From 5e7b27ddb6bb7a4b12da378da55581112a4457ba Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 5 Jul 2024 17:22:42 +0100 Subject: [PATCH 001/154] mieb ZeroshotClassification --- .../abstasks/AbsTaskZeroshotClassification.py | 60 +++++++++++++++++++ mteb/abstasks/TaskMetadata.py | 4 ++ mteb/abstasks/__init__.py | 1 + .../ZeroshotClassificationEvaluator.py | 42 +++++++++++++ mteb/evaluation/evaluators/__init__.py | 1 + mteb/models/__init__.py | 1 + mteb/models/clip_models.py | 60 +++++++++++++++++++ mteb/tasks/ZeroshotClassification/__init__.py | 3 + .../eng/RenderedSST2.py | 36 +++++++++++ .../ZeroshotClassification/eng/__init__.py | 0 mteb/tasks/__init__.py | 1 + .../no_revision_available/RenderedSST2.json | 19 ++++++ .../no_revision_available/model_meta.json | 1 + .../no_revision_available/RenderedSST2.json | 19 ++++++ .../no_revision_available/model_meta.json | 1 + .../no_revision_available/RenderedSST2.json | 19 ++++++ .../no_revision_available/model_meta.json | 1 + 17 files changed, 269 insertions(+) create mode 100644 mteb/abstasks/AbsTaskZeroshotClassification.py create mode 100644 mteb/evaluation/evaluators/ZeroshotClassificationEvaluator.py create mode 100644 mteb/models/clip_models.py create mode 100644 mteb/tasks/ZeroshotClassification/__init__.py create mode 100644 mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py create mode 100644 mteb/tasks/ZeroshotClassification/eng/__init__.py create mode 100644 results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/RenderedSST2.json create mode 100644 results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/RenderedSST2.json create mode 100644 results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/RenderedSST2.json create mode 100644 results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/model_meta.json diff --git a/mteb/abstasks/AbsTaskZeroshotClassification.py b/mteb/abstasks/AbsTaskZeroshotClassification.py new file mode 100644 index 0000000000..5361529f00 --- /dev/null +++ b/mteb/abstasks/AbsTaskZeroshotClassification.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import logging +from typing import Any + +import tqdm +from datasets import Dataset + +from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from mteb.load_results.mteb_results import ScoresDict + +from ..evaluation.evaluators import ZeroshotClassificationEvaluator +from .AbsTask import AbsTask + +logger = logging.getLogger(__name__) + + +class AbsTaskZeroshotClassification(AbsTask): + """Abstract class for ZeroshotClassification tasks + The similarity between an images and candidate text prompts, such as this is a dog/this is a cat. + + self.load_data() must generate a huggingface dataset with a split matching self.metadata_dict["eval_splits"], and assign it to self.dataset. It must contain the following columns: + png: list of Image.Image + cls: list of int + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def _add_main_score(self, scores) -> None: + scores["main_score"] = scores[self.metadata.main_score] + + def _evaluate_subset( + self, + model: EncoderWithQueryCorpusEncode | Encoder, + dataset: Dataset, + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ) -> ScoresDict: + + text_candidates = self.get_text_candidates() + + evaluator = ZeroshotClassificationEvaluator( + dataset["png"], + dataset["cls"], + text_candidates, + task_name=self.metadata.name, + **kwargs, + ) + metrics = evaluator(model, encode_kwargs=encode_kwargs) + + + scores = {"accuracy": metrics["accuracy"]} + self._add_main_score(scores) + return scores + + def get_text_candidates(self) -> list[str]: + """Return the text candidates for zeroshot classification""" + raise NotImplementedError("This method should be overridden by subclasses") diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 383d6e8904..d449e051c5 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -37,6 +37,7 @@ "Counterfactual Detection", "Emotion classification", "Reasoning as Retrieval", + "Rendered Texts Understanding", ] TASK_DOMAIN = Literal[ @@ -58,6 +59,7 @@ "Subtitles", "Web", "Programming", + "Movie", ] TEXT_CREATION_METHOD = Literal[ @@ -90,12 +92,14 @@ "Summarization", "InstructionRetrieval", "Speed", + "ZeroShotClassification", ] TASK_CATEGORY = Literal[ "s2s", # Sentence-to-sentence "s2p", # Sentence-to-paragraph "p2p", # Paragraph-to-paragraph + "i2t", ] ANNOTATOR_TYPE = Literal[ diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index ef3e8853d7..9f81557cc8 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -14,3 +14,4 @@ from .AbsTaskSTS import * from .AbsTaskSummarization import * from .MultilingualTask import * +from .AbsTaskZeroshotClassification import * \ No newline at end of file diff --git a/mteb/evaluation/evaluators/ZeroshotClassificationEvaluator.py b/mteb/evaluation/evaluators/ZeroshotClassificationEvaluator.py new file mode 100644 index 0000000000..46058149c4 --- /dev/null +++ b/mteb/evaluation/evaluators/ZeroshotClassificationEvaluator.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import logging +from typing import Any + +from sklearn import metrics + +from mteb.encoder_interface import Encoder +from PIL import Image +from .Evaluator import Evaluator + +logger = logging.getLogger(__name__) + + +class ZeroshotClassificationEvaluator(Evaluator): + def __init__( + self, + images:list[Image.Image], + labels:list[int], + text_candidates:list[str], + task_name: str | None = None, + **kwargs, + ): + super().__init__(**kwargs) + self.images = images + self.labels = labels + self.text_candidates = text_candidates + self.task_name = task_name + + def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): + if "batch_size" not in encode_kwargs: + encode_kwargs["batch_size"] = 32 + + text_embeddings = model.get_text_embeddings(self.text_candidates, batch_size=encode_kwargs["batch_size"]) + image_embeddings = model.get_image_embeddings(self.images, batch_size=encode_kwargs["batch_size"]) + probs = model.calculate_probs(text_embeddings, image_embeddings) + predictions = probs.argmax(dim=1) + + logger.info("Evaluating...") + accuracy = metrics.accuracy_score(self.labels, predictions) + + return {"accuracy": accuracy} diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py index a1dc8faaa5..6c83202d52 100644 --- a/mteb/evaluation/evaluators/__init__.py +++ b/mteb/evaluation/evaluators/__init__.py @@ -8,3 +8,4 @@ from .RetrievalEvaluator import * from .STSEvaluator import * from .SummarizationEvaluator import * +from .ZeroshotClassificationEvaluator import * \ No newline at end of file diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index 7d1e4c7239..ac64b7fb9f 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -8,6 +8,7 @@ from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode from mteb.model_meta import ModelMeta from mteb.models import ( + clip_models, bge_models, cohere_models, e5_instruct, diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py new file mode 100644 index 0000000000..a62f171269 --- /dev/null +++ b/mteb/models/clip_models.py @@ -0,0 +1,60 @@ +from PIL import Image +from transformers import AutoProcessor, AutoModel +from typing import Any +import torch +from tqdm import tqdm + +class CLIPModelWrapper: + + def __init__( + self, + model_name: str, + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + self.model = AutoModel.from_pretrained(model_name).to(self.device) + self.processor = AutoProcessor.from_pretrained(model_name) + + def preprocess( + self, + texts: list[str], + images: list[Image.Image], + ): + return self.processor(text=texts, images=images, return_tensors="pt", padding=True) + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i:i+batch_size] + inputs = self.processor(text=batch_texts, return_tensors="pt", padding=True) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + text_outputs = self.model.get_text_features(**inputs) + all_text_embeddings.append(text_outputs.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_image_embeddings(self, images: list[Image.Image], batch_size: int = 32): + all_image_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i:i+batch_size] + inputs = self.processor(images=batch_images, return_tensors="pt", padding=True) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model.get_image_features(**inputs) + all_image_embeddings.append(image_outputs.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm(dim=-1, keepdim=True) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits*100).softmax(dim=-1) + return probs \ No newline at end of file diff --git a/mteb/tasks/ZeroshotClassification/__init__.py b/mteb/tasks/ZeroshotClassification/__init__.py new file mode 100644 index 0000000000..8f87fc4cbe --- /dev/null +++ b/mteb/tasks/ZeroshotClassification/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .eng.RenderedSST2 import * \ No newline at end of file diff --git a/mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py b/mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py new file mode 100644 index 0000000000..365f28746d --- /dev/null +++ b/mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from ....abstasks import AbsTaskZeroshotClassification + + +class RenderedSST2(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="RenderedSST2", + description="RenderedSST2.", + reference="https://huggingface.co/datasets/clip-benchmark/wds_renderedsst2/commits/main", + dataset={ + "path": "clip-benchmark/wds_renderedsst2", + "revision": "66b9a461eda025201dd147e5f390f5984c33643a", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2016-01-01", "2016-12-31"), + form=["written"], + domains=["Movie"], + task_subtypes=[], + license="mit", + socioeconomic_status="mixed", + annotations_creators="human-annotated", + dialect=[], + text_creation="created", + bibtex_citation="""d""", + n_samples={"test": 1820}, + avg_character_length={"test": 10.0}, + ) + def get_text_candidates(self) -> list[str]: + return ["a negative review of a movie", "a positive review of a movie"] \ No newline at end of file diff --git a/mteb/tasks/ZeroshotClassification/eng/__init__.py b/mteb/tasks/ZeroshotClassification/eng/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/__init__.py b/mteb/tasks/__init__.py index dfe568bb89..e88221bff7 100644 --- a/mteb/tasks/__init__.py +++ b/mteb/tasks/__init__.py @@ -11,3 +11,4 @@ from .SpeedTask import * from .STS import * from .Summarization import * +from .ZeroshotClassification import * \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/RenderedSST2.json b/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/RenderedSST2.json new file mode 100644 index 0000000000..695287c385 --- /dev/null +++ b/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/RenderedSST2.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", + "evaluation_time": 11.55782675743103, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "accuracy": 0.6051619989017024, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6051619989017024 + } + ] + }, + "task_name": "RenderedSST2" +} \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/model_meta.json b/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000000..40ff52f432 --- /dev/null +++ b/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/RenderedSST2.json b/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/RenderedSST2.json new file mode 100644 index 0000000000..a73c6bd08e --- /dev/null +++ b/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/RenderedSST2.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", + "evaluation_time": 8.406261205673218, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "accuracy": 0.5848434925864909, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5848434925864909 + } + ] + }, + "task_name": "RenderedSST2" +} \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/model_meta.json b/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000000..40ff52f432 --- /dev/null +++ b/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/RenderedSST2.json b/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/RenderedSST2.json new file mode 100644 index 0000000000..070a2db249 --- /dev/null +++ b/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/RenderedSST2.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", + "evaluation_time": 27.11316442489624, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "accuracy": 0.6979681493684788, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6979681493684788 + } + ] + }, + "task_name": "RenderedSST2" +} \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/model_meta.json b/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000000..40ff52f432 --- /dev/null +++ b/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file From b6c3d484159e8e58644201b2763c9b29df219667 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 5 Jul 2024 17:34:54 +0100 Subject: [PATCH 002/154] mieb docs --- mieb-docs/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 mieb-docs/README.md diff --git a/mieb-docs/README.md b/mieb-docs/README.md new file mode 100644 index 0000000000..e69de29bb2 From 3e35fdb042c7aee1e7187977d61c84f52f3537a6 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Sat, 6 Jul 2024 01:19:52 +0800 Subject: [PATCH 003/154] mieb implementation demo --- mieb-docs/README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/mieb-docs/README.md b/mieb-docs/README.md index e69de29bb2..455bbffcb1 100644 --- a/mieb-docs/README.md +++ b/mieb-docs/README.md @@ -0,0 +1,34 @@ +To make a MIEB task `X` fully runable from scratch for dataset `Y` with model `Z`, we need to implemenet an `AbsTaskX` class for it, subclassing `AbsTask`; a task-specific `XEvaluator`, which will be called in `AbsTaskX`; a dataset-specific (e.g., Dataset `Y`) class `class Y(AbsTaskX)` subclassing the corresponding `AbsTaskX`, which is itself the subclass of `AbsTask`; and some model class `ZModelWrapper` that has needed functions. + +## Example + +Here is an example implementing zero-shot image classification from scratch. + +To solve this task, we basically need to encode the `images`, encode the `class label candidates with prompts` (things like "this is a dog pic", "this is a cat pic"), and similarity-compare them, to argmax out the class prediction for each image. + +#### ModelWrapper +Since we don't have an established class like `SentenceTransformer` or `DRES` anymore now, we first decide for this task so far, we need the model class to have `get_text_embeddings`, `get_image_embeddings`, and `calculate_probs`. As an example, [CLIPModelWrapper](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/models/clip_models.py) is first implemented. + +#### X Evaluator +With the model, [ZeroshotClassificationEvaluator](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/evaluation/evaluators/ZeroshotClassificationEvaluator.py) is implemented here, basically the pipeline of using the defined models to do zero-shot classification. + +#### AbsTask X +With the evaluator, [AbsTaskZeroshotClassification](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/abstasks/AbsTaskZeroshotClassification.py) is defined, operating on the dataset, calling the defined Evaluator, and gives out results. + +#### Dataset class +With all these, we can then define the dataset. Here I choose Rendered SST2 as an example, which is to classify SST2 movie reviews, with reviews rendered into images. [RenderedSST2](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py) is implemented like this, subclassing `AbsTaskZeroshotClassification`, and overwrite the `get_text_candidates` function, which gives `["a negative review of a movie", "a positive review of a movie"]` to be used in the evaluator. + +With all these, we can then +```python +from mteb.models.clip_models import CLIPModelWrapper +import mteb + +model_name = "openai/clip-vit-large-patch14" +model = CLIPModelWrapper(model_name) + +tasks = mteb.get_tasks(tasks=["RenderedSST2"]) +evaluation = mteb.MTEB(tasks=tasks) +results = evaluation.run(model, output_folder=f"results-mieb/{model_name}") +``` +And yeah, the results will be under [`results-mieb/openai/clip-vit-large-patch14`](https://github.com/embeddings-benchmark/mteb/tree/mieb/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available) and look legit with an `"accuracy": 0.6979681493684788,`, a bit higher than the original CLIP paper but might be resolution/layout difference of images in the remake of the dataset by the CLIP benchmark team. + From 4d709616f8706051795a9eb734e4fe6b1d401dce Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Thu, 11 Jul 2024 21:50:28 +0100 Subject: [PATCH 004/154] model meta; abstask column names; linear probe clf --- .../Image/AbsTaskImageClassification.py | 190 +++++++++++ .../Image/AbsTaskZeroshotClassification.py | 63 ++++ mteb/abstasks/Image/__init__.py | 0 .../Image/ClassificationEvaluator.py | 299 ++++++++++++++++++ .../Image/ZeroshotClassificationEvaluator.py | 42 +++ mteb/evaluation/evaluators/Image/__init__.py | 2 + .../Image/ImageClassification/__init__.py | 3 + .../eng/OxfordFlowersClassification.py | 36 +++ .../Image/ImageClassification/eng/__init__.py | 0 .../Image/ZeroshotClassification/__init__.py | 3 + .../eng/RenderedSST2.py | 41 +++ .../ZeroshotClassification/eng/__init__.py | 0 mteb/tasks/Image/__init__.py | 2 + .../RenderedSST2.json | 19 ++ .../model_meta.json | 1 + .../RenderedSST2.json | 19 ++ .../model_meta.json | 1 + .../RenderedSST2.json | 19 ++ .../model_meta.json | 1 + 19 files changed, 741 insertions(+) create mode 100644 mteb/abstasks/Image/AbsTaskImageClassification.py create mode 100644 mteb/abstasks/Image/AbsTaskZeroshotClassification.py create mode 100644 mteb/abstasks/Image/__init__.py create mode 100644 mteb/evaluation/evaluators/Image/ClassificationEvaluator.py create mode 100644 mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py create mode 100644 mteb/evaluation/evaluators/Image/__init__.py create mode 100644 mteb/tasks/Image/ImageClassification/__init__.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/__init__.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/__init__.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/__init__.py create mode 100644 mteb/tasks/Image/__init__.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/RenderedSST2.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RenderedSST2.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/RenderedSST2.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json diff --git a/mteb/abstasks/Image/AbsTaskImageClassification.py b/mteb/abstasks/Image/AbsTaskImageClassification.py new file mode 100644 index 0000000000..52ac7d81f9 --- /dev/null +++ b/mteb/abstasks/Image/AbsTaskImageClassification.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import logging +from collections import defaultdict +from typing import Any + +import numpy as np + +from mteb.encoder_interface import Encoder + +from ...evaluation.evaluators import ( + ImagekNNClassificationEvaluator, + ImagekNNClassificationEvaluatorPytorch, + ImagelogRegClassificationEvaluator, +) +from ...load_results.mteb_results import HFSubset, ScoresDict +from ..AbsTask import AbsTask + +logger = logging.getLogger(__name__) + + +class AbsTaskImageClassification(AbsTask): + """Abstract class for kNN classification tasks + The similarity is computed between pairs and the results are ranked. + + self.load_data() must generate a huggingface dataset with a split matching self.metadata_dict["eval_splits"], and assign it to self.dataset. It + must contain the following columns: + image: Image.Image + label: int + """ + + def __init__( + self, + method: str = "logReg", + n_experiments: int | None = None, + samples_per_label: int | None = None, + k: int = 3, + **kwargs, + ): + super().__init__(**kwargs) + self.method = method + + # Bootstrap parameters + self.n_experiments: int = ( # type: ignore + n_experiments + if n_experiments is not None + else self.metadata_dict.get("n_experiments", 10) + ) + self.samples_per_label: int = ( # type: ignore + samples_per_label + if samples_per_label is not None + else self.metadata_dict.get("samples_per_label", 8) + ) + + # kNN parameters + self.k = k + + # Run metadata validation by instantiating addressing the attribute + # This is quite hacky. Ideally, this would be done in the constructor of + # each concrete task, but then we have to duplicate the __init__ method's + # interface. + if hasattr(self, "metadata"): + self.metadata + + def _add_main_score(self, scores: dict[HFSubset, ScoresDict]) -> None: + scores["main_score"] = scores[self.metadata.main_score] + + def evaluate( + self, + model, + eval_split: str = "test", + train_split: str = "train", + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ) -> dict[HFSubset, ScoresDict]: + if not self.data_loaded: + self.load_data() + + scores = {} + hf_subsets = [l for l in self.dataset] if self.is_multilingual else ["default"] + + for hf_subset in hf_subsets: + logger.info( + f"\nTask: {self.metadata.name}, split: {eval_split}, subset: {hf_subset}. Running..." + ) + + if hf_subset not in self.dataset and hf_subset == "default": + ds = self.dataset + else: + ds = self.dataset[hf_subset] + scores[hf_subset] = self._evaluate_subset( + model, + ds, + eval_split, + train_split, + encode_kwargs=encode_kwargs, + **kwargs, + ) + self._add_main_score(scores[hf_subset]) + + return scores + + def _evaluate_subset( + self, + model: Encoder, + dataset, + eval_split: str = "test", + train_split: str = "train", + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ) -> ScoresDict: + train_split = dataset[train_split] + eval_split = dataset[eval_split] + params = {"k": self.k} + params.update(kwargs) + + scores = [] + test_cache, idxs = ( + None, + None, + ) # we store idxs to make the shuffling reproducible + for i in range(self.n_experiments): + logger.info( + "=" * 10 + f" Experiment {i+1}/{self.n_experiments} " + "=" * 10 + ) + # Bootstrap `self.samples_per_label` samples per label for each split + X_sampled, y_sampled, idxs = self._undersample_data( + train_split["image"], # type: ignore + train_split["label"], # type: ignore + self.samples_per_label, + idxs, + ) + + if self.method == "kNN": + evaluator = ImagekNNClassificationEvaluator( + X_sampled, + y_sampled, + eval_split["image"], # type: ignore + eval_split["label"], # type: ignore + task_name=self.metadata.name, + encode_kwargs=encode_kwargs, + **params, + ) + elif self.method == "kNN-pytorch": + evaluator = ImagekNNClassificationEvaluatorPytorch( + X_sampled, + y_sampled, + eval_split["image"], # type: ignore + eval_split["label"], # type: ignore + task_name=self.metadata.name, + encode_kwargs=encode_kwargs, + **params, + ) + elif self.method == "logReg": + evaluator = ImagelogRegClassificationEvaluator( + X_sampled, + y_sampled, + eval_split["image"], # type: ignore + eval_split["label"], # type: ignore + task_name=self.metadata.name, + encode_kwargs=encode_kwargs, + **params, + ) + else: + raise ValueError(f"Method {self.method} not supported") + + scores_exp, test_cache = evaluator(model, test_cache=test_cache) + scores.append(scores_exp) + + avg_scores: dict[str, Any] = { + k: np.mean([s[k] for s in scores]) for k in scores[0].keys() + } + avg_scores["scores_per_experiment"] = scores + return avg_scores + + def _undersample_data(self, X, y, samples_per_label: int, idxs=None): + """Undersample data to have samples_per_label samples of each label""" + X_sampled = [] + y_sampled = [] + if idxs is None: + idxs = np.arange(len(y)) + np.random.shuffle(idxs) + label_counter = defaultdict(int) + for i in idxs: + if label_counter[y[i]] < samples_per_label: + X_sampled.append(X[i]) + y_sampled.append(y[i]) + label_counter[y[i]] += 1 + return X_sampled, y_sampled, idxs diff --git a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py new file mode 100644 index 0000000000..d5cf76fc40 --- /dev/null +++ b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import logging +from typing import Any + +import tqdm +from datasets import Dataset + +from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from mteb.load_results.mteb_results import ScoresDict + +from ...evaluation.evaluators import ZeroshotClassificationEvaluator +from ..AbsTask import AbsTask + +logger = logging.getLogger(__name__) + + +class AbsTaskZeroshotClassification(AbsTask): + """Abstract class for ZeroshotClassification tasks + The similarity between an images and candidate text prompts, such as this is a dog/this is a cat. + + self.load_data() must generate a huggingface dataset with a split matching self.metadata_dict["eval_splits"], and assign it to self.dataset. It must contain the following columns: + image: list of Image.Image + labels: list of int + """ + + image_column_name: str = "image" + label_column_name: str = "label" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def _add_main_score(self, scores) -> None: + scores["main_score"] = scores[self.metadata.main_score] + + def _evaluate_subset( + self, + model: EncoderWithQueryCorpusEncode | Encoder, + dataset: Dataset, + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ) -> ScoresDict: + + text_candidates = self.get_text_candidates() + + evaluator = ZeroshotClassificationEvaluator( + dataset[self.image_column_name], + dataset[self.label_column_name], + text_candidates, + task_name=self.metadata.name, + **kwargs, + ) + metrics = evaluator(model, encode_kwargs=encode_kwargs) + + + scores = {"accuracy": metrics["accuracy"]} + self._add_main_score(scores) + return scores + + def get_text_candidates(self) -> list[str]: + """Return the text candidates for zeroshot classification""" + raise NotImplementedError("This method should be overridden by subclasses") diff --git a/mteb/abstasks/Image/__init__.py b/mteb/abstasks/Image/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py new file mode 100644 index 0000000000..984a929a6a --- /dev/null +++ b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import logging +from typing import Any + +import numpy as np +import torch +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import ( + accuracy_score, + average_precision_score, + f1_score, +) +from sklearn.neighbors import KNeighborsClassifier +from torch import Tensor + +from mteb.encoder_interface import Encoder +from mteb.evaluation.evaluators.model_encode import model_encode + +from ..Evaluator import Evaluator + +logger = logging.getLogger(__name__) + + +def dot_distance(a: np.ndarray, b: np.ndarray) -> float: + return -np.dot(a, b) + + +class ImagekNNClassificationEvaluator(Evaluator): + def __init__( + self, + images_train, + y_train, + images_test, + y_test, + task_name: str | None = None, + k: int = 1, + encode_kwargs: dict[str, Any] = {}, + limit: int | None = None, + **kwargs, + ): + super().__init__(**kwargs) + if limit is not None: + images_train = images_train[:limit] + y_train = y_train[:limit] + images_test = images_test[:limit] + y_test = y_test[:limit] + self.images_train = images_train + self.y_train = y_train + self.images_test = images_test + self.y_test = y_test + + self.task_name = task_name + self.encode_kwargs = encode_kwargs + + if "batch_size" not in self.encode_kwargs: + self.encode_kwargs["batch_size"] = 32 + + self.k = k + + def __call__(self, model, test_cache=None): + scores = {} + max_accuracy = 0 + max_f1 = 0 + max_ap = 0 + X_train = model.get_image_embeddings(self.images_train, batch_size=self.encode_kwargs["batch_size"]) + if test_cache is None: + X_test = model.get_image_embeddings(self.images_test, batch_size=self.encode_kwargs["batch_size"]) + test_cache = X_test + else: + X_test = test_cache + for metric in ["cosine", "euclidean"]: # TODO: "dot" + knn = KNeighborsClassifier(n_neighbors=self.k, n_jobs=-1, metric=metric) + knn.fit(X_train, self.y_train) + y_pred = knn.predict(X_test) + accuracy = accuracy_score(self.y_test, y_pred) + f1 = f1_score(self.y_test, y_pred, average="macro") + scores["accuracy_" + metric] = accuracy + scores["f1_" + metric] = f1 + max_accuracy = max(max_accuracy, accuracy) + max_f1 = max(max_f1, f1) # type: ignore + # if binary classification + if len(np.unique(self.y_train)) == 2: + ap = average_precision_score(self.y_test, y_pred) + scores["ap_" + metric] = ap + max_ap = max(max_ap, ap) + scores["accuracy"] = max_accuracy + scores["f1"] = max_f1 + if len(np.unique(self.y_train)) == 2: + scores["ap"] = max_ap + return scores, test_cache + + +class ImagekNNClassificationEvaluatorPytorch(Evaluator): + def __init__( + self, + images_train, + y_train, + images_test, + y_test, + task_name: str, + k: int = 1, + encode_kwargs: dict[str, Any] = {}, + limit: int | None = None, + **kwargs: Any, + ): + super().__init__(**kwargs) + if limit is not None: + images_train = images_train[:limit] + y_train = y_train[:limit] + images_test = images_test[:limit] + y_test = y_test[:limit] + + self.images_train = images_train + self.y_train = y_train + self.images_test = images_test + self.y_test = y_test + + self.task_name = task_name + self.encode_kwargs = encode_kwargs + + if "batch_size" not in self.encode_kwargs: + self.encode_kwargs["batch_size"] = 32 + + self.k = k + + def __call__(self, model: Encoder, test_cache=None): + scores = {} + max_accuracy = 0 + max_f1 = 0 + max_ap = 0 + X_train = model.get_image_embeddings(self.images_train, batch_size=self.encode_kwargs["batch_size"]) + + if test_cache is None: + X_test = model.get_image_embeddings(self.images_test, batch_size=self.encode_kwargs["batch_size"]) + test_cache = X_test + else: + X_test = test_cache + for metric in ["cosine", "euclidean", "dot"]: + if metric == "cosine": + distances = 1 - self._cos_sim(X_test, X_train) + elif metric == "euclidean": + distances = self._euclidean_dist(X_test, X_train) + elif metric == "dot": + distances = -self._dot_score(X_test, X_train) + neigh_indices = torch.topk( + distances, k=self.k, dim=1, largest=False + ).indices + y_train = torch.tensor(self.y_train) + y_pred = torch.mode( + y_train[neigh_indices], dim=1 + ).values # TODO: case where there is no majority + accuracy = accuracy_score(self.y_test, y_pred) + f1 = f1_score(self.y_test, y_pred, average="macro") + scores["accuracy_" + metric] = accuracy + scores["f1_" + metric] = f1 + max_accuracy = max(max_accuracy, accuracy) + max_f1 = max(max_f1, f1) # type: ignore + # if binary classification + if len(np.unique(self.y_train)) == 2: + ap = average_precision_score(self.y_test, y_pred) + scores["ap_" + metric] = ap + max_ap = max(max_ap, ap) + scores["accuracy"] = max_accuracy + scores["f1"] = max_f1 + if len(np.unique(self.y_train)) == 2: + scores["ap"] = max_ap + return scores, test_cache + + @staticmethod + def _cos_sim(a: Tensor, b: Tensor): + """Computes the cosine similarity cos_sim(a[i], b[j]) for all i and j. + + Return: + Matrix with res[i][j] = cos_sim(a[i], b[j]) + """ + if not isinstance(a, torch.Tensor): + a = torch.tensor(a) + + if not isinstance(b, torch.Tensor): + b = torch.tensor(b) + + if len(a.shape) == 1: + a = a.unsqueeze(0) + + if len(b.shape) == 1: + b = b.unsqueeze(0) + + a_norm = torch.nn.functional.normalize(a, p=2, dim=1) + b_norm = torch.nn.functional.normalize(b, p=2, dim=1) + return torch.mm(a_norm, b_norm.transpose(0, 1)) + + @staticmethod + def _euclidean_dist(a: Tensor, b: Tensor): + """Computes the euclidean distance euclidean_dist(a[i], b[j]) for all i and j. + + Returns: + Matrix with res[i][j] = euclidean_dist(a[i], b[j]) + """ + if not isinstance(a, torch.Tensor): + a = torch.tensor(a) + + if not isinstance(b, torch.Tensor): + b = torch.tensor(b) + + if len(a.shape) == 1: + a = a.unsqueeze(0) + + if len(b.shape) == 1: + b = b.unsqueeze(0) + + return torch.cdist(a, b, p=2) + + @staticmethod + def _dot_score(a: Tensor, b: Tensor): + """Computes the dot-product dot_prod(a[i], b[j]) for all i and j. + + Returns: + Matrix with res[i][j] = dot_prod(a[i], b[j]) + """ + if not isinstance(a, torch.Tensor): + a = torch.tensor(a) + + if not isinstance(b, torch.Tensor): + b = torch.tensor(b) + + if len(a.shape) == 1: + a = a.unsqueeze(0) + + if len(b.shape) == 1: + b = b.unsqueeze(0) + + return torch.mm(a, b.transpose(0, 1)) + + +class ImagelogRegClassificationEvaluator(Evaluator): + def __init__( + self, + images_train, + y_train, + images_test, + y_test, + task_name: str, + max_iter: int = 100, + encode_kwargs: dict[str, Any] = {}, + limit: int | None = None, + **kwargs, + ): + super().__init__(**kwargs) + self.encode_kwargs = encode_kwargs + + if "batch_size" not in self.encode_kwargs: + self.encode_kwargs["batch_size"] = 32 + + if limit is not None: + sentences_train = sentences_train[:limit] + y_train = y_train[:limit] + sentences_test = sentences_test[:limit] + y_test = y_test[:limit] + self.images_train = images_train + self.y_train = y_train + self.images_test = images_test + self.y_test = y_test + + self.max_iter = max_iter + self.task_name = task_name + + def __call__(self, model, test_cache=None): + scores = {} + clf = LogisticRegression( + random_state=self.seed, + n_jobs=-1, + max_iter=self.max_iter, + verbose=1 if logger.isEnabledFor(logging.DEBUG) else 0, + ) + X_train = model.get_image_embeddings(self.images_train, batch_size=self.encode_kwargs["batch_size"]) + + if test_cache is None: + X_test = model.get_image_embeddings(self.images_test, batch_size=self.encode_kwargs["batch_size"]) + test_cache = X_test + else: + X_test = test_cache + logger.info("Fitting logistic regression classifier...") + clf.fit(X_train, self.y_train) + logger.info("Evaluating...") + y_pred = clf.predict(X_test) + scores["accuracy"] = accuracy_score(self.y_test, y_pred) + scores["f1"] = f1_score(self.y_test, y_pred, average="macro") + scores["f1_weighted"] = f1_score(self.y_test, y_pred, average="weighted") + + # if binary classification + if len(np.unique(self.y_train)) == 2: + scores["ap"] = average_precision_score(self.y_test, y_pred, average="macro") + scores["ap_weighted"] = average_precision_score( + self.y_test, y_pred, average="weighted" + ) + + return scores, test_cache + \ No newline at end of file diff --git a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py new file mode 100644 index 0000000000..6100fb9706 --- /dev/null +++ b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import logging +from typing import Any + +from sklearn import metrics + +from mteb.encoder_interface import Encoder +from PIL import Image +from ..Evaluator import Evaluator + +logger = logging.getLogger(__name__) + + +class ZeroshotClassificationEvaluator(Evaluator): + def __init__( + self, + images:list[Image.Image], + labels:list[int], + text_candidates:list[str], + task_name: str | None = None, + **kwargs, + ): + super().__init__(**kwargs) + self.images = images + self.labels = labels + self.text_candidates = text_candidates + self.task_name = task_name + + def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): + if "batch_size" not in encode_kwargs: + encode_kwargs["batch_size"] = 32 + + text_embeddings = model.get_text_embeddings(self.text_candidates, batch_size=encode_kwargs["batch_size"]) + image_embeddings = model.get_image_embeddings(self.images, batch_size=encode_kwargs["batch_size"]) + probs = model.calculate_probs(text_embeddings, image_embeddings) + predictions = probs.argmax(dim=1) + + logger.info("Evaluating...") + accuracy = metrics.accuracy_score(self.labels, predictions) + + return {"accuracy": accuracy} diff --git a/mteb/evaluation/evaluators/Image/__init__.py b/mteb/evaluation/evaluators/Image/__init__.py new file mode 100644 index 0000000000..0fd2a7478b --- /dev/null +++ b/mteb/evaluation/evaluators/Image/__init__.py @@ -0,0 +1,2 @@ +# from .ClassificationEvaluator import * +# from .ZeroshotClassificationEvaluator import * \ No newline at end of file diff --git a/mteb/tasks/Image/ImageClassification/__init__.py b/mteb/tasks/Image/ImageClassification/__init__.py new file mode 100644 index 0000000000..a3861f99ad --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .eng.OxfordFlowersClassification import * \ No newline at end of file diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py new file mode 100644 index 0000000000..8f97aa0689 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + +class OxfordFlowersClassification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="OxfordFlowersClassification", + description="Classifying flowers", + reference="https://huggingface.co/datasets/nelorth/oxford-flowers/viewer/default/train", + dataset={ + "path": "nelorth/oxford-flowers", + "revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + }, + type="Classification", + category="s2s", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2012-01-01", + "2015-12-31", + ), # Estimated range for the collection of reviews + form=["written"], + domains=["Reviews"], + task_subtypes=["Sentiment/Hate speech"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + text_creation="found", + bibtex_citation="""d""", + n_samples={"test": 400000}, + avg_character_length={"test": 431.4}, + ) diff --git a/mteb/tasks/Image/ImageClassification/eng/__init__.py b/mteb/tasks/Image/ImageClassification/eng/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Image/ZeroshotClassification/__init__.py b/mteb/tasks/Image/ZeroshotClassification/__init__.py new file mode 100644 index 0000000000..8f87fc4cbe --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .eng.RenderedSST2 import * \ No newline at end of file diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py new file mode 100644 index 0000000000..3961e31b34 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class RenderedSST2(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="RenderedSST2", + description="RenderedSST2.", + reference="https://huggingface.co/datasets/clip-benchmark/wds_renderedsst2/commits/main", + dataset={ + "path": "clip-benchmark/wds_renderedsst2", + "revision": "66b9a461eda025201dd147e5f390f5984c33643a", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2016-01-01", "2016-12-31"), + form=["written"], + domains=["Movie"], + task_subtypes=[], + license="mit", + socioeconomic_status="mixed", + annotations_creators="human-annotated", + dialect=[], + text_creation="created", + bibtex_citation="""d""", + n_samples={"test": 1820}, + avg_character_length={"test": 10.0}, + ) + + # Override default column names in the subclass + image_column_name: str = "png" + label_column_name: str = "cls" + + def get_text_candidates(self) -> list[str]: + return ["a negative review of a movie", "a positive review of a movie"] \ No newline at end of file diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/__init__.py b/mteb/tasks/Image/ZeroshotClassification/eng/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Image/__init__.py b/mteb/tasks/Image/__init__.py new file mode 100644 index 0000000000..c9c76d48b3 --- /dev/null +++ b/mteb/tasks/Image/__init__.py @@ -0,0 +1,2 @@ +from .ZeroshotClassification import * +from .ImageClassification import * \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/RenderedSST2.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/RenderedSST2.json new file mode 100644 index 0000000000..773aaf9434 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/RenderedSST2.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", + "evaluation_time": 12.006900548934937, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "accuracy": 0.6051619989017024, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6051619989017024 + } + ] + }, + "task_name": "RenderedSST2" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json new file mode 100644 index 0000000000..0b8a2cecb0 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json @@ -0,0 +1 @@ +{"name": "openai/clip-vit-base-patch16", "revision": "57c216476eefef5ab752ec549e440a49ae4ae5f3", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "CLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RenderedSST2.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RenderedSST2.json new file mode 100644 index 0000000000..b293d8e5ff --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RenderedSST2.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", + "evaluation_time": 8.718077659606934, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "accuracy": 0.5848434925864909, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5848434925864909 + } + ] + }, + "task_name": "RenderedSST2" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json new file mode 100644 index 0000000000..cdfeb8c90d --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json @@ -0,0 +1 @@ +{"name": "openai/clip-vit-base-patch32", "revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "CLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/RenderedSST2.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/RenderedSST2.json new file mode 100644 index 0000000000..dafe3400a8 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/RenderedSST2.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", + "evaluation_time": 27.908196687698364, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "accuracy": 0.6979681493684788, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6979681493684788 + } + ] + }, + "task_name": "RenderedSST2" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json new file mode 100644 index 0000000000..8c47f12e0c --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json @@ -0,0 +1 @@ +{"name": "openai/clip-vit-large-patch14", "revision": "32bd64288804d66eefd0ccbe215aa642df71cc41", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "CLIPModelWrapper"} \ No newline at end of file From f5e504a016a9126451e0c660785f5faf7bc963a5 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Thu, 11 Jul 2024 22:03:32 +0100 Subject: [PATCH 005/154] model meta; abstask column names; linear probe clf --- .../abstasks/AbsTaskZeroshotClassification.py | 60 ------------------- mteb/abstasks/__init__.py | 3 +- .../ZeroshotClassificationEvaluator.py | 42 ------------- mteb/evaluation/evaluators/__init__.py | 3 +- mteb/models/__init__.py | 2 + mteb/models/clip_models.py | 48 ++++++++++++++- mteb/tasks/ZeroshotClassification/__init__.py | 3 - .../eng/RenderedSST2.py | 36 ----------- .../ZeroshotClassification/eng/__init__.py | 0 mteb/tasks/__init__.py | 3 +- .../no_revision_available/RenderedSST2.json | 19 ------ .../no_revision_available/model_meta.json | 1 - .../no_revision_available/RenderedSST2.json | 19 ------ .../no_revision_available/model_meta.json | 1 - .../no_revision_available/RenderedSST2.json | 19 ------ .../no_revision_available/model_meta.json | 1 - 16 files changed, 54 insertions(+), 206 deletions(-) delete mode 100644 mteb/abstasks/AbsTaskZeroshotClassification.py delete mode 100644 mteb/evaluation/evaluators/ZeroshotClassificationEvaluator.py delete mode 100644 mteb/tasks/ZeroshotClassification/__init__.py delete mode 100644 mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py delete mode 100644 mteb/tasks/ZeroshotClassification/eng/__init__.py delete mode 100644 results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/RenderedSST2.json delete mode 100644 results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/RenderedSST2.json delete mode 100644 results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/RenderedSST2.json delete mode 100644 results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/model_meta.json diff --git a/mteb/abstasks/AbsTaskZeroshotClassification.py b/mteb/abstasks/AbsTaskZeroshotClassification.py deleted file mode 100644 index 5361529f00..0000000000 --- a/mteb/abstasks/AbsTaskZeroshotClassification.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Any - -import tqdm -from datasets import Dataset - -from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode -from mteb.load_results.mteb_results import ScoresDict - -from ..evaluation.evaluators import ZeroshotClassificationEvaluator -from .AbsTask import AbsTask - -logger = logging.getLogger(__name__) - - -class AbsTaskZeroshotClassification(AbsTask): - """Abstract class for ZeroshotClassification tasks - The similarity between an images and candidate text prompts, such as this is a dog/this is a cat. - - self.load_data() must generate a huggingface dataset with a split matching self.metadata_dict["eval_splits"], and assign it to self.dataset. It must contain the following columns: - png: list of Image.Image - cls: list of int - """ - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def _add_main_score(self, scores) -> None: - scores["main_score"] = scores[self.metadata.main_score] - - def _evaluate_subset( - self, - model: EncoderWithQueryCorpusEncode | Encoder, - dataset: Dataset, - *, - encode_kwargs: dict[str, Any] = {}, - **kwargs, - ) -> ScoresDict: - - text_candidates = self.get_text_candidates() - - evaluator = ZeroshotClassificationEvaluator( - dataset["png"], - dataset["cls"], - text_candidates, - task_name=self.metadata.name, - **kwargs, - ) - metrics = evaluator(model, encode_kwargs=encode_kwargs) - - - scores = {"accuracy": metrics["accuracy"]} - self._add_main_score(scores) - return scores - - def get_text_candidates(self) -> list[str]: - """Return the text candidates for zeroshot classification""" - raise NotImplementedError("This method should be overridden by subclasses") diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index 9f81557cc8..3763090d84 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -14,4 +14,5 @@ from .AbsTaskSTS import * from .AbsTaskSummarization import * from .MultilingualTask import * -from .AbsTaskZeroshotClassification import * \ No newline at end of file +from .Image.AbsTaskZeroshotClassification import * +from .Image.AbsTaskImageClassification import * \ No newline at end of file diff --git a/mteb/evaluation/evaluators/ZeroshotClassificationEvaluator.py b/mteb/evaluation/evaluators/ZeroshotClassificationEvaluator.py deleted file mode 100644 index 46058149c4..0000000000 --- a/mteb/evaluation/evaluators/ZeroshotClassificationEvaluator.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Any - -from sklearn import metrics - -from mteb.encoder_interface import Encoder -from PIL import Image -from .Evaluator import Evaluator - -logger = logging.getLogger(__name__) - - -class ZeroshotClassificationEvaluator(Evaluator): - def __init__( - self, - images:list[Image.Image], - labels:list[int], - text_candidates:list[str], - task_name: str | None = None, - **kwargs, - ): - super().__init__(**kwargs) - self.images = images - self.labels = labels - self.text_candidates = text_candidates - self.task_name = task_name - - def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): - if "batch_size" not in encode_kwargs: - encode_kwargs["batch_size"] = 32 - - text_embeddings = model.get_text_embeddings(self.text_candidates, batch_size=encode_kwargs["batch_size"]) - image_embeddings = model.get_image_embeddings(self.images, batch_size=encode_kwargs["batch_size"]) - probs = model.calculate_probs(text_embeddings, image_embeddings) - predictions = probs.argmax(dim=1) - - logger.info("Evaluating...") - accuracy = metrics.accuracy_score(self.labels, predictions) - - return {"accuracy": accuracy} diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py index 6c83202d52..49aa7f5e6d 100644 --- a/mteb/evaluation/evaluators/__init__.py +++ b/mteb/evaluation/evaluators/__init__.py @@ -8,4 +8,5 @@ from .RetrievalEvaluator import * from .STSEvaluator import * from .SummarizationEvaluator import * -from .ZeroshotClassificationEvaluator import * \ No newline at end of file +from .Image.ZeroshotClassificationEvaluator import * +from .Image.ClassificationEvaluator import * \ No newline at end of file diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index ac64b7fb9f..aef5b9ba3a 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -22,6 +22,7 @@ salesforce_models, sentence_transformers_models, voyage_models, + clip_models, ) logger = logging.getLogger(__name__) @@ -132,6 +133,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe ru_sentence_models, nomic_models, cohere_models, + clip_models ] models = {} diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index a62f171269..c8536fa651 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -3,7 +3,9 @@ from typing import Any import torch from tqdm import tqdm - +from mteb.model_meta import ModelMeta +from functools import partial + class CLIPModelWrapper: def __init__( @@ -57,4 +59,46 @@ def calculate_probs(self, text_embeddings, image_embeddings): image_embeddings = image_embeddings / image_embeddings.norm(dim=-1, keepdim=True) logits = torch.matmul(image_embeddings, text_embeddings.T) probs = (logits*100).softmax(dim=-1) - return probs \ No newline at end of file + return probs + +clip_vit_large_patch14 = ModelMeta( + loader=partial( + CLIPModelWrapper, + model_name="openai/clip-vit-large-patch14", + ), + name="openai/clip-vit-large-patch14", + languages=["eng_Latn"], + open_source=True, + revision="32bd64288804d66eefd0ccbe215aa642df71cc41", + release_date="2021-02-26", +) + +clip_vit_base_patch32 = ModelMeta( + loader=partial( + CLIPModelWrapper, + model_name="openai/clip-vit-base-patch32", + ), + name="openai/clip-vit-base-patch32", + languages=["eng_Latn"], + open_source=True, + revision="3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", + release_date="2021-02-26", +) + +clip_vit_base_patch16 = ModelMeta( + loader=partial( + CLIPModelWrapper, + model_name="openai/clip-vit-base-patch16", + ), + name="openai/clip-vit-base-patch16", + languages=["eng_Latn"], + open_source=True, + revision="57c216476eefef5ab752ec549e440a49ae4ae5f3", + release_date="2021-02-26", +) + +if __name__ == "__main__": + import mteb + + mdl = mteb.get_model(clip_vit_base_patch16.name, clip_vit_base_patch16.revision) + emb = mdl.get_text_embeddings(["Hello, world!"]) \ No newline at end of file diff --git a/mteb/tasks/ZeroshotClassification/__init__.py b/mteb/tasks/ZeroshotClassification/__init__.py deleted file mode 100644 index 8f87fc4cbe..0000000000 --- a/mteb/tasks/ZeroshotClassification/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from __future__ import annotations - -from .eng.RenderedSST2 import * \ No newline at end of file diff --git a/mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py b/mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py deleted file mode 100644 index 365f28746d..0000000000 --- a/mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -from mteb.abstasks.TaskMetadata import TaskMetadata - -from ....abstasks import AbsTaskZeroshotClassification - - -class RenderedSST2(AbsTaskZeroshotClassification): - metadata = TaskMetadata( - name="RenderedSST2", - description="RenderedSST2.", - reference="https://huggingface.co/datasets/clip-benchmark/wds_renderedsst2/commits/main", - dataset={ - "path": "clip-benchmark/wds_renderedsst2", - "revision": "66b9a461eda025201dd147e5f390f5984c33643a", - }, - type="ZeroShotClassification", - category="i2t", - eval_splits=["test"], - eval_langs=["eng-Latn"], - main_score="accuracy", - date=("2016-01-01", "2016-12-31"), - form=["written"], - domains=["Movie"], - task_subtypes=[], - license="mit", - socioeconomic_status="mixed", - annotations_creators="human-annotated", - dialect=[], - text_creation="created", - bibtex_citation="""d""", - n_samples={"test": 1820}, - avg_character_length={"test": 10.0}, - ) - def get_text_candidates(self) -> list[str]: - return ["a negative review of a movie", "a positive review of a movie"] \ No newline at end of file diff --git a/mteb/tasks/ZeroshotClassification/eng/__init__.py b/mteb/tasks/ZeroshotClassification/eng/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mteb/tasks/__init__.py b/mteb/tasks/__init__.py index e88221bff7..df3098a718 100644 --- a/mteb/tasks/__init__.py +++ b/mteb/tasks/__init__.py @@ -11,4 +11,5 @@ from .SpeedTask import * from .STS import * from .Summarization import * -from .ZeroshotClassification import * \ No newline at end of file +from .Image.ZeroshotClassification import * +from .Image.ImageClassification import * \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/RenderedSST2.json b/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/RenderedSST2.json deleted file mode 100644 index 695287c385..0000000000 --- a/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/RenderedSST2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", - "evaluation_time": 11.55782675743103, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "accuracy": 0.6051619989017024, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6051619989017024 - } - ] - }, - "task_name": "RenderedSST2" -} \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/model_meta.json b/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f432..0000000000 --- a/results-mieb/openai/clip-vit-base-patch16/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/RenderedSST2.json b/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/RenderedSST2.json deleted file mode 100644 index a73c6bd08e..0000000000 --- a/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/RenderedSST2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", - "evaluation_time": 8.406261205673218, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "accuracy": 0.5848434925864909, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5848434925864909 - } - ] - }, - "task_name": "RenderedSST2" -} \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/model_meta.json b/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f432..0000000000 --- a/results-mieb/openai/clip-vit-base-patch32/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/RenderedSST2.json b/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/RenderedSST2.json deleted file mode 100644 index 070a2db249..0000000000 --- a/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/RenderedSST2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", - "evaluation_time": 27.11316442489624, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "accuracy": 0.6979681493684788, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6979681493684788 - } - ] - }, - "task_name": "RenderedSST2" -} \ No newline at end of file diff --git a/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/model_meta.json b/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f432..0000000000 --- a/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file From 46f3d91f12964317b0ddf785901ea4f118511a19 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Fri, 12 Jul 2024 16:30:59 +0000 Subject: [PATCH 006/154] fix: update naming as candidate_labels --- mieb-docs/README.md | 2 +- mteb/abstasks/Image/AbsTaskZeroshotClassification.py | 6 +++--- .../evaluators/Image/ZeroshotClassificationEvaluator.py | 6 +++--- mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mieb-docs/README.md b/mieb-docs/README.md index 455bbffcb1..ecc9617858 100644 --- a/mieb-docs/README.md +++ b/mieb-docs/README.md @@ -16,7 +16,7 @@ With the model, [ZeroshotClassificationEvaluator](https://github.com/embeddings- With the evaluator, [AbsTaskZeroshotClassification](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/abstasks/AbsTaskZeroshotClassification.py) is defined, operating on the dataset, calling the defined Evaluator, and gives out results. #### Dataset class -With all these, we can then define the dataset. Here I choose Rendered SST2 as an example, which is to classify SST2 movie reviews, with reviews rendered into images. [RenderedSST2](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py) is implemented like this, subclassing `AbsTaskZeroshotClassification`, and overwrite the `get_text_candidates` function, which gives `["a negative review of a movie", "a positive review of a movie"]` to be used in the evaluator. +With all these, we can then define the dataset. Here I choose Rendered SST2 as an example, which is to classify SST2 movie reviews, with reviews rendered into images. [RenderedSST2](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py) is implemented like this, subclassing `AbsTaskZeroshotClassification`, and overwrite the `get_candidate_labels` function, which gives `["a negative review of a movie", "a positive review of a movie"]` to be used in the evaluator. With all these, we can then ```python diff --git a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py index d5cf76fc40..9ba1e49589 100644 --- a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py +++ b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py @@ -42,12 +42,12 @@ def _evaluate_subset( **kwargs, ) -> ScoresDict: - text_candidates = self.get_text_candidates() + candidate_labels = self.get_candidate_labels() evaluator = ZeroshotClassificationEvaluator( dataset[self.image_column_name], dataset[self.label_column_name], - text_candidates, + candidate_labels, task_name=self.metadata.name, **kwargs, ) @@ -58,6 +58,6 @@ def _evaluate_subset( self._add_main_score(scores) return scores - def get_text_candidates(self) -> list[str]: + def get_candidate_labels(self) -> list[str]: """Return the text candidates for zeroshot classification""" raise NotImplementedError("This method should be overridden by subclasses") diff --git a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py index 6100fb9706..3c0fbad51c 100644 --- a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py @@ -17,21 +17,21 @@ def __init__( self, images:list[Image.Image], labels:list[int], - text_candidates:list[str], + candidate_labels:list[str], task_name: str | None = None, **kwargs, ): super().__init__(**kwargs) self.images = images self.labels = labels - self.text_candidates = text_candidates + self.candidate_labels = candidate_labels self.task_name = task_name def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): if "batch_size" not in encode_kwargs: encode_kwargs["batch_size"] = 32 - text_embeddings = model.get_text_embeddings(self.text_candidates, batch_size=encode_kwargs["batch_size"]) + text_embeddings = model.get_text_embeddings(self.candidate_labels, batch_size=encode_kwargs["batch_size"]) image_embeddings = model.get_image_embeddings(self.images, batch_size=encode_kwargs["batch_size"]) probs = model.calculate_probs(text_embeddings, image_embeddings) predictions = probs.argmax(dim=1) diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py index 3961e31b34..0274542704 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py @@ -37,5 +37,5 @@ class RenderedSST2(AbsTaskZeroshotClassification): image_column_name: str = "png" label_column_name: str = "cls" - def get_text_candidates(self) -> list[str]: + def get_candidate_labels(self) -> list[str]: return ["a negative review of a movie", "a positive review of a movie"] \ No newline at end of file From f8035ecd2a653fc4e246c8aadc573789746f2a28 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Mon, 15 Jul 2024 05:18:13 +0800 Subject: [PATCH 007/154] Update README.md --- mieb-docs/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mieb-docs/README.md b/mieb-docs/README.md index ecc9617858..6602128e3f 100644 --- a/mieb-docs/README.md +++ b/mieb-docs/README.md @@ -7,7 +7,7 @@ Here is an example implementing zero-shot image classification from scratch. To solve this task, we basically need to encode the `images`, encode the `class label candidates with prompts` (things like "this is a dog pic", "this is a cat pic"), and similarity-compare them, to argmax out the class prediction for each image. #### ModelWrapper -Since we don't have an established class like `SentenceTransformer` or `DRES` anymore now, we first decide for this task so far, we need the model class to have `get_text_embeddings`, `get_image_embeddings`, and `calculate_probs`. As an example, [CLIPModelWrapper](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/models/clip_models.py) is first implemented. +Since we don't have an established class like `SentenceTransformer` or `DRES` anymore now, we first decide for this task so far, we need the model class to have `get_text_embeddings`, `get_image_embeddings`, and `calculate_probs`. As an example, [CLIPModelWrapper](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/models/clip_models.py) is first implemented, with MetaData defined. #### X Evaluator With the model, [ZeroshotClassificationEvaluator](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/evaluation/evaluators/ZeroshotClassificationEvaluator.py) is implemented here, basically the pipeline of using the defined models to do zero-shot classification. @@ -20,15 +20,15 @@ With all these, we can then define the dataset. Here I choose Rendered SST2 as a With all these, we can then ```python -from mteb.models.clip_models import CLIPModelWrapper + import mteb model_name = "openai/clip-vit-large-patch14" -model = CLIPModelWrapper(model_name) +model = mteb.get_model(model_name = model_name) tasks = mteb.get_tasks(tasks=["RenderedSST2"]) evaluation = mteb.MTEB(tasks=tasks) results = evaluation.run(model, output_folder=f"results-mieb/{model_name}") ``` -And yeah, the results will be under [`results-mieb/openai/clip-vit-large-patch14`](https://github.com/embeddings-benchmark/mteb/tree/mieb/results-mieb/openai/clip-vit-large-patch14/no_model_name_available/no_revision_available) and look legit with an `"accuracy": 0.6979681493684788,`, a bit higher than the original CLIP paper but might be resolution/layout difference of images in the remake of the dataset by the CLIP benchmark team. +And yeah, the results will be under [`results-mieb/openai/clip-vit-large-patch14`](https://github.com/embeddings-benchmark/mteb/blob/mieb/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/RenderedSST2.json) and look legit with an `"accuracy": 0.6979681493684788,`, a bit higher than the original CLIP paper but might be resolution/layout difference of images in the remake of the dataset by the CLIP benchmark team. From 96870afd445eee76ca47ba61e9bc919686a3ce4d Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Mon, 15 Jul 2024 05:20:14 +0800 Subject: [PATCH 008/154] Update README.md --- mieb-docs/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mieb-docs/README.md b/mieb-docs/README.md index 6602128e3f..4c7f388702 100644 --- a/mieb-docs/README.md +++ b/mieb-docs/README.md @@ -10,13 +10,13 @@ To solve this task, we basically need to encode the `images`, encode the `class Since we don't have an established class like `SentenceTransformer` or `DRES` anymore now, we first decide for this task so far, we need the model class to have `get_text_embeddings`, `get_image_embeddings`, and `calculate_probs`. As an example, [CLIPModelWrapper](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/models/clip_models.py) is first implemented, with MetaData defined. #### X Evaluator -With the model, [ZeroshotClassificationEvaluator](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/evaluation/evaluators/ZeroshotClassificationEvaluator.py) is implemented here, basically the pipeline of using the defined models to do zero-shot classification. +With the model, [ZeroshotClassificationEvaluator](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py) is implemented here, basically the pipeline of using the defined models to do zero-shot classification. #### AbsTask X -With the evaluator, [AbsTaskZeroshotClassification](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/abstasks/AbsTaskZeroshotClassification.py) is defined, operating on the dataset, calling the defined Evaluator, and gives out results. +With the evaluator, [AbsTaskZeroshotClassification](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/abstasks/Image/AbsTaskZeroshotClassification.py) is defined, operating on the dataset, calling the defined Evaluator, and gives out results. #### Dataset class -With all these, we can then define the dataset. Here I choose Rendered SST2 as an example, which is to classify SST2 movie reviews, with reviews rendered into images. [RenderedSST2](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/tasks/ZeroshotClassification/eng/RenderedSST2.py) is implemented like this, subclassing `AbsTaskZeroshotClassification`, and overwrite the `get_candidate_labels` function, which gives `["a negative review of a movie", "a positive review of a movie"]` to be used in the evaluator. +With all these, we can then define the dataset. Here I choose Rendered SST2 as an example, which is to classify SST2 movie reviews, with reviews rendered into images. [RenderedSST2](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py) is implemented like this, subclassing `AbsTaskZeroshotClassification`, and overwrite the `get_candidate_labels` function, which gives `["a negative review of a movie", "a positive review of a movie"]` to be used in the evaluator. With all these, we can then ```python From 5c2df6b7d63a50a9146496cffa4b41eb5c490036 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 15 Jul 2024 01:02:20 +0100 Subject: [PATCH 009/154] i2tretrieval --- mteb/abstasks/Image/AbsTaskI2TRetrieval.py | 453 ++++++++++++++++++ mteb/abstasks/__init__.py | 3 +- .../evaluators/Image/I2TRetrievalEvaluator.py | 360 ++++++++++++++ mteb/evaluation/evaluators/__init__.py | 3 +- mteb/tasks/Image/I2TRetrieval/__init__.py | 3 + .../I2TRetrieval/eng/MSCOCOI2TRetrieval.py | 50 ++ mteb/tasks/Image/I2TRetrieval/eng/__init__.py | 0 mteb/tasks/__init__.py | 3 +- 8 files changed, 872 insertions(+), 3 deletions(-) create mode 100644 mteb/abstasks/Image/AbsTaskI2TRetrieval.py create mode 100644 mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py create mode 100644 mteb/tasks/Image/I2TRetrieval/__init__.py create mode 100644 mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py create mode 100644 mteb/tasks/Image/I2TRetrieval/eng/__init__.py diff --git a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py new file mode 100644 index 0000000000..becfd79241 --- /dev/null +++ b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py @@ -0,0 +1,453 @@ +from __future__ import annotations + +import json +import logging +import os +from collections import defaultdict +from pathlib import Path +from time import time +from typing import Any, Dict, Tuple, Union +from PIL import Image +import tqdm +from datasets import Features, Value, load_dataset + +from ...evaluation.evaluators import I2TRetrievalEvaluator +from ...load_results.mteb_results import ScoresDict +from ..AbsTask import AbsTask + +logger = logging.getLogger(__name__) + +class HFDataLoader: + def __init__( + self, + hf_repo: str | None = None, + hf_repo_qrels: str | None = None, + data_folder: str | None = None, + prefix: str | None = None, + corpus_file: str = "corpus.jsonl", + query_file: str = "queries.jsonl", + qrels_folder: str = "qrels", + qrels_file: str = "", + streaming: bool = False, + keep_in_memory: bool = False, + ): + self.corpus = {} + self.queries = {} + self.qrels = {} + self.hf_repo = hf_repo + if hf_repo: + # By default fetch qrels from same repo not a second repo with "-qrels" like in original + self.hf_repo_qrels = hf_repo_qrels if hf_repo_qrels else hf_repo + else: + # data folder would contain these files: + # (1) fiqa/corpus.jsonl (format: jsonlines) + # (2) fiqa/queries.jsonl (format: jsonlines) + # (3) fiqa/qrels/test.tsv (format: tsv ("\t")) + if prefix: + query_file = prefix + "-" + query_file + qrels_folder = prefix + "-" + qrels_folder + + self.corpus_file = ( + os.path.join(data_folder, corpus_file) if data_folder else corpus_file + ) + self.query_file = ( + os.path.join(data_folder, query_file) if data_folder else query_file + ) + self.qrels_folder = ( + os.path.join(data_folder, qrels_folder) if data_folder else None + ) + self.qrels_file = qrels_file + self.streaming = streaming + self.keep_in_memory = keep_in_memory + + @staticmethod + def check(fIn: str, ext: str): + if not os.path.exists(fIn): + raise ValueError( + "File {} not present! Please provide accurate file.".format(fIn) + ) + + if not fIn.endswith(ext): + raise ValueError( + "File {} must be present with extension {}".format(fIn, ext) + ) + + def load( + self, split="test" + ) -> Tuple[Dict[str, dict[str, str]], dict[str, str], dict[str, dict[str, int]]]: + if not self.hf_repo: + self.qrels_file = os.path.join(self.qrels_folder, split + ".tsv") + self.check(fIn=self.corpus_file, ext="jsonl") + self.check(fIn=self.query_file, ext="jsonl") + self.check(fIn=self.qrels_file, ext="tsv") + + if not len(self.corpus): + logger.info("Loading Corpus...") + self._load_corpus() + logger.info("Loaded %d %s Documents.", len(self.corpus), split.upper()) + logger.info("Doc Example: %s", self.corpus[0]) + + if not len(self.queries): + logger.info("Loading Queries...") + self._load_queries(split) + + self._load_qrels(split) + # filter queries with no qrels + qrels_dict = defaultdict(dict) + + def qrels_dict_init(row): + qrels_dict[row["query-id"]][row["corpus-id"]] = int(row["score"]) + + self.qrels.map(qrels_dict_init) + self.qrels = qrels_dict + self.queries = self.queries.filter(lambda x: x["id"] in self.qrels) + logger.info("Loaded %d %s Queries.", len(self.queries), split.upper()) + logger.info("Query Example: %s", self.queries[0]) + + return self.corpus, self.queries, self.qrels + + def load_corpus(self) -> dict[str, dict[str, str]]: + if not self.hf_repo: + self.check(fIn=self.corpus_file, ext="jsonl") + + if not len(self.corpus): + logger.info("Loading Corpus...") + self._load_corpus() + logger.info("Loaded %d %s Documents.", len(self.corpus)) + logger.info("Doc Example: %s", self.corpus[0]) + + return self.corpus + + def _load_corpus(self): + if self.hf_repo: + corpus_ds = load_dataset( + self.hf_repo, + "corpus", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )["corpus"] + else: + corpus_ds = load_dataset( + "json", + data_files=self.corpus_file, + streaming=self.streaming, + keep_in_memory=self.keep_in_memory, + ) + self.corpus = corpus_ds + + def _load_queries(self, split): + if self.hf_repo: + queries_ds = load_dataset( + self.hf_repo, + "query", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )[split] + else: + queries_ds = load_dataset( + "json", + data_files=self.query_file, + streaming=self.streaming, + keep_in_memory=self.keep_in_memory, + ) + self.queries = queries_ds + + def _load_qrels(self, split): + if self.hf_repo: + qrels_ds = load_dataset( + self.hf_repo_qrels, + "qrels", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )[split] + else: + qrels_ds = load_dataset( + "csv", + data_files=self.qrels_file, + delimiter="\t", + keep_in_memory=self.keep_in_memory, + ) + qrels_ds = qrels_ds.remove_columns("Q0") + features = Features( + { + "query-id": Value("string"), + "corpus-id": Value("string"), + "score": Value("float"), + } + ) + qrels_ds = qrels_ds.cast(features) + self.qrels = qrels_ds + +class AbsTaskI2TRetrieval(AbsTask): + """Abstract class for retrieval experiments. + + Child-classes must implement the following properties: + + self.corpus: dict[str, dict[str, str]] + Semantically, it should contain dict[split_name, dict[sample_id, dict[str, str]]] + E.g. {"test": {"document_one": {"_id": "d1", "title": "title", "text": "text"}}} + + self.queries: dict[str, dict[str, Union[str, List[str]]]] + Semantically, it should contain dict[split_name, dict[sample_id, str]] or dict[split_name, dict[sample_id, List[str]]] for conversations + E.g. {"test": {"q1": "query"}} + or {"test": {"q1": ["turn1", "turn2", "turn3"]}} + + self.relevant_docs: dict[str, dict[str, dict[str, int]]] + Semantically, it should contain dict[split_name, dict[sample_id, dict[doc_id, score]]] + E.g.: {"test": {"q1": {"document_one": 1}}} + """ + + ignore_identical_ids: bool = False + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = {}, {}, {} + dataset_path = self.metadata_dict["dataset"]["path"] + hf_repo_qrels = ( + dataset_path + "-qrels" if "clarin-knext" in dataset_path else None + ) + for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): + corpus, queries, qrels = HFDataLoader( + hf_repo=dataset_path, + streaming=False, + keep_in_memory=False, + ).load(split=split) + # Conversion from DataSet + queries = {query["id"]: query["image"] for query in queries} + corpus = { + doc["id"]: {"text": doc["text"]} + for doc in corpus + } + self.corpus[split], self.queries[split], self.relevant_docs[split] = ( + corpus, + queries, + qrels, + ) + + self.data_loaded = True + + def evaluate( + self, + model, + split: str = "test", + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ): + retriever = I2TRetrievalEvaluator( + retriever=model, + task_name=self.metadata.name, + encode_kwargs=encode_kwargs, + **kwargs, + ) + + scores = {} + hf_subsets = ( + [l for l in self.hf_subsets] if self.is_multilingual else ["default"] + ) + + for hf_subset in hf_subsets: + logger.info(f"Subset: {hf_subset}") + + if hf_subset == "default": + corpus, queries, relevant_docs = ( + self.corpus[split], + self.queries[split], + self.relevant_docs[split], + ) + else: + corpus, queries, relevant_docs = ( + self.corpus[hf_subset][split], + self.queries[hf_subset][split], + self.relevant_docs[hf_subset][split], + ) + scores[hf_subset] = self._evaluate_subset( + retriever, corpus, queries, relevant_docs, hf_subset, **kwargs + ) + return scores + + def _evaluate_subset( + self, retriever, corpus, queries, relevant_docs, hf_subset: str, **kwargs + ): + start_time = time() + results = retriever(corpus, queries) + end_time = time() + logger.info( + "Time taken to retrieve: {:.2f} seconds".format(end_time - start_time) + ) + + save_predictions = kwargs.get("save_predictions", False) + export_errors = kwargs.get("export_errors", False) + if save_predictions or export_errors: + output_folder = Path(kwargs.get("output_folder", "results")) + if not os.path.isdir(output_folder): + os.makedirs(output_folder) + + if save_predictions: + top_k = kwargs.get("top_k", None) + if top_k is not None: + for qid in list(results.keys()): + doc_ids = set( + sorted( + results[qid], key=lambda x: results[qid][x], reverse=True + )[:top_k] + ) + results[qid] = { + k: v for k, v in results[qid].items() if k in doc_ids + } + qrels_save_path = ( + output_folder / f"{self.metadata.name}_{hf_subset}_predictions.json" + ) + + with open(qrels_save_path, "w") as f: + json.dump(results, f) + + ndcg, _map, recall, precision, naucs = retriever.evaluate( + relevant_docs, + results, + retriever.k_values, + ignore_identical_ids=self.ignore_identical_ids, + ) + mrr, naucs_mrr = retriever.evaluate_custom( + relevant_docs, results, retriever.k_values, "mrr" + ) + scores = { + **{f"ndcg_at_{k.split('@')[1]}": v for (k, v) in ndcg.items()}, + **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()}, + **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()}, + **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()}, + **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, + **{ + k.replace("@", "_at_").replace("_P", "_precision").lower(): v + for k, v in naucs.items() + }, + **{ + k.replace("@", "_at_").replace("_P", "_precision").lower(): v + for k, v in naucs_mrr.items() + }, + } + self._add_main_score(scores) + + if export_errors: + errors = {} + + top_k = kwargs.get("top_k", 1) + if not save_predictions and top_k == 1: + for qid in results.keys(): + doc_scores = results[qid] + sorted_docs = sorted( + doc_scores.items(), key=lambda x: x[1], reverse=True + )[:top_k] + results[qid] = {doc_id: score for doc_id, score in sorted_docs} + for qid, retrieved_docs in results.items(): + expected_docs = relevant_docs[qid] + false_positives = [ + doc for doc in retrieved_docs if doc not in expected_docs + ] + false_negatives = [ + doc for doc in expected_docs if doc not in retrieved_docs + ] + if false_positives or false_negatives: + errors[qid] = { + "false_positives": false_positives, + "false_negatives": false_negatives, + } + + errors_save_path = ( + output_folder / f"{self.metadata.name}_{hf_subset}_errors.json" + ) + with open(errors_save_path, "w") as f: + json.dump(errors, f) + + return scores + + def _add_main_score(self, scores: ScoresDict) -> None: + scores["main_score"] = scores[self.metadata.main_score] + + def calculate_metadata_metrics(self) -> None: + self.load_data() + + all_details = {} + pbar_split = tqdm.tqdm( + self.metadata_dict["eval_splits"], desc="Processing Splits..." + ) + for split in pbar_split: + pbar_split.set_postfix_str(f"Split: {split}") + print(f"Processing metadata for split {split}") + all_details[split] = {} + if self.is_multilingual: + pbar_lang = tqdm.tqdm( + self.relevant_docs.keys(), desc="Processing Languages..." + ) + for lang in pbar_lang: + pbar_lang.set_postfix_str(f"Language: {lang}") + print(f"Processing metadata for language {lang}") + split_details = process_language( + self.relevant_docs[lang][split], + self.queries[lang][split], + self.corpus[lang][split], + lang, + ) + all_details[split][lang] = split_details + else: + split_details = process_language( + self.relevant_docs[split], self.queries[split], self.corpus[split] + ) + all_details[split] = split_details + + return all_details + + +def process_language(relevant_docs, queries, corpus, lang=None): + """We want to get three pieces of information: + - the number of documents (and their char length) in the corpus + - the number of queries (and their char length) + - the average number of relevant documents per query + """ + query_len, doc_len = calculate_length(queries, corpus) + num_documents = len(corpus) + num_queries = len(queries) + + # number of qrels that are not 0 + num_qrels_non_zero = sum( + sum(1 for doc_id in docs if docs[doc_id] != 0) + for docs in relevant_docs.values() + ) + qrels_per_doc = num_qrels_non_zero / num_queries if num_queries else 0 + + language_description = f" for language {lang}" if lang else "" + print(f"Average document character length{language_description} is {doc_len}") + print(f"Average query character length{language_description} is {query_len}") + print(f"Number of documents{language_description} is {num_documents}") + print(f"Number of queries{language_description} is {num_queries}") + print( + f"Average number of relevant documents per query{language_description} is {qrels_per_doc}" + ) + return { + "average_document_length": doc_len, + "average_query_length": query_len, + "num_documents": num_documents, + "num_queries": num_queries, + "average_relevant_docs_per_query": qrels_per_doc, + } + + +def calculate_length(queries, corpus): + queries_lens = [] + doc_lens = [] + for query in queries.values(): + queries_lens.append(len(query)) + + for doc in corpus.values(): + if isinstance(doc, dict): + doc_lens.append(len(doc.get("title", "")) + len(doc["text"])) + else: + doc_lens.append(len(doc)) + + doc_len = sum(doc_lens) / len(doc_lens) if doc_lens else 0 + query_len = sum(queries_lens) / len(queries_lens) if queries_lens else 0 + return query_len, doc_len diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index 3763090d84..a034bd949d 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -15,4 +15,5 @@ from .AbsTaskSummarization import * from .MultilingualTask import * from .Image.AbsTaskZeroshotClassification import * -from .Image.AbsTaskImageClassification import * \ No newline at end of file +from .Image.AbsTaskImageClassification import * +from .Image.AbsTaskI2TRetrieval import * \ No newline at end of file diff --git a/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py new file mode 100644 index 0000000000..4948a6a740 --- /dev/null +++ b/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import heapq +import json +import logging +import os +from collections import defaultdict +from typing import Any, Dict, List, Tuple, Union + +import numpy as np +import pytrec_eval +import torch +import tqdm +from mteb.encoder_interface import EncoderWithQueryCorpusEncode + +from ..Evaluator import Evaluator +from ..utils import ( + confidence_scores, + convert_conv_history_to_query, + cos_sim, + dot_score, + download, + hole, + mrr, + nAUC, + recall_cap, + top_k_accuracy, +) +from PIL import Image + +logger = logging.getLogger(__name__) + + +# Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/search/dense/exact_search.py#L12 +class DenseRetrievalExactSearch: + def __init__( + self, + model: EncoderWithQueryCorpusEncode, + encode_kwargs: dict[str, Any] = {}, + corpus_chunk_size: int = 50000, + previous_results: str | None = None, + **kwargs: Any, + ): + # Model is class that provides get_text_embeddings() and get_image_embeddings() + self.model = model + self.encode_kwargs = encode_kwargs + + if "batch_size" not in encode_kwargs: + encode_kwargs["batch_size"] = 128 + + self.score_functions = {"cos_sim": cos_sim, "dot": dot_score} + self.score_function_desc = { + "cos_sim": "Cosine Similarity", + "dot": "Dot Product", + } + self.corpus_chunk_size = corpus_chunk_size + self.previous_results = previous_results + self.batch_size = encode_kwargs.get("batch_size") + self.show_progress_bar = encode_kwargs.get("show_progress_bar") + self.save_corpus_embeddings = kwargs.get("save_corpus_embeddings", False) + self.corpus_embeddings = defaultdict(list) + self.results = {} + + if self.previous_results is not None: + self.previous_results = self.load_results_file() + + def search( + self, + corpus: dict[str, dict[str, str]], + queries: dict[str, Image.Image], + top_k: int, + score_function: str, + return_sorted: bool = False, + **kwargs, + ) -> dict[str, dict[str, float]]: + + if score_function not in self.score_functions: + raise ValueError( + f"score function: {score_function} must be either (cos_sim) for cosine similarity or (dot) for dot product" + ) + + logger.info("Encoding Queries.") + query_ids = list(queries.keys()) + self.results = {qid: {} for qid in query_ids} + queries = [queries[qid] for qid in queries] + query_embeddings = self.model.get_image_embeddings(queries, + batch_size=self.encode_kwargs["batch_size"]) + + logger.info("Sorting Corpus by document length (Longest first)...") + corpus_ids = sorted( + corpus, + key=lambda k: len(corpus[k].get("text", "")),# no "title" as in text retrieval. + reverse=True, + ) + + corpus = [corpus[cid] for cid in corpus_ids] + + logger.info("Encoding Corpus in batches... Warning: This might take a while!") + logger.info( + "Scoring Function: {} ({})".format( + self.score_function_desc[score_function], score_function + ) + ) + + itr = range(0, len(corpus), self.corpus_chunk_size) + + result_heaps = { + qid: [] for qid in query_ids + } # Keep only the top-k docs for each query + for batch_num, corpus_start_idx in enumerate(itr): + logger.info("Encoding Batch {}/{}...".format(batch_num + 1, len(itr))) + corpus_end_idx = min(corpus_start_idx + self.corpus_chunk_size, len(corpus)) + + # Encode chunk of corpus + if ( + self.save_corpus_embeddings + and "qid" in kwargs + and len(self.corpus_embeddings[kwargs["qid"]]) + ): + sub_corpus_embeddings = torch.tensor( + self.corpus_embeddings[kwargs["qid"]][batch_num] + ) + else: + # Encode chunk of corpus + texts = [doc["text"] for doc in corpus[corpus_start_idx:corpus_end_idx]] + sub_corpus_embeddings = self.model.get_text_embeddings( + # corpus[corpus_start_idx:corpus_end_idx], + texts, + batch_size=self.encode_kwargs["batch_size"] + ) + if self.save_corpus_embeddings and "qid" in kwargs: + self.corpus_embeddings[kwargs["qid"]].append(sub_corpus_embeddings) + + # Compute similarites using either cosine-similarity or dot product + cos_scores = self.score_functions[score_function]( + query_embeddings, sub_corpus_embeddings + ) + cos_scores[torch.isnan(cos_scores)] = -1 + + # Get top-k values + cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( + cos_scores, + min( + top_k + 1, + len(cos_scores[1]) if len(cos_scores) > 1 else len(cos_scores[-1]), + ), + dim=1, + largest=True, + sorted=return_sorted, + ) + cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() + cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() + + for query_itr in range(len(query_embeddings)): + query_id = query_ids[query_itr] + for sub_corpus_id, score in zip( + cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] + ): + corpus_id = corpus_ids[corpus_start_idx + sub_corpus_id] + if len(result_heaps[query_id]) < top_k: + # Push item on the heap + heapq.heappush(result_heaps[query_id], (score, corpus_id)) + else: + # If item is larger than the smallest in the heap, push it on the heap then pop the smallest element + heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) + + for qid in result_heaps: + for score, corpus_id in result_heaps[qid]: + self.results[qid][corpus_id] = score + + return self.results + + def load_results_file(self): + # load the first stage results from file in format {qid: {doc_id: score}} + if "https://" in self.previous_results: + # download the file + if not os.path.exists(self.previous_results): + url_descriptor = self.previous_results.split("https://")[-1].replace( + "/", "--" + ) + dest_file = os.path.join( + "results", f"cached_predictions--{url_descriptor}" + ) + os.makedirs(os.path.dirname(os.path.abspath(dest_file)), exist_ok=True) + download(self.previous_results, dest_file) + logger.info( + f"Downloaded the previous results at {self.previous_results} to {dest_file}" + ) + self.previous_results = dest_file + + with open(self.previous_results, "r") as f: + previous_results = json.load(f) + assert isinstance(previous_results, dict) + assert isinstance(previous_results[list(previous_results.keys())[0]], dict) + return previous_results + + +# Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/evaluation.py#L9 +class I2TRetrievalEvaluator(Evaluator): + def __init__( + self, + retriever=None, + task_name: str | None = None, + k_values: List[int] = [1, 3, 5, 10, 20, 100, 1000], + score_function: str = "cos_sim", + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ): + super().__init__(**kwargs) + + self.retriever = DenseRetrievalExactSearch( + retriever, encode_kwargs=encode_kwargs, **kwargs + ) + self.k_values = k_values + self.top_k = ( + max(k_values) if "top_k" not in kwargs else kwargs["top_k"] + ) # can lower it if reranking + self.score_function = score_function + self.task_name = task_name + + def __call__( + self, + corpus: dict[str, dict[str, str]], + queries: dict[str, Union[str, List[str]]], + ) -> dict[str, dict[str, float]]: + if not self.retriever: + raise ValueError("Model/Technique has not been provided!") + + + return self.retriever.search( + corpus, + queries, + self.top_k, + self.score_function, + prompt_name=self.task_name, # type: ignore + ) + + @staticmethod + def evaluate( + qrels: dict[str, dict[str, int]], + results: dict[str, dict[str, float]], + k_values: List[int], + ignore_identical_ids: bool = False, + ) -> Tuple[ + dict[str, float], + dict[str, float], + dict[str, float], + dict[str, float], + dict[str, float], + ]: + if ignore_identical_ids: + logger.debug( + "For evaluation, ``ignore_identical_ids=True`` is set to True, the evaluator will ignore identical query and document ids." + ) + # Remove identical ids from results dict + for qid, rels in results.items(): + for pid in list(rels): + if qid == pid: + results[qid].pop(pid) + else: + logger.debug( + "For evaluation, we DO NOT ignore identical query and document ids (default), please explicitly set ``ignore_identical_ids=True`` to ignore this." + ) + + all_ndcgs, all_aps, all_recalls, all_precisions = {}, {}, {}, {} + + for k in k_values: + all_ndcgs[f"NDCG@{k}"] = [] + all_aps[f"MAP@{k}"] = [] + all_recalls[f"Recall@{k}"] = [] + all_precisions[f"P@{k}"] = [] + + map_string = "map_cut." + ",".join([str(k) for k in k_values]) + ndcg_string = "ndcg_cut." + ",".join([str(k) for k in k_values]) + recall_string = "recall." + ",".join([str(k) for k in k_values]) + precision_string = "P." + ",".join([str(k) for k in k_values]) + evaluator = pytrec_eval.RelevanceEvaluator( + qrels, {map_string, ndcg_string, recall_string, precision_string} + ) + scores = evaluator.evaluate(results) + + for query_id in scores.keys(): + for k in k_values: + all_ndcgs[f"NDCG@{k}"].append(scores[query_id]["ndcg_cut_" + str(k)]) + all_aps[f"MAP@{k}"].append(scores[query_id]["map_cut_" + str(k)]) + all_recalls[f"Recall@{k}"].append(scores[query_id]["recall_" + str(k)]) + all_precisions[f"P@{k}"].append(scores[query_id]["P_" + str(k)]) + + ndcg, _map, recall, precision = ( + all_ndcgs.copy(), + all_aps.copy(), + all_recalls.copy(), + all_precisions.copy(), + ) + + for k in k_values: + ndcg[f"NDCG@{k}"] = round(sum(ndcg[f"NDCG@{k}"]) / len(scores), 5) + _map[f"MAP@{k}"] = round(sum(_map[f"MAP@{k}"]) / len(scores), 5) + recall[f"Recall@{k}"] = round(sum(recall[f"Recall@{k}"]) / len(scores), 5) + precision[f"P@{k}"] = round(sum(precision[f"P@{k}"]) / len(scores), 5) + + naucs = I2TRetrievalEvaluator.evaluate_abstention( + results, {**all_ndcgs, **all_aps, **all_recalls, **all_precisions} + ) + + return ndcg, _map, recall, precision, naucs + + @staticmethod + def evaluate_custom( + qrels: dict[str, dict[str, int]], + results: dict[str, dict[str, float]], + k_values: List[int], + metric: str, + output_type: str = "all", + ) -> Tuple[Dict[str, float]]: + if metric.lower() in ["mrr", "mrr@k", "mrr_cut"]: + metric_scores = mrr(qrels, results, k_values, output_type) + + elif metric.lower() in ["recall_cap", "r_cap", "r_cap@k"]: + metric_scores = recall_cap(qrels, results, k_values, output_type) + + elif metric.lower() in ["hole", "hole@k"]: + metric_scores = hole(qrels, results, k_values, output_type) + + elif metric.lower() in [ + "acc", + "top_k_acc", + "accuracy", + "accuracy@k", + "top_k_accuracy", + ]: + metric_scores = top_k_accuracy(qrels, results, k_values, output_type) + + naucs = I2TRetrievalEvaluator.evaluate_abstention(results, metric_scores) + metric_scores_avg = {k: sum(v) / len(v) for k, v in metric_scores.items()} + + return metric_scores_avg, naucs + + @staticmethod + def evaluate_abstention( + results: dict[str, dict[str, float]], + metric_scores: dict[str, list[float]], + ) -> Dict[str, float]: + """Computes normalized Area Under the Curve on a set of evaluated instances as presented in the paper https://arxiv.org/abs/2402.12997""" + all_sim_scores = [list(results[qid].values()) for qid in list(results.keys())] + all_conf_scores = [ + confidence_scores(sim_scores) for sim_scores in all_sim_scores + ] + conf_fcts = list(all_conf_scores[0].keys()) + all_conf_scores = { + fct: np.array([x[fct] for x in all_conf_scores]) for fct in conf_fcts + } + metric_scores = {k: np.array(v) for k, v in metric_scores.items()} + naucs = {} + + for metric_name, scores in metric_scores.items(): + for fct, conf_scores in all_conf_scores.items(): + naucs[f"nAUC_{metric_name}_{fct}"] = nAUC(conf_scores, scores) + + return naucs diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py index 49aa7f5e6d..24c06fe9d3 100644 --- a/mteb/evaluation/evaluators/__init__.py +++ b/mteb/evaluation/evaluators/__init__.py @@ -9,4 +9,5 @@ from .STSEvaluator import * from .SummarizationEvaluator import * from .Image.ZeroshotClassificationEvaluator import * -from .Image.ClassificationEvaluator import * \ No newline at end of file +from .Image.ClassificationEvaluator import * +from .Image.I2TRetrievalEvaluator import * \ No newline at end of file diff --git a/mteb/tasks/Image/I2TRetrieval/__init__.py b/mteb/tasks/Image/I2TRetrieval/__init__.py new file mode 100644 index 0000000000..ea4bd2bae0 --- /dev/null +++ b/mteb/tasks/Image/I2TRetrieval/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .eng.MSCOCOI2TRetrieval import * \ No newline at end of file diff --git a/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py b/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py new file mode 100644 index 0000000000..1f3c0d4ba1 --- /dev/null +++ b/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskI2TRetrieval + + +class MSCOCOI2TRetrieval(AbsTaskI2TRetrieval): + metadata = TaskMetadata( + name="MSCOCOI2TRetrieval", + description="Retrieve captions based on images.", + reference="https://link.springer.com/chapter/10.1007/978-3-319-10602-1_48", + dataset={ + "path": "MRBench/mbeir_mscoco_task3", + "revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", + "trust_remote_code": True, + }, + type="Retrieval", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2018-01-01", "2018-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Reasoning as Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + text_creation="found", + bibtex_citation="""@inproceedings{lin2014microsoft, + title={Microsoft coco: Common objects in context}, + author={Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\'a}r, Piotr and Zitnick, C Lawrence}, + booktitle={Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part V 13}, + pages={740--755}, + year={2014}, + organization={Springer} +}""", + n_samples={"test": 1172}, + avg_character_length={ + "test": { + "average_document_length": 30.94235294117647, + "average_query_length": 131.56569965870307, + "num_documents": 9350, + "num_queries": 1172, + "average_relevant_docs_per_query": 1.0, + } + }, + ) diff --git a/mteb/tasks/Image/I2TRetrieval/eng/__init__.py b/mteb/tasks/Image/I2TRetrieval/eng/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/__init__.py b/mteb/tasks/__init__.py index df3098a718..f6a0b49d91 100644 --- a/mteb/tasks/__init__.py +++ b/mteb/tasks/__init__.py @@ -12,4 +12,5 @@ from .STS import * from .Summarization import * from .Image.ZeroshotClassification import * -from .Image.ImageClassification import * \ No newline at end of file +from .Image.ImageClassification import * +from .Image.I2TRetrieval import * \ No newline at end of file From 976acc5f18bfca227ad451119799cd1a8f981d4c Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 15 Jul 2024 19:32:47 +0100 Subject: [PATCH 010/154] test load data ignore i2tretrieval --- tests/test_all_abstasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_all_abstasks.py b/tests/test_all_abstasks.py index b67124b4b7..db33e67e43 100644 --- a/tests/test_all_abstasks.py +++ b/tests/test_all_abstasks.py @@ -12,6 +12,7 @@ from mteb.abstasks import AbsTask from mteb.abstasks.AbsTaskInstructionRetrieval import AbsTaskInstructionRetrieval from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.Image.AbsTaskI2TRetrieval import AbsTaskI2TRetrieval from mteb.abstasks.AbsTaskSpeedTask import AbsTaskSpeedTask from mteb.abstasks.MultiSubsetLoader import MultiSubsetLoader from mteb.overview import TASKS_REGISTRY @@ -30,6 +31,7 @@ def test_load_data( # TODO: We skip because this load_data is completely different. if ( isinstance(task, AbsTaskRetrieval) + or isinstance(task, AbsTaskI2TRetrieval) or isinstance(task, AbsTaskInstructionRetrieval) or isinstance(task, MultiSubsetLoader) or isinstance(task, AbsTaskSpeedTask) From ddc4b6e38f0d4e2a4a4a08c0e35cb5b16efd6cff Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 15 Jul 2024 22:03:06 +0300 Subject: [PATCH 011/154] [MIEB] Add image clustering (#1088) * make lint * wip * add TinyImageNet and run * type hints * add accuracy * lint --- mteb/abstasks/Image/AbsTaskI2TRetrieval.py | 11 ++-- mteb/abstasks/Image/AbsTaskImageClustering.py | 50 ++++++++++++++++ .../Image/AbsTaskZeroshotClassification.py | 11 ++-- mteb/abstasks/__init__.py | 7 ++- .../Image/ClassificationEvaluator.py | 26 ++++++--- .../evaluators/Image/ClusteringEvaluator.py | 58 +++++++++++++++++++ .../evaluators/Image/I2TRetrievalEvaluator.py | 24 ++++---- .../Image/ZeroshotClassificationEvaluator.py | 19 +++--- mteb/evaluation/evaluators/Image/__init__.py | 2 +- mteb/evaluation/evaluators/__init__.py | 7 ++- mteb/models/__init__.py | 5 +- mteb/models/clip_models.py | 49 ++++++++++------ mteb/tasks/Image/Clustering/TinyImageNet.py | 36 ++++++++++++ mteb/tasks/Image/Clustering/__init__.py | 3 + mteb/tasks/Image/I2TRetrieval/__init__.py | 2 +- .../Image/ImageClassification/__init__.py | 2 +- .../eng/OxfordFlowersClassification.py | 1 + .../Image/ZeroshotClassification/__init__.py | 2 +- .../eng/RenderedSST2.py | 4 +- mteb/tasks/Image/__init__.py | 3 +- mteb/tasks/__init__.py | 6 +- .../TinyImageNetClustering.json | 19 ++++++ tests/test_all_abstasks.py | 2 +- 23 files changed, 271 insertions(+), 78 deletions(-) create mode 100644 mteb/abstasks/Image/AbsTaskImageClustering.py create mode 100644 mteb/evaluation/evaluators/Image/ClusteringEvaluator.py create mode 100644 mteb/tasks/Image/Clustering/TinyImageNet.py create mode 100644 mteb/tasks/Image/Clustering/__init__.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TinyImageNetClustering.json diff --git a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py index becfd79241..53e89e0bd9 100644 --- a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py @@ -6,8 +6,8 @@ from collections import defaultdict from pathlib import Path from time import time -from typing import Any, Dict, Tuple, Union -from PIL import Image +from typing import Any, Dict, Tuple + import tqdm from datasets import Features, Value, load_dataset @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) + class HFDataLoader: def __init__( self, @@ -178,6 +179,7 @@ def _load_qrels(self, split): qrels_ds = qrels_ds.cast(features) self.qrels = qrels_ds + class AbsTaskI2TRetrieval(AbsTask): """Abstract class for retrieval experiments. @@ -218,10 +220,7 @@ def load_data(self, **kwargs): ).load(split=split) # Conversion from DataSet queries = {query["id"]: query["image"] for query in queries} - corpus = { - doc["id"]: {"text": doc["text"]} - for doc in corpus - } + corpus = {doc["id"]: {"text": doc["text"]} for doc in corpus} self.corpus[split], self.queries[split], self.relevant_docs[split] = ( corpus, queries, diff --git a/mteb/abstasks/Image/AbsTaskImageClustering.py b/mteb/abstasks/Image/AbsTaskImageClustering.py new file mode 100644 index 0000000000..a2dfbb9d19 --- /dev/null +++ b/mteb/abstasks/Image/AbsTaskImageClustering.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import logging +from typing import Any + +from datasets import Dataset + +from mteb.abstasks import AbsTask +from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from mteb.evaluation.evaluators import ImageClusteringEvaluator +from mteb.load_results.mteb_results import HFSubset, ScoresDict + +logger = logging.getLogger(__name__) + + +class AbsTaskImageClustering(AbsTask): + """Abstract class for Clustering tasks + The similarity is computed between pairs and the results are ranked. + + self.load_data() must generate a huggingface dataset with a split matching self.metadata_dict["eval_splits"], and assign it to self.dataset. It must contain the following columns: + image: Image.Image + label: int + """ + + image_column_name: str = "image" + label_column_name: str = "label" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def _add_main_score(self, scores: dict[HFSubset, ScoresDict]) -> None: + scores["main_score"] = scores[self.metadata.main_score] + + def _evaluate_subset( + self, + model: EncoderWithQueryCorpusEncode | Encoder, + dataset: Dataset, + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ) -> ScoresDict: + evaluator = ImageClusteringEvaluator( + dataset[self.image_column_name], + dataset[self.label_column_name], + task_name=self.metadata.name, + **kwargs, + ) + metrics = evaluator(model, encode_kwargs=encode_kwargs) + self._add_main_score(metrics) + return scores diff --git a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py index 9ba1e49589..c2581f6214 100644 --- a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py +++ b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py @@ -3,7 +3,6 @@ import logging from typing import Any -import tqdm from datasets import Dataset from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode @@ -41,23 +40,21 @@ def _evaluate_subset( encode_kwargs: dict[str, Any] = {}, **kwargs, ) -> ScoresDict: - candidate_labels = self.get_candidate_labels() - + evaluator = ZeroshotClassificationEvaluator( - dataset[self.image_column_name], - dataset[self.label_column_name], + dataset[self.image_column_name], + dataset[self.label_column_name], candidate_labels, task_name=self.metadata.name, **kwargs, ) metrics = evaluator(model, encode_kwargs=encode_kwargs) - scores = {"accuracy": metrics["accuracy"]} self._add_main_score(scores) return scores - + def get_candidate_labels(self) -> list[str]: """Return the text candidates for zeroshot classification""" raise NotImplementedError("This method should be overridden by subclasses") diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index a034bd949d..57e5c3f9cb 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -13,7 +13,8 @@ from .AbsTaskSpeedTask import * from .AbsTaskSTS import * from .AbsTaskSummarization import * -from .MultilingualTask import * -from .Image.AbsTaskZeroshotClassification import * +from .Image.AbsTaskI2TRetrieval import * from .Image.AbsTaskImageClassification import * -from .Image.AbsTaskI2TRetrieval import * \ No newline at end of file +from .Image.AbsTaskImageClustering import * +from .Image.AbsTaskZeroshotClassification import * +from .MultilingualTask import * diff --git a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py index 984a929a6a..1252efaeae 100644 --- a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py @@ -15,7 +15,6 @@ from torch import Tensor from mteb.encoder_interface import Encoder -from mteb.evaluation.evaluators.model_encode import model_encode from ..Evaluator import Evaluator @@ -63,9 +62,13 @@ def __call__(self, model, test_cache=None): max_accuracy = 0 max_f1 = 0 max_ap = 0 - X_train = model.get_image_embeddings(self.images_train, batch_size=self.encode_kwargs["batch_size"]) + X_train = model.get_image_embeddings( + self.images_train, batch_size=self.encode_kwargs["batch_size"] + ) if test_cache is None: - X_test = model.get_image_embeddings(self.images_test, batch_size=self.encode_kwargs["batch_size"]) + X_test = model.get_image_embeddings( + self.images_test, batch_size=self.encode_kwargs["batch_size"] + ) test_cache = X_test else: X_test = test_cache @@ -129,10 +132,14 @@ def __call__(self, model: Encoder, test_cache=None): max_accuracy = 0 max_f1 = 0 max_ap = 0 - X_train = model.get_image_embeddings(self.images_train, batch_size=self.encode_kwargs["batch_size"]) + X_train = model.get_image_embeddings( + self.images_train, batch_size=self.encode_kwargs["batch_size"] + ) if test_cache is None: - X_test = model.get_image_embeddings(self.images_test, batch_size=self.encode_kwargs["batch_size"]) + X_test = model.get_image_embeddings( + self.images_test, batch_size=self.encode_kwargs["batch_size"] + ) test_cache = X_test else: X_test = test_cache @@ -273,10 +280,14 @@ def __call__(self, model, test_cache=None): max_iter=self.max_iter, verbose=1 if logger.isEnabledFor(logging.DEBUG) else 0, ) - X_train = model.get_image_embeddings(self.images_train, batch_size=self.encode_kwargs["batch_size"]) + X_train = model.get_image_embeddings( + self.images_train, batch_size=self.encode_kwargs["batch_size"] + ) if test_cache is None: - X_test = model.get_image_embeddings(self.images_test, batch_size=self.encode_kwargs["batch_size"]) + X_test = model.get_image_embeddings( + self.images_test, batch_size=self.encode_kwargs["batch_size"] + ) test_cache = X_test else: X_test = test_cache @@ -296,4 +307,3 @@ def __call__(self, model, test_cache=None): ) return scores, test_cache - \ No newline at end of file diff --git a/mteb/evaluation/evaluators/Image/ClusteringEvaluator.py b/mteb/evaluation/evaluators/Image/ClusteringEvaluator.py new file mode 100644 index 0000000000..b006470416 --- /dev/null +++ b/mteb/evaluation/evaluators/Image/ClusteringEvaluator.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import logging +from typing import Any + +import sklearn +import sklearn.cluster +from PIL import Image +from sklearn import metrics + +from mteb.encoder_interface import Encoder +from mteb.evaluation.evaluators.Evaluator import Evaluator + +logger = logging.getLogger(__name__) + + +class ImageClusteringEvaluator(Evaluator): + def __init__( + self, + images: list[Image.Image], + labels: list[int], + task_name: str | None = None, + clustering_batch_size: int = 500, + limit: int | None = None, + **kwargs, + ): + super().__init__(**kwargs) + if limit is not None: + images = images[:limit] + labels = labels[:limit] + self.images = images + self.labels = labels + self.clustering_batch_size = clustering_batch_size + self.task_name = task_name + + def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): + if "batch_size" not in encode_kwargs: + encode_kwargs["batch_size"] = 32 + + image_embeddings = model.get_image_embeddings( + self.images, + batch_size=encode_kwargs["batch_size"], + ) + + logger.info("Fitting Mini-Batch K-Means model...") + clustering_model = sklearn.cluster.MiniBatchKMeans( + n_clusters=len(set(self.labels)), + batch_size=self.clustering_batch_size, + n_init="auto", + ) + clustering_model.fit(image_embeddings) + cluster_assignment = clustering_model.labels_ + + logger.info("Evaluating...") + v_measure = metrics.cluster.v_measure_score(self.labels, cluster_assignment) + accuracy = metrics.accuracy_score(self.labels, cluster_assignment) + + return {"v_measure": v_measure, "accuracy": accuracy} diff --git a/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py index 4948a6a740..e23c4ca0d8 100644 --- a/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py @@ -10,13 +10,13 @@ import numpy as np import pytrec_eval import torch -import tqdm +from PIL import Image + from mteb.encoder_interface import EncoderWithQueryCorpusEncode from ..Evaluator import Evaluator from ..utils import ( confidence_scores, - convert_conv_history_to_query, cos_sim, dot_score, download, @@ -26,7 +26,6 @@ recall_cap, top_k_accuracy, ) -from PIL import Image logger = logging.getLogger(__name__) @@ -35,7 +34,7 @@ class DenseRetrievalExactSearch: def __init__( self, - model: EncoderWithQueryCorpusEncode, + model: EncoderWithQueryCorpusEncode, encode_kwargs: dict[str, Any] = {}, corpus_chunk_size: int = 50000, previous_results: str | None = None, @@ -47,7 +46,7 @@ def __init__( if "batch_size" not in encode_kwargs: encode_kwargs["batch_size"] = 128 - + self.score_functions = {"cos_sim": cos_sim, "dot": dot_score} self.score_function_desc = { "cos_sim": "Cosine Similarity", @@ -73,7 +72,6 @@ def search( return_sorted: bool = False, **kwargs, ) -> dict[str, dict[str, float]]: - if score_function not in self.score_functions: raise ValueError( f"score function: {score_function} must be either (cos_sim) for cosine similarity or (dot) for dot product" @@ -83,16 +81,19 @@ def search( query_ids = list(queries.keys()) self.results = {qid: {} for qid in query_ids} queries = [queries[qid] for qid in queries] - query_embeddings = self.model.get_image_embeddings(queries, - batch_size=self.encode_kwargs["batch_size"]) + query_embeddings = self.model.get_image_embeddings( + queries, batch_size=self.encode_kwargs["batch_size"] + ) logger.info("Sorting Corpus by document length (Longest first)...") corpus_ids = sorted( corpus, - key=lambda k: len(corpus[k].get("text", "")),# no "title" as in text retrieval. + key=lambda k: len( + corpus[k].get("text", "") + ), # no "title" as in text retrieval. reverse=True, ) - + corpus = [corpus[cid] for cid in corpus_ids] logger.info("Encoding Corpus in batches... Warning: This might take a while!") @@ -126,7 +127,7 @@ def search( sub_corpus_embeddings = self.model.get_text_embeddings( # corpus[corpus_start_idx:corpus_end_idx], texts, - batch_size=self.encode_kwargs["batch_size"] + batch_size=self.encode_kwargs["batch_size"], ) if self.save_corpus_embeddings and "qid" in kwargs: self.corpus_embeddings[kwargs["qid"]].append(sub_corpus_embeddings) @@ -226,7 +227,6 @@ def __call__( if not self.retriever: raise ValueError("Model/Technique has not been provided!") - return self.retriever.search( corpus, queries, diff --git a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py index 3c0fbad51c..03fb0f3f10 100644 --- a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py @@ -3,10 +3,11 @@ import logging from typing import Any +from PIL import Image from sklearn import metrics from mteb.encoder_interface import Encoder -from PIL import Image + from ..Evaluator import Evaluator logger = logging.getLogger(__name__) @@ -15,9 +16,9 @@ class ZeroshotClassificationEvaluator(Evaluator): def __init__( self, - images:list[Image.Image], - labels:list[int], - candidate_labels:list[str], + images: list[Image.Image], + labels: list[int], + candidate_labels: list[str], task_name: str | None = None, **kwargs, ): @@ -31,11 +32,15 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): if "batch_size" not in encode_kwargs: encode_kwargs["batch_size"] = 32 - text_embeddings = model.get_text_embeddings(self.candidate_labels, batch_size=encode_kwargs["batch_size"]) - image_embeddings = model.get_image_embeddings(self.images, batch_size=encode_kwargs["batch_size"]) + text_embeddings = model.get_text_embeddings( + self.candidate_labels, batch_size=encode_kwargs["batch_size"] + ) + image_embeddings = model.get_image_embeddings( + self.images, batch_size=encode_kwargs["batch_size"] + ) probs = model.calculate_probs(text_embeddings, image_embeddings) predictions = probs.argmax(dim=1) - + logger.info("Evaluating...") accuracy = metrics.accuracy_score(self.labels, predictions) diff --git a/mteb/evaluation/evaluators/Image/__init__.py b/mteb/evaluation/evaluators/Image/__init__.py index 0fd2a7478b..a5654a6e8c 100644 --- a/mteb/evaluation/evaluators/Image/__init__.py +++ b/mteb/evaluation/evaluators/Image/__init__.py @@ -1,2 +1,2 @@ # from .ClassificationEvaluator import * -# from .ZeroshotClassificationEvaluator import * \ No newline at end of file +# from .ZeroshotClassificationEvaluator import * diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py index 24c06fe9d3..2f6ca61148 100644 --- a/mteb/evaluation/evaluators/__init__.py +++ b/mteb/evaluation/evaluators/__init__.py @@ -3,11 +3,12 @@ from .BitextMiningEvaluator import * from .ClassificationEvaluator import * from .ClusteringEvaluator import * +from .Image.ClassificationEvaluator import * +from .Image.ClusteringEvaluator import * +from .Image.I2TRetrievalEvaluator import * +from .Image.ZeroshotClassificationEvaluator import * from .PairClassificationEvaluator import * from .RerankingEvaluator import * from .RetrievalEvaluator import * from .STSEvaluator import * from .SummarizationEvaluator import * -from .Image.ZeroshotClassificationEvaluator import * -from .Image.ClassificationEvaluator import * -from .Image.I2TRetrievalEvaluator import * \ No newline at end of file diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index aef5b9ba3a..6afaed17ac 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -8,8 +8,8 @@ from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode from mteb.model_meta import ModelMeta from mteb.models import ( - clip_models, bge_models, + clip_models, cohere_models, e5_instruct, e5_models, @@ -22,7 +22,6 @@ salesforce_models, sentence_transformers_models, voyage_models, - clip_models, ) logger = logging.getLogger(__name__) @@ -133,7 +132,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe ru_sentence_models, nomic_models, cohere_models, - clip_models + clip_models, ] models = {} diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index c8536fa651..df5d8d8871 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -1,13 +1,17 @@ -from PIL import Image -from transformers import AutoProcessor, AutoModel +from __future__ import annotations + +from functools import partial from typing import Any + import torch +from PIL import Image from tqdm import tqdm +from transformers import AutoModel, AutoProcessor + from mteb.model_meta import ModelMeta -from functools import partial + class CLIPModelWrapper: - def __init__( self, model_name: str, @@ -20,19 +24,23 @@ def __init__( self.processor = AutoProcessor.from_pretrained(model_name) def preprocess( - self, - texts: list[str], - images: list[Image.Image], + self, + texts: list[str], + images: list[Image.Image], ): - return self.processor(text=texts, images=images, return_tensors="pt", padding=True) + return self.processor( + text=texts, images=images, return_tensors="pt", padding=True + ) def get_text_embeddings(self, texts: list[str], batch_size: int = 32): all_text_embeddings = [] with torch.no_grad(): for i in tqdm(range(0, len(texts), batch_size)): - batch_texts = texts[i:i+batch_size] - inputs = self.processor(text=batch_texts, return_tensors="pt", padding=True) + batch_texts = texts[i : i + batch_size] + inputs = self.processor( + text=batch_texts, return_tensors="pt", padding=True + ) inputs = {k: v.to(self.device) for k, v in inputs.items()} text_outputs = self.model.get_text_features(**inputs) all_text_embeddings.append(text_outputs.cpu()) @@ -42,25 +50,30 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): def get_image_embeddings(self, images: list[Image.Image], batch_size: int = 32): all_image_embeddings = [] - + with torch.no_grad(): for i in tqdm(range(0, len(images), batch_size)): - batch_images = images[i:i+batch_size] - inputs = self.processor(images=batch_images, return_tensors="pt", padding=True) + batch_images = images[i : i + batch_size] + inputs = self.processor( + images=batch_images, return_tensors="pt", padding=True + ) inputs = {k: v.to(self.device) for k, v in inputs.items()} image_outputs = self.model.get_image_features(**inputs) all_image_embeddings.append(image_outputs.cpu()) all_image_embeddings = torch.cat(all_image_embeddings, dim=0) return all_image_embeddings - + def calculate_probs(self, text_embeddings, image_embeddings): text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) - image_embeddings = image_embeddings / image_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) logits = torch.matmul(image_embeddings, text_embeddings.T) - probs = (logits*100).softmax(dim=-1) + probs = (logits * 100).softmax(dim=-1) return probs - + + clip_vit_large_patch14 = ModelMeta( loader=partial( CLIPModelWrapper, @@ -101,4 +114,4 @@ def calculate_probs(self, text_embeddings, image_embeddings): import mteb mdl = mteb.get_model(clip_vit_base_patch16.name, clip_vit_base_patch16.revision) - emb = mdl.get_text_embeddings(["Hello, world!"]) \ No newline at end of file + emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/tasks/Image/Clustering/TinyImageNet.py b/mteb/tasks/Image/Clustering/TinyImageNet.py new file mode 100644 index 0000000000..37f66e016d --- /dev/null +++ b/mteb/tasks/Image/Clustering/TinyImageNet.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskImageClustering +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class TinyImageNet(AbsTaskImageClustering): + metadata = TaskMetadata( + name="TinyImageNetClustering", + description="Clustering over 200 classes.", + reference="https://huggingface.co/datasets/zh-plus/tiny-imagenet/viewer/default/valid", + dataset={ + "path": "zh-plus/tiny-imagenet", + "revision": "5a77092c28e51558c5586e9c5eb71a7e17a5e43f", + }, + type="Clustering", + category="s2s", + eval_splits=["valid"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2012-01-01", + "2015-12-31", + ), # Estimated range for the collection of reviews + form=["written"], + domains=["Reviews"], + task_subtypes=["Sentiment/Hate speech"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + text_creation="found", + bibtex_citation="""d""", + n_samples={"valid": 10000}, + avg_character_length={"valid": 431.4}, + ) diff --git a/mteb/tasks/Image/Clustering/__init__.py b/mteb/tasks/Image/Clustering/__init__.py new file mode 100644 index 0000000000..36ae667862 --- /dev/null +++ b/mteb/tasks/Image/Clustering/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .TinyImageNet import * diff --git a/mteb/tasks/Image/I2TRetrieval/__init__.py b/mteb/tasks/Image/I2TRetrieval/__init__.py index ea4bd2bae0..c5e58ad927 100644 --- a/mteb/tasks/Image/I2TRetrieval/__init__.py +++ b/mteb/tasks/Image/I2TRetrieval/__init__.py @@ -1,3 +1,3 @@ from __future__ import annotations -from .eng.MSCOCOI2TRetrieval import * \ No newline at end of file +from .eng.MSCOCOI2TRetrieval import * diff --git a/mteb/tasks/Image/ImageClassification/__init__.py b/mteb/tasks/Image/ImageClassification/__init__.py index a3861f99ad..72a89837fc 100644 --- a/mteb/tasks/Image/ImageClassification/__init__.py +++ b/mteb/tasks/Image/ImageClassification/__init__.py @@ -1,3 +1,3 @@ from __future__ import annotations -from .eng.OxfordFlowersClassification import * \ No newline at end of file +from .eng.OxfordFlowersClassification import * diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py index 8f97aa0689..662dd92e3c 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -4,6 +4,7 @@ from .....abstasks import AbsTaskImageClassification + class OxfordFlowersClassification(AbsTaskImageClassification): metadata = TaskMetadata( name="OxfordFlowersClassification", diff --git a/mteb/tasks/Image/ZeroshotClassification/__init__.py b/mteb/tasks/Image/ZeroshotClassification/__init__.py index 8f87fc4cbe..ff5f0b1a87 100644 --- a/mteb/tasks/Image/ZeroshotClassification/__init__.py +++ b/mteb/tasks/Image/ZeroshotClassification/__init__.py @@ -1,3 +1,3 @@ from __future__ import annotations -from .eng.RenderedSST2 import * \ No newline at end of file +from .eng.RenderedSST2 import * diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py index 0274542704..f2a741cf54 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py @@ -36,6 +36,6 @@ class RenderedSST2(AbsTaskZeroshotClassification): # Override default column names in the subclass image_column_name: str = "png" label_column_name: str = "cls" - + def get_candidate_labels(self) -> list[str]: - return ["a negative review of a movie", "a positive review of a movie"] \ No newline at end of file + return ["a negative review of a movie", "a positive review of a movie"] diff --git a/mteb/tasks/Image/__init__.py b/mteb/tasks/Image/__init__.py index c9c76d48b3..c5c14e726a 100644 --- a/mteb/tasks/Image/__init__.py +++ b/mteb/tasks/Image/__init__.py @@ -1,2 +1,3 @@ +from .Clustering import * +from .ImageClassification import * from .ZeroshotClassification import * -from .ImageClassification import * \ No newline at end of file diff --git a/mteb/tasks/__init__.py b/mteb/tasks/__init__.py index f6a0b49d91..9c34b18bbc 100644 --- a/mteb/tasks/__init__.py +++ b/mteb/tasks/__init__.py @@ -3,6 +3,9 @@ from .BitextMining import * from .Classification import * from .Clustering import * +from .Image.I2TRetrieval import * +from .Image.ImageClassification import * +from .Image.ZeroshotClassification import * from .InstructionRetrieval import * from .MultiLabelClassification import * from .PairClassification import * @@ -11,6 +14,3 @@ from .SpeedTask import * from .STS import * from .Summarization import * -from .Image.ZeroshotClassification import * -from .Image.ImageClassification import * -from .Image.I2TRetrieval import * \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TinyImageNetClustering.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TinyImageNetClustering.json new file mode 100644 index 0000000000..ec2d58ec81 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TinyImageNetClustering.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "5a77092c28e51558c5586e9c5eb71a7e17a5e43f", + "evaluation_time": 401.41785073280334, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "valid": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6290288786644828, + "v_measure": 0.6290288786644828 + } + ] + }, + "task_name": "TinyImageNetClustering" +} \ No newline at end of file diff --git a/tests/test_all_abstasks.py b/tests/test_all_abstasks.py index db33e67e43..4392fc9105 100644 --- a/tests/test_all_abstasks.py +++ b/tests/test_all_abstasks.py @@ -12,8 +12,8 @@ from mteb.abstasks import AbsTask from mteb.abstasks.AbsTaskInstructionRetrieval import AbsTaskInstructionRetrieval from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.Image.AbsTaskI2TRetrieval import AbsTaskI2TRetrieval from mteb.abstasks.AbsTaskSpeedTask import AbsTaskSpeedTask +from mteb.abstasks.Image.AbsTaskI2TRetrieval import AbsTaskI2TRetrieval from mteb.abstasks.MultiSubsetLoader import MultiSubsetLoader from mteb.overview import TASKS_REGISTRY From 9e50f22fd496ed64794ef0648d378029007d4124 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 15 Jul 2024 20:18:53 +0100 Subject: [PATCH 012/154] remove unused & fix typos --- mteb/abstasks/Image/AbsTaskI2TRetrieval.py | 4 +--- mteb/abstasks/Image/AbsTaskImageClustering.py | 2 +- mteb/evaluation/evaluators/Image/ClassificationEvaluator.py | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py index 53e89e0bd9..af01ccf555 100644 --- a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py @@ -209,9 +209,7 @@ def load_data(self, **kwargs): return self.corpus, self.queries, self.relevant_docs = {}, {}, {} dataset_path = self.metadata_dict["dataset"]["path"] - hf_repo_qrels = ( - dataset_path + "-qrels" if "clarin-knext" in dataset_path else None - ) + for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): corpus, queries, qrels = HFDataLoader( hf_repo=dataset_path, diff --git a/mteb/abstasks/Image/AbsTaskImageClustering.py b/mteb/abstasks/Image/AbsTaskImageClustering.py index a2dfbb9d19..020189da63 100644 --- a/mteb/abstasks/Image/AbsTaskImageClustering.py +++ b/mteb/abstasks/Image/AbsTaskImageClustering.py @@ -47,4 +47,4 @@ def _evaluate_subset( ) metrics = evaluator(model, encode_kwargs=encode_kwargs) self._add_main_score(metrics) - return scores + return metrics diff --git a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py index 1252efaeae..753b584992 100644 --- a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py @@ -260,9 +260,9 @@ def __init__( self.encode_kwargs["batch_size"] = 32 if limit is not None: - sentences_train = sentences_train[:limit] + images_train = images_train[:limit] y_train = y_train[:limit] - sentences_test = sentences_test[:limit] + images_test = images_test[:limit] y_test = y_test[:limit] self.images_train = images_train self.y_train = y_train From 68fa26a5c03b1f9a83e2e03daf1fbfcd35de0223 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Tue, 16 Jul 2024 20:46:55 +0100 Subject: [PATCH 013/154] T2I Retrieval --- mteb/abstasks/Image/AbsTaskI2TRetrieval.py | 10 +- mteb/abstasks/Image/AbsTaskT2IRetrieval.py | 451 ++++++++++++++++++ mteb/abstasks/TaskMetadata.py | 3 +- mteb/abstasks/__init__.py | 1 + .../evaluators/Image/I2TRetrievalEvaluator.py | 2 +- .../evaluators/Image/T2IRetrievalEvaluator.py | 353 ++++++++++++++ mteb/evaluation/evaluators/__init__.py | 1 + .../I2TRetrieval/eng/MSCOCOI2TRetrieval.py | 2 +- mteb/tasks/Image/T2IRetrieval/__init__.py | 3 + .../T2IRetrieval/eng/MSCOCOT2IRetrieval.py | 50 ++ mteb/tasks/Image/T2IRetrieval/eng/__init__.py | 0 .../eng/RenderedSST2.py | 2 +- mteb/tasks/Image/__init__.py | 2 + mteb/tasks/__init__.py | 1 + .../MSCOCOI2TRetrieval.json | 158 ++++++ .../MSCOCOT2IRetrieval.json | 158 ++++++ .../MSCOCOI2TRetrieval.json | 158 ++++++ .../MSCOCOT2IRetrieval.json | 158 ++++++ .../MSCOCOI2TRetrieval.json | 158 ++++++ .../MSCOCOT2IRetrieval.json | 158 ++++++ tests/test_all_abstasks.py | 2 + 21 files changed, 1824 insertions(+), 7 deletions(-) create mode 100644 mteb/abstasks/Image/AbsTaskT2IRetrieval.py create mode 100644 mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py create mode 100644 mteb/tasks/Image/T2IRetrieval/__init__.py create mode 100644 mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py create mode 100644 mteb/tasks/Image/T2IRetrieval/eng/__init__.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOT2IRetrieval.json diff --git a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py index af01ccf555..54e9606bb4 100644 --- a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py @@ -10,6 +10,7 @@ import tqdm from datasets import Features, Value, load_dataset +from PIL import Image from ...evaluation.evaluators import I2TRetrievalEvaluator from ...load_results.mteb_results import ScoresDict @@ -75,7 +76,9 @@ def check(fIn: str, ext: str): def load( self, split="test" - ) -> Tuple[Dict[str, dict[str, str]], dict[str, str], dict[str, dict[str, int]]]: + ) -> Tuple[ + Dict[str, dict[str, str]], dict[str, Image.Image], dict[str, dict[str, int]] + ]: if not self.hf_repo: self.qrels_file = os.path.join(self.qrels_folder, split + ".tsv") self.check(fIn=self.corpus_file, ext="jsonl") @@ -437,11 +440,12 @@ def calculate_length(queries, corpus): queries_lens = [] doc_lens = [] for query in queries.values(): - queries_lens.append(len(query)) + # for image append 1. Can perhaps be removed. + queries_lens.append(1.0) for doc in corpus.values(): if isinstance(doc, dict): - doc_lens.append(len(doc.get("title", "")) + len(doc["text"])) + doc_lens.append(len(doc["text"])) else: doc_lens.append(len(doc)) diff --git a/mteb/abstasks/Image/AbsTaskT2IRetrieval.py b/mteb/abstasks/Image/AbsTaskT2IRetrieval.py new file mode 100644 index 0000000000..1bdd550d6d --- /dev/null +++ b/mteb/abstasks/Image/AbsTaskT2IRetrieval.py @@ -0,0 +1,451 @@ +from __future__ import annotations + +import json +import logging +import os +from collections import defaultdict +from pathlib import Path +from time import time +from typing import Any, Dict, Tuple + +import tqdm +from datasets import Features, Value, load_dataset +from PIL import Image + +from ...evaluation.evaluators import T2IRetrievalEvaluator +from ...load_results.mteb_results import ScoresDict +from ..AbsTask import AbsTask + +logger = logging.getLogger(__name__) + + +class HFDataLoader: + def __init__( + self, + hf_repo: str | None = None, + hf_repo_qrels: str | None = None, + data_folder: str | None = None, + prefix: str | None = None, + corpus_file: str = "corpus.jsonl", + query_file: str = "queries.jsonl", + qrels_folder: str = "qrels", + qrels_file: str = "", + streaming: bool = False, + keep_in_memory: bool = False, + ): + self.corpus = {} + self.queries = {} + self.qrels = {} + self.hf_repo = hf_repo + if hf_repo: + # By default fetch qrels from same repo not a second repo with "-qrels" like in original + self.hf_repo_qrels = hf_repo_qrels if hf_repo_qrels else hf_repo + else: + # data folder would contain these files: + # (1) fiqa/corpus.jsonl (format: jsonlines) + # (2) fiqa/queries.jsonl (format: jsonlines) + # (3) fiqa/qrels/test.tsv (format: tsv ("\t")) + if prefix: + query_file = prefix + "-" + query_file + qrels_folder = prefix + "-" + qrels_folder + + self.corpus_file = ( + os.path.join(data_folder, corpus_file) if data_folder else corpus_file + ) + self.query_file = ( + os.path.join(data_folder, query_file) if data_folder else query_file + ) + self.qrels_folder = ( + os.path.join(data_folder, qrels_folder) if data_folder else None + ) + self.qrels_file = qrels_file + self.streaming = streaming + self.keep_in_memory = keep_in_memory + + @staticmethod + def check(fIn: str, ext: str): + if not os.path.exists(fIn): + raise ValueError( + "File {} not present! Please provide accurate file.".format(fIn) + ) + + if not fIn.endswith(ext): + raise ValueError( + "File {} must be present with extension {}".format(fIn, ext) + ) + + def load( + self, split="test" + ) -> Tuple[ + Dict[str, Image.Image], dict[str, dict[str, str]], dict[str, dict[str, int]] + ]: + if not self.hf_repo: + self.qrels_file = os.path.join(self.qrels_folder, split + ".tsv") + self.check(fIn=self.corpus_file, ext="jsonl") + self.check(fIn=self.query_file, ext="jsonl") + self.check(fIn=self.qrels_file, ext="tsv") + + if not len(self.corpus): + logger.info("Loading Corpus...") + self._load_corpus() + logger.info("Loaded %d %s Documents.", len(self.corpus), split.upper()) + logger.info("Doc Example: %s", self.corpus[0]) + + if not len(self.queries): + logger.info("Loading Queries...") + self._load_queries(split) + + self._load_qrels(split) + # filter queries with no qrels + qrels_dict = defaultdict(dict) + + def qrels_dict_init(row): + qrels_dict[row["query-id"]][row["corpus-id"]] = int(row["score"]) + + self.qrels.map(qrels_dict_init) + self.qrels = qrels_dict + self.queries = self.queries.filter(lambda x: x["id"] in self.qrels) + logger.info("Loaded %d %s Queries.", len(self.queries), split.upper()) + logger.info("Query Example: %s", self.queries[0]) + + return self.corpus, self.queries, self.qrels + + def load_corpus(self) -> dict[str, dict[str, str]]: + if not self.hf_repo: + self.check(fIn=self.corpus_file, ext="jsonl") + + if not len(self.corpus): + logger.info("Loading Corpus...") + self._load_corpus() + logger.info("Loaded %d %s Documents.", len(self.corpus)) + logger.info("Doc Example: %s", self.corpus[0]) + + return self.corpus + + def _load_corpus(self): + if self.hf_repo: + corpus_ds = load_dataset( + self.hf_repo, + "corpus", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )["corpus"] + else: + corpus_ds = load_dataset( + "json", + data_files=self.corpus_file, + streaming=self.streaming, + keep_in_memory=self.keep_in_memory, + ) + self.corpus = corpus_ds + + def _load_queries(self, split): + if self.hf_repo: + queries_ds = load_dataset( + self.hf_repo, + "query", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )[split] + else: + queries_ds = load_dataset( + "json", + data_files=self.query_file, + streaming=self.streaming, + keep_in_memory=self.keep_in_memory, + ) + self.queries = queries_ds + + def _load_qrels(self, split): + if self.hf_repo: + qrels_ds = load_dataset( + self.hf_repo_qrels, + "qrels", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )[split] + else: + qrels_ds = load_dataset( + "csv", + data_files=self.qrels_file, + delimiter="\t", + keep_in_memory=self.keep_in_memory, + ) + qrels_ds = qrels_ds.remove_columns("Q0") + features = Features( + { + "query-id": Value("string"), + "corpus-id": Value("string"), + "score": Value("float"), + } + ) + qrels_ds = qrels_ds.cast(features) + self.qrels = qrels_ds + + +class AbsTaskT2IRetrieval(AbsTask): + """Abstract class for retrieval experiments. + + Child-classes must implement the following properties: + + self.corpus: dict[str, dict[str, str]] + Semantically, it should contain dict[split_name, dict[sample_id, dict[str, str]]] + E.g. {"test": {"document_one": {"_id": "d1", "title": "title", "text": "text"}}} + + self.queries: dict[str, dict[str, Union[str, List[str]]]] + Semantically, it should contain dict[split_name, dict[sample_id, str]] or dict[split_name, dict[sample_id, List[str]]] for conversations + E.g. {"test": {"q1": "query"}} + or {"test": {"q1": ["turn1", "turn2", "turn3"]}} + + self.relevant_docs: dict[str, dict[str, dict[str, int]]] + Semantically, it should contain dict[split_name, dict[sample_id, dict[doc_id, score]]] + E.g.: {"test": {"q1": {"document_one": 1}}} + """ + + ignore_identical_ids: bool = False + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = {}, {}, {} + dataset_path = self.metadata_dict["dataset"]["path"] + + for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): + corpus, queries, qrels = HFDataLoader( + hf_repo=dataset_path, + streaming=False, + keep_in_memory=False, + ).load(split=split) + # Conversion from DataSet + queries = {query["id"]: {"text": query["text"]} for query in queries} + corpus = {image["id"]: image["image"] for image in corpus} + self.corpus[split], self.queries[split], self.relevant_docs[split] = ( + corpus, + queries, + qrels, + ) + + self.data_loaded = True + + def evaluate( + self, + model, + split: str = "test", + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ): + retriever = T2IRetrievalEvaluator( + retriever=model, + task_name=self.metadata.name, + encode_kwargs=encode_kwargs, + **kwargs, + ) + + scores = {} + hf_subsets = ( + [l for l in self.hf_subsets] if self.is_multilingual else ["default"] + ) + + for hf_subset in hf_subsets: + logger.info(f"Subset: {hf_subset}") + + if hf_subset == "default": + corpus, queries, relevant_docs = ( + self.corpus[split], + self.queries[split], + self.relevant_docs[split], + ) + else: + corpus, queries, relevant_docs = ( + self.corpus[hf_subset][split], + self.queries[hf_subset][split], + self.relevant_docs[hf_subset][split], + ) + scores[hf_subset] = self._evaluate_subset( + retriever, corpus, queries, relevant_docs, hf_subset, **kwargs + ) + return scores + + def _evaluate_subset( + self, retriever, corpus, queries, relevant_docs, hf_subset: str, **kwargs + ): + start_time = time() + results = retriever(corpus, queries) + end_time = time() + logger.info( + "Time taken to retrieve: {:.2f} seconds".format(end_time - start_time) + ) + + save_predictions = kwargs.get("save_predictions", False) + export_errors = kwargs.get("export_errors", False) + if save_predictions or export_errors: + output_folder = Path(kwargs.get("output_folder", "results")) + if not os.path.isdir(output_folder): + os.makedirs(output_folder) + + if save_predictions: + top_k = kwargs.get("top_k", None) + if top_k is not None: + for qid in list(results.keys()): + doc_ids = set( + sorted( + results[qid], key=lambda x: results[qid][x], reverse=True + )[:top_k] + ) + results[qid] = { + k: v for k, v in results[qid].items() if k in doc_ids + } + qrels_save_path = ( + output_folder / f"{self.metadata.name}_{hf_subset}_predictions.json" + ) + + with open(qrels_save_path, "w") as f: + json.dump(results, f) + + ndcg, _map, recall, precision, naucs = retriever.evaluate( + relevant_docs, + results, + retriever.k_values, + ignore_identical_ids=self.ignore_identical_ids, + ) + mrr, naucs_mrr = retriever.evaluate_custom( + relevant_docs, results, retriever.k_values, "mrr" + ) + scores = { + **{f"ndcg_at_{k.split('@')[1]}": v for (k, v) in ndcg.items()}, + **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()}, + **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()}, + **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()}, + **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, + **{ + k.replace("@", "_at_").replace("_P", "_precision").lower(): v + for k, v in naucs.items() + }, + **{ + k.replace("@", "_at_").replace("_P", "_precision").lower(): v + for k, v in naucs_mrr.items() + }, + } + self._add_main_score(scores) + + if export_errors: + errors = {} + + top_k = kwargs.get("top_k", 1) + if not save_predictions and top_k == 1: + for qid in results.keys(): + doc_scores = results[qid] + sorted_docs = sorted( + doc_scores.items(), key=lambda x: x[1], reverse=True + )[:top_k] + results[qid] = {doc_id: score for doc_id, score in sorted_docs} + for qid, retrieved_docs in results.items(): + expected_docs = relevant_docs[qid] + false_positives = [ + doc for doc in retrieved_docs if doc not in expected_docs + ] + false_negatives = [ + doc for doc in expected_docs if doc not in retrieved_docs + ] + if false_positives or false_negatives: + errors[qid] = { + "false_positives": false_positives, + "false_negatives": false_negatives, + } + + errors_save_path = ( + output_folder / f"{self.metadata.name}_{hf_subset}_errors.json" + ) + with open(errors_save_path, "w") as f: + json.dump(errors, f) + + return scores + + def _add_main_score(self, scores: ScoresDict) -> None: + scores["main_score"] = scores[self.metadata.main_score] + + def calculate_metadata_metrics(self) -> None: + self.load_data() + + all_details = {} + pbar_split = tqdm.tqdm( + self.metadata_dict["eval_splits"], desc="Processing Splits..." + ) + for split in pbar_split: + pbar_split.set_postfix_str(f"Split: {split}") + print(f"Processing metadata for split {split}") + all_details[split] = {} + if self.is_multilingual: + pbar_lang = tqdm.tqdm( + self.relevant_docs.keys(), desc="Processing Languages..." + ) + for lang in pbar_lang: + pbar_lang.set_postfix_str(f"Language: {lang}") + print(f"Processing metadata for language {lang}") + split_details = process_language( + self.relevant_docs[lang][split], + self.queries[lang][split], + self.corpus[lang][split], + lang, + ) + all_details[split][lang] = split_details + else: + split_details = process_language( + self.relevant_docs[split], self.queries[split], self.corpus[split] + ) + all_details[split] = split_details + + return all_details + + +def process_language(relevant_docs, queries, corpus, lang=None): + """We want to get three pieces of information: + - the number of documents (and their char length) in the corpus + - the number of queries (and their char length) + - the average number of relevant documents per query + """ + query_len, doc_len = calculate_length(queries, corpus) + num_documents = len(corpus) + num_queries = len(queries) + + # number of qrels that are not 0 + num_qrels_non_zero = sum( + sum(1 for doc_id in docs if docs[doc_id] != 0) + for docs in relevant_docs.values() + ) + qrels_per_doc = num_qrels_non_zero / num_queries if num_queries else 0 + + language_description = f" for language {lang}" if lang else "" + print(f"Average document character length{language_description} is {doc_len}") + print(f"Average query character length{language_description} is {query_len}") + print(f"Number of documents{language_description} is {num_documents}") + print(f"Number of queries{language_description} is {num_queries}") + print( + f"Average number of relevant documents per query{language_description} is {qrels_per_doc}" + ) + return { + "average_document_length": doc_len, + "average_query_length": query_len, + "num_documents": num_documents, + "num_queries": num_queries, + "average_relevant_docs_per_query": qrels_per_doc, + } + + +def calculate_length(queries, corpus): + queries_lens = [] + doc_lens = [] + for query in queries.values(): + queries_lens.append(len(query)) + + for doc in corpus.values(): + if isinstance(doc, Image.Image): + doc_lens.append(1.0) # for image append 1. Can perhaps be removed. + + doc_len = sum(doc_lens) / len(doc_lens) if doc_lens else 0 + query_len = sum(queries_lens) / len(queries_lens) if queries_lens else 0 + return query_len, doc_len diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index d449e051c5..05911a7a6e 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -38,6 +38,7 @@ "Emotion classification", "Reasoning as Retrieval", "Rendered Texts Understanding", + "Image Text Retrieval", ] TASK_DOMAIN = Literal[ @@ -59,7 +60,6 @@ "Subtitles", "Web", "Programming", - "Movie", ] TEXT_CREATION_METHOD = Literal[ @@ -100,6 +100,7 @@ "s2p", # Sentence-to-paragraph "p2p", # Paragraph-to-paragraph "i2t", + "t2i", ] ANNOTATOR_TYPE = Literal[ diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index 57e5c3f9cb..8d74dfd73a 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -16,5 +16,6 @@ from .Image.AbsTaskI2TRetrieval import * from .Image.AbsTaskImageClassification import * from .Image.AbsTaskImageClustering import * +from .Image.AbsTaskT2IRetrieval import * from .Image.AbsTaskZeroshotClassification import * from .MultilingualTask import * diff --git a/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py index e23c4ca0d8..5ceaa9d7d7 100644 --- a/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py @@ -221,7 +221,7 @@ def __init__( def __call__( self, - corpus: dict[str, dict[str, str]], + corpus: dict[str, Image.Image], queries: dict[str, Union[str, List[str]]], ) -> dict[str, dict[str, float]]: if not self.retriever: diff --git a/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py new file mode 100644 index 0000000000..cf8b458bbc --- /dev/null +++ b/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py @@ -0,0 +1,353 @@ +from __future__ import annotations + +import heapq +import json +import logging +import os +from collections import defaultdict +from typing import Any, Dict, List, Tuple + +import numpy as np +import pytrec_eval +import torch +from PIL import Image + +from mteb.encoder_interface import EncoderWithQueryCorpusEncode + +from ..Evaluator import Evaluator +from ..utils import ( + confidence_scores, + cos_sim, + dot_score, + download, + hole, + mrr, + nAUC, + recall_cap, + top_k_accuracy, +) + +logger = logging.getLogger(__name__) + + +# Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/search/dense/exact_search.py#L12 +class DenseRetrievalExactSearch: + def __init__( + self, + model: EncoderWithQueryCorpusEncode, + encode_kwargs: dict[str, Any] = {}, + corpus_chunk_size: int = 50000, + previous_results: str | None = None, + **kwargs: Any, + ): + # Model is class that provides get_text_embeddings() and get_image_embeddings() + self.model = model + self.encode_kwargs = encode_kwargs + + if "batch_size" not in encode_kwargs: + encode_kwargs["batch_size"] = 128 + + self.score_functions = {"cos_sim": cos_sim, "dot": dot_score} + self.score_function_desc = { + "cos_sim": "Cosine Similarity", + "dot": "Dot Product", + } + self.corpus_chunk_size = corpus_chunk_size + self.previous_results = previous_results + self.batch_size = encode_kwargs.get("batch_size") + self.show_progress_bar = encode_kwargs.get("show_progress_bar") + self.save_corpus_embeddings = kwargs.get("save_corpus_embeddings", False) + self.corpus_embeddings = defaultdict(list) + self.results = {} + + if self.previous_results is not None: + self.previous_results = self.load_results_file() + + def search( + self, + corpus: dict[str, Image.Image], + queries: dict[str, dict[str, str]], + top_k: int, + score_function: str, + return_sorted: bool = False, + **kwargs, + ) -> dict[str, dict[str, float]]: + if score_function not in self.score_functions: + raise ValueError( + f"score function: {score_function} must be either (cos_sim) for cosine similarity or (dot) for dot product" + ) + + logger.info("Encoding Queries.") + query_ids = list(queries.keys()) + self.results = {qid: {} for qid in query_ids} + queries = [queries[qid]["text"] for qid in queries] + query_embeddings = self.model.get_text_embeddings( + queries, batch_size=self.encode_kwargs["batch_size"] + ) + + logger.info("Preparing Corpus...") + corpus_ids = list(corpus.keys()) + corpus_images = [corpus[cid] for cid in corpus_ids] + + logger.info("Encoding Corpus in batches... Warning: This might take a while!") + logger.info( + "Scoring Function: {} ({})".format( + self.score_function_desc[score_function], score_function + ) + ) + + itr = range(0, len(corpus), self.corpus_chunk_size) + + result_heaps = { + qid: [] for qid in query_ids + } # Keep only the top-k docs for each query + for batch_num, corpus_start_idx in enumerate(itr): + logger.info("Encoding Batch {}/{}...".format(batch_num + 1, len(itr))) + corpus_end_idx = min(corpus_start_idx + self.corpus_chunk_size, len(corpus)) + + # Encode chunk of corpus + if ( + self.save_corpus_embeddings + and "qid" in kwargs + and len(self.corpus_embeddings[kwargs["qid"]]) + ): + sub_corpus_embeddings = torch.tensor( + self.corpus_embeddings[kwargs["qid"]][batch_num] + ) + else: + # Encode chunk of corpus + images = corpus_images[corpus_start_idx:corpus_end_idx] + sub_corpus_embeddings = self.model.get_image_embeddings( + # corpus[corpus_start_idx:corpus_end_idx], + images, + batch_size=self.encode_kwargs["batch_size"], + ) + if self.save_corpus_embeddings and "qid" in kwargs: + self.corpus_embeddings[kwargs["qid"]].append(sub_corpus_embeddings) + + # Compute similarites using either cosine-similarity or dot product + cos_scores = self.score_functions[score_function]( + query_embeddings, sub_corpus_embeddings + ) + cos_scores[torch.isnan(cos_scores)] = -1 + + # Get top-k values + cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( + cos_scores, + min( + top_k + 1, + len(cos_scores[1]) if len(cos_scores) > 1 else len(cos_scores[-1]), + ), + dim=1, + largest=True, + sorted=return_sorted, + ) + cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() + cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() + + for query_itr in range(len(query_embeddings)): + query_id = query_ids[query_itr] + for sub_corpus_id, score in zip( + cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] + ): + corpus_id = corpus_ids[corpus_start_idx + sub_corpus_id] + if len(result_heaps[query_id]) < top_k: + # Push item on the heap + heapq.heappush(result_heaps[query_id], (score, corpus_id)) + else: + # If item is larger than the smallest in the heap, push it on the heap then pop the smallest element + heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) + + for qid in result_heaps: + for score, corpus_id in result_heaps[qid]: + self.results[qid][corpus_id] = score + + return self.results + + def load_results_file(self): + # load the first stage results from file in format {qid: {doc_id: score}} + if "https://" in self.previous_results: + # download the file + if not os.path.exists(self.previous_results): + url_descriptor = self.previous_results.split("https://")[-1].replace( + "/", "--" + ) + dest_file = os.path.join( + "results", f"cached_predictions--{url_descriptor}" + ) + os.makedirs(os.path.dirname(os.path.abspath(dest_file)), exist_ok=True) + download(self.previous_results, dest_file) + logger.info( + f"Downloaded the previous results at {self.previous_results} to {dest_file}" + ) + self.previous_results = dest_file + + with open(self.previous_results, "r") as f: + previous_results = json.load(f) + assert isinstance(previous_results, dict) + assert isinstance(previous_results[list(previous_results.keys())[0]], dict) + return previous_results + + +# Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/evaluation.py#L9 +class T2IRetrievalEvaluator(Evaluator): + def __init__( + self, + retriever=None, + task_name: str | None = None, + k_values: List[int] = [1, 3, 5, 10, 20, 100, 1000], + score_function: str = "cos_sim", + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ): + super().__init__(**kwargs) + + self.retriever = DenseRetrievalExactSearch( + retriever, encode_kwargs=encode_kwargs, **kwargs + ) + self.k_values = k_values + self.top_k = ( + max(k_values) if "top_k" not in kwargs else kwargs["top_k"] + ) # can lower it if reranking + self.score_function = score_function + self.task_name = task_name + + def __call__( + self, + corpus: dict[str, dict[str, str]], + queries: dict[str, Image.Image], + ) -> dict[str, dict[str, float]]: + if not self.retriever: + raise ValueError("Model/Technique has not been provided!") + + return self.retriever.search( + corpus, + queries, + self.top_k, + self.score_function, + prompt_name=self.task_name, # type: ignore + ) + + @staticmethod + def evaluate( + qrels: dict[str, dict[str, int]], + results: dict[str, dict[str, float]], + k_values: List[int], + ignore_identical_ids: bool = False, + ) -> Tuple[ + dict[str, float], + dict[str, float], + dict[str, float], + dict[str, float], + dict[str, float], + ]: + if ignore_identical_ids: + logger.debug( + "For evaluation, ``ignore_identical_ids=True`` is set to True, the evaluator will ignore identical query and document ids." + ) + # Remove identical ids from results dict + for qid, rels in results.items(): + for pid in list(rels): + if qid == pid: + results[qid].pop(pid) + else: + logger.debug( + "For evaluation, we DO NOT ignore identical query and document ids (default), please explicitly set ``ignore_identical_ids=True`` to ignore this." + ) + + all_ndcgs, all_aps, all_recalls, all_precisions = {}, {}, {}, {} + + for k in k_values: + all_ndcgs[f"NDCG@{k}"] = [] + all_aps[f"MAP@{k}"] = [] + all_recalls[f"Recall@{k}"] = [] + all_precisions[f"P@{k}"] = [] + + map_string = "map_cut." + ",".join([str(k) for k in k_values]) + ndcg_string = "ndcg_cut." + ",".join([str(k) for k in k_values]) + recall_string = "recall." + ",".join([str(k) for k in k_values]) + precision_string = "P." + ",".join([str(k) for k in k_values]) + evaluator = pytrec_eval.RelevanceEvaluator( + qrels, {map_string, ndcg_string, recall_string, precision_string} + ) + scores = evaluator.evaluate(results) + + for query_id in scores.keys(): + for k in k_values: + all_ndcgs[f"NDCG@{k}"].append(scores[query_id]["ndcg_cut_" + str(k)]) + all_aps[f"MAP@{k}"].append(scores[query_id]["map_cut_" + str(k)]) + all_recalls[f"Recall@{k}"].append(scores[query_id]["recall_" + str(k)]) + all_precisions[f"P@{k}"].append(scores[query_id]["P_" + str(k)]) + + ndcg, _map, recall, precision = ( + all_ndcgs.copy(), + all_aps.copy(), + all_recalls.copy(), + all_precisions.copy(), + ) + + for k in k_values: + ndcg[f"NDCG@{k}"] = round(sum(ndcg[f"NDCG@{k}"]) / len(scores), 5) + _map[f"MAP@{k}"] = round(sum(_map[f"MAP@{k}"]) / len(scores), 5) + recall[f"Recall@{k}"] = round(sum(recall[f"Recall@{k}"]) / len(scores), 5) + precision[f"P@{k}"] = round(sum(precision[f"P@{k}"]) / len(scores), 5) + + naucs = T2IRetrievalEvaluator.evaluate_abstention( + results, {**all_ndcgs, **all_aps, **all_recalls, **all_precisions} + ) + + return ndcg, _map, recall, precision, naucs + + @staticmethod + def evaluate_custom( + qrels: dict[str, dict[str, int]], + results: dict[str, dict[str, float]], + k_values: List[int], + metric: str, + output_type: str = "all", + ) -> Tuple[Dict[str, float]]: + if metric.lower() in ["mrr", "mrr@k", "mrr_cut"]: + metric_scores = mrr(qrels, results, k_values, output_type) + + elif metric.lower() in ["recall_cap", "r_cap", "r_cap@k"]: + metric_scores = recall_cap(qrels, results, k_values, output_type) + + elif metric.lower() in ["hole", "hole@k"]: + metric_scores = hole(qrels, results, k_values, output_type) + + elif metric.lower() in [ + "acc", + "top_k_acc", + "accuracy", + "accuracy@k", + "top_k_accuracy", + ]: + metric_scores = top_k_accuracy(qrels, results, k_values, output_type) + + naucs = T2IRetrievalEvaluator.evaluate_abstention(results, metric_scores) + metric_scores_avg = {k: sum(v) / len(v) for k, v in metric_scores.items()} + + return metric_scores_avg, naucs + + @staticmethod + def evaluate_abstention( + results: dict[str, dict[str, float]], + metric_scores: dict[str, list[float]], + ) -> Dict[str, float]: + """Computes normalized Area Under the Curve on a set of evaluated instances as presented in the paper https://arxiv.org/abs/2402.12997""" + all_sim_scores = [list(results[qid].values()) for qid in list(results.keys())] + all_conf_scores = [ + confidence_scores(sim_scores) for sim_scores in all_sim_scores + ] + conf_fcts = list(all_conf_scores[0].keys()) + all_conf_scores = { + fct: np.array([x[fct] for x in all_conf_scores]) for fct in conf_fcts + } + metric_scores = {k: np.array(v) for k, v in metric_scores.items()} + naucs = {} + + for metric_name, scores in metric_scores.items(): + for fct, conf_scores in all_conf_scores.items(): + naucs[f"nAUC_{metric_name}_{fct}"] = nAUC(conf_scores, scores) + + return naucs diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py index 2f6ca61148..06f9862da1 100644 --- a/mteb/evaluation/evaluators/__init__.py +++ b/mteb/evaluation/evaluators/__init__.py @@ -6,6 +6,7 @@ from .Image.ClassificationEvaluator import * from .Image.ClusteringEvaluator import * from .Image.I2TRetrievalEvaluator import * +from .Image.T2IRetrievalEvaluator import * from .Image.ZeroshotClassificationEvaluator import * from .PairClassificationEvaluator import * from .RerankingEvaluator import * diff --git a/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py b/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py index 1f3c0d4ba1..782e471c62 100644 --- a/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py +++ b/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py @@ -23,7 +23,7 @@ class MSCOCOI2TRetrieval(AbsTaskI2TRetrieval): date=("2018-01-01", "2018-12-31"), form=["written"], domains=["Encyclopaedic"], - task_subtypes=["Reasoning as Retrieval"], + task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", socioeconomic_status="medium", annotations_creators="derived", diff --git a/mteb/tasks/Image/T2IRetrieval/__init__.py b/mteb/tasks/Image/T2IRetrieval/__init__.py new file mode 100644 index 0000000000..5fa74bdd4e --- /dev/null +++ b/mteb/tasks/Image/T2IRetrieval/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .eng.MSCOCOT2IRetrieval import * diff --git a/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py b/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py new file mode 100644 index 0000000000..fd75d30f72 --- /dev/null +++ b/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskT2IRetrieval + + +class MSCOCOT2IRetrieval(AbsTaskT2IRetrieval): + metadata = TaskMetadata( + name="MSCOCOT2IRetrieval", + description="Retrieve captions based on images.", + reference="https://link.springer.com/chapter/10.1007/978-3-319-10602-1_48", + dataset={ + "path": "MRBench/mbeir_mscoco_task0", + "revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", + "trust_remote_code": True, + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2018-01-01", "2018-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + text_creation="found", + bibtex_citation="""@inproceedings{lin2014microsoft, + title={Microsoft coco: Common objects in context}, + author={Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\'a}r, Piotr and Zitnick, C Lawrence}, + booktitle={Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part V 13}, + pages={740--755}, + year={2014}, + organization={Springer} +}""", + n_samples={"test": 1172}, + avg_character_length={ + "test": { + "average_document_length": 30.94235294117647, + "average_query_length": 131.56569965870307, + "num_documents": 9350, + "num_queries": 1172, + "average_relevant_docs_per_query": 1.0, + } + }, + ) diff --git a/mteb/tasks/Image/T2IRetrieval/eng/__init__.py b/mteb/tasks/Image/T2IRetrieval/eng/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py index f2a741cf54..c4540bf542 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py @@ -21,7 +21,7 @@ class RenderedSST2(AbsTaskZeroshotClassification): main_score="accuracy", date=("2016-01-01", "2016-12-31"), form=["written"], - domains=["Movie"], + domains=["Reviews"], task_subtypes=[], license="mit", socioeconomic_status="mixed", diff --git a/mteb/tasks/Image/__init__.py b/mteb/tasks/Image/__init__.py index c5c14e726a..08461edbb1 100644 --- a/mteb/tasks/Image/__init__.py +++ b/mteb/tasks/Image/__init__.py @@ -1,3 +1,5 @@ from .Clustering import * +from .I2TRetrieval import * from .ImageClassification import * +from .T2IRetrieval import * from .ZeroshotClassification import * diff --git a/mteb/tasks/__init__.py b/mteb/tasks/__init__.py index 9c34b18bbc..2726181eb2 100644 --- a/mteb/tasks/__init__.py +++ b/mteb/tasks/__init__.py @@ -5,6 +5,7 @@ from .Clustering import * from .Image.I2TRetrieval import * from .Image.ImageClassification import * +from .Image.T2IRetrieval import * from .Image.ZeroshotClassification import * from .InstructionRetrieval import * from .MultiLabelClassification import * diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOI2TRetrieval.json new file mode 100644 index 0000000000..d2ffdd60cd --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOI2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", + "evaluation_time": 35.03174376487732, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.46627, + "map_at_1": 0.10555, + "map_at_10": 0.35174, + "map_at_100": 0.39557, + "map_at_1000": 0.39923, + "map_at_20": 0.37565, + "map_at_3": 0.2358, + "map_at_5": 0.30315, + "mrr_at_1": 0.5274, + "mrr_at_10": 0.6307395238095252, + "mrr_at_100": 0.6373119196326027, + "mrr_at_1000": 0.6373802558284773, + "mrr_at_20": 0.6352388099361947, + "mrr_at_3": 0.6044666666666684, + "mrr_at_5": 0.6191566666666672, + "nauc_map_at_1000_diff1": 0.2869664571536092, + "nauc_map_at_1000_max": 0.3837078202810147, + "nauc_map_at_1000_std": -0.3159503626026489, + "nauc_map_at_100_diff1": 0.28703582233761415, + "nauc_map_at_100_max": 0.38340512757594436, + "nauc_map_at_100_std": -0.31691519501613774, + "nauc_map_at_10_diff1": 0.28144600872574277, + "nauc_map_at_10_max": 0.3815317371289302, + "nauc_map_at_10_std": -0.3130948877233665, + "nauc_map_at_1_diff1": 0.42136443963654857, + "nauc_map_at_1_max": 0.34593517564893767, + "nauc_map_at_1_std": -0.3210828465931971, + "nauc_map_at_20_diff1": 0.2839400296739511, + "nauc_map_at_20_max": 0.38270639817217694, + "nauc_map_at_20_std": -0.31693403230799216, + "nauc_map_at_3_diff1": 0.29969868166405594, + "nauc_map_at_3_max": 0.36804745199570993, + "nauc_map_at_3_std": -0.3005060420974389, + "nauc_map_at_5_diff1": 0.28066502811464294, + "nauc_map_at_5_max": 0.38500675433867, + "nauc_map_at_5_std": -0.30277657476472214, + "nauc_mrr_at_1000_diff1": 0.41411392018539944, + "nauc_mrr_at_1000_max": 0.34589612969127426, + "nauc_mrr_at_1000_std": -0.3570352162287009, + "nauc_mrr_at_100_diff1": 0.41413003302489404, + "nauc_mrr_at_100_max": 0.34588621354537913, + "nauc_mrr_at_100_std": -0.3571316999051599, + "nauc_mrr_at_10_diff1": 0.4136538823599215, + "nauc_mrr_at_10_max": 0.34527376482746647, + "nauc_mrr_at_10_std": -0.3592210063673948, + "nauc_mrr_at_1_diff1": 0.42218213910596014, + "nauc_mrr_at_1_max": 0.3463699419397979, + "nauc_mrr_at_1_std": -0.32142245164460015, + "nauc_mrr_at_20_diff1": 0.4140225618106111, + "nauc_mrr_at_20_max": 0.34565328848624477, + "nauc_mrr_at_20_std": -0.357990582740709, + "nauc_mrr_at_3_diff1": 0.41140479122082163, + "nauc_mrr_at_3_max": 0.3468454677490499, + "nauc_mrr_at_3_std": -0.356442538497874, + "nauc_mrr_at_5_diff1": 0.41259544348162186, + "nauc_mrr_at_5_max": 0.34704678498973585, + "nauc_mrr_at_5_std": -0.356703631914488, + "nauc_ndcg_at_1000_diff1": 0.3247556576373652, + "nauc_ndcg_at_1000_max": 0.376393317989791, + "nauc_ndcg_at_1000_std": -0.3324974788562371, + "nauc_ndcg_at_100_diff1": 0.3217721326814981, + "nauc_ndcg_at_100_max": 0.37324015047062437, + "nauc_ndcg_at_100_std": -0.3387959081771918, + "nauc_ndcg_at_10_diff1": 0.3087600262995678, + "nauc_ndcg_at_10_max": 0.36991072734632663, + "nauc_ndcg_at_10_std": -0.3387282627094938, + "nauc_ndcg_at_1_diff1": 0.42218213910596014, + "nauc_ndcg_at_1_max": 0.3463699419397979, + "nauc_ndcg_at_1_std": -0.32142245164460015, + "nauc_ndcg_at_20_diff1": 0.31310134298112313, + "nauc_ndcg_at_20_max": 0.3711638446713194, + "nauc_ndcg_at_20_std": -0.3446387196717444, + "nauc_ndcg_at_3_diff1": 0.32234088999598043, + "nauc_ndcg_at_3_max": 0.3637234182783304, + "nauc_ndcg_at_3_std": -0.3199344660388001, + "nauc_ndcg_at_5_diff1": 0.3087695940641351, + "nauc_ndcg_at_5_max": 0.3752423132579829, + "nauc_ndcg_at_5_std": -0.3254191806337329, + "nauc_precision_at_1000_diff1": 0.0318067226890696, + "nauc_precision_at_1000_max": 0.3495751633986902, + "nauc_precision_at_1000_std": 0.5565662931839553, + "nauc_precision_at_100_diff1": 0.2388655041840271, + "nauc_precision_at_100_max": 0.3006890192309006, + "nauc_precision_at_100_std": -0.26789110526496845, + "nauc_precision_at_10_diff1": 0.25885428174045677, + "nauc_precision_at_10_max": 0.3465562874468837, + "nauc_precision_at_10_std": -0.32815618004174235, + "nauc_precision_at_1_diff1": 0.42218213910596014, + "nauc_precision_at_1_max": 0.3463699419397979, + "nauc_precision_at_1_std": -0.32142245164460015, + "nauc_precision_at_20_diff1": 0.25349909583287034, + "nauc_precision_at_20_max": 0.33188675929576117, + "nauc_precision_at_20_std": -0.3268395370923763, + "nauc_precision_at_3_diff1": 0.2892533844448195, + "nauc_precision_at_3_max": 0.36389248373378874, + "nauc_precision_at_3_std": -0.3153927469706968, + "nauc_precision_at_5_diff1": 0.2692072552423276, + "nauc_precision_at_5_max": 0.37322473714076376, + "nauc_precision_at_5_std": -0.3176984049769347, + "nauc_recall_at_1000_diff1": 0.023861396410433965, + "nauc_recall_at_1000_max": 0.3630736245115491, + "nauc_recall_at_1000_std": 0.5760007706786342, + "nauc_recall_at_100_diff1": 0.23875325449572146, + "nauc_recall_at_100_max": 0.30099776552287444, + "nauc_recall_at_100_std": -0.2687324892664122, + "nauc_recall_at_10_diff1": 0.25859827210476144, + "nauc_recall_at_10_max": 0.3464336452165927, + "nauc_recall_at_10_std": -0.32824547353711414, + "nauc_recall_at_1_diff1": 0.42136443963654857, + "nauc_recall_at_1_max": 0.34593517564893767, + "nauc_recall_at_1_std": -0.3210828465931971, + "nauc_recall_at_20_diff1": 0.2532204432312222, + "nauc_recall_at_20_max": 0.3318583274471809, + "nauc_recall_at_20_std": -0.3269518502721138, + "nauc_recall_at_3_diff1": 0.28903591577358717, + "nauc_recall_at_3_max": 0.3637400527142261, + "nauc_recall_at_3_std": -0.31529036662690074, + "nauc_recall_at_5_diff1": 0.2691830355517802, + "nauc_recall_at_5_max": 0.3732658127670549, + "nauc_recall_at_5_std": -0.31779144735485815, + "ndcg_at_1": 0.5274, + "ndcg_at_10": 0.46627, + "ndcg_at_100": 0.58544, + "ndcg_at_1000": 0.62262, + "ndcg_at_20": 0.51262, + "ndcg_at_3": 0.45724, + "ndcg_at_5": 0.4002, + "precision_at_1": 0.5274, + "precision_at_10": 0.24096, + "precision_at_100": 0.04117, + "precision_at_1000": 0.00496, + "precision_at_20": 0.14745, + "precision_at_3": 0.43627, + "precision_at_5": 0.36168, + "recall_at_1": 0.10555, + "recall_at_10": 0.48216, + "recall_at_100": 0.82383, + "recall_at_1000": 0.99244, + "recall_at_20": 0.59011, + "recall_at_3": 0.2619, + "recall_at_5": 0.36183 + } + ] + }, + "task_name": "MSCOCOI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOT2IRetrieval.json new file mode 100644 index 0000000000..caa9409611 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOT2IRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", + "evaluation_time": 59.44574952125549, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.51224, + "map_at_1": 0.34778, + "map_at_10": 0.45378, + "map_at_100": 0.46526, + "map_at_1000": 0.46547, + "map_at_20": 0.4607, + "map_at_3": 0.42214, + "map_at_5": 0.43988, + "mrr_at_1": 0.3480591720746503, + "mrr_at_10": 0.4542383812326254, + "mrr_at_100": 0.46563385013928266, + "mrr_at_1000": 0.4658408234293717, + "mrr_at_20": 0.46113137131638215, + "mrr_at_3": 0.4225818587340632, + "mrr_at_5": 0.44035565050318304, + "nauc_map_at_1000_diff1": 0.5300274380820644, + "nauc_map_at_1000_max": 0.2559436002804574, + "nauc_map_at_1000_std": -0.2706192946414834, + "nauc_map_at_100_diff1": 0.5299324120035608, + "nauc_map_at_100_max": 0.2560347704220875, + "nauc_map_at_100_std": -0.27045953970788833, + "nauc_map_at_10_diff1": 0.5278510109426034, + "nauc_map_at_10_max": 0.2539268152655037, + "nauc_map_at_10_std": -0.27321216910868745, + "nauc_map_at_1_diff1": 0.5783541250432945, + "nauc_map_at_1_max": 0.2695828538669502, + "nauc_map_at_1_std": -0.2559433871779334, + "nauc_map_at_20_diff1": 0.5286608424283533, + "nauc_map_at_20_max": 0.25510619043899635, + "nauc_map_at_20_std": -0.27209215189802183, + "nauc_map_at_3_diff1": 0.5324437628964646, + "nauc_map_at_3_max": 0.2553342754483297, + "nauc_map_at_3_std": -0.27130950074984744, + "nauc_map_at_5_diff1": 0.5284529389089929, + "nauc_map_at_5_max": 0.25477174918600354, + "nauc_map_at_5_std": -0.27190725659741016, + "nauc_mrr_at_1000_diff1": 0.5293645653185072, + "nauc_mrr_at_1000_max": 0.2557502693986946, + "nauc_mrr_at_1000_std": -0.26978550875221907, + "nauc_mrr_at_100_diff1": 0.5292685848719979, + "nauc_mrr_at_100_max": 0.25584012455335126, + "nauc_mrr_at_100_std": -0.26961557231523003, + "nauc_mrr_at_10_diff1": 0.5271255164882128, + "nauc_mrr_at_10_max": 0.25379553348651324, + "nauc_mrr_at_10_std": -0.2722279666262872, + "nauc_mrr_at_1_diff1": 0.5776274975462988, + "nauc_mrr_at_1_max": 0.2692774729298938, + "nauc_mrr_at_1_std": -0.25521001402599525, + "nauc_mrr_at_20_diff1": 0.5279751430379428, + "nauc_mrr_at_20_max": 0.2549509074476351, + "nauc_mrr_at_20_std": -0.27112013741544844, + "nauc_mrr_at_3_diff1": 0.5316046890057915, + "nauc_mrr_at_3_max": 0.255056504420029, + "nauc_mrr_at_3_std": -0.27035519360749666, + "nauc_mrr_at_5_diff1": 0.5276492399505774, + "nauc_mrr_at_5_max": 0.25460202269239396, + "nauc_mrr_at_5_std": -0.2708629718337636, + "nauc_ndcg_at_1000_diff1": 0.5212393417971389, + "nauc_ndcg_at_1000_max": 0.2556420055322196, + "nauc_ndcg_at_1000_std": -0.2691695716908475, + "nauc_ndcg_at_100_diff1": 0.5190594914114796, + "nauc_ndcg_at_100_max": 0.2587996367020589, + "nauc_ndcg_at_100_std": -0.26338010950995205, + "nauc_ndcg_at_10_diff1": 0.5074477844063786, + "nauc_ndcg_at_10_max": 0.2469669155787985, + "nauc_ndcg_at_10_std": -0.2809472768896655, + "nauc_ndcg_at_1_diff1": 0.5776274975462988, + "nauc_ndcg_at_1_max": 0.2692774729298938, + "nauc_ndcg_at_1_std": -0.25521001402599525, + "nauc_ndcg_at_20_diff1": 0.5095606734495621, + "nauc_ndcg_at_20_max": 0.25133476467547433, + "nauc_ndcg_at_20_std": -0.27655628582798436, + "nauc_ndcg_at_3_diff1": 0.517781963819374, + "nauc_ndcg_at_3_max": 0.2506704219638103, + "nauc_ndcg_at_3_std": -0.2758702300013848, + "nauc_ndcg_at_5_diff1": 0.5099993994676364, + "nauc_ndcg_at_5_max": 0.2496258742411529, + "nauc_ndcg_at_5_std": -0.27713728292552275, + "nauc_precision_at_1000_diff1": -0.1842058861760832, + "nauc_precision_at_1000_max": 0.09117367390652957, + "nauc_precision_at_1000_std": 0.5620222894753265, + "nauc_precision_at_100_diff1": 0.23210239318675868, + "nauc_precision_at_100_max": 0.37569992320172535, + "nauc_precision_at_100_std": 0.2577714899683604, + "nauc_precision_at_10_diff1": 0.4239940327698815, + "nauc_precision_at_10_max": 0.21854025050104828, + "nauc_precision_at_10_std": -0.3069601314710065, + "nauc_precision_at_1_diff1": 0.5776274975462988, + "nauc_precision_at_1_max": 0.2692774729298938, + "nauc_precision_at_1_std": -0.25521001402599525, + "nauc_precision_at_20_diff1": 0.40073513220011464, + "nauc_precision_at_20_max": 0.2340653951866781, + "nauc_precision_at_20_std": -0.28105232486858844, + "nauc_precision_at_3_diff1": 0.4743236776776763, + "nauc_precision_at_3_max": 0.2369305838402195, + "nauc_precision_at_3_std": -0.2890312678079432, + "nauc_precision_at_5_diff1": 0.44949845794106563, + "nauc_precision_at_5_max": 0.2332288298110243, + "nauc_precision_at_5_std": -0.29233274328970654, + "nauc_recall_at_1000_diff1": 0.5481755072485809, + "nauc_recall_at_1000_max": 0.8579274209326658, + "nauc_recall_at_1000_std": 0.8989772097175315, + "nauc_recall_at_100_diff1": 0.39432018640739835, + "nauc_recall_at_100_max": 0.4852196943081552, + "nauc_recall_at_100_std": 0.16026301353344588, + "nauc_recall_at_10_diff1": 0.4289710956502324, + "nauc_recall_at_10_max": 0.21894642121454272, + "nauc_recall_at_10_std": -0.31340659744516125, + "nauc_recall_at_1_diff1": 0.5783541250432945, + "nauc_recall_at_1_max": 0.2695828538669502, + "nauc_recall_at_1_std": -0.2559433871779334, + "nauc_recall_at_20_diff1": 0.41279018466409206, + "nauc_recall_at_20_max": 0.23609265389037357, + "nauc_recall_at_20_std": -0.2972785442737482, + "nauc_recall_at_3_diff1": 0.4754634956167504, + "nauc_recall_at_3_max": 0.23707374848594062, + "nauc_recall_at_3_std": -0.2902811277316357, + "nauc_recall_at_5_diff1": 0.4515520952519273, + "nauc_recall_at_5_max": 0.23340217997407925, + "nauc_recall_at_5_std": -0.2947192867661317, + "ndcg_at_1": 0.34806, + "ndcg_at_10": 0.51224, + "ndcg_at_100": 0.56914, + "ndcg_at_1000": 0.57354, + "ndcg_at_20": 0.53735, + "ndcg_at_3": 0.44663, + "ndcg_at_5": 0.47858, + "precision_at_1": 0.34806, + "precision_at_10": 0.07004, + "precision_at_100": 0.00972, + "precision_at_1000": 0.00101, + "precision_at_20": 0.04002, + "precision_at_3": 0.17259, + "precision_at_5": 0.11915, + "recall_at_1": 0.34778, + "recall_at_10": 0.69854, + "recall_at_100": 0.96606, + "recall_at_1000": 0.99836, + "recall_at_20": 0.79752, + "recall_at_3": 0.51703, + "recall_at_5": 0.59457 + } + ] + }, + "task_name": "MSCOCOT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json new file mode 100644 index 0000000000..1476975b8f --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", + "evaluation_time": 24.560455560684204, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.44382, + "map_at_1": 0.10038, + "map_at_10": 0.33055, + "map_at_100": 0.3751, + "map_at_1000": 0.37878, + "map_at_20": 0.35503, + "map_at_3": 0.22144, + "map_at_5": 0.28304, + "mrr_at_1": 0.5016, + "mrr_at_10": 0.6070182539682555, + "mrr_at_100": 0.6142642221262432, + "mrr_at_1000": 0.6143593027903639, + "mrr_at_20": 0.6120301861073234, + "mrr_at_3": 0.5781000000000018, + "mrr_at_5": 0.5949800000000011, + "nauc_map_at_1000_diff1": 0.28351782466117825, + "nauc_map_at_1000_max": 0.3661635018482055, + "nauc_map_at_1000_std": -0.2823328432610589, + "nauc_map_at_100_diff1": 0.2836697696803503, + "nauc_map_at_100_max": 0.3658508272945635, + "nauc_map_at_100_std": -0.2831750575480581, + "nauc_map_at_10_diff1": 0.2751925036810592, + "nauc_map_at_10_max": 0.362845180918207, + "nauc_map_at_10_std": -0.2796511485424725, + "nauc_map_at_1_diff1": 0.42704782267482194, + "nauc_map_at_1_max": 0.3354423674380913, + "nauc_map_at_1_std": -0.295768978541717, + "nauc_map_at_20_diff1": 0.27970330738728366, + "nauc_map_at_20_max": 0.3631059439197211, + "nauc_map_at_20_std": -0.28526313274442944, + "nauc_map_at_3_diff1": 0.30553729889530606, + "nauc_map_at_3_max": 0.3491175423733284, + "nauc_map_at_3_std": -0.2785635560793982, + "nauc_map_at_5_diff1": 0.28121998853700064, + "nauc_map_at_5_max": 0.3642230800059721, + "nauc_map_at_5_std": -0.27715004713761787, + "nauc_mrr_at_1000_diff1": 0.41157652946293, + "nauc_mrr_at_1000_max": 0.33317963379765403, + "nauc_mrr_at_1000_std": -0.3099069801037426, + "nauc_mrr_at_100_diff1": 0.4115850028024918, + "nauc_mrr_at_100_max": 0.33318274824006155, + "nauc_mrr_at_100_std": -0.30988539187877806, + "nauc_mrr_at_10_diff1": 0.41067364563442044, + "nauc_mrr_at_10_max": 0.33216357042070327, + "nauc_mrr_at_10_std": -0.31053927104989015, + "nauc_mrr_at_1_diff1": 0.4269635838909405, + "nauc_mrr_at_1_max": 0.3359596131075358, + "nauc_mrr_at_1_std": -0.29597124038134076, + "nauc_mrr_at_20_diff1": 0.4111282203637099, + "nauc_mrr_at_20_max": 0.3329553429417173, + "nauc_mrr_at_20_std": -0.3108973352234124, + "nauc_mrr_at_3_diff1": 0.4106584396861111, + "nauc_mrr_at_3_max": 0.33525947044408083, + "nauc_mrr_at_3_std": -0.3108151055129786, + "nauc_mrr_at_5_diff1": 0.40928356526564613, + "nauc_mrr_at_5_max": 0.33241615563794474, + "nauc_mrr_at_5_std": -0.31330790888317744, + "nauc_ndcg_at_1000_diff1": 0.32243874847713305, + "nauc_ndcg_at_1000_max": 0.3624742162317057, + "nauc_ndcg_at_1000_std": -0.2869514637432924, + "nauc_ndcg_at_100_diff1": 0.32058430384405767, + "nauc_ndcg_at_100_max": 0.361149409268888, + "nauc_ndcg_at_100_std": -0.28782778350008814, + "nauc_ndcg_at_10_diff1": 0.300464637566366, + "nauc_ndcg_at_10_max": 0.35240653924539134, + "nauc_ndcg_at_10_std": -0.2926815629512613, + "nauc_ndcg_at_1_diff1": 0.4269635838909405, + "nauc_ndcg_at_1_max": 0.3359596131075358, + "nauc_ndcg_at_1_std": -0.29597124038134076, + "nauc_ndcg_at_20_diff1": 0.3069332270693507, + "nauc_ndcg_at_20_max": 0.35335596836456534, + "nauc_ndcg_at_20_std": -0.3019191697870882, + "nauc_ndcg_at_3_diff1": 0.32651722328846816, + "nauc_ndcg_at_3_max": 0.3454350647220891, + "nauc_ndcg_at_3_std": -0.29031057373881475, + "nauc_ndcg_at_5_diff1": 0.3080772824134777, + "nauc_ndcg_at_5_max": 0.35452785631989464, + "nauc_ndcg_at_5_std": -0.2931716669816145, + "nauc_precision_at_1000_diff1": 0.09001395234899717, + "nauc_precision_at_1000_max": 0.43535411629797816, + "nauc_precision_at_1000_std": 0.5700954653090525, + "nauc_precision_at_100_diff1": 0.247818689232099, + "nauc_precision_at_100_max": 0.3133668190372178, + "nauc_precision_at_100_std": -0.17862978392429757, + "nauc_precision_at_10_diff1": 0.24140633693516517, + "nauc_precision_at_10_max": 0.331551710651199, + "nauc_precision_at_10_std": -0.27078528866090024, + "nauc_precision_at_1_diff1": 0.4269635838909405, + "nauc_precision_at_1_max": 0.3359596131075358, + "nauc_precision_at_1_std": -0.29597124038134076, + "nauc_precision_at_20_diff1": 0.2415524145841425, + "nauc_precision_at_20_max": 0.3178040334889773, + "nauc_precision_at_20_std": -0.27782664848315153, + "nauc_precision_at_3_diff1": 0.2911132728042898, + "nauc_precision_at_3_max": 0.3430986077114997, + "nauc_precision_at_3_std": -0.28545786591563965, + "nauc_precision_at_5_diff1": 0.2627202382806198, + "nauc_precision_at_5_max": 0.3493999886119583, + "nauc_precision_at_5_std": -0.28416900168712844, + "nauc_recall_at_1000_diff1": 0.11234864316098189, + "nauc_recall_at_1000_max": 0.4494890548812042, + "nauc_recall_at_1000_std": 0.5881259911372041, + "nauc_recall_at_100_diff1": 0.24940335817964793, + "nauc_recall_at_100_max": 0.3135186357535106, + "nauc_recall_at_100_std": -0.17935112790907157, + "nauc_recall_at_10_diff1": 0.24176763561476253, + "nauc_recall_at_10_max": 0.3313976857716855, + "nauc_recall_at_10_std": -0.27091127395840575, + "nauc_recall_at_1_diff1": 0.42704782267482194, + "nauc_recall_at_1_max": 0.3354423674380913, + "nauc_recall_at_1_std": -0.295768978541717, + "nauc_recall_at_20_diff1": 0.24212418741953098, + "nauc_recall_at_20_max": 0.3177591546517272, + "nauc_recall_at_20_std": -0.277952723894512, + "nauc_recall_at_3_diff1": 0.29145311226379, + "nauc_recall_at_3_max": 0.34287227460196085, + "nauc_recall_at_3_std": -0.2854556141136081, + "nauc_recall_at_5_diff1": 0.26299935501097627, + "nauc_recall_at_5_max": 0.3492528893515414, + "nauc_recall_at_5_std": -0.2842098794048584, + "ndcg_at_1": 0.5016, + "ndcg_at_10": 0.44382, + "ndcg_at_100": 0.56713, + "ndcg_at_1000": 0.60567, + "ndcg_at_20": 0.49235, + "ndcg_at_3": 0.43251, + "ndcg_at_5": 0.37825, + "precision_at_1": 0.5016, + "precision_at_10": 0.23016, + "precision_at_100": 0.04069, + "precision_at_1000": 0.00495, + "precision_at_20": 0.14332, + "precision_at_3": 0.41153, + "precision_at_5": 0.3408, + "recall_at_1": 0.10038, + "recall_at_10": 0.46052, + "recall_at_100": 0.81409, + "recall_at_1000": 0.98992, + "recall_at_20": 0.57355, + "recall_at_3": 0.24704, + "recall_at_5": 0.34096 + } + ] + }, + "task_name": "MSCOCOI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json new file mode 100644 index 0000000000..2925dd6776 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", + "evaluation_time": 48.408275842666626, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.49354, + "map_at_1": 0.32148, + "map_at_10": 0.4321, + "map_at_100": 0.44382, + "map_at_1000": 0.44402, + "map_at_20": 0.43906, + "map_at_3": 0.39864, + "map_at_5": 0.41705, + "mrr_at_1": 0.3218993107340078, + "mrr_at_10": 0.43262941572023605, + "mrr_at_100": 0.44425445210134723, + "mrr_at_1000": 0.44445143330526093, + "mrr_at_20": 0.43955600062406863, + "mrr_at_3": 0.3991898101495421, + "mrr_at_5": 0.41756015961949616, + "nauc_map_at_1000_diff1": 0.49930616061028704, + "nauc_map_at_1000_max": 0.24685433707080706, + "nauc_map_at_1000_std": -0.22328503812884298, + "nauc_map_at_100_diff1": 0.4992272277806359, + "nauc_map_at_100_max": 0.24697921089776936, + "nauc_map_at_100_std": -0.22308864169984505, + "nauc_map_at_10_diff1": 0.49690012441978665, + "nauc_map_at_10_max": 0.24536246677307505, + "nauc_map_at_10_std": -0.22543751253448477, + "nauc_map_at_1_diff1": 0.5515444068560961, + "nauc_map_at_1_max": 0.25822323169218364, + "nauc_map_at_1_std": -0.21358954191507926, + "nauc_map_at_20_diff1": 0.49827147970901364, + "nauc_map_at_20_max": 0.24612833477596713, + "nauc_map_at_20_std": -0.2247932396699695, + "nauc_map_at_3_diff1": 0.5000595542031279, + "nauc_map_at_3_max": 0.24476462058347448, + "nauc_map_at_3_std": -0.2240119884784554, + "nauc_map_at_5_diff1": 0.49580203095577297, + "nauc_map_at_5_max": 0.24334951100406244, + "nauc_map_at_5_std": -0.2262968722182906, + "nauc_mrr_at_1000_diff1": 0.499078624321733, + "nauc_mrr_at_1000_max": 0.24705331263803362, + "nauc_mrr_at_1000_std": -0.22227497999784016, + "nauc_mrr_at_100_diff1": 0.49900036301502043, + "nauc_mrr_at_100_max": 0.2471775251171831, + "nauc_mrr_at_100_std": -0.2220739366647331, + "nauc_mrr_at_10_diff1": 0.4966506098098704, + "nauc_mrr_at_10_max": 0.2456726135596894, + "nauc_mrr_at_10_std": -0.22420370232311077, + "nauc_mrr_at_1_diff1": 0.5511594051722437, + "nauc_mrr_at_1_max": 0.25830642727292813, + "nauc_mrr_at_1_std": -0.21261765560734627, + "nauc_mrr_at_20_diff1": 0.49803403767882, + "nauc_mrr_at_20_max": 0.246377103277424, + "nauc_mrr_at_20_std": -0.22364539619421414, + "nauc_mrr_at_3_diff1": 0.49979377716896667, + "nauc_mrr_at_3_max": 0.24503825278773872, + "nauc_mrr_at_3_std": -0.22279957760963745, + "nauc_mrr_at_5_diff1": 0.4955529709824597, + "nauc_mrr_at_5_max": 0.24364677021673845, + "nauc_mrr_at_5_std": -0.22507060373310997, + "nauc_ndcg_at_1000_diff1": 0.49112188758844744, + "nauc_ndcg_at_1000_max": 0.24757056662063803, + "nauc_ndcg_at_1000_std": -0.22064747313445351, + "nauc_ndcg_at_100_diff1": 0.48918551868023125, + "nauc_ndcg_at_100_max": 0.2512228254623345, + "nauc_ndcg_at_100_std": -0.2143001857039737, + "nauc_ndcg_at_10_diff1": 0.477453906201686, + "nauc_ndcg_at_10_max": 0.24164298975701132, + "nauc_ndcg_at_10_std": -0.22983343834132502, + "nauc_ndcg_at_1_diff1": 0.5511594051722437, + "nauc_ndcg_at_1_max": 0.25830642727292813, + "nauc_ndcg_at_1_std": -0.21261765560734627, + "nauc_ndcg_at_20_diff1": 0.48210010047151386, + "nauc_ndcg_at_20_max": 0.24473128548110354, + "nauc_ndcg_at_20_std": -0.2275562152465016, + "nauc_ndcg_at_3_diff1": 0.48380834514144605, + "nauc_ndcg_at_3_max": 0.23983301673015683, + "nauc_ndcg_at_3_std": -0.22727992582610385, + "nauc_ndcg_at_5_diff1": 0.47554641186612, + "nauc_ndcg_at_5_max": 0.2368921319067536, + "nauc_ndcg_at_5_std": -0.2318280497980852, + "nauc_precision_at_1000_diff1": -0.08156277108785304, + "nauc_precision_at_1000_max": 0.18948123029517247, + "nauc_precision_at_1000_std": 0.5497574779729605, + "nauc_precision_at_100_diff1": 0.2394769875392392, + "nauc_precision_at_100_max": 0.41698415214593487, + "nauc_precision_at_100_std": 0.328556989104763, + "nauc_precision_at_10_diff1": 0.40269735640645404, + "nauc_precision_at_10_max": 0.22952547239545323, + "nauc_precision_at_10_std": -0.24130163140327587, + "nauc_precision_at_1_diff1": 0.5511594051722437, + "nauc_precision_at_1_max": 0.25830642727292813, + "nauc_precision_at_1_std": -0.21261765560734627, + "nauc_precision_at_20_diff1": 0.3999531202089017, + "nauc_precision_at_20_max": 0.24431433894217264, + "nauc_precision_at_20_std": -0.22380219125785444, + "nauc_precision_at_3_diff1": 0.4364725818613955, + "nauc_precision_at_3_max": 0.22523007776906476, + "nauc_precision_at_3_std": -0.23673214595260939, + "nauc_precision_at_5_diff1": 0.41023560618565763, + "nauc_precision_at_5_max": 0.21622732648955392, + "nauc_precision_at_5_std": -0.24767741784863434, + "nauc_recall_at_1000_diff1": 0.6939980469079802, + "nauc_recall_at_1000_max": 0.8686406283004262, + "nauc_recall_at_1000_std": 0.9562379494854356, + "nauc_recall_at_100_diff1": 0.38475736215115286, + "nauc_recall_at_100_max": 0.5134058933554025, + "nauc_recall_at_100_std": 0.2577215977974941, + "nauc_recall_at_10_diff1": 0.40577007213243543, + "nauc_recall_at_10_max": 0.22880905921602546, + "nauc_recall_at_10_std": -0.24719647545185264, + "nauc_recall_at_1_diff1": 0.5515444068560961, + "nauc_recall_at_1_max": 0.25822323169218364, + "nauc_recall_at_1_std": -0.21358954191507926, + "nauc_recall_at_20_diff1": 0.4096257462985691, + "nauc_recall_at_20_max": 0.24395998446094397, + "nauc_recall_at_20_std": -0.23983127974593152, + "nauc_recall_at_3_diff1": 0.4371184473204451, + "nauc_recall_at_3_max": 0.2247648394958751, + "nauc_recall_at_3_std": -0.23841868685719025, + "nauc_recall_at_5_diff1": 0.41182244342055924, + "nauc_recall_at_5_max": 0.21563596886073882, + "nauc_recall_at_5_std": -0.2510594326845239, + "ndcg_at_1": 0.3219, + "ndcg_at_10": 0.49354, + "ndcg_at_100": 0.5522, + "ndcg_at_1000": 0.55645, + "ndcg_at_20": 0.51874, + "ndcg_at_3": 0.4242, + "ndcg_at_5": 0.45735, + "precision_at_1": 0.3219, + "precision_at_10": 0.06912, + "precision_at_100": 0.00973, + "precision_at_1000": 0.00101, + "precision_at_20": 0.03958, + "precision_at_3": 0.16615, + "precision_at_5": 0.11589, + "recall_at_1": 0.32148, + "recall_at_10": 0.68941, + "recall_at_100": 0.96663, + "recall_at_1000": 0.998, + "recall_at_20": 0.78867, + "recall_at_3": 0.49757, + "recall_at_5": 0.57818 + } + ] + }, + "task_name": "MSCOCOT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOI2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOI2TRetrieval.json new file mode 100644 index 0000000000..49b5d4ec5b --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOI2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", + "evaluation_time": 78.54339241981506, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.48795, + "map_at_1": 0.11196, + "map_at_10": 0.37316, + "map_at_100": 0.4174, + "map_at_1000": 0.42082, + "map_at_20": 0.39776, + "map_at_3": 0.25241, + "map_at_5": 0.32523, + "mrr_at_1": 0.5596, + "mrr_at_10": 0.6581655555555556, + "mrr_at_100": 0.6643201403638, + "mrr_at_1000": 0.6643768592817573, + "mrr_at_20": 0.6623790460837746, + "mrr_at_3": 0.633833333333335, + "mrr_at_5": 0.6485333333333335, + "nauc_map_at_1000_diff1": 0.27585543400346596, + "nauc_map_at_1000_max": 0.39045174229638263, + "nauc_map_at_1000_std": -0.31907954307019376, + "nauc_map_at_100_diff1": 0.2756901587956695, + "nauc_map_at_100_max": 0.3901590379869748, + "nauc_map_at_100_std": -0.31982716652037885, + "nauc_map_at_10_diff1": 0.2695596721841738, + "nauc_map_at_10_max": 0.38999760608673667, + "nauc_map_at_10_std": -0.3152421322220176, + "nauc_map_at_1_diff1": 0.4535097760243967, + "nauc_map_at_1_max": 0.3847697525357455, + "nauc_map_at_1_std": -0.3475010070412092, + "nauc_map_at_20_diff1": 0.27324757934344923, + "nauc_map_at_20_max": 0.3887973382937881, + "nauc_map_at_20_std": -0.3206806647695715, + "nauc_map_at_3_diff1": 0.30732032953704846, + "nauc_map_at_3_max": 0.3835772487850095, + "nauc_map_at_3_std": -0.31666427177733625, + "nauc_map_at_5_diff1": 0.27329624635564115, + "nauc_map_at_5_max": 0.3924020258946327, + "nauc_map_at_5_std": -0.30667966704337424, + "nauc_mrr_at_1000_diff1": 0.4409454696413627, + "nauc_mrr_at_1000_max": 0.38085560117885164, + "nauc_mrr_at_1000_std": -0.37551707053781974, + "nauc_mrr_at_100_diff1": 0.4409658468757943, + "nauc_mrr_at_100_max": 0.3808934906228772, + "nauc_mrr_at_100_std": -0.375496240900825, + "nauc_mrr_at_10_diff1": 0.43980000162065963, + "nauc_mrr_at_10_max": 0.3798372891177225, + "nauc_mrr_at_10_std": -0.3781076819730193, + "nauc_mrr_at_1_diff1": 0.453795105129665, + "nauc_mrr_at_1_max": 0.38541783132147206, + "nauc_mrr_at_1_std": -0.34743272717366425, + "nauc_mrr_at_20_diff1": 0.4406709449591336, + "nauc_mrr_at_20_max": 0.38057516342819747, + "nauc_mrr_at_20_std": -0.376140377873905, + "nauc_mrr_at_3_diff1": 0.4382924266992488, + "nauc_mrr_at_3_max": 0.3805844116331195, + "nauc_mrr_at_3_std": -0.3769284278433346, + "nauc_mrr_at_5_diff1": 0.4397654551355224, + "nauc_mrr_at_5_max": 0.3802589096353254, + "nauc_mrr_at_5_std": -0.3774278739567175, + "nauc_ndcg_at_1000_diff1": 0.32227128701745533, + "nauc_ndcg_at_1000_max": 0.38730359922065866, + "nauc_ndcg_at_1000_std": -0.33970752042293817, + "nauc_ndcg_at_100_diff1": 0.31453883156173185, + "nauc_ndcg_at_100_max": 0.3835118980277628, + "nauc_ndcg_at_100_std": -0.342167313367944, + "nauc_ndcg_at_10_diff1": 0.3018499732014452, + "nauc_ndcg_at_10_max": 0.38168035853082877, + "nauc_ndcg_at_10_std": -0.3432797044973395, + "nauc_ndcg_at_1_diff1": 0.453795105129665, + "nauc_ndcg_at_1_max": 0.38541783132147206, + "nauc_ndcg_at_1_std": -0.34743272717366425, + "nauc_ndcg_at_20_diff1": 0.3081597647895536, + "nauc_ndcg_at_20_max": 0.3790139927724731, + "nauc_ndcg_at_20_std": -0.3496269372983925, + "nauc_ndcg_at_3_diff1": 0.33292580807819666, + "nauc_ndcg_at_3_max": 0.38163227948458556, + "nauc_ndcg_at_3_std": -0.33633460176008967, + "nauc_ndcg_at_5_diff1": 0.30594814905303774, + "nauc_ndcg_at_5_max": 0.38448832915244796, + "nauc_ndcg_at_5_std": -0.33317544529631354, + "nauc_precision_at_1000_diff1": 0.11133163483146782, + "nauc_precision_at_1000_max": 0.3092422898807732, + "nauc_precision_at_1000_std": 0.4494367261142218, + "nauc_precision_at_100_diff1": 0.20407789253226402, + "nauc_precision_at_100_max": 0.29949976205541556, + "nauc_precision_at_100_std": -0.25144385212643716, + "nauc_precision_at_10_diff1": 0.24180001061503437, + "nauc_precision_at_10_max": 0.35458118880548456, + "nauc_precision_at_10_std": -0.3254216181230694, + "nauc_precision_at_1_diff1": 0.453795105129665, + "nauc_precision_at_1_max": 0.38541783132147206, + "nauc_precision_at_1_std": -0.34743272717366425, + "nauc_precision_at_20_diff1": 0.23979773773669036, + "nauc_precision_at_20_max": 0.33042944980132744, + "nauc_precision_at_20_std": -0.3243391967972116, + "nauc_precision_at_3_diff1": 0.2972609148045276, + "nauc_precision_at_3_max": 0.3788843739773271, + "nauc_precision_at_3_std": -0.33157111929715655, + "nauc_precision_at_5_diff1": 0.25833278252283215, + "nauc_precision_at_5_max": 0.37596098373156045, + "nauc_precision_at_5_std": -0.3212248673681845, + "nauc_recall_at_1000_diff1": 0.12173947593223243, + "nauc_recall_at_1000_max": 0.31204928780024443, + "nauc_recall_at_1000_std": 0.45505045990026477, + "nauc_recall_at_100_diff1": 0.20458582828997776, + "nauc_recall_at_100_max": 0.29902721698634105, + "nauc_recall_at_100_std": -0.2531368692860601, + "nauc_recall_at_10_diff1": 0.24178112080035385, + "nauc_recall_at_10_max": 0.354323008010846, + "nauc_recall_at_10_std": -0.32570952801324166, + "nauc_recall_at_1_diff1": 0.4535097760243967, + "nauc_recall_at_1_max": 0.3847697525357455, + "nauc_recall_at_1_std": -0.3475010070412092, + "nauc_recall_at_20_diff1": 0.2397438401654046, + "nauc_recall_at_20_max": 0.33015488732418014, + "nauc_recall_at_20_std": -0.32472267420389017, + "nauc_recall_at_3_diff1": 0.297280489755177, + "nauc_recall_at_3_max": 0.3786095196183429, + "nauc_recall_at_3_std": -0.33164997286157694, + "nauc_recall_at_5_diff1": 0.2582794667970118, + "nauc_recall_at_5_max": 0.375682734001994, + "nauc_recall_at_5_std": -0.32143238857573325, + "ndcg_at_1": 0.5596, + "ndcg_at_10": 0.48795, + "ndcg_at_100": 0.60605, + "ndcg_at_1000": 0.64058, + "ndcg_at_20": 0.53528, + "ndcg_at_3": 0.48541, + "ndcg_at_5": 0.4244, + "precision_at_1": 0.5596, + "precision_at_10": 0.2492, + "precision_at_100": 0.04177, + "precision_at_1000": 0.00496, + "precision_at_20": 0.15213, + "precision_at_3": 0.46293, + "precision_at_5": 0.38288, + "recall_at_1": 0.11196, + "recall_at_10": 0.4986, + "recall_at_100": 0.83586, + "recall_at_1000": 0.99248, + "recall_at_20": 0.6088, + "recall_at_3": 0.27789, + "recall_at_5": 0.38304 + } + ] + }, + "task_name": "MSCOCOI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOT2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOT2IRetrieval.json new file mode 100644 index 0000000000..cfb8d64243 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOT2IRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", + "evaluation_time": 102.51849317550659, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.52318, + "map_at_1": 0.35829, + "map_at_10": 0.46471, + "map_at_100": 0.4758, + "map_at_1000": 0.47599, + "map_at_20": 0.47131, + "map_at_3": 0.43292, + "map_at_5": 0.45109, + "mrr_at_1": 0.35853923979201097, + "mrr_at_10": 0.4651022222478256, + "mrr_at_100": 0.47611295706758416, + "mrr_at_1000": 0.47629480544722647, + "mrr_at_20": 0.4716961247971015, + "mrr_at_3": 0.43328362019160016, + "mrr_at_5": 0.45149273785051924, + "nauc_map_at_1000_diff1": 0.5364389960050693, + "nauc_map_at_1000_max": 0.27063297333846825, + "nauc_map_at_1000_std": -0.27269894766508856, + "nauc_map_at_100_diff1": 0.5363642248904831, + "nauc_map_at_100_max": 0.2707039659874514, + "nauc_map_at_100_std": -0.27255201916852684, + "nauc_map_at_10_diff1": 0.5335239245537733, + "nauc_map_at_10_max": 0.26812050699895346, + "nauc_map_at_10_std": -0.2755012154401604, + "nauc_map_at_1_diff1": 0.5905593031641301, + "nauc_map_at_1_max": 0.2895208438254399, + "nauc_map_at_1_std": -0.253486375459467, + "nauc_map_at_20_diff1": 0.5348655481247181, + "nauc_map_at_20_max": 0.2693117996470176, + "nauc_map_at_20_std": -0.274396735120252, + "nauc_map_at_3_diff1": 0.5393950231886427, + "nauc_map_at_3_max": 0.27217487369108845, + "nauc_map_at_3_std": -0.27233322179292724, + "nauc_map_at_5_diff1": 0.5344534524377408, + "nauc_map_at_5_max": 0.2687999532811149, + "nauc_map_at_5_std": -0.27534165682431466, + "nauc_mrr_at_1000_diff1": 0.5360368110159093, + "nauc_mrr_at_1000_max": 0.27076830927500367, + "nauc_mrr_at_1000_std": -0.27230173708749605, + "nauc_mrr_at_100_diff1": 0.5359631708332794, + "nauc_mrr_at_100_max": 0.27084424913172367, + "nauc_mrr_at_100_std": -0.27214703103659804, + "nauc_mrr_at_10_diff1": 0.5330918181125404, + "nauc_mrr_at_10_max": 0.26837418551663855, + "nauc_mrr_at_10_std": -0.27488930900694575, + "nauc_mrr_at_1_diff1": 0.5901385036140094, + "nauc_mrr_at_1_max": 0.28945880832060467, + "nauc_mrr_at_1_std": -0.2534654694762397, + "nauc_mrr_at_20_diff1": 0.5344539765635075, + "nauc_mrr_at_20_max": 0.26958226442134897, + "nauc_mrr_at_20_std": -0.27382433915008214, + "nauc_mrr_at_3_diff1": 0.538923422488798, + "nauc_mrr_at_3_max": 0.2723350955816106, + "nauc_mrr_at_3_std": -0.2718935761838524, + "nauc_mrr_at_5_diff1": 0.533959843918952, + "nauc_mrr_at_5_max": 0.26905895384706296, + "nauc_mrr_at_5_std": -0.2747297894605437, + "nauc_ndcg_at_1000_diff1": 0.5269109631077971, + "nauc_ndcg_at_1000_max": 0.2691664483433101, + "nauc_ndcg_at_1000_std": -0.27198727370439346, + "nauc_ndcg_at_100_diff1": 0.5250688847438519, + "nauc_ndcg_at_100_max": 0.2716629727071101, + "nauc_ndcg_at_100_std": -0.2666531261354938, + "nauc_ndcg_at_10_diff1": 0.5100643637960702, + "nauc_ndcg_at_10_max": 0.25728575887213956, + "nauc_ndcg_at_10_std": -0.2851559836697289, + "nauc_ndcg_at_1_diff1": 0.5901385036140094, + "nauc_ndcg_at_1_max": 0.28945880832060467, + "nauc_ndcg_at_1_std": -0.2534654694762397, + "nauc_ndcg_at_20_diff1": 0.5141405251743207, + "nauc_ndcg_at_20_max": 0.2612554258422001, + "nauc_ndcg_at_20_std": -0.2812613383940013, + "nauc_ndcg_at_3_diff1": 0.5234248839202699, + "nauc_ndcg_at_3_max": 0.26623604986271715, + "nauc_ndcg_at_3_std": -0.2785040097130991, + "nauc_ndcg_at_5_diff1": 0.5137575676531829, + "nauc_ndcg_at_5_max": 0.2598174091166936, + "nauc_ndcg_at_5_std": -0.28407927219401274, + "nauc_precision_at_1000_diff1": -0.138201035126385, + "nauc_precision_at_1000_max": 0.2679227808873947, + "nauc_precision_at_1000_std": 0.5849397960968612, + "nauc_precision_at_100_diff1": 0.23962363566765063, + "nauc_precision_at_100_max": 0.4075927257460724, + "nauc_precision_at_100_std": 0.2742652644802499, + "nauc_precision_at_10_diff1": 0.4117501094295695, + "nauc_precision_at_10_max": 0.2126322069047596, + "nauc_precision_at_10_std": -0.31894957843005256, + "nauc_precision_at_1_diff1": 0.5901385036140094, + "nauc_precision_at_1_max": 0.28945880832060467, + "nauc_precision_at_1_std": -0.2534654694762397, + "nauc_precision_at_20_diff1": 0.3961829922935417, + "nauc_precision_at_20_max": 0.2209834925530759, + "nauc_precision_at_20_std": -0.2985831435827799, + "nauc_precision_at_3_diff1": 0.4756800519614237, + "nauc_precision_at_3_max": 0.24830374451491583, + "nauc_precision_at_3_std": -0.29648946830298284, + "nauc_precision_at_5_diff1": 0.44476767482754076, + "nauc_precision_at_5_max": 0.2298515223070273, + "nauc_precision_at_5_std": -0.31137543613145, + "nauc_recall_at_1000_diff1": 0.4494321491065472, + "nauc_recall_at_1000_max": 0.9332926952790045, + "nauc_recall_at_1000_std": 0.9655409418898824, + "nauc_recall_at_100_diff1": 0.4033274574532679, + "nauc_recall_at_100_max": 0.48101784678152426, + "nauc_recall_at_100_std": 0.17496166411367567, + "nauc_recall_at_10_diff1": 0.41658984626823475, + "nauc_recall_at_10_max": 0.21186835042904034, + "nauc_recall_at_10_std": -0.3250128096915894, + "nauc_recall_at_1_diff1": 0.5905593031641301, + "nauc_recall_at_1_max": 0.2895208438254399, + "nauc_recall_at_1_std": -0.253486375459467, + "nauc_recall_at_20_diff1": 0.4079171828509436, + "nauc_recall_at_20_max": 0.2192969350421769, + "nauc_recall_at_20_std": -0.31531173454663586, + "nauc_recall_at_3_diff1": 0.4765351052198576, + "nauc_recall_at_3_max": 0.2481479402236204, + "nauc_recall_at_3_std": -0.2973941516041928, + "nauc_recall_at_5_diff1": 0.4467425425914727, + "nauc_recall_at_5_max": 0.22955437013140248, + "nauc_recall_at_5_std": -0.3134480016683754, + "ndcg_at_1": 0.35854, + "ndcg_at_10": 0.52318, + "ndcg_at_100": 0.57825, + "ndcg_at_1000": 0.58224, + "ndcg_at_20": 0.54712, + "ndcg_at_3": 0.45743, + "ndcg_at_5": 0.49017, + "precision_at_1": 0.35854, + "precision_at_10": 0.07112, + "precision_at_100": 0.00975, + "precision_at_1000": 0.001, + "precision_at_20": 0.04033, + "precision_at_3": 0.17621, + "precision_at_5": 0.1217, + "recall_at_1": 0.35829, + "recall_at_10": 0.70943, + "recall_at_100": 0.96845, + "recall_at_1000": 0.99797, + "recall_at_20": 0.80373, + "recall_at_3": 0.52801, + "recall_at_5": 0.6074 + } + ] + }, + "task_name": "MSCOCOT2IRetrieval" +} \ No newline at end of file diff --git a/tests/test_all_abstasks.py b/tests/test_all_abstasks.py index 4392fc9105..ee67a12665 100644 --- a/tests/test_all_abstasks.py +++ b/tests/test_all_abstasks.py @@ -14,6 +14,7 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.AbsTaskSpeedTask import AbsTaskSpeedTask from mteb.abstasks.Image.AbsTaskI2TRetrieval import AbsTaskI2TRetrieval +from mteb.abstasks.Image.AbsTaskT2IRetrieval import AbsTaskT2IRetrieval from mteb.abstasks.MultiSubsetLoader import MultiSubsetLoader from mteb.overview import TASKS_REGISTRY @@ -32,6 +33,7 @@ def test_load_data( if ( isinstance(task, AbsTaskRetrieval) or isinstance(task, AbsTaskI2TRetrieval) + or isinstance(task, AbsTaskT2IRetrieval) or isinstance(task, AbsTaskInstructionRetrieval) or isinstance(task, MultiSubsetLoader) or isinstance(task, AbsTaskSpeedTask) From 1f624935aa6f645d20a979f694fdf62438f2c32d Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Wed, 17 Jul 2024 19:04:38 +0100 Subject: [PATCH 014/154] Any2AnyRetrieval --- .../abstasks/Image/AbsTaskAny2AnyRetrieval.py | 453 ++++++++++++++++++ mteb/abstasks/TaskMetadata.py | 5 + mteb/abstasks/__init__.py | 1 + .../Image/Any2AnyRetrievalEvaluator.py | 362 ++++++++++++++ mteb/evaluation/evaluators/__init__.py | 1 + mteb/models/clip_models.py | 35 ++ mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 3 + .../Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py | 49 ++ .../Image/Any2AnyRetrieval/eng/__init__.py | 0 mteb/tasks/Image/Clustering/__init__.py | 2 +- .../Clustering/{ => eng}/TinyImageNet.py | 0 mteb/tasks/Image/Clustering/eng/__init__.py | 0 mteb/tasks/Image/__init__.py | 1 + .../CIRRIT2TRetrieval.json | 158 ++++++ .../CIRRIT2TRetrieval.json | 158 ++++++ .../CIRRIT2TRetrieval.json | 158 ++++++ tests/test_all_abstasks.py | 2 + 17 files changed, 1387 insertions(+), 1 deletion(-) create mode 100644 mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py create mode 100644 mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/__init__.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/__init__.py rename mteb/tasks/Image/Clustering/{ => eng}/TinyImageNet.py (100%) create mode 100644 mteb/tasks/Image/Clustering/eng/__init__.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CIRRIT2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CIRRIT2TRetrieval.json diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py new file mode 100644 index 0000000000..25d08cbc94 --- /dev/null +++ b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py @@ -0,0 +1,453 @@ +from __future__ import annotations + +import json +import logging +import os +from collections import defaultdict +from pathlib import Path +from time import time +from typing import Any, Dict, Tuple + +import tqdm +from datasets import Features, Value, load_dataset +from PIL import Image + +from ...evaluation.evaluators import Any2AnyRetrievalEvaluator +from ...load_results.mteb_results import ScoresDict +from ..AbsTask import AbsTask + +logger = logging.getLogger(__name__) + + +class HFDataLoader: + def __init__( + self, + hf_repo: str | None = None, + hf_repo_qrels: str | None = None, + data_folder: str | None = None, + prefix: str | None = None, + corpus_file: str = "corpus.jsonl", + query_file: str = "queries.jsonl", + qrels_folder: str = "qrels", + qrels_file: str = "", + streaming: bool = False, + keep_in_memory: bool = False, + ): + self.corpus = {} + self.queries = {} + self.qrels = {} + self.hf_repo = hf_repo + if hf_repo: + # By default fetch qrels from same repo not a second repo with "-qrels" like in original + self.hf_repo_qrels = hf_repo_qrels if hf_repo_qrels else hf_repo + else: + # data folder would contain these files: + # (1) fiqa/corpus.jsonl (format: jsonlines) + # (2) fiqa/queries.jsonl (format: jsonlines) + # (3) fiqa/qrels/test.tsv (format: tsv ("\t")) + if prefix: + query_file = prefix + "-" + query_file + qrels_folder = prefix + "-" + qrels_folder + + self.corpus_file = ( + os.path.join(data_folder, corpus_file) if data_folder else corpus_file + ) + self.query_file = ( + os.path.join(data_folder, query_file) if data_folder else query_file + ) + self.qrels_folder = ( + os.path.join(data_folder, qrels_folder) if data_folder else None + ) + self.qrels_file = qrels_file + self.streaming = streaming + self.keep_in_memory = keep_in_memory + + @staticmethod + def check(fIn: str, ext: str): + if not os.path.exists(fIn): + raise ValueError( + "File {} not present! Please provide accurate file.".format(fIn) + ) + + if not fIn.endswith(ext): + raise ValueError( + "File {} must be present with extension {}".format(fIn, ext) + ) + + def load( + self, split="test" + ) -> Tuple[ + Dict[str, Dict[str, str | Image.Image]], + Dict[str, Dict[str, str | Image.Image]], + dict[str, dict[str, int]], + ]: + if not self.hf_repo: + self.qrels_file = os.path.join(self.qrels_folder, split + ".tsv") + self.check(fIn=self.corpus_file, ext="jsonl") + self.check(fIn=self.query_file, ext="jsonl") + self.check(fIn=self.qrels_file, ext="tsv") + + if not len(self.corpus): + logger.info("Loading Corpus...") + self._load_corpus() + logger.info("Loaded %d %s Documents.", len(self.corpus), split.upper()) + logger.info("Doc Example: %s", self.corpus[0]) + + if not len(self.queries): + logger.info("Loading Queries...") + self._load_queries(split) + + self._load_qrels(split) + # filter queries with no qrels + qrels_dict = defaultdict(dict) + + def qrels_dict_init(row): + qrels_dict[row["query-id"]][row["corpus-id"]] = int(row["score"]) + + self.qrels.map(qrels_dict_init) + self.qrels = qrels_dict + self.queries = self.queries.filter(lambda x: x["id"] in self.qrels) + logger.info("Loaded %d %s Queries.", len(self.queries), split.upper()) + logger.info("Query Example: %s", self.queries[0]) + + return self.corpus, self.queries, self.qrels + + def load_corpus(self) -> dict[str, dict[str, str]]: + if not self.hf_repo: + self.check(fIn=self.corpus_file, ext="jsonl") + + if not len(self.corpus): + logger.info("Loading Corpus...") + self._load_corpus() + logger.info("Loaded %d %s Documents.", len(self.corpus)) + logger.info("Doc Example: %s", self.corpus[0]) + + return self.corpus + + def _load_corpus(self): + if self.hf_repo: + corpus_ds = load_dataset( + self.hf_repo, + "corpus", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )["corpus"] + else: + corpus_ds = load_dataset( + "json", + data_files=self.corpus_file, + streaming=self.streaming, + keep_in_memory=self.keep_in_memory, + ) + self.corpus = corpus_ds + + def _load_queries(self, split): + if self.hf_repo: + queries_ds = load_dataset( + self.hf_repo, + "query", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )[split] + else: + queries_ds = load_dataset( + "json", + data_files=self.query_file, + streaming=self.streaming, + keep_in_memory=self.keep_in_memory, + ) + self.queries = queries_ds + + def _load_qrels(self, split): + if self.hf_repo: + qrels_ds = load_dataset( + self.hf_repo_qrels, + "qrels", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )[split] + else: + qrels_ds = load_dataset( + "csv", + data_files=self.qrels_file, + delimiter="\t", + keep_in_memory=self.keep_in_memory, + ) + qrels_ds = qrels_ds.remove_columns("Q0") + features = Features( + { + "query-id": Value("string"), + "corpus-id": Value("string"), + "score": Value("float"), + } + ) + qrels_ds = qrels_ds.cast(features) + self.qrels = qrels_ds + + +class AbsTaskAny2AnyRetrieval(AbsTask): + """Abstract class for retrieval experiments. + + Child-classes must implement the following properties: + + self.corpus: dict[str, dict[str, str]] + Semantically, it should contain dict[split_name, dict[sample_id, dict[str, str]]] + E.g. {"test": {"document_one": {"_id": "d1", "title": "title", "text": "text"}}} + + self.queries: dict[str, dict[str, Union[str, List[str]]]] + Semantically, it should contain dict[split_name, dict[sample_id, str]] or dict[split_name, dict[sample_id, List[str]]] for conversations + E.g. {"test": {"q1": "query"}} + or {"test": {"q1": ["turn1", "turn2", "turn3"]}} + + self.relevant_docs: dict[str, dict[str, dict[str, int]]] + Semantically, it should contain dict[split_name, dict[sample_id, dict[doc_id, score]]] + E.g.: {"test": {"q1": {"document_one": 1}}} + """ + + ignore_identical_ids: bool = False + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = {}, {}, {} + dataset_path = self.metadata_dict["dataset"]["path"] + + for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): + corpus, queries, qrels = HFDataLoader( + hf_repo=dataset_path, + streaming=False, + keep_in_memory=False, + ).load(split=split) + # Conversion from DataSet + queries = {query["id"]: query for query in queries} + corpus = {doc["id"]: doc for doc in corpus} + self.corpus[split], self.queries[split], self.relevant_docs[split] = ( + corpus, + queries, + qrels, + ) + + self.data_loaded = True + + def evaluate( + self, + model, + split: str = "test", + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ): + retriever = Any2AnyRetrievalEvaluator( + retriever=model, + task_name=self.metadata.name, + encode_kwargs=encode_kwargs, + **kwargs, + ) + + scores = {} + hf_subsets = ( + [l for l in self.hf_subsets] if self.is_multilingual else ["default"] + ) + + for hf_subset in hf_subsets: + logger.info(f"Subset: {hf_subset}") + + if hf_subset == "default": + corpus, queries, relevant_docs = ( + self.corpus[split], + self.queries[split], + self.relevant_docs[split], + ) + else: + corpus, queries, relevant_docs = ( + self.corpus[hf_subset][split], + self.queries[hf_subset][split], + self.relevant_docs[hf_subset][split], + ) + scores[hf_subset] = self._evaluate_subset( + retriever, corpus, queries, relevant_docs, hf_subset, **kwargs + ) + return scores + + def _evaluate_subset( + self, retriever, corpus, queries, relevant_docs, hf_subset: str, **kwargs + ): + start_time = time() + results = retriever(corpus, queries) + end_time = time() + logger.info( + "Time taken to retrieve: {:.2f} seconds".format(end_time - start_time) + ) + + save_predictions = kwargs.get("save_predictions", False) + export_errors = kwargs.get("export_errors", False) + if save_predictions or export_errors: + output_folder = Path(kwargs.get("output_folder", "results")) + if not os.path.isdir(output_folder): + os.makedirs(output_folder) + + if save_predictions: + top_k = kwargs.get("top_k", None) + if top_k is not None: + for qid in list(results.keys()): + doc_ids = set( + sorted( + results[qid], key=lambda x: results[qid][x], reverse=True + )[:top_k] + ) + results[qid] = { + k: v for k, v in results[qid].items() if k in doc_ids + } + qrels_save_path = ( + output_folder / f"{self.metadata.name}_{hf_subset}_predictions.json" + ) + + with open(qrels_save_path, "w") as f: + json.dump(results, f) + + ndcg, _map, recall, precision, naucs = retriever.evaluate( + relevant_docs, + results, + retriever.k_values, + ignore_identical_ids=self.ignore_identical_ids, + ) + mrr, naucs_mrr = retriever.evaluate_custom( + relevant_docs, results, retriever.k_values, "mrr" + ) + scores = { + **{f"ndcg_at_{k.split('@')[1]}": v for (k, v) in ndcg.items()}, + **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()}, + **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()}, + **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()}, + **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, + **{ + k.replace("@", "_at_").replace("_P", "_precision").lower(): v + for k, v in naucs.items() + }, + **{ + k.replace("@", "_at_").replace("_P", "_precision").lower(): v + for k, v in naucs_mrr.items() + }, + } + self._add_main_score(scores) + + if export_errors: + errors = {} + + top_k = kwargs.get("top_k", 1) + if not save_predictions and top_k == 1: + for qid in results.keys(): + doc_scores = results[qid] + sorted_docs = sorted( + doc_scores.items(), key=lambda x: x[1], reverse=True + )[:top_k] + results[qid] = {doc_id: score for doc_id, score in sorted_docs} + for qid, retrieved_docs in results.items(): + expected_docs = relevant_docs[qid] + false_positives = [ + doc for doc in retrieved_docs if doc not in expected_docs + ] + false_negatives = [ + doc for doc in expected_docs if doc not in retrieved_docs + ] + if false_positives or false_negatives: + errors[qid] = { + "false_positives": false_positives, + "false_negatives": false_negatives, + } + + errors_save_path = ( + output_folder / f"{self.metadata.name}_{hf_subset}_errors.json" + ) + with open(errors_save_path, "w") as f: + json.dump(errors, f) + + return scores + + def _add_main_score(self, scores: ScoresDict) -> None: + scores["main_score"] = scores[self.metadata.main_score] + + def calculate_metadata_metrics(self) -> None: + self.load_data() + + all_details = {} + pbar_split = tqdm.tqdm( + self.metadata_dict["eval_splits"], desc="Processing Splits..." + ) + for split in pbar_split: + pbar_split.set_postfix_str(f"Split: {split}") + print(f"Processing metadata for split {split}") + all_details[split] = {} + if self.is_multilingual: + pbar_lang = tqdm.tqdm( + self.relevant_docs.keys(), desc="Processing Languages..." + ) + for lang in pbar_lang: + pbar_lang.set_postfix_str(f"Language: {lang}") + print(f"Processing metadata for language {lang}") + split_details = process_language( + self.relevant_docs[lang][split], + self.queries[lang][split], + self.corpus[lang][split], + lang, + ) + all_details[split][lang] = split_details + else: + split_details = process_language( + self.relevant_docs[split], self.queries[split], self.corpus[split] + ) + all_details[split] = split_details + + return all_details + + +def process_language(relevant_docs, queries, corpus, lang=None): + """We want to get three pieces of information: + - the number of documents (and their char length) in the corpus + - the number of queries (and their char length) + - the average number of relevant documents per query + """ + query_len, doc_len = calculate_length(queries, corpus) + num_documents = len(corpus) + num_queries = len(queries) + + # number of qrels that are not 0 + num_qrels_non_zero = sum( + sum(1 for doc_id in docs if docs[doc_id] != 0) + for docs in relevant_docs.values() + ) + qrels_per_doc = num_qrels_non_zero / num_queries if num_queries else 0 + + language_description = f" for language {lang}" if lang else "" + print(f"Average document character length{language_description} is {doc_len}") + print(f"Average query character length{language_description} is {query_len}") + print(f"Number of documents{language_description} is {num_documents}") + print(f"Number of queries{language_description} is {num_queries}") + print( + f"Average number of relevant documents per query{language_description} is {qrels_per_doc}" + ) + return { + "average_document_length": doc_len, + "average_query_length": query_len, + "num_documents": num_documents, + "num_queries": num_queries, + "average_relevant_docs_per_query": qrels_per_doc, + } + + +def calculate_length(queries, corpus): + queries_lens = [] + doc_lens = [] + for query in queries.values(): + queries_lens.append(len(query)) + + for doc in corpus.values(): + if isinstance(doc, Image.Image): + doc_lens.append(1.0) # for image append 1. Can perhaps be removed. + + doc_len = sum(doc_lens) / len(doc_lens) if doc_lens else 0 + query_len = sum(queries_lens) / len(queries_lens) if queries_lens else 0 + return query_len, doc_len diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 05911a7a6e..fcbdd148ed 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -101,6 +101,11 @@ "p2p", # Paragraph-to-paragraph "i2t", "t2i", + "it2t", + "it2i", + "i2it", + "t2it", + "it2it", ] ANNOTATOR_TYPE = Literal[ diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index 8d74dfd73a..bd2c98c22c 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -13,6 +13,7 @@ from .AbsTaskSpeedTask import * from .AbsTaskSTS import * from .AbsTaskSummarization import * +from .Image.AbsTaskAny2AnyRetrieval import * from .Image.AbsTaskI2TRetrieval import * from .Image.AbsTaskImageClassification import * from .Image.AbsTaskImageClustering import * diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py new file mode 100644 index 0000000000..7a5ad8565d --- /dev/null +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -0,0 +1,362 @@ +from __future__ import annotations + +import heapq +import json +import logging +import os +from collections import defaultdict +from typing import Any, Dict, List, Tuple + +import numpy as np +import pytrec_eval +import torch +from PIL import Image + +from mteb.encoder_interface import EncoderWithQueryCorpusEncode + +from ..Evaluator import Evaluator +from ..utils import ( + confidence_scores, + cos_sim, + dot_score, + download, + hole, + mrr, + nAUC, + recall_cap, + top_k_accuracy, +) + +logger = logging.getLogger(__name__) + + +# Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/search/dense/exact_search.py#L12 +class DenseRetrievalExactSearch: + def __init__( + self, + model: EncoderWithQueryCorpusEncode, + encode_kwargs: dict[str, Any] = {}, + corpus_chunk_size: int = 50000, + previous_results: str | None = None, + **kwargs: Any, + ): + # Model is class that provides get_text_embeddings() and get_image_embeddings() + self.model = model + self.encode_kwargs = encode_kwargs + + if "batch_size" not in encode_kwargs: + encode_kwargs["batch_size"] = 128 + + self.score_functions = {"cos_sim": cos_sim, "dot": dot_score} + self.score_function_desc = { + "cos_sim": "Cosine Similarity", + "dot": "Dot Product", + } + self.corpus_chunk_size = corpus_chunk_size + self.previous_results = previous_results + self.batch_size = encode_kwargs.get("batch_size") + self.show_progress_bar = encode_kwargs.get("show_progress_bar") + self.save_corpus_embeddings = kwargs.get("save_corpus_embeddings", False) + self.corpus_embeddings = defaultdict(list) + self.results = {} + + if self.previous_results is not None: + self.previous_results = self.load_results_file() + + def search( + self, + corpus: dict[str, Dict[str, str | Image.Image]], + queries: dict[str, Dict[str, str | Image.Image]], + top_k: int, + score_function: str, + return_sorted: bool = False, + **kwargs, + ) -> dict[str, dict[str, float]]: + if score_function not in self.score_functions: + raise ValueError( + f"score function: {score_function} must be either (cos_sim) for cosine similarity or (dot) for dot product" + ) + + logger.info("Encoding Queries.") + query_ids = list(queries.keys()) + self.results = {qid: {} for qid in query_ids} + + q_modality = queries[query_ids[0]]["modality"] + + if q_modality == "text": + query_texts = [queries[qid]["text"] for qid in query_ids] + query_embeddings = self.model.get_text_embeddings( + texts=query_texts, batch_size=self.encode_kwargs["batch_size"] + ) + elif q_modality == "image": + query_images = [queries[qid]["image"] for qid in query_ids] + query_embeddings = self.model.get_image_embeddings( + images=query_images, batch_size=self.encode_kwargs["batch_size"] + ) + elif q_modality == "image,text": + query_texts = [queries[qid]["text"] for qid in query_ids] + query_images = [queries[qid]["image"] for qid in query_ids] + query_embeddings = self.model.get_fused_embeddings( + texts=query_texts, + images=query_images, + batch_size=self.encode_kwargs["batch_size"], + ) + else: + raise ValueError(f"Unsupported modality: {q_modality}") + + logger.info("Preparing Corpus...") + corpus_ids = list(corpus.keys()) + + corpus_modality = corpus[corpus_ids[0]]["modality"] + + if corpus_modality == "text": + corpus_texts = [corpus[cid]["text"] for cid in corpus_ids] + corpus_embeddings = self.model.get_text_embeddings( + texts=corpus_texts, batch_size=self.encode_kwargs["batch_size"] + ) + elif corpus_modality == "image": + corpus_images = [corpus[cid]["image"] for cid in corpus_ids] + corpus_embeddings = self.model.get_image_embeddings( + images=corpus_images, batch_size=self.encode_kwargs["batch_size"] + ) + elif corpus_modality == "image,text": + corpus_texts = [corpus[cid]["text"] for cid in corpus_ids] + corpus_images = [corpus[cid]["image"] for cid in corpus_ids] + corpus_embeddings = self.model.get_fused_embeddings( + texts=corpus_texts, + images=corpus_images, + batch_size=self.encode_kwargs["batch_size"], + ) + else: + raise ValueError(f"Unsupported modality: {corpus_modality}") + + logger.info("Encoding Corpus in batches... Warning: This might take a while!") + logger.info( + "Scoring Function: {} ({})".format( + self.score_function_desc[score_function], score_function + ) + ) + + result_heaps = { + qid: [] for qid in query_ids + } # Keep only the top-k docs for each query + + cos_scores = self.score_functions[score_function]( + query_embeddings, corpus_embeddings + ) + cos_scores[torch.isnan(cos_scores)] = -1 + + cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( + cos_scores, + top_k, + dim=1, + largest=True, + sorted=return_sorted, + ) + cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() + cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() + + for query_itr in range(len(query_embeddings)): + query_id = query_ids[query_itr] + for sub_corpus_id, score in zip( + cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] + ): + corpus_id = corpus_ids[sub_corpus_id] + if len(result_heaps[query_id]) < top_k: + heapq.heappush(result_heaps[query_id], (score, corpus_id)) + else: + heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) + + for qid in result_heaps: + for score, corpus_id in result_heaps[qid]: + self.results[qid][corpus_id] = score + + return self.results + + def load_results_file(self): + # load the first stage results from file in format {qid: {doc_id: score}} + if "https://" in self.previous_results: + # download the file + if not os.path.exists(self.previous_results): + url_descriptor = self.previous_results.split("https://")[-1].replace( + "/", "--" + ) + dest_file = os.path.join( + "results", f"cached_predictions--{url_descriptor}" + ) + os.makedirs(os.path.dirname(os.path.abspath(dest_file)), exist_ok=True) + download(self.previous_results, dest_file) + logger.info( + f"Downloaded the previous results at {self.previous_results} to {dest_file}" + ) + self.previous_results = dest_file + + with open(self.previous_results, "r") as f: + previous_results = json.load(f) + assert isinstance(previous_results, dict) + assert isinstance(previous_results[list(previous_results.keys())[0]], dict) + return previous_results + + +# Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/evaluation.py#L9 +class Any2AnyRetrievalEvaluator(Evaluator): + def __init__( + self, + retriever=None, + task_name: str | None = None, + k_values: List[int] = [1, 3, 5, 10, 20, 100, 1000], + score_function: str = "cos_sim", + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ): + super().__init__(**kwargs) + + self.retriever = DenseRetrievalExactSearch( + retriever, encode_kwargs=encode_kwargs, **kwargs + ) + self.k_values = k_values + self.top_k = ( + max(k_values) if "top_k" not in kwargs else kwargs["top_k"] + ) # can lower it if reranking + self.score_function = score_function + self.task_name = task_name + + def __call__( + self, + corpus: dict[str, Dict[str, str | Image.Image]], + queries: dict[str, Dict[str, str | Image.Image]], + ) -> dict[str, dict[str, float]]: + if not self.retriever: + raise ValueError("Model/Technique has not been provided!") + + return self.retriever.search( + corpus, + queries, + self.top_k, + self.score_function, + prompt_name=self.task_name, # type: ignore + ) + + @staticmethod + def evaluate( + qrels: dict[str, dict[str, int]], + results: dict[str, dict[str, float]], + k_values: List[int], + ignore_identical_ids: bool = False, + ) -> Tuple[ + dict[str, float], + dict[str, float], + dict[str, float], + dict[str, float], + dict[str, float], + ]: + if ignore_identical_ids: + logger.debug( + "For evaluation, ``ignore_identical_ids=True`` is set to True, the evaluator will ignore identical query and document ids." + ) + # Remove identical ids from results dict + for qid, rels in results.items(): + for pid in list(rels): + if qid == pid: + results[qid].pop(pid) + else: + logger.debug( + "For evaluation, we DO NOT ignore identical query and document ids (default), please explicitly set ``ignore_identical_ids=True`` to ignore this." + ) + + all_ndcgs, all_aps, all_recalls, all_precisions = {}, {}, {}, {} + + for k in k_values: + all_ndcgs[f"NDCG@{k}"] = [] + all_aps[f"MAP@{k}"] = [] + all_recalls[f"Recall@{k}"] = [] + all_precisions[f"P@{k}"] = [] + + map_string = "map_cut." + ",".join([str(k) for k in k_values]) + ndcg_string = "ndcg_cut." + ",".join([str(k) for k in k_values]) + recall_string = "recall." + ",".join([str(k) for k in k_values]) + precision_string = "P." + ",".join([str(k) for k in k_values]) + evaluator = pytrec_eval.RelevanceEvaluator( + qrels, {map_string, ndcg_string, recall_string, precision_string} + ) + scores = evaluator.evaluate(results) + + for query_id in scores.keys(): + for k in k_values: + all_ndcgs[f"NDCG@{k}"].append(scores[query_id]["ndcg_cut_" + str(k)]) + all_aps[f"MAP@{k}"].append(scores[query_id]["map_cut_" + str(k)]) + all_recalls[f"Recall@{k}"].append(scores[query_id]["recall_" + str(k)]) + all_precisions[f"P@{k}"].append(scores[query_id]["P_" + str(k)]) + + ndcg, _map, recall, precision = ( + all_ndcgs.copy(), + all_aps.copy(), + all_recalls.copy(), + all_precisions.copy(), + ) + + for k in k_values: + ndcg[f"NDCG@{k}"] = round(sum(ndcg[f"NDCG@{k}"]) / len(scores), 5) + _map[f"MAP@{k}"] = round(sum(_map[f"MAP@{k}"]) / len(scores), 5) + recall[f"Recall@{k}"] = round(sum(recall[f"Recall@{k}"]) / len(scores), 5) + precision[f"P@{k}"] = round(sum(precision[f"P@{k}"]) / len(scores), 5) + + naucs = Any2AnyRetrievalEvaluator.evaluate_abstention( + results, {**all_ndcgs, **all_aps, **all_recalls, **all_precisions} + ) + + return ndcg, _map, recall, precision, naucs + + @staticmethod + def evaluate_custom( + qrels: dict[str, dict[str, int]], + results: dict[str, dict[str, float]], + k_values: List[int], + metric: str, + output_type: str = "all", + ) -> Tuple[Dict[str, float]]: + if metric.lower() in ["mrr", "mrr@k", "mrr_cut"]: + metric_scores = mrr(qrels, results, k_values, output_type) + + elif metric.lower() in ["recall_cap", "r_cap", "r_cap@k"]: + metric_scores = recall_cap(qrels, results, k_values, output_type) + + elif metric.lower() in ["hole", "hole@k"]: + metric_scores = hole(qrels, results, k_values, output_type) + + elif metric.lower() in [ + "acc", + "top_k_acc", + "accuracy", + "accuracy@k", + "top_k_accuracy", + ]: + metric_scores = top_k_accuracy(qrels, results, k_values, output_type) + + naucs = Any2AnyRetrievalEvaluator.evaluate_abstention(results, metric_scores) + metric_scores_avg = {k: sum(v) / len(v) for k, v in metric_scores.items()} + + return metric_scores_avg, naucs + + @staticmethod + def evaluate_abstention( + results: dict[str, dict[str, float]], + metric_scores: dict[str, list[float]], + ) -> Dict[str, float]: + """Computes normalized Area Under the Curve on a set of evaluated instances as presented in the paper https://arxiv.org/abs/2402.12997""" + all_sim_scores = [list(results[qid].values()) for qid in list(results.keys())] + all_conf_scores = [ + confidence_scores(sim_scores) for sim_scores in all_sim_scores + ] + conf_fcts = list(all_conf_scores[0].keys()) + all_conf_scores = { + fct: np.array([x[fct] for x in all_conf_scores]) for fct in conf_fcts + } + metric_scores = {k: np.array(v) for k, v in metric_scores.items()} + naucs = {} + + for metric_name, scores in metric_scores.items(): + for fct, conf_scores in all_conf_scores.items(): + naucs[f"nAUC_{metric_name}_{fct}"] = nAUC(conf_scores, scores) + + return naucs diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py index 06f9862da1..10892fd6c5 100644 --- a/mteb/evaluation/evaluators/__init__.py +++ b/mteb/evaluation/evaluators/__init__.py @@ -3,6 +3,7 @@ from .BitextMiningEvaluator import * from .ClassificationEvaluator import * from .ClusteringEvaluator import * +from .Image.Any2AnyRetrievalEvaluator import * from .Image.ClassificationEvaluator import * from .Image.ClusteringEvaluator import * from .Image.I2TRetrievalEvaluator import * diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index df5d8d8871..733d83c927 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -73,6 +73,41 @@ def calculate_probs(self, text_embeddings, image_embeddings): probs = (logits * 100).softmax(dim=-1) return probs + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + else: + # to do: add other fusion mode + raise ValueError(f"fusion mode {fusion_mode} hasn't been implemented") + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + clip_vit_large_patch14 = ModelMeta( loader=partial( diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py new file mode 100644 index 0000000000..91a7f471a9 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .eng.CIRRIT2IRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py new file mode 100644 index 0000000000..2f1c72537f --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class CIRRIT2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="CIRRIT2TRetrieval", + description="Retrieve images based on texts and images.", + reference="https://openaccess.thecvf.com/content/ICCV2021/html/Liu_Image_Retrieval_on_Real-Life_Images_With_Pre-Trained_Vision-and-Language_Models_ICCV_2021_paper.html", + dataset={ + "path": "MRBench/mbeir_cirr_task7", + "revision": "503301cd99348035b9675883a543aa1ded0cf07c", + "trust_remote_code": True, + }, + type="Retrieval", + category="it2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2018-01-01", "2018-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + text_creation="found", + bibtex_citation="""@inproceedings{liu2021image, + title={Image retrieval on real-life images with pre-trained vision-and-language models}, + author={Liu, Zheyuan and Rodriguez-Opazo, Cristian and Teney, Damien and Gould, Stephen}, + booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision}, + pages={2125--2134}, + year={2021} +}""", + n_samples={"test": 1172}, + avg_character_length={ + "test": { + "average_document_length": 30.94235294117647, + "average_query_length": 131.56569965870307, + "num_documents": 9350, + "num_queries": 1172, + "average_relevant_docs_per_query": 1.0, + } + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Image/Clustering/__init__.py b/mteb/tasks/Image/Clustering/__init__.py index 36ae667862..8b99993d8c 100644 --- a/mteb/tasks/Image/Clustering/__init__.py +++ b/mteb/tasks/Image/Clustering/__init__.py @@ -1,3 +1,3 @@ from __future__ import annotations -from .TinyImageNet import * +from .eng.TinyImageNet import * diff --git a/mteb/tasks/Image/Clustering/TinyImageNet.py b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py similarity index 100% rename from mteb/tasks/Image/Clustering/TinyImageNet.py rename to mteb/tasks/Image/Clustering/eng/TinyImageNet.py diff --git a/mteb/tasks/Image/Clustering/eng/__init__.py b/mteb/tasks/Image/Clustering/eng/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Image/__init__.py b/mteb/tasks/Image/__init__.py index 08461edbb1..d9869877ed 100644 --- a/mteb/tasks/Image/__init__.py +++ b/mteb/tasks/Image/__init__.py @@ -1,3 +1,4 @@ +from .Any2AnyRetrieval import * from .Clustering import * from .I2TRetrieval import * from .ImageClassification import * diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CIRRIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CIRRIT2TRetrieval.json new file mode 100644 index 0000000000..f63b804ff7 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CIRRIT2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", + "evaluation_time": 123.18759036064148, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.09754, + "map_at_1": 0.00408, + "map_at_10": 0.06314, + "map_at_100": 0.07411, + "map_at_1000": 0.07553, + "map_at_20": 0.06877, + "map_at_3": 0.04189, + "map_at_5": 0.05327, + "mrr_at_1": 0.008633093525179856, + "mrr_at_10": 0.06345980358570272, + "mrr_at_100": 0.07443694089212283, + "mrr_at_1000": 0.07586721932715101, + "mrr_at_20": 0.06910360751975989, + "mrr_at_3": 0.04240607513988813, + "mrr_at_5": 0.053569144684252626, + "nauc_map_at_1000_diff1": -0.09430006303115672, + "nauc_map_at_1000_max": 0.07971589724669469, + "nauc_map_at_1000_std": 0.1097849476586519, + "nauc_map_at_100_diff1": -0.09304519843836945, + "nauc_map_at_100_max": 0.07978371174425837, + "nauc_map_at_100_std": 0.10767454860254984, + "nauc_map_at_10_diff1": -0.09265414471119784, + "nauc_map_at_10_max": 0.07156210824751408, + "nauc_map_at_10_std": 0.09100330279710303, + "nauc_map_at_1_diff1": -0.5319922978582364, + "nauc_map_at_1_max": 0.22971479979166354, + "nauc_map_at_1_std": 0.01727457819725688, + "nauc_map_at_20_diff1": -0.08883107818505538, + "nauc_map_at_20_max": 0.07564770187280079, + "nauc_map_at_20_std": 0.09984178318888319, + "nauc_map_at_3_diff1": -0.11451405772196703, + "nauc_map_at_3_max": 0.06998317415517456, + "nauc_map_at_3_std": 0.07144993107855412, + "nauc_map_at_5_diff1": -0.09303955529622057, + "nauc_map_at_5_max": 0.06894560968071584, + "nauc_map_at_5_std": 0.0765640043400923, + "nauc_mrr_at_1000_diff1": -0.09682010815892876, + "nauc_mrr_at_1000_max": 0.08150275059724531, + "nauc_mrr_at_1000_std": 0.10918420856774745, + "nauc_mrr_at_100_diff1": -0.09553424341368219, + "nauc_mrr_at_100_max": 0.08155065521338761, + "nauc_mrr_at_100_std": 0.10708594341679054, + "nauc_mrr_at_10_diff1": -0.09506902538489626, + "nauc_mrr_at_10_max": 0.07339968417405465, + "nauc_mrr_at_10_std": 0.0906630104753227, + "nauc_mrr_at_1_diff1": -0.5290373550368184, + "nauc_mrr_at_1_max": 0.2344229794592015, + "nauc_mrr_at_1_std": 0.005955042780908148, + "nauc_mrr_at_20_diff1": -0.09140047438145477, + "nauc_mrr_at_20_max": 0.07741254897582059, + "nauc_mrr_at_20_std": 0.0993326318120689, + "nauc_mrr_at_3_diff1": -0.11939291114310412, + "nauc_mrr_at_3_max": 0.07384253329310762, + "nauc_mrr_at_3_std": 0.07139864053407562, + "nauc_mrr_at_5_diff1": -0.09696157114944583, + "nauc_mrr_at_5_max": 0.07122590150036714, + "nauc_mrr_at_5_std": 0.07643234788433957, + "nauc_ndcg_at_1000_diff1": -0.10106932279250458, + "nauc_ndcg_at_1000_max": 0.10096178940020763, + "nauc_ndcg_at_1000_std": 0.19577796302889916, + "nauc_ndcg_at_100_diff1": -0.08696795064375475, + "nauc_ndcg_at_100_max": 0.09814750175500678, + "nauc_ndcg_at_100_std": 0.16801654804595473, + "nauc_ndcg_at_10_diff1": -0.07446639262352736, + "nauc_ndcg_at_10_max": 0.070454822695297, + "nauc_ndcg_at_10_std": 0.10445010583766168, + "nauc_ndcg_at_1_diff1": -0.5277319943271112, + "nauc_ndcg_at_1_max": 0.23186783701943348, + "nauc_ndcg_at_1_std": 0.003311740580321219, + "nauc_ndcg_at_20_diff1": -0.06726707326404968, + "nauc_ndcg_at_20_max": 0.0801126382265569, + "nauc_ndcg_at_20_std": 0.12521764835456067, + "nauc_ndcg_at_3_diff1": -0.09563193072427134, + "nauc_ndcg_at_3_max": 0.06551385540947947, + "nauc_ndcg_at_3_std": 0.0750635864151865, + "nauc_ndcg_at_5_diff1": -0.07109394463486146, + "nauc_ndcg_at_5_max": 0.06440364564768383, + "nauc_ndcg_at_5_std": 0.08123581037260018, + "nauc_precision_at_1000_diff1": -0.21894379652044482, + "nauc_precision_at_1000_max": 0.20110532137532738, + "nauc_precision_at_1000_std": 0.4985263956386437, + "nauc_precision_at_100_diff1": -0.11448917033346492, + "nauc_precision_at_100_max": 0.13704560799012916, + "nauc_precision_at_100_std": 0.25670911616573194, + "nauc_precision_at_10_diff1": -0.08078240677660208, + "nauc_precision_at_10_max": 0.07928001837810364, + "nauc_precision_at_10_std": 0.11439470716368563, + "nauc_precision_at_1_diff1": -0.5277319943271112, + "nauc_precision_at_1_max": 0.23186783701943348, + "nauc_precision_at_1_std": 0.003311740580321219, + "nauc_precision_at_20_diff1": -0.06791517816537197, + "nauc_precision_at_20_max": 0.0949095608045591, + "nauc_precision_at_20_std": 0.15010397292626543, + "nauc_precision_at_3_diff1": -0.11153702507708928, + "nauc_precision_at_3_max": 0.07582115029130086, + "nauc_precision_at_3_std": 0.06977064016082565, + "nauc_precision_at_5_diff1": -0.07729490099402637, + "nauc_precision_at_5_max": 0.07163492889720516, + "nauc_precision_at_5_std": 0.08001410639199522, + "nauc_recall_at_1000_diff1": -0.1855830858750129, + "nauc_recall_at_1000_max": 0.1980425044862128, + "nauc_recall_at_1000_std": 0.564878472724284, + "nauc_recall_at_100_diff1": -0.09436834890876052, + "nauc_recall_at_100_max": 0.1315423417990006, + "nauc_recall_at_100_std": 0.2717679584573868, + "nauc_recall_at_10_diff1": -0.05552800142913649, + "nauc_recall_at_10_max": 0.07015781570417769, + "nauc_recall_at_10_std": 0.12162556282236173, + "nauc_recall_at_1_diff1": -0.5319922978582364, + "nauc_recall_at_1_max": 0.22971479979166354, + "nauc_recall_at_1_std": 0.01727457819725688, + "nauc_recall_at_20_diff1": -0.045395108502877965, + "nauc_recall_at_20_max": 0.08780002237291626, + "nauc_recall_at_20_std": 0.1581707521402131, + "nauc_recall_at_3_diff1": -0.06721017685785699, + "nauc_recall_at_3_max": 0.05770885232307196, + "nauc_recall_at_3_std": 0.08003902316354816, + "nauc_recall_at_5_diff1": -0.04262981360165199, + "nauc_recall_at_5_max": 0.05798966850733496, + "nauc_recall_at_5_std": 0.08747605572883835, + "ndcg_at_1": 0.00839, + "ndcg_at_10": 0.09754, + "ndcg_at_100": 0.1599, + "ndcg_at_1000": 0.20034, + "ndcg_at_20": 0.11799, + "ndcg_at_3": 0.0533, + "ndcg_at_5": 0.0736, + "precision_at_1": 0.00839, + "precision_at_10": 0.02189, + "precision_at_100": 0.00533, + "precision_at_1000": 0.00086, + "precision_at_20": 0.01499, + "precision_at_3": 0.03181, + "precision_at_5": 0.02897, + "recall_at_1": 0.00408, + "recall_at_10": 0.20899, + "recall_at_100": 0.52254, + "recall_at_1000": 0.8458, + "recall_at_20": 0.28981, + "recall_at_3": 0.08597, + "recall_at_5": 0.13501 + } + ] + }, + "task_name": "CIRRIT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2TRetrieval.json new file mode 100644 index 0000000000..428f0fac5b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", + "evaluation_time": 76.20298647880554, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.10026, + "map_at_1": 0.00408, + "map_at_10": 0.06541, + "map_at_100": 0.07611, + "map_at_1000": 0.07756, + "map_at_20": 0.07065, + "map_at_3": 0.04442, + "map_at_5": 0.05543, + "mrr_at_1": 0.008872901678657074, + "mrr_at_10": 0.06594115184043225, + "mrr_at_100": 0.07663903441028781, + "mrr_at_1000": 0.07809622231266924, + "mrr_at_20": 0.07118418253647203, + "mrr_at_3": 0.044924060751398966, + "mrr_at_5": 0.05594324540367709, + "nauc_map_at_1000_diff1": -0.08655636473413195, + "nauc_map_at_1000_max": 0.11676320143589596, + "nauc_map_at_1000_std": 0.09629099312167273, + "nauc_map_at_100_diff1": -0.08516530982285186, + "nauc_map_at_100_max": 0.11685859248705843, + "nauc_map_at_100_std": 0.09453524737415578, + "nauc_map_at_10_diff1": -0.08765926721523226, + "nauc_map_at_10_max": 0.10978450131161009, + "nauc_map_at_10_std": 0.08130576460840168, + "nauc_map_at_1_diff1": -0.5318107924683155, + "nauc_map_at_1_max": 0.20301772439590268, + "nauc_map_at_1_std": -0.021059886256622317, + "nauc_map_at_20_diff1": -0.08592919050491332, + "nauc_map_at_20_max": 0.11502979273505931, + "nauc_map_at_20_std": 0.08621208379307907, + "nauc_map_at_3_diff1": -0.10585711421716114, + "nauc_map_at_3_max": 0.10824597492299746, + "nauc_map_at_3_std": 0.04942655635325061, + "nauc_map_at_5_diff1": -0.0903941122012634, + "nauc_map_at_5_max": 0.106932714398111, + "nauc_map_at_5_std": 0.06656522457054902, + "nauc_mrr_at_1000_diff1": -0.0907646058253511, + "nauc_mrr_at_1000_max": 0.12018754710642203, + "nauc_mrr_at_1000_std": 0.09796107072305628, + "nauc_mrr_at_100_diff1": -0.08936189065625093, + "nauc_mrr_at_100_max": 0.1202619669752152, + "nauc_mrr_at_100_std": 0.09620129145114814, + "nauc_mrr_at_10_diff1": -0.0919795422688847, + "nauc_mrr_at_10_max": 0.11332346465731229, + "nauc_mrr_at_10_std": 0.08317876347162799, + "nauc_mrr_at_1_diff1": -0.5216644065584233, + "nauc_mrr_at_1_max": 0.20209718707169744, + "nauc_mrr_at_1_std": -0.005366246310705697, + "nauc_mrr_at_20_diff1": -0.09013629049462879, + "nauc_mrr_at_20_max": 0.1184401494147931, + "nauc_mrr_at_20_std": 0.08802180434971997, + "nauc_mrr_at_3_diff1": -0.11096146519530939, + "nauc_mrr_at_3_max": 0.11435332435139552, + "nauc_mrr_at_3_std": 0.0525013678199003, + "nauc_mrr_at_5_diff1": -0.09504839521625083, + "nauc_mrr_at_5_max": 0.11098993126230935, + "nauc_mrr_at_5_std": 0.06863872006792304, + "nauc_ndcg_at_1000_diff1": -0.08344132954414758, + "nauc_ndcg_at_1000_max": 0.1323215007612839, + "nauc_ndcg_at_1000_std": 0.17697700366712762, + "nauc_ndcg_at_100_diff1": -0.06471556789805906, + "nauc_ndcg_at_100_max": 0.13367762668005959, + "nauc_ndcg_at_100_std": 0.1585535458458699, + "nauc_ndcg_at_10_diff1": -0.07011937821033006, + "nauc_ndcg_at_10_max": 0.11288519695312943, + "nauc_ndcg_at_10_std": 0.09866561563226296, + "nauc_ndcg_at_1_diff1": -0.5186323737973859, + "nauc_ndcg_at_1_max": 0.19513971405573238, + "nauc_ndcg_at_1_std": -0.03383801602207826, + "nauc_ndcg_at_20_diff1": -0.06694529638460721, + "nauc_ndcg_at_20_max": 0.12469099422056072, + "nauc_ndcg_at_20_std": 0.1092102137002337, + "nauc_ndcg_at_3_diff1": -0.08867212008892655, + "nauc_ndcg_at_3_max": 0.10990739346950268, + "nauc_ndcg_at_3_std": 0.0543947490857072, + "nauc_ndcg_at_5_diff1": -0.07145417341479289, + "nauc_ndcg_at_5_max": 0.10696288747042368, + "nauc_ndcg_at_5_std": 0.07571492220327662, + "nauc_precision_at_1000_diff1": -0.18314342593006072, + "nauc_precision_at_1000_max": 0.19346644885136122, + "nauc_precision_at_1000_std": 0.47116734223208256, + "nauc_precision_at_100_diff1": -0.06370954016671407, + "nauc_precision_at_100_max": 0.16241220181965255, + "nauc_precision_at_100_std": 0.258016779609776, + "nauc_precision_at_10_diff1": -0.07421873744815492, + "nauc_precision_at_10_max": 0.12076870044196765, + "nauc_precision_at_10_std": 0.11212464069366684, + "nauc_precision_at_1_diff1": -0.5186323737973859, + "nauc_precision_at_1_max": 0.19513971405573238, + "nauc_precision_at_1_std": -0.03383801602207826, + "nauc_precision_at_20_diff1": -0.06793713334562095, + "nauc_precision_at_20_max": 0.1402319646787195, + "nauc_precision_at_20_std": 0.1291706714805028, + "nauc_precision_at_3_diff1": -0.10265385272093064, + "nauc_precision_at_3_max": 0.11808887171461854, + "nauc_precision_at_3_std": 0.052103940853181874, + "nauc_precision_at_5_diff1": -0.07759274159246009, + "nauc_precision_at_5_max": 0.1124438194370848, + "nauc_precision_at_5_std": 0.07919386090270071, + "nauc_recall_at_1000_diff1": -0.14300824800294304, + "nauc_recall_at_1000_max": 0.19179659255731416, + "nauc_recall_at_1000_std": 0.545720347238759, + "nauc_recall_at_100_diff1": -0.041986859938095734, + "nauc_recall_at_100_max": 0.16127782528121198, + "nauc_recall_at_100_std": 0.27283270613912575, + "nauc_recall_at_10_diff1": -0.05065793884544601, + "nauc_recall_at_10_max": 0.11649110613991276, + "nauc_recall_at_10_std": 0.12019520504934413, + "nauc_recall_at_1_diff1": -0.5318107924683155, + "nauc_recall_at_1_max": 0.20301772439590268, + "nauc_recall_at_1_std": -0.021059886256622317, + "nauc_recall_at_20_diff1": -0.04704172155925913, + "nauc_recall_at_20_max": 0.1374810167985767, + "nauc_recall_at_20_std": 0.137082012873361, + "nauc_recall_at_3_diff1": -0.062451024117901044, + "nauc_recall_at_3_max": 0.11020315329162346, + "nauc_recall_at_3_std": 0.061213944422035795, + "nauc_recall_at_5_diff1": -0.04626447590534941, + "nauc_recall_at_5_max": 0.10547386265685323, + "nauc_recall_at_5_std": 0.08783936831294922, + "ndcg_at_1": 0.00839, + "ndcg_at_10": 0.10026, + "ndcg_at_100": 0.16319, + "ndcg_at_1000": 0.20471, + "ndcg_at_20": 0.11947, + "ndcg_at_3": 0.05637, + "ndcg_at_5": 0.07618, + "precision_at_1": 0.00839, + "precision_at_10": 0.02228, + "precision_at_100": 0.00544, + "precision_at_1000": 0.00088, + "precision_at_20": 0.01495, + "precision_at_3": 0.03341, + "precision_at_5": 0.02969, + "recall_at_1": 0.00408, + "recall_at_10": 0.21283, + "recall_at_100": 0.53345, + "recall_at_1000": 0.86631, + "recall_at_20": 0.28909, + "recall_at_3": 0.09053, + "recall_at_5": 0.13861 + } + ] + }, + "task_name": "CIRRIT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CIRRIT2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CIRRIT2TRetrieval.json new file mode 100644 index 0000000000..c812f33795 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CIRRIT2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", + "evaluation_time": 334.5525403022766, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.09282, + "map_at_1": 0.00396, + "map_at_10": 0.0602, + "map_at_100": 0.07143, + "map_at_1000": 0.07288, + "map_at_20": 0.06567, + "map_at_3": 0.04097, + "map_at_5": 0.05122, + "mrr_at_1": 0.00815347721822542, + "mrr_at_10": 0.06053176506413904, + "mrr_at_100": 0.07175520833724132, + "mrr_at_1000": 0.07319771612683665, + "mrr_at_20": 0.06600739568371021, + "mrr_at_3": 0.04128697042366111, + "mrr_at_5": 0.05150279776179057, + "nauc_map_at_1000_diff1": -0.12353623400385476, + "nauc_map_at_1000_max": 0.0912971224497447, + "nauc_map_at_1000_std": 0.08525451600633958, + "nauc_map_at_100_diff1": -0.12289322987256437, + "nauc_map_at_100_max": 0.09144178386642453, + "nauc_map_at_100_std": 0.08296012542760436, + "nauc_map_at_10_diff1": -0.12494376185642647, + "nauc_map_at_10_max": 0.08674812682258538, + "nauc_map_at_10_std": 0.06449719501276227, + "nauc_map_at_1_diff1": -0.5463082664585215, + "nauc_map_at_1_max": 0.26114918977811374, + "nauc_map_at_1_std": 0.031162171757824993, + "nauc_map_at_20_diff1": -0.1211136576549465, + "nauc_map_at_20_max": 0.08760779427458243, + "nauc_map_at_20_std": 0.07233682566935275, + "nauc_map_at_3_diff1": -0.1519614939106193, + "nauc_map_at_3_max": 0.08140281531605219, + "nauc_map_at_3_std": 0.0465481293396245, + "nauc_map_at_5_diff1": -0.1306452402075584, + "nauc_map_at_5_max": 0.07638304265278853, + "nauc_map_at_5_std": 0.05717760292127242, + "nauc_mrr_at_1000_diff1": -0.12585046105692907, + "nauc_mrr_at_1000_max": 0.09142429242891584, + "nauc_mrr_at_1000_std": 0.08671902673955421, + "nauc_mrr_at_100_diff1": -0.1251947063565411, + "nauc_mrr_at_100_max": 0.09156836634919437, + "nauc_mrr_at_100_std": 0.08442756201074159, + "nauc_mrr_at_10_diff1": -0.12735264435622265, + "nauc_mrr_at_10_max": 0.08687944105221006, + "nauc_mrr_at_10_std": 0.06615580242111993, + "nauc_mrr_at_1_diff1": -0.5415016019823545, + "nauc_mrr_at_1_max": 0.25890560141416386, + "nauc_mrr_at_1_std": 0.014078505026910177, + "nauc_mrr_at_20_diff1": -0.12354316783713191, + "nauc_mrr_at_20_max": 0.0877260670527992, + "nauc_mrr_at_20_std": 0.07382572978906658, + "nauc_mrr_at_3_diff1": -0.15456867301767724, + "nauc_mrr_at_3_max": 0.08127863156043737, + "nauc_mrr_at_3_std": 0.049917519287707324, + "nauc_mrr_at_5_diff1": -0.13341097010600572, + "nauc_mrr_at_5_max": 0.07607293298892381, + "nauc_mrr_at_5_std": 0.058415510075163025, + "nauc_ndcg_at_1000_diff1": -0.1197947190208689, + "nauc_ndcg_at_1000_max": 0.10761964394885151, + "nauc_ndcg_at_1000_std": 0.17544190656064562, + "nauc_ndcg_at_100_diff1": -0.11055695160977044, + "nauc_ndcg_at_100_max": 0.10882941855322274, + "nauc_ndcg_at_100_std": 0.14514994054402142, + "nauc_ndcg_at_10_diff1": -0.10490212058705983, + "nauc_ndcg_at_10_max": 0.0877933413318513, + "nauc_ndcg_at_10_std": 0.07407721145352236, + "nauc_ndcg_at_1_diff1": -0.5415016019823545, + "nauc_ndcg_at_1_max": 0.25890560141416386, + "nauc_ndcg_at_1_std": 0.014078505026910177, + "nauc_ndcg_at_20_diff1": -0.09739288742447834, + "nauc_ndcg_at_20_max": 0.08898965589119323, + "nauc_ndcg_at_20_std": 0.09116950469122251, + "nauc_ndcg_at_3_diff1": -0.13592258833726864, + "nauc_ndcg_at_3_max": 0.07511923847763913, + "nauc_ndcg_at_3_std": 0.05019467063066953, + "nauc_ndcg_at_5_diff1": -0.11148549384385414, + "nauc_ndcg_at_5_max": 0.07112590862724855, + "nauc_ndcg_at_5_std": 0.06408420927682207, + "nauc_precision_at_1000_diff1": -0.2052249721287206, + "nauc_precision_at_1000_max": 0.1810837917883838, + "nauc_precision_at_1000_std": 0.4888332886224504, + "nauc_precision_at_100_diff1": -0.12904477641430562, + "nauc_precision_at_100_max": 0.1472874080458991, + "nauc_precision_at_100_std": 0.2367939277259602, + "nauc_precision_at_10_diff1": -0.10753940868940802, + "nauc_precision_at_10_max": 0.09962493139437552, + "nauc_precision_at_10_std": 0.08001071204177711, + "nauc_precision_at_1_diff1": -0.5415016019823545, + "nauc_precision_at_1_max": 0.25890560141416386, + "nauc_precision_at_1_std": 0.014078505026910177, + "nauc_precision_at_20_diff1": -0.09444210001342718, + "nauc_precision_at_20_max": 0.09951342399887461, + "nauc_precision_at_20_std": 0.10807933554562363, + "nauc_precision_at_3_diff1": -0.1543208082621269, + "nauc_precision_at_3_max": 0.08611811554190649, + "nauc_precision_at_3_std": 0.048213763137672686, + "nauc_precision_at_5_diff1": -0.1195619408248877, + "nauc_precision_at_5_max": 0.0791887243073344, + "nauc_precision_at_5_std": 0.06718837781985118, + "nauc_recall_at_1000_diff1": -0.1684682707344514, + "nauc_recall_at_1000_max": 0.17380982864946737, + "nauc_recall_at_1000_std": 0.5550045573571595, + "nauc_recall_at_100_diff1": -0.10849369522179253, + "nauc_recall_at_100_max": 0.1424542327994153, + "nauc_recall_at_100_std": 0.2500201617623722, + "nauc_recall_at_10_diff1": -0.0824038257749242, + "nauc_recall_at_10_max": 0.09070695218424778, + "nauc_recall_at_10_std": 0.08439981274339317, + "nauc_recall_at_1_diff1": -0.5463082664585215, + "nauc_recall_at_1_max": 0.26114918977811374, + "nauc_recall_at_1_std": 0.031162171757824993, + "nauc_recall_at_20_diff1": -0.07172879582348955, + "nauc_recall_at_20_max": 0.09175525207752894, + "nauc_recall_at_20_std": 0.1135769541413721, + "nauc_recall_at_3_diff1": -0.1121549242083564, + "nauc_recall_at_3_max": 0.06611057402653997, + "nauc_recall_at_3_std": 0.05347844526372312, + "nauc_recall_at_5_diff1": -0.08645717088592683, + "nauc_recall_at_5_max": 0.06479040069227433, + "nauc_recall_at_5_std": 0.07221167912917376, + "ndcg_at_1": 0.00815, + "ndcg_at_10": 0.09282, + "ndcg_at_100": 0.15648, + "ndcg_at_1000": 0.19752, + "ndcg_at_20": 0.11286, + "ndcg_at_3": 0.05247, + "ndcg_at_5": 0.07096, + "precision_at_1": 0.00815, + "precision_at_10": 0.02082, + "precision_at_100": 0.00528, + "precision_at_1000": 0.00086, + "precision_at_20": 0.01439, + "precision_at_3": 0.03173, + "precision_at_5": 0.02806, + "recall_at_1": 0.00396, + "recall_at_10": 0.1982, + "recall_at_100": 0.51703, + "recall_at_1000": 0.84604, + "recall_at_20": 0.2777, + "recall_at_3": 0.08549, + "recall_at_5": 0.13046 + } + ] + }, + "task_name": "CIRRIT2TRetrieval" +} \ No newline at end of file diff --git a/tests/test_all_abstasks.py b/tests/test_all_abstasks.py index ee67a12665..f64d5616b1 100644 --- a/tests/test_all_abstasks.py +++ b/tests/test_all_abstasks.py @@ -13,6 +13,7 @@ from mteb.abstasks.AbsTaskInstructionRetrieval import AbsTaskInstructionRetrieval from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.AbsTaskSpeedTask import AbsTaskSpeedTask +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval from mteb.abstasks.Image.AbsTaskI2TRetrieval import AbsTaskI2TRetrieval from mteb.abstasks.Image.AbsTaskT2IRetrieval import AbsTaskT2IRetrieval from mteb.abstasks.MultiSubsetLoader import MultiSubsetLoader @@ -34,6 +35,7 @@ def test_load_data( isinstance(task, AbsTaskRetrieval) or isinstance(task, AbsTaskI2TRetrieval) or isinstance(task, AbsTaskT2IRetrieval) + or isinstance(task, AbsTaskAny2AnyRetrieval) or isinstance(task, AbsTaskInstructionRetrieval) or isinstance(task, MultiSubsetLoader) or isinstance(task, AbsTaskSpeedTask) From f2aba4f7f67b653e64b839da1d9c1d817c674547 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Fri, 19 Jul 2024 09:54:25 +0000 Subject: [PATCH 015/154] fix tests from merge --- mteb/abstasks/TaskMetadata.py | 7 +++- .../Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py | 35 ++++++++++-------- .../Image/Clustering/eng/TinyImageNet.py | 9 +++-- .../I2TRetrieval/eng/MSCOCOI2TRetrieval.py | 37 ++++++++++--------- .../eng/OxfordFlowersClassification.py | 9 +++-- .../T2IRetrieval/eng/MSCOCOT2IRetrieval.py | 37 ++++++++++--------- .../eng/RenderedSST2.py | 9 +++-- 7 files changed, 83 insertions(+), 60 deletions(-) diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 342505a757..836c505360 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -103,6 +103,11 @@ "it2it", ] +MODALITIES = Literal[ + "text", + "image", +] + ANNOTATOR_TYPE = Literal[ "expert-annotated", "human-annotated", "derived", "LM-generated" ] @@ -185,7 +190,7 @@ class TaskMetadata(BaseModel): name: str description: str type: TASK_TYPE - modalities: list[Literal["text"]] + modalities: list[MODALITIES] category: TASK_CATEGORY reference: STR_URL | None # URL to documentation, e.g. published paper diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py index 2f1c72537f..236449ab93 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py @@ -28,22 +28,25 @@ class CIRRIT2TRetrieval(AbsTaskAny2AnyRetrieval): socioeconomic_status="medium", annotations_creators="derived", dialect=[], - text_creation="found", + modalities=["text", "image"], + sample_creation="found", bibtex_citation="""@inproceedings{liu2021image, - title={Image retrieval on real-life images with pre-trained vision-and-language models}, - author={Liu, Zheyuan and Rodriguez-Opazo, Cristian and Teney, Damien and Gould, Stephen}, - booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision}, - pages={2125--2134}, - year={2021} -}""", - n_samples={"test": 1172}, - avg_character_length={ - "test": { - "average_document_length": 30.94235294117647, - "average_query_length": 131.56569965870307, - "num_documents": 9350, - "num_queries": 1172, - "average_relevant_docs_per_query": 1.0, - } + title={Image retrieval on real-life images with pre-trained vision-and-language models}, + author={Liu, Zheyuan and Rodriguez-Opazo, Cristian and Teney, Damien and Gould, Stephen}, + booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision}, + pages={2125--2134}, + year={2021} + }""", + descriptive_stats={ + "n_samples": {"test": 1172}, + "avg_character_length": { + "test": { + "average_document_length": 30.94235294117647, + "average_query_length": 131.56569965870307, + "num_documents": 9350, + "num_queries": 1172, + "average_relevant_docs_per_query": 1.0, + } + }, }, ) diff --git a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py index 37f66e016d..542d84b205 100644 --- a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py @@ -29,8 +29,11 @@ class TinyImageNet(AbsTaskImageClustering): socioeconomic_status="mixed", annotations_creators="derived", dialect=[], - text_creation="found", + modalities=["image"], + sample_creation="found", bibtex_citation="""d""", - n_samples={"valid": 10000}, - avg_character_length={"valid": 431.4}, + descriptive_stats={ + "n_samples": {"valid": 10000}, + "avg_character_length": {"valid": 431.4}, + }, ) diff --git a/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py b/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py index 782e471c62..5dc19270b7 100644 --- a/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py +++ b/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py @@ -28,23 +28,26 @@ class MSCOCOI2TRetrieval(AbsTaskI2TRetrieval): socioeconomic_status="medium", annotations_creators="derived", dialect=[], - text_creation="found", + modalities=["text", "image"], + sample_creation="found", bibtex_citation="""@inproceedings{lin2014microsoft, - title={Microsoft coco: Common objects in context}, - author={Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\'a}r, Piotr and Zitnick, C Lawrence}, - booktitle={Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part V 13}, - pages={740--755}, - year={2014}, - organization={Springer} -}""", - n_samples={"test": 1172}, - avg_character_length={ - "test": { - "average_document_length": 30.94235294117647, - "average_query_length": 131.56569965870307, - "num_documents": 9350, - "num_queries": 1172, - "average_relevant_docs_per_query": 1.0, - } + title={Microsoft coco: Common objects in context}, + author={Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\'a}r, Piotr and Zitnick, C Lawrence}, + booktitle={Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part V 13}, + pages={740--755}, + year={2014}, + organization={Springer} + }""", + descriptive_stats={ + "n_samples": {"test": 1172}, + "avg_character_length": { + "test": { + "average_document_length": 30.94235294117647, + "average_query_length": 131.56569965870307, + "num_documents": 9350, + "num_queries": 1172, + "average_relevant_docs_per_query": 1.0, + } + }, }, ) diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py index 662dd92e3c..39c6c1f45a 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -30,8 +30,11 @@ class OxfordFlowersClassification(AbsTaskImageClassification): socioeconomic_status="mixed", annotations_creators="derived", dialect=[], - text_creation="found", + modalities=["image"], + sample_creation="found", bibtex_citation="""d""", - n_samples={"test": 400000}, - avg_character_length={"test": 431.4}, + descriptive_stats={ + "n_samples": {"test": 400000}, + "avg_character_length": {"test": 431.4}, + }, ) diff --git a/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py b/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py index fd75d30f72..280ee91920 100644 --- a/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py +++ b/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py @@ -28,23 +28,26 @@ class MSCOCOT2IRetrieval(AbsTaskT2IRetrieval): socioeconomic_status="medium", annotations_creators="derived", dialect=[], - text_creation="found", + modalities=["text", "image"], + sample_creation="found", bibtex_citation="""@inproceedings{lin2014microsoft, - title={Microsoft coco: Common objects in context}, - author={Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\'a}r, Piotr and Zitnick, C Lawrence}, - booktitle={Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part V 13}, - pages={740--755}, - year={2014}, - organization={Springer} -}""", - n_samples={"test": 1172}, - avg_character_length={ - "test": { - "average_document_length": 30.94235294117647, - "average_query_length": 131.56569965870307, - "num_documents": 9350, - "num_queries": 1172, - "average_relevant_docs_per_query": 1.0, - } + title={Microsoft coco: Common objects in context}, + author={Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\'a}r, Piotr and Zitnick, C Lawrence}, + booktitle={Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part V 13}, + pages={740--755}, + year={2014}, + organization={Springer} + }""", + descriptive_stats={ + "n_samples": {"test": 1172}, + "avg_character_length": { + "test": { + "average_document_length": 30.94235294117647, + "average_query_length": 131.56569965870307, + "num_documents": 9350, + "num_queries": 1172, + "average_relevant_docs_per_query": 1.0, + } + }, }, ) diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py index c4540bf542..1cac41ad1b 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py @@ -27,10 +27,13 @@ class RenderedSST2(AbsTaskZeroshotClassification): socioeconomic_status="mixed", annotations_creators="human-annotated", dialect=[], - text_creation="created", + modalities=["text", "image"], + sample_creation="created", bibtex_citation="""d""", - n_samples={"test": 1820}, - avg_character_length={"test": 10.0}, + descriptive_stats={ + "n_samples": {"test": 1820}, + "avg_character_length": {"test": 10.0}, + }, ) # Override default column names in the subclass From b8561b8d74a773571a60fecb51e67a776e923507 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Fri, 19 Jul 2024 13:22:09 +0300 Subject: [PATCH 016/154] [MIEB] Add image text pair classification and tests (#1099) * add ImageTextPairClassification abstask and evaluator * dataset transform into sequence of images for each sample * fix processing logic; list of list images compatability * lint and docstrings * make lint * fix failing tests in TaskMetadata * add tests for mieb * skip gated repo --------- Co-authored-by: gowitheflow-1998 --- .../AbsTaskImageTextPairClassification.py | 69 ++++++++++++ mteb/abstasks/TaskMetadata.py | 2 + mteb/abstasks/__init__.py | 1 + .../ImageTextPairClassificationEvaluator.py | 103 ++++++++++++++++++ mteb/evaluation/evaluators/__init__.py | 1 + .../ImageTextPairClassification/Winoground.py | 51 +++++++++ .../ImageTextPairClassification/__init__.py | 3 + mteb/tasks/Image/__init__.py | 1 + mteb/tasks/__init__.py | 1 + .../Winoground.json | 21 ++++ .../Winoground.json | 21 ++++ .../Winoground.json | 21 ++++ tests/test_benchmark/mock_models.py | 17 +++ tests/test_benchmark/task_grid.py | 36 +++++- ...est_benchmark_integration_with_datasets.py | 2 +- tests/test_tasks/test_mieb_datasets.py | 25 +++++ 16 files changed, 373 insertions(+), 2 deletions(-) create mode 100644 mteb/abstasks/Image/AbsTaskImageTextPairClassification.py create mode 100644 mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py create mode 100644 mteb/tasks/Image/ImageTextPairClassification/Winoground.py create mode 100644 mteb/tasks/Image/ImageTextPairClassification/__init__.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Winoground.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Winoground.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Winoground.json create mode 100644 tests/test_tasks/test_mieb_datasets.py diff --git a/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py new file mode 100644 index 0000000000..b86114df91 --- /dev/null +++ b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import logging +from typing import Any, List, Union + +from datasets import Dataset +from tqdm import tqdm + +from mteb.abstasks import AbsTask +from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from mteb.evaluation.evaluators import ImageTextPairClassificationEvaluator +from mteb.load_results.mteb_results import ScoresDict + +logger = logging.getLogger(__name__) + + +class AbsTaskImageTextPairClassification(AbsTask): + """Abstract class for Image Text Pair Classification tasks, + e.g. Compositionality evaluation. + The similarity is computed between pairs and the results are ranked. + Note that the number of images and the number of captions can be different. + + self.load_data() must generate a huggingface dataset with a split matching self.metadata_dict["eval_splits"], and assign it to self.dataset. It must contain the following columns: + images: List[List[Image.Image]] + captions: List[List[str]] + """ + + # it can be ["image_0", "image_1"]; ["text_0", "text_1"] for datasets like WinoGround + images_column_names: Union[str, List[str]] = "image" + texts_column_names: Union[str, List[str]] = "caption" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def _preprocess_column( + self, dataset: Dataset, column_names: Union[str, List[str]] + ) -> List[List[Any]]: + """Group examples from the columns into a list of examples.""" + if isinstance(column_names, str): + return dataset[column_names] + + return [ + [example[col] for col in column_names] + for example in tqdm(dataset, desc=f"Processing columns {column_names}") + ] + + def _add_main_score(self, scores) -> None: + scores["main_score"] = scores[self.metadata.main_score] + + def _evaluate_subset( + self, + model: Encoder | EncoderWithQueryCorpusEncode, + dataset: Dataset, + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ) -> ScoresDict: + images = self._preprocess_column(dataset, self.images_column_names) + texts = self._preprocess_column(dataset, self.texts_column_names) + + evaluator = ImageTextPairClassificationEvaluator( + images, + texts, + task_name=self.metadata.name, + **kwargs, + ) + scores = evaluator(model, encode_kwargs=encode_kwargs) + self._add_main_score(scores) + return scores diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 836c505360..f0a3c2173d 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -39,6 +39,7 @@ "Reasoning as Retrieval", "Rendered Texts Understanding", "Image Text Retrieval", + "Caption Pairing", ] TASK_DOMAIN = Literal[ @@ -88,6 +89,7 @@ "InstructionRetrieval", "Speed", "ZeroShotClassification", + "ImageTextPairClassification", ] TASK_CATEGORY = Literal[ diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index bd2c98c22c..01366cf3f5 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -17,6 +17,7 @@ from .Image.AbsTaskI2TRetrieval import * from .Image.AbsTaskImageClassification import * from .Image.AbsTaskImageClustering import * +from .Image.AbsTaskImageTextPairClassification import * from .Image.AbsTaskT2IRetrieval import * from .Image.AbsTaskZeroshotClassification import * from .MultilingualTask import * diff --git a/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py new file mode 100644 index 0000000000..500aaf5f74 --- /dev/null +++ b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import itertools +import logging +from typing import Any, List + +import torch +import torch.nn.functional as F +from PIL import Image + +from mteb.encoder_interface import Encoder, EncoderWithSimilarity +from mteb.evaluation.evaluators.Evaluator import Evaluator + +logger = logging.getLogger(__name__) + + +class ImageTextPairClassificationEvaluator(Evaluator): + """Evaluate a model based on the similarity of the embeddings by calculating the accuracy of + identifying similar and dissimilar image caption pairs. + The goal is to find the correct image for each caption and the correct caption for each image. + This is done by computing the similarities between each image and each caption. + The results are written in a CSV. If a CSV already exists, then values are appended. + The labels need to be 0 for dissimilar pairs and 1 for similar pairs. + + Args: + images: Each row is a list of images. + texts: Each row is a list of captions. + batch_size: Batch size used to compute embeddings + """ + + def __init__( + self, + images: List[List[Image.Image]], + texts: List[List[str]], + task_name: str | None = None, + limit: int | None = None, + **kwargs, + ): + super().__init__(**kwargs) + if limit: + images = images[:limit] + texts = texts[:limit] + self.images = images + self.texts = texts + self.task_name = task_name + + assert len(self.images) == len(self.texts) + + def __call__( + self, + model: Encoder | EncoderWithSimilarity, + encode_kwargs: dict[str, Any] = {}, + ): + if "batch_size" not in encode_kwargs: + encode_kwargs["batch_size"] = 64 + + num_samples = len(self.images) + num_images_per_sample = len(self.images[0]) + num_texts_per_sample = len(self.texts[0]) + + images = list(itertools.chain.from_iterable(self.images)) + texts = list(itertools.chain.from_iterable(self.texts)) + + image_embeddings = F.normalize( + model.get_image_embeddings(images, batch_size=encode_kwargs["batch_size"]), + dim=-1, + ).view(num_samples, num_images_per_sample, -1) + text_embeddings = F.normalize( + model.get_text_embeddings(texts, batch_size=encode_kwargs["batch_size"]), + dim=-1, + ).view(num_samples, num_texts_per_sample, -1) + img_ground_truths = torch.arange(num_images_per_sample) + caption_ground_truths = torch.arange(num_texts_per_sample) + + image_score = [] + text_score = [] + score = [] + + for i in range(num_samples): + images_emb = image_embeddings[i] + texts_emb = text_embeddings[i] + scores = ( + images_emb @ texts_emb.t() + ) # shape = (num_images_per_sample x num_texts_per_sample) + + image_closest_text = scores.argmax(dim=1) # shape = (num_images_per_sample) + text_closest_image = scores.argmax(dim=0) # shape = (num_texts_per_sample) + pred_text_is_correct = ( + (image_closest_text == img_ground_truths).all().item() + ) + pred_image_is_correct = ( + (text_closest_image == caption_ground_truths).all().item() + ) + all_correct = pred_text_is_correct and pred_image_is_correct + image_score.append(pred_image_is_correct) + text_score.append(pred_text_is_correct) + score.append(all_correct) + + metrics = {} + metrics["image_acc"] = torch.Tensor(image_score).float().mean().item() + metrics["text_acc"] = torch.Tensor(text_score).float().mean().item() + metrics["accuracy"] = torch.Tensor(score).float().mean().item() + return metrics diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py index 10892fd6c5..8056c6e495 100644 --- a/mteb/evaluation/evaluators/__init__.py +++ b/mteb/evaluation/evaluators/__init__.py @@ -7,6 +7,7 @@ from .Image.ClassificationEvaluator import * from .Image.ClusteringEvaluator import * from .Image.I2TRetrievalEvaluator import * +from .Image.ImageTextPairClassificationEvaluator import * from .Image.T2IRetrievalEvaluator import * from .Image.ZeroshotClassificationEvaluator import * from .PairClassificationEvaluator import * diff --git a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py new file mode 100644 index 0000000000..22336f155e --- /dev/null +++ b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskImageTextPairClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class Winoground(AbsTaskImageTextPairClassification): + images_column_names = ["image_0", "image_1"] + texts_column_names = ["caption_0", "caption_1"] + + metadata = TaskMetadata( + name="Winoground", + description="Compositionality Evaluation of images to their captions.", + reference="https://openaccess.thecvf.com/content/CVPR2022/html/Thrush_Winoground_Probing_Vision_and_Language_Models_for_Visio-Linguistic_Compositionality_CVPR_2022_paper", + dataset={ + "path": "facebook/winoground", + "revision": "521ec2ba6f9a5d7380f7cca5a7b44aea5c1d677c", + "trust_remote_code": True, + }, + type="ImageTextPairClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2022-01-01", + "2022-04-07", + ), # Estimated range for the collection of data + form=["written"], + domains=["Social"], # Getty Images. Could be constructed? + task_subtypes=["Caption Pairing"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="expert-annotated", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@misc{thrush2022winogroundprobingvisionlanguage, + title={Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality}, + author={Tristan Thrush and Ryan Jiang and Max Bartolo and Amanpreet Singh and Adina Williams and Douwe Kiela and Candace Ross}, + year={2022}, + eprint={2204.03162}, + archivePrefix={arXiv}, + primaryClass={cs.CV}, + url={https://arxiv.org/abs/2204.03162}, + }""", + descriptive_stats={ + "n_samples": {"test": 400}, + "avg_character_length": {"test": 431.4}, + }, + ) diff --git a/mteb/tasks/Image/ImageTextPairClassification/__init__.py b/mteb/tasks/Image/ImageTextPairClassification/__init__.py new file mode 100644 index 0000000000..c6ea0b557b --- /dev/null +++ b/mteb/tasks/Image/ImageTextPairClassification/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .Winoground import * diff --git a/mteb/tasks/Image/__init__.py b/mteb/tasks/Image/__init__.py index d9869877ed..74a4980dae 100644 --- a/mteb/tasks/Image/__init__.py +++ b/mteb/tasks/Image/__init__.py @@ -2,5 +2,6 @@ from .Clustering import * from .I2TRetrieval import * from .ImageClassification import * +from .ImageTextPairClassification import * from .T2IRetrieval import * from .ZeroshotClassification import * diff --git a/mteb/tasks/__init__.py b/mteb/tasks/__init__.py index 2726181eb2..e6593154d6 100644 --- a/mteb/tasks/__init__.py +++ b/mteb/tasks/__init__.py @@ -5,6 +5,7 @@ from .Clustering import * from .Image.I2TRetrieval import * from .Image.ImageClassification import * +from .Image.ImageTextPairClassification import * from .Image.T2IRetrieval import * from .Image.ZeroshotClassification import * from .InstructionRetrieval import * diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Winoground.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Winoground.json new file mode 100644 index 0000000000..d077401528 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Winoground.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "521ec2ba6f9a5d7380f7cca5a7b44aea5c1d677c", + "evaluation_time": 48.52479600906372, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "accuracy": 0.07000000029802322, + "hf_subset": "default", + "image_acc": 0.10249999910593033, + "languages": [ + "eng-Latn" + ], + "main_score": 0.07000000029802322, + "text_acc": 0.25 + } + ] + }, + "task_name": "Winoground" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Winoground.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Winoground.json new file mode 100644 index 0000000000..13d93f9d05 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Winoground.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "521ec2ba6f9a5d7380f7cca5a7b44aea5c1d677c", + "evaluation_time": 46.73370671272278, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "accuracy": 0.08250000327825546, + "hf_subset": "default", + "image_acc": 0.10249999910593033, + "languages": [ + "eng-Latn" + ], + "main_score": 0.08250000327825546, + "text_acc": 0.3075000047683716 + } + ] + }, + "task_name": "Winoground" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Winoground.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Winoground.json new file mode 100644 index 0000000000..bc891ebea0 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Winoground.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "521ec2ba6f9a5d7380f7cca5a7b44aea5c1d677c", + "evaluation_time": 54.006736040115356, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "accuracy": 0.07999999821186066, + "hf_subset": "default", + "image_acc": 0.10999999940395355, + "languages": [ + "eng-Latn" + ], + "main_score": 0.07999999821186066, + "text_acc": 0.2849999964237213 + } + ] + }, + "task_name": "Winoground" +} \ No newline at end of file diff --git a/tests/test_benchmark/mock_models.py b/tests/test_benchmark/mock_models.py index 381b3710d4..0f2926825b 100644 --- a/tests/test_benchmark/mock_models.py +++ b/tests/test_benchmark/mock_models.py @@ -30,3 +30,20 @@ def __init__(self): def encode(self, sentences, prompt_name: str | None = None, **kwargs): return torch.randn(len(sentences), 10, dtype=torch.bfloat16) + + +class MockCLIPEncoder: + def __init__(self): + pass + + def get_text_embeddings(self, texts, **kwargs): + return torch.randn(len(texts), 10) + + def get_image_embeddings(self, images, **kwargs): + return torch.randn(len(images), 10) + + def get_fused_embeddings(self, texts, images, **kwargs): + return torch.randn(len(images), 10) + + def calculate_probs(self, text_embeddings, image_embeddings): + return torch.randn(image_embeddings.shape[0], text_embeddings.shape[1]) diff --git a/tests/test_benchmark/task_grid.py b/tests/test_benchmark/task_grid.py index 2be1a089e3..45e8a6025f 100644 --- a/tests/test_benchmark/task_grid.py +++ b/tests/test_benchmark/task_grid.py @@ -10,6 +10,9 @@ from mteb.tasks.Clustering.eng.TwentyNewsgroupsClustering import ( TwentyNewsgroupsClusteringFast, ) +from mteb.tasks.Image.Clustering import TinyImageNet +from mteb.tasks.Image.ImageTextPairClassification import Winoground +from mteb.tasks.Image.ZeroshotClassification import RenderedSST2 from .mock_tasks import ( MockBitextMiningTask, @@ -53,8 +56,39 @@ ] +def dataset_transform(self): + for split in self.metadata.eval_splits: + self.dataset[split] = self.dataset[split].select([0, 1]) + + +tiny_imagenet = TinyImageNet() +renderedSST2 = RenderedSST2() +winoground = Winoground() + +## method override to speed up tests +tiny_imagenet.dataset_transform = dataset_transform.__get__(tiny_imagenet) +renderedSST2.dataset_transform = dataset_transform.__get__(renderedSST2) +winoground.dataset_transform = dataset_transform.__get__(winoground) + + +MIEB_TASK_TEST_GRID = [ + tiny_imagenet, # image clustering + # winoground, # pair classification. Gated + renderedSST2, # zero shot classification + # The following takes a long time. Consider creating a mock class. + # "CIRRIT2TRetrieval", # it2i retrieval + # "MSCOCOI2TRetrieval", # i2t retrieval + # "MSCOCOT2IRetrieval", # t2i retrieval + # oxford_flowers, # image classification +] + +MIEB_TASK_TEST_GRID_AS_STRING = [ + t.metadata.name if isinstance(t, AbsTask) else t for t in MIEB_TASK_TEST_GRID +] + + # Mock tasks for testing - intended to be faster and avoid downloading data leading to false positive potential failures in CI -# Not all tasks are implemnted as Mock tasks yet +# Not all tasks are implemented as Mock tasks yet MOCK_TASK_TEST_GRID = [ MockBitextMiningTask(), MockClassificationTask(), diff --git a/tests/test_benchmark/test_benchmark_integration_with_datasets.py b/tests/test_benchmark/test_benchmark_integration_with_datasets.py index c4fb0951aa..369ad0acda 100644 --- a/tests/test_benchmark/test_benchmark_integration_with_datasets.py +++ b/tests/test_benchmark/test_benchmark_integration_with_datasets.py @@ -1,4 +1,4 @@ -"""test mteb.MTEB's integration with SentenceTransformers""" +"""test mteb.MTEB's integration with datasets""" from __future__ import annotations diff --git a/tests/test_tasks/test_mieb_datasets.py b/tests/test_tasks/test_mieb_datasets.py new file mode 100644 index 0000000000..2eca4de4d3 --- /dev/null +++ b/tests/test_tasks/test_mieb_datasets.py @@ -0,0 +1,25 @@ +"""test mteb.MTEB's integration with datasets""" + +from __future__ import annotations + +import logging +from typing import Union + +import pytest + +import mteb +from mteb import MTEB +from mteb.abstasks import AbsTask + +from ..test_benchmark.mock_models import MockCLIPEncoder +from ..test_benchmark.task_grid import MIEB_TASK_TEST_GRID + +logging.basicConfig(level=logging.INFO) + + +@pytest.mark.parametrize("task", MIEB_TASK_TEST_GRID) +@pytest.mark.parametrize("model", [MockCLIPEncoder()]) +def test_benchmark_sentence_transformer(task: Union[str, AbsTask], model: mteb.Encoder): + """Test that a task can be fetched and run""" + eval = MTEB(tasks=[task]) + eval.run(model, output_folder="tests/results", overwrite_results=True) From 3f888fa00e40ebc83f4d578586e9067ba584c0e9 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sat, 20 Jul 2024 17:24:33 +0300 Subject: [PATCH 017/154] [MIEB] Add image classification and zero shot classification tasks (#1101) * fix task metadata * use overrideable column names * add CIFAR datasets * add caltech101 dataset * add FGVC aircraft dataset * add food 101 dataset * add OxfordPets dataset * remove comments * correct cifar100 path * update cifar100 classification results * cifar zero shot results * add caltech101 zero shot * matching CLIP paper implementation * add aircraft and food zero shot * add oxford pets zero shot --- .../Image/AbsTaskImageClassification.py | 23 ++-- mteb/abstasks/TaskMetadata.py | 3 + .../Image/ImageClassification/__init__.py | 5 + .../Image/ImageClassification/eng/CIFAR.py | 88 +++++++++++++++ .../eng/Caltech101Classification.py | 50 +++++++++ .../eng/FGVCAircraftClassification.py | 49 +++++++++ .../eng/Food101Classification.py | 45 ++++++++ .../eng/OxfordFlowersClassification.py | 1 - .../eng/OxfordPetsClassification.py | 48 +++++++++ .../T2IRetrieval/eng/MSCOCOT2IRetrieval.py | 1 - .../Image/ZeroshotClassification/__init__.py | 5 + .../Image/ZeroshotClassification/eng/CIFAR.py | 100 ++++++++++++++++++ .../ZeroshotClassification/eng/Caltech101.py | 56 ++++++++++ .../eng/FGVCAircraft.py | 55 ++++++++++ .../ZeroshotClassification/eng/Food101.py | 53 ++++++++++ .../ZeroshotClassification/eng/OxfordPets.py | 54 ++++++++++ .../eng/RenderedSST2.py | 1 - .../CIFAR10.json | 48 +++++++++ .../CIFAR100.json | 48 +++++++++ .../CIFAR100ZeroShot.json | 19 ++++ .../CIFAR10ZeroShot.json | 19 ++++ .../Caltech101.json | 48 +++++++++ .../Caltech101ZeroShot.json | 19 ++++ .../FGVCAircraft.json | 73 +++++++++++++ .../FGVCAircraftZeroShot.json | 19 ++++ .../Food101Classification.json | 48 +++++++++ .../Food101ZeroShot.json | 19 ++++ .../OxfordPets.json | 73 +++++++++++++ .../OxfordPetsZeroShot.json | 19 ++++ 29 files changed, 1076 insertions(+), 13 deletions(-) create mode 100644 mteb/tasks/Image/ImageClassification/eng/CIFAR.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/Food101Classification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/Food101.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100ZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10ZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101ZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraft.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraftZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101Classification.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101ZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPets.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPetsZeroShot.json diff --git a/mteb/abstasks/Image/AbsTaskImageClassification.py b/mteb/abstasks/Image/AbsTaskImageClassification.py index 52ac7d81f9..9b69e67f9e 100644 --- a/mteb/abstasks/Image/AbsTaskImageClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageClassification.py @@ -29,6 +29,9 @@ class AbsTaskImageClassification(AbsTask): label: int """ + image_column_name: str = "image" + label_column_name: str = "label" + def __init__( self, method: str = "logReg", @@ -44,12 +47,12 @@ def __init__( self.n_experiments: int = ( # type: ignore n_experiments if n_experiments is not None - else self.metadata_dict.get("n_experiments", 10) + else self.metadata_dict.get("n_experiments", 5) ) self.samples_per_label: int = ( # type: ignore samples_per_label if samples_per_label is not None - else self.metadata_dict.get("samples_per_label", 8) + else self.metadata_dict.get("samples_per_label", 16) ) # kNN parameters @@ -126,8 +129,8 @@ def _evaluate_subset( ) # Bootstrap `self.samples_per_label` samples per label for each split X_sampled, y_sampled, idxs = self._undersample_data( - train_split["image"], # type: ignore - train_split["label"], # type: ignore + train_split[self.image_column_name], # type: ignore + train_split[self.label_column_name], # type: ignore self.samples_per_label, idxs, ) @@ -136,8 +139,8 @@ def _evaluate_subset( evaluator = ImagekNNClassificationEvaluator( X_sampled, y_sampled, - eval_split["image"], # type: ignore - eval_split["label"], # type: ignore + eval_split[self.image_column_name], # type: ignore + eval_split[self.label_column_name], # type: ignore task_name=self.metadata.name, encode_kwargs=encode_kwargs, **params, @@ -146,8 +149,8 @@ def _evaluate_subset( evaluator = ImagekNNClassificationEvaluatorPytorch( X_sampled, y_sampled, - eval_split["image"], # type: ignore - eval_split["label"], # type: ignore + eval_split[self.image_column_name], # type: ignore + eval_split[self.label_column_name], # type: ignore task_name=self.metadata.name, encode_kwargs=encode_kwargs, **params, @@ -156,8 +159,8 @@ def _evaluate_subset( evaluator = ImagelogRegClassificationEvaluator( X_sampled, y_sampled, - eval_split["image"], # type: ignore - eval_split["label"], # type: ignore + eval_split[self.image_column_name], # type: ignore + eval_split[self.label_column_name], # type: ignore task_name=self.metadata.name, encode_kwargs=encode_kwargs, **params, diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index f0a3c2173d..845a5e514a 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -39,6 +39,8 @@ "Reasoning as Retrieval", "Rendered Texts Understanding", "Image Text Retrieval", + "Object recognition", + "Scene recognition", "Caption Pairing", ] @@ -56,6 +58,7 @@ "Poetry", "Religious", "Reviews", + "Scene", "Social", "Spoken", "Subtitles", diff --git a/mteb/tasks/Image/ImageClassification/__init__.py b/mteb/tasks/Image/ImageClassification/__init__.py index 72a89837fc..6eed085634 100644 --- a/mteb/tasks/Image/ImageClassification/__init__.py +++ b/mteb/tasks/Image/ImageClassification/__init__.py @@ -1,3 +1,8 @@ from __future__ import annotations +from .eng.Caltech101Classification import * +from .eng.CIFAR import * +from .eng.FGVCAircraftClassification import * +from .eng.Food101Classification import * from .eng.OxfordFlowersClassification import * +from .eng.OxfordPetsClassification import * diff --git a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py new file mode 100644 index 0000000000..7a69afa137 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class CIFAR10Classification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="CIFAR10", + description="Classifying images from 10 classes.", + reference="https://huggingface.co/datasets/uoft-cs/cifar10", + dataset={ + "path": "uoft-cs/cifar10", + "revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + }, + type="Classification", + category="s2s", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2008-01-01", + "2009-01-01", + ), # Estimated range for the collection of reviews + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation=""" @TECHREPORT{Krizhevsky09learningmultiple, + author = {Alex Krizhevsky}, + title = {Learning multiple layers of features from tiny images}, + institution = {}, + year = {2009} + } + """, + descriptive_stats={ + "n_samples": {"test": 10000}, + "avg_character_length": {"test": 431.4}, + }, + ) + image_column_name: str = "img" + + +class CIFAR100Classification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="CIFAR100", + description="Classifying images from 100 classes.", + reference="https://huggingface.co/datasets/uoft-cs/cifar100", + dataset={ + "path": "uoft-cs/cifar100", + "revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + }, + type="Classification", + category="s2s", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2008-01-01", + "2009-01-01", + ), # Estimated range for the collection of reviews + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation=""" @TECHREPORT{Krizhevsky09learningmultiple, + author = {Alex Krizhevsky}, + title = {Learning multiple layers of features from tiny images}, + institution = {}, + year = {2009} + } + """, + descriptive_stats={ + "n_samples": {"test": 10000}, + "avg_character_length": {"test": 431.4}, + }, + ) + image_column_name: str = "img" + label_column_name: str = "fine_label" diff --git a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py new file mode 100644 index 0000000000..40a23b13e5 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class Caltech101Classification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="Caltech101", + description="Classifying images of 101 widely varied objects.", + reference="https://ieeexplore.ieee.org/document/1384978", + dataset={ + "path": "HuggingFaceM4/Caltech-101", + "name": "with_background_category", + "revision": "851374102055782c84f89b1b4e9d128a6568847b", + }, + type="Classification", + category="s2s", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2003-01-01", + "2004-01-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@INPROCEEDINGS{1384978, + author={Li Fei-Fei and Fergus, R. and Perona, P.}, + booktitle={2004 Conference on Computer Vision and Pattern Recognition Workshop}, + title={Learning Generative Visual Models from Few Training Examples: An Incremental Bayesian Approach Tested on 101 Object Categories}, + year={2004}, + volume={}, + number={}, + pages={178-178}, + keywords={Bayesian methods;Testing;Humans;Maximum likelihood estimation;Assembly;Shape;Machine vision;Image recognition;Parameter estimation;Image databases}, + doi={10.1109/CVPR.2004.383}} + """, + descriptive_stats={ + "n_samples": {"test": 6084}, + "avg_character_length": {"test": 431.4}, + }, + ) diff --git a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py new file mode 100644 index 0000000000..e8a469297a --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class FGVCAircraftClassification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="FGVCAircraft", + description="Classifying aircraft images from 41 manufacturers and 102 variants.", + reference="https://arxiv.org/abs/1306.5151", + dataset={ + "path": "HuggingFaceM4/FGVC-Aircraft", + "revision": "91860adfc9a09aabca5cddb5247442109b38e213", + }, + type="Classification", + category="s2s", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2009-01-01", + "2010-01-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@misc{maji2013finegrainedvisualclassificationaircraft, + title={Fine-Grained Visual Classification of Aircraft}, + author={Subhransu Maji and Esa Rahtu and Juho Kannala and Matthew Blaschko and Andrea Vedaldi}, + year={2013}, + eprint={1306.5151}, + archivePrefix={arXiv}, + primaryClass={cs.CV}, + url={https://arxiv.org/abs/1306.5151}, + } + """, + descriptive_stats={ + "n_samples": {"test": 3333}, + "avg_character_length": {"test": 431.4}, + }, + ) + label_column_name: str = "variant" ## could be family, manufacturer, or variant. Variant has the higher number of classes. diff --git a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py new file mode 100644 index 0000000000..466f78907b --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class Food101Classification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="Food101Classification", + description="Classifying food.", + reference="https://huggingface.co/datasets/ethz/food101", + dataset={ + "path": "ethz/food101", + "revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", + }, + type="Classification", + category="s2s", + eval_splits=["validation"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2013-01-01", + "2014-01-01", + ), # Estimated range for the collection of reviews + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation=""" @inproceedings{bossard14, + title = {Food-101 -- Mining Discriminative Components with Random Forests}, + author = {Bossard, Lukas and Guillaumin, Matthieu and Van Gool, Luc}, + booktitle = {European Conference on Computer Vision}, + year = {2014} + } + """, + descriptive_stats={ + "n_samples": {"validation": 25300}, + "avg_character_length": {"validation": 431.4}, + }, + ) diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py index 39c6c1f45a..5c19067710 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -23,7 +23,6 @@ class OxfordFlowersClassification(AbsTaskImageClassification): "2012-01-01", "2015-12-31", ), # Estimated range for the collection of reviews - form=["written"], domains=["Reviews"], task_subtypes=["Sentiment/Hate speech"], license="Not specified", diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py new file mode 100644 index 0000000000..769f40cce9 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class OxfordPetsClassification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="OxfordPets", + description="Classifying animal images.", + reference="https://arxiv.org/abs/1306.5151", + dataset={ + "path": "isaacchung/OxfordPets", + "revision": "557b480fae8d69247be74d9503b378a09425096f", + }, + type="Classification", + category="s2s", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2009-01-01", + "2010-01-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@misc{maji2013finegrainedvisualclassificationaircraft, + title={Fine-Grained Visual Classification of Aircraft}, + author={Subhransu Maji and Esa Rahtu and Juho Kannala and Matthew Blaschko and Andrea Vedaldi}, + year={2013}, + eprint={1306.5151}, + archivePrefix={arXiv}, + primaryClass={cs.CV}, + url={https://arxiv.org/abs/1306.5151}, + } + """, + descriptive_stats={ + "n_samples": {"test": 3669}, + "avg_character_length": {"test": 431.4}, + }, + ) diff --git a/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py b/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py index 280ee91920..9fd20f3450 100644 --- a/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py +++ b/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py @@ -21,7 +21,6 @@ class MSCOCOT2IRetrieval(AbsTaskT2IRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2018-01-01", "2018-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", diff --git a/mteb/tasks/Image/ZeroshotClassification/__init__.py b/mteb/tasks/Image/ZeroshotClassification/__init__.py index ff5f0b1a87..561f762e46 100644 --- a/mteb/tasks/Image/ZeroshotClassification/__init__.py +++ b/mteb/tasks/Image/ZeroshotClassification/__init__.py @@ -1,3 +1,8 @@ from __future__ import annotations +from .eng.Caltech101 import * +from .eng.CIFAR import * +from .eng.FGVCAircraft import * +from .eng.Food101 import * +from .eng.OxfordPets import * from .eng.RenderedSST2 import * diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py b/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py new file mode 100644 index 0000000000..aaeda85d3d --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class CIFAR10ZeroShotClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="CIFAR10ZeroShot", + description="Classifying images from 10 classes.", + reference="https://huggingface.co/datasets/uoft-cs/cifar10", + dataset={ + "path": "uoft-cs/cifar10", + "revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2008-01-01", + "2009-01-01", + ), # Estimated range for the collection of reviews + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation=""" @TECHREPORT{Krizhevsky09learningmultiple, + author = {Alex Krizhevsky}, + title = {Learning multiple layers of features from tiny images}, + institution = {}, + year = {2009} + } + """, + descriptive_stats={ + "n_samples": {"test": 10000}, + "avg_character_length": {"test": 431.4}, + }, + ) + image_column_name: str = "img" + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of a {name}." + for name in self.dataset["test"].features[self.label_column_name].names + ] + + +class CIFAR100ZeroShotClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="CIFAR100ZeroShot", + description="Classifying images from 100 classes.", + reference="https://huggingface.co/datasets/uoft-cs/cifar100", + dataset={ + "path": "uoft-cs/cifar100", + "revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2008-01-01", + "2009-01-01", + ), # Estimated range for the collection of reviews + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation=""" @TECHREPORT{Krizhevsky09learningmultiple, + author = {Alex Krizhevsky}, + title = {Learning multiple layers of features from tiny images}, + institution = {}, + year = {2009} + } + """, + descriptive_stats={ + "n_samples": {"test": 10000}, + "avg_character_length": {"test": 431.4}, + }, + ) + image_column_name: str = "img" + label_column_name: str = "fine_label" + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of a {name}." + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py b/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py new file mode 100644 index 0000000000..4b951a3632 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class Caltech101Classification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="Caltech101ZeroShot", + description="Classifying images of 101 widely varied objects.", + reference="https://ieeexplore.ieee.org/document/1384978", + dataset={ + "path": "HuggingFaceM4/Caltech-101", + "name": "with_background_category", + "revision": "851374102055782c84f89b1b4e9d128a6568847b", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2003-01-01", + "2004-01-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@INPROCEEDINGS{1384978, + author={Li Fei-Fei and Fergus, R. and Perona, P.}, + booktitle={2004 Conference on Computer Vision and Pattern Recognition Workshop}, + title={Learning Generative Visual Models from Few Training Examples: An Incremental Bayesian Approach Tested on 101 Object Categories}, + year={2004}, + volume={}, + number={}, + pages={178-178}, + keywords={Bayesian methods;Testing;Humans;Maximum likelihood estimation;Assembly;Shape;Machine vision;Image recognition;Parameter estimation;Image databases}, + doi={10.1109/CVPR.2004.383}} + """, + descriptive_stats={ + "n_samples": {"test": 6084}, + "avg_character_length": {"test": 431.4}, + }, + ) + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of a {name}." + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py b/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py new file mode 100644 index 0000000000..ec04c8b787 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class FGVCAircraftClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="FGVCAircraftZeroShot", + description="Classifying aircraft images from 41 manufacturers and 102 variants.", + reference="https://arxiv.org/abs/1306.5151", + dataset={ + "path": "HuggingFaceM4/FGVC-Aircraft", + "revision": "91860adfc9a09aabca5cddb5247442109b38e213", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2009-01-01", + "2010-01-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@misc{maji2013finegrainedvisualclassificationaircraft, + title={Fine-Grained Visual Classification of Aircraft}, + author={Subhransu Maji and Esa Rahtu and Juho Kannala and Matthew Blaschko and Andrea Vedaldi}, + year={2013}, + eprint={1306.5151}, + archivePrefix={arXiv}, + primaryClass={cs.CV}, + url={https://arxiv.org/abs/1306.5151}, + } + """, + descriptive_stats={ + "n_samples": {"test": 3333}, + "avg_character_length": {"test": 431.4}, + }, + ) + label_column_name: str = "variant" ## could be family, manufacturer, or variant. Variant has the higher number of classes. + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of a {name}, a type of aircraft." + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Food101.py b/mteb/tasks/Image/ZeroshotClassification/eng/Food101.py new file mode 100644 index 0000000000..78fca46f04 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Food101.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class Food101Classification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="Food101ZeroShot", + description="Classifying food.", + reference="https://huggingface.co/datasets/ethz/food101", + dataset={ + "path": "ethz/food101", + "revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["validation"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2013-01-01", + "2014-01-01", + ), # Estimated range for the collection of reviews + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation=""" @inproceedings{bossard14, + title = {Food-101 -- Mining Discriminative Components with Random Forests}, + author = {Bossard, Lukas and Guillaumin, Matthieu and Van Gool, Luc}, + booktitle = {European Conference on Computer Vision}, + year = {2014} + } + """, + descriptive_stats={ + "n_samples": {"validation": 25300}, + "avg_character_length": {"validation": 431.4}, + }, + ) + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of {name}, a type of food." + for name in self.dataset["validation"] + .features[self.label_column_name] + .names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py b/mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py new file mode 100644 index 0000000000..6600b01af7 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class OxfordPetsClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="OxfordPetsZeroShot", + description="Classifying animal images.", + reference="https://arxiv.org/abs/1306.5151", + dataset={ + "path": "isaacchung/OxfordPets", + "revision": "557b480fae8d69247be74d9503b378a09425096f", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2009-01-01", + "2010-01-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@misc{maji2013finegrainedvisualclassificationaircraft, + title={Fine-Grained Visual Classification of Aircraft}, + author={Subhransu Maji and Esa Rahtu and Juho Kannala and Matthew Blaschko and Andrea Vedaldi}, + year={2013}, + eprint={1306.5151}, + archivePrefix={arXiv}, + primaryClass={cs.CV}, + url={https://arxiv.org/abs/1306.5151}, + } + """, + descriptive_stats={ + "n_samples": {"test": 3669}, + "avg_character_length": {"test": 431.4}, + }, + ) + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of a {name}, a type of pet." + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py index 1cac41ad1b..b1206ca762 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py @@ -20,7 +20,6 @@ class RenderedSST2(AbsTaskZeroshotClassification): eval_langs=["eng-Latn"], main_score="accuracy", date=("2016-01-01", "2016-12-31"), - form=["written"], domains=["Reviews"], task_subtypes=[], license="mit", diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10.json new file mode 100644 index 0000000000..7f2341a9d9 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + "evaluation_time": 81.32431101799011, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.8985799999999999, + "f1": 0.899066397223649, + "f1_weighted": 0.899066397223649, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8985799999999999, + "scores_per_experiment": [ + { + "accuracy": 0.9015, + "f1": 0.9021178664975237, + "f1_weighted": 0.9021178664975236 + }, + { + "accuracy": 0.9052, + "f1": 0.9056178385863303, + "f1_weighted": 0.9056178385863304 + }, + { + "accuracy": 0.8925, + "f1": 0.8929011491758369, + "f1_weighted": 0.8929011491758371 + }, + { + "accuracy": 0.8964, + "f1": 0.8968836634711416, + "f1_weighted": 0.8968836634711413 + }, + { + "accuracy": 0.8973, + "f1": 0.897811468387413, + "f1_weighted": 0.8978114683874128 + } + ] + } + ] + }, + "task_name": "CIFAR10" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100.json new file mode 100644 index 0000000000..3fd76447e6 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + "evaluation_time": 112.59889626502991, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.67418, + "f1": 0.6752114591311454, + "f1_weighted": 0.6752114591311454, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.67418, + "scores_per_experiment": [ + { + "accuracy": 0.6725, + "f1": 0.6726574672039856, + "f1_weighted": 0.6726574672039854 + }, + { + "accuracy": 0.6809, + "f1": 0.6821528552320336, + "f1_weighted": 0.6821528552320336 + }, + { + "accuracy": 0.6778, + "f1": 0.6793950138107854, + "f1_weighted": 0.6793950138107854 + }, + { + "accuracy": 0.6707, + "f1": 0.6711676254605934, + "f1_weighted": 0.6711676254605933 + }, + { + "accuracy": 0.669, + "f1": 0.6706843339483296, + "f1_weighted": 0.6706843339483296 + } + ] + } + ] + }, + "task_name": "CIFAR100" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100ZeroShot.json new file mode 100644 index 0000000000..f076bff887 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + "evaluation_time": 41.24559950828552, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.6165, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6165 + } + ] + }, + "task_name": "CIFAR100ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10ZeroShot.json new file mode 100644 index 0000000000..04c094ec7e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + "evaluation_time": 37.38677930831909, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.883, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.883 + } + ] + }, + "task_name": "CIFAR10ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101.json new file mode 100644 index 0000000000..95f11f5a9e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", + "evaluation_time": 118.63952732086182, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.9175542406311636, + "f1": 0.8767202332660267, + "f1_weighted": 0.9187633243329824, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9175542406311636, + "scores_per_experiment": [ + { + "accuracy": 0.9119000657462196, + "f1": 0.8710825099573519, + "f1_weighted": 0.9133422092543803 + }, + { + "accuracy": 0.9275147928994083, + "f1": 0.8813945266330111, + "f1_weighted": 0.9291005925647753 + }, + { + "accuracy": 0.913379355687048, + "f1": 0.8738935806115447, + "f1_weighted": 0.9148291144424647 + }, + { + "accuracy": 0.9155161078238001, + "f1": 0.8709633024188326, + "f1_weighted": 0.9167341340128863 + }, + { + "accuracy": 0.9194608809993425, + "f1": 0.8862672467093923, + "f1_weighted": 0.9198105713904057 + } + ] + } + ] + }, + "task_name": "Caltech101" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101ZeroShot.json new file mode 100644 index 0000000000..d914345212 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", + "evaluation_time": 39.51401495933533, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.7899408284023669, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7899408284023669 + } + ] + }, + "task_name": "Caltech101ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraft.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraft.json new file mode 100644 index 0000000000..9243b849f8 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraft.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "91860adfc9a09aabca5cddb5247442109b38e213", + "evaluation_time": 779.7828733921051, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.3149414941494149, + "f1": 0.312693783815878, + "f1_weighted": 0.312678675983208, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.3149414941494149, + "scores_per_experiment": [ + { + "accuracy": 0.3114311431143114, + "f1": 0.31106599523075695, + "f1_weighted": 0.31096314031434213 + }, + { + "accuracy": 0.3069306930693069, + "f1": 0.3038393838854474, + "f1_weighted": 0.30395665785927795 + }, + { + "accuracy": 0.3096309630963096, + "f1": 0.3087612511688636, + "f1_weighted": 0.3086759599811729 + }, + { + "accuracy": 0.3171317131713171, + "f1": 0.31397818189887455, + "f1_weighted": 0.313804173735766 + }, + { + "accuracy": 0.32163216321632165, + "f1": 0.31683692491491705, + "f1_weighted": 0.3168587733660707 + }, + { + "accuracy": 0.3126312631263126, + "f1": 0.3114657147965915, + "f1_weighted": 0.31150939456099336 + }, + { + "accuracy": 0.32283228322832286, + "f1": 0.32236568456369397, + "f1_weighted": 0.32249530202709764 + }, + { + "accuracy": 0.3102310231023102, + "f1": 0.30481253675964476, + "f1_weighted": 0.3047149182274533 + }, + { + "accuracy": 0.3114311431143114, + "f1": 0.3087445080505303, + "f1_weighted": 0.3087629926503773 + }, + { + "accuracy": 0.32553255325532554, + "f1": 0.3250676568894592, + "f1_weighted": 0.3250454471095288 + } + ] + } + ] + }, + "task_name": "FGVCAircraft" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraftZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraftZeroShot.json new file mode 100644 index 0000000000..b78af9c07b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraftZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "91860adfc9a09aabca5cddb5247442109b38e213", + "evaluation_time": 91.58129215240479, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.18871887188718872, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.18871887188718872 + } + ] + }, + "task_name": "FGVCAircraftZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101Classification.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101Classification.json new file mode 100644 index 0000000000..d741fb416a --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101Classification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", + "evaluation_time": 1405.0914344787598, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "validation": [ + { + "accuracy": 0.7963564356435644, + "f1": 0.7966444203871001, + "f1_weighted": 0.7966444203871001, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7963564356435644, + "scores_per_experiment": [ + { + "accuracy": 0.7967524752475248, + "f1": 0.796877338175403, + "f1_weighted": 0.7968773381754028 + }, + { + "accuracy": 0.802930693069307, + "f1": 0.8032004030944163, + "f1_weighted": 0.8032004030944164 + }, + { + "accuracy": 0.7946930693069307, + "f1": 0.7948653096928756, + "f1_weighted": 0.7948653096928756 + }, + { + "accuracy": 0.7951683168316832, + "f1": 0.7952119577172861, + "f1_weighted": 0.7952119577172861 + }, + { + "accuracy": 0.7922376237623763, + "f1": 0.7930670932555197, + "f1_weighted": 0.7930670932555199 + } + ] + } + ] + }, + "task_name": "Food101Classification" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101ZeroShot.json new file mode 100644 index 0000000000..5d14882a38 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", + "evaluation_time": 264.61052894592285, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "validation": [ + { + "accuracy": 0.827089108910891, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.827089108910891 + } + ] + }, + "task_name": "Food101ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPets.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPets.json new file mode 100644 index 0000000000..e786b5adde --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPets.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "557b480fae8d69247be74d9503b378a09425096f", + "evaluation_time": 231.90859651565552, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.7442354865085855, + "f1": 0.7417313700884481, + "f1_weighted": 0.7420306227198182, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7442354865085855, + "scores_per_experiment": [ + { + "accuracy": 0.7391659852820932, + "f1": 0.7358160134713131, + "f1_weighted": 0.7358830566044606 + }, + { + "accuracy": 0.7320795857181793, + "f1": 0.7282184271306233, + "f1_weighted": 0.728373282427556 + }, + { + "accuracy": 0.7465249386753884, + "f1": 0.7452129062431734, + "f1_weighted": 0.7456542760421347 + }, + { + "accuracy": 0.7285363859362224, + "f1": 0.7259459516246249, + "f1_weighted": 0.7263860554430688 + }, + { + "accuracy": 0.7399836467702371, + "f1": 0.7391860302875606, + "f1_weighted": 0.7395108556167312 + }, + { + "accuracy": 0.7424366312346689, + "f1": 0.7384044479611193, + "f1_weighted": 0.738786323176385 + }, + { + "accuracy": 0.7519760152630145, + "f1": 0.751101111638505, + "f1_weighted": 0.751437383241543 + }, + { + "accuracy": 0.7467974925047697, + "f1": 0.7442267562085082, + "f1_weighted": 0.7445854619774289 + }, + { + "accuracy": 0.753338784409921, + "f1": 0.749093788861857, + "f1_weighted": 0.7492934083075202 + }, + { + "accuracy": 0.76151539929136, + "f1": 0.7601082674571955, + "f1_weighted": 0.7603961243613546 + } + ] + } + ] + }, + "task_name": "OxfordPets" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPetsZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPetsZeroShot.json new file mode 100644 index 0000000000..b0a0146b92 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPetsZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "557b480fae8d69247be74d9503b378a09425096f", + "evaluation_time": 36.26844310760498, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.874897792313982, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.874897792313982 + } + ] + }, + "task_name": "OxfordPetsZeroShot" +} \ No newline at end of file From 15721ff63dfee055627ac81961108af2d6c02072 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sun, 21 Jul 2024 05:13:34 +0300 Subject: [PATCH 018/154] [MIEB] Add CIFAR clustering (#1104) add CIFAR clustering --- mteb/tasks/Image/Clustering/__init__.py | 1 + mteb/tasks/Image/Clustering/eng/CIFAR.py | 88 +++++++++++++++++++ .../CIFAR100Clustering.json | 20 +++++ .../CIFAR10Clustering.json | 20 +++++ 4 files changed, 129 insertions(+) create mode 100644 mteb/tasks/Image/Clustering/eng/CIFAR.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100Clustering.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10Clustering.json diff --git a/mteb/tasks/Image/Clustering/__init__.py b/mteb/tasks/Image/Clustering/__init__.py index 8b99993d8c..fd9a71ec19 100644 --- a/mteb/tasks/Image/Clustering/__init__.py +++ b/mteb/tasks/Image/Clustering/__init__.py @@ -1,3 +1,4 @@ from __future__ import annotations +from .eng.CIFAR import * from .eng.TinyImageNet import * diff --git a/mteb/tasks/Image/Clustering/eng/CIFAR.py b/mteb/tasks/Image/Clustering/eng/CIFAR.py new file mode 100644 index 0000000000..0ba51f7213 --- /dev/null +++ b/mteb/tasks/Image/Clustering/eng/CIFAR.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClustering + + +class CIFAR10Clustering(AbsTaskImageClustering): + metadata = TaskMetadata( + name="CIFAR10Clustering", + description="Clustering images from 10 classes.", + reference="https://huggingface.co/datasets/uoft-cs/cifar10", + dataset={ + "path": "uoft-cs/cifar10", + "revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + }, + type="Clustering", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2008-01-01", + "2009-01-01", + ), # Estimated range for the collection of reviews + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation=""" @TECHREPORT{Krizhevsky09learningmultiple, + author = {Alex Krizhevsky}, + title = {Learning multiple layers of features from tiny images}, + institution = {}, + year = {2009} + } + """, + descriptive_stats={ + "n_samples": {"test": 10000}, + "avg_character_length": {"test": 431.4}, + }, + ) + image_column_name: str = "img" + + +class CIFAR100Clustering(AbsTaskImageClustering): + metadata = TaskMetadata( + name="CIFAR100Clustering", + description="Clustering images from 100 classes.", + reference="https://huggingface.co/datasets/uoft-cs/cifar100", + dataset={ + "path": "uoft-cs/cifar100", + "revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + }, + type="Clustering", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2008-01-01", + "2009-01-01", + ), # Estimated range for the collection of reviews + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation=""" @TECHREPORT{Krizhevsky09learningmultiple, + author = {Alex Krizhevsky}, + title = {Learning multiple layers of features from tiny images}, + institution = {}, + year = {2009} + } + """, + descriptive_stats={ + "n_samples": {"test": 10000}, + "avg_character_length": {"test": 431.4}, + }, + ) + image_column_name: str = "img" + label_column_name: str = "fine_label" diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100Clustering.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100Clustering.json new file mode 100644 index 0000000000..6ae63282d9 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100Clustering.json @@ -0,0 +1,20 @@ +{ + "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + "evaluation_time": 42.84400296211243, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.0149, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.0149, + "v_measure": 0.5807028840317499 + } + ] + }, + "task_name": "CIFAR100Clustering" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10Clustering.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10Clustering.json new file mode 100644 index 0000000000..2233e39698 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10Clustering.json @@ -0,0 +1,20 @@ +{ + "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + "evaluation_time": 42.9359176158905, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.137, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.137, + "v_measure": 0.7384717195071175 + } + ] + }, + "task_name": "CIFAR10Clustering" +} \ No newline at end of file From 4d99e905e1feca4b247c3e2cd820dd6720ad9dd0 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sun, 21 Jul 2024 05:18:31 +0300 Subject: [PATCH 019/154] [MIEB] Add more image classification and zero shot classification datasets (#1103) * update category to i2t * add MNIST linear probe and zero shot * add FER2013 linear probe and zero shot * add stanford cars linear probe and zero shot * add birdsnap linear probe and zero shot * add eurosat linear probe and zero shot * lint --- mteb/abstasks/TaskMetadata.py | 1 + .../Image/ImageClassification/__init__.py | 5 ++ .../eng/BirdsnapClassification.py | 49 +++++++++++++++++ .../Image/ImageClassification/eng/CIFAR.py | 4 +- .../eng/Caltech101Classification.py | 2 +- .../eng/EuroSATClassification.py | 49 +++++++++++++++++ .../eng/FER2013Classification.py | 50 +++++++++++++++++ .../eng/FGVCAircraftClassification.py | 2 +- .../eng/Food101Classification.py | 2 +- .../eng/MNISTClassification.py | 46 ++++++++++++++++ .../eng/OxfordFlowersClassification.py | 2 +- .../eng/OxfordPetsClassification.py | 2 +- .../eng/StanfordCarsClassification.py | 45 +++++++++++++++ .../Image/ZeroshotClassification/__init__.py | 5 ++ .../ZeroshotClassification/eng/Birdsnap.py | 55 +++++++++++++++++++ .../ZeroshotClassification/eng/EuroSAT.py | 55 +++++++++++++++++++ .../ZeroshotClassification/eng/FER2013.py | 54 ++++++++++++++++++ .../Image/ZeroshotClassification/eng/MNIST.py | 52 ++++++++++++++++++ .../eng/StanfordCars.py | 50 +++++++++++++++++ .../BirdsnapZeroShot.json | 19 +++++++ .../EuroSAT.json | 48 ++++++++++++++++ .../EuroSATZeroShot.json | 19 +++++++ .../FER2013.json | 48 ++++++++++++++++ .../FER2013ZeroShot.json | 19 +++++++ .../MNIST.json | 48 ++++++++++++++++ .../MNISTZeroShot.json | 19 +++++++ .../StanfordCars.json | 48 ++++++++++++++++ .../StanfordCarsZeroShot.json | 19 +++++++ 28 files changed, 810 insertions(+), 7 deletions(-) create mode 100644 mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BirdsnapZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSAT.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013ZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNIST.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNISTZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCars.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsZeroShot.json diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 845a5e514a..0b9e7e125d 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -42,6 +42,7 @@ "Object recognition", "Scene recognition", "Caption Pairing", + "Emotion recognition", ] TASK_DOMAIN = Literal[ diff --git a/mteb/tasks/Image/ImageClassification/__init__.py b/mteb/tasks/Image/ImageClassification/__init__.py index 6eed085634..be8b3c0df1 100644 --- a/mteb/tasks/Image/ImageClassification/__init__.py +++ b/mteb/tasks/Image/ImageClassification/__init__.py @@ -1,8 +1,13 @@ from __future__ import annotations +from .eng.BirdsnapClassification import * from .eng.Caltech101Classification import * from .eng.CIFAR import * +from .eng.EuroSATClassification import * +from .eng.FER2013Classification import * from .eng.FGVCAircraftClassification import * from .eng.Food101Classification import * +from .eng.MNISTClassification import * from .eng.OxfordFlowersClassification import * from .eng.OxfordPetsClassification import * +from .eng.StanfordCarsClassification import * diff --git a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py new file mode 100644 index 0000000000..eaee1b2bf9 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class BirdsnapClassification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="Birdsnap", + description="Classifying bird images from 500 species.", + reference="https://openaccess.thecvf.com/content_cvpr_2014/html/Berg_Birdsnap_Large-scale_Fine-grained_2014_CVPR_paper.html", + dataset={ + "path": "isaacchung/birdsnap", + "revision": "e09b9dea248d579376684268cbedba28cd66b9b4", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2013-01-01", + "2014-01-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@InProceedings{Berg_2014_CVPR, + author = {Berg, Thomas and Liu, Jiongxin and Woo Lee, Seung and Alexander, Michelle L. and Jacobs, David W. and Belhumeur, Peter N.}, + title = {Birdsnap: Large-scale Fine-grained Visual Categorization of Birds}, + booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, + month = {June}, + year = {2014} + } + """, + descriptive_stats={ + "n_samples": {"test": 1851}, + "avg_character_length": {"test": 431.4}, + }, + ) + + # Override default column name in the subclass + label_column_name: str = "common" diff --git a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py index 7a69afa137..ffd5bd6637 100644 --- a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py +++ b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py @@ -15,7 +15,7 @@ class CIFAR10Classification(AbsTaskImageClassification): "revision": "0b2714987fa478483af9968de7c934580d0bb9a2", }, type="Classification", - category="s2s", + category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", @@ -56,7 +56,7 @@ class CIFAR100Classification(AbsTaskImageClassification): "revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", }, type="Classification", - category="s2s", + category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py index 40a23b13e5..52e19d06b7 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py @@ -16,7 +16,7 @@ class Caltech101Classification(AbsTaskImageClassification): "revision": "851374102055782c84f89b1b4e9d128a6568847b", }, type="Classification", - category="s2s", + category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py new file mode 100644 index 0000000000..688ca27f75 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class EuroSATClassification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="EuroSAT", + description="Classifying satellite images.", + reference="https://ieeexplore.ieee.org/document/8736785", + dataset={ + "path": "timm/eurosat-rgb", + "revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2019-01-01", + "2019-03-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Scene recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@ARTICLE{8736785, + author={Helber, Patrick and Bischke, Benjamin and Dengel, Andreas and Borth, Damian}, + journal={IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing}, + title={EuroSAT: A Novel Dataset and Deep Learning Benchmark for Land Use and Land Cover Classification}, + year={2019}, + volume={12}, + number={7}, + pages={2217-2226}, + keywords={Satellites;Earth;Remote sensing;Machine learning;Spatial resolution;Feature extraction;Benchmark testing;Dataset;deep convolutional neural network;deep learning;earth observation;land cover classification;land use classification;machine learning;remote sensing;satellite image classification;satellite images}, + doi={10.1109/JSTARS.2019.2918242}} + """, + descriptive_stats={ + "n_samples": {"test": 5400}, + "avg_character_length": {"test": 431.4}, + }, + ) diff --git a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py new file mode 100644 index 0000000000..6fe912a2fc --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class FER2013Classification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="FER2013", + description="Classifying facial emotions.", + reference="https://arxiv.org/abs/1412.6572", + dataset={ + "path": "clip-benchmark/wds_fer2013", + "revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2014-01-01", + "2014-12-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Emotion recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@misc{goodfellow2015explainingharnessingadversarialexamples, + title={Explaining and Harnessing Adversarial Examples}, + author={Ian J. Goodfellow and Jonathon Shlens and Christian Szegedy}, + year={2015}, + eprint={1412.6572}, + archivePrefix={arXiv}, + primaryClass={stat.ML}, + url={https://arxiv.org/abs/1412.6572}, + } + """, + descriptive_stats={ + "n_samples": {"test": 7178}, + "avg_character_length": {"test": 431.4}, + }, + ) + image_column_name: str = "jpg" + label_column_name: str = "cls" diff --git a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py index e8a469297a..c5ab396ea4 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py @@ -15,7 +15,7 @@ class FGVCAircraftClassification(AbsTaskImageClassification): "revision": "91860adfc9a09aabca5cddb5247442109b38e213", }, type="Classification", - category="s2s", + category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py index 466f78907b..82ee062d08 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py @@ -15,7 +15,7 @@ class Food101Classification(AbsTaskImageClassification): "revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", }, type="Classification", - category="s2s", + category="i2t", eval_splits=["validation"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py new file mode 100644 index 0000000000..7351cf0f22 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class MNISTClassification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="MNIST", + description="Classifying handwritten digits.", + reference="https://en.wikipedia.org/wiki/MNIST_database", + dataset={ + "path": "ylecun/mnist", + "revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2010-01-01", + "2010-04-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@article{lecun2010mnist, + title={MNIST handwritten digit database}, + author={LeCun, Yann and Cortes, Corinna and Burges, CJ}, + journal={ATT Labs [Online]. Available: http://yann.lecun.com/exdb/mnist}, + volume={2}, + year={2010} + } + """, + descriptive_stats={ + "n_samples": {"test": 10000}, + "avg_character_length": {"test": 431.4}, + }, + ) diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py index 5c19067710..35ac96fd41 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -15,7 +15,7 @@ class OxfordFlowersClassification(AbsTaskImageClassification): "revision": "a37b1891609c0376fa81eced756e7863e1bd873b", }, type="Classification", - category="s2s", + category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py index 769f40cce9..bb85984e94 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py @@ -15,7 +15,7 @@ class OxfordPetsClassification(AbsTaskImageClassification): "revision": "557b480fae8d69247be74d9503b378a09425096f", }, type="Classification", - category="s2s", + category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py new file mode 100644 index 0000000000..a225df6f47 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class StanfordCarsClassification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="StanfordCars", + description="Classifying car images from 96 makes.", + reference="https://pure.mpg.de/rest/items/item_2029263/component/file_2029262/content", + dataset={ + "path": "isaacchung/StanfordCars", + "revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2013-01-01", + "2013-04-01", + ), # Estimated range for the collection of reviews + domains=["Scene"], + task_subtypes=["Scene recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{Krause2013CollectingAL, + title={Collecting a Large-scale Dataset of Fine-grained Cars}, + author={Jonathan Krause and Jia Deng and Michael Stark and Li Fei-Fei}, + year={2013}, + url={https://api.semanticscholar.org/CorpusID:16632981} + } + """, + descriptive_stats={ + "n_samples": {"test": 8041}, + "avg_character_length": {"test": 431.4}, + }, + ) diff --git a/mteb/tasks/Image/ZeroshotClassification/__init__.py b/mteb/tasks/Image/ZeroshotClassification/__init__.py index 561f762e46..574de5e5f9 100644 --- a/mteb/tasks/Image/ZeroshotClassification/__init__.py +++ b/mteb/tasks/Image/ZeroshotClassification/__init__.py @@ -1,8 +1,13 @@ from __future__ import annotations +from .eng.Birdsnap import * from .eng.Caltech101 import * from .eng.CIFAR import * +from .eng.EuroSAT import * +from .eng.FER2013 import * from .eng.FGVCAircraft import * from .eng.Food101 import * +from .eng.MNIST import * from .eng.OxfordPets import * from .eng.RenderedSST2 import * +from .eng.StanfordCars import * diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py b/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py new file mode 100644 index 0000000000..bb8482aab2 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class BirdsnapClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="BirdsnapZeroShot", + description="Classifying bird images from 500 species.", + reference="https://openaccess.thecvf.com/content_cvpr_2014/html/Berg_Birdsnap_Large-scale_Fine-grained_2014_CVPR_paper.html", + dataset={ + "path": "isaacchung/birdsnap", + "revision": "e09b9dea248d579376684268cbedba28cd66b9b4", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2013-01-01", + "2014-01-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@InProceedings{Berg_2014_CVPR, + author = {Berg, Thomas and Liu, Jiongxin and Woo Lee, Seung and Alexander, Michelle L. and Jacobs, David W. and Belhumeur, Peter N.}, + title = {Birdsnap: Large-scale Fine-grained Visual Categorization of Birds}, + booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, + month = {June}, + year = {2014} + } + """, + descriptive_stats={ + "n_samples": {"test": 1851}, + "avg_character_length": {"test": 431.4}, + }, + ) + + # Override default column name in the subclass + label_column_name: str = "common" + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of a {name}, a type of bird." + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py b/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py new file mode 100644 index 0000000000..a862085956 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class EuroSATClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="EuroSATZeroShot", + description="Classifying satellite images.", + reference="https://ieeexplore.ieee.org/document/8736785", + dataset={ + "path": "timm/eurosat-rgb", + "revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2019-01-01", + "2019-03-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Scene recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@ARTICLE{8736785, + author={Helber, Patrick and Bischke, Benjamin and Dengel, Andreas and Borth, Damian}, + journal={IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing}, + title={EuroSAT: A Novel Dataset and Deep Learning Benchmark for Land Use and Land Cover Classification}, + year={2019}, + volume={12}, + number={7}, + pages={2217-2226}, + keywords={Satellites;Earth;Remote sensing;Machine learning;Spatial resolution;Feature extraction;Benchmark testing;Dataset;deep convolutional neural network;deep learning;earth observation;land cover classification;land use classification;machine learning;remote sensing;satellite image classification;satellite images}, + doi={10.1109/JSTARS.2019.2918242}} + """, + descriptive_stats={ + "n_samples": {"test": 5400}, + "avg_character_length": {"test": 431.4}, + }, + ) + + def get_candidate_labels(self) -> list[str]: + return [ + f"a centered satellite photo of {name}." + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py b/mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py new file mode 100644 index 0000000000..0316c79d6c --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class FER2013Classification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="FER2013ZeroShot", + description="Classifying facial emotions.", + reference="https://arxiv.org/abs/1412.6572", + dataset={ + "path": "clip-benchmark/wds_fer2013", + "revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2014-01-01", + "2014-12-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Emotion recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@misc{goodfellow2015explainingharnessingadversarialexamples, + title={Explaining and Harnessing Adversarial Examples}, + author={Ian J. Goodfellow and Jonathon Shlens and Christian Szegedy}, + year={2015}, + eprint={1412.6572}, + archivePrefix={arXiv}, + primaryClass={stat.ML}, + url={https://arxiv.org/abs/1412.6572}, + } + """, + descriptive_stats={ + "n_samples": {"test": 7178}, + "avg_character_length": {"test": 431.4}, + }, + ) + image_column_name: str = "jpg" + label_column_name: str = "cls" + + def get_candidate_labels(self) -> list[str]: + labels = ["angry", "disgust", "fear", "happy", "neutral", "sad", "surprise"] + return [f"a photo of a {name} looking face." for name in labels] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py b/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py new file mode 100644 index 0000000000..324a7e5b8a --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class MNISTClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="MNISTZeroShot", + description="Classifying handwritten digits.", + reference="https://en.wikipedia.org/wiki/MNIST_database", + dataset={ + "path": "ylecun/mnist", + "revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2010-01-01", + "2010-04-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@article{lecun2010mnist, + title={MNIST handwritten digit database}, + author={LeCun, Yann and Cortes, Corinna and Burges, CJ}, + journal={ATT Labs [Online]. Available: http://yann.lecun.com/exdb/mnist}, + volume={2}, + year={2010} + } + """, + descriptive_stats={ + "n_samples": {"test": 10000}, + "avg_character_length": {"test": 431.4}, + }, + ) + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of the number: '{name}'." + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py b/mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py new file mode 100644 index 0000000000..394b3a073e --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class StanfordCarsClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="StanfordCarsZeroShot", + description="Classifying car images from 96 makes.", + reference="https://pure.mpg.de/rest/items/item_2029263/component/file_2029262/content", + dataset={ + "path": "isaacchung/StanfordCars", + "revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2013-01-01", + "2013-04-01", + ), # Estimated range for the collection of reviews + domains=["Scene"], + task_subtypes=["Scene recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@inproceedings{Krause2013CollectingAL, + title={Collecting a Large-scale Dataset of Fine-grained Cars}, + author={Jonathan Krause and Jia Deng and Michael Stark and Li Fei-Fei}, + year={2013}, + url={https://api.semanticscholar.org/CorpusID:16632981} + }""", + descriptive_stats={ + "n_samples": {"test": 8041}, + "avg_character_length": {"test": 431.4}, + }, + ) + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of a {name}." + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BirdsnapZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BirdsnapZeroShot.json new file mode 100644 index 0000000000..a3b1e48e41 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BirdsnapZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "e09b9dea248d579376684268cbedba28cd66b9b4", + "evaluation_time": 235.52802658081055, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.4057266342517558, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.4057266342517558 + } + ] + }, + "task_name": "BirdsnapZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSAT.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSAT.json new file mode 100644 index 0000000000..98c522b3fb --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSAT.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", + "evaluation_time": 46.82566833496094, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.8501481481481481, + "f1": 0.8469797204361242, + "f1_weighted": 0.8496265755865398, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8501481481481481, + "scores_per_experiment": [ + { + "accuracy": 0.8518518518518519, + "f1": 0.8495295548783852, + "f1_weighted": 0.8511665064639554 + }, + { + "accuracy": 0.8501851851851852, + "f1": 0.8471340637143676, + "f1_weighted": 0.8499896449882506 + }, + { + "accuracy": 0.8475925925925926, + "f1": 0.8422738482428335, + "f1_weighted": 0.8460446242004349 + }, + { + "accuracy": 0.8605555555555555, + "f1": 0.8580738304964756, + "f1_weighted": 0.8608069947542264 + }, + { + "accuracy": 0.8405555555555555, + "f1": 0.8378873048485588, + "f1_weighted": 0.840125107525832 + } + ] + } + ] + }, + "task_name": "EuroSAT" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json new file mode 100644 index 0000000000..a118df5f28 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", + "evaluation_time": 24.762829065322876, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.3672222222222222, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.3672222222222222 + } + ] + }, + "task_name": "EuroSATZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013.json new file mode 100644 index 0000000000..03d577f7b4 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", + "evaluation_time": 49.89526414871216, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.4285594873223739, + "f1": 0.3843822780757347, + "f1_weighted": 0.43760737475533196, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.4285594873223739, + "scores_per_experiment": [ + { + "accuracy": 0.4179437169127891, + "f1": 0.37722976261634705, + "f1_weighted": 0.43062554573695727 + }, + { + "accuracy": 0.45388687656728893, + "f1": 0.3994675689562795, + "f1_weighted": 0.459751540669273 + }, + { + "accuracy": 0.4360546113123433, + "f1": 0.39754111862374936, + "f1_weighted": 0.444949254019621 + }, + { + "accuracy": 0.45723042630259125, + "f1": 0.3974530741571466, + "f1_weighted": 0.45463208715209835 + }, + { + "accuracy": 0.37768180551685704, + "f1": 0.3502198660251512, + "f1_weighted": 0.3980784461987101 + } + ] + } + ] + }, + "task_name": "FER2013" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013ZeroShot.json new file mode 100644 index 0000000000..3366d7e8e7 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", + "evaluation_time": 30.47609853744507, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.4352187238785177, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.4352187238785177 + } + ] + }, + "task_name": "FER2013ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNIST.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNIST.json new file mode 100644 index 0000000000..8617dc7bad --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 69.79996991157532, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.89482, + "f1": 0.8938189564908757, + "f1_weighted": 0.8946237575172749, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89482, + "scores_per_experiment": [ + { + "accuracy": 0.8937, + "f1": 0.8927176424361276, + "f1_weighted": 0.893609834767085 + }, + { + "accuracy": 0.9019, + "f1": 0.9009269794036706, + "f1_weighted": 0.9015503965412863 + }, + { + "accuracy": 0.8788, + "f1": 0.878248230427133, + "f1_weighted": 0.8789836029821599 + }, + { + "accuracy": 0.8807, + "f1": 0.8791930140887333, + "f1_weighted": 0.8803275669342271 + }, + { + "accuracy": 0.919, + "f1": 0.9180089160987143, + "f1_weighted": 0.918647386361616 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNISTZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNISTZeroShot.json new file mode 100644 index 0000000000..46d3432848 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNISTZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 41.33473753929138, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.4899, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.4899 + } + ] + }, + "task_name": "MNISTZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCars.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCars.json new file mode 100644 index 0000000000..c7a9dd1630 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCars.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", + "evaluation_time": 660.5333795547485, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.7398830991170253, + "f1": 0.7403225810264707, + "f1_weighted": 0.7400357184409913, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7398830991170253, + "scores_per_experiment": [ + { + "accuracy": 0.7430667827384654, + "f1": 0.7436627162291564, + "f1_weighted": 0.7435113191449505 + }, + { + "accuracy": 0.7363512000994901, + "f1": 0.7365198863893867, + "f1_weighted": 0.7363691751162225 + }, + { + "accuracy": 0.734112672553165, + "f1": 0.7341057453057904, + "f1_weighted": 0.7342243485571237 + }, + { + "accuracy": 0.7404551672677527, + "f1": 0.7408828281148372, + "f1_weighted": 0.7406406001011914 + }, + { + "accuracy": 0.745429672926253, + "f1": 0.7464417290931831, + "f1_weighted": 0.7454331492854683 + } + ] + } + ] + }, + "task_name": "StanfordCars" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsZeroShot.json new file mode 100644 index 0000000000..2762ca15c7 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", + "evaluation_time": 131.17899537086487, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.5861211292127845, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5861211292127845 + } + ] + }, + "task_name": "StanfordCarsZeroShot" +} \ No newline at end of file From 6b49181ecdd8796f92c7194d43087c3780fb7c89 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sun, 21 Jul 2024 07:47:39 +0000 Subject: [PATCH 020/154] correct eurosat zero shot labels --- .../Image/ZeroshotClassification/eng/EuroSAT.py | 15 ++++++++++++--- .../EuroSATZeroShot.json | 6 +++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py b/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py index a862085956..aa71804767 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py @@ -49,7 +49,16 @@ class EuroSATClassification(AbsTaskZeroshotClassification): ) def get_candidate_labels(self) -> list[str]: - return [ - f"a centered satellite photo of {name}." - for name in self.dataset["test"].features[self.label_column_name].names + labels = [ + "annual crop land", + "forest land", + "brushland or shrubland", + "highway or road", + "industrial land", + "pasture land", + "permanent crop land", + "residential land", + "river", + "sea or lake", ] + return [f"a centered satellite photo of {name}." for name in labels] diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json index a118df5f28..af34ef21d1 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json @@ -1,17 +1,17 @@ { "dataset_revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", - "evaluation_time": 24.762829065322876, + "evaluation_time": 24.88299822807312, "kg_co2_emissions": null, "mteb_version": "1.12.80", "scores": { "test": [ { - "accuracy": 0.3672222222222222, + "accuracy": 0.4974074074074074, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.3672222222222222 + "main_score": 0.4974074074074074 } ] }, From b827a19155d1c5e9f38d74919adc643707734725 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sun, 21 Jul 2024 16:32:48 +0000 Subject: [PATCH 021/154] add abstask for image multilable and voc2007 --- .../AbsTaskImageMultilabelClassification.py | 203 ++++++++++++++++++ mteb/abstasks/__init__.py | 1 + .../ImageMultilabelClassification/__init__.py | 3 + .../eng/PascalVOC2007.py | 55 +++++ .../eng/__init__.py | 0 mteb/tasks/Image/__init__.py | 1 + .../VOC2007.json | 28 +++ 7 files changed, 291 insertions(+) create mode 100644 mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py create mode 100644 mteb/tasks/Image/ImageMultilabelClassification/__init__.py create mode 100644 mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py create mode 100644 mteb/tasks/Image/ImageMultilabelClassification/eng/__init__.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json diff --git a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py new file mode 100644 index 0000000000..91384cf581 --- /dev/null +++ b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import itertools +import logging +from collections import defaultdict +from typing import Any + +import numpy as np +from sklearn.base import ClassifierMixin, clone +from sklearn.metrics import f1_score, label_ranking_average_precision_score +from sklearn.model_selection import train_test_split +from sklearn.neighbors import KNeighborsClassifier +from sklearn.preprocessing import MultiLabelBinarizer + +from mteb.abstasks import AbsTask +from mteb.encoder_interface import Encoder +from mteb.load_results.mteb_results import HFSubset, ScoresDict + +logger = logging.getLogger(__name__) + + +def evaluate_classifier( + embeddings_train: np.ndarray, + y_train: np.ndarray, + embeddings_test: np.ndarray, + y_test: np.ndarray, + classifier: ClassifierMixin, +): + scores = {} + classifier = clone(classifier) + classifier.fit(embeddings_train, y_train) + y_pred = classifier.predict(embeddings_test) + accuracy = classifier.score(embeddings_test, y_test) + f1 = f1_score(y_test, y_pred, average="macro") + scores["accuracy"] = accuracy + scores["f1"] = f1 + lrap = label_ranking_average_precision_score(y_test, y_pred) + scores["lrap"] = lrap + return scores + + +class AbsTaskImageMultilabelClassification(AbsTask): + """Abstract class for image multioutput classification tasks + The similarity is computed between pairs and the results are ranked. + + self.load_data() must generate a huggingface dataset with a split matching self.metadata_dict["eval_splits"], and assign it to self.dataset. It must contain the following columns: + image: list[PIL.Image] + labels: list[Hashable] + """ + + image_column_name: str = "image" + label_column_name: str = "labels" + + classifier = KNeighborsClassifier(n_neighbors=5) + + def __init__( + self, + n_experiments=None, + samples_per_label=None, + batch_size=32, + **kwargs, + ): + super().__init__(**kwargs) + self.batch_size = batch_size + + # Bootstrap parameters + self.n_experiments = n_experiments or getattr(self, "n_experiments", 10) + self.samples_per_label = samples_per_label or getattr( + self, "samples_per_label", 8 + ) + # Run metadata validation by instantiating addressing the attribute + # This is quite hacky. Ideally, this would be done in the constructor of + # each concrete task, but then we have to duplicate the __init__ method's + # interface. + if hasattr(self, "metadata"): + self.metadata + + def _add_main_score(self, scores): + scores["main_score"] = scores[self.metadata.main_score] + + def evaluate( + self, + model: Encoder, + eval_split: str = "test", + train_split: str = "train", + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs: Any, + ) -> dict[HFSubset, ScoresDict]: + if not self.data_loaded: + self.load_data() + + scores = {} + hf_subsets = [l for l in self.dataset] if self.is_multilingual else ["default"] + + for hf_subset in hf_subsets: + logger.info( + f"\nTask: {self.metadata.name}, split: {eval_split}, subset: {hf_subset}. Running..." + ) + + if hf_subset not in self.dataset and hf_subset == "default": + ds = self.dataset + else: + ds = self.dataset[hf_subset] + scores[hf_subset] = self._evaluate_subset( + model, + ds, + eval_split, + train_split, + encode_kwargs=encode_kwargs, + **kwargs, + ) + self._add_main_score(scores[hf_subset]) + + return scores + + def _evaluate_subset( + self, + model: Encoder, + dataset, + eval_split: str = "test", + train_split: str = "train", + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs: Any, + ) -> ScoresDict: + train_split = dataset[train_split] + eval_split = dataset[eval_split] + params = { + "classifier_type": type(self.classifier).__name__, + "classifier_params": self.classifier.get_params(), + "batch_size": self.batch_size, + } + params.update(kwargs) + + scores = [] + # Bootstrap sample indices from training set for each experiment + train_samples = [] + for _ in range(self.n_experiments): + sample_indices, _ = self._undersample_data_indices( + train_split[self.label_column_name], self.samples_per_label, None + ) + train_samples.append(sample_indices) + # Encode all unique images at the indices + unique_train_indices = list(set(itertools.chain.from_iterable(train_samples))) + unique_train_images = train_split.select(unique_train_indices)[ + self.image_column_name + ] + + _unique_train_embeddings = model.get_image_embeddings( + unique_train_images, + **encode_kwargs, + ) + unique_train_embeddings = dict( + zip(unique_train_indices, _unique_train_embeddings) + ) + test_images = eval_split[self.image_column_name] + binarizer = MultiLabelBinarizer() + y_test = binarizer.fit_transform(eval_split[self.label_column_name]) + # Stratified subsampling of test set to 2000 examples. + try: + if len(test_images) > 2000: + test_images, _, y_test, _ = train_test_split( + test_images, y_test, stratify=y_test, train_size=2000 + ) + except ValueError: + logger.warning("Couldn't subsample, continuing with the entire test set.") + + X_test = model.get_image_embeddings(test_images, **encode_kwargs) + for i_experiment, sample_indices in enumerate(train_samples): + logger.info( + "=" * 10 + + f" Experiment {i_experiment+1}/{self.n_experiments} " + + "=" * 10 + ) + X_train = np.stack([unique_train_embeddings[idx] for idx in sample_indices]) + y_train = train_split.select(sample_indices)[self.label_column_name] + y_train = binarizer.transform(y_train) + scores_exp = evaluate_classifier( + X_train, y_train, X_test, y_test, self.classifier + ) + scores.append(scores_exp) + + avg_scores: dict[str, Any] = { + k: np.mean([s[k] for s in scores]) for k in scores[0].keys() + } + avg_scores["scores_per_experiment"] = scores + + return avg_scores + + def _undersample_data_indices(self, y, samples_per_label, idxs=None): + """Undersample data to have samples_per_label samples of each label""" + sample_indices = [] + if idxs is None: + idxs = np.arange(len(y)) + np.random.shuffle(idxs) + label_counter = defaultdict(int) + for i in idxs: + if any((label_counter[label] < samples_per_label) for label in y[i]): + sample_indices.append(i) + for label in y[i]: + label_counter[label] += 1 + return sample_indices, idxs diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index 01366cf3f5..273bc4640b 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -17,6 +17,7 @@ from .Image.AbsTaskI2TRetrieval import * from .Image.AbsTaskImageClassification import * from .Image.AbsTaskImageClustering import * +from .Image.AbsTaskImageMultilabelClassification import * from .Image.AbsTaskImageTextPairClassification import * from .Image.AbsTaskT2IRetrieval import * from .Image.AbsTaskZeroshotClassification import * diff --git a/mteb/tasks/Image/ImageMultilabelClassification/__init__.py b/mteb/tasks/Image/ImageMultilabelClassification/__init__.py new file mode 100644 index 0000000000..844f19a14c --- /dev/null +++ b/mteb/tasks/Image/ImageMultilabelClassification/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .eng.PascalVOC2007 import * diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py new file mode 100644 index 0000000000..06c71c1ffa --- /dev/null +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskImageMultilabelClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class VOC2007Classification(AbsTaskImageMultilabelClassification): + metadata = TaskMetadata( + name="VOC2007", + description="Classifying bird images from 500 species.", + reference="http://host.robots.ox.ac.uk/pascal/VOC/", + dataset={ + "path": "HuggingFaceM4/pascal_voc", + "name": "voc2007_main", + "revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + }, + type="MultilabelClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2005-01-01", + "2014-01-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@Article{Everingham10, + author = "Everingham, M. and Van~Gool, L. and Williams, C. K. I. and Winn, J. and Zisserman, A.", + title = "The Pascal Visual Object Classes (VOC) Challenge", + journal = "International Journal of Computer Vision", + volume = "88", + year = "2010", + number = "2", + month = jun, + pages = "303--338", + } + """, + descriptive_stats={ + "n_samples": {"test": 4952}, + "avg_character_length": {"test": 431.4}, + }, + ) + + # Override default column name in the subclass + label_column_name: str = "classes" + + # To be removed when we want full results + n_experiments: int = 1 diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/__init__.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Image/__init__.py b/mteb/tasks/Image/__init__.py index 74a4980dae..661136d16e 100644 --- a/mteb/tasks/Image/__init__.py +++ b/mteb/tasks/Image/__init__.py @@ -2,6 +2,7 @@ from .Clustering import * from .I2TRetrieval import * from .ImageClassification import * +from .ImageMultilabelClassification import * from .ImageTextPairClassification import * from .T2IRetrieval import * from .ZeroshotClassification import * diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json new file mode 100644 index 0000000000..dafbb1f01e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json @@ -0,0 +1,28 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 55.247910261154175, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.5026252019386107, + "f1": 0.6780762548765893, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.6642591096751196, + "main_score": 0.5026252019386107, + "scores_per_experiment": [ + { + "accuracy": 0.5026252019386107, + "f1": 0.6780762548765893, + "lrap": 0.6642591096751196 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file From 630d4a599211608e8ce9fe0e5ba15700a7407dbf Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sun, 21 Jul 2024 16:33:59 +0000 Subject: [PATCH 022/154] make lint --- .../Image/ImageMultilabelClassification/eng/PascalVOC2007.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index 06c71c1ffa..8054832b4a 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -50,6 +50,6 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): # Override default column name in the subclass label_column_name: str = "classes" - + # To be removed when we want full results n_experiments: int = 1 From 935f621a5f5c553eb98cfbcf55d022f58c4dcec0 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 22 Jul 2024 04:42:58 +0300 Subject: [PATCH 023/154] [MIEB] Add more image classification and zero shot datasets (#1105) * add STL10 linear probe and zero shot * add RESISC45 linear probe and zeor shot * add Describable textures linear probe and zero shot * fix spacing lint * add SUN397 linear probe and zero shot --- mteb/abstasks/TaskMetadata.py | 1 + .../Image/ImageClassification/__init__.py | 4 ++ .../eng/DTDClassification.py | 44 ++++++++++++++ .../eng/RESISC45Classification.py | 49 +++++++++++++++ .../eng/STL10Classification.py | 54 +++++++++++++++++ .../eng/SUN397Classification.py | 48 +++++++++++++++ .../Image/ZeroshotClassification/__init__.py | 4 ++ .../Image/ZeroshotClassification/eng/DTD.py | 50 ++++++++++++++++ .../ZeroshotClassification/eng/RESISC45.py | 55 +++++++++++++++++ .../Image/ZeroshotClassification/eng/STL10.py | 60 +++++++++++++++++++ .../ZeroshotClassification/eng/SUN397.py | 59 ++++++++++++++++++ .../DTD.json | 48 +++++++++++++++ .../DTDZeroShot.json | 19 ++++++ .../RESISC45.json | 48 +++++++++++++++ .../RESISC45ZeroShot.json | 19 ++++++ .../STL10.json | 48 +++++++++++++++ .../STL10ZeroShot.json | 19 ++++++ .../SUN397.json | 48 +++++++++++++++ .../SUN397ZeroShot.json | 19 ++++++ 19 files changed, 696 insertions(+) create mode 100644 mteb/tasks/Image/ImageClassification/eng/DTDClassification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/STL10Classification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/DTD.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/STL10.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTD.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTDZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45ZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10ZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 0b9e7e125d..16219482b1 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -43,6 +43,7 @@ "Scene recognition", "Caption Pairing", "Emotion recognition", + "Textures recognition", ] TASK_DOMAIN = Literal[ diff --git a/mteb/tasks/Image/ImageClassification/__init__.py b/mteb/tasks/Image/ImageClassification/__init__.py index be8b3c0df1..152406c954 100644 --- a/mteb/tasks/Image/ImageClassification/__init__.py +++ b/mteb/tasks/Image/ImageClassification/__init__.py @@ -3,6 +3,7 @@ from .eng.BirdsnapClassification import * from .eng.Caltech101Classification import * from .eng.CIFAR import * +from .eng.DTDClassification import * from .eng.EuroSATClassification import * from .eng.FER2013Classification import * from .eng.FGVCAircraftClassification import * @@ -10,4 +11,7 @@ from .eng.MNISTClassification import * from .eng.OxfordFlowersClassification import * from .eng.OxfordPetsClassification import * +from .eng.RESISC45Classification import * from .eng.StanfordCarsClassification import * +from .eng.STL10Classification import * +from .eng.SUN397Classification import * diff --git a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py new file mode 100644 index 0000000000..66c6c0de50 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class DTDClassification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="DTD", + description="Describable Textures Dataset in 47 categories.", + reference="https://www.robots.ox.ac.uk/~vgg/data/dtd/", + dataset={ + "path": "tanganke/dtd", + "revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2014-01-01", + "2014-03-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Textures recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@InProceedings{cimpoi14describing, + Author = {M. Cimpoi and S. Maji and I. Kokkinos and S. Mohamed and and A. Vedaldi}, + Title = {Describing Textures in the Wild}, + Booktitle = {Proceedings of the {IEEE} Conf. on Computer Vision and Pattern Recognition ({CVPR})}, + Year = {2014}} + """, + descriptive_stats={ + "n_samples": {"test": 1880}, + "avg_character_length": {"test": 456}, + }, + ) diff --git a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py new file mode 100644 index 0000000000..5426f6032c --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class RESISC45Classification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="RESISC45", + description="Remote Sensing Image Scene Classification by Northwestern Polytechnical University (NWPU).", + reference="https://ieeexplore.ieee.org/abstract/document/7891544", + dataset={ + "path": "timm/resisc45", + "revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2017-01-01", + "2017-03-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@ARTICLE{7891544, + author={Cheng, Gong and Han, Junwei and Lu, Xiaoqiang}, + journal={Proceedings of the IEEE}, + title={Remote Sensing Image Scene Classification: Benchmark and State of the Art}, + year={2017}, + volume={105}, + number={10}, + pages={1865-1883}, + keywords={Remote sensing;Benchmark testing;Spatial resolution;Social network services;Satellites;Image analysis;Machine learning;Unsupervised learning;Classification;Benchmark data set;deep learning;handcrafted features;remote sensing image;scene classification;unsupervised feature learning}, + doi={10.1109/JPROC.2017.2675998}} + """, + descriptive_stats={ + "n_samples": {"test": 6300}, + "avg_character_length": {"test": 256}, + }, + ) diff --git a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py new file mode 100644 index 0000000000..13685ffca1 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class STL10Classification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="STL10", + description="Classifying 96x96 images from 10 classes.", + reference="https://cs.stanford.edu/~acoates/stl10/", + dataset={ + "path": "tanganke/stl10", + "revision": "49ae7f94508f7feae62baf836db284306eab0b0f", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2011-01-01", + "2011-04-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@InProceedings{pmlr-v15-coates11a, + title = {An Analysis of Single-Layer Networks in Unsupervised Feature Learning}, + author = {Coates, Adam and Ng, Andrew and Lee, Honglak}, + booktitle = {Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics}, + pages = {215--223}, + year = {2011}, + editor = {Gordon, Geoffrey and Dunson, David and Dudík, Miroslav}, + volume = {15}, + series = {Proceedings of Machine Learning Research}, + address = {Fort Lauderdale, FL, USA}, + month = {11--13 Apr}, + publisher = {PMLR}, + pdf = {http://proceedings.mlr.press/v15/coates11a/coates11a.pdf}, + url = {https://proceedings.mlr.press/v15/coates11a.html}, + } + """, + descriptive_stats={ + "n_samples": {"test": 8000}, + "avg_character_length": {"test": 431.4}, + }, + ) diff --git a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py new file mode 100644 index 0000000000..006d2057e6 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskImageClassification + + +class SUN397Classification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="SUN397", + description="Large scale scene recognition in 397 categories.", + reference="https://ieeexplore.ieee.org/abstract/document/5539970", + dataset={ + "path": "dpdl-benchmark/sun397", + "revision": "7e6af6a2499ad708618be868e1471eac0aca1168", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2017-01-01", + "2017-03-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Scene recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@INPROCEEDINGS{5539970, + author={Xiao, Jianxiong and Hays, James and Ehinger, Krista A. and Oliva, Aude and Torralba, Antonio}, + booktitle={2010 IEEE Computer Society Conference on Computer Vision and Pattern Recognition}, + title={SUN database: Large-scale scene recognition from abbey to zoo}, + year={2010}, + volume={}, + number={}, + pages={3485-3492}, + doi={10.1109/CVPR.2010.5539970}} + """, + descriptive_stats={ + "n_samples": {"test": 21750}, + "avg_character_length": {"test": 256}, + }, + ) diff --git a/mteb/tasks/Image/ZeroshotClassification/__init__.py b/mteb/tasks/Image/ZeroshotClassification/__init__.py index 574de5e5f9..0de4d86ce1 100644 --- a/mteb/tasks/Image/ZeroshotClassification/__init__.py +++ b/mteb/tasks/Image/ZeroshotClassification/__init__.py @@ -3,6 +3,7 @@ from .eng.Birdsnap import * from .eng.Caltech101 import * from .eng.CIFAR import * +from .eng.DTD import * from .eng.EuroSAT import * from .eng.FER2013 import * from .eng.FGVCAircraft import * @@ -10,4 +11,7 @@ from .eng.MNIST import * from .eng.OxfordPets import * from .eng.RenderedSST2 import * +from .eng.RESISC45 import * from .eng.StanfordCars import * +from .eng.STL10 import * +from .eng.SUN397 import * diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/DTD.py b/mteb/tasks/Image/ZeroshotClassification/eng/DTD.py new file mode 100644 index 0000000000..a4dcc6e196 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/DTD.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class DTDClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="DTDZeroShot", + description="Describable Textures Dataset in 47 categories.", + reference="https://www.robots.ox.ac.uk/~vgg/data/dtd/", + dataset={ + "path": "tanganke/dtd", + "revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2014-01-01", + "2014-03-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Textures recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@InProceedings{cimpoi14describing, + Author = {M. Cimpoi and S. Maji and I. Kokkinos and S. Mohamed and and A. Vedaldi}, + Title = {Describing Textures in the Wild}, + Booktitle = {Proceedings of the {IEEE} Conf. on Computer Vision and Pattern Recognition ({CVPR})}, + Year = {2014}} + """, + descriptive_stats={ + "n_samples": {"test": 1880}, + "avg_character_length": {"test": 456}, + }, + ) + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of {name} texture." + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py b/mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py new file mode 100644 index 0000000000..93c16ea3cf --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class RESISC45Classification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="RESISC45ZeroShot", + description="Remote Sensing Image Scene Classification by Northwestern Polytechnical University (NWPU).", + reference="https://ieeexplore.ieee.org/abstract/document/7891544", + dataset={ + "path": "timm/resisc45", + "revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2017-01-01", + "2017-03-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@ARTICLE{7891544, + author={Cheng, Gong and Han, Junwei and Lu, Xiaoqiang}, + journal={Proceedings of the IEEE}, + title={Remote Sensing Image Scene Classification: Benchmark and State of the Art}, + year={2017}, + volume={105}, + number={10}, + pages={1865-1883}, + keywords={Remote sensing;Benchmark testing;Spatial resolution;Social network services;Satellites;Image analysis;Machine learning;Unsupervised learning;Classification;Benchmark data set;deep learning;handcrafted features;remote sensing image;scene classification;unsupervised feature learning}, + doi={10.1109/JPROC.2017.2675998}} + """, + descriptive_stats={ + "n_samples": {"test": 6300}, + "avg_character_length": {"test": 256}, + }, + ) + + def get_candidate_labels(self) -> list[str]: + return [ + f"satellite imagery of {name}." + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/STL10.py b/mteb/tasks/Image/ZeroshotClassification/eng/STL10.py new file mode 100644 index 0000000000..476646bdcb --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/STL10.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class STL10Classification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="STL10ZeroShot", + description="Classifying 96x96 images from 10 classes.", + reference="https://cs.stanford.edu/~acoates/stl10/", + dataset={ + "path": "tanganke/stl10", + "revision": "49ae7f94508f7feae62baf836db284306eab0b0f", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2011-01-01", + "2011-04-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@InProceedings{pmlr-v15-coates11a, + title = {An Analysis of Single-Layer Networks in Unsupervised Feature Learning}, + author = {Coates, Adam and Ng, Andrew and Lee, Honglak}, + booktitle = {Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics}, + pages = {215--223}, + year = {2011}, + editor = {Gordon, Geoffrey and Dunson, David and Dudík, Miroslav}, + volume = {15}, + series = {Proceedings of Machine Learning Research}, + address = {Fort Lauderdale, FL, USA}, + month = {11--13 Apr}, + publisher = {PMLR}, + pdf = {http://proceedings.mlr.press/v15/coates11a/coates11a.pdf}, + url = {https://proceedings.mlr.press/v15/coates11a.html}, + } + """, + descriptive_stats={ + "n_samples": {"test": 8000}, + "avg_character_length": {"test": 431.4}, + }, + ) + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of a {name}." + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py b/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py new file mode 100644 index 0000000000..a3852419b6 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskZeroshotClassification + + +class SUN397Classification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="SUN397ZeroShot", + description="Large scale scene recognition in 397 categories.", + reference="https://ieeexplore.ieee.org/abstract/document/5539970", + dataset={ + "path": "dpdl-benchmark/sun397", + "revision": "7e6af6a2499ad708618be868e1471eac0aca1168", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2017-01-01", + "2017-03-01", + ), # Estimated range for the collection of reviews + domains=["Encyclopaedic"], + task_subtypes=["Scene recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@INPROCEEDINGS{5539970, + author={Xiao, Jianxiong and Hays, James and Ehinger, Krista A. and Oliva, Aude and Torralba, Antonio}, + booktitle={2010 IEEE Computer Society Conference on Computer Vision and Pattern Recognition}, + title={SUN database: Large-scale scene recognition from abbey to zoo}, + year={2010}, + volume={}, + number={}, + pages={3485-3492}, + doi={10.1109/CVPR.2010.5539970}} + """, + descriptive_stats={ + "n_samples": {"test": 21750}, + "avg_character_length": {"test": 256}, + }, + ) + + def get_candidate_labels(self) -> list[str]: + """Convert labels as such: + - /b/boat_deck -> boat deck + - /c/church/outdoor -> church outdoor + """ + labels = [] + for name in self.dataset["test"].features[self.label_column_name].names: + name = " ".join(name.split("/")[1:]).replace("_", " ") + labels.append(f"a photo of a {name}.") + return labels diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTD.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTD.json new file mode 100644 index 0000000000..22f21ae589 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTD.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", + "evaluation_time": 124.488609790802, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.6753191489361702, + "f1": 0.6745404047233853, + "f1_weighted": 0.6745404047233852, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6753191489361702, + "scores_per_experiment": [ + { + "accuracy": 0.6808510638297872, + "f1": 0.6808368640933851, + "f1_weighted": 0.680836864093385 + }, + { + "accuracy": 0.676063829787234, + "f1": 0.6723410903725255, + "f1_weighted": 0.6723410903725255 + }, + { + "accuracy": 0.676595744680851, + "f1": 0.6772514855600662, + "f1_weighted": 0.6772514855600661 + }, + { + "accuracy": 0.6734042553191489, + "f1": 0.6732365886117732, + "f1_weighted": 0.6732365886117734 + }, + { + "accuracy": 0.6696808510638298, + "f1": 0.6690359949791764, + "f1_weighted": 0.6690359949791764 + } + ] + } + ] + }, + "task_name": "DTD" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTDZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTDZeroShot.json new file mode 100644 index 0000000000..c9dd323257 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTDZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", + "evaluation_time": 21.39889645576477, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.41914893617021276, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.41914893617021276 + } + ] + }, + "task_name": "DTDZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45.json new file mode 100644 index 0000000000..05a8755f9b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", + "evaluation_time": 124.30631566047668, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.8280000000000001, + "f1": 0.8274150272493415, + "f1_weighted": 0.8282609883607547, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8280000000000001, + "scores_per_experiment": [ + { + "accuracy": 0.8301587301587302, + "f1": 0.8298896051930552, + "f1_weighted": 0.8302023220255628 + }, + { + "accuracy": 0.8342857142857143, + "f1": 0.833295417662022, + "f1_weighted": 0.834228803768538 + }, + { + "accuracy": 0.8223809523809524, + "f1": 0.8212130407841506, + "f1_weighted": 0.8225055946019044 + }, + { + "accuracy": 0.8285714285714286, + "f1": 0.8283929971104949, + "f1_weighted": 0.8293046793995597 + }, + { + "accuracy": 0.8246031746031746, + "f1": 0.8242840754969853, + "f1_weighted": 0.8250635420082086 + } + ] + } + ] + }, + "task_name": "RESISC45" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45ZeroShot.json new file mode 100644 index 0000000000..807ddc393a --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", + "evaluation_time": 37.90462064743042, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.533968253968254, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.533968253968254 + } + ] + }, + "task_name": "RESISC45ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10.json new file mode 100644 index 0000000000..b11a0d7f99 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "49ae7f94508f7feae62baf836db284306eab0b0f", + "evaluation_time": 72.2589225769043, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.9773, + "f1": 0.9772715792109945, + "f1_weighted": 0.9772715792109942, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9773, + "scores_per_experiment": [ + { + "accuracy": 0.977, + "f1": 0.9769671771353311, + "f1_weighted": 0.9769671771353311 + }, + { + "accuracy": 0.97825, + "f1": 0.9781918264450462, + "f1_weighted": 0.9781918264450461 + }, + { + "accuracy": 0.97675, + "f1": 0.9767550808626206, + "f1_weighted": 0.9767550808626204 + }, + { + "accuracy": 0.978375, + "f1": 0.9783827689066221, + "f1_weighted": 0.978382768906622 + }, + { + "accuracy": 0.976125, + "f1": 0.976061042705352, + "f1_weighted": 0.9760610427053519 + } + ] + } + ] + }, + "task_name": "STL10" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10ZeroShot.json new file mode 100644 index 0000000000..a703124990 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "49ae7f94508f7feae62baf836db284306eab0b0f", + "evaluation_time": 34.58206105232239, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.973, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.973 + } + ] + }, + "task_name": "STL10ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397.json new file mode 100644 index 0000000000..84753b28c9 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "7e6af6a2499ad708618be868e1471eac0aca1168", + "evaluation_time": 2724.93390917778, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.6952091954022988, + "f1": 0.6758912178278318, + "f1_weighted": 0.7047976563694092, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6952091954022988, + "scores_per_experiment": [ + { + "accuracy": 0.6940689655172414, + "f1": 0.6720733245789509, + "f1_weighted": 0.7035562570363177 + }, + { + "accuracy": 0.6872183908045977, + "f1": 0.6717815223371335, + "f1_weighted": 0.69741352860682 + }, + { + "accuracy": 0.6966436781609195, + "f1": 0.6783298988455696, + "f1_weighted": 0.7056108128599463 + }, + { + "accuracy": 0.6977471264367816, + "f1": 0.6759683891558872, + "f1_weighted": 0.7076021298138837 + }, + { + "accuracy": 0.700367816091954, + "f1": 0.6813029542216178, + "f1_weighted": 0.7098055535300781 + } + ] + } + ] + }, + "task_name": "SUN397" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json new file mode 100644 index 0000000000..f62a34d6ac --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "7e6af6a2499ad708618be868e1471eac0aca1168", + "evaluation_time": 215.99848175048828, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "accuracy": 0.5687816091954023, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5687816091954023 + } + ] + }, + "task_name": "SUN397ZeroShot" +} \ No newline at end of file From 77b0e35d13b8a74409c2f428417a9fd273c783d9 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 22 Jul 2024 10:59:41 +0000 Subject: [PATCH 024/154] correct SUN397 zero shot captions --- mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py | 2 +- .../SUN397ZeroShot.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py b/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py index a3852419b6..9207321872 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py @@ -54,6 +54,6 @@ def get_candidate_labels(self) -> list[str]: """ labels = [] for name in self.dataset["test"].features[self.label_column_name].names: - name = " ".join(name.split("/")[1:]).replace("_", " ") + name = " ".join(name.split("/")[2:]).replace("_", " ") labels.append(f"a photo of a {name}.") return labels diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json index f62a34d6ac..525fe2dbbe 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json @@ -1,17 +1,17 @@ { "dataset_revision": "7e6af6a2499ad708618be868e1471eac0aca1168", - "evaluation_time": 215.99848175048828, + "evaluation_time": 211.204514503479, "kg_co2_emissions": null, "mteb_version": "1.12.80", "scores": { "test": [ { - "accuracy": 0.5687816091954023, + "accuracy": 0.6106206896551725, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.5687816091954023 + "main_score": 0.6106206896551725 } ] }, From a8841b238365e37a6951db11fe996ce29d03ba16 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Wed, 24 Jul 2024 22:25:56 +0100 Subject: [PATCH 025/154] add baai bge vista --- mteb/models/__init__.py | 2 + mteb/models/vista_models.py | 180 ++++++++++++++++++ .../CIRRIT2TRetrieval.json | 158 +++++++++++++++ .../model_meta.json | 1 + .../CIRRIT2TRetrieval.json | 158 +++++++++++++++ .../model_meta.json | 1 + 6 files changed, 500 insertions(+) create mode 100644 mteb/models/vista_models.py create mode 100644 results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json create mode 100644 results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json create mode 100644 results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json create mode 100644 results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index c28cd5b746..15fbaded54 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -24,6 +24,7 @@ ru_sentence_models, salesforce_models, sentence_transformers_models, + vista_models, voyage_models, ) @@ -138,6 +139,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe ru_sentence_models, salesforce_models, sentence_transformers_models, + vista_models, voyage_models, google_models, ] diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py new file mode 100644 index 0000000000..519273ac64 --- /dev/null +++ b/mteb/models/vista_models.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +from functools import partial + +import torch +from FlagEmbedding.visual.modeling import Visualized_BGE +from PIL import Image +from tqdm import tqdm + +from mteb.model_meta import ModelMeta + + +class VisualizedBGEWrapper(Visualized_BGE): + def __init__( + self, + model_name_bge: str = None, + model_weight=None, + normlized: bool = True, + sentence_pooling_method: str = "cls", + negatives_cross_device: bool = False, + temperature: float = 0.02, + from_pretrained=None, + ): + super().__init__( + model_name_bge=model_name_bge, + model_weight=model_weight, + normlized=normlized, + sentence_pooling_method=sentence_pooling_method, + negatives_cross_device=negatives_cross_device, + temperature=temperature, + from_pretrained=from_pretrained, + ) + self.eval() + + def encode_text(self, texts): + """Currently override Visualized_BGE's the original implementation + to fix attention_mask & embedding_output dtype misalignment + """ + input_ids = texts["input_ids"] + attention_mask = texts["attention_mask"] + + input_shape = input_ids.size() + device = input_ids.device + + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) + + head_mask = [None] * self.depth + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape + ) + + embedding_output = self.bge_embeddings( + input_ids=input_ids, + position_ids=None, + token_type_ids=token_type_ids, + inputs_embeds=None, + past_key_values_length=0, + ) + + # this line is missing in vista, currently override "encode_text" only to fix this. + extended_attention_mask = extended_attention_mask.to(embedding_output.dtype) + + encoder_outputs = self.bge_encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=False, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ) + sequence_output = encoder_outputs[0] + + t_reps = self.sentence_embedding( + sequence_output, texts["attention_mask"] + ) # tensor: reps with pooling + if self.normlized: + t_reps = torch.nn.functional.normalize(t_reps, dim=-1) + return t_reps.contiguous() + + def encode(self, images=None, texts=None): + if images is not None: + if isinstance(images, list): + images = [ + self.preprocess_val( + img if isinstance(img, Image.Image) else Image.open(img) + ) + for img in images + ] + images = torch.stack(images) + if texts is not None: + texts = self.tokenizer(texts, return_tensors="pt", padding=True) + return self.encode_mm(images.to(self.device), texts.to(self.device)) + else: + return self.encode_image(images.to(self.device)) + else: + if texts is not None: + texts = self.tokenizer(texts, return_tensors="pt", padding=True) + return self.encode_text(texts.to(self.device)) + else: + return None + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + with torch.no_grad(): + batch_embeddings = self.encode(texts=batch_texts) + all_text_embeddings.append(batch_embeddings.cpu()) + return torch.cat(all_text_embeddings, dim=0) + + def get_image_embeddings(self, images: list[Image.Image], batch_size: int = 32): + all_image_embeddings = [] + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + with torch.no_grad(): + batch_embeddings = self.encode(images=batch_images) + all_image_embeddings.append(batch_embeddings.cpu()) + return torch.cat(all_image_embeddings, dim=0) + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] = None, + batch_size: int = 32, + ): + all_embeddings = [] + assert len(texts) == len(images) + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + batch_images = images[i : i + batch_size] + with torch.no_grad(): + batch_embeddings = self.encode(images=batch_images, texts=batch_texts) + all_embeddings.append(batch_embeddings.cpu()) + return torch.cat(all_embeddings, dim=0) + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + +Visualized_BGE_base = ModelMeta( + loader=partial( + VisualizedBGEWrapper, + model_name_bge="BAAI/bge-base-en-v1.5", + model_weight="visualized_base_en_V1.5.pth", + ), + name="BAAI/bge-visualized-base", + languages=["eng_Latn"], + open_source=True, + revision="98db10b10d22620010d06f11733346e1c98c34aa", + release_date="2024-06-06", +) + +Visualized_BGE_base = ModelMeta( + loader=partial( + VisualizedBGEWrapper, + model_name_bge="BAAI/bge-m3", + model_weight="visualized_m3.pth", + ), + name="BAAI/bge-visualized-m3", + languages=["eng_Latn"], + open_source=True, + revision="98db10b10d22620010d06f11733346e1c98c34aa", + release_date="2024-06-06", +) + +if __name__ == "__main__": + import mteb + + mdl = mteb.get_model(Visualized_BGE_base.name, Visualized_BGE_base.name.revision) + emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json b/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json new file mode 100644 index 0000000000..d64eff78ed --- /dev/null +++ b/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", + "evaluation_time": 170.71532011032104, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.15458, + "map_at_1": 0.01763, + "map_at_10": 0.10611, + "map_at_100": 0.11949, + "map_at_1000": 0.12077, + "map_at_20": 0.11329, + "map_at_3": 0.07918, + "map_at_5": 0.09356, + "mrr_at_1": 0.02182254196642686, + "mrr_at_10": 0.10637946785428773, + "mrr_at_100": 0.11975036168307936, + "mrr_at_1000": 0.12103153782524949, + "mrr_at_20": 0.11357094520966761, + "mrr_at_3": 0.07957633892885677, + "mrr_at_5": 0.09383293365307707, + "nauc_map_at_1000_diff1": -0.1398093587192006, + "nauc_map_at_1000_max": -0.011571981373852963, + "nauc_map_at_1000_std": 0.016242835917060484, + "nauc_map_at_100_diff1": -0.13932006959252094, + "nauc_map_at_100_max": -0.011471402747623032, + "nauc_map_at_100_std": 0.01571359503578373, + "nauc_map_at_10_diff1": -0.13667051086803197, + "nauc_map_at_10_max": -0.011921920148726927, + "nauc_map_at_10_std": 0.007179174808104729, + "nauc_map_at_1_diff1": -0.4350127493981227, + "nauc_map_at_1_max": -0.15259852263953608, + "nauc_map_at_1_std": 0.0970061577980624, + "nauc_map_at_20_diff1": -0.13618307626254747, + "nauc_map_at_20_max": -0.01109762645661432, + "nauc_map_at_20_std": 0.010507207905837899, + "nauc_map_at_3_diff1": -0.15118399083124537, + "nauc_map_at_3_max": -0.011017137393102167, + "nauc_map_at_3_std": -0.012575390001070445, + "nauc_map_at_5_diff1": -0.1447814690330949, + "nauc_map_at_5_max": -0.013474206558154681, + "nauc_map_at_5_std": 0.00019795325134277809, + "nauc_mrr_at_1000_diff1": -0.14068353124277078, + "nauc_mrr_at_1000_max": -0.01256002651149935, + "nauc_mrr_at_1000_std": 0.017068554858189054, + "nauc_mrr_at_100_diff1": -0.14019671870704672, + "nauc_mrr_at_100_max": -0.012457880868046342, + "nauc_mrr_at_100_std": 0.016528429573257345, + "nauc_mrr_at_10_diff1": -0.13737917913365197, + "nauc_mrr_at_10_max": -0.01310536806893329, + "nauc_mrr_at_10_std": 0.007847056495460521, + "nauc_mrr_at_1_diff1": -0.43519962731124523, + "nauc_mrr_at_1_max": -0.12632792482537586, + "nauc_mrr_at_1_std": 0.07526013026415497, + "nauc_mrr_at_20_diff1": -0.13713739546158307, + "nauc_mrr_at_20_max": -0.012046383412244238, + "nauc_mrr_at_20_std": 0.011333750815465542, + "nauc_mrr_at_3_diff1": -0.15256650395775093, + "nauc_mrr_at_3_max": -0.01285034024259544, + "nauc_mrr_at_3_std": -0.011306437596869121, + "nauc_mrr_at_5_diff1": -0.14586054456834394, + "nauc_mrr_at_5_max": -0.014513358256981268, + "nauc_mrr_at_5_std": 0.0018048979112297826, + "nauc_ndcg_at_1000_diff1": -0.14124685268905293, + "nauc_ndcg_at_1000_max": -0.009951401237880533, + "nauc_ndcg_at_1000_std": 0.057750817408098905, + "nauc_ndcg_at_100_diff1": -0.13265632530925167, + "nauc_ndcg_at_100_max": -0.0040308311439358245, + "nauc_ndcg_at_100_std": 0.05781381999577725, + "nauc_ndcg_at_10_diff1": -0.11414370292413088, + "nauc_ndcg_at_10_max": -0.004236214064962545, + "nauc_ndcg_at_10_std": 0.01241974149052717, + "nauc_ndcg_at_1_diff1": -0.433649345616075, + "nauc_ndcg_at_1_max": -0.12218048475091685, + "nauc_ndcg_at_1_std": 0.07249798765763336, + "nauc_ndcg_at_20_diff1": -0.11345335703573077, + "nauc_ndcg_at_20_max": -0.002148129039034098, + "nauc_ndcg_at_20_std": 0.021976127484270407, + "nauc_ndcg_at_3_diff1": -0.13328966349326943, + "nauc_ndcg_at_3_max": -0.0031192254944289468, + "nauc_ndcg_at_3_std": -0.01881648682054409, + "nauc_ndcg_at_5_diff1": -0.1266611748844269, + "nauc_ndcg_at_5_max": -0.006687530952537075, + "nauc_ndcg_at_5_std": -0.0005726739692759715, + "nauc_precision_at_1000_diff1": -0.3860299158889171, + "nauc_precision_at_1000_max": -0.08377411665854645, + "nauc_precision_at_1000_std": 0.4127832513033843, + "nauc_precision_at_100_diff1": -0.16470827851656045, + "nauc_precision_at_100_max": 0.005707104600219941, + "nauc_precision_at_100_std": 0.17540404629596543, + "nauc_precision_at_10_diff1": -0.09948700092707702, + "nauc_precision_at_10_max": 0.004571406776459961, + "nauc_precision_at_10_std": 0.01970423348809215, + "nauc_precision_at_1_diff1": -0.433649345616075, + "nauc_precision_at_1_max": -0.12218048475091685, + "nauc_precision_at_1_std": 0.07249798765763336, + "nauc_precision_at_20_diff1": -0.09926523747780691, + "nauc_precision_at_20_max": 0.008832676339450061, + "nauc_precision_at_20_std": 0.04036571606012274, + "nauc_precision_at_3_diff1": -0.12470591591535352, + "nauc_precision_at_3_max": 0.009684827445411333, + "nauc_precision_at_3_std": -0.029676750441070248, + "nauc_precision_at_5_diff1": -0.11820147403739824, + "nauc_precision_at_5_max": 0.0012091241273901022, + "nauc_precision_at_5_std": -0.0027024137371278, + "nauc_recall_at_1000_diff1": -0.37734982882041684, + "nauc_recall_at_1000_max": -0.09944561157796639, + "nauc_recall_at_1000_std": 0.5693141145347029, + "nauc_recall_at_100_diff1": -0.14823260134885624, + "nauc_recall_at_100_max": 0.0070610596185191644, + "nauc_recall_at_100_std": 0.18890427192465584, + "nauc_recall_at_10_diff1": -0.0839031142002386, + "nauc_recall_at_10_max": 0.005932685093745821, + "nauc_recall_at_10_std": 0.022366941779985956, + "nauc_recall_at_1_diff1": -0.4350127493981227, + "nauc_recall_at_1_max": -0.15259852263953608, + "nauc_recall_at_1_std": 0.0970061577980624, + "nauc_recall_at_20_diff1": -0.08368591470272047, + "nauc_recall_at_20_max": 0.01010481550534646, + "nauc_recall_at_20_std": 0.043779155070841055, + "nauc_recall_at_3_diff1": -0.10537518414674711, + "nauc_recall_at_3_max": 0.00995675521551123, + "nauc_recall_at_3_std": -0.028879437342584083, + "nauc_recall_at_5_diff1": -0.10152978106995561, + "nauc_recall_at_5_max": 0.002831093824884129, + "nauc_recall_at_5_std": -0.0006341965296450934, + "ndcg_at_1": 0.02158, + "ndcg_at_10": 0.15458, + "ndcg_at_100": 0.22831, + "ndcg_at_1000": 0.26227, + "ndcg_at_20": 0.18104, + "ndcg_at_3": 0.09835, + "ndcg_at_5": 0.12423, + "precision_at_1": 0.02158, + "precision_at_10": 0.03199, + "precision_at_100": 0.00686, + "precision_at_1000": 0.00095, + "precision_at_20": 0.02127, + "precision_at_3": 0.0542, + "precision_at_5": 0.04523, + "recall_at_1": 0.01763, + "recall_at_10": 0.31007, + "recall_at_100": 0.6753, + "recall_at_1000": 0.94245, + "recall_at_20": 0.41535, + "recall_at_3": 0.15348, + "recall_at_5": 0.21643 + } + ] + }, + "task_name": "CIRRIT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json b/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json new file mode 100644 index 0000000000..746dfa90fd --- /dev/null +++ b/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json @@ -0,0 +1 @@ +{"name": "BAAI/bge-visualized", "revision": "98db10b10d22620010d06f11733346e1c98c34aa", "release_date": "2024-06-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "VisualizedBGEWrapper"} \ No newline at end of file diff --git a/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json b/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json new file mode 100644 index 0000000000..f7bfc2691e --- /dev/null +++ b/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", + "evaluation_time": 626.7223904132843, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.15486, + "map_at_1": 0.04269, + "map_at_10": 0.11306, + "map_at_100": 0.1261, + "map_at_1000": 0.12744, + "map_at_20": 0.11979, + "map_at_3": 0.08855, + "map_at_5": 0.10125, + "mrr_at_1": 0.045563549160671464, + "mrr_at_10": 0.11386700163678554, + "mrr_at_100": 0.126761432509432, + "mrr_at_1000": 0.1280940096115024, + "mrr_at_20": 0.12048092549117054, + "mrr_at_3": 0.08960831334932043, + "mrr_at_5": 0.1021622701838525, + "nauc_map_at_1000_diff1": -0.07791148946970866, + "nauc_map_at_1000_max": 5.143872615612968e-06, + "nauc_map_at_1000_std": 0.04083408199460632, + "nauc_map_at_100_diff1": -0.07731489078138438, + "nauc_map_at_100_max": 3.918221027999972e-05, + "nauc_map_at_100_std": 0.0404276988719768, + "nauc_map_at_10_diff1": -0.07729492524522034, + "nauc_map_at_10_max": 0.0012864822248743548, + "nauc_map_at_10_std": 0.029465548613110498, + "nauc_map_at_1_diff1": -0.1736241959423541, + "nauc_map_at_1_max": -0.07374443402682516, + "nauc_map_at_1_std": -0.04852583988286722, + "nauc_map_at_20_diff1": -0.07680229687469181, + "nauc_map_at_20_max": 0.0010002432624082133, + "nauc_map_at_20_std": 0.03666477523677474, + "nauc_map_at_3_diff1": -0.08922170584380983, + "nauc_map_at_3_max": -0.008861115632143349, + "nauc_map_at_3_std": 0.01094507983539612, + "nauc_map_at_5_diff1": -0.08011065395630808, + "nauc_map_at_5_max": 0.0016127729720874162, + "nauc_map_at_5_std": 0.020427568697340345, + "nauc_mrr_at_1000_diff1": -0.07770330269715102, + "nauc_mrr_at_1000_max": 0.0009131779135089956, + "nauc_mrr_at_1000_std": 0.038979658723853165, + "nauc_mrr_at_100_diff1": -0.0771211519155922, + "nauc_mrr_at_100_max": 0.0009403242167022057, + "nauc_mrr_at_100_std": 0.03858151021299973, + "nauc_mrr_at_10_diff1": -0.07755851577891724, + "nauc_mrr_at_10_max": 0.0023019907938825845, + "nauc_mrr_at_10_std": 0.02763783882670773, + "nauc_mrr_at_1_diff1": -0.1785604338186492, + "nauc_mrr_at_1_max": -0.05415389829551078, + "nauc_mrr_at_1_std": -0.055220086707242946, + "nauc_mrr_at_20_diff1": -0.07661887536133825, + "nauc_mrr_at_20_max": 0.0019978449944686866, + "nauc_mrr_at_20_std": 0.03474258189581495, + "nauc_mrr_at_3_diff1": -0.08926232455695343, + "nauc_mrr_at_3_max": -0.007285948429449863, + "nauc_mrr_at_3_std": 0.008561270537733506, + "nauc_mrr_at_5_diff1": -0.08043180840287993, + "nauc_mrr_at_5_max": 0.0024895489049257397, + "nauc_mrr_at_5_std": 0.018656997997976786, + "nauc_ndcg_at_1000_diff1": -0.07338267741428946, + "nauc_ndcg_at_1000_max": 0.0030417252830778666, + "nauc_ndcg_at_1000_std": 0.09821559898845032, + "nauc_ndcg_at_100_diff1": -0.062286101465836415, + "nauc_ndcg_at_100_max": 0.004482736065877528, + "nauc_ndcg_at_100_std": 0.10277505818112917, + "nauc_ndcg_at_10_diff1": -0.061254511757178484, + "nauc_ndcg_at_10_max": 0.01189902750651278, + "nauc_ndcg_at_10_std": 0.04977362354287121, + "nauc_ndcg_at_1_diff1": -0.18063459374565402, + "nauc_ndcg_at_1_max": -0.05604355931036713, + "nauc_ndcg_at_1_std": -0.054415423322380765, + "nauc_ndcg_at_20_diff1": -0.060054754551758655, + "nauc_ndcg_at_20_max": 0.01086549847664498, + "nauc_ndcg_at_20_std": 0.07039195680705373, + "nauc_ndcg_at_3_diff1": -0.07753134923526465, + "nauc_ndcg_at_3_max": -0.0002772674733751852, + "nauc_ndcg_at_3_std": 0.02141475124849136, + "nauc_ndcg_at_5_diff1": -0.06571664406877105, + "nauc_ndcg_at_5_max": 0.013233449066821962, + "nauc_ndcg_at_5_std": 0.0342274046868592, + "nauc_precision_at_1000_diff1": -0.19656576649998095, + "nauc_precision_at_1000_max": 0.0008923436399845466, + "nauc_precision_at_1000_std": 0.48358706669108, + "nauc_precision_at_100_diff1": -0.052150565375847446, + "nauc_precision_at_100_max": 0.009548025017867385, + "nauc_precision_at_100_std": 0.26024269132004735, + "nauc_precision_at_10_diff1": -0.046839952542220155, + "nauc_precision_at_10_max": 0.0331080559803692, + "nauc_precision_at_10_std": 0.07899523358542766, + "nauc_precision_at_1_diff1": -0.18063459374565402, + "nauc_precision_at_1_max": -0.05604355931036713, + "nauc_precision_at_1_std": -0.054415423322380765, + "nauc_precision_at_20_diff1": -0.0462560523472062, + "nauc_precision_at_20_max": 0.029442616882690585, + "nauc_precision_at_20_std": 0.12415894016012202, + "nauc_precision_at_3_diff1": -0.06709941561395527, + "nauc_precision_at_3_max": 0.024028017239341025, + "nauc_precision_at_3_std": 0.03720152948553412, + "nauc_precision_at_5_diff1": -0.05180083321883926, + "nauc_precision_at_5_max": 0.03899872250699904, + "nauc_precision_at_5_std": 0.05454906284290042, + "nauc_recall_at_1000_diff1": -0.17163650330108518, + "nauc_recall_at_1000_max": -0.043117719901198975, + "nauc_recall_at_1000_std": 0.6414764487355223, + "nauc_recall_at_100_diff1": -0.036997744471389225, + "nauc_recall_at_100_max": 0.0010422392633025453, + "nauc_recall_at_100_std": 0.27951527516127295, + "nauc_recall_at_10_diff1": -0.03608087156730888, + "nauc_recall_at_10_max": 0.026974486642378233, + "nauc_recall_at_10_std": 0.08294573285460217, + "nauc_recall_at_1_diff1": -0.1736241959423541, + "nauc_recall_at_1_max": -0.07374443402682516, + "nauc_recall_at_1_std": -0.04852583988286722, + "nauc_recall_at_20_diff1": -0.03472320683684399, + "nauc_recall_at_20_max": 0.023755047800559374, + "nauc_recall_at_20_std": 0.13014236460370307, + "nauc_recall_at_3_diff1": -0.05544788258407132, + "nauc_recall_at_3_max": 0.015039569782739954, + "nauc_recall_at_3_std": 0.04220096699168211, + "nauc_recall_at_5_diff1": -0.04120717983373631, + "nauc_recall_at_5_max": 0.03222632622898795, + "nauc_recall_at_5_std": 0.05790997866890883, + "ndcg_at_1": 0.0458, + "ndcg_at_10": 0.15486, + "ndcg_at_100": 0.22705, + "ndcg_at_1000": 0.26306, + "ndcg_at_20": 0.17901, + "ndcg_at_3": 0.10353, + "ndcg_at_5": 0.12623, + "precision_at_1": 0.0458, + "precision_at_10": 0.02976, + "precision_at_100": 0.00658, + "precision_at_1000": 0.00094, + "precision_at_20": 0.01966, + "precision_at_3": 0.05068, + "precision_at_5": 0.04158, + "recall_at_1": 0.04269, + "recall_at_10": 0.28921, + "recall_at_100": 0.64796, + "recall_at_1000": 0.93237, + "recall_at_20": 0.38413, + "recall_at_3": 0.14584, + "recall_at_5": 0.20064 + } + ] + }, + "task_name": "CIRRIT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json b/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json new file mode 100644 index 0000000000..992116b866 --- /dev/null +++ b/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json @@ -0,0 +1 @@ +{"name": "BAAI/bge-visualized-m3", "revision": "98db10b10d22620010d06f11733346e1c98c34aa", "release_date": "2024-06-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "VisualizedBGEWrapper"} \ No newline at end of file From 38acf7c0a93fbf8bb7a85eb9d93a1732c38310fa Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Thu, 25 Jul 2024 04:20:19 +0100 Subject: [PATCH 026/154] add e5-v --- mteb/models/__init__.py | 2 + mteb/models/e5_v.py | 111 ++++++++++++ .../CIRRIT2TRetrieval.json | 158 ++++++++++++++++++ .../model_meta.json | 1 + 4 files changed, 272 insertions(+) create mode 100644 mteb/models/e5_v.py create mode 100644 results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/CIRRIT2TRetrieval.json create mode 100644 results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index 15fbaded54..e6d72bf0ab 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -14,6 +14,7 @@ cohere_models, e5_instruct, e5_models, + e5_v, google_models, gritlm_models, gte_models, @@ -128,6 +129,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe cohere_models, e5_instruct, e5_models, + e5_v, gritlm_models, gte_models, llm2vec_models, diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py new file mode 100644 index 0000000000..5b3a07bd05 --- /dev/null +++ b/mteb/models/e5_v.py @@ -0,0 +1,111 @@ +import torch +from PIL import Image +from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration +from functools import partial +from typing import Any +from tqdm import tqdm +from mteb.model_meta import ModelMeta + +class E5VWrapper: + def __init__( + self, + model_name: str, + composed_prompt = None, + **kwargs: Any, + ): + self.model_name = model_name + self.processor = LlavaNextProcessor.from_pretrained(model_name) + self.model = LlavaNextForConditionalGeneration.from_pretrained(model_name, **kwargs) + self.model.eval() + self.template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n' + self.text_prompt = self.template.format('\nSummary above sentence in one word: ') + self.img_prompt = self.template.format('\nSummary above image in one word: ') + if not composed_prompt: + # default composed embedding, to_do: move it to get_fused_embedding with "prompt_name" like MTEB text ones. + self.composed_prompt = self.template.format('[INST] Modify this image with "{}" Describe modified image in one word: [/INST]') + else: + self.composed_prompt = self.template.format(composed_prompt) + def get_text_embeddings(self, texts: list[str], batch_size: int = 8): + + all_text_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + text_inputs = self.processor([self.text_prompt.replace('', text) for text in batch_texts], return_tensors="pt", padding=True).to('cuda') + text_outputs = self.model(**text_inputs, output_hidden_states=True, return_dict=True).hidden_states[-1][:, -1, :] + all_text_embeddings.append(text_outputs.cpu()) + return torch.cat(all_text_embeddings, dim=0) + + def get_image_embeddings(self, images: list[Image.Image], batch_size: int = 8): + all_image_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + img_inputs = self.processor([self.img_prompt]*len(batch_images), batch_images, return_tensors="pt", padding=True).to('cuda') + image_outputs = self.model(**img_inputs, output_hidden_states=True, return_dict=True).hidden_states[-1][:, -1, :] + all_image_embeddings.append(image_outputs.cpu()) + return torch.cat(all_image_embeddings, dim=0) + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] = None, + batch_size: int = 8, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + all_fused_embeddings = [] + + if texts is not None and images is not None: + if len(texts) != len(images): + raise ValueError( + "The number of texts and images must have the same length" + ) + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_texts = texts[i : i + batch_size] + batch_images = images[i : i + batch_size] + prompts = [self.composed_prompt.format(text) for text in batch_texts] + inputs = self.processor(prompts, batch_images, return_tensors="pt", padding=True).to('cuda') + outputs = self.model(**inputs, output_hidden_states=True, return_dict=True).hidden_states[-1][:, -1, :] + all_fused_embeddings.append(outputs.cpu()) + return torch.cat(all_fused_embeddings, dim=0) + + elif texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + return text_embeddings + elif images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + return image_embeddings + +e5_v = ModelMeta( + loader=partial( + E5VWrapper, + model_name="royokong/e5-v", + torch_dtype = torch.float16, + device_map = "auto" + ), + name="royokong/e5-v", + languages=["eng_Latn"], + open_source=True, + revision="0c1f22679417b3ae925d779442221c40cd1861ab", + release_date="2024-07-17", +) + +if __name__ == "__main__": + import mteb + + mdl = mteb.get_model(e5_v.name, e5_v.revision) + emb = mdl.get_text_embeddings(["Hello, world!"]) \ No newline at end of file diff --git a/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/CIRRIT2TRetrieval.json b/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/CIRRIT2TRetrieval.json new file mode 100644 index 0000000000..a91d4da661 --- /dev/null +++ b/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/CIRRIT2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", + "evaluation_time": 13414.161827325821, + "kg_co2_emissions": null, + "mteb_version": "1.12.67", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.2768, + "map_at_1": 0.12414, + "map_at_10": 0.22102, + "map_at_100": 0.23505, + "map_at_1000": 0.23583, + "map_at_20": 0.22904, + "map_at_3": 0.18959, + "map_at_5": 0.20727, + "mrr_at_1": 0.12541966426858514, + "mrr_at_10": 0.2214481747934993, + "mrr_at_100": 0.23534746672978193, + "mrr_at_1000": 0.23613361179255168, + "mrr_at_20": 0.229363862062874, + "mrr_at_3": 0.19044764188649146, + "mrr_at_5": 0.20782174260591652, + "nauc_map_at_1000_diff1": 0.051644929630880985, + "nauc_map_at_1000_max": -0.028815405528502636, + "nauc_map_at_1000_std": -0.020198524699442843, + "nauc_map_at_100_diff1": 0.05124473143253365, + "nauc_map_at_100_max": -0.028632119897176594, + "nauc_map_at_100_std": -0.019628560449452832, + "nauc_map_at_10_diff1": 0.05562870137097447, + "nauc_map_at_10_max": -0.028495628556100824, + "nauc_map_at_10_std": -0.02536320038491407, + "nauc_map_at_1_diff1": 0.0842856352950554, + "nauc_map_at_1_max": -0.06453708554638175, + "nauc_map_at_1_std": -0.05526923569827222, + "nauc_map_at_20_diff1": 0.052613598616759794, + "nauc_map_at_20_max": -0.029475951982590367, + "nauc_map_at_20_std": -0.023762823555117794, + "nauc_map_at_3_diff1": 0.05880862286832309, + "nauc_map_at_3_max": -0.03736131717268504, + "nauc_map_at_3_std": -0.042634580624996886, + "nauc_map_at_5_diff1": 0.05860660116822883, + "nauc_map_at_5_max": -0.03508785748513017, + "nauc_map_at_5_std": -0.03541676336950194, + "nauc_mrr_at_1000_diff1": 0.05207102176098774, + "nauc_mrr_at_1000_max": -0.027027039710112655, + "nauc_mrr_at_1000_std": -0.018340925011646518, + "nauc_mrr_at_100_diff1": 0.05167035757467952, + "nauc_mrr_at_100_max": -0.026832081188300435, + "nauc_mrr_at_100_std": -0.017754235557587542, + "nauc_mrr_at_10_diff1": 0.05607174629690726, + "nauc_mrr_at_10_max": -0.026650963919061615, + "nauc_mrr_at_10_std": -0.023439519375587647, + "nauc_mrr_at_1_diff1": 0.07953246778996732, + "nauc_mrr_at_1_max": -0.06299248593212067, + "nauc_mrr_at_1_std": -0.054162643797493096, + "nauc_mrr_at_20_diff1": 0.052842729664946166, + "nauc_mrr_at_20_max": -0.02756510867227389, + "nauc_mrr_at_20_std": -0.021727381415748443, + "nauc_mrr_at_3_diff1": 0.05837293843551822, + "nauc_mrr_at_3_max": -0.03597640902892992, + "nauc_mrr_at_3_std": -0.03964104042267918, + "nauc_mrr_at_5_diff1": 0.05833672640078505, + "nauc_mrr_at_5_max": -0.03310995104869811, + "nauc_mrr_at_5_std": -0.03382008427172164, + "nauc_ndcg_at_1000_diff1": 0.03606123281509481, + "nauc_ndcg_at_1000_max": -0.015599943386736242, + "nauc_ndcg_at_1000_std": 0.012256741773024273, + "nauc_ndcg_at_100_diff1": 0.02695005113788398, + "nauc_ndcg_at_100_max": -0.0070994293461191915, + "nauc_ndcg_at_100_std": 0.03257925388016779, + "nauc_ndcg_at_10_diff1": 0.04716531655680059, + "nauc_ndcg_at_10_max": -0.011574560261841896, + "nauc_ndcg_at_10_std": -0.002845570474735191, + "nauc_ndcg_at_1_diff1": 0.07696384973044536, + "nauc_ndcg_at_1_max": -0.0645711978776169, + "nauc_ndcg_at_1_std": -0.05613824052026943, + "nauc_ndcg_at_20_diff1": 0.037171068658800745, + "nauc_ndcg_at_20_max": -0.014733746284082054, + "nauc_ndcg_at_20_std": 0.0025827030969972587, + "nauc_ndcg_at_3_diff1": 0.05393062096446941, + "nauc_ndcg_at_3_max": -0.03021653468760562, + "nauc_ndcg_at_3_std": -0.038667943978515695, + "nauc_ndcg_at_5_diff1": 0.0530747472819436, + "nauc_ndcg_at_5_max": -0.02632200815887294, + "nauc_ndcg_at_5_std": -0.02619673280345486, + "nauc_precision_at_1000_diff1": -0.31337858572216665, + "nauc_precision_at_1000_max": 0.040106460630914706, + "nauc_precision_at_1000_std": 0.39471123050414453, + "nauc_precision_at_100_diff1": -0.13314928522696645, + "nauc_precision_at_100_max": 0.09704344396619104, + "nauc_precision_at_100_std": 0.32833150871848465, + "nauc_precision_at_10_diff1": 0.016706080957278685, + "nauc_precision_at_10_max": 0.028890104977194225, + "nauc_precision_at_10_std": 0.054075958829982056, + "nauc_precision_at_1_diff1": 0.07696384973044536, + "nauc_precision_at_1_max": -0.0645711978776169, + "nauc_precision_at_1_std": -0.05613824052026943, + "nauc_precision_at_20_diff1": -0.01980517203283842, + "nauc_precision_at_20_max": 0.02074496687832344, + "nauc_precision_at_20_std": 0.07551452450628639, + "nauc_precision_at_3_diff1": 0.033455817663258815, + "nauc_precision_at_3_max": -0.01298097522003606, + "nauc_precision_at_3_std": -0.02933261669672727, + "nauc_precision_at_5_diff1": 0.030996712811149657, + "nauc_precision_at_5_max": -0.00630530526298891, + "nauc_precision_at_5_std": -0.004748881667681358, + "nauc_recall_at_1000_diff1": -0.3250596534910208, + "nauc_recall_at_1000_max": 0.12736884877407081, + "nauc_recall_at_1000_std": 0.7188677940311853, + "nauc_recall_at_100_diff1": -0.1174097854649418, + "nauc_recall_at_100_max": 0.11271080733878815, + "nauc_recall_at_100_std": 0.36247172343121803, + "nauc_recall_at_10_diff1": 0.02773456006330092, + "nauc_recall_at_10_max": 0.030773925835657637, + "nauc_recall_at_10_std": 0.05648260910488339, + "nauc_recall_at_1_diff1": 0.0842856352950554, + "nauc_recall_at_1_max": -0.06453708554638175, + "nauc_recall_at_1_std": -0.05526923569827222, + "nauc_recall_at_20_diff1": -0.007102304486688877, + "nauc_recall_at_20_max": 0.0238251830397759, + "nauc_recall_at_20_std": 0.07965089580706912, + "nauc_recall_at_3_diff1": 0.043901759927141515, + "nauc_recall_at_3_max": -0.013952110738630563, + "nauc_recall_at_3_std": -0.028953041657461525, + "nauc_recall_at_5_diff1": 0.04156482887300926, + "nauc_recall_at_5_max": -0.006295432031863492, + "nauc_recall_at_5_std": -0.0038079303663845496, + "ndcg_at_1": 0.1259, + "ndcg_at_10": 0.2768, + "ndcg_at_100": 0.34982, + "ndcg_at_1000": 0.37092, + "ndcg_at_20": 0.30577, + "ndcg_at_3": 0.21148, + "ndcg_at_5": 0.24328, + "precision_at_1": 0.1259, + "precision_at_10": 0.04624, + "precision_at_100": 0.00818, + "precision_at_1000": 0.00099, + "precision_at_20": 0.02891, + "precision_at_3": 0.09297, + "precision_at_5": 0.07141, + "recall_at_1": 0.12414, + "recall_at_10": 0.45544, + "recall_at_100": 0.80791, + "recall_at_1000": 0.9741, + "recall_at_20": 0.56954, + "recall_at_3": 0.2741, + "recall_at_5": 0.35132 + } + ] + }, + "task_name": "CIRRIT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json b/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json new file mode 100644 index 0000000000..da7a8a862e --- /dev/null +++ b/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json @@ -0,0 +1 @@ +{"name": "royokong/e5-v", "revision": "0c1f22679417b3ae925d779442221c40cd1861ab", "release_date": "2024-07-17", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "E5VWrapper"} \ No newline at end of file From da7c8ba9b0d03233122fb7f82cf497815d2fd798 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Thu, 25 Jul 2024 16:38:04 +0100 Subject: [PATCH 027/154] linting --- mteb/models/e5_v.py | 81 +++++++++++++++++++++++++++++++-------------- 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index 5b3a07bd05..5cdbb4200d 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -1,50 +1,74 @@ -import torch -from PIL import Image -from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration from functools import partial from typing import Any + +import torch +from PIL import Image from tqdm import tqdm +from transformers import LlavaNextForConditionalGeneration, LlavaNextProcessor + from mteb.model_meta import ModelMeta + class E5VWrapper: def __init__( self, model_name: str, - composed_prompt = None, + composed_prompt=None, **kwargs: Any, ): self.model_name = model_name self.processor = LlavaNextProcessor.from_pretrained(model_name) - self.model = LlavaNextForConditionalGeneration.from_pretrained(model_name, **kwargs) + self.model = LlavaNextForConditionalGeneration.from_pretrained( + model_name, **kwargs + ) self.model.eval() - self.template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n' - self.text_prompt = self.template.format('\nSummary above sentence in one word: ') - self.img_prompt = self.template.format('\nSummary above image in one word: ') + self.template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n" + self.text_prompt = self.template.format( + "\nSummary above sentence in one word: " + ) + self.img_prompt = self.template.format( + "\nSummary above image in one word: " + ) if not composed_prompt: # default composed embedding, to_do: move it to get_fused_embedding with "prompt_name" like MTEB text ones. - self.composed_prompt = self.template.format('[INST] Modify this image with "{}" Describe modified image in one word: [/INST]') + self.composed_prompt = self.template.format( + '[INST] Modify this image with "{}" Describe modified image in one word: [/INST]' + ) else: self.composed_prompt = self.template.format(composed_prompt) + def get_text_embeddings(self, texts: list[str], batch_size: int = 8): - all_text_embeddings = [] - + with torch.no_grad(): for i in tqdm(range(0, len(texts), batch_size)): batch_texts = texts[i : i + batch_size] - text_inputs = self.processor([self.text_prompt.replace('', text) for text in batch_texts], return_tensors="pt", padding=True).to('cuda') - text_outputs = self.model(**text_inputs, output_hidden_states=True, return_dict=True).hidden_states[-1][:, -1, :] + text_inputs = self.processor( + [self.text_prompt.replace("", text) for text in batch_texts], + return_tensors="pt", + padding=True, + ).to("cuda") + text_outputs = self.model( + **text_inputs, output_hidden_states=True, return_dict=True + ).hidden_states[-1][:, -1, :] all_text_embeddings.append(text_outputs.cpu()) return torch.cat(all_text_embeddings, dim=0) def get_image_embeddings(self, images: list[Image.Image], batch_size: int = 8): all_image_embeddings = [] - + with torch.no_grad(): for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] - img_inputs = self.processor([self.img_prompt]*len(batch_images), batch_images, return_tensors="pt", padding=True).to('cuda') - image_outputs = self.model(**img_inputs, output_hidden_states=True, return_dict=True).hidden_states[-1][:, -1, :] + img_inputs = self.processor( + [self.img_prompt] * len(batch_images), + batch_images, + return_tensors="pt", + padding=True, + ).to("cuda") + image_outputs = self.model( + **img_inputs, output_hidden_states=True, return_dict=True + ).hidden_states[-1][:, -1, :] all_image_embeddings.append(image_outputs.cpu()) return torch.cat(all_image_embeddings, dim=0) @@ -65,7 +89,7 @@ def get_fused_embeddings( ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") - + all_fused_embeddings = [] if texts is not None and images is not None: @@ -77,25 +101,32 @@ def get_fused_embeddings( for i in tqdm(range(0, len(images), batch_size)): batch_texts = texts[i : i + batch_size] batch_images = images[i : i + batch_size] - prompts = [self.composed_prompt.format(text) for text in batch_texts] - inputs = self.processor(prompts, batch_images, return_tensors="pt", padding=True).to('cuda') - outputs = self.model(**inputs, output_hidden_states=True, return_dict=True).hidden_states[-1][:, -1, :] + prompts = [ + self.composed_prompt.format(text) for text in batch_texts + ] + inputs = self.processor( + prompts, batch_images, return_tensors="pt", padding=True + ).to("cuda") + outputs = self.model( + **inputs, output_hidden_states=True, return_dict=True + ).hidden_states[-1][:, -1, :] all_fused_embeddings.append(outputs.cpu()) return torch.cat(all_fused_embeddings, dim=0) - + elif texts is not None: text_embeddings = self.get_text_embeddings(texts, batch_size) return text_embeddings elif images is not None: image_embeddings = self.get_image_embeddings(images, batch_size) return image_embeddings - + + e5_v = ModelMeta( loader=partial( E5VWrapper, model_name="royokong/e5-v", - torch_dtype = torch.float16, - device_map = "auto" + torch_dtype=torch.float16, + device_map="auto", ), name="royokong/e5-v", languages=["eng_Latn"], @@ -108,4 +139,4 @@ def get_fused_embeddings( import mteb mdl = mteb.get_model(e5_v.name, e5_v.revision) - emb = mdl.get_text_embeddings(["Hello, world!"]) \ No newline at end of file + emb = mdl.get_text_embeddings(["Hello, world!"]) From fd64f753838e0a20150e6e3d24b22d8b160c0012 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 26 Jul 2024 19:24:36 +0100 Subject: [PATCH 028/154] memory issues for image linear probe & zeroshot --- .../Image/AbsTaskImageClassification.py | 47 ++++++--- .../Image/AbsTaskZeroshotClassification.py | 4 +- .../Image/ClassificationEvaluator.py | 99 ++++++++++++++----- .../Image/ZeroshotClassificationEvaluator.py | 46 ++++++++- mteb/models/clip_models.py | 33 +++++-- 5 files changed, 175 insertions(+), 54 deletions(-) diff --git a/mteb/abstasks/Image/AbsTaskImageClassification.py b/mteb/abstasks/Image/AbsTaskImageClassification.py index 9b69e67f9e..c5c28804e3 100644 --- a/mteb/abstasks/Image/AbsTaskImageClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageClassification.py @@ -129,10 +129,10 @@ def _evaluate_subset( ) # Bootstrap `self.samples_per_label` samples per label for each split X_sampled, y_sampled, idxs = self._undersample_data( - train_split[self.image_column_name], # type: ignore - train_split[self.label_column_name], # type: ignore + train_split, + self.label_column_name, self.samples_per_label, - idxs, + idxs=idxs, ) if self.method == "kNN": @@ -159,8 +159,13 @@ def _evaluate_subset( evaluator = ImagelogRegClassificationEvaluator( X_sampled, y_sampled, - eval_split[self.image_column_name], # type: ignore - eval_split[self.label_column_name], # type: ignore + # instead of slicing out the image column (huge memory), + # we pass the whole eval split in and do dataloader stype operations inside. + eval_split, + self.image_column_name, + self.label_column_name, + # eval_split[self.image_column_name], # type: ignore + # eval_split[self.label_column_name], # type: ignore task_name=self.metadata.name, encode_kwargs=encode_kwargs, **params, @@ -177,17 +182,29 @@ def _evaluate_subset( avg_scores["scores_per_experiment"] = scores return avg_scores - def _undersample_data(self, X, y, samples_per_label: int, idxs=None): - """Undersample data to have samples_per_label samples of each label""" - X_sampled = [] - y_sampled = [] + def _undersample_data( + self, dataset_split, label_column_name, samples_per_label, idxs=None + ): + """Undersample data to have samples_per_label samples of each label + without loading all images into memory. + """ if idxs is None: - idxs = np.arange(len(y)) + idxs = np.arange(len(dataset_split)) np.random.shuffle(idxs) + if not isinstance(idxs, list): + idxs = idxs.tolist() label_counter = defaultdict(int) + selected_indices = [] + for i in idxs: - if label_counter[y[i]] < samples_per_label: - X_sampled.append(X[i]) - y_sampled.append(y[i]) - label_counter[y[i]] += 1 - return X_sampled, y_sampled, idxs + label = dataset_split[i][label_column_name] + if label_counter[label] < samples_per_label: + selected_indices.append(i) + label_counter[label] += 1 + + undersampled_dataset = dataset_split.select(selected_indices) + return ( + undersampled_dataset[self.image_column_name], + undersampled_dataset[self.label_column_name], + idxs, + ) diff --git a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py index c2581f6214..a70480de38 100644 --- a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py +++ b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py @@ -43,7 +43,9 @@ def _evaluate_subset( candidate_labels = self.get_candidate_labels() evaluator = ZeroshotClassificationEvaluator( - dataset[self.image_column_name], + dataset, + self.image_column_name, + # dataset[self.image_column_name],#broken into dataset and self.image_column_name dataset[self.label_column_name], candidate_labels, task_name=self.metadata.name, diff --git a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py index 753b584992..d62af4d17a 100644 --- a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py @@ -13,6 +13,8 @@ ) from sklearn.neighbors import KNeighborsClassifier from torch import Tensor +from torch.utils.data import DataLoader +from torchvision import transforms from mteb.encoder_interface import Encoder @@ -25,13 +27,38 @@ def dot_distance(a: np.ndarray, b: np.ndarray) -> float: return -np.dot(a, b) +transform = transforms.Compose([transforms.PILToTensor()]) + + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, hf_dataset, image_column_name: str = "image", transform=None): + self.dataset = hf_dataset + self.transform = transform + self.image_column_name = image_column_name + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + image = self.dataset[idx][self.image_column_name] + if image.mode != "RGB": + image = image.convert("RGB") + image = self.transform(image) + return image + + +def custom_collate_fn(batch): + return batch + + class ImagekNNClassificationEvaluator(Evaluator): def __init__( self, images_train, y_train, - images_test, - y_test, + dataset_test, + image_column_name, + label_column_name, task_name: str | None = None, k: int = 1, encode_kwargs: dict[str, Any] = {}, @@ -39,16 +66,18 @@ def __init__( **kwargs, ): super().__init__(**kwargs) + if limit is not None: images_train = images_train[:limit] y_train = y_train[:limit] - images_test = images_test[:limit] - y_test = y_test[:limit] + dataset_test = dataset_test[:limit] + self.images_train = images_train self.y_train = y_train - self.images_test = images_test - self.y_test = y_test - + self.dataset_test = ImageDataset( + dataset_test, image_column_name=image_column_name, transform=transform + ) + self.y_test = dataset_test[label_column_name] self.task_name = task_name self.encode_kwargs = encode_kwargs @@ -65,9 +94,15 @@ def __call__(self, model, test_cache=None): X_train = model.get_image_embeddings( self.images_train, batch_size=self.encode_kwargs["batch_size"] ) + dataloader = DataLoader( + self.dataset_test, + batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + num_workers=16, + ) if test_cache is None: X_test = model.get_image_embeddings( - self.images_test, batch_size=self.encode_kwargs["batch_size"] + dataloader, batch_size=self.encode_kwargs["batch_size"] ) test_cache = X_test else: @@ -99,8 +134,9 @@ def __init__( self, images_train, y_train, - images_test, - y_test, + dataset_test, + image_column_name, + label_column_name, task_name: str, k: int = 1, encode_kwargs: dict[str, Any] = {}, @@ -111,14 +147,14 @@ def __init__( if limit is not None: images_train = images_train[:limit] y_train = y_train[:limit] - images_test = images_test[:limit] - y_test = y_test[:limit] + dataset_test = dataset_test[:limit] self.images_train = images_train self.y_train = y_train - self.images_test = images_test - self.y_test = y_test - + self.dataset_test = ImageDataset( + dataset_test, image_column_name=image_column_name, transform=transform + ) + self.y_test = dataset_test[label_column_name] self.task_name = task_name self.encode_kwargs = encode_kwargs @@ -136,9 +172,15 @@ def __call__(self, model: Encoder, test_cache=None): self.images_train, batch_size=self.encode_kwargs["batch_size"] ) + dataloader = DataLoader( + self.dataset_test, + batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + num_workers=16, + ) if test_cache is None: X_test = model.get_image_embeddings( - self.images_test, batch_size=self.encode_kwargs["batch_size"] + dataloader, batch_size=self.encode_kwargs["batch_size"] ) test_cache = X_test else: @@ -245,8 +287,9 @@ def __init__( self, images_train, y_train, - images_test, - y_test, + dataset_test, + image_column_name, + label_column_name, task_name: str, max_iter: int = 100, encode_kwargs: dict[str, Any] = {}, @@ -262,12 +305,14 @@ def __init__( if limit is not None: images_train = images_train[:limit] y_train = y_train[:limit] - images_test = images_test[:limit] - y_test = y_test[:limit] + dataset_test = dataset_test[:limit] + self.images_train = images_train self.y_train = y_train - self.images_test = images_test - self.y_test = y_test + self.dataset_test = ImageDataset( + dataset_test, image_column_name=image_column_name, transform=transform + ) + self.y_test = dataset_test[label_column_name] self.max_iter = max_iter self.task_name = task_name @@ -283,10 +328,16 @@ def __call__(self, model, test_cache=None): X_train = model.get_image_embeddings( self.images_train, batch_size=self.encode_kwargs["batch_size"] ) - + dataloader = DataLoader( + self.dataset_test, + batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=16, + ) if test_cache is None: X_test = model.get_image_embeddings( - self.images_test, batch_size=self.encode_kwargs["batch_size"] + dataloader, batch_size=self.encode_kwargs["batch_size"] ) test_cache = X_test else: diff --git a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py index 03fb0f3f10..3e10b88cd2 100644 --- a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py @@ -3,8 +3,10 @@ import logging from typing import Any -from PIL import Image +import torch from sklearn import metrics +from torch.utils.data import DataLoader +from torchvision import transforms from mteb.encoder_interface import Encoder @@ -12,18 +14,45 @@ logger = logging.getLogger(__name__) +transform = transforms.Compose([transforms.PILToTensor()]) + + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, hf_dataset, image_column_name: str = "image", transform=None): + self.dataset = hf_dataset + self.transform = transform + self.image_column_name = image_column_name + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + image = self.dataset[idx][self.image_column_name] + if image.mode != "RGB": + image = image.convert("RGB") + image = self.transform(image) + return image + + +def custom_collate_fn(batch): + return batch + class ZeroshotClassificationEvaluator(Evaluator): def __init__( self, - images: list[Image.Image], + dataset, + image_column_name: str, labels: list[int], candidate_labels: list[str], task_name: str | None = None, **kwargs, ): super().__init__(**kwargs) - self.images = images + self.dataset = ImageDataset( + dataset, image_column_name=image_column_name, transform=transform + ) + self.image_column_name = image_column_name self.labels = labels self.candidate_labels = candidate_labels self.task_name = task_name @@ -32,11 +61,20 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): if "batch_size" not in encode_kwargs: encode_kwargs["batch_size"] = 32 + dataloader = DataLoader( + self.dataset, + batch_size=encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=16, + ) + text_embeddings = model.get_text_embeddings( self.candidate_labels, batch_size=encode_kwargs["batch_size"] ) + image_embeddings = model.get_image_embeddings( - self.images, batch_size=encode_kwargs["batch_size"] + dataloader, batch_size=encode_kwargs["batch_size"] ) probs = model.calculate_probs(text_embeddings, image_embeddings) predictions = probs.argmax(dim=1) diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index 733d83c927..bb2c61e338 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -5,6 +5,7 @@ import torch from PIL import Image +from torch.utils.data import DataLoader from tqdm import tqdm from transformers import AutoModel, AutoProcessor @@ -48,18 +49,30 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): all_text_embeddings = torch.cat(all_text_embeddings, dim=0) return all_text_embeddings - def get_image_embeddings(self, images: list[Image.Image], batch_size: int = 32): + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): all_image_embeddings = [] - with torch.no_grad(): - for i in tqdm(range(0, len(images), batch_size)): - batch_images = images[i : i + batch_size] - inputs = self.processor( - images=batch_images, return_tensors="pt", padding=True - ) - inputs = {k: v.to(self.device) for k, v in inputs.items()} - image_outputs = self.model.get_image_features(**inputs) - all_image_embeddings.append(image_outputs.cpu()) + if isinstance(images, DataLoader): + with torch.no_grad(): + for batch in tqdm(images): + inputs = self.processor( + images=batch, return_tensors="pt", padding=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model.get_image_features(**inputs) + all_image_embeddings.append(image_outputs.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + inputs = self.processor( + images=batch_images, return_tensors="pt", padding=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model.get_image_features(**inputs) + all_image_embeddings.append(image_outputs.cpu()) all_image_embeddings = torch.cat(all_image_embeddings, dim=0) return all_image_embeddings From 2ad3a07fdba5d55dbde6cabec073b4f32548e4a4 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 26 Jul 2024 19:28:07 +0100 Subject: [PATCH 029/154] kknn linear probe arguments --- mteb/abstasks/Image/AbsTaskImageClassification.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mteb/abstasks/Image/AbsTaskImageClassification.py b/mteb/abstasks/Image/AbsTaskImageClassification.py index c5c28804e3..89899aefdd 100644 --- a/mteb/abstasks/Image/AbsTaskImageClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageClassification.py @@ -139,8 +139,9 @@ def _evaluate_subset( evaluator = ImagekNNClassificationEvaluator( X_sampled, y_sampled, - eval_split[self.image_column_name], # type: ignore - eval_split[self.label_column_name], # type: ignore + eval_split, + self.image_column_name, + self.label_column_name, task_name=self.metadata.name, encode_kwargs=encode_kwargs, **params, @@ -149,8 +150,9 @@ def _evaluate_subset( evaluator = ImagekNNClassificationEvaluatorPytorch( X_sampled, y_sampled, - eval_split[self.image_column_name], # type: ignore - eval_split[self.label_column_name], # type: ignore + eval_split, + self.image_column_name, + self.label_column_name, task_name=self.metadata.name, encode_kwargs=encode_kwargs, **params, @@ -164,8 +166,6 @@ def _evaluate_subset( eval_split, self.image_column_name, self.label_column_name, - # eval_split[self.image_column_name], # type: ignore - # eval_split[self.label_column_name], # type: ignore task_name=self.metadata.name, encode_kwargs=encode_kwargs, **params, From 12014412f2f0afddb002c385bab89d7c0f8780b5 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 26 Jul 2024 19:33:01 +0100 Subject: [PATCH 030/154] del comments --- mteb/abstasks/Image/AbsTaskImageClassification.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mteb/abstasks/Image/AbsTaskImageClassification.py b/mteb/abstasks/Image/AbsTaskImageClassification.py index 89899aefdd..958e5db424 100644 --- a/mteb/abstasks/Image/AbsTaskImageClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageClassification.py @@ -161,8 +161,6 @@ def _evaluate_subset( evaluator = ImagelogRegClassificationEvaluator( X_sampled, y_sampled, - # instead of slicing out the image column (huge memory), - # we pass the whole eval split in and do dataloader stype operations inside. eval_split, self.image_column_name, self.label_column_name, From c0f0021c03451179473a8b2e2124f0d367669835 Mon Sep 17 00:00:00 2001 From: Imene Kerboua <33312980+imenelydiaker@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:39:28 +0200 Subject: [PATCH 031/154] Add some classification and ZeroShot classification tasks (#1107) * Add Country211 classification task * Add imagenet1k classification task * Add UCF101 classification task * Add PatchCamelyon Classification task * Add GTSRB classification task * Add GSTRB Zero Shot Classification * Add country211 zero shot classification * Add results for classification tasks * Add zero shot classification tasks * Add PatchCamelyon tasks and results * Add linting * Add results and fix prompts for zero shot * Add results * Add results and linting --- mteb/abstasks/TaskMetadata.py | 2 + mteb/models/vista_models.py | 2 +- .../Image/ImageClassification/__init__.py | 5 + .../eng/Country211Classification.py | 46 + .../eng/GTSRBClassification.py | 50 + .../ImageClassification/eng/Imagenet1k.py | 47 + .../eng/PatchCamelyonClassification.py | 60 + .../eng/UCF101Classification.py | 50 + .../Image/ZeroshotClassification/__init__.py | 5 + .../ZeroshotClassification/eng/Country211.py | 55 + .../Image/ZeroshotClassification/eng/GTSRB.py | 60 + .../ZeroshotClassification/eng/Imagenet1k.py | 56 + .../eng/PatchCamelyon.py | 69 ++ .../ZeroshotClassification/eng/UCF101.py | 56 + .../eng/templates/Country211_labels.txt | 211 ++++ .../eng/templates/GTSRB_labels.txt | 43 + .../eng/templates/Imagenet1k_labels.txt | 1000 +++++++++++++++++ .../eng/templates/PatchCamelyon_labels.txt | 2 + .../Country211.json | 28 + .../Country211ZeroShot.json | 19 + .../GTSRB.json | 28 + .../GTSRBZeroShot.json | 19 + .../Imagenet1k.json | 28 + .../Imagenet1kZeroShot.json | 19 + .../PatchCamelyon.json | 32 + .../PatchCamelyonZeroShot.json | 19 + .../UCF101.json | 28 + .../UCF101ZeroShot.json | 19 + 28 files changed, 2057 insertions(+), 1 deletion(-) create mode 100644 mteb/tasks/Image/ImageClassification/eng/Country211Classification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py create mode 100644 mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/Country211.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/GTSRB.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/templates/Country211_labels.txt create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/templates/GTSRB_labels.txt create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/templates/Imagenet1k_labels.txt create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/templates/PatchCamelyon_labels.txt create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211ZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRB.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRBZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1k.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1kZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyon.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyonZeroShot.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101ZeroShot.json diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 16219482b1..4dc9e9b874 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -44,6 +44,8 @@ "Caption Pairing", "Emotion recognition", "Textures recognition", + "Activity recognition", + "Tumor detection", ] TASK_DOMAIN = Literal[ diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py index 519273ac64..35e6cc61b7 100644 --- a/mteb/models/vista_models.py +++ b/mteb/models/vista_models.py @@ -3,10 +3,10 @@ from functools import partial import torch -from FlagEmbedding.visual.modeling import Visualized_BGE from PIL import Image from tqdm import tqdm +from FlagEmbedding.visual.modeling import Visualized_BGE from mteb.model_meta import ModelMeta diff --git a/mteb/tasks/Image/ImageClassification/__init__.py b/mteb/tasks/Image/ImageClassification/__init__.py index 152406c954..c5a82f357d 100644 --- a/mteb/tasks/Image/ImageClassification/__init__.py +++ b/mteb/tasks/Image/ImageClassification/__init__.py @@ -3,15 +3,20 @@ from .eng.BirdsnapClassification import * from .eng.Caltech101Classification import * from .eng.CIFAR import * +from .eng.Country211Classification import * from .eng.DTDClassification import * from .eng.EuroSATClassification import * from .eng.FER2013Classification import * from .eng.FGVCAircraftClassification import * from .eng.Food101Classification import * +from .eng.GTSRBClassification import * +from .eng.Imagenet1k import * from .eng.MNISTClassification import * from .eng.OxfordFlowersClassification import * from .eng.OxfordPetsClassification import * +from .eng.PatchCamelyonClassification import * from .eng.RESISC45Classification import * from .eng.StanfordCarsClassification import * from .eng.STL10Classification import * from .eng.SUN397Classification import * +from .eng.UCF101Classification import * diff --git a/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py new file mode 100644 index 0000000000..f3a501828e --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class Country211Classification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="Country211", + description="Classifying images of 211 countries.", + reference="https://huggingface.co/datasets/clip-benchmark/wds_country211", + dataset={ + "path": "clip-benchmark/wds_country211", + "revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2020-01-01", + "2021-02-26", + ), # Estimated range for the collection of reviews + domains=["Scene"], + task_subtypes=["Scene recognition"], + license="CC BY-SA 4.0", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@article{radford2021learning, + title={Learning Transferable Visual Models From Natural Language Supervision}, + author={Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and others}, + journal={arXiv preprint arXiv:2103.00020}, + year={2021} + }""", + descriptive_stats={ + "n_samples": {"test": 21100}, + "avg_character_length": {"test": 0}, + }, + ) + + image_column_name: str = "jpg" + label_column_name: str = "cls" diff --git a/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py new file mode 100644 index 0000000000..dd5afb0058 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class GTSRBClassification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="GTSRB", + description="""The German Traffic Sign Recognition Benchmark (GTSRB) is a multi-class classification dataset for traffic signs. It consists of dataset of more than 50,000 traffic sign images. The dataset comprises 43 classes with unbalanced class frequencies.""", + reference="https://benchmark.ini.rub.de/", + dataset={ + "path": "clip-benchmark/wds_gtsrb", + "revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2011-01-01", + "2011-12-01", + ), # Estimated range for the collection of reviews + task_subtypes=["Activity recognition"], + domains=["Scene"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@INPROCEEDINGS{6033395, + author={Stallkamp, Johannes and Schlipsing, Marc and Salmen, Jan and Igel, Christian}, + booktitle={The 2011 International Joint Conference on Neural Networks}, + title={The German Traffic Sign Recognition Benchmark: A multi-class classification competition}, + year={2011}, + volume={}, + number={}, + pages={1453-1460}, + keywords={Humans;Training;Image color analysis;Benchmark testing;Lead;Histograms;Image resolution}, + doi={10.1109/IJCNN.2011.6033395}} +""", + descriptive_stats={ + "n_samples": {"test": 12630}, + "avg_character_length": {"test": 0}, + }, + ) + image_column_name = "webp" + label_column_name = "cls" diff --git a/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py new file mode 100644 index 0000000000..7f98ffd459 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class Imagenet1kClassification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="Imagenet1k", + description="ImageNet, a large-scale ontology of images built upon the backbone of the WordNet structure.", + reference="https://ieeexplore.ieee.org/document/5206848", + dataset={ + "path": "clip-benchmark/wds_imagenet1k", + "revision": "b24c7a5a3ef12df09089055d1795e2ce7c7e7397", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2010-01-01", + "2012-01-01", + ), # Estimated range for the collection of reviews + domains=["Scene"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="human-annotated", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@article{deng2009imagenet, + title={ImageNet: A large-scale hierarchical image database}, + author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Li, Kai and Fei-Fei, Li}, + journal={2009 IEEE Conference on Computer Vision and Pattern Recognition}, + pages={248--255}, + year={2009}, + organization={Ieee} + }""", + descriptive_stats={ + "n_samples": {"test": 37200}, + "avg_character_length": {"test": 0}, + }, + ) + image_column_name: str = "jpg" + label_column_name: str = "cls" diff --git a/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py new file mode 100644 index 0000000000..3685af5642 --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class PatchCamelyonClassification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="PatchCamelyon", + description="""Histopathology diagnosis classification dataset.""", + reference="https://link.springer.com/chapter/10.1007/978-3-030-00934-2_24", + dataset={ + "path": "clip-benchmark/wds_vtab-pcam", + "revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2018-01-01", + "2018-12-01", + ), # Estimated range for the collection of reviews + domains=["Medical"], + task_subtypes=["Tumor detection"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@InProceedings{10.1007/978-3-030-00934-2_24, +author="Veeling, Bastiaan S. +and Linmans, Jasper +and Winkens, Jim +and Cohen, Taco +and Welling, Max", +editor="Frangi, Alejandro F. +and Schnabel, Julia A. +and Davatzikos, Christos +and Alberola-L{\'o}pez, Carlos +and Fichtinger, Gabor", +title="Rotation Equivariant CNNs for Digital Pathology", +booktitle="Medical Image Computing and Computer Assisted Intervention -- MICCAI 2018", +year="2018", +publisher="Springer International Publishing", +address="Cham", +pages="210--218", +abstract="We propose a new model for digital pathology segmentation, based on the observation that histopathology images are inherently symmetric under rotation and reflection. Utilizing recent findings on rotation equivariant CNNs, the proposed model leverages these symmetries in a principled manner. We present a visual analysis showing improved stability on predictions, and demonstrate that exploiting rotation equivariance significantly improves tumor detection performance on a challenging lymph node metastases dataset. We further present a novel derived dataset to enable principled comparison of machine learning models, in combination with an initial benchmark. Through this dataset, the task of histopathology diagnosis becomes accessible as a challenging benchmark for fundamental machine learning research.", +isbn="978-3-030-00934-2" +} +""", + descriptive_stats={ + "n_samples": {"test": 32768}, + "avg_character_length": {"test": 0}, + }, + ) + image_column_name = "webp" + label_column_name = "cls" diff --git a/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py new file mode 100644 index 0000000000..65918b9d7f --- /dev/null +++ b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class UCF101Classification(AbsTaskImageClassification): + metadata = TaskMetadata( + name="UCF101", + description="""UCF101 is an action recognition data set of realistic +action videos collected from YouTube, having 101 action categories. This +version of the dataset does not contain images but images saved frame by +frame. Train and test splits are generated based on the authors' first +version train/test list.""", + reference="https://huggingface.co/datasets/flwrlabs/ucf101", + dataset={ + "path": "flwrlabs/ucf101", + "revision": "1098eed48f2929443f47c39f3b5c814e16369c11", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2012-01-01", + "2012-12-01", + ), # Estimated range for the collection of reviews + domains=["Scene"], + task_subtypes=["Activity recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@misc{soomro2012ucf101dataset101human, + title={UCF101: A Dataset of 101 Human Actions Classes From Videos in The Wild}, + author={Khurram Soomro and Amir Roshan Zamir and Mubarak Shah}, + year={2012}, + eprint={1212.0402}, + archivePrefix={arXiv}, + primaryClass={cs.CV}, + url={https://arxiv.org/abs/1212.0402}, +}""", + descriptive_stats={ + "n_samples": {"test": 697222}, + "avg_character_length": {"test": 0}, + }, + ) diff --git a/mteb/tasks/Image/ZeroshotClassification/__init__.py b/mteb/tasks/Image/ZeroshotClassification/__init__.py index 0de4d86ce1..99fd108065 100644 --- a/mteb/tasks/Image/ZeroshotClassification/__init__.py +++ b/mteb/tasks/Image/ZeroshotClassification/__init__.py @@ -3,15 +3,20 @@ from .eng.Birdsnap import * from .eng.Caltech101 import * from .eng.CIFAR import * +from .eng.Country211 import * from .eng.DTD import * from .eng.EuroSAT import * from .eng.FER2013 import * from .eng.FGVCAircraft import * from .eng.Food101 import * +from .eng.GTSRB import * +from .eng.Imagenet1k import * from .eng.MNIST import * from .eng.OxfordPets import * +from .eng.PatchCamelyon import * from .eng.RenderedSST2 import * from .eng.RESISC45 import * from .eng.StanfordCars import * from .eng.STL10 import * from .eng.SUN397 import * +from .eng.UCF101 import * diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Country211.py b/mteb/tasks/Image/ZeroshotClassification/eng/Country211.py new file mode 100644 index 0000000000..7de1b9949f --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Country211.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import os + +from mteb.abstasks import AbsTaskZeroshotClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class Country211Classification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="Country211ZeroShot", + description="Classifying images of 211 countries.", + reference="https://huggingface.co/datasets/clip-benchmark/wds_country211", + dataset={ + "path": "clip-benchmark/wds_country211", + "revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2020-01-01", + "2021-02-26", + ), # Estimated range for the collection of reviews + domains=["Scene"], + task_subtypes=["Scene recognition"], + license="CC BY-SA 4.0", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@article{radford2021learning, + title={Learning Transferable Visual Models From Natural Language Supervision}, + author={Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and others}, + journal={arXiv preprint arXiv:2103.00020}, + year={2021} + }""", + descriptive_stats={ + "n_samples": {"test": 21100}, + "avg_character_length": {"test": 0}, + }, + ) + + image_column_name: str = "jpg" + label_column_name: str = "cls" + + def get_candidate_labels(self) -> list[str]: + path = os.path.dirname(__file__) + with open(os.path.join(path, "templates/Country211_labels.txt")) as f: + labels = f.readlines() + + return [f"a photo showing the country of {c}." for c in labels] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/GTSRB.py b/mteb/tasks/Image/ZeroshotClassification/eng/GTSRB.py new file mode 100644 index 0000000000..2a33115840 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/GTSRB.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import os + +from mteb.abstasks import AbsTaskZeroshotClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class GTSRBClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="GTSRBZeroShot", + description="""The German Traffic Sign Recognition Benchmark (GTSRB) is a multi-class classification dataset for traffic signs. It consists of dataset of more than 50,000 traffic sign images. The dataset comprises 43 classes with unbalanced class frequencies.""", + reference="https://benchmark.ini.rub.de/", + dataset={ + "path": "clip-benchmark/wds_gtsrb", + "revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2011-01-01", + "2011-12-01", + ), # Estimated range for the collection of reviews + task_subtypes=["Activity recognition"], + domains=["Scene"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@INPROCEEDINGS{6033395, + author={Stallkamp, Johannes and Schlipsing, Marc and Salmen, Jan and Igel, Christian}, + booktitle={The 2011 International Joint Conference on Neural Networks}, + title={The German Traffic Sign Recognition Benchmark: A multi-class classification competition}, + year={2011}, + volume={}, + number={}, + pages={1453-1460}, + keywords={Humans;Training;Image color analysis;Benchmark testing;Lead;Histograms;Image resolution}, + doi={10.1109/IJCNN.2011.6033395}} + """, + descriptive_stats={ + "n_samples": {"test": 12630}, + "avg_character_length": {"test": 0}, + }, + ) + + image_column_name: str = "webp" + label_column_name: str = "cls" + + def get_candidate_labels(self) -> list[str]: + path = os.path.dirname(__file__) + with open(os.path.join(path, "templates/GTSRB_labels.txt")) as f: + labels = f.readlines() + + return [f"a close up photo of a '{c}' traffic sign." for c in labels] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py b/mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py new file mode 100644 index 0000000000..82a47567ec --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import os + +from mteb.abstasks import AbsTaskZeroshotClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class Imagenet1kClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="Imagenet1kZeroShot", + description="ImageNet, a large-scale ontology of images built upon the backbone of the WordNet structure.", + reference="https://ieeexplore.ieee.org/document/5206848", + dataset={ + "path": "clip-benchmark/wds_imagenet1k", + "revision": "b24c7a5a3ef12df09089055d1795e2ce7c7e7397", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2010-01-01", + "2012-01-01", + ), # Estimated range for the collection of reviews + domains=["Scene"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="human-annotated", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@article{deng2009imagenet, + title={ImageNet: A large-scale hierarchical image database}, + author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Li, Kai and Fei-Fei, Li}, + journal={2009 IEEE Conference on Computer Vision and Pattern Recognition}, + pages={248--255}, + year={2009}, + organization={Ieee} + }""", + descriptive_stats={ + "n_samples": {"test": 37200}, + "avg_character_length": {"test": 0}, + }, + ) + image_column_name: str = "jpg" + label_column_name: str = "cls" + + def get_candidate_labels(self) -> list[str]: + path = os.path.dirname(__file__) + with open(os.path.join(path, "templates/Imagenet1k_labels.txt")) as f: + labels = f.readlines() + + return [f"a photo of {c}." for c in labels] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py b/mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py new file mode 100644 index 0000000000..726bb2ff2a --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import os + +from mteb.abstasks import AbsTaskZeroshotClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class PatchCamelyonClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="PatchCamelyonZeroShot", + description="""Histopathology diagnosis classification dataset.""", + reference="https://link.springer.com/chapter/10.1007/978-3-030-00934-2_24", + dataset={ + "path": "clip-benchmark/wds_vtab-pcam", + "revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2018-01-01", + "2018-12-01", + ), # Estimated range for the collection of reviews + domains=["Medical"], + task_subtypes=["Tumor detection"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@InProceedings{10.1007/978-3-030-00934-2_24, +author="Veeling, Bastiaan S. +and Linmans, Jasper +and Winkens, Jim +and Cohen, Taco +and Welling, Max", +editor="Frangi, Alejandro F. +and Schnabel, Julia A. +and Davatzikos, Christos +and Alberola-L{\'o}pez, Carlos +and Fichtinger, Gabor", +title="Rotation Equivariant CNNs for Digital Pathology", +booktitle="Medical Image Computing and Computer Assisted Intervention -- MICCAI 2018", +year="2018", +publisher="Springer International Publishing", +address="Cham", +pages="210--218", +abstract="We propose a new model for digital pathology segmentation, based on the observation that histopathology images are inherently symmetric under rotation and reflection. Utilizing recent findings on rotation equivariant CNNs, the proposed model leverages these symmetries in a principled manner. We present a visual analysis showing improved stability on predictions, and demonstrate that exploiting rotation equivariance significantly improves tumor detection performance on a challenging lymph node metastases dataset. We further present a novel derived dataset to enable principled comparison of machine learning models, in combination with an initial benchmark. Through this dataset, the task of histopathology diagnosis becomes accessible as a challenging benchmark for fundamental machine learning research.", +isbn="978-3-030-00934-2" +} +""", + descriptive_stats={ + "n_samples": {"test": 32768}, + "avg_character_length": {"test": 0}, + }, + ) + image_column_name = "webp" + label_column_name = "cls" + + def get_candidate_labels(self) -> list[str]: + path = os.path.dirname(__file__) + with open(os.path.join(path, "templates/PatchCamelyon_labels.txt")) as f: + labels = f.readlines() + + return [f"histopathology image of {c}" for c in labels] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py b/mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py new file mode 100644 index 0000000000..d7d75defe2 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskZeroshotClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class UCF101Classification(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="UCF101ZeroShot", + description="""UCF101 is an action recognition data set of realistic +action videos collected from YouTube, having 101 action categories. This +version of the dataset does not contain images but images saved frame by +frame. Train and test splits are generated based on the authors' first +version train/test list.""", + reference="https://huggingface.co/datasets/flwrlabs/ucf101", + dataset={ + "path": "flwrlabs/ucf101", + "revision": "1098eed48f2929443f47c39f3b5c814e16369c11", + }, + type="Classification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=( + "2012-01-01", + "2012-12-01", + ), # Estimated range for the collection of reviews + domains=["Scene"], + task_subtypes=["Activity recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@misc{soomro2012ucf101dataset101human, + title={UCF101: A Dataset of 101 Human Actions Classes From Videos in The Wild}, + author={Khurram Soomro and Amir Roshan Zamir and Mubarak Shah}, + year={2012}, + eprint={1212.0402}, + archivePrefix={arXiv}, + primaryClass={cs.CV}, + url={https://arxiv.org/abs/1212.0402}, +}""", + descriptive_stats={ + "n_samples": {"test": 697222}, + "avg_character_length": {"test": 0}, + }, + ) + + def get_candidate_labels(self) -> list[str]: + return [ + f"a photo of {name}" + for name in self.dataset["test"].features[self.label_column_name].names + ] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/templates/Country211_labels.txt b/mteb/tasks/Image/ZeroshotClassification/eng/templates/Country211_labels.txt new file mode 100644 index 0000000000..b7c09926c8 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/templates/Country211_labels.txt @@ -0,0 +1,211 @@ +Andorra +United Arab Emirates +Afghanistan +Antigua and Barbuda +Anguilla +Albania +Armenia +Angola +Antarctica +Argentina +Austria +Australia +Aruba +Aland Islands +Azerbaijan +Bosnia and Herzegovina +Barbados +Bangladesh +Belgium +Burkina Faso +Bulgaria +Bahrain +Benin +Bermuda +Brunei Darussalam +Bolivia +Bonaire, Saint Eustatius and Saba +Brazil +Bahamas +Bhutan +Botswana +Belarus +Belize +Canada +DR Congo +Central African Republic +Switzerland +Cote d'Ivoire +Cook Islands +Chile +Cameroon +China +Colombia +Costa Rica +Cuba +Cabo Verde +Curacao +Cyprus +Czech Republic +Germany +Denmark +Dominica +Dominican Republic +Algeria +Ecuador +Estonia +Egypt +Spain +Ethiopia +Finland +Fiji +Falkland Islands +Faeroe Islands +France +Gabon +United Kingdom +Grenada +Georgia +French Guiana +Guernsey +Ghana +Gibraltar +Greenland +Gambia +Guadeloupe +Greece +South Georgia and South Sandwich Is. +Guatemala +Guam +Guyana +Hong Kong +Honduras +Croatia +Haiti +Hungary +Indonesia +Ireland +Israel +Isle of Man +India +Iraq +Iran +Iceland +Italy +Jersey +Jamaica +Jordan +Japan +Kenya +Kyrgyz Republic +Cambodia +St. Kitts and Nevis +North Korea +South Korea +Kuwait +Cayman Islands +Kazakhstan +Laos +Lebanon +St. Lucia +Liechtenstein +Sri Lanka +Liberia +Lithuania +Luxembourg +Latvia +Libya +Morocco +Monaco +Moldova +Montenegro +Saint-Martin +Madagascar +Macedonia +Mali +Myanmar +Mongolia +Macau +Martinique +Mauritania +Malta +Mauritius +Maldives +Malawi +Mexico +Malaysia +Mozambique +Namibia +New Caledonia +Nigeria +Nicaragua +Netherlands +Norway +Nepal +New Zealand +Oman +Panama +Peru +French Polynesia +Papua New Guinea +Philippines +Pakistan +Poland +Puerto Rico +Palestine +Portugal +Palau +Paraguay +Qatar +Reunion +Romania +Serbia +Russia +Rwanda +Saudi Arabia +Solomon Islands +Seychelles +Sudan +Sweden +Singapore +St. Helena +Slovenia +Svalbard and Jan Mayen Islands +Slovakia +Sierra Leone +San Marino +Senegal +Somalia +South Sudan +El Salvador +Sint Maarten +Syria +Eswatini +Togo +Thailand +Tajikistan +Timor-Leste +Turkmenistan +Tunisia +Tonga +Turkey +Trinidad and Tobago +Taiwan +Tanzania +Ukraine +Uganda +United States +Uruguay +Uzbekistan +Vatican +Venezuela +British Virgin Islands +United States Virgin Islands +Vietnam +Vanuatu +Samoa +Kosovo +Yemen +South Africa +Zambia +Zimbabwe \ No newline at end of file diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/templates/GTSRB_labels.txt b/mteb/tasks/Image/ZeroshotClassification/eng/templates/GTSRB_labels.txt new file mode 100644 index 0000000000..2049335509 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/templates/GTSRB_labels.txt @@ -0,0 +1,43 @@ +red and white circle 20 kph speed limit +red and white circle 30 kph speed limit +red and white circle 50 kph speed limit +red and white circle 60 kph speed limit +red and white circle 70 kph speed limit +red and white circle 80 kph speed limit +end / de-restriction of 80 kph speed limit +red and white circle 100 kph speed limit +red and white circle 120 kph speed limit +red and white circle red car and black car no passing +red and white circle red truck and black car no passing +red and white triangle road intersection warning +white and yellow diamond priority road +red and white upside down triangle yield right-of-way +stop +empty red and white circle +red and white circle no truck entry +red circle with white horizonal stripe no entry +red and white triangle with exclamation mark warning +red and white triangle with black left curve approaching warning +red and white triangle with black right curve approaching warning +red and white triangle with black double curve approaching warning +red and white triangle rough / bumpy road warning +red and white triangle car skidding / slipping warning +red and white triangle with merging / narrow lanes warning +red and white triangle with person digging / construction / road work warning +red and white triangle with traffic light approaching warning +red and white triangle with person walking warning +red and white triangle with child and person walking warning +red and white triangle with bicyle warning +red and white triangle with snowflake / ice warning +red and white triangle with deer warning +white circle with gray strike bar no speed limit +blue circle with white right turn arrow mandatory +blue circle with white left turn arrow mandatory +blue circle with white forward arrow mandatory +blue circle with white forward or right turn arrow mandatory +blue circle with white forward or left turn arrow mandatory +blue circle with white keep right arrow mandatory +blue circle with white keep left arrow mandatory +blue circle with white arrows indicating a traffic circle +white circle with gray strike bar indicating no passing for cars has ended +white circle with gray strike bar indicating no passing for trucks has ended \ No newline at end of file diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/templates/Imagenet1k_labels.txt b/mteb/tasks/Image/ZeroshotClassification/eng/templates/Imagenet1k_labels.txt new file mode 100644 index 0000000000..666b01ac0b --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/templates/Imagenet1k_labels.txt @@ -0,0 +1,1000 @@ +tench +goldfish +great white shark +tiger shark +hammerhead shark +electric ray +stingray +rooster +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +American robin +bulbul +jay +magpie +chickadee +American dipper +kite (bird of prey) +bald eagle +vulture +great grey owl +fire salamander +smooth newt +newt +spotted salamander +axolotl +American bullfrog +tree frog +tailed frog +loggerhead sea turtle +leatherback sea turtle +mud turtle +terrapin +box turtle +banded gecko +green iguana +Carolina anole +desert grassland whiptail lizard +agama +frilled-necked lizard +alligator lizard +Gila monster +European green lizard +chameleon +Komodo dragon +Nile crocodile +American alligator +triceratops +worm snake +ring-necked snake +eastern hog-nosed snake +smooth green snake +kingsnake +garter snake +water snake +vine snake +night snake +boa constrictor +African rock python +Indian cobra +green mamba +sea snake +Saharan horned viper +eastern diamondback rattlesnake +sidewinder rattlesnake +trilobite +harvestman +scorpion +yellow garden spider +barn spider +European garden spider +southern black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie grouse +peafowl +quail +partridge +african grey parrot +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +duck +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +red king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +great egret +bittern bird +crane bird +limpkin +common gallinule +American coot +bustard +ruddy turnstone +dunlin +common redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese Chin +Maltese +Pekingese +Shih Tzu +King Charles Spaniel +Papillon +toy terrier +Rhodesian Ridgeback +Afghan Hound +Basset Hound +Beagle +Bloodhound +Bluetick Coonhound +Black and Tan Coonhound +Treeing Walker Coonhound +English foxhound +Redbone Coonhound +borzoi +Irish Wolfhound +Italian Greyhound +Whippet +Ibizan Hound +Norwegian Elkhound +Otterhound +Saluki +Scottish Deerhound +Weimaraner +Staffordshire Bull Terrier +American Staffordshire Terrier +Bedlington Terrier +Border Terrier +Kerry Blue Terrier +Irish Terrier +Norfolk Terrier +Norwich Terrier +Yorkshire Terrier +Wire Fox Terrier +Lakeland Terrier +Sealyham Terrier +Airedale Terrier +Cairn Terrier +Australian Terrier +Dandie Dinmont Terrier +Boston Terrier +Miniature Schnauzer +Giant Schnauzer +Standard Schnauzer +Scottish Terrier +Tibetan Terrier +Australian Silky Terrier +Soft-coated Wheaten Terrier +West Highland White Terrier +Lhasa Apso +Flat-Coated Retriever +Curly-coated Retriever +Golden Retriever +Labrador Retriever +Chesapeake Bay Retriever +German Shorthaired Pointer +Vizsla +English Setter +Irish Setter +Gordon Setter +Brittany dog +Clumber Spaniel +English Springer Spaniel +Welsh Springer Spaniel +Cocker Spaniel +Sussex Spaniel +Irish Water Spaniel +Kuvasz +Schipperke +Groenendael dog +Malinois +Briard +Australian Kelpie +Komondor +Old English Sheepdog +Shetland Sheepdog +collie +Border Collie +Bouvier des Flandres dog +Rottweiler +German Shepherd Dog +Dobermann +Miniature Pinscher +Greater Swiss Mountain Dog +Bernese Mountain Dog +Appenzeller Sennenhund +Entlebucher Sennenhund +Boxer +Bullmastiff +Tibetan Mastiff +French Bulldog +Great Dane +St. Bernard +husky +Alaskan Malamute +Siberian Husky +Dalmatian +Affenpinscher +Basenji +pug +Leonberger +Newfoundland dog +Great Pyrenees dog +Samoyed +Pomeranian +Chow Chow +Keeshond +brussels griffon +Pembroke Welsh Corgi +Cardigan Welsh Corgi +Toy Poodle +Miniature Poodle +Standard Poodle +Mexican hairless dog (xoloitzcuintli) +grey wolf +Alaskan tundra wolf +red wolf or maned wolf +coyote +dingo +dhole +African wild dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby cat +tiger cat +Persian cat +Siamese cat +Egyptian Mau +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +polar bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +longhorn beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket insect +stick insect +cockroach +praying mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +red admiral butterfly +ringlet butterfly +monarch butterfly +small white butterfly +sulphur butterfly +gossamer-winged butterfly +starfish +sea urchin +sea cucumber +cottontail rabbit +hare +Angora rabbit +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +common sorrel horse +zebra +pig +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram (adult male sheep) +bighorn sheep +Alpine ibex +hartebeest +impala (antelope) +gazelle +arabian camel +llama +weasel +mink +European polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas monkey +baboon +macaque +langur +black-and-white colobus +proboscis monkey +marmoset +white-headed capuchin +howler monkey +titi monkey +Geoffroy's spider monkey +common squirrel monkey +ring-tailed lemur +indri +Asian elephant +African bush elephant +red panda +giant panda +snoek fish +eel +silver salmon +rock beauty fish +clownfish +sturgeon +gar fish +lionfish +pufferfish +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibious vehicle +analog clock +apiary +apron +trash can +assault rifle +backpack +bakery +balance beam +balloon +ballpoint pen +Band-Aid +banjo +baluster / handrail +barbell +barber chair +barbershop +barn +barometer +barrel +wheelbarrow +baseball +basketball +bassinet +bassoon +swimming cap +bath towel +bathtub +station wagon +lighthouse +beaker +military hat (bearskin or shako) +beer bottle +beer glass +bell tower +baby bib +tandem bicycle +bikini +ring binder +binoculars +birdhouse +boathouse +bobsleigh +bolo tie +poke bonnet +bookcase +bookstore +bottle cap +hunting bow +bow tie +brass memorial plaque +bra +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +high-speed train +butcher shop +taxicab +cauldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +tool kit +cardboard box / carton +car wheel +automated teller machine +cassette +cassette player +castle +catamaran +CD player +cello +mobile phone +chain +chain-link fence +chain mail +chainsaw +storage chest +chiffonier +bell or wind chime +china cabinet +Christmas stocking +church +movie theater +cleaver +cliff dwelling +cloak +clogs +cocktail shaker +coffee mug +coffeemaker +spiral or coil +combination lock +computer keyboard +candy store +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +construction crane +crash helmet +crate +infant bed +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +rotary dial telephone +diaper +digital clock +digital watch +dining table +dishcloth +dishwasher +disc brake +dock +dog sled +dome +doormat +drilling rig +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso machine +face powder +feather boa +filing cabinet +fireboat +fire truck +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster bed +freight car +French horn +frying pan +fur coat +garbage truck +gas mask or respirator +gas pump +goblet +go-kart +golf ball +golf cart +gondola +gong +gown +grand piano +greenhouse +radiator grille +grocery store +guillotine +hair clip +hair spray +half-track +hammer +hamper +hair dryer +hand-held computer +handkerchief +hard disk drive +harmonica +harp +combine harvester +hatchet +holster +home theater +honeycomb +hook +hoop skirt +gymnastic horizontal bar +horse-drawn vehicle +hourglass +iPod +clothes iron +carved pumpkin +jeans +jeep +T-shirt +jigsaw puzzle +rickshaw +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop computer +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +ocean liner +lipstick +slip-on shoe +lotion +music speaker +loupe magnifying glass +sawmill +magnetic compass +messenger bag +mailbox +tights +one-piece bathing suit +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine cabinet +megalith +microphone +microwave oven +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +ford model t +modem +monastery +monitor +moped +mortar and pestle +graduation cap +mosque +mosquito net +vespa +mountain bike +tent +computer mouse +mousetrap +moving van +muzzle +metal nail +neck brace +necklace +baby pacifier +notebook computer +obelisk +oboe +ocarina +odometer +oil filter +pipe organ +oscilloscope +overskirt +bullock cart +oxygen mask +product packet / packaging +paddle +paddle wheel +padlock +paintbrush +pajamas +palace +pan flute +paper towel +parachute +parallel bars +park bench +parking meter +railroad car +patio +payphone +pedestal +pencil case +pencil sharpener +perfume +Petri dish +photocopier +plectrum +Pickelhaube +picket fence +pickup truck +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate ship +drink pitcher +block plane +planetarium +plastic bag +plate rack +farm plow +plunger +Polaroid camera +pole +police van +poncho +pool table +soda bottle +plant pot +potter's wheel +power drill +prayer rug +printer +prison +missile +projector +hockey puck +punching bag +purse +quill +quilt +race car +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +fishing casting reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +eraser +rugby ball +ruler measuring stick +sneaker +safe +safety pin +salt shaker +sandal +sarong +saxophone +scabbard +weighing scale +school bus +schooner +scoreboard +CRT monitor +screw +screwdriver +seat belt +sewing machine +shield +shoe store +shoji screen / room divider +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +balaclava ski mask +sleeping bag +slide rule +sliding door +slot machine +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar thermal collector +sombrero +soup bowl +keyboard space bar +space heater +space shuttle +spatula +motorboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +through arch bridge +steel drum +stethoscope +scarf +stone wall +stopwatch +stove +strainer +tram +stretcher +couch +stupa +submarine +suit +sundial +sunglasses +sunglasses +sunscreen +suspension bridge +mop +sweatshirt +swim trunks / shorts +swing +electrical switch +syringe +table lamp +tank +tape player +teapot +teddy bear +television +tennis ball +thatched roof +front curtain +thimble +threshing machine +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toy store +tractor +semi-trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +hot tub +turnstile +typewriter keyboard +umbrella +unicycle +upright piano +vacuum cleaner +vase +vaulted or arched ceiling +velvet fabric +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +military aircraft +sink +washing machine +water bottle +water jug +water tower +whiskey jug +whistle +hair wig +window screen +window shade +Windsor tie +wine bottle +airplane wing +wok +wooden spoon +wool +split-rail fence +shipwreck +sailboat +yurt +website +comic book +crossword +traffic or street sign +traffic light +dust jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +popsicle +baguette +bagel +pretzel +cheeseburger +hot dog +mashed potatoes +cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith apple +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +cherimoya (custard apple) +pomegranate +hay +carbonara +chocolate syrup +dough +meatloaf +pizza +pot pie +burrito +red wine +espresso +tea cup +eggnog +mountain +bubble +cliff +coral reef +geyser +lakeshore +promontory +sandbar +beach +valley +volcano +baseball player +bridegroom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +rose hip +horse chestnut seed +coral fungus +agaric +gyromitra +stinkhorn mushroom +earth star fungus +hen of the woods mushroom +bolete +corn cob +toilet paper \ No newline at end of file diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/templates/PatchCamelyon_labels.txt b/mteb/tasks/Image/ZeroshotClassification/eng/templates/PatchCamelyon_labels.txt new file mode 100644 index 0000000000..4446eab039 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/templates/PatchCamelyon_labels.txt @@ -0,0 +1,2 @@ +lymph node +lymph node containing metastatic tumor tissue \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211.json new file mode 100644 index 0000000000..e59ca3c0c2 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211.json @@ -0,0 +1,28 @@ +{ + "dataset_revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", + "evaluation_time": 516.9000871181488, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "accuracy": 0.1495260663507109, + "f1": 0.1497060161021076, + "f1_weighted": 0.1497060161021076, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.1495260663507109, + "scores_per_experiment": [ + { + "accuracy": 0.1495260663507109, + "f1": 0.1497060161021076, + "f1_weighted": 0.1497060161021076 + } + ] + } + ] + }, + "task_name": "Country211" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211ZeroShot.json new file mode 100644 index 0000000000..b14c6829c9 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", + "evaluation_time": 167.4767632484436, + "kg_co2_emissions": null, + "mteb_version": "1.12.49", + "scores": { + "test": [ + { + "accuracy": 0.15962085308056873, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.15962085308056873 + } + ] + }, + "task_name": "Country211ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRB.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRB.json new file mode 100644 index 0000000000..c3d7abe851 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRB.json @@ -0,0 +1,28 @@ +{ + "dataset_revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", + "evaluation_time": 161.53320479393005, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "accuracy": 0.6921615201900237, + "f1": 0.6132213777667673, + "f1_weighted": 0.7046823400547864, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6921615201900237, + "scores_per_experiment": [ + { + "accuracy": 0.6921615201900237, + "f1": 0.6132213777667673, + "f1_weighted": 0.7046823400547864 + } + ] + } + ] + }, + "task_name": "GTSRB" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRBZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRBZeroShot.json new file mode 100644 index 0000000000..e341bc14a9 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRBZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", + "evaluation_time": 32.170368909835815, + "kg_co2_emissions": null, + "mteb_version": "1.12.49", + "scores": { + "test": [ + { + "accuracy": 0.34418052256532067, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.34418052256532067 + } + ] + }, + "task_name": "GTSRBZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1k.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1k.json new file mode 100644 index 0000000000..57d150dd7e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1k.json @@ -0,0 +1,28 @@ +{ + "dataset_revision": "b24c7a5a3ef12df09089055d1795e2ce7c7e7397", + "evaluation_time": 4185.878092765808, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "accuracy": 0.6102, + "f1": 0.607661279811048, + "f1_weighted": 0.6076612798110479, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6102, + "scores_per_experiment": [ + { + "accuracy": 0.6102, + "f1": 0.607661279811048, + "f1_weighted": 0.6076612798110479 + } + ] + } + ] + }, + "task_name": "Imagenet1k" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1kZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1kZeroShot.json new file mode 100644 index 0000000000..93f2b00122 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1kZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "b24c7a5a3ef12df09089055d1795e2ce7c7e7397", + "evaluation_time": 234.19821047782898, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "accuracy": 0.58842, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.58842 + } + ] + }, + "task_name": "Imagenet1kZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyon.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyon.json new file mode 100644 index 0000000000..09c443ccac --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyon.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", + "evaluation_time": 210.66001987457275, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "accuracy": 0.701171875, + "ap": 0.6377405975860886, + "ap_weighted": 0.6377405975860886, + "f1": 0.700650138300302, + "f1_weighted": 0.7006447988896325, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.701171875, + "scores_per_experiment": [ + { + "accuracy": 0.701171875, + "ap": 0.6377405975860886, + "ap_weighted": 0.6377405975860886, + "f1": 0.700650138300302, + "f1_weighted": 0.7006447988896325 + } + ] + } + ] + }, + "task_name": "PatchCamelyon" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyonZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyonZeroShot.json new file mode 100644 index 0000000000..edff52f341 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyonZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", + "evaluation_time": 104.11877298355103, + "kg_co2_emissions": null, + "mteb_version": "1.12.49", + "scores": { + "test": [ + { + "accuracy": 0.618804931640625, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.618804931640625 + } + ] + }, + "task_name": "PatchCamelyonZeroShot" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101.json new file mode 100644 index 0000000000..4ff30ca9fa --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101.json @@ -0,0 +1,28 @@ +{ + "dataset_revision": "1098eed48f2929443f47c39f3b5c814e16369c11", + "evaluation_time": 3175.5142464637756, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "accuracy": 0.7968753137451199, + "f1": 0.7654702260165476, + "f1_weighted": 0.7939622985652318, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7968753137451199, + "scores_per_experiment": [ + { + "accuracy": 0.7968753137451199, + "f1": 0.7654702260165476, + "f1_weighted": 0.7939622985652318 + } + ] + } + ] + }, + "task_name": "UCF101" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101ZeroShot.json new file mode 100644 index 0000000000..146378a6c6 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "1098eed48f2929443f47c39f3b5c814e16369c11", + "evaluation_time": 2312.6607372760773, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "accuracy": 0.5814977152183953, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5814977152183953 + } + ] + }, + "task_name": "UCF101ZeroShot" +} \ No newline at end of file From 6970a4ca6b0fe9bb53894a69c4b6ff4ff6a19837 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Tue, 30 Jul 2024 18:34:12 +0100 Subject: [PATCH 032/154] fix dependency & clip mock test --- .../Image/AbsTaskZeroshotClassification.py | 1 - .../Image/ZeroshotClassificationEvaluator.py | 2 + mteb/models/vista_models.py | 349 +++++++++--------- .../T2IRetrieval/eng/MSCOCOT2IRetrieval.py | 6 +- pyproject.toml | 1 + tests/test_benchmark/mock_models.py | 12 +- 6 files changed, 198 insertions(+), 173 deletions(-) diff --git a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py index a70480de38..dc5a8299ed 100644 --- a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py +++ b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py @@ -41,7 +41,6 @@ def _evaluate_subset( **kwargs, ) -> ScoresDict: candidate_labels = self.get_candidate_labels() - evaluator = ZeroshotClassificationEvaluator( dataset, self.image_column_name, diff --git a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py index 3e10b88cd2..d082f13517 100644 --- a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py @@ -76,10 +76,12 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): image_embeddings = model.get_image_embeddings( dataloader, batch_size=encode_kwargs["batch_size"] ) + probs = model.calculate_probs(text_embeddings, image_embeddings) predictions = probs.argmax(dim=1) logger.info("Evaluating...") + accuracy = metrics.accuracy_score(self.labels, predictions) return {"accuracy": accuracy} diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py index 35e6cc61b7..1f2e894f4c 100644 --- a/mteb/models/vista_models.py +++ b/mteb/models/vista_models.py @@ -3,178 +3,193 @@ from functools import partial import torch + +try: # a temporal fix for the dependency issues of vista models. + from FlagEmbedding.visual.modeling import Visualized_BGE +except ImportError: + Visualized_BGE = None from PIL import Image from tqdm import tqdm -from FlagEmbedding.visual.modeling import Visualized_BGE from mteb.model_meta import ModelMeta - -class VisualizedBGEWrapper(Visualized_BGE): - def __init__( - self, - model_name_bge: str = None, - model_weight=None, - normlized: bool = True, - sentence_pooling_method: str = "cls", - negatives_cross_device: bool = False, - temperature: float = 0.02, - from_pretrained=None, - ): - super().__init__( - model_name_bge=model_name_bge, - model_weight=model_weight, - normlized=normlized, - sentence_pooling_method=sentence_pooling_method, - negatives_cross_device=negatives_cross_device, - temperature=temperature, - from_pretrained=from_pretrained, - ) - self.eval() - - def encode_text(self, texts): - """Currently override Visualized_BGE's the original implementation - to fix attention_mask & embedding_output dtype misalignment - """ - input_ids = texts["input_ids"] - attention_mask = texts["attention_mask"] - - input_shape = input_ids.size() - device = input_ids.device - - token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) - - head_mask = [None] * self.depth - extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( - attention_mask, input_shape - ) - - embedding_output = self.bge_embeddings( - input_ids=input_ids, - position_ids=None, - token_type_ids=token_type_ids, - inputs_embeds=None, - past_key_values_length=0, - ) - - # this line is missing in vista, currently override "encode_text" only to fix this. - extended_attention_mask = extended_attention_mask.to(embedding_output.dtype) - - encoder_outputs = self.bge_encoder( - embedding_output, - attention_mask=extended_attention_mask, - head_mask=head_mask, - encoder_hidden_states=None, - encoder_attention_mask=None, - past_key_values=None, - use_cache=False, - output_attentions=False, - output_hidden_states=False, - return_dict=True, - ) - sequence_output = encoder_outputs[0] - - t_reps = self.sentence_embedding( - sequence_output, texts["attention_mask"] - ) # tensor: reps with pooling - if self.normlized: - t_reps = torch.nn.functional.normalize(t_reps, dim=-1) - return t_reps.contiguous() - - def encode(self, images=None, texts=None): - if images is not None: - if isinstance(images, list): - images = [ - self.preprocess_val( - img if isinstance(img, Image.Image) else Image.open(img) - ) - for img in images - ] - images = torch.stack(images) - if texts is not None: - texts = self.tokenizer(texts, return_tensors="pt", padding=True) - return self.encode_mm(images.to(self.device), texts.to(self.device)) - else: - return self.encode_image(images.to(self.device)) - else: - if texts is not None: - texts = self.tokenizer(texts, return_tensors="pt", padding=True) - return self.encode_text(texts.to(self.device)) +if Visualized_BGE is not None: + + class VisualizedBGEWrapper(Visualized_BGE): + def __init__( + self, + model_name_bge: str = None, + model_weight=None, + normlized: bool = True, + sentence_pooling_method: str = "cls", + negatives_cross_device: bool = False, + temperature: float = 0.02, + from_pretrained=None, + ): + super().__init__( + model_name_bge=model_name_bge, + model_weight=model_weight, + normlized=normlized, + sentence_pooling_method=sentence_pooling_method, + negatives_cross_device=negatives_cross_device, + temperature=temperature, + from_pretrained=from_pretrained, + ) + self.eval() + + def encode_text(self, texts): + """Currently override Visualized_BGE's the original implementation + to fix attention_mask & embedding_output dtype misalignment + """ + input_ids = texts["input_ids"] + attention_mask = texts["attention_mask"] + + input_shape = input_ids.size() + device = input_ids.device + + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) + + head_mask = [None] * self.depth + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape + ) + + embedding_output = self.bge_embeddings( + input_ids=input_ids, + position_ids=None, + token_type_ids=token_type_ids, + inputs_embeds=None, + past_key_values_length=0, + ) + + # this line is missing in vista, currently override "encode_text" only to fix this. + extended_attention_mask = extended_attention_mask.to(embedding_output.dtype) + + encoder_outputs = self.bge_encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=False, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ) + sequence_output = encoder_outputs[0] + + t_reps = self.sentence_embedding( + sequence_output, texts["attention_mask"] + ) # tensor: reps with pooling + if self.normlized: + t_reps = torch.nn.functional.normalize(t_reps, dim=-1) + return t_reps.contiguous() + + def encode(self, images=None, texts=None): + if images is not None: + if isinstance(images, list): + images = [ + self.preprocess_val( + img if isinstance(img, Image.Image) else Image.open(img) + ) + for img in images + ] + images = torch.stack(images) + if texts is not None: + texts = self.tokenizer(texts, return_tensors="pt", padding=True) + return self.encode_mm(images.to(self.device), texts.to(self.device)) + else: + return self.encode_image(images.to(self.device)) else: - return None - - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): - all_text_embeddings = [] - for i in tqdm(range(0, len(texts), batch_size)): - batch_texts = texts[i : i + batch_size] - with torch.no_grad(): - batch_embeddings = self.encode(texts=batch_texts) - all_text_embeddings.append(batch_embeddings.cpu()) - return torch.cat(all_text_embeddings, dim=0) - - def get_image_embeddings(self, images: list[Image.Image], batch_size: int = 32): - all_image_embeddings = [] - for i in tqdm(range(0, len(images), batch_size)): - batch_images = images[i : i + batch_size] - with torch.no_grad(): - batch_embeddings = self.encode(images=batch_images) - all_image_embeddings.append(batch_embeddings.cpu()) - return torch.cat(all_image_embeddings, dim=0) - - def get_fused_embeddings( - self, - texts: list[str] = None, - images: list[Image.Image] = None, - batch_size: int = 32, - ): - all_embeddings = [] - assert len(texts) == len(images) - for i in tqdm(range(0, len(texts), batch_size)): - batch_texts = texts[i : i + batch_size] - batch_images = images[i : i + batch_size] - with torch.no_grad(): - batch_embeddings = self.encode(images=batch_images, texts=batch_texts) - all_embeddings.append(batch_embeddings.cpu()) - return torch.cat(all_embeddings, dim=0) - - def calculate_probs(self, text_embeddings, image_embeddings): - text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) - image_embeddings = image_embeddings / image_embeddings.norm( - dim=-1, keepdim=True - ) - logits = torch.matmul(image_embeddings, text_embeddings.T) - probs = (logits * 100).softmax(dim=-1) - return probs - - -Visualized_BGE_base = ModelMeta( - loader=partial( - VisualizedBGEWrapper, - model_name_bge="BAAI/bge-base-en-v1.5", - model_weight="visualized_base_en_V1.5.pth", - ), - name="BAAI/bge-visualized-base", - languages=["eng_Latn"], - open_source=True, - revision="98db10b10d22620010d06f11733346e1c98c34aa", - release_date="2024-06-06", -) - -Visualized_BGE_base = ModelMeta( - loader=partial( - VisualizedBGEWrapper, - model_name_bge="BAAI/bge-m3", - model_weight="visualized_m3.pth", - ), - name="BAAI/bge-visualized-m3", - languages=["eng_Latn"], - open_source=True, - revision="98db10b10d22620010d06f11733346e1c98c34aa", - release_date="2024-06-06", -) + if texts is not None: + texts = self.tokenizer(texts, return_tensors="pt", padding=True) + return self.encode_text(texts.to(self.device)) + else: + return None + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + with torch.no_grad(): + batch_embeddings = self.encode(texts=batch_texts) + all_text_embeddings.append(batch_embeddings.cpu()) + return torch.cat(all_text_embeddings, dim=0) + + def get_image_embeddings(self, images: list[Image.Image], batch_size: int = 32): + all_image_embeddings = [] + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + with torch.no_grad(): + batch_embeddings = self.encode(images=batch_images) + all_image_embeddings.append(batch_embeddings.cpu()) + return torch.cat(all_image_embeddings, dim=0) + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] = None, + batch_size: int = 32, + ): + all_embeddings = [] + assert len(texts) == len(images) + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + batch_images = images[i : i + batch_size] + with torch.no_grad(): + batch_embeddings = self.encode( + images=batch_images, texts=batch_texts + ) + all_embeddings.append(batch_embeddings.cpu()) + return torch.cat(all_embeddings, dim=0) + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm( + dim=-1, keepdim=True + ) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + Visualized_BGE_base = ModelMeta( + loader=partial( + VisualizedBGEWrapper, + model_name_bge="BAAI/bge-base-en-v1.5", + model_weight="visualized_base_en_V1.5.pth", + ), + name="BAAI/bge-visualized-base", + languages=["eng_Latn"], + open_source=True, + revision="98db10b10d22620010d06f11733346e1c98c34aa", + release_date="2024-06-06", + ) + + Visualized_BGE_base = ModelMeta( + loader=partial( + VisualizedBGEWrapper, + model_name_bge="BAAI/bge-m3", + model_weight="visualized_m3.pth", + ), + name="BAAI/bge-visualized-m3", + languages=["eng_Latn"], + open_source=True, + revision="98db10b10d22620010d06f11733346e1c98c34aa", + release_date="2024-06-06", + ) if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(Visualized_BGE_base.name, Visualized_BGE_base.name.revision) - emb = mdl.get_text_embeddings(["Hello, world!"]) + if ( + Visualized_BGE is None + ): # a temporal fix to the dependency issues of Vista models. + print("Visualized_BGE module is not available.") + else: + import mteb + + mdl = mteb.get_model( + Visualized_BGE_base.name, Visualized_BGE_base.name.revision + ) + emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py b/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py index 9fd20f3450..b9a4a57456 100644 --- a/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py +++ b/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py @@ -8,7 +8,7 @@ class MSCOCOT2IRetrieval(AbsTaskT2IRetrieval): metadata = TaskMetadata( name="MSCOCOT2IRetrieval", - description="Retrieve captions based on images.", + description="Retrieve images based on captions.", reference="https://link.springer.com/chapter/10.1007/978-3-319-10602-1_48", dataset={ "path": "MRBench/mbeir_mscoco_task0", @@ -41,8 +41,8 @@ class MSCOCOT2IRetrieval(AbsTaskT2IRetrieval): "n_samples": {"test": 1172}, "avg_character_length": { "test": { - "average_document_length": 30.94235294117647, - "average_query_length": 131.56569965870307, + "average_document_length": 0.0, + "average_query_length": 0.0, "num_documents": 9350, "num_queries": 1172, "average_relevant_docs_per_query": 1.0, diff --git a/pyproject.toml b/pyproject.toml index 435c5fd993..cfc6f19ba7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "typing_extensions>=0.0.0", "eval_type_backport>=0.0.0", "polars>=0.20.22", + "torchvision>0.0.0", ] diff --git a/tests/test_benchmark/mock_models.py b/tests/test_benchmark/mock_models.py index 0f2926825b..93a5360356 100644 --- a/tests/test_benchmark/mock_models.py +++ b/tests/test_benchmark/mock_models.py @@ -4,6 +4,7 @@ import numpy as np import torch +from torch.utils.data import DataLoader import mteb @@ -40,10 +41,17 @@ def get_text_embeddings(self, texts, **kwargs): return torch.randn(len(texts), 10) def get_image_embeddings(self, images, **kwargs): - return torch.randn(len(images), 10) + if isinstance(images, DataLoader): + all_embeddings = [] + for batch in images: + batch_embeddings = torch.randn(len(batch), 10) + all_embeddings.append(batch_embeddings) + return torch.cat(all_embeddings, dim=0) + else: + return torch.randn(len(images), 10) def get_fused_embeddings(self, texts, images, **kwargs): return torch.randn(len(images), 10) def calculate_probs(self, text_embeddings, image_embeddings): - return torch.randn(image_embeddings.shape[0], text_embeddings.shape[1]) + return torch.randn(image_embeddings.shape[0], text_embeddings.shape[0]) From e94d6f3cd87ad97b5b877186a4afe4ead532e15b Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 31 Jul 2024 15:34:43 +0300 Subject: [PATCH 033/154] [MIEB] Add jina clip (#1120) * add jina clip and mscoco i2t and t2i results * make lint --- mteb/models/__init__.py | 2 + mteb/models/jina_clip.py | 176 ++++++++++++++++++ .../MSCOCOI2TRetrieval.json | 158 ++++++++++++++++ .../MSCOCOT2IRetrieval.json | 158 ++++++++++++++++ .../STS12.json | 26 +++ .../model_meta.json | 1 + 6 files changed, 521 insertions(+) create mode 100644 mteb/models/jina_clip.py create mode 100644 results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOI2TRetrieval.json create mode 100644 results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOT2IRetrieval.json create mode 100644 results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/STS12.json create mode 100644 results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/model_meta.json diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index e6d72bf0ab..ea3c26e236 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -18,6 +18,7 @@ google_models, gritlm_models, gte_models, + jina_clip, llm2vec_models, mxbai_models, nomic_models, @@ -130,6 +131,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe e5_instruct, e5_models, e5_v, + jina_clip, gritlm_models, gte_models, llm2vec_models, diff --git a/mteb/models/jina_clip.py b/mteb/models/jina_clip.py new file mode 100644 index 0000000000..969fd7ea9d --- /dev/null +++ b/mteb/models/jina_clip.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import AutoModel + +from mteb.model_meta import ModelMeta +from mteb.models.text_formatting_utils import corpus_to_texts + + +class JinaCLIPModelWrapper: + def __init__( + self, + model_name: str, + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + self.model = AutoModel.from_pretrained(model_name, trust_remote_code=True).to( + self.device + ) + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + text_outputs = self.model.encode_text( + batch_texts, convert_to_numpy=False, convert_to_tensor=True + ) + all_text_embeddings.append(text_outputs.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + with torch.no_grad(): + for batch in tqdm(images): + image_outputs = self.model.encode_image( + batch, convert_to_numpy=False, convert_to_tensor=True + ) + all_image_embeddings.append(image_outputs.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + image_outputs = self.model.encode_image( + batch_images, convert_to_numpy=False, convert_to_tensor=True + ) + all_image_embeddings.append(image_outputs.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.encode_text( + texts, + batch_size=batch_size, + convert_to_numpy=False, + convert_to_tensor=True, + ) + + if images is not None: + image_embeddings = self.encode_image( + images, + batch_size=batch_size, + convert_to_numpy=False, + convert_to_tensor=True, + ) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + else: + # to do: add other fusion mode + raise ValueError(f"fusion mode {fusion_mode} hasn't been implemented") + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + def encode( # type: ignore + self, + sentences: list[str], + *, + batch_size: int = 32, + **kwargs: Any, + ): + if "prompt_name" in kwargs: + kwargs.pop("prompt_name") + return self.model.encode_text(sentences, batch_size=batch_size, **kwargs) + + def encode_queries(self, queries: list[str], batch_size: int = 32, **kwargs: Any): + if "prompt_name" in kwargs: + kwargs.pop("prompt_name") + sentences = [ + "Represent this sentence for searching relevant passages: " + sentence + for sentence in queries + ] + emb = self.encode( + sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs + ) + return emb + + def encode_corpus( + self, + corpus: list[dict[str, str]] | dict[str, list[str]], + batch_size: int = 32, + **kwargs: Any, + ): + if "prompt_name" in kwargs: + kwargs.pop("prompt_name") + sentences = corpus_to_texts(corpus) + emb = self.encode( + sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs + ) + return emb + + +jina_clip_v1 = ModelMeta( + loader=partial( + JinaCLIPModelWrapper, + model_name="jinaai/jina-clip-v1", + ), + name="jinaai/jina-clip-v1", + languages=["eng_Latn"], + open_source=True, + revision="1cbe5e8b11ea3728df0b610d5453dfe739804aa9", + release_date="2024-05-30", +) + + +if __name__ == "__main__": + import mteb + + mdl = mteb.get_model(jina_clip_v1.name, jina_clip_v1.revision) + emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOI2TRetrieval.json b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOI2TRetrieval.json new file mode 100644 index 0000000000..2be160c172 --- /dev/null +++ b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOI2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", + "evaluation_time": 53.00429034233093, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.48279, + "map_at_1": 0.10886, + "map_at_10": 0.36754, + "map_at_100": 0.41438, + "map_at_1000": 0.41752, + "map_at_20": 0.39363, + "map_at_3": 0.2442, + "map_at_5": 0.31642, + "mrr_at_1": 0.544, + "mrr_at_10": 0.6441401587301602, + "mrr_at_100": 0.650529731505628, + "mrr_at_1000": 0.6505789992154296, + "mrr_at_20": 0.648565749577221, + "mrr_at_3": 0.6178333333333357, + "mrr_at_5": 0.6338033333333345, + "nauc_map_at_1000_diff1": 0.21096399203394334, + "nauc_map_at_1000_max": 0.2643789873573568, + "nauc_map_at_1000_std": -0.3656602505905553, + "nauc_map_at_100_diff1": 0.21094758478668932, + "nauc_map_at_100_max": 0.2647125569770621, + "nauc_map_at_100_std": -0.36658408307145496, + "nauc_map_at_10_diff1": 0.20614378960832005, + "nauc_map_at_10_max": 0.2556506736639637, + "nauc_map_at_10_std": -0.3625667597727067, + "nauc_map_at_1_diff1": 0.3621079914040161, + "nauc_map_at_1_max": 0.22314949459519212, + "nauc_map_at_1_std": -0.39289670930715714, + "nauc_map_at_20_diff1": 0.2088668661351603, + "nauc_map_at_20_max": 0.26068843280053283, + "nauc_map_at_20_std": -0.3674865778762073, + "nauc_map_at_3_diff1": 0.23125083203784524, + "nauc_map_at_3_max": 0.23835700021972306, + "nauc_map_at_3_std": -0.3525765196558998, + "nauc_map_at_5_diff1": 0.20396930020356405, + "nauc_map_at_5_max": 0.24892695318464714, + "nauc_map_at_5_std": -0.3551372962226835, + "nauc_mrr_at_1000_diff1": 0.34622127332034, + "nauc_mrr_at_1000_max": 0.2234325242238912, + "nauc_mrr_at_1000_std": -0.4210887792813959, + "nauc_mrr_at_100_diff1": 0.34617020154363415, + "nauc_mrr_at_100_max": 0.2234599441533202, + "nauc_mrr_at_100_std": -0.4210902614211229, + "nauc_mrr_at_10_diff1": 0.3457361472170574, + "nauc_mrr_at_10_max": 0.22270683416516163, + "nauc_mrr_at_10_std": -0.42354086081618936, + "nauc_mrr_at_1_diff1": 0.3620563318972254, + "nauc_mrr_at_1_max": 0.22304702323816303, + "nauc_mrr_at_1_std": -0.39325847713016515, + "nauc_mrr_at_20_diff1": 0.3461312655849626, + "nauc_mrr_at_20_max": 0.22332181681230495, + "nauc_mrr_at_20_std": -0.42256993930033726, + "nauc_mrr_at_3_diff1": 0.3436568869473764, + "nauc_mrr_at_3_max": 0.22268159334592896, + "nauc_mrr_at_3_std": -0.41793547872411135, + "nauc_mrr_at_5_diff1": 0.3463546807212511, + "nauc_mrr_at_5_max": 0.22201030832472302, + "nauc_mrr_at_5_std": -0.4215372140619334, + "nauc_ndcg_at_1000_diff1": 0.2474117966134631, + "nauc_ndcg_at_1000_max": 0.2633225510736001, + "nauc_ndcg_at_1000_std": -0.38255756076688163, + "nauc_ndcg_at_100_diff1": 0.24392830523701964, + "nauc_ndcg_at_100_max": 0.2722245687662711, + "nauc_ndcg_at_100_std": -0.38502007102726465, + "nauc_ndcg_at_10_diff1": 0.23223810393318697, + "nauc_ndcg_at_10_max": 0.25134167852059397, + "nauc_ndcg_at_10_std": -0.3890511501621063, + "nauc_ndcg_at_1_diff1": 0.3620563318972254, + "nauc_ndcg_at_1_max": 0.22304702323816303, + "nauc_ndcg_at_1_std": -0.39325847713016515, + "nauc_ndcg_at_20_diff1": 0.2368616298300584, + "nauc_ndcg_at_20_max": 0.2586300416187524, + "nauc_ndcg_at_20_std": -0.39570826135429343, + "nauc_ndcg_at_3_diff1": 0.25241282801383136, + "nauc_ndcg_at_3_max": 0.23505398892057644, + "nauc_ndcg_at_3_std": -0.37119292314398367, + "nauc_ndcg_at_5_diff1": 0.23258181324662106, + "nauc_ndcg_at_5_max": 0.24401204310948493, + "nauc_ndcg_at_5_std": -0.3766951207685948, + "nauc_precision_at_1000_diff1": -0.060776645601850005, + "nauc_precision_at_1000_max": 0.45061728395062717, + "nauc_precision_at_1000_std": 0.49837165139806966, + "nauc_precision_at_100_diff1": 0.16399269478318815, + "nauc_precision_at_100_max": 0.3174441756422564, + "nauc_precision_at_100_std": -0.2719045017872643, + "nauc_precision_at_10_diff1": 0.1834674594026287, + "nauc_precision_at_10_max": 0.2475111637901415, + "nauc_precision_at_10_std": -0.36964525963118233, + "nauc_precision_at_1_diff1": 0.3620563318972254, + "nauc_precision_at_1_max": 0.22304702323816303, + "nauc_precision_at_1_std": -0.39325847713016515, + "nauc_precision_at_20_diff1": 0.18190648500233034, + "nauc_precision_at_20_max": 0.2543371579459487, + "nauc_precision_at_20_std": -0.3684848720360581, + "nauc_precision_at_3_diff1": 0.2216308838348382, + "nauc_precision_at_3_max": 0.23889515868420422, + "nauc_precision_at_3_std": -0.3624204875957743, + "nauc_precision_at_5_diff1": 0.19295589426112914, + "nauc_precision_at_5_max": 0.24668879287833217, + "nauc_precision_at_5_std": -0.36421771939542763, + "nauc_recall_at_1000_diff1": -0.04813353912994217, + "nauc_recall_at_1000_max": 0.47849854227402444, + "nauc_recall_at_1000_std": 0.5228805808037352, + "nauc_recall_at_100_diff1": 0.16501981261932044, + "nauc_recall_at_100_max": 0.3183331361282548, + "nauc_recall_at_100_std": -0.2726147056630754, + "nauc_recall_at_10_diff1": 0.1835675928057196, + "nauc_recall_at_10_max": 0.24766746744342197, + "nauc_recall_at_10_std": -0.3696577837871595, + "nauc_recall_at_1_diff1": 0.3621079914040161, + "nauc_recall_at_1_max": 0.22314949459519212, + "nauc_recall_at_1_std": -0.39289670930715714, + "nauc_recall_at_20_diff1": 0.18207077982715833, + "nauc_recall_at_20_max": 0.254625446742885, + "nauc_recall_at_20_std": -0.36840440513586076, + "nauc_recall_at_3_diff1": 0.2218119797611259, + "nauc_recall_at_3_max": 0.23903410739410114, + "nauc_recall_at_3_std": -0.3625477824595922, + "nauc_recall_at_5_diff1": 0.1931427010978174, + "nauc_recall_at_5_max": 0.24689867912760904, + "nauc_recall_at_5_std": -0.364330180957868, + "ndcg_at_1": 0.544, + "ndcg_at_10": 0.48279, + "ndcg_at_100": 0.6064, + "ndcg_at_1000": 0.63736, + "ndcg_at_20": 0.53283, + "ndcg_at_3": 0.47129, + "ndcg_at_5": 0.41457, + "precision_at_1": 0.544, + "precision_at_10": 0.24992, + "precision_at_100": 0.04259, + "precision_at_1000": 0.00496, + "precision_at_20": 0.15413, + "precision_at_3": 0.45027, + "precision_at_5": 0.37588, + "recall_at_1": 0.10886, + "recall_at_10": 0.50001, + "recall_at_100": 0.85215, + "recall_at_1000": 0.99216, + "recall_at_20": 0.6168, + "recall_at_3": 0.27025, + "recall_at_5": 0.376 + } + ] + }, + "task_name": "MSCOCOI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOT2IRetrieval.json b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOT2IRetrieval.json new file mode 100644 index 0000000000..101f427b01 --- /dev/null +++ b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOT2IRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", + "evaluation_time": 93.58067321777344, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.58366, + "map_at_1": 0.41549, + "map_at_10": 0.52488, + "map_at_100": 0.53435, + "map_at_1000": 0.53448, + "map_at_20": 0.5309, + "map_at_3": 0.49362, + "map_at_5": 0.51155, + "mrr_at_1": 0.41601838042645817, + "mrr_at_10": 0.5255243392854887, + "mrr_at_100": 0.5348733342315144, + "mrr_at_1000": 0.5349968576566357, + "mrr_at_20": 0.5314956818071761, + "mrr_at_3": 0.49423596275545995, + "mrr_at_5": 0.5122092788907409, + "nauc_map_at_1000_diff1": 0.5624393151830621, + "nauc_map_at_1000_max": 0.1883984047435345, + "nauc_map_at_1000_std": -0.300281213090816, + "nauc_map_at_100_diff1": 0.5623891134436314, + "nauc_map_at_100_max": 0.18846608709434864, + "nauc_map_at_100_std": -0.3001965535748168, + "nauc_map_at_10_diff1": 0.5603327032580525, + "nauc_map_at_10_max": 0.18715837124430523, + "nauc_map_at_10_std": -0.302544478256696, + "nauc_map_at_1_diff1": 0.6091183832361869, + "nauc_map_at_1_max": 0.19445927993458345, + "nauc_map_at_1_std": -0.2806441521864675, + "nauc_map_at_20_diff1": 0.5614027805327534, + "nauc_map_at_20_max": 0.18786747432769882, + "nauc_map_at_20_std": -0.3015101845331886, + "nauc_map_at_3_diff1": 0.5621560332044606, + "nauc_map_at_3_max": 0.18742943309539936, + "nauc_map_at_3_std": -0.2996055329867434, + "nauc_map_at_5_diff1": 0.5598162631668463, + "nauc_map_at_5_max": 0.18579277507099531, + "nauc_map_at_5_std": -0.30325696648318895, + "nauc_mrr_at_1000_diff1": 0.5618384854801253, + "nauc_mrr_at_1000_max": 0.1893320924160437, + "nauc_mrr_at_1000_std": -0.29919748725758244, + "nauc_mrr_at_100_diff1": 0.5617865037742961, + "nauc_mrr_at_100_max": 0.18940792340998744, + "nauc_mrr_at_100_std": -0.29910507969649847, + "nauc_mrr_at_10_diff1": 0.5595973365694361, + "nauc_mrr_at_10_max": 0.18831353355793895, + "nauc_mrr_at_10_std": -0.3012225279840895, + "nauc_mrr_at_1_diff1": 0.6086677033863201, + "nauc_mrr_at_1_max": 0.19525350096143335, + "nauc_mrr_at_1_std": -0.2797335075064764, + "nauc_mrr_at_20_diff1": 0.5607498445253036, + "nauc_mrr_at_20_max": 0.18897520934910883, + "nauc_mrr_at_20_std": -0.30027624919809554, + "nauc_mrr_at_3_diff1": 0.5614777749783645, + "nauc_mrr_at_3_max": 0.18847496584680384, + "nauc_mrr_at_3_std": -0.29848619493001144, + "nauc_mrr_at_5_diff1": 0.5590866529097342, + "nauc_mrr_at_5_max": 0.18696239905084666, + "nauc_mrr_at_5_std": -0.3019481978381332, + "nauc_ndcg_at_1000_diff1": 0.5544996950498666, + "nauc_ndcg_at_1000_max": 0.1899712267352402, + "nauc_ndcg_at_1000_std": -0.3002746853968903, + "nauc_ndcg_at_100_diff1": 0.5531116875181972, + "nauc_ndcg_at_100_max": 0.19235157602831457, + "nauc_ndcg_at_100_std": -0.2968862811392692, + "nauc_ndcg_at_10_diff1": 0.5418234351636028, + "nauc_ndcg_at_10_max": 0.18525740550465958, + "nauc_ndcg_at_10_std": -0.3108723228378656, + "nauc_ndcg_at_1_diff1": 0.6086677033863201, + "nauc_ndcg_at_1_max": 0.19525350096143335, + "nauc_ndcg_at_1_std": -0.2797335075064764, + "nauc_ndcg_at_20_diff1": 0.5457713006659793, + "nauc_ndcg_at_20_max": 0.18823783560110394, + "nauc_ndcg_at_20_std": -0.30723165105906064, + "nauc_ndcg_at_3_diff1": 0.5465925294717622, + "nauc_ndcg_at_3_max": 0.18498021161717793, + "nauc_ndcg_at_3_std": -0.3052417229375767, + "nauc_ndcg_at_5_diff1": 0.5414141206084832, + "nauc_ndcg_at_5_max": 0.18187192357717108, + "nauc_ndcg_at_5_std": -0.3122724524680724, + "nauc_precision_at_1000_diff1": -0.2717903666306267, + "nauc_precision_at_1000_max": 0.3971860188038004, + "nauc_precision_at_1000_std": 0.552726119141797, + "nauc_precision_at_100_diff1": 0.15897061095455042, + "nauc_precision_at_100_max": 0.42848198127641873, + "nauc_precision_at_100_std": 0.24775900337098317, + "nauc_precision_at_10_diff1": 0.45241283409781885, + "nauc_precision_at_10_max": 0.18182339360260374, + "nauc_precision_at_10_std": -0.34048980216449526, + "nauc_precision_at_1_diff1": 0.6086677033863201, + "nauc_precision_at_1_max": 0.19525350096143335, + "nauc_precision_at_1_std": -0.2797335075064764, + "nauc_precision_at_20_diff1": 0.42977222993638897, + "nauc_precision_at_20_max": 0.20622305981256997, + "nauc_precision_at_20_std": -0.31448081933782945, + "nauc_precision_at_3_diff1": 0.4972012863500112, + "nauc_precision_at_3_max": 0.17750313343860488, + "nauc_precision_at_3_std": -0.3218876469389674, + "nauc_precision_at_5_diff1": 0.47444239516674885, + "nauc_precision_at_5_max": 0.16915063391060323, + "nauc_precision_at_5_std": -0.3422618390030155, + "nauc_recall_at_1000_diff1": 0.3832069088634286, + "nauc_recall_at_1000_max": 0.8901286684038919, + "nauc_recall_at_1000_std": 0.8681810401874042, + "nauc_recall_at_100_diff1": 0.4185975008775339, + "nauc_recall_at_100_max": 0.45763959222653156, + "nauc_recall_at_100_std": 0.1007829133427318, + "nauc_recall_at_10_diff1": 0.4592974110602662, + "nauc_recall_at_10_max": 0.1757829270013686, + "nauc_recall_at_10_std": -0.3502055124676758, + "nauc_recall_at_1_diff1": 0.6091183832361869, + "nauc_recall_at_1_max": 0.19445927993458345, + "nauc_recall_at_1_std": -0.2806441521864675, + "nauc_recall_at_20_diff1": 0.4522393445441416, + "nauc_recall_at_20_max": 0.19461709532525517, + "nauc_recall_at_20_std": -0.3418318635602553, + "nauc_recall_at_3_diff1": 0.49905878884342847, + "nauc_recall_at_3_max": 0.17580402067909676, + "nauc_recall_at_3_std": -0.32419684954654177, + "nauc_recall_at_5_diff1": 0.4770455272089403, + "nauc_recall_at_5_max": 0.1660363394348069, + "nauc_recall_at_5_std": -0.3461249105285916, + "ndcg_at_1": 0.41602, + "ndcg_at_10": 0.58366, + "ndcg_at_100": 0.62879, + "ndcg_at_1000": 0.63154, + "ndcg_at_20": 0.6053, + "ndcg_at_3": 0.51938, + "ndcg_at_5": 0.55166, + "precision_at_1": 0.41602, + "precision_at_10": 0.0772, + "precision_at_100": 0.00985, + "precision_at_1000": 0.00101, + "precision_at_20": 0.04291, + "precision_at_3": 0.1981, + "precision_at_5": 0.1346, + "recall_at_1": 0.41549, + "recall_at_10": 0.76983, + "recall_at_100": 0.9782, + "recall_at_1000": 0.99865, + "recall_at_20": 0.85476, + "recall_at_3": 0.59323, + "recall_at_5": 0.67157 + } + ] + }, + "task_name": "MSCOCOT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/STS12.json b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/STS12.json new file mode 100644 index 0000000000..a5b65a252d --- /dev/null +++ b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/STS12.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", + "evaluation_time": 5.472690582275391, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "cosine_pearson": 0.8338917619581533, + "cosine_spearman": 0.7351972128901206, + "euclidean_pearson": 0.7935770054482947, + "euclidean_spearman": 0.7351757102447369, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7351972128901206, + "manhattan_pearson": 0.7933747110044767, + "manhattan_spearman": 0.7344695134800555, + "pearson": 0.8338917619581533, + "spearman": 0.7351972128901206 + } + ] + }, + "task_name": "STS12" +} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/model_meta.json b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/model_meta.json new file mode 100644 index 0000000000..1cef0d24b8 --- /dev/null +++ b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/model_meta.json @@ -0,0 +1 @@ +{"name": "jinaai/jina-clip-v1", "revision": "1cbe5e8b11ea3728df0b610d5453dfe739804aa9", "release_date": "2024-05-30", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "JinaCLIPModelWrapper"} \ No newline at end of file From c129ae27d77efc00efb0954ac86e9fdbdbfca7f0 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 31 Jul 2024 15:57:53 +0300 Subject: [PATCH 034/154] [MIEB] Update `mieb` with the `main` branch and some fixes (#1126) * fix instruction retrival (#1072) * fix instruction retrival * fix test * add points * make nested results * add test * skip instruction test * fix instruction passes * fix unions * move do_length_ablation Co-authored-by: Kenneth Enevoldsen --------- Co-authored-by: Kenneth Enevoldsen * Update points table * fix: fix bug-causing spelling error in function name of e5-mistral-instruct (#1106) found bug * 1.12.85 Automatically generated by python-semantic-release * fix: MultilingualSentimentClassification (#1109) * Update points table * fix: Avoid spaces in dataset name for CQADupstack and ignore speed tasks * 1.12.86 Automatically generated by python-semantic-release * fix: Ensure that MLSUMClusteringP2P.v2 use the fast implementation as was intended (#1112) * fix: Ensure that MLSUMClusteringP2P.v2 use the fast implementation as was intended * fix: fixed formatting for cli * docs: improve searchability in the advanced usage documentation * 1.12.87 Automatically generated by python-semantic-release * docs: improve searchability in the advanced usage documentation (#1113) * docs: improve searchability in the advanced usage documentation * docs: update based on corrections * fix: export type for `mteb create_meta` (#1114) * fix export type * fix dataset version too * 1.12.88 Automatically generated by python-semantic-release * fix: Simplify models implementations (#1085) * Merge * Adapt * Simplify * Check for rev again * Rmv cmmnt * Simplify * simplify * Rmv comment Co-authored-by: Kenneth Enevoldsen * Use logging; change try except; add info * Lint * Rmv results * Update rev * format * Simplify models; Allow instructions * Jobs * Fix merge * Format * Adapt models * fix: ensure that e5 ignores the NQ * format --------- Co-authored-by: Kenneth Enevoldsen * 1.12.89 Automatically generated by python-semantic-release * fix: nomic models using prefix correctly (#1125) * fix: nomic models using prefix correctly * chore: remove comment * fix: handling in case not torch tensor * Fix typo --------- Co-authored-by: Niklas Muennighoff * 1.12.90 Automatically generated by python-semantic-release * refactor vista model wrapper to contain lib import * python 38 type hints --------- Co-authored-by: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Co-authored-by: Kenneth Enevoldsen Co-authored-by: github-actions[bot] Co-authored-by: anpalmak2003 <73543260+anpalmak2003@users.noreply.github.com> Co-authored-by: github-actions Co-authored-by: Niklas Muennighoff Co-authored-by: Zach Nussbaum Co-authored-by: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> --- README.md | 34 +- docs/mmteb/points/1072.jsonl | 1 + docs/mmteb/points_table.md | 184 ++++----- mteb/abstasks/AbsTaskInstructionRetrieval.py | 356 ++++++++---------- mteb/cli.py | 7 +- .../InstructionRetrievalEvaluator.py | 5 +- .../evaluators/RetrievalEvaluator.py | 65 ++-- mteb/evaluation/evaluators/utils.py | 2 +- mteb/load_results/mteb_results.py | 10 +- mteb/models/__init__.py | 3 +- mteb/models/cohere_models.py | 44 ++- mteb/models/e5_instruct.py | 275 +++----------- mteb/models/e5_models.py | 3 +- mteb/models/e5_v.py | 2 + mteb/models/google_models.py | 53 +-- mteb/models/gritlm_models.py | 19 +- mteb/models/gte_models.py | 24 +- mteb/models/nomic_models.py | 116 +++++- mteb/models/salesforce_models.py | 26 +- mteb/models/vista_models.py | 82 ++-- mteb/models/voyage_models.py | 8 +- .../MultilingualSentimentClassification.py | 4 +- .../multilingual/MLSUMClusteringP2P.py | 3 +- pyproject.toml | 2 +- .../mmteb/running_model/create_slurm_jobs.py | 26 +- tests/test_benchmark/mock_tasks.py | 71 ++++ tests/test_benchmark/task_grid.py | 2 + 27 files changed, 709 insertions(+), 718 deletions(-) create mode 100644 docs/mmteb/points/1072.jsonl diff --git a/README.md b/README.md index 0aaf487239..9589dfcca8 100644 --- a/README.md +++ b/README.md @@ -69,14 +69,15 @@ mteb run -m sentence-transformers/all-MiniLM-L6-v2 \ * Using multiple GPUs in parallel can be done by just having a custom encode function that distributes the inputs to multiple GPUs like e.g. [here](https://github.com/microsoft/unilm/blob/b60c741f746877293bb85eed6806736fc8fa0ffd/e5/mteb_eval.py#L60) or [here](https://github.com/ContextualAI/gritlm/blob/09d8630f0c95ac6a456354bcb6f964d7b9b6a609/gritlm/gritlm.py#L75). -
- -
- Advanced Usage (click to unfold) ## Advanced Usage +Click on each section below to see the details. +
+ +
+ Dataset selection ### Dataset selection @@ -127,6 +128,12 @@ from mteb import MTEB_MAIN_EN evaluation = mteb.MTEB(tasks=MTEB_MAIN_EN, task_langs=["en"]) ``` +
+ +
+ Passing in `encode` arguments + + ### Passing in `encode` arguments To pass in arguments to the model's `encode` function, you can use the encode keyword arguments (`encode_kwargs`): @@ -134,8 +141,13 @@ To pass in arguments to the model's `encode` function, you can use the encode ke ```python evaluation.run(model, encode_kwargs={"batch_size": 32} ``` +
+ -### Evaluation split +
+ Selecting evaluation split + +### Selecting evaluation split You can evaluate only on `test` splits of all tasks by doing the following: ```python @@ -144,6 +156,12 @@ evaluation.run(model, eval_splits=["test"]) Note that the public leaderboard uses the test splits for all datasets except MSMARCO, where the "dev" split is used. +
+ +
+ Using a custom model + + ### Using a custom model Models should implement the following interface, implementing an `encode` function taking as inputs a list of sentences, and returning a list of embeddings (embeddings can be `np.array`, `torch.tensor`, etc.). For inspiration, you can look at the [mteb/mtebscripts repo](https://github.com/embeddings-benchmark/mtebscripts) used for running diverse models via SLURM scripts for the paper. @@ -198,6 +216,12 @@ class MyModel(): pass ``` +
+ +
+ Evaluating on a custom dataset + + ### Evaluating on a custom dataset To evaluate on a custom task, you can run the following code on your custom task. See [how to add a new task](docs/adding_a_dataset.md), for how to create a new task in MTEB. diff --git a/docs/mmteb/points/1072.jsonl b/docs/mmteb/points/1072.jsonl new file mode 100644 index 0000000000..db16443a1b --- /dev/null +++ b/docs/mmteb/points/1072.jsonl @@ -0,0 +1 @@ +{"GitHub": "Samoed", "Bug fixes": 2} diff --git a/docs/mmteb/points_table.md b/docs/mmteb/points_table.md index bad296dbad..076c1c0902 100644 --- a/docs/mmteb/points_table.md +++ b/docs/mmteb/points_table.md @@ -2,95 +2,95 @@ _Note_: this table is **autogenerated** and should not be edited. It is intended to get an overview of contributions. - | GitHub | New dataset | Review PR | Bug fixes | New task | Dataset annotations | Coordination | Running Models | Paper writing | Total | -|:------------------|--------------:|------------:|------------:|-----------:|----------------------:|---------------:|-----------------:|----------------:|--------:| -| KennethEnevoldsen | 68 | 298 | 85 | 0 | 35 | 11 | 0 | 0 | 497 | -| isaac-chung | 120 | 194 | 50 | 2 | 1 | 4 | 0 | 12 | 383 | -| awinml | 300 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 302 | -| imenelydiaker | 120 | 144 | 24 | 0 | 0 | 0 | 0 | 0 | 288 | -| x-tabdeveloping | 144 | 32 | 10 | 12 | 0 | 1 | 0 | 0 | 199 | -| davidstap | 176 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 176 | -| jaygala24 | 149 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 149 | -| wissam-sib | 134 | 6 | 4 | 0 | 0 | 0 | 0 | 0 | 144 | -| gentaiscool | 110 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 110 | -| jupyterjazz | 108 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 108 | -| dokato | 92 | 6 | 10 | 0 | 0 | 0 | 0 | 0 | 108 | -| SaitejaUtpala | 102 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 102 | -| GabrielSequeira | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | -| MathieuCiancone | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | -| schmarion | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | -| digantamisra98 | 71 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 71 | -| shreeya-dhakal | 54 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 62 | -| Rysias | 58 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 58 | -| gowitheflow-1998 | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 50 | -| asparius | 34 | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 48 | -| orionw | 0 | 18 | 20 | 10 | 0 | 0 | 0 | 0 | 48 | -| Akash190104 | 46 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 46 | -| Muennighoff | 0 | 44 | 0 | 0 | 0 | 0 | 0 | 0 | 44 | -| staoxiao | 40 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 40 | -| akshita-sukhlecha | 36 | 0 | 4 | 0 | 0 | 0 | 0 | 0 | 40 | -| rafalposwiata | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | -| bp-high | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | -| rasdani | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| ShawonAshraf | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| jphme | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| loicmagne | 0 | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 28 | -| bjoernpl | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| violenil | 26 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 26 | -| kranthigv | 20 | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 26 | -| dwzhu-pku | 24 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 24 | -| MartinBernstorff | 2 | 8 | 13 | 0 | 0 | 0 | 0 | 0 | 23 | -| hgissbkh | 0 | 2 | 13 | 5 | 0 | 0 | 0 | 3 | 23 | -| taeminlee | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | -| jankounchained | 14 | 0 | 8 | 0 | 0 | 0 | 0 | 0 | 22 | -| crystina-z | 21 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 21 | -| mrshu | 16 | 4 | 0 | 0 | 1 | 0 | 0 | 0 | 21 | -| mmhamdy | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | -| Andrian0s | 14 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 20 | -| ManuelFay | 2 | 0 | 13 | 5 | 0 | 0 | 0 | 0 | 20 | -| rbroc | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | -| AlexeyVatolin | 0 | 0 | 20 | 0 | 0 | 0 | 0 | 0 | 20 | -| vaibhavad | 6 | 4 | 8 | 0 | 0 | 0 | 0 | 0 | 18 | -| manandey | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | -| thakur-nandan | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | -| dipam7 | 14 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | -| sted97 | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | -| PranjalChitale | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | -| Sakshamrzt | 12 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | -| taidnguyen | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 14 | -| artemsnegirev | 12 | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 14 | -| Samoed | 0 | 0 | 4 | 0 | 0 | 0 | 9 | 0 | 13 | -| mariyahendriksen | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | 12 | -| xhluca | 6 | 2 | 4 | 0 | 0 | 0 | 0 | 0 | 12 | -| anpalmak2003 | 9 | 0 | 0 | 0 | 3 | 0 | 0 | 0 | 12 | -| jordiclive | 2 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 12 | -| slvnwhrl | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | -| Art3mis07 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | -| guenthermi | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | -| henilp105 | 0 | 0 | 2 | 0 | 9 | 0 | 0 | 0 | 11 | -| MariyaTikhonova | 7 | 0 | 0 | 0 | 4 | 0 | 0 | 0 | 11 | -| ab1992ao | 8 | 0 | 0 | 0 | 3 | 0 | 0 | 0 | 11 | -| simon-clematide | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| ABorghini | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| ZhengLiu101 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| cassanof | 8 | 0 | 1 | 0 | 0 | 0 | 1 | 0 | 10 | -| Alenush | 6 | 0 | 0 | 0 | 4 | 0 | 0 | 0 | 10 | -| HLasse | 0 | 0 | 5 | 0 | 5 | 0 | 0 | 0 | 10 | -| Ruqyai | 2 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| malteos | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| xu3kev | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| guangyusong | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| ljvmiranda921 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| KranthiGV | 0 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 8 | -| izhx | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| marcobellagente93 | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| cslizc | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| bakrianoo | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| antoniolanza1996 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | -| achibb | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| hanhainebula | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| PhilipMay | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| NouamaneTazi | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| tomaarsen | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| Muenninghoff | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| MexicanLemonade | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | \ No newline at end of file + | GitHub | New dataset | Review PR | Bug fixes | Running Models | New task | Paper writing | Coordination | Dataset annotations | Total | +|:------------------|--------------:|------------:|------------:|-----------------:|-----------:|----------------:|---------------:|----------------------:|--------:| +| KennethEnevoldsen | 68 | 298 | 85 | 0 | 0 | 0 | 11 | 35 | 497 | +| isaac-chung | 120 | 194 | 50 | 0 | 2 | 12 | 4 | 1 | 383 | +| awinml | 300 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 302 | +| imenelydiaker | 120 | 144 | 24 | 0 | 0 | 0 | 0 | 0 | 288 | +| x-tabdeveloping | 144 | 32 | 10 | 0 | 12 | 0 | 1 | 0 | 199 | +| davidstap | 176 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 176 | +| jaygala24 | 149 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 149 | +| wissam-sib | 134 | 6 | 4 | 0 | 0 | 0 | 0 | 0 | 144 | +| gentaiscool | 110 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 110 | +| jupyterjazz | 108 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 108 | +| dokato | 92 | 6 | 10 | 0 | 0 | 0 | 0 | 0 | 108 | +| SaitejaUtpala | 102 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 102 | +| GabrielSequeira | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| MathieuCiancone | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| schmarion | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| digantamisra98 | 71 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 71 | +| shreeya-dhakal | 54 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 62 | +| Rysias | 58 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 58 | +| gowitheflow-1998 | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 50 | +| asparius | 34 | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 48 | +| orionw | 0 | 18 | 20 | 0 | 10 | 0 | 0 | 0 | 48 | +| Akash190104 | 46 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 46 | +| Muennighoff | 0 | 44 | 0 | 0 | 0 | 0 | 0 | 0 | 44 | +| staoxiao | 40 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 40 | +| akshita-sukhlecha | 36 | 0 | 4 | 0 | 0 | 0 | 0 | 0 | 40 | +| rafalposwiata | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | +| bp-high | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | +| rasdani | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| ShawonAshraf | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| jphme | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| loicmagne | 0 | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 28 | +| bjoernpl | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| violenil | 26 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 26 | +| kranthigv | 20 | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 26 | +| dwzhu-pku | 24 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 24 | +| MartinBernstorff | 2 | 8 | 13 | 0 | 0 | 0 | 0 | 0 | 23 | +| hgissbkh | 0 | 2 | 13 | 0 | 5 | 3 | 0 | 0 | 23 | +| taeminlee | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | +| jankounchained | 14 | 0 | 8 | 0 | 0 | 0 | 0 | 0 | 22 | +| crystina-z | 21 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 21 | +| mrshu | 16 | 4 | 0 | 0 | 0 | 0 | 0 | 1 | 21 | +| mmhamdy | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | +| Andrian0s | 14 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 20 | +| ManuelFay | 2 | 0 | 13 | 0 | 5 | 0 | 0 | 0 | 20 | +| rbroc | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | +| AlexeyVatolin | 0 | 0 | 20 | 0 | 0 | 0 | 0 | 0 | 20 | +| vaibhavad | 6 | 4 | 8 | 0 | 0 | 0 | 0 | 0 | 18 | +| manandey | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | +| thakur-nandan | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | +| dipam7 | 14 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| sted97 | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| PranjalChitale | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| Sakshamrzt | 12 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| Samoed | 0 | 0 | 6 | 9 | 0 | 0 | 0 | 0 | 15 | +| taidnguyen | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 14 | +| artemsnegirev | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | 14 | +| mariyahendriksen | 0 | 0 | 0 | 0 | 0 | 12 | 0 | 0 | 12 | +| xhluca | 6 | 2 | 4 | 0 | 0 | 0 | 0 | 0 | 12 | +| anpalmak2003 | 9 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | 12 | +| jordiclive | 2 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 12 | +| slvnwhrl | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| Art3mis07 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| guenthermi | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| henilp105 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 9 | 11 | +| MariyaTikhonova | 7 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | 11 | +| ab1992ao | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | 11 | +| simon-clematide | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| ABorghini | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| ZhengLiu101 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| cassanof | 8 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 10 | +| Alenush | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | 10 | +| HLasse | 0 | 0 | 5 | 0 | 0 | 0 | 0 | 5 | 10 | +| Ruqyai | 2 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| malteos | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| xu3kev | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| guangyusong | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| ljvmiranda921 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| KranthiGV | 0 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 8 | +| izhx | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| marcobellagente93 | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| cslizc | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| bakrianoo | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| antoniolanza1996 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | +| achibb | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| hanhainebula | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| PhilipMay | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| NouamaneTazi | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| tomaarsen | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| Muenninghoff | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| MexicanLemonade | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | \ No newline at end of file diff --git a/mteb/abstasks/AbsTaskInstructionRetrieval.py b/mteb/abstasks/AbsTaskInstructionRetrieval.py index 9f96973f6e..9b24b72a48 100644 --- a/mteb/abstasks/AbsTaskInstructionRetrieval.py +++ b/mteb/abstasks/AbsTaskInstructionRetrieval.py @@ -5,10 +5,10 @@ import os from collections import defaultdict from time import time -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Tuple, Union import tqdm -from datasets import Features, Value, load_dataset +from datasets import Dataset, Features, Value, load_dataset from mteb.encoder_interface import Encoder @@ -72,10 +72,11 @@ def __init__( def load( self, split="test" ) -> Tuple[ - Dict[str, Dict[str, str]], - Dict[str, str], + Dataset, + Dataset, Dict[str, Dict[str, int]], Dict[str, Dict[str, int]], + Dataset, ]: if not self.hf_repo: self.og_qrels_file = os.path.join(self.qrels_folder + "_og", split + ".tsv") @@ -223,28 +224,18 @@ class AbsTaskInstructionRetrieval(AbsTask): instruction: A relevant document will provide the projected or actual date of completion of the project, its estimated or actual total cost, or the estimated or ongoing electrical output of the finished project. Discussions of the social, political, or ecological impact of the project are not relevant. Child-classes must implement the following properties: - self.corpus = Dict[id, Dict[str, str]] #id => dict with document datas like title and text - self.queries = Dict[id, str] #id => query - self.relevant_docs = List[id, id, score] + self.corpus = Dict[corpus_id, Dict[str, str]] #id => dict with document datas like title and text + self.queries = Dict[query_id, str] #id => query + self.relevant_docs = Dict[query_id, Dict[corpus_id, int]] self.og_instructions = Dict[str, str] query => original instruction self.changed_instructions = Dict[str, str] query => changed instruction - self.top_ranked = Dict[id, List[id]] #id => list of top ranked document ids + self.top_ranked = Dict[query_id, List[corpus_id]] #id => list of top ranked document ids See https://arxiv.org/abs/2403.15246 for more details """ def __init__( self, - hf_repo: str = None, - hf_repo_qrels: str = None, - data_folder: str = None, - prefix: str = None, - corpus_file: str = "corpus.jsonl", - query_file: str = "queries.jsonl", - qrels_folder: str = "qrels", - qrels_file: str = "", - streaming: bool = False, - keep_in_memory: bool = False, **kwargs, ): super().__init__(**kwargs) @@ -271,14 +262,18 @@ def load_data(self, **kwargs): dataset_path + "-qrels" if "clarin-knext" in dataset_path else None ) for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): - corpus, queries, og_qrels, changed_qrels, top_ranked_init = ( - HFDataLoaderInstructions( - hf_repo=dataset_path, - hf_repo_qrels=hf_repo_qrels, - streaming=False, - keep_in_memory=False, - ).load(split=split) - ) + ( + corpus, + queries, + og_relevant_docs, + changed_relevant_docs, + top_ranked_init, + ) = HFDataLoaderInstructions( + hf_repo=dataset_path, + hf_repo_qrels=hf_repo_qrels, + streaming=False, + keep_in_memory=False, + ).load(split=split) # Conversion from DataSet top_ranked = defaultdict(list) @@ -311,7 +306,7 @@ def load_data(self, **kwargs): self.queries[split], self.og_relevant_docs[split], self.changed_relevant_docs[split], - ) = corpus, queries, og_qrels, changed_qrels + ) = corpus, queries, og_relevant_docs, changed_relevant_docs self.changed_instructions[split], self.og_instructions[split] = ( changed_instructions, og_instructions, @@ -326,205 +321,173 @@ def load_data(self, **kwargs): self.data_loaded = True - def evaluate( + def _evaluate_subset_lang( self, - model: Encoder, - split: str = "test", - *, - encode_kwargs: dict[str, Any] = {}, + retriever: InstructionRetrievalEvaluator, + corpus: Dict, + queries: Dict, + og_relevant_docs: Dict, + changed_relevant_docs: Dict, + og_instructions: Dict, + changed_instructions: Dict, + top_ranked: Dict, + lang: str, + split: str, + keywords: Union[Dict, None] = None, + short_instructions: Union[Dict, None] = None, **kwargs, - ): - retriever = InstructionRetrievalEvaluator( - model=model, - task_name=self.metadata.name, - encode_kwargs=encode_kwargs, - **kwargs, + ) -> Dict[str, Union[Dict[str, float], float]]: + corpus, queries = corpus[split], queries[split] + og_relevant_docs, changed_relevant_docs = ( + og_relevant_docs[split], + changed_relevant_docs[split], + ) + og_instructions, changed_instructions = ( + og_instructions[split], + changed_instructions[split], ) - scores_og = {} - scores_changed = {} - results_og = {} - results_changed = {} - scores_base = {} - results_base = {} - overall_changed_scores = {} - if self.is_multilingual: - for lang in self.hf_subsets: - logger.info(f"Language: {lang}") - corpus, queries = self.corpus[lang][split], self.queries[lang][split] - og_relevant_docs, changed_relevant_docs = ( - self.og_relevant_docs[lang][split], - self.changed_relevant_docs[lang][split], - ) - og_instructions, changed_instructions = ( - self.og_instructions[lang][split], - self.changed_instructions[lang][split], - ) - - top_ranked = self.top_ranked[lang][split] - scores_og[lang], results_og[lang] = self._evaluate_subset( - retriever, - corpus, - queries, - og_relevant_docs, - og_instructions, - top_ranked, - lang, - **kwargs, - ) - scores_changed[lang], results_changed[lang] = self._evaluate_subset( - retriever, - corpus, - queries, - changed_relevant_docs, - changed_instructions, - top_ranked, - lang, - **kwargs, - ) + top_ranked = top_ranked[split] + scores_og, results_og = self._evaluate_subset( + retriever, + corpus, + queries, + og_relevant_docs, + og_instructions, + top_ranked, + lang, + **kwargs, + ) + scores_changed, results_changed = self._evaluate_subset( + retriever, + corpus, + queries, + changed_relevant_docs, + changed_instructions, + top_ranked, + lang, + **kwargs, + ) - newly_irrelevant_qrels = self.create_qrel_diff( - self.og_relevant_docs[lang][split], - self.changed_relevant_docs[lang][split], - ) - overall_changed_scores[lang] = utils.evaluate_change( - results_og[lang], results_changed[lang], newly_irrelevant_qrels - ) + newly_irrelevant_qrels = self.create_qrel_diff( + og_relevant_docs, + changed_relevant_docs, + ) + overall_changed_scores = utils.evaluate_change( + results_og, results_changed, newly_irrelevant_qrels + ) - overall_changed_scores[lang]["individual"] = { - "original": scores_og[lang], - "changed": scores_changed[lang], - } + overall_changed_scores["individual"] = { + "original": scores_og, + "changed": scores_changed, + } - if self.do_length_ablation: - keywords, short_instructions = ( - self.keywords[lang][split], - self.short_instructions[lang][split], - ) - scores_base[lang], results_base[lang] = self._evaluate_subset( - retriever, - corpus, - queries, - og_relevant_docs, - defaultdict(str), - top_ranked, - lang, - **kwargs, - ) - scores_w_keywords = self._evaluate_subset( - retriever, - corpus, - queries, - og_relevant_docs, - keywords, - top_ranked, - lang, - **kwargs, - ) - scores_w_short_instr = self._evaluate_subset( - retriever, - corpus, - queries, - og_relevant_docs, - short_instructions, - top_ranked, - lang, - **kwargs, - ) - overall_changed_scores[lang]["length_ablation"] = { - "keywords": scores_w_keywords, - "short_instructions": scores_w_short_instr, - "base": scores_base[lang], - } - else: # seems like these two can be combined into one (with the new lang="default") - lang = "default" - corpus, queries = self.corpus[split], self.queries[split] - og_relevant_docs, changed_relevant_docs = ( - self.og_relevant_docs[split], - self.changed_relevant_docs[split], - ) - og_instructions, changed_instructions = ( - self.og_instructions[split], - self.changed_instructions[split], + if self.do_length_ablation: + keywords, short_instructions = ( + keywords[split], + short_instructions[split], ) - top_ranked = self.top_ranked[split] - - scores_og, results_og = self._evaluate_subset( + scores_base, results_base = self._evaluate_subset( retriever, corpus, queries, og_relevant_docs, - og_instructions, + defaultdict(str), top_ranked, - None, + lang, **kwargs, ) - - scores_changed, results_changed = self._evaluate_subset( + scores_w_keywords_scores, scores_w_keywords_results = self._evaluate_subset( retriever, corpus, queries, - changed_relevant_docs, - changed_instructions, + og_relevant_docs, + keywords, top_ranked, - None, + lang, **kwargs, ) - - newly_irrelevant_qrels = self.create_qrel_diff( - self.og_relevant_docs[split], self.changed_relevant_docs[split] - ) - overall_changed_scores[lang] = utils.evaluate_change( - results_og, results_changed, newly_irrelevant_qrels - ) - - overall_changed_scores[lang]["individual"] = { - "original": scores_og, - "changed": scores_changed, - } - - if self.do_length_ablation: - keywords, short_instructions = ( - self.keywords[split], - self.short_instructions[split], - ) - scores_w_keywords = self._evaluate_subset( - retriever, - corpus, - queries, - og_relevant_docs, - keywords, - top_ranked, - None, - **kwargs, - ) - scores_w_short_instr = self._evaluate_subset( + scores_w_short_instr_scores, scores_w_short_instr_result = ( + self._evaluate_subset( retriever, corpus, queries, og_relevant_docs, short_instructions, top_ranked, - None, + lang, **kwargs, ) - scores_base, results_base = self._evaluate_subset( + ) + overall_changed_scores["length_ablation"] = { + "keywords": scores_w_keywords_scores, + "short_instructions": scores_w_short_instr_scores, + "base": scores_base, + } + + return overall_changed_scores + + def evaluate( + self, + model: Encoder, + split: str = "test", + *, + encode_kwargs: Dict[str, Any] = {}, + **kwargs, + ) -> Dict[str, Dict[str, Any]]: + retriever = InstructionRetrievalEvaluator( + retriever=model, + task_name=self.metadata.name, + encode_kwargs=encode_kwargs, + **kwargs, + ) + scores = {} + if self.is_multilingual: + for lang in self.hf_subsets: + logger.info(f"Language: {lang}") + scores[lang] = self._evaluate_subset_lang( retriever, - corpus, - queries, - og_relevant_docs, - defaultdict(str), - top_ranked, - None, + corpus=self.corpus[lang], + queries=self.queries[lang], + og_relevant_docs=self.og_relevant_docs[lang], + changed_relevant_docs=self.changed_relevant_docs[lang], + og_instructions=self.og_instructions[lang], + changed_instructions=self.changed_instructions[lang], + top_ranked=self.top_ranked[lang], + lang=lang, + split=split, + keywords=self.keywords[lang] if self.do_length_ablation else None, + short_instructions=self.short_instructions[lang] + if self.do_length_ablation + else None, **kwargs, ) - overall_changed_scores[lang]["length_ablation"] = { - "keywords": scores_w_keywords, - "short_instructions": scores_w_short_instr, - "base": scores_base, - } + self._add_main_score(scores[lang]) + else: + lang = "default" + scores[lang] = self._evaluate_subset_lang( + retriever, + corpus=self.corpus, + queries=self.queries, + og_relevant_docs=self.og_relevant_docs, + changed_relevant_docs=self.changed_relevant_docs, + og_instructions=self.og_instructions, + changed_instructions=self.changed_instructions, + top_ranked=self.top_ranked, + lang=lang, + split=split, + keywords=self.keywords if self.do_length_ablation else None, + short_instructions=self.short_instructions + if self.do_length_ablation + else None, + **kwargs, + ) + self._add_main_score(scores[lang]) - return overall_changed_scores + return scores + + def _add_main_score(self, scores: dict[str, dict[str, float]]) -> None: + scores["main_score"] = scores[self.metadata.main_score] def _evaluate_subset( self, @@ -536,7 +499,7 @@ def _evaluate_subset( top_ranked: Dict[str, List[str]], lang=None, **kwargs, - ): + ) -> Tuple[Dict[str, float], Dict[str, Dict[str, float]]]: start_time = time() # do the results by query and relevant docs only @@ -589,13 +552,13 @@ def _evaluate_subset( with open(qrels_save_path, "w") as f: json.dump(results, f) - ndcg, _map, recall, precision = retriever.evaluate( + ndcg, _map, recall, precision, naucs = retriever.evaluate( relevant_docs, results, retriever.k_values, ignore_identical_ids=kwargs.get("ignore_identical_ids", True), ) - mrr = retriever.evaluate_custom( + mrr, naucs = retriever.evaluate_custom( relevant_docs, results, retriever.k_values, "mrr" ) scores = { @@ -603,6 +566,7 @@ def _evaluate_subset( **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()}, **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()}, **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()}, + **{f"naucs_at_{k.split('@')[1]}": v for (k, v) in naucs.items()}, **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, } return scores, results diff --git a/mteb/cli.py b/mteb/cli.py index 138ea46acc..f0acf437c3 100644 --- a/mteb/cli.py +++ b/mteb/cli.py @@ -292,7 +292,7 @@ def potentially_add_cqadupstack_to_results(results: list[mteb.MTEBResults]) -> N result = mteb.MTEBResults( task_name="CQADupstackRetrieval", - dataset_revision="CQADupstackRetrieval is a combined dataset", + dataset_revision="CQADupstackRetrieval_is_a_combined_dataset", mteb_version="NA", scores=scores, evaluation_time=evaluation_time, @@ -319,6 +319,11 @@ def create_meta(args: argparse.Namespace) -> None: ] task_results = [MTEBResults.from_disk(path) for path in json_files] + task_results = [ + results + for results in task_results + if results.task_name not in ["GPUSpeedTask", "CPUSpeedTask"] + ] potentially_add_cqadupstack_to_results( task_results ) # We should ideally find better way in the future to aggregate scores for tasks like CQADupstack diff --git a/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py b/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py index a1b1d1e582..8828aa2a22 100644 --- a/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py @@ -1,5 +1,5 @@ import logging -from typing import Dict +from typing import Dict, Union from .RetrievalEvaluator import ( RetrievalEvaluator, @@ -15,6 +15,7 @@ def __call__( corpus: Dict[str, Dict[str, str]], queries: Dict[str, str], instructions: Dict[str, str], + qid: Union[str, None] = None, **kwargs, ) -> Dict[str, Dict[str, float]]: if not self.retriever: @@ -31,5 +32,7 @@ def __call__( self.top_k, self.score_function, instructions=instructions, + request_qid=qid, + prompt_name=self.task_name, **kwargs, ) diff --git a/mteb/evaluation/evaluators/RetrievalEvaluator.py b/mteb/evaluation/evaluators/RetrievalEvaluator.py index bf5b62a720..3f767d83fe 100644 --- a/mteb/evaluation/evaluators/RetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/RetrievalEvaluator.py @@ -84,6 +84,8 @@ def search( top_k: int, score_function: str, prompt_name: str, + instructions: Dict[str, str] | None = None, + request_qid: Union[str, None] = None, return_sorted: bool = False, **kwargs, ) -> dict[str, dict[str, float]]: @@ -99,6 +101,8 @@ def search( query_ids = list(queries.keys()) self.results = {qid: {} for qid in query_ids} queries = [queries[qid] for qid in queries] # type: ignore + if instructions: + queries = [f"{query} {instructions[query]}".strip() for query in queries] if isinstance(queries[0], list): # type: ignore query_embeddings = self.encode_conversations( model=self.model, @@ -140,21 +144,22 @@ def search( # Encode chunk of corpus if ( self.save_corpus_embeddings - and "qid" in kwargs - and len(self.corpus_embeddings[kwargs["qid"]]) + and request_qid + and len(self.corpus_embeddings[request_qid]) ): sub_corpus_embeddings = torch.tensor( - self.corpus_embeddings[kwargs["qid"]][batch_num] + self.corpus_embeddings[request_qid][batch_num] ) else: # Encode chunk of corpus sub_corpus_embeddings = self.model.encode_corpus( corpus[corpus_start_idx:corpus_end_idx], # type: ignore prompt_name=prompt_name, + request_qid=request_qid, **self.encode_kwargs, ) - if self.save_corpus_embeddings and "qid" in kwargs: - self.corpus_embeddings[kwargs["qid"]].append(sub_corpus_embeddings) + if self.save_corpus_embeddings and request_qid: + self.corpus_embeddings[request_qid].append(sub_corpus_embeddings) # Compute similarites using either cosine-similarity or dot product cos_scores = self.score_functions[score_function]( @@ -224,7 +229,7 @@ def search_cross_encoder( corpus: Dict[str, Dict[str, str]], queries: Dict[str, Union[str, List[str]]], top_k: int, - instructions: Dict[str, str] | None = None, + instructions: Union[Dict[str, str], None] = None, **kwargs, ) -> Dict[str, Dict[str, float]]: """This function provides support for reranker (or cross-encoder) models that encoder query and document at the same time (typically with attention). @@ -244,7 +249,7 @@ def search_cross_encoder( top_n = [k for k, v in list(q_results_sorted.items())[:top_k]] query = queries[qid] query = ( - self.convert_conv_history_to_query([query])[0] + self.convert_conv_history_to_query(self.model, [query])[0] if isinstance(query, list) else query ) @@ -354,36 +359,28 @@ def encode_queries( "Queries will not be truncated. This could lead to memory issues. In that case please lower the batch_size." ) - if "instructions" in kwargs: - if kwargs["instructions"] is not None: - queries = [ - (query + " " + kwargs["instructions"][query]).strip() - for query in queries - ] - new_kwargs = { - k: v for k, v in kwargs.items() if k not in ["instructions", "qid"] - } - else: - # can't just delete, cuz assign by reference on kwargs - new_kwargs = kwargs - return model_encode( queries, model=self.model, prompt_name=prompt_name, batch_size=batch_size, - **new_kwargs, + **kwargs, ) def encode_corpus( - self, corpus: List[Dict[str, str]], prompt_name: str, batch_size: int, **kwargs + self, + corpus: List[Dict[str, str]], + prompt_name: str, + batch_size: int, + request_qid: Union[str, None] = None, + **kwargs, ): if ( - "qid" in kwargs + request_qid and self.save_corpus_embeddings and len(self.corpus_embeddings) > 0 ): - return self.corpus_embeddings[kwargs["qid"]] + return self.corpus_embeddings[request_qid] if isinstance(corpus, dict): sentences = [ @@ -400,24 +397,16 @@ def encode_corpus( for doc in corpus ] - if "instructions" in kwargs: # not used on the doc side - new_kwargs = { - k: v for k, v in kwargs.items() if k not in ["instructions", "qid"] - } - else: - # can't just delete, cuz assign by reference on kwargs - new_kwargs = kwargs - corpus_embeddings = model_encode( sentences, model=self.model, prompt_name=prompt_name, batch_size=batch_size, - **new_kwargs, + **kwargs, ) - if self.save_corpus_embeddings and "qid" in kwargs: - self.corpus_embeddings[kwargs["qid"]] = corpus_embeddings + if self.save_corpus_embeddings and request_qid: + self.corpus_embeddings[request_qid] = corpus_embeddings return corpus_embeddings def encode(self, sentences: List[str], prompt_name: str, **kwargs): @@ -434,9 +423,7 @@ def is_dres_compatible(model): def is_cross_encoder_compatible(model): op = getattr(model, "predict", None) - if not (callable(op)): - return False - return True + return callable(op) # Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/evaluation.py#L9 @@ -588,7 +575,7 @@ def evaluate_custom( k_values: List[int], metric: str, output_type: str = "all", - ) -> Tuple[Dict[str, float]]: + ) -> Tuple[Dict[str, float], Dict[str, float]]: if metric.lower() in ["mrr", "mrr@k", "mrr_cut"]: metric_scores = mrr(qrels, results, k_values, output_type) diff --git a/mteb/evaluation/evaluators/utils.py b/mteb/evaluation/evaluators/utils.py index e294f35c42..9823e9d603 100644 --- a/mteb/evaluation/evaluators/utils.py +++ b/mteb/evaluation/evaluators/utils.py @@ -182,7 +182,7 @@ def top_k_accuracy( results: dict[str, dict[str, float]], k_values: List[int], output_type: str = "mean", -) -> Tuple[Dict[str, float]]: +) -> Dict[str, float]: top_k_acc = {} for k in k_values: diff --git a/mteb/load_results/mteb_results.py b/mteb/load_results/mteb_results.py index 8d38b1be36..f6c2d254e6 100644 --- a/mteb/load_results/mteb_results.py +++ b/mteb/load_results/mteb_results.py @@ -38,7 +38,7 @@ class CQADupstackRetrievalDummy: }, dataset={ "revision": "revision not applicable", - "path": "CQADupstackRetrieval is a combined dataset", + "path": "CQADupstackRetrieval_is_a_combined_dataset", }, ) @@ -53,7 +53,7 @@ class ScalaNbClassificationDummy: hf_subsets_to_langscripts={ "default": ["nob-Latn"], }, - dataset={"revision": "revision not applicable"}, + dataset={"revision": "revision_not_applicable"}, ) @@ -67,7 +67,7 @@ class ScalaNnClassificationDummy: hf_subsets_to_langscripts={ "default": ["nno-Latn"], }, - dataset={"revision": "revision not applicable"}, + dataset={"revision": "revision_not_applicable"}, ) @@ -81,7 +81,7 @@ class ScalaDaClassificationDummy: hf_subsets_to_langscripts={ "default": ["dan-Latn"], }, - dataset={"revision": "revision not applicable"}, + dataset={"revision": "revision_not_applicable"}, ) @@ -95,7 +95,7 @@ class ScalaSvClassificationDummy: hf_subsets_to_langscripts={ "default": ["swe-Latn"], }, - dataset={"revision": "revision not applicable"}, + dataset={"revision": "revision_not_applicable"}, ) diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index ea3c26e236..50ae049323 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -131,9 +131,10 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe e5_instruct, e5_models, e5_v, - jina_clip, + google_models, gritlm_models, gte_models, + jina_clip, llm2vec_models, mxbai_models, nomic_models, diff --git a/mteb/models/cohere_models.py b/mteb/models/cohere_models.py index 0451532426..5f8c802278 100644 --- a/mteb/models/cohere_models.py +++ b/mteb/models/cohere_models.py @@ -10,23 +10,32 @@ from mteb.encoder_interface import Encoder from mteb.model_meta import ModelMeta -# Implementation follows that of https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/blob/main/src/seb/registered_models/cohere_models.py - +# Implementation follows https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/blob/main/src/seb/registered_models/cohere_models.py class CohereTextEmbeddingModel(Encoder): def __init__(self, model_name: str, sep: str = " ", **kwargs) -> None: self.model_name = model_name self.sep = sep - def _embed(self, sentences: list[str], input_type: str) -> torch.Tensor: + def _embed( + self, sentences: list[str], input_type: str, retries: int = 5 + ) -> torch.Tensor: import cohere # type: ignore client = cohere.Client() - response = client.embed( - texts=list(sentences), - model=self.model_name, - input_type=input_type, - ) + while retries > 0: # Cohere's API is not always reliable + try: + response = client.embed( + texts=list(sentences), + model=self.model_name, + input_type=input_type, + ) + break + except Exception as e: + print(f"Retrying... {retries} retries left.") + retries -= 1 + if retries == 0: + raise e return torch.tensor(response.embeddings) def encode( @@ -68,10 +77,10 @@ def encode_corpus(self, corpus: list[dict[str, str]], **kwargs: Any) -> np.ndarr cohere_mult_3 = ModelMeta( loader=partial(CohereTextEmbeddingModel, model_name="embed-multilingual-v3.0"), - name="cohere/embed-multilingual-v3.0", + name="embed-multilingual-v3.0", languages=[], # Unknown, but support >100 languages open_source=False, - revision=None, + revision="1", release_date="2023-11-02", n_parameters=None, memory_usage=None, @@ -82,6 +91,21 @@ def encode_corpus(self, corpus: list[dict[str, str]], **kwargs: Any) -> np.ndarr framework=[], ) +cohere_eng_3 = ModelMeta( + loader=partial(CohereTextEmbeddingModel, model_name="embed-english-v3.0"), + name="embed-english-v3.0", + languages=["eng-Latn"], + open_source=False, + revision="1", + release_date="2023-11-02", + n_parameters=None, + memory_usage=None, + max_tokens=None, + embed_dim=1024, + license=None, + similarity_fn_name="cosine", + framework=[], +) if __name__ == "__main__": import mteb diff --git a/mteb/models/e5_instruct.py b/mteb/models/e5_instruct.py index f88d3c864e..49650641fa 100644 --- a/mteb/models/e5_instruct.py +++ b/mteb/models/e5_instruct.py @@ -1,250 +1,59 @@ -from __future__ import annotations +from functools import partial -import logging -from itertools import islice -from typing import Any, Callable, Iterable, Literal, Optional, Sequence, Type, TypeVar - -import numpy as np import torch -from torch import Tensor -from tqdm import tqdm -from transformers import AutoModel, AutoTokenizer, BatchEncoding -from transformers.modeling_outputs import ModelOutput -from mteb.encoder_interface import Encoder from mteb.model_meta import ModelMeta from .e5_models import E5_PAPER_RELEASE_DATE, XLMR_LANGUAGES from .instructions import task_to_instruction -logger = logging.getLogger(__name__) - -T = TypeVar("T") -EncodeTypes = Literal["query", "passage"] - MISTRAL_LANGUAGES = ["eng_Latn", "fra_Latn", "deu_Latn", "ita_Latn", "spa_Latn"] -def batched(iterable: Iterable[T], n: int) -> Iterable[tuple[T, ...]]: - """batched('ABCDEFG', 3) --> ABC DEF G""" # noqa - if n < 1: - raise ValueError("n must be at least one") - it = iter(iterable) - while batch := tuple(islice(it, n)): - yield batch - - -class E5InstructWrapper(Encoder): - def __init__( - self, - model_name: str, - revision: str, - max_length: int, - max_batch_size: Optional[int] = None, - device: str = torch.device("cuda" if torch.cuda.is_available() else "cpu"), - **kwargs: Any, - ): - logger.info("Started loading e5 instruct model") - self.tokenizer = AutoTokenizer.from_pretrained( - model_name, revision=revision, **kwargs - ) - self.model = AutoModel.from_pretrained(model_name, **kwargs).to(device) - self.model.eval() - self.device = device - self.max_length = max_length - self.max_batch_size = max_batch_size - self.gpu_count = torch.cuda.device_count() - - if self.gpu_count > 1: - logger.info( - f"----------Using {self.gpu_count} data-parallel GPUs----------" - ) - self.model = torch.nn.DataParallel(self.model) - - def preprocess( - self, sentences: Sequence[str], instruction: str, encode_type: EncodeTypes - ) -> BatchEncoding: - if encode_type == "query": - sentences = [ - f"Instruction: {instruction}\nQuery: {sentence}" - for sentence in sentences - ] +def e5_instruction(instruction: str) -> str: + return f"Instruct: {instruction}\nQuery: " - batch_dict = self.tokenizer( - sentences, # type: ignore - max_length=512, - padding=True, - truncation=True, - return_tensors="pt", - ) - - return batch_dict.to(self.device) - def get_embedding_from_output( - self, output: ModelOutput, batch_dict: BatchEncoding - ) -> torch.Tensor: - return self.average_pool(output.last_hidden_state, batch_dict["attention_mask"]) # type: ignore - - @staticmethod - def average_pool( - last_hidden_states: torch.Tensor, attention_mask: torch.Tensor - ) -> Tensor: - last_hidden = last_hidden_states.masked_fill( - ~attention_mask[..., None].bool(), 0.0 +def e5_loader(**kwargs): + try: + from gritlm import GritLM + except ImportError: + raise ImportError( + "Please install `pip install gritlm` to use E5 Instruct models." ) - return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] - def encode( - self, - sentences: list[str], - *, - prompt_name: str | None = None, - batch_size: int = 32, - encode_type: EncodeTypes = "query", - **kwargs: Any, # noqa - ) -> np.ndarray: - if self.max_batch_size and batch_size > self.max_batch_size: - batch_size = self.max_batch_size - batch_size = batch_size * self.gpu_count - batched_embeddings = [] - if prompt_name is not None: - instruction = task_to_instruction( - prompt_name, is_query=encode_type == "query" - ) - else: - instruction = "" - for batch in tqdm(batched(sentences, batch_size)): - with torch.inference_mode(): - batch_dict = self.preprocess( - batch, instruction=instruction, encode_type=encode_type + class E5InstructWrapper(GritLM): + def encode(self, *args, **kwargs): + if "prompt_name" in kwargs: + if "instruction" in kwargs: + raise ValueError( + "Cannot specify both `prompt_name` and `instruction`." + ) + instruction = task_to_instruction( + kwargs.pop("prompt_name"), kwargs.pop("is_query", True) ) - outputs = self.model(**batch_dict) - embeddings = self.get_embedding_from_output(outputs, batch_dict) - batched_embeddings.append(embeddings.detach().cpu()) - - return torch.cat(batched_embeddings).to("cpu").detach().numpy() - - def encode_corpus( - self, - corpus: list[dict[str, str]] | dict[str, list[str]] | list[str], - prompt_name: str | None = None, - **kwargs: Any, - ) -> np.ndarray: - sep = " " - if isinstance(corpus, dict): - sentences = [ - (corpus["title"][i] + sep + corpus["text"][i]).strip() - if "title" in corpus - else corpus["text"][i].strip() # type: ignore - for i in range(len(corpus["text"])) # type: ignore - ] - else: - if isinstance(corpus[0], str): - sentences = corpus else: - sentences = [ - (doc["title"] + sep + doc["text"]).strip() - if "title" in doc - else doc["text"].strip() - for doc in corpus - ] - return self.encode( - sentences, encode_type="passage", prompt_name=prompt_name, **kwargs - ) + instruction = kwargs.pop("instruction", "") + if instruction: + kwargs["instruction"] = e5_instruction(instruction) + return super().encode(*args, **kwargs) - def encode_queries( - self, queries: list[str], prompt_name: str | None = None, **kwargs: Any - ) -> np.ndarray: - return self.encode( - queries, encode_type="query", prompt_name=prompt_name, **kwargs - ) + def encode_corpus(self, *args, **kwargs): + kwargs["is_query"] = False + return super().encode_corpus(*args, **kwargs) - -class E5MistralWrapper(E5InstructWrapper): - def __init__( - self, - name: str, - revision: str, - max_batch_size: int = 4, - torch_dtype=torch.float16, - **kwargs, - ): - assert ( - name == "intfloat/e5-mistral-7b-instruct" - ), f"Unexpected model name: {name}" - super().__init__( - model_name=name, - revision=revision, - max_length=4096, - max_batch_size=max_batch_size, - torch_dtype=torch_dtype, - **kwargs, - ) - - @staticmethod - def last_token_pool(last_hidden_states: Tensor, attention_mask: Tensor) -> Tensor: - left_padding = attention_mask[:, -1].sum() == attention_mask.shape[0] - if left_padding: - return last_hidden_states[:, -1] - sequence_lengths = attention_mask.sum(dim=1) - 1 - batch_size = last_hidden_states.shape[0] - return last_hidden_states[ - torch.arange(batch_size, device=last_hidden_states.device), - sequence_lengths, - ] - - def get_embbeding_from_output( - self, output: ModelOutput, batch_dict: BatchEncoding - ) -> torch.Tensor: - return self.last_token_pool( - output.last_hidden_state, # type: ignore - batch_dict["attention_mask"], # type: ignore - ) - - def preprocess( - self, sentences: Sequence[str], instruction: str, encode_type: EncodeTypes - ) -> BatchEncoding: - if encode_type == "query": - sentences = [ - f"Instruction: {instruction}\nQuery: {sentence}" - for sentence in sentences - ] - batch_dict: BatchEncoding = self.tokenizer( - sentences, # type: ignore - max_length=self.max_length - 1, - return_attention_mask=False, - padding=False, - truncation=True, - ) - # append eos_token_id to every input_ids - batch_dict["input_ids"] = [ - [*input_ids, self.tokenizer.eos_token_id] - for input_ids in batch_dict["input_ids"] # type: ignore - ] - batch_dict = self.tokenizer.pad( - batch_dict, padding=True, return_attention_mask=True, return_tensors="pt" - ) - - return batch_dict.to(self.device) - - -def _loader( - wrapper: Type[E5InstructWrapper], name: str, revision: str, **kwargs -) -> Callable[..., Encoder]: - _kwargs = kwargs - - def loader_inner(**kwargs: Any) -> Encoder: - return wrapper(name, revision=revision, **_kwargs, **kwargs) - - return loader_inner + return E5InstructWrapper(**kwargs) e5_instruct = ModelMeta( - loader=_loader( - E5InstructWrapper, - "intfloat/multilingual-e5-large-instruct", - "baa7be480a7de1539afce709c8f13f833a510e0a", - max_length=512, + loader=partial( + e5_loader, + model_name_or_path="intfloat/multilingual-e5-large-instruct", + attn="cccc", + pooling_method="mean", + mode="embedding", + torch_dtype=torch.float16, + normalized=True, ), name="intfloat/multilingual-e5-large-instruct", languages=XLMR_LANGUAGES, @@ -254,15 +63,19 @@ def loader_inner(**kwargs: Any) -> Encoder: ) e5_mistral = ModelMeta( - loader=_loader( - E5MistralWrapper, - "intfloat/e5-mistral-7b-instruct", - "07163b72af1488142a360786df853f237b1a3ca1", - max_batch_size=512, + loader=partial( + e5_loader, + model_name_or_path="intfloat/e5-mistral-7b-instruct", + attn="cccc", + pooling_method="lasttoken", + mode="embedding", torch_dtype=torch.float16, + # The ST script does not normalize while the HF one does so unclear what to do + # https://huggingface.co/intfloat/e5-mistral-7b-instruct#transformers + normalized=True, ), name="intfloat/e5-mistral-7b-instruct", - languages=XLMR_LANGUAGES, + languages=MISTRAL_LANGUAGES, open_source=True, revision="07163b72af1488142a360786df853f237b1a3ca1", release_date=E5_PAPER_RELEASE_DATE, diff --git a/mteb/models/e5_models.py b/mteb/models/e5_models.py index 3818e8a80a..b3a8fc74bd 100644 --- a/mteb/models/e5_models.py +++ b/mteb/models/e5_models.py @@ -157,6 +157,8 @@ def encode_corpus( batch_size: int = 32, **kwargs: Any, ): + if "request_qid" in kwargs: + kwargs.pop("request_qid") sentences = corpus_to_texts(corpus) sentences = ["passage: " + sentence for sentence in sentences] emb = self.mdl.encode(sentences, batch_size=batch_size, **kwargs) @@ -181,7 +183,6 @@ def encode_corpus( release_date=E5_PAPER_RELEASE_DATE, ) - e5_mult_large = ModelMeta( loader=partial(E5Wrapper, model_name="intfloat/multilingual-e5-large"), # type: ignore name="intfloat/multilingual-e5-large", diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index 5cdbb4200d..b3287f274c 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from functools import partial from typing import Any diff --git a/mteb/models/google_models.py b/mteb/models/google_models.py index 78cd6bd38f..fdb9a2a36c 100644 --- a/mteb/models/google_models.py +++ b/mteb/models/google_models.py @@ -3,7 +3,7 @@ from __future__ import annotations from functools import partial -from typing import Any +from typing import Any, List, Optional import numpy as np @@ -16,30 +16,41 @@ def __init__(self, model_name: str, sep: str = " ", **kwargs) -> None: self.model_name = model_name def _embed( - self, sentences: list[str], *, task_type: str, titles: list[str] | None = None - ) -> np.ndarray: + self, + texts: List[str], + task_type: str = "RETRIEVAL_DOCUMENT", + titles: List[str] | None = None, + dimensionality: Optional[int] = 768, + ) -> List[List[float]]: + """Embeds texts with a pre-trained, foundational model. + From https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings#generative-ai-get-text-embedding-python_vertex_ai_sdk + """ try: - import google.generativeai as genai + from vertexai.language_models import TextEmbeddingInput, TextEmbeddingModel except ImportError: raise ImportError( - "`google-generativeai` is required to run the google API, please install it using `pip install google-generativeai`" + "The `vertexai` package is required to run the google API, please install it using `pip install vertexai`" ) - + model = TextEmbeddingModel.from_pretrained(self.model_name) if titles: - result = genai.embed_content( # type: ignore - model=self.model_name, - content=sentences, - task_type=task_type, - title=titles, - ) + # Allow title-only embeddings by replacing text with a space + # Else Google throws google.api_core.exceptions.InvalidArgument: 400 The text content is empty. + inputs = [ + TextEmbeddingInput( + text if text else " ", task_type=task_type, title=title + ) + for text, title in zip(texts, titles) + ] else: - result = genai.embed_content( # type: ignore - model=self.model_name, - content=sentences, - task_type=task_type, - ) - - return np.asarray(result["embedding"]) + inputs = [TextEmbeddingInput(text, task_type=task_type) for text in texts] + kwargs = dict(output_dimensionality=dimensionality) if dimensionality else {} + try: + embeddings = model.get_embeddings(inputs, **kwargs) + # Except the very rare google.api_core.exceptions.InternalServerError + except Exception as e: + print("Retrying once after error:", e) + embeddings = model.get_embeddings(inputs, **kwargs) + return np.asarray([embedding.values for embedding in embeddings]) def encode( self, @@ -74,10 +85,10 @@ def encode_corpus(self, corpus: list[dict[str, str]], **kwargs: Any) -> np.ndarr for doc in corpus: titles.append(doc["title"]) sentences.append(doc["text"]) - return self._embed(sentences, task_type="RETRIEVAL_DOCUMENT") + return self._embed(sentences, task_type="RETRIEVAL_DOCUMENT", titles=titles) -name = "models/text-embedding-004" +name = "text-embedding-004" google_emb_004 = ModelMeta( loader=partial(GoogleTextEmbeddingModel, model_name=name), name=name, diff --git a/mteb/models/gritlm_models.py b/mteb/models/gritlm_models.py index 2b2fd83335..7c529f35f9 100644 --- a/mteb/models/gritlm_models.py +++ b/mteb/models/gritlm_models.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) -def gritlm_instruction(instruction): +def gritlm_instruction(instruction: str = "") -> str: return ( "<|user|>\n" + instruction + "\n<|embed|>\n" if instruction else "<|embed|>\n" ) @@ -19,28 +19,27 @@ def gritlm_loader(**kwargs): try: from gritlm import GritLM except ImportError: - raise ImportError( - "GritLM is not installed. Please install it with `pip install gritlm`." - ) + raise ImportError("Please install `pip install gritlm` to use GritLM models.") class GritLMWrapper(GritLM): def encode(self, *args, **kwargs): if "prompt_name" in kwargs: - instruction = gritlm_instruction( - task_to_instruction( - kwargs.pop("prompt_name"), kwargs.get("is_query", True) + if "instruction" in kwargs: + raise ValueError( + "Cannot specify both `prompt_name` and `instruction`." ) + instruction = task_to_instruction( + kwargs.pop("prompt_name"), kwargs.pop("is_query", True) ) else: - instruction = gritlm_instruction("") - kwargs["instruction"] = instruction + instruction = kwargs.pop("instruction", "") + kwargs["instruction"] = gritlm_instruction(instruction) return super().encode(*args, **kwargs) def encode_corpus(self, *args, **kwargs): kwargs["is_query"] = False return super().encode_corpus(*args, **kwargs) - kwargs.pop("device", None) # GritLM does automatic device placement return GritLMWrapper(**kwargs) diff --git a/mteb/models/gte_models.py b/mteb/models/gte_models.py index baa51e53a0..44ac2e65f5 100644 --- a/mteb/models/gte_models.py +++ b/mteb/models/gte_models.py @@ -7,6 +7,10 @@ from .instructions import task_to_instruction +def gte_instruction(instruction: str) -> str: + return f"Instruct: {instruction}\nQuery: " + + def gte_loader(**kwargs): try: from gritlm import GritLM @@ -16,23 +20,25 @@ def gte_loader(**kwargs): ) class GTEWrapper(GritLM): - def get_detailed_instruct(self, instruction: str, query: str) -> str: - return f"Instruct: {instruction}\nQuery: " - def encode(self, *args, **kwargs): - instruction = "" - if ("prompt_name" in kwargs) and (kwargs.get("is_query", True)): - instruction = self.get_detailed_instruct( - task_to_instruction(kwargs.pop("prompt_name")) + if "prompt_name" in kwargs: + if "instruction" in kwargs: + raise ValueError( + "Cannot specify both `prompt_name` and `instruction`." + ) + instruction = task_to_instruction( + kwargs.pop("prompt_name"), kwargs.pop("is_query", True) ) - kwargs["instruction"] = instruction + else: + instruction = kwargs.pop("instruction", "") + if instruction: + kwargs["instruction"] = gte_instruction(instruction) return super().encode(*args, **kwargs) def encode_corpus(self, *args, **kwargs): kwargs["is_query"] = False return super().encode_corpus(*args, **kwargs) - kwargs.pop("device", None) # GritLM does automatic device placement return GTEWrapper(**kwargs) diff --git a/mteb/models/nomic_models.py b/mteb/models/nomic_models.py index 84f2f5fd78..a013d1e621 100644 --- a/mteb/models/nomic_models.py +++ b/mteb/models/nomic_models.py @@ -1,31 +1,92 @@ from __future__ import annotations from functools import partial +from typing import Any, Optional +import torch +import torch.nn.functional as F from sentence_transformers import SentenceTransformer +import mteb from mteb.model_meta import ModelMeta +from mteb.models.text_formatting_utils import corpus_to_texts -class SentenceTransformerWithNormalization(SentenceTransformer): - def encode(self, sentences, *args, **kwargs): - if "normalize_embeddings" not in kwargs: - kwargs["normalize_embeddings"] = True +class NomicWrapper: + """following the hf model card documentation.""" - return super().encode(sentences, *args, **kwargs) + def __init__(self, model_name: str, revision: str, **kwargs: Any): + self.model_name = model_name + self.mdl = SentenceTransformer(model_name, revision=revision, **kwargs) + def to(self, device: torch.device) -> None: + self.mdl.to(device) -def sentence_transformers_loader( - model_name: str, revision: str | None, **kwargs -) -> SentenceTransformer: - return SentenceTransformerWithNormalization( - model_name_or_path=model_name, revision=revision, **kwargs - ) + def encode( # type: ignore + self, + sentences: list[str], + *, + prompt_name: str | None = None, + batch_size: int = 32, + input_type: Optional[str] = None, + **kwargs: Any, + ): + if prompt_name: + task = mteb.get_task(prompt_name) + task_type = task.metadata.type + if task_type in ["Classification", "MultilabelClassification"]: + input_type = "classification" + elif task_type == "Clustering": + input_type = "clustering" + + # default to search_document if input_type and prompt_name are not provided + if input_type is None: + input_type = "search_document" + + sentences = [f"{input_type}: {sentence}" for sentence in sentences] + + emb = self.mdl.encode(sentences, batch_size=batch_size, **kwargs) + # v1.5 has a non-trainable layer norm to unit normalize the embeddings for binary quantization + # the outputs are similar to if we just normalized but keeping the same for consistency + if self.model_name == "nomic-ai/nomic-embed-text-v1.5": + if not isinstance(emb, torch.Tensor): + emb = torch.tensor(emb) + emb = F.layer_norm(emb, normalized_shape=(emb.shape[1],)) + emb = F.normalize(emb, p=2, dim=1) + if kwargs.get("convert_to_tensor", False): + emb = emb.cpu().detach().numpy() + + return emb + + def encode_queries(self, queries: list[str], batch_size: int = 32, **kwargs: Any): + if "prompt_name" in kwargs: + kwargs.pop("prompt_name") + + emb = self.encode( + queries, batch_size=batch_size, input_type="search_query", **kwargs + ) + + return emb + + def encode_corpus( + self, + corpus: list[dict[str, str]] | dict[str, list[str]], + batch_size: int = 32, + **kwargs: Any, + ): + if "prompt_name" in kwargs: + kwargs.pop("prompt_name") + + sentences = corpus_to_texts(corpus) + emb = self.encode( + sentences, batch_size=batch_size, input_type="search_document", **kwargs + ) + return emb -nomic_embed = ModelMeta( +nomic_embed_v1_5 = ModelMeta( loader=partial( # type: ignore - sentence_transformers_loader, + NomicWrapper, trust_remote_code=True, model_name="nomic-ai/nomic-embed-text-v1.5", revision="b0753ae76394dd36bcfb912a46018088bca48be0", @@ -37,8 +98,33 @@ def sentence_transformers_loader( release_date="2024-02-10", # first commit ) +nomic_embed_v1 = ModelMeta( + loader=partial( # type: ignore + NomicWrapper, + trust_remote_code=True, + model_name="nomic-ai/nomic-embed-text-v1", + revision="0759316f275aa0cb93a5b830973843ca66babcf5", + ), + name="nomic-ai/nomic-embed-text-v1", + languages=["eng-Latn"], + open_source=True, + revision="0759316f275aa0cb93a5b830973843ca66babcf5", + release_date="2024-01-31", # first commit +) + if __name__ == "__main__": - import mteb + mdl = mteb.get_model(nomic_embed_v1_5.name, nomic_embed_v1_5.revision) + emb = mdl.encode(["test"], convert_to_tensor=True) + print(emb.shape) + emb = mdl.encode_queries(["test"], convert_to_tensor=True) + print(emb.shape) + emb = mdl.encode( + ["test"], + convert_to_tensor=True, + prompt_name="AmazonCounterfactualClassification", + ) + print(emb.shape) - mdl = mteb.get_model(nomic_embed.name, nomic_embed.revision) + mdl = mteb.get_model(nomic_embed_v1.name, nomic_embed_v1.revision) emb = mdl.encode(["test"], convert_to_tensor=True) + print(emb.shape) diff --git a/mteb/models/salesforce_models.py b/mteb/models/salesforce_models.py index 7e74ab833a..9d0e9354b0 100644 --- a/mteb/models/salesforce_models.py +++ b/mteb/models/salesforce_models.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from functools import partial import torch @@ -9,6 +7,10 @@ from .instructions import task_to_instruction +def sfr_instruction(instruction: str) -> str: + return f"Instruct: {instruction}\nQuery: " + + def sfr_loader(**kwargs): try: from gritlm import GritLM @@ -18,23 +20,25 @@ def sfr_loader(**kwargs): ) class SFRWrapper(GritLM): - def get_detailed_instruct(self, instruction: str, query: str) -> str: - return f"Instruct: {instruction}\nQuery: " - def encode(self, *args, **kwargs): - instruction = "" - if ("prompt_name" in kwargs) and (kwargs.get("is_query", True)): - instruction = self.get_detailed_instruct( - task_to_instruction(kwargs.pop("prompt_name")) + if "prompt_name" in kwargs: + if "instruction" in kwargs: + raise ValueError( + "Cannot specify both `prompt_name` and `instruction`." + ) + instruction = task_to_instruction( + kwargs.pop("prompt_name"), kwargs.pop("is_query", True) ) - kwargs["instruction"] = instruction + else: + instruction = kwargs.pop("instruction", "") + if instruction: + kwargs["instruction"] = sfr_instruction(instruction) return super().encode(*args, **kwargs) def encode_corpus(self, *args, **kwargs): kwargs["is_query"] = False return super().encode_corpus(*args, **kwargs) - kwargs.pop("device", None) # GritLM does automatic device placement return SFRWrapper(**kwargs) diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py index 1f2e894f4c..007bbfe37f 100644 --- a/mteb/models/vista_models.py +++ b/mteb/models/vista_models.py @@ -3,17 +3,19 @@ from functools import partial import torch - -try: # a temporal fix for the dependency issues of vista models. - from FlagEmbedding.visual.modeling import Visualized_BGE -except ImportError: - Visualized_BGE = None from PIL import Image from tqdm import tqdm from mteb.model_meta import ModelMeta -if Visualized_BGE is not None: + +def vista_loader(**kwargs): + try: # a temporal fix for the dependency issues of vista models. + from FlagEmbedding.visual.modeling import Visualized_BGE + except ImportError: + raise ImportError( + "Please install `pip install FlagEmbedding` to use VisualizedBGE models." + ) class VisualizedBGEWrapper(Visualized_BGE): def __init__( @@ -155,41 +157,37 @@ def calculate_probs(self, text_embeddings, image_embeddings): probs = (logits * 100).softmax(dim=-1) return probs - Visualized_BGE_base = ModelMeta( - loader=partial( - VisualizedBGEWrapper, - model_name_bge="BAAI/bge-base-en-v1.5", - model_weight="visualized_base_en_V1.5.pth", - ), - name="BAAI/bge-visualized-base", - languages=["eng_Latn"], - open_source=True, - revision="98db10b10d22620010d06f11733346e1c98c34aa", - release_date="2024-06-06", - ) - - Visualized_BGE_base = ModelMeta( - loader=partial( - VisualizedBGEWrapper, - model_name_bge="BAAI/bge-m3", - model_weight="visualized_m3.pth", - ), - name="BAAI/bge-visualized-m3", - languages=["eng_Latn"], - open_source=True, - revision="98db10b10d22620010d06f11733346e1c98c34aa", - release_date="2024-06-06", - ) + return VisualizedBGEWrapper(**kwargs) + + +visualized_bge_base = ModelMeta( + loader=partial( + vista_loader, + model_name_bge="BAAI/bge-base-en-v1.5", + model_weight="visualized_base_en_V1.5.pth", + ), + name="BAAI/bge-visualized-base", + languages=["eng_Latn"], + open_source=True, + revision="98db10b10d22620010d06f11733346e1c98c34aa", + release_date="2024-06-06", +) + +visualized_bge_m3 = ModelMeta( + loader=partial( + vista_loader, + model_name_bge="BAAI/bge-m3", + model_weight="visualized_m3.pth", + ), + name="BAAI/bge-visualized-m3", + languages=["eng_Latn"], + open_source=True, + revision="98db10b10d22620010d06f11733346e1c98c34aa", + release_date="2024-06-06", +) if __name__ == "__main__": - if ( - Visualized_BGE is None - ): # a temporal fix to the dependency issues of Vista models. - print("Visualized_BGE module is not available.") - else: - import mteb - - mdl = mteb.get_model( - Visualized_BGE_base.name, Visualized_BGE_base.name.revision - ) - emb = mdl.get_text_embeddings(["Hello, world!"]) + import mteb + + mdl = mteb.get_model(visualized_bge_base.name, visualized_bge_base.name.revision) + emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/models/voyage_models.py b/mteb/models/voyage_models.py index 11d06b5e1d..4720b1c4d0 100644 --- a/mteb/models/voyage_models.py +++ b/mteb/models/voyage_models.py @@ -107,7 +107,7 @@ def _batched_encode( ) -> np.ndarray: embeddings, index = [], 0 - while index < len(sentences) - 1: + while index < len(sentences): batch, batch_tokens = [], 0 while ( index < len(sentences) @@ -179,7 +179,7 @@ def _batched_encode( ) voyage_large_2 = ModelMeta( - name="voyage-large-2", # The release date is considered to be the date of publication of this post https://blog.voyageai.com/2023/10/29/voyage-embeddings/ + name="voyage-large-2", # Date of publication of this post https://blog.voyageai.com/2023/10/29/voyage-embeddings/ revision="1", release_date="2023-10-29", languages=None, # supported languages not specified @@ -200,13 +200,13 @@ def _batched_encode( open_source=False, ) -voyage_2_mult = ModelMeta( # reference: https://blog.voyageai.com/2024/06/10/voyage-multilingual-2-multilingual-embedding-model/" +voyage_multilingual_2 = ModelMeta( # reference: https://blog.voyageai.com/2024/06/10/voyage-multilingual-2-multilingual-embedding-model/" name="voyage-multilingual-2", revision="1", release_date="2024-06-10", languages=None, # supported languages not specified loader=partial(VoyageWrapper, model_name="voyage-multilingual-2"), - max_tokens=4000, + max_tokens=32000, embed_dim=1024, open_source=False, ) diff --git a/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py b/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py index 5f03798d51..68954c9dd3 100644 --- a/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py +++ b/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py @@ -96,8 +96,8 @@ class MultilingualSentimentClassification(AbsTaskClassification, MultilingualTas def dataset_transform(self): # create a train set from the test set for Welsh language (cym) lang = "cym" - _dataset = self.dataset[lang] if lang in self.dataset.keys(): + _dataset = self.dataset[lang] _dataset = _dataset.class_encode_column("label") _dataset = _dataset["test"].train_test_split( test_size=0.3, seed=self.seed, stratify_by_column="label" @@ -105,4 +105,4 @@ def dataset_transform(self): _dataset = self.stratified_subsampling( dataset_dict=_dataset, seed=self.seed, splits=["test"] ) - self.dataset[lang] = _dataset + self.dataset[lang] = _dataset diff --git a/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py b/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py index a90f2ef621..aebcbe8407 100644 --- a/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py @@ -5,6 +5,7 @@ from datasets import Dataset, DatasetDict from mteb.abstasks import AbsTaskClustering, MultilingualTask, TaskMetadata +from mteb.abstasks.AbsTaskClusteringFast import AbsTaskClusteringFast _LANGUAGES = { "de": ["deu-Latn"], @@ -91,7 +92,7 @@ def dataset_transform(self, lang): self.dataset[lang] = _dataset -class MLSUMClusteringP2PFast(AbsTaskClustering, MultilingualTask): +class MLSUMClusteringP2PFast(AbsTaskClusteringFast, MultilingualTask): max_document_to_embed = N_SAMPLES max_fraction_of_documents_to_embed = None diff --git a/pyproject.toml b/pyproject.toml index cfc6f19ba7..19d804e726 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mteb" -version = "1.12.84" +version = "1.12.90" description = "Massive Text Embedding Benchmark" readme = "README.md" authors = [ diff --git a/scripts/mmteb/running_model/create_slurm_jobs.py b/scripts/mmteb/running_model/create_slurm_jobs.py index bfb860bb79..177775144e 100644 --- a/scripts/mmteb/running_model/create_slurm_jobs.py +++ b/scripts/mmteb/running_model/create_slurm_jobs.py @@ -61,8 +61,8 @@ def run_slurm_jobs(files: list[Path]) -> None: slurm_prefix = """#!/bin/bash #SBATCH --job-name=mteb #SBATCH --nodes=1 -#SBATCH --partition=a3mixed -#SBATCH --gres=gpu:1 # number of gpus +#SBATCH --partition=a3 +#SBATCH --gres=gpu:8 # number of gpus #SBATCH --time 24:00:00 # maximum execution time (HH:MM:SS) #SBATCH --output=/data/niklas/jobs/%x-%j.out # output file name #SBATCH --exclusive @@ -104,10 +104,13 @@ def run_slurm_jobs(files: list[Path]) -> None: "Summarization", ], tasks=[ - "LivedoorNewsClustering", + # "LivedoorNewsClustering", # "FaithDial", - # "STS22", # "StatcanDialogueDatasetRetrieval", + # "STS22", + # "IN22GenBitextMining", + # "IN22ConvBitextMining", + # "FloresBitextMining", # "WikipediaRetrievalMultilingual" # "RARbMath" # "Touche2020", @@ -117,21 +120,6 @@ def run_slurm_jobs(files: list[Path]) -> None: ], ) - tasks = mteb.get_tasks( - task_types=[ - "BitextMining", - "Classification", - "MultilabelClassification", - "PairClassification", - "Reranking", - "STS", - "Summarization", - "Clustering", - "InstructionRetrieval", - "Retrieval", - ] - ) - # WE ALSO NEED TO RUN THESE retrieval_to_be_downsampled = [ "TopiOCQA", diff --git a/tests/test_benchmark/mock_tasks.py b/tests/test_benchmark/mock_tasks.py index ec67175a20..85a5b27196 100644 --- a/tests/test_benchmark/mock_tasks.py +++ b/tests/test_benchmark/mock_tasks.py @@ -8,6 +8,7 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.AbsTaskClustering import AbsTaskClustering from mteb.abstasks.AbsTaskClusteringFast import AbsTaskClusteringFast +from mteb.abstasks.AbsTaskInstructionRetrieval import AbsTaskInstructionRetrieval from mteb.abstasks.AbsTaskMultilabelClassification import ( AbsTaskMultilabelClassification, ) @@ -366,3 +367,73 @@ def load_data(self, **kwargs): } ) self.data_loaded = True + + +class MockInstructionRetrival(AbsTaskInstructionRetrieval): + do_length_ablation = True + metadata = TaskMetadata( + type="InstructionRetrieval", + name="MockInstructionRetrival", + main_score="p-MRR", + **general_args, # type: ignore + ) + + def load_data(self, **kwargs): + self.queries = { + "test": { + "q1": "This is a test sentence", + "q2": "This is another test sentence", + } + } + self.corpus = { + "test": { + "d1": {"text": "This is a positive sentence"}, + "d2": {"text": "This is another positive sentence"}, + } + } + + self.og_relevant_docs = { + "test": { + "q1": {"d1": 1, "d2": 0}, + "q2": {"d1": 0, "d2": 1}, + }, + } + self.og_instructions = { + "test": { + "This is a test sentence": "This is a test instruction", + "This is another test sentence": "This is another test instruction", + } + } + self.changed_instructions = { + "test": { + "This is a test sentence": "This is a changed test instruction", + "This is another test sentence": "This is changed another test instruction", + } + } + self.changed_relevant_docs = { + "test": { + "q1": {"d1": 0, "d2": 1}, + "q2": {"d1": 1, "d2": 0}, + } + } + + self.top_ranked = { + "test": { + "q1": ["d1", "d2"], + "q2": ["d2", "d1"], + } + } + + self.keywords = { + "test": { + "This is a test sentence": "test1", + "This is another test sentence": "test2", + } + } + self.short_instructions = { + "test": { + "This is a test sentence": "short1", + "This is another test sentence": "short2", + } + } + self.data_loaded = True diff --git a/tests/test_benchmark/task_grid.py b/tests/test_benchmark/task_grid.py index 45e8a6025f..081fb89afe 100644 --- a/tests/test_benchmark/task_grid.py +++ b/tests/test_benchmark/task_grid.py @@ -19,6 +19,7 @@ MockClassificationTask, MockClusteringFastTask, MockClusteringTask, + MockInstructionRetrival, MockMultilabelClassification, MockPairClassificationTask, MockRerankingTask, @@ -100,6 +101,7 @@ def dataset_transform(self): MockSTSTask(), MockMultilabelClassification(), MockSummarizationTask(), + MockInstructionRetrival(), ] MOCK_TASK_TEST_GRID_AS_STRING = [ From 4d50084b900e6f41ae07442ce4ca9dbc888c1d04 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Wed, 31 Jul 2024 17:59:26 +0100 Subject: [PATCH 035/154] image memoery issues for all retrieval Abstasks --- .../abstasks/Image/AbsTaskAny2AnyRetrieval.py | 6 +- mteb/abstasks/Image/AbsTaskI2TRetrieval.py | 6 +- mteb/abstasks/Image/AbsTaskT2IRetrieval.py | 6 +- .../Image/Any2AnyRetrievalEvaluator.py | 124 +++++++++----- .../evaluators/Image/I2TRetrievalEvaluator.py | 140 ++++++++-------- .../evaluators/Image/T2IRetrievalEvaluator.py | 131 ++++++++------- mteb/models/clip_models.py | 2 +- mteb/tasks/Image/T2IRetrieval/__init__.py | 1 + .../eng/Fashion200kT2IRetrieval.py | 51 ++++++ .../Fashion200kT2IRetrieval.json | 158 ++++++++++++++++++ 10 files changed, 444 insertions(+), 181 deletions(-) create mode 100644 mteb/tasks/Image/T2IRetrieval/eng/Fashion200kT2IRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kT2IRetrieval.json diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py index 25d08cbc94..64b10059df 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py @@ -221,9 +221,9 @@ def load_data(self, **kwargs): streaming=False, keep_in_memory=False, ).load(split=split) - # Conversion from DataSet - queries = {query["id"]: query for query in queries} - corpus = {doc["id"]: doc for doc in corpus} + # directly pass in corpus and queries datasets now to prevent loading into memory + # queries = {query["id"]: query for query in queries} + # corpus = {doc["id"]: doc for doc in corpus} self.corpus[split], self.queries[split], self.relevant_docs[split] = ( corpus, queries, diff --git a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py index 54e9606bb4..c20faff64c 100644 --- a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py @@ -219,9 +219,9 @@ def load_data(self, **kwargs): streaming=False, keep_in_memory=False, ).load(split=split) - # Conversion from DataSet - queries = {query["id"]: query["image"] for query in queries} - corpus = {doc["id"]: {"text": doc["text"]} for doc in corpus} + # Directly pass in queries and corpus datasets to prevent loading into memory + # queries = {query["id"]: query["image"] for query in queries} + # corpus = {doc["id"]: {"text": doc["text"]} for doc in corpus} self.corpus[split], self.queries[split], self.relevant_docs[split] = ( corpus, queries, diff --git a/mteb/abstasks/Image/AbsTaskT2IRetrieval.py b/mteb/abstasks/Image/AbsTaskT2IRetrieval.py index 1bdd550d6d..99369bd1b3 100644 --- a/mteb/abstasks/Image/AbsTaskT2IRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskT2IRetrieval.py @@ -219,9 +219,9 @@ def load_data(self, **kwargs): streaming=False, keep_in_memory=False, ).load(split=split) - # Conversion from DataSet - queries = {query["id"]: {"text": query["text"]} for query in queries} - corpus = {image["id"]: image["image"] for image in corpus} + # directly pass in queries and corpus dataset to prevent loading into memory + # queries = {query["id"]: {"text": query["text"]} for query in queries} + # corpus = {image["id"]: image["image"] for image in corpus} self.corpus[split], self.queries[split], self.relevant_docs[split] = ( corpus, queries, diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index 7a5ad8565d..7d3c26e107 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -10,7 +10,10 @@ import numpy as np import pytrec_eval import torch +from datasets import Dataset from PIL import Image +from torch.utils.data import DataLoader +from torchvision import transforms from mteb.encoder_interface import EncoderWithQueryCorpusEncode @@ -29,6 +32,29 @@ logger = logging.getLogger(__name__) +transform = transforms.Compose([transforms.PILToTensor()]) + + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, hf_dataset, image_column_name: str = "image", transform=None): + self.dataset = hf_dataset + self.transform = transform + self.image_column_name = image_column_name + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + image = self.dataset[idx][self.image_column_name] + if image.mode != "RGB": + image = image.convert("RGB") + image = self.transform(image) + return image + + +def custom_collate_fn(batch): + return batch + # Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/search/dense/exact_search.py#L12 class DenseRetrievalExactSearch: @@ -65,8 +91,8 @@ def __init__( def search( self, - corpus: dict[str, Dict[str, str | Image.Image]], - queries: dict[str, Dict[str, str | Image.Image]], + corpus: Dataset, # solve memoery issues + queries: Dataset, # solve memoery issues top_k: int, score_function: str, return_sorted: bool = False, @@ -78,64 +104,84 @@ def search( ) logger.info("Encoding Queries.") - query_ids = list(queries.keys()) + query_ids = list(queries["id"]) self.results = {qid: {} for qid in query_ids} - q_modality = queries[query_ids[0]]["modality"] + q_modality = queries[0]["modality"] if q_modality == "text": - query_texts = [queries[qid]["text"] for qid in query_ids] + query_texts = queries["text"] query_embeddings = self.model.get_text_embeddings( texts=query_texts, batch_size=self.encode_kwargs["batch_size"] ) - elif q_modality == "image": - query_images = [queries[qid]["image"] for qid in query_ids] - query_embeddings = self.model.get_image_embeddings( - images=query_images, batch_size=self.encode_kwargs["batch_size"] + else: + queries_dataset = ImageDataset( + queries, image_column_name="image", transform=transform ) - elif q_modality == "image,text": - query_texts = [queries[qid]["text"] for qid in query_ids] - query_images = [queries[qid]["image"] for qid in query_ids] - query_embeddings = self.model.get_fused_embeddings( - texts=query_texts, - images=query_images, + query_image_dataloader = DataLoader( + queries_dataset, batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=os.cpu_count(), ) - else: - raise ValueError(f"Unsupported modality: {q_modality}") + if q_modality == "image": + query_embeddings = self.model.get_image_embeddings( + images=query_image_dataloader, + batch_size=self.encode_kwargs["batch_size"], + ) + elif q_modality == "image,text": + query_texts = queries["text"] + query_embeddings = self.model.get_fused_embeddings( + texts=query_texts, + images=query_image_dataloader, + batch_size=self.encode_kwargs["batch_size"], + ) + else: + raise ValueError(f"Unsupported modality: {q_modality}") logger.info("Preparing Corpus...") - corpus_ids = list(corpus.keys()) + corpus_ids = list(corpus["id"]) - corpus_modality = corpus[corpus_ids[0]]["modality"] + corpus_modality = corpus[0]["modality"] + + logger.info("Encoding Corpus in batches... Warning: This might take a while!") + logger.info( + "Scoring Function: {} ({})".format( + self.score_function_desc[score_function], score_function + ) + ) if corpus_modality == "text": - corpus_texts = [corpus[cid]["text"] for cid in corpus_ids] + corpus_texts = corpus["text"] corpus_embeddings = self.model.get_text_embeddings( texts=corpus_texts, batch_size=self.encode_kwargs["batch_size"] ) - elif corpus_modality == "image": - corpus_images = [corpus[cid]["image"] for cid in corpus_ids] - corpus_embeddings = self.model.get_image_embeddings( - images=corpus_images, batch_size=self.encode_kwargs["batch_size"] + else: + corpus_dataset = ImageDataset( + corpus, image_column_name="image", transform=transform ) - elif corpus_modality == "image,text": - corpus_texts = [corpus[cid]["text"] for cid in corpus_ids] - corpus_images = [corpus[cid]["image"] for cid in corpus_ids] - corpus_embeddings = self.model.get_fused_embeddings( - texts=corpus_texts, - images=corpus_images, + corpus_image_dataloader = DataLoader( + corpus_dataset, batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=os.cpu_count(), ) - else: - raise ValueError(f"Unsupported modality: {corpus_modality}") - - logger.info("Encoding Corpus in batches... Warning: This might take a while!") - logger.info( - "Scoring Function: {} ({})".format( - self.score_function_desc[score_function], score_function - ) - ) + if corpus_modality == "image": + corpus_embeddings = self.model.get_image_embeddings( + images=corpus_image_dataloader, + batch_size=self.encode_kwargs["batch_size"], + ) + elif corpus_modality == "image,text": + corpus_texts = corpus["text"] + corpus_embeddings = self.model.get_fused_embeddings( + texts=corpus_texts, + images=corpus_image_dataloader, + batch_size=self.encode_kwargs["batch_size"], + ) + else: + raise ValueError(f"Unsupported modality: {corpus_modality}") result_heaps = { qid: [] for qid in query_ids diff --git a/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py index 5ceaa9d7d7..c0b1717315 100644 --- a/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py @@ -10,7 +10,10 @@ import numpy as np import pytrec_eval import torch +from datasets import Dataset from PIL import Image +from torch.utils.data import DataLoader +from torchvision import transforms from mteb.encoder_interface import EncoderWithQueryCorpusEncode @@ -29,6 +32,29 @@ logger = logging.getLogger(__name__) +transform = transforms.Compose([transforms.PILToTensor()]) + + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, hf_dataset, image_column_name: str = "image", transform=None): + self.dataset = hf_dataset + self.transform = transform + self.image_column_name = image_column_name + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + image = self.dataset[idx][self.image_column_name] + if image.mode != "RGB": + image = image.convert("RGB") + image = self.transform(image) + return image + + +def custom_collate_fn(batch): + return batch + # Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/search/dense/exact_search.py#L12 class DenseRetrievalExactSearch: @@ -65,8 +91,8 @@ def __init__( def search( self, - corpus: dict[str, dict[str, str]], - queries: dict[str, Image.Image], + corpus: Dataset, + queries: Dataset, top_k: int, score_function: str, return_sorted: bool = False, @@ -78,23 +104,24 @@ def search( ) logger.info("Encoding Queries.") - query_ids = list(queries.keys()) + query_ids = list(queries["id"]) self.results = {qid: {} for qid in query_ids} - queries = [queries[qid] for qid in queries] + queries_dataset = ImageDataset( + queries, image_column_name="image", transform=transform + ) + query_image_dataloader = DataLoader( + queries_dataset, + batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=os.cpu_count(), + ) query_embeddings = self.model.get_image_embeddings( - queries, batch_size=self.encode_kwargs["batch_size"] + images=query_image_dataloader, batch_size=self.encode_kwargs["batch_size"] ) logger.info("Sorting Corpus by document length (Longest first)...") - corpus_ids = sorted( - corpus, - key=lambda k: len( - corpus[k].get("text", "") - ), # no "title" as in text retrieval. - reverse=True, - ) - - corpus = [corpus[cid] for cid in corpus_ids] + corpus_ids = list(corpus["id"]) logger.info("Encoding Corpus in batches... Warning: This might take a while!") logger.info( @@ -103,67 +130,40 @@ def search( ) ) - itr = range(0, len(corpus), self.corpus_chunk_size) + corpus_texts = corpus["text"] + corpus_embeddings = self.model.get_text_embeddings( + texts=corpus_texts, batch_size=self.encode_kwargs["batch_size"] + ) result_heaps = { qid: [] for qid in query_ids } # Keep only the top-k docs for each query - for batch_num, corpus_start_idx in enumerate(itr): - logger.info("Encoding Batch {}/{}...".format(batch_num + 1, len(itr))) - corpus_end_idx = min(corpus_start_idx + self.corpus_chunk_size, len(corpus)) - - # Encode chunk of corpus - if ( - self.save_corpus_embeddings - and "qid" in kwargs - and len(self.corpus_embeddings[kwargs["qid"]]) - ): - sub_corpus_embeddings = torch.tensor( - self.corpus_embeddings[kwargs["qid"]][batch_num] - ) - else: - # Encode chunk of corpus - texts = [doc["text"] for doc in corpus[corpus_start_idx:corpus_end_idx]] - sub_corpus_embeddings = self.model.get_text_embeddings( - # corpus[corpus_start_idx:corpus_end_idx], - texts, - batch_size=self.encode_kwargs["batch_size"], - ) - if self.save_corpus_embeddings and "qid" in kwargs: - self.corpus_embeddings[kwargs["qid"]].append(sub_corpus_embeddings) - # Compute similarites using either cosine-similarity or dot product - cos_scores = self.score_functions[score_function]( - query_embeddings, sub_corpus_embeddings - ) - cos_scores[torch.isnan(cos_scores)] = -1 - - # Get top-k values - cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( - cos_scores, - min( - top_k + 1, - len(cos_scores[1]) if len(cos_scores) > 1 else len(cos_scores[-1]), - ), - dim=1, - largest=True, - sorted=return_sorted, - ) - cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() - cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() - - for query_itr in range(len(query_embeddings)): - query_id = query_ids[query_itr] - for sub_corpus_id, score in zip( - cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] - ): - corpus_id = corpus_ids[corpus_start_idx + sub_corpus_id] - if len(result_heaps[query_id]) < top_k: - # Push item on the heap - heapq.heappush(result_heaps[query_id], (score, corpus_id)) - else: - # If item is larger than the smallest in the heap, push it on the heap then pop the smallest element - heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) + cos_scores = self.score_functions[score_function]( + query_embeddings, corpus_embeddings + ) + cos_scores[torch.isnan(cos_scores)] = -1 + + cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( + cos_scores, + top_k, + dim=1, + largest=True, + sorted=return_sorted, + ) + cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() + cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() + + for query_itr in range(len(query_embeddings)): + query_id = query_ids[query_itr] + for sub_corpus_id, score in zip( + cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] + ): + corpus_id = corpus_ids[sub_corpus_id] + if len(result_heaps[query_id]) < top_k: + heapq.heappush(result_heaps[query_id], (score, corpus_id)) + else: + heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) for qid in result_heaps: for score, corpus_id in result_heaps[qid]: diff --git a/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py index cf8b458bbc..88f87e92e9 100644 --- a/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py @@ -10,7 +10,10 @@ import numpy as np import pytrec_eval import torch +from datasets import Dataset from PIL import Image +from torch.utils.data import DataLoader +from torchvision import transforms from mteb.encoder_interface import EncoderWithQueryCorpusEncode @@ -29,6 +32,29 @@ logger = logging.getLogger(__name__) +transform = transforms.Compose([transforms.PILToTensor()]) + + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, hf_dataset, image_column_name: str = "image", transform=None): + self.dataset = hf_dataset + self.transform = transform + self.image_column_name = image_column_name + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + image = self.dataset[idx][self.image_column_name] + if image.mode != "RGB": + image = image.convert("RGB") + image = self.transform(image) + return image + + +def custom_collate_fn(batch): + return batch + # Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/search/dense/exact_search.py#L12 class DenseRetrievalExactSearch: @@ -65,8 +91,8 @@ def __init__( def search( self, - corpus: dict[str, Image.Image], - queries: dict[str, dict[str, str]], + corpus: Dataset, + queries: Dataset, top_k: int, score_function: str, return_sorted: bool = False, @@ -78,16 +104,25 @@ def search( ) logger.info("Encoding Queries.") - query_ids = list(queries.keys()) + query_ids = list(queries["id"]) self.results = {qid: {} for qid in query_ids} - queries = [queries[qid]["text"] for qid in queries] + queries = queries["text"] query_embeddings = self.model.get_text_embeddings( queries, batch_size=self.encode_kwargs["batch_size"] ) logger.info("Preparing Corpus...") - corpus_ids = list(corpus.keys()) - corpus_images = [corpus[cid] for cid in corpus_ids] + corpus_ids = list(corpus["id"]) + corpus_dataset = ImageDataset( + corpus, image_column_name="image", transform=transform + ) + corpus_image_dataloader = DataLoader( + corpus_dataset, + batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=os.cpu_count(), + ) logger.info("Encoding Corpus in batches... Warning: This might take a while!") logger.info( @@ -96,67 +131,39 @@ def search( ) ) - itr = range(0, len(corpus), self.corpus_chunk_size) + corpus_embeddings = self.model.get_image_embeddings( + images=corpus_image_dataloader, batch_size=self.encode_kwargs["batch_size"] + ) result_heaps = { qid: [] for qid in query_ids } # Keep only the top-k docs for each query - for batch_num, corpus_start_idx in enumerate(itr): - logger.info("Encoding Batch {}/{}...".format(batch_num + 1, len(itr))) - corpus_end_idx = min(corpus_start_idx + self.corpus_chunk_size, len(corpus)) - - # Encode chunk of corpus - if ( - self.save_corpus_embeddings - and "qid" in kwargs - and len(self.corpus_embeddings[kwargs["qid"]]) - ): - sub_corpus_embeddings = torch.tensor( - self.corpus_embeddings[kwargs["qid"]][batch_num] - ) - else: - # Encode chunk of corpus - images = corpus_images[corpus_start_idx:corpus_end_idx] - sub_corpus_embeddings = self.model.get_image_embeddings( - # corpus[corpus_start_idx:corpus_end_idx], - images, - batch_size=self.encode_kwargs["batch_size"], - ) - if self.save_corpus_embeddings and "qid" in kwargs: - self.corpus_embeddings[kwargs["qid"]].append(sub_corpus_embeddings) - # Compute similarites using either cosine-similarity or dot product - cos_scores = self.score_functions[score_function]( - query_embeddings, sub_corpus_embeddings - ) - cos_scores[torch.isnan(cos_scores)] = -1 - - # Get top-k values - cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( - cos_scores, - min( - top_k + 1, - len(cos_scores[1]) if len(cos_scores) > 1 else len(cos_scores[-1]), - ), - dim=1, - largest=True, - sorted=return_sorted, - ) - cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() - cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() - - for query_itr in range(len(query_embeddings)): - query_id = query_ids[query_itr] - for sub_corpus_id, score in zip( - cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] - ): - corpus_id = corpus_ids[corpus_start_idx + sub_corpus_id] - if len(result_heaps[query_id]) < top_k: - # Push item on the heap - heapq.heappush(result_heaps[query_id], (score, corpus_id)) - else: - # If item is larger than the smallest in the heap, push it on the heap then pop the smallest element - heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) + cos_scores = self.score_functions[score_function]( + query_embeddings, corpus_embeddings + ) + cos_scores[torch.isnan(cos_scores)] = -1 + + cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( + cos_scores, + top_k, + dim=1, + largest=True, + sorted=return_sorted, + ) + cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() + cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() + + for query_itr in range(len(query_embeddings)): + query_id = query_ids[query_itr] + for sub_corpus_id, score in zip( + cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] + ): + corpus_id = corpus_ids[sub_corpus_id] + if len(result_heaps[query_id]) < top_k: + heapq.heappush(result_heaps[query_id], (score, corpus_id)) + else: + heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) for qid in result_heaps: for score, corpus_id in result_heaps[qid]: diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index bb2c61e338..94417b942b 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -89,7 +89,7 @@ def calculate_probs(self, text_embeddings, image_embeddings): def get_fused_embeddings( self, texts: list[str] = None, - images: list[Image.Image] = None, + images: list[Image.Image] | DataLoader = None, fusion_mode="sum", batch_size: int = 32, ): diff --git a/mteb/tasks/Image/T2IRetrieval/__init__.py b/mteb/tasks/Image/T2IRetrieval/__init__.py index 5fa74bdd4e..75f6eb9caf 100644 --- a/mteb/tasks/Image/T2IRetrieval/__init__.py +++ b/mteb/tasks/Image/T2IRetrieval/__init__.py @@ -1,3 +1,4 @@ from __future__ import annotations +from .eng.Fashion200kT2IRetrieval import * from .eng.MSCOCOT2IRetrieval import * diff --git a/mteb/tasks/Image/T2IRetrieval/eng/Fashion200kT2IRetrieval.py b/mteb/tasks/Image/T2IRetrieval/eng/Fashion200kT2IRetrieval.py new file mode 100644 index 0000000000..0eae629e5d --- /dev/null +++ b/mteb/tasks/Image/T2IRetrieval/eng/Fashion200kT2IRetrieval.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskT2IRetrieval + + +class Fashion200kT2IRetrieval(AbsTaskT2IRetrieval): + metadata = TaskMetadata( + name="Fashion200kT2IRetrieval", + description="Retrieve clothes based on descriptions.", + reference="https://openaccess.thecvf.com/content_iccv_2017/html/Han_Automatic_Spatially-Aware_Fashion_ICCV_2017_paper.html", + dataset={ + "path": "MRBench/mbeir_fashion200k_task0", + "revision": "1b86e2dde50e671d5c83d07a79e8b1d8c696964b", + # "trust_remote_code": True, + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2017-01-01", "2017-12-31"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="Apache-2.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@inproceedings{han2017automatic, + title={Automatic spatially-aware fashion concept discovery}, + author={Han, Xintong and Wu, Zuxuan and Huang, Phoenix X and Zhang, Xiao and Zhu, Menglong and Li, Yuan and Zhao, Yang and Davis, Larry S}, + booktitle={Proceedings of the IEEE international conference on computer vision}, + pages={1463--1471}, + year={2017} +}""", + descriptive_stats={ + "n_samples": {"test": 1719}, + "avg_character_length": { + "test": { + "average_document_length": 30.94235294117647, + "average_query_length": 131.56569965870307, + "num_documents": 201824, + "num_queries": 1719, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kT2IRetrieval.json new file mode 100644 index 0000000000..bd95aed3da --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kT2IRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "1b86e2dde50e671d5c83d07a79e8b1d8c696964b", + "evaluation_time": 787.4428789615631, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.01519, + "map_at_1": 0.00463, + "map_at_10": 0.0098, + "map_at_100": 0.01313, + "map_at_1000": 0.0145, + "map_at_20": 0.01107, + "map_at_3": 0.00718, + "map_at_5": 0.00837, + "mrr_at_1": 0.009307737056428156, + "mrr_at_10": 0.017149136171823776, + "mrr_at_100": 0.021056239330545873, + "mrr_at_1000": 0.022229945965150693, + "mrr_at_20": 0.01876052565704011, + "mrr_at_3": 0.012798138452588712, + "mrr_at_5": 0.015154159394997088, + "nauc_map_at_1000_diff1": 0.15317823848588835, + "nauc_map_at_1000_max": 0.3707778175232726, + "nauc_map_at_1000_std": 0.5513529953370644, + "nauc_map_at_100_diff1": 0.15847808580536737, + "nauc_map_at_100_max": 0.37859910108283945, + "nauc_map_at_100_std": 0.5591702162310715, + "nauc_map_at_10_diff1": 0.21041130697622099, + "nauc_map_at_10_max": 0.42580269159038553, + "nauc_map_at_10_std": 0.6098544986773251, + "nauc_map_at_1_diff1": 0.24460437684065364, + "nauc_map_at_1_max": 0.5398908169554762, + "nauc_map_at_1_std": 0.7089495012532402, + "nauc_map_at_20_diff1": 0.18101652805238996, + "nauc_map_at_20_max": 0.41197121831300915, + "nauc_map_at_20_std": 0.5931332163584082, + "nauc_map_at_3_diff1": 0.29239402842906054, + "nauc_map_at_3_max": 0.48418913696284516, + "nauc_map_at_3_std": 0.668531890584055, + "nauc_map_at_5_diff1": 0.2567059698570642, + "nauc_map_at_5_max": 0.45341283736125376, + "nauc_map_at_5_std": 0.6253794092756182, + "nauc_mrr_at_1000_diff1": 0.1307719512664129, + "nauc_mrr_at_1000_max": 0.326784704037806, + "nauc_mrr_at_1000_std": 0.4531970552488093, + "nauc_mrr_at_100_diff1": 0.13123410546023803, + "nauc_mrr_at_100_max": 0.3290602289954133, + "nauc_mrr_at_100_std": 0.4537819119706659, + "nauc_mrr_at_10_diff1": 0.1593237928308708, + "nauc_mrr_at_10_max": 0.3583121338641627, + "nauc_mrr_at_10_std": 0.4928534978178621, + "nauc_mrr_at_1_diff1": 0.20730859868154203, + "nauc_mrr_at_1_max": 0.468788203939954, + "nauc_mrr_at_1_std": 0.633374523001285, + "nauc_mrr_at_20_diff1": 0.14047651995233532, + "nauc_mrr_at_20_max": 0.3435170904987213, + "nauc_mrr_at_20_std": 0.4690886379389067, + "nauc_mrr_at_3_diff1": 0.22240178945439196, + "nauc_mrr_at_3_max": 0.4191234439740691, + "nauc_mrr_at_3_std": 0.5456838549504663, + "nauc_mrr_at_5_diff1": 0.19036504815271832, + "nauc_mrr_at_5_max": 0.38841387275222106, + "nauc_mrr_at_5_std": 0.5025611673628193, + "nauc_ndcg_at_1000_diff1": 0.06271178728730444, + "nauc_ndcg_at_1000_max": 0.25980377843991076, + "nauc_ndcg_at_1000_std": 0.42238971980153833, + "nauc_ndcg_at_100_diff1": 0.06892927634047888, + "nauc_ndcg_at_100_max": 0.26550937963515786, + "nauc_ndcg_at_100_std": 0.4232852028631629, + "nauc_ndcg_at_10_diff1": 0.1525986987396939, + "nauc_ndcg_at_10_max": 0.35518195150540305, + "nauc_ndcg_at_10_std": 0.5325073933143113, + "nauc_ndcg_at_1_diff1": 0.20730859868154203, + "nauc_ndcg_at_1_max": 0.468788203939954, + "nauc_ndcg_at_1_std": 0.633374523001285, + "nauc_ndcg_at_20_diff1": 0.11002233895191663, + "nauc_ndcg_at_20_max": 0.33186733998731466, + "nauc_ndcg_at_20_std": 0.49977699059530195, + "nauc_ndcg_at_3_diff1": 0.2510876023881896, + "nauc_ndcg_at_3_max": 0.4428573475936426, + "nauc_ndcg_at_3_std": 0.6108361525357452, + "nauc_ndcg_at_5_diff1": 0.21887224034279398, + "nauc_ndcg_at_5_max": 0.40876119909845304, + "nauc_ndcg_at_5_std": 0.5573993419624723, + "nauc_precision_at_1000_diff1": 0.031115583194632115, + "nauc_precision_at_1000_max": 0.1809611010103693, + "nauc_precision_at_1000_std": 0.2832925196998456, + "nauc_precision_at_100_diff1": 0.019172601222170217, + "nauc_precision_at_100_max": 0.20235981496845598, + "nauc_precision_at_100_std": 0.29677588925162657, + "nauc_precision_at_10_diff1": 0.06807082214371594, + "nauc_precision_at_10_max": 0.267528890941748, + "nauc_precision_at_10_std": 0.4088806442998267, + "nauc_precision_at_1_diff1": 0.20730859868154203, + "nauc_precision_at_1_max": 0.468788203939954, + "nauc_precision_at_1_std": 0.633374523001285, + "nauc_precision_at_20_diff1": 0.026159835125366106, + "nauc_precision_at_20_max": 0.2561727193975215, + "nauc_precision_at_20_std": 0.3703395011665188, + "nauc_precision_at_3_diff1": 0.23535261382907086, + "nauc_precision_at_3_max": 0.36679733902634626, + "nauc_precision_at_3_std": 0.5191545837207172, + "nauc_precision_at_5_diff1": 0.15226141418229086, + "nauc_precision_at_5_max": 0.3120957551523761, + "nauc_precision_at_5_std": 0.4277907224452402, + "nauc_recall_at_1000_diff1": 0.028915172975529115, + "nauc_recall_at_1000_max": 0.22768242611982248, + "nauc_recall_at_1000_std": 0.38989994849780985, + "nauc_recall_at_100_diff1": 0.020589093891737117, + "nauc_recall_at_100_max": 0.20835600133847137, + "nauc_recall_at_100_std": 0.37729386277822163, + "nauc_recall_at_10_diff1": 0.1037459686807867, + "nauc_recall_at_10_max": 0.29877440064977856, + "nauc_recall_at_10_std": 0.5155308636468926, + "nauc_recall_at_1_diff1": 0.24460437684065364, + "nauc_recall_at_1_max": 0.5398908169554762, + "nauc_recall_at_1_std": 0.7089495012532402, + "nauc_recall_at_20_diff1": 0.054939369566234304, + "nauc_recall_at_20_max": 0.28198381141530837, + "nauc_recall_at_20_std": 0.4785510498944293, + "nauc_recall_at_3_diff1": 0.2912211204998551, + "nauc_recall_at_3_max": 0.4274023242423866, + "nauc_recall_at_3_std": 0.6124458691991229, + "nauc_recall_at_5_diff1": 0.21817377658775405, + "nauc_recall_at_5_max": 0.3836703721551013, + "nauc_recall_at_5_std": 0.537587926520719, + "ndcg_at_1": 0.00931, + "ndcg_at_10": 0.01519, + "ndcg_at_100": 0.03573, + "ndcg_at_1000": 0.07765, + "ndcg_at_20": 0.0196, + "ndcg_at_3": 0.00981, + "ndcg_at_5": 0.01174, + "precision_at_1": 0.00931, + "precision_at_10": 0.00547, + "precision_at_100": 0.00267, + "precision_at_1000": 0.00096, + "precision_at_20": 0.00439, + "precision_at_3": 0.00679, + "precision_at_5": 0.00675, + "recall_at_1": 0.00463, + "recall_at_10": 0.02274, + "recall_at_100": 0.10759, + "recall_at_1000": 0.3845, + "recall_at_20": 0.03638, + "recall_at_3": 0.00971, + "recall_at_5": 0.01394 + } + ] + }, + "task_name": "Fashion200kT2IRetrieval" +} \ No newline at end of file From 33047fb3651560e6678e5fbbe9cd50f319a1a366 Mon Sep 17 00:00:00 2001 From: Imene Kerboua <33312980+imenelydiaker@users.noreply.github.com> Date: Wed, 31 Jul 2024 23:09:52 +0200 Subject: [PATCH 036/154] Add CLEVR and SciMMIR Image-Text Understanding tasks (#1127) * Add CLEVER and SciMMIR * Update metadata * remove useless comment * Add linting * fix typo and tests * Add CLEVR count task * add linting --- .../Image/ZeroshotClassification/__init__.py | 2 + .../Image/ZeroshotClassification/eng/CLEVR.py | 111 ++++++++++++++++++ .../eng/RenderedSST2.py | 5 +- .../ZeroshotClassification/eng/SciMMIR.py | 71 +++++++++++ .../CLEVR.json | 19 +++ .../CLEVRCount.json | 19 +++ .../SciMMIR.json | 19 +++ 7 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py create mode 100644 mteb/tasks/Image/ZeroshotClassification/eng/SciMMIR.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVR.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVRCount.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIR.json diff --git a/mteb/tasks/Image/ZeroshotClassification/__init__.py b/mteb/tasks/Image/ZeroshotClassification/__init__.py index 99fd108065..1baf75604b 100644 --- a/mteb/tasks/Image/ZeroshotClassification/__init__.py +++ b/mteb/tasks/Image/ZeroshotClassification/__init__.py @@ -3,6 +3,7 @@ from .eng.Birdsnap import * from .eng.Caltech101 import * from .eng.CIFAR import * +from .eng.CLEVR import * from .eng.Country211 import * from .eng.DTD import * from .eng.EuroSAT import * @@ -16,6 +17,7 @@ from .eng.PatchCamelyon import * from .eng.RenderedSST2 import * from .eng.RESISC45 import * +from .eng.SciMMIR import * from .eng.StanfordCars import * from .eng.STL10 import * from .eng.SUN397 import * diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py b/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py new file mode 100644 index 0000000000..dacfed5b45 --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskZeroshotClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class CLEVR(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="CLEVR", + description="CLEVR closest object distance identification task.", + reference="https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html", + dataset={ + "path": "clip-benchmark/wds_vtab-clevr_closest_object_distance", + "revision": "ec9c04224a95836ca0344a6000ec8d8bc8a6d4f2", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2016-01-01", "2016-12-20"), + domains=["Constructed"], + task_subtypes=["Object recognition"], + license="""""", + socioeconomic_status="mixed", + annotations_creators="human-annotated", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""\ +@InProceedings{Johnson_2017_CVPR, +author = {Johnson, Justin and Hariharan, Bharath and van der Maaten, Laurens and Fei-Fei, Li and Lawrence Zitnick, C. and Girshick, Ross}, +title = {CLEVR: A Diagnostic Dataset for Compositional Language and Elementary Visual Reasoning}, +booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, +month = {July}, +year = {2017} +}""", + descriptive_stats={ + "n_samples": {"test": 15000}, + "avg_character_length": {"test": 0}, + }, + ) + + image_column_name: str = "webp" + label_column_name: str = "cls" + + def get_candidate_labels(self) -> list[str]: + labels = [ + "very nearby", + "nearby", + "near", + "", # missing this class name in the original dataset: https://huggingface.co/datasets/clip-benchmark/wds_vtab-clevr_closest_object_distance/blob/main/classnames.txt + "distant", + "very distant", + ] + + return [f"{c} shapes." for c in labels] + + +class CLEVRCount(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="CLEVRCount", + description="CLEVR count objects task.", + reference="https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html", + dataset={ + "path": "clip-benchmark/wds_vtab-clevr_count_all", + "revision": "8b5dce4d5393a04fb58b9261b22a881b02e379b1", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2016-01-01", "2016-12-20"), + domains=["Constructed"], + task_subtypes=["Object recognition"], + license="""""", + socioeconomic_status="mixed", + annotations_creators="human-annotated", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""\ +@InProceedings{Johnson_2017_CVPR, +author = {Johnson, Justin and Hariharan, Bharath and van der Maaten, Laurens and Fei-Fei, Li and Lawrence Zitnick, C. and Girshick, Ross}, +title = {CLEVR: A Diagnostic Dataset for Compositional Language and Elementary Visual Reasoning}, +booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, +month = {July}, +year = {2017} +}""", + descriptive_stats={ + "n_samples": {"test": 15000}, + "avg_character_length": {"test": 0}, + }, + ) + + image_column_name: str = "webp" + label_column_name: str = "cls" + + def get_candidate_labels(self) -> list[str]: + labels = [ + "three", + "four", + "five", + "six", + "seven", + "eight", + "nine", + "ten", + ] + return [f"a picture of {c} objects" for c in labels] diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py index b1206ca762..df0257d8ea 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py @@ -1,15 +1,14 @@ from __future__ import annotations +from mteb.abstasks import AbsTaskZeroshotClassification from mteb.abstasks.TaskMetadata import TaskMetadata -from .....abstasks import AbsTaskZeroshotClassification - class RenderedSST2(AbsTaskZeroshotClassification): metadata = TaskMetadata( name="RenderedSST2", description="RenderedSST2.", - reference="https://huggingface.co/datasets/clip-benchmark/wds_renderedsst2/commits/main", + reference="https://huggingface.co/datasets/clip-benchmark/wds_renderedsst2", dataset={ "path": "clip-benchmark/wds_renderedsst2", "revision": "66b9a461eda025201dd147e5f390f5984c33643a", diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/SciMMIR.py b/mteb/tasks/Image/ZeroshotClassification/eng/SciMMIR.py new file mode 100644 index 0000000000..d3839b144e --- /dev/null +++ b/mteb/tasks/Image/ZeroshotClassification/eng/SciMMIR.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskZeroshotClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class SciMMIR(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + name="SciMMIR", + description="SciMMIR.", + reference="https://huggingface.co/datasets/m-a-p/SciMMIR", + dataset={ + "path": "m-a-p/SciMMIR", + "revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", + }, + type="ZeroShotClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2023-05-01", "2023-10-30"), + domains=["Academic"], + task_subtypes=["Caption Pairing", "Rendered Texts Understanding"], + license="""""", + socioeconomic_status="mixed", + annotations_creators="human-annotated", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""\ +@misc{wu2024scimmirbenchmarkingscientificmultimodal, + title={SciMMIR: Benchmarking Scientific Multi-modal Information Retrieval}, + author={Siwei Wu and Yizhi Li and Kang Zhu and Ge Zhang and Yiming Liang and Kaijing Ma and Chenghao Xiao and Haoran Zhang and Bohao Yang and Wenhu Chen and Wenhao Huang and Noura Al Moubayed and Jie Fu and Chenghua Lin}, + year={2024}, + eprint={2401.13478}, + archivePrefix={arXiv}, + primaryClass={cs.IR}, + url={https://arxiv.org/abs/2401.13478}, +}""", + descriptive_stats={ + "n_samples": {"test": 16263}, + "avg_character_length": {"test": 0}, + }, + ) + + label_column_name: str = "class" + + def dataset_transform(self): + class_code = { + "fig_result": 0, + "fig_illustration": 1, + "fig_architecture": 2, + "table_parameter": 3, + "table_result": 4, + } + for split in self.metadata.eval_splits: + self.dataset[split] = self.dataset[split].map( + lambda example: { + "image": example["image"], + "class": class_code[example[self.label_column_name]], + } + ) + + def get_candidate_labels(self) -> list[str]: + return [ + "a figure of results", + "a figure of an illustration", + "a figure of an architecture", + "a table of parameters", + "a table of results", + ] diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVR.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVR.json new file mode 100644 index 0000000000..468ad76812 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVR.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "ec9c04224a95836ca0344a6000ec8d8bc8a6d4f2", + "evaluation_time": 64.04479908943176, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "accuracy": 0.1634, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.1634 + } + ] + }, + "task_name": "CLEVER" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVRCount.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVRCount.json new file mode 100644 index 0000000000..565041de13 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVRCount.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "8b5dce4d5393a04fb58b9261b22a881b02e379b1", + "evaluation_time": 67.00646996498108, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "accuracy": 0.232, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.232 + } + ] + }, + "task_name": "CLEVRCount" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIR.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIR.json new file mode 100644 index 0000000000..b491c5cfea --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIR.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", + "evaluation_time": 94.48416376113892, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "accuracy": 0.4564963413884277, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.4564963413884277 + } + ] + }, + "task_name": "SciMMIR" +} \ No newline at end of file From da470bd2305227042ca712cdd3775457de297274 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Sun, 4 Aug 2024 23:20:21 +0100 Subject: [PATCH 037/154] add fashion200k & fashionIQ test passed --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 1 + .../eng/FashionIQIT2IRetrieval.py | 51 ++++++ mteb/tasks/Image/I2TRetrieval/__init__.py | 1 + .../eng/Fashion200kI2TRetrieval.py | 51 ++++++ .../Fashion200kI2TRetrieval.json | 158 ++++++++++++++++++ .../FashionIQIT2IRetrieval.json | 158 ++++++++++++++++++ 6 files changed, 420 insertions(+) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py create mode 100644 mteb/tasks/Image/I2TRetrieval/eng/Fashion200kI2TRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FashionIQIT2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 91a7f471a9..7087779f71 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -1,3 +1,4 @@ from __future__ import annotations from .eng.CIRRIT2IRetrieval import * +from .eng.FashionIQIT2IRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py new file mode 100644 index 0000000000..c38f5155dd --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class FashionIQIT2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="FashionIQIT2IRetrieval", + description="Retrieve clothes based on descriptions.", + reference="https://openaccess.thecvf.com/content/CVPR2021/html/Wu_Fashion_IQ_A_New_Dataset_Towards_Retrieving_Images_by_Natural_CVPR_2021_paper.html", + dataset={ + "path": "MRBench/mbeir_fashioniq_task7", + "revision": "e6f0ec70becc413d940cd62b2cfa3b1d3a08c31a", + # "trust_remote_code": True, + }, + type="Retrieval", + category="it2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2021-01-01", "2021-12-31"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="Apache-2.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@inproceedings{wu2021fashion, + title={Fashion iq: A new dataset towards retrieving images by natural language feedback}, + author={Wu, Hui and Gao, Yupeng and Guo, Xiaoxiao and Al-Halah, Ziad and Rennie, Steven and Grauman, Kristen and Feris, Rogerio}, + booktitle={Proceedings of the IEEE/CVF Conference on computer vision and pattern recognition}, + pages={11307--11317}, + year={2021} +}""", + descriptive_stats={ + "n_samples": {"test": 6000}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 74400, + "num_queries": 6000, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/mteb/tasks/Image/I2TRetrieval/__init__.py b/mteb/tasks/Image/I2TRetrieval/__init__.py index c5e58ad927..102974c6f2 100644 --- a/mteb/tasks/Image/I2TRetrieval/__init__.py +++ b/mteb/tasks/Image/I2TRetrieval/__init__.py @@ -1,3 +1,4 @@ from __future__ import annotations +from .eng.Fashion200kI2TRetrieval import * from .eng.MSCOCOI2TRetrieval import * diff --git a/mteb/tasks/Image/I2TRetrieval/eng/Fashion200kI2TRetrieval.py b/mteb/tasks/Image/I2TRetrieval/eng/Fashion200kI2TRetrieval.py new file mode 100644 index 0000000000..c2409da0f8 --- /dev/null +++ b/mteb/tasks/Image/I2TRetrieval/eng/Fashion200kI2TRetrieval.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskI2TRetrieval + + +class Fashion200kI2TRetrieval(AbsTaskI2TRetrieval): + metadata = TaskMetadata( + name="Fashion200kI2TRetrieval", + description="Retrieve clothes based on descriptions.", + reference="https://openaccess.thecvf.com/content_iccv_2017/html/Han_Automatic_Spatially-Aware_Fashion_ICCV_2017_paper.html", + dataset={ + "path": "MRBench/mbeir_fashion200k_task3", + "revision": "96a313715ecf67f5dfe70c4fa52406bc7bdfbeee", + # "trust_remote_code": True, + }, + type="Retrieval", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2017-01-01", "2017-12-31"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="Apache-2.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@inproceedings{han2017automatic, + title={Automatic spatially-aware fashion concept discovery}, + author={Han, Xintong and Wu, Zuxuan and Huang, Phoenix X and Zhang, Xiao and Zhu, Menglong and Li, Yuan and Zhao, Yang and Davis, Larry S}, + booktitle={Proceedings of the IEEE international conference on computer vision}, + pages={1463--1471}, + year={2017} +}""", + descriptive_stats={ + "n_samples": {"test": 4890}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 61700, + "num_queries": 4890, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kI2TRetrieval.json new file mode 100644 index 0000000000..ab412718ee --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kI2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "96a313715ecf67f5dfe70c4fa52406bc7bdfbeee", + "evaluation_time": 34.811583518981934, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.02127, + "map_at_1": 0.00614, + "map_at_10": 0.01488, + "map_at_100": 0.01919, + "map_at_1000": 0.02032, + "map_at_20": 0.01661, + "map_at_3": 0.01087, + "map_at_5": 0.01255, + "mrr_at_1": 0.006136224176723257, + "mrr_at_10": 0.014879856626635128, + "mrr_at_100": 0.019185020550758353, + "mrr_at_1000": 0.020315066221173354, + "mrr_at_20": 0.01661308681252838, + "mrr_at_3": 0.010874752846526219, + "mrr_at_5": 0.012551987454830582, + "nauc_map_at_1000_diff1": 0.07142806850553053, + "nauc_map_at_1000_max": 0.17906242598039115, + "nauc_map_at_1000_std": 0.3168950516726266, + "nauc_map_at_100_diff1": 0.07266830335912308, + "nauc_map_at_100_max": 0.17944952380795798, + "nauc_map_at_100_std": 0.316030703623398, + "nauc_map_at_10_diff1": 0.09164462564312548, + "nauc_map_at_10_max": 0.20279243640349265, + "nauc_map_at_10_std": 0.33321717908166465, + "nauc_map_at_1_diff1": 0.11815857703103975, + "nauc_map_at_1_max": 0.26408283394287474, + "nauc_map_at_1_std": 0.4157890704329977, + "nauc_map_at_20_diff1": 0.08132288730033431, + "nauc_map_at_20_max": 0.1912704962920035, + "nauc_map_at_20_std": 0.3217909720943865, + "nauc_map_at_3_diff1": 0.09133842779733792, + "nauc_map_at_3_max": 0.2256728647948325, + "nauc_map_at_3_std": 0.3398655747948414, + "nauc_map_at_5_diff1": 0.0823469463229123, + "nauc_map_at_5_max": 0.22508575065063585, + "nauc_map_at_5_std": 0.34776718853366784, + "nauc_mrr_at_1000_diff1": 0.07142806021147871, + "nauc_mrr_at_1000_max": 0.17906243579920242, + "nauc_mrr_at_1000_std": 0.316895066554052, + "nauc_mrr_at_100_diff1": 0.07266830335912308, + "nauc_mrr_at_100_max": 0.17944952380795798, + "nauc_mrr_at_100_std": 0.316030703623398, + "nauc_mrr_at_10_diff1": 0.09164462564312548, + "nauc_mrr_at_10_max": 0.20279243640349265, + "nauc_mrr_at_10_std": 0.33321717908166465, + "nauc_mrr_at_1_diff1": 0.11815857703103975, + "nauc_mrr_at_1_max": 0.26408283394287474, + "nauc_mrr_at_1_std": 0.4157890704329977, + "nauc_mrr_at_20_diff1": 0.08132288730033431, + "nauc_mrr_at_20_max": 0.1912704962920035, + "nauc_mrr_at_20_std": 0.3217909720943865, + "nauc_mrr_at_3_diff1": 0.09133842779733792, + "nauc_mrr_at_3_max": 0.2256728647948325, + "nauc_mrr_at_3_std": 0.3398655747948414, + "nauc_mrr_at_5_diff1": 0.0823469463229123, + "nauc_mrr_at_5_max": 0.22508575065063585, + "nauc_mrr_at_5_std": 0.34776718853366784, + "nauc_ndcg_at_1000_diff1": 0.041457607781455685, + "nauc_ndcg_at_1000_max": 0.14174752587899012, + "nauc_ndcg_at_1000_std": 0.30584900382090274, + "nauc_ndcg_at_100_diff1": 0.04809582600060403, + "nauc_ndcg_at_100_max": 0.14021333277597503, + "nauc_ndcg_at_100_std": 0.29106027529194606, + "nauc_ndcg_at_10_diff1": 0.09329799574658992, + "nauc_ndcg_at_10_max": 0.18548222751408638, + "nauc_ndcg_at_10_std": 0.3196240966651513, + "nauc_ndcg_at_1_diff1": 0.11815857703103975, + "nauc_ndcg_at_1_max": 0.26408283394287474, + "nauc_ndcg_at_1_std": 0.4157890704329977, + "nauc_ndcg_at_20_diff1": 0.07002390545831393, + "nauc_ndcg_at_20_max": 0.16326790533898922, + "nauc_ndcg_at_20_std": 0.296988919048661, + "nauc_ndcg_at_3_diff1": 0.08864420643603425, + "nauc_ndcg_at_3_max": 0.2246812088071025, + "nauc_ndcg_at_3_std": 0.3324288293295176, + "nauc_ndcg_at_5_diff1": 0.07762634832410889, + "nauc_ndcg_at_5_max": 0.22473611455942216, + "nauc_ndcg_at_5_std": 0.34600164677283235, + "nauc_precision_at_1000_diff1": 0.02402277161780163, + "nauc_precision_at_1000_max": 0.12476362384000816, + "nauc_precision_at_1000_std": 0.3107702224242301, + "nauc_precision_at_100_diff1": 0.032135602055711115, + "nauc_precision_at_100_max": 0.11791356699033088, + "nauc_precision_at_100_std": 0.27889601162358074, + "nauc_precision_at_10_diff1": 0.09648404787315049, + "nauc_precision_at_10_max": 0.1628069283520906, + "nauc_precision_at_10_std": 0.3024701967127499, + "nauc_precision_at_1_diff1": 0.11815857703103975, + "nauc_precision_at_1_max": 0.26408283394287474, + "nauc_precision_at_1_std": 0.4157890704329977, + "nauc_precision_at_20_diff1": 0.05755102665376828, + "nauc_precision_at_20_max": 0.134828413427724, + "nauc_precision_at_20_std": 0.2718561150616516, + "nauc_precision_at_3_diff1": 0.08406873162770592, + "nauc_precision_at_3_max": 0.2240122829921389, + "nauc_precision_at_3_std": 0.31986667568664484, + "nauc_precision_at_5_diff1": 0.07086103530059168, + "nauc_precision_at_5_max": 0.2253351962128875, + "nauc_precision_at_5_std": 0.3453532111047287, + "nauc_recall_at_1000_diff1": 0.02402277161780195, + "nauc_recall_at_1000_max": 0.12476362384000846, + "nauc_recall_at_1000_std": 0.31077022242423036, + "nauc_recall_at_100_diff1": 0.032135602055711136, + "nauc_recall_at_100_max": 0.11791356699033084, + "nauc_recall_at_100_std": 0.2788960116235806, + "nauc_recall_at_10_diff1": 0.09648404787315036, + "nauc_recall_at_10_max": 0.16280692835209049, + "nauc_recall_at_10_std": 0.30247019671274983, + "nauc_recall_at_1_diff1": 0.11815857703103975, + "nauc_recall_at_1_max": 0.26408283394287474, + "nauc_recall_at_1_std": 0.4157890704329977, + "nauc_recall_at_20_diff1": 0.05755102665376812, + "nauc_recall_at_20_max": 0.13482841342772403, + "nauc_recall_at_20_std": 0.27185611506165164, + "nauc_recall_at_3_diff1": 0.0840687316277059, + "nauc_recall_at_3_max": 0.22401228299213888, + "nauc_recall_at_3_std": 0.3198666756866447, + "nauc_recall_at_5_diff1": 0.07086103530059157, + "nauc_recall_at_5_max": 0.22533519621288747, + "nauc_recall_at_5_std": 0.34535321110472866, + "ndcg_at_1": 0.00614, + "ndcg_at_10": 0.02127, + "ndcg_at_100": 0.04791, + "ndcg_at_1000": 0.0874, + "ndcg_at_20": 0.02764, + "ndcg_at_3": 0.01249, + "ndcg_at_5": 0.01556, + "precision_at_1": 0.00614, + "precision_at_10": 0.00425, + "precision_at_100": 0.00181, + "precision_at_1000": 0.00051, + "precision_at_20": 0.0034, + "precision_at_3": 0.00573, + "precision_at_5": 0.00495, + "recall_at_1": 0.00614, + "recall_at_10": 0.04254, + "recall_at_100": 0.18061, + "recall_at_1000": 0.51237, + "recall_at_20": 0.06791, + "recall_at_3": 0.01718, + "recall_at_5": 0.02475 + } + ] + }, + "task_name": "Fashion200kI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FashionIQIT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FashionIQIT2IRetrieval.json new file mode 100644 index 0000000000..a22e7f02e6 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FashionIQIT2IRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e6f0ec70becc413d940cd62b2cfa3b1d3a08c31a", + "evaluation_time": 238.13813376426697, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.02256, + "map_at_1": 0.00083, + "map_at_10": 0.01391, + "map_at_100": 0.01746, + "map_at_1000": 0.01834, + "map_at_20": 0.01542, + "map_at_3": 0.00858, + "map_at_5": 0.01143, + "mrr_at_1": 0.0011660836248542396, + "mrr_at_10": 0.014161966635729764, + "mrr_at_100": 0.017712377694095203, + "mrr_at_1000": 0.018584663737948794, + "mrr_at_20": 0.01566681657909266, + "mrr_at_3": 0.008801154978066527, + "mrr_at_5": 0.01172469320895109, + "nauc_map_at_1000_diff1": -0.0598912640286578, + "nauc_map_at_1000_max": 0.0859123853979979, + "nauc_map_at_1000_std": 0.22907437716306384, + "nauc_map_at_100_diff1": -0.05872727439663456, + "nauc_map_at_100_max": 0.08223964815666353, + "nauc_map_at_100_std": 0.23237931413225082, + "nauc_map_at_10_diff1": -0.060422008312022546, + "nauc_map_at_10_max": 0.08379801841169691, + "nauc_map_at_10_std": 0.23414021829196138, + "nauc_map_at_1_diff1": -0.5746793234443401, + "nauc_map_at_1_max": -0.11897126039760639, + "nauc_map_at_1_std": 0.34186303568634, + "nauc_map_at_20_diff1": -0.057420791623517736, + "nauc_map_at_20_max": 0.07986494217564785, + "nauc_map_at_20_std": 0.23768374054528263, + "nauc_map_at_3_diff1": -0.05308375080151235, + "nauc_map_at_3_max": 0.06643197364615645, + "nauc_map_at_3_std": 0.2153150428142786, + "nauc_map_at_5_diff1": -0.04557552118534602, + "nauc_map_at_5_max": 0.08569842305795435, + "nauc_map_at_5_std": 0.22520034277664164, + "nauc_mrr_at_1000_diff1": -0.06692849899536789, + "nauc_mrr_at_1000_max": 0.07959932284987632, + "nauc_mrr_at_1000_std": 0.23483241832540724, + "nauc_mrr_at_100_diff1": -0.06593632580506924, + "nauc_mrr_at_100_max": 0.07585456666993269, + "nauc_mrr_at_100_std": 0.23821922385835617, + "nauc_mrr_at_10_diff1": -0.06928288793645844, + "nauc_mrr_at_10_max": 0.07594893566631603, + "nauc_mrr_at_10_std": 0.2411048130518643, + "nauc_mrr_at_1_diff1": -0.5746793234443403, + "nauc_mrr_at_1_max": -0.24917356412524466, + "nauc_mrr_at_1_std": 0.4815856420359519, + "nauc_mrr_at_20_diff1": -0.06542436073391363, + "nauc_mrr_at_20_max": 0.07256823322137557, + "nauc_mrr_at_20_std": 0.24438604121380278, + "nauc_mrr_at_3_diff1": -0.06574346236975877, + "nauc_mrr_at_3_max": 0.05465660858795071, + "nauc_mrr_at_3_std": 0.22718655887292788, + "nauc_mrr_at_5_diff1": -0.05700786192665374, + "nauc_mrr_at_5_max": 0.07490019732032167, + "nauc_mrr_at_5_std": 0.2363561699002534, + "nauc_ndcg_at_1000_diff1": -0.07572396773256204, + "nauc_ndcg_at_1000_max": 0.12148459423561712, + "nauc_ndcg_at_1000_std": 0.1883299605315341, + "nauc_ndcg_at_100_diff1": -0.058616309315249894, + "nauc_ndcg_at_100_max": 0.08313219905874299, + "nauc_ndcg_at_100_std": 0.22773559257768028, + "nauc_ndcg_at_10_diff1": -0.0612109564493037, + "nauc_ndcg_at_10_max": 0.08768057389756329, + "nauc_ndcg_at_10_std": 0.2352208258944654, + "nauc_ndcg_at_1_diff1": -0.5746793234443401, + "nauc_ndcg_at_1_max": -0.11897126039760639, + "nauc_ndcg_at_1_std": 0.34186303568634, + "nauc_ndcg_at_20_diff1": -0.0520605133348174, + "nauc_ndcg_at_20_max": 0.07900368270245652, + "nauc_ndcg_at_20_std": 0.24274695093730195, + "nauc_ndcg_at_3_diff1": -0.045307157277366054, + "nauc_ndcg_at_3_max": 0.0737406770633018, + "nauc_ndcg_at_3_std": 0.2103745920700806, + "nauc_ndcg_at_5_diff1": -0.03635638958328183, + "nauc_ndcg_at_5_max": 0.09290557427909102, + "nauc_ndcg_at_5_std": 0.22452211139249575, + "nauc_precision_at_1000_diff1": -0.09528976019730388, + "nauc_precision_at_1000_max": 0.15453704642183053, + "nauc_precision_at_1000_std": 0.1476052191408992, + "nauc_precision_at_100_diff1": -0.06103098984068198, + "nauc_precision_at_100_max": 0.08062010746375872, + "nauc_precision_at_100_std": 0.2240150722195767, + "nauc_precision_at_10_diff1": -0.06356559296982019, + "nauc_precision_at_10_max": 0.08804802619647352, + "nauc_precision_at_10_std": 0.23838131471222793, + "nauc_precision_at_1_diff1": -0.5746793234443401, + "nauc_precision_at_1_max": -0.11897126039760639, + "nauc_precision_at_1_std": 0.34186303568634, + "nauc_precision_at_20_diff1": -0.04729671581693161, + "nauc_precision_at_20_max": 0.07471004104058977, + "nauc_precision_at_20_std": 0.25043638477905833, + "nauc_precision_at_3_diff1": -0.036268500912781894, + "nauc_precision_at_3_max": 0.08353291703652879, + "nauc_precision_at_3_std": 0.2037605481596069, + "nauc_precision_at_5_diff1": -0.02651744785994212, + "nauc_precision_at_5_max": 0.09965707310149198, + "nauc_precision_at_5_std": 0.2260845282965032, + "nauc_recall_at_1000_diff1": -0.09421844317230951, + "nauc_recall_at_1000_max": 0.15558854134225528, + "nauc_recall_at_1000_std": 0.14658974505101816, + "nauc_recall_at_100_diff1": -0.05982995232397422, + "nauc_recall_at_100_max": 0.0820516416081276, + "nauc_recall_at_100_std": 0.2234736572809348, + "nauc_recall_at_10_diff1": -0.06399862735766773, + "nauc_recall_at_10_max": 0.09020731094094402, + "nauc_recall_at_10_std": 0.23574022327133567, + "nauc_recall_at_1_diff1": -0.5746793234443401, + "nauc_recall_at_1_max": -0.11897126039760639, + "nauc_recall_at_1_std": 0.34186303568634, + "nauc_recall_at_20_diff1": -0.046939197170608, + "nauc_recall_at_20_max": 0.076141622480933, + "nauc_recall_at_20_std": 0.24777983661827613, + "nauc_recall_at_3_diff1": -0.03626850091278174, + "nauc_recall_at_3_max": 0.08353291703652911, + "nauc_recall_at_3_std": 0.20376054815960698, + "nauc_recall_at_5_diff1": -0.026002358838177916, + "nauc_recall_at_5_max": 0.10141617506386659, + "nauc_recall_at_5_std": 0.22293448855893305, + "ndcg_at_1": 0.00083, + "ndcg_at_10": 0.02256, + "ndcg_at_100": 0.04358, + "ndcg_at_1000": 0.07302, + "ndcg_at_20": 0.0281, + "ndcg_at_3": 0.01136, + "ndcg_at_5": 0.01652, + "precision_at_1": 0.00083, + "precision_at_10": 0.0051, + "precision_at_100": 0.00158, + "precision_at_1000": 0.0004, + "precision_at_20": 0.00366, + "precision_at_3": 0.0065, + "precision_at_5": 0.00643, + "recall_at_1": 0.00083, + "recall_at_10": 0.05072, + "recall_at_100": 0.15775, + "recall_at_1000": 0.40346, + "recall_at_20": 0.0728, + "recall_at_3": 0.01949, + "recall_at_5": 0.03198 + } + ] + }, + "task_name": "FashionIQIT2IRetrieval" +} \ No newline at end of file From c59fd9540822f8f1a1fd1566fecd61092c2b4a3b Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Wed, 7 Aug 2024 07:55:00 +0100 Subject: [PATCH 038/154] clip text max seq truncation --- mteb/models/clip_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index 94417b942b..14d3cbcd9d 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -40,7 +40,7 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): for i in tqdm(range(0, len(texts), batch_size)): batch_texts = texts[i : i + batch_size] inputs = self.processor( - text=batch_texts, return_tensors="pt", padding=True + text=batch_texts, return_tensors="pt", padding=True, truncation = True ) inputs = {k: v.to(self.device) for k, v in inputs.items()} text_outputs = self.model.get_text_features(**inputs) From 8613945f32c8260f47ef42849c511ea17e632a8d Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Wed, 7 Aug 2024 08:00:13 +0100 Subject: [PATCH 039/154] add WebQA, NIGHTS, OVEN --- mteb/abstasks/TaskMetadata.py | 2 + mteb/models/clip_models.py | 2 +- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 5 + .../Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py | 4 +- .../eng/NIGHTSI2IRetrieval.py | 51 ++++++ .../eng/OVENIT2ITRetrieval.py | 51 ++++++ .../Any2AnyRetrieval/eng/OVENIT2TRetrieval.py | 51 ++++++ .../eng/WebQAT2ITRetrieval.py | 51 ++++++ .../Any2AnyRetrieval/eng/WebQAT2TRetrieval.py | 51 ++++++ .../NIGHTSI2IRetrieval.json | 158 ++++++++++++++++++ .../OVENIT2ITRetrieval.json | 158 ++++++++++++++++++ .../OVENIT2TRetrieval.json | 158 ++++++++++++++++++ .../WebQAT2ITRetrieval.json | 158 ++++++++++++++++++ .../WebQAT2TRetrieval.json | 158 ++++++++++++++++++ 14 files changed, 1055 insertions(+), 3 deletions(-) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2ITRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2ITRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2TRetrieval.json diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 4dc9e9b874..4b3b02891e 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -21,6 +21,7 @@ "Dialect pairing", "Dialog Systems", "Discourse coherence", + "Duplicate Image Retrieval", "Language identification", "Linguistic acceptability", "Political classification", @@ -103,6 +104,7 @@ "s2s", # Sentence-to-sentence "s2p", # Sentence-to-paragraph "p2p", # Paragraph-to-paragraph + "i2i", "i2t", "t2i", "it2t", diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index 14d3cbcd9d..0270e5abe3 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -40,7 +40,7 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): for i in tqdm(range(0, len(texts), batch_size)): batch_texts = texts[i : i + batch_size] inputs = self.processor( - text=batch_texts, return_tensors="pt", padding=True, truncation = True + text=batch_texts, return_tensors="pt", padding=True, truncation=True ) inputs = {k: v.to(self.device) for k, v in inputs.items()} text_outputs = self.model.get_text_features(**inputs) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 7087779f71..828013191e 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -2,3 +2,8 @@ from .eng.CIRRIT2IRetrieval import * from .eng.FashionIQIT2IRetrieval import * +from .eng.NIGHTSI2IRetrieval import * +from .eng.OVENIT2ITRetrieval import * +from .eng.OVENIT2TRetrieval import * +from .eng.WebQAT2ITRetrieval import * +from .eng.WebQAT2TRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py index 236449ab93..8bcfd80cd6 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py @@ -41,8 +41,8 @@ class CIRRIT2TRetrieval(AbsTaskAny2AnyRetrieval): "n_samples": {"test": 1172}, "avg_character_length": { "test": { - "average_document_length": 30.94235294117647, - "average_query_length": 131.56569965870307, + "average_document_length": 0.0, + "average_query_length": 0.0, "num_documents": 9350, "num_queries": 1172, "average_relevant_docs_per_query": 1.0, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py new file mode 100644 index 0000000000..291c0ad13a --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class NIGHTSI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="NIGHTSI2IRetrieval", + description="Retrieval identical image to the given image.", + reference="https://proceedings.neurips.cc/paper_files/paper/2023/hash/9f09f316a3eaf59d9ced5ffaefe97e0f-Abstract-Conference.html", + dataset={ + "path": "MRBench/mbeir_nights_task4", + "revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2023-01-01", "2023-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Duplicate Image Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@article{fu2024dreamsim, + title={DreamSim: Learning New Dimensions of Human Visual Similarity using Synthetic Data}, + author={Fu, Stephanie and Tamir, Netanel and Sundaram, Shobhita and Chai, Lucy and Zhang, Richard and Dekel, Tali and Isola, Phillip}, + journal={Advances in Neural Information Processing Systems}, + volume={36}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 2120}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 40038, + "num_queries": 2120, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py new file mode 100644 index 0000000000..429b33e6a0 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class OVENIT2ITRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="OVENIT2ITRetrieval", + description="Retrieval a Wiki image and passage to answer query about an image.", + reference="https://openaccess.thecvf.com/content/ICCV2023/html/Hu_Open-domain_Visual_Entity_Recognition_Towards_Recognizing_Millions_of_Wikipedia_Entities_ICCV_2023_paper.html", + dataset={ + "path": "MRBench/mbeir_oven_task8", + "revision": "350d14b7258189654e26a2be93dc0bd6bee09b76", + }, + type="Retrieval", + category="it2it", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2023-01-01", "2023-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{hu2023open, + title={Open-domain visual entity recognition: Towards recognizing millions of wikipedia entities}, + author={Hu, Hexiang and Luan, Yi and Chen, Yang and Khandelwal, Urvashi and Joshi, Mandar and Lee, Kenton and Toutanova, Kristina and Chang, Ming-Wei}, + booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision}, + pages={12065--12075}, + year={2023} +}""", + descriptive_stats={ + "n_samples": {"test": 14741}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 335135, + "num_queries": 14741, + "average_relevant_docs_per_query": 17.7, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py new file mode 100644 index 0000000000..d60b5b275b --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class OVENIT2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="OVENIT2TRetrieval", + description="Retrieval a Wiki passage to answer query about an image.", + reference="https://openaccess.thecvf.com/content/ICCV2023/html/Hu_Open-domain_Visual_Entity_Recognition_Towards_Recognizing_Millions_of_Wikipedia_Entities_ICCV_2023_paper.html", + dataset={ + "path": "MRBench/mbeir_oven_task6", + "revision": "2192074af29422bc1dc41cf07936f198b8c69bd0", + }, + type="Retrieval", + category="it2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2023-01-01", "2023-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{hu2023open, + title={Open-domain visual entity recognition: Towards recognizing millions of wikipedia entities}, + author={Hu, Hexiang and Luan, Yi and Chen, Yang and Khandelwal, Urvashi and Joshi, Mandar and Lee, Kenton and Toutanova, Kristina and Chang, Ming-Wei}, + booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision}, + pages={12065--12075}, + year={2023} +}""", + descriptive_stats={ + "n_samples": {"test": 50004}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 676667, + "num_queries": 50004, + "average_relevant_docs_per_query": 9.9, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py new file mode 100644 index 0000000000..4bea1f5464 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class WebQAT2ITRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="WebQAT2ITRetrieval", + description="Retrieve sources of information based on questions.", + reference="https://openaccess.thecvf.com/content/CVPR2022/html/Chang_WebQA_Multihop_and_Multimodal_QA_CVPR_2022_paper.html", + dataset={ + "path": "MRBench/mbeir_webqa_task2", + "revision": "53db4c9f9c93cb74926a1c9d04dea7d7acac2f21", + }, + type="Retrieval", + category="t2it", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2022-01-01", "2022-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@inproceedings{chang2022webqa, + title={Webqa: Multihop and multimodal qa}, + author={Chang, Yingshan and Narang, Mridu and Suzuki, Hisami and Cao, Guihong and Gao, Jianfeng and Bisk, Yonatan}, + booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, + pages={16495--16504}, + year={2022} + }""", + descriptive_stats={ + "n_samples": {"test": 2511}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 403196, + "num_queries": 2511, + "average_relevant_docs_per_query": 1.4, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py new file mode 100644 index 0000000000..6a5b28c0f3 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class WebQAT2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="WebQAT2TRetrieval", + description="Retrieve sources of information based on questions.", + reference="https://openaccess.thecvf.com/content/CVPR2022/html/Chang_WebQA_Multihop_and_Multimodal_QA_CVPR_2022_paper.html", + dataset={ + "path": "MRBench/mbeir_webqa_task1", + "revision": "468b42a2b2e767d80d2d93f5ae5d42f135a10478", + }, + type="Retrieval", + category="s2p", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2022-01-01", "2022-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text"], + sample_creation="created", + bibtex_citation="""@inproceedings{chang2022webqa, + title={Webqa: Multihop and multimodal qa}, + author={Chang, Yingshan and Narang, Mridu and Suzuki, Hisami and Cao, Guihong and Gao, Jianfeng and Bisk, Yonatan}, + booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, + pages={16495--16504}, + year={2022} + }""", + descriptive_stats={ + "n_samples": {"test": 2455}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 544457, + "num_queries": 2455, + "average_relevant_docs_per_query": 2.0, + } + }, + }, + ) diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json new file mode 100644 index 0000000000..06fb590415 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", + "evaluation_time": 140.29364442825317, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.23006, + "map_at_1": 0.07736, + "map_at_10": 0.16639, + "map_at_100": 0.18754, + "map_at_1000": 0.18791, + "map_at_20": 0.17939, + "map_at_3": 0.12469, + "map_at_5": 0.14561, + "mrr_at_1": 0.07735849056603773, + "mrr_at_10": 0.16638982479784364, + "mrr_at_100": 0.18754037945219226, + "mrr_at_1000": 0.18790557367103924, + "mrr_at_20": 0.179393828269821, + "mrr_at_3": 0.12468553459119512, + "mrr_at_5": 0.14560534591194946, + "nauc_map_at_1000_diff1": 0.07235874903557624, + "nauc_map_at_1000_max": 0.0012495691831918834, + "nauc_map_at_1000_std": -0.04266110409531828, + "nauc_map_at_100_diff1": 0.07206445308000643, + "nauc_map_at_100_max": 0.0019260185802715023, + "nauc_map_at_100_std": -0.04168632470080018, + "nauc_map_at_10_diff1": 0.07403831515427159, + "nauc_map_at_10_max": -0.0046099372581098605, + "nauc_map_at_10_std": -0.049713387866956435, + "nauc_map_at_1_diff1": 0.10866321569038064, + "nauc_map_at_1_max": -0.04068530891909402, + "nauc_map_at_1_std": -0.09376533797535455, + "nauc_map_at_20_diff1": 0.07037909843337932, + "nauc_map_at_20_max": 0.00034557195128515213, + "nauc_map_at_20_std": -0.04702226280108443, + "nauc_map_at_3_diff1": 0.09486061153270883, + "nauc_map_at_3_max": -0.02683999938507012, + "nauc_map_at_3_std": -0.06125207210727061, + "nauc_map_at_5_diff1": 0.09173894865921334, + "nauc_map_at_5_max": -0.019068286969482498, + "nauc_map_at_5_std": -0.055410455100472364, + "nauc_mrr_at_1000_diff1": 0.07235875136193966, + "nauc_mrr_at_1000_max": 0.0012495773151879567, + "nauc_mrr_at_1000_std": -0.042661096670129864, + "nauc_mrr_at_100_diff1": 0.07206445308000643, + "nauc_mrr_at_100_max": 0.0019260185802715023, + "nauc_mrr_at_100_std": -0.04168632470080018, + "nauc_mrr_at_10_diff1": 0.07403831515427159, + "nauc_mrr_at_10_max": -0.0046099372581098605, + "nauc_mrr_at_10_std": -0.049713387866956435, + "nauc_mrr_at_1_diff1": 0.10866321569038064, + "nauc_mrr_at_1_max": -0.04068530891909402, + "nauc_mrr_at_1_std": -0.09376533797535455, + "nauc_mrr_at_20_diff1": 0.07037909843337932, + "nauc_mrr_at_20_max": 0.00034557195128515213, + "nauc_mrr_at_20_std": -0.04702226280108443, + "nauc_mrr_at_3_diff1": 0.09486061153270883, + "nauc_mrr_at_3_max": -0.02683999938507012, + "nauc_mrr_at_3_std": -0.06125207210727061, + "nauc_mrr_at_5_diff1": 0.09173894865921334, + "nauc_mrr_at_5_max": -0.019068286969482498, + "nauc_mrr_at_5_std": -0.055410455100472364, + "nauc_ndcg_at_1000_diff1": 0.056957712662548704, + "nauc_ndcg_at_1000_max": 0.028286697913508196, + "nauc_ndcg_at_1000_std": -0.012334573800565744, + "nauc_ndcg_at_100_diff1": 0.049742644817624616, + "nauc_ndcg_at_100_max": 0.047234136999433886, + "nauc_ndcg_at_100_std": 0.01417643393391879, + "nauc_ndcg_at_10_diff1": 0.050417918199136984, + "nauc_ndcg_at_10_max": 0.01942596425857416, + "nauc_ndcg_at_10_std": -0.03280260244397793, + "nauc_ndcg_at_1_diff1": 0.10866321569038064, + "nauc_ndcg_at_1_max": -0.04068530891909402, + "nauc_ndcg_at_1_std": -0.09376533797535455, + "nauc_ndcg_at_20_diff1": 0.03740429843564625, + "nauc_ndcg_at_20_max": 0.03803214514992617, + "nauc_ndcg_at_20_std": -0.021565488128833982, + "nauc_ndcg_at_3_diff1": 0.09090936216756011, + "nauc_ndcg_at_3_max": -0.02488929764403297, + "nauc_ndcg_at_3_std": -0.053688777047987864, + "nauc_ndcg_at_5_diff1": 0.08633721570987683, + "nauc_ndcg_at_5_max": -0.01238793829885936, + "nauc_ndcg_at_5_std": -0.04527442021920161, + "nauc_precision_at_1000_diff1": -0.03953149887405619, + "nauc_precision_at_1000_max": 0.507263689789647, + "nauc_precision_at_1000_std": 0.7267946394243914, + "nauc_precision_at_100_diff1": -0.08156595971721974, + "nauc_precision_at_100_max": 0.4456795063704518, + "nauc_precision_at_100_std": 0.5793601391173772, + "nauc_precision_at_10_diff1": 0.00024134988468130694, + "nauc_precision_at_10_max": 0.07145495807777137, + "nauc_precision_at_10_std": 0.0010552900570005093, + "nauc_precision_at_1_diff1": 0.10866321569038064, + "nauc_precision_at_1_max": -0.04068530891909402, + "nauc_precision_at_1_std": -0.09376533797535455, + "nauc_precision_at_20_diff1": -0.04957649222390242, + "nauc_precision_at_20_max": 0.140936432573259, + "nauc_precision_at_20_std": 0.04479103118396489, + "nauc_precision_at_3_diff1": 0.08256318842375679, + "nauc_precision_at_3_max": -0.021108881570062847, + "nauc_precision_at_3_std": -0.038007584676221444, + "nauc_precision_at_5_diff1": 0.07592926310375633, + "nauc_precision_at_5_max": 0.0008620050605078056, + "nauc_precision_at_5_std": -0.026056300607536234, + "nauc_recall_at_1000_diff1": -0.03953149887405362, + "nauc_recall_at_1000_max": 0.5072636897896413, + "nauc_recall_at_1000_std": 0.7267946394243926, + "nauc_recall_at_100_diff1": -0.0815659597172199, + "nauc_recall_at_100_max": 0.4456795063704503, + "nauc_recall_at_100_std": 0.5793601391173749, + "nauc_recall_at_10_diff1": 0.00024134988468162648, + "nauc_recall_at_10_max": 0.0714549580777712, + "nauc_recall_at_10_std": 0.001055290057000327, + "nauc_recall_at_1_diff1": 0.10866321569038064, + "nauc_recall_at_1_max": -0.04068530891909402, + "nauc_recall_at_1_std": -0.09376533797535455, + "nauc_recall_at_20_diff1": -0.04957649222390187, + "nauc_recall_at_20_max": 0.1409364325732594, + "nauc_recall_at_20_std": 0.04479103118396537, + "nauc_recall_at_3_diff1": 0.08256318842375683, + "nauc_recall_at_3_max": -0.021108881570062934, + "nauc_recall_at_3_std": -0.03800758467622141, + "nauc_recall_at_5_diff1": 0.07592926310375617, + "nauc_recall_at_5_max": 0.0008620050605076114, + "nauc_recall_at_5_std": -0.026056300607536588, + "ndcg_at_1": 0.07736, + "ndcg_at_10": 0.23006, + "ndcg_at_100": 0.33167, + "ndcg_at_1000": 0.34104, + "ndcg_at_20": 0.2775, + "ndcg_at_3": 0.14127, + "ndcg_at_5": 0.17927, + "precision_at_1": 0.07736, + "precision_at_10": 0.04406, + "precision_at_100": 0.00911, + "precision_at_1000": 0.00098, + "precision_at_20": 0.03142, + "precision_at_3": 0.06321, + "precision_at_5": 0.05651, + "recall_at_1": 0.07736, + "recall_at_10": 0.44057, + "recall_at_100": 0.91085, + "recall_at_1000": 0.98396, + "recall_at_20": 0.6283, + "recall_at_3": 0.18962, + "recall_at_5": 0.28255 + } + ] + }, + "task_name": "NIGHTSI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2ITRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2ITRetrieval.json new file mode 100644 index 0000000000..5bf8b100e7 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2ITRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "350d14b7258189654e26a2be93dc0bd6bee09b76", + "evaluation_time": 1292.0344169139862, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.035, + "map_at_1": 0.01112, + "map_at_10": 0.01862, + "map_at_100": 0.02022, + "map_at_1000": 0.02041, + "map_at_20": 0.01946, + "map_at_3": 0.0157, + "map_at_5": 0.0173, + "mrr_at_1": 0.04097415372091446, + "mrr_at_10": 0.06899569067162872, + "mrr_at_100": 0.0753954947792008, + "mrr_at_1000": 0.07629663251285479, + "mrr_at_20": 0.07228274812120399, + "mrr_at_3": 0.05770752775704968, + "mrr_at_5": 0.06368066390792163, + "nauc_map_at_1000_diff1": 0.18464620105157786, + "nauc_map_at_1000_max": 0.09517461783393011, + "nauc_map_at_1000_std": 0.2040040891002083, + "nauc_map_at_100_diff1": 0.18507972397574288, + "nauc_map_at_100_max": 0.09527206757614491, + "nauc_map_at_100_std": 0.20298383306893306, + "nauc_map_at_10_diff1": 0.19701268529905605, + "nauc_map_at_10_max": 0.10283536792759419, + "nauc_map_at_10_std": 0.19184524055330834, + "nauc_map_at_1_diff1": 0.29584333547408176, + "nauc_map_at_1_max": 0.0916453292778421, + "nauc_map_at_1_std": 0.158857181905166, + "nauc_map_at_20_diff1": 0.18875941083901676, + "nauc_map_at_20_max": 0.09781871339391238, + "nauc_map_at_20_std": 0.1973870257544205, + "nauc_map_at_3_diff1": 0.22949511019464636, + "nauc_map_at_3_max": 0.11329425489178281, + "nauc_map_at_3_std": 0.18181067833411912, + "nauc_map_at_5_diff1": 0.20746294345468097, + "nauc_map_at_5_max": 0.1090653010040002, + "nauc_map_at_5_std": 0.18710071896768143, + "nauc_mrr_at_1000_diff1": 0.20282098428667883, + "nauc_mrr_at_1000_max": 0.06325911499231403, + "nauc_mrr_at_1000_std": 0.26265911763626243, + "nauc_mrr_at_100_diff1": 0.20294920425657323, + "nauc_mrr_at_100_max": 0.06321419720347958, + "nauc_mrr_at_100_std": 0.26095342846451874, + "nauc_mrr_at_10_diff1": 0.21082571157251911, + "nauc_mrr_at_10_max": 0.06496216667032441, + "nauc_mrr_at_10_std": 0.2456448673566505, + "nauc_mrr_at_1_diff1": 0.2888956197536302, + "nauc_mrr_at_1_max": 0.051945770424599746, + "nauc_mrr_at_1_std": 0.19366663227841058, + "nauc_mrr_at_20_diff1": 0.20578680515172237, + "nauc_mrr_at_20_max": 0.0643691629331365, + "nauc_mrr_at_20_std": 0.2526530831331751, + "nauc_mrr_at_3_diff1": 0.22957524087977552, + "nauc_mrr_at_3_max": 0.06920027956142867, + "nauc_mrr_at_3_std": 0.22811278430845391, + "nauc_mrr_at_5_diff1": 0.2164768879713726, + "nauc_mrr_at_5_max": 0.06709195985714196, + "nauc_mrr_at_5_std": 0.23643709402910953, + "nauc_ndcg_at_1000_diff1": 0.12300707877619335, + "nauc_ndcg_at_1000_max": 0.06750648751687409, + "nauc_ndcg_at_1000_std": 0.25758578795921744, + "nauc_ndcg_at_100_diff1": 0.131778013283211, + "nauc_ndcg_at_100_max": 0.06705475282373502, + "nauc_ndcg_at_100_std": 0.24966571751929542, + "nauc_ndcg_at_10_diff1": 0.1762416783033131, + "nauc_ndcg_at_10_max": 0.08358197089345713, + "nauc_ndcg_at_10_std": 0.23310907898873637, + "nauc_ndcg_at_1_diff1": 0.2888956197536302, + "nauc_ndcg_at_1_max": 0.051945770424599746, + "nauc_ndcg_at_1_std": 0.19366663227841058, + "nauc_ndcg_at_20_diff1": 0.15417408875873417, + "nauc_ndcg_at_20_max": 0.07760562462777675, + "nauc_ndcg_at_20_std": 0.23462305257844196, + "nauc_ndcg_at_3_diff1": 0.2171101478271009, + "nauc_ndcg_at_3_max": 0.08391708221792189, + "nauc_ndcg_at_3_std": 0.22973205684239975, + "nauc_ndcg_at_5_diff1": 0.19286258630447398, + "nauc_ndcg_at_5_max": 0.0858785578759677, + "nauc_ndcg_at_5_std": 0.23456022322849668, + "nauc_precision_at_1000_diff1": 0.09567903231176829, + "nauc_precision_at_1000_max": 0.0572561397518354, + "nauc_precision_at_1000_std": 0.565307711225196, + "nauc_precision_at_100_diff1": 0.10342329710952293, + "nauc_precision_at_100_max": 0.04018894578484745, + "nauc_precision_at_100_std": 0.4354605906817404, + "nauc_precision_at_10_diff1": 0.14942098424596895, + "nauc_precision_at_10_max": 0.06288408623104451, + "nauc_precision_at_10_std": 0.2973117922065519, + "nauc_precision_at_1_diff1": 0.2888956197536302, + "nauc_precision_at_1_max": 0.051945770424599746, + "nauc_precision_at_1_std": 0.19366663227841058, + "nauc_precision_at_20_diff1": 0.12836410396823858, + "nauc_precision_at_20_max": 0.059383711439420894, + "nauc_precision_at_20_std": 0.3294161555092745, + "nauc_precision_at_3_diff1": 0.1858984302659261, + "nauc_precision_at_3_max": 0.08228617635694004, + "nauc_precision_at_3_std": 0.2525153498411853, + "nauc_precision_at_5_diff1": 0.15962892065976547, + "nauc_precision_at_5_max": 0.07259812245110689, + "nauc_precision_at_5_std": 0.2693630038131517, + "nauc_recall_at_1000_diff1": 0.03339051340747988, + "nauc_recall_at_1000_max": 0.027844058563400814, + "nauc_recall_at_1000_std": 0.22858279511428947, + "nauc_recall_at_100_diff1": 0.05325867308988818, + "nauc_recall_at_100_max": 0.029638320585807856, + "nauc_recall_at_100_std": 0.25965889999647557, + "nauc_recall_at_10_diff1": 0.11051583333025253, + "nauc_recall_at_10_max": 0.08798717299067295, + "nauc_recall_at_10_std": 0.21440114661530302, + "nauc_recall_at_1_diff1": 0.29584333547408176, + "nauc_recall_at_1_max": 0.0916453292778421, + "nauc_recall_at_1_std": 0.158857181905166, + "nauc_recall_at_20_diff1": 0.07959909782666622, + "nauc_recall_at_20_max": 0.060615013502609366, + "nauc_recall_at_20_std": 0.23701031660389618, + "nauc_recall_at_3_diff1": 0.18117525603122883, + "nauc_recall_at_3_max": 0.12812464133898993, + "nauc_recall_at_3_std": 0.19577533427777125, + "nauc_recall_at_5_diff1": 0.1329832130842022, + "nauc_recall_at_5_max": 0.11245638518254758, + "nauc_recall_at_5_std": 0.2052166509941484, + "ndcg_at_1": 0.04097, + "ndcg_at_10": 0.035, + "ndcg_at_100": 0.0428, + "ndcg_at_1000": 0.05186, + "ndcg_at_20": 0.03662, + "ndcg_at_3": 0.03592, + "ndcg_at_5": 0.03512, + "precision_at_1": 0.04097, + "precision_at_10": 0.0147, + "precision_at_100": 0.00326, + "precision_at_1000": 0.00057, + "precision_at_20": 0.00972, + "precision_at_3": 0.02684, + "precision_at_5": 0.02136, + "recall_at_1": 0.01112, + "recall_at_10": 0.0387, + "recall_at_100": 0.08325, + "recall_at_1000": 0.13449, + "recall_at_20": 0.05102, + "recall_at_3": 0.02186, + "recall_at_5": 0.02891 + } + ] + }, + "task_name": "OVENIT2ITRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2TRetrieval.json new file mode 100644 index 0000000000..31fc4f04f4 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "2192074af29422bc1dc41cf07936f198b8c69bd0", + "evaluation_time": 1350.504133939743, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.00094, + "map_at_1": 0.00021, + "map_at_10": 0.0006, + "map_at_100": 0.00081, + "map_at_1000": 0.0009, + "map_at_20": 0.00068, + "map_at_3": 0.00046, + "map_at_5": 0.00052, + "mrr_at_1": 0.0002599792016638669, + "mrr_at_10": 0.0008833817104155483, + "mrr_at_100": 0.0012310911072741933, + "mrr_at_1000": 0.0013783242698433686, + "mrr_at_20": 0.0010165745852014807, + "mrr_at_3": 0.0006266165373436792, + "mrr_at_5": 0.0007476068581180172, + "nauc_map_at_1000_diff1": 0.3356025486850719, + "nauc_map_at_1000_max": -0.4169882172024213, + "nauc_map_at_1000_std": 0.14538374737049836, + "nauc_map_at_100_diff1": 0.3734562909349873, + "nauc_map_at_100_max": -0.43756259083510546, + "nauc_map_at_100_std": 0.14703254584037814, + "nauc_map_at_10_diff1": 0.4757961007140568, + "nauc_map_at_10_max": -0.457305835140058, + "nauc_map_at_10_std": 0.12131750979365002, + "nauc_map_at_1_diff1": 0.447023862894289, + "nauc_map_at_1_max": -0.5350799140255992, + "nauc_map_at_1_std": -0.1337349259821173, + "nauc_map_at_20_diff1": 0.4261599927770859, + "nauc_map_at_20_max": -0.4553914497074392, + "nauc_map_at_20_std": 0.14240410328361042, + "nauc_map_at_3_diff1": 0.48316789955802897, + "nauc_map_at_3_max": -0.44933732920300423, + "nauc_map_at_3_std": 0.07520438823121707, + "nauc_map_at_5_diff1": 0.4953864048755024, + "nauc_map_at_5_max": -0.45095276202437423, + "nauc_map_at_5_std": 0.08797594420893395, + "nauc_mrr_at_1000_diff1": 0.3233793837746254, + "nauc_mrr_at_1000_max": -0.40215331933996257, + "nauc_mrr_at_1000_std": 0.13645754139971128, + "nauc_mrr_at_100_diff1": 0.3631435594964905, + "nauc_mrr_at_100_max": -0.42486989417384147, + "nauc_mrr_at_100_std": 0.1353533277431553, + "nauc_mrr_at_10_diff1": 0.4571705267558742, + "nauc_mrr_at_10_max": -0.4452088246780154, + "nauc_mrr_at_10_std": 0.09358311806660385, + "nauc_mrr_at_1_diff1": 0.4390801123428319, + "nauc_mrr_at_1_max": -0.5385950516645776, + "nauc_mrr_at_1_std": -0.1013774439293419, + "nauc_mrr_at_20_diff1": 0.4123398002197073, + "nauc_mrr_at_20_max": -0.4436660692753614, + "nauc_mrr_at_20_std": 0.11881121462963219, + "nauc_mrr_at_3_diff1": 0.4566504542958695, + "nauc_mrr_at_3_max": -0.4392625768386175, + "nauc_mrr_at_3_std": 0.032253694253006206, + "nauc_mrr_at_5_diff1": 0.48230049629222144, + "nauc_mrr_at_5_max": -0.4428045340735895, + "nauc_mrr_at_5_std": 0.05184288616601152, + "nauc_ndcg_at_1000_diff1": 0.0804865683285438, + "nauc_ndcg_at_1000_max": -0.2846453341018818, + "nauc_ndcg_at_1000_std": 0.14182083062836548, + "nauc_ndcg_at_100_diff1": 0.22666071609718713, + "nauc_ndcg_at_100_max": -0.39369103560568086, + "nauc_ndcg_at_100_std": 0.17908268911241257, + "nauc_ndcg_at_10_diff1": 0.4669999755097327, + "nauc_ndcg_at_10_max": -0.4527155457158219, + "nauc_ndcg_at_10_std": 0.16289182127074892, + "nauc_ndcg_at_1_diff1": 0.4390801123428319, + "nauc_ndcg_at_1_max": -0.5385950516645776, + "nauc_ndcg_at_1_std": -0.1013774439293419, + "nauc_ndcg_at_20_diff1": 0.3635078941824615, + "nauc_ndcg_at_20_max": -0.44848318122661257, + "nauc_ndcg_at_20_std": 0.20109287009889576, + "nauc_ndcg_at_3_diff1": 0.47922450040931114, + "nauc_ndcg_at_3_max": -0.4322498148581518, + "nauc_ndcg_at_3_std": 0.08155045223711481, + "nauc_ndcg_at_5_diff1": 0.503620610286048, + "nauc_ndcg_at_5_max": -0.44037040269154254, + "nauc_ndcg_at_5_std": 0.10619497578100863, + "nauc_precision_at_1000_diff1": 0.019035664196766656, + "nauc_precision_at_1000_max": -0.22871440079272565, + "nauc_precision_at_1000_std": 0.14038857900152193, + "nauc_precision_at_100_diff1": 0.1566858798466843, + "nauc_precision_at_100_max": -0.3579552164851941, + "nauc_precision_at_100_std": 0.19897068898487488, + "nauc_precision_at_10_diff1": 0.4468305221392101, + "nauc_precision_at_10_max": -0.44023350004181505, + "nauc_precision_at_10_std": 0.18697492500703727, + "nauc_precision_at_1_diff1": 0.4390801123428319, + "nauc_precision_at_1_max": -0.5385950516645776, + "nauc_precision_at_1_std": -0.1013774439293419, + "nauc_precision_at_20_diff1": 0.3048328179348221, + "nauc_precision_at_20_max": -0.4341435816581416, + "nauc_precision_at_20_std": 0.23561115232580918, + "nauc_precision_at_3_diff1": 0.47986106476963836, + "nauc_precision_at_3_max": -0.4099968370570377, + "nauc_precision_at_3_std": 0.08388081246556922, + "nauc_precision_at_5_diff1": 0.5257145247079027, + "nauc_precision_at_5_max": -0.42785043345887885, + "nauc_precision_at_5_std": 0.11188569056651354, + "nauc_recall_at_1000_diff1": 0.01763504437122254, + "nauc_recall_at_1000_max": -0.25041869469339395, + "nauc_recall_at_1000_std": 0.12900976977544382, + "nauc_recall_at_100_diff1": 0.13782698515403186, + "nauc_recall_at_100_max": -0.3683545629769233, + "nauc_recall_at_100_std": 0.18322328259085513, + "nauc_recall_at_10_diff1": 0.46517127103809264, + "nauc_recall_at_10_max": -0.4578770140642293, + "nauc_recall_at_10_std": 0.2376092789995941, + "nauc_recall_at_1_diff1": 0.447023862894289, + "nauc_recall_at_1_max": -0.5350799140255992, + "nauc_recall_at_1_std": -0.1337349259821173, + "nauc_recall_at_20_diff1": 0.29085723394404345, + "nauc_recall_at_20_max": -0.44744217769546446, + "nauc_recall_at_20_std": 0.2712484276298942, + "nauc_recall_at_3_diff1": 0.5196230856918127, + "nauc_recall_at_3_max": -0.4244201041308224, + "nauc_recall_at_3_std": 0.16691297737481914, + "nauc_recall_at_5_diff1": 0.5373957061055848, + "nauc_recall_at_5_max": -0.4355135518338287, + "nauc_recall_at_5_std": 0.17644746841227857, + "ndcg_at_1": 0.00026, + "ndcg_at_10": 0.00094, + "ndcg_at_100": 0.00245, + "ndcg_at_1000": 0.00603, + "ndcg_at_20": 0.00126, + "ndcg_at_3": 0.00061, + "ndcg_at_5": 0.00074, + "precision_at_1": 0.00026, + "precision_at_10": 0.00027, + "precision_at_100": 0.00015, + "precision_at_1000": 6e-05, + "precision_at_20": 0.00023, + "precision_at_3": 0.00037, + "precision_at_5": 0.00034, + "recall_at_1": 0.00021, + "recall_at_10": 0.00166, + "recall_at_100": 0.00893, + "recall_at_1000": 0.03645, + "recall_at_20": 0.0028, + "recall_at_3": 0.00079, + "recall_at_5": 0.00109 + } + ] + }, + "task_name": "OVENIT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2ITRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2ITRetrieval.json new file mode 100644 index 0000000000..4c6ba8a3eb --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2ITRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "53db4c9f9c93cb74926a1c9d04dea7d7acac2f21", + "evaluation_time": 1701.370358467102, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.24052, + "map_at_1": 0.13361, + "map_at_10": 0.19599, + "map_at_100": 0.20547, + "map_at_1000": 0.20624, + "map_at_20": 0.20136, + "map_at_3": 0.17145, + "map_at_5": 0.18571, + "mrr_at_1": 0.16367980884109917, + "mrr_at_10": 0.23035169065634997, + "mrr_at_100": 0.2389849253733174, + "mrr_at_1000": 0.23958364576706842, + "mrr_at_20": 0.2352058042930738, + "mrr_at_3": 0.20483207221558458, + "mrr_at_5": 0.21974644895791826, + "nauc_map_at_1000_diff1": 0.36381862473180293, + "nauc_map_at_1000_max": -0.059600161024810354, + "nauc_map_at_1000_std": 0.13655474912550186, + "nauc_map_at_100_diff1": 0.36367553777720224, + "nauc_map_at_100_max": -0.059461863914186626, + "nauc_map_at_100_std": 0.13590446748202067, + "nauc_map_at_10_diff1": 0.36720760002091996, + "nauc_map_at_10_max": -0.05534352171819153, + "nauc_map_at_10_std": 0.12347691221697427, + "nauc_map_at_1_diff1": 0.45326896661199906, + "nauc_map_at_1_max": -0.02764769512868723, + "nauc_map_at_1_std": 0.0652048653787392, + "nauc_map_at_20_diff1": 0.36423569015898755, + "nauc_map_at_20_max": -0.05757039949977667, + "nauc_map_at_20_std": 0.12988237600501815, + "nauc_map_at_3_diff1": 0.3880170796310493, + "nauc_map_at_3_max": -0.046602902370452, + "nauc_map_at_3_std": 0.09557679001411416, + "nauc_map_at_5_diff1": 0.3737522906429374, + "nauc_map_at_5_max": -0.05285458276228204, + "nauc_map_at_5_std": 0.10793344879301892, + "nauc_mrr_at_1000_diff1": 0.3573332666479495, + "nauc_mrr_at_1000_max": -0.08009727757992803, + "nauc_mrr_at_1000_std": 0.13850009234942326, + "nauc_mrr_at_100_diff1": 0.35730299243761976, + "nauc_mrr_at_100_max": -0.08005647033421684, + "nauc_mrr_at_100_std": 0.13832388992979938, + "nauc_mrr_at_10_diff1": 0.3588306944358961, + "nauc_mrr_at_10_max": -0.0765992981251009, + "nauc_mrr_at_10_std": 0.1311338745029867, + "nauc_mrr_at_1_diff1": 0.4365726498076461, + "nauc_mrr_at_1_max": -0.0487958782703477, + "nauc_mrr_at_1_std": 0.07704304358420916, + "nauc_mrr_at_20_diff1": 0.357610013030439, + "nauc_mrr_at_20_max": -0.07903837125426354, + "nauc_mrr_at_20_std": 0.13393741454859437, + "nauc_mrr_at_3_diff1": 0.37733345474150354, + "nauc_mrr_at_3_max": -0.07021652088221061, + "nauc_mrr_at_3_std": 0.10434506999102883, + "nauc_mrr_at_5_diff1": 0.3642816751404842, + "nauc_mrr_at_5_max": -0.07777365989547423, + "nauc_mrr_at_5_std": 0.1146242099830958, + "nauc_ndcg_at_1000_diff1": 0.3236304186893325, + "nauc_ndcg_at_1000_max": -0.09260496608113483, + "nauc_ndcg_at_1000_std": 0.22257429357826702, + "nauc_ndcg_at_100_diff1": 0.32154613705297064, + "nauc_ndcg_at_100_max": -0.08750441934383102, + "nauc_ndcg_at_100_std": 0.2125151063859874, + "nauc_ndcg_at_10_diff1": 0.33334659337391326, + "nauc_ndcg_at_10_max": -0.07039991614529735, + "nauc_ndcg_at_10_std": 0.16000053754583443, + "nauc_ndcg_at_1_diff1": 0.43899642735143635, + "nauc_ndcg_at_1_max": -0.047721861162555626, + "nauc_ndcg_at_1_std": 0.07437915878107718, + "nauc_ndcg_at_20_diff1": 0.3255145799542806, + "nauc_ndcg_at_20_max": -0.07740934864013048, + "nauc_ndcg_at_20_std": 0.17623988789589784, + "nauc_ndcg_at_3_diff1": 0.3683119776482275, + "nauc_ndcg_at_3_max": -0.058131293476559404, + "nauc_ndcg_at_3_std": 0.10461441678807436, + "nauc_ndcg_at_5_diff1": 0.3469661061624626, + "nauc_ndcg_at_5_max": -0.0682112902662825, + "nauc_ndcg_at_5_std": 0.12503579582342164, + "nauc_precision_at_1000_diff1": 0.0828430329174661, + "nauc_precision_at_1000_max": -0.159449938520161, + "nauc_precision_at_1000_std": 0.4042474323720823, + "nauc_precision_at_100_diff1": 0.1296372825312727, + "nauc_precision_at_100_max": -0.14496151170022903, + "nauc_precision_at_100_std": 0.3906087071646669, + "nauc_precision_at_10_diff1": 0.22409492077465515, + "nauc_precision_at_10_max": -0.10925589146862036, + "nauc_precision_at_10_std": 0.2540176909879479, + "nauc_precision_at_1_diff1": 0.43899642735143635, + "nauc_precision_at_1_max": -0.047721861162555626, + "nauc_precision_at_1_std": 0.07437915878107718, + "nauc_precision_at_20_diff1": 0.18273565087641105, + "nauc_precision_at_20_max": -0.12370069538638741, + "nauc_precision_at_20_std": 0.28690928298508195, + "nauc_precision_at_3_diff1": 0.30938374042197886, + "nauc_precision_at_3_max": -0.08990275443773689, + "nauc_precision_at_3_std": 0.13718837605008682, + "nauc_precision_at_5_diff1": 0.26069624103883665, + "nauc_precision_at_5_max": -0.11012349271655618, + "nauc_precision_at_5_std": 0.17092390048598666, + "nauc_recall_at_1000_diff1": 0.17306295937133223, + "nauc_recall_at_1000_max": -0.2040107326619824, + "nauc_recall_at_1000_std": 0.5680970730286881, + "nauc_recall_at_100_diff1": 0.18974615129600783, + "nauc_recall_at_100_max": -0.14329014714781005, + "nauc_recall_at_100_std": 0.4331187602298659, + "nauc_recall_at_10_diff1": 0.24572008013380212, + "nauc_recall_at_10_max": -0.0817630066289336, + "nauc_recall_at_10_std": 0.23340550568856921, + "nauc_recall_at_1_diff1": 0.45326896661199906, + "nauc_recall_at_1_max": -0.02764769512868723, + "nauc_recall_at_1_std": 0.0652048653787392, + "nauc_recall_at_20_diff1": 0.22026766806324424, + "nauc_recall_at_20_max": -0.10008085428330954, + "nauc_recall_at_20_std": 0.2763531788296686, + "nauc_recall_at_3_diff1": 0.3237039549334668, + "nauc_recall_at_3_max": -0.06308781198222387, + "nauc_recall_at_3_std": 0.11812305027912111, + "nauc_recall_at_5_diff1": 0.27886114226778713, + "nauc_recall_at_5_max": -0.07965627595402587, + "nauc_recall_at_5_std": 0.15601728348224048, + "ndcg_at_1": 0.16328, + "ndcg_at_10": 0.24052, + "ndcg_at_100": 0.28761, + "ndcg_at_1000": 0.30845, + "ndcg_at_20": 0.25886, + "ndcg_at_3": 0.19115, + "ndcg_at_5": 0.21613, + "precision_at_1": 0.16328, + "precision_at_10": 0.04456, + "precision_at_100": 0.00744, + "precision_at_1000": 0.00098, + "precision_at_20": 0.02708, + "precision_at_3": 0.09067, + "precision_at_5": 0.06969, + "recall_at_1": 0.13361, + "recall_at_10": 0.3409, + "recall_at_100": 0.55078, + "recall_at_1000": 0.70291, + "recall_at_20": 0.40721, + "recall_at_3": 0.21545, + "recall_at_5": 0.27141 + } + ] + }, + "task_name": "WebQAT2ITRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2TRetrieval.json new file mode 100644 index 0000000000..58915fa1e1 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "468b42a2b2e767d80d2d93f5ae5d42f135a10478", + "evaluation_time": 318.56735610961914, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.21064, + "map_at_1": 0.0961, + "map_at_10": 0.15535, + "map_at_100": 0.16454, + "map_at_1000": 0.16555, + "map_at_20": 0.16023, + "map_at_3": 0.13528, + "map_at_5": 0.14522, + "mrr_at_1": 0.19389002036659878, + "mrr_at_10": 0.2656623993793037, + "mrr_at_100": 0.27305022327360007, + "mrr_at_1000": 0.2736181031441951, + "mrr_at_20": 0.26991093217110423, + "mrr_at_3": 0.24372029871011533, + "mrr_at_5": 0.2552274270196877, + "nauc_map_at_1000_diff1": 0.3577072821087506, + "nauc_map_at_1000_max": 0.19011419837219326, + "nauc_map_at_1000_std": 0.1988610404714397, + "nauc_map_at_100_diff1": 0.3578630736764876, + "nauc_map_at_100_max": 0.189817834110483, + "nauc_map_at_100_std": 0.19750527765359563, + "nauc_map_at_10_diff1": 0.35972052505814145, + "nauc_map_at_10_max": 0.18765097035594713, + "nauc_map_at_10_std": 0.18349339512654916, + "nauc_map_at_1_diff1": 0.41104840600884934, + "nauc_map_at_1_max": 0.2132812008152571, + "nauc_map_at_1_std": 0.10415969608061279, + "nauc_map_at_20_diff1": 0.3585489861669943, + "nauc_map_at_20_max": 0.1888402615468673, + "nauc_map_at_20_std": 0.18977266178160795, + "nauc_map_at_3_diff1": 0.36352191826392466, + "nauc_map_at_3_max": 0.20092109988099888, + "nauc_map_at_3_std": 0.15110133509151957, + "nauc_map_at_5_diff1": 0.36223938564701375, + "nauc_map_at_5_max": 0.1942064221941677, + "nauc_map_at_5_std": 0.16623308333665782, + "nauc_mrr_at_1000_diff1": 0.3761532158040219, + "nauc_mrr_at_1000_max": 0.18223027885145637, + "nauc_mrr_at_1000_std": 0.17085813127223065, + "nauc_mrr_at_100_diff1": 0.3761733367847822, + "nauc_mrr_at_100_max": 0.18193717387598093, + "nauc_mrr_at_100_std": 0.17076467545929344, + "nauc_mrr_at_10_diff1": 0.375757427090388, + "nauc_mrr_at_10_max": 0.1803972311772891, + "nauc_mrr_at_10_std": 0.1648971880152693, + "nauc_mrr_at_1_diff1": 0.40955729844120853, + "nauc_mrr_at_1_max": 0.2121400747392042, + "nauc_mrr_at_1_std": 0.10527479271269759, + "nauc_mrr_at_20_diff1": 0.3755902098099904, + "nauc_mrr_at_20_max": 0.18151951134518635, + "nauc_mrr_at_20_std": 0.1678452516111775, + "nauc_mrr_at_3_diff1": 0.38254901314999595, + "nauc_mrr_at_3_max": 0.18245118202851954, + "nauc_mrr_at_3_std": 0.14070604452851126, + "nauc_mrr_at_5_diff1": 0.3795585112521173, + "nauc_mrr_at_5_max": 0.18014590457007112, + "nauc_mrr_at_5_std": 0.15327419153894142, + "nauc_ndcg_at_1000_diff1": 0.34599492975328583, + "nauc_ndcg_at_1000_max": 0.1891205366172212, + "nauc_ndcg_at_1000_std": 0.28791695656426014, + "nauc_ndcg_at_100_diff1": 0.34912910612774883, + "nauc_ndcg_at_100_max": 0.18109853020411557, + "nauc_ndcg_at_100_std": 0.26671467734778753, + "nauc_ndcg_at_10_diff1": 0.3517277732735262, + "nauc_ndcg_at_10_max": 0.17272550706941595, + "nauc_ndcg_at_10_std": 0.21315749304817078, + "nauc_ndcg_at_1_diff1": 0.40955729844120853, + "nauc_ndcg_at_1_max": 0.2121400747392042, + "nauc_ndcg_at_1_std": 0.10527479271269759, + "nauc_ndcg_at_20_diff1": 0.34901296756219047, + "nauc_ndcg_at_20_max": 0.1758313664925407, + "nauc_ndcg_at_20_std": 0.2285421938559212, + "nauc_ndcg_at_3_diff1": 0.3614228822890276, + "nauc_ndcg_at_3_max": 0.1860348188440428, + "nauc_ndcg_at_3_std": 0.15802875724589438, + "nauc_ndcg_at_5_diff1": 0.358612584433648, + "nauc_ndcg_at_5_max": 0.1783684698176462, + "nauc_ndcg_at_5_std": 0.1806675247484184, + "nauc_precision_at_1000_diff1": 0.22394818622056822, + "nauc_precision_at_1000_max": 0.1745345166780263, + "nauc_precision_at_1000_std": 0.5296236186874234, + "nauc_precision_at_100_diff1": 0.2619757063234864, + "nauc_precision_at_100_max": 0.14566238107492874, + "nauc_precision_at_100_std": 0.4219600671015512, + "nauc_precision_at_10_diff1": 0.29206784963413884, + "nauc_precision_at_10_max": 0.1326876541958677, + "nauc_precision_at_10_std": 0.28378504557731654, + "nauc_precision_at_1_diff1": 0.40955729844120853, + "nauc_precision_at_1_max": 0.2121400747392042, + "nauc_precision_at_1_std": 0.10527479271269759, + "nauc_precision_at_20_diff1": 0.2768467143465054, + "nauc_precision_at_20_max": 0.13614135922924672, + "nauc_precision_at_20_std": 0.3133650863423826, + "nauc_precision_at_3_diff1": 0.33410422371740034, + "nauc_precision_at_3_max": 0.16730664435836548, + "nauc_precision_at_3_std": 0.18585733891306555, + "nauc_precision_at_5_diff1": 0.32446847913096094, + "nauc_precision_at_5_max": 0.1506483395267056, + "nauc_precision_at_5_std": 0.22620462778951125, + "nauc_recall_at_1000_diff1": 0.22464019658813997, + "nauc_recall_at_1000_max": 0.1835604946338646, + "nauc_recall_at_1000_std": 0.5264229949341164, + "nauc_recall_at_100_diff1": 0.26141381285720783, + "nauc_recall_at_100_max": 0.14837280463323577, + "nauc_recall_at_100_std": 0.41701657242690526, + "nauc_recall_at_10_diff1": 0.2915143454748924, + "nauc_recall_at_10_max": 0.13415867035178677, + "nauc_recall_at_10_std": 0.28071339987465604, + "nauc_recall_at_1_diff1": 0.41104840600884934, + "nauc_recall_at_1_max": 0.2132812008152571, + "nauc_recall_at_1_std": 0.10415969608061279, + "nauc_recall_at_20_diff1": 0.2757155375999912, + "nauc_recall_at_20_max": 0.13846791944824233, + "nauc_recall_at_20_std": 0.3096311569155981, + "nauc_recall_at_3_diff1": 0.33382352673735427, + "nauc_recall_at_3_max": 0.16883784484659523, + "nauc_recall_at_3_std": 0.1837381252088242, + "nauc_recall_at_5_diff1": 0.3248394719989662, + "nauc_recall_at_5_max": 0.1517889072443636, + "nauc_recall_at_5_std": 0.2240342787050381, + "ndcg_at_1": 0.19389, + "ndcg_at_10": 0.21064, + "ndcg_at_100": 0.25331, + "ndcg_at_1000": 0.27728, + "ndcg_at_20": 0.22613, + "ndcg_at_3": 0.17078, + "ndcg_at_5": 0.18803, + "precision_at_1": 0.19389, + "precision_at_10": 0.05263, + "precision_at_100": 0.00872, + "precision_at_1000": 0.00119, + "precision_at_20": 0.03134, + "precision_at_3": 0.11432, + "precision_at_5": 0.08236, + "recall_at_1": 0.0961, + "recall_at_10": 0.2609, + "recall_at_100": 0.43113, + "recall_at_1000": 0.58955, + "recall_at_20": 0.31039, + "recall_at_3": 0.16999, + "recall_at_5": 0.20418 + } + ] + }, + "task_name": "WebQAT2TRetrieval" +} \ No newline at end of file From f8aaf6dca94d70c38aa1c8efa7c5bdd820f30e1f Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Wed, 7 Aug 2024 08:00:51 +0100 Subject: [PATCH 040/154] any2any retrieval chunk encoding --- .../Image/Any2AnyRetrievalEvaluator.py | 117 +++++++++--------- 1 file changed, 61 insertions(+), 56 deletions(-) diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index 7d3c26e107..933f55dd41 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -62,7 +62,7 @@ def __init__( self, model: EncoderWithQueryCorpusEncode, encode_kwargs: dict[str, Any] = {}, - corpus_chunk_size: int = 50000, + corpus_chunk_size: int = 20000, previous_results: str | None = None, **kwargs: Any, ): @@ -152,66 +152,71 @@ def search( ) ) - if corpus_modality == "text": - corpus_texts = corpus["text"] - corpus_embeddings = self.model.get_text_embeddings( - texts=corpus_texts, batch_size=self.encode_kwargs["batch_size"] - ) - else: - corpus_dataset = ImageDataset( - corpus, image_column_name="image", transform=transform - ) - corpus_image_dataloader = DataLoader( - corpus_dataset, - batch_size=self.encode_kwargs["batch_size"], - shuffle=False, - collate_fn=custom_collate_fn, - num_workers=os.cpu_count(), + result_heaps = {qid: [] for qid in query_ids} + for chunk_start in range(0, len(corpus), self.corpus_chunk_size): + chunk = corpus.select( + range( + chunk_start, min(chunk_start + self.corpus_chunk_size, len(corpus)) + ) ) - if corpus_modality == "image": - corpus_embeddings = self.model.get_image_embeddings( - images=corpus_image_dataloader, - batch_size=self.encode_kwargs["batch_size"], + chunk_ids = corpus_ids[chunk_start : chunk_start + self.corpus_chunk_size] + + if corpus_modality == "text": + corpus_texts = chunk["text"] + sub_corpus_embeddings = self.model.get_text_embeddings( + texts=corpus_texts, batch_size=self.encode_kwargs["batch_size"] ) - elif corpus_modality == "image,text": - corpus_texts = corpus["text"] - corpus_embeddings = self.model.get_fused_embeddings( - texts=corpus_texts, - images=corpus_image_dataloader, + else: + corpus_dataset = ImageDataset( + chunk, image_column_name="image", transform=transform + ) + corpus_image_dataloader = DataLoader( + corpus_dataset, batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=os.cpu_count(), ) - else: - raise ValueError(f"Unsupported modality: {corpus_modality}") - - result_heaps = { - qid: [] for qid in query_ids - } # Keep only the top-k docs for each query - - cos_scores = self.score_functions[score_function]( - query_embeddings, corpus_embeddings - ) - cos_scores[torch.isnan(cos_scores)] = -1 - - cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( - cos_scores, - top_k, - dim=1, - largest=True, - sorted=return_sorted, - ) - cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() - cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() - - for query_itr in range(len(query_embeddings)): - query_id = query_ids[query_itr] - for sub_corpus_id, score in zip( - cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] - ): - corpus_id = corpus_ids[sub_corpus_id] - if len(result_heaps[query_id]) < top_k: - heapq.heappush(result_heaps[query_id], (score, corpus_id)) + if corpus_modality == "image": + sub_corpus_embeddings = self.model.get_image_embeddings( + images=corpus_image_dataloader, + batch_size=self.encode_kwargs["batch_size"], + ) + elif corpus_modality == "image,text": + corpus_texts = chunk["text"] + sub_corpus_embeddings = self.model.get_fused_embeddings( + texts=corpus_texts, + images=corpus_image_dataloader, + batch_size=self.encode_kwargs["batch_size"], + ) else: - heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) + raise ValueError(f"Unsupported modality: {corpus_modality}") + + cos_scores = self.score_functions[score_function]( + query_embeddings, sub_corpus_embeddings + ) + cos_scores[torch.isnan(cos_scores)] = -1 + + cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( + cos_scores, + top_k, + dim=1, + largest=True, + sorted=return_sorted, + ) + cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() + cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() + + for query_itr in range(len(query_embeddings)): + query_id = query_ids[query_itr] + for sub_corpus_id, score in zip( + cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] + ): + corpus_id = chunk_ids[sub_corpus_id] + if len(result_heaps[query_id]) < top_k: + heapq.heappush(result_heaps[query_id], (score, corpus_id)) + else: + heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) for qid in result_heaps: for score, corpus_id in result_heaps[qid]: From 697991209a762e615e690811772c37e0f8a3c590 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Sat, 10 Aug 2024 02:36:41 +0100 Subject: [PATCH 041/154] add nomic vision model; any2any topk bug --- .../Image/Any2AnyRetrievalEvaluator.py | 2 +- mteb/models/__init__.py | 2 + mteb/models/nomic_models_vision.py | 165 ++++++++++++++++++ .../Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py | 4 +- .../CIRRIT2IRetrieval.json | 158 +++++++++++++++++ .../MSCOCOI2TRetrieval.json | 158 +++++++++++++++++ .../MSCOCOT2IRetrieval.json | 158 +++++++++++++++++ .../NIGHTSI2IRetrieval.json | 158 +++++++++++++++++ .../WebQAT2ITRetrieval.json | 158 +++++++++++++++++ .../WebQAT2TRetrieval.json | 158 +++++++++++++++++ .../model_meta.json | 1 + 11 files changed, 1119 insertions(+), 3 deletions(-) create mode 100644 mteb/models/nomic_models_vision.py create mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/CIRRIT2IRetrieval.json create mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOI2TRetrieval.json create mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOT2IRetrieval.json create mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/NIGHTSI2IRetrieval.json create mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2ITRetrieval.json create mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2TRetrieval.json create mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/model_meta.json diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index 933f55dd41..467a24f969 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -199,7 +199,7 @@ def search( cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( cos_scores, - top_k, + min(top_k, cos_scores.size(1)), dim=1, largest=True, sorted=return_sorted, diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index 50ae049323..8bbfc8a3cb 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -22,6 +22,7 @@ llm2vec_models, mxbai_models, nomic_models, + nomic_models_vision, openai_models, ru_sentence_models, salesforce_models, @@ -138,6 +139,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe llm2vec_models, mxbai_models, nomic_models, + nomic_models_vision, cohere_models, clip_models, openai_models, diff --git a/mteb/models/nomic_models_vision.py b/mteb/models/nomic_models_vision.py new file mode 100644 index 0000000000..6fbf478c51 --- /dev/null +++ b/mteb/models/nomic_models_vision.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +import torch.nn.functional as F +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import AutoImageProcessor, AutoModel, AutoTokenizer + +from mteb.model_meta import ModelMeta + + +class NomicVisionModelWrapper: + def __init__( + self, + vision_model_name: str, + text_model_name: str, + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.vision_model_name = vision_model_name + self.text_model_name = text_model_name + self.device = device + self.processor = AutoImageProcessor.from_pretrained(self.vision_model_name) + self.vision_model = AutoModel.from_pretrained( + self.vision_model_name, trust_remote_code=True + ).to(self.device) + self.text_model = AutoModel.from_pretrained( + self.text_model_name, trust_remote_code=True + ).to(self.device) + self.tokenizer = AutoTokenizer.from_pretrained(self.text_model_name) + + self.text_model.eval() + self.vision_model.eval() + + def preprocess( + self, + texts: list[str], + images: list[Image.Image], + ): + return self.processor( + text=texts, images=images, return_tensors="pt", padding=True + ) + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + inputs = self.tokenizer( + batch_texts, padding=True, truncation=True, return_tensors="pt" + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + text_outputs = self.text_model(**inputs) + text_embeddings = self.mean_pooling( + text_outputs, inputs["attention_mask"] + ) + text_embeddings = F.layer_norm( + text_embeddings, normalized_shape=(text_embeddings.shape[1],) + ) + text_embeddings = F.normalize(text_embeddings, p=2, dim=1) + all_text_embeddings.append(text_embeddings.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def mean_pooling(self, model_output, attention_mask): + token_embeddings = model_output[0] + input_mask_expanded = ( + attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() + ) + return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp( + input_mask_expanded.sum(1), min=1e-9 + ) + + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + with torch.no_grad(): + for batch in tqdm(images): + inputs = self.processor(images=batch, return_tensors="pt") + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.vision_model(**inputs).last_hidden_state + img_embeddings = F.normalize(image_outputs[:, 0], p=2, dim=1) + all_image_embeddings.append(img_embeddings.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + inputs = self.processor(images=batch_images, return_tensors="pt") + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.vision_model(**inputs).last_hidden_state + img_embeddings = F.normalize(image_outputs[:, 0], p=2, dim=1) + all_image_embeddings.append(img_embeddings.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + # already normalized in the encoding functions + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + else: + # to do: add other fusion mode + raise ValueError(f"fusion mode {fusion_mode} hasn't been implemented") + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + +nomic_embed_vision_v1_5 = ModelMeta( + loader=partial( + NomicVisionModelWrapper, + vision_model_name="nomic-ai/nomic-embed-vision-v1.5", + text_model_name="nomic-ai/nomic-embed-text-v1.5", + ), + name="nomic-ai/nomic-embed-vision-v1.5", + languages=["eng_Latn"], + open_source=True, + revision="af2246fffdab78d8458418480e4886a8e48b70a7", + release_date="2024-06-08", +) + +if __name__ == "__main__": + import mteb + + mdl = mteb.get_model(nomic_embed_vision_v1_5.name, nomic_embed_vision_v1_5.revision) + emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py index 8bcfd80cd6..d56b868a0a 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py @@ -5,9 +5,9 @@ from .....abstasks import AbsTaskAny2AnyRetrieval -class CIRRIT2TRetrieval(AbsTaskAny2AnyRetrieval): +class CIRRIT2IRetrieval(AbsTaskAny2AnyRetrieval): metadata = TaskMetadata( - name="CIRRIT2TRetrieval", + name="CIRRIT2IRetrieval", description="Retrieve images based on texts and images.", reference="https://openaccess.thecvf.com/content/ICCV2021/html/Liu_Image_Retrieval_on_Real-Life_Images_With_Pre-Trained_Vision-and-Language_Models_ICCV_2021_paper.html", dataset={ diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/CIRRIT2IRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/CIRRIT2IRetrieval.json new file mode 100644 index 0000000000..8bab9a555f --- /dev/null +++ b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/CIRRIT2IRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", + "evaluation_time": 133.3731541633606, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.09422, + "map_at_1": 0.00408, + "map_at_10": 0.06275, + "map_at_100": 0.07352, + "map_at_1000": 0.07488, + "map_at_20": 0.06782, + "map_at_3": 0.04482, + "map_at_5": 0.05503, + "mrr_at_1": 0.008393285371702638, + "mrr_at_10": 0.06311579307982176, + "mrr_at_100": 0.0738817410625923, + "mrr_at_1000": 0.0752359451782398, + "mrr_at_20": 0.06819202642832276, + "mrr_at_3": 0.04532374100719432, + "mrr_at_5": 0.05547961630695447, + "nauc_map_at_1000_diff1": -0.10338892563096655, + "nauc_map_at_1000_max": 0.177335800834316, + "nauc_map_at_1000_std": 0.04510016072563672, + "nauc_map_at_100_diff1": -0.10237436191846207, + "nauc_map_at_100_max": 0.1764756777393816, + "nauc_map_at_100_std": 0.04276612836249515, + "nauc_map_at_10_diff1": -0.0966658193961709, + "nauc_map_at_10_max": 0.16285956921247727, + "nauc_map_at_10_std": 0.026961252449232854, + "nauc_map_at_1_diff1": -0.5280465285121293, + "nauc_map_at_1_max": 0.08621506021244016, + "nauc_map_at_1_std": 0.18339146766836595, + "nauc_map_at_20_diff1": -0.09819954169398601, + "nauc_map_at_20_max": 0.17274426550167696, + "nauc_map_at_20_std": 0.03509542919549203, + "nauc_map_at_3_diff1": -0.1096647340125396, + "nauc_map_at_3_max": 0.14731937096492292, + "nauc_map_at_3_std": 0.02285630960394209, + "nauc_map_at_5_diff1": -0.10387404872895724, + "nauc_map_at_5_max": 0.15482607000107218, + "nauc_map_at_5_std": 0.024648861558939302, + "nauc_mrr_at_1000_diff1": -0.10575125211055578, + "nauc_mrr_at_1000_max": 0.1758407777736024, + "nauc_mrr_at_1000_std": 0.04794265702233136, + "nauc_mrr_at_100_diff1": -0.10473322024101767, + "nauc_mrr_at_100_max": 0.17499228892267552, + "nauc_mrr_at_100_std": 0.045592396596819566, + "nauc_mrr_at_10_diff1": -0.09922185003516211, + "nauc_mrr_at_10_max": 0.16145148359497702, + "nauc_mrr_at_10_std": 0.02999573674643215, + "nauc_mrr_at_1_diff1": -0.5227107209935222, + "nauc_mrr_at_1_max": 0.06995285369312734, + "nauc_mrr_at_1_std": 0.17595155046188055, + "nauc_mrr_at_20_diff1": -0.10068826746108109, + "nauc_mrr_at_20_max": 0.171302451214174, + "nauc_mrr_at_20_std": 0.03802637704632661, + "nauc_mrr_at_3_diff1": -0.11370621646983334, + "nauc_mrr_at_3_max": 0.1478294385458323, + "nauc_mrr_at_3_std": 0.028261047534815342, + "nauc_mrr_at_5_diff1": -0.1069655081332803, + "nauc_mrr_at_5_max": 0.15385012052759187, + "nauc_mrr_at_5_std": 0.028281108584287444, + "nauc_ndcg_at_1000_diff1": -0.12001070980393722, + "nauc_ndcg_at_1000_max": 0.22010392536361548, + "nauc_ndcg_at_1000_std": 0.11525705750127722, + "nauc_ndcg_at_100_diff1": -0.10703657138256953, + "nauc_ndcg_at_100_max": 0.21257636133707714, + "nauc_ndcg_at_100_std": 0.08674070424465001, + "nauc_ndcg_at_10_diff1": -0.08128772448530021, + "nauc_ndcg_at_10_max": 0.17244036804629503, + "nauc_ndcg_at_10_std": 0.02819345677203796, + "nauc_ndcg_at_1_diff1": -0.5227107209935222, + "nauc_ndcg_at_1_max": 0.09562650925677493, + "nauc_ndcg_at_1_std": 0.16468243320939854, + "nauc_ndcg_at_20_diff1": -0.086056370146497, + "nauc_ndcg_at_20_max": 0.195869806760938, + "nauc_ndcg_at_20_std": 0.04757991969301295, + "nauc_ndcg_at_3_diff1": -0.09491160269887662, + "nauc_ndcg_at_3_max": 0.15135662699498084, + "nauc_ndcg_at_3_std": 0.019722501663945312, + "nauc_ndcg_at_5_diff1": -0.09008866995016263, + "nauc_ndcg_at_5_max": 0.16143679483180548, + "nauc_ndcg_at_5_std": 0.022764770078524912, + "nauc_precision_at_1000_diff1": -0.24045166016587438, + "nauc_precision_at_1000_max": 0.3195034508543935, + "nauc_precision_at_1000_std": 0.34953942622964457, + "nauc_precision_at_100_diff1": -0.14676866267060482, + "nauc_precision_at_100_max": 0.2595133723112115, + "nauc_precision_at_100_std": 0.16414102793250826, + "nauc_precision_at_10_diff1": -0.08958089915422018, + "nauc_precision_at_10_max": 0.18075236692646582, + "nauc_precision_at_10_std": 0.036349247845646616, + "nauc_precision_at_1_diff1": -0.5227107209935222, + "nauc_precision_at_1_max": 0.09562650925677493, + "nauc_precision_at_1_std": 0.16468243320939854, + "nauc_precision_at_20_diff1": -0.09690232093945902, + "nauc_precision_at_20_max": 0.2216329441415861, + "nauc_precision_at_20_std": 0.06967652721476236, + "nauc_precision_at_3_diff1": -0.11403461479629584, + "nauc_precision_at_3_max": 0.1515256188109501, + "nauc_precision_at_3_std": 0.02583697143100736, + "nauc_precision_at_5_diff1": -0.10407436535190769, + "nauc_precision_at_5_max": 0.16618197540243515, + "nauc_precision_at_5_std": 0.027731434908842483, + "nauc_recall_at_1000_diff1": -0.21373976975815448, + "nauc_recall_at_1000_max": 0.340364079088584, + "nauc_recall_at_1000_std": 0.3758106778790492, + "nauc_recall_at_100_diff1": -0.12841173327380748, + "nauc_recall_at_100_max": 0.26630249039089776, + "nauc_recall_at_100_std": 0.16610202623791748, + "nauc_recall_at_10_diff1": -0.06387695134566657, + "nauc_recall_at_10_max": 0.1841498177665857, + "nauc_recall_at_10_std": 0.03059539077954125, + "nauc_recall_at_1_diff1": -0.5280465285121293, + "nauc_recall_at_1_max": 0.08621506021244016, + "nauc_recall_at_1_std": 0.18339146766836595, + "nauc_recall_at_20_diff1": -0.07508476020191192, + "nauc_recall_at_20_max": 0.2266673632206196, + "nauc_recall_at_20_std": 0.06644902770116523, + "nauc_recall_at_3_diff1": -0.07292873956832151, + "nauc_recall_at_3_max": 0.15682509079744078, + "nauc_recall_at_3_std": 0.014168688229387593, + "nauc_recall_at_5_diff1": -0.07206865420575172, + "nauc_recall_at_5_max": 0.1701900002314879, + "nauc_recall_at_5_std": 0.019842639580852355, + "ndcg_at_1": 0.00839, + "ndcg_at_10": 0.09422, + "ndcg_at_100": 0.15655, + "ndcg_at_1000": 0.19407, + "ndcg_at_20": 0.11286, + "ndcg_at_3": 0.0573, + "ndcg_at_5": 0.07574, + "precision_at_1": 0.00839, + "precision_at_10": 0.02046, + "precision_at_100": 0.00519, + "precision_at_1000": 0.00082, + "precision_at_20": 0.01393, + "precision_at_3": 0.03421, + "precision_at_5": 0.02954, + "recall_at_1": 0.00408, + "recall_at_10": 0.1946, + "recall_at_100": 0.50923, + "recall_at_1000": 0.80791, + "recall_at_20": 0.26871, + "recall_at_3": 0.09305, + "recall_at_5": 0.13789 + } + ] + }, + "task_name": "CIRRIT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOI2TRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOI2TRetrieval.json new file mode 100644 index 0000000000..c4333ca68f --- /dev/null +++ b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOI2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", + "evaluation_time": 42.80281734466553, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.37288, + "map_at_1": 0.08071, + "map_at_10": 0.26786, + "map_at_100": 0.31168, + "map_at_1000": 0.31587, + "map_at_20": 0.29106, + "map_at_3": 0.17731, + "map_at_5": 0.2267, + "mrr_at_1": 0.4034, + "mrr_at_10": 0.5154798412698407, + "mrr_at_100": 0.5245965200676513, + "mrr_at_1000": 0.5247295619699504, + "mrr_at_20": 0.521172379941179, + "mrr_at_3": 0.482566666666666, + "mrr_at_5": 0.5011866666666658, + "nauc_map_at_1000_diff1": 0.18316596672602442, + "nauc_map_at_1000_max": 0.34801279014511405, + "nauc_map_at_1000_std": -0.18750172942216528, + "nauc_map_at_100_diff1": 0.18245509021092263, + "nauc_map_at_100_max": 0.3484150820371727, + "nauc_map_at_100_std": -0.18865972347643556, + "nauc_map_at_10_diff1": 0.1851275487650171, + "nauc_map_at_10_max": 0.3370878939903754, + "nauc_map_at_10_std": -0.19389877734594482, + "nauc_map_at_1_diff1": 0.2724994569940933, + "nauc_map_at_1_max": 0.2759939487280074, + "nauc_map_at_1_std": -0.18769761438653643, + "nauc_map_at_20_diff1": 0.18551346146324993, + "nauc_map_at_20_max": 0.34459316188937006, + "nauc_map_at_20_std": -0.19182380187451328, + "nauc_map_at_3_diff1": 0.21373477811733987, + "nauc_map_at_3_max": 0.3060468883646992, + "nauc_map_at_3_std": -0.19267528106941764, + "nauc_map_at_5_diff1": 0.1895485112324686, + "nauc_map_at_5_max": 0.32650067482878825, + "nauc_map_at_5_std": -0.19076772161486724, + "nauc_mrr_at_1000_diff1": 0.25557692572293067, + "nauc_mrr_at_1000_max": 0.2864330733081474, + "nauc_mrr_at_1000_std": -0.20284682486076736, + "nauc_mrr_at_100_diff1": 0.2555410395024608, + "nauc_mrr_at_100_max": 0.2864902886387769, + "nauc_mrr_at_100_std": -0.20281383228526584, + "nauc_mrr_at_10_diff1": 0.2552916397209132, + "nauc_mrr_at_10_max": 0.2866177928313412, + "nauc_mrr_at_10_std": -0.20398130147274418, + "nauc_mrr_at_1_diff1": 0.27278878441171606, + "nauc_mrr_at_1_max": 0.2760128871314018, + "nauc_mrr_at_1_std": -0.1875598816248055, + "nauc_mrr_at_20_diff1": 0.2552527730209484, + "nauc_mrr_at_20_max": 0.2869938940286829, + "nauc_mrr_at_20_std": -0.20347090553389235, + "nauc_mrr_at_3_diff1": 0.2551361915671908, + "nauc_mrr_at_3_max": 0.2853826392458793, + "nauc_mrr_at_3_std": -0.2014489869554216, + "nauc_mrr_at_5_diff1": 0.25424238709065144, + "nauc_mrr_at_5_max": 0.2834915887900367, + "nauc_mrr_at_5_std": -0.2041266754700247, + "nauc_ndcg_at_1000_diff1": 0.19931449927197542, + "nauc_ndcg_at_1000_max": 0.34824914996907946, + "nauc_ndcg_at_1000_std": -0.17612769107106158, + "nauc_ndcg_at_100_diff1": 0.18782042717250694, + "nauc_ndcg_at_100_max": 0.3567017765386365, + "nauc_ndcg_at_100_std": -0.18124611786715342, + "nauc_ndcg_at_10_diff1": 0.19614278113213443, + "nauc_ndcg_at_10_max": 0.3345885498609445, + "nauc_ndcg_at_10_std": -0.1995834302663533, + "nauc_ndcg_at_1_diff1": 0.27278878441171606, + "nauc_ndcg_at_1_max": 0.2760128871314018, + "nauc_ndcg_at_1_std": -0.1875598816248055, + "nauc_ndcg_at_20_diff1": 0.19575779535055157, + "nauc_ndcg_at_20_max": 0.3461848546945705, + "nauc_ndcg_at_20_std": -0.19512601196073395, + "nauc_ndcg_at_3_diff1": 0.21982547126571084, + "nauc_ndcg_at_3_max": 0.3043154912793067, + "nauc_ndcg_at_3_std": -0.19749127050913148, + "nauc_ndcg_at_5_diff1": 0.19883328350299154, + "nauc_ndcg_at_5_max": 0.32007392291215236, + "nauc_ndcg_at_5_std": -0.19678681748663923, + "nauc_precision_at_1000_diff1": 0.022351052380250144, + "nauc_precision_at_1000_max": 0.4725759260447294, + "nauc_precision_at_1000_std": 0.5797080918881626, + "nauc_precision_at_100_diff1": 0.08393111116724335, + "nauc_precision_at_100_max": 0.3771922887973945, + "nauc_precision_at_100_std": -0.08376254395248357, + "nauc_precision_at_10_diff1": 0.15560876491276518, + "nauc_precision_at_10_max": 0.33714785955779036, + "nauc_precision_at_10_std": -0.1875518639116136, + "nauc_precision_at_1_diff1": 0.27278878441171606, + "nauc_precision_at_1_max": 0.2760128871314018, + "nauc_precision_at_1_std": -0.1875598816248055, + "nauc_precision_at_20_diff1": 0.1477270394650292, + "nauc_precision_at_20_max": 0.35052230390958183, + "nauc_precision_at_20_std": -0.16921558322042235, + "nauc_precision_at_3_diff1": 0.19844496546548113, + "nauc_precision_at_3_max": 0.3091703704739135, + "nauc_precision_at_3_std": -0.19757336334013054, + "nauc_precision_at_5_diff1": 0.1653617822470633, + "nauc_precision_at_5_max": 0.3264198806421896, + "nauc_precision_at_5_std": -0.19150011703731976, + "nauc_recall_at_1000_diff1": 0.01459616156901875, + "nauc_recall_at_1000_max": 0.48367786796485557, + "nauc_recall_at_1000_std": 0.5837504997127877, + "nauc_recall_at_100_diff1": 0.08310515225683762, + "nauc_recall_at_100_max": 0.37760531087114085, + "nauc_recall_at_100_std": -0.08463401361245898, + "nauc_recall_at_10_diff1": 0.15530962605623758, + "nauc_recall_at_10_max": 0.33716708957281405, + "nauc_recall_at_10_std": -0.18785611276459221, + "nauc_recall_at_1_diff1": 0.2724994569940933, + "nauc_recall_at_1_max": 0.2759939487280074, + "nauc_recall_at_1_std": -0.18769761438653643, + "nauc_recall_at_20_diff1": 0.1472978746648726, + "nauc_recall_at_20_max": 0.35044375641725917, + "nauc_recall_at_20_std": -0.16960809708140606, + "nauc_recall_at_3_diff1": 0.1981017909103437, + "nauc_recall_at_3_max": 0.3091501498176776, + "nauc_recall_at_3_std": -0.19776288065176492, + "nauc_recall_at_5_diff1": 0.16510836445941884, + "nauc_recall_at_5_max": 0.32640908545278824, + "nauc_recall_at_5_std": -0.19170241637346824, + "ndcg_at_1": 0.4034, + "ndcg_at_10": 0.37288, + "ndcg_at_100": 0.50144, + "ndcg_at_1000": 0.54804, + "ndcg_at_20": 0.42119, + "ndcg_at_3": 0.35371, + "ndcg_at_5": 0.31191, + "precision_at_1": 0.4034, + "precision_at_10": 0.19746, + "precision_at_100": 0.03826, + "precision_at_1000": 0.0049, + "precision_at_20": 0.12689, + "precision_at_3": 0.33847, + "precision_at_5": 0.28372, + "recall_at_1": 0.08071, + "recall_at_10": 0.39509, + "recall_at_100": 0.76558, + "recall_at_1000": 0.97947, + "recall_at_20": 0.50782, + "recall_at_3": 0.20317, + "recall_at_5": 0.28382 + } + ] + }, + "task_name": "MSCOCOI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOT2IRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOT2IRetrieval.json new file mode 100644 index 0000000000..7e6d34afff --- /dev/null +++ b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOT2IRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", + "evaluation_time": 66.51581335067749, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.46897, + "map_at_1": 0.305, + "map_at_10": 0.40963, + "map_at_100": 0.42183, + "map_at_1000": 0.42212, + "map_at_20": 0.41684, + "map_at_3": 0.37632, + "map_at_5": 0.39495, + "mrr_at_1": 0.30525212624450804, + "mrr_at_10": 0.4099942929057418, + "mrr_at_100": 0.42214037857811404, + "mrr_at_1000": 0.4224150820003196, + "mrr_at_20": 0.4171927718574053, + "mrr_at_3": 0.3766643825493417, + "mrr_at_5": 0.39532696467679795, + "nauc_map_at_1000_diff1": 0.4478711106054124, + "nauc_map_at_1000_max": 0.28050562897695164, + "nauc_map_at_1000_std": -0.21959015997209919, + "nauc_map_at_100_diff1": 0.4477231775321896, + "nauc_map_at_100_max": 0.2806372368499711, + "nauc_map_at_100_std": -0.21940131954025208, + "nauc_map_at_10_diff1": 0.44612476561916625, + "nauc_map_at_10_max": 0.2799994443519687, + "nauc_map_at_10_std": -0.22187850998432965, + "nauc_map_at_1_diff1": 0.5029747421425568, + "nauc_map_at_1_max": 0.272549283076283, + "nauc_map_at_1_std": -0.2124194075262369, + "nauc_map_at_20_diff1": 0.4465097088138646, + "nauc_map_at_20_max": 0.28045224479942427, + "nauc_map_at_20_std": -0.22094628341471756, + "nauc_map_at_3_diff1": 0.4543183470256243, + "nauc_map_at_3_max": 0.27789569085174415, + "nauc_map_at_3_std": -0.22162450074705334, + "nauc_map_at_5_diff1": 0.44768741119006605, + "nauc_map_at_5_max": 0.27857801963069206, + "nauc_map_at_5_std": -0.22171216858692744, + "nauc_mrr_at_1000_diff1": 0.44762123092250417, + "nauc_mrr_at_1000_max": 0.2804655602576719, + "nauc_mrr_at_1000_std": -0.21917729645099004, + "nauc_mrr_at_100_diff1": 0.4474764210942955, + "nauc_mrr_at_100_max": 0.280601026351656, + "nauc_mrr_at_100_std": -0.2189669410434172, + "nauc_mrr_at_10_diff1": 0.4457989624406091, + "nauc_mrr_at_10_max": 0.2799902368295713, + "nauc_mrr_at_10_std": -0.22134188164578128, + "nauc_mrr_at_1_diff1": 0.5027796660701911, + "nauc_mrr_at_1_max": 0.27275111707746236, + "nauc_mrr_at_1_std": -0.21214467131036538, + "nauc_mrr_at_20_diff1": 0.44622688768256885, + "nauc_mrr_at_20_max": 0.28042386811339354, + "nauc_mrr_at_20_std": -0.22041389186782168, + "nauc_mrr_at_3_diff1": 0.4540565519372315, + "nauc_mrr_at_3_max": 0.27789412741657327, + "nauc_mrr_at_3_std": -0.22124654547810124, + "nauc_mrr_at_5_diff1": 0.4473821512144231, + "nauc_mrr_at_5_max": 0.2785729569663956, + "nauc_mrr_at_5_std": -0.22131319479803052, + "nauc_ndcg_at_1000_diff1": 0.43654475958434186, + "nauc_ndcg_at_1000_max": 0.2848558104719407, + "nauc_ndcg_at_1000_std": -0.21518136929725057, + "nauc_ndcg_at_100_diff1": 0.4327107582163226, + "nauc_ndcg_at_100_max": 0.28859948535481533, + "nauc_ndcg_at_100_std": -0.2081893113971199, + "nauc_ndcg_at_10_diff1": 0.42285553095601514, + "nauc_ndcg_at_10_max": 0.2852677139001271, + "nauc_ndcg_at_10_std": -0.2245997440948349, + "nauc_ndcg_at_1_diff1": 0.5027796660701911, + "nauc_ndcg_at_1_max": 0.27275111707746236, + "nauc_ndcg_at_1_std": -0.21214467131036538, + "nauc_ndcg_at_20_diff1": 0.4235238359251292, + "nauc_ndcg_at_20_max": 0.2872035072298614, + "nauc_ndcg_at_20_std": -0.22100876714130083, + "nauc_ndcg_at_3_diff1": 0.43942781361598426, + "nauc_ndcg_at_3_max": 0.27973046846426364, + "nauc_ndcg_at_3_std": -0.22444184638120696, + "nauc_ndcg_at_5_diff1": 0.427247141083044, + "nauc_ndcg_at_5_max": 0.2811124586942207, + "nauc_ndcg_at_5_std": -0.22465662590672983, + "nauc_precision_at_1000_diff1": -0.12179681349517751, + "nauc_precision_at_1000_max": 0.13023305110548689, + "nauc_precision_at_1000_std": 0.5260350786100252, + "nauc_precision_at_100_diff1": 0.18340202723844473, + "nauc_precision_at_100_max": 0.39983881724392506, + "nauc_precision_at_100_std": 0.221650593088434, + "nauc_precision_at_10_diff1": 0.3346141522540821, + "nauc_precision_at_10_max": 0.3061257203288402, + "nauc_precision_at_10_std": -0.22911572795263271, + "nauc_precision_at_1_diff1": 0.5027796660701911, + "nauc_precision_at_1_max": 0.27275111707746236, + "nauc_precision_at_1_std": -0.21214467131036538, + "nauc_precision_at_20_diff1": 0.3090330640648721, + "nauc_precision_at_20_max": 0.32131718134148296, + "nauc_precision_at_20_std": -0.20716374732844292, + "nauc_precision_at_3_diff1": 0.3960170209609372, + "nauc_precision_at_3_max": 0.285285655711078, + "nauc_precision_at_3_std": -0.23167624055155045, + "nauc_precision_at_5_diff1": 0.3625854653247323, + "nauc_precision_at_5_max": 0.28929006766488813, + "nauc_precision_at_5_std": -0.23215194678446605, + "nauc_recall_at_1000_diff1": 0.22969264464007677, + "nauc_recall_at_1000_max": 0.7235125847169891, + "nauc_recall_at_1000_std": 0.8279246959358108, + "nauc_recall_at_100_diff1": 0.2606980752081389, + "nauc_recall_at_100_max": 0.47215991575399363, + "nauc_recall_at_100_std": 0.16730099691638203, + "nauc_recall_at_10_diff1": 0.33881214716475055, + "nauc_recall_at_10_max": 0.30639423349407746, + "nauc_recall_at_10_std": -0.23428259082232605, + "nauc_recall_at_1_diff1": 0.5029747421425568, + "nauc_recall_at_1_max": 0.272549283076283, + "nauc_recall_at_1_std": -0.2124194075262369, + "nauc_recall_at_20_diff1": 0.31710792090285655, + "nauc_recall_at_20_max": 0.32370157795364835, + "nauc_recall_at_20_std": -0.21662838240185825, + "nauc_recall_at_3_diff1": 0.3971631468785135, + "nauc_recall_at_3_max": 0.28502696676818606, + "nauc_recall_at_3_std": -0.23298949690090923, + "nauc_recall_at_5_diff1": 0.3644953058714283, + "nauc_recall_at_5_max": 0.2889906359951118, + "nauc_recall_at_5_std": -0.23398910701923947, + "ndcg_at_1": 0.30525, + "ndcg_at_10": 0.46897, + "ndcg_at_100": 0.53034, + "ndcg_at_1000": 0.53666, + "ndcg_at_20": 0.49515, + "ndcg_at_3": 0.39993, + "ndcg_at_5": 0.4335, + "precision_at_1": 0.30525, + "precision_at_10": 0.06607, + "precision_at_100": 0.00955, + "precision_at_1000": 0.001, + "precision_at_20": 0.03824, + "precision_at_3": 0.15626, + "precision_at_5": 0.11012, + "recall_at_1": 0.305, + "recall_at_10": 0.65879, + "recall_at_100": 0.94938, + "recall_at_1000": 0.99673, + "recall_at_20": 0.76204, + "recall_at_3": 0.46797, + "recall_at_5": 0.54937 + } + ] + }, + "task_name": "MSCOCOT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/NIGHTSI2IRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/NIGHTSI2IRetrieval.json new file mode 100644 index 0000000000..5ba1e219c8 --- /dev/null +++ b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/NIGHTSI2IRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", + "evaluation_time": 199.96433472633362, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.22818, + "map_at_1": 0.06934, + "map_at_10": 0.16097, + "map_at_100": 0.18386, + "map_at_1000": 0.18414, + "map_at_20": 0.17653, + "map_at_3": 0.11635, + "map_at_5": 0.13781, + "mrr_at_1": 0.06933962264150943, + "mrr_at_10": 0.16097334531296778, + "mrr_at_100": 0.1838599085374125, + "mrr_at_1000": 0.18413700903303717, + "mrr_at_20": 0.17653316610403727, + "mrr_at_3": 0.1163522012578618, + "mrr_at_5": 0.13781446540880488, + "nauc_map_at_1000_diff1": 0.057301690218310136, + "nauc_map_at_1000_max": 0.07188842602959197, + "nauc_map_at_1000_std": -0.08154303633505418, + "nauc_map_at_100_diff1": 0.05698354071014854, + "nauc_map_at_100_max": 0.07244030136556417, + "nauc_map_at_100_std": -0.08128920818096987, + "nauc_map_at_10_diff1": 0.05865679320595122, + "nauc_map_at_10_max": 0.06134631667106162, + "nauc_map_at_10_std": -0.08503834658994171, + "nauc_map_at_1_diff1": 0.10488273656658804, + "nauc_map_at_1_max": 0.0639349917589788, + "nauc_map_at_1_std": -0.07575716092715594, + "nauc_map_at_20_diff1": 0.0574817213936499, + "nauc_map_at_20_max": 0.0694885498529206, + "nauc_map_at_20_std": -0.08453835749586952, + "nauc_map_at_3_diff1": 0.07568477970041008, + "nauc_map_at_3_max": 0.06567675283236628, + "nauc_map_at_3_std": -0.09191303028245797, + "nauc_map_at_5_diff1": 0.06830671022162287, + "nauc_map_at_5_max": 0.05388386482539364, + "nauc_map_at_5_std": -0.0869392787560929, + "nauc_mrr_at_1000_diff1": 0.057301690218310136, + "nauc_mrr_at_1000_max": 0.07188842602959197, + "nauc_mrr_at_1000_std": -0.08154303633505418, + "nauc_mrr_at_100_diff1": 0.05698354071014854, + "nauc_mrr_at_100_max": 0.07244030136556417, + "nauc_mrr_at_100_std": -0.08128920818096987, + "nauc_mrr_at_10_diff1": 0.05865679320595122, + "nauc_mrr_at_10_max": 0.06134631667106162, + "nauc_mrr_at_10_std": -0.08503834658994171, + "nauc_mrr_at_1_diff1": 0.10488273656658804, + "nauc_mrr_at_1_max": 0.0639349917589788, + "nauc_mrr_at_1_std": -0.07575716092715594, + "nauc_mrr_at_20_diff1": 0.0574817213936499, + "nauc_mrr_at_20_max": 0.0694885498529206, + "nauc_mrr_at_20_std": -0.08453835749586952, + "nauc_mrr_at_3_diff1": 0.07568477970041008, + "nauc_mrr_at_3_max": 0.06567675283236628, + "nauc_mrr_at_3_std": -0.09191303028245797, + "nauc_mrr_at_5_diff1": 0.06830671022162287, + "nauc_mrr_at_5_max": 0.05388386482539364, + "nauc_mrr_at_5_std": -0.0869392787560929, + "nauc_ndcg_at_1000_diff1": 0.03275032921060949, + "nauc_ndcg_at_1000_max": 0.09370875865082869, + "nauc_ndcg_at_1000_std": -0.06520945518508174, + "nauc_ndcg_at_100_diff1": 0.022434604110621165, + "nauc_ndcg_at_100_max": 0.1088264312707009, + "nauc_ndcg_at_100_std": -0.05625420269516149, + "nauc_ndcg_at_10_diff1": 0.036984429499972896, + "nauc_ndcg_at_10_max": 0.06396551203788203, + "nauc_ndcg_at_10_std": -0.08202279893586943, + "nauc_ndcg_at_1_diff1": 0.10488273656658804, + "nauc_ndcg_at_1_max": 0.0639349917589788, + "nauc_ndcg_at_1_std": -0.07575716092715594, + "nauc_ndcg_at_20_diff1": 0.029682814087520562, + "nauc_ndcg_at_20_max": 0.09156539071118802, + "nauc_ndcg_at_20_std": -0.07992527984856755, + "nauc_ndcg_at_3_diff1": 0.06755595595602068, + "nauc_ndcg_at_3_max": 0.06624062050199891, + "nauc_ndcg_at_3_std": -0.09524111086264671, + "nauc_ndcg_at_5_diff1": 0.05733520128202785, + "nauc_ndcg_at_5_max": 0.047767515545288526, + "nauc_ndcg_at_5_std": -0.08669006844506776, + "nauc_precision_at_1000_diff1": -0.5127678978568057, + "nauc_precision_at_1000_max": 0.6076477102468908, + "nauc_precision_at_1000_std": 0.49423490326363295, + "nauc_precision_at_100_diff1": -0.32390524343909355, + "nauc_precision_at_100_max": 0.524590549007864, + "nauc_precision_at_100_std": 0.27115080833171834, + "nauc_precision_at_10_diff1": -0.005594372755515605, + "nauc_precision_at_10_max": 0.07134091937034583, + "nauc_precision_at_10_std": -0.07449401017787786, + "nauc_precision_at_1_diff1": 0.10488273656658804, + "nauc_precision_at_1_max": 0.0639349917589788, + "nauc_precision_at_1_std": -0.07575716092715594, + "nauc_precision_at_20_diff1": -0.044341437676347024, + "nauc_precision_at_20_max": 0.16246650001126064, + "nauc_precision_at_20_std": -0.06513958194587852, + "nauc_precision_at_3_diff1": 0.05083674702012877, + "nauc_precision_at_3_max": 0.06741308965704264, + "nauc_precision_at_3_std": -0.10189783181241091, + "nauc_precision_at_5_diff1": 0.03711832073718323, + "nauc_precision_at_5_max": 0.03554341529116533, + "nauc_precision_at_5_std": -0.08532792633472566, + "nauc_recall_at_1000_diff1": -0.5127678978568214, + "nauc_recall_at_1000_max": 0.6076477102468842, + "nauc_recall_at_1000_std": 0.49423490326363095, + "nauc_recall_at_100_diff1": -0.32390524343909616, + "nauc_recall_at_100_max": 0.5245905490078637, + "nauc_recall_at_100_std": 0.27115080833171884, + "nauc_recall_at_10_diff1": -0.005594372755515973, + "nauc_recall_at_10_max": 0.0713409193703457, + "nauc_recall_at_10_std": -0.07449401017787816, + "nauc_recall_at_1_diff1": 0.10488273656658804, + "nauc_recall_at_1_max": 0.0639349917589788, + "nauc_recall_at_1_std": -0.07575716092715594, + "nauc_recall_at_20_diff1": -0.04434143767634782, + "nauc_recall_at_20_max": 0.1624665000112601, + "nauc_recall_at_20_std": -0.06513958194587831, + "nauc_recall_at_3_diff1": 0.05083674702012879, + "nauc_recall_at_3_max": 0.06741308965704265, + "nauc_recall_at_3_std": -0.10189783181241095, + "nauc_recall_at_5_diff1": 0.037118320737183075, + "nauc_recall_at_5_max": 0.03554341529116537, + "nauc_recall_at_5_std": -0.0853279263347257, + "ndcg_at_1": 0.06934, + "ndcg_at_10": 0.22818, + "ndcg_at_100": 0.33297, + "ndcg_at_1000": 0.34, + "ndcg_at_20": 0.28462, + "ndcg_at_3": 0.13301, + "ndcg_at_5": 0.17188, + "precision_at_1": 0.06934, + "precision_at_10": 0.04509, + "precision_at_100": 0.00925, + "precision_at_1000": 0.00098, + "precision_at_20": 0.03368, + "precision_at_3": 0.06053, + "precision_at_5": 0.05528, + "recall_at_1": 0.06934, + "recall_at_10": 0.45094, + "recall_at_100": 0.925, + "recall_at_1000": 0.97972, + "recall_at_20": 0.67358, + "recall_at_3": 0.1816, + "recall_at_5": 0.27642 + } + ] + }, + "task_name": "NIGHTSI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2ITRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2ITRetrieval.json new file mode 100644 index 0000000000..60695744d8 --- /dev/null +++ b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2ITRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "53db4c9f9c93cb74926a1c9d04dea7d7acac2f21", + "evaluation_time": 2691.2529063224792, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.52663, + "map_at_1": 0.32198, + "map_at_10": 0.45953, + "map_at_100": 0.47181, + "map_at_1000": 0.4723, + "map_at_20": 0.46711, + "map_at_3": 0.42417, + "map_at_5": 0.44499, + "mrr_at_1": 0.41178813221823973, + "mrr_at_10": 0.5210127818550758, + "mrr_at_100": 0.5288710554330768, + "mrr_at_1000": 0.5290959300255417, + "mrr_at_20": 0.526159218535241, + "mrr_at_3": 0.4948891543873623, + "mrr_at_5": 0.5105402893933366, + "nauc_map_at_1000_diff1": 0.43876271441341697, + "nauc_map_at_1000_max": 0.2713223520198593, + "nauc_map_at_1000_std": -0.16052486820215323, + "nauc_map_at_100_diff1": 0.43857076615081975, + "nauc_map_at_100_max": 0.27148142408822945, + "nauc_map_at_100_std": -0.1604368100365843, + "nauc_map_at_10_diff1": 0.43871121919352996, + "nauc_map_at_10_max": 0.269439210185352, + "nauc_map_at_10_std": -0.16631952646794543, + "nauc_map_at_1_diff1": 0.5024807336255694, + "nauc_map_at_1_max": 0.2285751039496304, + "nauc_map_at_1_std": -0.15553705788344513, + "nauc_map_at_20_diff1": 0.4381920616977959, + "nauc_map_at_20_max": 0.27000963658536997, + "nauc_map_at_20_std": -0.16422634459492544, + "nauc_map_at_3_diff1": 0.44957721321495686, + "nauc_map_at_3_max": 0.26041374421396873, + "nauc_map_at_3_std": -0.17098574514931533, + "nauc_map_at_5_diff1": 0.4379760197271896, + "nauc_map_at_5_max": 0.26650735933197567, + "nauc_map_at_5_std": -0.1678803021826141, + "nauc_mrr_at_1000_diff1": 0.4498242067689463, + "nauc_mrr_at_1000_max": 0.2763503353709589, + "nauc_mrr_at_1000_std": -0.1515498089863098, + "nauc_mrr_at_100_diff1": 0.4497295532312827, + "nauc_mrr_at_100_max": 0.27653912064968234, + "nauc_mrr_at_100_std": -0.15134474459586014, + "nauc_mrr_at_10_diff1": 0.44835612729964286, + "nauc_mrr_at_10_max": 0.2760509111175981, + "nauc_mrr_at_10_std": -0.15401033906315414, + "nauc_mrr_at_1_diff1": 0.5064389691968733, + "nauc_mrr_at_1_max": 0.25538371022385664, + "nauc_mrr_at_1_std": -0.15120047500046432, + "nauc_mrr_at_20_diff1": 0.4491194452623415, + "nauc_mrr_at_20_max": 0.2759581261363807, + "nauc_mrr_at_20_std": -0.15329529609279208, + "nauc_mrr_at_3_diff1": 0.4510015598661547, + "nauc_mrr_at_3_max": 0.26930524592262844, + "nauc_mrr_at_3_std": -0.16055986565559113, + "nauc_mrr_at_5_diff1": 0.4467399021960818, + "nauc_mrr_at_5_max": 0.27441954793660395, + "nauc_mrr_at_5_std": -0.156411952108089, + "nauc_ndcg_at_1000_diff1": 0.42307561167146884, + "nauc_ndcg_at_1000_max": 0.28854455791397793, + "nauc_ndcg_at_1000_std": -0.13415719921037747, + "nauc_ndcg_at_100_diff1": 0.41863495760271513, + "nauc_ndcg_at_100_max": 0.2934231665534968, + "nauc_ndcg_at_100_std": -0.12953624384754348, + "nauc_ndcg_at_10_diff1": 0.4165024926207999, + "nauc_ndcg_at_10_max": 0.2815688455123661, + "nauc_ndcg_at_10_std": -0.16033266463844695, + "nauc_ndcg_at_1_diff1": 0.4991940974293242, + "nauc_ndcg_at_1_max": 0.25257839379065944, + "nauc_ndcg_at_1_std": -0.1476551024812966, + "nauc_ndcg_at_20_diff1": 0.4142260718683874, + "nauc_ndcg_at_20_max": 0.2832424372665771, + "nauc_ndcg_at_20_std": -0.15369250623843952, + "nauc_ndcg_at_3_diff1": 0.43173077808948285, + "nauc_ndcg_at_3_max": 0.26535751742144664, + "nauc_ndcg_at_3_std": -0.17037703654971278, + "nauc_ndcg_at_5_diff1": 0.41505821729386394, + "nauc_ndcg_at_5_max": 0.2753571205036466, + "nauc_ndcg_at_5_std": -0.16436190189593947, + "nauc_precision_at_1000_diff1": -0.09195784735246411, + "nauc_precision_at_1000_max": 0.11346836723919133, + "nauc_precision_at_1000_std": 0.19491442787034385, + "nauc_precision_at_100_diff1": -0.032103070968056875, + "nauc_precision_at_100_max": 0.18552459477832725, + "nauc_precision_at_100_std": 0.17369141585226536, + "nauc_precision_at_10_diff1": 0.14619869365447397, + "nauc_precision_at_10_max": 0.23031678733137417, + "nauc_precision_at_10_std": -0.05167802110000886, + "nauc_precision_at_1_diff1": 0.4991940974293242, + "nauc_precision_at_1_max": 0.25257839379065944, + "nauc_precision_at_1_std": -0.1476551024812966, + "nauc_precision_at_20_diff1": 0.08041747973726056, + "nauc_precision_at_20_max": 0.20668859387848693, + "nauc_precision_at_20_std": 0.0015670064844935908, + "nauc_precision_at_3_diff1": 0.2766345000123392, + "nauc_precision_at_3_max": 0.25180850003621597, + "nauc_precision_at_3_std": -0.1259167659370286, + "nauc_precision_at_5_diff1": 0.19761775316424005, + "nauc_precision_at_5_max": 0.24663783968040612, + "nauc_precision_at_5_std": -0.09202309206478451, + "nauc_recall_at_1000_diff1": 0.21907469666289658, + "nauc_recall_at_1000_max": 0.45355511952440225, + "nauc_recall_at_1000_std": 0.2803857322711674, + "nauc_recall_at_100_diff1": 0.23460647409258342, + "nauc_recall_at_100_max": 0.4368628650169039, + "nauc_recall_at_100_std": 0.15794853700940753, + "nauc_recall_at_10_diff1": 0.309978003824074, + "nauc_recall_at_10_max": 0.29394118609593484, + "nauc_recall_at_10_std": -0.1475438686983404, + "nauc_recall_at_1_diff1": 0.5024807336255694, + "nauc_recall_at_1_max": 0.2285751039496304, + "nauc_recall_at_1_std": -0.15553705788344513, + "nauc_recall_at_20_diff1": 0.27182787851817175, + "nauc_recall_at_20_max": 0.3004972888603483, + "nauc_recall_at_20_std": -0.11618638451955582, + "nauc_recall_at_3_diff1": 0.3666928890381534, + "nauc_recall_at_3_max": 0.25984514795329117, + "nauc_recall_at_3_std": -0.1721856188134978, + "nauc_recall_at_5_diff1": 0.3190261983587683, + "nauc_recall_at_5_max": 0.27727115206859354, + "nauc_recall_at_5_std": -0.1578256131156148, + "ndcg_at_1": 0.41458, + "ndcg_at_10": 0.52663, + "ndcg_at_100": 0.57734, + "ndcg_at_1000": 0.58728, + "ndcg_at_20": 0.54976, + "ndcg_at_3": 0.46369, + "ndcg_at_5": 0.49654, + "precision_at_1": 0.41458, + "precision_at_10": 0.09271, + "precision_at_100": 0.01253, + "precision_at_1000": 0.00136, + "precision_at_20": 0.05247, + "precision_at_3": 0.23032, + "precision_at_5": 0.15978, + "recall_at_1": 0.32198, + "recall_at_10": 0.66169, + "recall_at_100": 0.87535, + "recall_at_1000": 0.94464, + "recall_at_20": 0.74313, + "recall_at_3": 0.50816, + "recall_at_5": 0.57965 + } + ] + }, + "task_name": "WebQAT2ITRetrieval" +} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2TRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2TRetrieval.json new file mode 100644 index 0000000000..febd96847e --- /dev/null +++ b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2TRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "468b42a2b2e767d80d2d93f5ae5d42f135a10478", + "evaluation_time": 988.9164459705353, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.64907, + "map_at_1": 0.31073, + "map_at_10": 0.55199, + "map_at_100": 0.56518, + "map_at_1000": 0.56572, + "map_at_20": 0.56038, + "map_at_3": 0.49469, + "map_at_5": 0.52906, + "mrr_at_1": 0.6276985743380855, + "mrr_at_10": 0.7316605243590977, + "mrr_at_100": 0.7344313618617866, + "mrr_at_1000": 0.7344774340987533, + "mrr_at_20": 0.7334756132529352, + "mrr_at_3": 0.7120162932790214, + "mrr_at_5": 0.7248065173116084, + "nauc_map_at_1000_diff1": 0.29580437839479934, + "nauc_map_at_1000_max": 0.2927721108826568, + "nauc_map_at_1000_std": 0.012872496436827125, + "nauc_map_at_100_diff1": 0.2957568562473137, + "nauc_map_at_100_max": 0.29338281472542477, + "nauc_map_at_100_std": 0.013473543179429661, + "nauc_map_at_10_diff1": 0.28970252769434723, + "nauc_map_at_10_max": 0.2864523752255613, + "nauc_map_at_10_std": 0.006314905902818341, + "nauc_map_at_1_diff1": 0.5159028191255995, + "nauc_map_at_1_max": 0.3174029367347432, + "nauc_map_at_1_std": -0.020320539407606746, + "nauc_map_at_20_diff1": 0.2947811042949462, + "nauc_map_at_20_max": 0.2929089074820099, + "nauc_map_at_20_std": 0.012461044754162466, + "nauc_map_at_3_diff1": 0.2905401755376613, + "nauc_map_at_3_max": 0.2661749034174475, + "nauc_map_at_3_std": -0.012626999843920064, + "nauc_map_at_5_diff1": 0.2861299068739539, + "nauc_map_at_5_max": 0.27446625170508415, + "nauc_map_at_5_std": -0.0058971920193630345, + "nauc_mrr_at_1000_diff1": 0.5086922608252236, + "nauc_mrr_at_1000_max": 0.3691168740765083, + "nauc_mrr_at_1000_std": 0.0006666279755451657, + "nauc_mrr_at_100_diff1": 0.5086748494269067, + "nauc_mrr_at_100_max": 0.36919780738198626, + "nauc_mrr_at_100_std": 0.0007592322141292633, + "nauc_mrr_at_10_diff1": 0.5085215685619419, + "nauc_mrr_at_10_max": 0.3718177164350851, + "nauc_mrr_at_10_std": 0.0027147806825914855, + "nauc_mrr_at_1_diff1": 0.5216513467070573, + "nauc_mrr_at_1_max": 0.320769879618688, + "nauc_mrr_at_1_std": -0.019386687001355655, + "nauc_mrr_at_20_diff1": 0.5083421770335178, + "nauc_mrr_at_20_max": 0.37019594976409614, + "nauc_mrr_at_20_std": 0.0017330204692732748, + "nauc_mrr_at_3_diff1": 0.5078153733991984, + "nauc_mrr_at_3_max": 0.3668773645861323, + "nauc_mrr_at_3_std": -0.008641807033782771, + "nauc_mrr_at_5_diff1": 0.5088459578154817, + "nauc_mrr_at_5_max": 0.3736712576479569, + "nauc_mrr_at_5_std": -2.2624894928488478e-05, + "nauc_ndcg_at_1000_diff1": 0.3374277453741463, + "nauc_ndcg_at_1000_max": 0.3321734054378378, + "nauc_ndcg_at_1000_std": 0.03673244516019259, + "nauc_ndcg_at_100_diff1": 0.3337261963315334, + "nauc_ndcg_at_100_max": 0.3432535360812789, + "nauc_ndcg_at_100_std": 0.049838336252328606, + "nauc_ndcg_at_10_diff1": 0.31654227932875967, + "nauc_ndcg_at_10_max": 0.32618988165002455, + "nauc_ndcg_at_10_std": 0.029231011592918396, + "nauc_ndcg_at_1_diff1": 0.5235838276940898, + "nauc_ndcg_at_1_max": 0.31696736904058387, + "nauc_ndcg_at_1_std": -0.02458631928633079, + "nauc_ndcg_at_20_diff1": 0.3275249333002614, + "nauc_ndcg_at_20_max": 0.34158128877849375, + "nauc_ndcg_at_20_std": 0.04494651591965775, + "nauc_ndcg_at_3_diff1": 0.32117487638130354, + "nauc_ndcg_at_3_max": 0.2941028830915832, + "nauc_ndcg_at_3_std": -0.007316961948953446, + "nauc_ndcg_at_5_diff1": 0.31284200905978193, + "nauc_ndcg_at_5_max": 0.3063806097099824, + "nauc_ndcg_at_5_std": 0.005868083161393718, + "nauc_precision_at_1000_diff1": -0.03635225074805508, + "nauc_precision_at_1000_max": 0.16765441647254156, + "nauc_precision_at_1000_std": 0.20388495902082046, + "nauc_precision_at_100_diff1": 0.09225951347257724, + "nauc_precision_at_100_max": 0.3981187839667785, + "nauc_precision_at_100_std": 0.30543394344414393, + "nauc_precision_at_10_diff1": 0.16625472536477842, + "nauc_precision_at_10_max": 0.3229284843568491, + "nauc_precision_at_10_std": 0.0995415625260059, + "nauc_precision_at_1_diff1": 0.5235838276940898, + "nauc_precision_at_1_max": 0.31696736904058387, + "nauc_precision_at_1_std": -0.02458631928633079, + "nauc_precision_at_20_diff1": 0.17259384065055555, + "nauc_precision_at_20_max": 0.38127315704848685, + "nauc_precision_at_20_std": 0.1789475498025293, + "nauc_precision_at_3_diff1": 0.2321551182804366, + "nauc_precision_at_3_max": 0.2757072760399245, + "nauc_precision_at_3_std": -3.369463890472521e-05, + "nauc_precision_at_5_diff1": 0.19327155462501952, + "nauc_precision_at_5_max": 0.2859361592299849, + "nauc_precision_at_5_std": 0.029196569686768183, + "nauc_recall_at_1000_diff1": 0.05823231057513774, + "nauc_recall_at_1000_max": 0.46812391396558123, + "nauc_recall_at_1000_std": 0.46536941182767955, + "nauc_recall_at_100_diff1": 0.14677934784136976, + "nauc_recall_at_100_max": 0.5113587224498323, + "nauc_recall_at_100_std": 0.38163612647621464, + "nauc_recall_at_10_diff1": 0.17727845662053254, + "nauc_recall_at_10_max": 0.34133843118145557, + "nauc_recall_at_10_std": 0.10554021248418982, + "nauc_recall_at_1_diff1": 0.5159028191255995, + "nauc_recall_at_1_max": 0.3174029367347432, + "nauc_recall_at_1_std": -0.020320539407606746, + "nauc_recall_at_20_diff1": 0.1891750473785528, + "nauc_recall_at_20_max": 0.411016904961582, + "nauc_recall_at_20_std": 0.1909091714508319, + "nauc_recall_at_3_diff1": 0.23406001763218115, + "nauc_recall_at_3_max": 0.28163137574773756, + "nauc_recall_at_3_std": 0.00345900087694381, + "nauc_recall_at_5_diff1": 0.1994571814043668, + "nauc_recall_at_5_max": 0.29485386492986626, + "nauc_recall_at_5_std": 0.032509102635484136, + "ndcg_at_1": 0.62688, + "ndcg_at_10": 0.64907, + "ndcg_at_100": 0.69217, + "ndcg_at_1000": 0.70137, + "ndcg_at_20": 0.6693, + "ndcg_at_3": 0.5656, + "ndcg_at_5": 0.61022, + "precision_at_1": 0.62688, + "precision_at_10": 0.15181, + "precision_at_100": 0.01853, + "precision_at_1000": 0.00197, + "precision_at_20": 0.08242, + "precision_at_3": 0.38031, + "precision_at_5": 0.26403, + "recall_at_1": 0.31073, + "recall_at_10": 0.74932, + "recall_at_100": 0.91236, + "recall_at_1000": 0.97101, + "recall_at_20": 0.81344, + "recall_at_3": 0.56456, + "recall_at_5": 0.65238 + } + ] + }, + "task_name": "WebQAT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/model_meta.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/model_meta.json new file mode 100644 index 0000000000..cdc58dc7d4 --- /dev/null +++ b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/model_meta.json @@ -0,0 +1 @@ +{"name": "nomic-ai/nomic-embed-vision-v1.5", "revision": "af2246fffdab78d8458418480e4886a8e48b70a7", "release_date": "2024-06-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "NomicVisionModelWrapper"} \ No newline at end of file From 621af0cc672e4ca6169d298da11634312f441643 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Sat, 10 Aug 2024 08:08:17 +0100 Subject: [PATCH 042/154] add cv recall --- .../abstasks/Image/AbsTaskAny2AnyRetrieval.py | 3 +- mteb/abstasks/Image/AbsTaskI2TRetrieval.py | 3 +- mteb/abstasks/Image/AbsTaskT2IRetrieval.py | 3 +- .../Image/Any2AnyRetrievalEvaluator.py | 56 +++++++++++- .../evaluators/Image/I2TRetrievalEvaluator.py | 36 +++++++- .../evaluators/Image/T2IRetrievalEvaluator.py | 36 +++++++- ...TRetrieval.json => CIRRIT2IRetrieval.json} | 90 ++++++++++++------- .../MSCOCOI2TRetrieval.json | 32 ++++++- .../MSCOCOT2IRetrieval.json | 32 ++++++- .../NIGHTSI2IRetrieval.json | 38 ++++++-- 10 files changed, 274 insertions(+), 55 deletions(-) rename results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/{CIRRIT2TRetrieval.json => CIRRIT2IRetrieval.json} (66%) diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py index 64b10059df..a751af9ece 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py @@ -308,7 +308,7 @@ def _evaluate_subset( with open(qrels_save_path, "w") as f: json.dump(results, f) - ndcg, _map, recall, precision, naucs = retriever.evaluate( + ndcg, _map, recall, precision, cv_recall, naucs = retriever.evaluate( relevant_docs, results, retriever.k_values, @@ -321,6 +321,7 @@ def _evaluate_subset( **{f"ndcg_at_{k.split('@')[1]}": v for (k, v) in ndcg.items()}, **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()}, **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()}, + **{f"cv_recall_at_{k.split('@')[1]}": v for (k, v) in cv_recall.items()}, **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()}, **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, **{ diff --git a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py index c20faff64c..4920e77cb3 100644 --- a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py @@ -306,7 +306,7 @@ def _evaluate_subset( with open(qrels_save_path, "w") as f: json.dump(results, f) - ndcg, _map, recall, precision, naucs = retriever.evaluate( + ndcg, _map, recall, precision, cv_recall, naucs = retriever.evaluate( relevant_docs, results, retriever.k_values, @@ -320,6 +320,7 @@ def _evaluate_subset( **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()}, **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()}, **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()}, + **{f"cv_recall_at_{k.split('@')[1]}": v for (k, v) in cv_recall.items()}, **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, **{ k.replace("@", "_at_").replace("_P", "_precision").lower(): v diff --git a/mteb/abstasks/Image/AbsTaskT2IRetrieval.py b/mteb/abstasks/Image/AbsTaskT2IRetrieval.py index 99369bd1b3..7269abcdfa 100644 --- a/mteb/abstasks/Image/AbsTaskT2IRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskT2IRetrieval.py @@ -306,7 +306,7 @@ def _evaluate_subset( with open(qrels_save_path, "w") as f: json.dump(results, f) - ndcg, _map, recall, precision, naucs = retriever.evaluate( + ndcg, _map, recall, precision, cv_recall, naucs = retriever.evaluate( relevant_docs, results, retriever.k_values, @@ -320,6 +320,7 @@ def _evaluate_subset( **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()}, **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()}, **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()}, + **{f"cv_recall_at_{k.split('@')[1]}": v for (k, v) in cv_recall.items()}, **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, **{ k.replace("@", "_at_").replace("_P", "_precision").lower(): v diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index 467a24f969..123fad09eb 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -315,13 +315,20 @@ def evaluate( "For evaluation, we DO NOT ignore identical query and document ids (default), please explicitly set ``ignore_identical_ids=True`` to ignore this." ) - all_ndcgs, all_aps, all_recalls, all_precisions = {}, {}, {}, {} + all_ndcgs, all_aps, all_recalls, all_precisions, all_cv_recalls = ( + {}, + {}, + {}, + {}, + {}, + ) for k in k_values: all_ndcgs[f"NDCG@{k}"] = [] all_aps[f"MAP@{k}"] = [] all_recalls[f"Recall@{k}"] = [] all_precisions[f"P@{k}"] = [] + all_cv_recalls[f"CV_Recall@{k}"] = [] # (new) CV-style Recall map_string = "map_cut." + ",".join([str(k) for k in k_values]) ndcg_string = "ndcg_cut." + ",".join([str(k) for k in k_values]) @@ -332,18 +339,35 @@ def evaluate( ) scores = evaluator.evaluate(results) + sorted_results = { + qid: sorted(rels.items(), key=lambda item: item[1], reverse=True) + for qid, rels in results.items() + } + for query_id in scores.keys(): + top_docs = [ + doc_id for doc_id, _ in sorted_results.get(query_id, []) + ] # Sorted list of doc IDs + relevant_docs = set(qrels.get(query_id, {}).keys()) + for k in k_values: + top_k_docs = top_docs[:k] all_ndcgs[f"NDCG@{k}"].append(scores[query_id]["ndcg_cut_" + str(k)]) all_aps[f"MAP@{k}"].append(scores[query_id]["map_cut_" + str(k)]) all_recalls[f"Recall@{k}"].append(scores[query_id]["recall_" + str(k)]) all_precisions[f"P@{k}"].append(scores[query_id]["P_" + str(k)]) - ndcg, _map, recall, precision = ( + if relevant_docs.intersection(top_k_docs): + all_cv_recalls[f"CV_Recall@{k}"].append(1.0) + else: + all_cv_recalls[f"CV_Recall@{k}"].append(0.0) + + ndcg, _map, recall, precision, cv_recall = ( all_ndcgs.copy(), all_aps.copy(), all_recalls.copy(), all_precisions.copy(), + all_cv_recalls.copy(), ) for k in k_values: @@ -351,12 +375,16 @@ def evaluate( _map[f"MAP@{k}"] = round(sum(_map[f"MAP@{k}"]) / len(scores), 5) recall[f"Recall@{k}"] = round(sum(recall[f"Recall@{k}"]) / len(scores), 5) precision[f"P@{k}"] = round(sum(precision[f"P@{k}"]) / len(scores), 5) + cv_recall[f"CV_Recall@{k}"] = round( + sum(cv_recall[f"CV_Recall@{k}"]) / len(scores), 5 + ) naucs = Any2AnyRetrievalEvaluator.evaluate_abstention( - results, {**all_ndcgs, **all_aps, **all_recalls, **all_precisions} + results, + {**all_ndcgs, **all_aps, **all_recalls, **all_precisions, **all_cv_recalls}, ) - return ndcg, _map, recall, precision, naucs + return ndcg, _map, recall, precision, cv_recall, naucs @staticmethod def evaluate_custom( @@ -411,3 +439,23 @@ def evaluate_abstention( naucs[f"nAUC_{metric_name}_{fct}"] = nAUC(conf_scores, scores) return naucs + + @staticmethod + def calculate_cv_style_recall( + qrels: dict[str, dict[str, int]], results: dict[str, dict[str, float]], k: int + ) -> dict[str, float]: + """Calculate CV-style recall: Recall is 1 if any relevant document is + retrieved in the top k, otherwise 0. + """ + cv_recalls = {} + for query_id, relevant_docs in qrels.items(): + retrieved_docs = list(results.get(query_id, {}).keys())[ + :k + ] # Retrieve top k documents + if any(doc_id in relevant_docs for doc_id in retrieved_docs): + cv_recalls[query_id] = ( + 1.0 # If any relevant doc is found in top k, recall is 1 + ) + else: + cv_recalls[query_id] = 0.0 # Otherwise, recall is 0 + return cv_recalls diff --git a/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py index c0b1717315..f8c655e883 100644 --- a/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py @@ -262,13 +262,20 @@ def evaluate( "For evaluation, we DO NOT ignore identical query and document ids (default), please explicitly set ``ignore_identical_ids=True`` to ignore this." ) - all_ndcgs, all_aps, all_recalls, all_precisions = {}, {}, {}, {} + all_ndcgs, all_aps, all_recalls, all_precisions, all_cv_recalls = ( + {}, + {}, + {}, + {}, + {}, + ) for k in k_values: all_ndcgs[f"NDCG@{k}"] = [] all_aps[f"MAP@{k}"] = [] all_recalls[f"Recall@{k}"] = [] all_precisions[f"P@{k}"] = [] + all_cv_recalls[f"CV_Recall@{k}"] = [] # (new) CV-style Recall map_string = "map_cut." + ",".join([str(k) for k in k_values]) ndcg_string = "ndcg_cut." + ",".join([str(k) for k in k_values]) @@ -279,18 +286,35 @@ def evaluate( ) scores = evaluator.evaluate(results) + sorted_results = { + qid: sorted(rels.items(), key=lambda item: item[1], reverse=True) + for qid, rels in results.items() + } + for query_id in scores.keys(): + top_docs = [ + doc_id for doc_id, _ in sorted_results.get(query_id, []) + ] # Sorted list of doc IDs + relevant_docs = set(qrels.get(query_id, {}).keys()) + for k in k_values: + top_k_docs = top_docs[:k] all_ndcgs[f"NDCG@{k}"].append(scores[query_id]["ndcg_cut_" + str(k)]) all_aps[f"MAP@{k}"].append(scores[query_id]["map_cut_" + str(k)]) all_recalls[f"Recall@{k}"].append(scores[query_id]["recall_" + str(k)]) all_precisions[f"P@{k}"].append(scores[query_id]["P_" + str(k)]) - ndcg, _map, recall, precision = ( + if relevant_docs.intersection(top_k_docs): + all_cv_recalls[f"CV_Recall@{k}"].append(1.0) + else: + all_cv_recalls[f"CV_Recall@{k}"].append(0.0) + + ndcg, _map, recall, precision, cv_recall = ( all_ndcgs.copy(), all_aps.copy(), all_recalls.copy(), all_precisions.copy(), + all_cv_recalls.copy(), ) for k in k_values: @@ -298,12 +322,16 @@ def evaluate( _map[f"MAP@{k}"] = round(sum(_map[f"MAP@{k}"]) / len(scores), 5) recall[f"Recall@{k}"] = round(sum(recall[f"Recall@{k}"]) / len(scores), 5) precision[f"P@{k}"] = round(sum(precision[f"P@{k}"]) / len(scores), 5) + cv_recall[f"CV_Recall@{k}"] = round( + sum(cv_recall[f"CV_Recall@{k}"]) / len(scores), 5 + ) naucs = I2TRetrievalEvaluator.evaluate_abstention( - results, {**all_ndcgs, **all_aps, **all_recalls, **all_precisions} + results, + {**all_ndcgs, **all_aps, **all_recalls, **all_precisions, **all_cv_recalls}, ) - return ndcg, _map, recall, precision, naucs + return ndcg, _map, recall, precision, cv_recall, naucs @staticmethod def evaluate_custom( diff --git a/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py index 88f87e92e9..85dfd5d753 100644 --- a/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py @@ -262,13 +262,20 @@ def evaluate( "For evaluation, we DO NOT ignore identical query and document ids (default), please explicitly set ``ignore_identical_ids=True`` to ignore this." ) - all_ndcgs, all_aps, all_recalls, all_precisions = {}, {}, {}, {} + all_ndcgs, all_aps, all_recalls, all_precisions, all_cv_recalls = ( + {}, + {}, + {}, + {}, + {}, + ) for k in k_values: all_ndcgs[f"NDCG@{k}"] = [] all_aps[f"MAP@{k}"] = [] all_recalls[f"Recall@{k}"] = [] all_precisions[f"P@{k}"] = [] + all_cv_recalls[f"CV_Recall@{k}"] = [] # (new) CV-style Recall map_string = "map_cut." + ",".join([str(k) for k in k_values]) ndcg_string = "ndcg_cut." + ",".join([str(k) for k in k_values]) @@ -279,18 +286,35 @@ def evaluate( ) scores = evaluator.evaluate(results) + sorted_results = { + qid: sorted(rels.items(), key=lambda item: item[1], reverse=True) + for qid, rels in results.items() + } + for query_id in scores.keys(): + top_docs = [ + doc_id for doc_id, _ in sorted_results.get(query_id, []) + ] # Sorted list of doc IDs + relevant_docs = set(qrels.get(query_id, {}).keys()) + for k in k_values: + top_k_docs = top_docs[:k] all_ndcgs[f"NDCG@{k}"].append(scores[query_id]["ndcg_cut_" + str(k)]) all_aps[f"MAP@{k}"].append(scores[query_id]["map_cut_" + str(k)]) all_recalls[f"Recall@{k}"].append(scores[query_id]["recall_" + str(k)]) all_precisions[f"P@{k}"].append(scores[query_id]["P_" + str(k)]) - ndcg, _map, recall, precision = ( + if relevant_docs.intersection(top_k_docs): + all_cv_recalls[f"CV_Recall@{k}"].append(1.0) + else: + all_cv_recalls[f"CV_Recall@{k}"].append(0.0) + + ndcg, _map, recall, precision, cv_recall = ( all_ndcgs.copy(), all_aps.copy(), all_recalls.copy(), all_precisions.copy(), + all_cv_recalls.copy(), ) for k in k_values: @@ -298,12 +322,16 @@ def evaluate( _map[f"MAP@{k}"] = round(sum(_map[f"MAP@{k}"]) / len(scores), 5) recall[f"Recall@{k}"] = round(sum(recall[f"Recall@{k}"]) / len(scores), 5) precision[f"P@{k}"] = round(sum(precision[f"P@{k}"]) / len(scores), 5) + cv_recall[f"CV_Recall@{k}"] = round( + sum(cv_recall[f"CV_Recall@{k}"]) / len(scores), 5 + ) naucs = T2IRetrievalEvaluator.evaluate_abstention( - results, {**all_ndcgs, **all_aps, **all_recalls, **all_precisions} + results, + {**all_ndcgs, **all_aps, **all_recalls, **all_precisions, **all_cv_recalls}, ) - return ndcg, _map, recall, precision, naucs + return ndcg, _map, recall, precision, cv_recall, naucs @staticmethod def evaluate_custom( diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2IRetrieval.json similarity index 66% rename from results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2TRetrieval.json rename to results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2IRetrieval.json index 428f0fac5b..0dbe5112b4 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2TRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2IRetrieval.json @@ -1,11 +1,18 @@ { "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", - "evaluation_time": 76.20298647880554, + "evaluation_time": 88.57046842575073, "kg_co2_emissions": null, - "mteb_version": "1.12.67", + "mteb_version": "1.12.84", "scores": { "test": [ { + "cv_recall_at_1": 0.00815, + "cv_recall_at_10": 0.21295, + "cv_recall_at_100": 0.53357, + "cv_recall_at_1000": 0.86643, + "cv_recall_at_20": 0.28921, + "cv_recall_at_3": 0.09065, + "cv_recall_at_5": 0.13861, "hf_subset": "default", "languages": [ "eng-Latn" @@ -18,13 +25,34 @@ "map_at_20": 0.07065, "map_at_3": 0.04442, "map_at_5": 0.05543, - "mrr_at_1": 0.008872901678657074, - "mrr_at_10": 0.06594115184043225, - "mrr_at_100": 0.07663903441028781, - "mrr_at_1000": 0.07809622231266924, - "mrr_at_20": 0.07118418253647203, - "mrr_at_3": 0.044924060751398966, - "mrr_at_5": 0.05594324540367709, + "mrr_at_1": 0.00815347721822542, + "mrr_at_10": 0.06562540443835395, + "mrr_at_100": 0.0763215755779849, + "mrr_at_1000": 0.07777876289048734, + "mrr_at_20": 0.07086673474032301, + "mrr_at_3": 0.04468425259792174, + "mrr_at_5": 0.0556314948041567, + "nauc_cv_recall_at_1000_diff1": -0.1429646050677094, + "nauc_cv_recall_at_1000_max": 0.1924743520635352, + "nauc_cv_recall_at_1000_std": 0.5456463071997951, + "nauc_cv_recall_at_100_diff1": -0.042346564842556896, + "nauc_cv_recall_at_100_max": 0.1612547792092197, + "nauc_cv_recall_at_100_std": 0.2725592737752734, + "nauc_cv_recall_at_10_diff1": -0.05093849270187104, + "nauc_cv_recall_at_10_max": 0.11611298165158403, + "nauc_cv_recall_at_10_std": 0.12039571371818639, + "nauc_cv_recall_at_1_diff1": -0.516982591265645, + "nauc_cv_recall_at_1_max": 0.17146735270443028, + "nauc_cv_recall_at_1_std": -0.044358338988936034, + "nauc_cv_recall_at_20_diff1": -0.04728747828920591, + "nauc_cv_recall_at_20_max": 0.13714658323389076, + "nauc_cv_recall_at_20_std": 0.13725377667220118, + "nauc_cv_recall_at_3_diff1": -0.06302926023881217, + "nauc_cv_recall_at_3_max": 0.1113801332475869, + "nauc_cv_recall_at_3_std": 0.061263225738675126, + "nauc_cv_recall_at_5_diff1": -0.04626447590534941, + "nauc_cv_recall_at_5_max": 0.10547386265685323, + "nauc_cv_recall_at_5_std": 0.08783936831294922, "nauc_map_at_1000_diff1": -0.08655636473413195, "nauc_map_at_1000_max": 0.11676320143589596, "nauc_map_at_1000_std": 0.09629099312167273, @@ -46,27 +74,27 @@ "nauc_map_at_5_diff1": -0.0903941122012634, "nauc_map_at_5_max": 0.106932714398111, "nauc_map_at_5_std": 0.06656522457054902, - "nauc_mrr_at_1000_diff1": -0.0907646058253511, - "nauc_mrr_at_1000_max": 0.12018754710642203, - "nauc_mrr_at_1000_std": 0.09796107072305628, - "nauc_mrr_at_100_diff1": -0.08936189065625093, - "nauc_mrr_at_100_max": 0.1202619669752152, - "nauc_mrr_at_100_std": 0.09620129145114814, - "nauc_mrr_at_10_diff1": -0.0919795422688847, - "nauc_mrr_at_10_max": 0.11332346465731229, - "nauc_mrr_at_10_std": 0.08317876347162799, - "nauc_mrr_at_1_diff1": -0.5216644065584233, - "nauc_mrr_at_1_max": 0.20209718707169744, - "nauc_mrr_at_1_std": -0.005366246310705697, - "nauc_mrr_at_20_diff1": -0.09013629049462879, - "nauc_mrr_at_20_max": 0.1184401494147931, - "nauc_mrr_at_20_std": 0.08802180434971997, - "nauc_mrr_at_3_diff1": -0.11096146519530939, - "nauc_mrr_at_3_max": 0.11435332435139552, - "nauc_mrr_at_3_std": 0.0525013678199003, - "nauc_mrr_at_5_diff1": -0.09504839521625083, - "nauc_mrr_at_5_max": 0.11098993126230935, - "nauc_mrr_at_5_std": 0.06863872006792304, + "nauc_mrr_at_1000_diff1": -0.0878544053521387, + "nauc_mrr_at_1000_max": 0.11835254552849916, + "nauc_mrr_at_1000_std": 0.09563928573449795, + "nauc_mrr_at_100_diff1": -0.08646539076860782, + "nauc_mrr_at_100_max": 0.11844073105135247, + "nauc_mrr_at_100_std": 0.0938876335450531, + "nauc_mrr_at_10_diff1": -0.08899929752404169, + "nauc_mrr_at_10_max": 0.11141118460268691, + "nauc_mrr_at_10_std": 0.08072528807311119, + "nauc_mrr_at_1_diff1": -0.516982591265645, + "nauc_mrr_at_1_max": 0.17146735270443028, + "nauc_mrr_at_1_std": -0.044358338988936034, + "nauc_mrr_at_20_diff1": -0.08723030616085735, + "nauc_mrr_at_20_max": 0.11660026484773199, + "nauc_mrr_at_20_std": 0.08564973677674821, + "nauc_mrr_at_3_diff1": -0.10793762866631498, + "nauc_mrr_at_3_max": 0.11188839551635246, + "nauc_mrr_at_3_std": 0.04886788042390466, + "nauc_mrr_at_5_diff1": -0.09176834345451616, + "nauc_mrr_at_5_max": 0.10901080188380181, + "nauc_mrr_at_5_std": 0.06597097726127213, "nauc_ndcg_at_1000_diff1": -0.08344132954414758, "nauc_ndcg_at_1000_max": 0.1323215007612839, "nauc_ndcg_at_1000_std": 0.17697700366712762, @@ -154,5 +182,5 @@ } ] }, - "task_name": "CIRRIT2TRetrieval" + "task_name": "CIRRIT2IRetrieval" } \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json index 1476975b8f..d71a300c97 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json @@ -1,11 +1,18 @@ { "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", - "evaluation_time": 24.560455560684204, + "evaluation_time": 31.064056158065796, "kg_co2_emissions": null, - "mteb_version": "1.12.67", + "mteb_version": "1.12.84", "scores": { "test": [ { + "cv_recall_at_1": 0.5016, + "cv_recall_at_10": 0.8362, + "cv_recall_at_100": 0.9864, + "cv_recall_at_1000": 0.9998, + "cv_recall_at_20": 0.9078, + "cv_recall_at_3": 0.6744, + "cv_recall_at_5": 0.747, "hf_subset": "default", "languages": [ "eng-Latn" @@ -25,6 +32,27 @@ "mrr_at_20": 0.6120301861073234, "mrr_at_3": 0.5781000000000018, "mrr_at_5": 0.5949800000000011, + "nauc_cv_recall_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.4323474487834336, + "nauc_cv_recall_at_100_max": 0.3997912890646433, + "nauc_cv_recall_at_100_std": -0.1196037238424793, + "nauc_cv_recall_at_10_diff1": 0.37501696719115774, + "nauc_cv_recall_at_10_max": 0.3127110388245365, + "nauc_cv_recall_at_10_std": -0.3305046673289374, + "nauc_cv_recall_at_1_diff1": 0.4269635838909405, + "nauc_cv_recall_at_1_max": 0.3359596131075358, + "nauc_cv_recall_at_1_std": -0.29597124038134076, + "nauc_cv_recall_at_20_diff1": 0.3767446240969275, + "nauc_cv_recall_at_20_max": 0.33553088625182514, + "nauc_cv_recall_at_20_std": -0.37383008156263287, + "nauc_cv_recall_at_3_diff1": 0.3859951165456735, + "nauc_cv_recall_at_3_max": 0.3344155089983177, + "nauc_cv_recall_at_3_std": -0.33527359778317994, + "nauc_cv_recall_at_5_diff1": 0.3725036329450592, + "nauc_cv_recall_at_5_max": 0.3198504947231606, + "nauc_cv_recall_at_5_std": -0.35528274781245933, "nauc_map_at_1000_diff1": 0.28351782466117825, "nauc_map_at_1000_max": 0.3661635018482055, "nauc_map_at_1000_std": -0.2823328432610589, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json index 2925dd6776..d3eb8cb7bb 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json @@ -1,11 +1,18 @@ { "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", - "evaluation_time": 48.408275842666626, + "evaluation_time": 64.15870261192322, "kg_co2_emissions": null, - "mteb_version": "1.12.67", + "mteb_version": "1.12.84", "scores": { "test": [ { + "cv_recall_at_1": 0.3219, + "cv_recall_at_10": 0.69043, + "cv_recall_at_100": 0.96699, + "cv_recall_at_1000": 0.99802, + "cv_recall_at_20": 0.78975, + "cv_recall_at_3": 0.49833, + "cv_recall_at_5": 0.57894, "hf_subset": "default", "languages": [ "eng-Latn" @@ -25,6 +32,27 @@ "mrr_at_20": 0.43955600062406863, "mrr_at_3": 0.3991898101495421, "mrr_at_5": 0.41756015961949616, + "nauc_cv_recall_at_1000_diff1": 0.6908755779988776, + "nauc_cv_recall_at_1000_max": 0.8673002265484557, + "nauc_cv_recall_at_1000_std": 0.9557913979496387, + "nauc_cv_recall_at_100_diff1": 0.3857607701244185, + "nauc_cv_recall_at_100_max": 0.515578931619228, + "nauc_cv_recall_at_100_std": 0.26327215392208675, + "nauc_cv_recall_at_10_diff1": 0.4056733846252647, + "nauc_cv_recall_at_10_max": 0.2300820556651551, + "nauc_cv_recall_at_10_std": -0.24485753427069776, + "nauc_cv_recall_at_1_diff1": 0.5511594051722437, + "nauc_cv_recall_at_1_max": 0.25830642727292813, + "nauc_cv_recall_at_1_std": -0.21261765560734627, + "nauc_cv_recall_at_20_diff1": 0.4094495789026411, + "nauc_cv_recall_at_20_max": 0.24534623596758162, + "nauc_cv_recall_at_20_std": -0.23697708049579708, + "nauc_cv_recall_at_3_diff1": 0.43694665288793116, + "nauc_cv_recall_at_3_max": 0.2252768452995081, + "nauc_cv_recall_at_3_std": -0.2370109213521609, + "nauc_cv_recall_at_5_diff1": 0.4117629943006741, + "nauc_cv_recall_at_5_max": 0.21644561968880394, + "nauc_cv_recall_at_5_std": -0.24928700760880004, "nauc_map_at_1000_diff1": 0.49930616061028704, "nauc_map_at_1000_max": 0.24685433707080706, "nauc_map_at_1000_std": -0.22328503812884298, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json index 06fb590415..0f91c2dce8 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json @@ -1,11 +1,18 @@ { "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", - "evaluation_time": 140.29364442825317, + "evaluation_time": 125.03077864646912, "kg_co2_emissions": null, "mteb_version": "1.12.84", "scores": { "test": [ { + "cv_recall_at_1": 0.07736, + "cv_recall_at_10": 0.44057, + "cv_recall_at_100": 0.91085, + "cv_recall_at_1000": 0.98396, + "cv_recall_at_20": 0.6283, + "cv_recall_at_3": 0.18962, + "cv_recall_at_5": 0.28255, "hf_subset": "default", "languages": [ "eng-Latn" @@ -21,10 +28,31 @@ "mrr_at_1": 0.07735849056603773, "mrr_at_10": 0.16638982479784364, "mrr_at_100": 0.18754037945219226, - "mrr_at_1000": 0.18790557367103924, + "mrr_at_1000": 0.18790557532212301, "mrr_at_20": 0.179393828269821, "mrr_at_3": 0.12468553459119512, "mrr_at_5": 0.14560534591194946, + "nauc_cv_recall_at_1000_diff1": -0.03953149887405362, + "nauc_cv_recall_at_1000_max": 0.5072636897896413, + "nauc_cv_recall_at_1000_std": 0.7267946394243926, + "nauc_cv_recall_at_100_diff1": -0.0815659597172199, + "nauc_cv_recall_at_100_max": 0.4456795063704503, + "nauc_cv_recall_at_100_std": 0.5793601391173749, + "nauc_cv_recall_at_10_diff1": 0.00024134988468162648, + "nauc_cv_recall_at_10_max": 0.0714549580777712, + "nauc_cv_recall_at_10_std": 0.001055290057000327, + "nauc_cv_recall_at_1_diff1": 0.10866321569038064, + "nauc_cv_recall_at_1_max": -0.04068530891909402, + "nauc_cv_recall_at_1_std": -0.09376533797535455, + "nauc_cv_recall_at_20_diff1": -0.04957649222390187, + "nauc_cv_recall_at_20_max": 0.1409364325732594, + "nauc_cv_recall_at_20_std": 0.04479103118396537, + "nauc_cv_recall_at_3_diff1": 0.08256318842375683, + "nauc_cv_recall_at_3_max": -0.021108881570062934, + "nauc_cv_recall_at_3_std": -0.03800758467622141, + "nauc_cv_recall_at_5_diff1": 0.07592926310375617, + "nauc_cv_recall_at_5_max": 0.0008620050605076114, + "nauc_cv_recall_at_5_std": -0.026056300607536588, "nauc_map_at_1000_diff1": 0.07235874903557624, "nauc_map_at_1000_max": 0.0012495691831918834, "nauc_map_at_1000_std": -0.04266110409531828, @@ -46,9 +74,9 @@ "nauc_map_at_5_diff1": 0.09173894865921334, "nauc_map_at_5_max": -0.019068286969482498, "nauc_map_at_5_std": -0.055410455100472364, - "nauc_mrr_at_1000_diff1": 0.07235875136193966, - "nauc_mrr_at_1000_max": 0.0012495773151879567, - "nauc_mrr_at_1000_std": -0.042661096670129864, + "nauc_mrr_at_1000_diff1": 0.07235874903557624, + "nauc_mrr_at_1000_max": 0.0012495691831918834, + "nauc_mrr_at_1000_std": -0.04266110409531828, "nauc_mrr_at_100_diff1": 0.07206445308000643, "nauc_mrr_at_100_max": 0.0019260185802715023, "nauc_mrr_at_100_std": -0.04168632470080018, From 2631eaa930c4b62a52fc2a3f0ee7d1d3ddffba6a Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Sat, 10 Aug 2024 08:10:36 +0100 Subject: [PATCH 043/154] add InfoSeek; VisualNews --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 4 + .../eng/InfoSeekIT2ITRetrieval.py | 52 +++++ .../eng/InfoSeekIT2TRetrieval.py | 52 +++++ .../eng/OVENIT2ITRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/OVENIT2TRetrieval.py | 2 +- .../eng/VisualNewsI2TRetrieval.py | 51 +++++ .../eng/VisualNewsT2IRetrieval.py | 51 +++++ .../InfoSeekIT2ITRetrieval.json | 186 ++++++++++++++++++ .../InfoSeekIT2TRetrieval.json | 186 ++++++++++++++++++ .../VisualNewsI2TRetrieval.json | 186 ++++++++++++++++++ .../VisualNewsT2IRetrieval.json | 186 ++++++++++++++++++ 11 files changed, 956 insertions(+), 2 deletions(-) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2ITRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsT2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 828013191e..3f5487a503 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -2,8 +2,12 @@ from .eng.CIRRIT2IRetrieval import * from .eng.FashionIQIT2IRetrieval import * +from .eng.InfoSeekIT2ITRetrieval import * +from .eng.InfoSeekIT2TRetrieval import * from .eng.NIGHTSI2IRetrieval import * from .eng.OVENIT2ITRetrieval import * from .eng.OVENIT2TRetrieval import * +from .eng.VisualNewsI2TRetrieval import * +from .eng.VisualNewsT2IRetrieval import * from .eng.WebQAT2ITRetrieval import * from .eng.WebQAT2TRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py new file mode 100644 index 0000000000..e34ddc5563 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class InfoSeekIT2ITRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="InfoSeekIT2ITRetrieval", + description="Retrieve source text and image information to answer questions about images.", + reference="https://aclanthology.org/2023.emnlp-main.925", + dataset={ + "path": "MRBench/mbeir_infoseek_task8", + "revision": "78ee7f7708aac75d3afac5dcab1c9e03cb62664c", + "trust_remote_code": True, + }, + type="Retrieval", + category="it2it", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2023-01-01", "2023-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@inproceedings{chen2023can, + title={Can Pre-trained Vision and Language Models Answer Visual Information-Seeking Questions?}, + author={Chen, Yang and Hu, Hexiang and Luan, Yi and Sun, Haitian and Changpinyo, Soravit and Ritter, Alan and Chang, Ming-Wei}, + booktitle={Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing}, + pages={14948--14968}, + year={2023} +}""", + descriptive_stats={ + "n_samples": {"test": 17593}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 481782, + "num_queries": 17593, + "average_relevant_docs_per_query": 7.5, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py new file mode 100644 index 0000000000..2932a5f6c9 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class InfoSeekIT2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="InfoSeekIT2TRetrieval", + description="Retrieve source information to answer questions about images.", + reference="https://aclanthology.org/2023.emnlp-main.925", + dataset={ + "path": "MRBench/mbeir_infoseek_task6", + "revision": "d4f4606f7a42bbf311c2957419ef3734fe81c47f", + "trust_remote_code": True, + }, + type="Retrieval", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2023-01-01", "2023-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@inproceedings{chen2023can, + title={Can Pre-trained Vision and Language Models Answer Visual Information-Seeking Questions?}, + author={Chen, Yang and Hu, Hexiang and Luan, Yi and Sun, Haitian and Changpinyo, Soravit and Ritter, Alan and Chang, Ming-Wei}, + booktitle={Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing}, + pages={14948--14968}, + year={2023} +}""", + descriptive_stats={ + "n_samples": {"test": 11323}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 611651, + "num_queries": 11323, + "average_relevant_docs_per_query": 6.5, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py index 429b33e6a0..7c425c1103 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py @@ -27,7 +27,7 @@ class OVENIT2ITRetrieval(AbsTaskAny2AnyRetrieval): socioeconomic_status="medium", annotations_creators="derived", dialect=[], - modalities=["image"], + modalities=["image", "text"], sample_creation="created", bibtex_citation="""@inproceedings{hu2023open, title={Open-domain visual entity recognition: Towards recognizing millions of wikipedia entities}, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py index d60b5b275b..c8d90fe636 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py @@ -27,7 +27,7 @@ class OVENIT2TRetrieval(AbsTaskAny2AnyRetrieval): socioeconomic_status="medium", annotations_creators="derived", dialect=[], - modalities=["image"], + modalities=["text"], sample_creation="created", bibtex_citation="""@inproceedings{hu2023open, title={Open-domain visual entity recognition: Towards recognizing millions of wikipedia entities}, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py new file mode 100644 index 0000000000..b3c80e9007 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class VisualNewsI2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VisualNewsI2TRetrieval", + description="Retrieval entity-rich captions for news images.", + reference="https://aclanthology.org/2021.emnlp-main.542/", + dataset={ + "path": "MRBench/mbeir_visualnews_task3", + "revision": "aaee58895a66e4d619168849267ed2bb40d37043", + }, + type="Retrieval", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2020-01-01", "2020-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@inproceedings{liu2021visual, + title={Visual News: Benchmark and Challenges in News Image Captioning}, + author={Liu, Fuxiao and Wang, Yinghan and Wang, Tianlu and Ordonez, Vicente}, + booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing}, + pages={6761--6771}, + year={2021} +}""", + descriptive_stats={ + "n_samples": {"test": 20000}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 537568, + "num_queries": 20000, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py new file mode 100644 index 0000000000..2324a3f2ba --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class VisualNewsT2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VisualNewsT2IRetrieval", + description="Retrieve news images with captions.", + reference="https://aclanthology.org/2021.emnlp-main.542/", + dataset={ + "path": "MRBench/mbeir_visualnews_task0", + "revision": "94c519d850dba2b0058c2fc9b5da6142a59aa285", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2020-01-01", "2020-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@inproceedings{liu2021visual, + title={Visual News: Benchmark and Challenges in News Image Captioning}, + author={Liu, Fuxiao and Wang, Yinghan and Wang, Tianlu and Ordonez, Vicente}, + booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing}, + pages={6761--6771}, + year={2021} +}""", + descriptive_stats={ + "n_samples": {"test": 19995}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 542246, + "num_queries": 19995, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2ITRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2ITRetrieval.json new file mode 100644 index 0000000000..c7dc42a9cd --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2ITRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "78ee7f7708aac75d3afac5dcab1c9e03cb62664c", + "evaluation_time": 1977.6153802871704, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.02302, + "cv_recall_at_10": 0.1159, + "cv_recall_at_100": 0.33792, + "cv_recall_at_1000": 0.64213, + "cv_recall_at_20": 0.17109, + "cv_recall_at_3": 0.05281, + "cv_recall_at_5": 0.07401, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.03561, + "map_at_1": 0.00363, + "map_at_10": 0.0159, + "map_at_100": 0.03009, + "map_at_1000": 0.03503, + "map_at_20": 0.02049, + "map_at_3": 0.00821, + "map_at_5": 0.01103, + "mrr_at_1": 0.023020519524811003, + "mrr_at_10": 0.045806435459990955, + "mrr_at_100": 0.05351402487304762, + "mrr_at_1000": 0.05465984282946314, + "mrr_at_20": 0.04955566772605788, + "mrr_at_3": 0.03543075844559389, + "mrr_at_5": 0.040216752875196694, + "nauc_cv_recall_at_1000_diff1": 0.02015559920712523, + "nauc_cv_recall_at_1000_max": 0.24843256485471527, + "nauc_cv_recall_at_1000_std": 0.551074214857492, + "nauc_cv_recall_at_100_diff1": -0.013440078556748954, + "nauc_cv_recall_at_100_max": 0.08259437484215827, + "nauc_cv_recall_at_100_std": 0.3728506524892255, + "nauc_cv_recall_at_10_diff1": -0.010908853697860047, + "nauc_cv_recall_at_10_max": 0.03265302320587433, + "nauc_cv_recall_at_10_std": 0.2912474705012547, + "nauc_cv_recall_at_1_diff1": -0.07459591752948071, + "nauc_cv_recall_at_1_max": 0.1041413767622372, + "nauc_cv_recall_at_1_std": 0.35757281008095554, + "nauc_cv_recall_at_20_diff1": -0.012006431352003777, + "nauc_cv_recall_at_20_max": 0.03928626639631902, + "nauc_cv_recall_at_20_std": 0.3109439121223359, + "nauc_cv_recall_at_3_diff1": -0.017901157268070282, + "nauc_cv_recall_at_3_max": 0.04809033608912219, + "nauc_cv_recall_at_3_std": 0.33002426201564206, + "nauc_cv_recall_at_5_diff1": -0.0005855123496424908, + "nauc_cv_recall_at_5_max": 0.04584585149215492, + "nauc_cv_recall_at_5_std": 0.3091096240828216, + "nauc_map_at_1000_diff1": -0.024361704214272237, + "nauc_map_at_1000_max": 0.09242420909635184, + "nauc_map_at_1000_std": 0.4217296486267511, + "nauc_map_at_100_diff1": -0.021467259941218432, + "nauc_map_at_100_max": 0.09288371297474067, + "nauc_map_at_100_std": 0.39700462700388234, + "nauc_map_at_10_diff1": -0.01621091913919308, + "nauc_map_at_10_max": 0.10737421656077656, + "nauc_map_at_10_std": 0.2883212294123066, + "nauc_map_at_1_diff1": -0.11122157215775409, + "nauc_map_at_1_max": 0.11283092055308541, + "nauc_map_at_1_std": 0.3101815193459182, + "nauc_map_at_20_diff1": -0.0132414014004414, + "nauc_map_at_20_max": 0.10305956628322452, + "nauc_map_at_20_std": 0.32172528986885807, + "nauc_map_at_3_diff1": -0.04703922992329759, + "nauc_map_at_3_max": 0.09191500246547152, + "nauc_map_at_3_std": 0.2776262040824528, + "nauc_map_at_5_diff1": -0.014823044890154886, + "nauc_map_at_5_max": 0.11547342983958507, + "nauc_map_at_5_std": 0.2786259705268041, + "nauc_mrr_at_1000_diff1": -0.025295693852781534, + "nauc_mrr_at_1000_max": 0.06665675577708217, + "nauc_mrr_at_1000_std": 0.3306288852027639, + "nauc_mrr_at_100_diff1": -0.025881583161203855, + "nauc_mrr_at_100_max": 0.06461279535497533, + "nauc_mrr_at_100_std": 0.32831954836747484, + "nauc_mrr_at_10_diff1": -0.026092451126092788, + "nauc_mrr_at_10_max": 0.061697627022831616, + "nauc_mrr_at_10_std": 0.3212451183051902, + "nauc_mrr_at_1_diff1": -0.07459591752948071, + "nauc_mrr_at_1_max": 0.1041413767622372, + "nauc_mrr_at_1_std": 0.35757281008095554, + "nauc_mrr_at_20_diff1": -0.025821452727717447, + "nauc_mrr_at_20_max": 0.061755410936926804, + "nauc_mrr_at_20_std": 0.32393209801181, + "nauc_mrr_at_3_diff1": -0.03487019953781371, + "nauc_mrr_at_3_max": 0.07256664246126977, + "nauc_mrr_at_3_std": 0.3399856220056522, + "nauc_mrr_at_5_diff1": -0.025993114561723766, + "nauc_mrr_at_5_max": 0.06852797283441574, + "nauc_mrr_at_5_std": 0.3305774742662973, + "nauc_ndcg_at_1000_diff1": -0.009096662902559042, + "nauc_ndcg_at_1000_max": 0.11414272124177291, + "nauc_ndcg_at_1000_std": 0.44903775651649097, + "nauc_ndcg_at_100_diff1": -0.0134914554783792, + "nauc_ndcg_at_100_max": 0.08546139791128299, + "nauc_ndcg_at_100_std": 0.39209880947251335, + "nauc_ndcg_at_10_diff1": -0.0170068734005736, + "nauc_ndcg_at_10_max": 0.07471253375985419, + "nauc_ndcg_at_10_std": 0.33715358748076135, + "nauc_ndcg_at_1_diff1": -0.07459591752948071, + "nauc_ndcg_at_1_max": 0.1041413767622372, + "nauc_ndcg_at_1_std": 0.35757281008095554, + "nauc_ndcg_at_20_diff1": -0.01646794403371852, + "nauc_ndcg_at_20_max": 0.07852251935576997, + "nauc_ndcg_at_20_std": 0.3362314720735628, + "nauc_ndcg_at_3_diff1": -0.040316335382471105, + "nauc_ndcg_at_3_max": 0.06681861033932048, + "nauc_ndcg_at_3_std": 0.34481018084001014, + "nauc_ndcg_at_5_diff1": -0.01906600466599756, + "nauc_ndcg_at_5_max": 0.07855526607330436, + "nauc_ndcg_at_5_std": 0.35145622550348826, + "nauc_precision_at_1000_diff1": -0.04214581973882055, + "nauc_precision_at_1000_max": 0.054818857222616854, + "nauc_precision_at_1000_std": 0.4210937554531033, + "nauc_precision_at_100_diff1": -0.04404957111240489, + "nauc_precision_at_100_max": 0.048214060241998896, + "nauc_precision_at_100_std": 0.5022884634473646, + "nauc_precision_at_10_diff1": -0.02670920513820409, + "nauc_precision_at_10_max": 0.0524986708658414, + "nauc_precision_at_10_std": 0.38931841492231234, + "nauc_precision_at_1_diff1": -0.07459591752948071, + "nauc_precision_at_1_max": 0.1041413767622372, + "nauc_precision_at_1_std": 0.35757281008095554, + "nauc_precision_at_20_diff1": -0.03507168094166081, + "nauc_precision_at_20_max": 0.03854551370879106, + "nauc_precision_at_20_std": 0.4348216482299613, + "nauc_precision_at_3_diff1": -0.03182756465463996, + "nauc_precision_at_3_max": 0.06796130945038668, + "nauc_precision_at_3_std": 0.3375923259123569, + "nauc_precision_at_5_diff1": -0.017097663262845838, + "nauc_precision_at_5_max": 0.06997218653624815, + "nauc_precision_at_5_std": 0.3613890823731953, + "nauc_recall_at_1000_diff1": 0.01878544994568831, + "nauc_recall_at_1000_max": 0.19492400747658695, + "nauc_recall_at_1000_std": 0.4947445992310574, + "nauc_recall_at_100_diff1": 0.0004998682480571065, + "nauc_recall_at_100_max": 0.10307683961504077, + "nauc_recall_at_100_std": 0.3729176792809507, + "nauc_recall_at_10_diff1": 0.012273865896795341, + "nauc_recall_at_10_max": 0.0923729011236101, + "nauc_recall_at_10_std": 0.2597789820846874, + "nauc_recall_at_1_diff1": -0.11122157215775409, + "nauc_recall_at_1_max": 0.11283092055308541, + "nauc_recall_at_1_std": 0.3101815193459182, + "nauc_recall_at_20_diff1": 0.009083054016587643, + "nauc_recall_at_20_max": 0.09732442344740161, + "nauc_recall_at_20_std": 0.28782323808504057, + "nauc_recall_at_3_diff1": -0.020078445600854207, + "nauc_recall_at_3_max": 0.05741724146915211, + "nauc_recall_at_3_std": 0.27282304342784597, + "nauc_recall_at_5_diff1": 0.029128946277282092, + "nauc_recall_at_5_max": 0.10730405854943821, + "nauc_recall_at_5_std": 0.26884834591228696, + "ndcg_at_1": 0.02302, + "ndcg_at_10": 0.03561, + "ndcg_at_100": 0.08069, + "ndcg_at_1000": 0.13787, + "ndcg_at_20": 0.04519, + "ndcg_at_3": 0.0251, + "ndcg_at_5": 0.02823, + "precision_at_1": 0.02302, + "precision_at_10": 0.02506, + "precision_at_100": 0.01596, + "precision_at_1000": 0.00403, + "precision_at_20": 0.02346, + "precision_at_3": 0.02384, + "precision_at_5": 0.02445, + "recall_at_1": 0.00363, + "recall_at_10": 0.04202, + "recall_at_100": 0.19135, + "recall_at_1000": 0.45742, + "recall_at_20": 0.07132, + "recall_at_3": 0.01285, + "recall_at_5": 0.02134 + } + ] + }, + "task_name": "InfoSeekIT2ITRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2TRetrieval.json new file mode 100644 index 0000000000..1819e46926 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "d4f4606f7a42bbf311c2957419ef3734fe81c47f", + "evaluation_time": 556.8317360877991, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.00185, + "cv_recall_at_10": 0.00742, + "cv_recall_at_100": 0.02994, + "cv_recall_at_1000": 0.08938, + "cv_recall_at_20": 0.01263, + "cv_recall_at_3": 0.00344, + "cv_recall_at_5": 0.00468, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.00178, + "map_at_1": 0.00031, + "map_at_10": 0.00101, + "map_at_100": 0.00121, + "map_at_1000": 0.00136, + "map_at_20": 0.00108, + "map_at_3": 0.00072, + "map_at_5": 0.0009, + "mrr_at_1": 0.0018546321646206836, + "mrr_at_10": 0.0031619866292656195, + "mrr_at_100": 0.0038811488094678136, + "mrr_at_1000": 0.004054925968264824, + "mrr_at_20": 0.0035053863113972037, + "mrr_at_3": 0.002546439400629986, + "mrr_at_5": 0.0028246342253230882, + "nauc_cv_recall_at_1000_diff1": 0.016795257748445515, + "nauc_cv_recall_at_1000_max": -0.11000237649698653, + "nauc_cv_recall_at_1000_std": -0.02941444476048924, + "nauc_cv_recall_at_100_diff1": 0.02698413117107532, + "nauc_cv_recall_at_100_max": -0.17015023428711334, + "nauc_cv_recall_at_100_std": -0.07555477868108984, + "nauc_cv_recall_at_10_diff1": -0.08516873412427065, + "nauc_cv_recall_at_10_max": -0.34819048962459037, + "nauc_cv_recall_at_10_std": -0.20008171938905003, + "nauc_cv_recall_at_1_diff1": -0.22629056772533634, + "nauc_cv_recall_at_1_max": -0.4457500290615541, + "nauc_cv_recall_at_1_std": -0.2617919564222382, + "nauc_cv_recall_at_20_diff1": -0.009662351956441516, + "nauc_cv_recall_at_20_max": -0.3111702270408559, + "nauc_cv_recall_at_20_std": -0.2084163850625874, + "nauc_cv_recall_at_3_diff1": -0.0562154615947782, + "nauc_cv_recall_at_3_max": -0.3912950585084963, + "nauc_cv_recall_at_3_std": -0.25332095011701883, + "nauc_cv_recall_at_5_diff1": -0.05391581003426526, + "nauc_cv_recall_at_5_max": -0.3124748627958558, + "nauc_cv_recall_at_5_std": -0.19528751853375279, + "nauc_map_at_1000_diff1": -0.010046715926208967, + "nauc_map_at_1000_max": -0.3727128923537285, + "nauc_map_at_1000_std": -0.10214863303623575, + "nauc_map_at_100_diff1": -0.011514482774858219, + "nauc_map_at_100_max": -0.39681497563871815, + "nauc_map_at_100_std": -0.11134665897764658, + "nauc_map_at_10_diff1": -0.044739667147889156, + "nauc_map_at_10_max": -0.4372186067013952, + "nauc_map_at_10_std": -0.0916840677417323, + "nauc_map_at_1_diff1": -0.1417401389289008, + "nauc_map_at_1_max": -0.4509225939541273, + "nauc_map_at_1_std": -0.1417396365832301, + "nauc_map_at_20_diff1": -0.025195053611041466, + "nauc_map_at_20_max": -0.4215612727662873, + "nauc_map_at_20_std": -0.1051435759810444, + "nauc_map_at_3_diff1": -0.11893535191042216, + "nauc_map_at_3_max": -0.4712135550112047, + "nauc_map_at_3_std": -0.0842650004835489, + "nauc_map_at_5_diff1": -0.059516500712779793, + "nauc_map_at_5_max": -0.4302726508985725, + "nauc_map_at_5_std": -0.07873538453894163, + "nauc_mrr_at_1000_diff1": -0.07607485688565918, + "nauc_mrr_at_1000_max": -0.3316597602298558, + "nauc_mrr_at_1000_std": -0.19792342004324956, + "nauc_mrr_at_100_diff1": -0.08219089640081807, + "nauc_mrr_at_100_max": -0.34407366018720503, + "nauc_mrr_at_100_std": -0.20666808516931123, + "nauc_mrr_at_10_diff1": -0.12069870481367113, + "nauc_mrr_at_10_max": -0.3800294117787652, + "nauc_mrr_at_10_std": -0.22619327527291744, + "nauc_mrr_at_1_diff1": -0.22629056772533634, + "nauc_mrr_at_1_max": -0.4457500290615541, + "nauc_mrr_at_1_std": -0.2617919564222382, + "nauc_mrr_at_20_diff1": -0.09926861453018676, + "nauc_mrr_at_20_max": -0.36925466427380327, + "nauc_mrr_at_20_std": -0.225543528899351, + "nauc_mrr_at_3_diff1": -0.12586977633057814, + "nauc_mrr_at_3_max": -0.4071395819966693, + "nauc_mrr_at_3_std": -0.2501530490148159, + "nauc_mrr_at_5_diff1": -0.11766245961586667, + "nauc_mrr_at_5_max": -0.37517494900469095, + "nauc_mrr_at_5_std": -0.22731118499071057, + "nauc_ndcg_at_1000_diff1": 0.02009277943675779, + "nauc_ndcg_at_1000_max": -0.18904948738068383, + "nauc_ndcg_at_1000_std": -0.03387648550118827, + "nauc_ndcg_at_100_diff1": 0.026232139318861668, + "nauc_ndcg_at_100_max": -0.27860403299468667, + "nauc_ndcg_at_100_std": -0.11772519727613279, + "nauc_ndcg_at_10_diff1": -0.04947234436473754, + "nauc_ndcg_at_10_max": -0.40364486097929114, + "nauc_ndcg_at_10_std": -0.1533786803793439, + "nauc_ndcg_at_1_diff1": -0.22629056772533634, + "nauc_ndcg_at_1_max": -0.4457500290615541, + "nauc_ndcg_at_1_std": -0.2617919564222382, + "nauc_ndcg_at_20_diff1": -0.00461252186000012, + "nauc_ndcg_at_20_max": -0.36649311208018615, + "nauc_ndcg_at_20_std": -0.16353687300636874, + "nauc_ndcg_at_3_diff1": -0.11267773936384334, + "nauc_ndcg_at_3_max": -0.42899411497666556, + "nauc_ndcg_at_3_std": -0.19653902566677903, + "nauc_ndcg_at_5_diff1": -0.06838328486298777, + "nauc_ndcg_at_5_max": -0.38111958871878415, + "nauc_ndcg_at_5_std": -0.1571783232595063, + "nauc_precision_at_1000_diff1": -0.03249085084774534, + "nauc_precision_at_1000_max": -0.16446381467440538, + "nauc_precision_at_1000_std": -0.10709319311891546, + "nauc_precision_at_100_diff1": -0.02883688252039862, + "nauc_precision_at_100_max": -0.2305183260721622, + "nauc_precision_at_100_std": -0.15283039340337035, + "nauc_precision_at_10_diff1": -0.06352844247941265, + "nauc_precision_at_10_max": -0.4007107530138077, + "nauc_precision_at_10_std": -0.17770052560462707, + "nauc_precision_at_1_diff1": -0.22629056772533634, + "nauc_precision_at_1_max": -0.4457500290615541, + "nauc_precision_at_1_std": -0.2617919564222382, + "nauc_precision_at_20_diff1": -0.02366990094318863, + "nauc_precision_at_20_max": -0.3483225640082979, + "nauc_precision_at_20_std": -0.21523236052685726, + "nauc_precision_at_3_diff1": -0.07889917386247514, + "nauc_precision_at_3_max": -0.43053862302789736, + "nauc_precision_at_3_std": -0.1807336392649264, + "nauc_precision_at_5_diff1": -0.049298631452126984, + "nauc_precision_at_5_max": -0.3765150851533156, + "nauc_precision_at_5_std": -0.14011440498471459, + "nauc_recall_at_1000_diff1": 0.052127753913729535, + "nauc_recall_at_1000_max": -0.13241816139211318, + "nauc_recall_at_1000_std": 0.034748550997898996, + "nauc_recall_at_100_diff1": 0.12778745661018148, + "nauc_recall_at_100_max": -0.1777150772680246, + "nauc_recall_at_100_std": -0.07989548943143837, + "nauc_recall_at_10_diff1": 0.026765977685626496, + "nauc_recall_at_10_max": -0.3841766539530331, + "nauc_recall_at_10_std": -0.12630049632674936, + "nauc_recall_at_1_diff1": -0.1417401389289008, + "nauc_recall_at_1_max": -0.4509225939541273, + "nauc_recall_at_1_std": -0.1417396365832301, + "nauc_recall_at_20_diff1": 0.10846163997925674, + "nauc_recall_at_20_max": -0.309999991338878, + "nauc_recall_at_20_std": -0.1682396965569195, + "nauc_recall_at_3_diff1": -0.0712472400268593, + "nauc_recall_at_3_max": -0.4618660243341875, + "nauc_recall_at_3_std": -0.10049121379738826, + "nauc_recall_at_5_diff1": 0.03522780744618103, + "nauc_recall_at_5_max": -0.34111771775889277, + "nauc_recall_at_5_std": -0.07960384930087543, + "ndcg_at_1": 0.00185, + "ndcg_at_10": 0.00178, + "ndcg_at_100": 0.00335, + "ndcg_at_1000": 0.00812, + "ndcg_at_20": 0.0021, + "ndcg_at_3": 0.00169, + "ndcg_at_5": 0.00165, + "precision_at_1": 0.00185, + "precision_at_10": 0.0011, + "precision_at_100": 0.00056, + "precision_at_1000": 0.00026, + "precision_at_20": 0.00091, + "precision_at_3": 0.00165, + "precision_at_5": 0.0014, + "recall_at_1": 0.00031, + "recall_at_10": 0.00183, + "recall_at_100": 0.00666, + "recall_at_1000": 0.02982, + "recall_at_20": 0.00261, + "recall_at_3": 0.00086, + "recall_at_5": 0.00127 + } + ] + }, + "task_name": "InfoSeekIT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsI2TRetrieval.json new file mode 100644 index 0000000000..22a50384e2 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "aaee58895a66e4d619168849267ed2bb40d37043", + "evaluation_time": 637.0611886978149, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.14225, + "cv_recall_at_10": 0.3711, + "cv_recall_at_100": 0.63325, + "cv_recall_at_1000": 0.8308, + "cv_recall_at_20": 0.4508, + "cv_recall_at_3": 0.2426, + "cv_recall_at_5": 0.2962, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.24665, + "map_at_1": 0.14215, + "map_at_10": 0.20805, + "map_at_100": 0.21816, + "map_at_1000": 0.21898, + "map_at_20": 0.2136, + "map_at_3": 0.18584, + "map_at_5": 0.19808, + "mrr_at_1": 0.14225, + "mrr_at_10": 0.20809535714285854, + "mrr_at_100": 0.21821117637519633, + "mrr_at_1000": 0.21903244279521142, + "mrr_at_20": 0.21365230710680844, + "mrr_at_3": 0.18589166666666862, + "mrr_at_5": 0.19812916666666722, + "nauc_cv_recall_at_1000_diff1": 0.1374166886265108, + "nauc_cv_recall_at_1000_max": 0.4326023231571799, + "nauc_cv_recall_at_1000_std": 0.5371027114383197, + "nauc_cv_recall_at_100_diff1": 0.1859457721091746, + "nauc_cv_recall_at_100_max": 0.37897087843161015, + "nauc_cv_recall_at_100_std": 0.3801615793616046, + "nauc_cv_recall_at_10_diff1": 0.24411979633870634, + "nauc_cv_recall_at_10_max": 0.31955568154632447, + "nauc_cv_recall_at_10_std": 0.15577379453785636, + "nauc_cv_recall_at_1_diff1": 0.43018478049486225, + "nauc_cv_recall_at_1_max": 0.3132376136367806, + "nauc_cv_recall_at_1_std": 0.04846489768168653, + "nauc_cv_recall_at_20_diff1": 0.22602718014627493, + "nauc_cv_recall_at_20_max": 0.3316118262662824, + "nauc_cv_recall_at_20_std": 0.21054304754429856, + "nauc_cv_recall_at_3_diff1": 0.3032334244956384, + "nauc_cv_recall_at_3_max": 0.308301563672542, + "nauc_cv_recall_at_3_std": 0.08475358897537363, + "nauc_cv_recall_at_5_diff1": 0.27115644663985733, + "nauc_cv_recall_at_5_max": 0.3128647962090153, + "nauc_cv_recall_at_5_std": 0.11554840469255882, + "nauc_map_at_1000_diff1": 0.3436467125060594, + "nauc_map_at_1000_max": 0.31323974428346524, + "nauc_map_at_1000_std": 0.09113461601378964, + "nauc_map_at_100_diff1": 0.34347632151089674, + "nauc_map_at_100_max": 0.31324681175801156, + "nauc_map_at_100_std": 0.0911844476249971, + "nauc_map_at_10_diff1": 0.34583327161961397, + "nauc_map_at_10_max": 0.31087876312089413, + "nauc_map_at_10_std": 0.0819667272939276, + "nauc_map_at_1_diff1": 0.4308567698038146, + "nauc_map_at_1_max": 0.31313330076808726, + "nauc_map_at_1_std": 0.048642072248903705, + "nauc_map_at_20_diff1": 0.3440655791308437, + "nauc_map_at_20_max": 0.3119640515148501, + "nauc_map_at_20_std": 0.08682389789331668, + "nauc_map_at_3_diff1": 0.36217978931296785, + "nauc_map_at_3_max": 0.30861261807449575, + "nauc_map_at_3_std": 0.06565290770639244, + "nauc_map_at_5_diff1": 0.3511167742174239, + "nauc_map_at_5_max": 0.3097889051312141, + "nauc_map_at_5_std": 0.07493591815578858, + "nauc_mrr_at_1000_diff1": 0.3433908476595087, + "nauc_mrr_at_1000_max": 0.3132831686651032, + "nauc_mrr_at_1000_std": 0.09105210117570409, + "nauc_mrr_at_100_diff1": 0.3432209969084996, + "nauc_mrr_at_100_max": 0.31329014350682705, + "nauc_mrr_at_100_std": 0.09110207533826178, + "nauc_mrr_at_10_diff1": 0.34557725180410864, + "nauc_mrr_at_10_max": 0.3109217636298925, + "nauc_mrr_at_10_std": 0.08188747150218671, + "nauc_mrr_at_1_diff1": 0.43018478049486225, + "nauc_mrr_at_1_max": 0.3132376136367806, + "nauc_mrr_at_1_std": 0.04846489768168653, + "nauc_mrr_at_20_diff1": 0.3438109296410951, + "nauc_mrr_at_20_max": 0.31200750537885574, + "nauc_mrr_at_20_std": 0.08674305106857981, + "nauc_mrr_at_3_diff1": 0.36191163048605396, + "nauc_mrr_at_3_max": 0.30865755294985336, + "nauc_mrr_at_3_std": 0.06557455228533765, + "nauc_mrr_at_5_diff1": 0.35085778529185385, + "nauc_mrr_at_5_max": 0.30983244732505644, + "nauc_mrr_at_5_std": 0.07485758391758571, + "nauc_ndcg_at_1000_diff1": 0.309216176636288, + "nauc_ndcg_at_1000_max": 0.32619689821644354, + "nauc_ndcg_at_1000_std": 0.14728501394403046, + "nauc_ndcg_at_100_diff1": 0.3055060583350281, + "nauc_ndcg_at_100_max": 0.3255442418423197, + "nauc_ndcg_at_100_std": 0.14920089955599194, + "nauc_ndcg_at_10_diff1": 0.31576995334315505, + "nauc_ndcg_at_10_max": 0.3131255785064659, + "nauc_ndcg_at_10_std": 0.10267909300234637, + "nauc_ndcg_at_1_diff1": 0.4308567698038146, + "nauc_ndcg_at_1_max": 0.31313330076808726, + "nauc_ndcg_at_1_std": 0.048642072248903705, + "nauc_ndcg_at_20_diff1": 0.3101266740624065, + "nauc_ndcg_at_20_max": 0.31681047684725033, + "nauc_ndcg_at_20_std": 0.11932009857835636, + "nauc_ndcg_at_3_diff1": 0.34498997548370164, + "nauc_ndcg_at_3_max": 0.3084426214505432, + "nauc_ndcg_at_3_std": 0.07112303928163569, + "nauc_ndcg_at_5_diff1": 0.32728459943297505, + "nauc_ndcg_at_5_max": 0.3105551753088202, + "nauc_ndcg_at_5_std": 0.08666800363917986, + "nauc_precision_at_1000_diff1": 0.13741668862651063, + "nauc_precision_at_1000_max": 0.43260232315717956, + "nauc_precision_at_1000_std": 0.5371027114383187, + "nauc_precision_at_100_diff1": 0.1859457721091744, + "nauc_precision_at_100_max": 0.37897087843161026, + "nauc_precision_at_100_std": 0.38016157936160416, + "nauc_precision_at_10_diff1": 0.24411979633870629, + "nauc_precision_at_10_max": 0.3195556815463242, + "nauc_precision_at_10_std": 0.15577379453785628, + "nauc_precision_at_1_diff1": 0.4308567698038146, + "nauc_precision_at_1_max": 0.31313330076808726, + "nauc_precision_at_1_std": 0.048642072248903705, + "nauc_precision_at_20_diff1": 0.22602718014627443, + "nauc_precision_at_20_max": 0.3316118262662818, + "nauc_precision_at_20_std": 0.21054304754429795, + "nauc_precision_at_3_diff1": 0.3032334244956383, + "nauc_precision_at_3_max": 0.3083015636725422, + "nauc_precision_at_3_std": 0.08475358897537373, + "nauc_precision_at_5_diff1": 0.2711564466398571, + "nauc_precision_at_5_max": 0.31286479620901536, + "nauc_precision_at_5_std": 0.1155484046925588, + "nauc_recall_at_1000_diff1": 0.1374166886265108, + "nauc_recall_at_1000_max": 0.4326023231571799, + "nauc_recall_at_1000_std": 0.5371027114383197, + "nauc_recall_at_100_diff1": 0.1859457721091746, + "nauc_recall_at_100_max": 0.37897087843161015, + "nauc_recall_at_100_std": 0.3801615793616046, + "nauc_recall_at_10_diff1": 0.24411979633870634, + "nauc_recall_at_10_max": 0.31955568154632447, + "nauc_recall_at_10_std": 0.15577379453785636, + "nauc_recall_at_1_diff1": 0.4308567698038146, + "nauc_recall_at_1_max": 0.31313330076808726, + "nauc_recall_at_1_std": 0.048642072248903705, + "nauc_recall_at_20_diff1": 0.22602718014627493, + "nauc_recall_at_20_max": 0.3316118262662824, + "nauc_recall_at_20_std": 0.21054304754429856, + "nauc_recall_at_3_diff1": 0.3032334244956384, + "nauc_recall_at_3_max": 0.308301563672542, + "nauc_recall_at_3_std": 0.08475358897537363, + "nauc_recall_at_5_diff1": 0.27115644663985733, + "nauc_recall_at_5_max": 0.3128647962090153, + "nauc_recall_at_5_std": 0.11554840469255882, + "ndcg_at_1": 0.14215, + "ndcg_at_10": 0.24665, + "ndcg_at_100": 0.30043, + "ndcg_at_1000": 0.32483, + "ndcg_at_20": 0.26683, + "ndcg_at_3": 0.20039, + "ndcg_at_5": 0.22246, + "precision_at_1": 0.14215, + "precision_at_10": 0.03711, + "precision_at_100": 0.00633, + "precision_at_1000": 0.00083, + "precision_at_20": 0.02254, + "precision_at_3": 0.08087, + "precision_at_5": 0.05924, + "recall_at_1": 0.14215, + "recall_at_10": 0.3711, + "recall_at_100": 0.63325, + "recall_at_1000": 0.8308, + "recall_at_20": 0.4508, + "recall_at_3": 0.2426, + "recall_at_5": 0.2962 + } + ] + }, + "task_name": "VisualNewsI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsT2IRetrieval.json new file mode 100644 index 0000000000..f6e3f67041 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "94c519d850dba2b0058c2fc9b5da6142a59aa285", + "evaluation_time": 2032.3265976905823, + "kg_co2_emissions": null, + "mteb_version": "1.12.84", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.14214, + "cv_recall_at_10": 0.38615, + "cv_recall_at_100": 0.64896, + "cv_recall_at_1000": 0.83731, + "cv_recall_at_20": 0.47277, + "cv_recall_at_3": 0.24846, + "cv_recall_at_5": 0.30413, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.25322, + "map_at_1": 0.14189, + "map_at_10": 0.21201, + "map_at_100": 0.22241, + "map_at_1000": 0.22322, + "map_at_20": 0.21802, + "map_at_3": 0.18826, + "map_at_5": 0.20104, + "mrr_at_1": 0.14213553388347086, + "mrr_at_10": 0.21213452172567082, + "mrr_at_100": 0.22253810455058598, + "mrr_at_1000": 0.22334262930582421, + "mrr_at_20": 0.21815238887195898, + "mrr_at_3": 0.18843044094357164, + "mrr_at_5": 0.2011486204884563, + "nauc_cv_recall_at_1000_diff1": 0.2738151451376307, + "nauc_cv_recall_at_1000_max": 0.5931214969636025, + "nauc_cv_recall_at_1000_std": 0.7501777008032257, + "nauc_cv_recall_at_100_diff1": 0.2743373648460172, + "nauc_cv_recall_at_100_max": 0.46575688991474795, + "nauc_cv_recall_at_100_std": 0.576698437786742, + "nauc_cv_recall_at_10_diff1": 0.28264363675897997, + "nauc_cv_recall_at_10_max": 0.35455403543397274, + "nauc_cv_recall_at_10_std": 0.3409080895780153, + "nauc_cv_recall_at_1_diff1": 0.43574527435374394, + "nauc_cv_recall_at_1_max": 0.30366809286647123, + "nauc_cv_recall_at_1_std": 0.19297182230123075, + "nauc_cv_recall_at_20_diff1": 0.27117097023604686, + "nauc_cv_recall_at_20_max": 0.38504583058673747, + "nauc_cv_recall_at_20_std": 0.4083714618245761, + "nauc_cv_recall_at_3_diff1": 0.3290103271052618, + "nauc_cv_recall_at_3_max": 0.3175734967043012, + "nauc_cv_recall_at_3_std": 0.2572722720243564, + "nauc_cv_recall_at_5_diff1": 0.30524189503214394, + "nauc_cv_recall_at_5_max": 0.3267082717032173, + "nauc_cv_recall_at_5_std": 0.28295915343267924, + "nauc_map_at_1000_diff1": 0.36629495836246867, + "nauc_map_at_1000_max": 0.32247650003298395, + "nauc_map_at_1000_std": 0.254260785723814, + "nauc_map_at_100_diff1": 0.3661110091807752, + "nauc_map_at_100_max": 0.32235486608870173, + "nauc_map_at_100_std": 0.25427301752910086, + "nauc_map_at_10_diff1": 0.3666371005486745, + "nauc_map_at_10_max": 0.3179934234214588, + "nauc_map_at_10_std": 0.2447703790002406, + "nauc_map_at_1_diff1": 0.43743514555841434, + "nauc_map_at_1_max": 0.30350123194006623, + "nauc_map_at_1_std": 0.1933581414080973, + "nauc_map_at_20_diff1": 0.3656297660643177, + "nauc_map_at_20_max": 0.3206298829457058, + "nauc_map_at_20_std": 0.2506027371516703, + "nauc_map_at_3_diff1": 0.37914731231852516, + "nauc_map_at_3_max": 0.31014982499900495, + "nauc_map_at_3_std": 0.22582773571427692, + "nauc_map_at_5_diff1": 0.37107276727666866, + "nauc_map_at_5_max": 0.3129587682028005, + "nauc_map_at_5_std": 0.23406545136024842, + "nauc_mrr_at_1000_diff1": 0.365694119656496, + "nauc_mrr_at_1000_max": 0.3225064538045288, + "nauc_mrr_at_1000_std": 0.25372446071677807, + "nauc_mrr_at_100_diff1": 0.36550943256415147, + "nauc_mrr_at_100_max": 0.32238352603484055, + "nauc_mrr_at_100_std": 0.2537390594412823, + "nauc_mrr_at_10_diff1": 0.36604292281022704, + "nauc_mrr_at_10_max": 0.318023509003043, + "nauc_mrr_at_10_std": 0.2442321947644308, + "nauc_mrr_at_1_diff1": 0.43574527435374394, + "nauc_mrr_at_1_max": 0.30366809286647123, + "nauc_mrr_at_1_std": 0.19297182230123075, + "nauc_mrr_at_20_diff1": 0.36503648073161493, + "nauc_mrr_at_20_max": 0.32065493374494664, + "nauc_mrr_at_20_std": 0.2500569829241322, + "nauc_mrr_at_3_diff1": 0.3783635531911808, + "nauc_mrr_at_3_max": 0.3099868426780373, + "nauc_mrr_at_3_std": 0.2250798843091142, + "nauc_mrr_at_5_diff1": 0.37029885694565495, + "nauc_mrr_at_5_max": 0.3128479822671293, + "nauc_mrr_at_5_std": 0.23345888219471445, + "nauc_ndcg_at_1000_diff1": 0.34587426274769756, + "nauc_ndcg_at_1000_max": 0.35296786555677756, + "nauc_ndcg_at_1000_std": 0.31690613840921944, + "nauc_ndcg_at_100_diff1": 0.34177689609578826, + "nauc_ndcg_at_100_max": 0.3503176936120679, + "nauc_ndcg_at_100_std": 0.31898929198056136, + "nauc_ndcg_at_10_diff1": 0.34181459809724857, + "nauc_ndcg_at_10_max": 0.328190780924741, + "nauc_ndcg_at_10_std": 0.2720579741726425, + "nauc_ndcg_at_1_diff1": 0.43743514555841434, + "nauc_ndcg_at_1_max": 0.30350123194006623, + "nauc_ndcg_at_1_std": 0.1933581414080973, + "nauc_ndcg_at_20_diff1": 0.33852580543057287, + "nauc_ndcg_at_20_max": 0.33728315846227197, + "nauc_ndcg_at_20_std": 0.29215613689258596, + "nauc_ndcg_at_3_diff1": 0.3646193686577447, + "nauc_ndcg_at_3_max": 0.31244948573813575, + "nauc_ndcg_at_3_std": 0.23509499886693777, + "nauc_ndcg_at_5_diff1": 0.35161503224004037, + "nauc_ndcg_at_5_max": 0.31712039236583617, + "nauc_ndcg_at_5_std": 0.24853289096733608, + "nauc_precision_at_1000_diff1": 0.27273524762384516, + "nauc_precision_at_1000_max": 0.5914908390552021, + "nauc_precision_at_1000_std": 0.7492366648909354, + "nauc_precision_at_100_diff1": 0.2743373648460168, + "nauc_precision_at_100_max": 0.46575688991474795, + "nauc_precision_at_100_std": 0.5766984377867421, + "nauc_precision_at_10_diff1": 0.28267830445192166, + "nauc_precision_at_10_max": 0.35462752640006245, + "nauc_precision_at_10_std": 0.34090808957801505, + "nauc_precision_at_1_diff1": 0.43743514555841434, + "nauc_precision_at_1_max": 0.30350123194006623, + "nauc_precision_at_1_std": 0.1933581414080973, + "nauc_precision_at_20_diff1": 0.2711325060911637, + "nauc_precision_at_20_max": 0.3851281672816333, + "nauc_precision_at_20_std": 0.4085615041031749, + "nauc_precision_at_3_diff1": 0.32949775513819174, + "nauc_precision_at_3_max": 0.3181893480026203, + "nauc_precision_at_3_std": 0.2578284990482501, + "nauc_precision_at_5_diff1": 0.30599236424623105, + "nauc_precision_at_5_max": 0.32728294271527764, + "nauc_precision_at_5_std": 0.28316104447138657, + "nauc_recall_at_1000_diff1": 0.2735240225347363, + "nauc_recall_at_1000_max": 0.5932571091213296, + "nauc_recall_at_1000_std": 0.7502902830528209, + "nauc_recall_at_100_diff1": 0.274464026101684, + "nauc_recall_at_100_max": 0.4658591197876539, + "nauc_recall_at_100_std": 0.5765276833211775, + "nauc_recall_at_10_diff1": 0.2826783044519219, + "nauc_recall_at_10_max": 0.3546275264000627, + "nauc_recall_at_10_std": 0.3409080895780153, + "nauc_recall_at_1_diff1": 0.43743514555841434, + "nauc_recall_at_1_max": 0.30350123194006623, + "nauc_recall_at_1_std": 0.1933581414080973, + "nauc_recall_at_20_diff1": 0.27116932602660276, + "nauc_recall_at_20_max": 0.385175879145329, + "nauc_recall_at_20_std": 0.4085181212052684, + "nauc_recall_at_3_diff1": 0.3294977551381917, + "nauc_recall_at_3_max": 0.3181893480026203, + "nauc_recall_at_3_std": 0.25782849904825034, + "nauc_recall_at_5_diff1": 0.3059923642462314, + "nauc_recall_at_5_max": 0.3272829427152781, + "nauc_recall_at_5_std": 0.2831610444713866, + "ndcg_at_1": 0.14189, + "ndcg_at_10": 0.25322, + "ndcg_at_100": 0.30753, + "ndcg_at_1000": 0.33093, + "ndcg_at_20": 0.2751, + "ndcg_at_3": 0.20363, + "ndcg_at_5": 0.22668, + "precision_at_1": 0.14189, + "precision_at_10": 0.03861, + "precision_at_100": 0.00649, + "precision_at_1000": 0.00084, + "precision_at_20": 0.02363, + "precision_at_3": 0.08274, + "precision_at_5": 0.06084, + "recall_at_1": 0.14189, + "recall_at_10": 0.38615, + "recall_at_100": 0.64889, + "recall_at_1000": 0.83726, + "recall_at_20": 0.47264, + "recall_at_3": 0.24821, + "recall_at_5": 0.30418 + } + ] + }, + "task_name": "VisualNewsT2IRetrieval" +} \ No newline at end of file From 494b563c8cbe8fd76b9ef7b6efc51a9a7a6df275 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 12 Aug 2024 10:35:23 +0300 Subject: [PATCH 044/154] [MIEB] Add Stanford Cars i2i Retrieval (#1147) * wip * add results * make lint * change back the order --- .../abstasks/Image/AbsTaskAny2AnyRetrieval.py | 4 +- mteb/abstasks/TaskMetadata.py | 1 - mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 1 + .../eng/StanfordCarsI2IRetrieval.py | 48 ++++++ .../I2TRetrieval/eng/MSCOCOI2TRetrieval.py | 3 +- .../eng/StanfordCarsClassification.py | 7 +- .../StanfordCarsI2IRetrieval.json | 158 ++++++++++++++++++ 7 files changed, 214 insertions(+), 8 deletions(-) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py index a751af9ece..21646e2d1a 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py @@ -173,7 +173,9 @@ def _load_qrels(self, split): delimiter="\t", keep_in_memory=self.keep_in_memory, ) - qrels_ds = qrels_ds.remove_columns("Q0") + + if "Q0" in qrels_ds.column_names: + qrels_ds = qrels_ds.remove_columns("Q0") features = Features( { "query-id": Value("string"), diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 4b3b02891e..fae6d29b58 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -183,7 +183,6 @@ class TaskMetadata(BaseModel): "Government", "Legal", "Medical", "Poetry", "Religious", "Reviews", "Web", "Spoken", "Written". A dataset can belong to multiple domains. task_subtypes: The subtypes of the task. E.g. includes "Sentiment/Hate speech", "Thematic Clustering". Feel free to update the list as needed. license: The license of the data. - socioeconomic_status: The socioeconomic status of the data. Includes "high", "medium", "low", "mixed". annotations_creators: The type of the annotators. Includes "expert-annotated" (annotated by experts), "human-annotated" (annotated e.g. by mturkers), "derived" (derived from structure in the data). dialect: The dialect of the data, if applicable. Ideally specified as a BCP-47 language tag. Empty list if no dialects are present. diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 3f5487a503..3f59cc04dc 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -7,6 +7,7 @@ from .eng.NIGHTSI2IRetrieval import * from .eng.OVENIT2ITRetrieval import * from .eng.OVENIT2TRetrieval import * +from .eng.StanfordCarsI2IRetrieval import * from .eng.VisualNewsI2TRetrieval import * from .eng.VisualNewsT2IRetrieval import * from .eng.WebQAT2ITRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py new file mode 100644 index 0000000000..ba3ace78ed --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class StanfordCarsI2I(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="StanfordCarsI2IRetrieval", + description="Retrieve car images from 196 makes.", + reference="https://pure.mpg.de/rest/items/item_2029263/component/file_2029262/content", + dataset={ + "path": "isaacchung/stanford_cars_retrieval", + "revision": "b27a0612211af3598bd11fe28af20928f20cce06", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="recall_at_1", + date=("2012-01-01", "2013-04-01"), + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{Krause2013CollectingAL, + title={Collecting a Large-scale Dataset of Fine-grained Cars}, + author={Jonathan Krause and Jia Deng and Michael Stark and Li Fei-Fei}, + year={2013}, + url={https://api.semanticscholar.org/CorpusID:16632981} + } + """, + descriptive_stats={ + "n_samples": {"default": 8041}, + "avg_character_length": { + "test": { + "average_document_length": 1074.894348894349, + "average_query_length": 77.06142506142506, + "num_documents": 8041, + "num_queries": 8041, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py b/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py index 5dc19270b7..8047b2665c 100644 --- a/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py +++ b/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py @@ -1,9 +1,8 @@ from __future__ import annotations +from mteb.abstasks import AbsTaskI2TRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata -from .....abstasks import AbsTaskI2TRetrieval - class MSCOCOI2TRetrieval(AbsTaskI2TRetrieval): metadata = TaskMetadata( diff --git a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py index a225df6f47..001fa6d965 100644 --- a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py @@ -8,7 +8,7 @@ class StanfordCarsClassification(AbsTaskImageClassification): metadata = TaskMetadata( name="StanfordCars", - description="Classifying car images from 96 makes.", + description="Classifying car images from 196 makes.", reference="https://pure.mpg.de/rest/items/item_2029263/component/file_2029262/content", dataset={ "path": "isaacchung/StanfordCars", @@ -23,10 +23,9 @@ class StanfordCarsClassification(AbsTaskImageClassification): "2013-01-01", "2013-04-01", ), # Estimated range for the collection of reviews - domains=["Scene"], - task_subtypes=["Scene recognition"], + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json new file mode 100644 index 0000000000..49c428cbfd --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b27a0612211af3598bd11fe28af20928f20cce06", + "evaluation_time": 205.83177590370178, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.0, + "map_at_1": 0.0, + "map_at_10": 0.06072, + "map_at_100": 0.17896, + "map_at_1000": 0.21707, + "map_at_20": 0.09946, + "map_at_3": 0.01597, + "map_at_5": 0.03084, + "mrr_at_1": 0.00012436264146250465, + "mrr_at_10": 0.3901051259122414, + "mrr_at_100": 0.3934181724945363, + "mrr_at_1000": 0.39343998588915124, + "mrr_at_20": 0.3926309438221267, + "mrr_at_3": 0.35457861791651374, + "mrr_at_5": 0.3791464577374308, + "nauc_map_at_1000_diff1": -0.20058034455775214, + "nauc_map_at_1000_max": -0.009541604862555151, + "nauc_map_at_1000_std": 0.36415192557738685, + "nauc_map_at_100_diff1": -0.1859159754121815, + "nauc_map_at_100_max": -0.00915256217544372, + "nauc_map_at_100_std": 0.3179916778555873, + "nauc_map_at_10_diff1": -0.17016138595951308, + "nauc_map_at_10_max": -0.008401015865521977, + "nauc_map_at_10_std": 0.13541784388452338, + "nauc_map_at_1_diff1": -0.5746685568283685, + "nauc_map_at_1_max": -0.4995495220405232, + "nauc_map_at_1_std": 1.0, + "nauc_map_at_20_diff1": -0.17026308934464754, + "nauc_map_at_20_max": -0.006465085107102209, + "nauc_map_at_20_std": 0.19410962638554546, + "nauc_map_at_3_diff1": -0.19618028548625588, + "nauc_map_at_3_max": 0.0015533961447322393, + "nauc_map_at_3_std": 0.08037524782358588, + "nauc_map_at_5_diff1": -0.17897720472394818, + "nauc_map_at_5_max": -0.0022562794225319126, + "nauc_map_at_5_std": 0.09354551152212666, + "nauc_mrr_at_1000_diff1": -0.26756915105219325, + "nauc_mrr_at_1000_max": 0.007219560942946209, + "nauc_mrr_at_1000_std": 0.1277621843185587, + "nauc_mrr_at_100_diff1": -0.26768624224236, + "nauc_mrr_at_100_max": 0.007215515428976401, + "nauc_mrr_at_100_std": 0.12791181627808856, + "nauc_mrr_at_10_diff1": -0.2689209194339867, + "nauc_mrr_at_10_max": 0.00554788070506897, + "nauc_mrr_at_10_std": 0.13204137790528978, + "nauc_mrr_at_1_diff1": -0.5746685568283684, + "nauc_mrr_at_1_max": -0.49954952204052305, + "nauc_mrr_at_1_std": 1.0, + "nauc_mrr_at_20_diff1": -0.26916203739306344, + "nauc_mrr_at_20_max": 0.0067661125465101725, + "nauc_mrr_at_20_std": 0.1304515877065281, + "nauc_mrr_at_3_diff1": -0.25798593374810697, + "nauc_mrr_at_3_max": 0.00846034337944975, + "nauc_mrr_at_3_std": 0.10915550317935098, + "nauc_mrr_at_5_diff1": -0.26767632665665964, + "nauc_mrr_at_5_max": 0.007533563370578295, + "nauc_mrr_at_5_std": 0.1314283638264671, + "nauc_ndcg_at_1000_diff1": -0.26848867699612694, + "nauc_ndcg_at_1000_max": -0.01219681131327712, + "nauc_ndcg_at_1000_std": 0.47014178875035684, + "nauc_ndcg_at_100_diff1": -0.21785067520456688, + "nauc_ndcg_at_100_max": -0.012635572435124356, + "nauc_ndcg_at_100_std": 0.3658418059055116, + "nauc_ndcg_at_10_diff1": -0.18666608258490472, + "nauc_ndcg_at_10_max": -0.006380936466543662, + "nauc_ndcg_at_10_std": 0.1570549915957106, + "nauc_ndcg_at_1_diff1": -0.5746685568283684, + "nauc_ndcg_at_1_max": -0.49954952204052305, + "nauc_ndcg_at_1_std": 1.0, + "nauc_ndcg_at_20_diff1": -0.1892271528105937, + "nauc_ndcg_at_20_max": -0.0045807039352219504, + "nauc_ndcg_at_20_std": 0.2166402891227918, + "nauc_ndcg_at_3_diff1": -0.2156887442186972, + "nauc_ndcg_at_3_max": 0.005069652317788833, + "nauc_ndcg_at_3_std": 0.09725117443347137, + "nauc_ndcg_at_5_diff1": -0.1949431835234068, + "nauc_ndcg_at_5_max": 0.0020311591774530764, + "nauc_ndcg_at_5_std": 0.11416484694774583, + "nauc_precision_at_1000_diff1": -0.2311605207327101, + "nauc_precision_at_1000_max": -0.011274296784979255, + "nauc_precision_at_1000_std": 0.5429105568395621, + "nauc_precision_at_100_diff1": -0.20218396795823576, + "nauc_precision_at_100_max": -0.013867657753386877, + "nauc_precision_at_100_std": 0.43295422350606705, + "nauc_precision_at_10_diff1": -0.17588408513054, + "nauc_precision_at_10_max": -0.009047036150405124, + "nauc_precision_at_10_std": 0.16785493183646077, + "nauc_precision_at_1_diff1": -0.5746685568283684, + "nauc_precision_at_1_max": -0.49954952204052305, + "nauc_precision_at_1_std": 1.0, + "nauc_precision_at_20_diff1": -0.1795363254595728, + "nauc_precision_at_20_max": -0.005308651370902496, + "nauc_precision_at_20_std": 0.23936954535875124, + "nauc_precision_at_3_diff1": -0.2138380176200969, + "nauc_precision_at_3_max": 0.004063308534515615, + "nauc_precision_at_3_std": 0.10034463101558477, + "nauc_precision_at_5_diff1": -0.18747136042736523, + "nauc_precision_at_5_max": 0.0007804267760293301, + "nauc_precision_at_5_std": 0.11857533805458287, + "nauc_recall_at_1000_diff1": -0.3090612097064354, + "nauc_recall_at_1000_max": -0.014613854656747456, + "nauc_recall_at_1000_std": 0.6433331116323254, + "nauc_recall_at_100_diff1": -0.21428842666117542, + "nauc_recall_at_100_max": -0.0157915060538706, + "nauc_recall_at_100_std": 0.42610561659215096, + "nauc_recall_at_10_diff1": -0.1808720571796819, + "nauc_recall_at_10_max": -0.012426875194332561, + "nauc_recall_at_10_std": 0.1595995698351979, + "nauc_recall_at_1_diff1": -0.5746685568283685, + "nauc_recall_at_1_max": -0.4995495220405232, + "nauc_recall_at_1_std": 1.0, + "nauc_recall_at_20_diff1": -0.18610900567530295, + "nauc_recall_at_20_max": -0.009094327927607726, + "nauc_recall_at_20_std": 0.22959563262774454, + "nauc_recall_at_3_diff1": -0.20479580828524666, + "nauc_recall_at_3_max": 0.0011739339718611925, + "nauc_recall_at_3_std": 0.08948104084187246, + "nauc_recall_at_5_diff1": -0.18850109914331253, + "nauc_recall_at_5_max": -0.0027946570149449752, + "nauc_recall_at_5_std": 0.1101593382209862, + "ndcg_at_1": 0.00012, + "ndcg_at_10": 0.37362, + "ndcg_at_100": 0.3811, + "ndcg_at_1000": 0.56344, + "ndcg_at_20": 0.34466, + "ndcg_at_3": 0.31444, + "ndcg_at_5": 0.36135, + "precision_at_1": 0.00012, + "precision_at_10": 0.41114, + "precision_at_100": 0.17081, + "precision_at_1000": 0.03375, + "precision_at_20": 0.35062, + "precision_at_3": 0.39216, + "precision_at_5": 0.42838, + "recall_at_1": 0.0, + "recall_at_10": 0.10276, + "recall_at_100": 0.42446, + "recall_at_1000": 0.83495, + "recall_at_20": 0.17504, + "recall_at_3": 0.02944, + "recall_at_5": 0.05356 + } + ] + }, + "task_name": "StanfordCarsI2IRetrieval" +} \ No newline at end of file From b58009c5aea697d4466898df32876df3ddfaf40a Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Tue, 13 Aug 2024 13:53:55 +0300 Subject: [PATCH 045/154] [MIEB] Add CUB200 i2i retrieval (#1154) * add cub200 and results * add skip_first_result * skipped self and rerun results --- .../abstasks/Image/AbsTaskAny2AnyRetrieval.py | 2 + .../Image/Any2AnyRetrievalEvaluator.py | 5 + mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 1 + .../eng/CUB200I2IRetrieval.py | 50 +++++ .../eng/StanfordCarsI2IRetrieval.py | 3 +- .../CUB200I2IRetrieval.json | 186 ++++++++++++++++++ .../StanfordCarsI2IRetrieval.json | 32 ++- 7 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CUB200I2IRetrieval.json diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py index 21646e2d1a..58d1407493 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py @@ -207,6 +207,7 @@ class AbsTaskAny2AnyRetrieval(AbsTask): """ ignore_identical_ids: bool = False + skip_first_result: bool = False def __init__(self, **kwargs): super().__init__(**kwargs) @@ -315,6 +316,7 @@ def _evaluate_subset( results, retriever.k_values, ignore_identical_ids=self.ignore_identical_ids, + skip_first_result=self.skip_first_result, ) mrr, naucs_mrr = retriever.evaluate_custom( relevant_docs, results, retriever.k_values, "mrr" diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index 123fad09eb..928da19d45 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -294,6 +294,7 @@ def evaluate( results: dict[str, dict[str, float]], k_values: List[int], ignore_identical_ids: bool = False, + skip_first_result: bool = False, ) -> Tuple[ dict[str, float], dict[str, float], @@ -344,6 +345,10 @@ def evaluate( for qid, rels in results.items() } + if skip_first_result: + for qid, rels in sorted_results.items(): + sorted_results[qid].pop(0) + for query_id in scores.keys(): top_docs = [ doc_id for doc_id, _ in sorted_results.get(query_id, []) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 3f59cc04dc..2fab5b0635 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -1,6 +1,7 @@ from __future__ import annotations from .eng.CIRRIT2IRetrieval import * +from .eng.CUB200I2IRetrieval import * from .eng.FashionIQIT2IRetrieval import * from .eng.InfoSeekIT2ITRetrieval import * from .eng.InfoSeekIT2TRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py new file mode 100644 index 0000000000..b66475e4cf --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class CUB200I2I(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="CUB200I2IRetrieval", + description="Retrieve bird images from 200 classes.", + reference="https://www.florian-schroff.de/publications/CUB-200.pdf", + dataset={ + "path": "isaacchung/cub200_retrieval", + "revision": "ad08c1307b15a226bf1b64e62656a17f1f85f7ec", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@article{article, + author = {Welinder, Peter and Branson, Steve and Mita, Takeshi and Wah, Catherine and Schroff, Florian and Belongie, Serge and Perona, Pietro}, + year = {2010}, + month = {09}, + pages = {}, + title = {Caltech-UCSD Birds 200} + } + """, + descriptive_stats={ + "n_samples": {"default": 5794}, + "avg_character_length": { + "test": { + "average_document_length": 1074.894348894349, + "average_query_length": 77.06142506142506, + "num_documents": 5794, + "num_queries": 5794, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + skip_first_result = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py index ba3ace78ed..848b2ebef6 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py @@ -17,7 +17,7 @@ class StanfordCarsI2I(AbsTaskAny2AnyRetrieval): category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], - main_score="recall_at_1", + main_score="cv_recall_at_1", date=("2012-01-01", "2013-04-01"), domains=["Encyclopaedic"], task_subtypes=["Object recognition"], @@ -46,3 +46,4 @@ class StanfordCarsI2I(AbsTaskAny2AnyRetrieval): }, }, ) + skip_first_result = True diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CUB200I2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CUB200I2IRetrieval.json new file mode 100644 index 0000000000..a46d52dd1e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CUB200I2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "ad08c1307b15a226bf1b64e62656a17f1f85f7ec", + "evaluation_time": 101.24021887779236, + "kg_co2_emissions": null, + "mteb_version": "1.12.80", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.5195, + "cv_recall_at_10": 0.8835, + "cv_recall_at_100": 0.99275, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.94184, + "cv_recall_at_3": 0.72489, + "cv_recall_at_5": 0.8029, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5195, + "map_at_1": 0.0, + "map_at_10": 0.06775, + "map_at_100": 0.17109, + "map_at_1000": 0.19946, + "map_at_20": 0.10637, + "map_at_3": 0.01856, + "map_at_5": 0.03526, + "mrr_at_1": 0.0, + "mrr_at_10": 0.345117417580307, + "mrr_at_100": 0.3515074205178541, + "mrr_at_1000": 0.3515481539320064, + "mrr_at_20": 0.3499334718788236, + "mrr_at_3": 0.30405016683925506, + "mrr_at_5": 0.33131975606949415, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.29829479476891246, + "nauc_cv_recall_at_100_max": -0.35475926675448466, + "nauc_cv_recall_at_100_std": 0.16066304907385462, + "nauc_cv_recall_at_10_diff1": -0.2786391142334684, + "nauc_cv_recall_at_10_max": -0.04765779245271946, + "nauc_cv_recall_at_10_std": 0.2248031704570365, + "nauc_cv_recall_at_1_diff1": -0.1779435022557211, + "nauc_cv_recall_at_1_max": -0.02718179344833177, + "nauc_cv_recall_at_1_std": 0.03655854432763116, + "nauc_cv_recall_at_20_diff1": -0.31932337225079194, + "nauc_cv_recall_at_20_max": -0.037830258082760186, + "nauc_cv_recall_at_20_std": 0.26690155092620627, + "nauc_cv_recall_at_3_diff1": -0.20332785145569104, + "nauc_cv_recall_at_3_max": -0.04697587564711363, + "nauc_cv_recall_at_3_std": 0.1258221057180091, + "nauc_cv_recall_at_5_diff1": -0.23347006556105487, + "nauc_cv_recall_at_5_max": -0.031142268954388815, + "nauc_cv_recall_at_5_std": 0.1516526846178296, + "nauc_map_at_1000_diff1": -0.2062688124757473, + "nauc_map_at_1000_max": -0.0035555471459080617, + "nauc_map_at_1000_std": 0.23080503561089502, + "nauc_map_at_100_diff1": -0.19641682305588465, + "nauc_map_at_100_max": -0.0026445524622137747, + "nauc_map_at_100_std": 0.20444263793369596, + "nauc_map_at_10_diff1": -0.18598512115387061, + "nauc_map_at_10_max": -0.005392449194288058, + "nauc_map_at_10_std": 0.1014171349773814, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": -0.18851990685363146, + "nauc_map_at_20_max": 0.0023419122967712775, + "nauc_map_at_20_std": 0.139746932457822, + "nauc_map_at_3_diff1": -0.15825409921589528, + "nauc_map_at_3_max": -0.011497137589137481, + "nauc_map_at_3_std": 0.056414830301161185, + "nauc_map_at_5_diff1": -0.173932631810766, + "nauc_map_at_5_max": -0.004713040702303579, + "nauc_map_at_5_std": 0.07084949917645643, + "nauc_mrr_at_1000_diff1": -0.19955607289570382, + "nauc_mrr_at_1000_max": -0.03238896690902184, + "nauc_mrr_at_1000_std": 0.08628495504725432, + "nauc_mrr_at_100_diff1": -0.1996164211591153, + "nauc_mrr_at_100_max": -0.03249899847707879, + "nauc_mrr_at_100_std": 0.08628990870899618, + "nauc_mrr_at_10_diff1": -0.20015919600929724, + "nauc_mrr_at_10_max": -0.03390649579122204, + "nauc_mrr_at_10_std": 0.09021220925342546, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": -0.2007342426577916, + "nauc_mrr_at_20_max": -0.032351629887101385, + "nauc_mrr_at_20_std": 0.08872643632144, + "nauc_mrr_at_3_diff1": -0.1896169258292634, + "nauc_mrr_at_3_max": -0.02905559729986968, + "nauc_mrr_at_3_std": 0.07153826033664305, + "nauc_mrr_at_5_diff1": -0.20025097945245238, + "nauc_mrr_at_5_max": -0.03291097332957811, + "nauc_mrr_at_5_std": 0.08662818761457174, + "nauc_ndcg_at_1000_diff1": -0.25364131167732773, + "nauc_ndcg_at_1000_max": -0.00969178999184677, + "nauc_ndcg_at_1000_std": 0.3093148634981729, + "nauc_ndcg_at_100_diff1": -0.21304640197927943, + "nauc_ndcg_at_100_max": -0.005994303743098639, + "nauc_ndcg_at_100_std": 0.24287453393772768, + "nauc_ndcg_at_10_diff1": -0.20032000568929073, + "nauc_ndcg_at_10_max": -0.00992656456191809, + "nauc_ndcg_at_10_std": 0.10587882848001441, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": -0.20182484259143996, + "nauc_ndcg_at_20_max": -0.004590142857157939, + "nauc_ndcg_at_20_std": 0.1429465836051686, + "nauc_ndcg_at_3_diff1": -0.17996517359211764, + "nauc_ndcg_at_3_max": -0.015207435524616678, + "nauc_ndcg_at_3_std": 0.060135755734251166, + "nauc_ndcg_at_5_diff1": -0.1920019943385643, + "nauc_ndcg_at_5_max": -0.01018908132779398, + "nauc_ndcg_at_5_std": 0.07561645521061498, + "nauc_precision_at_1000_diff1": -0.29703869433778635, + "nauc_precision_at_1000_max": -0.01298049022298684, + "nauc_precision_at_1000_std": 0.4308144194919067, + "nauc_precision_at_100_diff1": -0.21818017775568854, + "nauc_precision_at_100_max": -0.006397344189188299, + "nauc_precision_at_100_std": 0.2962068061740296, + "nauc_precision_at_10_diff1": -0.19845507492338102, + "nauc_precision_at_10_max": -0.008714316194455004, + "nauc_precision_at_10_std": 0.11541882266156184, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": -0.20066511163855275, + "nauc_precision_at_20_max": -0.0021458092054351886, + "nauc_precision_at_20_std": 0.16126858774816208, + "nauc_precision_at_3_diff1": -0.1802516939125809, + "nauc_precision_at_3_max": -0.01351037586517892, + "nauc_precision_at_3_std": 0.06347722377914423, + "nauc_precision_at_5_diff1": -0.19340808404985577, + "nauc_precision_at_5_max": -0.008276455491399038, + "nauc_precision_at_5_std": 0.08018404870245943, + "nauc_recall_at_1000_diff1": -0.2915096272448527, + "nauc_recall_at_1000_max": -0.013349971323190575, + "nauc_recall_at_1000_std": 0.5447028375955068, + "nauc_recall_at_100_diff1": -0.20185480358135358, + "nauc_recall_at_100_max": -0.004341021867975908, + "nauc_recall_at_100_std": 0.30547878506571907, + "nauc_recall_at_10_diff1": -0.18095986501038946, + "nauc_recall_at_10_max": -0.0073341347622717015, + "nauc_recall_at_10_std": 0.11791560298106384, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": -0.1845657988866603, + "nauc_recall_at_20_max": -0.0002001744986707724, + "nauc_recall_at_20_std": 0.16384027707020302, + "nauc_recall_at_3_diff1": -0.15968586730811474, + "nauc_recall_at_3_max": -0.01273857066006781, + "nauc_recall_at_3_std": 0.06752451141789033, + "nauc_recall_at_5_diff1": -0.1755997981980233, + "nauc_recall_at_5_max": -0.00769746739652761, + "nauc_recall_at_5_std": 0.08423476571748854, + "ndcg_at_1": 0.0, + "ndcg_at_10": 0.3074, + "ndcg_at_100": 0.37587, + "ndcg_at_1000": 0.53891, + "ndcg_at_20": 0.2812, + "ndcg_at_3": 0.26012, + "ndcg_at_5": 0.29826, + "precision_at_1": 0.0, + "precision_at_10": 0.33804, + "precision_at_100": 0.13023, + "precision_at_1000": 0.02473, + "precision_at_20": 0.28365, + "precision_at_3": 0.32419, + "precision_at_5": 0.35333, + "recall_at_1": 0.0, + "recall_at_10": 0.12092, + "recall_at_100": 0.46229, + "recall_at_1000": 0.87518, + "recall_at_20": 0.20234, + "recall_at_3": 0.03487, + "recall_at_5": 0.06331 + } + ] + }, + "task_name": "CUB200I2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json index 49c428cbfd..5b68c93a9f 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json @@ -1,16 +1,23 @@ { "dataset_revision": "b27a0612211af3598bd11fe28af20928f20cce06", - "evaluation_time": 205.83177590370178, + "evaluation_time": 211.27373456954956, "kg_co2_emissions": null, "mteb_version": "1.12.80", "scores": { "test": [ { + "cv_recall_at_1": 0.62418, + "cv_recall_at_10": 0.94242, + "cv_recall_at_100": 0.99689, + "cv_recall_at_1000": 0.99988, + "cv_recall_at_20": 0.97364, + "cv_recall_at_3": 0.8188, + "cv_recall_at_5": 0.88534, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.0, + "main_score": 0.62418, "map_at_1": 0.0, "map_at_10": 0.06072, "map_at_100": 0.17896, @@ -25,6 +32,27 @@ "mrr_at_20": 0.3926309438221267, "mrr_at_3": 0.35457861791651374, "mrr_at_5": 0.3791464577374308, + "nauc_cv_recall_at_1000_diff1": -1.7401334875861458, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 0.5541703308201117, + "nauc_cv_recall_at_100_diff1": -0.9997097911342215, + "nauc_cv_recall_at_100_max": 0.06743574922100379, + "nauc_cv_recall_at_100_std": 0.7587532067319325, + "nauc_cv_recall_at_10_diff1": -0.4239485200991442, + "nauc_cv_recall_at_10_max": -0.0024729549630218292, + "nauc_cv_recall_at_10_std": 0.35822657204730457, + "nauc_cv_recall_at_1_diff1": -0.22768537577324857, + "nauc_cv_recall_at_1_max": 0.012943261327137467, + "nauc_cv_recall_at_1_std": 0.06964226599453852, + "nauc_cv_recall_at_20_diff1": -0.5260289144807463, + "nauc_cv_recall_at_20_max": -0.015992094159549268, + "nauc_cv_recall_at_20_std": 0.47448921254663456, + "nauc_cv_recall_at_3_diff1": -0.30953137539602743, + "nauc_cv_recall_at_3_max": 0.013214322506575777, + "nauc_cv_recall_at_3_std": 0.18959869944763055, + "nauc_cv_recall_at_5_diff1": -0.3383429761997499, + "nauc_cv_recall_at_5_max": -0.014524830990478535, + "nauc_cv_recall_at_5_std": 0.22152839165884303, "nauc_map_at_1000_diff1": -0.20058034455775214, "nauc_map_at_1000_max": -0.009541604862555151, "nauc_map_at_1000_std": 0.36415192557738685, From fc2fcb963624e85c2c8b8c96e14350c208e33686 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Tue, 13 Aug 2024 12:33:51 +0000 Subject: [PATCH 046/154] consolidate i2t and t2i to any2any --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 4 + .../eng/Fashion200kI2TRetrieval.py | 5 +- .../eng/Fashion200kT2IRetrieval.py | 5 +- .../eng/MSCOCOI2TRetrieval.py | 4 +- .../eng/MSCOCOT2IRetrieval.py | 5 +- mteb/tasks/Image/I2TRetrieval/__init__.py | 4 - mteb/tasks/Image/I2TRetrieval/eng/__init__.py | 0 mteb/tasks/Image/T2IRetrieval/__init__.py | 4 - mteb/tasks/Image/T2IRetrieval/eng/__init__.py | 0 mteb/tasks/Image/__init__.py | 2 - mteb/tasks/__init__.py | 2 - .../Fashion200kI2TRetrieval.json | 186 ++++++++++++++++++ 12 files changed, 198 insertions(+), 23 deletions(-) rename mteb/tasks/Image/{I2TRetrieval => Any2AnyRetrieval}/eng/Fashion200kI2TRetrieval.py (94%) rename mteb/tasks/Image/{T2IRetrieval => Any2AnyRetrieval}/eng/Fashion200kT2IRetrieval.py (94%) rename mteb/tasks/Image/{I2TRetrieval => Any2AnyRetrieval}/eng/MSCOCOI2TRetrieval.py (95%) rename mteb/tasks/Image/{T2IRetrieval => Any2AnyRetrieval}/eng/MSCOCOT2IRetrieval.py (94%) delete mode 100644 mteb/tasks/Image/I2TRetrieval/__init__.py delete mode 100644 mteb/tasks/Image/I2TRetrieval/eng/__init__.py delete mode 100644 mteb/tasks/Image/T2IRetrieval/__init__.py delete mode 100644 mteb/tasks/Image/T2IRetrieval/eng/__init__.py create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Fashion200kI2TRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 2fab5b0635..9618f02342 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -2,9 +2,13 @@ from .eng.CIRRIT2IRetrieval import * from .eng.CUB200I2IRetrieval import * +from .eng.Fashion200kI2TRetrieval import * +from .eng.Fashion200kT2IRetrieval import * from .eng.FashionIQIT2IRetrieval import * from .eng.InfoSeekIT2ITRetrieval import * from .eng.InfoSeekIT2TRetrieval import * +from .eng.MSCOCOI2TRetrieval import * +from .eng.MSCOCOT2IRetrieval import * from .eng.NIGHTSI2IRetrieval import * from .eng.OVENIT2ITRetrieval import * from .eng.OVENIT2TRetrieval import * diff --git a/mteb/tasks/Image/I2TRetrieval/eng/Fashion200kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py similarity index 94% rename from mteb/tasks/Image/I2TRetrieval/eng/Fashion200kI2TRetrieval.py rename to mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py index c2409da0f8..2ff5afea56 100644 --- a/mteb/tasks/Image/I2TRetrieval/eng/Fashion200kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py @@ -1,11 +1,10 @@ from __future__ import annotations +from mteb.abstasks import AbsTaskAny2AnyRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata -from .....abstasks import AbsTaskI2TRetrieval - -class Fashion200kI2TRetrieval(AbsTaskI2TRetrieval): +class Fashion200kI2TRetrieval(AbsTaskAny2AnyRetrieval): metadata = TaskMetadata( name="Fashion200kI2TRetrieval", description="Retrieve clothes based on descriptions.", diff --git a/mteb/tasks/Image/T2IRetrieval/eng/Fashion200kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py similarity index 94% rename from mteb/tasks/Image/T2IRetrieval/eng/Fashion200kT2IRetrieval.py rename to mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py index 0eae629e5d..012ffb32a8 100644 --- a/mteb/tasks/Image/T2IRetrieval/eng/Fashion200kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py @@ -1,11 +1,10 @@ from __future__ import annotations +from mteb.abstasks import AbsTaskAny2AnyRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata -from .....abstasks import AbsTaskT2IRetrieval - -class Fashion200kT2IRetrieval(AbsTaskT2IRetrieval): +class Fashion200kT2IRetrieval(AbsTaskAny2AnyRetrieval): metadata = TaskMetadata( name="Fashion200kT2IRetrieval", description="Retrieve clothes based on descriptions.", diff --git a/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py similarity index 95% rename from mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py rename to mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py index 8047b2665c..e78a9c5873 100644 --- a/mteb/tasks/Image/I2TRetrieval/eng/MSCOCOI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py @@ -1,10 +1,10 @@ from __future__ import annotations -from mteb.abstasks import AbsTaskI2TRetrieval +from mteb.abstasks import AbsTaskAny2AnyRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata -class MSCOCOI2TRetrieval(AbsTaskI2TRetrieval): +class MSCOCOI2TRetrieval(AbsTaskAny2AnyRetrieval): metadata = TaskMetadata( name="MSCOCOI2TRetrieval", description="Retrieve captions based on images.", diff --git a/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py similarity index 94% rename from mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py rename to mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py index b9a4a57456..4846496215 100644 --- a/mteb/tasks/Image/T2IRetrieval/eng/MSCOCOT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py @@ -1,11 +1,10 @@ from __future__ import annotations +from mteb.abstasks import AbsTaskAny2AnyRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata -from .....abstasks import AbsTaskT2IRetrieval - -class MSCOCOT2IRetrieval(AbsTaskT2IRetrieval): +class MSCOCOT2IRetrieval(AbsTaskAny2AnyRetrieval): metadata = TaskMetadata( name="MSCOCOT2IRetrieval", description="Retrieve images based on captions.", diff --git a/mteb/tasks/Image/I2TRetrieval/__init__.py b/mteb/tasks/Image/I2TRetrieval/__init__.py deleted file mode 100644 index 102974c6f2..0000000000 --- a/mteb/tasks/Image/I2TRetrieval/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from __future__ import annotations - -from .eng.Fashion200kI2TRetrieval import * -from .eng.MSCOCOI2TRetrieval import * diff --git a/mteb/tasks/Image/I2TRetrieval/eng/__init__.py b/mteb/tasks/Image/I2TRetrieval/eng/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mteb/tasks/Image/T2IRetrieval/__init__.py b/mteb/tasks/Image/T2IRetrieval/__init__.py deleted file mode 100644 index 75f6eb9caf..0000000000 --- a/mteb/tasks/Image/T2IRetrieval/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from __future__ import annotations - -from .eng.Fashion200kT2IRetrieval import * -from .eng.MSCOCOT2IRetrieval import * diff --git a/mteb/tasks/Image/T2IRetrieval/eng/__init__.py b/mteb/tasks/Image/T2IRetrieval/eng/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mteb/tasks/Image/__init__.py b/mteb/tasks/Image/__init__.py index 661136d16e..255d9af637 100644 --- a/mteb/tasks/Image/__init__.py +++ b/mteb/tasks/Image/__init__.py @@ -1,8 +1,6 @@ from .Any2AnyRetrieval import * from .Clustering import * -from .I2TRetrieval import * from .ImageClassification import * from .ImageMultilabelClassification import * from .ImageTextPairClassification import * -from .T2IRetrieval import * from .ZeroshotClassification import * diff --git a/mteb/tasks/__init__.py b/mteb/tasks/__init__.py index e6593154d6..1505b15bc2 100644 --- a/mteb/tasks/__init__.py +++ b/mteb/tasks/__init__.py @@ -3,10 +3,8 @@ from .BitextMining import * from .Classification import * from .Clustering import * -from .Image.I2TRetrieval import * from .Image.ImageClassification import * from .Image.ImageTextPairClassification import * -from .Image.T2IRetrieval import * from .Image.ZeroshotClassification import * from .InstructionRetrieval import * from .MultiLabelClassification import * diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Fashion200kI2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Fashion200kI2TRetrieval.json new file mode 100644 index 0000000000..d8b3f67a4e --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Fashion200kI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "96a313715ecf67f5dfe70c4fa52406bc7bdfbeee", + "evaluation_time": 165.14273023605347, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.01657, + "cv_recall_at_10": 0.07486, + "cv_recall_at_100": 0.26652, + "cv_recall_at_1000": 0.62487, + "cv_recall_at_20": 0.11638, + "cv_recall_at_3": 0.03559, + "cv_recall_at_5": 0.05011, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.04138, + "map_at_1": 0.01657, + "map_at_10": 0.03121, + "map_at_100": 0.03747, + "map_at_1000": 0.03879, + "map_at_20": 0.03402, + "map_at_3": 0.02461, + "map_at_5": 0.02794, + "mrr_at_1": 0.01656780527715279, + "mrr_at_10": 0.031209923800433102, + "mrr_at_100": 0.03746585906947475, + "mrr_at_1000": 0.03878613232811227, + "mrr_at_20": 0.0340226319794799, + "mrr_at_3": 0.02461307697552327, + "mrr_at_5": 0.027936865071248384, + "nauc_cv_recall_at_1000_diff1": 0.05215643497408927, + "nauc_cv_recall_at_1000_max": 0.16019254559848906, + "nauc_cv_recall_at_1000_std": 0.3630252561689271, + "nauc_cv_recall_at_100_diff1": 0.09934080212830879, + "nauc_cv_recall_at_100_max": 0.11775386806178802, + "nauc_cv_recall_at_100_std": 0.2626429228260664, + "nauc_cv_recall_at_10_diff1": 0.14732289617853372, + "nauc_cv_recall_at_10_max": 0.1300838307363312, + "nauc_cv_recall_at_10_std": 0.22261060186404014, + "nauc_cv_recall_at_1_diff1": 0.43658812029041394, + "nauc_cv_recall_at_1_max": 0.20106424998008995, + "nauc_cv_recall_at_1_std": 0.21594406605223246, + "nauc_cv_recall_at_20_diff1": 0.12199658363483398, + "nauc_cv_recall_at_20_max": 0.13309204661129767, + "nauc_cv_recall_at_20_std": 0.24973972652762613, + "nauc_cv_recall_at_3_diff1": 0.24662246338856247, + "nauc_cv_recall_at_3_max": 0.15878837526882447, + "nauc_cv_recall_at_3_std": 0.18589391029851615, + "nauc_cv_recall_at_5_diff1": 0.19081976581565688, + "nauc_cv_recall_at_5_max": 0.15164531948619475, + "nauc_cv_recall_at_5_std": 0.21248017451063386, + "nauc_map_at_1000_diff1": 0.2406708231108197, + "nauc_map_at_1000_max": 0.16014121809590698, + "nauc_map_at_1000_std": 0.22704010525299115, + "nauc_map_at_100_diff1": 0.24219026893873466, + "nauc_map_at_100_max": 0.15936399671152812, + "nauc_map_at_100_std": 0.22429926115695684, + "nauc_map_at_10_diff1": 0.2674661720376913, + "nauc_map_at_10_max": 0.16657906217191382, + "nauc_map_at_10_std": 0.2149240991921386, + "nauc_map_at_1_diff1": 0.43658812029041394, + "nauc_map_at_1_max": 0.20106424998008995, + "nauc_map_at_1_std": 0.21594406605223246, + "nauc_map_at_20_diff1": 0.25261446616694033, + "nauc_map_at_20_max": 0.16383021382320623, + "nauc_map_at_20_std": 0.22094405123394173, + "nauc_map_at_3_diff1": 0.32374852815286687, + "nauc_map_at_3_max": 0.18065670256553057, + "nauc_map_at_3_std": 0.20149692020784588, + "nauc_map_at_5_diff1": 0.29208437849888963, + "nauc_map_at_5_max": 0.17535941533844787, + "nauc_map_at_5_std": 0.21150556697054126, + "nauc_mrr_at_1000_diff1": 0.2406708231108197, + "nauc_mrr_at_1000_max": 0.16014121809590698, + "nauc_mrr_at_1000_std": 0.22704010525299115, + "nauc_mrr_at_100_diff1": 0.24219026893873466, + "nauc_mrr_at_100_max": 0.15936399671152812, + "nauc_mrr_at_100_std": 0.22429926115695684, + "nauc_mrr_at_10_diff1": 0.2674661720376913, + "nauc_mrr_at_10_max": 0.16657906217191382, + "nauc_mrr_at_10_std": 0.2149240991921386, + "nauc_mrr_at_1_diff1": 0.43658812029041394, + "nauc_mrr_at_1_max": 0.20106424998008995, + "nauc_mrr_at_1_std": 0.21594406605223246, + "nauc_mrr_at_20_diff1": 0.25261446616694033, + "nauc_mrr_at_20_max": 0.16383021382320623, + "nauc_mrr_at_20_std": 0.22094405123394173, + "nauc_mrr_at_3_diff1": 0.32374852815286687, + "nauc_mrr_at_3_max": 0.18065670256553057, + "nauc_mrr_at_3_std": 0.20149692020784588, + "nauc_mrr_at_5_diff1": 0.29208437849888963, + "nauc_mrr_at_5_max": 0.17535941533844787, + "nauc_mrr_at_5_std": 0.21150556697054126, + "nauc_ndcg_at_1000_diff1": 0.1473174010573472, + "nauc_ndcg_at_1000_max": 0.1509480108109463, + "nauc_ndcg_at_1000_std": 0.27887845367426434, + "nauc_ndcg_at_100_diff1": 0.1627532520071558, + "nauc_ndcg_at_100_max": 0.1363846085513206, + "nauc_ndcg_at_100_std": 0.24358663136522796, + "nauc_ndcg_at_10_diff1": 0.21697906584798596, + "nauc_ndcg_at_10_max": 0.15163132959233716, + "nauc_ndcg_at_10_std": 0.21785059479581834, + "nauc_ndcg_at_1_diff1": 0.43658812029041394, + "nauc_ndcg_at_1_max": 0.20106424998008995, + "nauc_ndcg_at_1_std": 0.21594406605223246, + "nauc_ndcg_at_20_diff1": 0.19023028222093716, + "nauc_ndcg_at_20_max": 0.14880588752163815, + "nauc_ndcg_at_20_std": 0.233151196984513, + "nauc_ndcg_at_3_diff1": 0.29829003226909856, + "nauc_ndcg_at_3_max": 0.17360427192148528, + "nauc_ndcg_at_3_std": 0.19646665255827292, + "nauc_ndcg_at_5_diff1": 0.2548364192942451, + "nauc_ndcg_at_5_max": 0.16670644633246848, + "nauc_ndcg_at_5_std": 0.21170762231114157, + "nauc_precision_at_1000_diff1": 0.05215643497408827, + "nauc_precision_at_1000_max": 0.1601925455984888, + "nauc_precision_at_1000_std": 0.3630252561689268, + "nauc_precision_at_100_diff1": 0.09934080212830884, + "nauc_precision_at_100_max": 0.11775386806178809, + "nauc_precision_at_100_std": 0.26264292282606644, + "nauc_precision_at_10_diff1": 0.14732289617853392, + "nauc_precision_at_10_max": 0.13008383073633142, + "nauc_precision_at_10_std": 0.22261060186404052, + "nauc_precision_at_1_diff1": 0.43658812029041394, + "nauc_precision_at_1_max": 0.20106424998008995, + "nauc_precision_at_1_std": 0.21594406605223246, + "nauc_precision_at_20_diff1": 0.12199658363483433, + "nauc_precision_at_20_max": 0.13309204661129775, + "nauc_precision_at_20_std": 0.24973972652762635, + "nauc_precision_at_3_diff1": 0.24662246338856234, + "nauc_precision_at_3_max": 0.15878837526882422, + "nauc_precision_at_3_std": 0.18589391029851624, + "nauc_precision_at_5_diff1": 0.19081976581565693, + "nauc_precision_at_5_max": 0.1516453194861949, + "nauc_precision_at_5_std": 0.21248017451063406, + "nauc_recall_at_1000_diff1": 0.05215643497408927, + "nauc_recall_at_1000_max": 0.16019254559848906, + "nauc_recall_at_1000_std": 0.3630252561689271, + "nauc_recall_at_100_diff1": 0.09934080212830879, + "nauc_recall_at_100_max": 0.11775386806178802, + "nauc_recall_at_100_std": 0.2626429228260664, + "nauc_recall_at_10_diff1": 0.14732289617853372, + "nauc_recall_at_10_max": 0.1300838307363312, + "nauc_recall_at_10_std": 0.22261060186404014, + "nauc_recall_at_1_diff1": 0.43658812029041394, + "nauc_recall_at_1_max": 0.20106424998008995, + "nauc_recall_at_1_std": 0.21594406605223246, + "nauc_recall_at_20_diff1": 0.12199658363483398, + "nauc_recall_at_20_max": 0.13309204661129767, + "nauc_recall_at_20_std": 0.24973972652762613, + "nauc_recall_at_3_diff1": 0.24662246338856247, + "nauc_recall_at_3_max": 0.15878837526882447, + "nauc_recall_at_3_std": 0.18589391029851615, + "nauc_recall_at_5_diff1": 0.19081976581565688, + "nauc_recall_at_5_max": 0.15164531948619475, + "nauc_recall_at_5_std": 0.21248017451063386, + "ndcg_at_1": 0.01657, + "ndcg_at_10": 0.04138, + "ndcg_at_100": 0.07884, + "ndcg_at_1000": 0.1221, + "ndcg_at_20": 0.05179, + "ndcg_at_3": 0.02742, + "ndcg_at_5": 0.0334, + "precision_at_1": 0.01657, + "precision_at_10": 0.00749, + "precision_at_100": 0.00267, + "precision_at_1000": 0.00062, + "precision_at_20": 0.00582, + "precision_at_3": 0.01186, + "precision_at_5": 0.01002, + "recall_at_1": 0.01657, + "recall_at_10": 0.07486, + "recall_at_100": 0.26652, + "recall_at_1000": 0.62487, + "recall_at_20": 0.11638, + "recall_at_3": 0.03559, + "recall_at_5": 0.05011 + } + ] + }, + "task_name": "Fashion200kI2TRetrieval" +} \ No newline at end of file From 229c39237c2509ef93f23eb1162759519e57eef7 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Tue, 13 Aug 2024 12:50:34 +0000 Subject: [PATCH 047/154] remove abstask and evaluators --- mteb/abstasks/Image/AbsTaskI2TRetrieval.py | 455 ------------------ mteb/abstasks/Image/AbsTaskT2IRetrieval.py | 452 ----------------- mteb/abstasks/__init__.py | 2 - .../evaluators/Image/I2TRetrievalEvaluator.py | 388 --------------- .../evaluators/Image/T2IRetrievalEvaluator.py | 388 --------------- mteb/evaluation/evaluators/__init__.py | 2 - 6 files changed, 1687 deletions(-) delete mode 100644 mteb/abstasks/Image/AbsTaskI2TRetrieval.py delete mode 100644 mteb/abstasks/Image/AbsTaskT2IRetrieval.py delete mode 100644 mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py delete mode 100644 mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py diff --git a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py b/mteb/abstasks/Image/AbsTaskI2TRetrieval.py deleted file mode 100644 index 4920e77cb3..0000000000 --- a/mteb/abstasks/Image/AbsTaskI2TRetrieval.py +++ /dev/null @@ -1,455 +0,0 @@ -from __future__ import annotations - -import json -import logging -import os -from collections import defaultdict -from pathlib import Path -from time import time -from typing import Any, Dict, Tuple - -import tqdm -from datasets import Features, Value, load_dataset -from PIL import Image - -from ...evaluation.evaluators import I2TRetrievalEvaluator -from ...load_results.mteb_results import ScoresDict -from ..AbsTask import AbsTask - -logger = logging.getLogger(__name__) - - -class HFDataLoader: - def __init__( - self, - hf_repo: str | None = None, - hf_repo_qrels: str | None = None, - data_folder: str | None = None, - prefix: str | None = None, - corpus_file: str = "corpus.jsonl", - query_file: str = "queries.jsonl", - qrels_folder: str = "qrels", - qrels_file: str = "", - streaming: bool = False, - keep_in_memory: bool = False, - ): - self.corpus = {} - self.queries = {} - self.qrels = {} - self.hf_repo = hf_repo - if hf_repo: - # By default fetch qrels from same repo not a second repo with "-qrels" like in original - self.hf_repo_qrels = hf_repo_qrels if hf_repo_qrels else hf_repo - else: - # data folder would contain these files: - # (1) fiqa/corpus.jsonl (format: jsonlines) - # (2) fiqa/queries.jsonl (format: jsonlines) - # (3) fiqa/qrels/test.tsv (format: tsv ("\t")) - if prefix: - query_file = prefix + "-" + query_file - qrels_folder = prefix + "-" + qrels_folder - - self.corpus_file = ( - os.path.join(data_folder, corpus_file) if data_folder else corpus_file - ) - self.query_file = ( - os.path.join(data_folder, query_file) if data_folder else query_file - ) - self.qrels_folder = ( - os.path.join(data_folder, qrels_folder) if data_folder else None - ) - self.qrels_file = qrels_file - self.streaming = streaming - self.keep_in_memory = keep_in_memory - - @staticmethod - def check(fIn: str, ext: str): - if not os.path.exists(fIn): - raise ValueError( - "File {} not present! Please provide accurate file.".format(fIn) - ) - - if not fIn.endswith(ext): - raise ValueError( - "File {} must be present with extension {}".format(fIn, ext) - ) - - def load( - self, split="test" - ) -> Tuple[ - Dict[str, dict[str, str]], dict[str, Image.Image], dict[str, dict[str, int]] - ]: - if not self.hf_repo: - self.qrels_file = os.path.join(self.qrels_folder, split + ".tsv") - self.check(fIn=self.corpus_file, ext="jsonl") - self.check(fIn=self.query_file, ext="jsonl") - self.check(fIn=self.qrels_file, ext="tsv") - - if not len(self.corpus): - logger.info("Loading Corpus...") - self._load_corpus() - logger.info("Loaded %d %s Documents.", len(self.corpus), split.upper()) - logger.info("Doc Example: %s", self.corpus[0]) - - if not len(self.queries): - logger.info("Loading Queries...") - self._load_queries(split) - - self._load_qrels(split) - # filter queries with no qrels - qrels_dict = defaultdict(dict) - - def qrels_dict_init(row): - qrels_dict[row["query-id"]][row["corpus-id"]] = int(row["score"]) - - self.qrels.map(qrels_dict_init) - self.qrels = qrels_dict - self.queries = self.queries.filter(lambda x: x["id"] in self.qrels) - logger.info("Loaded %d %s Queries.", len(self.queries), split.upper()) - logger.info("Query Example: %s", self.queries[0]) - - return self.corpus, self.queries, self.qrels - - def load_corpus(self) -> dict[str, dict[str, str]]: - if not self.hf_repo: - self.check(fIn=self.corpus_file, ext="jsonl") - - if not len(self.corpus): - logger.info("Loading Corpus...") - self._load_corpus() - logger.info("Loaded %d %s Documents.", len(self.corpus)) - logger.info("Doc Example: %s", self.corpus[0]) - - return self.corpus - - def _load_corpus(self): - if self.hf_repo: - corpus_ds = load_dataset( - self.hf_repo, - "corpus", - keep_in_memory=self.keep_in_memory, - streaming=self.streaming, - )["corpus"] - else: - corpus_ds = load_dataset( - "json", - data_files=self.corpus_file, - streaming=self.streaming, - keep_in_memory=self.keep_in_memory, - ) - self.corpus = corpus_ds - - def _load_queries(self, split): - if self.hf_repo: - queries_ds = load_dataset( - self.hf_repo, - "query", - keep_in_memory=self.keep_in_memory, - streaming=self.streaming, - )[split] - else: - queries_ds = load_dataset( - "json", - data_files=self.query_file, - streaming=self.streaming, - keep_in_memory=self.keep_in_memory, - ) - self.queries = queries_ds - - def _load_qrels(self, split): - if self.hf_repo: - qrels_ds = load_dataset( - self.hf_repo_qrels, - "qrels", - keep_in_memory=self.keep_in_memory, - streaming=self.streaming, - )[split] - else: - qrels_ds = load_dataset( - "csv", - data_files=self.qrels_file, - delimiter="\t", - keep_in_memory=self.keep_in_memory, - ) - qrels_ds = qrels_ds.remove_columns("Q0") - features = Features( - { - "query-id": Value("string"), - "corpus-id": Value("string"), - "score": Value("float"), - } - ) - qrels_ds = qrels_ds.cast(features) - self.qrels = qrels_ds - - -class AbsTaskI2TRetrieval(AbsTask): - """Abstract class for retrieval experiments. - - Child-classes must implement the following properties: - - self.corpus: dict[str, dict[str, str]] - Semantically, it should contain dict[split_name, dict[sample_id, dict[str, str]]] - E.g. {"test": {"document_one": {"_id": "d1", "title": "title", "text": "text"}}} - - self.queries: dict[str, dict[str, Union[str, List[str]]]] - Semantically, it should contain dict[split_name, dict[sample_id, str]] or dict[split_name, dict[sample_id, List[str]]] for conversations - E.g. {"test": {"q1": "query"}} - or {"test": {"q1": ["turn1", "turn2", "turn3"]}} - - self.relevant_docs: dict[str, dict[str, dict[str, int]]] - Semantically, it should contain dict[split_name, dict[sample_id, dict[doc_id, score]]] - E.g.: {"test": {"q1": {"document_one": 1}}} - """ - - ignore_identical_ids: bool = False - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def load_data(self, **kwargs): - if self.data_loaded: - return - self.corpus, self.queries, self.relevant_docs = {}, {}, {} - dataset_path = self.metadata_dict["dataset"]["path"] - - for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): - corpus, queries, qrels = HFDataLoader( - hf_repo=dataset_path, - streaming=False, - keep_in_memory=False, - ).load(split=split) - # Directly pass in queries and corpus datasets to prevent loading into memory - # queries = {query["id"]: query["image"] for query in queries} - # corpus = {doc["id"]: {"text": doc["text"]} for doc in corpus} - self.corpus[split], self.queries[split], self.relevant_docs[split] = ( - corpus, - queries, - qrels, - ) - - self.data_loaded = True - - def evaluate( - self, - model, - split: str = "test", - *, - encode_kwargs: dict[str, Any] = {}, - **kwargs, - ): - retriever = I2TRetrievalEvaluator( - retriever=model, - task_name=self.metadata.name, - encode_kwargs=encode_kwargs, - **kwargs, - ) - - scores = {} - hf_subsets = ( - [l for l in self.hf_subsets] if self.is_multilingual else ["default"] - ) - - for hf_subset in hf_subsets: - logger.info(f"Subset: {hf_subset}") - - if hf_subset == "default": - corpus, queries, relevant_docs = ( - self.corpus[split], - self.queries[split], - self.relevant_docs[split], - ) - else: - corpus, queries, relevant_docs = ( - self.corpus[hf_subset][split], - self.queries[hf_subset][split], - self.relevant_docs[hf_subset][split], - ) - scores[hf_subset] = self._evaluate_subset( - retriever, corpus, queries, relevant_docs, hf_subset, **kwargs - ) - return scores - - def _evaluate_subset( - self, retriever, corpus, queries, relevant_docs, hf_subset: str, **kwargs - ): - start_time = time() - results = retriever(corpus, queries) - end_time = time() - logger.info( - "Time taken to retrieve: {:.2f} seconds".format(end_time - start_time) - ) - - save_predictions = kwargs.get("save_predictions", False) - export_errors = kwargs.get("export_errors", False) - if save_predictions or export_errors: - output_folder = Path(kwargs.get("output_folder", "results")) - if not os.path.isdir(output_folder): - os.makedirs(output_folder) - - if save_predictions: - top_k = kwargs.get("top_k", None) - if top_k is not None: - for qid in list(results.keys()): - doc_ids = set( - sorted( - results[qid], key=lambda x: results[qid][x], reverse=True - )[:top_k] - ) - results[qid] = { - k: v for k, v in results[qid].items() if k in doc_ids - } - qrels_save_path = ( - output_folder / f"{self.metadata.name}_{hf_subset}_predictions.json" - ) - - with open(qrels_save_path, "w") as f: - json.dump(results, f) - - ndcg, _map, recall, precision, cv_recall, naucs = retriever.evaluate( - relevant_docs, - results, - retriever.k_values, - ignore_identical_ids=self.ignore_identical_ids, - ) - mrr, naucs_mrr = retriever.evaluate_custom( - relevant_docs, results, retriever.k_values, "mrr" - ) - scores = { - **{f"ndcg_at_{k.split('@')[1]}": v for (k, v) in ndcg.items()}, - **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()}, - **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()}, - **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()}, - **{f"cv_recall_at_{k.split('@')[1]}": v for (k, v) in cv_recall.items()}, - **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, - **{ - k.replace("@", "_at_").replace("_P", "_precision").lower(): v - for k, v in naucs.items() - }, - **{ - k.replace("@", "_at_").replace("_P", "_precision").lower(): v - for k, v in naucs_mrr.items() - }, - } - self._add_main_score(scores) - - if export_errors: - errors = {} - - top_k = kwargs.get("top_k", 1) - if not save_predictions and top_k == 1: - for qid in results.keys(): - doc_scores = results[qid] - sorted_docs = sorted( - doc_scores.items(), key=lambda x: x[1], reverse=True - )[:top_k] - results[qid] = {doc_id: score for doc_id, score in sorted_docs} - for qid, retrieved_docs in results.items(): - expected_docs = relevant_docs[qid] - false_positives = [ - doc for doc in retrieved_docs if doc not in expected_docs - ] - false_negatives = [ - doc for doc in expected_docs if doc not in retrieved_docs - ] - if false_positives or false_negatives: - errors[qid] = { - "false_positives": false_positives, - "false_negatives": false_negatives, - } - - errors_save_path = ( - output_folder / f"{self.metadata.name}_{hf_subset}_errors.json" - ) - with open(errors_save_path, "w") as f: - json.dump(errors, f) - - return scores - - def _add_main_score(self, scores: ScoresDict) -> None: - scores["main_score"] = scores[self.metadata.main_score] - - def calculate_metadata_metrics(self) -> None: - self.load_data() - - all_details = {} - pbar_split = tqdm.tqdm( - self.metadata_dict["eval_splits"], desc="Processing Splits..." - ) - for split in pbar_split: - pbar_split.set_postfix_str(f"Split: {split}") - print(f"Processing metadata for split {split}") - all_details[split] = {} - if self.is_multilingual: - pbar_lang = tqdm.tqdm( - self.relevant_docs.keys(), desc="Processing Languages..." - ) - for lang in pbar_lang: - pbar_lang.set_postfix_str(f"Language: {lang}") - print(f"Processing metadata for language {lang}") - split_details = process_language( - self.relevant_docs[lang][split], - self.queries[lang][split], - self.corpus[lang][split], - lang, - ) - all_details[split][lang] = split_details - else: - split_details = process_language( - self.relevant_docs[split], self.queries[split], self.corpus[split] - ) - all_details[split] = split_details - - return all_details - - -def process_language(relevant_docs, queries, corpus, lang=None): - """We want to get three pieces of information: - - the number of documents (and their char length) in the corpus - - the number of queries (and their char length) - - the average number of relevant documents per query - """ - query_len, doc_len = calculate_length(queries, corpus) - num_documents = len(corpus) - num_queries = len(queries) - - # number of qrels that are not 0 - num_qrels_non_zero = sum( - sum(1 for doc_id in docs if docs[doc_id] != 0) - for docs in relevant_docs.values() - ) - qrels_per_doc = num_qrels_non_zero / num_queries if num_queries else 0 - - language_description = f" for language {lang}" if lang else "" - print(f"Average document character length{language_description} is {doc_len}") - print(f"Average query character length{language_description} is {query_len}") - print(f"Number of documents{language_description} is {num_documents}") - print(f"Number of queries{language_description} is {num_queries}") - print( - f"Average number of relevant documents per query{language_description} is {qrels_per_doc}" - ) - return { - "average_document_length": doc_len, - "average_query_length": query_len, - "num_documents": num_documents, - "num_queries": num_queries, - "average_relevant_docs_per_query": qrels_per_doc, - } - - -def calculate_length(queries, corpus): - queries_lens = [] - doc_lens = [] - for query in queries.values(): - # for image append 1. Can perhaps be removed. - queries_lens.append(1.0) - - for doc in corpus.values(): - if isinstance(doc, dict): - doc_lens.append(len(doc["text"])) - else: - doc_lens.append(len(doc)) - - doc_len = sum(doc_lens) / len(doc_lens) if doc_lens else 0 - query_len = sum(queries_lens) / len(queries_lens) if queries_lens else 0 - return query_len, doc_len diff --git a/mteb/abstasks/Image/AbsTaskT2IRetrieval.py b/mteb/abstasks/Image/AbsTaskT2IRetrieval.py deleted file mode 100644 index 7269abcdfa..0000000000 --- a/mteb/abstasks/Image/AbsTaskT2IRetrieval.py +++ /dev/null @@ -1,452 +0,0 @@ -from __future__ import annotations - -import json -import logging -import os -from collections import defaultdict -from pathlib import Path -from time import time -from typing import Any, Dict, Tuple - -import tqdm -from datasets import Features, Value, load_dataset -from PIL import Image - -from ...evaluation.evaluators import T2IRetrievalEvaluator -from ...load_results.mteb_results import ScoresDict -from ..AbsTask import AbsTask - -logger = logging.getLogger(__name__) - - -class HFDataLoader: - def __init__( - self, - hf_repo: str | None = None, - hf_repo_qrels: str | None = None, - data_folder: str | None = None, - prefix: str | None = None, - corpus_file: str = "corpus.jsonl", - query_file: str = "queries.jsonl", - qrels_folder: str = "qrels", - qrels_file: str = "", - streaming: bool = False, - keep_in_memory: bool = False, - ): - self.corpus = {} - self.queries = {} - self.qrels = {} - self.hf_repo = hf_repo - if hf_repo: - # By default fetch qrels from same repo not a second repo with "-qrels" like in original - self.hf_repo_qrels = hf_repo_qrels if hf_repo_qrels else hf_repo - else: - # data folder would contain these files: - # (1) fiqa/corpus.jsonl (format: jsonlines) - # (2) fiqa/queries.jsonl (format: jsonlines) - # (3) fiqa/qrels/test.tsv (format: tsv ("\t")) - if prefix: - query_file = prefix + "-" + query_file - qrels_folder = prefix + "-" + qrels_folder - - self.corpus_file = ( - os.path.join(data_folder, corpus_file) if data_folder else corpus_file - ) - self.query_file = ( - os.path.join(data_folder, query_file) if data_folder else query_file - ) - self.qrels_folder = ( - os.path.join(data_folder, qrels_folder) if data_folder else None - ) - self.qrels_file = qrels_file - self.streaming = streaming - self.keep_in_memory = keep_in_memory - - @staticmethod - def check(fIn: str, ext: str): - if not os.path.exists(fIn): - raise ValueError( - "File {} not present! Please provide accurate file.".format(fIn) - ) - - if not fIn.endswith(ext): - raise ValueError( - "File {} must be present with extension {}".format(fIn, ext) - ) - - def load( - self, split="test" - ) -> Tuple[ - Dict[str, Image.Image], dict[str, dict[str, str]], dict[str, dict[str, int]] - ]: - if not self.hf_repo: - self.qrels_file = os.path.join(self.qrels_folder, split + ".tsv") - self.check(fIn=self.corpus_file, ext="jsonl") - self.check(fIn=self.query_file, ext="jsonl") - self.check(fIn=self.qrels_file, ext="tsv") - - if not len(self.corpus): - logger.info("Loading Corpus...") - self._load_corpus() - logger.info("Loaded %d %s Documents.", len(self.corpus), split.upper()) - logger.info("Doc Example: %s", self.corpus[0]) - - if not len(self.queries): - logger.info("Loading Queries...") - self._load_queries(split) - - self._load_qrels(split) - # filter queries with no qrels - qrels_dict = defaultdict(dict) - - def qrels_dict_init(row): - qrels_dict[row["query-id"]][row["corpus-id"]] = int(row["score"]) - - self.qrels.map(qrels_dict_init) - self.qrels = qrels_dict - self.queries = self.queries.filter(lambda x: x["id"] in self.qrels) - logger.info("Loaded %d %s Queries.", len(self.queries), split.upper()) - logger.info("Query Example: %s", self.queries[0]) - - return self.corpus, self.queries, self.qrels - - def load_corpus(self) -> dict[str, dict[str, str]]: - if not self.hf_repo: - self.check(fIn=self.corpus_file, ext="jsonl") - - if not len(self.corpus): - logger.info("Loading Corpus...") - self._load_corpus() - logger.info("Loaded %d %s Documents.", len(self.corpus)) - logger.info("Doc Example: %s", self.corpus[0]) - - return self.corpus - - def _load_corpus(self): - if self.hf_repo: - corpus_ds = load_dataset( - self.hf_repo, - "corpus", - keep_in_memory=self.keep_in_memory, - streaming=self.streaming, - )["corpus"] - else: - corpus_ds = load_dataset( - "json", - data_files=self.corpus_file, - streaming=self.streaming, - keep_in_memory=self.keep_in_memory, - ) - self.corpus = corpus_ds - - def _load_queries(self, split): - if self.hf_repo: - queries_ds = load_dataset( - self.hf_repo, - "query", - keep_in_memory=self.keep_in_memory, - streaming=self.streaming, - )[split] - else: - queries_ds = load_dataset( - "json", - data_files=self.query_file, - streaming=self.streaming, - keep_in_memory=self.keep_in_memory, - ) - self.queries = queries_ds - - def _load_qrels(self, split): - if self.hf_repo: - qrels_ds = load_dataset( - self.hf_repo_qrels, - "qrels", - keep_in_memory=self.keep_in_memory, - streaming=self.streaming, - )[split] - else: - qrels_ds = load_dataset( - "csv", - data_files=self.qrels_file, - delimiter="\t", - keep_in_memory=self.keep_in_memory, - ) - qrels_ds = qrels_ds.remove_columns("Q0") - features = Features( - { - "query-id": Value("string"), - "corpus-id": Value("string"), - "score": Value("float"), - } - ) - qrels_ds = qrels_ds.cast(features) - self.qrels = qrels_ds - - -class AbsTaskT2IRetrieval(AbsTask): - """Abstract class for retrieval experiments. - - Child-classes must implement the following properties: - - self.corpus: dict[str, dict[str, str]] - Semantically, it should contain dict[split_name, dict[sample_id, dict[str, str]]] - E.g. {"test": {"document_one": {"_id": "d1", "title": "title", "text": "text"}}} - - self.queries: dict[str, dict[str, Union[str, List[str]]]] - Semantically, it should contain dict[split_name, dict[sample_id, str]] or dict[split_name, dict[sample_id, List[str]]] for conversations - E.g. {"test": {"q1": "query"}} - or {"test": {"q1": ["turn1", "turn2", "turn3"]}} - - self.relevant_docs: dict[str, dict[str, dict[str, int]]] - Semantically, it should contain dict[split_name, dict[sample_id, dict[doc_id, score]]] - E.g.: {"test": {"q1": {"document_one": 1}}} - """ - - ignore_identical_ids: bool = False - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def load_data(self, **kwargs): - if self.data_loaded: - return - self.corpus, self.queries, self.relevant_docs = {}, {}, {} - dataset_path = self.metadata_dict["dataset"]["path"] - - for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): - corpus, queries, qrels = HFDataLoader( - hf_repo=dataset_path, - streaming=False, - keep_in_memory=False, - ).load(split=split) - # directly pass in queries and corpus dataset to prevent loading into memory - # queries = {query["id"]: {"text": query["text"]} for query in queries} - # corpus = {image["id"]: image["image"] for image in corpus} - self.corpus[split], self.queries[split], self.relevant_docs[split] = ( - corpus, - queries, - qrels, - ) - - self.data_loaded = True - - def evaluate( - self, - model, - split: str = "test", - *, - encode_kwargs: dict[str, Any] = {}, - **kwargs, - ): - retriever = T2IRetrievalEvaluator( - retriever=model, - task_name=self.metadata.name, - encode_kwargs=encode_kwargs, - **kwargs, - ) - - scores = {} - hf_subsets = ( - [l for l in self.hf_subsets] if self.is_multilingual else ["default"] - ) - - for hf_subset in hf_subsets: - logger.info(f"Subset: {hf_subset}") - - if hf_subset == "default": - corpus, queries, relevant_docs = ( - self.corpus[split], - self.queries[split], - self.relevant_docs[split], - ) - else: - corpus, queries, relevant_docs = ( - self.corpus[hf_subset][split], - self.queries[hf_subset][split], - self.relevant_docs[hf_subset][split], - ) - scores[hf_subset] = self._evaluate_subset( - retriever, corpus, queries, relevant_docs, hf_subset, **kwargs - ) - return scores - - def _evaluate_subset( - self, retriever, corpus, queries, relevant_docs, hf_subset: str, **kwargs - ): - start_time = time() - results = retriever(corpus, queries) - end_time = time() - logger.info( - "Time taken to retrieve: {:.2f} seconds".format(end_time - start_time) - ) - - save_predictions = kwargs.get("save_predictions", False) - export_errors = kwargs.get("export_errors", False) - if save_predictions or export_errors: - output_folder = Path(kwargs.get("output_folder", "results")) - if not os.path.isdir(output_folder): - os.makedirs(output_folder) - - if save_predictions: - top_k = kwargs.get("top_k", None) - if top_k is not None: - for qid in list(results.keys()): - doc_ids = set( - sorted( - results[qid], key=lambda x: results[qid][x], reverse=True - )[:top_k] - ) - results[qid] = { - k: v for k, v in results[qid].items() if k in doc_ids - } - qrels_save_path = ( - output_folder / f"{self.metadata.name}_{hf_subset}_predictions.json" - ) - - with open(qrels_save_path, "w") as f: - json.dump(results, f) - - ndcg, _map, recall, precision, cv_recall, naucs = retriever.evaluate( - relevant_docs, - results, - retriever.k_values, - ignore_identical_ids=self.ignore_identical_ids, - ) - mrr, naucs_mrr = retriever.evaluate_custom( - relevant_docs, results, retriever.k_values, "mrr" - ) - scores = { - **{f"ndcg_at_{k.split('@')[1]}": v for (k, v) in ndcg.items()}, - **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()}, - **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()}, - **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()}, - **{f"cv_recall_at_{k.split('@')[1]}": v for (k, v) in cv_recall.items()}, - **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, - **{ - k.replace("@", "_at_").replace("_P", "_precision").lower(): v - for k, v in naucs.items() - }, - **{ - k.replace("@", "_at_").replace("_P", "_precision").lower(): v - for k, v in naucs_mrr.items() - }, - } - self._add_main_score(scores) - - if export_errors: - errors = {} - - top_k = kwargs.get("top_k", 1) - if not save_predictions and top_k == 1: - for qid in results.keys(): - doc_scores = results[qid] - sorted_docs = sorted( - doc_scores.items(), key=lambda x: x[1], reverse=True - )[:top_k] - results[qid] = {doc_id: score for doc_id, score in sorted_docs} - for qid, retrieved_docs in results.items(): - expected_docs = relevant_docs[qid] - false_positives = [ - doc for doc in retrieved_docs if doc not in expected_docs - ] - false_negatives = [ - doc for doc in expected_docs if doc not in retrieved_docs - ] - if false_positives or false_negatives: - errors[qid] = { - "false_positives": false_positives, - "false_negatives": false_negatives, - } - - errors_save_path = ( - output_folder / f"{self.metadata.name}_{hf_subset}_errors.json" - ) - with open(errors_save_path, "w") as f: - json.dump(errors, f) - - return scores - - def _add_main_score(self, scores: ScoresDict) -> None: - scores["main_score"] = scores[self.metadata.main_score] - - def calculate_metadata_metrics(self) -> None: - self.load_data() - - all_details = {} - pbar_split = tqdm.tqdm( - self.metadata_dict["eval_splits"], desc="Processing Splits..." - ) - for split in pbar_split: - pbar_split.set_postfix_str(f"Split: {split}") - print(f"Processing metadata for split {split}") - all_details[split] = {} - if self.is_multilingual: - pbar_lang = tqdm.tqdm( - self.relevant_docs.keys(), desc="Processing Languages..." - ) - for lang in pbar_lang: - pbar_lang.set_postfix_str(f"Language: {lang}") - print(f"Processing metadata for language {lang}") - split_details = process_language( - self.relevant_docs[lang][split], - self.queries[lang][split], - self.corpus[lang][split], - lang, - ) - all_details[split][lang] = split_details - else: - split_details = process_language( - self.relevant_docs[split], self.queries[split], self.corpus[split] - ) - all_details[split] = split_details - - return all_details - - -def process_language(relevant_docs, queries, corpus, lang=None): - """We want to get three pieces of information: - - the number of documents (and their char length) in the corpus - - the number of queries (and their char length) - - the average number of relevant documents per query - """ - query_len, doc_len = calculate_length(queries, corpus) - num_documents = len(corpus) - num_queries = len(queries) - - # number of qrels that are not 0 - num_qrels_non_zero = sum( - sum(1 for doc_id in docs if docs[doc_id] != 0) - for docs in relevant_docs.values() - ) - qrels_per_doc = num_qrels_non_zero / num_queries if num_queries else 0 - - language_description = f" for language {lang}" if lang else "" - print(f"Average document character length{language_description} is {doc_len}") - print(f"Average query character length{language_description} is {query_len}") - print(f"Number of documents{language_description} is {num_documents}") - print(f"Number of queries{language_description} is {num_queries}") - print( - f"Average number of relevant documents per query{language_description} is {qrels_per_doc}" - ) - return { - "average_document_length": doc_len, - "average_query_length": query_len, - "num_documents": num_documents, - "num_queries": num_queries, - "average_relevant_docs_per_query": qrels_per_doc, - } - - -def calculate_length(queries, corpus): - queries_lens = [] - doc_lens = [] - for query in queries.values(): - queries_lens.append(len(query)) - - for doc in corpus.values(): - if isinstance(doc, Image.Image): - doc_lens.append(1.0) # for image append 1. Can perhaps be removed. - - doc_len = sum(doc_lens) / len(doc_lens) if doc_lens else 0 - query_len = sum(queries_lens) / len(queries_lens) if queries_lens else 0 - return query_len, doc_len diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index 273bc4640b..6924ba76f6 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -14,11 +14,9 @@ from .AbsTaskSTS import * from .AbsTaskSummarization import * from .Image.AbsTaskAny2AnyRetrieval import * -from .Image.AbsTaskI2TRetrieval import * from .Image.AbsTaskImageClassification import * from .Image.AbsTaskImageClustering import * from .Image.AbsTaskImageMultilabelClassification import * from .Image.AbsTaskImageTextPairClassification import * -from .Image.AbsTaskT2IRetrieval import * from .Image.AbsTaskZeroshotClassification import * from .MultilingualTask import * diff --git a/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py deleted file mode 100644 index f8c655e883..0000000000 --- a/mteb/evaluation/evaluators/Image/I2TRetrievalEvaluator.py +++ /dev/null @@ -1,388 +0,0 @@ -from __future__ import annotations - -import heapq -import json -import logging -import os -from collections import defaultdict -from typing import Any, Dict, List, Tuple, Union - -import numpy as np -import pytrec_eval -import torch -from datasets import Dataset -from PIL import Image -from torch.utils.data import DataLoader -from torchvision import transforms - -from mteb.encoder_interface import EncoderWithQueryCorpusEncode - -from ..Evaluator import Evaluator -from ..utils import ( - confidence_scores, - cos_sim, - dot_score, - download, - hole, - mrr, - nAUC, - recall_cap, - top_k_accuracy, -) - -logger = logging.getLogger(__name__) - -transform = transforms.Compose([transforms.PILToTensor()]) - - -class ImageDataset(torch.utils.data.Dataset): - def __init__(self, hf_dataset, image_column_name: str = "image", transform=None): - self.dataset = hf_dataset - self.transform = transform - self.image_column_name = image_column_name - - def __len__(self): - return len(self.dataset) - - def __getitem__(self, idx): - image = self.dataset[idx][self.image_column_name] - if image.mode != "RGB": - image = image.convert("RGB") - image = self.transform(image) - return image - - -def custom_collate_fn(batch): - return batch - - -# Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/search/dense/exact_search.py#L12 -class DenseRetrievalExactSearch: - def __init__( - self, - model: EncoderWithQueryCorpusEncode, - encode_kwargs: dict[str, Any] = {}, - corpus_chunk_size: int = 50000, - previous_results: str | None = None, - **kwargs: Any, - ): - # Model is class that provides get_text_embeddings() and get_image_embeddings() - self.model = model - self.encode_kwargs = encode_kwargs - - if "batch_size" not in encode_kwargs: - encode_kwargs["batch_size"] = 128 - - self.score_functions = {"cos_sim": cos_sim, "dot": dot_score} - self.score_function_desc = { - "cos_sim": "Cosine Similarity", - "dot": "Dot Product", - } - self.corpus_chunk_size = corpus_chunk_size - self.previous_results = previous_results - self.batch_size = encode_kwargs.get("batch_size") - self.show_progress_bar = encode_kwargs.get("show_progress_bar") - self.save_corpus_embeddings = kwargs.get("save_corpus_embeddings", False) - self.corpus_embeddings = defaultdict(list) - self.results = {} - - if self.previous_results is not None: - self.previous_results = self.load_results_file() - - def search( - self, - corpus: Dataset, - queries: Dataset, - top_k: int, - score_function: str, - return_sorted: bool = False, - **kwargs, - ) -> dict[str, dict[str, float]]: - if score_function not in self.score_functions: - raise ValueError( - f"score function: {score_function} must be either (cos_sim) for cosine similarity or (dot) for dot product" - ) - - logger.info("Encoding Queries.") - query_ids = list(queries["id"]) - self.results = {qid: {} for qid in query_ids} - queries_dataset = ImageDataset( - queries, image_column_name="image", transform=transform - ) - query_image_dataloader = DataLoader( - queries_dataset, - batch_size=self.encode_kwargs["batch_size"], - shuffle=False, - collate_fn=custom_collate_fn, - num_workers=os.cpu_count(), - ) - query_embeddings = self.model.get_image_embeddings( - images=query_image_dataloader, batch_size=self.encode_kwargs["batch_size"] - ) - - logger.info("Sorting Corpus by document length (Longest first)...") - corpus_ids = list(corpus["id"]) - - logger.info("Encoding Corpus in batches... Warning: This might take a while!") - logger.info( - "Scoring Function: {} ({})".format( - self.score_function_desc[score_function], score_function - ) - ) - - corpus_texts = corpus["text"] - corpus_embeddings = self.model.get_text_embeddings( - texts=corpus_texts, batch_size=self.encode_kwargs["batch_size"] - ) - - result_heaps = { - qid: [] for qid in query_ids - } # Keep only the top-k docs for each query - - cos_scores = self.score_functions[score_function]( - query_embeddings, corpus_embeddings - ) - cos_scores[torch.isnan(cos_scores)] = -1 - - cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( - cos_scores, - top_k, - dim=1, - largest=True, - sorted=return_sorted, - ) - cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() - cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() - - for query_itr in range(len(query_embeddings)): - query_id = query_ids[query_itr] - for sub_corpus_id, score in zip( - cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] - ): - corpus_id = corpus_ids[sub_corpus_id] - if len(result_heaps[query_id]) < top_k: - heapq.heappush(result_heaps[query_id], (score, corpus_id)) - else: - heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) - - for qid in result_heaps: - for score, corpus_id in result_heaps[qid]: - self.results[qid][corpus_id] = score - - return self.results - - def load_results_file(self): - # load the first stage results from file in format {qid: {doc_id: score}} - if "https://" in self.previous_results: - # download the file - if not os.path.exists(self.previous_results): - url_descriptor = self.previous_results.split("https://")[-1].replace( - "/", "--" - ) - dest_file = os.path.join( - "results", f"cached_predictions--{url_descriptor}" - ) - os.makedirs(os.path.dirname(os.path.abspath(dest_file)), exist_ok=True) - download(self.previous_results, dest_file) - logger.info( - f"Downloaded the previous results at {self.previous_results} to {dest_file}" - ) - self.previous_results = dest_file - - with open(self.previous_results, "r") as f: - previous_results = json.load(f) - assert isinstance(previous_results, dict) - assert isinstance(previous_results[list(previous_results.keys())[0]], dict) - return previous_results - - -# Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/evaluation.py#L9 -class I2TRetrievalEvaluator(Evaluator): - def __init__( - self, - retriever=None, - task_name: str | None = None, - k_values: List[int] = [1, 3, 5, 10, 20, 100, 1000], - score_function: str = "cos_sim", - encode_kwargs: dict[str, Any] = {}, - **kwargs, - ): - super().__init__(**kwargs) - - self.retriever = DenseRetrievalExactSearch( - retriever, encode_kwargs=encode_kwargs, **kwargs - ) - self.k_values = k_values - self.top_k = ( - max(k_values) if "top_k" not in kwargs else kwargs["top_k"] - ) # can lower it if reranking - self.score_function = score_function - self.task_name = task_name - - def __call__( - self, - corpus: dict[str, Image.Image], - queries: dict[str, Union[str, List[str]]], - ) -> dict[str, dict[str, float]]: - if not self.retriever: - raise ValueError("Model/Technique has not been provided!") - - return self.retriever.search( - corpus, - queries, - self.top_k, - self.score_function, - prompt_name=self.task_name, # type: ignore - ) - - @staticmethod - def evaluate( - qrels: dict[str, dict[str, int]], - results: dict[str, dict[str, float]], - k_values: List[int], - ignore_identical_ids: bool = False, - ) -> Tuple[ - dict[str, float], - dict[str, float], - dict[str, float], - dict[str, float], - dict[str, float], - ]: - if ignore_identical_ids: - logger.debug( - "For evaluation, ``ignore_identical_ids=True`` is set to True, the evaluator will ignore identical query and document ids." - ) - # Remove identical ids from results dict - for qid, rels in results.items(): - for pid in list(rels): - if qid == pid: - results[qid].pop(pid) - else: - logger.debug( - "For evaluation, we DO NOT ignore identical query and document ids (default), please explicitly set ``ignore_identical_ids=True`` to ignore this." - ) - - all_ndcgs, all_aps, all_recalls, all_precisions, all_cv_recalls = ( - {}, - {}, - {}, - {}, - {}, - ) - - for k in k_values: - all_ndcgs[f"NDCG@{k}"] = [] - all_aps[f"MAP@{k}"] = [] - all_recalls[f"Recall@{k}"] = [] - all_precisions[f"P@{k}"] = [] - all_cv_recalls[f"CV_Recall@{k}"] = [] # (new) CV-style Recall - - map_string = "map_cut." + ",".join([str(k) for k in k_values]) - ndcg_string = "ndcg_cut." + ",".join([str(k) for k in k_values]) - recall_string = "recall." + ",".join([str(k) for k in k_values]) - precision_string = "P." + ",".join([str(k) for k in k_values]) - evaluator = pytrec_eval.RelevanceEvaluator( - qrels, {map_string, ndcg_string, recall_string, precision_string} - ) - scores = evaluator.evaluate(results) - - sorted_results = { - qid: sorted(rels.items(), key=lambda item: item[1], reverse=True) - for qid, rels in results.items() - } - - for query_id in scores.keys(): - top_docs = [ - doc_id for doc_id, _ in sorted_results.get(query_id, []) - ] # Sorted list of doc IDs - relevant_docs = set(qrels.get(query_id, {}).keys()) - - for k in k_values: - top_k_docs = top_docs[:k] - all_ndcgs[f"NDCG@{k}"].append(scores[query_id]["ndcg_cut_" + str(k)]) - all_aps[f"MAP@{k}"].append(scores[query_id]["map_cut_" + str(k)]) - all_recalls[f"Recall@{k}"].append(scores[query_id]["recall_" + str(k)]) - all_precisions[f"P@{k}"].append(scores[query_id]["P_" + str(k)]) - - if relevant_docs.intersection(top_k_docs): - all_cv_recalls[f"CV_Recall@{k}"].append(1.0) - else: - all_cv_recalls[f"CV_Recall@{k}"].append(0.0) - - ndcg, _map, recall, precision, cv_recall = ( - all_ndcgs.copy(), - all_aps.copy(), - all_recalls.copy(), - all_precisions.copy(), - all_cv_recalls.copy(), - ) - - for k in k_values: - ndcg[f"NDCG@{k}"] = round(sum(ndcg[f"NDCG@{k}"]) / len(scores), 5) - _map[f"MAP@{k}"] = round(sum(_map[f"MAP@{k}"]) / len(scores), 5) - recall[f"Recall@{k}"] = round(sum(recall[f"Recall@{k}"]) / len(scores), 5) - precision[f"P@{k}"] = round(sum(precision[f"P@{k}"]) / len(scores), 5) - cv_recall[f"CV_Recall@{k}"] = round( - sum(cv_recall[f"CV_Recall@{k}"]) / len(scores), 5 - ) - - naucs = I2TRetrievalEvaluator.evaluate_abstention( - results, - {**all_ndcgs, **all_aps, **all_recalls, **all_precisions, **all_cv_recalls}, - ) - - return ndcg, _map, recall, precision, cv_recall, naucs - - @staticmethod - def evaluate_custom( - qrels: dict[str, dict[str, int]], - results: dict[str, dict[str, float]], - k_values: List[int], - metric: str, - output_type: str = "all", - ) -> Tuple[Dict[str, float]]: - if metric.lower() in ["mrr", "mrr@k", "mrr_cut"]: - metric_scores = mrr(qrels, results, k_values, output_type) - - elif metric.lower() in ["recall_cap", "r_cap", "r_cap@k"]: - metric_scores = recall_cap(qrels, results, k_values, output_type) - - elif metric.lower() in ["hole", "hole@k"]: - metric_scores = hole(qrels, results, k_values, output_type) - - elif metric.lower() in [ - "acc", - "top_k_acc", - "accuracy", - "accuracy@k", - "top_k_accuracy", - ]: - metric_scores = top_k_accuracy(qrels, results, k_values, output_type) - - naucs = I2TRetrievalEvaluator.evaluate_abstention(results, metric_scores) - metric_scores_avg = {k: sum(v) / len(v) for k, v in metric_scores.items()} - - return metric_scores_avg, naucs - - @staticmethod - def evaluate_abstention( - results: dict[str, dict[str, float]], - metric_scores: dict[str, list[float]], - ) -> Dict[str, float]: - """Computes normalized Area Under the Curve on a set of evaluated instances as presented in the paper https://arxiv.org/abs/2402.12997""" - all_sim_scores = [list(results[qid].values()) for qid in list(results.keys())] - all_conf_scores = [ - confidence_scores(sim_scores) for sim_scores in all_sim_scores - ] - conf_fcts = list(all_conf_scores[0].keys()) - all_conf_scores = { - fct: np.array([x[fct] for x in all_conf_scores]) for fct in conf_fcts - } - metric_scores = {k: np.array(v) for k, v in metric_scores.items()} - naucs = {} - - for metric_name, scores in metric_scores.items(): - for fct, conf_scores in all_conf_scores.items(): - naucs[f"nAUC_{metric_name}_{fct}"] = nAUC(conf_scores, scores) - - return naucs diff --git a/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py deleted file mode 100644 index 85dfd5d753..0000000000 --- a/mteb/evaluation/evaluators/Image/T2IRetrievalEvaluator.py +++ /dev/null @@ -1,388 +0,0 @@ -from __future__ import annotations - -import heapq -import json -import logging -import os -from collections import defaultdict -from typing import Any, Dict, List, Tuple - -import numpy as np -import pytrec_eval -import torch -from datasets import Dataset -from PIL import Image -from torch.utils.data import DataLoader -from torchvision import transforms - -from mteb.encoder_interface import EncoderWithQueryCorpusEncode - -from ..Evaluator import Evaluator -from ..utils import ( - confidence_scores, - cos_sim, - dot_score, - download, - hole, - mrr, - nAUC, - recall_cap, - top_k_accuracy, -) - -logger = logging.getLogger(__name__) - -transform = transforms.Compose([transforms.PILToTensor()]) - - -class ImageDataset(torch.utils.data.Dataset): - def __init__(self, hf_dataset, image_column_name: str = "image", transform=None): - self.dataset = hf_dataset - self.transform = transform - self.image_column_name = image_column_name - - def __len__(self): - return len(self.dataset) - - def __getitem__(self, idx): - image = self.dataset[idx][self.image_column_name] - if image.mode != "RGB": - image = image.convert("RGB") - image = self.transform(image) - return image - - -def custom_collate_fn(batch): - return batch - - -# Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/search/dense/exact_search.py#L12 -class DenseRetrievalExactSearch: - def __init__( - self, - model: EncoderWithQueryCorpusEncode, - encode_kwargs: dict[str, Any] = {}, - corpus_chunk_size: int = 50000, - previous_results: str | None = None, - **kwargs: Any, - ): - # Model is class that provides get_text_embeddings() and get_image_embeddings() - self.model = model - self.encode_kwargs = encode_kwargs - - if "batch_size" not in encode_kwargs: - encode_kwargs["batch_size"] = 128 - - self.score_functions = {"cos_sim": cos_sim, "dot": dot_score} - self.score_function_desc = { - "cos_sim": "Cosine Similarity", - "dot": "Dot Product", - } - self.corpus_chunk_size = corpus_chunk_size - self.previous_results = previous_results - self.batch_size = encode_kwargs.get("batch_size") - self.show_progress_bar = encode_kwargs.get("show_progress_bar") - self.save_corpus_embeddings = kwargs.get("save_corpus_embeddings", False) - self.corpus_embeddings = defaultdict(list) - self.results = {} - - if self.previous_results is not None: - self.previous_results = self.load_results_file() - - def search( - self, - corpus: Dataset, - queries: Dataset, - top_k: int, - score_function: str, - return_sorted: bool = False, - **kwargs, - ) -> dict[str, dict[str, float]]: - if score_function not in self.score_functions: - raise ValueError( - f"score function: {score_function} must be either (cos_sim) for cosine similarity or (dot) for dot product" - ) - - logger.info("Encoding Queries.") - query_ids = list(queries["id"]) - self.results = {qid: {} for qid in query_ids} - queries = queries["text"] - query_embeddings = self.model.get_text_embeddings( - queries, batch_size=self.encode_kwargs["batch_size"] - ) - - logger.info("Preparing Corpus...") - corpus_ids = list(corpus["id"]) - corpus_dataset = ImageDataset( - corpus, image_column_name="image", transform=transform - ) - corpus_image_dataloader = DataLoader( - corpus_dataset, - batch_size=self.encode_kwargs["batch_size"], - shuffle=False, - collate_fn=custom_collate_fn, - num_workers=os.cpu_count(), - ) - - logger.info("Encoding Corpus in batches... Warning: This might take a while!") - logger.info( - "Scoring Function: {} ({})".format( - self.score_function_desc[score_function], score_function - ) - ) - - corpus_embeddings = self.model.get_image_embeddings( - images=corpus_image_dataloader, batch_size=self.encode_kwargs["batch_size"] - ) - - result_heaps = { - qid: [] for qid in query_ids - } # Keep only the top-k docs for each query - - cos_scores = self.score_functions[score_function]( - query_embeddings, corpus_embeddings - ) - cos_scores[torch.isnan(cos_scores)] = -1 - - cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( - cos_scores, - top_k, - dim=1, - largest=True, - sorted=return_sorted, - ) - cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() - cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() - - for query_itr in range(len(query_embeddings)): - query_id = query_ids[query_itr] - for sub_corpus_id, score in zip( - cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] - ): - corpus_id = corpus_ids[sub_corpus_id] - if len(result_heaps[query_id]) < top_k: - heapq.heappush(result_heaps[query_id], (score, corpus_id)) - else: - heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) - - for qid in result_heaps: - for score, corpus_id in result_heaps[qid]: - self.results[qid][corpus_id] = score - - return self.results - - def load_results_file(self): - # load the first stage results from file in format {qid: {doc_id: score}} - if "https://" in self.previous_results: - # download the file - if not os.path.exists(self.previous_results): - url_descriptor = self.previous_results.split("https://")[-1].replace( - "/", "--" - ) - dest_file = os.path.join( - "results", f"cached_predictions--{url_descriptor}" - ) - os.makedirs(os.path.dirname(os.path.abspath(dest_file)), exist_ok=True) - download(self.previous_results, dest_file) - logger.info( - f"Downloaded the previous results at {self.previous_results} to {dest_file}" - ) - self.previous_results = dest_file - - with open(self.previous_results, "r") as f: - previous_results = json.load(f) - assert isinstance(previous_results, dict) - assert isinstance(previous_results[list(previous_results.keys())[0]], dict) - return previous_results - - -# Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/evaluation.py#L9 -class T2IRetrievalEvaluator(Evaluator): - def __init__( - self, - retriever=None, - task_name: str | None = None, - k_values: List[int] = [1, 3, 5, 10, 20, 100, 1000], - score_function: str = "cos_sim", - encode_kwargs: dict[str, Any] = {}, - **kwargs, - ): - super().__init__(**kwargs) - - self.retriever = DenseRetrievalExactSearch( - retriever, encode_kwargs=encode_kwargs, **kwargs - ) - self.k_values = k_values - self.top_k = ( - max(k_values) if "top_k" not in kwargs else kwargs["top_k"] - ) # can lower it if reranking - self.score_function = score_function - self.task_name = task_name - - def __call__( - self, - corpus: dict[str, dict[str, str]], - queries: dict[str, Image.Image], - ) -> dict[str, dict[str, float]]: - if not self.retriever: - raise ValueError("Model/Technique has not been provided!") - - return self.retriever.search( - corpus, - queries, - self.top_k, - self.score_function, - prompt_name=self.task_name, # type: ignore - ) - - @staticmethod - def evaluate( - qrels: dict[str, dict[str, int]], - results: dict[str, dict[str, float]], - k_values: List[int], - ignore_identical_ids: bool = False, - ) -> Tuple[ - dict[str, float], - dict[str, float], - dict[str, float], - dict[str, float], - dict[str, float], - ]: - if ignore_identical_ids: - logger.debug( - "For evaluation, ``ignore_identical_ids=True`` is set to True, the evaluator will ignore identical query and document ids." - ) - # Remove identical ids from results dict - for qid, rels in results.items(): - for pid in list(rels): - if qid == pid: - results[qid].pop(pid) - else: - logger.debug( - "For evaluation, we DO NOT ignore identical query and document ids (default), please explicitly set ``ignore_identical_ids=True`` to ignore this." - ) - - all_ndcgs, all_aps, all_recalls, all_precisions, all_cv_recalls = ( - {}, - {}, - {}, - {}, - {}, - ) - - for k in k_values: - all_ndcgs[f"NDCG@{k}"] = [] - all_aps[f"MAP@{k}"] = [] - all_recalls[f"Recall@{k}"] = [] - all_precisions[f"P@{k}"] = [] - all_cv_recalls[f"CV_Recall@{k}"] = [] # (new) CV-style Recall - - map_string = "map_cut." + ",".join([str(k) for k in k_values]) - ndcg_string = "ndcg_cut." + ",".join([str(k) for k in k_values]) - recall_string = "recall." + ",".join([str(k) for k in k_values]) - precision_string = "P." + ",".join([str(k) for k in k_values]) - evaluator = pytrec_eval.RelevanceEvaluator( - qrels, {map_string, ndcg_string, recall_string, precision_string} - ) - scores = evaluator.evaluate(results) - - sorted_results = { - qid: sorted(rels.items(), key=lambda item: item[1], reverse=True) - for qid, rels in results.items() - } - - for query_id in scores.keys(): - top_docs = [ - doc_id for doc_id, _ in sorted_results.get(query_id, []) - ] # Sorted list of doc IDs - relevant_docs = set(qrels.get(query_id, {}).keys()) - - for k in k_values: - top_k_docs = top_docs[:k] - all_ndcgs[f"NDCG@{k}"].append(scores[query_id]["ndcg_cut_" + str(k)]) - all_aps[f"MAP@{k}"].append(scores[query_id]["map_cut_" + str(k)]) - all_recalls[f"Recall@{k}"].append(scores[query_id]["recall_" + str(k)]) - all_precisions[f"P@{k}"].append(scores[query_id]["P_" + str(k)]) - - if relevant_docs.intersection(top_k_docs): - all_cv_recalls[f"CV_Recall@{k}"].append(1.0) - else: - all_cv_recalls[f"CV_Recall@{k}"].append(0.0) - - ndcg, _map, recall, precision, cv_recall = ( - all_ndcgs.copy(), - all_aps.copy(), - all_recalls.copy(), - all_precisions.copy(), - all_cv_recalls.copy(), - ) - - for k in k_values: - ndcg[f"NDCG@{k}"] = round(sum(ndcg[f"NDCG@{k}"]) / len(scores), 5) - _map[f"MAP@{k}"] = round(sum(_map[f"MAP@{k}"]) / len(scores), 5) - recall[f"Recall@{k}"] = round(sum(recall[f"Recall@{k}"]) / len(scores), 5) - precision[f"P@{k}"] = round(sum(precision[f"P@{k}"]) / len(scores), 5) - cv_recall[f"CV_Recall@{k}"] = round( - sum(cv_recall[f"CV_Recall@{k}"]) / len(scores), 5 - ) - - naucs = T2IRetrievalEvaluator.evaluate_abstention( - results, - {**all_ndcgs, **all_aps, **all_recalls, **all_precisions, **all_cv_recalls}, - ) - - return ndcg, _map, recall, precision, cv_recall, naucs - - @staticmethod - def evaluate_custom( - qrels: dict[str, dict[str, int]], - results: dict[str, dict[str, float]], - k_values: List[int], - metric: str, - output_type: str = "all", - ) -> Tuple[Dict[str, float]]: - if metric.lower() in ["mrr", "mrr@k", "mrr_cut"]: - metric_scores = mrr(qrels, results, k_values, output_type) - - elif metric.lower() in ["recall_cap", "r_cap", "r_cap@k"]: - metric_scores = recall_cap(qrels, results, k_values, output_type) - - elif metric.lower() in ["hole", "hole@k"]: - metric_scores = hole(qrels, results, k_values, output_type) - - elif metric.lower() in [ - "acc", - "top_k_acc", - "accuracy", - "accuracy@k", - "top_k_accuracy", - ]: - metric_scores = top_k_accuracy(qrels, results, k_values, output_type) - - naucs = T2IRetrievalEvaluator.evaluate_abstention(results, metric_scores) - metric_scores_avg = {k: sum(v) / len(v) for k, v in metric_scores.items()} - - return metric_scores_avg, naucs - - @staticmethod - def evaluate_abstention( - results: dict[str, dict[str, float]], - metric_scores: dict[str, list[float]], - ) -> Dict[str, float]: - """Computes normalized Area Under the Curve on a set of evaluated instances as presented in the paper https://arxiv.org/abs/2402.12997""" - all_sim_scores = [list(results[qid].values()) for qid in list(results.keys())] - all_conf_scores = [ - confidence_scores(sim_scores) for sim_scores in all_sim_scores - ] - conf_fcts = list(all_conf_scores[0].keys()) - all_conf_scores = { - fct: np.array([x[fct] for x in all_conf_scores]) for fct in conf_fcts - } - metric_scores = {k: np.array(v) for k, v in metric_scores.items()} - naucs = {} - - for metric_name, scores in metric_scores.items(): - for fct, conf_scores in all_conf_scores.items(): - naucs[f"nAUC_{metric_name}_{fct}"] = nAUC(conf_scores, scores) - - return naucs diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py index 8056c6e495..111e899e42 100644 --- a/mteb/evaluation/evaluators/__init__.py +++ b/mteb/evaluation/evaluators/__init__.py @@ -6,9 +6,7 @@ from .Image.Any2AnyRetrievalEvaluator import * from .Image.ClassificationEvaluator import * from .Image.ClusteringEvaluator import * -from .Image.I2TRetrievalEvaluator import * from .Image.ImageTextPairClassificationEvaluator import * -from .Image.T2IRetrievalEvaluator import * from .Image.ZeroshotClassificationEvaluator import * from .PairClassificationEvaluator import * from .RerankingEvaluator import * From f10299c3674db957c3f77a76617f297d8d7c21d6 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Tue, 13 Aug 2024 13:18:28 +0000 Subject: [PATCH 048/154] remove references from test --- tests/test_tasks/test_all_abstasks.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_tasks/test_all_abstasks.py b/tests/test_tasks/test_all_abstasks.py index f691600511..c9f1f59ac6 100644 --- a/tests/test_tasks/test_all_abstasks.py +++ b/tests/test_tasks/test_all_abstasks.py @@ -14,8 +14,6 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.AbsTaskSpeedTask import AbsTaskSpeedTask from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval -from mteb.abstasks.Image.AbsTaskI2TRetrieval import AbsTaskI2TRetrieval -from mteb.abstasks.Image.AbsTaskT2IRetrieval import AbsTaskT2IRetrieval from mteb.abstasks.MultiSubsetLoader import MultiSubsetLoader from mteb.overview import TASKS_REGISTRY @@ -37,8 +35,6 @@ def test_load_data( # TODO: We skip because this load_data is completely different. if ( isinstance(task, AbsTaskRetrieval) - or isinstance(task, AbsTaskI2TRetrieval) - or isinstance(task, AbsTaskT2IRetrieval) or isinstance(task, AbsTaskAny2AnyRetrieval) or isinstance(task, AbsTaskInstructionRetrieval) or isinstance(task, MultiSubsetLoader) From f78e61aad105c4a1e7cbce7ad48f70c7c7d1ebaa Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 16 Aug 2024 23:35:36 +0100 Subject: [PATCH 049/154] tu-add berlin sketch retrieval --- .../Image/Any2AnyRetrievalEvaluator.py | 4 +- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 1 + .../eng/TUBerlinT2IRetrieval.py | 55 ++++++ mteb/tasks/__init__.py | 3 + .../TUBerlinT2IRetrieval.json | 186 ++++++++++++++++++ 5 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/TUBerlinT2IRetrieval.json diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index 928da19d45..ad72dc57b7 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -123,7 +123,7 @@ def search( batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=os.cpu_count(), + num_workers=max(1, os.cpu_count() // 2), ) if q_modality == "image": query_embeddings = self.model.get_image_embeddings( @@ -175,7 +175,7 @@ def search( batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=os.cpu_count(), + num_workers=max(1, os.cpu_count() // 2), ) if corpus_modality == "image": sub_corpus_embeddings = self.model.get_image_embeddings( diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 9618f02342..170a5acc8e 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -13,6 +13,7 @@ from .eng.OVENIT2ITRetrieval import * from .eng.OVENIT2TRetrieval import * from .eng.StanfordCarsI2IRetrieval import * +from .eng.TUBerlinT2IRetrieval import * from .eng.VisualNewsI2TRetrieval import * from .eng.VisualNewsT2IRetrieval import * from .eng.WebQAT2ITRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py new file mode 100644 index 0000000000..4d39d12a09 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +class TUBerlinT2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="TUBerlinT2IRetrieval", + description="Retrieve sketch images based on text descriptions.", + reference="https://dl.acm.org/doi/pdf/10.1145/2185520.2185540?casa_token=tq-eUx5UROYAAAAA:_694nPzE7tali6LCkxQc0M-mlo9xslasPMcVnFPMy9tDfvt7lg7p1RTe-k8VWCjuv9gmkQqasKUZ", + dataset={ + "path": "gowitheflow/tu-berlin", + "revision": "dcd8328b8b27cd39ed6c066862532a5dcd35f012", + "trust_remote_code": True, + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2012-01-01", "2012-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{eitz2012humans, + title={How do humans sketch objects?}, + author={Eitz, Mathias and Hays, James and Alexa, Marc}, + journal={ACM Transactions on graphics (TOG)}, + volume={31}, + number={4}, + pages={1--10}, + year={2012}, + publisher={Acm New York, NY, USA} +}""", + descriptive_stats={ + "n_samples": {"test": 250}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 20000, + "num_queries": 250, + "average_relevant_docs_per_query": 80.0, + } + }, + }, + ) diff --git a/mteb/tasks/__init__.py b/mteb/tasks/__init__.py index 1505b15bc2..608b849a70 100644 --- a/mteb/tasks/__init__.py +++ b/mteb/tasks/__init__.py @@ -3,7 +3,10 @@ from .BitextMining import * from .Classification import * from .Clustering import * +from .Image.Any2AnyRetrieval import * +from .Image.Clustering import * from .Image.ImageClassification import * +from .Image.ImageMultilabelClassification import * from .Image.ImageTextPairClassification import * from .Image.ZeroshotClassification import * from .InstructionRetrieval import * diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/TUBerlinT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/TUBerlinT2IRetrieval.json new file mode 100644 index 0000000000..d1daf9a0aa --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/TUBerlinT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "dcd8328b8b27cd39ed6c066862532a5dcd35f012", + "evaluation_time": 175.85956716537476, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.856, + "cv_recall_at_10": 0.992, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.996, + "cv_recall_at_3": 0.964, + "cv_recall_at_5": 0.976, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.81945, + "map_at_1": 0.0107, + "map_at_10": 0.09306, + "map_at_100": 0.46335, + "map_at_1000": 0.54192, + "map_at_20": 0.17108, + "map_at_3": 0.03029, + "map_at_5": 0.04878, + "mrr_at_1": 0.856, + "mrr_at_10": 0.908388888888889, + "mrr_at_100": 0.9086424259573667, + "mrr_at_1000": 0.9086424259573667, + "mrr_at_20": 0.9085994152046785, + "mrr_at_3": 0.9033333333333334, + "mrr_at_5": 0.9063333333333334, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": 0.5541549953314484, + "nauc_cv_recall_at_10_max": 1.0, + "nauc_cv_recall_at_10_std": 1.0, + "nauc_cv_recall_at_1_diff1": 0.13113248325048668, + "nauc_cv_recall_at_1_max": 0.43477145018370433, + "nauc_cv_recall_at_1_std": 0.23836989409984835, + "nauc_cv_recall_at_20_diff1": 0.5541549953314557, + "nauc_cv_recall_at_20_max": 1.0, + "nauc_cv_recall_at_20_std": 1.0, + "nauc_cv_recall_at_3_diff1": 0.04160182591555137, + "nauc_cv_recall_at_3_max": 0.4733893557422951, + "nauc_cv_recall_at_3_std": 0.4679427326486138, + "nauc_cv_recall_at_5_diff1": 0.09337068160597826, + "nauc_cv_recall_at_5_max": 0.3998599439775951, + "nauc_cv_recall_at_5_std": 0.39713663243075265, + "nauc_map_at_1000_diff1": -0.1659607050128913, + "nauc_map_at_1000_max": 0.4002951420709613, + "nauc_map_at_1000_std": 0.6008649703703033, + "nauc_map_at_100_diff1": -0.159320455241414, + "nauc_map_at_100_max": 0.37907101423707096, + "nauc_map_at_100_std": 0.5501993811365806, + "nauc_map_at_10_diff1": -0.03247236891366951, + "nauc_map_at_10_max": 0.39401250753640266, + "nauc_map_at_10_std": 0.2912329253162918, + "nauc_map_at_1_diff1": 0.13113248325048504, + "nauc_map_at_1_max": 0.43477145018370394, + "nauc_map_at_1_std": 0.23836989409984927, + "nauc_map_at_20_diff1": -0.07440129133273242, + "nauc_map_at_20_max": 0.3846356021740391, + "nauc_map_at_20_std": 0.34792679433137946, + "nauc_map_at_3_diff1": 0.08322035015793686, + "nauc_map_at_3_max": 0.423486714736962, + "nauc_map_at_3_std": 0.24833821828229646, + "nauc_map_at_5_diff1": 0.00919846951370846, + "nauc_map_at_5_max": 0.41473543872025803, + "nauc_map_at_5_std": 0.25080776636776136, + "nauc_mrr_at_1000_diff1": 0.11207727768308348, + "nauc_mrr_at_1000_max": 0.4261596090309955, + "nauc_mrr_at_1000_std": 0.24872225471043358, + "nauc_mrr_at_100_diff1": 0.11207727768308348, + "nauc_mrr_at_100_max": 0.4261596090309955, + "nauc_mrr_at_100_std": 0.24872225471043358, + "nauc_mrr_at_10_diff1": 0.11334039502197703, + "nauc_mrr_at_10_max": 0.4277992025202272, + "nauc_mrr_at_10_std": 0.25086882737971083, + "nauc_mrr_at_1_diff1": 0.13113248325048668, + "nauc_mrr_at_1_max": 0.43477145018370433, + "nauc_mrr_at_1_std": 0.23836989409984835, + "nauc_mrr_at_20_diff1": 0.11229206610732925, + "nauc_mrr_at_20_max": 0.4264384158299444, + "nauc_mrr_at_20_std": 0.24908727144083842, + "nauc_mrr_at_3_diff1": 0.10013605442176822, + "nauc_mrr_at_3_max": 0.42878048780487743, + "nauc_mrr_at_3_std": 0.26161937945910047, + "nauc_mrr_at_5_diff1": 0.10549139387956116, + "nauc_mrr_at_5_max": 0.4224472165945086, + "nauc_mrr_at_5_std": 0.2501225528005465, + "nauc_ndcg_at_1000_diff1": -0.15473148465972003, + "nauc_ndcg_at_1000_max": 0.46638630840908135, + "nauc_ndcg_at_1000_std": 0.6706615415351543, + "nauc_ndcg_at_100_diff1": -0.15091091689134778, + "nauc_ndcg_at_100_max": 0.41430352035241735, + "nauc_ndcg_at_100_std": 0.5945250349662444, + "nauc_ndcg_at_10_diff1": -0.02231970983029379, + "nauc_ndcg_at_10_max": 0.4180194276810624, + "nauc_ndcg_at_10_std": 0.30351802916358955, + "nauc_ndcg_at_1_diff1": 0.13113248325048668, + "nauc_ndcg_at_1_max": 0.43477145018370433, + "nauc_ndcg_at_1_std": 0.23836989409984835, + "nauc_ndcg_at_20_diff1": -0.05816593196928681, + "nauc_ndcg_at_20_max": 0.4138040923822943, + "nauc_ndcg_at_20_std": 0.3652054221361988, + "nauc_ndcg_at_3_diff1": 0.08447472013550292, + "nauc_ndcg_at_3_max": 0.4259852943789907, + "nauc_ndcg_at_3_std": 0.25416061971191134, + "nauc_ndcg_at_5_diff1": 0.00766324120279822, + "nauc_ndcg_at_5_max": 0.4151433003158982, + "nauc_ndcg_at_5_std": 0.2495238554669031, + "nauc_precision_at_1000_diff1": -0.1809233410743444, + "nauc_precision_at_1000_max": 0.42887321721343796, + "nauc_precision_at_1000_std": 0.7621261043128252, + "nauc_precision_at_100_diff1": -0.17465674567829512, + "nauc_precision_at_100_max": 0.3924712011609377, + "nauc_precision_at_100_std": 0.642807625809328, + "nauc_precision_at_10_diff1": -0.053504093361584064, + "nauc_precision_at_10_max": 0.41072806234342696, + "nauc_precision_at_10_std": 0.32022830196389684, + "nauc_precision_at_1_diff1": 0.13113248325048668, + "nauc_precision_at_1_max": 0.43477145018370433, + "nauc_precision_at_1_std": 0.23836989409984835, + "nauc_precision_at_20_diff1": -0.08543376918703843, + "nauc_precision_at_20_max": 0.4026196702671966, + "nauc_precision_at_20_std": 0.3896424104604883, + "nauc_precision_at_3_diff1": 0.07327497513632078, + "nauc_precision_at_3_max": 0.426716451181454, + "nauc_precision_at_3_std": 0.26320775746767655, + "nauc_precision_at_5_diff1": -0.031485413419408415, + "nauc_precision_at_5_max": 0.40567508192295, + "nauc_precision_at_5_std": 0.25015525472821193, + "nauc_recall_at_1000_diff1": -0.18092334107434152, + "nauc_recall_at_1000_max": 0.4288732172134386, + "nauc_recall_at_1000_std": 0.7621261043128265, + "nauc_recall_at_100_diff1": -0.1746567456782966, + "nauc_recall_at_100_max": 0.3924712011609363, + "nauc_recall_at_100_std": 0.6428076258093276, + "nauc_recall_at_10_diff1": -0.053504093361584064, + "nauc_recall_at_10_max": 0.41072806234342696, + "nauc_recall_at_10_std": 0.32022830196389684, + "nauc_recall_at_1_diff1": 0.13113248325048504, + "nauc_recall_at_1_max": 0.43477145018370394, + "nauc_recall_at_1_std": 0.23836989409984927, + "nauc_recall_at_20_diff1": -0.08543376918703843, + "nauc_recall_at_20_max": 0.4026196702671966, + "nauc_recall_at_20_std": 0.3896424104604883, + "nauc_recall_at_3_diff1": 0.07327497513632164, + "nauc_recall_at_3_max": 0.42671645118145396, + "nauc_recall_at_3_std": 0.2632077574676772, + "nauc_recall_at_5_diff1": -0.031485413419408415, + "nauc_recall_at_5_max": 0.40567508192295, + "nauc_recall_at_5_std": 0.25015525472821193, + "ndcg_at_1": 0.856, + "ndcg_at_10": 0.81945, + "ndcg_at_100": 0.63488, + "ndcg_at_1000": 0.81053, + "ndcg_at_20": 0.7844, + "ndcg_at_3": 0.84208, + "ndcg_at_5": 0.83105, + "precision_at_1": 0.856, + "precision_at_10": 0.8108, + "precision_at_100": 0.4738, + "precision_at_1000": 0.07223, + "precision_at_20": 0.7644, + "precision_at_3": 0.83867, + "precision_at_5": 0.824, + "recall_at_1": 0.0107, + "recall_at_10": 0.10135, + "recall_at_100": 0.59225, + "recall_at_1000": 0.9029, + "recall_at_20": 0.1911, + "recall_at_3": 0.03145, + "recall_at_5": 0.0515 + } + ] + }, + "task_name": "TUBerlinT2IRetrieval" +} \ No newline at end of file From df047e0201b185caa0238b8a2d6da9d44beaeba1 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Sat, 17 Aug 2024 19:52:53 +0100 Subject: [PATCH 050/154] XM3600; XFlickr30kCO; mutilingual --- .../Image/Any2AnyRetrievalEvaluator.py | 6 + mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 2 + .../multilingual/XFlickr30kCoT2IRetrieval.py | 187 + .../multilingual/XM3600T2IRetrieval.py | 420 ++ .../Any2AnyRetrieval/multilingual/__init__.py | 0 .../XFlickr30kCoT2IRetrieval.json | 1411 ++++ .../XM3600T2IRetrieval.json | 6311 +++++++++++++++++ 7 files changed, 8337 insertions(+) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/multilingual/__init__.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XFlickr30kCoT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XM3600T2IRetrieval.json diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index ad72dc57b7..fa96483972 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -1,6 +1,7 @@ from __future__ import annotations import heapq +import io import json import logging import os @@ -46,6 +47,11 @@ def __len__(self): def __getitem__(self, idx): image = self.dataset[idx][self.image_column_name] + if isinstance(image, bytes): + image = Image.open(io.BytesIO(image)) + else: + # Assume the image is already in a usable format (e.g., PIL Image) + image = image if image.mode != "RGB": image = image.convert("RGB") image = self.transform(image) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 170a5acc8e..8f62573137 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -18,3 +18,5 @@ from .eng.VisualNewsT2IRetrieval import * from .eng.WebQAT2ITRetrieval import * from .eng.WebQAT2TRetrieval import * +from .multilingual.XFlickr30kCoT2IRetrieval import * +from .multilingual.XM3600T2IRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py new file mode 100644 index 0000000000..1ed73db4c2 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +from datasets import DatasetDict, load_dataset + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval, MultilingualTask + +_LANGUAGES = { + "de": ["deu-Latn"], + "en": ["eng-Latn"], + "es": ["spa-Latn"], + "id": ["ind-Latn"], + "ja": ["jpn-Jpan"], + "ru": ["rus-Cyrl"], + "tr": ["tur-Latn"], + "zh": ["zho-Hans"], +} + + +def _load_xflickrco_data( + path: str, langs: list, splits: str, cache_dir: str = None, revision: str = None +): + corpus = {lang: {split: None for split in splits} for lang in langs} + queries = {lang: {split: None for split in splits} for lang in langs} + relevant_docs = {lang: {split: None for split in splits} for lang in langs} + + split = "test" + + for lang in langs: + lang_data = load_dataset( + path, + cache_dir=cache_dir, + revision=revision, + # trust_remote_code=True, + )[lang] + lang_corpus = lang_data.map( + lambda x: { + "id": "corpus-" + x["id"], + "text": None, + "modality": "image", + "image": x["image"]["bytes"], + }, + remove_columns=["sentences"], + ) + + lang_queries = lang_data.map( + lambda x: { + "id": "query-" + x["id"], + "text": x["sentences"], + "modality": "text", + "image": None, + }, + remove_columns=["sentences"], + ) + + relevant_docs[lang][split] = {} + for row in lang_data: + query_id = "query-" + row["id"] + corpus_id = "corpus-" + row["id"] + score = 1 + if query_id not in relevant_docs[lang][split]: + relevant_docs[lang][split][query_id] = {} + relevant_docs[lang][split][query_id][corpus_id] = score + + corpus[lang][split] = lang_corpus + queries[lang][split] = lang_queries + + corpus = DatasetDict({lang: DatasetDict(splits) for lang, splits in corpus.items()}) + queries = DatasetDict( + {lang: DatasetDict(splits) for lang, splits in queries.items()} + ) + relevant_docs = DatasetDict(relevant_docs) + return corpus, queries, relevant_docs + + +class XFlickr30kCoT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="XFlickr30kCoT2IRetrieval", + description="Retrieve images based on multilingual descriptions.", + reference="https://proceedings.mlr.press/v162/bugliarello22a/bugliarello22a.pdf", + dataset={ + "path": "floschne/xflickrco", + "revision": "0af2c2eba58b27a71898787e286be04befdd7a20", + # "trust_remote_code": True, + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=_LANGUAGES, + main_score="ndcg_at_10", + date=("2022-01-01", "2022-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@inproceedings{bugliarello2022iglue, + title={IGLUE: A benchmark for transfer learning across modalities, tasks, and languages}, + author={Bugliarello, Emanuele and Liu, Fangyu and Pfeiffer, Jonas and Reddy, Siva and Elliott, Desmond and Ponti, Edoardo Maria and Vuli{\'c}, Ivan}, + booktitle={International Conference on Machine Learning}, + pages={2370--2392}, + year={2022}, + organization={PMLR} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "dev": { + "de": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 2000, + "num_queries": 2000, + "average_relevant_docs_per_query": 1.0, + }, + "en": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 2000, + "num_queries": 2000, + "average_relevant_docs_per_query": 1.0, + }, + "es": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 2000, + "num_queries": 2000, + "average_relevant_docs_per_query": 1.0, + }, + "id": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 2000, + "num_queries": 2000, + "average_relevant_docs_per_query": 1.0, + }, + "ja": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 2000, + "num_queries": 2000, + "average_relevant_docs_per_query": 1.0, + }, + "ru": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 2000, + "num_queries": 2000, + "average_relevant_docs_per_query": 1.0, + }, + "tr": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 2000, + "num_queries": 2000, + "average_relevant_docs_per_query": 1.0, + }, + "zh": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 2000, + "num_queries": 2000, + "average_relevant_docs_per_query": 1.0, + }, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus, self.queries, self.relevant_docs = _load_xflickrco_data( + path=self.metadata_dict["dataset"]["path"], + langs=self.hf_subsets, + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py new file mode 100644 index 0000000000..350d91f3c7 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py @@ -0,0 +1,420 @@ +from __future__ import annotations + +from datasets import Dataset, DatasetDict, load_dataset + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval, MultilingualTask + +_LANGUAGES = { + "ar": ["ara-Arab"], + "bn": ["ben-Beng"], + "cs": ["ces-Latn"], + "da": ["dan-Latn"], + "de": ["deu-Latn"], + "el": ["ell-Grek"], + "en": ["eng-Latn"], + "es": ["spa-Latn"], + "fa": ["fas-Arab"], + "fi": ["fin-Latn"], + "fil": ["fil-Latn"], + "fr": ["fra-Latn"], + "he": ["heb-Hebr"], + "hi": ["hin-Deva"], + "hr": ["hrv-Latn"], + "hu": ["hun-Latn"], + "id": ["ind-Latn"], + "it": ["ita-Latn"], + "ja": ["jpn-Jpan"], + "ko": ["kor-Hang"], + "mi": ["mri-Latn"], + "nl": ["nld-Latn"], + "no": ["nor-Latn"], + "pl": ["pol-Latn"], + "pt": ["por-Latn"], + "quz": ["quz-Latn"], + "ro": ["ron-Latn"], + "ru": ["rus-Cyrl"], + "sv": ["swe-Latn"], + "sw": ["swa-Latn"], + "te": ["tel-Telu"], + "th": ["tha-Thai"], + "tr": ["tur-Latn"], + "uk": ["ukr-Cyrl"], + "vi": ["vie-Latn"], + "zh": ["zho-Hans"], +} + + +def _load_xm3600_data( + path: str, langs: list, splits: str, cache_dir: str = None, revision: str = None +): + corpus = {lang: {split: None for split in splits} for lang in langs} + queries = {lang: {split: None for split in splits} for lang in langs} + relevant_docs = {lang: {split: None for split in splits} for lang in langs} + + split = "test" + + for lang in langs: + lang_data = load_dataset( + path, + split=lang, + cache_dir=cache_dir, + revision=revision, + # trust_remote_code=True, + ) + lang_corpus = lang_data.map( + lambda x: { + "id": "corpus-" + x["image_id"], + "text": None, + "modality": "image", + "image": x["image"]["bytes"], + }, + remove_columns=[ + "captions", + "captions_tokenized", + "captions_tokenized_lowercase", + "image_locale", + "image_id", + ], + ) + + corpus[lang][split] = lang_corpus + + lang_data = lang_data.remove_columns(["image"]) + + queries[lang][split] = [] + relevant_docs[lang][split] = {} + + for row in lang_data: + image_id = "corpus-" + row["image_id"] + for idx, caption in enumerate(row["captions"]): + query_id = f"query-{row['image_id']}-{idx}" + queries[lang][split].append( + { + "id": query_id, + "text": caption, + "modality": "text", + "image": None, + } + ) + if query_id not in relevant_docs[lang][split]: + relevant_docs[lang][split][query_id] = {} + relevant_docs[lang][split][query_id][image_id] = 1 + + queries[lang][split] = Dataset.from_dict( + { + "id": [query["id"] for query in queries[lang][split]], + "text": [query["text"] for query in queries[lang][split]], + "modality": [query["modality"] for query in queries[lang][split]], + "image": [None for _ in queries[lang][split]], + } + ) + corpus = DatasetDict({lang: DatasetDict(splits) for lang, splits in corpus.items()}) + queries = DatasetDict( + {lang: DatasetDict(splits) for lang, splits in queries.items()} + ) + relevant_docs = DatasetDict(relevant_docs) + + return corpus, queries, relevant_docs + + +class XM3600T2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="XM3600T2IRetrieval", + description="Retrieve images based on multilingual descriptions.", + reference="https://aclanthology.org/2022.emnlp-main.45/", + dataset={ + "path": "floschne/xm3600", + "revision": "8d3e5665526c55a5855cd6ddfbaba2032bc7cee4", + # "trust_remote_code": True, + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=_LANGUAGES, + main_score="ndcg_at_10", + date=("2022-01-01", "2022-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@inproceedings{thapliyal2022crossmodal, + title={Crossmodal-3600: A Massively Multilingual Multimodal Evaluation Dataset}, + author={Thapliyal, Ashish V and Tuset, Jordi Pont and Chen, Xi and Soricut, Radu}, + booktitle={Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing}, + pages={715--729}, + year={2022} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "dev": { + "ar": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "bn": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "cs": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "da": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "de": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "el": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "en": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "es": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "fa": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "fi": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "fil": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "fr": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "he": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "hi": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "hr": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "hu": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "id": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "it": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "ja": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "ko": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "mi": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "nl": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "pl": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "pt": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "quz": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "ro": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "ru": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "sv": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "sw": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "te": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "th": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "tr": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "uk": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "vi": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "zh": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus, self.queries, self.relevant_docs = _load_xm3600_data( + path=self.metadata_dict["dataset"]["path"], + langs=self.hf_subsets, + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XFlickr30kCoT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XFlickr30kCoT2IRetrieval.json new file mode 100644 index 0000000000..81671f5ed3 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XFlickr30kCoT2IRetrieval.json @@ -0,0 +1,1411 @@ +{ + "dataset_revision": "0af2c2eba58b27a71898787e286be04befdd7a20", + "evaluation_time": 99.99835991859436, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.0965, + "cv_recall_at_10": 0.3335, + "cv_recall_at_100": 0.645, + "cv_recall_at_1000": 0.935, + "cv_recall_at_20": 0.4295, + "cv_recall_at_3": 0.189, + "cv_recall_at_5": 0.2405, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.20111, + "map_at_1": 0.0965, + "map_at_10": 0.16057, + "map_at_100": 0.17239, + "map_at_1000": 0.1735, + "map_at_20": 0.1672, + "map_at_3": 0.13667, + "map_at_5": 0.14822, + "mrr_at_1": 0.0965, + "mrr_at_10": 0.1605702380952379, + "mrr_at_100": 0.1723897851951646, + "mrr_at_1000": 0.17350047917826844, + "mrr_at_20": 0.1672007281246442, + "mrr_at_3": 0.13666666666666671, + "mrr_at_5": 0.14821666666666652, + "nauc_cv_recall_at_1000_diff1": -0.13237807943690208, + "nauc_cv_recall_at_1000_max": 0.6101845866551727, + "nauc_cv_recall_at_1000_std": 0.7255440637793579, + "nauc_cv_recall_at_100_diff1": -0.08269381110117098, + "nauc_cv_recall_at_100_max": 0.5258267043851802, + "nauc_cv_recall_at_100_std": 0.6066611861988931, + "nauc_cv_recall_at_10_diff1": 0.05244456199507041, + "nauc_cv_recall_at_10_max": 0.40815755564401374, + "nauc_cv_recall_at_10_std": 0.3812155724158006, + "nauc_cv_recall_at_1_diff1": 0.23466483899175744, + "nauc_cv_recall_at_1_max": 0.38016379563220937, + "nauc_cv_recall_at_1_std": 0.23962792171953437, + "nauc_cv_recall_at_20_diff1": 0.02385501952755473, + "nauc_cv_recall_at_20_max": 0.4362241829602174, + "nauc_cv_recall_at_20_std": 0.4356448904252889, + "nauc_cv_recall_at_3_diff1": 0.12159462941294655, + "nauc_cv_recall_at_3_max": 0.3403158731847171, + "nauc_cv_recall_at_3_std": 0.26052530716928135, + "nauc_cv_recall_at_5_diff1": 0.09419131251931254, + "nauc_cv_recall_at_5_max": 0.3499742250107925, + "nauc_cv_recall_at_5_std": 0.312421300241589, + "nauc_map_at_1000_diff1": 0.1461081483396935, + "nauc_map_at_1000_max": 0.37703355044770975, + "nauc_map_at_1000_std": 0.29448647756773627, + "nauc_map_at_100_diff1": 0.14547433469922097, + "nauc_map_at_100_max": 0.3772992484744435, + "nauc_map_at_100_std": 0.29507724913928524, + "nauc_map_at_10_diff1": 0.1518516384994729, + "nauc_map_at_10_max": 0.37112018036702726, + "nauc_map_at_10_std": 0.28302886799037996, + "nauc_map_at_1_diff1": 0.23466483899175744, + "nauc_map_at_1_max": 0.38016379563220937, + "nauc_map_at_1_std": 0.23962792171953437, + "nauc_map_at_20_diff1": 0.14803098009106574, + "nauc_map_at_20_max": 0.37413032174506977, + "nauc_map_at_20_std": 0.2893847241629189, + "nauc_map_at_3_diff1": 0.1730919004523251, + "nauc_map_at_3_max": 0.35789019817695994, + "nauc_map_at_3_std": 0.2519022240220413, + "nauc_map_at_5_diff1": 0.16283563503461893, + "nauc_map_at_5_max": 0.3603487503123098, + "nauc_map_at_5_std": 0.26862429359407647, + "nauc_mrr_at_1000_diff1": 0.1461081483396935, + "nauc_mrr_at_1000_max": 0.37703355044770975, + "nauc_mrr_at_1000_std": 0.29448647756773627, + "nauc_mrr_at_100_diff1": 0.14547433469922097, + "nauc_mrr_at_100_max": 0.3772992484744435, + "nauc_mrr_at_100_std": 0.29507724913928524, + "nauc_mrr_at_10_diff1": 0.1518516384994729, + "nauc_mrr_at_10_max": 0.37112018036702726, + "nauc_mrr_at_10_std": 0.28302886799037996, + "nauc_mrr_at_1_diff1": 0.23466483899175744, + "nauc_mrr_at_1_max": 0.38016379563220937, + "nauc_mrr_at_1_std": 0.23962792171953437, + "nauc_mrr_at_20_diff1": 0.14803098009106574, + "nauc_mrr_at_20_max": 0.37413032174506977, + "nauc_mrr_at_20_std": 0.2893847241629189, + "nauc_mrr_at_3_diff1": 0.1730919004523251, + "nauc_mrr_at_3_max": 0.35789019817695994, + "nauc_mrr_at_3_std": 0.2519022240220413, + "nauc_mrr_at_5_diff1": 0.16283563503461893, + "nauc_mrr_at_5_max": 0.3603487503123098, + "nauc_mrr_at_5_std": 0.26862429359407647, + "nauc_ndcg_at_1000_diff1": 0.1086985272853948, + "nauc_ndcg_at_1000_max": 0.39926536348764186, + "nauc_ndcg_at_1000_std": 0.3436142168990749, + "nauc_ndcg_at_100_diff1": 0.0903687422592335, + "nauc_ndcg_at_100_max": 0.40946033268315823, + "nauc_ndcg_at_100_std": 0.36639710955282306, + "nauc_ndcg_at_10_diff1": 0.11977407519483287, + "nauc_ndcg_at_10_max": 0.3812462038075193, + "nauc_ndcg_at_10_std": 0.31303685757698146, + "nauc_ndcg_at_1_diff1": 0.23466483899175744, + "nauc_ndcg_at_1_max": 0.38016379563220937, + "nauc_ndcg_at_1_std": 0.23962792171953437, + "nauc_ndcg_at_20_diff1": 0.10880436162955412, + "nauc_ndcg_at_20_max": 0.3910719259043727, + "nauc_ndcg_at_20_std": 0.33271214860697695, + "nauc_ndcg_at_3_diff1": 0.15734556970000205, + "nauc_ndcg_at_3_max": 0.3524947490091569, + "nauc_ndcg_at_3_std": 0.2545727797832577, + "nauc_ndcg_at_5_diff1": 0.14116730424839466, + "nauc_ndcg_at_5_max": 0.35683485916594976, + "nauc_ndcg_at_5_std": 0.281957030568669, + "nauc_precision_at_1000_diff1": -0.1323780794369055, + "nauc_precision_at_1000_max": 0.6101845866551743, + "nauc_precision_at_1000_std": 0.725544063779355, + "nauc_precision_at_100_diff1": -0.08269381110117134, + "nauc_precision_at_100_max": 0.5258267043851799, + "nauc_precision_at_100_std": 0.6066611861988925, + "nauc_precision_at_10_diff1": 0.05244456199507014, + "nauc_precision_at_10_max": 0.4081575556440135, + "nauc_precision_at_10_std": 0.3812155724158001, + "nauc_precision_at_1_diff1": 0.23466483899175744, + "nauc_precision_at_1_max": 0.38016379563220937, + "nauc_precision_at_1_std": 0.23962792171953437, + "nauc_precision_at_20_diff1": 0.023855019527554727, + "nauc_precision_at_20_max": 0.4362241829602172, + "nauc_precision_at_20_std": 0.4356448904252892, + "nauc_precision_at_3_diff1": 0.12159462941294678, + "nauc_precision_at_3_max": 0.34031587318471695, + "nauc_precision_at_3_std": 0.26052530716928124, + "nauc_precision_at_5_diff1": 0.09419131251931243, + "nauc_precision_at_5_max": 0.34997422501079256, + "nauc_precision_at_5_std": 0.31242130024158893, + "nauc_recall_at_1000_diff1": -0.13237807943690208, + "nauc_recall_at_1000_max": 0.6101845866551727, + "nauc_recall_at_1000_std": 0.7255440637793579, + "nauc_recall_at_100_diff1": -0.08269381110117098, + "nauc_recall_at_100_max": 0.5258267043851802, + "nauc_recall_at_100_std": 0.6066611861988931, + "nauc_recall_at_10_diff1": 0.05244456199507041, + "nauc_recall_at_10_max": 0.40815755564401374, + "nauc_recall_at_10_std": 0.3812155724158006, + "nauc_recall_at_1_diff1": 0.23466483899175744, + "nauc_recall_at_1_max": 0.38016379563220937, + "nauc_recall_at_1_std": 0.23962792171953437, + "nauc_recall_at_20_diff1": 0.02385501952755473, + "nauc_recall_at_20_max": 0.4362241829602174, + "nauc_recall_at_20_std": 0.4356448904252889, + "nauc_recall_at_3_diff1": 0.12159462941294655, + "nauc_recall_at_3_max": 0.3403158731847171, + "nauc_recall_at_3_std": 0.26052530716928135, + "nauc_recall_at_5_diff1": 0.09419131251931254, + "nauc_recall_at_5_max": 0.3499742250107925, + "nauc_recall_at_5_std": 0.312421300241589, + "ndcg_at_1": 0.0965, + "ndcg_at_10": 0.20111, + "ndcg_at_100": 0.2646, + "ndcg_at_1000": 0.29981, + "ndcg_at_20": 0.22534, + "ndcg_at_3": 0.15008, + "ndcg_at_5": 0.1711, + "precision_at_1": 0.0965, + "precision_at_10": 0.03335, + "precision_at_100": 0.00645, + "precision_at_1000": 0.00093, + "precision_at_20": 0.02147, + "precision_at_3": 0.063, + "precision_at_5": 0.0481, + "recall_at_1": 0.0965, + "recall_at_10": 0.3335, + "recall_at_100": 0.645, + "recall_at_1000": 0.935, + "recall_at_20": 0.4295, + "recall_at_3": 0.189, + "recall_at_5": 0.2405 + }, + { + "cv_recall_at_1": 0.5075, + "cv_recall_at_10": 0.8545, + "cv_recall_at_100": 0.9855, + "cv_recall_at_1000": 0.998, + "cv_recall_at_20": 0.9225, + "cv_recall_at_3": 0.6945, + "cv_recall_at_5": 0.756, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6747, + "map_at_1": 0.5075, + "map_at_10": 0.61801, + "map_at_100": 0.6245, + "map_at_1000": 0.62457, + "map_at_20": 0.62272, + "map_at_3": 0.59067, + "map_at_5": 0.60464, + "mrr_at_1": 0.5075, + "mrr_at_10": 0.6180101190476197, + "mrr_at_100": 0.6244980889716759, + "mrr_at_1000": 0.6245731391573968, + "mrr_at_20": 0.622716377877369, + "mrr_at_3": 0.5906666666666675, + "mrr_at_5": 0.604641666666668, + "nauc_cv_recall_at_1000_diff1": 0.6095938375349927, + "nauc_cv_recall_at_1000_max": 0.967320261437891, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.4820824881676851, + "nauc_cv_recall_at_100_max": 0.5075823432821354, + "nauc_cv_recall_at_100_std": 0.3155285102546801, + "nauc_cv_recall_at_10_diff1": 0.4956058624184979, + "nauc_cv_recall_at_10_max": 0.47403160032249275, + "nauc_cv_recall_at_10_std": 0.07836853215398068, + "nauc_cv_recall_at_1_diff1": 0.6850281471236896, + "nauc_cv_recall_at_1_max": 0.44724621432096096, + "nauc_cv_recall_at_1_std": 0.029021916824158608, + "nauc_cv_recall_at_20_diff1": 0.46023011355079824, + "nauc_cv_recall_at_20_max": 0.46001927652781566, + "nauc_cv_recall_at_20_std": 0.14276678413300842, + "nauc_cv_recall_at_3_diff1": 0.5766142565770788, + "nauc_cv_recall_at_3_max": 0.46684329587974965, + "nauc_cv_recall_at_3_std": 0.046707921695569345, + "nauc_cv_recall_at_5_diff1": 0.5441108037261881, + "nauc_cv_recall_at_5_max": 0.4786111324572869, + "nauc_cv_recall_at_5_std": 0.06760226952534688, + "nauc_map_at_1000_diff1": 0.6401875478503286, + "nauc_map_at_1000_max": 0.45045885067932817, + "nauc_map_at_1000_std": 0.03892977625614155, + "nauc_map_at_100_diff1": 0.6401443049412445, + "nauc_map_at_100_max": 0.4504404663927708, + "nauc_map_at_100_std": 0.03894741190570511, + "nauc_map_at_10_diff1": 0.6377847159976536, + "nauc_map_at_10_max": 0.4508785914698639, + "nauc_map_at_10_std": 0.03819626760294528, + "nauc_map_at_1_diff1": 0.6850281471236896, + "nauc_map_at_1_max": 0.44724621432096096, + "nauc_map_at_1_std": 0.029021916824158608, + "nauc_map_at_20_diff1": 0.6390353557528677, + "nauc_map_at_20_max": 0.45026491456488166, + "nauc_map_at_20_std": 0.039003664779637244, + "nauc_map_at_3_diff1": 0.6409558441584462, + "nauc_map_at_3_max": 0.4510104885839783, + "nauc_map_at_3_std": 0.035494081481255046, + "nauc_map_at_5_diff1": 0.6377571454432578, + "nauc_map_at_5_max": 0.45223368623817506, + "nauc_map_at_5_std": 0.03851371505521922, + "nauc_mrr_at_1000_diff1": 0.6401875478503286, + "nauc_mrr_at_1000_max": 0.45045885067932817, + "nauc_mrr_at_1000_std": 0.03892977625614155, + "nauc_mrr_at_100_diff1": 0.6401443049412445, + "nauc_mrr_at_100_max": 0.4504404663927708, + "nauc_mrr_at_100_std": 0.03894741190570511, + "nauc_mrr_at_10_diff1": 0.6377847159976536, + "nauc_mrr_at_10_max": 0.4508785914698639, + "nauc_mrr_at_10_std": 0.03819626760294528, + "nauc_mrr_at_1_diff1": 0.6850281471236896, + "nauc_mrr_at_1_max": 0.44724621432096096, + "nauc_mrr_at_1_std": 0.029021916824158608, + "nauc_mrr_at_20_diff1": 0.6390353557528677, + "nauc_mrr_at_20_max": 0.45026491456488166, + "nauc_mrr_at_20_std": 0.039003664779637244, + "nauc_mrr_at_3_diff1": 0.6409558441584462, + "nauc_mrr_at_3_max": 0.4510104885839783, + "nauc_mrr_at_3_std": 0.035494081481255046, + "nauc_mrr_at_5_diff1": 0.6377571454432578, + "nauc_mrr_at_5_max": 0.45223368623817506, + "nauc_mrr_at_5_std": 0.03851371505521922, + "nauc_ndcg_at_1000_diff1": 0.6324611400318849, + "nauc_ndcg_at_1000_max": 0.45241723555814617, + "nauc_ndcg_at_1000_std": 0.0438352694520655, + "nauc_ndcg_at_100_diff1": 0.6312415529234053, + "nauc_ndcg_at_100_max": 0.452206191978489, + "nauc_ndcg_at_100_std": 0.04483598941004991, + "nauc_ndcg_at_10_diff1": 0.6176965168162838, + "nauc_ndcg_at_10_max": 0.45412985687060525, + "nauc_ndcg_at_10_std": 0.04369653927305166, + "nauc_ndcg_at_1_diff1": 0.6850281471236896, + "nauc_ndcg_at_1_max": 0.44724621432096096, + "nauc_ndcg_at_1_std": 0.029021916824158608, + "nauc_ndcg_at_20_diff1": 0.6224722379002166, + "nauc_ndcg_at_20_max": 0.45166656786354653, + "nauc_ndcg_at_20_std": 0.04699416807281372, + "nauc_ndcg_at_3_diff1": 0.6262669267161668, + "nauc_ndcg_at_3_max": 0.45442926321108096, + "nauc_ndcg_at_3_std": 0.03802072419464461, + "nauc_ndcg_at_5_diff1": 0.6190724921092656, + "nauc_ndcg_at_5_max": 0.4572192055170717, + "nauc_ndcg_at_5_std": 0.044100513069060335, + "nauc_precision_at_1000_diff1": 0.6095938375350154, + "nauc_precision_at_1000_max": 0.9673202614379109, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.48208248816768146, + "nauc_precision_at_100_max": 0.5075823432821327, + "nauc_precision_at_100_std": 0.3155285102546789, + "nauc_precision_at_10_diff1": 0.49560586241849763, + "nauc_precision_at_10_max": 0.47403160032249225, + "nauc_precision_at_10_std": 0.07836853215397865, + "nauc_precision_at_1_diff1": 0.6850281471236896, + "nauc_precision_at_1_max": 0.44724621432096096, + "nauc_precision_at_1_std": 0.029021916824158608, + "nauc_precision_at_20_diff1": 0.4602301135507949, + "nauc_precision_at_20_max": 0.46001927652781444, + "nauc_precision_at_20_std": 0.14276678413300534, + "nauc_precision_at_3_diff1": 0.5766142565770792, + "nauc_precision_at_3_max": 0.4668432958797501, + "nauc_precision_at_3_std": 0.04670792169556997, + "nauc_precision_at_5_diff1": 0.544110803726187, + "nauc_precision_at_5_max": 0.4786111324572858, + "nauc_precision_at_5_std": 0.06760226952534575, + "nauc_recall_at_1000_diff1": 0.6095938375349927, + "nauc_recall_at_1000_max": 0.967320261437891, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.4820824881676851, + "nauc_recall_at_100_max": 0.5075823432821354, + "nauc_recall_at_100_std": 0.3155285102546801, + "nauc_recall_at_10_diff1": 0.4956058624184979, + "nauc_recall_at_10_max": 0.47403160032249275, + "nauc_recall_at_10_std": 0.07836853215398068, + "nauc_recall_at_1_diff1": 0.6850281471236896, + "nauc_recall_at_1_max": 0.44724621432096096, + "nauc_recall_at_1_std": 0.029021916824158608, + "nauc_recall_at_20_diff1": 0.46023011355079824, + "nauc_recall_at_20_max": 0.46001927652781566, + "nauc_recall_at_20_std": 0.14276678413300842, + "nauc_recall_at_3_diff1": 0.5766142565770788, + "nauc_recall_at_3_max": 0.46684329587974965, + "nauc_recall_at_3_std": 0.046707921695569345, + "nauc_recall_at_5_diff1": 0.5441108037261881, + "nauc_recall_at_5_max": 0.4786111324572869, + "nauc_recall_at_5_std": 0.06760226952534688, + "ndcg_at_1": 0.5075, + "ndcg_at_10": 0.6747, + "ndcg_at_100": 0.70392, + "ndcg_at_1000": 0.70559, + "ndcg_at_20": 0.69188, + "ndcg_at_3": 0.61737, + "ndcg_at_5": 0.64263, + "precision_at_1": 0.5075, + "precision_at_10": 0.08545, + "precision_at_100": 0.00986, + "precision_at_1000": 0.001, + "precision_at_20": 0.04612, + "precision_at_3": 0.2315, + "precision_at_5": 0.1512, + "recall_at_1": 0.5075, + "recall_at_10": 0.8545, + "recall_at_100": 0.9855, + "recall_at_1000": 0.998, + "recall_at_20": 0.9225, + "recall_at_3": 0.6945, + "recall_at_5": 0.756 + }, + { + "cv_recall_at_1": 0.178, + "cv_recall_at_10": 0.499, + "cv_recall_at_100": 0.806, + "cv_recall_at_1000": 0.9815, + "cv_recall_at_20": 0.6035, + "cv_recall_at_3": 0.3085, + "cv_recall_at_5": 0.386, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.32156, + "map_at_1": 0.178, + "map_at_10": 0.26683, + "map_at_100": 0.27926, + "map_at_1000": 0.28003, + "map_at_20": 0.27412, + "map_at_3": 0.23392, + "map_at_5": 0.25179, + "mrr_at_1": 0.178, + "mrr_at_10": 0.2668319444444442, + "mrr_at_100": 0.27925731838360435, + "mrr_at_1000": 0.28003361566253326, + "mrr_at_20": 0.2741153201610047, + "mrr_at_3": 0.2339166666666664, + "mrr_at_5": 0.2517916666666662, + "nauc_cv_recall_at_1000_diff1": 0.13902137431548597, + "nauc_cv_recall_at_1000_max": 0.6893027481262831, + "nauc_cv_recall_at_1000_std": 0.8832235596941536, + "nauc_cv_recall_at_100_diff1": 0.17696568592187603, + "nauc_cv_recall_at_100_max": 0.4787900664905917, + "nauc_cv_recall_at_100_std": 0.5594677155805847, + "nauc_cv_recall_at_10_diff1": 0.20089162625222612, + "nauc_cv_recall_at_10_max": 0.35988054818897564, + "nauc_cv_recall_at_10_std": 0.308875104393797, + "nauc_cv_recall_at_1_diff1": 0.39869161921412205, + "nauc_cv_recall_at_1_max": 0.3995277159429934, + "nauc_cv_recall_at_1_std": 0.19832125932977787, + "nauc_cv_recall_at_20_diff1": 0.20381530227560704, + "nauc_cv_recall_at_20_max": 0.39670392767516544, + "nauc_cv_recall_at_20_std": 0.36613850164004713, + "nauc_cv_recall_at_3_diff1": 0.25113374985451803, + "nauc_cv_recall_at_3_max": 0.3840655418755832, + "nauc_cv_recall_at_3_std": 0.25083235840824936, + "nauc_cv_recall_at_5_diff1": 0.2354295015175038, + "nauc_cv_recall_at_5_max": 0.3670337531198894, + "nauc_cv_recall_at_5_std": 0.25613869803100303, + "nauc_map_at_1000_diff1": 0.3176858451051277, + "nauc_map_at_1000_max": 0.39011754631587403, + "nauc_map_at_1000_std": 0.236531571726055, + "nauc_map_at_100_diff1": 0.3173466063685225, + "nauc_map_at_100_max": 0.39012399997496483, + "nauc_map_at_100_std": 0.23699486254878585, + "nauc_map_at_10_diff1": 0.31498941632282373, + "nauc_map_at_10_max": 0.3854108768250645, + "nauc_map_at_10_std": 0.23161137771783466, + "nauc_map_at_1_diff1": 0.39869161921412205, + "nauc_map_at_1_max": 0.3995277159429934, + "nauc_map_at_1_std": 0.19832125932977787, + "nauc_map_at_20_diff1": 0.31655878548331357, + "nauc_map_at_20_max": 0.3884827665912284, + "nauc_map_at_20_std": 0.2352857571016366, + "nauc_map_at_3_diff1": 0.32501618220535744, + "nauc_map_at_3_max": 0.3911459304357299, + "nauc_map_at_3_std": 0.22201571734296388, + "nauc_map_at_5_diff1": 0.3201466900694881, + "nauc_map_at_5_max": 0.38699141592725994, + "nauc_map_at_5_std": 0.22417441689275736, + "nauc_mrr_at_1000_diff1": 0.3176858451051277, + "nauc_mrr_at_1000_max": 0.39011754631587403, + "nauc_mrr_at_1000_std": 0.236531571726055, + "nauc_mrr_at_100_diff1": 0.3173466063685225, + "nauc_mrr_at_100_max": 0.39012399997496483, + "nauc_mrr_at_100_std": 0.23699486254878585, + "nauc_mrr_at_10_diff1": 0.31498941632282373, + "nauc_mrr_at_10_max": 0.3854108768250645, + "nauc_mrr_at_10_std": 0.23161137771783466, + "nauc_mrr_at_1_diff1": 0.39869161921412205, + "nauc_mrr_at_1_max": 0.3995277159429934, + "nauc_mrr_at_1_std": 0.19832125932977787, + "nauc_mrr_at_20_diff1": 0.31655878548331357, + "nauc_mrr_at_20_max": 0.3884827665912284, + "nauc_mrr_at_20_std": 0.2352857571016366, + "nauc_mrr_at_3_diff1": 0.32501618220535744, + "nauc_mrr_at_3_max": 0.3911459304357299, + "nauc_mrr_at_3_std": 0.22201571734296388, + "nauc_mrr_at_5_diff1": 0.3201466900694881, + "nauc_mrr_at_5_max": 0.38699141592725994, + "nauc_mrr_at_5_std": 0.22417441689275736, + "nauc_ndcg_at_1000_diff1": 0.3003864567165558, + "nauc_ndcg_at_1000_max": 0.3956273045582767, + "nauc_ndcg_at_1000_std": 0.2619661173143108, + "nauc_ndcg_at_100_diff1": 0.29160769223913835, + "nauc_ndcg_at_100_max": 0.3990188203836822, + "nauc_ndcg_at_100_std": 0.27953787343260206, + "nauc_ndcg_at_10_diff1": 0.2838945792321021, + "nauc_ndcg_at_10_max": 0.37855000399640953, + "nauc_ndcg_at_10_std": 0.25146628957781897, + "nauc_ndcg_at_1_diff1": 0.39869161921412205, + "nauc_ndcg_at_1_max": 0.3995277159429934, + "nauc_ndcg_at_1_std": 0.19832125932977787, + "nauc_ndcg_at_20_diff1": 0.2876980324046296, + "nauc_ndcg_at_20_max": 0.3888791107654943, + "nauc_ndcg_at_20_std": 0.2647289947812142, + "nauc_ndcg_at_3_diff1": 0.3040117361499693, + "nauc_ndcg_at_3_max": 0.38910655776005576, + "nauc_ndcg_at_3_std": 0.2301072351567442, + "nauc_ndcg_at_5_diff1": 0.29588345621924833, + "nauc_ndcg_at_5_max": 0.3815210586999878, + "nauc_ndcg_at_5_std": 0.23330023029333213, + "nauc_precision_at_1000_diff1": 0.13902137431548026, + "nauc_precision_at_1000_max": 0.6893027481262702, + "nauc_precision_at_1000_std": 0.8832235596941468, + "nauc_precision_at_100_diff1": 0.1769656859218761, + "nauc_precision_at_100_max": 0.4787900664905905, + "nauc_precision_at_100_std": 0.5594677155805846, + "nauc_precision_at_10_diff1": 0.200891626252226, + "nauc_precision_at_10_max": 0.3598805481889754, + "nauc_precision_at_10_std": 0.30887510439379645, + "nauc_precision_at_1_diff1": 0.39869161921412205, + "nauc_precision_at_1_max": 0.3995277159429934, + "nauc_precision_at_1_std": 0.19832125932977787, + "nauc_precision_at_20_diff1": 0.20381530227560674, + "nauc_precision_at_20_max": 0.39670392767516544, + "nauc_precision_at_20_std": 0.36613850164004663, + "nauc_precision_at_3_diff1": 0.2511337498545181, + "nauc_precision_at_3_max": 0.3840655418755833, + "nauc_precision_at_3_std": 0.2508323584082494, + "nauc_precision_at_5_diff1": 0.23542950151750375, + "nauc_precision_at_5_max": 0.36703375311988906, + "nauc_precision_at_5_std": 0.25613869803100286, + "nauc_recall_at_1000_diff1": 0.13902137431548597, + "nauc_recall_at_1000_max": 0.6893027481262831, + "nauc_recall_at_1000_std": 0.8832235596941536, + "nauc_recall_at_100_diff1": 0.17696568592187603, + "nauc_recall_at_100_max": 0.4787900664905917, + "nauc_recall_at_100_std": 0.5594677155805847, + "nauc_recall_at_10_diff1": 0.20089162625222612, + "nauc_recall_at_10_max": 0.35988054818897564, + "nauc_recall_at_10_std": 0.308875104393797, + "nauc_recall_at_1_diff1": 0.39869161921412205, + "nauc_recall_at_1_max": 0.3995277159429934, + "nauc_recall_at_1_std": 0.19832125932977787, + "nauc_recall_at_20_diff1": 0.20381530227560704, + "nauc_recall_at_20_max": 0.39670392767516544, + "nauc_recall_at_20_std": 0.36613850164004713, + "nauc_recall_at_3_diff1": 0.25113374985451803, + "nauc_recall_at_3_max": 0.3840655418755832, + "nauc_recall_at_3_std": 0.25083235840824936, + "nauc_recall_at_5_diff1": 0.2354295015175038, + "nauc_recall_at_5_max": 0.3670337531198894, + "nauc_recall_at_5_std": 0.25613869803100303, + "ndcg_at_1": 0.178, + "ndcg_at_10": 0.32156, + "ndcg_at_100": 0.38552, + "ndcg_at_1000": 0.40745, + "ndcg_at_20": 0.34802, + "ndcg_at_3": 0.253, + "ndcg_at_5": 0.28507, + "precision_at_1": 0.178, + "precision_at_10": 0.0499, + "precision_at_100": 0.00806, + "precision_at_1000": 0.00098, + "precision_at_20": 0.03017, + "precision_at_3": 0.10283, + "precision_at_5": 0.0772, + "recall_at_1": 0.178, + "recall_at_10": 0.499, + "recall_at_100": 0.806, + "recall_at_1000": 0.9815, + "recall_at_20": 0.6035, + "recall_at_3": 0.3085, + "recall_at_5": 0.386 + }, + { + "cv_recall_at_1": 0.03, + "cv_recall_at_10": 0.15, + "cv_recall_at_100": 0.449, + "cv_recall_at_1000": 0.8745, + "cv_recall_at_20": 0.216, + "cv_recall_at_3": 0.071, + "cv_recall_at_5": 0.095, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.08105, + "map_at_1": 0.03, + "map_at_10": 0.06018, + "map_at_100": 0.07, + "map_at_1000": 0.07157, + "map_at_20": 0.06471, + "map_at_3": 0.04742, + "map_at_5": 0.05287, + "mrr_at_1": 0.03, + "mrr_at_10": 0.06017539682539684, + "mrr_at_100": 0.06999993552071689, + "mrr_at_1000": 0.07157353469443933, + "mrr_at_20": 0.06470514698150148, + "mrr_at_3": 0.047416666666666635, + "mrr_at_5": 0.05286666666666666, + "nauc_cv_recall_at_1000_diff1": 0.0031622471635306543, + "nauc_cv_recall_at_1000_max": 0.1230754238939771, + "nauc_cv_recall_at_1000_std": 0.5455420913249333, + "nauc_cv_recall_at_100_diff1": 0.004456004806830121, + "nauc_cv_recall_at_100_max": 0.14632687230090322, + "nauc_cv_recall_at_100_std": 0.48531704526562575, + "nauc_cv_recall_at_10_diff1": 0.02403991533111579, + "nauc_cv_recall_at_10_max": 0.17673319221852626, + "nauc_cv_recall_at_10_std": 0.3569922386856162, + "nauc_cv_recall_at_1_diff1": 0.1459753152669708, + "nauc_cv_recall_at_1_max": 0.2915481620606387, + "nauc_cv_recall_at_1_std": 0.2551202933547983, + "nauc_cv_recall_at_20_diff1": 0.00797385989303483, + "nauc_cv_recall_at_20_max": 0.17344280364233017, + "nauc_cv_recall_at_20_std": 0.3785712496399003, + "nauc_cv_recall_at_3_diff1": 0.02156324045696237, + "nauc_cv_recall_at_3_max": 0.14944807022980386, + "nauc_cv_recall_at_3_std": 0.25913112158326945, + "nauc_cv_recall_at_5_diff1": 0.02042421589256204, + "nauc_cv_recall_at_5_max": 0.1657464024967167, + "nauc_cv_recall_at_5_std": 0.31874514566534384, + "nauc_map_at_1000_diff1": 0.05222864258329805, + "nauc_map_at_1000_max": 0.19626197508061652, + "nauc_map_at_1000_std": 0.31100022728590426, + "nauc_map_at_100_diff1": 0.05216544996314628, + "nauc_map_at_100_max": 0.19629835874395282, + "nauc_map_at_100_std": 0.3112490909791947, + "nauc_map_at_10_diff1": 0.05857667764737536, + "nauc_map_at_10_max": 0.20089763876749644, + "nauc_map_at_10_std": 0.294255605599976, + "nauc_map_at_1_diff1": 0.1459753152669708, + "nauc_map_at_1_max": 0.2915481620606387, + "nauc_map_at_1_std": 0.2551202933547983, + "nauc_map_at_20_diff1": 0.05318000546518001, + "nauc_map_at_20_max": 0.19925942267648766, + "nauc_map_at_20_std": 0.3021914243564095, + "nauc_map_at_3_diff1": 0.06724170695673946, + "nauc_map_at_3_max": 0.2011644471563847, + "nauc_map_at_3_std": 0.2575835953190111, + "nauc_map_at_5_diff1": 0.06244812656762225, + "nauc_map_at_5_max": 0.20323435608935306, + "nauc_map_at_5_std": 0.2823908454409408, + "nauc_mrr_at_1000_diff1": 0.05222864258329805, + "nauc_mrr_at_1000_max": 0.19626197508061652, + "nauc_mrr_at_1000_std": 0.31100022728590426, + "nauc_mrr_at_100_diff1": 0.05216544996314628, + "nauc_mrr_at_100_max": 0.19629835874395282, + "nauc_mrr_at_100_std": 0.3112490909791947, + "nauc_mrr_at_10_diff1": 0.05857667764737536, + "nauc_mrr_at_10_max": 0.20089763876749644, + "nauc_mrr_at_10_std": 0.294255605599976, + "nauc_mrr_at_1_diff1": 0.1459753152669708, + "nauc_mrr_at_1_max": 0.2915481620606387, + "nauc_mrr_at_1_std": 0.2551202933547983, + "nauc_mrr_at_20_diff1": 0.05318000546518001, + "nauc_mrr_at_20_max": 0.19925942267648766, + "nauc_mrr_at_20_std": 0.3021914243564095, + "nauc_mrr_at_3_diff1": 0.06724170695673946, + "nauc_mrr_at_3_max": 0.2011644471563847, + "nauc_mrr_at_3_std": 0.2575835953190111, + "nauc_mrr_at_5_diff1": 0.06244812656762225, + "nauc_mrr_at_5_max": 0.20323435608935306, + "nauc_mrr_at_5_std": 0.2823908454409408, + "nauc_ndcg_at_1000_diff1": 0.03313709100866658, + "nauc_ndcg_at_1000_max": 0.17555839361410389, + "nauc_ndcg_at_1000_std": 0.36791242069420094, + "nauc_ndcg_at_100_diff1": 0.029382656610136004, + "nauc_ndcg_at_100_max": 0.17392712886916573, + "nauc_ndcg_at_100_std": 0.3798645355648723, + "nauc_ndcg_at_10_diff1": 0.04341832341400407, + "nauc_ndcg_at_10_max": 0.1895060849776859, + "nauc_ndcg_at_10_std": 0.3185087090048503, + "nauc_ndcg_at_1_diff1": 0.1459753152669708, + "nauc_ndcg_at_1_max": 0.2915481620606387, + "nauc_ndcg_at_1_std": 0.2551202933547983, + "nauc_ndcg_at_20_diff1": 0.032429813463278674, + "nauc_ndcg_at_20_max": 0.18647413650324113, + "nauc_ndcg_at_20_std": 0.3346764917048847, + "nauc_ndcg_at_3_diff1": 0.05174277111392793, + "nauc_ndcg_at_3_max": 0.1836002702068179, + "nauc_ndcg_at_3_std": 0.2581059616380073, + "nauc_ndcg_at_5_diff1": 0.04662102581978742, + "nauc_ndcg_at_5_max": 0.18887628288344824, + "nauc_ndcg_at_5_std": 0.29523558230070585, + "nauc_precision_at_1000_diff1": 0.0031622471635302882, + "nauc_precision_at_1000_max": 0.12307542389397594, + "nauc_precision_at_1000_std": 0.5455420913249317, + "nauc_precision_at_100_diff1": 0.004456004806830124, + "nauc_precision_at_100_max": 0.14632687230090355, + "nauc_precision_at_100_std": 0.48531704526562597, + "nauc_precision_at_10_diff1": 0.024039915331115713, + "nauc_precision_at_10_max": 0.17673319221852626, + "nauc_precision_at_10_std": 0.3569922386856165, + "nauc_precision_at_1_diff1": 0.1459753152669708, + "nauc_precision_at_1_max": 0.2915481620606387, + "nauc_precision_at_1_std": 0.2551202933547983, + "nauc_precision_at_20_diff1": 0.007973859893034724, + "nauc_precision_at_20_max": 0.17344280364233036, + "nauc_precision_at_20_std": 0.3785712496399004, + "nauc_precision_at_3_diff1": 0.021563240456962503, + "nauc_precision_at_3_max": 0.1494480702298039, + "nauc_precision_at_3_std": 0.25913112158326945, + "nauc_precision_at_5_diff1": 0.020424215892562138, + "nauc_precision_at_5_max": 0.16574640249671674, + "nauc_precision_at_5_std": 0.31874514566534407, + "nauc_recall_at_1000_diff1": 0.0031622471635306543, + "nauc_recall_at_1000_max": 0.1230754238939771, + "nauc_recall_at_1000_std": 0.5455420913249333, + "nauc_recall_at_100_diff1": 0.004456004806830121, + "nauc_recall_at_100_max": 0.14632687230090322, + "nauc_recall_at_100_std": 0.48531704526562575, + "nauc_recall_at_10_diff1": 0.02403991533111579, + "nauc_recall_at_10_max": 0.17673319221852626, + "nauc_recall_at_10_std": 0.3569922386856162, + "nauc_recall_at_1_diff1": 0.1459753152669708, + "nauc_recall_at_1_max": 0.2915481620606387, + "nauc_recall_at_1_std": 0.2551202933547983, + "nauc_recall_at_20_diff1": 0.00797385989303483, + "nauc_recall_at_20_max": 0.17344280364233017, + "nauc_recall_at_20_std": 0.3785712496399003, + "nauc_recall_at_3_diff1": 0.02156324045696237, + "nauc_recall_at_3_max": 0.14944807022980386, + "nauc_recall_at_3_std": 0.25913112158326945, + "nauc_recall_at_5_diff1": 0.02042421589256204, + "nauc_recall_at_5_max": 0.1657464024967167, + "nauc_recall_at_5_std": 0.31874514566534384, + "ndcg_at_1": 0.03, + "ndcg_at_10": 0.08105, + "ndcg_at_100": 0.13952, + "ndcg_at_1000": 0.19087, + "ndcg_at_20": 0.09768, + "ndcg_at_3": 0.05345, + "ndcg_at_5": 0.0633, + "precision_at_1": 0.03, + "precision_at_10": 0.015, + "precision_at_100": 0.00449, + "precision_at_1000": 0.00087, + "precision_at_20": 0.0108, + "precision_at_3": 0.02367, + "precision_at_5": 0.019, + "recall_at_1": 0.03, + "recall_at_10": 0.15, + "recall_at_100": 0.449, + "recall_at_1000": 0.8745, + "recall_at_20": 0.216, + "recall_at_3": 0.071, + "recall_at_5": 0.095 + }, + { + "cv_recall_at_1": 0.016, + "cv_recall_at_10": 0.091, + "cv_recall_at_100": 0.3185, + "cv_recall_at_1000": 0.8135, + "cv_recall_at_20": 0.1365, + "cv_recall_at_3": 0.041, + "cv_recall_at_5": 0.059, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.04778, + "map_at_1": 0.016, + "map_at_10": 0.03469, + "map_at_100": 0.0421, + "map_at_1000": 0.0438, + "map_at_20": 0.03791, + "map_at_3": 0.02642, + "map_at_5": 0.03054, + "mrr_at_1": 0.016, + "mrr_at_10": 0.03468730158730162, + "mrr_at_100": 0.042098286930249154, + "mrr_at_1000": 0.0438005310087541, + "mrr_at_20": 0.03790599534839091, + "mrr_at_3": 0.026416666666666675, + "mrr_at_5": 0.030541666666666682, + "nauc_cv_recall_at_1000_diff1": 0.07106216173723574, + "nauc_cv_recall_at_1000_max": 0.3177797971022986, + "nauc_cv_recall_at_1000_std": 0.5407727380530455, + "nauc_cv_recall_at_100_diff1": 0.0713599535138732, + "nauc_cv_recall_at_100_max": 0.297997227166803, + "nauc_cv_recall_at_100_std": 0.48667707873788535, + "nauc_cv_recall_at_10_diff1": 0.05669645621349406, + "nauc_cv_recall_at_10_max": 0.2794030493252388, + "nauc_cv_recall_at_10_std": 0.40729514649557574, + "nauc_cv_recall_at_1_diff1": 0.14172759592165277, + "nauc_cv_recall_at_1_max": 0.19727662999731693, + "nauc_cv_recall_at_1_std": 0.400665749932922, + "nauc_cv_recall_at_20_diff1": 0.09607008604081496, + "nauc_cv_recall_at_20_max": 0.2698632401611685, + "nauc_cv_recall_at_20_std": 0.40934703849091997, + "nauc_cv_recall_at_3_diff1": 0.14946304815878855, + "nauc_cv_recall_at_3_max": 0.2802096762582865, + "nauc_cv_recall_at_3_std": 0.42041922163251666, + "nauc_cv_recall_at_5_diff1": 0.08865220811940362, + "nauc_cv_recall_at_5_max": 0.2713910856643915, + "nauc_cv_recall_at_5_std": 0.442433365318587, + "nauc_map_at_1000_diff1": 0.10616900969923737, + "nauc_map_at_1000_max": 0.2520076462657044, + "nauc_map_at_1000_std": 0.41663431232817416, + "nauc_map_at_100_diff1": 0.1055771817335392, + "nauc_map_at_100_max": 0.25226679270899655, + "nauc_map_at_100_std": 0.4159028968396924, + "nauc_map_at_10_diff1": 0.1051404762247085, + "nauc_map_at_10_max": 0.2502761119493815, + "nauc_map_at_10_std": 0.4074511071618613, + "nauc_map_at_1_diff1": 0.14172759592165277, + "nauc_map_at_1_max": 0.19727662999731693, + "nauc_map_at_1_std": 0.400665749932922, + "nauc_map_at_20_diff1": 0.11168787671435364, + "nauc_map_at_20_max": 0.24958328967583604, + "nauc_map_at_20_std": 0.4069329483284715, + "nauc_map_at_3_diff1": 0.14520944019216925, + "nauc_map_at_3_max": 0.24336265583486172, + "nauc_map_at_3_std": 0.40580671864195006, + "nauc_map_at_5_diff1": 0.12029564869169368, + "nauc_map_at_5_max": 0.24414012125666815, + "nauc_map_at_5_std": 0.41652591556544544, + "nauc_mrr_at_1000_diff1": 0.10616900969923737, + "nauc_mrr_at_1000_max": 0.2520076462657044, + "nauc_mrr_at_1000_std": 0.41663431232817416, + "nauc_mrr_at_100_diff1": 0.1055771817335392, + "nauc_mrr_at_100_max": 0.25226679270899655, + "nauc_mrr_at_100_std": 0.4159028968396924, + "nauc_mrr_at_10_diff1": 0.1051404762247085, + "nauc_mrr_at_10_max": 0.2502761119493815, + "nauc_mrr_at_10_std": 0.4074511071618613, + "nauc_mrr_at_1_diff1": 0.14172759592165277, + "nauc_mrr_at_1_max": 0.19727662999731693, + "nauc_mrr_at_1_std": 0.400665749932922, + "nauc_mrr_at_20_diff1": 0.11168787671435364, + "nauc_mrr_at_20_max": 0.24958328967583604, + "nauc_mrr_at_20_std": 0.4069329483284715, + "nauc_mrr_at_3_diff1": 0.14520944019216925, + "nauc_mrr_at_3_max": 0.24336265583486172, + "nauc_mrr_at_3_std": 0.40580671864195006, + "nauc_mrr_at_5_diff1": 0.12029564869169368, + "nauc_mrr_at_5_max": 0.24414012125666815, + "nauc_mrr_at_5_std": 0.41652591556544544, + "nauc_ndcg_at_1000_diff1": 0.09118363577195546, + "nauc_ndcg_at_1000_max": 0.2734412591888149, + "nauc_ndcg_at_1000_std": 0.45381717258062526, + "nauc_ndcg_at_100_diff1": 0.08635669876740394, + "nauc_ndcg_at_100_max": 0.2745828115432325, + "nauc_ndcg_at_100_std": 0.4479316944010123, + "nauc_ndcg_at_10_diff1": 0.08469399333358935, + "nauc_ndcg_at_10_max": 0.2633158764122716, + "nauc_ndcg_at_10_std": 0.40804473149726966, + "nauc_ndcg_at_1_diff1": 0.14172759592165277, + "nauc_ndcg_at_1_max": 0.19727662999731693, + "nauc_ndcg_at_1_std": 0.400665749932922, + "nauc_ndcg_at_20_diff1": 0.10288612513091308, + "nauc_ndcg_at_20_max": 0.2601636991744163, + "nauc_ndcg_at_20_std": 0.4082859431626536, + "nauc_ndcg_at_3_diff1": 0.14663138399660128, + "nauc_ndcg_at_3_max": 0.2558808244090084, + "nauc_ndcg_at_3_std": 0.41060566375848273, + "nauc_ndcg_at_5_diff1": 0.10881705441852337, + "nauc_ndcg_at_5_max": 0.2547659569303373, + "nauc_ndcg_at_5_std": 0.4260939220880413, + "nauc_precision_at_1000_diff1": 0.07106216173723427, + "nauc_precision_at_1000_max": 0.31777979710229853, + "nauc_precision_at_1000_std": 0.5407727380530446, + "nauc_precision_at_100_diff1": 0.0713599535138732, + "nauc_precision_at_100_max": 0.2979972271668032, + "nauc_precision_at_100_std": 0.4866770787378858, + "nauc_precision_at_10_diff1": 0.0566964562134939, + "nauc_precision_at_10_max": 0.27940304932523885, + "nauc_precision_at_10_std": 0.40729514649557574, + "nauc_precision_at_1_diff1": 0.14172759592165277, + "nauc_precision_at_1_max": 0.19727662999731693, + "nauc_precision_at_1_std": 0.400665749932922, + "nauc_precision_at_20_diff1": 0.09607008604081504, + "nauc_precision_at_20_max": 0.2698632401611685, + "nauc_precision_at_20_std": 0.40934703849091997, + "nauc_precision_at_3_diff1": 0.14946304815878844, + "nauc_precision_at_3_max": 0.2802096762582866, + "nauc_precision_at_3_std": 0.4204192216325167, + "nauc_precision_at_5_diff1": 0.0886522081194035, + "nauc_precision_at_5_max": 0.2713910856643913, + "nauc_precision_at_5_std": 0.4424333653185869, + "nauc_recall_at_1000_diff1": 0.07106216173723574, + "nauc_recall_at_1000_max": 0.3177797971022986, + "nauc_recall_at_1000_std": 0.5407727380530455, + "nauc_recall_at_100_diff1": 0.0713599535138732, + "nauc_recall_at_100_max": 0.297997227166803, + "nauc_recall_at_100_std": 0.48667707873788535, + "nauc_recall_at_10_diff1": 0.05669645621349406, + "nauc_recall_at_10_max": 0.2794030493252388, + "nauc_recall_at_10_std": 0.40729514649557574, + "nauc_recall_at_1_diff1": 0.14172759592165277, + "nauc_recall_at_1_max": 0.19727662999731693, + "nauc_recall_at_1_std": 0.400665749932922, + "nauc_recall_at_20_diff1": 0.09607008604081496, + "nauc_recall_at_20_max": 0.2698632401611685, + "nauc_recall_at_20_std": 0.40934703849091997, + "nauc_recall_at_3_diff1": 0.14946304815878855, + "nauc_recall_at_3_max": 0.2802096762582865, + "nauc_recall_at_3_std": 0.42041922163251666, + "nauc_recall_at_5_diff1": 0.08865220811940362, + "nauc_recall_at_5_max": 0.2713910856643915, + "nauc_recall_at_5_std": 0.442433365318587, + "ndcg_at_1": 0.016, + "ndcg_at_10": 0.04778, + "ndcg_at_100": 0.09216, + "ndcg_at_1000": 0.15112, + "ndcg_at_20": 0.05937, + "ndcg_at_3": 0.03014, + "ndcg_at_5": 0.03756, + "precision_at_1": 0.016, + "precision_at_10": 0.0091, + "precision_at_100": 0.00318, + "precision_at_1000": 0.00081, + "precision_at_20": 0.00683, + "precision_at_3": 0.01367, + "precision_at_5": 0.0118, + "recall_at_1": 0.016, + "recall_at_10": 0.091, + "recall_at_100": 0.3185, + "recall_at_1000": 0.8135, + "recall_at_20": 0.1365, + "recall_at_3": 0.041, + "recall_at_5": 0.059 + }, + { + "cv_recall_at_1": 0.005, + "cv_recall_at_10": 0.018, + "cv_recall_at_100": 0.087, + "cv_recall_at_1000": 0.5615, + "cv_recall_at_20": 0.028, + "cv_recall_at_3": 0.0085, + "cv_recall_at_5": 0.0135, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.01038, + "map_at_1": 0.005, + "map_at_10": 0.00807, + "map_at_100": 0.00989, + "map_at_1000": 0.01114, + "map_at_20": 0.00875, + "map_at_3": 0.00642, + "map_at_5": 0.00749, + "mrr_at_1": 0.005, + "mrr_at_10": 0.00806984126984127, + "mrr_at_100": 0.009885263099824498, + "mrr_at_1000": 0.011144283116471717, + "mrr_at_20": 0.008748844853546866, + "mrr_at_3": 0.006416666666666667, + "mrr_at_5": 0.0074916666666666664, + "nauc_cv_recall_at_1000_diff1": 0.01866337186197954, + "nauc_cv_recall_at_1000_max": 0.1432479371333068, + "nauc_cv_recall_at_1000_std": 0.1353675624298429, + "nauc_cv_recall_at_100_diff1": 0.09785535190548013, + "nauc_cv_recall_at_100_max": 0.26174174785427257, + "nauc_cv_recall_at_100_std": 0.32808582293237604, + "nauc_cv_recall_at_10_diff1": 0.15284112929672355, + "nauc_cv_recall_at_10_max": 0.5223668127478165, + "nauc_cv_recall_at_10_std": 0.5570759920102556, + "nauc_cv_recall_at_1_diff1": 0.12621411322779696, + "nauc_cv_recall_at_1_max": 0.5159109203112423, + "nauc_cv_recall_at_1_std": 0.5785350147571774, + "nauc_cv_recall_at_20_diff1": 0.18863314040400164, + "nauc_cv_recall_at_20_max": 0.43482885507301905, + "nauc_cv_recall_at_20_std": 0.46413718425389994, + "nauc_cv_recall_at_3_diff1": 0.10757745545226408, + "nauc_cv_recall_at_3_max": 0.48136807714768237, + "nauc_cv_recall_at_3_std": 0.5222304644959674, + "nauc_cv_recall_at_5_diff1": 0.14783014836677316, + "nauc_cv_recall_at_5_max": 0.5114529608760894, + "nauc_cv_recall_at_5_std": 0.5787099146369339, + "nauc_map_at_1000_diff1": 0.13484125064851551, + "nauc_map_at_1000_max": 0.43525690428320235, + "nauc_map_at_1000_std": 0.4860746717149452, + "nauc_map_at_100_diff1": 0.14322051241470532, + "nauc_map_at_100_max": 0.4553782899063439, + "nauc_map_at_100_std": 0.510862684307652, + "nauc_map_at_10_diff1": 0.14185029312751027, + "nauc_map_at_10_max": 0.5083078127015377, + "nauc_map_at_10_std": 0.5587400256321647, + "nauc_map_at_1_diff1": 0.12621411322779696, + "nauc_map_at_1_max": 0.5159109203112423, + "nauc_map_at_1_std": 0.5785350147571774, + "nauc_map_at_20_diff1": 0.1542361916482395, + "nauc_map_at_20_max": 0.4926807911639361, + "nauc_map_at_20_std": 0.5426787801006763, + "nauc_map_at_3_diff1": 0.12486976398969958, + "nauc_map_at_3_max": 0.48833538342526805, + "nauc_map_at_3_std": 0.5397990793751459, + "nauc_map_at_5_diff1": 0.14057476139155894, + "nauc_map_at_5_max": 0.503370319046921, + "nauc_map_at_5_std": 0.5624163389366535, + "nauc_mrr_at_1000_diff1": 0.13484125064851551, + "nauc_mrr_at_1000_max": 0.43525690428320235, + "nauc_mrr_at_1000_std": 0.4860746717149452, + "nauc_mrr_at_100_diff1": 0.14322051241470532, + "nauc_mrr_at_100_max": 0.4553782899063439, + "nauc_mrr_at_100_std": 0.510862684307652, + "nauc_mrr_at_10_diff1": 0.14185029312751027, + "nauc_mrr_at_10_max": 0.5083078127015377, + "nauc_mrr_at_10_std": 0.5587400256321647, + "nauc_mrr_at_1_diff1": 0.12621411322779696, + "nauc_mrr_at_1_max": 0.5159109203112423, + "nauc_mrr_at_1_std": 0.5785350147571774, + "nauc_mrr_at_20_diff1": 0.1542361916482395, + "nauc_mrr_at_20_max": 0.4926807911639361, + "nauc_mrr_at_20_std": 0.5426787801006763, + "nauc_mrr_at_3_diff1": 0.12486976398969958, + "nauc_mrr_at_3_max": 0.48833538342526805, + "nauc_mrr_at_3_std": 0.5397990793751459, + "nauc_mrr_at_5_diff1": 0.14057476139155894, + "nauc_mrr_at_5_max": 0.503370319046921, + "nauc_mrr_at_5_std": 0.5624163389366535, + "nauc_ndcg_at_1000_diff1": 0.06105664555017735, + "nauc_ndcg_at_1000_max": 0.2326125051202298, + "nauc_ndcg_at_1000_std": 0.24874451461947863, + "nauc_ndcg_at_100_diff1": 0.11970760208087682, + "nauc_ndcg_at_100_max": 0.34084016686829804, + "nauc_ndcg_at_100_std": 0.4020989248226949, + "nauc_ndcg_at_10_diff1": 0.14611669901831748, + "nauc_ndcg_at_10_max": 0.513929921408022, + "nauc_ndcg_at_10_std": 0.558476215872956, + "nauc_ndcg_at_1_diff1": 0.12621411322779696, + "nauc_ndcg_at_1_max": 0.5159109203112423, + "nauc_ndcg_at_1_std": 0.5785350147571774, + "nauc_ndcg_at_20_diff1": 0.17016830197419816, + "nauc_ndcg_at_20_max": 0.4698563274930774, + "nauc_ndcg_at_20_std": 0.5111408910283253, + "nauc_ndcg_at_3_diff1": 0.11980779651940149, + "nauc_ndcg_at_3_max": 0.4858343012706748, + "nauc_ndcg_at_3_std": 0.5340182129958824, + "nauc_ndcg_at_5_diff1": 0.14327180898186234, + "nauc_ndcg_at_5_max": 0.5062088605443746, + "nauc_ndcg_at_5_std": 0.5678637638968992, + "nauc_precision_at_1000_diff1": 0.01866337186197888, + "nauc_precision_at_1000_max": 0.14324793713330597, + "nauc_precision_at_1000_std": 0.1353675624298418, + "nauc_precision_at_100_diff1": 0.09785535190547996, + "nauc_precision_at_100_max": 0.26174174785427246, + "nauc_precision_at_100_std": 0.3280858229323761, + "nauc_precision_at_10_diff1": 0.15284112929672372, + "nauc_precision_at_10_max": 0.5223668127478164, + "nauc_precision_at_10_std": 0.5570759920102557, + "nauc_precision_at_1_diff1": 0.12621411322779696, + "nauc_precision_at_1_max": 0.5159109203112423, + "nauc_precision_at_1_std": 0.5785350147571774, + "nauc_precision_at_20_diff1": 0.18863314040400164, + "nauc_precision_at_20_max": 0.4348288550730195, + "nauc_precision_at_20_std": 0.4641371842539002, + "nauc_precision_at_3_diff1": 0.10757745545226413, + "nauc_precision_at_3_max": 0.48136807714768237, + "nauc_precision_at_3_std": 0.5222304644959676, + "nauc_precision_at_5_diff1": 0.147830148366773, + "nauc_precision_at_5_max": 0.5114529608760895, + "nauc_precision_at_5_std": 0.5787099146369338, + "nauc_recall_at_1000_diff1": 0.01866337186197954, + "nauc_recall_at_1000_max": 0.1432479371333068, + "nauc_recall_at_1000_std": 0.1353675624298429, + "nauc_recall_at_100_diff1": 0.09785535190548013, + "nauc_recall_at_100_max": 0.26174174785427257, + "nauc_recall_at_100_std": 0.32808582293237604, + "nauc_recall_at_10_diff1": 0.15284112929672355, + "nauc_recall_at_10_max": 0.5223668127478165, + "nauc_recall_at_10_std": 0.5570759920102556, + "nauc_recall_at_1_diff1": 0.12621411322779696, + "nauc_recall_at_1_max": 0.5159109203112423, + "nauc_recall_at_1_std": 0.5785350147571774, + "nauc_recall_at_20_diff1": 0.18863314040400164, + "nauc_recall_at_20_max": 0.43482885507301905, + "nauc_recall_at_20_std": 0.46413718425389994, + "nauc_recall_at_3_diff1": 0.10757745545226408, + "nauc_recall_at_3_max": 0.48136807714768237, + "nauc_recall_at_3_std": 0.5222304644959674, + "nauc_recall_at_5_diff1": 0.14783014836677316, + "nauc_recall_at_5_max": 0.5114529608760894, + "nauc_recall_at_5_std": 0.5787099146369339, + "ndcg_at_1": 0.005, + "ndcg_at_10": 0.01038, + "ndcg_at_100": 0.02307, + "ndcg_at_1000": 0.07725, + "ndcg_at_20": 0.01289, + "ndcg_at_3": 0.00695, + "ndcg_at_5": 0.00895, + "precision_at_1": 0.005, + "precision_at_10": 0.0018, + "precision_at_100": 0.00087, + "precision_at_1000": 0.00056, + "precision_at_20": 0.0014, + "precision_at_3": 0.00283, + "precision_at_5": 0.0027, + "recall_at_1": 0.005, + "recall_at_10": 0.018, + "recall_at_100": 0.087, + "recall_at_1000": 0.5615, + "recall_at_20": 0.028, + "recall_at_3": 0.0085, + "recall_at_5": 0.0135 + }, + { + "cv_recall_at_1": 0.0155, + "cv_recall_at_10": 0.073, + "cv_recall_at_100": 0.245, + "cv_recall_at_1000": 0.6955, + "cv_recall_at_20": 0.105, + "cv_recall_at_3": 0.0325, + "cv_recall_at_5": 0.0515, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.03951, + "map_at_1": 0.0155, + "map_at_10": 0.02936, + "map_at_100": 0.03458, + "map_at_1000": 0.03597, + "map_at_20": 0.03162, + "map_at_3": 0.02233, + "map_at_5": 0.02666, + "mrr_at_1": 0.0155, + "mrr_at_10": 0.029362896825396858, + "mrr_at_100": 0.03457677859623816, + "mrr_at_1000": 0.035970307503409196, + "mrr_at_20": 0.03161513074343963, + "mrr_at_3": 0.02233333333333334, + "mrr_at_5": 0.026658333333333357, + "nauc_cv_recall_at_1000_diff1": 0.10103909150196763, + "nauc_cv_recall_at_1000_max": 0.321531328425893, + "nauc_cv_recall_at_1000_std": 0.3990844717958969, + "nauc_cv_recall_at_100_diff1": 0.1785332076619936, + "nauc_cv_recall_at_100_max": 0.437202674721507, + "nauc_cv_recall_at_100_std": 0.515561476291904, + "nauc_cv_recall_at_10_diff1": 0.14435937678032573, + "nauc_cv_recall_at_10_max": 0.4617783593253233, + "nauc_cv_recall_at_10_std": 0.48276920362699444, + "nauc_cv_recall_at_1_diff1": 0.3766499043596423, + "nauc_cv_recall_at_1_max": 0.5343223382985537, + "nauc_cv_recall_at_1_std": 0.47144204886746255, + "nauc_cv_recall_at_20_diff1": 0.1347877465684938, + "nauc_cv_recall_at_20_max": 0.44873712779357744, + "nauc_cv_recall_at_20_std": 0.486303842507824, + "nauc_cv_recall_at_3_diff1": 0.2304802790448082, + "nauc_cv_recall_at_3_max": 0.5613134919815895, + "nauc_cv_recall_at_3_std": 0.49969660068935606, + "nauc_cv_recall_at_5_diff1": 0.18932690078435777, + "nauc_cv_recall_at_5_max": 0.4822874797137655, + "nauc_cv_recall_at_5_std": 0.49071978034859776, + "nauc_map_at_1000_diff1": 0.23946320151573663, + "nauc_map_at_1000_max": 0.4999034385390828, + "nauc_map_at_1000_std": 0.49153269673252975, + "nauc_map_at_100_diff1": 0.23981703740687996, + "nauc_map_at_100_max": 0.500941148105614, + "nauc_map_at_100_std": 0.49212619471780217, + "nauc_map_at_10_diff1": 0.24737658303862772, + "nauc_map_at_10_max": 0.5085696956319696, + "nauc_map_at_10_std": 0.4840574549189613, + "nauc_map_at_1_diff1": 0.3766499043596423, + "nauc_map_at_1_max": 0.5343223382985537, + "nauc_map_at_1_std": 0.47144204886746255, + "nauc_map_at_20_diff1": 0.23875732622736212, + "nauc_map_at_20_max": 0.5025212214500192, + "nauc_map_at_20_std": 0.4861404954526816, + "nauc_map_at_3_diff1": 0.2963179140519566, + "nauc_map_at_3_max": 0.5525251392621011, + "nauc_map_at_3_std": 0.4902626657429246, + "nauc_map_at_5_diff1": 0.26780211115410124, + "nauc_map_at_5_max": 0.5187074240818313, + "nauc_map_at_5_std": 0.48668004230259443, + "nauc_mrr_at_1000_diff1": 0.23946245396171295, + "nauc_mrr_at_1000_max": 0.49990313744248865, + "nauc_mrr_at_1000_std": 0.4915329100203209, + "nauc_mrr_at_100_diff1": 0.23981703740687996, + "nauc_mrr_at_100_max": 0.500941148105614, + "nauc_mrr_at_100_std": 0.49212619471780217, + "nauc_mrr_at_10_diff1": 0.24737658303862772, + "nauc_mrr_at_10_max": 0.5085696956319696, + "nauc_mrr_at_10_std": 0.4840574549189613, + "nauc_mrr_at_1_diff1": 0.3766499043596423, + "nauc_mrr_at_1_max": 0.5343223382985537, + "nauc_mrr_at_1_std": 0.47144204886746255, + "nauc_mrr_at_20_diff1": 0.23875732622736212, + "nauc_mrr_at_20_max": 0.5025212214500192, + "nauc_mrr_at_20_std": 0.4861404954526816, + "nauc_mrr_at_3_diff1": 0.2963179140519566, + "nauc_mrr_at_3_max": 0.5525251392621011, + "nauc_mrr_at_3_std": 0.4902626657429246, + "nauc_mrr_at_5_diff1": 0.26780211115410124, + "nauc_mrr_at_5_max": 0.5187074240818313, + "nauc_mrr_at_5_std": 0.48668004230259443, + "nauc_ndcg_at_1000_diff1": 0.18188268853902662, + "nauc_ndcg_at_1000_max": 0.4366951694757732, + "nauc_ndcg_at_1000_std": 0.4718476543887696, + "nauc_ndcg_at_100_diff1": 0.20134563640609476, + "nauc_ndcg_at_100_max": 0.46779496202101484, + "nauc_ndcg_at_100_std": 0.5044989040265137, + "nauc_ndcg_at_10_diff1": 0.20378966242608526, + "nauc_ndcg_at_10_max": 0.4891954411104291, + "nauc_ndcg_at_10_std": 0.4836847893872883, + "nauc_ndcg_at_1_diff1": 0.3766499043596423, + "nauc_ndcg_at_1_max": 0.5343223382985537, + "nauc_ndcg_at_1_std": 0.47144204886746255, + "nauc_ndcg_at_20_diff1": 0.18929143990336425, + "nauc_ndcg_at_20_max": 0.4776004370098488, + "nauc_ndcg_at_20_std": 0.4866188784612998, + "nauc_ndcg_at_3_diff1": 0.2746821234810608, + "nauc_ndcg_at_3_max": 0.5555235609881958, + "nauc_ndcg_at_3_std": 0.49347073414047715, + "nauc_ndcg_at_5_diff1": 0.237699805726334, + "nauc_ndcg_at_5_max": 0.5054343509360476, + "nauc_ndcg_at_5_std": 0.48831749596962815, + "nauc_precision_at_1000_diff1": 0.10103909150196676, + "nauc_precision_at_1000_max": 0.3215313284258918, + "nauc_precision_at_1000_std": 0.3990844717958955, + "nauc_precision_at_100_diff1": 0.17853320766199374, + "nauc_precision_at_100_max": 0.4372026747215069, + "nauc_precision_at_100_std": 0.5155614762919041, + "nauc_precision_at_10_diff1": 0.14435937678032562, + "nauc_precision_at_10_max": 0.46177835932532324, + "nauc_precision_at_10_std": 0.4827692036269943, + "nauc_precision_at_1_diff1": 0.3766499043596423, + "nauc_precision_at_1_max": 0.5343223382985537, + "nauc_precision_at_1_std": 0.47144204886746255, + "nauc_precision_at_20_diff1": 0.13478774656849388, + "nauc_precision_at_20_max": 0.4487371277935772, + "nauc_precision_at_20_std": 0.48630384250782405, + "nauc_precision_at_3_diff1": 0.23048027904480814, + "nauc_precision_at_3_max": 0.5613134919815899, + "nauc_precision_at_3_std": 0.49969660068935623, + "nauc_precision_at_5_diff1": 0.18932690078435782, + "nauc_precision_at_5_max": 0.48228747971376534, + "nauc_precision_at_5_std": 0.4907197803485978, + "nauc_recall_at_1000_diff1": 0.10103909150196763, + "nauc_recall_at_1000_max": 0.321531328425893, + "nauc_recall_at_1000_std": 0.3990844717958969, + "nauc_recall_at_100_diff1": 0.1785332076619936, + "nauc_recall_at_100_max": 0.437202674721507, + "nauc_recall_at_100_std": 0.515561476291904, + "nauc_recall_at_10_diff1": 0.14435937678032573, + "nauc_recall_at_10_max": 0.4617783593253233, + "nauc_recall_at_10_std": 0.48276920362699444, + "nauc_recall_at_1_diff1": 0.3766499043596423, + "nauc_recall_at_1_max": 0.5343223382985537, + "nauc_recall_at_1_std": 0.47144204886746255, + "nauc_recall_at_20_diff1": 0.1347877465684938, + "nauc_recall_at_20_max": 0.44873712779357744, + "nauc_recall_at_20_std": 0.486303842507824, + "nauc_recall_at_3_diff1": 0.2304802790448082, + "nauc_recall_at_3_max": 0.5613134919815895, + "nauc_recall_at_3_std": 0.49969660068935606, + "nauc_recall_at_5_diff1": 0.18932690078435777, + "nauc_recall_at_5_max": 0.4822874797137655, + "nauc_recall_at_5_std": 0.49071978034859776, + "ndcg_at_1": 0.0155, + "ndcg_at_10": 0.03951, + "ndcg_at_100": 0.07233, + "ndcg_at_1000": 0.125, + "ndcg_at_20": 0.04764, + "ndcg_at_3": 0.02492, + "ndcg_at_5": 0.03273, + "precision_at_1": 0.0155, + "precision_at_10": 0.0073, + "precision_at_100": 0.00245, + "precision_at_1000": 0.0007, + "precision_at_20": 0.00525, + "precision_at_3": 0.01083, + "precision_at_5": 0.0103, + "recall_at_1": 0.0155, + "recall_at_10": 0.073, + "recall_at_100": 0.245, + "recall_at_1000": 0.6955, + "recall_at_20": 0.105, + "recall_at_3": 0.0325, + "recall_at_5": 0.0515 + }, + { + "cv_recall_at_1": 0.004, + "cv_recall_at_10": 0.0285, + "cv_recall_at_100": 0.1545, + "cv_recall_at_1000": 0.676, + "cv_recall_at_20": 0.0525, + "cv_recall_at_3": 0.0125, + "cv_recall_at_5": 0.0185, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ], + "main_score": 0.01454, + "map_at_1": 0.004, + "map_at_10": 0.01031, + "map_at_100": 0.01413, + "map_at_1000": 0.01568, + "map_at_20": 0.01189, + "map_at_3": 0.00767, + "map_at_5": 0.00907, + "mrr_at_1": 0.004, + "mrr_at_10": 0.010310912698412694, + "mrr_at_100": 0.014128261725250548, + "mrr_at_1000": 0.015682993790635865, + "mrr_at_20": 0.011885865834801591, + "mrr_at_3": 0.007666666666666668, + "mrr_at_5": 0.009066666666666664, + "nauc_cv_recall_at_1000_diff1": 0.1463224055326704, + "nauc_cv_recall_at_1000_max": 0.20534263898503774, + "nauc_cv_recall_at_1000_std": 0.28937880833738816, + "nauc_cv_recall_at_100_diff1": 0.11011108120551756, + "nauc_cv_recall_at_100_max": 0.24934265204776085, + "nauc_cv_recall_at_100_std": 0.2998186932307211, + "nauc_cv_recall_at_10_diff1": 0.14244559614759988, + "nauc_cv_recall_at_10_max": 0.3921737534068603, + "nauc_cv_recall_at_10_std": 0.3703933835124436, + "nauc_cv_recall_at_1_diff1": 0.27867587872283334, + "nauc_cv_recall_at_1_max": 0.7182720686879527, + "nauc_cv_recall_at_1_std": 0.5737858867722029, + "nauc_cv_recall_at_20_diff1": 0.08256608787867177, + "nauc_cv_recall_at_20_max": 0.3323674089974063, + "nauc_cv_recall_at_20_std": 0.31922010553617747, + "nauc_cv_recall_at_3_diff1": 0.1741239602897772, + "nauc_cv_recall_at_3_max": 0.5191628655755299, + "nauc_cv_recall_at_3_std": 0.419125301851355, + "nauc_cv_recall_at_5_diff1": 0.13886250081581455, + "nauc_cv_recall_at_5_max": 0.43471671295658415, + "nauc_cv_recall_at_5_std": 0.3732079275411715, + "nauc_map_at_1000_diff1": 0.15194835205582552, + "nauc_map_at_1000_max": 0.40877748630514626, + "nauc_map_at_1000_std": 0.3731482014043823, + "nauc_map_at_100_diff1": 0.1513790258426513, + "nauc_map_at_100_max": 0.4168178083848067, + "nauc_map_at_100_std": 0.37797865889115057, + "nauc_map_at_10_diff1": 0.17883793655783767, + "nauc_map_at_10_max": 0.48375802812964486, + "nauc_map_at_10_std": 0.41700647583883665, + "nauc_map_at_1_diff1": 0.27867587872283334, + "nauc_map_at_1_max": 0.7182720686879527, + "nauc_map_at_1_std": 0.5737858867722029, + "nauc_map_at_20_diff1": 0.15818833869847163, + "nauc_map_at_20_max": 0.45599751664213256, + "nauc_map_at_20_std": 0.3962545579279782, + "nauc_map_at_3_diff1": 0.2025670489145016, + "nauc_map_at_3_max": 0.5618430722926705, + "nauc_map_at_3_std": 0.45777872399995345, + "nauc_map_at_5_diff1": 0.18630911748922815, + "nauc_map_at_5_max": 0.5126387924367496, + "nauc_map_at_5_std": 0.4302770226013669, + "nauc_mrr_at_1000_diff1": 0.15194835205582552, + "nauc_mrr_at_1000_max": 0.40877748630514626, + "nauc_mrr_at_1000_std": 0.3731482014043823, + "nauc_mrr_at_100_diff1": 0.1513790258426513, + "nauc_mrr_at_100_max": 0.4168178083848067, + "nauc_mrr_at_100_std": 0.37797865889115057, + "nauc_mrr_at_10_diff1": 0.17883793655783767, + "nauc_mrr_at_10_max": 0.48375802812964486, + "nauc_mrr_at_10_std": 0.41700647583883665, + "nauc_mrr_at_1_diff1": 0.27867587872283334, + "nauc_mrr_at_1_max": 0.7182720686879527, + "nauc_mrr_at_1_std": 0.5737858867722029, + "nauc_mrr_at_20_diff1": 0.15818833869847163, + "nauc_mrr_at_20_max": 0.45599751664213256, + "nauc_mrr_at_20_std": 0.3962545579279782, + "nauc_mrr_at_3_diff1": 0.2025670489145016, + "nauc_mrr_at_3_max": 0.5618430722926705, + "nauc_mrr_at_3_std": 0.45777872399995345, + "nauc_mrr_at_5_diff1": 0.18630911748922815, + "nauc_mrr_at_5_max": 0.5126387924367496, + "nauc_mrr_at_5_std": 0.4302770226013669, + "nauc_ndcg_at_1000_diff1": 0.13663974265837234, + "nauc_ndcg_at_1000_max": 0.27448743546385457, + "nauc_ndcg_at_1000_std": 0.3102772052076174, + "nauc_ndcg_at_100_diff1": 0.12070721039046628, + "nauc_ndcg_at_100_max": 0.30620942623287684, + "nauc_ndcg_at_100_std": 0.3236602531782627, + "nauc_ndcg_at_10_diff1": 0.16174756306693933, + "nauc_ndcg_at_10_max": 0.4415918844721654, + "nauc_ndcg_at_10_std": 0.3945274124410024, + "nauc_ndcg_at_1_diff1": 0.27867587872283334, + "nauc_ndcg_at_1_max": 0.7182720686879527, + "nauc_ndcg_at_1_std": 0.5737858867722029, + "nauc_ndcg_at_20_diff1": 0.11917990049729905, + "nauc_ndcg_at_20_max": 0.39048500817778287, + "nauc_ndcg_at_20_std": 0.3551788561055616, + "nauc_ndcg_at_3_diff1": 0.1922478271939251, + "nauc_ndcg_at_3_max": 0.5459087060547988, + "nauc_ndcg_at_3_std": 0.4436211622584078, + "nauc_ndcg_at_5_diff1": 0.16790385395307483, + "nauc_ndcg_at_5_max": 0.48136065712524156, + "nauc_ndcg_at_5_std": 0.40748577321050217, + "nauc_precision_at_1000_diff1": 0.1463224055326696, + "nauc_precision_at_1000_max": 0.20534263898503682, + "nauc_precision_at_1000_std": 0.289378808337388, + "nauc_precision_at_100_diff1": 0.11011108120551759, + "nauc_precision_at_100_max": 0.24934265204776102, + "nauc_precision_at_100_std": 0.29981869323072124, + "nauc_precision_at_10_diff1": 0.14244559614760016, + "nauc_precision_at_10_max": 0.3921737534068608, + "nauc_precision_at_10_std": 0.3703933835124439, + "nauc_precision_at_1_diff1": 0.27867587872283334, + "nauc_precision_at_1_max": 0.7182720686879527, + "nauc_precision_at_1_std": 0.5737858867722029, + "nauc_precision_at_20_diff1": 0.08256608787867174, + "nauc_precision_at_20_max": 0.3323674089974065, + "nauc_precision_at_20_std": 0.31922010553617763, + "nauc_precision_at_3_diff1": 0.1741239602897773, + "nauc_precision_at_3_max": 0.51916286557553, + "nauc_precision_at_3_std": 0.419125301851355, + "nauc_precision_at_5_diff1": 0.13886250081581444, + "nauc_precision_at_5_max": 0.43471671295658404, + "nauc_precision_at_5_std": 0.3732079275411712, + "nauc_recall_at_1000_diff1": 0.1463224055326704, + "nauc_recall_at_1000_max": 0.20534263898503774, + "nauc_recall_at_1000_std": 0.28937880833738816, + "nauc_recall_at_100_diff1": 0.11011108120551756, + "nauc_recall_at_100_max": 0.24934265204776085, + "nauc_recall_at_100_std": 0.2998186932307211, + "nauc_recall_at_10_diff1": 0.14244559614759988, + "nauc_recall_at_10_max": 0.3921737534068603, + "nauc_recall_at_10_std": 0.3703933835124436, + "nauc_recall_at_1_diff1": 0.27867587872283334, + "nauc_recall_at_1_max": 0.7182720686879527, + "nauc_recall_at_1_std": 0.5737858867722029, + "nauc_recall_at_20_diff1": 0.08256608787867177, + "nauc_recall_at_20_max": 0.3323674089974063, + "nauc_recall_at_20_std": 0.31922010553617747, + "nauc_recall_at_3_diff1": 0.1741239602897772, + "nauc_recall_at_3_max": 0.5191628655755299, + "nauc_recall_at_3_std": 0.419125301851355, + "nauc_recall_at_5_diff1": 0.13886250081581455, + "nauc_recall_at_5_max": 0.43471671295658415, + "nauc_recall_at_5_std": 0.3732079275411715, + "ndcg_at_1": 0.004, + "ndcg_at_10": 0.01454, + "ndcg_at_100": 0.03865, + "ndcg_at_1000": 0.09934, + "ndcg_at_20": 0.0205, + "ndcg_at_3": 0.0089, + "ndcg_at_5": 0.0114, + "precision_at_1": 0.004, + "precision_at_10": 0.00285, + "precision_at_100": 0.00154, + "precision_at_1000": 0.00068, + "precision_at_20": 0.00262, + "precision_at_3": 0.00417, + "precision_at_5": 0.0037, + "recall_at_1": 0.004, + "recall_at_10": 0.0285, + "recall_at_100": 0.1545, + "recall_at_1000": 0.676, + "recall_at_20": 0.0525, + "recall_at_3": 0.0125, + "recall_at_5": 0.0185 + } + ] + }, + "task_name": "XFlickr30kCoT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XM3600T2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XM3600T2IRetrieval.json new file mode 100644 index 0000000000..2ed56d8198 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XM3600T2IRetrieval.json @@ -0,0 +1,6311 @@ +{ + "dataset_revision": "8d3e5665526c55a5855cd6ddfbaba2032bc7cee4", + "evaluation_time": 1117.7640960216522, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.00231, + "cv_recall_at_10": 0.0076, + "cv_recall_at_100": 0.03909, + "cv_recall_at_1000": 0.31682, + "cv_recall_at_20": 0.01167, + "cv_recall_at_3": 0.00421, + "cv_recall_at_5": 0.00611, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.00466, + "map_at_1": 0.00231, + "map_at_10": 0.00375, + "map_at_100": 0.0046, + "map_at_1000": 0.00532, + "map_at_20": 0.00401, + "map_at_3": 0.0031, + "map_at_5": 0.00355, + "mrr_at_1": 0.0023075878919505905, + "mrr_at_10": 0.0037512308212729, + "mrr_at_100": 0.004595156575724333, + "mrr_at_1000": 0.005322854437046951, + "mrr_at_20": 0.004008086271771789, + "mrr_at_3": 0.003099407266639519, + "mrr_at_5": 0.0035473507986063975, + "nauc_cv_recall_at_1000_diff1": 0.02086085561171304, + "nauc_cv_recall_at_1000_max": 0.0659162316936909, + "nauc_cv_recall_at_1000_std": 0.043105470750601366, + "nauc_cv_recall_at_100_diff1": 0.121895728673165, + "nauc_cv_recall_at_100_max": 0.1638896678397787, + "nauc_cv_recall_at_100_std": 0.1526763117197299, + "nauc_cv_recall_at_10_diff1": 0.368207794124593, + "nauc_cv_recall_at_10_max": 0.3689105982750756, + "nauc_cv_recall_at_10_std": 0.33971394675418326, + "nauc_cv_recall_at_1_diff1": 0.6257674859330593, + "nauc_cv_recall_at_1_max": 0.691941523846895, + "nauc_cv_recall_at_1_std": 0.44349508908902135, + "nauc_cv_recall_at_20_diff1": 0.31845657498616187, + "nauc_cv_recall_at_20_max": 0.3064984576105181, + "nauc_cv_recall_at_20_std": 0.30547319280104007, + "nauc_cv_recall_at_3_diff1": 0.3879895446339794, + "nauc_cv_recall_at_3_max": 0.5360946810241998, + "nauc_cv_recall_at_3_std": 0.47829712213481257, + "nauc_cv_recall_at_5_diff1": 0.32946195996894806, + "nauc_cv_recall_at_5_max": 0.41995735231054454, + "nauc_cv_recall_at_5_std": 0.4275124824163301, + "nauc_map_at_1000_diff1": 0.3611149618443007, + "nauc_map_at_1000_max": 0.42087811795849395, + "nauc_map_at_1000_std": 0.33039782609147533, + "nauc_map_at_100_diff1": 0.40341655896005185, + "nauc_map_at_100_max": 0.46470868833845236, + "nauc_map_at_100_std": 0.3648103174068322, + "nauc_map_at_10_diff1": 0.4722863678408031, + "nauc_map_at_10_max": 0.5333535587266557, + "nauc_map_at_10_std": 0.40955214535310597, + "nauc_map_at_1_diff1": 0.6257674859330593, + "nauc_map_at_1_max": 0.691941523846895, + "nauc_map_at_1_std": 0.44349508908902135, + "nauc_map_at_20_diff1": 0.45771386169128625, + "nauc_map_at_20_max": 0.5132111217657302, + "nauc_map_at_20_std": 0.3993673769704931, + "nauc_map_at_3_diff1": 0.5049492976183295, + "nauc_map_at_3_max": 0.6133226050659266, + "nauc_map_at_3_std": 0.4539495984666943, + "nauc_map_at_5_diff1": 0.46805712656168214, + "nauc_map_at_5_max": 0.5564098464493543, + "nauc_map_at_5_std": 0.4345108623452712, + "nauc_mrr_at_1000_diff1": 0.3611149618443007, + "nauc_mrr_at_1000_max": 0.42087811795849395, + "nauc_mrr_at_1000_std": 0.33039782609147533, + "nauc_mrr_at_100_diff1": 0.40341655896005185, + "nauc_mrr_at_100_max": 0.46470868833845236, + "nauc_mrr_at_100_std": 0.3648103174068322, + "nauc_mrr_at_10_diff1": 0.4722863678408031, + "nauc_mrr_at_10_max": 0.5333535587266557, + "nauc_mrr_at_10_std": 0.40955214535310597, + "nauc_mrr_at_1_diff1": 0.6257674859330593, + "nauc_mrr_at_1_max": 0.691941523846895, + "nauc_mrr_at_1_std": 0.44349508908902135, + "nauc_mrr_at_20_diff1": 0.45771386169128625, + "nauc_mrr_at_20_max": 0.5132111217657302, + "nauc_mrr_at_20_std": 0.3993673769704931, + "nauc_mrr_at_3_diff1": 0.5049492976183295, + "nauc_mrr_at_3_max": 0.6133226050659266, + "nauc_mrr_at_3_std": 0.4539495984666943, + "nauc_mrr_at_5_diff1": 0.46805712656168214, + "nauc_mrr_at_5_max": 0.5564098464493543, + "nauc_mrr_at_5_std": 0.4345108623452712, + "nauc_ndcg_at_1000_diff1": 0.08034972935111898, + "nauc_ndcg_at_1000_max": 0.12690504164043506, + "nauc_ndcg_at_1000_std": 0.09785564212119391, + "nauc_ndcg_at_100_diff1": 0.23054253166440147, + "nauc_ndcg_at_100_max": 0.2790563706353781, + "nauc_ndcg_at_100_std": 0.23914173820141837, + "nauc_ndcg_at_10_diff1": 0.43064078578103404, + "nauc_ndcg_at_10_max": 0.47043726798904806, + "nauc_ndcg_at_10_std": 0.3846310774648549, + "nauc_ndcg_at_1_diff1": 0.6257674859330593, + "nauc_ndcg_at_1_max": 0.691941523846895, + "nauc_ndcg_at_1_std": 0.44349508908902135, + "nauc_ndcg_at_20_diff1": 0.39563088229431864, + "nauc_ndcg_at_20_max": 0.42274654860387567, + "nauc_ndcg_at_20_std": 0.3600233392323016, + "nauc_ndcg_at_3_diff1": 0.46834083139804267, + "nauc_ndcg_at_3_max": 0.5891689419024891, + "nauc_ndcg_at_3_std": 0.46133255794941463, + "nauc_ndcg_at_5_diff1": 0.4184780599766308, + "nauc_ndcg_at_5_max": 0.5080341676704876, + "nauc_ndcg_at_5_std": 0.43208940846013444, + "nauc_precision_at_1000_diff1": 0.02086085561171273, + "nauc_precision_at_1000_max": 0.06591623169369042, + "nauc_precision_at_1000_std": 0.04310547075060104, + "nauc_precision_at_100_diff1": 0.1218957286731651, + "nauc_precision_at_100_max": 0.1638896678397788, + "nauc_precision_at_100_std": 0.15267631171972992, + "nauc_precision_at_10_diff1": 0.36820779412459315, + "nauc_precision_at_10_max": 0.36891059827507583, + "nauc_precision_at_10_std": 0.3397139467541833, + "nauc_precision_at_1_diff1": 0.6257674859330593, + "nauc_precision_at_1_max": 0.691941523846895, + "nauc_precision_at_1_std": 0.44349508908902135, + "nauc_precision_at_20_diff1": 0.3184565749861623, + "nauc_precision_at_20_max": 0.3064984576105185, + "nauc_precision_at_20_std": 0.30547319280104024, + "nauc_precision_at_3_diff1": 0.3879895446339794, + "nauc_precision_at_3_max": 0.5360946810241999, + "nauc_precision_at_3_std": 0.4782971221348123, + "nauc_precision_at_5_diff1": 0.32946195996894834, + "nauc_precision_at_5_max": 0.4199573523105448, + "nauc_precision_at_5_std": 0.4275124824163303, + "nauc_recall_at_1000_diff1": 0.02086085561171304, + "nauc_recall_at_1000_max": 0.0659162316936909, + "nauc_recall_at_1000_std": 0.043105470750601366, + "nauc_recall_at_100_diff1": 0.121895728673165, + "nauc_recall_at_100_max": 0.1638896678397787, + "nauc_recall_at_100_std": 0.1526763117197299, + "nauc_recall_at_10_diff1": 0.368207794124593, + "nauc_recall_at_10_max": 0.3689105982750756, + "nauc_recall_at_10_std": 0.33971394675418326, + "nauc_recall_at_1_diff1": 0.6257674859330593, + "nauc_recall_at_1_max": 0.691941523846895, + "nauc_recall_at_1_std": 0.44349508908902135, + "nauc_recall_at_20_diff1": 0.31845657498616187, + "nauc_recall_at_20_max": 0.3064984576105181, + "nauc_recall_at_20_std": 0.30547319280104007, + "nauc_recall_at_3_diff1": 0.3879895446339794, + "nauc_recall_at_3_max": 0.5360946810241998, + "nauc_recall_at_3_std": 0.47829712213481257, + "nauc_recall_at_5_diff1": 0.32946195996894806, + "nauc_recall_at_5_max": 0.41995735231054454, + "nauc_recall_at_5_std": 0.4275124824163301, + "ndcg_at_1": 0.00231, + "ndcg_at_10": 0.00466, + "ndcg_at_100": 0.01051, + "ndcg_at_1000": 0.04216, + "ndcg_at_20": 0.00566, + "ndcg_at_3": 0.00338, + "ndcg_at_5": 0.00418, + "precision_at_1": 0.00231, + "precision_at_10": 0.00076, + "precision_at_100": 0.00039, + "precision_at_1000": 0.00032, + "precision_at_20": 0.00058, + "precision_at_3": 0.0014, + "precision_at_5": 0.00122, + "recall_at_1": 0.00231, + "recall_at_10": 0.0076, + "recall_at_100": 0.03909, + "recall_at_1000": 0.31682, + "recall_at_20": 0.01167, + "recall_at_3": 0.00421, + "recall_at_5": 0.00611 + }, + { + "cv_recall_at_1": 0.00028, + "cv_recall_at_10": 0.00333, + "cv_recall_at_100": 0.02972, + "cv_recall_at_1000": 0.28056, + "cv_recall_at_20": 0.00528, + "cv_recall_at_3": 0.00056, + "cv_recall_at_5": 0.00194, + "hf_subset": "bn", + "languages": [ + "ben-Beng" + ], + "main_score": 0.00141, + "map_at_1": 0.00028, + "map_at_10": 0.00085, + "map_at_100": 0.00146, + "map_at_1000": 0.00208, + "map_at_20": 0.00098, + "map_at_3": 0.00037, + "map_at_5": 0.00069, + "mrr_at_1": 0.0002777777777777778, + "mrr_at_10": 0.000849867724867725, + "mrr_at_100": 0.0014617241741730137, + "mrr_at_1000": 0.002083443888849066, + "mrr_at_20": 0.0009788853969388029, + "mrr_at_3": 0.00037037037037037035, + "mrr_at_5": 0.0006898148148148149, + "nauc_cv_recall_at_1000_diff1": 0.016552384422470947, + "nauc_cv_recall_at_1000_max": 0.10185474461891487, + "nauc_cv_recall_at_1000_std": -0.03476725571443525, + "nauc_cv_recall_at_100_diff1": 0.03830095614472818, + "nauc_cv_recall_at_100_max": 0.10712431887539524, + "nauc_cv_recall_at_100_std": -0.027134650153339247, + "nauc_cv_recall_at_10_diff1": -0.1874161524013952, + "nauc_cv_recall_at_10_max": -0.2595027278418746, + "nauc_cv_recall_at_10_std": -0.029380198551113474, + "nauc_cv_recall_at_1_diff1": -0.2057955460155622, + "nauc_cv_recall_at_1_max": -0.4995975315266971, + "nauc_cv_recall_at_1_std": -0.3184867185403811, + "nauc_cv_recall_at_20_diff1": 0.069747080338356, + "nauc_cv_recall_at_20_max": -0.09340092920791382, + "nauc_cv_recall_at_20_std": 0.01042181520342312, + "nauc_cv_recall_at_3_diff1": 0.22806546820499068, + "nauc_cv_recall_at_3_max": -0.40904212503353915, + "nauc_cv_recall_at_3_std": 0.0026831231553529154, + "nauc_cv_recall_at_5_diff1": -0.0941392924220936, + "nauc_cv_recall_at_5_max": -0.22921537812871332, + "nauc_cv_recall_at_5_std": -0.0891946797500862, + "nauc_map_at_1000_diff1": -0.0202846438557245, + "nauc_map_at_1000_max": -0.06063474992068149, + "nauc_map_at_1000_std": -0.06367760673418538, + "nauc_map_at_100_diff1": -0.027108134687678215, + "nauc_map_at_100_max": -0.11170282021618887, + "nauc_map_at_100_std": -0.07244311982233366, + "nauc_map_at_10_diff1": -0.13723183123677368, + "nauc_map_at_10_max": -0.3065024497854023, + "nauc_map_at_10_std": -0.11363684293498172, + "nauc_map_at_1_diff1": -0.2057955460155622, + "nauc_map_at_1_max": -0.4995975315266971, + "nauc_map_at_1_std": -0.3184867185403811, + "nauc_map_at_20_diff1": -0.057597244728021066, + "nauc_map_at_20_max": -0.23585465837190886, + "nauc_map_at_20_std": -0.08564537656879898, + "nauc_map_at_3_diff1": 0.011134961094714114, + "nauc_map_at_3_max": -0.4543198282801179, + "nauc_map_at_3_std": -0.15790179769251397, + "nauc_map_at_5_diff1": -0.09377965616407036, + "nauc_map_at_5_max": -0.3051161216085054, + "nauc_map_at_5_std": -0.14828307129364351, + "nauc_mrr_at_1000_diff1": -0.0202846438557245, + "nauc_mrr_at_1000_max": -0.06063474992068149, + "nauc_mrr_at_1000_std": -0.06367760673418538, + "nauc_mrr_at_100_diff1": -0.027108134687678215, + "nauc_mrr_at_100_max": -0.11170282021618887, + "nauc_mrr_at_100_std": -0.07244311982233366, + "nauc_mrr_at_10_diff1": -0.13723183123677368, + "nauc_mrr_at_10_max": -0.3065024497854023, + "nauc_mrr_at_10_std": -0.11363684293498172, + "nauc_mrr_at_1_diff1": -0.2057955460155622, + "nauc_mrr_at_1_max": -0.4995975315266971, + "nauc_mrr_at_1_std": -0.3184867185403811, + "nauc_mrr_at_20_diff1": -0.057597244728021066, + "nauc_mrr_at_20_max": -0.23585465837190886, + "nauc_mrr_at_20_std": -0.08564537656879898, + "nauc_mrr_at_3_diff1": 0.011134961094714114, + "nauc_mrr_at_3_max": -0.4543198282801179, + "nauc_mrr_at_3_std": -0.15790179769251397, + "nauc_mrr_at_5_diff1": -0.09377965616407036, + "nauc_mrr_at_5_max": -0.3051161216085054, + "nauc_mrr_at_5_std": -0.14828307129364351, + "nauc_ndcg_at_1000_diff1": 0.01392467040584063, + "nauc_ndcg_at_1000_max": 0.08760122688186933, + "nauc_ndcg_at_1000_std": -0.03651546233738003, + "nauc_ndcg_at_100_diff1": 0.023885256557399884, + "nauc_ndcg_at_100_max": 0.053667739728740935, + "nauc_ndcg_at_100_std": -0.03498338816742055, + "nauc_ndcg_at_10_diff1": -0.16040888984827062, + "nauc_ndcg_at_10_max": -0.27991923638992355, + "nauc_ndcg_at_10_std": -0.06988154388981663, + "nauc_ndcg_at_1_diff1": -0.2057955460155622, + "nauc_ndcg_at_1_max": -0.4995975315266971, + "nauc_ndcg_at_1_std": -0.3184867185403811, + "nauc_ndcg_at_20_diff1": 0.007121822135668618, + "nauc_ndcg_at_20_max": -0.1563107432414096, + "nauc_ndcg_at_20_std": -0.030328522176883817, + "nauc_ndcg_at_3_diff1": 0.08344513013147309, + "nauc_ndcg_at_3_max": -0.43922726053125843, + "nauc_ndcg_at_3_std": -0.1043734907432251, + "nauc_ndcg_at_5_diff1": -0.09195109198698515, + "nauc_ndcg_at_5_max": -0.268777979300934, + "nauc_ndcg_at_5_std": -0.12003272556917093, + "nauc_precision_at_1000_diff1": 0.016552384422470486, + "nauc_precision_at_1000_max": 0.10185474461891456, + "nauc_precision_at_1000_std": -0.03476725571443556, + "nauc_precision_at_100_diff1": 0.038300956144728254, + "nauc_precision_at_100_max": 0.10712431887539543, + "nauc_precision_at_100_std": -0.027134650153339105, + "nauc_precision_at_10_diff1": -0.1874161524013952, + "nauc_precision_at_10_max": -0.25950272784187467, + "nauc_precision_at_10_std": -0.0293801985511136, + "nauc_precision_at_1_diff1": -0.2057955460155622, + "nauc_precision_at_1_max": -0.4995975315266971, + "nauc_precision_at_1_std": -0.3184867185403811, + "nauc_precision_at_20_diff1": 0.06974708033835594, + "nauc_precision_at_20_max": -0.09340092920791385, + "nauc_precision_at_20_std": 0.010421815203423114, + "nauc_precision_at_3_diff1": 0.22806546820499055, + "nauc_precision_at_3_max": -0.409042125033539, + "nauc_precision_at_3_std": 0.0026831231553529475, + "nauc_precision_at_5_diff1": -0.09413929242209361, + "nauc_precision_at_5_max": -0.22921537812871337, + "nauc_precision_at_5_std": -0.08919467975008624, + "nauc_recall_at_1000_diff1": 0.016552384422470947, + "nauc_recall_at_1000_max": 0.10185474461891487, + "nauc_recall_at_1000_std": -0.03476725571443525, + "nauc_recall_at_100_diff1": 0.03830095614472818, + "nauc_recall_at_100_max": 0.10712431887539524, + "nauc_recall_at_100_std": -0.027134650153339247, + "nauc_recall_at_10_diff1": -0.1874161524013952, + "nauc_recall_at_10_max": -0.2595027278418746, + "nauc_recall_at_10_std": -0.029380198551113474, + "nauc_recall_at_1_diff1": -0.2057955460155622, + "nauc_recall_at_1_max": -0.4995975315266971, + "nauc_recall_at_1_std": -0.3184867185403811, + "nauc_recall_at_20_diff1": 0.069747080338356, + "nauc_recall_at_20_max": -0.09340092920791382, + "nauc_recall_at_20_std": 0.01042181520342312, + "nauc_recall_at_3_diff1": 0.22806546820499068, + "nauc_recall_at_3_max": -0.40904212503353915, + "nauc_recall_at_3_std": 0.0026831231553529154, + "nauc_recall_at_5_diff1": -0.0941392924220936, + "nauc_recall_at_5_max": -0.22921537812871332, + "nauc_recall_at_5_std": -0.0891946797500862, + "ndcg_at_1": 0.00028, + "ndcg_at_10": 0.00141, + "ndcg_at_100": 0.00615, + "ndcg_at_1000": 0.03451, + "ndcg_at_20": 0.0019, + "ndcg_at_3": 0.00042, + "ndcg_at_5": 0.00099, + "precision_at_1": 0.00028, + "precision_at_10": 0.00033, + "precision_at_100": 0.0003, + "precision_at_1000": 0.00028, + "precision_at_20": 0.00026, + "precision_at_3": 0.00019, + "precision_at_5": 0.00039, + "recall_at_1": 0.00028, + "recall_at_10": 0.00333, + "recall_at_100": 0.02972, + "recall_at_1000": 0.28056, + "recall_at_20": 0.00528, + "recall_at_3": 0.00056, + "recall_at_5": 0.00194 + }, + { + "cv_recall_at_1": 0.01762, + "cv_recall_at_10": 0.05162, + "cv_recall_at_100": 0.1221, + "cv_recall_at_1000": 0.42098, + "cv_recall_at_20": 0.06771, + "cv_recall_at_3": 0.03122, + "cv_recall_at_5": 0.03857, + "hf_subset": "cs", + "languages": [ + "ces-Latn" + ], + "main_score": 0.03255, + "map_at_1": 0.01762, + "map_at_10": 0.02672, + "map_at_100": 0.02906, + "map_at_1000": 0.02993, + "map_at_20": 0.02784, + "map_at_3": 0.02338, + "map_at_5": 0.02506, + "mrr_at_1": 0.01762175662550298, + "mrr_at_10": 0.026723522765565222, + "mrr_at_100": 0.029056966856710383, + "mrr_at_1000": 0.029929262097212285, + "mrr_at_20": 0.027837486156330166, + "mrr_at_3": 0.023380047176356323, + "mrr_at_5": 0.025058970445400314, + "nauc_cv_recall_at_1000_diff1": 0.130452831771586, + "nauc_cv_recall_at_1000_max": 0.2814459080268943, + "nauc_cv_recall_at_1000_std": 0.3298758134400155, + "nauc_cv_recall_at_100_diff1": 0.20707211860438848, + "nauc_cv_recall_at_100_max": 0.43864028814097267, + "nauc_cv_recall_at_100_std": 0.5045396386605944, + "nauc_cv_recall_at_10_diff1": 0.3018499527889426, + "nauc_cv_recall_at_10_max": 0.5588325128414509, + "nauc_cv_recall_at_10_std": 0.6018154506928893, + "nauc_cv_recall_at_1_diff1": 0.5227812130926891, + "nauc_cv_recall_at_1_max": 0.6728450969099916, + "nauc_cv_recall_at_1_std": 0.5969588549628254, + "nauc_cv_recall_at_20_diff1": 0.25978593693778945, + "nauc_cv_recall_at_20_max": 0.5488017653895927, + "nauc_cv_recall_at_20_std": 0.5966074067105238, + "nauc_cv_recall_at_3_diff1": 0.37876172952921305, + "nauc_cv_recall_at_3_max": 0.6228517843168137, + "nauc_cv_recall_at_3_std": 0.6315853508334851, + "nauc_cv_recall_at_5_diff1": 0.3407870427930846, + "nauc_cv_recall_at_5_max": 0.5891937579864478, + "nauc_cv_recall_at_5_std": 0.6205552352479096, + "nauc_map_at_1000_diff1": 0.3868841569445223, + "nauc_map_at_1000_max": 0.6000870613766743, + "nauc_map_at_1000_std": 0.5952722337740114, + "nauc_map_at_100_diff1": 0.39070161741982057, + "nauc_map_at_100_max": 0.6059801916711482, + "nauc_map_at_100_std": 0.6006444295906668, + "nauc_map_at_10_diff1": 0.4125970705509705, + "nauc_map_at_10_max": 0.6225128813748829, + "nauc_map_at_10_std": 0.609612837662559, + "nauc_map_at_1_diff1": 0.5227812130926891, + "nauc_map_at_1_max": 0.6728450969099916, + "nauc_map_at_1_std": 0.5969588549628254, + "nauc_map_at_20_diff1": 0.4011252779432377, + "nauc_map_at_20_max": 0.6183272754038512, + "nauc_map_at_20_std": 0.6084821498947195, + "nauc_map_at_3_diff1": 0.4438398201237769, + "nauc_map_at_3_max": 0.6437136529111982, + "nauc_map_at_3_std": 0.6152275089611549, + "nauc_map_at_5_diff1": 0.4264761762444746, + "nauc_map_at_5_max": 0.6316489699199355, + "nauc_map_at_5_std": 0.6130112758334513, + "nauc_mrr_at_1000_diff1": 0.3868841569445223, + "nauc_mrr_at_1000_max": 0.6000870613766743, + "nauc_mrr_at_1000_std": 0.5952722337740114, + "nauc_mrr_at_100_diff1": 0.39070161741982057, + "nauc_mrr_at_100_max": 0.6059801916711482, + "nauc_mrr_at_100_std": 0.6006444295906668, + "nauc_mrr_at_10_diff1": 0.4125970705509705, + "nauc_mrr_at_10_max": 0.6225128813748829, + "nauc_mrr_at_10_std": 0.609612837662559, + "nauc_mrr_at_1_diff1": 0.5227812130926891, + "nauc_mrr_at_1_max": 0.6728450969099916, + "nauc_mrr_at_1_std": 0.5969588549628254, + "nauc_mrr_at_20_diff1": 0.4011252779432377, + "nauc_mrr_at_20_max": 0.6183272754038512, + "nauc_mrr_at_20_std": 0.6084821498947195, + "nauc_mrr_at_3_diff1": 0.4438398201237769, + "nauc_mrr_at_3_max": 0.6437136529111982, + "nauc_mrr_at_3_std": 0.6152275089611549, + "nauc_mrr_at_5_diff1": 0.4264761762444746, + "nauc_mrr_at_5_max": 0.6316489699199355, + "nauc_mrr_at_5_std": 0.6130112758334513, + "nauc_ndcg_at_1000_diff1": 0.2525598663486497, + "nauc_ndcg_at_1000_max": 0.445635242025813, + "nauc_ndcg_at_1000_std": 0.474868093318983, + "nauc_ndcg_at_100_diff1": 0.30412139279929973, + "nauc_ndcg_at_100_max": 0.5326396104845605, + "nauc_ndcg_at_100_std": 0.5622182232718004, + "nauc_ndcg_at_10_diff1": 0.3717663380113828, + "nauc_ndcg_at_10_max": 0.5996211807967179, + "nauc_ndcg_at_10_std": 0.6075999987669697, + "nauc_ndcg_at_1_diff1": 0.5227812130926891, + "nauc_ndcg_at_1_max": 0.6728450969099916, + "nauc_ndcg_at_1_std": 0.5969588549628254, + "nauc_ndcg_at_20_diff1": 0.3443968913567033, + "nauc_ndcg_at_20_max": 0.5904537522253336, + "nauc_ndcg_at_20_std": 0.604579390794644, + "nauc_ndcg_at_3_diff1": 0.42347904622269117, + "nauc_ndcg_at_3_max": 0.6371236760291885, + "nauc_ndcg_at_3_std": 0.6203194196263287, + "nauc_ndcg_at_5_diff1": 0.3977540111389816, + "nauc_ndcg_at_5_max": 0.6176889358407618, + "nauc_ndcg_at_5_std": 0.615791094672046, + "nauc_precision_at_1000_diff1": 0.13045283177158534, + "nauc_precision_at_1000_max": 0.28144590802689373, + "nauc_precision_at_1000_std": 0.3298758134400153, + "nauc_precision_at_100_diff1": 0.2070721186043888, + "nauc_precision_at_100_max": 0.43864028814097294, + "nauc_precision_at_100_std": 0.5045396386605943, + "nauc_precision_at_10_diff1": 0.3018499527889427, + "nauc_precision_at_10_max": 0.558832512841451, + "nauc_precision_at_10_std": 0.6018154506928893, + "nauc_precision_at_1_diff1": 0.5227812130926891, + "nauc_precision_at_1_max": 0.6728450969099916, + "nauc_precision_at_1_std": 0.5969588549628254, + "nauc_precision_at_20_diff1": 0.2597859369377895, + "nauc_precision_at_20_max": 0.5488017653895926, + "nauc_precision_at_20_std": 0.596607406710524, + "nauc_precision_at_3_diff1": 0.3787617295292129, + "nauc_precision_at_3_max": 0.6228517843168137, + "nauc_precision_at_3_std": 0.6315853508334855, + "nauc_precision_at_5_diff1": 0.34078704279308475, + "nauc_precision_at_5_max": 0.5891937579864481, + "nauc_precision_at_5_std": 0.6205552352479098, + "nauc_recall_at_1000_diff1": 0.130452831771586, + "nauc_recall_at_1000_max": 0.2814459080268943, + "nauc_recall_at_1000_std": 0.3298758134400155, + "nauc_recall_at_100_diff1": 0.20707211860438848, + "nauc_recall_at_100_max": 0.43864028814097267, + "nauc_recall_at_100_std": 0.5045396386605944, + "nauc_recall_at_10_diff1": 0.3018499527889426, + "nauc_recall_at_10_max": 0.5588325128414509, + "nauc_recall_at_10_std": 0.6018154506928893, + "nauc_recall_at_1_diff1": 0.5227812130926891, + "nauc_recall_at_1_max": 0.6728450969099916, + "nauc_recall_at_1_std": 0.5969588549628254, + "nauc_recall_at_20_diff1": 0.25978593693778945, + "nauc_recall_at_20_max": 0.5488017653895927, + "nauc_recall_at_20_std": 0.5966074067105238, + "nauc_recall_at_3_diff1": 0.37876172952921305, + "nauc_recall_at_3_max": 0.6228517843168137, + "nauc_recall_at_3_std": 0.6315853508334851, + "nauc_recall_at_5_diff1": 0.3407870427930846, + "nauc_recall_at_5_max": 0.5891937579864478, + "nauc_recall_at_5_std": 0.6205552352479096, + "ndcg_at_1": 0.01762, + "ndcg_at_10": 0.03255, + "ndcg_at_100": 0.04634, + "ndcg_at_1000": 0.08096, + "ndcg_at_20": 0.03662, + "ndcg_at_3": 0.02538, + "ndcg_at_5": 0.02841, + "precision_at_1": 0.01762, + "precision_at_10": 0.00516, + "precision_at_100": 0.00122, + "precision_at_1000": 0.00042, + "precision_at_20": 0.00339, + "precision_at_3": 0.01041, + "precision_at_5": 0.00771, + "recall_at_1": 0.01762, + "recall_at_10": 0.05162, + "recall_at_100": 0.1221, + "recall_at_1000": 0.42098, + "recall_at_20": 0.06771, + "recall_at_3": 0.03122, + "recall_at_5": 0.03857 + }, + { + "cv_recall_at_1": 0.03373, + "cv_recall_at_10": 0.12252, + "cv_recall_at_100": 0.28621, + "cv_recall_at_1000": 0.64262, + "cv_recall_at_20": 0.1579, + "cv_recall_at_3": 0.06814, + "cv_recall_at_5": 0.08783, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.07269, + "map_at_1": 0.03373, + "map_at_10": 0.05742, + "map_at_100": 0.06286, + "map_at_1000": 0.06404, + "map_at_20": 0.05987, + "map_at_3": 0.04825, + "map_at_5": 0.05279, + "mrr_at_1": 0.03372797356828194, + "mrr_at_10": 0.057418679200755156, + "mrr_at_100": 0.0628598667569856, + "mrr_at_1000": 0.06403826011730678, + "mrr_at_20": 0.05987001474381017, + "mrr_at_3": 0.04825165198237881, + "mrr_at_5": 0.05279460352422895, + "nauc_cv_recall_at_1000_diff1": 0.11049894267922444, + "nauc_cv_recall_at_1000_max": 0.22451665754549788, + "nauc_cv_recall_at_1000_std": 0.48731589245366713, + "nauc_cv_recall_at_100_diff1": 0.12244855581388936, + "nauc_cv_recall_at_100_max": 0.23170326856395815, + "nauc_cv_recall_at_100_std": 0.4746093691675264, + "nauc_cv_recall_at_10_diff1": 0.17701327797996758, + "nauc_cv_recall_at_10_max": 0.3015191441210729, + "nauc_cv_recall_at_10_std": 0.4539356653217957, + "nauc_cv_recall_at_1_diff1": 0.3855984094356238, + "nauc_cv_recall_at_1_max": 0.4732925626042209, + "nauc_cv_recall_at_1_std": 0.4691654722763578, + "nauc_cv_recall_at_20_diff1": 0.15264152564379713, + "nauc_cv_recall_at_20_max": 0.28317527597564235, + "nauc_cv_recall_at_20_std": 0.45334028568180784, + "nauc_cv_recall_at_3_diff1": 0.24145222405286212, + "nauc_cv_recall_at_3_max": 0.39572480799685716, + "nauc_cv_recall_at_3_std": 0.4564294029992678, + "nauc_cv_recall_at_5_diff1": 0.2013629050318432, + "nauc_cv_recall_at_5_max": 0.33924257352135545, + "nauc_cv_recall_at_5_std": 0.4370634014365294, + "nauc_map_at_1000_diff1": 0.2571905858974858, + "nauc_map_at_1000_max": 0.375801399606642, + "nauc_map_at_1000_std": 0.4583349631543094, + "nauc_map_at_100_diff1": 0.25762509405172734, + "nauc_map_at_100_max": 0.37631945451142995, + "nauc_map_at_100_std": 0.4581332450756916, + "nauc_map_at_10_diff1": 0.2690954142123744, + "nauc_map_at_10_max": 0.38870643006079597, + "nauc_map_at_10_std": 0.45732740396293864, + "nauc_map_at_1_diff1": 0.3855984094356238, + "nauc_map_at_1_max": 0.4732925626042209, + "nauc_map_at_1_std": 0.4691654722763578, + "nauc_map_at_20_diff1": 0.2626524057901557, + "nauc_map_at_20_max": 0.3830007580042664, + "nauc_map_at_20_std": 0.4572309000202074, + "nauc_map_at_3_diff1": 0.3014548144513771, + "nauc_map_at_3_max": 0.42829192851781556, + "nauc_map_at_3_std": 0.4630579329096384, + "nauc_map_at_5_diff1": 0.28178452153706796, + "nauc_map_at_5_max": 0.40429588937500843, + "nauc_map_at_5_std": 0.4551840498859357, + "nauc_mrr_at_1000_diff1": 0.2571905858974858, + "nauc_mrr_at_1000_max": 0.375801399606642, + "nauc_mrr_at_1000_std": 0.4583349631543094, + "nauc_mrr_at_100_diff1": 0.25762509405172734, + "nauc_mrr_at_100_max": 0.37631945451142995, + "nauc_mrr_at_100_std": 0.4581332450756916, + "nauc_mrr_at_10_diff1": 0.2690954142123744, + "nauc_mrr_at_10_max": 0.38870643006079597, + "nauc_mrr_at_10_std": 0.45732740396293864, + "nauc_mrr_at_1_diff1": 0.3855984094356238, + "nauc_mrr_at_1_max": 0.4732925626042209, + "nauc_mrr_at_1_std": 0.4691654722763578, + "nauc_mrr_at_20_diff1": 0.2626524057901557, + "nauc_mrr_at_20_max": 0.3830007580042664, + "nauc_mrr_at_20_std": 0.4572309000202074, + "nauc_mrr_at_3_diff1": 0.3014548144513771, + "nauc_mrr_at_3_max": 0.42829192851781556, + "nauc_mrr_at_3_std": 0.4630579329096384, + "nauc_mrr_at_5_diff1": 0.28178452153706796, + "nauc_mrr_at_5_max": 0.40429588937500843, + "nauc_mrr_at_5_std": 0.4551840498859357, + "nauc_ndcg_at_1000_diff1": 0.19627469020943997, + "nauc_ndcg_at_1000_max": 0.31360426152518983, + "nauc_ndcg_at_1000_std": 0.4658878490099195, + "nauc_ndcg_at_100_diff1": 0.19799771617987885, + "nauc_ndcg_at_100_max": 0.31445442190760137, + "nauc_ndcg_at_100_std": 0.46270076157765944, + "nauc_ndcg_at_10_diff1": 0.23346397488496937, + "nauc_ndcg_at_10_max": 0.3556214902489086, + "nauc_ndcg_at_10_std": 0.455413729729024, + "nauc_ndcg_at_1_diff1": 0.3855984094356238, + "nauc_ndcg_at_1_max": 0.4732925626042209, + "nauc_ndcg_at_1_std": 0.4691654722763578, + "nauc_ndcg_at_20_diff1": 0.21803715738452054, + "nauc_ndcg_at_20_max": 0.34277709628744024, + "nauc_ndcg_at_20_std": 0.4551131267039593, + "nauc_ndcg_at_3_diff1": 0.28193549803328094, + "nauc_ndcg_at_3_max": 0.41770762655518223, + "nauc_ndcg_at_3_std": 0.4609494373853862, + "nauc_ndcg_at_5_diff1": 0.2535360527632714, + "nauc_ndcg_at_5_max": 0.3817724923731276, + "nauc_ndcg_at_5_std": 0.4490004569904931, + "nauc_precision_at_1000_diff1": 0.11049894267922375, + "nauc_precision_at_1000_max": 0.2245166575454968, + "nauc_precision_at_1000_std": 0.4873158924536665, + "nauc_precision_at_100_diff1": 0.12244855581388976, + "nauc_precision_at_100_max": 0.2317032685639583, + "nauc_precision_at_100_std": 0.47460936916752666, + "nauc_precision_at_10_diff1": 0.17701327797996744, + "nauc_precision_at_10_max": 0.30151914412107284, + "nauc_precision_at_10_std": 0.4539356653217955, + "nauc_precision_at_1_diff1": 0.3855984094356238, + "nauc_precision_at_1_max": 0.4732925626042209, + "nauc_precision_at_1_std": 0.4691654722763578, + "nauc_precision_at_20_diff1": 0.15264152564379704, + "nauc_precision_at_20_max": 0.2831752759756425, + "nauc_precision_at_20_std": 0.453340285681808, + "nauc_precision_at_3_diff1": 0.24145222405286218, + "nauc_precision_at_3_max": 0.3957248079968571, + "nauc_precision_at_3_std": 0.4564294029992675, + "nauc_precision_at_5_diff1": 0.20136290503184318, + "nauc_precision_at_5_max": 0.3392425735213555, + "nauc_precision_at_5_std": 0.4370634014365295, + "nauc_recall_at_1000_diff1": 0.11049894267922444, + "nauc_recall_at_1000_max": 0.22451665754549788, + "nauc_recall_at_1000_std": 0.48731589245366713, + "nauc_recall_at_100_diff1": 0.12244855581388936, + "nauc_recall_at_100_max": 0.23170326856395815, + "nauc_recall_at_100_std": 0.4746093691675264, + "nauc_recall_at_10_diff1": 0.17701327797996758, + "nauc_recall_at_10_max": 0.3015191441210729, + "nauc_recall_at_10_std": 0.4539356653217957, + "nauc_recall_at_1_diff1": 0.3855984094356238, + "nauc_recall_at_1_max": 0.4732925626042209, + "nauc_recall_at_1_std": 0.4691654722763578, + "nauc_recall_at_20_diff1": 0.15264152564379713, + "nauc_recall_at_20_max": 0.28317527597564235, + "nauc_recall_at_20_std": 0.45334028568180784, + "nauc_recall_at_3_diff1": 0.24145222405286212, + "nauc_recall_at_3_max": 0.39572480799685716, + "nauc_recall_at_3_std": 0.4564294029992678, + "nauc_recall_at_5_diff1": 0.2013629050318432, + "nauc_recall_at_5_max": 0.33924257352135545, + "nauc_recall_at_5_std": 0.4370634014365294, + "ndcg_at_1": 0.03373, + "ndcg_at_10": 0.07269, + "ndcg_at_100": 0.10483, + "ndcg_at_1000": 0.14699, + "ndcg_at_20": 0.08163, + "ndcg_at_3": 0.05333, + "ndcg_at_5": 0.06148, + "precision_at_1": 0.03373, + "precision_at_10": 0.01225, + "precision_at_100": 0.00286, + "precision_at_1000": 0.00064, + "precision_at_20": 0.0079, + "precision_at_3": 0.02271, + "precision_at_5": 0.01757, + "recall_at_1": 0.03373, + "recall_at_10": 0.12252, + "recall_at_100": 0.28621, + "recall_at_1000": 0.64262, + "recall_at_20": 0.1579, + "recall_at_3": 0.06814, + "recall_at_5": 0.08783 + }, + { + "cv_recall_at_1": 0.0995, + "cv_recall_at_10": 0.28694, + "cv_recall_at_100": 0.56878, + "cv_recall_at_1000": 0.86683, + "cv_recall_at_20": 0.36307, + "cv_recall_at_3": 0.17691, + "cv_recall_at_5": 0.22319, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.184, + "map_at_1": 0.0995, + "map_at_10": 0.15218, + "map_at_100": 0.16243, + "map_at_1000": 0.16364, + "map_at_20": 0.1574, + "map_at_3": 0.13311, + "map_at_5": 0.14368, + "mrr_at_1": 0.09950248756218906, + "mrr_at_10": 0.1521787518663603, + "mrr_at_100": 0.16243341041355344, + "mrr_at_1000": 0.16363593463741677, + "mrr_at_20": 0.15740114488028334, + "mrr_at_3": 0.13311350225616123, + "mrr_at_5": 0.1436769640171242, + "nauc_cv_recall_at_1000_diff1": 0.20657275023501265, + "nauc_cv_recall_at_1000_max": 0.5421799124337883, + "nauc_cv_recall_at_1000_std": 0.688794162507746, + "nauc_cv_recall_at_100_diff1": 0.18604652154929796, + "nauc_cv_recall_at_100_max": 0.37685297250936006, + "nauc_cv_recall_at_100_std": 0.4528221311140118, + "nauc_cv_recall_at_10_diff1": 0.22549313028612575, + "nauc_cv_recall_at_10_max": 0.3161762107844544, + "nauc_cv_recall_at_10_std": 0.26299428936782887, + "nauc_cv_recall_at_1_diff1": 0.3948317473041426, + "nauc_cv_recall_at_1_max": 0.375502342242583, + "nauc_cv_recall_at_1_std": 0.2023848866968725, + "nauc_cv_recall_at_20_diff1": 0.22005941352512404, + "nauc_cv_recall_at_20_max": 0.33182507120198707, + "nauc_cv_recall_at_20_std": 0.3187275546938682, + "nauc_cv_recall_at_3_diff1": 0.27936582648185887, + "nauc_cv_recall_at_3_max": 0.3486223843295035, + "nauc_cv_recall_at_3_std": 0.23765876004992492, + "nauc_cv_recall_at_5_diff1": 0.24286332116512424, + "nauc_cv_recall_at_5_max": 0.32743243092200136, + "nauc_cv_recall_at_5_std": 0.2431073992347361, + "nauc_map_at_1000_diff1": 0.3120378155628561, + "nauc_map_at_1000_max": 0.3549682164024318, + "nauc_map_at_1000_std": 0.2387843964547122, + "nauc_map_at_100_diff1": 0.3116763503762316, + "nauc_map_at_100_max": 0.3545353494643112, + "nauc_map_at_100_std": 0.23852763645291886, + "nauc_map_at_10_diff1": 0.3142076734004048, + "nauc_map_at_10_max": 0.3517818416606607, + "nauc_map_at_10_std": 0.22774463525698088, + "nauc_map_at_1_diff1": 0.3948317473041426, + "nauc_map_at_1_max": 0.375502342242583, + "nauc_map_at_1_std": 0.2023848866968725, + "nauc_map_at_20_diff1": 0.3127906116286042, + "nauc_map_at_20_max": 0.35330490228525435, + "nauc_map_at_20_std": 0.23389994953259716, + "nauc_map_at_3_diff1": 0.3335696361095251, + "nauc_map_at_3_max": 0.36261707635010537, + "nauc_map_at_3_std": 0.22179461625406885, + "nauc_map_at_5_diff1": 0.3192034566172024, + "nauc_map_at_5_max": 0.3546817463773264, + "nauc_map_at_5_std": 0.22391567292597164, + "nauc_mrr_at_1000_diff1": 0.3120378155628561, + "nauc_mrr_at_1000_max": 0.3549682164024318, + "nauc_mrr_at_1000_std": 0.2387843964547122, + "nauc_mrr_at_100_diff1": 0.3116763503762316, + "nauc_mrr_at_100_max": 0.3545353494643112, + "nauc_mrr_at_100_std": 0.23852763645291886, + "nauc_mrr_at_10_diff1": 0.3142076734004048, + "nauc_mrr_at_10_max": 0.3517818416606607, + "nauc_mrr_at_10_std": 0.22774463525698088, + "nauc_mrr_at_1_diff1": 0.3948317473041426, + "nauc_mrr_at_1_max": 0.375502342242583, + "nauc_mrr_at_1_std": 0.2023848866968725, + "nauc_mrr_at_20_diff1": 0.3127906116286042, + "nauc_mrr_at_20_max": 0.35330490228525435, + "nauc_mrr_at_20_std": 0.23389994953259716, + "nauc_mrr_at_3_diff1": 0.3335696361095251, + "nauc_mrr_at_3_max": 0.36261707635010537, + "nauc_mrr_at_3_std": 0.22179461625406885, + "nauc_mrr_at_5_diff1": 0.3192034566172024, + "nauc_mrr_at_5_max": 0.3546817463773264, + "nauc_mrr_at_5_std": 0.22391567292597164, + "nauc_ndcg_at_1000_diff1": 0.28551352733572516, + "nauc_ndcg_at_1000_max": 0.3656732372768745, + "nauc_ndcg_at_1000_std": 0.2905152999979401, + "nauc_ndcg_at_100_diff1": 0.2752630378426891, + "nauc_ndcg_at_100_max": 0.35575148947476287, + "nauc_ndcg_at_100_std": 0.2889703869897625, + "nauc_ndcg_at_10_diff1": 0.2855811374048819, + "nauc_ndcg_at_10_max": 0.34051231293075496, + "nauc_ndcg_at_10_std": 0.2386260521492016, + "nauc_ndcg_at_1_diff1": 0.3948317473041426, + "nauc_ndcg_at_1_max": 0.375502342242583, + "nauc_ndcg_at_1_std": 0.2023848866968725, + "nauc_ndcg_at_20_diff1": 0.2822639305782844, + "nauc_ndcg_at_20_max": 0.3455752287482097, + "nauc_ndcg_at_20_std": 0.258428457487018, + "nauc_ndcg_at_3_diff1": 0.31738521509897943, + "nauc_ndcg_at_3_max": 0.3584920009725775, + "nauc_ndcg_at_3_std": 0.22655842034674972, + "nauc_ndcg_at_5_diff1": 0.29541169832324277, + "nauc_ndcg_at_5_max": 0.3463061961496835, + "nauc_ndcg_at_5_std": 0.22992848838017527, + "nauc_precision_at_1000_diff1": 0.20657275023500984, + "nauc_precision_at_1000_max": 0.5421799124337869, + "nauc_precision_at_1000_std": 0.688794162507745, + "nauc_precision_at_100_diff1": 0.1860465215492983, + "nauc_precision_at_100_max": 0.37685297250936, + "nauc_precision_at_100_std": 0.4528221311140119, + "nauc_precision_at_10_diff1": 0.2254931302861256, + "nauc_precision_at_10_max": 0.3161762107844541, + "nauc_precision_at_10_std": 0.2629942893678289, + "nauc_precision_at_1_diff1": 0.3948317473041426, + "nauc_precision_at_1_max": 0.375502342242583, + "nauc_precision_at_1_std": 0.2023848866968725, + "nauc_precision_at_20_diff1": 0.22005941352512381, + "nauc_precision_at_20_max": 0.33182507120198695, + "nauc_precision_at_20_std": 0.31872755469386793, + "nauc_precision_at_3_diff1": 0.27936582648185876, + "nauc_precision_at_3_max": 0.3486223843295034, + "nauc_precision_at_3_std": 0.2376587600499251, + "nauc_precision_at_5_diff1": 0.24286332116512424, + "nauc_precision_at_5_max": 0.3274324309220014, + "nauc_precision_at_5_std": 0.24310739923473612, + "nauc_recall_at_1000_diff1": 0.20657275023501265, + "nauc_recall_at_1000_max": 0.5421799124337883, + "nauc_recall_at_1000_std": 0.688794162507746, + "nauc_recall_at_100_diff1": 0.18604652154929796, + "nauc_recall_at_100_max": 0.37685297250936006, + "nauc_recall_at_100_std": 0.4528221311140118, + "nauc_recall_at_10_diff1": 0.22549313028612575, + "nauc_recall_at_10_max": 0.3161762107844544, + "nauc_recall_at_10_std": 0.26299428936782887, + "nauc_recall_at_1_diff1": 0.3948317473041426, + "nauc_recall_at_1_max": 0.375502342242583, + "nauc_recall_at_1_std": 0.2023848866968725, + "nauc_recall_at_20_diff1": 0.22005941352512404, + "nauc_recall_at_20_max": 0.33182507120198707, + "nauc_recall_at_20_std": 0.3187275546938682, + "nauc_recall_at_3_diff1": 0.27936582648185887, + "nauc_recall_at_3_max": 0.3486223843295035, + "nauc_recall_at_3_std": 0.23765876004992492, + "nauc_recall_at_5_diff1": 0.24286332116512424, + "nauc_recall_at_5_max": 0.32743243092200136, + "nauc_recall_at_5_std": 0.2431073992347361, + "ndcg_at_1": 0.0995, + "ndcg_at_10": 0.184, + "ndcg_at_100": 0.24082, + "ndcg_at_1000": 0.2774, + "ndcg_at_20": 0.20317, + "ndcg_at_3": 0.14434, + "ndcg_at_5": 0.16339, + "precision_at_1": 0.0995, + "precision_at_10": 0.02869, + "precision_at_100": 0.00569, + "precision_at_1000": 0.00087, + "precision_at_20": 0.01815, + "precision_at_3": 0.05897, + "precision_at_5": 0.04464, + "recall_at_1": 0.0995, + "recall_at_10": 0.28694, + "recall_at_100": 0.56878, + "recall_at_1000": 0.86683, + "recall_at_20": 0.36307, + "recall_at_3": 0.17691, + "recall_at_5": 0.22319 + }, + { + "cv_recall_at_1": 0.00083, + "cv_recall_at_10": 0.00514, + "cv_recall_at_100": 0.03123, + "cv_recall_at_1000": 0.28415, + "cv_recall_at_20": 0.00805, + "cv_recall_at_3": 0.00236, + "cv_recall_at_5": 0.00292, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.00267, + "map_at_1": 0.00083, + "map_at_10": 0.00193, + "map_at_100": 0.00259, + "map_at_1000": 0.00325, + "map_at_20": 0.00212, + "map_at_3": 0.0015, + "map_at_5": 0.00164, + "mrr_at_1": 0.000832870627429206, + "mrr_at_10": 0.0019262887461110668, + "mrr_at_100": 0.0025882539691765576, + "mrr_at_1000": 0.003250035238441143, + "mrr_at_20": 0.002115760796454829, + "mrr_at_3": 0.001503794188413844, + "mrr_at_5": 0.001635665371090135, + "nauc_cv_recall_at_1000_diff1": -0.007626276133075307, + "nauc_cv_recall_at_1000_max": 0.012534558684651335, + "nauc_cv_recall_at_1000_std": 0.04417646187985337, + "nauc_cv_recall_at_100_diff1": 0.00847795729585559, + "nauc_cv_recall_at_100_max": 0.017779803226628355, + "nauc_cv_recall_at_100_std": 0.15477340945105372, + "nauc_cv_recall_at_10_diff1": 0.07691737696626728, + "nauc_cv_recall_at_10_max": 0.17591848148934253, + "nauc_cv_recall_at_10_std": 0.5213689918625686, + "nauc_cv_recall_at_1_diff1": 0.49849303801480216, + "nauc_cv_recall_at_1_max": 0.6149456523898669, + "nauc_cv_recall_at_1_std": 0.7802588913641183, + "nauc_cv_recall_at_20_diff1": 0.05901771001094816, + "nauc_cv_recall_at_20_max": 0.12694926843059573, + "nauc_cv_recall_at_20_std": 0.3765085181438246, + "nauc_cv_recall_at_3_diff1": 0.10624149252697057, + "nauc_cv_recall_at_3_max": 0.34846165139330765, + "nauc_cv_recall_at_3_std": 0.6460134983154191, + "nauc_cv_recall_at_5_diff1": 0.14246132052081384, + "nauc_cv_recall_at_5_max": 0.35420359563849213, + "nauc_cv_recall_at_5_std": 0.6560222993317575, + "nauc_map_at_1000_diff1": 0.11399284167745291, + "nauc_map_at_1000_max": 0.2162294325349823, + "nauc_map_at_1000_std": 0.4174091620453698, + "nauc_map_at_100_diff1": 0.13790384373461875, + "nauc_map_at_100_max": 0.2584907115040497, + "nauc_map_at_100_std": 0.49596050136186354, + "nauc_map_at_10_diff1": 0.18358668703366798, + "nauc_map_at_10_max": 0.3476649659861242, + "nauc_map_at_10_std": 0.6301256971193019, + "nauc_map_at_1_diff1": 0.49849303801480216, + "nauc_map_at_1_max": 0.6149456523898669, + "nauc_map_at_1_std": 0.7802588913641183, + "nauc_map_at_20_diff1": 0.16775456657418533, + "nauc_map_at_20_max": 0.319778921800124, + "nauc_map_at_20_std": 0.5843430754256731, + "nauc_map_at_3_diff1": 0.20590715961779868, + "nauc_map_at_3_max": 0.4189163654652013, + "nauc_map_at_3_std": 0.6778753784305693, + "nauc_map_at_5_diff1": 0.2140425392356912, + "nauc_map_at_5_max": 0.41903410657851026, + "nauc_map_at_5_std": 0.6833802853679313, + "nauc_mrr_at_1000_diff1": 0.11399287345428381, + "nauc_mrr_at_1000_max": 0.21622944086170898, + "nauc_mrr_at_1000_std": 0.4174091508705687, + "nauc_mrr_at_100_diff1": 0.13790384373461875, + "nauc_mrr_at_100_max": 0.2584907115040497, + "nauc_mrr_at_100_std": 0.49596050136186354, + "nauc_mrr_at_10_diff1": 0.18358668703366798, + "nauc_mrr_at_10_max": 0.3476649659861242, + "nauc_mrr_at_10_std": 0.6301256971193019, + "nauc_mrr_at_1_diff1": 0.49849303801480216, + "nauc_mrr_at_1_max": 0.6149456523898669, + "nauc_mrr_at_1_std": 0.7802588913641183, + "nauc_mrr_at_20_diff1": 0.16775456657418533, + "nauc_mrr_at_20_max": 0.319778921800124, + "nauc_mrr_at_20_std": 0.5843430754256731, + "nauc_mrr_at_3_diff1": 0.20590715961779868, + "nauc_mrr_at_3_max": 0.4189163654652013, + "nauc_mrr_at_3_std": 0.6778753784305693, + "nauc_mrr_at_5_diff1": 0.2140425392356912, + "nauc_mrr_at_5_max": 0.41903410657851026, + "nauc_mrr_at_5_std": 0.6833802853679313, + "nauc_ndcg_at_1000_diff1": 0.006198937508097668, + "nauc_ndcg_at_1000_max": 0.033653742379016145, + "nauc_ndcg_at_1000_std": 0.09181517437244709, + "nauc_ndcg_at_100_diff1": 0.04743129405503746, + "nauc_ndcg_at_100_max": 0.09152470836559619, + "nauc_ndcg_at_100_std": 0.26640497705006577, + "nauc_ndcg_at_10_diff1": 0.1360284394048584, + "nauc_ndcg_at_10_max": 0.2740083558852659, + "nauc_ndcg_at_10_std": 0.5837315004075809, + "nauc_ndcg_at_1_diff1": 0.49849303801480216, + "nauc_ndcg_at_1_max": 0.6149456523898669, + "nauc_ndcg_at_1_std": 0.7802588913641183, + "nauc_ndcg_at_20_diff1": 0.11145099981068309, + "nauc_ndcg_at_20_max": 0.2240742297112962, + "nauc_ndcg_at_20_std": 0.4852180698957463, + "nauc_ndcg_at_3_diff1": 0.1700577557184439, + "nauc_ndcg_at_3_max": 0.3936978605948217, + "nauc_ndcg_at_3_std": 0.6663133613132409, + "nauc_ndcg_at_5_diff1": 0.18572114707059503, + "nauc_ndcg_at_5_max": 0.3943678490772386, + "nauc_ndcg_at_5_std": 0.6729657879243428, + "nauc_precision_at_1000_diff1": -0.007626276133075511, + "nauc_precision_at_1000_max": 0.012534558684651226, + "nauc_precision_at_1000_std": 0.04417646187985327, + "nauc_precision_at_100_diff1": 0.008477957295855518, + "nauc_precision_at_100_max": 0.01777980322662837, + "nauc_precision_at_100_std": 0.15477340945105367, + "nauc_precision_at_10_diff1": 0.07691737696626709, + "nauc_precision_at_10_max": 0.1759184814893425, + "nauc_precision_at_10_std": 0.5213689918625685, + "nauc_precision_at_1_diff1": 0.49849303801480216, + "nauc_precision_at_1_max": 0.6149456523898669, + "nauc_precision_at_1_std": 0.7802588913641183, + "nauc_precision_at_20_diff1": 0.05901771001094845, + "nauc_precision_at_20_max": 0.12694926843059612, + "nauc_precision_at_20_std": 0.3765085181438252, + "nauc_precision_at_3_diff1": 0.10624149252697063, + "nauc_precision_at_3_max": 0.3484616513933078, + "nauc_precision_at_3_std": 0.6460134983154195, + "nauc_precision_at_5_diff1": 0.14246132052081387, + "nauc_precision_at_5_max": 0.3542035956384923, + "nauc_precision_at_5_std": 0.6560222993317575, + "nauc_recall_at_1000_diff1": -0.007626276133075307, + "nauc_recall_at_1000_max": 0.012534558684651335, + "nauc_recall_at_1000_std": 0.04417646187985337, + "nauc_recall_at_100_diff1": 0.00847795729585559, + "nauc_recall_at_100_max": 0.017779803226628355, + "nauc_recall_at_100_std": 0.15477340945105372, + "nauc_recall_at_10_diff1": 0.07691737696626728, + "nauc_recall_at_10_max": 0.17591848148934253, + "nauc_recall_at_10_std": 0.5213689918625686, + "nauc_recall_at_1_diff1": 0.49849303801480216, + "nauc_recall_at_1_max": 0.6149456523898669, + "nauc_recall_at_1_std": 0.7802588913641183, + "nauc_recall_at_20_diff1": 0.05901771001094816, + "nauc_recall_at_20_max": 0.12694926843059573, + "nauc_recall_at_20_std": 0.3765085181438246, + "nauc_recall_at_3_diff1": 0.10624149252697057, + "nauc_recall_at_3_max": 0.34846165139330765, + "nauc_recall_at_3_std": 0.6460134983154191, + "nauc_recall_at_5_diff1": 0.14246132052081384, + "nauc_recall_at_5_max": 0.35420359563849213, + "nauc_recall_at_5_std": 0.6560222993317575, + "ndcg_at_1": 0.00083, + "ndcg_at_10": 0.00267, + "ndcg_at_100": 0.00744, + "ndcg_at_1000": 0.03624, + "ndcg_at_20": 0.00339, + "ndcg_at_3": 0.00172, + "ndcg_at_5": 0.00196, + "precision_at_1": 0.00083, + "precision_at_10": 0.00051, + "precision_at_100": 0.00031, + "precision_at_1000": 0.00028, + "precision_at_20": 0.0004, + "precision_at_3": 0.00079, + "precision_at_5": 0.00058, + "recall_at_1": 0.00083, + "recall_at_10": 0.00514, + "recall_at_100": 0.03123, + "recall_at_1000": 0.28415, + "recall_at_20": 0.00805, + "recall_at_3": 0.00236, + "recall_at_5": 0.00292 + }, + { + "cv_recall_at_1": 0.39056, + "cv_recall_at_10": 0.76833, + "cv_recall_at_100": 0.95611, + "cv_recall_at_1000": 0.99917, + "cv_recall_at_20": 0.84819, + "cv_recall_at_3": 0.59597, + "cv_recall_at_5": 0.67222, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57352, + "map_at_1": 0.39056, + "map_at_10": 0.51189, + "map_at_100": 0.52037, + "map_at_1000": 0.52062, + "map_at_20": 0.5175, + "map_at_3": 0.48157, + "map_at_5": 0.49905, + "mrr_at_1": 0.39055555555555554, + "mrr_at_10": 0.5118883377425072, + "mrr_at_100": 0.5203659494810378, + "mrr_at_1000": 0.5206184290057001, + "mrr_at_20": 0.5175045521828792, + "mrr_at_3": 0.4815740740740779, + "mrr_at_5": 0.49905324074074237, + "nauc_cv_recall_at_1000_diff1": 0.14394646747577447, + "nauc_cv_recall_at_1000_max": 0.423591658885699, + "nauc_cv_recall_at_1000_std": 0.2710862122626985, + "nauc_cv_recall_at_100_diff1": 0.46606596224987895, + "nauc_cv_recall_at_100_max": 0.46268866196267544, + "nauc_cv_recall_at_100_std": 0.08952800529494662, + "nauc_cv_recall_at_10_diff1": 0.4379063393575716, + "nauc_cv_recall_at_10_max": 0.4216913978135716, + "nauc_cv_recall_at_10_std": -0.06510608864641969, + "nauc_cv_recall_at_1_diff1": 0.5635180937370017, + "nauc_cv_recall_at_1_max": 0.3921407024851276, + "nauc_cv_recall_at_1_std": -0.05834478567184907, + "nauc_cv_recall_at_20_diff1": 0.4242996361711508, + "nauc_cv_recall_at_20_max": 0.4116836539897339, + "nauc_cv_recall_at_20_std": -0.03936486874353415, + "nauc_cv_recall_at_3_diff1": 0.4587991676794355, + "nauc_cv_recall_at_3_max": 0.422630444047248, + "nauc_cv_recall_at_3_std": -0.0698509911455654, + "nauc_cv_recall_at_5_diff1": 0.4454212445561298, + "nauc_cv_recall_at_5_max": 0.41885208732865375, + "nauc_cv_recall_at_5_std": -0.07339352687217181, + "nauc_map_at_1000_diff1": 0.5211156484047063, + "nauc_map_at_1000_max": 0.4059355660415839, + "nauc_map_at_1000_std": -0.06225085095320318, + "nauc_map_at_100_diff1": 0.5210906627423454, + "nauc_map_at_100_max": 0.40600689920338445, + "nauc_map_at_100_std": -0.06217505336158905, + "nauc_map_at_10_diff1": 0.5194607406726276, + "nauc_map_at_10_max": 0.40661441290393446, + "nauc_map_at_10_std": -0.06299795452615677, + "nauc_map_at_1_diff1": 0.5635180937370017, + "nauc_map_at_1_max": 0.3921407024851276, + "nauc_map_at_1_std": -0.05834478567184907, + "nauc_map_at_20_diff1": 0.5202557335360662, + "nauc_map_at_20_max": 0.40603319833528057, + "nauc_map_at_20_std": -0.06228822965887942, + "nauc_map_at_3_diff1": 0.5191755804934863, + "nauc_map_at_3_max": 0.40734379586175073, + "nauc_map_at_3_std": -0.06333142407599146, + "nauc_map_at_5_diff1": 0.5184830281353593, + "nauc_map_at_5_max": 0.40646787276964275, + "nauc_map_at_5_std": -0.06391705205112995, + "nauc_mrr_at_1000_diff1": 0.5211156484047063, + "nauc_mrr_at_1000_max": 0.4059355660415839, + "nauc_mrr_at_1000_std": -0.06225085095320318, + "nauc_mrr_at_100_diff1": 0.5210906627423454, + "nauc_mrr_at_100_max": 0.40600689920338445, + "nauc_mrr_at_100_std": -0.06217505336158905, + "nauc_mrr_at_10_diff1": 0.5194607406726276, + "nauc_mrr_at_10_max": 0.40661441290393446, + "nauc_mrr_at_10_std": -0.06299795452615677, + "nauc_mrr_at_1_diff1": 0.5635180937370017, + "nauc_mrr_at_1_max": 0.3921407024851276, + "nauc_mrr_at_1_std": -0.05834478567184907, + "nauc_mrr_at_20_diff1": 0.5202557335360662, + "nauc_mrr_at_20_max": 0.40603319833528057, + "nauc_mrr_at_20_std": -0.06228822965887942, + "nauc_mrr_at_3_diff1": 0.5191755804934863, + "nauc_mrr_at_3_max": 0.40734379586175073, + "nauc_mrr_at_3_std": -0.06333142407599146, + "nauc_mrr_at_5_diff1": 0.5184830281353593, + "nauc_mrr_at_5_max": 0.40646787276964275, + "nauc_mrr_at_5_std": -0.06391705205112995, + "nauc_ndcg_at_1000_diff1": 0.5142678476811551, + "nauc_ndcg_at_1000_max": 0.4071623435820476, + "nauc_ndcg_at_1000_std": -0.0612249688199367, + "nauc_ndcg_at_100_diff1": 0.5134288832740771, + "nauc_ndcg_at_100_max": 0.4085381492895844, + "nauc_ndcg_at_100_std": -0.058269328250549, + "nauc_ndcg_at_10_diff1": 0.5042774894693469, + "nauc_ndcg_at_10_max": 0.40968828510708827, + "nauc_ndcg_at_10_std": -0.06365206462233498, + "nauc_ndcg_at_1_diff1": 0.5635180937370017, + "nauc_ndcg_at_1_max": 0.3921407024851276, + "nauc_ndcg_at_1_std": -0.05834478567184907, + "nauc_ndcg_at_20_diff1": 0.5064832037799302, + "nauc_ndcg_at_20_max": 0.40761011384041623, + "nauc_ndcg_at_20_std": -0.06039784256595193, + "nauc_ndcg_at_3_diff1": 0.5045191285336305, + "nauc_ndcg_at_3_max": 0.4111560098071898, + "nauc_ndcg_at_3_std": -0.06491911626024735, + "nauc_ndcg_at_5_diff1": 0.502323225972798, + "nauc_ndcg_at_5_max": 0.40944904239400765, + "nauc_ndcg_at_5_std": -0.06600665556081452, + "nauc_precision_at_1000_diff1": 0.143946467475703, + "nauc_precision_at_1000_max": 0.4235916588856638, + "nauc_precision_at_1000_std": 0.27108621226244745, + "nauc_precision_at_100_diff1": 0.4660659622498764, + "nauc_precision_at_100_max": 0.46268866196267266, + "nauc_precision_at_100_std": 0.08952800529494184, + "nauc_precision_at_10_diff1": 0.43790633935757056, + "nauc_precision_at_10_max": 0.421691397813571, + "nauc_precision_at_10_std": -0.06510608864641966, + "nauc_precision_at_1_diff1": 0.5635180937370017, + "nauc_precision_at_1_max": 0.3921407024851276, + "nauc_precision_at_1_std": -0.05834478567184907, + "nauc_precision_at_20_diff1": 0.424299636171151, + "nauc_precision_at_20_max": 0.4116836539897328, + "nauc_precision_at_20_std": -0.03936486874353411, + "nauc_precision_at_3_diff1": 0.4587991676794361, + "nauc_precision_at_3_max": 0.4226304440472487, + "nauc_precision_at_3_std": -0.06985099114556456, + "nauc_precision_at_5_diff1": 0.44542124455612886, + "nauc_precision_at_5_max": 0.4188520873286538, + "nauc_precision_at_5_std": -0.07339352687217225, + "nauc_recall_at_1000_diff1": 0.14394646747577447, + "nauc_recall_at_1000_max": 0.423591658885699, + "nauc_recall_at_1000_std": 0.2710862122626985, + "nauc_recall_at_100_diff1": 0.46606596224987895, + "nauc_recall_at_100_max": 0.46268866196267544, + "nauc_recall_at_100_std": 0.08952800529494662, + "nauc_recall_at_10_diff1": 0.4379063393575716, + "nauc_recall_at_10_max": 0.4216913978135716, + "nauc_recall_at_10_std": -0.06510608864641969, + "nauc_recall_at_1_diff1": 0.5635180937370017, + "nauc_recall_at_1_max": 0.3921407024851276, + "nauc_recall_at_1_std": -0.05834478567184907, + "nauc_recall_at_20_diff1": 0.4242996361711508, + "nauc_recall_at_20_max": 0.4116836539897339, + "nauc_recall_at_20_std": -0.03936486874353415, + "nauc_recall_at_3_diff1": 0.4587991676794355, + "nauc_recall_at_3_max": 0.422630444047248, + "nauc_recall_at_3_std": -0.0698509911455654, + "nauc_recall_at_5_diff1": 0.4454212445561298, + "nauc_recall_at_5_max": 0.41885208732865375, + "nauc_recall_at_5_std": -0.07339352687217181, + "ndcg_at_1": 0.39056, + "ndcg_at_10": 0.57352, + "ndcg_at_100": 0.614, + "ndcg_at_1000": 0.61973, + "ndcg_at_20": 0.5938, + "ndcg_at_3": 0.51098, + "ndcg_at_5": 0.54243, + "precision_at_1": 0.39056, + "precision_at_10": 0.07683, + "precision_at_100": 0.00956, + "precision_at_1000": 0.001, + "precision_at_20": 0.04241, + "precision_at_3": 0.19866, + "precision_at_5": 0.13444, + "recall_at_1": 0.39056, + "recall_at_10": 0.76833, + "recall_at_100": 0.95611, + "recall_at_1000": 0.99917, + "recall_at_20": 0.84819, + "recall_at_3": 0.59597, + "recall_at_5": 0.67222 + }, + { + "cv_recall_at_1": 0.15277, + "cv_recall_at_10": 0.43302, + "cv_recall_at_100": 0.74983, + "cv_recall_at_1000": 0.95774, + "cv_recall_at_20": 0.52624, + "cv_recall_at_3": 0.26759, + "cv_recall_at_5": 0.33237, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.27852, + "map_at_1": 0.15277, + "map_at_10": 0.23092, + "map_at_100": 0.24302, + "map_at_1000": 0.24395, + "map_at_20": 0.2374, + "map_at_3": 0.20283, + "map_at_5": 0.21761, + "mrr_at_1": 0.15277455305316925, + "mrr_at_10": 0.23091626403676438, + "mrr_at_100": 0.24301997050327284, + "mrr_at_1000": 0.24395317098725908, + "mrr_at_20": 0.2374002759347962, + "mrr_at_3": 0.20282872842659158, + "mrr_at_5": 0.21760699636251063, + "nauc_cv_recall_at_1000_diff1": 0.28871615017386665, + "nauc_cv_recall_at_1000_max": 0.4184062793922356, + "nauc_cv_recall_at_1000_std": 0.7481146617611558, + "nauc_cv_recall_at_100_diff1": 0.24780216183232057, + "nauc_cv_recall_at_100_max": 0.2647257211285579, + "nauc_cv_recall_at_100_std": 0.4019682823243577, + "nauc_cv_recall_at_10_diff1": 0.23715268833907996, + "nauc_cv_recall_at_10_max": 0.25727352781077684, + "nauc_cv_recall_at_10_std": 0.1883518291743347, + "nauc_cv_recall_at_1_diff1": 0.3951332983445778, + "nauc_cv_recall_at_1_max": 0.28679458916310435, + "nauc_cv_recall_at_1_std": 0.09567202538938674, + "nauc_cv_recall_at_20_diff1": 0.23858513986687463, + "nauc_cv_recall_at_20_max": 0.2509656308028913, + "nauc_cv_recall_at_20_std": 0.2432511638351338, + "nauc_cv_recall_at_3_diff1": 0.2612068631489378, + "nauc_cv_recall_at_3_max": 0.2554936434215696, + "nauc_cv_recall_at_3_std": 0.11764445499701637, + "nauc_cv_recall_at_5_diff1": 0.24864032378071985, + "nauc_cv_recall_at_5_max": 0.25485945507513963, + "nauc_cv_recall_at_5_std": 0.1447814692586401, + "nauc_map_at_1000_diff1": 0.31803830658466875, + "nauc_map_at_1000_max": 0.270883669496194, + "nauc_map_at_1000_std": 0.12967884875912955, + "nauc_map_at_100_diff1": 0.3177685807545578, + "nauc_map_at_100_max": 0.27073040716077845, + "nauc_map_at_100_std": 0.12997750284027942, + "nauc_map_at_10_diff1": 0.3167483618348145, + "nauc_map_at_10_max": 0.2711432056874237, + "nauc_map_at_10_std": 0.12326759319526, + "nauc_map_at_1_diff1": 0.3951332983445778, + "nauc_map_at_1_max": 0.28679458916310435, + "nauc_map_at_1_std": 0.09567202538938674, + "nauc_map_at_20_diff1": 0.31721758152947716, + "nauc_map_at_20_max": 0.27060117498496455, + "nauc_map_at_20_std": 0.12773963278126685, + "nauc_map_at_3_diff1": 0.324870380303839, + "nauc_map_at_3_max": 0.271729787112544, + "nauc_map_at_3_std": 0.10750958295910185, + "nauc_map_at_5_diff1": 0.319389243314658, + "nauc_map_at_5_max": 0.2711176076197904, + "nauc_map_at_5_std": 0.11561201659260371, + "nauc_mrr_at_1000_diff1": 0.31803830658466875, + "nauc_mrr_at_1000_max": 0.270883669496194, + "nauc_mrr_at_1000_std": 0.12967884875912955, + "nauc_mrr_at_100_diff1": 0.3177685807545578, + "nauc_mrr_at_100_max": 0.27073040716077845, + "nauc_mrr_at_100_std": 0.12997750284027942, + "nauc_mrr_at_10_diff1": 0.3167483618348145, + "nauc_mrr_at_10_max": 0.2711432056874237, + "nauc_mrr_at_10_std": 0.12326759319526, + "nauc_mrr_at_1_diff1": 0.3951332983445778, + "nauc_mrr_at_1_max": 0.28679458916310435, + "nauc_mrr_at_1_std": 0.09567202538938674, + "nauc_mrr_at_20_diff1": 0.31721758152947716, + "nauc_mrr_at_20_max": 0.27060117498496455, + "nauc_mrr_at_20_std": 0.12773963278126685, + "nauc_mrr_at_3_diff1": 0.324870380303839, + "nauc_mrr_at_3_max": 0.271729787112544, + "nauc_mrr_at_3_std": 0.10750958295910185, + "nauc_mrr_at_5_diff1": 0.319389243314658, + "nauc_mrr_at_5_max": 0.2711176076197904, + "nauc_mrr_at_5_std": 0.11561201659260371, + "nauc_ndcg_at_1000_diff1": 0.3049322282875534, + "nauc_ndcg_at_1000_max": 0.27115908501182373, + "nauc_ndcg_at_1000_std": 0.16206072135256805, + "nauc_ndcg_at_100_diff1": 0.2992437168898892, + "nauc_ndcg_at_100_max": 0.2674940082163727, + "nauc_ndcg_at_100_std": 0.1736118577818984, + "nauc_ndcg_at_10_diff1": 0.2930997019668874, + "nauc_ndcg_at_10_max": 0.266813494365646, + "nauc_ndcg_at_10_std": 0.14098806944662287, + "nauc_ndcg_at_1_diff1": 0.3951332983445778, + "nauc_ndcg_at_1_max": 0.28679458916310435, + "nauc_ndcg_at_1_std": 0.09567202538938674, + "nauc_ndcg_at_20_diff1": 0.2943634186381247, + "nauc_ndcg_at_20_max": 0.2650448328834689, + "nauc_ndcg_at_20_std": 0.15633248515225112, + "nauc_ndcg_at_3_diff1": 0.30640418992551965, + "nauc_ndcg_at_3_max": 0.2670751781101042, + "nauc_ndcg_at_3_std": 0.11046182320159958, + "nauc_ndcg_at_5_diff1": 0.29831489333197114, + "nauc_ndcg_at_5_max": 0.2662957662223934, + "nauc_ndcg_at_5_std": 0.12394467057830935, + "nauc_precision_at_1000_diff1": 0.2887161501738626, + "nauc_precision_at_1000_max": 0.4184062793922289, + "nauc_precision_at_1000_std": 0.7481146617611505, + "nauc_precision_at_100_diff1": 0.24780216183231976, + "nauc_precision_at_100_max": 0.26472572112855824, + "nauc_precision_at_100_std": 0.4019682823243572, + "nauc_precision_at_10_diff1": 0.23715268833907965, + "nauc_precision_at_10_max": 0.2572735278107764, + "nauc_precision_at_10_std": 0.1883518291743345, + "nauc_precision_at_1_diff1": 0.3951332983445778, + "nauc_precision_at_1_max": 0.28679458916310435, + "nauc_precision_at_1_std": 0.09567202538938674, + "nauc_precision_at_20_diff1": 0.2385851398668743, + "nauc_precision_at_20_max": 0.25096563080289136, + "nauc_precision_at_20_std": 0.2432511638351335, + "nauc_precision_at_3_diff1": 0.26120686314893776, + "nauc_precision_at_3_max": 0.25549364342156955, + "nauc_precision_at_3_std": 0.11764445499701617, + "nauc_precision_at_5_diff1": 0.2486403237807199, + "nauc_precision_at_5_max": 0.2548594550751396, + "nauc_precision_at_5_std": 0.1447814692586399, + "nauc_recall_at_1000_diff1": 0.28871615017386665, + "nauc_recall_at_1000_max": 0.4184062793922356, + "nauc_recall_at_1000_std": 0.7481146617611558, + "nauc_recall_at_100_diff1": 0.24780216183232057, + "nauc_recall_at_100_max": 0.2647257211285579, + "nauc_recall_at_100_std": 0.4019682823243577, + "nauc_recall_at_10_diff1": 0.23715268833907996, + "nauc_recall_at_10_max": 0.25727352781077684, + "nauc_recall_at_10_std": 0.1883518291743347, + "nauc_recall_at_1_diff1": 0.3951332983445778, + "nauc_recall_at_1_max": 0.28679458916310435, + "nauc_recall_at_1_std": 0.09567202538938674, + "nauc_recall_at_20_diff1": 0.23858513986687463, + "nauc_recall_at_20_max": 0.2509656308028913, + "nauc_recall_at_20_std": 0.2432511638351338, + "nauc_recall_at_3_diff1": 0.2612068631489378, + "nauc_recall_at_3_max": 0.2554936434215696, + "nauc_recall_at_3_std": 0.11764445499701637, + "nauc_recall_at_5_diff1": 0.24864032378071985, + "nauc_recall_at_5_max": 0.25485945507513963, + "nauc_recall_at_5_std": 0.1447814692586401, + "ndcg_at_1": 0.15277, + "ndcg_at_10": 0.27852, + "ndcg_at_100": 0.34335, + "ndcg_at_1000": 0.36944, + "ndcg_at_20": 0.30211, + "ndcg_at_3": 0.21944, + "ndcg_at_5": 0.24609, + "precision_at_1": 0.15277, + "precision_at_10": 0.0433, + "precision_at_100": 0.0075, + "precision_at_1000": 0.00096, + "precision_at_20": 0.02631, + "precision_at_3": 0.0892, + "precision_at_5": 0.06647, + "recall_at_1": 0.15277, + "recall_at_10": 0.43302, + "recall_at_100": 0.74983, + "recall_at_1000": 0.95774, + "recall_at_20": 0.52624, + "recall_at_3": 0.26759, + "recall_at_5": 0.33237 + }, + { + "cv_recall_at_1": 0.00028, + "cv_recall_at_10": 0.00373, + "cv_recall_at_100": 0.03271, + "cv_recall_at_1000": 0.29896, + "cv_recall_at_20": 0.00759, + "cv_recall_at_3": 0.0011, + "cv_recall_at_5": 0.00166, + "hf_subset": "fa", + "languages": [ + "fas-Arab" + ], + "main_score": 0.00162, + "map_at_1": 0.00028, + "map_at_10": 0.001, + "map_at_100": 0.00175, + "map_at_1000": 0.00244, + "map_at_20": 0.00124, + "map_at_3": 0.0006, + "map_at_5": 0.00072, + "mrr_at_1": 0.00027605244996549346, + "mrr_at_10": 0.001000032863386901, + "mrr_at_100": 0.0017508881788116945, + "mrr_at_1000": 0.0024357808783905394, + "mrr_at_20": 0.0012443930416931972, + "mrr_at_3": 0.0005981136415919026, + "mrr_at_5": 0.0007223372440763745, + "nauc_cv_recall_at_1000_diff1": 0.027557638498875554, + "nauc_cv_recall_at_1000_max": 0.03097565502070411, + "nauc_cv_recall_at_1000_std": 0.026461907925695066, + "nauc_cv_recall_at_100_diff1": -0.002745721981439647, + "nauc_cv_recall_at_100_max": 0.06018066655766445, + "nauc_cv_recall_at_100_std": 0.15216477829100694, + "nauc_cv_recall_at_10_diff1": -0.07750927677273792, + "nauc_cv_recall_at_10_max": 0.21309126607147533, + "nauc_cv_recall_at_10_std": 0.3776404599665452, + "nauc_cv_recall_at_1_diff1": 0.5492375204561938, + "nauc_cv_recall_at_1_max": 0.12346348800082675, + "nauc_cv_recall_at_1_std": 0.1717739449950981, + "nauc_cv_recall_at_20_diff1": 0.08359618948954435, + "nauc_cv_recall_at_20_max": 0.1892765672248886, + "nauc_cv_recall_at_20_std": 0.3033865628259771, + "nauc_cv_recall_at_3_diff1": 0.06821325088738682, + "nauc_cv_recall_at_3_max": 0.11353499778022445, + "nauc_cv_recall_at_3_std": 0.2872444300231949, + "nauc_cv_recall_at_5_diff1": -0.0650688349346964, + "nauc_cv_recall_at_5_max": 0.19116218917300562, + "nauc_cv_recall_at_5_std": 0.34639713863337207, + "nauc_map_at_1000_diff1": 0.04255967091580391, + "nauc_map_at_1000_max": 0.11000969859685499, + "nauc_map_at_1000_std": 0.20010807009438789, + "nauc_map_at_100_diff1": 0.04612963512398783, + "nauc_map_at_100_max": 0.13125045693129808, + "nauc_map_at_100_std": 0.25357706080615683, + "nauc_map_at_10_diff1": 0.04690727472110029, + "nauc_map_at_10_max": 0.18274617339818786, + "nauc_map_at_10_std": 0.3173527679515929, + "nauc_map_at_1_diff1": 0.5492375204561938, + "nauc_map_at_1_max": 0.12346348800082675, + "nauc_map_at_1_std": 0.1717739449950981, + "nauc_map_at_20_diff1": 0.08227213092585944, + "nauc_map_at_20_max": 0.17655587928495695, + "nauc_map_at_20_std": 0.3020484016197641, + "nauc_map_at_3_diff1": 0.18614921270224682, + "nauc_map_at_3_max": 0.12106983686463088, + "nauc_map_at_3_std": 0.2558291149549853, + "nauc_map_at_5_diff1": 0.09670003344083386, + "nauc_map_at_5_max": 0.16079189981757416, + "nauc_map_at_5_std": 0.2917500251732401, + "nauc_mrr_at_1000_diff1": 0.042559355600840974, + "nauc_mrr_at_1000_max": 0.11000950541871306, + "nauc_mrr_at_1000_std": 0.20010802605611105, + "nauc_mrr_at_100_diff1": 0.04612963512398783, + "nauc_mrr_at_100_max": 0.13125045693129808, + "nauc_mrr_at_100_std": 0.25357706080615683, + "nauc_mrr_at_10_diff1": 0.04690727472110029, + "nauc_mrr_at_10_max": 0.18274617339818786, + "nauc_mrr_at_10_std": 0.3173527679515929, + "nauc_mrr_at_1_diff1": 0.5492375204561938, + "nauc_mrr_at_1_max": 0.12346348800082675, + "nauc_mrr_at_1_std": 0.1717739449950981, + "nauc_mrr_at_20_diff1": 0.08227213092585944, + "nauc_mrr_at_20_max": 0.17655587928495695, + "nauc_mrr_at_20_std": 0.3020484016197641, + "nauc_mrr_at_3_diff1": 0.18614921270224682, + "nauc_mrr_at_3_max": 0.12106983686463088, + "nauc_mrr_at_3_std": 0.2558291149549853, + "nauc_mrr_at_5_diff1": 0.09670003344083386, + "nauc_mrr_at_5_max": 0.16079189981757416, + "nauc_mrr_at_5_std": 0.2917500251732401, + "nauc_ndcg_at_1000_diff1": 0.026082342552705223, + "nauc_ndcg_at_1000_max": 0.04196522349909082, + "nauc_ndcg_at_1000_std": 0.05286649907971509, + "nauc_ndcg_at_100_diff1": 0.007152590164762058, + "nauc_ndcg_at_100_max": 0.0817202656013075, + "nauc_ndcg_at_100_std": 0.18461686140719014, + "nauc_ndcg_at_10_diff1": -0.018790372369411834, + "nauc_ndcg_at_10_max": 0.19847190477287552, + "nauc_ndcg_at_10_std": 0.3478122953194488, + "nauc_ndcg_at_1_diff1": 0.5492375204561938, + "nauc_ndcg_at_1_max": 0.12346348800082675, + "nauc_ndcg_at_1_std": 0.1717739449950981, + "nauc_ndcg_at_20_diff1": 0.07476889360503366, + "nauc_ndcg_at_20_max": 0.18476999136430186, + "nauc_ndcg_at_20_std": 0.30587666233259075, + "nauc_ndcg_at_3_diff1": 0.14017557586032753, + "nauc_ndcg_at_3_max": 0.11832017157679414, + "nauc_ndcg_at_3_std": 0.26796122915840065, + "nauc_ndcg_at_5_diff1": 0.02805491053532623, + "nauc_ndcg_at_5_max": 0.173039085748848, + "nauc_ndcg_at_5_std": 0.31460287587606217, + "nauc_precision_at_1000_diff1": 0.027557638498875506, + "nauc_precision_at_1000_max": 0.03097565502070383, + "nauc_precision_at_1000_std": 0.026461907925694854, + "nauc_precision_at_100_diff1": -0.0027457219814395932, + "nauc_precision_at_100_max": 0.06018066655766441, + "nauc_precision_at_100_std": 0.1521647782910069, + "nauc_precision_at_10_diff1": -0.07750927677273797, + "nauc_precision_at_10_max": 0.21309126607147505, + "nauc_precision_at_10_std": 0.377640459966545, + "nauc_precision_at_1_diff1": 0.5492375204561938, + "nauc_precision_at_1_max": 0.12346348800082675, + "nauc_precision_at_1_std": 0.1717739449950981, + "nauc_precision_at_20_diff1": 0.0835961894895447, + "nauc_precision_at_20_max": 0.18927656722488895, + "nauc_precision_at_20_std": 0.3033865628259774, + "nauc_precision_at_3_diff1": 0.06821325088738678, + "nauc_precision_at_3_max": 0.11353499778022445, + "nauc_precision_at_3_std": 0.28724443002319483, + "nauc_precision_at_5_diff1": -0.0650688349346966, + "nauc_precision_at_5_max": 0.1911621891730056, + "nauc_precision_at_5_std": 0.34639713863337207, + "nauc_recall_at_1000_diff1": 0.027557638498875554, + "nauc_recall_at_1000_max": 0.03097565502070411, + "nauc_recall_at_1000_std": 0.026461907925695066, + "nauc_recall_at_100_diff1": -0.002745721981439647, + "nauc_recall_at_100_max": 0.06018066655766445, + "nauc_recall_at_100_std": 0.15216477829100694, + "nauc_recall_at_10_diff1": -0.07750927677273792, + "nauc_recall_at_10_max": 0.21309126607147533, + "nauc_recall_at_10_std": 0.3776404599665452, + "nauc_recall_at_1_diff1": 0.5492375204561938, + "nauc_recall_at_1_max": 0.12346348800082675, + "nauc_recall_at_1_std": 0.1717739449950981, + "nauc_recall_at_20_diff1": 0.08359618948954435, + "nauc_recall_at_20_max": 0.1892765672248886, + "nauc_recall_at_20_std": 0.3033865628259771, + "nauc_recall_at_3_diff1": 0.06821325088738682, + "nauc_recall_at_3_max": 0.11353499778022445, + "nauc_recall_at_3_std": 0.2872444300231949, + "nauc_recall_at_5_diff1": -0.0650688349346964, + "nauc_recall_at_5_max": 0.19116218917300562, + "nauc_recall_at_5_std": 0.34639713863337207, + "ndcg_at_1": 0.00028, + "ndcg_at_10": 0.00162, + "ndcg_at_100": 0.00695, + "ndcg_at_1000": 0.03721, + "ndcg_at_20": 0.00257, + "ndcg_at_3": 0.00073, + "ndcg_at_5": 0.00095, + "precision_at_1": 0.00028, + "precision_at_10": 0.00037, + "precision_at_100": 0.00033, + "precision_at_1000": 0.0003, + "precision_at_20": 0.00038, + "precision_at_3": 0.00037, + "precision_at_5": 0.00033, + "recall_at_1": 0.00028, + "recall_at_10": 0.00373, + "recall_at_100": 0.03271, + "recall_at_1000": 0.29896, + "recall_at_20": 0.00759, + "recall_at_3": 0.0011, + "recall_at_5": 0.00166 + }, + { + "cv_recall_at_1": 0.01389, + "cv_recall_at_10": 0.04546, + "cv_recall_at_100": 0.12165, + "cv_recall_at_1000": 0.41981, + "cv_recall_at_20": 0.06075, + "cv_recall_at_3": 0.02652, + "cv_recall_at_5": 0.03283, + "hf_subset": "fi", + "languages": [ + "fin-Latn" + ], + "main_score": 0.02784, + "map_at_1": 0.01389, + "map_at_10": 0.02244, + "map_at_100": 0.02485, + "map_at_1000": 0.02572, + "map_at_20": 0.02351, + "map_at_3": 0.01934, + "map_at_5": 0.02073, + "mrr_at_1": 0.013890837659604322, + "mrr_at_10": 0.022435918850960225, + "mrr_at_100": 0.024847508933185956, + "mrr_at_1000": 0.02572312343545724, + "mrr_at_20": 0.023512049607711832, + "mrr_at_3": 0.01933960057995416, + "mrr_at_5": 0.020728684345914582, + "nauc_cv_recall_at_1000_diff1": 0.08707531313449987, + "nauc_cv_recall_at_1000_max": 0.11996333114376777, + "nauc_cv_recall_at_1000_std": 0.30826029993220766, + "nauc_cv_recall_at_100_diff1": 0.10043742076486982, + "nauc_cv_recall_at_100_max": 0.1306432263472824, + "nauc_cv_recall_at_100_std": 0.48927315499614593, + "nauc_cv_recall_at_10_diff1": 0.1592530475063388, + "nauc_cv_recall_at_10_max": 0.20862327017350116, + "nauc_cv_recall_at_10_std": 0.5878117132491872, + "nauc_cv_recall_at_1_diff1": 0.350182350009386, + "nauc_cv_recall_at_1_max": 0.3046495935419812, + "nauc_cv_recall_at_1_std": 0.636084527606397, + "nauc_cv_recall_at_20_diff1": 0.1437524685823495, + "nauc_cv_recall_at_20_max": 0.17332115309790147, + "nauc_cv_recall_at_20_std": 0.5524658081664272, + "nauc_cv_recall_at_3_diff1": 0.22918673154466432, + "nauc_cv_recall_at_3_max": 0.24654796039781401, + "nauc_cv_recall_at_3_std": 0.623762262234279, + "nauc_cv_recall_at_5_diff1": 0.19788756712790903, + "nauc_cv_recall_at_5_max": 0.23318835036456873, + "nauc_cv_recall_at_5_std": 0.5975157204466219, + "nauc_map_at_1000_diff1": 0.2285067575085296, + "nauc_map_at_1000_max": 0.2343738982187907, + "nauc_map_at_1000_std": 0.5922158608918494, + "nauc_map_at_100_diff1": 0.2312473374345219, + "nauc_map_at_100_max": 0.23634822390739973, + "nauc_map_at_100_std": 0.5988199988119439, + "nauc_map_at_10_diff1": 0.24829523040656082, + "nauc_map_at_10_max": 0.2521165587215269, + "nauc_map_at_10_std": 0.6155248532278808, + "nauc_map_at_1_diff1": 0.350182350009386, + "nauc_map_at_1_max": 0.3046495935419812, + "nauc_map_at_1_std": 0.636084527606397, + "nauc_map_at_20_diff1": 0.24103792357800025, + "nauc_map_at_20_max": 0.24410149380833931, + "nauc_map_at_20_std": 0.6084242871609905, + "nauc_map_at_3_diff1": 0.2784663707486378, + "nauc_map_at_3_max": 0.26713667795648827, + "nauc_map_at_3_std": 0.6286305327651562, + "nauc_map_at_5_diff1": 0.26364300508564076, + "nauc_map_at_5_max": 0.26084655501595433, + "nauc_map_at_5_std": 0.6192192010943397, + "nauc_mrr_at_1000_diff1": 0.2285067575085296, + "nauc_mrr_at_1000_max": 0.2343738982187907, + "nauc_mrr_at_1000_std": 0.5922158608918494, + "nauc_mrr_at_100_diff1": 0.2312473374345219, + "nauc_mrr_at_100_max": 0.23634822390739973, + "nauc_mrr_at_100_std": 0.5988199988119439, + "nauc_mrr_at_10_diff1": 0.24829523040656082, + "nauc_mrr_at_10_max": 0.2521165587215269, + "nauc_mrr_at_10_std": 0.6155248532278808, + "nauc_mrr_at_1_diff1": 0.350182350009386, + "nauc_mrr_at_1_max": 0.3046495935419812, + "nauc_mrr_at_1_std": 0.636084527606397, + "nauc_mrr_at_20_diff1": 0.24103792357800025, + "nauc_mrr_at_20_max": 0.24410149380833931, + "nauc_mrr_at_20_std": 0.6084242871609905, + "nauc_mrr_at_3_diff1": 0.2784663707486378, + "nauc_mrr_at_3_max": 0.26713667795648827, + "nauc_mrr_at_3_std": 0.6286305327651562, + "nauc_mrr_at_5_diff1": 0.26364300508564076, + "nauc_mrr_at_5_max": 0.26084655501595433, + "nauc_mrr_at_5_std": 0.6192192010943397, + "nauc_ndcg_at_1000_diff1": 0.14334326058182448, + "nauc_ndcg_at_1000_max": 0.16778475378997937, + "nauc_ndcg_at_1000_std": 0.45092880887103054, + "nauc_ndcg_at_100_diff1": 0.16462476841543675, + "nauc_ndcg_at_100_max": 0.18467248424197302, + "nauc_ndcg_at_100_std": 0.5468753796421475, + "nauc_ndcg_at_10_diff1": 0.2143706326811892, + "nauc_ndcg_at_10_max": 0.2356068704911122, + "nauc_ndcg_at_10_std": 0.6052599730660827, + "nauc_ndcg_at_1_diff1": 0.350182350009386, + "nauc_ndcg_at_1_max": 0.3046495935419812, + "nauc_ndcg_at_1_std": 0.636084527606397, + "nauc_ndcg_at_20_diff1": 0.19970850602275628, + "nauc_ndcg_at_20_max": 0.2154759511247703, + "nauc_ndcg_at_20_std": 0.5864551079981218, + "nauc_ndcg_at_3_diff1": 0.2626559898455386, + "nauc_ndcg_at_3_max": 0.26041373891905834, + "nauc_ndcg_at_3_std": 0.627062901348483, + "nauc_ndcg_at_5_diff1": 0.24107407016068239, + "nauc_ndcg_at_5_max": 0.25125248027494673, + "nauc_ndcg_at_5_std": 0.6120704833304749, + "nauc_precision_at_1000_diff1": 0.08707531313449945, + "nauc_precision_at_1000_max": 0.11996333114376724, + "nauc_precision_at_1000_std": 0.3082602999322072, + "nauc_precision_at_100_diff1": 0.10043742076487006, + "nauc_precision_at_100_max": 0.13064322634728254, + "nauc_precision_at_100_std": 0.48927315499614615, + "nauc_precision_at_10_diff1": 0.15925304750633867, + "nauc_precision_at_10_max": 0.20862327017350096, + "nauc_precision_at_10_std": 0.587811713249187, + "nauc_precision_at_1_diff1": 0.350182350009386, + "nauc_precision_at_1_max": 0.3046495935419812, + "nauc_precision_at_1_std": 0.636084527606397, + "nauc_precision_at_20_diff1": 0.1437524685823496, + "nauc_precision_at_20_max": 0.1733211530979015, + "nauc_precision_at_20_std": 0.5524658081664269, + "nauc_precision_at_3_diff1": 0.22918673154466415, + "nauc_precision_at_3_max": 0.2465479603978137, + "nauc_precision_at_3_std": 0.6237622622342787, + "nauc_precision_at_5_diff1": 0.19788756712790923, + "nauc_precision_at_5_max": 0.23318835036456873, + "nauc_precision_at_5_std": 0.5975157204466223, + "nauc_recall_at_1000_diff1": 0.08707531313449987, + "nauc_recall_at_1000_max": 0.11996333114376777, + "nauc_recall_at_1000_std": 0.30826029993220766, + "nauc_recall_at_100_diff1": 0.10043742076486982, + "nauc_recall_at_100_max": 0.1306432263472824, + "nauc_recall_at_100_std": 0.48927315499614593, + "nauc_recall_at_10_diff1": 0.1592530475063388, + "nauc_recall_at_10_max": 0.20862327017350116, + "nauc_recall_at_10_std": 0.5878117132491872, + "nauc_recall_at_1_diff1": 0.350182350009386, + "nauc_recall_at_1_max": 0.3046495935419812, + "nauc_recall_at_1_std": 0.636084527606397, + "nauc_recall_at_20_diff1": 0.1437524685823495, + "nauc_recall_at_20_max": 0.17332115309790147, + "nauc_recall_at_20_std": 0.5524658081664272, + "nauc_recall_at_3_diff1": 0.22918673154466432, + "nauc_recall_at_3_max": 0.24654796039781401, + "nauc_recall_at_3_std": 0.623762262234279, + "nauc_recall_at_5_diff1": 0.19788756712790903, + "nauc_recall_at_5_max": 0.23318835036456873, + "nauc_recall_at_5_std": 0.5975157204466219, + "ndcg_at_1": 0.01389, + "ndcg_at_10": 0.02784, + "ndcg_at_100": 0.04256, + "ndcg_at_1000": 0.07711, + "ndcg_at_20": 0.03172, + "ndcg_at_3": 0.02118, + "ndcg_at_5": 0.02373, + "precision_at_1": 0.01389, + "precision_at_10": 0.00455, + "precision_at_100": 0.00122, + "precision_at_1000": 0.00042, + "precision_at_20": 0.00304, + "precision_at_3": 0.00884, + "precision_at_5": 0.00657, + "recall_at_1": 0.01389, + "recall_at_10": 0.04546, + "recall_at_100": 0.12165, + "recall_at_1000": 0.41981, + "recall_at_20": 0.06075, + "recall_at_3": 0.02652, + "recall_at_5": 0.03283 + }, + { + "cv_recall_at_1": 0.03798, + "cv_recall_at_10": 0.11197, + "cv_recall_at_100": 0.24926, + "cv_recall_at_1000": 0.57237, + "cv_recall_at_20": 0.14953, + "cv_recall_at_3": 0.06879, + "cv_recall_at_5": 0.08693, + "hf_subset": "fil", + "languages": [ + "fil-Latn" + ], + "main_score": 0.07134, + "map_at_1": 0.03798, + "map_at_10": 0.05877, + "map_at_100": 0.06375, + "map_at_1000": 0.06482, + "map_at_20": 0.06137, + "map_at_3": 0.0513, + "map_at_5": 0.05543, + "mrr_at_1": 0.03798002532001688, + "mrr_at_10": 0.058774591564013344, + "mrr_at_100": 0.06374552093791727, + "mrr_at_1000": 0.06482382251457994, + "mrr_at_20": 0.061370797279852826, + "mrr_at_3": 0.05129647864209684, + "mrr_at_5": 0.05543208139916527, + "nauc_cv_recall_at_1000_diff1": 0.20853496875406827, + "nauc_cv_recall_at_1000_max": 0.3881570464510469, + "nauc_cv_recall_at_1000_std": 0.49387254350944787, + "nauc_cv_recall_at_100_diff1": 0.23205637728352363, + "nauc_cv_recall_at_100_max": 0.3954508542415302, + "nauc_cv_recall_at_100_std": 0.4519956658040415, + "nauc_cv_recall_at_10_diff1": 0.3064742275003843, + "nauc_cv_recall_at_10_max": 0.4429589802430824, + "nauc_cv_recall_at_10_std": 0.40852835518045455, + "nauc_cv_recall_at_1_diff1": 0.5101715108551284, + "nauc_cv_recall_at_1_max": 0.5394259119087396, + "nauc_cv_recall_at_1_std": 0.37869340143950214, + "nauc_cv_recall_at_20_diff1": 0.28407658168080185, + "nauc_cv_recall_at_20_max": 0.43327377918876614, + "nauc_cv_recall_at_20_std": 0.4286340582660606, + "nauc_cv_recall_at_3_diff1": 0.3877289427470475, + "nauc_cv_recall_at_3_max": 0.5078791693039313, + "nauc_cv_recall_at_3_std": 0.4042514668481585, + "nauc_cv_recall_at_5_diff1": 0.3482188187657036, + "nauc_cv_recall_at_5_max": 0.4746387236244124, + "nauc_cv_recall_at_5_std": 0.4024974999774052, + "nauc_map_at_1000_diff1": 0.3928899992476722, + "nauc_map_at_1000_max": 0.4912539176811793, + "nauc_map_at_1000_std": 0.4005048908372273, + "nauc_map_at_100_diff1": 0.39371100428208494, + "nauc_map_at_100_max": 0.4917173814614342, + "nauc_map_at_100_std": 0.3997058234821897, + "nauc_map_at_10_diff1": 0.4057686405129143, + "nauc_map_at_10_max": 0.4988172418114415, + "nauc_map_at_10_std": 0.3942126514110771, + "nauc_map_at_1_diff1": 0.5101715108551284, + "nauc_map_at_1_max": 0.5394259119087396, + "nauc_map_at_1_std": 0.37869340143950214, + "nauc_map_at_20_diff1": 0.399727617762414, + "nauc_map_at_20_max": 0.49596156476295106, + "nauc_map_at_20_std": 0.3977598921807418, + "nauc_map_at_3_diff1": 0.437893265390524, + "nauc_map_at_3_max": 0.52071099682199, + "nauc_map_at_3_std": 0.39242515380387855, + "nauc_map_at_5_diff1": 0.42035957346223796, + "nauc_map_at_5_max": 0.5080647449043068, + "nauc_map_at_5_std": 0.39237505333497585, + "nauc_mrr_at_1000_diff1": 0.39288998966076943, + "nauc_mrr_at_1000_max": 0.49125389950626924, + "nauc_mrr_at_1000_std": 0.4005048696979699, + "nauc_mrr_at_100_diff1": 0.39371100428208494, + "nauc_mrr_at_100_max": 0.4917173814614342, + "nauc_mrr_at_100_std": 0.3997058234821897, + "nauc_mrr_at_10_diff1": 0.4057686405129143, + "nauc_mrr_at_10_max": 0.4988172418114415, + "nauc_mrr_at_10_std": 0.3942126514110771, + "nauc_mrr_at_1_diff1": 0.5101715108551284, + "nauc_mrr_at_1_max": 0.5394259119087396, + "nauc_mrr_at_1_std": 0.37869340143950214, + "nauc_mrr_at_20_diff1": 0.399727617762414, + "nauc_mrr_at_20_max": 0.49596156476295106, + "nauc_mrr_at_20_std": 0.3977598921807418, + "nauc_mrr_at_3_diff1": 0.437893265390524, + "nauc_mrr_at_3_max": 0.52071099682199, + "nauc_mrr_at_3_std": 0.39242515380387855, + "nauc_mrr_at_5_diff1": 0.42035957346223796, + "nauc_mrr_at_5_max": 0.5080647449043068, + "nauc_mrr_at_5_std": 0.39237505333497585, + "nauc_ndcg_at_1000_diff1": 0.31930736032054235, + "nauc_ndcg_at_1000_max": 0.44969017603521383, + "nauc_ndcg_at_1000_std": 0.4321298154567702, + "nauc_ndcg_at_100_diff1": 0.3274105911114975, + "nauc_ndcg_at_100_max": 0.4530516882928912, + "nauc_ndcg_at_100_std": 0.4196524139305105, + "nauc_ndcg_at_10_diff1": 0.3698265574948488, + "nauc_ndcg_at_10_max": 0.47919476729829397, + "nauc_ndcg_at_10_std": 0.3993731393346552, + "nauc_ndcg_at_1_diff1": 0.5101715108551284, + "nauc_ndcg_at_1_max": 0.5394259119087396, + "nauc_ndcg_at_1_std": 0.37869340143950214, + "nauc_ndcg_at_20_diff1": 0.35501548299345015, + "nauc_ndcg_at_20_max": 0.4721907577788585, + "nauc_ndcg_at_20_std": 0.4090085995177488, + "nauc_ndcg_at_3_diff1": 0.4219935975403448, + "nauc_ndcg_at_3_max": 0.5166402700042355, + "nauc_ndcg_at_3_std": 0.3961200978609036, + "nauc_ndcg_at_5_diff1": 0.3958647270105075, + "nauc_ndcg_at_5_max": 0.4969596142383925, + "nauc_ndcg_at_5_std": 0.39585335475850914, + "nauc_precision_at_1000_diff1": 0.2085349687540682, + "nauc_precision_at_1000_max": 0.38815704645104643, + "nauc_precision_at_1000_std": 0.4938725435094476, + "nauc_precision_at_100_diff1": 0.23205637728352368, + "nauc_precision_at_100_max": 0.39545085424153065, + "nauc_precision_at_100_std": 0.45199566580404166, + "nauc_precision_at_10_diff1": 0.3064742275003845, + "nauc_precision_at_10_max": 0.44295898024308256, + "nauc_precision_at_10_std": 0.4085283551804548, + "nauc_precision_at_1_diff1": 0.5101715108551284, + "nauc_precision_at_1_max": 0.5394259119087396, + "nauc_precision_at_1_std": 0.37869340143950214, + "nauc_precision_at_20_diff1": 0.2840765816808022, + "nauc_precision_at_20_max": 0.4332737791887661, + "nauc_precision_at_20_std": 0.4286340582660607, + "nauc_precision_at_3_diff1": 0.3877289427470472, + "nauc_precision_at_3_max": 0.5078791693039312, + "nauc_precision_at_3_std": 0.4042514668481582, + "nauc_precision_at_5_diff1": 0.34821881876570365, + "nauc_precision_at_5_max": 0.4746387236244124, + "nauc_precision_at_5_std": 0.40249749997740514, + "nauc_recall_at_1000_diff1": 0.20853496875406827, + "nauc_recall_at_1000_max": 0.3881570464510469, + "nauc_recall_at_1000_std": 0.49387254350944787, + "nauc_recall_at_100_diff1": 0.23205637728352363, + "nauc_recall_at_100_max": 0.3954508542415302, + "nauc_recall_at_100_std": 0.4519956658040415, + "nauc_recall_at_10_diff1": 0.3064742275003843, + "nauc_recall_at_10_max": 0.4429589802430824, + "nauc_recall_at_10_std": 0.40852835518045455, + "nauc_recall_at_1_diff1": 0.5101715108551284, + "nauc_recall_at_1_max": 0.5394259119087396, + "nauc_recall_at_1_std": 0.37869340143950214, + "nauc_recall_at_20_diff1": 0.28407658168080185, + "nauc_recall_at_20_max": 0.43327377918876614, + "nauc_recall_at_20_std": 0.4286340582660606, + "nauc_recall_at_3_diff1": 0.3877289427470475, + "nauc_recall_at_3_max": 0.5078791693039313, + "nauc_recall_at_3_std": 0.4042514668481585, + "nauc_recall_at_5_diff1": 0.3482188187657036, + "nauc_recall_at_5_max": 0.4746387236244124, + "nauc_recall_at_5_std": 0.4024974999774052, + "ndcg_at_1": 0.03798, + "ndcg_at_10": 0.07134, + "ndcg_at_100": 0.09895, + "ndcg_at_1000": 0.13722, + "ndcg_at_20": 0.08082, + "ndcg_at_3": 0.05578, + "ndcg_at_5": 0.06324, + "precision_at_1": 0.03798, + "precision_at_10": 0.0112, + "precision_at_100": 0.00249, + "precision_at_1000": 0.00057, + "precision_at_20": 0.00748, + "precision_at_3": 0.02293, + "precision_at_5": 0.01739, + "recall_at_1": 0.03798, + "recall_at_10": 0.11197, + "recall_at_100": 0.24926, + "recall_at_1000": 0.57237, + "recall_at_20": 0.14953, + "recall_at_3": 0.06879, + "recall_at_5": 0.08693 + }, + { + "cv_recall_at_1": 0.19224, + "cv_recall_at_10": 0.47349, + "cv_recall_at_100": 0.76337, + "cv_recall_at_1000": 0.95457, + "cv_recall_at_20": 0.564, + "cv_recall_at_3": 0.31873, + "cv_recall_at_5": 0.38636, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ], + "main_score": 0.32113, + "map_at_1": 0.19224, + "map_at_10": 0.27376, + "map_at_100": 0.28501, + "map_at_1000": 0.28586, + "map_at_20": 0.28005, + "map_at_3": 0.24689, + "map_at_5": 0.26217, + "mrr_at_1": 0.19224480261621116, + "mrr_at_10": 0.2737565488704229, + "mrr_at_100": 0.2850083273545978, + "mrr_at_1000": 0.28586338845015846, + "mrr_at_20": 0.28004661355585186, + "mrr_at_3": 0.24688546289807567, + "mrr_at_5": 0.26216810714007593, + "nauc_cv_recall_at_1000_diff1": 0.18221512084554836, + "nauc_cv_recall_at_1000_max": 0.46557401193098863, + "nauc_cv_recall_at_1000_std": 0.7395760302380457, + "nauc_cv_recall_at_100_diff1": 0.21033825061479602, + "nauc_cv_recall_at_100_max": 0.3345035605929724, + "nauc_cv_recall_at_100_std": 0.43863742408636125, + "nauc_cv_recall_at_10_diff1": 0.2360019555784446, + "nauc_cv_recall_at_10_max": 0.3163427257486659, + "nauc_cv_recall_at_10_std": 0.2284968215737257, + "nauc_cv_recall_at_1_diff1": 0.4095398577757327, + "nauc_cv_recall_at_1_max": 0.3589766108920974, + "nauc_cv_recall_at_1_std": 0.12674851925413028, + "nauc_cv_recall_at_20_diff1": 0.21034976734280142, + "nauc_cv_recall_at_20_max": 0.3025692961549244, + "nauc_cv_recall_at_20_std": 0.2732621004173901, + "nauc_cv_recall_at_3_diff1": 0.2922105553144808, + "nauc_cv_recall_at_3_max": 0.3357950932758947, + "nauc_cv_recall_at_3_std": 0.16981489233169808, + "nauc_cv_recall_at_5_diff1": 0.2650865302094447, + "nauc_cv_recall_at_5_max": 0.3296761807327087, + "nauc_cv_recall_at_5_std": 0.17915999497804208, + "nauc_map_at_1000_diff1": 0.3397297903402744, + "nauc_map_at_1000_max": 0.34529149765324946, + "nauc_map_at_1000_std": 0.1621579787020462, + "nauc_map_at_100_diff1": 0.33946264347274546, + "nauc_map_at_100_max": 0.34515605960451845, + "nauc_map_at_100_std": 0.1624126192568733, + "nauc_map_at_10_diff1": 0.3396071563022263, + "nauc_map_at_10_max": 0.34531124141719866, + "nauc_map_at_10_std": 0.15790447070890276, + "nauc_map_at_1_diff1": 0.4095398577757327, + "nauc_map_at_1_max": 0.3589766108920974, + "nauc_map_at_1_std": 0.12674851925413028, + "nauc_map_at_20_diff1": 0.3383203366998559, + "nauc_map_at_20_max": 0.34438683549639915, + "nauc_map_at_20_std": 0.16050069735291433, + "nauc_map_at_3_diff1": 0.35137520059375393, + "nauc_map_at_3_max": 0.3486786651913938, + "nauc_map_at_3_std": 0.14744928173306157, + "nauc_map_at_5_diff1": 0.34400402411460196, + "nauc_map_at_5_max": 0.3471475789241504, + "nauc_map_at_5_std": 0.1501588391853353, + "nauc_mrr_at_1000_diff1": 0.3397297903402744, + "nauc_mrr_at_1000_max": 0.34529149765324946, + "nauc_mrr_at_1000_std": 0.1621579787020462, + "nauc_mrr_at_100_diff1": 0.33946264347274546, + "nauc_mrr_at_100_max": 0.34515605960451845, + "nauc_mrr_at_100_std": 0.1624126192568733, + "nauc_mrr_at_10_diff1": 0.3396071563022263, + "nauc_mrr_at_10_max": 0.34531124141719866, + "nauc_mrr_at_10_std": 0.15790447070890276, + "nauc_mrr_at_1_diff1": 0.4095398577757327, + "nauc_mrr_at_1_max": 0.3589766108920974, + "nauc_mrr_at_1_std": 0.12674851925413028, + "nauc_mrr_at_20_diff1": 0.3383203366998559, + "nauc_mrr_at_20_max": 0.34438683549639915, + "nauc_mrr_at_20_std": 0.16050069735291433, + "nauc_mrr_at_3_diff1": 0.35137520059375393, + "nauc_mrr_at_3_max": 0.3486786651913938, + "nauc_mrr_at_3_std": 0.14744928173306157, + "nauc_mrr_at_5_diff1": 0.34400402411460196, + "nauc_mrr_at_5_max": 0.3471475789241504, + "nauc_mrr_at_5_std": 0.1501588391853353, + "nauc_ndcg_at_1000_diff1": 0.320807196056081, + "nauc_ndcg_at_1000_max": 0.3441773496778746, + "nauc_ndcg_at_1000_std": 0.1916687643243846, + "nauc_ndcg_at_100_diff1": 0.3136149630871213, + "nauc_ndcg_at_100_max": 0.3408155054521133, + "nauc_ndcg_at_100_std": 0.20215146786074856, + "nauc_ndcg_at_10_diff1": 0.3119374437666259, + "nauc_ndcg_at_10_max": 0.3378532278503251, + "nauc_ndcg_at_10_std": 0.17590910749683075, + "nauc_ndcg_at_1_diff1": 0.4095398577757327, + "nauc_ndcg_at_1_max": 0.3589766108920974, + "nauc_ndcg_at_1_std": 0.12674851925413028, + "nauc_ndcg_at_20_diff1": 0.30681576682350986, + "nauc_ndcg_at_20_max": 0.3346076731018436, + "nauc_ndcg_at_20_std": 0.1861829240387525, + "nauc_ndcg_at_3_diff1": 0.3348439226129218, + "nauc_ndcg_at_3_max": 0.3451279423006363, + "nauc_ndcg_at_3_std": 0.15367180839364783, + "nauc_ndcg_at_5_diff1": 0.32205950130045446, + "nauc_ndcg_at_5_max": 0.34235607549196584, + "nauc_ndcg_at_5_std": 0.15822721650377838, + "nauc_precision_at_1000_diff1": 0.18221512084554412, + "nauc_precision_at_1000_max": 0.4655740119309881, + "nauc_precision_at_1000_std": 0.7395760302380422, + "nauc_precision_at_100_diff1": 0.21033825061479558, + "nauc_precision_at_100_max": 0.3345035605929725, + "nauc_precision_at_100_std": 0.4386374240863622, + "nauc_precision_at_10_diff1": 0.23600195557844436, + "nauc_precision_at_10_max": 0.3163427257486658, + "nauc_precision_at_10_std": 0.2284968215737256, + "nauc_precision_at_1_diff1": 0.4095398577757327, + "nauc_precision_at_1_max": 0.3589766108920974, + "nauc_precision_at_1_std": 0.12674851925413028, + "nauc_precision_at_20_diff1": 0.21034976734280128, + "nauc_precision_at_20_max": 0.3025692961549245, + "nauc_precision_at_20_std": 0.2732621004173904, + "nauc_precision_at_3_diff1": 0.29221055531448104, + "nauc_precision_at_3_max": 0.33579509327589446, + "nauc_precision_at_3_std": 0.16981489233169803, + "nauc_precision_at_5_diff1": 0.26508653020944456, + "nauc_precision_at_5_max": 0.3296761807327088, + "nauc_precision_at_5_std": 0.17915999497804247, + "nauc_recall_at_1000_diff1": 0.18221512084554836, + "nauc_recall_at_1000_max": 0.46557401193098863, + "nauc_recall_at_1000_std": 0.7395760302380457, + "nauc_recall_at_100_diff1": 0.21033825061479602, + "nauc_recall_at_100_max": 0.3345035605929724, + "nauc_recall_at_100_std": 0.43863742408636125, + "nauc_recall_at_10_diff1": 0.2360019555784446, + "nauc_recall_at_10_max": 0.3163427257486659, + "nauc_recall_at_10_std": 0.2284968215737257, + "nauc_recall_at_1_diff1": 0.4095398577757327, + "nauc_recall_at_1_max": 0.3589766108920974, + "nauc_recall_at_1_std": 0.12674851925413028, + "nauc_recall_at_20_diff1": 0.21034976734280142, + "nauc_recall_at_20_max": 0.3025692961549244, + "nauc_recall_at_20_std": 0.2732621004173901, + "nauc_recall_at_3_diff1": 0.2922105553144808, + "nauc_recall_at_3_max": 0.3357950932758947, + "nauc_recall_at_3_std": 0.16981489233169808, + "nauc_recall_at_5_diff1": 0.2650865302094447, + "nauc_recall_at_5_max": 0.3296761807327087, + "nauc_recall_at_5_std": 0.17915999497804208, + "ndcg_at_1": 0.19224, + "ndcg_at_10": 0.32113, + "ndcg_at_100": 0.38071, + "ndcg_at_1000": 0.40466, + "ndcg_at_20": 0.34402, + "ndcg_at_3": 0.26529, + "ndcg_at_5": 0.29299, + "precision_at_1": 0.19224, + "precision_at_10": 0.04735, + "precision_at_100": 0.00763, + "precision_at_1000": 0.00095, + "precision_at_20": 0.0282, + "precision_at_3": 0.10624, + "precision_at_5": 0.07727, + "recall_at_1": 0.19224, + "recall_at_10": 0.47349, + "recall_at_100": 0.76337, + "recall_at_1000": 0.95457, + "recall_at_20": 0.564, + "recall_at_3": 0.31873, + "recall_at_5": 0.38636 + }, + { + "cv_recall_at_1": 0.00097, + "cv_recall_at_10": 0.00403, + "cv_recall_at_100": 0.03208, + "cv_recall_at_1000": 0.28361, + "cv_recall_at_20": 0.00722, + "cv_recall_at_3": 0.0025, + "cv_recall_at_5": 0.00306, + "hf_subset": "he", + "languages": [ + "heb-Hebr" + ], + "main_score": 0.00243, + "map_at_1": 0.00097, + "map_at_10": 0.00193, + "map_at_100": 0.00264, + "map_at_1000": 0.00328, + "map_at_20": 0.00214, + "map_at_3": 0.00167, + "map_at_5": 0.0018, + "mrr_at_1": 0.0009722222222222222, + "mrr_at_10": 0.001931437389770723, + "mrr_at_100": 0.0026372004826096546, + "mrr_at_1000": 0.003280471256251679, + "mrr_at_20": 0.0021394673002483886, + "mrr_at_3": 0.0016666666666666668, + "mrr_at_5": 0.001798611111111111, + "nauc_cv_recall_at_1000_diff1": -0.014688330434024493, + "nauc_cv_recall_at_1000_max": -0.040394933260628846, + "nauc_cv_recall_at_1000_std": 0.016267299925534613, + "nauc_cv_recall_at_100_diff1": 0.050751681017310306, + "nauc_cv_recall_at_100_max": 0.0670188410998714, + "nauc_cv_recall_at_100_std": 0.12506954631988185, + "nauc_cv_recall_at_10_diff1": 0.0967034593784406, + "nauc_cv_recall_at_10_max": 0.23868693504066313, + "nauc_cv_recall_at_10_std": 0.38581460544211377, + "nauc_cv_recall_at_1_diff1": 0.7857717812104719, + "nauc_cv_recall_at_1_max": 0.589482157231017, + "nauc_cv_recall_at_1_std": 0.7263214381540114, + "nauc_cv_recall_at_20_diff1": 0.06092237518317478, + "nauc_cv_recall_at_20_max": 0.16503787331530825, + "nauc_cv_recall_at_20_std": 0.2861860436317104, + "nauc_cv_recall_at_3_diff1": 0.1659660733983243, + "nauc_cv_recall_at_3_max": 0.2803267447753629, + "nauc_cv_recall_at_3_std": 0.3652923113615357, + "nauc_cv_recall_at_5_diff1": 0.15365758470131952, + "nauc_cv_recall_at_5_max": 0.2923384637900333, + "nauc_cv_recall_at_5_std": 0.3567334195185012, + "nauc_map_at_1000_diff1": 0.1962660554666269, + "nauc_map_at_1000_max": 0.22297679269240125, + "nauc_map_at_1000_std": 0.3176544914119965, + "nauc_map_at_100_diff1": 0.23879431747758573, + "nauc_map_at_100_max": 0.27444983312462656, + "nauc_map_at_100_std": 0.3793285530962337, + "nauc_map_at_10_diff1": 0.3146894224461796, + "nauc_map_at_10_max": 0.36308819726862906, + "nauc_map_at_10_std": 0.4853605633939986, + "nauc_map_at_1_diff1": 0.7857717812104719, + "nauc_map_at_1_max": 0.589482157231017, + "nauc_map_at_1_std": 0.7263214381540114, + "nauc_map_at_20_diff1": 0.28606662763634433, + "nauc_map_at_20_max": 0.33238947614972497, + "nauc_map_at_20_std": 0.4516059480170954, + "nauc_map_at_3_diff1": 0.3631309065975018, + "nauc_map_at_3_max": 0.38325060370271, + "nauc_map_at_3_std": 0.496590197656739, + "nauc_map_at_5_diff1": 0.3448749067208954, + "nauc_map_at_5_max": 0.38267862711114653, + "nauc_map_at_5_std": 0.485963329268937, + "nauc_mrr_at_1000_diff1": 0.1962660554666269, + "nauc_mrr_at_1000_max": 0.22297679269240125, + "nauc_mrr_at_1000_std": 0.3176544914119965, + "nauc_mrr_at_100_diff1": 0.23879431747758573, + "nauc_mrr_at_100_max": 0.27444983312462656, + "nauc_mrr_at_100_std": 0.3793285530962337, + "nauc_mrr_at_10_diff1": 0.3146894224461796, + "nauc_mrr_at_10_max": 0.36308819726862906, + "nauc_mrr_at_10_std": 0.4853605633939986, + "nauc_mrr_at_1_diff1": 0.7857717812104719, + "nauc_mrr_at_1_max": 0.589482157231017, + "nauc_mrr_at_1_std": 0.7263214381540114, + "nauc_mrr_at_20_diff1": 0.28606662763634433, + "nauc_mrr_at_20_max": 0.33238947614972497, + "nauc_mrr_at_20_std": 0.4516059480170954, + "nauc_mrr_at_3_diff1": 0.3631309065975018, + "nauc_mrr_at_3_max": 0.38325060370271, + "nauc_mrr_at_3_std": 0.496590197656739, + "nauc_mrr_at_5_diff1": 0.3448749067208954, + "nauc_mrr_at_5_max": 0.38267862711114653, + "nauc_mrr_at_5_std": 0.485963329268937, + "nauc_ndcg_at_1000_diff1": 0.009107777849244263, + "nauc_ndcg_at_1000_max": -0.005747819977854507, + "nauc_ndcg_at_1000_std": 0.053727994473211596, + "nauc_ndcg_at_100_diff1": 0.1010042139450688, + "nauc_ndcg_at_100_max": 0.12816600315656998, + "nauc_ndcg_at_100_std": 0.20279351156528072, + "nauc_ndcg_at_10_diff1": 0.2274399109660982, + "nauc_ndcg_at_10_max": 0.31451605513713937, + "nauc_ndcg_at_10_std": 0.44420690102477517, + "nauc_ndcg_at_1_diff1": 0.7857717812104719, + "nauc_ndcg_at_1_max": 0.589482157231017, + "nauc_ndcg_at_1_std": 0.7263214381540114, + "nauc_ndcg_at_20_diff1": 0.17585608075751075, + "nauc_ndcg_at_20_max": 0.25298732196021045, + "nauc_ndcg_at_20_std": 0.3727442874544738, + "nauc_ndcg_at_3_diff1": 0.29556646211677523, + "nauc_ndcg_at_3_max": 0.34815707856545386, + "nauc_ndcg_at_3_std": 0.4522308753168862, + "nauc_ndcg_at_5_diff1": 0.2746872655009156, + "nauc_ndcg_at_5_max": 0.3495516499369595, + "nauc_ndcg_at_5_std": 0.43928694839903326, + "nauc_precision_at_1000_diff1": -0.014688330434025001, + "nauc_precision_at_1000_max": -0.04039493326062917, + "nauc_precision_at_1000_std": 0.016267299925534114, + "nauc_precision_at_100_diff1": 0.05075168101731036, + "nauc_precision_at_100_max": 0.06701884109987172, + "nauc_precision_at_100_std": 0.12506954631988182, + "nauc_precision_at_10_diff1": 0.0967034593784405, + "nauc_precision_at_10_max": 0.23868693504066316, + "nauc_precision_at_10_std": 0.3858146054421139, + "nauc_precision_at_1_diff1": 0.7857717812104719, + "nauc_precision_at_1_max": 0.589482157231017, + "nauc_precision_at_1_std": 0.7263214381540114, + "nauc_precision_at_20_diff1": 0.060922375183175, + "nauc_precision_at_20_max": 0.16503787331530848, + "nauc_precision_at_20_std": 0.28618604363171074, + "nauc_precision_at_3_diff1": 0.16596607339832442, + "nauc_precision_at_3_max": 0.28032674477536285, + "nauc_precision_at_3_std": 0.36529231136153584, + "nauc_precision_at_5_diff1": 0.1536575847013195, + "nauc_precision_at_5_max": 0.2923384637900333, + "nauc_precision_at_5_std": 0.35673341951850135, + "nauc_recall_at_1000_diff1": -0.014688330434024493, + "nauc_recall_at_1000_max": -0.040394933260628846, + "nauc_recall_at_1000_std": 0.016267299925534613, + "nauc_recall_at_100_diff1": 0.050751681017310306, + "nauc_recall_at_100_max": 0.0670188410998714, + "nauc_recall_at_100_std": 0.12506954631988185, + "nauc_recall_at_10_diff1": 0.0967034593784406, + "nauc_recall_at_10_max": 0.23868693504066313, + "nauc_recall_at_10_std": 0.38581460544211377, + "nauc_recall_at_1_diff1": 0.7857717812104719, + "nauc_recall_at_1_max": 0.589482157231017, + "nauc_recall_at_1_std": 0.7263214381540114, + "nauc_recall_at_20_diff1": 0.06092237518317478, + "nauc_recall_at_20_max": 0.16503787331530825, + "nauc_recall_at_20_std": 0.2861860436317104, + "nauc_recall_at_3_diff1": 0.1659660733983243, + "nauc_recall_at_3_max": 0.2803267447753629, + "nauc_recall_at_3_std": 0.3652923113615357, + "nauc_recall_at_5_diff1": 0.15365758470131952, + "nauc_recall_at_5_max": 0.2923384637900333, + "nauc_recall_at_5_std": 0.3567334195185012, + "ndcg_at_1": 0.00097, + "ndcg_at_10": 0.00243, + "ndcg_at_100": 0.00755, + "ndcg_at_1000": 0.0361, + "ndcg_at_20": 0.00322, + "ndcg_at_3": 0.00188, + "ndcg_at_5": 0.00211, + "precision_at_1": 0.00097, + "precision_at_10": 0.0004, + "precision_at_100": 0.00032, + "precision_at_1000": 0.00028, + "precision_at_20": 0.00036, + "precision_at_3": 0.00083, + "precision_at_5": 0.00061, + "recall_at_1": 0.00097, + "recall_at_10": 0.00403, + "recall_at_100": 0.03208, + "recall_at_1000": 0.28361, + "recall_at_20": 0.00722, + "recall_at_3": 0.0025, + "recall_at_5": 0.00306 + }, + { + "cv_recall_at_1": 0.00047, + "cv_recall_at_10": 0.00212, + "cv_recall_at_100": 0.02975, + "cv_recall_at_1000": 0.28167, + "cv_recall_at_20": 0.006, + "cv_recall_at_3": 0.00082, + "cv_recall_at_5": 0.00106, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ], + "main_score": 0.00108, + "map_at_1": 0.00047, + "map_at_10": 0.00077, + "map_at_100": 0.00156, + "map_at_1000": 0.00218, + "map_at_20": 0.00104, + "map_at_3": 0.00059, + "map_at_5": 0.00065, + "mrr_at_1": 0.0004704222039280254, + "mrr_at_10": 0.0007740498684871258, + "mrr_at_100": 0.0015571687071699366, + "mrr_at_1000": 0.0021844098865008567, + "mrr_at_20": 0.0010434503539331948, + "mrr_at_3": 0.0005880277549100317, + "mrr_at_5": 0.000646830530401035, + "nauc_cv_recall_at_1000_diff1": 0.023466105738380243, + "nauc_cv_recall_at_1000_max": 0.02982050639025681, + "nauc_cv_recall_at_1000_std": 0.020550417425237402, + "nauc_cv_recall_at_100_diff1": 0.03784573719978466, + "nauc_cv_recall_at_100_max": -0.007253568714786409, + "nauc_cv_recall_at_100_std": 0.006491902302095944, + "nauc_cv_recall_at_10_diff1": -0.06717912818256067, + "nauc_cv_recall_at_10_max": 0.09469709173925962, + "nauc_cv_recall_at_10_std": 0.12422926632555775, + "nauc_cv_recall_at_1_diff1": 0.147582993880789, + "nauc_cv_recall_at_1_max": 0.08980850020969633, + "nauc_cv_recall_at_1_std": 0.20248893816056948, + "nauc_cv_recall_at_20_diff1": 0.055127169766363955, + "nauc_cv_recall_at_20_max": 0.1518853718623151, + "nauc_cv_recall_at_20_std": 0.1327060259635255, + "nauc_cv_recall_at_3_diff1": -0.0498531255070009, + "nauc_cv_recall_at_3_max": 0.025913192697909856, + "nauc_cv_recall_at_3_std": 0.011191172961024218, + "nauc_cv_recall_at_5_diff1": -0.12966772945517036, + "nauc_cv_recall_at_5_max": 0.16724926662252462, + "nauc_cv_recall_at_5_std": 0.1933515924059369, + "nauc_map_at_1000_diff1": 0.026214389853673414, + "nauc_map_at_1000_max": 0.05082203380750064, + "nauc_map_at_1000_std": 0.07111822507222437, + "nauc_map_at_100_diff1": 0.032677935166499354, + "nauc_map_at_100_max": 0.0576271300372991, + "nauc_map_at_100_std": 0.08628569066070878, + "nauc_map_at_10_diff1": 0.004674473643573394, + "nauc_map_at_10_max": 0.08824978358169214, + "nauc_map_at_10_std": 0.15106284212743892, + "nauc_map_at_1_diff1": 0.147582993880789, + "nauc_map_at_1_max": 0.08980850020969633, + "nauc_map_at_1_std": 0.20248893816056948, + "nauc_map_at_20_diff1": 0.03385553500941615, + "nauc_map_at_20_max": 0.10920469708203091, + "nauc_map_at_20_std": 0.14686018180633145, + "nauc_map_at_3_diff1": 0.055446138166487, + "nauc_map_at_3_max": 0.05999069003752935, + "nauc_map_at_3_std": 0.1132166477341151, + "nauc_map_at_5_diff1": 0.013222048944646094, + "nauc_map_at_5_max": 0.11471203870308802, + "nauc_map_at_5_std": 0.1784617761640255, + "nauc_mrr_at_1000_diff1": 0.026214308205751255, + "nauc_mrr_at_1000_max": 0.05082201653104815, + "nauc_mrr_at_1000_std": 0.07111832244532566, + "nauc_mrr_at_100_diff1": 0.032677935166499354, + "nauc_mrr_at_100_max": 0.0576271300372991, + "nauc_mrr_at_100_std": 0.08628569066070878, + "nauc_mrr_at_10_diff1": 0.004674473643573394, + "nauc_mrr_at_10_max": 0.08824978358169214, + "nauc_mrr_at_10_std": 0.15106284212743892, + "nauc_mrr_at_1_diff1": 0.147582993880789, + "nauc_mrr_at_1_max": 0.08980850020969633, + "nauc_mrr_at_1_std": 0.20248893816056948, + "nauc_mrr_at_20_diff1": 0.03385553500941615, + "nauc_mrr_at_20_max": 0.10920469708203091, + "nauc_mrr_at_20_std": 0.14686018180633145, + "nauc_mrr_at_3_diff1": 0.055446138166487, + "nauc_mrr_at_3_max": 0.05999069003752935, + "nauc_mrr_at_3_std": 0.1132166477341151, + "nauc_mrr_at_5_diff1": 0.013222048944646094, + "nauc_mrr_at_5_max": 0.11471203870308802, + "nauc_mrr_at_5_std": 0.1784617761640255, + "nauc_ndcg_at_1000_diff1": 0.022629861488434374, + "nauc_ndcg_at_1000_max": 0.029349628076607032, + "nauc_ndcg_at_1000_std": 0.023801430605690604, + "nauc_ndcg_at_100_diff1": 0.0353565114670267, + "nauc_ndcg_at_100_max": 0.011691000348249958, + "nauc_ndcg_at_100_std": 0.026352383777968207, + "nauc_ndcg_at_10_diff1": -0.03086303226289459, + "nauc_ndcg_at_10_max": 0.09027620565935654, + "nauc_ndcg_at_10_std": 0.13825251774960307, + "nauc_ndcg_at_1_diff1": 0.147582993880789, + "nauc_ndcg_at_1_max": 0.08980850020969633, + "nauc_ndcg_at_1_std": 0.20248893816056948, + "nauc_ndcg_at_20_diff1": 0.041403752773278076, + "nauc_ndcg_at_20_max": 0.13243074970635393, + "nauc_ndcg_at_20_std": 0.13752833968262493, + "nauc_ndcg_at_3_diff1": 0.021941826997649856, + "nauc_ndcg_at_3_max": 0.049147849974923004, + "nauc_ndcg_at_3_std": 0.08075399666994976, + "nauc_ndcg_at_5_diff1": -0.036412002682510894, + "nauc_ndcg_at_5_max": 0.1321204425287366, + "nauc_ndcg_at_5_std": 0.18232858864848792, + "nauc_precision_at_1000_diff1": 0.023466105738379983, + "nauc_precision_at_1000_max": 0.029820506390256532, + "nauc_precision_at_1000_std": 0.02055041742523727, + "nauc_precision_at_100_diff1": 0.037845737199784756, + "nauc_precision_at_100_max": -0.007253568714786299, + "nauc_precision_at_100_std": 0.006491902302096105, + "nauc_precision_at_10_diff1": -0.06717912818256085, + "nauc_precision_at_10_max": 0.09469709173925928, + "nauc_precision_at_10_std": 0.12422926632555738, + "nauc_precision_at_1_diff1": 0.147582993880789, + "nauc_precision_at_1_max": 0.08980850020969633, + "nauc_precision_at_1_std": 0.20248893816056948, + "nauc_precision_at_20_diff1": 0.055127169766364476, + "nauc_precision_at_20_max": 0.15188537186231557, + "nauc_precision_at_20_std": 0.13270602596352585, + "nauc_precision_at_3_diff1": -0.04985312550700086, + "nauc_precision_at_3_max": 0.025913192697909995, + "nauc_precision_at_3_std": 0.01119117296102431, + "nauc_precision_at_5_diff1": -0.1296677294551703, + "nauc_precision_at_5_max": 0.16724926662252482, + "nauc_precision_at_5_std": 0.193351592405937, + "nauc_recall_at_1000_diff1": 0.023466105738380243, + "nauc_recall_at_1000_max": 0.02982050639025681, + "nauc_recall_at_1000_std": 0.020550417425237402, + "nauc_recall_at_100_diff1": 0.03784573719978466, + "nauc_recall_at_100_max": -0.007253568714786409, + "nauc_recall_at_100_std": 0.006491902302095944, + "nauc_recall_at_10_diff1": -0.06717912818256067, + "nauc_recall_at_10_max": 0.09469709173925962, + "nauc_recall_at_10_std": 0.12422926632555775, + "nauc_recall_at_1_diff1": 0.147582993880789, + "nauc_recall_at_1_max": 0.08980850020969633, + "nauc_recall_at_1_std": 0.20248893816056948, + "nauc_recall_at_20_diff1": 0.055127169766363955, + "nauc_recall_at_20_max": 0.1518853718623151, + "nauc_recall_at_20_std": 0.1327060259635255, + "nauc_recall_at_3_diff1": -0.0498531255070009, + "nauc_recall_at_3_max": 0.025913192697909856, + "nauc_recall_at_3_std": 0.011191172961024218, + "nauc_recall_at_5_diff1": -0.12966772945517036, + "nauc_recall_at_5_max": 0.16724926662252462, + "nauc_recall_at_5_std": 0.1933515924059369, + "ndcg_at_1": 0.00047, + "ndcg_at_10": 0.00108, + "ndcg_at_100": 0.00627, + "ndcg_at_1000": 0.03476, + "ndcg_at_20": 0.00206, + "ndcg_at_3": 0.00065, + "ndcg_at_5": 0.00075, + "precision_at_1": 0.00047, + "precision_at_10": 0.00021, + "precision_at_100": 0.0003, + "precision_at_1000": 0.00028, + "precision_at_20": 0.0003, + "precision_at_3": 0.00027, + "precision_at_5": 0.00021, + "recall_at_1": 0.00047, + "recall_at_10": 0.00212, + "recall_at_100": 0.02975, + "recall_at_1000": 0.28167, + "recall_at_20": 0.006, + "recall_at_3": 0.00082, + "recall_at_5": 0.00106 + }, + { + "cv_recall_at_1": 0.01484, + "cv_recall_at_10": 0.05137, + "cv_recall_at_100": 0.14038, + "cv_recall_at_1000": 0.45989, + "cv_recall_at_20": 0.07019, + "cv_recall_at_3": 0.02816, + "cv_recall_at_5": 0.03777, + "hf_subset": "hr", + "languages": [ + "hrv-Latn" + ], + "main_score": 0.03088, + "map_at_1": 0.01484, + "map_at_10": 0.02461, + "map_at_100": 0.02739, + "map_at_1000": 0.02837, + "map_at_20": 0.02585, + "map_at_3": 0.02067, + "map_at_5": 0.02286, + "mrr_at_1": 0.014835164835164835, + "mrr_at_10": 0.02460895255538113, + "mrr_at_100": 0.027393640089861407, + "mrr_at_1000": 0.02836957940781095, + "mrr_at_20": 0.02585265651115727, + "mrr_at_3": 0.020673076923076916, + "mrr_at_5": 0.02285714285714286, + "nauc_cv_recall_at_1000_diff1": 0.09582476902539795, + "nauc_cv_recall_at_1000_max": 0.27929155755972995, + "nauc_cv_recall_at_1000_std": 0.3520941312199978, + "nauc_cv_recall_at_100_diff1": 0.16728310795758913, + "nauc_cv_recall_at_100_max": 0.37389641110769223, + "nauc_cv_recall_at_100_std": 0.45023838508861586, + "nauc_cv_recall_at_10_diff1": 0.19929435295839448, + "nauc_cv_recall_at_10_max": 0.46857517551499467, + "nauc_cv_recall_at_10_std": 0.4969495615891551, + "nauc_cv_recall_at_1_diff1": 0.4321169841695732, + "nauc_cv_recall_at_1_max": 0.5775298373232366, + "nauc_cv_recall_at_1_std": 0.5090480875294399, + "nauc_cv_recall_at_20_diff1": 0.1913912177336063, + "nauc_cv_recall_at_20_max": 0.4393900331688629, + "nauc_cv_recall_at_20_std": 0.4817797035122659, + "nauc_cv_recall_at_3_diff1": 0.2756104105178428, + "nauc_cv_recall_at_3_max": 0.5093405406820369, + "nauc_cv_recall_at_3_std": 0.526119876707219, + "nauc_cv_recall_at_5_diff1": 0.22791033490255386, + "nauc_cv_recall_at_5_max": 0.4876834890357833, + "nauc_cv_recall_at_5_std": 0.512941922579701, + "nauc_map_at_1000_diff1": 0.29128633313057856, + "nauc_map_at_1000_max": 0.5021598434492848, + "nauc_map_at_1000_std": 0.49977560680857286, + "nauc_map_at_100_diff1": 0.29528377391075455, + "nauc_map_at_100_max": 0.5065295243303385, + "nauc_map_at_100_std": 0.5028370765094626, + "nauc_map_at_10_diff1": 0.3082125854119504, + "nauc_map_at_10_max": 0.5252052250217453, + "nauc_map_at_10_std": 0.5108376017093625, + "nauc_map_at_1_diff1": 0.4321169841695732, + "nauc_map_at_1_max": 0.5775298373232366, + "nauc_map_at_1_std": 0.5090480875294399, + "nauc_map_at_20_diff1": 0.3023319120607131, + "nauc_map_at_20_max": 0.5179141232025187, + "nauc_map_at_20_std": 0.5080217864768897, + "nauc_map_at_3_diff1": 0.34768224809470033, + "nauc_map_at_3_max": 0.5429651809057904, + "nauc_map_at_3_std": 0.5195759536304023, + "nauc_map_at_5_diff1": 0.3226213994981184, + "nauc_map_at_5_max": 0.5323292959639635, + "nauc_map_at_5_std": 0.5155839093018375, + "nauc_mrr_at_1000_diff1": 0.29128634518308727, + "nauc_mrr_at_1000_max": 0.5021598494729698, + "nauc_mrr_at_1000_std": 0.4997756092105584, + "nauc_mrr_at_100_diff1": 0.29528377391075455, + "nauc_mrr_at_100_max": 0.5065295243303385, + "nauc_mrr_at_100_std": 0.5028370765094626, + "nauc_mrr_at_10_diff1": 0.3082125854119504, + "nauc_mrr_at_10_max": 0.5252052250217453, + "nauc_mrr_at_10_std": 0.5108376017093625, + "nauc_mrr_at_1_diff1": 0.4321169841695732, + "nauc_mrr_at_1_max": 0.5775298373232366, + "nauc_mrr_at_1_std": 0.5090480875294399, + "nauc_mrr_at_20_diff1": 0.3023319120607131, + "nauc_mrr_at_20_max": 0.5179141232025187, + "nauc_mrr_at_20_std": 0.5080217864768897, + "nauc_mrr_at_3_diff1": 0.34768224809470033, + "nauc_mrr_at_3_max": 0.5429651809057904, + "nauc_mrr_at_3_std": 0.5195759536304023, + "nauc_mrr_at_5_diff1": 0.3226213994981184, + "nauc_mrr_at_5_max": 0.5323292959639635, + "nauc_mrr_at_5_std": 0.5155839093018375, + "nauc_ndcg_at_1000_diff1": 0.18573460599963715, + "nauc_ndcg_at_1000_max": 0.38794287119574855, + "nauc_ndcg_at_1000_std": 0.43063491189756353, + "nauc_ndcg_at_100_diff1": 0.2290343646224961, + "nauc_ndcg_at_100_max": 0.441780450426168, + "nauc_ndcg_at_100_std": 0.47789482167649283, + "nauc_ndcg_at_10_diff1": 0.26575026349242603, + "nauc_ndcg_at_10_max": 0.503623708130739, + "nauc_ndcg_at_10_std": 0.505926394732924, + "nauc_ndcg_at_1_diff1": 0.4321169841695732, + "nauc_ndcg_at_1_max": 0.5775298373232366, + "nauc_ndcg_at_1_std": 0.5090480875294399, + "nauc_ndcg_at_20_diff1": 0.2538508507635941, + "nauc_ndcg_at_20_max": 0.48529750684276435, + "nauc_ndcg_at_20_std": 0.49782418515803956, + "nauc_ndcg_at_3_diff1": 0.32497674728583975, + "nauc_ndcg_at_3_max": 0.5324472658517485, + "nauc_ndcg_at_3_std": 0.5216821129287026, + "nauc_ndcg_at_5_diff1": 0.28955414564042403, + "nauc_ndcg_at_5_max": 0.5168867804137073, + "nauc_ndcg_at_5_std": 0.5148896151162353, + "nauc_precision_at_1000_diff1": 0.0958247690253974, + "nauc_precision_at_1000_max": 0.2792915575597295, + "nauc_precision_at_1000_std": 0.35209413121999733, + "nauc_precision_at_100_diff1": 0.16728310795758933, + "nauc_precision_at_100_max": 0.3738964111076925, + "nauc_precision_at_100_std": 0.45023838508861597, + "nauc_precision_at_10_diff1": 0.19929435295839446, + "nauc_precision_at_10_max": 0.46857517551499467, + "nauc_precision_at_10_std": 0.49694956158915504, + "nauc_precision_at_1_diff1": 0.4321169841695732, + "nauc_precision_at_1_max": 0.5775298373232366, + "nauc_precision_at_1_std": 0.5090480875294399, + "nauc_precision_at_20_diff1": 0.19139121773360634, + "nauc_precision_at_20_max": 0.4393900331688631, + "nauc_precision_at_20_std": 0.48177970351226623, + "nauc_precision_at_3_diff1": 0.2756104105178425, + "nauc_precision_at_3_max": 0.5093405406820367, + "nauc_precision_at_3_std": 0.5261198767072187, + "nauc_precision_at_5_diff1": 0.227910334902554, + "nauc_precision_at_5_max": 0.48768348903578324, + "nauc_precision_at_5_std": 0.5129419225797013, + "nauc_recall_at_1000_diff1": 0.09582476902539795, + "nauc_recall_at_1000_max": 0.27929155755972995, + "nauc_recall_at_1000_std": 0.3520941312199978, + "nauc_recall_at_100_diff1": 0.16728310795758913, + "nauc_recall_at_100_max": 0.37389641110769223, + "nauc_recall_at_100_std": 0.45023838508861586, + "nauc_recall_at_10_diff1": 0.19929435295839448, + "nauc_recall_at_10_max": 0.46857517551499467, + "nauc_recall_at_10_std": 0.4969495615891551, + "nauc_recall_at_1_diff1": 0.4321169841695732, + "nauc_recall_at_1_max": 0.5775298373232366, + "nauc_recall_at_1_std": 0.5090480875294399, + "nauc_recall_at_20_diff1": 0.1913912177336063, + "nauc_recall_at_20_max": 0.4393900331688629, + "nauc_recall_at_20_std": 0.4817797035122659, + "nauc_recall_at_3_diff1": 0.2756104105178428, + "nauc_recall_at_3_max": 0.5093405406820369, + "nauc_recall_at_3_std": 0.526119876707219, + "nauc_recall_at_5_diff1": 0.22791033490255386, + "nauc_recall_at_5_max": 0.4876834890357833, + "nauc_recall_at_5_std": 0.512941922579701, + "ndcg_at_1": 0.01484, + "ndcg_at_10": 0.03088, + "ndcg_at_100": 0.04805, + "ndcg_at_1000": 0.08534, + "ndcg_at_20": 0.03556, + "ndcg_at_3": 0.02259, + "ndcg_at_5": 0.02654, + "precision_at_1": 0.01484, + "precision_at_10": 0.00514, + "precision_at_100": 0.0014, + "precision_at_1000": 0.00046, + "precision_at_20": 0.00351, + "precision_at_3": 0.00939, + "precision_at_5": 0.00755, + "recall_at_1": 0.01484, + "recall_at_10": 0.05137, + "recall_at_100": 0.14038, + "recall_at_1000": 0.45989, + "recall_at_20": 0.07019, + "recall_at_3": 0.02816, + "recall_at_5": 0.03777 + }, + { + "cv_recall_at_1": 0.01677, + "cv_recall_at_10": 0.05058, + "cv_recall_at_100": 0.11572, + "cv_recall_at_1000": 0.40133, + "cv_recall_at_20": 0.06569, + "cv_recall_at_3": 0.03063, + "cv_recall_at_5": 0.03922, + "hf_subset": "hu", + "languages": [ + "hun-Latn" + ], + "main_score": 0.03203, + "map_at_1": 0.01677, + "map_at_10": 0.02629, + "map_at_100": 0.02846, + "map_at_1000": 0.02922, + "map_at_20": 0.02736, + "map_at_3": 0.02284, + "map_at_5": 0.02477, + "mrr_at_1": 0.01676829268292683, + "mrr_at_10": 0.02629199222187025, + "mrr_at_100": 0.028456210816696165, + "mrr_at_1000": 0.029224241038230778, + "mrr_at_20": 0.027362670619909213, + "mrr_at_3": 0.022842756836659275, + "mrr_at_5": 0.024769031781226896, + "nauc_cv_recall_at_1000_diff1": 0.07926305037393254, + "nauc_cv_recall_at_1000_max": 0.20067504942111922, + "nauc_cv_recall_at_1000_std": 0.2843937312557254, + "nauc_cv_recall_at_100_diff1": 0.1787803597914015, + "nauc_cv_recall_at_100_max": 0.40116950225577536, + "nauc_cv_recall_at_100_std": 0.524557608239799, + "nauc_cv_recall_at_10_diff1": 0.24628444574025682, + "nauc_cv_recall_at_10_max": 0.5619038337044497, + "nauc_cv_recall_at_10_std": 0.6709807774465408, + "nauc_cv_recall_at_1_diff1": 0.4976826031932956, + "nauc_cv_recall_at_1_max": 0.6802883947403277, + "nauc_cv_recall_at_1_std": 0.7427794453984465, + "nauc_cv_recall_at_20_diff1": 0.2206405756030709, + "nauc_cv_recall_at_20_max": 0.51654787140732, + "nauc_cv_recall_at_20_std": 0.6405806955555178, + "nauc_cv_recall_at_3_diff1": 0.3431538514634977, + "nauc_cv_recall_at_3_max": 0.6245233178374265, + "nauc_cv_recall_at_3_std": 0.7011957839961755, + "nauc_cv_recall_at_5_diff1": 0.29096232825573, + "nauc_cv_recall_at_5_max": 0.6048474626697937, + "nauc_cv_recall_at_5_std": 0.7052704818228328, + "nauc_map_at_1000_diff1": 0.34279126451861025, + "nauc_map_at_1000_max": 0.5960966964779011, + "nauc_map_at_1000_std": 0.6841950163672862, + "nauc_map_at_100_diff1": 0.34720535115453793, + "nauc_map_at_100_max": 0.6029773348188909, + "nauc_map_at_100_std": 0.6913385789082904, + "nauc_map_at_10_diff1": 0.36511963754672155, + "nauc_map_at_10_max": 0.6268488465681822, + "nauc_map_at_10_std": 0.7095996529439429, + "nauc_map_at_1_diff1": 0.4976826031932956, + "nauc_map_at_1_max": 0.6802883947403277, + "nauc_map_at_1_std": 0.7427794453984465, + "nauc_map_at_20_diff1": 0.3564859418420348, + "nauc_map_at_20_max": 0.6173916874697682, + "nauc_map_at_20_std": 0.7031393212610639, + "nauc_map_at_3_diff1": 0.4065773968949888, + "nauc_map_at_3_max": 0.6474856419723101, + "nauc_map_at_3_std": 0.7184473993852785, + "nauc_map_at_5_diff1": 0.381601324684726, + "nauc_map_at_5_max": 0.6388029077117123, + "nauc_map_at_5_std": 0.7182565082647873, + "nauc_mrr_at_1000_diff1": 0.34279126451861025, + "nauc_mrr_at_1000_max": 0.5960966964779011, + "nauc_mrr_at_1000_std": 0.6841950163672862, + "nauc_mrr_at_100_diff1": 0.34720535115453793, + "nauc_mrr_at_100_max": 0.6029773348188909, + "nauc_mrr_at_100_std": 0.6913385789082904, + "nauc_mrr_at_10_diff1": 0.36511963754672155, + "nauc_mrr_at_10_max": 0.6268488465681822, + "nauc_mrr_at_10_std": 0.7095996529439429, + "nauc_mrr_at_1_diff1": 0.4976826031932956, + "nauc_mrr_at_1_max": 0.6802883947403277, + "nauc_mrr_at_1_std": 0.7427794453984465, + "nauc_mrr_at_20_diff1": 0.3564859418420348, + "nauc_mrr_at_20_max": 0.6173916874697682, + "nauc_mrr_at_20_std": 0.7031393212610639, + "nauc_mrr_at_3_diff1": 0.4065773968949888, + "nauc_mrr_at_3_max": 0.6474856419723101, + "nauc_mrr_at_3_std": 0.7184473993852785, + "nauc_mrr_at_5_diff1": 0.381601324684726, + "nauc_mrr_at_5_max": 0.6388029077117123, + "nauc_mrr_at_5_std": 0.7182565082647873, + "nauc_ndcg_at_1000_diff1": 0.2043031642322968, + "nauc_ndcg_at_1000_max": 0.40307981800072007, + "nauc_ndcg_at_1000_std": 0.4961372678753577, + "nauc_ndcg_at_100_diff1": 0.2667995253569192, + "nauc_ndcg_at_100_max": 0.5144267445806551, + "nauc_ndcg_at_100_std": 0.620689438403104, + "nauc_ndcg_at_10_diff1": 0.3210474233046296, + "nauc_ndcg_at_10_max": 0.6036012095205587, + "nauc_ndcg_at_10_std": 0.6958951414348914, + "nauc_ndcg_at_1_diff1": 0.4976826031932956, + "nauc_ndcg_at_1_max": 0.6802883947403277, + "nauc_ndcg_at_1_std": 0.7427794453984465, + "nauc_ndcg_at_20_diff1": 0.3014415857234393, + "nauc_ndcg_at_20_max": 0.5787442865570296, + "nauc_ndcg_at_20_std": 0.6792556751813358, + "nauc_ndcg_at_3_diff1": 0.3864969421729367, + "nauc_ndcg_at_3_max": 0.640218475645307, + "nauc_ndcg_at_3_std": 0.7129925772036456, + "nauc_ndcg_at_5_diff1": 0.3505090436279808, + "nauc_ndcg_at_5_max": 0.6272360189711365, + "nauc_ndcg_at_5_std": 0.7136074643615019, + "nauc_precision_at_1000_diff1": 0.07926305037393205, + "nauc_precision_at_1000_max": 0.20067504942111877, + "nauc_precision_at_1000_std": 0.284393731255725, + "nauc_precision_at_100_diff1": 0.1787803597914016, + "nauc_precision_at_100_max": 0.40116950225577563, + "nauc_precision_at_100_std": 0.524557608239799, + "nauc_precision_at_10_diff1": 0.24628444574025693, + "nauc_precision_at_10_max": 0.5619038337044496, + "nauc_precision_at_10_std": 0.6709807774465409, + "nauc_precision_at_1_diff1": 0.4976826031932956, + "nauc_precision_at_1_max": 0.6802883947403277, + "nauc_precision_at_1_std": 0.7427794453984465, + "nauc_precision_at_20_diff1": 0.22064057560307082, + "nauc_precision_at_20_max": 0.51654787140732, + "nauc_precision_at_20_std": 0.6405806955555178, + "nauc_precision_at_3_diff1": 0.34315385146349775, + "nauc_precision_at_3_max": 0.6245233178374265, + "nauc_precision_at_3_std": 0.7011957839961754, + "nauc_precision_at_5_diff1": 0.2909623282557299, + "nauc_precision_at_5_max": 0.6048474626697936, + "nauc_precision_at_5_std": 0.7052704818228327, + "nauc_recall_at_1000_diff1": 0.07926305037393254, + "nauc_recall_at_1000_max": 0.20067504942111922, + "nauc_recall_at_1000_std": 0.2843937312557254, + "nauc_recall_at_100_diff1": 0.1787803597914015, + "nauc_recall_at_100_max": 0.40116950225577536, + "nauc_recall_at_100_std": 0.524557608239799, + "nauc_recall_at_10_diff1": 0.24628444574025682, + "nauc_recall_at_10_max": 0.5619038337044497, + "nauc_recall_at_10_std": 0.6709807774465408, + "nauc_recall_at_1_diff1": 0.4976826031932956, + "nauc_recall_at_1_max": 0.6802883947403277, + "nauc_recall_at_1_std": 0.7427794453984465, + "nauc_recall_at_20_diff1": 0.2206405756030709, + "nauc_recall_at_20_max": 0.51654787140732, + "nauc_recall_at_20_std": 0.6405806955555178, + "nauc_recall_at_3_diff1": 0.3431538514634977, + "nauc_recall_at_3_max": 0.6245233178374265, + "nauc_recall_at_3_std": 0.7011957839961755, + "nauc_recall_at_5_diff1": 0.29096232825573, + "nauc_recall_at_5_max": 0.6048474626697937, + "nauc_recall_at_5_std": 0.7052704818228328, + "ndcg_at_1": 0.01677, + "ndcg_at_10": 0.03203, + "ndcg_at_100": 0.04477, + "ndcg_at_1000": 0.07744, + "ndcg_at_20": 0.03587, + "ndcg_at_3": 0.02484, + "ndcg_at_5": 0.02835, + "precision_at_1": 0.01677, + "precision_at_10": 0.00506, + "precision_at_100": 0.00116, + "precision_at_1000": 0.0004, + "precision_at_20": 0.00328, + "precision_at_3": 0.01021, + "precision_at_5": 0.00784, + "recall_at_1": 0.01677, + "recall_at_10": 0.05058, + "recall_at_100": 0.11572, + "recall_at_1000": 0.40133, + "recall_at_20": 0.06569, + "recall_at_3": 0.03063, + "recall_at_5": 0.03922 + }, + { + "cv_recall_at_1": 0.05726, + "cv_recall_at_10": 0.1691, + "cv_recall_at_100": 0.38675, + "cv_recall_at_1000": 0.76172, + "cv_recall_at_20": 0.22214, + "cv_recall_at_3": 0.10342, + "cv_recall_at_5": 0.12939, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.10767, + "map_at_1": 0.05726, + "map_at_10": 0.08873, + "map_at_100": 0.09617, + "map_at_1000": 0.09752, + "map_at_20": 0.09236, + "map_at_3": 0.07758, + "map_at_5": 0.08348, + "mrr_at_1": 0.05725512208812798, + "mrr_at_10": 0.08872622722959524, + "mrr_at_100": 0.09617481516990756, + "mrr_at_1000": 0.09752354332896494, + "mrr_at_20": 0.09236356409778489, + "mrr_at_3": 0.07757975488820275, + "mrr_at_5": 0.08348068107400132, + "nauc_cv_recall_at_1000_diff1": 0.15030287323588018, + "nauc_cv_recall_at_1000_max": 0.3191062141405888, + "nauc_cv_recall_at_1000_std": 0.4750670986169835, + "nauc_cv_recall_at_100_diff1": 0.16397068588078975, + "nauc_cv_recall_at_100_max": 0.28952305269069245, + "nauc_cv_recall_at_100_std": 0.3689023094107915, + "nauc_cv_recall_at_10_diff1": 0.19309923383240687, + "nauc_cv_recall_at_10_max": 0.3141285805092174, + "nauc_cv_recall_at_10_std": 0.29483488469475855, + "nauc_cv_recall_at_1_diff1": 0.40092103200855395, + "nauc_cv_recall_at_1_max": 0.4284863429779795, + "nauc_cv_recall_at_1_std": 0.28322408485037104, + "nauc_cv_recall_at_20_diff1": 0.1615483611160991, + "nauc_cv_recall_at_20_max": 0.2894060506254236, + "nauc_cv_recall_at_20_std": 0.3123395488443825, + "nauc_cv_recall_at_3_diff1": 0.2672418202383049, + "nauc_cv_recall_at_3_max": 0.34414828955915494, + "nauc_cv_recall_at_3_std": 0.2533006977882842, + "nauc_cv_recall_at_5_diff1": 0.22732741825710243, + "nauc_cv_recall_at_5_max": 0.3295891147074863, + "nauc_cv_recall_at_5_std": 0.28159355610020537, + "nauc_map_at_1000_diff1": 0.2868239638816492, + "nauc_map_at_1000_max": 0.3617561081540899, + "nauc_map_at_1000_std": 0.2831757076010107, + "nauc_map_at_100_diff1": 0.2868253956883315, + "nauc_map_at_100_max": 0.36157898123651255, + "nauc_map_at_100_std": 0.28267515257161274, + "nauc_map_at_10_diff1": 0.29513212149528517, + "nauc_map_at_10_max": 0.3676838581354323, + "nauc_map_at_10_std": 0.2768791058387735, + "nauc_map_at_1_diff1": 0.40092103200855395, + "nauc_map_at_1_max": 0.4284863429779795, + "nauc_map_at_1_std": 0.28322408485037104, + "nauc_map_at_20_diff1": 0.2884731293833199, + "nauc_map_at_20_max": 0.3628246225023507, + "nauc_map_at_20_std": 0.2794205983087937, + "nauc_map_at_3_diff1": 0.3220621170390292, + "nauc_map_at_3_max": 0.38001448681516115, + "nauc_map_at_3_std": 0.2653154290950081, + "nauc_map_at_5_diff1": 0.306102269865566, + "nauc_map_at_5_max": 0.3727343689500744, + "nauc_map_at_5_std": 0.2740171214843683, + "nauc_mrr_at_1000_diff1": 0.28682396238682445, + "nauc_mrr_at_1000_max": 0.3617561074098036, + "nauc_mrr_at_1000_std": 0.28317571035510314, + "nauc_mrr_at_100_diff1": 0.2868253956883315, + "nauc_mrr_at_100_max": 0.36157898123651255, + "nauc_mrr_at_100_std": 0.28267515257161274, + "nauc_mrr_at_10_diff1": 0.29513212149528517, + "nauc_mrr_at_10_max": 0.3676838581354323, + "nauc_mrr_at_10_std": 0.2768791058387735, + "nauc_mrr_at_1_diff1": 0.40092103200855395, + "nauc_mrr_at_1_max": 0.4284863429779795, + "nauc_mrr_at_1_std": 0.28322408485037104, + "nauc_mrr_at_20_diff1": 0.2884731293833199, + "nauc_mrr_at_20_max": 0.3628246225023507, + "nauc_mrr_at_20_std": 0.2794205983087937, + "nauc_mrr_at_3_diff1": 0.3220621170390292, + "nauc_mrr_at_3_max": 0.38001448681516115, + "nauc_mrr_at_3_std": 0.2653154290950081, + "nauc_mrr_at_5_diff1": 0.306102269865566, + "nauc_mrr_at_5_max": 0.3727343689500744, + "nauc_mrr_at_5_std": 0.2740171214843683, + "nauc_ndcg_at_1000_diff1": 0.24161348000786495, + "nauc_ndcg_at_1000_max": 0.34037403947611655, + "nauc_ndcg_at_1000_std": 0.32039258837486695, + "nauc_ndcg_at_100_diff1": 0.23785912449290542, + "nauc_ndcg_at_100_max": 0.3328042419327395, + "nauc_ndcg_at_100_std": 0.3098300951168714, + "nauc_ndcg_at_10_diff1": 0.25966152812590537, + "nauc_ndcg_at_10_max": 0.348885270202277, + "nauc_ndcg_at_10_std": 0.28238321943430694, + "nauc_ndcg_at_1_diff1": 0.40092103200855395, + "nauc_ndcg_at_1_max": 0.4284863429779795, + "nauc_ndcg_at_1_std": 0.28322408485037104, + "nauc_ndcg_at_20_diff1": 0.24229561650784148, + "nauc_ndcg_at_20_max": 0.3361781384678798, + "nauc_ndcg_at_20_std": 0.2901134461141482, + "nauc_ndcg_at_3_diff1": 0.30481794493109293, + "nauc_ndcg_at_3_max": 0.3687831617083144, + "nauc_ndcg_at_3_std": 0.2615259423663829, + "nauc_ndcg_at_5_diff1": 0.2803326002641599, + "nauc_ndcg_at_5_max": 0.3584572010028402, + "nauc_ndcg_at_5_std": 0.27604262728435447, + "nauc_precision_at_1000_diff1": 0.15030287323587857, + "nauc_precision_at_1000_max": 0.3191062141405875, + "nauc_precision_at_1000_std": 0.47506709861698254, + "nauc_precision_at_100_diff1": 0.16397068588079, + "nauc_precision_at_100_max": 0.2895230526906928, + "nauc_precision_at_100_std": 0.3689023094107917, + "nauc_precision_at_10_diff1": 0.19309923383240699, + "nauc_precision_at_10_max": 0.31412858050921744, + "nauc_precision_at_10_std": 0.29483488469475866, + "nauc_precision_at_1_diff1": 0.40092103200855395, + "nauc_precision_at_1_max": 0.4284863429779795, + "nauc_precision_at_1_std": 0.28322408485037104, + "nauc_precision_at_20_diff1": 0.16154836111609902, + "nauc_precision_at_20_max": 0.28940605062542357, + "nauc_precision_at_20_std": 0.3123395488443823, + "nauc_precision_at_3_diff1": 0.2672418202383047, + "nauc_precision_at_3_max": 0.3441482895591548, + "nauc_precision_at_3_std": 0.2533006977882841, + "nauc_precision_at_5_diff1": 0.22732741825710256, + "nauc_precision_at_5_max": 0.32958911470748636, + "nauc_precision_at_5_std": 0.28159355610020526, + "nauc_recall_at_1000_diff1": 0.15030287323588018, + "nauc_recall_at_1000_max": 0.3191062141405888, + "nauc_recall_at_1000_std": 0.4750670986169835, + "nauc_recall_at_100_diff1": 0.16397068588078975, + "nauc_recall_at_100_max": 0.28952305269069245, + "nauc_recall_at_100_std": 0.3689023094107915, + "nauc_recall_at_10_diff1": 0.19309923383240687, + "nauc_recall_at_10_max": 0.3141285805092174, + "nauc_recall_at_10_std": 0.29483488469475855, + "nauc_recall_at_1_diff1": 0.40092103200855395, + "nauc_recall_at_1_max": 0.4284863429779795, + "nauc_recall_at_1_std": 0.28322408485037104, + "nauc_recall_at_20_diff1": 0.1615483611160991, + "nauc_recall_at_20_max": 0.2894060506254236, + "nauc_recall_at_20_std": 0.3123395488443825, + "nauc_recall_at_3_diff1": 0.2672418202383049, + "nauc_recall_at_3_max": 0.34414828955915494, + "nauc_recall_at_3_std": 0.2533006977882842, + "nauc_recall_at_5_diff1": 0.22732741825710243, + "nauc_recall_at_5_max": 0.3295891147074863, + "nauc_recall_at_5_std": 0.28159355610020537, + "ndcg_at_1": 0.05726, + "ndcg_at_10": 0.10767, + "ndcg_at_100": 0.15073, + "ndcg_at_1000": 0.19579, + "ndcg_at_20": 0.12102, + "ndcg_at_3": 0.08422, + "ndcg_at_5": 0.09488, + "precision_at_1": 0.05726, + "precision_at_10": 0.01691, + "precision_at_100": 0.00387, + "precision_at_1000": 0.00076, + "precision_at_20": 0.01111, + "precision_at_3": 0.03447, + "precision_at_5": 0.02588, + "recall_at_1": 0.05726, + "recall_at_10": 0.1691, + "recall_at_100": 0.38675, + "recall_at_1000": 0.76172, + "recall_at_20": 0.22214, + "recall_at_3": 0.10342, + "recall_at_5": 0.12939 + }, + { + "cv_recall_at_1": 0.08346, + "cv_recall_at_10": 0.25369, + "cv_recall_at_100": 0.52331, + "cv_recall_at_1000": 0.85763, + "cv_recall_at_20": 0.32523, + "cv_recall_at_3": 0.1504, + "cv_recall_at_5": 0.18841, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ], + "main_score": 0.15843, + "map_at_1": 0.08346, + "map_at_10": 0.12922, + "map_at_100": 0.13889, + "map_at_1000": 0.14021, + "map_at_20": 0.13422, + "map_at_3": 0.11191, + "map_at_5": 0.12055, + "mrr_at_1": 0.08346122063510801, + "mrr_at_10": 0.12921868822293828, + "mrr_at_100": 0.13888512076542645, + "mrr_at_1000": 0.14020972512942267, + "mrr_at_20": 0.13422039590574278, + "mrr_at_3": 0.11191122653759927, + "mrr_at_5": 0.12055247314366743, + "nauc_cv_recall_at_1000_diff1": 0.06115378612995843, + "nauc_cv_recall_at_1000_max": 0.2198677467029866, + "nauc_cv_recall_at_1000_std": 0.5562687441684824, + "nauc_cv_recall_at_100_diff1": 0.09190227414493322, + "nauc_cv_recall_at_100_max": 0.14206614111453056, + "nauc_cv_recall_at_100_std": 0.35163639320895096, + "nauc_cv_recall_at_10_diff1": 0.13034473302850733, + "nauc_cv_recall_at_10_max": 0.15694257987261587, + "nauc_cv_recall_at_10_std": 0.22673453590977974, + "nauc_cv_recall_at_1_diff1": 0.28469477006474864, + "nauc_cv_recall_at_1_max": 0.24277500552741196, + "nauc_cv_recall_at_1_std": 0.14989170009299607, + "nauc_cv_recall_at_20_diff1": 0.1139533115734007, + "nauc_cv_recall_at_20_max": 0.15001100076217513, + "nauc_cv_recall_at_20_std": 0.24987924114844598, + "nauc_cv_recall_at_3_diff1": 0.1723560568226035, + "nauc_cv_recall_at_3_max": 0.1899520873785719, + "nauc_cv_recall_at_3_std": 0.16313589784115362, + "nauc_cv_recall_at_5_diff1": 0.14723328888291767, + "nauc_cv_recall_at_5_max": 0.17217984968931596, + "nauc_cv_recall_at_5_std": 0.18099972391070385, + "nauc_map_at_1000_diff1": 0.2032600423187777, + "nauc_map_at_1000_max": 0.20174194367014686, + "nauc_map_at_1000_std": 0.17851924731958435, + "nauc_map_at_100_diff1": 0.2029155467410285, + "nauc_map_at_100_max": 0.20116705824454936, + "nauc_map_at_100_std": 0.17824653925030834, + "nauc_map_at_10_diff1": 0.20648651750802421, + "nauc_map_at_10_max": 0.203185940142513, + "nauc_map_at_10_std": 0.1707974532854906, + "nauc_map_at_1_diff1": 0.28469477006474864, + "nauc_map_at_1_max": 0.24277500552741196, + "nauc_map_at_1_std": 0.14989170009299607, + "nauc_map_at_20_diff1": 0.2034704607134265, + "nauc_map_at_20_max": 0.20159871772025179, + "nauc_map_at_20_std": 0.1739055372628376, + "nauc_map_at_3_diff1": 0.22302423083706688, + "nauc_map_at_3_max": 0.21488035339192169, + "nauc_map_at_3_std": 0.15477364907074237, + "nauc_map_at_5_diff1": 0.2121914715975426, + "nauc_map_at_5_max": 0.20762170454855294, + "nauc_map_at_5_std": 0.1610095680454263, + "nauc_mrr_at_1000_diff1": 0.2032600423187777, + "nauc_mrr_at_1000_max": 0.20174194367014686, + "nauc_mrr_at_1000_std": 0.17851924731958435, + "nauc_mrr_at_100_diff1": 0.2029155467410285, + "nauc_mrr_at_100_max": 0.20116705824454936, + "nauc_mrr_at_100_std": 0.17824653925030834, + "nauc_mrr_at_10_diff1": 0.20648651750802421, + "nauc_mrr_at_10_max": 0.203185940142513, + "nauc_mrr_at_10_std": 0.1707974532854906, + "nauc_mrr_at_1_diff1": 0.28469477006474864, + "nauc_mrr_at_1_max": 0.24277500552741196, + "nauc_mrr_at_1_std": 0.14989170009299607, + "nauc_mrr_at_20_diff1": 0.2034704607134265, + "nauc_mrr_at_20_max": 0.20159871772025179, + "nauc_mrr_at_20_std": 0.1739055372628376, + "nauc_mrr_at_3_diff1": 0.22302423083706688, + "nauc_mrr_at_3_max": 0.21488035339192169, + "nauc_mrr_at_3_std": 0.15477364907074237, + "nauc_mrr_at_5_diff1": 0.2121914715975426, + "nauc_mrr_at_5_max": 0.20762170454855294, + "nauc_mrr_at_5_std": 0.1610095680454263, + "nauc_ndcg_at_1000_diff1": 0.17430403953515997, + "nauc_ndcg_at_1000_max": 0.19342891016794667, + "nauc_ndcg_at_1000_std": 0.22824905385496075, + "nauc_ndcg_at_100_diff1": 0.1676830441466766, + "nauc_ndcg_at_100_max": 0.1817532485004325, + "nauc_ndcg_at_100_std": 0.2249159052021771, + "nauc_ndcg_at_10_diff1": 0.18074395377365038, + "nauc_ndcg_at_10_max": 0.1877903243250276, + "nauc_ndcg_at_10_std": 0.1882740651781376, + "nauc_ndcg_at_1_diff1": 0.28469477006474864, + "nauc_ndcg_at_1_max": 0.24277500552741196, + "nauc_ndcg_at_1_std": 0.14989170009299607, + "nauc_ndcg_at_20_diff1": 0.17271785778675547, + "nauc_ndcg_at_20_max": 0.18391476422554845, + "nauc_ndcg_at_20_std": 0.19791156758163084, + "nauc_ndcg_at_3_diff1": 0.20761307528172798, + "nauc_ndcg_at_3_max": 0.2073413273262493, + "nauc_ndcg_at_3_std": 0.15722376432472251, + "nauc_ndcg_at_5_diff1": 0.1912682684291303, + "nauc_ndcg_at_5_max": 0.19628937163214283, + "nauc_ndcg_at_5_std": 0.1672009773360082, + "nauc_precision_at_1000_diff1": 0.06115378612995736, + "nauc_precision_at_1000_max": 0.21986774670298528, + "nauc_precision_at_1000_std": 0.556268744168481, + "nauc_precision_at_100_diff1": 0.09190227414493372, + "nauc_precision_at_100_max": 0.14206614111453086, + "nauc_precision_at_100_std": 0.35163639320895096, + "nauc_precision_at_10_diff1": 0.13034473302850733, + "nauc_precision_at_10_max": 0.15694257987261595, + "nauc_precision_at_10_std": 0.22673453590977952, + "nauc_precision_at_1_diff1": 0.28469477006474864, + "nauc_precision_at_1_max": 0.24277500552741196, + "nauc_precision_at_1_std": 0.14989170009299607, + "nauc_precision_at_20_diff1": 0.11395331157340065, + "nauc_precision_at_20_max": 0.15001100076217486, + "nauc_precision_at_20_std": 0.24987924114844567, + "nauc_precision_at_3_diff1": 0.1723560568226034, + "nauc_precision_at_3_max": 0.1899520873785718, + "nauc_precision_at_3_std": 0.16313589784115368, + "nauc_precision_at_5_diff1": 0.1472332888829176, + "nauc_precision_at_5_max": 0.17217984968931588, + "nauc_precision_at_5_std": 0.1809997239107037, + "nauc_recall_at_1000_diff1": 0.06115378612995843, + "nauc_recall_at_1000_max": 0.2198677467029866, + "nauc_recall_at_1000_std": 0.5562687441684824, + "nauc_recall_at_100_diff1": 0.09190227414493322, + "nauc_recall_at_100_max": 0.14206614111453056, + "nauc_recall_at_100_std": 0.35163639320895096, + "nauc_recall_at_10_diff1": 0.13034473302850733, + "nauc_recall_at_10_max": 0.15694257987261587, + "nauc_recall_at_10_std": 0.22673453590977974, + "nauc_recall_at_1_diff1": 0.28469477006474864, + "nauc_recall_at_1_max": 0.24277500552741196, + "nauc_recall_at_1_std": 0.14989170009299607, + "nauc_recall_at_20_diff1": 0.1139533115734007, + "nauc_recall_at_20_max": 0.15001100076217513, + "nauc_recall_at_20_std": 0.24987924114844598, + "nauc_recall_at_3_diff1": 0.1723560568226035, + "nauc_recall_at_3_max": 0.1899520873785719, + "nauc_recall_at_3_std": 0.16313589784115362, + "nauc_recall_at_5_diff1": 0.14723328888291767, + "nauc_recall_at_5_max": 0.17217984968931596, + "nauc_recall_at_5_std": 0.18099972391070385, + "ndcg_at_1": 0.08346, + "ndcg_at_10": 0.15843, + "ndcg_at_100": 0.21249, + "ndcg_at_1000": 0.25338, + "ndcg_at_20": 0.17656, + "ndcg_at_3": 0.12175, + "ndcg_at_5": 0.13737, + "precision_at_1": 0.08346, + "precision_at_10": 0.02537, + "precision_at_100": 0.00523, + "precision_at_1000": 0.00086, + "precision_at_20": 0.01626, + "precision_at_3": 0.05013, + "precision_at_5": 0.03768, + "recall_at_1": 0.08346, + "recall_at_10": 0.25369, + "recall_at_100": 0.52331, + "recall_at_1000": 0.85763, + "recall_at_20": 0.32523, + "recall_at_3": 0.1504, + "recall_at_5": 0.18841 + }, + { + "cv_recall_at_1": 0.01754, + "cv_recall_at_10": 0.07474, + "cv_recall_at_100": 0.25678, + "cv_recall_at_1000": 0.67794, + "cv_recall_at_20": 0.11162, + "cv_recall_at_3": 0.03674, + "cv_recall_at_5": 0.05177, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.04196, + "map_at_1": 0.01754, + "map_at_10": 0.03201, + "map_at_100": 0.03789, + "map_at_1000": 0.03937, + "map_at_20": 0.03456, + "map_at_3": 0.02568, + "map_at_5": 0.02904, + "mrr_at_1": 0.01753653444676409, + "mrr_at_10": 0.03200936695717485, + "mrr_at_100": 0.03788702657630412, + "mrr_at_1000": 0.03936756710182338, + "mrr_at_20": 0.0345623431663912, + "mrr_at_3": 0.025678496868476012, + "mrr_at_5": 0.029039665970772448, + "nauc_cv_recall_at_1000_diff1": 0.10757708129019053, + "nauc_cv_recall_at_1000_max": 0.2961920238707485, + "nauc_cv_recall_at_1000_std": 0.4776809922444095, + "nauc_cv_recall_at_100_diff1": 0.09135429300688364, + "nauc_cv_recall_at_100_max": 0.22092085853045743, + "nauc_cv_recall_at_100_std": 0.39170988106526494, + "nauc_cv_recall_at_10_diff1": 0.11416323917396812, + "nauc_cv_recall_at_10_max": 0.21204992762585068, + "nauc_cv_recall_at_10_std": 0.32864092293250496, + "nauc_cv_recall_at_1_diff1": 0.29466646698439264, + "nauc_cv_recall_at_1_max": 0.29818456527762005, + "nauc_cv_recall_at_1_std": 0.29694254864600017, + "nauc_cv_recall_at_20_diff1": 0.09575980053421915, + "nauc_cv_recall_at_20_max": 0.20513323624093097, + "nauc_cv_recall_at_20_std": 0.35369955523262436, + "nauc_cv_recall_at_3_diff1": 0.16073635724839264, + "nauc_cv_recall_at_3_max": 0.2627261073098647, + "nauc_cv_recall_at_3_std": 0.3374073837314188, + "nauc_cv_recall_at_5_diff1": 0.16451856690252917, + "nauc_cv_recall_at_5_max": 0.2463348129018922, + "nauc_cv_recall_at_5_std": 0.34402122105642435, + "nauc_map_at_1000_diff1": 0.17404810974414622, + "nauc_map_at_1000_max": 0.2511168768175765, + "nauc_map_at_1000_std": 0.32791949498308215, + "nauc_map_at_100_diff1": 0.17408071817208495, + "nauc_map_at_100_max": 0.25012756041114087, + "nauc_map_at_100_std": 0.3262846189521321, + "nauc_map_at_10_diff1": 0.18821696482519734, + "nauc_map_at_10_max": 0.257032447064839, + "nauc_map_at_10_std": 0.3172818065044762, + "nauc_map_at_1_diff1": 0.29466646698439264, + "nauc_map_at_1_max": 0.29818456527762005, + "nauc_map_at_1_std": 0.29694254864600017, + "nauc_map_at_20_diff1": 0.17937160474901026, + "nauc_map_at_20_max": 0.2520240041798804, + "nauc_map_at_20_std": 0.32251538668153346, + "nauc_map_at_3_diff1": 0.21001530584534686, + "nauc_map_at_3_max": 0.278535322608543, + "nauc_map_at_3_std": 0.31651830882096854, + "nauc_map_at_5_diff1": 0.20592982486101274, + "nauc_map_at_5_max": 0.2700919615800627, + "nauc_map_at_5_std": 0.32095831129341523, + "nauc_mrr_at_1000_diff1": 0.17404810757470401, + "nauc_mrr_at_1000_max": 0.25111687217927603, + "nauc_mrr_at_1000_std": 0.32791949494354067, + "nauc_mrr_at_100_diff1": 0.17408071817208495, + "nauc_mrr_at_100_max": 0.25012756041114087, + "nauc_mrr_at_100_std": 0.3262846189521321, + "nauc_mrr_at_10_diff1": 0.18821696482519734, + "nauc_mrr_at_10_max": 0.257032447064839, + "nauc_mrr_at_10_std": 0.3172818065044762, + "nauc_mrr_at_1_diff1": 0.29466646698439264, + "nauc_mrr_at_1_max": 0.29818456527762005, + "nauc_mrr_at_1_std": 0.29694254864600017, + "nauc_mrr_at_20_diff1": 0.17937160474901026, + "nauc_mrr_at_20_max": 0.2520240041798804, + "nauc_mrr_at_20_std": 0.32251538668153346, + "nauc_mrr_at_3_diff1": 0.21001530584534686, + "nauc_mrr_at_3_max": 0.278535322608543, + "nauc_mrr_at_3_std": 0.31651830882096854, + "nauc_mrr_at_5_diff1": 0.20592982486101274, + "nauc_mrr_at_5_max": 0.2700919615800627, + "nauc_mrr_at_5_std": 0.32095831129341523, + "nauc_ndcg_at_1000_diff1": 0.1344421999066752, + "nauc_ndcg_at_1000_max": 0.25423268532569104, + "nauc_ndcg_at_1000_std": 0.3822515103732003, + "nauc_ndcg_at_100_diff1": 0.12834258133928364, + "nauc_ndcg_at_100_max": 0.23140922516545348, + "nauc_ndcg_at_100_std": 0.35678364039747357, + "nauc_ndcg_at_10_diff1": 0.1581123115368676, + "nauc_ndcg_at_10_max": 0.23894108805699588, + "nauc_ndcg_at_10_std": 0.32222987084202304, + "nauc_ndcg_at_1_diff1": 0.29466646698439264, + "nauc_ndcg_at_1_max": 0.29818456527762005, + "nauc_ndcg_at_1_std": 0.29694254864600017, + "nauc_ndcg_at_20_diff1": 0.1409982120405537, + "nauc_ndcg_at_20_max": 0.23033385189500785, + "nauc_ndcg_at_20_std": 0.3360002313034926, + "nauc_ndcg_at_3_diff1": 0.19365403664588626, + "nauc_ndcg_at_3_max": 0.2733960002514536, + "nauc_ndcg_at_3_std": 0.3232158137802028, + "nauc_ndcg_at_5_diff1": 0.19021691859943332, + "nauc_ndcg_at_5_max": 0.2614640537553007, + "nauc_ndcg_at_5_std": 0.3293040490494083, + "nauc_precision_at_1000_diff1": 0.10757708129018899, + "nauc_precision_at_1000_max": 0.29619202387074844, + "nauc_precision_at_1000_std": 0.4776809922444093, + "nauc_precision_at_100_diff1": 0.09135429300688377, + "nauc_precision_at_100_max": 0.2209208585304576, + "nauc_precision_at_100_std": 0.39170988106526494, + "nauc_precision_at_10_diff1": 0.11416323917396817, + "nauc_precision_at_10_max": 0.2120499276258509, + "nauc_precision_at_10_std": 0.3286409229325053, + "nauc_precision_at_1_diff1": 0.29466646698439264, + "nauc_precision_at_1_max": 0.29818456527762005, + "nauc_precision_at_1_std": 0.29694254864600017, + "nauc_precision_at_20_diff1": 0.09575980053421912, + "nauc_precision_at_20_max": 0.20513323624093116, + "nauc_precision_at_20_std": 0.3536995552326241, + "nauc_precision_at_3_diff1": 0.16073635724839253, + "nauc_precision_at_3_max": 0.2627261073098646, + "nauc_precision_at_3_std": 0.33740738373141876, + "nauc_precision_at_5_diff1": 0.16451856690252925, + "nauc_precision_at_5_max": 0.24633481290189227, + "nauc_precision_at_5_std": 0.34402122105642446, + "nauc_recall_at_1000_diff1": 0.10757708129019053, + "nauc_recall_at_1000_max": 0.2961920238707485, + "nauc_recall_at_1000_std": 0.4776809922444095, + "nauc_recall_at_100_diff1": 0.09135429300688364, + "nauc_recall_at_100_max": 0.22092085853045743, + "nauc_recall_at_100_std": 0.39170988106526494, + "nauc_recall_at_10_diff1": 0.11416323917396812, + "nauc_recall_at_10_max": 0.21204992762585068, + "nauc_recall_at_10_std": 0.32864092293250496, + "nauc_recall_at_1_diff1": 0.29466646698439264, + "nauc_recall_at_1_max": 0.29818456527762005, + "nauc_recall_at_1_std": 0.29694254864600017, + "nauc_recall_at_20_diff1": 0.09575980053421915, + "nauc_recall_at_20_max": 0.20513323624093097, + "nauc_recall_at_20_std": 0.35369955523262436, + "nauc_recall_at_3_diff1": 0.16073635724839264, + "nauc_recall_at_3_max": 0.2627261073098647, + "nauc_recall_at_3_std": 0.3374073837314188, + "nauc_recall_at_5_diff1": 0.16451856690252917, + "nauc_recall_at_5_max": 0.2463348129018922, + "nauc_recall_at_5_std": 0.34402122105642435, + "ndcg_at_1": 0.01754, + "ndcg_at_10": 0.04196, + "ndcg_at_100": 0.0774, + "ndcg_at_1000": 0.12779, + "ndcg_at_20": 0.05128, + "ndcg_at_3": 0.02851, + "ndcg_at_5": 0.03463, + "precision_at_1": 0.01754, + "precision_at_10": 0.00747, + "precision_at_100": 0.00257, + "precision_at_1000": 0.00068, + "precision_at_20": 0.00558, + "precision_at_3": 0.01225, + "precision_at_5": 0.01035, + "recall_at_1": 0.01754, + "recall_at_10": 0.07474, + "recall_at_100": 0.25678, + "recall_at_1000": 0.67794, + "recall_at_20": 0.11162, + "recall_at_3": 0.03674, + "recall_at_5": 0.05177 + }, + { + "cv_recall_at_1": 0.00078, + "cv_recall_at_10": 0.00405, + "cv_recall_at_100": 0.03608, + "cv_recall_at_1000": 0.30301, + "cv_recall_at_20": 0.00797, + "cv_recall_at_3": 0.0017, + "cv_recall_at_5": 0.00235, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.0021, + "map_at_1": 0.00078, + "map_at_10": 0.00152, + "map_at_100": 0.00235, + "map_at_1000": 0.00305, + "map_at_20": 0.00179, + "map_at_3": 0.00115, + "map_at_5": 0.00131, + "mrr_at_1": 0.000784313725490196, + "mrr_at_10": 0.0015218902375765124, + "mrr_at_100": 0.002352320429387841, + "mrr_at_1000": 0.0030529639414161465, + "mrr_at_20": 0.001791004357188416, + "mrr_at_3": 0.0011546840958605665, + "mrr_at_5": 0.0013050108932461874, + "nauc_cv_recall_at_1000_diff1": 0.008327387047313645, + "nauc_cv_recall_at_1000_max": 0.020073873859256595, + "nauc_cv_recall_at_1000_std": 0.044956744075964845, + "nauc_cv_recall_at_100_diff1": -0.01001893740545887, + "nauc_cv_recall_at_100_max": 0.05164234357197578, + "nauc_cv_recall_at_100_std": 0.18444138542480845, + "nauc_cv_recall_at_10_diff1": 0.04428884253529171, + "nauc_cv_recall_at_10_max": 0.18272068687952775, + "nauc_cv_recall_at_10_std": 0.28539775136969114, + "nauc_cv_recall_at_1_diff1": 0.17909847061980153, + "nauc_cv_recall_at_1_max": 0.7746176549503622, + "nauc_cv_recall_at_1_std": 0.9436544137375904, + "nauc_cv_recall_at_20_diff1": 0.09493417551144288, + "nauc_cv_recall_at_20_max": 0.15466225637461684, + "nauc_cv_recall_at_20_std": 0.23296546688542194, + "nauc_cv_recall_at_3_diff1": -0.002621204928690842, + "nauc_cv_recall_at_3_max": 0.5246331345070279, + "nauc_cv_recall_at_3_std": 0.5633320261707704, + "nauc_cv_recall_at_5_diff1": 0.012744834987925951, + "nauc_cv_recall_at_5_max": 0.3523238827773307, + "nauc_cv_recall_at_5_std": 0.5068270578063978, + "nauc_map_at_1000_diff1": 0.04187726128428896, + "nauc_map_at_1000_max": 0.25894178858660327, + "nauc_map_at_1000_std": 0.3620957970584589, + "nauc_map_at_100_diff1": 0.04651947833130733, + "nauc_map_at_100_max": 0.31442613006940473, + "nauc_map_at_100_std": 0.4353382498271757, + "nauc_map_at_10_diff1": 0.058577903756969585, + "nauc_map_at_10_max": 0.4602039446126341, + "nauc_map_at_10_std": 0.5739097636756447, + "nauc_map_at_1_diff1": 0.17909847061980153, + "nauc_map_at_1_max": 0.7746176549503622, + "nauc_map_at_1_std": 0.9436544137375904, + "nauc_map_at_20_diff1": 0.07270252474050658, + "nauc_map_at_20_max": 0.41226148400652984, + "nauc_map_at_20_std": 0.5140015311159577, + "nauc_map_at_3_diff1": 0.05353590069406826, + "nauc_map_at_3_max": 0.6316628782317711, + "nauc_map_at_3_std": 0.7147688210964356, + "nauc_map_at_5_diff1": 0.051837133080668826, + "nauc_map_at_5_max": 0.54565183991027, + "nauc_map_at_5_std": 0.6708672400517273, + "nauc_mrr_at_1000_diff1": 0.041877343672373425, + "nauc_mrr_at_1000_max": 0.2589420300194208, + "nauc_mrr_at_1000_std": 0.36209599021851346, + "nauc_mrr_at_100_diff1": 0.04651947833130733, + "nauc_mrr_at_100_max": 0.31442613006940473, + "nauc_mrr_at_100_std": 0.4353382498271757, + "nauc_mrr_at_10_diff1": 0.058577903756969585, + "nauc_mrr_at_10_max": 0.4602039446126341, + "nauc_mrr_at_10_std": 0.5739097636756447, + "nauc_mrr_at_1_diff1": 0.17909847061980153, + "nauc_mrr_at_1_max": 0.7746176549503622, + "nauc_mrr_at_1_std": 0.9436544137375904, + "nauc_mrr_at_20_diff1": 0.07270252474050658, + "nauc_mrr_at_20_max": 0.41226148400652984, + "nauc_mrr_at_20_std": 0.5140015311159577, + "nauc_mrr_at_3_diff1": 0.05353590069406826, + "nauc_mrr_at_3_max": 0.6316628782317711, + "nauc_mrr_at_3_std": 0.7147688210964356, + "nauc_mrr_at_5_diff1": 0.051837133080668826, + "nauc_mrr_at_5_max": 0.54565183991027, + "nauc_mrr_at_5_std": 0.6708672400517273, + "nauc_ndcg_at_1000_diff1": 0.011827898354997757, + "nauc_ndcg_at_1000_max": 0.0436259867857809, + "nauc_ndcg_at_1000_std": 0.08536272462093923, + "nauc_ndcg_at_100_diff1": 0.007417244212828439, + "nauc_ndcg_at_100_max": 0.11683503091805404, + "nauc_ndcg_at_100_std": 0.24690930985418497, + "nauc_ndcg_at_10_diff1": 0.05072202541744854, + "nauc_ndcg_at_10_max": 0.33967388199412657, + "nauc_ndcg_at_10_std": 0.4487116052117022, + "nauc_ndcg_at_1_diff1": 0.17909847061980153, + "nauc_ndcg_at_1_max": 0.7746176549503622, + "nauc_ndcg_at_1_std": 0.9436544137375904, + "nauc_ndcg_at_20_diff1": 0.08209430337287296, + "nauc_ndcg_at_20_max": 0.27299828714362034, + "nauc_ndcg_at_20_std": 0.36199893473900846, + "nauc_ndcg_at_3_diff1": 0.03415592930056411, + "nauc_ndcg_at_3_max": 0.5961127440833953, + "nauc_ndcg_at_3_std": 0.6640858782599793, + "nauc_ndcg_at_5_diff1": 0.03618641095466082, + "nauc_ndcg_at_5_max": 0.4749818186573876, + "nauc_ndcg_at_5_std": 0.6093292134220444, + "nauc_precision_at_1000_diff1": 0.008327387047313468, + "nauc_precision_at_1000_max": 0.02007387385925629, + "nauc_precision_at_1000_std": 0.044956744075964394, + "nauc_precision_at_100_diff1": -0.01001893740545882, + "nauc_precision_at_100_max": 0.051642343571975946, + "nauc_precision_at_100_std": 0.18444138542480845, + "nauc_precision_at_10_diff1": 0.0442888425352916, + "nauc_precision_at_10_max": 0.18272068687952764, + "nauc_precision_at_10_std": 0.28539775136969103, + "nauc_precision_at_1_diff1": 0.17909847061980153, + "nauc_precision_at_1_max": 0.7746176549503622, + "nauc_precision_at_1_std": 0.9436544137375904, + "nauc_precision_at_20_diff1": 0.09493417551144295, + "nauc_precision_at_20_max": 0.15466225637461695, + "nauc_precision_at_20_std": 0.23296546688542216, + "nauc_precision_at_3_diff1": -0.0026212049286907556, + "nauc_precision_at_3_max": 0.5246331345070279, + "nauc_precision_at_3_std": 0.5633320261707707, + "nauc_precision_at_5_diff1": 0.012744834987925856, + "nauc_precision_at_5_max": 0.35232388277733073, + "nauc_precision_at_5_std": 0.5068270578063978, + "nauc_recall_at_1000_diff1": 0.008327387047313645, + "nauc_recall_at_1000_max": 0.020073873859256595, + "nauc_recall_at_1000_std": 0.044956744075964845, + "nauc_recall_at_100_diff1": -0.01001893740545887, + "nauc_recall_at_100_max": 0.05164234357197578, + "nauc_recall_at_100_std": 0.18444138542480845, + "nauc_recall_at_10_diff1": 0.04428884253529171, + "nauc_recall_at_10_max": 0.18272068687952775, + "nauc_recall_at_10_std": 0.28539775136969114, + "nauc_recall_at_1_diff1": 0.17909847061980153, + "nauc_recall_at_1_max": 0.7746176549503622, + "nauc_recall_at_1_std": 0.9436544137375904, + "nauc_recall_at_20_diff1": 0.09493417551144288, + "nauc_recall_at_20_max": 0.15466225637461684, + "nauc_recall_at_20_std": 0.23296546688542194, + "nauc_recall_at_3_diff1": -0.002621204928690842, + "nauc_recall_at_3_max": 0.5246331345070279, + "nauc_recall_at_3_std": 0.5633320261707704, + "nauc_recall_at_5_diff1": 0.012744834987925951, + "nauc_recall_at_5_max": 0.3523238827773307, + "nauc_recall_at_5_std": 0.5068270578063978, + "ndcg_at_1": 0.00078, + "ndcg_at_10": 0.0021, + "ndcg_at_100": 0.00799, + "ndcg_at_1000": 0.03843, + "ndcg_at_20": 0.00309, + "ndcg_at_3": 0.00129, + "ndcg_at_5": 0.00156, + "precision_at_1": 0.00078, + "precision_at_10": 0.00041, + "precision_at_100": 0.00036, + "precision_at_1000": 0.0003, + "precision_at_20": 0.0004, + "precision_at_3": 0.00057, + "precision_at_5": 0.00047, + "recall_at_1": 0.00078, + "recall_at_10": 0.00405, + "recall_at_100": 0.03608, + "recall_at_1000": 0.30301, + "recall_at_20": 0.00797, + "recall_at_3": 0.0017, + "recall_at_5": 0.00235 + }, + { + "cv_recall_at_1": 0.00148, + "cv_recall_at_10": 0.00972, + "cv_recall_at_100": 0.04501, + "cv_recall_at_1000": 0.30917, + "cv_recall_at_20": 0.01479, + "cv_recall_at_3": 0.00528, + "cv_recall_at_5": 0.00676, + "hf_subset": "mi", + "languages": [ + "mri-Latn" + ], + "main_score": 0.00529, + "map_at_1": 0.00148, + "map_at_10": 0.00393, + "map_at_100": 0.0049, + "map_at_1000": 0.00561, + "map_at_20": 0.00429, + "map_at_3": 0.00321, + "map_at_5": 0.00355, + "mrr_at_1": 0.0014792899408284023, + "mrr_at_10": 0.0039252338016074265, + "mrr_at_100": 0.004904245024487178, + "mrr_at_1000": 0.005613644475039326, + "mrr_at_20": 0.00429167584331425, + "mrr_at_3": 0.0032051282051282055, + "mrr_at_5": 0.003553817976894901, + "nauc_cv_recall_at_1000_diff1": 0.004355331507438031, + "nauc_cv_recall_at_1000_max": 0.009966506525083177, + "nauc_cv_recall_at_1000_std": 0.04879354673716684, + "nauc_cv_recall_at_100_diff1": 0.031237245978576855, + "nauc_cv_recall_at_100_max": 0.10106260594284133, + "nauc_cv_recall_at_100_std": 0.1590760683957908, + "nauc_cv_recall_at_10_diff1": 0.1597156391580622, + "nauc_cv_recall_at_10_max": 0.32647292273706424, + "nauc_cv_recall_at_10_std": 0.3601123873766218, + "nauc_cv_recall_at_1_diff1": 0.18771392552872676, + "nauc_cv_recall_at_1_max": 0.03441764330158385, + "nauc_cv_recall_at_1_std": 0.15518249989400212, + "nauc_cv_recall_at_20_diff1": 0.09262905856022437, + "nauc_cv_recall_at_20_max": 0.29146045930012626, + "nauc_cv_recall_at_20_std": 0.36969914504123136, + "nauc_cv_recall_at_3_diff1": 0.17869623110167043, + "nauc_cv_recall_at_3_max": 0.3614519530503182, + "nauc_cv_recall_at_3_std": 0.3343149914188446, + "nauc_cv_recall_at_5_diff1": 0.2355772518637709, + "nauc_cv_recall_at_5_max": 0.37012544836471284, + "nauc_cv_recall_at_5_std": 0.3971712510046319, + "nauc_map_at_1000_diff1": 0.13344744017464022, + "nauc_map_at_1000_max": 0.21873607901173248, + "nauc_map_at_1000_std": 0.2599947155714711, + "nauc_map_at_100_diff1": 0.14674367263273896, + "nauc_map_at_100_max": 0.24234073023274486, + "nauc_map_at_100_std": 0.2833404271031057, + "nauc_map_at_10_diff1": 0.18487856478608633, + "nauc_map_at_10_max": 0.27962716852110747, + "nauc_map_at_10_std": 0.30848428841801184, + "nauc_map_at_1_diff1": 0.18771392552872676, + "nauc_map_at_1_max": 0.03441764330158385, + "nauc_map_at_1_std": 0.15518249989400212, + "nauc_map_at_20_diff1": 0.16789492237246179, + "nauc_map_at_20_max": 0.27544213437309595, + "nauc_map_at_20_std": 0.3146802255655496, + "nauc_map_at_3_diff1": 0.18426767070717054, + "nauc_map_at_3_max": 0.27566821370593203, + "nauc_map_at_3_std": 0.28178798599583, + "nauc_map_at_5_diff1": 0.20690189219095959, + "nauc_map_at_5_max": 0.28441408056847484, + "nauc_map_at_5_std": 0.3128851793422813, + "nauc_mrr_at_1000_diff1": 0.13344751137747338, + "nauc_mrr_at_1000_max": 0.21873628661038808, + "nauc_mrr_at_1000_std": 0.25999493540900376, + "nauc_mrr_at_100_diff1": 0.14674367263273896, + "nauc_mrr_at_100_max": 0.24234073023274486, + "nauc_mrr_at_100_std": 0.2833404271031057, + "nauc_mrr_at_10_diff1": 0.18487856478608633, + "nauc_mrr_at_10_max": 0.27962716852110747, + "nauc_mrr_at_10_std": 0.30848428841801184, + "nauc_mrr_at_1_diff1": 0.18771392552872676, + "nauc_mrr_at_1_max": 0.03441764330158385, + "nauc_mrr_at_1_std": 0.15518249989400212, + "nauc_mrr_at_20_diff1": 0.16789492237246179, + "nauc_mrr_at_20_max": 0.27544213437309595, + "nauc_mrr_at_20_std": 0.3146802255655496, + "nauc_mrr_at_3_diff1": 0.18426767070717054, + "nauc_mrr_at_3_max": 0.27566821370593203, + "nauc_mrr_at_3_std": 0.28178798599583, + "nauc_mrr_at_5_diff1": 0.20690189219095959, + "nauc_mrr_at_5_max": 0.28441408056847484, + "nauc_mrr_at_5_std": 0.3128851793422813, + "nauc_ndcg_at_1000_diff1": 0.02906027104190531, + "nauc_ndcg_at_1000_max": 0.055826066557204464, + "nauc_ndcg_at_1000_std": 0.09776181492411479, + "nauc_ndcg_at_100_diff1": 0.07528148312906342, + "nauc_ndcg_at_100_max": 0.16335625399710246, + "nauc_ndcg_at_100_std": 0.21626420423589257, + "nauc_ndcg_at_10_diff1": 0.17550276864424455, + "nauc_ndcg_at_10_max": 0.3020510586250383, + "nauc_ndcg_at_10_std": 0.33171510525118086, + "nauc_ndcg_at_1_diff1": 0.18771392552872676, + "nauc_ndcg_at_1_max": 0.03441764330158385, + "nauc_ndcg_at_1_std": 0.15518249989400212, + "nauc_ndcg_at_20_diff1": 0.13533400680107543, + "nauc_ndcg_at_20_max": 0.28723325888796997, + "nauc_ndcg_at_20_std": 0.34232923635660023, + "nauc_ndcg_at_3_diff1": 0.18236157753867158, + "nauc_ndcg_at_3_max": 0.30678150196283976, + "nauc_ndcg_at_3_std": 0.3006411799252318, + "nauc_ndcg_at_5_diff1": 0.21707062747772668, + "nauc_ndcg_at_5_max": 0.3176201891977177, + "nauc_ndcg_at_5_std": 0.344583002486531, + "nauc_precision_at_1000_diff1": 0.004355331507438161, + "nauc_precision_at_1000_max": 0.009966506525083213, + "nauc_precision_at_1000_std": 0.048793546737166654, + "nauc_precision_at_100_diff1": 0.031237245978576973, + "nauc_precision_at_100_max": 0.10106260594284158, + "nauc_precision_at_100_std": 0.15907606839579083, + "nauc_precision_at_10_diff1": 0.15971563915806258, + "nauc_precision_at_10_max": 0.3264729227370646, + "nauc_precision_at_10_std": 0.3601123873766221, + "nauc_precision_at_1_diff1": 0.18771392552872676, + "nauc_precision_at_1_max": 0.03441764330158385, + "nauc_precision_at_1_std": 0.15518249989400212, + "nauc_precision_at_20_diff1": 0.09262905856022462, + "nauc_precision_at_20_max": 0.2914604593001266, + "nauc_precision_at_20_std": 0.36969914504123164, + "nauc_precision_at_3_diff1": 0.1786962311016705, + "nauc_precision_at_3_max": 0.36145195305031846, + "nauc_precision_at_3_std": 0.3343149914188446, + "nauc_precision_at_5_diff1": 0.23557725186377082, + "nauc_precision_at_5_max": 0.37012544836471256, + "nauc_precision_at_5_std": 0.3971712510046316, + "nauc_recall_at_1000_diff1": 0.004355331507438031, + "nauc_recall_at_1000_max": 0.009966506525083177, + "nauc_recall_at_1000_std": 0.04879354673716684, + "nauc_recall_at_100_diff1": 0.031237245978576855, + "nauc_recall_at_100_max": 0.10106260594284133, + "nauc_recall_at_100_std": 0.1590760683957908, + "nauc_recall_at_10_diff1": 0.1597156391580622, + "nauc_recall_at_10_max": 0.32647292273706424, + "nauc_recall_at_10_std": 0.3601123873766218, + "nauc_recall_at_1_diff1": 0.18771392552872676, + "nauc_recall_at_1_max": 0.03441764330158385, + "nauc_recall_at_1_std": 0.15518249989400212, + "nauc_recall_at_20_diff1": 0.09262905856022437, + "nauc_recall_at_20_max": 0.29146045930012626, + "nauc_recall_at_20_std": 0.36969914504123136, + "nauc_recall_at_3_diff1": 0.17869623110167043, + "nauc_recall_at_3_max": 0.3614519530503182, + "nauc_recall_at_3_std": 0.3343149914188446, + "nauc_recall_at_5_diff1": 0.2355772518637709, + "nauc_recall_at_5_max": 0.37012544836471284, + "nauc_recall_at_5_std": 0.3971712510046319, + "ndcg_at_1": 0.00148, + "ndcg_at_10": 0.00529, + "ndcg_at_100": 0.01187, + "ndcg_at_1000": 0.04206, + "ndcg_at_20": 0.00659, + "ndcg_at_3": 0.00374, + "ndcg_at_5": 0.00436, + "precision_at_1": 0.00148, + "precision_at_10": 0.00097, + "precision_at_100": 0.00045, + "precision_at_1000": 0.00031, + "precision_at_20": 0.00074, + "precision_at_3": 0.00176, + "precision_at_5": 0.00135, + "recall_at_1": 0.00148, + "recall_at_10": 0.00972, + "recall_at_100": 0.04501, + "recall_at_1000": 0.30917, + "recall_at_20": 0.01479, + "recall_at_3": 0.00528, + "recall_at_5": 0.00676 + }, + { + "cv_recall_at_1": 0.07879, + "cv_recall_at_10": 0.24271, + "cv_recall_at_100": 0.49671, + "cv_recall_at_1000": 0.8341, + "cv_recall_at_20": 0.30686, + "cv_recall_at_3": 0.14915, + "cv_recall_at_5": 0.187, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ], + "main_score": 0.15295, + "map_at_1": 0.07879, + "map_at_10": 0.12519, + "map_at_100": 0.13416, + "map_at_1000": 0.13546, + "map_at_20": 0.12963, + "map_at_3": 0.10919, + "map_at_5": 0.11781, + "mrr_at_1": 0.07879389502419655, + "mrr_at_10": 0.12519405495581232, + "mrr_at_100": 0.13416070786638226, + "mrr_at_1000": 0.13545788899692368, + "mrr_at_20": 0.12963349463435273, + "mrr_at_3": 0.10919468916739085, + "mrr_at_5": 0.11780617942672854, + "nauc_cv_recall_at_1000_diff1": 0.08930409968796703, + "nauc_cv_recall_at_1000_max": 0.30663134466327585, + "nauc_cv_recall_at_1000_std": 0.5460102145860056, + "nauc_cv_recall_at_100_diff1": 0.09605523895794219, + "nauc_cv_recall_at_100_max": 0.2487062961121653, + "nauc_cv_recall_at_100_std": 0.39344458008528294, + "nauc_cv_recall_at_10_diff1": 0.13602886643850298, + "nauc_cv_recall_at_10_max": 0.29325465769882425, + "nauc_cv_recall_at_10_std": 0.290467383536801, + "nauc_cv_recall_at_1_diff1": 0.3277831672915696, + "nauc_cv_recall_at_1_max": 0.38358513881662565, + "nauc_cv_recall_at_1_std": 0.26455104901466153, + "nauc_cv_recall_at_20_diff1": 0.11704118112102552, + "nauc_cv_recall_at_20_max": 0.26934457082677205, + "nauc_cv_recall_at_20_std": 0.3076089089674812, + "nauc_cv_recall_at_3_diff1": 0.2015815540924677, + "nauc_cv_recall_at_3_max": 0.33519097609046333, + "nauc_cv_recall_at_3_std": 0.26655658100966784, + "nauc_cv_recall_at_5_diff1": 0.1604036961954942, + "nauc_cv_recall_at_5_max": 0.3133802482025979, + "nauc_cv_recall_at_5_std": 0.273309526612517, + "nauc_map_at_1000_diff1": 0.22963830503228963, + "nauc_map_at_1000_max": 0.338849193361561, + "nauc_map_at_1000_std": 0.27744821129645536, + "nauc_map_at_100_diff1": 0.22923942148213955, + "nauc_map_at_100_max": 0.3383619423952214, + "nauc_map_at_100_std": 0.27691713466932893, + "nauc_map_at_10_diff1": 0.2338906235270105, + "nauc_map_at_10_max": 0.34335803851093954, + "nauc_map_at_10_std": 0.27054230976216415, + "nauc_map_at_1_diff1": 0.3277831672915696, + "nauc_map_at_1_max": 0.38358513881662565, + "nauc_map_at_1_std": 0.26455104901466153, + "nauc_map_at_20_diff1": 0.23039337286476344, + "nauc_map_at_20_max": 0.33966045852700694, + "nauc_map_at_20_std": 0.27296817740708507, + "nauc_map_at_3_diff1": 0.2581545114721057, + "nauc_map_at_3_max": 0.3569969229868415, + "nauc_map_at_3_std": 0.2643599427994337, + "nauc_map_at_5_diff1": 0.24144220774040495, + "nauc_map_at_5_max": 0.34874293922106625, + "nauc_map_at_5_std": 0.2669519907389868, + "nauc_mrr_at_1000_diff1": 0.22963830503228963, + "nauc_mrr_at_1000_max": 0.338849193361561, + "nauc_mrr_at_1000_std": 0.27744821129645536, + "nauc_mrr_at_100_diff1": 0.22923942148213955, + "nauc_mrr_at_100_max": 0.3383619423952214, + "nauc_mrr_at_100_std": 0.27691713466932893, + "nauc_mrr_at_10_diff1": 0.2338906235270105, + "nauc_mrr_at_10_max": 0.34335803851093954, + "nauc_mrr_at_10_std": 0.27054230976216415, + "nauc_mrr_at_1_diff1": 0.3277831672915696, + "nauc_mrr_at_1_max": 0.38358513881662565, + "nauc_mrr_at_1_std": 0.26455104901466153, + "nauc_mrr_at_20_diff1": 0.23039337286476344, + "nauc_mrr_at_20_max": 0.33966045852700694, + "nauc_mrr_at_20_std": 0.27296817740708507, + "nauc_mrr_at_3_diff1": 0.2581545114721057, + "nauc_mrr_at_3_max": 0.3569969229868415, + "nauc_mrr_at_3_std": 0.2643599427994337, + "nauc_mrr_at_5_diff1": 0.24144220774040495, + "nauc_mrr_at_5_max": 0.34874293922106625, + "nauc_mrr_at_5_std": 0.2669519907389868, + "nauc_ndcg_at_1000_diff1": 0.1941231777187904, + "nauc_ndcg_at_1000_max": 0.3219632581148484, + "nauc_ndcg_at_1000_std": 0.3149786501614155, + "nauc_ndcg_at_100_diff1": 0.18513247666713342, + "nauc_ndcg_at_100_max": 0.30990446132316557, + "nauc_ndcg_at_100_std": 0.30800382186794323, + "nauc_ndcg_at_10_diff1": 0.20094888060006244, + "nauc_ndcg_at_10_max": 0.32679891787939697, + "nauc_ndcg_at_10_std": 0.27674862360276026, + "nauc_ndcg_at_1_diff1": 0.3277831672915696, + "nauc_ndcg_at_1_max": 0.38358513881662565, + "nauc_ndcg_at_1_std": 0.26455104901466153, + "nauc_ndcg_at_20_diff1": 0.19136917742345397, + "nauc_ndcg_at_20_max": 0.31632500954065434, + "nauc_ndcg_at_20_std": 0.2838398423680135, + "nauc_ndcg_at_3_diff1": 0.24077335740756323, + "nauc_ndcg_at_3_max": 0.35030162729309505, + "nauc_ndcg_at_3_std": 0.2649857421079281, + "nauc_ndcg_at_5_diff1": 0.215227244408146, + "nauc_ndcg_at_5_max": 0.33738550021643154, + "nauc_ndcg_at_5_std": 0.26893681994359964, + "nauc_precision_at_1000_diff1": 0.0893040996879657, + "nauc_precision_at_1000_max": 0.30663134466327596, + "nauc_precision_at_1000_std": 0.5460102145860033, + "nauc_precision_at_100_diff1": 0.09605523895794259, + "nauc_precision_at_100_max": 0.24870629611216516, + "nauc_precision_at_100_std": 0.3934445800852835, + "nauc_precision_at_10_diff1": 0.13602886643850304, + "nauc_precision_at_10_max": 0.29325465769882403, + "nauc_precision_at_10_std": 0.2904673835368009, + "nauc_precision_at_1_diff1": 0.3277831672915696, + "nauc_precision_at_1_max": 0.38358513881662565, + "nauc_precision_at_1_std": 0.26455104901466153, + "nauc_precision_at_20_diff1": 0.11704118112102543, + "nauc_precision_at_20_max": 0.2693445708267721, + "nauc_precision_at_20_std": 0.3076089089674813, + "nauc_precision_at_3_diff1": 0.20158155409246764, + "nauc_precision_at_3_max": 0.33519097609046333, + "nauc_precision_at_3_std": 0.2665565810096679, + "nauc_precision_at_5_diff1": 0.1604036961954938, + "nauc_precision_at_5_max": 0.31338024820259774, + "nauc_precision_at_5_std": 0.2733095266125168, + "nauc_recall_at_1000_diff1": 0.08930409968796703, + "nauc_recall_at_1000_max": 0.30663134466327585, + "nauc_recall_at_1000_std": 0.5460102145860056, + "nauc_recall_at_100_diff1": 0.09605523895794219, + "nauc_recall_at_100_max": 0.2487062961121653, + "nauc_recall_at_100_std": 0.39344458008528294, + "nauc_recall_at_10_diff1": 0.13602886643850298, + "nauc_recall_at_10_max": 0.29325465769882425, + "nauc_recall_at_10_std": 0.290467383536801, + "nauc_recall_at_1_diff1": 0.3277831672915696, + "nauc_recall_at_1_max": 0.38358513881662565, + "nauc_recall_at_1_std": 0.26455104901466153, + "nauc_recall_at_20_diff1": 0.11704118112102552, + "nauc_recall_at_20_max": 0.26934457082677205, + "nauc_recall_at_20_std": 0.3076089089674812, + "nauc_recall_at_3_diff1": 0.2015815540924677, + "nauc_recall_at_3_max": 0.33519097609046333, + "nauc_recall_at_3_std": 0.26655658100966784, + "nauc_recall_at_5_diff1": 0.1604036961954942, + "nauc_recall_at_5_max": 0.3133802482025979, + "nauc_recall_at_5_std": 0.273309526612517, + "ndcg_at_1": 0.07879, + "ndcg_at_10": 0.15295, + "ndcg_at_100": 0.20368, + "ndcg_at_1000": 0.24471, + "ndcg_at_20": 0.16916, + "ndcg_at_3": 0.11943, + "ndcg_at_5": 0.13499, + "precision_at_1": 0.07879, + "precision_at_10": 0.02427, + "precision_at_100": 0.00497, + "precision_at_1000": 0.00083, + "precision_at_20": 0.01534, + "precision_at_3": 0.04972, + "precision_at_5": 0.0374, + "recall_at_1": 0.07879, + "recall_at_10": 0.24271, + "recall_at_100": 0.49671, + "recall_at_1000": 0.8341, + "recall_at_20": 0.30686, + "recall_at_3": 0.14915, + "recall_at_5": 0.187 + }, + { + "cv_recall_at_1": 0.0305, + "cv_recall_at_10": 0.11438, + "cv_recall_at_100": 0.29197, + "cv_recall_at_1000": 0.65937, + "cv_recall_at_20": 0.15694, + "cv_recall_at_3": 0.06447, + "cv_recall_at_5": 0.08277, + "hf_subset": "no", + "languages": [ + "nor-Latn" + ], + "main_score": 0.06762, + "map_at_1": 0.0305, + "map_at_10": 0.05326, + "map_at_100": 0.05935, + "map_at_1000": 0.06061, + "map_at_20": 0.0562, + "map_at_3": 0.04497, + "map_at_5": 0.04907, + "mrr_at_1": 0.030500485234992374, + "mrr_at_10": 0.05326449818339451, + "mrr_at_100": 0.05935235207055774, + "mrr_at_1000": 0.06061448878521882, + "mrr_at_20": 0.05619837034936023, + "mrr_at_3": 0.04496510929340537, + "mrr_at_5": 0.04906881094320427, + "nauc_cv_recall_at_1000_diff1": 0.11112518129848828, + "nauc_cv_recall_at_1000_max": 0.17993525311133474, + "nauc_cv_recall_at_1000_std": 0.4386071263187699, + "nauc_cv_recall_at_100_diff1": 0.11926601259822187, + "nauc_cv_recall_at_100_max": 0.17193302058599702, + "nauc_cv_recall_at_100_std": 0.37419523754131717, + "nauc_cv_recall_at_10_diff1": 0.1517786413891697, + "nauc_cv_recall_at_10_max": 0.2442149898435886, + "nauc_cv_recall_at_10_std": 0.36620202281025915, + "nauc_cv_recall_at_1_diff1": 0.44248156992731186, + "nauc_cv_recall_at_1_max": 0.4793737235035624, + "nauc_cv_recall_at_1_std": 0.3974568606637231, + "nauc_cv_recall_at_20_diff1": 0.14704354887179238, + "nauc_cv_recall_at_20_max": 0.21188556513826365, + "nauc_cv_recall_at_20_std": 0.35708153462693637, + "nauc_cv_recall_at_3_diff1": 0.2398255875343225, + "nauc_cv_recall_at_3_max": 0.3218596579776256, + "nauc_cv_recall_at_3_std": 0.3679462784006674, + "nauc_cv_recall_at_5_diff1": 0.19407458813228443, + "nauc_cv_recall_at_5_max": 0.2903569698107385, + "nauc_cv_recall_at_5_std": 0.3734450090214543, + "nauc_map_at_1000_diff1": 0.2670329419285782, + "nauc_map_at_1000_max": 0.3327567801100508, + "nauc_map_at_1000_std": 0.37732823515389075, + "nauc_map_at_100_diff1": 0.2670297580050445, + "nauc_map_at_100_max": 0.33294280925431413, + "nauc_map_at_100_std": 0.37662222184363, + "nauc_map_at_10_diff1": 0.2779145274337388, + "nauc_map_at_10_max": 0.34846692225241066, + "nauc_map_at_10_std": 0.3773006555270141, + "nauc_map_at_1_diff1": 0.44248156992731186, + "nauc_map_at_1_max": 0.4793737235035624, + "nauc_map_at_1_std": 0.3974568606637231, + "nauc_map_at_20_diff1": 0.27307450094883884, + "nauc_map_at_20_max": 0.3398105447803915, + "nauc_map_at_20_std": 0.3757032305314629, + "nauc_map_at_3_diff1": 0.3209398158909183, + "nauc_map_at_3_max": 0.38457816609234413, + "nauc_map_at_3_std": 0.37935958905552525, + "nauc_map_at_5_diff1": 0.2966734685382508, + "nauc_map_at_5_max": 0.3669004169612993, + "nauc_map_at_5_std": 0.38033458614308907, + "nauc_mrr_at_1000_diff1": 0.2670329419285782, + "nauc_mrr_at_1000_max": 0.3327567801100508, + "nauc_mrr_at_1000_std": 0.37732823515389075, + "nauc_mrr_at_100_diff1": 0.2670297580050445, + "nauc_mrr_at_100_max": 0.33294280925431413, + "nauc_mrr_at_100_std": 0.37662222184363, + "nauc_mrr_at_10_diff1": 0.2779145274337388, + "nauc_mrr_at_10_max": 0.34846692225241066, + "nauc_mrr_at_10_std": 0.3773006555270141, + "nauc_mrr_at_1_diff1": 0.44248156992731186, + "nauc_mrr_at_1_max": 0.4793737235035624, + "nauc_mrr_at_1_std": 0.3974568606637231, + "nauc_mrr_at_20_diff1": 0.27307450094883884, + "nauc_mrr_at_20_max": 0.3398105447803915, + "nauc_mrr_at_20_std": 0.3757032305314629, + "nauc_mrr_at_3_diff1": 0.3209398158909183, + "nauc_mrr_at_3_max": 0.38457816609234413, + "nauc_mrr_at_3_std": 0.37935958905552525, + "nauc_mrr_at_5_diff1": 0.2966734685382508, + "nauc_mrr_at_5_max": 0.3669004169612993, + "nauc_mrr_at_5_std": 0.38033458614308907, + "nauc_ndcg_at_1000_diff1": 0.19975186137524023, + "nauc_ndcg_at_1000_max": 0.2650715883370969, + "nauc_ndcg_at_1000_std": 0.3899538430343936, + "nauc_ndcg_at_100_diff1": 0.19805283041177202, + "nauc_ndcg_at_100_max": 0.26046936636798446, + "nauc_ndcg_at_100_std": 0.3741037394930803, + "nauc_ndcg_at_10_diff1": 0.2288349265356686, + "nauc_ndcg_at_10_max": 0.3080841122158018, + "nauc_ndcg_at_10_std": 0.3728326724105545, + "nauc_ndcg_at_1_diff1": 0.44248156992731186, + "nauc_ndcg_at_1_max": 0.4793737235035624, + "nauc_ndcg_at_1_std": 0.3974568606637231, + "nauc_ndcg_at_20_diff1": 0.21954849145263852, + "nauc_ndcg_at_20_max": 0.28716962341639507, + "nauc_ndcg_at_20_std": 0.3682131493631691, + "nauc_ndcg_at_3_diff1": 0.2942502076310742, + "nauc_ndcg_at_3_max": 0.36392958959880967, + "nauc_ndcg_at_3_std": 0.3755897658861957, + "nauc_ndcg_at_5_diff1": 0.26012575265309584, + "nauc_ndcg_at_5_max": 0.33953946005404073, + "nauc_ndcg_at_5_std": 0.3777176090535697, + "nauc_precision_at_1000_diff1": 0.11112518129848781, + "nauc_precision_at_1000_max": 0.1799352531113345, + "nauc_precision_at_1000_std": 0.43860712631876997, + "nauc_precision_at_100_diff1": 0.11926601259822206, + "nauc_precision_at_100_max": 0.17193302058599713, + "nauc_precision_at_100_std": 0.3741952375413173, + "nauc_precision_at_10_diff1": 0.15177864138916966, + "nauc_precision_at_10_max": 0.24421498984358866, + "nauc_precision_at_10_std": 0.36620202281025893, + "nauc_precision_at_1_diff1": 0.44248156992731186, + "nauc_precision_at_1_max": 0.4793737235035624, + "nauc_precision_at_1_std": 0.3974568606637231, + "nauc_precision_at_20_diff1": 0.14704354887179236, + "nauc_precision_at_20_max": 0.21188556513826365, + "nauc_precision_at_20_std": 0.3570815346269364, + "nauc_precision_at_3_diff1": 0.23982558753432265, + "nauc_precision_at_3_max": 0.32185965797762567, + "nauc_precision_at_3_std": 0.3679462784006674, + "nauc_precision_at_5_diff1": 0.1940745881322843, + "nauc_precision_at_5_max": 0.29035696981073855, + "nauc_precision_at_5_std": 0.3734450090214543, + "nauc_recall_at_1000_diff1": 0.11112518129848828, + "nauc_recall_at_1000_max": 0.17993525311133474, + "nauc_recall_at_1000_std": 0.4386071263187699, + "nauc_recall_at_100_diff1": 0.11926601259822187, + "nauc_recall_at_100_max": 0.17193302058599702, + "nauc_recall_at_100_std": 0.37419523754131717, + "nauc_recall_at_10_diff1": 0.1517786413891697, + "nauc_recall_at_10_max": 0.2442149898435886, + "nauc_recall_at_10_std": 0.36620202281025915, + "nauc_recall_at_1_diff1": 0.44248156992731186, + "nauc_recall_at_1_max": 0.4793737235035624, + "nauc_recall_at_1_std": 0.3974568606637231, + "nauc_recall_at_20_diff1": 0.14704354887179238, + "nauc_recall_at_20_max": 0.21188556513826365, + "nauc_recall_at_20_std": 0.35708153462693637, + "nauc_recall_at_3_diff1": 0.2398255875343225, + "nauc_recall_at_3_max": 0.3218596579776256, + "nauc_recall_at_3_std": 0.3679462784006674, + "nauc_recall_at_5_diff1": 0.19407458813228443, + "nauc_recall_at_5_max": 0.2903569698107385, + "nauc_recall_at_5_std": 0.3734450090214543, + "ndcg_at_1": 0.0305, + "ndcg_at_10": 0.06762, + "ndcg_at_100": 0.10277, + "ndcg_at_1000": 0.14652, + "ndcg_at_20": 0.07836, + "ndcg_at_3": 0.04995, + "ndcg_at_5": 0.05742, + "precision_at_1": 0.0305, + "precision_at_10": 0.01144, + "precision_at_100": 0.00292, + "precision_at_1000": 0.00066, + "precision_at_20": 0.00785, + "precision_at_3": 0.02149, + "precision_at_5": 0.01655, + "recall_at_1": 0.0305, + "recall_at_10": 0.11438, + "recall_at_100": 0.29197, + "recall_at_1000": 0.65937, + "recall_at_20": 0.15694, + "recall_at_3": 0.06447, + "recall_at_5": 0.08277 + }, + { + "cv_recall_at_1": 0.01694, + "cv_recall_at_10": 0.06162, + "cv_recall_at_100": 0.1413, + "cv_recall_at_1000": 0.44602, + "cv_recall_at_20": 0.0815, + "cv_recall_at_3": 0.03375, + "cv_recall_at_5": 0.04397, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ], + "main_score": 0.03658, + "map_at_1": 0.01694, + "map_at_10": 0.02894, + "map_at_100": 0.03165, + "map_at_1000": 0.03254, + "map_at_20": 0.03029, + "map_at_3": 0.02439, + "map_at_5": 0.02669, + "mrr_at_1": 0.016944405545441816, + "mrr_at_10": 0.02894219163649213, + "mrr_at_100": 0.03165311711940447, + "mrr_at_1000": 0.0325447626187637, + "mrr_at_20": 0.03028990356706552, + "mrr_at_3": 0.024389674648742008, + "mrr_at_5": 0.02669327358446528, + "nauc_cv_recall_at_1000_diff1": 0.1377259253556431, + "nauc_cv_recall_at_1000_max": 0.3263785888666531, + "nauc_cv_recall_at_1000_std": 0.3424801457124148, + "nauc_cv_recall_at_100_diff1": 0.2423337950404231, + "nauc_cv_recall_at_100_max": 0.5241164351239225, + "nauc_cv_recall_at_100_std": 0.52779401007404, + "nauc_cv_recall_at_10_diff1": 0.3255266166865096, + "nauc_cv_recall_at_10_max": 0.6390632499675445, + "nauc_cv_recall_at_10_std": 0.6161215849797735, + "nauc_cv_recall_at_1_diff1": 0.5378727151451592, + "nauc_cv_recall_at_1_max": 0.7425241822187479, + "nauc_cv_recall_at_1_std": 0.6231210907908145, + "nauc_cv_recall_at_20_diff1": 0.2983831125608907, + "nauc_cv_recall_at_20_max": 0.6014093798271857, + "nauc_cv_recall_at_20_std": 0.590690017609021, + "nauc_cv_recall_at_3_diff1": 0.4092151134545967, + "nauc_cv_recall_at_3_max": 0.7160824725563123, + "nauc_cv_recall_at_3_std": 0.6627713649654221, + "nauc_cv_recall_at_5_diff1": 0.3815391729351457, + "nauc_cv_recall_at_5_max": 0.6942264335856264, + "nauc_cv_recall_at_5_std": 0.6582818015126044, + "nauc_map_at_1000_diff1": 0.40117554070848055, + "nauc_map_at_1000_max": 0.6732502978922448, + "nauc_map_at_1000_std": 0.6187333045457905, + "nauc_map_at_100_diff1": 0.4051924506529917, + "nauc_map_at_100_max": 0.67979233659632, + "nauc_map_at_100_std": 0.6244531722043362, + "nauc_map_at_10_diff1": 0.4237179846418767, + "nauc_map_at_10_max": 0.6997202433553061, + "nauc_map_at_10_std": 0.6377480420470368, + "nauc_map_at_1_diff1": 0.5378727151451592, + "nauc_map_at_1_max": 0.7425241822187479, + "nauc_map_at_1_std": 0.6231210907908145, + "nauc_map_at_20_diff1": 0.41474178486365765, + "nauc_map_at_20_max": 0.6907510609941931, + "nauc_map_at_20_std": 0.6326295788794856, + "nauc_map_at_3_diff1": 0.4575569051255311, + "nauc_map_at_3_max": 0.7238338855927117, + "nauc_map_at_3_std": 0.6481336001436105, + "nauc_map_at_5_diff1": 0.4429423480805897, + "nauc_map_at_5_max": 0.7148428260410172, + "nauc_map_at_5_std": 0.6477392623063669, + "nauc_mrr_at_1000_diff1": 0.40117554070848055, + "nauc_mrr_at_1000_max": 0.6732502978922448, + "nauc_mrr_at_1000_std": 0.6187333045457905, + "nauc_mrr_at_100_diff1": 0.4051924506529917, + "nauc_mrr_at_100_max": 0.67979233659632, + "nauc_mrr_at_100_std": 0.6244531722043362, + "nauc_mrr_at_10_diff1": 0.4237179846418767, + "nauc_mrr_at_10_max": 0.6997202433553061, + "nauc_mrr_at_10_std": 0.6377480420470368, + "nauc_mrr_at_1_diff1": 0.5378727151451592, + "nauc_mrr_at_1_max": 0.7425241822187479, + "nauc_mrr_at_1_std": 0.6231210907908145, + "nauc_mrr_at_20_diff1": 0.41474178486365765, + "nauc_mrr_at_20_max": 0.6907510609941931, + "nauc_mrr_at_20_std": 0.6326295788794856, + "nauc_mrr_at_3_diff1": 0.4575569051255311, + "nauc_mrr_at_3_max": 0.7238338855927117, + "nauc_mrr_at_3_std": 0.6481336001436105, + "nauc_mrr_at_5_diff1": 0.4429423480805897, + "nauc_mrr_at_5_max": 0.7148428260410172, + "nauc_mrr_at_5_std": 0.6477392623063669, + "nauc_ndcg_at_1000_diff1": 0.27181028944517127, + "nauc_ndcg_at_1000_max": 0.5132561629976885, + "nauc_ndcg_at_1000_std": 0.4959066930758658, + "nauc_ndcg_at_100_diff1": 0.32845283023793725, + "nauc_ndcg_at_100_max": 0.6096509413496082, + "nauc_ndcg_at_100_std": 0.5825751332255994, + "nauc_ndcg_at_10_diff1": 0.3858193527140121, + "nauc_ndcg_at_10_max": 0.6772961496311672, + "nauc_ndcg_at_10_std": 0.6306167576871402, + "nauc_ndcg_at_1_diff1": 0.5378727151451592, + "nauc_ndcg_at_1_max": 0.7425241822187479, + "nauc_ndcg_at_1_std": 0.6231210907908145, + "nauc_ndcg_at_20_diff1": 0.365546336861465, + "nauc_ndcg_at_20_max": 0.6547149548674186, + "nauc_ndcg_at_20_std": 0.6167961341010798, + "nauc_ndcg_at_3_diff1": 0.4418525175146829, + "nauc_ndcg_at_3_max": 0.7212285747452951, + "nauc_ndcg_at_3_std": 0.6528992424951905, + "nauc_ndcg_at_5_diff1": 0.42124824904040953, + "nauc_ndcg_at_5_max": 0.707684892712741, + "nauc_ndcg_at_5_std": 0.6516317812025026, + "nauc_precision_at_1000_diff1": 0.1377259253556427, + "nauc_precision_at_1000_max": 0.3263785888666525, + "nauc_precision_at_1000_std": 0.3424801457124145, + "nauc_precision_at_100_diff1": 0.24233379504042352, + "nauc_precision_at_100_max": 0.5241164351239226, + "nauc_precision_at_100_std": 0.5277940100740403, + "nauc_precision_at_10_diff1": 0.32552661668650973, + "nauc_precision_at_10_max": 0.6390632499675446, + "nauc_precision_at_10_std": 0.6161215849797738, + "nauc_precision_at_1_diff1": 0.5378727151451592, + "nauc_precision_at_1_max": 0.7425241822187479, + "nauc_precision_at_1_std": 0.6231210907908145, + "nauc_precision_at_20_diff1": 0.29838311256089073, + "nauc_precision_at_20_max": 0.601409379827186, + "nauc_precision_at_20_std": 0.5906900176090211, + "nauc_precision_at_3_diff1": 0.4092151134545966, + "nauc_precision_at_3_max": 0.7160824725563122, + "nauc_precision_at_3_std": 0.6627713649654222, + "nauc_precision_at_5_diff1": 0.3815391729351459, + "nauc_precision_at_5_max": 0.6942264335856266, + "nauc_precision_at_5_std": 0.6582818015126047, + "nauc_recall_at_1000_diff1": 0.1377259253556431, + "nauc_recall_at_1000_max": 0.3263785888666531, + "nauc_recall_at_1000_std": 0.3424801457124148, + "nauc_recall_at_100_diff1": 0.2423337950404231, + "nauc_recall_at_100_max": 0.5241164351239225, + "nauc_recall_at_100_std": 0.52779401007404, + "nauc_recall_at_10_diff1": 0.3255266166865096, + "nauc_recall_at_10_max": 0.6390632499675445, + "nauc_recall_at_10_std": 0.6161215849797735, + "nauc_recall_at_1_diff1": 0.5378727151451592, + "nauc_recall_at_1_max": 0.7425241822187479, + "nauc_recall_at_1_std": 0.6231210907908145, + "nauc_recall_at_20_diff1": 0.2983831125608907, + "nauc_recall_at_20_max": 0.6014093798271857, + "nauc_recall_at_20_std": 0.590690017609021, + "nauc_recall_at_3_diff1": 0.4092151134545967, + "nauc_recall_at_3_max": 0.7160824725563123, + "nauc_recall_at_3_std": 0.6627713649654221, + "nauc_recall_at_5_diff1": 0.3815391729351457, + "nauc_recall_at_5_max": 0.6942264335856264, + "nauc_recall_at_5_std": 0.6582818015126044, + "ndcg_at_1": 0.01694, + "ndcg_at_10": 0.03658, + "ndcg_at_100": 0.05231, + "ndcg_at_1000": 0.08767, + "ndcg_at_20": 0.04156, + "ndcg_at_3": 0.0268, + "ndcg_at_5": 0.03098, + "precision_at_1": 0.01694, + "precision_at_10": 0.00616, + "precision_at_100": 0.00141, + "precision_at_1000": 0.00045, + "precision_at_20": 0.00408, + "precision_at_3": 0.01125, + "precision_at_5": 0.00879, + "recall_at_1": 0.01694, + "recall_at_10": 0.06162, + "recall_at_100": 0.1413, + "recall_at_1000": 0.44602, + "recall_at_20": 0.0815, + "recall_at_3": 0.03375, + "recall_at_5": 0.04397 + }, + { + "cv_recall_at_1": 0.10534, + "cv_recall_at_10": 0.31092, + "cv_recall_at_100": 0.6206, + "cv_recall_at_1000": 0.90708, + "cv_recall_at_20": 0.39832, + "cv_recall_at_3": 0.18749, + "cv_recall_at_5": 0.2365, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ], + "main_score": 0.19699, + "map_at_1": 0.10534, + "map_at_10": 0.16193, + "map_at_100": 0.17328, + "map_at_1000": 0.17446, + "map_at_20": 0.16791, + "map_at_3": 0.14094, + "map_at_5": 0.15208, + "mrr_at_1": 0.10534308987988403, + "mrr_at_10": 0.16193067636185143, + "mrr_at_100": 0.17327529387246055, + "mrr_at_1000": 0.17445652475890244, + "mrr_at_20": 0.16791378478806276, + "mrr_at_3": 0.14094067835611443, + "mrr_at_5": 0.15207556721432272, + "nauc_cv_recall_at_1000_diff1": 0.188634082401284, + "nauc_cv_recall_at_1000_max": 0.4168648718922161, + "nauc_cv_recall_at_1000_std": 0.6453755575883677, + "nauc_cv_recall_at_100_diff1": 0.1751770945181202, + "nauc_cv_recall_at_100_max": 0.28556121125098244, + "nauc_cv_recall_at_100_std": 0.4132481269734251, + "nauc_cv_recall_at_10_diff1": 0.22174110466323108, + "nauc_cv_recall_at_10_max": 0.2843735838808934, + "nauc_cv_recall_at_10_std": 0.2187110961376596, + "nauc_cv_recall_at_1_diff1": 0.39297845070777243, + "nauc_cv_recall_at_1_max": 0.36221897239007084, + "nauc_cv_recall_at_1_std": 0.14795166958801662, + "nauc_cv_recall_at_20_diff1": 0.19741468067443296, + "nauc_cv_recall_at_20_max": 0.2793650184105488, + "nauc_cv_recall_at_20_std": 0.26820124036829546, + "nauc_cv_recall_at_3_diff1": 0.2781889669980125, + "nauc_cv_recall_at_3_max": 0.30514260823551453, + "nauc_cv_recall_at_3_std": 0.16817074987638658, + "nauc_cv_recall_at_5_diff1": 0.25413255473223123, + "nauc_cv_recall_at_5_max": 0.29288529852666495, + "nauc_cv_recall_at_5_std": 0.19417874420021805, + "nauc_map_at_1000_diff1": 0.3121601858930789, + "nauc_map_at_1000_max": 0.3271421192958233, + "nauc_map_at_1000_std": 0.1822483099304241, + "nauc_map_at_100_diff1": 0.3117424613332504, + "nauc_map_at_100_max": 0.32669216375125915, + "nauc_map_at_100_std": 0.1822541691667279, + "nauc_map_at_10_diff1": 0.31517810279756303, + "nauc_map_at_10_max": 0.32688790799889034, + "nauc_map_at_10_std": 0.17248238517072223, + "nauc_map_at_1_diff1": 0.39297845070777243, + "nauc_map_at_1_max": 0.36221897239007084, + "nauc_map_at_1_std": 0.14795166958801662, + "nauc_map_at_20_diff1": 0.31187388094996743, + "nauc_map_at_20_max": 0.3263220697760968, + "nauc_map_at_20_std": 0.1780222958748876, + "nauc_map_at_3_diff1": 0.332594494915826, + "nauc_map_at_3_max": 0.33455296392040723, + "nauc_map_at_3_std": 0.15889529679138556, + "nauc_map_at_5_diff1": 0.32306905528238067, + "nauc_map_at_5_max": 0.3297156250885414, + "nauc_map_at_5_std": 0.16704299359039254, + "nauc_mrr_at_1000_diff1": 0.3121601858930789, + "nauc_mrr_at_1000_max": 0.3271421192958233, + "nauc_mrr_at_1000_std": 0.1822483099304241, + "nauc_mrr_at_100_diff1": 0.3117424613332504, + "nauc_mrr_at_100_max": 0.32669216375125915, + "nauc_mrr_at_100_std": 0.1822541691667279, + "nauc_mrr_at_10_diff1": 0.31517810279756303, + "nauc_mrr_at_10_max": 0.32688790799889034, + "nauc_mrr_at_10_std": 0.17248238517072223, + "nauc_mrr_at_1_diff1": 0.39297845070777243, + "nauc_mrr_at_1_max": 0.36221897239007084, + "nauc_mrr_at_1_std": 0.14795166958801662, + "nauc_mrr_at_20_diff1": 0.31187388094996743, + "nauc_mrr_at_20_max": 0.3263220697760968, + "nauc_mrr_at_20_std": 0.1780222958748876, + "nauc_mrr_at_3_diff1": 0.332594494915826, + "nauc_mrr_at_3_max": 0.33455296392040723, + "nauc_mrr_at_3_std": 0.15889529679138556, + "nauc_mrr_at_5_diff1": 0.32306905528238067, + "nauc_mrr_at_5_max": 0.3297156250885414, + "nauc_mrr_at_5_std": 0.16704299359039254, + "nauc_ndcg_at_1000_diff1": 0.2856756508689451, + "nauc_ndcg_at_1000_max": 0.324123956897226, + "nauc_ndcg_at_1000_std": 0.22731027234399237, + "nauc_ndcg_at_100_diff1": 0.27435431907395347, + "nauc_ndcg_at_100_max": 0.31385085918658157, + "nauc_ndcg_at_100_std": 0.23333306719606933, + "nauc_ndcg_at_10_diff1": 0.28555354474720956, + "nauc_ndcg_at_10_max": 0.3132068665418705, + "nauc_ndcg_at_10_std": 0.18660288218305618, + "nauc_ndcg_at_1_diff1": 0.39297845070777243, + "nauc_ndcg_at_1_max": 0.36221897239007084, + "nauc_ndcg_at_1_std": 0.14795166958801662, + "nauc_ndcg_at_20_diff1": 0.2759260943286604, + "nauc_ndcg_at_20_max": 0.3112481235577066, + "nauc_ndcg_at_20_std": 0.2041925668301508, + "nauc_ndcg_at_3_diff1": 0.31634053385890426, + "nauc_ndcg_at_3_max": 0.3258569771631374, + "nauc_ndcg_at_3_std": 0.16167819626007002, + "nauc_ndcg_at_5_diff1": 0.30167602085250383, + "nauc_ndcg_at_5_max": 0.3183719831610955, + "nauc_ndcg_at_5_std": 0.17518979370699594, + "nauc_precision_at_1000_diff1": 0.18863408240128252, + "nauc_precision_at_1000_max": 0.4168648718922157, + "nauc_precision_at_1000_std": 0.6453755575883675, + "nauc_precision_at_100_diff1": 0.17517709451812002, + "nauc_precision_at_100_max": 0.28556121125098244, + "nauc_precision_at_100_std": 0.4132481269734251, + "nauc_precision_at_10_diff1": 0.22174110466323108, + "nauc_precision_at_10_max": 0.2843735838808934, + "nauc_precision_at_10_std": 0.21871109613765954, + "nauc_precision_at_1_diff1": 0.39297845070777243, + "nauc_precision_at_1_max": 0.36221897239007084, + "nauc_precision_at_1_std": 0.14795166958801662, + "nauc_precision_at_20_diff1": 0.197414680674433, + "nauc_precision_at_20_max": 0.27936501841054867, + "nauc_precision_at_20_std": 0.2682012403682955, + "nauc_precision_at_3_diff1": 0.27818896699801243, + "nauc_precision_at_3_max": 0.30514260823551453, + "nauc_precision_at_3_std": 0.16817074987638667, + "nauc_precision_at_5_diff1": 0.2541325547322312, + "nauc_precision_at_5_max": 0.2928852985266648, + "nauc_precision_at_5_std": 0.19417874420021808, + "nauc_recall_at_1000_diff1": 0.188634082401284, + "nauc_recall_at_1000_max": 0.4168648718922161, + "nauc_recall_at_1000_std": 0.6453755575883677, + "nauc_recall_at_100_diff1": 0.1751770945181202, + "nauc_recall_at_100_max": 0.28556121125098244, + "nauc_recall_at_100_std": 0.4132481269734251, + "nauc_recall_at_10_diff1": 0.22174110466323108, + "nauc_recall_at_10_max": 0.2843735838808934, + "nauc_recall_at_10_std": 0.2187110961376596, + "nauc_recall_at_1_diff1": 0.39297845070777243, + "nauc_recall_at_1_max": 0.36221897239007084, + "nauc_recall_at_1_std": 0.14795166958801662, + "nauc_recall_at_20_diff1": 0.19741468067443296, + "nauc_recall_at_20_max": 0.2793650184105488, + "nauc_recall_at_20_std": 0.26820124036829546, + "nauc_recall_at_3_diff1": 0.2781889669980125, + "nauc_recall_at_3_max": 0.30514260823551453, + "nauc_recall_at_3_std": 0.16817074987638658, + "nauc_recall_at_5_diff1": 0.25413255473223123, + "nauc_recall_at_5_max": 0.29288529852666495, + "nauc_recall_at_5_std": 0.19417874420021805, + "ndcg_at_1": 0.10534, + "ndcg_at_10": 0.19699, + "ndcg_at_100": 0.25954, + "ndcg_at_1000": 0.29488, + "ndcg_at_20": 0.21898, + "ndcg_at_3": 0.15287, + "ndcg_at_5": 0.173, + "precision_at_1": 0.10534, + "precision_at_10": 0.03109, + "precision_at_100": 0.00621, + "precision_at_1000": 0.00091, + "precision_at_20": 0.01992, + "precision_at_3": 0.0625, + "precision_at_5": 0.0473, + "recall_at_1": 0.10534, + "recall_at_10": 0.31092, + "recall_at_100": 0.6206, + "recall_at_1000": 0.90708, + "recall_at_20": 0.39832, + "recall_at_3": 0.18749, + "recall_at_5": 0.2365 + }, + { + "cv_recall_at_1": 0.01181, + "cv_recall_at_10": 0.04556, + "cv_recall_at_100": 0.11806, + "cv_recall_at_1000": 0.40236, + "cv_recall_at_20": 0.05708, + "cv_recall_at_3": 0.02333, + "cv_recall_at_5": 0.03236, + "hf_subset": "quz", + "languages": [ + "quz-Latn" + ], + "main_score": 0.02625, + "map_at_1": 0.01181, + "map_at_10": 0.02037, + "map_at_100": 0.02254, + "map_at_1000": 0.02335, + "map_at_20": 0.02118, + "map_at_3": 0.01667, + "map_at_5": 0.01865, + "mrr_at_1": 0.011805555555555555, + "mrr_at_10": 0.020373126102292763, + "mrr_at_100": 0.0225429811211209, + "mrr_at_1000": 0.023349026771023774, + "mrr_at_20": 0.021176072890936776, + "mrr_at_3": 0.016666666666666656, + "mrr_at_5": 0.01865277777777777, + "nauc_cv_recall_at_1000_diff1": 0.08667190720067616, + "nauc_cv_recall_at_1000_max": 0.16811377816078713, + "nauc_cv_recall_at_1000_std": 0.1776941853155657, + "nauc_cv_recall_at_100_diff1": 0.15789524491120618, + "nauc_cv_recall_at_100_max": 0.31809563437245164, + "nauc_cv_recall_at_100_std": 0.3395186658455805, + "nauc_cv_recall_at_10_diff1": 0.26798755947044317, + "nauc_cv_recall_at_10_max": 0.48983767104910125, + "nauc_cv_recall_at_10_std": 0.463757386768931, + "nauc_cv_recall_at_1_diff1": 0.5396391988509922, + "nauc_cv_recall_at_1_max": 0.7001152164649064, + "nauc_cv_recall_at_1_std": 0.5476285926229898, + "nauc_cv_recall_at_20_diff1": 0.23274755075248219, + "nauc_cv_recall_at_20_max": 0.43348759659406566, + "nauc_cv_recall_at_20_std": 0.4368803438053475, + "nauc_cv_recall_at_3_diff1": 0.37070540585431927, + "nauc_cv_recall_at_3_max": 0.5930676402570686, + "nauc_cv_recall_at_3_std": 0.5005478043108846, + "nauc_cv_recall_at_5_diff1": 0.33045137501425054, + "nauc_cv_recall_at_5_max": 0.5399457156971915, + "nauc_cv_recall_at_5_std": 0.4736610582099537, + "nauc_map_at_1000_diff1": 0.36268114588829053, + "nauc_map_at_1000_max": 0.5462730411169001, + "nauc_map_at_1000_std": 0.4733374126425002, + "nauc_map_at_100_diff1": 0.3684690660506013, + "nauc_map_at_100_max": 0.5545265516609, + "nauc_map_at_100_std": 0.4802315274638474, + "nauc_map_at_10_diff1": 0.39593461221974763, + "nauc_map_at_10_max": 0.5894828816343469, + "nauc_map_at_10_std": 0.4992925577248018, + "nauc_map_at_1_diff1": 0.5396391988509922, + "nauc_map_at_1_max": 0.7001152164649064, + "nauc_map_at_1_std": 0.5476285926229898, + "nauc_map_at_20_diff1": 0.38501958259804275, + "nauc_map_at_20_max": 0.5756154448581887, + "nauc_map_at_20_std": 0.4932341583371749, + "nauc_map_at_3_diff1": 0.44470157708016583, + "nauc_map_at_3_max": 0.6355059177771817, + "nauc_map_at_3_std": 0.5158617297200611, + "nauc_map_at_5_diff1": 0.4213217255125721, + "nauc_map_at_5_max": 0.6099601340775752, + "nauc_map_at_5_std": 0.5039017165794833, + "nauc_mrr_at_1000_diff1": 0.36268111735207437, + "nauc_mrr_at_1000_max": 0.5462730296720244, + "nauc_mrr_at_1000_std": 0.47333742053027184, + "nauc_mrr_at_100_diff1": 0.3684690660506013, + "nauc_mrr_at_100_max": 0.5545265516609, + "nauc_mrr_at_100_std": 0.4802315274638474, + "nauc_mrr_at_10_diff1": 0.39593461221974763, + "nauc_mrr_at_10_max": 0.5894828816343469, + "nauc_mrr_at_10_std": 0.4992925577248018, + "nauc_mrr_at_1_diff1": 0.5396391988509922, + "nauc_mrr_at_1_max": 0.7001152164649064, + "nauc_mrr_at_1_std": 0.5476285926229898, + "nauc_mrr_at_20_diff1": 0.38501958259804275, + "nauc_mrr_at_20_max": 0.5756154448581887, + "nauc_mrr_at_20_std": 0.4932341583371749, + "nauc_mrr_at_3_diff1": 0.44470157708016583, + "nauc_mrr_at_3_max": 0.6355059177771817, + "nauc_mrr_at_3_std": 0.5158617297200611, + "nauc_mrr_at_5_diff1": 0.4213217255125721, + "nauc_mrr_at_5_max": 0.6099601340775752, + "nauc_mrr_at_5_std": 0.5039017165794833, + "nauc_ndcg_at_1000_diff1": 0.2016366246175423, + "nauc_ndcg_at_1000_max": 0.33638671153257416, + "nauc_ndcg_at_1000_std": 0.31838659406129766, + "nauc_ndcg_at_100_diff1": 0.26037541254677204, + "nauc_ndcg_at_100_max": 0.4360310644584217, + "nauc_ndcg_at_100_std": 0.41292617667752046, + "nauc_ndcg_at_10_diff1": 0.34486121056368324, + "nauc_ndcg_at_10_max": 0.5497655909742565, + "nauc_ndcg_at_10_std": 0.48505160980469963, + "nauc_ndcg_at_1_diff1": 0.5396391988509922, + "nauc_ndcg_at_1_max": 0.7001152164649064, + "nauc_ndcg_at_1_std": 0.5476285926229898, + "nauc_ndcg_at_20_diff1": 0.32022522388215546, + "nauc_ndcg_at_20_max": 0.5162912607985376, + "nauc_ndcg_at_20_std": 0.4698969645910983, + "nauc_ndcg_at_3_diff1": 0.42087129551165053, + "nauc_ndcg_at_3_max": 0.6216838926839852, + "nauc_ndcg_at_3_std": 0.5107451829129399, + "nauc_ndcg_at_5_diff1": 0.3885307148680432, + "nauc_ndcg_at_5_max": 0.5848124141719698, + "nauc_ndcg_at_5_std": 0.4930263498076343, + "nauc_precision_at_1000_diff1": 0.08667190720067593, + "nauc_precision_at_1000_max": 0.16811377816078696, + "nauc_precision_at_1000_std": 0.17769418531556536, + "nauc_precision_at_100_diff1": 0.1578952449112062, + "nauc_precision_at_100_max": 0.3180956343724515, + "nauc_precision_at_100_std": 0.33951866584558027, + "nauc_precision_at_10_diff1": 0.26798755947044317, + "nauc_precision_at_10_max": 0.4898376710491013, + "nauc_precision_at_10_std": 0.463757386768931, + "nauc_precision_at_1_diff1": 0.5396391988509922, + "nauc_precision_at_1_max": 0.7001152164649064, + "nauc_precision_at_1_std": 0.5476285926229898, + "nauc_precision_at_20_diff1": 0.23274755075248224, + "nauc_precision_at_20_max": 0.43348759659406555, + "nauc_precision_at_20_std": 0.4368803438053475, + "nauc_precision_at_3_diff1": 0.37070540585431927, + "nauc_precision_at_3_max": 0.5930676402570686, + "nauc_precision_at_3_std": 0.5005478043108843, + "nauc_precision_at_5_diff1": 0.3304513750142505, + "nauc_precision_at_5_max": 0.539945715697192, + "nauc_precision_at_5_std": 0.47366105820995386, + "nauc_recall_at_1000_diff1": 0.08667190720067616, + "nauc_recall_at_1000_max": 0.16811377816078713, + "nauc_recall_at_1000_std": 0.1776941853155657, + "nauc_recall_at_100_diff1": 0.15789524491120618, + "nauc_recall_at_100_max": 0.31809563437245164, + "nauc_recall_at_100_std": 0.3395186658455805, + "nauc_recall_at_10_diff1": 0.26798755947044317, + "nauc_recall_at_10_max": 0.48983767104910125, + "nauc_recall_at_10_std": 0.463757386768931, + "nauc_recall_at_1_diff1": 0.5396391988509922, + "nauc_recall_at_1_max": 0.7001152164649064, + "nauc_recall_at_1_std": 0.5476285926229898, + "nauc_recall_at_20_diff1": 0.23274755075248219, + "nauc_recall_at_20_max": 0.43348759659406566, + "nauc_recall_at_20_std": 0.4368803438053475, + "nauc_recall_at_3_diff1": 0.37070540585431927, + "nauc_recall_at_3_max": 0.5930676402570686, + "nauc_recall_at_3_std": 0.5005478043108846, + "nauc_recall_at_5_diff1": 0.33045137501425054, + "nauc_recall_at_5_max": 0.5399457156971915, + "nauc_recall_at_5_std": 0.4736610582099537, + "ndcg_at_1": 0.01181, + "ndcg_at_10": 0.02625, + "ndcg_at_100": 0.04007, + "ndcg_at_1000": 0.07286, + "ndcg_at_20": 0.02917, + "ndcg_at_3": 0.01837, + "ndcg_at_5": 0.02202, + "precision_at_1": 0.01181, + "precision_at_10": 0.00456, + "precision_at_100": 0.00118, + "precision_at_1000": 0.0004, + "precision_at_20": 0.00285, + "precision_at_3": 0.00778, + "precision_at_5": 0.00647, + "recall_at_1": 0.01181, + "recall_at_10": 0.04556, + "recall_at_100": 0.11806, + "recall_at_1000": 0.40236, + "recall_at_20": 0.05708, + "recall_at_3": 0.02333, + "recall_at_5": 0.03236 + }, + { + "cv_recall_at_1": 0.03369, + "cv_recall_at_10": 0.11091, + "cv_recall_at_100": 0.26295, + "cv_recall_at_1000": 0.62488, + "cv_recall_at_20": 0.1446, + "cv_recall_at_3": 0.05981, + "cv_recall_at_5": 0.08044, + "hf_subset": "ro", + "languages": [ + "ron-Latn" + ], + "main_score": 0.06699, + "map_at_1": 0.03369, + "map_at_10": 0.05358, + "map_at_100": 0.05857, + "map_at_1000": 0.05976, + "map_at_20": 0.05587, + "map_at_3": 0.04478, + "map_at_5": 0.04947, + "mrr_at_1": 0.033693668398146845, + "mrr_at_10": 0.05357894947955309, + "mrr_at_100": 0.058565448685346744, + "mrr_at_1000": 0.059755076195600644, + "mrr_at_20": 0.05587114896195827, + "mrr_at_3": 0.04478450091253684, + "mrr_at_5": 0.04947353643127886, + "nauc_cv_recall_at_1000_diff1": -0.02750735436367819, + "nauc_cv_recall_at_1000_max": 0.2381781077979006, + "nauc_cv_recall_at_1000_std": 0.47236792234255026, + "nauc_cv_recall_at_100_diff1": 0.05755823909457514, + "nauc_cv_recall_at_100_max": 0.2768187610206619, + "nauc_cv_recall_at_100_std": 0.45525879717090767, + "nauc_cv_recall_at_10_diff1": 0.1244953534412135, + "nauc_cv_recall_at_10_max": 0.3349391617413094, + "nauc_cv_recall_at_10_std": 0.4056744665816779, + "nauc_cv_recall_at_1_diff1": 0.35164541350606676, + "nauc_cv_recall_at_1_max": 0.4524196178438682, + "nauc_cv_recall_at_1_std": 0.3814439266603003, + "nauc_cv_recall_at_20_diff1": 0.10413983945287474, + "nauc_cv_recall_at_20_max": 0.33140833112110973, + "nauc_cv_recall_at_20_std": 0.4386125720428651, + "nauc_cv_recall_at_3_diff1": 0.20245552150328452, + "nauc_cv_recall_at_3_max": 0.39518574373629967, + "nauc_cv_recall_at_3_std": 0.39401446691465314, + "nauc_cv_recall_at_5_diff1": 0.1570894857492748, + "nauc_cv_recall_at_5_max": 0.37154853140642485, + "nauc_cv_recall_at_5_std": 0.41356064041724844, + "nauc_map_at_1000_diff1": 0.21603653091241862, + "nauc_map_at_1000_max": 0.38683705895047166, + "nauc_map_at_1000_std": 0.4015751814700608, + "nauc_map_at_100_diff1": 0.21725138314361314, + "nauc_map_at_100_max": 0.3875455996624328, + "nauc_map_at_100_std": 0.4009878482275901, + "nauc_map_at_10_diff1": 0.2306561663320846, + "nauc_map_at_10_max": 0.3961959068993542, + "nauc_map_at_10_std": 0.39365347040267035, + "nauc_map_at_1_diff1": 0.35164541350606676, + "nauc_map_at_1_max": 0.4524196178438682, + "nauc_map_at_1_std": 0.3814439266603003, + "nauc_map_at_20_diff1": 0.22448119703894479, + "nauc_map_at_20_max": 0.394177621020983, + "nauc_map_at_20_std": 0.39904273371471993, + "nauc_map_at_3_diff1": 0.26782441090742687, + "nauc_map_at_3_max": 0.4198874910551511, + "nauc_map_at_3_std": 0.38756137620331704, + "nauc_map_at_5_diff1": 0.2454570039859637, + "nauc_map_at_5_max": 0.40897423169559444, + "nauc_map_at_5_std": 0.39559603455162656, + "nauc_mrr_at_1000_diff1": 0.2160365268297375, + "nauc_mrr_at_1000_max": 0.3868370537295254, + "nauc_mrr_at_1000_std": 0.4015751831776443, + "nauc_mrr_at_100_diff1": 0.21725138314361314, + "nauc_mrr_at_100_max": 0.3875455996624328, + "nauc_mrr_at_100_std": 0.4009878482275901, + "nauc_mrr_at_10_diff1": 0.2306561663320846, + "nauc_mrr_at_10_max": 0.3961959068993542, + "nauc_mrr_at_10_std": 0.39365347040267035, + "nauc_mrr_at_1_diff1": 0.35164541350606676, + "nauc_mrr_at_1_max": 0.4524196178438682, + "nauc_mrr_at_1_std": 0.3814439266603003, + "nauc_mrr_at_20_diff1": 0.22448119703894479, + "nauc_mrr_at_20_max": 0.394177621020983, + "nauc_mrr_at_20_std": 0.39904273371471993, + "nauc_mrr_at_3_diff1": 0.26782441090742687, + "nauc_mrr_at_3_max": 0.4198874910551511, + "nauc_mrr_at_3_std": 0.38756137620331704, + "nauc_mrr_at_5_diff1": 0.2454570039859637, + "nauc_mrr_at_5_max": 0.40897423169559444, + "nauc_mrr_at_5_std": 0.39559603455162656, + "nauc_ndcg_at_1000_diff1": 0.1256340994577069, + "nauc_ndcg_at_1000_max": 0.33073808738849186, + "nauc_ndcg_at_1000_std": 0.42844771829601985, + "nauc_ndcg_at_100_diff1": 0.1466673284994856, + "nauc_ndcg_at_100_max": 0.34043465035066833, + "nauc_ndcg_at_100_std": 0.4230647078541618, + "nauc_ndcg_at_10_diff1": 0.19009024385248605, + "nauc_ndcg_at_10_max": 0.3732493794963218, + "nauc_ndcg_at_10_std": 0.39822924971822304, + "nauc_ndcg_at_1_diff1": 0.35164541350606676, + "nauc_ndcg_at_1_max": 0.4524196178438682, + "nauc_ndcg_at_1_std": 0.3814439266603003, + "nauc_ndcg_at_20_diff1": 0.17574199361072354, + "nauc_ndcg_at_20_max": 0.3688816786833972, + "nauc_ndcg_at_20_std": 0.41355952693089276, + "nauc_ndcg_at_3_diff1": 0.24730011344565822, + "nauc_ndcg_at_3_max": 0.41211762593417495, + "nauc_ndcg_at_3_std": 0.38955204033843227, + "nauc_ndcg_at_5_diff1": 0.21484158800987502, + "nauc_ndcg_at_5_max": 0.3960497699441697, + "nauc_ndcg_at_5_std": 0.4016771024352979, + "nauc_precision_at_1000_diff1": -0.027507354363678547, + "nauc_precision_at_1000_max": 0.23817810779789986, + "nauc_precision_at_1000_std": 0.4723679223425496, + "nauc_precision_at_100_diff1": 0.05755823909457536, + "nauc_precision_at_100_max": 0.276818761020662, + "nauc_precision_at_100_std": 0.4552587971709079, + "nauc_precision_at_10_diff1": 0.12449535344121339, + "nauc_precision_at_10_max": 0.33493916174130933, + "nauc_precision_at_10_std": 0.4056744665816778, + "nauc_precision_at_1_diff1": 0.35164541350606676, + "nauc_precision_at_1_max": 0.4524196178438682, + "nauc_precision_at_1_std": 0.3814439266603003, + "nauc_precision_at_20_diff1": 0.10413983945287494, + "nauc_precision_at_20_max": 0.3314083311211099, + "nauc_precision_at_20_std": 0.43861257204286525, + "nauc_precision_at_3_diff1": 0.20245552150328444, + "nauc_precision_at_3_max": 0.3951857437362995, + "nauc_precision_at_3_std": 0.3940144669146529, + "nauc_precision_at_5_diff1": 0.15708948574927478, + "nauc_precision_at_5_max": 0.371548531406425, + "nauc_precision_at_5_std": 0.4135606404172485, + "nauc_recall_at_1000_diff1": -0.02750735436367819, + "nauc_recall_at_1000_max": 0.2381781077979006, + "nauc_recall_at_1000_std": 0.47236792234255026, + "nauc_recall_at_100_diff1": 0.05755823909457514, + "nauc_recall_at_100_max": 0.2768187610206619, + "nauc_recall_at_100_std": 0.45525879717090767, + "nauc_recall_at_10_diff1": 0.1244953534412135, + "nauc_recall_at_10_max": 0.3349391617413094, + "nauc_recall_at_10_std": 0.4056744665816779, + "nauc_recall_at_1_diff1": 0.35164541350606676, + "nauc_recall_at_1_max": 0.4524196178438682, + "nauc_recall_at_1_std": 0.3814439266603003, + "nauc_recall_at_20_diff1": 0.10413983945287474, + "nauc_recall_at_20_max": 0.33140833112110973, + "nauc_recall_at_20_std": 0.4386125720428651, + "nauc_recall_at_3_diff1": 0.20245552150328452, + "nauc_recall_at_3_max": 0.39518574373629967, + "nauc_recall_at_3_std": 0.39401446691465314, + "nauc_recall_at_5_diff1": 0.1570894857492748, + "nauc_recall_at_5_max": 0.37154853140642485, + "nauc_recall_at_5_std": 0.41356064041724844, + "ndcg_at_1": 0.03369, + "ndcg_at_10": 0.06699, + "ndcg_at_100": 0.09672, + "ndcg_at_1000": 0.1395, + "ndcg_at_20": 0.07545, + "ndcg_at_3": 0.04862, + "ndcg_at_5": 0.0571, + "precision_at_1": 0.03369, + "precision_at_10": 0.01109, + "precision_at_100": 0.00263, + "precision_at_1000": 0.00062, + "precision_at_20": 0.00723, + "precision_at_3": 0.01994, + "precision_at_5": 0.01609, + "recall_at_1": 0.03369, + "recall_at_10": 0.11091, + "recall_at_100": 0.26295, + "recall_at_1000": 0.62488, + "recall_at_20": 0.1446, + "recall_at_3": 0.05981, + "recall_at_5": 0.08044 + }, + { + "cv_recall_at_1": 0.00653, + "cv_recall_at_10": 0.01736, + "cv_recall_at_100": 0.07069, + "cv_recall_at_1000": 0.37639, + "cv_recall_at_20": 0.02528, + "cv_recall_at_3": 0.01097, + "cv_recall_at_5": 0.01333, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.01126, + "map_at_1": 0.00653, + "map_at_10": 0.0094, + "map_at_100": 0.01089, + "map_at_1000": 0.01175, + "map_at_20": 0.00995, + "map_at_3": 0.00836, + "map_at_5": 0.00889, + "mrr_at_1": 0.006527777777777778, + "mrr_at_10": 0.00940018738977073, + "mrr_at_100": 0.01088995737556746, + "mrr_at_1000": 0.011745707743364286, + "mrr_at_20": 0.009950996655443904, + "mrr_at_3": 0.008356481481481486, + "mrr_at_5": 0.00889120370370371, + "nauc_cv_recall_at_1000_diff1": 0.024532552702524496, + "nauc_cv_recall_at_1000_max": 0.14192208036422993, + "nauc_cv_recall_at_1000_std": 0.2185378722265516, + "nauc_cv_recall_at_100_diff1": 0.08880346939948125, + "nauc_cv_recall_at_100_max": 0.26940032461045954, + "nauc_cv_recall_at_100_std": 0.402973996899385, + "nauc_cv_recall_at_10_diff1": 0.3576538771129596, + "nauc_cv_recall_at_10_max": 0.5124894016635365, + "nauc_cv_recall_at_10_std": 0.6625961899651195, + "nauc_cv_recall_at_1_diff1": 0.5424532879676199, + "nauc_cv_recall_at_1_max": 0.5945287122721484, + "nauc_cv_recall_at_1_std": 0.7447093949271848, + "nauc_cv_recall_at_20_diff1": 0.2560805172825565, + "nauc_cv_recall_at_20_max": 0.4391653423045964, + "nauc_cv_recall_at_20_std": 0.5729942180170247, + "nauc_cv_recall_at_3_diff1": 0.4817462716475395, + "nauc_cv_recall_at_3_max": 0.6084202518060134, + "nauc_cv_recall_at_3_std": 0.7518212972051367, + "nauc_cv_recall_at_5_diff1": 0.45935627403631146, + "nauc_cv_recall_at_5_max": 0.6203604328772024, + "nauc_cv_recall_at_5_std": 0.7455197433145516, + "nauc_map_at_1000_diff1": 0.39858061256455646, + "nauc_map_at_1000_max": 0.5120458013119253, + "nauc_map_at_1000_std": 0.6542261352485719, + "nauc_map_at_100_diff1": 0.4183137782320978, + "nauc_map_at_100_max": 0.532310812827618, + "nauc_map_at_100_std": 0.6788068616607587, + "nauc_map_at_10_diff1": 0.4800763361335301, + "nauc_map_at_10_max": 0.581327495038239, + "nauc_map_at_10_std": 0.7316489298319947, + "nauc_map_at_1_diff1": 0.5424532879676199, + "nauc_map_at_1_max": 0.5945287122721484, + "nauc_map_at_1_std": 0.7447093949271848, + "nauc_map_at_20_diff1": 0.4564129368416336, + "nauc_map_at_20_max": 0.5651552122786756, + "nauc_map_at_20_std": 0.712024497097081, + "nauc_map_at_3_diff1": 0.5150370100048537, + "nauc_map_at_3_max": 0.6003811372725942, + "nauc_map_at_3_std": 0.7522800972464915, + "nauc_map_at_5_diff1": 0.5053000588806172, + "nauc_map_at_5_max": 0.6048658623537564, + "nauc_map_at_5_std": 0.7502147162144954, + "nauc_mrr_at_1000_diff1": 0.3985808585466461, + "nauc_mrr_at_1000_max": 0.5120460518793085, + "nauc_mrr_at_1000_std": 0.654226196079803, + "nauc_mrr_at_100_diff1": 0.4183137782320978, + "nauc_mrr_at_100_max": 0.532310812827618, + "nauc_mrr_at_100_std": 0.6788068616607587, + "nauc_mrr_at_10_diff1": 0.4800763361335301, + "nauc_mrr_at_10_max": 0.581327495038239, + "nauc_mrr_at_10_std": 0.7316489298319947, + "nauc_mrr_at_1_diff1": 0.5424532879676199, + "nauc_mrr_at_1_max": 0.5945287122721484, + "nauc_mrr_at_1_std": 0.7447093949271848, + "nauc_mrr_at_20_diff1": 0.4564129368416336, + "nauc_mrr_at_20_max": 0.5651552122786756, + "nauc_mrr_at_20_std": 0.712024497097081, + "nauc_mrr_at_3_diff1": 0.5150370100048537, + "nauc_mrr_at_3_max": 0.6003811372725942, + "nauc_mrr_at_3_std": 0.7522800972464915, + "nauc_mrr_at_5_diff1": 0.5053000588806172, + "nauc_mrr_at_5_max": 0.6048658623537564, + "nauc_mrr_at_5_std": 0.7502147162144954, + "nauc_ndcg_at_1000_diff1": 0.12888500419156781, + "nauc_ndcg_at_1000_max": 0.2535818040636635, + "nauc_ndcg_at_1000_std": 0.35265764685265366, + "nauc_ndcg_at_100_diff1": 0.23620062158291075, + "nauc_ndcg_at_100_max": 0.38965171262518355, + "nauc_ndcg_at_100_std": 0.529019771584767, + "nauc_ndcg_at_10_diff1": 0.4380991139616366, + "nauc_ndcg_at_10_max": 0.5591425095770631, + "nauc_ndcg_at_10_std": 0.7089649644315631, + "nauc_ndcg_at_1_diff1": 0.5424532879676199, + "nauc_ndcg_at_1_max": 0.5945287122721484, + "nauc_ndcg_at_1_std": 0.7447093949271848, + "nauc_ndcg_at_20_diff1": 0.3780217086880114, + "nauc_ndcg_at_20_max": 0.5173384500681644, + "nauc_ndcg_at_20_std": 0.6588354395696102, + "nauc_ndcg_at_3_diff1": 0.5049257793161163, + "nauc_ndcg_at_3_max": 0.6028096366542133, + "nauc_ndcg_at_3_std": 0.7522771466939655, + "nauc_ndcg_at_5_diff1": 0.490324978851244, + "nauc_ndcg_at_5_max": 0.6098454030674927, + "nauc_ndcg_at_5_std": 0.7488643686971155, + "nauc_precision_at_1000_diff1": 0.024532552702524055, + "nauc_precision_at_1000_max": 0.14192208036422974, + "nauc_precision_at_1000_std": 0.21853787222655116, + "nauc_precision_at_100_diff1": 0.08880346939948128, + "nauc_precision_at_100_max": 0.2694003246104596, + "nauc_precision_at_100_std": 0.40297399689938496, + "nauc_precision_at_10_diff1": 0.35765387711295954, + "nauc_precision_at_10_max": 0.5124894016635364, + "nauc_precision_at_10_std": 0.6625961899651195, + "nauc_precision_at_1_diff1": 0.5424532879676199, + "nauc_precision_at_1_max": 0.5945287122721484, + "nauc_precision_at_1_std": 0.7447093949271848, + "nauc_precision_at_20_diff1": 0.2560805172825565, + "nauc_precision_at_20_max": 0.4391653423045967, + "nauc_precision_at_20_std": 0.5729942180170249, + "nauc_precision_at_3_diff1": 0.4817462716475393, + "nauc_precision_at_3_max": 0.6084202518060132, + "nauc_precision_at_3_std": 0.7518212972051364, + "nauc_precision_at_5_diff1": 0.45935627403631163, + "nauc_precision_at_5_max": 0.6203604328772026, + "nauc_precision_at_5_std": 0.7455197433145517, + "nauc_recall_at_1000_diff1": 0.024532552702524496, + "nauc_recall_at_1000_max": 0.14192208036422993, + "nauc_recall_at_1000_std": 0.2185378722265516, + "nauc_recall_at_100_diff1": 0.08880346939948125, + "nauc_recall_at_100_max": 0.26940032461045954, + "nauc_recall_at_100_std": 0.402973996899385, + "nauc_recall_at_10_diff1": 0.3576538771129596, + "nauc_recall_at_10_max": 0.5124894016635365, + "nauc_recall_at_10_std": 0.6625961899651195, + "nauc_recall_at_1_diff1": 0.5424532879676199, + "nauc_recall_at_1_max": 0.5945287122721484, + "nauc_recall_at_1_std": 0.7447093949271848, + "nauc_recall_at_20_diff1": 0.2560805172825565, + "nauc_recall_at_20_max": 0.4391653423045964, + "nauc_recall_at_20_std": 0.5729942180170247, + "nauc_recall_at_3_diff1": 0.4817462716475395, + "nauc_recall_at_3_max": 0.6084202518060134, + "nauc_recall_at_3_std": 0.7518212972051367, + "nauc_recall_at_5_diff1": 0.45935627403631146, + "nauc_recall_at_5_max": 0.6203604328772024, + "nauc_recall_at_5_std": 0.7455197433145516, + "ndcg_at_1": 0.00653, + "ndcg_at_10": 0.01126, + "ndcg_at_100": 0.02124, + "ndcg_at_1000": 0.05644, + "ndcg_at_20": 0.01327, + "ndcg_at_3": 0.00902, + "ndcg_at_5": 0.00999, + "precision_at_1": 0.00653, + "precision_at_10": 0.00174, + "precision_at_100": 0.00071, + "precision_at_1000": 0.00038, + "precision_at_20": 0.00126, + "precision_at_3": 0.00366, + "precision_at_5": 0.00267, + "recall_at_1": 0.00653, + "recall_at_10": 0.01736, + "recall_at_100": 0.07069, + "recall_at_1000": 0.37639, + "recall_at_20": 0.02528, + "recall_at_3": 0.01097, + "recall_at_5": 0.01333 + }, + { + "cv_recall_at_1": 0.03231, + "cv_recall_at_10": 0.11646, + "cv_recall_at_100": 0.28654, + "cv_recall_at_1000": 0.66011, + "cv_recall_at_20": 0.15124, + "cv_recall_at_3": 0.06572, + "cv_recall_at_5": 0.08456, + "hf_subset": "sv", + "languages": [ + "swe-Latn" + ], + "main_score": 0.06925, + "map_at_1": 0.03231, + "map_at_10": 0.05478, + "map_at_100": 0.06028, + "map_at_1000": 0.06155, + "map_at_20": 0.05716, + "map_at_3": 0.04634, + "map_at_5": 0.05056, + "mrr_at_1": 0.03231128832668775, + "mrr_at_10": 0.05477609073786709, + "mrr_at_100": 0.06027529180422331, + "mrr_at_1000": 0.06154752807105916, + "mrr_at_20": 0.05716054677837161, + "mrr_at_3": 0.046335762408909624, + "mrr_at_5": 0.05056372886016759, + "nauc_cv_recall_at_1000_diff1": 0.06548065830489194, + "nauc_cv_recall_at_1000_max": 0.06237750952655973, + "nauc_cv_recall_at_1000_std": 0.4660590815935642, + "nauc_cv_recall_at_100_diff1": 0.0851957419101093, + "nauc_cv_recall_at_100_max": 0.09897022897570883, + "nauc_cv_recall_at_100_std": 0.42192223874633566, + "nauc_cv_recall_at_10_diff1": 0.14987876127649138, + "nauc_cv_recall_at_10_max": 0.1678938870971709, + "nauc_cv_recall_at_10_std": 0.3794087013292822, + "nauc_cv_recall_at_1_diff1": 0.36422330937694863, + "nauc_cv_recall_at_1_max": 0.2951200582537345, + "nauc_cv_recall_at_1_std": 0.3413342988414905, + "nauc_cv_recall_at_20_diff1": 0.1290383181182195, + "nauc_cv_recall_at_20_max": 0.13366207759059023, + "nauc_cv_recall_at_20_std": 0.3742757190843819, + "nauc_cv_recall_at_3_diff1": 0.2059850506998725, + "nauc_cv_recall_at_3_max": 0.23042996884778455, + "nauc_cv_recall_at_3_std": 0.3839113945804567, + "nauc_cv_recall_at_5_diff1": 0.17099262193583747, + "nauc_cv_recall_at_5_max": 0.19722962011318293, + "nauc_cv_recall_at_5_std": 0.38142108667220237, + "nauc_map_at_1000_diff1": 0.2249431356126328, + "nauc_map_at_1000_max": 0.2206935159089587, + "nauc_map_at_1000_std": 0.3697545426309123, + "nauc_map_at_100_diff1": 0.22568599685467652, + "nauc_map_at_100_max": 0.22145061764922253, + "nauc_map_at_100_std": 0.36889370714450276, + "nauc_map_at_10_diff1": 0.23874524889661422, + "nauc_map_at_10_max": 0.23508221440558008, + "nauc_map_at_10_std": 0.366064655472392, + "nauc_map_at_1_diff1": 0.36422330937694863, + "nauc_map_at_1_max": 0.2951200582537345, + "nauc_map_at_1_std": 0.3413342988414905, + "nauc_map_at_20_diff1": 0.23263970425299382, + "nauc_map_at_20_max": 0.22750740972522127, + "nauc_map_at_20_std": 0.3658400097951212, + "nauc_map_at_3_diff1": 0.26758268789384415, + "nauc_map_at_3_max": 0.2598950633798191, + "nauc_map_at_3_std": 0.36573044558195744, + "nauc_map_at_5_diff1": 0.24956071499912286, + "nauc_map_at_5_max": 0.24540260206801137, + "nauc_map_at_5_std": 0.36597438240167834, + "nauc_mrr_at_1000_diff1": 0.2249431249999408, + "nauc_mrr_at_1000_max": 0.22069352402740708, + "nauc_mrr_at_1000_std": 0.36975454576408934, + "nauc_mrr_at_100_diff1": 0.22568599685467652, + "nauc_mrr_at_100_max": 0.22145061764922253, + "nauc_mrr_at_100_std": 0.36889370714450276, + "nauc_mrr_at_10_diff1": 0.23874524889661422, + "nauc_mrr_at_10_max": 0.23508221440558008, + "nauc_mrr_at_10_std": 0.366064655472392, + "nauc_mrr_at_1_diff1": 0.36422330937694863, + "nauc_mrr_at_1_max": 0.2951200582537345, + "nauc_mrr_at_1_std": 0.3413342988414905, + "nauc_mrr_at_20_diff1": 0.23263970425299382, + "nauc_mrr_at_20_max": 0.22750740972522127, + "nauc_mrr_at_20_std": 0.3658400097951212, + "nauc_mrr_at_3_diff1": 0.26758268789384415, + "nauc_mrr_at_3_max": 0.2598950633798191, + "nauc_mrr_at_3_std": 0.36573044558195744, + "nauc_mrr_at_5_diff1": 0.24956071499912286, + "nauc_mrr_at_5_max": 0.24540260206801137, + "nauc_mrr_at_5_std": 0.36597438240167834, + "nauc_ndcg_at_1000_diff1": 0.1590560410499789, + "nauc_ndcg_at_1000_max": 0.16025781184624488, + "nauc_ndcg_at_1000_std": 0.4001104098966938, + "nauc_ndcg_at_100_diff1": 0.16297583355014392, + "nauc_ndcg_at_100_max": 0.16781657473650827, + "nauc_ndcg_at_100_std": 0.3889778490410969, + "nauc_ndcg_at_10_diff1": 0.2041448033747642, + "nauc_ndcg_at_10_max": 0.20982886824766298, + "nauc_ndcg_at_10_std": 0.37141720258175265, + "nauc_ndcg_at_1_diff1": 0.36422330937694863, + "nauc_ndcg_at_1_max": 0.2951200582537345, + "nauc_ndcg_at_1_std": 0.3413342988414905, + "nauc_ndcg_at_20_diff1": 0.19013064317884823, + "nauc_ndcg_at_20_max": 0.1907506207789357, + "nauc_ndcg_at_20_std": 0.369922491987889, + "nauc_ndcg_at_3_diff1": 0.24734016858623997, + "nauc_ndcg_at_3_max": 0.2503765220804391, + "nauc_ndcg_at_3_std": 0.3716434714330145, + "nauc_ndcg_at_5_diff1": 0.2216607014157429, + "nauc_ndcg_at_5_max": 0.22866568745631413, + "nauc_ndcg_at_5_std": 0.3715288106986452, + "nauc_precision_at_1000_diff1": 0.0654806583048908, + "nauc_precision_at_1000_max": 0.062377509526559224, + "nauc_precision_at_1000_std": 0.4660590815935634, + "nauc_precision_at_100_diff1": 0.0851957419101095, + "nauc_precision_at_100_max": 0.09897022897570915, + "nauc_precision_at_100_std": 0.4219222387463361, + "nauc_precision_at_10_diff1": 0.14987876127649127, + "nauc_precision_at_10_max": 0.16789388709717096, + "nauc_precision_at_10_std": 0.3794087013292824, + "nauc_precision_at_1_diff1": 0.36422330937694863, + "nauc_precision_at_1_max": 0.2951200582537345, + "nauc_precision_at_1_std": 0.3413342988414905, + "nauc_precision_at_20_diff1": 0.12903831811821959, + "nauc_precision_at_20_max": 0.13366207759059015, + "nauc_precision_at_20_std": 0.37427571908438173, + "nauc_precision_at_3_diff1": 0.20598505069987239, + "nauc_precision_at_3_max": 0.2304299688477843, + "nauc_precision_at_3_std": 0.38391139458045653, + "nauc_precision_at_5_diff1": 0.1709926219358375, + "nauc_precision_at_5_max": 0.19722962011318282, + "nauc_precision_at_5_std": 0.3814210866722025, + "nauc_recall_at_1000_diff1": 0.06548065830489194, + "nauc_recall_at_1000_max": 0.06237750952655973, + "nauc_recall_at_1000_std": 0.4660590815935642, + "nauc_recall_at_100_diff1": 0.0851957419101093, + "nauc_recall_at_100_max": 0.09897022897570883, + "nauc_recall_at_100_std": 0.42192223874633566, + "nauc_recall_at_10_diff1": 0.14987876127649138, + "nauc_recall_at_10_max": 0.1678938870971709, + "nauc_recall_at_10_std": 0.3794087013292822, + "nauc_recall_at_1_diff1": 0.36422330937694863, + "nauc_recall_at_1_max": 0.2951200582537345, + "nauc_recall_at_1_std": 0.3413342988414905, + "nauc_recall_at_20_diff1": 0.1290383181182195, + "nauc_recall_at_20_max": 0.13366207759059023, + "nauc_recall_at_20_std": 0.3742757190843819, + "nauc_recall_at_3_diff1": 0.2059850506998725, + "nauc_recall_at_3_max": 0.23042996884778455, + "nauc_recall_at_3_std": 0.3839113945804567, + "nauc_recall_at_5_diff1": 0.17099262193583747, + "nauc_recall_at_5_max": 0.19722962011318293, + "nauc_recall_at_5_std": 0.38142108667220237, + "ndcg_at_1": 0.03231, + "ndcg_at_10": 0.06925, + "ndcg_at_100": 0.10239, + "ndcg_at_1000": 0.14682, + "ndcg_at_20": 0.078, + "ndcg_at_3": 0.05129, + "ndcg_at_5": 0.05898, + "precision_at_1": 0.03231, + "precision_at_10": 0.01165, + "precision_at_100": 0.00287, + "precision_at_1000": 0.00066, + "precision_at_20": 0.00756, + "precision_at_3": 0.02191, + "precision_at_5": 0.01691, + "recall_at_1": 0.03231, + "recall_at_10": 0.11646, + "recall_at_100": 0.28654, + "recall_at_1000": 0.66011, + "recall_at_20": 0.15124, + "recall_at_3": 0.06572, + "recall_at_5": 0.08456 + }, + { + "cv_recall_at_1": 0.01093, + "cv_recall_at_10": 0.04116, + "cv_recall_at_100": 0.09523, + "cv_recall_at_1000": 0.34601, + "cv_recall_at_20": 0.05436, + "cv_recall_at_3": 0.02285, + "cv_recall_at_5": 0.03051, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ], + "main_score": 0.02425, + "map_at_1": 0.01093, + "map_at_10": 0.01906, + "map_at_100": 0.02086, + "map_at_1000": 0.02153, + "map_at_20": 0.01995, + "map_at_3": 0.01599, + "map_at_5": 0.01771, + "mrr_at_1": 0.010928186204938973, + "mrr_at_10": 0.01906333211683765, + "mrr_at_100": 0.020855486004402696, + "mrr_at_1000": 0.0215304990410364, + "mrr_at_20": 0.019949391755870887, + "mrr_at_3": 0.01599015990159901, + "mrr_at_5": 0.017707446305232282, + "nauc_cv_recall_at_1000_diff1": -0.029066888331591732, + "nauc_cv_recall_at_1000_max": 0.11440054004195425, + "nauc_cv_recall_at_1000_std": 0.17961864663480073, + "nauc_cv_recall_at_100_diff1": -0.027853539828646603, + "nauc_cv_recall_at_100_max": 0.28190708885837534, + "nauc_cv_recall_at_100_std": 0.41941312173899065, + "nauc_cv_recall_at_10_diff1": -0.030576462166714554, + "nauc_cv_recall_at_10_max": 0.45854089032862744, + "nauc_cv_recall_at_10_std": 0.5998171835323256, + "nauc_cv_recall_at_1_diff1": 0.0507230754331051, + "nauc_cv_recall_at_1_max": 0.5739380056956582, + "nauc_cv_recall_at_1_std": 0.640247504575004, + "nauc_cv_recall_at_20_diff1": -0.038133309010667873, + "nauc_cv_recall_at_20_max": 0.4169107885199648, + "nauc_cv_recall_at_20_std": 0.5630257929220277, + "nauc_cv_recall_at_3_diff1": -0.046575756766060704, + "nauc_cv_recall_at_3_max": 0.4955809528793124, + "nauc_cv_recall_at_3_std": 0.6080999130990666, + "nauc_cv_recall_at_5_diff1": -0.049400486753644265, + "nauc_cv_recall_at_5_max": 0.4755393479810073, + "nauc_cv_recall_at_5_std": 0.5964315119056258, + "nauc_map_at_1000_diff1": -0.015275603948263626, + "nauc_map_at_1000_max": 0.47149663581085133, + "nauc_map_at_1000_std": 0.5803479917322986, + "nauc_map_at_100_diff1": -0.015009992508063967, + "nauc_map_at_100_max": 0.48032511689420504, + "nauc_map_at_100_std": 0.5899444038260839, + "nauc_map_at_10_diff1": -0.012038249666452248, + "nauc_map_at_10_max": 0.5073521391195545, + "nauc_map_at_10_std": 0.6118480534729175, + "nauc_map_at_1_diff1": 0.0507230754331051, + "nauc_map_at_1_max": 0.5739380056956582, + "nauc_map_at_1_std": 0.640247504575004, + "nauc_map_at_20_diff1": -0.015037961413821672, + "nauc_map_at_20_max": 0.49703156017996536, + "nauc_map_at_20_std": 0.6049687714113716, + "nauc_map_at_3_diff1": -0.00927438069788099, + "nauc_map_at_3_max": 0.5256291385247881, + "nauc_map_at_3_std": 0.6179931686874017, + "nauc_map_at_5_diff1": -0.014151180614339051, + "nauc_map_at_5_max": 0.5137793474810228, + "nauc_map_at_5_std": 0.6122338853382189, + "nauc_mrr_at_1000_diff1": -0.015275754819867498, + "nauc_mrr_at_1000_max": 0.4714939865282889, + "nauc_mrr_at_1000_std": 0.5803448405218806, + "nauc_mrr_at_100_diff1": -0.015010146863816942, + "nauc_mrr_at_100_max": 0.48032239486348116, + "nauc_mrr_at_100_std": 0.5899411685976717, + "nauc_mrr_at_10_diff1": -0.012038249666452248, + "nauc_mrr_at_10_max": 0.5073521391195545, + "nauc_mrr_at_10_std": 0.6118480534729175, + "nauc_mrr_at_1_diff1": 0.0507230754331051, + "nauc_mrr_at_1_max": 0.5739380056956582, + "nauc_mrr_at_1_std": 0.640247504575004, + "nauc_mrr_at_20_diff1": -0.015037961413821672, + "nauc_mrr_at_20_max": 0.49703156017996536, + "nauc_mrr_at_20_std": 0.6049687714113716, + "nauc_mrr_at_3_diff1": -0.00927438069788099, + "nauc_mrr_at_3_max": 0.5256291385247881, + "nauc_mrr_at_3_std": 0.6179931686874017, + "nauc_mrr_at_5_diff1": -0.014151180614339051, + "nauc_mrr_at_5_max": 0.5137793474810228, + "nauc_mrr_at_5_std": 0.6122338853382189, + "nauc_ndcg_at_1000_diff1": -0.025189927360568656, + "nauc_ndcg_at_1000_max": 0.2755200466099936, + "nauc_ndcg_at_1000_std": 0.3708203542848814, + "nauc_ndcg_at_100_diff1": -0.023820824475258018, + "nauc_ndcg_at_100_max": 0.38842037833035764, + "nauc_ndcg_at_100_std": 0.5143521973344823, + "nauc_ndcg_at_10_diff1": -0.020566808342009864, + "nauc_ndcg_at_10_max": 0.4880795737018556, + "nauc_ndcg_at_10_std": 0.6067670926354549, + "nauc_ndcg_at_1_diff1": 0.0507230754331051, + "nauc_ndcg_at_1_max": 0.5739380056956582, + "nauc_ndcg_at_1_std": 0.640247504575004, + "nauc_ndcg_at_20_diff1": -0.026200186616833147, + "nauc_ndcg_at_20_max": 0.4635014088103323, + "nauc_ndcg_at_20_std": 0.5881263187112693, + "nauc_ndcg_at_3_diff1": -0.021590017901252544, + "nauc_ndcg_at_3_max": 0.5157085805179946, + "nauc_ndcg_at_3_std": 0.6146341160023117, + "nauc_ndcg_at_5_diff1": -0.0271432345961738, + "nauc_ndcg_at_5_max": 0.49984923150505894, + "nauc_ndcg_at_5_std": 0.6064767042528324, + "nauc_precision_at_1000_diff1": -0.02906688833159206, + "nauc_precision_at_1000_max": 0.11440054004195374, + "nauc_precision_at_1000_std": 0.17961864663480062, + "nauc_precision_at_100_diff1": -0.027853539828646398, + "nauc_precision_at_100_max": 0.2819070888583754, + "nauc_precision_at_100_std": 0.4194131217389907, + "nauc_precision_at_10_diff1": -0.030576462166714447, + "nauc_precision_at_10_max": 0.4585408903286276, + "nauc_precision_at_10_std": 0.5998171835323257, + "nauc_precision_at_1_diff1": 0.0507230754331051, + "nauc_precision_at_1_max": 0.5739380056956582, + "nauc_precision_at_1_std": 0.640247504575004, + "nauc_precision_at_20_diff1": -0.03813330901066789, + "nauc_precision_at_20_max": 0.4169107885199649, + "nauc_precision_at_20_std": 0.5630257929220278, + "nauc_precision_at_3_diff1": -0.04657575676606086, + "nauc_precision_at_3_max": 0.4955809528793123, + "nauc_precision_at_3_std": 0.6080999130990669, + "nauc_precision_at_5_diff1": -0.04940048675364416, + "nauc_precision_at_5_max": 0.4755393479810075, + "nauc_precision_at_5_std": 0.5964315119056262, + "nauc_recall_at_1000_diff1": -0.029066888331591732, + "nauc_recall_at_1000_max": 0.11440054004195425, + "nauc_recall_at_1000_std": 0.17961864663480073, + "nauc_recall_at_100_diff1": -0.027853539828646603, + "nauc_recall_at_100_max": 0.28190708885837534, + "nauc_recall_at_100_std": 0.41941312173899065, + "nauc_recall_at_10_diff1": -0.030576462166714554, + "nauc_recall_at_10_max": 0.45854089032862744, + "nauc_recall_at_10_std": 0.5998171835323256, + "nauc_recall_at_1_diff1": 0.0507230754331051, + "nauc_recall_at_1_max": 0.5739380056956582, + "nauc_recall_at_1_std": 0.640247504575004, + "nauc_recall_at_20_diff1": -0.038133309010667873, + "nauc_recall_at_20_max": 0.4169107885199648, + "nauc_recall_at_20_std": 0.5630257929220277, + "nauc_recall_at_3_diff1": -0.046575756766060704, + "nauc_recall_at_3_max": 0.4955809528793124, + "nauc_recall_at_3_std": 0.6080999130990666, + "nauc_recall_at_5_diff1": -0.049400486753644265, + "nauc_recall_at_5_max": 0.4755393479810073, + "nauc_recall_at_5_std": 0.5964315119056258, + "ndcg_at_1": 0.01093, + "ndcg_at_10": 0.02425, + "ndcg_at_100": 0.03484, + "ndcg_at_1000": 0.06352, + "ndcg_at_20": 0.02755, + "ndcg_at_3": 0.01774, + "ndcg_at_5": 0.02087, + "precision_at_1": 0.01093, + "precision_at_10": 0.00412, + "precision_at_100": 0.00095, + "precision_at_1000": 0.00035, + "precision_at_20": 0.00272, + "precision_at_3": 0.00762, + "precision_at_5": 0.0061, + "recall_at_1": 0.01093, + "recall_at_10": 0.04116, + "recall_at_100": 0.09523, + "recall_at_1000": 0.34601, + "recall_at_20": 0.05436, + "recall_at_3": 0.02285, + "recall_at_5": 0.03051 + }, + { + "cv_recall_at_1": 0.00028, + "cv_recall_at_10": 0.00306, + "cv_recall_at_100": 0.02806, + "cv_recall_at_1000": 0.28208, + "cv_recall_at_20": 0.00542, + "cv_recall_at_3": 0.00083, + "cv_recall_at_5": 0.00125, + "hf_subset": "te", + "languages": [ + "tel-Telu" + ], + "main_score": 0.00133, + "map_at_1": 0.00028, + "map_at_10": 0.00082, + "map_at_100": 0.00143, + "map_at_1000": 0.00209, + "map_at_20": 0.00097, + "map_at_3": 0.00051, + "map_at_5": 0.0006, + "mrr_at_1": 0.0002777777777777778, + "mrr_at_10": 0.0008230268959435623, + "mrr_at_100": 0.0014292767932411472, + "mrr_at_1000": 0.0020886165708559946, + "mrr_at_20": 0.0009712024307773786, + "mrr_at_3": 0.0005092592592592592, + "mrr_at_5": 0.000599537037037037, + "nauc_cv_recall_at_1000_diff1": -0.006836242757369639, + "nauc_cv_recall_at_1000_max": 0.049066917816044116, + "nauc_cv_recall_at_1000_std": 0.037085901708809384, + "nauc_cv_recall_at_100_diff1": 0.04301099549182174, + "nauc_cv_recall_at_100_max": 0.10517311457467186, + "nauc_cv_recall_at_100_std": 0.07455761674906419, + "nauc_cv_recall_at_10_diff1": -0.07094421543039736, + "nauc_cv_recall_at_10_max": -0.06462668000097577, + "nauc_cv_recall_at_10_std": -0.020635656267531922, + "nauc_cv_recall_at_1_diff1": 0.22806546820499068, + "nauc_cv_recall_at_1_max": 0.22806546820499068, + "nauc_cv_recall_at_1_std": 0.17171988194258117, + "nauc_cv_recall_at_20_diff1": -0.07966123850212935, + "nauc_cv_recall_at_20_max": -0.08551595082316851, + "nauc_cv_recall_at_20_std": 0.00919141675782415, + "nauc_cv_recall_at_3_diff1": -0.17789106519989273, + "nauc_cv_recall_at_3_max": -0.03063232269027806, + "nauc_cv_recall_at_3_std": 0.003353903944191056, + "nauc_cv_recall_at_5_diff1": -0.06949288972363836, + "nauc_cv_recall_at_5_max": -0.02900754255731451, + "nauc_cv_recall_at_5_std": -0.020033986226634583, + "nauc_map_at_1000_diff1": -0.011257585036496497, + "nauc_map_at_1000_max": 0.029375661768316776, + "nauc_map_at_1000_std": 0.04410516975714857, + "nauc_map_at_100_diff1": -0.011517552807642387, + "nauc_map_at_100_max": 0.025626619594148595, + "nauc_map_at_100_std": 0.04467982209656915, + "nauc_map_at_10_diff1": -0.03690924911344003, + "nauc_map_at_10_max": 0.0012078815651148187, + "nauc_map_at_10_std": 0.029082751539292167, + "nauc_map_at_1_diff1": 0.22806546820499068, + "nauc_map_at_1_max": 0.22806546820499068, + "nauc_map_at_1_std": 0.17171988194258117, + "nauc_map_at_20_diff1": -0.03897001588097872, + "nauc_map_at_20_max": -0.009199473447509016, + "nauc_map_at_20_std": 0.03966215121731595, + "nauc_map_at_3_diff1": -0.046161914286411206, + "nauc_map_at_3_max": 0.054821084469595376, + "nauc_map_at_3_std": 0.07165158426226309, + "nauc_map_at_5_diff1": -0.014985087429412576, + "nauc_map_at_5_max": 0.04412649837924862, + "nauc_map_at_5_std": 0.05532724260923882, + "nauc_mrr_at_1000_diff1": -0.011257585036496497, + "nauc_mrr_at_1000_max": 0.029375661768316776, + "nauc_mrr_at_1000_std": 0.04410516975714857, + "nauc_mrr_at_100_diff1": -0.011517552807642387, + "nauc_mrr_at_100_max": 0.025626619594148595, + "nauc_mrr_at_100_std": 0.04467982209656915, + "nauc_mrr_at_10_diff1": -0.03690924911344003, + "nauc_mrr_at_10_max": 0.0012078815651148187, + "nauc_mrr_at_10_std": 0.029082751539292167, + "nauc_mrr_at_1_diff1": 0.22806546820499068, + "nauc_mrr_at_1_max": 0.22806546820499068, + "nauc_mrr_at_1_std": 0.17171988194258117, + "nauc_mrr_at_20_diff1": -0.03897001588097872, + "nauc_mrr_at_20_max": -0.009199473447509016, + "nauc_mrr_at_20_std": 0.03966215121731595, + "nauc_mrr_at_3_diff1": -0.046161914286411206, + "nauc_mrr_at_3_max": 0.054821084469595376, + "nauc_mrr_at_3_std": 0.07165158426226309, + "nauc_mrr_at_5_diff1": -0.014985087429412576, + "nauc_mrr_at_5_max": 0.04412649837924862, + "nauc_mrr_at_5_std": 0.05532724260923882, + "nauc_ndcg_at_1000_diff1": -0.005808457844089288, + "nauc_ndcg_at_1000_max": 0.04742256068385995, + "nauc_ndcg_at_1000_std": 0.039097539233417467, + "nauc_ndcg_at_100_diff1": 0.024104271564372164, + "nauc_ndcg_at_100_max": 0.0760880166616617, + "nauc_ndcg_at_100_std": 0.06196169194032705, + "nauc_ndcg_at_10_diff1": -0.057071756465906585, + "nauc_ndcg_at_10_max": -0.034206271856895745, + "nauc_ndcg_at_10_std": 0.0021729394642180655, + "nauc_ndcg_at_1_diff1": 0.22806546820499068, + "nauc_ndcg_at_1_max": 0.22806546820499068, + "nauc_ndcg_at_1_std": 0.17171988194258117, + "nauc_ndcg_at_20_diff1": -0.06339489290133342, + "nauc_ndcg_at_20_max": -0.05349972674229658, + "nauc_ndcg_at_20_std": 0.02098634745170093, + "nauc_ndcg_at_3_diff1": -0.09337817648407122, + "nauc_ndcg_at_3_max": 0.024243382226872005, + "nauc_ndcg_at_3_std": 0.047640069545597234, + "nauc_ndcg_at_5_diff1": -0.038933138628667374, + "nauc_ndcg_at_5_max": 0.014222495494588525, + "nauc_ndcg_at_5_std": 0.025555236284224053, + "nauc_precision_at_1000_diff1": -0.006836242757369853, + "nauc_precision_at_1000_max": 0.04906691781604409, + "nauc_precision_at_1000_std": 0.037085901708809176, + "nauc_precision_at_100_diff1": 0.04301099549182199, + "nauc_precision_at_100_max": 0.10517311457467209, + "nauc_precision_at_100_std": 0.07455761674906439, + "nauc_precision_at_10_diff1": -0.07094421543039747, + "nauc_precision_at_10_max": -0.06462668000097574, + "nauc_precision_at_10_std": -0.020635656267531922, + "nauc_precision_at_1_diff1": 0.22806546820499068, + "nauc_precision_at_1_max": 0.22806546820499068, + "nauc_precision_at_1_std": 0.17171988194258117, + "nauc_precision_at_20_diff1": -0.07966123850212924, + "nauc_precision_at_20_max": -0.0855159508231683, + "nauc_precision_at_20_std": 0.009191416757824122, + "nauc_precision_at_3_diff1": -0.17789106519989278, + "nauc_precision_at_3_max": -0.030632322690278217, + "nauc_precision_at_3_std": 0.0033539039441908805, + "nauc_precision_at_5_diff1": -0.06949288972363828, + "nauc_precision_at_5_max": -0.029007542557314442, + "nauc_precision_at_5_std": -0.020033986226634496, + "nauc_recall_at_1000_diff1": -0.006836242757369639, + "nauc_recall_at_1000_max": 0.049066917816044116, + "nauc_recall_at_1000_std": 0.037085901708809384, + "nauc_recall_at_100_diff1": 0.04301099549182174, + "nauc_recall_at_100_max": 0.10517311457467186, + "nauc_recall_at_100_std": 0.07455761674906419, + "nauc_recall_at_10_diff1": -0.07094421543039736, + "nauc_recall_at_10_max": -0.06462668000097577, + "nauc_recall_at_10_std": -0.020635656267531922, + "nauc_recall_at_1_diff1": 0.22806546820499068, + "nauc_recall_at_1_max": 0.22806546820499068, + "nauc_recall_at_1_std": 0.17171988194258117, + "nauc_recall_at_20_diff1": -0.07966123850212935, + "nauc_recall_at_20_max": -0.08551595082316851, + "nauc_recall_at_20_std": 0.00919141675782415, + "nauc_recall_at_3_diff1": -0.17789106519989273, + "nauc_recall_at_3_max": -0.03063232269027806, + "nauc_recall_at_3_std": 0.003353903944191056, + "nauc_recall_at_5_diff1": -0.06949288972363836, + "nauc_recall_at_5_max": -0.02900754255731451, + "nauc_recall_at_5_std": -0.020033986226634583, + "ndcg_at_1": 0.00028, + "ndcg_at_10": 0.00133, + "ndcg_at_100": 0.00585, + "ndcg_at_1000": 0.03475, + "ndcg_at_20": 0.0019, + "ndcg_at_3": 0.00059, + "ndcg_at_5": 0.00076, + "precision_at_1": 0.00028, + "precision_at_10": 0.00031, + "precision_at_100": 0.00028, + "precision_at_1000": 0.00028, + "precision_at_20": 0.00027, + "precision_at_3": 0.00028, + "precision_at_5": 0.00025, + "recall_at_1": 0.00028, + "recall_at_10": 0.00306, + "recall_at_100": 0.02806, + "recall_at_1000": 0.28208, + "recall_at_20": 0.00542, + "recall_at_3": 0.00083, + "recall_at_5": 0.00125 + }, + { + "cv_recall_at_1": 0.01264, + "cv_recall_at_10": 0.03583, + "cv_recall_at_100": 0.11236, + "cv_recall_at_1000": 0.44097, + "cv_recall_at_20": 0.04778, + "cv_recall_at_3": 0.02125, + "cv_recall_at_5": 0.02681, + "hf_subset": "th", + "languages": [ + "tha-Thai" + ], + "main_score": 0.0227, + "map_at_1": 0.01264, + "map_at_10": 0.0187, + "map_at_100": 0.02089, + "map_at_1000": 0.02184, + "map_at_20": 0.01949, + "map_at_3": 0.01627, + "map_at_5": 0.01753, + "mrr_at_1": 0.012638888888888889, + "mrr_at_10": 0.018695436507936505, + "mrr_at_100": 0.02088866848548745, + "mrr_at_1000": 0.021838592527878096, + "mrr_at_20": 0.019494748513082492, + "mrr_at_3": 0.016273148148148144, + "mrr_at_5": 0.017530092592592594, + "nauc_cv_recall_at_1000_diff1": 0.022292591165605398, + "nauc_cv_recall_at_1000_max": 0.13098130226865806, + "nauc_cv_recall_at_1000_std": 0.2720168619540829, + "nauc_cv_recall_at_100_diff1": 0.09501884772423592, + "nauc_cv_recall_at_100_max": 0.2599535722147032, + "nauc_cv_recall_at_100_std": 0.3801763208649803, + "nauc_cv_recall_at_10_diff1": 0.2632091816890365, + "nauc_cv_recall_at_10_max": 0.47505631438715595, + "nauc_cv_recall_at_10_std": 0.506678688722355, + "nauc_cv_recall_at_1_diff1": 0.6077745704791587, + "nauc_cv_recall_at_1_max": 0.6941858785164394, + "nauc_cv_recall_at_1_std": 0.5627334833130381, + "nauc_cv_recall_at_20_diff1": 0.19312636886079568, + "nauc_cv_recall_at_20_max": 0.40874885343283757, + "nauc_cv_recall_at_20_std": 0.4715815139054417, + "nauc_cv_recall_at_3_diff1": 0.3669705785900801, + "nauc_cv_recall_at_3_max": 0.5861063323460141, + "nauc_cv_recall_at_3_std": 0.5506084376331697, + "nauc_cv_recall_at_5_diff1": 0.33259049284662684, + "nauc_cv_recall_at_5_max": 0.5517516067458997, + "nauc_cv_recall_at_5_std": 0.5278314943049669, + "nauc_map_at_1000_diff1": 0.3787159713673133, + "nauc_map_at_1000_max": 0.5428421168181844, + "nauc_map_at_1000_std": 0.5131407062499223, + "nauc_map_at_100_diff1": 0.3880841736654061, + "nauc_map_at_100_max": 0.5539388799369834, + "nauc_map_at_100_std": 0.5200669870443412, + "nauc_map_at_10_diff1": 0.431272322825696, + "nauc_map_at_10_max": 0.5984353858644095, + "nauc_map_at_10_std": 0.5422788490060564, + "nauc_map_at_1_diff1": 0.6077745704791587, + "nauc_map_at_1_max": 0.6941858785164394, + "nauc_map_at_1_std": 0.5627334833130381, + "nauc_map_at_20_diff1": 0.41359649406159166, + "nauc_map_at_20_max": 0.5820965759060719, + "nauc_map_at_20_std": 0.5346552811096857, + "nauc_map_at_3_diff1": 0.47600932948256175, + "nauc_map_at_3_max": 0.6379062326699059, + "nauc_map_at_3_std": 0.5565259241985268, + "nauc_map_at_5_diff1": 0.45640314603896, + "nauc_map_at_5_max": 0.6221316525944712, + "nauc_map_at_5_std": 0.5488202814490963, + "nauc_mrr_at_1000_diff1": 0.37871597713247773, + "nauc_mrr_at_1000_max": 0.5428421179864462, + "nauc_mrr_at_1000_std": 0.5131406973059689, + "nauc_mrr_at_100_diff1": 0.3880841736654061, + "nauc_mrr_at_100_max": 0.5539388799369834, + "nauc_mrr_at_100_std": 0.5200669870443412, + "nauc_mrr_at_10_diff1": 0.431272322825696, + "nauc_mrr_at_10_max": 0.5984353858644095, + "nauc_mrr_at_10_std": 0.5422788490060564, + "nauc_mrr_at_1_diff1": 0.6077745704791587, + "nauc_mrr_at_1_max": 0.6941858785164394, + "nauc_mrr_at_1_std": 0.5627334833130381, + "nauc_mrr_at_20_diff1": 0.41359649406159166, + "nauc_mrr_at_20_max": 0.5820965759060719, + "nauc_mrr_at_20_std": 0.5346552811096857, + "nauc_mrr_at_3_diff1": 0.47600932948256175, + "nauc_mrr_at_3_max": 0.6379062326699059, + "nauc_mrr_at_3_std": 0.5565259241985268, + "nauc_mrr_at_5_diff1": 0.45640314603896, + "nauc_mrr_at_5_max": 0.6221316525944712, + "nauc_mrr_at_5_std": 0.5488202814490963, + "nauc_ndcg_at_1000_diff1": 0.15985113642093954, + "nauc_ndcg_at_1000_max": 0.3001669091845696, + "nauc_ndcg_at_1000_std": 0.3763836071659746, + "nauc_ndcg_at_100_diff1": 0.23393916616983274, + "nauc_ndcg_at_100_max": 0.40375379090698243, + "nauc_ndcg_at_100_std": 0.450200694745624, + "nauc_ndcg_at_10_diff1": 0.36912578430047643, + "nauc_ndcg_at_10_max": 0.5538603225353859, + "nauc_ndcg_at_10_std": 0.5296354727215236, + "nauc_ndcg_at_1_diff1": 0.6077745704791587, + "nauc_ndcg_at_1_max": 0.6941858785164394, + "nauc_ndcg_at_1_std": 0.5627334833130381, + "nauc_ndcg_at_20_diff1": 0.32493358843185, + "nauc_ndcg_at_20_max": 0.5134958869109029, + "nauc_ndcg_at_20_std": 0.5102637742103265, + "nauc_ndcg_at_3_diff1": 0.4423071127667642, + "nauc_ndcg_at_3_max": 0.6220036053763677, + "nauc_ndcg_at_3_std": 0.5547130370697566, + "nauc_ndcg_at_5_diff1": 0.41467932527755136, + "nauc_ndcg_at_5_max": 0.5987024010782739, + "nauc_ndcg_at_5_std": 0.5420809148948331, + "nauc_precision_at_1000_diff1": 0.022292591165604964, + "nauc_precision_at_1000_max": 0.13098130226865767, + "nauc_precision_at_1000_std": 0.2720168619540827, + "nauc_precision_at_100_diff1": 0.0950188477242359, + "nauc_precision_at_100_max": 0.2599535722147032, + "nauc_precision_at_100_std": 0.3801763208649802, + "nauc_precision_at_10_diff1": 0.2632091816890364, + "nauc_precision_at_10_max": 0.475056314387156, + "nauc_precision_at_10_std": 0.5066786887223551, + "nauc_precision_at_1_diff1": 0.6077745704791587, + "nauc_precision_at_1_max": 0.6941858785164394, + "nauc_precision_at_1_std": 0.5627334833130381, + "nauc_precision_at_20_diff1": 0.19312636886079576, + "nauc_precision_at_20_max": 0.4087488534328376, + "nauc_precision_at_20_std": 0.47158151390544195, + "nauc_precision_at_3_diff1": 0.3669705785900801, + "nauc_precision_at_3_max": 0.5861063323460142, + "nauc_precision_at_3_std": 0.5506084376331697, + "nauc_precision_at_5_diff1": 0.33259049284662695, + "nauc_precision_at_5_max": 0.5517516067459, + "nauc_precision_at_5_std": 0.5278314943049669, + "nauc_recall_at_1000_diff1": 0.022292591165605398, + "nauc_recall_at_1000_max": 0.13098130226865806, + "nauc_recall_at_1000_std": 0.2720168619540829, + "nauc_recall_at_100_diff1": 0.09501884772423592, + "nauc_recall_at_100_max": 0.2599535722147032, + "nauc_recall_at_100_std": 0.3801763208649803, + "nauc_recall_at_10_diff1": 0.2632091816890365, + "nauc_recall_at_10_max": 0.47505631438715595, + "nauc_recall_at_10_std": 0.506678688722355, + "nauc_recall_at_1_diff1": 0.6077745704791587, + "nauc_recall_at_1_max": 0.6941858785164394, + "nauc_recall_at_1_std": 0.5627334833130381, + "nauc_recall_at_20_diff1": 0.19312636886079568, + "nauc_recall_at_20_max": 0.40874885343283757, + "nauc_recall_at_20_std": 0.4715815139054417, + "nauc_recall_at_3_diff1": 0.3669705785900801, + "nauc_recall_at_3_max": 0.5861063323460141, + "nauc_recall_at_3_std": 0.5506084376331697, + "nauc_recall_at_5_diff1": 0.33259049284662684, + "nauc_recall_at_5_max": 0.5517516067458997, + "nauc_recall_at_5_std": 0.5278314943049669, + "ndcg_at_1": 0.01264, + "ndcg_at_10": 0.0227, + "ndcg_at_100": 0.03713, + "ndcg_at_1000": 0.07513, + "ndcg_at_20": 0.02568, + "ndcg_at_3": 0.01754, + "ndcg_at_5": 0.01982, + "precision_at_1": 0.01264, + "precision_at_10": 0.00358, + "precision_at_100": 0.00112, + "precision_at_1000": 0.00044, + "precision_at_20": 0.00239, + "precision_at_3": 0.00708, + "precision_at_5": 0.00536, + "recall_at_1": 0.01264, + "recall_at_10": 0.03583, + "recall_at_100": 0.11236, + "recall_at_1000": 0.44097, + "recall_at_20": 0.04778, + "recall_at_3": 0.02125, + "recall_at_5": 0.02681 + }, + { + "cv_recall_at_1": 0.01452, + "cv_recall_at_10": 0.04853, + "cv_recall_at_100": 0.13563, + "cv_recall_at_1000": 0.44546, + "cv_recall_at_20": 0.06622, + "cv_recall_at_3": 0.02696, + "cv_recall_at_5": 0.03567, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.02939, + "map_at_1": 0.01452, + "map_at_10": 0.02354, + "map_at_100": 0.02629, + "map_at_1000": 0.02721, + "map_at_20": 0.02477, + "map_at_3": 0.01993, + "map_at_5": 0.02188, + "mrr_at_1": 0.014516798009124844, + "mrr_at_10": 0.02354140743813079, + "mrr_at_100": 0.02629381239415381, + "mrr_at_1000": 0.027214393245717273, + "mrr_at_20": 0.024767747558012896, + "mrr_at_3": 0.01993179409189363, + "mrr_at_5": 0.021881192681690413, + "nauc_cv_recall_at_1000_diff1": 0.012975759729651388, + "nauc_cv_recall_at_1000_max": 0.17599164905484435, + "nauc_cv_recall_at_1000_std": 0.29982526861184217, + "nauc_cv_recall_at_100_diff1": 0.04097170057080594, + "nauc_cv_recall_at_100_max": 0.2975198043678887, + "nauc_cv_recall_at_100_std": 0.4599486909929725, + "nauc_cv_recall_at_10_diff1": 0.15247718510875016, + "nauc_cv_recall_at_10_max": 0.4477304911588335, + "nauc_cv_recall_at_10_std": 0.5574011165041363, + "nauc_cv_recall_at_1_diff1": 0.37003413985130795, + "nauc_cv_recall_at_1_max": 0.5920701629678448, + "nauc_cv_recall_at_1_std": 0.6075379204919302, + "nauc_cv_recall_at_20_diff1": 0.10275027990254477, + "nauc_cv_recall_at_20_max": 0.3951312064574037, + "nauc_cv_recall_at_20_std": 0.5439696855040402, + "nauc_cv_recall_at_3_diff1": 0.26504280766216726, + "nauc_cv_recall_at_3_max": 0.5667941191367358, + "nauc_cv_recall_at_3_std": 0.576336612293182, + "nauc_cv_recall_at_5_diff1": 0.22435904340469076, + "nauc_cv_recall_at_5_max": 0.5158004698221501, + "nauc_cv_recall_at_5_std": 0.5663764917385143, + "nauc_map_at_1000_diff1": 0.23186302755386246, + "nauc_map_at_1000_max": 0.5002654710152582, + "nauc_map_at_1000_std": 0.561532936580193, + "nauc_map_at_100_diff1": 0.23567961199945364, + "nauc_map_at_100_max": 0.5068296025584254, + "nauc_map_at_100_std": 0.5675416011527985, + "nauc_map_at_10_diff1": 0.26566349671711437, + "nauc_map_at_10_max": 0.5390175619472563, + "nauc_map_at_10_std": 0.5826401711776862, + "nauc_map_at_1_diff1": 0.37003413985130795, + "nauc_map_at_1_max": 0.5920701629678448, + "nauc_map_at_1_std": 0.6075379204919302, + "nauc_map_at_20_diff1": 0.2504929196056003, + "nauc_map_at_20_max": 0.5243658129963331, + "nauc_map_at_20_std": 0.5787047908123439, + "nauc_map_at_3_diff1": 0.30807275588101796, + "nauc_map_at_3_max": 0.5785385671839364, + "nauc_map_at_3_std": 0.5898595966081458, + "nauc_map_at_5_diff1": 0.2890780673741935, + "nauc_map_at_5_max": 0.5587200102194135, + "nauc_map_at_5_std": 0.5851700030151188, + "nauc_mrr_at_1000_diff1": 0.23186302528567013, + "nauc_mrr_at_1000_max": 0.5002654647368839, + "nauc_mrr_at_1000_std": 0.561532932436037, + "nauc_mrr_at_100_diff1": 0.23567961199945364, + "nauc_mrr_at_100_max": 0.5068296025584254, + "nauc_mrr_at_100_std": 0.5675416011527985, + "nauc_mrr_at_10_diff1": 0.26566349671711437, + "nauc_mrr_at_10_max": 0.5390175619472563, + "nauc_mrr_at_10_std": 0.5826401711776862, + "nauc_mrr_at_1_diff1": 0.37003413985130795, + "nauc_mrr_at_1_max": 0.5920701629678448, + "nauc_mrr_at_1_std": 0.6075379204919302, + "nauc_mrr_at_20_diff1": 0.2504929196056003, + "nauc_mrr_at_20_max": 0.5243658129963331, + "nauc_mrr_at_20_std": 0.5787047908123439, + "nauc_mrr_at_3_diff1": 0.30807275588101796, + "nauc_mrr_at_3_max": 0.5785385671839364, + "nauc_mrr_at_3_std": 0.5898595966081458, + "nauc_mrr_at_5_diff1": 0.2890780673741935, + "nauc_mrr_at_5_max": 0.5587200102194135, + "nauc_mrr_at_5_std": 0.5851700030151188, + "nauc_ndcg_at_1000_diff1": 0.1054369793354991, + "nauc_ndcg_at_1000_max": 0.3286790257113219, + "nauc_ndcg_at_1000_std": 0.4337873128983138, + "nauc_ndcg_at_100_diff1": 0.13702123082185066, + "nauc_ndcg_at_100_max": 0.404377005984407, + "nauc_ndcg_at_100_std": 0.5170717047904392, + "nauc_ndcg_at_10_diff1": 0.2230863223743629, + "nauc_ndcg_at_10_max": 0.5055905424621192, + "nauc_ndcg_at_10_std": 0.5731628679097766, + "nauc_ndcg_at_1_diff1": 0.37003413985130795, + "nauc_ndcg_at_1_max": 0.5920701629678448, + "nauc_ndcg_at_1_std": 0.6075379204919302, + "nauc_ndcg_at_20_diff1": 0.1888436713819155, + "nauc_ndcg_at_20_max": 0.47161431523098796, + "nauc_ndcg_at_20_std": 0.5642877406496789, + "nauc_ndcg_at_3_diff1": 0.2943800217921929, + "nauc_ndcg_at_3_max": 0.5748548662193177, + "nauc_ndcg_at_3_std": 0.585584818460011, + "nauc_ndcg_at_5_diff1": 0.26661831436553673, + "nauc_ndcg_at_5_max": 0.5442655392840073, + "nauc_ndcg_at_5_std": 0.5786701870420777, + "nauc_precision_at_1000_diff1": 0.012975759729651133, + "nauc_precision_at_1000_max": 0.17599164905484405, + "nauc_precision_at_1000_std": 0.29982526861184183, + "nauc_precision_at_100_diff1": 0.040971700570806076, + "nauc_precision_at_100_max": 0.29751980436788894, + "nauc_precision_at_100_std": 0.4599486909929728, + "nauc_precision_at_10_diff1": 0.15247718510875025, + "nauc_precision_at_10_max": 0.4477304911588337, + "nauc_precision_at_10_std": 0.5574011165041366, + "nauc_precision_at_1_diff1": 0.37003413985130795, + "nauc_precision_at_1_max": 0.5920701629678448, + "nauc_precision_at_1_std": 0.6075379204919302, + "nauc_precision_at_20_diff1": 0.1027502799025449, + "nauc_precision_at_20_max": 0.3951312064574038, + "nauc_precision_at_20_std": 0.5439696855040405, + "nauc_precision_at_3_diff1": 0.26504280766216703, + "nauc_precision_at_3_max": 0.5667941191367358, + "nauc_precision_at_3_std": 0.5763366122931819, + "nauc_precision_at_5_diff1": 0.2243590434046909, + "nauc_precision_at_5_max": 0.5158004698221502, + "nauc_precision_at_5_std": 0.5663764917385142, + "nauc_recall_at_1000_diff1": 0.012975759729651388, + "nauc_recall_at_1000_max": 0.17599164905484435, + "nauc_recall_at_1000_std": 0.29982526861184217, + "nauc_recall_at_100_diff1": 0.04097170057080594, + "nauc_recall_at_100_max": 0.2975198043678887, + "nauc_recall_at_100_std": 0.4599486909929725, + "nauc_recall_at_10_diff1": 0.15247718510875016, + "nauc_recall_at_10_max": 0.4477304911588335, + "nauc_recall_at_10_std": 0.5574011165041363, + "nauc_recall_at_1_diff1": 0.37003413985130795, + "nauc_recall_at_1_max": 0.5920701629678448, + "nauc_recall_at_1_std": 0.6075379204919302, + "nauc_recall_at_20_diff1": 0.10275027990254477, + "nauc_recall_at_20_max": 0.3951312064574037, + "nauc_recall_at_20_std": 0.5439696855040402, + "nauc_recall_at_3_diff1": 0.26504280766216726, + "nauc_recall_at_3_max": 0.5667941191367358, + "nauc_recall_at_3_std": 0.576336612293182, + "nauc_recall_at_5_diff1": 0.22435904340469076, + "nauc_recall_at_5_max": 0.5158004698221501, + "nauc_recall_at_5_std": 0.5663764917385143, + "ndcg_at_1": 0.01452, + "ndcg_at_10": 0.02939, + "ndcg_at_100": 0.04623, + "ndcg_at_1000": 0.08222, + "ndcg_at_20": 0.03386, + "ndcg_at_3": 0.02173, + "ndcg_at_5": 0.02529, + "precision_at_1": 0.01452, + "precision_at_10": 0.00485, + "precision_at_100": 0.00136, + "precision_at_1000": 0.00045, + "precision_at_20": 0.00331, + "precision_at_3": 0.00899, + "precision_at_5": 0.00713, + "recall_at_1": 0.01452, + "recall_at_10": 0.04853, + "recall_at_100": 0.13563, + "recall_at_1000": 0.44546, + "recall_at_20": 0.06622, + "recall_at_3": 0.02696, + "recall_at_5": 0.03567 + }, + { + "cv_recall_at_1": 0.00236, + "cv_recall_at_10": 0.01067, + "cv_recall_at_100": 0.05295, + "cv_recall_at_1000": 0.33638, + "cv_recall_at_20": 0.01733, + "cv_recall_at_3": 0.00513, + "cv_recall_at_5": 0.00693, + "hf_subset": "uk", + "languages": [ + "ukr-Cyrl" + ], + "main_score": 0.00586, + "map_at_1": 0.00236, + "map_at_10": 0.00441, + "map_at_100": 0.00555, + "map_at_1000": 0.00632, + "map_at_20": 0.00486, + "map_at_3": 0.00351, + "map_at_5": 0.00393, + "mrr_at_1": 0.002356202356202356, + "mrr_at_10": 0.004410014410014408, + "mrr_at_100": 0.005552806872394621, + "mrr_at_1000": 0.006320104674748412, + "mrr_at_20": 0.0048556029825379664, + "mrr_at_3": 0.00351120351120351, + "mrr_at_5": 0.003927003927003925, + "nauc_cv_recall_at_1000_diff1": -0.08406797065643612, + "nauc_cv_recall_at_1000_max": -0.001525952193718598, + "nauc_cv_recall_at_1000_std": 0.14984248217962143, + "nauc_cv_recall_at_100_diff1": -0.07298074982572675, + "nauc_cv_recall_at_100_max": 0.04679622553449316, + "nauc_cv_recall_at_100_std": 0.2517498038656578, + "nauc_cv_recall_at_10_diff1": 0.08726660901806763, + "nauc_cv_recall_at_10_max": 0.2364171653297631, + "nauc_cv_recall_at_10_std": 0.5057754862103674, + "nauc_cv_recall_at_1_diff1": 0.15584461979096606, + "nauc_cv_recall_at_1_max": 0.28341285856642234, + "nauc_cv_recall_at_1_std": 0.5876382323289251, + "nauc_cv_recall_at_20_diff1": -0.012332708752283775, + "nauc_cv_recall_at_20_max": 0.1682570311545627, + "nauc_cv_recall_at_20_std": 0.42095305867101396, + "nauc_cv_recall_at_3_diff1": 0.14922454775842056, + "nauc_cv_recall_at_3_max": 0.3453267670397321, + "nauc_cv_recall_at_3_std": 0.616668865370652, + "nauc_cv_recall_at_5_diff1": 0.11501170879346388, + "nauc_cv_recall_at_5_max": 0.33071772697417046, + "nauc_cv_recall_at_5_std": 0.5981901963197561, + "nauc_map_at_1000_diff1": 0.0569558703850738, + "nauc_map_at_1000_max": 0.22432832656300244, + "nauc_map_at_1000_std": 0.4825080785510095, + "nauc_map_at_100_diff1": 0.07037303142604615, + "nauc_map_at_100_max": 0.2451547614749494, + "nauc_map_at_100_std": 0.5161914580691662, + "nauc_map_at_10_diff1": 0.12231154578051895, + "nauc_map_at_10_max": 0.300438045334167, + "nauc_map_at_10_std": 0.5873229977719833, + "nauc_map_at_1_diff1": 0.15584461979096606, + "nauc_map_at_1_max": 0.28341285856642234, + "nauc_map_at_1_std": 0.5876382323289251, + "nauc_map_at_20_diff1": 0.09475071247578104, + "nauc_map_at_20_max": 0.2790261663013074, + "nauc_map_at_20_std": 0.5613443798754721, + "nauc_map_at_3_diff1": 0.14692091393411147, + "nauc_map_at_3_max": 0.33111931125429245, + "nauc_map_at_3_std": 0.6209682999618753, + "nauc_map_at_5_diff1": 0.13374610850404176, + "nauc_map_at_5_max": 0.32939615249119725, + "nauc_map_at_5_std": 0.6172151637277775, + "nauc_mrr_at_1000_diff1": 0.05695589575990401, + "nauc_mrr_at_1000_max": 0.22432832213015588, + "nauc_mrr_at_1000_std": 0.48250801343703764, + "nauc_mrr_at_100_diff1": 0.07037303142604615, + "nauc_mrr_at_100_max": 0.2451547614749494, + "nauc_mrr_at_100_std": 0.5161914580691662, + "nauc_mrr_at_10_diff1": 0.12231154578051895, + "nauc_mrr_at_10_max": 0.300438045334167, + "nauc_mrr_at_10_std": 0.5873229977719833, + "nauc_mrr_at_1_diff1": 0.15584461979096606, + "nauc_mrr_at_1_max": 0.28341285856642234, + "nauc_mrr_at_1_std": 0.5876382323289251, + "nauc_mrr_at_20_diff1": 0.09475071247578104, + "nauc_mrr_at_20_max": 0.2790261663013074, + "nauc_mrr_at_20_std": 0.5613443798754721, + "nauc_mrr_at_3_diff1": 0.14692091393411147, + "nauc_mrr_at_3_max": 0.33111931125429245, + "nauc_mrr_at_3_std": 0.6209682999618753, + "nauc_mrr_at_5_diff1": 0.13374610850404176, + "nauc_mrr_at_5_max": 0.32939615249119725, + "nauc_mrr_at_5_std": 0.6172151637277775, + "nauc_ndcg_at_1000_diff1": -0.058503630663267324, + "nauc_ndcg_at_1000_max": 0.04506885875721012, + "nauc_ndcg_at_1000_std": 0.2191973371313843, + "nauc_ndcg_at_100_diff1": -0.02062968232376607, + "nauc_ndcg_at_100_max": 0.12258667087293763, + "nauc_ndcg_at_100_std": 0.35313324716473754, + "nauc_ndcg_at_10_diff1": 0.10804545321905491, + "nauc_ndcg_at_10_max": 0.2762775018336161, + "nauc_ndcg_at_10_std": 0.5558309811414158, + "nauc_ndcg_at_1_diff1": 0.15584461979096606, + "nauc_ndcg_at_1_max": 0.28341285856642234, + "nauc_ndcg_at_1_std": 0.5876382323289251, + "nauc_ndcg_at_20_diff1": 0.0457669467001654, + "nauc_ndcg_at_20_max": 0.2290101438279385, + "nauc_ndcg_at_20_std": 0.4974691435985739, + "nauc_ndcg_at_3_diff1": 0.14751080948533413, + "nauc_ndcg_at_3_max": 0.3362208528847951, + "nauc_ndcg_at_3_std": 0.6201048482107948, + "nauc_ndcg_at_5_diff1": 0.12715063122699638, + "nauc_ndcg_at_5_max": 0.3307088007409043, + "nauc_ndcg_at_5_std": 0.6113230989942977, + "nauc_precision_at_1000_diff1": -0.08406797065643624, + "nauc_precision_at_1000_max": -0.0015259521937187298, + "nauc_precision_at_1000_std": 0.14984248217962123, + "nauc_precision_at_100_diff1": -0.07298074982572668, + "nauc_precision_at_100_max": 0.04679622553449324, + "nauc_precision_at_100_std": 0.2517498038656581, + "nauc_precision_at_10_diff1": 0.08726660901806796, + "nauc_precision_at_10_max": 0.2364171653297635, + "nauc_precision_at_10_std": 0.5057754862103677, + "nauc_precision_at_1_diff1": 0.15584461979096606, + "nauc_precision_at_1_max": 0.28341285856642234, + "nauc_precision_at_1_std": 0.5876382323289251, + "nauc_precision_at_20_diff1": -0.012332708752283828, + "nauc_precision_at_20_max": 0.16825703115456306, + "nauc_precision_at_20_std": 0.4209530586710141, + "nauc_precision_at_3_diff1": 0.14922454775842028, + "nauc_precision_at_3_max": 0.3453267670397317, + "nauc_precision_at_3_std": 0.6166688653706518, + "nauc_precision_at_5_diff1": 0.11501170879346426, + "nauc_precision_at_5_max": 0.33071772697417073, + "nauc_precision_at_5_std": 0.5981901963197568, + "nauc_recall_at_1000_diff1": -0.08406797065643612, + "nauc_recall_at_1000_max": -0.001525952193718598, + "nauc_recall_at_1000_std": 0.14984248217962143, + "nauc_recall_at_100_diff1": -0.07298074982572675, + "nauc_recall_at_100_max": 0.04679622553449316, + "nauc_recall_at_100_std": 0.2517498038656578, + "nauc_recall_at_10_diff1": 0.08726660901806763, + "nauc_recall_at_10_max": 0.2364171653297631, + "nauc_recall_at_10_std": 0.5057754862103674, + "nauc_recall_at_1_diff1": 0.15584461979096606, + "nauc_recall_at_1_max": 0.28341285856642234, + "nauc_recall_at_1_std": 0.5876382323289251, + "nauc_recall_at_20_diff1": -0.012332708752283775, + "nauc_recall_at_20_max": 0.1682570311545627, + "nauc_recall_at_20_std": 0.42095305867101396, + "nauc_recall_at_3_diff1": 0.14922454775842056, + "nauc_recall_at_3_max": 0.3453267670397321, + "nauc_recall_at_3_std": 0.616668865370652, + "nauc_recall_at_5_diff1": 0.11501170879346388, + "nauc_recall_at_5_max": 0.33071772697417046, + "nauc_recall_at_5_std": 0.5981901963197561, + "ndcg_at_1": 0.00236, + "ndcg_at_10": 0.00586, + "ndcg_at_100": 0.01369, + "ndcg_at_1000": 0.04614, + "ndcg_at_20": 0.00752, + "ndcg_at_3": 0.00392, + "ndcg_at_5": 0.00467, + "precision_at_1": 0.00236, + "precision_at_10": 0.00107, + "precision_at_100": 0.00053, + "precision_at_1000": 0.00034, + "precision_at_20": 0.00087, + "precision_at_3": 0.00171, + "precision_at_5": 0.00139, + "recall_at_1": 0.00236, + "recall_at_10": 0.01067, + "recall_at_100": 0.05295, + "recall_at_1000": 0.33638, + "recall_at_20": 0.01733, + "recall_at_3": 0.00513, + "recall_at_5": 0.00693 + }, + { + "cv_recall_at_1": 0.00857, + "cv_recall_at_10": 0.03007, + "cv_recall_at_100": 0.07469, + "cv_recall_at_1000": 0.34544, + "cv_recall_at_20": 0.03905, + "cv_recall_at_3": 0.0166, + "cv_recall_at_5": 0.02095, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.01787, + "map_at_1": 0.00857, + "map_at_10": 0.01415, + "map_at_100": 0.01555, + "map_at_1000": 0.01628, + "map_at_20": 0.01476, + "map_at_3": 0.01195, + "map_at_5": 0.01294, + "mrr_at_1": 0.008571428571428572, + "mrr_at_10": 0.014153925062088327, + "mrr_at_100": 0.015554419065062603, + "mrr_at_1000": 0.016277461294299248, + "mrr_at_20": 0.014763666787850148, + "mrr_at_3": 0.011950113378684804, + "mrr_at_5": 0.012936507936507937, + "nauc_cv_recall_at_1000_diff1": 0.05577697623272914, + "nauc_cv_recall_at_1000_max": 0.1081174324422234, + "nauc_cv_recall_at_1000_std": 0.2069110458185291, + "nauc_cv_recall_at_100_diff1": 0.18158341409582918, + "nauc_cv_recall_at_100_max": 0.3224112137931103, + "nauc_cv_recall_at_100_std": 0.5094058372834869, + "nauc_cv_recall_at_10_diff1": 0.3869245702450141, + "nauc_cv_recall_at_10_max": 0.5598828167208352, + "nauc_cv_recall_at_10_std": 0.8236957411186805, + "nauc_cv_recall_at_1_diff1": 0.609533179160225, + "nauc_cv_recall_at_1_max": 0.6525099978279478, + "nauc_cv_recall_at_1_std": 0.9003666934978981, + "nauc_cv_recall_at_20_diff1": 0.3237865879367905, + "nauc_cv_recall_at_20_max": 0.5040690918235797, + "nauc_cv_recall_at_20_std": 0.7479556377839834, + "nauc_cv_recall_at_3_diff1": 0.42102380941908174, + "nauc_cv_recall_at_3_max": 0.5894469687306189, + "nauc_cv_recall_at_3_std": 0.8605149837033257, + "nauc_cv_recall_at_5_diff1": 0.43539074287665624, + "nauc_cv_recall_at_5_max": 0.5926827398520448, + "nauc_cv_recall_at_5_std": 0.8704922659846193, + "nauc_map_at_1000_diff1": 0.4336332621953282, + "nauc_map_at_1000_max": 0.5629566075355275, + "nauc_map_at_1000_std": 0.8006440442716728, + "nauc_map_at_100_diff1": 0.4465754563450948, + "nauc_map_at_100_max": 0.5796866198941129, + "nauc_map_at_100_std": 0.8225647482382251, + "nauc_map_at_10_diff1": 0.4852007113910197, + "nauc_map_at_10_max": 0.6142575371011583, + "nauc_map_at_10_std": 0.8658948991412293, + "nauc_map_at_1_diff1": 0.609533179160225, + "nauc_map_at_1_max": 0.6525099978279478, + "nauc_map_at_1_std": 0.9003666934978981, + "nauc_map_at_20_diff1": 0.46978800543511556, + "nauc_map_at_20_max": 0.6036917861122867, + "nauc_map_at_20_std": 0.8516885712299876, + "nauc_map_at_3_diff1": 0.5060955772253246, + "nauc_map_at_3_max": 0.6266019187130784, + "nauc_map_at_3_std": 0.8757357587001666, + "nauc_map_at_5_diff1": 0.5043082586342598, + "nauc_map_at_5_max": 0.6267534421460095, + "nauc_map_at_5_std": 0.8788762487633768, + "nauc_mrr_at_1000_diff1": 0.4336332621953282, + "nauc_mrr_at_1000_max": 0.5629566075355275, + "nauc_mrr_at_1000_std": 0.8006440442716728, + "nauc_mrr_at_100_diff1": 0.4465754563450948, + "nauc_mrr_at_100_max": 0.5796866198941129, + "nauc_mrr_at_100_std": 0.8225647482382251, + "nauc_mrr_at_10_diff1": 0.4852007113910197, + "nauc_mrr_at_10_max": 0.6142575371011583, + "nauc_mrr_at_10_std": 0.8658948991412293, + "nauc_mrr_at_1_diff1": 0.609533179160225, + "nauc_mrr_at_1_max": 0.6525099978279478, + "nauc_mrr_at_1_std": 0.9003666934978981, + "nauc_mrr_at_20_diff1": 0.46978800543511556, + "nauc_mrr_at_20_max": 0.6036917861122867, + "nauc_mrr_at_20_std": 0.8516885712299876, + "nauc_mrr_at_3_diff1": 0.5060955772253246, + "nauc_mrr_at_3_max": 0.6266019187130784, + "nauc_mrr_at_3_std": 0.8757357587001666, + "nauc_mrr_at_5_diff1": 0.5043082586342598, + "nauc_mrr_at_5_max": 0.6267534421460095, + "nauc_mrr_at_5_std": 0.8788762487633768, + "nauc_ndcg_at_1000_diff1": 0.19463394578176046, + "nauc_ndcg_at_1000_max": 0.2847639002989628, + "nauc_ndcg_at_1000_std": 0.441408571736638, + "nauc_ndcg_at_100_diff1": 0.3161729621111146, + "nauc_ndcg_at_100_max": 0.4586921287973759, + "nauc_ndcg_at_100_std": 0.6779271906930151, + "nauc_ndcg_at_10_diff1": 0.44654032280824246, + "nauc_ndcg_at_10_max": 0.5934056697433198, + "nauc_ndcg_at_10_std": 0.8502453582865719, + "nauc_ndcg_at_1_diff1": 0.609533179160225, + "nauc_ndcg_at_1_max": 0.6525099978279478, + "nauc_ndcg_at_1_std": 0.9003666934978981, + "nauc_ndcg_at_20_diff1": 0.40910476157401005, + "nauc_ndcg_at_20_max": 0.56404265530875, + "nauc_ndcg_at_20_std": 0.8114472859077251, + "nauc_ndcg_at_3_diff1": 0.4788979735449278, + "nauc_ndcg_at_3_max": 0.6150215956603844, + "nauc_ndcg_at_3_std": 0.8707748731872657, + "nauc_ndcg_at_5_diff1": 0.4798110373153733, + "nauc_ndcg_at_5_max": 0.6151902535439208, + "nauc_ndcg_at_5_std": 0.8757788094218073, + "nauc_precision_at_1000_diff1": 0.05577697623272853, + "nauc_precision_at_1000_max": 0.10811743244222277, + "nauc_precision_at_1000_std": 0.2069110458185286, + "nauc_precision_at_100_diff1": 0.18158341409582915, + "nauc_precision_at_100_max": 0.3224112137931104, + "nauc_precision_at_100_std": 0.5094058372834871, + "nauc_precision_at_10_diff1": 0.38692457024501414, + "nauc_precision_at_10_max": 0.5598828167208352, + "nauc_precision_at_10_std": 0.8236957411186803, + "nauc_precision_at_1_diff1": 0.609533179160225, + "nauc_precision_at_1_max": 0.6525099978279478, + "nauc_precision_at_1_std": 0.9003666934978981, + "nauc_precision_at_20_diff1": 0.32378658793679044, + "nauc_precision_at_20_max": 0.5040690918235794, + "nauc_precision_at_20_std": 0.7479556377839834, + "nauc_precision_at_3_diff1": 0.42102380941908174, + "nauc_precision_at_3_max": 0.5894469687306185, + "nauc_precision_at_3_std": 0.8605149837033252, + "nauc_precision_at_5_diff1": 0.4353907428766564, + "nauc_precision_at_5_max": 0.5926827398520451, + "nauc_precision_at_5_std": 0.8704922659846194, + "nauc_recall_at_1000_diff1": 0.05577697623272914, + "nauc_recall_at_1000_max": 0.1081174324422234, + "nauc_recall_at_1000_std": 0.2069110458185291, + "nauc_recall_at_100_diff1": 0.18158341409582918, + "nauc_recall_at_100_max": 0.3224112137931103, + "nauc_recall_at_100_std": 0.5094058372834869, + "nauc_recall_at_10_diff1": 0.3869245702450141, + "nauc_recall_at_10_max": 0.5598828167208352, + "nauc_recall_at_10_std": 0.8236957411186805, + "nauc_recall_at_1_diff1": 0.609533179160225, + "nauc_recall_at_1_max": 0.6525099978279478, + "nauc_recall_at_1_std": 0.9003666934978981, + "nauc_recall_at_20_diff1": 0.3237865879367905, + "nauc_recall_at_20_max": 0.5040690918235797, + "nauc_recall_at_20_std": 0.7479556377839834, + "nauc_recall_at_3_diff1": 0.42102380941908174, + "nauc_recall_at_3_max": 0.5894469687306189, + "nauc_recall_at_3_std": 0.8605149837033257, + "nauc_recall_at_5_diff1": 0.43539074287665624, + "nauc_recall_at_5_max": 0.5926827398520448, + "nauc_recall_at_5_std": 0.8704922659846193, + "ndcg_at_1": 0.00857, + "ndcg_at_10": 0.01787, + "ndcg_at_100": 0.02649, + "ndcg_at_1000": 0.0574, + "ndcg_at_20": 0.02013, + "ndcg_at_3": 0.01314, + "ndcg_at_5": 0.01492, + "precision_at_1": 0.00857, + "precision_at_10": 0.00301, + "precision_at_100": 0.00075, + "precision_at_1000": 0.00035, + "precision_at_20": 0.00195, + "precision_at_3": 0.00553, + "precision_at_5": 0.00419, + "recall_at_1": 0.00857, + "recall_at_10": 0.03007, + "recall_at_100": 0.07469, + "recall_at_1000": 0.34544, + "recall_at_20": 0.03905, + "recall_at_3": 0.0166, + "recall_at_5": 0.02095 + }, + { + "cv_recall_at_1": 0.00362, + "cv_recall_at_10": 0.01979, + "cv_recall_at_100": 0.11221, + "cv_recall_at_1000": 0.49582, + "cv_recall_at_20": 0.03415, + "cv_recall_at_3": 0.0085, + "cv_recall_at_5": 0.01213, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ], + "main_score": 0.0104, + "map_at_1": 0.00362, + "map_at_10": 0.00757, + "map_at_100": 0.01013, + "map_at_1000": 0.0113, + "map_at_20": 0.00851, + "map_at_3": 0.00574, + "map_at_5": 0.00656, + "mrr_at_1": 0.0036241984945637024, + "mrr_at_10": 0.007570050136958424, + "mrr_at_100": 0.010130797363797312, + "mrr_at_1000": 0.011298712371181118, + "mrr_at_20": 0.008506887845520186, + "mrr_at_3": 0.005738314283059196, + "mrr_at_5": 0.006560728556825576, + "nauc_cv_recall_at_1000_diff1": 0.06310131748808638, + "nauc_cv_recall_at_1000_max": 0.19115897801802068, + "nauc_cv_recall_at_1000_std": 0.3494729582464983, + "nauc_cv_recall_at_100_diff1": 0.06462051707840712, + "nauc_cv_recall_at_100_max": 0.20466155072653078, + "nauc_cv_recall_at_100_std": 0.36358143329435333, + "nauc_cv_recall_at_10_diff1": 0.11464586708775552, + "nauc_cv_recall_at_10_max": 0.3011761761750302, + "nauc_cv_recall_at_10_std": 0.44072601631113906, + "nauc_cv_recall_at_1_diff1": 0.3314490401058548, + "nauc_cv_recall_at_1_max": 0.576029066981452, + "nauc_cv_recall_at_1_std": 0.5506135812699169, + "nauc_cv_recall_at_20_diff1": 0.09732639529181819, + "nauc_cv_recall_at_20_max": 0.2762623071349517, + "nauc_cv_recall_at_20_std": 0.4160697700973022, + "nauc_cv_recall_at_3_diff1": 0.20311571079691665, + "nauc_cv_recall_at_3_max": 0.4890582156975279, + "nauc_cv_recall_at_3_std": 0.5122791391441451, + "nauc_cv_recall_at_5_diff1": 0.16597217434441056, + "nauc_cv_recall_at_5_max": 0.38658520019136333, + "nauc_cv_recall_at_5_std": 0.48447201176385574, + "nauc_map_at_1000_diff1": 0.16156489121310788, + "nauc_map_at_1000_max": 0.36931173274986095, + "nauc_map_at_1000_std": 0.4596945612412111, + "nauc_map_at_100_diff1": 0.16719444274697842, + "nauc_map_at_100_max": 0.37904946654327765, + "nauc_map_at_100_std": 0.4657260813330009, + "nauc_map_at_10_diff1": 0.20219953418305472, + "nauc_map_at_10_max": 0.43623717234748305, + "nauc_map_at_10_std": 0.4999483552034321, + "nauc_map_at_1_diff1": 0.3314490401058548, + "nauc_map_at_1_max": 0.576029066981452, + "nauc_map_at_1_std": 0.5506135812699169, + "nauc_map_at_20_diff1": 0.18651020724728606, + "nauc_map_at_20_max": 0.41297186154186527, + "nauc_map_at_20_std": 0.4871155315109689, + "nauc_map_at_3_diff1": 0.24822480464649008, + "nauc_map_at_3_max": 0.5220554202941391, + "nauc_map_at_3_std": 0.5308037116682419, + "nauc_map_at_5_diff1": 0.22892372186142276, + "nauc_map_at_5_max": 0.4760368182138216, + "nauc_map_at_5_std": 0.5172233572837502, + "nauc_mrr_at_1000_diff1": 0.16156489121310788, + "nauc_mrr_at_1000_max": 0.36931173274986095, + "nauc_mrr_at_1000_std": 0.4596945612412111, + "nauc_mrr_at_100_diff1": 0.16719444274697842, + "nauc_mrr_at_100_max": 0.37904946654327765, + "nauc_mrr_at_100_std": 0.4657260813330009, + "nauc_mrr_at_10_diff1": 0.20219953418305472, + "nauc_mrr_at_10_max": 0.43623717234748305, + "nauc_mrr_at_10_std": 0.4999483552034321, + "nauc_mrr_at_1_diff1": 0.3314490401058548, + "nauc_mrr_at_1_max": 0.576029066981452, + "nauc_mrr_at_1_std": 0.5506135812699169, + "nauc_mrr_at_20_diff1": 0.18651020724728606, + "nauc_mrr_at_20_max": 0.41297186154186527, + "nauc_mrr_at_20_std": 0.4871155315109689, + "nauc_mrr_at_3_diff1": 0.24822480464649008, + "nauc_mrr_at_3_max": 0.5220554202941391, + "nauc_mrr_at_3_std": 0.5308037116682419, + "nauc_mrr_at_5_diff1": 0.22892372186142276, + "nauc_mrr_at_5_max": 0.4760368182138216, + "nauc_mrr_at_5_std": 0.5172233572837502, + "nauc_ndcg_at_1000_diff1": 0.08511107747492294, + "nauc_ndcg_at_1000_max": 0.23496356891655748, + "nauc_ndcg_at_1000_std": 0.37818549814470126, + "nauc_ndcg_at_100_diff1": 0.09804413340784086, + "nauc_ndcg_at_100_max": 0.26285200146189763, + "nauc_ndcg_at_100_std": 0.3985480067269638, + "nauc_ndcg_at_10_diff1": 0.16419473245341326, + "nauc_ndcg_at_10_max": 0.3787350408792725, + "nauc_ndcg_at_10_std": 0.47506805045248357, + "nauc_ndcg_at_1_diff1": 0.3314490401058548, + "nauc_ndcg_at_1_max": 0.576029066981452, + "nauc_ndcg_at_1_std": 0.5506135812699169, + "nauc_ndcg_at_20_diff1": 0.1399569918152696, + "nauc_ndcg_at_20_max": 0.34241103145134444, + "nauc_ndcg_at_20_std": 0.45143958624730546, + "nauc_ndcg_at_3_diff1": 0.23294671622092797, + "nauc_ndcg_at_3_max": 0.5109715994212655, + "nauc_ndcg_at_3_std": 0.524721159753844, + "nauc_ndcg_at_5_diff1": 0.20545348119843998, + "nauc_ndcg_at_5_max": 0.4433749890215097, + "nauc_ndcg_at_5_std": 0.5052378279790504, + "nauc_precision_at_1000_diff1": 0.0631013174880857, + "nauc_precision_at_1000_max": 0.19115897801802026, + "nauc_precision_at_1000_std": 0.34947295824649804, + "nauc_precision_at_100_diff1": 0.06462051707840721, + "nauc_precision_at_100_max": 0.20466155072653078, + "nauc_precision_at_100_std": 0.3635814332943533, + "nauc_precision_at_10_diff1": 0.11464586708775586, + "nauc_precision_at_10_max": 0.30117617617503045, + "nauc_precision_at_10_std": 0.44072601631113917, + "nauc_precision_at_1_diff1": 0.3314490401058548, + "nauc_precision_at_1_max": 0.576029066981452, + "nauc_precision_at_1_std": 0.5506135812699169, + "nauc_precision_at_20_diff1": 0.09732639529181827, + "nauc_precision_at_20_max": 0.2762623071349518, + "nauc_precision_at_20_std": 0.4160697700973024, + "nauc_precision_at_3_diff1": 0.20311571079691687, + "nauc_precision_at_3_max": 0.4890582156975278, + "nauc_precision_at_3_std": 0.5122791391441456, + "nauc_precision_at_5_diff1": 0.16597217434441086, + "nauc_precision_at_5_max": 0.386585200191364, + "nauc_precision_at_5_std": 0.48447201176385596, + "nauc_recall_at_1000_diff1": 0.06310131748808638, + "nauc_recall_at_1000_max": 0.19115897801802068, + "nauc_recall_at_1000_std": 0.3494729582464983, + "nauc_recall_at_100_diff1": 0.06462051707840712, + "nauc_recall_at_100_max": 0.20466155072653078, + "nauc_recall_at_100_std": 0.36358143329435333, + "nauc_recall_at_10_diff1": 0.11464586708775552, + "nauc_recall_at_10_max": 0.3011761761750302, + "nauc_recall_at_10_std": 0.44072601631113906, + "nauc_recall_at_1_diff1": 0.3314490401058548, + "nauc_recall_at_1_max": 0.576029066981452, + "nauc_recall_at_1_std": 0.5506135812699169, + "nauc_recall_at_20_diff1": 0.09732639529181819, + "nauc_recall_at_20_max": 0.2762623071349517, + "nauc_recall_at_20_std": 0.4160697700973022, + "nauc_recall_at_3_diff1": 0.20311571079691665, + "nauc_recall_at_3_max": 0.4890582156975279, + "nauc_recall_at_3_std": 0.5122791391441451, + "nauc_recall_at_5_diff1": 0.16597217434441056, + "nauc_recall_at_5_max": 0.38658520019136333, + "nauc_recall_at_5_std": 0.48447201176385574, + "ndcg_at_1": 0.00362, + "ndcg_at_10": 0.0104, + "ndcg_at_100": 0.02767, + "ndcg_at_1000": 0.07249, + "ndcg_at_20": 0.01395, + "ndcg_at_3": 0.00645, + "ndcg_at_5": 0.00793, + "precision_at_1": 0.00362, + "precision_at_10": 0.00198, + "precision_at_100": 0.00112, + "precision_at_1000": 0.0005, + "precision_at_20": 0.00171, + "precision_at_3": 0.00283, + "precision_at_5": 0.00243, + "recall_at_1": 0.00362, + "recall_at_10": 0.01979, + "recall_at_100": 0.11221, + "recall_at_1000": 0.49582, + "recall_at_20": 0.03415, + "recall_at_3": 0.0085, + "recall_at_5": 0.01213 + } + ] + }, + "task_name": "XM3600T2IRetrieval" +} \ No newline at end of file From 17f1dd189392f3442c3670f5253b14a7ffc0c60b Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 19 Aug 2024 20:03:12 +0100 Subject: [PATCH 051/154] wit multilingual retrieval t2i --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 1 + .../multilingual/WITT2IRetrieval.py | 224 ++ .../WITT2IRetrieval.json | 1936 +++++++++++++++++ 3 files changed, 2161 insertions(+) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WITT2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 8f62573137..dc7c5f8476 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -18,5 +18,6 @@ from .eng.VisualNewsT2IRetrieval import * from .eng.WebQAT2ITRetrieval import * from .eng.WebQAT2TRetrieval import * +from .multilingual.WITT2IRetrieval import * from .multilingual.XFlickr30kCoT2IRetrieval import * from .multilingual.XM3600T2IRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py new file mode 100644 index 0000000000..17321d4e89 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +from datasets import Dataset, DatasetDict, load_dataset + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval, MultilingualTask + +_LANGUAGES = { + "ar": ["ara-Arab"], + "bg": ["bul-Cyrl"], + "da": ["dan-Latn"], + "el": ["ell-Grek"], + "et": ["est-Latn"], + "id": ["ind-Latn"], + "ko": ["kor-Hang"], + "ja": ["jpn-Jpan"], + "tr": ["tur-Latn"], + "vi": ["vie-Latn"], + "en": ["eng-Latn"], +} + + +def _load_wit_data( + path: str, langs: list, splits: str, cache_dir: str = None, revision: str = None +): + corpus = {lang: {split: None for split in splits} for lang in langs} + queries = {lang: {split: None for split in splits} for lang in langs} + relevant_docs = {lang: {split: None for split in splits} for lang in langs} + + split = "test" + + for lang in langs: + lang_data = load_dataset( + path, + split=lang, + cache_dir=cache_dir, + revision=revision, + # trust_remote_code=True, + ) + lang_corpus = lang_data.map( + lambda x: { + "id": "corpus-" + x["image_id"], + "text": None, + "modality": "image", + "image": x["image"], + }, + remove_columns=[ + "captions", + "image_id", + ], + ) + + corpus[lang][split] = lang_corpus + + lang_data = lang_data.remove_columns(["image"]) + + queries[lang][split] = [] + relevant_docs[lang][split] = {} + + for row in lang_data: + image_id = "corpus-" + row["image_id"] + for idx, caption in enumerate(row["captions"]): + query_id = f"query-{row['image_id']}-{idx}" + queries[lang][split].append( + { + "id": query_id, + "text": caption, + "modality": "text", + "image": None, + } + ) + if query_id not in relevant_docs[lang][split]: + relevant_docs[lang][split][query_id] = {} + relevant_docs[lang][split][query_id][image_id] = 1 + + queries[lang][split] = Dataset.from_dict( + { + "id": [query["id"] for query in queries[lang][split]], + "text": [query["text"] for query in queries[lang][split]], + "modality": [query["modality"] for query in queries[lang][split]], + "image": [None for _ in queries[lang][split]], + } + ) + corpus = DatasetDict({lang: DatasetDict(splits) for lang, splits in corpus.items()}) + queries = DatasetDict( + {lang: DatasetDict(splits) for lang, splits in queries.items()} + ) + relevant_docs = DatasetDict(relevant_docs) + + return corpus, queries, relevant_docs + + +class WITT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="WITT2IRetrieval", + description="Retrieve images based on multilingual descriptions.", + reference="https://aclanthology.org/2022.emnlp-main.45/", + dataset={ + "path": "mteb/wit", + "revision": "91ac153f1371a98b209ed763205e25e115ecd06e", + # "trust_remote_code": True, + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=_LANGUAGES, + main_score="ndcg_at_10", + date=("2022-01-01", "2022-12-31"), + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@inproceedings{thapliyal2022crossmodal, + title={Crossmodal-3600: A Massively Multilingual Multimodal Evaluation Dataset}, + author={Thapliyal, Ashish V and Tuset, Jordi Pont and Chen, Xi and Soricut, Radu}, + booktitle={Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing}, + pages={715--729}, + year={2022} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "dev": { + "ar": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "bg": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "da": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "el": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "et": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "id": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "ja": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "ko": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "tr": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "vi": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + "en": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 7200, + "num_queries": 3600, + "average_relevant_docs_per_query": 2.0, + }, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus, self.queries, self.relevant_docs = _load_wit_data( + path=self.metadata_dict["dataset"]["path"], + langs=self.hf_subsets, + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WITT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WITT2IRetrieval.json new file mode 100644 index 0000000000..3211c5c5af --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WITT2IRetrieval.json @@ -0,0 +1,1936 @@ +{ + "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", + "evaluation_time": 50.179383754730225, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.03933, + "cv_recall_at_10": 0.09888, + "cv_recall_at_100": 0.29775, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.12921, + "cv_recall_at_3": 0.06292, + "cv_recall_at_5": 0.07191, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.06583, + "map_at_1": 0.03933, + "map_at_10": 0.05581, + "map_at_100": 0.06157, + "map_at_1000": 0.06398, + "map_at_20": 0.05799, + "map_at_3": 0.05037, + "map_at_5": 0.05245, + "mrr_at_1": 0.03932584269662921, + "mrr_at_10": 0.05581148564294632, + "mrr_at_100": 0.06157040381700697, + "mrr_at_1000": 0.06397873885256421, + "mrr_at_20": 0.057929903295724096, + "mrr_at_3": 0.050374531835205995, + "mrr_at_5": 0.05245318352059926, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.10651632318768792, + "nauc_cv_recall_at_100_max": 0.28469315454649335, + "nauc_cv_recall_at_100_std": 0.3307774941906975, + "nauc_cv_recall_at_10_diff1": 0.24737175890918847, + "nauc_cv_recall_at_10_max": 0.37047222967534205, + "nauc_cv_recall_at_10_std": 0.35490096836353874, + "nauc_cv_recall_at_1_diff1": 0.6811989727471347, + "nauc_cv_recall_at_1_max": 0.7112499520870863, + "nauc_cv_recall_at_1_std": 0.5198436122503737, + "nauc_cv_recall_at_20_diff1": 0.2177190567014866, + "nauc_cv_recall_at_20_max": 0.29381449809900323, + "nauc_cv_recall_at_20_std": 0.33120286981015296, + "nauc_cv_recall_at_3_diff1": 0.45237935528383605, + "nauc_cv_recall_at_3_max": 0.5901481467285061, + "nauc_cv_recall_at_3_std": 0.5043744490014948, + "nauc_cv_recall_at_5_diff1": 0.36561326133619537, + "nauc_cv_recall_at_5_max": 0.5235444056882212, + "nauc_cv_recall_at_5_std": 0.47758334451301304, + "nauc_map_at_1000_diff1": 0.4572039928639722, + "nauc_map_at_1000_max": 0.556407100014074, + "nauc_map_at_1000_std": 0.4681785109479179, + "nauc_map_at_100_diff1": 0.4562262777878306, + "nauc_map_at_100_max": 0.5547399837702551, + "nauc_map_at_100_std": 0.4669314989959343, + "nauc_map_at_10_diff1": 0.4839164521569286, + "nauc_map_at_10_max": 0.581580308737203, + "nauc_map_at_10_std": 0.47985609505140064, + "nauc_map_at_1_diff1": 0.6811989727471347, + "nauc_map_at_1_max": 0.7112499520870863, + "nauc_map_at_1_std": 0.5198436122503737, + "nauc_map_at_20_diff1": 0.47296162994063123, + "nauc_map_at_20_max": 0.5646353533007439, + "nauc_map_at_20_std": 0.4719863696175586, + "nauc_map_at_3_diff1": 0.549209376368368, + "nauc_map_at_3_max": 0.6432214234915912, + "nauc_map_at_3_std": 0.5172682414970432, + "nauc_map_at_5_diff1": 0.5185926257506827, + "nauc_map_at_5_max": 0.6214937035692051, + "nauc_map_at_5_std": 0.5080343780782037, + "nauc_mrr_at_1000_diff1": 0.45723407696403573, + "nauc_mrr_at_1000_max": 0.5564216652366148, + "nauc_mrr_at_1000_std": 0.4681993792193247, + "nauc_mrr_at_100_diff1": 0.45625615571648387, + "nauc_mrr_at_100_max": 0.554754423070155, + "nauc_mrr_at_100_std": 0.46695221181507013, + "nauc_mrr_at_10_diff1": 0.4839164521569286, + "nauc_mrr_at_10_max": 0.581580308737203, + "nauc_mrr_at_10_std": 0.47985609505140064, + "nauc_mrr_at_1_diff1": 0.6811989727471347, + "nauc_mrr_at_1_max": 0.7112499520870863, + "nauc_mrr_at_1_std": 0.5198436122503737, + "nauc_mrr_at_20_diff1": 0.4738328510486203, + "nauc_mrr_at_20_max": 0.5652040793743005, + "nauc_mrr_at_20_std": 0.4724949134770215, + "nauc_mrr_at_3_diff1": 0.549209376368368, + "nauc_mrr_at_3_max": 0.6432214234915912, + "nauc_mrr_at_3_std": 0.5172682414970432, + "nauc_mrr_at_5_diff1": 0.5185926257506827, + "nauc_mrr_at_5_max": 0.6214937035692051, + "nauc_mrr_at_5_std": 0.5080343780782037, + "nauc_ndcg_at_1000_diff1": 0.36375745046225905, + "nauc_ndcg_at_1000_max": 0.4830894657999479, + "nauc_ndcg_at_1000_std": 0.4325926006081789, + "nauc_ndcg_at_100_diff1": 0.30353191564763154, + "nauc_ndcg_at_100_max": 0.43301860060603764, + "nauc_ndcg_at_100_std": 0.4049675638021951, + "nauc_ndcg_at_10_diff1": 0.4021193676211634, + "nauc_ndcg_at_10_max": 0.5101132679389379, + "nauc_ndcg_at_10_std": 0.4388204382824301, + "nauc_ndcg_at_1_diff1": 0.6811989727471347, + "nauc_ndcg_at_1_max": 0.7112499520870863, + "nauc_ndcg_at_1_std": 0.5198436122503737, + "nauc_ndcg_at_20_diff1": 0.3753829158483357, + "nauc_ndcg_at_20_max": 0.46551460951632206, + "nauc_ndcg_at_20_std": 0.4201020394377109, + "nauc_ndcg_at_3_diff1": 0.5198565864811237, + "nauc_ndcg_at_3_max": 0.6272080086003348, + "nauc_ndcg_at_3_std": 0.5136205735200569, + "nauc_ndcg_at_5_diff1": 0.4709900963415891, + "nauc_ndcg_at_5_max": 0.5914010152437607, + "nauc_ndcg_at_5_std": 0.4989203141368066, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.10651632318768808, + "nauc_precision_at_100_max": 0.2846931545464938, + "nauc_precision_at_100_std": 0.33077749419069763, + "nauc_precision_at_10_diff1": 0.24737175890918878, + "nauc_precision_at_10_max": 0.37047222967534255, + "nauc_precision_at_10_std": 0.3549009683635391, + "nauc_precision_at_1_diff1": 0.6811989727471347, + "nauc_precision_at_1_max": 0.7112499520870863, + "nauc_precision_at_1_std": 0.5198436122503737, + "nauc_precision_at_20_diff1": 0.21099288999578236, + "nauc_precision_at_20_max": 0.2900690915664105, + "nauc_precision_at_20_std": 0.3272258932695979, + "nauc_precision_at_3_diff1": 0.45237935528383627, + "nauc_precision_at_3_max": 0.5901481467285062, + "nauc_precision_at_3_std": 0.5043744490014948, + "nauc_precision_at_5_diff1": 0.36561326133619565, + "nauc_precision_at_5_max": 0.5235444056882217, + "nauc_precision_at_5_std": 0.47758334451301343, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.10651632318768792, + "nauc_recall_at_100_max": 0.28469315454649335, + "nauc_recall_at_100_std": 0.3307774941906975, + "nauc_recall_at_10_diff1": 0.24737175890918847, + "nauc_recall_at_10_max": 0.37047222967534205, + "nauc_recall_at_10_std": 0.35490096836353874, + "nauc_recall_at_1_diff1": 0.6811989727471347, + "nauc_recall_at_1_max": 0.7112499520870863, + "nauc_recall_at_1_std": 0.5198436122503737, + "nauc_recall_at_20_diff1": 0.21099288999578225, + "nauc_recall_at_20_max": 0.29006909156641014, + "nauc_recall_at_20_std": 0.3272258932695978, + "nauc_recall_at_3_diff1": 0.45237935528383605, + "nauc_recall_at_3_max": 0.5901481467285061, + "nauc_recall_at_3_std": 0.5043744490014948, + "nauc_recall_at_5_diff1": 0.36561326133619537, + "nauc_recall_at_5_max": 0.5235444056882212, + "nauc_recall_at_5_std": 0.47758334451301304, + "ndcg_at_1": 0.03933, + "ndcg_at_10": 0.06583, + "ndcg_at_100": 0.10342, + "ndcg_at_1000": 0.18752, + "ndcg_at_20": 0.07377, + "ndcg_at_3": 0.05362, + "ndcg_at_5": 0.05735, + "precision_at_1": 0.03933, + "precision_at_10": 0.00989, + "precision_at_100": 0.00298, + "precision_at_1000": 0.001, + "precision_at_20": 0.00652, + "precision_at_3": 0.02097, + "precision_at_5": 0.01438, + "recall_at_1": 0.03933, + "recall_at_10": 0.09888, + "recall_at_100": 0.29775, + "recall_at_1000": 1.0, + "recall_at_20": 0.13034, + "recall_at_3": 0.06292, + "recall_at_5": 0.07191 + }, + { + "cv_recall_at_1": 0.02558, + "cv_recall_at_10": 0.07093, + "cv_recall_at_100": 0.28256, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.1, + "cv_recall_at_3": 0.03837, + "cv_recall_at_5": 0.04419, + "hf_subset": "bg", + "languages": [ + "bul-Cyrl" + ], + "main_score": 0.04462, + "map_at_1": 0.02558, + "map_at_10": 0.03675, + "map_at_100": 0.04235, + "map_at_1000": 0.04478, + "map_at_20": 0.03855, + "map_at_3": 0.03178, + "map_at_5": 0.03306, + "mrr_at_1": 0.02558139534883721, + "mrr_at_10": 0.036754798818752314, + "mrr_at_100": 0.0423459890570135, + "mrr_at_1000": 0.044784919706231274, + "mrr_at_20": 0.03855251028621143, + "mrr_at_3": 0.03178294573643411, + "mrr_at_5": 0.03306201550387596, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2277560274329255, + "nauc_cv_recall_at_100_max": 0.3552178996443939, + "nauc_cv_recall_at_100_std": 0.43804062543965594, + "nauc_cv_recall_at_10_diff1": 0.35780107061012467, + "nauc_cv_recall_at_10_max": 0.39030644785284174, + "nauc_cv_recall_at_10_std": 0.5091951950102706, + "nauc_cv_recall_at_1_diff1": 0.7235285508695759, + "nauc_cv_recall_at_1_max": 0.6937824718881873, + "nauc_cv_recall_at_1_std": 0.5971534502524575, + "nauc_cv_recall_at_20_diff1": 0.26377596545634924, + "nauc_cv_recall_at_20_max": 0.37452031373821454, + "nauc_cv_recall_at_20_std": 0.4336176611901837, + "nauc_cv_recall_at_3_diff1": 0.5328845200055291, + "nauc_cv_recall_at_3_max": 0.6126545844817913, + "nauc_cv_recall_at_3_std": 0.5657893667016287, + "nauc_cv_recall_at_5_diff1": 0.4982630307994294, + "nauc_cv_recall_at_5_max": 0.493165096804259, + "nauc_cv_recall_at_5_std": 0.5330447799132926, + "nauc_map_at_1000_diff1": 0.5045425403634579, + "nauc_map_at_1000_max": 0.5414648881911014, + "nauc_map_at_1000_std": 0.5392505548104458, + "nauc_map_at_100_diff1": 0.5054810695846258, + "nauc_map_at_100_max": 0.5437676463628387, + "nauc_map_at_100_std": 0.5382516698611334, + "nauc_map_at_10_diff1": 0.5455350487951657, + "nauc_map_at_10_max": 0.5617908913164679, + "nauc_map_at_10_std": 0.5565122107198375, + "nauc_map_at_1_diff1": 0.7235285508695759, + "nauc_map_at_1_max": 0.6937824718881873, + "nauc_map_at_1_std": 0.5971534502524575, + "nauc_map_at_20_diff1": 0.5221237722219191, + "nauc_map_at_20_max": 0.5511832677721797, + "nauc_map_at_20_std": 0.5425866236786927, + "nauc_map_at_3_diff1": 0.6067588526703618, + "nauc_map_at_3_max": 0.6429417500507175, + "nauc_map_at_3_std": 0.5757638720739231, + "nauc_map_at_5_diff1": 0.5925839482550422, + "nauc_map_at_5_max": 0.6056554448369696, + "nauc_map_at_5_std": 0.5653784005755035, + "nauc_mrr_at_1000_diff1": 0.5045447401485028, + "nauc_mrr_at_1000_max": 0.5414661483330887, + "nauc_mrr_at_1000_std": 0.5392502061789687, + "nauc_mrr_at_100_diff1": 0.5054810695846258, + "nauc_mrr_at_100_max": 0.5437676463628387, + "nauc_mrr_at_100_std": 0.5382516698611334, + "nauc_mrr_at_10_diff1": 0.5455350487951657, + "nauc_mrr_at_10_max": 0.5617908913164679, + "nauc_mrr_at_10_std": 0.5565122107198375, + "nauc_mrr_at_1_diff1": 0.7235285508695759, + "nauc_mrr_at_1_max": 0.6937824718881873, + "nauc_mrr_at_1_std": 0.5971534502524575, + "nauc_mrr_at_20_diff1": 0.5221237722219191, + "nauc_mrr_at_20_max": 0.5511832677721797, + "nauc_mrr_at_20_std": 0.5425866236786927, + "nauc_mrr_at_3_diff1": 0.6067588526703618, + "nauc_mrr_at_3_max": 0.6429417500507175, + "nauc_mrr_at_3_std": 0.5757638720739231, + "nauc_mrr_at_5_diff1": 0.5925839482550422, + "nauc_mrr_at_5_max": 0.6056554448369696, + "nauc_mrr_at_5_std": 0.5653784005755035, + "nauc_ndcg_at_1000_diff1": 0.40681047732259257, + "nauc_ndcg_at_1000_max": 0.4677020272069214, + "nauc_ndcg_at_1000_std": 0.5095760841854711, + "nauc_ndcg_at_100_diff1": 0.362276233440421, + "nauc_ndcg_at_100_max": 0.4464708274109496, + "nauc_ndcg_at_100_std": 0.48668924230421035, + "nauc_ndcg_at_10_diff1": 0.47689523663253697, + "nauc_ndcg_at_10_max": 0.49869013966045334, + "nauc_ndcg_at_10_std": 0.5393206487389861, + "nauc_ndcg_at_1_diff1": 0.7235285508695759, + "nauc_ndcg_at_1_max": 0.6937824718881873, + "nauc_ndcg_at_1_std": 0.5971534502524575, + "nauc_ndcg_at_20_diff1": 0.41674993791448256, + "nauc_ndcg_at_20_max": 0.47634689041203726, + "nauc_ndcg_at_20_std": 0.49999937386166915, + "nauc_ndcg_at_3_diff1": 0.5844922883447777, + "nauc_ndcg_at_3_max": 0.6337591090258609, + "nauc_ndcg_at_3_std": 0.572655587911338, + "nauc_ndcg_at_5_diff1": 0.5629327805950496, + "nauc_ndcg_at_5_max": 0.5720266485267931, + "nauc_ndcg_at_5_std": 0.5555844489319993, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.22775602743292586, + "nauc_precision_at_100_max": 0.3552178996443944, + "nauc_precision_at_100_std": 0.43804062543965655, + "nauc_precision_at_10_diff1": 0.3578010706101251, + "nauc_precision_at_10_max": 0.3903064478528421, + "nauc_precision_at_10_std": 0.5091951950102711, + "nauc_precision_at_1_diff1": 0.7235285508695759, + "nauc_precision_at_1_max": 0.6937824718881873, + "nauc_precision_at_1_std": 0.5971534502524575, + "nauc_precision_at_20_diff1": 0.26377596545634974, + "nauc_precision_at_20_max": 0.3745203137382151, + "nauc_precision_at_20_std": 0.433617661190184, + "nauc_precision_at_3_diff1": 0.5328845200055288, + "nauc_precision_at_3_max": 0.612654584481791, + "nauc_precision_at_3_std": 0.5657893667016284, + "nauc_precision_at_5_diff1": 0.4982630307994295, + "nauc_precision_at_5_max": 0.4931650968042591, + "nauc_precision_at_5_std": 0.5330447799132926, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2277560274329255, + "nauc_recall_at_100_max": 0.3552178996443939, + "nauc_recall_at_100_std": 0.43804062543965594, + "nauc_recall_at_10_diff1": 0.35780107061012467, + "nauc_recall_at_10_max": 0.39030644785284174, + "nauc_recall_at_10_std": 0.5091951950102706, + "nauc_recall_at_1_diff1": 0.7235285508695759, + "nauc_recall_at_1_max": 0.6937824718881873, + "nauc_recall_at_1_std": 0.5971534502524575, + "nauc_recall_at_20_diff1": 0.26377596545634924, + "nauc_recall_at_20_max": 0.37452031373821454, + "nauc_recall_at_20_std": 0.4336176611901837, + "nauc_recall_at_3_diff1": 0.5328845200055291, + "nauc_recall_at_3_max": 0.6126545844817913, + "nauc_recall_at_3_std": 0.5657893667016287, + "nauc_recall_at_5_diff1": 0.4982630307994294, + "nauc_recall_at_5_max": 0.493165096804259, + "nauc_recall_at_5_std": 0.5330447799132926, + "ndcg_at_1": 0.02558, + "ndcg_at_10": 0.04462, + "ndcg_at_100": 0.08375, + "ndcg_at_1000": 0.1694, + "ndcg_at_20": 0.05169, + "ndcg_at_3": 0.0335, + "ndcg_at_5": 0.03585, + "precision_at_1": 0.02558, + "precision_at_10": 0.00709, + "precision_at_100": 0.00283, + "precision_at_1000": 0.001, + "precision_at_20": 0.005, + "precision_at_3": 0.01279, + "precision_at_5": 0.00884, + "recall_at_1": 0.02558, + "recall_at_10": 0.07093, + "recall_at_100": 0.28256, + "recall_at_1000": 1.0, + "recall_at_20": 0.1, + "recall_at_3": 0.03837, + "recall_at_5": 0.04419 + }, + { + "cv_recall_at_1": 0.29068, + "cv_recall_at_10": 0.52189, + "cv_recall_at_100": 0.77441, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.61167, + "cv_recall_at_3": 0.40067, + "cv_recall_at_5": 0.4624, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.39951, + "map_at_1": 0.29068, + "map_at_10": 0.36108, + "map_at_100": 0.37108, + "map_at_1000": 0.37212, + "map_at_20": 0.3673, + "map_at_3": 0.33895, + "map_at_5": 0.3532, + "mrr_at_1": 0.29068462401795736, + "mrr_at_10": 0.36107726292911485, + "mrr_at_100": 0.3710819986806254, + "mrr_at_1000": 0.3721214599242405, + "mrr_at_20": 0.36730148814055474, + "mrr_at_3": 0.33894500561167235, + "mrr_at_5": 0.35319865319865307, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4387577827219246, + "nauc_cv_recall_at_100_max": 0.6008487386838273, + "nauc_cv_recall_at_100_std": 0.533621040315399, + "nauc_cv_recall_at_10_diff1": 0.49950907077991114, + "nauc_cv_recall_at_10_max": 0.539881136077154, + "nauc_cv_recall_at_10_std": 0.3437086563088045, + "nauc_cv_recall_at_1_diff1": 0.7014840285137456, + "nauc_cv_recall_at_1_max": 0.5261619493150632, + "nauc_cv_recall_at_1_std": 0.1801679569228979, + "nauc_cv_recall_at_20_diff1": 0.4645735138658558, + "nauc_cv_recall_at_20_max": 0.576285247683802, + "nauc_cv_recall_at_20_std": 0.45524140316029943, + "nauc_cv_recall_at_3_diff1": 0.5749082348993855, + "nauc_cv_recall_at_3_max": 0.5187893702843904, + "nauc_cv_recall_at_3_std": 0.2655757143100922, + "nauc_cv_recall_at_5_diff1": 0.5501054370979939, + "nauc_cv_recall_at_5_max": 0.5141499178844948, + "nauc_cv_recall_at_5_std": 0.3210478752976409, + "nauc_map_at_1000_diff1": 0.631552054636958, + "nauc_map_at_1000_max": 0.5218272252876711, + "nauc_map_at_1000_std": 0.24075680609784053, + "nauc_map_at_100_diff1": 0.6309965116020844, + "nauc_map_at_100_max": 0.5219786854993752, + "nauc_map_at_100_std": 0.24136011393137455, + "nauc_map_at_10_diff1": 0.6318556518207006, + "nauc_map_at_10_max": 0.5199789751965203, + "nauc_map_at_10_std": 0.23535067632308174, + "nauc_map_at_1_diff1": 0.7014840285137456, + "nauc_map_at_1_max": 0.5261619493150632, + "nauc_map_at_1_std": 0.1801679569228979, + "nauc_map_at_20_diff1": 0.6303065708342569, + "nauc_map_at_20_max": 0.5221526521785125, + "nauc_map_at_20_std": 0.24159207985057737, + "nauc_map_at_3_diff1": 0.6440257440499185, + "nauc_map_at_3_max": 0.5181108294605662, + "nauc_map_at_3_std": 0.22082207870157222, + "nauc_map_at_5_diff1": 0.6384163044898216, + "nauc_map_at_5_max": 0.5165030019975038, + "nauc_map_at_5_std": 0.23301582972395324, + "nauc_mrr_at_1000_diff1": 0.631552054636958, + "nauc_mrr_at_1000_max": 0.5218272252876711, + "nauc_mrr_at_1000_std": 0.24075680609784053, + "nauc_mrr_at_100_diff1": 0.6309965116020844, + "nauc_mrr_at_100_max": 0.5219786854993752, + "nauc_mrr_at_100_std": 0.24136011393137455, + "nauc_mrr_at_10_diff1": 0.6318556518207006, + "nauc_mrr_at_10_max": 0.5199789751965203, + "nauc_mrr_at_10_std": 0.23535067632308174, + "nauc_mrr_at_1_diff1": 0.7014840285137456, + "nauc_mrr_at_1_max": 0.5261619493150632, + "nauc_mrr_at_1_std": 0.1801679569228979, + "nauc_mrr_at_20_diff1": 0.6303065708342569, + "nauc_mrr_at_20_max": 0.5221526521785125, + "nauc_mrr_at_20_std": 0.24159207985057737, + "nauc_mrr_at_3_diff1": 0.6440257440499185, + "nauc_mrr_at_3_max": 0.5181108294605662, + "nauc_mrr_at_3_std": 0.22082207870157222, + "nauc_mrr_at_5_diff1": 0.6384163044898216, + "nauc_mrr_at_5_max": 0.5165030019975038, + "nauc_mrr_at_5_std": 0.23301582972395324, + "nauc_ndcg_at_1000_diff1": 0.6127710409024325, + "nauc_ndcg_at_1000_max": 0.5263563070902447, + "nauc_ndcg_at_1000_std": 0.2637975600245607, + "nauc_ndcg_at_100_diff1": 0.5985083818914605, + "nauc_ndcg_at_100_max": 0.5320434101562074, + "nauc_ndcg_at_100_std": 0.28454891775164165, + "nauc_ndcg_at_10_diff1": 0.6000563468060606, + "nauc_ndcg_at_10_max": 0.5240039855705034, + "nauc_ndcg_at_10_std": 0.2613431364632493, + "nauc_ndcg_at_1_diff1": 0.7014840285137456, + "nauc_ndcg_at_1_max": 0.5261619493150632, + "nauc_ndcg_at_1_std": 0.1801679569228979, + "nauc_ndcg_at_20_diff1": 0.5941994895044833, + "nauc_ndcg_at_20_max": 0.5320728148915203, + "nauc_ndcg_at_20_std": 0.28502456673907356, + "nauc_ndcg_at_3_diff1": 0.6258340313166566, + "nauc_ndcg_at_3_max": 0.5180926268377164, + "nauc_ndcg_at_3_std": 0.23268002915426508, + "nauc_ndcg_at_5_diff1": 0.6157796708166019, + "nauc_ndcg_at_5_max": 0.5156928199908123, + "nauc_ndcg_at_5_std": 0.25510446898549743, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.43875778272192617, + "nauc_precision_at_100_max": 0.6008487386838282, + "nauc_precision_at_100_std": 0.5336210403154, + "nauc_precision_at_10_diff1": 0.49950907077991147, + "nauc_precision_at_10_max": 0.539881136077154, + "nauc_precision_at_10_std": 0.34370865630880415, + "nauc_precision_at_1_diff1": 0.7014840285137456, + "nauc_precision_at_1_max": 0.5261619493150632, + "nauc_precision_at_1_std": 0.1801679569228979, + "nauc_precision_at_20_diff1": 0.4645735138658555, + "nauc_precision_at_20_max": 0.5762852476838015, + "nauc_precision_at_20_std": 0.4552414031602992, + "nauc_precision_at_3_diff1": 0.5749082348993859, + "nauc_precision_at_3_max": 0.5187893702843903, + "nauc_precision_at_3_std": 0.2655757143100918, + "nauc_precision_at_5_diff1": 0.5501054370979936, + "nauc_precision_at_5_max": 0.5141499178844949, + "nauc_precision_at_5_std": 0.3210478752976408, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4387577827219246, + "nauc_recall_at_100_max": 0.6008487386838273, + "nauc_recall_at_100_std": 0.533621040315399, + "nauc_recall_at_10_diff1": 0.49950907077991114, + "nauc_recall_at_10_max": 0.539881136077154, + "nauc_recall_at_10_std": 0.3437086563088045, + "nauc_recall_at_1_diff1": 0.7014840285137456, + "nauc_recall_at_1_max": 0.5261619493150632, + "nauc_recall_at_1_std": 0.1801679569228979, + "nauc_recall_at_20_diff1": 0.4645735138658558, + "nauc_recall_at_20_max": 0.576285247683802, + "nauc_recall_at_20_std": 0.45524140316029943, + "nauc_recall_at_3_diff1": 0.5749082348993855, + "nauc_recall_at_3_max": 0.5187893702843904, + "nauc_recall_at_3_std": 0.2655757143100922, + "nauc_recall_at_5_diff1": 0.5501054370979939, + "nauc_recall_at_5_max": 0.5141499178844948, + "nauc_recall_at_5_std": 0.3210478752976409, + "ndcg_at_1": 0.29068, + "ndcg_at_10": 0.39951, + "ndcg_at_100": 0.45161, + "ndcg_at_1000": 0.48018, + "ndcg_at_20": 0.4222, + "ndcg_at_3": 0.35479, + "ndcg_at_5": 0.38034, + "precision_at_1": 0.29068, + "precision_at_10": 0.05219, + "precision_at_100": 0.00774, + "precision_at_1000": 0.001, + "precision_at_20": 0.03058, + "precision_at_3": 0.13356, + "precision_at_5": 0.09248, + "recall_at_1": 0.29068, + "recall_at_10": 0.52189, + "recall_at_100": 0.77441, + "recall_at_1000": 1.0, + "recall_at_20": 0.61167, + "recall_at_3": 0.40067, + "recall_at_5": 0.4624 + }, + { + "cv_recall_at_1": 0.04211, + "cv_recall_at_10": 0.08596, + "cv_recall_at_100": 0.32982, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.11404, + "cv_recall_at_3": 0.05789, + "cv_recall_at_5": 0.06842, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.0613, + "map_at_1": 0.04211, + "map_at_10": 0.05372, + "map_at_100": 0.0604, + "map_at_1000": 0.06307, + "map_at_20": 0.05556, + "map_at_3": 0.04883, + "map_at_5": 0.05129, + "mrr_at_1": 0.042105263157894736, + "mrr_at_10": 0.053719715956558056, + "mrr_at_100": 0.06039952708701682, + "mrr_at_1000": 0.06306899201660603, + "mrr_at_20": 0.05555911552041582, + "mrr_at_3": 0.04883040935672514, + "mrr_at_5": 0.05128654970760233, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.09380606219165781, + "nauc_cv_recall_at_100_max": 0.15657469352668574, + "nauc_cv_recall_at_100_std": 0.11667009268949023, + "nauc_cv_recall_at_10_diff1": 0.3468949694178718, + "nauc_cv_recall_at_10_max": 0.24647497850763583, + "nauc_cv_recall_at_10_std": 0.029695054839751628, + "nauc_cv_recall_at_1_diff1": 0.8321035685537966, + "nauc_cv_recall_at_1_max": 0.37766076379572494, + "nauc_cv_recall_at_1_std": -0.042046775780341786, + "nauc_cv_recall_at_20_diff1": 0.2297577258533434, + "nauc_cv_recall_at_20_max": 0.18198772477656933, + "nauc_cv_recall_at_20_std": 0.06291375040379016, + "nauc_cv_recall_at_3_diff1": 0.5682041775414459, + "nauc_cv_recall_at_3_max": 0.3961753299021879, + "nauc_cv_recall_at_3_std": 0.05106877739021546, + "nauc_cv_recall_at_5_diff1": 0.46971854726080636, + "nauc_cv_recall_at_5_max": 0.35717185059819867, + "nauc_cv_recall_at_5_std": 0.06804813110152522, + "nauc_map_at_1000_diff1": 0.5748181453726331, + "nauc_map_at_1000_max": 0.32974875316499175, + "nauc_map_at_1000_std": 0.020406796197221914, + "nauc_map_at_100_diff1": 0.5707707528476451, + "nauc_map_at_100_max": 0.32830437398801354, + "nauc_map_at_100_std": 0.021232965580687834, + "nauc_map_at_10_diff1": 0.6244299171747286, + "nauc_map_at_10_max": 0.34840847762878835, + "nauc_map_at_10_std": 0.007780667702480225, + "nauc_map_at_1_diff1": 0.8321035685537966, + "nauc_map_at_1_max": 0.37766076379572494, + "nauc_map_at_1_std": -0.042046775780341786, + "nauc_map_at_20_diff1": 0.601702734154098, + "nauc_map_at_20_max": 0.3375386968560606, + "nauc_map_at_20_std": 0.01285666527928734, + "nauc_map_at_3_diff1": 0.6993102606164114, + "nauc_map_at_3_max": 0.3881081411097846, + "nauc_map_at_3_std": 0.0075432713858572635, + "nauc_map_at_5_diff1": 0.6621187678192878, + "nauc_map_at_5_max": 0.3756754846678021, + "nauc_map_at_5_std": 0.014107047741541544, + "nauc_mrr_at_1000_diff1": 0.5748181453726331, + "nauc_mrr_at_1000_max": 0.32974875316499175, + "nauc_mrr_at_1000_std": 0.020406796197221914, + "nauc_mrr_at_100_diff1": 0.5707707528476451, + "nauc_mrr_at_100_max": 0.32830437398801354, + "nauc_mrr_at_100_std": 0.021232965580687834, + "nauc_mrr_at_10_diff1": 0.6244299171747286, + "nauc_mrr_at_10_max": 0.34840847762878835, + "nauc_mrr_at_10_std": 0.007780667702480225, + "nauc_mrr_at_1_diff1": 0.8321035685537966, + "nauc_mrr_at_1_max": 0.37766076379572494, + "nauc_mrr_at_1_std": -0.042046775780341786, + "nauc_mrr_at_20_diff1": 0.601702734154098, + "nauc_mrr_at_20_max": 0.3375386968560606, + "nauc_mrr_at_20_std": 0.01285666527928734, + "nauc_mrr_at_3_diff1": 0.6993102606164114, + "nauc_mrr_at_3_max": 0.3881081411097846, + "nauc_mrr_at_3_std": 0.0075432713858572635, + "nauc_mrr_at_5_diff1": 0.6621187678192878, + "nauc_mrr_at_5_max": 0.3756754846678021, + "nauc_mrr_at_5_std": 0.014107047741541544, + "nauc_ndcg_at_1000_diff1": 0.45496062445451224, + "nauc_ndcg_at_1000_max": 0.2865060854068369, + "nauc_ndcg_at_1000_std": 0.04176564515853639, + "nauc_ndcg_at_100_diff1": 0.35624810191070644, + "nauc_ndcg_at_100_max": 0.2512968056518267, + "nauc_ndcg_at_100_std": 0.06261282949917163, + "nauc_ndcg_at_10_diff1": 0.5326357528972646, + "nauc_ndcg_at_10_max": 0.3164974797165294, + "nauc_ndcg_at_10_std": 0.01633397875986684, + "nauc_ndcg_at_1_diff1": 0.8321035685537966, + "nauc_ndcg_at_1_max": 0.37766076379572494, + "nauc_ndcg_at_1_std": -0.042046775780341786, + "nauc_ndcg_at_20_diff1": 0.4695413875225568, + "nauc_ndcg_at_20_max": 0.28473639847072607, + "nauc_ndcg_at_20_std": 0.03105480362242873, + "nauc_ndcg_at_3_diff1": 0.6613777696246801, + "nauc_ndcg_at_3_max": 0.39048664131910743, + "nauc_ndcg_at_3_std": 0.020243838343380078, + "nauc_ndcg_at_5_diff1": 0.6033311906370703, + "nauc_ndcg_at_5_max": 0.370390668135692, + "nauc_ndcg_at_5_std": 0.030727359090790975, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.09380606219165794, + "nauc_precision_at_100_max": 0.1565746935266858, + "nauc_precision_at_100_std": 0.11667009268949041, + "nauc_precision_at_10_diff1": 0.346894969417872, + "nauc_precision_at_10_max": 0.24647497850763628, + "nauc_precision_at_10_std": 0.029695054839751957, + "nauc_precision_at_1_diff1": 0.8321035685537966, + "nauc_precision_at_1_max": 0.37766076379572494, + "nauc_precision_at_1_std": -0.042046775780341786, + "nauc_precision_at_20_diff1": 0.2297577258533437, + "nauc_precision_at_20_max": 0.1819877247765697, + "nauc_precision_at_20_std": 0.0629137504037904, + "nauc_precision_at_3_diff1": 0.5682041775414457, + "nauc_precision_at_3_max": 0.3961753299021877, + "nauc_precision_at_3_std": 0.051068777390215286, + "nauc_precision_at_5_diff1": 0.4697185472608067, + "nauc_precision_at_5_max": 0.3571718505981989, + "nauc_precision_at_5_std": 0.06804813110152524, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.09380606219165781, + "nauc_recall_at_100_max": 0.15657469352668574, + "nauc_recall_at_100_std": 0.11667009268949023, + "nauc_recall_at_10_diff1": 0.3468949694178718, + "nauc_recall_at_10_max": 0.24647497850763583, + "nauc_recall_at_10_std": 0.029695054839751628, + "nauc_recall_at_1_diff1": 0.8321035685537966, + "nauc_recall_at_1_max": 0.37766076379572494, + "nauc_recall_at_1_std": -0.042046775780341786, + "nauc_recall_at_20_diff1": 0.2297577258533434, + "nauc_recall_at_20_max": 0.18198772477656933, + "nauc_recall_at_20_std": 0.06291375040379016, + "nauc_recall_at_3_diff1": 0.5682041775414459, + "nauc_recall_at_3_max": 0.3961753299021879, + "nauc_recall_at_3_std": 0.05106877739021546, + "nauc_recall_at_5_diff1": 0.46971854726080636, + "nauc_recall_at_5_max": 0.35717185059819867, + "nauc_recall_at_5_std": 0.06804813110152522, + "ndcg_at_1": 0.04211, + "ndcg_at_10": 0.0613, + "ndcg_at_100": 0.10684, + "ndcg_at_1000": 0.18972, + "ndcg_at_20": 0.06826, + "ndcg_at_3": 0.05115, + "ndcg_at_5": 0.05553, + "precision_at_1": 0.04211, + "precision_at_10": 0.0086, + "precision_at_100": 0.0033, + "precision_at_1000": 0.001, + "precision_at_20": 0.0057, + "precision_at_3": 0.0193, + "precision_at_5": 0.01368, + "recall_at_1": 0.04211, + "recall_at_10": 0.08596, + "recall_at_100": 0.32982, + "recall_at_1000": 1.0, + "recall_at_20": 0.11404, + "recall_at_3": 0.05789, + "recall_at_5": 0.06842 + }, + { + "cv_recall_at_1": 0.14989, + "cv_recall_at_10": 0.31922, + "cv_recall_at_100": 0.58352, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.37643, + "cv_recall_at_3": 0.21053, + "cv_recall_at_5": 0.24256, + "hf_subset": "et", + "languages": [ + "est-Latn" + ], + "main_score": 0.22233, + "map_at_1": 0.14989, + "map_at_10": 0.193, + "map_at_100": 0.20171, + "map_at_1000": 0.20348, + "map_at_20": 0.19694, + "map_at_3": 0.17582, + "map_at_5": 0.18291, + "mrr_at_1": 0.14988558352402745, + "mrr_at_10": 0.1929978932839344, + "mrr_at_100": 0.20170528803138663, + "mrr_at_1000": 0.20348456836129458, + "mrr_at_20": 0.19694152960335984, + "mrr_at_3": 0.17581998474446983, + "mrr_at_5": 0.18291380625476733, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4882250360254275, + "nauc_cv_recall_at_100_max": 0.6143355183282306, + "nauc_cv_recall_at_100_std": 0.4515863451976158, + "nauc_cv_recall_at_10_diff1": 0.5174096468546779, + "nauc_cv_recall_at_10_max": 0.549513338317607, + "nauc_cv_recall_at_10_std": 0.30730731800583605, + "nauc_cv_recall_at_1_diff1": 0.6808299232010442, + "nauc_cv_recall_at_1_max": 0.5687023501836552, + "nauc_cv_recall_at_1_std": 0.1388330409326975, + "nauc_cv_recall_at_20_diff1": 0.5197372472881798, + "nauc_cv_recall_at_20_max": 0.5625030680575044, + "nauc_cv_recall_at_20_std": 0.36834068406172427, + "nauc_cv_recall_at_3_diff1": 0.5134289510178096, + "nauc_cv_recall_at_3_max": 0.49314510685331053, + "nauc_cv_recall_at_3_std": 0.14642685289874202, + "nauc_cv_recall_at_5_diff1": 0.5291372580273346, + "nauc_cv_recall_at_5_max": 0.5050836393241586, + "nauc_cv_recall_at_5_std": 0.18841237458492802, + "nauc_map_at_1000_diff1": 0.6011728516457758, + "nauc_map_at_1000_max": 0.5469848342131521, + "nauc_map_at_1000_std": 0.18180128661413641, + "nauc_map_at_100_diff1": 0.6007245774464908, + "nauc_map_at_100_max": 0.547240345099142, + "nauc_map_at_100_std": 0.18293810395039678, + "nauc_map_at_10_diff1": 0.5995452269902336, + "nauc_map_at_10_max": 0.5435080550775966, + "nauc_map_at_10_std": 0.17191793396850194, + "nauc_map_at_1_diff1": 0.6808299232010442, + "nauc_map_at_1_max": 0.5687023501836552, + "nauc_map_at_1_std": 0.1388330409326975, + "nauc_map_at_20_diff1": 0.5999647161769283, + "nauc_map_at_20_max": 0.5448508731630719, + "nauc_map_at_20_std": 0.1784683306043517, + "nauc_map_at_3_diff1": 0.5989549491315777, + "nauc_map_at_3_max": 0.5323565312763833, + "nauc_map_at_3_std": 0.13864403101476822, + "nauc_map_at_5_diff1": 0.6026925769676897, + "nauc_map_at_5_max": 0.5361290665852161, + "nauc_map_at_5_std": 0.1510969096382901, + "nauc_mrr_at_1000_diff1": 0.6011728516457758, + "nauc_mrr_at_1000_max": 0.5469848342131521, + "nauc_mrr_at_1000_std": 0.18180128661413641, + "nauc_mrr_at_100_diff1": 0.6007245774464908, + "nauc_mrr_at_100_max": 0.547240345099142, + "nauc_mrr_at_100_std": 0.18293810395039678, + "nauc_mrr_at_10_diff1": 0.5995452269902336, + "nauc_mrr_at_10_max": 0.5435080550775966, + "nauc_mrr_at_10_std": 0.17191793396850194, + "nauc_mrr_at_1_diff1": 0.6808299232010442, + "nauc_mrr_at_1_max": 0.5687023501836552, + "nauc_mrr_at_1_std": 0.1388330409326975, + "nauc_mrr_at_20_diff1": 0.5999647161769283, + "nauc_mrr_at_20_max": 0.5448508731630719, + "nauc_mrr_at_20_std": 0.1784683306043517, + "nauc_mrr_at_3_diff1": 0.5989549491315777, + "nauc_mrr_at_3_max": 0.5323565312763833, + "nauc_mrr_at_3_std": 0.13864403101476822, + "nauc_mrr_at_5_diff1": 0.6026925769676897, + "nauc_mrr_at_5_max": 0.5361290665852161, + "nauc_mrr_at_5_std": 0.1510969096382901, + "nauc_ndcg_at_1000_diff1": 0.5858679688974716, + "nauc_ndcg_at_1000_max": 0.5516847995363141, + "nauc_ndcg_at_1000_std": 0.2156924796629923, + "nauc_ndcg_at_100_diff1": 0.5728007301638065, + "nauc_ndcg_at_100_max": 0.5600893766197521, + "nauc_ndcg_at_100_std": 0.24765875357756895, + "nauc_ndcg_at_10_diff1": 0.5747391302933329, + "nauc_ndcg_at_10_max": 0.5438454792535534, + "nauc_ndcg_at_10_std": 0.2084742509312004, + "nauc_ndcg_at_1_diff1": 0.6808299232010442, + "nauc_ndcg_at_1_max": 0.5687023501836552, + "nauc_ndcg_at_1_std": 0.1388330409326975, + "nauc_ndcg_at_20_diff1": 0.5754484900044904, + "nauc_ndcg_at_20_max": 0.5480596643830482, + "nauc_ndcg_at_20_std": 0.22899177987152664, + "nauc_ndcg_at_3_diff1": 0.5748875227084549, + "nauc_ndcg_at_3_max": 0.5213466125963717, + "nauc_ndcg_at_3_std": 0.14067796344801495, + "nauc_ndcg_at_5_diff1": 0.581243380549002, + "nauc_ndcg_at_5_max": 0.5271206231905041, + "nauc_ndcg_at_5_std": 0.16123828946207489, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.488225036025428, + "nauc_precision_at_100_max": 0.6143355183282307, + "nauc_precision_at_100_std": 0.4515863451976166, + "nauc_precision_at_10_diff1": 0.517409646854678, + "nauc_precision_at_10_max": 0.5495133383176073, + "nauc_precision_at_10_std": 0.3073073180058359, + "nauc_precision_at_1_diff1": 0.6808299232010442, + "nauc_precision_at_1_max": 0.5687023501836552, + "nauc_precision_at_1_std": 0.1388330409326975, + "nauc_precision_at_20_diff1": 0.5197372472881798, + "nauc_precision_at_20_max": 0.5625030680575044, + "nauc_precision_at_20_std": 0.36834068406172404, + "nauc_precision_at_3_diff1": 0.5134289510178094, + "nauc_precision_at_3_max": 0.4931451068533103, + "nauc_precision_at_3_std": 0.14642685289874197, + "nauc_precision_at_5_diff1": 0.5291372580273341, + "nauc_precision_at_5_max": 0.5050836393241585, + "nauc_precision_at_5_std": 0.1884123745849278, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4882250360254275, + "nauc_recall_at_100_max": 0.6143355183282306, + "nauc_recall_at_100_std": 0.4515863451976158, + "nauc_recall_at_10_diff1": 0.5174096468546779, + "nauc_recall_at_10_max": 0.549513338317607, + "nauc_recall_at_10_std": 0.30730731800583605, + "nauc_recall_at_1_diff1": 0.6808299232010442, + "nauc_recall_at_1_max": 0.5687023501836552, + "nauc_recall_at_1_std": 0.1388330409326975, + "nauc_recall_at_20_diff1": 0.5197372472881798, + "nauc_recall_at_20_max": 0.5625030680575044, + "nauc_recall_at_20_std": 0.36834068406172427, + "nauc_recall_at_3_diff1": 0.5134289510178096, + "nauc_recall_at_3_max": 0.49314510685331053, + "nauc_recall_at_3_std": 0.14642685289874202, + "nauc_recall_at_5_diff1": 0.5291372580273346, + "nauc_recall_at_5_max": 0.5050836393241586, + "nauc_recall_at_5_std": 0.18841237458492802, + "ndcg_at_1": 0.14989, + "ndcg_at_10": 0.22233, + "ndcg_at_100": 0.27403, + "ndcg_at_1000": 0.32579, + "ndcg_at_20": 0.23674, + "ndcg_at_3": 0.1847, + "ndcg_at_5": 0.1977, + "precision_at_1": 0.14989, + "precision_at_10": 0.03192, + "precision_at_100": 0.00584, + "precision_at_1000": 0.001, + "precision_at_20": 0.01882, + "precision_at_3": 0.07018, + "precision_at_5": 0.04851, + "recall_at_1": 0.14989, + "recall_at_10": 0.31922, + "recall_at_100": 0.58352, + "recall_at_1000": 1.0, + "recall_at_20": 0.37643, + "recall_at_3": 0.21053, + "recall_at_5": 0.24256 + }, + { + "cv_recall_at_1": 0.29412, + "cv_recall_at_10": 0.58713, + "cv_recall_at_100": 0.84351, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.66371, + "cv_recall_at_3": 0.43507, + "cv_recall_at_5": 0.50055, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.43171, + "map_at_1": 0.29412, + "map_at_10": 0.3831, + "map_at_100": 0.39302, + "map_at_1000": 0.39377, + "map_at_20": 0.3884, + "map_at_3": 0.35664, + "map_at_5": 0.37151, + "mrr_at_1": 0.29411764705882354, + "mrr_at_10": 0.38309726406285766, + "mrr_at_100": 0.3930220896763865, + "mrr_at_1000": 0.39377684475076513, + "mrr_at_20": 0.3884071483670995, + "mrr_at_3": 0.3566407695153533, + "mrr_at_5": 0.3715131335553087, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4843906969178279, + "nauc_cv_recall_at_100_max": 0.43792503492328166, + "nauc_cv_recall_at_100_std": 0.39718120153890746, + "nauc_cv_recall_at_10_diff1": 0.512287603657462, + "nauc_cv_recall_at_10_max": 0.3918793835337743, + "nauc_cv_recall_at_10_std": 0.1831530751852317, + "nauc_cv_recall_at_1_diff1": 0.6952445853768835, + "nauc_cv_recall_at_1_max": 0.4071562479248181, + "nauc_cv_recall_at_1_std": 0.0314163160006179, + "nauc_cv_recall_at_20_diff1": 0.505832405815506, + "nauc_cv_recall_at_20_max": 0.4045125508969871, + "nauc_cv_recall_at_20_std": 0.2514925229932094, + "nauc_cv_recall_at_3_diff1": 0.590516922258488, + "nauc_cv_recall_at_3_max": 0.39106348574742505, + "nauc_cv_recall_at_3_std": 0.10230223861586185, + "nauc_cv_recall_at_5_diff1": 0.5608122121866207, + "nauc_cv_recall_at_5_max": 0.38005227936377856, + "nauc_cv_recall_at_5_std": 0.16948919006007676, + "nauc_map_at_1000_diff1": 0.6369696516571739, + "nauc_map_at_1000_max": 0.3998724277397566, + "nauc_map_at_1000_std": 0.08593593767386237, + "nauc_map_at_100_diff1": 0.6366365082322767, + "nauc_map_at_100_max": 0.39986303867167483, + "nauc_map_at_100_std": 0.08651676401231734, + "nauc_map_at_10_diff1": 0.6356948139146141, + "nauc_map_at_10_max": 0.3981409041198343, + "nauc_map_at_10_std": 0.0825337878498932, + "nauc_map_at_1_diff1": 0.6952445853768835, + "nauc_map_at_1_max": 0.4071562479248181, + "nauc_map_at_1_std": 0.0314163160006179, + "nauc_map_at_20_diff1": 0.6361343896322116, + "nauc_map_at_20_max": 0.3990389829224659, + "nauc_map_at_20_std": 0.08591182877461828, + "nauc_map_at_3_diff1": 0.6467022333496839, + "nauc_map_at_3_max": 0.3975578898342642, + "nauc_map_at_3_std": 0.0659588019641901, + "nauc_map_at_5_diff1": 0.6402241056103767, + "nauc_map_at_5_max": 0.39585631490830975, + "nauc_map_at_5_std": 0.08224554824992186, + "nauc_mrr_at_1000_diff1": 0.6369672954076999, + "nauc_mrr_at_1000_max": 0.3998832679414343, + "nauc_mrr_at_1000_std": 0.08592558754217373, + "nauc_mrr_at_100_diff1": 0.6366341543959031, + "nauc_mrr_at_100_max": 0.3998738564086212, + "nauc_mrr_at_100_std": 0.08650643954365476, + "nauc_mrr_at_10_diff1": 0.6356948139146141, + "nauc_mrr_at_10_max": 0.3981409041198343, + "nauc_mrr_at_10_std": 0.0825337878498932, + "nauc_mrr_at_1_diff1": 0.6952445853768835, + "nauc_mrr_at_1_max": 0.4071562479248181, + "nauc_mrr_at_1_std": 0.0314163160006179, + "nauc_mrr_at_20_diff1": 0.6361316614271182, + "nauc_mrr_at_20_max": 0.39904960011386953, + "nauc_mrr_at_20_std": 0.08590071405171072, + "nauc_mrr_at_3_diff1": 0.6467022333496839, + "nauc_mrr_at_3_max": 0.3975578898342642, + "nauc_mrr_at_3_std": 0.0659588019641901, + "nauc_mrr_at_5_diff1": 0.6402241056103767, + "nauc_mrr_at_5_max": 0.39585631490830975, + "nauc_mrr_at_5_std": 0.08224554824992186, + "nauc_ndcg_at_1000_diff1": 0.6227303173383151, + "nauc_ndcg_at_1000_max": 0.4014729497517295, + "nauc_ndcg_at_1000_std": 0.10451577191092994, + "nauc_ndcg_at_100_diff1": 0.6141022926671437, + "nauc_ndcg_at_100_max": 0.4033225984556337, + "nauc_ndcg_at_100_std": 0.12240480538607998, + "nauc_ndcg_at_10_diff1": 0.6076778215477938, + "nauc_ndcg_at_10_max": 0.3965483537094308, + "nauc_ndcg_at_10_std": 0.10579698991001121, + "nauc_ndcg_at_1_diff1": 0.6952445853768835, + "nauc_ndcg_at_1_max": 0.4071562479248181, + "nauc_ndcg_at_1_std": 0.0314163160006179, + "nauc_ndcg_at_20_diff1": 0.6089840297925048, + "nauc_ndcg_at_20_max": 0.39959392769169655, + "nauc_ndcg_at_20_std": 0.11850236148706383, + "nauc_ndcg_at_3_diff1": 0.6319939826876647, + "nauc_ndcg_at_3_max": 0.3957659622947088, + "nauc_ndcg_at_3_std": 0.07554519519230592, + "nauc_ndcg_at_5_diff1": 0.6201611375960527, + "nauc_ndcg_at_5_max": 0.39195675369259475, + "nauc_ndcg_at_5_std": 0.10395844860266833, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.48439069691782943, + "nauc_precision_at_100_max": 0.43792503492328155, + "nauc_precision_at_100_std": 0.39718120153891034, + "nauc_precision_at_10_diff1": 0.5122876036574616, + "nauc_precision_at_10_max": 0.39187938353377433, + "nauc_precision_at_10_std": 0.18315307518523186, + "nauc_precision_at_1_diff1": 0.6952445853768835, + "nauc_precision_at_1_max": 0.4071562479248181, + "nauc_precision_at_1_std": 0.0314163160006179, + "nauc_precision_at_20_diff1": 0.5058324058155061, + "nauc_precision_at_20_max": 0.4045125508969867, + "nauc_precision_at_20_std": 0.2514925229932092, + "nauc_precision_at_3_diff1": 0.5905169222584882, + "nauc_precision_at_3_max": 0.39106348574742467, + "nauc_precision_at_3_std": 0.10230223861586156, + "nauc_precision_at_5_diff1": 0.5608122121866198, + "nauc_precision_at_5_max": 0.380052279363778, + "nauc_precision_at_5_std": 0.16948919006007634, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4843906969178279, + "nauc_recall_at_100_max": 0.43792503492328166, + "nauc_recall_at_100_std": 0.39718120153890746, + "nauc_recall_at_10_diff1": 0.512287603657462, + "nauc_recall_at_10_max": 0.3918793835337743, + "nauc_recall_at_10_std": 0.1831530751852317, + "nauc_recall_at_1_diff1": 0.6952445853768835, + "nauc_recall_at_1_max": 0.4071562479248181, + "nauc_recall_at_1_std": 0.0314163160006179, + "nauc_recall_at_20_diff1": 0.505832405815506, + "nauc_recall_at_20_max": 0.4045125508969871, + "nauc_recall_at_20_std": 0.2514925229932094, + "nauc_recall_at_3_diff1": 0.590516922258488, + "nauc_recall_at_3_max": 0.39106348574742505, + "nauc_recall_at_3_std": 0.10230223861586185, + "nauc_recall_at_5_diff1": 0.5608122121866207, + "nauc_recall_at_5_max": 0.38005227936377856, + "nauc_recall_at_5_std": 0.16948919006007676, + "ndcg_at_1": 0.29412, + "ndcg_at_10": 0.43171, + "ndcg_at_100": 0.48443, + "ndcg_at_1000": 0.50446, + "ndcg_at_20": 0.45106, + "ndcg_at_3": 0.3768, + "ndcg_at_5": 0.40369, + "precision_at_1": 0.29412, + "precision_at_10": 0.05871, + "precision_at_100": 0.00844, + "precision_at_1000": 0.001, + "precision_at_20": 0.03319, + "precision_at_3": 0.14502, + "precision_at_5": 0.10011, + "recall_at_1": 0.29412, + "recall_at_10": 0.58713, + "recall_at_100": 0.84351, + "recall_at_1000": 1.0, + "recall_at_20": 0.66371, + "recall_at_3": 0.43507, + "recall_at_5": 0.50055 + }, + { + "cv_recall_at_1": 0.03222, + "cv_recall_at_10": 0.06767, + "cv_recall_at_100": 0.22879, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.0913, + "cv_recall_at_3": 0.04941, + "cv_recall_at_5": 0.05693, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.04892, + "map_at_1": 0.03222, + "map_at_10": 0.04304, + "map_at_100": 0.04719, + "map_at_1000": 0.04959, + "map_at_20": 0.0446, + "map_at_3": 0.03992, + "map_at_5": 0.04153, + "mrr_at_1": 0.0322234156820623, + "mrr_at_10": 0.043041276661040365, + "mrr_at_100": 0.04718814194903709, + "mrr_at_1000": 0.04959093601368786, + "mrr_at_20": 0.0445958309537952, + "mrr_at_3": 0.03992123165055496, + "mrr_at_5": 0.04153240243465807, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.10562240979373716, + "nauc_cv_recall_at_100_max": 0.2105611626269996, + "nauc_cv_recall_at_100_std": 0.25837760713964875, + "nauc_cv_recall_at_10_diff1": 0.29553317613513297, + "nauc_cv_recall_at_10_max": 0.47472584291680914, + "nauc_cv_recall_at_10_std": 0.4991933445782172, + "nauc_cv_recall_at_1_diff1": 0.6407063004327759, + "nauc_cv_recall_at_1_max": 0.7093671143201729, + "nauc_cv_recall_at_1_std": 0.6154096090896026, + "nauc_cv_recall_at_20_diff1": 0.19368466542337776, + "nauc_cv_recall_at_20_max": 0.36592455501695004, + "nauc_cv_recall_at_20_std": 0.46422209858794283, + "nauc_cv_recall_at_3_diff1": 0.44919576693768404, + "nauc_cv_recall_at_3_max": 0.6201389699342573, + "nauc_cv_recall_at_3_std": 0.5796657253268006, + "nauc_cv_recall_at_5_diff1": 0.37643444575546825, + "nauc_cv_recall_at_5_max": 0.5566219987238744, + "nauc_cv_recall_at_5_std": 0.5375968609046953, + "nauc_map_at_1000_diff1": 0.4518544643890885, + "nauc_map_at_1000_max": 0.5903397782167271, + "nauc_map_at_1000_std": 0.5543340345108702, + "nauc_map_at_100_diff1": 0.4527133546593165, + "nauc_map_at_100_max": 0.5902421317414485, + "nauc_map_at_100_std": 0.5539003872018832, + "nauc_map_at_10_diff1": 0.4873787589645924, + "nauc_map_at_10_max": 0.6303591553353276, + "nauc_map_at_10_std": 0.5814079879764974, + "nauc_map_at_1_diff1": 0.6407063004327759, + "nauc_map_at_1_max": 0.7093671143201729, + "nauc_map_at_1_std": 0.6154096090896026, + "nauc_map_at_20_diff1": 0.4687980014608742, + "nauc_map_at_20_max": 0.6102232719890351, + "nauc_map_at_20_std": 0.574625041111532, + "nauc_map_at_3_diff1": 0.5359507784595627, + "nauc_map_at_3_max": 0.6735009660285682, + "nauc_map_at_3_std": 0.6051326156020952, + "nauc_map_at_5_diff1": 0.5099922656257474, + "nauc_map_at_5_max": 0.6520574953798828, + "nauc_map_at_5_std": 0.5912656863022154, + "nauc_mrr_at_1000_diff1": 0.4518419414636818, + "nauc_mrr_at_1000_max": 0.5903318134461414, + "nauc_mrr_at_1000_std": 0.5543324172527735, + "nauc_mrr_at_100_diff1": 0.4527008589324545, + "nauc_mrr_at_100_max": 0.5902341876107777, + "nauc_mrr_at_100_std": 0.5538987762820369, + "nauc_mrr_at_10_diff1": 0.4873787589645924, + "nauc_mrr_at_10_max": 0.6303591553353276, + "nauc_mrr_at_10_std": 0.5814079879764974, + "nauc_mrr_at_1_diff1": 0.6407063004327759, + "nauc_mrr_at_1_max": 0.7093671143201729, + "nauc_mrr_at_1_std": 0.6154096090896026, + "nauc_mrr_at_20_diff1": 0.4687980014608742, + "nauc_mrr_at_20_max": 0.6102232719890351, + "nauc_mrr_at_20_std": 0.574625041111532, + "nauc_mrr_at_3_diff1": 0.5359507784595627, + "nauc_mrr_at_3_max": 0.6735009660285682, + "nauc_mrr_at_3_std": 0.6051326156020952, + "nauc_mrr_at_5_diff1": 0.5099922656257474, + "nauc_mrr_at_5_max": 0.6520574953798828, + "nauc_mrr_at_5_std": 0.5912656863022154, + "nauc_ndcg_at_1000_diff1": 0.35206988723325433, + "nauc_ndcg_at_1000_max": 0.4933585210026129, + "nauc_ndcg_at_1000_std": 0.487200281897772, + "nauc_ndcg_at_100_diff1": 0.2956713638094448, + "nauc_ndcg_at_100_max": 0.42359186477415095, + "nauc_ndcg_at_100_std": 0.4286119018031178, + "nauc_ndcg_at_10_diff1": 0.4247419124081797, + "nauc_ndcg_at_10_max": 0.5805727332491769, + "nauc_ndcg_at_10_std": 0.555289148310664, + "nauc_ndcg_at_1_diff1": 0.6407063004327759, + "nauc_ndcg_at_1_max": 0.7093671143201729, + "nauc_ndcg_at_1_std": 0.6154096090896026, + "nauc_ndcg_at_20_diff1": 0.3705827146953171, + "nauc_ndcg_at_20_max": 0.5241929646948014, + "nauc_ndcg_at_20_std": 0.5356342066296511, + "nauc_ndcg_at_3_diff1": 0.5099647348339971, + "nauc_ndcg_at_3_max": 0.6580326655970231, + "nauc_ndcg_at_3_std": 0.597873810995354, + "nauc_ndcg_at_5_diff1": 0.46849727115361317, + "nauc_ndcg_at_5_max": 0.6231169257330541, + "nauc_ndcg_at_5_std": 0.5751661711855892, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.10562240979373733, + "nauc_precision_at_100_max": 0.21056116262699995, + "nauc_precision_at_100_std": 0.2583776071396491, + "nauc_precision_at_10_diff1": 0.29553317613513325, + "nauc_precision_at_10_max": 0.47472584291680964, + "nauc_precision_at_10_std": 0.49919334457821773, + "nauc_precision_at_1_diff1": 0.6407063004327759, + "nauc_precision_at_1_max": 0.7093671143201729, + "nauc_precision_at_1_std": 0.6154096090896026, + "nauc_precision_at_20_diff1": 0.1936846654233782, + "nauc_precision_at_20_max": 0.3659245550169505, + "nauc_precision_at_20_std": 0.4642220985879432, + "nauc_precision_at_3_diff1": 0.4491957669376836, + "nauc_precision_at_3_max": 0.6201389699342572, + "nauc_precision_at_3_std": 0.5796657253268003, + "nauc_precision_at_5_diff1": 0.3764344457554687, + "nauc_precision_at_5_max": 0.556621998723875, + "nauc_precision_at_5_std": 0.5375968609046958, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.10562240979373716, + "nauc_recall_at_100_max": 0.2105611626269996, + "nauc_recall_at_100_std": 0.25837760713964875, + "nauc_recall_at_10_diff1": 0.29553317613513297, + "nauc_recall_at_10_max": 0.47472584291680914, + "nauc_recall_at_10_std": 0.4991933445782172, + "nauc_recall_at_1_diff1": 0.6407063004327759, + "nauc_recall_at_1_max": 0.7093671143201729, + "nauc_recall_at_1_std": 0.6154096090896026, + "nauc_recall_at_20_diff1": 0.19368466542337776, + "nauc_recall_at_20_max": 0.36592455501695004, + "nauc_recall_at_20_std": 0.46422209858794283, + "nauc_recall_at_3_diff1": 0.44919576693768404, + "nauc_recall_at_3_max": 0.6201389699342573, + "nauc_recall_at_3_std": 0.5796657253268006, + "nauc_recall_at_5_diff1": 0.37643444575546825, + "nauc_recall_at_5_max": 0.5566219987238744, + "nauc_recall_at_5_std": 0.5375968609046953, + "ndcg_at_1": 0.03222, + "ndcg_at_10": 0.04892, + "ndcg_at_100": 0.0784, + "ndcg_at_1000": 0.16893, + "ndcg_at_20": 0.05478, + "ndcg_at_3": 0.04236, + "ndcg_at_5": 0.04537, + "precision_at_1": 0.03222, + "precision_at_10": 0.00677, + "precision_at_100": 0.00229, + "precision_at_1000": 0.001, + "precision_at_20": 0.00456, + "precision_at_3": 0.01647, + "precision_at_5": 0.01139, + "recall_at_1": 0.03222, + "recall_at_10": 0.06767, + "recall_at_100": 0.22879, + "recall_at_1000": 1.0, + "recall_at_20": 0.0913, + "recall_at_3": 0.04941, + "recall_at_5": 0.05693 + }, + { + "cv_recall_at_1": 0.046, + "cv_recall_at_10": 0.183, + "cv_recall_at_100": 0.468, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.231, + "cv_recall_at_3": 0.088, + "cv_recall_at_5": 0.127, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.10243, + "map_at_1": 0.046, + "map_at_10": 0.07809, + "map_at_100": 0.0867, + "map_at_1000": 0.08868, + "map_at_20": 0.08144, + "map_at_3": 0.06217, + "map_at_5": 0.07087, + "mrr_at_1": 0.046, + "mrr_at_10": 0.07808690476190482, + "mrr_at_100": 0.0867040477420345, + "mrr_at_1000": 0.0886765889783381, + "mrr_at_20": 0.08143502048776828, + "mrr_at_3": 0.06216666666666669, + "mrr_at_5": 0.07086666666666668, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.20749990776317787, + "nauc_cv_recall_at_100_max": 0.436570055183973, + "nauc_cv_recall_at_100_std": 0.41855883918641856, + "nauc_cv_recall_at_10_diff1": 0.21364906028285896, + "nauc_cv_recall_at_10_max": 0.38371942001278025, + "nauc_cv_recall_at_10_std": 0.2998176500231185, + "nauc_cv_recall_at_1_diff1": 0.5176152868025337, + "nauc_cv_recall_at_1_max": 0.5964349459292354, + "nauc_cv_recall_at_1_std": 0.3364344792991215, + "nauc_cv_recall_at_20_diff1": 0.21708481966818757, + "nauc_cv_recall_at_20_max": 0.36739457109057766, + "nauc_cv_recall_at_20_std": 0.29985064332987743, + "nauc_cv_recall_at_3_diff1": 0.3207155401614752, + "nauc_cv_recall_at_3_max": 0.4184940361489865, + "nauc_cv_recall_at_3_std": 0.23899004805229662, + "nauc_cv_recall_at_5_diff1": 0.2426378699275454, + "nauc_cv_recall_at_5_max": 0.41560723773016267, + "nauc_cv_recall_at_5_std": 0.28245722259275363, + "nauc_map_at_1000_diff1": 0.35475021236774645, + "nauc_map_at_1000_max": 0.4769723827541295, + "nauc_map_at_1000_std": 0.306061384977434, + "nauc_map_at_100_diff1": 0.3527062886798938, + "nauc_map_at_100_max": 0.475180844474526, + "nauc_map_at_100_std": 0.3057957381898707, + "nauc_map_at_10_diff1": 0.3553754944785794, + "nauc_map_at_10_max": 0.47785190792936655, + "nauc_map_at_10_std": 0.2996686156665722, + "nauc_map_at_1_diff1": 0.5176152868025337, + "nauc_map_at_1_max": 0.5964349459292354, + "nauc_map_at_1_std": 0.3364344792991215, + "nauc_map_at_20_diff1": 0.35322811248117314, + "nauc_map_at_20_max": 0.47292251789868756, + "nauc_map_at_20_std": 0.2995973460187708, + "nauc_map_at_3_diff1": 0.40677657640678727, + "nauc_map_at_3_max": 0.49858974183751503, + "nauc_map_at_3_std": 0.2837240886193138, + "nauc_map_at_5_diff1": 0.3693536652542208, + "nauc_map_at_5_max": 0.4893966665377987, + "nauc_map_at_5_std": 0.29508609208289105, + "nauc_mrr_at_1000_diff1": 0.35475021236774645, + "nauc_mrr_at_1000_max": 0.4769723827541295, + "nauc_mrr_at_1000_std": 0.306061384977434, + "nauc_mrr_at_100_diff1": 0.3527062886798938, + "nauc_mrr_at_100_max": 0.475180844474526, + "nauc_mrr_at_100_std": 0.3057957381898707, + "nauc_mrr_at_10_diff1": 0.3553754944785794, + "nauc_mrr_at_10_max": 0.47785190792936655, + "nauc_mrr_at_10_std": 0.2996686156665722, + "nauc_mrr_at_1_diff1": 0.5176152868025337, + "nauc_mrr_at_1_max": 0.5964349459292354, + "nauc_mrr_at_1_std": 0.3364344792991215, + "nauc_mrr_at_20_diff1": 0.35322811248117314, + "nauc_mrr_at_20_max": 0.47292251789868756, + "nauc_mrr_at_20_std": 0.2995973460187708, + "nauc_mrr_at_3_diff1": 0.40677657640678727, + "nauc_mrr_at_3_max": 0.49858974183751503, + "nauc_mrr_at_3_std": 0.2837240886193138, + "nauc_mrr_at_5_diff1": 0.3693536652542208, + "nauc_mrr_at_5_max": 0.4893966665377987, + "nauc_mrr_at_5_std": 0.29508609208289105, + "nauc_ndcg_at_1000_diff1": 0.3216112624721145, + "nauc_ndcg_at_1000_max": 0.4624439762191791, + "nauc_ndcg_at_1000_std": 0.3221750936476479, + "nauc_ndcg_at_100_diff1": 0.29198957430111977, + "nauc_ndcg_at_100_max": 0.45015053169593955, + "nauc_ndcg_at_100_std": 0.33914260134993834, + "nauc_ndcg_at_10_diff1": 0.3001737819671239, + "nauc_ndcg_at_10_max": 0.4410967156973652, + "nauc_ndcg_at_10_std": 0.29866148848726565, + "nauc_ndcg_at_1_diff1": 0.5176152868025337, + "nauc_ndcg_at_1_max": 0.5964349459292354, + "nauc_ndcg_at_1_std": 0.3364344792991215, + "nauc_ndcg_at_20_diff1": 0.29716029150230633, + "nauc_ndcg_at_20_max": 0.43019187042701645, + "nauc_ndcg_at_20_std": 0.29866122236808, + "nauc_ndcg_at_3_diff1": 0.3787649140925966, + "nauc_ndcg_at_3_max": 0.472604051756553, + "nauc_ndcg_at_3_std": 0.2692416496275881, + "nauc_ndcg_at_5_diff1": 0.32430704106536345, + "nauc_ndcg_at_5_max": 0.46253507332181965, + "nauc_ndcg_at_5_std": 0.2898110469273884, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.20749990776317812, + "nauc_precision_at_100_max": 0.43657005518397296, + "nauc_precision_at_100_std": 0.41855883918641856, + "nauc_precision_at_10_diff1": 0.21364906028285893, + "nauc_precision_at_10_max": 0.3837194200127799, + "nauc_precision_at_10_std": 0.2998176500231184, + "nauc_precision_at_1_diff1": 0.5176152868025337, + "nauc_precision_at_1_max": 0.5964349459292354, + "nauc_precision_at_1_std": 0.3364344792991215, + "nauc_precision_at_20_diff1": 0.2170848196681876, + "nauc_precision_at_20_max": 0.36739457109057777, + "nauc_precision_at_20_std": 0.29985064332987715, + "nauc_precision_at_3_diff1": 0.3207155401614752, + "nauc_precision_at_3_max": 0.41849403614898656, + "nauc_precision_at_3_std": 0.23899004805229648, + "nauc_precision_at_5_diff1": 0.2426378699275453, + "nauc_precision_at_5_max": 0.41560723773016267, + "nauc_precision_at_5_std": 0.2824572225927536, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.20749990776317787, + "nauc_recall_at_100_max": 0.436570055183973, + "nauc_recall_at_100_std": 0.41855883918641856, + "nauc_recall_at_10_diff1": 0.21364906028285896, + "nauc_recall_at_10_max": 0.38371942001278025, + "nauc_recall_at_10_std": 0.2998176500231185, + "nauc_recall_at_1_diff1": 0.5176152868025337, + "nauc_recall_at_1_max": 0.5964349459292354, + "nauc_recall_at_1_std": 0.3364344792991215, + "nauc_recall_at_20_diff1": 0.21708481966818757, + "nauc_recall_at_20_max": 0.36739457109057766, + "nauc_recall_at_20_std": 0.29985064332987743, + "nauc_recall_at_3_diff1": 0.3207155401614752, + "nauc_recall_at_3_max": 0.4184940361489865, + "nauc_recall_at_3_std": 0.23899004805229662, + "nauc_recall_at_5_diff1": 0.2426378699275454, + "nauc_recall_at_5_max": 0.41560723773016267, + "nauc_recall_at_5_std": 0.28245722259275363, + "ndcg_at_1": 0.046, + "ndcg_at_10": 0.10243, + "ndcg_at_100": 0.15691, + "ndcg_at_1000": 0.22153, + "ndcg_at_20": 0.11458, + "ndcg_at_3": 0.0687, + "ndcg_at_5": 0.08458, + "precision_at_1": 0.046, + "precision_at_10": 0.0183, + "precision_at_100": 0.00468, + "precision_at_1000": 0.001, + "precision_at_20": 0.01155, + "precision_at_3": 0.02933, + "precision_at_5": 0.0254, + "recall_at_1": 0.046, + "recall_at_10": 0.183, + "recall_at_100": 0.468, + "recall_at_1000": 1.0, + "recall_at_20": 0.231, + "recall_at_3": 0.088, + "recall_at_5": 0.127 + }, + { + "cv_recall_at_1": 0.28433, + "cv_recall_at_10": 0.49653, + "cv_recall_at_100": 0.74202, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.55617, + "cv_recall_at_3": 0.38696, + "cv_recall_at_5": 0.42996, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.38454, + "map_at_1": 0.28433, + "map_at_10": 0.34956, + "map_at_100": 0.3582, + "map_at_1000": 0.3594, + "map_at_20": 0.35373, + "map_at_3": 0.33056, + "map_at_5": 0.34068, + "mrr_at_1": 0.2843273231622746, + "mrr_at_10": 0.3496736234947052, + "mrr_at_100": 0.35831473713940976, + "mrr_at_1000": 0.3595171973886647, + "mrr_at_20": 0.3538465479813173, + "mrr_at_3": 0.3310217290799815, + "mrr_at_5": 0.3407998150716597, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.23761896041641875, + "nauc_cv_recall_at_100_max": 0.43470547280407584, + "nauc_cv_recall_at_100_std": 0.4975354980788784, + "nauc_cv_recall_at_10_diff1": 0.4150816022911774, + "nauc_cv_recall_at_10_max": 0.5209102105032801, + "nauc_cv_recall_at_10_std": 0.4249162231453765, + "nauc_cv_recall_at_1_diff1": 0.6280767235329688, + "nauc_cv_recall_at_1_max": 0.5388327846189178, + "nauc_cv_recall_at_1_std": 0.3137589617260528, + "nauc_cv_recall_at_20_diff1": 0.38530925823819695, + "nauc_cv_recall_at_20_max": 0.5028179354666593, + "nauc_cv_recall_at_20_std": 0.4521541760355578, + "nauc_cv_recall_at_3_diff1": 0.4767108005085613, + "nauc_cv_recall_at_3_max": 0.5112648355110674, + "nauc_cv_recall_at_3_std": 0.37223839089566285, + "nauc_cv_recall_at_5_diff1": 0.45511911048701437, + "nauc_cv_recall_at_5_max": 0.5115753542928264, + "nauc_cv_recall_at_5_std": 0.37491215163806885, + "nauc_map_at_1000_diff1": 0.5431805028682132, + "nauc_map_at_1000_max": 0.5260011879130603, + "nauc_map_at_1000_std": 0.3491217816227124, + "nauc_map_at_100_diff1": 0.5423833566859384, + "nauc_map_at_100_max": 0.5257651543217859, + "nauc_map_at_100_std": 0.34955714965846457, + "nauc_map_at_10_diff1": 0.5451356885776881, + "nauc_map_at_10_max": 0.5286083509975998, + "nauc_map_at_10_std": 0.3479502149045429, + "nauc_map_at_1_diff1": 0.6280767235329688, + "nauc_map_at_1_max": 0.5388327846189178, + "nauc_map_at_1_std": 0.3137589617260528, + "nauc_map_at_20_diff1": 0.5435138517432606, + "nauc_map_at_20_max": 0.5272449970946642, + "nauc_map_at_20_std": 0.34943383178671283, + "nauc_map_at_3_diff1": 0.5566611680382575, + "nauc_map_at_3_max": 0.5277712012619854, + "nauc_map_at_3_std": 0.34333294348863985, + "nauc_map_at_5_diff1": 0.5505014537298937, + "nauc_map_at_5_max": 0.527187496253836, + "nauc_map_at_5_std": 0.3417995414399236, + "nauc_mrr_at_1000_diff1": 0.5431073194186685, + "nauc_mrr_at_1000_max": 0.5260286122417211, + "nauc_mrr_at_1000_std": 0.34880756967791593, + "nauc_mrr_at_100_diff1": 0.5423104335156301, + "nauc_mrr_at_100_max": 0.5257925119636787, + "nauc_mrr_at_100_std": 0.3492438326638032, + "nauc_mrr_at_10_diff1": 0.5450633720446688, + "nauc_mrr_at_10_max": 0.5286352598125965, + "nauc_mrr_at_10_std": 0.34764027112219087, + "nauc_mrr_at_1_diff1": 0.6280767235329688, + "nauc_mrr_at_1_max": 0.5388327846189178, + "nauc_mrr_at_1_std": 0.3137589617260528, + "nauc_mrr_at_20_diff1": 0.5434413327263256, + "nauc_mrr_at_20_max": 0.5272720939865494, + "nauc_mrr_at_20_std": 0.3491225367203654, + "nauc_mrr_at_3_diff1": 0.5563704316641994, + "nauc_mrr_at_3_max": 0.5278785986431076, + "nauc_mrr_at_3_std": 0.3420991513795755, + "nauc_mrr_at_5_diff1": 0.5504290598830466, + "nauc_mrr_at_5_max": 0.5272144010756797, + "nauc_mrr_at_5_std": 0.3414909249262113, + "nauc_ndcg_at_1000_diff1": 0.5192096295937189, + "nauc_ndcg_at_1000_max": 0.5201595378932853, + "nauc_ndcg_at_1000_std": 0.36109934162118995, + "nauc_ndcg_at_100_diff1": 0.49485884979265127, + "nauc_ndcg_at_100_max": 0.5128032894094785, + "nauc_ndcg_at_100_std": 0.37306503846461936, + "nauc_ndcg_at_10_diff1": 0.5127616820408724, + "nauc_ndcg_at_10_max": 0.5262872359686687, + "nauc_ndcg_at_10_std": 0.36597912477976347, + "nauc_ndcg_at_1_diff1": 0.6280767235329688, + "nauc_ndcg_at_1_max": 0.5388327846189178, + "nauc_ndcg_at_1_std": 0.3137589617260528, + "nauc_ndcg_at_20_diff1": 0.5067474987898707, + "nauc_ndcg_at_20_max": 0.5217141516107013, + "nauc_ndcg_at_20_std": 0.3717175599106268, + "nauc_ndcg_at_3_diff1": 0.5356901062750342, + "nauc_ndcg_at_3_max": 0.5234068589888287, + "nauc_ndcg_at_3_std": 0.3519483341630906, + "nauc_ndcg_at_5_diff1": 0.5255266240188277, + "nauc_ndcg_at_5_max": 0.5230445458245795, + "nauc_ndcg_at_5_std": 0.35040124016865504, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.23761896041641897, + "nauc_precision_at_100_max": 0.43470547280407584, + "nauc_precision_at_100_std": 0.49753549807887865, + "nauc_precision_at_10_diff1": 0.41508160229117763, + "nauc_precision_at_10_max": 0.5209102105032803, + "nauc_precision_at_10_std": 0.4249162231453769, + "nauc_precision_at_1_diff1": 0.6280767235329688, + "nauc_precision_at_1_max": 0.5388327846189178, + "nauc_precision_at_1_std": 0.3137589617260528, + "nauc_precision_at_20_diff1": 0.3853092582381967, + "nauc_precision_at_20_max": 0.5028179354666594, + "nauc_precision_at_20_std": 0.45215417603555763, + "nauc_precision_at_3_diff1": 0.4775004668645391, + "nauc_precision_at_3_max": 0.5109427978783841, + "nauc_precision_at_3_std": 0.3758320895768744, + "nauc_precision_at_5_diff1": 0.45511911048701426, + "nauc_precision_at_5_max": 0.5115753542928256, + "nauc_precision_at_5_std": 0.3749121516380687, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.23761896041641875, + "nauc_recall_at_100_max": 0.43470547280407584, + "nauc_recall_at_100_std": 0.4975354980788784, + "nauc_recall_at_10_diff1": 0.4150816022911774, + "nauc_recall_at_10_max": 0.5209102105032801, + "nauc_recall_at_10_std": 0.4249162231453765, + "nauc_recall_at_1_diff1": 0.6280767235329688, + "nauc_recall_at_1_max": 0.5388327846189178, + "nauc_recall_at_1_std": 0.3137589617260528, + "nauc_recall_at_20_diff1": 0.38530925823819695, + "nauc_recall_at_20_max": 0.5028179354666593, + "nauc_recall_at_20_std": 0.4521541760355578, + "nauc_recall_at_3_diff1": 0.4775004668645395, + "nauc_recall_at_3_max": 0.5109427978783846, + "nauc_recall_at_3_std": 0.37583208957687475, + "nauc_recall_at_5_diff1": 0.45511911048701437, + "nauc_recall_at_5_max": 0.5115753542928264, + "nauc_recall_at_5_std": 0.37491215163806885, + "ndcg_at_1": 0.28433, + "ndcg_at_10": 0.38454, + "ndcg_at_100": 0.43357, + "ndcg_at_1000": 0.46632, + "ndcg_at_20": 0.39967, + "ndcg_at_3": 0.34476, + "ndcg_at_5": 0.36302, + "precision_at_1": 0.28433, + "precision_at_10": 0.04965, + "precision_at_100": 0.00742, + "precision_at_1000": 0.001, + "precision_at_20": 0.02781, + "precision_at_3": 0.12853, + "precision_at_5": 0.08599, + "recall_at_1": 0.28433, + "recall_at_10": 0.49653, + "recall_at_100": 0.74202, + "recall_at_1000": 1.0, + "recall_at_20": 0.55617, + "recall_at_3": 0.38558, + "recall_at_5": 0.42996 + }, + { + "cv_recall_at_1": 0.21036, + "cv_recall_at_10": 0.41121, + "cv_recall_at_100": 0.66385, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.48626, + "cv_recall_at_3": 0.31818, + "cv_recall_at_5": 0.35729, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.30654, + "map_at_1": 0.2093, + "map_at_10": 0.27352, + "map_at_100": 0.28284, + "map_at_1000": 0.28417, + "map_at_20": 0.27883, + "map_at_3": 0.2574, + "map_at_5": 0.26638, + "mrr_at_1": 0.21035940803382663, + "mrr_at_10": 0.2740461089298299, + "mrr_at_100": 0.2833713578803151, + "mrr_at_1000": 0.2846954029031251, + "mrr_at_20": 0.2793545398253834, + "mrr_at_3": 0.25792811839323476, + "mrr_at_5": 0.2669133192389007, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.08412810930470697, + "nauc_cv_recall_at_100_max": -0.02893784821948192, + "nauc_cv_recall_at_100_std": 0.47651441730821087, + "nauc_cv_recall_at_10_diff1": 0.09489042949352793, + "nauc_cv_recall_at_10_max": 0.04818954906149728, + "nauc_cv_recall_at_10_std": 0.3784429655388849, + "nauc_cv_recall_at_1_diff1": 0.3793938217491008, + "nauc_cv_recall_at_1_max": 0.19745064963008244, + "nauc_cv_recall_at_1_std": 0.2226905726319812, + "nauc_cv_recall_at_20_diff1": -0.012006572717603949, + "nauc_cv_recall_at_20_max": -0.005659777804928359, + "nauc_cv_recall_at_20_std": 0.4477195025551248, + "nauc_cv_recall_at_3_diff1": 0.20273059052660355, + "nauc_cv_recall_at_3_max": 0.10868493007535936, + "nauc_cv_recall_at_3_std": 0.28516507041481937, + "nauc_cv_recall_at_5_diff1": 0.16823261516669927, + "nauc_cv_recall_at_5_max": 0.08532235034723983, + "nauc_cv_recall_at_5_std": 0.3037949441825064, + "nauc_map_at_1000_diff1": 0.2660247127381813, + "nauc_map_at_1000_max": 0.13870549236132962, + "nauc_map_at_1000_std": 0.27802370987169195, + "nauc_map_at_100_diff1": 0.26492589150670853, + "nauc_map_at_100_max": 0.1383701574159222, + "nauc_map_at_100_std": 0.27874235749011944, + "nauc_map_at_10_diff1": 0.2734729386436431, + "nauc_map_at_10_max": 0.1410624776447832, + "nauc_map_at_10_std": 0.2736177637841688, + "nauc_map_at_1_diff1": 0.3837519718710883, + "nauc_map_at_1_max": 0.19829121524445215, + "nauc_map_at_1_std": 0.22626211058018178, + "nauc_map_at_20_diff1": 0.2659316827657333, + "nauc_map_at_20_max": 0.13786512872317508, + "nauc_map_at_20_std": 0.2785818431897626, + "nauc_map_at_3_diff1": 0.2942392073848912, + "nauc_map_at_3_max": 0.1532573637149286, + "nauc_map_at_3_std": 0.25733381482426193, + "nauc_map_at_5_diff1": 0.284716350245029, + "nauc_map_at_5_max": 0.14692619982324098, + "nauc_map_at_5_std": 0.26223728974597177, + "nauc_mrr_at_1000_diff1": 0.26411706149667585, + "nauc_mrr_at_1000_max": 0.13885635065484084, + "nauc_mrr_at_1000_std": 0.2763953765129113, + "nauc_mrr_at_100_diff1": 0.26302522415884505, + "nauc_mrr_at_100_max": 0.13852085679095028, + "nauc_mrr_at_100_std": 0.2771184420870135, + "nauc_mrr_at_10_diff1": 0.2715777825185273, + "nauc_mrr_at_10_max": 0.14121035656644584, + "nauc_mrr_at_10_std": 0.27200881631607793, + "nauc_mrr_at_1_diff1": 0.3793938217491008, + "nauc_mrr_at_1_max": 0.19745064963008244, + "nauc_mrr_at_1_std": 0.2226905726319812, + "nauc_mrr_at_20_diff1": 0.26403967098494374, + "nauc_mrr_at_20_max": 0.1380162722929516, + "nauc_mrr_at_20_std": 0.27696502679263457, + "nauc_mrr_at_3_diff1": 0.292310398478914, + "nauc_mrr_at_3_max": 0.15339498925691814, + "nauc_mrr_at_3_std": 0.2557279132809406, + "nauc_mrr_at_5_diff1": 0.2828056401282569, + "nauc_mrr_at_5_max": 0.14706903825940273, + "nauc_mrr_at_5_std": 0.26063455175574307, + "nauc_ndcg_at_1000_diff1": 0.22736974070140006, + "nauc_ndcg_at_1000_max": 0.11943544918999241, + "nauc_ndcg_at_1000_std": 0.29876642126214314, + "nauc_ndcg_at_100_diff1": 0.19421368450323065, + "nauc_ndcg_at_100_max": 0.1048189143932704, + "nauc_ndcg_at_100_std": 0.3182914348541533, + "nauc_ndcg_at_10_diff1": 0.22660181037495555, + "nauc_ndcg_at_10_max": 0.11665523286258694, + "nauc_ndcg_at_10_std": 0.3001276869199722, + "nauc_ndcg_at_1_diff1": 0.3837519718710883, + "nauc_ndcg_at_1_max": 0.19829121524445215, + "nauc_ndcg_at_1_std": 0.22626211058018178, + "nauc_ndcg_at_20_diff1": 0.19891196886667445, + "nauc_ndcg_at_20_max": 0.10344495204601216, + "nauc_ndcg_at_20_std": 0.3181706232827814, + "nauc_ndcg_at_3_diff1": 0.2690721795734909, + "nauc_ndcg_at_3_max": 0.1409682595554232, + "nauc_ndcg_at_3_std": 0.2650702824215696, + "nauc_ndcg_at_5_diff1": 0.2528960713191041, + "nauc_ndcg_at_5_max": 0.130149050767249, + "nauc_ndcg_at_5_std": 0.2735581019527345, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": -0.0841281093047071, + "nauc_precision_at_100_max": -0.028937848219481813, + "nauc_precision_at_100_std": 0.47651441730821126, + "nauc_precision_at_10_diff1": 0.09489042949352763, + "nauc_precision_at_10_max": 0.04818954906149731, + "nauc_precision_at_10_std": 0.37844296553888496, + "nauc_precision_at_1_diff1": 0.3837519718710883, + "nauc_precision_at_1_max": 0.19829121524445215, + "nauc_precision_at_1_std": 0.22626211058018178, + "nauc_precision_at_20_diff1": -0.012006572717604604, + "nauc_precision_at_20_max": -0.00565977780492901, + "nauc_precision_at_20_std": 0.44771950255512455, + "nauc_precision_at_3_diff1": 0.20273059052660364, + "nauc_precision_at_3_max": 0.10868493007535932, + "nauc_precision_at_3_std": 0.2851650704148193, + "nauc_precision_at_5_diff1": 0.16823261516669877, + "nauc_precision_at_5_max": 0.08532235034723942, + "nauc_precision_at_5_std": 0.30379494418250624, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.08412810930470697, + "nauc_recall_at_100_max": -0.02893784821948192, + "nauc_recall_at_100_std": 0.47651441730821087, + "nauc_recall_at_10_diff1": 0.09489042949352793, + "nauc_recall_at_10_max": 0.04818954906149728, + "nauc_recall_at_10_std": 0.3784429655388849, + "nauc_recall_at_1_diff1": 0.3837519718710883, + "nauc_recall_at_1_max": 0.19829121524445215, + "nauc_recall_at_1_std": 0.22626211058018178, + "nauc_recall_at_20_diff1": -0.012006572717603949, + "nauc_recall_at_20_max": -0.005659777804928359, + "nauc_recall_at_20_std": 0.4477195025551248, + "nauc_recall_at_3_diff1": 0.20273059052660355, + "nauc_recall_at_3_max": 0.10868493007535936, + "nauc_recall_at_3_std": 0.28516507041481937, + "nauc_recall_at_5_diff1": 0.16823261516669927, + "nauc_recall_at_5_max": 0.08532235034723983, + "nauc_recall_at_5_std": 0.3037949441825064, + "ndcg_at_1": 0.2093, + "ndcg_at_10": 0.30654, + "ndcg_at_100": 0.35751, + "ndcg_at_1000": 0.39881, + "ndcg_at_20": 0.32565, + "ndcg_at_3": 0.27302, + "ndcg_at_5": 0.28916, + "precision_at_1": 0.2093, + "precision_at_10": 0.04112, + "precision_at_100": 0.00664, + "precision_at_1000": 0.001, + "precision_at_20": 0.02431, + "precision_at_3": 0.10606, + "precision_at_5": 0.07146, + "recall_at_1": 0.2093, + "recall_at_10": 0.41121, + "recall_at_100": 0.66385, + "recall_at_1000": 1.0, + "recall_at_20": 0.48626, + "recall_at_3": 0.31818, + "recall_at_5": 0.35729 + }, + { + "cv_recall_at_1": 0.594, + "cv_recall_at_10": 0.85, + "cv_recall_at_100": 0.996, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.904, + "cv_recall_at_3": 0.732, + "cv_recall_at_5": 0.789, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.71959, + "map_at_1": 0.594, + "map_at_10": 0.67823, + "map_at_100": 0.68461, + "map_at_1000": 0.68464, + "map_at_20": 0.6821, + "map_at_3": 0.657, + "map_at_5": 0.6702, + "mrr_at_1": 0.594, + "mrr_at_10": 0.6782289682539687, + "mrr_at_100": 0.684613528313254, + "mrr_at_1000": 0.6846440027954259, + "mrr_at_20": 0.6821040953886158, + "mrr_at_3": 0.657, + "mrr_at_5": 0.6702000000000002, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.7537348272642541, + "nauc_cv_recall_at_100_max": 0.967320261437891, + "nauc_cv_recall_at_100_std": 0.9346405228758147, + "nauc_cv_recall_at_10_diff1": 0.8035242434103493, + "nauc_cv_recall_at_10_max": 0.3245916042954772, + "nauc_cv_recall_at_10_std": -0.7203904978848026, + "nauc_cv_recall_at_1_diff1": 0.8321732542062084, + "nauc_cv_recall_at_1_max": 0.4253716822768802, + "nauc_cv_recall_at_1_std": -0.3308525033829497, + "nauc_cv_recall_at_20_diff1": 0.8025550497976981, + "nauc_cv_recall_at_20_max": 0.24774354186118897, + "nauc_cv_recall_at_20_std": -0.7750252878929338, + "nauc_cv_recall_at_3_diff1": 0.7885327415564996, + "nauc_cv_recall_at_3_max": 0.3675701548271744, + "nauc_cv_recall_at_3_std": -0.5087263348602445, + "nauc_cv_recall_at_5_diff1": 0.7794579521198232, + "nauc_cv_recall_at_5_max": 0.3440420490780201, + "nauc_cv_recall_at_5_std": -0.617837797693913, + "nauc_map_at_1000_diff1": 0.8144087540259416, + "nauc_map_at_1000_max": 0.4032590686849337, + "nauc_map_at_1000_std": -0.40569870835620436, + "nauc_map_at_100_diff1": 0.8144007244122586, + "nauc_map_at_100_max": 0.4033259669845395, + "nauc_map_at_100_std": -0.40553996166294154, + "nauc_map_at_10_diff1": 0.8141070805369576, + "nauc_map_at_10_max": 0.40229735186651844, + "nauc_map_at_10_std": -0.4127291380825015, + "nauc_map_at_1_diff1": 0.8321732542062084, + "nauc_map_at_1_max": 0.4253716822768802, + "nauc_map_at_1_std": -0.3308525033829497, + "nauc_map_at_20_diff1": 0.8143371892155288, + "nauc_map_at_20_max": 0.4014071959812905, + "nauc_map_at_20_std": -0.41027294967571937, + "nauc_map_at_3_diff1": 0.8116345917610723, + "nauc_map_at_3_max": 0.40462408831787977, + "nauc_map_at_3_std": -0.3966193654010294, + "nauc_map_at_5_diff1": 0.8111413316486993, + "nauc_map_at_5_max": 0.401958381543243, + "nauc_map_at_5_std": -0.411090564130863, + "nauc_mrr_at_1000_diff1": 0.81440362259585, + "nauc_mrr_at_1000_max": 0.4032546495314979, + "nauc_mrr_at_1000_std": -0.40568753541777147, + "nauc_mrr_at_100_diff1": 0.8143955935310161, + "nauc_mrr_at_100_max": 0.403321548820474, + "nauc_mrr_at_100_std": -0.4055287889306838, + "nauc_mrr_at_10_diff1": 0.8141070805369576, + "nauc_mrr_at_10_max": 0.40229735186651844, + "nauc_mrr_at_10_std": -0.4127291380825015, + "nauc_mrr_at_1_diff1": 0.8321732542062084, + "nauc_mrr_at_1_max": 0.4253716822768802, + "nauc_mrr_at_1_std": -0.3308525033829497, + "nauc_mrr_at_20_diff1": 0.8143321072331481, + "nauc_mrr_at_20_max": 0.40140280700044934, + "nauc_mrr_at_20_std": -0.4102619171951776, + "nauc_mrr_at_3_diff1": 0.8116345917610723, + "nauc_mrr_at_3_max": 0.40462408831787977, + "nauc_mrr_at_3_std": -0.3966193654010294, + "nauc_mrr_at_5_diff1": 0.8111413316486993, + "nauc_mrr_at_5_max": 0.401958381543243, + "nauc_mrr_at_5_std": -0.411090564130863, + "nauc_ndcg_at_1000_diff1": 0.8129387588004101, + "nauc_ndcg_at_1000_max": 0.3989000675515646, + "nauc_ndcg_at_1000_std": -0.41972484200714394, + "nauc_ndcg_at_100_diff1": 0.8127717229000625, + "nauc_ndcg_at_100_max": 0.40046696209809696, + "nauc_ndcg_at_100_std": -0.41599250354747924, + "nauc_ndcg_at_10_diff1": 0.811699922432113, + "nauc_ndcg_at_10_max": 0.3904109417859162, + "nauc_ndcg_at_10_std": -0.4591140511511065, + "nauc_ndcg_at_1_diff1": 0.8321732542062084, + "nauc_ndcg_at_1_max": 0.4253716822768802, + "nauc_ndcg_at_1_std": -0.3308525033829497, + "nauc_ndcg_at_20_diff1": 0.812209351956681, + "nauc_ndcg_at_20_max": 0.38571822662873284, + "nauc_ndcg_at_20_std": -0.45109142559304494, + "nauc_ndcg_at_3_diff1": 0.8062362518327737, + "nauc_ndcg_at_3_max": 0.396305885742386, + "nauc_ndcg_at_3_std": -0.42186974764971025, + "nauc_ndcg_at_5_diff1": 0.8047198234130771, + "nauc_ndcg_at_5_max": 0.390662297645285, + "nauc_ndcg_at_5_std": -0.45119130174831534, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.7537348272642461, + "nauc_precision_at_100_max": 0.9673202614378978, + "nauc_precision_at_100_std": 0.9346405228758466, + "nauc_precision_at_10_diff1": 0.8035242434103476, + "nauc_precision_at_10_max": 0.3245916042954751, + "nauc_precision_at_10_std": -0.7203904978848067, + "nauc_precision_at_1_diff1": 0.8321732542062084, + "nauc_precision_at_1_max": 0.4253716822768802, + "nauc_precision_at_1_std": -0.3308525033829497, + "nauc_precision_at_20_diff1": 0.802555049797697, + "nauc_precision_at_20_max": 0.2477435418611877, + "nauc_precision_at_20_std": -0.7750252878929389, + "nauc_precision_at_3_diff1": 0.7885327415565007, + "nauc_precision_at_3_max": 0.36757015482717503, + "nauc_precision_at_3_std": -0.5087263348602433, + "nauc_precision_at_5_diff1": 0.7794579521198226, + "nauc_precision_at_5_max": 0.3440420490780189, + "nauc_precision_at_5_std": -0.6178377976939148, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.7537348272642541, + "nauc_recall_at_100_max": 0.967320261437891, + "nauc_recall_at_100_std": 0.9346405228758147, + "nauc_recall_at_10_diff1": 0.8035242434103493, + "nauc_recall_at_10_max": 0.3245916042954772, + "nauc_recall_at_10_std": -0.7203904978848026, + "nauc_recall_at_1_diff1": 0.8321732542062084, + "nauc_recall_at_1_max": 0.4253716822768802, + "nauc_recall_at_1_std": -0.3308525033829497, + "nauc_recall_at_20_diff1": 0.8025550497976981, + "nauc_recall_at_20_max": 0.24774354186118897, + "nauc_recall_at_20_std": -0.7750252878929338, + "nauc_recall_at_3_diff1": 0.7885327415564996, + "nauc_recall_at_3_max": 0.3675701548271744, + "nauc_recall_at_3_std": -0.5087263348602445, + "nauc_recall_at_5_diff1": 0.7794579521198232, + "nauc_recall_at_5_max": 0.3440420490780201, + "nauc_recall_at_5_std": -0.617837797693913, + "ndcg_at_1": 0.594, + "ndcg_at_10": 0.71959, + "ndcg_at_100": 0.75084, + "ndcg_at_1000": 0.75141, + "ndcg_at_20": 0.7334, + "ndcg_at_3": 0.67635, + "ndcg_at_5": 0.69998, + "precision_at_1": 0.594, + "precision_at_10": 0.085, + "precision_at_100": 0.00996, + "precision_at_1000": 0.001, + "precision_at_20": 0.0452, + "precision_at_3": 0.244, + "precision_at_5": 0.1578, + "recall_at_1": 0.594, + "recall_at_10": 0.85, + "recall_at_100": 0.996, + "recall_at_1000": 1.0, + "recall_at_20": 0.904, + "recall_at_3": 0.732, + "recall_at_5": 0.789 + } + ] + }, + "task_name": "WITT2IRetrieval" +} \ No newline at end of file From c5a1c07b9b26844e20a54b2f1a11e7594363c7b4 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 19 Aug 2024 20:15:19 +0100 Subject: [PATCH 052/154] correct multilingual t2i meta --- .../multilingual/WITT2IRetrieval.py | 68 +++++++++---------- .../multilingual/XFlickr30kCoT2IRetrieval.py | 2 +- .../multilingual/XM3600T2IRetrieval.py | 2 +- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py index 17321d4e89..ba21f0fe7b 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py @@ -126,83 +126,83 @@ class WITT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): descriptive_stats={ "n_samples": None, "avg_character_length": { - "dev": { + "test": { "ar": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 7200, - "num_queries": 3600, - "average_relevant_docs_per_query": 2.0, + "num_documents": 792, + "num_queries": 890, + "average_relevant_docs_per_query": 0.89, }, "bg": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 7200, - "num_queries": 3600, - "average_relevant_docs_per_query": 2.0, + "num_documents": 806, + "num_queries": 890, + "average_relevant_docs_per_query": 0.91, }, "da": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 7200, - "num_queries": 3600, - "average_relevant_docs_per_query": 2.0, + "num_documents": 814, + "num_queries": 890, + "average_relevant_docs_per_query": 0.91, }, "el": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 7200, - "num_queries": 3600, - "average_relevant_docs_per_query": 2.0, + "num_documents": 541, + "num_queries": 890, + "average_relevant_docs_per_query": 0.61, }, "et": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 7200, - "num_queries": 3600, - "average_relevant_docs_per_query": 2.0, + "num_documents": 780, + "num_queries": 890, + "average_relevant_docs_per_query": 0.88, }, "id": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 7200, - "num_queries": 3600, - "average_relevant_docs_per_query": 2.0, + "num_documents": 854, + "num_queries": 890, + "average_relevant_docs_per_query": 0.96, }, "ja": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 7200, - "num_queries": 3600, - "average_relevant_docs_per_query": 2.0, + "num_documents": 842, + "num_queries": 890, + "average_relevant_docs_per_query": 0.95, }, "ko": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 7200, - "num_queries": 3600, - "average_relevant_docs_per_query": 2.0, + "num_documents": 889, + "num_queries": 890, + "average_relevant_docs_per_query": 1.0, }, "tr": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 7200, - "num_queries": 3600, - "average_relevant_docs_per_query": 2.0, + "num_documents": 681, + "num_queries": 890, + "average_relevant_docs_per_query": 0.77, }, "vi": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 7200, - "num_queries": 3600, - "average_relevant_docs_per_query": 2.0, + "num_documents": 869, + "num_queries": 890, + "average_relevant_docs_per_query": 0.98, }, "en": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 7200, - "num_queries": 3600, - "average_relevant_docs_per_query": 2.0, + "num_documents": 685, + "num_queries": 890, + "average_relevant_docs_per_query": 0.77, }, } }, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py index 1ed73db4c2..ae9ca88cd4 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py @@ -110,7 +110,7 @@ class XFlickr30kCoT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): descriptive_stats={ "n_samples": None, "avg_character_length": { - "dev": { + "test": { "de": { "average_document_length": 0.0, "average_query_length": 0.0, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py index 350d91f3c7..5537358750 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py @@ -154,7 +154,7 @@ class XM3600T2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): descriptive_stats={ "n_samples": None, "avg_character_length": { - "dev": { + "test": { "ar": { "average_document_length": 0.0, "average_query_length": 0.0, From 99cc566dbc2ebc530ca3259cdedc3c72a8663bc6 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 19 Aug 2024 20:17:36 +0100 Subject: [PATCH 053/154] meta --- .../multilingual/WITT2IRetrieval.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py index ba21f0fe7b..0cf391567a 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py @@ -95,7 +95,7 @@ class WITT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): metadata = TaskMetadata( name="WITT2IRetrieval", description="Retrieve images based on multilingual descriptions.", - reference="https://aclanthology.org/2022.emnlp-main.45/", + reference="https://proceedings.mlr.press/v162/bugliarello22a/bugliarello22a.pdf", dataset={ "path": "mteb/wit", "revision": "91ac153f1371a98b209ed763205e25e115ecd06e", @@ -116,12 +116,13 @@ class WITT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): dialect=[], modalities=["text", "image"], sample_creation="found", - bibtex_citation="""@inproceedings{thapliyal2022crossmodal, - title={Crossmodal-3600: A Massively Multilingual Multimodal Evaluation Dataset}, - author={Thapliyal, Ashish V and Tuset, Jordi Pont and Chen, Xi and Soricut, Radu}, - booktitle={Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing}, - pages={715--729}, - year={2022} + bibtex_citation="""@inproceedings{bugliarello2022iglue, + title={IGLUE: A benchmark for transfer learning across modalities, tasks, and languages}, + author={Bugliarello, Emanuele and Liu, Fangyu and Pfeiffer, Jonas and Reddy, Siva and Elliott, Desmond and Ponti, Edoardo Maria and Vuli{\'c}, Ivan}, + booktitle={International Conference on Machine Learning}, + pages={2370--2392}, + year={2022}, + organization={PMLR} }""", descriptive_stats={ "n_samples": None, From 848dea643645bde7929fc83040a66c3a059025c8 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 26 Aug 2024 18:34:14 +0100 Subject: [PATCH 054/154] add dinov2 model; 4 sizes --- mteb/models/__init__.py | 2 + mteb/models/dino_models.py | 162 +++++++++++++++ .../eng/MNISTClassification.py | 1 + .../MNIST.json | 48 +++++ .../NIGHTSI2IRetrieval.json | 186 ++++++++++++++++++ .../OxfordFlowersClassification.json | 48 +++++ .../model_meta.json | 1 + .../MNIST.json | 48 +++++ .../NIGHTSI2IRetrieval.json | 186 ++++++++++++++++++ .../OxfordFlowersClassification.json | 48 +++++ .../model_meta.json | 1 + .../MNIST.json | 48 +++++ .../NIGHTSI2IRetrieval.json | 186 ++++++++++++++++++ .../OxfordFlowersClassification.json | 48 +++++ .../model_meta.json | 1 + 15 files changed, 1014 insertions(+) create mode 100644 mteb/models/dino_models.py create mode 100644 results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/MNIST.json create mode 100644 results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/NIGHTSI2IRetrieval.json create mode 100644 results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/OxfordFlowersClassification.json create mode 100644 results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json create mode 100644 results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/MNIST.json create mode 100644 results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/NIGHTSI2IRetrieval.json create mode 100644 results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/OxfordFlowersClassification.json create mode 100644 results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json create mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/MNIST.json create mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/NIGHTSI2IRetrieval.json create mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/OxfordFlowersClassification.json create mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index 8bbfc8a3cb..e8222faf46 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -12,6 +12,7 @@ bm25, clip_models, cohere_models, + dino_models, e5_instruct, e5_models, e5_v, @@ -129,6 +130,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe bge_models, bm25, cohere_models, + dino_models, e5_instruct, e5_models, e5_v, diff --git a/mteb/models/dino_models.py b/mteb/models/dino_models.py new file mode 100644 index 0000000000..06f9ad8dce --- /dev/null +++ b/mteb/models/dino_models.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import AutoImageProcessor, AutoModel + +from mteb.model_meta import ModelMeta + + +class DINOModelWrapper: + """A wrapper class for DINO models that supports image encoding. + Text encoding and text-image fusion are not supported. + """ + + def __init__( + self, + model_name: str, + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + self.model = AutoModel.from_pretrained(model_name).to(self.device) + self.processor = AutoImageProcessor.from_pretrained(model_name) + + @staticmethod + def get_text_embeddings(texts: list[str], batch_size: int = 32): + raise ValueError("DINO models only support image encoding.") + + def get_image_embeddings( + self, + images: list[Image.Image] | DataLoader, + batch_size: int = 32, + pooling="cls", + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + with torch.no_grad(): + for batch in tqdm(images): + inputs = self.processor(images=batch, return_tensors="pt") + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model(**inputs) + features = image_outputs.last_hidden_state + if pooling == "cls": + features = features[:, 0, :] # TODO: confirm best practice + elif pooling == "mean": + features = features.mean(dim=1) + else: + raise ValueError( + "Pooling methods not implemented. Use cls or mean." + ) + all_image_embeddings.append(features.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + inputs = self.processor(images=batch_images, return_tensors="pt") + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model(**inputs) + features = image_outputs.last_hidden_state + if pooling == "cls": + features = features[:, 0, :] + elif pooling == "mean": + features = features.mean(dim=1) + else: + raise ValueError( + "Pooling methods not implemented. Use cls or mean." + ) + all_image_embeddings.append(features.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + @staticmethod + def calculate_probs(text_embeddings, image_embeddings): + raise ValueError("DINO models only support image encoding.") + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("images must be provided for DINO models") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + raise ValueError("DINO models only support image encoding.") + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + +dinov2_small = ModelMeta( + loader=partial( + DINOModelWrapper, + model_name="facebook/dinov2-small", + ), + name="facebook/dinov2-small", + languages=["eng_Latn"], + open_source=True, + revision="ed25f3a31f01632728cabb09d1542f84ab7b0056", + release_date="2023-07-18", +) + +dinov2_base = ModelMeta( + loader=partial( + DINOModelWrapper, + model_name="facebook/dinov2-base", + ), + name="facebook/dinov2-base", + languages=["eng_Latn"], + open_source=True, + revision="f9e44c814b77203eaa57a6bdbbd535f21ede1415", + release_date="2023-07-18", +) + +dinov2_large = ModelMeta( + loader=partial( + DINOModelWrapper, + model_name="facebook/dinov2-large", + ), + name="facebook/dinov2-large", + languages=["eng_Latn"], + open_source=True, + revision="47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", + release_date="2023-07-18", +) + +dinov2_giant = ModelMeta( + loader=partial( + DINOModelWrapper, + model_name="facebook/dinov2-giant", + ), + name="facebook/dinov2-giant", + languages=["eng_Latn"], + open_source=True, + revision="611a9d42f2335e0f921f1e313ad3c1b7178d206d", + release_date="2023-07-18", +) + +if __name__ == "__main__": + import mteb + + mdl = mteb.get_model(dinov2_base.name, dinov2_base.revision) diff --git a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py index 7351cf0f22..c64f28341e 100644 --- a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py @@ -13,6 +13,7 @@ class MNISTClassification(AbsTaskImageClassification): dataset={ "path": "ylecun/mnist", "revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "trust_remote_code": True, }, type="Classification", category="i2t", diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/MNIST.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/MNIST.json new file mode 100644 index 0000000000..1c4bf7a7c8 --- /dev/null +++ b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 77.37463116645813, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "accuracy": 0.86678, + "f1": 0.8645767524624768, + "f1_weighted": 0.8666851685943376, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.86678, + "scores_per_experiment": [ + { + "accuracy": 0.8689, + "f1": 0.8670967814543469, + "f1_weighted": 0.8697569866148404 + }, + { + "accuracy": 0.8349, + "f1": 0.8309818535418068, + "f1_weighted": 0.83417961551265 + }, + { + "accuracy": 0.8675, + "f1": 0.8665245386493894, + "f1_weighted": 0.8684789478397935 + }, + { + "accuracy": 0.8789, + "f1": 0.8764053509532486, + "f1_weighted": 0.877569373830515 + }, + { + "accuracy": 0.8837, + "f1": 0.8818752377135922, + "f1_weighted": 0.8834409191738892 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/NIGHTSI2IRetrieval.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/NIGHTSI2IRetrieval.json new file mode 100644 index 0000000000..ed5b188152 --- /dev/null +++ b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/NIGHTSI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", + "evaluation_time": 199.69204115867615, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.08491, + "cv_recall_at_10": 0.5033, + "cv_recall_at_100": 0.94811, + "cv_recall_at_1000": 0.99528, + "cv_recall_at_20": 0.73915, + "cv_recall_at_3": 0.21368, + "cv_recall_at_5": 0.32075, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.25976, + "map_at_1": 0.08491, + "map_at_10": 0.18614, + "map_at_100": 0.20832, + "map_at_1000": 0.20858, + "map_at_20": 0.20229, + "map_at_3": 0.13797, + "map_at_5": 0.16217, + "mrr_at_1": 0.08490566037735849, + "mrr_at_10": 0.18614180892482757, + "mrr_at_100": 0.20831945787163383, + "mrr_at_1000": 0.20858401988895778, + "mrr_at_20": 0.20229258313366166, + "mrr_at_3": 0.1379716981132077, + "mrr_at_5": 0.16216981132075434, + "nauc_cv_recall_at_1000_diff1": -0.9262371615312536, + "nauc_cv_recall_at_1000_max": 0.8307656395891835, + "nauc_cv_recall_at_1000_std": 0.9330065359477094, + "nauc_cv_recall_at_100_diff1": -0.47821492233256935, + "nauc_cv_recall_at_100_max": 0.653713606654785, + "nauc_cv_recall_at_100_std": 0.43830744418979667, + "nauc_cv_recall_at_10_diff1": -0.03606188090870156, + "nauc_cv_recall_at_10_max": 0.1143223995091099, + "nauc_cv_recall_at_10_std": -0.035964131725537925, + "nauc_cv_recall_at_1_diff1": 0.020150254896699814, + "nauc_cv_recall_at_1_max": 0.04705154577706238, + "nauc_cv_recall_at_1_std": -0.01978058015085107, + "nauc_cv_recall_at_20_diff1": -0.08100163008591106, + "nauc_cv_recall_at_20_max": 0.25923993548331703, + "nauc_cv_recall_at_20_std": -0.06323951198528052, + "nauc_cv_recall_at_3_diff1": 0.010570189395540861, + "nauc_cv_recall_at_3_max": 0.05478547707900785, + "nauc_cv_recall_at_3_std": -0.03527327833867772, + "nauc_cv_recall_at_5_diff1": -0.009090564990284886, + "nauc_cv_recall_at_5_max": 0.09606475056145755, + "nauc_cv_recall_at_5_std": -0.00945835120744882, + "nauc_map_at_1000_diff1": -0.006652494750020363, + "nauc_map_at_1000_max": 0.07794470071908056, + "nauc_map_at_1000_std": -0.021676746990813362, + "nauc_map_at_100_diff1": -0.007161878633089917, + "nauc_map_at_100_max": 0.07855793447874261, + "nauc_map_at_100_std": -0.021314555769920304, + "nauc_map_at_10_diff1": -0.0011759754726835372, + "nauc_map_at_10_max": 0.0688532273816496, + "nauc_map_at_10_std": -0.024844900739844897, + "nauc_map_at_1_diff1": 0.020150254896699814, + "nauc_map_at_1_max": 0.04705154577706238, + "nauc_map_at_1_std": -0.01978058015085107, + "nauc_map_at_20_diff1": -0.004673155845604268, + "nauc_map_at_20_max": 0.07787233394995861, + "nauc_map_at_20_std": -0.026666636294813202, + "nauc_map_at_3_diff1": 0.012874623252847255, + "nauc_map_at_3_max": 0.05289142275583296, + "nauc_map_at_3_std": -0.028444137731526885, + "nauc_map_at_5_diff1": 0.004486822990023081, + "nauc_map_at_5_max": 0.06691112758480544, + "nauc_map_at_5_std": -0.018957410356361237, + "nauc_mrr_at_1000_diff1": -0.006652494750020363, + "nauc_mrr_at_1000_max": 0.07794470071908056, + "nauc_mrr_at_1000_std": -0.021676746990813362, + "nauc_mrr_at_100_diff1": -0.007161878633089917, + "nauc_mrr_at_100_max": 0.07855793447874261, + "nauc_mrr_at_100_std": -0.021314555769920304, + "nauc_mrr_at_10_diff1": -0.0011759754726835372, + "nauc_mrr_at_10_max": 0.0688532273816496, + "nauc_mrr_at_10_std": -0.024844900739844897, + "nauc_mrr_at_1_diff1": 0.020150254896699814, + "nauc_mrr_at_1_max": 0.04705154577706238, + "nauc_mrr_at_1_std": -0.01978058015085107, + "nauc_mrr_at_20_diff1": -0.004673155845604268, + "nauc_mrr_at_20_max": 0.07787233394995861, + "nauc_mrr_at_20_std": -0.026666636294813202, + "nauc_mrr_at_3_diff1": 0.012874623252847255, + "nauc_mrr_at_3_max": 0.05289142275583296, + "nauc_mrr_at_3_std": -0.028444137731526885, + "nauc_mrr_at_5_diff1": 0.004486822990023081, + "nauc_mrr_at_5_max": 0.06691112758480544, + "nauc_mrr_at_5_std": -0.018957410356361237, + "nauc_ndcg_at_1000_diff1": -0.022411213599227923, + "nauc_ndcg_at_1000_max": 0.0986689095954933, + "nauc_ndcg_at_1000_std": -0.013359705039231978, + "nauc_ndcg_at_100_diff1": -0.0355840262798586, + "nauc_ndcg_at_100_max": 0.11558385385862016, + "nauc_ndcg_at_100_std": -0.0011908869944293231, + "nauc_ndcg_at_10_diff1": -0.012042479286208913, + "nauc_ndcg_at_10_max": 0.08275755290912527, + "nauc_ndcg_at_10_std": -0.0280393669746386, + "nauc_ndcg_at_1_diff1": 0.020150254896699814, + "nauc_ndcg_at_1_max": 0.04705154577706238, + "nauc_ndcg_at_1_std": -0.01978058015085107, + "nauc_ndcg_at_20_diff1": -0.022260903773348162, + "nauc_ndcg_at_20_max": 0.11601171113415618, + "nauc_ndcg_at_20_std": -0.03428564651751476, + "nauc_ndcg_at_3_diff1": 0.012064737993632191, + "nauc_ndcg_at_3_max": 0.05355497149851039, + "nauc_ndcg_at_3_std": -0.030649951361174387, + "nauc_ndcg_at_5_diff1": -0.00011617957032628835, + "nauc_ndcg_at_5_max": 0.07633348175009125, + "nauc_ndcg_at_5_std": -0.015978649311778212, + "nauc_precision_at_1000_diff1": -0.9262371615311449, + "nauc_precision_at_1000_max": 0.830765639589156, + "nauc_precision_at_1000_std": 0.933006535947729, + "nauc_precision_at_100_diff1": -0.47821492233257124, + "nauc_precision_at_100_max": 0.6537136066547907, + "nauc_precision_at_100_std": 0.4383074441897986, + "nauc_precision_at_10_diff1": -0.03606188090870162, + "nauc_precision_at_10_max": 0.11432239950910993, + "nauc_precision_at_10_std": -0.03596413172553808, + "nauc_precision_at_1_diff1": 0.020150254896699814, + "nauc_precision_at_1_max": 0.04705154577706238, + "nauc_precision_at_1_std": -0.01978058015085107, + "nauc_precision_at_20_diff1": -0.08100163008591092, + "nauc_precision_at_20_max": 0.2592399354833167, + "nauc_precision_at_20_std": -0.06323951198528094, + "nauc_precision_at_3_diff1": 0.010570189395540917, + "nauc_precision_at_3_max": 0.05478547707900796, + "nauc_precision_at_3_std": -0.03527327833867778, + "nauc_precision_at_5_diff1": -0.009090564990284862, + "nauc_precision_at_5_max": 0.0960647505614577, + "nauc_precision_at_5_std": -0.009458351207449148, + "nauc_recall_at_1000_diff1": -0.9262371615312536, + "nauc_recall_at_1000_max": 0.8307656395891835, + "nauc_recall_at_1000_std": 0.9330065359477094, + "nauc_recall_at_100_diff1": -0.47821492233256935, + "nauc_recall_at_100_max": 0.653713606654785, + "nauc_recall_at_100_std": 0.43830744418979667, + "nauc_recall_at_10_diff1": -0.03606188090870156, + "nauc_recall_at_10_max": 0.1143223995091099, + "nauc_recall_at_10_std": -0.035964131725537925, + "nauc_recall_at_1_diff1": 0.020150254896699814, + "nauc_recall_at_1_max": 0.04705154577706238, + "nauc_recall_at_1_std": -0.01978058015085107, + "nauc_recall_at_20_diff1": -0.08100163008591106, + "nauc_recall_at_20_max": 0.25923993548331703, + "nauc_recall_at_20_std": -0.06323951198528052, + "nauc_recall_at_3_diff1": 0.010570189395540861, + "nauc_recall_at_3_max": 0.05478547707900785, + "nauc_recall_at_3_std": -0.03527327833867772, + "nauc_recall_at_5_diff1": -0.009090564990284886, + "nauc_recall_at_5_max": 0.09606475056145755, + "nauc_recall_at_5_std": -0.00945835120744882, + "ndcg_at_1": 0.08491, + "ndcg_at_10": 0.25976, + "ndcg_at_100": 0.35919, + "ndcg_at_1000": 0.3654, + "ndcg_at_20": 0.31912, + "ndcg_at_3": 0.15726, + "ndcg_at_5": 0.20112, + "precision_at_1": 0.08491, + "precision_at_10": 0.05033, + "precision_at_100": 0.00948, + "precision_at_1000": 0.001, + "precision_at_20": 0.03696, + "precision_at_3": 0.07123, + "precision_at_5": 0.06415, + "recall_at_1": 0.08491, + "recall_at_10": 0.5033, + "recall_at_100": 0.94811, + "recall_at_1000": 0.99528, + "recall_at_20": 0.73915, + "recall_at_3": 0.21368, + "recall_at_5": 0.32075 + } + ] + }, + "task_name": "NIGHTSI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/OxfordFlowersClassification.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/OxfordFlowersClassification.json new file mode 100644 index 0000000000..d02203db75 --- /dev/null +++ b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/OxfordFlowersClassification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + "evaluation_time": 176.6004662513733, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "accuracy": 0.9950980392156863, + "f1": 0.9951496593312846, + "f1_weighted": 0.9950563841695577, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9950980392156863, + "scores_per_experiment": [ + { + "accuracy": 0.9950980392156863, + "f1": 0.9951492931717489, + "f1_weighted": 0.9950790860607716 + }, + { + "accuracy": 0.996078431372549, + "f1": 0.9961903907253213, + "f1_weighted": 0.9960625134874698 + }, + { + "accuracy": 0.9950980392156863, + "f1": 0.9951492931717489, + "f1_weighted": 0.9950790860607716 + }, + { + "accuracy": 0.9941176470588236, + "f1": 0.9941091110170156, + "f1_weighted": 0.9940389039060383 + }, + { + "accuracy": 0.9950980392156863, + "f1": 0.9951502085705881, + "f1_weighted": 0.9950223313327365 + } + ] + } + ] + }, + "task_name": "OxfordFlowersClassification" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json new file mode 100644 index 0000000000..6ae5f937c4 --- /dev/null +++ b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json @@ -0,0 +1 @@ +{"name": "facebook/dinov2-base", "revision": "f9e44c814b77203eaa57a6bdbbd535f21ede1415", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/MNIST.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/MNIST.json new file mode 100644 index 0000000000..8327e4aa22 --- /dev/null +++ b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 158.008056640625, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "accuracy": 0.8246600000000001, + "f1": 0.8221838432987795, + "f1_weighted": 0.8249356935234966, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8246600000000001, + "scores_per_experiment": [ + { + "accuracy": 0.8232, + "f1": 0.8218126192019535, + "f1_weighted": 0.8247912885329193 + }, + { + "accuracy": 0.8027, + "f1": 0.7986794416362917, + "f1_weighted": 0.8019845617911158 + }, + { + "accuracy": 0.8306, + "f1": 0.8297001183400369, + "f1_weighted": 0.8325294903919632 + }, + { + "accuracy": 0.839, + "f1": 0.8351043070584623, + "f1_weighted": 0.8372439745097275 + }, + { + "accuracy": 0.8278, + "f1": 0.8256227302571535, + "f1_weighted": 0.8281291523917571 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/NIGHTSI2IRetrieval.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/NIGHTSI2IRetrieval.json new file mode 100644 index 0000000000..e389cfeb4a --- /dev/null +++ b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/NIGHTSI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", + "evaluation_time": 508.99595761299133, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.07689, + "cv_recall_at_10": 0.48491, + "cv_recall_at_100": 0.94387, + "cv_recall_at_1000": 0.99575, + "cv_recall_at_20": 0.71698, + "cv_recall_at_3": 0.20142, + "cv_recall_at_5": 0.29528, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.24754, + "map_at_1": 0.07689, + "map_at_10": 0.17599, + "map_at_100": 0.19935, + "map_at_1000": 0.19966, + "map_at_20": 0.19239, + "map_at_3": 0.1298, + "map_at_5": 0.15107, + "mrr_at_1": 0.07688679245283019, + "mrr_at_10": 0.17599225067385418, + "mrr_at_100": 0.19935156300390885, + "mrr_at_1000": 0.19965908590632203, + "mrr_at_20": 0.19239222583377735, + "mrr_at_3": 0.12979559748427677, + "mrr_at_5": 0.15106918238993675, + "nauc_cv_recall_at_1000_diff1": 0.3851540616246801, + "nauc_cv_recall_at_1000_max": 0.8244631185807924, + "nauc_cv_recall_at_1000_std": 0.925562817719677, + "nauc_cv_recall_at_100_diff1": -0.4385715070341862, + "nauc_cv_recall_at_100_max": 0.6730613814153109, + "nauc_cv_recall_at_100_std": 0.45218479548682355, + "nauc_cv_recall_at_10_diff1": 0.012871715486821715, + "nauc_cv_recall_at_10_max": 0.08577496630146382, + "nauc_cv_recall_at_10_std": -0.08241386554096461, + "nauc_cv_recall_at_1_diff1": 0.08460726813618427, + "nauc_cv_recall_at_1_max": -0.0065678904232255786, + "nauc_cv_recall_at_1_std": -0.08707639987423882, + "nauc_cv_recall_at_20_diff1": -0.054615919794490805, + "nauc_cv_recall_at_20_max": 0.2859854431283007, + "nauc_cv_recall_at_20_std": -0.059189025260453935, + "nauc_cv_recall_at_3_diff1": 0.059515518902995126, + "nauc_cv_recall_at_3_max": 0.05847203984965998, + "nauc_cv_recall_at_3_std": -0.03672985512045184, + "nauc_cv_recall_at_5_diff1": 0.026571959649107007, + "nauc_cv_recall_at_5_max": 0.06516748857696698, + "nauc_cv_recall_at_5_std": -0.045787832815866014, + "nauc_map_at_1000_diff1": 0.047321574512003584, + "nauc_map_at_1000_max": 0.05438206812877677, + "nauc_map_at_1000_std": -0.05744428851185208, + "nauc_map_at_100_diff1": 0.04650275080611146, + "nauc_map_at_100_max": 0.05516022097644162, + "nauc_map_at_100_std": -0.0569522165815035, + "nauc_map_at_10_diff1": 0.05173169037370162, + "nauc_map_at_10_max": 0.043084012251647076, + "nauc_map_at_10_std": -0.06272816529553181, + "nauc_map_at_1_diff1": 0.08460726813618427, + "nauc_map_at_1_max": -0.0065678904232255786, + "nauc_map_at_1_std": -0.08707639987423882, + "nauc_map_at_20_diff1": 0.04752612557302544, + "nauc_map_at_20_max": 0.057067886386831156, + "nauc_map_at_20_std": -0.06083000950759207, + "nauc_map_at_3_diff1": 0.06836493519828814, + "nauc_map_at_3_max": 0.033865120590951175, + "nauc_map_at_3_std": -0.05438069275392832, + "nauc_map_at_5_diff1": 0.05660743085479349, + "nauc_map_at_5_max": 0.03746650010933686, + "nauc_map_at_5_std": -0.05718514991261317, + "nauc_mrr_at_1000_diff1": 0.047321574512003584, + "nauc_mrr_at_1000_max": 0.05438206812877677, + "nauc_mrr_at_1000_std": -0.05744428851185208, + "nauc_mrr_at_100_diff1": 0.04650275080611146, + "nauc_mrr_at_100_max": 0.05516022097644162, + "nauc_mrr_at_100_std": -0.0569522165815035, + "nauc_mrr_at_10_diff1": 0.05173169037370162, + "nauc_mrr_at_10_max": 0.043084012251647076, + "nauc_mrr_at_10_std": -0.06272816529553181, + "nauc_mrr_at_1_diff1": 0.08460726813618427, + "nauc_mrr_at_1_max": -0.0065678904232255786, + "nauc_mrr_at_1_std": -0.08707639987423882, + "nauc_mrr_at_20_diff1": 0.04752612557302544, + "nauc_mrr_at_20_max": 0.057067886386831156, + "nauc_mrr_at_20_std": -0.06083000950759207, + "nauc_mrr_at_3_diff1": 0.06836493519828814, + "nauc_mrr_at_3_max": 0.033865120590951175, + "nauc_mrr_at_3_std": -0.05438069275392832, + "nauc_mrr_at_5_diff1": 0.05660743085479349, + "nauc_mrr_at_5_max": 0.03746650010933686, + "nauc_mrr_at_5_std": -0.05718514991261317, + "nauc_ndcg_at_1000_diff1": 0.03410252328119208, + "nauc_ndcg_at_1000_max": 0.07841621563109125, + "nauc_ndcg_at_1000_std": -0.046895103992041366, + "nauc_ndcg_at_100_diff1": 0.014228107355737629, + "nauc_ndcg_at_100_max": 0.09921968541645529, + "nauc_ndcg_at_100_std": -0.031053659815862337, + "nauc_ndcg_at_10_diff1": 0.03861706367387873, + "nauc_ndcg_at_10_max": 0.057559536819589835, + "nauc_ndcg_at_10_std": -0.06759475458350936, + "nauc_ndcg_at_1_diff1": 0.08460726813618427, + "nauc_ndcg_at_1_max": -0.0065678904232255786, + "nauc_ndcg_at_1_std": -0.08707639987423882, + "nauc_ndcg_at_20_diff1": 0.02278919178762813, + "nauc_ndcg_at_20_max": 0.10903022698508856, + "nauc_ndcg_at_20_std": -0.06014408602190423, + "nauc_ndcg_at_3_diff1": 0.06540639729834469, + "nauc_ndcg_at_3_max": 0.04202468283776928, + "nauc_ndcg_at_3_std": -0.048475424580323556, + "nauc_ndcg_at_5_diff1": 0.04659329897322925, + "nauc_ndcg_at_5_max": 0.04714149856887462, + "nauc_ndcg_at_5_std": -0.0530008861423805, + "nauc_precision_at_1000_diff1": 0.3851540616246652, + "nauc_precision_at_1000_max": 0.8244631185807481, + "nauc_precision_at_1000_std": 0.925562817719698, + "nauc_precision_at_100_diff1": -0.43857150703418407, + "nauc_precision_at_100_max": 0.6730613814153111, + "nauc_precision_at_100_std": 0.4521847954868242, + "nauc_precision_at_10_diff1": 0.0128717154868219, + "nauc_precision_at_10_max": 0.08577496630146376, + "nauc_precision_at_10_std": -0.08241386554096451, + "nauc_precision_at_1_diff1": 0.08460726813618427, + "nauc_precision_at_1_max": -0.0065678904232255786, + "nauc_precision_at_1_std": -0.08707639987423882, + "nauc_precision_at_20_diff1": -0.05461591979449159, + "nauc_precision_at_20_max": 0.2859854431282994, + "nauc_precision_at_20_std": -0.0591890252604538, + "nauc_precision_at_3_diff1": 0.05951551890299547, + "nauc_precision_at_3_max": 0.05847203984966021, + "nauc_precision_at_3_std": -0.036729855120451856, + "nauc_precision_at_5_diff1": 0.026571959649107142, + "nauc_precision_at_5_max": 0.06516748857696697, + "nauc_precision_at_5_std": -0.04578783281586591, + "nauc_recall_at_1000_diff1": 0.3851540616246801, + "nauc_recall_at_1000_max": 0.8244631185807924, + "nauc_recall_at_1000_std": 0.925562817719677, + "nauc_recall_at_100_diff1": -0.4385715070341862, + "nauc_recall_at_100_max": 0.6730613814153109, + "nauc_recall_at_100_std": 0.45218479548682355, + "nauc_recall_at_10_diff1": 0.012871715486821715, + "nauc_recall_at_10_max": 0.08577496630146382, + "nauc_recall_at_10_std": -0.08241386554096461, + "nauc_recall_at_1_diff1": 0.08460726813618427, + "nauc_recall_at_1_max": -0.0065678904232255786, + "nauc_recall_at_1_std": -0.08707639987423882, + "nauc_recall_at_20_diff1": -0.054615919794490805, + "nauc_recall_at_20_max": 0.2859854431283007, + "nauc_recall_at_20_std": -0.059189025260453935, + "nauc_recall_at_3_diff1": 0.059515518902995126, + "nauc_recall_at_3_max": 0.05847203984965998, + "nauc_recall_at_3_std": -0.03672985512045184, + "nauc_recall_at_5_diff1": 0.026571959649107007, + "nauc_recall_at_5_max": 0.06516748857696698, + "nauc_recall_at_5_std": -0.045787832815866014, + "ndcg_at_1": 0.07689, + "ndcg_at_10": 0.24754, + "ndcg_at_100": 0.35089, + "ndcg_at_1000": 0.35778, + "ndcg_at_20": 0.30659, + "ndcg_at_3": 0.14811, + "ndcg_at_5": 0.18661, + "precision_at_1": 0.07689, + "precision_at_10": 0.04849, + "precision_at_100": 0.00944, + "precision_at_1000": 0.001, + "precision_at_20": 0.03585, + "precision_at_3": 0.06714, + "precision_at_5": 0.05906, + "recall_at_1": 0.07689, + "recall_at_10": 0.48491, + "recall_at_100": 0.94387, + "recall_at_1000": 0.99575, + "recall_at_20": 0.71698, + "recall_at_3": 0.20142, + "recall_at_5": 0.29528 + } + ] + }, + "task_name": "NIGHTSI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/OxfordFlowersClassification.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/OxfordFlowersClassification.json new file mode 100644 index 0000000000..918613b671 --- /dev/null +++ b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/OxfordFlowersClassification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + "evaluation_time": 242.11567449569702, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "accuracy": 0.996078431372549, + "f1": 0.9960915465032875, + "f1_weighted": 0.9960639651382607, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.996078431372549, + "scores_per_experiment": [ + { + "accuracy": 0.996078431372549, + "f1": 0.9960915465032875, + "f1_weighted": 0.9960639651382607 + }, + { + "accuracy": 0.996078431372549, + "f1": 0.9960915465032875, + "f1_weighted": 0.9960639651382607 + }, + { + "accuracy": 0.996078431372549, + "f1": 0.9960915465032875, + "f1_weighted": 0.9960639651382607 + }, + { + "accuracy": 0.996078431372549, + "f1": 0.9960915465032875, + "f1_weighted": 0.9960639651382607 + }, + { + "accuracy": 0.996078431372549, + "f1": 0.9960915465032875, + "f1_weighted": 0.9960639651382607 + } + ] + } + ] + }, + "task_name": "OxfordFlowersClassification" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json new file mode 100644 index 0000000000..1de6c7be30 --- /dev/null +++ b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json @@ -0,0 +1 @@ +{"name": "facebook/dinov2-large", "revision": "47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/MNIST.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/MNIST.json new file mode 100644 index 0000000000..2a1a442fe1 --- /dev/null +++ b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 53.76097369194031, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "accuracy": 0.8656600000000001, + "f1": 0.8637301373219758, + "f1_weighted": 0.8657989045644241, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8656600000000001, + "scores_per_experiment": [ + { + "accuracy": 0.8746, + "f1": 0.873539621523118, + "f1_weighted": 0.8756435923724779 + }, + { + "accuracy": 0.8274, + "f1": 0.8235176307759297, + "f1_weighted": 0.8265648513841878 + }, + { + "accuracy": 0.8719, + "f1": 0.8699976650278503, + "f1_weighted": 0.8715310177539896 + }, + { + "accuracy": 0.8787, + "f1": 0.8776085155251211, + "f1_weighted": 0.8795306016429042 + }, + { + "accuracy": 0.8757, + "f1": 0.8739872537578602, + "f1_weighted": 0.8757244596685608 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/NIGHTSI2IRetrieval.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/NIGHTSI2IRetrieval.json new file mode 100644 index 0000000000..d589588bc8 --- /dev/null +++ b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/NIGHTSI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", + "evaluation_time": 108.04806113243103, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.08208, + "cv_recall_at_10": 0.50566, + "cv_recall_at_100": 0.94104, + "cv_recall_at_1000": 0.99623, + "cv_recall_at_20": 0.71557, + "cv_recall_at_3": 0.21085, + "cv_recall_at_5": 0.31321, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.25876, + "map_at_1": 0.08208, + "map_at_10": 0.18424, + "map_at_100": 0.20549, + "map_at_1000": 0.20578, + "map_at_20": 0.19902, + "map_at_3": 0.13569, + "map_at_5": 0.15878, + "mrr_at_1": 0.0820754716981132, + "mrr_at_10": 0.1842383572926024, + "mrr_at_100": 0.20549248220385244, + "mrr_at_1000": 0.2057831548741384, + "mrr_at_20": 0.19901605762390864, + "mrr_at_3": 0.13569182389937115, + "mrr_at_5": 0.15878144654088, + "nauc_cv_recall_at_1000_diff1": -0.16304855275446334, + "nauc_cv_recall_at_1000_max": 0.7718837535013962, + "nauc_cv_recall_at_1000_std": 0.9489379084967337, + "nauc_cv_recall_at_100_diff1": -0.32002987861811505, + "nauc_cv_recall_at_100_max": 0.6210308123249305, + "nauc_cv_recall_at_100_std": 0.412720821662, + "nauc_cv_recall_at_10_diff1": -0.013857139967228161, + "nauc_cv_recall_at_10_max": 0.10358654643659096, + "nauc_cv_recall_at_10_std": -0.051303658487245135, + "nauc_cv_recall_at_1_diff1": 0.02898698222662205, + "nauc_cv_recall_at_1_max": 0.042667826269317664, + "nauc_cv_recall_at_1_std": -0.03785054078809807, + "nauc_cv_recall_at_20_diff1": -0.08321837405985853, + "nauc_cv_recall_at_20_max": 0.190030565990472, + "nauc_cv_recall_at_20_std": -0.03864319335986944, + "nauc_cv_recall_at_3_diff1": -0.02082871885741471, + "nauc_cv_recall_at_3_max": 0.05479275316599656, + "nauc_cv_recall_at_3_std": -0.04492840197809061, + "nauc_cv_recall_at_5_diff1": -0.017987852753899084, + "nauc_cv_recall_at_5_max": 0.058657445633728725, + "nauc_cv_recall_at_5_std": -0.05043285328578423, + "nauc_map_at_1000_diff1": -0.006510507398643199, + "nauc_map_at_1000_max": 0.0721993746295309, + "nauc_map_at_1000_std": -0.036777170133303096, + "nauc_map_at_100_diff1": -0.007006312689406556, + "nauc_map_at_100_max": 0.07287810601748373, + "nauc_map_at_100_std": -0.03630568232615193, + "nauc_map_at_10_diff1": -0.001302596447271446, + "nauc_map_at_10_max": 0.06448303401621405, + "nauc_map_at_10_std": -0.04200841918198155, + "nauc_map_at_1_diff1": 0.02898698222662205, + "nauc_map_at_1_max": 0.042667826269317664, + "nauc_map_at_1_std": -0.03785054078809807, + "nauc_map_at_20_diff1": -0.006772172318400011, + "nauc_map_at_20_max": 0.070340640864153, + "nauc_map_at_20_std": -0.04049217683866346, + "nauc_map_at_3_diff1": -0.0007283763921229228, + "nauc_map_at_3_max": 0.05298823394700488, + "nauc_map_at_3_std": -0.04169663282302213, + "nauc_map_at_5_diff1": -0.001801122555208517, + "nauc_map_at_5_max": 0.05479514191945225, + "nauc_map_at_5_std": -0.04264537947699984, + "nauc_mrr_at_1000_diff1": -0.006510507398643199, + "nauc_mrr_at_1000_max": 0.0721993746295309, + "nauc_mrr_at_1000_std": -0.036777170133303096, + "nauc_mrr_at_100_diff1": -0.007006312689406556, + "nauc_mrr_at_100_max": 0.07287810601748373, + "nauc_mrr_at_100_std": -0.03630568232615193, + "nauc_mrr_at_10_diff1": -0.001302596447271446, + "nauc_mrr_at_10_max": 0.06448303401621405, + "nauc_mrr_at_10_std": -0.04200841918198155, + "nauc_mrr_at_1_diff1": 0.02898698222662205, + "nauc_mrr_at_1_max": 0.042667826269317664, + "nauc_mrr_at_1_std": -0.03785054078809807, + "nauc_mrr_at_20_diff1": -0.006772172318400011, + "nauc_mrr_at_20_max": 0.070340640864153, + "nauc_mrr_at_20_std": -0.04049217683866346, + "nauc_mrr_at_3_diff1": -0.0007283763921229228, + "nauc_mrr_at_3_max": 0.05298823394700488, + "nauc_mrr_at_3_std": -0.04169663282302213, + "nauc_mrr_at_5_diff1": -0.001801122555208517, + "nauc_mrr_at_5_max": 0.05479514191945225, + "nauc_mrr_at_5_std": -0.04264537947699984, + "nauc_ndcg_at_1000_diff1": -0.016298224329016942, + "nauc_ndcg_at_1000_max": 0.09062695649942037, + "nauc_ndcg_at_1000_std": -0.02731715426532132, + "nauc_ndcg_at_100_diff1": -0.028264767453912674, + "nauc_ndcg_at_100_max": 0.10964458205541326, + "nauc_ndcg_at_100_std": -0.012838091552479945, + "nauc_ndcg_at_10_diff1": -0.006124175666309968, + "nauc_ndcg_at_10_max": 0.07645936667113105, + "nauc_ndcg_at_10_std": -0.04477882894615783, + "nauc_ndcg_at_1_diff1": 0.02898698222662205, + "nauc_ndcg_at_1_max": 0.042667826269317664, + "nauc_ndcg_at_1_std": -0.03785054078809807, + "nauc_ndcg_at_20_diff1": -0.02473085630921952, + "nauc_ndcg_at_20_max": 0.09743286781620096, + "nauc_ndcg_at_20_std": -0.040343009950076295, + "nauc_ndcg_at_3_diff1": -0.007308502881026242, + "nauc_ndcg_at_3_max": 0.05369162294449823, + "nauc_ndcg_at_3_std": -0.04274088343527608, + "nauc_ndcg_at_5_diff1": -0.007620898518130469, + "nauc_ndcg_at_5_max": 0.056222806231225546, + "nauc_ndcg_at_5_std": -0.04512864978013532, + "nauc_precision_at_1000_diff1": -0.16304855275437521, + "nauc_precision_at_1000_max": 0.771883753501379, + "nauc_precision_at_1000_std": 0.9489379084967005, + "nauc_precision_at_100_diff1": -0.3200298786181148, + "nauc_precision_at_100_max": 0.6210308123249351, + "nauc_precision_at_100_std": 0.4127208216619996, + "nauc_precision_at_10_diff1": -0.013857139967228201, + "nauc_precision_at_10_max": 0.10358654643659095, + "nauc_precision_at_10_std": -0.05130365848724515, + "nauc_precision_at_1_diff1": 0.02898698222662205, + "nauc_precision_at_1_max": 0.042667826269317664, + "nauc_precision_at_1_std": -0.03785054078809807, + "nauc_precision_at_20_diff1": -0.08321837405985895, + "nauc_precision_at_20_max": 0.19003056599047163, + "nauc_precision_at_20_std": -0.03864319335986984, + "nauc_precision_at_3_diff1": -0.020828718857414826, + "nauc_precision_at_3_max": 0.05479275316599652, + "nauc_precision_at_3_std": -0.04492840197809076, + "nauc_precision_at_5_diff1": -0.01798785275389913, + "nauc_precision_at_5_max": 0.05865744563372872, + "nauc_precision_at_5_std": -0.05043285328578428, + "nauc_recall_at_1000_diff1": -0.16304855275446334, + "nauc_recall_at_1000_max": 0.7718837535013962, + "nauc_recall_at_1000_std": 0.9489379084967337, + "nauc_recall_at_100_diff1": -0.32002987861811505, + "nauc_recall_at_100_max": 0.6210308123249305, + "nauc_recall_at_100_std": 0.412720821662, + "nauc_recall_at_10_diff1": -0.013857139967228161, + "nauc_recall_at_10_max": 0.10358654643659096, + "nauc_recall_at_10_std": -0.051303658487245135, + "nauc_recall_at_1_diff1": 0.02898698222662205, + "nauc_recall_at_1_max": 0.042667826269317664, + "nauc_recall_at_1_std": -0.03785054078809807, + "nauc_recall_at_20_diff1": -0.08321837405985853, + "nauc_recall_at_20_max": 0.190030565990472, + "nauc_recall_at_20_std": -0.03864319335986944, + "nauc_recall_at_3_diff1": -0.02082871885741471, + "nauc_recall_at_3_max": 0.05479275316599656, + "nauc_recall_at_3_std": -0.04492840197809061, + "nauc_recall_at_5_diff1": -0.017987852753899084, + "nauc_recall_at_5_max": 0.058657445633728725, + "nauc_recall_at_5_std": -0.05043285328578423, + "ndcg_at_1": 0.08208, + "ndcg_at_10": 0.25876, + "ndcg_at_100": 0.3553, + "ndcg_at_1000": 0.36245, + "ndcg_at_20": 0.3121, + "ndcg_at_3": 0.15486, + "ndcg_at_5": 0.19675, + "precision_at_1": 0.08208, + "precision_at_10": 0.05057, + "precision_at_100": 0.00941, + "precision_at_1000": 0.001, + "precision_at_20": 0.03578, + "precision_at_3": 0.07028, + "precision_at_5": 0.06264, + "recall_at_1": 0.08208, + "recall_at_10": 0.50566, + "recall_at_100": 0.94104, + "recall_at_1000": 0.99623, + "recall_at_20": 0.71557, + "recall_at_3": 0.21085, + "recall_at_5": 0.31321 + } + ] + }, + "task_name": "NIGHTSI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/OxfordFlowersClassification.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/OxfordFlowersClassification.json new file mode 100644 index 0000000000..546e17103a --- /dev/null +++ b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/OxfordFlowersClassification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + "evaluation_time": 154.27621984481812, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "accuracy": 0.9950980392156863, + "f1": 0.9951626579948023, + "f1_weighted": 0.9950347807569507, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9950980392156863, + "scores_per_experiment": [ + { + "accuracy": 0.9950980392156863, + "f1": 0.9951502085705881, + "f1_weighted": 0.9950223313327365 + }, + { + "accuracy": 0.996078431372549, + "f1": 0.9961903907253213, + "f1_weighted": 0.9960625134874698 + }, + { + "accuracy": 0.9941176470588236, + "f1": 0.9941722735369257, + "f1_weighted": 0.9940443962990739 + }, + { + "accuracy": 0.9950980392156863, + "f1": 0.9951502085705881, + "f1_weighted": 0.9950223313327365 + }, + { + "accuracy": 0.9950980392156863, + "f1": 0.9951502085705881, + "f1_weighted": 0.9950223313327365 + } + ] + } + ] + }, + "task_name": "OxfordFlowersClassification" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json new file mode 100644 index 0000000000..2eca9d0047 --- /dev/null +++ b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json @@ -0,0 +1 @@ +{"name": "facebook/dinov2-small", "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"} \ No newline at end of file From 8a274e2ae0463073191dde9cc168a2903fad1aa8 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 26 Aug 2024 20:17:20 +0100 Subject: [PATCH 055/154] cls evaluator channel bug fix --- .../Image/ClassificationEvaluator.py | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py index d62af4d17a..eaa6416dc7 100644 --- a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py @@ -5,6 +5,7 @@ import numpy as np import torch +from datasets import Dataset from sklearn.linear_model import LogisticRegression from sklearn.metrics import ( accuracy_score, @@ -74,6 +75,11 @@ def __init__( self.images_train = images_train self.y_train = y_train + self.dataset_train = ImageDataset( + Dataset.from_dict({"image": images_train, "label": y_train}), + image_column_name=image_column_name, + transform=transform, + ) self.dataset_test = ImageDataset( dataset_test, image_column_name=image_column_name, transform=transform ) @@ -91,8 +97,15 @@ def __call__(self, model, test_cache=None): max_accuracy = 0 max_f1 = 0 max_ap = 0 + dataloader_train = DataLoader( + self.dataset_train, + batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=16, + ) X_train = model.get_image_embeddings( - self.images_train, batch_size=self.encode_kwargs["batch_size"] + dataloader_train, batch_size=self.encode_kwargs["batch_size"] ) dataloader = DataLoader( self.dataset_test, @@ -150,6 +163,11 @@ def __init__( dataset_test = dataset_test[:limit] self.images_train = images_train + self.dataset_train = ImageDataset( + Dataset.from_dict({"image": images_train, "label": y_train}), + image_column_name=image_column_name, + transform=transform, + ) self.y_train = y_train self.dataset_test = ImageDataset( dataset_test, image_column_name=image_column_name, transform=transform @@ -168,8 +186,16 @@ def __call__(self, model: Encoder, test_cache=None): max_accuracy = 0 max_f1 = 0 max_ap = 0 + + dataloader_train = DataLoader( + self.dataset_train, + batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=16, + ) X_train = model.get_image_embeddings( - self.images_train, batch_size=self.encode_kwargs["batch_size"] + dataloader_train, batch_size=self.encode_kwargs["batch_size"] ) dataloader = DataLoader( @@ -309,6 +335,11 @@ def __init__( self.images_train = images_train self.y_train = y_train + self.dataset_train = ImageDataset( + Dataset.from_dict({"image": images_train, "label": y_train}), + image_column_name=image_column_name, + transform=transform, + ) self.dataset_test = ImageDataset( dataset_test, image_column_name=image_column_name, transform=transform ) @@ -325,8 +356,15 @@ def __call__(self, model, test_cache=None): max_iter=self.max_iter, verbose=1 if logger.isEnabledFor(logging.DEBUG) else 0, ) + dataloader_train = DataLoader( + self.dataset_train, + batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=16, + ) X_train = model.get_image_embeddings( - self.images_train, batch_size=self.encode_kwargs["batch_size"] + dataloader_train, batch_size=self.encode_kwargs["batch_size"] ) dataloader = DataLoader( self.dataset_test, From 65da41c180246b85e83fc41ee17ad78ef54f0891 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 26 Aug 2024 20:17:59 +0100 Subject: [PATCH 056/154] add ALIGN model --- mteb/models/__init__.py | 2 + mteb/models/align_models.py | 131 ++++++++++++ .../MNIST.json | 48 +++++ .../MNISTZeroShot.json | 19 ++ .../NIGHTSI2IRetrieval.json | 186 ++++++++++++++++++ .../OxfordFlowersClassification.json | 48 +++++ .../RenderedSST2.json | 19 ++ .../model_meta.json | 1 + 8 files changed, 454 insertions(+) create mode 100644 mteb/models/align_models.py create mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNIST.json create mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNISTZeroShot.json create mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/NIGHTSI2IRetrieval.json create mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/OxfordFlowersClassification.json create mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/RenderedSST2.json create mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/model_meta.json diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index e8222faf46..8e96542925 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -8,6 +8,7 @@ from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode from mteb.model_meta import ModelMeta from mteb.models import ( + align_models, bge_models, bm25, clip_models, @@ -127,6 +128,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe model_modules = [ + align_models, bge_models, bm25, cohere_models, diff --git a/mteb/models/align_models.py b/mteb/models/align_models.py new file mode 100644 index 0000000000..a47e199209 --- /dev/null +++ b/mteb/models/align_models.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import AutoModel, AutoProcessor + +from mteb.model_meta import ModelMeta + + +class ALIGNModelWrapper: + def __init__( + self, + model_name: str, + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + self.model = AutoModel.from_pretrained(model_name).to(self.device) + self.processor = AutoProcessor.from_pretrained(model_name) + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + inputs = self.processor( + text=batch_texts, return_tensors="pt", padding=True, truncation=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + text_outputs = self.model.get_text_features(**inputs) + all_text_embeddings.append(text_outputs.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + all_image_embeddings = [] + if isinstance(images, DataLoader): + with torch.no_grad(): + for batch in tqdm(images): + inputs = self.processor( + images=batch, return_tensors="pt", padding=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model.get_image_features(**inputs) + all_image_embeddings.append(image_outputs.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + inputs = self.processor( + images=batch_images, return_tensors="pt", padding=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model.get_image_features(**inputs) + all_image_embeddings.append(image_outputs.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + else: + # to do: add other fusion mode + raise ValueError(f"fusion mode {fusion_mode} hasn't been implemented") + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + +align_base = ModelMeta( + loader=partial( + ALIGNModelWrapper, + model_name="kakaobrain/align-base", + ), + name="kakaobrain/align-base", + languages=["eng_Latn"], + open_source=True, + revision="e96a37facc7b1f59090ece82293226b817afd6ba", + release_date="2023-02-24", +) + +if __name__ == "__main__": + import mteb + + mdl = mteb.get_model(align_base.name, align_base.revision) + emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNIST.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNIST.json new file mode 100644 index 0000000000..fdb32990a8 --- /dev/null +++ b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 88.12602519989014, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "accuracy": 0.8676, + "f1": 0.8647519484598748, + "f1_weighted": 0.8674255847302279, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8676, + "scores_per_experiment": [ + { + "accuracy": 0.8841, + "f1": 0.8822100989512647, + "f1_weighted": 0.8842956096110661 + }, + { + "accuracy": 0.8418, + "f1": 0.8348510573099901, + "f1_weighted": 0.8395228217754684 + }, + { + "accuracy": 0.8695, + "f1": 0.8673706569160476, + "f1_weighted": 0.8697790017131949 + }, + { + "accuracy": 0.8837, + "f1": 0.881759695651958, + "f1_weighted": 0.8836890214883635 + }, + { + "accuracy": 0.8589, + "f1": 0.8575682334701138, + "f1_weighted": 0.8598414690630468 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNISTZeroShot.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNISTZeroShot.json new file mode 100644 index 0000000000..d1cc3bbdbe --- /dev/null +++ b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNISTZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 52.55812096595764, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "accuracy": 0.3773, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.3773 + } + ] + }, + "task_name": "MNISTZeroShot" +} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/NIGHTSI2IRetrieval.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/NIGHTSI2IRetrieval.json new file mode 100644 index 0000000000..2a5b209388 --- /dev/null +++ b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/NIGHTSI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", + "evaluation_time": 239.57693338394165, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.07406, + "cv_recall_at_10": 0.45, + "cv_recall_at_100": 0.91651, + "cv_recall_at_1000": 0.98396, + "cv_recall_at_20": 0.66462, + "cv_recall_at_3": 0.18632, + "cv_recall_at_5": 0.27877, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.23151, + "map_at_1": 0.07406, + "map_at_10": 0.16568, + "map_at_100": 0.18777, + "map_at_1000": 0.1881, + "map_at_20": 0.18052, + "map_at_3": 0.12241, + "map_at_5": 0.14358, + "mrr_at_1": 0.0740566037735849, + "mrr_at_10": 0.16568134171907734, + "mrr_at_100": 0.18776707411027074, + "mrr_at_1000": 0.18810404013437873, + "mrr_at_20": 0.1805234731481328, + "mrr_at_3": 0.12240566037735866, + "mrr_at_5": 0.14358490566037718, + "nauc_cv_recall_at_1000_diff1": -0.41488163893008945, + "nauc_cv_recall_at_1000_max": 0.8024386225078316, + "nauc_cv_recall_at_1000_std": 0.6085296863843472, + "nauc_cv_recall_at_100_diff1": -0.1561532334214285, + "nauc_cv_recall_at_100_max": 0.3230256320984146, + "nauc_cv_recall_at_100_std": 0.42613429552612103, + "nauc_cv_recall_at_10_diff1": 0.035585056376463996, + "nauc_cv_recall_at_10_max": 0.041574167756887395, + "nauc_cv_recall_at_10_std": -0.031567607900982, + "nauc_cv_recall_at_1_diff1": 0.07258958982395641, + "nauc_cv_recall_at_1_max": -0.02721575557260758, + "nauc_cv_recall_at_1_std": -0.047981761598526164, + "nauc_cv_recall_at_20_diff1": -0.034360664129741786, + "nauc_cv_recall_at_20_max": 0.13687544288877665, + "nauc_cv_recall_at_20_std": 0.005421709114206555, + "nauc_cv_recall_at_3_diff1": 0.07796405401160618, + "nauc_cv_recall_at_3_max": 0.0022447113971254527, + "nauc_cv_recall_at_3_std": -0.02193083840413663, + "nauc_cv_recall_at_5_diff1": 0.058721315220139715, + "nauc_cv_recall_at_5_max": -0.01734274247588793, + "nauc_cv_recall_at_5_std": -0.06046753905236499, + "nauc_map_at_1000_diff1": 0.05722146065713886, + "nauc_map_at_1000_max": 0.004143611940867708, + "nauc_map_at_1000_std": -0.031865337917475284, + "nauc_map_at_100_diff1": 0.05693723750527088, + "nauc_map_at_100_max": 0.004399669635771658, + "nauc_map_at_100_std": -0.031274561293767475, + "nauc_map_at_10_diff1": 0.0624486940125799, + "nauc_map_at_10_max": -0.0022541784601356617, + "nauc_map_at_10_std": -0.03855323923750413, + "nauc_map_at_1_diff1": 0.07258958982395641, + "nauc_map_at_1_max": -0.02721575557260758, + "nauc_map_at_1_std": -0.047981761598526164, + "nauc_map_at_20_diff1": 0.05655287179993111, + "nauc_map_at_20_max": 0.005117092389360872, + "nauc_map_at_20_std": -0.03549998596719167, + "nauc_map_at_3_diff1": 0.07351556234707848, + "nauc_map_at_3_max": -0.007695467804342238, + "nauc_map_at_3_std": -0.031239859746265138, + "nauc_map_at_5_diff1": 0.06630205643433205, + "nauc_map_at_5_max": -0.01497880630240342, + "nauc_map_at_5_std": -0.045952638027913155, + "nauc_mrr_at_1000_diff1": 0.05722146065713886, + "nauc_mrr_at_1000_max": 0.004143611940867708, + "nauc_mrr_at_1000_std": -0.031865337917475284, + "nauc_mrr_at_100_diff1": 0.05693723750527088, + "nauc_mrr_at_100_max": 0.004399669635771658, + "nauc_mrr_at_100_std": -0.031274561293767475, + "nauc_mrr_at_10_diff1": 0.0624486940125799, + "nauc_mrr_at_10_max": -0.0022541784601356617, + "nauc_mrr_at_10_std": -0.03855323923750413, + "nauc_mrr_at_1_diff1": 0.07258958982395641, + "nauc_mrr_at_1_max": -0.02721575557260758, + "nauc_mrr_at_1_std": -0.047981761598526164, + "nauc_mrr_at_20_diff1": 0.05655287179993111, + "nauc_mrr_at_20_max": 0.005117092389360872, + "nauc_mrr_at_20_std": -0.03549998596719167, + "nauc_mrr_at_3_diff1": 0.07351556234707848, + "nauc_mrr_at_3_max": -0.007695467804342238, + "nauc_mrr_at_3_std": -0.031239859746265138, + "nauc_mrr_at_5_diff1": 0.06630205643433205, + "nauc_mrr_at_5_max": -0.01497880630240342, + "nauc_mrr_at_5_std": -0.045952638027913155, + "nauc_ndcg_at_1000_diff1": 0.042063964651544127, + "nauc_ndcg_at_1000_max": 0.02784904871625325, + "nauc_ndcg_at_1000_std": -0.012916227002940112, + "nauc_ndcg_at_100_diff1": 0.03566953457249385, + "nauc_ndcg_at_100_max": 0.03549475391663095, + "nauc_ndcg_at_100_std": 0.0044041422671843595, + "nauc_ndcg_at_10_diff1": 0.054097030876250334, + "nauc_ndcg_at_10_max": 0.011522468333852415, + "nauc_ndcg_at_10_std": -0.036407332290186015, + "nauc_ndcg_at_1_diff1": 0.07258958982395641, + "nauc_ndcg_at_1_max": -0.02721575557260758, + "nauc_ndcg_at_1_std": -0.047981761598526164, + "nauc_ndcg_at_20_diff1": 0.03398315242507301, + "nauc_ndcg_at_20_max": 0.03796440931298207, + "nauc_ndcg_at_20_std": -0.025618940226155185, + "nauc_ndcg_at_3_diff1": 0.07488855851721819, + "nauc_ndcg_at_3_max": -0.004377117227156189, + "nauc_ndcg_at_3_std": -0.028154289921466857, + "nauc_ndcg_at_5_diff1": 0.06380000100648368, + "nauc_ndcg_at_5_max": -0.01553121813581108, + "nauc_ndcg_at_5_std": -0.05057710684051813, + "nauc_precision_at_1000_diff1": -0.41488163893007834, + "nauc_precision_at_1000_max": 0.8024386225078226, + "nauc_precision_at_1000_std": 0.6085296863843523, + "nauc_precision_at_100_diff1": -0.1561532334214323, + "nauc_precision_at_100_max": 0.3230256320984115, + "nauc_precision_at_100_std": 0.42613429552612053, + "nauc_precision_at_10_diff1": 0.035585056376464114, + "nauc_precision_at_10_max": 0.041574167756887555, + "nauc_precision_at_10_std": -0.03156760790098189, + "nauc_precision_at_1_diff1": 0.07258958982395641, + "nauc_precision_at_1_max": -0.02721575557260758, + "nauc_precision_at_1_std": -0.047981761598526164, + "nauc_precision_at_20_diff1": -0.034360664129741654, + "nauc_precision_at_20_max": 0.13687544288877623, + "nauc_precision_at_20_std": 0.0054217091142065525, + "nauc_precision_at_3_diff1": 0.0779640540116062, + "nauc_precision_at_3_max": 0.0022447113971254535, + "nauc_precision_at_3_std": -0.021930838404136517, + "nauc_precision_at_5_diff1": 0.0587213152201399, + "nauc_precision_at_5_max": -0.017342742475887782, + "nauc_precision_at_5_std": -0.060467539052364704, + "nauc_recall_at_1000_diff1": -0.41488163893008945, + "nauc_recall_at_1000_max": 0.8024386225078316, + "nauc_recall_at_1000_std": 0.6085296863843472, + "nauc_recall_at_100_diff1": -0.1561532334214285, + "nauc_recall_at_100_max": 0.3230256320984146, + "nauc_recall_at_100_std": 0.42613429552612103, + "nauc_recall_at_10_diff1": 0.035585056376463996, + "nauc_recall_at_10_max": 0.041574167756887395, + "nauc_recall_at_10_std": -0.031567607900982, + "nauc_recall_at_1_diff1": 0.07258958982395641, + "nauc_recall_at_1_max": -0.02721575557260758, + "nauc_recall_at_1_std": -0.047981761598526164, + "nauc_recall_at_20_diff1": -0.034360664129741786, + "nauc_recall_at_20_max": 0.13687544288877665, + "nauc_recall_at_20_std": 0.005421709114206555, + "nauc_recall_at_3_diff1": 0.07796405401160618, + "nauc_recall_at_3_max": 0.0022447113971254527, + "nauc_recall_at_3_std": -0.02193083840413663, + "nauc_recall_at_5_diff1": 0.058721315220139715, + "nauc_recall_at_5_max": -0.01734274247588793, + "nauc_recall_at_5_std": -0.06046753905236499, + "ndcg_at_1": 0.07406, + "ndcg_at_10": 0.23151, + "ndcg_at_100": 0.33399, + "ndcg_at_1000": 0.34262, + "ndcg_at_20": 0.2857, + "ndcg_at_3": 0.13877, + "ndcg_at_5": 0.1769, + "precision_at_1": 0.07406, + "precision_at_10": 0.045, + "precision_at_100": 0.00917, + "precision_at_1000": 0.00098, + "precision_at_20": 0.03323, + "precision_at_3": 0.06211, + "precision_at_5": 0.05575, + "recall_at_1": 0.07406, + "recall_at_10": 0.45, + "recall_at_100": 0.91651, + "recall_at_1000": 0.98396, + "recall_at_20": 0.66462, + "recall_at_3": 0.18632, + "recall_at_5": 0.27877 + } + ] + }, + "task_name": "NIGHTSI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/OxfordFlowersClassification.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/OxfordFlowersClassification.json new file mode 100644 index 0000000000..46cd0c0c23 --- /dev/null +++ b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/OxfordFlowersClassification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + "evaluation_time": 173.65987730026245, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "accuracy": 0.9609803921568627, + "f1": 0.9607429107615776, + "f1_weighted": 0.9605803695939095, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9609803921568627, + "scores_per_experiment": [ + { + "accuracy": 0.9637254901960784, + "f1": 0.9638390179334648, + "f1_weighted": 0.9636168436654212 + }, + { + "accuracy": 0.9588235294117647, + "f1": 0.9585142741395463, + "f1_weighted": 0.9583485618271098 + }, + { + "accuracy": 0.953921568627451, + "f1": 0.9538204962747494, + "f1_weighted": 0.9536083701514289 + }, + { + "accuracy": 0.9637254901960784, + "f1": 0.962793648645596, + "f1_weighted": 0.9627911409488475 + }, + { + "accuracy": 0.9647058823529412, + "f1": 0.9647471168145315, + "f1_weighted": 0.9645369313767403 + } + ] + } + ] + }, + "task_name": "OxfordFlowersClassification" +} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/RenderedSST2.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/RenderedSST2.json new file mode 100644 index 0000000000..4987452960 --- /dev/null +++ b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/RenderedSST2.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", + "evaluation_time": 13.0860013961792, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "accuracy": 0.5744096650192202, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5744096650192202 + } + ] + }, + "task_name": "RenderedSST2" +} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/model_meta.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/model_meta.json new file mode 100644 index 0000000000..564d2686c3 --- /dev/null +++ b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/model_meta.json @@ -0,0 +1 @@ +{"name": "kakaobrain/align-base", "revision": "e96a37facc7b1f59090ece82293226b817afd6ba", "release_date": "2023-02-24", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "ALIGNModelWrapper"} \ No newline at end of file From 3e50e69b4358ff919f4c9ba1cbce4506899f4435 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Tue, 27 Aug 2024 10:49:36 +0000 Subject: [PATCH 057/154] add FORBI2IRetrieval --- .../abstasks/Image/AbsTaskAny2AnyRetrieval.py | 3 +- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 1 + .../Any2AnyRetrieval/eng/FORBI2IRetrieval.py | 49 +++++ .../FORBI2IRetrieval.json | 186 ++++++++++++++++++ 4 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py index 58d1407493..8fab9b776e 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py @@ -183,7 +183,8 @@ def _load_qrels(self, split): "score": Value("float"), } ) - qrels_ds = qrels_ds.cast(features) + # Some datasets may have extra columns, e.g. `difficulty` in qrels for FORB. + qrels_ds = qrels_ds.select_columns(["query-id", "corpus-id", "score"]).cast(features) self.qrels = qrels_ds diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index dc7c5f8476..af8d6b6e0a 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -21,3 +21,4 @@ from .multilingual.WITT2IRetrieval import * from .multilingual.XFlickr30kCoT2IRetrieval import * from .multilingual.XM3600T2IRetrieval import * +from .eng.FORBI2IRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py new file mode 100644 index 0000000000..f5f7428ff5 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class FORBI2I(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="FORBI2IRetrieval", + description="Retrieve flat object images from 8 classes.", + reference="https://github.com/pxiangwu/FORB", + dataset={ + "path": "isaacchung/forb_retrieval", + "revision": "ac01fba09e554b68ba4a79dc7ae45415e653a3aa", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2022-01-01", "2023-01-01"), + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@misc{wu2023forbflatobjectretrieval, + title={FORB: A Flat Object Retrieval Benchmark for Universal Image Embedding}, + author={Pengxiang Wu and Siman Wang and Kevin Dela Rosa and Derek Hao Hu}, + year={2023}, + eprint={2309.16249}, + archivePrefix={arXiv}, + primaryClass={cs.CV}, + url={https://arxiv.org/abs/2309.16249}, + } + """, + descriptive_stats={ + "n_samples": {"default": 13250}, + "avg_character_length": { + "test": { + "num_documents": 53984, + "num_queries": 13250, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json new file mode 100644 index 0000000000..e09859a559 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "ac01fba09e554b68ba4a79dc7ae45415e653a3aa", + "evaluation_time": 1184.2214772701263, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.00038, + "cv_recall_at_10": 0.00385, + "cv_recall_at_100": 0.03457, + "cv_recall_at_1000": 0.24649, + "cv_recall_at_20": 0.00823, + "cv_recall_at_3": 0.00083, + "cv_recall_at_5": 0.00166, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.00038, + "map_at_1": 0.00038, + "map_at_10": 0.00116, + "map_at_100": 0.00197, + "map_at_1000": 0.00259, + "map_at_20": 0.00144, + "map_at_3": 0.00065, + "map_at_5": 0.00088, + "mrr_at_1": 0.0003773584905660377, + "mrr_at_10": 0.0010433063791554353, + "mrr_at_100": 0.0018764402917165674, + "mrr_at_1000": 0.0024966567136127343, + "mrr_at_20": 0.0013360460673738318, + "mrr_at_3": 0.0005786163522012579, + "mrr_at_5": 0.0007635220125786163, + "nauc_cv_recall_at_1000_diff1": 0.22783742518067585, + "nauc_cv_recall_at_1000_max": -0.30094937055349125, + "nauc_cv_recall_at_1000_std": 0.01784167295863668, + "nauc_cv_recall_at_100_diff1": 0.1523855776857887, + "nauc_cv_recall_at_100_max": -0.2333491118159354, + "nauc_cv_recall_at_100_std": 0.07124160645261825, + "nauc_cv_recall_at_10_diff1": 0.07590081914171622, + "nauc_cv_recall_at_10_max": -0.13915413227271056, + "nauc_cv_recall_at_10_std": 0.11458514181095016, + "nauc_cv_recall_at_1_diff1": -0.0763616850013415, + "nauc_cv_recall_at_1_max": -0.2638046686342903, + "nauc_cv_recall_at_1_std": 0.037617386638046785, + "nauc_cv_recall_at_20_diff1": 0.07168369670369706, + "nauc_cv_recall_at_20_max": -0.1409353515014412, + "nauc_cv_recall_at_20_std": 0.12879237303781227, + "nauc_cv_recall_at_3_diff1": 0.08354269824621324, + "nauc_cv_recall_at_3_max": -0.2661170329536308, + "nauc_cv_recall_at_3_std": 0.02573359026270213, + "nauc_cv_recall_at_5_diff1": 0.05917506158987249, + "nauc_cv_recall_at_5_max": -0.23538307681049842, + "nauc_cv_recall_at_5_std": -0.008110349537771025, + "nauc_map_at_1000_diff1": 0.08679252942719358, + "nauc_map_at_1000_max": -0.19008245077835548, + "nauc_map_at_1000_std": 0.07439215236240897, + "nauc_map_at_100_diff1": 0.05299214493219657, + "nauc_map_at_100_max": -0.16858241572701993, + "nauc_map_at_100_std": 0.09152877376316754, + "nauc_map_at_10_diff1": -0.009168108240029638, + "nauc_map_at_10_max": -0.12428032980862758, + "nauc_map_at_10_std": 0.09071826395276378, + "nauc_map_at_1_diff1": -0.0763616850013415, + "nauc_map_at_1_max": -0.2638046686342903, + "nauc_map_at_1_std": 0.037617386638046785, + "nauc_map_at_20_diff1": 0.009578085831426258, + "nauc_map_at_20_max": -0.13374391534901683, + "nauc_map_at_20_std": 0.10562861158352993, + "nauc_map_at_3_diff1": -0.035427545355100974, + "nauc_map_at_3_max": -0.16112154547893748, + "nauc_map_at_3_std": 0.05967885079771333, + "nauc_map_at_5_diff1": -0.04279524182649304, + "nauc_map_at_5_max": -0.12683730006820387, + "nauc_map_at_5_std": 0.053172020254335796, + "nauc_mrr_at_1000_diff1": 0.11254472976600662, + "nauc_mrr_at_1000_max": -0.22511289948916516, + "nauc_mrr_at_1000_std": 0.06254278735292283, + "nauc_mrr_at_100_diff1": 0.0840199114831485, + "nauc_mrr_at_100_max": -0.2120464897313364, + "nauc_mrr_at_100_std": 0.07733222295763165, + "nauc_mrr_at_10_diff1": 0.04776673206413517, + "nauc_mrr_at_10_max": -0.20880049606815707, + "nauc_mrr_at_10_std": 0.061443566470517366, + "nauc_mrr_at_1_diff1": -0.0763616850013415, + "nauc_mrr_at_1_max": -0.2638046686342903, + "nauc_mrr_at_1_std": 0.037617386638046785, + "nauc_mrr_at_20_diff1": 0.05220974403818351, + "nauc_mrr_at_20_max": -0.19611442600889642, + "nauc_mrr_at_20_std": 0.08143751083549026, + "nauc_mrr_at_3_diff1": 0.034915598278134806, + "nauc_mrr_at_3_max": -0.26847563607517416, + "nauc_mrr_at_3_std": 0.02522135766031656, + "nauc_mrr_at_5_diff1": 0.03687636725458146, + "nauc_mrr_at_5_max": -0.2515841256355842, + "nauc_mrr_at_5_std": 0.010362513365887428, + "nauc_ndcg_at_1000_diff1": 0.20529190954097523, + "nauc_ndcg_at_1000_max": -0.27985057845510525, + "nauc_ndcg_at_1000_std": 0.028715467526608203, + "nauc_ndcg_at_100_diff1": 0.12489112426211123, + "nauc_ndcg_at_100_max": -0.21266051793182603, + "nauc_ndcg_at_100_std": 0.08036708473960381, + "nauc_ndcg_at_10_diff1": 0.011495883635102193, + "nauc_ndcg_at_10_max": -0.10236713608768004, + "nauc_ndcg_at_10_std": 0.11503781091876635, + "nauc_ndcg_at_1_diff1": -0.0763616850013415, + "nauc_ndcg_at_1_max": -0.2638046686342903, + "nauc_ndcg_at_1_std": 0.037617386638046785, + "nauc_ndcg_at_20_diff1": 0.04032171735248905, + "nauc_ndcg_at_20_max": -0.12785521817880435, + "nauc_ndcg_at_20_std": 0.13108451777376734, + "nauc_ndcg_at_3_diff1": -0.029332307317860733, + "nauc_ndcg_at_3_max": -0.1479246364175155, + "nauc_ndcg_at_3_std": 0.06371707260127392, + "nauc_ndcg_at_5_diff1": -0.044971001591395644, + "nauc_ndcg_at_5_max": -0.1026097612407452, + "nauc_ndcg_at_5_std": 0.05344340637950877, + "nauc_precision_at_1000_diff1": 0.22783742518067573, + "nauc_precision_at_1000_max": -0.30094937055349136, + "nauc_precision_at_1000_std": 0.017841672958636725, + "nauc_precision_at_100_diff1": 0.15238557768578878, + "nauc_precision_at_100_max": -0.23334911181593537, + "nauc_precision_at_100_std": 0.07124160645261826, + "nauc_precision_at_10_diff1": 0.03404639363855913, + "nauc_precision_at_10_max": -0.08089372393101929, + "nauc_precision_at_10_std": 0.14209820230748604, + "nauc_precision_at_1_diff1": -0.0763616850013415, + "nauc_precision_at_1_max": -0.2638046686342903, + "nauc_precision_at_1_std": 0.037617386638046785, + "nauc_precision_at_20_diff1": 0.06390467595189904, + "nauc_precision_at_20_max": -0.1269775837256385, + "nauc_precision_at_20_std": 0.14869868526965405, + "nauc_precision_at_3_diff1": -0.017729252234215863, + "nauc_precision_at_3_max": -0.12334110751068078, + "nauc_precision_at_3_std": 0.07159810943014612, + "nauc_precision_at_5_diff1": -0.04986634071689092, + "nauc_precision_at_5_max": -0.06921463991493515, + "nauc_precision_at_5_std": 0.053364338312017305, + "nauc_recall_at_1000_diff1": 0.22783742518067585, + "nauc_recall_at_1000_max": -0.30094937055349125, + "nauc_recall_at_1000_std": 0.01784167295863668, + "nauc_recall_at_100_diff1": 0.1523855776857887, + "nauc_recall_at_100_max": -0.2333491118159354, + "nauc_recall_at_100_std": 0.07124160645261825, + "nauc_recall_at_10_diff1": 0.034046393638559026, + "nauc_recall_at_10_max": -0.08089372393101936, + "nauc_recall_at_10_std": 0.1420982023074859, + "nauc_recall_at_1_diff1": -0.0763616850013415, + "nauc_recall_at_1_max": -0.2638046686342903, + "nauc_recall_at_1_std": 0.037617386638046785, + "nauc_recall_at_20_diff1": 0.06390467595189901, + "nauc_recall_at_20_max": -0.12697758372563853, + "nauc_recall_at_20_std": 0.14869868526965388, + "nauc_recall_at_3_diff1": -0.017729252234215977, + "nauc_recall_at_3_max": -0.12334110751068085, + "nauc_recall_at_3_std": 0.07159810943014594, + "nauc_recall_at_5_diff1": -0.049866340716890785, + "nauc_recall_at_5_max": -0.06921463991493515, + "nauc_recall_at_5_std": 0.053364338312017305, + "ndcg_at_1": 0.00038, + "ndcg_at_10": 0.00184, + "ndcg_at_100": 0.00747, + "ndcg_at_1000": 0.0321, + "ndcg_at_20": 0.00288, + "ndcg_at_3": 0.00074, + "ndcg_at_5": 0.00116, + "precision_at_1": 0.00038, + "precision_at_10": 0.00042, + "precision_at_100": 0.00035, + "precision_at_1000": 0.00025, + "precision_at_20": 0.00042, + "precision_at_3": 0.00033, + "precision_at_5": 0.00041, + "recall_at_1": 0.00038, + "recall_at_10": 0.00415, + "recall_at_100": 0.03457, + "recall_at_1000": 0.24649, + "recall_at_20": 0.0083, + "recall_at_3": 0.00098, + "recall_at_5": 0.00204 + } + ] + }, + "task_name": "FORBI2IRetrieval" +} \ No newline at end of file From 5038ae620b0f894ff20bb7dedd0b5bb7bf18535c Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 2 Sep 2024 00:49:50 +0100 Subject: [PATCH 058/154] forb & tuberlin new revision --- .../Any2AnyRetrieval/eng/FORBI2IRetrieval.py | 2 +- .../eng/TUBerlinT2IRetrieval.py | 6 +- .../TUBerlinT2IRetrieval.json | 186 ++++++++++++++++++ .../TUBerlinT2IRetrieval.json | 186 ++++++++++++++++++ 4 files changed, 376 insertions(+), 4 deletions(-) create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TUBerlinT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/TUBerlinT2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py index f5f7428ff5..49ce644c7d 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py @@ -11,7 +11,7 @@ class FORBI2I(AbsTaskAny2AnyRetrieval): reference="https://github.com/pxiangwu/FORB", dataset={ "path": "isaacchung/forb_retrieval", - "revision": "ac01fba09e554b68ba4a79dc7ae45415e653a3aa", + "revision": "336607d5bcc853fb7f7276c2c9721d4b5b1ca8e4", }, type="Retrieval", category="i2i", diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py index 4d39d12a09..467aa4f017 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py @@ -12,8 +12,8 @@ class TUBerlinT2IRetrieval(AbsTaskAny2AnyRetrieval): reference="https://dl.acm.org/doi/pdf/10.1145/2185520.2185540?casa_token=tq-eUx5UROYAAAAA:_694nPzE7tali6LCkxQc0M-mlo9xslasPMcVnFPMy9tDfvt7lg7p1RTe-k8VWCjuv9gmkQqasKUZ", dataset={ "path": "gowitheflow/tu-berlin", - "revision": "dcd8328b8b27cd39ed6c066862532a5dcd35f012", - "trust_remote_code": True, + "revision": "0cd78cd1ddbd3cafa9f319c638ebd77836ec9ff6", + # "trust_remote_code": True, }, type="Retrieval", category="t2i", @@ -45,7 +45,7 @@ class TUBerlinT2IRetrieval(AbsTaskAny2AnyRetrieval): "avg_character_length": { "test": { "average_document_length": 0.0, - "average_query_length": 0.0, + "average_query_length": 7.24, "num_documents": 20000, "num_queries": 250, "average_relevant_docs_per_query": 80.0, diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TUBerlinT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TUBerlinT2IRetrieval.json new file mode 100644 index 0000000000..81d2509bb5 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TUBerlinT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "0cd78cd1ddbd3cafa9f319c638ebd77836ec9ff6", + "evaluation_time": 213.69540762901306, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.892, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 0.988, + "cv_recall_at_5": 0.996, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.86686, + "map_at_1": 0.01115, + "map_at_10": 0.1005, + "map_at_100": 0.52616, + "map_at_1000": 0.5996, + "map_at_20": 0.19016, + "map_at_3": 0.03243, + "map_at_5": 0.05275, + "mrr_at_1": 0.892, + "mrr_at_10": 0.9413333333333334, + "mrr_at_100": 0.9413333333333334, + "mrr_at_1000": 0.9413333333333334, + "mrr_at_20": 0.9413333333333334, + "mrr_at_3": 0.9386666666666666, + "mrr_at_5": 0.9406666666666667, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": 0.04326570520655111, + "nauc_cv_recall_at_1_max": 0.3224325173726293, + "nauc_cv_recall_at_1_std": 0.0345532004050703, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": 0.460939931528164, + "nauc_cv_recall_at_3_max": -0.17086834733893533, + "nauc_cv_recall_at_3_std": -0.10768751945221451, + "nauc_cv_recall_at_5_diff1": 0.5541549953314557, + "nauc_cv_recall_at_5_max": -1.7399626517272968, + "nauc_cv_recall_at_5_std": -1.1517273576096612, + "nauc_map_at_1000_diff1": -0.15908458334668993, + "nauc_map_at_1000_max": 0.2802601284758, + "nauc_map_at_1000_std": 0.5666251153690053, + "nauc_map_at_100_diff1": -0.14422906770688157, + "nauc_map_at_100_max": 0.2646769819295198, + "nauc_map_at_100_std": 0.520091743009256, + "nauc_map_at_10_diff1": 0.02159195828127496, + "nauc_map_at_10_max": 0.2922120128550034, + "nauc_map_at_10_std": 0.2414778043850529, + "nauc_map_at_1_diff1": 0.0432657052065505, + "nauc_map_at_1_max": 0.3224325173726305, + "nauc_map_at_1_std": 0.03455320040506908, + "nauc_map_at_20_diff1": -0.03664228393755642, + "nauc_map_at_20_max": 0.2798113102736121, + "nauc_map_at_20_std": 0.29660406074745105, + "nauc_map_at_3_diff1": -0.04081880084720492, + "nauc_map_at_3_max": 0.29756873651497484, + "nauc_map_at_3_std": 0.1480977454027196, + "nauc_map_at_5_diff1": 0.04000144036378255, + "nauc_map_at_5_max": 0.29959598064961857, + "nauc_map_at_5_std": 0.21234921808246715, + "nauc_mrr_at_1000_diff1": 0.08232431911613593, + "nauc_mrr_at_1000_max": 0.2976966640972928, + "nauc_mrr_at_1000_std": 0.03936761305241755, + "nauc_mrr_at_100_diff1": 0.08232431911613593, + "nauc_mrr_at_100_max": 0.2976966640972928, + "nauc_mrr_at_100_std": 0.03936761305241755, + "nauc_mrr_at_10_diff1": 0.08232431911613593, + "nauc_mrr_at_10_max": 0.2976966640972928, + "nauc_mrr_at_10_std": 0.03936761305241755, + "nauc_mrr_at_1_diff1": 0.04326570520655111, + "nauc_mrr_at_1_max": 0.3224325173726293, + "nauc_mrr_at_1_std": 0.0345532004050703, + "nauc_mrr_at_20_diff1": 0.08232431911613593, + "nauc_mrr_at_20_max": 0.2976966640972928, + "nauc_mrr_at_20_std": 0.03936761305241755, + "nauc_mrr_at_3_diff1": 0.09841645170536006, + "nauc_mrr_at_3_max": 0.28575011259877825, + "nauc_mrr_at_3_std": 0.038641853990093455, + "nauc_mrr_at_5_diff1": 0.08767292202499989, + "nauc_mrr_at_5_max": 0.27459805882788724, + "nauc_mrr_at_5_std": 0.025865536256733646, + "nauc_ndcg_at_1000_diff1": -0.17908397271463292, + "nauc_ndcg_at_1000_max": 0.32950118161580466, + "nauc_ndcg_at_1000_std": 0.6445234074751405, + "nauc_ndcg_at_100_diff1": -0.1622574037823723, + "nauc_ndcg_at_100_max": 0.2902925476213027, + "nauc_ndcg_at_100_std": 0.5668207357528271, + "nauc_ndcg_at_10_diff1": 0.02841530472596724, + "nauc_ndcg_at_10_max": 0.31021101854725536, + "nauc_ndcg_at_10_std": 0.25227525115585936, + "nauc_ndcg_at_1_diff1": 0.04326570520655111, + "nauc_ndcg_at_1_max": 0.3224325173726293, + "nauc_ndcg_at_1_std": 0.0345532004050703, + "nauc_ndcg_at_20_diff1": -0.04345210224440129, + "nauc_ndcg_at_20_max": 0.3060058864100404, + "nauc_ndcg_at_20_std": 0.3203500202379029, + "nauc_ndcg_at_3_diff1": -0.027311698752615493, + "nauc_ndcg_at_3_max": 0.3062961102473417, + "nauc_ndcg_at_3_std": 0.15546986295776838, + "nauc_ndcg_at_5_diff1": 0.054176844078444426, + "nauc_ndcg_at_5_max": 0.29970785169971287, + "nauc_ndcg_at_5_std": 0.20921915554766976, + "nauc_precision_at_1000_diff1": -0.22031096869135272, + "nauc_precision_at_1000_max": 0.2986648920876661, + "nauc_precision_at_1000_std": 0.7259021291856423, + "nauc_precision_at_100_diff1": -0.1885686833994847, + "nauc_precision_at_100_max": 0.27498526176589005, + "nauc_precision_at_100_std": 0.6104621768161939, + "nauc_precision_at_10_diff1": 0.02876131690490984, + "nauc_precision_at_10_max": 0.306889992185712, + "nauc_precision_at_10_std": 0.28382126591470597, + "nauc_precision_at_1_diff1": 0.04326570520655111, + "nauc_precision_at_1_max": 0.3224325173726293, + "nauc_precision_at_1_std": 0.0345532004050703, + "nauc_precision_at_20_diff1": -0.06798464537403294, + "nauc_precision_at_20_max": 0.3008290397432531, + "nauc_precision_at_20_std": 0.3542598081165615, + "nauc_precision_at_3_diff1": -0.0445238095238101, + "nauc_precision_at_3_max": 0.30330827067669053, + "nauc_precision_at_3_std": 0.1895989974937346, + "nauc_precision_at_5_diff1": 0.07960984706120275, + "nauc_precision_at_5_max": 0.2913570788400933, + "nauc_precision_at_5_std": 0.2503577727023061, + "nauc_recall_at_1000_diff1": -0.22031096869135416, + "nauc_recall_at_1000_max": 0.29866489208766495, + "nauc_recall_at_1000_std": 0.7259021291856401, + "nauc_recall_at_100_diff1": -0.18856868339948582, + "nauc_recall_at_100_max": 0.2749852617658886, + "nauc_recall_at_100_std": 0.610462176816193, + "nauc_recall_at_10_diff1": 0.02876131690490984, + "nauc_recall_at_10_max": 0.306889992185712, + "nauc_recall_at_10_std": 0.28382126591470597, + "nauc_recall_at_1_diff1": 0.0432657052065505, + "nauc_recall_at_1_max": 0.3224325173726305, + "nauc_recall_at_1_std": 0.03455320040506908, + "nauc_recall_at_20_diff1": -0.06798464537403294, + "nauc_recall_at_20_max": 0.3008290397432531, + "nauc_recall_at_20_std": 0.3542598081165615, + "nauc_recall_at_3_diff1": -0.04452380952380912, + "nauc_recall_at_3_max": 0.303308270676692, + "nauc_recall_at_3_std": 0.18959899749373313, + "nauc_recall_at_5_diff1": 0.07960984706120275, + "nauc_recall_at_5_max": 0.2913570788400933, + "nauc_recall_at_5_std": 0.2503577727023061, + "ndcg_at_1": 0.892, + "ndcg_at_10": 0.86686, + "ndcg_at_100": 0.68398, + "ndcg_at_1000": 0.83983, + "ndcg_at_20": 0.843, + "ndcg_at_3": 0.89368, + "ndcg_at_5": 0.88679, + "precision_at_1": 0.892, + "precision_at_10": 0.8564, + "precision_at_100": 0.51112, + "precision_at_1000": 0.07312, + "precision_at_20": 0.827, + "precision_at_3": 0.89333, + "precision_at_5": 0.8832, + "recall_at_1": 0.01115, + "recall_at_10": 0.10705, + "recall_at_100": 0.6389, + "recall_at_1000": 0.914, + "recall_at_20": 0.20675, + "recall_at_3": 0.0335, + "recall_at_5": 0.0552 + } + ] + }, + "task_name": "TUBerlinT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/TUBerlinT2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/TUBerlinT2IRetrieval.json new file mode 100644 index 0000000000..355e3b33e2 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/TUBerlinT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "0cd78cd1ddbd3cafa9f319c638ebd77836ec9ff6", + "evaluation_time": 385.1187882423401, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.908, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 0.992, + "cv_recall_at_5": 0.992, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8924, + "map_at_1": 0.01135, + "map_at_10": 0.10502, + "map_at_100": 0.62701, + "map_at_1000": 0.6943, + "map_at_20": 0.20153, + "map_at_3": 0.03284, + "map_at_5": 0.05367, + "mrr_at_1": 0.908, + "mrr_at_10": 0.9484047619047621, + "mrr_at_100": 0.9484047619047621, + "mrr_at_1000": 0.9484047619047621, + "mrr_at_20": 0.9484047619047621, + "mrr_at_3": 0.9473333333333334, + "mrr_at_5": 0.9473333333333334, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": 0.11582024113993439, + "nauc_cv_recall_at_1_max": 0.373929281857669, + "nauc_cv_recall_at_1_std": 0.23413713311411463, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": 0.4561157796451851, + "nauc_cv_recall_at_3_max": 0.24042950513537323, + "nauc_cv_recall_at_3_std": -0.5144724556489358, + "nauc_cv_recall_at_5_diff1": 0.4561157796451851, + "nauc_cv_recall_at_5_max": 0.24042950513537323, + "nauc_cv_recall_at_5_std": -0.5144724556489358, + "nauc_map_at_1000_diff1": -0.06530005170454266, + "nauc_map_at_1000_max": 0.1502471022239646, + "nauc_map_at_1000_std": 0.4724049267752983, + "nauc_map_at_100_diff1": -0.0704630599295298, + "nauc_map_at_100_max": 0.13854288149819008, + "nauc_map_at_100_std": 0.43057914277907994, + "nauc_map_at_10_diff1": 0.006402040467115649, + "nauc_map_at_10_max": 0.255115488651325, + "nauc_map_at_10_std": 0.15962216044023922, + "nauc_map_at_1_diff1": 0.11582024113993612, + "nauc_map_at_1_max": 0.37392928185767066, + "nauc_map_at_1_std": 0.23413713311411455, + "nauc_map_at_20_diff1": -0.04061394388241786, + "nauc_map_at_20_max": 0.2114603378673776, + "nauc_map_at_20_std": 0.19804307292662585, + "nauc_map_at_3_diff1": 0.015403428628263226, + "nauc_map_at_3_max": 0.2868909673477487, + "nauc_map_at_3_std": 0.16916419193497864, + "nauc_map_at_5_diff1": 0.018750084565292698, + "nauc_map_at_5_max": 0.23531345421895392, + "nauc_map_at_5_std": 0.11997890223687437, + "nauc_mrr_at_1000_diff1": 0.110313776333482, + "nauc_mrr_at_1000_max": 0.3707442983346247, + "nauc_mrr_at_1000_std": 0.22493264341576957, + "nauc_mrr_at_100_diff1": 0.110313776333482, + "nauc_mrr_at_100_max": 0.3707442983346247, + "nauc_mrr_at_100_std": 0.22493264341576957, + "nauc_mrr_at_10_diff1": 0.110313776333482, + "nauc_mrr_at_10_max": 0.3707442983346247, + "nauc_mrr_at_10_std": 0.22493264341576957, + "nauc_mrr_at_1_diff1": 0.11582024113993439, + "nauc_mrr_at_1_max": 0.373929281857669, + "nauc_mrr_at_1_std": 0.23413713311411463, + "nauc_mrr_at_20_diff1": 0.110313776333482, + "nauc_mrr_at_20_max": 0.3707442983346247, + "nauc_mrr_at_20_std": 0.22493264341576957, + "nauc_mrr_at_3_diff1": 0.11721566263636403, + "nauc_mrr_at_3_max": 0.36825278634660735, + "nauc_mrr_at_3_std": 0.2090262265243671, + "nauc_mrr_at_5_diff1": 0.11721566263636403, + "nauc_mrr_at_5_max": 0.36825278634660735, + "nauc_mrr_at_5_std": 0.2090262265243671, + "nauc_ndcg_at_1000_diff1": -0.037292624079711435, + "nauc_ndcg_at_1000_max": 0.21046169942509146, + "nauc_ndcg_at_1000_std": 0.54841970255689, + "nauc_ndcg_at_100_diff1": -0.06417993720042019, + "nauc_ndcg_at_100_max": 0.15400863056231898, + "nauc_ndcg_at_100_std": 0.4723216905906371, + "nauc_ndcg_at_10_diff1": 0.01494953591412352, + "nauc_ndcg_at_10_max": 0.2680953405148115, + "nauc_ndcg_at_10_std": 0.16994841614578113, + "nauc_ndcg_at_1_diff1": 0.11582024113993439, + "nauc_ndcg_at_1_max": 0.373929281857669, + "nauc_ndcg_at_1_std": 0.23413713311411463, + "nauc_ndcg_at_20_diff1": -0.025310686952678893, + "nauc_ndcg_at_20_max": 0.2248683312661876, + "nauc_ndcg_at_20_std": 0.19959181596096173, + "nauc_ndcg_at_3_diff1": 0.0248235494869043, + "nauc_ndcg_at_3_max": 0.29864496147398095, + "nauc_ndcg_at_3_std": 0.16810172850714117, + "nauc_ndcg_at_5_diff1": 0.032312983699494084, + "nauc_ndcg_at_5_max": 0.2447447927622062, + "nauc_ndcg_at_5_std": 0.11460242029652785, + "nauc_precision_at_1000_diff1": -0.004062035055989619, + "nauc_precision_at_1000_max": 0.20159639972746157, + "nauc_precision_at_1000_std": 0.6755051245483236, + "nauc_precision_at_100_diff1": -0.0731911806199345, + "nauc_precision_at_100_max": 0.12152536286318481, + "nauc_precision_at_100_std": 0.5067829123595188, + "nauc_precision_at_10_diff1": 0.0010081351446029666, + "nauc_precision_at_10_max": 0.2530613833638247, + "nauc_precision_at_10_std": 0.17313261453427123, + "nauc_precision_at_1_diff1": 0.11582024113993439, + "nauc_precision_at_1_max": 0.373929281857669, + "nauc_precision_at_1_std": 0.23413713311411463, + "nauc_precision_at_20_diff1": -0.04926658558983306, + "nauc_precision_at_20_max": 0.19675521296851267, + "nauc_precision_at_20_std": 0.20747986966249193, + "nauc_precision_at_3_diff1": 0.0017235227188574813, + "nauc_precision_at_3_max": 0.28269138844566605, + "nauc_precision_at_3_std": 0.14890966990500618, + "nauc_precision_at_5_diff1": 0.023111522328179257, + "nauc_precision_at_5_max": 0.20941877145445104, + "nauc_precision_at_5_std": 0.08089615948310987, + "nauc_recall_at_1000_diff1": -0.004062035055987913, + "nauc_recall_at_1000_max": 0.20159639972746426, + "nauc_recall_at_1000_std": 0.6755051245483176, + "nauc_recall_at_100_diff1": -0.07319118061993815, + "nauc_recall_at_100_max": 0.1215253628631808, + "nauc_recall_at_100_std": 0.5067829123595168, + "nauc_recall_at_10_diff1": 0.0010081351446029666, + "nauc_recall_at_10_max": 0.2530613833638247, + "nauc_recall_at_10_std": 0.17313261453427123, + "nauc_recall_at_1_diff1": 0.11582024113993612, + "nauc_recall_at_1_max": 0.37392928185767066, + "nauc_recall_at_1_std": 0.23413713311411455, + "nauc_recall_at_20_diff1": -0.04926658558983306, + "nauc_recall_at_20_max": 0.19675521296851267, + "nauc_recall_at_20_std": 0.20747986966249193, + "nauc_recall_at_3_diff1": 0.001723522718856776, + "nauc_recall_at_3_max": 0.2826913884456665, + "nauc_recall_at_3_std": 0.1489096699050044, + "nauc_recall_at_5_diff1": 0.023111522328179257, + "nauc_recall_at_5_max": 0.20941877145445104, + "nauc_recall_at_5_std": 0.08089615948310987, + "ndcg_at_1": 0.908, + "ndcg_at_10": 0.8924, + "ndcg_at_100": 0.76172, + "ndcg_at_1000": 0.88769, + "ndcg_at_20": 0.87619, + "ndcg_at_3": 0.90306, + "ndcg_at_5": 0.89731, + "precision_at_1": 0.908, + "precision_at_10": 0.888, + "precision_at_100": 0.58372, + "precision_at_1000": 0.07593, + "precision_at_20": 0.8672, + "precision_at_3": 0.90133, + "precision_at_5": 0.8936, + "recall_at_1": 0.01135, + "recall_at_10": 0.111, + "recall_at_100": 0.72965, + "recall_at_1000": 0.9491, + "recall_at_20": 0.2168, + "recall_at_3": 0.0338, + "recall_at_5": 0.05585 + } + ] + }, + "task_name": "TUBerlinT2IRetrieval" +} \ No newline at end of file From 330f78ab934fb9ba43a5223e6e567b37ca9cfc87 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 2 Sep 2024 00:53:43 +0100 Subject: [PATCH 059/154] disable tokenization parallelism --- mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py | 6 ++++-- .../evaluators/Image/Any2AnyRetrievalEvaluator.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py index 8fab9b776e..8742c49522 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py @@ -183,8 +183,10 @@ def _load_qrels(self, split): "score": Value("float"), } ) - # Some datasets may have extra columns, e.g. `difficulty` in qrels for FORB. - qrels_ds = qrels_ds.select_columns(["query-id", "corpus-id", "score"]).cast(features) + # Some datasets may have extra columns, e.g. `difficulty` in qrels for FORB. + qrels_ds = qrels_ds.select_columns(["query-id", "corpus-id", "score"]).cast( + features + ) self.qrels = qrels_ds diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index fa96483972..540c37bbb3 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -31,6 +31,8 @@ top_k_accuracy, ) +os.environ["TOKENIZERS_PARALLELISM"] = "false" + logger = logging.getLogger(__name__) transform = transforms.Compose([transforms.PILToTensor()]) From 77effde2ce85fb345d9a1ec6e996244f5671effa Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 2 Sep 2024 00:55:47 +0100 Subject: [PATCH 060/154] add hateful meme retrieval i2tt2i --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 4 +- .../eng/HatefulMemesI2TRetrieval.py | 119 +++++++++++ .../eng/HatefulMemesT2IRetrieval.py | 119 +++++++++++ .../HatefulMemesI2TRetrieval.json | 186 ++++++++++++++++++ .../HatefulMemesT2IRetrieval.json | 186 ++++++++++++++++++ .../HatefulMemesI2TRetrieval.json | 186 ++++++++++++++++++ .../HatefulMemesT2IRetrieval.json | 186 ++++++++++++++++++ .../HatefulMemesI2TRetrieval.json | 186 ++++++++++++++++++ .../HatefulMemesT2IRetrieval.json | 186 ++++++++++++++++++ 9 files changed, 1357 insertions(+), 1 deletion(-) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesT2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index af8d6b6e0a..bdf8289f75 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -5,6 +5,9 @@ from .eng.Fashion200kI2TRetrieval import * from .eng.Fashion200kT2IRetrieval import * from .eng.FashionIQIT2IRetrieval import * +from .eng.FORBI2IRetrieval import * +from .eng.HatefulMemesI2TRetrieval import * +from .eng.HatefulMemesT2IRetrieval import * from .eng.InfoSeekIT2ITRetrieval import * from .eng.InfoSeekIT2TRetrieval import * from .eng.MSCOCOI2TRetrieval import * @@ -21,4 +24,3 @@ from .multilingual.WITT2IRetrieval import * from .multilingual.XFlickr30kCoT2IRetrieval import * from .multilingual.XM3600T2IRetrieval import * -from .eng.FORBI2IRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py new file mode 100644 index 0000000000..675dcfd5e8 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from datasets import concatenate_datasets, load_dataset + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): + corpus = {} + queries = {} + relevant_docs = {} + + dataset = load_dataset( + path, + cache_dir=cache_dir, + revision=revision, + ) + dataset_splits = [split for split in dataset] + shared_corpus = concatenate_datasets([dataset[split] for split in dataset_splits]) + + shared_corpus = shared_corpus.map( + lambda x: { + "id": "corpus-" + str(x["id"]), + # "text": x["text"], + "modality": "text", + "image": None, + }, + remove_columns=[ + "split", + "label", + ], + ) + + for split in splits: + corpus[split] = shared_corpus + split_dataset = dataset[split] + queries[split] = split_dataset.map( + lambda x: { + "id": "query-" + str(x["id"]), + "text": None, + "modality": "image", + # "image": x["image"], + }, + remove_columns=[ + "split", + "label", + ], + ) + relevant_docs[split] = {} + for example in split_dataset: + query_id = "query-" + str(example["id"]) + doc_id = "corpus-" + str(example["id"]) + if query_id not in relevant_docs[split]: + relevant_docs[split][query_id] = {} + relevant_docs[split][query_id][doc_id] = 1 + + return corpus, queries, relevant_docs + + +class HatefulMemesI2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="HatefulMemesI2TRetrieval", + description="Retrieve captions based on memes.", + reference="https://arxiv.org/pdf/2005.04790", + dataset={ + "path": "Ahren09/MMSoc_HatefulMemes", + "revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", + # "trust_remote_code": True, + }, + type="Retrieval", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2020-01-01", "2020-12-31"), + form=["found"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{kiela2020hateful, + title={The hateful memes challenge: Detecting hate speech in multimodal memes}, + author={Kiela, Douwe and Firooz, Hamed and Mohan, Aravind and Goswami, Vedanuj and Singh, Amanpreet and Ringshia, Pratik and Testuggine, Davide}, + journal={Advances in neural information processing systems}, + volume={33}, + pages={2611--2624}, + year={2020} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 61.0257, + "average_query_length": 0, + "num_documents": 10000, + "num_queries": 1000, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = _load_data( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py new file mode 100644 index 0000000000..cf0fb3adf2 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from datasets import concatenate_datasets, load_dataset + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): + corpus = {} + queries = {} + relevant_docs = {} + + dataset = load_dataset( + path, + cache_dir=cache_dir, + revision=revision, + ) + dataset_splits = [split for split in dataset] + shared_corpus = concatenate_datasets([dataset[split] for split in dataset_splits]) + + shared_corpus = shared_corpus.map( + lambda x: { + "id": "corpus-" + str(x["id"]), + "text": None, + "modality": "image", + # "image": None, + }, + remove_columns=[ + "split", + "label", + ], + ) + + for split in splits: + corpus[split] = shared_corpus + split_dataset = dataset[split] + queries[split] = split_dataset.map( + lambda x: { + "id": "query-" + str(x["id"]), + # "text": None, + "modality": "text", + "image": None, + }, + remove_columns=[ + "split", + "label", + ], + ) + relevant_docs[split] = {} + for example in split_dataset: + query_id = "query-" + str(example["id"]) + doc_id = "corpus-" + str(example["id"]) + if query_id not in relevant_docs[split]: + relevant_docs[split][query_id] = {} + relevant_docs[split][query_id][doc_id] = 1 + + return corpus, queries, relevant_docs + + +class HatefulMemesT2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="HatefulMemesT2IRetrieval", + description="Retrieve captions based on memes.", + reference="https://arxiv.org/pdf/2005.04790", + dataset={ + "path": "Ahren09/MMSoc_HatefulMemes", + "revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", + # "trust_remote_code": True, + }, + type="Retrieval", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2020-01-01", "2020-12-31"), + form=["found"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{kiela2020hateful, + title={The hateful memes challenge: Detecting hate speech in multimodal memes}, + author={Kiela, Douwe and Firooz, Hamed and Mohan, Aravind and Goswami, Vedanuj and Singh, Amanpreet and Ringshia, Pratik and Testuggine, Davide}, + journal={Advances in neural information processing systems}, + volume={33}, + pages={2611--2624}, + year={2020} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 0, + "average_query_length": 61.0257, + "num_documents": 10000, + "num_queries": 1000, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = _load_data( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesI2TRetrieval.json new file mode 100644 index 0000000000..e7c0e0194b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", + "evaluation_time": 13.294805765151978, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.343, + "cv_recall_at_10": 0.678, + "cv_recall_at_100": 0.798, + "cv_recall_at_1000": 0.886, + "cv_recall_at_20": 0.721, + "cv_recall_at_3": 0.557, + "cv_recall_at_5": 0.617, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.51779, + "map_at_1": 0.344, + "map_at_10": 0.46588, + "map_at_100": 0.47092, + "map_at_1000": 0.47125, + "map_at_20": 0.46902, + "map_at_3": 0.44583, + "map_at_5": 0.45763, + "mrr_at_1": 0.343, + "mrr_at_10": 0.46419642857142834, + "mrr_at_100": 0.469239410916577, + "mrr_at_1000": 0.4695660695365724, + "mrr_at_20": 0.4672927183884223, + "mrr_at_3": 0.4421666666666665, + "mrr_at_5": 0.4561666666666663, + "nauc_cv_recall_at_1000_diff1": 0.02351124577554979, + "nauc_cv_recall_at_1000_max": 0.6100895666505729, + "nauc_cv_recall_at_1000_std": 0.2871501822964356, + "nauc_cv_recall_at_100_diff1": 0.025343623449170807, + "nauc_cv_recall_at_100_max": 0.5897113711728056, + "nauc_cv_recall_at_100_std": 0.21345050979692376, + "nauc_cv_recall_at_10_diff1": 0.07681508015980319, + "nauc_cv_recall_at_10_max": 0.5990426933969243, + "nauc_cv_recall_at_10_std": 0.1857973138981508, + "nauc_cv_recall_at_1_diff1": 0.4267100244615221, + "nauc_cv_recall_at_1_max": 0.3618982353042344, + "nauc_cv_recall_at_1_std": 0.01272687102864061, + "nauc_cv_recall_at_20_diff1": 0.07174907627632691, + "nauc_cv_recall_at_20_max": 0.60599143151816, + "nauc_cv_recall_at_20_std": 0.19146863931819508, + "nauc_cv_recall_at_3_diff1": 0.15770090027886977, + "nauc_cv_recall_at_3_max": 0.547799106088024, + "nauc_cv_recall_at_3_std": 0.11460834402735758, + "nauc_cv_recall_at_5_diff1": 0.11310377114566393, + "nauc_cv_recall_at_5_max": 0.590776612134555, + "nauc_cv_recall_at_5_std": 0.1591778721919892, + "nauc_map_at_1000_diff1": 0.2931757212219033, + "nauc_map_at_1000_max": 0.47769835956247375, + "nauc_map_at_1000_std": 0.07142405784187013, + "nauc_map_at_100_diff1": 0.29296015626770694, + "nauc_map_at_100_max": 0.47772176532812277, + "nauc_map_at_100_std": 0.07134522164076948, + "nauc_map_at_10_diff1": 0.2922630116381693, + "nauc_map_at_10_max": 0.47864171579326814, + "nauc_map_at_10_std": 0.0714917138525951, + "nauc_map_at_1_diff1": 0.4251515375908431, + "nauc_map_at_1_max": 0.4001200664332334, + "nauc_map_at_1_std": 0.02200997556357167, + "nauc_map_at_20_diff1": 0.29275131026457724, + "nauc_map_at_20_max": 0.47842933833844037, + "nauc_map_at_20_std": 0.0715990326462406, + "nauc_map_at_3_diff1": 0.2996338562673293, + "nauc_map_at_3_max": 0.4770168797044735, + "nauc_map_at_3_std": 0.06405282046759102, + "nauc_map_at_5_diff1": 0.2923679786791551, + "nauc_map_at_5_max": 0.47682673138378284, + "nauc_map_at_5_std": 0.06873373261928388, + "nauc_mrr_at_1000_diff1": 0.294675550005923, + "nauc_mrr_at_1000_max": 0.4532811868558267, + "nauc_mrr_at_1000_std": 0.06531584821645785, + "nauc_mrr_at_100_diff1": 0.2944595453301148, + "nauc_mrr_at_100_max": 0.45332600827827807, + "nauc_mrr_at_100_std": 0.06524317581183853, + "nauc_mrr_at_10_diff1": 0.2927959210952178, + "nauc_mrr_at_10_max": 0.45399459720760665, + "nauc_mrr_at_10_std": 0.06574167283682725, + "nauc_mrr_at_1_diff1": 0.4267100244615221, + "nauc_mrr_at_1_max": 0.3618982353042344, + "nauc_mrr_at_1_std": 0.01272687102864061, + "nauc_mrr_at_20_diff1": 0.29404544778933245, + "nauc_mrr_at_20_max": 0.4541045696997424, + "nauc_mrr_at_20_std": 0.06560225616529214, + "nauc_mrr_at_3_diff1": 0.3017349546285236, + "nauc_mrr_at_3_max": 0.44735428178122444, + "nauc_mrr_at_3_std": 0.05620321719877866, + "nauc_mrr_at_5_diff1": 0.2944869030132802, + "nauc_mrr_at_5_max": 0.4544780453588209, + "nauc_mrr_at_5_std": 0.06394477273129469, + "nauc_ndcg_at_1000_diff1": 0.25563741681319463, + "nauc_ndcg_at_1000_max": 0.49813064292479103, + "nauc_ndcg_at_1000_std": 0.0945483999782709, + "nauc_ndcg_at_100_diff1": 0.2487088648098237, + "nauc_ndcg_at_100_max": 0.4997081881373343, + "nauc_ndcg_at_100_std": 0.09413719831820086, + "nauc_ndcg_at_10_diff1": 0.24794669752558104, + "nauc_ndcg_at_10_max": 0.506417052208621, + "nauc_ndcg_at_10_std": 0.09468363744731645, + "nauc_ndcg_at_1_diff1": 0.4251515375908431, + "nauc_ndcg_at_1_max": 0.4001200664332334, + "nauc_ndcg_at_1_std": 0.02200997556357167, + "nauc_ndcg_at_20_diff1": 0.24876495274829946, + "nauc_ndcg_at_20_max": 0.5049386724625328, + "nauc_ndcg_at_20_std": 0.09476708189800392, + "nauc_ndcg_at_3_diff1": 0.2628548139798052, + "nauc_ndcg_at_3_max": 0.5008682261479349, + "nauc_ndcg_at_3_std": 0.07843376700906236, + "nauc_ndcg_at_5_diff1": 0.24906238387697463, + "nauc_ndcg_at_5_max": 0.5015432023723324, + "nauc_ndcg_at_5_std": 0.08771207964905081, + "nauc_precision_at_1000_diff1": 0.02351124577554748, + "nauc_precision_at_1000_max": 0.6100895666505727, + "nauc_precision_at_1000_std": 0.2871501822964333, + "nauc_precision_at_100_diff1": 0.02534362344917089, + "nauc_precision_at_100_max": 0.5897113711728055, + "nauc_precision_at_100_std": 0.2134505097969231, + "nauc_precision_at_10_diff1": 0.09074798611182705, + "nauc_precision_at_10_max": 0.6065579975706014, + "nauc_precision_at_10_std": 0.18167834911065367, + "nauc_precision_at_1_diff1": 0.4251515375908431, + "nauc_precision_at_1_max": 0.4001200664332334, + "nauc_precision_at_1_std": 0.02200997556357167, + "nauc_precision_at_20_diff1": 0.07669661808676746, + "nauc_precision_at_20_max": 0.6072858332725597, + "nauc_precision_at_20_std": 0.18975907082787946, + "nauc_precision_at_3_diff1": 0.15301044037993555, + "nauc_precision_at_3_max": 0.572586687980361, + "nauc_precision_at_3_std": 0.12214031356821216, + "nauc_precision_at_5_diff1": 0.11060646785011727, + "nauc_precision_at_5_max": 0.5803579064397495, + "nauc_precision_at_5_std": 0.15018295439273746, + "nauc_recall_at_1000_diff1": 0.02351124577554979, + "nauc_recall_at_1000_max": 0.6100895666505729, + "nauc_recall_at_1000_std": 0.2871501822964356, + "nauc_recall_at_100_diff1": 0.025343623449170807, + "nauc_recall_at_100_max": 0.5897113711728056, + "nauc_recall_at_100_std": 0.21345050979692376, + "nauc_recall_at_10_diff1": 0.09074798611182784, + "nauc_recall_at_10_max": 0.6065579975706022, + "nauc_recall_at_10_std": 0.18167834911065472, + "nauc_recall_at_1_diff1": 0.4251515375908431, + "nauc_recall_at_1_max": 0.4001200664332334, + "nauc_recall_at_1_std": 0.02200997556357167, + "nauc_recall_at_20_diff1": 0.07669661808676843, + "nauc_recall_at_20_max": 0.6072858332725608, + "nauc_recall_at_20_std": 0.18975907082788046, + "nauc_recall_at_3_diff1": 0.15301044037993589, + "nauc_recall_at_3_max": 0.5725866879803613, + "nauc_recall_at_3_std": 0.12214031356821241, + "nauc_recall_at_5_diff1": 0.11060646785011774, + "nauc_recall_at_5_max": 0.5803579064397502, + "nauc_recall_at_5_std": 0.1501829543927384, + "ndcg_at_1": 0.344, + "ndcg_at_10": 0.51779, + "ndcg_at_100": 0.54301, + "ndcg_at_1000": 0.55369, + "ndcg_at_20": 0.52901, + "ndcg_at_3": 0.47663, + "ndcg_at_5": 0.49776, + "precision_at_1": 0.344, + "precision_at_10": 0.0678, + "precision_at_100": 0.00798, + "precision_at_1000": 0.00089, + "precision_at_20": 0.0361, + "precision_at_3": 0.18833, + "precision_at_5": 0.1232, + "recall_at_1": 0.344, + "recall_at_10": 0.678, + "recall_at_100": 0.798, + "recall_at_1000": 0.886, + "recall_at_20": 0.722, + "recall_at_3": 0.565, + "recall_at_5": 0.616 + } + ] + }, + "task_name": "HatefulMemesI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesT2IRetrieval.json new file mode 100644 index 0000000000..8542a96e83 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", + "evaluation_time": 64.34664630889893, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.402, + "cv_recall_at_10": 0.706, + "cv_recall_at_100": 0.817, + "cv_recall_at_1000": 0.883, + "cv_recall_at_20": 0.754, + "cv_recall_at_3": 0.593, + "cv_recall_at_5": 0.649, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.55779, + "map_at_1": 0.402, + "map_at_10": 0.50986, + "map_at_100": 0.51494, + "map_at_1000": 0.51523, + "map_at_20": 0.5134, + "map_at_3": 0.48833, + "map_at_5": 0.50178, + "mrr_at_1": 0.402, + "mrr_at_10": 0.5100047619047616, + "mrr_at_100": 0.5150915951115426, + "mrr_at_1000": 0.5153729730152096, + "mrr_at_20": 0.5135412349475424, + "mrr_at_3": 0.4889999999999997, + "mrr_at_5": 0.5019499999999996, + "nauc_cv_recall_at_1000_diff1": 0.25842166489054547, + "nauc_cv_recall_at_1000_max": 0.2840669665736568, + "nauc_cv_recall_at_1000_std": 0.3131370650874093, + "nauc_cv_recall_at_100_diff1": 0.2965754204791379, + "nauc_cv_recall_at_100_max": 0.3454573066555303, + "nauc_cv_recall_at_100_std": 0.2617113134786554, + "nauc_cv_recall_at_10_diff1": 0.36610371955486987, + "nauc_cv_recall_at_10_max": 0.39744200447768313, + "nauc_cv_recall_at_10_std": 0.18884133539018538, + "nauc_cv_recall_at_1_diff1": 0.4244055578313841, + "nauc_cv_recall_at_1_max": 0.23928406722368636, + "nauc_cv_recall_at_1_std": 0.043300432579218215, + "nauc_cv_recall_at_20_diff1": 0.3654705001675286, + "nauc_cv_recall_at_20_max": 0.39659048014895193, + "nauc_cv_recall_at_20_std": 0.19959623548971286, + "nauc_cv_recall_at_3_diff1": 0.4249958405050879, + "nauc_cv_recall_at_3_max": 0.37439871372221056, + "nauc_cv_recall_at_3_std": 0.11358881221021569, + "nauc_cv_recall_at_5_diff1": 0.4179313459067598, + "nauc_cv_recall_at_5_max": 0.40265448632879425, + "nauc_cv_recall_at_5_std": 0.1370580390717256, + "nauc_map_at_1000_diff1": 0.421562102725268, + "nauc_map_at_1000_max": 0.3036341729007964, + "nauc_map_at_1000_std": 0.08116520872126745, + "nauc_map_at_100_diff1": 0.4216137241609162, + "nauc_map_at_100_max": 0.30386546636442807, + "nauc_map_at_100_std": 0.0812544070904664, + "nauc_map_at_10_diff1": 0.4217182579707922, + "nauc_map_at_10_max": 0.3052606025415569, + "nauc_map_at_10_std": 0.08084695393081556, + "nauc_map_at_1_diff1": 0.4244055578313841, + "nauc_map_at_1_max": 0.23928406722368636, + "nauc_map_at_1_std": 0.043300432579218215, + "nauc_map_at_20_diff1": 0.42240694310701743, + "nauc_map_at_20_max": 0.3048275987004854, + "nauc_map_at_20_std": 0.0807437517104306, + "nauc_map_at_3_diff1": 0.4290903919098365, + "nauc_map_at_3_max": 0.3027723173840437, + "nauc_map_at_3_std": 0.07219734359468977, + "nauc_map_at_5_diff1": 0.42728854397252497, + "nauc_map_at_5_max": 0.30693836093800736, + "nauc_map_at_5_std": 0.07621156582526052, + "nauc_mrr_at_1000_diff1": 0.4211232600709866, + "nauc_mrr_at_1000_max": 0.30339937083832536, + "nauc_mrr_at_1000_std": 0.08097110091447081, + "nauc_mrr_at_100_diff1": 0.42117489295099, + "nauc_mrr_at_100_max": 0.3036308757904552, + "nauc_mrr_at_100_std": 0.08106111681094921, + "nauc_mrr_at_10_diff1": 0.4212911582115076, + "nauc_mrr_at_10_max": 0.3050328316378303, + "nauc_mrr_at_10_std": 0.08065462135727981, + "nauc_mrr_at_1_diff1": 0.4244055578313841, + "nauc_mrr_at_1_max": 0.23928406722368636, + "nauc_mrr_at_1_std": 0.043300432579218215, + "nauc_mrr_at_20_diff1": 0.42197662326971863, + "nauc_mrr_at_20_max": 0.30459799740710386, + "nauc_mrr_at_20_std": 0.08054991063228273, + "nauc_mrr_at_3_diff1": 0.4275714649549671, + "nauc_mrr_at_3_max": 0.3015989754499319, + "nauc_mrr_at_3_std": 0.07178001954853169, + "nauc_mrr_at_5_diff1": 0.4268594262944631, + "nauc_mrr_at_5_max": 0.3066866998974272, + "nauc_mrr_at_5_std": 0.07610582583795289, + "nauc_ndcg_at_1000_diff1": 0.4074447487238973, + "nauc_ndcg_at_1000_max": 0.31125512247814136, + "nauc_ndcg_at_1000_std": 0.10429888184923936, + "nauc_ndcg_at_100_diff1": 0.4071583158406819, + "nauc_ndcg_at_100_max": 0.3157770171610164, + "nauc_ndcg_at_100_std": 0.10582027042777999, + "nauc_ndcg_at_10_diff1": 0.41172173847499366, + "nauc_ndcg_at_10_max": 0.3252162868702745, + "nauc_ndcg_at_10_std": 0.10206715227531686, + "nauc_ndcg_at_1_diff1": 0.4244055578313841, + "nauc_ndcg_at_1_max": 0.23928406722368636, + "nauc_ndcg_at_1_std": 0.043300432579218215, + "nauc_ndcg_at_20_diff1": 0.4133242918402585, + "nauc_ndcg_at_20_max": 0.3233602336017414, + "nauc_ndcg_at_20_std": 0.10179080131066064, + "nauc_ndcg_at_3_diff1": 0.42939665072299477, + "nauc_ndcg_at_3_max": 0.32131749101164025, + "nauc_ndcg_at_3_std": 0.08252550533650273, + "nauc_ndcg_at_5_diff1": 0.42550410028071456, + "nauc_ndcg_at_5_max": 0.3289520097105336, + "nauc_ndcg_at_5_std": 0.08989807277937249, + "nauc_precision_at_1000_diff1": 0.2584216648905424, + "nauc_precision_at_1000_max": 0.28406696657365527, + "nauc_precision_at_1000_std": 0.313137065087408, + "nauc_precision_at_100_diff1": 0.2965754204791373, + "nauc_precision_at_100_max": 0.3454573066555294, + "nauc_precision_at_100_std": 0.26171131347865473, + "nauc_precision_at_10_diff1": 0.3661037195548693, + "nauc_precision_at_10_max": 0.3974420044776824, + "nauc_precision_at_10_std": 0.18884133539018413, + "nauc_precision_at_1_diff1": 0.4244055578313841, + "nauc_precision_at_1_max": 0.23928406722368636, + "nauc_precision_at_1_std": 0.043300432579218215, + "nauc_precision_at_20_diff1": 0.36547050016752725, + "nauc_precision_at_20_max": 0.3965904801489509, + "nauc_precision_at_20_std": 0.19959623548971295, + "nauc_precision_at_3_diff1": 0.4299202442181095, + "nauc_precision_at_3_max": 0.378034179370445, + "nauc_precision_at_3_std": 0.114845975410715, + "nauc_precision_at_5_diff1": 0.41793134590675957, + "nauc_precision_at_5_max": 0.4026544863287947, + "nauc_precision_at_5_std": 0.13705803907172523, + "nauc_recall_at_1000_diff1": 0.25842166489054547, + "nauc_recall_at_1000_max": 0.2840669665736568, + "nauc_recall_at_1000_std": 0.3131370650874093, + "nauc_recall_at_100_diff1": 0.2965754204791379, + "nauc_recall_at_100_max": 0.3454573066555303, + "nauc_recall_at_100_std": 0.2617113134786554, + "nauc_recall_at_10_diff1": 0.36610371955486987, + "nauc_recall_at_10_max": 0.39744200447768313, + "nauc_recall_at_10_std": 0.18884133539018538, + "nauc_recall_at_1_diff1": 0.4244055578313841, + "nauc_recall_at_1_max": 0.23928406722368636, + "nauc_recall_at_1_std": 0.043300432579218215, + "nauc_recall_at_20_diff1": 0.3654705001675286, + "nauc_recall_at_20_max": 0.39659048014895193, + "nauc_recall_at_20_std": 0.19959623548971286, + "nauc_recall_at_3_diff1": 0.42992024421810965, + "nauc_recall_at_3_max": 0.3780341793704453, + "nauc_recall_at_3_std": 0.11484597541071496, + "nauc_recall_at_5_diff1": 0.4179313459067598, + "nauc_recall_at_5_max": 0.40265448632879425, + "nauc_recall_at_5_std": 0.1370580390717256, + "ndcg_at_1": 0.402, + "ndcg_at_10": 0.55779, + "ndcg_at_100": 0.58173, + "ndcg_at_1000": 0.58994, + "ndcg_at_20": 0.57019, + "ndcg_at_3": 0.51483, + "ndcg_at_5": 0.53889, + "precision_at_1": 0.402, + "precision_at_10": 0.0706, + "precision_at_100": 0.00817, + "precision_at_1000": 0.00088, + "precision_at_20": 0.0377, + "precision_at_3": 0.197, + "precision_at_5": 0.1298, + "recall_at_1": 0.402, + "recall_at_10": 0.706, + "recall_at_100": 0.817, + "recall_at_1000": 0.883, + "recall_at_20": 0.754, + "recall_at_3": 0.591, + "recall_at_5": 0.649 + } + ] + }, + "task_name": "HatefulMemesT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesI2TRetrieval.json new file mode 100644 index 0000000000..1730af37d4 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", + "evaluation_time": 11.598620414733887, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.275, + "cv_recall_at_10": 0.573, + "cv_recall_at_100": 0.725, + "cv_recall_at_1000": 0.842, + "cv_recall_at_20": 0.621, + "cv_recall_at_3": 0.464, + "cv_recall_at_5": 0.515, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.43121, + "map_at_1": 0.279, + "map_at_10": 0.38507, + "map_at_100": 0.39114, + "map_at_1000": 0.39162, + "map_at_20": 0.3885, + "map_at_3": 0.3645, + "map_at_5": 0.37645, + "mrr_at_1": 0.275, + "mrr_at_10": 0.3829083333333332, + "mrr_at_100": 0.389097149571975, + "mrr_at_1000": 0.38958301943944784, + "mrr_at_20": 0.3864570007177203, + "mrr_at_3": 0.3629999999999999, + "mrr_at_5": 0.37474999999999964, + "nauc_cv_recall_at_1000_diff1": 0.031303926802318874, + "nauc_cv_recall_at_1000_max": 0.5619684912289413, + "nauc_cv_recall_at_1000_std": 0.37050428931457885, + "nauc_cv_recall_at_100_diff1": 0.042561843442901065, + "nauc_cv_recall_at_100_max": 0.5743835019605944, + "nauc_cv_recall_at_100_std": 0.32809410853463755, + "nauc_cv_recall_at_10_diff1": 0.11190351744901811, + "nauc_cv_recall_at_10_max": 0.5729760161495386, + "nauc_cv_recall_at_10_std": 0.24052820049848195, + "nauc_cv_recall_at_1_diff1": 0.3829883438391379, + "nauc_cv_recall_at_1_max": 0.4329626368982641, + "nauc_cv_recall_at_1_std": 0.08369053068986149, + "nauc_cv_recall_at_20_diff1": 0.09076044156560661, + "nauc_cv_recall_at_20_max": 0.5839339901644851, + "nauc_cv_recall_at_20_std": 0.26806324195952075, + "nauc_cv_recall_at_3_diff1": 0.12719746094572382, + "nauc_cv_recall_at_3_max": 0.5563392142827067, + "nauc_cv_recall_at_3_std": 0.20532706640279597, + "nauc_cv_recall_at_5_diff1": 0.11012794403209765, + "nauc_cv_recall_at_5_max": 0.5633962340746552, + "nauc_cv_recall_at_5_std": 0.2285752948260813, + "nauc_map_at_1000_diff1": 0.24302513335039774, + "nauc_map_at_1000_max": 0.4961908063983587, + "nauc_map_at_1000_std": 0.1429811933626351, + "nauc_map_at_100_diff1": 0.242827802636344, + "nauc_map_at_100_max": 0.4961973841716296, + "nauc_map_at_100_std": 0.1428596727473324, + "nauc_map_at_10_diff1": 0.24405062651754694, + "nauc_map_at_10_max": 0.49637204783980043, + "nauc_map_at_10_std": 0.14114302546813653, + "nauc_map_at_1_diff1": 0.3699809584569773, + "nauc_map_at_1_max": 0.4374252364103405, + "nauc_map_at_1_std": 0.0701724931549778, + "nauc_map_at_20_diff1": 0.2432461326883906, + "nauc_map_at_20_max": 0.49697515480820853, + "nauc_map_at_20_std": 0.14257792437817154, + "nauc_map_at_3_diff1": 0.24748123481892303, + "nauc_map_at_3_max": 0.4924971968626506, + "nauc_map_at_3_std": 0.13419322516706664, + "nauc_map_at_5_diff1": 0.24242153690056423, + "nauc_map_at_5_max": 0.4961183870348337, + "nauc_map_at_5_std": 0.13967033379573657, + "nauc_mrr_at_1000_diff1": 0.25115301640443705, + "nauc_mrr_at_1000_max": 0.4956146751056607, + "nauc_mrr_at_1000_std": 0.15010693213347828, + "nauc_mrr_at_100_diff1": 0.2509461328160055, + "nauc_mrr_at_100_max": 0.49562173640059265, + "nauc_mrr_at_100_std": 0.14997675975447758, + "nauc_mrr_at_10_diff1": 0.25258481872032185, + "nauc_mrr_at_10_max": 0.4961071238767585, + "nauc_mrr_at_10_std": 0.14802888717908258, + "nauc_mrr_at_1_diff1": 0.3829883438391379, + "nauc_mrr_at_1_max": 0.4329626368982641, + "nauc_mrr_at_1_std": 0.08369053068986149, + "nauc_mrr_at_20_diff1": 0.2513171572614364, + "nauc_mrr_at_20_max": 0.4964005836134918, + "nauc_mrr_at_20_std": 0.14965489299578572, + "nauc_mrr_at_3_diff1": 0.25549351845818574, + "nauc_mrr_at_3_max": 0.49385058969179646, + "nauc_mrr_at_3_std": 0.14122783678315726, + "nauc_mrr_at_5_diff1": 0.2515091731083041, + "nauc_mrr_at_5_max": 0.49544845579986435, + "nauc_mrr_at_5_std": 0.14676174165262207, + "nauc_ndcg_at_1000_diff1": 0.20883775445147526, + "nauc_ndcg_at_1000_max": 0.5104248735532774, + "nauc_ndcg_at_1000_std": 0.17388519073185565, + "nauc_ndcg_at_100_diff1": 0.20378718412019636, + "nauc_ndcg_at_100_max": 0.5127636544849964, + "nauc_ndcg_at_100_std": 0.17537566417326647, + "nauc_ndcg_at_10_diff1": 0.2093610810884053, + "nauc_ndcg_at_10_max": 0.5144665835322124, + "nauc_ndcg_at_10_std": 0.16593790274578846, + "nauc_ndcg_at_1_diff1": 0.3699809584569773, + "nauc_ndcg_at_1_max": 0.4374252364103405, + "nauc_ndcg_at_1_std": 0.0701724931549778, + "nauc_ndcg_at_20_diff1": 0.2068660199084453, + "nauc_ndcg_at_20_max": 0.516929381210776, + "nauc_ndcg_at_20_std": 0.17094758749205533, + "nauc_ndcg_at_3_diff1": 0.21501967088991664, + "nauc_ndcg_at_3_max": 0.5071457194184146, + "nauc_ndcg_at_3_std": 0.15279761060645786, + "nauc_ndcg_at_5_diff1": 0.20621710050182515, + "nauc_ndcg_at_5_max": 0.5135055727441509, + "nauc_ndcg_at_5_std": 0.16245661298705746, + "nauc_precision_at_1000_diff1": 0.03130392680231766, + "nauc_precision_at_1000_max": 0.5619684912289395, + "nauc_precision_at_1000_std": 0.37050428931457763, + "nauc_precision_at_100_diff1": 0.042561843442900246, + "nauc_precision_at_100_max": 0.5743835019605946, + "nauc_precision_at_100_std": 0.3280941085346374, + "nauc_precision_at_10_diff1": 0.10631618094334747, + "nauc_precision_at_10_max": 0.5696137757264226, + "nauc_precision_at_10_std": 0.24271559167529141, + "nauc_precision_at_1_diff1": 0.3699809584569773, + "nauc_precision_at_1_max": 0.4374252364103405, + "nauc_precision_at_1_std": 0.0701724931549778, + "nauc_precision_at_20_diff1": 0.09076044156560646, + "nauc_precision_at_20_max": 0.5839339901644841, + "nauc_precision_at_20_std": 0.26806324195951986, + "nauc_precision_at_3_diff1": 0.12543951694556468, + "nauc_precision_at_3_max": 0.5475888041194277, + "nauc_precision_at_3_std": 0.2047169339929046, + "nauc_precision_at_5_diff1": 0.10323701029939387, + "nauc_precision_at_5_max": 0.5633364797327312, + "nauc_precision_at_5_std": 0.22856093414302459, + "nauc_recall_at_1000_diff1": 0.031303926802318874, + "nauc_recall_at_1000_max": 0.5619684912289413, + "nauc_recall_at_1000_std": 0.37050428931457885, + "nauc_recall_at_100_diff1": 0.042561843442901065, + "nauc_recall_at_100_max": 0.5743835019605944, + "nauc_recall_at_100_std": 0.32809410853463755, + "nauc_recall_at_10_diff1": 0.1063161809433481, + "nauc_recall_at_10_max": 0.5696137757264229, + "nauc_recall_at_10_std": 0.24271559167529194, + "nauc_recall_at_1_diff1": 0.3699809584569773, + "nauc_recall_at_1_max": 0.4374252364103405, + "nauc_recall_at_1_std": 0.0701724931549778, + "nauc_recall_at_20_diff1": 0.09076044156560661, + "nauc_recall_at_20_max": 0.5839339901644851, + "nauc_recall_at_20_std": 0.26806324195952075, + "nauc_recall_at_3_diff1": 0.12543951694556424, + "nauc_recall_at_3_max": 0.5475888041194277, + "nauc_recall_at_3_std": 0.20471693399290464, + "nauc_recall_at_5_diff1": 0.10323701029939407, + "nauc_recall_at_5_max": 0.5633364797327317, + "nauc_recall_at_5_std": 0.22856093414302503, + "ndcg_at_1": 0.279, + "ndcg_at_10": 0.43121, + "ndcg_at_100": 0.46258, + "ndcg_at_1000": 0.47698, + "ndcg_at_20": 0.44331, + "ndcg_at_3": 0.38975, + "ndcg_at_5": 0.41101, + "precision_at_1": 0.279, + "precision_at_10": 0.0574, + "precision_at_100": 0.00725, + "precision_at_1000": 0.00084, + "precision_at_20": 0.03105, + "precision_at_3": 0.154, + "precision_at_5": 0.1026, + "recall_at_1": 0.279, + "recall_at_10": 0.574, + "recall_at_100": 0.725, + "recall_at_1000": 0.842, + "recall_at_20": 0.621, + "recall_at_3": 0.462, + "recall_at_5": 0.513 + } + ] + }, + "task_name": "HatefulMemesI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesT2IRetrieval.json new file mode 100644 index 0000000000..2555d35599 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", + "evaluation_time": 48.356528520584106, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.329, + "cv_recall_at_10": 0.599, + "cv_recall_at_100": 0.751, + "cv_recall_at_1000": 0.851, + "cv_recall_at_20": 0.657, + "cv_recall_at_3": 0.492, + "cv_recall_at_5": 0.542, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.46492, + "map_at_1": 0.329, + "map_at_10": 0.42189, + "map_at_100": 0.42825, + "map_at_1000": 0.42866, + "map_at_20": 0.42593, + "map_at_3": 0.4025, + "map_at_5": 0.41405, + "mrr_at_1": 0.329, + "mrr_at_10": 0.4219277777777777, + "mrr_at_100": 0.42830345620240934, + "mrr_at_1000": 0.4286957438222649, + "mrr_at_20": 0.42596962569164554, + "mrr_at_3": 0.4024999999999998, + "mrr_at_5": 0.4140499999999997, + "nauc_cv_recall_at_1000_diff1": 0.22118073062290955, + "nauc_cv_recall_at_1000_max": 0.3484340520825837, + "nauc_cv_recall_at_1000_std": 0.271289099588132, + "nauc_cv_recall_at_100_diff1": 0.3151051127355348, + "nauc_cv_recall_at_100_max": 0.3956778936713964, + "nauc_cv_recall_at_100_std": 0.2495175376887586, + "nauc_cv_recall_at_10_diff1": 0.3598026585943281, + "nauc_cv_recall_at_10_max": 0.36412098427583917, + "nauc_cv_recall_at_10_std": 0.0729252987471962, + "nauc_cv_recall_at_1_diff1": 0.48014966506514223, + "nauc_cv_recall_at_1_max": 0.2804395112575141, + "nauc_cv_recall_at_1_std": 0.04746098198717796, + "nauc_cv_recall_at_20_diff1": 0.3692325712994173, + "nauc_cv_recall_at_20_max": 0.37522127689761997, + "nauc_cv_recall_at_20_std": 0.14494647285037632, + "nauc_cv_recall_at_3_diff1": 0.4240862392440114, + "nauc_cv_recall_at_3_max": 0.35920484079276727, + "nauc_cv_recall_at_3_std": 0.04660884077150774, + "nauc_cv_recall_at_5_diff1": 0.3993186558000575, + "nauc_cv_recall_at_5_max": 0.3607991802708112, + "nauc_cv_recall_at_5_std": 0.033385030475325865, + "nauc_map_at_1000_diff1": 0.4437952396317114, + "nauc_map_at_1000_max": 0.31816421599945327, + "nauc_map_at_1000_std": 0.056598746601127375, + "nauc_map_at_100_diff1": 0.44391951463638446, + "nauc_map_at_100_max": 0.3183891752169483, + "nauc_map_at_100_std": 0.056738797910687, + "nauc_map_at_10_diff1": 0.4433494101595439, + "nauc_map_at_10_max": 0.3179603154080035, + "nauc_map_at_10_std": 0.05128662739139575, + "nauc_map_at_1_diff1": 0.48014966506514223, + "nauc_map_at_1_max": 0.2804395112575141, + "nauc_map_at_1_std": 0.04746098198717796, + "nauc_map_at_20_diff1": 0.444778154649432, + "nauc_map_at_20_max": 0.3182550979373629, + "nauc_map_at_20_std": 0.055205200384806766, + "nauc_map_at_3_diff1": 0.45290708022813964, + "nauc_map_at_3_max": 0.31707367227066335, + "nauc_map_at_3_std": 0.04922208476639303, + "nauc_map_at_5_diff1": 0.4474035304498997, + "nauc_map_at_5_max": 0.31734789204809866, + "nauc_map_at_5_std": 0.04660491752104569, + "nauc_mrr_at_1000_diff1": 0.44370331964018916, + "nauc_mrr_at_1000_max": 0.31808399595668846, + "nauc_mrr_at_1000_std": 0.05655108354905683, + "nauc_mrr_at_100_diff1": 0.4438120282477109, + "nauc_mrr_at_100_max": 0.3182912799047865, + "nauc_mrr_at_100_std": 0.05668195802059791, + "nauc_mrr_at_10_diff1": 0.4432637013304661, + "nauc_mrr_at_10_max": 0.31788692981966776, + "nauc_mrr_at_10_std": 0.051245648964999085, + "nauc_mrr_at_1_diff1": 0.48014966506514223, + "nauc_mrr_at_1_max": 0.2804395112575141, + "nauc_mrr_at_1_std": 0.04746098198717796, + "nauc_mrr_at_20_diff1": 0.44469195301952164, + "nauc_mrr_at_20_max": 0.3181812500366557, + "nauc_mrr_at_20_std": 0.05516412700119084, + "nauc_mrr_at_3_diff1": 0.45290708022813964, + "nauc_mrr_at_3_max": 0.31707367227066335, + "nauc_mrr_at_3_std": 0.04922208476639303, + "nauc_mrr_at_5_diff1": 0.4474035304498997, + "nauc_mrr_at_5_max": 0.31734789204809866, + "nauc_mrr_at_5_std": 0.04660491752104569, + "nauc_ndcg_at_1000_diff1": 0.4208294347119966, + "nauc_ndcg_at_1000_max": 0.32675359943049515, + "nauc_ndcg_at_1000_std": 0.0778020901079404, + "nauc_ndcg_at_100_diff1": 0.42354921239544435, + "nauc_ndcg_at_100_max": 0.33216360569482184, + "nauc_ndcg_at_100_std": 0.08196916027497256, + "nauc_ndcg_at_10_diff1": 0.4247919690071033, + "nauc_ndcg_at_10_max": 0.3292634407930656, + "nauc_ndcg_at_10_std": 0.0554815525525049, + "nauc_ndcg_at_1_diff1": 0.48014966506514223, + "nauc_ndcg_at_1_max": 0.2804395112575141, + "nauc_ndcg_at_1_std": 0.04746098198717796, + "nauc_ndcg_at_20_diff1": 0.42888603859808855, + "nauc_ndcg_at_20_max": 0.3307635310552114, + "nauc_ndcg_at_20_std": 0.07056726430076309, + "nauc_ndcg_at_3_diff1": 0.44540872613954735, + "nauc_ndcg_at_3_max": 0.32795961422684516, + "nauc_ndcg_at_3_std": 0.04864217823353319, + "nauc_ndcg_at_5_diff1": 0.43559223952131837, + "nauc_ndcg_at_5_max": 0.32832423954086903, + "nauc_ndcg_at_5_std": 0.04362509890485802, + "nauc_precision_at_1000_diff1": 0.22118073062290727, + "nauc_precision_at_1000_max": 0.3484340520825814, + "nauc_precision_at_1000_std": 0.2712890995881306, + "nauc_precision_at_100_diff1": 0.3179822082679228, + "nauc_precision_at_100_max": 0.39829408686551576, + "nauc_precision_at_100_std": 0.2503150183150184, + "nauc_precision_at_10_diff1": 0.35980265859432753, + "nauc_precision_at_10_max": 0.3641209842758386, + "nauc_precision_at_10_std": 0.07292529874719615, + "nauc_precision_at_1_diff1": 0.48014966506514223, + "nauc_precision_at_1_max": 0.2804395112575141, + "nauc_precision_at_1_std": 0.04746098198717796, + "nauc_precision_at_20_diff1": 0.3692325712994169, + "nauc_precision_at_20_max": 0.375221276897619, + "nauc_precision_at_20_std": 0.14494647285037554, + "nauc_precision_at_3_diff1": 0.42408623924401156, + "nauc_precision_at_3_max": 0.35920484079276777, + "nauc_precision_at_3_std": 0.046608840771507926, + "nauc_precision_at_5_diff1": 0.39931865580005693, + "nauc_precision_at_5_max": 0.360799180270811, + "nauc_precision_at_5_std": 0.033385030475325955, + "nauc_recall_at_1000_diff1": 0.22118073062290955, + "nauc_recall_at_1000_max": 0.3484340520825837, + "nauc_recall_at_1000_std": 0.271289099588132, + "nauc_recall_at_100_diff1": 0.3179822082679226, + "nauc_recall_at_100_max": 0.3982940868655151, + "nauc_recall_at_100_std": 0.25031501831501846, + "nauc_recall_at_10_diff1": 0.3598026585943281, + "nauc_recall_at_10_max": 0.36412098427583917, + "nauc_recall_at_10_std": 0.0729252987471962, + "nauc_recall_at_1_diff1": 0.48014966506514223, + "nauc_recall_at_1_max": 0.2804395112575141, + "nauc_recall_at_1_std": 0.04746098198717796, + "nauc_recall_at_20_diff1": 0.3692325712994173, + "nauc_recall_at_20_max": 0.37522127689761997, + "nauc_recall_at_20_std": 0.14494647285037632, + "nauc_recall_at_3_diff1": 0.4240862392440114, + "nauc_recall_at_3_max": 0.35920484079276727, + "nauc_recall_at_3_std": 0.04660884077150774, + "nauc_recall_at_5_diff1": 0.3993186558000575, + "nauc_recall_at_5_max": 0.3607991802708112, + "nauc_recall_at_5_std": 0.033385030475325865, + "ndcg_at_1": 0.329, + "ndcg_at_10": 0.46492, + "ndcg_at_100": 0.49677, + "ndcg_at_1000": 0.50914, + "ndcg_at_20": 0.4796, + "ndcg_at_3": 0.42556, + "ndcg_at_5": 0.44626, + "precision_at_1": 0.329, + "precision_at_10": 0.0599, + "precision_at_100": 0.0075, + "precision_at_1000": 0.00085, + "precision_at_20": 0.03285, + "precision_at_3": 0.164, + "precision_at_5": 0.1084, + "recall_at_1": 0.329, + "recall_at_10": 0.599, + "recall_at_100": 0.75, + "recall_at_1000": 0.851, + "recall_at_20": 0.657, + "recall_at_3": 0.492, + "recall_at_5": 0.542 + } + ] + }, + "task_name": "HatefulMemesT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesI2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesI2TRetrieval.json new file mode 100644 index 0000000000..52a575b78b --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", + "evaluation_time": 24.870660543441772, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.39, + "cv_recall_at_10": 0.724, + "cv_recall_at_100": 0.834, + "cv_recall_at_1000": 0.911, + "cv_recall_at_20": 0.758, + "cv_recall_at_3": 0.621, + "cv_recall_at_5": 0.678, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56877, + "map_at_1": 0.392, + "map_at_10": 0.51737, + "map_at_100": 0.52151, + "map_at_1000": 0.52183, + "map_at_20": 0.51954, + "map_at_3": 0.498, + "map_at_5": 0.5108, + "mrr_at_1": 0.39, + "mrr_at_10": 0.5160158730158726, + "mrr_at_100": 0.5203345965174644, + "mrr_at_1000": 0.5206527081735791, + "mrr_at_20": 0.518364903616335, + "mrr_at_3": 0.496833333333333, + "mrr_at_5": 0.5095833333333328, + "nauc_cv_recall_at_1000_diff1": 0.10263955769573768, + "nauc_cv_recall_at_1000_max": 0.6129732791993203, + "nauc_cv_recall_at_1000_std": 0.3754445598464104, + "nauc_cv_recall_at_100_diff1": 0.1065109821651072, + "nauc_cv_recall_at_100_max": 0.670416384844493, + "nauc_cv_recall_at_100_std": 0.3493289665729246, + "nauc_cv_recall_at_10_diff1": 0.1539117944599323, + "nauc_cv_recall_at_10_max": 0.6446834964052115, + "nauc_cv_recall_at_10_std": 0.236339029908953, + "nauc_cv_recall_at_1_diff1": 0.4762368553800783, + "nauc_cv_recall_at_1_max": 0.398969125502826, + "nauc_cv_recall_at_1_std": 0.021837207196719798, + "nauc_cv_recall_at_20_diff1": 0.1302823105717122, + "nauc_cv_recall_at_20_max": 0.6618096412777686, + "nauc_cv_recall_at_20_std": 0.28837236287373114, + "nauc_cv_recall_at_3_diff1": 0.22523976186403152, + "nauc_cv_recall_at_3_max": 0.6245459435051234, + "nauc_cv_recall_at_3_std": 0.15245202065735497, + "nauc_cv_recall_at_5_diff1": 0.16951071378184274, + "nauc_cv_recall_at_5_max": 0.6226364127498648, + "nauc_cv_recall_at_5_std": 0.19889403902865374, + "nauc_map_at_1000_diff1": 0.3561706156286026, + "nauc_map_at_1000_max": 0.49921751700988676, + "nauc_map_at_1000_std": 0.09758022590827435, + "nauc_map_at_100_diff1": 0.35594508335415453, + "nauc_map_at_100_max": 0.4994175748948008, + "nauc_map_at_100_std": 0.09772300134657558, + "nauc_map_at_10_diff1": 0.35503732771075197, + "nauc_map_at_10_max": 0.5003076952556401, + "nauc_map_at_10_std": 0.09670690179240216, + "nauc_map_at_1_diff1": 0.47169657717635144, + "nauc_map_at_1_max": 0.41488426907053433, + "nauc_map_at_1_std": 0.04491745319591042, + "nauc_map_at_20_diff1": 0.35511537035310814, + "nauc_map_at_20_max": 0.500525711086734, + "nauc_map_at_20_std": 0.09838319969822605, + "nauc_map_at_3_diff1": 0.36183536801796407, + "nauc_map_at_3_max": 0.5003507757379961, + "nauc_map_at_3_std": 0.08623036096074559, + "nauc_map_at_5_diff1": 0.3547091654543549, + "nauc_map_at_5_max": 0.4997381766720002, + "nauc_map_at_5_std": 0.09477623407261138, + "nauc_mrr_at_1000_diff1": 0.3591123682228567, + "nauc_mrr_at_1000_max": 0.49027183863310125, + "nauc_mrr_at_1000_std": 0.08439711102794764, + "nauc_mrr_at_100_diff1": 0.35888388249516046, + "nauc_mrr_at_100_max": 0.49047961412452795, + "nauc_mrr_at_100_std": 0.08455100972634555, + "nauc_mrr_at_10_diff1": 0.35816060914379055, + "nauc_mrr_at_10_max": 0.4917050030422624, + "nauc_mrr_at_10_std": 0.08386230024828238, + "nauc_mrr_at_1_diff1": 0.4762368553800783, + "nauc_mrr_at_1_max": 0.398969125502826, + "nauc_mrr_at_1_std": 0.021837207196719798, + "nauc_mrr_at_20_diff1": 0.35803929783268085, + "nauc_mrr_at_20_max": 0.49163331748152517, + "nauc_mrr_at_20_std": 0.08527826344844179, + "nauc_mrr_at_3_diff1": 0.3647354992093332, + "nauc_mrr_at_3_max": 0.49377814932302555, + "nauc_mrr_at_3_std": 0.07502909231738003, + "nauc_mrr_at_5_diff1": 0.3576362645252059, + "nauc_mrr_at_5_max": 0.4907831433335126, + "nauc_mrr_at_5_std": 0.08143862051903414, + "nauc_ndcg_at_1000_diff1": 0.32390740934623524, + "nauc_ndcg_at_1000_max": 0.5192713520679167, + "nauc_ndcg_at_1000_std": 0.12475783601928456, + "nauc_ndcg_at_100_diff1": 0.31759934966550735, + "nauc_ndcg_at_100_max": 0.5257628613485602, + "nauc_ndcg_at_100_std": 0.1300015379219445, + "nauc_ndcg_at_10_diff1": 0.31327913630562454, + "nauc_ndcg_at_10_max": 0.5294161890178075, + "nauc_ndcg_at_10_std": 0.12336856511354594, + "nauc_ndcg_at_1_diff1": 0.47169657717635144, + "nauc_ndcg_at_1_max": 0.41488426907053433, + "nauc_ndcg_at_1_std": 0.04491745319591042, + "nauc_ndcg_at_20_diff1": 0.3128923127075764, + "nauc_ndcg_at_20_max": 0.530861917752235, + "nauc_ndcg_at_20_std": 0.13092750080735302, + "nauc_ndcg_at_3_diff1": 0.3286920241406717, + "nauc_ndcg_at_3_max": 0.5277702997215936, + "nauc_ndcg_at_3_std": 0.1002345470431284, + "nauc_ndcg_at_5_diff1": 0.3136910095539595, + "nauc_ndcg_at_5_max": 0.5273652484740478, + "nauc_ndcg_at_5_std": 0.11725163266241763, + "nauc_precision_at_1000_diff1": 0.10263955769573535, + "nauc_precision_at_1000_max": 0.6129732791993188, + "nauc_precision_at_1000_std": 0.37544455984641006, + "nauc_precision_at_100_diff1": 0.10651098216510617, + "nauc_precision_at_100_max": 0.6704163848444932, + "nauc_precision_at_100_std": 0.34932896657292334, + "nauc_precision_at_10_diff1": 0.14952668399115804, + "nauc_precision_at_10_max": 0.64193653642222, + "nauc_precision_at_10_std": 0.23424384028837889, + "nauc_precision_at_1_diff1": 0.47169657717635144, + "nauc_precision_at_1_max": 0.41488426907053433, + "nauc_precision_at_1_std": 0.04491745319591042, + "nauc_precision_at_20_diff1": 0.13028231057171152, + "nauc_precision_at_20_max": 0.6618096412777668, + "nauc_precision_at_20_std": 0.28837236287373114, + "nauc_precision_at_3_diff1": 0.22446919505049567, + "nauc_precision_at_3_max": 0.6146379954166691, + "nauc_precision_at_3_std": 0.1448788900139171, + "nauc_precision_at_5_diff1": 0.16951071378184204, + "nauc_precision_at_5_max": 0.622636412749864, + "nauc_precision_at_5_std": 0.19889403902865282, + "nauc_recall_at_1000_diff1": 0.10263955769573768, + "nauc_recall_at_1000_max": 0.6129732791993203, + "nauc_recall_at_1000_std": 0.3754445598464104, + "nauc_recall_at_100_diff1": 0.1065109821651072, + "nauc_recall_at_100_max": 0.670416384844493, + "nauc_recall_at_100_std": 0.3493289665729246, + "nauc_recall_at_10_diff1": 0.14952668399115937, + "nauc_recall_at_10_max": 0.641936536422221, + "nauc_recall_at_10_std": 0.2342438402883794, + "nauc_recall_at_1_diff1": 0.47169657717635144, + "nauc_recall_at_1_max": 0.41488426907053433, + "nauc_recall_at_1_std": 0.04491745319591042, + "nauc_recall_at_20_diff1": 0.1302823105717122, + "nauc_recall_at_20_max": 0.6618096412777686, + "nauc_recall_at_20_std": 0.28837236287373114, + "nauc_recall_at_3_diff1": 0.2244691950504948, + "nauc_recall_at_3_max": 0.6146379954166685, + "nauc_recall_at_3_std": 0.14487889001391666, + "nauc_recall_at_5_diff1": 0.16951071378184274, + "nauc_recall_at_5_max": 0.6226364127498648, + "nauc_recall_at_5_std": 0.19889403902865374, + "ndcg_at_1": 0.392, + "ndcg_at_10": 0.56877, + "ndcg_at_100": 0.59092, + "ndcg_at_1000": 0.60039, + "ndcg_at_20": 0.57679, + "ndcg_at_3": 0.52981, + "ndcg_at_5": 0.55308, + "precision_at_1": 0.392, + "precision_at_10": 0.0726, + "precision_at_100": 0.00834, + "precision_at_1000": 0.00091, + "precision_at_20": 0.0379, + "precision_at_3": 0.207, + "precision_at_5": 0.1356, + "recall_at_1": 0.392, + "recall_at_10": 0.726, + "recall_at_100": 0.834, + "recall_at_1000": 0.911, + "recall_at_20": 0.758, + "recall_at_3": 0.621, + "recall_at_5": 0.678 + } + ] + }, + "task_name": "HatefulMemesI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesT2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesT2IRetrieval.json new file mode 100644 index 0000000000..2614fcd742 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", + "evaluation_time": 147.79884696006775, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.457, + "cv_recall_at_10": 0.761, + "cv_recall_at_100": 0.839, + "cv_recall_at_1000": 0.895, + "cv_recall_at_20": 0.793, + "cv_recall_at_3": 0.671, + "cv_recall_at_5": 0.724, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.61855, + "map_at_1": 0.457, + "map_at_10": 0.57175, + "map_at_100": 0.5752, + "map_at_1000": 0.5754, + "map_at_20": 0.57406, + "map_at_3": 0.5545, + "map_at_5": 0.5666, + "mrr_at_1": 0.457, + "mrr_at_10": 0.5717476190476187, + "mrr_at_100": 0.5751927754695753, + "mrr_at_1000": 0.5753988882730213, + "mrr_at_20": 0.5740526036656837, + "mrr_at_3": 0.5544999999999997, + "mrr_at_5": 0.5665999999999995, + "nauc_cv_recall_at_1000_diff1": 0.09881437072166871, + "nauc_cv_recall_at_1000_max": 0.3361460337345095, + "nauc_cv_recall_at_1000_std": 0.3314348351304203, + "nauc_cv_recall_at_100_diff1": 0.21713821991445592, + "nauc_cv_recall_at_100_max": 0.32597885179784103, + "nauc_cv_recall_at_100_std": 0.29378672410321527, + "nauc_cv_recall_at_10_diff1": 0.31827978460409795, + "nauc_cv_recall_at_10_max": 0.3669826124892289, + "nauc_cv_recall_at_10_std": 0.15130021075112898, + "nauc_cv_recall_at_1_diff1": 0.459490502212497, + "nauc_cv_recall_at_1_max": 0.19915884427724026, + "nauc_cv_recall_at_1_std": -0.0022919130648232137, + "nauc_cv_recall_at_20_diff1": 0.30115823543573067, + "nauc_cv_recall_at_20_max": 0.3577860510810276, + "nauc_cv_recall_at_20_std": 0.17380097220769894, + "nauc_cv_recall_at_3_diff1": 0.3679652871898125, + "nauc_cv_recall_at_3_max": 0.33460272118305795, + "nauc_cv_recall_at_3_std": 0.09750594972545565, + "nauc_cv_recall_at_5_diff1": 0.34694925594433695, + "nauc_cv_recall_at_5_max": 0.38095109388152243, + "nauc_cv_recall_at_5_std": 0.13474859644922935, + "nauc_map_at_1000_diff1": 0.41712920668130243, + "nauc_map_at_1000_max": 0.25742819455695226, + "nauc_map_at_1000_std": 0.043159224520371416, + "nauc_map_at_100_diff1": 0.41723483378100945, + "nauc_map_at_100_max": 0.25750564533870984, + "nauc_map_at_100_std": 0.04322917723700672, + "nauc_map_at_10_diff1": 0.41823666093016765, + "nauc_map_at_10_max": 0.259394302781376, + "nauc_map_at_10_std": 0.04212208767183869, + "nauc_map_at_1_diff1": 0.459490502212497, + "nauc_map_at_1_max": 0.19915884427724026, + "nauc_map_at_1_std": -0.0022919130648232137, + "nauc_map_at_20_diff1": 0.4179471732526579, + "nauc_map_at_20_max": 0.2581972993364297, + "nauc_map_at_20_std": 0.04229858410142156, + "nauc_map_at_3_diff1": 0.4228403396208564, + "nauc_map_at_3_max": 0.25461398394576956, + "nauc_map_at_3_std": 0.035903781369377205, + "nauc_map_at_5_diff1": 0.4204871609033417, + "nauc_map_at_5_max": 0.26207000402583563, + "nauc_map_at_5_std": 0.04155080339322991, + "nauc_mrr_at_1000_diff1": 0.4171205757774428, + "nauc_mrr_at_1000_max": 0.25744646575786206, + "nauc_mrr_at_1000_std": 0.04313917421147163, + "nauc_mrr_at_100_diff1": 0.4172264345923441, + "nauc_mrr_at_100_max": 0.2575237455468791, + "nauc_mrr_at_100_std": 0.04320914712121044, + "nauc_mrr_at_10_diff1": 0.41823666093016765, + "nauc_mrr_at_10_max": 0.259394302781376, + "nauc_mrr_at_10_std": 0.04212208767183869, + "nauc_mrr_at_1_diff1": 0.459490502212497, + "nauc_mrr_at_1_max": 0.19915884427724026, + "nauc_mrr_at_1_std": -0.0022919130648232137, + "nauc_mrr_at_20_diff1": 0.4179381540751488, + "nauc_mrr_at_20_max": 0.2582088368426858, + "nauc_mrr_at_20_std": 0.04227994399955885, + "nauc_mrr_at_3_diff1": 0.4228403396208564, + "nauc_mrr_at_3_max": 0.25461398394576956, + "nauc_mrr_at_3_std": 0.035903781369377205, + "nauc_mrr_at_5_diff1": 0.4204871609033417, + "nauc_mrr_at_5_max": 0.26207000402583563, + "nauc_mrr_at_5_std": 0.04155080339322991, + "nauc_ndcg_at_1000_diff1": 0.38960988590535767, + "nauc_ndcg_at_1000_max": 0.27080662641082004, + "nauc_ndcg_at_1000_std": 0.07097161120143067, + "nauc_ndcg_at_100_diff1": 0.39184562939358347, + "nauc_ndcg_at_100_max": 0.27174824704307293, + "nauc_ndcg_at_100_std": 0.07372029959444318, + "nauc_ndcg_at_10_diff1": 0.39942366314570593, + "nauc_ndcg_at_10_max": 0.28076942681445477, + "nauc_ndcg_at_10_std": 0.06280659045704115, + "nauc_ndcg_at_1_diff1": 0.459490502212497, + "nauc_ndcg_at_1_max": 0.19915884427724026, + "nauc_ndcg_at_1_std": -0.0022919130648232137, + "nauc_ndcg_at_20_diff1": 0.3980577456506629, + "nauc_ndcg_at_20_max": 0.2770711582775575, + "nauc_ndcg_at_20_std": 0.0645484759837998, + "nauc_ndcg_at_3_diff1": 0.41020207771802114, + "nauc_ndcg_at_3_max": 0.2730916384556409, + "nauc_ndcg_at_3_std": 0.050010910356776794, + "nauc_ndcg_at_5_diff1": 0.40510394288119245, + "nauc_ndcg_at_5_max": 0.2869883283095438, + "nauc_ndcg_at_5_std": 0.060887275366932087, + "nauc_precision_at_1000_diff1": 0.09881437072166596, + "nauc_precision_at_1000_max": 0.33614603373450663, + "nauc_precision_at_1000_std": 0.33143483513041805, + "nauc_precision_at_100_diff1": 0.21713821991445537, + "nauc_precision_at_100_max": 0.3259788517978413, + "nauc_precision_at_100_std": 0.2937867241032148, + "nauc_precision_at_10_diff1": 0.31827978460409695, + "nauc_precision_at_10_max": 0.36698261248922853, + "nauc_precision_at_10_std": 0.15130021075112876, + "nauc_precision_at_1_diff1": 0.459490502212497, + "nauc_precision_at_1_max": 0.19915884427724026, + "nauc_precision_at_1_std": -0.0022919130648232137, + "nauc_precision_at_20_diff1": 0.3011582354357301, + "nauc_precision_at_20_max": 0.3577860510810275, + "nauc_precision_at_20_std": 0.17380097220769716, + "nauc_precision_at_3_diff1": 0.3679652871898132, + "nauc_precision_at_3_max": 0.3346027211830583, + "nauc_precision_at_3_std": 0.09750594972545619, + "nauc_precision_at_5_diff1": 0.3469492559443356, + "nauc_precision_at_5_max": 0.38095109388152204, + "nauc_precision_at_5_std": 0.13474859644922846, + "nauc_recall_at_1000_diff1": 0.09881437072166871, + "nauc_recall_at_1000_max": 0.3361460337345095, + "nauc_recall_at_1000_std": 0.3314348351304203, + "nauc_recall_at_100_diff1": 0.21713821991445592, + "nauc_recall_at_100_max": 0.32597885179784103, + "nauc_recall_at_100_std": 0.29378672410321527, + "nauc_recall_at_10_diff1": 0.31827978460409795, + "nauc_recall_at_10_max": 0.3669826124892289, + "nauc_recall_at_10_std": 0.15130021075112898, + "nauc_recall_at_1_diff1": 0.459490502212497, + "nauc_recall_at_1_max": 0.19915884427724026, + "nauc_recall_at_1_std": -0.0022919130648232137, + "nauc_recall_at_20_diff1": 0.30115823543573067, + "nauc_recall_at_20_max": 0.3577860510810276, + "nauc_recall_at_20_std": 0.17380097220769894, + "nauc_recall_at_3_diff1": 0.3679652871898125, + "nauc_recall_at_3_max": 0.33460272118305795, + "nauc_recall_at_3_std": 0.09750594972545565, + "nauc_recall_at_5_diff1": 0.34694925594433695, + "nauc_recall_at_5_max": 0.38095109388152243, + "nauc_recall_at_5_std": 0.13474859644922935, + "ndcg_at_1": 0.457, + "ndcg_at_10": 0.61855, + "ndcg_at_100": 0.63523, + "ndcg_at_1000": 0.64198, + "ndcg_at_20": 0.62676, + "ndcg_at_3": 0.58456, + "ndcg_at_5": 0.60637, + "precision_at_1": 0.457, + "precision_at_10": 0.0761, + "precision_at_100": 0.00839, + "precision_at_1000": 0.0009, + "precision_at_20": 0.03965, + "precision_at_3": 0.22367, + "precision_at_5": 0.1448, + "recall_at_1": 0.457, + "recall_at_10": 0.761, + "recall_at_100": 0.839, + "recall_at_1000": 0.895, + "recall_at_20": 0.793, + "recall_at_3": 0.671, + "recall_at_5": 0.724 + } + ] + }, + "task_name": "HatefulMemesT2IRetrieval" +} \ No newline at end of file From 928b6f9ba022fe5df2862981b50ff8ee22868cde Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 2 Sep 2024 23:32:58 +0100 Subject: [PATCH 061/154] add memotion retrieval t2ii2t --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 2 + .../eng/HatefulMemesT2IRetrieval.py | 2 +- .../eng/MemotionI2TRetrieval.py | 147 ++++++++++++++ .../eng/MemotionT2IRetrieval.py | 146 ++++++++++++++ .../MemotionI2TRetrieval.json | 186 ++++++++++++++++++ .../MemotionT2IRetrieval.json | 186 ++++++++++++++++++ .../MemotionI2TRetrieval.json | 186 ++++++++++++++++++ .../MemotionT2IRetrieval.json | 186 ++++++++++++++++++ .../MemotionI2TRetrieval.json | 186 ++++++++++++++++++ .../MemotionT2IRetrieval.json | 186 ++++++++++++++++++ 10 files changed, 1412 insertions(+), 1 deletion(-) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionT2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index bdf8289f75..2fec8364a0 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -10,6 +10,8 @@ from .eng.HatefulMemesT2IRetrieval import * from .eng.InfoSeekIT2ITRetrieval import * from .eng.InfoSeekIT2TRetrieval import * +from .eng.MemotionI2TRetrieval import * +from .eng.MemotionT2IRetrieval import * from .eng.MSCOCOI2TRetrieval import * from .eng.MSCOCOT2IRetrieval import * from .eng.NIGHTSI2IRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py index cf0fb3adf2..8b22c7720b 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py @@ -70,7 +70,7 @@ class HatefulMemesT2IRetrieval(AbsTaskAny2AnyRetrieval): # "trust_remote_code": True, }, type="Retrieval", - category="i2t", + category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py new file mode 100644 index 0000000000..1c26c61066 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +from datasets import concatenate_datasets, load_dataset + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): + corpus = {} + queries = {} + relevant_docs = {} + + dataset = load_dataset( + path, + cache_dir=cache_dir, + revision=revision, + ) + dataset_splits = [split for split in dataset] + + def map_function(split_name): + return lambda x, idx: { + "id": f"corpus-{split_name}-{idx}", + "text": x["text_corrected"] if x["text_corrected"] else "", + "modality": "text", + "image": None, + } + + # Apply the map function to each split and concatenate + shared_corpus = concatenate_datasets( + [ + dataset[split].map( + map_function(split), + with_indices=True, + remove_columns=[ + "split", + "text_ocr", + "text_corrected", + "humor", + "sarcasm", + "offensive", + "motivational", + "sentiment", + ], + ) + for split in dataset_splits + ] + ) + # image corrupted & caption empty + shared_corpus = shared_corpus.select( + [i for i in range(len(shared_corpus)) if i not in [4578, 6781, 6784, 6786]] + ) + for split in splits: + corpus[split] = shared_corpus + split_dataset = dataset[split] + queries[split] = split_dataset.map( + lambda x, idx: { + "id": f"query-{split}-{idx}", + "text": None, + "modality": "image", + # "image": None, + }, + with_indices=True, + remove_columns=[ + "split", + "text_ocr", + "humor", + "sarcasm", + "offensive", + "motivational", + "sentiment", + "text_corrected", + ], + ) + if split == "test": + queries[split] = queries[split].select( + [i for i in range(len(queries[split])) if i not in [489, 492, 494]] + ) + relevant_docs[split] = {} + for index in range(len(split_dataset)): + if index not in [489, 492, 494]: + query_id = f"query-{split}-{index}" + doc_id = f"corpus-{split}-{index}" + if query_id not in relevant_docs[split]: + relevant_docs[split][query_id] = {} + relevant_docs[split][query_id][doc_id] = 1 + return corpus, queries, relevant_docs + + +class MemotionI2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="MemotionI2TRetrieval", + description="Retrieve captions based on memes.", + reference="https://aclanthology.org/2020.semeval-1.99/", + dataset={ + "path": "Ahren09/MMSoc_Memotion", + "revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", + # "trust_remote_code": True, + }, + type="Retrieval", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2020-01-01", "2020-12-31"), + form=["found"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@inproceedings{sharma2020semeval, + title={SemEval-2020 Task 8: Memotion Analysis-the Visuo-Lingual Metaphor!}, + author={Sharma, Chhavi and Bhageria, Deepesh and Scott, William and Pykl, Srinivas and Das, Amitava and Chakraborty, Tanmoy and Pulabaigari, Viswanath and Gamb{\"a}ck, Bj{\"o}rn}, + booktitle={Proceedings of the Fourteenth Workshop on Semantic Evaluation}, + pages={759--773}, + year={2020} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 83.80057388809182, + "average_query_length": 1.0, + "num_documents": 6988, + "num_queries": 697, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = _load_data( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py new file mode 100644 index 0000000000..245088eaf3 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +from datasets import concatenate_datasets, load_dataset + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): + corpus = {} + queries = {} + relevant_docs = {} + + dataset = load_dataset( + path, + cache_dir=cache_dir, + revision=revision, + ) + dataset_splits = [split for split in dataset] + + def map_function(split_name): + return lambda x, idx: { + "id": f"corpus-{split_name}-{idx}", + "text": None, + "modality": "image", + } + + # Apply the map function to each split and concatenate + shared_corpus = concatenate_datasets( + [ + dataset[split].map( + map_function(split), + with_indices=True, + remove_columns=[ + "split", + "text_ocr", + "text_corrected", + "humor", + "sarcasm", + "offensive", + "motivational", + "sentiment", + ], + ) + for split in dataset_splits + ] + ) + # image corrupted + shared_corpus = shared_corpus.select( + [i for i in range(len(shared_corpus)) if i not in [4578, 6781, 6784, 6786]] + ) + for split in splits: + corpus[split] = shared_corpus + split_dataset = dataset[split] + queries[split] = split_dataset.map( + lambda x, idx: { + "id": f"query-{split}-{idx}", + "text": x["text_corrected"], + "modality": "text", + "image": None, + }, + with_indices=True, + remove_columns=[ + "split", + "text_ocr", + "humor", + "sarcasm", + "offensive", + "motivational", + "sentiment", + "text_corrected", + ], + ) + if split == "test": + queries[split] = queries[split].select( + [i for i in range(len(queries[split])) if i not in [489, 492, 494]] + ) + relevant_docs[split] = {} + for index in range(len(split_dataset)): + if index not in [489, 492, 494]: + query_id = f"query-{split}-{index}" + doc_id = f"corpus-{split}-{index}" + if query_id not in relevant_docs[split]: + relevant_docs[split][query_id] = {} + relevant_docs[split][query_id][doc_id] = 1 + return corpus, queries, relevant_docs + + +class MemotionT2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="MemotionT2IRetrieval", + description="Retrieve memes based on captions.", + reference="https://aclanthology.org/2020.semeval-1.99/", + dataset={ + "path": "Ahren09/MMSoc_Memotion", + "revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", + # "trust_remote_code": True, + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2020-01-01", "2020-12-31"), + form=["found"], + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@inproceedings{sharma2020semeval, + title={SemEval-2020 Task 8: Memotion Analysis-the Visuo-Lingual Metaphor!}, + author={Sharma, Chhavi and Bhageria, Deepesh and Scott, William and Pykl, Srinivas and Das, Amitava and Chakraborty, Tanmoy and Pulabaigari, Viswanath and Gamb{\"a}ck, Bj{\"o}rn}, + booktitle={Proceedings of the Fourteenth Workshop on Semantic Evaluation}, + pages={759--773}, + year={2020} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 83.80057388809182, + "num_documents": 6988, + "num_queries": 697, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = _load_data( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionI2TRetrieval.json new file mode 100644 index 0000000000..4571e3c67d --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", + "evaluation_time": 10.613081693649292, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.6198, + "cv_recall_at_10": 0.82496, + "cv_recall_at_100": 0.94118, + "cv_recall_at_1000": 0.98852, + "cv_recall_at_20": 0.87088, + "cv_recall_at_3": 0.74892, + "cv_recall_at_5": 0.7891, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.72207, + "map_at_1": 0.61693, + "map_at_10": 0.68889, + "map_at_100": 0.69395, + "map_at_1000": 0.69417, + "map_at_20": 0.69209, + "map_at_3": 0.6748, + "map_at_5": 0.68412, + "mrr_at_1": 0.6197991391678622, + "mrr_at_10": 0.6908155131971484, + "mrr_at_100": 0.6958826981617491, + "mrr_at_1000": 0.6960943232255445, + "mrr_at_20": 0.6940146437225909, + "mrr_at_3": 0.676709708273553, + "mrr_at_5": 0.686035389765662, + "nauc_cv_recall_at_1000_diff1": 0.7115337193995271, + "nauc_cv_recall_at_1000_max": 0.9162400747714905, + "nauc_cv_recall_at_1000_std": 0.6394650706109806, + "nauc_cv_recall_at_100_diff1": 0.7333454397110322, + "nauc_cv_recall_at_100_max": 0.7444349099565744, + "nauc_cv_recall_at_100_std": 0.4724436931961091, + "nauc_cv_recall_at_10_diff1": 0.6871809913928805, + "nauc_cv_recall_at_10_max": 0.6133230953694784, + "nauc_cv_recall_at_10_std": 0.2527224723281998, + "nauc_cv_recall_at_1_diff1": 0.8178148233019549, + "nauc_cv_recall_at_1_max": 0.6052419836559261, + "nauc_cv_recall_at_1_std": 0.12987371517585286, + "nauc_cv_recall_at_20_diff1": 0.6981768420463863, + "nauc_cv_recall_at_20_max": 0.6659441432074131, + "nauc_cv_recall_at_20_std": 0.3204684923003592, + "nauc_cv_recall_at_3_diff1": 0.7321559190348417, + "nauc_cv_recall_at_3_max": 0.6343014239279325, + "nauc_cv_recall_at_3_std": 0.15185156111493697, + "nauc_cv_recall_at_5_diff1": 0.7072557341340876, + "nauc_cv_recall_at_5_max": 0.6257250342242687, + "nauc_cv_recall_at_5_std": 0.20404936155319015, + "nauc_map_at_1000_diff1": 0.7887351342678683, + "nauc_map_at_1000_max": 0.6168942239184106, + "nauc_map_at_1000_std": 0.1413630951928354, + "nauc_map_at_100_diff1": 0.7887041995063825, + "nauc_map_at_100_max": 0.6170199486176621, + "nauc_map_at_100_std": 0.1416616343081758, + "nauc_map_at_10_diff1": 0.7862784183796454, + "nauc_map_at_10_max": 0.6146388906025935, + "nauc_map_at_10_std": 0.14075448379320352, + "nauc_map_at_1_diff1": 0.8229562595133094, + "nauc_map_at_1_max": 0.6013677818784777, + "nauc_map_at_1_std": 0.12160003122737216, + "nauc_map_at_20_diff1": 0.7879869553581316, + "nauc_map_at_20_max": 0.6168049743962634, + "nauc_map_at_20_std": 0.14202953814505015, + "nauc_map_at_3_diff1": 0.790122035841549, + "nauc_map_at_3_max": 0.6181113964938436, + "nauc_map_at_3_std": 0.12999774139755105, + "nauc_map_at_5_diff1": 0.7867973534260535, + "nauc_map_at_5_max": 0.6154950412048648, + "nauc_map_at_5_std": 0.13770782443201052, + "nauc_mrr_at_1000_diff1": 0.7849851923349892, + "nauc_mrr_at_1000_max": 0.6191346898024174, + "nauc_mrr_at_1000_std": 0.14684978660096312, + "nauc_mrr_at_100_diff1": 0.7849572405285399, + "nauc_mrr_at_100_max": 0.6192591195776083, + "nauc_mrr_at_100_std": 0.1471451120883124, + "nauc_mrr_at_10_diff1": 0.7825858492004815, + "nauc_mrr_at_10_max": 0.6168284552461306, + "nauc_mrr_at_10_std": 0.14613846483667256, + "nauc_mrr_at_1_diff1": 0.8178148233019549, + "nauc_mrr_at_1_max": 0.6052419836559261, + "nauc_mrr_at_1_std": 0.12987371517585286, + "nauc_mrr_at_20_diff1": 0.784261954129683, + "nauc_mrr_at_20_max": 0.6190281187286515, + "nauc_mrr_at_20_std": 0.1474778049919308, + "nauc_mrr_at_3_diff1": 0.7866102375472549, + "nauc_mrr_at_3_max": 0.6202568371440428, + "nauc_mrr_at_3_std": 0.13510628896209134, + "nauc_mrr_at_5_diff1": 0.7831809807419776, + "nauc_mrr_at_5_max": 0.6176841717561736, + "nauc_mrr_at_5_std": 0.14298098309055324, + "nauc_ndcg_at_1000_diff1": 0.7814139580523798, + "nauc_ndcg_at_1000_max": 0.6221268142813257, + "nauc_ndcg_at_1000_std": 0.1552747016000708, + "nauc_ndcg_at_100_diff1": 0.780344212475686, + "nauc_ndcg_at_100_max": 0.6247734986989426, + "nauc_ndcg_at_100_std": 0.16332676375007413, + "nauc_ndcg_at_10_diff1": 0.7687973801203425, + "nauc_ndcg_at_10_max": 0.6150742030451716, + "nauc_ndcg_at_10_std": 0.15909617277981258, + "nauc_ndcg_at_1_diff1": 0.8229562595133094, + "nauc_ndcg_at_1_max": 0.6013677818784777, + "nauc_ndcg_at_1_std": 0.12160003122737216, + "nauc_ndcg_at_20_diff1": 0.7743559187463246, + "nauc_ndcg_at_20_max": 0.6230320610140855, + "nauc_ndcg_at_20_std": 0.1646520543777805, + "nauc_ndcg_at_3_diff1": 0.7772486128568934, + "nauc_ndcg_at_3_max": 0.6219190572302621, + "nauc_ndcg_at_3_std": 0.13473965557594939, + "nauc_ndcg_at_5_diff1": 0.7707816184815659, + "nauc_ndcg_at_5_max": 0.6178140545282956, + "nauc_ndcg_at_5_std": 0.15038650842581236, + "nauc_precision_at_1000_diff1": 0.7115337193995216, + "nauc_precision_at_1000_max": 0.9162400747714883, + "nauc_precision_at_1000_std": 0.6394650706109937, + "nauc_precision_at_100_diff1": 0.7333454397110306, + "nauc_precision_at_100_max": 0.7444349099565769, + "nauc_precision_at_100_std": 0.4724436931961141, + "nauc_precision_at_10_diff1": 0.6871809913928806, + "nauc_precision_at_10_max": 0.6133230953694784, + "nauc_precision_at_10_std": 0.2527224723281986, + "nauc_precision_at_1_diff1": 0.8229562595133094, + "nauc_precision_at_1_max": 0.6013677818784777, + "nauc_precision_at_1_std": 0.12160003122737216, + "nauc_precision_at_20_diff1": 0.6981768420463869, + "nauc_precision_at_20_max": 0.6659441432074145, + "nauc_precision_at_20_std": 0.32046849230036073, + "nauc_precision_at_3_diff1": 0.7321559190348422, + "nauc_precision_at_3_max": 0.6343014239279321, + "nauc_precision_at_3_std": 0.15185156111493722, + "nauc_precision_at_5_diff1": 0.7072557341340879, + "nauc_precision_at_5_max": 0.6257250342242688, + "nauc_precision_at_5_std": 0.20404936155319042, + "nauc_recall_at_1000_diff1": 0.7115337193995271, + "nauc_recall_at_1000_max": 0.9162400747714905, + "nauc_recall_at_1000_std": 0.6394650706109806, + "nauc_recall_at_100_diff1": 0.7333454397110322, + "nauc_recall_at_100_max": 0.7444349099565744, + "nauc_recall_at_100_std": 0.4724436931961091, + "nauc_recall_at_10_diff1": 0.6871809913928805, + "nauc_recall_at_10_max": 0.6133230953694784, + "nauc_recall_at_10_std": 0.2527224723281998, + "nauc_recall_at_1_diff1": 0.8229562595133094, + "nauc_recall_at_1_max": 0.6013677818784777, + "nauc_recall_at_1_std": 0.12160003122737216, + "nauc_recall_at_20_diff1": 0.6981768420463863, + "nauc_recall_at_20_max": 0.6659441432074131, + "nauc_recall_at_20_std": 0.3204684923003592, + "nauc_recall_at_3_diff1": 0.7321559190348417, + "nauc_recall_at_3_max": 0.6343014239279325, + "nauc_recall_at_3_std": 0.15185156111493697, + "nauc_recall_at_5_diff1": 0.7072557341340876, + "nauc_recall_at_5_max": 0.6257250342242687, + "nauc_recall_at_5_std": 0.20404936155319015, + "ndcg_at_1": 0.61693, + "ndcg_at_10": 0.72207, + "ndcg_at_100": 0.74685, + "ndcg_at_1000": 0.75279, + "ndcg_at_20": 0.73369, + "ndcg_at_3": 0.69382, + "ndcg_at_5": 0.71049, + "precision_at_1": 0.61693, + "precision_at_10": 0.0825, + "precision_at_100": 0.00941, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04354, + "precision_at_3": 0.24964, + "precision_at_5": 0.15782, + "recall_at_1": 0.61693, + "recall_at_10": 0.82496, + "recall_at_100": 0.94118, + "recall_at_1000": 0.98852, + "recall_at_20": 0.87088, + "recall_at_3": 0.74892, + "recall_at_5": 0.7891 + } + ] + }, + "task_name": "MemotionI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionT2IRetrieval.json new file mode 100644 index 0000000000..38858558f2 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", + "evaluation_time": 47.172343254089355, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.7561, + "cv_recall_at_10": 0.89957, + "cv_recall_at_100": 0.95409, + "cv_recall_at_1000": 0.98852, + "cv_recall_at_20": 0.92539, + "cv_recall_at_3": 0.85509, + "cv_recall_at_5": 0.87948, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8304, + "map_at_1": 0.75466, + "map_at_10": 0.80776, + "map_at_100": 0.81037, + "map_at_1000": 0.8105, + "map_at_20": 0.80956, + "map_at_3": 0.79914, + "map_at_5": 0.80495, + "mrr_at_1": 0.7560975609756098, + "mrr_at_10": 0.808956184099656, + "mrr_at_100": 0.8115696597483404, + "mrr_at_1000": 0.811693829214546, + "mrr_at_20": 0.810753790451444, + "mrr_at_3": 0.8003347680535627, + "mrr_at_5": 0.8061453849832614, + "nauc_cv_recall_at_1000_diff1": 0.6706645008987285, + "nauc_cv_recall_at_1000_max": 0.4840668585023845, + "nauc_cv_recall_at_1000_std": 0.5855825248516394, + "nauc_cv_recall_at_100_diff1": 0.7572865775123307, + "nauc_cv_recall_at_100_max": 0.7014046577380366, + "nauc_cv_recall_at_100_std": 0.6971837247747253, + "nauc_cv_recall_at_10_diff1": 0.8240165624741603, + "nauc_cv_recall_at_10_max": 0.6069417503163557, + "nauc_cv_recall_at_10_std": 0.43311682859231987, + "nauc_cv_recall_at_1_diff1": 0.8722966904073866, + "nauc_cv_recall_at_1_max": 0.5299326938244373, + "nauc_cv_recall_at_1_std": 0.22226251845301678, + "nauc_cv_recall_at_20_diff1": 0.7870728454870688, + "nauc_cv_recall_at_20_max": 0.6389615737600375, + "nauc_cv_recall_at_20_std": 0.5199494827002775, + "nauc_cv_recall_at_3_diff1": 0.8326936710385887, + "nauc_cv_recall_at_3_max": 0.5684788343399769, + "nauc_cv_recall_at_3_std": 0.31719170280557873, + "nauc_cv_recall_at_5_diff1": 0.8188729223764644, + "nauc_cv_recall_at_5_max": 0.5606570995953873, + "nauc_cv_recall_at_5_std": 0.3412401813639024, + "nauc_map_at_1000_diff1": 0.8592180617798573, + "nauc_map_at_1000_max": 0.5474455366412145, + "nauc_map_at_1000_std": 0.26314060985977905, + "nauc_map_at_100_diff1": 0.8591578945632715, + "nauc_map_at_100_max": 0.5476108921329297, + "nauc_map_at_100_std": 0.2634699740502252, + "nauc_map_at_10_diff1": 0.85968856552947, + "nauc_map_at_10_max": 0.5474144301133114, + "nauc_map_at_10_std": 0.26240890326221095, + "nauc_map_at_1_diff1": 0.8747133997178218, + "nauc_map_at_1_max": 0.532150450714701, + "nauc_map_at_1_std": 0.22774606041660467, + "nauc_map_at_20_diff1": 0.8590199843105324, + "nauc_map_at_20_max": 0.5476380049340864, + "nauc_map_at_20_std": 0.2630842730031387, + "nauc_map_at_3_diff1": 0.8601043415886783, + "nauc_map_at_3_max": 0.5455420632431127, + "nauc_map_at_3_std": 0.25409460690905056, + "nauc_map_at_5_diff1": 0.8585592346276657, + "nauc_map_at_5_max": 0.5439055718426468, + "nauc_map_at_5_std": 0.25610005430887645, + "nauc_mrr_at_1000_diff1": 0.8571279974501776, + "nauc_mrr_at_1000_max": 0.5466442414674472, + "nauc_mrr_at_1000_std": 0.2620528722535252, + "nauc_mrr_at_100_diff1": 0.8570690034765716, + "nauc_mrr_at_100_max": 0.5468110619980555, + "nauc_mrr_at_100_std": 0.26238479039821916, + "nauc_mrr_at_10_diff1": 0.8576335229981322, + "nauc_mrr_at_10_max": 0.5466254391056721, + "nauc_mrr_at_10_std": 0.2613342084530473, + "nauc_mrr_at_1_diff1": 0.8722966904073866, + "nauc_mrr_at_1_max": 0.5299326938244373, + "nauc_mrr_at_1_std": 0.22226251845301678, + "nauc_mrr_at_20_diff1": 0.8569401410607558, + "nauc_mrr_at_20_max": 0.5468420677776812, + "nauc_mrr_at_20_std": 0.2620021054881723, + "nauc_mrr_at_3_diff1": 0.8581353780119624, + "nauc_mrr_at_3_max": 0.5447757752206631, + "nauc_mrr_at_3_std": 0.25302149099138316, + "nauc_mrr_at_5_diff1": 0.856526428569613, + "nauc_mrr_at_5_max": 0.5431090179640301, + "nauc_mrr_at_5_std": 0.2550070511531312, + "nauc_ndcg_at_1000_diff1": 0.8530331244324216, + "nauc_ndcg_at_1000_max": 0.5523802225859098, + "nauc_ndcg_at_1000_std": 0.2828500300468584, + "nauc_ndcg_at_100_diff1": 0.8510912813706918, + "nauc_ndcg_at_100_max": 0.5586724956727237, + "nauc_ndcg_at_100_std": 0.29552202763851937, + "nauc_ndcg_at_10_diff1": 0.8537108262779001, + "nauc_ndcg_at_10_max": 0.5565012931379493, + "nauc_ndcg_at_10_std": 0.2884764507742189, + "nauc_ndcg_at_1_diff1": 0.8747133997178218, + "nauc_ndcg_at_1_max": 0.532150450714701, + "nauc_ndcg_at_1_std": 0.22774606041660467, + "nauc_ndcg_at_20_diff1": 0.8504706937817262, + "nauc_ndcg_at_20_max": 0.5581650724327955, + "nauc_ndcg_at_20_std": 0.29286385001715937, + "nauc_ndcg_at_3_diff1": 0.8544076459383539, + "nauc_ndcg_at_3_max": 0.5503441549211755, + "nauc_ndcg_at_3_std": 0.2669883605033744, + "nauc_ndcg_at_5_diff1": 0.8512619509841919, + "nauc_ndcg_at_5_max": 0.5472168141098742, + "nauc_ndcg_at_5_std": 0.2715472109001755, + "nauc_precision_at_1000_diff1": 0.67066450089872, + "nauc_precision_at_1000_max": 0.48406685850238695, + "nauc_precision_at_1000_std": 0.5855825248516409, + "nauc_precision_at_100_diff1": 0.7572865775123279, + "nauc_precision_at_100_max": 0.7014046577380433, + "nauc_precision_at_100_std": 0.6971837247747307, + "nauc_precision_at_10_diff1": 0.8240165624741588, + "nauc_precision_at_10_max": 0.6069417503163547, + "nauc_precision_at_10_std": 0.43311682859231926, + "nauc_precision_at_1_diff1": 0.8747133997178218, + "nauc_precision_at_1_max": 0.532150450714701, + "nauc_precision_at_1_std": 0.22774606041660467, + "nauc_precision_at_20_diff1": 0.7870728454870688, + "nauc_precision_at_20_max": 0.6389615737600398, + "nauc_precision_at_20_std": 0.5199494827002775, + "nauc_precision_at_3_diff1": 0.8326936710385894, + "nauc_precision_at_3_max": 0.5684788343399768, + "nauc_precision_at_3_std": 0.31719170280557873, + "nauc_precision_at_5_diff1": 0.8188729223764643, + "nauc_precision_at_5_max": 0.5606570995953888, + "nauc_precision_at_5_std": 0.3412401813639036, + "nauc_recall_at_1000_diff1": 0.6706645008987285, + "nauc_recall_at_1000_max": 0.4840668585023845, + "nauc_recall_at_1000_std": 0.5855825248516394, + "nauc_recall_at_100_diff1": 0.7572865775123307, + "nauc_recall_at_100_max": 0.7014046577380366, + "nauc_recall_at_100_std": 0.6971837247747253, + "nauc_recall_at_10_diff1": 0.8240165624741603, + "nauc_recall_at_10_max": 0.6069417503163557, + "nauc_recall_at_10_std": 0.43311682859231987, + "nauc_recall_at_1_diff1": 0.8747133997178218, + "nauc_recall_at_1_max": 0.532150450714701, + "nauc_recall_at_1_std": 0.22774606041660467, + "nauc_recall_at_20_diff1": 0.7870728454870688, + "nauc_recall_at_20_max": 0.6389615737600375, + "nauc_recall_at_20_std": 0.5199494827002775, + "nauc_recall_at_3_diff1": 0.8326936710385887, + "nauc_recall_at_3_max": 0.5684788343399769, + "nauc_recall_at_3_std": 0.31719170280557873, + "nauc_recall_at_5_diff1": 0.8188729223764644, + "nauc_recall_at_5_max": 0.5606570995953873, + "nauc_recall_at_5_std": 0.3412401813639024, + "ndcg_at_1": 0.75466, + "ndcg_at_10": 0.8304, + "ndcg_at_100": 0.84243, + "ndcg_at_1000": 0.84657, + "ndcg_at_20": 0.83694, + "ndcg_at_3": 0.81352, + "ndcg_at_5": 0.82377, + "precision_at_1": 0.75466, + "precision_at_10": 0.08996, + "precision_at_100": 0.00954, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04627, + "precision_at_3": 0.28503, + "precision_at_5": 0.1759, + "recall_at_1": 0.75466, + "recall_at_10": 0.89957, + "recall_at_100": 0.95409, + "recall_at_1000": 0.98852, + "recall_at_20": 0.92539, + "recall_at_3": 0.85509, + "recall_at_5": 0.87948 + } + ] + }, + "task_name": "MemotionT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionI2TRetrieval.json new file mode 100644 index 0000000000..6f097133d0 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", + "evaluation_time": 9.379658937454224, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.54089, + "cv_recall_at_10": 0.76327, + "cv_recall_at_100": 0.90961, + "cv_recall_at_1000": 0.97561, + "cv_recall_at_20": 0.81492, + "cv_recall_at_3": 0.66858, + "cv_recall_at_5": 0.70014, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.64846, + "map_at_1": 0.53945, + "map_at_10": 0.61219, + "map_at_100": 0.61816, + "map_at_1000": 0.61842, + "map_at_20": 0.61573, + "map_at_3": 0.5966, + "map_at_5": 0.60385, + "mrr_at_1": 0.5408895265423243, + "mrr_at_10": 0.6129033727312059, + "mrr_at_100": 0.6188781791949352, + "mrr_at_1000": 0.6191415311661694, + "mrr_at_20": 0.6164480169192975, + "mrr_at_3": 0.5973218555714968, + "mrr_at_5": 0.6045671927307505, + "nauc_cv_recall_at_1000_diff1": 0.6031068526863032, + "nauc_cv_recall_at_1000_max": 0.6187947052677972, + "nauc_cv_recall_at_1000_std": 0.13614884941799502, + "nauc_cv_recall_at_100_diff1": 0.6384413822343304, + "nauc_cv_recall_at_100_max": 0.6568440212607971, + "nauc_cv_recall_at_100_std": 0.36389752289108457, + "nauc_cv_recall_at_10_diff1": 0.6436032470889269, + "nauc_cv_recall_at_10_max": 0.5770073527151407, + "nauc_cv_recall_at_10_std": 0.1248391391939967, + "nauc_cv_recall_at_1_diff1": 0.7550158265678641, + "nauc_cv_recall_at_1_max": 0.5433476926735126, + "nauc_cv_recall_at_1_std": 0.08333119803942535, + "nauc_cv_recall_at_20_diff1": 0.6545130104125019, + "nauc_cv_recall_at_20_max": 0.626723137941904, + "nauc_cv_recall_at_20_std": 0.26198168043372594, + "nauc_cv_recall_at_3_diff1": 0.6830756998324176, + "nauc_cv_recall_at_3_max": 0.602332850939447, + "nauc_cv_recall_at_3_std": 0.14321032322791108, + "nauc_cv_recall_at_5_diff1": 0.6562142711894405, + "nauc_cv_recall_at_5_max": 0.5739787707563608, + "nauc_cv_recall_at_5_std": 0.12240833670188953, + "nauc_map_at_1000_diff1": 0.7273224554410895, + "nauc_map_at_1000_max": 0.5611946300973509, + "nauc_map_at_1000_std": 0.10422567634452316, + "nauc_map_at_100_diff1": 0.7272509146321907, + "nauc_map_at_100_max": 0.5612908433321292, + "nauc_map_at_100_std": 0.10454572772341748, + "nauc_map_at_10_diff1": 0.725549268371285, + "nauc_map_at_10_max": 0.5592550980691888, + "nauc_map_at_10_std": 0.0990385207650767, + "nauc_map_at_1_diff1": 0.758129112404582, + "nauc_map_at_1_max": 0.537597678667553, + "nauc_map_at_1_std": 0.08080106485075698, + "nauc_map_at_20_diff1": 0.726514594912638, + "nauc_map_at_20_max": 0.5609091488732393, + "nauc_map_at_20_std": 0.10421116489530967, + "nauc_map_at_3_diff1": 0.729828848291311, + "nauc_map_at_3_max": 0.5651513147787425, + "nauc_map_at_3_std": 0.10316360192499197, + "nauc_map_at_5_diff1": 0.7253689641307103, + "nauc_map_at_5_max": 0.5590662677071627, + "nauc_map_at_5_std": 0.09856405589506938, + "nauc_mrr_at_1000_diff1": 0.7255248516916601, + "nauc_mrr_at_1000_max": 0.5644856932022566, + "nauc_mrr_at_1000_std": 0.10568520167017303, + "nauc_mrr_at_100_diff1": 0.7254548437423526, + "nauc_mrr_at_100_max": 0.5645790694557368, + "nauc_mrr_at_100_std": 0.10600420119309457, + "nauc_mrr_at_10_diff1": 0.7237804426229136, + "nauc_mrr_at_10_max": 0.5624890366615415, + "nauc_mrr_at_10_std": 0.10046897577029043, + "nauc_mrr_at_1_diff1": 0.7550158265678641, + "nauc_mrr_at_1_max": 0.5433476926735126, + "nauc_mrr_at_1_std": 0.08333119803942535, + "nauc_mrr_at_20_diff1": 0.7247307286147782, + "nauc_mrr_at_20_max": 0.5641735376688276, + "nauc_mrr_at_20_std": 0.10565892278323516, + "nauc_mrr_at_3_diff1": 0.7281207899682096, + "nauc_mrr_at_3_max": 0.5682855124046605, + "nauc_mrr_at_3_std": 0.10455119493617923, + "nauc_mrr_at_5_diff1": 0.7236347661753463, + "nauc_mrr_at_5_max": 0.5622364615091957, + "nauc_mrr_at_5_std": 0.09996598700114626, + "nauc_ndcg_at_1000_diff1": 0.719295351046238, + "nauc_ndcg_at_1000_max": 0.5672932873311153, + "nauc_ndcg_at_1000_std": 0.11522678717518038, + "nauc_ndcg_at_100_diff1": 0.7170409644450718, + "nauc_ndcg_at_100_max": 0.5705269633295453, + "nauc_ndcg_at_100_std": 0.1259050314954282, + "nauc_ndcg_at_10_diff1": 0.709841562609803, + "nauc_ndcg_at_10_max": 0.5633088061734607, + "nauc_ndcg_at_10_std": 0.10457845445884532, + "nauc_ndcg_at_1_diff1": 0.758129112404582, + "nauc_ndcg_at_1_max": 0.537597678667553, + "nauc_ndcg_at_1_std": 0.08080106485075698, + "nauc_ndcg_at_20_diff1": 0.7137629800374607, + "nauc_ndcg_at_20_max": 0.5706475244876037, + "nauc_ndcg_at_20_std": 0.12599765657967227, + "nauc_ndcg_at_3_diff1": 0.7188796629698468, + "nauc_ndcg_at_3_max": 0.5739922976173544, + "nauc_ndcg_at_3_std": 0.11249256157795794, + "nauc_ndcg_at_5_diff1": 0.7101347667270254, + "nauc_ndcg_at_5_max": 0.5629182818624199, + "nauc_ndcg_at_5_std": 0.10415403042320356, + "nauc_precision_at_1000_diff1": 0.6031068526863047, + "nauc_precision_at_1000_max": 0.6187947052677932, + "nauc_precision_at_1000_std": 0.13614884941798686, + "nauc_precision_at_100_diff1": 0.6384413822343313, + "nauc_precision_at_100_max": 0.6568440212607987, + "nauc_precision_at_100_std": 0.3638975228910859, + "nauc_precision_at_10_diff1": 0.6436032470889286, + "nauc_precision_at_10_max": 0.5770073527151411, + "nauc_precision_at_10_std": 0.12483913919399606, + "nauc_precision_at_1_diff1": 0.758129112404582, + "nauc_precision_at_1_max": 0.537597678667553, + "nauc_precision_at_1_std": 0.08080106485075698, + "nauc_precision_at_20_diff1": 0.6545130104125023, + "nauc_precision_at_20_max": 0.6267231379419045, + "nauc_precision_at_20_std": 0.2619816804337271, + "nauc_precision_at_3_diff1": 0.6830756998324173, + "nauc_precision_at_3_max": 0.6023328509394469, + "nauc_precision_at_3_std": 0.14321032322791036, + "nauc_precision_at_5_diff1": 0.656214271189441, + "nauc_precision_at_5_max": 0.5739787707563612, + "nauc_precision_at_5_std": 0.12240833670189052, + "nauc_recall_at_1000_diff1": 0.6031068526863032, + "nauc_recall_at_1000_max": 0.6187947052677972, + "nauc_recall_at_1000_std": 0.13614884941799502, + "nauc_recall_at_100_diff1": 0.6384413822343304, + "nauc_recall_at_100_max": 0.6568440212607971, + "nauc_recall_at_100_std": 0.36389752289108457, + "nauc_recall_at_10_diff1": 0.6436032470889269, + "nauc_recall_at_10_max": 0.5770073527151407, + "nauc_recall_at_10_std": 0.1248391391939967, + "nauc_recall_at_1_diff1": 0.758129112404582, + "nauc_recall_at_1_max": 0.537597678667553, + "nauc_recall_at_1_std": 0.08080106485075698, + "nauc_recall_at_20_diff1": 0.6545130104125019, + "nauc_recall_at_20_max": 0.626723137941904, + "nauc_recall_at_20_std": 0.26198168043372594, + "nauc_recall_at_3_diff1": 0.6830756998324176, + "nauc_recall_at_3_max": 0.602332850939447, + "nauc_recall_at_3_std": 0.14321032322791108, + "nauc_recall_at_5_diff1": 0.6562142711894405, + "nauc_recall_at_5_max": 0.5739787707563608, + "nauc_recall_at_5_std": 0.12240833670188953, + "ndcg_at_1": 0.53945, + "ndcg_at_10": 0.64846, + "ndcg_at_100": 0.67903, + "ndcg_at_1000": 0.68716, + "ndcg_at_20": 0.66146, + "ndcg_at_3": 0.6151, + "ndcg_at_5": 0.62813, + "precision_at_1": 0.53945, + "precision_at_10": 0.07633, + "precision_at_100": 0.0091, + "precision_at_1000": 0.00098, + "precision_at_20": 0.04075, + "precision_at_3": 0.22286, + "precision_at_5": 0.14003, + "recall_at_1": 0.53945, + "recall_at_10": 0.76327, + "recall_at_100": 0.90961, + "recall_at_1000": 0.97561, + "recall_at_20": 0.81492, + "recall_at_3": 0.66858, + "recall_at_5": 0.70014 + } + ] + }, + "task_name": "MemotionI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionT2IRetrieval.json new file mode 100644 index 0000000000..df3428cbb9 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", + "evaluation_time": 35.66201996803284, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.66571, + "cv_recall_at_10": 0.8594, + "cv_recall_at_100": 0.93831, + "cv_recall_at_1000": 0.97561, + "cv_recall_at_20": 0.89813, + "cv_recall_at_3": 0.77618, + "cv_recall_at_5": 0.80344, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.75862, + "map_at_1": 0.66284, + "map_at_10": 0.72675, + "map_at_100": 0.73055, + "map_at_1000": 0.73072, + "map_at_20": 0.7296, + "map_at_3": 0.71258, + "map_at_5": 0.71911, + "mrr_at_1": 0.6657101865136298, + "mrr_at_10": 0.7285418232333586, + "mrr_at_100": 0.7323483961958512, + "mrr_at_1000": 0.732515458568796, + "mrr_at_20": 0.7313936571835862, + "mrr_at_3": 0.7147297943567668, + "mrr_at_5": 0.7208990913438542, + "nauc_cv_recall_at_1000_diff1": 0.7275510355633449, + "nauc_cv_recall_at_1000_max": 0.6270510775474021, + "nauc_cv_recall_at_1000_std": 0.6977342597863756, + "nauc_cv_recall_at_100_diff1": 0.67409025551175, + "nauc_cv_recall_at_100_max": 0.7187642379876352, + "nauc_cv_recall_at_100_std": 0.7412802975591734, + "nauc_cv_recall_at_10_diff1": 0.7440079572260425, + "nauc_cv_recall_at_10_max": 0.6116559774821306, + "nauc_cv_recall_at_10_std": 0.39207495561469424, + "nauc_cv_recall_at_1_diff1": 0.8221009762961692, + "nauc_cv_recall_at_1_max": 0.5448218971848086, + "nauc_cv_recall_at_1_std": 0.2320609220538416, + "nauc_cv_recall_at_20_diff1": 0.7031663816757069, + "nauc_cv_recall_at_20_max": 0.6291439507988603, + "nauc_cv_recall_at_20_std": 0.5077566395661355, + "nauc_cv_recall_at_3_diff1": 0.7702920446965164, + "nauc_cv_recall_at_3_max": 0.5372157631138539, + "nauc_cv_recall_at_3_std": 0.2549429305075186, + "nauc_cv_recall_at_5_diff1": 0.7650835866602818, + "nauc_cv_recall_at_5_max": 0.5442226414009286, + "nauc_cv_recall_at_5_std": 0.28724135516500227, + "nauc_map_at_1000_diff1": 0.8062791119801845, + "nauc_map_at_1000_max": 0.5506991188965215, + "nauc_map_at_1000_std": 0.25596886109538064, + "nauc_map_at_100_diff1": 0.8061675355941809, + "nauc_map_at_100_max": 0.550882192093261, + "nauc_map_at_100_std": 0.2563613670042926, + "nauc_map_at_10_diff1": 0.8066919819754693, + "nauc_map_at_10_max": 0.5503929884485892, + "nauc_map_at_10_std": 0.2528355504090529, + "nauc_map_at_1_diff1": 0.8276857924244969, + "nauc_map_at_1_max": 0.54640376432564, + "nauc_map_at_1_std": 0.23200436630899549, + "nauc_map_at_20_diff1": 0.8059529978723045, + "nauc_map_at_20_max": 0.5505629223268083, + "nauc_map_at_20_std": 0.25567849411462323, + "nauc_map_at_3_diff1": 0.8069803190393084, + "nauc_map_at_3_max": 0.5415664121870912, + "nauc_map_at_3_std": 0.23722873621401488, + "nauc_map_at_5_diff1": 0.8066155055231402, + "nauc_map_at_5_max": 0.543970624015228, + "nauc_map_at_5_std": 0.2448945089486877, + "nauc_mrr_at_1000_diff1": 0.8023950467283947, + "nauc_mrr_at_1000_max": 0.5512541672454327, + "nauc_mrr_at_1000_std": 0.25907486182158634, + "nauc_mrr_at_100_diff1": 0.8022857703546528, + "nauc_mrr_at_100_max": 0.5514376745671156, + "nauc_mrr_at_100_std": 0.259466912692116, + "nauc_mrr_at_10_diff1": 0.8028693166599788, + "nauc_mrr_at_10_max": 0.5509381510440614, + "nauc_mrr_at_10_std": 0.25587984389286694, + "nauc_mrr_at_1_diff1": 0.8221009762961692, + "nauc_mrr_at_1_max": 0.5448218971848086, + "nauc_mrr_at_1_std": 0.2320609220538416, + "nauc_mrr_at_20_diff1": 0.8020861161859157, + "nauc_mrr_at_20_max": 0.5511146841530047, + "nauc_mrr_at_20_std": 0.2587682482356528, + "nauc_mrr_at_3_diff1": 0.8029147313834037, + "nauc_mrr_at_3_max": 0.5434426114809214, + "nauc_mrr_at_3_std": 0.24285713945064882, + "nauc_mrr_at_5_diff1": 0.8028978569938292, + "nauc_mrr_at_5_max": 0.5444726737717525, + "nauc_mrr_at_5_std": 0.24782017249190733, + "nauc_ndcg_at_1000_diff1": 0.7990426525026079, + "nauc_ndcg_at_1000_max": 0.5568838813048612, + "nauc_ndcg_at_1000_std": 0.2780511118290086, + "nauc_ndcg_at_100_diff1": 0.7952466149552354, + "nauc_ndcg_at_100_max": 0.562318000567148, + "nauc_ndcg_at_100_std": 0.2899863705321415, + "nauc_ndcg_at_10_diff1": 0.7967440189968154, + "nauc_ndcg_at_10_max": 0.5591866658895446, + "nauc_ndcg_at_10_std": 0.2735502422966261, + "nauc_ndcg_at_1_diff1": 0.8276857924244969, + "nauc_ndcg_at_1_max": 0.54640376432564, + "nauc_ndcg_at_1_std": 0.23200436630899549, + "nauc_ndcg_at_20_diff1": 0.7935903921913575, + "nauc_ndcg_at_20_max": 0.5595600940092564, + "nauc_ndcg_at_20_std": 0.2842931614515233, + "nauc_ndcg_at_3_diff1": 0.7992628243453683, + "nauc_ndcg_at_3_max": 0.5391546646696291, + "nauc_ndcg_at_3_std": 0.23825763951943946, + "nauc_ndcg_at_5_diff1": 0.7980409986597947, + "nauc_ndcg_at_5_max": 0.5438585983101174, + "nauc_ndcg_at_5_std": 0.25319042848202133, + "nauc_precision_at_1000_diff1": 0.727551035563347, + "nauc_precision_at_1000_max": 0.6270510775474053, + "nauc_precision_at_1000_std": 0.697734259786382, + "nauc_precision_at_100_diff1": 0.6740902555117545, + "nauc_precision_at_100_max": 0.7187642379876428, + "nauc_precision_at_100_std": 0.7412802975591787, + "nauc_precision_at_10_diff1": 0.7440079572260412, + "nauc_precision_at_10_max": 0.6116559774821304, + "nauc_precision_at_10_std": 0.39207495561469397, + "nauc_precision_at_1_diff1": 0.8276857924244969, + "nauc_precision_at_1_max": 0.54640376432564, + "nauc_precision_at_1_std": 0.23200436630899549, + "nauc_precision_at_20_diff1": 0.7031663816757052, + "nauc_precision_at_20_max": 0.6291439507988585, + "nauc_precision_at_20_std": 0.5077566395661335, + "nauc_precision_at_3_diff1": 0.7724802552153682, + "nauc_precision_at_3_max": 0.5305632143015199, + "nauc_precision_at_3_std": 0.24149371118183455, + "nauc_precision_at_5_diff1": 0.7650835866602826, + "nauc_precision_at_5_max": 0.5442226414009279, + "nauc_precision_at_5_std": 0.28724135516500265, + "nauc_recall_at_1000_diff1": 0.7275510355633449, + "nauc_recall_at_1000_max": 0.6270510775474021, + "nauc_recall_at_1000_std": 0.6977342597863756, + "nauc_recall_at_100_diff1": 0.67409025551175, + "nauc_recall_at_100_max": 0.7187642379876352, + "nauc_recall_at_100_std": 0.7412802975591734, + "nauc_recall_at_10_diff1": 0.7440079572260425, + "nauc_recall_at_10_max": 0.6116559774821306, + "nauc_recall_at_10_std": 0.39207495561469424, + "nauc_recall_at_1_diff1": 0.8276857924244969, + "nauc_recall_at_1_max": 0.54640376432564, + "nauc_recall_at_1_std": 0.23200436630899549, + "nauc_recall_at_20_diff1": 0.7031663816757069, + "nauc_recall_at_20_max": 0.6291439507988603, + "nauc_recall_at_20_std": 0.5077566395661355, + "nauc_recall_at_3_diff1": 0.772480255215368, + "nauc_recall_at_3_max": 0.5305632143015196, + "nauc_recall_at_3_std": 0.24149371118183494, + "nauc_recall_at_5_diff1": 0.7650835866602818, + "nauc_recall_at_5_max": 0.5442226414009286, + "nauc_recall_at_5_std": 0.28724135516500227, + "ndcg_at_1": 0.66284, + "ndcg_at_10": 0.75862, + "ndcg_at_100": 0.77596, + "ndcg_at_1000": 0.78066, + "ndcg_at_20": 0.76862, + "ndcg_at_3": 0.72856, + "ndcg_at_5": 0.74036, + "precision_at_1": 0.66284, + "precision_at_10": 0.08594, + "precision_at_100": 0.00938, + "precision_at_1000": 0.00098, + "precision_at_20": 0.04491, + "precision_at_3": 0.25825, + "precision_at_5": 0.16069, + "recall_at_1": 0.66284, + "recall_at_10": 0.8594, + "recall_at_100": 0.93831, + "recall_at_1000": 0.97561, + "recall_at_20": 0.89813, + "recall_at_3": 0.77475, + "recall_at_5": 0.80344 + } + ] + }, + "task_name": "MemotionT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionI2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionI2TRetrieval.json new file mode 100644 index 0000000000..54da1c305d --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", + "evaluation_time": 19.325831651687622, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.66571, + "cv_recall_at_10": 0.86514, + "cv_recall_at_100": 0.9627, + "cv_recall_at_1000": 0.99139, + "cv_recall_at_20": 0.90387, + "cv_recall_at_3": 0.77762, + "cv_recall_at_5": 0.81492, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7633, + "map_at_1": 0.66571, + "map_at_10": 0.73104, + "map_at_100": 0.73511, + "map_at_1000": 0.73523, + "map_at_20": 0.73364, + "map_at_3": 0.71593, + "map_at_5": 0.72439, + "mrr_at_1": 0.6657101865136298, + "mrr_at_10": 0.7310355036323473, + "mrr_at_100": 0.7351134678303535, + "mrr_at_1000": 0.7352377709339848, + "mrr_at_20": 0.7336445586812597, + "mrr_at_3": 0.7159253945480629, + "mrr_at_5": 0.7243902439024387, + "nauc_cv_recall_at_1000_diff1": 0.6899284883121061, + "nauc_cv_recall_at_1000_max": 0.7332667407845029, + "nauc_cv_recall_at_1000_std": 0.9039084682819682, + "nauc_cv_recall_at_100_diff1": 0.7201839624155785, + "nauc_cv_recall_at_100_max": 0.6907472572724536, + "nauc_cv_recall_at_100_std": 0.3332725327931793, + "nauc_cv_recall_at_10_diff1": 0.7077842881191173, + "nauc_cv_recall_at_10_max": 0.6440495623955013, + "nauc_cv_recall_at_10_std": 0.22920450783080562, + "nauc_cv_recall_at_1_diff1": 0.8247920662804782, + "nauc_cv_recall_at_1_max": 0.6475202783168332, + "nauc_cv_recall_at_1_std": 0.18738440780550997, + "nauc_cv_recall_at_20_diff1": 0.7505468149526092, + "nauc_cv_recall_at_20_max": 0.71884925093909, + "nauc_cv_recall_at_20_std": 0.33390742592073563, + "nauc_cv_recall_at_3_diff1": 0.7864899607118706, + "nauc_cv_recall_at_3_max": 0.6854512548125032, + "nauc_cv_recall_at_3_std": 0.2128102726462487, + "nauc_cv_recall_at_5_diff1": 0.7639985062373839, + "nauc_cv_recall_at_5_max": 0.6612189478923202, + "nauc_cv_recall_at_5_std": 0.22878287705869715, + "nauc_map_at_1000_diff1": 0.8061418496180208, + "nauc_map_at_1000_max": 0.6558110507203014, + "nauc_map_at_1000_std": 0.19895557515971368, + "nauc_map_at_100_diff1": 0.8060998402466953, + "nauc_map_at_100_max": 0.6558378199002864, + "nauc_map_at_100_std": 0.19895388464990604, + "nauc_map_at_10_diff1": 0.8039242682913358, + "nauc_map_at_10_max": 0.6543112939934062, + "nauc_map_at_10_std": 0.19680737714854116, + "nauc_map_at_1_diff1": 0.8247920662804782, + "nauc_map_at_1_max": 0.6475202783168332, + "nauc_map_at_1_std": 0.18738440780550997, + "nauc_map_at_20_diff1": 0.8060382107450633, + "nauc_map_at_20_max": 0.6563399027924791, + "nauc_map_at_20_std": 0.19964630412672676, + "nauc_map_at_3_diff1": 0.8102967459949728, + "nauc_map_at_3_max": 0.6604941579272117, + "nauc_map_at_3_std": 0.1949003281855533, + "nauc_map_at_5_diff1": 0.80709000272496, + "nauc_map_at_5_max": 0.656058091855951, + "nauc_map_at_5_std": 0.19791335976271474, + "nauc_mrr_at_1000_diff1": 0.8061349415976898, + "nauc_mrr_at_1000_max": 0.6558011266170611, + "nauc_mrr_at_1000_std": 0.19893648517694507, + "nauc_mrr_at_100_diff1": 0.8060929351394873, + "nauc_mrr_at_100_max": 0.6558279017289936, + "nauc_mrr_at_100_std": 0.19893480501137517, + "nauc_mrr_at_10_diff1": 0.8039242682913358, + "nauc_mrr_at_10_max": 0.6543112939934062, + "nauc_mrr_at_10_std": 0.19680737714854116, + "nauc_mrr_at_1_diff1": 0.8247920662804782, + "nauc_mrr_at_1_max": 0.6475202783168332, + "nauc_mrr_at_1_std": 0.18738440780550997, + "nauc_mrr_at_20_diff1": 0.8060313484873587, + "nauc_mrr_at_20_max": 0.6563300579354437, + "nauc_mrr_at_20_std": 0.19962736007518378, + "nauc_mrr_at_3_diff1": 0.8102967459949728, + "nauc_mrr_at_3_max": 0.6604941579272117, + "nauc_mrr_at_3_std": 0.1949003281855533, + "nauc_mrr_at_5_diff1": 0.80709000272496, + "nauc_mrr_at_5_max": 0.656058091855951, + "nauc_mrr_at_5_std": 0.19791335976271474, + "nauc_ndcg_at_1000_diff1": 0.8006827413362921, + "nauc_ndcg_at_1000_max": 0.6578663386654083, + "nauc_ndcg_at_1000_std": 0.20815645113293776, + "nauc_ndcg_at_100_diff1": 0.7993042173321111, + "nauc_ndcg_at_100_max": 0.6583569899337858, + "nauc_ndcg_at_100_std": 0.2073560131798155, + "nauc_ndcg_at_10_diff1": 0.7895163176975294, + "nauc_ndcg_at_10_max": 0.6533321610894864, + "nauc_ndcg_at_10_std": 0.2020008736673157, + "nauc_ndcg_at_1_diff1": 0.8247920662804782, + "nauc_ndcg_at_1_max": 0.6475202783168332, + "nauc_ndcg_at_1_std": 0.18738440780550997, + "nauc_ndcg_at_20_diff1": 0.7981126769738102, + "nauc_ndcg_at_20_max": 0.6625089082782497, + "nauc_ndcg_at_20_std": 0.21380178270542483, + "nauc_ndcg_at_3_diff1": 0.8049686594776644, + "nauc_ndcg_at_3_max": 0.6660170615545122, + "nauc_ndcg_at_3_std": 0.19881314216111726, + "nauc_ndcg_at_5_diff1": 0.7986014653829432, + "nauc_ndcg_at_5_max": 0.6574369041289396, + "nauc_ndcg_at_5_std": 0.20400546764153435, + "nauc_precision_at_1000_diff1": 0.6899284883120989, + "nauc_precision_at_1000_max": 0.7332667407844798, + "nauc_precision_at_1000_std": 0.9039084682819483, + "nauc_precision_at_100_diff1": 0.7201839624155809, + "nauc_precision_at_100_max": 0.6907472572724539, + "nauc_precision_at_100_std": 0.33327253279318164, + "nauc_precision_at_10_diff1": 0.7077842881191184, + "nauc_precision_at_10_max": 0.6440495623955018, + "nauc_precision_at_10_std": 0.22920450783080423, + "nauc_precision_at_1_diff1": 0.8247920662804782, + "nauc_precision_at_1_max": 0.6475202783168332, + "nauc_precision_at_1_std": 0.18738440780550997, + "nauc_precision_at_20_diff1": 0.7505468149526078, + "nauc_precision_at_20_max": 0.7188492509390902, + "nauc_precision_at_20_std": 0.33390742592073597, + "nauc_precision_at_3_diff1": 0.786489960711871, + "nauc_precision_at_3_max": 0.6854512548125032, + "nauc_precision_at_3_std": 0.21281027264624763, + "nauc_precision_at_5_diff1": 0.763998506237385, + "nauc_precision_at_5_max": 0.661218947892321, + "nauc_precision_at_5_std": 0.22878287705869702, + "nauc_recall_at_1000_diff1": 0.6899284883121061, + "nauc_recall_at_1000_max": 0.7332667407845029, + "nauc_recall_at_1000_std": 0.9039084682819682, + "nauc_recall_at_100_diff1": 0.7201839624155785, + "nauc_recall_at_100_max": 0.6907472572724536, + "nauc_recall_at_100_std": 0.3332725327931793, + "nauc_recall_at_10_diff1": 0.7077842881191173, + "nauc_recall_at_10_max": 0.6440495623955013, + "nauc_recall_at_10_std": 0.22920450783080562, + "nauc_recall_at_1_diff1": 0.8247920662804782, + "nauc_recall_at_1_max": 0.6475202783168332, + "nauc_recall_at_1_std": 0.18738440780550997, + "nauc_recall_at_20_diff1": 0.7505468149526092, + "nauc_recall_at_20_max": 0.71884925093909, + "nauc_recall_at_20_std": 0.33390742592073563, + "nauc_recall_at_3_diff1": 0.7864899607118706, + "nauc_recall_at_3_max": 0.6854512548125032, + "nauc_recall_at_3_std": 0.2128102726462487, + "nauc_recall_at_5_diff1": 0.7639985062373839, + "nauc_recall_at_5_max": 0.6612189478923202, + "nauc_recall_at_5_std": 0.22878287705869715, + "ndcg_at_1": 0.66571, + "ndcg_at_10": 0.7633, + "ndcg_at_100": 0.78382, + "ndcg_at_1000": 0.78738, + "ndcg_at_20": 0.77298, + "ndcg_at_3": 0.73181, + "ndcg_at_5": 0.74712, + "precision_at_1": 0.66571, + "precision_at_10": 0.08651, + "precision_at_100": 0.00963, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04519, + "precision_at_3": 0.25921, + "precision_at_5": 0.16298, + "recall_at_1": 0.66571, + "recall_at_10": 0.86514, + "recall_at_100": 0.9627, + "recall_at_1000": 0.99139, + "recall_at_20": 0.90387, + "recall_at_3": 0.77762, + "recall_at_5": 0.81492 + } + ] + }, + "task_name": "MemotionI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionT2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionT2IRetrieval.json new file mode 100644 index 0000000000..cc31450ae6 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", + "evaluation_time": 104.71015620231628, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.80057, + "cv_recall_at_10": 0.92539, + "cv_recall_at_100": 0.97131, + "cv_recall_at_1000": 0.99139, + "cv_recall_at_20": 0.94692, + "cv_recall_at_3": 0.88235, + "cv_recall_at_5": 0.90387, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.86489, + "map_at_1": 0.79914, + "map_at_10": 0.84527, + "map_at_100": 0.84754, + "map_at_1000": 0.84764, + "map_at_20": 0.84696, + "map_at_3": 0.83716, + "map_at_5": 0.8424, + "mrr_at_1": 0.8005738880918221, + "mrr_at_10": 0.8461034820432242, + "mrr_at_100": 0.8483797957589674, + "mrr_at_1000": 0.8484791559369318, + "mrr_at_20": 0.8477932437401593, + "mrr_at_3": 0.8383548541367767, + "mrr_at_5": 0.8432329029172645, + "nauc_cv_recall_at_1000_diff1": 0.8175387148222494, + "nauc_cv_recall_at_1000_max": 0.7813301097103972, + "nauc_cv_recall_at_1000_std": 0.6193570292722026, + "nauc_cv_recall_at_100_diff1": 0.8263292349023955, + "nauc_cv_recall_at_100_max": 0.5635539288551284, + "nauc_cv_recall_at_100_std": 0.3522543346587812, + "nauc_cv_recall_at_10_diff1": 0.8295473346563693, + "nauc_cv_recall_at_10_max": 0.6923738450403997, + "nauc_cv_recall_at_10_std": 0.389947956274333, + "nauc_cv_recall_at_1_diff1": 0.893555032247032, + "nauc_cv_recall_at_1_max": 0.5814690622558578, + "nauc_cv_recall_at_1_std": 0.23603882918725352, + "nauc_cv_recall_at_20_diff1": 0.8490190393318788, + "nauc_cv_recall_at_20_max": 0.6828827824898038, + "nauc_cv_recall_at_20_std": 0.4875361417563047, + "nauc_cv_recall_at_3_diff1": 0.848738389948679, + "nauc_cv_recall_at_3_max": 0.6746014172072109, + "nauc_cv_recall_at_3_std": 0.28114809635552956, + "nauc_cv_recall_at_5_diff1": 0.8356168637564215, + "nauc_cv_recall_at_5_max": 0.6556750572467132, + "nauc_cv_recall_at_5_std": 0.2804939414519074, + "nauc_map_at_1000_diff1": 0.8805921183146161, + "nauc_map_at_1000_max": 0.613148597740671, + "nauc_map_at_1000_std": 0.2610442510570384, + "nauc_map_at_100_diff1": 0.8805784135077193, + "nauc_map_at_100_max": 0.6129179080060736, + "nauc_map_at_100_std": 0.26086055894944565, + "nauc_map_at_10_diff1": 0.8794830794153383, + "nauc_map_at_10_max": 0.6148311342515017, + "nauc_map_at_10_std": 0.2610332258087086, + "nauc_map_at_1_diff1": 0.8962682754647417, + "nauc_map_at_1_max": 0.5837819476683158, + "nauc_map_at_1_std": 0.24138500178899677, + "nauc_map_at_20_diff1": 0.8806350178255765, + "nauc_map_at_20_max": 0.6136795206917912, + "nauc_map_at_20_std": 0.26241957265256216, + "nauc_map_at_3_diff1": 0.8806059823643008, + "nauc_map_at_3_max": 0.6163799755044372, + "nauc_map_at_3_std": 0.25270335150389284, + "nauc_map_at_5_diff1": 0.8790830385851995, + "nauc_map_at_5_max": 0.6127564220549455, + "nauc_map_at_5_std": 0.2534054339713012, + "nauc_mrr_at_1000_diff1": 0.8785846990485243, + "nauc_mrr_at_1000_max": 0.6119022395966969, + "nauc_mrr_at_1000_std": 0.2583100845365849, + "nauc_mrr_at_100_diff1": 0.8785723188754336, + "nauc_mrr_at_100_max": 0.6116714134549016, + "nauc_mrr_at_100_std": 0.25812748443787986, + "nauc_mrr_at_10_diff1": 0.8775035197699672, + "nauc_mrr_at_10_max": 0.6136121471258943, + "nauc_mrr_at_10_std": 0.2583433756438642, + "nauc_mrr_at_1_diff1": 0.893555032247032, + "nauc_mrr_at_1_max": 0.5814690622558578, + "nauc_mrr_at_1_std": 0.23603882918725352, + "nauc_mrr_at_20_diff1": 0.8786372968530105, + "nauc_mrr_at_20_max": 0.612441351461196, + "nauc_mrr_at_20_std": 0.25970427563020676, + "nauc_mrr_at_3_diff1": 0.8781764404997089, + "nauc_mrr_at_3_max": 0.6156394406227684, + "nauc_mrr_at_3_std": 0.25193373541652625, + "nauc_mrr_at_5_diff1": 0.877139876269722, + "nauc_mrr_at_5_max": 0.6115522294704427, + "nauc_mrr_at_5_std": 0.2507355224549089, + "nauc_ndcg_at_1000_diff1": 0.8768801815238092, + "nauc_ndcg_at_1000_max": 0.6192014315122151, + "nauc_ndcg_at_1000_std": 0.27264225668380687, + "nauc_ndcg_at_100_diff1": 0.8760097922537312, + "nauc_ndcg_at_100_max": 0.6151797528361204, + "nauc_ndcg_at_100_std": 0.2708666025109306, + "nauc_ndcg_at_10_diff1": 0.8720122547537932, + "nauc_ndcg_at_10_max": 0.6265374448079035, + "nauc_ndcg_at_10_std": 0.2783746570432192, + "nauc_ndcg_at_1_diff1": 0.8962682754647417, + "nauc_ndcg_at_1_max": 0.5837819476683158, + "nauc_ndcg_at_1_std": 0.24138500178899677, + "nauc_ndcg_at_20_diff1": 0.8761941265114012, + "nauc_ndcg_at_20_max": 0.6224642007356991, + "nauc_ndcg_at_20_std": 0.2844421174644763, + "nauc_ndcg_at_3_diff1": 0.8746287968004955, + "nauc_ndcg_at_3_max": 0.6280145743162838, + "nauc_ndcg_at_3_std": 0.25660331622091054, + "nauc_ndcg_at_5_diff1": 0.8712648393471466, + "nauc_ndcg_at_5_max": 0.6210730988223663, + "nauc_ndcg_at_5_std": 0.25831577923580146, + "nauc_precision_at_1000_diff1": 0.8175387148222356, + "nauc_precision_at_1000_max": 0.7813301097103712, + "nauc_precision_at_1000_std": 0.6193570292721997, + "nauc_precision_at_100_diff1": 0.8263292349023921, + "nauc_precision_at_100_max": 0.5635539288551251, + "nauc_precision_at_100_std": 0.3522543346587806, + "nauc_precision_at_10_diff1": 0.8295473346563669, + "nauc_precision_at_10_max": 0.6923738450404027, + "nauc_precision_at_10_std": 0.3899479562743347, + "nauc_precision_at_1_diff1": 0.8962682754647417, + "nauc_precision_at_1_max": 0.5837819476683158, + "nauc_precision_at_1_std": 0.24138500178899677, + "nauc_precision_at_20_diff1": 0.8490190393318748, + "nauc_precision_at_20_max": 0.6828827824898049, + "nauc_precision_at_20_std": 0.4875361417562997, + "nauc_precision_at_3_diff1": 0.8519642374715442, + "nauc_precision_at_3_max": 0.6717930971542201, + "nauc_precision_at_3_std": 0.2712153905918118, + "nauc_precision_at_5_diff1": 0.8356168637564203, + "nauc_precision_at_5_max": 0.6556750572467118, + "nauc_precision_at_5_std": 0.2804939414519041, + "nauc_recall_at_1000_diff1": 0.8175387148222494, + "nauc_recall_at_1000_max": 0.7813301097103972, + "nauc_recall_at_1000_std": 0.6193570292722026, + "nauc_recall_at_100_diff1": 0.8263292349023955, + "nauc_recall_at_100_max": 0.5635539288551284, + "nauc_recall_at_100_std": 0.3522543346587812, + "nauc_recall_at_10_diff1": 0.8295473346563693, + "nauc_recall_at_10_max": 0.6923738450403997, + "nauc_recall_at_10_std": 0.389947956274333, + "nauc_recall_at_1_diff1": 0.8962682754647417, + "nauc_recall_at_1_max": 0.5837819476683158, + "nauc_recall_at_1_std": 0.24138500178899677, + "nauc_recall_at_20_diff1": 0.8490190393318788, + "nauc_recall_at_20_max": 0.6828827824898038, + "nauc_recall_at_20_std": 0.4875361417563047, + "nauc_recall_at_3_diff1": 0.8519642374715442, + "nauc_recall_at_3_max": 0.6717930971542223, + "nauc_recall_at_3_std": 0.27121539059181127, + "nauc_recall_at_5_diff1": 0.8356168637564215, + "nauc_recall_at_5_max": 0.6556750572467132, + "nauc_recall_at_5_std": 0.2804939414519074, + "ndcg_at_1": 0.79914, + "ndcg_at_10": 0.86489, + "ndcg_at_100": 0.87503, + "ndcg_at_1000": 0.87761, + "ndcg_at_20": 0.87058, + "ndcg_at_3": 0.84848, + "ndcg_at_5": 0.85793, + "precision_at_1": 0.79914, + "precision_at_10": 0.09254, + "precision_at_100": 0.00971, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04735, + "precision_at_3": 0.29364, + "precision_at_5": 0.18077, + "recall_at_1": 0.79914, + "recall_at_10": 0.92539, + "recall_at_100": 0.97131, + "recall_at_1000": 0.99139, + "recall_at_20": 0.94692, + "recall_at_3": 0.88092, + "recall_at_5": 0.90387 + } + ] + }, + "task_name": "MemotionT2IRetrieval" +} \ No newline at end of file From a7c6e7ce8cc39cc1f5c0339559b8f911eb723f1e Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Wed, 4 Sep 2024 01:47:54 +0100 Subject: [PATCH 062/154] add SciMMIR Retrieval i2tt2i --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 2 + .../eng/SciMMIRI2TRetrieval.py | 122 ++++++++++++ .../eng/SciMMIRT2IRetrieval.py | 122 ++++++++++++ .../SciMMIRI2TRetrieval.json | 186 ++++++++++++++++++ .../SciMMIRT2IRetrieval.json | 186 ++++++++++++++++++ .../SciMMIRI2TRetrieval.json | 186 ++++++++++++++++++ .../SciMMIRT2IRetrieval.json | 186 ++++++++++++++++++ .../SciMMIRI2TRetrieval.json | 186 ++++++++++++++++++ .../SciMMIRT2IRetrieval.json | 186 ++++++++++++++++++ 9 files changed, 1362 insertions(+) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRT2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 2fec8364a0..2cda0b0664 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -17,6 +17,8 @@ from .eng.NIGHTSI2IRetrieval import * from .eng.OVENIT2ITRetrieval import * from .eng.OVENIT2TRetrieval import * +from .eng.SciMMIRI2TRetrieval import * +from .eng.SciMMIRT2IRetrieval import * from .eng.StanfordCarsI2IRetrieval import * from .eng.TUBerlinT2IRetrieval import * from .eng.VisualNewsI2TRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py new file mode 100644 index 0000000000..ad657a9534 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): + corpus = {} + queries = {} + relevant_docs = {} + + dataset = load_dataset( + path, + cache_dir=cache_dir, + revision=revision, + ) + + for split in splits: + split_dataset = dataset[split] + + corpus[split] = split_dataset.map( + lambda x, idx: { + "id": f"corpus-{split}-{idx}", + # "text": None, + "modality": "text", + "image": None, + }, + with_indices=True, + remove_columns=[ + "file_name_index", + "class", + "super_class", + "sub_class", + "split", + ], + ) + + queries[split] = split_dataset.map( + lambda x, idx: { + "id": f"query-{split}-{idx}", + "text": None, + "modality": "image", + # "image": None, + }, + with_indices=True, + remove_columns=[ + "file_name_index", + "class", + "super_class", + "sub_class", + "split", + ], + ) + relevant_docs[split] = {} + for index in range(len(split_dataset)): + query_id = f"query-{split}-{index}" + doc_id = f"corpus-{split}-{index}" + if query_id not in relevant_docs[split]: + relevant_docs[split][query_id] = {} + relevant_docs[split][query_id][doc_id] = 1 + return corpus, queries, relevant_docs + + +class SciMMIRI2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="SciMMIRI2TRetrieval", + description="Retrieve captions based on figures and tables.", + reference="https://aclanthology.org/2024.findings-acl.746/", + dataset={ + "path": "m-a-p/SciMMIR", + "revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", + # "trust_remote_code": True, + }, + type="Retrieval", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2023-01-01", "2023-12-31"), + form=["found"], + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{wu2024scimmir, + title={SciMMIR: Benchmarking Scientific Multi-modal Information Retrieval}, + author={Wu, Siwei and Li, Yizhi and Zhu, Kang and Zhang, Ge and Liang, Yiming and Ma, Kaijing and Xiao, Chenghao and Zhang, Haoran and Yang, Bohao and Chen, Wenhu and others}, + journal={arXiv preprint arXiv:2401.13478}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 261.1932607759946, + "average_query_length": 1.0, + "num_documents": 16263, + "num_queries": 16263, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = _load_data( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py new file mode 100644 index 0000000000..8d4582886d --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from .....abstasks import AbsTaskAny2AnyRetrieval + + +def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): + corpus = {} + queries = {} + relevant_docs = {} + + dataset = load_dataset( + path, + cache_dir=cache_dir, + revision=revision, + ) + + for split in splits: + split_dataset = dataset[split] + + corpus[split] = split_dataset.map( + lambda x, idx: { + "id": f"corpus-{split}-{idx}", + "text": None, + "modality": "image", + # "image": None, + }, + with_indices=True, + remove_columns=[ + "file_name_index", + "class", + "super_class", + "sub_class", + "split", + ], + ) + + queries[split] = split_dataset.map( + lambda x, idx: { + "id": f"query-{split}-{idx}", + # "text": None, + "modality": "text", + "image": None, + }, + with_indices=True, + remove_columns=[ + "file_name_index", + "class", + "super_class", + "sub_class", + "split", + ], + ) + relevant_docs[split] = {} + for index in range(len(split_dataset)): + query_id = f"query-{split}-{index}" + doc_id = f"corpus-{split}-{index}" + if query_id not in relevant_docs[split]: + relevant_docs[split][query_id] = {} + relevant_docs[split][query_id][doc_id] = 1 + return corpus, queries, relevant_docs + + +class SciMMIRT2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="SciMMIRT2IRetrieval", + description="Retrieve figures and tables based on captions.", + reference="https://aclanthology.org/2024.findings-acl.746/", + dataset={ + "path": "m-a-p/SciMMIR", + "revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", + # "trust_remote_code": True, + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2023-01-01", "2023-12-31"), + form=["found"], + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{wu2024scimmir, + title={SciMMIR: Benchmarking Scientific Multi-modal Information Retrieval}, + author={Wu, Siwei and Li, Yizhi and Zhu, Kang and Zhang, Ge and Liang, Yiming and Ma, Kaijing and Xiao, Chenghao and Zhang, Haoran and Yang, Bohao and Chen, Wenhu and others}, + journal={arXiv preprint arXiv:2401.13478}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 261.1932607759946, + "num_documents": 16263, + "num_queries": 16263, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = _load_data( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRI2TRetrieval.json new file mode 100644 index 0000000000..5391e9d2ac --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", + "evaluation_time": 140.40432357788086, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.08836, + "cv_recall_at_10": 0.21085, + "cv_recall_at_100": 0.38431, + "cv_recall_at_1000": 0.6714, + "cv_recall_at_20": 0.25395, + "cv_recall_at_3": 0.14235, + "cv_recall_at_5": 0.16934, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.14411, + "map_at_1": 0.08836, + "map_at_10": 0.12346, + "map_at_100": 0.1294, + "map_at_1000": 0.13042, + "map_at_20": 0.12639, + "map_at_3": 0.11184, + "map_at_5": 0.11798, + "mrr_at_1": 0.08836008116583656, + "mrr_at_10": 0.12345752213857675, + "mrr_at_100": 0.12940234576740298, + "mrr_at_1000": 0.13042081759294002, + "mrr_at_20": 0.12639283297819012, + "mrr_at_3": 0.1118387341408919, + "mrr_at_5": 0.1179845866896225, + "nauc_cv_recall_at_1000_diff1": 0.1930440666440711, + "nauc_cv_recall_at_1000_max": 0.10121846044999482, + "nauc_cv_recall_at_1000_std": 0.42508401174919186, + "nauc_cv_recall_at_100_diff1": 0.2446500985422135, + "nauc_cv_recall_at_100_max": 0.1467798294776438, + "nauc_cv_recall_at_100_std": 0.414757382289627, + "nauc_cv_recall_at_10_diff1": 0.30827230657058935, + "nauc_cv_recall_at_10_max": 0.22109141920875744, + "nauc_cv_recall_at_10_std": 0.38971247695792116, + "nauc_cv_recall_at_1_diff1": 0.4980711735833261, + "nauc_cv_recall_at_1_max": 0.2971017528223884, + "nauc_cv_recall_at_1_std": 0.35834619013019836, + "nauc_cv_recall_at_20_diff1": 0.2825045381641615, + "nauc_cv_recall_at_20_max": 0.19613741590717187, + "nauc_cv_recall_at_20_std": 0.39649688417982515, + "nauc_cv_recall_at_3_diff1": 0.37694694761984543, + "nauc_cv_recall_at_3_max": 0.2635657508987931, + "nauc_cv_recall_at_3_std": 0.37644697851927716, + "nauc_cv_recall_at_5_diff1": 0.342725203515418, + "nauc_cv_recall_at_5_max": 0.2483062507893586, + "nauc_cv_recall_at_5_std": 0.3796606741574664, + "nauc_map_at_1000_diff1": 0.40552045901217515, + "nauc_map_at_1000_max": 0.26273915141358706, + "nauc_map_at_1000_std": 0.3729451955892119, + "nauc_map_at_100_diff1": 0.4056641617493706, + "nauc_map_at_100_max": 0.2629317107092913, + "nauc_map_at_100_std": 0.37288447789078516, + "nauc_map_at_10_diff1": 0.41197297657520887, + "nauc_map_at_10_max": 0.2684384296464497, + "nauc_map_at_10_std": 0.3709211802013581, + "nauc_map_at_1_diff1": 0.4980711735833261, + "nauc_map_at_1_max": 0.2971017528223884, + "nauc_map_at_1_std": 0.35834619013019836, + "nauc_map_at_20_diff1": 0.408279749615102, + "nauc_map_at_20_max": 0.26530210357036255, + "nauc_map_at_20_std": 0.3720042216340903, + "nauc_map_at_3_diff1": 0.43361382217865935, + "nauc_map_at_3_max": 0.2799005350406905, + "nauc_map_at_3_std": 0.36729622951789603, + "nauc_map_at_5_diff1": 0.42131281614217014, + "nauc_map_at_5_max": 0.2749144272701716, + "nauc_map_at_5_std": 0.36853037133934513, + "nauc_mrr_at_1000_diff1": 0.4055204594804454, + "nauc_mrr_at_1000_max": 0.2627391567916803, + "nauc_mrr_at_1000_std": 0.3729452011303819, + "nauc_mrr_at_100_diff1": 0.4056641617493706, + "nauc_mrr_at_100_max": 0.2629317107092913, + "nauc_mrr_at_100_std": 0.37288447789078516, + "nauc_mrr_at_10_diff1": 0.41197297657520887, + "nauc_mrr_at_10_max": 0.2684384296464497, + "nauc_mrr_at_10_std": 0.3709211802013581, + "nauc_mrr_at_1_diff1": 0.4980711735833261, + "nauc_mrr_at_1_max": 0.2971017528223884, + "nauc_mrr_at_1_std": 0.35834619013019836, + "nauc_mrr_at_20_diff1": 0.408279749615102, + "nauc_mrr_at_20_max": 0.26530210357036255, + "nauc_mrr_at_20_std": 0.3720042216340903, + "nauc_mrr_at_3_diff1": 0.43361382217865935, + "nauc_mrr_at_3_max": 0.2799005350406905, + "nauc_mrr_at_3_std": 0.36729622951789603, + "nauc_mrr_at_5_diff1": 0.42131281614217014, + "nauc_mrr_at_5_max": 0.2749144272701716, + "nauc_mrr_at_5_std": 0.36853037133934513, + "nauc_ndcg_at_1000_diff1": 0.3507824423697204, + "nauc_ndcg_at_1000_max": 0.22522557499324103, + "nauc_ndcg_at_1000_std": 0.385920157570721, + "nauc_ndcg_at_100_diff1": 0.35397834406230494, + "nauc_ndcg_at_100_max": 0.2288619650602957, + "nauc_ndcg_at_100_std": 0.385366160982851, + "nauc_ndcg_at_10_diff1": 0.37884486199522766, + "nauc_ndcg_at_10_max": 0.25371633138876953, + "nauc_ndcg_at_10_std": 0.37680379412540926, + "nauc_ndcg_at_1_diff1": 0.4980711735833261, + "nauc_ndcg_at_1_max": 0.2971017528223884, + "nauc_ndcg_at_1_std": 0.35834619013019836, + "nauc_ndcg_at_20_diff1": 0.3677982770691768, + "nauc_ndcg_at_20_max": 0.24398701478538692, + "nauc_ndcg_at_20_std": 0.37975850307644804, + "nauc_ndcg_at_3_diff1": 0.41691634606030453, + "nauc_ndcg_at_3_max": 0.2751132351903913, + "nauc_ndcg_at_3_std": 0.36996518625696995, + "nauc_ndcg_at_5_diff1": 0.3972950689280354, + "nauc_ndcg_at_5_max": 0.26688816758904627, + "nauc_ndcg_at_5_std": 0.3719255027762222, + "nauc_precision_at_1000_diff1": 0.19304406664407023, + "nauc_precision_at_1000_max": 0.10121846044999426, + "nauc_precision_at_1000_std": 0.42508401174919064, + "nauc_precision_at_100_diff1": 0.2446500985422135, + "nauc_precision_at_100_max": 0.14677982947764376, + "nauc_precision_at_100_std": 0.41475738228962666, + "nauc_precision_at_10_diff1": 0.30827230657058935, + "nauc_precision_at_10_max": 0.2210914192087575, + "nauc_precision_at_10_std": 0.38971247695792105, + "nauc_precision_at_1_diff1": 0.4980711735833261, + "nauc_precision_at_1_max": 0.2971017528223884, + "nauc_precision_at_1_std": 0.35834619013019836, + "nauc_precision_at_20_diff1": 0.2825045381641612, + "nauc_precision_at_20_max": 0.1961374159071717, + "nauc_precision_at_20_std": 0.39649688417982526, + "nauc_precision_at_3_diff1": 0.3769469476198457, + "nauc_precision_at_3_max": 0.2635657508987932, + "nauc_precision_at_3_std": 0.3764469785192774, + "nauc_precision_at_5_diff1": 0.34272520351541796, + "nauc_precision_at_5_max": 0.24830625078935847, + "nauc_precision_at_5_std": 0.37966067415746646, + "nauc_recall_at_1000_diff1": 0.1930440666440711, + "nauc_recall_at_1000_max": 0.10121846044999482, + "nauc_recall_at_1000_std": 0.42508401174919186, + "nauc_recall_at_100_diff1": 0.2446500985422135, + "nauc_recall_at_100_max": 0.1467798294776438, + "nauc_recall_at_100_std": 0.414757382289627, + "nauc_recall_at_10_diff1": 0.30827230657058935, + "nauc_recall_at_10_max": 0.22109141920875744, + "nauc_recall_at_10_std": 0.38971247695792116, + "nauc_recall_at_1_diff1": 0.4980711735833261, + "nauc_recall_at_1_max": 0.2971017528223884, + "nauc_recall_at_1_std": 0.35834619013019836, + "nauc_recall_at_20_diff1": 0.2825045381641615, + "nauc_recall_at_20_max": 0.19613741590717187, + "nauc_recall_at_20_std": 0.39649688417982515, + "nauc_recall_at_3_diff1": 0.37694694761984543, + "nauc_recall_at_3_max": 0.2635657508987931, + "nauc_recall_at_3_std": 0.37644697851927716, + "nauc_recall_at_5_diff1": 0.342725203515418, + "nauc_recall_at_5_max": 0.2483062507893586, + "nauc_recall_at_5_std": 0.3796606741574664, + "ndcg_at_1": 0.08836, + "ndcg_at_10": 0.14411, + "ndcg_at_100": 0.17846, + "ndcg_at_1000": 0.21287, + "ndcg_at_20": 0.15494, + "ndcg_at_3": 0.11966, + "ndcg_at_5": 0.13076, + "precision_at_1": 0.08836, + "precision_at_10": 0.02108, + "precision_at_100": 0.00384, + "precision_at_1000": 0.00067, + "precision_at_20": 0.0127, + "precision_at_3": 0.04745, + "precision_at_5": 0.03387, + "recall_at_1": 0.08836, + "recall_at_10": 0.21085, + "recall_at_100": 0.38431, + "recall_at_1000": 0.6714, + "recall_at_20": 0.25395, + "recall_at_3": 0.14235, + "recall_at_5": 0.16934 + } + ] + }, + "task_name": "SciMMIRI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRT2IRetrieval.json new file mode 100644 index 0000000000..72e576558c --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", + "evaluation_time": 137.33402299880981, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.07846, + "cv_recall_at_10": 0.18699, + "cv_recall_at_100": 0.35153, + "cv_recall_at_1000": 0.64207, + "cv_recall_at_20": 0.22603, + "cv_recall_at_3": 0.12593, + "cv_recall_at_5": 0.15046, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.12785, + "map_at_1": 0.07846, + "map_at_10": 0.10952, + "map_at_100": 0.11508, + "map_at_1000": 0.11612, + "map_at_20": 0.11218, + "map_at_3": 0.09912, + "map_at_5": 0.10464, + "mrr_at_1": 0.07846030867613601, + "mrr_at_10": 0.10951404932220245, + "mrr_at_100": 0.1150779301551405, + "mrr_at_1000": 0.11611561738842863, + "mrr_at_20": 0.11217670727748709, + "mrr_at_3": 0.09912070343724981, + "mrr_at_5": 0.10464243989423845, + "nauc_cv_recall_at_1000_diff1": 0.14280409251991044, + "nauc_cv_recall_at_1000_max": 0.18628330671583407, + "nauc_cv_recall_at_1000_std": 0.3870589592403038, + "nauc_cv_recall_at_100_diff1": 0.18605678508837606, + "nauc_cv_recall_at_100_max": 0.17998073338336004, + "nauc_cv_recall_at_100_std": 0.33735207281947194, + "nauc_cv_recall_at_10_diff1": 0.2650604850881054, + "nauc_cv_recall_at_10_max": 0.23823992507714759, + "nauc_cv_recall_at_10_std": 0.27980801514303494, + "nauc_cv_recall_at_1_diff1": 0.4827365014175455, + "nauc_cv_recall_at_1_max": 0.33010914760725707, + "nauc_cv_recall_at_1_std": 0.2512977037559647, + "nauc_cv_recall_at_20_diff1": 0.23941515017415582, + "nauc_cv_recall_at_20_max": 0.22007084090482062, + "nauc_cv_recall_at_20_std": 0.29443736255648456, + "nauc_cv_recall_at_3_diff1": 0.35336896283747304, + "nauc_cv_recall_at_3_max": 0.295773329151323, + "nauc_cv_recall_at_3_std": 0.2681076282273982, + "nauc_cv_recall_at_5_diff1": 0.31570322218368735, + "nauc_cv_recall_at_5_max": 0.2776172685924335, + "nauc_cv_recall_at_5_std": 0.27866786119872977, + "nauc_map_at_1000_diff1": 0.376660313469499, + "nauc_map_at_1000_max": 0.29209998722994507, + "nauc_map_at_1000_std": 0.26716382098614505, + "nauc_map_at_100_diff1": 0.37688184035666233, + "nauc_map_at_100_max": 0.29208878660890697, + "nauc_map_at_100_std": 0.2668800438584805, + "nauc_map_at_10_diff1": 0.3851603824358598, + "nauc_map_at_10_max": 0.29771555101574054, + "nauc_map_at_10_std": 0.26295031900912746, + "nauc_map_at_1_diff1": 0.4827365014175455, + "nauc_map_at_1_max": 0.33010914760725707, + "nauc_map_at_1_std": 0.2512977037559647, + "nauc_map_at_20_diff1": 0.3807333533485515, + "nauc_map_at_20_max": 0.2947438323961747, + "nauc_map_at_20_std": 0.26459412628505796, + "nauc_map_at_3_diff1": 0.41141276578806035, + "nauc_map_at_3_max": 0.3122153310864702, + "nauc_map_at_3_std": 0.25861944245595125, + "nauc_map_at_5_diff1": 0.3982000648043669, + "nauc_map_at_5_max": 0.30657629290318295, + "nauc_map_at_5_std": 0.2624903009814347, + "nauc_mrr_at_1000_diff1": 0.37666471757312453, + "nauc_mrr_at_1000_max": 0.29209929641607063, + "nauc_mrr_at_1000_std": 0.26716424106866365, + "nauc_mrr_at_100_diff1": 0.3768862418339056, + "nauc_mrr_at_100_max": 0.29208808952380083, + "nauc_mrr_at_100_std": 0.2668804612792214, + "nauc_mrr_at_10_diff1": 0.3852100780007191, + "nauc_mrr_at_10_max": 0.29770795553077667, + "nauc_mrr_at_10_std": 0.2629548450361711, + "nauc_mrr_at_1_diff1": 0.4827365014175455, + "nauc_mrr_at_1_max": 0.33010914760725707, + "nauc_mrr_at_1_std": 0.2512977037559647, + "nauc_mrr_at_20_diff1": 0.38073780181367767, + "nauc_mrr_at_20_max": 0.2947431394977995, + "nauc_mrr_at_20_std": 0.2645945387197461, + "nauc_mrr_at_3_diff1": 0.41141276578806035, + "nauc_mrr_at_3_max": 0.3122153310864702, + "nauc_mrr_at_3_std": 0.25861944245595125, + "nauc_mrr_at_5_diff1": 0.3982000648043669, + "nauc_mrr_at_5_max": 0.30657629290318295, + "nauc_mrr_at_5_std": 0.2624903009814347, + "nauc_ndcg_at_1000_diff1": 0.31030920764011694, + "nauc_ndcg_at_1000_max": 0.259488547988665, + "nauc_ndcg_at_1000_std": 0.294399902820702, + "nauc_ndcg_at_100_diff1": 0.31373988925006974, + "nauc_ndcg_at_100_max": 0.25646300051736803, + "nauc_ndcg_at_100_std": 0.2875697891829784, + "nauc_ndcg_at_10_diff1": 0.3465105960548437, + "nauc_ndcg_at_10_max": 0.2792501524340969, + "nauc_ndcg_at_10_std": 0.2683925570046724, + "nauc_ndcg_at_1_diff1": 0.4827365014175455, + "nauc_ndcg_at_1_max": 0.33010914760725707, + "nauc_ndcg_at_1_std": 0.2512977037559647, + "nauc_ndcg_at_20_diff1": 0.3341800450298754, + "nauc_ndcg_at_20_max": 0.270914009208363, + "nauc_ndcg_at_20_std": 0.273906970807198, + "nauc_ndcg_at_3_diff1": 0.3941121974458003, + "nauc_ndcg_at_3_max": 0.3073570658308402, + "nauc_ndcg_at_3_std": 0.2613677969210819, + "nauc_ndcg_at_5_diff1": 0.37280888984298527, + "nauc_ndcg_at_5_max": 0.2978307511070134, + "nauc_ndcg_at_5_std": 0.26738827173898944, + "nauc_precision_at_1000_diff1": 0.14280409251990966, + "nauc_precision_at_1000_max": 0.18628330671583412, + "nauc_precision_at_1000_std": 0.38705895924030337, + "nauc_precision_at_100_diff1": 0.18605678508837586, + "nauc_precision_at_100_max": 0.17998073338336013, + "nauc_precision_at_100_std": 0.33735207281947227, + "nauc_precision_at_10_diff1": 0.2647678271567299, + "nauc_precision_at_10_max": 0.2383047527423176, + "nauc_precision_at_10_std": 0.27977418385850095, + "nauc_precision_at_1_diff1": 0.4827365014175455, + "nauc_precision_at_1_max": 0.33010914760725707, + "nauc_precision_at_1_std": 0.2512977037559647, + "nauc_precision_at_20_diff1": 0.239415150174156, + "nauc_precision_at_20_max": 0.22007084090482062, + "nauc_precision_at_20_std": 0.2944373625564846, + "nauc_precision_at_3_diff1": 0.3533689628374732, + "nauc_precision_at_3_max": 0.295773329151323, + "nauc_precision_at_3_std": 0.26810762822739814, + "nauc_precision_at_5_diff1": 0.31570322218368735, + "nauc_precision_at_5_max": 0.2776172685924336, + "nauc_precision_at_5_std": 0.2786678611987297, + "nauc_recall_at_1000_diff1": 0.14280409251991044, + "nauc_recall_at_1000_max": 0.18628330671583407, + "nauc_recall_at_1000_std": 0.3870589592403038, + "nauc_recall_at_100_diff1": 0.18605678508837606, + "nauc_recall_at_100_max": 0.17998073338336004, + "nauc_recall_at_100_std": 0.33735207281947194, + "nauc_recall_at_10_diff1": 0.2647678271567298, + "nauc_recall_at_10_max": 0.2383047527423176, + "nauc_recall_at_10_std": 0.2797741838585011, + "nauc_recall_at_1_diff1": 0.4827365014175455, + "nauc_recall_at_1_max": 0.33010914760725707, + "nauc_recall_at_1_std": 0.2512977037559647, + "nauc_recall_at_20_diff1": 0.23941515017415582, + "nauc_recall_at_20_max": 0.22007084090482062, + "nauc_recall_at_20_std": 0.29443736255648456, + "nauc_recall_at_3_diff1": 0.35336896283747304, + "nauc_recall_at_3_max": 0.295773329151323, + "nauc_recall_at_3_std": 0.2681076282273982, + "nauc_recall_at_5_diff1": 0.31570322218368735, + "nauc_recall_at_5_max": 0.2776172685924335, + "nauc_recall_at_5_std": 0.27866786119872977, + "ndcg_at_1": 0.07846, + "ndcg_at_10": 0.12785, + "ndcg_at_100": 0.16029, + "ndcg_at_1000": 0.19513, + "ndcg_at_20": 0.13765, + "ndcg_at_3": 0.106, + "ndcg_at_5": 0.11603, + "precision_at_1": 0.07846, + "precision_at_10": 0.01871, + "precision_at_100": 0.00352, + "precision_at_1000": 0.00064, + "precision_at_20": 0.0113, + "precision_at_3": 0.04198, + "precision_at_5": 0.03009, + "recall_at_1": 0.07846, + "recall_at_10": 0.18705, + "recall_at_100": 0.35153, + "recall_at_1000": 0.64207, + "recall_at_20": 0.22603, + "recall_at_3": 0.12593, + "recall_at_5": 0.15046 + } + ] + }, + "task_name": "SciMMIRT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRI2TRetrieval.json new file mode 100644 index 0000000000..300c9a1314 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", + "evaluation_time": 114.45678329467773, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.05817, + "cv_recall_at_10": 0.14579, + "cv_recall_at_100": 0.30197, + "cv_recall_at_1000": 0.60315, + "cv_recall_at_20": 0.18434, + "cv_recall_at_3": 0.09291, + "cv_recall_at_5": 0.11369, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.09715, + "map_at_1": 0.05817, + "map_at_10": 0.08221, + "map_at_100": 0.08759, + "map_at_1000": 0.08863, + "map_at_20": 0.08488, + "map_at_3": 0.0733, + "map_at_5": 0.07796, + "mrr_at_1": 0.05816884953575601, + "mrr_at_10": 0.08220717687144535, + "mrr_at_100": 0.08758833735797077, + "mrr_at_1000": 0.08862821378958152, + "mrr_at_20": 0.08488199256182855, + "mrr_at_3": 0.07329520998585749, + "mrr_at_5": 0.07796224558814507, + "nauc_cv_recall_at_1000_diff1": 0.14684226988015606, + "nauc_cv_recall_at_1000_max": 0.134802691044856, + "nauc_cv_recall_at_1000_std": 0.389831948813318, + "nauc_cv_recall_at_100_diff1": 0.1806306486342646, + "nauc_cv_recall_at_100_max": 0.14414724950638658, + "nauc_cv_recall_at_100_std": 0.39548007814807806, + "nauc_cv_recall_at_10_diff1": 0.27063104148951167, + "nauc_cv_recall_at_10_max": 0.21400035358270772, + "nauc_cv_recall_at_10_std": 0.41381715017793386, + "nauc_cv_recall_at_1_diff1": 0.48484300258564605, + "nauc_cv_recall_at_1_max": 0.3213938518834158, + "nauc_cv_recall_at_1_std": 0.4507495622214212, + "nauc_cv_recall_at_20_diff1": 0.23455604464471214, + "nauc_cv_recall_at_20_max": 0.18500843309451123, + "nauc_cv_recall_at_20_std": 0.40437096458544736, + "nauc_cv_recall_at_3_diff1": 0.3408987613305649, + "nauc_cv_recall_at_3_max": 0.2664202105680823, + "nauc_cv_recall_at_3_std": 0.40308964378875, + "nauc_cv_recall_at_5_diff1": 0.3075078599041494, + "nauc_cv_recall_at_5_max": 0.24843826848027892, + "nauc_cv_recall_at_5_std": 0.4110419152792929, + "nauc_map_at_1000_diff1": 0.3681055550235476, + "nauc_map_at_1000_max": 0.2670263591078058, + "nauc_map_at_1000_std": 0.42310319341347463, + "nauc_map_at_100_diff1": 0.368442314606953, + "nauc_map_at_100_max": 0.2672938366227504, + "nauc_map_at_100_std": 0.4230389418150572, + "nauc_map_at_10_diff1": 0.38017416191050696, + "nauc_map_at_10_max": 0.27588764093276885, + "nauc_map_at_10_std": 0.4252284193888985, + "nauc_map_at_1_diff1": 0.48484300258564605, + "nauc_map_at_1_max": 0.3213938518834158, + "nauc_map_at_1_std": 0.4507495622214212, + "nauc_map_at_20_diff1": 0.37306566641438116, + "nauc_map_at_20_max": 0.27061824646863875, + "nauc_map_at_20_std": 0.42367498722326263, + "nauc_map_at_3_diff1": 0.40484680089911823, + "nauc_map_at_3_max": 0.292228620983057, + "nauc_map_at_3_std": 0.42377965324762107, + "nauc_map_at_5_diff1": 0.3909036060274109, + "nauc_map_at_5_max": 0.28493006995037873, + "nauc_map_at_5_std": 0.425098683635399, + "nauc_mrr_at_1000_diff1": 0.36810555556946195, + "nauc_mrr_at_1000_max": 0.26702636048225037, + "nauc_mrr_at_1000_std": 0.42310319409545427, + "nauc_mrr_at_100_diff1": 0.368442314606953, + "nauc_mrr_at_100_max": 0.2672938366227504, + "nauc_mrr_at_100_std": 0.4230389418150572, + "nauc_mrr_at_10_diff1": 0.38017416191050696, + "nauc_mrr_at_10_max": 0.27588764093276885, + "nauc_mrr_at_10_std": 0.4252284193888985, + "nauc_mrr_at_1_diff1": 0.48484300258564605, + "nauc_mrr_at_1_max": 0.3213938518834158, + "nauc_mrr_at_1_std": 0.4507495622214212, + "nauc_mrr_at_20_diff1": 0.37306566641438116, + "nauc_mrr_at_20_max": 0.27061824646863875, + "nauc_mrr_at_20_std": 0.42367498722326263, + "nauc_mrr_at_3_diff1": 0.40484680089911823, + "nauc_mrr_at_3_max": 0.292228620983057, + "nauc_mrr_at_3_std": 0.42377965324762107, + "nauc_mrr_at_5_diff1": 0.3909036060274109, + "nauc_mrr_at_5_max": 0.28493006995037873, + "nauc_mrr_at_5_std": 0.425098683635399, + "nauc_ndcg_at_1000_diff1": 0.2943590352399392, + "nauc_ndcg_at_1000_max": 0.2217944789510595, + "nauc_ndcg_at_1000_std": 0.41258004401152276, + "nauc_ndcg_at_100_diff1": 0.2995056724630319, + "nauc_ndcg_at_100_max": 0.2231781872995249, + "nauc_ndcg_at_100_std": 0.4130007443807324, + "nauc_ndcg_at_10_diff1": 0.3432523114481406, + "nauc_ndcg_at_10_max": 0.2554435336092466, + "nauc_ndcg_at_10_std": 0.42092555323891523, + "nauc_ndcg_at_1_diff1": 0.48484300258564605, + "nauc_ndcg_at_1_max": 0.3213938518834158, + "nauc_ndcg_at_1_std": 0.4507495622214212, + "nauc_ndcg_at_20_diff1": 0.32406899107370635, + "nauc_ndcg_at_20_max": 0.24102103282365073, + "nauc_ndcg_at_20_std": 0.4166142389037552, + "nauc_ndcg_at_3_diff1": 0.3853732743990311, + "nauc_ndcg_at_3_max": 0.2844243743500187, + "nauc_ndcg_at_3_std": 0.41745995065792757, + "nauc_ndcg_at_5_diff1": 0.3641878536983579, + "nauc_ndcg_at_5_max": 0.27331942089003486, + "nauc_ndcg_at_5_std": 0.42032916876913506, + "nauc_precision_at_1000_diff1": 0.14684226988015617, + "nauc_precision_at_1000_max": 0.13480269104485515, + "nauc_precision_at_1000_std": 0.3898319488133176, + "nauc_precision_at_100_diff1": 0.18063064863426476, + "nauc_precision_at_100_max": 0.14414724950638685, + "nauc_precision_at_100_std": 0.39548007814807845, + "nauc_precision_at_10_diff1": 0.27063104148951156, + "nauc_precision_at_10_max": 0.21400035358270783, + "nauc_precision_at_10_std": 0.4138171501779341, + "nauc_precision_at_1_diff1": 0.48484300258564605, + "nauc_precision_at_1_max": 0.3213938518834158, + "nauc_precision_at_1_std": 0.4507495622214212, + "nauc_precision_at_20_diff1": 0.23455604464471222, + "nauc_precision_at_20_max": 0.1850084330945111, + "nauc_precision_at_20_std": 0.40437096458544713, + "nauc_precision_at_3_diff1": 0.3408987613305649, + "nauc_precision_at_3_max": 0.2664202105680823, + "nauc_precision_at_3_std": 0.40308964378875006, + "nauc_precision_at_5_diff1": 0.30750785990414936, + "nauc_precision_at_5_max": 0.24843826848027897, + "nauc_precision_at_5_std": 0.41104191527929257, + "nauc_recall_at_1000_diff1": 0.14684226988015606, + "nauc_recall_at_1000_max": 0.134802691044856, + "nauc_recall_at_1000_std": 0.389831948813318, + "nauc_recall_at_100_diff1": 0.1806306486342646, + "nauc_recall_at_100_max": 0.14414724950638658, + "nauc_recall_at_100_std": 0.39548007814807806, + "nauc_recall_at_10_diff1": 0.27063104148951167, + "nauc_recall_at_10_max": 0.21400035358270772, + "nauc_recall_at_10_std": 0.41381715017793386, + "nauc_recall_at_1_diff1": 0.48484300258564605, + "nauc_recall_at_1_max": 0.3213938518834158, + "nauc_recall_at_1_std": 0.4507495622214212, + "nauc_recall_at_20_diff1": 0.23455604464471214, + "nauc_recall_at_20_max": 0.18500843309451123, + "nauc_recall_at_20_std": 0.40437096458544736, + "nauc_recall_at_3_diff1": 0.3408987613305649, + "nauc_recall_at_3_max": 0.2664202105680823, + "nauc_recall_at_3_std": 0.40308964378875, + "nauc_recall_at_5_diff1": 0.3075078599041494, + "nauc_recall_at_5_max": 0.24843826848027892, + "nauc_recall_at_5_std": 0.4110419152792929, + "ndcg_at_1": 0.05817, + "ndcg_at_10": 0.09715, + "ndcg_at_100": 0.12811, + "ndcg_at_1000": 0.16402, + "ndcg_at_20": 0.1069, + "ndcg_at_3": 0.07833, + "ndcg_at_5": 0.08681, + "precision_at_1": 0.05817, + "precision_at_10": 0.01458, + "precision_at_100": 0.00302, + "precision_at_1000": 0.0006, + "precision_at_20": 0.00922, + "precision_at_3": 0.03097, + "precision_at_5": 0.02274, + "recall_at_1": 0.05817, + "recall_at_10": 0.14579, + "recall_at_100": 0.30197, + "recall_at_1000": 0.60315, + "recall_at_20": 0.18434, + "recall_at_3": 0.09291, + "recall_at_5": 0.11369 + } + ] + }, + "task_name": "SciMMIRI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRT2IRetrieval.json new file mode 100644 index 0000000000..d46edfa129 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", + "evaluation_time": 109.91747164726257, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.04895, + "cv_recall_at_10": 0.1207, + "cv_recall_at_100": 0.2657, + "cv_recall_at_1000": 0.56601, + "cv_recall_at_20": 0.15544, + "cv_recall_at_3": 0.07963, + "cv_recall_at_5": 0.0966, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.08144, + "map_at_1": 0.04895, + "map_at_10": 0.0693, + "map_at_100": 0.07419, + "map_at_1000": 0.0752, + "map_at_20": 0.07168, + "map_at_3": 0.06228, + "map_at_5": 0.06612, + "mrr_at_1": 0.048945459017401466, + "mrr_at_10": 0.06929704099967894, + "mrr_at_100": 0.07418728836197862, + "mrr_at_1000": 0.07520233371125491, + "mrr_at_20": 0.07167806562939899, + "mrr_at_3": 0.062278382422267485, + "mrr_at_5": 0.06612146180491527, + "nauc_cv_recall_at_1000_diff1": 0.08565868923582504, + "nauc_cv_recall_at_1000_max": 0.10138064328519379, + "nauc_cv_recall_at_1000_std": 0.3288030738840006, + "nauc_cv_recall_at_100_diff1": 0.15064453104841738, + "nauc_cv_recall_at_100_max": 0.143295352377065, + "nauc_cv_recall_at_100_std": 0.27989257851203553, + "nauc_cv_recall_at_10_diff1": 0.248178863037523, + "nauc_cv_recall_at_10_max": 0.20552793051112445, + "nauc_cv_recall_at_10_std": 0.23682614089723797, + "nauc_cv_recall_at_1_diff1": 0.4931665319076531, + "nauc_cv_recall_at_1_max": 0.34046617367806836, + "nauc_cv_recall_at_1_std": 0.22520394629166143, + "nauc_cv_recall_at_20_diff1": 0.20448507514630834, + "nauc_cv_recall_at_20_max": 0.17905271116474095, + "nauc_cv_recall_at_20_std": 0.24941512163240517, + "nauc_cv_recall_at_3_diff1": 0.33601488304743465, + "nauc_cv_recall_at_3_max": 0.2672543870064712, + "nauc_cv_recall_at_3_std": 0.22899599659848674, + "nauc_cv_recall_at_5_diff1": 0.2836379140928627, + "nauc_cv_recall_at_5_max": 0.2313767233406441, + "nauc_cv_recall_at_5_std": 0.21663135292170893, + "nauc_map_at_1000_diff1": 0.3568321384391532, + "nauc_map_at_1000_max": 0.26999090632663203, + "nauc_map_at_1000_std": 0.23150834718439864, + "nauc_map_at_100_diff1": 0.357872984312932, + "nauc_map_at_100_max": 0.2707815994312529, + "nauc_map_at_100_std": 0.2310206556257606, + "nauc_map_at_10_diff1": 0.37342184657421573, + "nauc_map_at_10_max": 0.27996073103876784, + "nauc_map_at_10_std": 0.2267919116158103, + "nauc_map_at_1_diff1": 0.4931665319076531, + "nauc_map_at_1_max": 0.34046617367806836, + "nauc_map_at_1_std": 0.22520394629166143, + "nauc_map_at_20_diff1": 0.36463069830617245, + "nauc_map_at_20_max": 0.2747543332241589, + "nauc_map_at_20_std": 0.2289897405088031, + "nauc_map_at_3_diff1": 0.40611981629392646, + "nauc_map_at_3_max": 0.3014379705164891, + "nauc_map_at_3_std": 0.22655778598644558, + "nauc_map_at_5_diff1": 0.38456853729603835, + "nauc_map_at_5_max": 0.28753313941804676, + "nauc_map_at_5_std": 0.2226991023456713, + "nauc_mrr_at_1000_diff1": 0.356832161495167, + "nauc_mrr_at_1000_max": 0.2699909040097771, + "nauc_mrr_at_1000_std": 0.23150833671626686, + "nauc_mrr_at_100_diff1": 0.357872984312932, + "nauc_mrr_at_100_max": 0.2707815994312529, + "nauc_mrr_at_100_std": 0.2310206556257606, + "nauc_mrr_at_10_diff1": 0.37342184657421573, + "nauc_mrr_at_10_max": 0.27996073103876784, + "nauc_mrr_at_10_std": 0.2267919116158103, + "nauc_mrr_at_1_diff1": 0.4931665319076531, + "nauc_mrr_at_1_max": 0.34046617367806836, + "nauc_mrr_at_1_std": 0.22520394629166143, + "nauc_mrr_at_20_diff1": 0.36463069830617245, + "nauc_mrr_at_20_max": 0.2747543332241589, + "nauc_mrr_at_20_std": 0.2289897405088031, + "nauc_mrr_at_3_diff1": 0.40611981629392646, + "nauc_mrr_at_3_max": 0.3014379705164891, + "nauc_mrr_at_3_std": 0.22655778598644558, + "nauc_mrr_at_5_diff1": 0.38456853729603835, + "nauc_mrr_at_5_max": 0.28753313941804676, + "nauc_mrr_at_5_std": 0.2226991023456713, + "nauc_ndcg_at_1000_diff1": 0.26011437580815167, + "nauc_ndcg_at_1000_max": 0.21035983312198475, + "nauc_ndcg_at_1000_std": 0.25938262677666973, + "nauc_ndcg_at_100_diff1": 0.27692777560932097, + "nauc_ndcg_at_100_max": 0.22175842905077686, + "nauc_ndcg_at_100_std": 0.2476316343627969, + "nauc_ndcg_at_10_diff1": 0.3306820261454687, + "nauc_ndcg_at_10_max": 0.2548593820745378, + "nauc_ndcg_at_10_std": 0.2297778323461325, + "nauc_ndcg_at_1_diff1": 0.4931665319076531, + "nauc_ndcg_at_1_max": 0.34046617367806836, + "nauc_ndcg_at_1_std": 0.22520394629166143, + "nauc_ndcg_at_20_diff1": 0.3072405340031752, + "nauc_ndcg_at_20_max": 0.24080054220757713, + "nauc_ndcg_at_20_std": 0.23564872251630792, + "nauc_ndcg_at_3_diff1": 0.3846824095684635, + "nauc_ndcg_at_3_max": 0.29104440148011873, + "nauc_ndcg_at_3_std": 0.22727424904293786, + "nauc_ndcg_at_5_diff1": 0.3518141711145917, + "nauc_ndcg_at_5_max": 0.26947351331044006, + "nauc_ndcg_at_5_std": 0.22086120599448097, + "nauc_precision_at_1000_diff1": 0.08565868923582474, + "nauc_precision_at_1000_max": 0.10138064328519283, + "nauc_precision_at_1000_std": 0.328803073884, + "nauc_precision_at_100_diff1": 0.15064453104841719, + "nauc_precision_at_100_max": 0.1432953523770651, + "nauc_precision_at_100_std": 0.2798925785120355, + "nauc_precision_at_10_diff1": 0.24817886303752315, + "nauc_precision_at_10_max": 0.2055279305111246, + "nauc_precision_at_10_std": 0.23682614089723822, + "nauc_precision_at_1_diff1": 0.4931665319076531, + "nauc_precision_at_1_max": 0.34046617367806836, + "nauc_precision_at_1_std": 0.22520394629166143, + "nauc_precision_at_20_diff1": 0.20448507514630834, + "nauc_precision_at_20_max": 0.17905271116474109, + "nauc_precision_at_20_std": 0.24941512163240534, + "nauc_precision_at_3_diff1": 0.33601488304743476, + "nauc_precision_at_3_max": 0.26725438700647136, + "nauc_precision_at_3_std": 0.22899599659848702, + "nauc_precision_at_5_diff1": 0.28363791409286265, + "nauc_precision_at_5_max": 0.23137672334064413, + "nauc_precision_at_5_std": 0.21663135292170888, + "nauc_recall_at_1000_diff1": 0.08565868923582504, + "nauc_recall_at_1000_max": 0.10138064328519379, + "nauc_recall_at_1000_std": 0.3288030738840006, + "nauc_recall_at_100_diff1": 0.15064453104841738, + "nauc_recall_at_100_max": 0.143295352377065, + "nauc_recall_at_100_std": 0.27989257851203553, + "nauc_recall_at_10_diff1": 0.248178863037523, + "nauc_recall_at_10_max": 0.20552793051112445, + "nauc_recall_at_10_std": 0.23682614089723797, + "nauc_recall_at_1_diff1": 0.4931665319076531, + "nauc_recall_at_1_max": 0.34046617367806836, + "nauc_recall_at_1_std": 0.22520394629166143, + "nauc_recall_at_20_diff1": 0.20448507514630834, + "nauc_recall_at_20_max": 0.17905271116474095, + "nauc_recall_at_20_std": 0.24941512163240517, + "nauc_recall_at_3_diff1": 0.33601488304743465, + "nauc_recall_at_3_max": 0.2672543870064712, + "nauc_recall_at_3_std": 0.22899599659848674, + "nauc_recall_at_5_diff1": 0.2836379140928627, + "nauc_recall_at_5_max": 0.2313767233406441, + "nauc_recall_at_5_std": 0.21663135292170893, + "ndcg_at_1": 0.04895, + "ndcg_at_10": 0.08144, + "ndcg_at_100": 0.10998, + "ndcg_at_1000": 0.14567, + "ndcg_at_20": 0.09018, + "ndcg_at_3": 0.06673, + "ndcg_at_5": 0.07369, + "precision_at_1": 0.04895, + "precision_at_10": 0.01207, + "precision_at_100": 0.00266, + "precision_at_1000": 0.00057, + "precision_at_20": 0.00777, + "precision_at_3": 0.02654, + "precision_at_5": 0.01932, + "recall_at_1": 0.04895, + "recall_at_10": 0.1207, + "recall_at_100": 0.2657, + "recall_at_1000": 0.56601, + "recall_at_20": 0.15544, + "recall_at_3": 0.07963, + "recall_at_5": 0.0966 + } + ] + }, + "task_name": "SciMMIRT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRI2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRI2TRetrieval.json new file mode 100644 index 0000000000..92da9c357d --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", + "evaluation_time": 283.1252419948578, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.10533, + "cv_recall_at_10": 0.24633, + "cv_recall_at_100": 0.4378, + "cv_recall_at_1000": 0.70805, + "cv_recall_at_20": 0.2957, + "cv_recall_at_3": 0.17137, + "cv_recall_at_5": 0.20138, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.17056, + "map_at_1": 0.10533, + "map_at_10": 0.14695, + "map_at_100": 0.15368, + "map_at_1000": 0.15468, + "map_at_20": 0.15037, + "map_at_3": 0.13406, + "map_at_5": 0.14093, + "mrr_at_1": 0.10533111971960893, + "mrr_at_10": 0.1469541826855974, + "mrr_at_100": 0.15368428460896544, + "mrr_at_1000": 0.15467910679952085, + "mrr_at_20": 0.15036538821910328, + "mrr_at_3": 0.1340568570784395, + "mrr_at_5": 0.1409252085511078, + "nauc_cv_recall_at_1000_diff1": 0.23028863328141164, + "nauc_cv_recall_at_1000_max": 0.092046541316207, + "nauc_cv_recall_at_1000_std": 0.4571359824346522, + "nauc_cv_recall_at_100_diff1": 0.2559857960706079, + "nauc_cv_recall_at_100_max": 0.14529317511624498, + "nauc_cv_recall_at_100_std": 0.4106083321055665, + "nauc_cv_recall_at_10_diff1": 0.31671357009704, + "nauc_cv_recall_at_10_max": 0.2194327767019012, + "nauc_cv_recall_at_10_std": 0.35938923946956497, + "nauc_cv_recall_at_1_diff1": 0.5069311041965368, + "nauc_cv_recall_at_1_max": 0.3013928019939395, + "nauc_cv_recall_at_1_std": 0.2965515266383622, + "nauc_cv_recall_at_20_diff1": 0.2913557570660773, + "nauc_cv_recall_at_20_max": 0.1915193655073638, + "nauc_cv_recall_at_20_std": 0.37274328079150615, + "nauc_cv_recall_at_3_diff1": 0.38300334076067827, + "nauc_cv_recall_at_3_max": 0.27189162202407996, + "nauc_cv_recall_at_3_std": 0.3332969427195455, + "nauc_cv_recall_at_5_diff1": 0.34020077997092296, + "nauc_cv_recall_at_5_max": 0.2493568151718566, + "nauc_cv_recall_at_5_std": 0.33961563695736396, + "nauc_map_at_1000_diff1": 0.41636126447481425, + "nauc_map_at_1000_max": 0.26831410429735236, + "nauc_map_at_1000_std": 0.3272211690775936, + "nauc_map_at_100_diff1": 0.4163608476581626, + "nauc_map_at_100_max": 0.2684712221457793, + "nauc_map_at_100_std": 0.327070263065989, + "nauc_map_at_10_diff1": 0.4217520673857625, + "nauc_map_at_10_max": 0.2739379505279472, + "nauc_map_at_10_std": 0.3236386178749276, + "nauc_map_at_1_diff1": 0.5069311041965368, + "nauc_map_at_1_max": 0.3013928019939395, + "nauc_map_at_1_std": 0.2965515266383622, + "nauc_map_at_20_diff1": 0.4181470924103441, + "nauc_map_at_20_max": 0.270497163275252, + "nauc_map_at_20_std": 0.32538970678533347, + "nauc_map_at_3_diff1": 0.44235606110109965, + "nauc_map_at_3_max": 0.2868603836336605, + "nauc_map_at_3_std": 0.3166462314562751, + "nauc_map_at_5_diff1": 0.4273046860941681, + "nauc_map_at_5_max": 0.27968527276930205, + "nauc_map_at_5_std": 0.3192981186456483, + "nauc_mrr_at_1000_diff1": 0.4163612644311463, + "nauc_mrr_at_1000_max": 0.2683141057970173, + "nauc_mrr_at_1000_std": 0.32722116697250775, + "nauc_mrr_at_100_diff1": 0.4163608476581626, + "nauc_mrr_at_100_max": 0.2684712221457793, + "nauc_mrr_at_100_std": 0.327070263065989, + "nauc_mrr_at_10_diff1": 0.4217520673857625, + "nauc_mrr_at_10_max": 0.2739379505279472, + "nauc_mrr_at_10_std": 0.3236386178749276, + "nauc_mrr_at_1_diff1": 0.5069311041965368, + "nauc_mrr_at_1_max": 0.3013928019939395, + "nauc_mrr_at_1_std": 0.2965515266383622, + "nauc_mrr_at_20_diff1": 0.4181470924103441, + "nauc_mrr_at_20_max": 0.270497163275252, + "nauc_mrr_at_20_std": 0.32538970678533347, + "nauc_mrr_at_3_diff1": 0.44235606110109965, + "nauc_mrr_at_3_max": 0.2868603836336605, + "nauc_mrr_at_3_std": 0.3166462314562751, + "nauc_mrr_at_5_diff1": 0.4273046860941681, + "nauc_mrr_at_5_max": 0.27968527276930205, + "nauc_mrr_at_5_std": 0.3192981186456483, + "nauc_ndcg_at_1000_diff1": 0.3701149218651449, + "nauc_ndcg_at_1000_max": 0.23194294078168726, + "nauc_ndcg_at_1000_std": 0.3532591641869934, + "nauc_ndcg_at_100_diff1": 0.3683136220697507, + "nauc_ndcg_at_100_max": 0.23463564798854547, + "nauc_ndcg_at_100_std": 0.35003745669206504, + "nauc_ndcg_at_10_diff1": 0.3893629910934037, + "nauc_ndcg_at_10_max": 0.257886567721901, + "nauc_ndcg_at_10_std": 0.3345022150293404, + "nauc_ndcg_at_1_diff1": 0.5069311041965368, + "nauc_ndcg_at_1_max": 0.3013928019939395, + "nauc_ndcg_at_1_std": 0.2965515266383622, + "nauc_ndcg_at_20_diff1": 0.3786251554746303, + "nauc_ndcg_at_20_max": 0.24722749345684428, + "nauc_ndcg_at_20_std": 0.33980295740359023, + "nauc_ndcg_at_3_diff1": 0.4249771061750892, + "nauc_ndcg_at_3_max": 0.2825108192119049, + "nauc_ndcg_at_3_std": 0.3215594005999232, + "nauc_ndcg_at_5_diff1": 0.40095083808527676, + "nauc_ndcg_at_5_max": 0.27067795407149536, + "nauc_ndcg_at_5_std": 0.32555508887366974, + "nauc_precision_at_1000_diff1": 0.23028863328141058, + "nauc_precision_at_1000_max": 0.09204654131620628, + "nauc_precision_at_1000_std": 0.4571359824346514, + "nauc_precision_at_100_diff1": 0.25598579607060845, + "nauc_precision_at_100_max": 0.1452931751162452, + "nauc_precision_at_100_std": 0.4106083321055668, + "nauc_precision_at_10_diff1": 0.31671357009703993, + "nauc_precision_at_10_max": 0.21943277670190117, + "nauc_precision_at_10_std": 0.35938923946956486, + "nauc_precision_at_1_diff1": 0.5069311041965368, + "nauc_precision_at_1_max": 0.3013928019939395, + "nauc_precision_at_1_std": 0.2965515266383622, + "nauc_precision_at_20_diff1": 0.29135575706607697, + "nauc_precision_at_20_max": 0.19151936550736376, + "nauc_precision_at_20_std": 0.37274328079150626, + "nauc_precision_at_3_diff1": 0.3830033407606782, + "nauc_precision_at_3_max": 0.27189162202407974, + "nauc_precision_at_3_std": 0.33329694271954546, + "nauc_precision_at_5_diff1": 0.3402007799709232, + "nauc_precision_at_5_max": 0.24935681517185637, + "nauc_precision_at_5_std": 0.33961563695736424, + "nauc_recall_at_1000_diff1": 0.23028863328141164, + "nauc_recall_at_1000_max": 0.092046541316207, + "nauc_recall_at_1000_std": 0.4571359824346522, + "nauc_recall_at_100_diff1": 0.2559857960706079, + "nauc_recall_at_100_max": 0.14529317511624498, + "nauc_recall_at_100_std": 0.4106083321055665, + "nauc_recall_at_10_diff1": 0.31671357009704, + "nauc_recall_at_10_max": 0.2194327767019012, + "nauc_recall_at_10_std": 0.35938923946956497, + "nauc_recall_at_1_diff1": 0.5069311041965368, + "nauc_recall_at_1_max": 0.3013928019939395, + "nauc_recall_at_1_std": 0.2965515266383622, + "nauc_recall_at_20_diff1": 0.2913557570660773, + "nauc_recall_at_20_max": 0.1915193655073638, + "nauc_recall_at_20_std": 0.37274328079150615, + "nauc_recall_at_3_diff1": 0.38300334076067827, + "nauc_recall_at_3_max": 0.27189162202407996, + "nauc_recall_at_3_std": 0.3332969427195455, + "nauc_recall_at_5_diff1": 0.34020077997092296, + "nauc_recall_at_5_max": 0.2493568151718566, + "nauc_recall_at_5_std": 0.33961563695736396, + "ndcg_at_1": 0.10533, + "ndcg_at_10": 0.17056, + "ndcg_at_100": 0.20873, + "ndcg_at_1000": 0.24132, + "ndcg_at_20": 0.18302, + "ndcg_at_3": 0.14362, + "ndcg_at_5": 0.15599, + "precision_at_1": 0.10533, + "precision_at_10": 0.02463, + "precision_at_100": 0.00438, + "precision_at_1000": 0.00071, + "precision_at_20": 0.01479, + "precision_at_3": 0.05712, + "precision_at_5": 0.04028, + "recall_at_1": 0.10533, + "recall_at_10": 0.24633, + "recall_at_100": 0.4378, + "recall_at_1000": 0.70805, + "recall_at_20": 0.2957, + "recall_at_3": 0.17137, + "recall_at_5": 0.20138 + } + ] + }, + "task_name": "SciMMIRI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRT2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRT2IRetrieval.json new file mode 100644 index 0000000000..586fd3ba1d --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", + "evaluation_time": 278.578586101532, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.09592, + "cv_recall_at_10": 0.22597, + "cv_recall_at_100": 0.40638, + "cv_recall_at_1000": 0.69796, + "cv_recall_at_20": 0.27104, + "cv_recall_at_3": 0.15255, + "cv_recall_at_5": 0.1833, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.15526, + "map_at_1": 0.09592, + "map_at_10": 0.13334, + "map_at_100": 0.13961, + "map_at_1000": 0.14066, + "map_at_20": 0.13642, + "map_at_3": 0.1207, + "map_at_5": 0.12773, + "mrr_at_1": 0.09592326139088729, + "mrr_at_10": 0.1333431667364518, + "mrr_at_100": 0.13960663411040436, + "mrr_at_1000": 0.14065717797325847, + "mrr_at_20": 0.13641789220130707, + "mrr_at_3": 0.12070343725019922, + "mrr_at_5": 0.12772858636167966, + "nauc_cv_recall_at_1000_diff1": 0.16617922291887638, + "nauc_cv_recall_at_1000_max": 0.16984497409382054, + "nauc_cv_recall_at_1000_std": 0.42084793152355615, + "nauc_cv_recall_at_100_diff1": 0.21608124742748158, + "nauc_cv_recall_at_100_max": 0.20246475499276076, + "nauc_cv_recall_at_100_std": 0.35902357900333665, + "nauc_cv_recall_at_10_diff1": 0.2896832447371392, + "nauc_cv_recall_at_10_max": 0.2679650963958003, + "nauc_cv_recall_at_10_std": 0.2892498692096226, + "nauc_cv_recall_at_1_diff1": 0.4832228454253711, + "nauc_cv_recall_at_1_max": 0.3539648181240017, + "nauc_cv_recall_at_1_std": 0.24684187585451275, + "nauc_cv_recall_at_20_diff1": 0.2572076651086662, + "nauc_cv_recall_at_20_max": 0.23989562009918416, + "nauc_cv_recall_at_20_std": 0.3134865111627789, + "nauc_cv_recall_at_3_diff1": 0.37048797651088045, + "nauc_cv_recall_at_3_max": 0.32499007199890534, + "nauc_cv_recall_at_3_std": 0.26812218154000894, + "nauc_cv_recall_at_5_diff1": 0.33336929117413305, + "nauc_cv_recall_at_5_max": 0.298643405877764, + "nauc_cv_recall_at_5_std": 0.2752327353435506, + "nauc_map_at_1000_diff1": 0.3946988397633675, + "nauc_map_at_1000_max": 0.3183753649842543, + "nauc_map_at_1000_std": 0.26834083843885537, + "nauc_map_at_100_diff1": 0.39476113058528917, + "nauc_map_at_100_max": 0.3184429744599427, + "nauc_map_at_100_std": 0.26806520363704717, + "nauc_map_at_10_diff1": 0.40163266202098985, + "nauc_map_at_10_max": 0.32402854649481405, + "nauc_map_at_10_std": 0.2635706102905207, + "nauc_map_at_1_diff1": 0.4832228454253711, + "nauc_map_at_1_max": 0.3539648181240017, + "nauc_map_at_1_std": 0.24684187585451275, + "nauc_map_at_20_diff1": 0.3971216898413058, + "nauc_map_at_20_max": 0.3203661920396288, + "nauc_map_at_20_std": 0.26622152413508027, + "nauc_map_at_3_diff1": 0.42552280749830074, + "nauc_map_at_3_max": 0.33981729678252126, + "nauc_map_at_3_std": 0.25792014034448607, + "nauc_map_at_5_diff1": 0.41199461734235987, + "nauc_map_at_5_max": 0.33105191712818083, + "nauc_map_at_5_std": 0.2603994992627051, + "nauc_mrr_at_1000_diff1": 0.3946988543879861, + "nauc_mrr_at_1000_max": 0.31837534752269314, + "nauc_mrr_at_1000_std": 0.26834083921874824, + "nauc_mrr_at_100_diff1": 0.39476113058528917, + "nauc_mrr_at_100_max": 0.3184429744599427, + "nauc_mrr_at_100_std": 0.26806520363704717, + "nauc_mrr_at_10_diff1": 0.40163266202098985, + "nauc_mrr_at_10_max": 0.32402854649481405, + "nauc_mrr_at_10_std": 0.2635706102905207, + "nauc_mrr_at_1_diff1": 0.4832228454253711, + "nauc_mrr_at_1_max": 0.3539648181240017, + "nauc_mrr_at_1_std": 0.24684187585451275, + "nauc_mrr_at_20_diff1": 0.3971216898413058, + "nauc_mrr_at_20_max": 0.3203661920396288, + "nauc_mrr_at_20_std": 0.26622152413508027, + "nauc_mrr_at_3_diff1": 0.42552280749830074, + "nauc_mrr_at_3_max": 0.33981729678252126, + "nauc_mrr_at_3_std": 0.25792014034448607, + "nauc_mrr_at_5_diff1": 0.41199461734235987, + "nauc_mrr_at_5_max": 0.33105191712818083, + "nauc_mrr_at_5_std": 0.2603994992627051, + "nauc_ndcg_at_1000_diff1": 0.3388507558240032, + "nauc_ndcg_at_1000_max": 0.28376369750843505, + "nauc_ndcg_at_1000_std": 0.2984960891174002, + "nauc_ndcg_at_100_diff1": 0.3398066680113286, + "nauc_ndcg_at_100_max": 0.284463506938026, + "nauc_ndcg_at_100_std": 0.29289785303561827, + "nauc_ndcg_at_10_diff1": 0.3668889675479959, + "nauc_ndcg_at_10_max": 0.30698993721798895, + "nauc_ndcg_at_10_std": 0.2714358842514923, + "nauc_ndcg_at_1_diff1": 0.4832228454253711, + "nauc_ndcg_at_1_max": 0.3539648181240017, + "nauc_ndcg_at_1_std": 0.24684187585451275, + "nauc_ndcg_at_20_diff1": 0.35340066661737557, + "nauc_ndcg_at_20_max": 0.295972831155534, + "nauc_ndcg_at_20_std": 0.28017707571664463, + "nauc_ndcg_at_3_diff1": 0.409522030418501, + "nauc_ndcg_at_3_max": 0.33553325712974746, + "nauc_ndcg_at_3_std": 0.26089360395464967, + "nauc_ndcg_at_5_diff1": 0.3881318919670405, + "nauc_ndcg_at_5_max": 0.32138515823147434, + "nauc_ndcg_at_5_std": 0.26489458853923464, + "nauc_precision_at_1000_diff1": 0.16617922291887616, + "nauc_precision_at_1000_max": 0.1698449740938206, + "nauc_precision_at_1000_std": 0.42084793152355576, + "nauc_precision_at_100_diff1": 0.21608124742748158, + "nauc_precision_at_100_max": 0.20246475499276068, + "nauc_precision_at_100_std": 0.3590235790033364, + "nauc_precision_at_10_diff1": 0.289683244737139, + "nauc_precision_at_10_max": 0.26796509639579996, + "nauc_precision_at_10_std": 0.2892498692096226, + "nauc_precision_at_1_diff1": 0.4832228454253711, + "nauc_precision_at_1_max": 0.3539648181240017, + "nauc_precision_at_1_std": 0.24684187585451275, + "nauc_precision_at_20_diff1": 0.2572076651086662, + "nauc_precision_at_20_max": 0.23989562009918394, + "nauc_precision_at_20_std": 0.31348651116277904, + "nauc_precision_at_3_diff1": 0.3704879765108802, + "nauc_precision_at_3_max": 0.32499007199890523, + "nauc_precision_at_3_std": 0.2681221815400088, + "nauc_precision_at_5_diff1": 0.333369291174133, + "nauc_precision_at_5_max": 0.29864340587776383, + "nauc_precision_at_5_std": 0.27523273534355047, + "nauc_recall_at_1000_diff1": 0.16617922291887638, + "nauc_recall_at_1000_max": 0.16984497409382054, + "nauc_recall_at_1000_std": 0.42084793152355615, + "nauc_recall_at_100_diff1": 0.21608124742748158, + "nauc_recall_at_100_max": 0.20246475499276076, + "nauc_recall_at_100_std": 0.35902357900333665, + "nauc_recall_at_10_diff1": 0.2896832447371392, + "nauc_recall_at_10_max": 0.2679650963958003, + "nauc_recall_at_10_std": 0.2892498692096226, + "nauc_recall_at_1_diff1": 0.4832228454253711, + "nauc_recall_at_1_max": 0.3539648181240017, + "nauc_recall_at_1_std": 0.24684187585451275, + "nauc_recall_at_20_diff1": 0.2572076651086662, + "nauc_recall_at_20_max": 0.23989562009918416, + "nauc_recall_at_20_std": 0.3134865111627789, + "nauc_recall_at_3_diff1": 0.37048797651088045, + "nauc_recall_at_3_max": 0.32499007199890534, + "nauc_recall_at_3_std": 0.26812218154000894, + "nauc_recall_at_5_diff1": 0.33336929117413305, + "nauc_recall_at_5_max": 0.298643405877764, + "nauc_recall_at_5_std": 0.2752327353435506, + "ndcg_at_1": 0.09592, + "ndcg_at_10": 0.15526, + "ndcg_at_100": 0.19112, + "ndcg_at_1000": 0.22617, + "ndcg_at_20": 0.16658, + "ndcg_at_3": 0.12888, + "ndcg_at_5": 0.14154, + "precision_at_1": 0.09592, + "precision_at_10": 0.0226, + "precision_at_100": 0.00406, + "precision_at_1000": 0.0007, + "precision_at_20": 0.01355, + "precision_at_3": 0.05085, + "precision_at_5": 0.03666, + "recall_at_1": 0.09592, + "recall_at_10": 0.22597, + "recall_at_100": 0.40638, + "recall_at_1000": 0.69796, + "recall_at_20": 0.27104, + "recall_at_3": 0.15255, + "recall_at_5": 0.1833 + } + ] + }, + "task_name": "SciMMIRT2IRetrieval" +} \ No newline at end of file From 6ef5377d7b729943c0614d24a5c9aceeafa3faba Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Sat, 7 Sep 2024 19:38:10 +0100 Subject: [PATCH 063/154] ruff update --- docs/create_tasks_table.py | 8 +-- docs/mmteb/create_points_table.py | 2 + docs/mmteb/validate_points.py | 23 ++++---- mteb/abstasks/AbsTask.py | 2 +- mteb/abstasks/AbsTaskBitextMining.py | 2 +- mteb/abstasks/AbsTaskClassification.py | 2 +- mteb/abstasks/AbsTaskClusteringFast.py | 4 +- mteb/abstasks/AbsTaskInstructionRetrieval.py | 46 ++++++++-------- .../AbsTaskMultilabelClassification.py | 2 +- mteb/abstasks/AbsTaskRetrieval.py | 14 ++--- mteb/abstasks/AbsTaskSummarization.py | 9 +--- .../abstasks/Image/AbsTaskAny2AnyRetrieval.py | 18 +++---- .../Image/AbsTaskImageClassification.py | 4 +- .../AbsTaskImageMultilabelClassification.py | 2 +- .../AbsTaskImageTextPairClassification.py | 10 ++-- mteb/abstasks/TaskMetadata.py | 18 +++---- mteb/abstasks/stratification.py | 7 ++- mteb/evaluation/MTEB.py | 14 ++--- .../evaluators/BitextMiningEvaluator.py | 2 +- .../Image/Any2AnyRetrievalEvaluator.py | 24 ++++----- .../ImageTextPairClassificationEvaluator.py | 6 +-- .../InstructionRetrievalEvaluator.py | 13 ++--- .../evaluators/PairClassificationEvaluator.py | 2 +- .../evaluators/RerankingEvaluator.py | 14 ++--- .../evaluators/RetrievalEvaluator.py | 53 +++++++++---------- mteb/evaluation/evaluators/utils.py | 39 +++++++------- mteb/load_results/__init__.py | 2 + mteb/load_results/mteb_results.py | 6 +-- mteb/models/bm25.py | 8 +-- mteb/models/e5_instruct.py | 2 + mteb/models/google_models.py | 12 ++--- mteb/models/gritlm_models.py | 2 + mteb/models/instructions.py | 1 + mteb/models/llm2vec_models.py | 12 +++-- mteb/models/nomic_models.py | 4 +- mteb/models/ru_sentence_models.py | 1 + mteb/models/salesforce_models.py | 2 + mteb/models/sentence_transformers_models.py | 1 + mteb/overview.py | 11 ++-- mteb/requires_package.py | 4 +- .../multilingual/BibleNLPBitextMining.py | 2 +- .../FilipinoShopeeReviewsClassification.py | 2 + .../heb/HebrewSentimentAnalysis.py | 2 + mteb/tasks/Classification/kor/KorFin.py | 2 + .../multilingual/IndicLangClassification.py | 2 +- .../Classification/multilingual/NaijaSenti.py | 2 + mteb/tasks/Classification/mya/MyanmarNews.py | 2 + .../pol/PolishClassification.py | 6 +-- mteb/tasks/Classification/ron/Moroco.py | 2 + .../tha/WongnaiReviewsClassification .py | 2 + .../Clustering/deu/BlurbsClusteringS2S.py | 2 +- .../eng/ArXivHierarchicalClustering.py | 4 +- .../Clustering/eng/BigPatentClustering.py | 4 +- .../Clustering/eng/MedrxivClusteringP2P.py | 2 +- .../Clustering/eng/MedrxivClusteringS2S.py | 2 +- mteb/tasks/Clustering/eng/RedditClustering.py | 2 +- .../Clustering/eng/RedditClusteringP2P.py | 2 +- .../Clustering/eng/StackExchangeClustering.py | 2 +- .../eng/StackExchangeClusteringP2P.py | 2 +- .../eng/TwentyNewsgroupsClustering.py | 2 +- .../Clustering/fra/AlloProfClusteringP2P.py | 2 + .../Clustering/fra/AlloProfClusteringS2S.py | 2 + mteb/tasks/Clustering/fra/HALClusteringS2S.py | 8 +-- .../Clustering/jpn/LivedoorNewsClustering.py | 2 + .../Clustering/jpn/MewsC16JaClustering.py | 2 + .../multilingual/MLSUMClusteringP2P.py | 2 +- .../multilingual/MLSUMClusteringS2S.py | 2 +- .../multilingual/SIB200ClusteringS2S.py | 4 +- .../multilingual/WikiClusteringP2P.py | 6 +-- mteb/tasks/Clustering/nob/snl_clustering.py | 2 +- mteb/tasks/Clustering/nob/vg_clustering.py | 2 +- mteb/tasks/Clustering/pol/PolishClustering.py | 18 +++---- mteb/tasks/Clustering/zho/CMTEBClustering.py | 8 +-- .../eng/HatefulMemesI2TRetrieval.py | 2 +- .../eng/HatefulMemesT2IRetrieval.py | 2 +- .../eng/MemotionI2TRetrieval.py | 2 +- .../eng/MemotionT2IRetrieval.py | 2 +- .../eng/MNISTClassification.py | 1 - mteb/tasks/Image/__init__.py | 2 + .../MultiLabelClassification/__init__.py | 2 + mteb/tasks/PairClassification/rus/TERRa.py | 2 +- mteb/tasks/Retrieval/dan/DanFeverRetrieval.py | 2 + mteb/tasks/Retrieval/dan/TV2Nordretrieval.py | 2 + .../Retrieval/dan/TwitterHjerneRetrieval.py | 2 + .../tasks/Retrieval/deu/GermanDPRRetrieval.py | 10 ++-- mteb/tasks/Retrieval/eng/BrightRetrieval.py | 2 + mteb/tasks/Retrieval/eng/FEVERRetrieval.py | 4 +- .../tasks/Retrieval/eng/FaithDialRetrieval.py | 10 ++-- mteb/tasks/Retrieval/eng/HagridRetrieval.py | 9 ++-- mteb/tasks/Retrieval/eng/HotpotQARetrieval.py | 2 +- .../Retrieval/eng/LEMBNarrativeQARetrieval.py | 2 + .../Retrieval/eng/LEMBNeedleRetrieval.py | 2 + .../Retrieval/eng/LEMBPasskeyRetrieval.py | 2 + .../tasks/Retrieval/eng/LEMBQMSumRetrieval.py | 2 + .../eng/LEMBSummScreenFDRetrieval.py | 2 + .../Retrieval/eng/LEMBWikimQARetrieval.py | 2 + mteb/tasks/Retrieval/eng/MLQuestions.py | 8 +-- .../Retrieval/eng/NarrativeQARetrieval.py | 2 +- mteb/tasks/Retrieval/eng/QuoraRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/SCIDOCSRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py | 6 ++- .../multilingual/BelebeleRetrieval.py | 2 +- ...CrossLingualSemanticDiscriminationWMT19.py | 6 +-- ...CrossLingualSemanticDiscriminationWMT21.py | 6 +-- .../Retrieval/multilingual/MLQARetrieval.py | 6 +-- .../Retrieval/multilingual/XPQARetrieval.py | 4 +- mteb/tasks/Retrieval/nob/norquad.py | 2 + mteb/tasks/Retrieval/nob/snl_retrieval.py | 2 + mteb/tasks/Retrieval/swe/SweFaqRetrieval.py | 2 + mteb/tasks/Retrieval/swe/SwednRetrieval.py | 2 + mteb/tasks/Retrieval/tur/TurHistQuad.py | 2 + mteb/tasks/STS/deu/GermanSTSBenchmarkSTS.py | 2 +- mteb/tasks/STS/jpn/JSTS.py | 2 +- .../STSBenchmarkMultilingualSTS.py | 2 +- mteb/tasks/STS/multilingual/SemRel24STS.py | 6 +-- mteb/tasks/STS/rus/RuSTSBenchmarkSTS.py | 2 +- mteb/tasks/SpeedTask/__init__.py | 2 + pyproject.toml | 21 +++++++- scripts/data/arxiv/script_raw.py | 2 +- scripts/data/bucc/create_data.py | 6 +-- scripts/data/create_task_table.py | 4 +- .../data/medicalqaretrieval/create_data.py | 3 +- scripts/data/mind/prepare_data.py | 2 +- scripts/data/redditp2p/script_clustering.py | 2 +- .../data/wikipedia_reranking/create_data.py | 4 +- .../data/wikipedia_retrieval/create_data.py | 4 +- scripts/mmteb/create_dataset_citations_bib.py | 10 ++-- .../result_analysis/create_result_tables.py | 2 +- scripts/mmteb/running_baseline_model.py | 2 + scripts/mmteb/running_model/check_results.py | 4 +- .../task_selection/benchmark_construction.py | 3 +- scripts/run_mteb_bright.py | 2 + tests/test_TaskMetadata.py | 4 +- tests/test_benchmark/mock_tasks.py | 36 ++++++------- tests/test_benchmark/test_benchmark.py | 3 +- ...est_benchmark_integration_with_datasets.py | 3 +- ...k_integration_with_sentencetransformers.py | 3 +- tests/test_cli.py | 1 + tests/test_encoder_interfaces.py | 2 + .../test_ClusteringEvaluator.py | 4 +- tests/test_langscripts.py | 6 +-- .../test_mteb_load_results.py | 2 + tests/test_load_results/test_mteb_results.py | 2 + .../test_clustering_fast_datasets.py | 2 + tests/test_tasks/test_mieb_datasets.py | 3 +- tests/test_tasks/test_retrieval_abstask.py | 2 + 146 files changed, 451 insertions(+), 349 deletions(-) diff --git a/docs/create_tasks_table.py b/docs/create_tasks_table.py index 95a47fd071..acbc9bcebd 100644 --- a/docs/create_tasks_table.py +++ b/docs/create_tasks_table.py @@ -88,10 +88,10 @@ def create_task_lang_table(tasks: list[mteb.AbsTask]) -> str: task_names_md = " | ".join(sorted(get_args(TASK_TYPE))) horizontal_line_md = "---|---" * len(sorted(get_args(TASK_TYPE))) - table = """ -| Language | {} | -|{}| -""".format(task_names_md, horizontal_line_md) + table = f""" +| Language | {task_names_md} | +|{horizontal_line_md}| +""" for row in df.iter_rows(): table += f"| {row[-1]} " diff --git a/docs/mmteb/create_points_table.py b/docs/mmteb/create_points_table.py index c30dd1600d..dd3657eb86 100644 --- a/docs/mmteb/create_points_table.py +++ b/docs/mmteb/create_points_table.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json from pathlib import Path diff --git a/docs/mmteb/validate_points.py b/docs/mmteb/validate_points.py index 5098283021..6843d6ffb3 100644 --- a/docs/mmteb/validate_points.py +++ b/docs/mmteb/validate_points.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import logging import os -from typing import Optional from jsonlines import Reader from pydantic import BaseModel, ConfigDict, Field, ValidationError, conint, constr @@ -21,17 +22,17 @@ class JsonObject(BaseModel): model_config = ConfigDict(extra="forbid") GitHub: constr(min_length=1) - new_dataset: Optional[conint(ge=1)] = Field(alias="New dataset", default=None) - new_task: Optional[conint(ge=2)] = Field(alias="New task", default=None) - dataset_annotations: Optional[conint(ge=1)] = Field( + new_dataset: conint(ge=1) | None = Field(alias="New dataset", default=None) + new_task: conint(ge=2) | None = Field(alias="New task", default=None) + dataset_annotations: conint(ge=1) | None = Field( alias="Dataset annotations", default=None ) - bug_fixes: Optional[conint(ge=1)] = Field(alias="Bug fixes", default=None) - running_models: Optional[conint(ge=1)] = Field(alias="Running Models", default=None) - review_pr: Optional[conint(ge=2)] = Field(alias="Review PR", default=None) - paper_writing: Optional[int] = Field(alias="Paper writing", default=None) - Ideation: Optional[int] = None - Coordination: Optional[int] = None + bug_fixes: conint(ge=1) | None = Field(alias="Bug fixes", default=None) + running_models: conint(ge=1) | None = Field(alias="Running Models", default=None) + review_pr: conint(ge=2) | None = Field(alias="Review PR", default=None) + paper_writing: int | None = Field(alias="Paper writing", default=None) + Ideation: int | None = None + Coordination: int | None = None def check_max_points(obj: JsonObject, commit_n: str): @@ -46,7 +47,7 @@ def validate_jsonl_files(folder_path): if filename.endswith(".jsonl"): file_path = os.path.join(folder_path, filename) commit_n = os.path.splitext(filename)[0] - with open(file_path, "r", encoding="utf-8") as file: + with open(file_path, encoding="utf-8") as file: try: # Read JSONL file reader = Reader(file) diff --git a/mteb/abstasks/AbsTask.py b/mteb/abstasks/AbsTask.py index f474e4b919..c772125b8d 100644 --- a/mteb/abstasks/AbsTask.py +++ b/mteb/abstasks/AbsTask.py @@ -105,7 +105,7 @@ def evaluate( scores = {} hf_subsets = ( - [l for l in self.dataset.keys()] if self.is_multilingual else ["default"] + list(self.dataset.keys()) if self.is_multilingual else ["default"] ) for hf_subset in hf_subsets: diff --git a/mteb/abstasks/AbsTaskBitextMining.py b/mteb/abstasks/AbsTaskBitextMining.py index 08bceee37c..1820f9873b 100644 --- a/mteb/abstasks/AbsTaskBitextMining.py +++ b/mteb/abstasks/AbsTaskBitextMining.py @@ -40,7 +40,7 @@ def evaluate( if not self.data_loaded: self.load_data() - hf_subsets = [l for l in self.dataset] if self.is_multilingual else ["default"] + hf_subsets = list(self.dataset) if self.is_multilingual else ["default"] scores = {} if self.parallel_subsets: diff --git a/mteb/abstasks/AbsTaskClassification.py b/mteb/abstasks/AbsTaskClassification.py index 5ce6f64ceb..d5996e01e4 100644 --- a/mteb/abstasks/AbsTaskClassification.py +++ b/mteb/abstasks/AbsTaskClassification.py @@ -78,7 +78,7 @@ def evaluate( self.load_data() scores = {} - hf_subsets = [l for l in self.dataset] if self.is_multilingual else ["default"] + hf_subsets = list(self.dataset) if self.is_multilingual else ["default"] for hf_subset in hf_subsets: logger.info( diff --git a/mteb/abstasks/AbsTaskClusteringFast.py b/mteb/abstasks/AbsTaskClusteringFast.py index 7ac4480f2c..bc3ac608f0 100644 --- a/mteb/abstasks/AbsTaskClusteringFast.py +++ b/mteb/abstasks/AbsTaskClusteringFast.py @@ -4,7 +4,7 @@ import logging import random from collections import defaultdict -from typing import Any, Dict, Optional +from typing import Any, Dict import numpy as np import sklearn @@ -30,7 +30,7 @@ def evaluate_clustering_bootstrapped( n_clusters: int, cluster_size: int, kmean_batch_size: int, - max_depth: Optional[int], + max_depth: int | None, rng_state: random.Random = random.Random(), ) -> dict[str, list[float]]: """Bootstrapped evaluation of clustering performance using V-measure. diff --git a/mteb/abstasks/AbsTaskInstructionRetrieval.py b/mteb/abstasks/AbsTaskInstructionRetrieval.py index 9b24b72a48..1bdff2cefb 100644 --- a/mteb/abstasks/AbsTaskInstructionRetrieval.py +++ b/mteb/abstasks/AbsTaskInstructionRetrieval.py @@ -5,7 +5,7 @@ import os from collections import defaultdict from time import time -from typing import Any, Dict, List, Tuple, Union +from typing import Any import tqdm from datasets import Dataset, Features, Value, load_dataset @@ -71,11 +71,11 @@ def __init__( def load( self, split="test" - ) -> Tuple[ + ) -> tuple[ Dataset, Dataset, - Dict[str, Dict[str, int]], - Dict[str, Dict[str, int]], + dict[str, dict[str, int]], + dict[str, dict[str, int]], Dataset, ]: if not self.hf_repo: @@ -324,19 +324,19 @@ def load_data(self, **kwargs): def _evaluate_subset_lang( self, retriever: InstructionRetrievalEvaluator, - corpus: Dict, - queries: Dict, - og_relevant_docs: Dict, - changed_relevant_docs: Dict, - og_instructions: Dict, - changed_instructions: Dict, - top_ranked: Dict, + corpus: dict, + queries: dict, + og_relevant_docs: dict, + changed_relevant_docs: dict, + og_instructions: dict, + changed_instructions: dict, + top_ranked: dict, lang: str, split: str, - keywords: Union[Dict, None] = None, - short_instructions: Union[Dict, None] = None, + keywords: dict | None = None, + short_instructions: dict | None = None, **kwargs, - ) -> Dict[str, Union[Dict[str, float], float]]: + ) -> dict[str, dict[str, float] | float]: corpus, queries = corpus[split], queries[split] og_relevant_docs, changed_relevant_docs = ( og_relevant_docs[split], @@ -432,9 +432,9 @@ def evaluate( model: Encoder, split: str = "test", *, - encode_kwargs: Dict[str, Any] = {}, + encode_kwargs: dict[str, Any] = {}, **kwargs, - ) -> Dict[str, Dict[str, Any]]: + ) -> dict[str, dict[str, Any]]: retriever = InstructionRetrievalEvaluator( retriever=model, task_name=self.metadata.name, @@ -492,14 +492,14 @@ def _add_main_score(self, scores: dict[str, dict[str, float]]) -> None: def _evaluate_subset( self, retriever: InstructionRetrievalEvaluator, - corpus: Dict[str, Dict[str, str]], - queries: Dict[str, str], - relevant_docs: Dict[str, Dict[str, int]], - instructions: Dict[str, str], - top_ranked: Dict[str, List[str]], + corpus: dict[str, dict[str, str]], + queries: dict[str, str], + relevant_docs: dict[str, dict[str, int]], + instructions: dict[str, str], + top_ranked: dict[str, list[str]], lang=None, **kwargs, - ) -> Tuple[Dict[str, float], Dict[str, Dict[str, float]]]: + ) -> tuple[dict[str, float], dict[str, dict[str, float]]]: start_time = time() # do the results by query and relevant docs only @@ -524,7 +524,7 @@ def _evaluate_subset( end_time = time() logger.info( - "Time taken to retrieve: {:.2f} seconds".format(end_time - start_time) + f"Time taken to retrieve: {end_time - start_time:.2f} seconds" ) if kwargs.get("save_predictions", False): diff --git a/mteb/abstasks/AbsTaskMultilabelClassification.py b/mteb/abstasks/AbsTaskMultilabelClassification.py index c5245fde3f..bbd7347a99 100644 --- a/mteb/abstasks/AbsTaskMultilabelClassification.py +++ b/mteb/abstasks/AbsTaskMultilabelClassification.py @@ -90,7 +90,7 @@ def evaluate( self.load_data() scores = {} - hf_subsets = [l for l in self.dataset] if self.is_multilingual else ["default"] + hf_subsets = list(self.dataset) if self.is_multilingual else ["default"] for hf_subset in hf_subsets: logger.info( diff --git a/mteb/abstasks/AbsTaskRetrieval.py b/mteb/abstasks/AbsTaskRetrieval.py index 1dedf6883d..95f05f57af 100644 --- a/mteb/abstasks/AbsTaskRetrieval.py +++ b/mteb/abstasks/AbsTaskRetrieval.py @@ -6,7 +6,7 @@ from collections import defaultdict from pathlib import Path from time import time -from typing import Any, Dict, Tuple +from typing import Any import tqdm from datasets import Features, Value, load_dataset @@ -66,17 +66,17 @@ def __init__( def check(fIn: str, ext: str): if not os.path.exists(fIn): raise ValueError( - "File {} not present! Please provide accurate file.".format(fIn) + f"File {fIn} not present! Please provide accurate file." ) if not fIn.endswith(ext): raise ValueError( - "File {} must be present with extension {}".format(fIn, ext) + f"File {fIn} must be present with extension {ext}" ) def load( self, split="test" - ) -> Tuple[Dict[str, dict[str, str]], dict[str, str], dict[str, dict[str, int]]]: + ) -> tuple[dict[str, dict[str, str]], dict[str, str], dict[str, dict[str, int]]]: if not self.hf_repo: self.qrels_file = os.path.join(self.qrels_folder, split + ".tsv") self.check(fIn=self.corpus_file, ext="jsonl") @@ -265,7 +265,7 @@ def evaluate( scores = {} hf_subsets = ( - [l for l in self.hf_subsets] if self.is_multilingual else ["default"] + list(self.hf_subsets) if self.is_multilingual else ["default"] ) for hf_subset in hf_subsets: @@ -295,7 +295,7 @@ def _evaluate_subset( results = retriever(corpus, queries) end_time = time() logger.info( - "Time taken to retrieve: {:.2f} seconds".format(end_time - start_time) + f"Time taken to retrieve: {end_time - start_time:.2f} seconds" ) save_predictions = kwargs.get("save_predictions", False) @@ -360,7 +360,7 @@ def _evaluate_subset( sorted_docs = sorted( doc_scores.items(), key=lambda x: x[1], reverse=True )[:top_k] - results[qid] = {doc_id: score for doc_id, score in sorted_docs} + results[qid] = dict(sorted_docs) for qid, retrieved_docs in results.items(): expected_docs = relevant_docs[qid] false_positives = [ diff --git a/mteb/abstasks/AbsTaskSummarization.py b/mteb/abstasks/AbsTaskSummarization.py index c500193b15..9cbd3dd29c 100644 --- a/mteb/abstasks/AbsTaskSummarization.py +++ b/mteb/abstasks/AbsTaskSummarization.py @@ -38,13 +38,8 @@ def max_score(self): def _evaluate_subset( self, model: Encoder, data_split, *, encode_kwargs: dict[str, Any], **kwargs ) -> ScoresDict: - normalized_scores = list( - map( - lambda x: (np.array(x) - self.min_score) - / (self.max_score - self.min_score), - data_split["relevance"], - ) - ) + normalized_scores = [(np.array(x) - self.min_score) + / (self.max_score - self.min_score) for x in data_split["relevance"]] evaluator = SummarizationEvaluator( machine_summaries=data_split["machine_summaries"], human_summaries=data_split["human_summaries"], diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py index 8742c49522..8f3ca548ce 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py @@ -6,7 +6,7 @@ from collections import defaultdict from pathlib import Path from time import time -from typing import Any, Dict, Tuple +from typing import Any import tqdm from datasets import Features, Value, load_dataset @@ -66,19 +66,19 @@ def __init__( def check(fIn: str, ext: str): if not os.path.exists(fIn): raise ValueError( - "File {} not present! Please provide accurate file.".format(fIn) + f"File {fIn} not present! Please provide accurate file." ) if not fIn.endswith(ext): raise ValueError( - "File {} must be present with extension {}".format(fIn, ext) + f"File {fIn} must be present with extension {ext}" ) def load( self, split="test" - ) -> Tuple[ - Dict[str, Dict[str, str | Image.Image]], - Dict[str, Dict[str, str | Image.Image]], + ) -> tuple[ + dict[str, dict[str, str | Image.Image]], + dict[str, dict[str, str | Image.Image]], dict[str, dict[str, int]], ]: if not self.hf_repo: @@ -255,7 +255,7 @@ def evaluate( scores = {} hf_subsets = ( - [l for l in self.hf_subsets] if self.is_multilingual else ["default"] + list(self.hf_subsets) if self.is_multilingual else ["default"] ) for hf_subset in hf_subsets: @@ -285,7 +285,7 @@ def _evaluate_subset( results = retriever(corpus, queries) end_time = time() logger.info( - "Time taken to retrieve: {:.2f} seconds".format(end_time - start_time) + f"Time taken to retrieve: {end_time - start_time:.2f} seconds" ) save_predictions = kwargs.get("save_predictions", False) @@ -352,7 +352,7 @@ def _evaluate_subset( sorted_docs = sorted( doc_scores.items(), key=lambda x: x[1], reverse=True )[:top_k] - results[qid] = {doc_id: score for doc_id, score in sorted_docs} + results[qid] = dict(sorted_docs) for qid, retrieved_docs in results.items(): expected_docs = relevant_docs[qid] false_positives = [ diff --git a/mteb/abstasks/Image/AbsTaskImageClassification.py b/mteb/abstasks/Image/AbsTaskImageClassification.py index 958e5db424..9d52760600 100644 --- a/mteb/abstasks/Image/AbsTaskImageClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageClassification.py @@ -52,7 +52,7 @@ def __init__( self.samples_per_label: int = ( # type: ignore samples_per_label if samples_per_label is not None - else self.metadata_dict.get("samples_per_label", 16) + else self.metadata_dict.get("samples_per_label", 64) ) # kNN parameters @@ -81,7 +81,7 @@ def evaluate( self.load_data() scores = {} - hf_subsets = [l for l in self.dataset] if self.is_multilingual else ["default"] + hf_subsets = list(self.dataset) if self.is_multilingual else ["default"] for hf_subset in hf_subsets: logger.info( diff --git a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py index 91384cf581..380e9a4b23 100644 --- a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py @@ -91,7 +91,7 @@ def evaluate( self.load_data() scores = {} - hf_subsets = [l for l in self.dataset] if self.is_multilingual else ["default"] + hf_subsets = list(self.dataset) if self.is_multilingual else ["default"] for hf_subset in hf_subsets: logger.info( diff --git a/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py index b86114df91..de6e3e32dd 100644 --- a/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Any, List, Union +from typing import Any from datasets import Dataset from tqdm import tqdm @@ -26,15 +26,15 @@ class AbsTaskImageTextPairClassification(AbsTask): """ # it can be ["image_0", "image_1"]; ["text_0", "text_1"] for datasets like WinoGround - images_column_names: Union[str, List[str]] = "image" - texts_column_names: Union[str, List[str]] = "caption" + images_column_names: str | list[str] = "image" + texts_column_names: str | list[str] = "caption" def __init__(self, **kwargs): super().__init__(**kwargs) def _preprocess_column( - self, dataset: Dataset, column_names: Union[str, List[str]] - ) -> List[List[Any]]: + self, dataset: Dataset, column_names: str | list[str] + ) -> list[list[Any]]: """Group examples from the columns into a list of examples.""" if isinstance(column_names, str): return dataset[column_names] diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index fae6d29b58..5bc650c1ba 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -2,7 +2,7 @@ import logging from datetime import date -from typing import Any, Dict, List, Mapping, Optional, Union +from typing import Any, Dict, List, Mapping, Union from pydantic import AnyUrl, BaseModel, BeforeValidator, TypeAdapter, field_validator from typing_extensions import Annotated, Literal @@ -219,7 +219,7 @@ class TaskMetadata(BaseModel): sample_creation: SAMPLE_CREATION_METHOD | None bibtex_citation: str | None - descriptive_stats: dict[METRIC_NAME, Optional[dict[SPLIT_NAME, METRIC_VALUE]]] + descriptive_stats: dict[METRIC_NAME, dict[SPLIT_NAME, METRIC_VALUE] | None] @field_validator("dataset") def _check_dataset_path_is_specified(cls, dataset): @@ -227,7 +227,7 @@ def _check_dataset_path_is_specified(cls, dataset): if "path" not in dataset or dataset["path"] is None: raise ValueError( "You must specify the path to the dataset in the dataset dictionary. " - "See https://huggingface.co/docs/datasets/main/en/package_reference/loading_methods#datasets.load_dataset" + + "See https://huggingface.co/docs/datasets/main/en/package_reference/loading_methods#datasets.load_dataset" ) return dataset @@ -285,13 +285,13 @@ def get_lang(lang: str) -> str: if isinstance(self.eval_langs, dict): return sorted( - set( + { get_lang(lang) for langs in self.eval_langs.values() for lang in langs - ) + } ) - return sorted(set([get_lang(lang) for lang in self.eval_langs])) + return sorted({get_lang(lang) for lang in self.eval_langs}) @property def scripts(self) -> set[str]: @@ -301,10 +301,10 @@ def get_script(lang: str) -> str: return lang.split("-")[1] if isinstance(self.eval_langs, dict): - return set( + return { get_script(lang) for langs in self.eval_langs.values() for lang in langs - ) - return set(get_script(lang) for lang in self.eval_langs) + } + return {get_script(lang) for lang in self.eval_langs} def is_filled(self) -> bool: """Check if all the metadata fields are filled.""" diff --git a/mteb/abstasks/stratification.py b/mteb/abstasks/stratification.py index c6dce6d99a..c739ebd812 100644 --- a/mteb/abstasks/stratification.py +++ b/mteb/abstasks/stratification.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """The following code is a copy of https://github.com/scikit-multilearn/scikit-multilearn/blob/master/skmultilearn/model_selection/iterative_stratification.py from the amazing scikit-multilearn library. Please try it out: https://github.com/scikit-multilearn/scikit-multilearn @@ -36,6 +35,7 @@ publisher = {PMLR}, } """ +from __future__ import annotations import itertools @@ -178,7 +178,7 @@ def __init__( self._rng_state = check_random_state(random_state) need_shuffle = shuffle or random_state is not None self.order = order - super(IterativeStratification, self).__init__( + super().__init__( n_splits, shuffle=need_shuffle, random_state=self._rng_state if need_shuffle else None, @@ -369,5 +369,4 @@ def _iter_test_indices(self, X, y=None, groups=None): ) self._distribute_negative_evidence(rows_used, folds) - for fold in folds: - yield fold + yield from folds diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py index 24213da4c0..c6b55ee10a 100644 --- a/mteb/evaluation/MTEB.py +++ b/mteb/evaluation/MTEB.py @@ -114,11 +114,11 @@ def available_tasks(self): @property def available_task_types(self): - return set([x.metadata_dict["type"] for x in self.tasks_cls]) + return {x.metadata_dict["type"] for x in self.tasks_cls} @property def available_task_categories(self): - return set([x.metadata_dict["category"] for x in self.tasks_cls]) + return {x.metadata_dict["category"] for x in self.tasks_cls} def _extend_lang_code(self): # add all possible language codes @@ -181,7 +181,7 @@ def print_selected_tasks(self): def select_tasks(self, **kwargs): """Select the tasks to be evaluated.""" # Get all existing tasks - tasks_categories_cls = [cls for cls in AbsTask.__subclasses__()] + tasks_categories_cls = list(AbsTask.__subclasses__()) self.tasks_cls = [ cls(hf_subsets=self._task_langs, **kwargs) for cat_cls in tasks_categories_cls @@ -197,14 +197,14 @@ def select_tasks(self, **kwargs): ) ) if len(self.tasks) != len(self._tasks): - tasks_known = set([x.metadata_dict["name"] for x in self.tasks_cls]) + tasks_known = {x.metadata_dict["name"] for x in self.tasks_cls} tasks_unknown = ( - set(x for x in self._tasks if isinstance(x, str)) - tasks_known + {x for x in self._tasks if isinstance(x, str)} - tasks_known ) if tasks_unknown: unknown_str, known_str = ( - ",".join(sorted(list(tasks_unknown))), - ",".join(sorted(list(tasks_known))), + ",".join(sorted(tasks_unknown)), + ",".join(sorted(tasks_known)), ) logger.warning( f"WARNING: Unknown tasks: {unknown_str}. Known tasks: {known_str}." diff --git a/mteb/evaluation/evaluators/BitextMiningEvaluator.py b/mteb/evaluation/evaluators/BitextMiningEvaluator.py index 8f233bac91..ae3f3dd73e 100644 --- a/mteb/evaluation/evaluators/BitextMiningEvaluator.py +++ b/mteb/evaluation/evaluators/BitextMiningEvaluator.py @@ -43,7 +43,7 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): return scores def compute_metrics(self, model: Encoder, encode_kwargs: dict[str, Any] = {}): - pair_elements = set(p for pair in self.pairs for p in pair) + pair_elements = {p for pair in self.pairs for p in pair} subsets = [ col for col in self.sentences.features.keys() if col in pair_elements ] diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index 540c37bbb3..7f76c250e7 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -6,7 +6,7 @@ import logging import os from collections import defaultdict -from typing import Any, Dict, List, Tuple +from typing import Any import numpy as np import pytrec_eval @@ -155,9 +155,7 @@ def search( logger.info("Encoding Corpus in batches... Warning: This might take a while!") logger.info( - "Scoring Function: {} ({})".format( - self.score_function_desc[score_function], score_function - ) + f"Scoring Function: {self.score_function_desc[score_function]} ({score_function})" ) result_heaps = {qid: [] for qid in query_ids} @@ -250,7 +248,7 @@ def load_results_file(self): ) self.previous_results = dest_file - with open(self.previous_results, "r") as f: + with open(self.previous_results) as f: previous_results = json.load(f) assert isinstance(previous_results, dict) assert isinstance(previous_results[list(previous_results.keys())[0]], dict) @@ -263,7 +261,7 @@ def __init__( self, retriever=None, task_name: str | None = None, - k_values: List[int] = [1, 3, 5, 10, 20, 100, 1000], + k_values: list[int] = [1, 3, 5, 10, 20, 100, 1000], score_function: str = "cos_sim", encode_kwargs: dict[str, Any] = {}, **kwargs, @@ -282,8 +280,8 @@ def __init__( def __call__( self, - corpus: dict[str, Dict[str, str | Image.Image]], - queries: dict[str, Dict[str, str | Image.Image]], + corpus: dict[str, dict[str, str | Image.Image]], + queries: dict[str, dict[str, str | Image.Image]], ) -> dict[str, dict[str, float]]: if not self.retriever: raise ValueError("Model/Technique has not been provided!") @@ -300,10 +298,10 @@ def __call__( def evaluate( qrels: dict[str, dict[str, int]], results: dict[str, dict[str, float]], - k_values: List[int], + k_values: list[int], ignore_identical_ids: bool = False, skip_first_result: bool = False, - ) -> Tuple[ + ) -> tuple[ dict[str, float], dict[str, float], dict[str, float], @@ -403,10 +401,10 @@ def evaluate( def evaluate_custom( qrels: dict[str, dict[str, int]], results: dict[str, dict[str, float]], - k_values: List[int], + k_values: list[int], metric: str, output_type: str = "all", - ) -> Tuple[Dict[str, float]]: + ) -> tuple[dict[str, float]]: if metric.lower() in ["mrr", "mrr@k", "mrr_cut"]: metric_scores = mrr(qrels, results, k_values, output_type) @@ -434,7 +432,7 @@ def evaluate_custom( def evaluate_abstention( results: dict[str, dict[str, float]], metric_scores: dict[str, list[float]], - ) -> Dict[str, float]: + ) -> dict[str, float]: """Computes normalized Area Under the Curve on a set of evaluated instances as presented in the paper https://arxiv.org/abs/2402.12997""" all_sim_scores = [list(results[qid].values()) for qid in list(results.keys())] all_conf_scores = [ diff --git a/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py index 500aaf5f74..403b3758f2 100644 --- a/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py @@ -2,7 +2,7 @@ import itertools import logging -from typing import Any, List +from typing import Any import torch import torch.nn.functional as F @@ -30,8 +30,8 @@ class ImageTextPairClassificationEvaluator(Evaluator): def __init__( self, - images: List[List[Image.Image]], - texts: List[List[str]], + images: list[list[Image.Image]], + texts: list[list[str]], task_name: str | None = None, limit: int | None = None, **kwargs, diff --git a/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py b/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py index 8828aa2a22..10c78d5d0d 100644 --- a/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py @@ -1,5 +1,6 @@ +from __future__ import annotations + import logging -from typing import Dict, Union from .RetrievalEvaluator import ( RetrievalEvaluator, @@ -12,12 +13,12 @@ class InstructionRetrievalEvaluator(RetrievalEvaluator): # only added to extend the RetrievalEvaluator to pass along the instructions def __call__( self, - corpus: Dict[str, Dict[str, str]], - queries: Dict[str, str], - instructions: Dict[str, str], - qid: Union[str, None] = None, + corpus: dict[str, dict[str, str]], + queries: dict[str, str], + instructions: dict[str, str], + qid: str | None = None, **kwargs, - ) -> Dict[str, Dict[str, float]]: + ) -> dict[str, dict[str, float]]: if not self.retriever: raise ValueError("Model/Technique has not been provided!") diff --git a/mteb/evaluation/evaluators/PairClassificationEvaluator.py b/mteb/evaluation/evaluators/PairClassificationEvaluator.py index b38bf53c1f..9212a9ff41 100644 --- a/mteb/evaluation/evaluators/PairClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/PairClassificationEvaluator.py @@ -96,7 +96,7 @@ def compute_metrics( prompt_name=self.task_name, **encode_kwargs, ) - emb_dict = {sent: emb for sent, emb in zip(sentences, embeddings)} + emb_dict = dict(zip(sentences, embeddings)) embeddings1 = [emb_dict[sent] for sent in self.sentences1] embeddings2 = [emb_dict[sent] for sent in self.sentences2] diff --git a/mteb/evaluation/evaluators/RerankingEvaluator.py b/mteb/evaluation/evaluators/RerankingEvaluator.py index bce203fd02..60720954f5 100644 --- a/mteb/evaluation/evaluators/RerankingEvaluator.py +++ b/mteb/evaluation/evaluators/RerankingEvaluator.py @@ -2,7 +2,7 @@ import logging from functools import partial -from typing import Any, Callable, Dict, List +from typing import Any, Callable import numpy as np import torch @@ -459,8 +459,8 @@ def _compute_sim_scores_instance( return sim_scores def _compute_metrics_instance( - self, sim_scores: torch.Tensor, is_relevant: List[bool] - ) -> Dict[str, float]: + self, sim_scores: torch.Tensor, is_relevant: list[bool] + ) -> dict[str, float]: """Computes metrics for a single instance = (query, positives, negatives) Args: @@ -478,7 +478,7 @@ def _compute_metrics_instance( return {"mrr": mrr, "ap": ap} @staticmethod - def conf_scores(sim_scores: torch.Tensor) -> Dict[str, float]: + def conf_scores(sim_scores: torch.Tensor) -> dict[str, float]: """Computes confidence scores for a single instance = (query, positives, negatives) Args: @@ -494,10 +494,10 @@ def conf_scores(sim_scores: torch.Tensor) -> Dict[str, float]: @staticmethod def nAUC_scores( - all_conf_scores: List[Dict[str, float]], - metrics: List[float], + all_conf_scores: list[dict[str, float]], + metrics: list[float], metric_name: str, - ) -> Dict[str, float]: + ) -> dict[str, float]: """Computes normalized Area Under the Curve on a set of evaluated instances as presented in the paper https://arxiv.org/abs/2402.12997 Args: diff --git a/mteb/evaluation/evaluators/RetrievalEvaluator.py b/mteb/evaluation/evaluators/RetrievalEvaluator.py index 3f767d83fe..6de33c71d7 100644 --- a/mteb/evaluation/evaluators/RetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/RetrievalEvaluator.py @@ -5,7 +5,7 @@ import logging import os from collections import defaultdict -from typing import Any, Dict, List, Tuple, Union +from typing import Any import numpy as np import pytrec_eval @@ -80,12 +80,12 @@ def __init__( def search( self, corpus: dict[str, dict[str, str]], - queries: dict[str, Union[str, List[str]]], + queries: dict[str, str | list[str]], top_k: int, score_function: str, prompt_name: str, - instructions: Dict[str, str] | None = None, - request_qid: Union[str, None] = None, + instructions: dict[str, str] | None = None, + request_qid: str | None = None, return_sorted: bool = False, **kwargs, ) -> dict[str, dict[str, float]]: @@ -127,9 +127,7 @@ def search( logger.info("Encoding Corpus in batches... Warning: This might take a while!") logger.info( - "Scoring Function: {} ({})".format( - self.score_function_desc[score_function], score_function - ) + f"Scoring Function: {self.score_function_desc[score_function]} ({score_function})" ) itr = range(0, len(corpus), self.corpus_chunk_size) @@ -138,7 +136,7 @@ def search( qid: [] for qid in query_ids } # Keep only the top-k docs for each query for batch_num, corpus_start_idx in enumerate(itr): - logger.info("Encoding Batch {}/{}...".format(batch_num + 1, len(itr))) + logger.info(f"Encoding Batch {batch_num + 1}/{len(itr)}...") corpus_end_idx = min(corpus_start_idx + self.corpus_chunk_size, len(corpus)) # Encode chunk of corpus @@ -218,7 +216,7 @@ def load_results_file(self): ) self.previous_results = dest_file - with open(self.previous_results, "r") as f: + with open(self.previous_results) as f: previous_results = json.load(f) assert isinstance(previous_results, dict) assert isinstance(previous_results[list(previous_results.keys())[0]], dict) @@ -226,12 +224,12 @@ def load_results_file(self): def search_cross_encoder( self, - corpus: Dict[str, Dict[str, str]], - queries: Dict[str, Union[str, List[str]]], + corpus: dict[str, dict[str, str]], + queries: dict[str, str | list[str]], top_k: int, - instructions: Union[Dict[str, str], None] = None, + instructions: dict[str, str] | None = None, **kwargs, - ) -> Dict[str, Dict[str, float]]: + ) -> dict[str, dict[str, float]]: """This function provides support for reranker (or cross-encoder) models that encoder query and document at the same time (typically with attention). Some notable examples include MonoBERT, MonoT5, RankLlama, etc. Note: you must provide the path to the results to rerank to the __init__ function as `previous_results` @@ -240,12 +238,9 @@ def search_cross_encoder( for qid in queries.keys(): q_results = self.previous_results[qid] # take the top-k only - q_results_sorted = { - k: v - for k, v in sorted( + q_results_sorted = dict(sorted( q_results.items(), key=lambda item: item[1], reverse=True - ) - } + )) top_n = [k for k, v in list(q_results_sorted.items())[:top_k]] query = queries[qid] query = ( @@ -347,7 +342,7 @@ def __init__(self, model, **kwargs): self.corpus_embeddings = {} def encode_queries( - self, queries: List[str], *, prompt_name: str, batch_size: int, **kwargs + self, queries: list[str], *, prompt_name: str, batch_size: int, **kwargs ): if self.use_sbert_model: if isinstance(self.model._first_module(), Transformer): @@ -369,10 +364,10 @@ def encode_queries( def encode_corpus( self, - corpus: List[Dict[str, str]], + corpus: list[dict[str, str]], prompt_name: str, batch_size: int, - request_qid: Union[str, None] = None, + request_qid: str | None = None, **kwargs, ): if ( @@ -409,7 +404,7 @@ def encode_corpus( self.corpus_embeddings[request_qid] = corpus_embeddings return corpus_embeddings - def encode(self, sentences: List[str], prompt_name: str, **kwargs): + def encode(self, sentences: list[str], prompt_name: str, **kwargs): return self.encode_queries(sentences, prompt_name=prompt_name, **kwargs) @@ -432,7 +427,7 @@ def __init__( self, retriever=None, task_name: str | None = None, - k_values: List[int] = [1, 3, 5, 10, 20, 100, 1000], + k_values: list[int] = [1, 3, 5, 10, 20, 100, 1000], score_function: str = "cos_sim", encode_kwargs: dict[str, Any] = {}, **kwargs, @@ -471,7 +466,7 @@ def __init__( def __call__( self, corpus: dict[str, dict[str, str]], - queries: dict[str, Union[str, List[str]]], + queries: dict[str, str | list[str]], ) -> dict[str, dict[str, float]]: if not self.retriever: raise ValueError("Model/Technique has not been provided!") @@ -502,9 +497,9 @@ def __call__( def evaluate( qrels: dict[str, dict[str, int]], results: dict[str, dict[str, float]], - k_values: List[int], + k_values: list[int], ignore_identical_ids: bool = False, - ) -> Tuple[ + ) -> tuple[ dict[str, float], dict[str, float], dict[str, float], @@ -572,10 +567,10 @@ def evaluate( def evaluate_custom( qrels: dict[str, dict[str, int]], results: dict[str, dict[str, float]], - k_values: List[int], + k_values: list[int], metric: str, output_type: str = "all", - ) -> Tuple[Dict[str, float], Dict[str, float]]: + ) -> tuple[dict[str, float], dict[str, float]]: if metric.lower() in ["mrr", "mrr@k", "mrr_cut"]: metric_scores = mrr(qrels, results, k_values, output_type) @@ -603,7 +598,7 @@ def evaluate_custom( def evaluate_abstention( results: dict[str, dict[str, float]], metric_scores: dict[str, list[float]], - ) -> Dict[str, float]: + ) -> dict[str, float]: """Computes normalized Area Under the Curve on a set of evaluated instances as presented in the paper https://arxiv.org/abs/2402.12997""" all_sim_scores = [list(results[qid].values()) for qid in list(results.keys())] all_conf_scores = [ diff --git a/mteb/evaluation/evaluators/utils.py b/mteb/evaluation/evaluators/utils.py index 9823e9d603..0cdfb6bd72 100644 --- a/mteb/evaluation/evaluators/utils.py +++ b/mteb/evaluation/evaluators/utils.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from typing import Dict, List, Tuple, Union import numpy as np import pandas as pd @@ -57,9 +56,9 @@ def dot_score(a: torch.Tensor, b: torch.Tensor): def mrr( qrels: dict[str, dict[str, int]], results: dict[str, dict[str, float]], - k_values: List[int], + k_values: list[int], output_type: str = "mean", -) -> Tuple[Dict[str, float]]: +) -> tuple[dict[str, float]]: MRR = {} for k in k_values: @@ -74,9 +73,9 @@ def mrr( )[0:k_max] for query_id in top_hits: - query_relevant_docs = set( - [doc_id for doc_id in qrels[query_id] if qrels[query_id][doc_id] > 0] - ) + query_relevant_docs = { + doc_id for doc_id in qrels[query_id] if qrels[query_id][doc_id] > 0 + } for k in k_values: rr = 0 for rank, hit in enumerate(top_hits[query_id][0:k]): @@ -99,9 +98,9 @@ def mrr( def recall_cap( qrels: dict[str, dict[str, int]], results: dict[str, dict[str, float]], - k_values: List[int], + k_values: list[int], output_type: str = "mean", -) -> Tuple[Dict[str, float]]: +) -> tuple[dict[str, float]]: capped_recall = {} for k in k_values: @@ -140,9 +139,9 @@ def recall_cap( def hole( qrels: dict[str, dict[str, int]], results: dict[str, dict[str, float]], - k_values: List[int], + k_values: list[int], output_type: str = "mean", -) -> Tuple[Dict[str, float]]: +) -> tuple[dict[str, float]]: Hole = {} for k in k_values: @@ -180,9 +179,9 @@ def hole( def top_k_accuracy( qrels: dict[str, dict[str, int]], results: dict[str, dict[str, float]], - k_values: List[int], + k_values: list[int], output_type: str = "mean", -) -> Dict[str, float]: +) -> dict[str, float]: top_k_acc = {} for k in k_values: @@ -200,9 +199,9 @@ def top_k_accuracy( ] for query_id in top_hits: - query_relevant_docs = set( - [doc_id for doc_id in qrels[query_id] if qrels[query_id][doc_id] > 0] - ) + query_relevant_docs = { + doc_id for doc_id in qrels[query_id] if qrels[query_id][doc_id] > 0 + } for k in k_values: for relevant_doc_id in query_relevant_docs: if relevant_doc_id in top_hits[query_id][0:k]: @@ -224,7 +223,7 @@ def top_k_accuracy( def get_rank_from_dict( dict_of_results: dict[str, float], doc_id: str -) -> Tuple[int, float]: +) -> tuple[int, float]: tuple_of_id_score = dict_of_results.items() sorted_by_score = sorted(tuple_of_id_score, key=lambda x: x[1], reverse=True) for i, (id, score) in enumerate(sorted_by_score): @@ -237,7 +236,7 @@ def get_rank_from_dict( def evaluate_change( original_run: dict[str, dict[str, float]], new_run: dict[str, dict[str, float]], - changed_qrels: dict[str, List[str]], + changed_qrels: dict[str, list[str]], ) -> dict[str, float]: changes = [] for qid in changed_qrels.keys(): @@ -294,7 +293,7 @@ def download(url: str, fname: str): bar.update(size) -def convert_conv_history_to_query(conversations: List[List[Union[str, dict]]]) -> str: +def convert_conv_history_to_query(conversations: list[list[str | dict]]) -> str: conversations_converted = [] for conversation in conversations: @@ -307,7 +306,7 @@ def convert_conv_history_to_query(conversations: List[List[Union[str, dict]]]) - for i, turn in enumerate(conversation): error_msg = ( "When converting conversations lists of dictionary to string, each turn in the conversation " - "must be a dictionary with 'role' and 'content' keys" + + "must be a dictionary with 'role' and 'content' keys" ) if not isinstance(turn, dict): raise ValueError(f"Turn {i} is not a dictionary. " + error_msg) @@ -334,7 +333,7 @@ def convert_conv_history_to_query(conversations: List[List[Union[str, dict]]]) - return conversations_converted -def confidence_scores(sim_scores: List[float]) -> Dict[str, float]: +def confidence_scores(sim_scores: list[float]) -> dict[str, float]: """Computes confidence scores for a single instance = (query, positives, negatives) Args: diff --git a/mteb/load_results/__init__.py b/mteb/load_results/__init__.py index 63e9595a32..3b08f6eb4d 100644 --- a/mteb/load_results/__init__.py +++ b/mteb/load_results/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from .load_results import load_results from .mteb_results import MTEBResults diff --git a/mteb/load_results/mteb_results.py b/mteb/load_results/mteb_results.py index f6c2d254e6..5c091f060d 100644 --- a/mteb/load_results/mteb_results.py +++ b/mteb/load_results/mteb_results.py @@ -6,7 +6,7 @@ from collections import defaultdict from importlib.metadata import version from pathlib import Path -from typing import Any, Callable, Type +from typing import Any, Callable import numpy as np from packaging.version import Version @@ -166,7 +166,7 @@ class MTEBResults(BaseModel): @classmethod def from_task_results( cls, - task: AbsTask | Type[AbsTask], + task: AbsTask | type[AbsTask], scores: dict[Split, dict[HFSubset, ScoresDict]], evaluation_time: float, kg_co2_emissions: float | None = None, @@ -449,7 +449,7 @@ def validate_and_filter_scores(self): task = get_task(self.task_name) splits = task.metadata.eval_splits - hf_subsets = set([s for s in task.metadata.hf_subsets_to_langscripts]) + hf_subsets = set(task.metadata.hf_subsets_to_langscripts) new_scores = {} seen_splits = set() diff --git a/mteb/models/bm25.py b/mteb/models/bm25.py index 9a1ed9ccf7..4ef5a55ca6 100644 --- a/mteb/models/bm25.py +++ b/mteb/models/bm25.py @@ -2,7 +2,7 @@ import logging from functools import partial -from typing import Any, List, Optional, Union +from typing import Any from mteb.evaluation.evaluators.RetrievalEvaluator import DRESModel from mteb.model_meta import ModelMeta @@ -27,7 +27,7 @@ def __init__( self, previous_results: str = None, stopwords: str = "en", - stemmer_language: Optional[str] = "english", + stemmer_language: str | None = "english", **kwargs, ): super().__init__( @@ -50,7 +50,7 @@ def name(self): def search( self, corpus: dict[str, dict[str, str]], - queries: dict[str, Union[str, List[str]]], + queries: dict[str, str | list[str]], top_k: int, score_function: str, return_sorted: bool = False, @@ -106,7 +106,7 @@ def search( return self.results - def encode(self, texts: List[str], **kwargs): + def encode(self, texts: list[str], **kwargs): """Encode input text as term vectors""" return bm25s.tokenize(texts, stopwords=self.stopwords, stemmer=self.stemmer) diff --git a/mteb/models/e5_instruct.py b/mteb/models/e5_instruct.py index 49650641fa..d21017c920 100644 --- a/mteb/models/e5_instruct.py +++ b/mteb/models/e5_instruct.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from functools import partial import torch diff --git a/mteb/models/google_models.py b/mteb/models/google_models.py index fdb9a2a36c..5a72b97e81 100644 --- a/mteb/models/google_models.py +++ b/mteb/models/google_models.py @@ -3,7 +3,7 @@ from __future__ import annotations from functools import partial -from typing import Any, List, Optional +from typing import Any import numpy as np @@ -17,11 +17,11 @@ def __init__(self, model_name: str, sep: str = " ", **kwargs) -> None: def _embed( self, - texts: List[str], + texts: list[str], task_type: str = "RETRIEVAL_DOCUMENT", - titles: List[str] | None = None, - dimensionality: Optional[int] = 768, - ) -> List[List[float]]: + titles: list[str] | None = None, + dimensionality: int | None = 768, + ) -> list[list[float]]: """Embeds texts with a pre-trained, foundational model. From https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings#generative-ai-get-text-embedding-python_vertex_ai_sdk """ @@ -43,7 +43,7 @@ def _embed( ] else: inputs = [TextEmbeddingInput(text, task_type=task_type) for text in texts] - kwargs = dict(output_dimensionality=dimensionality) if dimensionality else {} + kwargs = {"output_dimensionality": dimensionality} if dimensionality else {} try: embeddings = model.get_embeddings(inputs, **kwargs) # Except the very rare google.api_core.exceptions.InternalServerError diff --git a/mteb/models/gritlm_models.py b/mteb/models/gritlm_models.py index 7c529f35f9..2ef3a079b0 100644 --- a/mteb/models/gritlm_models.py +++ b/mteb/models/gritlm_models.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging from functools import partial diff --git a/mteb/models/instructions.py b/mteb/models/instructions.py index ec05b34741..5ddb2394d6 100644 --- a/mteb/models/instructions.py +++ b/mteb/models/instructions.py @@ -1,4 +1,5 @@ """This specifies the default instructions for tasks within MTEB. These are optional to use and some models might want to use their own instructions.""" +from __future__ import annotations import mteb diff --git a/mteb/models/llm2vec_models.py b/mteb/models/llm2vec_models.py index 91344e101b..66bd4f447a 100644 --- a/mteb/models/llm2vec_models.py +++ b/mteb/models/llm2vec_models.py @@ -1,5 +1,7 @@ +from __future__ import annotations + import logging -from typing import Any, Callable, Dict, List, Literal, Type, Union +from typing import Any, Callable, Literal import numpy as np import torch @@ -53,7 +55,7 @@ def __init__(self, *args, **kwargs): def encode( self, - sentences: List[str], + sentences: list[str], *, prompt_name: str = None, **kwargs: Any, # noqa @@ -73,7 +75,7 @@ def encode( def encode_corpus( self, - corpus: Union[List[Dict[str, str]], Dict[str, List[str]], List[str]], + corpus: list[dict[str, str]] | dict[str, list[str]] | list[str], prompt_name: str = None, **kwargs: Any, ) -> np.ndarray: @@ -81,11 +83,11 @@ def encode_corpus( sentences = [["", sentence] for sentence in sentences] return self.model.encode(sentences, **kwargs) - def encode_queries(self, queries: List[str], **kwargs: Any) -> np.ndarray: + def encode_queries(self, queries: list[str], **kwargs: Any) -> np.ndarray: return self.encode(queries, **kwargs) -def _loader(wrapper: Type[LLM2VecWrapper], **kwargs) -> Callable[..., Encoder]: +def _loader(wrapper: type[LLM2VecWrapper], **kwargs) -> Callable[..., Encoder]: _kwargs = kwargs def loader_inner(**kwargs: Any) -> Encoder: diff --git a/mteb/models/nomic_models.py b/mteb/models/nomic_models.py index a013d1e621..d05d65c16d 100644 --- a/mteb/models/nomic_models.py +++ b/mteb/models/nomic_models.py @@ -1,7 +1,7 @@ from __future__ import annotations from functools import partial -from typing import Any, Optional +from typing import Any import torch import torch.nn.functional as F @@ -28,7 +28,7 @@ def encode( # type: ignore *, prompt_name: str | None = None, batch_size: int = 32, - input_type: Optional[str] = None, + input_type: str | None = None, **kwargs: Any, ): if prompt_name: diff --git a/mteb/models/ru_sentence_models.py b/mteb/models/ru_sentence_models.py index 18fb5fd524..4cb26dd2f0 100644 --- a/mteb/models/ru_sentence_models.py +++ b/mteb/models/ru_sentence_models.py @@ -1,4 +1,5 @@ """Sentence models for evaluation on the Russian part of MTEB""" +from __future__ import annotations from functools import partial diff --git a/mteb/models/salesforce_models.py b/mteb/models/salesforce_models.py index 9d0e9354b0..1ce5700133 100644 --- a/mteb/models/salesforce_models.py +++ b/mteb/models/salesforce_models.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from functools import partial import torch diff --git a/mteb/models/sentence_transformers_models.py b/mteb/models/sentence_transformers_models.py index d1fb638be2..98a25e46ac 100644 --- a/mteb/models/sentence_transformers_models.py +++ b/mteb/models/sentence_transformers_models.py @@ -1,4 +1,5 @@ """Implementation of Sentence Transformers model validated in MTEB.""" +from __future__ import annotations from mteb.model_meta import ModelMeta diff --git a/mteb/overview.py b/mteb/overview.py index 184706219d..0fae9df4f4 100644 --- a/mteb/overview.py +++ b/mteb/overview.py @@ -5,7 +5,6 @@ import difflib import logging from collections import Counter -from typing import Dict, Set, Type from mteb.abstasks import AbsTask from mteb.abstasks.TaskMetadata import TASK_CATEGORY, TASK_DOMAIN, TASK_TYPE @@ -23,8 +22,8 @@ # Create task registry -def create_task_list() -> list[Type[AbsTask]]: - tasks_categories_cls = [cls for cls in AbsTask.__subclasses__()] +def create_task_list() -> list[type[AbsTask]]: + tasks_categories_cls = list(AbsTask.__subclasses__()) tasks = [ cls for cat_cls in tasks_categories_cls @@ -34,7 +33,7 @@ def create_task_list() -> list[Type[AbsTask]]: return tasks -def create_name_to_task_mapping() -> dict[str, Type[AbsTask]]: +def create_name_to_task_mapping() -> dict[str, type[AbsTask]]: tasks = create_task_list() return {cls.metadata.name: cls for cls in tasks} @@ -117,7 +116,7 @@ def _extract_property_from_task(task, property): raise KeyError("Property neither in Task attribute or metadata keys.") @property - def languages(self) -> Set: + def languages(self) -> set: """Return all languages from tasks""" langs = set() for task in self: @@ -125,7 +124,7 @@ def languages(self) -> Set: langs.add(lg) return langs - def count_languages(self) -> Dict: + def count_languages(self) -> dict: """Summarize count of all languages from tasks""" langs = [] for task in self: diff --git a/mteb/requires_package.py b/mteb/requires_package.py index 6256174fbd..a91c2ba093 100644 --- a/mteb/requires_package.py +++ b/mteb/requires_package.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import importlib.util @@ -11,5 +13,5 @@ def requires_package(obj, package_name: str, model_name: str) -> None: name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__ raise ImportError( f"{name} requires the `{package_name}` library but it was not found in your environment. " - f"If you want to load {model_name} models, please `pip install {package_name}` else they will not be available." + + f"If you want to load {model_name} models, please `pip install {package_name}` else they will not be available." ) diff --git a/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py b/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py index e48346ebb2..f85c6ff105 100644 --- a/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py @@ -923,7 +923,7 @@ def load_data(self, **kwargs: Any) -> None: def dataset_transform(self) -> None: # Convert to standard format for lang in self.hf_subsets: - l1, l2 = [l.split("_")[0] for l in lang.split("-")] + l1, l2 = (l.split("_")[0] for l in lang.split("-")) self.dataset[lang] = self.dataset[lang].rename_columns( {l1: "sentence1", l2: "sentence2"} diff --git a/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py b/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py index e5df76e4c5..06a68b08d1 100644 --- a/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py +++ b/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata diff --git a/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py b/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py index 513d7980c8..1f6453c844 100644 --- a/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py +++ b/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from mteb.abstasks import AbsTaskClassification # type: ignore from mteb.abstasks.TaskMetadata import TaskMetadata # type: ignore diff --git a/mteb/tasks/Classification/kor/KorFin.py b/mteb/tasks/Classification/kor/KorFin.py index 7907b80fcd..5da2539e19 100644 --- a/mteb/tasks/Classification/kor/KorFin.py +++ b/mteb/tasks/Classification/kor/KorFin.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from mteb.abstasks import AbsTaskClassification # type: ignore from mteb.abstasks.TaskMetadata import TaskMetadata # type: ignore diff --git a/mteb/tasks/Classification/multilingual/IndicLangClassification.py b/mteb/tasks/Classification/multilingual/IndicLangClassification.py index 2ee372e07b..e94131d6f0 100644 --- a/mteb/tasks/Classification/multilingual/IndicLangClassification.py +++ b/mteb/tasks/Classification/multilingual/IndicLangClassification.py @@ -113,7 +113,7 @@ def load_data(self, **kwargs: Any) -> None: if self.data_loaded: return - labels = sorted(list(_LANGUAGES.keys())) + labels = sorted(_LANGUAGES.keys()) data = datasets.load_dataset(**self.metadata_dict["dataset"])["train"]["data"][ 0 diff --git a/mteb/tasks/Classification/multilingual/NaijaSenti.py b/mteb/tasks/Classification/multilingual/NaijaSenti.py index 89bce608b6..c861afb52b 100644 --- a/mteb/tasks/Classification/multilingual/NaijaSenti.py +++ b/mteb/tasks/Classification/multilingual/NaijaSenti.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Any import datasets diff --git a/mteb/tasks/Classification/mya/MyanmarNews.py b/mteb/tasks/Classification/mya/MyanmarNews.py index 2bc7aa21ff..5337a43a03 100644 --- a/mteb/tasks/Classification/mya/MyanmarNews.py +++ b/mteb/tasks/Classification/mya/MyanmarNews.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from mteb.abstasks import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata diff --git a/mteb/tasks/Classification/pol/PolishClassification.py b/mteb/tasks/Classification/pol/PolishClassification.py index fe95d6a504..79b84b1c82 100644 --- a/mteb/tasks/Classification/pol/PolishClassification.py +++ b/mteb/tasks/Classification/pol/PolishClassification.py @@ -47,7 +47,7 @@ class PolEmo2InClassification(AbsTaskClassification): metadata = TaskMetadata( name="PolEmo2.0-IN", description="A collection of Polish online reviews from four domains: medicine, hotels, products and " - "school. The PolEmo2.0-IN task is to predict the sentiment of in-domain (medicine and hotels) reviews.", + + "school. The PolEmo2.0-IN task is to predict the sentiment of in-domain (medicine and hotels) reviews.", reference="https://aclanthology.org/K19-1092.pdf", dataset={ "path": "PL-MTEB/polemo2_in", @@ -89,8 +89,8 @@ class PolEmo2OutClassification(AbsTaskClassification): metadata = TaskMetadata( name="PolEmo2.0-OUT", description="A collection of Polish online reviews from four domains: medicine, hotels, products and " - "school. The PolEmo2.0-OUT task is to predict the sentiment of out-of-domain (products and " - "school) reviews using models train on reviews from medicine and hotels domains.", + + "school. The PolEmo2.0-OUT task is to predict the sentiment of out-of-domain (products and " + + "school) reviews using models train on reviews from medicine and hotels domains.", reference="https://aclanthology.org/K19-1092.pdf", dataset={ "path": "PL-MTEB/polemo2_out", diff --git a/mteb/tasks/Classification/ron/Moroco.py b/mteb/tasks/Classification/ron/Moroco.py index e7d4d04bf0..5e06017a14 100644 --- a/mteb/tasks/Classification/ron/Moroco.py +++ b/mteb/tasks/Classification/ron/Moroco.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from mteb.abstasks import AbsTaskClassification # type: ignore from mteb.abstasks.TaskMetadata import TaskMetadata # type: ignore diff --git a/mteb/tasks/Classification/tha/WongnaiReviewsClassification .py b/mteb/tasks/Classification/tha/WongnaiReviewsClassification .py index daeadba09b..bc164bcd85 100644 --- a/mteb/tasks/Classification/tha/WongnaiReviewsClassification .py +++ b/mteb/tasks/Classification/tha/WongnaiReviewsClassification .py @@ -1,3 +1,5 @@ +from __future__ import annotations + from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata diff --git a/mteb/tasks/Clustering/deu/BlurbsClusteringS2S.py b/mteb/tasks/Clustering/deu/BlurbsClusteringS2S.py index 895c6c46d3..7b9dc43e5d 100644 --- a/mteb/tasks/Clustering/deu/BlurbsClusteringS2S.py +++ b/mteb/tasks/Clustering/deu/BlurbsClusteringS2S.py @@ -99,7 +99,7 @@ class BlurbsClusteringS2SFast(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( diff --git a/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py b/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py index 0aa183f068..7a70251c20 100644 --- a/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py +++ b/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py @@ -45,7 +45,7 @@ class ArXivHierarchicalClusteringP2P(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( @@ -89,7 +89,7 @@ class ArXivHierarchicalClusteringS2S(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( diff --git a/mteb/tasks/Clustering/eng/BigPatentClustering.py b/mteb/tasks/Clustering/eng/BigPatentClustering.py index beed4dd594..756ec4db32 100644 --- a/mteb/tasks/Clustering/eng/BigPatentClustering.py +++ b/mteb/tasks/Clustering/eng/BigPatentClustering.py @@ -16,7 +16,7 @@ class BigPatentClustering(AbsTaskClustering): metadata = TaskMetadata( name="BigPatentClustering", description="Clustering of documents from the Big Patent dataset. Test set only includes documents" - "belonging to a single category, with a total of 9 categories.", + + "belonging to a single category, with a total of 9 categories.", reference="https://www.kaggle.com/datasets/big_patent", dataset={ "path": "jinaai/big-patent-clustering", @@ -61,7 +61,7 @@ class BigPatentClusteringFast(AbsTaskClusteringFast): metadata = TaskMetadata( name="BigPatentClustering.v2", description="Clustering of documents from the Big Patent dataset. Test set only includes documents" - "belonging to a single category, with a total of 9 categories.", + + "belonging to a single category, with a total of 9 categories.", reference="https://huggingface.co/datasets/NortheasternUniversity/big_patent", dataset={ "path": "mteb/big-patent", diff --git a/mteb/tasks/Clustering/eng/MedrxivClusteringP2P.py b/mteb/tasks/Clustering/eng/MedrxivClusteringP2P.py index 5f8d8c49fb..ac338f152f 100644 --- a/mteb/tasks/Clustering/eng/MedrxivClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/MedrxivClusteringP2P.py @@ -42,7 +42,7 @@ class MedrxivClusteringP2PFast(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( diff --git a/mteb/tasks/Clustering/eng/MedrxivClusteringS2S.py b/mteb/tasks/Clustering/eng/MedrxivClusteringS2S.py index 9bb34a03b0..e0062db193 100644 --- a/mteb/tasks/Clustering/eng/MedrxivClusteringS2S.py +++ b/mteb/tasks/Clustering/eng/MedrxivClusteringS2S.py @@ -42,7 +42,7 @@ class MedrxivClusteringS2SFast(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( diff --git a/mteb/tasks/Clustering/eng/RedditClustering.py b/mteb/tasks/Clustering/eng/RedditClustering.py index d102cfddba..7c2d44b9f5 100644 --- a/mteb/tasks/Clustering/eng/RedditClustering.py +++ b/mteb/tasks/Clustering/eng/RedditClustering.py @@ -54,7 +54,7 @@ class RedditFastClusteringS2S(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( diff --git a/mteb/tasks/Clustering/eng/RedditClusteringP2P.py b/mteb/tasks/Clustering/eng/RedditClusteringP2P.py index c66424c595..12148f91ed 100644 --- a/mteb/tasks/Clustering/eng/RedditClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/RedditClusteringP2P.py @@ -99,7 +99,7 @@ class RedditFastClusteringP2P(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( diff --git a/mteb/tasks/Clustering/eng/StackExchangeClustering.py b/mteb/tasks/Clustering/eng/StackExchangeClustering.py index 9b437bffc3..8d1118c357 100644 --- a/mteb/tasks/Clustering/eng/StackExchangeClustering.py +++ b/mteb/tasks/Clustering/eng/StackExchangeClustering.py @@ -54,7 +54,7 @@ class StackExchangeClusteringFast(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( diff --git a/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py b/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py index 2c1a299833..4131ed30e8 100644 --- a/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py @@ -56,7 +56,7 @@ class StackExchangeClusteringP2PFast(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( diff --git a/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py b/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py index 8f38e167cc..df7b68f5ba 100644 --- a/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py +++ b/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py @@ -99,7 +99,7 @@ class TwentyNewsgroupsClusteringFast(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( diff --git a/mteb/tasks/Clustering/fra/AlloProfClusteringP2P.py b/mteb/tasks/Clustering/fra/AlloProfClusteringP2P.py index 0983564b05..a900043551 100644 --- a/mteb/tasks/Clustering/fra/AlloProfClusteringP2P.py +++ b/mteb/tasks/Clustering/fra/AlloProfClusteringP2P.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets import numpy as np diff --git a/mteb/tasks/Clustering/fra/AlloProfClusteringS2S.py b/mteb/tasks/Clustering/fra/AlloProfClusteringS2S.py index 25a984762e..c46e239689 100644 --- a/mteb/tasks/Clustering/fra/AlloProfClusteringS2S.py +++ b/mteb/tasks/Clustering/fra/AlloProfClusteringS2S.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets import numpy as np diff --git a/mteb/tasks/Clustering/fra/HALClusteringS2S.py b/mteb/tasks/Clustering/fra/HALClusteringS2S.py index fd43a35ff5..b967e23fa1 100644 --- a/mteb/tasks/Clustering/fra/HALClusteringS2S.py +++ b/mteb/tasks/Clustering/fra/HALClusteringS2S.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from collections import Counter import datasets @@ -109,18 +111,18 @@ def dataset_transform(self): labels_count = Counter(self.dataset["test"]["labels"]) # keep classes with more than 2 samples after stratified_subsampling - frequent_labels = set( + frequent_labels = { label for label, count in labels_count.items() if count > len(self.dataset["test"]) * 2 / NUM_SAMPLES - ) + } self.dataset["test"] = self.dataset["test"].filter( lambda row: row["labels"] in frequent_labels ) self.dataset["test"] = self.dataset["test"].cast( datasets.Features( sentences=datasets.Value("string"), - labels=datasets.ClassLabel(names=sorted(list(frequent_labels))), + labels=datasets.ClassLabel(names=sorted(frequent_labels)), ) ) for split in self.metadata.eval_splits: diff --git a/mteb/tasks/Clustering/jpn/LivedoorNewsClustering.py b/mteb/tasks/Clustering/jpn/LivedoorNewsClustering.py index 4917abb6a8..e9ca78b325 100644 --- a/mteb/tasks/Clustering/jpn/LivedoorNewsClustering.py +++ b/mteb/tasks/Clustering/jpn/LivedoorNewsClustering.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from mteb.abstasks.AbsTaskClusteringFast import AbsTaskClusteringFast from mteb.abstasks.TaskMetadata import TaskMetadata diff --git a/mteb/tasks/Clustering/jpn/MewsC16JaClustering.py b/mteb/tasks/Clustering/jpn/MewsC16JaClustering.py index 99d4585e3c..6dd93313d2 100644 --- a/mteb/tasks/Clustering/jpn/MewsC16JaClustering.py +++ b/mteb/tasks/Clustering/jpn/MewsC16JaClustering.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from mteb.abstasks.AbsTaskClusteringFast import AbsTaskClusteringFast from mteb.abstasks.TaskMetadata import TaskMetadata diff --git a/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py b/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py index aebcbe8407..28b85b88ce 100644 --- a/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py @@ -157,7 +157,7 @@ def dataset_transform(self, lang): ["summary", "url", "date", "title"] ).rename_columns({"topic": "labels", "text": "sentences"}) - lang_dict = dict() + lang_dict = {} for split in self.metadata.eval_splits: labels = _dataset[split]["labels"] sentences = _dataset[split]["sentences"] diff --git a/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py b/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py index 6576583426..326170ba4f 100644 --- a/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py +++ b/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py @@ -147,7 +147,7 @@ def dataset_transform(self, lang): ["summary", "url", "date", "title"] ).rename_columns({"topic": "labels", "text": "sentences"}) - lang_dict = dict() + lang_dict = {} for split in self.metadata.eval_splits: labels = _dataset[split]["labels"] sentences = _dataset[split]["sentences"] diff --git a/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py b/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py index ee6744c594..c280a05dac 100644 --- a/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py +++ b/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py @@ -250,11 +250,11 @@ class SIB200ClusteringFast(MultilingualTask, AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for lang in self.hf_subsets: labels = [] sentences = [] - ds[lang] = dict() + ds[lang] = {} for split in ["train", "validation", "test"]: labels.extend(self.dataset[lang][split]["category"]) sentences.extend(self.dataset[lang][split]["text"]) diff --git a/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py b/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py index adfc6d611f..7629af5e1d 100644 --- a/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py @@ -91,12 +91,12 @@ class WikiClusteringFastP2P(AbsTaskClusteringFast, MultilingualTask): ) def dataset_transform(self): - ds = dict() + ds = {} for lang in self.hf_subsets: labels = [] sentences = [] - ds[lang] = dict() - lang_dict = dict() + ds[lang] = {} + lang_dict = {} for split in self.metadata.eval_splits: labels.extend( itertools.chain.from_iterable(self.dataset[lang][split]["labels"]) diff --git a/mteb/tasks/Clustering/nob/snl_clustering.py b/mteb/tasks/Clustering/nob/snl_clustering.py index ba4e401432..c3543e1376 100644 --- a/mteb/tasks/Clustering/nob/snl_clustering.py +++ b/mteb/tasks/Clustering/nob/snl_clustering.py @@ -78,7 +78,7 @@ def dataset_transform(self): rng = random.Random(42) # local only seed pairs = list(zip(documents, labels)) rng.shuffle(pairs) - documents, labels = [list(collection) for collection in zip(*pairs)] + documents, labels = (list(collection) for collection in zip(*pairs)) # reduce size of dataset to not have too large datasets in the clustering task documents_batched = list(batched(documents, 512))[:4] diff --git a/mteb/tasks/Clustering/nob/vg_clustering.py b/mteb/tasks/Clustering/nob/vg_clustering.py index 899fcb717a..08ad9ceef8 100644 --- a/mteb/tasks/Clustering/nob/vg_clustering.py +++ b/mteb/tasks/Clustering/nob/vg_clustering.py @@ -82,7 +82,7 @@ def dataset_transform(self): # resampling changes scores from 12.68, 11.30, 12.65 (sample model) pairs = list(zip(documents, labels)) rng.shuffle(pairs) - documents, labels = [list(collection) for collection in zip(*pairs)] + documents, labels = (list(collection) for collection in zip(*pairs)) # reduce size of dataset to not have too large datasets in the clustering task documents_batched = list(batched(documents, 512))[:4] diff --git a/mteb/tasks/Clustering/pol/PolishClustering.py b/mteb/tasks/Clustering/pol/PolishClustering.py index ce43ac00cb..7c11cb6148 100644 --- a/mteb/tasks/Clustering/pol/PolishClustering.py +++ b/mteb/tasks/Clustering/pol/PolishClustering.py @@ -20,7 +20,7 @@ class EightTagsClustering(AbsTaskClustering): metadata = TaskMetadata( name="EightTagsClustering", description="Clustering of headlines from social media posts in Polish belonging to 8 categories: film, history, " - "food, medicine, motorization, work, sport and technology.", + + "food, medicine, motorization, work, sport and technology.", reference="https://aclanthology.org/2020.lrec-1.207.pdf", dataset={ "path": "PL-MTEB/8tags-clustering", @@ -83,7 +83,7 @@ class EightTagsClusteringFast(AbsTaskClusteringFast): metadata = TaskMetadata( name="EightTagsClustering.v2", description="Clustering of headlines from social media posts in Polish belonging to 8 categories: film, history, " - "food, medicine, motorization, work, sport and technology.", + + "food, medicine, motorization, work, sport and technology.", reference="https://aclanthology.org/2020.lrec-1.207.pdf", dataset={ "path": "PL-MTEB/8tags-clustering", @@ -139,7 +139,7 @@ class EightTagsClusteringFast(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(chain.from_iterable(self.dataset[split]["labels"])) sentences = list(chain.from_iterable(self.dataset[split]["sentences"])) @@ -161,7 +161,7 @@ class PlscClusteringS2S(AbsTaskClusteringFast): metadata = TaskMetadata( name="PlscClusteringS2S", description="Clustering of Polish article titles from Library of Science (https://bibliotekanauki.pl/), either " - "on the scientific field or discipline.", + + "on the scientific field or discipline.", reference="https://huggingface.co/datasets/rafalposwiata/plsc", dataset={ "path": "PL-MTEB/plsc-clustering-s2s", @@ -192,7 +192,7 @@ class PlscClusteringS2SFast(AbsTaskClusteringFast): metadata = TaskMetadata( name="PlscClusteringS2S.v2", description="Clustering of Polish article titles from Library of Science (https://bibliotekanauki.pl/), either " - "on the scientific field or discipline.", + + "on the scientific field or discipline.", reference="https://huggingface.co/datasets/rafalposwiata/plsc", dataset={ "path": "PL-MTEB/plsc-clustering-s2s", @@ -219,7 +219,7 @@ class PlscClusteringS2SFast(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = self.dataset[split]["labels"] sentences = self.dataset[split]["sentences"] @@ -250,7 +250,7 @@ class PlscClusteringP2P(AbsTaskClusteringFast): metadata = TaskMetadata( name="PlscClusteringP2P", description="Clustering of Polish article titles+abstracts from Library of Science " - "(https://bibliotekanauki.pl/), either on the scientific field or discipline.", + + "(https://bibliotekanauki.pl/), either on the scientific field or discipline.", reference="https://huggingface.co/datasets/rafalposwiata/plsc", dataset={ "path": "PL-MTEB/plsc-clustering-p2p", @@ -281,7 +281,7 @@ class PlscClusteringP2PFast(AbsTaskClusteringFast): metadata = TaskMetadata( name="PlscClusteringP2P.v2", description="Clustering of Polish article titles+abstracts from Library of Science " - "(https://bibliotekanauki.pl/), either on the scientific field or discipline.", + + "(https://bibliotekanauki.pl/), either on the scientific field or discipline.", reference="https://huggingface.co/datasets/rafalposwiata/plsc", dataset={ "path": "PL-MTEB/plsc-clustering-p2p", @@ -308,7 +308,7 @@ class PlscClusteringP2PFast(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = self.dataset[split]["labels"] sentences = self.dataset[split]["sentences"] diff --git a/mteb/tasks/Clustering/zho/CMTEBClustering.py b/mteb/tasks/Clustering/zho/CMTEBClustering.py index 4cc376d45e..28360bf3b3 100644 --- a/mteb/tasks/Clustering/zho/CMTEBClustering.py +++ b/mteb/tasks/Clustering/zho/CMTEBClustering.py @@ -54,7 +54,7 @@ class CLSClusteringFastS2S(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( @@ -114,7 +114,7 @@ class CLSClusteringFastP2P(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( @@ -244,7 +244,7 @@ class ThuNewsClusteringFastS2S(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( @@ -304,7 +304,7 @@ class ThuNewsClusteringFastP2P(AbsTaskClusteringFast): ) def dataset_transform(self): - ds = dict() + ds = {} for split in self.metadata.eval_splits: labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"])) sentences = list( diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py index 675dcfd5e8..e4ac242609 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py @@ -17,7 +17,7 @@ def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = No cache_dir=cache_dir, revision=revision, ) - dataset_splits = [split for split in dataset] + dataset_splits = list(dataset) shared_corpus = concatenate_datasets([dataset[split] for split in dataset_splits]) shared_corpus = shared_corpus.map( diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py index 8b22c7720b..89097e06d2 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py @@ -17,7 +17,7 @@ def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = No cache_dir=cache_dir, revision=revision, ) - dataset_splits = [split for split in dataset] + dataset_splits = list(dataset) shared_corpus = concatenate_datasets([dataset[split] for split in dataset_splits]) shared_corpus = shared_corpus.map( diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py index 1c26c61066..b6f01bb325 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py @@ -17,7 +17,7 @@ def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = No cache_dir=cache_dir, revision=revision, ) - dataset_splits = [split for split in dataset] + dataset_splits = list(dataset) def map_function(split_name): return lambda x, idx: { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py index 245088eaf3..5a44585737 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py @@ -17,7 +17,7 @@ def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = No cache_dir=cache_dir, revision=revision, ) - dataset_splits = [split for split in dataset] + dataset_splits = list(dataset) def map_function(split_name): return lambda x, idx: { diff --git a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py index c64f28341e..7351cf0f22 100644 --- a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py @@ -13,7 +13,6 @@ class MNISTClassification(AbsTaskImageClassification): dataset={ "path": "ylecun/mnist", "revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "trust_remote_code": True, }, type="Classification", category="i2t", diff --git a/mteb/tasks/Image/__init__.py b/mteb/tasks/Image/__init__.py index 255d9af637..bcaa875644 100644 --- a/mteb/tasks/Image/__init__.py +++ b/mteb/tasks/Image/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from .Any2AnyRetrieval import * from .Clustering import * from .ImageClassification import * diff --git a/mteb/tasks/MultiLabelClassification/__init__.py b/mteb/tasks/MultiLabelClassification/__init__.py index d6e3132c9e..0cf8c1bf6a 100644 --- a/mteb/tasks/MultiLabelClassification/__init__.py +++ b/mteb/tasks/MultiLabelClassification/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from .kor.KorHateSpeechMLClassification import * from .mlt.MalteseNewsClassification import * from .multilingual.MultiEURLEXMultilabelClassification import * diff --git a/mteb/tasks/PairClassification/rus/TERRa.py b/mteb/tasks/PairClassification/rus/TERRa.py index e11dd72925..fdb711c8d6 100644 --- a/mteb/tasks/PairClassification/rus/TERRa.py +++ b/mteb/tasks/PairClassification/rus/TERRa.py @@ -13,7 +13,7 @@ class TERRa(AbsTaskPairClassification): "revision": "7b58f24536063837d644aab9a023c62199b2a612", }, description="Textual Entailment Recognition for Russian. This task requires to recognize, given two text fragments, " - "whether the meaning of one text is entailed (can be inferred) from the other text.", + + "whether the meaning of one text is entailed (can be inferred) from the other text.", reference="https://arxiv.org/pdf/2010.15925", type="PairClassification", category="s2s", diff --git a/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py b/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py index 8d73d56a3f..757fddb0fe 100644 --- a/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py +++ b/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks import AbsTaskRetrieval, TaskMetadata diff --git a/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py b/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py index 73ffe5e6ca..89632f6c5a 100644 --- a/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py +++ b/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks import AbsTaskRetrieval, TaskMetadata diff --git a/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py b/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py index ea18d9af3f..344cc63a0d 100644 --- a/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py +++ b/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks import AbsTaskRetrieval, TaskMetadata diff --git a/mteb/tasks/Retrieval/deu/GermanDPRRetrieval.py b/mteb/tasks/Retrieval/deu/GermanDPRRetrieval.py index 27bb0c24ea..09c0e5cc51 100644 --- a/mteb/tasks/Retrieval/deu/GermanDPRRetrieval.py +++ b/mteb/tasks/Retrieval/deu/GermanDPRRetrieval.py @@ -57,7 +57,7 @@ class GermanDPR(AbsTaskRetrieval): @staticmethod def _format_documents(docs, id_prefix="", existing_docs=None): if existing_docs is None: - existing_docs = dict() + existing_docs = {} result = {} for i, (title, content) in enumerate(zip(docs["title"], docs["text"])): formatted_content = content.split("==\n")[-1].replace("\n", " ").lstrip() @@ -77,10 +77,10 @@ def load_data(self, **kwargs): split=self._EVAL_SPLIT, **self.metadata_dict["dataset"], ) - corpus = dict() - queries = dict() - relevant_docs = dict() - all_docs = dict() + corpus = {} + queries = {} + relevant_docs = {} + all_docs = {} for i, row in enumerate(data): q_id = f"q_{i}" queries[q_id] = row["question"] diff --git a/mteb/tasks/Retrieval/eng/BrightRetrieval.py b/mteb/tasks/Retrieval/eng/BrightRetrieval.py index 0eff276ea5..b1dfdcd42a 100644 --- a/mteb/tasks/Retrieval/eng/BrightRetrieval.py +++ b/mteb/tasks/Retrieval/eng/BrightRetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from collections import defaultdict import datasets diff --git a/mteb/tasks/Retrieval/eng/FEVERRetrieval.py b/mteb/tasks/Retrieval/eng/FEVERRetrieval.py index 3a54df8168..8a28b3d0e0 100644 --- a/mteb/tasks/Retrieval/eng/FEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/FEVERRetrieval.py @@ -16,8 +16,8 @@ class FEVER(AbsTaskRetrieval): }, description=( "FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences" - " extracted from Wikipedia and subsequently verified without knowledge of the sentence they were" - " derived from." + + " extracted from Wikipedia and subsequently verified without knowledge of the sentence they were" + + " derived from." ), reference="https://fever.ai/", type="Retrieval", diff --git a/mteb/tasks/Retrieval/eng/FaithDialRetrieval.py b/mteb/tasks/Retrieval/eng/FaithDialRetrieval.py index ddfe7cda6c..5cd0a7fdd2 100644 --- a/mteb/tasks/Retrieval/eng/FaithDialRetrieval.py +++ b/mteb/tasks/Retrieval/eng/FaithDialRetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datasets import load_dataset from mteb.abstasks.TaskMetadata import TaskMetadata @@ -16,10 +18,10 @@ class FaithDialRetrieval(AbsTaskRetrieval): reference="https://mcgill-nlp.github.io/FaithDial", description=( "FaithDial is a faithful knowledge-grounded dialogue benchmark." - "It was curated by asking annotators to amend hallucinated utterances in Wizard of Wikipedia (WoW)." - "It consists of conversation histories along with manually labelled relevant passage." - "For the purpose of retrieval, we only consider the instances marked as 'Edification' in the VRM field," - "as the gold passage associated with these instances is non-ambiguous." + + "It was curated by asking annotators to amend hallucinated utterances in Wizard of Wikipedia (WoW)." + + "It consists of conversation histories along with manually labelled relevant passage." + + "For the purpose of retrieval, we only consider the instances marked as 'Edification' in the VRM field," + + "as the gold passage associated with these instances is non-ambiguous." ), type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/HagridRetrieval.py b/mteb/tasks/Retrieval/eng/HagridRetrieval.py index 3ba62b98f3..1b02000a6d 100644 --- a/mteb/tasks/Retrieval/eng/HagridRetrieval.py +++ b/mteb/tasks/Retrieval/eng/HagridRetrieval.py @@ -1,7 +1,6 @@ from __future__ import annotations import uuid -from typing import Dict, List import datasets @@ -21,8 +20,8 @@ class HagridRetrieval(AbsTaskRetrieval): reference="https://github.com/project-miracl/hagrid", description=( "HAGRID (Human-in-the-loop Attributable Generative Retrieval for Information-seeking Dataset)" - "is a dataset for generative information-seeking scenarios. It consists of queries" - "along with a set of manually labelled relevant passages" + + "is a dataset for generative information-seeking scenarios. It consists of queries" + + "along with a set of manually labelled relevant passages" ), type="Retrieval", category="s2p", @@ -87,7 +86,7 @@ def load_data(self, **kwargs): self.data_loaded = True - def preprocess_data(self, dataset: Dict) -> List[Dict]: + def preprocess_data(self, dataset: dict) -> list[dict]: """Preprocessed the data in a format easirer to handle for the loading of queries and corpus ------ @@ -111,7 +110,7 @@ def preprocess_data(self, dataset: Dict) -> List[Dict]: return preprocessed_data - def get_best_answer(self, data: Dict) -> str: + def get_best_answer(self, data: dict) -> str: """Get the best answer among available answers of a query. WARNING : May return None if no good answer available diff --git a/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py b/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py index 3e12ee26b2..ec36211ef7 100644 --- a/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py @@ -14,7 +14,7 @@ class HotpotQA(AbsTaskRetrieval): }, description=( "HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong" - " supervision for supporting facts to enable more explainable question answering systems." + + " supervision for supporting facts to enable more explainable question answering systems." ), reference="https://hotpotqa.github.io/", type="Retrieval", diff --git a/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py b/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py index 2b439ea919..4fe17228a5 100644 --- a/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks.TaskMetadata import TaskMetadata diff --git a/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py index 6995272f33..fa43550350 100644 --- a/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks.TaskMetadata import TaskMetadata diff --git a/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py index e8ebddc6ad..0b38895f06 100644 --- a/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks.TaskMetadata import TaskMetadata diff --git a/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py index 3e57bd3590..9a09eb8079 100644 --- a/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks.TaskMetadata import TaskMetadata diff --git a/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py index 10b1876a19..3c8478c536 100644 --- a/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks.TaskMetadata import TaskMetadata diff --git a/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py b/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py index 6be77111d4..6afae35e5f 100644 --- a/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks.TaskMetadata import TaskMetadata diff --git a/mteb/tasks/Retrieval/eng/MLQuestions.py b/mteb/tasks/Retrieval/eng/MLQuestions.py index ef711c1d12..01ebe4dde4 100644 --- a/mteb/tasks/Retrieval/eng/MLQuestions.py +++ b/mteb/tasks/Retrieval/eng/MLQuestions.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import csv from huggingface_hub import snapshot_download @@ -19,7 +21,7 @@ class MLQuestionsRetrieval(AbsTaskRetrieval): reference="https://github.com/McGill-NLP/MLQuestions", description=( "MLQuestions is a domain adaptation dataset for the machine learning domain" - "It consists of ML questions along with passages from Wikipedia machine learning pages (https://en.wikipedia.org/wiki/Category:Machine_learning)" + + "It consists of ML questions along with passages from Wikipedia machine learning pages (https://en.wikipedia.org/wiki/Category:Machine_learning)" ), type="Retrieval", category="s2p", @@ -98,7 +100,7 @@ def _load_data_for_split(self, download_dir, split): queries, corpus, qrels = {}, {}, {} dataset_path = f"{download_dir}/{split}.csv" - with open(dataset_path, "r") as csvfile: + with open(dataset_path) as csvfile: reader = csv.DictReader(csvfile) for i, row in enumerate(reader): query_id = f"Q{str(i)}" @@ -109,7 +111,7 @@ def _load_data_for_split(self, download_dir, split): # Same corpus for all splits corpus_path = f"{download_dir}/test_passages.csv" - with open(corpus_path, "r") as csvfile: + with open(corpus_path) as csvfile: reader = csv.DictReader(csvfile) for i, row in enumerate(reader): doc_id = f"C{str(i)}" diff --git a/mteb/tasks/Retrieval/eng/NarrativeQARetrieval.py b/mteb/tasks/Retrieval/eng/NarrativeQARetrieval.py index 96d3156878..4f9bb143b6 100644 --- a/mteb/tasks/Retrieval/eng/NarrativeQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/NarrativeQARetrieval.py @@ -19,7 +19,7 @@ class NarrativeQARetrieval(AbsTaskRetrieval): reference="https://metatext.io/datasets/narrativeqa", description=( "NarrativeQA is a dataset for the task of question answering on long narratives. It consists of " - "realistic QA instances collected from literature (fiction and non-fiction) and movie scripts. " + + "realistic QA instances collected from literature (fiction and non-fiction) and movie scripts. " ), type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/QuoraRetrieval.py b/mteb/tasks/Retrieval/eng/QuoraRetrieval.py index a4ca9d1ce1..40e25ba4f2 100644 --- a/mteb/tasks/Retrieval/eng/QuoraRetrieval.py +++ b/mteb/tasks/Retrieval/eng/QuoraRetrieval.py @@ -16,7 +16,7 @@ class QuoraRetrieval(AbsTaskRetrieval): }, description=( "QuoraRetrieval is based on questions that are marked as duplicates on the Quora platform. Given a" - " question, find other (duplicate) questions." + + " question, find other (duplicate) questions." ), reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs", type="Retrieval", diff --git a/mteb/tasks/Retrieval/eng/SCIDOCSRetrieval.py b/mteb/tasks/Retrieval/eng/SCIDOCSRetrieval.py index d4d7bbf0a7..08eef53093 100644 --- a/mteb/tasks/Retrieval/eng/SCIDOCSRetrieval.py +++ b/mteb/tasks/Retrieval/eng/SCIDOCSRetrieval.py @@ -14,7 +14,7 @@ class SCIDOCS(AbsTaskRetrieval): }, description=( "SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation" - " prediction, to document classification and recommendation." + + " prediction, to document classification and recommendation." ), reference="https://allenai.org/data/scidocs", type="Retrieval", diff --git a/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py b/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py index dc20c81fd1..7f521a4595 100644 --- a/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datasets import load_dataset from mteb.abstasks.TaskMetadata import TaskMetadata @@ -22,8 +24,8 @@ class TopiOCQARetrieval(AbsTaskRetrieval): reference="https://mcgill-nlp.github.io/topiocqa", description=( "TopiOCQA (Human-in-the-loop Attributable Generative Retrieval for Information-seeking Dataset)" - "is information-seeking conversational dataset with challenging topic switching phenomena." - "It consists of conversation histories along with manually labelled relevant/gold passage." + + "is information-seeking conversational dataset with challenging topic switching phenomena." + + "It consists of conversation histories along with manually labelled relevant/gold passage." ), type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py b/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py index 89ee686fdb..84cde66010 100644 --- a/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py @@ -184,7 +184,7 @@ class BelebeleRetrieval(MultilingualTask, AbsTaskRetrieval): }, description=( "Belebele is a multiple-choice machine reading comprehension (MRC) dataset spanning 122 language variants " - "(including 115 distinct languages and their scripts)" + + "(including 115 distinct languages and their scripts)" ), type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py index f5daa7f57c..f5e86e4ad0 100644 --- a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py +++ b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py @@ -1,4 +1,4 @@ -from typing import Dict, List +from __future__ import annotations import datasets @@ -10,14 +10,14 @@ } -def _build_lang_pair(langs: List[str]) -> str: +def _build_lang_pair(langs: list[str]) -> str: """Builds a language pair separated by a dash. e.g., ['eng-Latn', 'deu-Latn'] -> 'eng-deu'. """ return langs[0].split("-")[0] + "-" + langs[1].split("-")[0] -def extend_lang_pairs() -> Dict[str, List[str]]: +def extend_lang_pairs() -> dict[str, list[str]]: eval_langs = {} for langs in _LANGUAGES.values(): lang_pair = _build_lang_pair(langs) diff --git a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py index dea6bb121d..17e2c6e0c6 100644 --- a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py +++ b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py @@ -1,4 +1,4 @@ -from typing import Dict, List +from __future__ import annotations import datasets @@ -10,14 +10,14 @@ } -def _build_lang_pair(langs: List[str]) -> str: +def _build_lang_pair(langs: list[str]) -> str: """Builds a language pair separated by a dash. e.g., ['eng-Latn', 'deu-Latn'] -> 'eng-deu'. """ return langs[0].split("-")[0] + "-" + langs[1].split("-")[0] -def extend_lang_pairs() -> Dict[str, List[str]]: +def extend_lang_pairs() -> dict[str, list[str]]: eval_langs = {} for langs in _LANGUAGES.values(): lang_pair = _build_lang_pair(langs) diff --git a/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py b/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py index 22e7455384..cd861b888c 100644 --- a/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py @@ -1,4 +1,4 @@ -from typing import Dict, List +from __future__ import annotations import datasets @@ -57,14 +57,14 @@ } -def _build_lang_pair(langs: List[str]) -> str: +def _build_lang_pair(langs: list[str]) -> str: """Builds a language pair separated by a dash. e.g., ['eng-Latn', 'deu-Latn'] -> 'eng-deu'. """ return langs[0].split("-")[0] + "-" + langs[1].split("-")[0] -def extend_lang_pairs() -> Dict[str, List[str]]: +def extend_lang_pairs() -> dict[str, list[str]]: eval_langs = {} for langs in _LANGUAGES.values(): lang_pair = _build_lang_pair(langs) diff --git a/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py b/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py index fa1212fe6c..745734116a 100644 --- a/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py @@ -1,4 +1,4 @@ -from typing import List +from __future__ import annotations import datasets @@ -423,7 +423,7 @@ def load_data(self, **kwargs): self.data_loaded = True -def _load_dataset_csv(path: str, revision: str, eval_splits: List[str]): +def _load_dataset_csv(path: str, revision: str, eval_splits: list[str]): data_files = { eval_split: f"https://huggingface.co/datasets/{path}/resolve/{revision}/{eval_split}.csv" for eval_split in eval_splits diff --git a/mteb/tasks/Retrieval/nob/norquad.py b/mteb/tasks/Retrieval/nob/norquad.py index 8a6603f850..3cafda9f68 100644 --- a/mteb/tasks/Retrieval/nob/norquad.py +++ b/mteb/tasks/Retrieval/nob/norquad.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks import AbsTaskRetrieval, TaskMetadata diff --git a/mteb/tasks/Retrieval/nob/snl_retrieval.py b/mteb/tasks/Retrieval/nob/snl_retrieval.py index 65f7f00b70..6845115d53 100644 --- a/mteb/tasks/Retrieval/nob/snl_retrieval.py +++ b/mteb/tasks/Retrieval/nob/snl_retrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks import AbsTaskRetrieval, TaskMetadata diff --git a/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py b/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py index 8540614450..e459809c4b 100644 --- a/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py +++ b/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks import AbsTaskRetrieval, TaskMetadata diff --git a/mteb/tasks/Retrieval/swe/SwednRetrieval.py b/mteb/tasks/Retrieval/swe/SwednRetrieval.py index a5dae8a541..45bb8aa337 100644 --- a/mteb/tasks/Retrieval/swe/SwednRetrieval.py +++ b/mteb/tasks/Retrieval/swe/SwednRetrieval.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks import AbsTaskRetrieval, TaskMetadata diff --git a/mteb/tasks/Retrieval/tur/TurHistQuad.py b/mteb/tasks/Retrieval/tur/TurHistQuad.py index 312e6d94ab..0781ed7dd3 100644 --- a/mteb/tasks/Retrieval/tur/TurHistQuad.py +++ b/mteb/tasks/Retrieval/tur/TurHistQuad.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datasets from mteb.abstasks import AbsTaskRetrieval, TaskMetadata diff --git a/mteb/tasks/STS/deu/GermanSTSBenchmarkSTS.py b/mteb/tasks/STS/deu/GermanSTSBenchmarkSTS.py index 0a48d8a34d..7b2b5b59fb 100644 --- a/mteb/tasks/STS/deu/GermanSTSBenchmarkSTS.py +++ b/mteb/tasks/STS/deu/GermanSTSBenchmarkSTS.py @@ -13,7 +13,7 @@ class GermanSTSBenchmarkSTS(AbsTaskSTS): "revision": "e36907544d44c3a247898ed81540310442329e20", }, description="Semantic Textual Similarity Benchmark (STSbenchmark) dataset translated into German. " - "Translations were originally done by T-Systems on site services GmbH.", + + "Translations were originally done by T-Systems on site services GmbH.", reference="https://github.com/t-systems-on-site-services-gmbh/german-STSbenchmark", type="STS", category="s2s", diff --git a/mteb/tasks/STS/jpn/JSTS.py b/mteb/tasks/STS/jpn/JSTS.py index 07b4a3fb67..63ab4a546d 100644 --- a/mteb/tasks/STS/jpn/JSTS.py +++ b/mteb/tasks/STS/jpn/JSTS.py @@ -15,7 +15,7 @@ class JSTS(AbsTaskSTS): "trust_remote_code": True, }, description="Japanese Semantic Textual Similarity Benchmark dataset construct from YJ Image Captions Dataset" - "(Miyazaki and Shimizu, 2016) and annotated by crowdsource annotators.", + + "(Miyazaki and Shimizu, 2016) and annotated by crowdsource annotators.", reference="https://aclanthology.org/2022.lrec-1.317.pdf#page=2.00", type="STS", category="s2s", diff --git a/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py b/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py index 91b6887610..bd7d3cd3c6 100644 --- a/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py +++ b/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py @@ -30,7 +30,7 @@ class STSBenchmarkMultilingualSTS(AbsTaskSTS, MultilingualTask): }, description=( "Semantic Textual Similarity Benchmark (STSbenchmark) dataset," - "but translated using DeepL API." + + "but translated using DeepL API." ), reference="https://github.com/PhilipMay/stsb-multi-mt/", type="STS", diff --git a/mteb/tasks/STS/multilingual/SemRel24STS.py b/mteb/tasks/STS/multilingual/SemRel24STS.py index 701745d48d..943851d9e1 100644 --- a/mteb/tasks/STS/multilingual/SemRel24STS.py +++ b/mteb/tasks/STS/multilingual/SemRel24STS.py @@ -29,9 +29,9 @@ class SemRel24STS(AbsTaskSTS, MultilingualTask): }, description=( "SemRel2024 is a collection of Semantic Textual Relatedness (STR) datasets for 14 languages, " - "including African and Asian languages. The datasets are composed of sentence pairs, each assigned a " - "relatedness score between 0 (completely) unrelated and 1 (maximally related) with a large range of " - "expected relatedness values." + + "including African and Asian languages. The datasets are composed of sentence pairs, each assigned a " + + "relatedness score between 0 (completely) unrelated and 1 (maximally related) with a large range of " + + "expected relatedness values." ), reference="https://huggingface.co/datasets/SemRel/SemRel2024", type="STS", diff --git a/mteb/tasks/STS/rus/RuSTSBenchmarkSTS.py b/mteb/tasks/STS/rus/RuSTSBenchmarkSTS.py index 08f27d87c2..35bf75ba04 100644 --- a/mteb/tasks/STS/rus/RuSTSBenchmarkSTS.py +++ b/mteb/tasks/STS/rus/RuSTSBenchmarkSTS.py @@ -13,7 +13,7 @@ class RuSTSBenchmarkSTS(AbsTaskSTS): "revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", }, description="Semantic Textual Similarity Benchmark (STSbenchmark) dataset translated into Russian and verified. " - "The dataset was checked with RuCOLA model to ensure that the translation is good and filtered.", + + "The dataset was checked with RuCOLA model to ensure that the translation is good and filtered.", reference="https://github.com/PhilipMay/stsb-multi-mt/", type="STS", category="s2s", diff --git a/mteb/tasks/SpeedTask/__init__.py b/mteb/tasks/SpeedTask/__init__.py index c48548e329..5e9d2ce9bb 100644 --- a/mteb/tasks/SpeedTask/__init__.py +++ b/mteb/tasks/SpeedTask/__init__.py @@ -1,2 +1,4 @@ +from __future__ import annotations + from .CPUSpeedTask import * from .GPUSpeedTask import * diff --git a/pyproject.toml b/pyproject.toml index 19d804e726..8916c12d96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ homepage = "https://github.com/embeddings-benchmark/mteb" mteb = "mteb.cli:main" [project.optional-dependencies] -dev = ["ruff>=0.0.254", "pytest", "pytest-xdist", "pytest-coverage"] +dev = ["ruff>=0.6.0", "pytest", "pytest-xdist", "pytest-coverage"] codecarbon = ["codecarbon"] speedtask = ["GPUtil>=1.4.0", "psutil>=5.9.8"] @@ -88,7 +88,18 @@ exclude = ["tests", "results"] target-version = "py38" [tool.ruff.lint] -select = ["F", "I", "E", "D"] +select = [ + "F", + "I", + "E", + "D", + "UP", + "FA", + "C4", + "ISC", +] +unfixable = ["ISC001"] + ignore = ["E501", # line too long "E741", # ambiguous variable name "F403", # undefined import @@ -103,6 +114,12 @@ ignore = ["E501", # line too long "D415", # First line should end with a period ] +[tool.ruff.lint.flake8-implicit-str-concat] +allow-multiline = false + +[tool.ruff.lint.isort] +required-imports = ["from __future__ import annotations"] + [tool.ruff.lint.pydocstyle] convention = "google" diff --git a/scripts/data/arxiv/script_raw.py b/scripts/data/arxiv/script_raw.py index 39cfa77acb..faef32a74c 100644 --- a/scripts/data/arxiv/script_raw.py +++ b/scripts/data/arxiv/script_raw.py @@ -10,7 +10,7 @@ import jsonlines from tqdm import tqdm -with open("archive/arxiv-metadata-oai-snapshot.json", "r") as file: +with open("archive/arxiv-metadata-oai-snapshot.json") as file: old_lines = file.readlines() new_lines = [] split = 0 diff --git a/scripts/data/bucc/create_data.py b/scripts/data/bucc/create_data.py index c642140133..daacc91319 100644 --- a/scripts/data/bucc/create_data.py +++ b/scripts/data/bucc/create_data.py @@ -9,13 +9,13 @@ repo_name = "bucc-bitext-mining" # create_repo(repo_name, organization="mteb", repo_type="dataset") -with open("bucc-data/zh-en/zh-en.training.zh", "r") as f: +with open("bucc-data/zh-en/zh-en.training.zh") as f: sentence1 = f.readlines() -with open("bucc-data/zh-en/zh-en.training.en", "r") as f: +with open("bucc-data/zh-en/zh-en.training.en") as f: sentence2 = f.readlines() -with open("bucc-data/zh-en/zh-en.training.gold", "r") as f: +with open("bucc-data/zh-en/zh-en.training.gold") as f: gold = f.readlines() diff --git a/scripts/data/create_task_table.py b/scripts/data/create_task_table.py index 07746e3376..aa9b5f6ccc 100644 --- a/scripts/data/create_task_table.py +++ b/scripts/data/create_task_table.py @@ -16,9 +16,9 @@ "sentence1", "sentence2", "sent1", - "sent2" "query", + "sent2query", "positive", - "negative" "queries", + "negativequeries", "corpus", "machine_summaries", "human_summaries", diff --git a/scripts/data/medicalqaretrieval/create_data.py b/scripts/data/medicalqaretrieval/create_data.py index 9005539d76..ce178ba591 100644 --- a/scripts/data/medicalqaretrieval/create_data.py +++ b/scripts/data/medicalqaretrieval/create_data.py @@ -2,13 +2,12 @@ import os import uuid -from typing import Dict from datasets import load_dataset from huggingface_hub import create_repo, upload_file -def preprocess_data(example: Dict) -> Dict: +def preprocess_data(example: dict) -> dict: """Preprocessed the data in a format easier to handle for the loading of queries and corpus ------ diff --git a/scripts/data/mind/prepare_data.py b/scripts/data/mind/prepare_data.py index 1c441bdf5e..884271d11c 100644 --- a/scripts/data/mind/prepare_data.py +++ b/scripts/data/mind/prepare_data.py @@ -32,7 +32,7 @@ def proc_row(row): elif label == "0": negatives.append(idx) else: - raise Exception("Unknown label: {}".format(label)) + raise Exception(f"Unknown label: {label}") row["positive"] = [df_news.loc[idx]["text"] for idx in positives] row["negative"] = [df_news.loc[idx]["text"] for idx in negatives] queries = row["query"].split() diff --git a/scripts/data/redditp2p/script_clustering.py b/scripts/data/redditp2p/script_clustering.py index 5b9aa775d9..406ace3777 100644 --- a/scripts/data/redditp2p/script_clustering.py +++ b/scripts/data/redditp2p/script_clustering.py @@ -25,7 +25,7 @@ ) unique, counts = np.unique(ds["subreddit"], return_counts=True) -unique_to_count = {k: v for k, v in zip(unique, counts)} +unique_to_count = dict(zip(unique, counts)) # Check top subreddits :) # sorted(unique_to_count, key=lambda x: unique_to_count[x], reverse=True)[:10] diff --git a/scripts/data/wikipedia_reranking/create_data.py b/scripts/data/wikipedia_reranking/create_data.py index ad0a3e668b..fdebd831d7 100644 --- a/scripts/data/wikipedia_reranking/create_data.py +++ b/scripts/data/wikipedia_reranking/create_data.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from collections import defaultdict from time import sleep @@ -62,7 +64,7 @@ def map_corpus_to_query(example, negatives_dict): repo_id=repo_id, filename="README.md", repo_type="dataset" ) -with open(readme_path, "r") as f: +with open(readme_path) as f: readme_content = f.read() readme = """ diff --git a/scripts/data/wikipedia_retrieval/create_data.py b/scripts/data/wikipedia_retrieval/create_data.py index cc58377148..8ccb7a8afa 100644 --- a/scripts/data/wikipedia_retrieval/create_data.py +++ b/scripts/data/wikipedia_retrieval/create_data.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from time import sleep from datasets import DatasetDict, Features, Value, load_dataset @@ -87,7 +89,7 @@ def apply_query_id(example, queries_dict): repo_id=repo_id, filename="README.md", repo_type="dataset" ) - with open(readme_path, "r") as f: + with open(readme_path) as f: readme_content = f.read() readme = """ diff --git a/scripts/mmteb/create_dataset_citations_bib.py b/scripts/mmteb/create_dataset_citations_bib.py index ec74af2819..beaf26f9f9 100644 --- a/scripts/mmteb/create_dataset_citations_bib.py +++ b/scripts/mmteb/create_dataset_citations_bib.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from pathlib import Path import bibtexparser @@ -69,10 +71,10 @@ def task_to_tex_row(task: mteb.AbsTask) -> str: ) n_samples = ( "{:.2f}".format( - ( + sum(task.metadata.n_samples.values()) / len(task.metadata.n_samples.keys()) - ) + ) if task.metadata.n_samples else "" @@ -80,10 +82,10 @@ def task_to_tex_row(task: mteb.AbsTask) -> str: avg_character_length = ( "{:.2f}".format( - ( + sum(task.metadata.avg_character_length.values()) / len(task.metadata.avg_character_length.keys()) - ) + ) if task.metadata.avg_character_length else "" diff --git a/scripts/mmteb/result_analysis/create_result_tables.py b/scripts/mmteb/result_analysis/create_result_tables.py index d8af8455b7..ab52a731de 100644 --- a/scripts/mmteb/result_analysis/create_result_tables.py +++ b/scripts/mmteb/result_analysis/create_result_tables.py @@ -20,7 +20,7 @@ def filter_results( iter_models = models if models is not None else results.keys() if tasks is not None: - task_names = set(t.metadata.name for t in tasks) + task_names = {t.metadata.name for t in tasks} for mdl in iter_models: if isinstance(mdl, mteb.ModelMeta): diff --git a/scripts/mmteb/running_baseline_model.py b/scripts/mmteb/running_baseline_model.py index 9e37d6f68c..3ee0da2dea 100644 --- a/scripts/mmteb/running_baseline_model.py +++ b/scripts/mmteb/running_baseline_model.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import mteb baseline_models = [ diff --git a/scripts/mmteb/running_model/check_results.py b/scripts/mmteb/running_model/check_results.py index 01cded9a56..a4b166e0c8 100644 --- a/scripts/mmteb/running_model/check_results.py +++ b/scripts/mmteb/running_model/check_results.py @@ -43,7 +43,7 @@ def filter_results( iter_models = models if models is not None else results.keys() if tasks is not None: - task_names = set(t.metadata.name for t in tasks) + task_names = {t.metadata.name for t in tasks} for mdl in iter_models: if isinstance(mdl, mteb.ModelMeta): @@ -192,7 +192,7 @@ def normalize_results(results): nans = wide_table[wide_table.isna().any(axis=1)] # create list of model names x task names which is missing -t_names = set([t.metadata.name for t in tasks_which_should_be_there]) +t_names = {t.metadata.name for t in tasks_which_should_be_there} sav_str = "" diff --git a/scripts/mmteb/task_selection/benchmark_construction.py b/scripts/mmteb/task_selection/benchmark_construction.py index b7a0a80765..e527958156 100644 --- a/scripts/mmteb/task_selection/benchmark_construction.py +++ b/scripts/mmteb/task_selection/benchmark_construction.py @@ -27,6 +27,7 @@ - MTEB(Scandinavian, v2) - an version of the Mainland Scandinavian benchmark - shows an example of a semi-local language benchmark - MTEB(Code) or some other domain specific benchmark - shows an example of a domain specific benchmark """ +from __future__ import annotations import mteb from mteb.benchmarks import MTEB_MAIN_EN, MTEB_MAIN_RU @@ -178,7 +179,7 @@ "yue", "jpn", "kor", - "mon" + "mon", # south asian - indic # (south asian - dravidian) "hin", diff --git a/scripts/run_mteb_bright.py b/scripts/run_mteb_bright.py index 58d2fbfe7d..e6420bb9e7 100644 --- a/scripts/run_mteb_bright.py +++ b/scripts/run_mteb_bright.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from sentence_transformers import SentenceTransformer from mteb import MTEB diff --git a/tests/test_TaskMetadata.py b/tests/test_TaskMetadata.py index 65a4ecf52b..3502acfa65 100644 --- a/tests/test_TaskMetadata.py +++ b/tests/test_TaskMetadata.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import pytest @@ -316,7 +318,7 @@ def test_given_none_revision_path_then_it_logs_warning(caplog): assert len(warning_logs) == 1 assert ( warning_logs[0].message == "Revision missing for the dataset test/dataset. " - "It is encourage to specify a dataset revision for reproducability." + + "It is encourage to specify a dataset revision for reproducability." ) diff --git a/tests/test_benchmark/mock_tasks.py b/tests/test_benchmark/mock_tasks.py index 85a5b27196..0b41f42152 100644 --- a/tests/test_benchmark/mock_tasks.py +++ b/tests/test_benchmark/mock_tasks.py @@ -19,27 +19,27 @@ from mteb.abstasks.AbsTaskSummarization import AbsTaskSummarization from mteb.abstasks.TaskMetadata import TaskMetadata -general_args = dict( - description="a mock task for testing", - reference="https://github.com/embeddings-benchmark/mteb", - dataset={ +general_args = { + "description": "a mock task for testing", + "reference": "https://github.com/embeddings-benchmark/mteb", + "dataset": { "path": "NA", "revision": "NA", }, - category="s2s", - eval_splits=["test"], - eval_langs=["eng-Latn"], - date=("2022-12-22", "2022-12-22"), - dialect=["Written"], - domains=[], - task_subtypes=[], - license="NA", - annotations_creators="derived", - modalities=["text"], - sample_creation="found", - bibtex_citation="", - descriptive_stats={}, -) + "category": "s2s", + "eval_splits": ["test"], + "eval_langs": ["eng-Latn"], + "date": ("2022-12-22", "2022-12-22"), + "dialect": ["Written"], + "domains": [], + "task_subtypes": [], + "license": "NA", + "annotations_creators": "derived", + "modalities": ["text"], + "sample_creation": "found", + "bibtex_citation": "", + "descriptive_stats": {}, +} class MockClassificationTask(AbsTaskClassification): diff --git a/tests/test_benchmark/test_benchmark.py b/tests/test_benchmark/test_benchmark.py index f30e854aae..b2c81f9084 100644 --- a/tests/test_benchmark/test_benchmark.py +++ b/tests/test_benchmark/test_benchmark.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -from typing import Union import numpy as np import pytest @@ -31,7 +30,7 @@ def test_mulitple_mteb_tasks(tasks: list[mteb.AbsTask], model: mteb.Encoder): "model", [MockNumpyEncoder(), MockTorchEncoder(), MockTorchbf16Encoder()] ) def test_benchmark_encoders_on_task( - task: Union[str, mteb.AbsTask], model: mteb.Encoder + task: str | mteb.AbsTask, model: mteb.Encoder ): """Test that a task can be fetched and run using a variety of encoders""" if isinstance(task, str): diff --git a/tests/test_benchmark/test_benchmark_integration_with_datasets.py b/tests/test_benchmark/test_benchmark_integration_with_datasets.py index 369ad0acda..31a4b3d30c 100644 --- a/tests/test_benchmark/test_benchmark_integration_with_datasets.py +++ b/tests/test_benchmark/test_benchmark_integration_with_datasets.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -from typing import Union import pytest @@ -19,7 +18,7 @@ @pytest.mark.parametrize("task", TASK_TEST_GRID) @pytest.mark.parametrize("model", [MockNumpyEncoder()]) -def test_benchmark_sentence_transformer(task: Union[str, AbsTask], model: mteb.Encoder): +def test_benchmark_sentence_transformer(task: str | AbsTask, model: mteb.Encoder): """Test that a task can be fetched and run""" eval = MTEB(tasks=[task]) eval.run(model, output_folder="tests/results", overwrite_results=True) diff --git a/tests/test_benchmark/test_benchmark_integration_with_sentencetransformers.py b/tests/test_benchmark/test_benchmark_integration_with_sentencetransformers.py index 376466b727..4ca0056cd7 100644 --- a/tests/test_benchmark/test_benchmark_integration_with_sentencetransformers.py +++ b/tests/test_benchmark/test_benchmark_integration_with_sentencetransformers.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -from typing import Union import pytest from sentence_transformers import SentenceTransformer @@ -23,7 +22,7 @@ "average_word_embeddings_levy_dependency", ], ) -def test_benchmark_sentence_transformer(task: Union[str, AbsTask], model_name: str): +def test_benchmark_sentence_transformer(task: str | AbsTask, model_name: str): """Test that a task can be fetched and run""" if isinstance(model_name, str): model = SentenceTransformer(model_name) diff --git a/tests/test_cli.py b/tests/test_cli.py index ec2cbd77fc..cc49f745d3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,5 @@ """tests for the MTEB CLI""" +from __future__ import annotations import subprocess from argparse import Namespace diff --git a/tests/test_encoder_interfaces.py b/tests/test_encoder_interfaces.py index 42a308435f..941b75dca1 100644 --- a/tests/test_encoder_interfaces.py +++ b/tests/test_encoder_interfaces.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from sentence_transformers import SentenceTransformer from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode diff --git a/tests/test_evaluators/test_ClusteringEvaluator.py b/tests/test_evaluators/test_ClusteringEvaluator.py index 9ead694af6..a0209857ba 100644 --- a/tests/test_evaluators/test_ClusteringEvaluator.py +++ b/tests/test_evaluators/test_ClusteringEvaluator.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import List - import numpy as np from mteb.evaluation.evaluators import ClusteringEvaluator @@ -12,7 +10,7 @@ def test_clustering_v_measure(self): class Model: def encode( self, - sentences: List[str], + sentences: list[str], prompt_name: str | None = None, batch_size=32, ) -> np.ndarray: diff --git a/tests/test_langscripts.py b/tests/test_langscripts.py index f4e4ab9482..511fbb06f9 100644 --- a/tests/test_langscripts.py +++ b/tests/test_langscripts.py @@ -17,21 +17,21 @@ class LangScriptTestCase: test_cases = [ LangScriptTestCase( - args=dict(languages=["fra"], scripts=None), + args={"languages": ["fra"], "scripts": None}, contains_language=["fra", "fra-Latn"], not_contains_language=["eng"], contains_script=[], not_contains_script=["Latn"], ), LangScriptTestCase( - args=dict(languages=["fra", "eng"], scripts=["Latn"]), + args={"languages": ["fra", "eng"], "scripts": ["Latn"]}, contains_language=["fra", "fra-Latn", "eng", "eng-Latn"], not_contains_language=["deu"], contains_script=["Latn"], not_contains_script=["Cyrl"], ), LangScriptTestCase( - args=dict(languages=["fra-Latn"]), + args={"languages": ["fra-Latn"]}, contains_language=["fra", "fra-Latn"], not_contains_language=["eng", "eng-Latn"], contains_script=["Latn"], diff --git a/tests/test_load_results/test_mteb_load_results.py b/tests/test_load_results/test_mteb_load_results.py index 1e44543828..d5d2ec87ef 100644 --- a/tests/test_load_results/test_mteb_load_results.py +++ b/tests/test_load_results/test_mteb_load_results.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os from pathlib import Path diff --git a/tests/test_load_results/test_mteb_results.py b/tests/test_load_results/test_mteb_results.py index 8908a5f173..734d4e23d2 100644 --- a/tests/test_load_results/test_mteb_results.py +++ b/tests/test_load_results/test_mteb_results.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import glob import json import os diff --git a/tests/test_tasks/test_clustering_fast_datasets.py b/tests/test_tasks/test_clustering_fast_datasets.py index fd188dabd4..c25b4482ef 100644 --- a/tests/test_tasks/test_clustering_fast_datasets.py +++ b/tests/test_tasks/test_clustering_fast_datasets.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from mteb.abstasks.AbsTaskClusteringFast import AbsTaskClusteringFast diff --git a/tests/test_tasks/test_mieb_datasets.py b/tests/test_tasks/test_mieb_datasets.py index 2eca4de4d3..c2122945d6 100644 --- a/tests/test_tasks/test_mieb_datasets.py +++ b/tests/test_tasks/test_mieb_datasets.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -from typing import Union import pytest @@ -19,7 +18,7 @@ @pytest.mark.parametrize("task", MIEB_TASK_TEST_GRID) @pytest.mark.parametrize("model", [MockCLIPEncoder()]) -def test_benchmark_sentence_transformer(task: Union[str, AbsTask], model: mteb.Encoder): +def test_benchmark_sentence_transformer(task: str | AbsTask, model: mteb.Encoder): """Test that a task can be fetched and run""" eval = MTEB(tasks=[task]) eval.run(model, output_folder="tests/results", overwrite_results=True) diff --git a/tests/test_tasks/test_retrieval_abstask.py b/tests/test_tasks/test_retrieval_abstask.py index af787fa16e..585e8a1466 100644 --- a/tests/test_tasks/test_retrieval_abstask.py +++ b/tests/test_tasks/test_retrieval_abstask.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from mteb.abstasks import AbsTaskRetrieval From e696f1a55486a4a519f71ea1fcc43eab097b4e52 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Thu, 12 Sep 2024 05:54:00 +0100 Subject: [PATCH 064/154] Visual STS Abstask&evaluator --- mteb/abstasks/Image/AbsTaskVisualSTS.py | 88 +++++ mteb/abstasks/TaskMetadata.py | 1 + mteb/abstasks/__init__.py | 1 + .../evaluators/Image/VisualSTSEvaluator.py | 141 ++++++++ mteb/evaluation/evaluators/__init__.py | 1 + mteb/tasks/Image/VisualSTS/__init__.py | 1 + .../STSBenchmarkMultilingualVisualSTS.py | 67 ++++ .../Image/VisualSTS/multilingual/__init__.py | 0 mteb/tasks/Image/__init__.py | 1 + mteb/tasks/__init__.py | 1 + .../STSBenchmarkMultilingualVisualSTS.json | 313 ++++++++++++++++++ .../STSBenchmarkMultilingualVisualSTS.json | 313 ++++++++++++++++++ 12 files changed, 928 insertions(+) create mode 100644 mteb/abstasks/Image/AbsTaskVisualSTS.py create mode 100644 mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py create mode 100644 mteb/tasks/Image/VisualSTS/__init__.py create mode 100644 mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py create mode 100644 mteb/tasks/Image/VisualSTS/multilingual/__init__.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STSBenchmarkMultilingualVisualSTS.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STSBenchmarkMultilingualVisualSTS.json diff --git a/mteb/abstasks/Image/AbsTaskVisualSTS.py b/mteb/abstasks/Image/AbsTaskVisualSTS.py new file mode 100644 index 0000000000..c084be038e --- /dev/null +++ b/mteb/abstasks/Image/AbsTaskVisualSTS.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import logging +from typing import Any + +from ...evaluation.evaluators import VisualSTSEvaluator +from ...load_results.mteb_results import ScoresDict +from ..AbsTask import AbsTask, DescriptiveStatistics + +logger = logging.getLogger(__name__) + + +class VisualSTSDescriptiveStatistics(DescriptiveStatistics): + """Descriptive statistics for STS + + Attributes: + num_samples: number of samples in the dataset + avg_score: Average score + """ + + # TODO: what are useful stats for visual STS tasks? + # average_pixel_width; average_pixel_height; average non-white boxes? + + num_samples: int + avg_score: float + + +class AbsTaskVisualSTS(AbsTask): + """Abstract class for visual STS experiments. + + self.load_data() must generate a huggingface dataset with a split matching self.metadata_dict["eval_splits"], and assign it to self.dataset. It must contain the following columns: + sentence1: PIL.Image + sentence2: PIL.Image + score: float + """ + + sentences_column_names = ["sentence1", "sentence2"] + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @property + def min_score(self) -> int: + return self.metadata_dict["min_score"] + + @property + def max_score(self) -> int: + return self.metadata_dict["max_score"] + + def _evaluate_subset( + self, model, data_split, *, encode_kwargs: dict[str, Any] = {}, **kwargs + ) -> ScoresDict: + def normalize(x): + return (x - self.min_score) / (self.max_score - self.min_score) + + normalized_scores = list(map(normalize, data_split["score"])) + evaluator = VisualSTSEvaluator( + data_split, + self.sentences_column_names, + normalized_scores, + task_name=self.metadata.name, + **kwargs, + ) + scores = evaluator(model, encode_kwargs=encode_kwargs) + + self._add_main_score(scores) + return scores + + def _add_main_score(self, scores: ScoresDict) -> None: + scores["main_score"] = scores[self.metadata.main_score] + + def _calculate_metrics_from_split( + self, split: str, hf_subset: str | None = None, compute_overall: bool = False + ) -> VisualSTSDescriptiveStatistics: + if hf_subset: + score = self.dataset[hf_subset][split]["score"] + elif compute_overall: + score = [] + for hf_subset in self.metadata.eval_langs: + score.extend(self.dataset[hf_subset][split]["score"]) + else: + score = self.dataset[split]["score"] + + avg_score = sum(score) / len(score) + return VisualSTSDescriptiveStatistics( + num_samples=len(score), + avg_score=avg_score, + ) diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 3fc5ed7068..290932dd26 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -83,6 +83,7 @@ "machine-translated and verified", "machine-translated and localized", "LM-generated and verified", + "rendered", ] TASK_TYPE = Literal[ diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index 6924ba76f6..f188430f48 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -18,5 +18,6 @@ from .Image.AbsTaskImageClustering import * from .Image.AbsTaskImageMultilabelClassification import * from .Image.AbsTaskImageTextPairClassification import * +from .Image.AbsTaskVisualSTS import * from .Image.AbsTaskZeroshotClassification import * from .MultilingualTask import * diff --git a/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py b/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py new file mode 100644 index 0000000000..d47e060e75 --- /dev/null +++ b/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import logging +from typing import Any +import os + +import numpy as np +from scipy.stats import pearsonr, spearmanr +from sklearn.metrics.pairwise import ( + paired_cosine_distances, + paired_euclidean_distances, + paired_manhattan_distances, +) +import math +import torch +from torch.utils.data import DataLoader +from torchvision import transforms + +from ..Evaluator import Evaluator + +logger = logging.getLogger(__name__) + +transform = transforms.Compose([transforms.PILToTensor()]) + + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, hf_dataset, image_column_name: str = "image", transform=None): + self.dataset = hf_dataset + self.transform = transform + self.image_column_name = image_column_name + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + image = self.dataset[idx][self.image_column_name] + if image.mode != "RGB": + image = image.convert("RGB") + image = self.transform(image) + return image + + +def custom_collate_fn(batch): + return batch + + +class VisualSTSEvaluator(Evaluator): + def __init__( + self, + dataset, + sentences_column_names: list[str], + gold_scores: list[float], + task_name: str | None = None, + **kwargs, + ): + super().__init__(**kwargs) + self.sentence1_dataset = ImageDataset( + dataset, image_column_name=sentences_column_names[0], transform=transform + ) + self.sentence2_dataset = ImageDataset( + dataset, image_column_name=sentences_column_names[1], transform=transform + ) + self.gold_scores = gold_scores + self.task_name = task_name + # TODO use task_name for prompts with interleaved encoding. + + def __call__( + self, + model, # TODO: model type + *, + encode_kwargs: dict[str, Any] = {}, + ): + if "batch_size" not in encode_kwargs: + encode_kwargs["batch_size"] = 32 + + sentence1_dataloader = DataLoader( + self.sentence1_dataset, + batch_size=encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=math.floor(os.cpu_count() / 2), + ) + sentence2_dataloader = DataLoader( + self.sentence2_dataset, + batch_size=encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=math.floor(os.cpu_count() / 2), + ) + + embeddings1 = model.get_image_embeddings( + sentence1_dataloader, batch_size=encode_kwargs["batch_size"] + ) + embeddings2 = model.get_image_embeddings( + sentence2_dataloader, batch_size=encode_kwargs["batch_size"] + ) + + logger.info("Evaluating...") + cosine_scores = 1 - (paired_cosine_distances(embeddings1, embeddings2)) + manhattan_distances = -paired_manhattan_distances(embeddings1, embeddings2) + euclidean_distances = -paired_euclidean_distances(embeddings1, embeddings2) + + cosine_pearson, _ = pearsonr(self.gold_scores, cosine_scores) + cosine_spearman, _ = spearmanr(self.gold_scores, cosine_scores) + + manhatten_pearson, _ = pearsonr(self.gold_scores, manhattan_distances) + manhatten_spearman, _ = spearmanr(self.gold_scores, manhattan_distances) + + euclidean_pearson, _ = pearsonr(self.gold_scores, euclidean_distances) + euclidean_spearman, _ = spearmanr(self.gold_scores, euclidean_distances) + + similarity_scores = None + if hasattr(model, "similarity_pairwise"): + similarity_scores = model.similarity_pairwise(embeddings1, embeddings2) # type: ignore + elif hasattr(model, "similarity"): + _similarity_scores = [ + float(model.similarity(e1, e2)) # type: ignore + for e1, e2 in zip(embeddings1, embeddings2) + ] + similarity_scores = np.array(_similarity_scores) + + if similarity_scores is not None: + pearson = pearsonr(self.gold_scores, similarity_scores) + spearman = spearmanr(self.gold_scores, similarity_scores) + else: + # if model does not have a similarity function, we assume the cosine similarity + pearson = cosine_pearson + spearman = cosine_spearman + + return { + # using the models own similarity score + "pearson": pearson, + "spearman": spearman, + # generic similarity scores + "cosine_pearson": cosine_pearson, + "cosine_spearman": cosine_spearman, + "manhattan_pearson": manhatten_pearson, + "manhattan_spearman": manhatten_spearman, + "euclidean_pearson": euclidean_pearson, + "euclidean_spearman": euclidean_spearman, + } diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py index 111e899e42..ce7da0db59 100644 --- a/mteb/evaluation/evaluators/__init__.py +++ b/mteb/evaluation/evaluators/__init__.py @@ -7,6 +7,7 @@ from .Image.ClassificationEvaluator import * from .Image.ClusteringEvaluator import * from .Image.ImageTextPairClassificationEvaluator import * +from .Image.VisualSTSEvaluator import * from .Image.ZeroshotClassificationEvaluator import * from .PairClassificationEvaluator import * from .RerankingEvaluator import * diff --git a/mteb/tasks/Image/VisualSTS/__init__.py b/mteb/tasks/Image/VisualSTS/__init__.py new file mode 100644 index 0000000000..6cb9ac8ef9 --- /dev/null +++ b/mteb/tasks/Image/VisualSTS/__init__.py @@ -0,0 +1 @@ +from .multilingual.STSBenchmarkMultilingualVisualSTS import * diff --git a/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py b/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py new file mode 100644 index 0000000000..2283643f83 --- /dev/null +++ b/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS +from mteb.abstasks.MultilingualTask import MultilingualTask +from mteb.abstasks.TaskMetadata import TaskMetadata + +_LANGUAGES = { + "en": ["eng-Latn"], + "de": ["deu-Latn"], + "es": ["spa-Latn"], + "fr": ["fra-Latn"], + "it": ["ita-Latn"], + "nl": ["nld-Latn"], + "pl": ["pol-Latn"], + "pt": ["por-Latn"], + "ru": ["rus-Cyrl"], + "zh": ["cmn-Hans"], +} + +_SPLITS = ["dev", "test"] + + +class STSBenchmarkMultilingualVisualSTS(AbsTaskVisualSTS, MultilingualTask): + metadata = TaskMetadata( + name="STSBenchmarkMultilingualVisualSTS", + dataset={ + "path": "Pixel-Linguist/rendered-stsb", + "revision": "9f1ab21f17f497974996ab74b3ff911165a7dbf9", + }, + description=( + "Semantic Textual Similarity Benchmark (STSbenchmark) dataset, " + + "translated into target languages using DeepL API," + + "then rendered into images." + + "built upon multi-sts created by Philip May" + ), + reference="https://arxiv.org/abs/2402.08183/", + type="STS", + category="i2i", + modalities=["image"], + eval_splits=_SPLITS, + eval_langs=_LANGUAGES, + main_score="cosine_spearman", + date=("2012-01-01", "2017-12-31"), + domains=["News", "Social", "Web", "Spoken", "Written"], + task_subtypes=[], + license="Not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="rendered", + bibtex_citation="""@article{xiao2024pixel, + title={Pixel Sentence Representation Learning}, + author={Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al}, + journal={arXiv preprint arXiv:2402.08183}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"dev": 30000, "test": 27580}, + "avg_character_length": {"dev": 1.0, "test": 1.0}, + }, + ) + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 0 + metadata_dict["max_score"] = 5 + return metadata_dict diff --git a/mteb/tasks/Image/VisualSTS/multilingual/__init__.py b/mteb/tasks/Image/VisualSTS/multilingual/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Image/__init__.py b/mteb/tasks/Image/__init__.py index bcaa875644..cf632fe736 100644 --- a/mteb/tasks/Image/__init__.py +++ b/mteb/tasks/Image/__init__.py @@ -5,4 +5,5 @@ from .ImageClassification import * from .ImageMultilabelClassification import * from .ImageTextPairClassification import * +from .VisualSTS import * from .ZeroshotClassification import * diff --git a/mteb/tasks/__init__.py b/mteb/tasks/__init__.py index 608b849a70..0d7d1d5fc0 100644 --- a/mteb/tasks/__init__.py +++ b/mteb/tasks/__init__.py @@ -8,6 +8,7 @@ from .Image.ImageClassification import * from .Image.ImageMultilabelClassification import * from .Image.ImageTextPairClassification import * +from .Image.VisualSTS import * from .Image.ZeroshotClassification import * from .InstructionRetrieval import * from .MultiLabelClassification import * diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STSBenchmarkMultilingualVisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STSBenchmarkMultilingualVisualSTS.json new file mode 100644 index 0000000000..90c2bf295e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STSBenchmarkMultilingualVisualSTS.json @@ -0,0 +1,313 @@ +{ + "dataset_revision": "9f1ab21f17f497974996ab74b3ff911165a7dbf9", + "evaluation_time": 227.7307116985321, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "dev": [ + { + "cosine_pearson": 0.5272718426731863, + "cosine_spearman": 0.537680296325815, + "euclidean_pearson": 0.6007770904077094, + "euclidean_spearman": 0.6010636781456551, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.537680296325815, + "manhattan_pearson": 0.6100414071389302, + "manhattan_spearman": 0.6089001105999241, + "pearson": 0.5272718426731863, + "spearman": 0.537680296325815 + }, + { + "cosine_pearson": 0.2676685900225375, + "cosine_spearman": 0.29324273637437337, + "euclidean_pearson": 0.3597775948582906, + "euclidean_spearman": 0.34396994265808895, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.29324273637437337, + "manhattan_pearson": 0.36893705209216576, + "manhattan_spearman": 0.3506676102021138, + "pearson": 0.2676685900225375, + "spearman": 0.29324273637437337 + }, + { + "cosine_pearson": 0.17206470202776458, + "cosine_spearman": 0.20995341198163026, + "euclidean_pearson": 0.26850806204192573, + "euclidean_spearman": 0.2644582230111772, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.20995341198163026, + "manhattan_pearson": 0.2776953897517844, + "manhattan_spearman": 0.27065854111745996, + "pearson": 0.17206470202776458, + "spearman": 0.20995341198163026 + }, + { + "cosine_pearson": 0.23513332207947069, + "cosine_spearman": 0.2548454596994308, + "euclidean_pearson": 0.3185683710098647, + "euclidean_spearman": 0.2921945764011797, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ], + "main_score": 0.2548454596994308, + "manhattan_pearson": 0.3263260541428151, + "manhattan_spearman": 0.30069700086416457, + "pearson": 0.23513332207947069, + "spearman": 0.2548454596994308 + }, + { + "cosine_pearson": 0.17437752754674288, + "cosine_spearman": 0.2022199564598799, + "euclidean_pearson": 0.25458208115505687, + "euclidean_spearman": 0.24701419915645423, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ], + "main_score": 0.2022199564598799, + "manhattan_pearson": 0.2643892014665793, + "manhattan_spearman": 0.25371535347866575, + "pearson": 0.17437752754674288, + "spearman": 0.2022199564598799 + }, + { + "cosine_pearson": 0.20169834707998577, + "cosine_spearman": 0.2315465811887652, + "euclidean_pearson": 0.2856781367609151, + "euclidean_spearman": 0.27242351901545375, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ], + "main_score": 0.2315465811887652, + "manhattan_pearson": 0.28888139925688416, + "manhattan_spearman": 0.27547663872028577, + "pearson": 0.20169834707998577, + "spearman": 0.2315465811887652 + }, + { + "cosine_pearson": 0.2812070785959071, + "cosine_spearman": 0.2957475897374985, + "euclidean_pearson": 0.3611073985811877, + "euclidean_spearman": 0.34699149889488523, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ], + "main_score": 0.2957475897374985, + "manhattan_pearson": 0.37331542624423614, + "manhattan_spearman": 0.3583753122357121, + "pearson": 0.2812070785959071, + "spearman": 0.2957475897374985 + }, + { + "cosine_pearson": 0.13670550978570914, + "cosine_spearman": 0.14557883052048473, + "euclidean_pearson": 0.20315124815132346, + "euclidean_spearman": 0.19102861115778288, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ], + "main_score": 0.14557883052048473, + "manhattan_pearson": 0.20680263501202406, + "manhattan_spearman": 0.19173115079101838, + "pearson": 0.13670550978570914, + "spearman": 0.14557883052048473 + }, + { + "cosine_pearson": 0.12452547178134374, + "cosine_spearman": 0.18614027821316195, + "euclidean_pearson": 0.20199310870295473, + "euclidean_spearman": 0.19732036928609661, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.18614027821316195, + "manhattan_pearson": 0.20622105611449384, + "manhattan_spearman": 0.1996718640479634, + "pearson": 0.12452547178134374, + "spearman": 0.18614027821316195 + }, + { + "cosine_pearson": 0.07938008146809811, + "cosine_spearman": 0.18403175659612492, + "euclidean_pearson": 0.14422004233703312, + "euclidean_spearman": 0.18924749482955278, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ], + "main_score": 0.18403175659612492, + "manhattan_pearson": 0.13883666866119507, + "manhattan_spearman": 0.18413299674895794, + "pearson": 0.07938008146809811, + "spearman": 0.18403175659612492 + } + ], + "test": [ + { + "cosine_pearson": 0.45584849857229814, + "cosine_spearman": 0.47171211498989196, + "euclidean_pearson": 0.5493184633612522, + "euclidean_spearman": 0.542186090785119, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.47171211498989196, + "manhattan_pearson": 0.558028494795885, + "manhattan_spearman": 0.5508031599036809, + "pearson": 0.45584849857229814, + "spearman": 0.47171211498989196 + }, + { + "cosine_pearson": 0.2736341916716048, + "cosine_spearman": 0.294077550139935, + "euclidean_pearson": 0.34400702776748815, + "euclidean_spearman": 0.33227797643753454, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.294077550139935, + "manhattan_pearson": 0.35534149649541, + "manhattan_spearman": 0.33973816803083806, + "pearson": 0.2736341916716048, + "spearman": 0.294077550139935 + }, + { + "cosine_pearson": 0.1787373679481151, + "cosine_spearman": 0.23429932660594835, + "euclidean_pearson": 0.2730687899228438, + "euclidean_spearman": 0.27917737631797584, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.23429932660594835, + "manhattan_pearson": 0.28368410011635753, + "manhattan_spearman": 0.28781260934323166, + "pearson": 0.1787373679481151, + "spearman": 0.23429932660594835 + }, + { + "cosine_pearson": 0.24359460439610567, + "cosine_spearman": 0.2684595434289802, + "euclidean_pearson": 0.31871428107193633, + "euclidean_spearman": 0.29373936931462125, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ], + "main_score": 0.2684595434289802, + "manhattan_pearson": 0.32834491701105845, + "manhattan_spearman": 0.3020391568058281, + "pearson": 0.24359460439610567, + "spearman": 0.2684595434289802 + }, + { + "cosine_pearson": 0.17151725238870894, + "cosine_spearman": 0.20546160529404361, + "euclidean_pearson": 0.2528804508579695, + "euclidean_spearman": 0.24624455796714606, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ], + "main_score": 0.20546160529404361, + "manhattan_pearson": 0.264170451940936, + "manhattan_spearman": 0.25410292788867, + "pearson": 0.17151725238870894, + "spearman": 0.20546160529404361 + }, + { + "cosine_pearson": 0.13346252416405158, + "cosine_spearman": 0.1905489071972136, + "euclidean_pearson": 0.21314765423638377, + "euclidean_spearman": 0.2125972363673462, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ], + "main_score": 0.1905489071972136, + "manhattan_pearson": 0.2222995027779861, + "manhattan_spearman": 0.2185386986595845, + "pearson": 0.13346252416405158, + "spearman": 0.1905489071972136 + }, + { + "cosine_pearson": 0.29193603521517864, + "cosine_spearman": 0.30368819951978826, + "euclidean_pearson": 0.3540877734657154, + "euclidean_spearman": 0.3343842768900143, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ], + "main_score": 0.30368819951978826, + "manhattan_pearson": 0.36455550272672194, + "manhattan_spearman": 0.3449415822927682, + "pearson": 0.29193603521517864, + "spearman": 0.30368819951978826 + }, + { + "cosine_pearson": 0.10018100867037781, + "cosine_spearman": 0.140073723116761, + "euclidean_pearson": 0.1815827550610446, + "euclidean_spearman": 0.17476484700015202, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ], + "main_score": 0.140073723116761, + "manhattan_pearson": 0.19128411004281437, + "manhattan_spearman": 0.18039207613584918, + "pearson": 0.10018100867037781, + "spearman": 0.140073723116761 + }, + { + "cosine_pearson": 0.11943191616611451, + "cosine_spearman": 0.18592383434972357, + "euclidean_pearson": 0.19435454798231933, + "euclidean_spearman": 0.19704910739182657, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.18592383434972357, + "manhattan_pearson": 0.1994075652457647, + "manhattan_spearman": 0.20203574788795442, + "pearson": 0.11943191616611451, + "spearman": 0.18592383434972357 + }, + { + "cosine_pearson": 0.02043380644449274, + "cosine_spearman": 0.10518089601052212, + "euclidean_pearson": 0.0834029290349068, + "euclidean_spearman": 0.11379865455499177, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ], + "main_score": 0.10518089601052212, + "manhattan_pearson": 0.07858627139925128, + "manhattan_spearman": 0.11183544342388632, + "pearson": 0.02043380644449274, + "spearman": 0.10518089601052212 + } + ] + }, + "task_name": "STSBenchmarkMultilingualVisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STSBenchmarkMultilingualVisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STSBenchmarkMultilingualVisualSTS.json new file mode 100644 index 0000000000..254c21fe5f --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STSBenchmarkMultilingualVisualSTS.json @@ -0,0 +1,313 @@ +{ + "dataset_revision": "9f1ab21f17f497974996ab74b3ff911165a7dbf9", + "evaluation_time": 803.8532636165619, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "dev": [ + { + "cosine_pearson": 0.6064690420220407, + "cosine_spearman": 0.6197864969896985, + "euclidean_pearson": 0.6942088223171196, + "euclidean_spearman": 0.6870127125489691, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6197864969896985, + "manhattan_pearson": 0.6990269458595981, + "manhattan_spearman": 0.6915353669156528, + "pearson": 0.6064690420220407, + "spearman": 0.6197864969896985 + }, + { + "cosine_pearson": 0.47849293571967316, + "cosine_spearman": 0.48272931892528914, + "euclidean_pearson": 0.5514231774961209, + "euclidean_spearman": 0.5427657874720075, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.48272931892528914, + "manhattan_pearson": 0.5563708024997025, + "manhattan_spearman": 0.547306668508022, + "pearson": 0.47849293571967316, + "spearman": 0.48272931892528914 + }, + { + "cosine_pearson": 0.5296312367505166, + "cosine_spearman": 0.5415773190695478, + "euclidean_pearson": 0.6074187893745651, + "euclidean_spearman": 0.6023347969959988, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.5415773190695478, + "manhattan_pearson": 0.6164718955947857, + "manhattan_spearman": 0.6112736003257329, + "pearson": 0.5296312367505166, + "spearman": 0.5415773190695478 + }, + { + "cosine_pearson": 0.5471814676359532, + "cosine_spearman": 0.5599694424985604, + "euclidean_pearson": 0.6252995994474644, + "euclidean_spearman": 0.6210724060203441, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ], + "main_score": 0.5599694424985604, + "manhattan_pearson": 0.6312980999587994, + "manhattan_spearman": 0.6269701559513498, + "pearson": 0.5471814676359532, + "spearman": 0.5599694424985604 + }, + { + "cosine_pearson": 0.4535418234458108, + "cosine_spearman": 0.4697046074838273, + "euclidean_pearson": 0.5456851627302479, + "euclidean_spearman": 0.5362763883603435, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ], + "main_score": 0.4697046074838273, + "manhattan_pearson": 0.5555002671807903, + "manhattan_spearman": 0.5443832355991821, + "pearson": 0.4535418234458108, + "spearman": 0.4697046074838273 + }, + { + "cosine_pearson": 0.4444109218978589, + "cosine_spearman": 0.45599958771514254, + "euclidean_pearson": 0.5267559388120012, + "euclidean_spearman": 0.5176849288935731, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ], + "main_score": 0.45599958771514254, + "manhattan_pearson": 0.5308716107005126, + "manhattan_spearman": 0.5222130897679905, + "pearson": 0.4444109218978589, + "spearman": 0.45599958771514254 + }, + { + "cosine_pearson": 0.3063101737717443, + "cosine_spearman": 0.3367199070808206, + "euclidean_pearson": 0.3987785895110705, + "euclidean_spearman": 0.3913098100074221, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ], + "main_score": 0.3367199070808206, + "manhattan_pearson": 0.3998630367465825, + "manhattan_spearman": 0.3923105563563955, + "pearson": 0.3063101737717443, + "spearman": 0.3367199070808206 + }, + { + "cosine_pearson": 0.4632062714190491, + "cosine_spearman": 0.47564584417433386, + "euclidean_pearson": 0.5393215479135585, + "euclidean_spearman": 0.5319489388628749, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ], + "main_score": 0.47564584417433386, + "manhattan_pearson": 0.5485030654666205, + "manhattan_spearman": 0.5401675722048989, + "pearson": 0.4632062714190491, + "spearman": 0.47564584417433386 + }, + { + "cosine_pearson": 0.1800744555427423, + "cosine_spearman": 0.2183834305276247, + "euclidean_pearson": 0.25850374844167806, + "euclidean_spearman": 0.2451706003193563, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.2183834305276247, + "manhattan_pearson": 0.2605585663635114, + "manhattan_spearman": 0.24598330621017067, + "pearson": 0.1800744555427423, + "spearman": 0.2183834305276247 + }, + { + "cosine_pearson": 0.10264590141987182, + "cosine_spearman": 0.19151948112441222, + "euclidean_pearson": 0.16974507374451053, + "euclidean_spearman": 0.20056773053396862, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ], + "main_score": 0.19151948112441222, + "manhattan_pearson": 0.17342254049511108, + "manhattan_spearman": 0.20243639613422254, + "pearson": 0.10264590141987182, + "spearman": 0.19151948112441222 + } + ], + "test": [ + { + "cosine_pearson": 0.5270768320200866, + "cosine_spearman": 0.5673533844224959, + "euclidean_pearson": 0.6443058202600288, + "euclidean_spearman": 0.645019056836489, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5673533844224959, + "manhattan_pearson": 0.6530080093827622, + "manhattan_spearman": 0.6525615100978652, + "pearson": 0.5270768320200866, + "spearman": 0.5673533844224959 + }, + { + "cosine_pearson": 0.3521438273165664, + "cosine_spearman": 0.3749686752380741, + "euclidean_pearson": 0.45537539942154803, + "euclidean_spearman": 0.44106091613516796, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.3749686752380741, + "manhattan_pearson": 0.46237570766054453, + "manhattan_spearman": 0.4471662661275312, + "pearson": 0.3521438273165664, + "spearman": 0.3749686752380741 + }, + { + "cosine_pearson": 0.4096357505353805, + "cosine_spearman": 0.44184121082999916, + "euclidean_pearson": 0.51269336299542, + "euclidean_spearman": 0.5063138290175788, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.44184121082999916, + "manhattan_pearson": 0.5215447226048767, + "manhattan_spearman": 0.5134022523664801, + "pearson": 0.4096357505353805, + "spearman": 0.44184121082999916 + }, + { + "cosine_pearson": 0.4471367141662311, + "cosine_spearman": 0.4753040189693729, + "euclidean_pearson": 0.5453656618359074, + "euclidean_spearman": 0.5399611013119631, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ], + "main_score": 0.4753040189693729, + "manhattan_pearson": 0.5502662673469212, + "manhattan_spearman": 0.5436264459052782, + "pearson": 0.4471367141662311, + "spearman": 0.4753040189693729 + }, + { + "cosine_pearson": 0.316221141637704, + "cosine_spearman": 0.3688720060740888, + "euclidean_pearson": 0.4439254366261284, + "euclidean_spearman": 0.4427471555272557, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ], + "main_score": 0.3688720060740888, + "manhattan_pearson": 0.45081804481326126, + "manhattan_spearman": 0.44797869193706497, + "pearson": 0.316221141637704, + "spearman": 0.3688720060740888 + }, + { + "cosine_pearson": 0.287912292278141, + "cosine_spearman": 0.3250557807992596, + "euclidean_pearson": 0.39531965148425147, + "euclidean_spearman": 0.3852550773539582, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ], + "main_score": 0.3250557807992596, + "manhattan_pearson": 0.4010420046702985, + "manhattan_spearman": 0.39002819841596115, + "pearson": 0.287912292278141, + "spearman": 0.3250557807992596 + }, + { + "cosine_pearson": 0.198384733263269, + "cosine_spearman": 0.23406616776235958, + "euclidean_pearson": 0.30015046447501326, + "euclidean_spearman": 0.28485294933685945, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ], + "main_score": 0.23406616776235958, + "manhattan_pearson": 0.3048021063858739, + "manhattan_spearman": 0.29003810288565623, + "pearson": 0.198384733263269, + "spearman": 0.23406616776235958 + }, + { + "cosine_pearson": 0.31449680168072336, + "cosine_spearman": 0.3548920430387672, + "euclidean_pearson": 0.41260364320400034, + "euclidean_spearman": 0.4094693280962031, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ], + "main_score": 0.3548920430387672, + "manhattan_pearson": 0.4236962018619726, + "manhattan_spearman": 0.41740681554571923, + "pearson": 0.31449680168072336, + "spearman": 0.3548920430387672 + }, + { + "cosine_pearson": 0.07318840017959619, + "cosine_spearman": 0.14060633900590963, + "euclidean_pearson": 0.15202005150811604, + "euclidean_spearman": 0.16390557574378006, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.14060633900590963, + "manhattan_pearson": 0.1551360292588062, + "manhattan_spearman": 0.16695666252040703, + "pearson": 0.07318840017959619, + "spearman": 0.14060633900590963 + }, + { + "cosine_pearson": 0.010396153007584142, + "cosine_spearman": 0.12123555650303279, + "euclidean_pearson": 0.07498960212905466, + "euclidean_spearman": 0.13153048990121405, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ], + "main_score": 0.12123555650303279, + "manhattan_pearson": 0.07621513791751623, + "manhattan_spearman": 0.1314945508379666, + "pearson": 0.010396153007584142, + "spearman": 0.12123555650303279 + } + ] + }, + "task_name": "STSBenchmarkMultilingualVisualSTS" +} \ No newline at end of file From 4efb4109ea097a97f2863a20a1b08f88bb4fccef Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 13 Sep 2024 18:57:37 +0100 Subject: [PATCH 065/154] add visual STS17 --- mteb/tasks/Image/VisualSTS/__init__.py | 1 + .../STS17MultilingualVisualSTS.py | 66 +++++++ .../STS17MultilingualVisualSTS.json | 183 ++++++++++++++++++ .../STS17MultilingualVisualSTS.json | 183 ++++++++++++++++++ .../STS17MultilingualVisualSTS.json | 183 ++++++++++++++++++ .../STS17MultilingualVisualSTS.json | 183 ++++++++++++++++++ .../STS17MultilingualVisualSTS.json | 183 ++++++++++++++++++ 7 files changed, 982 insertions(+) create mode 100644 mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py create mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/STS17MultilingualVisualSTS.json create mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/STS17MultilingualVisualSTS.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS17MultilingualVisualSTS.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS17MultilingualVisualSTS.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS17MultilingualVisualSTS.json diff --git a/mteb/tasks/Image/VisualSTS/__init__.py b/mteb/tasks/Image/VisualSTS/__init__.py index 6cb9ac8ef9..a71adb8198 100644 --- a/mteb/tasks/Image/VisualSTS/__init__.py +++ b/mteb/tasks/Image/VisualSTS/__init__.py @@ -1 +1,2 @@ +from .multilingual.STS17MultilingualVisualSTS import * from .multilingual.STSBenchmarkMultilingualVisualSTS import * diff --git a/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py b/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py new file mode 100644 index 0000000000..dc9e464dcf --- /dev/null +++ b/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS +from mteb.abstasks.MultilingualTask import MultilingualTask +from mteb.abstasks.TaskMetadata import TaskMetadata + +_LANGUAGES = { + "ko-ko": ["kor-Hang"], + "ar-ar": ["ara-Arab"], + "en-ar": ["eng-Latn", "ara-Arab"], + "en-de": ["eng-Latn", "deu-Latn"], + "en-en": ["eng-Latn"], + "en-tr": ["eng-Latn", "tur-Latn"], + "es-en": ["spa-Latn", "eng-Latn"], + "es-es": ["spa-Latn"], + "fr-en": ["fra-Latn", "eng-Latn"], + "it-en": ["ita-Latn", "eng-Latn"], + "nl-en": ["nld-Latn", "eng-Latn"], +} + +_SPLITS = ["test"] + + +class STS17MultilingualVisualSTS(AbsTaskVisualSTS, MultilingualTask): + metadata = TaskMetadata( + name="STS17MultilingualVisualSTS", + dataset={ + "path": "Pixel-Linguist/rendered-sts17", + "revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", + }, + description=( + "Semantic Textual Similarity 17 (STS-17) dataset, " + + "rendered into images." + ), + reference="https://arxiv.org/abs/2402.08183/", + type="STS", + category="i2i", + modalities=["image"], + eval_splits=_SPLITS, + eval_langs=_LANGUAGES, + main_score="cosine_spearman", + date=("2012-01-01", "2017-12-31"), + domains=["News", "Social", "Web", "Spoken", "Written"], + task_subtypes=[], + license="Not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="rendered", + bibtex_citation="""@article{xiao2024pixel, + title={Pixel Sentence Representation Learning}, + author={Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al}, + journal={arXiv preprint arXiv:2402.08183}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 10692}, + "avg_character_length": {"dev": 1.0, "test": 1.0}, + }, + ) + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 0 + metadata_dict["max_score"] = 5 + return metadata_dict diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/STS17MultilingualVisualSTS.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/STS17MultilingualVisualSTS.json new file mode 100644 index 0000000000..7d7cd4eef5 --- /dev/null +++ b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/STS17MultilingualVisualSTS.json @@ -0,0 +1,183 @@ +{ + "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", + "evaluation_time": 59.09429907798767, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.09188810319729813, + "cosine_spearman": 0.14346112848523435, + "euclidean_pearson": 0.13082342242931153, + "euclidean_spearman": 0.1468297994946006, + "hf_subset": "ko-ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.14346112848523435, + "manhattan_pearson": 0.13001422733840193, + "manhattan_spearman": 0.14598848320610913, + "pearson": 0.09188810319729813, + "spearman": 0.14346112848523435 + }, + { + "cosine_pearson": 0.27592688750203054, + "cosine_spearman": 0.32201421731334023, + "euclidean_pearson": 0.3443186066520749, + "euclidean_spearman": 0.31841020538106835, + "hf_subset": "ar-ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.32201421731334023, + "manhattan_pearson": 0.3436882295376996, + "manhattan_spearman": 0.31904272427881586, + "pearson": 0.27592688750203054, + "spearman": 0.32201421731334023 + }, + { + "cosine_pearson": 0.11597578416806917, + "cosine_spearman": 0.12294269023753306, + "euclidean_pearson": 0.09300559683007897, + "euclidean_spearman": 0.1211560040681997, + "hf_subset": "en-ar", + "languages": [ + "eng-Latn", + "ara-Arab" + ], + "main_score": 0.12294269023753306, + "manhattan_pearson": 0.09641990638923058, + "manhattan_spearman": 0.12381604457848083, + "pearson": 0.11597578416806917, + "spearman": 0.12294269023753306 + }, + { + "cosine_pearson": 0.2786109978789415, + "cosine_spearman": 0.3115303408523445, + "euclidean_pearson": 0.3040768135184991, + "euclidean_spearman": 0.31091107800856443, + "hf_subset": "en-de", + "languages": [ + "eng-Latn", + "deu-Latn" + ], + "main_score": 0.3115303408523445, + "manhattan_pearson": 0.3032684317040234, + "manhattan_spearman": 0.3090748156840843, + "pearson": 0.2786109978789415, + "spearman": 0.3115303408523445 + }, + { + "cosine_pearson": 0.3800008798157441, + "cosine_spearman": 0.42635650979604245, + "euclidean_pearson": 0.41789590060032134, + "euclidean_spearman": 0.4265325634164343, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.42635650979604245, + "manhattan_pearson": 0.41787926409683357, + "manhattan_spearman": 0.4250380296258589, + "pearson": 0.3800008798157441, + "spearman": 0.42635650979604245 + }, + { + "cosine_pearson": 0.27677049410622256, + "cosine_spearman": 0.252972959285875, + "euclidean_pearson": 0.27226752994699055, + "euclidean_spearman": 0.2526223218944939, + "hf_subset": "en-tr", + "languages": [ + "eng-Latn", + "tur-Latn" + ], + "main_score": 0.252972959285875, + "manhattan_pearson": 0.2685258863887162, + "manhattan_spearman": 0.2487845341304967, + "pearson": 0.27677049410622256, + "spearman": 0.252972959285875 + }, + { + "cosine_pearson": 0.152944923551701, + "cosine_spearman": 0.11263692422646802, + "euclidean_pearson": 0.14461819162711873, + "euclidean_spearman": 0.11089890891421848, + "hf_subset": "es-en", + "languages": [ + "spa-Latn", + "eng-Latn" + ], + "main_score": 0.11263692422646802, + "manhattan_pearson": 0.14243235664712942, + "manhattan_spearman": 0.10971716614273982, + "pearson": 0.152944923551701, + "spearman": 0.11263692422646802 + }, + { + "cosine_pearson": 0.3281526313284088, + "cosine_spearman": 0.35267614845127765, + "euclidean_pearson": 0.3866300928881876, + "euclidean_spearman": 0.35291147634567144, + "hf_subset": "es-es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.35267614845127765, + "manhattan_pearson": 0.38706458323264775, + "manhattan_spearman": 0.35592459624839945, + "pearson": 0.3281526313284088, + "spearman": 0.35267614845127765 + }, + { + "cosine_pearson": 0.18223800347528196, + "cosine_spearman": 0.20383511232761864, + "euclidean_pearson": 0.19417671758750618, + "euclidean_spearman": 0.20500559982563873, + "hf_subset": "fr-en", + "languages": [ + "fra-Latn", + "eng-Latn" + ], + "main_score": 0.20383511232761864, + "manhattan_pearson": 0.1925857152284029, + "manhattan_spearman": 0.20148375862469953, + "pearson": 0.18223800347528196, + "spearman": 0.20383511232761864 + }, + { + "cosine_pearson": 0.29206885904401914, + "cosine_spearman": 0.2793232914240653, + "euclidean_pearson": 0.29837255050901446, + "euclidean_spearman": 0.27901538978665513, + "hf_subset": "it-en", + "languages": [ + "ita-Latn", + "eng-Latn" + ], + "main_score": 0.2793232914240653, + "manhattan_pearson": 0.29586809569328343, + "manhattan_spearman": 0.2752967375515661, + "pearson": 0.29206885904401914, + "spearman": 0.2793232914240653 + }, + { + "cosine_pearson": 0.2843218379631862, + "cosine_spearman": 0.2952572972591813, + "euclidean_pearson": 0.29344751852344736, + "euclidean_spearman": 0.2932607415729033, + "hf_subset": "nl-en", + "languages": [ + "nld-Latn", + "eng-Latn" + ], + "main_score": 0.2952572972591813, + "manhattan_pearson": 0.2935374530503633, + "manhattan_spearman": 0.2921060143334773, + "pearson": 0.2843218379631862, + "spearman": 0.2952572972591813 + } + ] + }, + "task_name": "STS17MultilingualVisualSTS" +} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/STS17MultilingualVisualSTS.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/STS17MultilingualVisualSTS.json new file mode 100644 index 0000000000..c6710d98f7 --- /dev/null +++ b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/STS17MultilingualVisualSTS.json @@ -0,0 +1,183 @@ +{ + "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", + "evaluation_time": 78.60265827178955, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.09597727663103678, + "cosine_spearman": 0.1770016412213255, + "euclidean_pearson": 0.15275214888889865, + "euclidean_spearman": 0.17834721249044283, + "hf_subset": "ko-ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.1770016412213255, + "manhattan_pearson": 0.15188457950850526, + "manhattan_spearman": 0.17694304714523107, + "pearson": 0.09597727663103678, + "spearman": 0.1770016412213255 + }, + { + "cosine_pearson": 0.0922905025352107, + "cosine_spearman": 0.17678403329083517, + "euclidean_pearson": 0.19665930504248638, + "euclidean_spearman": 0.20254389410280788, + "hf_subset": "ar-ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.17678403329083517, + "manhattan_pearson": 0.20372253572199442, + "manhattan_spearman": 0.2072891316208028, + "pearson": 0.0922905025352107, + "spearman": 0.17678403329083517 + }, + { + "cosine_pearson": 0.21248211257701216, + "cosine_spearman": 0.21536141561958996, + "euclidean_pearson": 0.21166158764988635, + "euclidean_spearman": 0.2300915821447306, + "hf_subset": "en-ar", + "languages": [ + "eng-Latn", + "ara-Arab" + ], + "main_score": 0.21536141561958996, + "manhattan_pearson": 0.19881626986599565, + "manhattan_spearman": 0.21889019768380685, + "pearson": 0.21248211257701216, + "spearman": 0.21536141561958996 + }, + { + "cosine_pearson": 0.1999261135570124, + "cosine_spearman": 0.21224263369270732, + "euclidean_pearson": 0.17992560914365363, + "euclidean_spearman": 0.17445760591179463, + "hf_subset": "en-de", + "languages": [ + "eng-Latn", + "deu-Latn" + ], + "main_score": 0.21224263369270732, + "manhattan_pearson": 0.19511854135088197, + "manhattan_spearman": 0.19271029174298845, + "pearson": 0.1999261135570124, + "spearman": 0.21224263369270732 + }, + { + "cosine_pearson": 0.760796960762554, + "cosine_spearman": 0.7757114712740603, + "euclidean_pearson": 0.7805797751436612, + "euclidean_spearman": 0.7855424130677757, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7757114712740603, + "manhattan_pearson": 0.7798776498566931, + "manhattan_spearman": 0.785226823499344, + "pearson": 0.760796960762554, + "spearman": 0.7757114712740603 + }, + { + "cosine_pearson": 0.1983656963882651, + "cosine_spearman": 0.19407395142116296, + "euclidean_pearson": 0.23667518822462527, + "euclidean_spearman": 0.2165070550530714, + "hf_subset": "en-tr", + "languages": [ + "eng-Latn", + "tur-Latn" + ], + "main_score": 0.19407395142116296, + "manhattan_pearson": 0.23941732565618717, + "manhattan_spearman": 0.22726074763183676, + "pearson": 0.1983656963882651, + "spearman": 0.19407395142116296 + }, + { + "cosine_pearson": 0.26668276308314165, + "cosine_spearman": 0.28381663186603195, + "euclidean_pearson": 0.259253083456217, + "euclidean_spearman": 0.263978807455943, + "hf_subset": "es-en", + "languages": [ + "spa-Latn", + "eng-Latn" + ], + "main_score": 0.28381663186603195, + "manhattan_pearson": 0.23501156058662798, + "manhattan_spearman": 0.24264285305871922, + "pearson": 0.26668276308314165, + "spearman": 0.28381663186603195 + }, + { + "cosine_pearson": 0.4476771003066796, + "cosine_spearman": 0.5430314210401244, + "euclidean_pearson": 0.5767962443120037, + "euclidean_spearman": 0.5685268143569289, + "hf_subset": "es-es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.5430314210401244, + "manhattan_pearson": 0.5881074786408972, + "manhattan_spearman": 0.5770066935318731, + "pearson": 0.4476771003066796, + "spearman": 0.5430314210401244 + }, + { + "cosine_pearson": 0.3357957852619192, + "cosine_spearman": 0.3419618626112132, + "euclidean_pearson": 0.33039599951923004, + "euclidean_spearman": 0.3368547700336451, + "hf_subset": "fr-en", + "languages": [ + "fra-Latn", + "eng-Latn" + ], + "main_score": 0.3419618626112132, + "manhattan_pearson": 0.34797372878650235, + "manhattan_spearman": 0.35782629267058474, + "pearson": 0.3357957852619192, + "spearman": 0.3419618626112132 + }, + { + "cosine_pearson": 0.2919124060662022, + "cosine_spearman": 0.30848592016783905, + "euclidean_pearson": 0.27358870246148104, + "euclidean_spearman": 0.28561893813755396, + "hf_subset": "it-en", + "languages": [ + "ita-Latn", + "eng-Latn" + ], + "main_score": 0.30848592016783905, + "manhattan_pearson": 0.28058310574571027, + "manhattan_spearman": 0.29789118242713974, + "pearson": 0.2919124060662022, + "spearman": 0.30848592016783905 + }, + { + "cosine_pearson": 0.2095390948752094, + "cosine_spearman": 0.1954844816520889, + "euclidean_pearson": 0.18902206597452498, + "euclidean_spearman": 0.17608168134008192, + "hf_subset": "nl-en", + "languages": [ + "nld-Latn", + "eng-Latn" + ], + "main_score": 0.1954844816520889, + "manhattan_pearson": 0.2084658256986222, + "manhattan_spearman": 0.20003458362715085, + "pearson": 0.2095390948752094, + "spearman": 0.1954844816520889 + } + ] + }, + "task_name": "STS17MultilingualVisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS17MultilingualVisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS17MultilingualVisualSTS.json new file mode 100644 index 0000000000..a7d95e63bd --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS17MultilingualVisualSTS.json @@ -0,0 +1,183 @@ +{ + "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", + "evaluation_time": 67.66748046875, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.061602218313769115, + "cosine_spearman": 0.10536378343822136, + "euclidean_pearson": 0.09492865294256914, + "euclidean_spearman": 0.1121225391929482, + "hf_subset": "ko-ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.10536378343822136, + "manhattan_pearson": 0.0937650446851091, + "manhattan_spearman": 0.11075594240825876, + "pearson": 0.061602218313769115, + "spearman": 0.10536378343822136 + }, + { + "cosine_pearson": 0.22474277017008987, + "cosine_spearman": 0.3625061930482899, + "euclidean_pearson": 0.34914286152218044, + "euclidean_spearman": 0.35956084669160554, + "hf_subset": "ar-ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.3625061930482899, + "manhattan_pearson": 0.35031626840872193, + "manhattan_spearman": 0.3618855939595577, + "pearson": 0.22474277017008987, + "spearman": 0.3625061930482899 + }, + { + "cosine_pearson": 0.13201283590883084, + "cosine_spearman": 0.1313064418882118, + "euclidean_pearson": 0.13748213800122056, + "euclidean_spearman": 0.12376607271918363, + "hf_subset": "en-ar", + "languages": [ + "eng-Latn", + "ara-Arab" + ], + "main_score": 0.1313064418882118, + "manhattan_pearson": 0.159407808813381, + "manhattan_spearman": 0.15133400989777218, + "pearson": 0.13201283590883084, + "spearman": 0.1313064418882118 + }, + { + "cosine_pearson": 0.4077979312610406, + "cosine_spearman": 0.41573294231401264, + "euclidean_pearson": 0.4486770405893728, + "euclidean_spearman": 0.46190588723590936, + "hf_subset": "en-de", + "languages": [ + "eng-Latn", + "deu-Latn" + ], + "main_score": 0.41573294231401264, + "manhattan_pearson": 0.4515925499294072, + "manhattan_spearman": 0.4601457354285419, + "pearson": 0.4077979312610406, + "spearman": 0.41573294231401264 + }, + { + "cosine_pearson": 0.7487757113443495, + "cosine_spearman": 0.7817926247120496, + "euclidean_pearson": 0.7996271438124265, + "euclidean_spearman": 0.8014141466616177, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7817926247120496, + "manhattan_pearson": 0.8076051409003571, + "manhattan_spearman": 0.8078643207886379, + "pearson": 0.7487757113443495, + "spearman": 0.7817926247120496 + }, + { + "cosine_pearson": 0.34213443280278477, + "cosine_spearman": 0.3541791366106406, + "euclidean_pearson": 0.34915230983683704, + "euclidean_spearman": 0.3607616615863365, + "hf_subset": "en-tr", + "languages": [ + "eng-Latn", + "tur-Latn" + ], + "main_score": 0.3541791366106406, + "manhattan_pearson": 0.3731936697763999, + "manhattan_spearman": 0.3784834597423862, + "pearson": 0.34213443280278477, + "spearman": 0.3541791366106406 + }, + { + "cosine_pearson": 0.20904811500829665, + "cosine_spearman": 0.24627188746430106, + "euclidean_pearson": 0.23028650420666652, + "euclidean_spearman": 0.27021390375087223, + "hf_subset": "es-en", + "languages": [ + "spa-Latn", + "eng-Latn" + ], + "main_score": 0.24627188746430106, + "manhattan_pearson": 0.283890229592864, + "manhattan_spearman": 0.3048350457527673, + "pearson": 0.20904811500829665, + "spearman": 0.24627188746430106 + }, + { + "cosine_pearson": 0.6123145532680564, + "cosine_spearman": 0.6295394162055459, + "euclidean_pearson": 0.6826256087559528, + "euclidean_spearman": 0.6501694437580297, + "hf_subset": "es-es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.6295394162055459, + "manhattan_pearson": 0.6940458973588839, + "manhattan_spearman": 0.6610441303288633, + "pearson": 0.6123145532680564, + "spearman": 0.6295394162055459 + }, + { + "cosine_pearson": 0.3960565459527028, + "cosine_spearman": 0.3872111026208792, + "euclidean_pearson": 0.40972175388780063, + "euclidean_spearman": 0.39840742096400916, + "hf_subset": "fr-en", + "languages": [ + "fra-Latn", + "eng-Latn" + ], + "main_score": 0.3872111026208792, + "manhattan_pearson": 0.42718862954472175, + "manhattan_spearman": 0.41936894929062085, + "pearson": 0.3960565459527028, + "spearman": 0.3872111026208792 + }, + { + "cosine_pearson": 0.3038988006923432, + "cosine_spearman": 0.3139631866240913, + "euclidean_pearson": 0.33027917052156686, + "euclidean_spearman": 0.3509951815614502, + "hf_subset": "it-en", + "languages": [ + "ita-Latn", + "eng-Latn" + ], + "main_score": 0.3139631866240913, + "manhattan_pearson": 0.359871587744587, + "manhattan_spearman": 0.3636218394711262, + "pearson": 0.3038988006923432, + "spearman": 0.3139631866240913 + }, + { + "cosine_pearson": 0.39175159861451403, + "cosine_spearman": 0.38628124736383135, + "euclidean_pearson": 0.4207412968225121, + "euclidean_spearman": 0.4079931177581394, + "hf_subset": "nl-en", + "languages": [ + "nld-Latn", + "eng-Latn" + ], + "main_score": 0.38628124736383135, + "manhattan_pearson": 0.44162755285128313, + "manhattan_spearman": 0.4344688146099064, + "pearson": 0.39175159861451403, + "spearman": 0.38628124736383135 + } + ] + }, + "task_name": "STS17MultilingualVisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS17MultilingualVisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS17MultilingualVisualSTS.json new file mode 100644 index 0000000000..8fe733385b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS17MultilingualVisualSTS.json @@ -0,0 +1,183 @@ +{ + "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", + "evaluation_time": 52.30275201797485, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.04942380420066665, + "cosine_spearman": 0.18100901146250903, + "euclidean_pearson": 0.10583208785240501, + "euclidean_spearman": 0.18141227558130985, + "hf_subset": "ko-ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.18100901146250903, + "manhattan_pearson": 0.10771252455967777, + "manhattan_spearman": 0.18258653950535597, + "pearson": 0.04942380420066665, + "spearman": 0.18100901146250903 + }, + { + "cosine_pearson": 0.232334349420455, + "cosine_spearman": 0.28303067418377864, + "euclidean_pearson": 0.3164072647785108, + "euclidean_spearman": 0.2888667664816579, + "hf_subset": "ar-ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.28303067418377864, + "manhattan_pearson": 0.30513106540241797, + "manhattan_spearman": 0.2763182836682906, + "pearson": 0.232334349420455, + "spearman": 0.28303067418377864 + }, + { + "cosine_pearson": 0.09559095557157546, + "cosine_spearman": 0.08247355658408094, + "euclidean_pearson": 0.08598225245654229, + "euclidean_spearman": 0.07747906144678571, + "hf_subset": "en-ar", + "languages": [ + "eng-Latn", + "ara-Arab" + ], + "main_score": 0.08247355658408094, + "manhattan_pearson": 0.07391796156253816, + "manhattan_spearman": 0.07205903670762905, + "pearson": 0.09559095557157546, + "spearman": 0.08247355658408094 + }, + { + "cosine_pearson": 0.2489932748844445, + "cosine_spearman": 0.2215127409184059, + "euclidean_pearson": 0.25256785548840116, + "euclidean_spearman": 0.20598504223777503, + "hf_subset": "en-de", + "languages": [ + "eng-Latn", + "deu-Latn" + ], + "main_score": 0.2215127409184059, + "manhattan_pearson": 0.25919799382395436, + "manhattan_spearman": 0.21472583541264484, + "pearson": 0.2489932748844445, + "spearman": 0.2215127409184059 + }, + { + "cosine_pearson": 0.7018077649163341, + "cosine_spearman": 0.710059231126493, + "euclidean_pearson": 0.7476029594905387, + "euclidean_spearman": 0.7415144009008715, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.710059231126493, + "manhattan_pearson": 0.75001718011639, + "manhattan_spearman": 0.7459299640830576, + "pearson": 0.7018077649163341, + "spearman": 0.710059231126493 + }, + { + "cosine_pearson": 0.20500049148877042, + "cosine_spearman": 0.1797047388493705, + "euclidean_pearson": 0.15313682625154354, + "euclidean_spearman": 0.11880071563811015, + "hf_subset": "en-tr", + "languages": [ + "eng-Latn", + "tur-Latn" + ], + "main_score": 0.1797047388493705, + "manhattan_pearson": 0.15802674968664282, + "manhattan_spearman": 0.12885078297440422, + "pearson": 0.20500049148877042, + "spearman": 0.1797047388493705 + }, + { + "cosine_pearson": 0.12063182375204703, + "cosine_spearman": 0.12152613636208974, + "euclidean_pearson": 0.11799595218875049, + "euclidean_spearman": 0.12272133423994501, + "hf_subset": "es-en", + "languages": [ + "spa-Latn", + "eng-Latn" + ], + "main_score": 0.12152613636208974, + "manhattan_pearson": 0.13792614785112425, + "manhattan_spearman": 0.14357713355532858, + "pearson": 0.12063182375204703, + "spearman": 0.12152613636208974 + }, + { + "cosine_pearson": 0.44396659087851953, + "cosine_spearman": 0.4756388184990541, + "euclidean_pearson": 0.5318750782307595, + "euclidean_spearman": 0.4929346496915896, + "hf_subset": "es-es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.4756388184990541, + "manhattan_pearson": 0.5387926098506205, + "manhattan_spearman": 0.5017886694568034, + "pearson": 0.44396659087851953, + "spearman": 0.4756388184990541 + }, + { + "cosine_pearson": 0.20645976694323198, + "cosine_spearman": 0.19479103027395156, + "euclidean_pearson": 0.2831839920669268, + "euclidean_spearman": 0.2661811577393992, + "hf_subset": "fr-en", + "languages": [ + "fra-Latn", + "eng-Latn" + ], + "main_score": 0.19479103027395156, + "manhattan_pearson": 0.2876400969391301, + "manhattan_spearman": 0.2737037982439158, + "pearson": 0.20645976694323198, + "spearman": 0.19479103027395156 + }, + { + "cosine_pearson": 0.23084128283452515, + "cosine_spearman": 0.2273932393567347, + "euclidean_pearson": 0.3150620968342629, + "euclidean_spearman": 0.29362092114125965, + "hf_subset": "it-en", + "languages": [ + "ita-Latn", + "eng-Latn" + ], + "main_score": 0.2273932393567347, + "manhattan_pearson": 0.3143620843939853, + "manhattan_spearman": 0.2921852000229986, + "pearson": 0.23084128283452515, + "spearman": 0.2273932393567347 + }, + { + "cosine_pearson": 0.24590072426750986, + "cosine_spearman": 0.2504966252659302, + "euclidean_pearson": 0.2797988081265482, + "euclidean_spearman": 0.25981516645706415, + "hf_subset": "nl-en", + "languages": [ + "nld-Latn", + "eng-Latn" + ], + "main_score": 0.2504966252659302, + "manhattan_pearson": 0.30580381481568664, + "manhattan_spearman": 0.2951646576903725, + "pearson": 0.24590072426750986, + "spearman": 0.2504966252659302 + } + ] + }, + "task_name": "STS17MultilingualVisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS17MultilingualVisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS17MultilingualVisualSTS.json new file mode 100644 index 0000000000..1a8e2b87dd --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS17MultilingualVisualSTS.json @@ -0,0 +1,183 @@ +{ + "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", + "evaluation_time": 157.03906059265137, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.07658049577791673, + "cosine_spearman": 0.11072720883928548, + "euclidean_pearson": 0.1078991145581461, + "euclidean_spearman": 0.11751897940369754, + "hf_subset": "ko-ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.11072720883928548, + "manhattan_pearson": 0.10677695356608186, + "manhattan_spearman": 0.11595152029180494, + "pearson": 0.07658049577791673, + "spearman": 0.11072720883928548 + }, + { + "cosine_pearson": 0.29431080534890053, + "cosine_spearman": 0.39115449274164804, + "euclidean_pearson": 0.39214676108210245, + "euclidean_spearman": 0.3865876678489144, + "hf_subset": "ar-ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.39115449274164804, + "manhattan_pearson": 0.3971512172702036, + "manhattan_spearman": 0.39399986650659113, + "pearson": 0.29431080534890053, + "spearman": 0.39115449274164804 + }, + { + "cosine_pearson": 0.21213474813505678, + "cosine_spearman": 0.18952058277809578, + "euclidean_pearson": 0.19447082478701705, + "euclidean_spearman": 0.1638631082203305, + "hf_subset": "en-ar", + "languages": [ + "eng-Latn", + "ara-Arab" + ], + "main_score": 0.18952058277809578, + "manhattan_pearson": 0.2078261530517104, + "manhattan_spearman": 0.18518725377919415, + "pearson": 0.21213474813505678, + "spearman": 0.18952058277809578 + }, + { + "cosine_pearson": 0.44553015622322445, + "cosine_spearman": 0.457078250950972, + "euclidean_pearson": 0.4972439157265751, + "euclidean_spearman": 0.5086700340533576, + "hf_subset": "en-de", + "languages": [ + "eng-Latn", + "deu-Latn" + ], + "main_score": 0.457078250950972, + "manhattan_pearson": 0.49637286566496275, + "manhattan_spearman": 0.5067761122462164, + "pearson": 0.44553015622322445, + "spearman": 0.457078250950972 + }, + { + "cosine_pearson": 0.7227717533500028, + "cosine_spearman": 0.7526100073476201, + "euclidean_pearson": 0.7691084686390968, + "euclidean_spearman": 0.7745390617932849, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7526100073476201, + "manhattan_pearson": 0.7737367747407056, + "manhattan_spearman": 0.7768901802649685, + "pearson": 0.7227717533500028, + "spearman": 0.7526100073476201 + }, + { + "cosine_pearson": 0.3868402813159765, + "cosine_spearman": 0.39701995157424336, + "euclidean_pearson": 0.40798422293969827, + "euclidean_spearman": 0.41023229143697265, + "hf_subset": "en-tr", + "languages": [ + "eng-Latn", + "tur-Latn" + ], + "main_score": 0.39701995157424336, + "manhattan_pearson": 0.41772293269264277, + "manhattan_spearman": 0.42169259775526863, + "pearson": 0.3868402813159765, + "spearman": 0.39701995157424336 + }, + { + "cosine_pearson": 0.3733755575936537, + "cosine_spearman": 0.3675712091760419, + "euclidean_pearson": 0.40908137483711915, + "euclidean_spearman": 0.3963751320439048, + "hf_subset": "es-en", + "languages": [ + "spa-Latn", + "eng-Latn" + ], + "main_score": 0.3675712091760419, + "manhattan_pearson": 0.4485532114750566, + "manhattan_spearman": 0.43086210741364434, + "pearson": 0.3733755575936537, + "spearman": 0.3675712091760419 + }, + { + "cosine_pearson": 0.685661509838329, + "cosine_spearman": 0.7010571783071657, + "euclidean_pearson": 0.7446173298221099, + "euclidean_spearman": 0.7165276802273365, + "hf_subset": "es-es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.7010571783071657, + "manhattan_pearson": 0.7504667158699667, + "manhattan_spearman": 0.7237947747962055, + "pearson": 0.685661509838329, + "spearman": 0.7010571783071657 + }, + { + "cosine_pearson": 0.4394450113967274, + "cosine_spearman": 0.4406322289098256, + "euclidean_pearson": 0.4661773590688362, + "euclidean_spearman": 0.4746951448866896, + "hf_subset": "fr-en", + "languages": [ + "fra-Latn", + "eng-Latn" + ], + "main_score": 0.4406322289098256, + "manhattan_pearson": 0.47363965221268445, + "manhattan_spearman": 0.4882408949499833, + "pearson": 0.4394450113967274, + "spearman": 0.4406322289098256 + }, + { + "cosine_pearson": 0.40548839627224753, + "cosine_spearman": 0.40166825690676916, + "euclidean_pearson": 0.4454998522066273, + "euclidean_spearman": 0.439994899428145, + "hf_subset": "it-en", + "languages": [ + "ita-Latn", + "eng-Latn" + ], + "main_score": 0.40166825690676916, + "manhattan_pearson": 0.45497014049754414, + "manhattan_spearman": 0.4528287470788499, + "pearson": 0.40548839627224753, + "spearman": 0.40166825690676916 + }, + { + "cosine_pearson": 0.4129668284215636, + "cosine_spearman": 0.41634720800262875, + "euclidean_pearson": 0.44891021370876105, + "euclidean_spearman": 0.44611833648675253, + "hf_subset": "nl-en", + "languages": [ + "nld-Latn", + "eng-Latn" + ], + "main_score": 0.41634720800262875, + "manhattan_pearson": 0.460244992998331, + "manhattan_spearman": 0.4585112812933756, + "pearson": 0.4129668284215636, + "spearman": 0.41634720800262875 + } + ] + }, + "task_name": "STS17MultilingualVisualSTS" +} \ No newline at end of file From 2aa93be190d9822f3dca03700d80162b3307fe8a Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Mon, 16 Sep 2024 00:28:02 +0100 Subject: [PATCH 066/154] add visual STS 12-16 --- mteb/tasks/Image/VisualSTS/__init__.py | 5 ++ .../Image/VisualSTS/en/STS12VisualSTS.py | 46 ++++++++++++++++++ .../Image/VisualSTS/en/STS13VisualSTS.py | 46 ++++++++++++++++++ .../Image/VisualSTS/en/STS14VisualSTS.py | 47 +++++++++++++++++++ .../Image/VisualSTS/en/STS15VisualSTS.py | 46 ++++++++++++++++++ .../Image/VisualSTS/en/STS16VisualSTS.py | 46 ++++++++++++++++++ mteb/tasks/Image/VisualSTS/en/__init__.py | 0 .../STSBenchmarkMultilingualVisualSTS.py | 2 +- .../STS12VisualSTS.json | 26 ++++++++++ .../STS13VisualSTS.json | 26 ++++++++++ .../STS14VisualSTS.json | 26 ++++++++++ .../STS15VisualSTS.json | 26 ++++++++++ .../STS16VisualSTS.json | 26 ++++++++++ .../STS12VisualSTS.json | 26 ++++++++++ .../STS13VisualSTS.json | 26 ++++++++++ .../STS14VisualSTS.json | 26 ++++++++++ .../STS15VisualSTS.json | 26 ++++++++++ .../STS16VisualSTS.json | 26 ++++++++++ .../STS12VisualSTS.json | 26 ++++++++++ .../STS13VisualSTS.json | 26 ++++++++++ .../STS14VisualSTS.json | 26 ++++++++++ .../STS15VisualSTS.json | 26 ++++++++++ .../STS16VisualSTS.json | 26 ++++++++++ 23 files changed, 627 insertions(+), 1 deletion(-) create mode 100644 mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py create mode 100644 mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py create mode 100644 mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py create mode 100644 mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py create mode 100644 mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py create mode 100644 mteb/tasks/Image/VisualSTS/en/__init__.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS12VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS13VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS14VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS15VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS16VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS12VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS13VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS14VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS15VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS16VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS12VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS13VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS14VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS15VisualSTS.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS16VisualSTS.json diff --git a/mteb/tasks/Image/VisualSTS/__init__.py b/mteb/tasks/Image/VisualSTS/__init__.py index a71adb8198..cc7823118b 100644 --- a/mteb/tasks/Image/VisualSTS/__init__.py +++ b/mteb/tasks/Image/VisualSTS/__init__.py @@ -1,2 +1,7 @@ +from .en.STS12VisualSTS import * +from .en.STS13VisualSTS import * +from .en.STS14VisualSTS import * +from .en.STS15VisualSTS import * +from .en.STS16VisualSTS import * from .multilingual.STS17MultilingualVisualSTS import * from .multilingual.STSBenchmarkMultilingualVisualSTS import * diff --git a/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py new file mode 100644 index 0000000000..1f88b8045a --- /dev/null +++ b/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata +from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS + + +class STS12VisualSTS(AbsTaskVisualSTS): + metadata = TaskMetadata( + name="STS12VisualSTS", + dataset={ + "path": "Pixel-Linguist/rendered-sts12", + "revision": "820c25edfba736f3789201b2476208cc62c2ccb9", + }, + description="SemEval-2012 Task 6." + "then rendered into images.", + reference="https://arxiv.org/abs/2402.08183/", + type="STS", + category="i2i", + modalities=["image"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cosine_spearman", + date=("2005-01-01", "2012-12-31"), + domains=["Encyclopaedic", "News", "Written"], + task_subtypes=[], + license="Not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="rendered", + bibtex_citation="""@article{xiao2024pixel, + title={Pixel Sentence Representation Learning}, + author={Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al}, + journal={arXiv preprint arXiv:2402.08183}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 5342}, + "avg_character_length": {"dev": 1.0, "test": 1.0}, + }, + ) + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 0 + metadata_dict["max_score"] = 5 + return metadata_dict diff --git a/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py new file mode 100644 index 0000000000..122a5d6d30 --- /dev/null +++ b/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata +from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS + + +class STS13VisualSTS(AbsTaskVisualSTS): + metadata = TaskMetadata( + name="STS13VisualSTS", + dataset={ + "path": "Pixel-Linguist/rendered-sts13", + "revision": "561ee9ca47ff3e4a657283c59416deca8dc169f2", + }, + description="SemEval STS 2013 dataset." + "then rendered into images.", + reference="https://arxiv.org/abs/2402.08183/", + type="STS", + category="i2i", + modalities=["image"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cosine_spearman", + date=("2012-01-01", "2012-12-31"), + domains=["Web", "News", "Non-fiction", "Written"], + task_subtypes=[], + license="Not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="rendered", + bibtex_citation="""@article{xiao2024pixel, + title={Pixel Sentence Representation Learning}, + author={Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al}, + journal={arXiv preprint arXiv:2402.08183}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 1500}, + "avg_character_length": {"dev": 1.0, "test": 1.0}, + }, + ) + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 0 + metadata_dict["max_score"] = 5 + return metadata_dict diff --git a/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py new file mode 100644 index 0000000000..cbbcc94445 --- /dev/null +++ b/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata +from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS + + +class STS14VisualSTS(AbsTaskVisualSTS): + metadata = TaskMetadata( + name="STS14VisualSTS", + dataset={ + "path": "Pixel-Linguist/rendered-sts14", + "revision": "824e95e45471024a684b901e0645579ffd9ca288", + }, + description="SemEval STS 2014 dataset. Currently only the English dataset." + + "rendered into images.", + reference="https://arxiv.org/abs/2402.08183/", + type="STS", + category="i2i", + modalities=["image"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cosine_spearman", + date=("2012-01-01", "2012-08-31"), + domains=["Blog", "Web", "Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="Not specified", + annotations_creators="derived", + dialect=[], + sample_creation="rendered", + bibtex_citation="""@article{xiao2024pixel, + title={Pixel Sentence Representation Learning}, + author={Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al}, + journal={arXiv preprint arXiv:2402.08183}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 3750}, + "avg_character_length": {"dev": 1.0, "test": 1.0}, + }, + ) + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 0 + metadata_dict["max_score"] = 5 + return metadata_dict diff --git a/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py new file mode 100644 index 0000000000..9eb99af506 --- /dev/null +++ b/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata +from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS + + +class STS15VisualSTS(AbsTaskVisualSTS): + metadata = TaskMetadata( + name="STS15VisualSTS", + dataset={ + "path": "Pixel-Linguist/rendered-sts15", + "revision": "1f8d08d9b9daac7118dfdefeb94b0aac4baf2e5f", + }, + description="SemEval STS 2015 dataset" + "rendered into images.", + reference="https://arxiv.org/abs/2402.08183/", + type="STS", + category="i2i", + modalities=["image"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cosine_spearman", + date=("2008-01-01", "2014-07-28"), + domains=["Blog", "News", "Web", "Written", "Spoken"], + task_subtypes=[], + license="Not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="rendered", + bibtex_citation="""@article{xiao2024pixel, + title={Pixel Sentence Representation Learning}, + author={Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al}, + journal={arXiv preprint arXiv:2402.08183}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 3000}, + "avg_character_length": {"dev": 1.0, "test": 1.0}, + }, + ) + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 0 + metadata_dict["max_score"] = 5 + return metadata_dict diff --git a/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py new file mode 100644 index 0000000000..7db7b4f906 --- /dev/null +++ b/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata +from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS + + +class STS16VisualSTS(AbsTaskVisualSTS): + metadata = TaskMetadata( + name="STS16VisualSTS", + dataset={ + "path": "Pixel-Linguist/rendered-sts16", + "revision": "fc354f19598af93f32c0af1b94046ffdeaacde15", + }, + description="SemEval STS 2016 dataset" + "rendered into images.", + reference="https://arxiv.org/abs/2402.08183/", + type="STS", + category="i2i", + modalities=["image"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cosine_spearman", + date=("2015-10-01", "2015-12-31"), + domains=["Blog", "Web", "Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="Not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="rendered", + bibtex_citation="""@article{xiao2024pixel, + title={Pixel Sentence Representation Learning}, + author={Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al}, + journal={arXiv preprint arXiv:2402.08183}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 1186}, + "avg_character_length": {"dev": 1.0, "test": 1.0}, + }, + ) + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 0 + metadata_dict["max_score"] = 5 + return metadata_dict diff --git a/mteb/tasks/Image/VisualSTS/en/__init__.py b/mteb/tasks/Image/VisualSTS/en/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py b/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py index 2283643f83..8cf063d059 100644 --- a/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py @@ -54,7 +54,7 @@ class STSBenchmarkMultilingualVisualSTS(AbsTaskVisualSTS, MultilingualTask): year={2024} }""", descriptive_stats={ - "n_samples": {"dev": 30000, "test": 27580}, + "n_samples": {"dev": 15000, "test": 13790}, "avg_character_length": {"dev": 1.0, "test": 1.0}, }, ) diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS12VisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS12VisualSTS.json new file mode 100644 index 0000000000..a1444b9c04 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS12VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "820c25edfba736f3789201b2476208cc62c2ccb9", + "evaluation_time": 36.103975772857666, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.59081142663465, + "cosine_spearman": 0.6381747892969383, + "euclidean_pearson": 0.6340957480798639, + "euclidean_spearman": 0.6676750629657797, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6381747892969383, + "manhattan_pearson": 0.6404929502036195, + "manhattan_spearman": 0.6719835643946208, + "pearson": 0.59081142663465, + "spearman": 0.6381747892969383 + } + ] + }, + "task_name": "STS12VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS13VisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS13VisualSTS.json new file mode 100644 index 0000000000..e4453a611c --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS13VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "561ee9ca47ff3e4a657283c59416deca8dc169f2", + "evaluation_time": 16.974658966064453, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.6075975743163279, + "cosine_spearman": 0.6325573553595002, + "euclidean_pearson": 0.6692213417308934, + "euclidean_spearman": 0.6743831446100913, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6325573553595002, + "manhattan_pearson": 0.6754706863972464, + "manhattan_spearman": 0.6786762052263963, + "pearson": 0.6075975743163279, + "spearman": 0.6325573553595002 + } + ] + }, + "task_name": "STS13VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS14VisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS14VisualSTS.json new file mode 100644 index 0000000000..cf1e991fb4 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS14VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "824e95e45471024a684b901e0645579ffd9ca288", + "evaluation_time": 40.73365497589111, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.5761138227493786, + "cosine_spearman": 0.5698651220984589, + "euclidean_pearson": 0.6271652197991896, + "euclidean_spearman": 0.6147902301712311, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5698651220984589, + "manhattan_pearson": 0.6340773461052611, + "manhattan_spearman": 0.6220267895742038, + "pearson": 0.5761138227493786, + "spearman": 0.5698651220984589 + } + ] + }, + "task_name": "STS14VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS15VisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS15VisualSTS.json new file mode 100644 index 0000000000..0d385e0c8f --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS15VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "1f8d08d9b9daac7118dfdefeb94b0aac4baf2e5f", + "evaluation_time": 33.43548011779785, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.714887829276996, + "cosine_spearman": 0.7331854831535017, + "euclidean_pearson": 0.7553837552629206, + "euclidean_spearman": 0.7613317705180044, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7331854831535017, + "manhattan_pearson": 0.7598332173640413, + "manhattan_spearman": 0.7656311405677237, + "pearson": 0.714887829276996, + "spearman": 0.7331854831535017 + } + ] + }, + "task_name": "STS15VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS16VisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS16VisualSTS.json new file mode 100644 index 0000000000..032ec34621 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS16VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fc354f19598af93f32c0af1b94046ffdeaacde15", + "evaluation_time": 13.63206934928894, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.6780449758797138, + "cosine_spearman": 0.6891106167366204, + "euclidean_pearson": 0.7192011457180031, + "euclidean_spearman": 0.7222807032346467, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6891106167366204, + "manhattan_pearson": 0.723392009670844, + "manhattan_spearman": 0.7257618725946727, + "pearson": 0.6780449758797138, + "spearman": 0.6891106167366204 + } + ] + }, + "task_name": "STS16VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS12VisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS12VisualSTS.json new file mode 100644 index 0000000000..76c2dea8f9 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS12VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "820c25edfba736f3789201b2476208cc62c2ccb9", + "evaluation_time": 24.613746643066406, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.4927841964614398, + "cosine_spearman": 0.5381410964527167, + "euclidean_pearson": 0.5376663970806768, + "euclidean_spearman": 0.5680000317402323, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5381410964527167, + "manhattan_pearson": 0.5482646153361562, + "manhattan_spearman": 0.5781763600741524, + "pearson": 0.4927841964614398, + "spearman": 0.5381410964527167 + } + ] + }, + "task_name": "STS12VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS13VisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS13VisualSTS.json new file mode 100644 index 0000000000..1bbbf70ea2 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS13VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "561ee9ca47ff3e4a657283c59416deca8dc169f2", + "evaluation_time": 12.163323640823364, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.5190223767985211, + "cosine_spearman": 0.5250273114471853, + "euclidean_pearson": 0.5926139149679672, + "euclidean_spearman": 0.5939351043456808, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5250273114471853, + "manhattan_pearson": 0.5985603216705444, + "manhattan_spearman": 0.5996096714526287, + "pearson": 0.5190223767985211, + "spearman": 0.5250273114471853 + } + ] + }, + "task_name": "STS13VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS14VisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS14VisualSTS.json new file mode 100644 index 0000000000..798c057d23 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS14VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "824e95e45471024a684b901e0645579ffd9ca288", + "evaluation_time": 29.798357725143433, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.4329815257276047, + "cosine_spearman": 0.43691481964575507, + "euclidean_pearson": 0.4900456996545259, + "euclidean_spearman": 0.48614470617248245, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.43691481964575507, + "manhattan_pearson": 0.4968577536111171, + "manhattan_spearman": 0.4933346299127321, + "pearson": 0.4329815257276047, + "spearman": 0.43691481964575507 + } + ] + }, + "task_name": "STS14VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS15VisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS15VisualSTS.json new file mode 100644 index 0000000000..a9ee924256 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS15VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "1f8d08d9b9daac7118dfdefeb94b0aac4baf2e5f", + "evaluation_time": 24.508333921432495, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.5863061376001493, + "cosine_spearman": 0.5956425557069953, + "euclidean_pearson": 0.6310648275006788, + "euclidean_spearman": 0.6394200753868249, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5956425557069953, + "manhattan_pearson": 0.6352520023800736, + "manhattan_spearman": 0.644021485552327, + "pearson": 0.5863061376001493, + "spearman": 0.5956425557069953 + } + ] + }, + "task_name": "STS15VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS16VisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS16VisualSTS.json new file mode 100644 index 0000000000..5f8705554d --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS16VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fc354f19598af93f32c0af1b94046ffdeaacde15", + "evaluation_time": 10.809600830078125, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.516953879631008, + "cosine_spearman": 0.5301267122042563, + "euclidean_pearson": 0.5692230389664796, + "euclidean_spearman": 0.5712672683254012, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5301267122042563, + "manhattan_pearson": 0.572549410410549, + "manhattan_spearman": 0.5746695336316615, + "pearson": 0.516953879631008, + "spearman": 0.5301267122042563 + } + ] + }, + "task_name": "STS16VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS12VisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS12VisualSTS.json new file mode 100644 index 0000000000..9c16b4a119 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS12VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "820c25edfba736f3789201b2476208cc62c2ccb9", + "evaluation_time": 87.90539646148682, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.4405211392977615, + "cosine_spearman": 0.5389032781902258, + "euclidean_pearson": 0.5399657862990608, + "euclidean_spearman": 0.5974928829513158, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5389032781902258, + "manhattan_pearson": 0.5436282415345026, + "manhattan_spearman": 0.5999630768327725, + "pearson": 0.4405211392977615, + "spearman": 0.5389032781902258 + } + ] + }, + "task_name": "STS12VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS13VisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS13VisualSTS.json new file mode 100644 index 0000000000..c0b48421cf --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS13VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "561ee9ca47ff3e4a657283c59416deca8dc169f2", + "evaluation_time": 42.32037687301636, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.6245913368723488, + "cosine_spearman": 0.6677606349866827, + "euclidean_pearson": 0.6979935416065853, + "euclidean_spearman": 0.7086329137634516, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6677606349866827, + "manhattan_pearson": 0.7114180697383635, + "manhattan_spearman": 0.719152354710675, + "pearson": 0.6245913368723488, + "spearman": 0.6677606349866827 + } + ] + }, + "task_name": "STS13VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS14VisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS14VisualSTS.json new file mode 100644 index 0000000000..8d36ba6127 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS14VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "824e95e45471024a684b901e0645579ffd9ca288", + "evaluation_time": 106.82659244537354, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.5453585198353686, + "cosine_spearman": 0.5598248304719118, + "euclidean_pearson": 0.6150781984784081, + "euclidean_spearman": 0.6101318963139828, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5598248304719118, + "manhattan_pearson": 0.6264295102013351, + "manhattan_spearman": 0.6204680830805613, + "pearson": 0.5453585198353686, + "spearman": 0.5598248304719118 + } + ] + }, + "task_name": "STS14VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS15VisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS15VisualSTS.json new file mode 100644 index 0000000000..007949f7e9 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS15VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "1f8d08d9b9daac7118dfdefeb94b0aac4baf2e5f", + "evaluation_time": 85.88353157043457, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.6929759616177938, + "cosine_spearman": 0.7202791619453712, + "euclidean_pearson": 0.7518673935267203, + "euclidean_spearman": 0.7550769421651201, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7202791619453712, + "manhattan_pearson": 0.756112007453974, + "manhattan_spearman": 0.7594251311725545, + "pearson": 0.6929759616177938, + "spearman": 0.7202791619453712 + } + ] + }, + "task_name": "STS15VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS16VisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS16VisualSTS.json new file mode 100644 index 0000000000..ddfcfad75d --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS16VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fc354f19598af93f32c0af1b94046ffdeaacde15", + "evaluation_time": 33.86677646636963, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.6849728559170047, + "cosine_spearman": 0.7049303888061984, + "euclidean_pearson": 0.7334630202312067, + "euclidean_spearman": 0.7352518330560981, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7049303888061984, + "manhattan_pearson": 0.7392596027406768, + "manhattan_spearman": 0.7411917628510096, + "pearson": 0.6849728559170047, + "spearman": 0.7049303888061984 + } + ] + }, + "task_name": "STS16VisualSTS" +} \ No newline at end of file From 15a511baf8fdadeb1e449fa5fdb452c58c52946f Mon Sep 17 00:00:00 2001 From: Jamie-Stirling <36764530+Jamie-Stirling@users.noreply.github.com> Date: Fri, 20 Sep 2024 14:19:59 +0100 Subject: [PATCH 067/154] [mieb] Add blip and blip2 models, and ImageNetDog15Clustering task (#1226) * wip: start adding BLIP models * add other blip variants * wip: add blip2_models.py * make lint * wip: implement blip2 wrapper * feat: add blip2 models, still mismatched names * fix: remove projections from image and text embeddings * make lint * wip: add coco BLIP2 * fix: BLIP2 better zero-shot classification without text_proj and vision_proj * tidy blip2 * add imagenet-dog-15 dataset * tidy and lint * remove unused import * add cluster_accuracy, ari and nmi to Image.ClusteringEvaluator * add imagenet-10 clustering task * add results forclip on ImageNet10Clustering and ImageNetDog15Clustering --- .../abstasks/Image/AbsTaskAny2AnyRetrieval.py | 2 +- .../Image/AbsTaskImageClassification.py | 2 +- mteb/abstasks/Image/AbsTaskImageClustering.py | 2 +- .../AbsTaskImageMultilabelClassification.py | 2 +- .../AbsTaskImageTextPairClassification.py | 2 +- .../Image/AbsTaskZeroshotClassification.py | 2 +- .../evaluators/Image/ClusteringEvaluator.py | 21 +- .../evaluators/Image/VisualSTSEvaluator.py | 6 +- mteb/models/__init__.py | 4 + mteb/models/blip2_models.py | 265 ++++++++++++++++++ mteb/models/blip_models.py | 253 +++++++++++++++++ mteb/models/instructions.py | 2 - mteb/models/ru_sentence_models.py | 2 - mteb/models/sentence_transformers_models.py | 2 - .../Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py | 3 +- .../eng/FashionIQIT2IRetrieval.py | 3 +- .../eng/HatefulMemesI2TRetrieval.py | 3 +- .../eng/HatefulMemesT2IRetrieval.py | 3 +- .../eng/InfoSeekIT2ITRetrieval.py | 3 +- .../eng/InfoSeekIT2TRetrieval.py | 3 +- .../eng/MemotionI2TRetrieval.py | 3 +- .../eng/MemotionT2IRetrieval.py | 3 +- .../eng/NIGHTSI2IRetrieval.py | 3 +- .../eng/OVENIT2ITRetrieval.py | 3 +- .../Any2AnyRetrieval/eng/OVENIT2TRetrieval.py | 3 +- .../eng/SciMMIRI2TRetrieval.py | 3 +- .../eng/SciMMIRT2IRetrieval.py | 3 +- .../eng/TUBerlinT2IRetrieval.py | 3 +- .../eng/VisualNewsI2TRetrieval.py | 3 +- .../eng/VisualNewsT2IRetrieval.py | 3 +- .../eng/WebQAT2ITRetrieval.py | 3 +- .../Any2AnyRetrieval/eng/WebQAT2TRetrieval.py | 3 +- .../multilingual/WITT2IRetrieval.py | 3 +- .../multilingual/XFlickr30kCoT2IRetrieval.py | 3 +- .../multilingual/XM3600T2IRetrieval.py | 3 +- mteb/tasks/Image/Clustering/__init__.py | 1 + mteb/tasks/Image/Clustering/eng/CIFAR.py | 3 +- mteb/tasks/Image/Clustering/eng/ImageNet.py | 86 ++++++ .../eng/BirdsnapClassification.py | 3 +- .../Image/ImageClassification/eng/CIFAR.py | 3 +- .../eng/Caltech101Classification.py | 3 +- .../eng/DTDClassification.py | 3 +- .../eng/EuroSATClassification.py | 3 +- .../eng/FER2013Classification.py | 3 +- .../eng/FGVCAircraftClassification.py | 3 +- .../eng/Food101Classification.py | 3 +- .../eng/MNISTClassification.py | 3 +- .../eng/OxfordFlowersClassification.py | 3 +- .../eng/OxfordPetsClassification.py | 3 +- .../eng/RESISC45Classification.py | 3 +- .../eng/STL10Classification.py | 3 +- .../eng/SUN397Classification.py | 3 +- .../eng/StanfordCarsClassification.py | 3 +- mteb/tasks/Image/VisualSTS/__init__.py | 2 + .../Image/VisualSTS/en/STS12VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS13VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS14VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS15VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS16VisualSTS.py | 2 +- .../ZeroshotClassification/eng/Birdsnap.py | 3 +- .../Image/ZeroshotClassification/eng/CIFAR.py | 3 +- .../ZeroshotClassification/eng/Caltech101.py | 3 +- .../Image/ZeroshotClassification/eng/DTD.py | 3 +- .../ZeroshotClassification/eng/EuroSAT.py | 3 +- .../ZeroshotClassification/eng/FER2013.py | 3 +- .../eng/FGVCAircraft.py | 3 +- .../ZeroshotClassification/eng/Food101.py | 3 +- .../Image/ZeroshotClassification/eng/MNIST.py | 3 +- .../ZeroshotClassification/eng/OxfordPets.py | 3 +- .../ZeroshotClassification/eng/RESISC45.py | 3 +- .../Image/ZeroshotClassification/eng/STL10.py | 3 +- .../ZeroshotClassification/eng/SUN397.py | 3 +- .../eng/StanfordCars.py | 3 +- .../ImageNet10Clustering.json | 23 ++ .../ImageNetDog15Clustering.json | 23 ++ 75 files changed, 742 insertions(+), 123 deletions(-) create mode 100644 mteb/models/blip2_models.py create mode 100644 mteb/models/blip_models.py create mode 100644 mteb/tasks/Image/Clustering/eng/ImageNet.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNet10Clustering.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNetDog15Clustering.json diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py index 9c5987f4b1..c640988e91 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py @@ -12,9 +12,9 @@ from datasets import Features, Value, load_dataset from PIL import Image -from ..AbsTask import AbsTask from ...evaluation.evaluators import Any2AnyRetrievalEvaluator from ...load_results.mteb_results import ScoresDict +from ..AbsTask import AbsTask logger = logging.getLogger(__name__) diff --git a/mteb/abstasks/Image/AbsTaskImageClassification.py b/mteb/abstasks/Image/AbsTaskImageClassification.py index 3a95f2bd29..715f007e10 100644 --- a/mteb/abstasks/Image/AbsTaskImageClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageClassification.py @@ -6,7 +6,6 @@ import numpy as np -from ..AbsTask import AbsTask from ...encoder_interface import Encoder from ...evaluation.evaluators import ( ImagekNNClassificationEvaluator, @@ -14,6 +13,7 @@ ImagelogRegClassificationEvaluator, ) from ...load_results.mteb_results import HFSubset, ScoresDict +from ..AbsTask import AbsTask logger = logging.getLogger(__name__) diff --git a/mteb/abstasks/Image/AbsTaskImageClustering.py b/mteb/abstasks/Image/AbsTaskImageClustering.py index 5370b16b15..3d6f7e88d2 100644 --- a/mteb/abstasks/Image/AbsTaskImageClustering.py +++ b/mteb/abstasks/Image/AbsTaskImageClustering.py @@ -5,10 +5,10 @@ from datasets import Dataset -from ..AbsTask import AbsTask from ...encoder_interface import Encoder, EncoderWithQueryCorpusEncode from ...evaluation.evaluators import ImageClusteringEvaluator from ...load_results.mteb_results import HFSubset, ScoresDict +from ..AbsTask import AbsTask logger = logging.getLogger(__name__) diff --git a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py index 5669575a18..6a0d649f10 100644 --- a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py @@ -12,9 +12,9 @@ from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import MultiLabelBinarizer -from ..AbsTask import AbsTask from ...encoder_interface import Encoder from ...load_results.mteb_results import HFSubset, ScoresDict +from ..AbsTask import AbsTask logger = logging.getLogger(__name__) diff --git a/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py index 492de11659..81f3094b5c 100644 --- a/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py @@ -6,10 +6,10 @@ from datasets import Dataset from tqdm import tqdm -from ..AbsTask import AbsTask from ...encoder_interface import Encoder, EncoderWithQueryCorpusEncode from ...evaluation.evaluators import ImageTextPairClassificationEvaluator from ...load_results.mteb_results import ScoresDict +from ..AbsTask import AbsTask logger = logging.getLogger(__name__) diff --git a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py index 9d5a55e235..4f23bb46b4 100644 --- a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py +++ b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py @@ -5,10 +5,10 @@ from datasets import Dataset -from ..AbsTask import AbsTask from ...encoder_interface import Encoder, EncoderWithQueryCorpusEncode from ...evaluation.evaluators import ZeroshotClassificationEvaluator from ...load_results.mteb_results import ScoresDict +from ..AbsTask import AbsTask logger = logging.getLogger(__name__) diff --git a/mteb/evaluation/evaluators/Image/ClusteringEvaluator.py b/mteb/evaluation/evaluators/Image/ClusteringEvaluator.py index b006470416..f53befe8ef 100644 --- a/mteb/evaluation/evaluators/Image/ClusteringEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ClusteringEvaluator.py @@ -6,6 +6,7 @@ import sklearn import sklearn.cluster from PIL import Image +from scipy.optimize import linear_sum_assignment from sklearn import metrics from mteb.encoder_interface import Encoder @@ -53,6 +54,24 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): logger.info("Evaluating...") v_measure = metrics.cluster.v_measure_score(self.labels, cluster_assignment) + nmi = metrics.cluster.normalized_mutual_info_score( + self.labels, cluster_assignment + ) + ari = metrics.cluster.adjusted_rand_score(self.labels, cluster_assignment) + accuracy = metrics.accuracy_score(self.labels, cluster_assignment) - return {"v_measure": v_measure, "accuracy": accuracy} + matrix = metrics.confusion_matrix(self.labels, cluster_assignment) + + # get linear sum assignment + row_ind, col_ind = linear_sum_assignment(matrix, maximize=True) + total_correct = matrix[row_ind, col_ind].sum() + clustering_accuracy = total_correct / len(self.labels) + + return { + "v_measure": v_measure, + "accuracy": accuracy, + "nmi": nmi, + "ari": ari, + "cluster_accuracy": clustering_accuracy, + } diff --git a/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py b/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py index d47e060e75..a442eb6a9a 100644 --- a/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py +++ b/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py @@ -1,18 +1,18 @@ from __future__ import annotations import logging -from typing import Any +import math import os +from typing import Any import numpy as np +import torch from scipy.stats import pearsonr, spearmanr from sklearn.metrics.pairwise import ( paired_cosine_distances, paired_euclidean_distances, paired_manhattan_distances, ) -import math -import torch from torch.utils.data import DataLoader from torchvision import transforms diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index 8e96542925..eabe5a2d3f 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -10,6 +10,8 @@ from mteb.models import ( align_models, bge_models, + blip2_models, + blip_models, bm25, clip_models, cohere_models, @@ -130,6 +132,8 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe model_modules = [ align_models, bge_models, + blip_models, + blip2_models, bm25, cohere_models, dino_models, diff --git a/mteb/models/blip2_models.py b/mteb/models/blip2_models.py new file mode 100644 index 0000000000..cb289b3f96 --- /dev/null +++ b/mteb/models/blip2_models.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import Blip2Processor + +from mteb.model_meta import ModelMeta + + +def blip2_loader(**kwargs): + try: # a temporal fix for the dependency issues of vista models. + from lavis.models.blip2_models.blip2_image_text_matching import ( + Blip2ITM, + ) + except ImportError: + raise ImportError( + "Please install `pip install salesforce-lavis` to use BLIP-2 models." + ) + + class BLIP2ModelWrapper: + def __init__( + self, + model_name: str, + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + model_type = "coco" if "coco" in model_name else "pretrain" + self.model = Blip2ITM.from_pretrained(model_type).to(self.device).float() + # print numbr of parameters + print( + f"Number of parameters: {sum(p.numel() for p in self.model.parameters())}" + ) + self.processor = Blip2Processor.from_pretrained(model_name) + + def preprocess( + self, + texts: list[str], + images: list[Image.Image], + ): + return self.processor( + text=texts, images=images, return_tensors="pt", padding=True + ) + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + text_tokens = self.model.tokenizer( + batch_texts, + padding="max_length", + truncation=True, + max_length=self.model.max_txt_len, + return_tensors="pt", + ).to(self.device) + text_outputs = self.model.forward_text(text_tokens) + # text_outputs = normalize(self.model.text_proj(text_outputs)) + all_text_embeddings.append(text_outputs.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + with torch.no_grad(): + for batch in tqdm(images): + inputs = self.processor( + images=batch, return_tensors="pt", padding=True + ) + image_outputs = self.model.forward_image( + inputs["pixel_values"].to(self.device) + ) + image_outputs = image_outputs[0][:, 0, :] + # image_outputs = normalize(self.model.vision_proj(image_outputs), dim=-1) + all_image_embeddings.append(image_outputs.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + inputs = self.processor( + images=batch_images, return_tensors="pt", padding=True + )["pixel_values"].to(self.device) + image_outputs = self.model.forward_image(inputs) + image_outputs = image_outputs[0][:, 0, :] + # image_outputs = normalize(self.model.vision_proj(image_outputs), dim=-1) + all_image_embeddings.append(image_outputs.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def get_multimodal_embeddings(self, texts, images, batch_size=32): + all_multimodal_embeddings = [] + + with torch.no_grad(): + if isinstance(images, DataLoader): + # check dataloader batch size is the same as batch size + if images.batch_size != batch_size: + raise ValueError( + "Image DataLoader batch size must be the same as the given batch size: " + + str(batch_size) + ) + for batch_images, i in tqdm( + zip(images, range(0, len(texts), batch_size)) + ): + batch_texts = texts[i : i + batch_size] + + image_inputs = self.processor( + images=batch_images, return_tensors="pt", padding=True + )["pixel_values"].to(self.device) + multimodal_outputs = self.model.extract_features( + {"text_input": batch_texts, "image": image_inputs} + ).multimodal_embeds[:, 0, :] + + # multimodal_outputs = normalize(self.model.text_proj(multimodal_outputs), dim=-1) + + all_multimodal_embeddings.append(multimodal_outputs.cpu()) + else: + for i in tqdm(range(0, len(texts), batch_size)): + batch_images = images[i : i + batch_size] + batch_texts = texts[i : i + batch_size] + + image_inputs = self.processor( + images=batch_images, return_tensors="pt", padding=True + )["pixel_values"].to(self.device) + multimodal_outputs = self.model.extract_features( + {"text_input": batch_texts, "image": image_inputs} + ).multimodal_embeds[:, 0, :] + + # multimodal_outputs = normalize(self.model.text_proj(multimodal_outputs), dim=-1) + + all_multimodal_embeddings.append(multimodal_outputs.cpu()) + + return torch.cat(all_multimodal_embeddings, dim=0) + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm( + dim=-1, keepdim=True + ) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + # TODO: find out if BLIP has a prescribed way of fusing text and image embeddings + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + elif fusion_mode == "multimodal": + fused_embeddings = self.get_multimodal_embeddings( + texts, images, batch_size + ) + else: + # to do: add other fusion mode + raise ValueError( + f"fusion mode {fusion_mode} hasn't been implemented" + ) + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + return BLIP2ModelWrapper(**kwargs) + + +blip2_opt_2_7b = ModelMeta( + loader=partial( + blip2_loader, + model_name="Salesforce/blip2-opt-2.7b", + ), + name="Salesforce/blip2-opt-2.7b", + languages=["eng_Latn"], + open_source=True, + revision="51572668da0eb669e01a189dc22abe6088589a24", + release_date="2024-03-22", +) + +blip2_opt_6_7b_coco = ModelMeta( + loader=partial( + blip2_loader, + model_name="Salesforce/blip2-opt-6.7b-coco", + ), + name="Salesforce/blip2-opt-6.7b-coco", + languages=["eng_Latn"], + open_source=True, + revision="0d580de59320a25a4d2c386387bcef310d5f286e", + release_date="2024-03-31", +) + + +if __name__ == "__main__": + import mteb + + mdl = mteb.get_model(blip2_opt_2_7b.name, blip2_opt_2_7b.revision, device="cpu") + emb = mdl.get_text_embeddings(["Hello, world!"]) + emb2 = mdl.get_text_embeddings(["Hello there, world!"]) + emb3 = mdl.get_text_embeddings(["Goodbye, person!"]) + + sim = torch.nn.functional.cosine_similarity(emb, emb2) + print(sim) + + sim = torch.nn.functional.cosine_similarity(emb, emb3) + print(sim) + + cat_img = Image.open("cat.jpg") + cat_text = "An image of a cat" + + multi_cat_emb = mdl.get_fused_embeddings( + ["A photo of an animal"], [cat_img], fusion_mode="multimodal" + ) + multi_conflicting_emb = mdl.get_fused_embeddings( + ["A photo of a dog"], [cat_img], fusion_mode="multimodal" + ) + image_cat_emb = mdl.get_image_embeddings([cat_img]) + text_cat_emb = mdl.get_text_embeddings(["An photo of a cat"]) + text_dog_emb = mdl.get_text_embeddings(["An image of a dog"]) + + print(multi_cat_emb.shape) + + sim1 = torch.nn.functional.cosine_similarity(image_cat_emb, text_cat_emb) + sim2 = torch.nn.functional.cosine_similarity(image_cat_emb, text_dog_emb) + sim3 = torch.nn.functional.cosine_similarity(multi_cat_emb, text_cat_emb) + sim4 = torch.nn.functional.cosine_similarity(multi_cat_emb, text_dog_emb) + sim5 = torch.nn.functional.cosine_similarity(multi_conflicting_emb, text_cat_emb) + + print(sim1, sim2) + + print(sim3, sim4, sim5) diff --git a/mteb/models/blip_models.py b/mteb/models/blip_models.py new file mode 100644 index 0000000000..dff6014246 --- /dev/null +++ b/mteb/models/blip_models.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.nn.functional import normalize +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import BlipForImageTextRetrieval, BlipProcessor + +from mteb.model_meta import ModelMeta + + +class BLIPModelWrapper: + def __init__( + self, + model_name: str, + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + self.model = BlipForImageTextRetrieval.from_pretrained(model_name).to( + self.device + ) + self.processor = BlipProcessor.from_pretrained(model_name) + + def preprocess( + self, + texts: list[str], + images: list[Image.Image], + ): + return self.processor( + text=texts, images=images, return_tensors="pt", padding=True + ) + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + inputs = self.processor( + text=batch_texts, return_tensors="pt", padding=True, truncation=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + # different to CLIPModelWrapper: text_encoder instead of get_text_features and apply projection and normalization + text_outputs = self.model.text_encoder(**inputs) + text_outputs = text_outputs[0] + text_outputs = normalize( + self.model.text_proj(text_outputs[:, 0, :]), dim=-1 + ) + all_text_embeddings.append(text_outputs.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + with torch.no_grad(): + for batch in tqdm(images): + inputs = self.processor( + images=batch, return_tensors="pt", padding=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model.vision_model(**inputs) + image_outputs = image_outputs[0] + image_outputs = normalize( + self.model.vision_proj(image_outputs[:, 0, :]), dim=-1 + ) + all_image_embeddings.append(image_outputs.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + inputs = self.processor( + images=batch_images, return_tensors="pt", padding=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model.get_image_features(**inputs) + image_outputs = self.model.vision_model(**inputs) + image_outputs = image_outputs[0] + image_outputs = normalize( + self.model.vision_proj(image_outputs[:, 0, :]), dim=-1 + ) + all_image_embeddings.append(image_outputs.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + else: + # to do: add other fusion mode + raise ValueError(f"fusion mode {fusion_mode} hasn't been implemented") + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + +# in descending order of usage (downloads from huggingface) +blip_image_captioning_large = ModelMeta( + loader=partial( + BLIPModelWrapper, + model_name="Salesforce/blip-image-captioning-large", + ), + name="Salesforce/blip-image-captioning-large", + languages=["eng_Latn"], + open_source=True, + revision="2227ac38c9f16105cb0412e7cab4759978a8fd90", + release_date="2023-12-07", +) + +blip_image_captioning_base = ModelMeta( + loader=partial( + BLIPModelWrapper, + model_name="Salesforce/blip-image-captioning-base", + ), + name="Salesforce/blip-image-captioning-base", + languages=["eng_Latn"], + open_source=True, + revision="89b09ea1789f7addf2f6d6f0dfc4ce10ab58ef84", + release_date="2023-08-01", +) + + +blip_vqa_base = ModelMeta( + loader=partial( + BLIPModelWrapper, + model_name="Salesforce/blip-vqa-base", + ), + name="Salesforce/blip-vqa-base", + languages=["eng_Latn"], + open_source=True, + revision="c7df8e7cd7aa2ee9af18f56e2b29e59a92651b64", + release_date="2023-12-07", +) + +blip_vqa_capfilt_large = ModelMeta( + loader=partial( + BLIPModelWrapper, + model_name="Salesforce/blip-vqa-capfilt-large", + ), + name="Salesforce/blip-vqa-capfilt-large", + languages=["eng_Latn"], + open_source=True, + revision="e53f95265aeab69013fabb5380500ab984adbbb4", + release_date="2023-01-22", +) + +blip_itm_base_coco = ModelMeta( + loader=partial( + BLIPModelWrapper, + model_name="Salesforce/blip-itm-base-coco", + ), + name="Salesforce/blip-itm-base-coco", + languages=["eng_Latn"], + open_source=True, + revision="7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f", + release_date="2023-08-01", +) + +blip_itm_large_coco = ModelMeta( + loader=partial( + BLIPModelWrapper, + model_name="Salesforce/blip-itm-large-coco", + ), + name="Salesforce/blip-itm-large-coco", + languages=["eng_Latn"], + open_source=True, + revision="fef05cafc05298067cbbca00b125749394a77a6f", + release_date="2023-08-01", +) + +blip_itm_base_flickr = ModelMeta( + loader=partial( + BLIPModelWrapper, + model_name="Salesforce/blip-itm-base-flickr", + ), + name="Salesforce/blip-itm-base-flickr", + languages=["eng_Latn"], + open_source=True, + revision="1de29e660d91ae1786c1876212ea805a22eab251", + release_date="2023-08-01", +) + +blip_itm_large_flickr = ModelMeta( + loader=partial( + BLIPModelWrapper, + model_name="Salesforce/blip-itm-large-flickr", + ), + name="Salesforce/blip-itm-large-flickr", + languages=["eng_Latn"], + open_source=True, + revision="bda12e6506758f54261b5ab174b2c55a3ba143fb", + release_date="2023-08-01", +) + + +if __name__ == "__main__": + import mteb + + mdl = mteb.get_model(blip_itm_base_coco.name, blip_itm_base_coco.revision) + emb = mdl.get_text_embeddings(["Hello, world!"]) + emb2 = mdl.get_text_embeddings(["Hello there, world!"]) + emb3 = mdl.get_text_embeddings(["Goodbye, person!"]) + + sim = torch.nn.functional.cosine_similarity(emb, emb2) + print(sim) + + sim = torch.nn.functional.cosine_similarity(emb, emb3) + print(sim) diff --git a/mteb/models/instructions.py b/mteb/models/instructions.py index 99054e41d7..4a31f8da02 100644 --- a/mteb/models/instructions.py +++ b/mteb/models/instructions.py @@ -2,8 +2,6 @@ from __future__ import annotations -from __future__ import annotations - import mteb # Prompts from diff --git a/mteb/models/ru_sentence_models.py b/mteb/models/ru_sentence_models.py index cffe7f7be4..30214c21f2 100644 --- a/mteb/models/ru_sentence_models.py +++ b/mteb/models/ru_sentence_models.py @@ -2,8 +2,6 @@ from __future__ import annotations -from __future__ import annotations - from functools import partial from mteb.model_meta import ModelMeta diff --git a/mteb/models/sentence_transformers_models.py b/mteb/models/sentence_transformers_models.py index 33ea17b165..a3603d9eb3 100644 --- a/mteb/models/sentence_transformers_models.py +++ b/mteb/models/sentence_transformers_models.py @@ -2,8 +2,6 @@ from __future__ import annotations -from __future__ import annotations - from mteb.model_meta import ModelMeta paraphrase_langs = [ diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py index eb65b82e79..417e5d6caa 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class CIRRIT2IRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py index b336549557..a58ed15dd5 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class FashionIQIT2IRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py index 1fcf9f0cb9..817ea1c674 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py @@ -2,9 +2,8 @@ from datasets import concatenate_datasets, load_dataset -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py index 5b2b9bcaef..0a55e446ed 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py @@ -2,9 +2,8 @@ from datasets import concatenate_datasets, load_dataset -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py index 5029c51ec9..f7cb041bcb 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class InfoSeekIT2ITRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py index cd08aa77b2..cc2b23ea88 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class InfoSeekIT2TRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py index af68e278b9..9247a12f88 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py @@ -2,9 +2,8 @@ from datasets import concatenate_datasets, load_dataset -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py index 7478ddddeb..f214bd2ea5 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py @@ -2,9 +2,8 @@ from datasets import concatenate_datasets, load_dataset -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py index 82dcf0894a..73d3f7c280 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class NIGHTSI2IRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py index 51d031241c..0f53eb7e6a 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class OVENIT2ITRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py index cfa07350ba..3df5b92625 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class OVENIT2TRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py index fa0f5b5707..eb2c24aeb2 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py @@ -2,9 +2,8 @@ from datasets import load_dataset -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py index c6004e7840..e92bd637f5 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py @@ -2,9 +2,8 @@ from datasets import load_dataset -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py index 018f708ce5..7c7bddfe4c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class TUBerlinT2IRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py index c1f1b306ca..2de1713097 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class VisualNewsI2TRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py index 7457f00d03..091d7a7f00 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class VisualNewsT2IRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py index 7086c1c205..50725b79b9 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class WebQAT2ITRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py index 6a4efb261a..14c9c02148 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class WebQAT2TRetrieval(AbsTaskAny2AnyRetrieval): diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py index a0395594a2..5de06b937f 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py @@ -2,10 +2,9 @@ from datasets import Dataset, DatasetDict, load_dataset -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval from mteb.abstasks.MultilingualTask import MultilingualTask +from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { "ar": ["ara-Arab"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py index 92f4a9c2c0..65c886f314 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py @@ -2,10 +2,9 @@ from datasets import DatasetDict, load_dataset -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval from mteb.abstasks.MultilingualTask import MultilingualTask +from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { "de": ["deu-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py index 8cb7f0e9d1..687c9f0446 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py @@ -2,10 +2,9 @@ from datasets import Dataset, DatasetDict, load_dataset -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval from mteb.abstasks.MultilingualTask import MultilingualTask +from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { "ar": ["ara-Arab"], diff --git a/mteb/tasks/Image/Clustering/__init__.py b/mteb/tasks/Image/Clustering/__init__.py index fd9a71ec19..804870ebeb 100644 --- a/mteb/tasks/Image/Clustering/__init__.py +++ b/mteb/tasks/Image/Clustering/__init__.py @@ -1,4 +1,5 @@ from __future__ import annotations from .eng.CIFAR import * +from .eng.ImageNet import * from .eng.TinyImageNet import * diff --git a/mteb/tasks/Image/Clustering/eng/CIFAR.py b/mteb/tasks/Image/Clustering/eng/CIFAR.py index 01b493233c..e7f7a1d633 100644 --- a/mteb/tasks/Image/Clustering/eng/CIFAR.py +++ b/mteb/tasks/Image/Clustering/eng/CIFAR.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClustering import AbsTaskImageClustering +from mteb.abstasks.TaskMetadata import TaskMetadata class CIFAR10Clustering(AbsTaskImageClustering): diff --git a/mteb/tasks/Image/Clustering/eng/ImageNet.py b/mteb/tasks/Image/Clustering/eng/ImageNet.py new file mode 100644 index 0000000000..dcf8587322 --- /dev/null +++ b/mteb/tasks/Image/Clustering/eng/ImageNet.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskImageClustering import AbsTaskImageClustering +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class ImageNetDog15Clustering(AbsTaskImageClustering): + metadata = TaskMetadata( + name="ImageNetDog15Clustering", + description="Clustering images from a 15-class dogs-only subset of the dog classes in ImageNet.", + reference="http://vision.stanford.edu/aditya86/ImageNetDogs/main.html", + dataset={ + "path": "JamieSJS/imagenet-dog-15", + "revision": "bfb6ad3b2109d26c9daddf14f98d315daa35ee72", + }, + type="Clustering", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-06-20", "2009-06-20"), # Conference date + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation=""" @INPROCEEDINGS{5206848, + author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Kai Li and Li Fei-Fei}, + booktitle={2009 IEEE Conference on Computer Vision and Pattern Recognition}, + title={ImageNet: A large-scale hierarchical image database}, + year={2009}, + volume={}, + number={}, + pages={248-255}, + keywords={Large-scale systems;Image databases;Explosions;Internet;Robustness;Information retrieval;Image retrieval;Multimedia databases;Ontologies;Spine}, + doi={10.1109/CVPR.2009.5206848}} + """, + descriptive_stats={ + "n_samples": {"test": 1076, "train": 1500}, + # "avg_character_length": {"test": 431.4}, + }, + ) + + +class ImageNet10Clustering(AbsTaskImageClustering): + metadata = TaskMetadata( + name="ImageNet10Clustering", + description="Clustering images from an 10-class subset of ImageNet which are generally easy to distinguish.", + reference="https://www.kaggle.com/datasets/liusha249/imagenet10", + dataset={ + "path": "JamieSJS/imagenet-10", + "revision": "88f8a6d47c257895094c5ad81e67ba751771fc99", + }, + type="Clustering", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-06-20", "2009-06-20"), # Conference date + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation=""" @INPROCEEDINGS{5206848, + author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Kai Li and Li Fei-Fei}, + booktitle={2009 IEEE Conference on Computer Vision and Pattern Recognition}, + title={ImageNet: A large-scale hierarchical image database}, + year={2009}, + volume={}, + number={}, + pages={248-255}, + keywords={Large-scale systems;Image databases;Explosions;Internet;Robustness;Information retrieval;Image retrieval;Multimedia databases;Ontologies;Spine}, + doi={10.1109/CVPR.2009.5206848}} + """, + descriptive_stats={ + "n_samples": {"test": 13000}, + # "avg_character_length": {"test": 431.4}, + }, + ) diff --git a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py index a104d51e13..38016e5e79 100644 --- a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class BirdsnapClassification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py index 75e3cdf6fc..9b4f45e387 100644 --- a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py +++ b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class CIFAR10Classification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py index 0175cd8663..fe62f955b3 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class Caltech101Classification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py index 2f921e5587..25f6ba0401 100644 --- a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class DTDClassification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py index b849d93c0b..4930c13d1b 100644 --- a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class EuroSATClassification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py index 2081683154..9db8b017f7 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class FER2013Classification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py index bb09f32426..9b061e6dd1 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class FGVCAircraftClassification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py index 533b2c2145..04389db8f1 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class Food101Classification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py index 82de6fab16..f3831abdb4 100644 --- a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class MNISTClassification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py index dce55d9362..c0a10de48d 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class OxfordFlowersClassification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py index 0277098d64..cf537648ed 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class OxfordPetsClassification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py index e883db4c6e..afbc8fe1da 100644 --- a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class RESISC45Classification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py index 9b9fcf3ef4..9531e1c1f6 100644 --- a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class STL10Classification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py index 414f3560e6..eef0ccbfcb 100644 --- a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class SUN397Classification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py index 1fa4f64af2..e4561b2165 100644 --- a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py @@ -1,8 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.TaskMetadata import TaskMetadata class StanfordCarsClassification(AbsTaskImageClassification): diff --git a/mteb/tasks/Image/VisualSTS/__init__.py b/mteb/tasks/Image/VisualSTS/__init__.py index cc7823118b..eb785d5d85 100644 --- a/mteb/tasks/Image/VisualSTS/__init__.py +++ b/mteb/tasks/Image/VisualSTS/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from .en.STS12VisualSTS import * from .en.STS13VisualSTS import * from .en.STS14VisualSTS import * diff --git a/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py index 1f88b8045a..8d78bb7238 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py @@ -1,7 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS +from mteb.abstasks.TaskMetadata import TaskMetadata class STS12VisualSTS(AbsTaskVisualSTS): diff --git a/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py index 122a5d6d30..1b02248d35 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py @@ -1,7 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS +from mteb.abstasks.TaskMetadata import TaskMetadata class STS13VisualSTS(AbsTaskVisualSTS): diff --git a/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py index cbbcc94445..a427fdae0b 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py @@ -1,7 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS +from mteb.abstasks.TaskMetadata import TaskMetadata class STS14VisualSTS(AbsTaskVisualSTS): diff --git a/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py index 9eb99af506..12c9a74c81 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py @@ -1,7 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS +from mteb.abstasks.TaskMetadata import TaskMetadata class STS15VisualSTS(AbsTaskVisualSTS): diff --git a/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py index 7db7b4f906..ae1e2900dd 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py @@ -1,7 +1,7 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS +from mteb.abstasks.TaskMetadata import TaskMetadata class STS16VisualSTS(AbsTaskVisualSTS): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py b/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py index 9273b66add..ed31e3f89f 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class BirdsnapClassification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py b/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py index 517bf565cc..81103a0f1d 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class CIFAR10ZeroShotClassification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py b/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py index f07c423939..ab7ca141cb 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class Caltech101Classification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/DTD.py b/mteb/tasks/Image/ZeroshotClassification/eng/DTD.py index caea933534..27ef0a6f3d 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/DTD.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/DTD.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class DTDClassification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py b/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py index 275487580d..de6fb4c434 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class EuroSATClassification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py b/mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py index febbb27e5e..9cfa0dd3e9 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class FER2013Classification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py b/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py index 833afde477..c15e0b6d4b 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class FGVCAircraftClassification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Food101.py b/mteb/tasks/Image/ZeroshotClassification/eng/Food101.py index a2b93c2471..fd073ac412 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Food101.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Food101.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class Food101Classification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py b/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py index f343cb9211..253fa938ac 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class MNISTClassification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py b/mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py index 2145fe8bff..3da580af1b 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class OxfordPetsClassification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py b/mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py index 7ba9824455..d6fb98ba6c 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class RESISC45Classification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/STL10.py b/mteb/tasks/Image/ZeroshotClassification/eng/STL10.py index 11c53d5032..8b0f42d08d 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/STL10.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/STL10.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class STL10Classification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py b/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py index c3e67879b0..64252584b8 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class SUN397Classification(AbsTaskZeroshotClassification): diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py b/mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py index 0e881b65f0..c8cc639a4e 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py @@ -1,10 +1,9 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( AbsTaskZeroshotClassification, ) +from mteb.abstasks.TaskMetadata import TaskMetadata class StanfordCarsClassification(AbsTaskZeroshotClassification): diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNet10Clustering.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNet10Clustering.json new file mode 100644 index 0000000000..d502635992 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNet10Clustering.json @@ -0,0 +1,23 @@ +{ + "dataset_revision": "88f8a6d47c257895094c5ad81e67ba751771fc99", + "evaluation_time": 33.32936453819275, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.1993076923076923, + "ari": 0.9672782515730578, + "cluster_accuracy": 0.985, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.1993076923076923, + "nmi": 0.9644473066207006, + "v_measure": 0.9644473066207006 + } + ] + }, + "task_name": "ImageNet10Clustering" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNetDog15Clustering.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNetDog15Clustering.json new file mode 100644 index 0000000000..fe53c8ed7e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNetDog15Clustering.json @@ -0,0 +1,23 @@ +{ + "dataset_revision": "bfb6ad3b2109d26c9daddf14f98d315daa35ee72", + "evaluation_time": 4.18316650390625, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.026022304832713755, + "ari": 0.36465670607270784, + "cluster_accuracy": 0.4656133828996282, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.026022304832713755, + "nmi": 0.5160500208664386, + "v_measure": 0.5160500208664386 + } + ] + }, + "task_name": "ImageNetDog15Clustering" +} \ No newline at end of file From 99e631f0f5b4b648d4fc4431cd3fed685452c9d3 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Mon, 23 Sep 2024 05:28:24 +0800 Subject: [PATCH 068/154] [mieb] add 3 compositionality evaluation tasks (#1229) * linting & update unavailable dataset path * add aro visual relation&attribution; sugarcrepe * correct reference --- .../AbsTaskImageTextPairClassification.py | 1 - mteb/models/blip2_models.py | 2 +- .../fil/FilipinoHateSpeechClassification.py | 4 +- .../AROVisualAttribution.py | 49 +++++++++++++++ .../AROVisualRelation.py | 49 +++++++++++++++ .../ImageTextPairClassification/SugarCrepe.py | 61 +++++++++++++++++++ .../ImageTextPairClassification/__init__.py | 3 + .../AROVisualAttribution.json | 21 +++++++ .../AROVisualRelation.json | 21 +++++++ .../SugarCrepe.json | 21 +++++++ .../AROVisualAttribution.json | 21 +++++++ .../AROVisualRelation.json | 21 +++++++ .../SugarCrepe.json | 21 +++++++ .../AROVisualAttribution.json | 21 +++++++ .../AROVisualRelation.json | 21 +++++++ .../MNIST.json | 48 +++++++++++++++ .../SugarCrepe.json | 21 +++++++ 17 files changed, 402 insertions(+), 4 deletions(-) create mode 100644 mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py create mode 100644 mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py create mode 100644 mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualAttribution.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualRelation.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SugarCrepe.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualAttribution.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualRelation.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SugarCrepe.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualAttribution.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualRelation.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MNIST.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SugarCrepe.json diff --git a/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py index 81f3094b5c..c6d4a6a2de 100644 --- a/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py @@ -62,7 +62,6 @@ def _evaluate_subset( ) -> ScoresDict: images = self._preprocess_column(dataset, self.images_column_names) texts = self._preprocess_column(dataset, self.texts_column_names) - evaluator = ImageTextPairClassificationEvaluator( images, texts, diff --git a/mteb/models/blip2_models.py b/mteb/models/blip2_models.py index cb289b3f96..9f950ad525 100644 --- a/mteb/models/blip2_models.py +++ b/mteb/models/blip2_models.py @@ -13,7 +13,7 @@ def blip2_loader(**kwargs): - try: # a temporal fix for the dependency issues of vista models. + try: # a temporal fix for the dependency issues. from lavis.models.blip2_models.blip2_image_text_matching import ( Blip2ITM, ) diff --git a/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py b/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py index a9cf4cea25..7ae4285e2d 100644 --- a/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py +++ b/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py @@ -12,8 +12,8 @@ class FilipinoHateSpeechClassification(AbsTaskClassification): description="Filipino Twitter dataset for sentiment classification.", reference="https://pcj.csp.org.ph/index.php/pcj/issue/download/29/PCJ%20V14%20N1%20pp1-14%202019", dataset={ - "path": "hate-speech-filipino/hate_speech_filipino", - "revision": "1994e9bb7f3ec07518e3f0d9e870cb293e234686", + "path": "jcblaise/hatespeech_filipino", + "revision": "b01711587b073e55569de75ef04d7da4592a3618", "trust_remote_code": True, }, type="Classification", diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py b/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py new file mode 100644 index 0000000000..7d49f49d28 --- /dev/null +++ b/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskImageTextPairClassification import ( + AbsTaskImageTextPairClassification, +) +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class AROVisualAttribution(AbsTaskImageTextPairClassification): + images_column_names = ["image"] + texts_column_names = ["true_caption", "false_caption"] + + metadata = TaskMetadata( + name="AROVisualAttribution", + description="Compositionality Evaluation of images to their captions.", + reference="https://openreview.net/forum?id=KRLUvxh8uaX", + dataset={ + "path": "gowitheflow/ARO-Visual-Attribution", + "revision": "18f7e01358d91df599d723f00e16a18640e19398", + }, + type="ImageTextPairClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="text_acc", + date=( + "2022-01-01", + "2022-12-31", + ), # Estimated range for the collection of data + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Caption Pairing"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="expert-annotated", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@inproceedings{yuksekgonul2023and, + title={When and why vision-language models behave like bags-of-words, and what to do about it?}, + author={Yuksekgonul, Mert and Bianchi, Federico and Kalluri, Pratyusha and Jurafsky, Dan and Zou, James}, + booktitle={The Eleventh International Conference on Learning Representations}, + year={2023} +}""", + descriptive_stats={ + "n_samples": {"test": 28748}, + "avg_character_length": {"test": 1}, + }, + ) diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py b/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py new file mode 100644 index 0000000000..980638ee96 --- /dev/null +++ b/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskImageTextPairClassification import ( + AbsTaskImageTextPairClassification, +) +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class AROVisualRelation(AbsTaskImageTextPairClassification): + images_column_names = ["image"] + texts_column_names = ["true_caption", "false_caption"] + + metadata = TaskMetadata( + name="AROVisualRelation", + description="Compositionality Evaluation of images to their captions.", + reference="https://openreview.net/forum?id=KRLUvxh8uaX", + dataset={ + "path": "gowitheflow/ARO-Visual-Relation", + "revision": "3867ad4f46a1ac2e63be034d1fc77dd8c2ef7209", + }, + type="ImageTextPairClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="text_acc", + date=( + "2022-01-01", + "2022-12-31", + ), # Estimated range for the collection of data + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Caption Pairing"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="expert-annotated", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@inproceedings{yuksekgonul2023and, + title={When and why vision-language models behave like bags-of-words, and what to do about it?}, + author={Yuksekgonul, Mert and Bianchi, Federico and Kalluri, Pratyusha and Jurafsky, Dan and Zou, James}, + booktitle={The Eleventh International Conference on Learning Representations}, + year={2023} +}""", + descriptive_stats={ + "n_samples": {"test": 23937}, + "avg_character_length": {"test": 1}, + }, + ) diff --git a/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py b/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py new file mode 100644 index 0000000000..1ecdf253c0 --- /dev/null +++ b/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import datasets + +from mteb.abstasks.Image.AbsTaskImageTextPairClassification import ( + AbsTaskImageTextPairClassification, +) +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class SugarCrepe(AbsTaskImageTextPairClassification): + images_column_names = ["images"] + texts_column_names = ["caption", "negative_caption"] + + metadata = TaskMetadata( + name="SugarCrepe", + description="Compositionality Evaluation of images to their captions.", + reference="https://proceedings.neurips.cc/paper_files/paper/2023/hash/63461de0b4cb760fc498e85b18a7fe81-Abstract-Datasets_and_Benchmarks.html", + dataset={ + "path": "yjkimstats/SUGARCREPE_fmt", + "revision": "134abf9ade6a32f9fdae0e89022ff227a70b87e5", + }, + type="ImageTextPairClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="text_acc", + date=( + "2022-01-01", + "2022-12-31", + ), # Estimated range for the collection of data + form=["written"], + domains=["Encyclopaedic"], + task_subtypes=["Caption Pairing"], + license="Not specified", + socioeconomic_status="mixed", + annotations_creators="expert-annotated", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@article{hsieh2024sugarcrepe, + title={Sugarcrepe: Fixing hackable benchmarks for vision-language compositionality}, + author={Hsieh, Cheng-Yu and Zhang, Jieyu and Ma, Zixian and Kembhavi, Aniruddha and Krishna, Ranjay}, + journal={Advances in neural information processing systems}, + volume={36}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 7511}, + "avg_character_length": {"test": 1}, + }, + ) + + def load_data(self, **kwargs): + """Load dataset from HuggingFace hub""" + if self.data_loaded: + return + self.dataset = datasets.load_dataset(**self.metadata_dict["dataset"]) # type: ignore + self.dataset = datasets.DatasetDict({"test": self.dataset["train"]}) + self.dataset_transform() + self.data_loaded = True diff --git a/mteb/tasks/Image/ImageTextPairClassification/__init__.py b/mteb/tasks/Image/ImageTextPairClassification/__init__.py index c6ea0b557b..d35de07c28 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/__init__.py +++ b/mteb/tasks/Image/ImageTextPairClassification/__init__.py @@ -1,3 +1,6 @@ from __future__ import annotations +from .AROVisualAttribution import * +from .AROVisualRelation import * +from .SugarCrepe import * from .Winoground import * diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualAttribution.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualAttribution.json new file mode 100644 index 0000000000..98049aafaa --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualAttribution.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "18f7e01358d91df599d723f00e16a18640e19398", + "evaluation_time": 339.30732822418213, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.6186517477035522, + "text_acc": 0.6186517477035522 + } + ] + }, + "task_name": "AROVisualAttribution" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualRelation.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualRelation.json new file mode 100644 index 0000000000..3bc832d5d9 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualRelation.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "3867ad4f46a1ac2e63be034d1fc77dd8c2ef7209", + "evaluation_time": 278.83220171928406, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.5365334153175354, + "text_acc": 0.5365334153175354 + } + ] + }, + "task_name": "AROVisualRelation" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SugarCrepe.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SugarCrepe.json new file mode 100644 index 0000000000..94abe47cb0 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SugarCrepe.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "134abf9ade6a32f9fdae0e89022ff227a70b87e5", + "evaluation_time": 126.86821699142456, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.7690054774284363, + "text_acc": 0.7690054774284363 + } + ] + }, + "task_name": "SugarCrepe" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualAttribution.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualAttribution.json new file mode 100644 index 0000000000..4966616b48 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualAttribution.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "18f7e01358d91df599d723f00e16a18640e19398", + "evaluation_time": 280.1469085216522, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.6139209866523743, + "text_acc": 0.6139209866523743 + } + ] + }, + "task_name": "AROVisualAttribution" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualRelation.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualRelation.json new file mode 100644 index 0000000000..949b1aea3b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualRelation.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "3867ad4f46a1ac2e63be034d1fc77dd8c2ef7209", + "evaluation_time": 238.80001974105835, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.5179429054260254, + "text_acc": 0.5179429054260254 + } + ] + }, + "task_name": "AROVisualRelation" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SugarCrepe.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SugarCrepe.json new file mode 100644 index 0000000000..722f2da7e1 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SugarCrepe.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "134abf9ade6a32f9fdae0e89022ff227a70b87e5", + "evaluation_time": 113.4538505077362, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.7660764455795288, + "text_acc": 0.7660764455795288 + } + ] + }, + "task_name": "SugarCrepe" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualAttribution.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualAttribution.json new file mode 100644 index 0000000000..d8cd1fe3e8 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualAttribution.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "18f7e01358d91df599d723f00e16a18640e19398", + "evaluation_time": 570.5447673797607, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.616216778755188, + "text_acc": 0.616216778755188 + } + ] + }, + "task_name": "AROVisualAttribution" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualRelation.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualRelation.json new file mode 100644 index 0000000000..c92ce1a787 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualRelation.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "3867ad4f46a1ac2e63be034d1fc77dd8c2ef7209", + "evaluation_time": 476.9429078102112, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.5326064229011536, + "text_acc": 0.5326064229011536 + } + ] + }, + "task_name": "AROVisualRelation" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MNIST.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MNIST.json new file mode 100644 index 0000000000..a77358f9a5 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 162.98263120651245, + "kg_co2_emissions": null, + "mteb_version": "1.14.1", + "scores": { + "test": [ + { + "accuracy": 0.9369799999999999, + "f1": 0.9364180161805986, + "f1_weighted": 0.9369031495544664, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9369799999999999, + "scores_per_experiment": [ + { + "accuracy": 0.9396, + "f1": 0.9390095708059392, + "f1_weighted": 0.9393176829760612 + }, + { + "accuracy": 0.9176, + "f1": 0.9161560756580137, + "f1_weighted": 0.9172825619473937 + }, + { + "accuracy": 0.9493, + "f1": 0.9490741086414646, + "f1_weighted": 0.9492979321844203 + }, + { + "accuracy": 0.9303, + "f1": 0.9302718818210677, + "f1_weighted": 0.9305303676361367 + }, + { + "accuracy": 0.9481, + "f1": 0.9475784439765083, + "f1_weighted": 0.9480872030283201 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SugarCrepe.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SugarCrepe.json new file mode 100644 index 0000000000..74c0442e9f --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SugarCrepe.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "134abf9ade6a32f9fdae0e89022ff227a70b87e5", + "evaluation_time": 189.15511345863342, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.7671415209770203, + "text_acc": 0.7671415209770203 + } + ] + }, + "task_name": "SugarCrepe" +} \ No newline at end of file From d5bfecea0b9a038c9d004b1f061ee681f25aede6 Mon Sep 17 00:00:00 2001 From: Jamie-Stirling <36764530+Jamie-Stirling@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:25:29 +0100 Subject: [PATCH 069/154] add SOPI2IRetrieval dataset/task (#1232) * wip: start adding BLIP models * add other blip variants * wip: add blip2_models.py * make lint * wip: implement blip2 wrapper * feat: add blip2 models, still mismatched names * fix: remove projections from image and text embeddings * make lint * wip: add coco BLIP2 * fix: BLIP2 better zero-shot classification without text_proj and vision_proj * tidy blip2 * add imagenet-dog-15 dataset * tidy and lint * remove unused import * add cluster_accuracy, ari and nmi to Image.ClusteringEvaluator * add imagenet-10 clustering task * add SOPI2IRetrieval * add results forclip on ImageNet10Clustering and ImageNetDog15Clustering * add SOPI2IRetrieval results for clip 32 * add results for clip vit 32/SOPI2IRetrieval * resolve conflict * change reference --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 1 + .../Any2AnyRetrieval/eng/SOPI2IRetrieval.py | 41 ++++ .../SOPI2IRetrieval.json | 186 ++++++++++++++++++ 3 files changed, 228 insertions(+) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 2cda0b0664..7ce85ca362 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -19,6 +19,7 @@ from .eng.OVENIT2TRetrieval import * from .eng.SciMMIRI2TRetrieval import * from .eng.SciMMIRT2IRetrieval import * +from .eng.SOPI2IRetrieval import * from .eng.StanfordCarsI2IRetrieval import * from .eng.TUBerlinT2IRetrieval import * from .eng.VisualNewsI2TRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py new file mode 100644 index 0000000000..da8efd4b32 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class SOPI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="SOPI2IRetrieval", + description="Retrieve product photos of 22634 online products.", + reference="https://paperswithcode.com/dataset/stanford-online-products", + dataset={ + "path": "JamieSJS/stanford-online-products", + "revision": "0b3a1622902e6258425e673405bdfb1e5dfa8618", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{oh2016deep, + title={Deep metric learning via lifted structured feature embedding}, + author={Oh Song, Hyun and Xiang, Yu and Jegelka, Stefanie and Savarese, Silvio}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={4004--4012}, + year={2016} +} + """, + descriptive_stats={ + "n_samples": {"default": 120053}, + }, + ) + skip_first_result = True diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json new file mode 100644 index 0000000000..6c038f69cf --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "0b3a1622902e6258425e673405bdfb1e5dfa8618", + "evaluation_time": 510.43445205688477, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.50296, + "cv_recall_at_10": 0.67867, + "cv_recall_at_100": 0.80207, + "cv_recall_at_1000": 0.90797, + "cv_recall_at_20": 0.71918, + "cv_recall_at_3": 0.59883, + "cv_recall_at_5": 0.63223, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.50296, + "map_at_1": 0.01743, + "map_at_10": 0.18702, + "map_at_100": 0.19992, + "map_at_1000": 0.20163, + "map_at_20": 0.19415, + "map_at_3": 0.13926, + "map_at_5": 0.16856, + "mrr_at_1": 0.020809401784925335, + "mrr_at_10": 0.30757755224265354, + "mrr_at_100": 0.3127412453176731, + "mrr_at_1000": 0.3131742235270453, + "mrr_at_20": 0.3106655733183114, + "mrr_at_3": 0.28876174486759937, + "mrr_at_5": 0.30027318782951246, + "nauc_cv_recall_at_1000_diff1": -0.16859046840753472, + "nauc_cv_recall_at_1000_max": 0.025825747180792717, + "nauc_cv_recall_at_1000_std": 0.10093428974009021, + "nauc_cv_recall_at_100_diff1": -0.21029734576768694, + "nauc_cv_recall_at_100_max": 0.014647522560512015, + "nauc_cv_recall_at_100_std": 0.0507444053462594, + "nauc_cv_recall_at_10_diff1": -0.27159676443421377, + "nauc_cv_recall_at_10_max": 0.005491087955239775, + "nauc_cv_recall_at_10_std": 0.02474251556064112, + "nauc_cv_recall_at_1_diff1": -0.2932928014798219, + "nauc_cv_recall_at_1_max": -0.002633536711876493, + "nauc_cv_recall_at_1_std": -0.008456158086326292, + "nauc_cv_recall_at_20_diff1": -0.25166875961614193, + "nauc_cv_recall_at_20_max": 0.006747419978420269, + "nauc_cv_recall_at_20_std": 0.035987294094302555, + "nauc_cv_recall_at_3_diff1": -0.29779984100144863, + "nauc_cv_recall_at_3_max": -0.0035652648610872604, + "nauc_cv_recall_at_3_std": 0.014056845458600668, + "nauc_cv_recall_at_5_diff1": -0.2888857057827477, + "nauc_cv_recall_at_5_max": 7.1464099594732e-05, + "nauc_cv_recall_at_5_std": 0.01823034964853647, + "nauc_map_at_1000_diff1": -0.2714835648506667, + "nauc_map_at_1000_max": 0.0048783333927581995, + "nauc_map_at_1000_std": 0.10933153351257194, + "nauc_map_at_100_diff1": -0.27106477649697597, + "nauc_map_at_100_max": 0.004961279421023488, + "nauc_map_at_100_std": 0.10869233131948694, + "nauc_map_at_10_diff1": -0.27101224264468865, + "nauc_map_at_10_max": 0.003129881807984945, + "nauc_map_at_10_std": 0.09970024568152357, + "nauc_map_at_1_diff1": -0.5746988023835725, + "nauc_map_at_1_max": 0.009513570955681795, + "nauc_map_at_1_std": 0.24050607736711668, + "nauc_map_at_20_diff1": -0.27103409286809227, + "nauc_map_at_20_max": 0.004505986343978161, + "nauc_map_at_20_std": 0.10422239017835391, + "nauc_map_at_3_diff1": -0.25758853794935105, + "nauc_map_at_3_max": 0.0003165186809392022, + "nauc_map_at_3_std": 0.073787815755662, + "nauc_map_at_5_diff1": -0.26840871915761827, + "nauc_map_at_5_max": 0.002542520569357188, + "nauc_map_at_5_std": 0.0864360224226272, + "nauc_mrr_at_1000_diff1": -0.352973203440373, + "nauc_mrr_at_1000_max": 0.004279169460655329, + "nauc_mrr_at_1000_std": 0.05010168406176362, + "nauc_mrr_at_100_diff1": -0.35282838896266405, + "nauc_mrr_at_100_max": 0.004246244348126371, + "nauc_mrr_at_100_std": 0.0500103093583224, + "nauc_mrr_at_10_diff1": -0.3544263028903223, + "nauc_mrr_at_10_max": 0.003929367832392052, + "nauc_mrr_at_10_std": 0.0478252958551306, + "nauc_mrr_at_1_diff1": -0.5746988023835726, + "nauc_mrr_at_1_max": 0.029047376939523493, + "nauc_mrr_at_1_std": 0.2870259521668249, + "nauc_mrr_at_20_diff1": -0.35339280008168317, + "nauc_mrr_at_20_max": 0.004175461034327089, + "nauc_mrr_at_20_std": 0.049148248185835845, + "nauc_mrr_at_3_diff1": -0.3591723252785509, + "nauc_mrr_at_3_max": 0.00403104720219049, + "nauc_mrr_at_3_std": 0.0404960811268073, + "nauc_mrr_at_5_diff1": -0.3585645562893135, + "nauc_mrr_at_5_max": 0.0020548539639942804, + "nauc_mrr_at_5_std": 0.045522727902755446, + "nauc_ndcg_at_1000_diff1": -0.2685009406444735, + "nauc_ndcg_at_1000_max": 0.005836111353149481, + "nauc_ndcg_at_1000_std": 0.11311294743168826, + "nauc_ndcg_at_100_diff1": -0.2626615425913708, + "nauc_ndcg_at_100_max": 0.006500040666350577, + "nauc_ndcg_at_100_std": 0.10412717822836703, + "nauc_ndcg_at_10_diff1": -0.26986440689503255, + "nauc_ndcg_at_10_max": 0.002259064902287912, + "nauc_ndcg_at_10_std": 0.08351960465910635, + "nauc_ndcg_at_1_diff1": -0.5746988023835727, + "nauc_ndcg_at_1_max": 0.01860068455415359, + "nauc_ndcg_at_1_std": 0.24591999227187353, + "nauc_ndcg_at_20_diff1": -0.26557544406085987, + "nauc_ndcg_at_20_max": 0.0047811746525403785, + "nauc_ndcg_at_20_std": 0.09053231502866856, + "nauc_ndcg_at_3_diff1": -0.3079122639483788, + "nauc_ndcg_at_3_max": 0.001817933912863451, + "nauc_ndcg_at_3_std": 0.05331981166099078, + "nauc_ndcg_at_5_diff1": -0.29161633386458363, + "nauc_ndcg_at_5_max": 0.00323633286273378, + "nauc_ndcg_at_5_std": 0.07076540546903416, + "nauc_precision_at_1000_diff1": -0.13140300790068282, + "nauc_precision_at_1000_max": 0.005922440105917975, + "nauc_precision_at_1000_std": 0.08342438590882045, + "nauc_precision_at_100_diff1": -0.17052567611996527, + "nauc_precision_at_100_max": 0.011026201381103823, + "nauc_precision_at_100_std": 0.09223669236894032, + "nauc_precision_at_10_diff1": -0.23209493838034498, + "nauc_precision_at_10_max": 0.004908243372756972, + "nauc_precision_at_10_std": 0.07007542762563608, + "nauc_precision_at_1_diff1": -0.5746988023835727, + "nauc_precision_at_1_max": 0.01860068455415359, + "nauc_precision_at_1_std": 0.24591999227187353, + "nauc_precision_at_20_diff1": -0.2109568972104465, + "nauc_precision_at_20_max": 0.007714408837495573, + "nauc_precision_at_20_std": 0.07965290786741164, + "nauc_precision_at_3_diff1": -0.2870525654193057, + "nauc_precision_at_3_max": -0.0007584139080774421, + "nauc_precision_at_3_std": 0.0318750842480405, + "nauc_precision_at_5_diff1": -0.25948022131281406, + "nauc_precision_at_5_max": 0.004692362608818381, + "nauc_precision_at_5_std": 0.04879698010109281, + "nauc_recall_at_1000_diff1": -0.12358858307301535, + "nauc_recall_at_1000_max": 0.009306080801565777, + "nauc_recall_at_1000_std": 0.15591617596817545, + "nauc_recall_at_100_diff1": -0.14715964976867443, + "nauc_recall_at_100_max": 0.01041920443322213, + "nauc_recall_at_100_std": 0.11606682272338587, + "nauc_recall_at_10_diff1": -0.18845161575089034, + "nauc_recall_at_10_max": 0.000650022994426268, + "nauc_recall_at_10_std": 0.07614315809536469, + "nauc_recall_at_1_diff1": -0.5746988023835725, + "nauc_recall_at_1_max": 0.009513570955681795, + "nauc_recall_at_1_std": 0.24050607736711668, + "nauc_recall_at_20_diff1": -0.17338195050795077, + "nauc_recall_at_20_max": 0.006581585832690798, + "nauc_recall_at_20_std": 0.08663740977783436, + "nauc_recall_at_3_diff1": -0.19307909748063756, + "nauc_recall_at_3_max": 0.0003553588908448416, + "nauc_recall_at_3_std": 0.0442289537166574, + "nauc_recall_at_5_diff1": -0.19899176174509764, + "nauc_recall_at_5_max": -0.0005095021106718202, + "nauc_recall_at_5_std": 0.060914430592624226, + "ndcg_at_1": 0.03583, + "ndcg_at_10": 0.27789, + "ndcg_at_100": 0.32166, + "ndcg_at_1000": 0.35384, + "ndcg_at_20": 0.29399, + "ndcg_at_3": 0.26171, + "ndcg_at_5": 0.27031, + "precision_at_1": 0.03583, + "precision_at_10": 0.13151, + "precision_at_100": 0.01932, + "precision_at_1000": 0.0027, + "precision_at_20": 0.07531, + "precision_at_3": 0.26493, + "precision_at_5": 0.21197, + "recall_at_1": 0.01743, + "recall_at_10": 0.3639, + "recall_at_100": 0.50674, + "recall_at_1000": 0.67803, + "recall_at_20": 0.40787, + "recall_at_3": 0.24952, + "recall_at_5": 0.30796 + } + ] + }, + "task_name": "SOPI2IRetrieval" +} \ No newline at end of file From a7883b50eb7b061b3b56a03e8988c2d763a06cb1 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Thu, 26 Sep 2024 23:30:31 +0800 Subject: [PATCH 070/154] Image text pair cls (#1233) * fix ImageTextPair dataloading for large datasets; more compositionality evaluation datasets * fix meta data * fix validate points --------- Co-authored-by: Isaac Chung --- {mieb-docs => docs/mieb-docs}/README.md | 0 docs/mmteb/validate_points.py | 1 + .../AbsTaskImageTextPairClassification.py | 20 +-- .../ImageTextPairClassificationEvaluator.py | 156 +++++++++++++----- .../Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py | 2 - .../eng/Fashion200kI2TRetrieval.py | 1 - .../eng/Fashion200kT2IRetrieval.py | 1 - .../eng/FashionIQIT2IRetrieval.py | 1 - .../eng/HatefulMemesI2TRetrieval.py | 2 - .../eng/HatefulMemesT2IRetrieval.py | 2 - .../eng/InfoSeekIT2ITRetrieval.py | 2 - .../eng/InfoSeekIT2TRetrieval.py | 2 - .../eng/MSCOCOI2TRetrieval.py | 2 - .../eng/MSCOCOT2IRetrieval.py | 1 - .../eng/MemotionI2TRetrieval.py | 2 - .../eng/MemotionT2IRetrieval.py | 2 - .../eng/NIGHTSI2IRetrieval.py | 2 - .../eng/OVENIT2ITRetrieval.py | 2 - .../Any2AnyRetrieval/eng/OVENIT2TRetrieval.py | 2 - .../eng/SciMMIRI2TRetrieval.py | 2 - .../eng/SciMMIRT2IRetrieval.py | 2 - .../eng/TUBerlinT2IRetrieval.py | 2 - .../eng/VisualNewsI2TRetrieval.py | 2 - .../eng/VisualNewsT2IRetrieval.py | 2 - .../eng/WebQAT2ITRetrieval.py | 2 - .../Any2AnyRetrieval/eng/WebQAT2TRetrieval.py | 2 - mteb/tasks/Image/Clustering/eng/CIFAR.py | 1 - mteb/tasks/Image/Clustering/eng/ImageNet.py | 1 - .../Image/Clustering/eng/TinyImageNet.py | 2 - .../eng/BirdsnapClassification.py | 1 - .../Image/ImageClassification/eng/CIFAR.py | 1 - .../eng/Caltech101Classification.py | 1 - .../eng/Country211Classification.py | 1 - .../eng/DTDClassification.py | 1 - .../eng/EuroSATClassification.py | 1 - .../eng/FER2013Classification.py | 1 - .../eng/FGVCAircraftClassification.py | 1 - .../eng/Food101Classification.py | 1 - .../eng/GTSRBClassification.py | 1 - .../ImageClassification/eng/Imagenet1k.py | 1 - .../eng/MNISTClassification.py | 1 - .../eng/OxfordFlowersClassification.py | 1 - .../eng/OxfordPetsClassification.py | 1 - .../eng/PatchCamelyonClassification.py | 1 - .../eng/RESISC45Classification.py | 1 - .../eng/STL10Classification.py | 1 - .../eng/SUN397Classification.py | 1 - .../eng/UCF101Classification.py | 1 - .../eng/PascalVOC2007.py | 1 - .../AROCocoOrder.py | 55 ++++++ .../AROFlickrOrder.py | 55 ++++++ .../AROVisualAttribution.py | 4 +- .../AROVisualRelation.py | 4 +- .../ImageTextPairClassification/SugarCrepe.py | 4 +- .../ImageTextPairClassification/Winoground.py | 4 +- .../ImageTextPairClassification/__init__.py | 2 + .../AROCocoOrder.json | 21 +++ .../AROFlickrOrder.json | 21 +++ .../AROCocoOrder.json | 21 +++ .../AROFlickrOrder.json | 21 +++ .../AROCocoOrder.json | 21 +++ .../AROFlickrOrder.json | 21 +++ 62 files changed, 358 insertions(+), 137 deletions(-) rename {mieb-docs => docs/mieb-docs}/README.md (100%) create mode 100644 mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py create mode 100644 mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROCocoOrder.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROFlickrOrder.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROCocoOrder.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROFlickrOrder.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROCocoOrder.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROFlickrOrder.json diff --git a/mieb-docs/README.md b/docs/mieb-docs/README.md similarity index 100% rename from mieb-docs/README.md rename to docs/mieb-docs/README.md diff --git a/docs/mmteb/validate_points.py b/docs/mmteb/validate_points.py index 2a898de9bf..5b2a4614ac 100644 --- a/docs/mmteb/validate_points.py +++ b/docs/mmteb/validate_points.py @@ -2,6 +2,7 @@ import logging import os +from typing import Optional from jsonlines import Reader from pydantic import BaseModel, ConfigDict, Field, ValidationError, conint, constr diff --git a/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py index c6d4a6a2de..49523c58f9 100644 --- a/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py @@ -4,7 +4,6 @@ from typing import Any from datasets import Dataset -from tqdm import tqdm from ...encoder_interface import Encoder, EncoderWithQueryCorpusEncode from ...evaluation.evaluators import ImageTextPairClassificationEvaluator @@ -32,18 +31,6 @@ class AbsTaskImageTextPairClassification(AbsTask): def __init__(self, **kwargs): super().__init__(**kwargs) - def _preprocess_column( - self, dataset: Dataset, column_names: str | list[str] - ) -> list[list[Any]]: - """Group examples from the columns into a list of examples.""" - if isinstance(column_names, str): - return dataset[column_names] - - return [ - [example[col] for col in column_names] - for example in tqdm(dataset, desc=f"Processing columns {column_names}") - ] - def _add_main_score(self, scores) -> None: scores["main_score"] = scores[self.metadata.main_score] @@ -60,11 +47,10 @@ def _evaluate_subset( encode_kwargs: dict[str, Any] = {}, **kwargs, ) -> ScoresDict: - images = self._preprocess_column(dataset, self.images_column_names) - texts = self._preprocess_column(dataset, self.texts_column_names) evaluator = ImageTextPairClassificationEvaluator( - images, - texts, + dataset, + images_column_names=self.images_column_names, + texts_column_names=self.texts_column_names, task_name=self.metadata.name, **kwargs, ) diff --git a/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py index 403b3758f2..b548da365e 100644 --- a/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py @@ -1,18 +1,58 @@ from __future__ import annotations -import itertools import logging from typing import Any import torch import torch.nn.functional as F -from PIL import Image +from torch.utils.data import DataLoader +from torchvision import transforms from mteb.encoder_interface import Encoder, EncoderWithSimilarity from mteb.evaluation.evaluators.Evaluator import Evaluator logger = logging.getLogger(__name__) +transform = transforms.Compose([transforms.PILToTensor()]) + + +class ImageTextDataset(torch.utils.data.Dataset): + def __init__( + self, hf_dataset, images_column_names, texts_column_names, transform=None + ): + self.dataset = hf_dataset + self.transform = transform + self.images_column_names = images_column_names + self.texts_column_names = texts_column_names + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + data = self.dataset[idx] + + # Get images + if isinstance(self.images_column_names, str): + images = data[self.images_column_names] + else: + images = [data[col] for col in self.images_column_names] + + # Apply transforms to images + if self.transform is not None: + images = [self.transform(img) for img in images] + + # Get texts + if isinstance(self.texts_column_names, str): + texts = data[self.texts_column_names] + else: + texts = [data[col] for col in self.texts_column_names] + + return images, texts + + +def custom_collate_fn(batch): + return batch + class ImageTextPairClassificationEvaluator(Evaluator): """Evaluate a model based on the similarity of the embeddings by calculating the accuracy of @@ -30,21 +70,22 @@ class ImageTextPairClassificationEvaluator(Evaluator): def __init__( self, - images: list[list[Image.Image]], - texts: list[list[str]], + dataset, + images_column_names: str | list[str], + texts_column_names: str | list[str], task_name: str | None = None, + transform=None, limit: int | None = None, **kwargs, ): super().__init__(**kwargs) if limit: - images = images[:limit] - texts = texts[:limit] - self.images = images - self.texts = texts + dataset = dataset.select(range(limit)) + self.dataset = dataset + self.images_column_names = images_column_names + self.texts_column_names = texts_column_names self.task_name = task_name - - assert len(self.images) == len(self.texts) + self.transform = transform def __call__( self, @@ -54,21 +95,31 @@ def __call__( if "batch_size" not in encode_kwargs: encode_kwargs["batch_size"] = 64 - num_samples = len(self.images) - num_images_per_sample = len(self.images[0]) - num_texts_per_sample = len(self.texts[0]) - - images = list(itertools.chain.from_iterable(self.images)) - texts = list(itertools.chain.from_iterable(self.texts)) - - image_embeddings = F.normalize( - model.get_image_embeddings(images, batch_size=encode_kwargs["batch_size"]), - dim=-1, - ).view(num_samples, num_images_per_sample, -1) - text_embeddings = F.normalize( - model.get_text_embeddings(texts, batch_size=encode_kwargs["batch_size"]), - dim=-1, - ).view(num_samples, num_texts_per_sample, -1) + data_loader = DataLoader( + ImageTextDataset( + self.dataset, + self.images_column_names, + self.texts_column_names, + transform=self.transform, + ), + batch_size=encode_kwargs["batch_size"], + shuffle=False, + # collate_fn=lambda x: x, # Identity collate function + collate_fn=custom_collate_fn, + num_workers=4, + ) + + num_images_per_sample = ( + len(self.images_column_names) + if isinstance(self.images_column_names, list) + else 1 + ) + num_texts_per_sample = ( + len(self.texts_column_names) + if isinstance(self.texts_column_names, list) + else 1 + ) + img_ground_truths = torch.arange(num_images_per_sample) caption_ground_truths = torch.arange(num_texts_per_sample) @@ -76,25 +127,42 @@ def __call__( text_score = [] score = [] - for i in range(num_samples): - images_emb = image_embeddings[i] - texts_emb = text_embeddings[i] - scores = ( - images_emb @ texts_emb.t() - ) # shape = (num_images_per_sample x num_texts_per_sample) - - image_closest_text = scores.argmax(dim=1) # shape = (num_images_per_sample) - text_closest_image = scores.argmax(dim=0) # shape = (num_texts_per_sample) - pred_text_is_correct = ( - (image_closest_text == img_ground_truths).all().item() - ) - pred_image_is_correct = ( - (text_closest_image == caption_ground_truths).all().item() - ) - all_correct = pred_text_is_correct and pred_image_is_correct - image_score.append(pred_image_is_correct) - text_score.append(pred_text_is_correct) - score.append(all_correct) + for batch in data_loader: + images_list, texts_list = zip(*batch) + images = [img for images in images_list for img in images] + texts = [txt for texts in texts_list for txt in texts] + images_emb = F.normalize( + model.get_image_embeddings(images, batch_size=len(images)), + dim=-1, + ).view(len(batch), num_images_per_sample, -1) + texts_emb = F.normalize( + model.get_text_embeddings(texts, batch_size=len(texts)), + dim=-1, + ).view(len(batch), num_texts_per_sample, -1) + for i in range(len(batch)): + img_emb = images_emb[i] + txt_emb = texts_emb[i] + + scores = ( + img_emb @ txt_emb.t() + ) # shape = (num_images_per_sample x num_texts_per_sample) + + image_closest_text = scores.argmax( + dim=1 + ) # shape = (num_images_per_sample) + text_closest_image = scores.argmax( + dim=0 + ) # shape = (num_texts_per_sample) + pred_text_is_correct = ( + (image_closest_text == img_ground_truths).all().item() + ) + pred_image_is_correct = ( + (text_closest_image == caption_ground_truths).all().item() + ) + all_correct = pred_text_is_correct and pred_image_is_correct + image_score.append(pred_image_is_correct) + text_score.append(pred_text_is_correct) + score.append(all_correct) metrics = {} metrics["image_acc"] = torch.Tensor(image_score).float().mean().item() diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py index 417e5d6caa..2e45933ea3 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py @@ -20,11 +20,9 @@ class CIRRIT2IRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2018-01-01", "2018-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py index f0e095423d..3e24c8691f 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py @@ -23,7 +23,6 @@ class Fashion200kI2TRetrieval(AbsTaskAny2AnyRetrieval): domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="Apache-2.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py index 868266d67c..f54a3a38b2 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py @@ -23,7 +23,6 @@ class Fashion200kT2IRetrieval(AbsTaskAny2AnyRetrieval): domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="Apache-2.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py index a58ed15dd5..6072354fe6 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py @@ -23,7 +23,6 @@ class FashionIQIT2IRetrieval(AbsTaskAny2AnyRetrieval): domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="Apache-2.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py index 817ea1c674..c92a497914 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py @@ -74,11 +74,9 @@ class HatefulMemesI2TRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2020-01-01", "2020-12-31"), - form=["found"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="MIT", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py index 0a55e446ed..067396752a 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py @@ -74,11 +74,9 @@ class HatefulMemesT2IRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2020-01-01", "2020-12-31"), - form=["found"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="MIT", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py index f7cb041bcb..e35da59fcb 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py @@ -20,11 +20,9 @@ class InfoSeekIT2ITRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2023-01-01", "2023-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py index cc2b23ea88..4d88a7ac80 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py @@ -20,11 +20,9 @@ class InfoSeekIT2TRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2023-01-01", "2023-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py index 4155a3d31f..dff57f5a53 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py @@ -20,11 +20,9 @@ class MSCOCOI2TRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2018-01-01", "2018-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py index 1c5e440e2b..9ce5fd839e 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py @@ -23,7 +23,6 @@ class MSCOCOT2IRetrieval(AbsTaskAny2AnyRetrieval): domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py index 9247a12f88..5eda9cd295 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py @@ -103,11 +103,9 @@ class MemotionI2TRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2020-01-01", "2020-12-31"), - form=["found"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="MIT", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py index f214bd2ea5..b82b6367a5 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py @@ -102,11 +102,9 @@ class MemotionT2IRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2020-01-01", "2020-12-31"), - form=["found"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="MIT", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py index 73d3f7c280..3c7798c77c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py @@ -19,11 +19,9 @@ class NIGHTSI2IRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2023-01-01", "2023-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Duplicate Image Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py index 0f53eb7e6a..0a720ec995 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py @@ -19,11 +19,9 @@ class OVENIT2ITRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2023-01-01", "2023-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py index 3df5b92625..2c171c778d 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py @@ -19,11 +19,9 @@ class OVENIT2TRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2023-01-01", "2023-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py index eb2c24aeb2..cc96d134a0 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py @@ -79,11 +79,9 @@ class SciMMIRI2TRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2023-01-01", "2023-12-31"), - form=["found"], domains=["Academic"], task_subtypes=["Image Text Retrieval"], license="MIT", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py index e92bd637f5..41c2c98e79 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py @@ -79,11 +79,9 @@ class SciMMIRT2IRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2023-01-01", "2023-12-31"), - form=["found"], domains=["Academic"], task_subtypes=["Image Text Retrieval"], license="MIT", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py index 7c7bddfe4c..754fa14911 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py @@ -20,11 +20,9 @@ class TUBerlinT2IRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2012-01-01", "2012-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py index 2de1713097..bf99c199a8 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py @@ -19,11 +19,9 @@ class VisualNewsI2TRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2020-01-01", "2020-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py index 091d7a7f00..8bd3f8278f 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py @@ -19,11 +19,9 @@ class VisualNewsT2IRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2020-01-01", "2020-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py index 50725b79b9..b3f21869ed 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py @@ -19,11 +19,9 @@ class WebQAT2ITRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2022-01-01", "2022-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py index 14c9c02148..f53415087e 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py @@ -19,11 +19,9 @@ class WebQAT2TRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2022-01-01", "2022-12-31"), - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], license="CC BY-SA 4.0", - socioeconomic_status="medium", annotations_creators="derived", dialect=[], modalities=["text"], diff --git a/mteb/tasks/Image/Clustering/eng/CIFAR.py b/mteb/tasks/Image/Clustering/eng/CIFAR.py index e7f7a1d633..61250cc3f5 100644 --- a/mteb/tasks/Image/Clustering/eng/CIFAR.py +++ b/mteb/tasks/Image/Clustering/eng/CIFAR.py @@ -25,7 +25,6 @@ class CIFAR10Clustering(AbsTaskImageClustering): domains=["Web"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Clustering/eng/ImageNet.py b/mteb/tasks/Image/Clustering/eng/ImageNet.py index dcf8587322..daf8ab8dae 100644 --- a/mteb/tasks/Image/Clustering/eng/ImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/ImageNet.py @@ -22,7 +22,6 @@ class ImageNetDog15Clustering(AbsTaskImageClustering): domains=["Web"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py index 9bafc348af..14123e2111 100644 --- a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py @@ -22,11 +22,9 @@ class TinyImageNet(AbsTaskImageClustering): "2012-01-01", "2015-12-31", ), # Estimated range for the collection of reviews - form=["written"], domains=["Reviews"], task_subtypes=["Sentiment/Hate speech"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py index 38016e5e79..f29259ae2a 100644 --- a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py @@ -25,7 +25,6 @@ class BirdsnapClassification(AbsTaskImageClassification): domains=["Encyclopaedic"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py index 9b4f45e387..2fe4fc2808 100644 --- a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py +++ b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py @@ -25,7 +25,6 @@ class CIFAR10Classification(AbsTaskImageClassification): domains=["Web"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py index fe62f955b3..5c79a41046 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py @@ -26,7 +26,6 @@ class Caltech101Classification(AbsTaskImageClassification): domains=["Encyclopaedic"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py index f02f0e3196..5f34c09a14 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py @@ -25,7 +25,6 @@ class Country211Classification(AbsTaskImageClassification): domains=["Scene"], task_subtypes=["Scene recognition"], license="CC BY-SA 4.0", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py index 25f6ba0401..aabb03f02a 100644 --- a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py @@ -25,7 +25,6 @@ class DTDClassification(AbsTaskImageClassification): domains=["Encyclopaedic"], task_subtypes=["Textures recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py index 4930c13d1b..6ef26a0dba 100644 --- a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py @@ -25,7 +25,6 @@ class EuroSATClassification(AbsTaskImageClassification): domains=["Encyclopaedic"], task_subtypes=["Scene recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py index 9db8b017f7..ea987fb4e2 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py @@ -25,7 +25,6 @@ class FER2013Classification(AbsTaskImageClassification): domains=["Encyclopaedic"], task_subtypes=["Emotion recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py index 9b061e6dd1..74659b5e92 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py @@ -25,7 +25,6 @@ class FGVCAircraftClassification(AbsTaskImageClassification): domains=["Encyclopaedic"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py index 04389db8f1..34b2592e20 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py @@ -25,7 +25,6 @@ class Food101Classification(AbsTaskImageClassification): domains=["Web"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py index e1a222d249..6596151327 100644 --- a/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py @@ -25,7 +25,6 @@ class GTSRBClassification(AbsTaskImageClassification): task_subtypes=["Activity recognition"], domains=["Scene"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py index d15116039c..d3b8474808 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py +++ b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py @@ -25,7 +25,6 @@ class Imagenet1kClassification(AbsTaskImageClassification): domains=["Scene"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="human-annotated", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py index f3831abdb4..4ea68ddea3 100644 --- a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py @@ -25,7 +25,6 @@ class MNISTClassification(AbsTaskImageClassification): domains=["Encyclopaedic"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py index c0a10de48d..d07badc30f 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -25,7 +25,6 @@ class OxfordFlowersClassification(AbsTaskImageClassification): domains=["Reviews"], task_subtypes=["Sentiment/Hate speech"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py index cf537648ed..603dad1278 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py @@ -25,7 +25,6 @@ class OxfordPetsClassification(AbsTaskImageClassification): domains=["Encyclopaedic"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py index 6998c76280..a6f9466672 100644 --- a/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py @@ -25,7 +25,6 @@ class PatchCamelyonClassification(AbsTaskImageClassification): domains=["Medical"], task_subtypes=["Tumor detection"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py index afbc8fe1da..c767e3b334 100644 --- a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py @@ -25,7 +25,6 @@ class RESISC45Classification(AbsTaskImageClassification): domains=["Encyclopaedic"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py index 9531e1c1f6..02593fe4e3 100644 --- a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py @@ -25,7 +25,6 @@ class STL10Classification(AbsTaskImageClassification): domains=["Encyclopaedic"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py index eef0ccbfcb..d23844ec4f 100644 --- a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py @@ -25,7 +25,6 @@ class SUN397Classification(AbsTaskImageClassification): domains=["Encyclopaedic"], task_subtypes=["Scene recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py index 5a76409103..ef82d99d9e 100644 --- a/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py @@ -29,7 +29,6 @@ class UCF101Classification(AbsTaskImageClassification): domains=["Scene"], task_subtypes=["Activity recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index a29108cf6b..fa0628b351 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -28,7 +28,6 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): domains=["Encyclopaedic"], task_subtypes=["Object recognition"], license="Not specified", - socioeconomic_status="mixed", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py b/mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py new file mode 100644 index 0000000000..c72ef004bd --- /dev/null +++ b/mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskImageTextPairClassification import ( + AbsTaskImageTextPairClassification, +) +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class AROCocoOrder(AbsTaskImageTextPairClassification): + images_column_names = ["images"] + texts_column_names = [ + "correct_caption", + "hard_text_1", + "hard_text_2", + "hard_text_3", + "hard_text_4", + ] + + metadata = TaskMetadata( + name="AROCocoOrder", + description="Compositionality Evaluation of images to their captions." + + "Each capation has four hard negatives created by order permutations.", + reference="https://proceedings.neurips.cc/paper_files/paper/2023/hash/63461de0b4cb760fc498e85b18a7fe81-Abstract-Datasets_and_Benchmarks.html", + dataset={ + "path": "gowitheflow/ARO-COCO-order", + "revision": "853ec8757226585a38a80886c51fe0f3f268787c", + }, + type="ImageTextPairClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="text_acc", + date=( + "2022-01-01", + "2022-12-31", + ), # Estimated range for the collection of data + domains=["Encyclopaedic"], + task_subtypes=["Caption Pairing"], + license="MIT", + annotations_creators="expert-annotated", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@article{hsieh2024sugarcrepe, + title={Sugarcrepe: Fixing hackable benchmarks for vision-language compositionality}, + author={Hsieh, Cheng-Yu and Zhang, Jieyu and Ma, Zixian and Kembhavi, Aniruddha and Krishna, Ranjay}, + journal={Advances in neural information processing systems}, + volume={36}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 25010}, + "avg_character_length": {"test": 1}, + }, + ) diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py b/mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py new file mode 100644 index 0000000000..bd8ec152bf --- /dev/null +++ b/mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskImageTextPairClassification import ( + AbsTaskImageTextPairClassification, +) +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class AROFlickrOrder(AbsTaskImageTextPairClassification): + images_column_names = ["images"] + texts_column_names = [ + "correct_caption", + "hard_text_1", + "hard_text_2", + "hard_text_3", + "hard_text_4", + ] + + metadata = TaskMetadata( + name="AROFlickrOrder", + description="Compositionality Evaluation of images to their captions." + + "Each capation has four hard negatives created by order permutations.", + reference="https://proceedings.neurips.cc/paper_files/paper/2023/hash/63461de0b4cb760fc498e85b18a7fe81-Abstract-Datasets_and_Benchmarks.html", + dataset={ + "path": "gowitheflow/ARO-Flickr-Order", + "revision": "1f9485f69c87947812378a1aedf86410c86a0aa8", + }, + type="ImageTextPairClassification", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="text_acc", + date=( + "2022-01-01", + "2022-12-31", + ), # Estimated range for the collection of data + domains=["Encyclopaedic"], + task_subtypes=["Caption Pairing"], + license="MIT", + annotations_creators="expert-annotated", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@article{hsieh2024sugarcrepe, + title={Sugarcrepe: Fixing hackable benchmarks for vision-language compositionality}, + author={Hsieh, Cheng-Yu and Zhang, Jieyu and Ma, Zixian and Kembhavi, Aniruddha and Krishna, Ranjay}, + journal={Advances in neural information processing systems}, + volume={36}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 5000}, + "avg_character_length": {"test": 1}, + }, + ) diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py b/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py index 7d49f49d28..83755ca489 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py +++ b/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py @@ -27,11 +27,9 @@ class AROVisualAttribution(AbsTaskImageTextPairClassification): "2022-01-01", "2022-12-31", ), # Estimated range for the collection of data - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Caption Pairing"], - license="Not specified", - socioeconomic_status="mixed", + license="MIT", annotations_creators="expert-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py b/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py index 980638ee96..6d222b90bd 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py +++ b/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py @@ -27,11 +27,9 @@ class AROVisualRelation(AbsTaskImageTextPairClassification): "2022-01-01", "2022-12-31", ), # Estimated range for the collection of data - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Caption Pairing"], - license="Not specified", - socioeconomic_status="mixed", + license="MIT", annotations_creators="expert-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py b/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py index 1ecdf253c0..1db8fd6563 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py +++ b/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py @@ -29,11 +29,9 @@ class SugarCrepe(AbsTaskImageTextPairClassification): "2022-01-01", "2022-12-31", ), # Estimated range for the collection of data - form=["written"], domains=["Encyclopaedic"], task_subtypes=["Caption Pairing"], - license="Not specified", - socioeconomic_status="mixed", + license="MIT", annotations_creators="expert-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py index c8efea08e2..e598b9f4c1 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py +++ b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py @@ -28,11 +28,9 @@ class Winoground(AbsTaskImageTextPairClassification): "2022-01-01", "2022-04-07", ), # Estimated range for the collection of data - form=["written"], domains=["Social"], # Getty Images. Could be constructed? task_subtypes=["Caption Pairing"], - license="Not specified", - socioeconomic_status="mixed", + license="META Images Reseaerch License", annotations_creators="expert-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/__init__.py b/mteb/tasks/Image/ImageTextPairClassification/__init__.py index d35de07c28..69f0a9fbc1 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/__init__.py +++ b/mteb/tasks/Image/ImageTextPairClassification/__init__.py @@ -1,5 +1,7 @@ from __future__ import annotations +from .AROCocoOrder import * +from .AROFlickrOrder import * from .AROVisualAttribution import * from .AROVisualRelation import * from .SugarCrepe import * diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROCocoOrder.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROCocoOrder.json new file mode 100644 index 0000000000..8924145dd2 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROCocoOrder.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "853ec8757226585a38a80886c51fe0f3f268787c", + "evaluation_time": 207.2516052722931, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.4812075197696686, + "text_acc": 0.4812075197696686 + } + ] + }, + "task_name": "AROCocoOrder" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROFlickrOrder.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROFlickrOrder.json new file mode 100644 index 0000000000..b69a24d891 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROFlickrOrder.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "1f9485f69c87947812378a1aedf86410c86a0aa8", + "evaluation_time": 37.06100630760193, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.5583999752998352, + "text_acc": 0.5583999752998352 + } + ] + }, + "task_name": "AROFlickrOrder" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROCocoOrder.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROCocoOrder.json new file mode 100644 index 0000000000..ed54c38473 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROCocoOrder.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "853ec8757226585a38a80886c51fe0f3f268787c", + "evaluation_time": 158.77182126045227, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.46009597182273865, + "text_acc": 0.46009597182273865 + } + ] + }, + "task_name": "AROCocoOrder" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROFlickrOrder.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROFlickrOrder.json new file mode 100644 index 0000000000..6cdefe46fb --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROFlickrOrder.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "1f9485f69c87947812378a1aedf86410c86a0aa8", + "evaluation_time": 29.562106609344482, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.5612000226974487, + "text_acc": 0.5612000226974487 + } + ] + }, + "task_name": "AROFlickrOrder" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROCocoOrder.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROCocoOrder.json new file mode 100644 index 0000000000..5d7cf611c6 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROCocoOrder.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "853ec8757226585a38a80886c51fe0f3f268787c", + "evaluation_time": 432.3775689601898, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.4538184702396393, + "text_acc": 0.4538184702396393 + } + ] + }, + "task_name": "AROCocoOrder" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROFlickrOrder.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROFlickrOrder.json new file mode 100644 index 0000000000..b67b51a5cd --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROFlickrOrder.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "1f9485f69c87947812378a1aedf86410c86a0aa8", + "evaluation_time": 84.40965294837952, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.5440000295639038, + "text_acc": 0.5440000295639038 + } + ] + }, + "task_name": "AROFlickrOrder" +} \ No newline at end of file From 305afbaf1dde8b76d059d7047e4dbf883d77722d Mon Sep 17 00:00:00 2001 From: Jamie-Stirling <36764530+Jamie-Stirling@users.noreply.github.com> Date: Sun, 29 Sep 2024 18:14:45 +0100 Subject: [PATCH 071/154] Add RP2kI2IRetrieval and METI2IRetrieval (#1239) * wip: start adding BLIP models * add other blip variants * wip: add blip2_models.py * make lint * wip: implement blip2 wrapper * feat: add blip2 models, still mismatched names * fix: remove projections from image and text embeddings * make lint * wip: add coco BLIP2 * fix: BLIP2 better zero-shot classification without text_proj and vision_proj * tidy blip2 * add imagenet-dog-15 dataset * tidy and lint * remove unused import * add cluster_accuracy, ari and nmi to Image.ClusteringEvaluator * add imagenet-10 clustering task * add SOPI2IRetrieval * add results forclip on ImageNet10Clustering and ImageNetDog15Clustering * add SOPI2IRetrieval results for clip 32 * add results for clip vit 32/SOPI2IRetrieval * resolve conflict * add RP2kI2IRetrieval dataset * add RP2kI2IRetrieval results with clip-vit-base-patch32 * update image retrieval __init__.py * add RP2kI2IRetrieval and METI2IRetrieval * add METI2IRetreival * add SOP results * make lign * new revision for METI2IRetrieval * make lint * reset corpus chunk size * remove wrong classification import * add Flickr30k T2I and I2T * add Flickr30k T2I retriebal * reduced-size MET revision * fix: add Flickr30k T2I * make lint --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 4 + .../eng/Flickr30kI2TRetrieval.py | 43 +++ .../eng/Flickr30kT2IRetrieval.py | 57 +++ .../Any2AnyRetrieval/eng/METI2IRetrieval.py | 40 ++ .../Any2AnyRetrieval/eng/RP2kI2IRetrieval.py | 40 ++ .../Any2AnyRetrieval/eng/SOPI2IRetrieval.py | 4 +- .../Fashion200kI2TRetrieval.json | 186 ++++++++++ .../MNIST.json | 48 +++ .../MNISTZeroShot.json | 19 + .../NIGHTSI2IRetrieval.json | 186 ++++++++++ .../OxfordFlowersClassification.json | 48 +++ .../RenderedSST2.json | 19 + .../model_meta.json | 1 + .../RenderedSST2.json | 19 + .../model_meta.json | 1 + .../MNIST.json | 48 +++ .../MNISTZeroShot.json | 19 + .../NIGHTSI2IRetrieval.json | 186 ++++++++++ .../OxfordFlowersClassification.json | 48 +++ .../RenderedSST2.json | 19 + .../model_meta.json | 1 + .../Flickr30kI2TRetrieval.json | 186 ++++++++++ .../Flickr30kT2IRetrieval.json | 186 ++++++++++ .../RP2kI2IRetrieval.json | 186 ++++++++++ .../SOPI2IRetrieval.json | 342 +++++++++--------- 25 files changed, 1763 insertions(+), 173 deletions(-) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py create mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/Fashion200kI2TRetrieval.json create mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNIST.json create mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNISTZeroShot.json create mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/NIGHTSI2IRetrieval.json create mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/OxfordFlowersClassification.json create mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/RenderedSST2.json create mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/model_meta.json create mode 100644 results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/RenderedSST2.json create mode 100644 results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/model_meta.json create mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNIST.json create mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNISTZeroShot.json create mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/NIGHTSI2IRetrieval.json create mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/OxfordFlowersClassification.json create mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/RenderedSST2.json create mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/model_meta.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RP2kI2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 7ce85ca362..3e56bdf82c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -5,6 +5,8 @@ from .eng.Fashion200kI2TRetrieval import * from .eng.Fashion200kT2IRetrieval import * from .eng.FashionIQIT2IRetrieval import * +from .eng.Flickr30kI2TRetrieval import * +from .eng.Flickr30kT2IRetrieval import * from .eng.FORBI2IRetrieval import * from .eng.HatefulMemesI2TRetrieval import * from .eng.HatefulMemesT2IRetrieval import * @@ -12,11 +14,13 @@ from .eng.InfoSeekIT2TRetrieval import * from .eng.MemotionI2TRetrieval import * from .eng.MemotionT2IRetrieval import * +from .eng.METI2IRetrieval import * from .eng.MSCOCOI2TRetrieval import * from .eng.MSCOCOT2IRetrieval import * from .eng.NIGHTSI2IRetrieval import * from .eng.OVENIT2ITRetrieval import * from .eng.OVENIT2TRetrieval import * +from .eng.RP2kI2IRetrieval import * from .eng.SciMMIRI2TRetrieval import * from .eng.SciMMIRT2IRetrieval import * from .eng.SOPI2IRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py new file mode 100644 index 0000000000..f7278bcf37 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class Flickr30kI2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="Flickr30kI2TRetrieval", + description="Retrieve captions based on images.", + reference="https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31", + dataset={ + "path": "JamieSJS/flickr30k", + "revision": "a4cf34ac79215f9e2cd6a10342d84f606fc41cc3", + }, + type="Retrieval", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2018-01-01", "2018-12-31"), + form=["written"], + domains=["Web"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{Young2014FromID, + title={From image descriptions to visual denotations: New similarity metrics for semantic inference over event descriptions}, + author={Peter Young and Alice Lai and Micah Hodosh and J. Hockenmaier}, + journal={Transactions of the Association for Computational Linguistics}, + year={2014}, + volume={2}, + pages={67-78}, + url={https://api.semanticscholar.org/CorpusID:3104920} +}""", + descriptive_stats={ + "n_samples": {"default": 155070}, # qrels + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py new file mode 100644 index 0000000000..44164c90b6 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="Flickr30kT2IRetrieval", + description="Retrieve images based on captions.", + reference="https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31", + dataset={ + "path": "JamieSJS/flickr30k", + "revision": "a4cf34ac79215f9e2cd6a10342d84f606fc41cc3", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2018-01-01", "2018-12-31"), + form=["written"], + domains=["Web"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{Young2014FromID, + title={From image descriptions to visual denotations: New similarity metrics for semantic inference over event descriptions}, + author={Peter Young and Alice Lai and Micah Hodosh and J. Hockenmaier}, + journal={Transactions of the Association for Computational Linguistics}, + year={2014}, + volume={2}, + pages={67-78}, + url={https://api.semanticscholar.org/CorpusID:3104920} +}""", + descriptive_stats={ + "n_samples": {"default": 31014}, # qrels + }, + ) + + def load_data(self, **kwargs): + super().load_data(**kwargs) + # swap corpus and query + for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): + self.queries[split], self.corpus[split] = ( + self.corpus[split], + self.queries[split], + ) + self.relevant_docs[split] = { + cid: {qid: score} + for qid, cid_score in self.relevant_docs[split].items() + for cid, score in cid_score.items() + } diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py new file mode 100644 index 0000000000..e46b2635e5 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class METI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="METI2IRetrieval", + description="Retrieve photos of more than 224k artworks.", + reference="https://arxiv.org/abs/2202.01747", + dataset={ + "path": "JamieSJS/met", + "revision": "08ceaa61c0d172214abb3b8e82971d8f69d2aec0", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2021-12-06", "2021-12-14"), # conference dates + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{ypsilantis2021met, + title={The met dataset: Instance-level recognition for artworks}, + author={Ypsilantis, Nikolaos-Antonios and Garcia, Noa and Han, Guangxing and Ibrahimi, Sarah and Van Noord, Nanne and Tolias, Giorgos}, + booktitle={Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)}, + year={2021} +} + """, + descriptive_stats={ + # "n_samples": {"default": 397121}, + }, + ) + skip_first_result = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py new file mode 100644 index 0000000000..1335e11659 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class RP2kI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="RP2kI2IRetrieval", + description="Retrieve photos of 39457 products.", + reference="https://arxiv.org/abs/2006.12634", + dataset={ + "path": "JamieSJS/rp2k", + "revision": "f8f82d4eb1aa4dc4dbf2c768596c8110a3703765", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@article{peng2020rp2k, + title={RP2K: A large-scale retail product dataset for fine-grained image classification}, + author={Peng, Jingtian and Xiao, Chang and Li, Yifan}, + journal={arXiv preprint arXiv:2006.12634}, + year={2020} +} + """, + descriptive_stats={ + "n_samples": {"default": 4409419}, + }, + ) + skip_first_result = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py index da8efd4b32..9f3d771b0a 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py @@ -18,8 +18,8 @@ class SOPI2IRetrieval(AbsTaskAny2AnyRetrieval): eval_splits=["test"], eval_langs=["eng-Latn"], main_score="cv_recall_at_1", - date=("2009-01-01", "2010-04-01"), - domains=["Web"], + date=("2019-07-17", "2019-07-17"), + domains=["Encyclopaedic"], task_subtypes=["Object recognition"], license="Not specified", annotations_creators="derived", diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/Fashion200kI2TRetrieval.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/Fashion200kI2TRetrieval.json new file mode 100644 index 0000000000..9225a6062a --- /dev/null +++ b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/Fashion200kI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "96a313715ecf67f5dfe70c4fa52406bc7bdfbeee", + "evaluation_time": 181.5803382396698, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.03886, + "cv_recall_at_10": 0.15484, + "cv_recall_at_100": 0.39701, + "cv_recall_at_1000": 0.7658, + "cv_recall_at_20": 0.21334, + "cv_recall_at_3": 0.07568, + "cv_recall_at_5": 0.1035, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.08809, + "map_at_1": 0.03886, + "map_at_10": 0.06791, + "map_at_100": 0.07632, + "map_at_1000": 0.07774, + "map_at_20": 0.07197, + "map_at_3": 0.05489, + "map_at_5": 0.06124, + "mrr_at_1": 0.03886275311924729, + "mrr_at_10": 0.06791249874191167, + "mrr_at_100": 0.07631679834894635, + "mrr_at_1000": 0.07773539145986881, + "mrr_at_20": 0.0719660846478045, + "mrr_at_3": 0.05488511624735805, + "mrr_at_5": 0.06123610827026652, + "nauc_cv_recall_at_1000_diff1": 0.0218733952063869, + "nauc_cv_recall_at_1000_max": 0.16706592598167, + "nauc_cv_recall_at_1000_std": 0.33597938688032986, + "nauc_cv_recall_at_100_diff1": 0.07193545061359337, + "nauc_cv_recall_at_100_max": 0.03727221469537029, + "nauc_cv_recall_at_100_std": 0.1561269787000631, + "nauc_cv_recall_at_10_diff1": 0.0797701915257812, + "nauc_cv_recall_at_10_max": 0.020430887616611652, + "nauc_cv_recall_at_10_std": 0.06318639559887353, + "nauc_cv_recall_at_1_diff1": 0.18888757745716797, + "nauc_cv_recall_at_1_max": 0.03510598124425557, + "nauc_cv_recall_at_1_std": -0.0011818918393113542, + "nauc_cv_recall_at_20_diff1": 0.07098442271206883, + "nauc_cv_recall_at_20_max": 0.017202068195442434, + "nauc_cv_recall_at_20_std": 0.09427426491566825, + "nauc_cv_recall_at_3_diff1": 0.14514396355155457, + "nauc_cv_recall_at_3_max": 0.022088685493555183, + "nauc_cv_recall_at_3_std": 0.02219092083634193, + "nauc_cv_recall_at_5_diff1": 0.11843084765533575, + "nauc_cv_recall_at_5_max": 0.028181652993821218, + "nauc_cv_recall_at_5_std": 0.03378133752500267, + "nauc_map_at_1000_diff1": 0.1318292645003046, + "nauc_map_at_1000_max": 0.025968459599028312, + "nauc_map_at_1000_std": 0.03596161753365205, + "nauc_map_at_100_diff1": 0.13207865359812196, + "nauc_map_at_100_max": 0.02503274769433611, + "nauc_map_at_100_std": 0.034811909972038466, + "nauc_map_at_10_diff1": 0.13675377201318661, + "nauc_map_at_10_max": 0.023988302044714947, + "nauc_map_at_10_std": 0.024038623147938202, + "nauc_map_at_1_diff1": 0.18888757745716797, + "nauc_map_at_1_max": 0.03510598124425557, + "nauc_map_at_1_std": -0.0011818918393113542, + "nauc_map_at_20_diff1": 0.13291861173066144, + "nauc_map_at_20_max": 0.02328223207465244, + "nauc_map_at_20_std": 0.030123558300912807, + "nauc_map_at_3_diff1": 0.161277155146534, + "nauc_map_at_3_max": 0.024402585884692875, + "nauc_map_at_3_std": 0.010083727677318252, + "nauc_map_at_5_diff1": 0.1491311529395408, + "nauc_map_at_5_max": 0.026749124262949692, + "nauc_map_at_5_std": 0.015564562618014838, + "nauc_mrr_at_1000_diff1": 0.13183050469281174, + "nauc_mrr_at_1000_max": 0.025968889811347606, + "nauc_mrr_at_1000_std": 0.035962054245207556, + "nauc_mrr_at_100_diff1": 0.13207988989141017, + "nauc_mrr_at_100_max": 0.02503317589140467, + "nauc_mrr_at_100_std": 0.03481234450834155, + "nauc_mrr_at_10_diff1": 0.13675377201318661, + "nauc_mrr_at_10_max": 0.023988302044714947, + "nauc_mrr_at_10_std": 0.024038623147938202, + "nauc_mrr_at_1_diff1": 0.18888757745716797, + "nauc_mrr_at_1_max": 0.03510598124425557, + "nauc_mrr_at_1_std": -0.0011818918393113542, + "nauc_mrr_at_20_diff1": 0.13291861173066144, + "nauc_mrr_at_20_max": 0.02328223207465244, + "nauc_mrr_at_20_std": 0.030123558300912807, + "nauc_mrr_at_3_diff1": 0.161277155146534, + "nauc_mrr_at_3_max": 0.024402585884692875, + "nauc_mrr_at_3_std": 0.010083727677318252, + "nauc_mrr_at_5_diff1": 0.1491311529395408, + "nauc_mrr_at_5_max": 0.026749124262949692, + "nauc_mrr_at_5_std": 0.015564562618014838, + "nauc_ndcg_at_1000_diff1": 0.10002387998068846, + "nauc_ndcg_at_1000_max": 0.04997596715187386, + "nauc_ndcg_at_1000_std": 0.10395875200499329, + "nauc_ndcg_at_100_diff1": 0.10528225366714257, + "nauc_ndcg_at_100_max": 0.028766853087252058, + "nauc_ndcg_at_100_std": 0.08065845933284391, + "nauc_ndcg_at_10_diff1": 0.11527617893060818, + "nauc_ndcg_at_10_max": 0.022532453546031717, + "nauc_ndcg_at_10_std": 0.038512971156843226, + "nauc_ndcg_at_1_diff1": 0.18888757745716797, + "nauc_ndcg_at_1_max": 0.03510598124425557, + "nauc_ndcg_at_1_std": -0.0011818918393113542, + "nauc_ndcg_at_20_diff1": 0.10745173167011737, + "nauc_ndcg_at_20_max": 0.02078812380101046, + "nauc_ndcg_at_20_std": 0.05495270418039658, + "nauc_ndcg_at_3_diff1": 0.15602598314224742, + "nauc_ndcg_at_3_max": 0.023546212034123336, + "nauc_ndcg_at_3_std": 0.013879753309827844, + "nauc_ndcg_at_5_diff1": 0.13832101043102465, + "nauc_ndcg_at_5_max": 0.027109131323799444, + "nauc_ndcg_at_5_std": 0.021889859067966443, + "nauc_precision_at_1000_diff1": 0.02187339520638577, + "nauc_precision_at_1000_max": 0.1670659259816692, + "nauc_precision_at_1000_std": 0.3359793868803299, + "nauc_precision_at_100_diff1": 0.0719354506135936, + "nauc_precision_at_100_max": 0.03727221469537065, + "nauc_precision_at_100_std": 0.15612697870006317, + "nauc_precision_at_10_diff1": 0.07977019152578149, + "nauc_precision_at_10_max": 0.020430887616611822, + "nauc_precision_at_10_std": 0.06318639559887362, + "nauc_precision_at_1_diff1": 0.18888757745716797, + "nauc_precision_at_1_max": 0.03510598124425557, + "nauc_precision_at_1_std": -0.0011818918393113542, + "nauc_precision_at_20_diff1": 0.0709844227120689, + "nauc_precision_at_20_max": 0.017202068195442757, + "nauc_precision_at_20_std": 0.09427426491566834, + "nauc_precision_at_3_diff1": 0.14514396355155454, + "nauc_precision_at_3_max": 0.022088685493555214, + "nauc_precision_at_3_std": 0.022190920836341964, + "nauc_precision_at_5_diff1": 0.11843084765533585, + "nauc_precision_at_5_max": 0.028181652993821266, + "nauc_precision_at_5_std": 0.03378133752500258, + "nauc_recall_at_1000_diff1": 0.0218733952063869, + "nauc_recall_at_1000_max": 0.16706592598167, + "nauc_recall_at_1000_std": 0.33597938688032986, + "nauc_recall_at_100_diff1": 0.07193545061359337, + "nauc_recall_at_100_max": 0.03727221469537029, + "nauc_recall_at_100_std": 0.1561269787000631, + "nauc_recall_at_10_diff1": 0.0797701915257812, + "nauc_recall_at_10_max": 0.020430887616611652, + "nauc_recall_at_10_std": 0.06318639559887353, + "nauc_recall_at_1_diff1": 0.18888757745716797, + "nauc_recall_at_1_max": 0.03510598124425557, + "nauc_recall_at_1_std": -0.0011818918393113542, + "nauc_recall_at_20_diff1": 0.07098442271206883, + "nauc_recall_at_20_max": 0.017202068195442434, + "nauc_recall_at_20_std": 0.09427426491566825, + "nauc_recall_at_3_diff1": 0.14514396355155457, + "nauc_recall_at_3_max": 0.022088685493555183, + "nauc_recall_at_3_std": 0.02219092083634193, + "nauc_recall_at_5_diff1": 0.11843084765533575, + "nauc_recall_at_5_max": 0.028181652993821218, + "nauc_recall_at_5_std": 0.03378133752500267, + "ndcg_at_1": 0.03886, + "ndcg_at_10": 0.08809, + "ndcg_at_100": 0.13619, + "ndcg_at_1000": 0.18105, + "ndcg_at_20": 0.10288, + "ndcg_at_3": 0.06022, + "ndcg_at_5": 0.07167, + "precision_at_1": 0.03886, + "precision_at_10": 0.01548, + "precision_at_100": 0.00397, + "precision_at_1000": 0.00077, + "precision_at_20": 0.01067, + "precision_at_3": 0.02523, + "precision_at_5": 0.0207, + "recall_at_1": 0.03886, + "recall_at_10": 0.15484, + "recall_at_100": 0.39701, + "recall_at_1000": 0.7658, + "recall_at_20": 0.21334, + "recall_at_3": 0.07568, + "recall_at_5": 0.1035 + } + ] + }, + "task_name": "Fashion200kI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNIST.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNIST.json new file mode 100644 index 0000000000..e0ac824a95 --- /dev/null +++ b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 297.4648160934448, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.85734, + "f1": 0.8534562890978588, + "f1_weighted": 0.8556807192174982, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.85734, + "scores_per_experiment": [ + { + "accuracy": 0.8733, + "f1": 0.8705401169868281, + "f1_weighted": 0.8720762577071195 + }, + { + "accuracy": 0.8022, + "f1": 0.7946331419529729, + "f1_weighted": 0.7995418875379899 + }, + { + "accuracy": 0.8581, + "f1": 0.8542121810766361, + "f1_weighted": 0.8563303310872925 + }, + { + "accuracy": 0.878, + "f1": 0.8754710153202587, + "f1_weighted": 0.8767339121591414 + }, + { + "accuracy": 0.8751, + "f1": 0.8724249901525976, + "f1_weighted": 0.8737212075959484 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNISTZeroShot.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNISTZeroShot.json new file mode 100644 index 0000000000..b058c0b7a3 --- /dev/null +++ b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNISTZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 231.2214720249176, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.5762, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5762 + } + ] + }, + "task_name": "MNISTZeroShot" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/NIGHTSI2IRetrieval.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/NIGHTSI2IRetrieval.json new file mode 100644 index 0000000000..238904b1c3 --- /dev/null +++ b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/NIGHTSI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", + "evaluation_time": 1032.8710505962372, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.07925, + "cv_recall_at_10": 0.48821, + "cv_recall_at_100": 0.9316, + "cv_recall_at_1000": 0.99481, + "cv_recall_at_20": 0.69764, + "cv_recall_at_3": 0.20142, + "cv_recall_at_5": 0.30708, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.24968, + "map_at_1": 0.07925, + "map_at_10": 0.17764, + "map_at_100": 0.19909, + "map_at_1000": 0.19941, + "map_at_20": 0.1922, + "map_at_3": 0.12995, + "map_at_5": 0.15373, + "mrr_at_1": 0.07924528301886792, + "mrr_at_10": 0.1776362683438151, + "mrr_at_100": 0.19909399968942385, + "mrr_at_1000": 0.19940749680294112, + "mrr_at_20": 0.1921998785828074, + "mrr_at_3": 0.12995283018867934, + "mrr_at_5": 0.15372641509433924, + "nauc_cv_recall_at_1000_diff1": -0.6306765130294512, + "nauc_cv_recall_at_1000_max": 0.4255156608097682, + "nauc_cv_recall_at_1000_std": 0.8273066802478575, + "nauc_cv_recall_at_100_diff1": -0.20477156379793215, + "nauc_cv_recall_at_100_max": 0.3289094948324168, + "nauc_cv_recall_at_100_std": 0.3540262081844245, + "nauc_cv_recall_at_10_diff1": -0.0002766589364904998, + "nauc_cv_recall_at_10_max": 0.05573488312411784, + "nauc_cv_recall_at_10_std": -0.08384768736288326, + "nauc_cv_recall_at_1_diff1": 0.0778329308648601, + "nauc_cv_recall_at_1_max": 0.017467451160770153, + "nauc_cv_recall_at_1_std": -0.09230422783548616, + "nauc_cv_recall_at_20_diff1": -0.03663843577705176, + "nauc_cv_recall_at_20_max": 0.09299368334901667, + "nauc_cv_recall_at_20_std": -0.06832906783953004, + "nauc_cv_recall_at_3_diff1": 0.031859526758706465, + "nauc_cv_recall_at_3_max": 0.0014239615902498597, + "nauc_cv_recall_at_3_std": -0.11472573740325614, + "nauc_cv_recall_at_5_diff1": 0.008440048599030739, + "nauc_cv_recall_at_5_max": 0.007294935526259778, + "nauc_cv_recall_at_5_std": -0.09334608645063842, + "nauc_map_at_1000_diff1": 0.03606146943658301, + "nauc_map_at_1000_max": 0.023462582774634224, + "nauc_map_at_1000_std": -0.09234216583449868, + "nauc_map_at_100_diff1": 0.035779413386981485, + "nauc_map_at_100_max": 0.0238874941734106, + "nauc_map_at_100_std": -0.09185556818208644, + "nauc_map_at_10_diff1": 0.03806883475990293, + "nauc_map_at_10_max": 0.020307608194289003, + "nauc_map_at_10_std": -0.09595197128913463, + "nauc_map_at_1_diff1": 0.0778329308648601, + "nauc_map_at_1_max": 0.017467451160770153, + "nauc_map_at_1_std": -0.09230422783548616, + "nauc_map_at_20_diff1": 0.03544017675471238, + "nauc_map_at_20_max": 0.02225673468383308, + "nauc_map_at_20_std": -0.09543789353271984, + "nauc_map_at_3_diff1": 0.05000879822749758, + "nauc_map_at_3_max": 0.00721164104466641, + "nauc_map_at_3_std": -0.10583005601628986, + "nauc_map_at_5_diff1": 0.04007888239793958, + "nauc_map_at_5_max": 0.009848450000869181, + "nauc_map_at_5_std": -0.09855947438234418, + "nauc_mrr_at_1000_diff1": 0.03606146943658301, + "nauc_mrr_at_1000_max": 0.023462582774634224, + "nauc_mrr_at_1000_std": -0.09234216583449868, + "nauc_mrr_at_100_diff1": 0.035779413386981485, + "nauc_mrr_at_100_max": 0.0238874941734106, + "nauc_mrr_at_100_std": -0.09185556818208644, + "nauc_mrr_at_10_diff1": 0.03806883475990293, + "nauc_mrr_at_10_max": 0.020307608194289003, + "nauc_mrr_at_10_std": -0.09595197128913463, + "nauc_mrr_at_1_diff1": 0.0778329308648601, + "nauc_mrr_at_1_max": 0.017467451160770153, + "nauc_mrr_at_1_std": -0.09230422783548616, + "nauc_mrr_at_20_diff1": 0.03544017675471238, + "nauc_mrr_at_20_max": 0.02225673468383308, + "nauc_mrr_at_20_std": -0.09543789353271984, + "nauc_mrr_at_3_diff1": 0.05000879822749758, + "nauc_mrr_at_3_max": 0.00721164104466641, + "nauc_mrr_at_3_std": -0.10583005601628986, + "nauc_mrr_at_5_diff1": 0.04007888239793958, + "nauc_mrr_at_5_max": 0.009848450000869181, + "nauc_mrr_at_5_std": -0.09855947438234418, + "nauc_ndcg_at_1000_diff1": 0.023142396023156372, + "nauc_ndcg_at_1000_max": 0.035375916635258325, + "nauc_ndcg_at_1000_std": -0.07958245047320736, + "nauc_ndcg_at_100_diff1": 0.015119475959438647, + "nauc_ndcg_at_100_max": 0.04731669040790856, + "nauc_ndcg_at_100_std": -0.0635412170264141, + "nauc_ndcg_at_10_diff1": 0.025144530735346976, + "nauc_ndcg_at_10_max": 0.03090893341127276, + "nauc_ndcg_at_10_std": -0.09226635650097872, + "nauc_ndcg_at_1_diff1": 0.0778329308648601, + "nauc_ndcg_at_1_max": 0.017467451160770153, + "nauc_ndcg_at_1_std": -0.09230422783548616, + "nauc_ndcg_at_20_diff1": 0.016460308621990286, + "nauc_ndcg_at_20_max": 0.038935887157304706, + "nauc_ndcg_at_20_std": -0.08916836990042935, + "nauc_ndcg_at_3_diff1": 0.04402254814866491, + "nauc_ndcg_at_3_max": 0.005282526107844626, + "nauc_ndcg_at_3_std": -0.10876252624429479, + "nauc_ndcg_at_5_diff1": 0.029223974622285064, + "nauc_ndcg_at_5_max": 0.0089255171139576, + "nauc_ndcg_at_5_std": -0.09706274219479245, + "nauc_precision_at_1000_diff1": -0.6306765130294122, + "nauc_precision_at_1000_max": 0.42551566080976494, + "nauc_precision_at_1000_std": 0.8273066802478539, + "nauc_precision_at_100_diff1": -0.20477156379793393, + "nauc_precision_at_100_max": 0.3289094948324158, + "nauc_precision_at_100_std": 0.3540262081844231, + "nauc_precision_at_10_diff1": -0.00027665893649036046, + "nauc_precision_at_10_max": 0.055734883124117916, + "nauc_precision_at_10_std": -0.08384768736288313, + "nauc_precision_at_1_diff1": 0.0778329308648601, + "nauc_precision_at_1_max": 0.017467451160770153, + "nauc_precision_at_1_std": -0.09230422783548616, + "nauc_precision_at_20_diff1": -0.036638435777051476, + "nauc_precision_at_20_max": 0.09299368334901673, + "nauc_precision_at_20_std": -0.06832906783952929, + "nauc_precision_at_3_diff1": 0.03185952675870671, + "nauc_precision_at_3_max": 0.0014239615902501407, + "nauc_precision_at_3_std": -0.11472573740325583, + "nauc_precision_at_5_diff1": 0.00844004859903081, + "nauc_precision_at_5_max": 0.0072949355262599425, + "nauc_precision_at_5_std": -0.09334608645063842, + "nauc_recall_at_1000_diff1": -0.6306765130294512, + "nauc_recall_at_1000_max": 0.4255156608097682, + "nauc_recall_at_1000_std": 0.8273066802478575, + "nauc_recall_at_100_diff1": -0.20477156379793215, + "nauc_recall_at_100_max": 0.3289094948324168, + "nauc_recall_at_100_std": 0.3540262081844245, + "nauc_recall_at_10_diff1": -0.0002766589364904998, + "nauc_recall_at_10_max": 0.05573488312411784, + "nauc_recall_at_10_std": -0.08384768736288326, + "nauc_recall_at_1_diff1": 0.0778329308648601, + "nauc_recall_at_1_max": 0.017467451160770153, + "nauc_recall_at_1_std": -0.09230422783548616, + "nauc_recall_at_20_diff1": -0.03663843577705176, + "nauc_recall_at_20_max": 0.09299368334901667, + "nauc_recall_at_20_std": -0.06832906783953004, + "nauc_recall_at_3_diff1": 0.031859526758706465, + "nauc_recall_at_3_max": 0.0014239615902498597, + "nauc_recall_at_3_std": -0.11472573740325614, + "nauc_recall_at_5_diff1": 0.008440048599030739, + "nauc_recall_at_5_max": 0.007294935526259778, + "nauc_recall_at_5_std": -0.09334608645063842, + "ndcg_at_1": 0.07925, + "ndcg_at_10": 0.24968, + "ndcg_at_100": 0.34784, + "ndcg_at_1000": 0.35593, + "ndcg_at_20": 0.30266, + "ndcg_at_3": 0.14817, + "ndcg_at_5": 0.19136, + "precision_at_1": 0.07925, + "precision_at_10": 0.04882, + "precision_at_100": 0.00932, + "precision_at_1000": 0.00099, + "precision_at_20": 0.03488, + "precision_at_3": 0.06714, + "precision_at_5": 0.06142, + "recall_at_1": 0.07925, + "recall_at_10": 0.48821, + "recall_at_100": 0.9316, + "recall_at_1000": 0.99481, + "recall_at_20": 0.69764, + "recall_at_3": 0.20142, + "recall_at_5": 0.30708 + } + ] + }, + "task_name": "NIGHTSI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/OxfordFlowersClassification.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/OxfordFlowersClassification.json new file mode 100644 index 0000000000..6b9742156a --- /dev/null +++ b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/OxfordFlowersClassification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + "evaluation_time": 351.2027907371521, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.8762745098039215, + "f1": 0.8707019249469784, + "f1_weighted": 0.8703980266170556, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8762745098039215, + "scores_per_experiment": [ + { + "accuracy": 0.8784313725490196, + "f1": 0.8709034893034996, + "f1_weighted": 0.8708997095974342 + }, + { + "accuracy": 0.8823529411764706, + "f1": 0.879309449928005, + "f1_weighted": 0.8791989291482171 + }, + { + "accuracy": 0.865686274509804, + "f1": 0.8593490468264394, + "f1_weighted": 0.858799777548312 + }, + { + "accuracy": 0.8715686274509804, + "f1": 0.866195738562844, + "f1_weighted": 0.8655759996721402 + }, + { + "accuracy": 0.8833333333333333, + "f1": 0.8777519001141039, + "f1_weighted": 0.877515717119174 + } + ] + } + ] + }, + "task_name": "OxfordFlowersClassification" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/RenderedSST2.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/RenderedSST2.json new file mode 100644 index 0000000000..bce1d97cbb --- /dev/null +++ b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/RenderedSST2.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", + "evaluation_time": 34.560497760772705, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.49203734211971445, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.49203734211971445 + } + ] + }, + "task_name": "RenderedSST2" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/model_meta.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/model_meta.json new file mode 100644 index 0000000000..2445c6c1a5 --- /dev/null +++ b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/model_meta.json @@ -0,0 +1 @@ +{"name": "Salesforce/blip-itm-base-coco", "revision": "7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f", "release_date": "2023-08-01", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "BLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/RenderedSST2.json b/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/RenderedSST2.json new file mode 100644 index 0000000000..79964f28b8 --- /dev/null +++ b/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/RenderedSST2.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", + "evaluation_time": 76.84005379676819, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.500823723228995, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.500823723228995 + } + ] + }, + "task_name": "RenderedSST2" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/model_meta.json b/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/model_meta.json new file mode 100644 index 0000000000..69acd258db --- /dev/null +++ b/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/model_meta.json @@ -0,0 +1 @@ +{"name": "Salesforce/blip-itm-large-coco", "revision": "fef05cafc05298067cbbca00b125749394a77a6f", "release_date": "2023-08-01", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "BLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNIST.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNIST.json new file mode 100644 index 0000000000..ac83e833d4 --- /dev/null +++ b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 716.1278786659241, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.9285, + "f1": 0.9276672727830793, + "f1_weighted": 0.9282674252781511, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9285, + "scores_per_experiment": [ + { + "accuracy": 0.9284, + "f1": 0.9278192557850607, + "f1_weighted": 0.9285648611883027 + }, + { + "accuracy": 0.9127, + "f1": 0.9108312967490848, + "f1_weighted": 0.9117442017991534 + }, + { + "accuracy": 0.9321, + "f1": 0.931401132729107, + "f1_weighted": 0.9320989666694753 + }, + { + "accuracy": 0.9354, + "f1": 0.9350616561525831, + "f1_weighted": 0.9354103679494147 + }, + { + "accuracy": 0.9339, + "f1": 0.9332230224995612, + "f1_weighted": 0.9335187287844099 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNISTZeroShot.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNISTZeroShot.json new file mode 100644 index 0000000000..b0fd292049 --- /dev/null +++ b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNISTZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "evaluation_time": 636.4780406951904, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.5049, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5049 + } + ] + }, + "task_name": "MNISTZeroShot" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/NIGHTSI2IRetrieval.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/NIGHTSI2IRetrieval.json new file mode 100644 index 0000000000..d452d34222 --- /dev/null +++ b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/NIGHTSI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", + "evaluation_time": 2631.542476415634, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.05943, + "cv_recall_at_10": 0.40142, + "cv_recall_at_100": 0.88679, + "cv_recall_at_1000": 0.99104, + "cv_recall_at_20": 0.59764, + "cv_recall_at_3": 0.16698, + "cv_recall_at_5": 0.24717, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.20235, + "map_at_1": 0.05943, + "map_at_10": 0.14229, + "map_at_100": 0.16407, + "map_at_1000": 0.16457, + "map_at_20": 0.1558, + "map_at_3": 0.10409, + "map_at_5": 0.12215, + "mrr_at_1": 0.059433962264150944, + "mrr_at_10": 0.14228998203054774, + "mrr_at_100": 0.16406642527076057, + "mrr_at_1000": 0.1645732756182432, + "mrr_at_20": 0.15579824218803512, + "mrr_at_3": 0.10408805031446555, + "mrr_at_5": 0.12215408805031443, + "nauc_cv_recall_at_1000_diff1": -0.26463216865694905, + "nauc_cv_recall_at_1000_max": 0.5278883483217763, + "nauc_cv_recall_at_1000_std": 0.7581453634085121, + "nauc_cv_recall_at_100_diff1": 0.014236330594184277, + "nauc_cv_recall_at_100_max": 0.3171598451327442, + "nauc_cv_recall_at_100_std": 0.37266908975979723, + "nauc_cv_recall_at_10_diff1": -0.002210170307048272, + "nauc_cv_recall_at_10_max": 0.10455296298298879, + "nauc_cv_recall_at_10_std": 0.02712166932863189, + "nauc_cv_recall_at_1_diff1": 0.06336429572276098, + "nauc_cv_recall_at_1_max": 0.07874966460960554, + "nauc_cv_recall_at_1_std": -0.029650640329470464, + "nauc_cv_recall_at_20_diff1": -0.015820807363866755, + "nauc_cv_recall_at_20_max": 0.12973690402299562, + "nauc_cv_recall_at_20_std": 0.04547749025861378, + "nauc_cv_recall_at_3_diff1": -0.0053137478758161424, + "nauc_cv_recall_at_3_max": 0.05351014259433647, + "nauc_cv_recall_at_3_std": -0.003411847027194816, + "nauc_cv_recall_at_5_diff1": 0.0004749523652767695, + "nauc_cv_recall_at_5_max": 0.0679335317764752, + "nauc_cv_recall_at_5_std": -0.0006472184948264357, + "nauc_map_at_1000_diff1": 0.016633966898488534, + "nauc_map_at_1000_max": 0.08068623366557101, + "nauc_map_at_1000_std": -0.0017960556231574202, + "nauc_map_at_100_diff1": 0.01661909546774381, + "nauc_map_at_100_max": 0.0811695484351021, + "nauc_map_at_100_std": -0.0010905676199076005, + "nauc_map_at_10_diff1": 0.017095173553333582, + "nauc_map_at_10_max": 0.07644073407248878, + "nauc_map_at_10_std": -0.007324964157984001, + "nauc_map_at_1_diff1": 0.06336429572276098, + "nauc_map_at_1_max": 0.07874966460960554, + "nauc_map_at_1_std": -0.029650640329470464, + "nauc_map_at_20_diff1": 0.016082386042899254, + "nauc_map_at_20_max": 0.07877478749954128, + "nauc_map_at_20_std": -0.0062248086434086415, + "nauc_map_at_3_diff1": 0.017880266235441065, + "nauc_map_at_3_max": 0.06255089186760056, + "nauc_map_at_3_std": -0.015893007578431876, + "nauc_map_at_5_diff1": 0.018500079315787177, + "nauc_map_at_5_max": 0.06573480071106033, + "nauc_map_at_5_std": -0.01479218234208926, + "nauc_mrr_at_1000_diff1": 0.016633966898488534, + "nauc_mrr_at_1000_max": 0.08068623366557101, + "nauc_mrr_at_1000_std": -0.0017960556231574202, + "nauc_mrr_at_100_diff1": 0.01661909546774381, + "nauc_mrr_at_100_max": 0.0811695484351021, + "nauc_mrr_at_100_std": -0.0010905676199076005, + "nauc_mrr_at_10_diff1": 0.017095173553333582, + "nauc_mrr_at_10_max": 0.07644073407248878, + "nauc_mrr_at_10_std": -0.007324964157984001, + "nauc_mrr_at_1_diff1": 0.06336429572276098, + "nauc_mrr_at_1_max": 0.07874966460960554, + "nauc_mrr_at_1_std": -0.029650640329470464, + "nauc_mrr_at_20_diff1": 0.016082386042899254, + "nauc_mrr_at_20_max": 0.07877478749954128, + "nauc_mrr_at_20_std": -0.0062248086434086415, + "nauc_mrr_at_3_diff1": 0.017880266235441065, + "nauc_mrr_at_3_max": 0.06255089186760056, + "nauc_mrr_at_3_std": -0.015893007578431876, + "nauc_mrr_at_5_diff1": 0.018500079315787177, + "nauc_mrr_at_5_max": 0.06573480071106033, + "nauc_mrr_at_5_std": -0.01479218234208926, + "nauc_ndcg_at_1000_diff1": 0.009960736287420825, + "nauc_ndcg_at_1000_max": 0.09531647626699342, + "nauc_ndcg_at_1000_std": 0.020883086622059264, + "nauc_ndcg_at_100_diff1": 0.011572508647446316, + "nauc_ndcg_at_100_max": 0.10937633012181702, + "nauc_ndcg_at_100_std": 0.042541337399181214, + "nauc_ndcg_at_10_diff1": 0.00952815561968612, + "nauc_ndcg_at_10_max": 0.08583341700567972, + "nauc_ndcg_at_10_std": 0.0047805496391580115, + "nauc_ndcg_at_1_diff1": 0.06336429572276098, + "nauc_ndcg_at_1_max": 0.07874966460960554, + "nauc_ndcg_at_1_std": -0.029650640329470464, + "nauc_ndcg_at_20_diff1": 0.005486565789239601, + "nauc_ndcg_at_20_max": 0.09377542572693116, + "nauc_ndcg_at_20_std": 0.009702452362775931, + "nauc_ndcg_at_3_diff1": 0.009923888782701359, + "nauc_ndcg_at_3_max": 0.0594695188640034, + "nauc_ndcg_at_3_std": -0.011751091386382948, + "nauc_ndcg_at_5_diff1": 0.01169898585355613, + "nauc_ndcg_at_5_max": 0.06611397346239468, + "nauc_ndcg_at_5_std": -0.009801904972316896, + "nauc_precision_at_1000_diff1": -0.26463216865691536, + "nauc_precision_at_1000_max": 0.5278883483217904, + "nauc_precision_at_1000_std": 0.7581453634085238, + "nauc_precision_at_100_diff1": 0.01423633059418428, + "nauc_precision_at_100_max": 0.31715984513274537, + "nauc_precision_at_100_std": 0.37266908975979857, + "nauc_precision_at_10_diff1": -0.002210170307048316, + "nauc_precision_at_10_max": 0.10455296298298879, + "nauc_precision_at_10_std": 0.027121669328632015, + "nauc_precision_at_1_diff1": 0.06336429572276098, + "nauc_precision_at_1_max": 0.07874966460960554, + "nauc_precision_at_1_std": -0.029650640329470464, + "nauc_precision_at_20_diff1": -0.015820807363866707, + "nauc_precision_at_20_max": 0.12973690402299493, + "nauc_precision_at_20_std": 0.04547749025861332, + "nauc_precision_at_3_diff1": -0.0053137478758161095, + "nauc_precision_at_3_max": 0.05351014259433666, + "nauc_precision_at_3_std": -0.0034118470271947503, + "nauc_precision_at_5_diff1": 0.00047495236527695035, + "nauc_precision_at_5_max": 0.06793353177647533, + "nauc_precision_at_5_std": -0.000647218494826177, + "nauc_recall_at_1000_diff1": -0.26463216865694905, + "nauc_recall_at_1000_max": 0.5278883483217763, + "nauc_recall_at_1000_std": 0.7581453634085121, + "nauc_recall_at_100_diff1": 0.014236330594184277, + "nauc_recall_at_100_max": 0.3171598451327442, + "nauc_recall_at_100_std": 0.37266908975979723, + "nauc_recall_at_10_diff1": -0.002210170307048272, + "nauc_recall_at_10_max": 0.10455296298298879, + "nauc_recall_at_10_std": 0.02712166932863189, + "nauc_recall_at_1_diff1": 0.06336429572276098, + "nauc_recall_at_1_max": 0.07874966460960554, + "nauc_recall_at_1_std": -0.029650640329470464, + "nauc_recall_at_20_diff1": -0.015820807363866755, + "nauc_recall_at_20_max": 0.12973690402299562, + "nauc_recall_at_20_std": 0.04547749025861378, + "nauc_recall_at_3_diff1": -0.0053137478758161424, + "nauc_recall_at_3_max": 0.05351014259433647, + "nauc_recall_at_3_std": -0.003411847027194816, + "nauc_recall_at_5_diff1": 0.0004749523652767695, + "nauc_recall_at_5_max": 0.0679335317764752, + "nauc_recall_at_5_std": -0.0006472184948264357, + "ndcg_at_1": 0.05943, + "ndcg_at_10": 0.20235, + "ndcg_at_100": 0.30717, + "ndcg_at_1000": 0.32046, + "ndcg_at_20": 0.25183, + "ndcg_at_3": 0.12012, + "ndcg_at_5": 0.15292, + "precision_at_1": 0.05943, + "precision_at_10": 0.04014, + "precision_at_100": 0.00887, + "precision_at_1000": 0.00099, + "precision_at_20": 0.02988, + "precision_at_3": 0.05566, + "precision_at_5": 0.04943, + "recall_at_1": 0.05943, + "recall_at_10": 0.40142, + "recall_at_100": 0.88679, + "recall_at_1000": 0.99104, + "recall_at_20": 0.59764, + "recall_at_3": 0.16698, + "recall_at_5": 0.24717 + } + ] + }, + "task_name": "NIGHTSI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/OxfordFlowersClassification.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/OxfordFlowersClassification.json new file mode 100644 index 0000000000..f49aa0a7d7 --- /dev/null +++ b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/OxfordFlowersClassification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + "evaluation_time": 684.7742538452148, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.966078431372549, + "f1": 0.9665318717720475, + "f1_weighted": 0.96610958525568, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.966078431372549, + "scores_per_experiment": [ + { + "accuracy": 0.9686274509803922, + "f1": 0.969270715608082, + "f1_weighted": 0.9689539772636104 + }, + { + "accuracy": 0.9696078431372549, + "f1": 0.9698978889543093, + "f1_weighted": 0.9692619935891078 + }, + { + "accuracy": 0.9647058823529412, + "f1": 0.9649920270608272, + "f1_weighted": 0.9646380547095591 + }, + { + "accuracy": 0.9598039215686275, + "f1": 0.9602800604286633, + "f1_weighted": 0.9600010155147554 + }, + { + "accuracy": 0.9676470588235294, + "f1": 0.9682186668083551, + "f1_weighted": 0.9676928852013674 + } + ] + } + ] + }, + "task_name": "OxfordFlowersClassification" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/RenderedSST2.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/RenderedSST2.json new file mode 100644 index 0000000000..c593e878e3 --- /dev/null +++ b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/RenderedSST2.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", + "evaluation_time": 117.32345771789551, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.5178473366282262, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5178473366282262 + } + ] + }, + "task_name": "RenderedSST2" +} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/model_meta.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/model_meta.json new file mode 100644 index 0000000000..92e563becc --- /dev/null +++ b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/model_meta.json @@ -0,0 +1 @@ +{"name": "Salesforce/blip2-opt-2.7b", "revision": "51572668da0eb669e01a189dc22abe6088589a24", "release_date": "2024-03-22", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "blip2_loader"} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json new file mode 100644 index 0000000000..714a658b14 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "a4cf34ac79215f9e2cd6a10342d84f606fc41cc3", + "evaluation_time": 2905.3177580833435, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.4111, + "cv_recall_at_10": 0.74505, + "cv_recall_at_100": 0.94796, + "cv_recall_at_1000": 0.99607, + "cv_recall_at_20": 0.82443, + "cv_recall_at_3": 0.58196, + "cv_recall_at_5": 0.65757, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.33866, + "map_at_1": 0.08222, + "map_at_10": 0.2347, + "map_at_100": 0.26833, + "map_at_1000": 0.27226, + "map_at_20": 0.25195, + "map_at_3": 0.16585, + "map_at_5": 0.20389, + "mrr_at_1": 0.4111046624105243, + "mrr_at_10": 0.5151111351658093, + "mrr_at_100": 0.5239514165075607, + "mrr_at_1000": 0.5242073041502724, + "mrr_at_20": 0.5206470443576335, + "mrr_at_3": 0.4860761806496125, + "mrr_at_5": 0.503347412996305, + "nauc_cv_recall_at_1000_diff1": 0.18030127835739224, + "nauc_cv_recall_at_1000_max": 0.5581466344401298, + "nauc_cv_recall_at_1000_std": 0.8631168401994536, + "nauc_cv_recall_at_100_diff1": 0.272670704205089, + "nauc_cv_recall_at_100_max": 0.4612079444879557, + "nauc_cv_recall_at_100_std": 0.5695575488041834, + "nauc_cv_recall_at_10_diff1": 0.28543669447055997, + "nauc_cv_recall_at_10_max": 0.36347027563947776, + "nauc_cv_recall_at_10_std": 0.2385430096241168, + "nauc_cv_recall_at_1_diff1": 0.3784400483602523, + "nauc_cv_recall_at_1_max": 0.3504797969786945, + "nauc_cv_recall_at_1_std": 0.10634115715764078, + "nauc_cv_recall_at_20_diff1": 0.27508492044537997, + "nauc_cv_recall_at_20_max": 0.3714030729491561, + "nauc_cv_recall_at_20_std": 0.2959015251319611, + "nauc_cv_recall_at_3_diff1": 0.31233109201191445, + "nauc_cv_recall_at_3_max": 0.35059358000714524, + "nauc_cv_recall_at_3_std": 0.15142403433656834, + "nauc_cv_recall_at_5_diff1": 0.2974464195367057, + "nauc_cv_recall_at_5_max": 0.35509762440464737, + "nauc_cv_recall_at_5_std": 0.18640081229405794, + "nauc_map_at_1000_diff1": 0.2662124495945085, + "nauc_map_at_1000_max": 0.40406570060222413, + "nauc_map_at_1000_std": 0.1959460147481183, + "nauc_map_at_100_diff1": 0.2659471804759087, + "nauc_map_at_100_max": 0.40363986652542505, + "nauc_map_at_100_std": 0.1947488241847008, + "nauc_map_at_10_diff1": 0.2666647469717836, + "nauc_map_at_10_max": 0.39968979149613193, + "nauc_map_at_10_std": 0.16723592929025385, + "nauc_map_at_1_diff1": 0.37844004836025164, + "nauc_map_at_1_max": 0.3504797969786941, + "nauc_map_at_1_std": 0.1063411571576403, + "nauc_map_at_20_diff1": 0.2650108128343263, + "nauc_map_at_20_max": 0.4008348432286667, + "nauc_map_at_20_std": 0.17971414027637928, + "nauc_map_at_3_diff1": 0.28590676523718034, + "nauc_map_at_3_max": 0.3781596069511839, + "nauc_map_at_3_std": 0.13040101399163928, + "nauc_map_at_5_diff1": 0.27462944218025, + "nauc_map_at_5_max": 0.39938962890452623, + "nauc_map_at_5_std": 0.14980070399423312, + "nauc_mrr_at_1000_diff1": 0.3494548271148969, + "nauc_mrr_at_1000_max": 0.3521197992435183, + "nauc_mrr_at_1000_std": 0.1344856075273458, + "nauc_mrr_at_100_diff1": 0.349406446391289, + "nauc_mrr_at_100_max": 0.35218688878029347, + "nauc_mrr_at_100_std": 0.1347720116595227, + "nauc_mrr_at_10_diff1": 0.34832734062778187, + "nauc_mrr_at_10_max": 0.35184655254909514, + "nauc_mrr_at_10_std": 0.13412235150155113, + "nauc_mrr_at_1_diff1": 0.3784400483602523, + "nauc_mrr_at_1_max": 0.3504797969786945, + "nauc_mrr_at_1_std": 0.10634115715764078, + "nauc_mrr_at_20_diff1": 0.34874544122097734, + "nauc_mrr_at_20_max": 0.35198594125683075, + "nauc_mrr_at_20_std": 0.1348277834635207, + "nauc_mrr_at_3_diff1": 0.3502458632268556, + "nauc_mrr_at_3_max": 0.3503121014809114, + "nauc_mrr_at_3_std": 0.12461253730902709, + "nauc_mrr_at_5_diff1": 0.3483495916749157, + "nauc_mrr_at_5_max": 0.3512186490276025, + "nauc_mrr_at_5_std": 0.13065045260516017, + "nauc_ndcg_at_1000_diff1": 0.28085910013736953, + "nauc_ndcg_at_1000_max": 0.400173338146715, + "nauc_ndcg_at_1000_std": 0.2454880347293459, + "nauc_ndcg_at_100_diff1": 0.27614261400212725, + "nauc_ndcg_at_100_max": 0.3975804199137475, + "nauc_ndcg_at_100_std": 0.24550705671790238, + "nauc_ndcg_at_10_diff1": 0.2770040634103947, + "nauc_ndcg_at_10_max": 0.386892601870249, + "nauc_ndcg_at_10_std": 0.17593863508925003, + "nauc_ndcg_at_1_diff1": 0.3784400483602523, + "nauc_ndcg_at_1_max": 0.3504797969786945, + "nauc_ndcg_at_1_std": 0.10634115715764078, + "nauc_ndcg_at_20_diff1": 0.27393587281733567, + "nauc_ndcg_at_20_max": 0.3893030504620739, + "nauc_ndcg_at_20_std": 0.1976666061871801, + "nauc_ndcg_at_3_diff1": 0.29659323480247735, + "nauc_ndcg_at_3_max": 0.3710317896505282, + "nauc_ndcg_at_3_std": 0.1331916818585782, + "nauc_ndcg_at_5_diff1": 0.28580433201485833, + "nauc_ndcg_at_5_max": 0.3859484521144905, + "nauc_ndcg_at_5_std": 0.15313433521572764, + "nauc_precision_at_1000_diff1": 0.12752694892887595, + "nauc_precision_at_1000_max": 0.3590535825037547, + "nauc_precision_at_1000_std": 0.6227671512776303, + "nauc_precision_at_100_diff1": 0.18161894144710583, + "nauc_precision_at_100_max": 0.3507884567772516, + "nauc_precision_at_100_std": 0.39162073570098294, + "nauc_precision_at_10_diff1": 0.22500485735582146, + "nauc_precision_at_10_max": 0.37378539642037556, + "nauc_precision_at_10_std": 0.2035430362585049, + "nauc_precision_at_1_diff1": 0.3784400483602523, + "nauc_precision_at_1_max": 0.3504797969786945, + "nauc_precision_at_1_std": 0.10634115715764078, + "nauc_precision_at_20_diff1": 0.20677311842444712, + "nauc_precision_at_20_max": 0.3598552830533254, + "nauc_precision_at_20_std": 0.24219283310766115, + "nauc_precision_at_3_diff1": 0.26529469191714483, + "nauc_precision_at_3_max": 0.37325220096807005, + "nauc_precision_at_3_std": 0.14099019547945105, + "nauc_precision_at_5_diff1": 0.24620492543282482, + "nauc_precision_at_5_max": 0.3867515487955738, + "nauc_precision_at_5_std": 0.16846852569552279, + "nauc_recall_at_1000_diff1": 0.1275269489288761, + "nauc_recall_at_1000_max": 0.35905358250375646, + "nauc_recall_at_1000_std": 0.6227671512776317, + "nauc_recall_at_100_diff1": 0.1816189414471066, + "nauc_recall_at_100_max": 0.35078845677725146, + "nauc_recall_at_100_std": 0.3916207357009826, + "nauc_recall_at_10_diff1": 0.22500485735582146, + "nauc_recall_at_10_max": 0.37378539642037556, + "nauc_recall_at_10_std": 0.2035430362585049, + "nauc_recall_at_1_diff1": 0.37844004836025164, + "nauc_recall_at_1_max": 0.3504797969786941, + "nauc_recall_at_1_std": 0.1063411571576403, + "nauc_recall_at_20_diff1": 0.20677311842444712, + "nauc_recall_at_20_max": 0.3598552830533254, + "nauc_recall_at_20_std": 0.24219283310766115, + "nauc_recall_at_3_diff1": 0.2652946919171448, + "nauc_recall_at_3_max": 0.3732522009680695, + "nauc_recall_at_3_std": 0.1409901954794505, + "nauc_recall_at_5_diff1": 0.24620492543282482, + "nauc_recall_at_5_max": 0.3867515487955738, + "nauc_recall_at_5_std": 0.16846852569552279, + "ndcg_at_1": 0.4111, + "ndcg_at_10": 0.33866, + "ndcg_at_100": 0.44523, + "ndcg_at_1000": 0.49782, + "ndcg_at_20": 0.37695, + "ndcg_at_3": 0.33845, + "ndcg_at_5": 0.29078, + "precision_at_1": 0.4111, + "precision_at_10": 0.17182, + "precision_at_100": 0.03266, + "precision_at_1000": 0.00451, + "precision_at_20": 0.10825, + "precision_at_3": 0.31717, + "precision_at_5": 0.25644, + "recall_at_1": 0.08222, + "recall_at_10": 0.34363, + "recall_at_100": 0.65316, + "recall_at_1000": 0.90261, + "recall_at_20": 0.43299, + "recall_at_3": 0.1903, + "recall_at_5": 0.25644 + } + ] + }, + "task_name": "Flickr30kI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json new file mode 100644 index 0000000000..60797e4e6b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "a4cf34ac79215f9e2cd6a10342d84f606fc41cc3", + "evaluation_time": 3504.4338762760162, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.2175, + "cv_recall_at_10": 0.50718, + "cv_recall_at_100": 0.80839, + "cv_recall_at_1000": 0.96492, + "cv_recall_at_20": 0.60324, + "cv_recall_at_3": 0.34774, + "cv_recall_at_5": 0.41235, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.3504, + "map_at_1": 0.2175, + "map_at_10": 0.3017, + "map_at_100": 0.31355, + "map_at_1000": 0.31429, + "map_at_20": 0.30838, + "map_at_3": 0.2743, + "map_at_5": 0.28901, + "mrr_at_1": 0.21749532469207455, + "mrr_at_10": 0.3017006349410669, + "mrr_at_100": 0.3135464607899773, + "mrr_at_1000": 0.31428602518127013, + "mrr_at_20": 0.30838107727990743, + "mrr_at_3": 0.2743008533780535, + "mrr_at_5": 0.28900711506631577, + "nauc_cv_recall_at_1000_diff1": 0.2632892740429544, + "nauc_cv_recall_at_1000_max": 0.4683201344263211, + "nauc_cv_recall_at_1000_std": 0.7033795380897453, + "nauc_cv_recall_at_100_diff1": 0.2647647634897656, + "nauc_cv_recall_at_100_max": 0.34490671196475553, + "nauc_cv_recall_at_100_std": 0.3944496412003796, + "nauc_cv_recall_at_10_diff1": 0.2995130617977379, + "nauc_cv_recall_at_10_max": 0.28870905172779393, + "nauc_cv_recall_at_10_std": 0.13919504988489, + "nauc_cv_recall_at_1_diff1": 0.46334150143887837, + "nauc_cv_recall_at_1_max": 0.3188934433784608, + "nauc_cv_recall_at_1_std": 0.03870010544883344, + "nauc_cv_recall_at_20_diff1": 0.2830079535289093, + "nauc_cv_recall_at_20_max": 0.29612199469809664, + "nauc_cv_recall_at_20_std": 0.19159692443206613, + "nauc_cv_recall_at_3_diff1": 0.3535295484138601, + "nauc_cv_recall_at_3_max": 0.29806929455574155, + "nauc_cv_recall_at_3_std": 0.07658894036209522, + "nauc_cv_recall_at_5_diff1": 0.32210572836847934, + "nauc_cv_recall_at_5_max": 0.2888711825797017, + "nauc_cv_recall_at_5_std": 0.09843675944719517, + "nauc_map_at_1000_diff1": 0.39832464563018605, + "nauc_map_at_1000_max": 0.3074377337641309, + "nauc_map_at_1000_std": 0.07177029946282956, + "nauc_map_at_100_diff1": 0.3980484146317034, + "nauc_map_at_100_max": 0.3074089025502927, + "nauc_map_at_100_std": 0.0721090495580087, + "nauc_map_at_10_diff1": 0.39759464570461306, + "nauc_map_at_10_max": 0.3059467184032683, + "nauc_map_at_10_std": 0.06729002523587067, + "nauc_map_at_1_diff1": 0.46334150143887837, + "nauc_map_at_1_max": 0.3188934433784608, + "nauc_map_at_1_std": 0.03870010544883344, + "nauc_map_at_20_diff1": 0.39724585077726443, + "nauc_map_at_20_max": 0.30662830181613326, + "nauc_map_at_20_std": 0.07029104503264444, + "nauc_map_at_3_diff1": 0.4087813508135372, + "nauc_map_at_3_max": 0.3082838424431037, + "nauc_map_at_3_std": 0.05612029637486883, + "nauc_map_at_5_diff1": 0.40044699102255726, + "nauc_map_at_5_max": 0.3058538687154621, + "nauc_map_at_5_std": 0.06168460440199823, + "nauc_mrr_at_1000_diff1": 0.3983251214518209, + "nauc_mrr_at_1000_max": 0.3074398467997274, + "nauc_mrr_at_1000_std": 0.0717737083834719, + "nauc_mrr_at_100_diff1": 0.3980488893862368, + "nauc_mrr_at_100_max": 0.3074110112149868, + "nauc_mrr_at_100_std": 0.07211245146163399, + "nauc_mrr_at_10_diff1": 0.39759509662724174, + "nauc_mrr_at_10_max": 0.3059487942694512, + "nauc_mrr_at_10_std": 0.06729335845497784, + "nauc_mrr_at_1_diff1": 0.46334150143887837, + "nauc_mrr_at_1_max": 0.3188934433784608, + "nauc_mrr_at_1_std": 0.03870010544883344, + "nauc_mrr_at_20_diff1": 0.3972463051807796, + "nauc_mrr_at_20_max": 0.30663039310948675, + "nauc_mrr_at_20_std": 0.07029440252517179, + "nauc_mrr_at_3_diff1": 0.4087846165854502, + "nauc_mrr_at_3_max": 0.3082911152291177, + "nauc_mrr_at_3_std": 0.0561317481536657, + "nauc_mrr_at_5_diff1": 0.40044780533219726, + "nauc_mrr_at_5_max": 0.30585567761349425, + "nauc_mrr_at_5_std": 0.061687450057817825, + "nauc_ndcg_at_1000_diff1": 0.3822651584130574, + "nauc_ndcg_at_1000_max": 0.3096027445146429, + "nauc_ndcg_at_1000_std": 0.09905420670861774, + "nauc_ndcg_at_100_diff1": 0.3751136878938184, + "nauc_ndcg_at_100_max": 0.3096593022400737, + "nauc_ndcg_at_100_std": 0.1114725296151852, + "nauc_ndcg_at_10_diff1": 0.3722441527278821, + "nauc_ndcg_at_10_max": 0.30140653687780977, + "nauc_ndcg_at_10_std": 0.08492137004866748, + "nauc_ndcg_at_1_diff1": 0.46334150143887837, + "nauc_ndcg_at_1_max": 0.3188934433784608, + "nauc_ndcg_at_1_std": 0.03870010544883344, + "nauc_ndcg_at_20_diff1": 0.3702205468461526, + "nauc_ndcg_at_20_max": 0.30363863160972876, + "nauc_ndcg_at_20_std": 0.09630513131602707, + "nauc_ndcg_at_3_diff1": 0.39360911171689245, + "nauc_ndcg_at_3_max": 0.30546309152189277, + "nauc_ndcg_at_3_std": 0.061672579869672305, + "nauc_ndcg_at_5_diff1": 0.37922470007538506, + "nauc_ndcg_at_5_max": 0.3012774286231696, + "nauc_ndcg_at_5_std": 0.07144813701782884, + "nauc_precision_at_1000_diff1": 0.2632892740429434, + "nauc_precision_at_1000_max": 0.46832013442631265, + "nauc_precision_at_1000_std": 0.7033795380897392, + "nauc_precision_at_100_diff1": 0.26476476348976524, + "nauc_precision_at_100_max": 0.3449067119647543, + "nauc_precision_at_100_std": 0.3944496412003801, + "nauc_precision_at_10_diff1": 0.2995130617977381, + "nauc_precision_at_10_max": 0.28870905172779415, + "nauc_precision_at_10_std": 0.13919504988489012, + "nauc_precision_at_1_diff1": 0.46334150143887837, + "nauc_precision_at_1_max": 0.3188934433784608, + "nauc_precision_at_1_std": 0.03870010544883344, + "nauc_precision_at_20_diff1": 0.2830079535289092, + "nauc_precision_at_20_max": 0.29612199469809636, + "nauc_precision_at_20_std": 0.19159692443206594, + "nauc_precision_at_3_diff1": 0.3535203790281254, + "nauc_precision_at_3_max": 0.29804922704221115, + "nauc_precision_at_3_std": 0.07655745610155888, + "nauc_precision_at_5_diff1": 0.32210572836847917, + "nauc_precision_at_5_max": 0.2888711825797015, + "nauc_precision_at_5_std": 0.09843675944719478, + "nauc_recall_at_1000_diff1": 0.2632892740429544, + "nauc_recall_at_1000_max": 0.4683201344263211, + "nauc_recall_at_1000_std": 0.7033795380897453, + "nauc_recall_at_100_diff1": 0.2647647634897656, + "nauc_recall_at_100_max": 0.34490671196475553, + "nauc_recall_at_100_std": 0.3944496412003796, + "nauc_recall_at_10_diff1": 0.2995130617977379, + "nauc_recall_at_10_max": 0.28870905172779393, + "nauc_recall_at_10_std": 0.13919504988489, + "nauc_recall_at_1_diff1": 0.46334150143887837, + "nauc_recall_at_1_max": 0.3188934433784608, + "nauc_recall_at_1_std": 0.03870010544883344, + "nauc_recall_at_20_diff1": 0.2830079535289093, + "nauc_recall_at_20_max": 0.29612199469809664, + "nauc_recall_at_20_std": 0.19159692443206613, + "nauc_recall_at_3_diff1": 0.3535203790281251, + "nauc_recall_at_3_max": 0.29804922704221065, + "nauc_recall_at_3_std": 0.07655745610155873, + "nauc_recall_at_5_diff1": 0.32210572836847934, + "nauc_recall_at_5_max": 0.2888711825797017, + "nauc_recall_at_5_std": 0.09843675944719517, + "ndcg_at_1": 0.2175, + "ndcg_at_10": 0.3504, + "ndcg_at_100": 0.41258, + "ndcg_at_1000": 0.43242, + "ndcg_at_20": 0.3747, + "ndcg_at_3": 0.29313, + "ndcg_at_5": 0.31969, + "precision_at_1": 0.2175, + "precision_at_10": 0.05072, + "precision_at_100": 0.00808, + "precision_at_1000": 0.00096, + "precision_at_20": 0.03016, + "precision_at_3": 0.11591, + "precision_at_5": 0.08247, + "recall_at_1": 0.2175, + "recall_at_10": 0.50718, + "recall_at_100": 0.80839, + "recall_at_1000": 0.96492, + "recall_at_20": 0.60324, + "recall_at_3": 0.34773, + "recall_at_5": 0.41235 + } + ] + }, + "task_name": "Flickr30kT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RP2kI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RP2kI2IRetrieval.json new file mode 100644 index 0000000000..eafeff620f --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RP2kI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f8f82d4eb1aa4dc4dbf2c768596c8110a3703765", + "evaluation_time": 324.14615392684937, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.55887, + "cv_recall_at_10": 0.77913, + "cv_recall_at_100": 0.90418, + "cv_recall_at_1000": 0.95933, + "cv_recall_at_20": 0.82554, + "cv_recall_at_3": 0.6808, + "cv_recall_at_5": 0.72537, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.55887, + "map_at_1": 0.06083, + "map_at_10": 0.12431, + "map_at_100": 0.17952, + "map_at_1000": 0.2009, + "map_at_20": 0.14322, + "map_at_3": 0.09136, + "map_at_5": 0.10532, + "mrr_at_1": 0.9998166867438328, + "mrr_at_10": 0.9998952495679044, + "mrr_at_100": 0.9998952495679044, + "mrr_at_1000": 0.9998952495679044, + "mrr_at_20": 0.9998952495679044, + "mrr_at_3": 0.9998952495679044, + "mrr_at_5": 0.9998952495679044, + "nauc_cv_recall_at_1000_diff1": -0.6534775051605857, + "nauc_cv_recall_at_1000_max": 0.016875483355911193, + "nauc_cv_recall_at_1000_std": 0.1834914123790724, + "nauc_cv_recall_at_100_diff1": -0.46190450882692813, + "nauc_cv_recall_at_100_max": 0.0044555049072486745, + "nauc_cv_recall_at_100_std": 0.3217134902493942, + "nauc_cv_recall_at_10_diff1": -0.3197493491364096, + "nauc_cv_recall_at_10_max": 0.004081705009985084, + "nauc_cv_recall_at_10_std": 0.3833079134677583, + "nauc_cv_recall_at_1_diff1": -0.2732166234186543, + "nauc_cv_recall_at_1_max": -0.010034973300874108, + "nauc_cv_recall_at_1_std": 0.3494947305628685, + "nauc_cv_recall_at_20_diff1": -0.3517887547295718, + "nauc_cv_recall_at_20_max": 0.006552844210252164, + "nauc_cv_recall_at_20_std": 0.3793238219190742, + "nauc_cv_recall_at_3_diff1": -0.28039644458970836, + "nauc_cv_recall_at_3_max": 0.00026932680904395343, + "nauc_cv_recall_at_3_std": 0.3777753345990772, + "nauc_cv_recall_at_5_diff1": -0.29514123944001847, + "nauc_cv_recall_at_5_max": 0.0002103622905348435, + "nauc_cv_recall_at_5_std": 0.3862948975183731, + "nauc_map_at_1000_diff1": 0.08734717040767775, + "nauc_map_at_1000_max": -0.010675299588909455, + "nauc_map_at_1000_std": 0.41490480554200815, + "nauc_map_at_100_diff1": 0.135072420725819, + "nauc_map_at_100_max": -0.00837091280878698, + "nauc_map_at_100_std": 0.35005558541793835, + "nauc_map_at_10_diff1": 0.26950863371478956, + "nauc_map_at_10_max": -0.004745789557209113, + "nauc_map_at_10_std": 0.14275927317472537, + "nauc_map_at_1_diff1": 0.40347502895694837, + "nauc_map_at_1_max": -0.006070639716117982, + "nauc_map_at_1_std": 0.01609734274601474, + "nauc_map_at_20_diff1": 0.22773030861533414, + "nauc_map_at_20_max": -0.005347115187225529, + "nauc_map_at_20_std": 0.2096685956925957, + "nauc_map_at_3_diff1": 0.3350402994842465, + "nauc_map_at_3_max": -0.002361981770631997, + "nauc_map_at_3_std": 0.05928879397905913, + "nauc_map_at_5_diff1": 0.30786980093898875, + "nauc_map_at_5_max": -0.0033167682834259605, + "nauc_map_at_5_std": 0.08902137115070434, + "nauc_mrr_at_1000_diff1": 0.7693741473921198, + "nauc_mrr_at_1000_max": -0.07329673170966303, + "nauc_mrr_at_1000_std": -0.41562237775388855, + "nauc_mrr_at_100_diff1": 0.7693741473921198, + "nauc_mrr_at_100_max": -0.07329673170966303, + "nauc_mrr_at_100_std": -0.41562237775388855, + "nauc_mrr_at_10_diff1": 0.7693741473921198, + "nauc_mrr_at_10_max": -0.07329673170966303, + "nauc_mrr_at_10_std": -0.41562237775388855, + "nauc_mrr_at_1_diff1": 0.7976521093473791, + "nauc_mrr_at_1_max": -0.05649010070984422, + "nauc_mrr_at_1_std": -0.43184586481189485, + "nauc_mrr_at_20_diff1": 0.7693741473921198, + "nauc_mrr_at_20_max": -0.07329673170966303, + "nauc_mrr_at_20_std": -0.41562237775388855, + "nauc_mrr_at_3_diff1": 0.7693741473921198, + "nauc_mrr_at_3_max": -0.07329673170966303, + "nauc_mrr_at_3_std": -0.41562237775388855, + "nauc_mrr_at_5_diff1": 0.7693741473921198, + "nauc_mrr_at_5_max": -0.07329673170966303, + "nauc_mrr_at_5_std": -0.41562237775388855, + "nauc_ndcg_at_1000_diff1": 0.009920350828696455, + "nauc_ndcg_at_1000_max": -0.010774540419665588, + "nauc_ndcg_at_1000_std": 0.43694481975295574, + "nauc_ndcg_at_100_diff1": 0.018084517333648395, + "nauc_ndcg_at_100_max": -0.0147514814443795, + "nauc_ndcg_at_100_std": 0.46744646393565903, + "nauc_ndcg_at_10_diff1": -0.16407446859272273, + "nauc_ndcg_at_10_max": -0.01784761939283373, + "nauc_ndcg_at_10_std": 0.45860760438003595, + "nauc_ndcg_at_1_diff1": 0.7976521093473791, + "nauc_ndcg_at_1_max": -0.11251153162885205, + "nauc_ndcg_at_1_std": -0.43184586481189485, + "nauc_ndcg_at_20_diff1": -0.1180606939919965, + "nauc_ndcg_at_20_max": -0.017164859556287968, + "nauc_ndcg_at_20_std": 0.48562601519497933, + "nauc_ndcg_at_3_diff1": -0.21591401825959575, + "nauc_ndcg_at_3_max": -0.013782741144658254, + "nauc_ndcg_at_3_std": 0.38832357405024914, + "nauc_ndcg_at_5_diff1": -0.19548651074158818, + "nauc_ndcg_at_5_max": -0.017383534701624292, + "nauc_ndcg_at_5_std": 0.42227867362587224, + "nauc_precision_at_1000_diff1": -0.26457930829821386, + "nauc_precision_at_1000_max": -0.019625850952636456, + "nauc_precision_at_1000_std": 0.44487310862894697, + "nauc_precision_at_100_diff1": -0.28070930263126453, + "nauc_precision_at_100_max": -0.0183983445570901, + "nauc_precision_at_100_std": 0.5216852457299462, + "nauc_precision_at_10_diff1": -0.2642289169634242, + "nauc_precision_at_10_max": -0.016037018440131376, + "nauc_precision_at_10_std": 0.43775176956816353, + "nauc_precision_at_1_diff1": 0.7976521093473791, + "nauc_precision_at_1_max": -0.11251153162885205, + "nauc_precision_at_1_std": -0.43184586481189485, + "nauc_precision_at_20_diff1": -0.26818857979333577, + "nauc_precision_at_20_max": -0.015097011478949736, + "nauc_precision_at_20_std": 0.47166450463929854, + "nauc_precision_at_3_diff1": -0.2643174309717343, + "nauc_precision_at_3_max": -0.01229104802996264, + "nauc_precision_at_3_std": 0.37315288392656526, + "nauc_precision_at_5_diff1": -0.2618738011679915, + "nauc_precision_at_5_max": -0.016386565682558307, + "nauc_precision_at_5_std": 0.4022744328150495, + "nauc_recall_at_1000_diff1": 0.02988142348682087, + "nauc_recall_at_1000_max": -0.006736994916843787, + "nauc_recall_at_1000_std": 0.38393510595562075, + "nauc_recall_at_100_diff1": 0.12754685099331003, + "nauc_recall_at_100_max": -0.005988478824620124, + "nauc_recall_at_100_std": 0.3383849668377813, + "nauc_recall_at_10_diff1": 0.27252371808491316, + "nauc_recall_at_10_max": -0.00514889254539631, + "nauc_recall_at_10_std": 0.1385644816305786, + "nauc_recall_at_1_diff1": 0.40347502895694837, + "nauc_recall_at_1_max": -0.006070639716117982, + "nauc_recall_at_1_std": 0.01609734274601474, + "nauc_recall_at_20_diff1": 0.23198661607389173, + "nauc_recall_at_20_max": -0.0036289993571175373, + "nauc_recall_at_20_std": 0.20018947547570426, + "nauc_recall_at_3_diff1": 0.3355751433312504, + "nauc_recall_at_3_max": -0.001792243329487425, + "nauc_recall_at_3_std": 0.0593931739832833, + "nauc_recall_at_5_diff1": 0.30950726685875857, + "nauc_recall_at_5_max": -0.003205556081532953, + "nauc_recall_at_5_std": 0.08819687899167265, + "ndcg_at_1": 0.99982, + "ndcg_at_10": 0.5575, + "ndcg_at_100": 0.36797, + "ndcg_at_1000": 0.41673, + "ndcg_at_20": 0.475, + "ndcg_at_3": 0.7517, + "ndcg_at_5": 0.66112, + "precision_at_1": 0.99982, + "precision_at_10": 0.42989, + "precision_at_100": 0.15119, + "precision_at_1000": 0.03396, + "precision_at_20": 0.32737, + "precision_at_3": 0.67203, + "precision_at_5": 0.557, + "recall_at_1": 0.06083, + "recall_at_10": 0.13643, + "recall_at_100": 0.26298, + "recall_at_1000": 0.44653, + "recall_at_20": 0.16902, + "recall_at_3": 0.09249, + "recall_at_5": 0.10951 + } + ] + }, + "task_name": "RP2kI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json index 6c038f69cf..77f92463bd 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json @@ -1,184 +1,184 @@ { - "dataset_revision": "0b3a1622902e6258425e673405bdfb1e5dfa8618", - "evaluation_time": 510.43445205688477, + "dataset_revision": "4ac3894bdabee3c3938cf0133ab991c4b501891d", + "evaluation_time": 1411.4596996307373, "kg_co2_emissions": null, "mteb_version": "1.12.90", "scores": { "test": [ { - "cv_recall_at_1": 0.50296, - "cv_recall_at_10": 0.67867, - "cv_recall_at_100": 0.80207, - "cv_recall_at_1000": 0.90797, - "cv_recall_at_20": 0.71918, - "cv_recall_at_3": 0.59883, - "cv_recall_at_5": 0.63223, + "cv_recall_at_1": 0.4856, + "cv_recall_at_10": 0.65892, + "cv_recall_at_100": 0.80265, + "cv_recall_at_1000": 0.91906, + "cv_recall_at_20": 0.70484, + "cv_recall_at_3": 0.57481, + "cv_recall_at_5": 0.61136, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.50296, - "map_at_1": 0.01743, - "map_at_10": 0.18702, - "map_at_100": 0.19992, - "map_at_1000": 0.20163, - "map_at_20": 0.19415, - "map_at_3": 0.13926, - "map_at_5": 0.16856, - "mrr_at_1": 0.020809401784925335, - "mrr_at_10": 0.30757755224265354, - "mrr_at_100": 0.3127412453176731, - "mrr_at_1000": 0.3131742235270453, - "mrr_at_20": 0.3106655733183114, - "mrr_at_3": 0.28876174486759937, - "mrr_at_5": 0.30027318782951246, - "nauc_cv_recall_at_1000_diff1": -0.16859046840753472, - "nauc_cv_recall_at_1000_max": 0.025825747180792717, - "nauc_cv_recall_at_1000_std": 0.10093428974009021, - "nauc_cv_recall_at_100_diff1": -0.21029734576768694, - "nauc_cv_recall_at_100_max": 0.014647522560512015, - "nauc_cv_recall_at_100_std": 0.0507444053462594, - "nauc_cv_recall_at_10_diff1": -0.27159676443421377, - "nauc_cv_recall_at_10_max": 0.005491087955239775, - "nauc_cv_recall_at_10_std": 0.02474251556064112, - "nauc_cv_recall_at_1_diff1": -0.2932928014798219, - "nauc_cv_recall_at_1_max": -0.002633536711876493, - "nauc_cv_recall_at_1_std": -0.008456158086326292, - "nauc_cv_recall_at_20_diff1": -0.25166875961614193, - "nauc_cv_recall_at_20_max": 0.006747419978420269, - "nauc_cv_recall_at_20_std": 0.035987294094302555, - "nauc_cv_recall_at_3_diff1": -0.29779984100144863, - "nauc_cv_recall_at_3_max": -0.0035652648610872604, - "nauc_cv_recall_at_3_std": 0.014056845458600668, - "nauc_cv_recall_at_5_diff1": -0.2888857057827477, - "nauc_cv_recall_at_5_max": 7.1464099594732e-05, - "nauc_cv_recall_at_5_std": 0.01823034964853647, - "nauc_map_at_1000_diff1": -0.2714835648506667, - "nauc_map_at_1000_max": 0.0048783333927581995, - "nauc_map_at_1000_std": 0.10933153351257194, - "nauc_map_at_100_diff1": -0.27106477649697597, - "nauc_map_at_100_max": 0.004961279421023488, - "nauc_map_at_100_std": 0.10869233131948694, - "nauc_map_at_10_diff1": -0.27101224264468865, - "nauc_map_at_10_max": 0.003129881807984945, - "nauc_map_at_10_std": 0.09970024568152357, - "nauc_map_at_1_diff1": -0.5746988023835725, - "nauc_map_at_1_max": 0.009513570955681795, - "nauc_map_at_1_std": 0.24050607736711668, - "nauc_map_at_20_diff1": -0.27103409286809227, - "nauc_map_at_20_max": 0.004505986343978161, - "nauc_map_at_20_std": 0.10422239017835391, - "nauc_map_at_3_diff1": -0.25758853794935105, - "nauc_map_at_3_max": 0.0003165186809392022, - "nauc_map_at_3_std": 0.073787815755662, - "nauc_map_at_5_diff1": -0.26840871915761827, - "nauc_map_at_5_max": 0.002542520569357188, - "nauc_map_at_5_std": 0.0864360224226272, - "nauc_mrr_at_1000_diff1": -0.352973203440373, - "nauc_mrr_at_1000_max": 0.004279169460655329, - "nauc_mrr_at_1000_std": 0.05010168406176362, - "nauc_mrr_at_100_diff1": -0.35282838896266405, - "nauc_mrr_at_100_max": 0.004246244348126371, - "nauc_mrr_at_100_std": 0.0500103093583224, - "nauc_mrr_at_10_diff1": -0.3544263028903223, - "nauc_mrr_at_10_max": 0.003929367832392052, - "nauc_mrr_at_10_std": 0.0478252958551306, - "nauc_mrr_at_1_diff1": -0.5746988023835726, - "nauc_mrr_at_1_max": 0.029047376939523493, - "nauc_mrr_at_1_std": 0.2870259521668249, - "nauc_mrr_at_20_diff1": -0.35339280008168317, - "nauc_mrr_at_20_max": 0.004175461034327089, - "nauc_mrr_at_20_std": 0.049148248185835845, - "nauc_mrr_at_3_diff1": -0.3591723252785509, - "nauc_mrr_at_3_max": 0.00403104720219049, - "nauc_mrr_at_3_std": 0.0404960811268073, - "nauc_mrr_at_5_diff1": -0.3585645562893135, - "nauc_mrr_at_5_max": 0.0020548539639942804, - "nauc_mrr_at_5_std": 0.045522727902755446, - "nauc_ndcg_at_1000_diff1": -0.2685009406444735, - "nauc_ndcg_at_1000_max": 0.005836111353149481, - "nauc_ndcg_at_1000_std": 0.11311294743168826, - "nauc_ndcg_at_100_diff1": -0.2626615425913708, - "nauc_ndcg_at_100_max": 0.006500040666350577, - "nauc_ndcg_at_100_std": 0.10412717822836703, - "nauc_ndcg_at_10_diff1": -0.26986440689503255, - "nauc_ndcg_at_10_max": 0.002259064902287912, - "nauc_ndcg_at_10_std": 0.08351960465910635, - "nauc_ndcg_at_1_diff1": -0.5746988023835727, - "nauc_ndcg_at_1_max": 0.01860068455415359, - "nauc_ndcg_at_1_std": 0.24591999227187353, - "nauc_ndcg_at_20_diff1": -0.26557544406085987, - "nauc_ndcg_at_20_max": 0.0047811746525403785, - "nauc_ndcg_at_20_std": 0.09053231502866856, - "nauc_ndcg_at_3_diff1": -0.3079122639483788, - "nauc_ndcg_at_3_max": 0.001817933912863451, - "nauc_ndcg_at_3_std": 0.05331981166099078, - "nauc_ndcg_at_5_diff1": -0.29161633386458363, - "nauc_ndcg_at_5_max": 0.00323633286273378, - "nauc_ndcg_at_5_std": 0.07076540546903416, - "nauc_precision_at_1000_diff1": -0.13140300790068282, - "nauc_precision_at_1000_max": 0.005922440105917975, - "nauc_precision_at_1000_std": 0.08342438590882045, - "nauc_precision_at_100_diff1": -0.17052567611996527, - "nauc_precision_at_100_max": 0.011026201381103823, - "nauc_precision_at_100_std": 0.09223669236894032, - "nauc_precision_at_10_diff1": -0.23209493838034498, - "nauc_precision_at_10_max": 0.004908243372756972, - "nauc_precision_at_10_std": 0.07007542762563608, - "nauc_precision_at_1_diff1": -0.5746988023835727, - "nauc_precision_at_1_max": 0.01860068455415359, - "nauc_precision_at_1_std": 0.24591999227187353, - "nauc_precision_at_20_diff1": -0.2109568972104465, - "nauc_precision_at_20_max": 0.007714408837495573, - "nauc_precision_at_20_std": 0.07965290786741164, - "nauc_precision_at_3_diff1": -0.2870525654193057, - "nauc_precision_at_3_max": -0.0007584139080774421, - "nauc_precision_at_3_std": 0.0318750842480405, - "nauc_precision_at_5_diff1": -0.25948022131281406, - "nauc_precision_at_5_max": 0.004692362608818381, - "nauc_precision_at_5_std": 0.04879698010109281, - "nauc_recall_at_1000_diff1": -0.12358858307301535, - "nauc_recall_at_1000_max": 0.009306080801565777, - "nauc_recall_at_1000_std": 0.15591617596817545, - "nauc_recall_at_100_diff1": -0.14715964976867443, - "nauc_recall_at_100_max": 0.01041920443322213, - "nauc_recall_at_100_std": 0.11606682272338587, - "nauc_recall_at_10_diff1": -0.18845161575089034, - "nauc_recall_at_10_max": 0.000650022994426268, - "nauc_recall_at_10_std": 0.07614315809536469, - "nauc_recall_at_1_diff1": -0.5746988023835725, - "nauc_recall_at_1_max": 0.009513570955681795, - "nauc_recall_at_1_std": 0.24050607736711668, - "nauc_recall_at_20_diff1": -0.17338195050795077, - "nauc_recall_at_20_max": 0.006581585832690798, - "nauc_recall_at_20_std": 0.08663740977783436, - "nauc_recall_at_3_diff1": -0.19307909748063756, - "nauc_recall_at_3_max": 0.0003553588908448416, - "nauc_recall_at_3_std": 0.0442289537166574, - "nauc_recall_at_5_diff1": -0.19899176174509764, - "nauc_recall_at_5_max": -0.0005095021106718202, - "nauc_recall_at_5_std": 0.060914430592624226, - "ndcg_at_1": 0.03583, - "ndcg_at_10": 0.27789, - "ndcg_at_100": 0.32166, - "ndcg_at_1000": 0.35384, - "ndcg_at_20": 0.29399, - "ndcg_at_3": 0.26171, - "ndcg_at_5": 0.27031, - "precision_at_1": 0.03583, - "precision_at_10": 0.13151, - "precision_at_100": 0.01932, - "precision_at_1000": 0.0027, - "precision_at_20": 0.07531, - "precision_at_3": 0.26493, - "precision_at_5": 0.21197, - "recall_at_1": 0.01743, - "recall_at_10": 0.3639, - "recall_at_100": 0.50674, - "recall_at_1000": 0.67803, - "recall_at_20": 0.40787, - "recall_at_3": 0.24952, - "recall_at_5": 0.30796 + "main_score": 0.4856, + "map_at_1": 0.18847, + "map_at_10": 0.37878, + "map_at_100": 0.39701, + "map_at_1000": 0.39955, + "map_at_20": 0.38884, + "map_at_3": 0.31574, + "map_at_5": 0.35297, + "mrr_at_1": 0.9996334952062839, + "mrr_at_10": 0.9998132769138076, + "mrr_at_100": 0.9998132769138076, + "mrr_at_1000": 0.9998132769138076, + "mrr_at_20": 0.9998132769138076, + "mrr_at_3": 0.999811194500207, + "mrr_at_5": 0.9998132769138076, + "nauc_cv_recall_at_1000_diff1": -0.4588738282028226, + "nauc_cv_recall_at_1000_max": -0.00045154026330132926, + "nauc_cv_recall_at_1000_std": 0.13389839300845494, + "nauc_cv_recall_at_100_diff1": -0.40349410035175093, + "nauc_cv_recall_at_100_max": -0.0015857946226738907, + "nauc_cv_recall_at_100_std": 0.1564430560934902, + "nauc_cv_recall_at_10_diff1": -0.4005181992520635, + "nauc_cv_recall_at_10_max": -0.003957796532778069, + "nauc_cv_recall_at_10_std": 0.14566052153014458, + "nauc_cv_recall_at_1_diff1": -0.40652510568056555, + "nauc_cv_recall_at_1_max": -0.00544324116729814, + "nauc_cv_recall_at_1_std": 0.11520889475416408, + "nauc_cv_recall_at_20_diff1": -0.3973805526446551, + "nauc_cv_recall_at_20_max": -0.0046404467731546205, + "nauc_cv_recall_at_20_std": 0.14736528921128944, + "nauc_cv_recall_at_3_diff1": -0.4068997028240004, + "nauc_cv_recall_at_3_max": -0.00581379693442588, + "nauc_cv_recall_at_3_std": 0.13232980933178856, + "nauc_cv_recall_at_5_diff1": -0.4027824500878659, + "nauc_cv_recall_at_5_max": -0.00622124305658329, + "nauc_cv_recall_at_5_std": 0.13795375991899358, + "nauc_map_at_1000_diff1": -0.2815134589062327, + "nauc_map_at_1000_max": -0.009309150035044157, + "nauc_map_at_1000_std": 0.21821074035964452, + "nauc_map_at_100_diff1": -0.2802916020865774, + "nauc_map_at_100_max": -0.00929273233548749, + "nauc_map_at_100_std": 0.21738215325378304, + "nauc_map_at_10_diff1": -0.2714998577314571, + "nauc_map_at_10_max": -0.00972009151554314, + "nauc_map_at_10_std": 0.20451609552118097, + "nauc_map_at_1_diff1": 0.006605172707482505, + "nauc_map_at_1_max": -0.00535293182577402, + "nauc_map_at_1_std": 0.08912323354604411, + "nauc_map_at_20_diff1": -0.276631619640432, + "nauc_map_at_20_max": -0.009275901369492237, + "nauc_map_at_20_std": 0.21273445874298258, + "nauc_map_at_3_diff1": -0.2144365987666858, + "nauc_map_at_3_max": -0.008565184161123921, + "nauc_map_at_3_std": 0.1490404208758112, + "nauc_map_at_5_diff1": -0.2525988859250142, + "nauc_map_at_5_max": -0.00899732355635905, + "nauc_map_at_5_std": 0.18031989165161078, + "nauc_mrr_at_1000_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.17398421274750922, + "nauc_mrr_at_1000_std": -0.3793244588442443, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_100_max": 0.17398421274750922, + "nauc_mrr_at_100_std": -0.3793244588442443, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_10_max": 0.17398421274750922, + "nauc_mrr_at_10_std": -0.3793244588442443, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_1_max": 0.17317138790943362, + "nauc_mrr_at_1_std": -0.36469583799829147, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_20_max": 0.17398421274750922, + "nauc_mrr_at_20_std": -0.3793244588442443, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_3_max": 0.17341951004597636, + "nauc_mrr_at_3_std": -0.3878434792194687, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_5_max": 0.17398421274750922, + "nauc_mrr_at_5_std": -0.3793244588442443, + "nauc_ndcg_at_1000_diff1": -0.3092675026331808, + "nauc_ndcg_at_1000_max": -0.009597931782047127, + "nauc_ndcg_at_1000_std": 0.23456527611896652, + "nauc_ndcg_at_100_diff1": -0.29597233628106967, + "nauc_ndcg_at_100_max": -0.009309685183289425, + "nauc_ndcg_at_100_std": 0.22752615017083713, + "nauc_ndcg_at_10_diff1": -0.2930380020320867, + "nauc_ndcg_at_10_max": -0.01000547366692856, + "nauc_ndcg_at_10_std": 0.20753291775628463, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_1_max": 0.17610023993447146, + "nauc_ndcg_at_1_std": -0.36469583799829147, + "nauc_ndcg_at_20_diff1": -0.28823218777084936, + "nauc_ndcg_at_20_max": -0.00923331182370813, + "nauc_ndcg_at_20_std": 0.215531645591675, + "nauc_ndcg_at_3_diff1": -0.3656155045971337, + "nauc_ndcg_at_3_max": -0.007681435647815146, + "nauc_ndcg_at_3_std": 0.14037876964394946, + "nauc_ndcg_at_5_diff1": -0.3365107912821576, + "nauc_ndcg_at_5_max": -0.008614755602549906, + "nauc_ndcg_at_5_std": 0.18007814023447868, + "nauc_precision_at_1000_diff1": -0.19806317219535188, + "nauc_precision_at_1000_max": -0.0015221284698821193, + "nauc_precision_at_1000_std": 0.10530172591311254, + "nauc_precision_at_100_diff1": -0.2572749977303199, + "nauc_precision_at_100_max": -0.003787347697600317, + "nauc_precision_at_100_std": 0.1656157857683021, + "nauc_precision_at_10_diff1": -0.3101437993491914, + "nauc_precision_at_10_max": -0.007420244456215511, + "nauc_precision_at_10_std": 0.17668167529658718, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_1_max": 0.17610023993447146, + "nauc_precision_at_1_std": -0.36469583799829147, + "nauc_precision_at_20_diff1": -0.29472889748829945, + "nauc_precision_at_20_max": -0.005520841437776455, + "nauc_precision_at_20_std": 0.18138281023915606, + "nauc_precision_at_3_diff1": -0.35841782500948777, + "nauc_precision_at_3_max": -0.00748476584740033, + "nauc_precision_at_3_std": 0.12872707729510002, + "nauc_precision_at_5_diff1": -0.3295916303527328, + "nauc_precision_at_5_max": -0.007145664789792643, + "nauc_precision_at_5_std": 0.15297931876842116, + "nauc_recall_at_1000_diff1": -0.25792280904882936, + "nauc_recall_at_1000_max": -0.00797884188493623, + "nauc_recall_at_1000_std": 0.24244025939864458, + "nauc_recall_at_100_diff1": -0.2524975921657061, + "nauc_recall_at_100_max": -0.007586553109911903, + "nauc_recall_at_100_std": 0.23439228371650886, + "nauc_recall_at_10_diff1": -0.2562100752922837, + "nauc_recall_at_10_max": -0.009867846823865567, + "nauc_recall_at_10_std": 0.20691424138363798, + "nauc_recall_at_1_diff1": 0.006605172707482505, + "nauc_recall_at_1_max": -0.00535293182577402, + "nauc_recall_at_1_std": 0.08912323354604411, + "nauc_recall_at_20_diff1": -0.2560836034546459, + "nauc_recall_at_20_max": -0.007754724440480409, + "nauc_recall_at_20_std": 0.21910103605653863, + "nauc_recall_at_3_diff1": -0.21039105024165025, + "nauc_recall_at_3_max": -0.008374958963214659, + "nauc_recall_at_3_std": 0.1489992487923535, + "nauc_recall_at_5_diff1": -0.242995017130598, + "nauc_recall_at_5_max": -0.00909414443674345, + "nauc_recall_at_5_std": 0.18059416206675072, + "ndcg_at_1": 0.99963, + "ndcg_at_10": 0.52441, + "ndcg_at_100": 0.5623, + "ndcg_at_1000": 0.59803, + "ndcg_at_20": 0.53321, + "ndcg_at_3": 0.69421, + "ndcg_at_5": 0.59023, + "precision_at_1": 0.99963, + "precision_at_10": 0.24637, + "precision_at_100": 0.03343, + "precision_at_1000": 0.00451, + "precision_at_20": 0.13636, + "precision_at_3": 0.59241, + "precision_at_5": 0.42271, + "recall_at_1": 0.18847, + "recall_at_10": 0.41136, + "recall_at_100": 0.52984, + "recall_at_1000": 0.6822, + "recall_at_20": 0.44681, + "recall_at_3": 0.31926, + "recall_at_5": 0.3657 } ] }, From f1fe91f586629a7c88d6f4e32258cf2bdec99d71 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Tue, 8 Oct 2024 06:42:37 +0300 Subject: [PATCH 072/154] [MIEB] Adding DataComp CLIP models (#1283) * adding data comp CLIP models * update model and caltech101 results * make lint --- mteb/encoder_interface.py | 1 + mteb/models/__init__.py | 2 + mteb/models/datacomp_clip.py | 151 ++++++++++++++++++ .../Caltech101.json | 48 ++++++ .../model_meta.json | 1 + 5 files changed, 203 insertions(+) create mode 100644 mteb/models/datacomp_clip.py create mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/Caltech101.json create mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json diff --git a/mteb/encoder_interface.py b/mteb/encoder_interface.py index 5920143105..21ef24cbba 100644 --- a/mteb/encoder_interface.py +++ b/mteb/encoder_interface.py @@ -22,6 +22,7 @@ def __init__(self, device: str | None = None) -> None: Args: device: The device to use for encoding. Can be ignored if the encoder is not using a device (e.g. for API) """ + self.device = device def encode( self, sentences: Sequence[str], *, prompt_name: str | None = None, **kwargs: Any diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index eabe5a2d3f..33b83f9a11 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -15,6 +15,7 @@ bm25, clip_models, cohere_models, + datacomp_clip, dino_models, e5_instruct, e5_models, @@ -136,6 +137,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe blip2_models, bm25, cohere_models, + datacomp_clip, dino_models, e5_instruct, e5_models, diff --git a/mteb/models/datacomp_clip.py b/mteb/models/datacomp_clip.py new file mode 100644 index 0000000000..cca9bf5c1a --- /dev/null +++ b/mteb/models/datacomp_clip.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import AutoModel, AutoProcessor + +from mteb.model_meta import ModelMeta + + +class DataCLIPModelWrapper: + def __init__( + self, + model_name: str = "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + self.model = AutoModel.from_pretrained(model_name, trust_remote_code=True).to( + self.device + ) + self.processor = AutoProcessor.from_pretrained(model_name) + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + inputs = self.processor( + text=batch_texts, return_tensors="pt", padding=True, truncation=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + text_outputs = self.model.get_text_features(**inputs) + all_text_embeddings.append(text_outputs.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + all_image_embeddings = [] + if isinstance(images, DataLoader): + with torch.no_grad(): + for batch in tqdm(images): + inputs = self.processor( + images=batch, return_tensors="pt", padding=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model.get_image_features(**inputs) + all_image_embeddings.append(image_outputs.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + inputs = self.processor( + images=batch_images, return_tensors="pt", padding=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model.get_image_features(**inputs) + all_image_embeddings.append(image_outputs.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + else: + # to do: add other fusion mode + raise ValueError(f"fusion mode {fusion_mode} hasn't been implemented") + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + +datacomp_clip_vit_large_patch14 = ModelMeta( + loader=partial( + DataCLIPModelWrapper, + model_name="laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + ), + name="laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + languages=["eng_Latn"], + open_source=True, + revision="84c9828e63dc9a9351d1fe637c346d4c1c4db341", + release_date="2023-04-26", +) + +datacomp_clip_vit_base_patch32 = ModelMeta( + loader=partial( + DataCLIPModelWrapper, + model_name="laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", + ), + name="laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", + languages=["eng_Latn"], + open_source=True, + revision="f0e2ffa09cbadab3db6a261ec1ec56407ce42912", + release_date="2023-04-26", +) + +datacomp_clip_vit_base_patch16 = ModelMeta( + loader=partial( + DataCLIPModelWrapper, + model_name="laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", + ), + name="laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", + languages=["eng_Latn"], + open_source=True, + revision="d110532e8d4ff91c574ee60a342323f28468b287", + release_date="2023-04-26", +) diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/Caltech101.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/Caltech101.json new file mode 100644 index 0000000000..84d016d4f2 --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/Caltech101.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", + "evaluation_time": 336.7735936641693, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.9634451019066403, + "f1": 0.9390393783666946, + "f1_weighted": 0.9629340674668043, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9634451019066403, + "scores_per_experiment": [ + { + "accuracy": 0.9640039447731755, + "f1": 0.9387417673075573, + "f1_weighted": 0.963701162627355 + }, + { + "accuracy": 0.965483234714004, + "f1": 0.9412686220165988, + "f1_weighted": 0.9654077609459234 + }, + { + "accuracy": 0.9598948060486522, + "f1": 0.9360200534285099, + "f1_weighted": 0.9589670335376868 + }, + { + "accuracy": 0.9617028270874425, + "f1": 0.9417554557342505, + "f1_weighted": 0.9606672076394908 + }, + { + "accuracy": 0.9661406969099277, + "f1": 0.9374109933465566, + "f1_weighted": 0.9659271725835652 + } + ] + } + ] + }, + "task_name": "Caltech101" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json new file mode 100644 index 0000000000..76c7344f9e --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json @@ -0,0 +1 @@ +{"name": "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "revision": "84c9828e63dc9a9351d1fe637c346d4c1c4db341", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DataCLIPModelWrapper"} \ No newline at end of file From b0bc4e20c2dbe123b99e19cff436d9d1355d44cd Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:38:35 +0800 Subject: [PATCH 073/154] [mieb] Any2TextMultipleChoice Abstask&Evaluator & four tasks in CV-bench (#1287) * fix ImageTextPair dataloading for large datasets; more compositionality evaluation datasets * fix meta data * fix validate points * CV-Bench * evaluator args comment * fix --------- Co-authored-by: Isaac Chung --- .../Image/AbsTaskAny2TextMultipleChoice.py | 65 +++++ mteb/abstasks/TaskMetadata.py | 1 + mteb/abstasks/__init__.py | 1 + .../Image/Any2TextMultipleChoiceEvaluator.py | 99 +++++++ mteb/evaluation/evaluators/__init__.py | 1 + .../Image/Any2TextMultipleChoice/__init__.py | 3 + .../Any2TextMultipleChoice/eng/CVBench.py | 258 ++++++++++++++++++ .../Any2TextMultipleChoice/eng/__init__.py | 0 mteb/tasks/Image/__init__.py | 1 + .../CVBenchCount.json | 19 ++ .../CVBenchDepth.json | 19 ++ .../CVBenchDistance.json | 19 ++ .../CVBenchRelation.json | 19 ++ .../CVBenchCount.json | 19 ++ .../CVBenchDepth.json | 19 ++ .../CVBenchDistance.json | 19 ++ .../CVBenchRelation.json | 19 ++ .../CVBenchCount.json | 19 ++ .../CVBenchDepth.json | 19 ++ .../CVBenchDistance.json | 19 ++ .../CVBenchRelation.json | 19 ++ 21 files changed, 657 insertions(+) create mode 100644 mteb/abstasks/Image/AbsTaskAny2TextMultipleChoice.py create mode 100644 mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py create mode 100644 mteb/tasks/Image/Any2TextMultipleChoice/__init__.py create mode 100644 mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py create mode 100644 mteb/tasks/Image/Any2TextMultipleChoice/eng/__init__.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchCount.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDepth.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDistance.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchRelation.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchCount.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDepth.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDistance.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchRelation.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchCount.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDepth.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDistance.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchRelation.json diff --git a/mteb/abstasks/Image/AbsTaskAny2TextMultipleChoice.py b/mteb/abstasks/Image/AbsTaskAny2TextMultipleChoice.py new file mode 100644 index 0000000000..6172eae3ff --- /dev/null +++ b/mteb/abstasks/Image/AbsTaskAny2TextMultipleChoice.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import logging +from typing import Any + +from datasets import Dataset + +from ...encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from ...evaluation.evaluators import Any2TextMultipleChoiceEvaluator +from ...load_results.mteb_results import ScoresDict +from ..AbsTask import AbsTask + +logger = logging.getLogger(__name__) + + +class AbsTaskAny2TextMultipleChoice(AbsTask): + """Abstract class for Any to Text Multiple Choice tasks, + where the queries and be either text or image, or both. + This task assess interleaved encoding of queries, + the similarity computed between the queries and the candidate choices is ranked. + + self.load_data() must generate a huggingface dataset with a split matching self.metadata_dict["eval_splits"], and assign it to self.dataset. + """ + + query_modalities: list[str] | str = ["image", "text"] + query_column_names: dict = {"image": "image", "text": "question"} + label_column_name: str = "answer" + choices_column_name: str = "choices" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def _add_main_score(self, scores) -> None: + scores["main_score"] = scores[self.metadata.main_score] + + def _calculate_metrics_from_split( + self, split: str, hf_subset: str | None = None, compute_overall: bool = False + ): + pass + + def _evaluate_subset( + self, + model: Encoder | EncoderWithQueryCorpusEncode, + dataset: Dataset, + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ) -> ScoresDict: + for modality in self.query_modalities: + if modality not in self.query_column_names: + raise KeyError( + f"query column name of modality {modality} is not defined" + ) + evaluator = Any2TextMultipleChoiceEvaluator( + dataset, + query_modalities=self.query_modalities, + query_column_names=self.query_column_names, + label_column_name=self.label_column_name, + choices_column_name=self.choices_column_name, + task_name=self.metadata.name, + **kwargs, + ) + scores = evaluator(model, encode_kwargs=encode_kwargs) + self._add_main_score(scores) + return scores diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 290932dd26..09582d779b 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -100,6 +100,7 @@ "Speed", "ZeroShotClassification", "ImageTextPairClassification", + "Any2TextMutipleChoice", ] TASK_CATEGORY = Literal[ diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index f188430f48..f70cbd5324 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -14,6 +14,7 @@ from .AbsTaskSTS import * from .AbsTaskSummarization import * from .Image.AbsTaskAny2AnyRetrieval import * +from .Image.AbsTaskAny2TextMultipleChoice import * from .Image.AbsTaskImageClassification import * from .Image.AbsTaskImageClustering import * from .Image.AbsTaskImageMultilabelClassification import * diff --git a/mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py b/mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py new file mode 100644 index 0000000000..f682225ba5 --- /dev/null +++ b/mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import logging +from typing import Any + +import numpy as np +import torch +from sklearn.metrics import accuracy_score +from sklearn.metrics.pairwise import cosine_similarity +from torchvision import transforms +from tqdm import tqdm + +from mteb.encoder_interface import Encoder, EncoderWithSimilarity +from mteb.evaluation.evaluators.Evaluator import Evaluator + +logger = logging.getLogger(__name__) + +transform = transforms.Compose([transforms.PILToTensor()]) + + +class Any2TextMultipleChoiceEvaluator(Evaluator): + """Evaluate a model based on the similarity of queries (can be interleaved) and candidate answers. + The goal is to find the correct text in multiple candidates that + forms the correct answer of the interleaved query. + + Args: + query_modalities: the modality of queries; supports image and text or either at the moment, + query_column_names: column names of queries; should align with query modalities. + label_column_name: column name of labels; + choices_column_names: column name of candidate choices; + """ + + def __init__( + self, + dataset, + query_modalities: str | list[str], + query_column_names: dict, + label_column_name: str, + choices_column_name: str, + task_name: str | None = None, + transform=None, + limit: int | None = None, + **kwargs, + ): + super().__init__(**kwargs) + if limit: + dataset = dataset.select(range(limit)) + self.dataset = dataset + self.query_modalities = query_modalities + self.query_column_names = query_column_names + self.label_column_name = label_column_name + self.choices_column_name = choices_column_name + self.task_name = task_name + self.transform = transform + + def __call__( + self, + model: Encoder | EncoderWithSimilarity, + encode_kwargs: dict[str, Any] = {}, + ): + if "batch_size" not in encode_kwargs: + encode_kwargs["batch_size"] = 64 + + label_list = list( + set([x for n in self.dataset[self.choices_column_name] for x in n]) + ) + label_embeddings = model.get_text_embeddings(label_list) + label_embedding_dict = {} + for label, embedding in zip(label_list, label_embeddings): + label_embedding_dict[label] = embedding + + if "text" in self.query_modalities: + questions = self.dataset[self.query_column_names["text"]] + else: + questions = None + if "image" in self.query_modalities: + images = self.dataset[self.query_column_names["image"]] + query_embeddings = model.get_fused_embeddings( + texts=questions, + images=images, + batch_size=encode_kwargs["batch_size"], + ) + + answers = self.dataset[self.label_column_name] + choices = self.dataset[self.choices_column_name] + + # note that answers are the indeces + predictions = [] + for q_embedding, choice in tqdm(zip(query_embeddings, choices)): + choice_embeddings = torch.vstack( + [label_embedding_dict[c] for c in choice] + ) # (choice_size, embedding_dim) + q_embedding = q_embedding[np.newaxis, :] + cos_sim = cosine_similarity(q_embedding, choice_embeddings) + predictions.append(np.argmax(cos_sim)) + + metrics = {} + metrics["accuracy"] = accuracy_score(predictions, answers) + return metrics diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py index ce7da0db59..2fb90b655f 100644 --- a/mteb/evaluation/evaluators/__init__.py +++ b/mteb/evaluation/evaluators/__init__.py @@ -4,6 +4,7 @@ from .ClassificationEvaluator import * from .ClusteringEvaluator import * from .Image.Any2AnyRetrievalEvaluator import * +from .Image.Any2TextMultipleChoiceEvaluator import * from .Image.ClassificationEvaluator import * from .Image.ClusteringEvaluator import * from .Image.ImageTextPairClassificationEvaluator import * diff --git a/mteb/tasks/Image/Any2TextMultipleChoice/__init__.py b/mteb/tasks/Image/Any2TextMultipleChoice/__init__.py new file mode 100644 index 0000000000..e1433ec949 --- /dev/null +++ b/mteb/tasks/Image/Any2TextMultipleChoice/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .eng.CVBench import * diff --git a/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py b/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py new file mode 100644 index 0000000000..e42ec28f76 --- /dev/null +++ b/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +import datasets + +from mteb.abstasks.Image.AbsTaskAny2TextMultipleChoice import ( + AbsTaskAny2TextMultipleChoice, +) +from mteb.abstasks.TaskMetadata import TaskMetadata + + +def transform_choices(example): + mapping = {"(A)": 0, "(B)": 1, "(C)": 2, "(D)": 3, "(E)": 4, "(F)": 5} + example["answer"] = mapping[example["answer"]] + return example + + +class CVBenchCount(AbsTaskAny2TextMultipleChoice): + metadata = TaskMetadata( + name="CVBenchCount", + description="count the number of objects in the image.", + reference="https://arxiv.org/pdf/2406.16860", + dataset={ + "path": "nyu-visionx/CV-Bench", + "revision": "22409a927ab5cf68e3655023d51694587455fc99", + }, + type="Any2TextMutipleChoice", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-01-01", "2024-06-24"), + domains=["Academic"], + task_subtypes=["Question answering"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{wu2024scimmir, + title={placeholder}, + author={placeholder and others}, + journal={arXiv preprint arXiv:2401.13478}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 788}, + "avg_character_length": { + "test": { + # to do + } + }, + }, + ) + + def load_data(self, **kwargs): + self.dataset = datasets.load_dataset(**self.metadata_dict["dataset"]) + self.dataset_transform() + self.dataset = self.dataset.filter(lambda example: example["task"] == "Count") + self.dataset = self.dataset.map( + transform_choices, + remove_columns=[ + "idx", + "type", + "filename", + "source", + "source_dataset", + "source_filename", + "target_class", + "target_size", + "bbox", + "prompt", + ], + ) + self.data_loaded = True + + +class CVBenchRelation(AbsTaskAny2TextMultipleChoice): + metadata = TaskMetadata( + name="CVBenchRelation", + description="decide the relation of the objects in the image.", + reference="https://arxiv.org/pdf/2406.16860", + dataset={ + "path": "nyu-visionx/CV-Bench", + "revision": "22409a927ab5cf68e3655023d51694587455fc99", + }, + type="Any2TextMutipleChoice", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-01-01", "2024-06-24"), + domains=["Academic"], + task_subtypes=["Question answering"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{wu2024scimmir, + title={placeholder}, + author={placeholder and others}, + journal={arXiv preprint arXiv:2401.13478}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 650}, + "avg_character_length": { + "test": { + # to do + } + }, + }, + ) + + def load_data(self, **kwargs): + self.dataset = datasets.load_dataset(**self.metadata_dict["dataset"]) + self.dataset_transform() + self.dataset = self.dataset.filter( + lambda example: example["task"] == "Relation" + ) + self.dataset = self.dataset.map( + transform_choices, + remove_columns=[ + "idx", + "type", + "filename", + "source", + "source_dataset", + "source_filename", + "target_class", + "target_size", + "bbox", + "prompt", + ], + ) + self.data_loaded = True + + +class CVBenchDepth(AbsTaskAny2TextMultipleChoice): + metadata = TaskMetadata( + name="CVBenchDepth", + description="judge the depth of the objects in the image with similarity matching.", + reference="https://arxiv.org/pdf/2406.16860", + dataset={ + "path": "nyu-visionx/CV-Bench", + "revision": "22409a927ab5cf68e3655023d51694587455fc99", + }, + type="Any2TextMutipleChoice", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-01-01", "2024-06-24"), + domains=["Academic"], + task_subtypes=["Question answering"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{wu2024scimmir, + title={placeholder}, + author={placeholder and others}, + journal={arXiv preprint arXiv:2401.13478}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 600}, + "avg_character_length": { + "test": { + # to do + } + }, + }, + ) + + def load_data(self, **kwargs): + self.dataset = datasets.load_dataset(**self.metadata_dict["dataset"]) + self.dataset_transform() + self.dataset = self.dataset.filter(lambda example: example["task"] == "Depth") + self.dataset = self.dataset.map( + transform_choices, + remove_columns=[ + "idx", + "type", + "filename", + "source", + "source_dataset", + "source_filename", + "target_class", + "target_size", + "bbox", + "prompt", + ], + ) + self.data_loaded = True + + +class CVBenchDistance(AbsTaskAny2TextMultipleChoice): + metadata = TaskMetadata( + name="CVBenchDistance", + description="judge the distance of the objects in the image with similarity matching.", + reference="https://arxiv.org/pdf/2406.16860", + dataset={ + "path": "nyu-visionx/CV-Bench", + "revision": "22409a927ab5cf68e3655023d51694587455fc99", + }, + type="Any2TextMutipleChoice", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-01-01", "2024-06-24"), + domains=["Academic"], + task_subtypes=["Question answering"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{tong2024cambrian, + title={Cambrian-1: A fully open, vision-centric exploration of multimodal llms}, + author={Tong, Shengbang and Brown, Ellis and Wu, Penghao and Woo, Sanghyun and Middepogu, Manoj and Akula, Sai Charitha and Yang, Jihan and Yang, Shusheng and Iyer, Adithya and Pan, Xichen and others}, + journal={arXiv preprint arXiv:2406.16860}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"test": 600}, + "avg_character_length": { + "test": { + # to do + } + }, + }, + ) + + def load_data(self, **kwargs): + self.dataset = datasets.load_dataset(**self.metadata_dict["dataset"]) + self.dataset_transform() + self.dataset = self.dataset.filter( + lambda example: example["task"] == "Distance" + ) + self.dataset = self.dataset.map( + transform_choices, + remove_columns=[ + "idx", + "type", + "filename", + "source", + "source_dataset", + "source_filename", + "target_class", + "target_size", + "bbox", + "prompt", + ], + ) + self.data_loaded = True diff --git a/mteb/tasks/Image/Any2TextMultipleChoice/eng/__init__.py b/mteb/tasks/Image/Any2TextMultipleChoice/eng/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Image/__init__.py b/mteb/tasks/Image/__init__.py index cf632fe736..845cc136f3 100644 --- a/mteb/tasks/Image/__init__.py +++ b/mteb/tasks/Image/__init__.py @@ -1,6 +1,7 @@ from __future__ import annotations from .Any2AnyRetrieval import * +from .Any2TextMultipleChoice import * from .Clustering import * from .ImageClassification import * from .ImageMultilabelClassification import * diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchCount.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchCount.json new file mode 100644 index 0000000000..2a11efd947 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchCount.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 12.502729654312134, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.2017766497461929, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.2017766497461929 + } + ] + }, + "task_name": "CVBenchCount" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDepth.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDepth.json new file mode 100644 index 0000000000..b420a0688b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDepth.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 9.667015790939331, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5116666666666667, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5116666666666667 + } + ] + }, + "task_name": "CVBenchDepth" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDistance.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDistance.json new file mode 100644 index 0000000000..742d2cc2d1 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDistance.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 9.924455404281616, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.4633333333333333, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.4633333333333333 + } + ] + }, + "task_name": "CVBenchDistance" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchRelation.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchRelation.json new file mode 100644 index 0000000000..7c9c4d1238 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchRelation.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 6.079412221908569, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5030769230769231, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5030769230769231 + } + ] + }, + "task_name": "CVBenchRelation" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchCount.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchCount.json new file mode 100644 index 0000000000..a888119c3f --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchCount.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 11.933090686798096, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.06598984771573604, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.06598984771573604 + } + ] + }, + "task_name": "CVBenchCount" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDepth.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDepth.json new file mode 100644 index 0000000000..cb647a3405 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDepth.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 8.507234334945679, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.4533333333333333, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.4533333333333333 + } + ] + }, + "task_name": "CVBenchDepth" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDistance.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDistance.json new file mode 100644 index 0000000000..d8246da01d --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDistance.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 8.697332382202148, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.46166666666666667, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.46166666666666667 + } + ] + }, + "task_name": "CVBenchDistance" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchRelation.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchRelation.json new file mode 100644 index 0000000000..02145addc1 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchRelation.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 4.9446187019348145, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.4907692307692308, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.4907692307692308 + } + ] + }, + "task_name": "CVBenchRelation" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchCount.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchCount.json new file mode 100644 index 0000000000..d69a7b62ff --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchCount.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 18.0276620388031, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.027918781725888325, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.027918781725888325 + } + ] + }, + "task_name": "CVBenchCount" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDepth.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDepth.json new file mode 100644 index 0000000000..f6e87d0e07 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDepth.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 14.530367136001587, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5266666666666666, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5266666666666666 + } + ] + }, + "task_name": "CVBenchDepth" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDistance.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDistance.json new file mode 100644 index 0000000000..f8d0a0fc67 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDistance.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 14.808384895324707, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.47, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.47 + } + ] + }, + "task_name": "CVBenchDistance" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchRelation.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchRelation.json new file mode 100644 index 0000000000..21cf37be5b --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchRelation.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "default", + "evaluation_time": 11.672912359237671, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5092307692307693, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5092307692307693 + } + ] + }, + "task_name": "CVBenchRelation" +} \ No newline at end of file From 1b70f6db68512de83dd36f72bec507094e0a4cc3 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Tue, 15 Oct 2024 22:02:59 +0800 Subject: [PATCH 074/154] [mieb] adding 10 tasks (#1290) * fix ImageTextPair dataloading for large datasets; more compositionality evaluation datasets * add vidore benchmark 10 tasks * fix reference * fix old metadata * fix meta --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 1 + .../eng/VidoreBenchRetrieval.py | 621 ++++++++++++++++++ .../VidoreArxivQARetrieval.json | 186 ++++++ .../VidoreDocVQARetrieval.json | 186 ++++++ .../VidoreInfoVQARetrieval.json | 186 ++++++ .../VidoreShiftProjectRetrieval.json | 186 ++++++ .../VidoreSyntheticDocQAAIRetrieval.json | 186 ++++++ .../VidoreSyntheticDocQAEnergyRetrieval.json | 186 ++++++ ...theticDocQAGovernmentReportsRetrieval.json | 186 ++++++ ...heticDocQAHealthcareIndustryRetrieval.json | 186 ++++++ .../VidoreTabfquadRetrieval.json | 186 ++++++ .../VidoreTatdqaRetrieval.json | 186 ++++++ .../VidoreArxivQARetrieval.json | 186 ++++++ .../VidoreDocVQARetrieval.json | 186 ++++++ .../VidoreInfoVQARetrieval.json | 186 ++++++ .../VidoreShiftProjectRetrieval.json | 186 ++++++ .../VidoreSyntheticDocQAAIRetrieval.json | 186 ++++++ .../VidoreSyntheticDocQAEnergyRetrieval.json | 186 ++++++ ...theticDocQAGovernmentReportsRetrieval.json | 186 ++++++ ...heticDocQAHealthcareIndustryRetrieval.json | 186 ++++++ .../VidoreTabfquadRetrieval.json | 186 ++++++ .../VidoreTatdqaRetrieval.json | 186 ++++++ .../VidoreArxivQARetrieval.json | 186 ++++++ .../VidoreDocVQARetrieval.json | 186 ++++++ .../VidoreInfoVQARetrieval.json | 186 ++++++ .../VidoreShiftProjectRetrieval.json | 186 ++++++ .../VidoreSyntheticDocQAAIRetrieval.json | 186 ++++++ .../VidoreSyntheticDocQAEnergyRetrieval.json | 186 ++++++ ...theticDocQAGovernmentReportsRetrieval.json | 186 ++++++ ...heticDocQAHealthcareIndustryRetrieval.json | 186 ++++++ .../VidoreTabfquadRetrieval.json | 186 ++++++ .../VidoreTatdqaRetrieval.json | 186 ++++++ 32 files changed, 6202 insertions(+) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreArxivQARetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreDocVQARetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreInfoVQARetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreShiftProjectRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAAIRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAEnergyRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTabfquadRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTatdqaRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreArxivQARetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreDocVQARetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreInfoVQARetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreShiftProjectRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAAIRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAEnergyRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAGovernmentReportsRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTabfquadRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTatdqaRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreArxivQARetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreDocVQARetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreInfoVQARetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreShiftProjectRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAAIRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAEnergyRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAGovernmentReportsRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTabfquadRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTatdqaRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 3e56bdf82c..1bf12cf0e7 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -26,6 +26,7 @@ from .eng.SOPI2IRetrieval import * from .eng.StanfordCarsI2IRetrieval import * from .eng.TUBerlinT2IRetrieval import * +from .eng.VidoreBenchRetrieval import * from .eng.VisualNewsI2TRetrieval import * from .eng.VisualNewsT2IRetrieval import * from .eng.WebQAT2ITRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py new file mode 100644 index 0000000000..6365b7adb3 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py @@ -0,0 +1,621 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): + corpus = {} + queries = {} + relevant_docs = {} + + dataset = load_dataset( + path, + cache_dir=cache_dir, + revision=revision, + ) + + for split in splits: + split_dataset = dataset[split] + split_dataset = split_dataset.rename_column("query", "text") + corpus[split] = split_dataset.map( + lambda x, idx: { + "id": f"corpus-{split}-{idx}", + "modality": "image", + "text": None, + }, + with_indices=True, + ) + + queries[split] = split_dataset.map( + lambda x, idx: { + "id": f"query-{split}-{idx}", + "image": None, + "modality": "text", + }, + with_indices=True, + ) + relevant_docs[split] = {} + for index in range(len(split_dataset)): + query_id = f"query-{split}-{index}" + doc_id = f"corpus-{split}-{index}" + if query_id not in relevant_docs[split]: + relevant_docs[split][query_id] = {} + relevant_docs[split][query_id][doc_id] = 1 + return corpus, queries, relevant_docs + + +def _load_data_qc_unmatched( + path: str, splits: str, cache_dir: str = None, revision: str = None, num_queries=100 +): + corpus = {} + queries = {} + relevant_docs = {} + + dataset = load_dataset( + path, + cache_dir=cache_dir, + revision=revision, + ) + + for split in splits: + split_dataset = dataset[split] + split_dataset = split_dataset.rename_column("query", "text") + corpus[split] = split_dataset.map( + lambda x, idx: { + "id": f"corpus-{split}-{idx}", + "modality": "image", + "text": None, + }, + with_indices=True, + ) + + split_dataset = split_dataset.select(range(num_queries)) + queries[split] = split_dataset.map( + lambda x, idx: { + "id": f"query-{split}-{idx}", + "image": None, + "modality": "text", + }, + with_indices=True, + ) + relevant_docs[split] = {} + for index in range(len(queries[split])): + query_id = f"query-{split}-{index}" + doc_id = f"corpus-{split}-{index}" + if query_id not in relevant_docs[split]: + relevant_docs[split][query_id] = {} + relevant_docs[split][query_id][doc_id] = 1 + return corpus, queries, relevant_docs + + +class VidoreArxivQARetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VidoreArxivQARetrieval", + description="Retrieve associated pages according to questions.", + reference="https://arxiv.org/pdf/2407.01449", + dataset={ + "path": "vidore/arxivqa_test_subsampled", + "revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_5", + date=("2024-01-01", "2024-07-01"), + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{faysse2024colpali, + title={ColPali: Efficient Document Retrieval with Vision Language Models}, + author={Faysse, Manuel and Sibille, Hugues and Wu, Tony and Viaud, Gautier and Hudelot, C{\'e}line and Colombo, Pierre}, + journal={arXiv preprint arXiv:2407.01449}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 99.328, + "num_documents": 500, + "num_queries": 500, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + self.corpus, self.queries, self.relevant_docs = _load_data( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True + + +class VidoreDocVQARetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VidoreDocVQARetrieval", + description="Retrieve associated pages according to questions.", + reference="https://arxiv.org/pdf/2407.01449", + dataset={ + "path": "vidore/docvqa_test_subsampled", + "revision": "b1d89eda849e636676df6ead8002602fb1858600", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_5", + date=("2024-01-01", "2024-07-01"), + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{faysse2024colpali, + title={ColPali: Efficient Document Retrieval with Vision Language Models}, + author={Faysse, Manuel and Sibille, Hugues and Wu, Tony and Viaud, Gautier and Hudelot, C{\'e}line and Colombo, Pierre}, + journal={arXiv preprint arXiv:2407.01449}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 41.746, + "num_documents": 500, + "num_queries": 500, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + self.corpus, self.queries, self.relevant_docs = _load_data( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True + + +class VidoreInfoVQARetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VidoreInfoVQARetrieval", + description="Retrieve associated pages according to questions.", + reference="https://arxiv.org/pdf/2407.01449", + dataset={ + "path": "vidore/infovqa_test_subsampled", + "revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_5", + date=("2024-01-01", "2024-07-01"), + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{faysse2024colpali, + title={ColPali: Efficient Document Retrieval with Vision Language Models}, + author={Faysse, Manuel and Sibille, Hugues and Wu, Tony and Viaud, Gautier and Hudelot, C{\'e}line and Colombo, Pierre}, + journal={arXiv preprint arXiv:2407.01449}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 64.934, + "num_documents": 500, + "num_queries": 500, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + self.corpus, self.queries, self.relevant_docs = _load_data( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True + + +class VidoreTabfquadRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VidoreTabfquadRetrieval", + description="Retrieve associated pages according to questions.", + reference="https://arxiv.org/pdf/2407.01449", + dataset={ + "path": "vidore/tabfquad_test_subsampled", + "revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_5", + date=("2024-01-01", "2024-07-01"), + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{faysse2024colpali, + title={ColPali: Efficient Document Retrieval with Vision Language Models}, + author={Faysse, Manuel and Sibille, Hugues and Wu, Tony and Viaud, Gautier and Hudelot, C{\'e}line and Colombo, Pierre}, + journal={arXiv preprint arXiv:2407.01449}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 100.63214285714285, + "num_documents": 280, + "num_queries": 280, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + self.corpus, self.queries, self.relevant_docs = _load_data( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True + + +class VidoreTatdqaRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VidoreTatdqaRetrieval", + description="Retrieve associated pages according to questions.", + reference="https://arxiv.org/pdf/2407.01449", + dataset={ + "path": "vidore/tatdqa_test", + "revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_5", + date=("2024-01-01", "2024-07-01"), + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{faysse2024colpali, + title={ColPali: Efficient Document Retrieval with Vision Language Models}, + author={Faysse, Manuel and Sibille, Hugues and Wu, Tony and Viaud, Gautier and Hudelot, C{\'e}line and Colombo, Pierre}, + journal={arXiv preprint arXiv:2407.01449}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 72.76368009621167, + "num_documents": 1663, + "num_queries": 1663, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + self.corpus, self.queries, self.relevant_docs = _load_data( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True + + +class VidoreShiftProjectRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VidoreShiftProjectRetrieval", + description="Retrieve associated pages according to questions.", + reference="https://arxiv.org/pdf/2407.01449", + dataset={ + "path": "vidore/shiftproject_test", + "revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_5", + date=("2024-01-01", "2024-07-01"), + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{faysse2024colpali, + title={ColPali: Efficient Document Retrieval with Vision Language Models}, + author={Faysse, Manuel and Sibille, Hugues and Wu, Tony and Viaud, Gautier and Hudelot, C{\'e}line and Colombo, Pierre}, + journal={arXiv preprint arXiv:2407.01449}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 97.7, + "num_documents": 1000, + "num_queries": 100, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + self.corpus, self.queries, self.relevant_docs = _load_data_qc_unmatched( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True + + +class VidoreSyntheticDocQAAIRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VidoreSyntheticDocQAAIRetrieval", + description="Retrieve associated pages according to questions.", + reference="https://arxiv.org/pdf/2407.01449", + dataset={ + "path": "vidore/syntheticDocQA_artificial_intelligence_test", + "revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_5", + date=("2024-01-01", "2024-07-01"), + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{faysse2024colpali, + title={ColPali: Efficient Document Retrieval with Vision Language Models}, + author={Faysse, Manuel and Sibille, Hugues and Wu, Tony and Viaud, Gautier and Hudelot, C{\'e}line and Colombo, Pierre}, + journal={arXiv preprint arXiv:2407.01449}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 77.71, + "num_documents": 1000, + "num_queries": 100, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + self.corpus, self.queries, self.relevant_docs = _load_data_qc_unmatched( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True + + +class VidoreSyntheticDocQAEnergyRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VidoreSyntheticDocQAEnergyRetrieval", + description="Retrieve associated pages according to questions.", + reference="https://arxiv.org/pdf/2407.01449", + dataset={ + "path": "vidore/syntheticDocQA_energy_test", + "revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_5", + date=("2024-01-01", "2024-07-01"), + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{faysse2024colpali, + title={ColPali: Efficient Document Retrieval with Vision Language Models}, + author={Faysse, Manuel and Sibille, Hugues and Wu, Tony and Viaud, Gautier and Hudelot, C{\'e}line and Colombo, Pierre}, + journal={arXiv preprint arXiv:2407.01449}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 83.69, + "num_documents": 1000, + "num_queries": 100, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + self.corpus, self.queries, self.relevant_docs = _load_data_qc_unmatched( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True + + +class VidoreSyntheticDocQAGovernmentReportsRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VidoreSyntheticDocQAGovernmentReportsRetrieval", + description="Retrieve associated pages according to questions.", + reference="https://arxiv.org/pdf/2407.01449", + dataset={ + "path": "vidore/syntheticDocQA_government_reports_test", + "revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_5", + date=("2024-01-01", "2024-07-01"), + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{faysse2024colpali, + title={ColPali: Efficient Document Retrieval with Vision Language Models}, + author={Faysse, Manuel and Sibille, Hugues and Wu, Tony and Viaud, Gautier and Hudelot, C{\'e}line and Colombo, Pierre}, + journal={arXiv preprint arXiv:2407.01449}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 82.53, + "num_documents": 1000, + "num_queries": 100, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + self.corpus, self.queries, self.relevant_docs = _load_data_qc_unmatched( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True + + +class VidoreSyntheticDocQAHealthcareIndustryRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VidoreSyntheticDocQAHealthcareIndustryRetrieval", + description="Retrieve associated pages according to questions.", + reference="https://arxiv.org/pdf/2407.01449", + dataset={ + "path": "vidore/syntheticDocQA_healthcare_industry_test", + "revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_5", + date=("2024-01-01", "2024-07-01"), + domains=["Academic"], + task_subtypes=["Image Text Retrieval"], + license="MIT", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{faysse2024colpali, + title={ColPali: Efficient Document Retrieval with Vision Language Models}, + author={Faysse, Manuel and Sibille, Hugues and Wu, Tony and Viaud, Gautier and Hudelot, C{\'e}line and Colombo, Pierre}, + journal={arXiv preprint arXiv:2407.01449}, + year={2024} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 1.0, + "average_query_length": 80.43, + "num_documents": 1000, + "num_queries": 100, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + self.corpus, self.queries, self.relevant_docs = _load_data_qc_unmatched( + path=self.metadata_dict["dataset"]["path"], + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreArxivQARetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..8a05dc0907 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", + "evaluation_time": 11.539915561676025, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.198, + "cv_recall_at_10": 0.366, + "cv_recall_at_100": 0.716, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.444, + "cv_recall_at_3": 0.28, + "cv_recall_at_5": 0.318, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.262, + "map_at_1": 0.198, + "map_at_10": 0.25016, + "map_at_100": 0.26189, + "map_at_1000": 0.26345, + "map_at_20": 0.25574, + "map_at_3": 0.23467, + "map_at_5": 0.24337, + "mrr_at_1": 0.198, + "mrr_at_10": 0.2501595238095238, + "mrr_at_100": 0.2618894833637717, + "mrr_at_1000": 0.2634476289251727, + "mrr_at_20": 0.2557436838874919, + "mrr_at_3": 0.23466666666666666, + "mrr_at_5": 0.24336666666666668, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2685481985413355, + "nauc_cv_recall_at_100_max": 0.25195531990315806, + "nauc_cv_recall_at_100_std": 0.06048858485371907, + "nauc_cv_recall_at_10_diff1": 0.4109315183914633, + "nauc_cv_recall_at_10_max": 0.32723380983407496, + "nauc_cv_recall_at_10_std": -0.01634389694395826, + "nauc_cv_recall_at_1_diff1": 0.5591586841257287, + "nauc_cv_recall_at_1_max": 0.30669556981183665, + "nauc_cv_recall_at_1_std": -0.06708263760728826, + "nauc_cv_recall_at_20_diff1": 0.3963498460589525, + "nauc_cv_recall_at_20_max": 0.3259290041660787, + "nauc_cv_recall_at_20_std": 0.0020150856999482376, + "nauc_cv_recall_at_3_diff1": 0.4703111905693055, + "nauc_cv_recall_at_3_max": 0.3366472987317727, + "nauc_cv_recall_at_3_std": 0.0015140510643050556, + "nauc_cv_recall_at_5_diff1": 0.4413490717899683, + "nauc_cv_recall_at_5_max": 0.34234892983316073, + "nauc_cv_recall_at_5_std": -0.00039372925890506795, + "nauc_map_at_1000_diff1": 0.5023400105417865, + "nauc_map_at_1000_max": 0.321188547000857, + "nauc_map_at_1000_std": -0.034383841529415766, + "nauc_map_at_100_diff1": 0.5016743216328943, + "nauc_map_at_100_max": 0.3209011408009578, + "nauc_map_at_100_std": -0.034423076879957805, + "nauc_map_at_10_diff1": 0.5044594445998427, + "nauc_map_at_10_max": 0.3232179253193846, + "nauc_map_at_10_std": -0.03552375018397788, + "nauc_map_at_1_diff1": 0.5591586841257287, + "nauc_map_at_1_max": 0.30669556981183665, + "nauc_map_at_1_std": -0.06708263760728826, + "nauc_map_at_20_diff1": 0.5034779946433506, + "nauc_map_at_20_max": 0.3231314045091189, + "nauc_map_at_20_std": -0.03407625748520336, + "nauc_map_at_3_diff1": 0.5189393862869124, + "nauc_map_at_3_max": 0.3252757143116887, + "nauc_map_at_3_std": -0.034379690779757784, + "nauc_map_at_5_diff1": 0.5103116101493478, + "nauc_map_at_5_max": 0.3256348361969301, + "nauc_map_at_5_std": -0.033658858852439, + "nauc_mrr_at_1000_diff1": 0.5023400105417865, + "nauc_mrr_at_1000_max": 0.321188547000857, + "nauc_mrr_at_1000_std": -0.034383841529415766, + "nauc_mrr_at_100_diff1": 0.5016743216328943, + "nauc_mrr_at_100_max": 0.3209011408009578, + "nauc_mrr_at_100_std": -0.034423076879957805, + "nauc_mrr_at_10_diff1": 0.5044594445998427, + "nauc_mrr_at_10_max": 0.3232179253193846, + "nauc_mrr_at_10_std": -0.03552375018397788, + "nauc_mrr_at_1_diff1": 0.5591586841257287, + "nauc_mrr_at_1_max": 0.30669556981183665, + "nauc_mrr_at_1_std": -0.06708263760728826, + "nauc_mrr_at_20_diff1": 0.5034779946433506, + "nauc_mrr_at_20_max": 0.3231314045091189, + "nauc_mrr_at_20_std": -0.03407625748520336, + "nauc_mrr_at_3_diff1": 0.5189393862869124, + "nauc_mrr_at_3_max": 0.3252757143116887, + "nauc_mrr_at_3_std": -0.034379690779757784, + "nauc_mrr_at_5_diff1": 0.5103116101493478, + "nauc_mrr_at_5_max": 0.3256348361969301, + "nauc_mrr_at_5_std": -0.033658858852439, + "nauc_ndcg_at_1000_diff1": 0.4823895239759644, + "nauc_ndcg_at_1000_max": 0.3190692347281697, + "nauc_ndcg_at_1000_std": -0.026762242805733943, + "nauc_ndcg_at_100_diff1": 0.4613737853231418, + "nauc_ndcg_at_100_max": 0.3119611918607358, + "nauc_ndcg_at_100_std": -0.019659627911515334, + "nauc_ndcg_at_10_diff1": 0.47935455871116944, + "nauc_ndcg_at_10_max": 0.3247619541581183, + "nauc_ndcg_at_10_std": -0.02933370691404022, + "nauc_ndcg_at_1_diff1": 0.5591586841257287, + "nauc_ndcg_at_1_max": 0.30669556981183665, + "nauc_ndcg_at_1_std": -0.06708263760728826, + "nauc_ndcg_at_20_diff1": 0.4754922425300419, + "nauc_ndcg_at_20_max": 0.3244000467022679, + "nauc_ndcg_at_20_std": -0.024220056034281272, + "nauc_ndcg_at_3_diff1": 0.5057760785119431, + "nauc_ndcg_at_3_max": 0.3285522047251961, + "nauc_ndcg_at_3_std": -0.02459832888889593, + "nauc_ndcg_at_5_diff1": 0.4915665657301162, + "nauc_ndcg_at_5_max": 0.33021385983758084, + "nauc_ndcg_at_5_std": -0.024268471559681672, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.26854819854133494, + "nauc_precision_at_100_max": 0.25195531990315767, + "nauc_precision_at_100_std": 0.060488584853718386, + "nauc_precision_at_10_diff1": 0.41093151839146325, + "nauc_precision_at_10_max": 0.327233809834075, + "nauc_precision_at_10_std": -0.016343896943957985, + "nauc_precision_at_1_diff1": 0.5591586841257287, + "nauc_precision_at_1_max": 0.30669556981183665, + "nauc_precision_at_1_std": -0.06708263760728826, + "nauc_precision_at_20_diff1": 0.3963498460589526, + "nauc_precision_at_20_max": 0.32592900416607873, + "nauc_precision_at_20_std": 0.002015085699948558, + "nauc_precision_at_3_diff1": 0.4703111905693055, + "nauc_precision_at_3_max": 0.3366472987317729, + "nauc_precision_at_3_std": 0.0015140510643054945, + "nauc_precision_at_5_diff1": 0.44134907178996824, + "nauc_precision_at_5_max": 0.3423489298331607, + "nauc_precision_at_5_std": -0.00039372925890499764, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2685481985413355, + "nauc_recall_at_100_max": 0.25195531990315806, + "nauc_recall_at_100_std": 0.06048858485371907, + "nauc_recall_at_10_diff1": 0.4109315183914633, + "nauc_recall_at_10_max": 0.32723380983407496, + "nauc_recall_at_10_std": -0.01634389694395826, + "nauc_recall_at_1_diff1": 0.5591586841257287, + "nauc_recall_at_1_max": 0.30669556981183665, + "nauc_recall_at_1_std": -0.06708263760728826, + "nauc_recall_at_20_diff1": 0.3963498460589525, + "nauc_recall_at_20_max": 0.3259290041660787, + "nauc_recall_at_20_std": 0.0020150856999482376, + "nauc_recall_at_3_diff1": 0.4703111905693055, + "nauc_recall_at_3_max": 0.3366472987317727, + "nauc_recall_at_3_std": 0.0015140510643050556, + "nauc_recall_at_5_diff1": 0.4413490717899683, + "nauc_recall_at_5_max": 0.34234892983316073, + "nauc_recall_at_5_std": -0.00039372925890506795, + "ndcg_at_1": 0.198, + "ndcg_at_10": 0.27792, + "ndcg_at_100": 0.34666, + "ndcg_at_1000": 0.38404, + "ndcg_at_20": 0.29785, + "ndcg_at_3": 0.24633, + "ndcg_at_5": 0.262, + "precision_at_1": 0.198, + "precision_at_10": 0.0366, + "precision_at_100": 0.00716, + "precision_at_1000": 0.001, + "precision_at_20": 0.0222, + "precision_at_3": 0.09333, + "precision_at_5": 0.0636, + "recall_at_1": 0.198, + "recall_at_10": 0.366, + "recall_at_100": 0.716, + "recall_at_1000": 1.0, + "recall_at_20": 0.444, + "recall_at_3": 0.28, + "recall_at_5": 0.318 + } + ] + }, + "task_name": "VidoreArxivQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreDocVQARetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..ad62f9a1d4 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b1d89eda849e636676df6ead8002602fb1858600", + "evaluation_time": 16.344624996185303, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.098, + "cv_recall_at_10": 0.246, + "cv_recall_at_100": 0.608, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.332, + "cv_recall_at_3": 0.16, + "cv_recall_at_5": 0.19, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.14602, + "map_at_1": 0.098, + "map_at_10": 0.13877, + "map_at_100": 0.15117, + "map_at_1000": 0.15306, + "map_at_20": 0.14451, + "map_at_3": 0.12467, + "map_at_5": 0.13147, + "mrr_at_1": 0.098, + "mrr_at_10": 0.13876746031746037, + "mrr_at_100": 0.15116731214168472, + "mrr_at_1000": 0.1530579249837991, + "mrr_at_20": 0.14450965011081118, + "mrr_at_3": 0.1246666666666667, + "mrr_at_5": 0.13146666666666668, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.18735699318384363, + "nauc_cv_recall_at_100_max": 0.14111611551337905, + "nauc_cv_recall_at_100_std": 0.42355006141766416, + "nauc_cv_recall_at_10_diff1": 0.3071288237751918, + "nauc_cv_recall_at_10_max": 0.1993356269317047, + "nauc_cv_recall_at_10_std": 0.26737311867149083, + "nauc_cv_recall_at_1_diff1": 0.4351971000366878, + "nauc_cv_recall_at_1_max": 0.19066601687629722, + "nauc_cv_recall_at_1_std": 0.22446789287220115, + "nauc_cv_recall_at_20_diff1": 0.22081974452146985, + "nauc_cv_recall_at_20_max": 0.14423736839025114, + "nauc_cv_recall_at_20_std": 0.2758524374737312, + "nauc_cv_recall_at_3_diff1": 0.3795897987402058, + "nauc_cv_recall_at_3_max": 0.22304501459517587, + "nauc_cv_recall_at_3_std": 0.23199416192963584, + "nauc_cv_recall_at_5_diff1": 0.3728078287626318, + "nauc_cv_recall_at_5_max": 0.23064920215896276, + "nauc_cv_recall_at_5_std": 0.2503690750424564, + "nauc_map_at_1000_diff1": 0.38116462145375835, + "nauc_map_at_1000_max": 0.19729004096849725, + "nauc_map_at_1000_std": 0.23852591704752155, + "nauc_map_at_100_diff1": 0.3800813688282042, + "nauc_map_at_100_max": 0.1968220239656418, + "nauc_map_at_100_std": 0.2396456603690272, + "nauc_map_at_10_diff1": 0.3932756999618319, + "nauc_map_at_10_max": 0.2036729162444215, + "nauc_map_at_10_std": 0.23368952621988348, + "nauc_map_at_1_diff1": 0.4351971000366878, + "nauc_map_at_1_max": 0.19066601687629722, + "nauc_map_at_1_std": 0.22446789287220115, + "nauc_map_at_20_diff1": 0.3830786234108412, + "nauc_map_at_20_max": 0.19759003746990553, + "nauc_map_at_20_std": 0.23508413618482943, + "nauc_map_at_3_diff1": 0.4092011738960796, + "nauc_map_at_3_max": 0.20665475087093785, + "nauc_map_at_3_std": 0.2255308517840538, + "nauc_map_at_5_diff1": 0.4066964428867831, + "nauc_map_at_5_max": 0.21030017113953264, + "nauc_map_at_5_std": 0.23113798288129628, + "nauc_mrr_at_1000_diff1": 0.38116462145375835, + "nauc_mrr_at_1000_max": 0.19729004096849725, + "nauc_mrr_at_1000_std": 0.23852591704752155, + "nauc_mrr_at_100_diff1": 0.3800813688282042, + "nauc_mrr_at_100_max": 0.1968220239656418, + "nauc_mrr_at_100_std": 0.2396456603690272, + "nauc_mrr_at_10_diff1": 0.3932756999618319, + "nauc_mrr_at_10_max": 0.2036729162444215, + "nauc_mrr_at_10_std": 0.23368952621988348, + "nauc_mrr_at_1_diff1": 0.4351971000366878, + "nauc_mrr_at_1_max": 0.19066601687629722, + "nauc_mrr_at_1_std": 0.22446789287220115, + "nauc_mrr_at_20_diff1": 0.3830786234108412, + "nauc_mrr_at_20_max": 0.19759003746990553, + "nauc_mrr_at_20_std": 0.23508413618482943, + "nauc_mrr_at_3_diff1": 0.4092011738960796, + "nauc_mrr_at_3_max": 0.20665475087093785, + "nauc_mrr_at_3_std": 0.2255308517840538, + "nauc_mrr_at_5_diff1": 0.4066964428867831, + "nauc_mrr_at_5_max": 0.21030017113953264, + "nauc_mrr_at_5_std": 0.23113798288129628, + "nauc_ndcg_at_1000_diff1": 0.3523903000677059, + "nauc_ndcg_at_1000_max": 0.19100960870984232, + "nauc_ndcg_at_1000_std": 0.2535966606069548, + "nauc_ndcg_at_100_diff1": 0.3262818126767828, + "nauc_ndcg_at_100_max": 0.18243597433523048, + "nauc_ndcg_at_100_std": 0.28083778118767516, + "nauc_ndcg_at_10_diff1": 0.36743814573481537, + "nauc_ndcg_at_10_max": 0.2030744688835133, + "nauc_ndcg_at_10_std": 0.24356124176904942, + "nauc_ndcg_at_1_diff1": 0.4351971000366878, + "nauc_ndcg_at_1_max": 0.19066601687629722, + "nauc_ndcg_at_1_std": 0.22446789287220115, + "nauc_ndcg_at_20_diff1": 0.334829441547401, + "nauc_ndcg_at_20_max": 0.1833490765926271, + "nauc_ndcg_at_20_std": 0.2475227625096276, + "nauc_ndcg_at_3_diff1": 0.40063489059083385, + "nauc_ndcg_at_3_max": 0.21142675686936951, + "nauc_ndcg_at_3_std": 0.22731281521503882, + "nauc_ndcg_at_5_diff1": 0.3964257193277851, + "nauc_ndcg_at_5_max": 0.21658505356493776, + "nauc_ndcg_at_5_std": 0.23672134038325543, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.18735699318384358, + "nauc_precision_at_100_max": 0.14111611551337946, + "nauc_precision_at_100_std": 0.42355006141766455, + "nauc_precision_at_10_diff1": 0.3071288237751922, + "nauc_precision_at_10_max": 0.19933562693170523, + "nauc_precision_at_10_std": 0.26737311867149116, + "nauc_precision_at_1_diff1": 0.4351971000366878, + "nauc_precision_at_1_max": 0.19066601687629722, + "nauc_precision_at_1_std": 0.22446789287220115, + "nauc_precision_at_20_diff1": 0.22081974452147027, + "nauc_precision_at_20_max": 0.14423736839025153, + "nauc_precision_at_20_std": 0.2758524374737314, + "nauc_precision_at_3_diff1": 0.3795897987402061, + "nauc_precision_at_3_max": 0.2230450145951759, + "nauc_precision_at_3_std": 0.231994161929636, + "nauc_precision_at_5_diff1": 0.372807828762632, + "nauc_precision_at_5_max": 0.23064920215896323, + "nauc_precision_at_5_std": 0.25036907504245653, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.18735699318384363, + "nauc_recall_at_100_max": 0.14111611551337905, + "nauc_recall_at_100_std": 0.42355006141766416, + "nauc_recall_at_10_diff1": 0.3071288237751918, + "nauc_recall_at_10_max": 0.1993356269317047, + "nauc_recall_at_10_std": 0.26737311867149083, + "nauc_recall_at_1_diff1": 0.4351971000366878, + "nauc_recall_at_1_max": 0.19066601687629722, + "nauc_recall_at_1_std": 0.22446789287220115, + "nauc_recall_at_20_diff1": 0.22081974452146985, + "nauc_recall_at_20_max": 0.14423736839025114, + "nauc_recall_at_20_std": 0.2758524374737312, + "nauc_recall_at_3_diff1": 0.3795897987402058, + "nauc_recall_at_3_max": 0.22304501459517587, + "nauc_recall_at_3_std": 0.23199416192963584, + "nauc_recall_at_5_diff1": 0.3728078287626318, + "nauc_recall_at_5_max": 0.23064920215896276, + "nauc_recall_at_5_std": 0.2503690750424564, + "ndcg_at_1": 0.098, + "ndcg_at_10": 0.16395, + "ndcg_at_100": 0.23576, + "ndcg_at_1000": 0.28608, + "ndcg_at_20": 0.18543, + "ndcg_at_3": 0.13371, + "ndcg_at_5": 0.14602, + "precision_at_1": 0.098, + "precision_at_10": 0.0246, + "precision_at_100": 0.00608, + "precision_at_1000": 0.001, + "precision_at_20": 0.0166, + "precision_at_3": 0.05333, + "precision_at_5": 0.038, + "recall_at_1": 0.098, + "recall_at_10": 0.246, + "recall_at_100": 0.608, + "recall_at_1000": 1.0, + "recall_at_20": 0.332, + "recall_at_3": 0.16, + "recall_at_5": 0.19 + } + ] + }, + "task_name": "VidoreDocVQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreInfoVQARetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..e29196c1c3 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", + "evaluation_time": 17.864064693450928, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.392, + "cv_recall_at_10": 0.716, + "cv_recall_at_100": 0.922, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.784, + "cv_recall_at_3": 0.564, + "cv_recall_at_5": 0.624, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5185, + "map_at_1": 0.392, + "map_at_10": 0.4957, + "map_at_100": 0.50427, + "map_at_1000": 0.50467, + "map_at_20": 0.5007, + "map_at_3": 0.46933, + "map_at_5": 0.48313, + "mrr_at_1": 0.392, + "mrr_at_10": 0.4957047619047617, + "mrr_at_100": 0.5042708815113673, + "mrr_at_1000": 0.5046663761343821, + "mrr_at_20": 0.5007034324722403, + "mrr_at_3": 0.4693333333333335, + "mrr_at_5": 0.4831333333333333, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.43401805166511004, + "nauc_cv_recall_at_100_max": 0.4512916277622163, + "nauc_cv_recall_at_100_std": 0.5630012688836221, + "nauc_cv_recall_at_10_diff1": 0.4999622889121859, + "nauc_cv_recall_at_10_max": 0.3837631140307873, + "nauc_cv_recall_at_10_std": 0.21789089428073674, + "nauc_cv_recall_at_1_diff1": 0.6361117715774349, + "nauc_cv_recall_at_1_max": 0.38531576607776064, + "nauc_cv_recall_at_1_std": 0.059550515336028, + "nauc_cv_recall_at_20_diff1": 0.44793391539966904, + "nauc_cv_recall_at_20_max": 0.3685975839229269, + "nauc_cv_recall_at_20_std": 0.2803279768177026, + "nauc_cv_recall_at_3_diff1": 0.5806573026979864, + "nauc_cv_recall_at_3_max": 0.3603360079382735, + "nauc_cv_recall_at_3_std": 0.13183122045245887, + "nauc_cv_recall_at_5_diff1": 0.5356571704849744, + "nauc_cv_recall_at_5_max": 0.3265211246652782, + "nauc_cv_recall_at_5_std": 0.10480821680055523, + "nauc_map_at_1000_diff1": 0.5943168242782374, + "nauc_map_at_1000_max": 0.37326847099761423, + "nauc_map_at_1000_std": 0.10119065227206148, + "nauc_map_at_100_diff1": 0.594054309790089, + "nauc_map_at_100_max": 0.3733454665922043, + "nauc_map_at_100_std": 0.10165465316992746, + "nauc_map_at_10_diff1": 0.5946248202240149, + "nauc_map_at_10_max": 0.37243402579793333, + "nauc_map_at_10_std": 0.09948573630126686, + "nauc_map_at_1_diff1": 0.6361117715774349, + "nauc_map_at_1_max": 0.38531576607776064, + "nauc_map_at_1_std": 0.059550515336028, + "nauc_map_at_20_diff1": 0.592838076133879, + "nauc_map_at_20_max": 0.37203415673242324, + "nauc_map_at_20_std": 0.10081523194886777, + "nauc_map_at_3_diff1": 0.6067771499819062, + "nauc_map_at_3_max": 0.3721820842902545, + "nauc_map_at_3_std": 0.09344892818883076, + "nauc_map_at_5_diff1": 0.5979515676651626, + "nauc_map_at_5_max": 0.3662453984051962, + "nauc_map_at_5_std": 0.08812192427868058, + "nauc_mrr_at_1000_diff1": 0.5943168242782374, + "nauc_mrr_at_1000_max": 0.37326847099761423, + "nauc_mrr_at_1000_std": 0.10119065227206148, + "nauc_mrr_at_100_diff1": 0.594054309790089, + "nauc_mrr_at_100_max": 0.3733454665922043, + "nauc_mrr_at_100_std": 0.10165465316992746, + "nauc_mrr_at_10_diff1": 0.5946248202240149, + "nauc_mrr_at_10_max": 0.37243402579793333, + "nauc_mrr_at_10_std": 0.09948573630126686, + "nauc_mrr_at_1_diff1": 0.6361117715774349, + "nauc_mrr_at_1_max": 0.38531576607776064, + "nauc_mrr_at_1_std": 0.059550515336028, + "nauc_mrr_at_20_diff1": 0.592838076133879, + "nauc_mrr_at_20_max": 0.37203415673242324, + "nauc_mrr_at_20_std": 0.10081523194886777, + "nauc_mrr_at_3_diff1": 0.6067771499819062, + "nauc_mrr_at_3_max": 0.3721820842902545, + "nauc_mrr_at_3_std": 0.09344892818883076, + "nauc_mrr_at_5_diff1": 0.5979515676651626, + "nauc_mrr_at_5_max": 0.3662453984051962, + "nauc_mrr_at_5_std": 0.08812192427868058, + "nauc_ndcg_at_1000_diff1": 0.5844068303992299, + "nauc_ndcg_at_1000_max": 0.37439145340533564, + "nauc_ndcg_at_1000_std": 0.11647849865946318, + "nauc_ndcg_at_100_diff1": 0.5785820083831996, + "nauc_ndcg_at_100_max": 0.3770927197960715, + "nauc_ndcg_at_100_std": 0.1320926286188547, + "nauc_ndcg_at_10_diff1": 0.5760468027551768, + "nauc_ndcg_at_10_max": 0.37332452932466365, + "nauc_ndcg_at_10_std": 0.12141024137245926, + "nauc_ndcg_at_1_diff1": 0.6361117715774349, + "nauc_ndcg_at_1_max": 0.38531576607776064, + "nauc_ndcg_at_1_std": 0.059550515336028, + "nauc_ndcg_at_20_diff1": 0.5696852943787447, + "nauc_ndcg_at_20_max": 0.37086868649526816, + "nauc_ndcg_at_20_std": 0.12776127821364855, + "nauc_ndcg_at_3_diff1": 0.60009384166449, + "nauc_ndcg_at_3_max": 0.3691552827726549, + "nauc_ndcg_at_3_std": 0.10303937890816356, + "nauc_ndcg_at_5_diff1": 0.5837010291294129, + "nauc_ndcg_at_5_max": 0.35739896590967324, + "nauc_ndcg_at_5_std": 0.09270634380545034, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.4340180516651088, + "nauc_precision_at_100_max": 0.45129162776221715, + "nauc_precision_at_100_std": 0.5630012688836226, + "nauc_precision_at_10_diff1": 0.49996228891218547, + "nauc_precision_at_10_max": 0.3837631140307867, + "nauc_precision_at_10_std": 0.21789089428073644, + "nauc_precision_at_1_diff1": 0.6361117715774349, + "nauc_precision_at_1_max": 0.38531576607776064, + "nauc_precision_at_1_std": 0.059550515336028, + "nauc_precision_at_20_diff1": 0.4479339153996688, + "nauc_precision_at_20_max": 0.3685975839229259, + "nauc_precision_at_20_std": 0.2803279768177021, + "nauc_precision_at_3_diff1": 0.5806573026979863, + "nauc_precision_at_3_max": 0.36033600793827303, + "nauc_precision_at_3_std": 0.13183122045245868, + "nauc_precision_at_5_diff1": 0.5356571704849745, + "nauc_precision_at_5_max": 0.326521124665278, + "nauc_precision_at_5_std": 0.10480821680055544, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.43401805166511004, + "nauc_recall_at_100_max": 0.4512916277622163, + "nauc_recall_at_100_std": 0.5630012688836221, + "nauc_recall_at_10_diff1": 0.4999622889121859, + "nauc_recall_at_10_max": 0.3837631140307873, + "nauc_recall_at_10_std": 0.21789089428073674, + "nauc_recall_at_1_diff1": 0.6361117715774349, + "nauc_recall_at_1_max": 0.38531576607776064, + "nauc_recall_at_1_std": 0.059550515336028, + "nauc_recall_at_20_diff1": 0.44793391539966904, + "nauc_recall_at_20_max": 0.3685975839229269, + "nauc_recall_at_20_std": 0.2803279768177026, + "nauc_recall_at_3_diff1": 0.5806573026979864, + "nauc_recall_at_3_max": 0.3603360079382735, + "nauc_recall_at_3_std": 0.13183122045245887, + "nauc_recall_at_5_diff1": 0.5356571704849744, + "nauc_recall_at_5_max": 0.3265211246652782, + "nauc_recall_at_5_std": 0.10480821680055523, + "ndcg_at_1": 0.392, + "ndcg_at_10": 0.54854, + "ndcg_at_100": 0.59176, + "ndcg_at_1000": 0.60188, + "ndcg_at_20": 0.56608, + "ndcg_at_3": 0.49371, + "ndcg_at_5": 0.5185, + "precision_at_1": 0.392, + "precision_at_10": 0.0716, + "precision_at_100": 0.00922, + "precision_at_1000": 0.001, + "precision_at_20": 0.0392, + "precision_at_3": 0.188, + "precision_at_5": 0.1248, + "recall_at_1": 0.392, + "recall_at_10": 0.716, + "recall_at_100": 0.922, + "recall_at_1000": 1.0, + "recall_at_20": 0.784, + "recall_at_3": 0.564, + "recall_at_5": 0.624 + } + ] + }, + "task_name": "VidoreInfoVQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreShiftProjectRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..840a9706ec --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", + "evaluation_time": 31.411139011383057, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.06, + "cv_recall_at_10": 0.12, + "cv_recall_at_100": 0.4, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.16, + "cv_recall_at_3": 0.08, + "cv_recall_at_5": 0.09, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.07518, + "map_at_1": 0.06, + "map_at_10": 0.07425, + "map_at_100": 0.08209, + "map_at_1000": 0.08411, + "map_at_20": 0.07705, + "map_at_3": 0.06833, + "map_at_5": 0.07033, + "mrr_at_1": 0.06, + "mrr_at_10": 0.07425, + "mrr_at_100": 0.08209457915336821, + "mrr_at_1000": 0.08410506965250342, + "mrr_at_20": 0.07704554655870446, + "mrr_at_3": 0.06833333333333333, + "mrr_at_5": 0.07033333333333333, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.08204170957775479, + "nauc_cv_recall_at_100_max": -0.16466271884655007, + "nauc_cv_recall_at_100_std": 0.041310504634397276, + "nauc_cv_recall_at_10_diff1": 0.2861577101696523, + "nauc_cv_recall_at_10_max": -0.02165671500331717, + "nauc_cv_recall_at_10_std": 0.16344422329637, + "nauc_cv_recall_at_1_diff1": 0.4233968339146767, + "nauc_cv_recall_at_1_max": 0.10759323852964853, + "nauc_cv_recall_at_1_std": 0.23982649136928724, + "nauc_cv_recall_at_20_diff1": 0.2683976033184821, + "nauc_cv_recall_at_20_max": -0.05797741588569673, + "nauc_cv_recall_at_20_std": 0.05742049469964647, + "nauc_cv_recall_at_3_diff1": 0.4125972632143814, + "nauc_cv_recall_at_3_max": -0.06298631607190777, + "nauc_cv_recall_at_3_std": 0.0455795546015561, + "nauc_cv_recall_at_5_diff1": 0.3028947917598306, + "nauc_cv_recall_at_5_max": -0.06382851861789338, + "nauc_cv_recall_at_5_std": 0.1516262707569388, + "nauc_map_at_1000_diff1": 0.37410025658803814, + "nauc_map_at_1000_max": 0.01858867554088589, + "nauc_map_at_1000_std": 0.15072481017384068, + "nauc_map_at_100_diff1": 0.37370530168851285, + "nauc_map_at_100_max": 0.017909234398799562, + "nauc_map_at_100_std": 0.14943430467788055, + "nauc_map_at_10_diff1": 0.39419977929478384, + "nauc_map_at_10_max": 0.029271888806436316, + "nauc_map_at_10_std": 0.16444181569745317, + "nauc_map_at_1_diff1": 0.4233968339146767, + "nauc_map_at_1_max": 0.10759323852964853, + "nauc_map_at_1_std": 0.23982649136928724, + "nauc_map_at_20_diff1": 0.38728572779256304, + "nauc_map_at_20_max": 0.024433234986002847, + "nauc_map_at_20_std": 0.1509822547622029, + "nauc_map_at_3_diff1": 0.4250001636050705, + "nauc_map_at_3_max": 0.024383699699621193, + "nauc_map_at_3_std": 0.14415569967344424, + "nauc_map_at_5_diff1": 0.39657196047289084, + "nauc_map_at_5_max": 0.021683704286766203, + "nauc_map_at_5_std": 0.16849250442206679, + "nauc_mrr_at_1000_diff1": 0.37410025658803814, + "nauc_mrr_at_1000_max": 0.01858867554088589, + "nauc_mrr_at_1000_std": 0.15072481017384068, + "nauc_mrr_at_100_diff1": 0.37370530168851285, + "nauc_mrr_at_100_max": 0.017909234398799562, + "nauc_mrr_at_100_std": 0.14943430467788055, + "nauc_mrr_at_10_diff1": 0.39419977929478384, + "nauc_mrr_at_10_max": 0.029271888806436316, + "nauc_mrr_at_10_std": 0.16444181569745317, + "nauc_mrr_at_1_diff1": 0.4233968339146767, + "nauc_mrr_at_1_max": 0.10759323852964853, + "nauc_mrr_at_1_std": 0.23982649136928724, + "nauc_mrr_at_20_diff1": 0.38728572779256304, + "nauc_mrr_at_20_max": 0.024433234986002847, + "nauc_mrr_at_20_std": 0.1509822547622029, + "nauc_mrr_at_3_diff1": 0.4250001636050705, + "nauc_mrr_at_3_max": 0.024383699699621193, + "nauc_mrr_at_3_std": 0.14415569967344424, + "nauc_mrr_at_5_diff1": 0.39657196047289084, + "nauc_mrr_at_5_max": 0.021683704286766203, + "nauc_mrr_at_5_std": 0.16849250442206679, + "nauc_ndcg_at_1000_diff1": 0.31537358926976555, + "nauc_ndcg_at_1000_max": -0.016070596404182663, + "nauc_ndcg_at_1000_std": 0.12905512978048142, + "nauc_ndcg_at_100_diff1": 0.26988366255315605, + "nauc_ndcg_at_100_max": -0.04728349318783632, + "nauc_ndcg_at_100_std": 0.1074580962286981, + "nauc_ndcg_at_10_diff1": 0.3613398039023956, + "nauc_ndcg_at_10_max": 0.011330093365220005, + "nauc_ndcg_at_10_std": 0.16061426691048258, + "nauc_ndcg_at_1_diff1": 0.4233968339146767, + "nauc_ndcg_at_1_max": 0.10759323852964853, + "nauc_ndcg_at_1_std": 0.23982649136928724, + "nauc_ndcg_at_20_diff1": 0.3466254194336954, + "nauc_ndcg_at_20_max": -0.004343325835300405, + "nauc_ndcg_at_20_std": 0.11916220300381282, + "nauc_ndcg_at_3_diff1": 0.4217185598163835, + "nauc_ndcg_at_3_max": -0.0006190144676542667, + "nauc_ndcg_at_3_std": 0.11591042417905126, + "nauc_ndcg_at_5_diff1": 0.3704432033581334, + "nauc_ndcg_at_5_max": -0.004218379208381653, + "nauc_ndcg_at_5_std": 0.16140422923218145, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.08204170957775474, + "nauc_precision_at_100_max": -0.16466271884655004, + "nauc_precision_at_100_std": 0.041310504634397435, + "nauc_precision_at_10_diff1": 0.2861577101696519, + "nauc_precision_at_10_max": -0.02165671500331726, + "nauc_precision_at_10_std": 0.16344422329636993, + "nauc_precision_at_1_diff1": 0.4233968339146767, + "nauc_precision_at_1_max": 0.10759323852964853, + "nauc_precision_at_1_std": 0.23982649136928724, + "nauc_precision_at_20_diff1": 0.2683976033184821, + "nauc_precision_at_20_max": -0.05797741588569691, + "nauc_precision_at_20_std": 0.057420494699646496, + "nauc_precision_at_3_diff1": 0.41259726321438156, + "nauc_precision_at_3_max": -0.06298631607190772, + "nauc_precision_at_3_std": 0.04557955460155622, + "nauc_precision_at_5_diff1": 0.30289479175983075, + "nauc_precision_at_5_max": -0.0638285186178934, + "nauc_precision_at_5_std": 0.15162627075693896, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.08204170957775479, + "nauc_recall_at_100_max": -0.16466271884655007, + "nauc_recall_at_100_std": 0.041310504634397276, + "nauc_recall_at_10_diff1": 0.2861577101696523, + "nauc_recall_at_10_max": -0.02165671500331717, + "nauc_recall_at_10_std": 0.16344422329637, + "nauc_recall_at_1_diff1": 0.4233968339146767, + "nauc_recall_at_1_max": 0.10759323852964853, + "nauc_recall_at_1_std": 0.23982649136928724, + "nauc_recall_at_20_diff1": 0.2683976033184821, + "nauc_recall_at_20_max": -0.05797741588569673, + "nauc_recall_at_20_std": 0.05742049469964647, + "nauc_recall_at_3_diff1": 0.4125972632143814, + "nauc_recall_at_3_max": -0.06298631607190777, + "nauc_recall_at_3_std": 0.0455795546015561, + "nauc_recall_at_5_diff1": 0.3028947917598306, + "nauc_recall_at_5_max": -0.06382851861789338, + "nauc_recall_at_5_std": 0.1516262707569388, + "ndcg_at_1": 0.06, + "ndcg_at_10": 0.08479, + "ndcg_at_100": 0.13719, + "ndcg_at_1000": 0.20844, + "ndcg_at_20": 0.09493, + "ndcg_at_3": 0.07131, + "ndcg_at_5": 0.07518, + "precision_at_1": 0.06, + "precision_at_10": 0.012, + "precision_at_100": 0.004, + "precision_at_1000": 0.001, + "precision_at_20": 0.008, + "precision_at_3": 0.02667, + "precision_at_5": 0.018, + "recall_at_1": 0.06, + "recall_at_10": 0.12, + "recall_at_100": 0.4, + "recall_at_1000": 1.0, + "recall_at_20": 0.16, + "recall_at_3": 0.08, + "recall_at_5": 0.09 + } + ] + }, + "task_name": "VidoreShiftProjectRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAAIRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..0884644b3a --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", + "evaluation_time": 29.046104431152344, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.16, + "cv_recall_at_10": 0.38, + "cv_recall_at_100": 0.7, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.44, + "cv_recall_at_3": 0.23, + "cv_recall_at_5": 0.27, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.21534, + "map_at_1": 0.15, + "map_at_10": 0.21204, + "map_at_100": 0.22219, + "map_at_1000": 0.22351, + "map_at_20": 0.21614, + "map_at_3": 0.19, + "map_at_5": 0.197, + "mrr_at_1": 0.16, + "mrr_at_10": 0.21411904761904757, + "mrr_at_100": 0.22427953818055787, + "mrr_at_1000": 0.22560495232991717, + "mrr_at_20": 0.21819018889607122, + "mrr_at_3": 0.19, + "mrr_at_5": 0.19949999999999998, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.04487479733381322, + "nauc_cv_recall_at_100_max": 0.38578634480273827, + "nauc_cv_recall_at_100_std": 0.21624932444604553, + "nauc_cv_recall_at_10_diff1": 0.053816962545161474, + "nauc_cv_recall_at_10_max": 0.3068385621084916, + "nauc_cv_recall_at_10_std": 0.11257245340888422, + "nauc_cv_recall_at_1_diff1": -0.04307497311414982, + "nauc_cv_recall_at_1_max": 0.32458903057305266, + "nauc_cv_recall_at_1_std": 0.02273774773390678, + "nauc_cv_recall_at_20_diff1": 0.011758104087278764, + "nauc_cv_recall_at_20_max": 0.27744427624367063, + "nauc_cv_recall_at_20_std": 0.13502897113326723, + "nauc_cv_recall_at_3_diff1": -0.0619880721770566, + "nauc_cv_recall_at_3_max": 0.3106114915879866, + "nauc_cv_recall_at_3_std": 0.06766217720901142, + "nauc_cv_recall_at_5_diff1": 0.04367159641421238, + "nauc_cv_recall_at_5_max": 0.37758740712327205, + "nauc_cv_recall_at_5_std": 0.11848442228189042, + "nauc_map_at_1000_diff1": -0.011906178432101485, + "nauc_map_at_1000_max": 0.34347834826166185, + "nauc_map_at_1000_std": 0.06868788719712396, + "nauc_map_at_100_diff1": -0.011461416682847331, + "nauc_map_at_100_max": 0.3439169246976087, + "nauc_map_at_100_std": 0.06902969086073914, + "nauc_map_at_10_diff1": -0.006776007026008395, + "nauc_map_at_10_max": 0.34593482568066836, + "nauc_map_at_10_std": 0.06713240165892259, + "nauc_map_at_1_diff1": -0.002036085072069493, + "nauc_map_at_1_max": 0.3561939320633002, + "nauc_map_at_1_std": 0.029170446527567652, + "nauc_map_at_20_diff1": -0.011132446959779519, + "nauc_map_at_20_max": 0.3416196002890793, + "nauc_map_at_20_std": 0.06811661532008675, + "nauc_map_at_3_diff1": -0.0345761769966832, + "nauc_map_at_3_max": 0.34568362190229734, + "nauc_map_at_3_std": 0.05712912954633583, + "nauc_map_at_5_diff1": -0.008473691506968436, + "nauc_map_at_5_max": 0.3577332062251081, + "nauc_map_at_5_std": 0.0680679378630188, + "nauc_mrr_at_1000_diff1": -0.04170970645246049, + "nauc_mrr_at_1000_max": 0.31914921880257574, + "nauc_mrr_at_1000_std": 0.0616313297999736, + "nauc_mrr_at_100_diff1": -0.04115911433035388, + "nauc_mrr_at_100_max": 0.31967350861649435, + "nauc_mrr_at_100_std": 0.06200175938599041, + "nauc_mrr_at_10_diff1": -0.03629641083965268, + "nauc_mrr_at_10_max": 0.32191326930625896, + "nauc_mrr_at_10_std": 0.0601941819647312, + "nauc_mrr_at_1_diff1": -0.04307497311414982, + "nauc_mrr_at_1_max": 0.32458903057305266, + "nauc_mrr_at_1_std": 0.02273774773390678, + "nauc_mrr_at_20_diff1": -0.0405265668427798, + "nauc_mrr_at_20_max": 0.31763501020716245, + "nauc_mrr_at_20_std": 0.06120366174527031, + "nauc_mrr_at_3_diff1": -0.07018191812825324, + "nauc_mrr_at_3_max": 0.3105577791902151, + "nauc_mrr_at_3_std": 0.04388481381592876, + "nauc_mrr_at_5_diff1": -0.03854217469274165, + "nauc_mrr_at_5_max": 0.33172955029216, + "nauc_mrr_at_5_std": 0.059700767552818744, + "nauc_ndcg_at_1000_diff1": -0.008833068456348002, + "nauc_ndcg_at_1000_max": 0.3380747213218557, + "nauc_ndcg_at_1000_std": 0.08270631330777752, + "nauc_ndcg_at_100_diff1": -0.0014110765231264717, + "nauc_ndcg_at_100_max": 0.34496092616016066, + "nauc_ndcg_at_100_std": 0.09604559110919922, + "nauc_ndcg_at_10_diff1": 0.009681017763689211, + "nauc_ndcg_at_10_max": 0.3360390307609726, + "nauc_ndcg_at_10_std": 0.08059365276563066, + "nauc_ndcg_at_1_diff1": -0.002036085072069493, + "nauc_ndcg_at_1_max": 0.3561939320633002, + "nauc_ndcg_at_1_std": 0.029170446527567652, + "nauc_ndcg_at_20_diff1": -0.0038663194029288084, + "nauc_ndcg_at_20_max": 0.3251206918661345, + "nauc_ndcg_at_20_std": 0.08619257062666767, + "nauc_ndcg_at_3_diff1": -0.03739285196077174, + "nauc_ndcg_at_3_max": 0.344486029969986, + "nauc_ndcg_at_3_std": 0.06494039425865014, + "nauc_ndcg_at_5_diff1": 0.0052680993713773135, + "nauc_ndcg_at_5_max": 0.3631429134715458, + "nauc_ndcg_at_5_std": 0.0824333133139938, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.044874797333814595, + "nauc_precision_at_100_max": 0.3857863448027387, + "nauc_precision_at_100_std": 0.21624932444604655, + "nauc_precision_at_10_diff1": 0.05381696254516126, + "nauc_precision_at_10_max": 0.3068385621084915, + "nauc_precision_at_10_std": 0.11257245340888383, + "nauc_precision_at_1_diff1": -0.002036085072069493, + "nauc_precision_at_1_max": 0.3561939320633002, + "nauc_precision_at_1_std": 0.029170446527567652, + "nauc_precision_at_20_diff1": 0.01175810408727945, + "nauc_precision_at_20_max": 0.27744427624367113, + "nauc_precision_at_20_std": 0.1350289711332675, + "nauc_precision_at_3_diff1": -0.042789959261921945, + "nauc_precision_at_3_max": 0.34194524323028996, + "nauc_precision_at_3_std": 0.08451653486700217, + "nauc_precision_at_5_diff1": 0.04367159641421233, + "nauc_precision_at_5_max": 0.3775874071232719, + "nauc_precision_at_5_std": 0.11848442228189054, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.04487479733381322, + "nauc_recall_at_100_max": 0.38578634480273827, + "nauc_recall_at_100_std": 0.21624932444604553, + "nauc_recall_at_10_diff1": 0.053816962545161474, + "nauc_recall_at_10_max": 0.3068385621084916, + "nauc_recall_at_10_std": 0.11257245340888422, + "nauc_recall_at_1_diff1": -0.002036085072069493, + "nauc_recall_at_1_max": 0.3561939320633002, + "nauc_recall_at_1_std": 0.029170446527567652, + "nauc_recall_at_20_diff1": 0.011758104087278764, + "nauc_recall_at_20_max": 0.27744427624367063, + "nauc_recall_at_20_std": 0.13502897113326723, + "nauc_recall_at_3_diff1": -0.04278995926192189, + "nauc_recall_at_3_max": 0.3419452432302902, + "nauc_recall_at_3_std": 0.08451653486700222, + "nauc_recall_at_5_diff1": 0.04367159641421238, + "nauc_recall_at_5_max": 0.37758740712327205, + "nauc_recall_at_5_std": 0.11848442228189042, + "ndcg_at_1": 0.15, + "ndcg_at_10": 0.25127, + "ndcg_at_100": 0.31342, + "ndcg_at_1000": 0.35084, + "ndcg_at_20": 0.26638, + "ndcg_at_3": 0.20286, + "ndcg_at_5": 0.21534, + "precision_at_1": 0.15, + "precision_at_10": 0.038, + "precision_at_100": 0.007, + "precision_at_1000": 0.001, + "precision_at_20": 0.022, + "precision_at_3": 0.08, + "precision_at_5": 0.054, + "recall_at_1": 0.15, + "recall_at_10": 0.38, + "recall_at_100": 0.7, + "recall_at_1000": 1.0, + "recall_at_20": 0.44, + "recall_at_3": 0.24, + "recall_at_5": 0.27 + } + ] + }, + "task_name": "VidoreSyntheticDocQAAIRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAEnergyRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..5e1e99143f --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", + "evaluation_time": 28.03231382369995, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.16, + "cv_recall_at_10": 0.51, + "cv_recall_at_100": 0.83, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.59, + "cv_recall_at_3": 0.34, + "cv_recall_at_5": 0.4, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.27804, + "map_at_1": 0.13, + "map_at_10": 0.25134, + "map_at_100": 0.26306, + "map_at_1000": 0.26398, + "map_at_20": 0.2567, + "map_at_3": 0.22, + "map_at_5": 0.237, + "mrr_at_1": 0.16, + "mrr_at_10": 0.26677380952380947, + "mrr_at_100": 0.2784318900512844, + "mrr_at_1000": 0.27935058562216325, + "mrr_at_20": 0.2720564384445963, + "mrr_at_3": 0.2383333333333333, + "mrr_at_5": 0.2523333333333333, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.048383351715331105, + "nauc_cv_recall_at_100_max": 0.2678934231148774, + "nauc_cv_recall_at_100_std": 0.3938294537644398, + "nauc_cv_recall_at_10_diff1": -0.21793680922956815, + "nauc_cv_recall_at_10_max": 0.2884105380773526, + "nauc_cv_recall_at_10_std": 0.04301118694721539, + "nauc_cv_recall_at_1_diff1": -0.1065447841450301, + "nauc_cv_recall_at_1_max": 0.4035566139191886, + "nauc_cv_recall_at_1_std": -0.08908818558918431, + "nauc_cv_recall_at_20_diff1": -0.17946498011651307, + "nauc_cv_recall_at_20_max": 0.2712518526127342, + "nauc_cv_recall_at_20_std": 0.0032283152596590085, + "nauc_cv_recall_at_3_diff1": -0.24440234240440936, + "nauc_cv_recall_at_3_max": 0.3217626328201594, + "nauc_cv_recall_at_3_std": 0.022841092768753537, + "nauc_cv_recall_at_5_diff1": -0.17510298661174065, + "nauc_cv_recall_at_5_max": 0.33388259526261593, + "nauc_cv_recall_at_5_std": 0.047528321318228595, + "nauc_map_at_1000_diff1": -0.12118347269446686, + "nauc_map_at_1000_max": 0.40798998686985033, + "nauc_map_at_1000_std": 0.033248252345893674, + "nauc_map_at_100_diff1": -0.12178214546037262, + "nauc_map_at_100_max": 0.4069924113722096, + "nauc_map_at_100_std": 0.034050458694374126, + "nauc_map_at_10_diff1": -0.12829595983501216, + "nauc_map_at_10_max": 0.4085183545597175, + "nauc_map_at_10_std": 0.03090773118576432, + "nauc_map_at_1_diff1": 0.013386760985874355, + "nauc_map_at_1_max": 0.5047570011864522, + "nauc_map_at_1_std": 0.05936737480692162, + "nauc_map_at_20_diff1": -0.12576195523349035, + "nauc_map_at_20_max": 0.40697775682648785, + "nauc_map_at_20_std": 0.028255055376397914, + "nauc_map_at_3_diff1": -0.15234727486013372, + "nauc_map_at_3_max": 0.4206173299043492, + "nauc_map_at_3_std": 0.02386471304818614, + "nauc_map_at_5_diff1": -0.12167568813457902, + "nauc_map_at_5_max": 0.4171512252509577, + "nauc_map_at_5_std": 0.030444733533239992, + "nauc_mrr_at_1000_diff1": -0.1568288547366136, + "nauc_mrr_at_1000_max": 0.3562663780279398, + "nauc_mrr_at_1000_std": -0.025261925833143874, + "nauc_mrr_at_100_diff1": -0.1572874396536292, + "nauc_mrr_at_100_max": 0.3554807097494433, + "nauc_mrr_at_100_std": -0.024336608137159148, + "nauc_mrr_at_10_diff1": -0.16286454153286034, + "nauc_mrr_at_10_max": 0.3577889522492966, + "nauc_mrr_at_10_std": -0.026007433851469176, + "nauc_mrr_at_1_diff1": -0.1065447841450301, + "nauc_mrr_at_1_max": 0.4035566139191886, + "nauc_mrr_at_1_std": -0.08908818558918431, + "nauc_mrr_at_20_diff1": -0.16059994761425353, + "nauc_mrr_at_20_max": 0.3561607628353277, + "nauc_mrr_at_20_std": -0.029019937409929492, + "nauc_mrr_at_3_diff1": -0.17650657774505316, + "nauc_mrr_at_3_max": 0.36178524794668954, + "nauc_mrr_at_3_std": -0.03691059193084039, + "nauc_mrr_at_5_diff1": -0.15646123751391736, + "nauc_mrr_at_5_max": 0.36587355422576173, + "nauc_mrr_at_5_std": -0.027926371058717044, + "nauc_ndcg_at_1000_diff1": -0.12540293991394347, + "nauc_ndcg_at_1000_max": 0.388934244167919, + "nauc_ndcg_at_1000_std": 0.042555564548196194, + "nauc_ndcg_at_100_diff1": -0.12334124629330959, + "nauc_ndcg_at_100_max": 0.3765402027977492, + "nauc_ndcg_at_100_std": 0.06951668374021025, + "nauc_ndcg_at_10_diff1": -0.1554041086320955, + "nauc_ndcg_at_10_max": 0.3738909303116026, + "nauc_ndcg_at_10_std": 0.034099449761324147, + "nauc_ndcg_at_1_diff1": 0.013386760985874355, + "nauc_ndcg_at_1_max": 0.5047570011864522, + "nauc_ndcg_at_1_std": 0.05936737480692162, + "nauc_ndcg_at_20_diff1": -0.14450745932102255, + "nauc_ndcg_at_20_max": 0.3700666109489038, + "nauc_ndcg_at_20_std": 0.023554961081632526, + "nauc_ndcg_at_3_diff1": -0.19126335236855005, + "nauc_ndcg_at_3_max": 0.3961688610945783, + "nauc_ndcg_at_3_std": 0.024002257480038775, + "nauc_ndcg_at_5_diff1": -0.139901062146425, + "nauc_ndcg_at_5_max": 0.3917329472525927, + "nauc_ndcg_at_5_std": 0.0347569625654395, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": -0.04838335171533055, + "nauc_precision_at_100_max": 0.26789342311487824, + "nauc_precision_at_100_std": 0.3938294537644401, + "nauc_precision_at_10_diff1": -0.21793680922956724, + "nauc_precision_at_10_max": 0.2884105380773535, + "nauc_precision_at_10_std": 0.04301118694721595, + "nauc_precision_at_1_diff1": 0.013386760985874355, + "nauc_precision_at_1_max": 0.5047570011864522, + "nauc_precision_at_1_std": 0.05936737480692162, + "nauc_precision_at_20_diff1": -0.17946498011651218, + "nauc_precision_at_20_max": 0.27125185261273455, + "nauc_precision_at_20_std": 0.003228315259659776, + "nauc_precision_at_3_diff1": -0.2801586984596132, + "nauc_precision_at_3_max": 0.3391380756480404, + "nauc_precision_at_3_std": 0.026454181145238276, + "nauc_precision_at_5_diff1": -0.17510298661174045, + "nauc_precision_at_5_max": 0.33388259526261604, + "nauc_precision_at_5_std": 0.04752832131822858, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.048383351715331105, + "nauc_recall_at_100_max": 0.2678934231148774, + "nauc_recall_at_100_std": 0.3938294537644398, + "nauc_recall_at_10_diff1": -0.21793680922956815, + "nauc_recall_at_10_max": 0.2884105380773526, + "nauc_recall_at_10_std": 0.04301118694721539, + "nauc_recall_at_1_diff1": 0.013386760985874355, + "nauc_recall_at_1_max": 0.5047570011864522, + "nauc_recall_at_1_std": 0.05936737480692162, + "nauc_recall_at_20_diff1": -0.17946498011651307, + "nauc_recall_at_20_max": 0.2712518526127342, + "nauc_recall_at_20_std": 0.0032283152596590085, + "nauc_recall_at_3_diff1": -0.28015869845961333, + "nauc_recall_at_3_max": 0.3391380756480409, + "nauc_recall_at_3_std": 0.026454181145238197, + "nauc_recall_at_5_diff1": -0.17510298661174065, + "nauc_recall_at_5_max": 0.33388259526261593, + "nauc_recall_at_5_std": 0.047528321318228595, + "ndcg_at_1": 0.13, + "ndcg_at_10": 0.31327, + "ndcg_at_100": 0.37801, + "ndcg_at_1000": 0.40025, + "ndcg_at_20": 0.33322, + "ndcg_at_3": 0.24833, + "ndcg_at_5": 0.27804, + "precision_at_1": 0.13, + "precision_at_10": 0.051, + "precision_at_100": 0.0083, + "precision_at_1000": 0.001, + "precision_at_20": 0.0295, + "precision_at_3": 0.11, + "precision_at_5": 0.08, + "recall_at_1": 0.13, + "recall_at_10": 0.51, + "recall_at_100": 0.83, + "recall_at_1000": 1.0, + "recall_at_20": 0.59, + "recall_at_3": 0.33, + "recall_at_5": 0.4 + } + ] + }, + "task_name": "VidoreSyntheticDocQAEnergyRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..3b7e8c92eb --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", + "evaluation_time": 29.03953790664673, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.17, + "cv_recall_at_10": 0.53, + "cv_recall_at_100": 0.88, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.65, + "cv_recall_at_3": 0.41, + "cv_recall_at_5": 0.47, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.34996, + "map_at_1": 0.19, + "map_at_10": 0.3125, + "map_at_100": 0.3268, + "map_at_1000": 0.32732, + "map_at_20": 0.32092, + "map_at_3": 0.28667, + "map_at_5": 0.30617, + "mrr_at_1": 0.17, + "mrr_at_10": 0.3043015873015873, + "mrr_at_100": 0.31867453920623073, + "mrr_at_1000": 0.31919269880635714, + "mrr_at_20": 0.3127636329958311, + "mrr_at_3": 0.2816666666666666, + "mrr_at_5": 0.29616666666666663, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.3516544702513509, + "nauc_cv_recall_at_100_max": 0.6257954183900725, + "nauc_cv_recall_at_100_std": -0.08725739739102838, + "nauc_cv_recall_at_10_diff1": 0.11492941483731742, + "nauc_cv_recall_at_10_max": 0.6216305378768097, + "nauc_cv_recall_at_10_std": 0.1785041718786511, + "nauc_cv_recall_at_1_diff1": 0.33847338601734545, + "nauc_cv_recall_at_1_max": 0.42578705146775714, + "nauc_cv_recall_at_1_std": -0.028346687691376937, + "nauc_cv_recall_at_20_diff1": 0.045601757669460355, + "nauc_cv_recall_at_20_max": 0.550069167548214, + "nauc_cv_recall_at_20_std": 0.09712751240947172, + "nauc_cv_recall_at_3_diff1": -0.004456385548393712, + "nauc_cv_recall_at_3_max": 0.5421160726972114, + "nauc_cv_recall_at_3_std": 0.12001240037717818, + "nauc_cv_recall_at_5_diff1": 0.08508814725932576, + "nauc_cv_recall_at_5_max": 0.6091588760400324, + "nauc_cv_recall_at_5_std": 0.19719405582879526, + "nauc_map_at_1000_diff1": 0.1425413131213182, + "nauc_map_at_1000_max": 0.49699745466836864, + "nauc_map_at_1000_std": 0.09344073064154622, + "nauc_map_at_100_diff1": 0.14260967351190185, + "nauc_map_at_100_max": 0.4972447058805711, + "nauc_map_at_100_std": 0.09313436868733994, + "nauc_map_at_10_diff1": 0.1409243269612414, + "nauc_map_at_10_max": 0.502989362838475, + "nauc_map_at_10_std": 0.10005581107730499, + "nauc_map_at_1_diff1": 0.23836026432162497, + "nauc_map_at_1_max": 0.3865115262387973, + "nauc_map_at_1_std": 0.04561733896053668, + "nauc_map_at_20_diff1": 0.1371717127110203, + "nauc_map_at_20_max": 0.4980711323828558, + "nauc_map_at_20_std": 0.09531305443499136, + "nauc_map_at_3_diff1": 0.11010341190945848, + "nauc_map_at_3_max": 0.4888800460910493, + "nauc_map_at_3_std": 0.07477345077436877, + "nauc_map_at_5_diff1": 0.13706143128873957, + "nauc_map_at_5_max": 0.5032336982522804, + "nauc_map_at_5_std": 0.10533815409735035, + "nauc_mrr_at_1000_diff1": 0.18678624669182986, + "nauc_mrr_at_1000_max": 0.5091271845519432, + "nauc_mrr_at_1000_std": 0.06389258559560088, + "nauc_mrr_at_100_diff1": 0.18678259303086617, + "nauc_mrr_at_100_max": 0.5093634612617053, + "nauc_mrr_at_100_std": 0.06362384958529906, + "nauc_mrr_at_10_diff1": 0.18388219199565162, + "nauc_mrr_at_10_max": 0.515153193106594, + "nauc_mrr_at_10_std": 0.07149646969980394, + "nauc_mrr_at_1_diff1": 0.33847338601734545, + "nauc_mrr_at_1_max": 0.42578705146775714, + "nauc_mrr_at_1_std": -0.028346687691376937, + "nauc_mrr_at_20_diff1": 0.18056301217995185, + "nauc_mrr_at_20_max": 0.5100760746918761, + "nauc_mrr_at_20_std": 0.06626317479181122, + "nauc_mrr_at_3_diff1": 0.15186627578465683, + "nauc_mrr_at_3_max": 0.4937757023080455, + "nauc_mrr_at_3_std": 0.054727704395902466, + "nauc_mrr_at_5_diff1": 0.17992256895127598, + "nauc_mrr_at_5_max": 0.5144597679413234, + "nauc_mrr_at_5_std": 0.07481339912415413, + "nauc_ndcg_at_1000_diff1": 0.14018428326853596, + "nauc_ndcg_at_1000_max": 0.5095848675975442, + "nauc_ndcg_at_1000_std": 0.09683580225656489, + "nauc_ndcg_at_100_diff1": 0.14984320552095684, + "nauc_ndcg_at_100_max": 0.5158634936681875, + "nauc_ndcg_at_100_std": 0.08711396818198522, + "nauc_ndcg_at_10_diff1": 0.13101689912462652, + "nauc_ndcg_at_10_max": 0.53491224497677, + "nauc_ndcg_at_10_std": 0.12134567341471547, + "nauc_ndcg_at_1_diff1": 0.23836026432162497, + "nauc_ndcg_at_1_max": 0.3865115262387973, + "nauc_ndcg_at_1_std": 0.04561733896053668, + "nauc_ndcg_at_20_diff1": 0.11588990092348003, + "nauc_ndcg_at_20_max": 0.5156627793963243, + "nauc_ndcg_at_20_std": 0.10148201064617794, + "nauc_ndcg_at_3_diff1": 0.07592271092042169, + "nauc_ndcg_at_3_max": 0.510637228334191, + "nauc_ndcg_at_3_std": 0.08301779806797376, + "nauc_ndcg_at_5_diff1": 0.1204085206167412, + "nauc_ndcg_at_5_max": 0.5345240398050349, + "nauc_ndcg_at_5_std": 0.13407080761621448, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3516544702513525, + "nauc_precision_at_100_max": 0.6257954183900729, + "nauc_precision_at_100_std": -0.08725739739102806, + "nauc_precision_at_10_diff1": 0.11492941483731785, + "nauc_precision_at_10_max": 0.6216305378768097, + "nauc_precision_at_10_std": 0.17850417187865175, + "nauc_precision_at_1_diff1": 0.23836026432162497, + "nauc_precision_at_1_max": 0.3865115262387973, + "nauc_precision_at_1_std": 0.04561733896053668, + "nauc_precision_at_20_diff1": 0.04560175766946091, + "nauc_precision_at_20_max": 0.5500691675482139, + "nauc_precision_at_20_std": 0.09712751240947225, + "nauc_precision_at_3_diff1": -0.01078784757981495, + "nauc_precision_at_3_max": 0.5642507723995873, + "nauc_precision_at_3_std": 0.10406797116374848, + "nauc_precision_at_5_diff1": 0.08190430824600467, + "nauc_precision_at_5_max": 0.6161347517730504, + "nauc_precision_at_5_std": 0.2138509579760776, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.3516544702513509, + "nauc_recall_at_100_max": 0.6257954183900725, + "nauc_recall_at_100_std": -0.08725739739102838, + "nauc_recall_at_10_diff1": 0.11492941483731742, + "nauc_recall_at_10_max": 0.6216305378768097, + "nauc_recall_at_10_std": 0.1785041718786511, + "nauc_recall_at_1_diff1": 0.23836026432162497, + "nauc_recall_at_1_max": 0.3865115262387973, + "nauc_recall_at_1_std": 0.04561733896053668, + "nauc_recall_at_20_diff1": 0.045601757669460355, + "nauc_recall_at_20_max": 0.550069167548214, + "nauc_recall_at_20_std": 0.09712751240947172, + "nauc_recall_at_3_diff1": -0.010787847579814863, + "nauc_recall_at_3_max": 0.5642507723995882, + "nauc_recall_at_3_std": 0.10406797116374862, + "nauc_recall_at_5_diff1": 0.08190430824600382, + "nauc_recall_at_5_max": 0.6161347517730497, + "nauc_recall_at_5_std": 0.21385095797607692, + "ndcg_at_1": 0.19, + "ndcg_at_10": 0.36581, + "ndcg_at_100": 0.43883, + "ndcg_at_1000": 0.4539, + "ndcg_at_20": 0.39628, + "ndcg_at_3": 0.31595, + "ndcg_at_5": 0.34996, + "precision_at_1": 0.19, + "precision_at_10": 0.053, + "precision_at_100": 0.0088, + "precision_at_1000": 0.001, + "precision_at_20": 0.0325, + "precision_at_3": 0.13333, + "precision_at_5": 0.096, + "recall_at_1": 0.19, + "recall_at_10": 0.53, + "recall_at_100": 0.88, + "recall_at_1000": 1.0, + "recall_at_20": 0.65, + "recall_at_3": 0.4, + "recall_at_5": 0.48 + } + ] + }, + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..956d9f38be --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", + "evaluation_time": 27.550007104873657, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.19, + "cv_recall_at_10": 0.54, + "cv_recall_at_100": 0.82, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.65, + "cv_recall_at_3": 0.38, + "cv_recall_at_5": 0.45, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.33111, + "map_at_1": 0.19, + "map_at_10": 0.30356, + "map_at_100": 0.31497, + "map_at_1000": 0.31591, + "map_at_20": 0.3098, + "map_at_3": 0.27833, + "map_at_5": 0.29133, + "mrr_at_1": 0.19, + "mrr_at_10": 0.3056468253968253, + "mrr_at_100": 0.3170461377031122, + "mrr_at_1000": 0.31798498793135255, + "mrr_at_20": 0.3123555641554867, + "mrr_at_3": 0.2783333333333333, + "mrr_at_5": 0.2938333333333333, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.15676285934574333, + "nauc_cv_recall_at_100_max": 0.45654807798215596, + "nauc_cv_recall_at_100_std": -0.14313250357968876, + "nauc_cv_recall_at_10_diff1": -0.09325942596760317, + "nauc_cv_recall_at_10_max": 0.31298102303353603, + "nauc_cv_recall_at_10_std": -0.25989830791031104, + "nauc_cv_recall_at_1_diff1": 0.002084979738704942, + "nauc_cv_recall_at_1_max": 0.38790711751551116, + "nauc_cv_recall_at_1_std": -0.11869251593159925, + "nauc_cv_recall_at_20_diff1": 0.06229961754414476, + "nauc_cv_recall_at_20_max": 0.31618520628204094, + "nauc_cv_recall_at_20_std": -0.27548213849784403, + "nauc_cv_recall_at_3_diff1": -0.2096405271229176, + "nauc_cv_recall_at_3_max": 0.3323369635848518, + "nauc_cv_recall_at_3_std": -0.21148597717880088, + "nauc_cv_recall_at_5_diff1": -0.13304524323970407, + "nauc_cv_recall_at_5_max": 0.37726707261179854, + "nauc_cv_recall_at_5_std": -0.24580632488705587, + "nauc_map_at_1000_diff1": -0.09235421018350352, + "nauc_map_at_1000_max": 0.3155831508446477, + "nauc_map_at_1000_std": -0.21978739885425327, + "nauc_map_at_100_diff1": -0.09169127502452307, + "nauc_map_at_100_max": 0.31590394921168813, + "nauc_map_at_100_std": -0.21997561809528432, + "nauc_map_at_10_diff1": -0.09858535432501385, + "nauc_map_at_10_max": 0.31377485989975207, + "nauc_map_at_10_std": -0.21996261057713298, + "nauc_map_at_1_diff1": 0.002084979738704942, + "nauc_map_at_1_max": 0.27975720074655724, + "nauc_map_at_1_std": -0.2348460646007433, + "nauc_map_at_20_diff1": -0.08808591850158354, + "nauc_map_at_20_max": 0.3147501333995878, + "nauc_map_at_20_std": -0.22056529702822625, + "nauc_map_at_3_diff1": -0.12443445016908257, + "nauc_map_at_3_max": 0.31066801703822294, + "nauc_map_at_3_std": -0.2136006611262196, + "nauc_map_at_5_diff1": -0.10518620663878896, + "nauc_map_at_5_max": 0.32470654018637624, + "nauc_map_at_5_std": -0.20637715997684672, + "nauc_mrr_at_1000_diff1": -0.08252339841089382, + "nauc_mrr_at_1000_max": 0.37186520512978016, + "nauc_mrr_at_1000_std": -0.16387062255497364, + "nauc_mrr_at_100_diff1": -0.08188638538702586, + "nauc_mrr_at_100_max": 0.37203061930902376, + "nauc_mrr_at_100_std": -0.1642176357964918, + "nauc_mrr_at_10_diff1": -0.08858050769501902, + "nauc_mrr_at_10_max": 0.36905843834344043, + "nauc_mrr_at_10_std": -0.16523125576816933, + "nauc_mrr_at_1_diff1": 0.002084979738704942, + "nauc_mrr_at_1_max": 0.38790711751551116, + "nauc_mrr_at_1_std": -0.11869251593159925, + "nauc_mrr_at_20_diff1": -0.07936545859048873, + "nauc_mrr_at_20_max": 0.3691865273941238, + "nauc_mrr_at_20_std": -0.16584492656687405, + "nauc_mrr_at_3_diff1": -0.11139584441954048, + "nauc_mrr_at_3_max": 0.36619428184673924, + "nauc_mrr_at_3_std": -0.15469519776158758, + "nauc_mrr_at_5_diff1": -0.09444148817962006, + "nauc_mrr_at_5_max": 0.37868655568518383, + "nauc_mrr_at_5_std": -0.1628485012637727, + "nauc_ndcg_at_1000_diff1": -0.08845801918452714, + "nauc_ndcg_at_1000_max": 0.321947402125894, + "nauc_ndcg_at_1000_std": -0.2209402250110133, + "nauc_ndcg_at_100_diff1": -0.06956430063277862, + "nauc_ndcg_at_100_max": 0.33184409702496304, + "nauc_ndcg_at_100_std": -0.21689400558975963, + "nauc_ndcg_at_10_diff1": -0.10082683668530741, + "nauc_ndcg_at_10_max": 0.31499392517072217, + "nauc_ndcg_at_10_std": -0.22898881208713334, + "nauc_ndcg_at_1_diff1": 0.002084979738704942, + "nauc_ndcg_at_1_max": 0.27975720074655724, + "nauc_ndcg_at_1_std": -0.2348460646007433, + "nauc_ndcg_at_20_diff1": -0.06058784979969239, + "nauc_ndcg_at_20_max": 0.3204646041250894, + "nauc_ndcg_at_20_std": -0.23011656634574992, + "nauc_ndcg_at_3_diff1": -0.15159027688467022, + "nauc_ndcg_at_3_max": 0.31394762477267585, + "nauc_ndcg_at_3_std": -0.2132489721131679, + "nauc_ndcg_at_5_diff1": -0.11540331552795374, + "nauc_ndcg_at_5_max": 0.3388623148185459, + "nauc_ndcg_at_5_std": -0.20140518922691222, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.15676285934574297, + "nauc_precision_at_100_max": 0.4565480779821573, + "nauc_precision_at_100_std": -0.14313250357968957, + "nauc_precision_at_10_diff1": -0.09325942596760275, + "nauc_precision_at_10_max": 0.3129810230335366, + "nauc_precision_at_10_std": -0.25989830791031043, + "nauc_precision_at_1_diff1": 0.002084979738704942, + "nauc_precision_at_1_max": 0.27975720074655724, + "nauc_precision_at_1_std": -0.2348460646007433, + "nauc_precision_at_20_diff1": 0.0800708902797296, + "nauc_precision_at_20_max": 0.3399220845574146, + "nauc_precision_at_20_std": -0.2660620768936003, + "nauc_precision_at_3_diff1": -0.21856608849740672, + "nauc_precision_at_3_max": 0.32083866878791373, + "nauc_precision_at_3_std": -0.21388382290171126, + "nauc_precision_at_5_diff1": -0.13304524323970335, + "nauc_precision_at_5_max": 0.3772670726117992, + "nauc_precision_at_5_std": -0.1891180514633659, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.15676285934574333, + "nauc_recall_at_100_max": 0.45654807798215596, + "nauc_recall_at_100_std": -0.14313250357968876, + "nauc_recall_at_10_diff1": -0.09325942596760317, + "nauc_recall_at_10_max": 0.31298102303353603, + "nauc_recall_at_10_std": -0.25989830791031104, + "nauc_recall_at_1_diff1": 0.002084979738704942, + "nauc_recall_at_1_max": 0.27975720074655724, + "nauc_recall_at_1_std": -0.2348460646007433, + "nauc_recall_at_20_diff1": 0.08007089027972884, + "nauc_recall_at_20_max": 0.3399220845574144, + "nauc_recall_at_20_std": -0.2660620768936013, + "nauc_recall_at_3_diff1": -0.2185660884974068, + "nauc_recall_at_3_max": 0.3208386687879141, + "nauc_recall_at_3_std": -0.21388382290171132, + "nauc_recall_at_5_diff1": -0.13304524323970407, + "nauc_recall_at_5_max": 0.37726707261179854, + "nauc_recall_at_5_std": -0.1891180514633668, + "ndcg_at_1": 0.19, + "ndcg_at_10": 0.36044, + "ndcg_at_100": 0.41927, + "ndcg_at_1000": 0.44269, + "ndcg_at_20": 0.38488, + "ndcg_at_3": 0.30702, + "ndcg_at_5": 0.33111, + "precision_at_1": 0.19, + "precision_at_10": 0.054, + "precision_at_100": 0.0082, + "precision_at_1000": 0.001, + "precision_at_20": 0.032, + "precision_at_3": 0.13, + "precision_at_5": 0.09, + "recall_at_1": 0.19, + "recall_at_10": 0.54, + "recall_at_100": 0.82, + "recall_at_1000": 1.0, + "recall_at_20": 0.64, + "recall_at_3": 0.39, + "recall_at_5": 0.45 + } + ] + }, + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTabfquadRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..d165b2f16a --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", + "evaluation_time": 4.353839874267578, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.02143, + "cv_recall_at_10": 0.175, + "cv_recall_at_100": 0.64643, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.24643, + "cv_recall_at_3": 0.07857, + "cv_recall_at_5": 0.11786, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.06876, + "map_at_1": 0.01429, + "map_at_10": 0.05641, + "map_at_100": 0.06963, + "map_at_1000": 0.07195, + "map_at_20": 0.06128, + "map_at_3": 0.03988, + "map_at_5": 0.05042, + "mrr_at_1": 0.02142857142857143, + "mrr_at_10": 0.06293792517006803, + "mrr_at_100": 0.07592145747016359, + "mrr_at_1000": 0.07823686336276094, + "mrr_at_20": 0.06762745758958719, + "mrr_at_3": 0.04642857142857143, + "mrr_at_5": 0.055535714285714285, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.07013675223542849, + "nauc_cv_recall_at_100_max": 0.11614992524518984, + "nauc_cv_recall_at_100_std": 0.265093833161118, + "nauc_cv_recall_at_10_diff1": -0.062031339049582676, + "nauc_cv_recall_at_10_max": 0.21408400197229802, + "nauc_cv_recall_at_10_std": 0.20956948829108032, + "nauc_cv_recall_at_1_diff1": -0.3614614077452824, + "nauc_cv_recall_at_1_max": 0.3357928628924066, + "nauc_cv_recall_at_1_std": 0.37492174224130215, + "nauc_cv_recall_at_20_diff1": -0.04239040201350441, + "nauc_cv_recall_at_20_max": 0.16552719658634857, + "nauc_cv_recall_at_20_std": 0.21981990556410375, + "nauc_cv_recall_at_3_diff1": -0.19656316315827993, + "nauc_cv_recall_at_3_max": 0.37258823816376824, + "nauc_cv_recall_at_3_std": 0.19223357806668773, + "nauc_cv_recall_at_5_diff1": -0.1539878825273585, + "nauc_cv_recall_at_5_max": 0.33965772853090626, + "nauc_cv_recall_at_5_std": 0.2498221799453256, + "nauc_map_at_1000_diff1": -0.11477836932696846, + "nauc_map_at_1000_max": 0.2906056774186307, + "nauc_map_at_1000_std": 0.23287433742435157, + "nauc_map_at_100_diff1": -0.1113113159044201, + "nauc_map_at_100_max": 0.28669249289561527, + "nauc_map_at_100_std": 0.2326195062209629, + "nauc_map_at_10_diff1": -0.12493110967755462, + "nauc_map_at_10_max": 0.29585346508604804, + "nauc_map_at_10_std": 0.2185152261808974, + "nauc_map_at_1_diff1": -0.25248188891870144, + "nauc_map_at_1_max": 0.5492353099007244, + "nauc_map_at_1_std": 0.47719345317950096, + "nauc_map_at_20_diff1": -0.12048170875102637, + "nauc_map_at_20_max": 0.2853357651323181, + "nauc_map_at_20_std": 0.22407682393584646, + "nauc_map_at_3_diff1": -0.1865611571869657, + "nauc_map_at_3_max": 0.31550725043951156, + "nauc_map_at_3_std": 0.21905898465814205, + "nauc_map_at_5_diff1": -0.14819065296774128, + "nauc_map_at_5_max": 0.31966766311722844, + "nauc_map_at_5_std": 0.2253066719087251, + "nauc_mrr_at_1000_diff1": -0.1731260258556916, + "nauc_mrr_at_1000_max": 0.2870994898422976, + "nauc_mrr_at_1000_std": 0.23989941101230317, + "nauc_mrr_at_100_diff1": -0.16901002627023246, + "nauc_mrr_at_100_max": 0.283614353441143, + "nauc_mrr_at_100_std": 0.23956904850637092, + "nauc_mrr_at_10_diff1": -0.1833916598209548, + "nauc_mrr_at_10_max": 0.2930079237890623, + "nauc_mrr_at_10_std": 0.23234705273326456, + "nauc_mrr_at_1_diff1": -0.3614614077452824, + "nauc_mrr_at_1_max": 0.3357928628924066, + "nauc_mrr_at_1_std": 0.37492174224130215, + "nauc_mrr_at_20_diff1": -0.177786791550534, + "nauc_mrr_at_20_max": 0.2824709513782039, + "nauc_mrr_at_20_std": 0.23149134244179065, + "nauc_mrr_at_3_diff1": -0.2519659036965182, + "nauc_mrr_at_3_max": 0.35485335699985543, + "nauc_mrr_at_3_std": 0.22600152731625767, + "nauc_mrr_at_5_diff1": -0.22520366189697133, + "nauc_mrr_at_5_max": 0.3373958293786979, + "nauc_mrr_at_5_std": 0.2458651801062478, + "nauc_ndcg_at_1000_diff1": -0.08616081793508518, + "nauc_ndcg_at_1000_max": 0.2636258033034474, + "nauc_ndcg_at_1000_std": 0.2400451291595649, + "nauc_ndcg_at_100_diff1": -0.043638278763048495, + "nauc_ndcg_at_100_max": 0.22150656020775003, + "nauc_ndcg_at_100_std": 0.2443691153022479, + "nauc_ndcg_at_10_diff1": -0.0959049965441373, + "nauc_ndcg_at_10_max": 0.25740388210300413, + "nauc_ndcg_at_10_std": 0.20599782546699, + "nauc_ndcg_at_1_diff1": -0.25248188891870144, + "nauc_ndcg_at_1_max": 0.5492353099007244, + "nauc_ndcg_at_1_std": 0.47719345317950096, + "nauc_ndcg_at_20_diff1": -0.08485754868327161, + "nauc_ndcg_at_20_max": 0.23109618682790573, + "nauc_ndcg_at_20_std": 0.22020858825779455, + "nauc_ndcg_at_3_diff1": -0.18560257748100978, + "nauc_ndcg_at_3_max": 0.28745044081818694, + "nauc_ndcg_at_3_std": 0.20898634536396835, + "nauc_ndcg_at_5_diff1": -0.13078634593787122, + "nauc_ndcg_at_5_max": 0.29665194805795475, + "nauc_ndcg_at_5_std": 0.2203277736803905, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.07013675223542876, + "nauc_precision_at_100_max": 0.11614992524519006, + "nauc_precision_at_100_std": 0.2650938331611187, + "nauc_precision_at_10_diff1": -0.05865474703310428, + "nauc_precision_at_10_max": 0.20959556527170534, + "nauc_precision_at_10_std": 0.1914168228190715, + "nauc_precision_at_1_diff1": -0.25248188891870144, + "nauc_precision_at_1_max": 0.5492353099007244, + "nauc_precision_at_1_std": 0.47719345317950096, + "nauc_precision_at_20_diff1": -0.0423904020135042, + "nauc_precision_at_20_max": 0.16552719658634873, + "nauc_precision_at_20_std": 0.219819905564104, + "nauc_precision_at_3_diff1": -0.18520867380540046, + "nauc_precision_at_3_max": 0.24521306437056364, + "nauc_precision_at_3_std": 0.1969778276459253, + "nauc_precision_at_5_diff1": -0.10594039054470711, + "nauc_precision_at_5_max": 0.26743679342240495, + "nauc_precision_at_5_std": 0.21703597122302157, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.07013675223542849, + "nauc_recall_at_100_max": 0.11614992524518984, + "nauc_recall_at_100_std": 0.265093833161118, + "nauc_recall_at_10_diff1": -0.058654747033104304, + "nauc_recall_at_10_max": 0.209595565271705, + "nauc_recall_at_10_std": 0.19141682281907146, + "nauc_recall_at_1_diff1": -0.25248188891870144, + "nauc_recall_at_1_max": 0.5492353099007244, + "nauc_recall_at_1_std": 0.47719345317950096, + "nauc_recall_at_20_diff1": -0.04239040201350441, + "nauc_recall_at_20_max": 0.16552719658634857, + "nauc_recall_at_20_std": 0.21981990556410375, + "nauc_recall_at_3_diff1": -0.18520867380540046, + "nauc_recall_at_3_max": 0.2452130643705637, + "nauc_recall_at_3_std": 0.19697782764592536, + "nauc_recall_at_5_diff1": -0.10594039054470714, + "nauc_recall_at_5_max": 0.26743679342240495, + "nauc_recall_at_5_std": 0.21703597122302157, + "ndcg_at_1": 0.01429, + "ndcg_at_10": 0.08358, + "ndcg_at_100": 0.17244, + "ndcg_at_1000": 0.22086, + "ndcg_at_20": 0.10211, + "ndcg_at_3": 0.0497, + "ndcg_at_5": 0.06876, + "precision_at_1": 0.01429, + "precision_at_10": 0.01714, + "precision_at_100": 0.00646, + "precision_at_1000": 0.001, + "precision_at_20": 0.01232, + "precision_at_3": 0.02619, + "precision_at_5": 0.025, + "recall_at_1": 0.01429, + "recall_at_10": 0.17143, + "recall_at_100": 0.64643, + "recall_at_1000": 1.0, + "recall_at_20": 0.24643, + "recall_at_3": 0.07857, + "recall_at_5": 0.125 + } + ] + }, + "task_name": "VidoreTabfquadRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTatdqaRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..0053f94100 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", + "evaluation_time": 56.91961908340454, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.00301, + "cv_recall_at_10": 0.03548, + "cv_recall_at_100": 0.14251, + "cv_recall_at_1000": 0.73121, + "cv_recall_at_20": 0.05412, + "cv_recall_at_3": 0.01263, + "cv_recall_at_5": 0.02345, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.01095, + "map_at_1": 0.0018, + "map_at_10": 0.00933, + "map_at_100": 0.01243, + "map_at_1000": 0.01412, + "map_at_20": 0.0105, + "map_at_3": 0.00471, + "map_at_5": 0.00754, + "mrr_at_1": 0.003006614552014432, + "mrr_at_10": 0.010695673338487527, + "mrr_at_100": 0.013802284461660666, + "mrr_at_1000": 0.01548110419231038, + "mrr_at_20": 0.011912744158405977, + "mrr_at_3": 0.006714772499498898, + "mrr_at_5": 0.009089997995590298, + "nauc_cv_recall_at_1000_diff1": -0.014566063340413188, + "nauc_cv_recall_at_1000_max": 0.05409691540398872, + "nauc_cv_recall_at_1000_std": 0.11465856299781937, + "nauc_cv_recall_at_100_diff1": -0.06896710247347743, + "nauc_cv_recall_at_100_max": 0.08860267679149006, + "nauc_cv_recall_at_100_std": 0.17026666742389082, + "nauc_cv_recall_at_10_diff1": 0.10177862892497945, + "nauc_cv_recall_at_10_max": 0.2097063707163848, + "nauc_cv_recall_at_10_std": 0.38240798275194327, + "nauc_cv_recall_at_1_diff1": 0.03526380003576067, + "nauc_cv_recall_at_1_max": 0.4020982414202427, + "nauc_cv_recall_at_1_std": 0.6686073305917339, + "nauc_cv_recall_at_20_diff1": 0.009867753517689027, + "nauc_cv_recall_at_20_max": 0.18905020883601725, + "nauc_cv_recall_at_20_std": 0.2702661332401687, + "nauc_cv_recall_at_3_diff1": 0.04287210481309383, + "nauc_cv_recall_at_3_max": 0.3989496498413514, + "nauc_cv_recall_at_3_std": 0.6033952370820275, + "nauc_cv_recall_at_5_diff1": 0.07819466098025143, + "nauc_cv_recall_at_5_max": 0.33271295458325373, + "nauc_cv_recall_at_5_std": 0.5060975464431268, + "nauc_map_at_1000_diff1": 0.002376257165185258, + "nauc_map_at_1000_max": 0.2819036286133092, + "nauc_map_at_1000_std": 0.41437523372664636, + "nauc_map_at_100_diff1": -0.0012297099110992408, + "nauc_map_at_100_max": 0.28939605934379836, + "nauc_map_at_100_std": 0.4281412770351935, + "nauc_map_at_10_diff1": 0.039876616061044146, + "nauc_map_at_10_max": 0.3563897574654661, + "nauc_map_at_10_std": 0.5389632898745574, + "nauc_map_at_1_diff1": -0.02568513786845483, + "nauc_map_at_1_max": 0.6995235045831298, + "nauc_map_at_1_std": 1.0, + "nauc_map_at_20_diff1": 0.021159229553580298, + "nauc_map_at_20_max": 0.3323683590354517, + "nauc_map_at_20_std": 0.4834271083888167, + "nauc_map_at_3_diff1": -0.06702435727404064, + "nauc_map_at_3_max": 0.5885596535419393, + "nauc_map_at_3_std": 0.7767529197688164, + "nauc_map_at_5_diff1": 0.034734102998953825, + "nauc_map_at_5_max": 0.44997314380901593, + "nauc_map_at_5_std": 0.6196877602314378, + "nauc_mrr_at_1000_diff1": 0.022796670054829382, + "nauc_mrr_at_1000_max": 0.25534052859583023, + "nauc_mrr_at_1000_std": 0.3901775596025565, + "nauc_mrr_at_100_diff1": 0.020228157587260442, + "nauc_mrr_at_100_max": 0.2612247454263831, + "nauc_mrr_at_100_std": 0.4016538436087405, + "nauc_mrr_at_10_diff1": 0.0699129110691725, + "nauc_mrr_at_10_max": 0.3166979674099861, + "nauc_mrr_at_10_std": 0.4943672231606845, + "nauc_mrr_at_1_diff1": 0.03526380003576067, + "nauc_mrr_at_1_max": 0.4020982414202427, + "nauc_mrr_at_1_std": 0.6686073305917339, + "nauc_mrr_at_20_diff1": 0.045078949690099346, + "nauc_mrr_at_20_max": 0.29840241395898265, + "nauc_mrr_at_20_std": 0.4477899188338933, + "nauc_mrr_at_3_diff1": 0.03638945853763958, + "nauc_mrr_at_3_max": 0.414453160995652, + "nauc_mrr_at_3_std": 0.6142739363580281, + "nauc_mrr_at_5_diff1": 0.05644557824415222, + "nauc_mrr_at_5_max": 0.37386357142111964, + "nauc_mrr_at_5_std": 0.5528710632147819, + "nauc_ndcg_at_1000_diff1": -0.012699300979067456, + "nauc_ndcg_at_1000_max": 0.13586894754526818, + "nauc_ndcg_at_1000_std": 0.217056132320633, + "nauc_ndcg_at_100_diff1": -0.043325013809925546, + "nauc_ndcg_at_100_max": 0.15772228815085845, + "nauc_ndcg_at_100_std": 0.25872834420062885, + "nauc_ndcg_at_10_diff1": 0.0562938241735581, + "nauc_ndcg_at_10_max": 0.2794645518226739, + "nauc_ndcg_at_10_std": 0.46133489790312227, + "nauc_ndcg_at_1_diff1": -0.02568513786845483, + "nauc_ndcg_at_1_max": 0.6995235045831298, + "nauc_ndcg_at_1_std": 1.0, + "nauc_ndcg_at_20_diff1": 0.012327678642763356, + "nauc_ndcg_at_20_max": 0.2450981482790461, + "nauc_ndcg_at_20_std": 0.36548222828664273, + "nauc_ndcg_at_3_diff1": -0.07882852915289862, + "nauc_ndcg_at_3_max": 0.5781964658113339, + "nauc_ndcg_at_3_std": 0.7570757867717248, + "nauc_ndcg_at_5_diff1": 0.05284682509921638, + "nauc_ndcg_at_5_max": 0.408496056286339, + "nauc_ndcg_at_5_std": 0.5689798228201582, + "nauc_precision_at_1000_diff1": -0.014566063340413198, + "nauc_precision_at_1000_max": 0.05409691540398878, + "nauc_precision_at_1000_std": 0.11465856299781925, + "nauc_precision_at_100_diff1": -0.07070737778348025, + "nauc_precision_at_100_max": 0.09083326876918361, + "nauc_precision_at_100_std": 0.17229679067374504, + "nauc_precision_at_10_diff1": 0.07230484504920205, + "nauc_precision_at_10_max": 0.20179917541371403, + "nauc_precision_at_10_std": 0.38483848287524763, + "nauc_precision_at_1_diff1": -0.02568513786845483, + "nauc_precision_at_1_max": 0.6995235045831298, + "nauc_precision_at_1_std": 1.0, + "nauc_precision_at_20_diff1": -0.001104068362403356, + "nauc_precision_at_20_max": 0.1744671111021371, + "nauc_precision_at_20_std": 0.2658456862734212, + "nauc_precision_at_3_diff1": -0.09810504569367917, + "nauc_precision_at_3_max": 0.5627605752906821, + "nauc_precision_at_3_std": 0.7280169253981901, + "nauc_precision_at_5_diff1": 0.07493463474156249, + "nauc_precision_at_5_max": 0.36014761171237997, + "nauc_precision_at_5_std": 0.5109657682867575, + "nauc_recall_at_1000_diff1": -0.014566063340413188, + "nauc_recall_at_1000_max": 0.05409691540398872, + "nauc_recall_at_1000_std": 0.11465856299781937, + "nauc_recall_at_100_diff1": -0.07070737778348046, + "nauc_recall_at_100_max": 0.09083326876918348, + "nauc_recall_at_100_std": 0.17229679067374493, + "nauc_recall_at_10_diff1": 0.07230484504920151, + "nauc_recall_at_10_max": 0.2017991754137134, + "nauc_recall_at_10_std": 0.3848384828752471, + "nauc_recall_at_1_diff1": -0.02568513786845483, + "nauc_recall_at_1_max": 0.6995235045831298, + "nauc_recall_at_1_std": 1.0, + "nauc_recall_at_20_diff1": -0.0011040683624037103, + "nauc_recall_at_20_max": 0.17446711110213683, + "nauc_recall_at_20_std": 0.2658456862734206, + "nauc_recall_at_3_diff1": -0.09810504569367938, + "nauc_recall_at_3_max": 0.5627605752906818, + "nauc_recall_at_3_std": 0.7280169253981897, + "nauc_recall_at_5_diff1": 0.07493463474156249, + "nauc_recall_at_5_max": 0.3601476117123801, + "nauc_recall_at_5_std": 0.5109657682867574, + "ndcg_at_1": 0.0018, + "ndcg_at_10": 0.01526, + "ndcg_at_100": 0.03553, + "ndcg_at_1000": 0.10362, + "ndcg_at_20": 0.01972, + "ndcg_at_3": 0.00581, + "ndcg_at_5": 0.01095, + "precision_at_1": 0.0018, + "precision_at_10": 0.00349, + "precision_at_100": 0.00142, + "precision_at_1000": 0.00073, + "precision_at_20": 0.00265, + "precision_at_3": 0.00301, + "precision_at_5": 0.00433, + "recall_at_1": 0.0018, + "recall_at_10": 0.03488, + "recall_at_100": 0.14191, + "recall_at_1000": 0.73121, + "recall_at_20": 0.05292, + "recall_at_3": 0.00902, + "recall_at_5": 0.02165 + } + ] + }, + "task_name": "VidoreTatdqaRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreArxivQARetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..6c91af01bf --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", + "evaluation_time": 11.5097815990448, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.118, + "cv_recall_at_10": 0.276, + "cv_recall_at_100": 0.65, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.366, + "cv_recall_at_3": 0.178, + "cv_recall_at_5": 0.216, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.16803, + "map_at_1": 0.118, + "map_at_10": 0.16063, + "map_at_100": 0.17306, + "map_at_1000": 0.17487, + "map_at_20": 0.16669, + "map_at_3": 0.14367, + "map_at_5": 0.15227, + "mrr_at_1": 0.118, + "mrr_at_10": 0.16062539682539684, + "mrr_at_100": 0.17306236569997144, + "mrr_at_1000": 0.17487418229364743, + "mrr_at_20": 0.16669178075966007, + "mrr_at_3": 0.1436666666666667, + "mrr_at_5": 0.15226666666666672, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.18296688095044317, + "nauc_cv_recall_at_100_max": 0.13051346732850505, + "nauc_cv_recall_at_100_std": 0.0815916673447794, + "nauc_cv_recall_at_10_diff1": 0.21002008066013042, + "nauc_cv_recall_at_10_max": 0.11474944173514898, + "nauc_cv_recall_at_10_std": -0.08737055848984453, + "nauc_cv_recall_at_1_diff1": 0.485761413165958, + "nauc_cv_recall_at_1_max": 0.27947811827209385, + "nauc_cv_recall_at_1_std": -0.14702257939228888, + "nauc_cv_recall_at_20_diff1": 0.15293554287329153, + "nauc_cv_recall_at_20_max": 0.06147670746099138, + "nauc_cv_recall_at_20_std": -0.0882638469126888, + "nauc_cv_recall_at_3_diff1": 0.32625290643148597, + "nauc_cv_recall_at_3_max": 0.1682279703995215, + "nauc_cv_recall_at_3_std": -0.11057711023736298, + "nauc_cv_recall_at_5_diff1": 0.25332857375468115, + "nauc_cv_recall_at_5_max": 0.14352947807462518, + "nauc_cv_recall_at_5_std": -0.11418604942446668, + "nauc_map_at_1000_diff1": 0.36779035073667693, + "nauc_map_at_1000_max": 0.20151349002499713, + "nauc_map_at_1000_std": -0.11756000190443665, + "nauc_map_at_100_diff1": 0.3669486782669981, + "nauc_map_at_100_max": 0.20130857725652113, + "nauc_map_at_100_std": -0.11682548553478848, + "nauc_map_at_10_diff1": 0.37227377992149197, + "nauc_map_at_10_max": 0.20513150531011887, + "nauc_map_at_10_std": -0.12196059276209226, + "nauc_map_at_1_diff1": 0.485761413165958, + "nauc_map_at_1_max": 0.27947811827209385, + "nauc_map_at_1_std": -0.14702257939228888, + "nauc_map_at_20_diff1": 0.36526766556678214, + "nauc_map_at_20_max": 0.1986343132176448, + "nauc_map_at_20_std": -0.12266526320796677, + "nauc_map_at_3_diff1": 0.4086126400407416, + "nauc_map_at_3_max": 0.22074868328572508, + "nauc_map_at_3_std": -0.12634281959678811, + "nauc_map_at_5_diff1": 0.384865760968298, + "nauc_map_at_5_max": 0.2118497650368132, + "nauc_map_at_5_std": -0.12708878494906578, + "nauc_mrr_at_1000_diff1": 0.36779035073667693, + "nauc_mrr_at_1000_max": 0.20151349002499713, + "nauc_mrr_at_1000_std": -0.11756000190443665, + "nauc_mrr_at_100_diff1": 0.3669486782669981, + "nauc_mrr_at_100_max": 0.20130857725652113, + "nauc_mrr_at_100_std": -0.11682548553478848, + "nauc_mrr_at_10_diff1": 0.37227377992149197, + "nauc_mrr_at_10_max": 0.20513150531011887, + "nauc_mrr_at_10_std": -0.12196059276209226, + "nauc_mrr_at_1_diff1": 0.485761413165958, + "nauc_mrr_at_1_max": 0.27947811827209385, + "nauc_mrr_at_1_std": -0.14702257939228888, + "nauc_mrr_at_20_diff1": 0.36526766556678214, + "nauc_mrr_at_20_max": 0.1986343132176448, + "nauc_mrr_at_20_std": -0.12266526320796677, + "nauc_mrr_at_3_diff1": 0.4086126400407416, + "nauc_mrr_at_3_max": 0.22074868328572508, + "nauc_mrr_at_3_std": -0.12634281959678811, + "nauc_mrr_at_5_diff1": 0.384865760968298, + "nauc_mrr_at_5_max": 0.2118497650368132, + "nauc_mrr_at_5_std": -0.12708878494906578, + "nauc_ndcg_at_1000_diff1": 0.3339421857835854, + "nauc_ndcg_at_1000_max": 0.18217616137925813, + "nauc_ndcg_at_1000_std": -0.10173651860955586, + "nauc_ndcg_at_100_diff1": 0.3131503412000536, + "nauc_ndcg_at_100_max": 0.17582437121383968, + "nauc_ndcg_at_100_std": -0.07731985140025688, + "nauc_ndcg_at_10_diff1": 0.32246312193822635, + "nauc_ndcg_at_10_max": 0.1773668049229422, + "nauc_ndcg_at_10_std": -0.11172669555093219, + "nauc_ndcg_at_1_diff1": 0.485761413165958, + "nauc_ndcg_at_1_max": 0.27947811827209385, + "nauc_ndcg_at_1_std": -0.14702257939228888, + "nauc_ndcg_at_20_diff1": 0.3005922257636208, + "nauc_ndcg_at_20_max": 0.1575132660989431, + "nauc_ndcg_at_20_std": -0.11244207879437416, + "nauc_ndcg_at_3_diff1": 0.38501728903920196, + "nauc_ndcg_at_3_max": 0.2055141399629287, + "nauc_ndcg_at_3_std": -0.12170959892325568, + "nauc_ndcg_at_5_diff1": 0.34602861150963293, + "nauc_ndcg_at_5_max": 0.19130504508028406, + "nauc_ndcg_at_5_std": -0.12304138641095921, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.18296688095044372, + "nauc_precision_at_100_max": 0.13051346732850508, + "nauc_precision_at_100_std": 0.0815916673447802, + "nauc_precision_at_10_diff1": 0.210020080660131, + "nauc_precision_at_10_max": 0.11474944173514932, + "nauc_precision_at_10_std": -0.08737055848984408, + "nauc_precision_at_1_diff1": 0.485761413165958, + "nauc_precision_at_1_max": 0.27947811827209385, + "nauc_precision_at_1_std": -0.14702257939228888, + "nauc_precision_at_20_diff1": 0.15293554287329167, + "nauc_precision_at_20_max": 0.06147670746099171, + "nauc_precision_at_20_std": -0.0882638469126885, + "nauc_precision_at_3_diff1": 0.32625290643148613, + "nauc_precision_at_3_max": 0.1682279703995217, + "nauc_precision_at_3_std": -0.11057711023736295, + "nauc_precision_at_5_diff1": 0.2533285737546817, + "nauc_precision_at_5_max": 0.14352947807462554, + "nauc_precision_at_5_std": -0.11418604942446653, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.18296688095044317, + "nauc_recall_at_100_max": 0.13051346732850505, + "nauc_recall_at_100_std": 0.0815916673447794, + "nauc_recall_at_10_diff1": 0.21002008066013042, + "nauc_recall_at_10_max": 0.11474944173514898, + "nauc_recall_at_10_std": -0.08737055848984453, + "nauc_recall_at_1_diff1": 0.485761413165958, + "nauc_recall_at_1_max": 0.27947811827209385, + "nauc_recall_at_1_std": -0.14702257939228888, + "nauc_recall_at_20_diff1": 0.15293554287329153, + "nauc_recall_at_20_max": 0.06147670746099138, + "nauc_recall_at_20_std": -0.0882638469126888, + "nauc_recall_at_3_diff1": 0.32625290643148597, + "nauc_recall_at_3_max": 0.1682279703995215, + "nauc_recall_at_3_std": -0.11057711023736298, + "nauc_recall_at_5_diff1": 0.25332857375468115, + "nauc_recall_at_5_max": 0.14352947807462518, + "nauc_recall_at_5_std": -0.11418604942446668, + "ndcg_at_1": 0.118, + "ndcg_at_10": 0.18779, + "ndcg_at_100": 0.26119, + "ndcg_at_1000": 0.3067, + "ndcg_at_20": 0.21032, + "ndcg_at_3": 0.15245, + "ndcg_at_5": 0.16803, + "precision_at_1": 0.118, + "precision_at_10": 0.0276, + "precision_at_100": 0.0065, + "precision_at_1000": 0.001, + "precision_at_20": 0.0183, + "precision_at_3": 0.05933, + "precision_at_5": 0.0432, + "recall_at_1": 0.118, + "recall_at_10": 0.276, + "recall_at_100": 0.65, + "recall_at_1000": 1.0, + "recall_at_20": 0.366, + "recall_at_3": 0.178, + "recall_at_5": 0.216 + } + ] + }, + "task_name": "VidoreArxivQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreDocVQARetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..e07536809f --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b1d89eda849e636676df6ead8002602fb1858600", + "evaluation_time": 16.02955412864685, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.058, + "cv_recall_at_10": 0.172, + "cv_recall_at_100": 0.54, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.262, + "cv_recall_at_3": 0.1, + "cv_recall_at_5": 0.13, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.0948, + "map_at_1": 0.058, + "map_at_10": 0.08928, + "map_at_100": 0.10155, + "map_at_1000": 0.10376, + "map_at_20": 0.09538, + "map_at_3": 0.07667, + "map_at_5": 0.08327, + "mrr_at_1": 0.058, + "mrr_at_10": 0.08927777777777782, + "mrr_at_100": 0.10155225471194389, + "mrr_at_1000": 0.10375791922323688, + "mrr_at_20": 0.09537714343619923, + "mrr_at_3": 0.07666666666666667, + "mrr_at_5": 0.0832666666666667, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.032680392320302315, + "nauc_cv_recall_at_100_max": 0.008841099163679788, + "nauc_cv_recall_at_100_std": 0.25023478091745127, + "nauc_cv_recall_at_10_diff1": 0.05107536651136432, + "nauc_cv_recall_at_10_max": 0.09401370828820849, + "nauc_cv_recall_at_10_std": 0.14515449080905074, + "nauc_cv_recall_at_1_diff1": 0.14420399137699735, + "nauc_cv_recall_at_1_max": 0.17870525429531003, + "nauc_cv_recall_at_1_std": 0.07417447702228848, + "nauc_cv_recall_at_20_diff1": 0.06076049285975611, + "nauc_cv_recall_at_20_max": 0.07384829898852548, + "nauc_cv_recall_at_20_std": 0.17957633881371404, + "nauc_cv_recall_at_3_diff1": 0.10039173598068159, + "nauc_cv_recall_at_3_max": 0.2013791253018513, + "nauc_cv_recall_at_3_std": 0.1424953045344781, + "nauc_cv_recall_at_5_diff1": 0.027530164983994042, + "nauc_cv_recall_at_5_max": 0.12327013050972653, + "nauc_cv_recall_at_5_std": 0.10974009984106002, + "nauc_map_at_1000_diff1": 0.10296205110535936, + "nauc_map_at_1000_max": 0.15862975165487792, + "nauc_map_at_1000_std": 0.12497468408354453, + "nauc_map_at_100_diff1": 0.10261504617825842, + "nauc_map_at_100_max": 0.15700943002591616, + "nauc_map_at_100_std": 0.12577033745669342, + "nauc_map_at_10_diff1": 0.10233479631218023, + "nauc_map_at_10_max": 0.1655998866614918, + "nauc_map_at_10_std": 0.11614651176369198, + "nauc_map_at_1_diff1": 0.14420399137699735, + "nauc_map_at_1_max": 0.17870525429531003, + "nauc_map_at_1_std": 0.07417447702228848, + "nauc_map_at_20_diff1": 0.10286768495886522, + "nauc_map_at_20_max": 0.16077790911145834, + "nauc_map_at_20_std": 0.12225269367497407, + "nauc_map_at_3_diff1": 0.12587697297045058, + "nauc_map_at_3_max": 0.19920439565567363, + "nauc_map_at_3_std": 0.11490533241562752, + "nauc_map_at_5_diff1": 0.1008074992224546, + "nauc_map_at_5_max": 0.17466259834098666, + "nauc_map_at_5_std": 0.10658693949141812, + "nauc_mrr_at_1000_diff1": 0.10296205110535936, + "nauc_mrr_at_1000_max": 0.15862975165487792, + "nauc_mrr_at_1000_std": 0.12497468408354453, + "nauc_mrr_at_100_diff1": 0.10261504617825842, + "nauc_mrr_at_100_max": 0.15700943002591616, + "nauc_mrr_at_100_std": 0.12577033745669342, + "nauc_mrr_at_10_diff1": 0.10233479631218023, + "nauc_mrr_at_10_max": 0.1655998866614918, + "nauc_mrr_at_10_std": 0.11614651176369198, + "nauc_mrr_at_1_diff1": 0.14420399137699735, + "nauc_mrr_at_1_max": 0.17870525429531003, + "nauc_mrr_at_1_std": 0.07417447702228848, + "nauc_mrr_at_20_diff1": 0.10286768495886522, + "nauc_mrr_at_20_max": 0.16077790911145834, + "nauc_mrr_at_20_std": 0.12225269367497407, + "nauc_mrr_at_3_diff1": 0.12587697297045058, + "nauc_mrr_at_3_max": 0.19920439565567363, + "nauc_mrr_at_3_std": 0.11490533241562752, + "nauc_mrr_at_5_diff1": 0.1008074992224546, + "nauc_mrr_at_5_max": 0.17466259834098666, + "nauc_mrr_at_5_std": 0.10658693949141812, + "nauc_ndcg_at_1000_diff1": 0.08993939510748966, + "nauc_ndcg_at_1000_max": 0.13722131092451595, + "nauc_ndcg_at_1000_std": 0.14394044997396738, + "nauc_ndcg_at_100_diff1": 0.07881330019181253, + "nauc_ndcg_at_100_max": 0.10857797405282689, + "nauc_ndcg_at_100_std": 0.16533729610286177, + "nauc_ndcg_at_10_diff1": 0.08389649942218956, + "nauc_ndcg_at_10_max": 0.14191890938892987, + "nauc_ndcg_at_10_std": 0.12645751380780731, + "nauc_ndcg_at_1_diff1": 0.14420399137699735, + "nauc_ndcg_at_1_max": 0.17870525429531003, + "nauc_ndcg_at_1_std": 0.07417447702228848, + "nauc_ndcg_at_20_diff1": 0.08576639195224166, + "nauc_ndcg_at_20_max": 0.12935318913387056, + "nauc_ndcg_at_20_std": 0.14299439642394743, + "nauc_ndcg_at_3_diff1": 0.11823031088822708, + "nauc_ndcg_at_3_max": 0.2001720221732457, + "nauc_ndcg_at_3_std": 0.12353115467979815, + "nauc_ndcg_at_5_diff1": 0.07772393650891742, + "nauc_ndcg_at_5_max": 0.1591303206130916, + "nauc_ndcg_at_5_std": 0.10837344904056297, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.03268039232030201, + "nauc_precision_at_100_max": 0.008841099163679927, + "nauc_precision_at_100_std": 0.25023478091745144, + "nauc_precision_at_10_diff1": 0.051075366511364524, + "nauc_precision_at_10_max": 0.09401370828820864, + "nauc_precision_at_10_std": 0.1451544908090511, + "nauc_precision_at_1_diff1": 0.14420399137699735, + "nauc_precision_at_1_max": 0.17870525429531003, + "nauc_precision_at_1_std": 0.07417447702228848, + "nauc_precision_at_20_diff1": 0.060760492859756446, + "nauc_precision_at_20_max": 0.07384829898852595, + "nauc_precision_at_20_std": 0.17957633881371443, + "nauc_precision_at_3_diff1": 0.1003917359806816, + "nauc_precision_at_3_max": 0.20137912530185134, + "nauc_precision_at_3_std": 0.14249530453447826, + "nauc_precision_at_5_diff1": 0.027530164983994375, + "nauc_precision_at_5_max": 0.12327013050972686, + "nauc_precision_at_5_std": 0.10974009984106038, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.032680392320302315, + "nauc_recall_at_100_max": 0.008841099163679788, + "nauc_recall_at_100_std": 0.25023478091745127, + "nauc_recall_at_10_diff1": 0.05107536651136432, + "nauc_recall_at_10_max": 0.09401370828820849, + "nauc_recall_at_10_std": 0.14515449080905074, + "nauc_recall_at_1_diff1": 0.14420399137699735, + "nauc_recall_at_1_max": 0.17870525429531003, + "nauc_recall_at_1_std": 0.07417447702228848, + "nauc_recall_at_20_diff1": 0.06076049285975611, + "nauc_recall_at_20_max": 0.07384829898852548, + "nauc_recall_at_20_std": 0.17957633881371404, + "nauc_recall_at_3_diff1": 0.10039173598068159, + "nauc_recall_at_3_max": 0.2013791253018513, + "nauc_recall_at_3_std": 0.1424953045344781, + "nauc_recall_at_5_diff1": 0.027530164983994042, + "nauc_recall_at_5_max": 0.12327013050972653, + "nauc_recall_at_5_std": 0.10974009984106002, + "ndcg_at_1": 0.058, + "ndcg_at_10": 0.10879, + "ndcg_at_100": 0.18103, + "ndcg_at_1000": 0.23998, + "ndcg_at_20": 0.13135, + "ndcg_at_3": 0.08267, + "ndcg_at_5": 0.0948, + "precision_at_1": 0.058, + "precision_at_10": 0.0172, + "precision_at_100": 0.0054, + "precision_at_1000": 0.001, + "precision_at_20": 0.0131, + "precision_at_3": 0.03333, + "precision_at_5": 0.026, + "recall_at_1": 0.058, + "recall_at_10": 0.172, + "recall_at_100": 0.54, + "recall_at_1000": 1.0, + "recall_at_20": 0.262, + "recall_at_3": 0.1, + "recall_at_5": 0.13 + } + ] + }, + "task_name": "VidoreDocVQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreInfoVQARetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..79c911b0de --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", + "evaluation_time": 15.756592273712158, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.274, + "cv_recall_at_10": 0.548, + "cv_recall_at_100": 0.834, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.628, + "cv_recall_at_3": 0.392, + "cv_recall_at_5": 0.458, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.36997, + "map_at_1": 0.274, + "map_at_10": 0.35329, + "map_at_100": 0.36413, + "map_at_1000": 0.36503, + "map_at_20": 0.35891, + "map_at_3": 0.32567, + "map_at_5": 0.34087, + "mrr_at_1": 0.274, + "mrr_at_10": 0.35328888888888876, + "mrr_at_100": 0.3641305092820594, + "mrr_at_1000": 0.3650337866097008, + "mrr_at_20": 0.3589137667801754, + "mrr_at_3": 0.3256666666666667, + "mrr_at_5": 0.34086666666666665, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4108195982105783, + "nauc_cv_recall_at_100_max": 0.3513699524165519, + "nauc_cv_recall_at_100_std": 0.3340927699263115, + "nauc_cv_recall_at_10_diff1": 0.3707691359200985, + "nauc_cv_recall_at_10_max": 0.2438081840142132, + "nauc_cv_recall_at_10_std": 0.15858559250312437, + "nauc_cv_recall_at_1_diff1": 0.4619750027514469, + "nauc_cv_recall_at_1_max": 0.2773571434618564, + "nauc_cv_recall_at_1_std": 0.09954311870165566, + "nauc_cv_recall_at_20_diff1": 0.4034132420804166, + "nauc_cv_recall_at_20_max": 0.2844867960110151, + "nauc_cv_recall_at_20_std": 0.15804197550908236, + "nauc_cv_recall_at_3_diff1": 0.4065778999927638, + "nauc_cv_recall_at_3_max": 0.2496459326186513, + "nauc_cv_recall_at_3_std": 0.13659247619736797, + "nauc_cv_recall_at_5_diff1": 0.39779343598416905, + "nauc_cv_recall_at_5_max": 0.25945904777912304, + "nauc_cv_recall_at_5_std": 0.17263041234541332, + "nauc_map_at_1000_diff1": 0.4289400319756985, + "nauc_map_at_1000_max": 0.2689506737673011, + "nauc_map_at_1000_std": 0.12803419134610242, + "nauc_map_at_100_diff1": 0.42885442258511014, + "nauc_map_at_100_max": 0.2692582970619958, + "nauc_map_at_100_std": 0.12843203024529296, + "nauc_map_at_10_diff1": 0.4262212910098287, + "nauc_map_at_10_max": 0.26616493787234696, + "nauc_map_at_10_std": 0.1265953804223453, + "nauc_map_at_1_diff1": 0.4619750027514469, + "nauc_map_at_1_max": 0.2773571434618564, + "nauc_map_at_1_std": 0.09954311870165566, + "nauc_map_at_20_diff1": 0.4284448343118344, + "nauc_map_at_20_max": 0.2690760269120174, + "nauc_map_at_20_std": 0.12663495864569618, + "nauc_map_at_3_diff1": 0.43187220501498585, + "nauc_map_at_3_max": 0.2654933953846897, + "nauc_map_at_3_std": 0.11997884284630515, + "nauc_map_at_5_diff1": 0.4296497686024672, + "nauc_map_at_5_max": 0.2687269018079568, + "nauc_map_at_5_std": 0.12984081889984647, + "nauc_mrr_at_1000_diff1": 0.4289400319756985, + "nauc_mrr_at_1000_max": 0.2689506737673011, + "nauc_mrr_at_1000_std": 0.12803419134610242, + "nauc_mrr_at_100_diff1": 0.42885442258511014, + "nauc_mrr_at_100_max": 0.2692582970619958, + "nauc_mrr_at_100_std": 0.12843203024529296, + "nauc_mrr_at_10_diff1": 0.4262212910098287, + "nauc_mrr_at_10_max": 0.26616493787234696, + "nauc_mrr_at_10_std": 0.1265953804223453, + "nauc_mrr_at_1_diff1": 0.4619750027514469, + "nauc_mrr_at_1_max": 0.2773571434618564, + "nauc_mrr_at_1_std": 0.09954311870165566, + "nauc_mrr_at_20_diff1": 0.4284448343118344, + "nauc_mrr_at_20_max": 0.2690760269120174, + "nauc_mrr_at_20_std": 0.12663495864569618, + "nauc_mrr_at_3_diff1": 0.43187220501498585, + "nauc_mrr_at_3_max": 0.2654933953846897, + "nauc_mrr_at_3_std": 0.11997884284630515, + "nauc_mrr_at_5_diff1": 0.4296497686024672, + "nauc_mrr_at_5_max": 0.2687269018079568, + "nauc_mrr_at_5_std": 0.12984081889984647, + "nauc_ndcg_at_1000_diff1": 0.42490331218837485, + "nauc_ndcg_at_1000_max": 0.26923686243472, + "nauc_ndcg_at_1000_std": 0.13661973225278048, + "nauc_ndcg_at_100_diff1": 0.4237501775481191, + "nauc_ndcg_at_100_max": 0.2754123277695727, + "nauc_ndcg_at_100_std": 0.14932637461067172, + "nauc_ndcg_at_10_diff1": 0.4129110395057173, + "nauc_ndcg_at_10_max": 0.2608695707815544, + "nauc_ndcg_at_10_std": 0.1347290633153979, + "nauc_ndcg_at_1_diff1": 0.4619750027514469, + "nauc_ndcg_at_1_max": 0.2773571434618564, + "nauc_ndcg_at_1_std": 0.09954311870165566, + "nauc_ndcg_at_20_diff1": 0.4212506902996852, + "nauc_ndcg_at_20_max": 0.2709472172227333, + "nauc_ndcg_at_20_std": 0.13434839262447948, + "nauc_ndcg_at_3_diff1": 0.4249611000998995, + "nauc_ndcg_at_3_max": 0.2613207228374443, + "nauc_ndcg_at_3_std": 0.12453307166402308, + "nauc_ndcg_at_5_diff1": 0.42116772431902333, + "nauc_ndcg_at_5_max": 0.26628491565434564, + "nauc_ndcg_at_5_std": 0.14088721346867775, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.4108195982105766, + "nauc_precision_at_100_max": 0.3513699524165505, + "nauc_precision_at_100_std": 0.3340927699263106, + "nauc_precision_at_10_diff1": 0.37076913592009925, + "nauc_precision_at_10_max": 0.24380818401421336, + "nauc_precision_at_10_std": 0.1585855925031248, + "nauc_precision_at_1_diff1": 0.4619750027514469, + "nauc_precision_at_1_max": 0.2773571434618564, + "nauc_precision_at_1_std": 0.09954311870165566, + "nauc_precision_at_20_diff1": 0.40341324208041696, + "nauc_precision_at_20_max": 0.2844867960110156, + "nauc_precision_at_20_std": 0.1580419755090828, + "nauc_precision_at_3_diff1": 0.4065778999927632, + "nauc_precision_at_3_max": 0.2496459326186512, + "nauc_precision_at_3_std": 0.136592476197368, + "nauc_precision_at_5_diff1": 0.397793435984169, + "nauc_precision_at_5_max": 0.25945904777912304, + "nauc_precision_at_5_std": 0.1726304123454132, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4108195982105783, + "nauc_recall_at_100_max": 0.3513699524165519, + "nauc_recall_at_100_std": 0.3340927699263115, + "nauc_recall_at_10_diff1": 0.3707691359200985, + "nauc_recall_at_10_max": 0.2438081840142132, + "nauc_recall_at_10_std": 0.15858559250312437, + "nauc_recall_at_1_diff1": 0.4619750027514469, + "nauc_recall_at_1_max": 0.2773571434618564, + "nauc_recall_at_1_std": 0.09954311870165566, + "nauc_recall_at_20_diff1": 0.4034132420804166, + "nauc_recall_at_20_max": 0.2844867960110151, + "nauc_recall_at_20_std": 0.15804197550908236, + "nauc_recall_at_3_diff1": 0.4065778999927638, + "nauc_recall_at_3_max": 0.2496459326186513, + "nauc_recall_at_3_std": 0.13659247619736797, + "nauc_recall_at_5_diff1": 0.39779343598416905, + "nauc_recall_at_5_max": 0.25945904777912304, + "nauc_recall_at_5_std": 0.17263041234541332, + "ndcg_at_1": 0.274, + "ndcg_at_10": 0.39949, + "ndcg_at_100": 0.45791, + "ndcg_at_1000": 0.47973, + "ndcg_at_20": 0.41982, + "ndcg_at_3": 0.34269, + "ndcg_at_5": 0.36997, + "precision_at_1": 0.274, + "precision_at_10": 0.0548, + "precision_at_100": 0.00834, + "precision_at_1000": 0.001, + "precision_at_20": 0.0314, + "precision_at_3": 0.13067, + "precision_at_5": 0.0916, + "recall_at_1": 0.274, + "recall_at_10": 0.548, + "recall_at_100": 0.834, + "recall_at_1000": 1.0, + "recall_at_20": 0.628, + "recall_at_3": 0.392, + "recall_at_5": 0.458 + } + ] + }, + "task_name": "VidoreInfoVQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreShiftProjectRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..65d569d994 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", + "evaluation_time": 30.601240396499634, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.01, + "cv_recall_at_10": 0.03, + "cv_recall_at_100": 0.19, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.05, + "cv_recall_at_3": 0.01, + "cv_recall_at_5": 0.01, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.01, + "map_at_1": 0.01, + "map_at_10": 0.01243, + "map_at_100": 0.01593, + "map_at_1000": 0.01858, + "map_at_20": 0.01345, + "map_at_3": 0.01, + "map_at_5": 0.01, + "mrr_at_1": 0.01, + "mrr_at_10": 0.01242857142857143, + "mrr_at_100": 0.01592918539143402, + "mrr_at_1000": 0.018579131316456693, + "mrr_at_20": 0.013454887218045114, + "mrr_at_3": 0.01, + "mrr_at_5": 0.01, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.03801725152590248, + "nauc_cv_recall_at_100_max": -0.08736737679283049, + "nauc_cv_recall_at_100_std": -0.27136683873354306, + "nauc_cv_recall_at_10_diff1": 0.04588140595653343, + "nauc_cv_recall_at_10_max": -0.35032644665056795, + "nauc_cv_recall_at_10_std": -0.1259279134245595, + "nauc_cv_recall_at_1_diff1": 0.6619264824255433, + "nauc_cv_recall_at_1_max": -0.5747249798765763, + "nauc_cv_recall_at_1_std": -0.5747249798765763, + "nauc_cv_recall_at_20_diff1": 0.05999463375368926, + "nauc_cv_recall_at_20_max": -0.2735175744566677, + "nauc_cv_recall_at_20_std": -0.20461497182720695, + "nauc_cv_recall_at_3_diff1": 0.6619264824255433, + "nauc_cv_recall_at_3_max": -0.5747249798765763, + "nauc_cv_recall_at_3_std": -0.5747249798765763, + "nauc_cv_recall_at_5_diff1": 0.6619264824255433, + "nauc_cv_recall_at_5_max": -0.5747249798765763, + "nauc_cv_recall_at_5_std": -0.5747249798765763, + "nauc_map_at_1000_diff1": 0.37055532373093525, + "nauc_map_at_1000_max": -0.4409673650509871, + "nauc_map_at_1000_std": -0.39942667927544195, + "nauc_map_at_100_diff1": 0.38208823611071047, + "nauc_map_at_100_max": -0.44308890084107116, + "nauc_map_at_100_std": -0.4127580396465241, + "nauc_map_at_10_diff1": 0.47941859496868144, + "nauc_map_at_10_max": -0.5205598166840916, + "nauc_map_at_10_std": -0.443181012123399, + "nauc_map_at_1_diff1": 0.6619264824255433, + "nauc_map_at_1_max": -0.5747249798765763, + "nauc_map_at_1_std": -0.5747249798765763, + "nauc_map_at_20_diff1": 0.45017633719278727, + "nauc_map_at_20_max": -0.4924253686557989, + "nauc_map_at_20_std": -0.4344798043441707, + "nauc_map_at_3_diff1": 0.6619264824255433, + "nauc_map_at_3_max": -0.5747249798765763, + "nauc_map_at_3_std": -0.5747249798765763, + "nauc_map_at_5_diff1": 0.6619264824255433, + "nauc_map_at_5_max": -0.5747249798765763, + "nauc_map_at_5_std": -0.5747249798765763, + "nauc_mrr_at_1000_diff1": 0.37055532373093525, + "nauc_mrr_at_1000_max": -0.4409673650509871, + "nauc_mrr_at_1000_std": -0.39942667927544195, + "nauc_mrr_at_100_diff1": 0.38208823611071047, + "nauc_mrr_at_100_max": -0.44308890084107116, + "nauc_mrr_at_100_std": -0.4127580396465241, + "nauc_mrr_at_10_diff1": 0.47941859496868144, + "nauc_mrr_at_10_max": -0.5205598166840916, + "nauc_mrr_at_10_std": -0.443181012123399, + "nauc_mrr_at_1_diff1": 0.6619264824255433, + "nauc_mrr_at_1_max": -0.5747249798765763, + "nauc_mrr_at_1_std": -0.5747249798765763, + "nauc_mrr_at_20_diff1": 0.45017633719278727, + "nauc_mrr_at_20_max": -0.4924253686557989, + "nauc_mrr_at_20_std": -0.4344798043441707, + "nauc_mrr_at_3_diff1": 0.6619264824255433, + "nauc_mrr_at_3_max": -0.5747249798765763, + "nauc_mrr_at_3_std": -0.5747249798765763, + "nauc_mrr_at_5_diff1": 0.6619264824255433, + "nauc_mrr_at_5_max": -0.5747249798765763, + "nauc_mrr_at_5_std": -0.5747249798765763, + "nauc_ndcg_at_1000_diff1": 0.1597681443227891, + "nauc_ndcg_at_1000_max": -0.3000341630497847, + "nauc_ndcg_at_1000_std": -0.2766624383259684, + "nauc_ndcg_at_100_diff1": 0.13848351210393903, + "nauc_ndcg_at_100_max": -0.21869254918959663, + "nauc_ndcg_at_100_std": -0.30606324798461026, + "nauc_ndcg_at_10_diff1": 0.3058903797405613, + "nauc_ndcg_at_10_max": -0.454780829335424, + "nauc_ndcg_at_10_std": -0.3164679670512058, + "nauc_ndcg_at_1_diff1": 0.6619264824255433, + "nauc_ndcg_at_1_max": -0.5747249798765763, + "nauc_ndcg_at_1_std": -0.5747249798765763, + "nauc_ndcg_at_20_diff1": 0.2573632473234855, + "nauc_ndcg_at_20_max": -0.38893757347888275, + "nauc_ndcg_at_20_std": -0.31827945516426087, + "nauc_ndcg_at_3_diff1": 0.6619264824255433, + "nauc_ndcg_at_3_max": -0.5747249798765763, + "nauc_ndcg_at_3_std": -0.5747249798765763, + "nauc_ndcg_at_5_diff1": 0.6619264824255433, + "nauc_ndcg_at_5_max": -0.5747249798765763, + "nauc_ndcg_at_5_std": -0.5747249798765763, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.03801725152590239, + "nauc_precision_at_100_max": -0.08736737679283053, + "nauc_precision_at_100_std": -0.2713668387335432, + "nauc_precision_at_10_diff1": 0.04588140595653325, + "nauc_precision_at_10_max": -0.3503264466505679, + "nauc_precision_at_10_std": -0.12592791342455947, + "nauc_precision_at_1_diff1": 0.6619264824255433, + "nauc_precision_at_1_max": -0.5747249798765763, + "nauc_precision_at_1_std": -0.5747249798765763, + "nauc_precision_at_20_diff1": 0.0599946337536892, + "nauc_precision_at_20_max": -0.2735175744566675, + "nauc_precision_at_20_std": -0.20461497182720684, + "nauc_precision_at_3_diff1": 0.6619264824255435, + "nauc_precision_at_3_max": -0.5747249798765764, + "nauc_precision_at_3_std": -0.5747249798765764, + "nauc_precision_at_5_diff1": 0.6619264824255435, + "nauc_precision_at_5_max": -0.5747249798765764, + "nauc_precision_at_5_std": -0.5747249798765764, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.03801725152590248, + "nauc_recall_at_100_max": -0.08736737679283049, + "nauc_recall_at_100_std": -0.27136683873354306, + "nauc_recall_at_10_diff1": 0.04588140595653343, + "nauc_recall_at_10_max": -0.35032644665056795, + "nauc_recall_at_10_std": -0.1259279134245595, + "nauc_recall_at_1_diff1": 0.6619264824255433, + "nauc_recall_at_1_max": -0.5747249798765763, + "nauc_recall_at_1_std": -0.5747249798765763, + "nauc_recall_at_20_diff1": 0.05999463375368926, + "nauc_recall_at_20_max": -0.2735175744566677, + "nauc_recall_at_20_std": -0.20461497182720695, + "nauc_recall_at_3_diff1": 0.6619264824255433, + "nauc_recall_at_3_max": -0.5747249798765763, + "nauc_recall_at_3_std": -0.5747249798765763, + "nauc_recall_at_5_diff1": 0.6619264824255433, + "nauc_recall_at_5_max": -0.5747249798765763, + "nauc_recall_at_5_std": -0.5747249798765763, + "ndcg_at_1": 0.01, + "ndcg_at_10": 0.01622, + "ndcg_at_100": 0.04451, + "ndcg_at_1000": 0.14068, + "ndcg_at_20": 0.02081, + "ndcg_at_3": 0.01, + "ndcg_at_5": 0.01, + "precision_at_1": 0.01, + "precision_at_10": 0.003, + "precision_at_100": 0.0019, + "precision_at_1000": 0.001, + "precision_at_20": 0.0025, + "precision_at_3": 0.00333, + "precision_at_5": 0.002, + "recall_at_1": 0.01, + "recall_at_10": 0.03, + "recall_at_100": 0.19, + "recall_at_1000": 1.0, + "recall_at_20": 0.05, + "recall_at_3": 0.01, + "recall_at_5": 0.01 + } + ] + }, + "task_name": "VidoreShiftProjectRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAAIRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..a33ab39de5 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", + "evaluation_time": 29.120087385177612, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.08, + "cv_recall_at_10": 0.16, + "cv_recall_at_100": 0.48, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.23, + "cv_recall_at_3": 0.11, + "cv_recall_at_5": 0.12, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.10323, + "map_at_1": 0.08, + "map_at_10": 0.10186, + "map_at_100": 0.11176, + "map_at_1000": 0.11365, + "map_at_20": 0.10621, + "map_at_3": 0.095, + "map_at_5": 0.0975, + "mrr_at_1": 0.08, + "mrr_at_10": 0.10019444444444443, + "mrr_at_100": 0.11011084461491963, + "mrr_at_1000": 0.11200037839646133, + "mrr_at_20": 0.1045761630010856, + "mrr_at_3": 0.09333333333333332, + "mrr_at_5": 0.09583333333333333, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.04469672912035571, + "nauc_cv_recall_at_100_max": 0.06390917751667213, + "nauc_cv_recall_at_100_std": 0.16791044776119407, + "nauc_cv_recall_at_10_diff1": 0.07295667537256094, + "nauc_cv_recall_at_10_max": 0.32518436011676144, + "nauc_cv_recall_at_10_std": 0.1959402365954832, + "nauc_cv_recall_at_1_diff1": 0.23302924604239317, + "nauc_cv_recall_at_1_max": 0.4590823718808693, + "nauc_cv_recall_at_1_std": 0.150757982291387, + "nauc_cv_recall_at_20_diff1": -0.024984365705221116, + "nauc_cv_recall_at_20_max": 0.20837083021918498, + "nauc_cv_recall_at_20_std": 0.10311008068821408, + "nauc_cv_recall_at_3_diff1": 0.16787880313058365, + "nauc_cv_recall_at_3_max": 0.48519012507235065, + "nauc_cv_recall_at_3_std": 0.11757304275612139, + "nauc_cv_recall_at_5_diff1": 0.12994029001990343, + "nauc_cv_recall_at_5_max": 0.42870344043218656, + "nauc_cv_recall_at_5_std": 0.19900957255236476, + "nauc_map_at_1000_diff1": 0.1675620920313474, + "nauc_map_at_1000_max": 0.4200132379612108, + "nauc_map_at_1000_std": 0.14740733681367998, + "nauc_map_at_100_diff1": 0.16626251906483952, + "nauc_map_at_100_max": 0.4170654384966692, + "nauc_map_at_100_std": 0.14678438936186355, + "nauc_map_at_10_diff1": 0.17286908282947616, + "nauc_map_at_10_max": 0.43939582397261495, + "nauc_map_at_10_std": 0.1561951662475019, + "nauc_map_at_1_diff1": 0.23302924604239317, + "nauc_map_at_1_max": 0.4590823718808693, + "nauc_map_at_1_std": 0.150757982291387, + "nauc_map_at_20_diff1": 0.1595907524800354, + "nauc_map_at_20_max": 0.42461718707559404, + "nauc_map_at_20_std": 0.1458996645672452, + "nauc_map_at_3_diff1": 0.1958075136945926, + "nauc_map_at_3_max": 0.47399824594194356, + "nauc_map_at_3_std": 0.13179877215936048, + "nauc_map_at_5_diff1": 0.18401107555819254, + "nauc_map_at_5_max": 0.45735415651289435, + "nauc_map_at_5_std": 0.1558277355145401, + "nauc_mrr_at_1000_diff1": 0.15244811169195893, + "nauc_mrr_at_1000_max": 0.40811312724788, + "nauc_mrr_at_1000_std": 0.14373520730250974, + "nauc_mrr_at_100_diff1": 0.1512175820307389, + "nauc_mrr_at_100_max": 0.40519414293681005, + "nauc_mrr_at_100_std": 0.1431171513061672, + "nauc_mrr_at_10_diff1": 0.15761340785428135, + "nauc_mrr_at_10_max": 0.4271793359905257, + "nauc_mrr_at_10_std": 0.15239727016735158, + "nauc_mrr_at_1_diff1": 0.23302924604239317, + "nauc_mrr_at_1_max": 0.4590823718808693, + "nauc_mrr_at_1_std": 0.150757982291387, + "nauc_mrr_at_20_diff1": 0.14432706409099771, + "nauc_mrr_at_20_max": 0.41254188560602034, + "nauc_mrr_at_20_std": 0.14202270786213608, + "nauc_mrr_at_3_diff1": 0.18007488166369212, + "nauc_mrr_at_3_max": 0.4615940482656942, + "nauc_mrr_at_3_std": 0.12752192131605505, + "nauc_mrr_at_5_diff1": 0.16841716593017853, + "nauc_mrr_at_5_max": 0.44497302190406995, + "nauc_mrr_at_5_std": 0.15190871517562562, + "nauc_ndcg_at_1000_diff1": 0.1490204225947559, + "nauc_ndcg_at_1000_max": 0.37537786682774904, + "nauc_ndcg_at_1000_std": 0.1497468010938436, + "nauc_ndcg_at_100_diff1": 0.12563396873682928, + "nauc_ndcg_at_100_max": 0.31099215070250275, + "nauc_ndcg_at_100_std": 0.14960003826861942, + "nauc_ndcg_at_10_diff1": 0.14300628393164974, + "nauc_ndcg_at_10_max": 0.40712871839693465, + "nauc_ndcg_at_10_std": 0.16798748927277138, + "nauc_ndcg_at_1_diff1": 0.23302924604239317, + "nauc_ndcg_at_1_max": 0.4590823718808693, + "nauc_ndcg_at_1_std": 0.150757982291387, + "nauc_ndcg_at_20_diff1": 0.10129856953452061, + "nauc_ndcg_at_20_max": 0.3585785212661761, + "nauc_ndcg_at_20_std": 0.134777637721659, + "nauc_ndcg_at_3_diff1": 0.1877518282172568, + "nauc_ndcg_at_3_max": 0.4772264027867892, + "nauc_ndcg_at_3_std": 0.12769553934109137, + "nauc_ndcg_at_5_diff1": 0.16861518269930026, + "nauc_ndcg_at_5_max": 0.44980817963915065, + "nauc_ndcg_at_5_std": 0.16726310730787342, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.044696729120355265, + "nauc_precision_at_100_max": 0.06390917751667215, + "nauc_precision_at_100_std": 0.1679104477611941, + "nauc_precision_at_10_diff1": 0.07295667537256093, + "nauc_precision_at_10_max": 0.3251843601167612, + "nauc_precision_at_10_std": 0.195940236595483, + "nauc_precision_at_1_diff1": 0.23302924604239317, + "nauc_precision_at_1_max": 0.4590823718808693, + "nauc_precision_at_1_std": 0.150757982291387, + "nauc_precision_at_20_diff1": -0.024984365705221136, + "nauc_precision_at_20_max": 0.20837083021918515, + "nauc_precision_at_20_std": 0.10311008068821399, + "nauc_precision_at_3_diff1": 0.16787880313058348, + "nauc_precision_at_3_max": 0.48519012507235065, + "nauc_precision_at_3_std": 0.11757304275612139, + "nauc_precision_at_5_diff1": 0.12994029001990304, + "nauc_precision_at_5_max": 0.4287034404321862, + "nauc_precision_at_5_std": 0.19900957255236448, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.04469672912035571, + "nauc_recall_at_100_max": 0.06390917751667213, + "nauc_recall_at_100_std": 0.16791044776119407, + "nauc_recall_at_10_diff1": 0.07295667537256094, + "nauc_recall_at_10_max": 0.32518436011676144, + "nauc_recall_at_10_std": 0.1959402365954832, + "nauc_recall_at_1_diff1": 0.23302924604239317, + "nauc_recall_at_1_max": 0.4590823718808693, + "nauc_recall_at_1_std": 0.150757982291387, + "nauc_recall_at_20_diff1": -0.024984365705221116, + "nauc_recall_at_20_max": 0.20837083021918498, + "nauc_recall_at_20_std": 0.10311008068821408, + "nauc_recall_at_3_diff1": 0.16787880313058365, + "nauc_recall_at_3_max": 0.48519012507235065, + "nauc_recall_at_3_std": 0.11757304275612139, + "nauc_recall_at_5_diff1": 0.12994029001990343, + "nauc_recall_at_5_max": 0.42870344043218656, + "nauc_recall_at_5_std": 0.19900957255236476, + "ndcg_at_1": 0.08, + "ndcg_at_10": 0.11518, + "ndcg_at_100": 0.17682, + "ndcg_at_1000": 0.23957, + "ndcg_at_20": 0.13223, + "ndcg_at_3": 0.09893, + "ndcg_at_5": 0.10323, + "precision_at_1": 0.08, + "precision_at_10": 0.016, + "precision_at_100": 0.0048, + "precision_at_1000": 0.001, + "precision_at_20": 0.0115, + "precision_at_3": 0.03667, + "precision_at_5": 0.024, + "recall_at_1": 0.08, + "recall_at_10": 0.16, + "recall_at_100": 0.48, + "recall_at_1000": 1.0, + "recall_at_20": 0.23, + "recall_at_3": 0.11, + "recall_at_5": 0.12 + } + ] + }, + "task_name": "VidoreSyntheticDocQAAIRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAEnergyRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..289c1e6c15 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", + "evaluation_time": 27.4805850982666, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.09, + "cv_recall_at_10": 0.22, + "cv_recall_at_100": 0.61, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.3, + "cv_recall_at_3": 0.18, + "cv_recall_at_5": 0.22, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.15421, + "map_at_1": 0.08, + "map_at_10": 0.13233, + "map_at_100": 0.14599, + "map_at_1000": 0.14767, + "map_at_20": 0.13794, + "map_at_3": 0.12333, + "map_at_5": 0.13233, + "mrr_at_1": 0.09, + "mrr_at_10": 0.13733333333333334, + "mrr_at_100": 0.1508502521768428, + "mrr_at_1000": 0.15253465640562683, + "mrr_at_20": 0.14279348326252353, + "mrr_at_3": 0.12833333333333333, + "mrr_at_5": 0.13733333333333334, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.4314655628238636, + "nauc_cv_recall_at_100_max": -0.06355656942577056, + "nauc_cv_recall_at_100_std": 0.27635852733672706, + "nauc_cv_recall_at_10_diff1": -0.2979457376386859, + "nauc_cv_recall_at_10_max": 0.1128445784255835, + "nauc_cv_recall_at_10_std": 0.12398607701152999, + "nauc_cv_recall_at_1_diff1": -0.19631517753331543, + "nauc_cv_recall_at_1_max": 0.236860149658647, + "nauc_cv_recall_at_1_std": -0.01994454878812264, + "nauc_cv_recall_at_20_diff1": -0.3405350628293475, + "nauc_cv_recall_at_20_max": 0.03631941629509504, + "nauc_cv_recall_at_20_std": 0.10158086745034434, + "nauc_cv_recall_at_3_diff1": -0.34073606287057506, + "nauc_cv_recall_at_3_max": 0.12274146581061644, + "nauc_cv_recall_at_3_std": 0.26448093183173693, + "nauc_cv_recall_at_5_diff1": -0.2979457376386859, + "nauc_cv_recall_at_5_max": 0.1128445784255835, + "nauc_cv_recall_at_5_std": 0.12398607701152999, + "nauc_map_at_1000_diff1": -0.2604944977994006, + "nauc_map_at_1000_max": 0.17830635109139642, + "nauc_map_at_1000_std": 0.12090084293431434, + "nauc_map_at_100_diff1": -0.2621590656772411, + "nauc_map_at_100_max": 0.17648762923000524, + "nauc_map_at_100_std": 0.121268509592713, + "nauc_map_at_10_diff1": -0.2616638520052834, + "nauc_map_at_10_max": 0.18996089626586538, + "nauc_map_at_10_std": 0.11845610393114783, + "nauc_map_at_1_diff1": -0.15840488328414284, + "nauc_map_at_1_max": 0.30346122887040494, + "nauc_map_at_1_std": 0.049771934531795, + "nauc_map_at_20_diff1": -0.2646839857229457, + "nauc_map_at_20_max": 0.18237211042993076, + "nauc_map_at_20_std": 0.1192844198516302, + "nauc_map_at_3_diff1": -0.2756486484329484, + "nauc_map_at_3_max": 0.1953966113057568, + "nauc_map_at_3_std": 0.16172914388782036, + "nauc_map_at_5_diff1": -0.2616638520052834, + "nauc_map_at_5_max": 0.18996089626586538, + "nauc_map_at_5_std": 0.11845610393114783, + "nauc_mrr_at_1000_diff1": -0.27452629073850626, + "nauc_mrr_at_1000_max": 0.1574894915596897, + "nauc_mrr_at_1000_std": 0.09317346271772621, + "nauc_mrr_at_100_diff1": -0.27604407977587014, + "nauc_mrr_at_100_max": 0.15586164065749025, + "nauc_mrr_at_100_std": 0.09368786780755935, + "nauc_mrr_at_10_diff1": -0.2754707494354536, + "nauc_mrr_at_10_max": 0.16860312970455407, + "nauc_mrr_at_10_std": 0.09050459282085065, + "nauc_mrr_at_1_diff1": -0.19631517753331543, + "nauc_mrr_at_1_max": 0.236860149658647, + "nauc_mrr_at_1_std": -0.01994454878812264, + "nauc_mrr_at_20_diff1": -0.27839677071526, + "nauc_mrr_at_20_max": 0.1615180503347429, + "nauc_mrr_at_20_std": 0.09183806368518946, + "nauc_mrr_at_3_diff1": -0.2852138636025044, + "nauc_mrr_at_3_max": 0.17441047601220877, + "nauc_mrr_at_3_std": 0.1301479492243458, + "nauc_mrr_at_5_diff1": -0.2754707494354536, + "nauc_mrr_at_5_max": 0.16860312970455407, + "nauc_mrr_at_5_std": 0.09050459282085065, + "nauc_ndcg_at_1000_diff1": -0.2708524099375955, + "nauc_ndcg_at_1000_max": 0.1462212343162209, + "nauc_ndcg_at_1000_std": 0.13236351721468018, + "nauc_ndcg_at_100_diff1": -0.2999893850603643, + "nauc_ndcg_at_100_max": 0.1105185531422724, + "nauc_ndcg_at_100_std": 0.15311561566657694, + "nauc_ndcg_at_10_diff1": -0.27466649873010107, + "nauc_ndcg_at_10_max": 0.16425858048656408, + "nauc_ndcg_at_10_std": 0.12216702114512916, + "nauc_ndcg_at_1_diff1": -0.15840488328414284, + "nauc_ndcg_at_1_max": 0.30346122887040494, + "nauc_ndcg_at_1_std": 0.049771934531795, + "nauc_ndcg_at_20_diff1": -0.2886967979788647, + "nauc_ndcg_at_20_max": 0.13646757078462018, + "nauc_ndcg_at_20_std": 0.11774155369915566, + "nauc_ndcg_at_3_diff1": -0.29680730637047387, + "nauc_ndcg_at_3_max": 0.17213099438008325, + "nauc_ndcg_at_3_std": 0.19399951682309413, + "nauc_ndcg_at_5_diff1": -0.27466649873010107, + "nauc_ndcg_at_5_max": 0.16425858048656408, + "nauc_ndcg_at_5_std": 0.12216702114512916, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": -0.43146556282386345, + "nauc_precision_at_100_max": -0.06355656942577005, + "nauc_precision_at_100_std": 0.27635852733672717, + "nauc_precision_at_10_diff1": -0.29794573763868604, + "nauc_precision_at_10_max": 0.11284457842558356, + "nauc_precision_at_10_std": 0.12398607701152996, + "nauc_precision_at_1_diff1": -0.15840488328414284, + "nauc_precision_at_1_max": 0.30346122887040494, + "nauc_precision_at_1_std": 0.049771934531795, + "nauc_precision_at_20_diff1": -0.34053506282934753, + "nauc_precision_at_20_max": 0.036319416295095025, + "nauc_precision_at_20_std": 0.1015808674503442, + "nauc_precision_at_3_diff1": -0.3407360628705749, + "nauc_precision_at_3_max": 0.1227414658106165, + "nauc_precision_at_3_std": 0.26448093183173715, + "nauc_precision_at_5_diff1": -0.29794573763868604, + "nauc_precision_at_5_max": 0.11284457842558356, + "nauc_precision_at_5_std": 0.12398607701152996, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.4314655628238636, + "nauc_recall_at_100_max": -0.06355656942577056, + "nauc_recall_at_100_std": 0.27635852733672706, + "nauc_recall_at_10_diff1": -0.2979457376386859, + "nauc_recall_at_10_max": 0.1128445784255835, + "nauc_recall_at_10_std": 0.12398607701152999, + "nauc_recall_at_1_diff1": -0.15840488328414284, + "nauc_recall_at_1_max": 0.30346122887040494, + "nauc_recall_at_1_std": 0.049771934531795, + "nauc_recall_at_20_diff1": -0.3405350628293475, + "nauc_recall_at_20_max": 0.03631941629509504, + "nauc_recall_at_20_std": 0.10158086745034434, + "nauc_recall_at_3_diff1": -0.34073606287057506, + "nauc_recall_at_3_max": 0.12274146581061644, + "nauc_recall_at_3_std": 0.26448093183173693, + "nauc_recall_at_5_diff1": -0.2979457376386859, + "nauc_recall_at_5_max": 0.1128445784255835, + "nauc_recall_at_5_std": 0.12398607701152999, + "ndcg_at_1": 0.08, + "ndcg_at_10": 0.15421, + "ndcg_at_100": 0.2321, + "ndcg_at_1000": 0.28117, + "ndcg_at_20": 0.17448, + "ndcg_at_3": 0.13786, + "ndcg_at_5": 0.15421, + "precision_at_1": 0.08, + "precision_at_10": 0.022, + "precision_at_100": 0.0061, + "precision_at_1000": 0.001, + "precision_at_20": 0.015, + "precision_at_3": 0.06, + "precision_at_5": 0.044, + "recall_at_1": 0.08, + "recall_at_10": 0.22, + "recall_at_100": 0.61, + "recall_at_1000": 1.0, + "recall_at_20": 0.3, + "recall_at_3": 0.18, + "recall_at_5": 0.22 + } + ] + }, + "task_name": "VidoreSyntheticDocQAEnergyRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..1f71d1c048 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", + "evaluation_time": 28.769615650177002, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.06, + "cv_recall_at_10": 0.16, + "cv_recall_at_100": 0.42, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.23, + "cv_recall_at_3": 0.09, + "cv_recall_at_5": 0.09, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.08123, + "map_at_1": 0.06, + "map_at_10": 0.08121, + "map_at_100": 0.09109, + "map_at_1000": 0.0932, + "map_at_20": 0.08728, + "map_at_3": 0.07, + "map_at_5": 0.075, + "mrr_at_1": 0.06, + "mrr_at_10": 0.08202777777777777, + "mrr_at_100": 0.09121283057539557, + "mrr_at_1000": 0.09323295120835003, + "mrr_at_20": 0.08731412520228309, + "mrr_at_3": 0.07333333333333333, + "mrr_at_5": 0.07333333333333333, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.16483915286310502, + "nauc_cv_recall_at_100_max": 0.43260232781190855, + "nauc_cv_recall_at_100_std": 0.2827721181014594, + "nauc_cv_recall_at_10_diff1": 0.058591949608234696, + "nauc_cv_recall_at_10_max": 0.34500307266861274, + "nauc_cv_recall_at_10_std": 0.20777001075434018, + "nauc_cv_recall_at_1_diff1": 0.1739558179053752, + "nauc_cv_recall_at_1_max": 0.33887845452106247, + "nauc_cv_recall_at_1_std": 0.3941507915213307, + "nauc_cv_recall_at_20_diff1": 0.19142477997590027, + "nauc_cv_recall_at_20_max": 0.3804243376397552, + "nauc_cv_recall_at_20_std": 0.09292109638352061, + "nauc_cv_recall_at_3_diff1": 0.37417643025370423, + "nauc_cv_recall_at_3_max": 0.5592523030140416, + "nauc_cv_recall_at_3_std": 0.42455952061532964, + "nauc_cv_recall_at_5_diff1": 0.37417643025370423, + "nauc_cv_recall_at_5_max": 0.5592523030140416, + "nauc_cv_recall_at_5_std": 0.42455952061532964, + "nauc_map_at_1000_diff1": 0.22211997448946572, + "nauc_map_at_1000_max": 0.4226703021544981, + "nauc_map_at_1000_std": 0.35509700498369534, + "nauc_map_at_100_diff1": 0.22279977014716346, + "nauc_map_at_100_max": 0.4252031145284334, + "nauc_map_at_100_std": 0.3563523572828241, + "nauc_map_at_10_diff1": 0.22192935627611637, + "nauc_map_at_10_max": 0.42415004690257796, + "nauc_map_at_10_std": 0.3662952971659471, + "nauc_map_at_1_diff1": 0.1739558179053752, + "nauc_map_at_1_max": 0.33887845452106247, + "nauc_map_at_1_std": 0.3941507915213307, + "nauc_map_at_20_diff1": 0.2235370957493086, + "nauc_map_at_20_max": 0.42363627165734064, + "nauc_map_at_20_std": 0.3493697046949435, + "nauc_map_at_3_diff1": 0.2436659128368275, + "nauc_map_at_3_max": 0.433324389589482, + "nauc_map_at_3_std": 0.45655257004867944, + "nauc_map_at_5_diff1": 0.24691888024326974, + "nauc_map_at_5_max": 0.44856452911188616, + "nauc_map_at_5_std": 0.408639656560236, + "nauc_mrr_at_1000_diff1": 0.24970464634892656, + "nauc_mrr_at_1000_max": 0.4293471710105555, + "nauc_mrr_at_1000_std": 0.3328740535666148, + "nauc_mrr_at_100_diff1": 0.24973309243013533, + "nauc_mrr_at_100_max": 0.43118294840439664, + "nauc_mrr_at_100_std": 0.3336076391475648, + "nauc_mrr_at_10_diff1": 0.23498849214811451, + "nauc_mrr_at_10_max": 0.424442907589755, + "nauc_mrr_at_10_std": 0.3482117751660398, + "nauc_mrr_at_1_diff1": 0.1739558179053752, + "nauc_mrr_at_1_max": 0.33887845452106247, + "nauc_mrr_at_1_std": 0.3941507915213307, + "nauc_mrr_at_20_diff1": 0.25119430080323674, + "nauc_mrr_at_20_max": 0.43033456107778606, + "nauc_mrr_at_20_std": 0.3270902377687649, + "nauc_mrr_at_3_diff1": 0.29341171305217445, + "nauc_mrr_at_3_max": 0.4590823718808692, + "nauc_mrr_at_3_std": 0.3990413932726785, + "nauc_mrr_at_5_diff1": 0.29341171305217445, + "nauc_mrr_at_5_max": 0.4590823718808692, + "nauc_mrr_at_5_std": 0.3990413932726785, + "nauc_ndcg_at_1000_diff1": 0.2138824549051445, + "nauc_ndcg_at_1000_max": 0.4161443897516234, + "nauc_ndcg_at_1000_std": 0.3158045285204317, + "nauc_ndcg_at_100_diff1": 0.21009464889600732, + "nauc_ndcg_at_100_max": 0.4347151408518815, + "nauc_ndcg_at_100_std": 0.3206930265791543, + "nauc_ndcg_at_10_diff1": 0.2036918358650856, + "nauc_ndcg_at_10_max": 0.41509671780130775, + "nauc_ndcg_at_10_std": 0.30358830108810564, + "nauc_ndcg_at_1_diff1": 0.1739558179053752, + "nauc_ndcg_at_1_max": 0.33887845452106247, + "nauc_ndcg_at_1_std": 0.3941507915213307, + "nauc_ndcg_at_20_diff1": 0.2130246179818501, + "nauc_ndcg_at_20_max": 0.4125222189697398, + "nauc_ndcg_at_20_std": 0.25841208063962756, + "nauc_ndcg_at_3_diff1": 0.25874820773692664, + "nauc_ndcg_at_3_max": 0.45375846655090646, + "nauc_ndcg_at_3_std": 0.47005365661247767, + "nauc_ndcg_at_5_diff1": 0.26232291688461556, + "nauc_ndcg_at_5_max": 0.4758317739319828, + "nauc_ndcg_at_5_std": 0.3924146371401771, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.1842618546314115, + "nauc_precision_at_100_max": 0.4588010385315887, + "nauc_precision_at_100_std": 0.30947983001149604, + "nauc_precision_at_10_diff1": 0.155266606188892, + "nauc_precision_at_10_max": 0.3810704566071967, + "nauc_precision_at_10_std": 0.16837012397943735, + "nauc_precision_at_1_diff1": 0.1739558179053752, + "nauc_precision_at_1_max": 0.33887845452106247, + "nauc_precision_at_1_std": 0.3941507915213307, + "nauc_precision_at_20_diff1": 0.19142477997590027, + "nauc_precision_at_20_max": 0.3804243376397553, + "nauc_precision_at_20_std": 0.0929210963835207, + "nauc_precision_at_3_diff1": 0.2959484840354171, + "nauc_precision_at_3_max": 0.504158840890797, + "nauc_precision_at_3_std": 0.5033539039441909, + "nauc_precision_at_5_diff1": 0.2952508720150253, + "nauc_precision_at_5_max": 0.5357123691977462, + "nauc_precision_at_5_std": 0.35025489669975846, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.18426185463141187, + "nauc_recall_at_100_max": 0.4588010385315888, + "nauc_recall_at_100_std": 0.30947983001149604, + "nauc_recall_at_10_diff1": 0.15526660618889215, + "nauc_recall_at_10_max": 0.38107045660719685, + "nauc_recall_at_10_std": 0.16837012397943757, + "nauc_recall_at_1_diff1": 0.1739558179053752, + "nauc_recall_at_1_max": 0.33887845452106247, + "nauc_recall_at_1_std": 0.3941507915213307, + "nauc_recall_at_20_diff1": 0.19142477997590027, + "nauc_recall_at_20_max": 0.3804243376397552, + "nauc_recall_at_20_std": 0.09292109638352061, + "nauc_recall_at_3_diff1": 0.29594848403541707, + "nauc_recall_at_3_max": 0.5041588408907967, + "nauc_recall_at_3_std": 0.503353903944191, + "nauc_recall_at_5_diff1": 0.2952508720150256, + "nauc_recall_at_5_max": 0.5357123691977462, + "nauc_recall_at_5_std": 0.3502548966997586, + "ndcg_at_1": 0.06, + "ndcg_at_10": 0.09692, + "ndcg_at_100": 0.14955, + "ndcg_at_1000": 0.22068, + "ndcg_at_20": 0.11779, + "ndcg_at_3": 0.07262, + "ndcg_at_5": 0.08123, + "precision_at_1": 0.06, + "precision_at_10": 0.015, + "precision_at_100": 0.0041, + "precision_at_1000": 0.001, + "precision_at_20": 0.0115, + "precision_at_3": 0.02667, + "precision_at_5": 0.02, + "recall_at_1": 0.06, + "recall_at_10": 0.15, + "recall_at_100": 0.41, + "recall_at_1000": 1.0, + "recall_at_20": 0.23, + "recall_at_3": 0.08, + "recall_at_5": 0.1 + } + ] + }, + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..f817e7bf63 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", + "evaluation_time": 27.634584188461304, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.07, + "cv_recall_at_10": 0.24, + "cv_recall_at_100": 0.49, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.26, + "cv_recall_at_3": 0.13, + "cv_recall_at_5": 0.16, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.1179, + "map_at_1": 0.07, + "map_at_10": 0.1134, + "map_at_100": 0.11961, + "map_at_1000": 0.12196, + "map_at_20": 0.11461, + "map_at_3": 0.095, + "map_at_5": 0.104, + "mrr_at_1": 0.07, + "mrr_at_10": 0.11423412698412698, + "mrr_at_100": 0.1204383669219322, + "mrr_at_1000": 0.1227870262150322, + "mrr_at_20": 0.11548412698412697, + "mrr_at_3": 0.09833333333333333, + "mrr_at_5": 0.10483333333333335, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.0033593137970868337, + "nauc_cv_recall_at_100_max": 0.13656340870765996, + "nauc_cv_recall_at_100_std": 0.0730551166465286, + "nauc_cv_recall_at_10_diff1": 0.0789300263599329, + "nauc_cv_recall_at_10_max": 0.32840582314881384, + "nauc_cv_recall_at_10_std": -0.11701713395638631, + "nauc_cv_recall_at_1_diff1": 0.13193299858177768, + "nauc_cv_recall_at_1_max": 0.7504695465521868, + "nauc_cv_recall_at_1_std": -0.1696883744106713, + "nauc_cv_recall_at_20_diff1": 0.05911942412766328, + "nauc_cv_recall_at_20_max": 0.3305917724263536, + "nauc_cv_recall_at_20_std": -0.08836113439912116, + "nauc_cv_recall_at_3_diff1": 0.15529090461373152, + "nauc_cv_recall_at_3_max": 0.7226388484699247, + "nauc_cv_recall_at_3_std": -0.008103691432920732, + "nauc_cv_recall_at_5_diff1": 0.12473114149638939, + "nauc_cv_recall_at_5_max": 0.5643916116146874, + "nauc_cv_recall_at_5_std": -0.03466354278691037, + "nauc_map_at_1000_diff1": 0.10475800103203144, + "nauc_map_at_1000_max": 0.5801798373932328, + "nauc_map_at_1000_std": -0.11603267202556924, + "nauc_map_at_100_diff1": 0.10355861111113708, + "nauc_map_at_100_max": 0.5772673044620018, + "nauc_map_at_100_std": -0.1162521804291923, + "nauc_map_at_10_diff1": 0.11088361581323901, + "nauc_map_at_10_max": 0.5871221018084627, + "nauc_map_at_10_std": -0.1280790298966015, + "nauc_map_at_1_diff1": 0.13193299858177768, + "nauc_map_at_1_max": 0.6216796350952509, + "nauc_map_at_1_std": -0.2189045191460004, + "nauc_map_at_20_diff1": 0.10849886911360809, + "nauc_map_at_20_max": 0.5864430025825756, + "nauc_map_at_20_std": -0.12474158742824958, + "nauc_map_at_3_diff1": 0.11585374782188503, + "nauc_map_at_3_max": 0.6925014276719429, + "nauc_map_at_3_std": -0.14349933375309332, + "nauc_map_at_5_diff1": 0.1203639033404092, + "nauc_map_at_5_max": 0.6400549402202009, + "nauc_map_at_5_std": -0.11654861027090663, + "nauc_mrr_at_1000_diff1": 0.10924678698408471, + "nauc_mrr_at_1000_max": 0.6234840055556127, + "nauc_mrr_at_1000_std": -0.09032182856834724, + "nauc_mrr_at_100_diff1": 0.10801566961185796, + "nauc_mrr_at_100_max": 0.6202581345282157, + "nauc_mrr_at_100_std": -0.09072222954701539, + "nauc_mrr_at_10_diff1": 0.11531417186020634, + "nauc_mrr_at_10_max": 0.630524081718047, + "nauc_mrr_at_10_std": -0.10245821572444079, + "nauc_mrr_at_1_diff1": 0.13193299858177768, + "nauc_mrr_at_1_max": 0.7504695465521868, + "nauc_mrr_at_1_std": -0.1696883744106713, + "nauc_mrr_at_20_diff1": 0.112947332125841, + "nauc_mrr_at_20_max": 0.6295858860926723, + "nauc_mrr_at_20_std": -0.09929781553533608, + "nauc_mrr_at_3_diff1": 0.13664548907836976, + "nauc_mrr_at_3_max": 0.7357647047609058, + "nauc_mrr_at_3_std": -0.08623869826941129, + "nauc_mrr_at_5_diff1": 0.12530857755601577, + "nauc_mrr_at_5_max": 0.6851435714471256, + "nauc_mrr_at_5_std": -0.0898469382407397, + "nauc_ndcg_at_1000_diff1": 0.09128770677873728, + "nauc_ndcg_at_1000_max": 0.5149398611965009, + "nauc_ndcg_at_1000_std": -0.08505448929111248, + "nauc_ndcg_at_100_diff1": 0.07184603454814685, + "nauc_ndcg_at_100_max": 0.443191212813763, + "nauc_ndcg_at_100_std": -0.06142226399285712, + "nauc_ndcg_at_10_diff1": 0.10049597269892058, + "nauc_ndcg_at_10_max": 0.5040530206853383, + "nauc_ndcg_at_10_std": -0.12066788476859182, + "nauc_ndcg_at_1_diff1": 0.13193299858177768, + "nauc_ndcg_at_1_max": 0.6216796350952509, + "nauc_ndcg_at_1_std": -0.2189045191460004, + "nauc_ndcg_at_20_diff1": 0.09281328000703318, + "nauc_ndcg_at_20_max": 0.5026776186328519, + "nauc_ndcg_at_20_std": -0.1098844443956209, + "nauc_ndcg_at_3_diff1": 0.11282399978682046, + "nauc_ndcg_at_3_max": 0.7058460909341273, + "nauc_ndcg_at_3_std": -0.12929104036372097, + "nauc_ndcg_at_5_diff1": 0.12132540415555444, + "nauc_ndcg_at_5_max": 0.6182882597405424, + "nauc_ndcg_at_5_std": -0.08996362964673281, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.003359313797086568, + "nauc_precision_at_100_max": 0.1365634087076601, + "nauc_precision_at_100_std": 0.07305511664652851, + "nauc_precision_at_10_diff1": 0.07893002635993276, + "nauc_precision_at_10_max": 0.3284058231488137, + "nauc_precision_at_10_std": -0.1170171339563864, + "nauc_precision_at_1_diff1": 0.13193299858177768, + "nauc_precision_at_1_max": 0.6216796350952509, + "nauc_precision_at_1_std": -0.2189045191460004, + "nauc_precision_at_20_diff1": 0.05911942412766315, + "nauc_precision_at_20_max": 0.3305917724263538, + "nauc_precision_at_20_std": -0.08836113439912127, + "nauc_precision_at_3_diff1": 0.10591413136195617, + "nauc_precision_at_3_max": 0.7362809212396925, + "nauc_precision_at_3_std": -0.09688655103781642, + "nauc_precision_at_5_diff1": 0.12473114149638945, + "nauc_precision_at_5_max": 0.5643916116146872, + "nauc_precision_at_5_std": -0.03466354278691048, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.0033593137970868337, + "nauc_recall_at_100_max": 0.13656340870765996, + "nauc_recall_at_100_std": 0.0730551166465286, + "nauc_recall_at_10_diff1": 0.0789300263599329, + "nauc_recall_at_10_max": 0.32840582314881384, + "nauc_recall_at_10_std": -0.11701713395638631, + "nauc_recall_at_1_diff1": 0.13193299858177768, + "nauc_recall_at_1_max": 0.6216796350952509, + "nauc_recall_at_1_std": -0.2189045191460004, + "nauc_recall_at_20_diff1": 0.05911942412766328, + "nauc_recall_at_20_max": 0.3305917724263536, + "nauc_recall_at_20_std": -0.08836113439912116, + "nauc_recall_at_3_diff1": 0.10591413136195621, + "nauc_recall_at_3_max": 0.736280921239693, + "nauc_recall_at_3_std": -0.09688655103781627, + "nauc_recall_at_5_diff1": 0.12473114149638939, + "nauc_recall_at_5_max": 0.5643916116146874, + "nauc_recall_at_5_std": -0.03466354278691037, + "ndcg_at_1": 0.07, + "ndcg_at_10": 0.1425, + "ndcg_at_100": 0.1882, + "ndcg_at_1000": 0.25263, + "ndcg_at_20": 0.14734, + "ndcg_at_3": 0.10155, + "ndcg_at_5": 0.1179, + "precision_at_1": 0.07, + "precision_at_10": 0.024, + "precision_at_100": 0.0049, + "precision_at_1000": 0.001, + "precision_at_20": 0.013, + "precision_at_3": 0.04, + "precision_at_5": 0.032, + "recall_at_1": 0.07, + "recall_at_10": 0.24, + "recall_at_100": 0.49, + "recall_at_1000": 1.0, + "recall_at_20": 0.26, + "recall_at_3": 0.12, + "recall_at_5": 0.16 + } + ] + }, + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTabfquadRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..fc0eead64f --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", + "evaluation_time": 4.014543533325195, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.01429, + "cv_recall_at_10": 0.12857, + "cv_recall_at_100": 0.51429, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.21071, + "cv_recall_at_3": 0.05714, + "cv_recall_at_5": 0.08571, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.05553, + "map_at_1": 0.02143, + "map_at_10": 0.04813, + "map_at_100": 0.06068, + "map_at_1000": 0.06382, + "map_at_20": 0.05355, + "map_at_3": 0.03631, + "map_at_5": 0.04345, + "mrr_at_1": 0.014285714285714285, + "mrr_at_10": 0.0428798185941043, + "mrr_at_100": 0.05584756483687463, + "mrr_at_1000": 0.058994237118739384, + "mrr_at_20": 0.048727962983132145, + "mrr_at_3": 0.03095238095238095, + "mrr_at_5": 0.03755952380952381, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.14435075722453997, + "nauc_cv_recall_at_100_max": 0.04018409055787388, + "nauc_cv_recall_at_100_std": 0.35451437181270307, + "nauc_cv_recall_at_10_diff1": -0.14477967959256025, + "nauc_cv_recall_at_10_max": 0.10806498404409637, + "nauc_cv_recall_at_10_std": 0.2212874319053606, + "nauc_cv_recall_at_1_diff1": -0.3902602629460694, + "nauc_cv_recall_at_1_max": 0.07713979071639386, + "nauc_cv_recall_at_1_std": 0.3971022269922189, + "nauc_cv_recall_at_20_diff1": -0.13442302587360852, + "nauc_cv_recall_at_20_max": 0.07322101291703624, + "nauc_cv_recall_at_20_std": 0.2198268315137276, + "nauc_cv_recall_at_3_diff1": -0.17822645559431183, + "nauc_cv_recall_at_3_max": 0.08101354977193453, + "nauc_cv_recall_at_3_std": 0.2634156157767643, + "nauc_cv_recall_at_5_diff1": -0.16759458009122633, + "nauc_cv_recall_at_5_max": 0.10308782756461854, + "nauc_cv_recall_at_5_std": 0.24713800196762364, + "nauc_map_at_1000_diff1": -0.19046764108103575, + "nauc_map_at_1000_max": 0.08958064836736834, + "nauc_map_at_1000_std": 0.25301683753256515, + "nauc_map_at_100_diff1": -0.18963136394230734, + "nauc_map_at_100_max": 0.08780243681091121, + "nauc_map_at_100_std": 0.25502426729338673, + "nauc_map_at_10_diff1": -0.1859454096855409, + "nauc_map_at_10_max": 0.0922310989379099, + "nauc_map_at_10_std": 0.25011689318446306, + "nauc_map_at_1_diff1": -0.29433861014220575, + "nauc_map_at_1_max": 0.1435470888113763, + "nauc_map_at_1_std": 0.28816742688489394, + "nauc_map_at_20_diff1": -0.18776838654510583, + "nauc_map_at_20_max": 0.08945737330394286, + "nauc_map_at_20_std": 0.24766535799913875, + "nauc_map_at_3_diff1": -0.21081430588483685, + "nauc_map_at_3_max": 0.06450931835476176, + "nauc_map_at_3_std": 0.27496734067306816, + "nauc_map_at_5_diff1": -0.2022784493753467, + "nauc_map_at_5_max": 0.09233031083797995, + "nauc_map_at_5_std": 0.2453719801081337, + "nauc_mrr_at_1000_diff1": -0.20178163215368516, + "nauc_mrr_at_1000_max": 0.06804656436386369, + "nauc_mrr_at_1000_std": 0.26218897036305217, + "nauc_mrr_at_100_diff1": -0.2005749976351999, + "nauc_mrr_at_100_max": 0.06662175737366387, + "nauc_mrr_at_100_std": 0.2641890624642852, + "nauc_mrr_at_10_diff1": -0.20573494584516494, + "nauc_mrr_at_10_max": 0.07332293876449567, + "nauc_mrr_at_10_std": 0.2588373545696161, + "nauc_mrr_at_1_diff1": -0.3902602629460694, + "nauc_mrr_at_1_max": 0.07713979071639386, + "nauc_mrr_at_1_std": 0.3971022269922189, + "nauc_mrr_at_20_diff1": -0.19903140504273983, + "nauc_mrr_at_20_max": 0.06777812049985836, + "nauc_mrr_at_20_std": 0.256438663304796, + "nauc_mrr_at_3_diff1": -0.23416957338341837, + "nauc_mrr_at_3_max": 0.0601277579410126, + "nauc_mrr_at_3_std": 0.2875121256527214, + "nauc_mrr_at_5_diff1": -0.21939272971424953, + "nauc_mrr_at_5_max": 0.07277131754103458, + "nauc_mrr_at_5_std": 0.27610187703812117, + "nauc_ndcg_at_1000_diff1": -0.17606361925913822, + "nauc_ndcg_at_1000_max": 0.08433762641379804, + "nauc_ndcg_at_1000_std": 0.2549701974882767, + "nauc_ndcg_at_100_diff1": -0.1661933244126871, + "nauc_ndcg_at_100_max": 0.06822003904242817, + "nauc_ndcg_at_100_std": 0.2860804140807971, + "nauc_ndcg_at_10_diff1": -0.1543556954363095, + "nauc_ndcg_at_10_max": 0.09069581700525794, + "nauc_ndcg_at_10_std": 0.24111594833834304, + "nauc_ndcg_at_1_diff1": -0.29433861014220575, + "nauc_ndcg_at_1_max": 0.1435470888113763, + "nauc_ndcg_at_1_std": 0.28816742688489394, + "nauc_ndcg_at_20_diff1": -0.1613278895889277, + "nauc_ndcg_at_20_max": 0.08190405817102135, + "nauc_ndcg_at_20_std": 0.23431193680408277, + "nauc_ndcg_at_3_diff1": -0.19150484931331083, + "nauc_ndcg_at_3_max": 0.04565509051495962, + "nauc_ndcg_at_3_std": 0.272087289057183, + "nauc_ndcg_at_5_diff1": -0.18240460530220248, + "nauc_ndcg_at_5_max": 0.08876459284421898, + "nauc_ndcg_at_5_std": 0.22815212336675553, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": -0.1443507572245399, + "nauc_precision_at_100_max": 0.04018409055787383, + "nauc_precision_at_100_std": 0.3545143718127031, + "nauc_precision_at_10_diff1": -0.11212161916576463, + "nauc_precision_at_10_max": 0.09143561500960012, + "nauc_precision_at_10_std": 0.2314019548194914, + "nauc_precision_at_1_diff1": -0.29433861014220575, + "nauc_precision_at_1_max": 0.1435470888113763, + "nauc_precision_at_1_std": 0.28816742688489394, + "nauc_precision_at_20_diff1": -0.1344230258736084, + "nauc_precision_at_20_max": 0.0732210129170362, + "nauc_precision_at_20_std": 0.21982683151372764, + "nauc_precision_at_3_diff1": -0.15702583689767838, + "nauc_precision_at_3_max": 0.011900440347858922, + "nauc_precision_at_3_std": 0.26697075395760667, + "nauc_precision_at_5_diff1": -0.15362944005283702, + "nauc_precision_at_5_max": 0.08550907102020577, + "nauc_precision_at_5_std": 0.20099688344925784, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.14435075722453997, + "nauc_recall_at_100_max": 0.04018409055787388, + "nauc_recall_at_100_std": 0.35451437181270307, + "nauc_recall_at_10_diff1": -0.1121216191657646, + "nauc_recall_at_10_max": 0.09143561500960022, + "nauc_recall_at_10_std": 0.2314019548194913, + "nauc_recall_at_1_diff1": -0.29433861014220575, + "nauc_recall_at_1_max": 0.1435470888113763, + "nauc_recall_at_1_std": 0.28816742688489394, + "nauc_recall_at_20_diff1": -0.13442302587360852, + "nauc_recall_at_20_max": 0.07322101291703624, + "nauc_recall_at_20_std": 0.2198268315137276, + "nauc_recall_at_3_diff1": -0.1570258368976784, + "nauc_recall_at_3_max": 0.011900440347858999, + "nauc_recall_at_3_std": 0.2669707539576066, + "nauc_recall_at_5_diff1": -0.1536294400528369, + "nauc_recall_at_5_max": 0.08550907102020605, + "nauc_recall_at_5_std": 0.20099688344925803, + "ndcg_at_1": 0.02143, + "ndcg_at_10": 0.06767, + "ndcg_at_100": 0.14238, + "ndcg_at_1000": 0.20878, + "ndcg_at_20": 0.0875, + "ndcg_at_3": 0.04247, + "ndcg_at_5": 0.05553, + "precision_at_1": 0.02143, + "precision_at_10": 0.01321, + "precision_at_100": 0.00514, + "precision_at_1000": 0.001, + "precision_at_20": 0.01054, + "precision_at_3": 0.02024, + "precision_at_5": 0.01857, + "recall_at_1": 0.02143, + "recall_at_10": 0.13214, + "recall_at_100": 0.51429, + "recall_at_1000": 1.0, + "recall_at_20": 0.21071, + "recall_at_3": 0.06071, + "recall_at_5": 0.09286 + } + ] + }, + "task_name": "VidoreTabfquadRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTatdqaRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..6e2e9d79d4 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", + "evaluation_time": 57.08458185195923, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.0018, + "cv_recall_at_10": 0.01443, + "cv_recall_at_100": 0.089, + "cv_recall_at_1000": 0.64702, + "cv_recall_at_20": 0.02044, + "cv_recall_at_3": 0.00481, + "cv_recall_at_5": 0.00782, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.00494, + "map_at_1": 0.0018, + "map_at_10": 0.00453, + "map_at_100": 0.0064, + "map_at_1000": 0.00793, + "map_at_20": 0.00509, + "map_at_3": 0.00301, + "map_at_5": 0.00382, + "mrr_at_1": 0.0018039687312086591, + "mrr_at_10": 0.004618494020177725, + "mrr_at_100": 0.006420684518018152, + "mrr_at_1000": 0.007956473201255023, + "mrr_at_20": 0.005077383723695083, + "mrr_at_3": 0.003106835037081579, + "mrr_at_5": 0.00373822409300461, + "nauc_cv_recall_at_1000_diff1": 0.08556738813980604, + "nauc_cv_recall_at_1000_max": 0.04177557742932173, + "nauc_cv_recall_at_1000_std": 0.09542880648474937, + "nauc_cv_recall_at_100_diff1": -0.03785066090099726, + "nauc_cv_recall_at_100_max": 0.16260798258815512, + "nauc_cv_recall_at_100_std": 0.16515780334855118, + "nauc_cv_recall_at_10_diff1": -0.04756502325362798, + "nauc_cv_recall_at_10_max": 0.37839103716358846, + "nauc_cv_recall_at_10_std": 0.5507149715917287, + "nauc_cv_recall_at_1_diff1": 0.22229315572956015, + "nauc_cv_recall_at_1_max": 0.3351554829783855, + "nauc_cv_recall_at_1_std": 0.7742753455023496, + "nauc_cv_recall_at_20_diff1": -0.05830832523955373, + "nauc_cv_recall_at_20_max": 0.2981752507205586, + "nauc_cv_recall_at_20_std": 0.43391229719546554, + "nauc_cv_recall_at_3_diff1": -0.00034359410196027406, + "nauc_cv_recall_at_3_max": 0.406924879634949, + "nauc_cv_recall_at_3_std": 0.7463139581208836, + "nauc_cv_recall_at_5_diff1": 0.0003485640898375281, + "nauc_cv_recall_at_5_max": 0.5266418196067134, + "nauc_cv_recall_at_5_std": 0.7917952078057013, + "nauc_map_at_1000_diff1": 0.03434531985076361, + "nauc_map_at_1000_max": 0.31700764572606704, + "nauc_map_at_1000_std": 0.513352390609002, + "nauc_map_at_100_diff1": 0.03249238585044048, + "nauc_map_at_100_max": 0.344444287197186, + "nauc_map_at_100_std": 0.5550391228334784, + "nauc_map_at_10_diff1": 0.058183011455032234, + "nauc_map_at_10_max": 0.4398961980755496, + "nauc_map_at_10_std": 0.7447981009503175, + "nauc_map_at_1_diff1": 0.10976975438838904, + "nauc_map_at_1_max": 0.47413777599313356, + "nauc_map_at_1_std": 0.8871376727511748, + "nauc_map_at_20_diff1": 0.04397629778875386, + "nauc_map_at_20_max": 0.39610041742289226, + "nauc_map_at_20_std": 0.6896304269055668, + "nauc_map_at_3_diff1": 0.16301622153895354, + "nauc_map_at_3_max": 0.4091613868613357, + "nauc_map_at_3_std": 0.8758514400262921, + "nauc_map_at_5_diff1": 0.08874006709245287, + "nauc_map_at_5_max": 0.4728023227574577, + "nauc_map_at_5_std": 0.8436644205016807, + "nauc_mrr_at_1000_diff1": 0.01740259977732265, + "nauc_mrr_at_1000_max": 0.29799538890335353, + "nauc_mrr_at_1000_std": 0.471596671070976, + "nauc_mrr_at_100_diff1": 0.013919150972653904, + "nauc_mrr_at_100_max": 0.3236918247489658, + "nauc_mrr_at_100_std": 0.5110336712019508, + "nauc_mrr_at_10_diff1": 0.027150120334202502, + "nauc_mrr_at_10_max": 0.4057588635600142, + "nauc_mrr_at_10_std": 0.6752326785079271, + "nauc_mrr_at_1_diff1": 0.22229315572956015, + "nauc_mrr_at_1_max": 0.3351554829783855, + "nauc_mrr_at_1_std": 0.7742753455023496, + "nauc_mrr_at_20_diff1": 0.016326440037051686, + "nauc_mrr_at_20_max": 0.37567945892777016, + "nauc_mrr_at_20_std": 0.6305420118844842, + "nauc_mrr_at_3_diff1": 0.06993606678947581, + "nauc_mrr_at_3_max": 0.38672053784189653, + "nauc_mrr_at_3_std": 0.7380649388523338, + "nauc_mrr_at_5_diff1": 0.05434084587610187, + "nauc_mrr_at_5_max": 0.4449723918087936, + "nauc_mrr_at_5_std": 0.7605202238131582, + "nauc_ndcg_at_1000_diff1": 0.050164927261478795, + "nauc_ndcg_at_1000_max": 0.117452719821097, + "nauc_ndcg_at_1000_std": 0.1867385570636576, + "nauc_ndcg_at_100_diff1": -0.016674132560073465, + "nauc_ndcg_at_100_max": 0.21961935814992392, + "nauc_ndcg_at_100_std": 0.27453278925206187, + "nauc_ndcg_at_10_diff1": 0.020330391687192594, + "nauc_ndcg_at_10_max": 0.4189996415078504, + "nauc_ndcg_at_10_std": 0.6632133533180021, + "nauc_ndcg_at_1_diff1": 0.10976975438838904, + "nauc_ndcg_at_1_max": 0.47413777599313356, + "nauc_ndcg_at_1_std": 0.8871376727511748, + "nauc_ndcg_at_20_diff1": 0.004206349712176916, + "nauc_ndcg_at_20_max": 0.3318577423266575, + "nauc_ndcg_at_20_std": 0.5628695087328316, + "nauc_ndcg_at_3_diff1": 0.17023097167757442, + "nauc_ndcg_at_3_max": 0.38994775524514685, + "nauc_ndcg_at_3_std": 0.8747791276963487, + "nauc_ndcg_at_5_diff1": 0.06334559288223734, + "nauc_ndcg_at_5_max": 0.4816104113947934, + "nauc_ndcg_at_5_std": 0.828927419007028, + "nauc_precision_at_1000_diff1": 0.08556738813980541, + "nauc_precision_at_1000_max": 0.04177557742932134, + "nauc_precision_at_1000_std": 0.09542880648474839, + "nauc_precision_at_100_diff1": -0.035702893863874686, + "nauc_precision_at_100_max": 0.16550643303366588, + "nauc_precision_at_100_std": 0.1588177693069593, + "nauc_precision_at_10_diff1": -0.02465221955110585, + "nauc_precision_at_10_max": 0.390557146877249, + "nauc_precision_at_10_std": 0.5605789784335422, + "nauc_precision_at_1_diff1": 0.10976975438838904, + "nauc_precision_at_1_max": 0.47413777599313356, + "nauc_precision_at_1_std": 0.8871376727511748, + "nauc_precision_at_20_diff1": -0.02927689545315649, + "nauc_precision_at_20_max": 0.26369251710115277, + "nauc_precision_at_20_std": 0.43725836390939754, + "nauc_precision_at_3_diff1": 0.18284774781549482, + "nauc_precision_at_3_max": 0.35347854339802, + "nauc_precision_at_3_std": 0.8730298818450715, + "nauc_precision_at_5_diff1": 0.023455296242707004, + "nauc_precision_at_5_max": 0.4960652977597613, + "nauc_precision_at_5_std": 0.8066669786767225, + "nauc_recall_at_1000_diff1": 0.08556738813980604, + "nauc_recall_at_1000_max": 0.04177557742932173, + "nauc_recall_at_1000_std": 0.09542880648474937, + "nauc_recall_at_100_diff1": -0.03570289386387487, + "nauc_recall_at_100_max": 0.16550643303366577, + "nauc_recall_at_100_std": 0.15881776930695907, + "nauc_recall_at_10_diff1": -0.024652219551105883, + "nauc_recall_at_10_max": 0.39055714687724913, + "nauc_recall_at_10_std": 0.5605789784335424, + "nauc_recall_at_1_diff1": 0.10976975438838904, + "nauc_recall_at_1_max": 0.47413777599313356, + "nauc_recall_at_1_std": 0.8871376727511748, + "nauc_recall_at_20_diff1": -0.029276895453156492, + "nauc_recall_at_20_max": 0.26369251710115293, + "nauc_recall_at_20_std": 0.43725836390939793, + "nauc_recall_at_3_diff1": 0.18284774781549484, + "nauc_recall_at_3_max": 0.35347854339801976, + "nauc_recall_at_3_std": 0.8730298818450716, + "nauc_recall_at_5_diff1": 0.023455296242707074, + "nauc_recall_at_5_max": 0.4960652977597618, + "nauc_recall_at_5_std": 0.8066669786767229, + "ndcg_at_1": 0.0018, + "ndcg_at_10": 0.00668, + "ndcg_at_100": 0.02049, + "ndcg_at_1000": 0.08436, + "ndcg_at_20": 0.00868, + "ndcg_at_3": 0.00346, + "ndcg_at_5": 0.00494, + "precision_at_1": 0.0018, + "precision_at_10": 0.00138, + "precision_at_100": 0.0009, + "precision_at_1000": 0.00065, + "precision_at_20": 0.00108, + "precision_at_3": 0.0016, + "precision_at_5": 0.00168, + "recall_at_1": 0.0018, + "recall_at_10": 0.01383, + "recall_at_100": 0.0902, + "recall_at_1000": 0.64702, + "recall_at_20": 0.02165, + "recall_at_3": 0.00481, + "recall_at_5": 0.00842 + } + ] + }, + "task_name": "VidoreTatdqaRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreArxivQARetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..2c60f777b4 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", + "evaluation_time": 15.369404315948486, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.218, + "cv_recall_at_10": 0.426, + "cv_recall_at_100": 0.766, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.526, + "cv_recall_at_3": 0.304, + "cv_recall_at_5": 0.352, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.28621, + "map_at_1": 0.218, + "map_at_10": 0.27448, + "map_at_100": 0.28695, + "map_at_1000": 0.28821, + "map_at_20": 0.28155, + "map_at_3": 0.254, + "map_at_5": 0.2646, + "mrr_at_1": 0.218, + "mrr_at_10": 0.27447857142857146, + "mrr_at_100": 0.2869511549000328, + "mrr_at_1000": 0.28820781348047175, + "mrr_at_20": 0.28154940605490136, + "mrr_at_3": 0.2539999999999999, + "mrr_at_5": 0.26459999999999995, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.24209378407851656, + "nauc_cv_recall_at_100_max": 0.2393564651499565, + "nauc_cv_recall_at_100_std": 0.2226853098689175, + "nauc_cv_recall_at_10_diff1": 0.3765744283484118, + "nauc_cv_recall_at_10_max": 0.3556539153880319, + "nauc_cv_recall_at_10_std": 0.05377148482652398, + "nauc_cv_recall_at_1_diff1": 0.5788284986102574, + "nauc_cv_recall_at_1_max": 0.5023162373698774, + "nauc_cv_recall_at_1_std": 0.06590280225727854, + "nauc_cv_recall_at_20_diff1": 0.29657056660798986, + "nauc_cv_recall_at_20_max": 0.31536989534027415, + "nauc_cv_recall_at_20_std": 0.09941715784373036, + "nauc_cv_recall_at_3_diff1": 0.475473591093467, + "nauc_cv_recall_at_3_max": 0.4033426469319063, + "nauc_cv_recall_at_3_std": 0.026381936653145647, + "nauc_cv_recall_at_5_diff1": 0.4553467933992285, + "nauc_cv_recall_at_5_max": 0.39863326940504556, + "nauc_cv_recall_at_5_std": 0.028559611372210154, + "nauc_map_at_1000_diff1": 0.510784304760984, + "nauc_map_at_1000_max": 0.44976634773529395, + "nauc_map_at_1000_std": 0.05570654098630355, + "nauc_map_at_100_diff1": 0.5097698322258946, + "nauc_map_at_100_max": 0.44894547649673194, + "nauc_map_at_100_std": 0.05591543784058362, + "nauc_map_at_10_diff1": 0.5142652560397565, + "nauc_map_at_10_max": 0.4518266232736721, + "nauc_map_at_10_std": 0.052221764538752394, + "nauc_map_at_1_diff1": 0.5788284986102574, + "nauc_map_at_1_max": 0.5023162373698774, + "nauc_map_at_1_std": 0.06590280225727854, + "nauc_map_at_20_diff1": 0.5082674529304618, + "nauc_map_at_20_max": 0.4483239199812672, + "nauc_map_at_20_std": 0.05508802163522663, + "nauc_map_at_3_diff1": 0.5326564812078188, + "nauc_map_at_3_max": 0.4609798017337326, + "nauc_map_at_3_std": 0.04817950286142923, + "nauc_map_at_5_diff1": 0.5274072636450247, + "nauc_map_at_5_max": 0.4593382703749472, + "nauc_map_at_5_std": 0.04884229953125181, + "nauc_mrr_at_1000_diff1": 0.510784304760984, + "nauc_mrr_at_1000_max": 0.44976634773529395, + "nauc_mrr_at_1000_std": 0.05570654098630355, + "nauc_mrr_at_100_diff1": 0.5097698322258946, + "nauc_mrr_at_100_max": 0.44894547649673194, + "nauc_mrr_at_100_std": 0.05591543784058362, + "nauc_mrr_at_10_diff1": 0.5142652560397565, + "nauc_mrr_at_10_max": 0.4518266232736721, + "nauc_mrr_at_10_std": 0.052221764538752394, + "nauc_mrr_at_1_diff1": 0.5788284986102574, + "nauc_mrr_at_1_max": 0.5023162373698774, + "nauc_mrr_at_1_std": 0.06590280225727854, + "nauc_mrr_at_20_diff1": 0.5082674529304618, + "nauc_mrr_at_20_max": 0.4483239199812672, + "nauc_mrr_at_20_std": 0.05508802163522663, + "nauc_mrr_at_3_diff1": 0.5326564812078188, + "nauc_mrr_at_3_max": 0.4609798017337326, + "nauc_mrr_at_3_std": 0.04817950286142923, + "nauc_mrr_at_5_diff1": 0.5274072636450247, + "nauc_mrr_at_5_max": 0.4593382703749472, + "nauc_mrr_at_5_std": 0.04884229953125181, + "nauc_ndcg_at_1000_diff1": 0.4874518894504884, + "nauc_ndcg_at_1000_max": 0.4332576903427314, + "nauc_ndcg_at_1000_std": 0.061892632637893304, + "nauc_ndcg_at_100_diff1": 0.4647491919732108, + "nauc_ndcg_at_100_max": 0.41517598386589954, + "nauc_ndcg_at_100_std": 0.07455149159361915, + "nauc_ndcg_at_10_diff1": 0.4788542200767202, + "nauc_ndcg_at_10_max": 0.42663150440804193, + "nauc_ndcg_at_10_std": 0.051773329518349155, + "nauc_ndcg_at_1_diff1": 0.5788284986102574, + "nauc_ndcg_at_1_max": 0.5023162373698774, + "nauc_ndcg_at_1_std": 0.06590280225727854, + "nauc_ndcg_at_20_diff1": 0.4584286443534341, + "nauc_ndcg_at_20_max": 0.41589497776504764, + "nauc_ndcg_at_20_std": 0.06316090723521364, + "nauc_ndcg_at_3_diff1": 0.5171896708549963, + "nauc_ndcg_at_3_max": 0.4455034140521195, + "nauc_ndcg_at_3_std": 0.042281025973323616, + "nauc_ndcg_at_5_diff1": 0.5079788124522473, + "nauc_ndcg_at_5_max": 0.4428814009365794, + "nauc_ndcg_at_5_std": 0.04328737509334253, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.24209378407851703, + "nauc_precision_at_100_max": 0.23935646514995684, + "nauc_precision_at_100_std": 0.2226853098689183, + "nauc_precision_at_10_diff1": 0.37657442834841215, + "nauc_precision_at_10_max": 0.3556539153880323, + "nauc_precision_at_10_std": 0.05377148482652421, + "nauc_precision_at_1_diff1": 0.5788284986102574, + "nauc_precision_at_1_max": 0.5023162373698774, + "nauc_precision_at_1_std": 0.06590280225727854, + "nauc_precision_at_20_diff1": 0.29657056660798986, + "nauc_precision_at_20_max": 0.3153698953402739, + "nauc_precision_at_20_std": 0.09941715784373033, + "nauc_precision_at_3_diff1": 0.4754735910934663, + "nauc_precision_at_3_max": 0.4033426469319066, + "nauc_precision_at_3_std": 0.026381936653145834, + "nauc_precision_at_5_diff1": 0.45534679339922834, + "nauc_precision_at_5_max": 0.3986332694050452, + "nauc_precision_at_5_std": 0.028559611372210206, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.24209378407851656, + "nauc_recall_at_100_max": 0.2393564651499565, + "nauc_recall_at_100_std": 0.2226853098689175, + "nauc_recall_at_10_diff1": 0.3765744283484118, + "nauc_recall_at_10_max": 0.3556539153880319, + "nauc_recall_at_10_std": 0.05377148482652398, + "nauc_recall_at_1_diff1": 0.5788284986102574, + "nauc_recall_at_1_max": 0.5023162373698774, + "nauc_recall_at_1_std": 0.06590280225727854, + "nauc_recall_at_20_diff1": 0.29657056660798986, + "nauc_recall_at_20_max": 0.31536989534027415, + "nauc_recall_at_20_std": 0.09941715784373036, + "nauc_recall_at_3_diff1": 0.475473591093467, + "nauc_recall_at_3_max": 0.4033426469319063, + "nauc_recall_at_3_std": 0.026381936653145647, + "nauc_recall_at_5_diff1": 0.4553467933992285, + "nauc_recall_at_5_max": 0.39863326940504556, + "nauc_recall_at_5_std": 0.028559611372210154, + "ndcg_at_1": 0.218, + "ndcg_at_10": 0.31015, + "ndcg_at_100": 0.37861, + "ndcg_at_1000": 0.40927, + "ndcg_at_20": 0.3356, + "ndcg_at_3": 0.26676, + "ndcg_at_5": 0.28621, + "precision_at_1": 0.218, + "precision_at_10": 0.0426, + "precision_at_100": 0.00766, + "precision_at_1000": 0.001, + "precision_at_20": 0.0263, + "precision_at_3": 0.10133, + "precision_at_5": 0.0704, + "recall_at_1": 0.218, + "recall_at_10": 0.426, + "recall_at_100": 0.766, + "recall_at_1000": 1.0, + "recall_at_20": 0.526, + "recall_at_3": 0.304, + "recall_at_5": 0.352 + } + ] + }, + "task_name": "VidoreArxivQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreDocVQARetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..61c6949996 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b1d89eda849e636676df6ead8002602fb1858600", + "evaluation_time": 20.387259244918823, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.106, + "cv_recall_at_10": 0.308, + "cv_recall_at_100": 0.618, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.388, + "cv_recall_at_3": 0.174, + "cv_recall_at_5": 0.228, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.16692, + "map_at_1": 0.106, + "map_at_10": 0.15761, + "map_at_100": 0.16868, + "map_at_1000": 0.17047, + "map_at_20": 0.16321, + "map_at_3": 0.135, + "map_at_5": 0.147, + "mrr_at_1": 0.106, + "mrr_at_10": 0.15760873015873023, + "mrr_at_100": 0.16867595360076532, + "mrr_at_1000": 0.17047065030110975, + "mrr_at_20": 0.16321167112900875, + "mrr_at_3": 0.13500000000000004, + "mrr_at_5": 0.14700000000000008, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.18647709959934627, + "nauc_cv_recall_at_100_max": 0.15991502588767584, + "nauc_cv_recall_at_100_std": 0.3570537403017011, + "nauc_cv_recall_at_10_diff1": 0.2136343326568294, + "nauc_cv_recall_at_10_max": 0.12466152380234613, + "nauc_cv_recall_at_10_std": 0.24616034754709049, + "nauc_cv_recall_at_1_diff1": 0.3991979396235373, + "nauc_cv_recall_at_1_max": 0.3656858545246218, + "nauc_cv_recall_at_1_std": 0.33274666969460814, + "nauc_cv_recall_at_20_diff1": 0.22020310825017214, + "nauc_cv_recall_at_20_max": 0.12771428127540788, + "nauc_cv_recall_at_20_std": 0.23684483088985434, + "nauc_cv_recall_at_3_diff1": 0.32715242283767193, + "nauc_cv_recall_at_3_max": 0.2647489995641667, + "nauc_cv_recall_at_3_std": 0.2599008028700173, + "nauc_cv_recall_at_5_diff1": 0.258351245261759, + "nauc_cv_recall_at_5_max": 0.21310908211317878, + "nauc_cv_recall_at_5_std": 0.27940771223338784, + "nauc_map_at_1000_diff1": 0.3320639485473742, + "nauc_map_at_1000_max": 0.28235594429927185, + "nauc_map_at_1000_std": 0.29583968480955875, + "nauc_map_at_100_diff1": 0.33145311782359116, + "nauc_map_at_100_max": 0.28158165538707064, + "nauc_map_at_100_std": 0.2958828545201129, + "nauc_map_at_10_diff1": 0.33156624683420866, + "nauc_map_at_10_max": 0.279553774946279, + "nauc_map_at_10_std": 0.2941832501123395, + "nauc_map_at_1_diff1": 0.3991979396235373, + "nauc_map_at_1_max": 0.3656858545246218, + "nauc_map_at_1_std": 0.33274666969460814, + "nauc_map_at_20_diff1": 0.3316839346518803, + "nauc_map_at_20_max": 0.27915064588763655, + "nauc_map_at_20_std": 0.29232482968981444, + "nauc_map_at_3_diff1": 0.36732346501976904, + "nauc_map_at_3_max": 0.31561466377660846, + "nauc_map_at_3_std": 0.2963195329433932, + "nauc_map_at_5_diff1": 0.3434991787671332, + "nauc_map_at_5_max": 0.2997608630800974, + "nauc_map_at_5_std": 0.3022477486176086, + "nauc_mrr_at_1000_diff1": 0.3320639485473742, + "nauc_mrr_at_1000_max": 0.28235594429927185, + "nauc_mrr_at_1000_std": 0.29583968480955875, + "nauc_mrr_at_100_diff1": 0.33145311782359116, + "nauc_mrr_at_100_max": 0.28158165538707064, + "nauc_mrr_at_100_std": 0.2958828545201129, + "nauc_mrr_at_10_diff1": 0.33156624683420866, + "nauc_mrr_at_10_max": 0.279553774946279, + "nauc_mrr_at_10_std": 0.2941832501123395, + "nauc_mrr_at_1_diff1": 0.3991979396235373, + "nauc_mrr_at_1_max": 0.3656858545246218, + "nauc_mrr_at_1_std": 0.33274666969460814, + "nauc_mrr_at_20_diff1": 0.3316839346518803, + "nauc_mrr_at_20_max": 0.27915064588763655, + "nauc_mrr_at_20_std": 0.29232482968981444, + "nauc_mrr_at_3_diff1": 0.36732346501976904, + "nauc_mrr_at_3_max": 0.31561466377660846, + "nauc_mrr_at_3_std": 0.2963195329433932, + "nauc_mrr_at_5_diff1": 0.3434991787671332, + "nauc_mrr_at_5_max": 0.2997608630800974, + "nauc_mrr_at_5_std": 0.3022477486176086, + "nauc_ndcg_at_1000_diff1": 0.30916637131727903, + "nauc_ndcg_at_1000_max": 0.259081036791414, + "nauc_ndcg_at_1000_std": 0.29352066034827357, + "nauc_ndcg_at_100_diff1": 0.29165565271392513, + "nauc_ndcg_at_100_max": 0.24382913622899394, + "nauc_ndcg_at_100_std": 0.3018042533076364, + "nauc_ndcg_at_10_diff1": 0.29439259518708877, + "nauc_ndcg_at_10_max": 0.23166412777636708, + "nauc_ndcg_at_10_std": 0.2789810354221307, + "nauc_ndcg_at_1_diff1": 0.3991979396235373, + "nauc_ndcg_at_1_max": 0.3656858545246218, + "nauc_ndcg_at_1_std": 0.33274666969460814, + "nauc_ndcg_at_20_diff1": 0.29550536417143825, + "nauc_ndcg_at_20_max": 0.23119634036974532, + "nauc_ndcg_at_20_std": 0.2745837194582885, + "nauc_ndcg_at_3_diff1": 0.355735107849091, + "nauc_ndcg_at_3_max": 0.30075662999643216, + "nauc_ndcg_at_3_std": 0.2856705283412383, + "nauc_ndcg_at_5_diff1": 0.31771622814966893, + "nauc_ndcg_at_5_max": 0.2735109839698581, + "nauc_ndcg_at_5_std": 0.2949705422143009, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.18647709959934627, + "nauc_precision_at_100_max": 0.15991502588767598, + "nauc_precision_at_100_std": 0.35705374030170134, + "nauc_precision_at_10_diff1": 0.21363433265682943, + "nauc_precision_at_10_max": 0.1246615238023463, + "nauc_precision_at_10_std": 0.2461603475470906, + "nauc_precision_at_1_diff1": 0.3991979396235373, + "nauc_precision_at_1_max": 0.3656858545246218, + "nauc_precision_at_1_std": 0.33274666969460814, + "nauc_precision_at_20_diff1": 0.22020310825017242, + "nauc_precision_at_20_max": 0.12771428127540807, + "nauc_precision_at_20_std": 0.23684483088985492, + "nauc_precision_at_3_diff1": 0.32715242283767215, + "nauc_precision_at_3_max": 0.2647489995641667, + "nauc_precision_at_3_std": 0.2599008028700174, + "nauc_precision_at_5_diff1": 0.2583512452617593, + "nauc_precision_at_5_max": 0.21310908211317917, + "nauc_precision_at_5_std": 0.27940771223338806, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.18647709959934627, + "nauc_recall_at_100_max": 0.15991502588767584, + "nauc_recall_at_100_std": 0.3570537403017011, + "nauc_recall_at_10_diff1": 0.2136343326568294, + "nauc_recall_at_10_max": 0.12466152380234613, + "nauc_recall_at_10_std": 0.24616034754709049, + "nauc_recall_at_1_diff1": 0.3991979396235373, + "nauc_recall_at_1_max": 0.3656858545246218, + "nauc_recall_at_1_std": 0.33274666969460814, + "nauc_recall_at_20_diff1": 0.22020310825017214, + "nauc_recall_at_20_max": 0.12771428127540788, + "nauc_recall_at_20_std": 0.23684483088985434, + "nauc_recall_at_3_diff1": 0.32715242283767193, + "nauc_recall_at_3_max": 0.2647489995641667, + "nauc_recall_at_3_std": 0.2599008028700173, + "nauc_recall_at_5_diff1": 0.258351245261759, + "nauc_recall_at_5_max": 0.21310908211317878, + "nauc_recall_at_5_std": 0.27940771223338784, + "ndcg_at_1": 0.106, + "ndcg_at_10": 0.1927, + "ndcg_at_100": 0.25482, + "ndcg_at_1000": 0.30362, + "ndcg_at_20": 0.213, + "ndcg_at_3": 0.14498, + "ndcg_at_5": 0.16692, + "precision_at_1": 0.106, + "precision_at_10": 0.0308, + "precision_at_100": 0.00618, + "precision_at_1000": 0.001, + "precision_at_20": 0.0194, + "precision_at_3": 0.058, + "precision_at_5": 0.0456, + "recall_at_1": 0.106, + "recall_at_10": 0.308, + "recall_at_100": 0.618, + "recall_at_1000": 1.0, + "recall_at_20": 0.388, + "recall_at_3": 0.174, + "recall_at_5": 0.228 + } + ] + }, + "task_name": "VidoreDocVQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreInfoVQARetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..8bb3c53692 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", + "evaluation_time": 19.91568922996521, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.514, + "cv_recall_at_10": 0.78, + "cv_recall_at_100": 0.952, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.85, + "cv_recall_at_3": 0.666, + "cv_recall_at_5": 0.722, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.62564, + "map_at_1": 0.514, + "map_at_10": 0.60106, + "map_at_100": 0.60859, + "map_at_1000": 0.60883, + "map_at_20": 0.6059, + "map_at_3": 0.58033, + "map_at_5": 0.59343, + "mrr_at_1": 0.514, + "mrr_at_10": 0.6010595238095238, + "mrr_at_100": 0.6085922116441255, + "mrr_at_1000": 0.608826236252889, + "mrr_at_20": 0.6058996230893289, + "mrr_at_3": 0.5803333333333334, + "mrr_at_5": 0.5934333333333331, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5994786803610316, + "nauc_cv_recall_at_100_max": 0.4174447556800473, + "nauc_cv_recall_at_100_std": 0.7542600373482705, + "nauc_cv_recall_at_10_diff1": 0.6065006723234561, + "nauc_cv_recall_at_10_max": 0.41218528307135877, + "nauc_cv_recall_at_10_std": 0.13799323039829378, + "nauc_cv_recall_at_1_diff1": 0.7065975273128384, + "nauc_cv_recall_at_1_max": 0.4003963436889161, + "nauc_cv_recall_at_1_std": 0.05878106742438773, + "nauc_cv_recall_at_20_diff1": 0.5701204035144813, + "nauc_cv_recall_at_20_max": 0.4218939147412955, + "nauc_cv_recall_at_20_std": 0.23806703547022504, + "nauc_cv_recall_at_3_diff1": 0.643071383708003, + "nauc_cv_recall_at_3_max": 0.3869760755586455, + "nauc_cv_recall_at_3_std": 0.140661474025036, + "nauc_cv_recall_at_5_diff1": 0.6101338410105434, + "nauc_cv_recall_at_5_max": 0.3596857211946283, + "nauc_cv_recall_at_5_std": 0.1005976210307328, + "nauc_map_at_1000_diff1": 0.6767499194189067, + "nauc_map_at_1000_max": 0.39488883206709513, + "nauc_map_at_1000_std": 0.0840509486527422, + "nauc_map_at_100_diff1": 0.6767211533807833, + "nauc_map_at_100_max": 0.3949524090616767, + "nauc_map_at_100_std": 0.08452792320253084, + "nauc_map_at_10_diff1": 0.676363167757303, + "nauc_map_at_10_max": 0.3950306149617231, + "nauc_map_at_10_std": 0.08043584269066946, + "nauc_map_at_1_diff1": 0.7065975273128384, + "nauc_map_at_1_max": 0.4003963436889161, + "nauc_map_at_1_std": 0.05878106742438773, + "nauc_map_at_20_diff1": 0.6757578583739451, + "nauc_map_at_20_max": 0.3948965844015537, + "nauc_map_at_20_std": 0.08276739928464345, + "nauc_map_at_3_diff1": 0.6814372098114474, + "nauc_map_at_3_max": 0.39504568161146403, + "nauc_map_at_3_std": 0.08606245763290074, + "nauc_map_at_5_diff1": 0.6757588337318032, + "nauc_map_at_5_max": 0.3903524306747273, + "nauc_map_at_5_std": 0.07749784338609057, + "nauc_mrr_at_1000_diff1": 0.6767499194189067, + "nauc_mrr_at_1000_max": 0.39488883206709513, + "nauc_mrr_at_1000_std": 0.0840509486527422, + "nauc_mrr_at_100_diff1": 0.6767211533807833, + "nauc_mrr_at_100_max": 0.3949524090616767, + "nauc_mrr_at_100_std": 0.08452792320253084, + "nauc_mrr_at_10_diff1": 0.676363167757303, + "nauc_mrr_at_10_max": 0.3950306149617231, + "nauc_mrr_at_10_std": 0.08043584269066946, + "nauc_mrr_at_1_diff1": 0.7065975273128384, + "nauc_mrr_at_1_max": 0.4003963436889161, + "nauc_mrr_at_1_std": 0.05878106742438773, + "nauc_mrr_at_20_diff1": 0.6757578583739451, + "nauc_mrr_at_20_max": 0.3948965844015537, + "nauc_mrr_at_20_std": 0.08276739928464345, + "nauc_mrr_at_3_diff1": 0.6814372098114474, + "nauc_mrr_at_3_max": 0.39504568161146403, + "nauc_mrr_at_3_std": 0.08606245763290074, + "nauc_mrr_at_5_diff1": 0.6757588337318032, + "nauc_mrr_at_5_max": 0.3903524306747273, + "nauc_mrr_at_5_std": 0.07749784338609057, + "nauc_ndcg_at_1000_diff1": 0.67013446159532, + "nauc_ndcg_at_1000_max": 0.3951244287190329, + "nauc_ndcg_at_1000_std": 0.09644033556766034, + "nauc_ndcg_at_100_diff1": 0.6686093666822559, + "nauc_ndcg_at_100_max": 0.39599855847861803, + "nauc_ndcg_at_100_std": 0.11257804884700358, + "nauc_ndcg_at_10_diff1": 0.6633460215679075, + "nauc_ndcg_at_10_max": 0.3969814632280433, + "nauc_ndcg_at_10_std": 0.09075371352008957, + "nauc_ndcg_at_1_diff1": 0.7065975273128384, + "nauc_ndcg_at_1_max": 0.4003963436889161, + "nauc_ndcg_at_1_std": 0.05878106742438773, + "nauc_ndcg_at_20_diff1": 0.6610383819053706, + "nauc_ndcg_at_20_max": 0.39712671597648946, + "nauc_ndcg_at_20_std": 0.10152856644348263, + "nauc_ndcg_at_3_diff1": 0.6724945628522271, + "nauc_ndcg_at_3_max": 0.3931631826532289, + "nauc_ndcg_at_3_std": 0.09856146625556418, + "nauc_ndcg_at_5_diff1": 0.6619718384007528, + "nauc_ndcg_at_5_max": 0.3841315571764587, + "nauc_ndcg_at_5_std": 0.08283508024988509, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5994786803610277, + "nauc_precision_at_100_max": 0.4174447556800509, + "nauc_precision_at_100_std": 0.7542600373482679, + "nauc_precision_at_10_diff1": 0.606500672323457, + "nauc_precision_at_10_max": 0.4121852830713588, + "nauc_precision_at_10_std": 0.1379932303982931, + "nauc_precision_at_1_diff1": 0.7065975273128384, + "nauc_precision_at_1_max": 0.4003963436889161, + "nauc_precision_at_1_std": 0.05878106742438773, + "nauc_precision_at_20_diff1": 0.5701204035144817, + "nauc_precision_at_20_max": 0.42189391474129495, + "nauc_precision_at_20_std": 0.23806703547022473, + "nauc_precision_at_3_diff1": 0.6430713837080028, + "nauc_precision_at_3_max": 0.38697607555864566, + "nauc_precision_at_3_std": 0.14066147402503643, + "nauc_precision_at_5_diff1": 0.6101338410105439, + "nauc_precision_at_5_max": 0.3596857211946276, + "nauc_precision_at_5_std": 0.10059762103073269, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5994786803610316, + "nauc_recall_at_100_max": 0.4174447556800473, + "nauc_recall_at_100_std": 0.7542600373482705, + "nauc_recall_at_10_diff1": 0.6065006723234561, + "nauc_recall_at_10_max": 0.41218528307135877, + "nauc_recall_at_10_std": 0.13799323039829378, + "nauc_recall_at_1_diff1": 0.7065975273128384, + "nauc_recall_at_1_max": 0.4003963436889161, + "nauc_recall_at_1_std": 0.05878106742438773, + "nauc_recall_at_20_diff1": 0.5701204035144813, + "nauc_recall_at_20_max": 0.4218939147412955, + "nauc_recall_at_20_std": 0.23806703547022504, + "nauc_recall_at_3_diff1": 0.643071383708003, + "nauc_recall_at_3_max": 0.3869760755586455, + "nauc_recall_at_3_std": 0.140661474025036, + "nauc_recall_at_5_diff1": 0.6101338410105434, + "nauc_recall_at_5_max": 0.3596857211946283, + "nauc_recall_at_5_std": 0.1005976210307328, + "ndcg_at_1": 0.514, + "ndcg_at_10": 0.64427, + "ndcg_at_100": 0.68104, + "ndcg_at_1000": 0.68721, + "ndcg_at_20": 0.66194, + "ndcg_at_3": 0.60231, + "ndcg_at_5": 0.62564, + "precision_at_1": 0.514, + "precision_at_10": 0.078, + "precision_at_100": 0.00952, + "precision_at_1000": 0.001, + "precision_at_20": 0.0425, + "precision_at_3": 0.222, + "precision_at_5": 0.1444, + "recall_at_1": 0.514, + "recall_at_10": 0.78, + "recall_at_100": 0.952, + "recall_at_1000": 1.0, + "recall_at_20": 0.85, + "recall_at_3": 0.666, + "recall_at_5": 0.722 + } + ] + }, + "task_name": "VidoreInfoVQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreShiftProjectRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..1217f26233 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", + "evaluation_time": 36.15191340446472, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.11, + "cv_recall_at_10": 0.31, + "cv_recall_at_100": 0.59, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.37, + "cv_recall_at_3": 0.18, + "cv_recall_at_5": 0.24, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.17302, + "map_at_1": 0.11, + "map_at_10": 0.16052, + "map_at_100": 0.16994, + "map_at_1000": 0.17162, + "map_at_20": 0.16487, + "map_at_3": 0.13833, + "map_at_5": 0.15133, + "mrr_at_1": 0.11, + "mrr_at_10": 0.1605238095238095, + "mrr_at_100": 0.16993682614552097, + "mrr_at_1000": 0.171616148007682, + "mrr_at_20": 0.16487272857861085, + "mrr_at_3": 0.13833333333333334, + "mrr_at_5": 0.15133333333333332, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.09664401954598185, + "nauc_cv_recall_at_100_max": 0.2321892379708571, + "nauc_cv_recall_at_100_std": 0.29961700441692257, + "nauc_cv_recall_at_10_diff1": 0.08412546889495404, + "nauc_cv_recall_at_10_max": 0.21329174341530308, + "nauc_cv_recall_at_10_std": 0.15528994178307814, + "nauc_cv_recall_at_1_diff1": 0.11281677026448891, + "nauc_cv_recall_at_1_max": 0.3319324558975262, + "nauc_cv_recall_at_1_std": 0.26501748999672853, + "nauc_cv_recall_at_20_diff1": -0.002768382977056451, + "nauc_cv_recall_at_20_max": 0.14356416249885756, + "nauc_cv_recall_at_20_std": 0.08242468561877275, + "nauc_cv_recall_at_3_diff1": 0.11991720169806679, + "nauc_cv_recall_at_3_max": 0.3230712556572991, + "nauc_cv_recall_at_3_std": 0.3017752517278882, + "nauc_cv_recall_at_5_diff1": 0.07033309369757972, + "nauc_cv_recall_at_5_max": 0.2757458662832493, + "nauc_cv_recall_at_5_std": 0.2587766594775941, + "nauc_map_at_1000_diff1": 0.07924737253621075, + "nauc_map_at_1000_max": 0.2907190868329696, + "nauc_map_at_1000_std": 0.24198829664232294, + "nauc_map_at_100_diff1": 0.07793045130849639, + "nauc_map_at_100_max": 0.29059223848457494, + "nauc_map_at_100_std": 0.24230159395580023, + "nauc_map_at_10_diff1": 0.0896249074215925, + "nauc_map_at_10_max": 0.29429957243051497, + "nauc_map_at_10_std": 0.24209507427102459, + "nauc_map_at_1_diff1": 0.11281677026448891, + "nauc_map_at_1_max": 0.3319324558975262, + "nauc_map_at_1_std": 0.26501748999672853, + "nauc_map_at_20_diff1": 0.08024968159849469, + "nauc_map_at_20_max": 0.2872360074786645, + "nauc_map_at_20_std": 0.23456233639857504, + "nauc_map_at_3_diff1": 0.10023862092088413, + "nauc_map_at_3_max": 0.3218243621616418, + "nauc_map_at_3_std": 0.2780094235646185, + "nauc_map_at_5_diff1": 0.08321524099693559, + "nauc_map_at_5_max": 0.30548488940425605, + "nauc_map_at_5_std": 0.26643845322657783, + "nauc_mrr_at_1000_diff1": 0.07924737253621075, + "nauc_mrr_at_1000_max": 0.2907190868329696, + "nauc_mrr_at_1000_std": 0.24198829664232294, + "nauc_mrr_at_100_diff1": 0.07793045130849639, + "nauc_mrr_at_100_max": 0.29059223848457494, + "nauc_mrr_at_100_std": 0.24230159395580023, + "nauc_mrr_at_10_diff1": 0.0896249074215925, + "nauc_mrr_at_10_max": 0.29429957243051497, + "nauc_mrr_at_10_std": 0.24209507427102459, + "nauc_mrr_at_1_diff1": 0.11281677026448891, + "nauc_mrr_at_1_max": 0.3319324558975262, + "nauc_mrr_at_1_std": 0.26501748999672853, + "nauc_mrr_at_20_diff1": 0.08024968159849469, + "nauc_mrr_at_20_max": 0.2872360074786645, + "nauc_mrr_at_20_std": 0.23456233639857504, + "nauc_mrr_at_3_diff1": 0.10023862092088413, + "nauc_mrr_at_3_max": 0.3218243621616418, + "nauc_mrr_at_3_std": 0.2780094235646185, + "nauc_mrr_at_5_diff1": 0.08321524099693559, + "nauc_mrr_at_5_max": 0.30548488940425605, + "nauc_mrr_at_5_std": 0.26643845322657783, + "nauc_ndcg_at_1000_diff1": 0.06615008028731253, + "nauc_ndcg_at_1000_max": 0.2748275557288526, + "nauc_ndcg_at_1000_std": 0.23349999155014844, + "nauc_ndcg_at_100_diff1": 0.038763196864327186, + "nauc_ndcg_at_100_max": 0.26890993061914903, + "nauc_ndcg_at_100_std": 0.2432662566601855, + "nauc_ndcg_at_10_diff1": 0.08806371511158409, + "nauc_ndcg_at_10_max": 0.2702528801580752, + "nauc_ndcg_at_10_std": 0.21641774547281548, + "nauc_ndcg_at_1_diff1": 0.11281677026448891, + "nauc_ndcg_at_1_max": 0.3319324558975262, + "nauc_ndcg_at_1_std": 0.26501748999672853, + "nauc_ndcg_at_20_diff1": 0.0580246437643237, + "nauc_ndcg_at_20_max": 0.24637629255838372, + "nauc_ndcg_at_20_std": 0.19127929130989058, + "nauc_ndcg_at_3_diff1": 0.10541266833562696, + "nauc_ndcg_at_3_max": 0.3219743407161455, + "nauc_ndcg_at_3_std": 0.28479647710896433, + "nauc_ndcg_at_5_diff1": 0.07896740745121697, + "nauc_ndcg_at_5_max": 0.29633957889754337, + "nauc_ndcg_at_5_std": 0.26451689568166686, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": -0.0966440195459817, + "nauc_precision_at_100_max": 0.23218923797085725, + "nauc_precision_at_100_std": 0.2996170044169224, + "nauc_precision_at_10_diff1": 0.08412546889495406, + "nauc_precision_at_10_max": 0.21329174341530296, + "nauc_precision_at_10_std": 0.15528994178307795, + "nauc_precision_at_1_diff1": 0.11281677026448891, + "nauc_precision_at_1_max": 0.3319324558975262, + "nauc_precision_at_1_std": 0.26501748999672853, + "nauc_precision_at_20_diff1": -0.0027683829770564048, + "nauc_precision_at_20_max": 0.14356416249885745, + "nauc_precision_at_20_std": 0.082424685618773, + "nauc_precision_at_3_diff1": 0.11991720169806704, + "nauc_precision_at_3_max": 0.32307125565729955, + "nauc_precision_at_3_std": 0.3017752517278884, + "nauc_precision_at_5_diff1": 0.07033309369757954, + "nauc_precision_at_5_max": 0.27574586628324943, + "nauc_precision_at_5_std": 0.2587766594775939, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.09664401954598185, + "nauc_recall_at_100_max": 0.2321892379708571, + "nauc_recall_at_100_std": 0.29961700441692257, + "nauc_recall_at_10_diff1": 0.08412546889495404, + "nauc_recall_at_10_max": 0.21329174341530308, + "nauc_recall_at_10_std": 0.15528994178307814, + "nauc_recall_at_1_diff1": 0.11281677026448891, + "nauc_recall_at_1_max": 0.3319324558975262, + "nauc_recall_at_1_std": 0.26501748999672853, + "nauc_recall_at_20_diff1": -0.002768382977056451, + "nauc_recall_at_20_max": 0.14356416249885756, + "nauc_recall_at_20_std": 0.08242468561877275, + "nauc_recall_at_3_diff1": 0.11991720169806679, + "nauc_recall_at_3_max": 0.3230712556572991, + "nauc_recall_at_3_std": 0.3017752517278882, + "nauc_recall_at_5_diff1": 0.07033309369757972, + "nauc_recall_at_5_max": 0.2757458662832493, + "nauc_recall_at_5_std": 0.2587766594775941, + "ndcg_at_1": 0.11, + "ndcg_at_10": 0.19548, + "ndcg_at_100": 0.25056, + "ndcg_at_1000": 0.30114, + "ndcg_at_20": 0.21089, + "ndcg_at_3": 0.14893, + "ndcg_at_5": 0.17302, + "precision_at_1": 0.11, + "precision_at_10": 0.031, + "precision_at_100": 0.0059, + "precision_at_1000": 0.001, + "precision_at_20": 0.0185, + "precision_at_3": 0.06, + "precision_at_5": 0.048, + "recall_at_1": 0.11, + "recall_at_10": 0.31, + "recall_at_100": 0.59, + "recall_at_1000": 1.0, + "recall_at_20": 0.37, + "recall_at_3": 0.18, + "recall_at_5": 0.24 + } + ] + }, + "task_name": "VidoreShiftProjectRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAAIRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..34b30646b3 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", + "evaluation_time": 34.92671084403992, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.22, + "cv_recall_at_10": 0.53, + "cv_recall_at_100": 0.88, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.61, + "cv_recall_at_3": 0.39, + "cv_recall_at_5": 0.44, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.34875, + "map_at_1": 0.24, + "map_at_10": 0.33169, + "map_at_100": 0.34394, + "map_at_1000": 0.34446, + "map_at_20": 0.33755, + "map_at_3": 0.30667, + "map_at_5": 0.31817, + "mrr_at_1": 0.22, + "mrr_at_10": 0.3172222222222222, + "mrr_at_100": 0.3304223584351638, + "mrr_at_1000": 0.3309363490911448, + "mrr_at_20": 0.32343802396433974, + "mrr_at_3": 0.29333333333333333, + "mrr_at_5": 0.3048333333333333, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.25588609608654145, + "nauc_cv_recall_at_100_max": 0.34254692968501516, + "nauc_cv_recall_at_100_std": -0.030862233534839804, + "nauc_cv_recall_at_10_diff1": 0.015945236360637088, + "nauc_cv_recall_at_10_max": 0.5467978941291292, + "nauc_cv_recall_at_10_std": 0.1269708174682812, + "nauc_cv_recall_at_1_diff1": 0.2607763309196009, + "nauc_cv_recall_at_1_max": 0.31143363271902275, + "nauc_cv_recall_at_1_std": -0.02879385896758557, + "nauc_cv_recall_at_20_diff1": 0.0002417174021419322, + "nauc_cv_recall_at_20_max": 0.5049627604127324, + "nauc_cv_recall_at_20_std": 0.06206094300001453, + "nauc_cv_recall_at_3_diff1": -0.01054156480798838, + "nauc_cv_recall_at_3_max": 0.5327823263875415, + "nauc_cv_recall_at_3_std": -0.0017849520779171265, + "nauc_cv_recall_at_5_diff1": 0.09731429764576903, + "nauc_cv_recall_at_5_max": 0.5683040141984653, + "nauc_cv_recall_at_5_std": 0.06458474708983654, + "nauc_map_at_1000_diff1": 0.10443462676821944, + "nauc_map_at_1000_max": 0.4090433812678768, + "nauc_map_at_1000_std": -0.016759060369677045, + "nauc_map_at_100_diff1": 0.10470542206159071, + "nauc_map_at_100_max": 0.4090673255951132, + "nauc_map_at_100_std": -0.016784069113126018, + "nauc_map_at_10_diff1": 0.10170960093274041, + "nauc_map_at_10_max": 0.41526994097301995, + "nauc_map_at_10_std": -0.013590212809351342, + "nauc_map_at_1_diff1": 0.19137910376228143, + "nauc_map_at_1_max": 0.28789240354660905, + "nauc_map_at_1_std": -0.04003414809489571, + "nauc_map_at_20_diff1": 0.10300773263289635, + "nauc_map_at_20_max": 0.4121548149249788, + "nauc_map_at_20_std": -0.0161035883846474, + "nauc_map_at_3_diff1": 0.09119254288953736, + "nauc_map_at_3_max": 0.4142859798145018, + "nauc_map_at_3_std": -0.027759707068641875, + "nauc_map_at_5_diff1": 0.1003239990473492, + "nauc_map_at_5_max": 0.4226964161286666, + "nauc_map_at_5_std": -0.024585700506800114, + "nauc_mrr_at_1000_diff1": 0.13095044354935767, + "nauc_mrr_at_1000_max": 0.4088406531555248, + "nauc_mrr_at_1000_std": -0.006095326008767425, + "nauc_mrr_at_100_diff1": 0.13118474764897708, + "nauc_mrr_at_100_max": 0.4088652545299387, + "nauc_mrr_at_100_std": -0.0061360067675142686, + "nauc_mrr_at_10_diff1": 0.1271792759098105, + "nauc_mrr_at_10_max": 0.4164808103965064, + "nauc_mrr_at_10_std": -0.0011619168678242003, + "nauc_mrr_at_1_diff1": 0.2607763309196009, + "nauc_mrr_at_1_max": 0.31143363271902275, + "nauc_mrr_at_1_std": -0.02879385896758557, + "nauc_mrr_at_20_diff1": 0.1273336734023317, + "nauc_mrr_at_20_max": 0.4119379927819901, + "nauc_mrr_at_20_std": -0.006324913135721497, + "nauc_mrr_at_3_diff1": 0.11325609802287717, + "nauc_mrr_at_3_max": 0.4132688811453676, + "nauc_mrr_at_3_std": -0.020022157412317262, + "nauc_mrr_at_5_diff1": 0.13761856788427215, + "nauc_mrr_at_5_max": 0.4200430161805268, + "nauc_mrr_at_5_std": -0.006370825854209519, + "nauc_ndcg_at_1000_diff1": 0.09617174046670891, + "nauc_ndcg_at_1000_max": 0.4203264186005738, + "nauc_ndcg_at_1000_std": -0.0068412291888518975, + "nauc_ndcg_at_100_diff1": 0.10442963683136622, + "nauc_ndcg_at_100_max": 0.4169754620603158, + "nauc_ndcg_at_100_std": -0.007949442426679318, + "nauc_ndcg_at_10_diff1": 0.07882991025385962, + "nauc_ndcg_at_10_max": 0.4457128887951948, + "nauc_ndcg_at_10_std": 0.013698345756966162, + "nauc_ndcg_at_1_diff1": 0.19137910376228143, + "nauc_ndcg_at_1_max": 0.28789240354660905, + "nauc_ndcg_at_1_std": -0.04003414809489571, + "nauc_ndcg_at_20_diff1": 0.0853973025330627, + "nauc_ndcg_at_20_max": 0.4381885352579316, + "nauc_ndcg_at_20_std": 0.005313883337564515, + "nauc_ndcg_at_3_diff1": 0.06342948608957481, + "nauc_ndcg_at_3_max": 0.44683332935967496, + "nauc_ndcg_at_3_std": -0.020959173735207975, + "nauc_ndcg_at_5_diff1": 0.0800650733044633, + "nauc_ndcg_at_5_max": 0.463683639664684, + "nauc_ndcg_at_5_std": -0.014099286510659888, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.25588609608654206, + "nauc_precision_at_100_max": 0.34254692968501455, + "nauc_precision_at_100_std": -0.03086223353483784, + "nauc_precision_at_10_diff1": 0.012461448695507674, + "nauc_precision_at_10_max": 0.5267566891722935, + "nauc_precision_at_10_std": 0.10683226362146125, + "nauc_precision_at_1_diff1": 0.19137910376228143, + "nauc_precision_at_1_max": 0.28789240354660905, + "nauc_precision_at_1_std": -0.04003414809489571, + "nauc_precision_at_20_diff1": 0.03819647417161561, + "nauc_precision_at_20_max": 0.5101375257668525, + "nauc_precision_at_20_std": 0.08176168353690481, + "nauc_precision_at_3_diff1": -0.01054156480798847, + "nauc_precision_at_3_max": 0.5327823263875415, + "nauc_precision_at_3_std": -0.001784952077917081, + "nauc_precision_at_5_diff1": 0.028488281046093265, + "nauc_precision_at_5_max": 0.5765255520175396, + "nauc_precision_at_5_std": 0.016899827739208093, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.25588609608654145, + "nauc_recall_at_100_max": 0.34254692968501516, + "nauc_recall_at_100_std": -0.030862233534839804, + "nauc_recall_at_10_diff1": 0.012461448695507143, + "nauc_recall_at_10_max": 0.5267566891722932, + "nauc_recall_at_10_std": 0.10683226362146066, + "nauc_recall_at_1_diff1": 0.19137910376228143, + "nauc_recall_at_1_max": 0.28789240354660905, + "nauc_recall_at_1_std": -0.04003414809489571, + "nauc_recall_at_20_diff1": 0.03819647417161486, + "nauc_recall_at_20_max": 0.5101375257668519, + "nauc_recall_at_20_std": 0.08176168353690391, + "nauc_recall_at_3_diff1": -0.01054156480798838, + "nauc_recall_at_3_max": 0.5327823263875415, + "nauc_recall_at_3_std": -0.0017849520779171265, + "nauc_recall_at_5_diff1": 0.028488281046092762, + "nauc_recall_at_5_max": 0.5765255520175391, + "nauc_recall_at_5_std": 0.0168998277392075, + "ndcg_at_1": 0.24, + "ndcg_at_10": 0.38125, + "ndcg_at_100": 0.44966, + "ndcg_at_1000": 0.4647, + "ndcg_at_20": 0.40182, + "ndcg_at_3": 0.32809, + "ndcg_at_5": 0.34875, + "precision_at_1": 0.24, + "precision_at_10": 0.054, + "precision_at_100": 0.0088, + "precision_at_1000": 0.001, + "precision_at_20": 0.031, + "precision_at_3": 0.13, + "precision_at_5": 0.088, + "recall_at_1": 0.24, + "recall_at_10": 0.54, + "recall_at_100": 0.88, + "recall_at_1000": 1.0, + "recall_at_20": 0.62, + "recall_at_3": 0.39, + "recall_at_5": 0.44 + } + ] + }, + "task_name": "VidoreSyntheticDocQAAIRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAEnergyRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..ab864d679f --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", + "evaluation_time": 34.53333258628845, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.37, + "cv_recall_at_10": 0.76, + "cv_recall_at_100": 0.95, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.83, + "cv_recall_at_3": 0.64, + "cv_recall_at_5": 0.68, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.53058, + "map_at_1": 0.32, + "map_at_10": 0.4915, + "map_at_100": 0.4978, + "map_at_1000": 0.4981, + "map_at_20": 0.49541, + "map_at_3": 0.46667, + "map_at_5": 0.47917, + "mrr_at_1": 0.37, + "mrr_at_10": 0.5126746031746032, + "mrr_at_100": 0.5198506079345971, + "mrr_at_1000": 0.5201440447016661, + "mrr_at_20": 0.5174591278604437, + "mrr_at_3": 0.49166666666666664, + "mrr_at_5": 0.5011666666666666, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.21447245564892614, + "nauc_cv_recall_at_100_max": 0.30924369747899044, + "nauc_cv_recall_at_100_std": 0.4916900093370688, + "nauc_cv_recall_at_10_diff1": -0.03153932681491796, + "nauc_cv_recall_at_10_max": 0.47771913126243837, + "nauc_cv_recall_at_10_std": -0.2724971878515191, + "nauc_cv_recall_at_1_diff1": 0.13080609566591367, + "nauc_cv_recall_at_1_max": 0.2643805743088836, + "nauc_cv_recall_at_1_std": -0.02580341869180856, + "nauc_cv_recall_at_20_diff1": -0.19083415568584217, + "nauc_cv_recall_at_20_max": 0.34146978580135867, + "nauc_cv_recall_at_20_std": -0.23689557090613594, + "nauc_cv_recall_at_3_diff1": -0.1465704432239113, + "nauc_cv_recall_at_3_max": 0.3345095159024136, + "nauc_cv_recall_at_3_std": -0.2043045088772514, + "nauc_cv_recall_at_5_diff1": -0.10203164950590858, + "nauc_cv_recall_at_5_max": 0.3911270817497059, + "nauc_cv_recall_at_5_std": -0.19333494575357663, + "nauc_map_at_1000_diff1": 0.12011397834904365, + "nauc_map_at_1000_max": 0.28323827692841, + "nauc_map_at_1000_std": -0.1905001367686242, + "nauc_map_at_100_diff1": 0.11966738540821883, + "nauc_map_at_100_max": 0.28323118429537103, + "nauc_map_at_100_std": -0.1898627518331248, + "nauc_map_at_10_diff1": 0.12065719889856055, + "nauc_map_at_10_max": 0.29021890060830574, + "nauc_map_at_10_std": -0.19420513505740164, + "nauc_map_at_1_diff1": 0.2819891897677404, + "nauc_map_at_1_max": 0.2161778871882692, + "nauc_map_at_1_std": -0.16618056298833367, + "nauc_map_at_20_diff1": 0.11805338930333743, + "nauc_map_at_20_max": 0.2847181951397561, + "nauc_map_at_20_std": -0.1908796432928892, + "nauc_map_at_3_diff1": 0.09382723677706396, + "nauc_map_at_3_max": 0.27825677782425196, + "nauc_map_at_3_std": -0.19836780866192627, + "nauc_map_at_5_diff1": 0.10911512170369399, + "nauc_map_at_5_max": 0.28478822185507247, + "nauc_map_at_5_std": -0.18782586739147047, + "nauc_mrr_at_1000_diff1": 0.010318386026540968, + "nauc_mrr_at_1000_max": 0.28476023988898364, + "nauc_mrr_at_1000_std": -0.11479441647303681, + "nauc_mrr_at_100_diff1": 0.009982532144116743, + "nauc_mrr_at_100_max": 0.2847509614579322, + "nauc_mrr_at_100_std": -0.11424005887976545, + "nauc_mrr_at_10_diff1": 0.01425407904727165, + "nauc_mrr_at_10_max": 0.29299876743388203, + "nauc_mrr_at_10_std": -0.11910098135182312, + "nauc_mrr_at_1_diff1": 0.13080609566591367, + "nauc_mrr_at_1_max": 0.2643805743088836, + "nauc_mrr_at_1_std": -0.02580341869180856, + "nauc_mrr_at_20_diff1": 0.00914945533225785, + "nauc_mrr_at_20_max": 0.2862026177193902, + "nauc_mrr_at_20_std": -0.11575968447469559, + "nauc_mrr_at_3_diff1": -0.006611838316696035, + "nauc_mrr_at_3_max": 0.2755136338315525, + "nauc_mrr_at_3_std": -0.11688827568051624, + "nauc_mrr_at_5_diff1": 0.004146095220944025, + "nauc_mrr_at_5_max": 0.2862054857241672, + "nauc_mrr_at_5_std": -0.11479469794696757, + "nauc_ndcg_at_1000_diff1": 0.09693517981423239, + "nauc_ndcg_at_1000_max": 0.2911411346799712, + "nauc_ndcg_at_1000_std": -0.19237592779877516, + "nauc_ndcg_at_100_diff1": 0.08831100693949918, + "nauc_ndcg_at_100_max": 0.29170420800681346, + "nauc_ndcg_at_100_std": -0.17476915296633083, + "nauc_ndcg_at_10_diff1": 0.08365375220406525, + "nauc_ndcg_at_10_max": 0.32139912966631373, + "nauc_ndcg_at_10_std": -0.20956662213502888, + "nauc_ndcg_at_1_diff1": 0.2819891897677404, + "nauc_ndcg_at_1_max": 0.2161778871882692, + "nauc_ndcg_at_1_std": -0.16618056298833367, + "nauc_ndcg_at_20_diff1": 0.07085280370170333, + "nauc_ndcg_at_20_max": 0.29872845359082356, + "nauc_ndcg_at_20_std": -0.197926152444139, + "nauc_ndcg_at_3_diff1": 0.03496567870813315, + "nauc_ndcg_at_3_max": 0.29633703404607786, + "nauc_ndcg_at_3_std": -0.20824883400477262, + "nauc_ndcg_at_5_diff1": 0.06009325276331838, + "nauc_ndcg_at_5_max": 0.30874738372680843, + "nauc_ndcg_at_5_std": -0.18973188186410675, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": -0.21447245564892456, + "nauc_precision_at_100_max": 0.30924369747899483, + "nauc_precision_at_100_std": 0.491690009337072, + "nauc_precision_at_10_diff1": -0.059197421722878354, + "nauc_precision_at_10_max": 0.45963608692732916, + "nauc_precision_at_10_std": -0.28781808822541777, + "nauc_precision_at_1_diff1": 0.2819891897677404, + "nauc_precision_at_1_max": 0.2161778871882692, + "nauc_precision_at_1_std": -0.16618056298833367, + "nauc_precision_at_20_diff1": -0.19083415568583906, + "nauc_precision_at_20_max": 0.3414697858013586, + "nauc_precision_at_20_std": -0.23689557090613314, + "nauc_precision_at_3_diff1": -0.14959494821291394, + "nauc_precision_at_3_max": 0.35252824393989307, + "nauc_precision_at_3_std": -0.23916075150033597, + "nauc_precision_at_5_diff1": -0.10203164950590708, + "nauc_precision_at_5_max": 0.3911270817497069, + "nauc_precision_at_5_std": -0.1933349457535751, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.21447245564892614, + "nauc_recall_at_100_max": 0.30924369747899044, + "nauc_recall_at_100_std": 0.4916900093370688, + "nauc_recall_at_10_diff1": -0.059197421722880034, + "nauc_recall_at_10_max": 0.4596360869273292, + "nauc_recall_at_10_std": -0.28781808822542065, + "nauc_recall_at_1_diff1": 0.2819891897677404, + "nauc_recall_at_1_max": 0.2161778871882692, + "nauc_recall_at_1_std": -0.16618056298833367, + "nauc_recall_at_20_diff1": -0.19083415568584217, + "nauc_recall_at_20_max": 0.34146978580135867, + "nauc_recall_at_20_std": -0.23689557090613594, + "nauc_recall_at_3_diff1": -0.1495949482129152, + "nauc_recall_at_3_max": 0.3525282439398921, + "nauc_recall_at_3_std": -0.23916075150033683, + "nauc_recall_at_5_diff1": -0.10203164950590858, + "nauc_recall_at_5_max": 0.3911270817497059, + "nauc_recall_at_5_std": -0.19333494575357663, + "ndcg_at_1": 0.32, + "ndcg_at_10": 0.55999, + "ndcg_at_100": 0.59578, + "ndcg_at_1000": 0.6024, + "ndcg_at_20": 0.57487, + "ndcg_at_3": 0.50904, + "ndcg_at_5": 0.53058, + "precision_at_1": 0.32, + "precision_at_10": 0.077, + "precision_at_100": 0.0095, + "precision_at_1000": 0.001, + "precision_at_20": 0.0415, + "precision_at_3": 0.21, + "precision_at_5": 0.136, + "recall_at_1": 0.32, + "recall_at_10": 0.77, + "recall_at_100": 0.95, + "recall_at_1000": 1.0, + "recall_at_20": 0.83, + "recall_at_3": 0.63, + "recall_at_5": 0.68 + } + ] + }, + "task_name": "VidoreSyntheticDocQAEnergyRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..b846e73a5f --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", + "evaluation_time": 35.05716419219971, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.29, + "cv_recall_at_10": 0.68, + "cv_recall_at_100": 0.92, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.77, + "cv_recall_at_3": 0.59, + "cv_recall_at_5": 0.61, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.47815, + "map_at_1": 0.29, + "map_at_10": 0.43789, + "map_at_100": 0.44824, + "map_at_1000": 0.44875, + "map_at_20": 0.44399, + "map_at_3": 0.41833, + "map_at_5": 0.42983, + "mrr_at_1": 0.29, + "mrr_at_10": 0.4382341269841271, + "mrr_at_100": 0.4484619359551061, + "mrr_at_1000": 0.44896652588458674, + "mrr_at_20": 0.4437531526780754, + "mrr_at_3": 0.42333333333333345, + "mrr_at_5": 0.42833333333333345, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.14869281045751423, + "nauc_cv_recall_at_100_max": 0.21381886087768426, + "nauc_cv_recall_at_100_std": 0.2429971988795516, + "nauc_cv_recall_at_10_diff1": 0.08117960058047077, + "nauc_cv_recall_at_10_max": 0.5350701402805611, + "nauc_cv_recall_at_10_std": 0.12226176490912774, + "nauc_cv_recall_at_1_diff1": 0.19073389832836563, + "nauc_cv_recall_at_1_max": 0.19907146683515795, + "nauc_cv_recall_at_1_std": -0.10344353941458466, + "nauc_cv_recall_at_20_diff1": 0.1277948121125314, + "nauc_cv_recall_at_20_max": 0.45941227814954944, + "nauc_cv_recall_at_20_std": 0.01828517714464683, + "nauc_cv_recall_at_3_diff1": 0.07260774502179126, + "nauc_cv_recall_at_3_max": 0.49208329053369904, + "nauc_cv_recall_at_3_std": -0.014791553553347924, + "nauc_cv_recall_at_5_diff1": 0.07505325336515935, + "nauc_cv_recall_at_5_max": 0.5005665251612711, + "nauc_cv_recall_at_5_std": 0.04027616213194748, + "nauc_map_at_1000_diff1": 0.11877256344149041, + "nauc_map_at_1000_max": 0.2720490218954375, + "nauc_map_at_1000_std": -0.0901853515205092, + "nauc_map_at_100_diff1": 0.11885317591762472, + "nauc_map_at_100_max": 0.2721652636804168, + "nauc_map_at_100_std": -0.08947323825938826, + "nauc_map_at_10_diff1": 0.1172708279300533, + "nauc_map_at_10_max": 0.282532047752819, + "nauc_map_at_10_std": -0.08414817563361583, + "nauc_map_at_1_diff1": 0.20611787975770224, + "nauc_map_at_1_max": 0.12768704586349439, + "nauc_map_at_1_std": -0.19106355507328007, + "nauc_map_at_20_diff1": 0.12091348717143217, + "nauc_map_at_20_max": 0.27613935786590416, + "nauc_map_at_20_std": -0.08964440713899123, + "nauc_map_at_3_diff1": 0.09281666092911399, + "nauc_map_at_3_max": 0.27717924962051305, + "nauc_map_at_3_std": -0.09299918442034921, + "nauc_map_at_5_diff1": 0.12036062962477545, + "nauc_map_at_5_max": 0.2796734132716185, + "nauc_map_at_5_std": -0.09220654472338634, + "nauc_mrr_at_1000_diff1": 0.12026493721452002, + "nauc_mrr_at_1000_max": 0.2935938299086023, + "nauc_mrr_at_1000_std": -0.06270575154374189, + "nauc_mrr_at_100_diff1": 0.1203394458874957, + "nauc_mrr_at_100_max": 0.2936737480473231, + "nauc_mrr_at_100_std": -0.06204189363139713, + "nauc_mrr_at_10_diff1": 0.11879442086655934, + "nauc_mrr_at_10_max": 0.3035665197689515, + "nauc_mrr_at_10_std": -0.05735455948583704, + "nauc_mrr_at_1_diff1": 0.19073389832836563, + "nauc_mrr_at_1_max": 0.19907146683515795, + "nauc_mrr_at_1_std": -0.10344353941458466, + "nauc_mrr_at_20_diff1": 0.12052296633532071, + "nauc_mrr_at_20_max": 0.2972690785119719, + "nauc_mrr_at_20_std": -0.06412275539703116, + "nauc_mrr_at_3_diff1": 0.11476782259264771, + "nauc_mrr_at_3_max": 0.30347475442774224, + "nauc_mrr_at_3_std": -0.0737158617353772, + "nauc_mrr_at_5_diff1": 0.11562569511923856, + "nauc_mrr_at_5_max": 0.3041525691607059, + "nauc_mrr_at_5_std": -0.06131800461266482, + "nauc_ndcg_at_1000_diff1": 0.11377699350015925, + "nauc_ndcg_at_1000_max": 0.28899299562034736, + "nauc_ndcg_at_1000_std": -0.07328852794331638, + "nauc_ndcg_at_100_diff1": 0.11528044837168852, + "nauc_ndcg_at_100_max": 0.2870892189870652, + "nauc_ndcg_at_100_std": -0.05994758581865143, + "nauc_ndcg_at_10_diff1": 0.10730189616771577, + "nauc_ndcg_at_10_max": 0.33739176053782455, + "nauc_ndcg_at_10_std": -0.041229286834314025, + "nauc_ndcg_at_1_diff1": 0.20611787975770224, + "nauc_ndcg_at_1_max": 0.12768704586349439, + "nauc_ndcg_at_1_std": -0.19106355507328007, + "nauc_ndcg_at_20_diff1": 0.12519909516253155, + "nauc_ndcg_at_20_max": 0.3147205255038511, + "nauc_ndcg_at_20_std": -0.05945710494891262, + "nauc_ndcg_at_3_diff1": 0.06440362309001588, + "nauc_ndcg_at_3_max": 0.32086799317238685, + "nauc_ndcg_at_3_std": -0.06743828423819527, + "nauc_ndcg_at_5_diff1": 0.11596750866018518, + "nauc_ndcg_at_5_max": 0.3279860926092747, + "nauc_ndcg_at_5_std": -0.06638587402429165, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.14869281045751595, + "nauc_precision_at_100_max": 0.21381886087768318, + "nauc_precision_at_100_std": 0.2429971988795515, + "nauc_precision_at_10_diff1": 0.08117960058047159, + "nauc_precision_at_10_max": 0.5350701402805608, + "nauc_precision_at_10_std": 0.12226176490912925, + "nauc_precision_at_1_diff1": 0.20611787975770224, + "nauc_precision_at_1_max": 0.12768704586349439, + "nauc_precision_at_1_std": -0.19106355507328007, + "nauc_precision_at_20_diff1": 0.18957203134418324, + "nauc_precision_at_20_max": 0.46979181156396266, + "nauc_precision_at_20_std": 0.07613483562850694, + "nauc_precision_at_3_diff1": -0.017881488497581227, + "nauc_precision_at_3_max": 0.44996483976980833, + "nauc_precision_at_3_std": 0.006988992695283658, + "nauc_precision_at_5_diff1": 0.11437405777928256, + "nauc_precision_at_5_max": 0.4810632864658649, + "nauc_precision_at_5_std": 0.012368089099468108, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.14869281045751423, + "nauc_recall_at_100_max": 0.21381886087768426, + "nauc_recall_at_100_std": 0.2429971988795516, + "nauc_recall_at_10_diff1": 0.08117960058047077, + "nauc_recall_at_10_max": 0.5350701402805611, + "nauc_recall_at_10_std": 0.12226176490912774, + "nauc_recall_at_1_diff1": 0.20611787975770224, + "nauc_recall_at_1_max": 0.12768704586349439, + "nauc_recall_at_1_std": -0.19106355507328007, + "nauc_recall_at_20_diff1": 0.1895720313441832, + "nauc_recall_at_20_max": 0.46979181156396255, + "nauc_recall_at_20_std": 0.07613483562850668, + "nauc_recall_at_3_diff1": -0.017881488497581748, + "nauc_recall_at_3_max": 0.4499648397698083, + "nauc_recall_at_3_std": 0.006988992695282862, + "nauc_recall_at_5_diff1": 0.1143740577792816, + "nauc_recall_at_5_max": 0.4810632864658645, + "nauc_recall_at_5_std": 0.012368089099467303, + "ndcg_at_1": 0.29, + "ndcg_at_10": 0.49761, + "ndcg_at_100": 0.54922, + "ndcg_at_1000": 0.56002, + "ndcg_at_20": 0.52184, + "ndcg_at_3": 0.4575, + "ndcg_at_5": 0.47815, + "precision_at_1": 0.29, + "precision_at_10": 0.068, + "precision_at_100": 0.0092, + "precision_at_1000": 0.001, + "precision_at_20": 0.039, + "precision_at_3": 0.19, + "precision_at_5": 0.124, + "recall_at_1": 0.29, + "recall_at_10": 0.68, + "recall_at_100": 0.92, + "recall_at_1000": 1.0, + "recall_at_20": 0.78, + "recall_at_3": 0.57, + "recall_at_5": 0.62 + } + ] + }, + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..8570c3f0ee --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", + "evaluation_time": 33.74739336967468, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.29, + "cv_recall_at_10": 0.85, + "cv_recall_at_100": 0.93, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.87, + "cv_recall_at_3": 0.6, + "cv_recall_at_5": 0.72, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.52595, + "map_at_1": 0.31, + "map_at_10": 0.47932, + "map_at_100": 0.48276, + "map_at_1000": 0.48316, + "map_at_20": 0.48087, + "map_at_3": 0.43833, + "map_at_5": 0.46133, + "mrr_at_1": 0.29, + "mrr_at_10": 0.4757420634920635, + "mrr_at_100": 0.4791640089167549, + "mrr_at_1000": 0.4795730200133059, + "mrr_at_20": 0.47728968253968257, + "mrr_at_3": 0.43, + "mrr_at_5": 0.4575000000000001, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.19654528478058125, + "nauc_cv_recall_at_100_max": 0.42957182873149247, + "nauc_cv_recall_at_100_std": -0.07476323862878458, + "nauc_cv_recall_at_10_diff1": -0.2242434103481935, + "nauc_cv_recall_at_10_max": 0.44301985030914476, + "nauc_cv_recall_at_10_std": -0.05893263911487101, + "nauc_cv_recall_at_1_diff1": -0.13760421960633507, + "nauc_cv_recall_at_1_max": 0.07348598272049224, + "nauc_cv_recall_at_1_std": -0.00712882710877292, + "nauc_cv_recall_at_20_diff1": -0.30885729097237585, + "nauc_cv_recall_at_20_max": 0.4254239798563282, + "nauc_cv_recall_at_20_std": -0.1016811079019473, + "nauc_cv_recall_at_3_diff1": -0.3601513802315228, + "nauc_cv_recall_at_3_max": 0.33622736717126733, + "nauc_cv_recall_at_3_std": -0.06294152567527489, + "nauc_cv_recall_at_5_diff1": -0.39176968864468903, + "nauc_cv_recall_at_5_max": 0.27016559829059883, + "nauc_cv_recall_at_5_std": -0.07577838827838879, + "nauc_map_at_1000_diff1": -0.2638744951639972, + "nauc_map_at_1000_max": 0.16549033577792002, + "nauc_map_at_1000_std": -0.09178021864822189, + "nauc_map_at_100_diff1": -0.2638803838632255, + "nauc_map_at_100_max": 0.165875694571672, + "nauc_map_at_100_std": -0.09179531209394068, + "nauc_map_at_10_diff1": -0.2613495470274504, + "nauc_map_at_10_max": 0.16861705649237943, + "nauc_map_at_10_std": -0.08941924748102244, + "nauc_map_at_1_diff1": -0.1842908425991907, + "nauc_map_at_1_max": 0.04688277290022446, + "nauc_map_at_1_std": -0.09131855277841287, + "nauc_map_at_20_diff1": -0.2643001111188259, + "nauc_map_at_20_max": 0.16699166558456383, + "nauc_map_at_20_std": -0.0906144307284239, + "nauc_map_at_3_diff1": -0.2878495963843492, + "nauc_map_at_3_max": 0.17777726007291886, + "nauc_map_at_3_std": -0.0957122389311466, + "nauc_map_at_5_diff1": -0.2680322640463455, + "nauc_map_at_5_max": 0.1687211954163152, + "nauc_map_at_5_std": -0.0887669041098019, + "nauc_mrr_at_1000_diff1": -0.23448658723913543, + "nauc_mrr_at_1000_max": 0.18899168193132979, + "nauc_mrr_at_1000_std": -0.03492752674627976, + "nauc_mrr_at_100_diff1": -0.2345317236965136, + "nauc_mrr_at_100_max": 0.1893521317106688, + "nauc_mrr_at_100_std": -0.03501909631084891, + "nauc_mrr_at_10_diff1": -0.232273562005334, + "nauc_mrr_at_10_max": 0.19185458070516728, + "nauc_mrr_at_10_std": -0.033212636577323054, + "nauc_mrr_at_1_diff1": -0.13760421960633507, + "nauc_mrr_at_1_max": 0.07348598272049224, + "nauc_mrr_at_1_std": -0.00712882710877292, + "nauc_mrr_at_20_diff1": -0.23513440790266207, + "nauc_mrr_at_20_max": 0.1903108519693569, + "nauc_mrr_at_20_std": -0.034173102757696694, + "nauc_mrr_at_3_diff1": -0.24945724900105762, + "nauc_mrr_at_3_max": 0.20986215996182014, + "nauc_mrr_at_3_std": -0.03673862796290598, + "nauc_mrr_at_5_diff1": -0.2496203340274323, + "nauc_mrr_at_5_max": 0.18933274365636593, + "nauc_mrr_at_5_std": -0.03722532416877058, + "nauc_ndcg_at_1000_diff1": -0.26887041578092546, + "nauc_ndcg_at_1000_max": 0.18120174826484248, + "nauc_ndcg_at_1000_std": -0.09304740019552571, + "nauc_ndcg_at_100_diff1": -0.26678413669480144, + "nauc_ndcg_at_100_max": 0.1904171225816268, + "nauc_ndcg_at_100_std": -0.0923756902429794, + "nauc_ndcg_at_10_diff1": -0.2614009801076366, + "nauc_ndcg_at_10_max": 0.20542522951307385, + "nauc_ndcg_at_10_std": -0.08560121250285908, + "nauc_ndcg_at_1_diff1": -0.1842908425991907, + "nauc_ndcg_at_1_max": 0.04688277290022446, + "nauc_ndcg_at_1_std": -0.09131855277841287, + "nauc_ndcg_at_20_diff1": -0.2727678331984996, + "nauc_ndcg_at_20_max": 0.19931315260195256, + "nauc_ndcg_at_20_std": -0.09099972088577328, + "nauc_ndcg_at_3_diff1": -0.316753883236658, + "nauc_ndcg_at_3_max": 0.21041750335678436, + "nauc_ndcg_at_3_std": -0.09777860382526975, + "nauc_ndcg_at_5_diff1": -0.28017894422744943, + "nauc_ndcg_at_5_max": 0.1933173642946018, + "nauc_ndcg_at_5_std": -0.08364682820592939, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": -0.19654528478057492, + "nauc_precision_at_100_max": 0.42957182873149663, + "nauc_precision_at_100_std": -0.07476323862878233, + "nauc_precision_at_10_diff1": -0.2242434103481926, + "nauc_precision_at_10_max": 0.4430198503091455, + "nauc_precision_at_10_std": -0.058932639114869526, + "nauc_precision_at_1_diff1": -0.1842908425991907, + "nauc_precision_at_1_max": 0.04688277290022446, + "nauc_precision_at_1_std": -0.09131855277841287, + "nauc_precision_at_20_diff1": -0.3088572909723734, + "nauc_precision_at_20_max": 0.42542397985632724, + "nauc_precision_at_20_std": -0.1016811079019465, + "nauc_precision_at_3_diff1": -0.4065932375473025, + "nauc_precision_at_3_max": 0.3102790511645081, + "nauc_precision_at_3_std": -0.10454419592037588, + "nauc_precision_at_5_diff1": -0.3156478937728926, + "nauc_precision_at_5_max": 0.276175213675214, + "nauc_precision_at_5_std": -0.06089743589743516, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.19654528478058125, + "nauc_recall_at_100_max": 0.42957182873149247, + "nauc_recall_at_100_std": -0.07476323862878458, + "nauc_recall_at_10_diff1": -0.2242434103481935, + "nauc_recall_at_10_max": 0.44301985030914476, + "nauc_recall_at_10_std": -0.05893263911487101, + "nauc_recall_at_1_diff1": -0.1842908425991907, + "nauc_recall_at_1_max": 0.04688277290022446, + "nauc_recall_at_1_std": -0.09131855277841287, + "nauc_recall_at_20_diff1": -0.30885729097237585, + "nauc_recall_at_20_max": 0.4254239798563282, + "nauc_recall_at_20_std": -0.1016811079019473, + "nauc_recall_at_3_diff1": -0.4065932375473035, + "nauc_recall_at_3_max": 0.3102790511645078, + "nauc_recall_at_3_std": -0.10454419592037704, + "nauc_recall_at_5_diff1": -0.315647893772894, + "nauc_recall_at_5_max": 0.2761752136752141, + "nauc_recall_at_5_std": -0.06089743589743614, + "ndcg_at_1": 0.31, + "ndcg_at_10": 0.56863, + "ndcg_at_100": 0.58571, + "ndcg_at_1000": 0.59506, + "ndcg_at_20": 0.57389, + "ndcg_at_3": 0.48464, + "ndcg_at_5": 0.52595, + "precision_at_1": 0.31, + "precision_at_10": 0.085, + "precision_at_100": 0.0093, + "precision_at_1000": 0.001, + "precision_at_20": 0.0435, + "precision_at_3": 0.20667, + "precision_at_5": 0.144, + "recall_at_1": 0.31, + "recall_at_10": 0.85, + "recall_at_100": 0.93, + "recall_at_1000": 1.0, + "recall_at_20": 0.87, + "recall_at_3": 0.62, + "recall_at_5": 0.72 + } + ] + }, + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTabfquadRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..96af80c3d8 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", + "evaluation_time": 6.725373029708862, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.04643, + "cv_recall_at_10": 0.29643, + "cv_recall_at_100": 0.73929, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.4, + "cv_recall_at_3": 0.16071, + "cv_recall_at_5": 0.24286, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.14514, + "map_at_1": 0.05, + "map_at_10": 0.12193, + "map_at_100": 0.13702, + "map_at_1000": 0.1386, + "map_at_20": 0.1291, + "map_at_3": 0.09762, + "map_at_5": 0.11423, + "mrr_at_1": 0.04642857142857143, + "mrr_at_10": 0.12090561224489796, + "mrr_at_100": 0.1360089000569665, + "mrr_at_1000": 0.13759077042079312, + "mrr_at_20": 0.1280908305844728, + "mrr_at_3": 0.09464285714285713, + "mrr_at_5": 0.11374999999999996, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.12803173158202777, + "nauc_cv_recall_at_100_max": 0.0654788997984262, + "nauc_cv_recall_at_100_std": 0.4255586622450532, + "nauc_cv_recall_at_10_diff1": 0.014522791365928616, + "nauc_cv_recall_at_10_max": 0.3343785342270934, + "nauc_cv_recall_at_10_std": 0.2151285548560583, + "nauc_cv_recall_at_1_diff1": -0.08249571732265588, + "nauc_cv_recall_at_1_max": 0.5310313512620999, + "nauc_cv_recall_at_1_std": 0.2604487007492106, + "nauc_cv_recall_at_20_diff1": 0.00324591731646297, + "nauc_cv_recall_at_20_max": 0.25144824922760045, + "nauc_cv_recall_at_20_std": 0.23772436368986324, + "nauc_cv_recall_at_3_diff1": -0.0041901637103114115, + "nauc_cv_recall_at_3_max": 0.3307836904883966, + "nauc_cv_recall_at_3_std": 0.14303974845346734, + "nauc_cv_recall_at_5_diff1": 0.029390791630302882, + "nauc_cv_recall_at_5_max": 0.3490070031290577, + "nauc_cv_recall_at_5_std": 0.15756508227080204, + "nauc_map_at_1000_diff1": 0.07024249062149376, + "nauc_map_at_1000_max": 0.3532682565434852, + "nauc_map_at_1000_std": 0.11365370017831007, + "nauc_map_at_100_diff1": 0.07061643502220898, + "nauc_map_at_100_max": 0.3507764960731763, + "nauc_map_at_100_std": 0.11587548078896048, + "nauc_map_at_10_diff1": 0.07629220002560363, + "nauc_map_at_10_max": 0.36801613967093916, + "nauc_map_at_10_std": 0.10835899650696748, + "nauc_map_at_1_diff1": 0.13227797155889448, + "nauc_map_at_1_max": 0.3912185212158381, + "nauc_map_at_1_std": -0.04321744796657597, + "nauc_map_at_20_diff1": 0.067463263265385, + "nauc_map_at_20_max": 0.35562412424426926, + "nauc_map_at_20_std": 0.11282823760206927, + "nauc_map_at_3_diff1": 0.09211535320973883, + "nauc_map_at_3_max": 0.3902951639128921, + "nauc_map_at_3_std": 0.08363924252859689, + "nauc_map_at_5_diff1": 0.07955415447338117, + "nauc_map_at_5_max": 0.36905360018748185, + "nauc_map_at_5_std": 0.09305798236023993, + "nauc_mrr_at_1000_diff1": -0.0011983533381621932, + "nauc_mrr_at_1000_max": 0.3807218888272681, + "nauc_mrr_at_1000_std": 0.20305842835780713, + "nauc_mrr_at_100_diff1": -0.00023939715846331688, + "nauc_mrr_at_100_max": 0.37798336319707565, + "nauc_mrr_at_100_std": 0.20460984418849354, + "nauc_mrr_at_10_diff1": -0.0012782569765985967, + "nauc_mrr_at_10_max": 0.39359082264653156, + "nauc_mrr_at_10_std": 0.1979779592188029, + "nauc_mrr_at_1_diff1": -0.08249571732265588, + "nauc_mrr_at_1_max": 0.5310313512620999, + "nauc_mrr_at_1_std": 0.2604487007492106, + "nauc_mrr_at_20_diff1": -0.0021143734778184373, + "nauc_mrr_at_20_max": 0.3826280903094471, + "nauc_mrr_at_20_std": 0.20021923515610932, + "nauc_mrr_at_3_diff1": -0.017440468685185215, + "nauc_mrr_at_3_max": 0.3941611708358232, + "nauc_mrr_at_3_std": 0.1815424142915358, + "nauc_mrr_at_5_diff1": 0.0009224287977095485, + "nauc_mrr_at_5_max": 0.39848050580147903, + "nauc_mrr_at_5_std": 0.1819622237637238, + "nauc_ndcg_at_1000_diff1": 0.06292042013878293, + "nauc_ndcg_at_1000_max": 0.3324384230789723, + "nauc_ndcg_at_1000_std": 0.14549363875421784, + "nauc_ndcg_at_100_diff1": 0.07282815521201402, + "nauc_ndcg_at_100_max": 0.2898769800095464, + "nauc_ndcg_at_100_std": 0.18831946927410187, + "nauc_ndcg_at_10_diff1": 0.06810378455819563, + "nauc_ndcg_at_10_max": 0.3597482951290414, + "nauc_ndcg_at_10_std": 0.14694597443117657, + "nauc_ndcg_at_1_diff1": 0.13227797155889448, + "nauc_ndcg_at_1_max": 0.3912185212158381, + "nauc_ndcg_at_1_std": -0.04321744796657597, + "nauc_ndcg_at_20_diff1": 0.0447225718184246, + "nauc_ndcg_at_20_max": 0.32188100789256113, + "nauc_ndcg_at_20_std": 0.15945765527960085, + "nauc_ndcg_at_3_diff1": 0.08841552608454324, + "nauc_ndcg_at_3_max": 0.3946137737235812, + "nauc_ndcg_at_3_std": 0.10748417874736685, + "nauc_ndcg_at_5_diff1": 0.0714709503254886, + "nauc_ndcg_at_5_max": 0.36215797961625046, + "nauc_ndcg_at_5_std": 0.12058489753937601, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.12803173158202766, + "nauc_precision_at_100_max": 0.06547889979842647, + "nauc_precision_at_100_std": 0.42555866224505257, + "nauc_precision_at_10_diff1": 0.05654763349367328, + "nauc_precision_at_10_max": 0.34480950971017543, + "nauc_precision_at_10_std": 0.2104931050718133, + "nauc_precision_at_1_diff1": 0.13227797155889448, + "nauc_precision_at_1_max": 0.3912185212158381, + "nauc_precision_at_1_std": -0.04321744796657597, + "nauc_precision_at_20_diff1": 0.003245917316463151, + "nauc_precision_at_20_max": 0.2514482492276001, + "nauc_precision_at_20_std": 0.23772436368986333, + "nauc_precision_at_3_diff1": 0.08234531736637818, + "nauc_precision_at_3_max": 0.40334195160981035, + "nauc_precision_at_3_std": 0.15114164611284883, + "nauc_precision_at_5_diff1": 0.05837426155139814, + "nauc_precision_at_5_max": 0.34929757001537187, + "nauc_precision_at_5_std": 0.1668692992839126, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.12803173158202777, + "nauc_recall_at_100_max": 0.0654788997984262, + "nauc_recall_at_100_std": 0.4255586622450532, + "nauc_recall_at_10_diff1": 0.05654763349367305, + "nauc_recall_at_10_max": 0.34480950971017527, + "nauc_recall_at_10_std": 0.21049310507181299, + "nauc_recall_at_1_diff1": 0.13227797155889448, + "nauc_recall_at_1_max": 0.3912185212158381, + "nauc_recall_at_1_std": -0.04321744796657597, + "nauc_recall_at_20_diff1": 0.00324591731646297, + "nauc_recall_at_20_max": 0.25144824922760045, + "nauc_recall_at_20_std": 0.23772436368986324, + "nauc_recall_at_3_diff1": 0.08234531736637819, + "nauc_recall_at_3_max": 0.40334195160981046, + "nauc_recall_at_3_std": 0.1511416461128487, + "nauc_recall_at_5_diff1": 0.05837426155139821, + "nauc_recall_at_5_max": 0.3492975700153716, + "nauc_recall_at_5_std": 0.16686929928391245, + "ndcg_at_1": 0.05, + "ndcg_at_10": 0.16371, + "ndcg_at_100": 0.25125, + "ndcg_at_1000": 0.28643, + "ndcg_at_20": 0.18987, + "ndcg_at_3": 0.11548, + "ndcg_at_5": 0.14514, + "precision_at_1": 0.05, + "precision_at_10": 0.02964, + "precision_at_100": 0.00739, + "precision_at_1000": 0.001, + "precision_at_20": 0.02, + "precision_at_3": 0.05595, + "precision_at_5": 0.04786, + "recall_at_1": 0.05, + "recall_at_10": 0.29643, + "recall_at_100": 0.73929, + "recall_at_1000": 1.0, + "recall_at_20": 0.4, + "recall_at_3": 0.16786, + "recall_at_5": 0.23929 + } + ] + }, + "task_name": "VidoreTabfquadRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTatdqaRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..b76decc1ed --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", + "evaluation_time": 59.205832719802856, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.01022, + "cv_recall_at_10": 0.08539, + "cv_recall_at_100": 0.25256, + "cv_recall_at_1000": 0.80878, + "cv_recall_at_20": 0.12267, + "cv_recall_at_3": 0.03307, + "cv_recall_at_5": 0.05171, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.02729, + "map_at_1": 0.00722, + "map_at_10": 0.02527, + "map_at_100": 0.0308, + "map_at_1000": 0.03258, + "map_at_20": 0.02783, + "map_at_3": 0.01523, + "map_at_5": 0.02019, + "mrr_at_1": 0.010222489476849068, + "mrr_at_10": 0.02842921093071426, + "mrr_at_100": 0.03400601407873887, + "mrr_at_1000": 0.03579084760325294, + "mrr_at_20": 0.030996115929951835, + "mrr_at_3": 0.01964321507316095, + "mrr_at_5": 0.02382240930046101, + "nauc_cv_recall_at_1000_diff1": 0.062245112432012605, + "nauc_cv_recall_at_1000_max": 0.0824049719625231, + "nauc_cv_recall_at_1000_std": 0.1926981253481964, + "nauc_cv_recall_at_100_diff1": -0.0016464566256863254, + "nauc_cv_recall_at_100_max": 0.13593753936259395, + "nauc_cv_recall_at_100_std": 0.20986830714623048, + "nauc_cv_recall_at_10_diff1": -0.07914455925364039, + "nauc_cv_recall_at_10_max": 0.221889572021891, + "nauc_cv_recall_at_10_std": 0.22793307551587869, + "nauc_cv_recall_at_1_diff1": -0.07096976801316256, + "nauc_cv_recall_at_1_max": 0.1406929737474586, + "nauc_cv_recall_at_1_std": 0.3511188157780023, + "nauc_cv_recall_at_20_diff1": -0.04273344090166819, + "nauc_cv_recall_at_20_max": 0.19950094761618484, + "nauc_cv_recall_at_20_std": 0.23339685125761483, + "nauc_cv_recall_at_3_diff1": -0.029092895639082766, + "nauc_cv_recall_at_3_max": 0.3024227478832245, + "nauc_cv_recall_at_3_std": 0.3651209878641501, + "nauc_cv_recall_at_5_diff1": -0.06245270317059542, + "nauc_cv_recall_at_5_max": 0.2780656666101435, + "nauc_cv_recall_at_5_std": 0.2916408018352315, + "nauc_map_at_1000_diff1": -0.07915406935803154, + "nauc_map_at_1000_max": 0.28295299157468146, + "nauc_map_at_1000_std": 0.3338660650388641, + "nauc_map_at_100_diff1": -0.08057721702418681, + "nauc_map_at_100_max": 0.2851732241233053, + "nauc_map_at_100_std": 0.33535597233102077, + "nauc_map_at_10_diff1": -0.10153434963322266, + "nauc_map_at_10_max": 0.3179042357801275, + "nauc_map_at_10_std": 0.35665721016705615, + "nauc_map_at_1_diff1": -0.19014926326861104, + "nauc_map_at_1_max": 0.390026592747675, + "nauc_map_at_1_std": 0.6195628746702934, + "nauc_map_at_20_diff1": -0.08822801352495452, + "nauc_map_at_20_max": 0.3006216111365114, + "nauc_map_at_20_std": 0.344373361564743, + "nauc_map_at_3_diff1": -0.12322204243809397, + "nauc_map_at_3_max": 0.4422719884723983, + "nauc_map_at_3_std": 0.4833438290593387, + "nauc_map_at_5_diff1": -0.09916631427213753, + "nauc_map_at_5_max": 0.367562851560209, + "nauc_map_at_5_std": 0.41793713089980417, + "nauc_mrr_at_1000_diff1": -0.05131706920876975, + "nauc_mrr_at_1000_max": 0.210064636684804, + "nauc_mrr_at_1000_std": 0.2869649829077407, + "nauc_mrr_at_100_diff1": -0.052613232566971556, + "nauc_mrr_at_100_max": 0.21221535560957858, + "nauc_mrr_at_100_std": 0.28811513687379225, + "nauc_mrr_at_10_diff1": -0.06605985271574645, + "nauc_mrr_at_10_max": 0.23039009119491277, + "nauc_mrr_at_10_std": 0.2967871584573022, + "nauc_mrr_at_1_diff1": -0.07096976801316256, + "nauc_mrr_at_1_max": 0.1406929737474586, + "nauc_mrr_at_1_std": 0.3511188157780023, + "nauc_mrr_at_20_diff1": -0.05738441359665133, + "nauc_mrr_at_20_max": 0.22292825397434873, + "nauc_mrr_at_20_std": 0.2933713427853174, + "nauc_mrr_at_3_diff1": -0.041994622737094436, + "nauc_mrr_at_3_max": 0.2490547970067084, + "nauc_mrr_at_3_std": 0.3664687256018941, + "nauc_mrr_at_5_diff1": -0.055577165183634286, + "nauc_mrr_at_5_max": 0.24481872513306857, + "nauc_mrr_at_5_std": 0.3301532065538261, + "nauc_ndcg_at_1000_diff1": -0.027271831337053593, + "nauc_ndcg_at_1000_max": 0.19320311929839745, + "nauc_ndcg_at_1000_std": 0.26230675769834827, + "nauc_ndcg_at_100_diff1": -0.03857256879121206, + "nauc_ndcg_at_100_max": 0.20053651893359686, + "nauc_ndcg_at_100_std": 0.2634249219923024, + "nauc_ndcg_at_10_diff1": -0.09208853881933711, + "nauc_ndcg_at_10_max": 0.2741867263963051, + "nauc_ndcg_at_10_std": 0.3018685154562808, + "nauc_ndcg_at_1_diff1": -0.19014926326861104, + "nauc_ndcg_at_1_max": 0.390026592747675, + "nauc_ndcg_at_1_std": 0.6195628746702934, + "nauc_ndcg_at_20_diff1": -0.0662883493156438, + "nauc_ndcg_at_20_max": 0.2455835992143969, + "nauc_ndcg_at_20_std": 0.2862069918150026, + "nauc_ndcg_at_3_diff1": -0.115158159385534, + "nauc_ndcg_at_3_max": 0.4486641436064538, + "nauc_ndcg_at_3_std": 0.46223973106257327, + "nauc_ndcg_at_5_diff1": -0.08719685556011549, + "nauc_ndcg_at_5_max": 0.34464533880726306, + "nauc_ndcg_at_5_std": 0.382579857386717, + "nauc_precision_at_1000_diff1": 0.06224511243200888, + "nauc_precision_at_1000_max": 0.08240497196252142, + "nauc_precision_at_1000_std": 0.19269812534819536, + "nauc_precision_at_100_diff1": -0.0008778967359981768, + "nauc_precision_at_100_max": 0.13264567779183334, + "nauc_precision_at_100_std": 0.21307362293607623, + "nauc_precision_at_10_diff1": -0.08303221130704748, + "nauc_precision_at_10_max": 0.22554733634346374, + "nauc_precision_at_10_std": 0.2456046858526632, + "nauc_precision_at_1_diff1": -0.19014926326861104, + "nauc_precision_at_1_max": 0.390026592747675, + "nauc_precision_at_1_std": 0.6195628746702934, + "nauc_precision_at_20_diff1": -0.04471530791368069, + "nauc_precision_at_20_max": 0.19301794526935728, + "nauc_precision_at_20_std": 0.23610474317523472, + "nauc_precision_at_3_diff1": -0.10217447389281792, + "nauc_precision_at_3_max": 0.458977611263278, + "nauc_precision_at_3_std": 0.42722577233039744, + "nauc_precision_at_5_diff1": -0.07232585588874545, + "nauc_precision_at_5_max": 0.3115793713279293, + "nauc_precision_at_5_std": 0.3367719489231344, + "nauc_recall_at_1000_diff1": 0.062245112432012605, + "nauc_recall_at_1000_max": 0.0824049719625231, + "nauc_recall_at_1000_std": 0.1926981253481964, + "nauc_recall_at_100_diff1": -0.0008778967359985224, + "nauc_recall_at_100_max": 0.13264567779183312, + "nauc_recall_at_100_std": 0.2130736229360761, + "nauc_recall_at_10_diff1": -0.08303221130704774, + "nauc_recall_at_10_max": 0.2255473363434636, + "nauc_recall_at_10_std": 0.24560468585266304, + "nauc_recall_at_1_diff1": -0.19014926326861104, + "nauc_recall_at_1_max": 0.390026592747675, + "nauc_recall_at_1_std": 0.6195628746702934, + "nauc_recall_at_20_diff1": -0.044715307913680905, + "nauc_recall_at_20_max": 0.19301794526935703, + "nauc_recall_at_20_std": 0.2361047431752346, + "nauc_recall_at_3_diff1": -0.10217447389281765, + "nauc_recall_at_3_max": 0.45897761126327813, + "nauc_recall_at_3_std": 0.42722577233039744, + "nauc_recall_at_5_diff1": -0.07232585588874574, + "nauc_recall_at_5_max": 0.31157937132792907, + "nauc_recall_at_5_std": 0.3367719489231341, + "ndcg_at_1": 0.00722, + "ndcg_at_10": 0.03933, + "ndcg_at_100": 0.07208, + "ndcg_at_1000": 0.13746, + "ndcg_at_20": 0.04872, + "ndcg_at_3": 0.01824, + "ndcg_at_5": 0.02729, + "precision_at_1": 0.00722, + "precision_at_10": 0.0086, + "precision_at_100": 0.00253, + "precision_at_1000": 0.00081, + "precision_at_20": 0.00616, + "precision_at_3": 0.00902, + "precision_at_5": 0.00986, + "recall_at_1": 0.00722, + "recall_at_10": 0.08599, + "recall_at_100": 0.25316, + "recall_at_1000": 0.80878, + "recall_at_20": 0.12327, + "recall_at_3": 0.02706, + "recall_at_5": 0.04931 + } + ] + }, + "task_name": "VidoreTatdqaRetrieval" +} \ No newline at end of file From 6e7dd3d6725db7b5f260b146f2266e63cc1ea50b Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Fri, 18 Oct 2024 16:33:33 +0300 Subject: [PATCH 075/154] [mieb] Adding MOCOv3 models (#1293) * add moco models first try * add as a timm model * add large model results * make lint --- .../Image/Any2TextMultipleChoiceEvaluator.py | 2 +- mteb/models/__init__.py | 2 + mteb/models/moco_models.py | 144 ++++++++++++++++++ .../OxfordFlowersClassification.json | 48 ++++++ .../model_meta.json | 1 + .../OxfordFlowersClassification.json | 48 ++++++ .../model_meta.json | 1 + 7 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 mteb/models/moco_models.py create mode 100644 results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json create mode 100644 results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json create mode 100644 results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json create mode 100644 results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json diff --git a/mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py b/mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py index f682225ba5..a93714e770 100644 --- a/mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py @@ -62,7 +62,7 @@ def __call__( encode_kwargs["batch_size"] = 64 label_list = list( - set([x for n in self.dataset[self.choices_column_name] for x in n]) + {x for n in self.dataset[self.choices_column_name] for x in n} ) label_embeddings = model.get_text_embeddings(label_list) label_embedding_dict = {} diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index 33b83f9a11..d8512a5a23 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -25,6 +25,7 @@ gte_models, jina_clip, llm2vec_models, + moco_models, mxbai_models, nomic_models, nomic_models_vision, @@ -147,6 +148,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe gte_models, jina_clip, llm2vec_models, + moco_models, mxbai_models, nomic_models, nomic_models_vision, diff --git a/mteb/models/moco_models.py b/mteb/models/moco_models.py new file mode 100644 index 0000000000..3fa7dfe203 --- /dev/null +++ b/mteb/models/moco_models.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm + +from mteb.model_meta import ModelMeta + + +def mocov3_loader(**kwargs): + try: + import timm + except ImportError: + raise ImportError("Please install `pip install timm` to use MOCOv3 models.") + + class MOCOv3Wrapper: + """A wrapper class for MOCOv3 models that supports image encoding. + Text encoding and text-image fusion are not supported. + """ + + def __init__( + self, + model_name: str = "nyu-visionx/moco-v3-vit-b", + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + name = "vit_base_patch16_224" + if "vit-l" in model_name: + name = "vit_large_patch16_224" + model = timm.create_model( + name, + pretrained=True, + num_classes=0, + pretrained_cfg_overlay={"hf_hub_id": model_name}, + ) + + self.model = model.eval() + + # get model specific transforms (normalization, resize) + data_config = timm.data.resolve_model_data_config(self.model) + self.processor = timm.data.create_transform( + **data_config, is_training=False + ) + + @staticmethod + def get_text_embeddings(texts: list[str], batch_size: int = 32): + raise ValueError("MOCO models only support image encoding.") + + def get_image_embeddings( + self, + images: list[Image.Image] | DataLoader, + batch_size: int = 32, + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + import torchvision.transforms.functional as F + + with torch.no_grad(): + for batch in tqdm(images): + inputs = torch.vstack( + [ + self.processor(F.to_pil_image(b.to("cpu"))).unsqueeze(0) + for b in batch + ] + ) + output = self.model( + inputs + ) # output is (batch_size, num_features) shaped tensor + all_image_embeddings.append(output) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + output = self.model( + self.processor(batch_images) + ) # output is (batch_size, num_features) shaped tensor + all_image_embeddings.append(output) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + @staticmethod + def calculate_probs(text_embeddings, image_embeddings): + raise ValueError("MOCO models only support image encoding.") + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("images must be provided for MOCO models") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + raise ValueError("MOCO models only support image encoding.") + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + return MOCOv3Wrapper(**kwargs) + + +mocov3_vit_base = ModelMeta( + loader=partial( + mocov3_loader, + model_name="nyu-visionx/moco-v3-vit-b", + ), + name="nyu-visionx/moco-v3-vit-b", + languages=["eng_Latn"], + open_source=True, + revision="7d091cd70772c5c0ecf7f00b5f12ca609a99d69d", + release_date="2024-06-03", +) + +mocov3_vit_large = ModelMeta( + loader=partial( + mocov3_loader, + model_name="nyu-visionx/moco-v3-vit-l", + ), + name="nyu-visionx/moco-v3-vit-l", + languages=["eng_Latn"], + open_source=True, + revision="7bf75358d616f39b9716148bf4e3425f3bd35b47", + release_date="2024-06-03", +) diff --git a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json new file mode 100644 index 0000000000..f4ea1b602c --- /dev/null +++ b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + "evaluation_time": 347.9395191669464, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.8880392156862745, + "f1": 0.8858313070145011, + "f1_weighted": 0.8855993382917493, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8880392156862745, + "scores_per_experiment": [ + { + "accuracy": 0.8921568627450981, + "f1": 0.8908725994288734, + "f1_weighted": 0.890707976140322 + }, + { + "accuracy": 0.888235294117647, + "f1": 0.8862588054389066, + "f1_weighted": 0.8857789823937094 + }, + { + "accuracy": 0.8833333333333333, + "f1": 0.8807271651302992, + "f1_weighted": 0.8807519823317017 + }, + { + "accuracy": 0.884313725490196, + "f1": 0.8818842242107174, + "f1_weighted": 0.8815393805409137 + }, + { + "accuracy": 0.8921568627450981, + "f1": 0.8894137408637092, + "f1_weighted": 0.8892183700520992 + } + ] + } + ] + }, + "task_name": "OxfordFlowersClassification" +} \ No newline at end of file diff --git a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json new file mode 100644 index 0000000000..d97b884a3b --- /dev/null +++ b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json @@ -0,0 +1 @@ +{"name": "nyu-visionx/moco-v3-vit-b", "revision": "7d091cd70772c5c0ecf7f00b5f12ca609a99d69d", "release_date": "2024-06-03", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "mocov3_loader"} \ No newline at end of file diff --git a/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json b/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json new file mode 100644 index 0000000000..a1fb3e1e31 --- /dev/null +++ b/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + "evaluation_time": 780.7572722434998, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.8990196078431373, + "f1": 0.8976575858316652, + "f1_weighted": 0.8973426503552833, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8990196078431373, + "scores_per_experiment": [ + { + "accuracy": 0.8960784313725491, + "f1": 0.8954744443526484, + "f1_weighted": 0.8949013011291541 + }, + { + "accuracy": 0.9009803921568628, + "f1": 0.89924527322374, + "f1_weighted": 0.89919388345273 + }, + { + "accuracy": 0.8960784313725491, + "f1": 0.8937528098526334, + "f1_weighted": 0.8936883759623736 + }, + { + "accuracy": 0.8990196078431373, + "f1": 0.8985132758082764, + "f1_weighted": 0.8977617544247366 + }, + { + "accuracy": 0.9029411764705882, + "f1": 0.901302125921028, + "f1_weighted": 0.901167936807422 + } + ] + } + ] + }, + "task_name": "OxfordFlowersClassification" +} \ No newline at end of file diff --git a/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json b/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json new file mode 100644 index 0000000000..b9e1a399c2 --- /dev/null +++ b/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json @@ -0,0 +1 @@ +{"name": "nyu-visionx/moco-v3-vit-l", "revision": "7bf75358d616f39b9716148bf4e3425f3bd35b47", "release_date": "2024-06-03", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "mocov3_loader"} \ No newline at end of file From 053b5be16c445ae401e64ce151f356c56cb4a787 Mon Sep 17 00:00:00 2001 From: Jamie-Stirling <36764530+Jamie-Stirling@users.noreply.github.com> Date: Sun, 20 Oct 2024 08:18:39 +0100 Subject: [PATCH 076/154] [mieb] Add more Any2AnyRetrieval datasets (#1285) * wip: start adding BLIP models * add other blip variants * wip: add blip2_models.py * make lint * wip: implement blip2 wrapper * feat: add blip2 models, still mismatched names * fix: remove projections from image and text embeddings * make lint * wip: add coco BLIP2 * fix: BLIP2 better zero-shot classification without text_proj and vision_proj * tidy blip2 * add imagenet-dog-15 dataset * tidy and lint * remove unused import * add cluster_accuracy, ari and nmi to Image.ClusteringEvaluator * add imagenet-10 clustering task * add SOPI2IRetrieval * add results forclip on ImageNet10Clustering and ImageNetDog15Clustering * add SOPI2IRetrieval results for clip 32 * add results for clip vit 32/SOPI2IRetrieval * resolve conflict * add RP2kI2IRetrieval dataset * add RP2kI2IRetrieval results with clip-vit-base-patch32 * update image retrieval __init__.py * add RP2kI2IRetrieval and METI2IRetrieval * add METI2IRetreival * add SOP results * make lign * new revision for METI2IRetrieval * make lint * reset corpus chunk size * remove wrong classification import * add Flickr30k T2I and I2T * add Flickr30k T2I retriebal * reduced-size MET revision * fix: add Flickr30k T2I * make lint * add two landmark datasets and results * add Sketchy i2i retrieval * add task metadata * add BLINKIT2IRetrieval dataset * add BLINKIT2TRetrieval * add ImageCoDeT2IRetrieval * make lint * add vizwiz retrieval and results * fix vizwiz duplicate texts * add new vizwiz results * add VQA2 results * add GLD v2 I2T retrieval * add gld v2 i2i retrieval * make lint * remove GLDv2I2IRetrieval --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 9 + .../eng/BLINKIT2IRetrieval.py | 49 +++++ .../eng/BLINKIT2TRetrieval.py | 49 +++++ .../Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py | 50 +++++ .../eng/ImageCoDeT2IRetrieval.py | 50 +++++ .../eng/ROxfordI2IRetrieval.py | 50 +++++ .../Any2AnyRetrieval/eng/RP2kI2IRetrieval.py | 11 +- .../eng/RParisI2IRetrieval.py | 50 +++++ .../Any2AnyRetrieval/eng/SOPI2IRetrieval.py | 11 +- .../eng/SketchyI2IRetrieval.py | 49 +++++ .../Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py | 50 +++++ .../eng/VizWizIT2TRetrieval.py | 51 +++++ .../BLINKIT2IRetrieval.json | 186 ++++++++++++++++++ .../BLINKIT2TRetrieval.json | 186 ++++++++++++++++++ .../GLDv2I2TRetrieval.json | 186 ++++++++++++++++++ .../ImageCoDeT2IRetrieval.json | 186 ++++++++++++++++++ .../ROxfordI2IRetrieval.json | 186 ++++++++++++++++++ .../RParisI2IRetrieval.json | 186 ++++++++++++++++++ .../VQA2IT2TRetrieval.json | 186 ++++++++++++++++++ .../VizWizIT2TRetrieval.json | 186 ++++++++++++++++++ 20 files changed, 1965 insertions(+), 2 deletions(-) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordI2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisI2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VQA2IT2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VizWizIT2TRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 1bf12cf0e7..d4822a3473 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -1,5 +1,7 @@ from __future__ import annotations +from .eng.BLINKIT2IRetrieval import * +from .eng.BLINKIT2TRetrieval import * from .eng.CIRRIT2IRetrieval import * from .eng.CUB200I2IRetrieval import * from .eng.Fashion200kI2TRetrieval import * @@ -8,8 +10,10 @@ from .eng.Flickr30kI2TRetrieval import * from .eng.Flickr30kT2IRetrieval import * from .eng.FORBI2IRetrieval import * +from .eng.GLDv2I2TRetrieval import * from .eng.HatefulMemesI2TRetrieval import * from .eng.HatefulMemesT2IRetrieval import * +from .eng.ImageCoDeT2IRetrieval import * from .eng.InfoSeekIT2ITRetrieval import * from .eng.InfoSeekIT2TRetrieval import * from .eng.MemotionI2TRetrieval import * @@ -20,15 +24,20 @@ from .eng.NIGHTSI2IRetrieval import * from .eng.OVENIT2ITRetrieval import * from .eng.OVENIT2TRetrieval import * +from .eng.ROxfordI2IRetrieval import * from .eng.RP2kI2IRetrieval import * +from .eng.RParisI2IRetrieval import * from .eng.SciMMIRI2TRetrieval import * from .eng.SciMMIRT2IRetrieval import * +from .eng.SketchyI2IRetrieval import * from .eng.SOPI2IRetrieval import * from .eng.StanfordCarsI2IRetrieval import * from .eng.TUBerlinT2IRetrieval import * from .eng.VidoreBenchRetrieval import * from .eng.VisualNewsI2TRetrieval import * from .eng.VisualNewsT2IRetrieval import * +from .eng.VizWizIT2TRetrieval import * +from .eng.VQA2IT2TRetrieval import * from .eng.WebQAT2ITRetrieval import * from .eng.WebQAT2TRetrieval import * from .multilingual.WITT2IRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py new file mode 100644 index 0000000000..ca265fd898 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class BLINKIT2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="BLINKIT2IRetrieval", + description="Retrieve images based on images and specific retrieval instructions.", + reference="https://arxiv.org/abs/2404.12390", + dataset={ + "path": "JamieSJS/blink-it2i", + "revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26", + "trust_remote_code": True, + }, + type="Retrieval", + category="it2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2018-01-01", "2018-12-31"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{fu2024blink, + title={Blink: Multimodal large language models can see but not perceive}, + author={Fu, Xingyu and Hu, Yushi and Li, Bangzheng and Feng, Yu and Wang, Haoyu and Lin, Xudong and Roth, Dan and Smith, Noah A and Ma, Wei-Chiu and Krishna, Ranjay}, + journal={arXiv preprint arXiv:2404.12390}, + year={2024} +} +""", + descriptive_stats={ + "n_samples": {"test": 402}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 804, + "num_queries": 402, + "average_relevant_docs_per_query": 1, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py new file mode 100644 index 0000000000..2c652c6388 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class BLINKIT2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="BLINKIT2TRetrieval", + description="Retrieve images based on images and specific retrieval instructions.", + reference="https://arxiv.org/abs/2404.12390", + dataset={ + "path": "JamieSJS/blink-it2t", + "revision": "4ab83c87ac5b24e3b730f86d585671493a3a423c", + "trust_remote_code": True, + }, + type="Retrieval", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2018-01-01", "2018-12-31"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{fu2024blink, + title={Blink: Multimodal large language models can see but not perceive}, + author={Fu, Xingyu and Hu, Yushi and Li, Bangzheng and Feng, Yu and Wang, Haoyu and Lin, Xudong and Roth, Dan and Smith, Noah A and Ma, Wei-Chiu and Krishna, Ranjay}, + journal={arXiv preprint arXiv:2404.12390}, + year={2024} +} +""", + descriptive_stats={ + "n_samples": {"test": 1073}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 3080, + "num_queries": 1073, + "average_relevant_docs_per_query": 1, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py new file mode 100644 index 0000000000..9539ef31b3 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class GLDv2I2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="GLDv2I2TRetrieval", + description="Retrieve names of landmarks based on their image.", + reference="https://openaccess.thecvf.com/content_CVPR_2020/html/Weyand_Google_Landmarks_Dataset_v2_-_A_Large-Scale_Benchmark_for_Instance-Level_CVPR_2020_paper.html", + dataset={ + "path": "JamieSJS/gld-v2-i2t", + "revision": "d8c3e53160860f76de73ed3041a8593672fe5928", + }, + type="Retrieval", + category="i2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2017-01-01", "2017-12-31"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="Apache-2.0", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@InProceedings{Weyand_2020_CVPR, +author = {Weyand, Tobias and Araujo, Andre and Cao, Bingyi and Sim, Jack}, +title = {Google Landmarks Dataset v2 - A Large-Scale Benchmark for Instance-Level Recognition and Retrieval}, +booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, +month = {June}, +year = {2020} +} + +""", + descriptive_stats={ + "n_samples": {"test": 1972}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 674, + "num_queries": 1972, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py new file mode 100644 index 0000000000..3fae916f6b --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class ImageCoDeT2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="ImageCoDeT2IRetrieval", + description="Retrieve a specific video frame based on a precise caption.", + reference="https://aclanthology.org/2022.acl-long.241.pdf", + dataset={ + "path": "JamieSJS/imagecode", + "revision": "a424cd523ffb157b69a875fb5e71c1d51be54089", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2022-05-22", "2022-05-27"), # conference dates + form=["written"], + domains=["Web"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{krojer2022image, + title={Image retrieval from contextual descriptions}, + author={Krojer, Benno and Adlakha, Vaibhav and Vineet, Vibhav and Goyal, Yash and Ponti, Edoardo and Reddy, Siva}, + journal={arXiv preprint arXiv:2203.15867}, + year={2022} +} +""", + descriptive_stats={ + "n_samples": {"test": 2302}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 23020, + "num_queries": 2302, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py new file mode 100644 index 0000000000..39502a46d8 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class ROxfordI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="ROxfordI2IRetrieval", + description="Retrieve photos of landmarks in Oxford, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-oxford", + "revision": "d8daad98b4e4896a7f7fa1b3485a22420242d4fc", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image retrieval benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 3555537}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 5063, + "num_queries": 5063, + "average_relevant_docs_per_query": 702, + } + }, + }, + ) + skip_first_result = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py index 1335e11659..321bb818be 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py @@ -34,7 +34,16 @@ class RP2kI2IRetrieval(AbsTaskAny2AnyRetrieval): } """, descriptive_stats={ - "n_samples": {"default": 4409419}, + "n_samples": {"test": 39457}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 39457, + "num_queries": 39457, + "average_relevant_docs_per_query": 111.8, + } + }, }, ) skip_first_result = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py new file mode 100644 index 0000000000..a112ded273 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class RParisI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="RParisI2IRetrieval", + description="Retrieve photos of landmarks in Paris.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-paris", + "revision": "bafc3a08fdffd72558021ce3a41250833d7e0e88", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image retrieval benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 6392}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 6392, + "num_queries": 6392, + "average_relevant_docs_per_query": 734, + } + }, + }, + ) + skip_first_result = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py index 9f3d771b0a..09b33aac7d 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py @@ -35,7 +35,16 @@ class SOPI2IRetrieval(AbsTaskAny2AnyRetrieval): } """, descriptive_stats={ - "n_samples": {"default": 120053}, + "n_samples": {"test": 120053}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 120053, + "num_queries": 120053, + "average_relevant_docs_per_query": 7, + } + }, }, ) skip_first_result = True diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py new file mode 100644 index 0000000000..c89091f41d --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class SketchyI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="SketchyI2IRetrieval", + description="Retrieve photos from sketches.", + reference="https://arxiv.org/abs/2202.01747", + dataset={ + "path": "JamieSJS/sketchy", + "revision": "c8b8c1b7a2f0a92f1bfaaa1c9afc22aa42c61d5b", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2021-12-06", "2021-12-14"), # conference dates + domains=["Encyclopaedic"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{ypsilantis2021met, + title={The met dataset: Instance-level recognition for artworks}, + author={Ypsilantis, Nikolaos-Antonios and Garcia, Noa and Han, Guangxing and Ibrahimi, Sarah and Van Noord, Nanne and Tolias, Giorgos}, + booktitle={Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)}, + year={2021} +} + """, + descriptive_stats={ + "n_samples": {"test": 452886}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 7.24, + "num_documents": 25000, + "num_queries": 452886, + "average_relevant_docs_per_query": 3623.0, + } + }, + }, + ) + skip_first_result = False diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py new file mode 100644 index 0000000000..cb5c3b38e5 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class VQA2IT2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VQA2IT2TRetrieval", + description="Retrieve the correct answer for a question about an image.", + reference="https://openaccess.thecvf.com/content_cvpr_2017/html/Goyal_Making_the_v_CVPR_2017_paper.html", + dataset={ + "path": "JamieSJS/vqa-2", + "revision": "69882b6ba0b443dd62e633e546725b0f13b7e3aa", + "trust_remote_code": True, + }, + type="Retrieval", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2017-07-01", "2017-07-01"), + domains=["Web"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@InProceedings{Goyal_2017_CVPR, +author = {Goyal, Yash and Khot, Tejas and Summers-Stay, Douglas and Batra, Dhruv and Parikh, Devi}, +title = {Making the v in VQA Matter: Elevating the Role of Image Understanding in Visual Question Answering}, +booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, +month = {July}, +year = {2017} +} +""", + descriptive_stats={ + "n_samples": {"test": 4319}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 2091, + "num_queries": 4319, + "average_relevant_docs_per_query": 1, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py new file mode 100644 index 0000000000..5565ca9f50 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class VizWizIT2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="VizWizIT2TRetrieval", + description="Retrieve the correct answer for a question about an image.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/papers/Gurari_VizWiz_Grand_Challenge_CVPR_2018_paper.pdf", + dataset={ + "path": "JamieSJS/vizwiz", + "revision": "044af162d55f82ab603fa16ffcf7f1e4dbf300e9", + "trust_remote_code": True, + }, + type="Retrieval", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2018-01-01", "2018-01-01"), + domains=["Web"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@inproceedings{gurari2018vizwiz, + title={Vizwiz grand challenge: Answering visual questions from blind people}, + author={Gurari, Danna and Li, Qing and Stangl, Abigale J and Guo, Anhong and Lin, Chi and Grauman, Kristen and Luo, Jiebo and Bigham, Jeffrey P}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={3608--3617}, + year={2018} +} + +""", + descriptive_stats={ + "n_samples": {"test": 214354}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 2143540, + "num_queries": 214354, + "average_relevant_docs_per_query": 1, + } + }, + }, + ) diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IRetrieval.json new file mode 100644 index 0000000000..1032c117ac --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26", + "evaluation_time": 17.393863439559937, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.31592, + "cv_recall_at_10": 0.56965, + "cv_recall_at_100": 0.87065, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.6393, + "cv_recall_at_3": 0.48507, + "cv_recall_at_5": 0.52736, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.44916, + "map_at_1": 0.31592, + "map_at_10": 0.40997, + "map_at_100": 0.42021, + "map_at_1000": 0.42102, + "map_at_20": 0.41454, + "map_at_3": 0.39469, + "map_at_5": 0.40427, + "mrr_at_1": 0.31592039800995025, + "mrr_at_10": 0.40997295269683326, + "mrr_at_100": 0.4202098745202131, + "mrr_at_1000": 0.42101657433711814, + "mrr_at_20": 0.41453745832471395, + "mrr_at_3": 0.3946932006633499, + "mrr_at_5": 0.4042703150912106, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.17659898624319145, + "nauc_cv_recall_at_100_max": 0.2661970196010441, + "nauc_cv_recall_at_100_std": -0.2790205688665586, + "nauc_cv_recall_at_10_diff1": 0.3814466970679155, + "nauc_cv_recall_at_10_max": 0.45289099967438606, + "nauc_cv_recall_at_10_std": -0.6696974920638539, + "nauc_cv_recall_at_1_diff1": 0.4348231455825518, + "nauc_cv_recall_at_1_max": 0.3562517088470133, + "nauc_cv_recall_at_1_std": -0.4716841092376067, + "nauc_cv_recall_at_20_diff1": 0.34765332655935005, + "nauc_cv_recall_at_20_max": 0.40507962086487265, + "nauc_cv_recall_at_20_std": -0.60412556449698, + "nauc_cv_recall_at_3_diff1": 0.4000259511772347, + "nauc_cv_recall_at_3_max": 0.46517944743370765, + "nauc_cv_recall_at_3_std": -0.6578600193275457, + "nauc_cv_recall_at_5_diff1": 0.4149222268518541, + "nauc_cv_recall_at_5_max": 0.4447326055964038, + "nauc_cv_recall_at_5_std": -0.6819130948957665, + "nauc_map_at_1000_diff1": 0.41844540783755707, + "nauc_map_at_1000_max": 0.40386353368016625, + "nauc_map_at_1000_std": -0.5528994661016456, + "nauc_map_at_100_diff1": 0.41795524243083654, + "nauc_map_at_100_max": 0.40346749605366333, + "nauc_map_at_100_std": -0.5524674902038105, + "nauc_map_at_10_diff1": 0.42120255585772387, + "nauc_map_at_10_max": 0.40787732867834814, + "nauc_map_at_10_std": -0.5613450956416107, + "nauc_map_at_1_diff1": 0.4348231455825518, + "nauc_map_at_1_max": 0.3562517088470133, + "nauc_map_at_1_std": -0.4716841092376067, + "nauc_map_at_20_diff1": 0.4188754182377973, + "nauc_map_at_20_max": 0.4046917634724885, + "nauc_map_at_20_std": -0.5563336557097699, + "nauc_map_at_3_diff1": 0.4214894251666346, + "nauc_map_at_3_max": 0.41188287008473307, + "nauc_map_at_3_std": -0.5590768607537941, + "nauc_map_at_5_diff1": 0.4246473310855139, + "nauc_map_at_5_max": 0.4070463857404686, + "nauc_map_at_5_std": -0.5636557349011302, + "nauc_mrr_at_1000_diff1": 0.41844540783755707, + "nauc_mrr_at_1000_max": 0.40386353368016625, + "nauc_mrr_at_1000_std": -0.5528994661016456, + "nauc_mrr_at_100_diff1": 0.41795524243083654, + "nauc_mrr_at_100_max": 0.40346749605366333, + "nauc_mrr_at_100_std": -0.5524674902038105, + "nauc_mrr_at_10_diff1": 0.42120255585772387, + "nauc_mrr_at_10_max": 0.40787732867834814, + "nauc_mrr_at_10_std": -0.5613450956416107, + "nauc_mrr_at_1_diff1": 0.4348231455825518, + "nauc_mrr_at_1_max": 0.3562517088470133, + "nauc_mrr_at_1_std": -0.4716841092376067, + "nauc_mrr_at_20_diff1": 0.4188754182377973, + "nauc_mrr_at_20_max": 0.4046917634724885, + "nauc_mrr_at_20_std": -0.5563336557097699, + "nauc_mrr_at_3_diff1": 0.4214894251666346, + "nauc_mrr_at_3_max": 0.41188287008473307, + "nauc_mrr_at_3_std": -0.5590768607537941, + "nauc_mrr_at_5_diff1": 0.4246473310855139, + "nauc_mrr_at_5_max": 0.4070463857404686, + "nauc_mrr_at_5_std": -0.5636557349011302, + "nauc_ndcg_at_1000_diff1": 0.410108945334173, + "nauc_ndcg_at_1000_max": 0.4059872490668924, + "nauc_ndcg_at_1000_std": -0.5567117452158448, + "nauc_ndcg_at_100_diff1": 0.3974622734711721, + "nauc_ndcg_at_100_max": 0.39776281190453455, + "nauc_ndcg_at_100_std": -0.5422948577574471, + "nauc_ndcg_at_10_diff1": 0.41250730473823416, + "nauc_ndcg_at_10_max": 0.4193748171136443, + "nauc_ndcg_at_10_std": -0.5882737032592944, + "nauc_ndcg_at_1_diff1": 0.4348231455825518, + "nauc_ndcg_at_1_max": 0.3562517088470133, + "nauc_ndcg_at_1_std": -0.4716841092376067, + "nauc_ndcg_at_20_diff1": 0.4050302095189553, + "nauc_ndcg_at_20_max": 0.40785434428016737, + "nauc_ndcg_at_20_std": -0.5711108827183825, + "nauc_ndcg_at_3_diff1": 0.41605079061886857, + "nauc_ndcg_at_3_max": 0.4259171131235778, + "nauc_ndcg_at_3_std": -0.5847193268248356, + "nauc_ndcg_at_5_diff1": 0.42200838628819426, + "nauc_ndcg_at_5_max": 0.41721735277614064, + "nauc_ndcg_at_5_std": -0.5934002799239759, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.17659898624319106, + "nauc_precision_at_100_max": 0.26619701960104514, + "nauc_precision_at_100_std": -0.279020568866558, + "nauc_precision_at_10_diff1": 0.38144669706791556, + "nauc_precision_at_10_max": 0.4528909996743864, + "nauc_precision_at_10_std": -0.6696974920638536, + "nauc_precision_at_1_diff1": 0.4348231455825518, + "nauc_precision_at_1_max": 0.3562517088470133, + "nauc_precision_at_1_std": -0.4716841092376067, + "nauc_precision_at_20_diff1": 0.34765332655935016, + "nauc_precision_at_20_max": 0.40507962086487265, + "nauc_precision_at_20_std": -0.60412556449698, + "nauc_precision_at_3_diff1": 0.4000259511772348, + "nauc_precision_at_3_max": 0.4651794474337073, + "nauc_precision_at_3_std": -0.6578600193275455, + "nauc_precision_at_5_diff1": 0.41492222685185387, + "nauc_precision_at_5_max": 0.444732605596404, + "nauc_precision_at_5_std": -0.6819130948957663, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.17659898624319145, + "nauc_recall_at_100_max": 0.2661970196010441, + "nauc_recall_at_100_std": -0.2790205688665586, + "nauc_recall_at_10_diff1": 0.3814466970679155, + "nauc_recall_at_10_max": 0.45289099967438606, + "nauc_recall_at_10_std": -0.6696974920638539, + "nauc_recall_at_1_diff1": 0.4348231455825518, + "nauc_recall_at_1_max": 0.3562517088470133, + "nauc_recall_at_1_std": -0.4716841092376067, + "nauc_recall_at_20_diff1": 0.34765332655935005, + "nauc_recall_at_20_max": 0.40507962086487265, + "nauc_recall_at_20_std": -0.60412556449698, + "nauc_recall_at_3_diff1": 0.4000259511772347, + "nauc_recall_at_3_max": 0.46517944743370765, + "nauc_recall_at_3_std": -0.6578600193275457, + "nauc_recall_at_5_diff1": 0.4149222268518541, + "nauc_recall_at_5_max": 0.4447326055964038, + "nauc_recall_at_5_std": -0.6819130948957665, + "ndcg_at_1": 0.31592, + "ndcg_at_10": 0.44916, + "ndcg_at_100": 0.50879, + "ndcg_at_1000": 0.52627, + "ndcg_at_20": 0.46643, + "ndcg_at_3": 0.41809, + "ndcg_at_5": 0.43543, + "precision_at_1": 0.31592, + "precision_at_10": 0.05697, + "precision_at_100": 0.00871, + "precision_at_1000": 0.001, + "precision_at_20": 0.03197, + "precision_at_3": 0.16169, + "precision_at_5": 0.10547, + "recall_at_1": 0.31592, + "recall_at_10": 0.56965, + "recall_at_100": 0.87065, + "recall_at_1000": 1.0, + "recall_at_20": 0.6393, + "recall_at_3": 0.48507, + "recall_at_5": 0.52736 + } + ] + }, + "task_name": "BLINKIT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json new file mode 100644 index 0000000000..d701facb1e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "4ab83c87ac5b24e3b730f86d585671493a3a423c", + "evaluation_time": 13.768992900848389, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.00093, + "cv_recall_at_10": 0.00373, + "cv_recall_at_100": 0.06151, + "cv_recall_at_1000": 0.49301, + "cv_recall_at_20": 0.01025, + "cv_recall_at_3": 0.00093, + "cv_recall_at_5": 0.00093, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.00209, + "map_at_1": 0.00093, + "map_at_10": 0.00139, + "map_at_100": 0.00274, + "map_at_1000": 0.00397, + "map_at_20": 0.00176, + "map_at_3": 0.00093, + "map_at_5": 0.00093, + "mrr_at_1": 0.0009319664492078285, + "mrr_at_10": 0.0013147383837039009, + "mrr_at_100": 0.002685356046665323, + "mrr_at_1000": 0.003948307638249348, + "mrr_at_20": 0.0016977247814228705, + "mrr_at_3": 0.0009319664492078285, + "mrr_at_5": 0.0009319664492078285, + "nauc_cv_recall_at_1000_diff1": 0.21618180421784144, + "nauc_cv_recall_at_1000_max": 0.1966391139887544, + "nauc_cv_recall_at_1000_std": 0.6300616078101204, + "nauc_cv_recall_at_100_diff1": 0.17443594463110845, + "nauc_cv_recall_at_100_max": 0.3594175633623019, + "nauc_cv_recall_at_100_std": 0.5893797181186331, + "nauc_cv_recall_at_10_diff1": -0.05990075938028686, + "nauc_cv_recall_at_10_max": 0.43603863739514936, + "nauc_cv_recall_at_10_std": 0.6462545358013111, + "nauc_cv_recall_at_1_diff1": -0.49952309145315554, + "nauc_cv_recall_at_1_max": 0.6610449563695936, + "nauc_cv_recall_at_1_std": 1.0, + "nauc_cv_recall_at_20_diff1": 0.1175713728405099, + "nauc_cv_recall_at_20_max": 0.36443381136528874, + "nauc_cv_recall_at_20_std": 0.3549079762661633, + "nauc_cv_recall_at_3_diff1": -0.49952309145315554, + "nauc_cv_recall_at_3_max": 0.6610449563695936, + "nauc_cv_recall_at_3_std": 1.0, + "nauc_cv_recall_at_5_diff1": -0.49952309145315554, + "nauc_cv_recall_at_5_max": 0.6610449563695936, + "nauc_cv_recall_at_5_std": 1.0, + "nauc_map_at_1000_diff1": 0.3461462162176906, + "nauc_map_at_1000_max": 0.30890589543493097, + "nauc_map_at_1000_std": 0.707200194236762, + "nauc_map_at_100_diff1": 0.4005511014650457, + "nauc_map_at_100_max": 0.30811121696833377, + "nauc_map_at_100_std": 0.7144966661138995, + "nauc_map_at_10_diff1": 0.5208347947487291, + "nauc_map_at_10_max": 0.206028460069979, + "nauc_map_at_10_std": 0.8872521734282164, + "nauc_map_at_1_diff1": 0.6610449563695936, + "nauc_map_at_1_max": 0.09839818901557038, + "nauc_map_at_1_std": 1.0, + "nauc_map_at_20_diff1": 0.4542718670585475, + "nauc_map_at_20_max": 0.26815935842012756, + "nauc_map_at_20_std": 0.7972838331802248, + "nauc_map_at_3_diff1": 0.6610449563695936, + "nauc_map_at_3_max": 0.09839818901557038, + "nauc_map_at_3_std": 1.0, + "nauc_map_at_5_diff1": 0.6610449563695936, + "nauc_map_at_5_max": 0.09839818901557038, + "nauc_map_at_5_std": 1.0, + "nauc_mrr_at_1000_diff1": -0.013155580619472381, + "nauc_mrr_at_1000_max": 0.4252494452276833, + "nauc_mrr_at_1000_std": 0.687428882142861, + "nauc_mrr_at_100_diff1": -0.05688061033793547, + "nauc_mrr_at_100_max": 0.4528834485316897, + "nauc_mrr_at_100_std": 0.6942429482097452, + "nauc_mrr_at_10_diff1": -0.3361389575479778, + "nauc_mrr_at_10_max": 0.5699031562786795, + "nauc_mrr_at_10_std": 0.8567106980461006, + "nauc_mrr_at_1_diff1": -0.49952309145315554, + "nauc_mrr_at_1_max": 0.6610449563695936, + "nauc_mrr_at_1_std": 1.0, + "nauc_mrr_at_20_diff1": -0.2128688275475949, + "nauc_mrr_at_20_max": 0.5173348447205537, + "nauc_mrr_at_20_std": 0.6977606046374574, + "nauc_mrr_at_3_diff1": -0.49952309145315554, + "nauc_mrr_at_3_max": 0.6610449563695936, + "nauc_mrr_at_3_std": 1.0, + "nauc_mrr_at_5_diff1": -0.49952309145315554, + "nauc_mrr_at_5_max": 0.6610449563695936, + "nauc_mrr_at_5_std": 1.0, + "nauc_ndcg_at_1000_diff1": 0.2267671022987118, + "nauc_ndcg_at_1000_max": 0.2471557214473064, + "nauc_ndcg_at_1000_std": 0.6348682667422092, + "nauc_ndcg_at_100_diff1": 0.2912265250539601, + "nauc_ndcg_at_100_max": 0.3559110476318641, + "nauc_ndcg_at_100_std": 0.5901042407967306, + "nauc_ndcg_at_10_diff1": 0.42096720717204206, + "nauc_ndcg_at_10_max": 0.2838485324864617, + "nauc_ndcg_at_10_std": 0.7948899807288888, + "nauc_ndcg_at_1_diff1": 0.6610449563695936, + "nauc_ndcg_at_1_max": 0.09839818901557038, + "nauc_ndcg_at_1_std": 1.0, + "nauc_ndcg_at_20_diff1": 0.3330912830350423, + "nauc_ndcg_at_20_max": 0.36298197701132934, + "nauc_ndcg_at_20_std": 0.6588880474018856, + "nauc_ndcg_at_3_diff1": 0.6610449563695936, + "nauc_ndcg_at_3_max": 0.09839818901557038, + "nauc_ndcg_at_3_std": 1.0, + "nauc_ndcg_at_5_diff1": 0.6610449563695936, + "nauc_ndcg_at_5_max": 0.09839818901557038, + "nauc_ndcg_at_5_std": 1.0, + "nauc_precision_at_1000_diff1": 0.21618180421784106, + "nauc_precision_at_1000_max": 0.19663911398875364, + "nauc_precision_at_1000_std": 0.6300616078101199, + "nauc_precision_at_100_diff1": 0.2633008669431452, + "nauc_precision_at_100_max": 0.35813105058108047, + "nauc_precision_at_100_std": 0.5625569512563295, + "nauc_precision_at_10_diff1": 0.31227200390917714, + "nauc_precision_at_10_max": 0.3688258547365643, + "nauc_precision_at_10_std": 0.6910567239150112, + "nauc_precision_at_1_diff1": 0.6610449563695936, + "nauc_precision_at_1_max": 0.09839818901557038, + "nauc_precision_at_1_std": 1.0, + "nauc_precision_at_20_diff1": 0.24987842391547985, + "nauc_precision_at_20_max": 0.4262053224877857, + "nauc_precision_at_20_std": 0.5596936671568136, + "nauc_precision_at_3_diff1": 0.6610449563695937, + "nauc_precision_at_3_max": 0.09839818901557038, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 0.6610449563695937, + "nauc_precision_at_5_max": 0.09839818901557036, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": 0.21618180421784144, + "nauc_recall_at_1000_max": 0.1966391139887544, + "nauc_recall_at_1000_std": 0.6300616078101204, + "nauc_recall_at_100_diff1": 0.2633008669431453, + "nauc_recall_at_100_max": 0.3581310505810804, + "nauc_recall_at_100_std": 0.5625569512563297, + "nauc_recall_at_10_diff1": 0.3122720039091771, + "nauc_recall_at_10_max": 0.3688258547365644, + "nauc_recall_at_10_std": 0.691056723915011, + "nauc_recall_at_1_diff1": 0.6610449563695936, + "nauc_recall_at_1_max": 0.09839818901557038, + "nauc_recall_at_1_std": 1.0, + "nauc_recall_at_20_diff1": 0.24987842391547993, + "nauc_recall_at_20_max": 0.4262053224877856, + "nauc_recall_at_20_std": 0.5596936671568133, + "nauc_recall_at_3_diff1": 0.6610449563695936, + "nauc_recall_at_3_max": 0.09839818901557038, + "nauc_recall_at_3_std": 1.0, + "nauc_recall_at_5_diff1": 0.6610449563695936, + "nauc_recall_at_5_max": 0.09839818901557038, + "nauc_recall_at_5_std": 1.0, + "ndcg_at_1": 0.00093, + "ndcg_at_10": 0.00209, + "ndcg_at_100": 0.0128, + "ndcg_at_1000": 0.06231, + "ndcg_at_20": 0.00349, + "ndcg_at_3": 0.00093, + "ndcg_at_5": 0.00093, + "precision_at_1": 0.00093, + "precision_at_10": 0.00047, + "precision_at_100": 0.00065, + "precision_at_1000": 0.00049, + "precision_at_20": 0.00051, + "precision_at_3": 0.00031, + "precision_at_5": 0.00019, + "recall_at_1": 0.00093, + "recall_at_10": 0.00466, + "recall_at_100": 0.06524, + "recall_at_1000": 0.49301, + "recall_at_20": 0.01025, + "recall_at_3": 0.00093, + "recall_at_5": 0.00093 + } + ] + }, + "task_name": "BLINKIT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2TRetrieval.json new file mode 100644 index 0000000000..5454017683 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "d8c3e53160860f76de73ed3041a8593672fe5928", + "evaluation_time": 7.942275285720825, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.45833, + "cv_recall_at_10": 0.75939, + "cv_recall_at_100": 0.94894, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.82923, + "cv_recall_at_3": 0.61737, + "cv_recall_at_5": 0.67547, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.59091, + "map_at_1": 0.41862, + "map_at_10": 0.53693, + "map_at_100": 0.54643, + "map_at_1000": 0.54678, + "map_at_20": 0.54267, + "map_at_3": 0.50797, + "map_at_5": 0.52367, + "mrr_at_1": 0.4583333333333333, + "mrr_at_10": 0.5535262500931519, + "mrr_at_100": 0.5614522827750004, + "mrr_at_1000": 0.5617553026170138, + "mrr_at_20": 0.5583628555240837, + "mrr_at_3": 0.528951486697966, + "mrr_at_5": 0.542243740219093, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4805358684051926, + "nauc_cv_recall_at_100_max": 0.5649594006452747, + "nauc_cv_recall_at_100_std": 0.32360974466378195, + "nauc_cv_recall_at_10_diff1": 0.5013034837210644, + "nauc_cv_recall_at_10_max": 0.4802128509459776, + "nauc_cv_recall_at_10_std": 0.09412450554170432, + "nauc_cv_recall_at_1_diff1": 0.636898011304436, + "nauc_cv_recall_at_1_max": 0.44436198490104667, + "nauc_cv_recall_at_1_std": -0.06652018253703539, + "nauc_cv_recall_at_20_diff1": 0.47564828778050955, + "nauc_cv_recall_at_20_max": 0.5259584288342394, + "nauc_cv_recall_at_20_std": 0.1768940709215821, + "nauc_cv_recall_at_3_diff1": 0.5205869375457675, + "nauc_cv_recall_at_3_max": 0.450051437202706, + "nauc_cv_recall_at_3_std": -0.029414267124329728, + "nauc_cv_recall_at_5_diff1": 0.5001251296481215, + "nauc_cv_recall_at_5_max": 0.4693890552265276, + "nauc_cv_recall_at_5_std": 0.013796336174200773, + "nauc_map_at_1000_diff1": 0.5818434859115754, + "nauc_map_at_1000_max": 0.4430387835643193, + "nauc_map_at_1000_std": -0.04099022306215756, + "nauc_map_at_100_diff1": 0.5817200322199132, + "nauc_map_at_100_max": 0.44314688497711546, + "nauc_map_at_100_std": -0.040679605719069825, + "nauc_map_at_10_diff1": 0.5806678508477698, + "nauc_map_at_10_max": 0.4427501241681824, + "nauc_map_at_10_std": -0.04102488719045883, + "nauc_map_at_1_diff1": 0.6363475951973468, + "nauc_map_at_1_max": 0.3978201419843627, + "nauc_map_at_1_std": -0.07535068057179833, + "nauc_map_at_20_diff1": 0.5811344454158822, + "nauc_map_at_20_max": 0.4440615143544749, + "nauc_map_at_20_std": -0.03955530709812017, + "nauc_map_at_3_diff1": 0.583539903148956, + "nauc_map_at_3_max": 0.4347198346189301, + "nauc_map_at_3_std": -0.05914956536629059, + "nauc_map_at_5_diff1": 0.5808229323614098, + "nauc_map_at_5_max": 0.44138202029415413, + "nauc_map_at_5_std": -0.05072257931247744, + "nauc_mrr_at_1000_diff1": 0.5881203904403035, + "nauc_mrr_at_1000_max": 0.4484481284666203, + "nauc_mrr_at_1000_std": -0.041015672924594425, + "nauc_mrr_at_100_diff1": 0.5880202802699318, + "nauc_mrr_at_100_max": 0.4485244130022475, + "nauc_mrr_at_100_std": -0.04073803891469471, + "nauc_mrr_at_10_diff1": 0.5872948949505256, + "nauc_mrr_at_10_max": 0.44803331196596113, + "nauc_mrr_at_10_std": -0.040942756000374045, + "nauc_mrr_at_1_diff1": 0.636898011304436, + "nauc_mrr_at_1_max": 0.44436198490104667, + "nauc_mrr_at_1_std": -0.06652018253703539, + "nauc_mrr_at_20_diff1": 0.5873601429560038, + "nauc_mrr_at_20_max": 0.4491342783654011, + "nauc_mrr_at_20_std": -0.039490970673790156, + "nauc_mrr_at_3_diff1": 0.5876254651867967, + "nauc_mrr_at_3_max": 0.4441519579742525, + "nauc_mrr_at_3_std": -0.05530094979107773, + "nauc_mrr_at_5_diff1": 0.5853848397651132, + "nauc_mrr_at_5_max": 0.44757418986482955, + "nauc_mrr_at_5_std": -0.047728167237815616, + "nauc_ndcg_at_1000_diff1": 0.5746984338116712, + "nauc_ndcg_at_1000_max": 0.44886091973812225, + "nauc_ndcg_at_1000_std": -0.02776328674880611, + "nauc_ndcg_at_100_diff1": 0.5718934034391514, + "nauc_ndcg_at_100_max": 0.4519133095623923, + "nauc_ndcg_at_100_std": -0.01859333456676269, + "nauc_ndcg_at_10_diff1": 0.5648965674150932, + "nauc_ndcg_at_10_max": 0.4509288057994942, + "nauc_ndcg_at_10_std": -0.01760326906724774, + "nauc_ndcg_at_1_diff1": 0.636898011304436, + "nauc_ndcg_at_1_max": 0.44436198490104667, + "nauc_ndcg_at_1_std": -0.06652018253703539, + "nauc_ndcg_at_20_diff1": 0.5653913906886788, + "nauc_ndcg_at_20_max": 0.4567155165752001, + "nauc_ndcg_at_20_std": -0.010364885098699228, + "nauc_ndcg_at_3_diff1": 0.5691469740706386, + "nauc_ndcg_at_3_max": 0.4377098844996301, + "nauc_ndcg_at_3_std": -0.05395678784871217, + "nauc_ndcg_at_5_diff1": 0.5637652891437743, + "nauc_ndcg_at_5_max": 0.44804388763273256, + "nauc_ndcg_at_5_std": -0.03867023871561297, + "nauc_precision_at_1000_diff1": -0.1269295688966779, + "nauc_precision_at_1000_max": 0.09576039350822045, + "nauc_precision_at_1000_std": 0.09159920254296734, + "nauc_precision_at_100_diff1": -0.007437154335484724, + "nauc_precision_at_100_max": 0.20039321410244404, + "nauc_precision_at_100_std": 0.1426867944558165, + "nauc_precision_at_10_diff1": 0.2452553975868521, + "nauc_precision_at_10_max": 0.37544079604903624, + "nauc_precision_at_10_std": 0.10584035906560733, + "nauc_precision_at_1_diff1": 0.636898011304436, + "nauc_precision_at_1_max": 0.44436198490104667, + "nauc_precision_at_1_std": -0.06652018253703539, + "nauc_precision_at_20_diff1": 0.16916910475445357, + "nauc_precision_at_20_max": 0.34642136478127744, + "nauc_precision_at_20_std": 0.14261345610270082, + "nauc_precision_at_3_diff1": 0.377439212377142, + "nauc_precision_at_3_max": 0.4175722537265246, + "nauc_precision_at_3_std": -0.001966043546048812, + "nauc_precision_at_5_diff1": 0.32001958998919133, + "nauc_precision_at_5_max": 0.4091917985758222, + "nauc_precision_at_5_std": 0.037734817146087056, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.46741327337617883, + "nauc_recall_at_100_max": 0.5601227994679975, + "nauc_recall_at_100_std": 0.32393562317395125, + "nauc_recall_at_10_diff1": 0.48441585851399666, + "nauc_recall_at_10_max": 0.46628557295962697, + "nauc_recall_at_10_std": 0.08341081487340105, + "nauc_recall_at_1_diff1": 0.6363475951973468, + "nauc_recall_at_1_max": 0.3978201419843627, + "nauc_recall_at_1_std": -0.07535068057179833, + "nauc_recall_at_20_diff1": 0.4621173128401141, + "nauc_recall_at_20_max": 0.5117026275838669, + "nauc_recall_at_20_std": 0.1626250437132122, + "nauc_recall_at_3_diff1": 0.5164962617526161, + "nauc_recall_at_3_max": 0.43204598915930476, + "nauc_recall_at_3_std": -0.04234979378579455, + "nauc_recall_at_5_diff1": 0.49384971068639183, + "nauc_recall_at_5_max": 0.4557447504137923, + "nauc_recall_at_5_std": -0.0012052087838667518, + "ndcg_at_1": 0.45833, + "ndcg_at_10": 0.59091, + "ndcg_at_100": 0.6353, + "ndcg_at_1000": 0.64262, + "ndcg_at_20": 0.61038, + "ndcg_at_3": 0.5348, + "ndcg_at_5": 0.56109, + "precision_at_1": 0.45833, + "precision_at_10": 0.08474, + "precision_at_100": 0.01079, + "precision_at_1000": 0.00114, + "precision_at_20": 0.0466, + "precision_at_3": 0.22418, + "precision_at_5": 0.14906, + "recall_at_1": 0.41862, + "recall_at_10": 0.74169, + "recall_at_100": 0.9464, + "recall_at_1000": 1.0, + "recall_at_20": 0.81641, + "recall_at_3": 0.59165, + "recall_at_5": 0.65278 + } + ] + }, + "task_name": "GLDv2I2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IRetrieval.json new file mode 100644 index 0000000000..13200b7196 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "a424cd523ffb157b69a875fb5e71c1d51be54089", + "evaluation_time": 82.47818303108215, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.0404, + "cv_recall_at_10": 0.19939, + "cv_recall_at_100": 0.4748, + "cv_recall_at_1000": 0.67811, + "cv_recall_at_20": 0.29018, + "cv_recall_at_3": 0.08905, + "cv_recall_at_5": 0.12858, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.1073, + "map_at_1": 0.0404, + "map_at_10": 0.07947, + "map_at_100": 0.09013, + "map_at_1000": 0.09115, + "map_at_20": 0.0858, + "map_at_3": 0.06118, + "map_at_5": 0.07015, + "mrr_at_1": 0.04039965247610773, + "mrr_at_10": 0.07948695406340925, + "mrr_at_100": 0.09015226912980688, + "mrr_at_1000": 0.09117557368465125, + "mrr_at_20": 0.08582291845210348, + "mrr_at_3": 0.06117868520127424, + "mrr_at_5": 0.07017086591369824, + "nauc_cv_recall_at_1000_diff1": 0.10207641080088928, + "nauc_cv_recall_at_1000_max": 0.47551950475797095, + "nauc_cv_recall_at_1000_std": 0.667878501495164, + "nauc_cv_recall_at_100_diff1": 0.10625659946342164, + "nauc_cv_recall_at_100_max": 0.429394897766257, + "nauc_cv_recall_at_100_std": 0.6428953991022838, + "nauc_cv_recall_at_10_diff1": 0.1561354794756864, + "nauc_cv_recall_at_10_max": 0.3230948830435383, + "nauc_cv_recall_at_10_std": 0.4849191311442782, + "nauc_cv_recall_at_1_diff1": 0.31664206223566443, + "nauc_cv_recall_at_1_max": 0.17966444353568511, + "nauc_cv_recall_at_1_std": 0.39566281510890866, + "nauc_cv_recall_at_20_diff1": 0.12129977218912101, + "nauc_cv_recall_at_20_max": 0.38830333447601323, + "nauc_cv_recall_at_20_std": 0.5541829899702385, + "nauc_cv_recall_at_3_diff1": 0.19875202658849309, + "nauc_cv_recall_at_3_max": 0.28302032898711493, + "nauc_cv_recall_at_3_std": 0.4420405932660536, + "nauc_cv_recall_at_5_diff1": 0.1777398492807893, + "nauc_cv_recall_at_5_max": 0.28188387839125345, + "nauc_cv_recall_at_5_std": 0.4596852170744158, + "nauc_map_at_1000_diff1": 0.21083155153769165, + "nauc_map_at_1000_max": 0.2761233937186182, + "nauc_map_at_1000_std": 0.4596747781481127, + "nauc_map_at_100_diff1": 0.21045007881515884, + "nauc_map_at_100_max": 0.2758409491731245, + "nauc_map_at_100_std": 0.45980619206042345, + "nauc_map_at_10_diff1": 0.2198233112203957, + "nauc_map_at_10_max": 0.25983030622113884, + "nauc_map_at_10_std": 0.4388620519482371, + "nauc_map_at_1_diff1": 0.31664206223566443, + "nauc_map_at_1_max": 0.17966444353568511, + "nauc_map_at_1_std": 0.39566281510890866, + "nauc_map_at_20_diff1": 0.21187076846238379, + "nauc_map_at_20_max": 0.2726996242735469, + "nauc_map_at_20_std": 0.4532884201759696, + "nauc_map_at_3_diff1": 0.2424283784615785, + "nauc_map_at_3_max": 0.2405242743804034, + "nauc_map_at_3_std": 0.4204628045064827, + "nauc_map_at_5_diff1": 0.22933322343545606, + "nauc_map_at_5_max": 0.24325667796160028, + "nauc_map_at_5_std": 0.4279703891935171, + "nauc_mrr_at_1000_diff1": 0.2111114841525623, + "nauc_mrr_at_1000_max": 0.27633286230256027, + "nauc_mrr_at_1000_std": 0.4599361510805012, + "nauc_mrr_at_100_diff1": 0.2107295407122133, + "nauc_mrr_at_100_max": 0.27604998501829475, + "nauc_mrr_at_100_std": 0.46006698826843867, + "nauc_mrr_at_10_diff1": 0.22007340887962168, + "nauc_mrr_at_10_max": 0.25997366670711813, + "nauc_mrr_at_10_std": 0.4390697248762385, + "nauc_mrr_at_1_diff1": 0.31664206223566443, + "nauc_mrr_at_1_max": 0.17966444353568511, + "nauc_mrr_at_1_std": 0.39566281510890866, + "nauc_mrr_at_20_diff1": 0.21215153596461336, + "nauc_mrr_at_20_max": 0.2729107668453846, + "nauc_mrr_at_20_std": 0.45355197906522926, + "nauc_mrr_at_3_diff1": 0.2424283784615785, + "nauc_mrr_at_3_max": 0.2405242743804034, + "nauc_mrr_at_3_std": 0.4204628045064827, + "nauc_mrr_at_5_diff1": 0.22960397137313956, + "nauc_mrr_at_5_max": 0.24341652796565305, + "nauc_mrr_at_5_std": 0.42819917818577374, + "nauc_ndcg_at_1000_diff1": 0.17173880474374906, + "nauc_ndcg_at_1000_max": 0.33895563139815, + "nauc_ndcg_at_1000_std": 0.5246453040907454, + "nauc_ndcg_at_100_diff1": 0.1683122269839326, + "nauc_ndcg_at_100_max": 0.33447412438250224, + "nauc_ndcg_at_100_std": 0.5261247976469097, + "nauc_ndcg_at_10_diff1": 0.1941087589715255, + "nauc_ndcg_at_10_max": 0.2852009961647385, + "nauc_ndcg_at_10_std": 0.4571235834817824, + "nauc_ndcg_at_1_diff1": 0.31664206223566443, + "nauc_ndcg_at_1_max": 0.17966444353568511, + "nauc_ndcg_at_1_std": 0.39566281510890866, + "nauc_ndcg_at_20_diff1": 0.17466391431078573, + "nauc_ndcg_at_20_max": 0.31877860873395164, + "nauc_ndcg_at_20_std": 0.49309984249311956, + "nauc_ndcg_at_3_diff1": 0.22782771989085718, + "nauc_ndcg_at_3_max": 0.25457117569804816, + "nauc_ndcg_at_3_std": 0.4275096555899013, + "nauc_ndcg_at_5_diff1": 0.21040054798962565, + "nauc_ndcg_at_5_max": 0.25741771300479144, + "nauc_ndcg_at_5_std": 0.4391783586085749, + "nauc_precision_at_1000_diff1": 0.10207641080088917, + "nauc_precision_at_1000_max": 0.47551950475797033, + "nauc_precision_at_1000_std": 0.6678785014951635, + "nauc_precision_at_100_diff1": 0.10625659946342192, + "nauc_precision_at_100_max": 0.4293948977662575, + "nauc_precision_at_100_std": 0.6428953991022842, + "nauc_precision_at_10_diff1": 0.15613547947568646, + "nauc_precision_at_10_max": 0.32309488304353845, + "nauc_precision_at_10_std": 0.4849191311442785, + "nauc_precision_at_1_diff1": 0.31664206223566443, + "nauc_precision_at_1_max": 0.17966444353568511, + "nauc_precision_at_1_std": 0.39566281510890866, + "nauc_precision_at_20_diff1": 0.12129977218912126, + "nauc_precision_at_20_max": 0.3883033344760135, + "nauc_precision_at_20_std": 0.5541829899702387, + "nauc_precision_at_3_diff1": 0.19875202658849306, + "nauc_precision_at_3_max": 0.283020328987115, + "nauc_precision_at_3_std": 0.44204059326605394, + "nauc_precision_at_5_diff1": 0.17773984928078942, + "nauc_precision_at_5_max": 0.2818838783912535, + "nauc_precision_at_5_std": 0.45968521707441584, + "nauc_recall_at_1000_diff1": 0.10207641080088928, + "nauc_recall_at_1000_max": 0.47551950475797095, + "nauc_recall_at_1000_std": 0.667878501495164, + "nauc_recall_at_100_diff1": 0.10625659946342164, + "nauc_recall_at_100_max": 0.429394897766257, + "nauc_recall_at_100_std": 0.6428953991022838, + "nauc_recall_at_10_diff1": 0.1561354794756864, + "nauc_recall_at_10_max": 0.3230948830435383, + "nauc_recall_at_10_std": 0.4849191311442782, + "nauc_recall_at_1_diff1": 0.31664206223566443, + "nauc_recall_at_1_max": 0.17966444353568511, + "nauc_recall_at_1_std": 0.39566281510890866, + "nauc_recall_at_20_diff1": 0.12129977218912101, + "nauc_recall_at_20_max": 0.38830333447601323, + "nauc_recall_at_20_std": 0.5541829899702385, + "nauc_recall_at_3_diff1": 0.19875202658849309, + "nauc_recall_at_3_max": 0.28302032898711493, + "nauc_recall_at_3_std": 0.4420405932660536, + "nauc_recall_at_5_diff1": 0.1777398492807893, + "nauc_recall_at_5_max": 0.28188387839125345, + "nauc_recall_at_5_std": 0.4596852170744158, + "ndcg_at_1": 0.0404, + "ndcg_at_10": 0.1073, + "ndcg_at_100": 0.16373, + "ndcg_at_1000": 0.19005, + "ndcg_at_20": 0.13029, + "ndcg_at_3": 0.06831, + "ndcg_at_5": 0.08453, + "precision_at_1": 0.0404, + "precision_at_10": 0.01994, + "precision_at_100": 0.00475, + "precision_at_1000": 0.00068, + "precision_at_20": 0.01451, + "precision_at_3": 0.02968, + "precision_at_5": 0.02572, + "recall_at_1": 0.0404, + "recall_at_10": 0.19939, + "recall_at_100": 0.4748, + "recall_at_1000": 0.67811, + "recall_at_20": 0.29018, + "recall_at_3": 0.08905, + "recall_at_5": 0.12858 + } + ] + }, + "task_name": "ImageCoDeT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordI2IRetrieval.json new file mode 100644 index 0000000000..e47a943219 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "d8daad98b4e4896a7f7fa1b3485a22420242d4fc", + "evaluation_time": 29.71091365814209, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.80946, + "cv_recall_at_10": 0.99953, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 0.97636, + "cv_recall_at_5": 0.99291, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.80946, + "map_at_1": 0.00083, + "map_at_10": 0.00583, + "map_at_100": 0.04516, + "map_at_1000": 0.27358, + "map_at_20": 0.01067, + "map_at_3": 0.00211, + "map_at_5": 0.00324, + "mrr_at_1": 0.9985815602836879, + "mrr_at_10": 0.9992119779353821, + "mrr_at_100": 0.9992119779353821, + "mrr_at_1000": 0.9992119779353821, + "mrr_at_20": 0.9992119779353821, + "mrr_at_3": 0.9992119779353821, + "mrr_at_5": 0.9992119779353821, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": 0.5540638703677596, + "nauc_cv_recall_at_10_max": 0.7221878767269868, + "nauc_cv_recall_at_10_std": 0.869246700256431, + "nauc_cv_recall_at_1_diff1": -0.24158864035258856, + "nauc_cv_recall_at_1_max": -0.0420448021592233, + "nauc_cv_recall_at_1_std": 0.04484895202286055, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": -0.2813966714480976, + "nauc_cv_recall_at_3_max": -0.04571848249527183, + "nauc_cv_recall_at_3_std": 0.06606763479339506, + "nauc_cv_recall_at_5_diff1": -0.057737057928295976, + "nauc_cv_recall_at_5_max": 0.017855647720182007, + "nauc_cv_recall_at_5_std": 0.08427153819844212, + "nauc_map_at_1000_diff1": -0.3857995513654731, + "nauc_map_at_1000_max": 0.03808456962512866, + "nauc_map_at_1000_std": -0.06663520492889344, + "nauc_map_at_100_diff1": -0.34437629024420324, + "nauc_map_at_100_max": 0.020553049295469906, + "nauc_map_at_100_std": 0.0556790533768489, + "nauc_map_at_10_diff1": -0.18993364966776907, + "nauc_map_at_10_max": 0.006866799821313959, + "nauc_map_at_10_std": 0.13195237855420855, + "nauc_map_at_1_diff1": -0.05431107143472811, + "nauc_map_at_1_max": 0.016884860597154504, + "nauc_map_at_1_std": 0.11466554606849116, + "nauc_map_at_20_diff1": -0.2414863261116838, + "nauc_map_at_20_max": 0.008935096908909975, + "nauc_map_at_20_std": 0.11557906533166638, + "nauc_map_at_3_diff1": -0.12217591294530498, + "nauc_map_at_3_max": 0.008152934040333987, + "nauc_map_at_3_std": 0.13797559913063503, + "nauc_map_at_5_diff1": -0.13888462292383946, + "nauc_map_at_5_max": 0.007634414930511178, + "nauc_map_at_5_std": 0.13391202111166425, + "nauc_mrr_at_1000_diff1": 0.9476986801026335, + "nauc_mrr_at_1000_max": 1.0, + "nauc_mrr_at_1000_std": 0.9166563630181881, + "nauc_mrr_at_100_diff1": 0.9476986801026335, + "nauc_mrr_at_100_max": 1.0, + "nauc_mrr_at_100_std": 0.9166563630181881, + "nauc_mrr_at_10_diff1": 0.9476986801026335, + "nauc_mrr_at_10_max": 1.0, + "nauc_mrr_at_10_std": 0.9166563630181881, + "nauc_mrr_at_1_diff1": 0.9564155667521437, + "nauc_mrr_at_1_max": 1.0, + "nauc_mrr_at_1_std": 0.9073959589089956, + "nauc_mrr_at_20_diff1": 0.9476986801026335, + "nauc_mrr_at_20_max": 1.0, + "nauc_mrr_at_20_std": 0.9166563630181881, + "nauc_mrr_at_3_diff1": 0.9476986801026335, + "nauc_mrr_at_3_max": 1.0, + "nauc_mrr_at_3_std": 0.9166563630181881, + "nauc_mrr_at_5_diff1": 0.9476986801026335, + "nauc_mrr_at_5_max": 1.0, + "nauc_mrr_at_5_std": 0.9166563630181881, + "nauc_ndcg_at_1000_diff1": -0.29077931122633116, + "nauc_ndcg_at_1000_max": 0.014960510380277934, + "nauc_ndcg_at_1000_std": -0.12483981180901933, + "nauc_ndcg_at_100_diff1": -0.3609816949819758, + "nauc_ndcg_at_100_max": 0.021419925854651446, + "nauc_ndcg_at_100_std": -0.17265752707444715, + "nauc_ndcg_at_10_diff1": -0.27457750496652206, + "nauc_ndcg_at_10_max": 0.0008466224260104365, + "nauc_ndcg_at_10_std": -0.045841557226464144, + "nauc_ndcg_at_1_diff1": 0.9564155667521437, + "nauc_ndcg_at_1_max": 1.0, + "nauc_ndcg_at_1_std": 0.9073959589089956, + "nauc_ndcg_at_20_diff1": -0.3094417772089175, + "nauc_ndcg_at_20_max": 0.009714640004268442, + "nauc_ndcg_at_20_std": -0.08586776230182966, + "nauc_ndcg_at_3_diff1": -0.23757585938494089, + "nauc_ndcg_at_3_max": -0.016387555353346198, + "nauc_ndcg_at_3_std": 0.030386940725562046, + "nauc_ndcg_at_5_diff1": -0.23196460003691158, + "nauc_ndcg_at_5_max": -0.013062943721282708, + "nauc_ndcg_at_5_std": -0.010500172553175749, + "nauc_precision_at_1000_diff1": -0.17619733662463302, + "nauc_precision_at_1000_max": 0.012339801300641195, + "nauc_precision_at_1000_std": -0.1876530479926194, + "nauc_precision_at_100_diff1": -0.3626475300981416, + "nauc_precision_at_100_max": 0.025040872201569076, + "nauc_precision_at_100_std": -0.18897442356689456, + "nauc_precision_at_10_diff1": -0.2782683949086005, + "nauc_precision_at_10_max": 0.006150543325428016, + "nauc_precision_at_10_std": -0.05974378846400665, + "nauc_precision_at_1_diff1": 0.9564155667521437, + "nauc_precision_at_1_max": 1.0, + "nauc_precision_at_1_std": 0.9073959589089956, + "nauc_precision_at_20_diff1": -0.31218967700544853, + "nauc_precision_at_20_max": 0.015101101880288523, + "nauc_precision_at_20_std": -0.10187042490803115, + "nauc_precision_at_3_diff1": -0.2418495400445544, + "nauc_precision_at_3_max": -0.017073069963463458, + "nauc_precision_at_3_std": 0.024904433150907852, + "nauc_precision_at_5_diff1": -0.23061542055201162, + "nauc_precision_at_5_max": -0.013293477378375076, + "nauc_precision_at_5_std": -0.02217796284197703, + "nauc_recall_at_1000_diff1": -0.1758800893625114, + "nauc_recall_at_1000_max": 0.025501344233019, + "nauc_recall_at_1000_std": 0.042539963927428814, + "nauc_recall_at_100_diff1": -0.2486522818740174, + "nauc_recall_at_100_max": 0.019163648439019607, + "nauc_recall_at_100_std": 0.08018065043566762, + "nauc_recall_at_10_diff1": -0.1643890021643603, + "nauc_recall_at_10_max": 0.015612163534741345, + "nauc_recall_at_10_std": 0.12932048693738263, + "nauc_recall_at_1_diff1": -0.05431107143472811, + "nauc_recall_at_1_max": 0.016884860597154504, + "nauc_recall_at_1_std": 0.11466554606849116, + "nauc_recall_at_20_diff1": -0.19900512249205043, + "nauc_recall_at_20_max": 0.013811903907386002, + "nauc_recall_at_20_std": 0.11661397478238389, + "nauc_recall_at_3_diff1": -0.11946531860608983, + "nauc_recall_at_3_max": 0.009964580844462759, + "nauc_recall_at_3_std": 0.13655833592145109, + "nauc_recall_at_5_diff1": -0.1255934399523269, + "nauc_recall_at_5_max": 0.012044411079830297, + "nauc_recall_at_5_std": 0.1305906448336473, + "ndcg_at_1": 0.99858, + "ndcg_at_10": 0.82209, + "ndcg_at_100": 0.74273, + "ndcg_at_1000": 0.56583, + "ndcg_at_20": 0.79415, + "ndcg_at_3": 0.89211, + "ndcg_at_5": 0.8591, + "precision_at_1": 0.99858, + "precision_at_10": 0.79054, + "precision_at_100": 0.72617, + "precision_at_1000": 0.49807, + "precision_at_20": 0.76664, + "precision_at_3": 0.86383, + "precision_at_5": 0.82752, + "recall_at_1": 0.00083, + "recall_at_10": 0.00647, + "recall_at_100": 0.05838, + "recall_at_1000": 0.3868, + "recall_at_20": 0.01248, + "recall_at_3": 0.00214, + "recall_at_5": 0.00341 + } + ] + }, + "task_name": "ROxfordI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisI2IRetrieval.json new file mode 100644 index 0000000000..0f309a07f3 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "bafc3a08fdffd72558021ce3a41250833d7e0e88", + "evaluation_time": 43.56014585494995, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.78348, + "cv_recall_at_10": 0.96558, + "cv_recall_at_100": 0.99812, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.98217, + "cv_recall_at_3": 0.89972, + "cv_recall_at_5": 0.93367, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.78348, + "map_at_1": 0.00183, + "map_at_10": 0.01259, + "map_at_100": 0.08554, + "map_at_1000": 0.22541, + "map_at_20": 0.02257, + "map_at_3": 0.0046, + "map_at_5": 0.00704, + "mrr_at_1": 0.9870150187734669, + "mrr_at_10": 0.9935075093867334, + "mrr_at_100": 0.9935075093867334, + "mrr_at_1000": 0.9935075093867334, + "mrr_at_20": 0.9935075093867334, + "mrr_at_3": 0.9935075093867334, + "mrr_at_5": 0.9935075093867334, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.12564420871970045, + "nauc_cv_recall_at_100_max": 0.3655746256621164, + "nauc_cv_recall_at_100_std": -0.2306365991736563, + "nauc_cv_recall_at_10_diff1": -0.5058662936618029, + "nauc_cv_recall_at_10_max": 0.09207458773253613, + "nauc_cv_recall_at_10_std": 0.4663743642051124, + "nauc_cv_recall_at_1_diff1": -0.4013157463788695, + "nauc_cv_recall_at_1_max": 0.013502025515305102, + "nauc_cv_recall_at_1_std": 0.32242775071432206, + "nauc_cv_recall_at_20_diff1": -0.5198855681306671, + "nauc_cv_recall_at_20_max": 0.10151852705481003, + "nauc_cv_recall_at_20_std": 0.41412564884036973, + "nauc_cv_recall_at_3_diff1": -0.5198700551458323, + "nauc_cv_recall_at_3_max": 0.025885833103837443, + "nauc_cv_recall_at_3_std": 0.4187563971498901, + "nauc_cv_recall_at_5_diff1": -0.5209006482350635, + "nauc_cv_recall_at_5_max": 0.051797269624278094, + "nauc_cv_recall_at_5_std": 0.4360707529546872, + "nauc_map_at_1000_diff1": -0.32518052685711313, + "nauc_map_at_1000_max": 0.0038005812075390727, + "nauc_map_at_1000_std": 0.43958508353436493, + "nauc_map_at_100_diff1": -0.324729620306574, + "nauc_map_at_100_max": 0.003712984860206863, + "nauc_map_at_100_std": 0.3530965583894132, + "nauc_map_at_10_diff1": -0.2865369016119553, + "nauc_map_at_10_max": 0.012519773293503903, + "nauc_map_at_10_std": 0.23475872496725028, + "nauc_map_at_1_diff1": -0.0855732613663, + "nauc_map_at_1_max": 0.023221460373481606, + "nauc_map_at_1_std": 0.03199412568851411, + "nauc_map_at_20_diff1": -0.3089685409009752, + "nauc_map_at_20_max": 0.008164069309369689, + "nauc_map_at_20_std": 0.28138880870717525, + "nauc_map_at_3_diff1": -0.21596259885289204, + "nauc_map_at_3_max": 0.026694429615608712, + "nauc_map_at_3_std": 0.13815328440691188, + "nauc_map_at_5_diff1": -0.2539278390954876, + "nauc_map_at_5_max": 0.01952941733527699, + "nauc_map_at_5_std": 0.18318190468395143, + "nauc_mrr_at_1000_diff1": 1.0, + "nauc_mrr_at_1000_max": -0.13028347793646336, + "nauc_mrr_at_1000_std": -0.28303491792776114, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_100_max": -0.13028347793646336, + "nauc_mrr_at_100_std": -0.28303491792776114, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_10_max": -0.13028347793646336, + "nauc_mrr_at_10_std": -0.28303491792776114, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_1_max": -0.1302834779364721, + "nauc_mrr_at_1_std": -0.2830349179277583, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_20_max": -0.13028347793646336, + "nauc_mrr_at_20_std": -0.28303491792776114, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_3_max": -0.13028347793646336, + "nauc_mrr_at_3_std": -0.28303491792776114, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_5_max": -0.13028347793646336, + "nauc_mrr_at_5_std": -0.28303491792776114, + "nauc_ndcg_at_1000_diff1": -0.3461037602745701, + "nauc_ndcg_at_1000_max": 0.01170706578078914, + "nauc_ndcg_at_1000_std": 0.40425946623874776, + "nauc_ndcg_at_100_diff1": -0.39901516979641344, + "nauc_ndcg_at_100_max": -0.00685883548003498, + "nauc_ndcg_at_100_std": 0.465350992380673, + "nauc_ndcg_at_10_diff1": -0.3955198632793018, + "nauc_ndcg_at_10_max": -0.0037363142445774643, + "nauc_ndcg_at_10_std": 0.38110734984115535, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_1_max": -0.14486457971944697, + "nauc_ndcg_at_1_std": -0.2830349179277583, + "nauc_ndcg_at_20_diff1": -0.41278062763010503, + "nauc_ndcg_at_20_max": -0.004138702255930161, + "nauc_ndcg_at_20_std": 0.4191652248206659, + "nauc_ndcg_at_3_diff1": -0.34367743707797543, + "nauc_ndcg_at_3_max": 0.004103711367444123, + "nauc_ndcg_at_3_std": 0.3101278349486688, + "nauc_ndcg_at_5_diff1": -0.37388771810166976, + "nauc_ndcg_at_5_max": -0.004694760292318884, + "nauc_ndcg_at_5_std": 0.3429402757237829, + "nauc_precision_at_1000_diff1": -0.1804983224177905, + "nauc_precision_at_1000_max": -0.01148320196422333, + "nauc_precision_at_1000_std": 0.2504460959613102, + "nauc_precision_at_100_diff1": -0.3911848294276243, + "nauc_precision_at_100_max": -0.007414470176849031, + "nauc_precision_at_100_std": 0.4673613835957893, + "nauc_precision_at_10_diff1": -0.4052976521224889, + "nauc_precision_at_10_max": -0.00433906156493395, + "nauc_precision_at_10_std": 0.39212256495307285, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_1_max": -0.14486457971944697, + "nauc_precision_at_1_std": -0.2830349179277583, + "nauc_precision_at_20_diff1": -0.4173913678934324, + "nauc_precision_at_20_max": -0.004543026048413097, + "nauc_precision_at_20_std": 0.4299497661264597, + "nauc_precision_at_3_diff1": -0.37619357586668295, + "nauc_precision_at_3_max": 0.006637408448532989, + "nauc_precision_at_3_std": 0.3255087096886432, + "nauc_precision_at_5_diff1": -0.393543490911112, + "nauc_precision_at_5_max": -0.0054501833188476565, + "nauc_precision_at_5_std": 0.3553468241787662, + "nauc_recall_at_1000_diff1": -0.28865460441970253, + "nauc_recall_at_1000_max": 0.02031269144932309, + "nauc_recall_at_1000_std": 0.3581111189760722, + "nauc_recall_at_100_diff1": -0.31737603693043515, + "nauc_recall_at_100_max": 0.010815413934469631, + "nauc_recall_at_100_std": 0.32072251393381573, + "nauc_recall_at_10_diff1": -0.2835809857944858, + "nauc_recall_at_10_max": 0.01631628873177338, + "nauc_recall_at_10_std": 0.2203863362341707, + "nauc_recall_at_1_diff1": -0.0855732613663, + "nauc_recall_at_1_max": 0.023221460373481606, + "nauc_recall_at_1_std": 0.03199412568851411, + "nauc_recall_at_20_diff1": -0.30410786586422606, + "nauc_recall_at_20_max": 0.012058890403941508, + "nauc_recall_at_20_std": 0.2609223499236706, + "nauc_recall_at_3_diff1": -0.22686364377869106, + "nauc_recall_at_3_max": 0.02838523233952593, + "nauc_recall_at_3_std": 0.13768932686394217, + "nauc_recall_at_5_diff1": -0.2568046605350692, + "nauc_recall_at_5_max": 0.02092009985156803, + "nauc_recall_at_5_std": 0.17610456507060399, + "ndcg_at_1": 0.98702, + "ndcg_at_10": 0.76647, + "ndcg_at_100": 0.59394, + "ndcg_at_1000": 0.44945, + "ndcg_at_20": 0.71423, + "ndcg_at_3": 0.8678, + "ndcg_at_5": 0.82287, + "precision_at_1": 0.98702, + "precision_at_10": 0.72226, + "precision_at_100": 0.5574, + "precision_at_1000": 0.25148, + "precision_at_20": 0.67042, + "precision_at_3": 0.83521, + "precision_at_5": 0.7832, + "recall_at_1": 0.00183, + "recall_at_10": 0.01349, + "recall_at_100": 0.10451, + "recall_at_1000": 0.40829, + "recall_at_20": 0.02511, + "recall_at_3": 0.00467, + "recall_at_5": 0.0073 + } + ] + }, + "task_name": "RParisI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VQA2IT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VQA2IT2TRetrieval.json new file mode 100644 index 0000000000..63970704c1 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VQA2IT2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "69882b6ba0b443dd62e633e546725b0f13b7e3aa", + "evaluation_time": 1096.5730390548706, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.00224, + "cv_recall_at_10": 0.01687, + "cv_recall_at_100": 0.07996, + "cv_recall_at_1000": 0.2859, + "cv_recall_at_20": 0.02905, + "cv_recall_at_3": 0.00618, + "cv_recall_at_5": 0.00962, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.0082, + "map_at_1": 0.00224, + "map_at_10": 0.00561, + "map_at_100": 0.0076, + "map_at_1000": 0.00818, + "map_at_20": 0.00644, + "map_at_3": 0.00389, + "map_at_5": 0.00467, + "mrr_at_1": 0.0022392864140627186, + "mrr_at_10": 0.005608912626509676, + "mrr_at_100": 0.007598006845600478, + "mrr_at_1000": 0.008184917799157795, + "mrr_at_20": 0.006437587783214147, + "mrr_at_3": 0.003891537674438876, + "mrr_at_5": 0.00466502452329641, + "nauc_cv_recall_at_1000_diff1": 0.008825837565798213, + "nauc_cv_recall_at_1000_max": 0.06850469648818859, + "nauc_cv_recall_at_1000_std": -0.0651814166780974, + "nauc_cv_recall_at_100_diff1": 0.045034516026494435, + "nauc_cv_recall_at_100_max": 0.004398660634268218, + "nauc_cv_recall_at_100_std": -0.03975561918800372, + "nauc_cv_recall_at_10_diff1": 0.06821726635054984, + "nauc_cv_recall_at_10_max": -0.01603124686173661, + "nauc_cv_recall_at_10_std": -0.08733387779771963, + "nauc_cv_recall_at_1_diff1": 0.17890721176822888, + "nauc_cv_recall_at_1_max": -0.08979014629444568, + "nauc_cv_recall_at_1_std": -0.07645555550866222, + "nauc_cv_recall_at_20_diff1": 0.06573644523017931, + "nauc_cv_recall_at_20_max": -0.008446727739883213, + "nauc_cv_recall_at_20_std": -0.10159027094805619, + "nauc_cv_recall_at_3_diff1": 0.10725987074701412, + "nauc_cv_recall_at_3_max": -0.04337993473508564, + "nauc_cv_recall_at_3_std": -0.06088188555814698, + "nauc_cv_recall_at_5_diff1": 0.08502413140032329, + "nauc_cv_recall_at_5_max": -0.018521868603006558, + "nauc_cv_recall_at_5_std": -0.07681198564756143, + "nauc_map_at_1000_diff1": 0.0857862921438581, + "nauc_map_at_1000_max": -0.022301252824256254, + "nauc_map_at_1000_std": -0.07002071019417072, + "nauc_map_at_100_diff1": 0.09059194638791825, + "nauc_map_at_100_max": -0.02730484396719757, + "nauc_map_at_100_std": -0.07112702467096156, + "nauc_map_at_10_diff1": 0.10561558040018261, + "nauc_map_at_10_max": -0.03887680003815409, + "nauc_map_at_10_std": -0.07384905648271843, + "nauc_map_at_1_diff1": 0.17890721176822888, + "nauc_map_at_1_max": -0.08979014629444568, + "nauc_map_at_1_std": -0.07645555550866222, + "nauc_map_at_20_diff1": 0.09999678477824628, + "nauc_map_at_20_max": -0.033642117064318564, + "nauc_map_at_20_std": -0.08006217604646698, + "nauc_map_at_3_diff1": 0.13264050928045265, + "nauc_map_at_3_max": -0.05878443289453297, + "nauc_map_at_3_std": -0.06106821971711554, + "nauc_map_at_5_diff1": 0.11792502863936245, + "nauc_map_at_5_max": -0.04439031923384928, + "nauc_map_at_5_std": -0.06823912896230401, + "nauc_mrr_at_1000_diff1": 0.08578620852479406, + "nauc_mrr_at_1000_max": -0.022301287770317674, + "nauc_mrr_at_1000_std": -0.07002051705337815, + "nauc_mrr_at_100_diff1": 0.09059184648214072, + "nauc_mrr_at_100_max": -0.02730487433721151, + "nauc_mrr_at_100_std": -0.07112681168567145, + "nauc_mrr_at_10_diff1": 0.10561558040018261, + "nauc_mrr_at_10_max": -0.03887680003815409, + "nauc_mrr_at_10_std": -0.07384905648271843, + "nauc_mrr_at_1_diff1": 0.17890721176822888, + "nauc_mrr_at_1_max": -0.08979014629444568, + "nauc_mrr_at_1_std": -0.07645555550866222, + "nauc_mrr_at_20_diff1": 0.09999678477824628, + "nauc_mrr_at_20_max": -0.033642117064318564, + "nauc_mrr_at_20_std": -0.08006217604646698, + "nauc_mrr_at_3_diff1": 0.13264050928045265, + "nauc_mrr_at_3_max": -0.05878443289453297, + "nauc_mrr_at_3_std": -0.06106821971711554, + "nauc_mrr_at_5_diff1": 0.11792502863936245, + "nauc_mrr_at_5_max": -0.04439031923384928, + "nauc_mrr_at_5_std": -0.06823912896230401, + "nauc_ndcg_at_1000_diff1": 0.03027392253878934, + "nauc_ndcg_at_1000_max": 0.03872756726005947, + "nauc_ndcg_at_1000_std": -0.06339954815558649, + "nauc_ndcg_at_100_diff1": 0.060190435328092255, + "nauc_ndcg_at_100_max": -0.006149217698640977, + "nauc_ndcg_at_100_std": -0.05423336919167363, + "nauc_ndcg_at_10_diff1": 0.08801115068128108, + "nauc_ndcg_at_10_max": -0.027958780559606267, + "nauc_ndcg_at_10_std": -0.0798528780495694, + "nauc_ndcg_at_1_diff1": 0.17890721176822888, + "nauc_ndcg_at_1_max": -0.08979014629444568, + "nauc_ndcg_at_1_std": -0.07645555550866222, + "nauc_ndcg_at_20_diff1": 0.08098580648408388, + "nauc_ndcg_at_20_max": -0.019828494542212433, + "nauc_ndcg_at_20_std": -0.09119555503912527, + "nauc_ndcg_at_3_diff1": 0.12374455660074443, + "nauc_ndcg_at_3_max": -0.05335042807216969, + "nauc_ndcg_at_3_std": -0.06082434994775198, + "nauc_ndcg_at_5_diff1": 0.10480672753283997, + "nauc_ndcg_at_5_max": -0.03413521896711701, + "nauc_ndcg_at_5_std": -0.07135709274825974, + "nauc_precision_at_1000_diff1": 0.00882583756579781, + "nauc_precision_at_1000_max": 0.06850469648818812, + "nauc_precision_at_1000_std": -0.06518141667809778, + "nauc_precision_at_100_diff1": 0.045034516026494456, + "nauc_precision_at_100_max": 0.004398660634268171, + "nauc_precision_at_100_std": -0.039755619188003716, + "nauc_precision_at_10_diff1": 0.06821726635055005, + "nauc_precision_at_10_max": -0.01603124686173649, + "nauc_precision_at_10_std": -0.08733387779771956, + "nauc_precision_at_1_diff1": 0.17890721176822888, + "nauc_precision_at_1_max": -0.08979014629444568, + "nauc_precision_at_1_std": -0.07645555550866222, + "nauc_precision_at_20_diff1": 0.06573644523017923, + "nauc_precision_at_20_max": -0.008446727739883295, + "nauc_precision_at_20_std": -0.10159027094805619, + "nauc_precision_at_3_diff1": 0.10725987074701417, + "nauc_precision_at_3_max": -0.04337993473508582, + "nauc_precision_at_3_std": -0.060881885558147046, + "nauc_precision_at_5_diff1": 0.0850241314003234, + "nauc_precision_at_5_max": -0.01852186860300637, + "nauc_precision_at_5_std": -0.0768119856475613, + "nauc_recall_at_1000_diff1": 0.008825837565798213, + "nauc_recall_at_1000_max": 0.06850469648818859, + "nauc_recall_at_1000_std": -0.0651814166780974, + "nauc_recall_at_100_diff1": 0.045034516026494435, + "nauc_recall_at_100_max": 0.004398660634268218, + "nauc_recall_at_100_std": -0.03975561918800372, + "nauc_recall_at_10_diff1": 0.06821726635054984, + "nauc_recall_at_10_max": -0.01603124686173661, + "nauc_recall_at_10_std": -0.08733387779771963, + "nauc_recall_at_1_diff1": 0.17890721176822888, + "nauc_recall_at_1_max": -0.08979014629444568, + "nauc_recall_at_1_std": -0.07645555550866222, + "nauc_recall_at_20_diff1": 0.06573644523017931, + "nauc_recall_at_20_max": -0.008446727739883213, + "nauc_recall_at_20_std": -0.10159027094805619, + "nauc_recall_at_3_diff1": 0.10725987074701412, + "nauc_recall_at_3_max": -0.04337993473508564, + "nauc_recall_at_3_std": -0.06088188555814698, + "nauc_recall_at_5_diff1": 0.08502413140032329, + "nauc_recall_at_5_max": -0.018521868603006558, + "nauc_recall_at_5_std": -0.07681198564756143, + "ndcg_at_1": 0.00224, + "ndcg_at_10": 0.0082, + "ndcg_at_100": 0.02041, + "ndcg_at_1000": 0.04417, + "ndcg_at_20": 0.01126, + "ndcg_at_3": 0.00448, + "ndcg_at_5": 0.00588, + "precision_at_1": 0.00224, + "precision_at_10": 0.00169, + "precision_at_100": 0.0008, + "precision_at_1000": 0.00029, + "precision_at_20": 0.00145, + "precision_at_3": 0.00206, + "precision_at_5": 0.00192, + "recall_at_1": 0.00224, + "recall_at_10": 0.01687, + "recall_at_100": 0.07996, + "recall_at_1000": 0.2859, + "recall_at_20": 0.02905, + "recall_at_3": 0.00618, + "recall_at_5": 0.00962 + } + ] + }, + "task_name": "VQA2IT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VizWizIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VizWizIT2TRetrieval.json new file mode 100644 index 0000000000..03ec1474f1 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VizWizIT2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "044af162d55f82ab603fa16ffcf7f1e4dbf300e9", + "evaluation_time": 30.84842872619629, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.01343, + "cv_recall_at_10": 0.07131, + "cv_recall_at_100": 0.39569, + "cv_recall_at_1000": 0.91433, + "cv_recall_at_20": 0.12063, + "cv_recall_at_3": 0.0345, + "cv_recall_at_5": 0.04561, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.03809, + "map_at_1": 0.01343, + "map_at_10": 0.02804, + "map_at_100": 0.03742, + "map_at_1000": 0.03951, + "map_at_20": 0.0312, + "map_at_3": 0.02227, + "map_at_5": 0.0247, + "mrr_at_1": 0.013429034498726558, + "mrr_at_10": 0.028036417160056885, + "mrr_at_100": 0.037420008046085314, + "mrr_at_1000": 0.039505486644354056, + "mrr_at_20": 0.031201242210305073, + "mrr_at_3": 0.022265956625762126, + "mrr_at_5": 0.024697074940186765, + "nauc_cv_recall_at_1000_diff1": 0.050583394811374374, + "nauc_cv_recall_at_1000_max": -0.18376667036878835, + "nauc_cv_recall_at_1000_std": 0.1298217845164567, + "nauc_cv_recall_at_100_diff1": 0.027698576038987024, + "nauc_cv_recall_at_100_max": -0.009117749373629837, + "nauc_cv_recall_at_100_std": 0.006829941177572575, + "nauc_cv_recall_at_10_diff1": 0.16070351829818008, + "nauc_cv_recall_at_10_max": -0.06294639384504126, + "nauc_cv_recall_at_10_std": -0.008037027217108009, + "nauc_cv_recall_at_1_diff1": 0.38564376924174965, + "nauc_cv_recall_at_1_max": -0.14158948678291924, + "nauc_cv_recall_at_1_std": -0.04672599956819156, + "nauc_cv_recall_at_20_diff1": 0.08493892106912637, + "nauc_cv_recall_at_20_max": -0.015163802751554916, + "nauc_cv_recall_at_20_std": 0.021963421659727856, + "nauc_cv_recall_at_3_diff1": 0.24266018554188226, + "nauc_cv_recall_at_3_max": -0.09291021576633819, + "nauc_cv_recall_at_3_std": -0.02148263688905047, + "nauc_cv_recall_at_5_diff1": 0.21381393860129796, + "nauc_cv_recall_at_5_max": -0.11435865792008686, + "nauc_cv_recall_at_5_std": -0.019942360767102027, + "nauc_map_at_1000_diff1": 0.20370724207547133, + "nauc_map_at_1000_max": -0.07702226966924787, + "nauc_map_at_1000_std": -0.014549893563493205, + "nauc_map_at_100_diff1": 0.20105115841633822, + "nauc_map_at_100_max": -0.0738589428173749, + "nauc_map_at_100_std": -0.015012934920928511, + "nauc_map_at_10_diff1": 0.2501026724956375, + "nauc_map_at_10_max": -0.09835226805301363, + "nauc_map_at_10_std": -0.024470727792863956, + "nauc_map_at_1_diff1": 0.38564376924174965, + "nauc_map_at_1_max": -0.14158948678291924, + "nauc_map_at_1_std": -0.04672599956819156, + "nauc_map_at_20_diff1": 0.2244173819284951, + "nauc_map_at_20_max": -0.08444395428873437, + "nauc_map_at_20_std": -0.016020286898738696, + "nauc_map_at_3_diff1": 0.2910544924422312, + "nauc_map_at_3_max": -0.10878289402033367, + "nauc_map_at_3_std": -0.03140611440353479, + "nauc_map_at_5_diff1": 0.2743756933858256, + "nauc_map_at_5_max": -0.11562364386231114, + "nauc_map_at_5_std": -0.02997097324144088, + "nauc_mrr_at_1000_diff1": 0.20370724207547133, + "nauc_mrr_at_1000_max": -0.07702226966924787, + "nauc_mrr_at_1000_std": -0.014549893563493205, + "nauc_mrr_at_100_diff1": 0.20105115841633822, + "nauc_mrr_at_100_max": -0.0738589428173749, + "nauc_mrr_at_100_std": -0.015012934920928511, + "nauc_mrr_at_10_diff1": 0.2501026724956375, + "nauc_mrr_at_10_max": -0.09835226805301363, + "nauc_mrr_at_10_std": -0.024470727792863956, + "nauc_mrr_at_1_diff1": 0.38564376924174965, + "nauc_mrr_at_1_max": -0.14158948678291924, + "nauc_mrr_at_1_std": -0.04672599956819156, + "nauc_mrr_at_20_diff1": 0.2244173819284951, + "nauc_mrr_at_20_max": -0.08444395428873437, + "nauc_mrr_at_20_std": -0.016020286898738696, + "nauc_mrr_at_3_diff1": 0.2910544924422312, + "nauc_mrr_at_3_max": -0.10878289402033367, + "nauc_mrr_at_3_std": -0.03140611440353479, + "nauc_mrr_at_5_diff1": 0.2743756933858256, + "nauc_mrr_at_5_max": -0.11562364386231114, + "nauc_mrr_at_5_std": -0.02997097324144088, + "nauc_ndcg_at_1000_diff1": 0.13406325540576058, + "nauc_ndcg_at_1000_max": -0.07106041428299001, + "nauc_ndcg_at_1000_std": 0.012519681774848902, + "nauc_ndcg_at_100_diff1": 0.10175722112103643, + "nauc_ndcg_at_100_max": -0.03450295253160438, + "nauc_ndcg_at_100_std": -0.0014070304908460896, + "nauc_ndcg_at_10_diff1": 0.21195276364011353, + "nauc_ndcg_at_10_max": -0.08376357345515494, + "nauc_ndcg_at_10_std": -0.017452778189587872, + "nauc_ndcg_at_1_diff1": 0.38564376924174965, + "nauc_ndcg_at_1_max": -0.14158948678291924, + "nauc_ndcg_at_1_std": -0.04672599956819156, + "nauc_ndcg_at_20_diff1": 0.15751370600120412, + "nauc_ndcg_at_20_max": -0.05224769346714892, + "nauc_ndcg_at_20_std": 0.0017388040866098441, + "nauc_ndcg_at_3_diff1": 0.27420783124692033, + "nauc_ndcg_at_3_max": -0.10323505279682847, + "nauc_ndcg_at_3_std": -0.028002766346549326, + "nauc_ndcg_at_5_diff1": 0.2515210601839793, + "nauc_ndcg_at_5_max": -0.11470427341623049, + "nauc_ndcg_at_5_std": -0.026194743288170313, + "nauc_precision_at_1000_diff1": 0.050583394811373666, + "nauc_precision_at_1000_max": -0.18376667036878533, + "nauc_precision_at_1000_std": 0.1298217845164564, + "nauc_precision_at_100_diff1": 0.027698576038987212, + "nauc_precision_at_100_max": -0.009117749373629547, + "nauc_precision_at_100_std": 0.006829941177572595, + "nauc_precision_at_10_diff1": 0.16070351829818025, + "nauc_precision_at_10_max": -0.06294639384504118, + "nauc_precision_at_10_std": -0.008037027217107877, + "nauc_precision_at_1_diff1": 0.38564376924174965, + "nauc_precision_at_1_max": -0.14158948678291924, + "nauc_precision_at_1_std": -0.04672599956819156, + "nauc_precision_at_20_diff1": 0.08493892106912652, + "nauc_precision_at_20_max": -0.015163802751554795, + "nauc_precision_at_20_std": 0.021963421659728005, + "nauc_precision_at_3_diff1": 0.24266018554188226, + "nauc_precision_at_3_max": -0.09291021576633826, + "nauc_precision_at_3_std": -0.02148263688905037, + "nauc_precision_at_5_diff1": 0.2138139386012981, + "nauc_precision_at_5_max": -0.11435865792008668, + "nauc_precision_at_5_std": -0.019942360767101795, + "nauc_recall_at_1000_diff1": 0.050583394811374374, + "nauc_recall_at_1000_max": -0.18376667036878835, + "nauc_recall_at_1000_std": 0.1298217845164567, + "nauc_recall_at_100_diff1": 0.027698576038987024, + "nauc_recall_at_100_max": -0.009117749373629837, + "nauc_recall_at_100_std": 0.006829941177572575, + "nauc_recall_at_10_diff1": 0.16070351829818008, + "nauc_recall_at_10_max": -0.06294639384504126, + "nauc_recall_at_10_std": -0.008037027217108009, + "nauc_recall_at_1_diff1": 0.38564376924174965, + "nauc_recall_at_1_max": -0.14158948678291924, + "nauc_recall_at_1_std": -0.04672599956819156, + "nauc_recall_at_20_diff1": 0.08493892106912637, + "nauc_recall_at_20_max": -0.015163802751554916, + "nauc_recall_at_20_std": 0.021963421659727856, + "nauc_recall_at_3_diff1": 0.24266018554188226, + "nauc_recall_at_3_max": -0.09291021576633819, + "nauc_recall_at_3_std": -0.02148263688905047, + "nauc_recall_at_5_diff1": 0.21381393860129796, + "nauc_recall_at_5_max": -0.11435865792008686, + "nauc_recall_at_5_std": -0.019942360767102027, + "ndcg_at_1": 0.01343, + "ndcg_at_10": 0.03809, + "ndcg_at_100": 0.09961, + "ndcg_at_1000": 0.16333, + "ndcg_at_20": 0.05024, + "ndcg_at_3": 0.02539, + "ndcg_at_5": 0.02987, + "precision_at_1": 0.01343, + "precision_at_10": 0.00713, + "precision_at_100": 0.00396, + "precision_at_1000": 0.00091, + "precision_at_20": 0.00603, + "precision_at_3": 0.0115, + "precision_at_5": 0.00912, + "recall_at_1": 0.01343, + "recall_at_10": 0.07131, + "recall_at_100": 0.39569, + "recall_at_1000": 0.91433, + "recall_at_20": 0.12063, + "recall_at_3": 0.0345, + "recall_at_5": 0.04561 + } + ] + }, + "task_name": "VizWizIT2TRetrieval" +} \ No newline at end of file From a3ec14d954ac4eadaee8a72b0f466e7befef2cd1 Mon Sep 17 00:00:00 2001 From: Jamie-Stirling <36764530+Jamie-Stirling@users.noreply.github.com> Date: Sun, 20 Oct 2024 21:00:37 +0100 Subject: [PATCH 077/154] [mieb] Add any2any multiple choice evaluator and abstask (and one task) (#1301) * wip: start adding BLIP models * add other blip variants * wip: add blip2_models.py * make lint * wip: implement blip2 wrapper * feat: add blip2 models, still mismatched names * fix: remove projections from image and text embeddings * make lint * wip: add coco BLIP2 * fix: BLIP2 better zero-shot classification without text_proj and vision_proj * tidy blip2 * add imagenet-dog-15 dataset * tidy and lint * remove unused import * add cluster_accuracy, ari and nmi to Image.ClusteringEvaluator * add imagenet-10 clustering task * add SOPI2IRetrieval * add results forclip on ImageNet10Clustering and ImageNetDog15Clustering * add SOPI2IRetrieval results for clip 32 * add results for clip vit 32/SOPI2IRetrieval * resolve conflict * add RP2kI2IRetrieval dataset * add RP2kI2IRetrieval results with clip-vit-base-patch32 * update image retrieval __init__.py * add RP2kI2IRetrieval and METI2IRetrieval * add METI2IRetreival * add SOP results * make lign * new revision for METI2IRetrieval * make lint * reset corpus chunk size * remove wrong classification import * add Flickr30k T2I and I2T * add Flickr30k T2I retriebal * reduced-size MET revision * fix: add Flickr30k T2I * make lint * add two landmark datasets and results * add Sketchy i2i retrieval * add task metadata * add BLINKIT2IRetrieval dataset * add BLINKIT2TRetrieval * add ImageCoDeT2IRetrieval * make lint * add vizwiz retrieval and results * fix vizwiz duplicate texts * add new vizwiz results * add VQA2 results * add GLD v2 I2T retrieval * add gld v2 i2i retrieval * make lint * add AbsTaskAny2AnyMultiChoice * make lint * remove GLDv2I2IRetrieval * exclude AbsTaskAny2AnyMultiChoice from test_load_data --- .../Image/AbsTaskAny2AnyMultiChoice.py | 450 ++++++++++++++++ mteb/abstasks/__init__.py | 1 + .../Image/Any2AnyMultiChoiceEvaluator.py | 486 ++++++++++++++++++ mteb/evaluation/evaluators/__init__.py | 1 + .../Image/Any2AnyMultiChoice/__init__.py | 3 + .../eng/ImageCoDeT2IMultiChoice.py | 50 ++ mteb/tasks/Image/__init__.py | 1 + mteb/tasks/__init__.py | 1 + .../ImageCoDeT2IMultiChoice.json | 33 ++ tests/test_tasks/test_all_abstasks.py | 2 + 10 files changed, 1028 insertions(+) create mode 100644 mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py create mode 100644 mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py create mode 100644 mteb/tasks/Image/Any2AnyMultiChoice/__init__.py create mode 100644 mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IMultiChoice.json diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py b/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py new file mode 100644 index 0000000000..a8d0dde0ea --- /dev/null +++ b/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py @@ -0,0 +1,450 @@ +from __future__ import annotations + +import json +import logging +import os +from collections import defaultdict +from pathlib import Path +from time import time +from typing import Any + +import tqdm +from datasets import Features, Value, load_dataset +from PIL import Image + +from ...evaluation.evaluators import Any2AnyMultiChoiceEvaluator +from ...load_results.mteb_results import ScoresDict +from ..AbsTask import AbsTask + +logger = logging.getLogger(__name__) + + +class HFDataLoader: + def __init__( + self, + hf_repo: str | None = None, + hf_repo_qrels: str | None = None, + data_folder: str | None = None, + prefix: str | None = None, + corpus_file: str = "corpus.jsonl", + query_file: str = "queries.jsonl", + qrels_folder: str = "qrels", + qrels_file: str = "", + streaming: bool = False, + keep_in_memory: bool = False, + ): + self.corpus = {} + self.queries = {} + self.qrels = {} + self.hf_repo = hf_repo + if hf_repo: + # By default fetch qrels from same repo not a second repo with "-qrels" like in original + self.hf_repo_qrels = hf_repo_qrels if hf_repo_qrels else hf_repo + else: + # data folder would contain these files: + # (1) fiqa/corpus.jsonl (format: jsonlines) + # (2) fiqa/queries.jsonl (format: jsonlines) + # (3) fiqa/qrels/test.tsv (format: tsv ("\t")) + if prefix: + query_file = prefix + "-" + query_file + qrels_folder = prefix + "-" + qrels_folder + + self.corpus_file = ( + os.path.join(data_folder, corpus_file) if data_folder else corpus_file + ) + self.query_file = ( + os.path.join(data_folder, query_file) if data_folder else query_file + ) + self.qrels_folder = ( + os.path.join(data_folder, qrels_folder) if data_folder else None + ) + self.qrels_file = qrels_file + self.streaming = streaming + self.keep_in_memory = keep_in_memory + + @staticmethod + def check(fIn: str, ext: str): + if not os.path.exists(fIn): + raise ValueError(f"File {fIn} not present! Please provide accurate file.") + + if not fIn.endswith(ext): + raise ValueError(f"File {fIn} must be present with extension {ext}") + + def load( + self, split="test" + ) -> tuple[ + dict[str, dict[str, str | Image.Image]], + dict[str, dict[str, str | Image.Image]], + dict[str, dict[str, int]], + ]: + if not self.hf_repo: + self.qrels_file = os.path.join(self.qrels_folder, split + ".tsv") + self.check(fIn=self.corpus_file, ext="jsonl") + self.check(fIn=self.query_file, ext="jsonl") + self.check(fIn=self.qrels_file, ext="tsv") + + if not len(self.corpus): + logger.info("Loading Corpus...") + self._load_corpus() + logger.info("Loaded %d %s Documents.", len(self.corpus), split.upper()) + logger.info("Doc Example: %s", self.corpus[0]) + + if not len(self.queries): + logger.info("Loading Queries...") + self._load_queries(split) + + self._load_qrels(split) + # filter queries with no qrels + qrels_dict = defaultdict(dict) + + def qrels_dict_init(row): + qrels_dict[row["query-id"]][row["corpus-id"]] = int(row["score"]) + + self.qrels.map(qrels_dict_init) + self.qrels = qrels_dict + self.queries = self.queries.filter(lambda x: x["id"] in self.qrels) + logger.info("Loaded %d %s Queries.", len(self.queries), split.upper()) + logger.info("Query Example: %s", self.queries[0]) + + return self.corpus, self.queries, self.qrels + + def load_corpus(self) -> dict[str, dict[str, str]]: + if not self.hf_repo: + self.check(fIn=self.corpus_file, ext="jsonl") + + if not len(self.corpus): + logger.info("Loading Corpus...") + self._load_corpus() + logger.info("Loaded %d %s Documents.", len(self.corpus)) + logger.info("Doc Example: %s", self.corpus[0]) + + return self.corpus + + def _load_corpus(self): + if self.hf_repo: + corpus_ds = load_dataset( + self.hf_repo, + "corpus", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )["corpus"] + else: + corpus_ds = load_dataset( + "json", + data_files=self.corpus_file, + streaming=self.streaming, + keep_in_memory=self.keep_in_memory, + ) + self.corpus = corpus_ds + + def _load_queries(self, split): + if self.hf_repo: + queries_ds = load_dataset( + self.hf_repo, + "query", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )[split] + else: + queries_ds = load_dataset( + "json", + data_files=self.query_file, + streaming=self.streaming, + keep_in_memory=self.keep_in_memory, + ) + self.queries = queries_ds + + def _load_qrels(self, split): + if self.hf_repo: + qrels_ds = load_dataset( + self.hf_repo_qrels, + "qrels", + keep_in_memory=self.keep_in_memory, + streaming=self.streaming, + )[split] + else: + qrels_ds = load_dataset( + "csv", + data_files=self.qrels_file, + delimiter="\t", + keep_in_memory=self.keep_in_memory, + ) + + if "Q0" in qrels_ds.column_names: + qrels_ds = qrels_ds.remove_columns("Q0") + features = Features( + { + "query-id": Value("string"), + "corpus-id": Value("string"), + "score": Value("float"), + } + ) + # Some datasets may have extra columns, e.g. `difficulty` in qrels for FORB. + qrels_ds = qrels_ds.select_columns(["query-id", "corpus-id", "score"]).cast( + features + ) + self.qrels = qrels_ds + + +class AbsTaskAny2AnyMultiChoice(AbsTask): + """Abstract class for Any2Any multiple choice experiments + + This is NOT a retrieval task: there is one correct answer among a set of candidates, which are a subset of the corpus, indicated in qrels with a relevance of 0 + + Child-classes must implement the following properties: + + self.corpus: dict[str, dict[str, str]] + Semantically, it should contain dict[split_name, dict[sample_id, dict[str, str]]] + E.g. {"test": {"document_one": {"_id": "d1", "title": "title", "text": "text"}}} + + self.queries: dict[str, dict[str, Union[str, List[str]]]] + Semantically, it should contain dict[split_name, dict[sample_id, str]] or dict[split_name, dict[sample_id, List[str]]] for conversations + E.g. {"test": {"q1": "query"}} + or {"test": {"q1": ["turn1", "turn2", "turn3"]}} + + self.relevant_docs: dict[str, dict[str, dict[str, int]]] + Semantically, it should contain dict[split_name, dict[sample_id, dict[doc_id, score]]] + E.g.: {"test": {"q1": {"document_one": 1}}} for hard positive samples (the correct choice) + E.g.: {"test": {"q1": {"document_two": 0}}} for hard negative samples (incorrect choices from the same query) + """ + + ignore_identical_ids: bool = False + skip_first_result: bool = False + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = {}, {}, {} + dataset_path = self.metadata_dict["dataset"]["path"] + + for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): + corpus, queries, qrels = HFDataLoader( + hf_repo=dataset_path, + streaming=False, + keep_in_memory=False, + ).load(split=split) + # directly pass in corpus and queries datasets now to prevent loading into memory + # queries = {query["id"]: query for query in queries} + # corpus = {doc["id"]: doc for doc in corpus} + self.corpus[split], self.queries[split], self.relevant_docs[split] = ( + corpus, + queries, + qrels, + ) + + self.data_loaded = True + + def evaluate( + self, + model, + split: str = "test", + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ): + retriever = Any2AnyMultiChoiceEvaluator( + retriever=model, + task_name=self.metadata.name, + encode_kwargs=encode_kwargs, + **kwargs, + ) + + scores = {} + hf_subsets = list(self.hf_subsets) if self.is_multilingual else ["default"] + + for hf_subset in hf_subsets: + logger.info(f"Subset: {hf_subset}") + + if hf_subset == "default": + corpus, queries, relevant_docs = ( + self.corpus[split], + self.queries[split], + self.relevant_docs[split], + ) + else: + corpus, queries, relevant_docs = ( + self.corpus[hf_subset][split], + self.queries[hf_subset][split], + self.relevant_docs[hf_subset][split], + ) + scores[hf_subset] = self._evaluate_subset( + retriever, corpus, queries, relevant_docs, hf_subset, **kwargs + ) + return scores + + def _evaluate_subset( + self, retriever, corpus, queries, relevant_docs, hf_subset: str, **kwargs + ): + start_time = time() + results = retriever(corpus, queries, relevant_docs) + end_time = time() + logger.info(f"Time taken to retrieve: {end_time - start_time:.2f} seconds") + + save_predictions = kwargs.get("save_predictions", False) + export_errors = kwargs.get("export_errors", False) + if save_predictions or export_errors: + output_folder = Path(kwargs.get("output_folder", "results")) + if not os.path.isdir(output_folder): + os.makedirs(output_folder) + + if save_predictions: + top_k = kwargs.get("top_k", None) + if top_k is not None: + for qid in list(results.keys()): + doc_ids = set( + sorted( + results[qid], key=lambda x: results[qid][x], reverse=True + )[:top_k] + ) + results[qid] = { + k: v for k, v in results[qid].items() if k in doc_ids + } + qrels_save_path = ( + output_folder / f"{self.metadata.name}_{hf_subset}_predictions.json" + ) + + with open(qrels_save_path, "w") as f: + json.dump(results, f) + + ndcg, _map, recall, precision, cv_recall, naucs = retriever.evaluate( + relevant_docs, + results, + retriever.k_values, + ignore_identical_ids=self.ignore_identical_ids, + skip_first_result=self.skip_first_result, + ) + mrr, naucs_mrr = retriever.evaluate_custom( + relevant_docs, results, retriever.k_values, "mrr" + ) + scores = { + **{f"ndcg_at_{k.split('@')[1]}": v for (k, v) in ndcg.items()}, + **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, + "accuracy": recall["Recall@1"], + } + self._add_main_score(scores) + + if export_errors: + errors = {} + + top_k = kwargs.get("top_k", 1) + if not save_predictions and top_k == 1: + for qid in results.keys(): + doc_scores = results[qid] + sorted_docs = sorted( + doc_scores.items(), key=lambda x: x[1], reverse=True + )[:top_k] + results[qid] = dict(sorted_docs) + for qid, retrieved_docs in results.items(): + expected_docs = relevant_docs[qid] + false_positives = [ + doc for doc in retrieved_docs if doc not in expected_docs + ] + false_negatives = [ + doc for doc in expected_docs if doc not in retrieved_docs + ] + if false_positives or false_negatives: + errors[qid] = { + "false_positives": false_positives, + "false_negatives": false_negatives, + } + + errors_save_path = ( + output_folder / f"{self.metadata.name}_{hf_subset}_errors.json" + ) + with open(errors_save_path, "w") as f: + json.dump(errors, f) + + return scores + + def _add_main_score(self, scores: ScoresDict) -> None: + scores["main_score"] = scores[self.metadata.main_score] + + def _calculate_metrics_from_split( + self, split: str, hf_subset: str | None = None, compute_overall: bool = False + ): + pass + + def calculate_metadata_metrics(self) -> None: + self.load_data() + + all_details = {} + pbar_split = tqdm.tqdm( + self.metadata_dict["eval_splits"], desc="Processing Splits..." + ) + for split in pbar_split: + pbar_split.set_postfix_str(f"Split: {split}") + print(f"Processing metadata for split {split}") + all_details[split] = {} + if self.is_multilingual: + pbar_lang = tqdm.tqdm( + self.relevant_docs.keys(), desc="Processing Languages..." + ) + for lang in pbar_lang: + pbar_lang.set_postfix_str(f"Language: {lang}") + print(f"Processing metadata for language {lang}") + split_details = process_language( + self.relevant_docs[lang][split], + self.queries[lang][split], + self.corpus[lang][split], + lang, + ) + all_details[split][lang] = split_details + else: + split_details = process_language( + self.relevant_docs[split], self.queries[split], self.corpus[split] + ) + all_details[split] = split_details + + return all_details + + +def process_language(relevant_docs, queries, corpus, lang=None): + """We want to get three pieces of information: + - the number of documents (and their char length) in the corpus + - the number of queries (and their char length) + - the average number of relevant documents per query + """ + query_len, doc_len = calculate_length(queries, corpus) + num_documents = len(corpus) + num_queries = len(queries) + + # number of qrels that are not 0 + num_qrels_non_zero = sum( + sum(1 for doc_id in docs if docs[doc_id] != 0) + for docs in relevant_docs.values() + ) + qrels_per_doc = num_qrels_non_zero / num_queries if num_queries else 0 + + language_description = f" for language {lang}" if lang else "" + print(f"Average document character length{language_description} is {doc_len}") + print(f"Average query character length{language_description} is {query_len}") + print(f"Number of documents{language_description} is {num_documents}") + print(f"Number of queries{language_description} is {num_queries}") + print( + f"Average number of relevant documents per query{language_description} is {qrels_per_doc}" + ) + return { + "average_document_length": doc_len, + "average_query_length": query_len, + "num_documents": num_documents, + "num_queries": num_queries, + "average_relevant_docs_per_query": qrels_per_doc, + } + + +def calculate_length(queries, corpus): + queries_lens = [] + doc_lens = [] + for query in queries.values(): + queries_lens.append(len(query)) + + for doc in corpus.values(): + if isinstance(doc, Image.Image): + doc_lens.append(1.0) # for image append 1. Can perhaps be removed. + + doc_len = sum(doc_lens) / len(doc_lens) if doc_lens else 0 + query_len = sum(queries_lens) / len(queries_lens) if queries_lens else 0 + return query_len, doc_len diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index f70cbd5324..c874bd2214 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -13,6 +13,7 @@ from .AbsTaskSpeedTask import * from .AbsTaskSTS import * from .AbsTaskSummarization import * +from .Image.AbsTaskAny2AnyMultiChoice import * from .Image.AbsTaskAny2AnyRetrieval import * from .Image.AbsTaskAny2TextMultipleChoice import * from .Image.AbsTaskImageClassification import * diff --git a/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py new file mode 100644 index 0000000000..20e8547536 --- /dev/null +++ b/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py @@ -0,0 +1,486 @@ +from __future__ import annotations + +import heapq +import io +import json +import logging +import os +from collections import defaultdict +from typing import Any + +import numpy as np +import pytrec_eval +import torch +from datasets import Dataset +from PIL import Image +from torch.utils.data import DataLoader +from torchvision import transforms + +from mteb.encoder_interface import EncoderWithQueryCorpusEncode + +from ..Evaluator import Evaluator +from ..utils import ( + confidence_scores, + cos_sim, + dot_score, + download, + hole, + mrr, + nAUC, + recall_cap, + top_k_accuracy, +) + +os.environ["TOKENIZERS_PARALLELISM"] = "false" + +logger = logging.getLogger(__name__) + +transform = transforms.Compose([transforms.PILToTensor()]) + + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, hf_dataset, image_column_name: str = "image", transform=None): + self.dataset = hf_dataset + self.transform = transform + self.image_column_name = image_column_name + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + image = self.dataset[idx][self.image_column_name] + if isinstance(image, bytes): + image = Image.open(io.BytesIO(image)) + else: + # Assume the image is already in a usable format (e.g., PIL Image) + image = image + if image.mode != "RGB": + image = image.convert("RGB") + image = self.transform(image) + return image + + +def custom_collate_fn(batch): + return batch + + +# Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/search/dense/exact_search.py#L12 +class Any2AnyMultiChoiceSearch: + def __init__( + self, + model: EncoderWithQueryCorpusEncode, + encode_kwargs: dict[str, Any] = {}, + corpus_chunk_size: int = 20000, + previous_results: str | None = None, + **kwargs: Any, + ): + # Model is class that provides get_text_embeddings() and get_image_embeddings() + self.model = model + self.encode_kwargs = encode_kwargs + + if "batch_size" not in encode_kwargs: + encode_kwargs["batch_size"] = 128 + + self.score_functions = {"cos_sim": cos_sim, "dot": dot_score} + self.score_function_desc = { + "cos_sim": "Cosine Similarity", + "dot": "Dot Product", + } + self.corpus_chunk_size = corpus_chunk_size + self.previous_results = previous_results + self.batch_size = encode_kwargs.get("batch_size") + self.show_progress_bar = encode_kwargs.get("show_progress_bar") + self.save_corpus_embeddings = kwargs.get("save_corpus_embeddings", False) + self.corpus_embeddings = defaultdict(list) + self.results = {} + + if self.previous_results is not None: + self.previous_results = self.load_results_file() + + def search( + self, + corpus: Dataset, # solve memoery issues + queries: Dataset, # solve memoery issues + qrels: Dataset, + top_k: int, + score_function: str, + return_sorted: bool = False, + **kwargs, + ) -> dict[str, dict[str, float]]: + if score_function not in self.score_functions: + raise ValueError( + f"score function: {score_function} must be either (cos_sim) for cosine similarity or (dot) for dot product" + ) + + logger.info("Encoding Queries.") + query_ids = list(queries["id"]) + self.results = {qid: {} for qid in query_ids} + + q_modality = queries[0]["modality"] + + if q_modality == "text": + query_texts = queries["text"] + query_embeddings = self.model.get_text_embeddings( + texts=query_texts, batch_size=self.encode_kwargs["batch_size"] + ) + else: + queries_dataset = ImageDataset( + queries, image_column_name="image", transform=transform + ) + query_image_dataloader = DataLoader( + queries_dataset, + batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=max(1, os.cpu_count() // 2), + ) + if q_modality == "image": + query_embeddings = self.model.get_image_embeddings( + images=query_image_dataloader, + batch_size=self.encode_kwargs["batch_size"], + ) + elif q_modality == "image,text": + query_texts = queries["text"] + query_embeddings = self.model.get_fused_embeddings( + texts=query_texts, + images=query_image_dataloader, + batch_size=self.encode_kwargs["batch_size"], + ) + else: + raise ValueError(f"Unsupported modality: {q_modality}") + + logger.info("Preparing Corpus...") + corpus_ids = list(corpus["id"]) + + corpus_modality = corpus[0]["modality"] + + logger.info("Encoding Corpus in batches... Warning: This might take a while!") + logger.info( + f"Scoring Function: {self.score_function_desc[score_function]} ({score_function})" + ) + + result_heaps = {qid: [] for qid in query_ids} + for chunk_start in range(0, len(corpus), self.corpus_chunk_size): + chunk = corpus.select( + range( + chunk_start, min(chunk_start + self.corpus_chunk_size, len(corpus)) + ) + ) + chunk_ids = corpus_ids[chunk_start : chunk_start + self.corpus_chunk_size] + + if corpus_modality == "text": + corpus_texts = chunk["text"] + sub_corpus_embeddings = self.model.get_text_embeddings( + texts=corpus_texts, batch_size=self.encode_kwargs["batch_size"] + ) + else: + corpus_dataset = ImageDataset( + chunk, image_column_name="image", transform=transform + ) + corpus_image_dataloader = DataLoader( + corpus_dataset, + batch_size=self.encode_kwargs["batch_size"], + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=max(1, os.cpu_count() // 2), + ) + if corpus_modality == "image": + sub_corpus_embeddings = self.model.get_image_embeddings( + images=corpus_image_dataloader, + batch_size=self.encode_kwargs["batch_size"], + ) + elif corpus_modality == "image,text": + corpus_texts = chunk["text"] + sub_corpus_embeddings = self.model.get_fused_embeddings( + texts=corpus_texts, + images=corpus_image_dataloader, + batch_size=self.encode_kwargs["batch_size"], + ) + else: + raise ValueError(f"Unsupported modality: {corpus_modality}") + + cos_scores = self.score_functions[score_function]( + query_embeddings, sub_corpus_embeddings + ) + cos_scores[torch.isnan(cos_scores)] = -1 + + for query_idx in range(len(query_embeddings)): + query_id = query_ids[query_idx] + # discount answers which aren't a multiple choice (where there is a qrel entry for both query and corpus id) + for c_idx, c_id in enumerate(chunk_ids): + if c_id not in qrels[query_id]: + cos_scores[query_idx, c_idx] = -1 + + cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( + cos_scores, + min(top_k, cos_scores.size(1)), + dim=1, + largest=True, + sorted=return_sorted, + ) + cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() + cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() + + for query_itr in range(len(query_embeddings)): + query_id = query_ids[query_itr] + for sub_corpus_id, score in zip( + cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] + ): + corpus_id = chunk_ids[sub_corpus_id] + if len(result_heaps[query_id]) < top_k: + heapq.heappush(result_heaps[query_id], (score, corpus_id)) + else: + heapq.heappushpop(result_heaps[query_id], (score, corpus_id)) + + for qid in result_heaps: + for score, corpus_id in result_heaps[qid]: + self.results[qid][corpus_id] = score + + return self.results + + def load_results_file(self): + # load the first stage results from file in format {qid: {doc_id: score}} + if "https://" in self.previous_results: + # download the file + if not os.path.exists(self.previous_results): + url_descriptor = self.previous_results.split("https://")[-1].replace( + "/", "--" + ) + dest_file = os.path.join( + "results", f"cached_predictions--{url_descriptor}" + ) + os.makedirs(os.path.dirname(os.path.abspath(dest_file)), exist_ok=True) + download(self.previous_results, dest_file) + logger.info( + f"Downloaded the previous results at {self.previous_results} to {dest_file}" + ) + self.previous_results = dest_file + + with open(self.previous_results) as f: + previous_results = json.load(f) + assert isinstance(previous_results, dict) + assert isinstance(previous_results[list(previous_results.keys())[0]], dict) + return previous_results + + +class Any2AnyMultiChoiceEvaluator(Evaluator): + def __init__( + self, + retriever=None, + task_name: str | None = None, + k_values: list[int] = [1, 3, 5, 10, 20, 100, 1000], + score_function: str = "cos_sim", + encode_kwargs: dict[str, Any] = {}, + **kwargs, + ): + super().__init__(**kwargs) + + self.retriever = Any2AnyMultiChoiceSearch( + retriever, encode_kwargs=encode_kwargs, **kwargs + ) + self.k_values = k_values + self.top_k = ( + max(k_values) if "top_k" not in kwargs else kwargs["top_k"] + ) # can lower it if reranking + self.score_function = score_function + self.task_name = task_name + + def __call__( + self, + corpus: dict[str, dict[str, str | Image.Image]], + queries: dict[str, dict[str, str | Image.Image]], + qrels: dict[str, dict[str, int]], + ) -> dict[str, dict[str, float]]: + if not self.retriever: + raise ValueError("Model/Technique has not been provided!") + + return self.retriever.search( + corpus, + queries, + qrels, + self.top_k, + self.score_function, + prompt_name=self.task_name, # type: ignore + ) + + @staticmethod + def evaluate( + qrels: dict[str, dict[str, int]], + results: dict[str, dict[str, float]], + k_values: list[int], + ignore_identical_ids: bool = False, + skip_first_result: bool = False, + ) -> tuple[ + dict[str, float], + dict[str, float], + dict[str, float], + dict[str, float], + dict[str, float], + ]: + if ignore_identical_ids: + logger.debug( + "For evaluation, ``ignore_identical_ids=True`` is set to True, the evaluator will ignore identical query and document ids." + ) + # Remove identical ids from results dict + for qid, rels in results.items(): + for pid in list(rels): + if qid == pid: + results[qid].pop(pid) + else: + logger.debug( + "For evaluation, we DO NOT ignore identical query and document ids (default), please explicitly set ``ignore_identical_ids=True`` to ignore this." + ) + + all_ndcgs, all_aps, all_recalls, all_precisions, all_cv_recalls = ( + {}, + {}, + {}, + {}, + {}, + ) + + for k in k_values: + all_ndcgs[f"NDCG@{k}"] = [] + all_aps[f"MAP@{k}"] = [] + all_recalls[f"Recall@{k}"] = [] + all_precisions[f"P@{k}"] = [] + all_cv_recalls[f"CV_Recall@{k}"] = [] # (new) CV-style Recall + + map_string = "map_cut." + ",".join([str(k) for k in k_values]) + ndcg_string = "ndcg_cut." + ",".join([str(k) for k in k_values]) + recall_string = "recall." + ",".join([str(k) for k in k_values]) + precision_string = "P." + ",".join([str(k) for k in k_values]) + evaluator = pytrec_eval.RelevanceEvaluator( + qrels, {map_string, ndcg_string, recall_string, precision_string} + ) + scores = evaluator.evaluate(results) + + sorted_results = { + qid: sorted(rels.items(), key=lambda item: item[1], reverse=True) + for qid, rels in results.items() + } + + if skip_first_result: + for qid, rels in sorted_results.items(): + sorted_results[qid].pop(0) + + for query_id in scores.keys(): + top_docs = [ + doc_id for doc_id, _ in sorted_results.get(query_id, []) + ] # Sorted list of doc IDs + # we need to discount qrels that have a ground truth score of 0 + relevant_docs = { + key + for key in qrels.get(query_id, {}).keys() + if qrels[query_id][key] != 0 + } + + for k in k_values: + top_k_docs = top_docs[:k] + all_ndcgs[f"NDCG@{k}"].append(scores[query_id]["ndcg_cut_" + str(k)]) + all_aps[f"MAP@{k}"].append(scores[query_id]["map_cut_" + str(k)]) + all_recalls[f"Recall@{k}"].append(scores[query_id]["recall_" + str(k)]) + all_precisions[f"P@{k}"].append(scores[query_id]["P_" + str(k)]) + + if relevant_docs.intersection(top_k_docs): + all_cv_recalls[f"CV_Recall@{k}"].append(1.0) + else: + all_cv_recalls[f"CV_Recall@{k}"].append(0.0) + + ndcg, _map, recall, precision, cv_recall = ( + all_ndcgs.copy(), + all_aps.copy(), + all_recalls.copy(), + all_precisions.copy(), + all_cv_recalls.copy(), + ) + + for k in k_values: + ndcg[f"NDCG@{k}"] = round(sum(ndcg[f"NDCG@{k}"]) / len(scores), 5) + _map[f"MAP@{k}"] = round(sum(_map[f"MAP@{k}"]) / len(scores), 5) + recall[f"Recall@{k}"] = round(sum(recall[f"Recall@{k}"]) / len(scores), 5) + precision[f"P@{k}"] = round(sum(precision[f"P@{k}"]) / len(scores), 5) + cv_recall[f"CV_Recall@{k}"] = round( + sum(cv_recall[f"CV_Recall@{k}"]) / len(scores), 5 + ) + + naucs = Any2AnyMultiChoiceEvaluator.evaluate_abstention( + results, + {**all_ndcgs, **all_aps, **all_recalls, **all_precisions, **all_cv_recalls}, + ) + + return ndcg, _map, recall, precision, cv_recall, naucs + + @staticmethod + def evaluate_custom( + qrels: dict[str, dict[str, int]], + results: dict[str, dict[str, float]], + k_values: list[int], + metric: str, + output_type: str = "all", + ) -> tuple[dict[str, float]]: + if metric.lower() in ["mrr", "mrr@k", "mrr_cut"]: + metric_scores = mrr(qrels, results, k_values, output_type) + + elif metric.lower() in ["recall_cap", "r_cap", "r_cap@k"]: + metric_scores = recall_cap(qrels, results, k_values, output_type) + + elif metric.lower() in ["hole", "hole@k"]: + metric_scores = hole(qrels, results, k_values, output_type) + + elif metric.lower() in [ + "acc", + "top_k_acc", + "accuracy", + "accuracy@k", + "top_k_accuracy", + ]: + metric_scores = top_k_accuracy(qrels, results, k_values, output_type) + + naucs = Any2AnyMultiChoiceEvaluator.evaluate_abstention(results, metric_scores) + metric_scores_avg = {k: sum(v) / len(v) for k, v in metric_scores.items()} + + return metric_scores_avg, naucs + + @staticmethod + def evaluate_abstention( + results: dict[str, dict[str, float]], + metric_scores: dict[str, list[float]], + ) -> dict[str, float]: + """Computes normalized Area Under the Curve on a set of evaluated instances as presented in the paper https://arxiv.org/abs/2402.12997""" + all_sim_scores = [list(results[qid].values()) for qid in list(results.keys())] + all_conf_scores = [ + confidence_scores(sim_scores) for sim_scores in all_sim_scores + ] + conf_fcts = list(all_conf_scores[0].keys()) + all_conf_scores = { + fct: np.array([x[fct] for x in all_conf_scores]) for fct in conf_fcts + } + metric_scores = {k: np.array(v) for k, v in metric_scores.items()} + naucs = {} + + for metric_name, scores in metric_scores.items(): + for fct, conf_scores in all_conf_scores.items(): + naucs[f"nAUC_{metric_name}_{fct}"] = nAUC(conf_scores, scores) + + return naucs + + @staticmethod + def calculate_cv_style_recall( + qrels: dict[str, dict[str, int]], results: dict[str, dict[str, float]], k: int + ) -> dict[str, float]: + """Calculate CV-style recall: Recall is 1 if any relevant document is + retrieved in the top k, otherwise 0. + """ + cv_recalls = {} + for query_id, relevant_docs in qrels.items(): + retrieved_docs = list(results.get(query_id, {}).keys())[ + :k + ] # Retrieve top k documents + if any(doc_id in relevant_docs for doc_id in retrieved_docs): + cv_recalls[query_id] = ( + 1.0 # If any relevant doc is found in top k, recall is 1 + ) + else: + cv_recalls[query_id] = 0.0 # Otherwise, recall is 0 + return cv_recalls diff --git a/mteb/evaluation/evaluators/__init__.py b/mteb/evaluation/evaluators/__init__.py index 2fb90b655f..d6ad94a88d 100644 --- a/mteb/evaluation/evaluators/__init__.py +++ b/mteb/evaluation/evaluators/__init__.py @@ -3,6 +3,7 @@ from .BitextMiningEvaluator import * from .ClassificationEvaluator import * from .ClusteringEvaluator import * +from .Image.Any2AnyMultiChoiceEvaluator import * from .Image.Any2AnyRetrievalEvaluator import * from .Image.Any2TextMultipleChoiceEvaluator import * from .Image.ClassificationEvaluator import * diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py b/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py new file mode 100644 index 0000000000..b317e8cabd --- /dev/null +++ b/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .eng.ImageCoDeT2IMultiChoice import * diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py new file mode 100644 index 0000000000..1f00290cdd --- /dev/null +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class ImageCoDeT2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="ImageCoDeT2IMultiChoice", + description="Identify the correct image from a set of similar images based on a precise caption.", + reference="https://aclanthology.org/2022.acl-long.241.pdf", + dataset={ + "path": "JamieSJS/imagecode-multi", + "revision": "d28adfd8b34fefa546fdf94bdc352622b2575f6c", + }, + type="Retrieval", + category="t2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_1", + date=("2022-05-22", "2022-05-27"), # conference dates + form=["written"], + domains=["Web"], + task_subtypes=["Image Text Retrieval"], + license="CC BY-SA 4.0", + socioeconomic_status="medium", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{krojer2022image, + title={Image retrieval from contextual descriptions}, + author={Krojer, Benno and Adlakha, Vaibhav and Vineet, Vibhav and Goyal, Yash and Ponti, Edoardo and Reddy, Siva}, + journal={arXiv preprint arXiv:2203.15867}, + year={2022} +} +""", + descriptive_stats={ + "n_samples": {"test": 2302}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 23020, + "num_queries": 2302, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/mteb/tasks/Image/__init__.py b/mteb/tasks/Image/__init__.py index 845cc136f3..8f1c2d27f7 100644 --- a/mteb/tasks/Image/__init__.py +++ b/mteb/tasks/Image/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +from .Any2AnyMultiChoice import * from .Any2AnyRetrieval import * from .Any2TextMultipleChoice import * from .Clustering import * diff --git a/mteb/tasks/__init__.py b/mteb/tasks/__init__.py index 0d7d1d5fc0..8d49517136 100644 --- a/mteb/tasks/__init__.py +++ b/mteb/tasks/__init__.py @@ -3,6 +3,7 @@ from .BitextMining import * from .Classification import * from .Clustering import * +from .Image.Any2AnyMultiChoice import * from .Image.Any2AnyRetrieval import * from .Image.Clustering import * from .Image.ImageClassification import * diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IMultiChoice.json new file mode 100644 index 0000000000..1f3e5fbd9e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IMultiChoice.json @@ -0,0 +1,33 @@ +{ + "dataset_revision": "d28adfd8b34fefa546fdf94bdc352622b2575f6c", + "evaluation_time": 459.3987202644348, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.13206, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.13206, + "mrr_at_1": 0.13205907906168549, + "mrr_at_10": 0.32183470550108295, + "mrr_at_100": 0.32183470550108295, + "mrr_at_1000": 0.32183470550108295, + "mrr_at_20": 0.32183470550108295, + "mrr_at_3": 0.2158268172603526, + "mrr_at_5": 0.2607225600926729, + "ndcg_at_1": 0.13206, + "ndcg_at_10": 0.47717, + "ndcg_at_100": 0.47717, + "ndcg_at_1000": 0.47717, + "ndcg_at_20": 0.47717, + "ndcg_at_3": 0.24566, + "ndcg_at_5": 0.32738 + } + ] + }, + "task_name": "ImageCoDeT2IMultiChoice" +} \ No newline at end of file diff --git a/tests/test_tasks/test_all_abstasks.py b/tests/test_tasks/test_all_abstasks.py index c9f1f59ac6..d4c8e44a88 100644 --- a/tests/test_tasks/test_all_abstasks.py +++ b/tests/test_tasks/test_all_abstasks.py @@ -13,6 +13,7 @@ from mteb.abstasks.AbsTaskInstructionRetrieval import AbsTaskInstructionRetrieval from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.AbsTaskSpeedTask import AbsTaskSpeedTask +from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval from mteb.abstasks.MultiSubsetLoader import MultiSubsetLoader from mteb.overview import TASKS_REGISTRY @@ -39,6 +40,7 @@ def test_load_data( or isinstance(task, AbsTaskInstructionRetrieval) or isinstance(task, MultiSubsetLoader) or isinstance(task, AbsTaskSpeedTask) + or isinstance(task, AbsTaskAny2AnyMultiChoice) ): pytest.skip() with patch.object(task, "dataset_transform") as mock_dataset_transform: From b73a1334311279054a25932b44686ce6994edb29 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Tue, 22 Oct 2024 11:18:12 +0300 Subject: [PATCH 078/154] [mieb] Fix FORB dataset (#1306) * correct format * update results * add more results * add more results --- .../Any2AnyRetrieval/eng/FORBI2IRetrieval.py | 2 +- .../Caltech101.json | 48 +++ .../FORBI2IRetrieval.json | 186 ++++++++++ .../FORBI2IRetrieval.json | 344 +++++++++--------- .../FORBI2IRetrieval.json | 186 ++++++++++ 5 files changed, 593 insertions(+), 173 deletions(-) create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Caltech101.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/FORBI2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/FORBI2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py index 051aa324e6..6f19a91f44 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py @@ -11,7 +11,7 @@ class FORBI2I(AbsTaskAny2AnyRetrieval): reference="https://github.com/pxiangwu/FORB", dataset={ "path": "isaacchung/forb_retrieval", - "revision": "336607d5bcc853fb7f7276c2c9721d4b5b1ca8e4", + "revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", }, type="Retrieval", category="i2i", diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Caltech101.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Caltech101.json new file mode 100644 index 0000000000..9cf71ae0fc --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Caltech101.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", + "evaluation_time": 140.4232726097107, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.9374424720578567, + "f1": 0.8896507530328425, + "f1_weighted": 0.9381916904354753, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9374424720578567, + "scores_per_experiment": [ + { + "accuracy": 0.9316239316239316, + "f1": 0.8916896755795425, + "f1_weighted": 0.9320804382676573 + }, + { + "accuracy": 0.9421433267587114, + "f1": 0.8911440787847722, + "f1_weighted": 0.9432545002991353 + }, + { + "accuracy": 0.9349112426035503, + "f1": 0.8901121996677432, + "f1_weighted": 0.9358989633433914 + }, + { + "accuracy": 0.9373767258382643, + "f1": 0.8843712891021908, + "f1_weighted": 0.9381450174798306 + }, + { + "accuracy": 0.9411571334648258, + "f1": 0.8909365220299629, + "f1_weighted": 0.9415795327873617 + } + ] + } + ] + }, + "task_name": "Caltech101" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/FORBI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/FORBI2IRetrieval.json new file mode 100644 index 0000000000..4d3811487d --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/FORBI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", + "evaluation_time": 1242.6785640716553, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.40136, + "cv_recall_at_10": 0.55162, + "cv_recall_at_100": 0.66423, + "cv_recall_at_1000": 0.7397, + "cv_recall_at_20": 0.58755, + "cv_recall_at_3": 0.48317, + "cv_recall_at_5": 0.51185, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.40136, + "map_at_1": 0.40023, + "map_at_10": 0.44883, + "map_at_100": 0.45322, + "map_at_1000": 0.45356, + "map_at_20": 0.45134, + "map_at_3": 0.43694, + "map_at_5": 0.4435, + "mrr_at_1": 0.40135849056603773, + "mrr_at_10": 0.44943953279424914, + "mrr_at_100": 0.4538207184180781, + "mrr_at_1000": 0.4541651229469619, + "mrr_at_20": 0.4519378833805158, + "mrr_at_3": 0.43757232704402443, + "mrr_at_5": 0.44411194968553264, + "nauc_cv_recall_at_1000_diff1": 0.9500334258065578, + "nauc_cv_recall_at_1000_max": -1.2528093194175578, + "nauc_cv_recall_at_1000_std": -1.1856652591596875, + "nauc_cv_recall_at_100_diff1": 0.877926997353908, + "nauc_cv_recall_at_100_max": -0.8870905020174381, + "nauc_cv_recall_at_100_std": -0.8732695918632056, + "nauc_cv_recall_at_10_diff1": 0.8177473807784005, + "nauc_cv_recall_at_10_max": -0.5466647529055957, + "nauc_cv_recall_at_10_std": -0.6554690765334921, + "nauc_cv_recall_at_1_diff1": 0.8246695670414056, + "nauc_cv_recall_at_1_max": -0.3034632255124627, + "nauc_cv_recall_at_1_std": -0.4858988248874936, + "nauc_cv_recall_at_20_diff1": 0.82479953127787, + "nauc_cv_recall_at_20_max": -0.6318860267460972, + "nauc_cv_recall_at_20_std": -0.7024579748752124, + "nauc_cv_recall_at_3_diff1": 0.8072308741555094, + "nauc_cv_recall_at_3_max": -0.4123387172398377, + "nauc_cv_recall_at_3_std": -0.5690279516462133, + "nauc_cv_recall_at_5_diff1": 0.8047845690821774, + "nauc_cv_recall_at_5_max": -0.46313010588875514, + "nauc_cv_recall_at_5_std": -0.5999518918827577, + "nauc_map_at_1000_diff1": 0.821123720578351, + "nauc_map_at_1000_max": -0.380874046553554, + "nauc_map_at_1000_std": -0.5404325379702113, + "nauc_map_at_100_diff1": 0.8209485062871567, + "nauc_map_at_100_max": -0.3800979301805527, + "nauc_map_at_100_std": -0.5397866691260815, + "nauc_map_at_10_diff1": 0.8195428750181966, + "nauc_map_at_10_max": -0.37103423612650216, + "nauc_map_at_10_std": -0.5343816263585759, + "nauc_map_at_1_diff1": 0.8270196566959886, + "nauc_map_at_1_max": -0.30320990154210953, + "nauc_map_at_1_std": -0.4833533184781571, + "nauc_map_at_20_diff1": 0.820009618850825, + "nauc_map_at_20_max": -0.3759310074719576, + "nauc_map_at_20_std": -0.5369871242494627, + "nauc_map_at_3_diff1": 0.8186425667711692, + "nauc_map_at_3_max": -0.3502082517227007, + "nauc_map_at_3_std": -0.5210663781673923, + "nauc_map_at_5_diff1": 0.8178707187282951, + "nauc_map_at_5_max": -0.36111543033237625, + "nauc_map_at_5_std": -0.5277312572627604, + "nauc_mrr_at_1000_diff1": 0.8198732381938787, + "nauc_mrr_at_1000_max": -0.3810706994001967, + "nauc_mrr_at_1000_std": -0.5418434797910091, + "nauc_mrr_at_100_diff1": 0.8196987253945341, + "nauc_mrr_at_100_max": -0.38029428979297347, + "nauc_mrr_at_100_std": -0.5411966208887677, + "nauc_mrr_at_10_diff1": 0.8182978327883684, + "nauc_mrr_at_10_max": -0.3712361113968832, + "nauc_mrr_at_10_std": -0.5358019360627169, + "nauc_mrr_at_1_diff1": 0.8246695670414056, + "nauc_mrr_at_1_max": -0.3034632255124627, + "nauc_mrr_at_1_std": -0.4858988248874936, + "nauc_mrr_at_20_diff1": 0.8187629322864929, + "nauc_mrr_at_20_max": -0.3761251367912585, + "nauc_mrr_at_20_std": -0.538391953833271, + "nauc_mrr_at_3_diff1": 0.8173376655828167, + "nauc_mrr_at_3_max": -0.35042983995133714, + "nauc_mrr_at_3_std": -0.5225353792328652, + "nauc_mrr_at_5_diff1": 0.8166387092255286, + "nauc_mrr_at_5_max": -0.3613393270069024, + "nauc_mrr_at_5_std": -0.5291442800691877, + "nauc_ndcg_at_1000_diff1": 0.8334868024040154, + "nauc_ndcg_at_1000_max": -0.48731436283957213, + "nauc_ndcg_at_1000_std": -0.6163695321444129, + "nauc_ndcg_at_100_diff1": 0.8281459653974711, + "nauc_ndcg_at_100_max": -0.4631929003672613, + "nauc_ndcg_at_100_std": -0.594729965146729, + "nauc_ndcg_at_10_diff1": 0.8186924353738807, + "nauc_ndcg_at_10_max": -0.4106178964102518, + "nauc_ndcg_at_10_std": -0.5618271856100127, + "nauc_ndcg_at_1_diff1": 0.8270196566959886, + "nauc_ndcg_at_1_max": -0.30320990154210953, + "nauc_ndcg_at_1_std": -0.4833533184781571, + "nauc_ndcg_at_20_diff1": 0.8203675518215456, + "nauc_ndcg_at_20_max": -0.42922604110746493, + "nauc_ndcg_at_20_std": -0.5718977180047882, + "nauc_ndcg_at_3_diff1": 0.8158251981916631, + "nauc_ndcg_at_3_max": -0.3659119877900009, + "nauc_ndcg_at_3_std": -0.5331616044670827, + "nauc_ndcg_at_5_diff1": 0.8145904125253537, + "nauc_ndcg_at_5_max": -0.38591697079578874, + "nauc_ndcg_at_5_std": -0.5454027961145099, + "nauc_precision_at_1000_diff1": 0.9500334258065563, + "nauc_precision_at_1000_max": -1.2528093194175558, + "nauc_precision_at_1000_std": -1.1856652591596857, + "nauc_precision_at_100_diff1": 0.877926997353908, + "nauc_precision_at_100_max": -0.8870905020174389, + "nauc_precision_at_100_std": -0.8732695918632062, + "nauc_precision_at_10_diff1": 0.817762219827385, + "nauc_precision_at_10_max": -0.5465388231157847, + "nauc_precision_at_10_std": -0.6552180490907682, + "nauc_precision_at_1_diff1": 0.8270196566959886, + "nauc_precision_at_1_max": -0.30320990154210953, + "nauc_precision_at_1_std": -0.4833533184781571, + "nauc_precision_at_20_diff1": 0.824799531277869, + "nauc_precision_at_20_max": -0.6318860267460973, + "nauc_precision_at_20_std": -0.7024579748752124, + "nauc_precision_at_3_diff1": 0.8075292153589316, + "nauc_precision_at_3_max": -0.4121216998390528, + "nauc_precision_at_3_std": -0.568597762738296, + "nauc_precision_at_5_diff1": 0.8047485091566745, + "nauc_precision_at_5_max": -0.46287592219959217, + "nauc_precision_at_5_std": -0.5997842327013363, + "nauc_recall_at_1000_diff1": 0.9500334258065578, + "nauc_recall_at_1000_max": -1.2528093194175578, + "nauc_recall_at_1000_std": -1.1856652591596875, + "nauc_recall_at_100_diff1": 0.877926997353908, + "nauc_recall_at_100_max": -0.8870905020174381, + "nauc_recall_at_100_std": -0.8732695918632056, + "nauc_recall_at_10_diff1": 0.8177622198273848, + "nauc_recall_at_10_max": -0.5465388231157855, + "nauc_recall_at_10_std": -0.6552180490907689, + "nauc_recall_at_1_diff1": 0.8270196566959886, + "nauc_recall_at_1_max": -0.30320990154210953, + "nauc_recall_at_1_std": -0.4833533184781571, + "nauc_recall_at_20_diff1": 0.82479953127787, + "nauc_recall_at_20_max": -0.6318860267460972, + "nauc_recall_at_20_std": -0.7024579748752124, + "nauc_recall_at_3_diff1": 0.8075292153589314, + "nauc_recall_at_3_max": -0.41212169983905295, + "nauc_recall_at_3_std": -0.5685977627382963, + "nauc_recall_at_5_diff1": 0.8047485091566744, + "nauc_recall_at_5_max": -0.4628759221995928, + "nauc_recall_at_5_std": -0.5997842327013364, + "ndcg_at_1": 0.40023, + "ndcg_at_10": 0.4735, + "ndcg_at_100": 0.49668, + "ndcg_at_1000": 0.50618, + "ndcg_at_20": 0.48261, + "ndcg_at_3": 0.44879, + "ndcg_at_5": 0.46062, + "precision_at_1": 0.40023, + "precision_at_10": 0.05515, + "precision_at_100": 0.00664, + "precision_at_1000": 0.00074, + "precision_at_20": 0.02938, + "precision_at_3": 0.16101, + "precision_at_5": 0.10235, + "recall_at_1": 0.40023, + "recall_at_10": 0.55155, + "recall_at_100": 0.66423, + "recall_at_1000": 0.7397, + "recall_at_20": 0.58755, + "recall_at_3": 0.48302, + "recall_at_5": 0.51177 + } + ] + }, + "task_name": "FORBI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json index e09859a559..9e86b31861 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json @@ -1,184 +1,184 @@ { - "dataset_revision": "ac01fba09e554b68ba4a79dc7ae45415e653a3aa", - "evaluation_time": 1184.2214772701263, + "dataset_revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", + "evaluation_time": 1013.2197253704071, "kg_co2_emissions": null, - "mteb_version": "1.12.90", + "mteb_version": "1.14.21", "scores": { "test": [ { - "cv_recall_at_1": 0.00038, - "cv_recall_at_10": 0.00385, - "cv_recall_at_100": 0.03457, - "cv_recall_at_1000": 0.24649, - "cv_recall_at_20": 0.00823, - "cv_recall_at_3": 0.00083, - "cv_recall_at_5": 0.00166, + "cv_recall_at_1": 0.2997, + "cv_recall_at_10": 0.43985, + "cv_recall_at_100": 0.57849, + "cv_recall_at_1000": 0.70921, + "cv_recall_at_20": 0.47864, + "cv_recall_at_3": 0.37042, + "cv_recall_at_5": 0.40053, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.00038, - "map_at_1": 0.00038, - "map_at_10": 0.00116, - "map_at_100": 0.00197, - "map_at_1000": 0.00259, - "map_at_20": 0.00144, - "map_at_3": 0.00065, - "map_at_5": 0.00088, - "mrr_at_1": 0.0003773584905660377, - "mrr_at_10": 0.0010433063791554353, - "mrr_at_100": 0.0018764402917165674, - "mrr_at_1000": 0.0024966567136127343, - "mrr_at_20": 0.0013360460673738318, - "mrr_at_3": 0.0005786163522012579, - "mrr_at_5": 0.0007635220125786163, - "nauc_cv_recall_at_1000_diff1": 0.22783742518067585, - "nauc_cv_recall_at_1000_max": -0.30094937055349125, - "nauc_cv_recall_at_1000_std": 0.01784167295863668, - "nauc_cv_recall_at_100_diff1": 0.1523855776857887, - "nauc_cv_recall_at_100_max": -0.2333491118159354, - "nauc_cv_recall_at_100_std": 0.07124160645261825, - "nauc_cv_recall_at_10_diff1": 0.07590081914171622, - "nauc_cv_recall_at_10_max": -0.13915413227271056, - "nauc_cv_recall_at_10_std": 0.11458514181095016, - "nauc_cv_recall_at_1_diff1": -0.0763616850013415, - "nauc_cv_recall_at_1_max": -0.2638046686342903, - "nauc_cv_recall_at_1_std": 0.037617386638046785, - "nauc_cv_recall_at_20_diff1": 0.07168369670369706, - "nauc_cv_recall_at_20_max": -0.1409353515014412, - "nauc_cv_recall_at_20_std": 0.12879237303781227, - "nauc_cv_recall_at_3_diff1": 0.08354269824621324, - "nauc_cv_recall_at_3_max": -0.2661170329536308, - "nauc_cv_recall_at_3_std": 0.02573359026270213, - "nauc_cv_recall_at_5_diff1": 0.05917506158987249, - "nauc_cv_recall_at_5_max": -0.23538307681049842, - "nauc_cv_recall_at_5_std": -0.008110349537771025, - "nauc_map_at_1000_diff1": 0.08679252942719358, - "nauc_map_at_1000_max": -0.19008245077835548, - "nauc_map_at_1000_std": 0.07439215236240897, - "nauc_map_at_100_diff1": 0.05299214493219657, - "nauc_map_at_100_max": -0.16858241572701993, - "nauc_map_at_100_std": 0.09152877376316754, - "nauc_map_at_10_diff1": -0.009168108240029638, - "nauc_map_at_10_max": -0.12428032980862758, - "nauc_map_at_10_std": 0.09071826395276378, - "nauc_map_at_1_diff1": -0.0763616850013415, - "nauc_map_at_1_max": -0.2638046686342903, - "nauc_map_at_1_std": 0.037617386638046785, - "nauc_map_at_20_diff1": 0.009578085831426258, - "nauc_map_at_20_max": -0.13374391534901683, - "nauc_map_at_20_std": 0.10562861158352993, - "nauc_map_at_3_diff1": -0.035427545355100974, - "nauc_map_at_3_max": -0.16112154547893748, - "nauc_map_at_3_std": 0.05967885079771333, - "nauc_map_at_5_diff1": -0.04279524182649304, - "nauc_map_at_5_max": -0.12683730006820387, - "nauc_map_at_5_std": 0.053172020254335796, - "nauc_mrr_at_1000_diff1": 0.11254472976600662, - "nauc_mrr_at_1000_max": -0.22511289948916516, - "nauc_mrr_at_1000_std": 0.06254278735292283, - "nauc_mrr_at_100_diff1": 0.0840199114831485, - "nauc_mrr_at_100_max": -0.2120464897313364, - "nauc_mrr_at_100_std": 0.07733222295763165, - "nauc_mrr_at_10_diff1": 0.04776673206413517, - "nauc_mrr_at_10_max": -0.20880049606815707, - "nauc_mrr_at_10_std": 0.061443566470517366, - "nauc_mrr_at_1_diff1": -0.0763616850013415, - "nauc_mrr_at_1_max": -0.2638046686342903, - "nauc_mrr_at_1_std": 0.037617386638046785, - "nauc_mrr_at_20_diff1": 0.05220974403818351, - "nauc_mrr_at_20_max": -0.19611442600889642, - "nauc_mrr_at_20_std": 0.08143751083549026, - "nauc_mrr_at_3_diff1": 0.034915598278134806, - "nauc_mrr_at_3_max": -0.26847563607517416, - "nauc_mrr_at_3_std": 0.02522135766031656, - "nauc_mrr_at_5_diff1": 0.03687636725458146, - "nauc_mrr_at_5_max": -0.2515841256355842, - "nauc_mrr_at_5_std": 0.010362513365887428, - "nauc_ndcg_at_1000_diff1": 0.20529190954097523, - "nauc_ndcg_at_1000_max": -0.27985057845510525, - "nauc_ndcg_at_1000_std": 0.028715467526608203, - "nauc_ndcg_at_100_diff1": 0.12489112426211123, - "nauc_ndcg_at_100_max": -0.21266051793182603, - "nauc_ndcg_at_100_std": 0.08036708473960381, - "nauc_ndcg_at_10_diff1": 0.011495883635102193, - "nauc_ndcg_at_10_max": -0.10236713608768004, - "nauc_ndcg_at_10_std": 0.11503781091876635, - "nauc_ndcg_at_1_diff1": -0.0763616850013415, - "nauc_ndcg_at_1_max": -0.2638046686342903, - "nauc_ndcg_at_1_std": 0.037617386638046785, - "nauc_ndcg_at_20_diff1": 0.04032171735248905, - "nauc_ndcg_at_20_max": -0.12785521817880435, - "nauc_ndcg_at_20_std": 0.13108451777376734, - "nauc_ndcg_at_3_diff1": -0.029332307317860733, - "nauc_ndcg_at_3_max": -0.1479246364175155, - "nauc_ndcg_at_3_std": 0.06371707260127392, - "nauc_ndcg_at_5_diff1": -0.044971001591395644, - "nauc_ndcg_at_5_max": -0.1026097612407452, - "nauc_ndcg_at_5_std": 0.05344340637950877, - "nauc_precision_at_1000_diff1": 0.22783742518067573, - "nauc_precision_at_1000_max": -0.30094937055349136, - "nauc_precision_at_1000_std": 0.017841672958636725, - "nauc_precision_at_100_diff1": 0.15238557768578878, - "nauc_precision_at_100_max": -0.23334911181593537, - "nauc_precision_at_100_std": 0.07124160645261826, - "nauc_precision_at_10_diff1": 0.03404639363855913, - "nauc_precision_at_10_max": -0.08089372393101929, - "nauc_precision_at_10_std": 0.14209820230748604, - "nauc_precision_at_1_diff1": -0.0763616850013415, - "nauc_precision_at_1_max": -0.2638046686342903, - "nauc_precision_at_1_std": 0.037617386638046785, - "nauc_precision_at_20_diff1": 0.06390467595189904, - "nauc_precision_at_20_max": -0.1269775837256385, - "nauc_precision_at_20_std": 0.14869868526965405, - "nauc_precision_at_3_diff1": -0.017729252234215863, - "nauc_precision_at_3_max": -0.12334110751068078, - "nauc_precision_at_3_std": 0.07159810943014612, - "nauc_precision_at_5_diff1": -0.04986634071689092, - "nauc_precision_at_5_max": -0.06921463991493515, - "nauc_precision_at_5_std": 0.053364338312017305, - "nauc_recall_at_1000_diff1": 0.22783742518067585, - "nauc_recall_at_1000_max": -0.30094937055349125, - "nauc_recall_at_1000_std": 0.01784167295863668, - "nauc_recall_at_100_diff1": 0.1523855776857887, - "nauc_recall_at_100_max": -0.2333491118159354, - "nauc_recall_at_100_std": 0.07124160645261825, - "nauc_recall_at_10_diff1": 0.034046393638559026, - "nauc_recall_at_10_max": -0.08089372393101936, - "nauc_recall_at_10_std": 0.1420982023074859, - "nauc_recall_at_1_diff1": -0.0763616850013415, - "nauc_recall_at_1_max": -0.2638046686342903, - "nauc_recall_at_1_std": 0.037617386638046785, - "nauc_recall_at_20_diff1": 0.06390467595189901, - "nauc_recall_at_20_max": -0.12697758372563853, - "nauc_recall_at_20_std": 0.14869868526965388, - "nauc_recall_at_3_diff1": -0.017729252234215977, - "nauc_recall_at_3_max": -0.12334110751068085, - "nauc_recall_at_3_std": 0.07159810943014594, - "nauc_recall_at_5_diff1": -0.049866340716890785, - "nauc_recall_at_5_max": -0.06921463991493515, - "nauc_recall_at_5_std": 0.053364338312017305, - "ndcg_at_1": 0.00038, - "ndcg_at_10": 0.00184, - "ndcg_at_100": 0.00747, - "ndcg_at_1000": 0.0321, - "ndcg_at_20": 0.00288, - "ndcg_at_3": 0.00074, - "ndcg_at_5": 0.00116, - "precision_at_1": 0.00038, - "precision_at_10": 0.00042, - "precision_at_100": 0.00035, - "precision_at_1000": 0.00025, - "precision_at_20": 0.00042, - "precision_at_3": 0.00033, - "precision_at_5": 0.00041, - "recall_at_1": 0.00038, - "recall_at_10": 0.00415, - "recall_at_100": 0.03457, - "recall_at_1000": 0.24649, - "recall_at_20": 0.0083, - "recall_at_3": 0.00098, - "recall_at_5": 0.00204 + "main_score": 0.2997, + "map_at_1": 0.29872, + "map_at_10": 0.34317, + "map_at_100": 0.34826, + "map_at_1000": 0.34882, + "map_at_20": 0.34587, + "map_at_3": 0.33092, + "map_at_5": 0.33784, + "mrr_at_1": 0.2996981132075472, + "mrr_at_10": 0.34371009284216936, + "mrr_at_100": 0.3488068311925895, + "mrr_at_1000": 0.3493644373926243, + "mrr_at_20": 0.3464172969474597, + "mrr_at_3": 0.3314716981132089, + "mrr_at_5": 0.33839622641509476, + "nauc_cv_recall_at_1000_diff1": 0.8843727921961846, + "nauc_cv_recall_at_1000_max": -1.1122059360557914, + "nauc_cv_recall_at_1000_std": -0.9675872247776091, + "nauc_cv_recall_at_100_diff1": 0.7785055286507491, + "nauc_cv_recall_at_100_max": -0.7106703205453045, + "nauc_cv_recall_at_100_std": -0.6128728702791083, + "nauc_cv_recall_at_10_diff1": 0.7465191137528097, + "nauc_cv_recall_at_10_max": -0.45743970495195085, + "nauc_cv_recall_at_10_std": -0.44231729089667093, + "nauc_cv_recall_at_1_diff1": 0.809005297864194, + "nauc_cv_recall_at_1_max": -0.2993329488342445, + "nauc_cv_recall_at_1_std": -0.32620741505098666, + "nauc_cv_recall_at_20_diff1": 0.7469876391343502, + "nauc_cv_recall_at_20_max": -0.5177087431978706, + "nauc_cv_recall_at_20_std": -0.4798695500368616, + "nauc_cv_recall_at_3_diff1": 0.7660283014554723, + "nauc_cv_recall_at_3_max": -0.37414600202429155, + "nauc_cv_recall_at_3_std": -0.3812954637897764, + "nauc_cv_recall_at_5_diff1": 0.7506021664679934, + "nauc_cv_recall_at_5_max": -0.40594245769432563, + "nauc_cv_recall_at_5_std": -0.40001908508400597, + "nauc_map_at_1000_diff1": 0.787800072995429, + "nauc_map_at_1000_max": -0.3566392040651437, + "nauc_map_at_1000_std": -0.36714262152547383, + "nauc_map_at_100_diff1": 0.7874967433485706, + "nauc_map_at_100_max": -0.355689418488149, + "nauc_map_at_100_std": -0.3663899874287131, + "nauc_map_at_10_diff1": 0.7868312493681175, + "nauc_map_at_10_max": -0.34775384038463175, + "nauc_map_at_10_std": -0.36125695985470363, + "nauc_map_at_1_diff1": 0.8126115518742145, + "nauc_map_at_1_max": -0.29913151762869594, + "nauc_map_at_1_std": -0.32523258687122936, + "nauc_map_at_20_diff1": 0.7869431680525095, + "nauc_map_at_20_max": -0.35188890045617494, + "nauc_map_at_20_std": -0.36378010134017036, + "nauc_map_at_3_diff1": 0.7911567236759326, + "nauc_map_at_3_max": -0.333149500020068, + "nauc_map_at_3_std": -0.35085539025987095, + "nauc_map_at_5_diff1": 0.787330722378602, + "nauc_map_at_5_max": -0.34076571407782574, + "nauc_map_at_5_std": -0.35536942851206027, + "nauc_mrr_at_1000_diff1": 0.7860249244782958, + "nauc_mrr_at_1000_max": -0.35671253205293696, + "nauc_mrr_at_1000_std": -0.36767791330321364, + "nauc_mrr_at_100_diff1": 0.7857232346858175, + "nauc_mrr_at_100_max": -0.35576342624777996, + "nauc_mrr_at_100_std": -0.3669254208694941, + "nauc_mrr_at_10_diff1": 0.7850630778152726, + "nauc_mrr_at_10_max": -0.347832581099885, + "nauc_mrr_at_10_std": -0.3617931109970315, + "nauc_mrr_at_1_diff1": 0.809005297864194, + "nauc_mrr_at_1_max": -0.2993329488342445, + "nauc_mrr_at_1_std": -0.32620741505098666, + "nauc_mrr_at_20_diff1": 0.7851670975927381, + "nauc_mrr_at_20_max": -0.3519692193968048, + "nauc_mrr_at_20_std": -0.36432365017895374, + "nauc_mrr_at_3_diff1": 0.7893467497060561, + "nauc_mrr_at_3_max": -0.3332399381221576, + "nauc_mrr_at_3_std": -0.35142740018662944, + "nauc_mrr_at_5_diff1": 0.7855592591112479, + "nauc_mrr_at_5_max": -0.34085078593741425, + "nauc_mrr_at_5_std": -0.3559324773796398, + "nauc_ndcg_at_1000_diff1": 0.7921287476026665, + "nauc_ndcg_at_1000_max": -0.4495754958173373, + "nauc_ndcg_at_1000_std": -0.4379750085804611, + "nauc_ndcg_at_100_diff1": 0.7823270946335801, + "nauc_ndcg_at_100_max": -0.41962827268592534, + "nauc_ndcg_at_100_std": -0.4109985043880115, + "nauc_ndcg_at_10_diff1": 0.7766582486969618, + "nauc_ndcg_at_10_max": -0.37418523977947854, + "nauc_ndcg_at_10_std": -0.38075972176781303, + "nauc_ndcg_at_1_diff1": 0.8126115518742145, + "nauc_ndcg_at_1_max": -0.29913151762869594, + "nauc_ndcg_at_1_std": -0.32523258687122936, + "nauc_ndcg_at_20_diff1": 0.776952347684857, + "nauc_ndcg_at_20_max": -0.38921670424824983, + "nauc_ndcg_at_20_std": -0.3900253543363332, + "nauc_ndcg_at_3_diff1": 0.7846298919901769, + "nauc_ndcg_at_3_max": -0.3438430556258442, + "nauc_ndcg_at_3_std": -0.35876507125063356, + "nauc_ndcg_at_5_diff1": 0.7779001958221061, + "nauc_ndcg_at_5_max": -0.35740366313832533, + "nauc_ndcg_at_5_std": -0.36678964402271136, + "nauc_precision_at_1000_diff1": 0.8843727921961839, + "nauc_precision_at_1000_max": -1.1122059360557912, + "nauc_precision_at_1000_std": -0.967587224777609, + "nauc_precision_at_100_diff1": 0.778505528650749, + "nauc_precision_at_100_max": -0.7106703205453052, + "nauc_precision_at_100_std": -0.6128728702791086, + "nauc_precision_at_10_diff1": 0.7465191137528097, + "nauc_precision_at_10_max": -0.4574397049519504, + "nauc_precision_at_10_std": -0.44231729089667066, + "nauc_precision_at_1_diff1": 0.8126115518742145, + "nauc_precision_at_1_max": -0.29913151762869594, + "nauc_precision_at_1_std": -0.32523258687122936, + "nauc_precision_at_20_diff1": 0.7471224397052061, + "nauc_precision_at_20_max": -0.5176185970858633, + "nauc_precision_at_20_std": -0.47970254372347515, + "nauc_precision_at_3_diff1": 0.7662084419673741, + "nauc_precision_at_3_max": -0.3741336169388402, + "nauc_precision_at_3_std": -0.3811257749890647, + "nauc_precision_at_5_diff1": 0.7506575148707284, + "nauc_precision_at_5_max": -0.40590652706796215, + "nauc_precision_at_5_std": -0.39985853688778367, + "nauc_recall_at_1000_diff1": 0.8843727921961846, + "nauc_recall_at_1000_max": -1.1122059360557914, + "nauc_recall_at_1000_std": -0.9675872247776091, + "nauc_recall_at_100_diff1": 0.7785055286507491, + "nauc_recall_at_100_max": -0.7106703205453045, + "nauc_recall_at_100_std": -0.6128728702791083, + "nauc_recall_at_10_diff1": 0.7465191137528097, + "nauc_recall_at_10_max": -0.45743970495195085, + "nauc_recall_at_10_std": -0.44231729089667093, + "nauc_recall_at_1_diff1": 0.8126115518742145, + "nauc_recall_at_1_max": -0.29913151762869594, + "nauc_recall_at_1_std": -0.32523258687122936, + "nauc_recall_at_20_diff1": 0.7471224397052065, + "nauc_recall_at_20_max": -0.5176185970858634, + "nauc_recall_at_20_std": -0.47970254372347554, + "nauc_recall_at_3_diff1": 0.7662084419673746, + "nauc_recall_at_3_max": -0.37413361693884, + "nauc_recall_at_3_std": -0.38112577498906464, + "nauc_recall_at_5_diff1": 0.750657514870729, + "nauc_recall_at_5_max": -0.4059065270679627, + "nauc_recall_at_5_std": -0.39985853688778417, + "ndcg_at_1": 0.29872, + "ndcg_at_10": 0.36632, + "ndcg_at_100": 0.39432, + "ndcg_at_1000": 0.41057, + "ndcg_at_20": 0.37613, + "ndcg_at_3": 0.34107, + "ndcg_at_5": 0.35351, + "precision_at_1": 0.29872, + "precision_at_10": 0.04398, + "precision_at_100": 0.00578, + "precision_at_1000": 0.00071, + "precision_at_20": 0.02393, + "precision_at_3": 0.12345, + "precision_at_5": 0.08009, + "recall_at_1": 0.29872, + "recall_at_10": 0.43985, + "recall_at_100": 0.57849, + "recall_at_1000": 0.70921, + "recall_at_20": 0.47857, + "recall_at_3": 0.37034, + "recall_at_5": 0.40045 } ] }, diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/FORBI2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/FORBI2IRetrieval.json new file mode 100644 index 0000000000..67200bf463 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/FORBI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", + "evaluation_time": 2148.242924451828, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.47955, + "cv_recall_at_10": 0.64309, + "cv_recall_at_100": 0.72468, + "cv_recall_at_1000": 0.75774, + "cv_recall_at_20": 0.67275, + "cv_recall_at_3": 0.56981, + "cv_recall_at_5": 0.60325, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.47955, + "map_at_1": 0.47826, + "map_at_10": 0.53199, + "map_at_100": 0.53545, + "map_at_1000": 0.53562, + "map_at_20": 0.53409, + "map_at_3": 0.51886, + "map_at_5": 0.52653, + "mrr_at_1": 0.47954716981132073, + "mrr_at_10": 0.5327790955375838, + "mrr_at_100": 0.536231063428097, + "mrr_at_1000": 0.5363963198824263, + "mrr_at_20": 0.5348666258075478, + "mrr_at_3": 0.5196729559748401, + "mrr_at_5": 0.527325786163518, + "nauc_cv_recall_at_1000_diff1": 0.9783622452451709, + "nauc_cv_recall_at_1000_max": -1.3700380127097909, + "nauc_cv_recall_at_1000_std": -1.3622593419555964, + "nauc_cv_recall_at_100_diff1": 0.9367206943403367, + "nauc_cv_recall_at_100_max": -1.1415729064069615, + "nauc_cv_recall_at_100_std": -1.1585751691509931, + "nauc_cv_recall_at_10_diff1": 0.8585716756688088, + "nauc_cv_recall_at_10_max": -0.7579079603295624, + "nauc_cv_recall_at_10_std": -0.8979314486065754, + "nauc_cv_recall_at_1_diff1": 0.8389554763575047, + "nauc_cv_recall_at_1_max": -0.3861612888474326, + "nauc_cv_recall_at_1_std": -0.6155125338004539, + "nauc_cv_recall_at_20_diff1": 0.8841302115144418, + "nauc_cv_recall_at_20_max": -0.8720455440695557, + "nauc_cv_recall_at_20_std": -0.9664298519627592, + "nauc_cv_recall_at_3_diff1": 0.8259560459790033, + "nauc_cv_recall_at_3_max": -0.546319243284802, + "nauc_cv_recall_at_3_std": -0.7634227610142231, + "nauc_cv_recall_at_5_diff1": 0.8299350588708171, + "nauc_cv_recall_at_5_max": -0.6287235649657446, + "nauc_cv_recall_at_5_std": -0.8161938597529815, + "nauc_map_at_1000_diff1": 0.840854892934761, + "nauc_map_at_1000_max": -0.4924278647794887, + "nauc_map_at_1000_std": -0.7022277605230981, + "nauc_map_at_100_diff1": 0.8407441848445416, + "nauc_map_at_100_max": -0.49192560404032804, + "nauc_map_at_100_std": -0.7017634806879928, + "nauc_map_at_10_diff1": 0.8385260397871099, + "nauc_map_at_10_max": -0.4821878658539431, + "nauc_map_at_10_std": -0.6956217111863439, + "nauc_map_at_1_diff1": 0.8418694895112901, + "nauc_map_at_1_max": -0.38442688222770177, + "nauc_map_at_1_std": -0.6125805636780283, + "nauc_map_at_20_diff1": 0.8399206633399918, + "nauc_map_at_20_max": -0.4878232150574702, + "nauc_map_at_20_std": -0.6989058410509833, + "nauc_map_at_3_diff1": 0.8341959402895627, + "nauc_map_at_3_max": -0.45174026022682956, + "nauc_map_at_3_std": -0.676617769201114, + "nauc_map_at_5_diff1": 0.8350923824381429, + "nauc_map_at_5_max": -0.46831637558370504, + "nauc_map_at_5_std": -0.6870541683615817, + "nauc_mrr_at_1000_diff1": 0.8393074209785932, + "nauc_mrr_at_1000_max": -0.4936213541118185, + "nauc_mrr_at_1000_std": -0.7041826758357784, + "nauc_mrr_at_100_diff1": 0.8391972194458248, + "nauc_mrr_at_100_max": -0.4931184490320341, + "nauc_mrr_at_100_std": -0.7037174874420373, + "nauc_mrr_at_10_diff1": 0.8369860440129665, + "nauc_mrr_at_10_max": -0.4833850308584579, + "nauc_mrr_at_10_std": -0.6975876349014124, + "nauc_mrr_at_1_diff1": 0.8389554763575047, + "nauc_mrr_at_1_max": -0.3861612888474326, + "nauc_mrr_at_1_std": -0.6155125338004539, + "nauc_mrr_at_20_diff1": 0.8383772686177878, + "nauc_mrr_at_20_max": -0.4890112340146167, + "nauc_mrr_at_20_std": -0.7008535231923642, + "nauc_mrr_at_3_diff1": 0.8326910963164327, + "nauc_mrr_at_3_max": -0.45292910686532045, + "nauc_mrr_at_3_std": -0.6786157396873091, + "nauc_mrr_at_5_diff1": 0.8335794976849904, + "nauc_mrr_at_5_max": -0.46951576427639513, + "nauc_mrr_at_5_std": -0.689014845411203, + "nauc_ndcg_at_1000_diff1": 0.8571833725542346, + "nauc_ndcg_at_1000_max": -0.6105533656454696, + "nauc_ndcg_at_1000_std": -0.7896000730053025, + "nauc_ndcg_at_100_diff1": 0.854211660299563, + "nauc_ndcg_at_100_max": -0.5955309613288091, + "nauc_ndcg_at_100_std": -0.7753517475583974, + "nauc_ndcg_at_10_diff1": 0.8421946492855936, + "nauc_ndcg_at_10_max": -0.5410015654361832, + "nauc_ndcg_at_10_std": -0.7393165425948717, + "nauc_ndcg_at_1_diff1": 0.8418694895112901, + "nauc_ndcg_at_1_max": -0.38442688222770177, + "nauc_ndcg_at_1_std": -0.6125805636780283, + "nauc_ndcg_at_20_diff1": 0.8475183312428272, + "nauc_ndcg_at_20_max": -0.563016555679603, + "nauc_ndcg_at_20_std": -0.7522312081819034, + "nauc_ndcg_at_3_diff1": 0.8321161349170787, + "nauc_ndcg_at_3_max": -0.47507879358661737, + "nauc_ndcg_at_3_std": -0.6980061182251038, + "nauc_ndcg_at_5_diff1": 0.8337229410549184, + "nauc_ndcg_at_5_max": -0.5059407346638893, + "nauc_ndcg_at_5_std": -0.7175685317009618, + "nauc_precision_at_1000_diff1": 0.9783622452451708, + "nauc_precision_at_1000_max": -1.370038012709789, + "nauc_precision_at_1000_std": -1.362259341955595, + "nauc_precision_at_100_diff1": 0.9367206943403371, + "nauc_precision_at_100_max": -1.141572906406963, + "nauc_precision_at_100_std": -1.1585751691509945, + "nauc_precision_at_10_diff1": 0.8585920210973836, + "nauc_precision_at_10_max": -0.7576550732857764, + "nauc_precision_at_10_std": -0.8975426783303393, + "nauc_precision_at_1_diff1": 0.8418694895112901, + "nauc_precision_at_1_max": -0.38442688222770177, + "nauc_precision_at_1_std": -0.6125805636780283, + "nauc_precision_at_20_diff1": 0.8841302115144408, + "nauc_precision_at_20_max": -0.8720455440695557, + "nauc_precision_at_20_std": -0.9664298519627592, + "nauc_precision_at_3_diff1": 0.8259800230617228, + "nauc_precision_at_3_max": -0.5459658712057691, + "nauc_precision_at_3_std": -0.7627160737347208, + "nauc_precision_at_5_diff1": 0.8298537192402428, + "nauc_precision_at_5_max": -0.6284652527439748, + "nauc_precision_at_5_std": -0.8158750015206129, + "nauc_recall_at_1000_diff1": 0.9783622452451709, + "nauc_recall_at_1000_max": -1.3700380127097909, + "nauc_recall_at_1000_std": -1.3622593419555964, + "nauc_recall_at_100_diff1": 0.9367206943403367, + "nauc_recall_at_100_max": -1.1415729064069615, + "nauc_recall_at_100_std": -1.1585751691509931, + "nauc_recall_at_10_diff1": 0.8585920210973839, + "nauc_recall_at_10_max": -0.7576550732857767, + "nauc_recall_at_10_std": -0.8975426783303397, + "nauc_recall_at_1_diff1": 0.8418694895112901, + "nauc_recall_at_1_max": -0.38442688222770177, + "nauc_recall_at_1_std": -0.6125805636780283, + "nauc_recall_at_20_diff1": 0.8841302115144418, + "nauc_recall_at_20_max": -0.8720455440695557, + "nauc_recall_at_20_std": -0.9664298519627592, + "nauc_recall_at_3_diff1": 0.8259800230617225, + "nauc_recall_at_3_max": -0.5459658712057688, + "nauc_recall_at_3_std": -0.76271607373472, + "nauc_recall_at_5_diff1": 0.829853719240243, + "nauc_recall_at_5_max": -0.628465252743975, + "nauc_recall_at_5_std": -0.8158750015206132, + "ndcg_at_1": 0.47826, + "ndcg_at_10": 0.55876, + "ndcg_at_100": 0.57601, + "ndcg_at_1000": 0.58025, + "ndcg_at_20": 0.56632, + "ndcg_at_3": 0.5319, + "ndcg_at_5": 0.54573, + "precision_at_1": 0.47826, + "precision_at_10": 0.0643, + "precision_at_100": 0.00725, + "precision_at_1000": 0.00076, + "precision_at_20": 0.03364, + "precision_at_3": 0.18986, + "precision_at_5": 0.12063, + "recall_at_1": 0.47826, + "recall_at_10": 0.64302, + "recall_at_100": 0.72468, + "recall_at_1000": 0.75774, + "recall_at_20": 0.67275, + "recall_at_3": 0.56958, + "recall_at_5": 0.60317 + } + ] + }, + "task_name": "FORBI2IRetrieval" +} \ No newline at end of file From a6f306f898224411ab10bc1738fb35132af75450 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:36:06 +0800 Subject: [PATCH 079/154] [mieb] run tasks fix (#1302) * fix ImageTextPair dataloading for large datasets; more compositionality evaluation datasets * fix e5v&vista * task type fix for running tasks * fix wrong meta * run mieb script * script * lint * align --- mteb/abstasks/TaskMetadata.py | 10 ++- mteb/models/e5_v.py | 70 ++++++++++++----- mteb/models/vista_models.py | 75 +++++++++++++------ .../eng/ImageCoDeT2IMultiChoice.py | 2 +- .../eng/BLINKIT2IRetrieval.py | 2 +- .../eng/BLINKIT2TRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py | 2 +- .../eng/CUB200I2IRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/FORBI2IRetrieval.py | 2 +- .../eng/Fashion200kI2TRetrieval.py | 2 +- .../eng/Fashion200kT2IRetrieval.py | 2 +- .../eng/FashionIQIT2IRetrieval.py | 2 +- .../eng/Flickr30kI2TRetrieval.py | 2 +- .../eng/Flickr30kT2IRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py | 2 +- .../eng/HatefulMemesI2TRetrieval.py | 2 +- .../eng/HatefulMemesT2IRetrieval.py | 2 +- .../eng/ImageCoDeT2IRetrieval.py | 2 +- .../eng/InfoSeekIT2ITRetrieval.py | 2 +- .../eng/InfoSeekIT2TRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/METI2IRetrieval.py | 2 +- .../eng/MSCOCOI2TRetrieval.py | 2 +- .../eng/MSCOCOT2IRetrieval.py | 2 +- .../eng/MemotionI2TRetrieval.py | 2 +- .../eng/MemotionT2IRetrieval.py | 2 +- .../eng/NIGHTSI2IRetrieval.py | 2 +- .../eng/OVENIT2ITRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/OVENIT2TRetrieval.py | 2 +- .../eng/ROxfordI2IRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/RP2kI2IRetrieval.py | 2 +- .../eng/RParisI2IRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/SOPI2IRetrieval.py | 2 +- .../eng/SciMMIRI2TRetrieval.py | 2 +- .../eng/SciMMIRT2IRetrieval.py | 2 +- .../eng/SketchyI2IRetrieval.py | 2 +- .../eng/StanfordCarsI2IRetrieval.py | 2 +- .../eng/TUBerlinT2IRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py | 2 +- .../eng/VidoreBenchRetrieval.py | 2 +- .../eng/VisualNewsI2TRetrieval.py | 2 +- .../eng/VisualNewsT2IRetrieval.py | 2 +- .../eng/VizWizIT2TRetrieval.py | 2 +- .../eng/WebQAT2ITRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/WebQAT2TRetrieval.py | 2 +- .../multilingual/WITT2IRetrieval.py | 2 +- .../multilingual/XFlickr30kCoT2IRetrieval.py | 2 +- .../multilingual/XM3600T2IRetrieval.py | 2 +- .../Any2TextMultipleChoice/eng/CVBench.py | 24 +++--- mteb/tasks/Image/Clustering/eng/CIFAR.py | 2 +- mteb/tasks/Image/Clustering/eng/ImageNet.py | 2 +- .../Image/Clustering/eng/TinyImageNet.py | 2 +- .../eng/BirdsnapClassification.py | 2 +- .../Image/ImageClassification/eng/CIFAR.py | 2 +- .../eng/Caltech101Classification.py | 2 +- .../eng/Country211Classification.py | 2 +- .../eng/DTDClassification.py | 2 +- .../eng/EuroSATClassification.py | 2 +- .../eng/FER2013Classification.py | 2 +- .../eng/FGVCAircraftClassification.py | 2 +- .../eng/Food101Classification.py | 2 +- .../eng/GTSRBClassification.py | 2 +- .../ImageClassification/eng/Imagenet1k.py | 2 +- .../eng/MNISTClassification.py | 2 +- .../eng/OxfordFlowersClassification.py | 2 +- .../eng/OxfordPetsClassification.py | 2 +- .../eng/PatchCamelyonClassification.py | 2 +- .../eng/RESISC45Classification.py | 2 +- .../eng/STL10Classification.py | 2 +- .../eng/SUN397Classification.py | 2 +- .../eng/StanfordCarsClassification.py | 2 +- .../eng/UCF101Classification.py | 2 +- .../eng/PascalVOC2007.py | 2 +- .../Image/VisualSTS/en/STS12VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS13VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS14VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS15VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS16VisualSTS.py | 2 +- .../STS17MultilingualVisualSTS.py | 2 +- .../STSBenchmarkMultilingualVisualSTS.py | 2 +- .../ZeroshotClassification/eng/Country211.py | 2 +- .../ZeroshotClassification/eng/Imagenet1k.py | 2 +- .../ZeroshotClassification/eng/UCF101.py | 2 +- .../model_meta.json | 2 +- .../model_meta.json | 2 +- scripts/run_mieb.py | 45 +++++++++++ 85 files changed, 250 insertions(+), 134 deletions(-) create mode 100644 scripts/run_mieb.py diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 09582d779b..c7b6839015 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -98,9 +98,15 @@ "Summarization", "InstructionRetrieval", "Speed", - "ZeroShotClassification", - "ImageTextPairClassification", + "Any2AnyMultiChoice", + "Any2AnyRetrieval", "Any2TextMutipleChoice", + "ImageClustering", + "ImageClassification", + "ImageMultilabelClassification", + "ImageTextPairClassification", + "VisualSTS", + "ZeroShotClassification", ] TASK_CATEGORY = Literal[ diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index b3287f274c..70bc20cabf 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -5,6 +5,7 @@ import torch from PIL import Image +from torch.utils.data import DataLoader from tqdm import tqdm from transformers import LlavaNextForConditionalGeneration, LlavaNextProcessor @@ -56,9 +57,23 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 8): all_text_embeddings.append(text_outputs.cpu()) return torch.cat(all_text_embeddings, dim=0) - def get_image_embeddings(self, images: list[Image.Image], batch_size: int = 8): + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 8 + ): all_image_embeddings = [] + if isinstance(images, DataLoader): + for batch_images in tqdm(images): + img_inputs = self.processor( + [self.img_prompt] * len(batch_images), + batch_images, + return_tensors="pt", + padding=True, + ).to("cuda") + image_outputs = self.model( + **img_inputs, output_hidden_states=True, return_dict=True + ).hidden_states[-1][:, -1, :] + all_image_embeddings.append(image_outputs.cpu()) with torch.no_grad(): for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] @@ -95,24 +110,41 @@ def get_fused_embeddings( all_fused_embeddings = [] if texts is not None and images is not None: - if len(texts) != len(images): - raise ValueError( - "The number of texts and images must have the same length" - ) - with torch.no_grad(): - for i in tqdm(range(0, len(images), batch_size)): - batch_texts = texts[i : i + batch_size] - batch_images = images[i : i + batch_size] - prompts = [ - self.composed_prompt.format(text) for text in batch_texts - ] - inputs = self.processor( - prompts, batch_images, return_tensors="pt", padding=True - ).to("cuda") - outputs = self.model( - **inputs, output_hidden_states=True, return_dict=True - ).hidden_states[-1][:, -1, :] - all_fused_embeddings.append(outputs.cpu()) + if isinstance(images, DataLoader): + with torch.no_grad(): + for index, batch_images in enumerate(tqdm(images)): + batch_texts = texts[ + index * batch_size : (index + 1) * batch_size + ] + prompts = [ + self.composed_prompt.format(text) for text in batch_texts + ] + inputs = self.processor( + prompts, batch_images, return_tensors="pt", padding=True + ).to("cuda") + outputs = self.model( + **inputs, output_hidden_states=True, return_dict=True + ).hidden_states[-1][:, -1, :] + all_fused_embeddings.append(outputs.cpu()) + else: + if len(texts) != len(images): + raise ValueError( + "The number of texts and images must have the same length" + ) + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_texts = texts[i : i + batch_size] + batch_images = images[i : i + batch_size] + prompts = [ + self.composed_prompt.format(text) for text in batch_texts + ] + inputs = self.processor( + prompts, batch_images, return_tensors="pt", padding=True + ).to("cuda") + outputs = self.model( + **inputs, output_hidden_states=True, return_dict=True + ).hidden_states[-1][:, -1, :] + all_fused_embeddings.append(outputs.cpu()) return torch.cat(all_fused_embeddings, dim=0) elif texts is not None: diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py index 007bbfe37f..c86fdcd5b6 100644 --- a/mteb/models/vista_models.py +++ b/mteb/models/vista_models.py @@ -1,13 +1,18 @@ from __future__ import annotations from functools import partial +from typing import Any import torch from PIL import Image +from torch.utils.data import DataLoader +from torchvision import transforms from tqdm import tqdm from mteb.model_meta import ModelMeta +tensor_to_image = transforms.Compose([transforms.ToPILImage()]) + def vista_loader(**kwargs): try: # a temporal fix for the dependency issues of vista models. @@ -27,6 +32,7 @@ def __init__( negatives_cross_device: bool = False, temperature: float = 0.02, from_pretrained=None, + **kwargs: Any, ): super().__init__( model_name_bge=model_name_bge, @@ -88,15 +94,21 @@ def encode_text(self, texts): t_reps = torch.nn.functional.normalize(t_reps, dim=-1) return t_reps.contiguous() - def encode(self, images=None, texts=None): + def encode(self, images=None, texts=None, tensors=False): if images is not None: if isinstance(images, list): - images = [ - self.preprocess_val( - img if isinstance(img, Image.Image) else Image.open(img) - ) - for img in images - ] + if not tensors: + images = [ + self.preprocess_val( + img if isinstance(img, Image.Image) else Image.open(img) + ) + for img in images + ] + else: + images = [ + self.preprocess_val(tensor_to_image(image)) + for image in images + ] images = torch.stack(images) if texts is not None: texts = self.tokenizer(texts, return_tensors="pt", padding=True) @@ -119,31 +131,52 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): all_text_embeddings.append(batch_embeddings.cpu()) return torch.cat(all_text_embeddings, dim=0) - def get_image_embeddings(self, images: list[Image.Image], batch_size: int = 32): + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): all_image_embeddings = [] - for i in tqdm(range(0, len(images), batch_size)): - batch_images = images[i : i + batch_size] + + if isinstance(images, DataLoader): with torch.no_grad(): - batch_embeddings = self.encode(images=batch_images) - all_image_embeddings.append(batch_embeddings.cpu()) + for batch in tqdm(images): + batch_embeddings = self.encode(images=batch, tensors=True) + all_image_embeddings.append(batch_embeddings.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + batch_embeddings = self.encode(images=batch_images) + all_image_embeddings.append(batch_embeddings.cpu()) return torch.cat(all_image_embeddings, dim=0) def get_fused_embeddings( self, texts: list[str] = None, - images: list[Image.Image] = None, + images: list[Image.Image] | DataLoader = None, batch_size: int = 32, ): all_embeddings = [] - assert len(texts) == len(images) - for i in tqdm(range(0, len(texts), batch_size)): - batch_texts = texts[i : i + batch_size] - batch_images = images[i : i + batch_size] + + if isinstance(images, DataLoader): + with torch.no_grad(): + for index, batch_images in enumerate(tqdm(images)): + batch_texts = texts[ + index * batch_size : (index + 1) * batch_size + ] + batch_embeddings = self.encode( + images=batch_images, texts=batch_texts, tensors=True + ) + all_embeddings.append(batch_embeddings.cpu()) + else: + assert len(texts) == len(images) with torch.no_grad(): - batch_embeddings = self.encode( - images=batch_images, texts=batch_texts - ) - all_embeddings.append(batch_embeddings.cpu()) + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + batch_images = images[i : i + batch_size] + batch_embeddings = self.encode( + images=batch_images, texts=batch_texts + ) + all_embeddings.append(batch_embeddings.cpu()) return torch.cat(all_embeddings, dim=0) def calculate_probs(self, text_embeddings, image_embeddings): diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py index 1f00290cdd..46fbb5b990 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py @@ -13,7 +13,7 @@ class ImageCoDeT2IMultiChoice(AbsTaskAny2AnyMultiChoice): "path": "JamieSJS/imagecode-multi", "revision": "d28adfd8b34fefa546fdf94bdc352622b2575f6c", }, - type="Retrieval", + type="Any2AnyMultiChoice", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py index ca265fd898..cd1ce67e4a 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py @@ -14,7 +14,7 @@ class BLINKIT2IRetrieval(AbsTaskAny2AnyRetrieval): "revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26", "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="it2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py index 2c652c6388..d0fb86ba90 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py @@ -14,7 +14,7 @@ class BLINKIT2TRetrieval(AbsTaskAny2AnyRetrieval): "revision": "4ab83c87ac5b24e3b730f86d585671493a3a423c", "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="it2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py index 2e45933ea3..b215dfda06 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py @@ -14,7 +14,7 @@ class CIRRIT2IRetrieval(AbsTaskAny2AnyRetrieval): "revision": "503301cd99348035b9675883a543aa1ded0cf07c", "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="it2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py index c33f706c63..048f5a33bf 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py @@ -13,7 +13,7 @@ class CUB200I2I(AbsTaskAny2AnyRetrieval): "path": "isaacchung/cub200_retrieval", "revision": "ad08c1307b15a226bf1b64e62656a17f1f85f7ec", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py index 6f19a91f44..c358ee6507 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py @@ -13,7 +13,7 @@ class FORBI2I(AbsTaskAny2AnyRetrieval): "path": "isaacchung/forb_retrieval", "revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py index 3e24c8691f..04fad6e352 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py @@ -14,7 +14,7 @@ class Fashion200kI2TRetrieval(AbsTaskAny2AnyRetrieval): "revision": "96a313715ecf67f5dfe70c4fa52406bc7bdfbeee", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py index f54a3a38b2..54a1c24cf1 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py @@ -14,7 +14,7 @@ class Fashion200kT2IRetrieval(AbsTaskAny2AnyRetrieval): "revision": "1b86e2dde50e671d5c83d07a79e8b1d8c696964b", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py index 6072354fe6..45b8e10576 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py @@ -14,7 +14,7 @@ class FashionIQIT2IRetrieval(AbsTaskAny2AnyRetrieval): "revision": "e6f0ec70becc413d940cd62b2cfa3b1d3a08c31a", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="it2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py index f7278bcf37..267cf860b9 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py @@ -13,7 +13,7 @@ class Flickr30kI2TRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/flickr30k", "revision": "a4cf34ac79215f9e2cd6a10342d84f606fc41cc3", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py index 44164c90b6..576e6afa50 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py @@ -13,7 +13,7 @@ class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/flickr30k", "revision": "a4cf34ac79215f9e2cd6a10342d84f606fc41cc3", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py index 9539ef31b3..67b238a470 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py @@ -13,7 +13,7 @@ class GLDv2I2TRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/gld-v2-i2t", "revision": "d8c3e53160860f76de73ed3041a8593672fe5928", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py index c92a497914..40323d3636 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py @@ -68,7 +68,7 @@ class HatefulMemesI2TRetrieval(AbsTaskAny2AnyRetrieval): "revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py index 067396752a..fec70177db 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py @@ -68,7 +68,7 @@ class HatefulMemesT2IRetrieval(AbsTaskAny2AnyRetrieval): "revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py index 3fae916f6b..6e34640459 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py @@ -13,7 +13,7 @@ class ImageCoDeT2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/imagecode", "revision": "a424cd523ffb157b69a875fb5e71c1d51be54089", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py index e35da59fcb..abc71666aa 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py @@ -14,7 +14,7 @@ class InfoSeekIT2ITRetrieval(AbsTaskAny2AnyRetrieval): "revision": "78ee7f7708aac75d3afac5dcab1c9e03cb62664c", "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="it2it", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py index 4d88a7ac80..a856969e75 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py @@ -14,7 +14,7 @@ class InfoSeekIT2TRetrieval(AbsTaskAny2AnyRetrieval): "revision": "d4f4606f7a42bbf311c2957419ef3734fe81c47f", "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="it2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py index e46b2635e5..b0578c5944 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py @@ -13,7 +13,7 @@ class METI2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/met", "revision": "08ceaa61c0d172214abb3b8e82971d8f69d2aec0", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py index dff57f5a53..8652b2e8e0 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py @@ -14,7 +14,7 @@ class MSCOCOI2TRetrieval(AbsTaskAny2AnyRetrieval): "revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py index 9ce5fd839e..4797e98911 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py @@ -14,7 +14,7 @@ class MSCOCOT2IRetrieval(AbsTaskAny2AnyRetrieval): "revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py index 5eda9cd295..c9d671d9d6 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py @@ -97,7 +97,7 @@ class MemotionI2TRetrieval(AbsTaskAny2AnyRetrieval): "revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py index b82b6367a5..331e628f24 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py @@ -96,7 +96,7 @@ class MemotionT2IRetrieval(AbsTaskAny2AnyRetrieval): "revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py index 3c7798c77c..ae0d91a6b5 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py @@ -13,7 +13,7 @@ class NIGHTSI2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "MRBench/mbeir_nights_task4", "revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py index 0a720ec995..9bac08fa34 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py @@ -13,7 +13,7 @@ class OVENIT2ITRetrieval(AbsTaskAny2AnyRetrieval): "path": "MRBench/mbeir_oven_task8", "revision": "350d14b7258189654e26a2be93dc0bd6bee09b76", }, - type="Retrieval", + type="Any2AnyRetrieval", category="it2it", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py index 2c171c778d..0877cfdf33 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py @@ -13,7 +13,7 @@ class OVENIT2TRetrieval(AbsTaskAny2AnyRetrieval): "path": "MRBench/mbeir_oven_task6", "revision": "2192074af29422bc1dc41cf07936f198b8c69bd0", }, - type="Retrieval", + type="Any2AnyRetrieval", category="it2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py index 39502a46d8..dc43e34e29 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py @@ -13,7 +13,7 @@ class ROxfordI2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/r-oxford", "revision": "d8daad98b4e4896a7f7fa1b3485a22420242d4fc", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py index 321bb818be..61cd189fce 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py @@ -13,7 +13,7 @@ class RP2kI2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/rp2k", "revision": "f8f82d4eb1aa4dc4dbf2c768596c8110a3703765", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py index a112ded273..258ec836c6 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py @@ -13,7 +13,7 @@ class RParisI2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/r-paris", "revision": "bafc3a08fdffd72558021ce3a41250833d7e0e88", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py index 09b33aac7d..5d754fe0e6 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py @@ -13,7 +13,7 @@ class SOPI2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/stanford-online-products", "revision": "0b3a1622902e6258425e673405bdfb1e5dfa8618", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py index cc96d134a0..0f7acedab0 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py @@ -73,7 +73,7 @@ class SciMMIRI2TRetrieval(AbsTaskAny2AnyRetrieval): "revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py index 41c2c98e79..987a00ea6d 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py @@ -73,7 +73,7 @@ class SciMMIRT2IRetrieval(AbsTaskAny2AnyRetrieval): "revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py index c89091f41d..5a4b13ec94 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py @@ -13,7 +13,7 @@ class SketchyI2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/sketchy", "revision": "c8b8c1b7a2f0a92f1bfaaa1c9afc22aa42c61d5b", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py index 4a053f8cc0..e98633e899 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py @@ -13,7 +13,7 @@ class StanfordCarsI2I(AbsTaskAny2AnyRetrieval): "path": "isaacchung/stanford_cars_retrieval", "revision": "b27a0612211af3598bd11fe28af20928f20cce06", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py index 754fa14911..fe1c2891db 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py @@ -14,7 +14,7 @@ class TUBerlinT2IRetrieval(AbsTaskAny2AnyRetrieval): "revision": "0cd78cd1ddbd3cafa9f319c638ebd77836ec9ff6", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py index cb5c3b38e5..58e1c5d31e 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py @@ -14,7 +14,7 @@ class VQA2IT2TRetrieval(AbsTaskAny2AnyRetrieval): "revision": "69882b6ba0b443dd62e633e546725b0f13b7e3aa", "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="it2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py index 6365b7adb3..fc73789541 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py @@ -100,7 +100,7 @@ class VidoreArxivQARetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/arxivqa_test_subsampled", "revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py index bf99c199a8..a36f5ea5fe 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py @@ -13,7 +13,7 @@ class VisualNewsI2TRetrieval(AbsTaskAny2AnyRetrieval): "path": "MRBench/mbeir_visualnews_task3", "revision": "aaee58895a66e4d619168849267ed2bb40d37043", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py index 8bd3f8278f..aae9882d52 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py @@ -13,7 +13,7 @@ class VisualNewsT2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "MRBench/mbeir_visualnews_task0", "revision": "94c519d850dba2b0058c2fc9b5da6142a59aa285", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py index 5565ca9f50..076f003b2b 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py @@ -14,7 +14,7 @@ class VizWizIT2TRetrieval(AbsTaskAny2AnyRetrieval): "revision": "044af162d55f82ab603fa16ffcf7f1e4dbf300e9", "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="it2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py index b3f21869ed..fabbf48ed4 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py @@ -13,7 +13,7 @@ class WebQAT2ITRetrieval(AbsTaskAny2AnyRetrieval): "path": "MRBench/mbeir_webqa_task2", "revision": "53db4c9f9c93cb74926a1c9d04dea7d7acac2f21", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2it", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py index f53415087e..a98ee514a9 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py @@ -13,7 +13,7 @@ class WebQAT2TRetrieval(AbsTaskAny2AnyRetrieval): "path": "MRBench/mbeir_webqa_task1", "revision": "468b42a2b2e767d80d2d93f5ae5d42f135a10478", }, - type="Retrieval", + type="Any2AnyRetrieval", category="s2p", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py index 5de06b937f..ee8b8c4148 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py @@ -101,7 +101,7 @@ class WITT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): "revision": "91ac153f1371a98b209ed763205e25e115ecd06e", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=_LANGUAGES, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py index 65c886f314..507639a4df 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py @@ -84,7 +84,7 @@ class XFlickr30kCoT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): "revision": "0af2c2eba58b27a71898787e286be04befdd7a20", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=_LANGUAGES, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py index 687c9f0446..7e78db8193 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py @@ -129,7 +129,7 @@ class XM3600T2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): "revision": "8d3e5665526c55a5855cd6ddfbaba2032bc7cee4", # "trust_remote_code": True, }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=_LANGUAGES, diff --git a/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py b/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py index e42ec28f76..bb3c5db181 100644 --- a/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py +++ b/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py @@ -36,10 +36,10 @@ class CVBenchCount(AbsTaskAny2TextMultipleChoice): dialect=[], modalities=["text", "image"], sample_creation="found", - bibtex_citation="""@article{wu2024scimmir, - title={placeholder}, - author={placeholder and others}, - journal={arXiv preprint arXiv:2401.13478}, + bibtex_citation="""@article{tong2024cambrian, + title={Cambrian-1: A fully open, vision-centric exploration of multimodal llms}, + author={Tong, Shengbang and Brown, Ellis and Wu, Penghao and Woo, Sanghyun and Middepogu, Manoj and Akula, Sai Charitha and Yang, Jihan and Yang, Shusheng and Iyer, Adithya and Pan, Xichen and others}, + journal={arXiv preprint arXiv:2406.16860}, year={2024} }""", descriptive_stats={ @@ -96,10 +96,10 @@ class CVBenchRelation(AbsTaskAny2TextMultipleChoice): dialect=[], modalities=["text", "image"], sample_creation="found", - bibtex_citation="""@article{wu2024scimmir, - title={placeholder}, - author={placeholder and others}, - journal={arXiv preprint arXiv:2401.13478}, + bibtex_citation="""@article{tong2024cambrian, + title={Cambrian-1: A fully open, vision-centric exploration of multimodal llms}, + author={Tong, Shengbang and Brown, Ellis and Wu, Penghao and Woo, Sanghyun and Middepogu, Manoj and Akula, Sai Charitha and Yang, Jihan and Yang, Shusheng and Iyer, Adithya and Pan, Xichen and others}, + journal={arXiv preprint arXiv:2406.16860}, year={2024} }""", descriptive_stats={ @@ -158,10 +158,10 @@ class CVBenchDepth(AbsTaskAny2TextMultipleChoice): dialect=[], modalities=["text", "image"], sample_creation="found", - bibtex_citation="""@article{wu2024scimmir, - title={placeholder}, - author={placeholder and others}, - journal={arXiv preprint arXiv:2401.13478}, + bibtex_citation="""@article{tong2024cambrian, + title={Cambrian-1: A fully open, vision-centric exploration of multimodal llms}, + author={Tong, Shengbang and Brown, Ellis and Wu, Penghao and Woo, Sanghyun and Middepogu, Manoj and Akula, Sai Charitha and Yang, Jihan and Yang, Shusheng and Iyer, Adithya and Pan, Xichen and others}, + journal={arXiv preprint arXiv:2406.16860}, year={2024} }""", descriptive_stats={ diff --git a/mteb/tasks/Image/Clustering/eng/CIFAR.py b/mteb/tasks/Image/Clustering/eng/CIFAR.py index 61250cc3f5..e9a0429cf9 100644 --- a/mteb/tasks/Image/Clustering/eng/CIFAR.py +++ b/mteb/tasks/Image/Clustering/eng/CIFAR.py @@ -13,7 +13,7 @@ class CIFAR10Clustering(AbsTaskImageClustering): "path": "uoft-cs/cifar10", "revision": "0b2714987fa478483af9968de7c934580d0bb9a2", }, - type="Clustering", + type="ImageClustering", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Clustering/eng/ImageNet.py b/mteb/tasks/Image/Clustering/eng/ImageNet.py index daf8ab8dae..381b3d9d91 100644 --- a/mteb/tasks/Image/Clustering/eng/ImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/ImageNet.py @@ -13,7 +13,7 @@ class ImageNetDog15Clustering(AbsTaskImageClustering): "path": "JamieSJS/imagenet-dog-15", "revision": "bfb6ad3b2109d26c9daddf14f98d315daa35ee72", }, - type="Clustering", + type="ImageClustering", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py index 14123e2111..96c557e3ab 100644 --- a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py @@ -13,7 +13,7 @@ class TinyImageNet(AbsTaskImageClustering): "path": "zh-plus/tiny-imagenet", "revision": "5a77092c28e51558c5586e9c5eb71a7e17a5e43f", }, - type="Clustering", + type="ImageClustering", category="s2s", eval_splits=["valid"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py index f29259ae2a..cf9a18cc46 100644 --- a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py @@ -13,7 +13,7 @@ class BirdsnapClassification(AbsTaskImageClassification): "path": "isaacchung/birdsnap", "revision": "e09b9dea248d579376684268cbedba28cd66b9b4", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py index 2fe4fc2808..7560bff77e 100644 --- a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py +++ b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py @@ -13,7 +13,7 @@ class CIFAR10Classification(AbsTaskImageClassification): "path": "uoft-cs/cifar10", "revision": "0b2714987fa478483af9968de7c934580d0bb9a2", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py index 5c79a41046..0e00980428 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py @@ -14,7 +14,7 @@ class Caltech101Classification(AbsTaskImageClassification): "name": "with_background_category", "revision": "851374102055782c84f89b1b4e9d128a6568847b", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py index 5f34c09a14..7fcbd4b209 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py @@ -13,7 +13,7 @@ class Country211Classification(AbsTaskImageClassification): "path": "clip-benchmark/wds_country211", "revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py index aabb03f02a..6362785cb7 100644 --- a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py @@ -13,7 +13,7 @@ class DTDClassification(AbsTaskImageClassification): "path": "tanganke/dtd", "revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py index 6ef26a0dba..45638643de 100644 --- a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py @@ -13,7 +13,7 @@ class EuroSATClassification(AbsTaskImageClassification): "path": "timm/eurosat-rgb", "revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py index ea987fb4e2..49323aa4b6 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py @@ -13,7 +13,7 @@ class FER2013Classification(AbsTaskImageClassification): "path": "clip-benchmark/wds_fer2013", "revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py index 74659b5e92..3db3ef2d8c 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py @@ -13,7 +13,7 @@ class FGVCAircraftClassification(AbsTaskImageClassification): "path": "HuggingFaceM4/FGVC-Aircraft", "revision": "91860adfc9a09aabca5cddb5247442109b38e213", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py index 34b2592e20..c508486997 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py @@ -13,7 +13,7 @@ class Food101Classification(AbsTaskImageClassification): "path": "ethz/food101", "revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["validation"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py index 6596151327..29c5ccc4c0 100644 --- a/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py @@ -13,7 +13,7 @@ class GTSRBClassification(AbsTaskImageClassification): "path": "clip-benchmark/wds_gtsrb", "revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py index d3b8474808..c8bfd62ce8 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py +++ b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py @@ -13,7 +13,7 @@ class Imagenet1kClassification(AbsTaskImageClassification): "path": "clip-benchmark/wds_imagenet1k", "revision": "b24c7a5a3ef12df09089055d1795e2ce7c7e7397", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py index 4ea68ddea3..5e9b9a86af 100644 --- a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py @@ -13,7 +13,7 @@ class MNISTClassification(AbsTaskImageClassification): "path": "ylecun/mnist", "revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py index d07badc30f..9e4a6f6aaa 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -13,7 +13,7 @@ class OxfordFlowersClassification(AbsTaskImageClassification): "path": "nelorth/oxford-flowers", "revision": "a37b1891609c0376fa81eced756e7863e1bd873b", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py index 603dad1278..09935a2735 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py @@ -13,7 +13,7 @@ class OxfordPetsClassification(AbsTaskImageClassification): "path": "isaacchung/OxfordPets", "revision": "557b480fae8d69247be74d9503b378a09425096f", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py index a6f9466672..0032266eaa 100644 --- a/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py @@ -13,7 +13,7 @@ class PatchCamelyonClassification(AbsTaskImageClassification): "path": "clip-benchmark/wds_vtab-pcam", "revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py index c767e3b334..d73abc76b9 100644 --- a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py @@ -13,7 +13,7 @@ class RESISC45Classification(AbsTaskImageClassification): "path": "timm/resisc45", "revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py index 02593fe4e3..fe25c9d3d7 100644 --- a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py @@ -13,7 +13,7 @@ class STL10Classification(AbsTaskImageClassification): "path": "tanganke/stl10", "revision": "49ae7f94508f7feae62baf836db284306eab0b0f", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py index d23844ec4f..f7593a3373 100644 --- a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py @@ -13,7 +13,7 @@ class SUN397Classification(AbsTaskImageClassification): "path": "dpdl-benchmark/sun397", "revision": "7e6af6a2499ad708618be868e1471eac0aca1168", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py index e4561b2165..4c049f540d 100644 --- a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py @@ -13,7 +13,7 @@ class StanfordCarsClassification(AbsTaskImageClassification): "path": "isaacchung/StanfordCars", "revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py index ef82d99d9e..41c923b538 100644 --- a/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py @@ -17,7 +17,7 @@ class UCF101Classification(AbsTaskImageClassification): "path": "flwrlabs/ucf101", "revision": "1098eed48f2929443f47c39f3b5c814e16369c11", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index fa0628b351..1a02997aec 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -16,7 +16,7 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): "name": "voc2007_main", "revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", }, - type="MultilabelClassification", + type="ImageMultilabelClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py index 8d78bb7238..c036b54042 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py @@ -13,7 +13,7 @@ class STS12VisualSTS(AbsTaskVisualSTS): }, description="SemEval-2012 Task 6." + "then rendered into images.", reference="https://arxiv.org/abs/2402.08183/", - type="STS", + type="VisualSTS", category="i2i", modalities=["image"], eval_splits=["test"], diff --git a/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py index 1b02248d35..cf4c0aa6c4 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py @@ -13,7 +13,7 @@ class STS13VisualSTS(AbsTaskVisualSTS): }, description="SemEval STS 2013 dataset." + "then rendered into images.", reference="https://arxiv.org/abs/2402.08183/", - type="STS", + type="VisualSTS", category="i2i", modalities=["image"], eval_splits=["test"], diff --git a/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py index a427fdae0b..46dce36f80 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py @@ -14,7 +14,7 @@ class STS14VisualSTS(AbsTaskVisualSTS): description="SemEval STS 2014 dataset. Currently only the English dataset." + "rendered into images.", reference="https://arxiv.org/abs/2402.08183/", - type="STS", + type="VisualSTS", category="i2i", modalities=["image"], eval_splits=["test"], diff --git a/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py index 12c9a74c81..a9aca02c39 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py @@ -13,7 +13,7 @@ class STS15VisualSTS(AbsTaskVisualSTS): }, description="SemEval STS 2015 dataset" + "rendered into images.", reference="https://arxiv.org/abs/2402.08183/", - type="STS", + type="VisualSTS", category="i2i", modalities=["image"], eval_splits=["test"], diff --git a/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py index ae1e2900dd..b64e040282 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py @@ -13,7 +13,7 @@ class STS16VisualSTS(AbsTaskVisualSTS): }, description="SemEval STS 2016 dataset" + "rendered into images.", reference="https://arxiv.org/abs/2402.08183/", - type="STS", + type="VisualSTS", category="i2i", modalities=["image"], eval_splits=["test"], diff --git a/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py b/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py index dc9e464dcf..b72988676a 100644 --- a/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py @@ -33,7 +33,7 @@ class STS17MultilingualVisualSTS(AbsTaskVisualSTS, MultilingualTask): + "rendered into images." ), reference="https://arxiv.org/abs/2402.08183/", - type="STS", + type="VisualSTS", category="i2i", modalities=["image"], eval_splits=_SPLITS, diff --git a/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py b/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py index 8cf063d059..339be27c37 100644 --- a/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py @@ -34,7 +34,7 @@ class STSBenchmarkMultilingualVisualSTS(AbsTaskVisualSTS, MultilingualTask): + "built upon multi-sts created by Philip May" ), reference="https://arxiv.org/abs/2402.08183/", - type="STS", + type="VisualSTS", category="i2i", modalities=["image"], eval_splits=_SPLITS, diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Country211.py b/mteb/tasks/Image/ZeroshotClassification/eng/Country211.py index ce3e7657d8..0a60e33003 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Country211.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Country211.py @@ -17,7 +17,7 @@ class Country211Classification(AbsTaskZeroshotClassification): "path": "clip-benchmark/wds_country211", "revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", }, - type="Classification", + type="ZeroShotClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py b/mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py index 6c96fad3ab..87dc8e277e 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py @@ -17,7 +17,7 @@ class Imagenet1kClassification(AbsTaskZeroshotClassification): "path": "clip-benchmark/wds_imagenet1k", "revision": "b24c7a5a3ef12df09089055d1795e2ce7c7e7397", }, - type="Classification", + type="ZeroShotClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py b/mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py index 9274e7c1f5..b95021184c 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py @@ -19,7 +19,7 @@ class UCF101Classification(AbsTaskZeroshotClassification): "path": "flwrlabs/ucf101", "revision": "1098eed48f2929443f47c39f3b5c814e16369c11", }, - type="Classification", + type="ZeroShotClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json b/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json index 746dfa90fd..a73369f513 100644 --- a/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json +++ b/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json @@ -1 +1 @@ -{"name": "BAAI/bge-visualized", "revision": "98db10b10d22620010d06f11733346e1c98c34aa", "release_date": "2024-06-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "VisualizedBGEWrapper"} \ No newline at end of file +{"name": "BAAI/bge-visualized-base", "revision": "98db10b10d22620010d06f11733346e1c98c34aa", "release_date": "2024-06-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "vista_loader"} \ No newline at end of file diff --git a/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json b/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json index 992116b866..a2b05c9208 100644 --- a/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json +++ b/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json @@ -1 +1 @@ -{"name": "BAAI/bge-visualized-m3", "revision": "98db10b10d22620010d06f11733346e1c98c34aa", "release_date": "2024-06-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "VisualizedBGEWrapper"} \ No newline at end of file +{"name": "BAAI/bge-visualized-m3", "revision": "98db10b10d22620010d06f11733346e1c98c34aa", "release_date": "2024-06-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "vista_loader"} \ No newline at end of file diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py new file mode 100644 index 0000000000..8b00d64e9d --- /dev/null +++ b/scripts/run_mieb.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import mteb + +for model_name in [ + "openai/clip-vit-base-patch32", + "openai/clip-vit-base-patch16", + "openai/clip-vit-large-patch14", + "royokong/e5-v", + "BAAI/bge-visualized-base", + "BAAI/bge-visualized-m3", + "google/siglip-so400m-patch14-384", + "kakaobrain/align-base", + "jinaai/jina-clip-v1", + "nomic-ai/nomic-embed-vision-v1.5", + "Salesforce/blip-image-captioning-large", + "Salesforce/blip-image-captioning-base", + "Salesforce/blip2-opt-2.7b", + "Salesforce/blip2-opt-6.7b-coco", + "facebook/dinov2-small", + "facebook/dinov2-base", + "facebook/dinov2-large", + "facebook/dinov2-giant", + "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + "nyu-visionx/moco-v3-vit-b", + "nyu-visionx/moco-v3-vit-l", +]: + model = mteb.get_model(model_name) + tasks = mteb.get_tasks( + task_types=[ + "Any2AnyRetrieval", + "AbsTaskAny2AnyMultiChoice", + "Any2TextMutipleChoice", + "ImageClustering", + "ImageClassification", + "ImageMultilabelClassification", + "ImageTextPairClassification", + "VisualSTS", + "ZeroShotClassification", + ] + ) + evaluation = mteb.MTEB(tasks=tasks) + results = evaluation.run(model, output_folder="results-mieb-final") From 22751ca22779c73e60c5400710867bb7aedca4f5 Mon Sep 17 00:00:00 2001 From: Jamie-Stirling <36764530+Jamie-Stirling@users.noreply.github.com> Date: Tue, 22 Oct 2024 11:42:53 +0100 Subject: [PATCH 080/154] [mieb] split RParisI2IRetrieval and ROxfordI2IRetrieval into easy, medium and hard versions (#1305) * wip: start adding BLIP models * add other blip variants * wip: add blip2_models.py * make lint * wip: implement blip2 wrapper * feat: add blip2 models, still mismatched names * fix: remove projections from image and text embeddings * make lint * wip: add coco BLIP2 * fix: BLIP2 better zero-shot classification without text_proj and vision_proj * tidy blip2 * add imagenet-dog-15 dataset * tidy and lint * remove unused import * add cluster_accuracy, ari and nmi to Image.ClusteringEvaluator * add imagenet-10 clustering task * add SOPI2IRetrieval * add results forclip on ImageNet10Clustering and ImageNetDog15Clustering * add SOPI2IRetrieval results for clip 32 * add results for clip vit 32/SOPI2IRetrieval * resolve conflict * add RP2kI2IRetrieval dataset * add RP2kI2IRetrieval results with clip-vit-base-patch32 * update image retrieval __init__.py * fix ImageTextPair dataloading for large datasets; more compositionality evaluation datasets * add RP2kI2IRetrieval and METI2IRetrieval * add METI2IRetreival * add SOP results * make lign * new revision for METI2IRetrieval * make lint * reset corpus chunk size * remove wrong classification import * add Flickr30k T2I and I2T * add Flickr30k T2I retriebal * reduced-size MET revision * fix: add Flickr30k T2I * make lint * add two landmark datasets and results * add Sketchy i2i retrieval * add task metadata * add BLINKIT2IRetrieval dataset * add BLINKIT2TRetrieval * add ImageCoDeT2IRetrieval * make lint * add vizwiz retrieval and results * fix vizwiz duplicate texts * add new vizwiz results * add VQA2 results * add GLD v2 I2T retrieval * add gld v2 i2i retrieval * make lint * add AbsTaskAny2AnyMultiChoice * make lint * remove GLDv2I2IRetrieval * exclude AbsTaskAny2AnyMultiChoice from test_load_data * fix e5v&vista * remove duplicate corpus entries from BLINKIT2TRetreival dataset * task type fix for running tasks * update BLINKIT2T metadata * fix wrong meta * run mieb script * split ROxford, RParis into easy, medium and hard * make lint --------- Co-authored-by: gowitheflow-1998 --- .../eng/BLINKIT2TRetrieval.py | 4 +- .../eng/ROxfordI2IRetrieval.py | 110 +++++- .../eng/RParisI2IRetrieval.py | 114 +++++- .../BLINKIT2TRetrieval.json | 342 +++++++++--------- .../ROxfordEasyI2IRetrieval.json | 186 ++++++++++ .../ROxfordHardI2IRetrieval.json | 186 ++++++++++ .../ROxfordI2IRetrieval.json | 186 ---------- .../ROxfordMediumI2IRetrieval.json | 186 ++++++++++ .../RParisEasyI2IRetrieval.json | 186 ++++++++++ .../RParisHardI2IRetrieval.json | 186 ++++++++++ .../RParisI2IRetrieval.json | 186 ---------- .../RParisMediumI2IRetrieval.json | 186 ++++++++++ 12 files changed, 1493 insertions(+), 565 deletions(-) create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordI2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisI2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py index d0fb86ba90..36d7434575 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py @@ -11,7 +11,7 @@ class BLINKIT2TRetrieval(AbsTaskAny2AnyRetrieval): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2t", - "revision": "4ab83c87ac5b24e3b730f86d585671493a3a423c", + "revision": "302cf2008f204285985099dcd46425b00356c610", "trust_remote_code": True, }, type="Any2AnyRetrieval", @@ -40,7 +40,7 @@ class BLINKIT2TRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 3080, + "num_documents": 26, "num_queries": 1073, "average_relevant_docs_per_query": 1, } diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py index dc43e34e29..7779883a84 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py @@ -4,14 +4,14 @@ from mteb.abstasks.TaskMetadata import TaskMetadata -class ROxfordI2IRetrieval(AbsTaskAny2AnyRetrieval): +class ROxfordEasyI2IRetrieval(AbsTaskAny2AnyRetrieval): metadata = TaskMetadata( - name="ROxfordI2IRetrieval", + name="ROxfordEasyI2IRetrieval", description="Retrieve photos of landmarks in Oxford, UK.", reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", dataset={ - "path": "JamieSJS/r-oxford", - "revision": "d8daad98b4e4896a7f7fa1b3485a22420242d4fc", + "path": "JamieSJS/r-oxford-easy", + "revision": "3f018eb7ad32218a5a4ebd704493e0834a265cf5", }, type="Any2AnyRetrieval", category="i2i", @@ -35,16 +35,108 @@ class ROxfordI2IRetrieval(AbsTaskAny2AnyRetrieval): } """, descriptive_stats={ - "n_samples": {"test": 3555537}, + "n_samples": {"test": 70}, "avg_character_length": { "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 5063, - "num_queries": 5063, - "average_relevant_docs_per_query": 702, + "num_documents": 4993, + "num_queries": 70, + "average_relevant_docs_per_query": 43.3, } }, }, ) - skip_first_result = True + skip_first_result = False + + +class ROxfordMediumI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="ROxfordMediumI2IRetrieval", + description="Retrieve photos of landmarks in Oxford, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-oxford-medium", + "revision": "3bd28e9c45e15f299117c634799f7035c4de2d31", + }, + type="Any2AnyRetrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image retrieval benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 4993, + "num_queries": 70, + "average_relevant_docs_per_query": 78.9, + } + }, + }, + ) + skip_first_result = False + + +class ROxfordHardI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="ROxfordHardI2IRetrieval", + description="Retrieve photos of landmarks in Oxford, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-oxford-hard", + "revision": "f20b30211b7ba3fc64a02bd83998fe75f3023719", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image retrieval benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 4993, + "num_queries": 70, + "average_relevant_docs_per_query": 35.7, + } + }, + }, + ) + skip_first_result = False diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py index 258ec836c6..ed49982fe5 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py @@ -4,14 +4,14 @@ from mteb.abstasks.TaskMetadata import TaskMetadata -class RParisI2IRetrieval(AbsTaskAny2AnyRetrieval): +class RParisEasyI2IRetrieval(AbsTaskAny2AnyRetrieval): metadata = TaskMetadata( - name="RParisI2IRetrieval", - description="Retrieve photos of landmarks in Paris.", - reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + name="RParisEasyI2IRetrieval", + description="Retrieve photos of landmarks in Paris, France.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", dataset={ - "path": "JamieSJS/r-paris", - "revision": "bafc3a08fdffd72558021ce3a41250833d7e0e88", + "path": "JamieSJS/r-paris-easy", + "revision": "a7293da8a341de665ee4dcb2f209281df342d80b", }, type="Any2AnyRetrieval", category="i2i", @@ -35,16 +35,108 @@ class RParisI2IRetrieval(AbsTaskAny2AnyRetrieval): } """, descriptive_stats={ - "n_samples": {"test": 6392}, + "n_samples": {"test": 70}, "avg_character_length": { "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 6392, - "num_queries": 6392, - "average_relevant_docs_per_query": 734, + "num_documents": 6322, + "num_queries": 70, + "average_relevant_docs_per_query": 98.2, } }, }, ) - skip_first_result = True + skip_first_result = False + + +class RParisMediumI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="RParisMediumI2IRetrieval", + description="Retrieve photos of landmarks in Paris, France.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-paris-medium", + "revision": "900267b49003a086979e8d52f6942624236bfc34", + }, + type="Any2AnyRetrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image retrieval benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 6322, + "num_queries": 70, + "average_relevant_docs_per_query": 147.9, + } + }, + }, + ) + skip_first_result = False + + +class RParisHardI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="RParisHardI2IRetrieval", + description="Retrieve photos of landmarks in Paris, France.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-paris-hard", + "revision": "fd121b6592fe946616fa85116703b94a4c61fd63", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image retrieval benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 6322, + "num_queries": 70, + "average_relevant_docs_per_query": 35.7, + } + }, + }, + ) + skip_first_result = False diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json index d701facb1e..21d3b7f0dc 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json @@ -1,184 +1,184 @@ { - "dataset_revision": "4ab83c87ac5b24e3b730f86d585671493a3a423c", - "evaluation_time": 13.768992900848389, + "dataset_revision": "302cf2008f204285985099dcd46425b00356c610", + "evaluation_time": 11.171209335327148, "kg_co2_emissions": null, "mteb_version": "1.12.90", "scores": { "test": [ { - "cv_recall_at_1": 0.00093, - "cv_recall_at_10": 0.00373, - "cv_recall_at_100": 0.06151, - "cv_recall_at_1000": 0.49301, - "cv_recall_at_20": 0.01025, - "cv_recall_at_3": 0.00093, - "cv_recall_at_5": 0.00093, + "cv_recall_at_1": 0.08947, + "cv_recall_at_10": 0.64678, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.96645, + "cv_recall_at_3": 0.2479, + "cv_recall_at_5": 0.37651, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.00209, - "map_at_1": 0.00093, - "map_at_10": 0.00139, - "map_at_100": 0.00274, - "map_at_1000": 0.00397, - "map_at_20": 0.00176, - "map_at_3": 0.00093, - "map_at_5": 0.00093, - "mrr_at_1": 0.0009319664492078285, - "mrr_at_10": 0.0013147383837039009, - "mrr_at_100": 0.002685356046665323, - "mrr_at_1000": 0.003948307638249348, - "mrr_at_20": 0.0016977247814228705, - "mrr_at_3": 0.0009319664492078285, - "mrr_at_5": 0.0009319664492078285, - "nauc_cv_recall_at_1000_diff1": 0.21618180421784144, - "nauc_cv_recall_at_1000_max": 0.1966391139887544, - "nauc_cv_recall_at_1000_std": 0.6300616078101204, - "nauc_cv_recall_at_100_diff1": 0.17443594463110845, - "nauc_cv_recall_at_100_max": 0.3594175633623019, - "nauc_cv_recall_at_100_std": 0.5893797181186331, - "nauc_cv_recall_at_10_diff1": -0.05990075938028686, - "nauc_cv_recall_at_10_max": 0.43603863739514936, - "nauc_cv_recall_at_10_std": 0.6462545358013111, - "nauc_cv_recall_at_1_diff1": -0.49952309145315554, - "nauc_cv_recall_at_1_max": 0.6610449563695936, - "nauc_cv_recall_at_1_std": 1.0, - "nauc_cv_recall_at_20_diff1": 0.1175713728405099, - "nauc_cv_recall_at_20_max": 0.36443381136528874, - "nauc_cv_recall_at_20_std": 0.3549079762661633, - "nauc_cv_recall_at_3_diff1": -0.49952309145315554, - "nauc_cv_recall_at_3_max": 0.6610449563695936, - "nauc_cv_recall_at_3_std": 1.0, - "nauc_cv_recall_at_5_diff1": -0.49952309145315554, - "nauc_cv_recall_at_5_max": 0.6610449563695936, - "nauc_cv_recall_at_5_std": 1.0, - "nauc_map_at_1000_diff1": 0.3461462162176906, - "nauc_map_at_1000_max": 0.30890589543493097, - "nauc_map_at_1000_std": 0.707200194236762, - "nauc_map_at_100_diff1": 0.4005511014650457, - "nauc_map_at_100_max": 0.30811121696833377, - "nauc_map_at_100_std": 0.7144966661138995, - "nauc_map_at_10_diff1": 0.5208347947487291, - "nauc_map_at_10_max": 0.206028460069979, - "nauc_map_at_10_std": 0.8872521734282164, - "nauc_map_at_1_diff1": 0.6610449563695936, - "nauc_map_at_1_max": 0.09839818901557038, - "nauc_map_at_1_std": 1.0, - "nauc_map_at_20_diff1": 0.4542718670585475, - "nauc_map_at_20_max": 0.26815935842012756, - "nauc_map_at_20_std": 0.7972838331802248, - "nauc_map_at_3_diff1": 0.6610449563695936, - "nauc_map_at_3_max": 0.09839818901557038, - "nauc_map_at_3_std": 1.0, - "nauc_map_at_5_diff1": 0.6610449563695936, - "nauc_map_at_5_max": 0.09839818901557038, - "nauc_map_at_5_std": 1.0, - "nauc_mrr_at_1000_diff1": -0.013155580619472381, - "nauc_mrr_at_1000_max": 0.4252494452276833, - "nauc_mrr_at_1000_std": 0.687428882142861, - "nauc_mrr_at_100_diff1": -0.05688061033793547, - "nauc_mrr_at_100_max": 0.4528834485316897, - "nauc_mrr_at_100_std": 0.6942429482097452, - "nauc_mrr_at_10_diff1": -0.3361389575479778, - "nauc_mrr_at_10_max": 0.5699031562786795, - "nauc_mrr_at_10_std": 0.8567106980461006, - "nauc_mrr_at_1_diff1": -0.49952309145315554, - "nauc_mrr_at_1_max": 0.6610449563695936, - "nauc_mrr_at_1_std": 1.0, - "nauc_mrr_at_20_diff1": -0.2128688275475949, - "nauc_mrr_at_20_max": 0.5173348447205537, - "nauc_mrr_at_20_std": 0.6977606046374574, - "nauc_mrr_at_3_diff1": -0.49952309145315554, - "nauc_mrr_at_3_max": 0.6610449563695936, - "nauc_mrr_at_3_std": 1.0, - "nauc_mrr_at_5_diff1": -0.49952309145315554, - "nauc_mrr_at_5_max": 0.6610449563695936, - "nauc_mrr_at_5_std": 1.0, - "nauc_ndcg_at_1000_diff1": 0.2267671022987118, - "nauc_ndcg_at_1000_max": 0.2471557214473064, - "nauc_ndcg_at_1000_std": 0.6348682667422092, - "nauc_ndcg_at_100_diff1": 0.2912265250539601, - "nauc_ndcg_at_100_max": 0.3559110476318641, - "nauc_ndcg_at_100_std": 0.5901042407967306, - "nauc_ndcg_at_10_diff1": 0.42096720717204206, - "nauc_ndcg_at_10_max": 0.2838485324864617, - "nauc_ndcg_at_10_std": 0.7948899807288888, - "nauc_ndcg_at_1_diff1": 0.6610449563695936, - "nauc_ndcg_at_1_max": 0.09839818901557038, - "nauc_ndcg_at_1_std": 1.0, - "nauc_ndcg_at_20_diff1": 0.3330912830350423, - "nauc_ndcg_at_20_max": 0.36298197701132934, - "nauc_ndcg_at_20_std": 0.6588880474018856, - "nauc_ndcg_at_3_diff1": 0.6610449563695936, - "nauc_ndcg_at_3_max": 0.09839818901557038, - "nauc_ndcg_at_3_std": 1.0, - "nauc_ndcg_at_5_diff1": 0.6610449563695936, - "nauc_ndcg_at_5_max": 0.09839818901557038, - "nauc_ndcg_at_5_std": 1.0, - "nauc_precision_at_1000_diff1": 0.21618180421784106, - "nauc_precision_at_1000_max": 0.19663911398875364, - "nauc_precision_at_1000_std": 0.6300616078101199, - "nauc_precision_at_100_diff1": 0.2633008669431452, - "nauc_precision_at_100_max": 0.35813105058108047, - "nauc_precision_at_100_std": 0.5625569512563295, - "nauc_precision_at_10_diff1": 0.31227200390917714, - "nauc_precision_at_10_max": 0.3688258547365643, - "nauc_precision_at_10_std": 0.6910567239150112, - "nauc_precision_at_1_diff1": 0.6610449563695936, - "nauc_precision_at_1_max": 0.09839818901557038, - "nauc_precision_at_1_std": 1.0, - "nauc_precision_at_20_diff1": 0.24987842391547985, - "nauc_precision_at_20_max": 0.4262053224877857, - "nauc_precision_at_20_std": 0.5596936671568136, - "nauc_precision_at_3_diff1": 0.6610449563695937, - "nauc_precision_at_3_max": 0.09839818901557038, - "nauc_precision_at_3_std": 1.0, - "nauc_precision_at_5_diff1": 0.6610449563695937, - "nauc_precision_at_5_max": 0.09839818901557036, - "nauc_precision_at_5_std": 1.0, - "nauc_recall_at_1000_diff1": 0.21618180421784144, - "nauc_recall_at_1000_max": 0.1966391139887544, - "nauc_recall_at_1000_std": 0.6300616078101204, - "nauc_recall_at_100_diff1": 0.2633008669431453, - "nauc_recall_at_100_max": 0.3581310505810804, - "nauc_recall_at_100_std": 0.5625569512563297, - "nauc_recall_at_10_diff1": 0.3122720039091771, - "nauc_recall_at_10_max": 0.3688258547365644, - "nauc_recall_at_10_std": 0.691056723915011, - "nauc_recall_at_1_diff1": 0.6610449563695936, - "nauc_recall_at_1_max": 0.09839818901557038, - "nauc_recall_at_1_std": 1.0, - "nauc_recall_at_20_diff1": 0.24987842391547993, - "nauc_recall_at_20_max": 0.4262053224877856, - "nauc_recall_at_20_std": 0.5596936671568133, - "nauc_recall_at_3_diff1": 0.6610449563695936, - "nauc_recall_at_3_max": 0.09839818901557038, - "nauc_recall_at_3_std": 1.0, - "nauc_recall_at_5_diff1": 0.6610449563695936, - "nauc_recall_at_5_max": 0.09839818901557038, - "nauc_recall_at_5_std": 1.0, - "ndcg_at_1": 0.00093, - "ndcg_at_10": 0.00209, - "ndcg_at_100": 0.0128, - "ndcg_at_1000": 0.06231, - "ndcg_at_20": 0.00349, - "ndcg_at_3": 0.00093, - "ndcg_at_5": 0.00093, - "precision_at_1": 0.00093, - "precision_at_10": 0.00047, - "precision_at_100": 0.00065, - "precision_at_1000": 0.00049, - "precision_at_20": 0.00051, - "precision_at_3": 0.00031, - "precision_at_5": 0.00019, - "recall_at_1": 0.00093, - "recall_at_10": 0.00466, - "recall_at_100": 0.06524, - "recall_at_1000": 0.49301, - "recall_at_20": 0.01025, - "recall_at_3": 0.00093, - "recall_at_5": 0.00093 + "main_score": 0.32202, + "map_at_1": 0.08947, + "map_at_10": 0.22464, + "map_at_100": 0.24864, + "map_at_1000": 0.24864, + "map_at_20": 0.24715, + "map_at_3": 0.16061, + "map_at_5": 0.18987, + "mrr_at_1": 0.08946877912395154, + "mrr_at_10": 0.2246397875708215, + "mrr_at_100": 0.24864178823066596, + "mrr_at_1000": 0.24864178823066596, + "mrr_at_20": 0.24714520392298733, + "mrr_at_3": 0.16060888474681598, + "mrr_at_5": 0.18987263125194162, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": 0.6643245257587762, + "nauc_cv_recall_at_10_max": 0.3394378418139761, + "nauc_cv_recall_at_10_std": 0.5631230256620064, + "nauc_cv_recall_at_1_diff1": 0.3501554820571643, + "nauc_cv_recall_at_1_max": 0.3599782655731798, + "nauc_cv_recall_at_1_std": 0.7041481187202615, + "nauc_cv_recall_at_20_diff1": 0.5587451940825555, + "nauc_cv_recall_at_20_max": 0.47434883444027837, + "nauc_cv_recall_at_20_std": 0.6875358512675467, + "nauc_cv_recall_at_3_diff1": 0.5321534332531633, + "nauc_cv_recall_at_3_max": 0.3540044845277292, + "nauc_cv_recall_at_3_std": 0.7076694823976849, + "nauc_cv_recall_at_5_diff1": 0.6570524014466579, + "nauc_cv_recall_at_5_max": 0.3001299949877772, + "nauc_cv_recall_at_5_std": 0.6030407237879231, + "nauc_map_at_1000_diff1": 0.4915782753126093, + "nauc_map_at_1000_max": 0.3523024760071231, + "nauc_map_at_1000_std": 0.6868659279458145, + "nauc_map_at_100_diff1": 0.4915782753126093, + "nauc_map_at_100_max": 0.3523024760071231, + "nauc_map_at_100_std": 0.6868659279458145, + "nauc_map_at_10_diff1": 0.507291819494973, + "nauc_map_at_10_max": 0.34661422520261387, + "nauc_map_at_10_std": 0.6748191769380557, + "nauc_map_at_1_diff1": 0.3501554820571643, + "nauc_map_at_1_max": 0.3599782655731798, + "nauc_map_at_1_std": 0.7041481187202615, + "nauc_map_at_20_diff1": 0.49198266429986276, + "nauc_map_at_20_max": 0.35311180756517996, + "nauc_map_at_20_std": 0.6868445762814864, + "nauc_map_at_3_diff1": 0.4648021929978972, + "nauc_map_at_3_max": 0.3663517861281038, + "nauc_map_at_3_std": 0.7205919836574776, + "nauc_map_at_5_diff1": 0.5087022522085294, + "nauc_map_at_5_max": 0.3467565269949981, + "nauc_map_at_5_std": 0.683363887437365, + "nauc_mrr_at_1000_diff1": 0.4915782753126093, + "nauc_mrr_at_1000_max": 0.3523024760071231, + "nauc_mrr_at_1000_std": 0.6868659279458145, + "nauc_mrr_at_100_diff1": 0.4915782753126093, + "nauc_mrr_at_100_max": 0.3523024760071231, + "nauc_mrr_at_100_std": 0.6868659279458145, + "nauc_mrr_at_10_diff1": 0.507291819494973, + "nauc_mrr_at_10_max": 0.34661422520261387, + "nauc_mrr_at_10_std": 0.6748191769380557, + "nauc_mrr_at_1_diff1": 0.3501554820571643, + "nauc_mrr_at_1_max": 0.3599782655731798, + "nauc_mrr_at_1_std": 0.7041481187202615, + "nauc_mrr_at_20_diff1": 0.49198266429986276, + "nauc_mrr_at_20_max": 0.35311180756517996, + "nauc_mrr_at_20_std": 0.6868445762814864, + "nauc_mrr_at_3_diff1": 0.4648021929978972, + "nauc_mrr_at_3_max": 0.3663517861281038, + "nauc_mrr_at_3_std": 0.7205919836574776, + "nauc_mrr_at_5_diff1": 0.5087022522085294, + "nauc_mrr_at_5_max": 0.3467565269949981, + "nauc_mrr_at_5_std": 0.683363887437365, + "nauc_ndcg_at_1000_diff1": 0.5054976057073902, + "nauc_ndcg_at_1000_max": 0.3504516506472678, + "nauc_ndcg_at_1000_std": 0.6795326611960923, + "nauc_ndcg_at_100_diff1": 0.5054976057073902, + "nauc_ndcg_at_100_max": 0.3504516506472678, + "nauc_ndcg_at_100_std": 0.6795326611960923, + "nauc_ndcg_at_10_diff1": 0.5529277638560192, + "nauc_ndcg_at_10_max": 0.3418676893611852, + "nauc_ndcg_at_10_std": 0.6438431211574437, + "nauc_ndcg_at_1_diff1": 0.3501554820571643, + "nauc_ndcg_at_1_max": 0.3599782655731798, + "nauc_ndcg_at_1_std": 0.7041481187202615, + "nauc_ndcg_at_20_diff1": 0.5075033561523076, + "nauc_ndcg_at_20_max": 0.35530204332453247, + "nauc_ndcg_at_20_std": 0.6798053401234159, + "nauc_ndcg_at_3_diff1": 0.48649313476516426, + "nauc_ndcg_at_3_max": 0.36277467680108344, + "nauc_ndcg_at_3_std": 0.7169907526506151, + "nauc_ndcg_at_5_diff1": 0.5565020379741223, + "nauc_ndcg_at_5_max": 0.33223528863871, + "nauc_ndcg_at_5_std": 0.6584725779305518, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": 0.6643245257587758, + "nauc_precision_at_10_max": 0.3394378418139761, + "nauc_precision_at_10_std": 0.563123025662006, + "nauc_precision_at_1_diff1": 0.3501554820571643, + "nauc_precision_at_1_max": 0.3599782655731798, + "nauc_precision_at_1_std": 0.7041481187202615, + "nauc_precision_at_20_diff1": 0.5587451940825576, + "nauc_precision_at_20_max": 0.47434883444027603, + "nauc_precision_at_20_std": 0.6875358512675491, + "nauc_precision_at_3_diff1": 0.5321534332531633, + "nauc_precision_at_3_max": 0.3540044845277291, + "nauc_precision_at_3_std": 0.7076694823976849, + "nauc_precision_at_5_diff1": 0.6570524014466578, + "nauc_precision_at_5_max": 0.30012999498777754, + "nauc_precision_at_5_std": 0.6030407237879232, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.6643245257587762, + "nauc_recall_at_10_max": 0.3394378418139761, + "nauc_recall_at_10_std": 0.5631230256620064, + "nauc_recall_at_1_diff1": 0.3501554820571643, + "nauc_recall_at_1_max": 0.3599782655731798, + "nauc_recall_at_1_std": 0.7041481187202615, + "nauc_recall_at_20_diff1": 0.5587451940825555, + "nauc_recall_at_20_max": 0.47434883444027837, + "nauc_recall_at_20_std": 0.6875358512675467, + "nauc_recall_at_3_diff1": 0.5321534332531633, + "nauc_recall_at_3_max": 0.3540044845277292, + "nauc_recall_at_3_std": 0.7076694823976849, + "nauc_recall_at_5_diff1": 0.6570524014466579, + "nauc_recall_at_5_max": 0.3001299949877772, + "nauc_recall_at_5_std": 0.6030407237879231, + "ndcg_at_1": 0.08947, + "ndcg_at_10": 0.32202, + "ndcg_at_100": 0.41063, + "ndcg_at_1000": 0.41063, + "ndcg_at_20": 0.40326, + "ndcg_at_3": 0.18308, + "ndcg_at_5": 0.23594, + "precision_at_1": 0.08947, + "precision_at_10": 0.06468, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.04832, + "precision_at_3": 0.08263, + "precision_at_5": 0.0753, + "recall_at_1": 0.08947, + "recall_at_10": 0.64678, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.96645, + "recall_at_3": 0.2479, + "recall_at_5": 0.37651 } ] }, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json new file mode 100644 index 0000000000..319e3d389b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "3f018eb7ad32218a5a4ebd704493e0834a265cf5", + "evaluation_time": 17.977893829345703, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.73529, + "cv_recall_at_10": 0.88235, + "cv_recall_at_100": 0.95588, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.89706, + "cv_recall_at_3": 0.79412, + "cv_recall_at_5": 0.82353, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.73529, + "map_at_1": 0.12315, + "map_at_10": 0.25156, + "map_at_100": 0.39829, + "map_at_1000": 0.46268, + "map_at_20": 0.30547, + "map_at_3": 0.16397, + "map_at_5": 0.20244, + "mrr_at_1": 0.7352941176470589, + "mrr_at_10": 0.7796977124183007, + "mrr_at_100": 0.782038983404294, + "mrr_at_1000": 0.7822887502278202, + "mrr_at_20": 0.7806168300653595, + "mrr_at_3": 0.7647058823529411, + "mrr_at_5": 0.7713235294117647, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.43205924601836393, + "nauc_cv_recall_at_100_max": 0.46045430774676144, + "nauc_cv_recall_at_100_std": 0.092928936942703, + "nauc_cv_recall_at_10_diff1": 0.5238482274120124, + "nauc_cv_recall_at_10_max": 0.5447795999610322, + "nauc_cv_recall_at_10_std": -0.06773626862669024, + "nauc_cv_recall_at_1_diff1": 0.12227878218725667, + "nauc_cv_recall_at_1_max": 0.6674736854281768, + "nauc_cv_recall_at_1_std": 0.14078948712021733, + "nauc_cv_recall_at_20_diff1": 0.5375955313486179, + "nauc_cv_recall_at_20_max": 0.46954147068219054, + "nauc_cv_recall_at_20_std": -0.09380403989753433, + "nauc_cv_recall_at_3_diff1": 0.032732730804488214, + "nauc_cv_recall_at_3_max": 0.6199038499058639, + "nauc_cv_recall_at_3_std": 0.1529241887407745, + "nauc_cv_recall_at_5_diff1": 0.05658962411663845, + "nauc_cv_recall_at_5_max": 0.5877996333799903, + "nauc_cv_recall_at_5_std": 0.07730142560650717, + "nauc_map_at_1000_diff1": -0.11043090289526158, + "nauc_map_at_1000_max": 0.3448001394797035, + "nauc_map_at_1000_std": 0.27617723792125437, + "nauc_map_at_100_diff1": -0.01003127759400255, + "nauc_map_at_100_max": 0.1251706076455483, + "nauc_map_at_100_std": 0.03701229955981713, + "nauc_map_at_10_diff1": 0.2139884419568207, + "nauc_map_at_10_max": -0.26177042855989124, + "nauc_map_at_10_std": -0.26537536804238954, + "nauc_map_at_1_diff1": 0.21399681007835022, + "nauc_map_at_1_max": -0.3465990811601411, + "nauc_map_at_1_std": -0.29206561782006085, + "nauc_map_at_20_diff1": 0.1442752699133972, + "nauc_map_at_20_max": -0.15930665035054287, + "nauc_map_at_20_std": -0.21656927646466867, + "nauc_map_at_3_diff1": 0.23213875592486613, + "nauc_map_at_3_max": -0.3123546266529488, + "nauc_map_at_3_std": -0.28815041773010014, + "nauc_map_at_5_diff1": 0.2555448109113392, + "nauc_map_at_5_max": -0.29279690087015, + "nauc_map_at_5_std": -0.2774467752271305, + "nauc_mrr_at_1000_diff1": 0.12342303172342535, + "nauc_mrr_at_1000_max": 0.6362159972261828, + "nauc_mrr_at_1000_std": 0.12097023869090642, + "nauc_mrr_at_100_diff1": 0.12414954187540776, + "nauc_mrr_at_100_max": 0.6357802506977609, + "nauc_mrr_at_100_std": 0.12039621076657632, + "nauc_mrr_at_10_diff1": 0.12909230332110294, + "nauc_mrr_at_10_max": 0.6368549836172475, + "nauc_mrr_at_10_std": 0.11895636170173347, + "nauc_mrr_at_1_diff1": 0.12227878218725667, + "nauc_mrr_at_1_max": 0.6674736854281768, + "nauc_mrr_at_1_std": 0.14078948712021733, + "nauc_mrr_at_20_diff1": 0.1279569336937743, + "nauc_mrr_at_20_max": 0.6347997662966113, + "nauc_mrr_at_20_std": 0.11886311296272292, + "nauc_mrr_at_3_diff1": 0.0819493669888214, + "nauc_mrr_at_3_max": 0.6460493610936507, + "nauc_mrr_at_3_std": 0.14625466858168346, + "nauc_mrr_at_5_diff1": 0.08926776001340989, + "nauc_mrr_at_5_max": 0.6409743370271209, + "nauc_mrr_at_5_std": 0.13086158199344464, + "nauc_ndcg_at_1000_diff1": 0.0019895005695910535, + "nauc_ndcg_at_1000_max": 0.45479785106891296, + "nauc_ndcg_at_1000_std": 0.24337684988960884, + "nauc_ndcg_at_100_diff1": -0.09459953169491665, + "nauc_ndcg_at_100_max": 0.3603949417192614, + "nauc_ndcg_at_100_std": 0.25977974826865124, + "nauc_ndcg_at_10_diff1": -0.13743096053058393, + "nauc_ndcg_at_10_max": 0.5086455404107587, + "nauc_ndcg_at_10_std": 0.2942246989065215, + "nauc_ndcg_at_1_diff1": 0.12227878218725667, + "nauc_ndcg_at_1_max": 0.6674736854281768, + "nauc_ndcg_at_1_std": 0.14078948712021733, + "nauc_ndcg_at_20_diff1": -0.12980804316788766, + "nauc_ndcg_at_20_max": 0.4455786946909776, + "nauc_ndcg_at_20_std": 0.31146671038942925, + "nauc_ndcg_at_3_diff1": -0.12622874187697183, + "nauc_ndcg_at_3_max": 0.6321053376737653, + "nauc_ndcg_at_3_std": 0.2970450461706155, + "nauc_ndcg_at_5_diff1": -0.1167444253887786, + "nauc_ndcg_at_5_max": 0.5830509606014207, + "nauc_ndcg_at_5_std": 0.3092017793979728, + "nauc_precision_at_1000_diff1": -0.20870904079910094, + "nauc_precision_at_1000_max": 0.553838513916762, + "nauc_precision_at_1000_std": 0.5530198613029336, + "nauc_precision_at_100_diff1": -0.35505246743099134, + "nauc_precision_at_100_max": 0.7356593773083951, + "nauc_precision_at_100_std": 0.6881226249275548, + "nauc_precision_at_10_diff1": -0.32094129231704527, + "nauc_precision_at_10_max": 0.6957334058235605, + "nauc_precision_at_10_std": 0.4986922302670923, + "nauc_precision_at_1_diff1": 0.12227878218725667, + "nauc_precision_at_1_max": 0.6674736854281768, + "nauc_precision_at_1_std": 0.14078948712021733, + "nauc_precision_at_20_diff1": -0.348557232862726, + "nauc_precision_at_20_max": 0.7349093491157456, + "nauc_precision_at_20_std": 0.5403176104598045, + "nauc_precision_at_3_diff1": -0.19598105078336095, + "nauc_precision_at_3_max": 0.6900728385079788, + "nauc_precision_at_3_std": 0.38284792684568514, + "nauc_precision_at_5_diff1": -0.24325694519449578, + "nauc_precision_at_5_max": 0.6894112316594818, + "nauc_precision_at_5_std": 0.43885055756284386, + "nauc_recall_at_1000_diff1": 0.1247114496786093, + "nauc_recall_at_1000_max": 0.1972100012754324, + "nauc_recall_at_1000_std": 0.45358985850344147, + "nauc_recall_at_100_diff1": -0.007219441799189107, + "nauc_recall_at_100_max": -0.2726450323976995, + "nauc_recall_at_100_std": -0.09774220148121496, + "nauc_recall_at_10_diff1": 0.2493200864593003, + "nauc_recall_at_10_max": -0.35624261159143006, + "nauc_recall_at_10_std": -0.3178591018768707, + "nauc_recall_at_1_diff1": 0.21399681007835022, + "nauc_recall_at_1_max": -0.3465990811601411, + "nauc_recall_at_1_std": -0.29206561782006085, + "nauc_recall_at_20_diff1": 0.17625014384578389, + "nauc_recall_at_20_max": -0.3372766383490987, + "nauc_recall_at_20_std": -0.30264233629183895, + "nauc_recall_at_3_diff1": 0.23996176062420918, + "nauc_recall_at_3_max": -0.3204556669767577, + "nauc_recall_at_3_std": -0.292832765895384, + "nauc_recall_at_5_diff1": 0.28900424501627087, + "nauc_recall_at_5_max": -0.31760291427784704, + "nauc_recall_at_5_std": -0.2855625586155051, + "ndcg_at_1": 0.73529, + "ndcg_at_10": 0.60178, + "ndcg_at_100": 0.61347, + "ndcg_at_1000": 0.69898, + "ndcg_at_20": 0.58929, + "ndcg_at_3": 0.62141, + "ndcg_at_5": 0.61571, + "precision_at_1": 0.73529, + "precision_at_10": 0.45, + "precision_at_100": 0.19691, + "precision_at_1000": 0.04228, + "precision_at_20": 0.37647, + "precision_at_3": 0.54412, + "precision_at_5": 0.51176, + "recall_at_1": 0.12315, + "recall_at_10": 0.32137, + "recall_at_100": 0.68206, + "recall_at_1000": 0.93425, + "recall_at_20": 0.43253, + "recall_at_3": 0.16748, + "recall_at_5": 0.22328 + } + ] + }, + "task_name": "ROxfordEasyI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json new file mode 100644 index 0000000000..f957d7315e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f20b30211b7ba3fc64a02bd83998fe75f3023719", + "evaluation_time": 17.18506622314453, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.04286, + "cv_recall_at_10": 0.44286, + "cv_recall_at_100": 0.77143, + "cv_recall_at_1000": 0.92857, + "cv_recall_at_20": 0.52857, + "cv_recall_at_3": 0.18571, + "cv_recall_at_5": 0.27143, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.04286, + "map_at_1": 0.00178, + "map_at_10": 0.02316, + "map_at_100": 0.03694, + "map_at_1000": 0.05036, + "map_at_20": 0.02814, + "map_at_3": 0.01285, + "map_at_5": 0.01752, + "mrr_at_1": 0.04285714285714286, + "mrr_at_10": 0.13929705215419497, + "mrr_at_100": 0.15254576393638014, + "mrr_at_1000": 0.15348328701607364, + "mrr_at_20": 0.14564255189255187, + "mrr_at_3": 0.1, + "mrr_at_5": 0.11857142857142858, + "nauc_cv_recall_at_1000_diff1": 0.47861811391223064, + "nauc_cv_recall_at_1000_max": 0.5808590102707742, + "nauc_cv_recall_at_1000_std": 0.09140989729224985, + "nauc_cv_recall_at_100_diff1": 0.5586401902191371, + "nauc_cv_recall_at_100_max": -0.12197159565580537, + "nauc_cv_recall_at_100_std": -0.46324143692564745, + "nauc_cv_recall_at_10_diff1": 0.33702203892735166, + "nauc_cv_recall_at_10_max": -0.2566107451434116, + "nauc_cv_recall_at_10_std": -0.2562935040214977, + "nauc_cv_recall_at_1_diff1": 0.5001341561577676, + "nauc_cv_recall_at_1_max": -0.12592791342455964, + "nauc_cv_recall_at_1_std": 0.13701815580001792, + "nauc_cv_recall_at_20_diff1": 0.33340522734848327, + "nauc_cv_recall_at_20_max": -0.27021038803160713, + "nauc_cv_recall_at_20_std": -0.2604850885276762, + "nauc_cv_recall_at_3_diff1": 0.42356508767210377, + "nauc_cv_recall_at_3_max": -0.30471252842923874, + "nauc_cv_recall_at_3_std": -0.2722848548580372, + "nauc_cv_recall_at_5_diff1": 0.37677403680683497, + "nauc_cv_recall_at_5_max": -0.30530641993804797, + "nauc_cv_recall_at_5_std": -0.2763347032980382, + "nauc_map_at_1000_diff1": 0.32492788395298233, + "nauc_map_at_1000_max": -0.264513294885872, + "nauc_map_at_1000_std": -0.06025413474178467, + "nauc_map_at_100_diff1": 0.33932455574204345, + "nauc_map_at_100_max": -0.3183944729391321, + "nauc_map_at_100_std": -0.11067793875024108, + "nauc_map_at_10_diff1": 0.3627322367418886, + "nauc_map_at_10_max": -0.270787009945954, + "nauc_map_at_10_std": -0.12449041958590679, + "nauc_map_at_1_diff1": 0.8349360037495761, + "nauc_map_at_1_max": 0.04030601005296298, + "nauc_map_at_1_std": 0.28342764489523986, + "nauc_map_at_20_diff1": 0.36471735689250456, + "nauc_map_at_20_max": -0.27443485632176895, + "nauc_map_at_20_std": -0.10351956211548237, + "nauc_map_at_3_diff1": 0.33740985231621823, + "nauc_map_at_3_max": -0.26557891525076527, + "nauc_map_at_3_std": -0.13603337759846046, + "nauc_map_at_5_diff1": 0.3822623605794258, + "nauc_map_at_5_max": -0.26535510732907475, + "nauc_map_at_5_std": -0.15188815608303466, + "nauc_mrr_at_1000_diff1": 0.42803033998914947, + "nauc_mrr_at_1000_max": -0.25330248029839103, + "nauc_mrr_at_1000_std": -0.1498777504287341, + "nauc_mrr_at_100_diff1": 0.428756525260202, + "nauc_mrr_at_100_max": -0.255106401697983, + "nauc_mrr_at_100_std": -0.15247667584774333, + "nauc_mrr_at_10_diff1": 0.4269277134986295, + "nauc_mrr_at_10_max": -0.26024547551332144, + "nauc_mrr_at_10_std": -0.15027887241014917, + "nauc_mrr_at_1_diff1": 0.5001341561577676, + "nauc_mrr_at_1_max": -0.12592791342455964, + "nauc_mrr_at_1_std": 0.13701815580001792, + "nauc_mrr_at_20_diff1": 0.4273548864254061, + "nauc_mrr_at_20_max": -0.2649857851418372, + "nauc_mrr_at_20_std": -0.150811901899745, + "nauc_mrr_at_3_diff1": 0.46234460520174797, + "nauc_mrr_at_3_max": -0.26846705418134004, + "nauc_mrr_at_3_std": -0.14951136379707816, + "nauc_mrr_at_5_diff1": 0.4315603993226756, + "nauc_mrr_at_5_max": -0.2751708043421038, + "nauc_mrr_at_5_std": -0.15676414005890574, + "nauc_ndcg_at_1000_diff1": 0.3031539661086623, + "nauc_ndcg_at_1000_max": -0.06324949257415059, + "nauc_ndcg_at_1000_std": 0.03291344536352667, + "nauc_ndcg_at_100_diff1": 0.3537878100611368, + "nauc_ndcg_at_100_max": -0.3440256642252378, + "nauc_ndcg_at_100_std": -0.13928638317122793, + "nauc_ndcg_at_10_diff1": 0.31423572757944224, + "nauc_ndcg_at_10_max": -0.27799943432453955, + "nauc_ndcg_at_10_std": -0.12000726018713354, + "nauc_ndcg_at_1_diff1": 0.5001341561577676, + "nauc_ndcg_at_1_max": -0.12592791342455964, + "nauc_ndcg_at_1_std": 0.13701815580001792, + "nauc_ndcg_at_20_diff1": 0.3527561180413498, + "nauc_ndcg_at_20_max": -0.29516361937414604, + "nauc_ndcg_at_20_std": -0.10951825882002067, + "nauc_ndcg_at_3_diff1": 0.3265974284434805, + "nauc_ndcg_at_3_max": -0.27320998825572146, + "nauc_ndcg_at_3_std": -0.12237689567430368, + "nauc_ndcg_at_5_diff1": 0.32240119227390085, + "nauc_ndcg_at_5_max": -0.30077766985994786, + "nauc_ndcg_at_5_std": -0.15147497522509035, + "nauc_precision_at_1000_diff1": -0.12639282009702774, + "nauc_precision_at_1000_max": 0.22826737103796205, + "nauc_precision_at_1000_std": 0.32628514757600596, + "nauc_precision_at_100_diff1": 0.2772154726137188, + "nauc_precision_at_100_max": -0.23028476826992986, + "nauc_precision_at_100_std": -0.1303180540104937, + "nauc_precision_at_10_diff1": 0.29107138713345343, + "nauc_precision_at_10_max": -0.30600784337992426, + "nauc_precision_at_10_std": -0.12021874830794935, + "nauc_precision_at_1_diff1": 0.5001341561577676, + "nauc_precision_at_1_max": -0.12592791342455964, + "nauc_precision_at_1_std": 0.13701815580001792, + "nauc_precision_at_20_diff1": 0.27727122281419164, + "nauc_precision_at_20_max": -0.3127822679663893, + "nauc_precision_at_20_std": -0.07951548670204237, + "nauc_precision_at_3_diff1": 0.3064711218627153, + "nauc_precision_at_3_max": -0.29553472432214506, + "nauc_precision_at_3_std": -0.17296643483519816, + "nauc_precision_at_5_diff1": 0.27289717372431765, + "nauc_precision_at_5_max": -0.33688627375325547, + "nauc_precision_at_5_std": -0.1943667406192727, + "nauc_recall_at_1000_diff1": 0.2612937021120264, + "nauc_recall_at_1000_max": -0.04177364682142833, + "nauc_recall_at_1000_std": -0.10225779216767208, + "nauc_recall_at_100_diff1": 0.23045925470740378, + "nauc_recall_at_100_max": -0.32266175361589855, + "nauc_recall_at_100_std": -0.10122639227831537, + "nauc_recall_at_10_diff1": 0.3214491777002914, + "nauc_recall_at_10_max": -0.2191819309348708, + "nauc_recall_at_10_std": -0.13294514318629588, + "nauc_recall_at_1_diff1": 0.8349360037495761, + "nauc_recall_at_1_max": 0.04030601005296298, + "nauc_recall_at_1_std": 0.28342764489523986, + "nauc_recall_at_20_diff1": 0.30304432643409973, + "nauc_recall_at_20_max": -0.2369864752715589, + "nauc_recall_at_20_std": -0.11319354561879752, + "nauc_recall_at_3_diff1": 0.2610862099372524, + "nauc_recall_at_3_max": -0.26808957441898906, + "nauc_recall_at_3_std": -0.1754514836876096, + "nauc_recall_at_5_diff1": 0.3659777743786138, + "nauc_recall_at_5_max": -0.23557478979416202, + "nauc_recall_at_5_std": -0.1749293090721626, + "ndcg_at_1": 0.04286, + "ndcg_at_10": 0.08519, + "ndcg_at_100": 0.12707, + "ndcg_at_1000": 0.26609, + "ndcg_at_20": 0.08785, + "ndcg_at_3": 0.07857, + "ndcg_at_5": 0.08201, + "precision_at_1": 0.04286, + "precision_at_10": 0.07571, + "precision_at_100": 0.03571, + "precision_at_1000": 0.01857, + "precision_at_20": 0.05714, + "precision_at_3": 0.08571, + "precision_at_5": 0.08, + "recall_at_1": 0.00178, + "recall_at_10": 0.07379, + "recall_at_100": 0.22471, + "recall_at_1000": 0.60888, + "recall_at_20": 0.09447, + "recall_at_3": 0.02931, + "recall_at_5": 0.04839 + } + ] + }, + "task_name": "ROxfordHardI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordI2IRetrieval.json deleted file mode 100644 index e47a943219..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "d8daad98b4e4896a7f7fa1b3485a22420242d4fc", - "evaluation_time": 29.71091365814209, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.80946, - "cv_recall_at_10": 0.99953, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 0.97636, - "cv_recall_at_5": 0.99291, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.80946, - "map_at_1": 0.00083, - "map_at_10": 0.00583, - "map_at_100": 0.04516, - "map_at_1000": 0.27358, - "map_at_20": 0.01067, - "map_at_3": 0.00211, - "map_at_5": 0.00324, - "mrr_at_1": 0.9985815602836879, - "mrr_at_10": 0.9992119779353821, - "mrr_at_100": 0.9992119779353821, - "mrr_at_1000": 0.9992119779353821, - "mrr_at_20": 0.9992119779353821, - "mrr_at_3": 0.9992119779353821, - "mrr_at_5": 0.9992119779353821, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": 0.5540638703677596, - "nauc_cv_recall_at_10_max": 0.7221878767269868, - "nauc_cv_recall_at_10_std": 0.869246700256431, - "nauc_cv_recall_at_1_diff1": -0.24158864035258856, - "nauc_cv_recall_at_1_max": -0.0420448021592233, - "nauc_cv_recall_at_1_std": 0.04484895202286055, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": -0.2813966714480976, - "nauc_cv_recall_at_3_max": -0.04571848249527183, - "nauc_cv_recall_at_3_std": 0.06606763479339506, - "nauc_cv_recall_at_5_diff1": -0.057737057928295976, - "nauc_cv_recall_at_5_max": 0.017855647720182007, - "nauc_cv_recall_at_5_std": 0.08427153819844212, - "nauc_map_at_1000_diff1": -0.3857995513654731, - "nauc_map_at_1000_max": 0.03808456962512866, - "nauc_map_at_1000_std": -0.06663520492889344, - "nauc_map_at_100_diff1": -0.34437629024420324, - "nauc_map_at_100_max": 0.020553049295469906, - "nauc_map_at_100_std": 0.0556790533768489, - "nauc_map_at_10_diff1": -0.18993364966776907, - "nauc_map_at_10_max": 0.006866799821313959, - "nauc_map_at_10_std": 0.13195237855420855, - "nauc_map_at_1_diff1": -0.05431107143472811, - "nauc_map_at_1_max": 0.016884860597154504, - "nauc_map_at_1_std": 0.11466554606849116, - "nauc_map_at_20_diff1": -0.2414863261116838, - "nauc_map_at_20_max": 0.008935096908909975, - "nauc_map_at_20_std": 0.11557906533166638, - "nauc_map_at_3_diff1": -0.12217591294530498, - "nauc_map_at_3_max": 0.008152934040333987, - "nauc_map_at_3_std": 0.13797559913063503, - "nauc_map_at_5_diff1": -0.13888462292383946, - "nauc_map_at_5_max": 0.007634414930511178, - "nauc_map_at_5_std": 0.13391202111166425, - "nauc_mrr_at_1000_diff1": 0.9476986801026335, - "nauc_mrr_at_1000_max": 1.0, - "nauc_mrr_at_1000_std": 0.9166563630181881, - "nauc_mrr_at_100_diff1": 0.9476986801026335, - "nauc_mrr_at_100_max": 1.0, - "nauc_mrr_at_100_std": 0.9166563630181881, - "nauc_mrr_at_10_diff1": 0.9476986801026335, - "nauc_mrr_at_10_max": 1.0, - "nauc_mrr_at_10_std": 0.9166563630181881, - "nauc_mrr_at_1_diff1": 0.9564155667521437, - "nauc_mrr_at_1_max": 1.0, - "nauc_mrr_at_1_std": 0.9073959589089956, - "nauc_mrr_at_20_diff1": 0.9476986801026335, - "nauc_mrr_at_20_max": 1.0, - "nauc_mrr_at_20_std": 0.9166563630181881, - "nauc_mrr_at_3_diff1": 0.9476986801026335, - "nauc_mrr_at_3_max": 1.0, - "nauc_mrr_at_3_std": 0.9166563630181881, - "nauc_mrr_at_5_diff1": 0.9476986801026335, - "nauc_mrr_at_5_max": 1.0, - "nauc_mrr_at_5_std": 0.9166563630181881, - "nauc_ndcg_at_1000_diff1": -0.29077931122633116, - "nauc_ndcg_at_1000_max": 0.014960510380277934, - "nauc_ndcg_at_1000_std": -0.12483981180901933, - "nauc_ndcg_at_100_diff1": -0.3609816949819758, - "nauc_ndcg_at_100_max": 0.021419925854651446, - "nauc_ndcg_at_100_std": -0.17265752707444715, - "nauc_ndcg_at_10_diff1": -0.27457750496652206, - "nauc_ndcg_at_10_max": 0.0008466224260104365, - "nauc_ndcg_at_10_std": -0.045841557226464144, - "nauc_ndcg_at_1_diff1": 0.9564155667521437, - "nauc_ndcg_at_1_max": 1.0, - "nauc_ndcg_at_1_std": 0.9073959589089956, - "nauc_ndcg_at_20_diff1": -0.3094417772089175, - "nauc_ndcg_at_20_max": 0.009714640004268442, - "nauc_ndcg_at_20_std": -0.08586776230182966, - "nauc_ndcg_at_3_diff1": -0.23757585938494089, - "nauc_ndcg_at_3_max": -0.016387555353346198, - "nauc_ndcg_at_3_std": 0.030386940725562046, - "nauc_ndcg_at_5_diff1": -0.23196460003691158, - "nauc_ndcg_at_5_max": -0.013062943721282708, - "nauc_ndcg_at_5_std": -0.010500172553175749, - "nauc_precision_at_1000_diff1": -0.17619733662463302, - "nauc_precision_at_1000_max": 0.012339801300641195, - "nauc_precision_at_1000_std": -0.1876530479926194, - "nauc_precision_at_100_diff1": -0.3626475300981416, - "nauc_precision_at_100_max": 0.025040872201569076, - "nauc_precision_at_100_std": -0.18897442356689456, - "nauc_precision_at_10_diff1": -0.2782683949086005, - "nauc_precision_at_10_max": 0.006150543325428016, - "nauc_precision_at_10_std": -0.05974378846400665, - "nauc_precision_at_1_diff1": 0.9564155667521437, - "nauc_precision_at_1_max": 1.0, - "nauc_precision_at_1_std": 0.9073959589089956, - "nauc_precision_at_20_diff1": -0.31218967700544853, - "nauc_precision_at_20_max": 0.015101101880288523, - "nauc_precision_at_20_std": -0.10187042490803115, - "nauc_precision_at_3_diff1": -0.2418495400445544, - "nauc_precision_at_3_max": -0.017073069963463458, - "nauc_precision_at_3_std": 0.024904433150907852, - "nauc_precision_at_5_diff1": -0.23061542055201162, - "nauc_precision_at_5_max": -0.013293477378375076, - "nauc_precision_at_5_std": -0.02217796284197703, - "nauc_recall_at_1000_diff1": -0.1758800893625114, - "nauc_recall_at_1000_max": 0.025501344233019, - "nauc_recall_at_1000_std": 0.042539963927428814, - "nauc_recall_at_100_diff1": -0.2486522818740174, - "nauc_recall_at_100_max": 0.019163648439019607, - "nauc_recall_at_100_std": 0.08018065043566762, - "nauc_recall_at_10_diff1": -0.1643890021643603, - "nauc_recall_at_10_max": 0.015612163534741345, - "nauc_recall_at_10_std": 0.12932048693738263, - "nauc_recall_at_1_diff1": -0.05431107143472811, - "nauc_recall_at_1_max": 0.016884860597154504, - "nauc_recall_at_1_std": 0.11466554606849116, - "nauc_recall_at_20_diff1": -0.19900512249205043, - "nauc_recall_at_20_max": 0.013811903907386002, - "nauc_recall_at_20_std": 0.11661397478238389, - "nauc_recall_at_3_diff1": -0.11946531860608983, - "nauc_recall_at_3_max": 0.009964580844462759, - "nauc_recall_at_3_std": 0.13655833592145109, - "nauc_recall_at_5_diff1": -0.1255934399523269, - "nauc_recall_at_5_max": 0.012044411079830297, - "nauc_recall_at_5_std": 0.1305906448336473, - "ndcg_at_1": 0.99858, - "ndcg_at_10": 0.82209, - "ndcg_at_100": 0.74273, - "ndcg_at_1000": 0.56583, - "ndcg_at_20": 0.79415, - "ndcg_at_3": 0.89211, - "ndcg_at_5": 0.8591, - "precision_at_1": 0.99858, - "precision_at_10": 0.79054, - "precision_at_100": 0.72617, - "precision_at_1000": 0.49807, - "precision_at_20": 0.76664, - "precision_at_3": 0.86383, - "precision_at_5": 0.82752, - "recall_at_1": 0.00083, - "recall_at_10": 0.00647, - "recall_at_100": 0.05838, - "recall_at_1000": 0.3868, - "recall_at_20": 0.01248, - "recall_at_3": 0.00214, - "recall_at_5": 0.00341 - } - ] - }, - "task_name": "ROxfordI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json new file mode 100644 index 0000000000..27241284b1 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "3bd28e9c45e15f299117c634799f7035c4de2d31", + "evaluation_time": 17.523592710494995, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.75714, + "cv_recall_at_10": 0.87143, + "cv_recall_at_100": 0.94286, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.88571, + "cv_recall_at_3": 0.82857, + "cv_recall_at_5": 0.85714, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.75714, + "map_at_1": 0.04928, + "map_at_10": 0.14238, + "map_at_100": 0.25975, + "map_at_1000": 0.33, + "map_at_20": 0.1832, + "map_at_3": 0.08106, + "map_at_5": 0.10574, + "mrr_at_1": 0.7571428571428571, + "mrr_at_10": 0.7992857142857144, + "mrr_at_100": 0.8018753503269448, + "mrr_at_1000": 0.8022392070599913, + "mrr_at_20": 0.8001785714285715, + "mrr_at_3": 0.7904761904761906, + "mrr_at_5": 0.796904761904762, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.15581232492997169, + "nauc_cv_recall_at_100_max": 0.2811624649859951, + "nauc_cv_recall_at_100_std": -0.12196545284780497, + "nauc_cv_recall_at_10_diff1": 0.394540014958862, + "nauc_cv_recall_at_10_max": 0.4610535313601873, + "nauc_cv_recall_at_10_std": -0.14937493321936154, + "nauc_cv_recall_at_1_diff1": 0.3099268419602683, + "nauc_cv_recall_at_1_max": 0.6277204689460957, + "nauc_cv_recall_at_1_std": 0.11781811503259924, + "nauc_cv_recall_at_20_diff1": 0.39261983863312777, + "nauc_cv_recall_at_20_max": 0.3849074513526345, + "nauc_cv_recall_at_20_std": -0.18147840531561454, + "nauc_cv_recall_at_3_diff1": 0.29657669519420665, + "nauc_cv_recall_at_3_max": 0.5705233706385782, + "nauc_cv_recall_at_3_std": 0.04756418696510771, + "nauc_cv_recall_at_5_diff1": 0.37162293488824066, + "nauc_cv_recall_at_5_max": 0.5234207968901848, + "nauc_cv_recall_at_5_std": -0.06068999028182738, + "nauc_map_at_1000_diff1": -0.14810539444535628, + "nauc_map_at_1000_max": 0.42639970063449095, + "nauc_map_at_1000_std": 0.35965851851847036, + "nauc_map_at_100_diff1": -0.04654719636323819, + "nauc_map_at_100_max": 0.20124568398625026, + "nauc_map_at_100_std": 0.09779920001336984, + "nauc_map_at_10_diff1": 0.19553982602379108, + "nauc_map_at_10_max": -0.16962211647877953, + "nauc_map_at_10_std": -0.2049718676191143, + "nauc_map_at_1_diff1": 0.19831336261662813, + "nauc_map_at_1_max": -0.26048526225990315, + "nauc_map_at_1_std": -0.24848660044643464, + "nauc_map_at_20_diff1": 0.11533311201293248, + "nauc_map_at_20_max": -0.06940468788280867, + "nauc_map_at_20_std": -0.15394999023265438, + "nauc_map_at_3_diff1": 0.22661360439380449, + "nauc_map_at_3_max": -0.24582246289343465, + "nauc_map_at_3_std": -0.24358060859761363, + "nauc_map_at_5_diff1": 0.2391073300885911, + "nauc_map_at_5_max": -0.21313902222536138, + "nauc_map_at_5_std": -0.22555952910665222, + "nauc_mrr_at_1000_diff1": 0.3118072480358942, + "nauc_mrr_at_1000_max": 0.5896434235555521, + "nauc_mrr_at_1000_std": 0.07565750084460038, + "nauc_mrr_at_100_diff1": 0.3114581767470047, + "nauc_mrr_at_100_max": 0.5888175740478946, + "nauc_mrr_at_100_std": 0.07590573329917173, + "nauc_mrr_at_10_diff1": 0.31501956699434985, + "nauc_mrr_at_10_max": 0.5906694492065899, + "nauc_mrr_at_10_std": 0.07282707502971797, + "nauc_mrr_at_1_diff1": 0.3099268419602683, + "nauc_mrr_at_1_max": 0.6277204689460957, + "nauc_mrr_at_1_std": 0.11781811503259924, + "nauc_mrr_at_20_diff1": 0.31460997067981183, + "nauc_mrr_at_20_max": 0.5883036453949262, + "nauc_mrr_at_20_std": 0.07253787758405468, + "nauc_mrr_at_3_diff1": 0.298829153668517, + "nauc_mrr_at_3_max": 0.6036890630335515, + "nauc_mrr_at_3_std": 0.09929911274966623, + "nauc_mrr_at_5_diff1": 0.31309074738187426, + "nauc_mrr_at_5_max": 0.5968819745150035, + "nauc_mrr_at_5_std": 0.08124058142418439, + "nauc_ndcg_at_1000_diff1": 0.023759962305708315, + "nauc_ndcg_at_1000_max": 0.42118350886771483, + "nauc_ndcg_at_1000_std": 0.21503403862781068, + "nauc_ndcg_at_100_diff1": -0.10602196843045252, + "nauc_ndcg_at_100_max": 0.4040222401408388, + "nauc_ndcg_at_100_std": 0.3422251986340049, + "nauc_ndcg_at_10_diff1": -0.14230825595892613, + "nauc_ndcg_at_10_max": 0.5717619317460108, + "nauc_ndcg_at_10_std": 0.35188240284777716, + "nauc_ndcg_at_1_diff1": 0.3099268419602683, + "nauc_ndcg_at_1_max": 0.6277204689460957, + "nauc_ndcg_at_1_std": 0.11781811503259924, + "nauc_ndcg_at_20_diff1": -0.15743789243142528, + "nauc_ndcg_at_20_max": 0.5724054251457065, + "nauc_ndcg_at_20_std": 0.3797961049903619, + "nauc_ndcg_at_3_diff1": 0.02676887075350004, + "nauc_ndcg_at_3_max": 0.5535721465155271, + "nauc_ndcg_at_3_std": 0.22786549855388236, + "nauc_ndcg_at_5_diff1": -0.04449400646171614, + "nauc_ndcg_at_5_max": 0.555861797739005, + "nauc_ndcg_at_5_std": 0.27833851228984907, + "nauc_precision_at_1000_diff1": -0.19203752528557633, + "nauc_precision_at_1000_max": 0.4830680560318716, + "nauc_precision_at_1000_std": 0.5059538376358517, + "nauc_precision_at_100_diff1": -0.29506043721080727, + "nauc_precision_at_100_max": 0.658086670179558, + "nauc_precision_at_100_std": 0.6166722456503947, + "nauc_precision_at_10_diff1": -0.2282600490658613, + "nauc_precision_at_10_max": 0.5965276467257976, + "nauc_precision_at_10_std": 0.42049443291187055, + "nauc_precision_at_1_diff1": 0.3099268419602683, + "nauc_precision_at_1_max": 0.6277204689460957, + "nauc_precision_at_1_std": 0.11781811503259924, + "nauc_precision_at_20_diff1": -0.2777369235938004, + "nauc_precision_at_20_max": 0.6422763272584597, + "nauc_precision_at_20_std": 0.46973331485105685, + "nauc_precision_at_3_diff1": -0.03905783426100687, + "nauc_precision_at_3_max": 0.548195217505554, + "nauc_precision_at_3_std": 0.2602422302381689, + "nauc_precision_at_5_diff1": -0.12812907648800725, + "nauc_precision_at_5_max": 0.5661902630852285, + "nauc_precision_at_5_std": 0.33254144330289526, + "nauc_recall_at_1000_diff1": 0.11552874226597364, + "nauc_recall_at_1000_max": 0.1991754113531379, + "nauc_recall_at_1000_std": 0.05667927260800616, + "nauc_recall_at_100_diff1": -0.01602892704072587, + "nauc_recall_at_100_max": -0.12257633928876718, + "nauc_recall_at_100_std": -0.01682143204953458, + "nauc_recall_at_10_diff1": 0.22298978148074566, + "nauc_recall_at_10_max": -0.22859971193805304, + "nauc_recall_at_10_std": -0.23715467924390157, + "nauc_recall_at_1_diff1": 0.19831336261662813, + "nauc_recall_at_1_max": -0.26048526225990315, + "nauc_recall_at_1_std": -0.24848660044643464, + "nauc_recall_at_20_diff1": 0.16037912600194174, + "nauc_recall_at_20_max": -0.1835918931237162, + "nauc_recall_at_20_std": -0.20833669696201074, + "nauc_recall_at_3_diff1": 0.2179234117693627, + "nauc_recall_at_3_max": -0.2525912771978034, + "nauc_recall_at_3_std": -0.24319572308223014, + "nauc_recall_at_5_diff1": 0.2530123469958082, + "nauc_recall_at_5_max": -0.22138719273185706, + "nauc_recall_at_5_std": -0.2215811361906131, + "ndcg_at_1": 0.75714, + "ndcg_at_10": 0.58385, + "ndcg_at_100": 0.48629, + "ndcg_at_1000": 0.59891, + "ndcg_at_20": 0.52663, + "ndcg_at_3": 0.6553, + "ndcg_at_5": 0.63094, + "precision_at_1": 0.75714, + "precision_at_10": 0.51286, + "precision_at_100": 0.227, + "precision_at_1000": 0.05964, + "precision_at_20": 0.42286, + "precision_at_3": 0.61429, + "precision_at_5": 0.57714, + "recall_at_1": 0.04928, + "recall_at_10": 0.17721, + "recall_at_100": 0.44393, + "recall_at_1000": 0.76857, + "recall_at_20": 0.24576, + "recall_at_3": 0.08507, + "recall_at_5": 0.11871 + } + ] + }, + "task_name": "ROxfordMediumI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json new file mode 100644 index 0000000000..bd4a6c53b9 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "a7293da8a341de665ee4dcb2f209281df342d80b", + "evaluation_time": 22.57354736328125, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.94286, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94286, + "map_at_1": 0.02765, + "map_at_10": 0.11901, + "map_at_100": 0.46776, + "map_at_1000": 0.58942, + "map_at_20": 0.19679, + "map_at_3": 0.05048, + "map_at_5": 0.07407, + "mrr_at_1": 0.9428571428571428, + "mrr_at_10": 0.9666666666666666, + "mrr_at_100": 0.9666666666666666, + "mrr_at_1000": 0.9666666666666666, + "mrr_at_20": 0.9666666666666666, + "mrr_at_3": 0.9666666666666666, + "mrr_at_5": 0.9666666666666666, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": -0.303338001867413, + "nauc_cv_recall_at_1_max": -0.11951447245564903, + "nauc_cv_recall_at_1_std": 0.38328664799253065, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_map_at_1000_diff1": -0.2774518254418231, + "nauc_map_at_1000_max": 0.25032079891265085, + "nauc_map_at_1000_std": 0.45081359384807485, + "nauc_map_at_100_diff1": -0.04907209740285267, + "nauc_map_at_100_max": 0.18686662832088577, + "nauc_map_at_100_std": 0.31839565681325593, + "nauc_map_at_10_diff1": 0.4545859442374117, + "nauc_map_at_10_max": -0.04495045809094941, + "nauc_map_at_10_std": 0.002947268581625076, + "nauc_map_at_1_diff1": 0.7049183369958152, + "nauc_map_at_1_max": -0.2616079702049637, + "nauc_map_at_1_std": -0.3618487407893112, + "nauc_map_at_20_diff1": 0.30890872824690435, + "nauc_map_at_20_max": 0.07581745603833642, + "nauc_map_at_20_std": 0.15307053171724933, + "nauc_map_at_3_diff1": 0.5935344781456238, + "nauc_map_at_3_max": -0.20146382202873225, + "nauc_map_at_3_std": -0.22821424999017603, + "nauc_map_at_5_diff1": 0.5703897004682528, + "nauc_map_at_5_max": -0.16308005302080678, + "nauc_map_at_5_std": -0.1620575175690372, + "nauc_mrr_at_1000_diff1": -0.39652527677737137, + "nauc_mrr_at_1000_max": -0.10590903027877008, + "nauc_mrr_at_1000_std": 0.452714419100977, + "nauc_mrr_at_100_diff1": -0.39652527677737137, + "nauc_mrr_at_100_max": -0.10590903027877008, + "nauc_mrr_at_100_std": 0.452714419100977, + "nauc_mrr_at_10_diff1": -0.39652527677737137, + "nauc_mrr_at_10_max": -0.10590903027877008, + "nauc_mrr_at_10_std": 0.452714419100977, + "nauc_mrr_at_1_diff1": -0.303338001867413, + "nauc_mrr_at_1_max": -0.11951447245564903, + "nauc_mrr_at_1_std": 0.38328664799253065, + "nauc_mrr_at_20_diff1": -0.39652527677737137, + "nauc_mrr_at_20_max": -0.10590903027877008, + "nauc_mrr_at_20_std": 0.452714419100977, + "nauc_mrr_at_3_diff1": -0.39652527677737137, + "nauc_mrr_at_3_max": -0.10590903027877008, + "nauc_mrr_at_3_std": 0.452714419100977, + "nauc_mrr_at_5_diff1": -0.39652527677737137, + "nauc_mrr_at_5_max": -0.10590903027877008, + "nauc_mrr_at_5_std": 0.452714419100977, + "nauc_ndcg_at_1000_diff1": -0.49032575238867226, + "nauc_ndcg_at_1000_max": 0.30827149571860624, + "nauc_ndcg_at_1000_std": 0.5422742306681069, + "nauc_ndcg_at_100_diff1": -0.2712801258824707, + "nauc_ndcg_at_100_max": 0.2469510949945573, + "nauc_ndcg_at_100_std": 0.4367311454283413, + "nauc_ndcg_at_10_diff1": -0.519586478031318, + "nauc_ndcg_at_10_max": 0.35401668386388874, + "nauc_ndcg_at_10_std": 0.5596598049318611, + "nauc_ndcg_at_1_diff1": -0.303338001867413, + "nauc_ndcg_at_1_max": -0.11951447245564903, + "nauc_ndcg_at_1_std": 0.38328664799253065, + "nauc_ndcg_at_20_diff1": -0.4606517361381654, + "nauc_ndcg_at_20_max": 0.38378869676003413, + "nauc_ndcg_at_20_std": 0.474186229263476, + "nauc_ndcg_at_3_diff1": -0.585149663762276, + "nauc_ndcg_at_3_max": 0.217548589813707, + "nauc_ndcg_at_3_std": 0.550167698282911, + "nauc_ndcg_at_5_diff1": -0.5267224367233376, + "nauc_ndcg_at_5_max": 0.2591401820511711, + "nauc_ndcg_at_5_std": 0.5702945720424282, + "nauc_precision_at_1000_diff1": -0.4499561707406712, + "nauc_precision_at_1000_max": 0.15634621233710536, + "nauc_precision_at_1000_std": 0.17441880422894795, + "nauc_precision_at_100_diff1": -0.4968893518170479, + "nauc_precision_at_100_max": 0.19447257428460527, + "nauc_precision_at_100_std": 0.25819757590889797, + "nauc_precision_at_10_diff1": -0.6167580528722199, + "nauc_precision_at_10_max": 0.3956822166454678, + "nauc_precision_at_10_std": 0.587829591941972, + "nauc_precision_at_1_diff1": -0.303338001867413, + "nauc_precision_at_1_max": -0.11951447245564903, + "nauc_precision_at_1_std": 0.38328664799253065, + "nauc_precision_at_20_diff1": -0.6031642916224456, + "nauc_precision_at_20_max": 0.4138166765495704, + "nauc_precision_at_20_std": 0.5161953717797159, + "nauc_precision_at_3_diff1": -0.6298090956856782, + "nauc_precision_at_3_max": 0.24751396267538517, + "nauc_precision_at_3_std": 0.557806449101913, + "nauc_precision_at_5_diff1": -0.5860255772185323, + "nauc_precision_at_5_max": 0.29785990954926145, + "nauc_precision_at_5_std": 0.5944106636660691, + "nauc_recall_at_1000_diff1": -0.12209326635132679, + "nauc_recall_at_1000_max": 0.008543724924431836, + "nauc_recall_at_1000_std": 0.25794589209932395, + "nauc_recall_at_100_diff1": 0.16704714994186, + "nauc_recall_at_100_max": 0.053397233537995845, + "nauc_recall_at_100_std": 0.10488570154536475, + "nauc_recall_at_10_diff1": 0.5240194164262459, + "nauc_recall_at_10_max": -0.03784601294172749, + "nauc_recall_at_10_std": -0.07182820315172336, + "nauc_recall_at_1_diff1": 0.7049183369958152, + "nauc_recall_at_1_max": -0.2616079702049637, + "nauc_recall_at_1_std": -0.3618487407893112, + "nauc_recall_at_20_diff1": 0.40266587722079766, + "nauc_recall_at_20_max": 0.03757413630827814, + "nauc_recall_at_20_std": 0.042791117828726286, + "nauc_recall_at_3_diff1": 0.6179795805128587, + "nauc_recall_at_3_max": -0.1966728040865525, + "nauc_recall_at_3_std": -0.2470220135910208, + "nauc_recall_at_5_diff1": 0.6188095385694616, + "nauc_recall_at_5_max": -0.1447272403059871, + "nauc_recall_at_5_std": -0.20646683990606104, + "ndcg_at_1": 0.94286, + "ndcg_at_10": 0.85179, + "ndcg_at_100": 0.71775, + "ndcg_at_1000": 0.83657, + "ndcg_at_20": 0.82388, + "ndcg_at_3": 0.8936, + "ndcg_at_5": 0.88343, + "precision_at_1": 0.94286, + "precision_at_10": 0.81429, + "precision_at_100": 0.519, + "precision_at_1000": 0.09099, + "precision_at_20": 0.77143, + "precision_at_3": 0.87619, + "precision_at_5": 0.86286, + "recall_at_1": 0.02765, + "recall_at_10": 0.1307, + "recall_at_100": 0.60631, + "recall_at_1000": 0.93409, + "recall_at_20": 0.22272, + "recall_at_3": 0.05209, + "recall_at_5": 0.07985 + } + ] + }, + "task_name": "RParisEasyI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json new file mode 100644 index 0000000000..5b15d0ae51 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fd121b6592fe946616fa85116703b94a4c61fd63", + "evaluation_time": 22.44923186302185, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.04286, + "cv_recall_at_10": 0.41429, + "cv_recall_at_100": 0.97143, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.6, + "cv_recall_at_3": 0.17143, + "cv_recall_at_5": 0.27143, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.04286, + "map_at_1": 0.00032, + "map_at_10": 0.00319, + "map_at_100": 0.04427, + "map_at_1000": 0.14044, + "map_at_20": 0.00602, + "map_at_3": 0.00096, + "map_at_5": 0.00146, + "mrr_at_1": 0.04285714285714286, + "mrr_at_10": 0.13515873015873012, + "mrr_at_100": 0.15938730320565236, + "mrr_at_1000": 0.15962520658618265, + "mrr_at_20": 0.14815883634769078, + "mrr_at_3": 0.0976190476190476, + "mrr_at_5": 0.11833333333333333, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.6909430438842217, + "nauc_cv_recall_at_100_max": 0.6790382819794577, + "nauc_cv_recall_at_100_std": 0.4143323996265152, + "nauc_cv_recall_at_10_diff1": 0.2550034187718293, + "nauc_cv_recall_at_10_max": -0.26788387263688956, + "nauc_cv_recall_at_10_std": -0.3843993125496647, + "nauc_cv_recall_at_1_diff1": 0.3989804132009661, + "nauc_cv_recall_at_1_max": -0.12905822377247125, + "nauc_cv_recall_at_1_std": -0.401663536356319, + "nauc_cv_recall_at_20_diff1": 0.12233388457787381, + "nauc_cv_recall_at_20_max": -0.21169910528770744, + "nauc_cv_recall_at_20_std": -0.2534876818046898, + "nauc_cv_recall_at_3_diff1": 0.35100458047053923, + "nauc_cv_recall_at_3_max": -0.1396262752446388, + "nauc_cv_recall_at_3_std": -0.314985425775557, + "nauc_cv_recall_at_5_diff1": 0.35853258558905104, + "nauc_cv_recall_at_5_max": -0.1935699390601907, + "nauc_cv_recall_at_5_std": -0.4389893304719292, + "nauc_map_at_1000_diff1": -0.048721647577313475, + "nauc_map_at_1000_max": 0.02185883585100512, + "nauc_map_at_1000_std": -0.014590760062175558, + "nauc_map_at_100_diff1": 0.20175041133547802, + "nauc_map_at_100_max": -0.2999406736946551, + "nauc_map_at_100_std": -0.21513200915624464, + "nauc_map_at_10_diff1": 0.4340980160657004, + "nauc_map_at_10_max": -0.30137691467001876, + "nauc_map_at_10_std": -0.4166728188377905, + "nauc_map_at_1_diff1": 0.3225400054210336, + "nauc_map_at_1_max": -0.20455804791357643, + "nauc_map_at_1_std": -0.3502703918125217, + "nauc_map_at_20_diff1": 0.4482686549814925, + "nauc_map_at_20_max": -0.33767777626178874, + "nauc_map_at_20_std": -0.39261152175985364, + "nauc_map_at_3_diff1": 0.3906475942798504, + "nauc_map_at_3_max": -0.1421273626640729, + "nauc_map_at_3_std": -0.3164227766935927, + "nauc_map_at_5_diff1": 0.40607500232859556, + "nauc_map_at_5_max": -0.21636348183658996, + "nauc_map_at_5_std": -0.385395422874435, + "nauc_mrr_at_1000_diff1": 0.33380716561285934, + "nauc_mrr_at_1000_max": -0.19720559281432273, + "nauc_mrr_at_1000_std": -0.3739725633585593, + "nauc_mrr_at_100_diff1": 0.3320423802626735, + "nauc_mrr_at_100_max": -0.19584551091041755, + "nauc_mrr_at_100_std": -0.37267220068248286, + "nauc_mrr_at_10_diff1": 0.33810313585026747, + "nauc_mrr_at_10_max": -0.20898927155954197, + "nauc_mrr_at_10_std": -0.39317334419879707, + "nauc_mrr_at_1_diff1": 0.3989804132009661, + "nauc_mrr_at_1_max": -0.12905822377247125, + "nauc_mrr_at_1_std": -0.401663536356319, + "nauc_mrr_at_20_diff1": 0.3270365458084192, + "nauc_mrr_at_20_max": -0.206603016304459, + "nauc_mrr_at_20_std": -0.3731460382346558, + "nauc_mrr_at_3_diff1": 0.3569994314378864, + "nauc_mrr_at_3_max": -0.17174174325462235, + "nauc_mrr_at_3_std": -0.3561501867618589, + "nauc_mrr_at_5_diff1": 0.3571575774665211, + "nauc_mrr_at_5_max": -0.18834228874209263, + "nauc_mrr_at_5_std": -0.40200287994954503, + "nauc_ndcg_at_1000_diff1": -0.0743652757576749, + "nauc_ndcg_at_1000_max": 0.09391843454021216, + "nauc_ndcg_at_1000_std": 0.08425685683454673, + "nauc_ndcg_at_100_diff1": 0.023516077444591114, + "nauc_ndcg_at_100_max": -0.11892313686780007, + "nauc_ndcg_at_100_std": -0.16710001090586352, + "nauc_ndcg_at_10_diff1": 0.3882520721308202, + "nauc_ndcg_at_10_max": -0.2924404653976282, + "nauc_ndcg_at_10_std": -0.4189010025783995, + "nauc_ndcg_at_1_diff1": 0.3989804132009661, + "nauc_ndcg_at_1_max": -0.12905822377247125, + "nauc_ndcg_at_1_std": -0.401663536356319, + "nauc_ndcg_at_20_diff1": 0.3805388505094879, + "nauc_ndcg_at_20_max": -0.30519760291519316, + "nauc_ndcg_at_20_std": -0.3772566200501027, + "nauc_ndcg_at_3_diff1": 0.412201762045363, + "nauc_ndcg_at_3_max": -0.17116948069763432, + "nauc_ndcg_at_3_std": -0.3608861222611007, + "nauc_ndcg_at_5_diff1": 0.35696244084680845, + "nauc_ndcg_at_5_max": -0.25186160853169576, + "nauc_ndcg_at_5_std": -0.400509495428499, + "nauc_precision_at_1000_diff1": -0.22093427992384315, + "nauc_precision_at_1000_max": 0.2196784647696767, + "nauc_precision_at_1000_std": 0.04361074681539203, + "nauc_precision_at_100_diff1": -0.016149306619419775, + "nauc_precision_at_100_max": 0.034799312614065006, + "nauc_precision_at_100_std": -0.13882957890950748, + "nauc_precision_at_10_diff1": 0.38489780804914814, + "nauc_precision_at_10_max": -0.30500607571435273, + "nauc_precision_at_10_std": -0.41532331666205985, + "nauc_precision_at_1_diff1": 0.3989804132009661, + "nauc_precision_at_1_max": -0.12905822377247125, + "nauc_precision_at_1_std": -0.401663536356319, + "nauc_precision_at_20_diff1": 0.3692528574157947, + "nauc_precision_at_20_max": -0.30111599366142605, + "nauc_precision_at_20_std": -0.35015003540161194, + "nauc_precision_at_3_diff1": 0.428985570425912, + "nauc_precision_at_3_max": -0.1563930424168448, + "nauc_precision_at_3_std": -0.3449801647848643, + "nauc_precision_at_5_diff1": 0.34983601813446513, + "nauc_precision_at_5_max": -0.2741390952059421, + "nauc_precision_at_5_std": -0.4086645123951, + "nauc_recall_at_1000_diff1": -0.08562125010420926, + "nauc_recall_at_1000_max": 0.07813883934551541, + "nauc_recall_at_1000_std": 0.23218650354926193, + "nauc_recall_at_100_diff1": -0.01041195221018056, + "nauc_recall_at_100_max": -0.23261397534923414, + "nauc_recall_at_100_std": -0.05979942942574269, + "nauc_recall_at_10_diff1": 0.3361904724346995, + "nauc_recall_at_10_max": -0.28110372866797806, + "nauc_recall_at_10_std": -0.36784022778417585, + "nauc_recall_at_1_diff1": 0.3225400054210336, + "nauc_recall_at_1_max": -0.20455804791357643, + "nauc_recall_at_1_std": -0.3502703918125217, + "nauc_recall_at_20_diff1": 0.31944679632949374, + "nauc_recall_at_20_max": -0.32588194303425905, + "nauc_recall_at_20_std": -0.25546971593163326, + "nauc_recall_at_3_diff1": 0.36197730614062007, + "nauc_recall_at_3_max": -0.07376636861137077, + "nauc_recall_at_3_std": -0.2538555661579059, + "nauc_recall_at_5_diff1": 0.3711994561836934, + "nauc_recall_at_5_max": -0.1684006987187415, + "nauc_recall_at_5_std": -0.373838797717674, + "ndcg_at_1": 0.04286, + "ndcg_at_10": 0.08888, + "ndcg_at_100": 0.20767, + "ndcg_at_1000": 0.46304, + "ndcg_at_20": 0.0984, + "ndcg_at_3": 0.06137, + "ndcg_at_5": 0.07145, + "precision_at_1": 0.04286, + "precision_at_10": 0.10143, + "precision_at_100": 0.202, + "precision_at_1000": 0.09564, + "precision_at_20": 0.10857, + "precision_at_3": 0.06667, + "precision_at_5": 0.08, + "recall_at_1": 0.00032, + "recall_at_10": 0.0087, + "recall_at_100": 0.18965, + "recall_at_1000": 0.67973, + "recall_at_20": 0.01989, + "recall_at_3": 0.00187, + "recall_at_5": 0.00327 + } + ] + }, + "task_name": "RParisHardI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisI2IRetrieval.json deleted file mode 100644 index 0f309a07f3..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "bafc3a08fdffd72558021ce3a41250833d7e0e88", - "evaluation_time": 43.56014585494995, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.78348, - "cv_recall_at_10": 0.96558, - "cv_recall_at_100": 0.99812, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.98217, - "cv_recall_at_3": 0.89972, - "cv_recall_at_5": 0.93367, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.78348, - "map_at_1": 0.00183, - "map_at_10": 0.01259, - "map_at_100": 0.08554, - "map_at_1000": 0.22541, - "map_at_20": 0.02257, - "map_at_3": 0.0046, - "map_at_5": 0.00704, - "mrr_at_1": 0.9870150187734669, - "mrr_at_10": 0.9935075093867334, - "mrr_at_100": 0.9935075093867334, - "mrr_at_1000": 0.9935075093867334, - "mrr_at_20": 0.9935075093867334, - "mrr_at_3": 0.9935075093867334, - "mrr_at_5": 0.9935075093867334, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.12564420871970045, - "nauc_cv_recall_at_100_max": 0.3655746256621164, - "nauc_cv_recall_at_100_std": -0.2306365991736563, - "nauc_cv_recall_at_10_diff1": -0.5058662936618029, - "nauc_cv_recall_at_10_max": 0.09207458773253613, - "nauc_cv_recall_at_10_std": 0.4663743642051124, - "nauc_cv_recall_at_1_diff1": -0.4013157463788695, - "nauc_cv_recall_at_1_max": 0.013502025515305102, - "nauc_cv_recall_at_1_std": 0.32242775071432206, - "nauc_cv_recall_at_20_diff1": -0.5198855681306671, - "nauc_cv_recall_at_20_max": 0.10151852705481003, - "nauc_cv_recall_at_20_std": 0.41412564884036973, - "nauc_cv_recall_at_3_diff1": -0.5198700551458323, - "nauc_cv_recall_at_3_max": 0.025885833103837443, - "nauc_cv_recall_at_3_std": 0.4187563971498901, - "nauc_cv_recall_at_5_diff1": -0.5209006482350635, - "nauc_cv_recall_at_5_max": 0.051797269624278094, - "nauc_cv_recall_at_5_std": 0.4360707529546872, - "nauc_map_at_1000_diff1": -0.32518052685711313, - "nauc_map_at_1000_max": 0.0038005812075390727, - "nauc_map_at_1000_std": 0.43958508353436493, - "nauc_map_at_100_diff1": -0.324729620306574, - "nauc_map_at_100_max": 0.003712984860206863, - "nauc_map_at_100_std": 0.3530965583894132, - "nauc_map_at_10_diff1": -0.2865369016119553, - "nauc_map_at_10_max": 0.012519773293503903, - "nauc_map_at_10_std": 0.23475872496725028, - "nauc_map_at_1_diff1": -0.0855732613663, - "nauc_map_at_1_max": 0.023221460373481606, - "nauc_map_at_1_std": 0.03199412568851411, - "nauc_map_at_20_diff1": -0.3089685409009752, - "nauc_map_at_20_max": 0.008164069309369689, - "nauc_map_at_20_std": 0.28138880870717525, - "nauc_map_at_3_diff1": -0.21596259885289204, - "nauc_map_at_3_max": 0.026694429615608712, - "nauc_map_at_3_std": 0.13815328440691188, - "nauc_map_at_5_diff1": -0.2539278390954876, - "nauc_map_at_5_max": 0.01952941733527699, - "nauc_map_at_5_std": 0.18318190468395143, - "nauc_mrr_at_1000_diff1": 1.0, - "nauc_mrr_at_1000_max": -0.13028347793646336, - "nauc_mrr_at_1000_std": -0.28303491792776114, - "nauc_mrr_at_100_diff1": 1.0, - "nauc_mrr_at_100_max": -0.13028347793646336, - "nauc_mrr_at_100_std": -0.28303491792776114, - "nauc_mrr_at_10_diff1": 1.0, - "nauc_mrr_at_10_max": -0.13028347793646336, - "nauc_mrr_at_10_std": -0.28303491792776114, - "nauc_mrr_at_1_diff1": 1.0, - "nauc_mrr_at_1_max": -0.1302834779364721, - "nauc_mrr_at_1_std": -0.2830349179277583, - "nauc_mrr_at_20_diff1": 1.0, - "nauc_mrr_at_20_max": -0.13028347793646336, - "nauc_mrr_at_20_std": -0.28303491792776114, - "nauc_mrr_at_3_diff1": 1.0, - "nauc_mrr_at_3_max": -0.13028347793646336, - "nauc_mrr_at_3_std": -0.28303491792776114, - "nauc_mrr_at_5_diff1": 1.0, - "nauc_mrr_at_5_max": -0.13028347793646336, - "nauc_mrr_at_5_std": -0.28303491792776114, - "nauc_ndcg_at_1000_diff1": -0.3461037602745701, - "nauc_ndcg_at_1000_max": 0.01170706578078914, - "nauc_ndcg_at_1000_std": 0.40425946623874776, - "nauc_ndcg_at_100_diff1": -0.39901516979641344, - "nauc_ndcg_at_100_max": -0.00685883548003498, - "nauc_ndcg_at_100_std": 0.465350992380673, - "nauc_ndcg_at_10_diff1": -0.3955198632793018, - "nauc_ndcg_at_10_max": -0.0037363142445774643, - "nauc_ndcg_at_10_std": 0.38110734984115535, - "nauc_ndcg_at_1_diff1": 1.0, - "nauc_ndcg_at_1_max": -0.14486457971944697, - "nauc_ndcg_at_1_std": -0.2830349179277583, - "nauc_ndcg_at_20_diff1": -0.41278062763010503, - "nauc_ndcg_at_20_max": -0.004138702255930161, - "nauc_ndcg_at_20_std": 0.4191652248206659, - "nauc_ndcg_at_3_diff1": -0.34367743707797543, - "nauc_ndcg_at_3_max": 0.004103711367444123, - "nauc_ndcg_at_3_std": 0.3101278349486688, - "nauc_ndcg_at_5_diff1": -0.37388771810166976, - "nauc_ndcg_at_5_max": -0.004694760292318884, - "nauc_ndcg_at_5_std": 0.3429402757237829, - "nauc_precision_at_1000_diff1": -0.1804983224177905, - "nauc_precision_at_1000_max": -0.01148320196422333, - "nauc_precision_at_1000_std": 0.2504460959613102, - "nauc_precision_at_100_diff1": -0.3911848294276243, - "nauc_precision_at_100_max": -0.007414470176849031, - "nauc_precision_at_100_std": 0.4673613835957893, - "nauc_precision_at_10_diff1": -0.4052976521224889, - "nauc_precision_at_10_max": -0.00433906156493395, - "nauc_precision_at_10_std": 0.39212256495307285, - "nauc_precision_at_1_diff1": 1.0, - "nauc_precision_at_1_max": -0.14486457971944697, - "nauc_precision_at_1_std": -0.2830349179277583, - "nauc_precision_at_20_diff1": -0.4173913678934324, - "nauc_precision_at_20_max": -0.004543026048413097, - "nauc_precision_at_20_std": 0.4299497661264597, - "nauc_precision_at_3_diff1": -0.37619357586668295, - "nauc_precision_at_3_max": 0.006637408448532989, - "nauc_precision_at_3_std": 0.3255087096886432, - "nauc_precision_at_5_diff1": -0.393543490911112, - "nauc_precision_at_5_max": -0.0054501833188476565, - "nauc_precision_at_5_std": 0.3553468241787662, - "nauc_recall_at_1000_diff1": -0.28865460441970253, - "nauc_recall_at_1000_max": 0.02031269144932309, - "nauc_recall_at_1000_std": 0.3581111189760722, - "nauc_recall_at_100_diff1": -0.31737603693043515, - "nauc_recall_at_100_max": 0.010815413934469631, - "nauc_recall_at_100_std": 0.32072251393381573, - "nauc_recall_at_10_diff1": -0.2835809857944858, - "nauc_recall_at_10_max": 0.01631628873177338, - "nauc_recall_at_10_std": 0.2203863362341707, - "nauc_recall_at_1_diff1": -0.0855732613663, - "nauc_recall_at_1_max": 0.023221460373481606, - "nauc_recall_at_1_std": 0.03199412568851411, - "nauc_recall_at_20_diff1": -0.30410786586422606, - "nauc_recall_at_20_max": 0.012058890403941508, - "nauc_recall_at_20_std": 0.2609223499236706, - "nauc_recall_at_3_diff1": -0.22686364377869106, - "nauc_recall_at_3_max": 0.02838523233952593, - "nauc_recall_at_3_std": 0.13768932686394217, - "nauc_recall_at_5_diff1": -0.2568046605350692, - "nauc_recall_at_5_max": 0.02092009985156803, - "nauc_recall_at_5_std": 0.17610456507060399, - "ndcg_at_1": 0.98702, - "ndcg_at_10": 0.76647, - "ndcg_at_100": 0.59394, - "ndcg_at_1000": 0.44945, - "ndcg_at_20": 0.71423, - "ndcg_at_3": 0.8678, - "ndcg_at_5": 0.82287, - "precision_at_1": 0.98702, - "precision_at_10": 0.72226, - "precision_at_100": 0.5574, - "precision_at_1000": 0.25148, - "precision_at_20": 0.67042, - "precision_at_3": 0.83521, - "precision_at_5": 0.7832, - "recall_at_1": 0.00183, - "recall_at_10": 0.01349, - "recall_at_100": 0.10451, - "recall_at_1000": 0.40829, - "recall_at_20": 0.02511, - "recall_at_3": 0.00467, - "recall_at_5": 0.0073 - } - ] - }, - "task_name": "RParisI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json new file mode 100644 index 0000000000..700a4a2774 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "900267b49003a086979e8d52f6942624236bfc34", + "evaluation_time": 23.04814124107361, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.98571, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.98571, + "map_at_1": 0.0055, + "map_at_10": 0.0505, + "map_at_100": 0.32128, + "map_at_1000": 0.53905, + "map_at_20": 0.09538, + "map_at_3": 0.01586, + "map_at_5": 0.02626, + "mrr_at_1": 0.9857142857142858, + "mrr_at_10": 0.9928571428571429, + "mrr_at_100": 0.9928571428571429, + "mrr_at_1000": 0.9928571428571429, + "mrr_at_20": 0.9928571428571429, + "mrr_at_3": 0.9928571428571429, + "mrr_at_5": 0.9928571428571429, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": 0.8692810457516342, + "nauc_cv_recall_at_1_max": -1.151727357609713, + "nauc_cv_recall_at_1_std": -0.5634920634920669, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_map_at_1000_diff1": -0.5202144222378728, + "nauc_map_at_1000_max": 0.11126568280766698, + "nauc_map_at_1000_std": 0.32813155264948524, + "nauc_map_at_100_diff1": -0.17082735069359992, + "nauc_map_at_100_max": -0.07951969633170079, + "nauc_map_at_100_std": 0.13798406208276, + "nauc_map_at_10_diff1": 0.08925396593997569, + "nauc_map_at_10_max": -0.078676032579074, + "nauc_map_at_10_std": 0.13021224342356594, + "nauc_map_at_1_diff1": 0.20034146659034022, + "nauc_map_at_1_max": -0.1488526539306114, + "nauc_map_at_1_std": 0.036837238597286596, + "nauc_map_at_20_diff1": 0.044029095752394776, + "nauc_map_at_20_max": -0.044115438949865464, + "nauc_map_at_20_std": 0.16589359171211718, + "nauc_map_at_3_diff1": 0.12936241106392424, + "nauc_map_at_3_max": -0.11171026414768116, + "nauc_map_at_3_std": 0.09557523635145118, + "nauc_map_at_5_diff1": 0.12090379123610429, + "nauc_map_at_5_max": -0.11656881579212829, + "nauc_map_at_5_std": 0.09921919672363319, + "nauc_mrr_at_1000_diff1": 0.8692810457516355, + "nauc_mrr_at_1000_max": -1.1517273576096931, + "nauc_mrr_at_1000_std": -0.5634920634920526, + "nauc_mrr_at_100_diff1": 0.8692810457516355, + "nauc_mrr_at_100_max": -1.1517273576096931, + "nauc_mrr_at_100_std": -0.5634920634920526, + "nauc_mrr_at_10_diff1": 0.8692810457516355, + "nauc_mrr_at_10_max": -1.1517273576096931, + "nauc_mrr_at_10_std": -0.5634920634920526, + "nauc_mrr_at_1_diff1": 0.8692810457516342, + "nauc_mrr_at_1_max": -1.151727357609713, + "nauc_mrr_at_1_std": -0.5634920634920669, + "nauc_mrr_at_20_diff1": 0.8692810457516355, + "nauc_mrr_at_20_max": -1.1517273576096931, + "nauc_mrr_at_20_std": -0.5634920634920526, + "nauc_mrr_at_3_diff1": 0.8692810457516355, + "nauc_mrr_at_3_max": -1.1517273576096931, + "nauc_mrr_at_3_std": -0.5634920634920526, + "nauc_mrr_at_5_diff1": 0.8692810457516355, + "nauc_mrr_at_5_max": -1.1517273576096931, + "nauc_mrr_at_5_std": -0.5634920634920526, + "nauc_ndcg_at_1000_diff1": -0.44820569134469457, + "nauc_ndcg_at_1000_max": 0.14278328260517373, + "nauc_ndcg_at_1000_std": 0.35909204639356485, + "nauc_ndcg_at_100_diff1": -0.6018888319473394, + "nauc_ndcg_at_100_max": 0.21192336225080224, + "nauc_ndcg_at_100_std": 0.25475321800547673, + "nauc_ndcg_at_10_diff1": -0.5815570895956712, + "nauc_ndcg_at_10_max": 0.2384364105122267, + "nauc_ndcg_at_10_std": 0.47588628797056887, + "nauc_ndcg_at_1_diff1": 0.8692810457516342, + "nauc_ndcg_at_1_max": -1.151727357609713, + "nauc_ndcg_at_1_std": -0.5634920634920669, + "nauc_ndcg_at_20_diff1": -0.5953841029416077, + "nauc_ndcg_at_20_max": 0.32157620990503083, + "nauc_ndcg_at_20_std": 0.47071964995241883, + "nauc_ndcg_at_3_diff1": -0.47333802126140007, + "nauc_ndcg_at_3_max": 0.13407004770022382, + "nauc_ndcg_at_3_std": 0.46679765020950414, + "nauc_ndcg_at_5_diff1": -0.5587389095148461, + "nauc_ndcg_at_5_max": 0.07339891244234566, + "nauc_ndcg_at_5_std": 0.47215237618630984, + "nauc_precision_at_1000_diff1": -0.3950971876695561, + "nauc_precision_at_1000_max": 0.24406161426957987, + "nauc_precision_at_1000_std": 0.12174947607128742, + "nauc_precision_at_100_diff1": -0.5472068296947269, + "nauc_precision_at_100_max": 0.23378022737023205, + "nauc_precision_at_100_std": 0.1829472692928344, + "nauc_precision_at_10_diff1": -0.5950517610712319, + "nauc_precision_at_10_max": 0.28672134365425656, + "nauc_precision_at_10_std": 0.48780862485384807, + "nauc_precision_at_1_diff1": 0.8692810457516342, + "nauc_precision_at_1_max": -1.151727357609713, + "nauc_precision_at_1_std": -0.5634920634920669, + "nauc_precision_at_20_diff1": -0.5947364591012498, + "nauc_precision_at_20_max": 0.35422908868857506, + "nauc_precision_at_20_std": 0.4705389149562903, + "nauc_precision_at_3_diff1": -0.510094916344916, + "nauc_precision_at_3_max": 0.22297297297297156, + "nauc_precision_at_3_std": 0.5162886100386108, + "nauc_precision_at_5_diff1": -0.6028283796740131, + "nauc_precision_at_5_max": 0.11490891658676922, + "nauc_precision_at_5_std": 0.504458293384468, + "nauc_recall_at_1000_diff1": -0.2460203893111549, + "nauc_recall_at_1000_max": 0.07681666757366377, + "nauc_recall_at_1000_std": 0.29402261311504047, + "nauc_recall_at_100_diff1": -0.10390008545301123, + "nauc_recall_at_100_max": -0.09447732649357599, + "nauc_recall_at_100_std": 0.0928830407002215, + "nauc_recall_at_10_diff1": 0.10590480631127752, + "nauc_recall_at_10_max": -0.08162621409653509, + "nauc_recall_at_10_std": 0.11847839880663995, + "nauc_recall_at_1_diff1": 0.20034146659034022, + "nauc_recall_at_1_max": -0.1488526539306114, + "nauc_recall_at_1_std": 0.036837238597286596, + "nauc_recall_at_20_diff1": 0.06586007946887822, + "nauc_recall_at_20_max": -0.05281307817884481, + "nauc_recall_at_20_std": 0.14970188120798883, + "nauc_recall_at_3_diff1": 0.13126830970729877, + "nauc_recall_at_3_max": -0.10596900293507791, + "nauc_recall_at_3_std": 0.09476793236811286, + "nauc_recall_at_5_diff1": 0.12983221018552543, + "nauc_recall_at_5_max": -0.11532911944953175, + "nauc_recall_at_5_std": 0.09212571294283788, + "ndcg_at_1": 0.98571, + "ndcg_at_10": 0.92841, + "ndcg_at_100": 0.77588, + "ndcg_at_1000": 0.7685, + "ndcg_at_20": 0.89894, + "ndcg_at_3": 0.95291, + "ndcg_at_5": 0.94991, + "precision_at_1": 0.98571, + "precision_at_10": 0.91571, + "precision_at_100": 0.721, + "precision_at_1000": 0.18663, + "precision_at_20": 0.88, + "precision_at_3": 0.94286, + "precision_at_5": 0.94286, + "recall_at_1": 0.0055, + "recall_at_10": 0.05165, + "recall_at_100": 0.36172, + "recall_at_1000": 0.79065, + "recall_at_20": 0.0987, + "recall_at_3": 0.01595, + "recall_at_5": 0.0266 + } + ] + }, + "task_name": "RParisMediumI2IRetrieval" +} \ No newline at end of file From 8065568f3e41df34fda1121f5e273d65b85500fb Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Tue, 22 Oct 2024 22:54:38 +0800 Subject: [PATCH 081/154] [mieb] run tasks small fix (#1310) * fix ImageTextPair dataloading for large datasets; more compositionality evaluation datasets * fix e5v&vista * task type fix for running tasks * fix wrong meta * run mieb script * script * lint * align * fix * linting --- mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py | 2 +- mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py | 2 +- .../Image/ImageClassification/eng/Caltech101Classification.py | 1 + .../ImageClassification/eng/FGVCAircraftClassification.py | 1 + .../Image/ImageClassification/eng/MNISTClassification.py | 2 +- .../Image/ImageMultilabelClassification/eng/PascalVOC2007.py | 1 + mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py | 4 ++-- mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py | 1 + mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py | 1 + mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py | 2 +- scripts/run_mieb.py | 2 +- 11 files changed, 12 insertions(+), 7 deletions(-) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py index 7779883a84..2fb96a2365 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py @@ -105,7 +105,7 @@ class ROxfordHardI2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/r-oxford-hard", "revision": "f20b30211b7ba3fc64a02bd83998fe75f3023719", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py index ed49982fe5..e625258053 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py @@ -105,7 +105,7 @@ class RParisHardI2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/r-paris-hard", "revision": "fd121b6592fe946616fa85116703b94a4c61fd63", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py index 0e00980428..63b0b622ad 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py @@ -13,6 +13,7 @@ class Caltech101Classification(AbsTaskImageClassification): "path": "HuggingFaceM4/Caltech-101", "name": "with_background_category", "revision": "851374102055782c84f89b1b4e9d128a6568847b", + "trust_remote_code": True, }, type="ImageClassification", category="i2t", diff --git a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py index 3db3ef2d8c..aac18c7e57 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py @@ -12,6 +12,7 @@ class FGVCAircraftClassification(AbsTaskImageClassification): dataset={ "path": "HuggingFaceM4/FGVC-Aircraft", "revision": "91860adfc9a09aabca5cddb5247442109b38e213", + "trust_remote_code": True, }, type="ImageClassification", category="i2t", diff --git a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py index 5e9b9a86af..0956e0d1c3 100644 --- a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py @@ -11,7 +11,7 @@ class MNISTClassification(AbsTaskImageClassification): reference="https://en.wikipedia.org/wiki/MNIST_database", dataset={ "path": "ylecun/mnist", - "revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "revision": "77f3279092a1c1579b2250db8eafed0ad422088c", }, type="ImageClassification", category="i2t", diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index 1a02997aec..057d631784 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -15,6 +15,7 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): "path": "HuggingFaceM4/pascal_voc", "name": "voc2007_main", "revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "trust_remote_code": True, }, type="ImageMultilabelClassification", category="i2t", diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py b/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py index 3137ad510d..fe89f06d42 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py @@ -8,7 +8,7 @@ class CLEVR(AbsTaskZeroshotClassification): metadata = TaskMetadata( - name="CLEVR", + name="CLEVRZeroShot", description="CLEVR closest object distance identification task.", reference="https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html", dataset={ @@ -61,7 +61,7 @@ def get_candidate_labels(self) -> list[str]: class CLEVRCount(AbsTaskZeroshotClassification): metadata = TaskMetadata( - name="CLEVRCount", + name="CLEVRCountZeroShot", description="CLEVR count objects task.", reference="https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html", dataset={ diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py b/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py index ab7ca141cb..00bfdac874 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py @@ -15,6 +15,7 @@ class Caltech101Classification(AbsTaskZeroshotClassification): "path": "HuggingFaceM4/Caltech-101", "name": "with_background_category", "revision": "851374102055782c84f89b1b4e9d128a6568847b", + "trust_remote_code": True, }, type="ZeroShotClassification", category="i2t", diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py b/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py index c15e0b6d4b..b973c85607 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py @@ -14,6 +14,7 @@ class FGVCAircraftClassification(AbsTaskZeroshotClassification): dataset={ "path": "HuggingFaceM4/FGVC-Aircraft", "revision": "91860adfc9a09aabca5cddb5247442109b38e213", + "trust_remote_code": True, }, type="ZeroShotClassification", category="i2t", diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py b/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py index 253fa938ac..c274e72d61 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py @@ -13,7 +13,7 @@ class MNISTClassification(AbsTaskZeroshotClassification): reference="https://en.wikipedia.org/wiki/MNIST_database", dataset={ "path": "ylecun/mnist", - "revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "revision": "77f3279092a1c1579b2250db8eafed0ad422088c", }, type="ZeroShotClassification", category="i2t", diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index 8b00d64e9d..cf05096996 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -9,7 +9,6 @@ "royokong/e5-v", "BAAI/bge-visualized-base", "BAAI/bge-visualized-m3", - "google/siglip-so400m-patch14-384", "kakaobrain/align-base", "jinaai/jina-clip-v1", "nomic-ai/nomic-embed-vision-v1.5", @@ -26,6 +25,7 @@ "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "nyu-visionx/moco-v3-vit-b", "nyu-visionx/moco-v3-vit-l", + # "google/siglip-so400m-patch14-384",# haven't pushed ]: model = mteb.get_model(model_name) tasks = mteb.get_tasks( From 2011aa1210e291e11a824b8be10ca9e55c97eed8 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Fri, 25 Oct 2024 18:41:43 +0300 Subject: [PATCH 082/154] [mieb] Add VLM2vec (#1323) * wip vlm2vec model * making i2t classification work wit Calteh101 * test vlm2vec on other task types * move peft into class --- .../Image/ClassificationEvaluator.py | 4 + mteb/models/__init__.py | 3 +- mteb/models/vlm2vec_models.py | 302 ++++++++++++++++++ .../Caltech101.json | 28 ++ .../STS12.json | 26 ++ .../model_meta.json | 1 + 6 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 mteb/models/vlm2vec_models.py create mode 100644 results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/Caltech101.json create mode 100644 results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/STS12.json create mode 100644 results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/model_meta.json diff --git a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py index eaa6416dc7..e129ab0a4c 100644 --- a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py @@ -381,6 +381,10 @@ def __call__(self, model, test_cache=None): else: X_test = test_cache logger.info("Fitting logistic regression classifier...") + if X_train.dtype == torch.bfloat16: + X_train = X_train.to(torch.float32) + if X_test.dtype == torch.bfloat16: + X_test = X_test.to(torch.float32) clf.fit(X_train, self.y_train) logger.info("Evaluating...") y_pred = clf.predict(X_test) diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index d8512a5a23..25c7eb8f44 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -34,6 +34,7 @@ salesforce_models, sentence_transformers_models, vista_models, + vlm2vec_models, voyage_models, ) @@ -160,7 +161,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe sentence_transformers_models, vista_models, voyage_models, - google_models, + vlm2vec_models, ] models = {} diff --git a/mteb/models/vlm2vec_models.py b/mteb/models/vlm2vec_models.py new file mode 100644 index 0000000000..321cba24a4 --- /dev/null +++ b/mteb/models/vlm2vec_models.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +import logging +from functools import partial +from typing import Any, Literal + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import AutoConfig, AutoModelForCausalLM, AutoProcessor + +from mteb.model_meta import ModelMeta + +logging.basicConfig(level=logging.WARNING) +logger = logging.getLogger(__name__) + +EncodeTypes = Literal["query", "passage"] + + +class VLM2VecWrapper: + """Adapted from https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/src/model.py""" + + def __init__( + self, + model_name: str = "TIGER-Lab/VLM2Vec-LoRA", + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs, + ): + try: + import flash_attn # noqa + from peft import LoraConfig, PeftModel # noqa + except ImportError: + logger.warning( + "VLM2Vec models were trained with flash attention enabled. For optimal performance, please install the `flash_attn` package with `pip install flash-attn --no-build-isolation`." + ) + + self.pooling = "last" + self.normalize = True + self.temperature = 1.0 + self.hidden_size = 4096 + self.device = device + + # Loading the base model + base_model_name = "microsoft/Phi-3.5-vision-instruct" + config = AutoConfig.from_pretrained(base_model_name, trust_remote_code=True) + config.use_cache = False + config.padding_side = "right" + + checkpoint_path = model_name if model_name else base_model_name + base_model = AutoModelForCausalLM.from_pretrained( + checkpoint_path, + config=config, + attn_implementation="flash_attention_2", + torch_dtype=torch.bfloat16, + trust_remote_code=True, + ) + base_model.padding_side = "right" + + # Building the model on top of the base + if "LoRA" in model_name: + lora_config = LoraConfig.from_pretrained(checkpoint_path) + lora_model = PeftModel.from_pretrained( + base_model, checkpoint_path, config=lora_config + ) + lora_model = lora_model.merge_and_unload() + model = lora_model + else: + model = base_model + + model.eval() + model.to(device) + self.mdl = model + + self.processor = AutoProcessor.from_pretrained( + base_model_name, + trust_remote_code=True, + num_crops=4, + ) + + def encode( + self, + sentences: list[str], + *, + prompt_name: str = None, + **kwargs: Any, # noqa + ): + return self.get_text_embeddings(texts=sentences) + + def encode_input(self, input): + hidden_states = self.mdl(**input, return_dict=True, output_hidden_states=True) + hidden_states = hidden_states.hidden_states[-1] + pooled_output = self._pooling(hidden_states, input["attention_mask"]) + return pooled_output + + def _pooling(self, last_hidden_state, attention_mask): + if self.pooling == "last": + sequence_lengths = attention_mask.sum(dim=1) - 1 + batch_size = last_hidden_state.shape[0] + reps = last_hidden_state[ + torch.arange(batch_size, device=last_hidden_state.device), + sequence_lengths, + ] + else: + raise NotImplementedError + if self.normalize: + reps = torch.nn.functional.normalize(reps, p=2, dim=-1) + return reps + + # reference: https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/src/collator.py + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + text = "<|image_1|> Represent the given image." + all_image_embeddings = [] + if isinstance(images, DataLoader): + import torchvision.transforms.functional as F + + with torch.no_grad(): + for batch in tqdm(images): + input_ids, pixel_values, image_sizes = [], [], [] + for b in batch: + inputs = self.processor( + text, + [F.to_pil_image(b.to("cpu"))], + return_tensors="pt", + max_length=256, + truncation=True, + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + input_ids.append(inputs["input_ids"].squeeze(0).unsqueeze(1)) + pixel_values.append(inputs["pixel_values"]) + image_sizes.append(inputs["image_sizes"]) + + input_ids = torch._C._nn.pad_sequence( + input_ids, + batch_first=True, + padding_value=self.processor.tokenizer.pad_token_id, + ).squeeze(2) + attention_mask = input_ids.ne(self.processor.tokenizer.pad_token_id) + + pixel_values = torch.cat(pixel_values, dim=0) + image_sizes = torch.cat(image_sizes, dim=0) + inputs = { + "input_ids": input_ids, + "attention_mask": attention_mask, + "pixel_values": pixel_values, + "image_sizes": image_sizes, + } + + image_outputs = self.encode_input(inputs) + all_image_embeddings.append(image_outputs.cpu()) + + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + input_ids, pixel_values, image_sizes = [], [], [] + for b in batch_images: + inputs = self.processor( + text, + [b], + return_tensors="pt", + max_length=256, + truncation=True, + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + input_ids.append(inputs["input_ids"].squeeze(0).unsqueeze(1)) + pixel_values.append(inputs["pixel_values"]) + image_sizes.append(inputs["image_sizes"]) + + input_ids = torch._C._nn.pad_sequence( + input_ids, + batch_first=True, + padding_value=self.processor.tokenizer.pad_token_id, + ).squeeze(2) + attention_mask = input_ids.ne(self.processor.tokenizer.pad_token_id) + + pixel_values = torch.cat(pixel_values, dim=0) + image_sizes = torch.cat(image_sizes, dim=0) + inputs = { + "input_ids": input_ids, + "attention_mask": attention_mask, + "pixel_values": pixel_values, + "image_sizes": image_sizes, + } + + image_outputs = self.encode_input(inputs) + all_image_embeddings.append(image_outputs.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(texts), batch_size)): + input_ids = [] + batch_texts = texts[i : i + batch_size] + for text in batch_texts: + inputs = self.processor( + text, + None, + return_tensors="pt", + max_length=256, + truncation=True, + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + input_ids.append(inputs["input_ids"].squeeze(0).unsqueeze(1)) + + input_ids = torch._C._nn.pad_sequence( + input_ids, + batch_first=True, + padding_value=self.processor.tokenizer.pad_token_id, + ).squeeze(2) + attention_mask = input_ids.ne(self.processor.tokenizer.pad_token_id) + inputs = { + "input_ids": input_ids, + "attention_mask": attention_mask, + } + + text_outputs = self.encode_input(inputs) + all_text_embeddings.append(text_outputs.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + texts = iter(texts) + all_fused_embeddings = [] + if isinstance(images, DataLoader): + import torchvision.transforms.functional as F + + for batch in images: + for b in batch: + text = next(texts) + inputs = self.processor( + f"<|image_1|> Represent the given image with the following question: {text}", + [F.to_pil_image(b.to("cpu"))], + ) + inputs = { + key: value.to(self.device) for key, value in inputs.items() + } + outputs = self.encode_input(inputs) + all_fused_embeddings.append(outputs.cpu()) + + fused_embeddings = torch.cat(all_fused_embeddings, dim=0) + + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + +vlm2vec_lora = ModelMeta( + loader=partial( + VLM2VecWrapper, + model_name="TIGER-Lab/VLM2Vec-LoRA", + ), + name="TIGER-Lab/VLM2Vec-LoRA", + languages=["eng_Latn"], + open_source=True, + revision="7403b6327958071c1e33c822c7453adadccc7298", + release_date="2024-10-08", +) + +vlm2vec_full = ModelMeta( + loader=partial( + VLM2VecWrapper, + model_name="TIGER-Lab/VLM2Vec-Full", + ), + name="TIGER-Lab/VLM2Vec-Full", + languages=["eng_Latn"], + open_source=True, + revision="e9afa98002097ac2471827ba23ea1f2ddd229480", + release_date="2024-10-08", +) diff --git a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/Caltech101.json b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/Caltech101.json new file mode 100644 index 0000000000..cac75612d1 --- /dev/null +++ b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/Caltech101.json @@ -0,0 +1,28 @@ +{ + "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", + "evaluation_time": 1317.9743084907532, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.9301446416831032, + "f1": 0.8863632422649081, + "f1_weighted": 0.9270094006117223, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9301446416831032, + "scores_per_experiment": [ + { + "accuracy": 0.9301446416831032, + "f1": 0.8863632422649081, + "f1_weighted": 0.9270094006117223 + } + ] + } + ] + }, + "task_name": "Caltech101" +} \ No newline at end of file diff --git a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/STS12.json b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/STS12.json new file mode 100644 index 0000000000..2f7a702114 --- /dev/null +++ b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/STS12.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", + "evaluation_time": 33.679136514663696, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cosine_pearson": 0.6128856131150828, + "cosine_spearman": 0.5375376750091784, + "euclidean_pearson": 0.5866571133163221, + "euclidean_spearman": 0.5376001641683719, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5375376750091784, + "manhattan_pearson": 0.5912422177023093, + "manhattan_spearman": 0.5413588869937086, + "pearson": 0.6128856131150828, + "spearman": 0.5375376750091784 + } + ] + }, + "task_name": "STS12" +} \ No newline at end of file diff --git a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/model_meta.json b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/model_meta.json new file mode 100644 index 0000000000..07f5788002 --- /dev/null +++ b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/model_meta.json @@ -0,0 +1 @@ +{"name": "TIGER-Lab/VLM2Vec-LoRA", "revision": "7403b6327958071c1e33c822c7453adadccc7298", "release_date": "2024-10-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "VLM2VecWrapper"} \ No newline at end of file From 93260cb27b8a4ba9f016aedf3dddf55b696685e0 Mon Sep 17 00:00:00 2001 From: Kenneth Enevoldsen Date: Sun, 27 Oct 2024 23:12:13 +0100 Subject: [PATCH 083/154] feat: Merge main into MIEB (#1329) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: OpenAI BadRequestError by limiting input dimensions to 2048 elem… (#1203) * fix: OpenAI BadRequestError by limiting input dimensions to 2048 elements (#1201) Fix OpenAI BadRequestError by limiting input dimensions to 2048 elements - Ensure the 'sentences' list passed to OpenAI API does not exceed 2048 elements - Reference: OpenAI's Embedding API documentation on input limits Co-authored-by: Ali Shiraee * fix ruff formatting * Added minor test fixes to ensure reproducility across systems * Ensure that tmp.json is not created within repo when running tests * format * fixes path issues * Rerun CI --------- Co-authored-by: HSILA Co-authored-by: Ali Shiraee * fix: Ensure STS pearson and spearman does not use the p-value only the correlation (#1207) Fixes #1206 * 1.14.16 Automatically generated by python-semantic-release * fix: Normalize licenses including casing, uses of "-" etc. * fix: Normalize licenses including casing, uses of "-" etc. (#1210) * fix: Normalize licenses including casing, uses of "-" etc. * fix tests * 1.14.17 Automatically generated by python-semantic-release * fix: Normalize benchmarks no only include task objects and added getter for benchmarks (#1208) * Normalize benchmarks to only include tasks - Force benchmarks to only include tasks. This fixes a few bugs where benchmarks can reference a task which is not implemented - implements `mteb.get_benchmark`, which makes it easier to fetch benchmarks - Added tests + updated docs A few outstanding issues: I would like `mteb.MTEB(benchmark)` to always reproduce the benchmark. Currently this is not possible as MTEB(eng) required the split to be specified. A solution it to allow "eval_splits) to be specified when initializing a task and then pass it on to the `load_data()`. This way we can write the following: `mteb.get_tasks(tasks=[...], eval_splits=["test"], ...)` I would also love the aggregation to be a part of the benchmark (such that it is clear how it should be aggregated). This is especially relevant for MTEB(eng) as it average the CQAD datasets before creating the global average. This way we can also create a result object for the benchmark itself. A complimenting solution for this is to allow nested benchmarks. * fix error in tests * format * Added corrections based on review * added example and formatted * 1.14.18 Automatically generated by python-semantic-release * docs: Fix broken links in docs (#1212) * Added fixes for broken links in adding_a_dataset and adding_a_model docs. * Updated link name * Mismatch of the category of AmazonPolarityClassification (#1220) Fixes #1219 * Update tasks table * fix: Ensure that results are returned even when hitting cache (#1215) Fixes #1122 * 1.14.19 Automatically generated by python-semantic-release * fix: Allow benchmark to specify eval_splits (#1217) * fix: Allow benchmark to specify eval_splits This PR allow for benchmarks to specify specific eval. splits. This allow us to fully specify a benchmark within the benchmark object. To do this it add the following: - added eval_splits to the Abstask object, which default to metadata.eval_splits - use the task.eval_splits unless overwritten in mteb.MTEB.run - added eval_splits arg to mteb.get_tasks, which filter the tasks based on splits - updated documentation - renamed the "Advanced Usage" to "Usage Documentation" to make it more accicible - added tests where relevant * Added correction based on feedback * 1.14.20 Automatically generated by python-semantic-release * Update points table * Update points table * docs: clarify adding a model (#1222) * fix: Add RepLLaMA style models (#1223) * init commit * working and reproducing * lint * update hashes * warning * add pyproject * Update points table * 1.14.21 Automatically generated by python-semantic-release * docs: Update points (#1228) * Fix case * Fix casing * Fix case * Fix case * Create 971.jsonl * Update contrib * Add contributors * Update points table * docs: Add MTEB(code) dataset (#1237) * docs: Add MTEB(code) dataset * Fix linting * Update points table * Update of my affiliation (#1242) Update points.md * Add contributor (#1243) * fix: @mrshu's name in `points.md` (#1246) * Use the diacritic character to be inline with Slovak spelling. Signed-off-by: mr.Shu * docs: Create benchmarks overview table (#1245) * fix get_benchmarks method * add create benchmark script * make lint * 1.14.22 Automatically generated by python-semantic-release * docs: Update affiliation (#1247) Update points.md * Added author-information * Add final author list * Update points table * docs: Added coordination point for Jimmy Lee (#1253) docs: Added coordination point for Jimmy lee for his work on the coordination of Crystina and Nandan * Update points table * fix: Add multilingual Benchmark (#1252) * fix: Add multilingual bench * Update mteb/benchmarks/benchmarks.py Co-authored-by: Niklas Muennighoff * format --------- Co-authored-by: Niklas Muennighoff * 1.14.23 Automatically generated by python-semantic-release * docs: Small point changes & more contributors (#1254) * Update points.md * Fix format * Fix attribution * Update points table * fix: Downsample large retrieval datasets (#1236) * most tasks * lint * fix other issues * refactor * lint and docs * add polish * keep case sensitive mteb paths * add potential points * fix points * fix test about metadata * update tasks and stats * lint * Update points table * Update tasks table * 1.14.24 Automatically generated by python-semantic-release * fix: Get meta from CrossEncoder (#1255) * remove indent after return * handle cross encoders for model meta * make lint * update filename since we now have model name * 1.14.25 Automatically generated by python-semantic-release * fix: Add listing all available benchmarks CLI option (#1256) * add benchmarks.md in README * add cli option * add benchmark cli test case * correct typo * 1.14.26 Automatically generated by python-semantic-release * docs: Update affiliation (#1248) * Update points.md * Update points.md --------- Co-authored-by: Kenneth Enevoldsen * docs: Update mteb(eng) calculation (#1258) * Update mteb(eng) calculation * Fixed citations * Update MTEB(eng) + MTEB(multilingual) * feat: leverage SentenceTransformers' query/passage specific prompts (#1221) * feat: leverage SentenceTransformer models' query/passage specific prompts * refactor: remove E5Wrapper fix: wrong e5 revisions * fix: default prompt_type to None * fix: e4ce987 revision no longer exists for multilingual-e5-small on the Hub * fix: keep `prompt_name` in kwargs when model doesn't have a `prompts` attr * feat: use Enum for `prompt_type` * docs: specify how to use prompts with Sentence Transformers * feat: readd arctic models due to metadata * 1.15.0 Automatically generated by python-semantic-release * fix: Add Touche2020v3 and JMTEB (#1262) * add datasets * fix metrics * add Touche2020v3 * fix metadata * Apply suggestions from code review Co-authored-by: Kenneth Enevoldsen * upd name and supress * add benchmark class --------- Co-authored-by: Kenneth Enevoldsen * Update tasks table * 1.15.1 Automatically generated by python-semantic-release * fix: Select benchmarks CLI option (#1261) * add test case for a list of Benchmarks * add selecting benchmarks CLI option * typos * use a separate attribute for benchmarks * try fixing tests * should accept string as well * revert filename change * use Benchmark and avoid circular import * fix: derive `results_directory` path from `results_repo` name (#1275) fix: don't hardcode repo name when downloading results * 1.15.2 Automatically generated by python-semantic-release * fix: sorting benchmark tasks by MTEB, then alphabetical (#1271) * sorted * fixed formatting * efficiency changes * fix test * make lint --------- Co-authored-by: Isaac Chung * 1.15.3 Automatically generated by python-semantic-release * ci: Removed 3.8 dependency (#1281) Changes include: - remove 3.8 from tests (added 3.11 and 3.12) - changed other CI to 3.9 - updated lint rules to use 3.8 * Update points table * fix: Allow Numpy >=2.0 (#1264) Allow Numpy >=2.0 * 1.15.4 Automatically generated by python-semantic-release * docs: points for paper writing (#1286) * Create 1004.jsonl * Create 1006.jsonl * Update docs/mmteb/points/1004.jsonl * Update docs/mmteb/points/1006.jsonl --------- Co-authored-by: Kenneth Enevoldsen * Update points table * Update points table * Update points table * docs: Fix a link in the README (#1289) * Fix a link in the README And fix some typos. * Update README.md * Update points table * fix: Update benchmarks (#1288) * make benchmark var name uppercase * update touche to v3 * add MIRACLRetrievalHardNegatives to multilingual * add mteb(indic) * add eu benchmark * 1.15.5 Automatically generated by python-semantic-release * fix: Allow numpy<2.0.0 (#1291) * 1.15.6 Automatically generated by python-semantic-release * fix: Add metadata dict to QBQTC in C-MTEB (#1292) * fix QBQTC in C-MTEB * make lint --------- Co-authored-by: Isaac Chung * 1.15.7 Automatically generated by python-semantic-release * fix: Remove non-existent eval split of CMNLI (#1294) fix eval_splits of CMNLI * 1.15.8 Automatically generated by python-semantic-release * Leaderboard (#1235) * Add leaderboard dev * Renamed MTEBResults to TaskResult * Moved model and model meta loading utilities into overview.py * Added get_model_metas to retrieve filtered metadata for models * Restructured results object and made it into a class instead of a dict * Added utilities for filtering models on BenchmarkResults objects * Added to_table utility function to BenchmarkResults * Added serialization utilities to BenchmarkResults * Attempted fixing tests * Added get_model_metas to __init__ * Added get_benchmarks to __init__ and made it return all benchmarks by default * Added get_benchmarks to __init__ * Made tasks hashable * Added task filtering based on task objects on BenchmarkResults * Added BenchmarkResults to __init__ * Added additional arguments to get_scores on two classes * Made get_scores smarter on BenchmarkResult * Added basic multilingual benchmark * Modified benchmark to be able to easily access results * Added useful properties and filtering functions to BenchmarkResults * Added minimal functioning example * Added smarter table, task-list updating and tried fixing dropdown scrolling * Made restrict_results into a private function Co-authored-by: Kenneth Enevoldsen * Removed old leaderboard scripts * Hardcoded max and min model size * Removed redundant utils file * Ran linting * added leaderboard dependencies as optional * Fixed union type error on Python 3.9 * Removed references to Dict in task aggregation * Fixed name errors in _restrict_task_results * Fixed _restrict_task_results * Made hf_subsets={'default'} when the task is monolingual in _restric_task_results * Task dropdown now gets filtered based on the other criteria * Ran linting again * Introduced hotfix for reranking test * Added BenchmarkResults to __all__ in __init__ * Fixed validate_and_filter_scores method, and replaced _restric_task_results with it --------- Co-authored-by: Kenneth Enevoldsen * feat: Use prompts instead of encode_corpus and encode_queries (#1278) * add prompt per task type * fix prompt * upd test * lint * fix test * fix DeprecatedSummarizationEvaluator * fix prompts * add test * lint * logger info * use task type only in model_encode * lint * update interface * add prompt types to docs * fix test * mock tasks * mock task registry * remove last task_type * fix tests * lint * fix test * fix * use wrapper and new prompts * fix tests * lint * fix test * remove conftest * validate task to prompt_name * override model prompts * task to prompt name optional * fix tests * fix models * remove task_to_prompt_name * remove from mteb __init__ * update docs * load existing model prompts if model_prompts is None * fix * lint * change wrapper loader * add wrapper class * lint * add wrapper file * update logging * upd logging * refactor reranking * lint * remove prints * 1.16.0 Automatically generated by python-semantic-release * fix: Add Retrieval SK Quad dataset for Slovak search evaluation (#1276) * Add Retrieval SK Quad dataset for Slovak search evaluation This commit introduces the Retrieval SK Quad dataset, designed to assess Slovak search performance. The dataset is derived from SK-QuAD and includes questions with their best answers categorized post-annotation. This addition provides a significant resource for advancing Slovak language search evaluation and supporting further research and development. * Add Retrieval SK Quad dataset for Slovak search evaluation 2 Added the requested changes on the SKQuadRetrieval.py file * add task to init * add missing task metadata --------- Co-authored-by: Isaac Chung * Update tasks table * 1.16.1 Automatically generated by python-semantic-release * fix: Add Slovak Hate Speech and Offensive Language Dataset (#1274) * Add Slovak Hate Speech and Offensive Language Dataset This commit introduces the Slovak Hate Speech and Offensive Language Database to MTEB. The dataset includes posts from a social network, annotated by humans for hate speech and offensive content. Additionally, the corresponding task has been added to the tasks.md table to reflect this update. * Add Slovak Hate Speech and Offensive Language Dataset - Updated __init__.py to include the new SlovakHateSpeechClassification task. - Modified SlovakHateSpeechClassification.py as per review suggestions to enhance functionality and readability. * Did requested changes: - Updated __init__.py to include the new SlovakHateSpeechClassification task. - Modified SlovakHateSpeechClassification.py as per review suggestions to enhance functionality and readability. * resolve linting issues by running `make lint` * Update tasks table * WIP: Leaderboard UI improvements (#1312) * Fixed typos in task_results * Fixed typos in task_results * Added Tailwind, reorganized layout and fixed scrolling * Ran linting * 1.16.2 Automatically generated by python-semantic-release * fix: remove duplicate multilingual * 1.16.3 Automatically generated by python-semantic-release * fix: Re-upload dataset to hub to avoid using script upload (#1322) * fix dataset upload * add linting * Update tasks table * 1.16.4 Automatically generated by python-semantic-release * fix: Add implementations of common reranker models (#1309) * init * revert * revert * add metadata * lint * add reqs * change to float16 * benchmark lint fix * 1.16.5 Automatically generated by python-semantic-release * Add multilingual mFollowIR dataset (#1308) * add mFollowIR * paper name * edit warning->info * convert to parquet * lint * Update tasks table * Cache the embeddings when requested (#1307) * add caching * update test to use close * change from json to pkl * fix for window * cleanup on Windows again * infer dimension * move cachewrapper * add wrapper * fix * updates * fix tests * fix lint * lint * add test * WIP: Leaderboard UI improvements (#1320) * Fixed typos in task_results * Fixed typos in task_results * Added Tailwind, reorganized layout and fixed scrolling * Ran linting * Removed faux benchmark * Updated layout * Changed table number format * Table highlights highest values by making them bold * Added rank to table, removed organization from model_name * Added mean rank to table * Ran linting * feat: Update metadata for all models (#1316) * Added model meta * format * fixed metadata * Metadata update for voyage models * Update mteb/models/cohere_models.py Co-authored-by: Roman Solomatin * Update mteb/models/cohere_models.py Co-authored-by: Roman Solomatin * Added corrections from review * fix spelling error --------- Co-authored-by: Roman Solomatin * resolved bugs from pytest --collect-only * Avoid wrapping all models with the SentenceTransformerWrapper * Added normalize_embeddings_to_numpy to ensure standard embeddings during evaluations * fixed moved on correction from @Samoed * conditionally set .predict method on SentenceTransformerWrapper --------- Signed-off-by: mr.Shu Co-authored-by: HSILA Co-authored-by: Ali Shiraee Co-authored-by: github-actions Co-authored-by: Thomas van Dongen Co-authored-by: github-actions[bot] Co-authored-by: Niklas Muennighoff Co-authored-by: Orion Weller <31665361+orionw@users.noreply.github.com> Co-authored-by: John Yang Co-authored-by: Imene Kerboua <33312980+imenelydiaker@users.noreply.github.com> Co-authored-by: Marek Šuppa Co-authored-by: Isaac Chung Co-authored-by: Xa9aX ツ Co-authored-by: Roman Solomatin Co-authored-by: Daniel Buades Marcos Co-authored-by: Daniel Buades Marcos Co-authored-by: Sathvik Nallamalli Co-authored-by: Michael Graczyk Co-authored-by: Mariya Hendriksen <35101262+mariyahendriksen@users.noreply.github.com> Co-authored-by: Santiago Castro Co-authored-by: Joey Xia <77958037+ZiyiXia@users.noreply.github.com> Co-authored-by: Márton Kardos Co-authored-by: Oliver --- .github/pull_request_template.md | 4 +- .github/workflows/lint.yml | 2 +- .github/workflows/mmteb.yml | 4 +- .github/workflows/test.yml | 4 +- README.md | 155 +- .../__init__.py | 0 docs/adding_a_dataset.md | 4 +- docs/adding_a_model.md | 29 +- docs/benchmarks.md | 22 + docs/create_benchmarks_table.py | 58 + docs/mmteb/final_author_list.md | 82 + docs/mmteb/points.md | 92 +- docs/mmteb/points/1004.jsonl | 1 + docs/mmteb/points/1006.jsonl | 1 + docs/mmteb/points/121.jsonl | 1 + docs/mmteb/points/1236.jsonl | 3 + docs/mmteb/points/1237.jsonl | 1 + docs/mmteb/points/1252.jsonl | 1 + docs/mmteb/points/676.jsonl | 4 +- docs/mmteb/points/690.jsonl | 4 +- docs/mmteb/points/724.jsonl | 2 +- docs/mmteb/points/736.jsonl | 2 +- docs/mmteb/points/971.jsonl | 6 + docs/mmteb/points_table.md | 194 +- docs/reproducible_workflow.md | 12 +- docs/tasks.md | 74 +- mteb/__init__.py | 13 +- mteb/__main__.py | 5 + mteb/abstasks/AbsTask.py | 30 +- mteb/abstasks/AbsTaskBitextMining.py | 4 +- mteb/abstasks/AbsTaskClassification.py | 6 +- mteb/abstasks/AbsTaskClustering.py | 6 +- mteb/abstasks/AbsTaskClusteringFast.py | 19 +- mteb/abstasks/AbsTaskInstructionRetrieval.py | 24 +- .../AbsTaskMultilabelClassification.py | 23 +- mteb/abstasks/AbsTaskPairClassification.py | 6 +- mteb/abstasks/AbsTaskReranking.py | 6 +- mteb/abstasks/AbsTaskRetrieval.py | 14 +- mteb/abstasks/AbsTaskSTS.py | 2 +- mteb/abstasks/AbsTaskSpeedTask.py | 16 +- mteb/abstasks/AbsTaskSummarization.py | 2 +- .../Image/AbsTaskAny2AnyMultiChoice.py | 4 +- .../abstasks/Image/AbsTaskAny2AnyRetrieval.py | 3 +- .../Image/AbsTaskAny2TextMultipleChoice.py | 7 +- .../Image/AbsTaskImageClassification.py | 5 +- mteb/abstasks/Image/AbsTaskImageClustering.py | 9 +- .../AbsTaskImageMultilabelClassification.py | 5 +- .../AbsTaskImageTextPairClassification.py | 7 +- mteb/abstasks/Image/AbsTaskVisualSTS.py | 3 +- .../Image/AbsTaskZeroshotClassification.py | 7 +- mteb/abstasks/TaskMetadata.py | 47 +- mteb/abstasks/stratification.py | 22 +- mteb/benchmarks.py | 460 ----- mteb/benchmarks/__init__.py | 4 + mteb/benchmarks/benchmarks.py | 861 ++++++++ mteb/benchmarks/get_benchmark.py | 35 + mteb/cli.py | 53 +- mteb/create_meta.py | 16 +- mteb/encoder_interface.py | 79 +- mteb/evaluation/MTEB.py | 70 +- .../evaluators/BitextMiningEvaluator.py | 8 +- .../evaluators/ClassificationEvaluator.py | 58 +- .../evaluators/ClusteringEvaluator.py | 13 +- .../Image/Any2AnyMultiChoiceEvaluator.py | 4 +- .../Image/Any2AnyRetrievalEvaluator.py | 4 +- .../InstructionRetrievalEvaluator.py | 15 +- .../evaluators/PairClassificationEvaluator.py | 13 +- .../evaluators/RerankingEvaluator.py | 156 +- .../evaluators/RetrievalEvaluator.py | 205 +- mteb/evaluation/evaluators/STSEvaluator.py | 22 +- .../evaluators/SummarizationEvaluator.py | 50 +- mteb/evaluation/evaluators/model_encode.py | 44 - mteb/evaluation/evaluators/utils.py | 24 +- mteb/leaderboard/__init__.py | 5 + mteb/leaderboard/app.py | 223 +++ mteb/leaderboard/table.py | 89 + mteb/load_results/__init__.py | 5 +- mteb/load_results/benchmark_results.py | 316 +++ mteb/load_results/load_results.py | 61 +- .../{mteb_results.py => task_results.py} | 78 +- mteb/model_meta.py | 77 +- mteb/models/__init__.py | 175 +- mteb/models/arctic_models.py | 68 +- mteb/models/bge_models.py | 120 +- mteb/models/bm25.py | 34 +- mteb/models/cache_wrapper.py | 294 +++ mteb/models/cohere_models.py | 93 +- mteb/models/e5_instruct.py | 56 +- mteb/models/e5_models.py | 192 +- mteb/models/google_models.py | 79 +- mteb/models/gritlm_models.py | 61 +- mteb/models/gte_models.py | 86 +- mteb/models/llm2vec_models.py | 164 +- mteb/models/mxbai_models.py | 75 +- mteb/models/nomic_models.py | 106 +- mteb/models/openai_models.py | 51 +- mteb/models/overview.py | 189 ++ mteb/models/promptriever_models.py | 140 ++ mteb/models/repllama_models.py | 182 ++ mteb/models/rerankers_custom.py | 251 +++ mteb/models/rerankers_monot5_based.py | 578 ++++++ mteb/models/ru_sentence_models.py | 163 +- mteb/models/salesforce_models.py | 49 +- mteb/models/sentence_transformer_wrapper.py | 191 ++ mteb/models/sentence_transformers_models.py | 44 +- mteb/models/text_formatting_utils.py | 23 - mteb/models/voyage_models.py | 145 +- mteb/models/wrapper.py | 7 + mteb/normalize_embeddings.py | 31 + mteb/overview.py | 19 +- mteb/task_aggregation.py | 22 +- mteb/task_selection.py | 10 +- .../dan/BornholmskBitextMining.py | 2 +- .../kat/TbilisiCityHallBitextMining.py | 2 +- .../multilingual/BUCCBitextMining.py | 2 +- .../multilingual/BUCCBitextMiningFast.py | 2 +- .../multilingual/BibleNLPBitextMining.py | 2 +- .../multilingual/DiaBLaBitextMining.py | 2 +- .../multilingual/FloresBitextMining.py | 2 +- .../multilingual/IN22ConvBitextMining.py | 2 +- .../multilingual/IN22GenBitextMining.py | 2 +- .../multilingual/IWSLT2017BitextMining.py | 2 +- .../IndicGenBenchFloresBitextMining.py | 2 +- .../multilingual/LinceMTBitextMining.py | 2 +- .../multilingual/NTREXBitextMining.py | 2 +- .../multilingual/NollySentiBitextMining.py | 2 +- .../NorwegianCourtsBitextMining.py | 2 +- .../NusaTranslationBitextMining.py | 2 +- .../multilingual/NusaXBitextMining.py | 2 +- .../multilingual/PhincBitextMining.py | 2 +- .../multilingual/RomaTalesBitextMining.py | 2 +- .../multilingual/TatoebaBitextMining.py | 2 +- .../BitextMining/srn/SRNCorpusBitextMining.py | 2 +- .../BitextMining/vie/VieMedEVBitextMining.py | 2 +- mteb/tasks/Classification/__init__.py | 1 + mteb/tasks/Classification/ara/AJGT.py | 2 +- .../ara/HotelReviewSentimentClassification.py | 2 +- ...nlineStoreReviewSentimentClassification.py | 2 +- ...RestaurantReviewSentimentClassification.py | 2 +- .../ara/TweetEmotionClassification.py | 2 +- .../ara/TweetSarcasmClassification.py | 2 +- .../ben/BengaliDocumentClassification.py | 2 +- .../ben/BengaliHateSpeechClassification.py | 2 +- .../ben/BengaliSentimentAnalysis.py | 2 +- ...SFDCZMovieReviewSentimentClassification.py | 2 +- ...echProductReviewSentimentClassification.py | 2 +- .../ces/CzechSoMeSentimentClassification.py | 2 +- .../ces/CzechSubjectivityClassification.py | 2 +- .../dan/AngryTweetsClassification.py | 2 +- .../dan/DKHateClassification.py | 2 +- .../DanishPoliticalCommentsClassification.py | 2 +- .../dan/LccSentimentClassification.py | 2 +- ...liticiansTwitterSentimentClassification.py | 2 +- .../eng/AmazonPolarityClassification.py | 4 +- .../Classification/eng/ArxivClassification.py | 2 +- .../eng/Banking77Classification.py | 2 +- .../eng/EmotionClassification.py | 2 +- .../eng/FrenkEnClassification.py | 2 +- .../Classification/eng/ImdbClassification.py | 2 +- .../Classification/eng/NewsClassification.py | 2 +- .../eng/PatentClassification.py | 2 +- .../eng/PoemSentimentClassification.py | 2 +- .../eng/ToxicConversationsClassification.py | 2 +- .../TweetSentimentExtractionClassification.py | 2 +- .../eng/TweetTopicSingleClassification.py | 2 +- .../eng/YahooAnswersTopicsClassification.py | 2 +- .../eng/YelpReviewFullClassification.py | 2 +- .../Classification/est/estonian_valence.py | 2 +- .../fas/PersianFoodSentimentClassification.py | 2 +- .../fil/FilipinoHateSpeechClassification.py | 6 +- .../FilipinoShopeeReviewsClassification.py | 2 +- .../fin/FinToxicityClassification.py | 2 +- .../Classification/fra/FrenchBookReviews.py | 2 +- .../fra/MovieReviewSentimentClassification.py | 2 +- .../guj/GujaratiNewsClassification.py | 2 +- .../heb/HebrewSentimentAnalysis.py | 2 +- .../hin/HindiDiscourseClassification.py | 2 +- .../hin/SentimentAnalysisHindi.py | 2 +- .../hrv/FrenkHrClassification.py | 2 +- ...esianMongabayConservationClassification.py | 2 +- .../ita/ItaCaseholdClassification.py | 2 +- ...lianLinguistAcceptabilityClassification.py | 2 +- .../jav/JavaneseIMDBClassification.py | 2 +- .../Classification/jpn/WRIMEClassification.py | 2 +- .../kan/KannadaNewsClassification.py | 2 +- .../kat/GeorgianSentimentClassification.py | 2 +- mteb/tasks/Classification/kor/KlueTC.py | 2 +- mteb/tasks/Classification/kor/KorFin.py | 2 +- .../kor/KorSarcasmClassification.py | 2 +- .../kur/KurdishSentimentClassification.py | 2 +- .../mal/MalayalamNewsClassification.py | 2 +- .../mar/MarathiNewsClassification.py | 2 +- .../MacedonianTweetSentimentClassification.py | 2 +- .../multilingual/AfriSentiClassification.py | 2 +- .../AmazonCounterfactualClassification.py | 2 +- .../CyrillicTurkicLangClassification.py | 2 +- .../multilingual/HinDialectClassification.py | 2 +- .../multilingual/IndicLangClassification.py | 2 +- .../IndicSentimentClassification.py | 2 +- .../multilingual/LanguageClassification.py | 2 +- .../multilingual/MTOPDomainClassification.py | 2 +- .../multilingual/MTOPIntentClassification.py | 2 +- .../MassiveIntentClassification.py | 2 +- .../MassiveScenarioClassification.py | 2 +- .../MultilingualSentimentClassification.py | 2 +- .../Classification/multilingual/NaijaSenti.py | 2 +- .../NusaParagraphEmotionClassification.py | 2 +- .../NusaParagraphTopicClassification.py | 2 +- .../Classification/multilingual/NusaXSenti.py | 2 +- .../multilingual/ScalaClassification.py | 2 +- .../SouthAfricanLangClassification.py | 2 +- .../SwissJudgementClassification.py | 2 +- .../multilingual/TurkicClassification.py | 2 +- mteb/tasks/Classification/mya/MyanmarNews.py | 2 +- .../nep/NepaliNewsClassification.py | 2 +- .../DutchBookReviewSentimentClassification.py | 2 +- .../ory/OdiaNewsClassification.py | 2 +- .../pan/PunjabiNewsClassification.py | 2 +- .../por/HateSpeechPortugueseClassification.py | 2 +- mteb/tasks/Classification/ron/Moroco.py | 2 +- .../ron/RomanianReviewsSentiment.py | 2 +- .../ron/RomanianSentimentClassification.py | 2 +- .../rus/KinopoiskClassification.py | 2 +- .../rus/RuSciBenchGRNTIClassification.py | 2 +- .../rus/RuSciBenchOECDClassification.py | 2 +- .../san/SanskritShlokasClassification.py | 2 +- ...SFDSKMovieReviewSentimentClassification.py | 2 +- .../slk/SlovakHateSpeechClassification.py | 34 + .../slv/FrenkSlClassification.py | 2 +- .../spa/SpanishNewsClassification.py | 2 +- .../spa/SpanishSentimentClassification.py | 2 +- .../ssw/SiswatiNewsClassification.py | 2 +- ...lovakMovieReviewSentimentClassification.py | 2 +- .../swa/SwahiliNewsClassification.py | 2 +- .../Classification/swe/DalajClassification.py | 2 +- .../swe/SwedishSentimentClassification.py | 2 +- .../tam/TamilNewsClassification.py | 2 +- .../TeluguAndhraJyotiNewsClassification.py | 2 +- .../tsn/TswanaNewsClassification.py | 2 +- .../TurkishMovieSentimentClassification.py | 2 +- .../TurkishProductSentimentClassification.py | 2 +- .../urd/UrduRomanSentimentClassification.py | 2 +- .../vie/VieStudentFeedbackClassification.py | 2 +- .../zho/YueOpenriceReviewClassification.py | 2 +- .../zul/IsiZuluNewsClassification.py | 2 +- .../eng/ArXivHierarchicalClustering.py | 8 +- .../Clustering/eng/ArxivClusteringP2P.py | 4 +- mteb/tasks/Clustering/eng/RedditClustering.py | 2 +- .../Clustering/eng/RedditClusteringP2P.py | 2 +- .../Clustering/eng/StackExchangeClustering.py | 2 +- .../eng/StackExchangeClusteringP2P.py | 2 +- .../eng/TwentyNewsgroupsClustering.py | 4 +- mteb/tasks/Clustering/fra/HALClusteringS2S.py | 2 +- .../multilingual/IndicReviewsClusteringP2P.py | 2 +- .../multilingual/MLSUMClusteringP2P.py | 18 +- .../multilingual/MLSUMClusteringS2S.py | 18 +- .../multilingual/MasakhaNEWSClusteringP2P.py | 2 +- .../nob/SNLHierarchicalClustering.py | 4 +- .../nob/VGHierarchicalClustering.py | 4 +- mteb/tasks/Clustering/nob/snl_clustering.py | 5 +- mteb/tasks/Clustering/nob/vg_clustering.py | 3 +- mteb/tasks/Clustering/pol/PolishClustering.py | 4 +- .../Clustering/rom/RomaniBibleClustering.py | 2 +- .../rus/RuSciBenchGRNTIClusteringP2P.py | 2 +- .../rus/RuSciBenchOECDClusteringP2P.py | 2 +- mteb/tasks/Clustering/zho/CMTEBClustering.py | 8 +- .../eng/ImageCoDeT2IMultiChoice.py | 6 +- .../eng/BLINKIT2IRetrieval.py | 2 +- .../eng/BLINKIT2TRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py | 2 +- .../eng/CUB200I2IRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/FORBI2IRetrieval.py | 2 +- .../eng/Fashion200kI2TRetrieval.py | 3 +- .../eng/Fashion200kT2IRetrieval.py | 2 +- .../eng/FashionIQIT2IRetrieval.py | 2 +- .../eng/Flickr30kI2TRetrieval.py | 6 +- .../eng/Flickr30kT2IRetrieval.py | 6 +- .../Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py | 2 +- .../eng/HatefulMemesI2TRetrieval.py | 2 +- .../eng/HatefulMemesT2IRetrieval.py | 2 +- .../eng/ImageCoDeT2IRetrieval.py | 6 +- .../eng/InfoSeekIT2ITRetrieval.py | 2 +- .../eng/InfoSeekIT2TRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/METI2IRetrieval.py | 2 +- .../eng/MSCOCOI2TRetrieval.py | 2 +- .../eng/MSCOCOT2IRetrieval.py | 2 +- .../eng/MemotionI2TRetrieval.py | 2 +- .../eng/MemotionT2IRetrieval.py | 2 +- .../eng/NIGHTSI2IRetrieval.py | 2 +- .../eng/OVENIT2ITRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/OVENIT2TRetrieval.py | 2 +- .../eng/ROxfordI2IRetrieval.py | 6 +- .../Any2AnyRetrieval/eng/RP2kI2IRetrieval.py | 2 +- .../eng/RParisI2IRetrieval.py | 6 +- .../Any2AnyRetrieval/eng/SOPI2IRetrieval.py | 2 +- .../eng/SciMMIRI2TRetrieval.py | 2 +- .../eng/SciMMIRT2IRetrieval.py | 2 +- .../eng/SketchyI2IRetrieval.py | 2 +- .../eng/StanfordCarsI2IRetrieval.py | 2 +- .../eng/TUBerlinT2IRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py | 2 +- .../eng/VidoreBenchRetrieval.py | 20 +- .../eng/VisualNewsI2TRetrieval.py | 2 +- .../eng/VisualNewsT2IRetrieval.py | 2 +- .../eng/VizWizIT2TRetrieval.py | 2 +- .../eng/WebQAT2ITRetrieval.py | 2 +- .../Any2AnyRetrieval/eng/WebQAT2TRetrieval.py | 2 +- .../multilingual/WITT2IRetrieval.py | 6 +- .../multilingual/XFlickr30kCoT2IRetrieval.py | 6 +- .../multilingual/XM3600T2IRetrieval.py | 6 +- .../Any2TextMultipleChoice/eng/CVBench.py | 8 +- mteb/tasks/Image/Clustering/eng/CIFAR.py | 5 +- mteb/tasks/Image/Clustering/eng/ImageNet.py | 5 +- .../Image/Clustering/eng/TinyImageNet.py | 2 +- .../eng/BirdsnapClassification.py | 2 +- .../Image/ImageClassification/eng/CIFAR.py | 5 +- .../eng/Caltech101Classification.py | 2 +- .../eng/Country211Classification.py | 2 +- .../eng/DTDClassification.py | 2 +- .../eng/EuroSATClassification.py | 2 +- .../eng/FER2013Classification.py | 2 +- .../eng/FGVCAircraftClassification.py | 2 +- .../eng/Food101Classification.py | 2 +- .../eng/GTSRBClassification.py | 2 +- .../ImageClassification/eng/Imagenet1k.py | 2 +- .../eng/MNISTClassification.py | 2 +- .../eng/OxfordFlowersClassification.py | 2 +- .../eng/OxfordPetsClassification.py | 2 +- .../eng/PatchCamelyonClassification.py | 2 +- .../eng/RESISC45Classification.py | 2 +- .../eng/STL10Classification.py | 2 +- .../eng/SUN397Classification.py | 2 +- .../eng/StanfordCarsClassification.py | 2 +- .../eng/UCF101Classification.py | 2 +- .../eng/PascalVOC2007.py | 2 +- .../AROCocoOrder.py | 2 +- .../AROFlickrOrder.py | 2 +- .../AROVisualAttribution.py | 2 +- .../AROVisualRelation.py | 2 +- .../ImageTextPairClassification/SugarCrepe.py | 2 +- .../ImageTextPairClassification/Winoground.py | 2 +- .../Image/VisualSTS/en/STS12VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS13VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS14VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS15VisualSTS.py | 2 +- .../Image/VisualSTS/en/STS16VisualSTS.py | 2 +- .../STS17MultilingualVisualSTS.py | 2 +- .../STSBenchmarkMultilingualVisualSTS.py | 2 +- .../ZeroshotClassification/eng/Birdsnap.py | 3 +- .../Image/ZeroshotClassification/eng/CIFAR.py | 6 +- .../Image/ZeroshotClassification/eng/CLEVR.py | 6 +- .../ZeroshotClassification/eng/Caltech101.py | 3 +- .../ZeroshotClassification/eng/Country211.py | 3 +- .../Image/ZeroshotClassification/eng/DTD.py | 3 +- .../ZeroshotClassification/eng/EuroSAT.py | 3 +- .../ZeroshotClassification/eng/FER2013.py | 3 +- .../eng/FGVCAircraft.py | 3 +- .../ZeroshotClassification/eng/Food101.py | 3 +- .../Image/ZeroshotClassification/eng/GTSRB.py | 3 +- .../ZeroshotClassification/eng/Imagenet1k.py | 3 +- .../Image/ZeroshotClassification/eng/MNIST.py | 3 +- .../ZeroshotClassification/eng/OxfordPets.py | 3 +- .../eng/PatchCamelyon.py | 3 +- .../ZeroshotClassification/eng/RESISC45.py | 3 +- .../eng/RenderedSST2.py | 1 - .../Image/ZeroshotClassification/eng/STL10.py | 3 +- .../ZeroshotClassification/eng/SUN397.py | 3 +- .../ZeroshotClassification/eng/SciMMIR.py | 3 +- .../eng/StanfordCars.py | 3 +- .../ZeroshotClassification/eng/UCF101.py | 3 +- mteb/tasks/InstructionRetrieval/__init__.py | 1 + .../eng/Core17InstructionRetrieval.py | 4 +- .../eng/News21InstructionRetrieval.py | 4 +- .../eng/Robust04InstructionRetrieval.py | 4 +- .../multilingual/__init__.py | 0 .../multilingual/mFollowIR.py | 368 ++++ .../MultiEURLEXMultilabelClassification.py | 2 +- .../por/BrazilianToxicTweetsClassification.py | 2 +- mteb/tasks/PairClassification/ara/ArEntail.py | 2 +- .../PairClassification/ces/CTKFactsNLI.py | 2 +- .../eng/SprintDuplicateQuestionsPC.py | 2 +- mteb/tasks/PairClassification/fas/FarsTail.py | 2 +- .../hye/ArmenianParaphrasePC.py | 2 +- mteb/tasks/PairClassification/ind/IndoNLI.py | 2 +- mteb/tasks/PairClassification/kor/KlueNLI.py | 2 +- .../multilingual/PawsXPairClassification.py | 2 +- .../PairClassification/multilingual/XNLI.py | 4 +- .../multilingual/XStance.py | 2 +- mteb/tasks/PairClassification/pol/PolishPC.py | 6 +- .../tasks/PairClassification/por/Assin2RTE.py | 2 +- mteb/tasks/PairClassification/por/SickBrPC.py | 2 +- .../zho/CMTEBPairClassification.py | 2 +- mteb/tasks/Reranking/__init__.py | 1 + .../eng/WebLINXCandidatesReranking.py | 2 +- mteb/tasks/Reranking/fra/AlloprofReranking.py | 2 +- mteb/tasks/Reranking/fra/SyntecReranking.py | 2 +- mteb/tasks/Reranking/jpn/MMarcoReranking.py | 2 +- .../Reranking/multilingual/ESCIReranking.py | 86 + .../Reranking/multilingual/MIRACLReranking.py | 8 +- mteb/tasks/Reranking/zho/CMTEBReranking.py | 2 +- mteb/tasks/Retrieval/__init__.py | 3 + .../Retrieval/ara/SadeemQuestionRetrieval.py | 2 +- mteb/tasks/Retrieval/code/AppsRetrieval.py | 2 +- .../code/COIRCodeSearchNetRetrieval.py | 2 +- .../Retrieval/code/CodeEditSearchRetrieval.py | 2 +- .../Retrieval/code/CodeFeedbackMTRetrieval.py | 2 +- .../Retrieval/code/CodeFeedbackSTRetrieval.py | 2 +- .../code/CodeSearchNetCCRetrieval.py | 2 +- .../Retrieval/code/CodeSearchNetRetrieval.py | 2 +- .../code/CodeTransOceanContestRetrieval.py | 2 +- .../code/CodeTransOceanDLRetrieval.py | 2 +- mteb/tasks/Retrieval/code/CosQARetrieval.py | 2 +- .../code/StackOverflowQARetrieval.py | 2 +- .../code/SyntheticText2SqlRetrieval.py | 2 +- mteb/tasks/Retrieval/dan/DanFeverRetrieval.py | 16 +- mteb/tasks/Retrieval/dan/TV2Nordretrieval.py | 8 +- .../Retrieval/dan/TwitterHjerneRetrieval.py | 8 +- .../Retrieval/deu/GerDaLIRSmallRetrieval.py | 2 +- .../tasks/Retrieval/deu/LegalQuADRetrieval.py | 2 +- .../Retrieval/eng/AILACasedocsRetrieval.py | 2 +- .../Retrieval/eng/AILAStatutesRetrieval.py | 2 +- .../Retrieval/eng/ARCChallengeRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/AlphaNLIRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/BrightRetrieval.py | 6 +- .../Retrieval/eng/ClimateFEVERRetrieval.py | 45 + mteb/tasks/Retrieval/eng/DBPediaRetrieval.py | 47 + mteb/tasks/Retrieval/eng/FEVERRetrieval.py | 62 + .../Retrieval/eng/FeedbackQARetrieval.py | 2 +- .../tasks/Retrieval/eng/HellaSwagRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/HotpotQARetrieval.py | 63 + .../Retrieval/eng/LEMBNarrativeQARetrieval.py | 2 +- .../Retrieval/eng/LEMBNeedleRetrieval.py | 2 +- .../Retrieval/eng/LEMBPasskeyRetrieval.py | 2 +- .../tasks/Retrieval/eng/LEMBQMSumRetrieval.py | 2 +- .../eng/LEMBSummScreenFDRetrieval.py | 2 +- .../Retrieval/eng/LEMBWikimQARetrieval.py | 2 +- .../LegalBenchConsumerContractsQARetrieval.py | 2 +- .../LegalBenchCorporateLobbyingRetrieval.py | 2 +- .../eng/LegalSummarizationRetrieval.py | 2 +- .../tasks/Retrieval/eng/LitSearchRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/MSMARCORetrieval.py | 59 + .../tasks/Retrieval/eng/MedicalQARetrieval.py | 2 +- mteb/tasks/Retrieval/eng/NQRetrieval.py | 43 + mteb/tasks/Retrieval/eng/PiqaRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/QuailRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/QuoraRetrieval.py | 49 + mteb/tasks/Retrieval/eng/RARbCodeRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/RARbMathRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/SiqaRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/SpartQARetrieval.py | 2 +- .../Retrieval/eng/TempReasonL1Retrieval.py | 2 +- .../eng/TempReasonL2ContextRetrieval.py | 2 +- .../eng/TempReasonL2FactRetrieval.py | 2 +- .../eng/TempReasonL2PureRetrieval.py | 2 +- .../eng/TempReasonL3ContextRetrieval.py | 2 +- .../eng/TempReasonL3FactRetrieval.py | 2 +- .../eng/TempReasonL3PureRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py | 52 + .../Retrieval/eng/Touche2020Retrieval.py | 60 +- .../Retrieval/eng/WinoGrandeRetrieval.py | 2 +- mteb/tasks/Retrieval/est/estqa.py | 2 +- mteb/tasks/Retrieval/fra/SyntecRetrieval.py | 2 +- mteb/tasks/Retrieval/hun/HunSum2.py | 2 +- mteb/tasks/Retrieval/jpn/JaQuADRetrieval.py | 2 +- mteb/tasks/Retrieval/jpn/JaqketRetrieval.py | 46 + .../Retrieval/kat/GeorgianFAQRetrieval.py | 2 +- .../multilingual/BelebeleRetrieval.py | 2 +- ...CrossLingualSemanticDiscriminationWMT19.py | 2 +- ...CrossLingualSemanticDiscriminationWMT21.py | 2 +- .../multilingual/IndicQARetrieval.py | 2 +- .../Retrieval/multilingual/MIRACLRetrieval.py | 329 ++- .../multilingual/MintakaRetrieval.py | 2 +- .../Retrieval/multilingual/MrTidyRetrieval.py | 131 ++ .../multilingual/NeuCLIR2022Retrieval.py | 150 ++ .../multilingual/NeuCLIR2023Retrieval.py | 152 ++ .../multilingual/PublicHealthQARetrieval.py | 2 +- .../Retrieval/multilingual/XPQARetrieval.py | 2 +- .../Retrieval/multilingual/XQuADRetrieval.py | 2 +- mteb/tasks/Retrieval/nob/norquad.py | 8 +- mteb/tasks/Retrieval/nob/snl_retrieval.py | 8 +- .../tasks/Retrieval/pol/DBPediaPLRetrieval.py | 48 + .../Retrieval/pol/HotpotQAPLRetrieval.py | 46 + .../tasks/Retrieval/pol/MSMARCOPLRetrieval.py | 48 + mteb/tasks/Retrieval/pol/NQPLRetrieval.py | 46 + mteb/tasks/Retrieval/pol/QuoraPLRetrieval.py | 46 + .../Retrieval/pol/TRECCOVIDPLRetrieval.py | 2 +- mteb/tasks/Retrieval/rus/RiaNewsRetrieval.py | 45 + mteb/tasks/Retrieval/slk/SKQuadRetrieval.py | 80 + mteb/tasks/Retrieval/swe/SweFaqRetrieval.py | 8 +- mteb/tasks/Retrieval/swe/SwednRetrieval.py | 8 +- mteb/tasks/Retrieval/tur/TurHistQuad.py | 8 +- mteb/tasks/Retrieval/zho/LeCaRDv2Retrieval.py | 2 +- mteb/tasks/STS/eng/STS12STS.py | 2 +- mteb/tasks/STS/eng/STS13STS.py | 2 +- mteb/tasks/STS/eng/STS14STS.py | 2 +- mteb/tasks/STS/eng/STS15STS.py | 2 +- mteb/tasks/STS/eng/STS16STS.py | 2 +- mteb/tasks/STS/jpn/JSTS.py | 2 +- mteb/tasks/STS/kor/KlueSTS.py | 2 +- mteb/tasks/STS/kor/KorSTS.py | 2 +- .../STS/multilingual/IndicCrosslingualSTS.py | 2 +- .../STS/multilingual/STS17CrosslingualSTS.py | 2 +- .../STS/multilingual/STS22CrosslingualSTS.py | 4 +- .../STSBenchmarkMultilingualSTS.py | 2 +- mteb/tasks/STS/multilingual/SemRel24STS.py | 2 +- mteb/tasks/STS/pol/PolishSTS.py | 4 +- mteb/tasks/STS/por/Assin2STS.py | 2 +- mteb/tasks/STS/por/SickBrSTS.py | 2 +- mteb/tasks/STS/rus/RUParaPhraserSTS.py | 2 +- mteb/tasks/STS/zho/CMTEBSTS.py | 7 + mteb/tasks/SpeedTask/CPUSpeedTask.py | 2 +- mteb/tasks/SpeedTask/GPUSpeedTask.py | 2 +- pyproject.toml | 19 +- scripts/data/run_mteb_english.py | 120 -- scripts/mmteb_create_author_list.ipynb | 848 +++++--- scripts/running_model/check_results.py | 4 +- scripts/running_model/check_run.sh | 2 +- scripts/running_model/create_slurm_jobs.py | 2 +- scripts/task_selection/mteb_lite_results.csv | 26 +- scripts/task_selection/mteb_lite_tasks.csv | 66 +- scripts/task_selection/mult_results.csv | 26 +- .../task_selection_eng_lite.ipynb | 1774 +++++++++++------ .../task_selection/task_selection_mult.ipynb | 470 ++++- tasks_per_language.pdf | Bin 0 -> 18297 bytes tests/test_TaskMetadata.py | 6 +- tests/test_benchmark/mock_models.py | 118 +- tests/test_benchmark/mock_tasks.py | 10 +- tests/test_benchmark/task_grid.py | 2 + tests/test_benchmark/test_benchmark.py | 232 ++- tests/test_cli.py | 21 +- tests/test_embedding_caching.py | 98 + tests/test_encoder_interfaces.py | 3 +- .../test_ClusteringEvaluator.py | 2 +- .../test_InstructionRetrievalEvaluator.py | 4 +- .../test_RetrievalEvaluator.py | 6 +- .../test_mteb_load_results.py | 19 +- tests/test_load_results/test_mteb_results.py | 10 +- tests/test_overview.py | 18 +- tests/test_reproducible_workflow.py | 49 +- tests/test_task_aggregation.py | 33 +- tests/test_tasks/test_mteb_rerank.py | 22 +- 541 files changed, 11124 insertions(+), 3819 deletions(-) rename scripts/mmteb_create_author_list.py => docs/__init__.py (100%) create mode 100644 docs/benchmarks.md create mode 100644 docs/create_benchmarks_table.py create mode 100644 docs/mmteb/final_author_list.md create mode 100644 docs/mmteb/points/1004.jsonl create mode 100644 docs/mmteb/points/1006.jsonl create mode 100644 docs/mmteb/points/121.jsonl create mode 100644 docs/mmteb/points/1236.jsonl create mode 100644 docs/mmteb/points/1237.jsonl create mode 100644 docs/mmteb/points/1252.jsonl create mode 100644 docs/mmteb/points/971.jsonl create mode 100644 mteb/__main__.py delete mode 100644 mteb/benchmarks.py create mode 100644 mteb/benchmarks/__init__.py create mode 100644 mteb/benchmarks/benchmarks.py create mode 100644 mteb/benchmarks/get_benchmark.py delete mode 100644 mteb/evaluation/evaluators/model_encode.py create mode 100644 mteb/leaderboard/__init__.py create mode 100644 mteb/leaderboard/app.py create mode 100644 mteb/leaderboard/table.py create mode 100644 mteb/load_results/benchmark_results.py rename mteb/load_results/{mteb_results.py => task_results.py} (89%) create mode 100644 mteb/models/cache_wrapper.py create mode 100644 mteb/models/overview.py create mode 100644 mteb/models/promptriever_models.py create mode 100644 mteb/models/repllama_models.py create mode 100644 mteb/models/rerankers_custom.py create mode 100644 mteb/models/rerankers_monot5_based.py create mode 100644 mteb/models/sentence_transformer_wrapper.py delete mode 100644 mteb/models/text_formatting_utils.py create mode 100644 mteb/models/wrapper.py create mode 100644 mteb/normalize_embeddings.py create mode 100644 mteb/tasks/Classification/slk/SlovakHateSpeechClassification.py create mode 100644 mteb/tasks/InstructionRetrieval/multilingual/__init__.py create mode 100644 mteb/tasks/InstructionRetrieval/multilingual/mFollowIR.py create mode 100644 mteb/tasks/Reranking/multilingual/ESCIReranking.py create mode 100644 mteb/tasks/Retrieval/jpn/JaqketRetrieval.py create mode 100644 mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py create mode 100644 mteb/tasks/Retrieval/slk/SKQuadRetrieval.py delete mode 100644 scripts/data/run_mteb_english.py create mode 100644 tasks_per_language.pdf create mode 100644 tests/test_embedding_caching.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1e4c1d5928..be1d1c7418 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -34,6 +34,6 @@ see also https://github.com/embeddings-benchmark/mteb/blob/main/docs/reproducibl - [ ] I have filled out the ModelMeta object to the extent possible - [ ] I have ensured that my model can be loaded using - - [ ] `mteb.get_model(model_name, revision_id)` and - - [ ] `mteb.get_model_meta(model_name, revision_id)` + - [ ] `mteb.get_model(model_name, revision)` and + - [ ] `mteb.get_model_meta(model_name, revision)` - [ ] I have tested the implementation works on a representative set of tasks. \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 861c2e6ba6..23f0a095ea 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.9" cache: "pip" - name: Install dependencies diff --git a/.github/workflows/mmteb.yml b/.github/workflows/mmteb.yml index 6ae21152f2..522f0ab4af 100644 --- a/.github/workflows/mmteb.yml +++ b/.github/workflows/mmteb.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.9" cache: "pip" - name: Install dependencies @@ -38,7 +38,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.9" cache: "pip" - name: Install dependencies diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e56a85ce99..1fdfff47a2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,11 +16,11 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] #, macos-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12"] include: # Add Windows with Python 3.8 only to avoid tests taking too long - os: windows-latest - python-version: "3.8" + python-version: "3.9" steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index e08545aec6..d20afdbedc 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@

Installation | - Usage | + Usage | Leaderboard | Documentation | Citing @@ -36,9 +36,9 @@ pip install mteb ``` -## Usage +## Example Usage -* Using a python script (see [scripts/run_mteb_english.py](https://github.com/embeddings-benchmark/mteb/blob/main/scripts/run_mteb_english.py) and [mteb/mtebscripts](https://github.com/embeddings-benchmark/mtebscripts) for more): +* Using a Python script: ```python import mteb @@ -55,6 +55,37 @@ evaluation = mteb.MTEB(tasks=tasks) results = evaluation.run(model, output_folder=f"results/{model_name}") ``` +

+ Running SentneceTransformermer model with prompts + +Prompts can be passed to the SentenceTransformer model using the `prompts` parameter. The following code shows how to use prompts with SentenceTransformer: + +```python +from sentence_transformers import SentenceTransformer + + +model = SentenceTransformer("average_word_embeddings_komninos", prompts={"query": "Query:", "passage": "Passage:"}) +evaluation = mteb.MTEB(tasks=tasks) +``` + +In prompts the key can be: +1. Prompt types (`passage`, `query`) - they will be used in reranking and retrieval tasks +2. Task type - these prompts will be used in all tasks of the given type + 1. `BitextMining` + 2. `Classification` + 3. `MultilabelClassification` + 4. `Clustering` + 5. `PairClassification` + 6. `Reranking` + 7. `Retrieval` + 8. `STS` + 9. `Summarization` + 10. `InstructionRetrieval` +3. Pair of task type and prompt type like `Retrival-query` - these prompts will be used in all classification tasks +4. Task name - these prompts will be used in the specific task +5. Pair of task name and prompt type like `NFCorpus-query` - these prompts will be used in the specific task +
+ * Using CLI ```bash @@ -71,17 +102,17 @@ mteb run -m sentence-transformers/all-MiniLM-L6-v2 \ -## Advanced Usage +## Usage Documentation Click on each section below to see the details.
- Dataset selection + Task selection -### Dataset selection +### Task selection -Datasets can be selected by providing the list of datasets, but also +Tasks can be selected by providing the list of datasets, but also * by their task (e.g. "Clustering" or "Classification") @@ -121,11 +152,33 @@ evaluation = mteb.MTEB(tasks=[ # for an example of a HF subset see "Subset" in the dataset viewer at: https://huggingface.co/datasets/mteb/bucc-bitext-mining ``` -There are also presets available for certain task collections, e.g. to select the 56 English datasets that form the "Overall MTEB English leaderboard": +
+ +
+ Running a benchmark + +### Running a Benchmark + +`mteb` comes with a set of predefined benchmarks. These can be fetched using `get_benchmark` and run in a similar fashion to other sets of tasks. +For instance to select the 56 English datasets that form the "Overall MTEB English leaderboard": + +```python +import mteb +benchmark = mteb.get_benchmark("MTEB(eng)") +evaluation = mteb.MTEB(tasks=benchmark) +``` + +The benchmark specified not only a list of tasks, but also what splits and language to run on. To get an overview of all available benchmarks simply run: ```python -from mteb import MTEB_MAIN_EN -evaluation = mteb.MTEB(tasks=MTEB_MAIN_EN, task_langs=["en"]) +import mteb +benchmarks = mteb.get_benchmarks() +``` + +Generally we use the naming scheme for benchmarks `MTEB(*)`, where the "*" denotes the target of the benchmark. In the case of a language, we use the three-letter language code. For large groups of languages, we use the group notation, e.g., `MTEB(Scandinavian)` for Scandinavian languages. External benchmarks implemented in MTEB like `CoIR` use their original name. When using a benchmark from MTEB please cite `mteb` along with the citations of the benchmark which you can access using: + +```python +benchmark.citation ```
@@ -139,7 +192,7 @@ evaluation = mteb.MTEB(tasks=MTEB_MAIN_EN, task_langs=["en"]) To pass in arguments to the model's `encode` function, you can use the encode keyword arguments (`encode_kwargs`): ```python -evaluation.run(model, encode_kwargs={"batch_size": 32} +evaluation.run(model, encode_kwargs={"batch_size": 32}) ```

@@ -167,55 +220,35 @@ Note that the public leaderboard uses the test splits for all datasets except MS Models should implement the following interface, implementing an `encode` function taking as inputs a list of sentences, and returning a list of embeddings (embeddings can be `np.array`, `torch.tensor`, etc.). For inspiration, you can look at the [mteb/mtebscripts repo](https://github.com/embeddings-benchmark/mtebscripts) used for running diverse models via SLURM scripts for the paper. ```python -class MyModel(): +from mteb.encoder_interface import PromptType + +class CustomModel: def encode( - self, sentences: list[str], **kwargs: Any - ) -> torch.Tensor | np.ndarray: + self, + sentences: list[str], + task_name: str, + prompt_type: PromptType | None = None, + **kwargs, + ) -> np.ndarray: """Encodes the given sentences using the encoder. - + Args: sentences: The sentences to encode. + task_name: The name of the task. + prompt_type: The prompt type to use. **kwargs: Additional arguments to pass to the encoder. - + Returns: The encoded sentences. """ pass -model = MyModel() +model = CustomModel() tasks = mteb.get_task("Banking77Classification") evaluation = MTEB(tasks=tasks) evaluation.run(model) ``` -If you'd like to use different encoding functions for query and corpus when evaluating on Retrieval or Reranking tasks, you can add separate methods for `encode_queries` and `encode_corpus`. If these methods exist, they will be automatically used for those tasks. You can refer to the `DRESModel` at `mteb/evaluation/evaluators/RetrievalEvaluator.py` for an example of these functions. - -```python -class MyModel(): - def encode_queries(self, queries: list[str], **kwargs) -> list[np.ndarray] | list[torch.Tensor]: - """ - Returns a list of embeddings for the given sentences. - Args: - queries: List of sentences to encode - - Returns: - List of embeddings for the given sentences - """ - pass - - def encode_corpus(self, corpus: list[str] | list[dict[str, str]], **kwargs) -> list[np.ndarray] | list[torch.Tensor]: - """ - Returns a list of embeddings for the given sentences. - Args: - corpus: List of sentences to encode - or list of dictionaries with keys "title" and "text" - - Returns: - List of embeddings for the given sentences - """ - pass -``` -
@@ -297,7 +330,7 @@ from sentence_transformers import SentenceTransformer model = SentenceTransformer("all-MiniLM-L6-v2") -tasks = mteb.get_tasks( tasks=["NFCorpus"], languages=["eng"]) +tasks = mteb.get_tasks(tasks=["NFCorpus"], languages=["eng"]) evaluation = MTEB(tasks=tasks) evaluation.run( @@ -309,7 +342,7 @@ evaluation.run( ``` CLI: -``` +```bash mteb run -t NFCorpus -m all-MiniLM-L6-v2 --output_folder results --save_predictions ``` @@ -318,9 +351,11 @@ mteb run -t NFCorpus -m all-MiniLM-L6-v2 --output_folder results --save_predicti
Fetching result from the results repository -Multiple models have already been run on tasks avaiable within MTEB. These results are available results [repository](https://github.com/embeddings-benchmark/results). +### Fetching results from the results repository -To make the results more easily accecible we have designed custom functionality for retrieving from the repository. For instance, you are selecting the best model for your French and English retrieval task on legal documents you could fetch the relevant tasks and create a dataframe of the results using the following code: +Multiple models have already been run on tasks available within MTEB. These results are available results [repository](https://github.com/embeddings-benchmark/results). + +To make the results more easily accessible, we have designed custom functionality for retrieving from the repository. For instance, if you are selecting the best model for your French and English retrieval task on legal documents you could fetch the relevant tasks and create a dataframe of the results using the following code: ```python import mteb @@ -345,6 +380,26 @@ df = results_to_dataframe(results)
+
+ Caching Embeddings To Re-Use Them + + +### Caching Embeddings To Re-Use Them + +There are times you may want to cache the embeddings so you can re-use them. This may be true if you have multiple query sets for the same corpus (e.g. Wikipedia) or are doing some optimization over the queries (e.g. prompting, other experiments). You can setup a cache by using a simple wrapper, which will save the cache per task in the `cache_embeddings/{task_name}` folder: + +```python +# define your task and model above as normal +... +# wrap the model with the cache wrapper +from mteb.models.cache_wrapper import CachedEmbeddingWrapper +model_with_cached_emb = CachedEmbeddingWrapper(model, cache_path='path_to_cache_dir') +# run as normal +evaluation.run(model, ...) +``` + +
+
@@ -354,6 +409,7 @@ df = results_to_dataframe(results) | Documentation | | | ------------------------------ | ---------------------- | | 📋 [Tasks] | Overview of available tasks | +| 📐 [Benchmarks] | Overview of available benchmarks | | 📈 [Leaderboard] | The interactive leaderboard of the benchmark | | 🤖 [Adding a model] | Information related to how to submit a model to the leaderboard | | 👩‍🔬 [Reproducible workflows] | Information related to how to reproduce and create reproducible workflows with MTEB | @@ -363,6 +419,7 @@ df = results_to_dataframe(results) | 🌐 [MMTEB] | An open-source effort to extend MTEB to cover a broad set of languages |   [Tasks]: docs/tasks.md +[Benchmarks]: docs/benchmarks.md [Contributing]: CONTRIBUTING.md [Adding a model]: docs/adding_a_model.md [Adding a dataset]: docs/adding_a_dataset.md diff --git a/scripts/mmteb_create_author_list.py b/docs/__init__.py similarity index 100% rename from scripts/mmteb_create_author_list.py rename to docs/__init__.py diff --git a/docs/adding_a_dataset.md b/docs/adding_a_dataset.md index 3e593912e3..4dc1b70a2f 100644 --- a/docs/adding_a_dataset.md +++ b/docs/adding_a_dataset.md @@ -77,7 +77,7 @@ evaluation = MTEB(tasks=[MindSmallReranking()]) evaluation.run(model) ``` -> **Note:** for multilingual / crosslingual tasks, make sure your class also inherits from the `MultilingualTask` class like in [this](https://github.com/embeddings-benchmark/mteb-draft/blob/main/mteb/tasks/Classification/MTOPIntentClassification.py) example. +> **Note:** for multilingual / crosslingual tasks, make sure your class also inherits from the `MultilingualTask` class like in [this](https://github.com/embeddings-benchmark/mteb/blob/main/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py) example. @@ -104,7 +104,7 @@ class VGClustering(AbsTaskClustering): form="Written", domains=["Academic", "Non-fiction"], task_subtypes=["Scientific Reranking"], - license="cc-by-nc", + license="cc-by-nc-4.0", annotations_creators="derived", dialect=[], text_creation="found", diff --git a/docs/adding_a_model.md b/docs/adding_a_model.md index bae2a257e0..5018f49b65 100644 --- a/docs/adding_a_model.md +++ b/docs/adding_a_model.md @@ -29,10 +29,7 @@ mteb run -m {model_name} -t {task_names} These will save the results in a folder called `results/{model_name}/{model_revision}`. -For reference you can also look at [scripts/run_mteb_english.py](https://github.com/embeddings-benchmark/mteb/blob/main/scripts/run_mteb_english.py) for all MTEB English datasets used in the main ranking, or [scripts/run_mteb_chinese.py](https://github.com/embeddings-benchmark/mteb/blob/main/scripts/run_mteb_chinese.py) for the Chinese ones. -Advanced scripts with different models are available in the [mteb/mtebscripts repo](https://github.com/embeddings-benchmark/mtebscripts). - -2. **Format the results using the CLI:** +1. **Format the results using the CLI:** ```bash mteb create_meta --results_folder results/{model_name}/{model_revision} --output_path model_card.md @@ -44,12 +41,28 @@ If readme of model exists: mteb create_meta --results_folder results/{model_name}/{model_revision} --output_path model_card.md --from_existing your_existing_readme.md ``` -3. **Add the frontmatter to model repository:** +2. **Add the frontmatter to model repository:** Copy the content of the `model_card.md` file to the top of a `README.md` file of your model on the Hub. See [here](https://huggingface.co/Muennighoff/SGPT-5.8B-weightedmean-msmarco-specb-bitfit/blob/main/README.md) for an example. -4. **Wait for a refresh the leaderboard:** +3. **Wait for a refresh the leaderboard:** + +The leaderboard [automatically refreshes daily](https://github.com/embeddings-benchmark/leaderboard/commits/main/) so once submitted you only need to wait for the automatic refresh. You can find the workflows for the leaderboard refresh [here](https://github.com/embeddings-benchmark/leaderboard/tree/main/.github/workflows). If you experience issues with the leaderboard please create an [issue](https://github.com/embeddings-benchmark/mteb/issues). + +**Notes:** +- We remove models with scores that cannot be reproduced, so please ensure that your model is accessible and scores can be reproduced. +- An alternative way of submitting to the leaderboard is by opening a PR with your results [here](https://github.com/embeddings-benchmark/results) & checking that they are displayed correctly by [locally running the leaderboard](https://github.com/embeddings-benchmark/leaderboard?tab=readme-ov-file#developer-setup) + +- ##### Using Prompts with Sentence Transformers + + If your model uses Sentence Transformers and requires different prompts for encoding the queries and corpus, you can take advantage of the `prompts` [parameter](https://sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer). + + Internally, `mteb` uses the prompt named `query` for encoding the queries and `passage` as the prompt name for encoding the corpus. This is aligned with the default names used by Sentence Transformers. + + ###### Adding the prompts in the model configuration (Preferred) + + You can directly add the prompts when saving and uploading your model to the Hub. For an example, refer to this [configuration file](https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5/blob/3b5a16eaf17e47bd997da998988dce5877a57092/config_sentence_transformers.json). -The leaderboard will then automatically refresh daily so once submitted all you have to do is wait for the automatic refresh. + ###### Instantiating the Model with Prompts -You can find the workflows for the leaderboard refresh [here](https://github.com/embeddings-benchmark/leaderboard/tree/main/.github/workflows). If you experience issues with the leaderboard please create an [issue](https://github.com/embeddings-benchmark/mteb/issues). + If you are unable to directly add the prompts in the model configuration, you can instantiate the model using the `sentence_transformers_loader` and pass `prompts` as an argument. For more details, see the `mteb/models/bge_models.py` file. \ No newline at end of file diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000000..a5abe50215 --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,22 @@ +## Available benchmarks +The following table gives you an overview of the benchmarks in MTEB. + +
+ + + +| Name | # Tasks | Task Types | Domains | Languages | +|------|---------|------------|---------|-----------| +| [CoIR](https://github.com/CoIR-team/coir) | 10 | {'Retrieval': 10} | [Written, Programming] | python,c++,sql,go,eng,php,javascript,ruby,java | +| [MINERSBitextMining](https://arxiv.org/pdf/2406.07424) | 7 | {'BitextMining': 7} | [Written, Social, Reviews] | sun,kaz,tzl,ido,abs,arq,yue,tam,nij,glg,slk,hsb,ber,xho,cbk,pol,uzb,ina,kab,swh,amh,fao,kzj,lfn,uig,sqi,deu,ang,ind,bug,pms,ibo,cym,eus,spa,ceb,tgl,ron,isl,ita,csb,cha,fin,est,pes,jpn,tel,tha,oci,cmn,min,fry,bbc,epo,lit,rus,bos,hrv,war,ara,bjn,mkd,srp,ast,nno,urd,pam,aze,eng,ace,bew,kor,dan,awa,mui,hye,ban,cor,ben,gle,swe,mad,bul,lat,cat,nob,fra,pcm,ell,mar,vie,tat,ukr,gsw,kat,arz,dsb,lvs,nld,tur,bel,max,nds,afr,khm,dtp,yor,ces,gla,zsm,mak,ile,nov,orv,bre,swg,rej,mhr,mon,mal,jav,heb,slv,bhp,kur,wuu,tuk,por,hun,hin,hau,yid | +| [MTEB(Retrieval w/Instructions)](https://arxiv.org/abs/2403.15246) | 3 | {'InstructionRetrieval': 3} | [Written, News] | eng | +| [MTEB(Scandinavian)](https://kennethenevoldsen.github.io/scandinavian-embedding-benchmark/) | 28 | {'BitextMining': 2, 'Classification': 13, 'Retrieval': 7, 'Clustering': 6} | [Encyclopaedic, Spoken, Non-fiction, Government, News, Fiction, Social, Blog, Reviews, Written, Web, Legal] | nob,fao,swe,isl,dan,nno | +| MTEB(code) | 12 | {'Retrieval': 12} | [Written, Programming] | python,c++,sql,c,go,eng,shell,typescript,php,scala,rust,swift,javascript,ruby,java | +| [MTEB(deu)](https://arxiv.org/html/2401.02709v1) | 19 | {'Classification': 6, 'Clustering': 4, 'PairClassification': 2, 'Reranking': 1, 'Retrieval': 4, 'STS': 2} | [Encyclopaedic, Spoken, News, Reviews, Written, Web] | eng,deu,pol,fra | +| MTEB(eng) | 67 | {'Classification': 12, 'Retrieval': 26, 'Clustering': 11, 'Reranking': 4, 'STS': 10, 'PairClassification': 3, 'Summarization': 1} | [Encyclopaedic, Spoken, Non-fiction, Blog, News, Medical, Social, Programming, Written, Reviews, Web, Academic] | tur,fra,eng,cmn,pol,ita,nld,spa,deu,ara | +| [MTEB(fra)](https://arxiv.org/abs/2405.20468) | 26 | {'Classification': 6, 'Clustering': 7, 'PairClassification': 2, 'Reranking': 2, 'Retrieval': 5, 'STS': 3, 'Summarization': 1} | [Encyclopaedic, Spoken, Non-fiction, News, Social, Reviews, Written, Web, Legal, Academic] | eng,deu,pol,fra | +| MTEB(kor) | 6 | {'Classification': 1, 'Reranking': 1, 'Retrieval': 2, 'STS': 2} | [Encyclopaedic, Spoken, News, Reviews, Written, Web] | kor | +| [MTEB(law)](https://aclanthology.org/2023.eacl-main.148/) | 8 | {'Retrieval': 8} | [Written, Legal] | eng,deu,zho | +| [MTEB(pol)](https://arxiv.org/abs/2405.10138) | 18 | {'Classification': 7, 'Clustering': 3, 'PairClassification': 4, 'STS': 4} | [Spoken, Non-fiction, News, Fiction, Social, Written, Web, Legal, Academic] | pol,deu,eng,fra | +| [MTEB(rus)](https://aclanthology.org/2023.eacl-main.148/) | 23 | {'Classification': 9, 'Clustering': 3, 'MultilabelClassification': 2, 'PairClassification': 1, 'Reranking': 2, 'Retrieval': 3, 'STS': 3} | [Encyclopaedic, Spoken, Blog, News, Social, Reviews, Written, Web, Academic] | rus | + \ No newline at end of file diff --git a/docs/create_benchmarks_table.py b/docs/create_benchmarks_table.py new file mode 100644 index 0000000000..7fddf07c75 --- /dev/null +++ b/docs/create_benchmarks_table.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from collections import Counter +from pathlib import Path + +import mteb +from docs.create_tasks_table import insert_tables + + +def benchmark_to_markdown_row(b: mteb.Benchmark) -> str: + name = b.name + name_w_reference = f"[{name}]({b.reference})" if b.reference else name + num_tasks = len(b.tasks) + n_tasks = f"{num_tasks}" + + agg_domains = set() + agg_langs = set() + for t in b.tasks: + if t.metadata.domains: + agg_domains.update(t.metadata.domains) + if t.metadata.languages: + agg_langs.update(t.languages) + + langs = ",".join(list(agg_langs)) + domains = "[" + ", ".join(agg_domains) + "]" if agg_domains else "" + + task_types = dict(Counter([t.metadata.type for t in b.tasks])) + + return f"| {name_w_reference} | {n_tasks} | {task_types} | {domains} | {langs} |" + + +def create_benchmarks_table(benchmarks: list[mteb.Benchmark]) -> str: + table = """ +| Name | # Tasks | Task Types | Domains | Languages | +|------|---------|------------|---------|-----------| +""" + for benchmark in benchmarks: + table += benchmark_to_markdown_row(benchmark) + "\n" + return table + + +def main(): + benchmarks = mteb.get_benchmarks() + benchmarks = sorted(benchmarks, key=lambda x: x.name) + + benchmarks_table = create_benchmarks_table(benchmarks) + + file_path = Path(__file__).parent / "benchmarks.md" + + insert_tables( + file_path, + tables=[benchmarks_table], + tags=["BENCHMARKS TABLE"], + ) + + +if __name__ == "__main__": + main() diff --git a/docs/mmteb/final_author_list.md b/docs/mmteb/final_author_list.md new file mode 100644 index 0000000000..b3aeae7457 --- /dev/null +++ b/docs/mmteb/final_author_list.md @@ -0,0 +1,82 @@ +Kenneth Enevoldsen ~Kenneth_Enevoldsen1 +Isaac Chung ~Isaac_Kwan_Yin_Chung1 +Imene Kerboua ~Imene_Kerboua1 +Márton Kardos ~Márton_Kardos1 +Ashwin Mathur ~Ashwin_Mathur1 +David Stap ~David_Stap +Jay Gala ~Jay_Gala1 +Wissam Siblini ~Wissam_Siblini1 +Dominik Krzemiński ~Dominik_Krzemiński1 +Genta Indra Winata ~Genta_Indra_Winata1 +Saba Sturua ~Saba_Sturua1 +Saiteja Utpala ~Saiteja_Utpala1 +Mathieu Ciancone ~Mathieu_Ciancone1 +Marion Schaeffer ~Marion_Schaeffer1 +Diganta Misra ~Diganta_Misra1 +Shreeya Dhakal (added via. search) +Jonathan Rystrøm ~Jonathan_Rystrøm1 +Roman Solomatin ~Roman_Solomatin1 +Ömer Çağatan ~Ömer_Veysel_Çağatan1 +Akash Kundu ~Akash_Kundu2 +Martin Bernstorff ~Martin_Bernstorff1 +Shitao Xiao ~Shitao_Xiao1 +Akshita Sukhlecha ~Akshita_Sukhlecha1 +Bhavish Pahwa ~Bhavish_Pahwa1 +Rafał Poświata ~Rafał_Poświata1 +Kranthi Kiran GV ~Kranthi_Kiran_GV1 +Shawon Ashraf ~Shawon_Ashraf1 +Daniel Auras ~Daniel_Auras1 +Björn Plüster ~Björn_Plüster1 +Jan Philipp Harries ~Jan_Philipp_Harries1 +Loic Magne Individual Contributor +Isabelle Mohr ~Isabelle_Mohr1 +Dawei Zhu ~Dawei_Zhu2 +Hippolyte Gisserot-Boukhlef ~Hippolyte_Gisserot-Boukhlef1 +Tom Aarsen ~Tom_Aarsen1 +Jan Kostkan ~Jan_Kostkan1 +Konrad Wojtasik Wrocław University of Science and Technology +Taemin Lee ~Taemin_Lee1 +Marek Suppa ~Marek_Suppa1 +Xinyu Zhang ~Crystina_Zhang1 +Roberta Rocca ~Roberta_Rocca1 +Mohammed Hamdy ~Mohammed_Hamdy1 +Andrianos Michail ~Andrianos_Michail1 +John Yang ~John_Yang3 +Manuel Faysse ~Manuel_Faysse1 +Aleksei Vatolin ~Aleksei_Vatolin1 +Nandan Thakur ~Nandan_Thakur1 +Manan Dey ~Manan_Dey2 +Dipam Vasani ~Dipam_Vasani1 +Pranjal Chitale ~Pranjal_A_Chitale1 +Simone Tedeschi ~Simone_Tedeschi1 +Nguyen Tai ~Nguyen_Tai1 +Artem Snegirev ~Artem_Snegirev1 +Mariya Hendriksen ~Mariya_Hendriksen1 +Michael Günther ~Michael_Günther1 +Mengzhou Xia ~Mengzhou_Xia1 +Weijia Shi ~Weijia_Shi1 +Xing Han Lù ~Xing_Han_Lù1 +Jordan Clive ~Jordan_Clive1 +Gayatri K ~Gayatri_K1 +Anna Maksimova ~Anna_Maksimova1 +Silvan Wehrli ~Silvan_Wehrli1 +Maria Tikhonova ~Maria_Tikhonova1 +Henil Panchal ~Henil_Shalin_Panchal1 +Aleksandr Abramov ~Aleksandr_Abramov1 +Malte Ostendorff ~Malte_Ostendorff1 +Zheng Liu ~Zheng_Liu4 +Simon Clematide ~Simon_Clematide1 +Lester James Miranda ~Lester_James_Validad_Miranda1 +Alena Fenogenova ~Alena_Fenogenova1 +Guangyu Song ~Guangyu_Song1 +Ruqiya Bin Safi ~Ruqiya_Bin_Safi1 +Wen-Ding Li ~Wen-Ding_Li1 +Alessia Borghini ~Alessia_Borghini1 +Federico Cassano ~Federico_Cassano1 +Lasse Hansen ~Lasse_Hansen2 +Sara Hooker ~Sara_Hooker2 +Chenghao Xiao ~Chenghao_Xiao1 +Vaibhav Adlakha ~Vaibhav_Adlakha1 +Orion Weller ~Orion_Weller1 +Siva Reddy ~Siva_Reddy1 +Niklas Muennighoff ~Niklas_Muennighoff1 diff --git a/docs/mmteb/points.md b/docs/mmteb/points.md index 12f404699b..75fad31051 100644 --- a/docs/mmteb/points.md +++ b/docs/mmteb/points.md @@ -37,52 +37,51 @@ Please also add your first name and last name are as you want them to appear in | GitHub | First name | Last name | Email | User on openreview | Affiliations | | ----------------- | ---------- | ---------- | ---------------------------- | -------------------- | ----------------------------------------------------- | -| KennethEnevoldsen | Kenneth | Enevoldsen | kennethcenevoldsen@gmail.com | ~Kenneth_Enevoldsen1 | Aarhus University, Denmark | -| x-tabdeveloping | Márton | Kardos | martonkardos@cas.au.dk | ~Márton_Kardos1 | Aarhus University, Denmark | -| imenelydiaker | Imene | Kerboua | | | Esker, Lyon, France && INSA Lyon, LIRIS, Lyon, France | -| wissam-sib | Wissam | Siblini | wissam.siblini92@gmail.com | ~Wissam_Siblini1 | N/A | -| GabrielSequeira | Gabriel | Sequeira | | | N/A | -| schmarion | Marion | Schaeffer | | ~Marion_Schaeffer1 | Wikit, Lyon, France | -| MathieuCiancone | Mathieu | Ciancone | | | Wikit, Lyon, France | -| MartinBernstorff | Martin | Bernstorff | martinbernstorff@gmail.com | ~Martin_Bernstorff1 | Aarhus University, Denmark | +| KennethEnevoldsen | Kenneth | Enevoldsen | kennethcenevoldsen@gmail.com | ~Kenneth_Enevoldsen1 | Aarhus University | +| x-tabdeveloping | Márton | Kardos | martonkardos@cas.au.dk | ~Márton_Kardos1 | Aarhus University | +| imenelydiaker | Imene | Kerboua | | ~Imene_Kerboua1 | INSA Lyon, LIRIS | +| wissam-sib | Wissam | Siblini | wissam.siblini92@gmail.com | ~Wissam_Siblini1 | Individual Contributor | +| GabrielSequeira | Gabriel | Sequeira | | | Individual Contributor | +| schmarion | Marion | Schaeffer | | ~Marion_Schaeffer1 | Wikit | +| MathieuCiancone | Mathieu | Ciancone | | ~Mathieu_Ciancone1 | Wikit | +| MartinBernstorff | Martin | Bernstorff | martinbernstorff@gmail.com | ~Martin_Bernstorff1 | Aarhus University | | staoxiao | Shitao | Xiao | 2906698981@qq.com | ~Shitao_Xiao1 | Beijing Academy of Artificial Intelligence | | ZhengLiu101 | Zheng | Liu | zhengliu1026@gmail.com | ~Zheng_Liu4 | Beijing Academy of Artificial Intelligence | -| achibb | Aaron | Chibb | | | N/A | -| cassanof | Federico | Cassano | federico.cassanno@federico.codes | ~Federico_Cassano1 | Northeastern University, Boston, USA | +| achibb | Aaron | Chibb | ~Aaron_Chibb1 | | Individual Contributor | +| cassanof | Federico | Cassano | federico.cassanno@federico.codes | ~Federico_Cassano1 | Northeastern University && Cursor AI | | taidnguyen | Nguyen | Tai | taing@seas.upenn.edu | ~Nguyen_Tai1 | University of Pennsylvania | | xu3kev | Wen-Ding | Li | wl678@cornell.edu | ~Wen-Ding_Li1 | Cornell University | -| Rysias | Jonathan | Rystrøm | jonathan.rystroem@gmail.com | | University of Oxford, UK | +| Rysias | Jonathan | Rystrøm | jonathan.rystroem@gmail.com | ~Jonathan_Rystrøm1 | University of Oxford | | taeminlee | Taemin | Lee | taeminlee@korea.ac.kr | ~Taemin_Lee1 | Korea University Human-Inspired AI Research | -| izhx | Xin | Zhang | zhangxin2023@stu.hit.edu.cn | | Harbin Institute of Technology, Shenzhen | +| izhx | Xin | Zhang | zhangxin2023@stu.hit.edu.cn | ~Xin_Zhang15 | Harbin Institute of Technology | | orionw | Orion | Weller | oweller@cs.jhu.edu | ~Orion_Weller1 | Johns Hopkins University | -| slvnwhrl | Silvan | Wehrli | wehrlis@rki.de | ~Silvan_Wehrli1 | Robert Koch Institute, Berlin, Germany | -| manandey | Manan | Dey | manandey1@gmail.com | ~Manan_Dey2 | Salesforce, India | -| isaac-chung | Isaac | Chung | chungisaac1217@gmail.com | ~Isaac_Kwan_Yin_Chung1 | N/A | +| slvnwhrl | Silvan | Wehrli | wehrlis@rki.de | ~Silvan_Wehrli1 | Robert Koch Institute | +| manandey | Manan | Dey | manandey1@gmail.com | ~Manan_Dey2 | Salesforce | +| isaac-chung | Isaac | Chung | chungisaac1217@gmail.com | ~Isaac_Kwan_Yin_Chung1 | Individual Contributor | | asparius | Ömer | Çağatan | ocagatan19@ku.edu.tr | ~Ömer_Veysel_Çağatan1 | Koç University,Turkey | -| rafalposwiata | Rafał | Poświata | rposwiata@opi.org.pl | ~Rafał_Poświata1 | National Information Processing Institute, Warsaw, Poland | -| rbroc | Roberta | Rocca | roberta.rocca@cas.au.dk | ~Roberta_Rocca1 | Aarhus University, Denmark | -| awinml | Ashwin | Mathur | ashwinxmathur@gmail.com | | N/A | +| rafalposwiata | Rafał | Poświata | rposwiata@opi.org.pl | ~Rafał_Poświata1 | National Information Processing Institute | +| rbroc | Roberta | Rocca | roberta.rocca@cas.au.dk | ~Roberta_Rocca1 | Aarhus University | +| awinml | Ashwin | Mathur | ashwinxmathur@gmail.com | ~Ashwin_Mathur1 | Individual Contributor | | guangyusong | Guangyu | Song | guangysong@gmail.com | ~Guangyu_Song1 | Tano Labs | -| davidstap | David | Stap | dd.stap@gmail.com | ~David_Stap | University of Amsterdam. | -| HLasse | Lasse | Hansen | lasseh0310@gmail.com | ~Lasse_Hansen2 | Aarhus University, Denmark | +| davidstap | David | Stap | dd.stap@gmail.com | ~David_Stap | University of Amsterdam | +| HLasse | Lasse | Hansen | lasseh0310@gmail.com | ~Lasse_Hansen2 | Aarhus University | | jaygala24 | Jay | Gala | jaygala24@gmail.com | ~Jay_Gala1 | MBZUAI | -| digantamisra98 | Diganta | Misra | diganta.misra@mila.quebec | ~Diganta_Misra1 | Mila - Quebec AI Institute | -| PranjalChitale | Pranjal | Chitale | cs21s022@smail.iitm.ac.in | ~Pranjal_A_Chitale1 | Indian Institute of Technology Madras | -| Akash190104 | Akash | Kundu | akashkundu2xx4@gmail.com |~Akash_Kundu2 | Heritage Institute of Technology, Kolkata && Apart Research | +| digantamisra98 | Diganta | Misra | diganta.misra@mila.quebec | ~Diganta_Misra1 | Max Planck Institute for Intelligent Systems && ELLIS Institute Tübingen | +| PranjalChitale | Pranjal | Chitale | cs21s022@smail.iitm.ac.in | ~Pranjal_A_Chitale1 | Indian Institute of Technology | +| Akash190104 | Akash | Kundu | akashkundu2xx4@gmail.com |~Akash_Kundu2 | Heritage Institute of Technology && Apart Research | | dwzhu-pku | Dawei | Zhu | dwzhu@pku.edu.cn | ~Dawei_Zhu2 | Peking University | | ljvmiranda921 | Lester James | Miranda | ljm@allenai.org | ~Lester_James_Validad_Miranda1 | Allen Institute for AI | -| Sakshamrzt | Saksham | Thakur | sthakur5@alumni.ncsu.edu | ~Saksham_Thakur1 | N/A | | Andrian0s | Andrianos | Michail | andrianos.michail@cl.uzh.ch | ~Andrianos_Michail1 | University of Zurich| | simon-clematide | Simon | Clematide | simon.clematide@cl.uzh.ch | ~Simon_Clematide1 | University of Zurich| | SaitejaUtpala | Saiteja | Utpala | saitejautpala@gmail.com | ~Saiteja_Utpala1 | Microsoft Research| | mmhamdy | Mohammed | Hamdy | mhamdy.res@gmail.com | ~Mohammed_Hamdy1 | Cohere For AI Community| | jupyterjazz | Saba | Sturua | saba.sturua@jina.ai | ~Saba_Sturua1 | Jina AI | -| Ruqyai | Ruqiya | Bin Safi | ruqiya.binsafi@libfstudy.ac.uk | ~Ruqiya_Bin_Safi1 | LIBF : The London Institute of Banking and Finance -| kranthigv | Kranthi Kiran | GV | kranthi.gv@nyu.edu | ~Kranthi_Kiran_GV1 | New York University| +| Ruqyai | Ruqiya | Bin Safi | ruqiya.binsafi@libfstudy.ac.uk | ~Ruqiya_Bin_Safi1 | The London Institute of Banking and Finance +| KranthiGV | Kranthi Kiran | GV | kranthi.gv@nyu.edu | ~Kranthi_Kiran_GV1 | New York University| | shreeya-dhakal | Shreeya | Dhakal | ssdhakal57@gmail.com | | Individual Contributor | | dipam7 | Dipam | Vasani | dipam44@gmail.com | ~Dipam_Vasani1 | Individual Contributor | -| Art3mis07 | Gayatri | K | gayatrikrishnakumar0707@gmail.com | ~Gayatri_K1 | R. V. College of Engineering, Bengaluru | -| jankounchained | Jan | Kostkan | jan.kostkan@cas.au.dk | ~Jan_Kostkan1 | Aarhus University, Denmark | +| Art3mis07 | Gayatri | K | gayatrikrishnakumar0707@gmail.com | ~Gayatri_K1 | R. V. College of Engineering | +| jankounchained | Jan | Kostkan | jan.kostkan@cas.au.dk | ~Jan_Kostkan1 | Aarhus University | | bp-high | Bhavish | Pahwa | t-bpahwa@microsoft.com | ~Bhavish_Pahwa1 | Microsoft Research | | rasdani | Daniel | Auras | daniel@ellamind.com | ~Daniel_Auras1 | ellamind, Germany | | ShawonAshraf | Shawon | Ashraf | shawon@ellamind.com | ~Shawon_Ashraf1 | ellamind, Germany | @@ -92,28 +91,37 @@ Please also add your first name and last name are as you want them to appear in | ManuelFay | Manuel | Faysse | manuel.faysse@centralesupelec.fr | ~Manuel_Faysse1 | CentraleSupélec && Illuin Technology | | hgissbkh | Hippolyte | Gisserot-Boukhlef | hippolyte.gisserot-boukhlef@centralesupelec.fr | ~Hippolyte_Gisserot-Boukhlef1 | CentraleSupélec && Artefact Research Center | | sted97 | Simone | Tedeschi | tedeschi@diag.uniroma1.it | ~Simone_Tedeschi1 | Sapienza University of Rome | -| gentaiscool | Genta Indra | Winata | gentaindrawinata@gmail.com | ~Genta_Indra_Winata1 | N/A | +| gentaiscool | Genta Indra | Winata | gentaindrawinata@gmail.com | ~Genta_Indra_Winata1 | Individual Contributor | | henilp105 | Henil | Panchal | henilp105@gmail.com | ~Henil_Shalin_Panchal1 | Nirma University | | ABorghini | Alessia | Borghini | borghini.alessia99@gmail.com | ~Alessia_Borghini1 | Sapienza University of Rome | | jordiclive | Jordan | Clive | jordan.clive19@imperial.ac.uk | ~Jordan_Clive1 | Imperial College London | | gowitheflow-1998 | Chenghao | Xiao | chenghao.xiao@durham.ac.uk | ~Chenghao_Xiao1 | Durham University | | mariyahendriksen | Mariya | Hendriksen | mariya.hendriksen@gmail.com | ~Mariya_Hendriksen1 | University of Amsterdam | | dokato | Dominik | Krzemiński | dkk33@cantab.ac.uk | ~Dominik_Krzemiński1 | Cohere For AI Community | -| Samoed | Roman | Solomatin | risolomatin@gmail.com | ~Roman_Solomatin1 | ITMO | -| Alenush | Alena | Fenogenova | alenush93@gmail.com | ~Alena_Fenogenova1 | SaluteDevices, Russia | -| ab1992ao | Aleksandr | Abramov | andril772@gmail.com | ~Aleksandr_Abramov1 | SaluteDevices, Russia | -| artemsnegirev | Artem | Snegirev | artem.s.snegirev@gmail.com | ~Artem_Snegirev1 | SaluteDevices, Russia | -| anpalmak2003 | Anna | Maksimova | anpalmak@gmail.com | ~Anna_Maksimova1 | SaluteDevices, Russia | -| MariyaTikhonova | Maria | Tikhonova | m_tikhonova94@mail.ru | ~Maria_Tikhonova1 | SaluteDevices, HSE University, Russia | -| vaibhavad | Vaibhav | Adlakha | vaibhav.adlakha@mila.quebec | ~Vaibhav_Adlakha1 | McGill University && Mila - Quebec AI Institute && ServiceNow Research | -| sivareddyg | Siva | Reddy | siva.reddy@mila.quebec | ~Siva_Reddy1 | McGill University && Mila - Quebec AI Institute && ServiceNow Research && Facebook CIFAR AI Chair | +| Samoed | Roman | Solomatin | risolomatin@gmail.com | ~Roman_Solomatin1 | AI Talent Hub && ITMO University | +| Alenush | Alena | Fenogenova | alenush93@gmail.com | ~Alena_Fenogenova1 | SaluteDevices | +| ab1992ao | Aleksandr | Abramov | andril772@gmail.com | ~Aleksandr_Abramov1 | SaluteDevices | +| artemsnegirev | Artem | Snegirev | artem.s.snegirev@gmail.com | ~Artem_Snegirev1 | SaluteDevices | +| anpalmak2003 | Anna | Maksimova | anpalmak@gmail.com | ~Anna_Maksimova1 | SaluteDevices | +| MariyaTikhonova | Maria | Tikhonova | m_tikhonova94@mail.ru | ~Maria_Tikhonova1 | SaluteDevices && HSE University | +| vaibhavad | Vaibhav | Adlakha | vaibhav.adlakha@mila.quebec | ~Vaibhav_Adlakha1 | Mila, McGill University && ServiceNow Research | +| sivareddyg | Siva | Reddy | siva.reddy@mila.quebec | ~Siva_Reddy1 | Mila, McGill University && ServiceNow Research | | guenthermi | Michael | Günther | michael.guenther@jina.ai | ~Michael_Günther1 | Jina AI | | violenil | Isabelle | Mohr | isabelle.mohr@jina.ai | ~Isabelle_Mohr1 | Jina AI | -| akshita-sukhlecha | Akshita | Sukhlecha | sukhlecha.akshita@gmail.com | | N/A | -| Muennighoff | Niklas | Muennighoff | n.muennighoff@gmail.com | | Contextual AI | +| akshita-sukhlecha | Akshita | Sukhlecha | sukhlecha.akshita@gmail.com | ~Akshita_Sukhlecha1 | Individual Contributor | +| Muennighoff | Niklas | Muennighoff | n.muennighoff@gmail.com | ~Niklas_Muennighoff1 | Stanford University && Contextual AI | | AlexeyVatolin | Aleksei | Vatolin | vatolinalex@gmail.com | ~Aleksei_Vatolin1 | FRC CSC RAS | -| xhluca | Xing Han | Lù | xing.han.lu@mail.mcgill.ca | ~Xing_Han_Lù1 | McGill University && Mila - Quebec AI Institute | +| xhluca | Xing Han | Lù | xing.han.lu@mail.mcgill.ca | ~Xing_Han_Lù1 | Mila, McGill University | | crystina-z | Xinyu | Zhang | xinyucrystina.zhang@uwaterloo.ca | ~Crystina_Zhang1 | University of Waterloo | | tomaarsen | Tom | Aarsen | | ~Tom_Aarsen1 | Hugging Face | -| crystina-z | Xinyu | Zhang | xinyucrystina.zhang@uwaterloo.ca | ~Crystina_Zhang1 | University of Waterloo | -| mrshu | Marek | Suppa | marek.suppa@fmph.uniba.sk | ~Marek_Suppa1 | Comenius University in Bratislava && Cisco Systems | +| mrshu | Marek | Suppa | marek.suppa@fmph.uniba.sk | ~Marek_Suppa1 | Comenius University Bratislava && Cisco Systems | +| swj0419 | Weijia | Shi | swj0419@uw.edu | ~Weijia_Shi1 | University of Washington | +| xiamengzhou | Mengzhou | Xia | mengzhou@princeton.edu | ~Mengzhou_Xia1 | Princeton University | +| john-b-yang | John | Yang | johnby@stanford.edu | ~John_Yang3 | Stanford University | +| thakur-nandan | Nandan | Thakur | | ~Nandan_Thakur1 | University of Waterloo | +| loicmagne | Loic | Magne | | ~Loïc_Magne1 |Individual Contributor | +| sarahooker | Sara | Hooker | | ~Sara_Hooker2 | Cohere For AI | +| kwojtasi | Konrad | Wojtasik | | ~Konrad_Wojtasik1 | Wrocław University of Science and Technology | +| tmp_handle | Jimmy | Lin | | ~Jimmy_Lin2 | University of Waterloo | +| hongjin-su | Hongjin | Su | | ~Hongjin_SU1 | University of Hong Kong | +| howard-yen | Howard | Yen | | ~Howard_Yen1 | Princeton University | diff --git a/docs/mmteb/points/1004.jsonl b/docs/mmteb/points/1004.jsonl new file mode 100644 index 0000000000..1e80779272 --- /dev/null +++ b/docs/mmteb/points/1004.jsonl @@ -0,0 +1 @@ +{"GitHub": "mariyahendriksen", "Paper writing": 6} diff --git a/docs/mmteb/points/1006.jsonl b/docs/mmteb/points/1006.jsonl new file mode 100644 index 0000000000..1e80779272 --- /dev/null +++ b/docs/mmteb/points/1006.jsonl @@ -0,0 +1 @@ +{"GitHub": "mariyahendriksen", "Paper writing": 6} diff --git a/docs/mmteb/points/121.jsonl b/docs/mmteb/points/121.jsonl new file mode 100644 index 0000000000..edaddb4009 --- /dev/null +++ b/docs/mmteb/points/121.jsonl @@ -0,0 +1 @@ +{"GitHub": "kwojtasi", "New dataset": 22} \ No newline at end of file diff --git a/docs/mmteb/points/1236.jsonl b/docs/mmteb/points/1236.jsonl new file mode 100644 index 0000000000..8e477bc4fe --- /dev/null +++ b/docs/mmteb/points/1236.jsonl @@ -0,0 +1,3 @@ +{"GitHub": "orionw", "Coordination": 25} +{"GitHub": "KennethEnevoldsen", "Review PR": 2, "Bug fixes": 2} +{"GitHub": "vaibhavad", "Coordination": 25} \ No newline at end of file diff --git a/docs/mmteb/points/1237.jsonl b/docs/mmteb/points/1237.jsonl new file mode 100644 index 0000000000..ef73ff16d5 --- /dev/null +++ b/docs/mmteb/points/1237.jsonl @@ -0,0 +1 @@ +{"GitHub": "john-b-yang", "Paper writing": 20} \ No newline at end of file diff --git a/docs/mmteb/points/1252.jsonl b/docs/mmteb/points/1252.jsonl new file mode 100644 index 0000000000..3eefa84e1b --- /dev/null +++ b/docs/mmteb/points/1252.jsonl @@ -0,0 +1 @@ +{"GitHub": "tmp_handle", "Coordination": 10} \ No newline at end of file diff --git a/docs/mmteb/points/676.jsonl b/docs/mmteb/points/676.jsonl index a714dbfa1f..b8077d25a1 100644 --- a/docs/mmteb/points/676.jsonl +++ b/docs/mmteb/points/676.jsonl @@ -1,3 +1,3 @@ {"GitHub": "wissam-sib", "New dataset": 6} -{"GitHub": "kranthigv", "Review PR": 2} -{"GitHub": "KennethEnevoldsen", "Review PR": 2} \ No newline at end of file +{"GitHub": "KranthiGV", "Review PR": 2} +{"GitHub": "KennethEnevoldsen", "Review PR": 2} diff --git a/docs/mmteb/points/690.jsonl b/docs/mmteb/points/690.jsonl index e2b6194c27..68e60e0c84 100644 --- a/docs/mmteb/points/690.jsonl +++ b/docs/mmteb/points/690.jsonl @@ -1,3 +1,3 @@ -{"GitHub": "kranthigv", "New dataset": 20} +{"GitHub": "KranthiGV", "New dataset": 20} {"GitHub": "awinml", "Review PR": 2} -{"GitHub": "KennethEnevoldsen", "Review PR": 2} \ No newline at end of file +{"GitHub": "KennethEnevoldsen", "Review PR": 2} diff --git a/docs/mmteb/points/724.jsonl b/docs/mmteb/points/724.jsonl index c2d55561f4..29706ede79 100644 --- a/docs/mmteb/points/724.jsonl +++ b/docs/mmteb/points/724.jsonl @@ -1,3 +1,3 @@ {"GitHub": "Art3mis07", "New dataset": 12} {"GitHub": "imenelydiaker", "Review PR": 2} -{"GitHub": "kranthigv", "Review PR": 2} \ No newline at end of file +{"GitHub": "KranthiGV", "Review PR": 2} diff --git a/docs/mmteb/points/736.jsonl b/docs/mmteb/points/736.jsonl index 2a95159593..8dc6c9f612 100644 --- a/docs/mmteb/points/736.jsonl +++ b/docs/mmteb/points/736.jsonl @@ -1,3 +1,3 @@ {"GitHub": "wissam-sib", "New dataset": 6} {"GitHub": "isaac-chung", "Review PR": 2} -{"GitHub": "kranthigv", "Review PR": 2} \ No newline at end of file +{"GitHub": "KranthiGV", "Review PR": 2} diff --git a/docs/mmteb/points/971.jsonl b/docs/mmteb/points/971.jsonl new file mode 100644 index 0000000000..24ce9bdbd8 --- /dev/null +++ b/docs/mmteb/points/971.jsonl @@ -0,0 +1,6 @@ +{"GitHub": "swj0419", "New dataset": 10} +{"GitHub": "xiamengzhou", "New dataset": 10} +{"GitHub": "hongjin-su", "New dataset": 10} +{"GitHub": "howard-yen", "New dataset": 10} +{"GitHub": "KennethEnevoldsen", "Review PR": 2} +{"GitHub": "Muennighoff", "Review PR": 2} diff --git a/docs/mmteb/points_table.md b/docs/mmteb/points_table.md index f15ccbb27a..44ee346804 100644 --- a/docs/mmteb/points_table.md +++ b/docs/mmteb/points_table.md @@ -2,97 +2,103 @@ _Note_: this table is **autogenerated** and should not be edited. It is intended to get an overview of contributions. - | GitHub | Bug fixes | Review PR | New dataset | Paper writing | Dataset annotations | Coordination | Running Models | New task | Total | -|:------------------|------------:|------------:|--------------:|----------------:|----------------------:|---------------:|-----------------:|-----------:|--------:| -| KennethEnevoldsen | 85 | 322 | 68 | 0 | 35 | 81 | 0 | 0 | 591 | -| isaac-chung | 50 | 194 | 120 | 12 | 1 | 54 | 0 | 2 | 433 | -| imenelydiaker | 24 | 144 | 120 | 0 | 0 | 70 | 0 | 0 | 358 | -| awinml | 0 | 2 | 300 | 0 | 0 | 0 | 0 | 0 | 302 | -| x-tabdeveloping | 10 | 32 | 144 | 0 | 0 | 41 | 0 | 12 | 239 | -| davidstap | 0 | 0 | 176 | 0 | 0 | 0 | 0 | 0 | 176 | -| jaygala24 | 0 | 0 | 149 | 0 | 0 | 0 | 0 | 0 | 149 | -| wissam-sib | 4 | 6 | 134 | 0 | 0 | 0 | 0 | 0 | 144 | -| Muennighoff | 0 | 46 | 0 | 0 | 0 | 70 | 24 | 0 | 140 | -| dokato | 12 | 6 | 94 | 0 | 0 | 0 | 0 | 0 | 112 | -| gentaiscool | 0 | 0 | 110 | 0 | 0 | 0 | 0 | 0 | 110 | -| jupyterjazz | 0 | 0 | 108 | 0 | 0 | 0 | 0 | 0 | 108 | -| SaitejaUtpala | 0 | 0 | 102 | 0 | 0 | 0 | 0 | 0 | 102 | -| orionw | 20 | 20 | 0 | 0 | 0 | 50 | 0 | 10 | 100 | -| MathieuCiancone | 0 | 0 | 88 | 0 | 0 | 0 | 0 | 0 | 88 | -| schmarion | 0 | 0 | 88 | 0 | 0 | 0 | 0 | 0 | 88 | -| GabrielSequeira | 0 | 0 | 88 | 0 | 0 | 0 | 0 | 0 | 88 | -| digantamisra98 | 0 | 0 | 71 | 0 | 0 | 0 | 0 | 0 | 71 | -| vaibhavad | 8 | 4 | 6 | 0 | 0 | 50 | 0 | 0 | 68 | -| shreeya-dhakal | 0 | 8 | 54 | 0 | 0 | 0 | 0 | 0 | 62 | -| Rysias | 0 | 0 | 58 | 0 | 0 | 0 | 0 | 0 | 58 | -| Samoed | 22 | 2 | 18 | 0 | 0 | 0 | 9 | 0 | 51 | -| sivareddyg | 0 | 0 | 0 | 0 | 0 | 50 | 0 | 0 | 50 | -| gowitheflow-1998 | 0 | 0 | 50 | 0 | 0 | 0 | 0 | 0 | 50 | -| asparius | 0 | 14 | 34 | 0 | 0 | 0 | 0 | 0 | 48 | -| Akash190104 | 0 | 0 | 46 | 0 | 0 | 0 | 0 | 0 | 46 | -| MartinBernstorff | 13 | 8 | 2 | 0 | 0 | 20 | 0 | 0 | 43 | -| staoxiao | 0 | 0 | 40 | 0 | 0 | 0 | 0 | 0 | 40 | -| akshita-sukhlecha | 4 | 0 | 36 | 0 | 0 | 0 | 0 | 0 | 40 | -| rafalposwiata | 0 | 0 | 36 | 0 | 0 | 0 | 0 | 0 | 36 | -| bp-high | 0 | 0 | 36 | 0 | 0 | 0 | 0 | 0 | 36 | -| jphme | 0 | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 28 | -| rasdani | 0 | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 28 | -| loicmagne | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| bjoernpl | 0 | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 28 | -| ShawonAshraf | 0 | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 28 | -| violenil | 0 | 0 | 26 | 0 | 0 | 0 | 0 | 0 | 26 | -| kranthigv | 0 | 6 | 20 | 0 | 0 | 0 | 0 | 0 | 26 | -| dwzhu-pku | 0 | 0 | 24 | 0 | 0 | 0 | 0 | 0 | 24 | -| hgissbkh | 13 | 2 | 0 | 3 | 0 | 0 | 0 | 5 | 23 | -| tomaarsen | 0 | 2 | 0 | 0 | 0 | 20 | 0 | 0 | 22 | -| jankounchained | 8 | 0 | 14 | 0 | 0 | 0 | 0 | 0 | 22 | -| taeminlee | 0 | 0 | 22 | 0 | 0 | 0 | 0 | 0 | 22 | -| mrshu | 0 | 4 | 16 | 0 | 1 | 0 | 0 | 0 | 21 | -| crystina-z | 0 | 0 | 21 | 0 | 0 | 0 | 0 | 0 | 21 | -| rbroc | 0 | 0 | 20 | 0 | 0 | 0 | 0 | 0 | 20 | -| AlexeyVatolin | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | -| ManuelFay | 13 | 0 | 2 | 0 | 0 | 0 | 0 | 5 | 20 | -| mmhamdy | 0 | 0 | 20 | 0 | 0 | 0 | 0 | 0 | 20 | -| Andrian0s | 2 | 4 | 14 | 0 | 0 | 0 | 0 | 0 | 20 | -| manandey | 0 | 0 | 18 | 0 | 0 | 0 | 0 | 0 | 18 | -| thakur-nandan | 0 | 0 | 18 | 0 | 0 | 0 | 0 | 0 | 18 | -| PranjalChitale | 0 | 0 | 16 | 0 | 0 | 0 | 0 | 0 | 16 | -| sted97 | 0 | 0 | 16 | 0 | 0 | 0 | 0 | 0 | 16 | -| Sakshamrzt | 0 | 4 | 12 | 0 | 0 | 0 | 0 | 0 | 16 | -| dipam7 | 0 | 2 | 14 | 0 | 0 | 0 | 0 | 0 | 16 | -| taidnguyen | 0 | 0 | 14 | 0 | 0 | 0 | 0 | 0 | 14 | -| artemsnegirev | 0 | 0 | 12 | 0 | 2 | 0 | 0 | 0 | 14 | -| Art3mis07 | 0 | 0 | 12 | 0 | 0 | 0 | 0 | 0 | 12 | -| jordiclive | 10 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 12 | -| guenthermi | 0 | 0 | 12 | 0 | 0 | 0 | 0 | 0 | 12 | -| anpalmak2003 | 0 | 0 | 9 | 0 | 3 | 0 | 0 | 0 | 12 | -| mariyahendriksen | 0 | 0 | 0 | 12 | 0 | 0 | 0 | 0 | 12 | -| slvnwhrl | 0 | 0 | 12 | 0 | 0 | 0 | 0 | 0 | 12 | -| xhluca | 4 | 2 | 6 | 0 | 0 | 0 | 0 | 0 | 12 | -| henilp105 | 2 | 0 | 0 | 0 | 9 | 0 | 0 | 0 | 11 | -| MariyaTikhonova | 0 | 0 | 7 | 0 | 4 | 0 | 0 | 0 | 11 | -| ab1992ao | 0 | 0 | 8 | 0 | 3 | 0 | 0 | 0 | 11 | -| simon-clematide | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 10 | -| sarahooker | 0 | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 10 | -| ABorghini | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 10 | -| xu3kev | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 10 | -| malteos | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 10 | -| ljvmiranda921 | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 10 | -| Alenush | 0 | 0 | 6 | 0 | 4 | 0 | 0 | 0 | 10 | -| guangyusong | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 10 | -| HLasse | 5 | 0 | 0 | 0 | 5 | 0 | 0 | 0 | 10 | -| cassanof | 1 | 0 | 8 | 0 | 0 | 0 | 1 | 0 | 10 | -| ZhengLiu101 | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 10 | -| Ruqyai | 0 | 8 | 2 | 0 | 0 | 0 | 0 | 0 | 10 | -| KranthiGV | 0 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 8 | -| izhx | 0 | 0 | 6 | 0 | 0 | 0 | 0 | 0 | 6 | -| marcobellagente93 | 0 | 0 | 6 | 0 | 0 | 0 | 0 | 0 | 6 | -| NouamaneTazi | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| MexicanLemonade | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | -| monikernemo | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | -| PhilipMay | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| achibb | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | -| bakrianoo | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | -| cslizc | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | -| hanhainebula | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | -| antoniolanza1996 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | + | GitHub | Paper writing | New dataset | Review PR | Bug fixes | Coordination | Dataset annotations | New task | Running Models | Total | +|:------------------|----------------:|--------------:|------------:|------------:|---------------:|----------------------:|-----------:|-----------------:|--------:| +| KennethEnevoldsen | 0 | 68 | 326 | 87 | 81 | 35 | 0 | 0 | 597 | +| isaac-chung | 12 | 120 | 194 | 50 | 54 | 1 | 2 | 0 | 433 | +| imenelydiaker | 0 | 120 | 144 | 24 | 70 | 0 | 0 | 0 | 358 | +| awinml | 0 | 300 | 2 | 0 | 0 | 0 | 0 | 0 | 302 | +| x-tabdeveloping | 0 | 144 | 32 | 10 | 41 | 0 | 12 | 0 | 239 | +| davidstap | 0 | 176 | 0 | 0 | 0 | 0 | 0 | 0 | 176 | +| jaygala24 | 0 | 149 | 0 | 0 | 0 | 0 | 0 | 0 | 149 | +| wissam-sib | 0 | 134 | 6 | 4 | 0 | 0 | 0 | 0 | 144 | +| Muennighoff | 0 | 0 | 48 | 0 | 70 | 0 | 0 | 24 | 142 | +| orionw | 0 | 0 | 20 | 20 | 75 | 0 | 10 | 0 | 125 | +| dokato | 0 | 94 | 6 | 12 | 0 | 0 | 0 | 0 | 112 | +| gentaiscool | 0 | 110 | 0 | 0 | 0 | 0 | 0 | 0 | 110 | +| jupyterjazz | 0 | 108 | 0 | 0 | 0 | 0 | 0 | 0 | 108 | +| SaitejaUtpala | 0 | 102 | 0 | 0 | 0 | 0 | 0 | 0 | 102 | +| vaibhavad | 0 | 6 | 4 | 8 | 75 | 0 | 0 | 0 | 93 | +| schmarion | 0 | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| MathieuCiancone | 0 | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| GabrielSequeira | 0 | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| digantamisra98 | 0 | 71 | 0 | 0 | 0 | 0 | 0 | 0 | 71 | +| shreeya-dhakal | 0 | 54 | 8 | 0 | 0 | 0 | 0 | 0 | 62 | +| Rysias | 0 | 58 | 0 | 0 | 0 | 0 | 0 | 0 | 58 | +| Samoed | 0 | 18 | 2 | 22 | 0 | 0 | 0 | 9 | 51 | +| sivareddyg | 0 | 0 | 0 | 0 | 50 | 0 | 0 | 0 | 50 | +| gowitheflow-1998 | 0 | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 50 | +| asparius | 0 | 34 | 14 | 0 | 0 | 0 | 0 | 0 | 48 | +| Akash190104 | 0 | 46 | 0 | 0 | 0 | 0 | 0 | 0 | 46 | +| MartinBernstorff | 0 | 2 | 8 | 13 | 20 | 0 | 0 | 0 | 43 | +| akshita-sukhlecha | 0 | 36 | 0 | 4 | 0 | 0 | 0 | 0 | 40 | +| staoxiao | 0 | 40 | 0 | 0 | 0 | 0 | 0 | 0 | 40 | +| bp-high | 0 | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | +| rafalposwiata | 0 | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | +| KranthiGV | 0 | 20 | 14 | 0 | 0 | 0 | 0 | 0 | 34 | +| loicmagne | 0 | 0 | 0 | 28 | 0 | 0 | 0 | 0 | 28 | +| ShawonAshraf | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| bjoernpl | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| jphme | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| rasdani | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| violenil | 0 | 26 | 0 | 0 | 0 | 0 | 0 | 0 | 26 | +| mariyahendriksen | 24 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 24 | +| dwzhu-pku | 0 | 24 | 0 | 0 | 0 | 0 | 0 | 0 | 24 | +| hgissbkh | 3 | 0 | 2 | 13 | 0 | 0 | 5 | 0 | 23 | +| taeminlee | 0 | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | +| kwojtasi | 0 | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | +| jankounchained | 0 | 14 | 0 | 8 | 0 | 0 | 0 | 0 | 22 | +| tomaarsen | 0 | 0 | 2 | 0 | 20 | 0 | 0 | 0 | 22 | +| crystina-z | 0 | 21 | 0 | 0 | 0 | 0 | 0 | 0 | 21 | +| mrshu | 0 | 16 | 4 | 0 | 0 | 1 | 0 | 0 | 21 | +| john-b-yang | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | +| rbroc | 0 | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | +| mmhamdy | 0 | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | +| ManuelFay | 0 | 2 | 0 | 13 | 0 | 0 | 5 | 0 | 20 | +| AlexeyVatolin | 0 | 0 | 0 | 20 | 0 | 0 | 0 | 0 | 20 | +| Andrian0s | 0 | 14 | 4 | 2 | 0 | 0 | 0 | 0 | 20 | +| thakur-nandan | 0 | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | +| manandey | 0 | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | +| PranjalChitale | 0 | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| dipam7 | 0 | 14 | 2 | 0 | 0 | 0 | 0 | 0 | 16 | +| sted97 | 0 | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| Sakshamrzt | 0 | 12 | 4 | 0 | 0 | 0 | 0 | 0 | 16 | +| taidnguyen | 0 | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 14 | +| artemsnegirev | 0 | 12 | 0 | 0 | 0 | 2 | 0 | 0 | 14 | +| slvnwhrl | 0 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| anpalmak2003 | 0 | 9 | 0 | 0 | 0 | 3 | 0 | 0 | 12 | +| Art3mis07 | 0 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| guenthermi | 0 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| jordiclive | 0 | 2 | 0 | 10 | 0 | 0 | 0 | 0 | 12 | +| xhluca | 0 | 6 | 2 | 4 | 0 | 0 | 0 | 0 | 12 | +| henilp105 | 0 | 0 | 0 | 2 | 0 | 9 | 0 | 0 | 11 | +| MariyaTikhonova | 0 | 7 | 0 | 0 | 0 | 4 | 0 | 0 | 11 | +| ab1992ao | 0 | 8 | 0 | 0 | 0 | 3 | 0 | 0 | 11 | +| tmp_handle | 0 | 0 | 0 | 0 | 10 | 0 | 0 | 0 | 10 | +| swj0419 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| Ruqyai | 0 | 2 | 8 | 0 | 0 | 0 | 0 | 0 | 10 | +| ZhengLiu101 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| Alenush | 0 | 6 | 0 | 0 | 0 | 4 | 0 | 0 | 10 | +| ABorghini | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| simon-clematide | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| sarahooker | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| guangyusong | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| HLasse | 0 | 0 | 0 | 5 | 0 | 5 | 0 | 0 | 10 | +| cassanof | 0 | 8 | 0 | 1 | 0 | 0 | 0 | 1 | 10 | +| hongjin-su | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| xiamengzhou | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| xu3kev | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| howard-yen | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| malteos | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| ljvmiranda921 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| marcobellagente93 | 0 | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| izhx | 0 | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| MexicanLemonade | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| antoniolanza1996 | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 2 | +| achibb | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| NouamaneTazi | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | +| PhilipMay | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | +| cslizc | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| bakrianoo | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| hanhainebula | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| monikernemo | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | diff --git a/docs/reproducible_workflow.md b/docs/reproducible_workflow.md index fd4b628509..9030e5d860 100644 --- a/docs/reproducible_workflow.md +++ b/docs/reproducible_workflow.md @@ -8,13 +8,13 @@ This section introduces how MTEB uses reproducible workflows. The main goal is t Using a reproducible workflow: -```{python} +```python import mteb model_name = "intfloat/multilingual-e5-small" revision = "4dc6d853a804b9c8886ede6dda8a073b7dc08a81" -model = mteb.get_model(model_name, revision_id=revision) # load model using registry implementation if available, otherwise use SentenceTransformers +model = mteb.get_model(model_name, revision=revision) # load model using registry implementation if available, otherwise use SentenceTransformers tasks = mteb.get_tasks(tasks = ["MIRACLReranking"], languages = ["eng"]) @@ -40,12 +40,16 @@ You may additionally want to specify parameters like whether the model is open-s 2. **If your model is not compatible with SentenceTransformer** -Additionally specify the `loader` in the ModelMeta object. This is a function that loads the model and returns a mteb compatible `Encoder` model. For the `Encoder` class, see `mteb/encoder_interface.py`. +Additionally specify the `loader` in the ModelMeta object. This is a function that loads the model and returns a mteb compatible `Encoder` model. For the `Encoder` class, see `mteb/encoder_interface.py`. Loader should contain: + - loader_function (for `SentenceTransformers` models, this is `sentence_transformers_loader`) + - `model_name`: The name of the model + - `revision`: The revision id of the model + - Optional `model_prompts`: A dictionary of prompts to be used in encoding. 3. **Submit a pull request** Submit a pull request with the new model. The model will be reviewed and added to the model repository. Please include the checklist in the pull request: - [ ] I have filled out the ModelMeta object to the extent possible -- [ ] I have ensured that my model can be loaded using `mteb.get_model(model_name, revision_id)` and `mteb.get_model_meta(model_name, revision_id)` +- [ ] I have ensured that my model can be loaded using `mteb.get_model(model_name, revision)` and `mteb.get_model_meta(model_name, revision)` - [ ] I have tested the implementation works for a representative set of tasks. \ No newline at end of file diff --git a/docs/tasks.md b/docs/tasks.md index df9e603abf..d90ac1816b 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -22,7 +22,7 @@ The following tables give you an overview of the tasks in MTEB. | [AlloprofRetrieval](https://huggingface.co/datasets/antoinelb7/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Retrieval | s2p | [Encyclopaedic, Written] | {'train': 2048} | {'test': {'average_document_length': 3505.705399061033, 'average_query_length': 170.71286701208982, 'num_documents': 2556, 'num_queries': 2316, 'average_relevant_docs_per_query': 1.0}} | | [AlphaNLI](https://leaderboard.allenai.org/anli/submissions/get-started) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 1532} | {'test': {'average_document_length': 43.42647308646886, 'average_query_length': 103.05483028720627, 'num_documents': 241347, 'num_queries': 1532, 'average_relevant_docs_per_query': 1.0}} | | [AmazonCounterfactualClassification](https://arxiv.org/abs/2104.06893) | ['deu', 'eng', 'jpn'] | Classification | s2s | [Reviews, Written] | {'validation': 335, 'test': 670} | {'validation': 109.2, 'test': 106.1} | -| [AmazonPolarityClassification](https://huggingface.co/datasets/amazon_polarity) (Julian McAuley, 2013) | ['eng'] | Classification | s2s | [Reviews, Written] | {'test': 400000} | {'test': 431.4} | +| [AmazonPolarityClassification](https://huggingface.co/datasets/amazon_polarity) (Julian McAuley, 2013) | ['eng'] | Classification | p2p | [Reviews, Written] | {'test': 400000} | {'test': 431.4} | | [AmazonReviewsClassification](https://arxiv.org/abs/2010.02573) (Phillip Keung, 2020) | ['cmn', 'deu', 'eng', 'fra', 'jpn', 'spa'] | Classification | s2s | [Reviews, Written] | {'validation': 30000, 'test': 30000} | {'validation': 159.2, 'test': 160.4} | | [AngryTweetsClassification](https://aclanthology.org/2021.nodalida-main.53/) (Pauli et al., 2021) | ['dan'] | Classification | s2s | [Social, Written] | {'test': 1050} | {'test': 156.1} | | [AppsRetrieval](https://arxiv.org/abs/2105.09938) (Dan Hendrycks, 2021) | ['eng', 'python'] | Retrieval | p2p | [Programming, Written] | {'test': 1000} | {'test': {'average_document_length': 575.0086708499715, 'average_query_length': 1669.8284196547145, 'num_documents': 8765, 'num_queries': 3765, 'average_relevant_docs_per_query': 1.0}} | @@ -121,6 +121,7 @@ The following tables give you an overview of the tasks in MTEB. | [CanadaTaxCourtOutcomesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 244} | {'test': 622.6} | | [CataloniaTweetClassification](https://aclanthology.org/2020.lrec-1.171/) | ['cat', 'spa'] | Classification | s2s | [Social, Government, Written] | {'validation': 2000, 'test': 2000} | {'validation': 202.61, 'test': 200.49} | | [ClimateFEVER](https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 538.241873443325, 'average_query_length': 123.39934853420195, 'num_documents': 5416593, 'num_queries': 1535, 'average_relevant_docs_per_query': 3.0495114006514656}} | +| [ClimateFEVERHardNegatives](https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | | {'test': 1000} | {'test': {'average_document_length': 1245.4236333727013, 'average_query_length': 121.879, 'num_documents': 47416, 'num_queries': 1000, 'average_relevant_docs_per_query': 3.048}} | | [CmedqaRetrieval](https://aclanthology.org/2022.emnlp-main.357.pdf) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 307.7710222897771, 'average_query_length': 48.470367591897976, 'num_documents': 100001, 'num_queries': 3999, 'average_relevant_docs_per_query': 1.86271567891973}} | | [Cmnli](https://huggingface.co/datasets/clue/viewer/cmnli) | ['cmn'] | PairClassification | s2s | | None | None | | [CodeEditSearchRetrieval](https://huggingface.co/datasets/cassanof/CodeEditSearch/viewer) (Niklas Muennighoff, 2023) | ['c', 'c++', 'go', 'java', 'javascript', 'php', 'python', 'ruby', 'rust', 'scala', 'shell', 'swift', 'typescript'] | Retrieval | p2p | [Programming, Written] | {'train': 13000} | {'train': {'python': {'average_document_length': 597.592, 'average_query_length': 69.519, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'javascript': {'average_document_length': 582.554, 'average_query_length': 56.88, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'typescript': {'average_document_length': 580.877, 'average_query_length': 60.092, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'go': {'average_document_length': 548.498, 'average_query_length': 70.797, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'ruby': {'average_document_length': 518.895, 'average_query_length': 66.9, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'java': {'average_document_length': 620.332, 'average_query_length': 62.984, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'php': {'average_document_length': 545.452, 'average_query_length': 61.927, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'c': {'average_document_length': 475.868, 'average_query_length': 97.588, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'c++': {'average_document_length': 544.446, 'average_query_length': 114.48, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'rust': {'average_document_length': 609.548, 'average_query_length': 67.503, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'swift': {'average_document_length': 574.62, 'average_query_length': 57.279, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'scala': {'average_document_length': 495.485, 'average_query_length': 64.833, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'shell': {'average_document_length': 486.519, 'average_query_length': 72.059, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}}} | @@ -156,6 +157,8 @@ The following tables give you an overview of the tasks in MTEB. | [CzechSubjectivityClassification](https://arxiv.org/abs/2009.08712) | ['ces'] | Classification | s2s | [Reviews, Written] | {'validation': 500, 'test': 2000} | {'validation': 108.2, 'test': 108.3} | | [DBPedia](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['eng'] | Retrieval | s2p | [Written, Encyclopaedic] | None | {'test': {'average_document_length': 1122.7690155333814, 'average_query_length': 48.7264325323475, 'num_documents': 48605, 'num_queries': 541, 'average_relevant_docs_per_query': 1.3752310536044363}} | | [DBPedia-PL](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['pol'] | Retrieval | s2p | [Written, Encyclopaedic] | None | {'test': {'average_document_length': 311.7007956561823, 'average_query_length': 35.45, 'num_documents': 4635922, 'num_queries': 400, 'average_relevant_docs_per_query': 38.215}} | +| [DBPedia-PLHardNegatives](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['pol'] | Retrieval | s2p | [Written, Encyclopaedic] | {'test': 400} | {'test': {'average_document_length': 363.468546000768, 'average_query_length': 35.45, 'num_documents': 88542, 'num_queries': 400, 'average_relevant_docs_per_query': 38.215}} | +| [DBPediaHardNegatives](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['eng'] | Retrieval | s2p | [Written, Encyclopaedic] | {'test': 400} | {'test': {'average_document_length': 338.58561119129564, 'average_query_length': 34.085, 'num_documents': 90070, 'num_queries': 400, 'average_relevant_docs_per_query': 38.215}} | | [DBpediaClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [Encyclopaedic, Written] | {'test': 70000} | {'test': 281.4} | | [DKHateClassification](https://aclanthology.org/2020.lrec-1.430/) | ['dan'] | Classification | s2s | [Social, Written] | {'test': 329} | {'test': 104.0} | | [DalajClassification](https://spraakbanken.gu.se/en/resources/superlim) | ['swe'] | Classification | s2s | [Non-fiction, Written] | {'test': 444} | {'test': 243.8} | @@ -171,12 +174,14 @@ The following tables give you an overview of the tasks in MTEB. | [Diversity6LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 300} | {'test': 301.01} | | [DuRetrieval](https://aclanthology.org/2022.emnlp-main.357.pdf) (Yifu Qiu, 2022) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 331.3219967800322, 'average_query_length': 9.289, 'num_documents': 100001, 'num_queries': 2000, 'average_relevant_docs_per_query': 4.9195}} | | [DutchBookReviewSentimentClassification](https://github.com/benjaminvdb/DBRD) (Benjamin et al., 2019) | ['nld'] | Classification | s2s | [Reviews, Written] | {'test': 2224} | {'test': 1443.0} | +| [ESCIReranking](https://github.com/amazon-science/esci-data/) (Chandan K. Reddy, 2022) | ['eng', 'jpn', 'spa'] | Reranking | s2p | [Written] | | | | [EcomRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 32.98041664189015, 'average_query_length': 6.798, 'num_documents': 100902, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}} | | [EightTagsClustering.v2](https://aclanthology.org/2020.lrec-1.207.pdf) | ['pol'] | Clustering | s2s | [Social, Written] | {'test': 2048} | {'test': 78.73} | | [EmotionClassification](https://www.aclweb.org/anthology/D18-1404) | ['eng'] | Classification | s2s | [Social, Written] | {'validation': 2000, 'test': 2000} | {'validation': 95.3, 'test': 95.6} | | [EstQA](https://www.semanticscholar.org/paper/Extractive-Question-Answering-for-Estonian-Language-182912IAPM-Alum%C3%A4e/ea4f60ab36cadca059c880678bc4c51e293a85d6?utm_source=direct_link) | ['est'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 603} | {'test': {'average_document_length': 785.595041322314, 'average_query_length': 55.32006633499171, 'num_documents': 121, 'num_queries': 603, 'average_relevant_docs_per_query': 1.0}} | | [EstonianValenceClassification](https://figshare.com/articles/dataset/Estonian_Valence_Corpus_Eesti_valentsikorpus/24517054) | ['est'] | Classification | s2s | [News, Written] | {'train': 3270, 'test': 818} | {'train': 226.70642201834863, 'test': 231.5085574572127} | | [FEVER](https://fever.ai/) | ['eng'] | Retrieval | s2p | | None | {'train': {'average_document_length': 538.2340070317589, 'average_query_length': 47.56034058828886, 'num_documents': 5416568, 'num_queries': 109810, 'average_relevant_docs_per_query': 1.2757034878426372}, 'dev': {'average_document_length': 538.2340070317589, 'average_query_length': 47.326282628262824, 'num_documents': 5416568, 'num_queries': 6666, 'average_relevant_docs_per_query': 1.211971197119712}, 'test': {'average_document_length': 538.2340070317589, 'average_query_length': 49.60546054605461, 'num_documents': 5416568, 'num_queries': 6666, 'average_relevant_docs_per_query': 1.1906690669066906}} | +| [FEVERHardNegatives](https://fever.ai/) | ['eng'] | Retrieval | s2p | | {'test': 1000} | {'test': {'average_document_length': 695.4370242764114, 'average_query_length': 49.62, 'num_documents': 163698, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.171}} | | [FQuADRetrieval](https://huggingface.co/datasets/manu/fquad2_test) | ['fra'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 400, 'validation': 100} | {'test': {'average_document_length': 896.3308550185874, 'average_query_length': 58.52, 'num_documents': 269, 'num_queries': 400, 'average_relevant_docs_per_query': 1.0}, 'validation': {'average_document_length': 895.1340206185567, 'average_query_length': 54.13, 'num_documents': 97, 'num_queries': 100, 'average_relevant_docs_per_query': 1.0}} | | [FaithDial](https://mcgill-nlp.github.io/FaithDial) (Dziri et al., 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 2042} | {'test': {'average_document_length': 140.61062447018932, 'average_query_length': 4.926542605288932, 'num_documents': 3539, 'num_queries': 2042, 'average_relevant_docs_per_query': 1.0}} | | [FalseFriendsGermanEnglish](https://drive.google.com/file/d/1jgq0nBnV-UiYNxbKNrrr2gxDEHm-DMKH/view?usp=share_link) | ['deu'] | PairClassification | s2s | [Written] | {'test': 1524} | {'test': 40.3} | @@ -221,6 +226,8 @@ The following tables give you an overview of the tasks in MTEB. | [HotelReviewSentimentClassification](https://link.springer.com/chapter/10.1007/978-3-319-67056-0_3) (Elnagar et al., 2018) | ['ara'] | Classification | s2s | [Reviews, Written] | {'train': 2048} | {'train': 137.2} | | [HotpotQA](https://hotpotqa.github.io/) | ['eng'] | Retrieval | s2p | [Web, Written] | None | {'train': {'average_document_length': 287.9079517072212, 'average_query_length': 105.54965882352941, 'num_documents': 5233329, 'num_queries': 85000, 'average_relevant_docs_per_query': 2.0}, 'dev': {'average_document_length': 287.9079517072212, 'average_query_length': 105.35634294106848, 'num_documents': 5233329, 'num_queries': 5447, 'average_relevant_docs_per_query': 2.0}, 'test': {'average_document_length': 287.9079517072212, 'average_query_length': 92.17096556380824, 'num_documents': 5233329, 'num_queries': 7405, 'average_relevant_docs_per_query': 2.0}} | | [HotpotQA-PL](https://hotpotqa.github.io/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | {'test': {'average_document_length': 292.26835882093405, 'average_query_length': 94.64064821066847, 'num_documents': 5233329, 'num_queries': 7405, 'average_relevant_docs_per_query': 2.0}} | +| [HotpotQA-PLHardNegatives](https://hotpotqa.github.io/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | {'test': 1000} | {'test': {'average_document_length': 438.3888210025661, 'average_query_length': 95.161, 'num_documents': 212774, 'num_queries': 1000, 'average_relevant_docs_per_query': 2.0}} | +| [HotpotQAHardNegatives](https://hotpotqa.github.io/) | ['eng'] | Retrieval | s2p | [Web, Written] | {'test': 1000} | {'test': {'average_document_length': 373.558822095461, 'average_query_length': 92.584, 'num_documents': 225621, 'num_queries': 1000, 'average_relevant_docs_per_query': 2.0}} | | [HunSum2AbstractiveRetrieval](https://arxiv.org/abs/2404.03555) (Botond Barta, 2024) | ['hun'] | Retrieval | s2p | [News, Written] | {'test': 1998} | {'test': {'average_document_length': 2511.0315315315315, 'average_query_length': 201.2112112112112, 'num_documents': 1998, 'num_queries': 1998, 'average_relevant_docs_per_query': 1.0}} | | [IFlyTek](https://www.cluebenchmarks.com/introduce.html) | ['cmn'] | Classification | s2s | | None | None | | [IN22ConvBitextMining](https://huggingface.co/datasets/ai4bharat/IN22-Conv) (Jay Gala, 2023) | ['asm', 'ben', 'brx', 'doi', 'eng', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Social, Spoken, Fiction, Spoken] | | | @@ -248,6 +255,7 @@ The following tables give you an overview of the tasks in MTEB. | [JSTS](https://aclanthology.org/2022.lrec-1.317.pdf#page=2.00) | ['jpn'] | STS | s2s | [Web, Written] | {'valudtion': 1457} | {'valudtion': 46.34} | | [JaGovFaqsRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Web, Written] | {'test': 2048} | {'test': {'average_document_length': 210.02601561814512, 'average_query_length': 59.48193359375, 'num_documents': 22794, 'num_queries': 2048, 'average_relevant_docs_per_query': 1.0}} | | [JaQuADRetrieval](https://arxiv.org/abs/2202.01764) (ByungHoon So, 2022) | ['jpn'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Written] | {'validation': 2048} | {'validation': {'average_document_length': 155.80922362309224, 'average_query_length': 30.826171875, 'num_documents': 3014, 'num_queries': 2048, 'average_relevant_docs_per_query': 2.0}} | +| [JaqketRetrieval](https://github.com/kumapo/JAQKET-dataset) | ['jpn'] | Retrieval | s2p | [Encyclopaedic, Non-fiction, Written] | | | | [JavaneseIMDBClassification](https://github.com/w11wo/nlp-datasets#javanese-imdb) (Wongso et al., 2021) | ['jav'] | Classification | s2s | [Reviews, Written] | {'test': 25000} | {'test': 481.83} | | [KLUE-NLI](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | PairClassification | s2s | [News, Encyclopaedic, Written] | {'validation': 2000} | {'validation': 35.01} | | [KLUE-STS](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | STS | s2s | [Reviews, News, Spoken, Written, Spoken] | {'validation': 519} | {'validation': 33.178227360308284} | @@ -299,14 +307,17 @@ The following tables give you an overview of the tasks in MTEB. | [MAUDLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 2048} | {'test': 1802.93} | | [MIRACLReranking](https://project-miracl.github.io/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Reranking | s2s | [Encyclopaedic, Written] | {'dev': 44608} | {'dev': 506.3} | | [MIRACLRetrieval](http://miracl.ai/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | None | {'dev': {'ar': {'average_document_length': 318.6539598547405, 'average_query_length': 29.480662983425415, 'num_documents': 2061414, 'num_queries': 2896, 'average_relevant_docs_per_query': 1.953729281767956}, 'bn': {'average_document_length': 383.2428136511194, 'average_query_length': 46.98053527980535, 'num_documents': 297265, 'num_queries': 411, 'average_relevant_docs_per_query': 2.099756690997567}, 'de': {'average_document_length': 414.28004442393404, 'average_query_length': 46.0, 'num_documents': 15866222, 'num_queries': 305, 'average_relevant_docs_per_query': 2.6590163934426227}, 'en': {'average_document_length': 401.0042914921588, 'average_query_length': 40.247809762202756, 'num_documents': 32893221, 'num_queries': 799, 'average_relevant_docs_per_query': 2.911138923654568}, 'es': {'average_document_length': 403.71153493754986, 'average_query_length': 47.373456790123456, 'num_documents': 10373953, 'num_queries': 648, 'average_relevant_docs_per_query': 4.609567901234568}, 'fa': {'average_document_length': 262.6478385010321, 'average_query_length': 41.1503164556962, 'num_documents': 2207172, 'num_queries': 632, 'average_relevant_docs_per_query': 2.079113924050633}, 'fi': {'average_document_length': 359.87767671935734, 'average_query_length': 38.63493312352478, 'num_documents': 1883509, 'num_queries': 1271, 'average_relevant_docs_per_query': 1.925255704169945}, 'fr': {'average_document_length': 343.6283550271699, 'average_query_length': 43.883381924198254, 'num_documents': 14636953, 'num_queries': 343, 'average_relevant_docs_per_query': 2.131195335276968}, 'hi': {'average_document_length': 370.96196845914386, 'average_query_length': 53.34, 'num_documents': 506264, 'num_queries': 350, 'average_relevant_docs_per_query': 2.1485714285714286}, 'id': {'average_document_length': 350.2785651811673, 'average_query_length': 37.958333333333336, 'num_documents': 1446315, 'num_queries': 960, 'average_relevant_docs_per_query': 3.216666666666667}, 'ja': {'average_document_length': 145.8538220556965, 'average_query_length': 17.71395348837209, 'num_documents': 6953614, 'num_queries': 860, 'average_relevant_docs_per_query': 2.0813953488372094}, 'ko': {'average_document_length': 173.97649170809927, 'average_query_length': 21.624413145539908, 'num_documents': 1486752, 'num_queries': 213, 'average_relevant_docs_per_query': 2.568075117370892}, 'ru': {'average_document_length': 332.2475377512674, 'average_query_length': 44.13258785942492, 'num_documents': 9543918, 'num_queries': 1252, 'average_relevant_docs_per_query': 2.8434504792332267}, 'sw': {'average_document_length': 228.71348655286377, 'average_query_length': 38.97095435684647, 'num_documents': 131924, 'num_queries': 482, 'average_relevant_docs_per_query': 1.887966804979253}, 'te': {'average_document_length': 396.2108674545774, 'average_query_length': 38.11231884057971, 'num_documents': 518079, 'num_queries': 828, 'average_relevant_docs_per_query': 1.0314009661835748}, 'th': {'average_document_length': 356.8283496198581, 'average_query_length': 42.87585266030014, 'num_documents': 542166, 'num_queries': 733, 'average_relevant_docs_per_query': 1.8321964529331514}, 'yo': {'average_document_length': 159.35250698366738, 'average_query_length': 37.6890756302521, 'num_documents': 49043, 'num_queries': 119, 'average_relevant_docs_per_query': 1.2100840336134453}, 'zh': {'average_document_length': 119.9458931721347, 'average_query_length': 10.867684478371501, 'num_documents': 4934368, 'num_queries': 393, 'average_relevant_docs_per_query': 2.5292620865139948}}} | +| [MIRACLRetrievalHardNegatives](http://miracl.ai/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | None | {'dev': {'average_document_length': 417.6655323669399, 'average_query_length': 37.46957385337667, 'num_documents': 2449382, 'num_queries': 11076, 'average_relevant_docs_per_query': 2.3643011917659806, 'hf_subset_descriptive_stats': {'ar': {'average_document_length': 438.1872433017704, 'average_query_length': 29.584, 'num_documents': 192103, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.982}, 'bn': {'average_document_length': 383.2428136511194, 'average_query_length': 46.98053527980535, 'num_documents': 297265, 'num_queries': 411, 'average_relevant_docs_per_query': 2.099756690997567}, 'de': {'average_document_length': 513.7796484139344, 'average_query_length': 46.0, 'num_documents': 71277, 'num_queries': 305, 'average_relevant_docs_per_query': 2.6590163934426227}, 'en': {'average_document_length': 529.2486406963214, 'average_query_length': 40.247809762202756, 'num_documents': 178768, 'num_queries': 799, 'average_relevant_docs_per_query': 2.911138923654568}, 'es': {'average_document_length': 535.8023645655877, 'average_query_length': 47.373456790123456, 'num_documents': 146750, 'num_queries': 648, 'average_relevant_docs_per_query': 4.609567901234568}, 'fa': {'average_document_length': 411.2648282882721, 'average_query_length': 41.1503164556962, 'num_documents': 133596, 'num_queries': 632, 'average_relevant_docs_per_query': 2.079113924050633}, 'fi': {'average_document_length': 462.9445310289844, 'average_query_length': 38.646, 'num_documents': 194415, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.918}, 'fr': {'average_document_length': 460.40909271865917, 'average_query_length': 43.883381924198254, 'num_documents': 75357, 'num_queries': 343, 'average_relevant_docs_per_query': 2.131195335276968}, 'hi': {'average_document_length': 498.6759426632417, 'average_query_length': 53.34, 'num_documents': 63066, 'num_queries': 350, 'average_relevant_docs_per_query': 2.1485714285714286}, 'id': {'average_document_length': 494.1689807519638, 'average_query_length': 37.958333333333336, 'num_documents': 168173, 'num_queries': 960, 'average_relevant_docs_per_query': 3.216666666666667}, 'ja': {'average_document_length': 206.13654293407583, 'average_query_length': 17.71395348837209, 'num_documents': 185319, 'num_queries': 860, 'average_relevant_docs_per_query': 2.0813953488372094}, 'ko': {'average_document_length': 257.82646155267594, 'average_query_length': 21.624413145539908, 'num_documents': 43293, 'num_queries': 213, 'average_relevant_docs_per_query': 2.568075117370892}, 'ru': {'average_document_length': 476.0820349224605, 'average_query_length': 44.055, 'num_documents': 219114, 'num_queries': 1000, 'average_relevant_docs_per_query': 2.833}, 'sw': {'average_document_length': 228.71348655286377, 'average_query_length': 38.97095435684647, 'num_documents': 131924, 'num_queries': 482, 'average_relevant_docs_per_query': 1.887966804979253}, 'te': {'average_document_length': 601.7099283059209, 'average_query_length': 38.11231884057971, 'num_documents': 101961, 'num_queries': 828, 'average_relevant_docs_per_query': 1.0314009661835748}, 'th': {'average_document_length': 478.8818849711528, 'average_query_length': 42.87585266030014, 'num_documents': 116649, 'num_queries': 733, 'average_relevant_docs_per_query': 1.8321964529331514}, 'yo': {'average_document_length': 159.35250698366738, 'average_query_length': 37.6890756302521, 'num_documents': 49043, 'num_queries': 119, 'average_relevant_docs_per_query': 1.2100840336134453}, 'zh': {'average_document_length': 147.36211243527777, 'average_query_length': 10.867684478371501, 'num_documents': 81309, 'num_queries': 393, 'average_relevant_docs_per_query': 2.5292620865139948}}}} | | [MLQARetrieval](https://huggingface.co/datasets/mlqa) | ['ara', 'deu', 'eng', 'hin', 'spa', 'vie', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 158083, 'validation': 15747} | {'validation': {'ara-ara': {'average_document_length': 693.8883826879271, 'average_query_length': 42.321083172147, 'num_documents': 439, 'num_queries': 517, 'average_relevant_docs_per_query': 1.0}, 'ara-deu': {'average_document_length': 759.3882352941176, 'average_query_length': 55.14492753623188, 'num_documents': 170, 'num_queries': 207, 'average_relevant_docs_per_query': 1.0}, 'ara-eng': {'average_document_length': 693.8883826879271, 'average_query_length': 50.029013539651835, 'num_documents': 439, 'num_queries': 517, 'average_relevant_docs_per_query': 1.0}, 'ara-spa': {'average_document_length': 654.3071428571428, 'average_query_length': 53.68944099378882, 'num_documents': 140, 'num_queries': 161, 'average_relevant_docs_per_query': 1.0}, 'ara-hin': {'average_document_length': 626.5935483870968, 'average_query_length': 51.956989247311824, 'num_documents': 155, 'num_queries': 186, 'average_relevant_docs_per_query': 1.0}, 'ara-vie': {'average_document_length': 804.6216216216217, 'average_query_length': 49.57055214723926, 'num_documents': 148, 'num_queries': 163, 'average_relevant_docs_per_query': 1.0}, 'ara-zho': {'average_document_length': 787.3161290322581, 'average_query_length': 15.617021276595745, 'num_documents': 155, 'num_queries': 188, 'average_relevant_docs_per_query': 1.0}, 'deu-ara': {'average_document_length': 702.1675977653631, 'average_query_length': 43.06280193236715, 'num_documents': 179, 'num_queries': 207, 'average_relevant_docs_per_query': 1.0}, 'deu-deu': {'average_document_length': 721.405701754386, 'average_query_length': 52.572265625, 'num_documents': 456, 'num_queries': 512, 'average_relevant_docs_per_query': 1.0}, 'deu-eng': {'average_document_length': 721.405701754386, 'average_query_length': 48.33984375, 'num_documents': 456, 'num_queries': 512, 'average_relevant_docs_per_query': 1.0}, 'deu-spa': {'average_document_length': 677.2762430939226, 'average_query_length': 50.60204081632653, 'num_documents': 181, 'num_queries': 196, 'average_relevant_docs_per_query': 1.0}, 'deu-hin': {'average_document_length': 685.917808219178, 'average_query_length': 47.01840490797546, 'num_documents': 146, 'num_queries': 163, 'average_relevant_docs_per_query': 1.0}, 'deu-vie': {'average_document_length': 921.6196319018405, 'average_query_length': 46.81868131868132, 'num_documents': 163, 'num_queries': 182, 'average_relevant_docs_per_query': 1.0}, 'deu-zho': {'average_document_length': 736.6347305389221, 'average_query_length': 14.936842105263159, 'num_documents': 167, 'num_queries': 190, 'average_relevant_docs_per_query': 1.0}, 'eng-ara': {'average_document_length': 979.3447488584475, 'average_query_length': 42.321083172147, 'num_documents': 438, 'num_queries': 517, 'average_relevant_docs_per_query': 1.0}, 'eng-deu': {'average_document_length': 947.3109619686801, 'average_query_length': 52.572265625, 'num_documents': 447, 'num_queries': 512, 'average_relevant_docs_per_query': 1.0}, 'eng-eng': {'average_document_length': 940.2842535787321, 'average_query_length': 49.01480836236934, 'num_documents': 978, 'num_queries': 1148, 'average_relevant_docs_per_query': 1.0}, 'eng-spa': {'average_document_length': 904.3166287015945, 'average_query_length': 52.146, 'num_documents': 439, 'num_queries': 500, 'average_relevant_docs_per_query': 1.0}, 'eng-hin': {'average_document_length': 926.9621749408983, 'average_query_length': 49.3905325443787, 'num_documents': 423, 'num_queries': 507, 'average_relevant_docs_per_query': 1.0}, 'eng-vie': {'average_document_length': 1011.8296460176991, 'average_query_length': 48.082191780821915, 'num_documents': 452, 'num_queries': 511, 'average_relevant_docs_per_query': 1.0}, 'eng-zho': {'average_document_length': 1001.5046511627907, 'average_query_length': 15.39484126984127, 'num_documents': 430, 'num_queries': 504, 'average_relevant_docs_per_query': 1.0}, 'spa-ara': {'average_document_length': 674.3586206896551, 'average_query_length': 41.36024844720497, 'num_documents': 145, 'num_queries': 161, 'average_relevant_docs_per_query': 1.0}, 'spa-deu': {'average_document_length': 544.0489130434783, 'average_query_length': 51.86734693877551, 'num_documents': 184, 'num_queries': 196, 'average_relevant_docs_per_query': 1.0}, 'spa-eng': {'average_document_length': 641.8215859030837, 'average_query_length': 49.156, 'num_documents': 454, 'num_queries': 500, 'average_relevant_docs_per_query': 1.0}, 'spa-spa': {'average_document_length': 641.8215859030837, 'average_query_length': 52.146, 'num_documents': 454, 'num_queries': 500, 'average_relevant_docs_per_query': 1.0}, 'spa-hin': {'average_document_length': 703.3212121212122, 'average_query_length': 48.080213903743314, 'num_documents': 165, 'num_queries': 187, 'average_relevant_docs_per_query': 1.0}, 'spa-vie': {'average_document_length': 737.8579545454545, 'average_query_length': 48.82539682539682, 'num_documents': 176, 'num_queries': 189, 'average_relevant_docs_per_query': 1.0}, 'spa-zho': {'average_document_length': 605.52, 'average_query_length': 15.590062111801242, 'num_documents': 150, 'num_queries': 161, 'average_relevant_docs_per_query': 1.0}, 'hin-ara': {'average_document_length': 670.0394736842105, 'average_query_length': 43.623655913978496, 'num_documents': 152, 'num_queries': 186, 'average_relevant_docs_per_query': 1.0}, 'hin-deu': {'average_document_length': 596.9718309859155, 'average_query_length': 51.41717791411043, 'num_documents': 142, 'num_queries': 163, 'average_relevant_docs_per_query': 1.0}, 'hin-eng': {'average_document_length': 691.5482352941176, 'average_query_length': 49.75936883629191, 'num_documents': 425, 'num_queries': 507, 'average_relevant_docs_per_query': 1.0}, 'hin-spa': {'average_document_length': 718.4904458598726, 'average_query_length': 52.75935828877005, 'num_documents': 157, 'num_queries': 187, 'average_relevant_docs_per_query': 1.0}, 'hin-hin': {'average_document_length': 691.5482352941176, 'average_query_length': 49.3905325443787, 'num_documents': 425, 'num_queries': 507, 'average_relevant_docs_per_query': 1.0}, 'hin-vie': {'average_document_length': 778.484076433121, 'average_query_length': 48.35028248587571, 'num_documents': 157, 'num_queries': 177, 'average_relevant_docs_per_query': 1.0}, 'hin-zho': {'average_document_length': 685.0679012345679, 'average_query_length': 15.97883597883598, 'num_documents': 162, 'num_queries': 189, 'average_relevant_docs_per_query': 1.0}, 'vie-ara': {'average_document_length': 886.6052631578947, 'average_query_length': 41.214723926380366, 'num_documents': 152, 'num_queries': 163, 'average_relevant_docs_per_query': 1.0}, 'vie-deu': {'average_document_length': 981.4534161490683, 'average_query_length': 51.27472527472528, 'num_documents': 161, 'num_queries': 182, 'average_relevant_docs_per_query': 1.0}, 'vie-eng': {'average_document_length': 892.7250554323725, 'average_query_length': 48.09001956947162, 'num_documents': 451, 'num_queries': 511, 'average_relevant_docs_per_query': 1.0}, 'vie-spa': {'average_document_length': 936.6746987951807, 'average_query_length': 51.851851851851855, 'num_documents': 166, 'num_queries': 189, 'average_relevant_docs_per_query': 1.0}, 'vie-hin': {'average_document_length': 869.0509554140127, 'average_query_length': 46.44632768361582, 'num_documents': 157, 'num_queries': 177, 'average_relevant_docs_per_query': 1.0}, 'vie-vie': {'average_document_length': 892.7250554323725, 'average_query_length': 48.082191780821915, 'num_documents': 451, 'num_queries': 511, 'average_relevant_docs_per_query': 1.0}, 'vie-zho': {'average_document_length': 960.7349397590361, 'average_query_length': 15.048913043478262, 'num_documents': 166, 'num_queries': 184, 'average_relevant_docs_per_query': 1.0}, 'zho-ara': {'average_document_length': 238.75155279503105, 'average_query_length': 44.34574468085106, 'num_documents': 161, 'num_queries': 188, 'average_relevant_docs_per_query': 1.0}, 'zho-deu': {'average_document_length': 257.109756097561, 'average_query_length': 53.84736842105263, 'num_documents': 164, 'num_queries': 190, 'average_relevant_docs_per_query': 1.0}, 'zho-eng': {'average_document_length': 246.65237020316027, 'average_query_length': 50.15079365079365, 'num_documents': 443, 'num_queries': 504, 'average_relevant_docs_per_query': 1.0}, 'zho-spa': {'average_document_length': 249.6081081081081, 'average_query_length': 52.857142857142854, 'num_documents': 148, 'num_queries': 161, 'average_relevant_docs_per_query': 1.0}, 'zho-hin': {'average_document_length': 238.5521472392638, 'average_query_length': 52.05291005291005, 'num_documents': 163, 'num_queries': 189, 'average_relevant_docs_per_query': 1.0}, 'zho-vie': {'average_document_length': 268.32142857142856, 'average_query_length': 49.33695652173913, 'num_documents': 168, 'num_queries': 184, 'average_relevant_docs_per_query': 1.0}, 'zho-zho': {'average_document_length': 246.65237020316027, 'average_query_length': 15.39484126984127, 'num_documents': 443, 'num_queries': 504, 'average_relevant_docs_per_query': 1.0}}, 'test': {'ara-ara': {'average_document_length': 698.5714593198451, 'average_query_length': 41.26176636039752, 'num_documents': 4646, 'num_queries': 5333, 'average_relevant_docs_per_query': 1.000375023438965}, 'ara-deu': {'average_document_length': 592.5728542914171, 'average_query_length': 51.27730582524272, 'num_documents': 1503, 'num_queries': 1648, 'average_relevant_docs_per_query': 1.0006067961165048}, 'ara-eng': {'average_document_length': 698.5714593198451, 'average_query_length': 48.556451612903224, 'num_documents': 4646, 'num_queries': 5332, 'average_relevant_docs_per_query': 1.000562640660165}, 'ara-spa': {'average_document_length': 713.4833239118146, 'average_query_length': 51.406471183013146, 'num_documents': 1769, 'num_queries': 1978, 'average_relevant_docs_per_query': 1.0}, 'ara-hin': {'average_document_length': 702.1388888888889, 'average_query_length': 48.71818678317859, 'num_documents': 1512, 'num_queries': 1831, 'average_relevant_docs_per_query': 1.0}, 'ara-vie': {'average_document_length': 745.4528096017458, 'average_query_length': 48.815828041035665, 'num_documents': 1833, 'num_queries': 2047, 'average_relevant_docs_per_query': 1.0}, 'ara-zho': {'average_document_length': 774.4593639575971, 'average_query_length': 14.985355648535565, 'num_documents': 1698, 'num_queries': 1912, 'average_relevant_docs_per_query': 1.0}, 'deu-ara': {'average_document_length': 719.6800267201069, 'average_query_length': 39.54578532443905, 'num_documents': 1497, 'num_queries': 1649, 'average_relevant_docs_per_query': 1.0}, 'deu-deu': {'average_document_length': 725.5304712558599, 'average_query_length': 51.610680257035234, 'num_documents': 4053, 'num_queries': 4513, 'average_relevant_docs_per_query': 1.0008863283846665}, 'deu-eng': {'average_document_length': 725.5304712558599, 'average_query_length': 47.07777531575449, 'num_documents': 4053, 'num_queries': 4513, 'average_relevant_docs_per_query': 1.0008863283846665}, 'deu-spa': {'average_document_length': 740.5414052697616, 'average_query_length': 50.098591549295776, 'num_documents': 1594, 'num_queries': 1775, 'average_relevant_docs_per_query': 1.0005633802816902}, 'deu-hin': {'average_document_length': 674.3714063714064, 'average_query_length': 45.146153846153844, 'num_documents': 1287, 'num_queries': 1430, 'average_relevant_docs_per_query': 1.0}, 'deu-vie': {'average_document_length': 760.1198945981555, 'average_query_length': 46.64358208955224, 'num_documents': 1518, 'num_queries': 1675, 'average_relevant_docs_per_query': 1.0}, 'deu-zho': {'average_document_length': 771.3367697594501, 'average_query_length': 14.942592592592593, 'num_documents': 1455, 'num_queries': 1620, 'average_relevant_docs_per_query': 1.0006172839506173}, 'eng-ara': {'average_document_length': 1008.3584455058619, 'average_query_length': 41.26176636039752, 'num_documents': 4606, 'num_queries': 5333, 'average_relevant_docs_per_query': 1.000375023438965}, 'eng-deu': {'average_document_length': 910.3226686507936, 'average_query_length': 51.610680257035234, 'num_documents': 4032, 'num_queries': 4513, 'average_relevant_docs_per_query': 1.0008863283846665}, 'eng-eng': {'average_document_length': 983.0993344090359, 'average_query_length': 47.960714902434816, 'num_documents': 9916, 'num_queries': 11582, 'average_relevant_docs_per_query': 1.000690726990157}, 'eng-spa': {'average_document_length': 967.4622376109068, 'average_query_length': 50.923252713768804, 'num_documents': 4621, 'num_queries': 5251, 'average_relevant_docs_per_query': 1.000380879832413}, 'eng-hin': {'average_document_length': 986.0465631929046, 'average_query_length': 47.328315703824245, 'num_documents': 4059, 'num_queries': 4916, 'average_relevant_docs_per_query': 1.000406834825061}, 'eng-vie': {'average_document_length': 1048.6062197940744, 'average_query_length': 48.094085532302095, 'num_documents': 4759, 'num_queries': 5495, 'average_relevant_docs_per_query': 1.0}, 'eng-zho': {'average_document_length': 1063.8536257833482, 'average_query_length': 15.019080996884735, 'num_documents': 4468, 'num_queries': 5136, 'average_relevant_docs_per_query': 1.0001947040498442}, 'spa-ara': {'average_document_length': 645.5182320441988, 'average_query_length': 40.78412537917088, 'num_documents': 1810, 'num_queries': 1978, 'average_relevant_docs_per_query': 1.0}, 'spa-deu': {'average_document_length': 586.6057810578105, 'average_query_length': 51.870913190529876, 'num_documents': 1626, 'num_queries': 1774, 'average_relevant_docs_per_query': 1.0011273957158964}, 'spa-eng': {'average_document_length': 630.6735979836169, 'average_query_length': 47.827907862173994, 'num_documents': 4761, 'num_queries': 5253, 'average_relevant_docs_per_query': 1.0}, 'spa-spa': {'average_document_length': 630.6735979836169, 'average_query_length': 50.923252713768804, 'num_documents': 4761, 'num_queries': 5251, 'average_relevant_docs_per_query': 1.000380879832413}, 'spa-hin': {'average_document_length': 613.3478260869565, 'average_query_length': 46.36680208937899, 'num_documents': 1518, 'num_queries': 1723, 'average_relevant_docs_per_query': 1.0}, 'spa-vie': {'average_document_length': 659.6179295624333, 'average_query_length': 48.1595639246779, 'num_documents': 1874, 'num_queries': 2018, 'average_relevant_docs_per_query': 1.0}, 'spa-zho': {'average_document_length': 668.6646171045277, 'average_query_length': 15.115562403697997, 'num_documents': 1789, 'num_queries': 1947, 'average_relevant_docs_per_query': 1.0}, 'hin-ara': {'average_document_length': 765.0352862849534, 'average_query_length': 42.04642271982523, 'num_documents': 1502, 'num_queries': 1831, 'average_relevant_docs_per_query': 1.0}, 'hin-deu': {'average_document_length': 719.676862745098, 'average_query_length': 51.002799160251925, 'num_documents': 1275, 'num_queries': 1429, 'average_relevant_docs_per_query': 1.000699790062981}, 'hin-eng': {'average_document_length': 760.9956086850451, 'average_query_length': 47.91232709519935, 'num_documents': 4099, 'num_queries': 4916, 'average_relevant_docs_per_query': 1.000406834825061}, 'hin-spa': {'average_document_length': 753.5010281014394, 'average_query_length': 50.46689895470383, 'num_documents': 1459, 'num_queries': 1722, 'average_relevant_docs_per_query': 1.0005807200929153}, 'hin-hin': {'average_document_length': 760.9956086850451, 'average_query_length': 47.328315703824245, 'num_documents': 4099, 'num_queries': 4916, 'average_relevant_docs_per_query': 1.000406834825061}, 'hin-vie': {'average_document_length': 789.9253822629969, 'average_query_length': 48.21160760143811, 'num_documents': 1635, 'num_queries': 1947, 'average_relevant_docs_per_query': 1.0}, 'hin-zho': {'average_document_length': 834.2057448229793, 'average_query_length': 15.101301641199774, 'num_documents': 1497, 'num_queries': 1767, 'average_relevant_docs_per_query': 1.0}, 'vie-ara': {'average_document_length': 992.2129527991218, 'average_query_length': 41.82462139716659, 'num_documents': 1822, 'num_queries': 2047, 'average_relevant_docs_per_query': 1.0}, 'vie-deu': {'average_document_length': 861.0610079575597, 'average_query_length': 51.58721624850657, 'num_documents': 1508, 'num_queries': 1674, 'average_relevant_docs_per_query': 1.0005973715651135}, 'vie-eng': {'average_document_length': 913.8633993743483, 'average_query_length': 48.11086837793555, 'num_documents': 4795, 'num_queries': 5493, 'average_relevant_docs_per_query': 1.0003640997633352}, 'vie-spa': {'average_document_length': 940.0322580645161, 'average_query_length': 51.13386217154189, 'num_documents': 1829, 'num_queries': 2017, 'average_relevant_docs_per_query': 1.0004957858205255}, 'vie-hin': {'average_document_length': 838.1713414634146, 'average_query_length': 47.484334874165384, 'num_documents': 1640, 'num_queries': 1947, 'average_relevant_docs_per_query': 1.0}, 'vie-vie': {'average_document_length': 913.8633993743483, 'average_query_length': 48.094085532302095, 'num_documents': 4795, 'num_queries': 5495, 'average_relevant_docs_per_query': 1.0}, 'vie-zho': {'average_document_length': 999.064534883721, 'average_query_length': 15.045805455481215, 'num_documents': 1720, 'num_queries': 1943, 'average_relevant_docs_per_query': 1.0}, 'zho-ara': {'average_document_length': 253.71303841676368, 'average_query_length': 42.04866562009419, 'num_documents': 1718, 'num_queries': 1911, 'average_relevant_docs_per_query': 1.000523286237572}, 'zho-deu': {'average_document_length': 241.84631147540983, 'average_query_length': 52.25107958050586, 'num_documents': 1464, 'num_queries': 1621, 'average_relevant_docs_per_query': 1.0}, 'zho-eng': {'average_document_length': 247.55609326880776, 'average_query_length': 48.64167478091529, 'num_documents': 4546, 'num_queries': 5135, 'average_relevant_docs_per_query': 1.0003894839337877}, 'zho-spa': {'average_document_length': 254.44552196235026, 'average_query_length': 51.90446841294299, 'num_documents': 1753, 'num_queries': 1947, 'average_relevant_docs_per_query': 1.0}, 'zho-hin': {'average_document_length': 229.60590163934427, 'average_query_length': 49.06625141562854, 'num_documents': 1525, 'num_queries': 1766, 'average_relevant_docs_per_query': 1.0005662514156286}, 'zho-vie': {'average_document_length': 266.1140401146132, 'average_query_length': 49.27328872876994, 'num_documents': 1745, 'num_queries': 1943, 'average_relevant_docs_per_query': 1.0}, 'zho-zho': {'average_document_length': 247.55609326880776, 'average_query_length': 15.019080996884735, 'num_documents': 4546, 'num_queries': 5136, 'average_relevant_docs_per_query': 1.0001947040498442}}} | | [MLQuestions](https://github.com/McGill-NLP/MLQuestions) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Academic, Written] | {'dev': 1500, 'test': 1500} | {'dev': {'average_document_length': 258.8772727272727, 'average_query_length': 45.05533333333333, 'num_documents': 11000, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'test': {'average_document_length': 258.8772727272727, 'average_query_length': 45.75333333333333, 'num_documents': 11000, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}} | -| [MLSUMClusteringP2P.v2](https://huggingface.co/datasets/mlsum) (Scialom et al., 2020) | ['deu', 'fra', 'rus', 'spa'] | Clustering | p2p | [News, Written] | {'validation': 2048, 'test': 2048} | {'validation': 4613, 'test': 4810} | -| [MLSUMClusteringS2S.v2](https://huggingface.co/datasets/mlsum) (Scialom et al., 2020) | ['deu', 'fra', 'rus', 'spa'] | Clustering | s2s | [News, Written] | {'validation': 750, 'test': 756} | {'validation': 4613, 'test': 4810} | +| [MLSUMClusteringP2P.v2](https://huggingface.co/datasets/mteb/mlsum) (Scialom et al., 2020) | ['deu', 'fra', 'rus', 'spa'] | Clustering | p2p | [News, Written] | {'validation': 2048, 'test': 2048} | {'validation': 4613, 'test': 4810} | +| [MLSUMClusteringS2S.v2](https://huggingface.co/datasets/mteb/mlsum) (Scialom et al., 2020) | ['deu', 'fra', 'rus', 'spa'] | Clustering | s2s | [News, Written] | {'validation': 750, 'test': 756} | {'validation': 4613, 'test': 4810} | | [MMarcoReranking](https://github.com/unicamp-dl/mMARCO) (Luiz Henrique Bonifacio, 2021) | ['cmn'] | Reranking | s2s | | None | None | | [MMarcoRetrieval](https://arxiv.org/abs/2309.07597) (Shitao Xiao, 2024) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 114.41787048392986, 'average_query_length': 10.51131805157593, 'num_documents': 106813, 'num_queries': 6980, 'average_relevant_docs_per_query': 1.0654727793696275}} | | [MSMARCO](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | None | {'train': {'average_document_length': 335.79716603691344, 'average_query_length': 33.21898281898998, 'num_documents': 8841823, 'num_queries': 502939, 'average_relevant_docs_per_query': 1.0592755781516248}, 'dev': {'average_document_length': 335.79716603691344, 'average_query_length': 33.2621776504298, 'num_documents': 8841823, 'num_queries': 6980, 'average_relevant_docs_per_query': 1.0654727793696275}, 'test': {'average_document_length': 335.79716603691344, 'average_query_length': 32.74418604651163, 'num_documents': 8841823, 'num_queries': 43, 'average_relevant_docs_per_query': 95.3953488372093}} | | [MSMARCO-PL](https://microsoft.github.io/msmarco/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | {'test': {'average_document_length': 349.3574939240471, 'average_query_length': 33.02325581395349, 'num_documents': 8841823, 'num_queries': 43, 'average_relevant_docs_per_query': 95.3953488372093}} | +| [MSMARCO-PLHardNegatives](https://microsoft.github.io/msmarco/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | {'test': 43} | {'test': {'average_document_length': 382.3476426537285, 'average_query_length': 33.02325581395349, 'num_documents': 9481, 'num_queries': 43, 'average_relevant_docs_per_query': 95.3953488372093}} | +| [MSMARCOHardNegatives](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | {'test': 43} | {'test': {'average_document_length': 355.2909668633681, 'average_query_length': 32.74418604651163, 'num_documents': 8812, 'num_queries': 43, 'average_relevant_docs_per_query': 95.3953488372093}} | | [MSMARCOv2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | None | None | | [MTOPDomainClassification](https://arxiv.org/pdf/2008.09335.pdf) | ['deu', 'eng', 'fra', 'hin', 'spa', 'tha'] | Classification | s2s | [Spoken, Spoken] | {'validation': 2235, 'test': 4386} | {'validation': {'num_samples': 10837, 'average_text_length': 39.85374181046415, 'unique_labels': 11, 'labels': {'1': {'count': 1688}, '10': {'count': 754}, '7': {'count': 849}, '3': {'count': 681}, '6': {'count': 985}, '2': {'count': 647}, '9': {'count': 872}, '0': {'count': 833}, '5': {'count': 1182}, '4': {'count': 982}, '8': {'count': 1364}}, 'hf_subset_descriptive_stats': {}, 'en': {'num_samples': 2235, 'average_text_length': 36.53825503355705, 'unique_labels': 11, 'labels': {'1': {'count': 329}, '10': {'count': 185}, '7': {'count': 183}, '3': {'count': 134}, '6': {'count': 186}, '2': {'count': 123}, '9': {'count': 196}, '0': {'count': 176}, '5': {'count': 228}, '4': {'count': 207}, '8': {'count': 288}}}, 'de': {'num_samples': 1815, 'average_text_length': 42.824793388429754, 'unique_labels': 11, 'labels': {'0': {'count': 99}, '1': {'count': 303}, '2': {'count': 104}, '3': {'count': 122}, '6': {'count': 165}, '4': {'count': 157}, '7': {'count': 141}, '5': {'count': 203}, '8': {'count': 220}, '10': {'count': 133}, '9': {'count': 168}}}, 'es': {'num_samples': 1527, 'average_text_length': 44.34839554682384, 'unique_labels': 11, 'labels': {'1': {'count': 197}, '6': {'count': 166}, '4': {'count': 138}, '10': {'count': 103}, '3': {'count': 104}, '5': {'count': 190}, '2': {'count': 115}, '8': {'count': 212}, '7': {'count': 82}, '9': {'count': 76}, '0': {'count': 144}}}, 'fr': {'num_samples': 1577, 'average_text_length': 43.12492073557387, 'unique_labels': 11, 'labels': {'0': {'count': 125}, '1': {'count': 278}, '2': {'count': 92}, '3': {'count': 89}, '4': {'count': 137}, '7': {'count': 145}, '6': {'count': 138}, '5': {'count': 168}, '8': {'count': 203}, '9': {'count': 124}, '10': {'count': 78}}}, 'hi': {'num_samples': 2012, 'average_text_length': 39.139662027833005, 'unique_labels': 11, 'labels': {'0': {'count': 161}, '1': {'count': 304}, '3': {'count': 126}, '4': {'count': 193}, '2': {'count': 109}, '10': {'count': 154}, '5': {'count': 208}, '6': {'count': 167}, '7': {'count': 172}, '8': {'count': 235}, '9': {'count': 183}}}, 'th': {'num_samples': 1671, 'average_text_length': 34.726511071214844, 'unique_labels': 11, 'labels': {'0': {'count': 128}, '1': {'count': 277}, '2': {'count': 104}, '3': {'count': 106}, '4': {'count': 150}, '5': {'count': 185}, '6': {'count': 163}, '7': {'count': 126}, '8': {'count': 206}, '9': {'count': 125}, '10': {'count': 101}}}}, 'test': {'num_samples': 19680, 'average_text_length': 39.71443089430894, 'unique_labels': 11, 'labels': {'2': {'count': 977}, '5': {'count': 2372}, '6': {'count': 2014}, '8': {'count': 2572}, '9': {'count': 1317}, '1': {'count': 3065}, '10': {'count': 1330}, '3': {'count': 1351}, '0': {'count': 1459}, '7': {'count': 1535}, '4': {'count': 1688}}, 'hf_subset_descriptive_stats': {}, 'en': {'num_samples': 4386, 'average_text_length': 36.79343365253078, 'unique_labels': 11, 'labels': {'2': {'count': 197}, '5': {'count': 487}, '6': {'count': 418}, '8': {'count': 613}, '9': {'count': 346}, '1': {'count': 613}, '10': {'count': 358}, '3': {'count': 290}, '0': {'count': 341}, '7': {'count': 354}, '4': {'count': 369}}}, 'de': {'num_samples': 3549, 'average_text_length': 42.67258382642998, 'unique_labels': 11, 'labels': {'0': {'count': 193}, '10': {'count': 264}, '1': {'count': 553}, '2': {'count': 163}, '3': {'count': 256}, '5': {'count': 439}, '4': {'count': 306}, '6': {'count': 353}, '7': {'count': 279}, '8': {'count': 452}, '9': {'count': 291}}}, 'es': {'num_samples': 2998, 'average_text_length': 43.552034689793196, 'unique_labels': 11, 'labels': {'1': {'count': 401}, '6': {'count': 352}, '4': {'count': 246}, '10': {'count': 206}, '3': {'count': 231}, '5': {'count': 404}, '2': {'count': 177}, '8': {'count': 435}, '7': {'count': 156}, '9': {'count': 126}, '0': {'count': 264}}}, 'fr': {'num_samples': 3193, 'average_text_length': 43.854995302223614, 'unique_labels': 11, 'labels': {'0': {'count': 253}, '1': {'count': 551}, '2': {'count': 159}, '3': {'count': 190}, '4': {'count': 280}, '6': {'count': 330}, '5': {'count': 356}, '7': {'count': 272}, '8': {'count': 462}, '10': {'count': 159}, '9': {'count': 181}}}, 'hi': {'num_samples': 2789, 'average_text_length': 37.395123700250984, 'unique_labels': 11, 'labels': {'0': {'count': 208}, '1': {'count': 470}, '5': {'count': 335}, '3': {'count': 195}, '4': {'count': 242}, '2': {'count': 132}, '6': {'count': 267}, '7': {'count': 262}, '8': {'count': 265}, '10': {'count': 186}, '9': {'count': 227}}}, 'th': {'num_samples': 2765, 'average_text_length': 33.94792043399638, 'unique_labels': 11, 'labels': {'0': {'count': 200}, '1': {'count': 477}, '2': {'count': 149}, '3': {'count': 189}, '4': {'count': 245}, '6': {'count': 294}, '5': {'count': 351}, '7': {'count': 212}, '8': {'count': 345}, '9': {'count': 146}, '10': {'count': 157}}}}, 'train': {'num_samples': 73928, 'average_text_length': 39.73095444215994, 'unique_labels': 11, 'labels': {'0': {'count': 5262}, '5': {'count': 8334}, '6': {'count': 6961}, '9': {'count': 5313}, '1': {'count': 11107}, '8': {'count': 9698}, '10': {'count': 5084}, '2': {'count': 4770}, '4': {'count': 6644}, '3': {'count': 5191}, '7': {'count': 5564}}, 'hf_subset_descriptive_stats': {}, 'en': {'num_samples': 15667, 'average_text_length': 36.57222186761984, 'unique_labels': 11, 'labels': {'0': {'count': 1165}, '5': {'count': 1657}, '6': {'count': 1402}, '9': {'count': 1303}, '1': {'count': 2187}, '8': {'count': 2157}, '10': {'count': 1219}, '2': {'count': 929}, '4': {'count': 1353}, '3': {'count': 1064}, '7': {'count': 1231}}}, 'de': {'num_samples': 13424, 'average_text_length': 43.226013110846246, 'unique_labels': 11, 'labels': {'0': {'count': 761}, '10': {'count': 996}, '4': {'count': 1185}, '1': {'count': 2016}, '7': {'count': 1029}, '5': {'count': 1484}, '2': {'count': 814}, '3': {'count': 980}, '6': {'count': 1265}, '8': {'count': 1767}, '9': {'count': 1127}}}, 'es': {'num_samples': 10934, 'average_text_length': 43.60691421254801, 'unique_labels': 11, 'labels': {'1': {'count': 1459}, '6': {'count': 1188}, '4': {'count': 928}, '10': {'count': 743}, '3': {'count': 830}, '5': {'count': 1396}, '2': {'count': 823}, '8': {'count': 1555}, '7': {'count': 525}, '9': {'count': 560}, '0': {'count': 927}}}, 'fr': {'num_samples': 11814, 'average_text_length': 43.594802776367025, 'unique_labels': 11, 'labels': {'0': {'count': 861}, '10': {'count': 668}, '1': {'count': 1968}, '7': {'count': 975}, '5': {'count': 1261}, '2': {'count': 799}, '3': {'count': 734}, '4': {'count': 1082}, '6': {'count': 1113}, '8': {'count': 1656}, '9': {'count': 697}}}, 'hi': {'num_samples': 11330, 'average_text_length': 37.592144748455425, 'unique_labels': 11, 'labels': {'0': {'count': 794}, '1': {'count': 1741}, '7': {'count': 974}, '2': {'count': 670}, '3': {'count': 831}, '5': {'count': 1272}, '6': {'count': 940}, '4': {'count': 1073}, '10': {'count': 786}, '8': {'count': 1281}, '9': {'count': 968}}}, 'th': {'num_samples': 10759, 'average_text_length': 34.04043126684636, 'unique_labels': 11, 'labels': {'0': {'count': 754}, '10': {'count': 672}, '1': {'count': 1736}, '7': {'count': 830}, '2': {'count': 735}, '3': {'count': 752}, '5': {'count': 1264}, '6': {'count': 1053}, '4': {'count': 1023}, '8': {'count': 1282}, '9': {'count': 658}}}}} | | [MTOPIntentClassification](https://arxiv.org/pdf/2008.09335.pdf) | ['deu', 'eng', 'fra', 'hin', 'spa', 'tha'] | Classification | s2s | [Spoken, Spoken] | {'validation': 2235, 'test': 4386} | {'validation': 36.5, 'test': 36.8} | @@ -328,6 +339,7 @@ The following tables give you an overview of the tasks in MTEB. | MintakaRetrieval | ['ara', 'deu', 'fra', 'hin', 'ita', 'jpn', 'por', 'spa'] | Retrieval | s2p | [Encyclopaedic, Written] | None | {'test': {'ar': {'average_document_length': 12.736418511066399, 'average_query_length': 55.275533363595095, 'num_documents': 1491, 'num_queries': 2203, 'average_relevant_docs_per_query': 1.0}, 'de': {'average_document_length': 14.40060422960725, 'average_query_length': 65.41322662173546, 'num_documents': 1655, 'num_queries': 2374, 'average_relevant_docs_per_query': 1.0}, 'es': {'average_document_length': 14.291789722386296, 'average_query_length': 64.88325082508251, 'num_documents': 1693, 'num_queries': 2424, 'average_relevant_docs_per_query': 1.0}, 'fr': {'average_document_length': 14.407234539089849, 'average_query_length': 68.88452088452088, 'num_documents': 1714, 'num_queries': 2442, 'average_relevant_docs_per_query': 1.0}, 'hi': {'average_document_length': 12.71038961038961, 'average_query_length': 58.404637247569184, 'num_documents': 770, 'num_queries': 1337, 'average_relevant_docs_per_query': 1.0}, 'it': {'average_document_length': 14.365985576923077, 'average_query_length': 64.39707724425887, 'num_documents': 1664, 'num_queries': 2395, 'average_relevant_docs_per_query': 1.0004175365344468}, 'ja': {'average_document_length': 9.167713567839195, 'average_query_length': 29.961937716262977, 'num_documents': 1592, 'num_queries': 2312, 'average_relevant_docs_per_query': 1.0}, 'pt': {'average_document_length': 14.244471744471744, 'average_query_length': 60.42225998300765, 'num_documents': 1628, 'num_queries': 2354, 'average_relevant_docs_per_query': 1.0004248088360237}}} | | [Moroco](https://huggingface.co/datasets/moroco) (Andrei M. Butnaru, 2019) | ['ron'] | Classification | s2s | [News, Written] | {'test': 2048} | {'test': 1710.94} | | [MovieReviewSentimentClassification](https://github.com/TheophileBlard/french-sentiment-analysis-with-bert) (Théophile Blard, 2020) | ['fra'] | Classification | s2s | [Reviews, Written] | {'validation': 1024, 'test': 1024} | {'validation': 550.3, 'test': 558.1} | +| [MrTidyRetrieval](https://huggingface.co/datasets/castorini/mr-tydi) (Xinyu Zhang, 2021) | ['ara', 'ben', 'eng', 'fin', 'ind', 'jpn', 'kor', 'rus', 'swa', 'tel', 'tha'] | Retrieval | s2p | [Encyclopaedic, Written] | | | | [MultiEURLEXMultilabelClassification](https://huggingface.co/datasets/coastalcph/multi_eurlex) (Chalkidis et al., 2021) | ['bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'est', 'fin', 'fra', 'hrv', 'hun', 'ita', 'lav', 'lit', 'mlt', 'nld', 'pol', 'por', 'ron', 'slk', 'slv', 'spa', 'swe'] | MultilabelClassification | p2p | [Legal, Government, Written] | {'test': 5000} | {'test': {'average_text_length': 12014.408930434782, 'average_label_per_text': 3.5938, 'num_samples': 115000, 'unique_labels': 21, 'labels': {'18': {'count': 50784}, '15': {'count': 30981}, '5': {'count': 24978}, '6': {'count': 45080}, '3': {'count': 63687}, '17': {'count': 37743}, '1': {'count': 15019}, '20': {'count': 14030}, '0': {'count': 17802}, '2': {'count': 22402}, '19': {'count': 10212}, '9': {'count': 3772}, '4': {'count': 9062}, '10': {'count': 7705}, '11': {'count': 12213}, '7': {'count': 14306}, '12': {'count': 11799}, '8': {'count': 13800}, '13': {'count': 2346}, '14': {'count': 4255}, '16': {'count': 1311}}, 'hf_subset_descriptive_stats': {'en': {'average_text_length': 11720.2926, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'de': {'average_text_length': 12865.4162, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'fr': {'average_text_length': 13081.1098, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'it': {'average_text_length': 12763.4786, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'es': {'average_text_length': 13080.29, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'pl': {'average_text_length': 12282.5926, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'ro': {'average_text_length': 12836.9322, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'nl': {'average_text_length': 12857.9742, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'el': {'average_text_length': 12998.143, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'hu': {'average_text_length': 12424.641, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'pt': {'average_text_length': 12482.4616, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'cs': {'average_text_length': 10783.4676, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'sv': {'average_text_length': 11612.4774, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'bg': {'average_text_length': 12235.4268, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'da': {'average_text_length': 11773.958, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'fi': {'average_text_length': 12087.6862, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'sk': {'average_text_length': 11130.814, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'lt': {'average_text_length': 11245.3566, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'hr': {'average_text_length': 11022.142, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'sl': {'average_text_length': 10620.0594, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'et': {'average_text_length': 10898.4312, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'lv': {'average_text_length': 10938.5102, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'mt': {'average_text_length': 12589.7442, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}}}} | | [MultiHateClassification](https://aclanthology.org/2022.woah-1.15/) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'nld', 'pol', 'por', 'spa'] | Classification | s2s | [Constructed, Written] | {'test': 10000} | {'test': 45.9} | | [MultiLongDocRetrieval](https://arxiv.org/abs/2402.03216) (Jianlv Chen, 2024) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'jpn', 'kor', 'por', 'rus', 'spa', 'tha'] | Retrieval | s2p | [Encyclopaedic, Written, Web, Non-fiction, Fiction] | None | {'dev': {'ar': {'average_document_length': 29234.48153016958, 'average_query_length': 69.27, 'num_documents': 7607, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'de': {'average_document_length': 33771.2111, 'average_query_length': 153.63, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'en': {'average_document_length': 13332.76764, 'average_query_length': 81.22, 'num_documents': 200000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'es': {'average_document_length': 36567.1736990891, 'average_query_length': 123.11, 'num_documents': 9551, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'fr': {'average_document_length': 36009.4934, 'average_query_length': 142.165, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'hi': {'average_document_length': 18688.50788229112, 'average_query_length': 77.995, 'num_documents': 3806, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'it': {'average_document_length': 36633.9969, 'average_query_length': 99.615, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ja': {'average_document_length': 14480.7508, 'average_query_length': 61.625, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ko': {'average_document_length': 13813.441224093263, 'average_query_length': 58.845, 'num_documents': 6176, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'pt': {'average_document_length': 32127.576952351956, 'average_query_length': 122.275, 'num_documents': 6569, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ru': {'average_document_length': 35934.8756, 'average_query_length': 87.875, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'th': {'average_document_length': 25993.2696, 'average_query_length': 107.81, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'zh': {'average_document_length': 6039.059725, 'average_query_length': 26.79, 'num_documents': 200000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}}, 'test': {'ar': {'average_document_length': 29234.48153016958, 'average_query_length': 75.77, 'num_documents': 7607, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'de': {'average_document_length': 33771.2111, 'average_query_length': 123.65, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'en': {'average_document_length': 13332.76764, 'average_query_length': 81.33, 'num_documents': 200000, 'num_queries': 800, 'average_relevant_docs_per_query': 1.0}, 'es': {'average_document_length': 36567.1736990891, 'average_query_length': 131.985, 'num_documents': 9551, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'fr': {'average_document_length': 36009.4934, 'average_query_length': 149.795, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'hi': {'average_document_length': 18688.50788229112, 'average_query_length': 103.76, 'num_documents': 3806, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'it': {'average_document_length': 36633.9969, 'average_query_length': 114.595, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ja': {'average_document_length': 14480.7508, 'average_query_length': 55.73, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ko': {'average_document_length': 13813.441224093263, 'average_query_length': 58.72, 'num_documents': 6176, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'pt': {'average_document_length': 32127.576952351956, 'average_query_length': 113.455, 'num_documents': 6569, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ru': {'average_document_length': 35934.8756, 'average_query_length': 94.87, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'th': {'average_document_length': 25993.2696, 'average_query_length': 97.99, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'zh': {'average_document_length': 6039.059725, 'average_query_length': 24.70875, 'num_documents': 200000, 'num_queries': 800, 'average_relevant_docs_per_query': 1.0}}} | @@ -341,13 +353,17 @@ The following tables give you an overview of the tasks in MTEB. | [NLPJournalTitleIntroRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Academic, Written] | {'test': 404} | {'test': {'average_document_length': 2052.8611111111113, 'average_query_length': 27.60891089108911, 'num_documents': 504, 'num_queries': 404, 'average_relevant_docs_per_query': 1.0}} | | [NQ](https://ai.google.com/research/NaturalQuestions/) (Tom Kwiatkowski, 2019) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 492.2287851281462, 'average_query_length': 48.17902665121669, 'num_documents': 2681468, 'num_queries': 3452, 'average_relevant_docs_per_query': 1.2169756662804172}} | | [NQ-PL](https://ai.google.com/research/NaturalQuestions/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | {'test': {'average_document_length': 502.14302128535564, 'average_query_length': 48.31662804171495, 'num_documents': 2681468, 'num_queries': 3452, 'average_relevant_docs_per_query': 1.2169756662804172}} | +| [NQ-PLHardNegatives](https://ai.google.com/research/NaturalQuestions/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | {'test': 1000} | {'test': {'average_document_length': 610.7449138094336, 'average_query_length': 48.381, 'num_documents': 184765, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.213}} | +| [NQHardNegatives](https://ai.google.com/research/NaturalQuestions/) (Tom Kwiatkowski, 2019) | ['eng'] | Retrieval | s2p | | {'test': 1000} | {'test': {'average_document_length': 602.7903551179953, 'average_query_length': 47.878, 'num_documents': 198779, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.213}} | | [NTREXBitextMining](https://huggingface.co/datasets/davidstap/NTREX) | ['afr', 'amh', 'arb', 'aze', 'bak', 'bel', 'bem', 'ben', 'bod', 'bos', 'bul', 'cat', 'ces', 'ckb', 'cym', 'dan', 'deu', 'div', 'dzo', 'ell', 'eng', 'eus', 'ewe', 'fao', 'fas', 'fij', 'fil', 'fin', 'fra', 'fuc', 'gle', 'glg', 'guj', 'hau', 'heb', 'hin', 'hmn', 'hrv', 'hun', 'hye', 'ibo', 'ind', 'isl', 'ita', 'jpn', 'kan', 'kat', 'kaz', 'khm', 'kin', 'kir', 'kmr', 'kor', 'lao', 'lav', 'lit', 'ltz', 'mal', 'mar', 'mey', 'mkd', 'mlg', 'mlt', 'mon', 'mri', 'msa', 'mya', 'nde', 'nep', 'nld', 'nno', 'nob', 'nso', 'nya', 'orm', 'pan', 'pol', 'por', 'prs', 'pus', 'ron', 'rus', 'shi', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'spa', 'sqi', 'srp', 'ssw', 'swa', 'swe', 'tah', 'tam', 'tat', 'tel', 'tgk', 'tha', 'tir', 'ton', 'tsn', 'tuk', 'tur', 'uig', 'ukr', 'urd', 'uzb', 'ven', 'vie', 'wol', 'xho', 'yor', 'yue', 'zho', 'zul'] | BitextMining | s2s | [News, Written] | {'test': 3826252} | {'test': 120} | | [NYSJudicialEthicsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 292} | {'test': 159.45} | | [NaijaSenti](https://github.com/hausanlp/NaijaSenti) | ['hau', 'ibo', 'pcm', 'yor'] | Classification | s2s | [Social, Written] | {'test': 4800} | {'test': 72.81} | | [NarrativeQARetrieval](https://metatext.io/datasets/narrativeqa) (Tomáš Kočiský, 2017) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 326753.5323943662, 'average_query_length': 47.730889457232166, 'num_documents': 355, 'num_queries': 10557, 'average_relevant_docs_per_query': 1.0}} | | [NepaliNewsClassification](https://github.com/goru001/nlp-for-nepali) | ['nep'] | Classification | s2s | [News, Written] | {'train': 5975, 'test': 1495} | {'train': 196.61, 'test': 196.017} | | [NeuCLIR2022Retrieval](https://neuclir.github.io/) (Lawrie et al., 2023) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'fas': 2232130, 'zho': 3179323, 'rus': 4627657} | {'test': {'fas': {'average_document_length': 2032.093148525817, 'average_query_length': 85.4298245614035, 'num_documents': 2232016, 'num_queries': 114, 'average_relevant_docs_per_query': 12.912280701754385}, 'rus': {'average_document_length': 1757.9129983233004, 'average_query_length': 85.58771929824562, 'num_documents': 4627543, 'num_queries': 114, 'average_relevant_docs_per_query': 16.57017543859649}, 'zho': {'average_document_length': 743.1426659901881, 'average_query_length': 24.17543859649123, 'num_documents': 3179209, 'num_queries': 114, 'average_relevant_docs_per_query': 18.710526315789473}}} | +| [NeuCLIR2022RetrievalHardNegatives](https://neuclir.github.io/) (Lawrie et al., 2023) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | None | {'test': {'average_document_length': 2066.9453653646488, 'average_query_length': 63.529411764705884, 'num_documents': 27931, 'num_queries': 136, 'average_relevant_docs_per_query': 40.39705882352941, 'hf_subset_descriptive_stats': {'fas': {'average_document_length': 2816.847782031074, 'average_query_length': 83.26666666666667, 'num_documents': 8882, 'num_queries': 45, 'average_relevant_docs_per_query': 32.71111111111111}, 'rus': {'average_document_length': 2446.5574277854193, 'average_query_length': 85.56818181818181, 'num_documents': 8724, 'num_queries': 44, 'average_relevant_docs_per_query': 42.93181818181818}, 'zho': {'average_document_length': 1101.0984987893462, 'average_query_length': 24.0, 'num_documents': 10325, 'num_queries': 47, 'average_relevant_docs_per_query': 45.38297872340426}}}} | | [NeuCLIR2023Retrieval](https://neuclir.github.io/) (Dawn Lawrie, 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'fas': 2232092, 'zho': 3179285, 'rus': 4627619} | {'test': {'fas': {'average_document_length': 2032.093148525817, 'average_query_length': 65.48684210526316, 'num_documents': 2232016, 'num_queries': 76, 'average_relevant_docs_per_query': 66.28947368421052}, 'rus': {'average_document_length': 1757.9129983233004, 'average_query_length': 74.4342105263158, 'num_documents': 4627543, 'num_queries': 76, 'average_relevant_docs_per_query': 62.223684210526315}, 'zho': {'average_document_length': 743.1426659901881, 'average_query_length': 22.210526315789473, 'num_documents': 3179209, 'num_queries': 76, 'average_relevant_docs_per_query': 53.68421052631579}}} | +| [NeuCLIR2023RetrievalHardNegatives](https://neuclir.github.io/) (Dawn Lawrie, 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | None | {'test': {'average_document_length': 2236.175955333482, 'average_query_length': 54.10267857142857, 'num_documents': 49433, 'num_queries': 224, 'average_relevant_docs_per_query': 61.816964285714285, 'hf_subset_descriptive_stats': {'fas': {'average_document_length': 2895.869857421016, 'average_query_length': 65.89189189189189, 'num_documents': 15921, 'num_queries': 74, 'average_relevant_docs_per_query': 68.08108108108108}, 'rus': {'average_document_length': 2724.294762109928, 'average_query_length': 74.41333333333333, 'num_documents': 16247, 'num_queries': 75, 'average_relevant_docs_per_query': 63.053333333333335}, 'zho': {'average_document_length': 1168.4984071821605, 'average_query_length': 22.16, 'num_documents': 17265, 'num_queries': 75, 'average_relevant_docs_per_query': 54.4}}}} | | [News21InstructionRetrieval](https://arxiv.org/abs/2403.15246) (Orion Weller, 2024) | ['eng'] | InstructionRetrieval | s2p | [News, Written] | {'eng': 61906} | {'eng': 2983.724665391969} | | [NewsClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [News, Written] | {'test': 7600} | {'test': 235.29} | | [NoRecClassification](https://aclanthology.org/L18-1661/) | ['nob'] | Classification | s2s | [Written, Reviews] | {'test': 2050} | {'test': 82} | @@ -398,7 +414,9 @@ The following tables give you an overview of the tasks in MTEB. | [QBQTC](https://github.com/CLUEbenchmark/QBQTC/tree/main/dataset) | ['cmn'] | STS | s2s | | None | None | | [Quail](https://text-machine.cs.uml.edu/lab2/projects/quail/) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 2720} | {'test': {'average_document_length': 27.50788422240522, 'average_query_length': 1957.3632352941177, 'num_documents': 32787, 'num_queries': 2720, 'average_relevant_docs_per_query': 1.0}} | | [Quora-PL](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2s | | None | {'validation': {'average_document_length': 65.82473022253414, 'average_query_length': 54.6006, 'num_documents': 522931, 'num_queries': 5000, 'average_relevant_docs_per_query': 1.5252}, 'test': {'average_document_length': 65.82473022253414, 'average_query_length': 54.5354, 'num_documents': 522931, 'num_queries': 10000, 'average_relevant_docs_per_query': 1.5675}} | +| [Quora-PLHardNegatives](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2s | | {'test': 1000} | {'test': {'average_document_length': 67.77529631287385, 'average_query_length': 53.846, 'num_documents': 172031, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.641}} | | [QuoraRetrieval](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (DataCanary et al., 2017) | ['eng'] | Retrieval | s2s | | None | {'dev': {'average_document_length': 62.158154708747425, 'average_query_length': 51.5342, 'num_documents': 522931, 'num_queries': 5000, 'average_relevant_docs_per_query': 1.5252}, 'test': {'average_document_length': 62.158154708747425, 'average_query_length': 51.5396, 'num_documents': 522931, 'num_queries': 10000, 'average_relevant_docs_per_query': 1.5675}} | +| [QuoraRetrievalHardNegatives](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (DataCanary et al., 2017) | ['eng'] | Retrieval | s2s | | {'test': 1000} | {'test': {'average_document_length': 58.96963812985781, 'average_query_length': 51.228, 'num_documents': 177163, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.641}} | | [RARbCode](https://arxiv.org/abs/2404.06347) (Xiao et al., 2024) | ['eng'] | Retrieval | s2p | [Programming, Written] | {'test': 1484} | {'test': {'average_document_length': 793.6813076734267, 'average_query_length': 375.7506738544474, 'num_documents': 301482, 'num_queries': 1484, 'average_relevant_docs_per_query': 1.0}} | | [RARbMath](https://arxiv.org/abs/2404.06347) (Xiao et al., 2024) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 6319} | {'test': {'average_document_length': 504.0197829347469, 'average_query_length': 210.30732710871973, 'num_documents': 389376, 'num_queries': 6319, 'average_relevant_docs_per_query': 1.0}} | | [RTE3](https://aclanthology.org/W07-1401/) | ['deu', 'eng', 'fra', 'ita'] | PairClassification | s2s | [News, Web, Encyclopaedic, Written] | {'test': 1923} | {'test': 124.79} | @@ -407,6 +425,7 @@ The following tables give you an overview of the tasks in MTEB. | [RedditClusteringP2P.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | p2p | [Web, Social, Written] | {'test': 18375} | {'test': 727.7} | | [RestaurantReviewSentimentClassification](https://link.springer.com/chapter/10.1007/978-3-319-18117-2_2) (ElSahar et al., 2015) | ['ara'] | Classification | s2s | [Reviews, Written] | {'train': 2048} | {'train': 231.4} | | [RiaNewsRetrieval](https://arxiv.org/abs/1901.07786) (Gavrilov et al., 2019) | ['rus'] | Retrieval | s2p | [News, Written] | {'test': 10000} | {'test': {'average_document_length': 1165.6429557148213, 'average_query_length': 62.4029, 'num_documents': 704344, 'num_queries': 10000, 'average_relevant_docs_per_query': 1.0}} | +| [RiaNewsRetrievalHardNegatives](https://arxiv.org/abs/1901.07786) (Gavrilov et al., 2019) | ['rus'] | Retrieval | s2p | [News, Written] | {'test': 1000} | {'test': {'average_document_length': 1225.7253146619116, 'average_query_length': 62.338, 'num_documents': 191237, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}} | | [Robust04InstructionRetrieval](https://arxiv.org/abs/2403.15246) (Orion Weller, 2024) | ['eng'] | InstructionRetrieval | s2p | [News, Written] | {'eng': 95088} | {'eng': 2471.0398058252426} | | [RomaTalesBitextMining](https://idoc.pub/documents/idocpub-zpnxm9g35ylv) | ['hun', 'rom'] | BitextMining | s2s | [Fiction, Written] | {'test': 215} | {'test': 316.8046511627907} | | [RomaniBibleClustering](https://romani.global.bible/info) | ['rom'] | Clustering | p2p | [Religious, Written] | {'test': 2048} | {'test': 132.2} | @@ -442,6 +461,7 @@ The following tables give you an overview of the tasks in MTEB. | [SICK-R-PL](https://aclanthology.org/2020.lrec-1.207) | ['pol'] | STS | s2s | [Web, Written] | {'test': 9812} | {'test': 42.8} | | [SICKFr](https://huggingface.co/datasets/Lajavaness/SICK-fr) | ['fra'] | STS | s2s | | None | None | | [SIQA](https://leaderboard.allenai.org/socialiqa/submissions/get-started) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 0} | {'test': {'average_document_length': 22.967085695044617, 'average_query_length': 127.75383828045035, 'num_documents': 71276, 'num_queries': 1954, 'average_relevant_docs_per_query': 1.0}} | +| [SKQuadRetrieval](https://huggingface.co/datasets/TUKE-KEMT/retrieval-skquad) | ['slk'] | Retrieval | s2s | [Encyclopaedic] | {'test': 1134} | {'test': {'average_document_length': 1180.5071792496526, 'average_query_length': 53.63403880070547, 'num_documents': 6477, 'num_queries': 1134, 'average_relevant_docs_per_query': 11}} | | [SNLHierarchicalClusteringP2P](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | p2p | [Encyclopaedic, Non-fiction, Written] | {'test': 1300} | {'test': 1986.9453846153847} | | [SNLHierarchicalClusteringS2S](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | s2s | [Encyclopaedic, Non-fiction, Written] | {'test': 1300} | {'test': 242.22384615384615} | | [SNLRetrieval](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Written] | {'test': 2048} | {'test': {'average_document_length': 1986.9453846153847, 'average_query_length': 14.906153846153845, 'num_documents': 1300, 'num_queries': 1300, 'average_relevant_docs_per_query': 1.0}} | @@ -469,6 +489,7 @@ The following tables give you an overview of the tasks in MTEB. | [SinhalaNewsClassification](https://huggingface.co/datasets/NLPC-UOM/Sinhala-News-Category-classification) (Nisansa de Silva, 2015) | ['sin'] | Classification | s2s | [News, Written] | {'train': 3327} | {'train': 148.04} | | [SinhalaNewsSourceClassification](https://huggingface.co/datasets/NLPC-UOM/Sinhala-News-Source-classification) (Dhananjaya et al., 2022) | ['sin'] | Classification | s2s | [News, Written] | {'train': 24094} | {'train': 56.08} | | [SiswatiNewsClassification](https://huggingface.co/datasets/dsfsi/za-isizulu-siswati-news) (Madodonga et al., 2023) | ['ssw'] | Classification | s2s | [News, Written] | {'train': 80} | {'train': 354.2} | +| [SlovakHateSpeechClassification](https://huggingface.co/datasets/TUKE-KEMT/hate_speech_slovak) | ['slk'] | Classification | s2s | [Social, Written] | {'test': 1319} | {'test': 92.71} | | [SlovakMovieReviewSentimentClassification](https://arxiv.org/pdf/2304.01922) ({ {S, 2023) | ['svk'] | Classification | s2s | [Reviews, Written] | {'test': 2048} | {'test': 366.17} | | [SlovakSumRetrieval](https://huggingface.co/datasets/NaiveNeuron/slovaksum) | ['slk'] | Retrieval | s2s | [News, Social, Web, Written] | {'test': 600} | {'test': {'average_document_length': 2156.445, 'average_query_length': 143.59833333333333, 'num_documents': 600, 'num_queries': 600, 'average_relevant_docs_per_query': 1.0}} | | [SouthAfricanLangClassification](https://www.kaggle.com/competitions/south-african-language-identification/) (ExploreAI Academy et al., 2022) | ['afr', 'eng', 'nbl', 'nso', 'sot', 'ssw', 'tsn', 'tso', 'ven', 'xho', 'zul'] | Classification | s2s | [Web, Non-fiction, Written] | {'test': 2048} | {'test': 247.49} | @@ -524,7 +545,8 @@ The following tables give you an overview of the tasks in MTEB. | [ThuNewsClusteringP2P.v2](http://thuctc.thunlp.org/) (Sun et al., 2016) | ['cmn'] | Clustering | p2p | [News, Written] | {'test': 2048} | {} | | [ThuNewsClusteringS2S.v2](http://thuctc.thunlp.org/) (Sun et al., 2016) | ['cmn'] | Clustering | s2s | [News, Written] | {'test': 2048} | {} | | [TopiOCQA](https://mcgill-nlp.github.io/topiocqa) (Vaibhav Adlakha, 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | {'dev': 2514} | {'validation': {'average_document_length': 478.8968086416064, 'average_query_length': 12.579952267303103, 'num_documents': 25700592, 'num_queries': 2514, 'average_relevant_docs_per_query': 1.0}} | -| [Touche2020](https://webis.de/events/touche-20/shared-task-1.html) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1719.3347658445412, 'average_query_length': 43.42857142857143, 'num_documents': 382545, 'num_queries': 49, 'average_relevant_docs_per_query': 19.020408163265305}} | +| [TopiOCQAHardNegatives](https://mcgill-nlp.github.io/topiocqa) (Vaibhav Adlakha, 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 1000} | {'validation': {'average_document_length': 538.7586536643946, 'average_query_length': 12.85, 'num_documents': 89933, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}} | +| [Touche2020Retrieval.v3](https://github.com/castorini/touche-error-analysis) | ['eng'] | Retrieval | s2p | [Academic] | | | | [ToxicChatClassification](https://aclanthology.org/2023.findings-emnlp.311/) (Zi Lin, 2023) | ['eng'] | Classification | s2s | [Constructed, Written] | {'test': 1427} | {'test': 189.4} | | [ToxicConversationsClassification](https://www.kaggle.com/competitions/jigsaw-unintended-bias-in-toxicity-classification/overview) (cjadams, 2019) | ['eng'] | Classification | s2s | [Social, Written] | {'test': 50000} | {'test': 296.6} | | [TswanaNewsClassification](https://link.springer.com/chapter/10.1007/978-3-031-49002-6_17) (Vukosi Marivate, 2023) | ['tsn'] | Classification | s2s | [News, Written] | {'validation': 487, 'test': 487} | {'validation': 2417.72, 'test': 2369.52} | @@ -571,6 +593,8 @@ The following tables give you an overview of the tasks in MTEB. | [YelpReviewFullClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [Reviews, Written] | {'test': 50000} | {} | | [YueOpenriceReviewClassification](https://github.com/Christainx/Dataset_Cantonese_Openrice) (Xiang et al., 2019) | ['yue'] | Classification | s2s | [Reviews, Spoken] | {'test': 6161} | {'test': 173.0} | | [indonli](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) | ['ind'] | PairClassification | s2s | [Encyclopaedic, Web, News, Written] | {'test_expert': 2040} | {'test_expert': 145.88} | +| [mFollowIRCrossLingualInstructionRetrieval](https://neuclir.github.io/) (Weller et al., 2024) | ['eng', 'fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'eng-fas': 80, 'eng-rus': 80, 'eng-zho': 86} | {'test': {'num_docs': 121635, 'num_queries': 123, 'average_document_length': 2331.0777818884367, 'average_query_length': 81.8780487804878, 'average_instruction_length': 389.9512195121951, 'average_changed_instruction_length': 450.5528455284553, 'average_relevant_docs_per_query': 10.30952380952381, 'average_top_ranked_per_query': 1024.3902439024391, 'hf_subset_descriptive_stats': {'eng-fas': {'num_docs': 41189, 'num_queries': 40, 'average_document_length': 3145.4990895627475, 'average_query_length': 80.075, 'average_instruction_length': 396.875, 'average_changed_instruction_length': 463.175, 'average_relevant_docs_per_query': 10.465116279069768, 'average_top_ranked_per_query': 1075}, 'eng-rus': {'num_docs': 39326, 'num_queries': 40, 'average_document_length': 2784.0813456746173, 'average_query_length': 81.875, 'average_instruction_length': 371.125, 'average_changed_instruction_length': 431.8, 'average_relevant_docs_per_query': 9.775, 'average_top_ranked_per_query': 1000}, 'eng-zho': {'num_docs': 41120, 'num_queries': 43, 'average_document_length': 1082.0501215953307, 'average_query_length': 83.55813953488372, 'average_instruction_length': 401.0232558139535, 'average_changed_instruction_length': 456.25581395348837, 'average_relevant_docs_per_query': 10.651162790697674, 'average_top_ranked_per_query': 1000}}}} | +| [mFollowIRInstructionRetrieval](https://neuclir.github.io/) (Weller et al., 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'fas': 80, 'rus': 80, 'zho': 86} | {'test': {'num_docs': 121635, 'num_queries': 123, 'average_document_length': 2331.0777818884367, 'average_query_length': 57.113821138211385, 'average_instruction_length': 281.0650406504065, 'average_changed_instruction_length': 326.9430894308943, 'average_relevant_docs_per_query': 10.30952380952381, 'average_top_ranked_per_query': 1024.3902439024391, 'hf_subset_descriptive_stats': {'fas': {'num_docs': 41189, 'num_queries': 40, 'average_document_length': 3145.4990895627475, 'average_query_length': 72.65, 'average_instruction_length': 358.925, 'average_changed_instruction_length': 415.325, 'average_relevant_docs_per_query': 10.465116279069768, 'average_top_ranked_per_query': 1075}, 'rus': {'num_docs': 39326, 'num_queries': 40, 'average_document_length': 2784.0813456746173, 'average_query_length': 77.5, 'average_instruction_length': 387, 'average_changed_instruction_length': 458, 'average_relevant_docs_per_query': 9.775, 'average_top_ranked_per_query': 1000}, 'zho': {'num_docs': 41120, 'num_queries': 43, 'average_document_length': 1082.0501215953307, 'average_query_length': 23.697674418604652, 'average_instruction_length': 110.09302325581395, 'average_changed_instruction_length': 122.81395348837209, 'average_relevant_docs_per_query': 10.651162790697674, 'average_top_ranked_per_query': 1000}}}} |
@@ -646,7 +670,7 @@ The following tables give you an overview of the tasks in MTEB. | apu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | apw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | apz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ara | 2 | 12 | 0 | 0 | 0 | 2 | 1 | 7 | 2 | 0 | 0 | +| ara | 2 | 12 | 0 | 0 | 0 | 2 | 1 | 9 | 2 | 0 | 0 | | arb | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 2 | 1 | 0 | 0 | | are | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | arl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -693,7 +717,7 @@ The following tables give you an overview of the tasks in MTEB. | bef | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | bel | 4 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | bem | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ben | 7 | 9 | 2 | 0 | 0 | 1 | 2 | 4 | 1 | 0 | 0 | +| ben | 7 | 9 | 2 | 0 | 0 | 1 | 2 | 6 | 1 | 0 | 0 | | beo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | ber | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | beu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -835,7 +859,7 @@ The following tables give you an overview of the tasks in MTEB. | dah | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | dan | 5 | 9 | 2 | 0 | 1 | 0 | 1 | 5 | 0 | 0 | 0 | | ded | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| deu | 6 | 14 | 7 | 0 | 1 | 6 | 2 | 17 | 4 | 0 | 0 | +| deu | 6 | 14 | 7 | 0 | 1 | 6 | 2 | 18 | 4 | 0 | 0 | | dgc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | dgr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | dgz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -863,7 +887,7 @@ The following tables give you an overview of the tasks in MTEB. | ell | 3 | 6 | 1 | 0 | 1 | 2 | 0 | 3 | 0 | 0 | 0 | | emi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | emp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| eng | 16 | 143 | 16 | 3 | 1 | 8 | 7 | 80 | 13 | 2 | 1 | +| eng | 16 | 143 | 16 | 3 | 1 | 8 | 8 | 91 | 13 | 2 | 1 | | enq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | epo | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | eri | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -877,14 +901,14 @@ The following tables give you an overview of the tasks in MTEB. | fai | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | fao | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | | far | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fas | 1 | 4 | 0 | 0 | 0 | 1 | 2 | 4 | 0 | 0 | 0 | +| fas | 1 | 4 | 0 | 0 | 0 | 1 | 2 | 9 | 0 | 0 | 0 | | ffm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | fij | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | fil | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fin | 3 | 5 | 1 | 0 | 1 | 1 | 2 | 3 | 1 | 0 | 0 | +| fin | 3 | 5 | 1 | 0 | 1 | 1 | 2 | 5 | 1 | 0 | 0 | | fon | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | for | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fra | 7 | 13 | 8 | 0 | 1 | 5 | 3 | 13 | 4 | 0 | 1 | +| fra | 7 | 13 | 8 | 0 | 1 | 5 | 3 | 14 | 4 | 0 | 1 | | fry | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | fuc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | fue | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -941,7 +965,7 @@ The following tables give you an overview of the tasks in MTEB. | hch | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | heb | 4 | 5 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | | heg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hin | 9 | 12 | 2 | 0 | 0 | 1 | 2 | 9 | 2 | 0 | 0 | +| hin | 9 | 12 | 2 | 0 | 0 | 1 | 2 | 10 | 2 | 0 | 0 | | hix | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | hla | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | hlt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -973,7 +997,7 @@ The following tables give you an overview of the tasks in MTEB. | imo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | ina | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | inb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ind | 6 | 7 | 1 | 0 | 0 | 1 | 1 | 2 | 1 | 0 | 0 | +| ind | 6 | 7 | 1 | 0 | 0 | 1 | 1 | 4 | 1 | 0 | 0 | | ino | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | iou | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | ipi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -990,7 +1014,7 @@ The following tables give you an overview of the tasks in MTEB. | jid | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | jiv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | jni | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| jpn | 5 | 8 | 3 | 0 | 0 | 1 | 2 | 10 | 2 | 0 | 0 | +| jpn | 5 | 8 | 3 | 0 | 0 | 1 | 3 | 13 | 2 | 0 | 0 | | jvn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | kab | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | kac | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | @@ -1046,7 +1070,7 @@ The following tables give you an overview of the tasks in MTEB. | knj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | knv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | kon | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kor | 4 | 8 | 1 | 0 | 1 | 2 | 1 | 6 | 3 | 0 | 0 | +| kor | 4 | 8 | 1 | 0 | 1 | 2 | 1 | 8 | 3 | 0 | 0 | | kos | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | kpf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | kpg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -1322,7 +1346,7 @@ The following tables give you an overview of the tasks in MTEB. | poe | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | poh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | poi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pol | 4 | 11 | 4 | 0 | 1 | 4 | 0 | 13 | 4 | 0 | 0 | +| pol | 4 | 11 | 4 | 0 | 1 | 4 | 0 | 18 | 4 | 0 | 0 | | pon | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | por | 4 | 9 | 1 | 0 | 2 | 2 | 1 | 5 | 3 | 0 | 0 | | poy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -1370,7 +1394,7 @@ The following tables give you an overview of the tasks in MTEB. | ruf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | rug | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | run | 1 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rus | 5 | 13 | 6 | 0 | 2 | 4 | 2 | 9 | 4 | 0 | 0 | +| rus | 5 | 13 | 6 | 0 | 2 | 4 | 2 | 16 | 4 | 0 | 0 | | rwo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | sab | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | sag | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -1393,7 +1417,7 @@ The following tables give you an overview of the tasks in MTEB. | sim | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | sin | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | | sja | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| slk | 3 | 3 | 1 | 0 | 1 | 0 | 0 | 2 | 0 | 0 | 0 | +| slk | 3 | 4 | 1 | 0 | 1 | 0 | 0 | 3 | 0 | 0 | 0 | | sll | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | slv | 3 | 4 | 1 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | | smk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -1409,7 +1433,7 @@ The following tables give you an overview of the tasks in MTEB. | soq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | sot | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | | soy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| spa | 4 | 13 | 4 | 0 | 1 | 2 | 1 | 11 | 4 | 0 | 0 | +| spa | 4 | 13 | 4 | 0 | 1 | 2 | 2 | 12 | 4 | 0 | 0 | | spl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | spm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | spp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -1433,7 +1457,7 @@ The following tables give you an overview of the tasks in MTEB. | sus | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | suz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | svk | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| swa | 1 | 7 | 2 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | +| swa | 1 | 7 | 2 | 0 | 0 | 1 | 1 | 3 | 0 | 0 | 0 | | swe | 4 | 8 | 3 | 0 | 1 | 1 | 1 | 4 | 0 | 0 | 0 | | swg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | swh | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | @@ -1458,7 +1482,7 @@ The following tables give you an overview of the tasks in MTEB. | tcz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | tdt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | tee | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tel | 7 | 7 | 2 | 0 | 0 | 0 | 1 | 3 | 2 | 0 | 0 | +| tel | 7 | 7 | 2 | 0 | 0 | 0 | 1 | 5 | 2 | 0 | 0 | | ter | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | tet | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | tew | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -1467,7 +1491,7 @@ The following tables give you an overview of the tasks in MTEB. | tgl | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | | tgo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | tgp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tha | 4 | 8 | 1 | 0 | 0 | 1 | 1 | 4 | 0 | 0 | 0 | +| tha | 4 | 8 | 1 | 0 | 0 | 1 | 1 | 6 | 0 | 0 | 0 | | tif | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | tim | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | tir | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | @@ -1591,7 +1615,7 @@ The following tables give you an overview of the tasks in MTEB. | yle | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | yml | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | yon | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yor | 4 | 5 | 3 | 0 | 0 | 0 | 1 | 2 | 0 | 0 | 0 | +| yor | 4 | 5 | 3 | 0 | 0 | 0 | 1 | 3 | 0 | 0 | 0 | | yrb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | yre | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | yss | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -1616,7 +1640,7 @@ The following tables give you an overview of the tasks in MTEB. | zaw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | zca | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | zga | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zho | 2 | 2 | 1 | 0 | 0 | 1 | 1 | 8 | 0 | 0 | 0 | +| zho | 2 | 2 | 1 | 0 | 0 | 1 | 1 | 13 | 0 | 0 | 0 | | zia | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | ziw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | zlm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -1635,7 +1659,7 @@ The following tables give you an overview of the tasks in MTEB. | zty | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | zul | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | | zyp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| Total | 1394 | 794 | 304 | 3 | 28 | 67 | 47 | 398 | 85 | 2 | 2 | +| Total | 1394 | 795 | 304 | 3 | 28 | 67 | 50 | 456 | 85 | 2 | 2 |
diff --git a/mteb/__init__.py b/mteb/__init__.py index be5edd97ed..bef5f7408d 100644 --- a/mteb/__init__.py +++ b/mteb/__init__.py @@ -2,7 +2,7 @@ from importlib.metadata import version -from mteb.benchmarks import ( +from mteb.benchmarks.benchmarks import ( MTEB_MAIN_EN, MTEB_MAIN_RU, MTEB_RETRIEVAL_LAW, @@ -10,11 +10,12 @@ CoIR, ) from mteb.evaluation import * -from mteb.load_results import load_results -from mteb.models import get_model, get_model_meta +from mteb.load_results import BenchmarkResults, load_results +from mteb.models import get_model, get_model_meta, get_model_metas from mteb.overview import TASKS_REGISTRY, get_task, get_tasks -from .benchmarks import Benchmark +from .benchmarks.benchmarks import Benchmark +from .benchmarks.get_benchmark import get_benchmark, get_benchmarks __version__ = version("mteb") # fetch version from install metadata @@ -30,6 +31,10 @@ "get_task", "get_model", "get_model_meta", + "get_model_metas", "load_results", "Benchmark", + "get_benchmark", + "get_benchmarks", + "BenchmarkResults", ] diff --git a/mteb/__main__.py b/mteb/__main__.py new file mode 100644 index 0000000000..709f6d4345 --- /dev/null +++ b/mteb/__main__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from mteb.cli import main + +main() diff --git a/mteb/abstasks/AbsTask.py b/mteb/abstasks/AbsTask.py index 928020ff10..5fa983bbe5 100644 --- a/mteb/abstasks/AbsTask.py +++ b/mteb/abstasks/AbsTask.py @@ -3,7 +3,8 @@ import logging import random from abc import ABC, abstractmethod -from typing import Any, Dict, Sequence, TypedDict +from collections.abc import Sequence +from typing import Any, TypedDict import datasets import numpy as np @@ -14,12 +15,12 @@ from mteb.abstasks.stratification import _iterative_train_test_split from mteb.abstasks.TaskMetadata import HFSubset, TaskMetadata -from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from mteb.encoder_interface import Encoder from mteb.languages import LanguageScripts logger = logging.getLogger(__name__) -ScoresDict = Dict[str, Any] +ScoresDict = dict[str, Any] # ^ e.g {'main_score': 0.5, 'hf_subset': 'en-de', 'languages': ['eng-Latn', 'deu-Latn']} @@ -60,12 +61,13 @@ class DescriptiveStatistics(TypedDict): class AbsTask(ABC): metadata: TaskMetadata + _eval_splits: list[str] | None = None superseded_by: None | str = None + dataset: dict[HFSubset, DatasetDict] | None = None # type: ignore + data_loaded: bool = False + is_multilingual: bool = False def __init__(self, seed: int = 42, **kwargs: Any): - self.dataset = None - self.data_loaded = False - self.is_multilingual = False self.save_suffix = kwargs.get("save_suffix", "") self.seed = seed @@ -89,7 +91,7 @@ def dataset_transform(self): def evaluate( self, - model: Encoder | EncoderWithQueryCorpusEncode, + model: Encoder, split: str = "test", *, encode_kwargs: dict[str, Any] = {}, @@ -255,6 +257,11 @@ def languages(self) -> list[str]: return self.metadata.languages + def filter_eval_splits(self, eval_splits: list[str] | None) -> AbsTask: + """Filter the evaluation splits of the task.""" + self._eval_splits = eval_splits + return self + def filter_languages( self, languages: list[str] | None, script: list[str] | None = None ) -> AbsTask: @@ -285,6 +292,12 @@ def filter_languages( self.hf_subsets = subsets_to_keep return self + @property + def eval_splits(self) -> list[str]: + if self._eval_splits: + return self._eval_splits + return self.metadata.eval_splits + def __repr__(self) -> str: """Format the representation of the task such that it appears as: @@ -297,3 +310,6 @@ def __repr__(self) -> str: return ( f"{self.__class__.__name__}(name='{self.metadata.name}', languages={langs})" ) + + def __hash__(self) -> int: + return hash(self.metadata) diff --git a/mteb/abstasks/AbsTaskBitextMining.py b/mteb/abstasks/AbsTaskBitextMining.py index 973d69ee7f..ea4667d9de 100644 --- a/mteb/abstasks/AbsTaskBitextMining.py +++ b/mteb/abstasks/AbsTaskBitextMining.py @@ -8,7 +8,7 @@ from mteb.encoder_interface import Encoder from ..evaluation.evaluators import BitextMiningEvaluator -from ..load_results.mteb_results import HFSubset, ScoresDict +from ..load_results.task_results import HFSubset, ScoresDict from .AbsTask import AbsTask, DescriptiveStatistics logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ class AbsTaskBitextMining(AbsTask): """Abstract class for BitextMining tasks The similarity is computed between pairs and the results are ranked. - self.load_data() must generate a huggingface dataset with a split matching self.metadata_dict["eval_splits"], and assign it to self.dataset. It must contain the following columns: + self.load_data() must generate a huggingface dataset with a split matching self.metadata.eval_splits, and assign it to self.dataset. It must contain the following columns: id: str sentence1: str sentence2: str diff --git a/mteb/abstasks/AbsTaskClassification.py b/mteb/abstasks/AbsTaskClassification.py index 36c0a76b96..eacd25af42 100644 --- a/mteb/abstasks/AbsTaskClassification.py +++ b/mteb/abstasks/AbsTaskClassification.py @@ -14,7 +14,7 @@ kNNClassificationEvaluatorPytorch, logRegClassificationEvaluator, ) -from ..load_results.mteb_results import HFSubset, ScoresDict +from ..load_results.task_results import HFSubset, ScoresDict from .AbsTask import AbsTask, DescriptiveStatistics logger = logging.getLogger(__name__) @@ -216,7 +216,7 @@ def calculate_metadata_metrics( ) for split in pbar_split: pbar_split.set_postfix_str(f"Split: {split}") - print(f"Processing metadata for split {split}") + logger.info(f"Processing metadata for split {split}") if self.is_multilingual: all_details[split] = self._calculate_metrics_from_split( split, compute_overall=True @@ -228,7 +228,7 @@ def calculate_metadata_metrics( ) for hf_subset in pbar_subset: pbar_subset.set_postfix_str(f"Language: {hf_subset}") - print(f"Processing metadata for language {hf_subset}") + logger.info(f"Processing metadata for language {hf_subset}") split_details = self._calculate_metrics_from_split(split, hf_subset) all_details[split][hf_subset] = split_details else: diff --git a/mteb/abstasks/AbsTaskClustering.py b/mteb/abstasks/AbsTaskClustering.py index 87113b2b26..8aaff2a484 100644 --- a/mteb/abstasks/AbsTaskClustering.py +++ b/mteb/abstasks/AbsTaskClustering.py @@ -8,8 +8,8 @@ import tqdm from datasets import Dataset -from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode -from mteb.load_results.mteb_results import ScoresDict +from mteb.encoder_interface import Encoder +from mteb.load_results.task_results import ScoresDict from ..evaluation.evaluators import ClusteringEvaluator from .AbsTask import AbsTask, DescriptiveStatistics @@ -52,7 +52,7 @@ def _add_main_score(self, scores) -> None: def _evaluate_subset( self, - model: EncoderWithQueryCorpusEncode | Encoder, + model: Encoder, dataset: Dataset, *, encode_kwargs: dict[str, Any] = {}, diff --git a/mteb/abstasks/AbsTaskClusteringFast.py b/mteb/abstasks/AbsTaskClusteringFast.py index 4afa632307..22ed1bb693 100644 --- a/mteb/abstasks/AbsTaskClusteringFast.py +++ b/mteb/abstasks/AbsTaskClusteringFast.py @@ -4,7 +4,7 @@ import logging import random from collections import Counter, defaultdict -from typing import Any, Dict +from typing import Any import numpy as np import sklearn @@ -13,15 +13,15 @@ from sklearn.metrics.cluster import v_measure_score from mteb.encoder_interface import Encoder +from mteb.normalize_embeddings import normalize_embeddings_to_numpy -from ..evaluation.evaluators.model_encode import model_encode -from ..load_results.mteb_results import HFSubset +from ..load_results.task_results import HFSubset from .AbsTask import AbsTask, DescriptiveStatistics logger = logging.getLogger(__name__) -MultilingualDataset = Dict[HFSubset, DatasetDict] +MultilingualDataset = dict[HFSubset, DatasetDict] def evaluate_clustering_bootstrapped( @@ -174,11 +174,12 @@ def _evaluate_subset( ) downsampled_dataset = dataset.select(example_indices) # type: ignore - embeddings = model_encode( - downsampled_dataset["sentences"], # type: ignore - model=model, - prompt_name=self.metadata.name, - **encode_kwargs, + embeddings = normalize_embeddings_to_numpy( + model.encode( + downsampled_dataset["sentences"], # type: ignore + task_name=self.metadata.name, + **encode_kwargs, + ) ) labels = [] diff --git a/mteb/abstasks/AbsTaskInstructionRetrieval.py b/mteb/abstasks/AbsTaskInstructionRetrieval.py index 1bcb36d78d..a0107abc75 100644 --- a/mteb/abstasks/AbsTaskInstructionRetrieval.py +++ b/mteb/abstasks/AbsTaskInstructionRetrieval.py @@ -248,12 +248,12 @@ class AbsTaskInstructionRetrieval(AbsTask): instruction: A relevant document will provide the projected or actual date of completion of the project, its estimated or actual total cost, or the estimated or ongoing electrical output of the finished project. Discussions of the social, political, or ecological impact of the project are not relevant. Child-classes must implement the following properties: - self.corpus = Dict[corpus_id, Dict[str, str]] #id => dict with document datas like title and text - self.queries = Dict[query_id, str] #id => query - self.relevant_docs = Dict[query_id, Dict[corpus_id, int]] - self.og_instructions = Dict[str, str] query => original instruction - self.changed_instructions = Dict[str, str] query => changed instruction - self.top_ranked = Dict[query_id, List[corpus_id]] #id => list of top ranked document ids + self.corpus = dict[corpus_id, dict[str, str]] #id => dict with document datas like title and text + self.queries = dict[query_id, str] #id => query + self.relevant_docs = dict[query_id, dict[corpus_id, int]] + self.og_instructions = dict[str, str] query => original instruction + self.changed_instructions = dict[str, str] query => changed instruction + self.top_ranked = dict[query_id, list[corpus_id]] #id => list of top ranked document ids See https://arxiv.org/abs/2403.15246 for more details """ @@ -372,6 +372,7 @@ def _evaluate_subset_lang( ) top_ranked = top_ranked[split] + kwargs["prediction_name"] = "og" # for naming predictions, as needed scores_og, results_og = self._evaluate_subset( retriever, corpus, @@ -382,6 +383,7 @@ def _evaluate_subset_lang( lang, **kwargs, ) + kwargs["prediction_name"] = "changed" # for naming predictions, as needed scores_changed, results_changed = self._evaluate_subset( retriever, corpus, @@ -411,6 +413,7 @@ def _evaluate_subset_lang( keywords[split], short_instructions[split], ) + kwargs["prediction_name"] = "base" # for naming predictions, as needed scores_base, results_base = self._evaluate_subset( retriever, corpus, @@ -421,6 +424,7 @@ def _evaluate_subset_lang( lang, **kwargs, ) + kwargs["prediction_name"] = "keywords" # for naming predictions, as needed scores_w_keywords_scores, scores_w_keywords_results = self._evaluate_subset( retriever, corpus, @@ -431,6 +435,9 @@ def _evaluate_subset_lang( lang, **kwargs, ) + kwargs["prediction_name"] = ( + "short_instr" # for naming predictions, as needed + ) ( scores_w_short_instr_scores, scores_w_short_instr_result, @@ -572,6 +579,11 @@ def _evaluate_subset( else: qrels_save_path = f"{output_folder}/{self.metadata_dict['name']}_{lang}_predictions.json" + if kwargs.get("prediction_name", None): + qrels_save_path = qrels_save_path.replace( + ".json", f"_{kwargs['prediction_name']}.json" + ) + with open(qrels_save_path, "w") as f: json.dump(results, f) diff --git a/mteb/abstasks/AbsTaskMultilabelClassification.py b/mteb/abstasks/AbsTaskMultilabelClassification.py index 01cba996f8..f31cd90bb3 100644 --- a/mteb/abstasks/AbsTaskMultilabelClassification.py +++ b/mteb/abstasks/AbsTaskMultilabelClassification.py @@ -13,9 +13,9 @@ from sklearn.preprocessing import MultiLabelBinarizer from mteb.encoder_interface import Encoder +from mteb.normalize_embeddings import normalize_embeddings_to_numpy -from ..evaluation.evaluators.model_encode import model_encode -from ..load_results.mteb_results import HFSubset, ScoresDict +from ..load_results.task_results import HFSubset, ScoresDict from .AbsTask import AbsTask, DescriptiveStatistics logger = logging.getLogger(__name__) @@ -162,11 +162,12 @@ def _evaluate_subset( unique_train_indices = list(set(itertools.chain.from_iterable(train_samples))) unique_train_sentences = train_split.select(unique_train_indices)["text"] - _unique_train_embeddings = model_encode( - unique_train_sentences, - model=model, - prompt_name=self.metadata.name, - **encode_kwargs, + _unique_train_embeddings = normalize_embeddings_to_numpy( + model.encode( + unique_train_sentences, + task_name=self.metadata.name, + **encode_kwargs, + ) ) unique_train_embeddings = dict( zip(unique_train_indices, _unique_train_embeddings) @@ -183,8 +184,12 @@ def _evaluate_subset( except ValueError: logger.warning("Couldn't subsample, continuing with the entire test set.") - X_test = model_encode( - test_text, model=model, prompt_name=self.metadata.name, **encode_kwargs + X_test = normalize_embeddings_to_numpy( + model.encode( + test_text, + task_name=self.metadata.name, + **encode_kwargs, + ) ) for i_experiment, sample_indices in enumerate(train_samples): logger.info( diff --git a/mteb/abstasks/AbsTaskPairClassification.py b/mteb/abstasks/AbsTaskPairClassification.py index f06fcdcf4c..8ce32a8fcc 100644 --- a/mteb/abstasks/AbsTaskPairClassification.py +++ b/mteb/abstasks/AbsTaskPairClassification.py @@ -5,9 +5,9 @@ from datasets import Dataset -from ..encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from ..encoder_interface import Encoder from ..evaluation.evaluators import PairClassificationEvaluator -from ..load_results.mteb_results import ScoresDict +from ..load_results.task_results import ScoresDict from .AbsTask import AbsTask, DescriptiveStatistics logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def _add_main_score(self, scores: ScoresDict) -> None: def _evaluate_subset( self, - model: Encoder | EncoderWithQueryCorpusEncode, + model: Encoder, dataset: Dataset, *, encode_kwargs: dict[str, str] = {}, diff --git a/mteb/abstasks/AbsTaskReranking.py b/mteb/abstasks/AbsTaskReranking.py index 0fba84b040..bcbc4571d5 100644 --- a/mteb/abstasks/AbsTaskReranking.py +++ b/mteb/abstasks/AbsTaskReranking.py @@ -4,8 +4,8 @@ from datasets import Dataset -from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode -from mteb.load_results.mteb_results import ScoresDict +from mteb.encoder_interface import Encoder +from mteb.load_results.task_results import ScoresDict from ..evaluation.evaluators import RerankingEvaluator from .AbsTask import AbsTask, DescriptiveStatistics @@ -45,7 +45,7 @@ def __init__(self, **kwargs): def _evaluate_subset( self, - model: Encoder | EncoderWithQueryCorpusEncode, + model: Encoder, data_split: Dataset, *, encode_kwargs: dict[str, Any] = {}, diff --git a/mteb/abstasks/AbsTaskRetrieval.py b/mteb/abstasks/AbsTaskRetrieval.py index a31aee761e..6fa901c791 100644 --- a/mteb/abstasks/AbsTaskRetrieval.py +++ b/mteb/abstasks/AbsTaskRetrieval.py @@ -13,7 +13,7 @@ from mteb.abstasks.TaskMetadata import HFSubset from ..evaluation.evaluators import RetrievalEvaluator -from ..load_results.mteb_results import ScoresDict +from ..load_results.task_results import ScoresDict from .AbsTask import AbsTask, DescriptiveStatistics logger = logging.getLogger(__name__) @@ -219,8 +219,8 @@ class AbsTaskRetrieval(AbsTask): Semantically, it should contain dict[split_name, dict[sample_id, dict[str, str]]] E.g. {"test": {"document_one": {"_id": "d1", "title": "title", "text": "text"}}} - self.queries: dict[str, dict[str, Union[str, List[str]]]] - Semantically, it should contain dict[split_name, dict[sample_id, str]] or dict[split_name, dict[sample_id, List[str]]] for conversations + self.queries: dict[str, dict[str, Union[str, list[str]]]] + Semantically, it should contain dict[split_name, dict[sample_id, str]] or dict[split_name, dict[sample_id, list[str]]] for conversations E.g. {"test": {"q1": "query"}} or {"test": {"q1": ["turn1", "turn2", "turn3"]}} @@ -252,8 +252,7 @@ def load_data(self, **kwargs): # Conversion from DataSet queries = {query["id"]: query["text"] for query in queries} corpus = { - doc["id"]: {"title": doc["title"], "text": doc["text"]} - for doc in corpus + doc["id"]: doc.get("title", "") + " " + doc["text"] for doc in corpus } self.corpus[split], self.queries[split], self.relevant_docs[split] = ( corpus, @@ -447,10 +446,7 @@ def calculate_length( queries_lens.append(len(query)) for doc in corpus.values(): - if isinstance(doc, dict): - doc_lens.append(len(doc.get("title", "")) + len(doc["text"])) - else: - doc_lens.append(len(doc)) + doc_lens.append(len(doc)) doc_len = sum(doc_lens) / len(doc_lens) if doc_lens else 0 query_len = sum(queries_lens) / len(queries_lens) if queries_lens else 0 diff --git a/mteb/abstasks/AbsTaskSTS.py b/mteb/abstasks/AbsTaskSTS.py index 422162e8c3..157f285951 100644 --- a/mteb/abstasks/AbsTaskSTS.py +++ b/mteb/abstasks/AbsTaskSTS.py @@ -4,7 +4,7 @@ from typing import Any from ..evaluation.evaluators import STSEvaluator -from ..load_results.mteb_results import ScoresDict +from ..load_results.task_results import ScoresDict from .AbsTask import AbsTask, DescriptiveStatistics logger = logging.getLogger(__name__) diff --git a/mteb/abstasks/AbsTaskSpeedTask.py b/mteb/abstasks/AbsTaskSpeedTask.py index e764f607db..7a73da445b 100644 --- a/mteb/abstasks/AbsTaskSpeedTask.py +++ b/mteb/abstasks/AbsTaskSpeedTask.py @@ -7,8 +7,8 @@ import numpy as np -from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode -from mteb.load_results.mteb_results import ScoresDict +from mteb.encoder_interface import Encoder +from mteb.load_results.task_results import ScoresDict from .AbsTask import AbsTask @@ -39,7 +39,9 @@ def load_data(self, **kwargs): def _get_time_taken(self, model: Encoder, data_split) -> float: start = time.time() - model.encode(data_split["text"], device=self.device) + model.encode( + data_split["text"], device=self.device, task_name=self.metadata.name + ) time_taken = time.time() - start return time_taken @@ -82,10 +84,10 @@ def get_system_info(self) -> dict[str, str]: info["gpu_info"] = list_gpus return info - def _evaluate_subset( - self, model: EncoderWithQueryCorpusEncode | Encoder, data_split, **kwargs - ) -> ScoresDict: - model.encode(["encode this"], device=self.device) # ensure model is loaded + def _evaluate_subset(self, model: Encoder, data_split, **kwargs) -> ScoresDict: + model.encode( + ["encode this"], device=self.device, task_name=self.metadata.name + ) # ensure model is loaded timings = [] for _ in range(self.num_loops): diff --git a/mteb/abstasks/AbsTaskSummarization.py b/mteb/abstasks/AbsTaskSummarization.py index 4717d2a8cb..ff03fbaab3 100644 --- a/mteb/abstasks/AbsTaskSummarization.py +++ b/mteb/abstasks/AbsTaskSummarization.py @@ -6,7 +6,7 @@ import numpy as np from mteb.encoder_interface import Encoder -from mteb.load_results.mteb_results import ScoresDict +from mteb.load_results.task_results import ScoresDict from ..evaluation.evaluators import SummarizationEvaluator from .AbsTask import AbsTask, DescriptiveStatistics diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py b/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py index a8d0dde0ea..4bef3f30b3 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py @@ -12,9 +12,9 @@ from datasets import Features, Value, load_dataset from PIL import Image +from mteb.abstasks.AbsTask import AbsTask, ScoresDict + from ...evaluation.evaluators import Any2AnyMultiChoiceEvaluator -from ...load_results.mteb_results import ScoresDict -from ..AbsTask import AbsTask logger = logging.getLogger(__name__) diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py index c640988e91..e288f3e37d 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py @@ -13,8 +13,7 @@ from PIL import Image from ...evaluation.evaluators import Any2AnyRetrievalEvaluator -from ...load_results.mteb_results import ScoresDict -from ..AbsTask import AbsTask +from ..AbsTask import AbsTask, ScoresDict logger = logging.getLogger(__name__) diff --git a/mteb/abstasks/Image/AbsTaskAny2TextMultipleChoice.py b/mteb/abstasks/Image/AbsTaskAny2TextMultipleChoice.py index 6172eae3ff..50991b6aee 100644 --- a/mteb/abstasks/Image/AbsTaskAny2TextMultipleChoice.py +++ b/mteb/abstasks/Image/AbsTaskAny2TextMultipleChoice.py @@ -5,10 +5,9 @@ from datasets import Dataset -from ...encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from ...encoder_interface import Encoder from ...evaluation.evaluators import Any2TextMultipleChoiceEvaluator -from ...load_results.mteb_results import ScoresDict -from ..AbsTask import AbsTask +from ..AbsTask import AbsTask, ScoresDict logger = logging.getLogger(__name__) @@ -40,7 +39,7 @@ def _calculate_metrics_from_split( def _evaluate_subset( self, - model: Encoder | EncoderWithQueryCorpusEncode, + model: Encoder, dataset: Dataset, *, encode_kwargs: dict[str, Any] = {}, diff --git a/mteb/abstasks/Image/AbsTaskImageClassification.py b/mteb/abstasks/Image/AbsTaskImageClassification.py index 715f007e10..04a6980283 100644 --- a/mteb/abstasks/Image/AbsTaskImageClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageClassification.py @@ -6,14 +6,15 @@ import numpy as np +from mteb.abstasks.TaskMetadata import HFSubset + from ...encoder_interface import Encoder from ...evaluation.evaluators import ( ImagekNNClassificationEvaluator, ImagekNNClassificationEvaluatorPytorch, ImagelogRegClassificationEvaluator, ) -from ...load_results.mteb_results import HFSubset, ScoresDict -from ..AbsTask import AbsTask +from ..AbsTask import AbsTask, ScoresDict logger = logging.getLogger(__name__) diff --git a/mteb/abstasks/Image/AbsTaskImageClustering.py b/mteb/abstasks/Image/AbsTaskImageClustering.py index 3d6f7e88d2..8152bf10f7 100644 --- a/mteb/abstasks/Image/AbsTaskImageClustering.py +++ b/mteb/abstasks/Image/AbsTaskImageClustering.py @@ -5,10 +5,11 @@ from datasets import Dataset -from ...encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from mteb.abstasks.TaskMetadata import HFSubset + +from ...encoder_interface import Encoder from ...evaluation.evaluators import ImageClusteringEvaluator -from ...load_results.mteb_results import HFSubset, ScoresDict -from ..AbsTask import AbsTask +from ..AbsTask import AbsTask, ScoresDict logger = logging.getLogger(__name__) @@ -38,7 +39,7 @@ def _calculate_metrics_from_split( def _evaluate_subset( self, - model: EncoderWithQueryCorpusEncode | Encoder, + model: Encoder, dataset: Dataset, *, encode_kwargs: dict[str, Any] = {}, diff --git a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py index 6a0d649f10..3cbd33ab53 100644 --- a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py @@ -12,9 +12,10 @@ from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import MultiLabelBinarizer +from mteb.abstasks.TaskMetadata import HFSubset + from ...encoder_interface import Encoder -from ...load_results.mteb_results import HFSubset, ScoresDict -from ..AbsTask import AbsTask +from ..AbsTask import AbsTask, ScoresDict logger = logging.getLogger(__name__) diff --git a/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py index 49523c58f9..b635610127 100644 --- a/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageTextPairClassification.py @@ -5,10 +5,9 @@ from datasets import Dataset -from ...encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from ...encoder_interface import Encoder from ...evaluation.evaluators import ImageTextPairClassificationEvaluator -from ...load_results.mteb_results import ScoresDict -from ..AbsTask import AbsTask +from ..AbsTask import AbsTask, ScoresDict logger = logging.getLogger(__name__) @@ -41,7 +40,7 @@ def _calculate_metrics_from_split( def _evaluate_subset( self, - model: Encoder | EncoderWithQueryCorpusEncode, + model: Encoder, dataset: Dataset, *, encode_kwargs: dict[str, Any] = {}, diff --git a/mteb/abstasks/Image/AbsTaskVisualSTS.py b/mteb/abstasks/Image/AbsTaskVisualSTS.py index c084be038e..45de465eac 100644 --- a/mteb/abstasks/Image/AbsTaskVisualSTS.py +++ b/mteb/abstasks/Image/AbsTaskVisualSTS.py @@ -4,8 +4,7 @@ from typing import Any from ...evaluation.evaluators import VisualSTSEvaluator -from ...load_results.mteb_results import ScoresDict -from ..AbsTask import AbsTask, DescriptiveStatistics +from ..AbsTask import AbsTask, DescriptiveStatistics, ScoresDict logger = logging.getLogger(__name__) diff --git a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py index 4f23bb46b4..36bdd27103 100644 --- a/mteb/abstasks/Image/AbsTaskZeroshotClassification.py +++ b/mteb/abstasks/Image/AbsTaskZeroshotClassification.py @@ -5,10 +5,9 @@ from datasets import Dataset -from ...encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from ...encoder_interface import Encoder from ...evaluation.evaluators import ZeroshotClassificationEvaluator -from ...load_results.mteb_results import ScoresDict -from ..AbsTask import AbsTask +from ..AbsTask import AbsTask, ScoresDict logger = logging.getLogger(__name__) @@ -38,7 +37,7 @@ def _calculate_metrics_from_split( def _evaluate_subset( self, - model: EncoderWithQueryCorpusEncode | Encoder, + model: Encoder, dataset: Dataset, *, encode_kwargs: dict[str, Any] = {}, diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index c7b6839015..872273e1d2 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -1,11 +1,12 @@ from __future__ import annotations import logging +from collections.abc import Mapping from datetime import date -from typing import Any, Dict, List, Mapping, Union +from typing import Annotated, Any, Union from pydantic import AnyUrl, BaseModel, BeforeValidator, TypeAdapter, field_validator -from typing_extensions import Annotated, Literal +from typing_extensions import Literal from ..languages import ( ISO_LANGUAGE_SCRIPT, @@ -71,7 +72,6 @@ "Web", "Written", "Programming", - None, ] SAMPLE_CREATION_METHOD = Literal[ @@ -149,7 +149,7 @@ SPLIT_NAME = str HFSubset = str LANGUAGES = Union[ - List[ISO_LANGUAGE_SCRIPT], Mapping[HFSubset, List[ISO_LANGUAGE_SCRIPT]] + list[ISO_LANGUAGE_SCRIPT], Mapping[HFSubset, list[ISO_LANGUAGE_SCRIPT]] ] PROGRAMMING_LANGS = [ @@ -169,9 +169,35 @@ "sql", ] +LICENSES = ( # this list can be extended as needed + Literal[ # we use lowercase for the licenses similar to the huggingface datasets + "not specified", # or none found + "mit", + "cc-by-2.0", + "cc-by-3.0", + "cc-by-4.0", + "cc-by-sa-3.0", + "cc-by-sa-4.0", + "cc-by-nc-4.0", + "cc-by-nc-sa-3.0", + "cc-by-nc-sa-4.0", + "cc-by-nc-nd-4.0", + "openrail", + "openrail++", + "odc-by", + "afl-3.0", + "apache-2.0", + "cc-by-nd-2.1-jp", + "cc0-1.0", + "bsd-3-clause", + "gpl-3.0", + "cdla-sharing-1.0", + "mpl-2.0", + ] +) METRIC_NAME = str -METRIC_VALUE = Union[int, float, Dict[str, Any]] +METRIC_VALUE = Union[int, float, dict[str, Any]] logger = logging.getLogger(__name__) @@ -197,7 +223,7 @@ class TaskMetadata(BaseModel): domains: The domains of the data. These includes "Non-fiction", "Social", "Fiction", "News", "Academic", "Blog", "Encyclopaedic", "Government", "Legal", "Medical", "Poetry", "Religious", "Reviews", "Web", "Spoken", "Written". A dataset can belong to multiple domains. task_subtypes: The subtypes of the task. E.g. includes "Sentiment/Hate speech", "Thematic Clustering". Feel free to update the list as needed. - license: The license of the data. + license: The license of the data specified as lowercase, e.g. "cc-by-nc-4.0". If the license is not specified, use "not specified". For custom licenses a URL is used. annotations_creators: The type of the annotators. Includes "expert-annotated" (annotated by experts), "human-annotated" (annotated e.g. by mturkers), "derived" (derived from structure in the data). dialect: The dialect of the data, if applicable. Ideally specified as a BCP-47 language tag. Empty list if no dialects are present. @@ -214,8 +240,8 @@ class TaskMetadata(BaseModel): name: str description: str - type: TASK_TYPE | None = None - modalities: list[Literal["text", "image"]] = ["text"] + type: TASK_TYPE + modalities: list[MODALITIES] = ["text"] category: TASK_CATEGORY | None = None reference: STR_URL | None = None @@ -226,7 +252,7 @@ class TaskMetadata(BaseModel): date: tuple[STR_DATE, STR_DATE] | None = None domains: list[TASK_DOMAIN] | None = None task_subtypes: list[TASK_SUBTYPE] | None = None - license: str | None = None + license: LICENSES | STR_URL | None = None annotations_creators: ANNOTATOR_TYPE | None = None dialect: list[str] | None = None @@ -359,3 +385,6 @@ def intext_citation(self, include_cite: bool = True) -> str: ) return f"\\cite{{{cite}}}" return cite + + def __hash__(self) -> int: + return hash(self.model_dump_json()) diff --git a/mteb/abstasks/stratification.py b/mteb/abstasks/stratification.py index cb1bb91ac6..b44250aba9 100644 --- a/mteb/abstasks/stratification.py +++ b/mteb/abstasks/stratification.py @@ -113,7 +113,7 @@ def _get_most_desired_combination(samples_with_combination): Parameters ---------- - samples_with_combination : Dict[Combination, List[int]], :code:`(n_combinations)` + samples_with_combination : dict[Combination, list[int]], :code:`(n_combinations)` map from each label combination present in y to list of sample indexes that have this combination assigned Returns: @@ -155,7 +155,7 @@ class IterativeStratification(_BaseKFold): order : int, >= 1 the order of label relationship to take into account when balancing sample distribution across labels - sample_distribution_per_fold : None or List[float], :code:`(n_splits)` + sample_distribution_per_fold : None or list[float], :code:`(n_splits)` desired percentage of samples in each of the folds, if None and equal distribution of samples per fold is assumed i.e. 1/n_splits for each fold. The value is held in :code:`self.percentage_per_fold`. @@ -195,7 +195,7 @@ def __init__( def _prepare_stratification(self, y): """Prepares variables for performing stratification - For the purpose of clarity, the type Combination denotes List[int], :code:`(self.order)` and represents a + For the purpose of clarity, the type Combination denotes list[int], :code:`(self.order)` and represents a label combination of the order we want to preserve among folds in stratification. The total number of combinations present in :code:`(y)` will be denoted as :code:`(n_combinations)`. @@ -208,7 +208,7 @@ def _prepare_stratification(self, y): self.desired_samples_per_fold: np.array[Float], :code:`(n_splits)` number of samples desired per fold - self.desired_samples_per_combination_per_fold: Dict[Combination, np.array[Float]], :code:`(n_combinations, n_splits)` + self.desired_samples_per_combination_per_fold: dict[Combination, np.array[Float]], :code:`(n_combinations, n_splits)` number of samples evidencing each combination desired per each fold Parameters @@ -218,22 +218,22 @@ def _prepare_stratification(self, y): Returns: ------- - rows : List[List[int]], :code:`(n_samples, n_labels)` + rows : list[list[int]], :code:`(n_samples, n_labels)` list of label indices assigned to each sample - rows_used : Dict[int, bool], :code:`(n_samples)` + rows_used : dict[int, bool], :code:`(n_samples)` boolean map from a given sample index to boolean value whether it has been already assigned to a fold or not - all_combinations : List[Combination], :code:`(n_combinations)` + all_combinations : list[Combination], :code:`(n_combinations)` list of all label combinations of order self.order present in y - per_row_combinations : List[Combination], :code:`(n_samples)` + per_row_combinations : list[Combination], :code:`(n_samples)` list of all label combinations of order self.order present in y per row - samples_with_combination : Dict[Combination, List[int]], :code:`(n_combinations)` + samples_with_combination : dict[Combination, list[int]], :code:`(n_combinations)` map from each label combination present in y to list of sample indexes that have this combination assigned - folds: List[List[int]] (n_splits) + folds: list[list[int]] (n_splits) list of lists to be populated with samples """ @@ -353,7 +353,7 @@ def _iter_test_indices(self, X, y=None, groups=None): Yields: ------ - fold : List[int] + fold : list[int] indexes of test samples for a given fold, yielded for each of the folds """ ( diff --git a/mteb/benchmarks.py b/mteb/benchmarks.py deleted file mode 100644 index 9485230a62..0000000000 --- a/mteb/benchmarks.py +++ /dev/null @@ -1,460 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Sequence - -from mteb.abstasks.AbsTask import AbsTask -from mteb.overview import get_tasks - - -@dataclass -class Benchmark: - name: str - tasks: Sequence[str] | Sequence[AbsTask] - description: str | None = None - reference: str | None = None - citation: str | None = None - - def __iter__(self): - return iter(self.tasks) - - def __len__(self) -> int: - return len(self.tasks) - - def __getitem__(self, index): - return self.tasks[index] - - -MTEB_MAIN_EN = Benchmark( - name="MTEB(eng)", - tasks=[ - "AmazonCounterfactualClassification", - "AmazonPolarityClassification", - "AmazonReviewsClassification", - "ArguAna", - "ArxivClusteringP2P", - "ArxivClusteringS2S", - "AskUbuntuDupQuestions", - "BIOSSES", - "Banking77Classification", - "BiorxivClusteringP2P", - "BiorxivClusteringS2S", - "CQADupstackAndroidRetrieval", - "CQADupstackEnglishRetrieval", - "CQADupstackGamingRetrieval", - "CQADupstackGisRetrieval", - "CQADupstackMathematicaRetrieval", - "CQADupstackPhysicsRetrieval", - "CQADupstackProgrammersRetrieval", - "CQADupstackStatsRetrieval", - "CQADupstackTexRetrieval", - "CQADupstackUnixRetrieval", - "CQADupstackWebmastersRetrieval", - "CQADupstackWordpressRetrieval", - "ClimateFEVER", - "DBPedia", - "EmotionClassification", - "FEVER", - "FiQA2018", - "HotpotQA", - "ImdbClassification", - "MSMARCO", - "MTOPDomainClassification", - "MTOPIntentClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "MedrxivClusteringP2P", - "MedrxivClusteringS2S", - "MindSmallReranking", - "NFCorpus", - "NQ", - "QuoraRetrieval", - "RedditClustering", - "RedditClusteringP2P", - "SCIDOCS", - "SICK-R", - "STS12", - "STS13", - "STS14", - "STS15", - "STS16", - "STS17", - "STS22", - "STSBenchmark", - "SciDocsRR", - "SciFact", - "SprintDuplicateQuestions", - "StackExchangeClustering", - "StackExchangeClusteringP2P", - "StackOverflowDupQuestions", - "SummEval", - "TRECCOVID", - "Touche2020", - "ToxicConversationsClassification", - "TweetSentimentExtractionClassification", - "TwentyNewsgroupsClustering", - "TwitterSemEval2015", - "TwitterURLCorpus", - ], - description="Main English benchmarks from MTEB", - citation="""@inproceedings{muennighoff-etal-2023-mteb, - title = "{MTEB}: Massive Text Embedding Benchmark", - author = "Muennighoff, Niklas and - Tazi, Nouamane and - Magne, Loic and - Reimers, Nils", - editor = "Vlachos, Andreas and - Augenstein, Isabelle", - booktitle = "Proceedings of the 17th Conference of the European Chapter of the Association for Computational Linguistics", - month = may, - year = "2023", - address = "Dubrovnik, Croatia", - publisher = "Association for Computational Linguistics", - url = "https://aclanthology.org/2023.eacl-main.148", - doi = "10.18653/v1/2023.eacl-main.148", - pages = "2014--2037", -} -""", -) - -MTEB_MAIN_RU = Benchmark( - name="MTEB(rus)", - tasks=get_tasks( - languages=["rus"], - tasks=[ - # Classification - "GeoreviewClassification", - "HeadlineClassification", - "InappropriatenessClassification", - "KinopoiskClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "RuReviewsClassification", - "RuSciBenchGRNTIClassification", - "RuSciBenchOECDClassification", - # Clustering - "GeoreviewClusteringP2P", - "RuSciBenchGRNTIClusteringP2P", - "RuSciBenchOECDClusteringP2P", - # MultiLabelClassification - "CEDRClassification", - "SensitiveTopicsClassification", - # PairClassification - "TERRa", - # Reranking - "MIRACLReranking", - "RuBQReranking", - # Retrieval - "MIRACLRetrieval", - "RiaNewsRetrieval", - "RuBQRetrieval", - # STS - "RUParaPhraserSTS", - "RuSTSBenchmarkSTS", - "STS22", - ], - ), - description="Main Russian benchmarks from MTEB", - reference="https://aclanthology.org/2023.eacl-main.148/", - citation="""@misc{snegirev2024russianfocusedembeddersexplorationrumteb, - title={The Russian-focused embedders' exploration: ruMTEB benchmark and Russian embedding model design}, - author={Artem Snegirev and Maria Tikhonova and Anna Maksimova and Alena Fenogenova and Alexander Abramov}, - year={2024}, - eprint={2408.12503}, - archivePrefix={arXiv}, - primaryClass={cs.CL}, - url={https://arxiv.org/abs/2408.12503}, -} -""", -) - -MTEB_RETRIEVAL_WITH_INSTRUCTIONS = Benchmark( - name="MTEB(Retrieval w/Instructions)", - tasks=[ - "Robust04InstructionRetrieval", - "News21InstructionRetrieval", - "Core17InstructionRetrieval", - ], - description="Retrieval w/Instructions is the task of finding relevant documents for a query that has detailed instructions.", - reference="https://arxiv.org/abs/2403.15246", - citation="""@misc{weller2024followir, - title={FollowIR: Evaluating and Teaching Information Retrieval Models to Follow Instructions}, - author={Orion Weller and Benjamin Chang and Sean MacAvaney and Kyle Lo and Arman Cohan and Benjamin Van Durme and Dawn Lawrie and Luca Soldaini}, - year={2024}, - eprint={2403.15246}, - archivePrefix={arXiv}, - primaryClass={cs.IR} -}""", -) - -MTEB_RETRIEVAL_LAW = Benchmark( - name="MTEB(law)", - tasks=[ - "LegalSummarization", - "LegalBenchConsumerContractsQA", - "LegalBenchCorporateLobbying", - "AILACasedocs", - "AILAStatutes", - "LeCaRDv2", - "LegalQuAD", - "GerDaLIRSmall", - ], - description="Legal benchmarks from MTEB", - reference="https://aclanthology.org/2023.eacl-main.148/", - citation=None, -) - -MTEB_MINERS_BITEXT_MINING = Benchmark( - name="MINERSBitextMining", - tasks=[ - "BUCCBitextMining", - "LinceMTBitextMining", - "NollySentiBitextMining", - "NusaXBitextMining", - "NusaTranslationBitextMining", - "PhincBitextMining", - "TatoebaBitextMining", - ], - description="BitextMining benchmark from MINERS", - reference="https://arxiv.org/pdf/2406.07424", - citation=""" - @article{winata2024miners, - title={MINERS: Multilingual Language Models as Semantic Retrievers}, - author={Winata, Genta Indra and Zhang, Ruochen and Adelani, David Ifeoluwa}, - journal={arXiv preprint arXiv:2406.07424}, - year={2024} - } - """, -) -SEB = Benchmark( - name="MTEB(Scandinavian)", - tasks=[ - "BornholmBitextMining", - "NorwegianCourtsBitextMining", - "AngryTweetsClassification", - "DanishPoliticalCommentsClassification", - "DKHateClassification", - "LccSentimentClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "NordicLangClassification", - "ScalaClassification", - "NoRecClassification", - "NorwegianParliamentClassification", - "DalajClassification", - "SwedishSentimentClassification", - "SweRecClassification", - "DanFEVER", - "TV2Nordretrieval", - "TwitterHjerneRetrieval", - "NorQuadRetrieval", - "SNLRetrieval", - "SwednRetrieval", - "SweFaqRetrieval", - "WikiClusteringP2P.v2", - "SNLHierarchicalClusteringP2P", - "SNLHierarchicalClusteringS2S", - "VGHierarchicalClusteringP2P", - "VGHierarchicalClusteringS2S", - "SwednClusteringP2P", - "SwednClusteringS2S", - ], - description="A curated selection of tasks coverering the Scandinavian languages; Danish, Swedish and Norwegian, including Bokmål and Nynorsk.", - reference="https://kennethenevoldsen.github.io/scandinavian-embedding-benchmark/", - citation="""@misc{enevoldsen2024scandinavian, - title={The Scandinavian Embedding Benchmarks: Comprehensive Assessment of Multilingual and Monolingual Text Embedding}, - author={Kenneth Enevoldsen and Márton Kardos and Niklas Muennighoff and Kristoffer Laigaard Nielbo}, - year={2024}, - eprint={2406.02396}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -}""", -) - -CoIR = Benchmark( - name="CoIR", - tasks=[ - "AppsRetrieval", - "CosQA", - "SyntheticText2SQL", - "COIRCodeSearchNetRetrieval", - "CodeSearchNetCCRetrieval", - "CodeTransOceanDL", - "CodeTransOceanContest", - "StackOverflowQA", - "CodeFeedbackMT", - "CodeFeedbackST", - ], - description="CoIR: A Comprehensive Benchmark for Code Information Retrieval Models", - reference="https://github.com/CoIR-team/coir", - citation="""@misc{li2024coircomprehensivebenchmarkcode, - title={CoIR: A Comprehensive Benchmark for Code Information Retrieval Models}, - author={Xiangyang Li and Kuicai Dong and Yi Quan Lee and Wei Xia and Yichun Yin and Hao Zhang and Yong Liu and Yasheng Wang and Ruiming Tang}, - year={2024}, - eprint={2407.02883}, - archivePrefix={arXiv}, - primaryClass={cs.IR}, - url={https://arxiv.org/abs/2407.02883}, - }""", -) - -MTEB_FRA = Benchmark( - name="MTEB(fra)", - tasks=get_tasks( - languages=["fra"], - tasks=[ - # Classification - "AmazonReviewsClassification", - "MasakhaNEWSClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "MTOPDomainClassification", - "MTOPIntentClassification", - # Clustering - "AlloProfClusteringP2P", - "AlloProfClusteringS2S", - "HALClusteringS2S", - "MasakhaNEWSClusteringP2P", - "MasakhaNEWSClusteringS2S", - "MLSUMClusteringP2P", - "MLSUMClusteringS2S", - # Pair Classification - "OpusparcusPC", - "PawsXPairClassification", - # Reranking - "SyntecReranking", - "AlloprofReranking", - # Retrieval - "AlloprofRetrieval", - "BSARDRetrieval", - "SyntecRetrieval", - "XPQARetrieval", - "MintakaRetrieval", - # STS - "SummEvalFr", - "STSBenchmarkMultilingualSTS", - "STS22", - "SICKFr", - ], - ), - description="Main French benchmarks from MTEB", - reference="https://arxiv.org/abs/2405.20468", - citation="""@misc{ciancone2024mtebfrenchresourcesfrenchsentence, - title={MTEB-French: Resources for French Sentence Embedding Evaluation and Analysis}, - author={Mathieu Ciancone and Imene Kerboua and Marion Schaeffer and Wissam Siblini}, - year={2024}, - eprint={2405.20468}, - archivePrefix={arXiv}, - primaryClass={cs.CL}, - url={https://arxiv.org/abs/2405.20468}, -}""", -) - - -MTEB_DEU = Benchmark( - name="MTEB(deu)", - tasks=get_tasks( - languages=["deu"], - tasks=[ - # Classification - "AmazonCounterfactualClassification", - "AmazonReviewsClassification", - "MTOPDomainClassification", - "MTOPIntentClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - # Clustering - "BlurbsClusteringP2P", - "BlurbsClusteringS2S", - "TenKGnadClusteringP2P", - "TenKGnadClusteringS2S", - # Pair Classification - "FalseFriendsGermanEnglish", - "PawsXPairClassification", - # Reranking - "MIRACLReranking", - # Retrieval - "GermanQuAD-Retrieval", - "GermanDPR", - "XMarket", - "GerDaLIR", - # STS - "GermanSTSBenchmark", - "STS22", - ], - ), - description="Main German benchmarks from MTEB", - reference="https://arxiv.org/html/2401.02709v1", - citation="""@misc{wehrli2024germantextembeddingclustering, - title={German Text Embedding Clustering Benchmark}, - author={Silvan Wehrli and Bert Arnrich and Christopher Irrgang}, - year={2024}, - eprint={2401.02709}, - archivePrefix={arXiv}, - primaryClass={cs.CL}, - url={https://arxiv.org/abs/2401.02709}, -}""", -) - - -MTEB_KOR = Benchmark( - name="MTEB(kor)", - tasks=get_tasks( - languages=["kor"], - tasks=[ # @KennethEnevoldsen: We could probably expand this to a more solid benchamrk, but for now I have left it as is. - # Classification - "KLUE-TC", - # Reranking - "MIRACLReranking", - # Retrieval - "MIRACLRetrieval", - "Ko-StrategyQA", - # STS - "KLUE-STS", - "KorSTS", - ], - ), - description="Main Korean benchmarks from MTEB", - reference=None, - citation=None, -) - - -MTEB_pol = Benchmark( - name="MTEB(pol)", - tasks=get_tasks( - languages=["pol"], - tasks=[ - # Classification - "CBD", - "PolEmo2.0-IN", - "PolEmo2.0-OUT", - "AllegroReviews", - "PAC", - "MassiveIntentClassification", - "MassiveScenarioClassification", - # Clustering - "EightTagsClustering", - "PlscClusteringS2S", - "PlscClusteringP2P", - # Pair Classification - "SICK-E-PL", - "PpcPC", - "CDSC-E", - "PSC", - # STS - "SICK-R-PL", - "CDSC-R", - "STS22", - "STSBenchmarkMultilingualSTS", - ], - ), - description="Main Polish benchmarks from MTEB", - reference="https://arxiv.org/abs/2405.10138", - citation="""@article{poswiata2024plmteb, - title={PL-MTEB: Polish Massive Text Embedding Benchmark}, - author={Rafał Poświata and Sławomir Dadas and Michał Perełkiewicz}, - journal={arXiv preprint arXiv:2405.10138}, - year={2024} -}""", -) diff --git a/mteb/benchmarks/__init__.py b/mteb/benchmarks/__init__.py new file mode 100644 index 0000000000..653b97c6f7 --- /dev/null +++ b/mteb/benchmarks/__init__.py @@ -0,0 +1,4 @@ +from __future__ import annotations + +from mteb.benchmarks.benchmarks import * +from mteb.benchmarks.get_benchmark import * diff --git a/mteb/benchmarks/benchmarks.py b/mteb/benchmarks/benchmarks.py new file mode 100644 index 0000000000..9c24c525ac --- /dev/null +++ b/mteb/benchmarks/benchmarks.py @@ -0,0 +1,861 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Annotated + +from pydantic import AnyUrl, BeforeValidator, TypeAdapter + +from mteb.abstasks.AbsTask import AbsTask +from mteb.load_results.benchmark_results import BenchmarkResults +from mteb.load_results.load_results import load_results +from mteb.overview import get_tasks + +http_url_adapter = TypeAdapter(AnyUrl) +UrlString = Annotated[ + str, BeforeValidator(lambda value: str(http_url_adapter.validate_python(value))) +] # Allows the type to be a string, but ensures that the string is a URL + + +@dataclass +class Benchmark: + """A benchmark object intended to run a certain benchmark within MTEB. + + Args: + name: The name of the benchmark + tasks: The tasks within the benchmark. + description: A description of the benchmark, should include its intended goal and potentially a description of its construction + reference: A link reference, to a source containing additional information typically to a paper, leaderboard or github. + citation: A bibtex citation + + Example: + >>> Benchmark( + ... name="MTEB(custom)", + ... tasks=mteb.get_tasks( + ... tasks=["AmazonCounterfactualClassification", "AmazonPolarityClassification"], + ... languages=["eng"], + ... ), + ... description="A custom benchmark" + ... ) + """ + + name: str + tasks: Sequence[AbsTask] + description: str | None = None + reference: UrlString | None = None + citation: str | None = None + + def __iter__(self): + return iter(self.tasks) + + def __len__(self) -> int: + return len(self.tasks) + + def __getitem__(self, index): + return self.tasks[index] + + def load_results( + self, base_results: None | BenchmarkResults = None + ) -> BenchmarkResults: + if base_results is None: + base_results = load_results() + return base_results.select_tasks(self.tasks) + + +MTEB_MAIN_EN = Benchmark( + name="MTEB(eng)", + tasks=get_tasks( + tasks=[ + "AmazonCounterfactualClassification", + "AmazonPolarityClassification", + "AmazonReviewsClassification", + "ArguAna", + "ArxivClusteringP2P", + "ArxivClusteringS2S", + "AskUbuntuDupQuestions", + "BIOSSES", + "Banking77Classification", + "BiorxivClusteringP2P", + "BiorxivClusteringS2S", + "CQADupstackAndroidRetrieval", + "CQADupstackEnglishRetrieval", + "CQADupstackGamingRetrieval", + "CQADupstackGisRetrieval", + "CQADupstackMathematicaRetrieval", + "CQADupstackPhysicsRetrieval", + "CQADupstackProgrammersRetrieval", + "CQADupstackStatsRetrieval", + "CQADupstackTexRetrieval", + "CQADupstackUnixRetrieval", + "CQADupstackWebmastersRetrieval", + "CQADupstackWordpressRetrieval", + "ClimateFEVER", + "DBPedia", + "EmotionClassification", + "FEVER", + "FiQA2018", + "HotpotQA", + "ImdbClassification", + "MSMARCO", + "MTOPDomainClassification", + "MTOPIntentClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MedrxivClusteringP2P", + "MedrxivClusteringS2S", + "MindSmallReranking", + "NFCorpus", + "NQ", + "QuoraRetrieval", + "RedditClustering", + "RedditClusteringP2P", + "SCIDOCS", + "SICK-R", + "STS12", + "STS13", + "STS14", + "STS15", + "STS16", + "STS17", + "STS22", + "STSBenchmark", + "SciDocsRR", + "SciFact", + "SprintDuplicateQuestions", + "StackExchangeClustering", + "StackExchangeClusteringP2P", + "StackOverflowDupQuestions", + "SummEval", + "TRECCOVID", + "Touche2020Retrieval.v3", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "TwentyNewsgroupsClustering", + "TwitterSemEval2015", + "TwitterURLCorpus", + ], + languages=["eng"], + eval_splits=["test"], + ), + description="Main English benchmarks from MTEB", + citation="""@inproceedings{muennighoff-etal-2023-mteb, + title = "{MTEB}: Massive Text Embedding Benchmark", + author = "Muennighoff, Niklas and + Tazi, Nouamane and + Magne, Loic and + Reimers, Nils", + editor = "Vlachos, Andreas and + Augenstein, Isabelle", + booktitle = "Proceedings of the 17th Conference of the European Chapter of the Association for Computational Linguistics", + month = may, + year = "2023", + address = "Dubrovnik, Croatia", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.eacl-main.148", + doi = "10.18653/v1/2023.eacl-main.148", + pages = "2014--2037", +} +""", +) + +MTEB_MAIN_RU = Benchmark( + name="MTEB(rus)", + tasks=get_tasks( + languages=["rus"], + tasks=[ + # Classification + "GeoreviewClassification", + "HeadlineClassification", + "InappropriatenessClassification", + "KinopoiskClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "RuReviewsClassification", + "RuSciBenchGRNTIClassification", + "RuSciBenchOECDClassification", + # Clustering + "GeoreviewClusteringP2P", + "RuSciBenchGRNTIClusteringP2P", + "RuSciBenchOECDClusteringP2P", + # MultiLabelClassification + "CEDRClassification", + "SensitiveTopicsClassification", + # PairClassification + "TERRa", + # Reranking + "MIRACLReranking", + "RuBQReranking", + # Retrieval + "MIRACLRetrieval", + "RiaNewsRetrieval", + "RuBQRetrieval", + # STS + "RUParaPhraserSTS", + "RuSTSBenchmarkSTS", + "STS22", + ], + ), + description="Main Russian benchmarks from MTEB", + reference="https://aclanthology.org/2023.eacl-main.148/", + citation="""@misc{snegirev2024russianfocusedembeddersexplorationrumteb, + title={The Russian-focused embedders' exploration: ruMTEB benchmark and Russian embedding model design}, + author={Artem Snegirev and Maria Tikhonova and Anna Maksimova and Alena Fenogenova and Alexander Abramov}, + year={2024}, + eprint={2408.12503}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2408.12503}, +} +""", +) + +MTEB_RETRIEVAL_WITH_INSTRUCTIONS = Benchmark( + name="MTEB(Retrieval w/Instructions)", + tasks=get_tasks( + tasks=[ + "Robust04InstructionRetrieval", + "News21InstructionRetrieval", + "Core17InstructionRetrieval", + ] + ), + description="Retrieval w/Instructions is the task of finding relevant documents for a query that has detailed instructions.", + reference="https://arxiv.org/abs/2403.15246", + citation="""@misc{weller2024followir, + title={FollowIR: Evaluating and Teaching Information Retrieval Models to Follow Instructions}, + author={Orion Weller and Benjamin Chang and Sean MacAvaney and Kyle Lo and Arman Cohan and Benjamin Van Durme and Dawn Lawrie and Luca Soldaini}, + year={2024}, + eprint={2403.15246}, + archivePrefix={arXiv}, + primaryClass={cs.IR} +}""", +) + +MTEB_RETRIEVAL_LAW = Benchmark( + name="MTEB(law)", # This benchmark is likely in the need of an update + tasks=get_tasks( + tasks=[ + "AILACasedocs", + "AILAStatutes", + "LegalSummarization", + "GerDaLIRSmall", + "LeCaRDv2", + "LegalBenchConsumerContractsQA", + "LegalBenchCorporateLobbying", + "LegalQuAD", + ] + ), + description="Legal benchmarks from MTEB.", + reference="https://aclanthology.org/2023.eacl-main.148/", + citation=None, +) + +MTEB_MINERS_BITEXT_MINING = Benchmark( + name="MINERSBitextMining", + tasks=get_tasks( + tasks=[ + "BUCC", + "LinceMTBitextMining", + "NollySentiBitextMining", + "NusaXBitextMining", + "NusaTranslationBitextMining", + "PhincBitextMining", + "Tatoeba", + ] + ), + description="BitextMining benchmark from MINERS", + reference="https://arxiv.org/pdf/2406.07424", + citation=""" + @article{winata2024miners, + title={MINERS: Multilingual Language Models as Semantic Retrievers}, + author={Winata, Genta Indra and Zhang, Ruochen and Adelani, David Ifeoluwa}, + journal={arXiv preprint arXiv:2406.07424}, + year={2024} + } + """, +) + +SEB = Benchmark( + name="MTEB(Scandinavian)", + tasks=get_tasks( + tasks=[ + # Bitext + "BornholmBitextMining", + "NorwegianCourtsBitextMining", + # Classification + "AngryTweetsClassification", + "DanishPoliticalCommentsClassification", + "DalajClassification", + "DKHateClassification", + "LccSentimentClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "NordicLangClassification", + "NoRecClassification", + "NorwegianParliamentClassification", + "ScalaClassification", + "SwedishSentimentClassification", + "SweRecClassification", + # Retrieval + "DanFEVER", + "NorQuadRetrieval", + "SNLRetrieval", + "SwednRetrieval", + "SweFaqRetrieval", + "TV2Nordretrieval", + "TwitterHjerneRetrieval", + # Clustering + "SNLHierarchicalClusteringS2S", + "SNLHierarchicalClusteringP2P", + "SwednClusteringP2P", + "SwednClusteringS2S", + "VGHierarchicalClusteringS2S", + "VGHierarchicalClusteringP2P", + ], + languages=["dan", "swe", "nno", "nob"], + ), + description="A curated selection of tasks coverering the Scandinavian languages; Danish, Swedish and Norwegian, including Bokmål and Nynorsk.", + reference="https://kennethenevoldsen.github.io/scandinavian-embedding-benchmark/", + citation="""@misc{enevoldsen2024scandinavian, + title={The Scandinavian Embedding Benchmarks: Comprehensive Assessment of Multilingual and Monolingual Text Embedding}, + author={Kenneth Enevoldsen and Márton Kardos and Niklas Muennighoff and Kristoffer Laigaard Nielbo}, + year={2024}, + eprint={2406.02396}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +}""", +) + +CoIR = Benchmark( + name="CoIR", + tasks=get_tasks( + tasks=[ + "AppsRetrieval", + "CodeFeedbackMT", + "CodeFeedbackST", + "CodeSearchNetCCRetrieval", + "CodeTransOceanContest", + "CodeTransOceanDL", + "CosQA", + "COIRCodeSearchNetRetrieval", + "StackOverflowQA", + "SyntheticText2SQL", + ] + ), + description="CoIR: A Comprehensive Benchmark for Code Information Retrieval Models", + reference="https://github.com/CoIR-team/coir", + citation="""@misc{li2024coircomprehensivebenchmarkcode, + title={CoIR: A Comprehensive Benchmark for Code Information Retrieval Models}, + author={Xiangyang Li and Kuicai Dong and Yi Quan Lee and Wei Xia and Yichun Yin and Hao Zhang and Yong Liu and Yasheng Wang and Ruiming Tang}, + year={2024}, + eprint={2407.02883}, + archivePrefix={arXiv}, + primaryClass={cs.IR}, + url={https://arxiv.org/abs/2407.02883}, + }""", +) + +MTEB_FRA = Benchmark( + name="MTEB(fra)", + tasks=get_tasks( + languages=["fra"], + tasks=[ + # Classification + "AmazonReviewsClassification", + "MasakhaNEWSClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MTOPDomainClassification", + "MTOPIntentClassification", + # Clustering + "AlloProfClusteringP2P", + "AlloProfClusteringS2S", + "HALClusteringS2S", + "MasakhaNEWSClusteringP2P", + "MasakhaNEWSClusteringS2S", + "MLSUMClusteringP2P", + "MLSUMClusteringS2S", + # Pair Classification + "OpusparcusPC", + "PawsXPairClassification", + # Reranking + "AlloprofReranking", + "SyntecReranking", + # Retrieval + "AlloprofRetrieval", + "BSARDRetrieval", + "MintakaRetrieval", + "SyntecRetrieval", + "XPQARetrieval", + # STS + "SICKFr", + "STS22", + "STSBenchmarkMultilingualSTS", + "SummEvalFr", + ], + ), + description="Main French benchmarks from MTEB", + reference="https://arxiv.org/abs/2405.20468", + citation="""@misc{ciancone2024mtebfrenchresourcesfrenchsentence, + title={MTEB-French: Resources for French Sentence Embedding Evaluation and Analysis}, + author={Mathieu Ciancone and Imene Kerboua and Marion Schaeffer and Wissam Siblini}, + year={2024}, + eprint={2405.20468}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2405.20468}, +}""", +) + + +MTEB_DEU = Benchmark( + name="MTEB(deu)", + tasks=get_tasks( + languages=["deu"], + tasks=[ + # Classification + "AmazonCounterfactualClassification", + "AmazonReviewsClassification", + "MTOPDomainClassification", + "MTOPIntentClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + # Clustering + "BlurbsClusteringP2P", + "BlurbsClusteringS2S", + "TenKGnadClusteringP2P", + "TenKGnadClusteringS2S", + # Pair Classification + "FalseFriendsGermanEnglish", + "PawsXPairClassification", + # Reranking + "MIRACLReranking", + # Retrieval + "GermanQuAD-Retrieval", + "GermanDPR", + "XMarket", + "GerDaLIR", + # STS + "GermanSTSBenchmark", + "STS22", + ], + ), + description="Main German benchmarks from MTEB", + reference="https://arxiv.org/html/2401.02709v1", + citation="""@misc{wehrli2024germantextembeddingclustering, + title={German Text Embedding Clustering Benchmark}, + author={Silvan Wehrli and Bert Arnrich and Christopher Irrgang}, + year={2024}, + eprint={2401.02709}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2401.02709}, +}""", +) + + +MTEB_KOR = Benchmark( + name="MTEB(kor)", + tasks=get_tasks( + languages=["kor"], + tasks=[ # @KennethEnevoldsen: We could probably expand this to a more solid benchamrk, but for now I have left it as is. + # Classification + "KLUE-TC", + # Reranking + "MIRACLReranking", + # Retrieval + "MIRACLRetrieval", + "Ko-StrategyQA", + # STS + "KLUE-STS", + "KorSTS", + ], + ), + description="Main Korean benchmarks from MTEB", + reference=None, + citation=None, +) + + +MTEB_POL = Benchmark( + name="MTEB(pol)", + tasks=get_tasks( + languages=["pol"], + tasks=[ + # Classification + "AllegroReviews", + "CBD", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "PolEmo2.0-IN", + "PolEmo2.0-OUT", + "PAC", + # Clustering + "EightTagsClustering", + "PlscClusteringS2S", + "PlscClusteringP2P", + # Pair Classification + "CDSC-E", + "PpcPC", + "PSC", + "SICK-E-PL", + # STS + "CDSC-R", + "STS22", + "STSBenchmarkMultilingualSTS", + "SICK-R-PL", + ], + ), + description="Main Polish benchmarks from MTEB", + reference="https://arxiv.org/abs/2405.10138", + citation="""@article{poswiata2024plmteb, + title={PL-MTEB: Polish Massive Text Embedding Benchmark}, + author={Rafał Poświata and Sławomir Dadas and Michał Perełkiewicz}, + journal={arXiv preprint arXiv:2405.10138}, + year={2024} +}""", +) + +MTEB_code = Benchmark( + name="MTEB(code)", + tasks=get_tasks( + tasks=[ + # Retrieval + "AppsRetrieval", + "CodeEditSearchRetrieval", + "CodeFeedbackMT", + "CodeFeedbackST", + "CodeSearchNetCCRetrieval", + "CodeSearchNetRetrieval", + "CodeTransOceanContest", + "CodeTransOceanDL", + "CosQA", + "COIRCodeSearchNetRetrieval", + "StackOverflowQA", + "SyntheticText2SQL", + ], + languages=[ + "c", + "c++", + "go", + "java", + "javascript", + "php", + "python", + "ruby", + "rust", + "scala", + "shell", + "swift", + "typescript", + ], + ), + description="Main code benchmarks from MTEB", + reference=None, + citation=None, +) + + +MTEB_multilingual = Benchmark( + name="MTEB(Multilingual)", + tasks=get_tasks( + tasks=[ + "BornholmBitextMining", + "BibleNLPBitextMining", + "BUCC.v2", + "DiaBlaBitextMining", + "FloresBitextMining", + "IN22GenBitextMining", + "IndicGenBenchFloresBitextMining", + "NollySentiBitextMining", + "NorwegianCourtsBitextMining", + "NTREXBitextMining", + "NusaTranslationBitextMining", + "NusaXBitextMining", + "Tatoeba", + "BulgarianStoreReviewSentimentClassfication", + "CzechProductReviewSentimentClassification", + "GreekLegalCodeClassification", + "DBpediaClassification", + "FinancialPhrasebankClassification", + "PoemSentimentClassification", + "ToxicConversationsClassification", + "TweetTopicSingleClassification", + "EstonianValenceClassification", + "FilipinoShopeeReviewsClassification", + "GujaratiNewsClassification", + "SentimentAnalysisHindi", + "IndonesianIdClickbaitClassification", + "ItaCaseholdClassification", + "KorSarcasmClassification", + "KurdishSentimentClassification", + "MacedonianTweetSentimentClassification", + "AfriSentiClassification", + "AmazonCounterfactualClassification", + "CataloniaTweetClassification", + "CyrillicTurkicLangClassification", + "IndicLangClassification", + "MasakhaNEWSClassification", + "MassiveIntentClassification", + "MultiHateClassification", + "NordicLangClassification", + "NusaParagraphEmotionClassification", + "NusaX-senti", + "ScalaClassification", + "SwissJudgementClassification", + "NepaliNewsClassification", + "OdiaNewsClassification", + "PunjabiNewsClassification", + "PolEmo2.0-OUT", + "PAC", + "SinhalaNewsClassification", + "CSFDSKMovieReviewSentimentClassification", + "SiswatiNewsClassification", + "SlovakMovieReviewSentimentClassification", + "SwahiliNewsClassification", + "DalajClassification", + "TswanaNewsClassification", + "IsiZuluNewsClassification", + "WikiCitiesClustering", + "MasakhaNEWSClusteringS2S", + "RomaniBibleClustering", + "ArXivHierarchicalClusteringP2P", + "ArXivHierarchicalClusteringS2S", + "BigPatentClustering.v2", + "BiorxivClusteringP2P.v2", + "MedrxivClusteringP2P.v2", + "StackExchangeClustering.v2", + "AlloProfClusteringS2S.v2", + "HALClusteringS2S.v2", + "SIB200ClusteringS2S", + "WikiClusteringP2P.v2", + "SNLHierarchicalClusteringP2P", + "PlscClusteringP2P.v2", + "SwednClusteringP2P", + "CLSClusteringP2P.v2", + "StackOverflowQA", + "TwitterHjerneRetrieval", + "AILAStatutes", + "ArguAna", + "HagridRetrieval", + "LegalBenchCorporateLobbying", + "LEMBPasskeyRetrieval", + "SCIDOCS", + "SpartQA", + "TempReasonL1", + "TRECCOVID", + "WinoGrande", + "BelebeleRetrieval", + "MLQARetrieval", + "StatcanDialogueDatasetRetrieval", + "WikipediaRetrievalMultilingual", + "CovidRetrieval", + "Core17InstructionRetrieval", + "News21InstructionRetrieval", + "Robust04InstructionRetrieval", + "KorHateSpeechMLClassification", + "MalteseNewsClassification", + "MultiEURLEXMultilabelClassification", + "BrazilianToxicTweetsClassification", + "CEDRClassification", + "CTKFactsNLI", + "SprintDuplicateQuestions", + "TwitterURLCorpus", + "ArmenianParaphrasePC", + "indonli", + "OpusparcusPC", + "PawsXPairClassification", + "RTE3", + "XNLI", + "PpcPC", + "TERRa", + "WebLINXCandidatesReranking", + "AlloprofReranking", + "VoyageMMarcoReranking", + "WikipediaRerankingMultilingual", + "RuBQReranking", + "T2Reranking", + "GermanSTSBenchmark", + "SICK-R", + "STS12", + "STS13", + "STS14", + "STS15", + "STSBenchmark", + "FaroeseSTS", + "FinParaSTS", + "JSICK", + "IndicCrosslingualSTS", + "SemRel24STS", + "STS17", + "STS22.v2", + "STSES", + "STSB", + "MIRACLRetrievalHardNegatives", + ], + ), + description="The Multilingual benchmarks from MMTEB. Currently under development.", + reference=None, + citation=None, +) + +MTEB_JPN = Benchmark( + name="MTEB(jpn)", + tasks=get_tasks( + languages=["jpn"], + tasks=[ + # clustering + "LivedoorNewsClustering.v2", + "MewsC16JaClustering", + # classification + "AmazonReviewsClassification", + "AmazonCounterfactualClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + # STS + "JSTS", + "JSICK", + # pair classification + "PawsXPairClassification", + # retrieval + "JaqketRetrieval", + "MrTidyRetrieval", + "JaGovFaqsRetrieval", + "NLPJournalTitleAbsRetrieval", + "NLPJournalAbsIntroRetrieval", + "NLPJournalTitleIntroRetrieval", + # reranking + "ESCIReranking", + ], + ), + description="Main Japanese benchmarks from MTEB", + reference="https://github.com/sbintuitions/JMTEB", + citation=None, +) + + +MTEB_INDIC = Benchmark( + name="MTEB(indic)", + tasks=get_tasks( + tasks=[ + # Bitext + "IN22ConvBitextMining", + "IN22GenBitextMining", + "IndicGenBenchFloresBitextMining", + "LinceMTBitextMining", + # clustering + "SIB200ClusteringS2S", + # classification + "BengaliSentimentAnalysis", + "GujaratiNewsClassification", + "HindiDiscourseClassification", + "SentimentAnalysisHindi", + "MalayalamNewsClassification", + "IndicLangClassification", + "MTOPIntentClassification", + "MultiHateClassification", + "TweetSentimentClassification", + "NepaliNewsClassification", + "PunjabiNewsClassification", + "SanskritShlokasClassification", + "UrduRomanSentimentClassification", + # STS + "IndicCrosslingualSTS", + # pair classification + "XNLI", + # retrieval + "BelebeleRetrieval", + "XQuADRetrieval", + # reranking + "WikipediaRerankingMultilingual", + ], + ), + description="Main Indic benchmark from MMTEB", + reference=None, + citation=None, +) + + +MTEB_EU = Benchmark( + name="MTEB(Europe)", + tasks=get_tasks( + tasks=[ + "BornholmBitextMining", + "BibleNLPBitextMining", + "BUCC.v2", + "DiaBlaBitextMining", + "FloresBitextMining", + "NorwegianCourtsBitextMining", + "NTREXBitextMining", + "BulgarianStoreReviewSentimentClassfication", + "CzechProductReviewSentimentClassification", + "GreekLegalCodeClassification", + "DBpediaClassification", + "FinancialPhrasebankClassification", + "PoemSentimentClassification", + "ToxicChatClassification", + "ToxicConversationsClassification", + "EstonianValenceClassification", + "ItaCaseholdClassification", + "AmazonCounterfactualClassification", + "MassiveScenarioClassification", + "MultiHateClassification", + "NordicLangClassification", + "ScalaClassification", + "SwissJudgementClassification", + "TweetSentimentClassification", + "CBD", + "PolEmo2.0-OUT", + "CSFDSKMovieReviewSentimentClassification", + "DalajClassification", + "WikiCitiesClustering", + "RomaniBibleClustering", + "BigPatentClustering.v2", + "BiorxivClusteringP2P.v2", + "AlloProfClusteringS2S.v2", + "HALClusteringS2S.v2", + "SIB200ClusteringS2S", + "WikiClusteringP2P.v2", + "StackOverflowQA", + "TwitterHjerneRetrieval", + "LegalQuAD", + "ArguAna", + "HagridRetrieval", + "LegalBenchCorporateLobbying", + "LEMBPasskeyRetrieval", + "SCIDOCS", + "SpartQA", + "TempReasonL1", + "WinoGrande", + "AlloprofRetrieval", + "BelebeleRetrieval", + "StatcanDialogueDatasetRetrieval", + "WikipediaRetrievalMultilingual", + "Core17InstructionRetrieval", + "News21InstructionRetrieval", + "Robust04InstructionRetrieval", + "MalteseNewsClassification", + "MultiEURLEXMultilabelClassification", + "CTKFactsNLI", + "SprintDuplicateQuestions", + "OpusparcusPC", + "RTE3", + "XNLI", + "PSC", + "WebLINXCandidatesReranking", + "AlloprofReranking", + "WikipediaRerankingMultilingual", + "SICK-R", + "STS12", + "STS14", + "STS15", + "STSBenchmark", + "FinParaSTS", + "STS17", + "SICK-R-PL", + "STSES", + ] + ), + description="Main European benchmark from MMTEB", + reference=None, + citation=None, +) diff --git a/mteb/benchmarks/get_benchmark.py b/mteb/benchmarks/get_benchmark.py new file mode 100644 index 0000000000..b60b40fc59 --- /dev/null +++ b/mteb/benchmarks/get_benchmark.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import difflib + +import mteb.benchmarks.benchmarks as benchmark_module +from mteb.benchmarks.benchmarks import Benchmark + +BENCHMARK_REGISTRY = { + inst.name: inst + for nam, inst in benchmark_module.__dict__.items() + if isinstance(inst, Benchmark) +} + + +def get_benchmark( + benchmark_name: str, +) -> Benchmark: + if benchmark_name not in BENCHMARK_REGISTRY: + close_matches = difflib.get_close_matches( + benchmark_name, BENCHMARK_REGISTRY.keys() + ) + if close_matches: + suggestion = f"KeyError: '{benchmark_name}' not found. Did you mean: {close_matches[0]}?" + else: + suggestion = f"KeyError: '{benchmark_name}' not found and no similar keys were found." + raise KeyError(suggestion) + return BENCHMARK_REGISTRY[benchmark_name] + + +def get_benchmarks( + names: list[str] | None = None, +) -> list[Benchmark]: + if names is None: + names = list(BENCHMARK_REGISTRY.keys()) + return [BENCHMARK_REGISTRY[name] for name in names] diff --git a/mteb/cli.py b/mteb/cli.py index 24e99bd241..65d6938416 100644 --- a/mteb/cli.py +++ b/mteb/cli.py @@ -30,6 +30,14 @@ mteb available_tasks --task_types Clustering # list tasks of type Clustering ``` +## Listing Available Benchmarks + +To list the available benchmarks within MTEB, use the `mteb available_benchmarks` command. For example: + +```bash +mteb available_benchmarks # list all available benchmarks +``` + ## Creating Model Metadata @@ -114,12 +122,16 @@ def run(args: argparse.Namespace) -> None: model = mteb.get_model(args.model, args.model_revision, device=device) - tasks = mteb.get_tasks( - categories=args.categories, - task_types=args.task_types, - languages=args.languages, - tasks=args.tasks, - ) + if args.benchmarks: + tasks = mteb.get_benchmarks(names=args.benchmarks) + else: + tasks = mteb.get_tasks( + categories=args.categories, + task_types=args.task_types, + languages=args.languages, + tasks=args.tasks, + ) + eval = mteb.MTEB(tasks=tasks) encode_kwargs = {} @@ -144,6 +156,12 @@ def run(args: argparse.Namespace) -> None: _save_model_metadata(model, Path(args.output_folder)) +def available_benchmarks(args: argparse.Namespace) -> None: + benchmarks = mteb.get_benchmarks(names=args.benchmarks) + eval = mteb.MTEB(tasks=benchmarks) + eval.mteb_benchmarks() + + def available_tasks(args: argparse.Namespace) -> None: tasks = mteb.get_tasks( categories=args.categories, @@ -155,6 +173,18 @@ def available_tasks(args: argparse.Namespace) -> None: eval.mteb_tasks() +def add_benchmark_selection_args(parser: argparse.ArgumentParser) -> None: + """Adds arguments to the parser for filtering benchmarks by name.""" + parser.add_argument( + "-b", + "--benchmarks", + nargs="+", + type=str, + default=None, + help="List of benchmark to be evaluated.", + ) + + def add_task_selection_args(parser: argparse.ArgumentParser) -> None: """Adds arguments to the parser for filtering tasks by type, category, language, and task name.""" parser.add_argument( @@ -198,6 +228,15 @@ def add_available_tasks_parser(subparsers) -> None: parser.set_defaults(func=available_tasks) +def add_available_benchmarks_parser(subparsers) -> None: + parser = subparsers.add_parser( + "available_benchmarks", help="List the available benchmarks within MTEB" + ) + add_benchmark_selection_args(parser) + + parser.set_defaults(func=available_benchmarks) + + def add_run_parser(subparsers) -> None: parser = subparsers.add_parser("run", help="Run a model on a set of tasks") @@ -209,6 +248,7 @@ def add_run_parser(subparsers) -> None: ) add_task_selection_args(parser) + add_benchmark_selection_args(parser) parser.add_argument( "--device", type=int, default=None, help="Device to use for computation" @@ -321,6 +361,7 @@ def main(): ) add_run_parser(subparsers) add_available_tasks_parser(subparsers) + add_available_benchmarks_parser(subparsers) add_create_meta_parser(subparsers) args = parser.parse_args() diff --git a/mteb/create_meta.py b/mteb/create_meta.py index 5fde28394c..e810751a08 100644 --- a/mteb/create_meta.py +++ b/mteb/create_meta.py @@ -7,8 +7,8 @@ import yaml import mteb -from mteb import MTEBResults -from mteb.load_results.mteb_results import CQADupstackRetrievalDummy +from mteb import TaskResult +from mteb.load_results.task_results import CQADupstackRetrievalDummy def generate_readme(results_folder: Path, from_existing: Path | None = None) -> str: @@ -45,7 +45,7 @@ def load_model_name(results_folder: Path) -> str: return "PLACEHOLDER" -def process_task_result(task_result: MTEBResults) -> list[dict[str, Any]]: +def process_task_result(task_result: TaskResult) -> list[dict[str, Any]]: # CQADupstackRetrieval is a combined dataset (special case atm.) task = ( CQADupstackRetrievalDummy() @@ -84,13 +84,13 @@ def process_task_result(task_result: MTEBResults) -> list[dict[str, Any]]: return yaml_results -def get_task_results(results_folder: Path) -> list[MTEBResults]: +def get_task_results(results_folder: Path) -> list[TaskResult]: json_files = [ r for r in results_folder.glob("*.json") if r.is_file() and r.name != "model_meta.json" and "predictions" not in r.name ] - task_results = [MTEBResults.from_disk(path) for path in json_files] + task_results = [TaskResult.from_disk(path) for path in json_files] task_results = [ results for results in task_results @@ -102,8 +102,8 @@ def get_task_results(results_folder: Path) -> list[MTEBResults]: def potentially_add_cqadupstack_to_results( - results: list[MTEBResults], -) -> list[MTEBResults]: + results: list[TaskResult], +) -> list[TaskResult]: task_list_cqa = { "CQADupstackAndroidRetrieval", "CQADupstackEnglishRetrieval", @@ -128,7 +128,7 @@ def potentially_add_cqadupstack_to_results( main_scores = [r.get_score(splits=["test"]) for r in cqa_results] main_score = float(sum(main_scores) / len(main_scores)) - combined_result = MTEBResults( + combined_result = TaskResult( task_name="CQADupstackRetrieval", dataset_revision="CQADupstackRetrieval_is_a_combined_dataset", mteb_version="NA", diff --git a/mteb/encoder_interface.py b/mteb/encoder_interface.py index 21ef24cbba..c8a45f711d 100644 --- a/mteb/encoder_interface.py +++ b/mteb/encoder_interface.py @@ -1,11 +1,18 @@ from __future__ import annotations -from typing import Any, Dict, List, Protocol, Sequence, Union, runtime_checkable +from collections.abc import Sequence +from enum import Enum +from typing import Any, Protocol, Union, runtime_checkable import numpy as np import torch -Corpus = Union[List[Dict[str, str]], Dict[str, List[str]]] +Corpus = Union[list[dict[str, str]], dict[str, list[str]]] + + +class PromptType(str, Enum): + query = "query" + passage = "passage" @runtime_checkable @@ -25,16 +32,30 @@ def __init__(self, device: str | None = None) -> None: self.device = device def encode( - self, sentences: Sequence[str], *, prompt_name: str | None = None, **kwargs: Any - ) -> torch.Tensor | np.ndarray: + self, + sentences: Sequence[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: """Encodes the given sentences using the encoder. Args: sentences: The sentences to encode. - prompt_name: The name of the prompt. This will just be the name of the task. Sentence-transformers uses this to + task_name: The name of the task. Sentence-transformers uses this to determine which prompt to use from a specified dictionary. + prompt_type: The name type of prompt. (query or passage) **kwargs: Additional arguments to pass to the encoder. + The order of priorities for prompt selection are: + 1. Composed prompt of task name + prompt type (query or passage) + 2. Specific task prompt + 3. Composed prompt of task type + prompt type (query or passage) + 4. Specific task type prompt + 5. Specific prompt type (query or passage) + + Returns: The encoded sentences. """ @@ -88,43 +109,6 @@ def similarity_pairwise( ... -@runtime_checkable -class EncoderWithQueryCorpusEncode(Encoder, Protocol): - """The optional interface for an encoder that supports encoding queries and a corpus.""" - - def encode_queries( - self, queries: Sequence[str], *, prompt_name: str | None = None, **kwargs: Any - ) -> torch.Tensor | np.ndarray: - """Encodes the given queries using the encoder. - - Args: - queries: The queries to encode. - prompt_name: The name of the prompt. This will just be the name of the task. Sentence-transformers uses this to - determine which prompt to use from a specified dictionary. - **kwargs: Additional arguments to pass to the encoder. - - Returns: - The encoded queries. - """ - ... - - def encode_corpus( - self, corpus: Corpus, *, prompt_name: str | None = None, **kwargs: Any - ) -> torch.Tensor | np.ndarray: - """Encodes the given corpus using the encoder. - - Args: - corpus: The corpus to encode. - prompt_name: The name of the prompt. This will just be the name of the task. Sentence-transformers uses this to - determine which prompt to use from a specified dictionary. - **kwargs: Additional arguments to pass to the encoder. - - Returns: - The encoded corpus. - """ - ... - - @runtime_checkable class EncoderWithConversationEncode(Encoder, Protocol): """The optional interface for an encoder that supports encoding conversations.""" @@ -133,17 +117,24 @@ def encode_conversations( self, conversations: Sequence[Sequence[str]], *, - prompt_name: str | None = None, + task_name: str | None = None, **kwargs: Any, ) -> torch.Tensor | np.ndarray: """Encodes the given conversations using the encoder. Args: conversations: The conversations to encode. - prompt_name: The name of the prompt. This will just be the name of the task. Sentence-transformers uses this to + task_name: The name of the task. Sentence-transformers uses this to determine which prompt to use from a specified dictionary. **kwargs: Additional arguments to pass to the encoder. + The order of priorities for prompt selection are: + 1. Composed prompt of task name + prompt type (query or passage) + 2. Specific task prompt + 3. Composed prompt of task type + prompt type (query or passage) + 4. Specific task type prompt + 5. Specific prompt type (query or passage) + Returns: The encoded conversations. """ diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py index 0ac12d4bd2..d6c1b43ab4 100644 --- a/mteb/evaluation/MTEB.py +++ b/mteb/evaluation/MTEB.py @@ -4,14 +4,16 @@ import logging import os import traceback +from collections.abc import Iterable from copy import copy from datetime import datetime +from itertools import chain from pathlib import Path from time import time -from typing import Any, Iterable +from typing import Any import datasets -from sentence_transformers import SentenceTransformer +from sentence_transformers import CrossEncoder, SentenceTransformer from mteb.encoder_interface import Encoder from mteb.model_meta import ModelMeta @@ -19,7 +21,8 @@ from ..abstasks import * from ..abstasks import AbsTask -from ..load_results.mteb_results import MTEBResults +from ..load_results.task_results import TaskResult +from ..models.sentence_transformer_wrapper import SentenceTransformerWrapper from ..tasks import * from . import LangMapping @@ -52,12 +55,17 @@ def __init__( err_logs_path: Path to save error logs. kwargs: Additional arguments to be passed to the tasks """ + from mteb.benchmarks import Benchmark + self.deprecation_warning( task_types, task_categories, task_langs, tasks, version ) if tasks is not None: self._tasks = tasks + if isinstance(tasks[0], Benchmark): + self.benchmarks = tasks + self._tasks = list(chain.from_iterable(tasks)) assert ( task_types is None and task_categories is None ), "Cannot specify both `tasks` and `task_types`/`task_categories`" @@ -114,7 +122,8 @@ def available_tasks(self): @property def available_task_types(self): - return {x.metadata_dict["type"] for x in self.tasks_cls} + # sort the task types + return sorted({x.metadata_dict["type"] for x in self.tasks_cls}) @property def available_task_categories(self): @@ -148,7 +157,7 @@ def _display_tasks(self, task_list, name=None): console = Console() if name: console.rule(f"[bold]{name}\n", style="grey15") - for task_type in self.available_task_types: + for task_type in self.available_task_types: # iterate through sorted task_types current_type_tasks = list( filter(lambda x: x.metadata.type == task_type, task_list) ) @@ -156,7 +165,9 @@ def _display_tasks(self, task_list, name=None): continue else: console.print(f"[bold]{task_type}[/]") - for task in current_type_tasks: + for ( + task + ) in current_type_tasks: # will be sorted as input to this function prefix = " - " name = f"{task.metadata.name}" category = f", [italic grey39]{task.metadata.category}[/]" @@ -168,6 +179,35 @@ def _display_tasks(self, task_list, name=None): console.print(f"{prefix}{name}{category}{multilingual}") console.print("\n") + def mteb_benchmarks(self): + """Get all benchmarks available in the MTEB.""" + from mteb.overview import MTEBTasks + + # get all the MTEB specific benchmarks: + sorted_mteb_benchmarks = sorted( + self.benchmarks, key=lambda obj: obj.name.lower() + ) + + mteb_b, remaining_b = [], [] + for b in sorted_mteb_benchmarks: + if "MTEB" in b.name: + mteb_b.append(b) + else: + remaining_b.append(b) + + # place mteb first, then remaining + sorted_mteb_benchmarks = mteb_b + remaining_b + + # task ordering within each benchmark should be alphabetical + for st in sorted_mteb_benchmarks: + st.tasks = MTEBTasks( + sorted(st.tasks, key=lambda obj: obj.metadata.name.lower()) + ) + + for benchmark in sorted_mteb_benchmarks: + name = benchmark.name + self._display_tasks(benchmark.tasks, name=name) + @classmethod def mteb_tasks(cls): """Get all tasks available in the MTEB.""" @@ -278,7 +318,7 @@ def run( co2_tracker: bool = False, encode_kwargs: dict[str, Any] = {}, **kwargs, - ): + ) -> list[TaskResult]: """Run the evaluation pipeline on the selected tasks. Args: @@ -297,7 +337,7 @@ def run( kwargs: Additional arguments to be passed to `_run_eval` method and task.load_data. Returns: - A list of MTEBResults objects, one for each task evaluated. + A list of TaskResult objects, one for each task evaluated. """ if "batch_size" in kwargs: logger.warning( @@ -313,6 +353,8 @@ def run( meta = self.create_model_meta(model) output_path = self.create_output_folder(meta, output_folder) + if isinstance(model, (SentenceTransformer, CrossEncoder)): + model = SentenceTransformerWrapper(model) if output_path: self._save_model_metadata(meta, output_path) @@ -335,15 +377,15 @@ def run( save_path = output_path / f"{task.metadata.name}{task.save_suffix}.json" if save_path.exists() and not overwrite_results: logger.info( - f"{task.metadata.name} results already exists. Skipping. Set overwrite_results=True to overwrite." + f"{task.metadata.name} results already exists. Loading results from disk. Set overwrite_results=True to overwrite." ) - del self.tasks[0] + mteb_results = TaskResult.from_disk(save_path) + evaluation_results.append(mteb_results) + del self.tasks[0] # empty memory continue try: task_eval_splits = ( - eval_splits - if eval_splits is not None - else task.metadata_dict.get("eval_splits", []) + eval_splits if eval_splits is not None else task.eval_splits ) # load data @@ -398,7 +440,7 @@ def run( if verbosity >= 1: logger.info(f"Scores: {results}") - mteb_task_result = MTEBResults.from_task_results( + mteb_task_result = TaskResult.from_task_results( task, task_results, evaluation_time=evaluation_time, diff --git a/mteb/evaluation/evaluators/BitextMiningEvaluator.py b/mteb/evaluation/evaluators/BitextMiningEvaluator.py index 60769489b3..122abdf799 100644 --- a/mteb/evaluation/evaluators/BitextMiningEvaluator.py +++ b/mteb/evaluation/evaluators/BitextMiningEvaluator.py @@ -9,9 +9,9 @@ from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score from mteb.encoder_interface import Encoder +from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator -from .model_encode import model_encode from .utils import cos_sim logger = logging.getLogger(__name__) @@ -52,12 +52,12 @@ def compute_metrics(self, model: Encoder, encode_kwargs: dict[str, Any] = {}): embeddings = {} for sub in tqdm.tqdm(subsets, desc=f"Encoding {n_subsets}x{self.n} sentences"): - embeddings[sub] = model_encode( + emb = model.encode( self.sentences[sub], - model=model, - prompt_name=self.task_name, + task_name=self.task_name, **encode_kwargs, ) + embeddings[sub] = normalize_embeddings_to_numpy(emb) scores = {} for i, (key1, key2) in enumerate( diff --git a/mteb/evaluation/evaluators/ClassificationEvaluator.py b/mteb/evaluation/evaluators/ClassificationEvaluator.py index edfd8e4f8f..a5bb725315 100644 --- a/mteb/evaluation/evaluators/ClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/ClassificationEvaluator.py @@ -15,7 +15,7 @@ from torch import Tensor from mteb.encoder_interface import Encoder -from mteb.evaluation.evaluators.model_encode import model_encode +from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator @@ -63,19 +63,19 @@ def __call__(self, model, test_cache=None): max_accuracy = 0 max_f1 = 0 max_ap = 0 - X_train = model_encode( + emb = model.encode( self.sentences_train, - model=model, - prompt_name=self.task_name, + task_name=self.task_name, **self.encode_kwargs, ) + X_train = normalize_embeddings_to_numpy(emb) if test_cache is None: - X_test = model_encode( + emb = model.encode( self.sentences_test, - model=model, - prompt_name=self.task_name, + task_name=self.task_name, **self.encode_kwargs, ) + X_test = normalize_embeddings_to_numpy(emb) test_cache = X_test else: X_test = test_cache @@ -139,19 +139,21 @@ def __call__(self, model: Encoder, test_cache=None): max_accuracy = 0 max_f1 = 0 max_ap = 0 - X_train = model_encode( - self.sentences_train, - model=model, - prompt_name=self.task_name, - **self.encode_kwargs, + X_train = normalize_embeddings_to_numpy( + model.encode( + self.sentences_train, + task_name=self.task_name, + **self.encode_kwargs, + ) ) if test_cache is None: - X_test = model_encode( - self.sentences_test, - model=model, - prompt_name=self.task_name, - **self.encode_kwargs, + X_test = normalize_embeddings_to_numpy( + model.encode( + self.sentences_test, + task_name=self.task_name, + **self.encode_kwargs, + ) ) test_cache = X_test else: @@ -293,18 +295,20 @@ def __call__(self, model, test_cache=None): max_iter=self.max_iter, verbose=1 if logger.isEnabledFor(logging.DEBUG) else 0, ) - X_train = model_encode( - self.sentences_train, - model=model, - prompt_name=self.task_name, - **self.encode_kwargs, + X_train = normalize_embeddings_to_numpy( + model.encode( + self.sentences_train, + task_name=self.task_name, + **self.encode_kwargs, + ) ) if test_cache is None: - X_test = model_encode( - self.sentences_test, - model=model, - prompt_name=self.task_name, - **self.encode_kwargs, + X_test = normalize_embeddings_to_numpy( + model.encode( + self.sentences_test, + task_name=self.task_name, + **self.encode_kwargs, + ) ) test_cache = X_test else: diff --git a/mteb/evaluation/evaluators/ClusteringEvaluator.py b/mteb/evaluation/evaluators/ClusteringEvaluator.py index f8a68a9a90..138ac565d6 100644 --- a/mteb/evaluation/evaluators/ClusteringEvaluator.py +++ b/mteb/evaluation/evaluators/ClusteringEvaluator.py @@ -8,9 +8,9 @@ from sklearn import metrics from mteb.encoder_interface import Encoder +from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator -from .model_encode import model_encode logger = logging.getLogger(__name__) @@ -38,11 +38,12 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): if "batch_size" not in encode_kwargs: encode_kwargs["batch_size"] = 32 - corpus_embeddings = model_encode( - self.sentences, - model=model, - prompt_name=self.task_name, - **encode_kwargs, + corpus_embeddings = normalize_embeddings_to_numpy( + model.encode( + self.sentences, + task_name=self.task_name, + **encode_kwargs, + ) ) logger.info("Fitting Mini-Batch K-Means model...") diff --git a/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py index 20e8547536..a391af472b 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py @@ -16,7 +16,7 @@ from torch.utils.data import DataLoader from torchvision import transforms -from mteb.encoder_interface import EncoderWithQueryCorpusEncode +from mteb.encoder_interface import Encoder from ..Evaluator import Evaluator from ..utils import ( @@ -68,7 +68,7 @@ def custom_collate_fn(batch): class Any2AnyMultiChoiceSearch: def __init__( self, - model: EncoderWithQueryCorpusEncode, + model: Encoder, encode_kwargs: dict[str, Any] = {}, corpus_chunk_size: int = 20000, previous_results: str | None = None, diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index 3b96babb81..cb0119ea6e 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -16,7 +16,7 @@ from torch.utils.data import DataLoader from torchvision import transforms -from mteb.encoder_interface import EncoderWithQueryCorpusEncode +from mteb.encoder_interface import Encoder from ..Evaluator import Evaluator from ..utils import ( @@ -68,7 +68,7 @@ def custom_collate_fn(batch): class Any2AnyDenseRetrievalExactSearch: def __init__( self, - model: EncoderWithQueryCorpusEncode, + model: Encoder, encode_kwargs: dict[str, Any] = {}, corpus_chunk_size: int = 20000, previous_results: str | None = None, diff --git a/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py b/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py index 10c78d5d0d..f17dad9872 100644 --- a/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py @@ -26,6 +26,19 @@ def __call__( return self.retriever.search_cross_encoder( corpus, queries, self.top_k, instructions=instructions, **kwargs ) + elif ( + hasattr(self.retriever.model, "mteb_model_meta") + and self.retriever.model.mteb_model_meta.name == "bm25s" + ): + return self.retriever.model.search( + corpus, + queries, + self.top_k, + self.score_function, + task_name=self.task_name, # type: ignore + instructions=instructions, + **kwargs, + ) else: return self.retriever.search( corpus, @@ -34,6 +47,6 @@ def __call__( self.score_function, instructions=instructions, request_qid=qid, - prompt_name=self.task_name, + task_name=self.task_name, **kwargs, ) diff --git a/mteb/evaluation/evaluators/PairClassificationEvaluator.py b/mteb/evaluation/evaluators/PairClassificationEvaluator.py index 1f4449ffbf..c402383fcd 100644 --- a/mteb/evaluation/evaluators/PairClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/PairClassificationEvaluator.py @@ -13,7 +13,7 @@ ) from mteb.encoder_interface import Encoder, EncoderWithSimilarity -from mteb.evaluation.evaluators.model_encode import model_encode +from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator @@ -90,11 +90,12 @@ def compute_metrics( logger.warning( f"Found {n_duplicates}/{total_sents} duplicates in the input data. Only encoding unique sentences." ) - embeddings = model_encode( - sentences, - model=model, - prompt_name=self.task_name, - **encode_kwargs, + embeddings = normalize_embeddings_to_numpy( + model.encode( + sentences, + task_name=self.task_name, + **encode_kwargs, + ) ) emb_dict = dict(zip(sentences, embeddings)) embeddings1 = [emb_dict[sent] for sent in self.sentences1] diff --git a/mteb/evaluation/evaluators/RerankingEvaluator.py b/mteb/evaluation/evaluators/RerankingEvaluator.py index 60720954f5..44994681be 100644 --- a/mteb/evaluation/evaluators/RerankingEvaluator.py +++ b/mteb/evaluation/evaluators/RerankingEvaluator.py @@ -1,8 +1,7 @@ from __future__ import annotations import logging -from functools import partial -from typing import Any, Callable +from typing import Any import numpy as np import torch @@ -10,10 +9,10 @@ from sklearn.metrics import average_precision_score from mteb.evaluation.evaluators.RetrievalEvaluator import RetrievalEvaluator +from mteb.normalize_embeddings import normalize_embeddings_to_numpy -from ...encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from ...encoder_interface import Encoder, PromptType from .Evaluator import Evaluator -from .model_encode import model_encode from .utils import confidence_scores, cos_sim, nAUC logger = logging.getLogger(__name__) @@ -70,40 +69,28 @@ def __init__( if len(sample["positive"]) > 0 and len(sample["negative"]) > 0 ] - def __call__(self, model): + def __call__(self, model: Encoder): scores = self.compute_metrics(model) return scores - def compute_metrics(self, model): + def compute_metrics(self, model: Encoder): return ( self.compute_metrics_batched(model) if self.use_batched_encoding else self.compute_metrics_individual(model) ) - def compute_metrics_batched(self, model: Encoder | EncoderWithQueryCorpusEncode): + def compute_metrics_batched(self, model: Encoder): """Computes the metrices in a batched way, by batching all queries and all documents together """ - # using encode_queries and encode_corpus functions if they exists, - # which can be defined by users to add different instructions for query and passage conveniently - encode_queries_func = ( - model.encode_queries - if isinstance(model, EncoderWithQueryCorpusEncode) - else partial(model_encode, model=model) - ) - encode_corpus_func = ( - model.encode_corpus - if isinstance(model, EncoderWithQueryCorpusEncode) - else partial(model_encode, model=model) - ) - logger.info("Encoding queries...") if isinstance(self.samples[0]["query"], str): - all_query_embs = np.asarray( - encode_queries_func( + all_query_embs = normalize_embeddings_to_numpy( + model.encode( [sample["query"] for sample in self.samples], - prompt_name=self.task_name, + task_name=self.task_name, + prompt_type=PromptType.query, **self.encode_kwargs, ) ) @@ -114,8 +101,9 @@ def compute_metrics_batched(self, model: Encoder | EncoderWithQueryCorpusEncode) ] all_query_embs = self._encode_unique_texts( all_query_flattened, - encode_queries_func, - prompt_name=self.task_name, + model, + task_name=self.task_name, + prompt_type=PromptType.query, **self.encode_kwargs, ) else: @@ -125,58 +113,44 @@ def compute_metrics_batched(self, model: Encoder | EncoderWithQueryCorpusEncode) if self.evaluator_type == "standard": results = self._encode_candidates( - encode_queries_func=encode_queries_func, - encode_corpus_func=encode_corpus_func, + model=model, batched=True, all_query_embs=all_query_embs, ) elif self.evaluator_type == "miracl": results = self._encode_candidates_miracl( - encode_queries_func=encode_queries_func, - encode_corpus_func=encode_corpus_func, + model=model, batched=True, all_query_embs=all_query_embs, ) return results - def compute_metrics_individual(self, model): + def compute_metrics_individual(self, model: Encoder): """Embeds every (query, positive, negative) tuple individually. Is slower than the batched version, but saves memory as only the embeddings for one tuple are needed. Useful when you have a really large test set """ - # using encode_queries and encode_corpus functions if they exists, - # which can be defined by users to add different instructions for query and passage conveniently - encode_queries_func = ( - model.encode_queries if hasattr(model, "encode_queries") else model.encode - ) - encode_corpus_func = ( - model.encode_corpus if hasattr(model, "encode_corpus") else model.encode - ) if self.evaluator_type == "standard": results = self._encode_candidates( - encode_queries_func=encode_queries_func, - encode_corpus_func=encode_corpus_func, + model=model, batched=False, ) elif self.evaluator_type == "miracl": results = self._encode_candidates_miracl( - encode_queries_func=encode_queries_func, - encode_corpus_func=encode_corpus_func, + model=model, batched=False, ) return results - def _encode_candidates( - self, encode_corpus_func, batched, all_query_embs=None, encode_queries_func=None - ): + def _encode_candidates(self, model: Encoder, batched: bool, all_query_embs=None): all_mrr_scores = [] all_ap_scores = [] all_conf_scores = [] logger.info("Encoding candidates...") if batched: self._encode_candidates_batched( - encode_corpus_func=encode_corpus_func, + model=model, all_query_embs=all_query_embs, all_mrr_scores=all_mrr_scores, all_ap_scores=all_ap_scores, @@ -184,8 +158,7 @@ def _encode_candidates( ) else: self._encode_candidates_individual( - encode_queries_func=encode_queries_func, - encode_corpus_func=encode_corpus_func, + model=model, all_mrr_scores=all_mrr_scores, all_ap_scores=all_ap_scores, all_conf_scores=all_conf_scores, @@ -196,7 +169,7 @@ def _encode_candidates( def _encode_candidates_batched( self, all_query_embs, - encode_corpus_func, + model: Encoder, all_mrr_scores, all_ap_scores, all_conf_scores, @@ -208,8 +181,9 @@ def _encode_candidates_batched( all_docs_embs = self._encode_unique_texts( all_docs, - encode_corpus_func, - prompt_name=self.task_name, + model, + task_name=self.task_name, + prompt_type=PromptType.passage, **self.encode_kwargs, ) @@ -242,8 +216,7 @@ def _encode_candidates_batched( def _encode_candidates_individual( self, - encode_queries_func, - encode_corpus_func, + model: Encoder, all_mrr_scores, all_ap_scores, all_conf_scores, @@ -260,10 +233,24 @@ def _encode_candidates_individual( is_relevant = [True] * len(positive) + [False] * len(negative) if isinstance(query, str): - # .encoding interface requires List[str] as input + # .encoding interface requires list[str] as input query = [query] - query_emb = np.asarray(encode_queries_func(query, **self.encode_kwargs)) - docs_emb = np.asarray(encode_corpus_func(docs, **self.encode_kwargs)) + query_emb = np.asarray( + model.encode( + query, + task_name=self.task_name, + prompt_type=PromptType.query, + **self.encode_kwargs, + ) + ) + docs_emb = np.asarray( + model.encode( + docs, + task_name=self.task_name, + prompt_type=PromptType.passage, + **self.encode_kwargs, + ) + ) self._apply_sim_scores( query_emb, docs_emb, @@ -285,29 +272,30 @@ def _collect_results(self, all_mrr_scores, all_ap_scores, all_conf_scores): def _encode_candidates_miracl( self, - encode_corpus_func, - encode_queries_func, + model: Encoder, batched, all_query_embs=None, ): if batched: return self._encode_candidates_miracl_batched( - all_query_embs=all_query_embs, encode_corpus_func=encode_corpus_func + model=model, all_query_embs=all_query_embs ) else: return self._encode_candidates_miracl_individual( - encode_queries_func=encode_queries_func, - encode_corpus_func=encode_corpus_func, + model=model, ) - def _encode_candidates_miracl_batched(self, all_query_embs, encode_corpus_func): + def _encode_candidates_miracl_batched(self, all_query_embs, model: Encoder): all_docs = [] for sample in self.samples: all_docs.extend(sample["candidates"]) all_docs_embs = np.asarray( - encode_corpus_func( - all_docs, prompt_name=self.task_name, **self.encode_kwargs + model.encode( + all_docs, + task_name=self.task_name, + prompt_type=PromptType.passage, + **self.encode_kwargs, ) ) @@ -337,9 +325,7 @@ def _encode_candidates_miracl_batched(self, all_query_embs, encode_corpus_func): scores_miracl = self._collect_miracl_results(results, qrels) return scores_miracl - def _encode_candidates_miracl_individual( - self, encode_queries_func, encode_corpus_func - ): + def _encode_candidates_miracl_individual(self, model: Encoder): results, qrels = {}, {} for i, instance in enumerate(tqdm.tqdm(self.samples, desc="Samples")): query = instance["query"] @@ -347,11 +333,23 @@ def _encode_candidates_miracl_individual( docs = list(instance["candidates"]) if isinstance(query, str): - # .encoding interface requires List[str] as input + # .encoding interface requires list[str] as input query_emb = np.asarray( - encode_queries_func([query], **self.encode_kwargs) + model.encode( + [query], + task_name=self.task_name, + prompt_type=PromptType.query, + **self.encode_kwargs, + ) + ) + docs_emb = np.asarray( + model.encode( + docs, + task_name=self.task_name, + prompt_type=PromptType.passage, + **self.encode_kwargs, + ) ) - docs_emb = np.asarray(encode_corpus_func(docs, **self.encode_kwargs)) fake_qid = str(i) results[fake_qid] = self.rerank(query_emb, docs_emb) @@ -420,8 +418,9 @@ def _apply_sim_scores( @staticmethod def _encode_unique_texts( all_texts: list[str], - encode_fn: Callable, - prompt_name: str | None, + model: Encoder, + task_name: str | None, + prompt_type: PromptType | None, **encode_kwargs: Any, ): index_map, all_unique_texts, all_texts_indexes = {}, [], [] @@ -434,8 +433,13 @@ def _encode_unique_texts( logger.warning( f"A total on {len(all_texts) - len(all_unique_texts)}/{len(all_texts)} duplicate texts were found during encoding. Only encoding unique text and duplicating embeddings across." ) - all_unique_texts_embs = np.asarray( - encode_fn(all_unique_texts, prompt_name=prompt_name, **encode_kwargs) + all_unique_texts_embs = normalize_embeddings_to_numpy( + model.encode( + all_unique_texts, + task_name=task_name, + prompt_type=prompt_type, + **encode_kwargs, + ) ) return all_unique_texts_embs[all_texts_indexes] @@ -547,8 +551,8 @@ def ap_score(is_relevant, pred_scores): """Computes AP score Args: - is_relevant (`List[bool]` of length `num_pos+num_neg`): True if the document is relevant - pred_scores (`List[float]` of length `num_pos+num_neg`): Predicted similarity scores + is_relevant (`list[bool]` of length `num_pos+num_neg`): True if the document is relevant + pred_scores (`list[float]` of length `num_pos+num_neg`): Predicted similarity scores Returns: ap_score (`float`): AP score diff --git a/mteb/evaluation/evaluators/RetrievalEvaluator.py b/mteb/evaluation/evaluators/RetrievalEvaluator.py index 2ea70cb9bd..f81647c3ad 100644 --- a/mteb/evaluation/evaluators/RetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/RetrievalEvaluator.py @@ -5,6 +5,7 @@ import logging import os from collections import defaultdict +from pathlib import Path from typing import Any import numpy as np @@ -12,13 +13,12 @@ import torch import tqdm from sentence_transformers import CrossEncoder, SentenceTransformer -from sentence_transformers.models import Transformer, WordEmbeddings -from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from mteb.encoder_interface import Encoder, PromptType from mteb.model_meta import ModelMeta +from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator -from .model_encode import model_encode from .utils import ( confidence_scores, convert_conv_history_to_query, @@ -35,14 +35,36 @@ logger = logging.getLogger(__name__) +def corpus_to_str( + corpus: list[dict[str, str]] | dict[str, list[str]] | list[str], +) -> list[str]: + if isinstance(corpus, dict): + sentences = [ + (corpus["title"][i] + " " + corpus["text"][i]).strip() + if "title" in corpus + else corpus["text"][i].strip() + for i in range(len(corpus["text"])) + ] + elif isinstance(corpus, list) and isinstance(corpus[0], dict): + sentences = [ + (doc["title"] + " " + doc["text"]).strip() + if "title" in doc + else doc["text"].strip() + for doc in corpus + ] + else: + sentences = corpus + return sentences + + # Adapted from https://github.com/beir-cellar/beir/blob/f062f038c4bfd19a8ca942a9910b1e0d218759d4/beir/retrieval/search/dense/exact_search.py#L12 class DenseRetrievalExactSearch: def __init__( self, - model: EncoderWithQueryCorpusEncode, + model: Encoder, encode_kwargs: dict[str, Any] = {}, corpus_chunk_size: int = 50000, - previous_results: str | None = None, + previous_results: str | Path | None = None, **kwargs: Any, ): # Model is class that provides encode_corpus() and encode_queries() @@ -62,7 +84,10 @@ def __init__( "dot": "Dot Product", } self.corpus_chunk_size = corpus_chunk_size - self.previous_results = previous_results + if isinstance(previous_results, Path): + self.previous_results = str(previous_results) + else: + self.previous_results = previous_results self.batch_size = encode_kwargs.get("batch_size") self.show_progress_bar = encode_kwargs.get("show_progress_bar") self.save_corpus_embeddings = kwargs.get("save_corpus_embeddings", False) @@ -83,13 +108,13 @@ def search( queries: dict[str, str | list[str]], top_k: int, score_function: str, - prompt_name: str, + task_name: str, instructions: dict[str, str] | None = None, request_qid: str | None = None, return_sorted: bool = False, **kwargs, ) -> dict[str, dict[str, float]]: - # Create embeddings for all queries using model.encode_queries() + # Create embeddings for all queries using model.encode # Runs semantic search against the corpus embeddings # Returns a ranked list with the corpus ids if score_function not in self.score_functions: @@ -107,20 +132,22 @@ def search( query_embeddings = self.encode_conversations( model=self.model, conversations=queries, # type: ignore - prompt_name=prompt_name, + task_name=task_name, **self.encode_kwargs, ) else: - query_embeddings = self.model.encode_queries( - queries, # type: ignore - prompt_name=prompt_name, - **self.encode_kwargs, + query_embeddings = normalize_embeddings_to_numpy( + self.model.encode( + queries, # type: ignore + task_name=task_name, + prompt_type=PromptType.query, + **self.encode_kwargs, + ) ) logger.info("Sorting Corpus by document length (Longest first)...") corpus_ids = sorted( corpus, - key=lambda k: len(corpus[k].get("title", "") + corpus[k].get("text", "")), reverse=True, ) corpus = [corpus[cid] for cid in corpus_ids] # type: ignore @@ -150,11 +177,14 @@ def search( ) else: # Encode chunk of corpus - sub_corpus_embeddings = self.model.encode_corpus( - corpus[corpus_start_idx:corpus_end_idx], # type: ignore - prompt_name=prompt_name, - request_qid=request_qid, - **self.encode_kwargs, + sub_corpus_embeddings = normalize_embeddings_to_numpy( + self.model.encode( + corpus[corpus_start_idx:corpus_end_idx], # type: ignore + task_name=task_name, + prompt_type=PromptType.passage, + request_qid=request_qid, + **self.encode_kwargs, + ) ) if self.save_corpus_embeddings and request_qid: self.corpus_embeddings[request_qid].append(sub_corpus_embeddings) @@ -232,11 +262,18 @@ def search_cross_encoder( ) -> dict[str, dict[str, float]]: """This function provides support for reranker (or cross-encoder) models that encoder query and document at the same time (typically with attention). Some notable examples include MonoBERT, MonoT5, RankLlama, etc. - Note: you must provide the path to the results to rerank to the __init__ function as `previous_results` + Note: you must provide the path to the results to rerank to the __init__ function as `previous_results` or else rerank all documents in the corpus """ pairs = [] # create the pairs for reranking for qid in queries.keys(): - q_results = self.previous_results[qid] + if self.previous_results is None: + # try to use all of them + logging.info( + f"previous_results is None. Using all the documents to rerank: {len(corpus)}" + ) + q_results = {doc_id: 0.0 for doc_id in corpus.keys()} + else: + q_results = self.previous_results[qid] # take the top-k only q_results_sorted = dict( sorted(q_results.items(), key=lambda item: item[1], reverse=True) @@ -249,13 +286,10 @@ def search_cross_encoder( else query ) for doc_id in top_n: - corpus_item = ( - corpus[doc_id].get("title", "") + " " + corpus[doc_id]["text"] - ).strip() pairs.append( ( query, - corpus_item, + corpus[doc_id], instructions[query] if instructions is not None else None, qid, doc_id, @@ -284,7 +318,7 @@ def search_cross_encoder( len(queries_in_pair) == len(corpus_in_pair) == len(instructions_in_pair) ) - if isinstance(self.model, CrossEncoder): + if isinstance(self.model.model, CrossEncoder): # can't take instructions, so add them here queries_in_pair = [ f"{q} {i}".strip() @@ -308,16 +342,25 @@ def predict(self, queries, passages, **kwargs): ) def encode_conversations( - self, model: Encoder, conversations: list[list[str]], prompt_name: str, **kwargs + self, + model: Encoder, + conversations: list[list[str]], + task_name: str, + **kwargs, ): if callable(getattr(self.model, "encode_conversations", None)): return model.encode_conversations( # type: ignore - conversations, prompt_name=prompt_name, **kwargs + conversations, task_name=task_name, **kwargs ) - # otherwise fallback to default implementation - # TODO: add a warning here + logger.warning( + "Model doesn't have encode_conversations fallback to default implementation" + ) queries = self.convert_conv_history_to_query(model, conversations) # type: ignore - return model.encode_queries(queries, prompt_name=prompt_name, **kwargs) # type: ignore + return normalize_embeddings_to_numpy( + model.encode( + queries, task_name=task_name, prompt_type=PromptType.query, **kwargs + ) + ) # type: ignore @staticmethod def convert_conv_history_to_query( @@ -329,7 +372,7 @@ def convert_conv_history_to_query( class DRESModel: - """Dense Retrieval Exact Search (DRES) requires an encode_queries & encode_corpus method. + """Dense Retrieval Exact Search (DRES). This class converts a model with just an .encode method into DRES format. """ @@ -341,32 +384,12 @@ def __init__(self, model, **kwargs): self.save_corpus_embeddings = kwargs.get("save_corpus_embeddings", False) self.corpus_embeddings = {} - def encode_queries( - self, queries: list[str], *, prompt_name: str, batch_size: int, **kwargs - ): - if self.use_sbert_model: - if isinstance(self.model._first_module(), Transformer): - logger.info( - f"Queries will be truncated to {self.model.get_max_seq_length()} tokens." - ) - elif isinstance(self.model._first_module(), WordEmbeddings): - logger.warning( - "Queries will not be truncated. This could lead to memory issues. In that case please lower the batch_size." - ) - - return model_encode( - queries, - model=self.model, - prompt_name=prompt_name, - batch_size=batch_size, - **kwargs, - ) - def encode_corpus( self, corpus: list[dict[str, str]], - prompt_name: str, + task_name: str, batch_size: int, + prompt_type: PromptType = PromptType.passage, request_qid: str | None = None, **kwargs, ): @@ -377,46 +400,40 @@ def encode_corpus( ): return self.corpus_embeddings[request_qid] - if isinstance(corpus, dict): - sentences = [ - (corpus["title"][i] + " " + corpus["text"][i]).strip() - if "title" in corpus - else corpus["text"][i].strip() - for i in range(len(corpus["text"])) - ] - else: - sentences = [ - (doc["title"] + " " + doc["text"]).strip() - if "title" in doc - else doc["text"].strip() - for doc in corpus - ] - - corpus_embeddings = model_encode( - sentences, - model=self.model, - prompt_name=prompt_name, - batch_size=batch_size, - **kwargs, + sentences = corpus_to_str(corpus) + corpus_embeddings = normalize_embeddings_to_numpy( + self.model.encode( + sentences, + task_name=task_name, + prompt_type=prompt_type, + batch_size=batch_size, + **kwargs, + ) ) if self.save_corpus_embeddings and request_qid: self.corpus_embeddings[request_qid] = corpus_embeddings return corpus_embeddings - def encode(self, sentences: list[str], prompt_name: str, **kwargs): - return self.encode_queries(sentences, prompt_name=prompt_name, **kwargs) - - -def is_dres_compatible(model): - for method in ["encode_queries", "encode_corpus"]: - op = getattr(model, method, None) - if not (callable(op)): - return False - return True + def encode( + self, + sentences: list[str], + task_name: str, + prompt_type: PromptType | None = None, + **kwargs, + ): + if prompt_type and prompt_type == PromptType.passage: + return self.encode_corpus( + sentences, task_name, prompt_type=prompt_type, **kwargs + ) + return normalize_embeddings_to_numpy( + self.model.encode( + sentences, task_name=task_name, prompt_type=prompt_type, **kwargs + ) + ) -def is_cross_encoder_compatible(model): +def is_cross_encoder_compatible(model) -> bool: op = getattr(model, "predict", None) return callable(op) @@ -425,7 +442,7 @@ def is_cross_encoder_compatible(model): class RetrievalEvaluator(Evaluator): def __init__( self, - retriever=None, + retriever, task_name: str | None = None, k_values: list[int] = [1, 3, 5, 10, 20, 100, 1000], score_function: str = "cos_sim", @@ -442,17 +459,7 @@ def __init__( retriever, encode_kwargs=encode_kwargs, **kwargs ) self.is_cross_encoder = True - elif is_dres_compatible(retriever): - logger.info( - "The custom encode_queries and encode_corpus functions of the model will be used" - ) - self.retriever = DenseRetrievalExactSearch( - retriever, encode_kwargs=encode_kwargs, **kwargs - ) else: - logger.info( - "The model does not have the optional encode_queries and encode_corpus functions. Wrapping it in DRESModel." - ) self.retriever = DenseRetrievalExactSearch( DRESModel(retriever), encode_kwargs=encode_kwargs, **kwargs ) @@ -482,7 +489,7 @@ def __call__( queries, self.top_k, self.score_function, - prompt_name=self.task_name, # type: ignore + task_name=self.task_name, # type: ignore ) else: return self.retriever.search( @@ -490,7 +497,7 @@ def __call__( queries, self.top_k, self.score_function, - prompt_name=self.task_name, # type: ignore + task_name=self.task_name, # type: ignore ) @staticmethod diff --git a/mteb/evaluation/evaluators/STSEvaluator.py b/mteb/evaluation/evaluators/STSEvaluator.py index c24670b44a..3d769fa957 100644 --- a/mteb/evaluation/evaluators/STSEvaluator.py +++ b/mteb/evaluation/evaluators/STSEvaluator.py @@ -12,9 +12,9 @@ ) from mteb.encoder_interface import Encoder, EncoderWithSimilarity +from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator -from .model_encode import model_encode logger = logging.getLogger(__name__) @@ -45,11 +45,19 @@ def __call__( *, encode_kwargs: dict[str, Any] = {}, ): - embeddings1 = model_encode( - self.sentences1, model=model, prompt_name=self.task_name, **encode_kwargs + embeddings1 = normalize_embeddings_to_numpy( + model.encode( + self.sentences1, + task_name=self.task_name, + **encode_kwargs, + ) ) - embeddings2 = model_encode( - self.sentences2, model=model, prompt_name=self.task_name, **encode_kwargs + embeddings2 = normalize_embeddings_to_numpy( + model.encode( + self.sentences2, + task_name=self.task_name, + **encode_kwargs, + ) ) logger.info("Evaluating...") @@ -77,8 +85,8 @@ def __call__( similarity_scores = np.array(_similarity_scores) if similarity_scores is not None: - pearson = pearsonr(self.gold_scores, similarity_scores) - spearman = spearmanr(self.gold_scores, similarity_scores) + pearson, _ = pearsonr(self.gold_scores, similarity_scores) + spearman, _ = spearmanr(self.gold_scores, similarity_scores) else: # if model does not have a similarity function, we assume the cosine similarity pearson = cosine_pearson diff --git a/mteb/evaluation/evaluators/SummarizationEvaluator.py b/mteb/evaluation/evaluators/SummarizationEvaluator.py index bffa2c1f24..153ef8c42b 100644 --- a/mteb/evaluation/evaluators/SummarizationEvaluator.py +++ b/mteb/evaluation/evaluators/SummarizationEvaluator.py @@ -10,9 +10,9 @@ from scipy.stats import pearsonr, spearmanr from mteb.encoder_interface import Encoder, EncoderWithSimilarity +from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator -from .model_encode import model_encode from .utils import cos_sim, dot_score # if later than python 3.13 use typing module @@ -75,27 +75,29 @@ def __call__( ] logger.info("Encoding human summaries...") - embs_human_summaries_all = model_encode( - [ - summary - for human_summaries in self.human_summaries - for summary in human_summaries - ], - model=model, - prompt_name=self.task_name, - **encode_kwargs, + embs_human_summaries_all = normalize_embeddings_to_numpy( + model.encode( + [ + summary + for human_summaries in self.human_summaries + for summary in human_summaries + ], + task_name=self.task_name, + **encode_kwargs, + ) ) logger.info("Encoding machine summaries...") - embs_machine_summaries_all = model_encode( - [ - summary - for machine_summaries in self.machine_summaries - for summary in machine_summaries - ], - model=model, - prompt_name=self.task_name, - **encode_kwargs, + embs_machine_summaries_all = normalize_embeddings_to_numpy( + model.encode( + [ + summary + for machine_summaries in self.machine_summaries + for summary in machine_summaries + ], + task_name=self.task_name, + **encode_kwargs, + ) ) # Split the embeddings into the original human & machine summaries @@ -236,26 +238,24 @@ def __call__( ] logger.info("Encoding human summaries...") - embs_human_summaries_all = model_encode( + embs_human_summaries_all = model.encode( [ summary for human_summaries in self.human_summaries for summary in human_summaries ], - model=model, - prompt_name=self.task_name, + task_name=self.task_name, **encode_kwargs, ) logger.info("Encoding machine summaries...") - embs_machine_summaries_all = model_encode( + embs_machine_summaries_all = model.encode( [ summary for machine_summaries in self.machine_summaries for summary in machine_summaries ], - model=model, - prompt_name=self.task_name, + task_name=self.task_name, **encode_kwargs, ) diff --git a/mteb/evaluation/evaluators/model_encode.py b/mteb/evaluation/evaluators/model_encode.py deleted file mode 100644 index 7c87ce09e4..0000000000 --- a/mteb/evaluation/evaluators/model_encode.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Sequence - -import numpy as np -import torch - -from mteb.encoder_interface import Encoder - -logger = logging.getLogger(__name__) - - -def model_encode( - sentences: Sequence[str], *, model: Encoder, prompt_name: str | None, **kwargs -) -> np.ndarray: - """A wrapper function around the model.encode method that handles the prompt_name argument and standardizes the output to a numpy array. - - Args: - sentences: The sentences to encode - model: The model to use for encoding - prompt_name: The prompt name to use for encoding - **kwargs: Additional arguments to pass to the model.encode method - """ - kwargs["prompt_name"] = prompt_name - if hasattr(model, "prompts"): - # check if prompts is an empty dict - if not model.prompts: # type: ignore - logger.info( - "Model does not support prompts. Removing prompt_name argument." - ) - kwargs.pop("prompt_name") - elif prompt_name not in model.prompts: # type: ignore - logger.info( - f"Prompt {prompt_name} not found in model prompts. Removing prompt_name argument." - ) - kwargs.pop("prompt_name") - logger.info(f"Encoding {len(sentences)} sentences.") - - embeddings = model.encode(sentences, **kwargs) - if isinstance(embeddings, torch.Tensor): - embeddings = embeddings.cpu().detach().float() - - return np.asarray(embeddings) diff --git a/mteb/evaluation/evaluators/utils.py b/mteb/evaluation/evaluators/utils.py index 0cdfb6bd72..95d84bd2f2 100644 --- a/mteb/evaluation/evaluators/utils.py +++ b/mteb/evaluation/evaluators/utils.py @@ -7,6 +7,7 @@ import requests import torch import tqdm +from packaging.version import Version from sklearn.metrics import auc @@ -281,13 +282,16 @@ def rank_score(x: dict[str, float]) -> float: def download(url: str, fname: str): resp = requests.get(url, stream=True) total = int(resp.headers.get("content-length", 0)) - with open(fname, "wb") as file, tqdm.tqdm( - desc=fname, - total=total, - unit="iB", - unit_scale=True, - unit_divisor=1024, - ) as bar: + with ( + open(fname, "wb") as file, + tqdm.tqdm( + desc=fname, + total=total, + unit="iB", + unit_scale=True, + unit_divisor=1024, + ) as bar, + ): for data in resp.iter_content(chunk_size=1024): size = file.write(data) bar.update(size) @@ -395,7 +399,11 @@ def abstention_curve( Returns: abst_curve: Abstention curve of length `len(abstention_rates)` """ - conf_scores_argsort = np.argsort(conf_scores) + # argsort stable=True is default in numpy >2.0.0 + if Version(np.__version__) < Version("2.0.0"): + conf_scores_argsort = np.argsort(conf_scores) + else: + conf_scores_argsort = np.argsort(conf_scores, stable=True) abst_curve = np.zeros(len(abstention_rates)) for i, rate in enumerate(abstention_rates): diff --git a/mteb/leaderboard/__init__.py b/mteb/leaderboard/__init__.py new file mode 100644 index 0000000000..1dc3560a64 --- /dev/null +++ b/mteb/leaderboard/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from mteb.leaderboard.app import demo + +__all__ = ["demo"] diff --git a/mteb/leaderboard/app.py b/mteb/leaderboard/app.py new file mode 100644 index 0000000000..7d49d009d1 --- /dev/null +++ b/mteb/leaderboard/app.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +from collections import defaultdict +from pathlib import Path + +import gradio as gr +from gradio_rangeslider import RangeSlider + +import mteb +from mteb.leaderboard.table import scores_to_tables + + +def load_results(): + results_cache_path = Path(__file__).parent.joinpath("__cached_results.json") + if not results_cache_path.exists(): + all_results = mteb.load_results() + all_results.to_disk(results_cache_path) + return all_results + else: + return mteb.BenchmarkResults.from_disk(results_cache_path) + + +all_results = load_results().filter_models() + +# Model sizes in million parameters +min_model_size, max_model_size = 8, 46703 + +benchmarks = mteb.get_benchmarks() + +default_benchmark = mteb.get_benchmark("MTEB(Multilingual)") +default_results = default_benchmark.load_results(base_results=all_results) + +benchmark_select = gr.Dropdown( + [bench.name for bench in benchmarks], + value=default_benchmark.name, + label="Prebuilt Benchmarks", + info="Select one of our expert-selected benchmarks from MTEB publications.", +) +lang_select = gr.Dropdown( + default_results.languages, + value=default_results.languages, + multiselect=True, + label="Language", + info="Select languages to include.", +) +type_select = gr.Dropdown( + default_results.task_types, + value=default_results.task_types, + multiselect=True, + label="Task Type", + info="Select task types to include.", +) +domain_select = gr.Dropdown( + default_results.domains, + value=default_results.domains, + multiselect=True, + label="Domain", + info="Select domains to include.", +) +task_select = gr.Dropdown( + default_results.task_names, + value=default_results.task_names, + allow_custom_value=True, + multiselect=True, + label="Task", + info="Select specific tasks to include", +) + +head = """ + +""" + +with gr.Blocks(fill_width=True, theme=gr.themes.Base(), head=head) as demo: + with gr.Row(): + with gr.Column(scale=1): + gr.Markdown( + """ + ### Model Selection + Select models to rank based on an assortment of criteria. + """, + ) + with gr.Group(): + with gr.Row(elem_classes="overflow-y-scroll max-h-80"): + with gr.Column(): + availability = gr.Radio( + [ + ("Only Open", True), + ("Only Proprietary", False), + ("Both", None), + ], + value=None, + label="Availability", + interactive=True, + ) + compatibility = gr.CheckboxGroup( + [ + ( + "Should be sentence-transformers compatible", + "sbert_compatible", + ) + ], + value=[], + label="Compatibility", + interactive=True, + ) + instructions = gr.Radio( + [ + ("Only Instruction-tuned", True), + ("Only non-instruction", False), + ("Both", None), + ], + value=None, + label="Instructions", + interactive=True, + ) + model_size = RangeSlider( + minimum=0, + maximum=8000, + value=(0, 8000), + label="Model Size (#M Parameters)", + interactive=True, + ) + with gr.Column(scale=2): + gr.Markdown( + """ + ### Benchmarks + Select one of the hand-curated benchmarks from our publication. + Or create one from scratch based on your use case. + """ + ) + with gr.Group(): + with gr.Row(elem_classes="overflow-y-scroll max-h-80"): + with gr.Column(): + benchmark_select.render() + with gr.Accordion("Select Languages", open=False): + lang_select.render() + with gr.Accordion("Select Task Types", open=False): + type_select.render() + with gr.Accordion("Select Domains", open=False): + domain_select.render() + with gr.Accordion("Add and remove tasks:", open=False): + task_select.render() + default_scores = default_results.get_scores(format="long") + scores = gr.State(default_scores) + summary, per_task = scores_to_tables(default_scores) + with gr.Tab("Summary"): + summary_table = gr.DataFrame(summary) + with gr.Tab("Performance per task"): + per_task_table = gr.DataFrame(per_task) + + @gr.on(inputs=[scores], outputs=[summary_table, per_task_table]) + def update_tables(scores): + summary, per_task = scores_to_tables(scores) + return summary, per_task + + @gr.on( + inputs=[benchmark_select], + outputs=[ + lang_select, + type_select, + domain_select, + ], + ) + def on_select_benchmark(benchmark_name): + benchmark = mteb.get_benchmark(benchmark_name) + benchmark_results = benchmark.load_results(base_results=all_results) + return ( + benchmark_results.languages, + benchmark_results.task_types, + benchmark_results.domains, + ) + + @gr.on( + inputs=[benchmark_select, lang_select, type_select, domain_select], + outputs=[task_select], + ) + def update_task_list(benchmark_name, languages, task_types, domains): + benchmark = mteb.get_benchmark(benchmark_name) + benchmark_results = benchmark.load_results(base_results=all_results) + task_to_lang_set = defaultdict(set) + task_to_type = {} + task_to_domains = defaultdict(set) + for model_res in benchmark_results: + for task_res in model_res: + task_to_lang_set[task_res.task_name] |= set(task_res.languages) + task_to_domains[task_res.task_name] |= set(task_res.domains) + task_to_type[task_res.task_name] = task_res.task_type + res = [] + for task_name in benchmark_results.task_names: + if not (task_to_domains[task_name] & set(domains)): + continue + if not (task_to_lang_set[task_name] & set(languages)): + continue + if task_to_type[task_name] not in task_types: + continue + res.append(task_name) + return res + + @gr.on( + inputs=[ + benchmark_select, + task_select, + lang_select, + type_select, + domain_select, + ], + outputs=[scores], + ) + def update_scores(benchmark_name, task_names, languages, task_types, domains): + benchmark = mteb.get_benchmark(benchmark_name) + benchmark_results = benchmark.load_results(base_results=all_results) + benchmark_results = benchmark_results.filter_tasks( + languages=languages, + task_names=task_names, + task_types=task_types, + domains=domains, + ) + scores = benchmark_results.get_scores(languages=languages, format="long") + return scores + + +if __name__ == "__main__": + demo.launch() diff --git a/mteb/leaderboard/table.py b/mteb/leaderboard/table.py new file mode 100644 index 0000000000..570d5bc6dd --- /dev/null +++ b/mteb/leaderboard/table.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import gradio as gr +import numpy as np +import pandas as pd + +from mteb.overview import get_task + + +def format_scores(score: float) -> float: + return score * 100 + + +def scores_to_tables(scores_long: list[dict]): + if not scores_long: + return gr.DataFrame(), gr.DataFrame() + data = pd.DataFrame.from_records(scores_long) + data["task_type"] = data["task_name"].map( + lambda task_name: get_task(task_name).metadata.type + ) + mean_per_type = ( + data.groupby(["model_name", "model_revision", "task_type"])[["score"]] + .agg(np.nanmean) + .reset_index() + ) + typed_mean = ( + mean_per_type.groupby(["model_name", "model_revision"])[["score"]] + .agg(np.nanmean) + .rename(columns={"score": "mean_by_task_type"}) + ) + mean_per_type = mean_per_type.pivot( + index=["model_name", "model_revision"], columns="task_type", values="score" + ) + per_task = data.pivot( + index=["model_name", "model_revision"], columns="task_name", values="score" + ) + to_remove = per_task.isna().any(axis="columns") + overall_mean = ( + data.groupby(["model_name", "model_revision"])[["score"]] + .agg(np.nanmean) + .rename(columns={"score": "mean"}) + ) + per_task = per_task[~to_remove] + mean_per_type = mean_per_type[~to_remove] + overall_mean = overall_mean[~to_remove] + mean_rank = per_task.rank(ascending=False, numeric_only=True).mean( + axis=1, skipna=True + ) + joint_table = overall_mean.join([typed_mean, mean_per_type]) + joint_table.insert(0, "mean_rank", mean_rank) + joint_table = joint_table.reset_index() + joint_table = joint_table.sort_values("mean", ascending=False) + joint_table["model_name"] = joint_table["model_name"].map( + lambda name: name.split("/")[-1] + ) + joint_table = joint_table.rename( + columns={ + "model_name": "Model", + "mean_by_task_type": "Mean by Task Type", + "mean": "Mean", + "mean_rank": "Mean Rank", + } + ) + joint_table = joint_table.drop(columns=["model_revision"]) + joint_table.insert( + 0, "Rank", joint_table["Mean"].rank(ascending=False).map(int).map(str) + ) + per_task = per_task.rename( + columns={ + "model_name": "Model", + } + ) + per_task = per_task.reset_index().drop(columns=["model_revision"]) + numerics = joint_table.select_dtypes("number").columns + to_format = ["Mean", "Mean by Task Type", *mean_per_type.columns] + joint_table[to_format] = joint_table[to_format].map(format_scores) + joint_table = joint_table.style.highlight_max( + subset=to_format, + props="font-weight: bold", + ).format("{:.2f}", subset=numerics) + joint_table = joint_table.highlight_min( + subset=["Mean Rank"], props="font-weight: bold" + ) + numerics = per_task.select_dtypes("number").columns + per_task[numerics] = per_task[numerics].map(format_scores) + per_task = per_task.style.highlight_max( + subset=numerics, props="font-weight: bold" + ).format("{:.2f}", subset=numerics) + return joint_table, per_task diff --git a/mteb/load_results/__init__.py b/mteb/load_results/__init__.py index 3b08f6eb4d..aee4201d39 100644 --- a/mteb/load_results/__init__.py +++ b/mteb/load_results/__init__.py @@ -1,6 +1,7 @@ from __future__ import annotations +from .benchmark_results import BenchmarkResults, ModelResult from .load_results import load_results -from .mteb_results import MTEBResults +from .task_results import TaskResult -__all__ = ["load_results", "MTEBResults"] +__all__ = ["load_results", "TaskResult", "ModelResult", "BenchmarkResults"] diff --git a/mteb/load_results/benchmark_results.py b/mteb/load_results/benchmark_results.py new file mode 100644 index 0000000000..e3060dfbbb --- /dev/null +++ b/mteb/load_results/benchmark_results.py @@ -0,0 +1,316 @@ +from __future__ import annotations + +import json +from collections import defaultdict +from collections.abc import Iterable +from pathlib import Path +from typing import Any, Callable, Literal + +import numpy as np +from pydantic import BaseModel, ConfigDict + +from mteb.abstasks.AbsTask import AbsTask, ScoresDict +from mteb.abstasks.TaskMetadata import ( + ISO_LANGUAGE_SCRIPT, + TASK_DOMAIN, + TASK_TYPE, +) +from mteb.languages import ISO_LANGUAGE +from mteb.load_results.task_results import TaskResult +from mteb.models.overview import get_model_metas + +Split = str +Score = Any + + +class ModelResult(BaseModel): + model_name: str + model_revision: str | None + task_results: list[TaskResult] + model_config = ConfigDict( + protected_namespaces=(), + ) + + def __repr__(self) -> str: + n_entries = len(self.task_results) + return f"ModelResult(model_name={self.model_name}, model_revision={self.model_revision}, task_results=[...](#{n_entries}))" + + def filter_tasks( + self, + task_names: list[str] | None = None, + languages: list[str] | None = None, + domains: list[TASK_DOMAIN] | None = None, + task_types: list[TASK_TYPE] | None = None, + ) -> ModelResult: + new_task_results = [] + for task_result in self.task_results: + if (task_names is not None) and (task_result.task_name not in task_names): + continue + if languages is not None: + task_languages = task_result.languages + if not any(lang in task_languages for lang in languages): + continue + if domains is not None: + task_domains = task_result.domains + if not any(domain in task_domains for domain in domains): + continue + if (task_types is not None) and (task_result.task_type not in task_types): + continue + new_task_results.append(task_result) + return type(self)( + model_name=self.model_name, + model_revision=self.model_revision, + task_results=new_task_results, + ) + + def select_tasks(self, tasks: list[AbsTask]) -> ModelResult: + task_name_to_task = {task.metadata.name: task for task in tasks} + new_task_results = [ + task_res.validate_and_filter_scores(task_name_to_task[task_res.task_name]) + for task_res in self.task_results + if task_res.task_name in task_name_to_task + ] + return type(self)( + model_name=self.model_name, + model_revision=self.model_revision, + task_results=new_task_results, + ) + + def get_scores( + self, + splits: list[Split] | None = None, + languages: list[ISO_LANGUAGE | ISO_LANGUAGE_SCRIPT] | None = None, + scripts: list[ISO_LANGUAGE_SCRIPT] | None = None, + getter: Callable[[ScoresDict], Score] = lambda scores: scores["main_score"], + aggregation: Callable[[list[Score]], Any] = np.mean, + format: Literal["wide", "long"] = "wide", + ) -> dict | list: + if format == "wide": + scores = { + res.task_name: res.get_score( + splits=splits, + languages=languages, + scripts=scripts, + getter=getter, + aggregation=aggregation, + ) + for res in self.task_results + } + return scores + if format == "long": + entries = [] + for task_res in self.task_results: + entry = dict( # noqa + model_name=self.model_name, + model_revision=self.model_revision, + task_name=task_res.task_name, + score=task_res.get_score( + splits=splits, + languages=languages, + getter=getter, + aggregation=aggregation, + ), + mteb_version=task_res.mteb_version, + dataset_revision=task_res.dataset_revision, + evaluation_time=task_res.evaluation_time, + kg_co2_emissions=task_res.kg_co2_emissions, + ) + entries.append(entry) + return entries + + def __iter__(self): + return iter(self.task_results) + + def __getitem__(self, index) -> TaskResult: + return self.task_results[index] + + @property + def languages(self) -> list[str]: + langs = [] + for task_res in self.task_results: + langs.extend(task_res.languages) + return list(set(langs)) + + @property + def domains(self) -> list[str]: + ds = [] + for task_res in self.task_results: + ds.extend(task_res.domains) + return list(set(ds)) + + @property + def task_types(self) -> list[str]: + return list({task_res.task_type for task_res in self.task_results}) + + @property + def task_names(self) -> list[str]: + return [task_res.task_name for task_res in self.task_results] + + +class BenchmarkResults(BaseModel): + model_results: list[ModelResult] + model_config = ConfigDict( + protected_namespaces=(), + ) + + def __repr__(self) -> str: + n_models = len(self.model_results) + return f"BenchmarkResults(model_results=[...](#{n_models}))" + + def filter_tasks( + self, + task_names: list[str] | None = None, + languages: list[str] | None = None, + domains: list[TASK_DOMAIN] | None = None, + task_types: list[TASK_TYPE] | None = None, + ) -> BenchmarkResults: + model_results = [ + res.filter_tasks( + task_names=task_names, + languages=languages, + domains=domains, + task_types=task_types, + ) + for res in self.model_results + ] + return type(self)( + model_results=[res for res in model_results if res.task_results] + ) + + def select_tasks(self, tasks: list[AbsTask]) -> BenchmarkResults: + new_model_results = [ + model_res.select_tasks(tasks) for model_res in self.model_results + ] + return type(self)(model_results=new_model_results) + + def filter_models( + self, + model_names: Iterable[str] | None = None, + languages: Iterable[str] | None = None, + open_source: bool | None = None, + frameworks: Iterable[str] | None = None, + n_parameters_range: tuple[int | None, int | None] = (None, None), + ) -> BenchmarkResults: + model_metas = get_model_metas( + model_names, languages, open_source, frameworks, n_parameters_range + ) + model_revision_pairs = {(meta.name, meta.revision) for meta in model_metas} + new_model_results = [] + for model_res in self: + if (model_res.model_name, model_res.model_revision) in model_revision_pairs: + new_model_results.append(model_res) + return type(self)(model_results=new_model_results) + + def get_scores( + self, + splits: list[Split] | None = None, + languages: list[ISO_LANGUAGE | ISO_LANGUAGE_SCRIPT] | None = None, + scripts: list[ISO_LANGUAGE_SCRIPT] | None = None, + getter: Callable[[ScoresDict], Score] = lambda scores: scores["main_score"], + aggregation: Callable[[list[Score]], Any] = np.mean, + format: Literal["wide", "long"] = "wide", + ) -> list[dict]: + entries = [] + if format == "wide": + for model_res in self: + model_scores = model_res.get_scores( + splits=splits, + languages=languages, + scripts=scripts, + getter=getter, + aggregation=aggregation, + format="wide", + ) + entries.append( + { + "model": model_res.model_name, + "revision": model_res.model_revision, + **model_scores, + } + ) + if format == "long": + for model_res in self: + entries.extend( + model_res.get_scores( + splits=splits, + languages=languages, + scripts=scripts, + getter=getter, + aggregation=aggregation, + format="long", + ) + ) + return entries + + def __iter__(self): + return iter(self.model_results) + + def __getitem__(self, index) -> ModelResult: + return self.model_results[index] + + def to_legacy_dict(self) -> dict[str, dict[str, list[TaskResult]]]: + res = defaultdict(dict) + for model_res in self: + res[model_res.model_name][model_res.model_revision] = model_res.task_results + return res + + @classmethod + def from_legacy_dict(cls, legacy: dict[str, dict[str, list[TaskResult]]]): + model_results = [] + for model_name, revisions in legacy.items(): + for model_revision, results in revisions.items(): + model_results.append( + ModelResult( + model_name=model_name, + model_revision=model_revision, + task_results=results, + ) + ) + return cls(model_results=model_results) + + def to_dict(self) -> dict: + return self.model_dump() + + @classmethod + def from_dict(cls, data: dict) -> TaskResult: + return cls.model_validate(data) + + def to_disk(self, path: Path | str) -> None: + path = Path(path) + with path.open("w") as out_file: + out_file.write(self.model_dump_json(indent=2)) + + @classmethod + def from_disk(cls, path: Path | str) -> BenchmarkResults: + path = Path(path) + with path.open() as in_file: + data = json.loads(in_file.read()) + return cls.from_dict(data) + + @property + def languages(self) -> list[str]: + langs = [] + for model_res in self.model_results: + langs.extend(model_res.languages) + return list(set(langs)) + + @property + def domains(self) -> list[str]: + ds = [] + for model_res in self.model_results: + ds.extend(model_res.domains) + return list(set(ds)) + + @property + def task_types(self) -> list[str]: + ts = [] + for model_res in self.model_results: + ts.extend(model_res.task_types) + return list(set(ts)) + + @property + def task_names(self) -> list[str]: + names = [] + for model_res in self.model_results: + names.extend(model_res.task_names) + return list(set(names)) diff --git a/mteb/load_results/load_results.py b/mteb/load_results/load_results.py index 3f9530ab21..8601420427 100644 --- a/mteb/load_results/load_results.py +++ b/mteb/load_results/load_results.py @@ -4,20 +4,18 @@ import logging import os import subprocess -from collections import defaultdict +from collections.abc import Sequence from pathlib import Path -from typing import Dict, List, Sequence from mteb.abstasks.AbsTask import AbsTask -from mteb.load_results.mteb_results import MTEBResults +from mteb.load_results.benchmark_results import BenchmarkResults, ModelResult +from mteb.load_results.task_results import TaskResult from mteb.model_meta import ModelMeta logger = logging.getLogger(__name__) MODEL_NAME = str REVISION = str -RESULTS = Dict[MODEL_NAME, Dict[REVISION, List[MTEBResults]]] - def download_of_results( results_repo: str, cache_directory: Path | None = None, download_latest: bool = True @@ -44,7 +42,7 @@ def download_of_results( cache_directory.mkdir(parents=True) # if "results" folder already exists update it - results_directory = cache_directory / "results" + results_directory = cache_directory / os.path.basename(results_repo) if results_directory.exists(): if download_latest: logger.info( @@ -92,7 +90,7 @@ def load_results( tasks: Sequence[AbsTask] | Sequence[str] | None = None, validate_and_filter: bool = True, require_model_meta: bool = True, -) -> RESULTS: +) -> BenchmarkResults: """Loads the results from the latest version of the results repository. The results are cached locally in the MTEB_CACHE directory. This directory can be set using the MTEB_CACHE environment variable or defaults to "~/.cache/mteb". @@ -105,31 +103,6 @@ def load_results( extract the model name and revision from the path. validate_and_filter: If True it will validate that the results object for the task contains the correct splits and filter out splits from the results object that are not default in the task metadata. Defaults to True. - - Returns: - A dictionary where the keys are the model names and the values are dictionaries where the keys are the revisions and the values are lists of MTEBResults objects. - - Example: - >>> results = load_results() - >>> results - {'mixedbread-ai/mxbai-embed-large-v1': - {'990580e27d329c7408b3741ecff85876e128e203': [ - MTEBResults(task_name=TwentyNewsgroupsClustering.v2, scores=...), - MTEBResults(task_name=MedrxivClusteringP2P, scores=...), - MTEBResults(task_name=StackExchangeClustering, scores=...), - MTEBResults(task_name=BiorxivClusteringP2P.v2, scores=...), - MTEBResults(task_name=MedrxivClusteringS2S.v2, scores=...), - MTEBResults(task_name=MedrxivClusteringS2S, scores=...), - ... - ]}, - 'intfloat/multilingual-e5-small': - {'e4ce9877abf3edfe10b0d82785e83bdcb973e22e': [ - MTEBResults(task_name=IndicGenBenchFloresBitextMining, scores=...), - MTEBResults(task_name=PpcPC, scores=...), - MTEBResults(task_name=TwentyNewsgroupsClustering.v2, scores=...), - ... - ]}, - ... """ repo_directory = download_of_results(results_repo, download_latest=download_latest) model_paths = [p for p in (repo_directory / "results").glob("*") if p.is_dir()] @@ -144,16 +117,15 @@ def load_results( else: models_to_keep = None + task_names = {} if tasks is not None: - task_names = {} for task in tasks: if isinstance(task, AbsTask): task_names[task.metadata.name] = task else: task_names[task] = None - results = defaultdict(dict) - + model_results = [] for model_path in model_paths: model_revisions = model_path.glob("*") @@ -174,7 +146,7 @@ def load_results( task_json_files = [ f for f in revision_path.glob("*.json") if "model_meta.json" != f.name ] - _results = [MTEBResults.from_disk(f) for f in task_json_files] + _results = [TaskResult.from_disk(f) for f in task_json_files] # filter out tasks that are not in the tasks list if tasks is not None: @@ -184,14 +156,23 @@ def load_results( filtered_results = [] for r in _results: try: - r.validate_and_filter_scores(task_names[r.task_name]) + if task_names: + task = task_names[r.task_name] + else: + task = None + r = r.validate_and_filter_scores(task=task) filtered_results.append(r) except Exception as e: logger.warning( f"Validation failed for {r.task_name} in {model_name} {revision}: {e}" ) _results = filtered_results + model_results.append( + ModelResult( + model_name=model_name, + model_revision=revision, + task_results=_results, + ) + ) - results[model_name][revision] = _results - - return dict(results) + return BenchmarkResults(model_results=model_results) diff --git a/mteb/load_results/mteb_results.py b/mteb/load_results/task_results.py similarity index 89% rename from mteb/load_results/mteb_results.py rename to mteb/load_results/task_results.py index 49cf3a710a..b6da0ba304 100644 --- a/mteb/load_results/mteb_results.py +++ b/mteb/load_results/task_results.py @@ -4,6 +4,7 @@ import logging from argparse import Namespace from collections import defaultdict +from functools import cached_property from importlib.metadata import version from pathlib import Path from typing import Any, Callable @@ -13,10 +14,7 @@ from pydantic import BaseModel, field_validator from mteb.abstasks.AbsTask import AbsTask, ScoresDict -from mteb.abstasks.TaskMetadata import ( - ISO_LANGUAGE_SCRIPT, - HFSubset, -) +from mteb.abstasks.TaskMetadata import ISO_LANGUAGE_SCRIPT, HFSubset from mteb.languages import ISO_LANGUAGE, LanguageScripts Split = str @@ -116,7 +114,7 @@ class ScalaSvClassificationDummy: } -class MTEBResults(BaseModel): +class TaskResult(BaseModel): """A class to represent the MTEB result. Attributes: @@ -142,7 +140,7 @@ class MTEBResults(BaseModel): ... }, ... } >>> sample_task = ... # some MTEB task - >>> mteb_results = MTEBResults.from_task_results(sample_task, scores) + >>> mteb_results = TaskResult.from_task_results(sample_task, scores) >>> mteb_results.get_score() # get the main score for all languages 0.55 >>> mteb_results.get_score(languages=["fra"]) # get the main score for French @@ -170,7 +168,7 @@ def from_task_results( scores: dict[Split, dict[HFSubset, ScoresDict]], evaluation_time: float, kg_co2_emissions: float | None = None, - ) -> MTEBResults: + ) -> TaskResult: task_meta = task.metadata subset2langscripts = task_meta.hf_subsets_to_langscripts flat_scores = defaultdict(list) @@ -184,7 +182,7 @@ def from_task_results( } flat_scores[split].append(_scores) - return MTEBResults( + return TaskResult( dataset_revision=task.metadata.dataset["revision"], task_name=task.metadata.name, mteb_version=version("mteb"), @@ -219,11 +217,36 @@ def _validate_scores_dict(scores: ScoresDict) -> None: except Exception as e: raise ValueError(f"Scores are not json serializable: {e}") + @property + def languages(self) -> list[str]: + langs = [] + for split, split_res in self.scores.items(): + for entry in split_res: + langs.extend([lang.split("-")[0] for lang in entry["languages"]]) + return list(set(langs)) + + @cached_property + def task(self) -> AbsTask: + from mteb.overview import get_task + + return get_task(self.task_name) + + @property + def domains(self) -> list[str]: + doms = self.task.metadata.domains + if doms is None: + doms = [] + return doms + + @property + def task_type(self) -> str: + return self.task.metadata.type + def to_dict(self) -> dict: return self.model_dump() @classmethod - def from_dict(cls, data: dict) -> MTEBResults: + def from_dict(cls, data: dict) -> TaskResult: return cls.model_validate(data) def _round_scores(self, scores: dict[Split, list[ScoresDict]], n: int) -> None: @@ -249,8 +272,8 @@ def to_disk(self, path: Path) -> None: json.dump(json_obj, f, indent=2) @classmethod - def from_disk(cls, path: Path, load_historic_data: bool = True) -> MTEBResults: # type: ignore - """Load MTEBResults from disk. + def from_disk(cls, path: Path, load_historic_data: bool = True) -> TaskResult: # type: ignore + """Load TaskResult from disk. Args: path: The path to the file to load. @@ -264,7 +287,7 @@ def from_disk(cls, path: Path, load_historic_data: bool = True) -> MTEBResults: return cls.model_validate(data) except Exception as e: raise ValueError( - f"Error loading MTEBResults from disk. You can try to load historic data by setting `load_historic_data=True`. Error: {e}" + f"Error loading TaskResult from disk. You can try to load historic data by setting `load_historic_data=True`. Error: {e}" ) pre_1_11_load = ( @@ -280,7 +303,7 @@ def from_disk(cls, path: Path, load_historic_data: bool = True) -> MTEBResults: if not pre_1_11_load: raise e logger.debug( - f"Could not load MTEBResults from disk, got error: {e}. Attempting to load from disk using format from before v1.11.0" + f"Could not load TaskResult from disk, got error: {e}. Attempting to load from disk using format from before v1.11.0" ) obj = cls._convert_from_before_v1_11_0(data) @@ -294,7 +317,7 @@ def from_disk(cls, path: Path, load_historic_data: bool = True) -> MTEBResults: return obj @classmethod - def _fix_pair_classification_scores(cls, obj: MTEBResults) -> None: + def _fix_pair_classification_scores(cls, obj: TaskResult) -> None: from mteb import get_task task_name = obj.task_name @@ -314,7 +337,7 @@ def _fix_pair_classification_scores(cls, obj: MTEBResults) -> None: hf_subset_scores.pop(key) @classmethod - def _convert_from_before_v1_11_0(cls, data: dict) -> MTEBResults: + def _convert_from_before_v1_11_0(cls, data: dict) -> TaskResult: from mteb.overview import TASKS_REGISTRY # in case the task name is not found in the registry, try to find a lower case version @@ -394,7 +417,7 @@ def _convert_from_before_v1_11_0(cls, data: dict) -> MTEBResults: if "test" in scores and "fr" in scores["test"]: scores["test"]["fra-fra"] = scores["test"].pop("fr") - result: MTEBResults = MTEBResults.from_task_results( + result: TaskResult = TaskResult.from_task_results( task, # type: ignore scores, evaluation_time, @@ -444,11 +467,12 @@ def get_score( return aggregation(values) def __repr__(self) -> str: - return f"MTEBResults(task_name={self.task_name}, scores=...)" + return f"TaskResult(task_name={self.task_name}, scores=...)" - def validate_and_filter_scores(self, task: AbsTask | None = None) -> None: + def validate_and_filter_scores(self, task: AbsTask | None = None) -> AbsTask: """This ensures that the scores are correct for the given task, by removing any splits besides those specified in the task metadata. Additionally it also ensure that all of the splits required as well as the languages are present in the scores. + Returns new TaskResult object. Args: task: The task to validate the scores against. E.g. if the task supplied is limited to certain splits and languages, @@ -459,30 +483,32 @@ def validate_and_filter_scores(self, task: AbsTask | None = None) -> None: if task is None: task = get_task(self.task_name) splits = task.metadata.eval_splits - hf_subsets = set(task.metadata.hf_subsets_to_langscripts) - + if task.is_multilingual: + hf_subsets = getattr( + task, "hf_subsets", task.metadata.hf_subsets_to_langscripts.keys() + ) + hf_subsets = set(hf_subsets) + else: + hf_subsets = {"default"} new_scores = {} seen_splits = set() for split in self.scores: if split not in splits: continue new_scores[split] = [] - seen_subsets = set() for _scores in self.scores[split]: if _scores["hf_subset"] not in hf_subsets: continue new_scores[split].append(_scores) seen_subsets.add(_scores["hf_subset"]) - if seen_subsets != hf_subsets: raise ValueError( f"Missing subsets {hf_subsets - seen_subsets} for split {split}" ) - seen_splits.add(split) - if seen_splits != set(splits): raise ValueError(f"Missing splits {set(splits) - seen_splits}") - - self.scores = new_scores + new_res = {**self.to_dict(), "scores": new_scores} + new_res = TaskResult.from_dict(new_res) + return new_res diff --git a/mteb/model_meta.py b/mteb/model_meta.py index fbb93c5f8a..7acb806b81 100644 --- a/mteb/model_meta.py +++ b/mteb/model_meta.py @@ -1,35 +1,44 @@ from __future__ import annotations -from datetime import date +import logging from functools import partial -from typing import Any, Callable, Literal +from typing import TYPE_CHECKING, Any, Callable, Literal -from pydantic import BaseModel, BeforeValidator, TypeAdapter -from sentence_transformers import SentenceTransformer -from typing_extensions import Annotated +from pydantic import BaseModel -from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from mteb.abstasks.TaskMetadata import STR_DATE, STR_URL +from mteb.encoder_interface import Encoder from .languages import ISO_LANGUAGE_SCRIPT -Frameworks = Literal["Sentence Transformers", "PyTorch"] +if TYPE_CHECKING: + from .models.sentence_transformer_wrapper import SentenceTransformerWrapper -pastdate_adapter = TypeAdapter(date) -STR_DATE = Annotated[ - str, BeforeValidator(lambda value: str(pastdate_adapter.validate_python(value))) -] # Allows the type to be a string, but ensures that the string is a valid date +logger = logging.getLogger(__name__) + + +FRAMEWORKS = Literal[ + "Sentence Transformers", + "PyTorch", + "GritLM", + "LLM2Vec", + "TensorFlow", + "API", + "Tevatron", +] +DISTANCE_METRICS = Literal["cosine"] def sentence_transformers_loader( - model_name: str, revision: str | None, **kwargs -) -> SentenceTransformer: - return SentenceTransformer( - model_name_or_path=model_name, revision=revision, **kwargs - ) + model_name: str, revision: str | None = None, **kwargs +) -> SentenceTransformerWrapper: + from .models.sentence_transformer_wrapper import SentenceTransformerWrapper + + return SentenceTransformerWrapper(model=model_name, revision=revision, **kwargs) def get_loader_name( - loader: Callable[..., Encoder | EncoderWithQueryCorpusEncode] | None, + loader: Callable[..., Encoder] | None, ) -> str | None: if loader is None: return None @@ -52,27 +61,39 @@ class ModelMeta(BaseModel): embed_dim: The dimension of the embeddings produced by the model. Currently all models are assumed to produce fixed-size embeddings. revision: The revision number of the model. If None it is assumed that the metadata (including the loader) is valid for all revisions of the model. release_date: The date the model's revision was released. - license: The license under which the model is released. Required if open_source is True. - open_source: Whether the model is open source or proprietary. - distance_metric: The distance metric used by the model. + license: The license under which the model is released. Required if open_weights is True. + open_weights: Whether the model is open source or proprietary. + public_training_data: Whether the training data used to train the model is publicly available. + public_training_code: Whether the code used to train the model is publicly available. + similarity_fn_name: The distance metric used by the model. framework: The framework the model is implemented in, can be a list of frameworks e.g. `["Sentence Transformers", "PyTorch"]`. + reference: A URL to the model's page on huggingface or another source. languages: The languages the model is intended for specified as a 3 letter language code followed by a script code e.g. "eng-Latn" for English in the Latin script. + use_instuctions: Whether the model uses instructions E.g. for prompt-based models. This also include models that require a specific format for + input such as "query: {document}" or "passage: {document}". + zero_shot_benchmarks: A list of benchmarks on which the model has been evaluated in a zero-shot setting. By default we assume that all models + are evaluated non-zero-shot unless specified otherwise. """ name: str | None revision: str | None release_date: STR_DATE | None languages: list[ISO_LANGUAGE_SCRIPT] | None - loader: Callable[..., Encoder | EncoderWithQueryCorpusEncode] | None = None + loader: Callable[..., Encoder] | None = None n_parameters: int | None = None memory_usage: float | None = None max_tokens: int | None = None embed_dim: int | None = None license: str | None = None - open_source: bool | None = None - similarity_fn_name: str | None = None - framework: list[Frameworks] = [] + open_weights: bool | None = None + public_training_data: bool | None = None + public_training_code: bool | None = None + framework: list[FRAMEWORKS] = [] + reference: STR_URL | None = None + similarity_fn_name: DISTANCE_METRICS | None = None + use_instuctions: bool | None = None + zero_shot_benchmarks: list[str] | None = None def to_dict(self): dict_repr = self.model_dump() @@ -80,19 +101,21 @@ def to_dict(self): dict_repr["loader"] = get_loader_name(loader) return dict_repr - def load_model(self, **kwargs: Any) -> Encoder | EncoderWithQueryCorpusEncode: + def load_model(self, **kwargs: Any) -> Encoder: if self.loader is None: + logger.warning( + f"Loader not specified for model {self.name}, loading using sentence transformers." + ) loader = partial( sentence_transformers_loader, model_name=self.name, revision=self.revision, - trust_remote_code=True, **kwargs, ) else: loader = self.loader - model: Encoder | EncoderWithQueryCorpusEncode = loader(**kwargs) # type: ignore + model: Encoder = loader(**kwargs) # type: ignore return model def model_name_as_path(self) -> str: diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index 25c7eb8f44..3804aebcd8 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -1,171 +1,24 @@ from __future__ import annotations import logging -from typing import Any -from sentence_transformers import SentenceTransformer - -from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode -from mteb.model_meta import ModelMeta -from mteb.models import ( - align_models, - bge_models, - blip2_models, - blip_models, - bm25, - clip_models, - cohere_models, - datacomp_clip, - dino_models, - e5_instruct, - e5_models, - e5_v, - google_models, - gritlm_models, - gte_models, - jina_clip, - llm2vec_models, - moco_models, - mxbai_models, - nomic_models, - nomic_models_vision, - openai_models, - ru_sentence_models, - salesforce_models, - sentence_transformers_models, - vista_models, - vlm2vec_models, - voyage_models, +from mteb.models.overview import ( + MODEL_REGISTRY, + ModelMeta, + get_model, + get_model_meta, + get_model_metas, + model_meta_from_sentence_transformers, ) logger = logging.getLogger(__name__) -def get_model( - model_name: str, revision: str | None = None, **kwargs: Any -) -> Encoder | EncoderWithQueryCorpusEncode: - """A function to fetch a model object by name. - - Args: - model_name: Name of the model to fetch - revision: Revision of the model to fetch - **kwargs: Additional keyword arguments to pass to the model loader - - Returns: - A model object - """ - meta = get_model_meta(model_name, revision) - model = meta.load_model(**kwargs) - - # If revision not available in the modelmeta, try to extract it from sentence-transformers - if meta.revision is None and isinstance(model, SentenceTransformer): - _meta = model_meta_from_sentence_transformers(model) - meta.revision = _meta.revision if _meta.revision else meta.revision - - model.mteb_model_meta = meta # type: ignore - return model - - -def get_model_meta(model_name: str, revision: str | None = None) -> ModelMeta: - """A function to fetch a model metadata object by name. - - Args: - model_name: Name of the model to fetch - revision: Revision of the model to fetch - - Returns: - A model metadata object - """ - if model_name in models: - if revision and (not models[model_name].revision == revision): - raise ValueError( - f"Model revision {revision} not found for model {model_name}. Expected {models[model_name].revision}." - ) - return models[model_name] - else: # assume it is a sentence-transformers model - logger.info( - "Model not found in model registry, assuming it is a sentence-transformers model." - ) - logger.info( - f"Attempting to extract metadata by loading the model ({model_name}) using sentence-transformers." - ) - model = SentenceTransformer( - model_name, revision=revision, trust_remote_code=True - ) - meta = model_meta_from_sentence_transformers(model) - - meta.revision = revision - meta.name = model_name - return meta - - -def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMeta: - try: - name = ( - model.model_card_data.model_name - if model.model_card_data.model_name - else model.model_card_data.base_model - ) - languages = ( - [model.model_card_data.language] - if isinstance(model.model_card_data.language, str) - else model.model_card_data.language - ) - meta = ModelMeta( - name=name, - revision=model.model_card_data.base_model_revision, - release_date=None, - languages=languages, - framework=["Sentence Transformers"], - similarity_fn_name=model.similarity_fn_name, - ) - except AttributeError as e: - logger.warning( - f"Failed to extract metadata from model: {e}. Upgrading to sentence-transformers v3.0.0 or above is recommended." - ) - meta = ModelMeta( - name=None, - revision=None, - languages=None, - release_date=None, - ) - return meta - - -model_modules = [ - align_models, - bge_models, - blip_models, - blip2_models, - bm25, - cohere_models, - datacomp_clip, - dino_models, - e5_instruct, - e5_models, - e5_v, - google_models, - gritlm_models, - gte_models, - jina_clip, - llm2vec_models, - moco_models, - mxbai_models, - nomic_models, - nomic_models_vision, - cohere_models, - clip_models, - openai_models, - ru_sentence_models, - salesforce_models, - sentence_transformers_models, - vista_models, - voyage_models, - vlm2vec_models, +__all__ = [ + "MODEL_REGISTRY", + "ModelMeta", + "get_model", + "get_model_meta", + "get_model_metas", + "model_meta_from_sentence_transformers", ] -models = {} - -for module in model_modules: - for mdl in vars(module).values(): - if isinstance(mdl, ModelMeta): - models[mdl.name] = mdl diff --git a/mteb/models/arctic_models.py b/mteb/models/arctic_models.py index 043d54a362..3c350a0ad7 100644 --- a/mteb/models/arctic_models.py +++ b/mteb/models/arctic_models.py @@ -1,66 +1,20 @@ from __future__ import annotations -from functools import partial -from typing import Any - -import torch -from sentence_transformers import SentenceTransformer - from mteb.model_meta import ModelMeta -from mteb.models.text_formatting_utils import corpus_to_texts - - -class ArcticWrapper: - """following the hf model card documentation.""" - - def __init__(self, model_name: str, **kwargs: Any): - self.model_name = model_name - self.mdl = SentenceTransformer(model_name) - - def to(self, device: torch.device) -> None: - self.mdl.to(device) - - def encode( # type: ignore - self, - sentences: list[str], - *, - batch_size: int = 32, - **kwargs: Any, - ): - return self.mdl.encode(sentences, batch_size=batch_size, **kwargs) - - def encode_queries(self, queries: list[str], batch_size: int = 32, **kwargs: Any): - if "prompt_name" in kwargs: - kwargs.pop("prompt_name") - sentences = [ - "Represent this sentence for searching relevant passages: " + sentence - for sentence in queries - ] - emb = self.mdl.encode( - sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs - ) - return emb - - def encode_corpus( - self, - corpus: list[dict[str, str]] | dict[str, list[str]], - batch_size: int = 32, - **kwargs: Any, - ): - if "prompt_name" in kwargs: - kwargs.pop("prompt_name") - sentences = corpus_to_texts(corpus) - emb = self.mdl.encode( - sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs - ) - return emb - arctic_m_v1_5 = ModelMeta( - loader=partial(ArcticWrapper, model_name="Snowflake/snowflake-arctic-embed-m-v1.5"), # type: ignore name="Snowflake/snowflake-arctic-embed-m-v1.5", - languages=["eng_Latn"], - open_source=True, revision="97eab2e17fcb7ccb8bb94d6e547898fa1a6a0f47", release_date="2024-07-08", # initial commit of hf model. + languages=["eng_Latn"], + open_weights=True, + framework=["Sentence Transformers", "PyTorch"], + n_parameters=109_000_000, + memory_usage=None, + max_tokens=512, + embed_dim=256, + license="apache-2.0", + reference="https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5", + similarity_fn_name="cosine_similarity", + use_instuctions=False, ) diff --git a/mteb/models/bge_models.py b/mteb/models/bge_models.py index 0635967156..61200d72e0 100644 --- a/mteb/models/bge_models.py +++ b/mteb/models/bge_models.py @@ -1,84 +1,76 @@ from __future__ import annotations from functools import partial -from typing import Any -import torch -from sentence_transformers import SentenceTransformer +from mteb.model_meta import ModelMeta, sentence_transformers_loader -from mteb.model_meta import ModelMeta -from mteb.models.text_formatting_utils import corpus_to_texts - - -class BGEWrapper: - """following the hf model card documentation.""" - - def __init__(self, model_name: str, **kwargs: Any): - self.model_name = model_name - self.mdl = SentenceTransformer(model_name) - - def to(self, device: torch.device) -> None: - self.mdl.to(device) - - def encode( # type: ignore - self, - sentences: list[str], - *, - batch_size: int = 32, - **kwargs: Any, - ): - if "request_qid" in kwargs: - kwargs.pop("request_qid") - - return self.mdl.encode(sentences, batch_size=batch_size, **kwargs) - - def encode_queries(self, queries: list[str], batch_size: int = 32, **kwargs: Any): - if "prompt_name" in kwargs: - kwargs.pop("prompt_name") - if "request_qid" in kwargs: - kwargs.pop("request_qid") - - sentences = [ - "Represent this sentence for searching relevant passages: " + sentence - for sentence in queries - ] - emb = self.mdl.encode( - sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs - ) - return emb - - def encode_corpus( - self, - corpus: list[dict[str, str]] | dict[str, list[str]], - batch_size: int = 32, - **kwargs: Any, - ): - if "prompt_name" in kwargs: - kwargs.pop("prompt_name") - if "request_qid" in kwargs: - kwargs.pop("request_qid") - - sentences = corpus_to_texts(corpus) - emb = self.mdl.encode( - sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs - ) - return emb +model_prompts = {"query": "Represent this sentence for searching relevant passages: "} +bge_small_en_v1_5 = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="BAAI/bge-small-en-v1.5", + revision="5c38ec7c405ec4b44b94cc5a9bb96e735b38267a", + model_prompts=model_prompts, + ), + name="BAAI/bge-small-en-v1.5", + languages=["eng_Latn"], + open_weights=True, + revision="5c38ec7c405ec4b44b94cc5a9bb96e735b38267a", + release_date="2023-09-12", # initial commit of hf model. + n_parameters=24_000_000, + memory_usage=None, + embed_dim=512, + license="mit", + max_tokens=512, + reference="https://huggingface.co/BAAI/bge-small-en-v1.5", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, +) bge_base_en_v1_5 = ModelMeta( - loader=partial(BGEWrapper, model_name="BAAI/bge-base-en-v1.5"), # type: ignore + loader=partial( + sentence_transformers_loader, + model_name="BAAI/bge-base-en-v1.5", + revision="a5beb1e3e68b9ab74eb54cfd186867f64f240e1a", + model_prompts=model_prompts, + ), name="BAAI/bge-base-en-v1.5", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="a5beb1e3e68b9ab74eb54cfd186867f64f240e1a", release_date="2023-09-11", # initial commit of hf model. + n_parameters=438_000_000, + memory_usage=None, + embed_dim=768, + license="mit", + max_tokens=512, + reference="https://huggingface.co/BAAI/bge-base-en-v1.5", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) bge_large_en_v1_5 = ModelMeta( - loader=partial(BGEWrapper, model_name="BAAI/bge-large-en-v1.5"), # type: ignore + loader=partial( + sentence_transformers_loader, + model_name="BAAI/bge-large-en-v1.5", + revision="d4aa6901d3a41ba39fb536a557fa166f842b0e09", + model_prompts=model_prompts, + ), name="BAAI/bge-large-en-v1.5", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="d4aa6901d3a41ba39fb536a557fa166f842b0e09", release_date="2023-09-12", # initial commit of hf model. + n_parameters=1_340_000_000, + memory_usage=None, + embed_dim=1024, + license="mit", + max_tokens=512, + reference="https://huggingface.co/BAAI/bge-large-en-v1.5", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) diff --git a/mteb/models/bm25.py b/mteb/models/bm25.py index 4ef5a55ca6..180929329b 100644 --- a/mteb/models/bm25.py +++ b/mteb/models/bm25.py @@ -2,11 +2,11 @@ import logging from functools import partial -from typing import Any from mteb.evaluation.evaluators.RetrievalEvaluator import DRESModel from mteb.model_meta import ModelMeta -from mteb.models.text_formatting_utils import corpus_to_texts + +from .wrapper import Wrapper logger = logging.getLogger(__name__) @@ -20,7 +20,7 @@ def bm25_loader(**kwargs): "bm25s or Stemmer is not installed. Please install it with `pip install bm25s Stemmer`." ) - class BM25Search(DRESModel): + class BM25Search(DRESModel, Wrapper): """BM25 search""" def __init__( @@ -110,23 +110,6 @@ def encode(self, texts: list[str], **kwargs): """Encode input text as term vectors""" return bm25s.tokenize(texts, stopwords=self.stopwords, stemmer=self.stemmer) - def encode_queries( - self, - queries: list[str], - batch_size: int = 32, - **kwargs: Any, - ): - return self.encode(queries, kwargs=kwargs) - - def encode_corpus( - self, - corpus: list[dict[str, str]] | dict[str, list[str]], - batch_size: int = 32, - **kwargs: Any, - ): - sentences = corpus_to_texts(corpus) - return self.encode(sentences, kwargs=kwargs) - return BM25Search(**kwargs) @@ -134,7 +117,16 @@ def encode_corpus( loader=partial(bm25_loader, model_name="bm25s"), # type: ignore name="bm25s", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="0_1_10", release_date="2024-07-10", ## release of version 0.1.10 + n_parameters=None, + memory_usage=None, + embed_dim=None, + license=None, + max_tokens=None, + reference=None, + similarity_fn_name=None, + framework=[], + use_instuctions=False, ) diff --git a/mteb/models/cache_wrapper.py b/mteb/models/cache_wrapper.py new file mode 100644 index 0000000000..61abccb9da --- /dev/null +++ b/mteb/models/cache_wrapper.py @@ -0,0 +1,294 @@ +from __future__ import annotations + +import hashlib +import json +import logging +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +from mteb.encoder_interface import Encoder +from mteb.models.wrapper import Wrapper + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +class TextVectorMap: + def __init__( + self, + directory: str | Path, + initial_vectors: int = 100000, + ): + self.directory = Path(directory) + self.directory.mkdir(parents=True, exist_ok=True) + self.vectors_file = self.directory / "vectors.npy" + self.index_file = self.directory / "index.json" + self.dimension_file = self.directory / "dimension" + self.hash_to_index: dict[str, int] = {} + self.vectors: np.memmap | None = None + self.vector_dim: int | None = None + self.initial_vectors = initial_vectors + logger.info(f"Initialized TextVectorMap in directory: {self.directory}") + self._initialize_vectors_file() + + def _hash_text(self, text: str) -> str: + return hashlib.sha256(text.encode()).hexdigest() + + def add(self, text: str, vector: np.ndarray) -> None: + try: + if self.vector_dim is None: + self.vector_dim = vector.shape[0] + self._initialize_vectors_file() + self._save_dimension() + logger.info(f"Initialized vector dimension to {self.vector_dim}") + + text_hash = self._hash_text(text) + if text_hash in self.hash_to_index: + logger.warning( + "Hash collision or duplicate text. Overwriting existing vector." + ) + index = self.hash_to_index[text_hash] + else: + index = len(self.hash_to_index) + if index >= len(self.vectors): + self._double_vectors_file() + self.hash_to_index[text_hash] = index + + self.vectors[index] = vector + logger.debug( + f"Added new text-vector pair. Total pairs: {len(self.hash_to_index)}" + ) + except Exception as e: + logger.error(f"Error adding text-vector pair: {str(e)}") + raise + + def _initialize_vectors_file(self): + if self.vector_dim is None: + logger.info("Vector dimension not set. Waiting for first add() call.") + return + + if not self.vectors_file.exists(): + logger.info( + f"Creating initial vectors file with {self.initial_vectors} vectors" + ) + self.vectors = np.memmap( + self.vectors_file, + dtype="float32", + mode="w+", + shape=(self.initial_vectors, self.vector_dim), + ) + else: + self.vectors = np.memmap(self.vectors_file, dtype="float32", mode="r+") + self.vectors = self.vectors.reshape(-1, self.vector_dim) + logger.info(f"Vectors file initialized with shape: {self.vectors.shape}") + + def _double_vectors_file(self): + current_size = len(self.vectors) + new_size = current_size * 2 + logger.info(f"Doubling vectors file from {current_size} to {new_size} vectors") + self.vectors.flush() + new_vectors = np.memmap( + self.vectors_file, + dtype="float32", + mode="r+", + shape=(new_size, self.vector_dim), + ) + new_vectors[:current_size] = self.vectors[:] + self.vectors = new_vectors + + def _save_dimension(self): + with open(self.dimension_file, "w") as f: + f.write(str(self.vector_dim)) + logger.info( + f"Saved vector dimension {self.vector_dim} to {self.dimension_file}" + ) + + def _load_dimension(self): + if self.dimension_file.exists(): + with open(self.dimension_file) as f: + self.vector_dim = int(f.read().strip()) + logger.info( + f"Loaded vector dimension {self.vector_dim} from {self.dimension_file}" + ) + else: + logger.warning( + "Dimension file not found. Vector dimension remains uninitialized." + ) + + def save(self) -> None: + try: + if self.vectors is not None: + self.vectors.flush() + + # Convert hash_to_index dict to a format suitable for JSON + # JSON doesn't support integer keys, so we keep everything as strings + serializable_index = { + str(hash_): int(index) # Ensure indices are serialized as integers + for hash_, index in self.hash_to_index.items() + } + + with open(self.index_file, "w", encoding="utf-8") as f: + json.dump(serializable_index, f, indent=2) + + self._save_dimension() + logger.info(f"Saved TextVectorMap to {self.directory}") + except Exception as e: + logger.error(f"Error saving TextVectorMap: {str(e)}") + raise + + def load(self, name: str | None = None) -> None: + name_details = name if name else "" + try: + self._load_dimension() + if self.index_file.exists() and self.vectors_file.exists(): + with open(self.index_file, encoding="utf-8") as f: + # Load and convert the JSON data back to the expected format + loaded_index = json.load(f) + self.hash_to_index = { + str(hash_): int(index) # Ensure we maintain the correct types + for hash_, index in loaded_index.items() + } + + if self.vector_dim is not None: + self.vectors = np.memmap( + self.vectors_file, dtype="float32", mode="r+" + ) + self.vectors = self.vectors.reshape(-1, self.vector_dim) + logger.info(f"Loaded vectors file with shape: {self.vectors.shape}") + else: + logger.warning( + "Vector dimension not set. Unable to load vectors file." + ) + + logger.info( + f"Loaded TextVectorMap ({name_details}) from {self.directory}" + ) + else: + logger.warning( + f"No existing files found. Initialized empty TextVectorMap ({name_details})." + ) + except Exception as e: + logger.error(f"Error loading TextVectorMap ({name_details}): {str(e)}") + raise + + def get_vector(self, text: str) -> np.ndarray | None: + try: + text_hash = self._hash_text(text) + if text_hash not in self.hash_to_index: + logger.debug(f"Text hash not found in index: {text_hash}") + return None + index = self.hash_to_index[text_hash] + return self.vectors[index] + except Exception as e: + logger.error(f"Error retrieving vector for text: {str(e)}") + raise + + def __contains__(self, text: str) -> bool: + return self._hash_text(text) in self.hash_to_index + + def __del__(self): + self.close() + + def close(self): + if hasattr(self, "vectors") and self.vectors is not None: + self.vectors.flush() + del self.vectors + self.vectors = None + logger.info(f"Closed TextVectorMap in directory: {self.directory}") + + +class CachedEmbeddingWrapper(Wrapper, Encoder): + def __init__(self, model: Encoder, cache_path: str | Path): + self._model = model + self.cache_path = Path(cache_path) + self.cache_path.mkdir(parents=True, exist_ok=True) + + if hasattr(model, "encode"): + self.cache = TextVectorMap(self.cache_path / "cache") + self.cache.load(name="cache") + else: + logger.error("Model must have an 'encode' method.") + raise ValueError("Invalid model encoding method") + + logger.info("Initialized CachedEmbeddingWrapper") + + def encode(self, texts: list[str], batch_size: int = 32, **kwargs) -> np.ndarray: + """Encode texts using the wrapped model, with caching""" + try: + results = [] + uncached_texts = [] + uncached_indices = [] + + # Check cache for each text + for i, text in enumerate(texts): + vector = self.cache.get_vector(text) + if vector is not None: + results.append(vector) + else: + uncached_texts.append(text) + uncached_indices.append(i) + + # Encode any texts not found in cache + if uncached_texts: + logger.info(f"Encoding {len(uncached_texts)} new texts") + new_vectors = self._model.encode( + uncached_texts, batch_size=batch_size, **kwargs + ) + if isinstance(new_vectors, torch.Tensor): + new_vectors = new_vectors.cpu().numpy() + + # Add new vectors to cache + for text, vector in zip(uncached_texts, new_vectors): + self.cache.add(text, vector) + results.extend(new_vectors) + self.cache.save() + else: + logger.info("All texts found in cache") + + # Reconstruct results in original order + final_results = [None] * len(texts) + uncached_idx = 0 + for i in range(len(texts)): + if i in uncached_indices: + final_results[i] = results[ + len(texts) - len(uncached_texts) + uncached_idx + ] + uncached_idx += 1 + else: + final_results[i] = results[i - uncached_idx] + + return np.array(final_results) + except Exception as e: + logger.error(f"Error in cached encoding: {str(e)}") + raise + + def __getattr__(self, name: str) -> Any: + """Check for attributes in this class first, then fall back to model attributes""" + try: + # First try to get the attribute from this class's __dict__ + return self.__dict__[name] + except KeyError: + # If not found, try the model's attributes + try: + return getattr(self._model, name) + except AttributeError: + raise AttributeError( + f"Neither {self.__class__.__name__} nor the wrapped model " + f"has attribute '{name}'" + ) + + def __dir__(self) -> list[str]: + """Return all attributes from both this class and the wrapped model""" + return list(set(super().__dir__() + dir(self._model))) + + def __del__(self): + self.close() + + def close(self): + self.cache.close() + logger.info("Closed CachedEmbeddingWrapper") diff --git a/mteb/models/cohere_models.py b/mteb/models/cohere_models.py index c47ec4200d..276bc6587c 100644 --- a/mteb/models/cohere_models.py +++ b/mteb/models/cohere_models.py @@ -6,16 +6,30 @@ import numpy as np import torch -import mteb -from mteb.encoder_interface import Encoder +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta +from mteb.models.sentence_transformer_wrapper import ( + get_prompt_name, + validate_task_to_prompt_name, +) + +from .wrapper import Wrapper # Implementation follows https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/blob/main/src/seb/registered_models/cohere_models.py -class CohereTextEmbeddingModel(Encoder): - def __init__(self, model_name: str, sep: str = " ", **kwargs) -> None: +class CohereTextEmbeddingModel(Wrapper): + def __init__( + self, + model_name: str, + sep: str = " ", + model_prompts: dict[str, str] | None = None, + **kwargs, + ) -> None: self.model_name = model_name self.sep = sep + self.model_prompts = ( + validate_task_to_prompt_name(model_prompts) if model_prompts else None + ) def _embed( self, sentences: list[str], cohere_task_type: str, retries: int = 5 @@ -41,46 +55,35 @@ def _embed( def encode( self, sentences: list[str], - prompt_name: str | None = None, - # search_document is recommended if unknown (https://cohere.com/blog/introducing-embed-v3) - cohere_task_type: str = "search_document", - **kwargs: Any, # noqa: ARG002 + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, ) -> np.ndarray: - if prompt_name: - task = mteb.get_task(prompt_name) - task_type = task.metadata.type - if task_type in ["Classification", "MultilabelClassification"]: - cohere_task_type = "classification" - elif task_type == "Clustering": - cohere_task_type = "clustering" + cohere_task_type = get_prompt_name(self.model_prompts, task_name, prompt_type) + if cohere_task_type is None: + # search_document is recommended if unknown (https://cohere.com/blog/introducing-embed-v3) + cohere_task_type = "search_document" return self._embed(sentences, cohere_task_type=cohere_task_type).numpy() - def encode_queries(self, queries: list[str], **kwargs: Any) -> np.ndarray: # noqa: ARG002 - return self._embed(queries, cohere_task_type="search_query").numpy() - - def encode_corpus(self, corpus: list[dict[str, str]], **kwargs: Any) -> np.ndarray: # noqa: ARG002 - if isinstance(corpus, dict): - sentences = [ - (corpus["title"][i] + self.sep + corpus["text"][i]).strip() # type: ignore - if "title" in corpus - else corpus["text"][i].strip() # type: ignore - for i in range(len(corpus["text"])) # type: ignore - ] - else: - sentences = [ - (doc["title"] + self.sep + doc["text"]).strip() - if "title" in doc - else doc["text"].strip() - for doc in corpus - ] - return self._embed(sentences, cohere_task_type="search_document").numpy() +model_prompts = { + "Classification": "classification", + "MultilabelClassification": "classification", + "Clustering": "clustering", + PromptType.query.value: "search_query", + PromptType.passage.value: "search_document", +} cohere_mult_3 = ModelMeta( - loader=partial(CohereTextEmbeddingModel, model_name="embed-multilingual-v3.0"), + loader=partial( + CohereTextEmbeddingModel, + model_name="embed-multilingual-v3.0", + model_prompts=model_prompts, + ), name="embed-multilingual-v3.0", languages=[], # Unknown, but support >100 languages - open_source=False, + open_weights=False, revision="1", release_date="2023-11-02", n_parameters=None, @@ -89,14 +92,19 @@ def encode_corpus(self, corpus: list[dict[str, str]], **kwargs: Any) -> np.ndarr embed_dim=1024, license=None, similarity_fn_name="cosine", - framework=[], + framework=["API"], + use_instuctions=False, ) cohere_eng_3 = ModelMeta( - loader=partial(CohereTextEmbeddingModel, model_name="embed-english-v3.0"), + loader=partial( + CohereTextEmbeddingModel, + model_name="embed-multilingual-v3.0", + model_prompts=model_prompts, + ), name="embed-english-v3.0", languages=["eng-Latn"], - open_source=False, + open_weights=False, revision="1", release_date="2023-11-02", n_parameters=None, @@ -105,9 +113,6 @@ def encode_corpus(self, corpus: list[dict[str, str]], **kwargs: Any) -> np.ndarr embed_dim=1024, license=None, similarity_fn_name="cosine", - framework=[], + framework=["API"], + use_instuctions=False, ) - -if __name__ == "__main__": - mdl = mteb.get_model(cohere_mult_3.name, cohere_mult_3.revision) - emb = mdl.encode(["Hello, world!"]) diff --git a/mteb/models/e5_instruct.py b/mteb/models/e5_instruct.py index d21017c920..0be991347e 100644 --- a/mteb/models/e5_instruct.py +++ b/mteb/models/e5_instruct.py @@ -1,13 +1,18 @@ from __future__ import annotations +from collections.abc import Sequence from functools import partial +from typing import Any +import numpy as np import torch from mteb.model_meta import ModelMeta +from ..encoder_interface import PromptType from .e5_models import E5_PAPER_RELEASE_DATE, XLMR_LANGUAGES from .instructions import task_to_instruction +from .wrapper import Wrapper MISTRAL_LANGUAGES = ["eng_Latn", "fra_Latn", "deu_Latn", "ita_Latn", "spa_Latn"] @@ -24,25 +29,24 @@ def e5_loader(**kwargs): "Please install `pip install gritlm` to use E5 Instruct models." ) - class E5InstructWrapper(GritLM): - def encode(self, *args, **kwargs): - if "prompt_name" in kwargs: - if "instruction" in kwargs: - raise ValueError( - "Cannot specify both `prompt_name` and `instruction`." - ) + class E5InstructWrapper(GritLM, Wrapper): + def encode( + self, + sentences: Sequence[str], + *args, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + if "instruction" in kwargs: + instruction = kwargs.pop("instruction", "") + else: instruction = task_to_instruction( - kwargs.pop("prompt_name"), kwargs.pop("is_query", True) + task_name, prompt_type == PromptType.query ) - else: - instruction = kwargs.pop("instruction", "") if instruction: kwargs["instruction"] = e5_instruction(instruction) - return super().encode(*args, **kwargs) - - def encode_corpus(self, *args, **kwargs): - kwargs["is_query"] = False - return super().encode_corpus(*args, **kwargs) + return super().encode(sentences, *args, **kwargs) return E5InstructWrapper(**kwargs) @@ -59,9 +63,18 @@ def encode_corpus(self, *args, **kwargs): ), name="intfloat/multilingual-e5-large-instruct", languages=XLMR_LANGUAGES, - open_source=True, + open_weights=True, revision="baa7be480a7de1539afce709c8f13f833a510e0a", release_date=E5_PAPER_RELEASE_DATE, + framework=["GritLM", "PyTorch"], + similarity_fn_name="cosine", + use_instuctions=True, + reference="https://huggingface.co/intfloat/multilingual-e5-large-instruct", + n_parameters=560_000_000, + memory_usage=None, + embed_dim=1024, + license="mit", + max_tokens=514, ) e5_mistral = ModelMeta( @@ -78,7 +91,16 @@ def encode_corpus(self, *args, **kwargs): ), name="intfloat/e5-mistral-7b-instruct", languages=MISTRAL_LANGUAGES, - open_source=True, + open_weights=True, revision="07163b72af1488142a360786df853f237b1a3ca1", release_date=E5_PAPER_RELEASE_DATE, + framework=["GritLM", "PyTorch"], + similarity_fn_name="cosine", + use_instuctions=True, + reference="https://huggingface.co/intfloat/e5-mistral-7b-instruct", + n_parameters=7_111_000_000, + memory_usage=None, + embed_dim=4096, + license="mit", + max_tokens=32768, ) diff --git a/mteb/models/e5_models.py b/mteb/models/e5_models.py index b3a8fc74bd..5549c7dd86 100644 --- a/mteb/models/e5_models.py +++ b/mteb/models/e5_models.py @@ -1,13 +1,9 @@ from __future__ import annotations from functools import partial -from typing import Any -import torch -from sentence_transformers import SentenceTransformer - -from mteb.model_meta import ModelMeta -from mteb.models.text_formatting_utils import corpus_to_texts +from mteb.encoder_interface import PromptType +from mteb.model_meta import ModelMeta, sentence_transformers_loader E5_PAPER_RELEASE_DATE = "2024-02-08" XLMR_LANGUAGES = [ @@ -112,118 +108,166 @@ "zho_Hans", ] - -class E5Wrapper: - """following the implementation within the Scandinavian Embedding Benchmark and the intfloat/multilingual-e5-small documentation.""" - - def __init__( - self, - model_name: str, - sep: str = " ", - prompt_name: str | None = None, - **kwargs: Any, - ): - self.model_name = model_name - self.mdl = SentenceTransformer(model_name) - self.sep = sep - - def to(self, device: torch.device) -> None: - self.mdl.to(device) - - def encode( # type: ignore - self, - sentences: list[str], - *, - batch_size: int = 32, - **kwargs: Any, - ): - return self.encode_queries(sentences, batch_size=batch_size, **kwargs) - - def encode_queries( - self, - queries: list[str], - batch_size: int = 32, - prompt_name: str | None = None, - **kwargs: Any, - ): - sentences = ["query: " + sentence for sentence in queries] - emb = self.mdl.encode(sentences, batch_size=batch_size, **kwargs) - return emb - - def encode_corpus( - self, - corpus: list[dict[str, str]] | dict[str, list[str]], - prompt_name: str | None = None, - batch_size: int = 32, - **kwargs: Any, - ): - if "request_qid" in kwargs: - kwargs.pop("request_qid") - sentences = corpus_to_texts(corpus) - sentences = ["passage: " + sentence for sentence in sentences] - emb = self.mdl.encode(sentences, batch_size=batch_size, **kwargs) - return emb - +model_prompts = { + PromptType.query.value: "query: ", + PromptType.passage.value: "passage: ", +} e5_mult_small = ModelMeta( - loader=partial(E5Wrapper, model_name="intfloat/multilingual-e5-small"), # type: ignore + loader=partial( + sentence_transformers_loader, + model_name="intfloat/multilingual-e5-small", + revision="fd1525a9fd15316a2d503bf26ab031a61d056e98", + model_prompts=model_prompts, + ), name="intfloat/multilingual-e5-small", languages=XLMR_LANGUAGES, - open_source=True, - revision="e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + open_weights=True, + revision="fd1525a9fd15316a2d503bf26ab031a61d056e98", release_date=E5_PAPER_RELEASE_DATE, + n_parameters=118_000_000, + memory_usage=None, + embed_dim=384, + license="mit", + max_tokens=512, + reference="https://huggingface.co/intfloat/multilingual-e5-small", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=True, ) e5_mult_base = ModelMeta( - loader=partial(E5Wrapper, model_name="intfloat/multilingual-e5-base"), # type: ignore + loader=partial( + sentence_transformers_loader, + model_name="intfloat/multilingual-e5-base", + model_prompts=model_prompts, + ), name="intfloat/multilingual-e5-base", languages=XLMR_LANGUAGES, - open_source=True, + open_weights=True, revision="d13f1b27baf31030b7fd040960d60d909913633f", release_date=E5_PAPER_RELEASE_DATE, + n_parameters=278_000_000, + memory_usage=None, + embed_dim=768, + license="mit", + max_tokens=514, + reference="https://huggingface.co/intfloat/multilingual-e5-base", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=True, ) e5_mult_large = ModelMeta( - loader=partial(E5Wrapper, model_name="intfloat/multilingual-e5-large"), # type: ignore + loader=partial( + sentence_transformers_loader, + model_name="intfloat/multilingual-e5-large", + revision="ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb", + model_prompts=model_prompts, + ), name="intfloat/multilingual-e5-large", languages=XLMR_LANGUAGES, - open_source=True, - revision="4dc6d853a804b9c8886ede6dda8a073b7dc08a81", + open_weights=True, + revision="ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb", release_date=E5_PAPER_RELEASE_DATE, + n_parameters=560_000_000, + memory_usage=None, + embed_dim=1024, + license="mit", + max_tokens=514, + reference="https://huggingface.co/intfloat/multilingual-e5-large", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=True, ) e5_eng_small_v2 = ModelMeta( - loader=partial(E5Wrapper, model_name="intfloat/e5-small-v2"), # type: ignore + loader=partial( + sentence_transformers_loader, + model_name="intfloat/e5-small-v2", + model_prompts=model_prompts, + ), name="intfloat/e5-small-v2", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="dca8b1a9dae0d4575df2bf423a5edb485a431236", release_date=E5_PAPER_RELEASE_DATE, + n_parameters=33_000_000, + memory_usage=None, + embed_dim=384, + license="mit", + max_tokens=512, + reference="https://huggingface.co/intfloat/e5-small-v2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=True, ) e5_eng_small = ModelMeta( - loader=partial(E5Wrapper, model_name="intfloat/e5-small"), # type: ignore + loader=partial( + sentence_transformers_loader, + model_name="intfloat/e5-small", + revision="e272f3049e853b47cb5ca3952268c6662abda68f", + model_prompts=model_prompts, + ), name="intfloat/e5-small", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="e272f3049e853b47cb5ca3952268c6662abda68f", release_date=E5_PAPER_RELEASE_DATE, + n_parameters=33_000_000, + memory_usage=None, + embed_dim=384, + license="mit", + max_tokens=512, + reference="https://huggingface.co/intfloat/e5-small", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=True, ) e5_eng_base_v2 = ModelMeta( - loader=partial(E5Wrapper, model_name="intfloat/e5-base-v2"), # type: ignore + loader=partial( + sentence_transformers_loader, + model_name="intfloat/e5-base-v2", + revision="1c644c92ad3ba1efdad3f1451a637716616a20e8", + model_prompts=model_prompts, + ), name="intfloat/e5-base-v2", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="1c644c92ad3ba1efdad3f1451a637716616a20e8", release_date=E5_PAPER_RELEASE_DATE, + n_parameters=278_000_000, + memory_usage=None, + embed_dim=768, + license="mit", + max_tokens=514, + reference="https://huggingface.co/intfloat/e5-base-v2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=True, ) e5_eng_large_v2 = ModelMeta( - loader=partial(E5Wrapper, model_name="intfloat/e5-large-v2"), # type: ignore + loader=partial( + sentence_transformers_loader, + model_name="intfloat/e5-large-v2", + revision="b322e09026e4ea05f42beadf4d661fb4e101d311", + model_prompts=model_prompts, + ), name="intfloat/e5-large-v2", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="b322e09026e4ea05f42beadf4d661fb4e101d311", release_date=E5_PAPER_RELEASE_DATE, + n_parameters=560_000_000, + memory_usage=None, + embed_dim=1024, + license="mit", + max_tokens=514, + reference="https://huggingface.co/intfloat/e5-large-v2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=True, ) diff --git a/mteb/models/google_models.py b/mteb/models/google_models.py index 44c7e21d12..3ec6f384c0 100644 --- a/mteb/models/google_models.py +++ b/mteb/models/google_models.py @@ -5,13 +5,28 @@ import numpy as np -from mteb.encoder_interface import Encoder +from mteb.encoder_interface import Encoder, PromptType from mteb.model_meta import ModelMeta +from mteb.models.sentence_transformer_wrapper import ( + get_prompt_name, + validate_task_to_prompt_name, +) + +from .wrapper import Wrapper -class GoogleTextEmbeddingModel(Encoder): - def __init__(self, model_name: str, sep: str = " ", **kwargs) -> None: +class GoogleTextEmbeddingModel(Encoder, Wrapper): + def __init__( + self, + model_name: str, + sep: str = " ", + model_prompts: dict[str, str] | None = None, + **kwargs, + ) -> None: self.model_name = model_name + self.model_prompts = ( + validate_task_to_prompt_name(model_prompts) if model_prompts else None + ) def _embed( self, @@ -55,47 +70,31 @@ def _embed( def encode( self, sentences: list[str], - prompt_name: str | None = None, - google_task_type: str | None = None, # Optional + task_name: str, + prompt_type: PromptType | None = None, **kwargs: Any, ) -> np.ndarray: - if prompt_name and google_task_type is None: - task = mteb.get_task(prompt_name) - task_type = task.metadata.type - if task_type in ["Classification", "MultilabelClassification"]: - google_task_type = "CLASSIFICATION" - elif task_type == "Clustering": - google_task_type = "CLUSTERING" - elif task_type == "STS": - google_task_type = "SIMILARITY" + google_task_type = get_prompt_name(self.model_prompts, task_name, prompt_type) return self._embed(sentences, google_task_type=google_task_type) - def encode_queries(self, queries: list[str], **kwargs: Any) -> np.ndarray: - return self._embed(queries, google_task_type="RETRIEVAL_QUERY") - - def encode_corpus(self, corpus: list[dict[str, str]], **kwargs: Any) -> np.ndarray: - if isinstance(corpus, dict): - sentences, titles = [], [] - - for i in range(len(corpus["text"])): # type: ignore - titles.append(corpus["title"][i]) # type: ignore - sentences.append(corpus["text"][i]) # type: ignore - else: - sentences, titles = [], [] - for doc in corpus: - titles.append(doc["title"]) - sentences.append(doc["text"]) - return self._embed( - sentences, google_task_type="RETRIEVAL_DOCUMENT", titles=titles - ) - name = "text-embedding-004" google_emb_004 = ModelMeta( - loader=partial(GoogleTextEmbeddingModel, model_name=name), + loader=partial( + GoogleTextEmbeddingModel, + model_name=name, + model_prompts={ + "Classification": "CLASSIFICATION", + "MultilabelClassification": "CLASSIFICATION", + "Clustering": "CLUSTERING", + "STS": "SIMILARITY", + PromptType.query.value: "RETRIEVAL_QUERY", + PromptType.passage.value: "RETRIEVAL_DOCUMENT", + }, + ), name=name, languages=["eng-Latn"], - open_source=False, + open_weights=False, revision="1", # revision is intended for implementation release_date=None, # couldnt figure this out n_parameters=None, @@ -104,12 +103,6 @@ def encode_corpus(self, corpus: list[dict[str, str]], **kwargs: Any) -> np.ndarr embed_dim=768, license=None, similarity_fn_name="cosine", # assumed - framework=[], + framework=["API"], + use_instuctions=True, ) - - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(google_emb_004.name, google_emb_004.revision) - emb = mdl.encode(["Hello, world!"]) diff --git a/mteb/models/gritlm_models.py b/mteb/models/gritlm_models.py index 2ef3a079b0..3f5dc173a1 100644 --- a/mteb/models/gritlm_models.py +++ b/mteb/models/gritlm_models.py @@ -1,13 +1,18 @@ from __future__ import annotations import logging +from collections.abc import Sequence from functools import partial +from typing import Any + +import numpy as np from mteb.model_meta import ModelMeta +from ..encoder_interface import PromptType from .instructions import task_to_instruction +from .wrapper import Wrapper -logging.basicConfig(level=logging.WARNING) logger = logging.getLogger(__name__) @@ -23,24 +28,24 @@ def gritlm_loader(**kwargs): except ImportError: raise ImportError("Please install `pip install gritlm` to use GritLM models.") - class GritLMWrapper(GritLM): - def encode(self, *args, **kwargs): - if "prompt_name" in kwargs: - if "instruction" in kwargs: - raise ValueError( - "Cannot specify both `prompt_name` and `instruction`." - ) + class GritLMWrapper(GritLM, Wrapper): + def encode( + self, + sentences: Sequence[str], + *args, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + if "instruction" in kwargs: + instruction = kwargs.pop("instruction", "") + else: instruction = task_to_instruction( - kwargs.pop("prompt_name"), kwargs.pop("is_query", True) + task_name, prompt_type == PromptType.query ) - else: - instruction = kwargs.pop("instruction", "") - kwargs["instruction"] = gritlm_instruction(instruction) - return super().encode(*args, **kwargs) - - def encode_corpus(self, *args, **kwargs): - kwargs["is_query"] = False - return super().encode_corpus(*args, **kwargs) + if instruction: + kwargs["instruction"] = gritlm_instruction(instruction) + return super().encode(sentences, *args, **kwargs) return GritLMWrapper(**kwargs) @@ -54,9 +59,18 @@ def encode_corpus(self, *args, **kwargs): ), name="GritLM/GritLM-7B", languages=["eng_Latn", "fra_Latn", "deu_Latn", "ita_Latn", "spa_Latn"], - open_source=True, + open_weights=True, revision="13f00a0e36500c80ce12870ea513846a066004af", release_date="2024-02-15", + n_parameters=7_240_000_000, + memory_usage=None, + embed_dim=4096, + license="apache-2.0", + max_tokens=4096, + reference="https://huggingface.co/GritLM/GritLM-7B", + similarity_fn_name="cosine", + framework=["GritLM", "PyTorch"], + use_instuctions=True, ) gritlm8x7b = ModelMeta( loader=partial( @@ -67,7 +81,16 @@ def encode_corpus(self, *args, **kwargs): ), name="GritLM/GritLM-8x7B", languages=["eng_Latn", "fra_Latn", "deu_Latn", "ita_Latn", "spa_Latn"], - open_source=True, + open_weights=True, revision="7f089b13e3345510281733ca1e6ff871b5b4bc76", release_date="2024-02-15", + n_parameters=57_920_000_000, + memory_usage=None, + embed_dim=4096, + license="apache-2.0", + max_tokens=4096, + reference="https://huggingface.co/GritLM/GritLM-8x7B", + similarity_fn_name="cosine", + framework=["GritLM", "PyTorch"], + use_instuctions=True, ) diff --git a/mteb/models/gte_models.py b/mteb/models/gte_models.py index 4df3b6e5ed..b6cc9bfb2e 100644 --- a/mteb/models/gte_models.py +++ b/mteb/models/gte_models.py @@ -1,10 +1,16 @@ from __future__ import annotations +from collections.abc import Sequence from functools import partial +from typing import Any +import numpy as np + +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta from .instructions import task_to_instruction +from .wrapper import Wrapper def gte_instruction(instruction: str) -> str: @@ -19,25 +25,24 @@ def gte_loader(**kwargs): "Please install `pip install gritlm` to use gte-Qwen2-7B-instruct." ) - class GTEWrapper(GritLM): - def encode(self, *args, **kwargs): - if "prompt_name" in kwargs: - if "instruction" in kwargs: - raise ValueError( - "Cannot specify both `prompt_name` and `instruction`." - ) + class GTEWrapper(GritLM, Wrapper): + def encode( + self, + sentences: Sequence[str], + *args, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + if "instruction" in kwargs: + instruction = kwargs.pop("instruction", "") + else: instruction = task_to_instruction( - kwargs.pop("prompt_name"), kwargs.pop("is_query", True) + task_name, prompt_type == PromptType.query ) - else: - instruction = kwargs.pop("instruction", "") if instruction: kwargs["instruction"] = gte_instruction(instruction) - return super().encode(*args, **kwargs) - - def encode_corpus(self, *args, **kwargs): - kwargs["is_query"] = False - return super().encode_corpus(*args, **kwargs) + return super().encode(sentences, *args, **kwargs) return GTEWrapper(**kwargs) @@ -56,48 +61,15 @@ def encode_corpus(self, *args, **kwargs): ), name="Alibaba-NLP/gte-Qwen2-7B-instruct", languages=None, - open_source=True, + open_weights=True, revision="e26182b2122f4435e8b3ebecbf363990f409b45b", release_date="2024-06-15", # initial commit of hf model. + n_parameters=7_613_000_000, + memory_usage=None, + embed_dim=3584, + license="apache-2.0", + reference="https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=True, ) - - -if __name__ == "__main__": - # Verify it reproduces https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct#sentence-transformers - from sentence_transformers import SentenceTransformer - - model = SentenceTransformer( - "Alibaba-NLP/gte-Qwen2-7B-instruct", trust_remote_code=True - ) - # Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████| 7/7 [00:10<00:00, 1.52s/it] - # Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. - # In case you want to reduce the maximum length: - model.max_seq_length = 8192 - queries = ["how much protein should a female eat", "summit define"] - documents = [ - "As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", - "Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments.", - ] - query_embeddings = model.encode(queries, prompt_name="query") - document_embeddings = model.encode(documents) - scores = (query_embeddings @ document_embeddings.T) * 100 - print(scores.tolist()) - # [[70.39706420898438, 3.4318461418151855], [4.516170978546143, 81.91815948486328]] - - import mteb - - model_mteb = mteb.get_model( - "Alibaba-NLP/gte-Qwen2-7B-instruct" - ) # gte_Qwen2_7B_instruct.name, gte_Qwen2_7B_instruct.revision) - # Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████| 7/7 [00:01<00:00, 5.71it/s] - # Created GritLM: torch.float32 dtype, lasttoken pool, embedding mode, cccc attn - # Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. - # ----------Using 8 data-parallel GPUs---------- - query_embeddings_mteb = model_mteb.encode( - queries, - instruction="Given a web search query, retrieve relevant passages that answer the query", - ) - document_embeddings_mteb = model_mteb.encode_corpus(documents) - scores_mteb = (query_embeddings_mteb @ document_embeddings_mteb.T) * 100 - print(scores_mteb.tolist()) - # [[70.39706420898438, 3.4318461418151855], [4.516170978546143, 81.91815948486328]] diff --git a/mteb/models/llm2vec_models.py b/mteb/models/llm2vec_models.py index b426779588..79f8d7950c 100644 --- a/mteb/models/llm2vec_models.py +++ b/mteb/models/llm2vec_models.py @@ -1,22 +1,20 @@ from __future__ import annotations import logging -from typing import Any, Callable, Literal +from typing import Any, Callable import numpy as np import torch -from mteb.encoder_interface import Encoder +from mteb.encoder_interface import Encoder, PromptType from mteb.model_meta import ModelMeta -from mteb.models.text_formatting_utils import corpus_to_texts from .instructions import task_to_instruction +from .sentence_transformer_wrapper import validate_task_to_prompt_name +from .wrapper import Wrapper -logging.basicConfig(level=logging.WARNING) logger = logging.getLogger(__name__) -EncodeTypes = Literal["query", "passage"] - def llm2vec_instruction(instruction): if len(instruction) > 0 and instruction[-1] != ":": @@ -24,8 +22,14 @@ def llm2vec_instruction(instruction): return instruction -class LLM2VecWrapper: - def __init__(self, *args, **kwargs): +class LLM2VecWrapper(Wrapper): + def __init__( + self, + model_prompts: dict[str, str] | None = None, + device: str | None = None, + *args, + **kwargs, + ): try: from llm2vec import LLM2Vec except ImportError: @@ -41,12 +45,12 @@ def __init__(self, *args, **kwargs): logger.warning( "LLM2Vec models were trained with flash attention enabled. For optimal performance, please install the `flash_attn` package with `pip install flash-attn --no-build-isolation`." ) - self.task_to_instructions = None - if "task_to_instructions" in kwargs: - self.task_to_instructions = kwargs.pop("task_to_instructions") + self.model_prompts = ( + validate_task_to_prompt_name(model_prompts) if model_prompts else None + ) - if "device" in kwargs: - kwargs["device_map"] = kwargs.pop("device") + if device: + kwargs["device_map"] = device elif torch.cuda.device_count() > 1: # bug fix for multi-gpu kwargs["device_map"] = None @@ -57,37 +61,17 @@ def encode( self, sentences: list[str], *, - prompt_name: str = None, + task_name: str, + prompt_type: PromptType | None = None, **kwargs: Any, # noqa ) -> np.ndarray: - if prompt_name is not None: - instruction = ( - self.task_to_instructions[prompt_name] - if self.task_to_instructions - and prompt_name in self.task_to_instructions - else llm2vec_instruction(task_to_instruction(prompt_name)) - ) - else: - instruction = "" + instruction = llm2vec_instruction( + task_to_instruction(task_name, prompt_type == PromptType.query) + ) sentences = [[instruction, sentence] for sentence in sentences] return self.model.encode(sentences, **kwargs) - def encode_corpus( - self, - corpus: list[dict[str, str]] | dict[str, list[str]] | list[str], - prompt_name: str = None, - **kwargs: Any, - ) -> np.ndarray: - sentences = corpus_to_texts(corpus, sep=" ") - sentences = [["", sentence] for sentence in sentences] - if "request_qid" in kwargs: - kwargs.pop("request_qid") - return self.model.encode(sentences, **kwargs) - - def encode_queries(self, queries: list[str], **kwargs: Any) -> np.ndarray: - return self.encode(queries, **kwargs) - def _loader(wrapper: type[LLM2VecWrapper], **kwargs) -> Callable[..., Encoder]: _kwargs = kwargs @@ -108,9 +92,18 @@ def loader_inner(**kwargs: Any) -> Encoder: ), name="McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised", languages=["eng_Latn"], - open_source=True, - revision=None, # TODO: Not sure what to put here as a model is made of two peft repos, each with a different revision + open_weights=True, + revision="baa8ebf04a1c2500e61288e7dad65e8ae42601a7", # TODO: Not sure what to put here as a model is made of two peft repos, each with a different revision release_date="2024-04-09", + n_parameters=7_505_000_000, + memory_usage=None, + max_tokens=8192, + embed_dim=4096, + license="mit", + reference="https://huggingface.co/McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised", + similarity_fn_name="cosine", + framework=["LLM2Vec", "PyTorch"], + use_instuctions=True, ) llm2vec_llama3_8b_unsupervised = ModelMeta( @@ -123,9 +116,18 @@ def loader_inner(**kwargs: Any) -> Encoder: ), name="McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-unsup-simcse", languages=["eng_Latn"], - open_source=True, - revision=None, + open_weights=True, + revision="1cb7b735326d13a8541db8f57f35da5373f5e9c6", release_date="2024-04-09", + n_parameters=7_505_000_000, + memory_usage=None, + max_tokens=8192, + embed_dim=4096, + license="mit", + reference="https://huggingface.co/McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-unsup-simcse", + similarity_fn_name="cosine", + framework=["LLM2Vec", "PyTorch"], + use_instuctions=True, ) @@ -139,9 +141,18 @@ def loader_inner(**kwargs: Any) -> Encoder: ), name="McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised", languages=["eng_Latn"], - open_source=True, - revision=None, + open_weights=True, + revision="0ae69bdd5816105778b971c3138e8f8a18eaa3ae", release_date="2024-04-09", + n_parameters=7_111_000_000, + memory_usage=None, + max_tokens=32768, + embed_dim=4096, + license="mit", + reference="https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised", + similarity_fn_name="cosine", + framework=["LLM2Vec", "PyTorch"], + use_instuctions=True, ) llm2vec_mistral7b_unsupervised = ModelMeta( @@ -154,9 +165,18 @@ def loader_inner(**kwargs: Any) -> Encoder: ), name="McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-unsup-simcse", languages=["eng_Latn"], - open_source=True, - revision=None, + open_weights=True, + revision="2c055a5d77126c0d3dc6cd8ffa30e2908f4f45f8", release_date="2024-04-09", + n_parameters=7_111_000_000, + memory_usage=None, + max_tokens=32768, + embed_dim=4096, + license="mit", + reference="https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-unsup-simcse", + similarity_fn_name="cosine", + framework=["LLM2Vec", "PyTorch"], + use_instuctions=True, ) llm2vec_llama2_7b_supervised = ModelMeta( @@ -169,9 +189,18 @@ def loader_inner(**kwargs: Any) -> Encoder: ), name="McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised", languages=["eng_Latn"], - open_source=True, - revision=None, + open_weights=True, + revision="2c055a5d77126c0d3dc6cd8ffa30e2908f4f45f8", release_date="2024-04-09", + n_parameters=7_111_000_000, + memory_usage=None, + max_tokens=32768, + embed_dim=4096, + license="mit", + reference="https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised", + similarity_fn_name="cosine", + framework=["LLM2Vec", "PyTorch"], + use_instuctions=True, ) llm2vec_llama2_7b_unsupervised = ModelMeta( @@ -184,9 +213,18 @@ def loader_inner(**kwargs: Any) -> Encoder: ), name="McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-unsup-simcse", languages=["eng_Latn"], - open_source=True, - revision=None, + open_weights=True, + revision="a76944871d169ebe7c97eb921764cd063afed785", release_date="2024-04-09", + n_parameters=7_111_000_000, + memory_usage=None, + max_tokens=32768, + embed_dim=4096, + license="mit", + reference="https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-unsup-simcse", + similarity_fn_name="cosine", + framework=["LLM2Vec", "PyTorch"], + use_instuctions=True, ) llm2vec_sheared_llama_supervised = ModelMeta( @@ -199,9 +237,18 @@ def loader_inner(**kwargs: Any) -> Encoder: ), name="McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-supervised", languages=["eng_Latn"], - open_source=True, - revision=None, + open_weights=True, + revision="a5943d406c6b016fef3f07906aac183cf1a0b47d", release_date="2024-04-09", + n_parameters=7_111_000_000, + memory_usage=None, + max_tokens=32768, + embed_dim=4096, + license="mit", + reference="https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-supervised", + similarity_fn_name="cosine", + framework=["LLM2Vec", "PyTorch"], + use_instuctions=True, ) llm2vec_sheared_llama_unsupervised = ModelMeta( @@ -214,7 +261,16 @@ def loader_inner(**kwargs: Any) -> Encoder: ), name="McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-unsup-simcse", languages=["eng_Latn"], - open_source=True, - revision=None, + open_weights=True, + revision="a5943d406c6b016fef3f07906aac183cf1a0b47d", release_date="2024-04-09", + n_parameters=7_111_000_000, + memory_usage=None, + max_tokens=32768, + embed_dim=4096, + license="mit", + reference="https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-unsup-simcse", + similarity_fn_name="cosine", + framework=["LLM2Vec", "PyTorch"], + use_instuctions=True, ) diff --git a/mteb/models/mxbai_models.py b/mteb/models/mxbai_models.py index fee824c30c..b5451e30ec 100644 --- a/mteb/models/mxbai_models.py +++ b/mteb/models/mxbai_models.py @@ -1,67 +1,30 @@ from __future__ import annotations from functools import partial -from typing import Any - -import torch -from sentence_transformers import SentenceTransformer - -from mteb.model_meta import ModelMeta -from mteb.models.text_formatting_utils import corpus_to_texts - - -class MxbaiWrapper: - """following the hf model card documentation.""" - - def __init__(self, model_name: str, **kwargs: Any): - self.model_name = model_name - self.mdl = SentenceTransformer(model_name) - - def to(self, device: torch.device) -> None: - self.mdl.to(device) - - def encode( # type: ignore - self, - sentences: list[str], - *, - batch_size: int = 32, - **kwargs: Any, - ): - if "request_qid" in kwargs: - kwargs.pop("request_qid") - - return self.mdl.encode(sentences, batch_size=batch_size, **kwargs) - - def encode_queries(self, queries: list[str], batch_size: int = 32, **kwargs: Any): - sentences = [ - "Represent this sentence for searching relevant passages: " + sentence - for sentence in queries - ] - if "request_qid" in kwargs: - kwargs.pop("request_qid") - - emb = self.mdl.encode(sentences, batch_size=batch_size, **kwargs) - return emb - - def encode_corpus( - self, - corpus: list[dict[str, str]] | dict[str, list[str]], - batch_size: int = 32, - **kwargs: Any, - ): - sentences = corpus_to_texts(corpus) - if "request_qid" in kwargs: - kwargs.pop("request_qid") - - emb = self.mdl.encode(sentences, batch_size=batch_size, **kwargs) - return emb +from mteb.model_meta import ModelMeta, sentence_transformers_loader mxbai_embed_large_v1 = ModelMeta( - loader=partial(MxbaiWrapper, model_name="mixedbread-ai/mxbai-embed-large-v1"), # type: ignore + loader=partial( + sentence_transformers_loader, + model_name="mixedbread-ai/mxbai-embed-large-v1", + revision="990580e27d329c7408b3741ecff85876e128e203", + model_prompts={ + "query": "Represent this sentence for searching relevant passages: " + }, + ), name="mixedbread-ai/mxbai-embed-large-v1", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="990580e27d329c7408b3741ecff85876e128e203", release_date="2024-03-07", # initial commit of hf model. + n_parameters=335_000_000, + memory_usage=None, + max_tokens=512, + embed_dim=1024, + license="apache-2.0", + reference="https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=True, ) diff --git a/mteb/models/nomic_models.py b/mteb/models/nomic_models.py index d05d65c16d..00c9341b3b 100644 --- a/mteb/models/nomic_models.py +++ b/mteb/models/nomic_models.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from functools import partial from typing import Any @@ -7,37 +8,47 @@ import torch.nn.functional as F from sentence_transformers import SentenceTransformer -import mteb +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta -from mteb.models.text_formatting_utils import corpus_to_texts +from mteb.models.sentence_transformer_wrapper import ( + get_prompt_name, + validate_task_to_prompt_name, +) + +from .wrapper import Wrapper + +logger = logging.getLogger(__name__) -class NomicWrapper: +class NomicWrapper(Wrapper): """following the hf model card documentation.""" - def __init__(self, model_name: str, revision: str, **kwargs: Any): + def __init__( + self, + model_name: str, + revision: str, + model_prompts: dict[str, str] | None = None, + **kwargs: Any, + ): self.model_name = model_name - self.mdl = SentenceTransformer(model_name, revision=revision, **kwargs) + self.model = SentenceTransformer(model_name, revision=revision, **kwargs) + self.model_prompts = ( + validate_task_to_prompt_name(model_prompts) if model_prompts else None + ) def to(self, device: torch.device) -> None: - self.mdl.to(device) + self.model.to(device) def encode( # type: ignore self, sentences: list[str], *, - prompt_name: str | None = None, + task_name: str, + prompt_type: PromptType | None = None, batch_size: int = 32, - input_type: str | None = None, **kwargs: Any, ): - if prompt_name: - task = mteb.get_task(prompt_name) - task_type = task.metadata.type - if task_type in ["Classification", "MultilabelClassification"]: - input_type = "classification" - elif task_type == "Clustering": - input_type = "clustering" + input_type = get_prompt_name(self.model_prompts, task_name, prompt_type) # default to search_document if input_type and prompt_name are not provided if input_type is None: @@ -45,7 +56,7 @@ def encode( # type: ignore sentences = [f"{input_type}: {sentence}" for sentence in sentences] - emb = self.mdl.encode(sentences, batch_size=batch_size, **kwargs) + emb = self.model.encode(sentences, batch_size=batch_size, **kwargs) # v1.5 has a non-trainable layer norm to unit normalize the embeddings for binary quantization # the outputs are similar to if we just normalized but keeping the same for consistency if self.model_name == "nomic-ai/nomic-embed-text-v1.5": @@ -58,31 +69,14 @@ def encode( # type: ignore return emb - def encode_queries(self, queries: list[str], batch_size: int = 32, **kwargs: Any): - if "prompt_name" in kwargs: - kwargs.pop("prompt_name") - - emb = self.encode( - queries, batch_size=batch_size, input_type="search_query", **kwargs - ) - - return emb - - def encode_corpus( - self, - corpus: list[dict[str, str]] | dict[str, list[str]], - batch_size: int = 32, - **kwargs: Any, - ): - if "prompt_name" in kwargs: - kwargs.pop("prompt_name") - - sentences = corpus_to_texts(corpus) - emb = self.encode( - sentences, batch_size=batch_size, input_type="search_document", **kwargs - ) - return emb +model_prompts = { + "Classification": "classification: ", + "MultilabelClassification": "classification: ", + "Clustering": "clustering: ", + PromptType.query.value: "search_query: ", + PromptType.passage.value: "search_document: ", +} nomic_embed_v1_5 = ModelMeta( loader=partial( # type: ignore @@ -90,10 +84,11 @@ def encode_corpus( trust_remote_code=True, model_name="nomic-ai/nomic-embed-text-v1.5", revision="b0753ae76394dd36bcfb912a46018088bca48be0", + model_prompts=model_prompts, ), name="nomic-ai/nomic-embed-text-v1.5", languages=["eng-Latn"], - open_source=True, + open_weights=True, revision="b0753ae76394dd36bcfb912a46018088bca48be0", release_date="2024-02-10", # first commit ) @@ -104,27 +99,20 @@ def encode_corpus( trust_remote_code=True, model_name="nomic-ai/nomic-embed-text-v1", revision="0759316f275aa0cb93a5b830973843ca66babcf5", + model_prompts=model_prompts, ), name="nomic-ai/nomic-embed-text-v1", languages=["eng-Latn"], - open_source=True, + open_weights=True, revision="0759316f275aa0cb93a5b830973843ca66babcf5", release_date="2024-01-31", # first commit + n_parameters=None, + memory_usage=None, + max_tokens=8192, + embed_dim=768, + license="apache-2.0", + reference="https://huggingface.co/nomic-ai/nomic-embed-text-v1", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=True, ) - -if __name__ == "__main__": - mdl = mteb.get_model(nomic_embed_v1_5.name, nomic_embed_v1_5.revision) - emb = mdl.encode(["test"], convert_to_tensor=True) - print(emb.shape) - emb = mdl.encode_queries(["test"], convert_to_tensor=True) - print(emb.shape) - emb = mdl.encode( - ["test"], - convert_to_tensor=True, - prompt_name="AmazonCounterfactualClassification", - ) - print(emb.shape) - - mdl = mteb.get_model(nomic_embed_v1.name, nomic_embed_v1.revision) - emb = mdl.encode(["test"], convert_to_tensor=True) - print(emb.shape) diff --git a/mteb/models/openai_models.py b/mteb/models/openai_models.py index 03de44ba3c..d1eaf61644 100644 --- a/mteb/models/openai_models.py +++ b/mteb/models/openai_models.py @@ -7,13 +7,14 @@ import numpy as np from mteb.model_meta import ModelMeta -from mteb.models.text_formatting_utils import corpus_to_texts from mteb.requires_package import requires_package +from .wrapper import Wrapper + logger = logging.getLogger(__name__) -class OpenAIWrapper: +class OpenAIWrapper(Wrapper): def __init__(self, model_name: str, embed_dim: int | None = None, **kwargs) -> None: requires_package(self, "openai", "Openai text embedding") from openai import OpenAI @@ -31,23 +32,24 @@ def encode(self, sentences: list[str], **kwargs: Any) -> np.ndarray: "Reducing embedding size available only for text-embedding-3-* models" ) - return self._to_numpy( - self._client.embeddings.create( - input=sentences, + max_batch_size = 2048 + sublists = [ + sentences[i : i + max_batch_size] + for i in range(0, len(sentences), max_batch_size) + ] + + all_embeddings = [] + + for sublist in sublists: + response = self._client.embeddings.create( + input=sublist, model=self._model_name, encoding_format="float", dimensions=self._embed_dim or NotGiven(), ) - ) - - def encode_queries(self, queries: list[str], **kwargs: Any) -> np.ndarray: - return self.encode(queries, **kwargs) + all_embeddings.extend(self._to_numpy(response)) - def encode_corpus( - self, corpus: list[dict[str, str]] | dict[str, list[str]], **kwargs: Any - ) -> np.ndarray: - sentences = corpus_to_texts(corpus) - return self.encode(sentences, **kwargs) + return np.array(all_embeddings) def _to_numpy(self, embedding_response) -> np.ndarray: return np.array([e.embedding for e in embedding_response.data]) @@ -61,7 +63,14 @@ def _to_numpy(self, embedding_response) -> np.ndarray: loader=partial(OpenAIWrapper, model_name="text-embedding-3-small"), max_tokens=8191, embed_dim=1536, - open_source=False, + open_weights=False, + n_parameters=None, + memory_usage=None, + license=None, + reference="https://openai.com/index/new-embedding-models-and-api-updates/", + similarity_fn_name="cosine", + framework=["API"], + use_instuctions=False, ) text_embedding_3_large = ModelMeta( name="text-embedding-3-large", @@ -71,7 +80,11 @@ def _to_numpy(self, embedding_response) -> np.ndarray: loader=partial(OpenAIWrapper, model_name="text-embedding-3-large"), max_tokens=8191, embed_dim=3072, - open_source=False, + open_weights=False, + framework=["API"], + use_instuctions=False, + n_parameters=None, + memory_usage=None, ) text_embedding_ada_002 = ModelMeta( name="text-embedding-ada-002", @@ -81,5 +94,9 @@ def _to_numpy(self, embedding_response) -> np.ndarray: loader=partial(OpenAIWrapper, model_name="text-embedding-ada-002"), max_tokens=8191, embed_dim=1536, - open_source=False, + open_weights=False, + framework=["API"], + use_instuctions=False, + n_parameters=None, + memory_usage=None, ) diff --git a/mteb/models/overview.py b/mteb/models/overview.py new file mode 100644 index 0000000000..0c2175ebdf --- /dev/null +++ b/mteb/models/overview.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import logging +from collections.abc import Iterable +from typing import Any + +from sentence_transformers import SentenceTransformer + +from mteb.encoder_interface import Encoder +from mteb.model_meta import ModelMeta +from mteb.models import ( + bge_models, + bm25, + cohere_models, + e5_instruct, + e5_models, + google_models, + gritlm_models, + gte_models, + llm2vec_models, + mxbai_models, + nomic_models, + openai_models, + promptriever_models, + repllama_models, + rerankers_custom, + rerankers_monot5_based, + ru_sentence_models, + salesforce_models, + sentence_transformers_models, + voyage_models, +) + +logger = logging.getLogger(__name__) + +model_modules = [ + bge_models, + bm25, + cohere_models, + e5_instruct, + e5_models, + google_models, + gritlm_models, + gte_models, + llm2vec_models, + mxbai_models, + nomic_models, + openai_models, + ru_sentence_models, + salesforce_models, + sentence_transformers_models, + voyage_models, + google_models, + repllama_models, + promptriever_models, + rerankers_monot5_based, + rerankers_custom, +] +MODEL_REGISTRY = {} + +for module in model_modules: + for mdl in vars(module).values(): + if isinstance(mdl, ModelMeta): + MODEL_REGISTRY[mdl.name] = mdl + + +def get_model_metas( + model_names: Iterable[str] | None = None, + languages: Iterable[str] | None = None, + open_source: bool | None = None, + frameworks: Iterable[str] | None = None, + n_parameters_range: tuple[int | None, int | None] = (None, None), +) -> list[ModelMeta]: + """Load all models' metadata that fit the specified criteria.""" + res = [] + model_names = set(model_names) if model_names is not None else None + languages = set(languages) if languages is not None else None + frameworks = set(frameworks) if frameworks is not None else None + for model_meta in MODEL_REGISTRY.values(): + if (model_names is not None) and (model_meta.name not in model_names): + continue + if languages is not None: + if (model_meta.languages is None) or not ( + languages <= set(model_meta.languages) + ): + continue + if (open_source is not None) and (model_meta.open_source != open_source): + continue + if (frameworks is not None) and not (frameworks <= set(model_meta.framework)): + continue + upper, lower = n_parameters_range + n_parameters = model_meta.n_parameters + if upper is not None: + if (n_parameters is None) or (n_parameters > upper): + continue + if lower is not None: + if (n_parameters is None) or (n_parameters < lower): + continue + res.append(model_meta) + return res + + +def get_model(model_name: str, revision: str | None = None, **kwargs: Any) -> Encoder: + """A function to fetch a model object by name. + + Args: + model_name: Name of the model to fetch + revision: Revision of the model to fetch + **kwargs: Additional keyword arguments to pass to the model loader + + Returns: + A model object + """ + meta = get_model_meta(model_name, revision) + model = meta.load_model(**kwargs) + + # If revision not available in the modelmeta, try to extract it from sentence-transformers + if meta.revision is None and isinstance(model, SentenceTransformer): + _meta = model_meta_from_sentence_transformers(model) + meta.revision = _meta.revision if _meta.revision else meta.revision + + model.mteb_model_meta = meta # type: ignore + return model + + +def get_model_meta(model_name: str, revision: str | None = None) -> ModelMeta: + """A function to fetch a model metadata object by name. + + Args: + model_name: Name of the model to fetch + revision: Revision of the model to fetch + + Returns: + A model metadata object + """ + if model_name in MODEL_REGISTRY: + if revision and (not MODEL_REGISTRY[model_name].revision == revision): + raise ValueError( + f"Model revision {revision} not found for model {model_name}. Expected {MODEL_REGISTRY[model_name].revision}." + ) + return MODEL_REGISTRY[model_name] + else: # assume it is a sentence-transformers model + logger.info( + "Model not found in model registry, assuming it is a sentence-transformers model." + ) + logger.info( + f"Attempting to extract metadata by loading the model ({model_name}) using sentence-transformers." + ) + model = SentenceTransformer( + model_name, revision=revision, trust_remote_code=True + ) + meta = model_meta_from_sentence_transformers(model) + + meta.revision = revision + meta.name = model_name + return meta + + +def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMeta: + try: + name = ( + model.model_card_data.model_name + if model.model_card_data.model_name + else model.model_card_data.base_model + ) + languages = ( + [model.model_card_data.language] + if isinstance(model.model_card_data.language, str) + else model.model_card_data.language + ) + meta = ModelMeta( + name=name, + revision=model.model_card_data.base_model_revision, + release_date=None, + languages=languages, + framework=["Sentence Transformers"], + similarity_fn_name=model.similarity_fn_name, + ) + except AttributeError as e: + logger.warning( + f"Failed to extract metadata from model: {e}. Upgrading to sentence-transformers v3.0.0 or above is recommended." + ) + meta = ModelMeta( + name=None, + revision=None, + languages=None, + release_date=None, + ) + return meta diff --git a/mteb/models/promptriever_models.py b/mteb/models/promptriever_models.py new file mode 100644 index 0000000000..b3ed5ca876 --- /dev/null +++ b/mteb/models/promptriever_models.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import logging +from typing import Any, Callable + +import numpy as np +import torch + +from mteb.encoder_interface import Encoder +from mteb.model_meta import ModelMeta + +from .repllama_models import RepLLaMAWrapper +from .wrapper import Wrapper + +logger = logging.getLogger(__name__) + + +class PromptrieverWrapper(RepLLaMAWrapper, Wrapper): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def encode_queries(self, queries: list[str], **kwargs: Any) -> np.ndarray: + queries = [f"query: {query}" for query in queries] + if "instruction" in kwargs: + end_punct_list = [ + "?" if query.strip()[-1] not in ["?", ".", "!"] else "" + for query in queries + ] + queries = [ + f"{query}{end_punct_list[i]} {kwargs['instruction']}" + for i, query in enumerate(queries) + ] + return self.encode(queries, **kwargs) + + +def _loader(wrapper: type[PromptrieverWrapper], **kwargs) -> Callable[..., Encoder]: + _kwargs = kwargs + + def loader_inner(**kwargs: Any) -> Encoder: + return wrapper(**_kwargs, **kwargs) + + return loader_inner + + +promptriever_llama2 = ModelMeta( + loader=_loader( + RepLLaMAWrapper, + base_model_name_or_path="meta-llama/Llama-2-7b-hf", + peft_model_name_or_path="samaya-ai/promptriever-llama2-7b-v1", + device_map="auto", + torch_dtype=torch.bfloat16, + ), + name="samaya-ai/promptriever-llama2-7b-v1", + languages=["eng_Latn"], + open_weights=True, + revision="01c7f73d771dfac7d292323805ebc428287df4f9-30b14e3813c0fa45facfd01a594580c3fe5ecf23", # base-peft revision + release_date="2024-09-15", + n_parameters=7_000_000, + memory_usage=None, + max_tokens=4096, + embed_dim=4096, + license="apache-2.0", + reference="https://huggingface.co/samaya-ai/promptriever-llama2-7b-v1", + similarity_fn_name="cosine", + framework=["PyTorch", "Tevatron"], + use_instuctions=True, +) + +promptriever_llama3 = ModelMeta( + loader=_loader( + RepLLaMAWrapper, + base_model_name_or_path="meta-llama/Meta-Llama-3.1-8B", + peft_model_name_or_path="samaya-ai/promptriever-llama3.1-8b-v1", + device_map="auto", + torch_dtype=torch.bfloat16, + ), + name="samaya-ai/promptriever-llama3.1-8b-v1", + languages=["eng_Latn"], + open_weights=True, + revision="48d6d0fc4e02fb1269b36940650a1b7233035cbb-2ead22cfb1b0e0c519c371c63c2ab90ffc511b8a", # base-peft revision + release_date="2024-09-15", + n_parameters=8_000_000, + memory_usage=None, + max_tokens=8192, + embed_dim=4096, + license="apache-2.0", + reference="https://huggingface.co/samaya-ai/promptriever-llama3.1-8b-v1", + similarity_fn_name="cosine", + framework=["PyTorch", "Tevatron"], + use_instuctions=True, +) + + +promptriever_llama3_instruct = ModelMeta( + loader=_loader( + RepLLaMAWrapper, + base_model_name_or_path="meta-llama/Meta-Llama-3.1-8B-Instruct", + peft_model_name_or_path="samaya-ai/promptriever-llama3.1-8b-instruct-v1", + device_map="auto", + torch_dtype=torch.bfloat16, + ), + name="samaya-ai/promptriever-llama3.1-8b-instruct-v1", + languages=["eng_Latn"], + open_weights=True, + revision="5206a32e0bd3067aef1ce90f5528ade7d866253f-8b677258615625122c2eb7329292b8c402612c21", # base-peft revision + release_date="2024-09-15", + n_parameters=8_000_000, + memory_usage=None, + max_tokens=8192, + embed_dim=4096, + license="apache-2.0", + reference="https://huggingface.co/samaya-ai/promptriever-llama3.1-8b-instruct-v1", + similarity_fn_name="cosine", + framework=["PyTorch", "Tevatron"], + use_instuctions=True, +) + +promptriever_mistral_v1 = ModelMeta( + loader=_loader( + RepLLaMAWrapper, + base_model_name_or_path="mistralai/Mistral-7B-v0.1", + peft_model_name_or_path="samaya-ai/promptriever-mistral-v0.1-7b-v1", + device_map="auto", + torch_dtype=torch.bfloat16, + ), + name="samaya-ai/promptriever-mistral-v0.1-7b-v1", + languages=["eng_Latn"], + open_weights=True, + revision="7231864981174d9bee8c7687c24c8344414eae6b-876d63e49b6115ecb6839893a56298fadee7e8f5", # base-peft revision + release_date="2024-09-15", + n_parameters=7_000_000, + memory_usage=None, + max_tokens=4096, + embed_dim=4096, + license="apache-2.0", + reference="https://huggingface.co/samaya-ai/promptriever-mistral-v0.1-7b-v1", + similarity_fn_name="cosine", + framework=["PyTorch", "Tevatron"], + use_instuctions=True, +) diff --git a/mteb/models/repllama_models.py b/mteb/models/repllama_models.py new file mode 100644 index 0000000000..6f2f93f169 --- /dev/null +++ b/mteb/models/repllama_models.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import logging +from typing import Any, Callable + +import numpy as np +import torch +import torch.nn.functional as F +import tqdm +from transformers import AutoModel, AutoTokenizer + +from mteb.encoder_interface import Encoder, PromptType +from mteb.model_meta import ModelMeta +from mteb.models.sentence_transformer_wrapper import ( + get_prompt_name, + validate_task_to_prompt_name, +) + +from .wrapper import Wrapper + +logger = logging.getLogger(__name__) + + +class RepLLaMAWrapper(Wrapper): + def __init__( + self, + base_model_name_or_path: str, + peft_model_name_or_path: str, + torch_dtype: torch.dtype, + device_map: str, + model_prompts: dict[str, str] | None = None, + **kwargs, + ): + try: + from peft import PeftModel + except ImportError: + raise ImportError( + "To use the RepLLaMA based models `peft` is required. Please install it with `pip install 'mteb[peft]'`." + ) + + self.base_model = AutoModel.from_pretrained( + base_model_name_or_path, + torch_dtype=torch_dtype, + device_map=device_map, + ) + self.model = PeftModel.from_pretrained(self.base_model, peft_model_name_or_path) + self.model = self.model.merge_and_unload() + + self.tokenizer = AutoTokenizer.from_pretrained(base_model_name_or_path) + self.tokenizer.pad_token_id = self.tokenizer.eos_token_id + self.tokenizer.pad_token = self.tokenizer.eos_token + self.tokenizer.padding_side = "right" + # set the max_length for the evals as they did, although the model can handle longer + self.model.config.max_length = 512 + self.tokenizer.model_max_length = 512 + self.model_prompts = ( + validate_task_to_prompt_name(model_prompts) if model_prompts else None + ) + + def create_batch_dict(self, tokenizer, input_texts): + max_length = self.model.config.max_length + batch_dict = tokenizer( + input_texts, + max_length=max_length - 1, + return_token_type_ids=False, + return_attention_mask=False, + padding=False, + truncation=True, + ) + batch_dict["input_ids"] = [ + input_ids + [tokenizer.eos_token_id] + for input_ids in batch_dict["input_ids"] + ] + return tokenizer.pad( + batch_dict, + padding=True, + pad_to_multiple_of=8, + return_attention_mask=True, + return_tensors="pt", + ) + + def encode( + self, + sentences: list[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, # noqa + ) -> np.ndarray: + batch_size = 16 if "batch_size" not in kwargs else kwargs.pop("batch_size") + all_embeddings = [] + prompt = get_prompt_name(self.model_prompts, task_name, prompt_type) + if prompt: + sentences = [f"{prompt}{sentence}".strip() for sentence in sentences] + for i in tqdm.tqdm(range(0, len(sentences), batch_size)): + batch_texts = sentences[i : i + batch_size] + + batch_dict = self.create_batch_dict(self.tokenizer, batch_texts) + batch_dict = { + key: value.to(self.model.device) for key, value in batch_dict.items() + } + + with torch.cuda.amp.autocast(): + with torch.no_grad(): + outputs = self.model(**batch_dict) + last_hidden_state = outputs.last_hidden_state + sequence_lengths = batch_dict["attention_mask"].sum(dim=1) - 1 + batch_size = last_hidden_state.shape[0] + reps = last_hidden_state[ + torch.arange(batch_size, device=last_hidden_state.device), + sequence_lengths, + ] + embeddings = F.normalize(reps, p=2, dim=-1) + all_embeddings.append(embeddings.cpu().numpy()) + + return np.concatenate(all_embeddings, axis=0) + + +def _loader(wrapper: type[RepLLaMAWrapper], **kwargs) -> Callable[..., Encoder]: + _kwargs = kwargs + + def loader_inner(**kwargs: Any) -> Encoder: + return wrapper(**_kwargs, **kwargs) + + return loader_inner + + +model_prompts = { + PromptType.query.value: "query: ", + PromptType.passage.value: "passage: ", +} + +repllama_llama2_original = ModelMeta( + loader=_loader( + RepLLaMAWrapper, + base_model_name_or_path="meta-llama/Llama-2-7b-hf", + peft_model_name_or_path="castorini/repllama-v1-7b-lora-passage", + device_map="auto", + torch_dtype=torch.bfloat16, + model_prompts=model_prompts, + ), + name="castorini/repllama-v1-7b-lora-passage", + languages=["eng_Latn"], + open_weights=True, + revision="01c7f73d771dfac7d292323805ebc428287df4f9-6097554dfe6e7d93e92f55010b678bcca1e233a8", # base-peft revision + release_date="2023-10-11", + n_parameters=7_000_000, + memory_usage=None, + max_tokens=4096, + embed_dim=4096, + license="apache-2.0", + reference="https://huggingface.co/samaya-ai/castorini/repllama-v1-7b-lora-passage", + similarity_fn_name="cosine", + framework=["PyTorch", "Tevatron"], + use_instuctions=True, +) + + +repllama_llama2_reproduced = ModelMeta( + loader=_loader( + RepLLaMAWrapper, + base_model_name_or_path="meta-llama/Llama-2-7b-hf", + peft_model_name_or_path="samaya-ai/RepLLaMA-reproduced", + device_map="auto", + torch_dtype=torch.bfloat16, + model_prompts=model_prompts, + ), + name="samaya-ai/RepLLaMA-reproduced", + languages=["eng_Latn"], + open_weights=True, + revision="01c7f73d771dfac7d292323805ebc428287df4f9-ad5c1d0938a1e02954bcafb4d811ba2f34052e71", # base-peft revision + release_date="2024-09-15", + n_parameters=7_000_000, + memory_usage=None, + max_tokens=4096, + embed_dim=4096, + license="apache-2.0", + reference="https://huggingface.co/samaya-ai/RepLLaMA-reproduced", + similarity_fn_name="cosine", + framework=["PyTorch", "Tevatron"], + use_instuctions=True, +) diff --git a/mteb/models/rerankers_custom.py b/mteb/models/rerankers_custom.py new file mode 100644 index 0000000000..ded6b0be5a --- /dev/null +++ b/mteb/models/rerankers_custom.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import logging +from functools import partial +from typing import Any, Callable + +import torch +from sentence_transformers import CrossEncoder +from transformers import AutoModelForSequenceClassification, AutoTokenizer + +from mteb.encoder_interface import Encoder +from mteb.evaluation.evaluators.RetrievalEvaluator import DenseRetrievalExactSearch +from mteb.model_meta import ModelMeta + +logger = logging.getLogger(__name__) + + +class RerankerWrapper(DenseRetrievalExactSearch): + def __init__( + self, + model_name_or_path: str, + batch_size: int = 4, + fp_options: bool = None, + silent: bool = False, + ): + self.model_name_or_path = model_name_or_path + self.batch_size = batch_size + self.fp_options = fp_options if fp_options is not None else torch.float32 + if self.fp_options == "auto": + self.fp_options = torch.float32 + elif self.fp_options == "float16": + self.fp_options = torch.float16 + elif self.fp_options == "float32": + self.fp_options = torch.float32 + elif self.fp_options == "bfloat16": + self.fp_options = torch.bfloat16 + print(f"Using fp_options of {self.fp_options}") + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.silent = silent + self.first_print = True # for debugging + + def predict(self, input_to_rerank, **kwargs) -> list: + pass + + +class BGEReranker(RerankerWrapper): + name: str = "BGE" + + def __init__( + self, + model_name_or_path="BAAI/bge-reranker-v2-m3", + torch_compile=False, + **kwargs, + ): + super().__init__(model_name_or_path, **kwargs) + if not self.device: + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model_args = {} + if self.fp_options: + model_args["torch_dtype"] = self.fp_options + + try: + from FlagEmbedding import FlagReranker + except ImportError: + raise ImportError( + "FlagEmbedding is not installed. Please install it via `pip install mteb[flagembedding]`" + ) + + self.model = FlagReranker(model_name_or_path, use_fp16=True) + + @torch.inference_mode() + def predict(self, input_to_rerank, **kwargs): + queries, passages, instructions = list(zip(*input_to_rerank)) + if instructions is not None and instructions[0] is not None: + assert len(instructions) == len(queries) + queries = [f"{q} {i}".strip() for i, q in zip(instructions, queries)] + + assert len(queries) == len(passages) + query_passage_tuples = list(zip(queries, passages)) + scores = self.model.compute_score(query_passage_tuples, normalize=True) + assert len(scores) == len( + queries + ), f"Expected {len(queries)} scores, got {len(scores)}" + return scores + + +class MonoBERTReranker(RerankerWrapper): + name: str = "MonoBERT" + + def __init__( + self, + model_name_or_path="castorini/monobert-large-msmarco", + torch_compile=False, + **kwargs, + ): + super().__init__(model_name_or_path, **kwargs) + if not self.device: + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model_args = {} + if self.fp_options: + model_args["torch_dtype"] = self.fp_options + self.model = AutoModelForSequenceClassification.from_pretrained( + model_name_or_path, + **model_args, + ) + self.model.to(self.device) + self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) + self.max_length = self.tokenizer.model_max_length + logger.info(f"Using max_length of {self.max_length}") + + self.model.eval() + + @torch.inference_mode() + def predict(self, input_to_rerank, **kwargs): + queries, passages, instructions = list(zip(*input_to_rerank)) + if instructions is not None and instructions[0] is not None: + queries = [f"{q} {i}".strip() for i, q in zip(instructions, queries)] + + tokens = self.tokenizer( + queries, + passages, + padding=True, + truncation="only_second", + return_tensors="pt", + max_length=self.max_length, + ).to(self.device) + output = self.model(**tokens)[0] + batch_scores = torch.nn.functional.log_softmax(output, dim=1) + return batch_scores[:, 1].exp().tolist() + + +class JinaReranker(RerankerWrapper): + name = "Jina" + + def __init__( + self, + model_name_or_path="jinaai/jina-reranker-v2-base-multilingual", + torch_compile=False, + **kwargs, + ): + super().__init__(model_name_or_path, **kwargs) + if not self.device: + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model_args = {} + if self.fp_options: + model_args["torch_dtype"] = self.fp_options + + self.model = CrossEncoder( + model_name_or_path, + automodel_args={"torch_dtype": "auto"}, + trust_remote_code=True, + ) + + def predict(self, input_to_rerank, **kwargs): + queries, passages, instructions = list(zip(*input_to_rerank)) + if instructions is not None and instructions[0] is not None: + queries = [f"{q} {i}".strip() for i, q in zip(instructions, queries)] + + if self.first_print: + logger.info(f"Using {queries[0]}") + self.first_print = False + + sentence_pairs = list(zip(queries, passages)) + scores = self.model.predict(sentence_pairs, convert_to_tensor=True).tolist() + return scores + + +def _loader(wrapper: type[RerankerWrapper], **kwargs) -> Callable[..., Encoder]: + _kwargs = kwargs + + def loader_inner(**kwargs: Any) -> Encoder: + return wrapper(**_kwargs, **kwargs) + + return loader_inner() + + +monobert_large = ModelMeta( + loader=partial( + _loader, + wrapper=MonoBERTReranker, + model_name_or_path="castorini/monobert-large-msmarco", + fp_options="float1616", + ), + name="castorini/monobert-large-msmarco", + languages=["eng_Latn"], + open_source=True, + revision="0a97706f3827389da43b83348d5d18c9d53876fa", + release_date="2020-05-28", +) + +# languages unclear: https://huggingface.co/jinaai/jina-reranker-v2-base-multilingual/discussions/28 +jina_reranker_multilingual = ModelMeta( + loader=partial( + _loader, + wrapper=JinaReranker, + model_name_or_path="jinaai/jina-reranker-v2-base-multilingual", + fp_options="float1616", + ), + name="jinaai/jina-reranker-v2-base-multilingual", + languages=["eng_Latn"], + open_source=True, + revision="126747772a932960028d9f4dc93bd5d9c4869be4", + release_date="2024-09-26", +) + +bge_reranker_v2_m3 = ModelMeta( + loader=partial( + _loader, + wrapper=BGEReranker, + model_name_or_path="BAAI/bge-reranker-v2-m3", + fp_options="float1616", + ), + name="BAAI/bge-reranker-v2-m3", + languages=[ + "eng_Latn", + "ara_Arab", + "ben_Beng", + "spa_Latn", + "fas_Arab", + "fin_Latn", + "fra_Latn", + "hin_Deva", + "ind_Latn", + "jpn_Jpan", + "kor_Hang", + "rus_Cyrl", + "swa_Latn", + "tel_Telu", + "tha_Thai", + "zho_Hans", + "deu_Latn", + "yor_Latn", + "dan_Latn", + "heb_Hebr", + "hun_Latn", + "ita_Latn", + "khm_Khmr", + "msa_Latn", + "nld_Latn", + "nob_Latn", + "pol_Latn", + "por_Latn", + "swe_Latn", + "tur_Latn", + "vie_Latn", + "zho_Hant", + ], + open_source=True, + revision="953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e", + release_date="2024-06-24", +) diff --git a/mteb/models/rerankers_monot5_based.py b/mteb/models/rerankers_monot5_based.py new file mode 100644 index 0000000000..cd54bbd5cf --- /dev/null +++ b/mteb/models/rerankers_monot5_based.py @@ -0,0 +1,578 @@ +from __future__ import annotations + +import logging +from functools import partial + +import torch +from transformers import ( + AutoModelForCausalLM, + AutoModelForSeq2SeqLM, + AutoTokenizer, +) + +from mteb.model_meta import ModelMeta +from mteb.models.rerankers_custom import RerankerWrapper, _loader + +logger = logging.getLogger(__name__) + + +# Based on https://github.com/castorini/pygaggle/blob/f54ae53d6183c1b66444fa5a0542301e0d1090f5/pygaggle/rerank/base.py#L63 +prediction_tokens = { + "castorini/monot5-small-msmarco-10k": ["▁false", "▁true"], + "castorini/monot5-small-msmarco-100k": ["▁false", "▁true"], + "castorini/monot5-base-msmarco": ["▁false", "▁true"], + "castorini/monot5-base-msmarco-10k": ["▁false", "▁true"], + "castorini/monot5-large-msmarco": ["▁false", "▁true"], + "castorini/monot5-large-msmarco-10k": ["▁false", "▁true"], + "castorini/monot5-base-med-msmarco": ["▁false", "▁true"], + "castorini/monot5-3b-med-msmarco": ["▁false", "▁true"], + "castorini/monot5-3b-msmarco-10k": ["▁false", "▁true"], + "castorini/monot5-3b-msmarco": ["▁false", "▁true"], + "unicamp-dl/mt5-base-en-msmarco": ["▁no", "▁yes"], + "unicamp-dl/mt5-base-mmarco-v2": ["▁no", "▁yes"], + "unicamp-dl/mt5-base-mmarco-v1": ["▁no", "▁yes"], + "unicamp-dl/mt5-13b-mmarco-100k": ["▁", "▁true"], +} + + +def chunks(lst, n): + for i in range(0, len(lst), n): + yield lst[i : i + n] + + +class MonoT5Reranker(RerankerWrapper): + name: str = "MonoT5" + prompt_template: str = "Query: {query} Document: {text} Relevant:" + + def __init__( + self, + model_name_or_path="castorini/monot5-base-msmarco-10k", + **kwargs, + ): + super().__init__(model_name_or_path, **kwargs) + if not self.device: + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model_args = {} + if self.fp_options: + model_args["torch_dtype"] = self.fp_options + self.model = AutoModelForSeq2SeqLM.from_pretrained( + model_name_or_path, **model_args + ) + logger.info(f"Using model {model_name_or_path}") + + if "torch_compile" in kwargs and kwargs["torch_compile"]: + self.torch_compile = kwargs["torch_compile"] + self.model = torch.compile(self.model) + else: + self.torch_compile = False + + self.model.to(self.device) + self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) + self.token_false_id, self.token_true_id = self.get_prediction_tokens( + model_name_or_path, + self.tokenizer, + kwargs["token_false"] if "token_false" in kwargs else None, + kwargs["token_true"] if "token_true" in kwargs else None, + ) + logger.info(f"Using max_length of {self.tokenizer.model_max_length}") + logger.info(f"Using token_false_id of {self.token_false_id}") + logger.info(f"Using token_true_id of {self.token_true_id}") + self.max_length = min( + 2048, self.tokenizer.model_max_length + ) # sometimes it's a v large number/max int + logger.info(f"Using max_length of {self.max_length}") + + self.model.eval() + + def get_prediction_tokens( + self, model_name_or_path, tokenizer, token_false=None, token_true=None + ): + if not (token_false and token_true): + if model_name_or_path in prediction_tokens: + token_false, token_true = prediction_tokens[model_name_or_path] + token_false_id = tokenizer.get_vocab()[token_false] + token_true_id = tokenizer.get_vocab()[token_true] + return token_false_id, token_true_id + else: + raise Exception(f"We don't know the indexes for the non-relevant/relevant tokens for\ + the checkpoint {model_name_or_path} and you did not provide any.") + else: + token_false_id = tokenizer.get_vocab()[token_false] + token_true_id = tokenizer.get_vocab()[token_true] + return token_false_id, token_true_id + + @torch.inference_mode() + def predict(self, input_to_rerank, **kwargs): + queries, passages, instructions = list(zip(*input_to_rerank)) + + if instructions is not None and instructions[0] is not None: + queries = [f"{q} {i}".strip() for i, q in zip(instructions, queries)] + + prompts = [ + self.prompt_template.format(query=query, text=text) + for (query, text) in zip(queries, passages) + ] + + tokens = self.tokenizer( + prompts, + padding=True, + truncation=True, + return_tensors="pt", + max_length=self.max_length, + pad_to_multiple_of=(8 if self.torch_compile else None), + ).to(self.device) + output = self.model.generate( + **tokens, + max_new_tokens=1, + return_dict_in_generate=True, + output_scores=True, + ) + batch_scores = output.scores[0] + batch_scores = batch_scores[:, [self.token_false_id, self.token_true_id]] + batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1) + return batch_scores[:, 1].exp().tolist() + + +class LlamaReranker(RerankerWrapper): + name: str = "LLAMA-Based" + + def __init__( + self, model_name_or_path: str, is_classification: bool = False, **kwargs + ): + if "torch_compile" in kwargs: + del kwargs["torch_compile"] + super().__init__(model_name_or_path, **kwargs) + + if "chat" in model_name_or_path: + self.template = """[INST] <> +You are an expert at finding information. Determine if the following document is relevant to the query (true/false). +<>Query: {query} +Document: {text} +Relevant: [/INST]""" + else: + self.template = """Determine if the following document is relevant to the query (true/false). + +Query: {query} +Document: {text} +Relevant: """ + + self.query_instruct_template = "{query} {instruction}" + logger.info(f"Using query_instruct_template of {self.query_instruct_template}") + self.is_classification = is_classification + + model_args = {} + if self.fp_options: + model_args["torch_dtype"] = self.fp_options + + logger.info(self.template) + logger.info(model_name_or_path) + if not self.device: + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + self.model = AutoModelForCausalLM.from_pretrained( + model_name_or_path, **model_args + ) + self.model.to(self.device) + + self.tokenizer = AutoTokenizer.from_pretrained( + model_name_or_path, padding_side="left" + ) + self.tokenizer.pad_token = self.tokenizer.eos_token + self.tokenizer.padding_side = "left" + + self.token_false_id = self.tokenizer.get_vocab()["false"] + self.token_true_id = self.tokenizer.get_vocab()["true"] + self.max_length = min(2048, self.tokenizer.model_max_length) + logger.info(f"Using max_length of {self.max_length}") + self.gpu_count = torch.cuda.device_count() + if self.gpu_count > 1: + logger.info(f"Using {self.gpu_count} GPUs") + self.model = torch.nn.DataParallel(self.model) + self.model.eval() + + @torch.inference_mode() + def predict(self, input_to_rerank, **kwargs): + queries, passages, instructions = list(zip(*input_to_rerank)) + if instructions is not None and instructions[0] is not None: + # logger.info(f"Adding instructions to LLAMA queries") + queries = [ + self.query_instruct_template.format(instruction=i, query=q).strip() + for i, q in zip(instructions, queries) + ] + + prompts = [ + self.template.format(query=query, text=text) + for (query, text) in zip(queries, passages) + ] + assert "{query}" not in prompts[0], "Query not replaced" + assert "{text}" not in prompts[0], "Text not replaced" + assert "{instruction}" not in prompts[0], "Instruction not replaced" + + tokens = self.tokenizer( + prompts, + padding=True, + truncation=True, + return_tensors="pt", + max_length=self.max_length, + pad_to_multiple_of=None, + ).to(self.device) + if "token_type_ids" in tokens: + del tokens["token_type_ids"] + if not self.is_classification: + batch_scores = self.model(**tokens).logits[:, -1, :] + true_vector = batch_scores[:, self.token_true_id] + false_vector = batch_scores[:, self.token_false_id] + batch_scores = torch.stack([false_vector, true_vector], dim=1) + batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1) + scores = batch_scores[:, 1].exp().tolist() + else: + batch_scores = self.model(**tokens).logits + batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1) + scores = batch_scores[:, 1].exp().tolist() + + return scores + + +class MistralReranker(LlamaReranker): + name: str = "Mistral" + + def __init__(self, model_name_or_path: str, **kwargs): + # use the base class for everything except template + super().__init__(model_name_or_path, **kwargs) + self.template = """[INST] You are an expert Google searcher, whose job is to determine if the following document is relevant to the query (true/false). +Query: {query} +Document: {text} +Relevant (either "true" or "false"): [/INST]""" + self.max_length = min(2048, self.tokenizer.model_max_length) + logger.info(f"Using max_length of {self.max_length}") + logger.info(f"Using template of {self.template}") + + +class FollowIRReranker(LlamaReranker): + name: str = "FollowIR" + + def __init__(self, model_name_or_path: str, **kwargs): + # use the base class for everything except template + super().__init__(model_name_or_path, **kwargs) + self.template = """ [INST] You are an expert Google searcher, whose job is to determine if the following document is relevant to the query (true/false). Answer using only one word, one of those two choices. + +Query: {query} +Document: {text} +Relevant (only output one word, either "true" or "false"): [/INST] """ + self.max_length = min(2048, self.tokenizer.model_max_length) + logger.info(f"Using template of {self.template}") + + +class FLANT5Reranker(MonoT5Reranker): + name: str = "FLAN-T5" + prompt_template: str = """Is the following passage relevant to the query? +Query: {query} +Passage: {text}""" + + def get_prediction_tokens(self, *args, **kwargs): + yes_token_id, *_ = self.tokenizer.encode("yes") + no_token_id, *_ = self.tokenizer.encode("no") + return no_token_id, yes_token_id + + +monot5_small = ModelMeta( + loader=partial( + _loader, + wrapper=MonoT5Reranker, + model_name_or_path="castorini/monot5-small-msmarco-10k", + fp_options="float16", + ), + name="castorini/monot5-small-msmarco-10k", + languages=["eng_Latn"], + open_source=True, + revision="77f8e3f7b1eb1afe353aa21a7c3a2fc8feca702e", + release_date="2022-03-28", +) + +monot5_base = ModelMeta( + loader=partial( + _loader, + wrapper=MonoT5Reranker, + model_name_or_path="castorini/monot5-base-msmarco-10k", + fp_options="float16", + ), + name="castorini/monot5-base-msmarco-10k", + languages=["eng_Latn"], + open_source=True, + revision="f15657ab3d2a5dd0b9a30c8c0b6a0a73c9cb5884", + release_date="2022-03-28", +) + +monot5_large = ModelMeta( + loader=partial( + _loader, + wrapper=MonoT5Reranker, + model_name_or_path="castorini/monot5-large-msmarco-10k", + fp_options="float16", + ), + name="castorini/monot5-large-msmarco-10k", + languages=["eng_Latn"], + open_source=True, + revision="48cfad1d8dd587670393f27ee8ec41fde63e3d98", + release_date="2022-03-28", +) + +monot5_3b = ModelMeta( + loader=partial( + _loader, + wrapper=MonoT5Reranker, + model_name_or_path="castorini/monot5-3b-msmarco-10k", + fp_options="float16", + ), + name="castorini/monot5-3b-msmarco-10k", + languages=["eng_Latn"], + open_source=True, + revision="bc0c419a438c81f592f878ce32430a1823f5db6c", + release_date="2022-03-28", +) + +flant5_base = ModelMeta( + loader=partial( + _loader, + wrapper=FLANT5Reranker, + model_name_or_path="google/flan-t5-base", + fp_options="float16", + ), + name="google/flan-t5-base", + languages=["eng_Latn"], + open_source=True, + revision="7bcac572ce56db69c1ea7c8af255c5d7c9672fc2", + release_date="2022-10-21", +) + +flant5_large = ModelMeta( + loader=partial( + _loader, + wrapper=FLANT5Reranker, + model_name_or_path="google/flan-t5-large", + fp_options="float16", + ), + name="google/flan-t5-large", + languages=["eng_Latn"], + open_source=True, + revision="0613663d0d48ea86ba8cb3d7a44f0f65dc596a2a", + release_date="2022-10-21", +) + +flant5_xl = ModelMeta( + loader=partial( + _loader, + wrapper=FLANT5Reranker, + model_name_or_path="google/flan-t5-xl", + fp_options="float16", + ), + name="google/flan-t5-xl", + languages=["eng_Latn"], + open_source=True, + revision="7d6315df2c2fb742f0f5b556879d730926ca9001", + release_date="2022-10-21", +) + +flant5_xxl = ModelMeta( + loader=partial( + _loader, + wrapper=FLANT5Reranker, + model_name_or_path="google/flan-t5-xxl", + fp_options="float16", + ), + name="google/flan-t5-xxl", + languages=["eng_Latn"], + open_source=True, + revision="ae7c9136adc7555eeccc78cdd960dfd60fb346ce", + release_date="2022-10-21", +) + + +llama2_7b = ModelMeta( + loader=partial( + _loader, + wrapper=LlamaReranker, + model_name_or_path="meta-llama/Llama-2-7b-hf", + fp_options="float16", + ), + name="meta-llama/Llama-2-7b-hf", + languages=["eng_Latn"], + open_source=True, + revision="01c7f73d771dfac7d292323805ebc428287df4f9", + release_date="2023-07-18", +) + +llama2_7b_chat = ModelMeta( + loader=partial( + _loader, + wrapper=LlamaReranker, + model_name_or_path="meta-llama/Llama-2-7b-chat-hf", + fp_options="float16", + ), + name="meta-llama/Llama-2-7b-chat-hf", + languages=["eng_Latn"], + open_source=True, + revision="f5db02db724555f92da89c216ac04704f23d4590", + release_date="2023-07-18", +) + +mistral_7b = ModelMeta( + loader=partial( + _loader, + wrapper=MistralReranker, + model_name_or_path="mistralai/Mistral-7B-Instruct-v0.2", + fp_options="float16", + ), + name="mistralai/Mistral-7B-Instruct-v0.2", + languages=["eng_Latn"], + open_source=True, + revision="3ad372fc79158a2148299e3318516c786aeded6c", + release_date="2023-12-11", +) + +followir_7b = ModelMeta( + loader=partial( + _loader, + wrapper=FollowIRReranker, + model_name_or_path="jhu-clsp/FollowIR-7B", + fp_options="float16", + ), + name="jhu-clsp/FollowIR-7B", + languages=["eng_Latn"], + open_source=True, + revision="4d25d437e38b510c01852070c0731e8f6e1875d1", + release_date="2024-04-29", +) + + +mt5_languages = [ + "afr_Latn", + "sqi_Latn", + "amh_Ethi", + "ara_Arab", + "hye_Armn", + "aze_Latn", + "eus_Latn", + "bel_Cyrl", + "ben_Beng", + "bul_Cyrl", + "mya_Mymr", + "cat_Latn", + "ceb_Latn", + "nya_Latn", + "zho_Hans", + "cos_Latn", + "ces_Latn", + "dan_Latn", + "nld_Latn", + "eng_Latn", + "epo_Latn", + "est_Latn", + "fil_Latn", + "fin_Latn", + "fra_Latn", + "glg_Latn", + "kat_Geor", + "deu_Latn", + "ell_Grek", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "haw_Latn", + "heb_Hebr", + "hin_Deva", + "hmn_Latn", + "hun_Latn", + "isl_Latn", + "ibo_Latn", + "ind_Latn", + "gle_Latn", + "ita_Latn", + "jpn_Jpan", + "jav_Latn", + "kan_Knda", + "kaz_Cyrl", + "khm_Khmr", + "kor_Hang", + "kur_Latn", + "kir_Cyrl", + "lao_Laoo", + "lat_Latn", + "lav_Latn", + "lit_Latn", + "ltz_Latn", + "mkd_Cyrl", + "mlg_Latn", + "msa_Latn", + "mal_Mlym", + "mlt_Latn", + "mri_Latn", + "mar_Deva", + "mon_Cyrl", + "nep_Deva", + "nor_Latn", + "pus_Arab", + "fas_Arab", + "pol_Latn", + "por_Latn", + "pan_Guru", + "ron_Latn", + "rus_Cyrl", + "smo_Latn", + "gla_Latn", + "srp_Cyrl", + "sna_Latn", + "snd_Arab", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "som_Latn", + "sot_Latn", + "spa_Latn", + "sun_Latn", + "swa_Latn", + "swe_Latn", + "tgk_Cyrl", + "tam_Taml", + "tel_Telu", + "tha_Thai", + "tur_Latn", + "ukr_Cyrl", + "urd_Arab", + "uzb_Latn", + "vie_Latn", + "cym_Latn", + "fry_Latn", + "xho_Latn", + "yid_Hebr", + "yor_Latn", + "zul_Latn", +] + +mt5_base_mmarco_v2 = ModelMeta( + loader=partial( + _loader, + wrapper=MonoT5Reranker, + model_name_or_path="unicamp-dl/mt5-base-mmarco-v2", + fp_options="float16", + ), + name="unicamp-dl/mt5-base-mmarco-v2", + languages=mt5_languages, + open_source=True, + revision="cc0a949b9f21efcaba45c8cabb998ad02ce8d4e7", + release_date="2022-01-05", +) + +mt5_13b_mmarco_100k = ModelMeta( + loader=partial( + _loader, + wrapper=MonoT5Reranker, + model_name_or_path="unicamp-dl/mt5-13b-mmarco-100k", + fp_options="float16", + ), + name="unicamp-dl/mt5-13b-mmarco-100k", + languages=mt5_languages, + open_source=True, + revision="e1a4317e102a525ea9e16745ad21394a4f1bffbc", + release_date="2022-11-04", +) diff --git a/mteb/models/ru_sentence_models.py b/mteb/models/ru_sentence_models.py index 30214c21f2..1328005a33 100644 --- a/mteb/models/ru_sentence_models.py +++ b/mteb/models/ru_sentence_models.py @@ -4,103 +4,234 @@ from functools import partial -from mteb.model_meta import ModelMeta - -from .e5_models import E5Wrapper +from mteb.model_meta import ModelMeta, sentence_transformers_loader rubert_tiny2 = ModelMeta( name="cointegrated/rubert-tiny2", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="dad72b8f77c5eef6995dd3e4691b758ba56b90c3", release_date="2021-10-28", + n_parameters=29_400_000, + memory_usage=None, + embed_dim=312, + license="mit", + max_tokens=2048, + reference="https://huggingface.co/cointegrated/rubert-tiny2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) rubert_tiny = ModelMeta( name="cointegrated/rubert-tiny", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="5441c5ea8026d4f6d7505ec004845409f1259fb1", release_date="2021-05-24", + n_parameters=29_400_000, + memory_usage=None, + embed_dim=312, + license="mit", + max_tokens=2048, + reference="https://huggingface.co/cointegrated/rubert-tiny", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) sbert_large_nlu_ru = ModelMeta( name="ai-forever/sbert_large_nlu_ru", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="af977d5dfa46a3635e29bf0ef383f2df2a08d47a", release_date="2020-11-20", + n_parameters=427_000_000, + memory_usage=None, + embed_dim=1024, + license="mit", + max_tokens=512, # best guess + reference="https://huggingface.co/ai-forever/sbert_large_nlu_ru", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) sbert_large_mt_nlu_ru = ModelMeta( name="ai-forever/sbert_large_mt_nlu_ru", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="05300876c2b83f46d3ddd422a7f17e45cf633bb0", release_date="2021-05-18", + n_parameters=427_000_000, + memory_usage=None, + embed_dim=1024, + license="Not specified", + max_tokens=512, # best guess + reference="https://huggingface.co/ai-forever/sbert_large_mt_nlu_ru", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) user_base_ru = ModelMeta( - loader=partial(E5Wrapper, model_name="deepvk/USER-base"), # type: ignore + loader=partial( + sentence_transformers_loader, + model_name="deepvk/USER-base", + revision="436a489a2087d61aa670b3496a9915f84e46c861", + prompts={"query": "query: ", "passage": "passage: "}, + ), name="deepvk/USER-base", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="436a489a2087d61aa670b3496a9915f84e46c861", release_date="2024-06-10", + n_parameters=427_000_000, + memory_usage=None, + embed_dim=1024, + license="Not specified", + max_tokens=512, # best guess + reference="https://huggingface.co/ai-forever/sbert_large_mt_nlu_ru", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) deberta_v1_ru = ModelMeta( name="deepvk/deberta-v1-base", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="bdd30b0e19757e6940c92c7aff19e8fc0a60dff4", release_date="2023-02-07", + n_parameters=124_000_000, + memory_usage=None, + embed_dim=768, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/deepvk/deberta-v1-base", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) rubert_base_cased = ModelMeta( name="DeepPavlov/rubert-base-cased", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="4036cab694767a299f2b9e6492909664d9414229", release_date="2020-03-04", + n_parameters=1280_000_000, + memory_usage=None, + embed_dim=768, + license="Not specified", + max_tokens=512, # best guess + reference="https://huggingface.co/DeepPavlov/rubert-base-cased", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) distilrubert_small_cased_conversational = ModelMeta( name="DeepPavlov/distilrubert-small-cased-conversational", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="e348066b4a7279b97138038299bddc6580a9169a", release_date="2022-06-28", + n_parameters=107_000_000, + memory_usage=None, + embed_dim=768, + license="Not specified", + max_tokens=512, + reference="https://huggingface.co/DeepPavlov/distilrubert-small-cased-conversational", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) rubert_base_cased_sentence = ModelMeta( name="DeepPavlov/rubert-base-cased-sentence", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="78b5122d6365337dd4114281b0d08cd1edbb3bc8", release_date="2020-03-04", + n_parameters=107_000_000, + memory_usage=None, + embed_dim=768, + license="Not specified", + max_tokens=512, + reference="https://huggingface.co/DeepPavlov/rubert-base-cased-sentence", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) labse_en_ru = ModelMeta( name="cointegrated/LaBSE-en-ru", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="cf0714e606d4af551e14ad69a7929cd6b0da7f7e", release_date="2021-06-10", + n_parameters=129_000_000, + memory_usage=None, + embed_dim=768, + license="Not specified", + max_tokens=512, + reference="https://huggingface.co/cointegrated/LaBSE-en-ru", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) rubert_tiny_turbo = ModelMeta( name="sergeyzh/rubert-tiny-turbo", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054", release_date="2024-06-21", + n_parameters=129_000_000, + memory_usage=None, + embed_dim=312, + license="mit", + max_tokens=512, + reference="https://huggingface.co/sergeyzh/rubert-tiny-turbo", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) labse_ru_turbo = ModelMeta( name="sergeyzh/LaBSE-ru-turbo", languages=["rus_Cyrl"], - open_source=True, + open_weights=True, revision="1940b046c6b5e125df11722b899130329d0a46da", release_date="2024-06-27", + n_parameters=129_000_000, + memory_usage=None, + embed_dim=312, + license="mit", + max_tokens=512, + reference="https://huggingface.co/sergeyzh/LaBSE-ru-turbo", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, +) + + +rosberta_ru_en = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="ai-forever/ru-en-RoSBERTa", + revision="89fb1651989adbb1cfcfdedafd7d102951ad0555", + prompts={ + "Classification": "classification: ", + "Clustering": "clustering: ", + "query": "search_query: ", + "passage": "search_document: ", + }, + ), + name="ai-forever/ru-en-RoSBERTa", + languages=["rus_Cyrl"], + open_weights=True, + revision="89fb1651989adbb1cfcfdedafd7d102951ad0555", + release_date="2024-07-29", ) diff --git a/mteb/models/salesforce_models.py b/mteb/models/salesforce_models.py index 1ce5700133..fe70f597ae 100644 --- a/mteb/models/salesforce_models.py +++ b/mteb/models/salesforce_models.py @@ -1,12 +1,17 @@ from __future__ import annotations +from collections.abc import Sequence from functools import partial +from typing import Any +import numpy as np import torch from mteb.model_meta import ModelMeta +from ..encoder_interface import PromptType from .instructions import task_to_instruction +from .wrapper import Wrapper def sfr_instruction(instruction: str) -> str: @@ -21,26 +26,25 @@ def sfr_loader(**kwargs): "Please install `pip install gritlm` to use SFR_Embedding_2_R." ) - class SFRWrapper(GritLM): - def encode(self, *args, **kwargs): - if "prompt_name" in kwargs: - if "instruction" in kwargs: - raise ValueError( - "Cannot specify both `prompt_name` and `instruction`." - ) + class SFRWrapper(GritLM, Wrapper): + def encode( + self, + sentences: Sequence[str], + *args, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + if "instruction" in kwargs: + instruction = kwargs.pop("instruction", "") + else: instruction = task_to_instruction( - kwargs.pop("prompt_name"), kwargs.pop("is_query", True) + task_name, prompt_type == PromptType.query ) - else: - instruction = kwargs.pop("instruction", "") if instruction: kwargs["instruction"] = sfr_instruction(instruction) return super().encode(*args, **kwargs) - def encode_corpus(self, *args, **kwargs): - kwargs["is_query"] = False - return super().encode_corpus(*args, **kwargs) - return SFRWrapper(**kwargs) @@ -58,13 +62,16 @@ def encode_corpus(self, *args, **kwargs): ), name="Salesforce/SFR-Embedding-2_R", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="91762139d94ed4371a9fa31db5551272e0b83818", release_date="2024-06-14", # initial commit of hf model. + n_parameters=7_110_000_000, + memory_usage=None, + embed_dim=4096, + license="cc-by-nc-4.0", + max_tokens=32768, + reference="https://huggingface.co/Salesforce/SFR-Embedding-2_R", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=True, ) - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(SFR_Embedding_2_R.name, SFR_Embedding_2_R.revision) - emb = mdl.encode(["Hello, world!"]) diff --git a/mteb/models/sentence_transformer_wrapper.py b/mteb/models/sentence_transformer_wrapper.py new file mode 100644 index 0000000000..13f274c186 --- /dev/null +++ b/mteb/models/sentence_transformer_wrapper.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import logging +from collections.abc import Sequence +from typing import Any, get_args + +import numpy as np +from sentence_transformers import CrossEncoder, SentenceTransformer + +import mteb +from mteb.abstasks.TaskMetadata import TASK_TYPE +from mteb.encoder_interface import PromptType + +from .wrapper import Wrapper + +logger = logging.getLogger(__name__) + + +class SentenceTransformerWrapper(Wrapper): + def __init__( + self, + model: str | SentenceTransformer | CrossEncoder, + revision: str | None = None, + model_prompts: dict[str, str] | None = None, + **kwargs, + ) -> None: + """Wrapper for SentenceTransformer models. + + Args: + model: The SentenceTransformer model to use. Can be a string (model name), a SentenceTransformer model, or a CrossEncoder model. + revision: The revision of the model to use. + model_prompts: A dictionary mapping task names to prompt names. + First priority is given to the composed prompt of task name + prompt type (query or passage), then to the specific task prompt, + then to the composed prompt of task type + prompt type, then to the specific task type prompt, + and finally to the specific prompt type. + **kwargs: Additional arguments to pass to the SentenceTransformer model. + """ + if isinstance(model, str): + self.model = SentenceTransformer( + model, revision=revision, trust_remote_code=True, **kwargs + ) + else: + self.model = model + + if ( + model_prompts is None + and hasattr(self.model, "prompts") + and len(self.model.prompts) > 0 + ): + try: + model_prompts = validate_task_to_prompt_name(self.model.prompts) + except ValueError: + model_prompts = None + elif model_prompts is not None and hasattr(self.model, "prompts"): + logger.info(f"Model prompts will be overwritten with {model_prompts}") + self.model.prompts = model_prompts + self.model_prompts = validate_task_to_prompt_name(model_prompts) + + if isinstance(self.model, CrossEncoder): + self.predict = self._predict + + def encode( + self, + sentences: Sequence[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + """Encodes the given sentences using the encoder. + + Args: + sentences: The sentences to encode. + task_name: The name of the task. Sentence-transformers uses this to + determine which prompt to use from a specified dictionary. + prompt_type: The name type of prompt. (query or passage) + **kwargs: Additional arguments to pass to the encoder. + + The order of priorities for prompt selection are: + 1. Composed prompt of task name + prompt type (query or passage) + 2. Specific task prompt + 3. Composed prompt of task type + prompt type (query or passage) + 4. Specific task type prompt + 5. Specific prompt type (query or passage) + + + Returns: + The encoded sentences. + """ + prompt_name = None + if self.model_prompts is not None: + prompt_name = get_prompt_name(self.model_prompts, task_name, prompt_type) + if prompt_name: + logger.info( + f"Using prompt_nane={prompt_name} for task={task_name} prompt_type={prompt_type}" + ) + else: + logger.info( + f"No model prompts found for task={task_name} prompt_type={prompt_type}" + ) + logger.info(f"Encoding {len(sentences)} sentences.") + + embeddings = self.model.encode( + sentences, + prompt_name=prompt_name, + **kwargs, + ) + return embeddings + + def _predict( + self, + sentences: Sequence[str], + **kwargs: Any, + ) -> np.ndarray: + return self.model.predict( + sentences, + convert_to_numpy=True, + **kwargs, + ) + + +def get_prompt_name( + task_to_prompt: dict[str, str] | None, + task_name: str, + prompt_type: PromptType | None, +) -> str | None: + """A wrapper function around the model.encode method that handles the prompt_name argument and standardizes the output to a numpy array. + The order of priorities for prompt selection are: + 1. Composed prompt of task name + prompt type (query or passage) + 2. Specific task prompt + 3. Composed prompt of task type + prompt type (query or passage) + 4. Specific task type prompt + 5. Specific prompt type (query or passage) + + + Args: + task_to_prompt: The tasks names and their corresponding prompt_names + task_name: The task name to use for building the encoding prompt + prompt_type: The prompt type (e.g. "query" | "passage") to use for building the encoding prompt + """ + import mteb + + task = mteb.get_task(task_name=task_name) + task_type = task.metadata.type + prompt_type_value = prompt_type.value if prompt_type else None + + if ( + task_name + and prompt_type + and f"{task_name}-{prompt_type_value}" in task_to_prompt + ): + return f"{task_name}-{prompt_type_value}" + if task_name and task_name in task_to_prompt: + return task_name + if ( + task_type + and prompt_type + and f"{task_type}-{prompt_type_value}" in task_to_prompt + ): + return f"{task_type}-{prompt_type_value}" + if task_type and task_type in task_to_prompt: + return task_type + if prompt_type and prompt_type_value in task_to_prompt: + return prompt_type_value + logger.info( + "No combination of task name and prompt type was found in model prompts." + ) + return None + + +def validate_task_to_prompt_name( + task_to_prompt_name: dict[str, str] | None, +) -> dict[str, str] | None: + if task_to_prompt_name is None: + return task_to_prompt_name + task_types = get_args(TASK_TYPE) + prompt_types = [e.value for e in PromptType] + for task_name in task_to_prompt_name: + if "-" in task_name: + task_name, prompt_type = task_name.split("-") + if prompt_type not in prompt_types: + raise ValueError( + f"Prompt type {prompt_type} is not valid. Valid prompt types are {prompt_types}" + ) + if task_name not in task_types and task_name not in prompt_types: + task = mteb.get_task(task_name=task_name) + if not task: + raise ValueError( + f"Task name {task_name} is not valid. Valid task names are task types [{task_types}], prompt types [{prompt_types}] and task names" + ) + return task_to_prompt_name diff --git a/mteb/models/sentence_transformers_models.py b/mteb/models/sentence_transformers_models.py index a3603d9eb3..9a33e0f64f 100644 --- a/mteb/models/sentence_transformers_models.py +++ b/mteb/models/sentence_transformers_models.py @@ -63,31 +63,67 @@ all_MiniLM_L6_v2 = ModelMeta( name="sentence-transformers/all-MiniLM-L6-v2", languages=["eng-Latn"], - open_source=True, + open_weights=True, revision="8b3219a92973c328a8e22fadcfa821b5dc75636a", # can be any release_date="2021-08-30", + n_parameters=22_700_000, + memory_usage=None, + embed_dim=384, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) paraphrase_multilingual_MiniLM_L12_v2 = ModelMeta( name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", languages=paraphrase_langs, - open_source=True, + open_weights=True, revision="bf3bf13ab40c3157080a7ab344c831b9ad18b5eb", # can be any release_date="2019-11-01", # release date of paper + n_parameters=118_000_000, + memory_usage=None, + embed_dim=768, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) paraphrase_multilingual_mpnet_base_v2 = ModelMeta( name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2", languages=paraphrase_langs, - open_source=True, + open_weights=True, revision="79f2382ceacceacdf38563d7c5d16b9ff8d725d6", # can be any release_date="2019-11-01", # release date of paper + n_parameters=278_000_000, + memory_usage=None, + embed_dim=768, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) labse = ModelMeta( name="sentence-transformers/LaBSE", languages=paraphrase_langs, - open_source=True, + open_weights=True, revision="e34fab64a3011d2176c99545a93d5cbddc9a91b7", # can be any release_date="2019-11-01", # release date of paper + n_parameters=471_000_000, + memory_usage=None, + embed_dim=768, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/sentence-transformers/LaBSE", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instuctions=False, ) diff --git a/mteb/models/text_formatting_utils.py b/mteb/models/text_formatting_utils.py deleted file mode 100644 index 48f13ded33..0000000000 --- a/mteb/models/text_formatting_utils.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - - -def corpus_to_texts( - corpus: list[dict[str, str]] | dict[str, list[str]] | list[str], - sep: str = "\n", -) -> list[str]: - if isinstance(corpus, dict): - return [ - (corpus["title"][i] + sep + corpus["text"][i]).strip() # type: ignore - if "title" in corpus - else corpus["text"][i].strip() # type: ignore - for i in range(len(corpus["text"])) # type: ignore - ] - else: - if isinstance(corpus[0], str): - return corpus - return [ - (doc["title"] + sep + doc["text"]).strip() - if "title" in doc - else doc["text"].strip() - for doc in corpus - ] diff --git a/mteb/models/voyage_models.py b/mteb/models/voyage_models.py index 1daad08d2c..4e79f189ee 100644 --- a/mteb/models/voyage_models.py +++ b/mteb/models/voyage_models.py @@ -6,10 +6,16 @@ import numpy as np +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta -from mteb.models.text_formatting_utils import corpus_to_texts +from mteb.models.sentence_transformer_wrapper import ( + get_prompt_name, + validate_task_to_prompt_name, +) from mteb.requires_package import requires_package +from .wrapper import Wrapper + def token_limit(max_tpm: int, interval: int = 60): limit_interval_start_ts = time.time() @@ -62,13 +68,14 @@ def wrapper(*args, **kwargs): return decorator -class VoyageWrapper: +class VoyageWrapper(Wrapper): def __init__( self, model_name: str, max_retries: int = 5, max_rpm: int = 300, max_tpm: int = 1_000_000, + model_prompts: dict[str, str] | None = None, **kwargs, ) -> None: requires_package(self, "voyageai", "Voyage") @@ -78,26 +85,23 @@ def __init__( self._embed_func = rate_limit(max_rpm)(token_limit(max_tpm)(self._client.embed)) self._model_name = model_name self._max_tpm = max_tpm + self.model_prompts = ( + validate_task_to_prompt_name(model_prompts) if model_prompts else None + ) def encode( - self, sentences: list[str], *, batch_size: int = 32, **kwargs: Any - ) -> np.ndarray: - return self._batched_encode(sentences, batch_size, "document") - - def encode_queries( - self, queries: list[str], *, batch_size: int = 32, **kwargs: Any - ) -> np.ndarray: - return self._batched_encode(queries, batch_size, "query") - - def encode_corpus( self, - corpus: list[dict[str, str]] | dict[str, list[str]], + sentences: list[str], *, batch_size: int = 32, + task_name: str, + prompt_type: PromptType | None = None, **kwargs: Any, ) -> np.ndarray: - sentences = corpus_to_texts(corpus) - return self._batched_encode(sentences, batch_size, "document") + input_type = ( + get_prompt_name(self.model_prompts, task_name, prompt_type) or "document" + ) + return self._batched_encode(sentences, batch_size, input_type) def _batched_encode( self, @@ -134,15 +138,31 @@ def _batched_encode( return np.array(embeddings) +model_prompts = { + PromptType.query.value: "query", + PromptType.passage.value: "document", +} + voyage_large_2_instruct = ModelMeta( name="voyage-large-2-instruct", revision="1", release_date="2024-05-05", languages=None, # supported languages not specified - loader=partial(VoyageWrapper, model_name="voyage-large-2-instruct"), + loader=partial( + VoyageWrapper, + model_name="voyage-large-2-instruct", + model_prompts=model_prompts, + ), max_tokens=16000, embed_dim=1024, - open_source=False, + open_weights=False, + n_parameters=None, + memory_usage=None, + license=None, + reference="https://blog.voyageai.com/2024/05/05/voyage-large-2-instruct-instruction-tuned-and-rank-1-on-mteb/", + similarity_fn_name="cosine", + framework=["API"], + use_instuctions=True, ) voyage_finance_2 = ModelMeta( @@ -150,10 +170,21 @@ def _batched_encode( revision="1", release_date="2024-05-30", languages=None, # supported languages not specified - loader=partial(VoyageWrapper, model_name="voyage-finance-2"), + loader=partial( + VoyageWrapper, + model_name="voyage-finance-2", + model_prompts=model_prompts, + ), max_tokens=32000, embed_dim=1024, - open_source=False, + open_weights=False, + n_parameters=None, + memory_usage=None, + license=None, + reference="https://blog.voyageai.com/2024/06/03/domain-specific-embeddings-finance-edition-voyage-finance-2/", + similarity_fn_name="cosine", + framework=["API"], + use_instuctions=False, ) voyage_law_2 = ModelMeta( @@ -161,10 +192,21 @@ def _batched_encode( revision="1", release_date="2024-04-15", languages=None, # supported languages not specified - loader=partial(VoyageWrapper, model_name="voyage-law-2"), + loader=partial( + VoyageWrapper, + model_name="voyage-law-2", + model_prompts=model_prompts, + ), max_tokens=16000, embed_dim=1024, - open_source=False, + open_weights=False, + n_parameters=None, + memory_usage=None, + license=None, + reference="https://blog.voyageai.com/2024/04/15/domain-specific-embeddings-and-retrieval-legal-edition-voyage-law-2/", + similarity_fn_name="cosine", + framework=["API"], + use_instuctions=False, ) voyage_code_2 = ModelMeta( @@ -172,10 +214,21 @@ def _batched_encode( revision="1", release_date="2024-01-23", languages=None, # supported languages not specified - loader=partial(VoyageWrapper, model_name="voyage-code-2"), + loader=partial( + VoyageWrapper, + model_name="voyage-code-2", + model_prompts=model_prompts, + ), max_tokens=16000, embed_dim=1536, - open_source=False, + open_weights=False, + n_parameters=None, + memory_usage=None, + license=None, + reference="https://blog.voyageai.com/2024/01/23/voyage-code-2-elevate-your-code-retrieval/", + similarity_fn_name="cosine", + framework=["API"], + use_instuctions=False, ) voyage_large_2 = ModelMeta( @@ -183,10 +236,21 @@ def _batched_encode( revision="1", release_date="2023-10-29", languages=None, # supported languages not specified - loader=partial(VoyageWrapper, model_name="voyage-large-2"), + loader=partial( + VoyageWrapper, + model_name="voyage-large-2", + model_prompts=model_prompts, + ), max_tokens=16000, embed_dim=1536, - open_source=False, + open_weights=False, + n_parameters=None, + memory_usage=None, + license=None, + reference="https://blog.voyageai.com/2023/10/29/voyage-embeddings/", + similarity_fn_name="cosine", + framework=["API"], + use_instuctions=False, ) voyage_2 = ModelMeta( @@ -194,19 +258,40 @@ def _batched_encode( revision="1", release_date="2023-10-29", languages=None, # supported languages not specified - loader=partial(VoyageWrapper, model_name="voyage-2"), + loader=partial( + VoyageWrapper, + model_name="voyage-2", + model_prompts=model_prompts, + ), max_tokens=4000, embed_dim=1024, - open_source=False, + open_weights=False, + n_parameters=None, + memory_usage=None, + license=None, + reference="https://blog.voyageai.com/2023/10/29/voyage-embeddings/", + similarity_fn_name="cosine", + framework=["API"], + use_instuctions=False, ) -# see https://blog.voyageai.com/2024/06/10/voyage-multilingual-2-multilingual-embedding-model/" voyage_multilingual_2 = ModelMeta( name="voyage-multilingual-2", revision="1", release_date="2024-06-10", languages=None, # supported languages not specified - loader=partial(VoyageWrapper, model_name="voyage-multilingual-2"), + loader=partial( + VoyageWrapper, + model_name="voyage-multilingual-2", + model_prompts=model_prompts, + ), max_tokens=32000, embed_dim=1024, - open_source=False, + open_weights=False, + n_parameters=None, + memory_usage=None, + license=None, + reference="https://blog.voyageai.com/2024/06/10/voyage-multilingual-2-multilingual-embedding-model/", + similarity_fn_name="cosine", + framework=["API"], + use_instuctions=False, ) diff --git a/mteb/models/wrapper.py b/mteb/models/wrapper.py new file mode 100644 index 0000000000..c310ab38d9 --- /dev/null +++ b/mteb/models/wrapper.py @@ -0,0 +1,7 @@ +from __future__ import annotations + + +class Wrapper: + """Class to indicate that this is a wrapper for a model.""" + + pass diff --git a/mteb/normalize_embeddings.py b/mteb/normalize_embeddings.py new file mode 100644 index 0000000000..b9ee635806 --- /dev/null +++ b/mteb/normalize_embeddings.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import numpy as np +import torch + + +def normalize_embeddings_to_numpy( + embeddings: torch.Tensor | np.ndarray | list[np.ndarray] | list[torch.Tensor], +) -> np.ndarray: + """Normalize embeddings to be numpy arrays + + + Args: + embeddings: embeddings to normalize + + Returns: + Normalized embeddings + """ + if isinstance(embeddings, torch.Tensor): + embeddings = embeddings.cpu().detach().float().numpy() + elif isinstance(embeddings, list): + if isinstance(embeddings[0], torch.Tensor): + embeddings = [ + embedding.cpu().detach().float().numpy() for embedding in embeddings + ] + elif isinstance(embeddings[0], np.ndarray): + embeddings = embeddings + + numpy_embeddings = np.array(embeddings) + + return numpy_embeddings diff --git a/mteb/overview.py b/mteb/overview.py index 993767da48..7b1bfbb426 100644 --- a/mteb/overview.py +++ b/mteb/overview.py @@ -231,6 +231,7 @@ def get_tasks( categories: list[TASK_CATEGORY] | None = None, tasks: list[str] | None = None, exclude_superseeded: bool = True, + eval_splits: list[str] | None = None, ) -> MTEBTasks: """Get a list of tasks based on the specified filters. @@ -245,6 +246,7 @@ def get_tasks( paragraph). tasks: A list of task names to include. If None, all tasks which pass the filters are included. exclude_superseeded: A boolean flag to exclude datasets which are superseeded by another. + eval_splits: A list of evaluation splits to include. If None, all splits are included. Returns: A list of all initialized tasks objects which pass all of the filters (AND operation). @@ -253,12 +255,18 @@ def get_tasks( >>> get_tasks(languages=["eng", "deu"], script=["Latn"], domains=["Legal"]) >>> get_tasks(languages=["eng"], script=["Latn"], task_types=["Classification"]) >>> get_tasks(languages=["eng"], script=["Latn"], task_types=["Clustering"], exclude_superseeded=False) + >>> get_tasks(languages=["eng"], tasks=["WikipediaRetrievalMultilingual"], eval_splits=["test"]) """ if tasks: - _tasks = [get_task(task, languages, script) for task in tasks] + _tasks = [ + get_task(task, languages, script, eval_splits=eval_splits) for task in tasks + ] return MTEBTasks(_tasks) - _tasks = [cls().filter_languages(languages, script) for cls in create_task_list()] + _tasks = [ + cls().filter_languages(languages, script).filter_eval_splits(eval_splits) + for cls in create_task_list() + ] if languages: _tasks = filter_tasks_by_languages(_tasks, languages) @@ -280,6 +288,7 @@ def get_task( task_name: str, languages: list[str] | None = None, script: list[str] | None = None, + eval_splits: list[str] | None = None, ) -> AbsTask: """Get a task by name. @@ -288,6 +297,7 @@ def get_task( languages: A list of languages either specified as 3 letter languages codes (ISO 639-3, e.g. "eng") or as script languages codes e.g. "eng-Latn". For multilingual tasks this will also remove languages that are not in the specified list. script: A list of script codes (ISO 15924 codes). If None, all scripts are included. For multilingual tasks this will also remove scripts + eval_splits: A list of evaluation splits to include. If None, all splits are included. Returns: An initialized task object. @@ -306,4 +316,7 @@ def get_task( f"KeyError: '{task_name}' not found and no similar keys were found." ) raise KeyError(suggestion) - return TASKS_REGISTRY[task_name]().filter_languages(languages, script) + task = TASKS_REGISTRY[task_name]() + if eval_splits: + task.filter_eval_splits(eval_splits=eval_splits) + return task.filter_languages(languages, script) diff --git a/mteb/task_aggregation.py b/mteb/task_aggregation.py index 899b6ae553..e5ce47a4d0 100644 --- a/mteb/task_aggregation.py +++ b/mteb/task_aggregation.py @@ -2,28 +2,30 @@ import logging from collections import defaultdict -from typing import Dict import numpy as np -from mteb.load_results.load_results import MODEL_NAME, RESULTS, REVISION -from mteb.load_results.mteb_results import MTEBResults +from mteb.load_results.benchmark_results import BenchmarkResults +from mteb.load_results.task_results import TaskResult from mteb.overview import get_task logger = logging.getLogger(__name__) -AGGREGATION = Dict[MODEL_NAME, Dict[REVISION, Dict[str, float]]] +REVISION = str +MODEL_NAME = str +AGGREGATION = dict[MODEL_NAME, dict[REVISION, dict[str, float]]] -def mean(results: RESULTS) -> AGGREGATION: +def mean(results: BenchmarkResults) -> AGGREGATION: """Calculate the mean of the main score of the given results.""" + results = results.to_legacy_dict() unique_tasks = set() for model, revisions in results.items(): for revision, res in revisions.items(): for result in res: unique_tasks.add(result.task_name) - def _mean(model_name: str, rev: str, results: list[MTEBResults]) -> float: + def _mean(model_name: str, rev: str, results: list[TaskResult]) -> float: """Calculate the mean of the main score of the given results.""" scores: list[float] = [result.get_score() for result in results] @@ -43,9 +45,10 @@ def _mean(model_name: str, rev: str, results: list[MTEBResults]) -> float: def task_category_weighted_mean( - results: RESULTS, + results: BenchmarkResults, ) -> AGGREGATION: """Calculate the mean of the main score of the given results, weighted by the number of tasks of each type.""" + results = results.to_legacy_dict() unique_tasks = set() task_types = defaultdict(set) for model, revisions in results.items(): @@ -57,7 +60,7 @@ def task_category_weighted_mean( task_types[task_type].add(task_name) def _task_category_weighted_mean( - model: str, rev: str, results: list[MTEBResults] + model: str, rev: str, results: list[TaskResult] ) -> dict[str, float]: """Calculate the mean of the main score of the given results, weighted by the number of tasks of each type.""" _task_types = {task_type: [] for task_type in task_types.keys()} @@ -92,7 +95,7 @@ def _task_category_weighted_mean( def borda_count( - results: RESULTS, + results: BenchmarkResults, ) -> AGGREGATION: """Calculate the Borda count of the given results. @@ -103,6 +106,7 @@ def borda_count( # consider each model a candidate and each task a voter # each voter ranks the candidates + results = results.to_legacy_dict() n_candidates = sum(len(revs) for revs in results.values()) candidate_scores = { model: {revision: 0.0 for revision in revisions} diff --git a/mteb/task_selection.py b/mteb/task_selection.py index d5a499c415..20d91a97b7 100644 --- a/mteb/task_selection.py +++ b/mteb/task_selection.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Callable, List +from typing import Any, Callable import pandas as pd from scipy.stats import pearsonr, spearmanr @@ -14,7 +14,7 @@ MODEL_NAME = str REVISION = str -METRIC = Callable[[List[float], List[float]], float] +METRIC = Callable[[list[float], list[float]], float] def spearman(x: list[float], y: list[float]) -> float: @@ -52,8 +52,8 @@ def results_to_dataframe( for task_result in tasks_results: data.append( { - "model": model_name, - "revision": rev, + "Model": model_name, + "Revision": rev, "task": task_result.task_name, "main_score": task_result.get_score(**kwargs), } @@ -63,7 +63,7 @@ def results_to_dataframe( if drop_na: df = df.dropna(axis=1) return df.pivot_table( - index=["model", "revision"], + index=["Model", "Revision"], columns=["task"], values="main_score", ) diff --git a/mteb/tasks/BitextMining/dan/BornholmskBitextMining.py b/mteb/tasks/BitextMining/dan/BornholmskBitextMining.py index 16fb64c2d1..53a9ccaf2b 100644 --- a/mteb/tasks/BitextMining/dan/BornholmskBitextMining.py +++ b/mteb/tasks/BitextMining/dan/BornholmskBitextMining.py @@ -22,7 +22,7 @@ class BornholmBitextMining(AbsTaskBitextMining): main_score="f1", date=("2019-01-01", "2019-12-31"), domains=["Web", "Social", "Fiction", "Written"], - license="CC-BY-4.0", + license="cc-by-4.0", task_subtypes=["Dialect pairing"], annotations_creators="expert-annotated", dialect=["da-dan-bornholm"], diff --git a/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py b/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py index 447b34fe37..9d5bfbaa8d 100644 --- a/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py +++ b/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py @@ -37,7 +37,7 @@ class TbilisiCityHallBitextMining(AbsTaskBitextMining, MultilingualTask): reference="https://huggingface.co/datasets/jupyterjazz/tbilisi-city-hall-titles", date=("2024-05-02", "2024-05-03"), task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], bibtex_citation="", diff --git a/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py b/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py index cc899896f1..d36b8f074d 100644 --- a/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py @@ -34,7 +34,7 @@ class BUCCBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2017-01-01", "2018-12-31"), domains=["Written"], task_subtypes=[], - license="Unknown", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="human-translated", diff --git a/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py b/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py index b2587afe43..25722426c8 100644 --- a/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py +++ b/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py @@ -34,7 +34,7 @@ class BUCCBitextMiningFast(AbsTaskBitextMining, MultilingualTask): date=("2017-01-01", "2018-12-31"), domains=["Written"], task_subtypes=[], - license="Unknown", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="human-translated", diff --git a/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py b/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py index cca93bb568..58c127963b 100644 --- a/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py @@ -880,7 +880,7 @@ class BibleNLPBitextMining(AbsTaskBitextMining, MultilingualTask): date=("1997-01-01", "2020-12-31"), domains=["Religious", "Written"], task_subtypes=[], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py b/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py index 417d53dfb9..b600162a60 100644 --- a/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py @@ -29,7 +29,7 @@ class DiaBLaBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2016-01-01", "2017-12-31"), domains=["Social", "Written"], task_subtypes=[], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py b/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py index 1cc34e6bc7..59abd3bf0f 100644 --- a/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py @@ -255,7 +255,7 @@ class FloresBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2022-01-01", "2022-12-31"), domains=["Non-fiction", "Encyclopaedic", "Written"], task_subtypes=[], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py b/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py index 2b9a936f64..4676aa1906 100644 --- a/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py @@ -87,7 +87,7 @@ class IN22ConvBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2022-10-01", "2023-03-01"), domains=["Social", "Spoken", "Fiction", "Spoken"], task_subtypes=[], - license="CC-BY-4.0", + license="cc-by-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py b/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py index 429520e782..174a27bc8f 100644 --- a/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py @@ -81,7 +81,7 @@ class IN22GenBitextMining(AbsTaskBitextMining, MultilingualTask): "Written", ], task_subtypes=[], - license="CC-BY-4.0", + license="cc-by-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py b/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py index 2c99749299..3ceffef598 100644 --- a/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py @@ -55,7 +55,7 @@ class IWSLT2017BitextMining(AbsTaskBitextMining, MultilingualTask): date=("2007-01-01", "2017-12-14"), # rough estimate domains=["Non-fiction", "Fiction", "Written"], task_subtypes=[], - license="CC-BY-NC-ND-4.0", + license="cc-by-nc-nd-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py b/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py index e04a96af93..acb6c72779 100644 --- a/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py @@ -116,7 +116,7 @@ class IndicGenBenchFloresBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2023-10-01", "2024-05-01"), domains=["Web", "News", "Written"], task_subtypes=[], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="human-translated and localized", diff --git a/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py b/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py index f0db3f0388..9350f36583 100644 --- a/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py @@ -27,7 +27,7 @@ class LinceMTBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2019-01-01", "2020-01-01"), domains=["Social", "Written"], task_subtypes=[], - license="Unknown", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py b/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py index 420ab2c336..1144d7d285 100644 --- a/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py @@ -266,7 +266,7 @@ class NTREXBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2019-08-01", "2022-11-01"), domains=["News", "Written"], task_subtypes=[], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="human-translated and localized", diff --git a/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py b/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py index ae74b96f0d..019a3b5e71 100644 --- a/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py @@ -30,7 +30,7 @@ class NollySentiBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2022-01-01", "2023-01-01"), domains=["Social", "Reviews", "Written"], task_subtypes=[], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/BitextMining/multilingual/NorwegianCourtsBitextMining.py b/mteb/tasks/BitextMining/multilingual/NorwegianCourtsBitextMining.py index e0c922aaa5..e013d17a1d 100644 --- a/mteb/tasks/BitextMining/multilingual/NorwegianCourtsBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NorwegianCourtsBitextMining.py @@ -22,7 +22,7 @@ class NorwegianCourtsBitextMining(AbsTaskBitextMining): date=("2020-01-01", "2020-12-31"), domains=["Legal", "Written"], task_subtypes=[], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py b/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py index fa48a010a5..e4d53dee43 100644 --- a/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py @@ -37,7 +37,7 @@ class NusaTranslationBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2021-08-01", "2022-07-01"), domains=["Social", "Written"], task_subtypes=[], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py b/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py index 539bb15d98..3ea0bc4b1b 100644 --- a/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py @@ -37,7 +37,7 @@ class NusaXBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2021-08-01", "2022-07-01"), domains=["Reviews", "Written"], task_subtypes=[], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py b/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py index 3b3c6dbc6e..4f22ce44b0 100644 --- a/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py @@ -27,7 +27,7 @@ class PhincBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2019-01-01", "2020-01-01"), domains=["Social", "Written"], task_subtypes=[], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py b/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py index 44b39e6176..2b67e06db1 100644 --- a/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py @@ -28,7 +28,7 @@ class RomaTalesBitextMining(AbsTaskBitextMining, MultilingualTask): ), # Broad historical range for the creation of folk tales domains=["Fiction", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="expert-annotated", dialect=["Lovari"], sample_creation="created", diff --git a/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py b/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py index 558e6f6f7a..d1f539f8c2 100644 --- a/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py @@ -141,7 +141,7 @@ class TatoebaBitextMining(AbsTaskBitextMining, MultilingualTask): "Written" ], # Tatoeba corpus includes a wide range of topics and domains task_subtypes=[], - license="CC BY 2.0", + license="cc-by-2.0", annotations_creators="human-annotated", dialect=[], # No specific dialect mentioned sample_creation="found", diff --git a/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py b/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py index db970d97bc..22e7c3d7f7 100644 --- a/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py +++ b/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py @@ -42,7 +42,7 @@ class SRNCorpusBitextMining(AbsTaskBitextMining, MultilingualTask): date=("2022-04-01", "2022-07-31"), domains=["Social", "Web", "Written"], task_subtypes=[], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py b/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py index ab8c12412e..c8328040d2 100644 --- a/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py +++ b/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py @@ -29,7 +29,7 @@ class VieMedEVBitextMining(AbsTaskBitextMining): date=("2024-08-28", "2022-03-28"), domains=["Medical", "Written"], task_subtypes=[], - license="cc-by-nc", + license="cc-by-nc-4.0", # version is assumed, but was previously unspecified annotations_creators="expert-annotated", dialect=[], sample_creation="human-translated and localized", diff --git a/mteb/tasks/Classification/__init__.py b/mteb/tasks/Classification/__init__.py index d8f87f8ea9..3e80ae2181 100644 --- a/mteb/tasks/Classification/__init__.py +++ b/mteb/tasks/Classification/__init__.py @@ -118,6 +118,7 @@ from .sin.SinhalaNewsClassification import * from .sin.SinhalaNewsSourceClassification import * from .slk.CSFDSKMovieReviewSentimentClassification import * +from .slk.SlovakHateSpeechClassification import * from .slv.FrenkSlClassification import * from .spa.SpanishNewsClassification import * from .spa.SpanishSentimentClassification import * diff --git a/mteb/tasks/Classification/ara/AJGT.py b/mteb/tasks/Classification/ara/AJGT.py index 06b41a46e1..b39b0f5031 100644 --- a/mteb/tasks/Classification/ara/AJGT.py +++ b/mteb/tasks/Classification/ara/AJGT.py @@ -22,7 +22,7 @@ class AJGT(AbsTaskClassification): date=("2021-01-01", "2022-01-25"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="AFL", + license="afl-3.0", annotations_creators="human-annotated", dialect=["ara-arab-MSA", "ara-arab-JO"], sample_creation="found", diff --git a/mteb/tasks/Classification/ara/HotelReviewSentimentClassification.py b/mteb/tasks/Classification/ara/HotelReviewSentimentClassification.py index 0705ab42aa..416cf44bb7 100644 --- a/mteb/tasks/Classification/ara/HotelReviewSentimentClassification.py +++ b/mteb/tasks/Classification/ara/HotelReviewSentimentClassification.py @@ -25,7 +25,7 @@ class HotelReviewSentimentClassification(AbsTaskClassification): date=("2016-06-01", "2016-07-31"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=["ara-arab-EG", "ara-arab-JO", "ara-arab-LB", "ara-arab-SA"], sample_creation="found", diff --git a/mteb/tasks/Classification/ara/OnlineStoreReviewSentimentClassification.py b/mteb/tasks/Classification/ara/OnlineStoreReviewSentimentClassification.py index f297cf22fa..81ec501a5e 100644 --- a/mteb/tasks/Classification/ara/OnlineStoreReviewSentimentClassification.py +++ b/mteb/tasks/Classification/ara/OnlineStoreReviewSentimentClassification.py @@ -24,7 +24,7 @@ class OnlineStoreReviewSentimentClassification(AbsTaskClassification): date=("2024-05-01", "2024-05-15"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=["ara-Arab-SA"], sample_creation="found", diff --git a/mteb/tasks/Classification/ara/RestaurantReviewSentimentClassification.py b/mteb/tasks/Classification/ara/RestaurantReviewSentimentClassification.py index 645b3061c0..22bd5574e0 100644 --- a/mteb/tasks/Classification/ara/RestaurantReviewSentimentClassification.py +++ b/mteb/tasks/Classification/ara/RestaurantReviewSentimentClassification.py @@ -24,7 +24,7 @@ class RestaurantReviewSentimentClassification(AbsTaskClassification): date=("2014-01-01", "2015-01-01"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="None specified", + license="not specified", annotations_creators="derived", dialect=["ara-arab-EG", "ara-arab-JO", "ara-arab-SA"], sample_creation="found", diff --git a/mteb/tasks/Classification/ara/TweetEmotionClassification.py b/mteb/tasks/Classification/ara/TweetEmotionClassification.py index eb92bc416b..22e3af698c 100644 --- a/mteb/tasks/Classification/ara/TweetEmotionClassification.py +++ b/mteb/tasks/Classification/ara/TweetEmotionClassification.py @@ -25,7 +25,7 @@ class TweetEmotionClassification(AbsTaskClassification): date=("2014-01-01", "2016-08-31"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=["ara-arab-EG", "ara-arab-LB", "ara-arab-JO", "ara-arab-SA"], sample_creation="found", diff --git a/mteb/tasks/Classification/ara/TweetSarcasmClassification.py b/mteb/tasks/Classification/ara/TweetSarcasmClassification.py index e84b990396..3c804780f5 100644 --- a/mteb/tasks/Classification/ara/TweetSarcasmClassification.py +++ b/mteb/tasks/Classification/ara/TweetSarcasmClassification.py @@ -22,7 +22,7 @@ class TweetSarcasmClassification(AbsTaskClassification): date=("2020-01-01", "2021-01-01"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="MIT", + license="mit", annotations_creators="human-annotated", dialect=["ara-arab-EG", "ara-arab-LB", "ara-arab-MA", "ara-arab-SA"], sample_creation="found", diff --git a/mteb/tasks/Classification/ben/BengaliDocumentClassification.py b/mteb/tasks/Classification/ben/BengaliDocumentClassification.py index 455424bc63..c5ed6aa451 100644 --- a/mteb/tasks/Classification/ben/BengaliDocumentClassification.py +++ b/mteb/tasks/Classification/ben/BengaliDocumentClassification.py @@ -23,7 +23,7 @@ class BengaliDocumentClassification(AbsTaskClassification): dialect=[], domains=["News", "Written"], task_subtypes=[], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="derived", sample_creation="found", bibtex_citation=""" diff --git a/mteb/tasks/Classification/ben/BengaliHateSpeechClassification.py b/mteb/tasks/Classification/ben/BengaliHateSpeechClassification.py index 81a288bff0..59fa5721b1 100644 --- a/mteb/tasks/Classification/ben/BengaliHateSpeechClassification.py +++ b/mteb/tasks/Classification/ben/BengaliHateSpeechClassification.py @@ -23,7 +23,7 @@ class BengaliHateSpeechClassification(AbsTaskClassification): dialect=[], domains=["News", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="MIT", + license="mit", annotations_creators="expert-annotated", sample_creation="found", bibtex_citation="""@inproceedings{karim2020BengaliNLP, diff --git a/mteb/tasks/Classification/ben/BengaliSentimentAnalysis.py b/mteb/tasks/Classification/ben/BengaliSentimentAnalysis.py index 0c139a01a1..c2fb31f72c 100644 --- a/mteb/tasks/Classification/ben/BengaliSentimentAnalysis.py +++ b/mteb/tasks/Classification/ben/BengaliSentimentAnalysis.py @@ -23,7 +23,7 @@ class BengaliSentimentAnalysis(AbsTaskClassification): dialect=[], domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="human-annotated", sample_creation="found", bibtex_citation="""@inproceedings{sazzed2020cross, diff --git a/mteb/tasks/Classification/ces/CSFDCZMovieReviewSentimentClassification.py b/mteb/tasks/Classification/ces/CSFDCZMovieReviewSentimentClassification.py index 0de075da37..69d4e358e8 100644 --- a/mteb/tasks/Classification/ces/CSFDCZMovieReviewSentimentClassification.py +++ b/mteb/tasks/Classification/ces/CSFDCZMovieReviewSentimentClassification.py @@ -24,7 +24,7 @@ class CSFDCZMovieReviewSentimentClassification(AbsTaskClassification): main_score="accuracy", domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/ces/CzechProductReviewSentimentClassification.py b/mteb/tasks/Classification/ces/CzechProductReviewSentimentClassification.py index b85c6829d9..8405140936 100644 --- a/mteb/tasks/Classification/ces/CzechProductReviewSentimentClassification.py +++ b/mteb/tasks/Classification/ces/CzechProductReviewSentimentClassification.py @@ -23,7 +23,7 @@ class CzechProductReviewSentimentClassification(AbsTaskClassification): dialect=[], domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="derived", sample_creation="found", bibtex_citation=""" diff --git a/mteb/tasks/Classification/ces/CzechSoMeSentimentClassification.py b/mteb/tasks/Classification/ces/CzechSoMeSentimentClassification.py index d3462f4879..add6a19e9e 100644 --- a/mteb/tasks/Classification/ces/CzechSoMeSentimentClassification.py +++ b/mteb/tasks/Classification/ces/CzechSoMeSentimentClassification.py @@ -23,7 +23,7 @@ class CzechSoMeSentimentClassification(AbsTaskClassification): dialect=[], domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="derived", sample_creation="found", bibtex_citation=""" diff --git a/mteb/tasks/Classification/ces/CzechSubjectivityClassification.py b/mteb/tasks/Classification/ces/CzechSubjectivityClassification.py index 2b7e51c1db..5603b606ab 100644 --- a/mteb/tasks/Classification/ces/CzechSubjectivityClassification.py +++ b/mteb/tasks/Classification/ces/CzechSubjectivityClassification.py @@ -22,7 +22,7 @@ class CzechSubjectivityClassification(AbsTaskClassification): main_score="accuracy", domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/dan/AngryTweetsClassification.py b/mteb/tasks/Classification/dan/AngryTweetsClassification.py index 24709f9804..fc1f177e4f 100644 --- a/mteb/tasks/Classification/dan/AngryTweetsClassification.py +++ b/mteb/tasks/Classification/dan/AngryTweetsClassification.py @@ -22,7 +22,7 @@ class AngryTweetsClassification(AbsTaskClassification): date=("2021-01-01", "2021-12-31"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC-BY-4.0", + license="cc-by-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/dan/DKHateClassification.py b/mteb/tasks/Classification/dan/DKHateClassification.py index 7646016118..65a407a56e 100644 --- a/mteb/tasks/Classification/dan/DKHateClassification.py +++ b/mteb/tasks/Classification/dan/DKHateClassification.py @@ -22,7 +22,7 @@ class DKHateClassification(AbsTaskClassification): date=("2018-01-01", "2018-12-31"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC-BY-4.0", + license="cc-by-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/dan/DanishPoliticalCommentsClassification.py b/mteb/tasks/Classification/dan/DanishPoliticalCommentsClassification.py index 29594192f1..bb2b9bff15 100644 --- a/mteb/tasks/Classification/dan/DanishPoliticalCommentsClassification.py +++ b/mteb/tasks/Classification/dan/DanishPoliticalCommentsClassification.py @@ -26,7 +26,7 @@ class DanishPoliticalCommentsClassification(AbsTaskClassification): ), # Estimated range for the collection of comments domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/dan/LccSentimentClassification.py b/mteb/tasks/Classification/dan/LccSentimentClassification.py index 070818e62a..dbaf564a5b 100644 --- a/mteb/tasks/Classification/dan/LccSentimentClassification.py +++ b/mteb/tasks/Classification/dan/LccSentimentClassification.py @@ -22,7 +22,7 @@ class LccSentimentClassification(AbsTaskClassification): date=("2006-01-01", "2006-12-31"), domains=["News", "Web", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC-BY-4.0", + license="cc-by-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/deu/GermanPoliticiansTwitterSentimentClassification.py b/mteb/tasks/Classification/deu/GermanPoliticiansTwitterSentimentClassification.py index 45bf7d6ee6..ec2021eaac 100644 --- a/mteb/tasks/Classification/deu/GermanPoliticiansTwitterSentimentClassification.py +++ b/mteb/tasks/Classification/deu/GermanPoliticiansTwitterSentimentClassification.py @@ -22,7 +22,7 @@ class GermanPoliticiansTwitterSentimentClassification(AbsTaskClassification): date=("2021-01-01", "2021-12-31"), domains=["Social", "Government", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/AmazonPolarityClassification.py b/mteb/tasks/Classification/eng/AmazonPolarityClassification.py index aa4957032b..38bf1616aa 100644 --- a/mteb/tasks/Classification/eng/AmazonPolarityClassification.py +++ b/mteb/tasks/Classification/eng/AmazonPolarityClassification.py @@ -14,7 +14,7 @@ class AmazonPolarityClassification(AbsTaskClassification): "revision": "e2d317d38cd51312af73b3d32a06d1a08b442046", }, type="Classification", - category="s2s", + category="p2p", modalities=["text"], eval_splits=["test"], eval_langs=["eng-Latn"], @@ -25,7 +25,7 @@ class AmazonPolarityClassification(AbsTaskClassification): ), # Estimated range for the collection of reviews domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/ArxivClassification.py b/mteb/tasks/Classification/eng/ArxivClassification.py index f3e84f9bf4..ca4a6657cd 100644 --- a/mteb/tasks/Classification/eng/ArxivClassification.py +++ b/mteb/tasks/Classification/eng/ArxivClassification.py @@ -23,7 +23,7 @@ class ArxivClassification(AbsTaskClassification): date=("1998-11-11", "2019-03-28"), domains=["Academic", "Written"], task_subtypes=["Topic classification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/Banking77Classification.py b/mteb/tasks/Classification/eng/Banking77Classification.py index 6de6fe211f..ca8d759196 100644 --- a/mteb/tasks/Classification/eng/Banking77Classification.py +++ b/mteb/tasks/Classification/eng/Banking77Classification.py @@ -25,7 +25,7 @@ class Banking77Classification(AbsTaskClassification): ), # Estimated range for the collection of queries domains=["Written"], task_subtypes=[], - license="MIT", + license="mit", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/EmotionClassification.py b/mteb/tasks/Classification/eng/EmotionClassification.py index 564e9aaeb8..24879e89bb 100644 --- a/mteb/tasks/Classification/eng/EmotionClassification.py +++ b/mteb/tasks/Classification/eng/EmotionClassification.py @@ -25,7 +25,7 @@ class EmotionClassification(AbsTaskClassification): ), # Estimated range for the collection of Twitter messages domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/FrenkEnClassification.py b/mteb/tasks/Classification/eng/FrenkEnClassification.py index 9feeb77fd1..2ed1fce68c 100644 --- a/mteb/tasks/Classification/eng/FrenkEnClassification.py +++ b/mteb/tasks/Classification/eng/FrenkEnClassification.py @@ -23,7 +23,7 @@ class FrenkEnClassification(AbsTaskClassification): date=("2021-05-28", "2021-05-28"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/ImdbClassification.py b/mteb/tasks/Classification/eng/ImdbClassification.py index cb47142a2c..ce48c718d1 100644 --- a/mteb/tasks/Classification/eng/ImdbClassification.py +++ b/mteb/tasks/Classification/eng/ImdbClassification.py @@ -25,7 +25,7 @@ class ImdbClassification(AbsTaskClassification): ), # Estimated range for the collection of movie reviews domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/NewsClassification.py b/mteb/tasks/Classification/eng/NewsClassification.py index 20d64c8199..1ba06bb94c 100644 --- a/mteb/tasks/Classification/eng/NewsClassification.py +++ b/mteb/tasks/Classification/eng/NewsClassification.py @@ -25,7 +25,7 @@ class NewsClassification(AbsTaskClassification): ), # Estimated range for the collection of news articles domains=["News", "Written"], task_subtypes=["Topic classification"], - license="Apache 2.0", + license="apache-2.0", annotations_creators="expert-annotated", dialect=["eng-Latn-US", "en-Latn-GB", "en-Latn-AU"], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/PatentClassification.py b/mteb/tasks/Classification/eng/PatentClassification.py index 601d3c510b..6ae0eabd58 100644 --- a/mteb/tasks/Classification/eng/PatentClassification.py +++ b/mteb/tasks/Classification/eng/PatentClassification.py @@ -23,7 +23,7 @@ class PatentClassification(AbsTaskClassification): date=("2021-11-05", "2022-10-22"), domains=["Legal", "Written"], task_subtypes=["Topic classification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/PoemSentimentClassification.py b/mteb/tasks/Classification/eng/PoemSentimentClassification.py index 2910b4369c..54a28138ac 100644 --- a/mteb/tasks/Classification/eng/PoemSentimentClassification.py +++ b/mteb/tasks/Classification/eng/PoemSentimentClassification.py @@ -23,7 +23,7 @@ class PoemSentimentClassification(AbsTaskClassification): date=("1700-01-01", "1900-01-01"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC-BY-4.0", + license="cc-by-4.0", annotations_creators="human-annotated", dialect=["eng-Latn-US", "en-Latn-GB"], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/ToxicConversationsClassification.py b/mteb/tasks/Classification/eng/ToxicConversationsClassification.py index b443d94ca9..d346f8743d 100644 --- a/mteb/tasks/Classification/eng/ToxicConversationsClassification.py +++ b/mteb/tasks/Classification/eng/ToxicConversationsClassification.py @@ -25,7 +25,7 @@ class ToxicConversationsClassification(AbsTaskClassification): ), # Estimated range for the collection of comments domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/TweetSentimentExtractionClassification.py b/mteb/tasks/Classification/eng/TweetSentimentExtractionClassification.py index 289a2236ab..c865339e30 100644 --- a/mteb/tasks/Classification/eng/TweetSentimentExtractionClassification.py +++ b/mteb/tasks/Classification/eng/TweetSentimentExtractionClassification.py @@ -25,7 +25,7 @@ class TweetSentimentExtractionClassification(AbsTaskClassification): ), # Estimated range for the collection of tweets domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/TweetTopicSingleClassification.py b/mteb/tasks/Classification/eng/TweetTopicSingleClassification.py index 2f7308516e..946644e38d 100644 --- a/mteb/tasks/Classification/eng/TweetTopicSingleClassification.py +++ b/mteb/tasks/Classification/eng/TweetTopicSingleClassification.py @@ -28,7 +28,7 @@ class TweetTopicSingleClassification(AbsTaskClassification): date=("2019-09-01", "2021-08-31"), domains=["Social", "News", "Written"], task_subtypes=["Topic classification"], - license="Other", + license="not specified", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/YahooAnswersTopicsClassification.py b/mteb/tasks/Classification/eng/YahooAnswersTopicsClassification.py index f56bfec7cd..be3b475548 100644 --- a/mteb/tasks/Classification/eng/YahooAnswersTopicsClassification.py +++ b/mteb/tasks/Classification/eng/YahooAnswersTopicsClassification.py @@ -23,7 +23,7 @@ class YahooAnswersTopicsClassification(AbsTaskClassification): date=("2022-01-25", "2022-01-25"), domains=["Web", "Written"], task_subtypes=["Topic classification"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/eng/YelpReviewFullClassification.py b/mteb/tasks/Classification/eng/YelpReviewFullClassification.py index f1faf5190a..e945a17408 100644 --- a/mteb/tasks/Classification/eng/YelpReviewFullClassification.py +++ b/mteb/tasks/Classification/eng/YelpReviewFullClassification.py @@ -22,7 +22,7 @@ class YelpReviewFullClassification(AbsTaskClassification): date=("2015-01-01", "2015-12-31"), # reviews from 2015 domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Other", + license="https://huggingface.co/datasets/Yelp/yelp_review_full#licensing-information", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/est/estonian_valence.py b/mteb/tasks/Classification/est/estonian_valence.py index d92254d37a..48b8c58764 100644 --- a/mteb/tasks/Classification/est/estonian_valence.py +++ b/mteb/tasks/Classification/est/estonian_valence.py @@ -26,7 +26,7 @@ class EstonianValenceClassification(AbsTaskClassification): domains=["News", "Written"], task_subtypes=["Sentiment/Hate speech"], dialect=[], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="human-annotated", sample_creation="found", bibtex_citation=""" diff --git a/mteb/tasks/Classification/fas/PersianFoodSentimentClassification.py b/mteb/tasks/Classification/fas/PersianFoodSentimentClassification.py index ed797f5211..7ff7aef939 100644 --- a/mteb/tasks/Classification/fas/PersianFoodSentimentClassification.py +++ b/mteb/tasks/Classification/fas/PersianFoodSentimentClassification.py @@ -24,7 +24,7 @@ class PersianFoodSentimentClassification(AbsTaskClassification): date=("2020-01-01", "2020-05-31"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py b/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py index 7ae4285e2d..385e71fb2e 100644 --- a/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py +++ b/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py @@ -12,8 +12,8 @@ class FilipinoHateSpeechClassification(AbsTaskClassification): description="Filipino Twitter dataset for sentiment classification.", reference="https://pcj.csp.org.ph/index.php/pcj/issue/download/29/PCJ%20V14%20N1%20pp1-14%202019", dataset={ - "path": "jcblaise/hatespeech_filipino", - "revision": "b01711587b073e55569de75ef04d7da4592a3618", + "path": "legacy-datasets/hate_speech_filipino", + "revision": "1994e9bb7f3ec07518e3f0d9e870cb293e234686", "trust_remote_code": True, }, type="Classification", @@ -25,7 +25,7 @@ class FilipinoHateSpeechClassification(AbsTaskClassification): main_score="accuracy", domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py b/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py index 06a68b08d1..0c05524caa 100644 --- a/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py +++ b/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py @@ -20,7 +20,7 @@ class FilipinoShopeeReviewsClassification(AbsTaskClassification): eval_splits=["validation", "test"], eval_langs=["fil-Latn"], domains=["Social", "Written"], - license="MPL-2.0", + license="mpl-2.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/fin/FinToxicityClassification.py b/mteb/tasks/Classification/fin/FinToxicityClassification.py index 6e07973b24..b6daff471b 100644 --- a/mteb/tasks/Classification/fin/FinToxicityClassification.py +++ b/mteb/tasks/Classification/fin/FinToxicityClassification.py @@ -26,7 +26,7 @@ class FinToxicityClassification(AbsTaskClassification): date=("2023-03-13", "2023-09-25"), domains=["News", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="ccy-by-sa-4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="machine-translated", diff --git a/mteb/tasks/Classification/fra/FrenchBookReviews.py b/mteb/tasks/Classification/fra/FrenchBookReviews.py index 5a2c3dfcea..855e6df775 100644 --- a/mteb/tasks/Classification/fra/FrenchBookReviews.py +++ b/mteb/tasks/Classification/fra/FrenchBookReviews.py @@ -22,7 +22,7 @@ class FrenchBookReviews(AbsTaskClassification): date=("2022-01-01", "2023-01-01"), domains=["Reviews", "Written"], task_subtypes=[], - license="CC0", + license="cc0-1.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/fra/MovieReviewSentimentClassification.py b/mteb/tasks/Classification/fra/MovieReviewSentimentClassification.py index afccd0962f..b88dd76d57 100644 --- a/mteb/tasks/Classification/fra/MovieReviewSentimentClassification.py +++ b/mteb/tasks/Classification/fra/MovieReviewSentimentClassification.py @@ -24,7 +24,7 @@ class MovieReviewSentimentClassification(AbsTaskClassification): date=("2006-01-01", "2020-01-01"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/guj/GujaratiNewsClassification.py b/mteb/tasks/Classification/guj/GujaratiNewsClassification.py index 091998f4a5..a11c26b4d5 100644 --- a/mteb/tasks/Classification/guj/GujaratiNewsClassification.py +++ b/mteb/tasks/Classification/guj/GujaratiNewsClassification.py @@ -22,7 +22,7 @@ class GujaratiNewsClassification(AbsTaskClassification): main_score="accuracy", domains=["News", "Written"], task_subtypes=["Topic classification"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py b/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py index 5530bf77c8..e1ada66712 100644 --- a/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py +++ b/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py @@ -26,7 +26,7 @@ class HebrewSentimentAnalysis(AbsTaskClassification): date=("2015-10-01", "2015-10-31"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="MIT", + license="mit", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/hin/HindiDiscourseClassification.py b/mteb/tasks/Classification/hin/HindiDiscourseClassification.py index 51576e7b0f..e68ee15b2b 100644 --- a/mteb/tasks/Classification/hin/HindiDiscourseClassification.py +++ b/mteb/tasks/Classification/hin/HindiDiscourseClassification.py @@ -24,7 +24,7 @@ class HindiDiscourseClassification(AbsTaskClassification): domains=["Fiction", "Social", "Written"], dialect=[], task_subtypes=["Discourse coherence"], - license="MIT", + license="mit", annotations_creators="expert-annotated", sample_creation="found", bibtex_citation=""" diff --git a/mteb/tasks/Classification/hin/SentimentAnalysisHindi.py b/mteb/tasks/Classification/hin/SentimentAnalysisHindi.py index c037680961..8465b7d142 100644 --- a/mteb/tasks/Classification/hin/SentimentAnalysisHindi.py +++ b/mteb/tasks/Classification/hin/SentimentAnalysisHindi.py @@ -23,7 +23,7 @@ class SentimentAnalysisHindi(AbsTaskClassification): dialect=[], domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="derived", sample_creation="found", bibtex_citation="""@misc{OdiaGenAI, diff --git a/mteb/tasks/Classification/hrv/FrenkHrClassification.py b/mteb/tasks/Classification/hrv/FrenkHrClassification.py index 0438365934..c05aab9579 100644 --- a/mteb/tasks/Classification/hrv/FrenkHrClassification.py +++ b/mteb/tasks/Classification/hrv/FrenkHrClassification.py @@ -23,7 +23,7 @@ class FrenkHrClassification(AbsTaskClassification): date=("2021-05-28", "2021-05-28"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/ind/IndonesianMongabayConservationClassification.py b/mteb/tasks/Classification/ind/IndonesianMongabayConservationClassification.py index e627a1ebc3..331f864e62 100644 --- a/mteb/tasks/Classification/ind/IndonesianMongabayConservationClassification.py +++ b/mteb/tasks/Classification/ind/IndonesianMongabayConservationClassification.py @@ -27,7 +27,7 @@ class IndonesianMongabayConservationClassification(AbsTaskClassification): main_score="f1", domains=["Web", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/ita/ItaCaseholdClassification.py b/mteb/tasks/Classification/ita/ItaCaseholdClassification.py index 3c76cae4ec..866802eede 100644 --- a/mteb/tasks/Classification/ita/ItaCaseholdClassification.py +++ b/mteb/tasks/Classification/ita/ItaCaseholdClassification.py @@ -23,7 +23,7 @@ class ItaCaseholdClassification(AbsTaskClassification): domains=["Legal", "Government", "Written"], dialect=[], task_subtypes=[], - license="Apache 2.0", + license="apache-2.0", annotations_creators="expert-annotated", sample_creation="found", bibtex_citation=""" diff --git a/mteb/tasks/Classification/ita/ItalianLinguistAcceptabilityClassification.py b/mteb/tasks/Classification/ita/ItalianLinguistAcceptabilityClassification.py index ff1760d4e4..73c317d391 100644 --- a/mteb/tasks/Classification/ita/ItalianLinguistAcceptabilityClassification.py +++ b/mteb/tasks/Classification/ita/ItalianLinguistAcceptabilityClassification.py @@ -24,7 +24,7 @@ class ItalianLinguisticAcceptabilityClassification(AbsTaskClassification): domains=["Non-fiction", "Spoken", "Written"], dialect=[], task_subtypes=["Linguistic acceptability"], - license="Unknown", + license="not specified", annotations_creators="expert-annotated", sample_creation="found", bibtex_citation=""" diff --git a/mteb/tasks/Classification/jav/JavaneseIMDBClassification.py b/mteb/tasks/Classification/jav/JavaneseIMDBClassification.py index 367c608ec4..8cff5c0b85 100644 --- a/mteb/tasks/Classification/jav/JavaneseIMDBClassification.py +++ b/mteb/tasks/Classification/jav/JavaneseIMDBClassification.py @@ -23,7 +23,7 @@ class JavaneseIMDBClassification(AbsTaskClassification): main_score="accuracy", domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="MIT", + license="mit", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/jpn/WRIMEClassification.py b/mteb/tasks/Classification/jpn/WRIMEClassification.py index 8a679d8e94..a7fd229dde 100644 --- a/mteb/tasks/Classification/jpn/WRIMEClassification.py +++ b/mteb/tasks/Classification/jpn/WRIMEClassification.py @@ -24,7 +24,7 @@ class WRIMEClassification(AbsTaskClassification): date=("2011-06-01", "2020-05-31"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="The dataset is available for research purposes only. Redistribution of the dataset is prohibited.", + license="https://huggingface.co/datasets/shunk031/wrime#licensing-information", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/kan/KannadaNewsClassification.py b/mteb/tasks/Classification/kan/KannadaNewsClassification.py index 3d5ef90395..0e6bf8ea80 100644 --- a/mteb/tasks/Classification/kan/KannadaNewsClassification.py +++ b/mteb/tasks/Classification/kan/KannadaNewsClassification.py @@ -22,7 +22,7 @@ class KannadaNewsClassification(AbsTaskClassification): date=("2019-03-17", "2020-08-06"), domains=["News", "Written"], task_subtypes=["Topic classification"], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/kat/GeorgianSentimentClassification.py b/mteb/tasks/Classification/kat/GeorgianSentimentClassification.py index 46c5aa6ba2..00481fb835 100644 --- a/mteb/tasks/Classification/kat/GeorgianSentimentClassification.py +++ b/mteb/tasks/Classification/kat/GeorgianSentimentClassification.py @@ -22,7 +22,7 @@ class GeorgianSentimentClassification(AbsTaskClassification): date=("2022-01-01", "2022-06-25"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/kor/KlueTC.py b/mteb/tasks/Classification/kor/KlueTC.py index 305ec79c8f..a9e31046aa 100644 --- a/mteb/tasks/Classification/kor/KlueTC.py +++ b/mteb/tasks/Classification/kor/KlueTC.py @@ -23,7 +23,7 @@ class KlueTC(AbsTaskClassification): date=("2016-01-01", "2020-12-31"), # from 2016 to 2020 domains=["News", "Written"], task_subtypes=["Topic classification"], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/kor/KorFin.py b/mteb/tasks/Classification/kor/KorFin.py index ef25728155..9c439e51b4 100644 --- a/mteb/tasks/Classification/kor/KorFin.py +++ b/mteb/tasks/Classification/kor/KorFin.py @@ -27,7 +27,7 @@ class KorFin(AbsTaskClassification): ), # Assumed date based on the citations in the paper domains=["News", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/kor/KorSarcasmClassification.py b/mteb/tasks/Classification/kor/KorSarcasmClassification.py index 0158d518ca..a09eaf9786 100644 --- a/mteb/tasks/Classification/kor/KorSarcasmClassification.py +++ b/mteb/tasks/Classification/kor/KorSarcasmClassification.py @@ -31,7 +31,7 @@ class KorSarcasmClassification(AbsTaskClassification): date=("2018-10-31", "2019-09-28"), # estimated based on git history domains=["Social", "Written"], task_subtypes=["Topic classification"], - license="MIT", + license="mit", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/kur/KurdishSentimentClassification.py b/mteb/tasks/Classification/kur/KurdishSentimentClassification.py index 7913fe2393..4db5a52616 100644 --- a/mteb/tasks/Classification/kur/KurdishSentimentClassification.py +++ b/mteb/tasks/Classification/kur/KurdishSentimentClassification.py @@ -22,7 +22,7 @@ class KurdishSentimentClassification(AbsTaskClassification): date=("2023-01-01", "2024-01-02"), domains=["Web", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="derived", dialect=["Sorani"], sample_creation="found", diff --git a/mteb/tasks/Classification/mal/MalayalamNewsClassification.py b/mteb/tasks/Classification/mal/MalayalamNewsClassification.py index 76cef19161..8a5c3b3772 100644 --- a/mteb/tasks/Classification/mal/MalayalamNewsClassification.py +++ b/mteb/tasks/Classification/mal/MalayalamNewsClassification.py @@ -22,7 +22,7 @@ class MalayalamNewsClassification(AbsTaskClassification): main_score="accuracy", domains=["News", "Written"], task_subtypes=["Topic classification"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/mar/MarathiNewsClassification.py b/mteb/tasks/Classification/mar/MarathiNewsClassification.py index b0cab3ac0d..18aff925cb 100644 --- a/mteb/tasks/Classification/mar/MarathiNewsClassification.py +++ b/mteb/tasks/Classification/mar/MarathiNewsClassification.py @@ -22,7 +22,7 @@ class MarathiNewsClassification(AbsTaskClassification): main_score="f1", domains=["News", "Written"], task_subtypes=["Topic classification"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/mkd/MacedonianTweetSentimentClassification.py b/mteb/tasks/Classification/mkd/MacedonianTweetSentimentClassification.py index c901a90673..f66105763c 100644 --- a/mteb/tasks/Classification/mkd/MacedonianTweetSentimentClassification.py +++ b/mteb/tasks/Classification/mkd/MacedonianTweetSentimentClassification.py @@ -22,7 +22,7 @@ class MacedonianTweetSentimentClassification(AbsTaskClassification): main_score="accuracy", domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY-NC-SA 3.0", + license="cc-by-nc-sa-3.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/AfriSentiClassification.py b/mteb/tasks/Classification/multilingual/AfriSentiClassification.py index 6a44bbfd10..c21f8c5e50 100644 --- a/mteb/tasks/Classification/multilingual/AfriSentiClassification.py +++ b/mteb/tasks/Classification/multilingual/AfriSentiClassification.py @@ -48,7 +48,7 @@ class AfriSentiClassification(MultilingualTask, AbsTaskClassification): date=("2023-02-16", "2023-09-03"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Creative Commons Attribution 4.0 International License", + license="cc-by-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py b/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py index 832f9034e0..21adc105eb 100644 --- a/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py +++ b/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py @@ -34,7 +34,7 @@ class AmazonCounterfactualClassification(MultilingualTask, AbsTaskClassification ), # Estimated range for the collection of Amazon reviews domains=["Reviews", "Written"], task_subtypes=["Counterfactual Detection"], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/CyrillicTurkicLangClassification.py b/mteb/tasks/Classification/multilingual/CyrillicTurkicLangClassification.py index 4b912a1f42..4c234bee96 100644 --- a/mteb/tasks/Classification/multilingual/CyrillicTurkicLangClassification.py +++ b/mteb/tasks/Classification/multilingual/CyrillicTurkicLangClassification.py @@ -32,7 +32,7 @@ class CyrillicTurkicLangClassification(AbsTaskClassification): date=("1998-01-01", "2012-05-01"), domains=["Web", "Written"], task_subtypes=["Language identification"], - license="CC BY-NC 4.0 DEED", + license="cc-by-nc-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/HinDialectClassification.py b/mteb/tasks/Classification/multilingual/HinDialectClassification.py index a101a453d2..af19ef5d14 100644 --- a/mteb/tasks/Classification/multilingual/HinDialectClassification.py +++ b/mteb/tasks/Classification/multilingual/HinDialectClassification.py @@ -46,7 +46,7 @@ class HinDialectClassification(AbsTaskClassification): date=("2010-01-01", "2023-01-01"), domains=["Social", "Spoken", "Written"], task_subtypes=["Language identification"], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/IndicLangClassification.py b/mteb/tasks/Classification/multilingual/IndicLangClassification.py index f644a73553..407d472253 100644 --- a/mteb/tasks/Classification/multilingual/IndicLangClassification.py +++ b/mteb/tasks/Classification/multilingual/IndicLangClassification.py @@ -80,7 +80,7 @@ class IndicLangClassification(AbsTaskClassification): date=("2022-08-01", "2023-01-01"), domains=["Web", "Non-fiction", "Written"], task_subtypes=["Language identification"], - license="CC0", + license="cc0-1.0", annotations_creators="expert-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py b/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py index cd05298e22..bd9058918b 100644 --- a/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py +++ b/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py @@ -40,7 +40,7 @@ class IndicSentimentClassification(MultilingualTask, AbsTaskClassification): date=("2022-08-01", "2022-12-20"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC0", + license="cc0-1.0", annotations_creators="human-annotated", dialect=[], sample_creation="machine-translated and verified", diff --git a/mteb/tasks/Classification/multilingual/LanguageClassification.py b/mteb/tasks/Classification/multilingual/LanguageClassification.py index bf067de1b1..2d0514578c 100644 --- a/mteb/tasks/Classification/multilingual/LanguageClassification.py +++ b/mteb/tasks/Classification/multilingual/LanguageClassification.py @@ -45,7 +45,7 @@ class LanguageClassification(AbsTaskClassification): date=("2021-11-01", "2021-11-30"), domains=["Reviews", "Web", "Non-fiction", "Fiction", "Government", "Written"], task_subtypes=["Language identification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py b/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py index 98b9b57f21..4f5c793aeb 100644 --- a/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py +++ b/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py @@ -33,7 +33,7 @@ class MTOPDomainClassification(MultilingualTask, AbsTaskClassification): date=("2020-01-01", "2020-12-31"), domains=["Spoken", "Spoken"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py b/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py index 657b5b9015..be9bb79131 100644 --- a/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py +++ b/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py @@ -33,7 +33,7 @@ class MTOPIntentClassification(MultilingualTask, AbsTaskClassification): date=("2020-01-01", "2020-12-31"), domains=["Spoken", "Spoken"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py b/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py index 8b2162876a..9af5992499 100644 --- a/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py +++ b/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py @@ -78,7 +78,7 @@ class MassiveIntentClassification(MultilingualTask, AbsTaskClassification): date=("2022-01-01", "2022-04-22"), domains=["Spoken"], task_subtypes=[], - license="Apache 2.0", + license="apache-2.0", annotations_creators="human-annotated", dialect=[], sample_creation="human-translated and localized", # with the exception of the English data diff --git a/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py b/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py index 8386f2c77f..d59ae1e41f 100644 --- a/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py +++ b/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py @@ -78,7 +78,7 @@ class MassiveScenarioClassification(MultilingualTask, AbsTaskClassification): date=("2022-01-01", "2022-04-22"), domains=["Spoken"], task_subtypes=[], - license="Apache 2.0", + license="apache-2.0", annotations_creators="human-annotated", dialect=[], sample_creation="human-translated and localized", # with the exception of the English data diff --git a/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py b/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py index 921aa31dde..0f762f747f 100644 --- a/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py +++ b/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py @@ -60,7 +60,7 @@ class MultilingualSentimentClassification(AbsTaskClassification, MultilingualTas date=("2022-08-01", "2022-08-01"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=["ar-dz"], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/NaijaSenti.py b/mteb/tasks/Classification/multilingual/NaijaSenti.py index 6f5c98eea6..20d1498345 100644 --- a/mteb/tasks/Classification/multilingual/NaijaSenti.py +++ b/mteb/tasks/Classification/multilingual/NaijaSenti.py @@ -33,7 +33,7 @@ class NaijaSenti(AbsTaskClassification, MultilingualTask): date=("2022-05-01", "2023-05-08"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC-BY-4.0 license", + license="cc-by-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py b/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py index 3143fcb436..2a866c9792 100644 --- a/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py +++ b/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py @@ -36,7 +36,7 @@ class NusaParagraphEmotionClassification(MultilingualTask, AbsTaskClassification date=("2021-08-01", "2022-07-01"), domains=["Non-fiction", "Fiction", "Written"], task_subtypes=["Emotion classification"], - license="Apache 2.0", + license="apache-2.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py b/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py index abf247e351..6d7d745a43 100644 --- a/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py +++ b/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py @@ -36,7 +36,7 @@ class NusaParagraphTopicClassification(MultilingualTask, AbsTaskClassification): date=("2021-08-01", "2022-07-01"), domains=["Non-fiction", "Fiction", "Written"], task_subtypes=["Topic classification"], - license="Apache 2.0", + license="apache-2.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/NusaXSenti.py b/mteb/tasks/Classification/multilingual/NusaXSenti.py index 09f7bac19a..a701bf02a7 100644 --- a/mteb/tasks/Classification/multilingual/NusaXSenti.py +++ b/mteb/tasks/Classification/multilingual/NusaXSenti.py @@ -37,7 +37,7 @@ class NusaXSentiClassification(AbsTaskClassification, MultilingualTask): date=("2022-05-01", "2023-05-08"), domains=["Reviews", "Web", "Social", "Constructed", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC-BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/ScalaClassification.py b/mteb/tasks/Classification/multilingual/ScalaClassification.py index be0a76b70d..2354e851f9 100644 --- a/mteb/tasks/Classification/multilingual/ScalaClassification.py +++ b/mteb/tasks/Classification/multilingual/ScalaClassification.py @@ -34,7 +34,7 @@ class ScalaClassification(AbsTaskClassification, MultilingualTask): ), # derived from dependency treebank, this a the best guess domains=["Fiction", "News", "Non-fiction", "Blog", "Spoken", "Web", "Written"], task_subtypes=["Linguistic acceptability"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/Classification/multilingual/SouthAfricanLangClassification.py b/mteb/tasks/Classification/multilingual/SouthAfricanLangClassification.py index 604d0b08dd..b23756ac97 100644 --- a/mteb/tasks/Classification/multilingual/SouthAfricanLangClassification.py +++ b/mteb/tasks/Classification/multilingual/SouthAfricanLangClassification.py @@ -36,7 +36,7 @@ class SouthAfricanLangClassification(AbsTaskClassification): date=("2010-01-01", "2023-01-01"), domains=["Web", "Non-fiction", "Written"], task_subtypes=["Language identification"], - license="MIT", + license="mit", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py b/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py index 567471a055..ca8ecb30bd 100644 --- a/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py +++ b/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py @@ -30,7 +30,7 @@ class SwissJudgementClassification(MultilingualTask, AbsTaskClassification): task_subtypes=[ "Political classification", ], - license="CC-BY-4.0", + license="cc-by-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/multilingual/TurkicClassification.py b/mteb/tasks/Classification/multilingual/TurkicClassification.py index 564cd2846c..327765c092 100644 --- a/mteb/tasks/Classification/multilingual/TurkicClassification.py +++ b/mteb/tasks/Classification/multilingual/TurkicClassification.py @@ -32,7 +32,7 @@ class TurkicClassification(MultilingualTask, AbsTaskClassification): date=("2023-02-16", "2023-09-03"), domains=["News", "Written"], task_subtypes=["Topic classification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/mya/MyanmarNews.py b/mteb/tasks/Classification/mya/MyanmarNews.py index a8bd7ad04a..70a603e8b2 100644 --- a/mteb/tasks/Classification/mya/MyanmarNews.py +++ b/mteb/tasks/Classification/mya/MyanmarNews.py @@ -23,7 +23,7 @@ class MyanmarNews(AbsTaskClassification): date=("2017-10-01", "2017-10-31"), domains=["News", "Written"], task_subtypes=["Topic classification"], - license="GPL 3.0", + license="gpl-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/nep/NepaliNewsClassification.py b/mteb/tasks/Classification/nep/NepaliNewsClassification.py index 478c877df7..5985cf232c 100644 --- a/mteb/tasks/Classification/nep/NepaliNewsClassification.py +++ b/mteb/tasks/Classification/nep/NepaliNewsClassification.py @@ -22,7 +22,7 @@ class NepaliNewsClassification(AbsTaskClassification): main_score="accuracy", domains=["News", "Written"], task_subtypes=["Topic classification"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/nld/DutchBookReviewSentimentClassification.py b/mteb/tasks/Classification/nld/DutchBookReviewSentimentClassification.py index 23bce772bb..efd7076d59 100644 --- a/mteb/tasks/Classification/nld/DutchBookReviewSentimentClassification.py +++ b/mteb/tasks/Classification/nld/DutchBookReviewSentimentClassification.py @@ -23,7 +23,7 @@ class DutchBookReviewSentimentClassification(AbsTaskClassification): main_score="accuracy", domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/ory/OdiaNewsClassification.py b/mteb/tasks/Classification/ory/OdiaNewsClassification.py index 5de5940b4c..4459cb325c 100644 --- a/mteb/tasks/Classification/ory/OdiaNewsClassification.py +++ b/mteb/tasks/Classification/ory/OdiaNewsClassification.py @@ -22,7 +22,7 @@ class OdiaNewsClassification(AbsTaskClassification): main_score="f1", domains=["News", "Written"], task_subtypes=["Topic classification"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/pan/PunjabiNewsClassification.py b/mteb/tasks/Classification/pan/PunjabiNewsClassification.py index 347608bdac..68e4492e22 100644 --- a/mteb/tasks/Classification/pan/PunjabiNewsClassification.py +++ b/mteb/tasks/Classification/pan/PunjabiNewsClassification.py @@ -22,7 +22,7 @@ class PunjabiNewsClassification(AbsTaskClassification): main_score="accuracy", domains=["News", "Written"], task_subtypes=["Topic classification"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/por/HateSpeechPortugueseClassification.py b/mteb/tasks/Classification/por/HateSpeechPortugueseClassification.py index 2c1261c9ac..43a6b086dd 100644 --- a/mteb/tasks/Classification/por/HateSpeechPortugueseClassification.py +++ b/mteb/tasks/Classification/por/HateSpeechPortugueseClassification.py @@ -23,7 +23,7 @@ class HateSpeechPortugueseClassification(AbsTaskClassification): date=("2017-03-08", "2017-03-09"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/ron/Moroco.py b/mteb/tasks/Classification/ron/Moroco.py index 18e29d4237..d761823d56 100644 --- a/mteb/tasks/Classification/ron/Moroco.py +++ b/mteb/tasks/Classification/ron/Moroco.py @@ -25,7 +25,7 @@ class Moroco(AbsTaskClassification): date=("2017-10-01", "2017-10-31"), domains=["News", "Written"], task_subtypes=["Topic classification"], - license="CC BY-4.0", + license="cc-by-4.0", annotations_creators="derived", dialect=[ "ron-Latn-ron", diff --git a/mteb/tasks/Classification/ron/RomanianReviewsSentiment.py b/mteb/tasks/Classification/ron/RomanianReviewsSentiment.py index de8069d096..bc4aabcbd0 100644 --- a/mteb/tasks/Classification/ron/RomanianReviewsSentiment.py +++ b/mteb/tasks/Classification/ron/RomanianReviewsSentiment.py @@ -25,7 +25,7 @@ class RomanianReviewsSentiment(AbsTaskClassification): main_score="accuracy", domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC-BY-4.0", + license="cc-by-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/ron/RomanianSentimentClassification.py b/mteb/tasks/Classification/ron/RomanianSentimentClassification.py index 9c30fc8135..880ab33774 100644 --- a/mteb/tasks/Classification/ron/RomanianSentimentClassification.py +++ b/mteb/tasks/Classification/ron/RomanianSentimentClassification.py @@ -25,7 +25,7 @@ class RomanianSentimentClassification(AbsTaskClassification): main_score="accuracy", domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/rus/KinopoiskClassification.py b/mteb/tasks/Classification/rus/KinopoiskClassification.py index e15ac0785a..619d807afb 100644 --- a/mteb/tasks/Classification/rus/KinopoiskClassification.py +++ b/mteb/tasks/Classification/rus/KinopoiskClassification.py @@ -22,7 +22,7 @@ class KinopoiskClassification(AbsTaskClassification): date=("2004-07-01", "2012-12-01"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/rus/RuSciBenchGRNTIClassification.py b/mteb/tasks/Classification/rus/RuSciBenchGRNTIClassification.py index d0d0729273..e6e276664a 100644 --- a/mteb/tasks/Classification/rus/RuSciBenchGRNTIClassification.py +++ b/mteb/tasks/Classification/rus/RuSciBenchGRNTIClassification.py @@ -22,7 +22,7 @@ class RuSciBenchGRNTIClassification(AbsTaskClassification): date=("1999-01-01", "2024-01-01"), domains=["Academic", "Written"], task_subtypes=["Topic classification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/rus/RuSciBenchOECDClassification.py b/mteb/tasks/Classification/rus/RuSciBenchOECDClassification.py index 22899e1933..b08367e4af 100644 --- a/mteb/tasks/Classification/rus/RuSciBenchOECDClassification.py +++ b/mteb/tasks/Classification/rus/RuSciBenchOECDClassification.py @@ -22,7 +22,7 @@ class RuSciBenchOECDClassification(AbsTaskClassification): date=("1999-01-01", "2024-01-01"), domains=["Academic", "Written"], task_subtypes=["Topic classification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/san/SanskritShlokasClassification.py b/mteb/tasks/Classification/san/SanskritShlokasClassification.py index 188a8df20a..4e22db6f07 100644 --- a/mteb/tasks/Classification/san/SanskritShlokasClassification.py +++ b/mteb/tasks/Classification/san/SanskritShlokasClassification.py @@ -22,7 +22,7 @@ class SanskritShlokasClassification(AbsTaskClassification): main_score="accuracy", domains=["Religious", "Written"], task_subtypes=["Topic classification"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/slk/CSFDSKMovieReviewSentimentClassification.py b/mteb/tasks/Classification/slk/CSFDSKMovieReviewSentimentClassification.py index 47d7560830..587fa1ff7b 100644 --- a/mteb/tasks/Classification/slk/CSFDSKMovieReviewSentimentClassification.py +++ b/mteb/tasks/Classification/slk/CSFDSKMovieReviewSentimentClassification.py @@ -24,7 +24,7 @@ class CSFDSKMovieReviewSentimentClassification(AbsTaskClassification): main_score="accuracy", domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/slk/SlovakHateSpeechClassification.py b/mteb/tasks/Classification/slk/SlovakHateSpeechClassification.py new file mode 100644 index 0000000000..ad1d29dcf7 --- /dev/null +++ b/mteb/tasks/Classification/slk/SlovakHateSpeechClassification.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class SlovakHateSpeechClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="SlovakHateSpeechClassification", + description="The dataset contains posts from a social network with human annotations for hateful or offensive language in Slovak.", + reference="https://huggingface.co/datasets/TUKE-KEMT/hate_speech_slovak", + dataset={ + "path": "TUKE-KEMT/hate_speech_slovak", + "revision": "f9301b9937128c9c0b636fa6da203aeb046479f4", + }, + type="Classification", + category="s2s", + modalities=["text"], + date=("2024-05-25", "2024-06-06"), + eval_splits=["test"], + eval_langs=["slk-Latn"], + main_score="accuracy", + domains=["Social", "Written"], + task_subtypes=["Sentiment/Hate speech"], + license="cc-by-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="", + descriptive_stats={ + "n_samples": {"test": 1319}, + "avg_character_length": {"test": 92.71}, + }, + ) diff --git a/mteb/tasks/Classification/slv/FrenkSlClassification.py b/mteb/tasks/Classification/slv/FrenkSlClassification.py index 22cfb2091f..63a5af2ec0 100644 --- a/mteb/tasks/Classification/slv/FrenkSlClassification.py +++ b/mteb/tasks/Classification/slv/FrenkSlClassification.py @@ -23,7 +23,7 @@ class FrenkSlClassification(AbsTaskClassification): date=("2021-05-28", "2021-05-28"), domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/spa/SpanishNewsClassification.py b/mteb/tasks/Classification/spa/SpanishNewsClassification.py index 557dd4c9a1..6804034ef2 100644 --- a/mteb/tasks/Classification/spa/SpanishNewsClassification.py +++ b/mteb/tasks/Classification/spa/SpanishNewsClassification.py @@ -22,7 +22,7 @@ class SpanishNewsClassification(AbsTaskClassification): main_score="accuracy", domains=["News", "Written"], task_subtypes=[], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/spa/SpanishSentimentClassification.py b/mteb/tasks/Classification/spa/SpanishSentimentClassification.py index a5aee78b6c..0b0e147273 100644 --- a/mteb/tasks/Classification/spa/SpanishSentimentClassification.py +++ b/mteb/tasks/Classification/spa/SpanishSentimentClassification.py @@ -22,7 +22,7 @@ class SpanishSentimentClassification(AbsTaskClassification): main_score="accuracy", domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/ssw/SiswatiNewsClassification.py b/mteb/tasks/Classification/ssw/SiswatiNewsClassification.py index 02c9f4f9a9..516e9b3a20 100644 --- a/mteb/tasks/Classification/ssw/SiswatiNewsClassification.py +++ b/mteb/tasks/Classification/ssw/SiswatiNewsClassification.py @@ -24,7 +24,7 @@ class SiswatiNewsClassification(AbsTaskClassification): date=("2022-08-01", "2022-08-01"), domains=["News", "Written"], task_subtypes=["Topic classification"], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/svk/SlovakMovieReviewSentimentClassification.py b/mteb/tasks/Classification/svk/SlovakMovieReviewSentimentClassification.py index e1428ca45e..10df4d28e9 100644 --- a/mteb/tasks/Classification/svk/SlovakMovieReviewSentimentClassification.py +++ b/mteb/tasks/Classification/svk/SlovakMovieReviewSentimentClassification.py @@ -23,7 +23,7 @@ class SlovakMovieReviewSentimentClassification(AbsTaskClassification): dialect=[], domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="derived", sample_creation="found", bibtex_citation=""" diff --git a/mteb/tasks/Classification/swa/SwahiliNewsClassification.py b/mteb/tasks/Classification/swa/SwahiliNewsClassification.py index 835482f7ae..c297a4528d 100644 --- a/mteb/tasks/Classification/swa/SwahiliNewsClassification.py +++ b/mteb/tasks/Classification/swa/SwahiliNewsClassification.py @@ -23,7 +23,7 @@ class SwahiliNewsClassification(AbsTaskClassification): dialect=[], domains=["News", "Written"], task_subtypes=[], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="derived", sample_creation="found", bibtex_citation=""" diff --git a/mteb/tasks/Classification/swe/DalajClassification.py b/mteb/tasks/Classification/swe/DalajClassification.py index 22d63b0f7c..b43dd8a168 100644 --- a/mteb/tasks/Classification/swe/DalajClassification.py +++ b/mteb/tasks/Classification/swe/DalajClassification.py @@ -25,7 +25,7 @@ class DalajClassification(AbsTaskClassification): date=("2017-01-01", "2020-12-31"), domains=["Non-fiction", "Written"], task_subtypes=["Linguistic acceptability"], - license="CC-BY-4.0", + license="cc-by-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/Classification/swe/SwedishSentimentClassification.py b/mteb/tasks/Classification/swe/SwedishSentimentClassification.py index 6487ef143b..0b62ba0668 100644 --- a/mteb/tasks/Classification/swe/SwedishSentimentClassification.py +++ b/mteb/tasks/Classification/swe/SwedishSentimentClassification.py @@ -25,7 +25,7 @@ class SwedishSentimentClassification(AbsTaskClassification): date=("2021-01-01", "2022-01-01"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/tam/TamilNewsClassification.py b/mteb/tasks/Classification/tam/TamilNewsClassification.py index a5af3f19d9..73cf10adba 100644 --- a/mteb/tasks/Classification/tam/TamilNewsClassification.py +++ b/mteb/tasks/Classification/tam/TamilNewsClassification.py @@ -22,7 +22,7 @@ class TamilNewsClassification(AbsTaskClassification): main_score="f1", domains=["News", "Written"], task_subtypes=["Topic classification"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/tel/TeluguAndhraJyotiNewsClassification.py b/mteb/tasks/Classification/tel/TeluguAndhraJyotiNewsClassification.py index 1a00410bdb..0458fd0e66 100644 --- a/mteb/tasks/Classification/tel/TeluguAndhraJyotiNewsClassification.py +++ b/mteb/tasks/Classification/tel/TeluguAndhraJyotiNewsClassification.py @@ -22,7 +22,7 @@ class TeluguAndhraJyotiNewsClassification(AbsTaskClassification): main_score="f1", domains=["News", "Written"], task_subtypes=["Topic classification"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/tsn/TswanaNewsClassification.py b/mteb/tasks/Classification/tsn/TswanaNewsClassification.py index e7a0d2c3e4..d49e3c5cd1 100644 --- a/mteb/tasks/Classification/tsn/TswanaNewsClassification.py +++ b/mteb/tasks/Classification/tsn/TswanaNewsClassification.py @@ -22,7 +22,7 @@ class TswanaNewsClassification(AbsTaskClassification): main_score="accuracy", date=("2015-01-01", "2023-01-01"), domains=["News", "Written"], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/tur/TurkishMovieSentimentClassification.py b/mteb/tasks/Classification/tur/TurkishMovieSentimentClassification.py index b096ccda46..70dab008d8 100644 --- a/mteb/tasks/Classification/tur/TurkishMovieSentimentClassification.py +++ b/mteb/tasks/Classification/tur/TurkishMovieSentimentClassification.py @@ -22,7 +22,7 @@ class TurkishMovieSentimentClassification(AbsTaskClassification): date=("2013-01-01", "2013-08-11"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/tur/TurkishProductSentimentClassification.py b/mteb/tasks/Classification/tur/TurkishProductSentimentClassification.py index 5add75fae3..d755d51ba7 100644 --- a/mteb/tasks/Classification/tur/TurkishProductSentimentClassification.py +++ b/mteb/tasks/Classification/tur/TurkishProductSentimentClassification.py @@ -22,7 +22,7 @@ class TurkishProductSentimentClassification(AbsTaskClassification): date=("2013-01-01", "2013-08-11"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/urd/UrduRomanSentimentClassification.py b/mteb/tasks/Classification/urd/UrduRomanSentimentClassification.py index 04857de0f7..6b1eddda97 100644 --- a/mteb/tasks/Classification/urd/UrduRomanSentimentClassification.py +++ b/mteb/tasks/Classification/urd/UrduRomanSentimentClassification.py @@ -23,7 +23,7 @@ class UrduRomanSentimentClassification(AbsTaskClassification): main_score="f1", domains=["Social", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/vie/VieStudentFeedbackClassification.py b/mteb/tasks/Classification/vie/VieStudentFeedbackClassification.py index c5455db7e6..dc3dac9dea 100644 --- a/mteb/tasks/Classification/vie/VieStudentFeedbackClassification.py +++ b/mteb/tasks/Classification/vie/VieStudentFeedbackClassification.py @@ -25,7 +25,7 @@ class VieStudentFeedbackClassification(AbsTaskClassification): date=("2021-12-26", "2021-12-26"), domains=["Reviews", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="MIT", + license="mit", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/Classification/zho/YueOpenriceReviewClassification.py b/mteb/tasks/Classification/zho/YueOpenriceReviewClassification.py index d445a30f54..926199dc83 100644 --- a/mteb/tasks/Classification/zho/YueOpenriceReviewClassification.py +++ b/mteb/tasks/Classification/zho/YueOpenriceReviewClassification.py @@ -22,7 +22,7 @@ class YueOpenriceReviewClassification(AbsTaskClassification): date=("2019-01-01", "2019-05-01"), domains=["Reviews", "Spoken"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Classification/zul/IsiZuluNewsClassification.py b/mteb/tasks/Classification/zul/IsiZuluNewsClassification.py index 428d21e122..18f8a21e5c 100644 --- a/mteb/tasks/Classification/zul/IsiZuluNewsClassification.py +++ b/mteb/tasks/Classification/zul/IsiZuluNewsClassification.py @@ -24,7 +24,7 @@ class IsiZuluNewsClassification(AbsTaskClassification): date=("2022-08-01", "2022-08-01"), domains=["News", "Written"], task_subtypes=["Topic classification"], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py b/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py index ca16fec0c7..7a7748fd7c 100644 --- a/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py +++ b/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py @@ -33,11 +33,11 @@ class ArXivHierarchicalClusteringP2P(AbsTaskClusteringFast): date=("1991-01-01", "2021-01-01"), # 1991-01-01 is the first arxiv paper domains=["Academic", "Written"], task_subtypes=[], - license="CC0", + license="cc0-1.0", annotations_creators="derived", dialect=["Thematic clustering"], sample_creation="found", - bibtex_citation="@misc{arXiv.org e-Print archive, url={https://arxiv.org/} }", + bibtex_citation="", descriptive_stats={ "n_samples": {"test": N_SAMPLES}, "test": { @@ -213,11 +213,11 @@ class ArXivHierarchicalClusteringS2S(AbsTaskClusteringFast): date=("1991-01-01", "2021-01-01"), # 1991-01-01 is the first arxiv paper domains=["Academic", "Written"], task_subtypes=["Thematic clustering"], - license="CC0", + license="cc0-1.0", annotations_creators="derived", dialect=[], sample_creation="found", - bibtex_citation="@misc{arXiv.org e-Print archive, url={https://arxiv.org/} }", + bibtex_citation="", descriptive_stats={ "n_samples": {"test": N_SAMPLES}, "avg_character_length": {"test": 1009.98}, diff --git a/mteb/tasks/Clustering/eng/ArxivClusteringP2P.py b/mteb/tasks/Clustering/eng/ArxivClusteringP2P.py index 55eb6a1c83..e74170eea1 100644 --- a/mteb/tasks/Clustering/eng/ArxivClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/ArxivClusteringP2P.py @@ -25,7 +25,7 @@ class ArxivClusteringP2P(AbsTaskClustering): date=("1991-01-01", "2021-01-01"), # 1991-01-01 is the first arxiv paper domains=["Academic", "Written"], task_subtypes=[], - license="CC0", + license="cc0-1.0", annotations_creators="derived", dialect=[], sample_creation="found", @@ -66,7 +66,7 @@ class ArxivClusteringP2PFast(AbsTaskClustering): date=("1991-01-01", "2021-01-01"), # 1991-01-01 is the first arxiv paper domains=["Academic", "Written"], task_subtypes=[], - license="CC0", + license="cc0-1.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/eng/RedditClustering.py b/mteb/tasks/Clustering/eng/RedditClustering.py index 7c2d44b9f5..58d40fbd43 100644 --- a/mteb/tasks/Clustering/eng/RedditClustering.py +++ b/mteb/tasks/Clustering/eng/RedditClustering.py @@ -30,7 +30,7 @@ class RedditFastClusteringS2S(AbsTaskClusteringFast): date=("2021-01-01", "2021-04-14"), domains=["Web", "Social", "Written"], task_subtypes=["Thematic clustering"], - license="Not specified", # derived from pushshift + license="not specified", # derived from pushshift annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/eng/RedditClusteringP2P.py b/mteb/tasks/Clustering/eng/RedditClusteringP2P.py index 12148f91ed..d81559e246 100644 --- a/mteb/tasks/Clustering/eng/RedditClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/RedditClusteringP2P.py @@ -75,7 +75,7 @@ class RedditFastClusteringP2P(AbsTaskClusteringFast): date=("2021-01-01", "2021-04-14"), domains=["Web", "Social", "Written"], task_subtypes=["Thematic clustering"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/eng/StackExchangeClustering.py b/mteb/tasks/Clustering/eng/StackExchangeClustering.py index 8d1118c357..881d77e20e 100644 --- a/mteb/tasks/Clustering/eng/StackExchangeClustering.py +++ b/mteb/tasks/Clustering/eng/StackExchangeClustering.py @@ -30,7 +30,7 @@ class StackExchangeClusteringFast(AbsTaskClusteringFast): date=("2021-01-01", "2021-04-14"), domains=["Web", "Written"], task_subtypes=["Thematic clustering"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py b/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py index 4131ed30e8..993b1e0db8 100644 --- a/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py @@ -32,7 +32,7 @@ class StackExchangeClusteringP2PFast(AbsTaskClusteringFast): date=("2021-01-01", "2021-04-14"), domains=["Web", "Written"], task_subtypes=["Thematic clustering"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py b/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py index df7b68f5ba..6e9aef97df 100644 --- a/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py +++ b/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py @@ -31,7 +31,7 @@ class TwentyNewsgroupsClustering(AbsTaskClustering): date=("1995-01-01", "1995-01-01"), domains=["News", "Written"], task_subtypes=["Thematic clustering"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", @@ -74,7 +74,7 @@ class TwentyNewsgroupsClusteringFast(AbsTaskClusteringFast): date=("1995-01-01", "1995-01-01"), domains=["News", "Written"], task_subtypes=["Thematic clustering"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/fra/HALClusteringS2S.py b/mteb/tasks/Clustering/fra/HALClusteringS2S.py index b967e23fa1..442176d640 100644 --- a/mteb/tasks/Clustering/fra/HALClusteringS2S.py +++ b/mteb/tasks/Clustering/fra/HALClusteringS2S.py @@ -84,7 +84,7 @@ class HALClusteringS2SFast(AbsTaskClusteringFast): date=("2000-03-29", "2024-05-24"), domains=["Academic", "Written"], task_subtypes=["Thematic clustering"], - license="Apache-2.0", + license="apache-2.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py b/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py index 2e4d6c040f..0c65e82772 100644 --- a/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py @@ -45,7 +45,7 @@ class IndicReviewsClusteringP2P(AbsTaskClustering, MultilingualTask): date=("2022-08-01", "2022-12-20"), domains=["Reviews", "Written"], task_subtypes=["Thematic clustering"], - license="CC0", + license="cc0-1.0", annotations_creators="human-annotated", dialect=[], sample_creation="machine-translated and verified", diff --git a/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py b/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py index a892f666e6..9f823e774f 100644 --- a/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py @@ -26,11 +26,10 @@ class MLSUMClusteringP2P(AbsTaskClustering, MultilingualTask): metadata = TaskMetadata( name="MLSUMClusteringP2P", description="Clustering of newspaper article contents and titles from MLSUM dataset. Clustering of 10 sets on the newpaper article topics.", - reference="https://huggingface.co/datasets/reciTAL/mlsum", + reference="https://huggingface.co/datasets/mteb/mlsum", dataset={ - "path": "reciTAL/mlsum", - "revision": "b5d54f8f3b61ae17845046286940f03c6bc79bc7", - "trust_remote_code": True, + "path": "mteb/mlsum", + "revision": "b4efe498c4d0b9d7bdd2905f6fff4e22ae251d00", }, type="Clustering", category="p2p", @@ -41,7 +40,7 @@ class MLSUMClusteringP2P(AbsTaskClustering, MultilingualTask): date=("2010-01-01", "2018-09-30"), domains=["News", "Written"], task_subtypes=["Topic classification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", @@ -101,11 +100,10 @@ class MLSUMClusteringP2PFast(AbsTaskClusteringFast, MultilingualTask): metadata = TaskMetadata( name="MLSUMClusteringP2P.v2", description="Clustering of newspaper article contents and titles from MLSUM dataset. Clustering of 10 sets on the newpaper article topics.", - reference="https://huggingface.co/datasets/mlsum", + reference="https://huggingface.co/datasets/mteb/mlsum", dataset={ - "path": "reciTAL/mlsum", - "revision": "b5d54f8f3b61ae17845046286940f03c6bc79bc7", - "trust_remote_code": True, + "path": "mteb/mlsum", + "revision": "b4efe498c4d0b9d7bdd2905f6fff4e22ae251d00", }, type="Clustering", category="p2p", @@ -116,7 +114,7 @@ class MLSUMClusteringP2PFast(AbsTaskClusteringFast, MultilingualTask): date=("2010-01-01", "2018-09-30"), domains=["News", "Written"], task_subtypes=["Topic classification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py b/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py index 8bb21b456f..9e3978ff30 100644 --- a/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py +++ b/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py @@ -26,11 +26,10 @@ class MLSUMClusteringS2S(AbsTaskClustering, MultilingualTask): metadata = TaskMetadata( name="MLSUMClusteringS2S", description="Clustering of newspaper article contents and titles from MLSUM dataset. Clustering of 10 sets on the newpaper article topics.", - reference="https://huggingface.co/datasets/reciTAL/mlsum", + reference="https://huggingface.co/datasets/mteb/mlsum", dataset={ - "path": "reciTAL/mlsum", - "revision": "b5d54f8f3b61ae17845046286940f03c6bc79bc7", - "trust_remote_code": True, + "path": "mteb/mlsum", + "revision": "b4efe498c4d0b9d7bdd2905f6fff4e22ae251d00", }, type="Clustering", category="s2s", @@ -41,7 +40,7 @@ class MLSUMClusteringS2S(AbsTaskClustering, MultilingualTask): date=("2010-01-01", "2018-09-30"), domains=["News", "Written"], task_subtypes=["Topic classification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", @@ -96,11 +95,10 @@ class MLSUMClusteringS2SFast(AbsTaskClusteringFast, MultilingualTask): metadata = TaskMetadata( name="MLSUMClusteringS2S.v2", description="Clustering of newspaper article contents and titles from MLSUM dataset. Clustering of 10 sets on the newpaper article topics.", - reference="https://huggingface.co/datasets/mlsum", + reference="https://huggingface.co/datasets/mteb/mlsum", dataset={ - "path": "reciTAL/mlsum", - "revision": "b5d54f8f3b61ae17845046286940f03c6bc79bc7", - "trust_remote_code": True, + "path": "mteb/mlsum", + "revision": "b4efe498c4d0b9d7bdd2905f6fff4e22ae251d00", }, type="Clustering", category="s2s", @@ -111,7 +109,7 @@ class MLSUMClusteringS2SFast(AbsTaskClusteringFast, MultilingualTask): date=("2010-01-01", "2018-09-30"), domains=["News", "Written"], task_subtypes=["Topic classification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py b/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py index 3fda487885..c74b3ac52d 100644 --- a/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py @@ -49,7 +49,7 @@ class MasakhaNEWSClusteringP2P(AbsTaskClustering, MultilingualTask): ), # best guess (not found in paper, dataset or datasheet) domains=["News", "Written", "Non-fiction"], task_subtypes=["Thematic clustering"], - license="AFL-3.0", + license="afl-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/nob/SNLHierarchicalClustering.py b/mteb/tasks/Clustering/nob/SNLHierarchicalClustering.py index 3cdaa8c88b..3ec8e8e3ef 100644 --- a/mteb/tasks/Clustering/nob/SNLHierarchicalClustering.py +++ b/mteb/tasks/Clustering/nob/SNLHierarchicalClustering.py @@ -31,7 +31,7 @@ class SNLHierarchicalClusteringP2P(AbsTaskClusteringFast): main_score="v_measure", date=("2020-01-01", "2024-12-31"), # best guess domains=["Encyclopaedic", "Non-fiction", "Written"], - license="CC-BY-NC", + license="cc-by-nc-4.0", # version assumed (not specified beforehand) annotations_creators="derived", dialect=[], task_subtypes=["Thematic clustering"], @@ -76,7 +76,7 @@ class SNLHierarchicalClusteringS2S(AbsTaskClusteringFast): main_score="v_measure", date=("2020-01-01", "2024-12-31"), # best guess domains=["Encyclopaedic", "Non-fiction", "Written"], - license="CC-BY-NC", + license="cc-by-nc-4.0", # version assumed (not specified beforehand) annotations_creators="derived", dialect=[], task_subtypes=["Thematic clustering"], diff --git a/mteb/tasks/Clustering/nob/VGHierarchicalClustering.py b/mteb/tasks/Clustering/nob/VGHierarchicalClustering.py index 16f02b39e7..ff708d00a3 100644 --- a/mteb/tasks/Clustering/nob/VGHierarchicalClustering.py +++ b/mteb/tasks/Clustering/nob/VGHierarchicalClustering.py @@ -31,7 +31,7 @@ class VGHierarchicalClusteringP2P(AbsTaskClusteringFast): main_score="v_measure", date=("2020-01-01", "2024-12-31"), # best guess domains=["News", "Non-fiction", "Written"], - license="CC-BY-NC 4.0", + license="cc-by-nc-4.0", annotations_creators="derived", dialect=[], task_subtypes=["Thematic clustering"], @@ -79,7 +79,7 @@ class VGHierarchicalClusteringS2S(AbsTaskClusteringFast): main_score="v_measure", date=("2020-01-01", "2024-12-31"), # best guess domains=["News", "Non-fiction", "Written"], - license="CC-BY-NC 4.0", + license="cc-by-nc-4.0", annotations_creators="derived", dialect=[], task_subtypes=["Thematic clustering"], diff --git a/mteb/tasks/Clustering/nob/snl_clustering.py b/mteb/tasks/Clustering/nob/snl_clustering.py index cdfd2eb9c5..0acb6d52b1 100644 --- a/mteb/tasks/Clustering/nob/snl_clustering.py +++ b/mteb/tasks/Clustering/nob/snl_clustering.py @@ -1,8 +1,9 @@ from __future__ import annotations import random +from collections.abc import Iterable from itertools import islice -from typing import Iterable, TypeVar +from typing import TypeVar import datasets @@ -39,7 +40,7 @@ class SNLClustering(AbsTaskClustering): main_score="v_measure", date=("2020-01-01", "2024-12-31"), # best guess domains=["Encyclopaedic", "Non-fiction", "Written"], - license="CC-BY-NC", + license="cc-by-nc-4.0", # version is assumed (not specified before) annotations_creators="derived", dialect=[], task_subtypes=["Thematic clustering"], diff --git a/mteb/tasks/Clustering/nob/vg_clustering.py b/mteb/tasks/Clustering/nob/vg_clustering.py index 769f69da1a..6c6c692fb7 100644 --- a/mteb/tasks/Clustering/nob/vg_clustering.py +++ b/mteb/tasks/Clustering/nob/vg_clustering.py @@ -1,8 +1,9 @@ from __future__ import annotations import random +from collections.abc import Iterable from itertools import islice -from typing import Iterable, TypeVar +from typing import TypeVar import datasets diff --git a/mteb/tasks/Clustering/pol/PolishClustering.py b/mteb/tasks/Clustering/pol/PolishClustering.py index 7c11cb6148..cf87fe9fa8 100644 --- a/mteb/tasks/Clustering/pol/PolishClustering.py +++ b/mteb/tasks/Clustering/pol/PolishClustering.py @@ -35,7 +35,7 @@ class EightTagsClustering(AbsTaskClustering): date=("2019-01-01", "2020-05-01"), domains=["Social", "Written"], task_subtypes=["Topic classification", "Thematic clustering"], - license="GPL-3.0", + license="gpl-3.0", annotations_creators="derived", dialect=[], sample_creation="found", @@ -98,7 +98,7 @@ class EightTagsClusteringFast(AbsTaskClusteringFast): date=("2019-01-01", "2020-05-01"), domains=["Social", "Written"], task_subtypes=["Topic classification", "Thematic clustering"], - license="GPL-3.0", + license="gpl-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/rom/RomaniBibleClustering.py b/mteb/tasks/Clustering/rom/RomaniBibleClustering.py index 8d242b76be..8801261ea8 100644 --- a/mteb/tasks/Clustering/rom/RomaniBibleClustering.py +++ b/mteb/tasks/Clustering/rom/RomaniBibleClustering.py @@ -22,7 +22,7 @@ class RomaniBibleClustering(AbsTaskClustering): date=("2020-01-01", "2020-12-31"), domains=["Religious", "Written"], task_subtypes=["Thematic clustering"], - license="MIT", + license="mit", annotations_creators="derived", dialect=["Kalderash"], sample_creation="human-translated and localized", diff --git a/mteb/tasks/Clustering/rus/RuSciBenchGRNTIClusteringP2P.py b/mteb/tasks/Clustering/rus/RuSciBenchGRNTIClusteringP2P.py index 16788dc46d..dab6be4db9 100644 --- a/mteb/tasks/Clustering/rus/RuSciBenchGRNTIClusteringP2P.py +++ b/mteb/tasks/Clustering/rus/RuSciBenchGRNTIClusteringP2P.py @@ -27,7 +27,7 @@ class RuSciBenchGRNTIClusteringP2P(AbsTaskClusteringFast): date=("1999-01-01", "2024-01-01"), domains=["Academic", "Written"], task_subtypes=["Thematic clustering"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/rus/RuSciBenchOECDClusteringP2P.py b/mteb/tasks/Clustering/rus/RuSciBenchOECDClusteringP2P.py index 56145b0943..25a27ea264 100644 --- a/mteb/tasks/Clustering/rus/RuSciBenchOECDClusteringP2P.py +++ b/mteb/tasks/Clustering/rus/RuSciBenchOECDClusteringP2P.py @@ -27,7 +27,7 @@ class RuSciBenchOECDClusteringP2P(AbsTaskClusteringFast): date=("1999-01-01", "2024-01-01"), domains=["Academic", "Written"], task_subtypes=["Thematic clustering"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Clustering/zho/CMTEBClustering.py b/mteb/tasks/Clustering/zho/CMTEBClustering.py index 28360bf3b3..4ce12d9bbd 100644 --- a/mteb/tasks/Clustering/zho/CMTEBClustering.py +++ b/mteb/tasks/Clustering/zho/CMTEBClustering.py @@ -35,7 +35,7 @@ class CLSClusteringFastS2S(AbsTaskClusteringFast): date=("2022-01-01", "2022-09-12"), domains=["Academic", "Written"], task_subtypes=["Thematic clustering", "Topic classification"], - license="Apache-2.0", + license="apache-2.0", annotations_creators="derived", dialect=[], sample_creation="found", @@ -95,7 +95,7 @@ class CLSClusteringFastP2P(AbsTaskClusteringFast): date=("2022-01-01", "2022-09-12"), domains=["Academic", "Written"], task_subtypes=["Thematic clustering", "Topic classification"], - license="Apache-2.0", + license="apache-2.0", annotations_creators="derived", dialect=[], sample_creation="found", @@ -225,7 +225,7 @@ class ThuNewsClusteringFastS2S(AbsTaskClusteringFast): date=("2006-01-01", "2007-01-01"), domains=["News", "Written"], task_subtypes=["Thematic clustering", "Topic classification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", @@ -285,7 +285,7 @@ class ThuNewsClusteringFastP2P(AbsTaskClusteringFast): date=("2006-01-01", "2007-01-01"), domains=["News", "Written"], task_subtypes=["Thematic clustering", "Topic classification"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py index 46fbb5b990..7c9ab2e721 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py @@ -19,11 +19,9 @@ class ImageCoDeT2IMultiChoice(AbsTaskAny2AnyMultiChoice): eval_langs=["eng-Latn"], main_score="ndcg_at_1", date=("2022-05-22", "2022-05-27"), # conference dates - form=["written"], - domains=["Web"], + domains=["Web", "Written"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", - socioeconomic_status="medium", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py index cd1ce67e4a..6b133f47c3 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py @@ -22,7 +22,7 @@ class BLINKIT2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2018-01-01", "2018-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py index 36d7434575..da63f01df4 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py @@ -22,7 +22,7 @@ class BLINKIT2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2018-01-01", "2018-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py index b215dfda06..81e9a13328 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py @@ -22,7 +22,7 @@ class CIRRIT2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2018-01-01", "2018-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py index 048f5a33bf..95a3c9a77c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CUB200I2IRetrieval.py @@ -21,7 +21,7 @@ class CUB200I2I(AbsTaskAny2AnyRetrieval): date=("2009-01-01", "2010-04-01"), domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py index c358ee6507..a07a88b7b8 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py @@ -21,7 +21,7 @@ class FORBI2I(AbsTaskAny2AnyRetrieval): date=("2022-01-01", "2023-01-01"), domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py index 04fad6e352..0acb328f14 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py @@ -12,7 +12,6 @@ class Fashion200kI2TRetrieval(AbsTaskAny2AnyRetrieval): dataset={ "path": "MRBench/mbeir_fashion200k_task3", "revision": "96a313715ecf67f5dfe70c4fa52406bc7bdfbeee", - # "trust_remote_code": True, }, type="Any2AnyRetrieval", category="i2t", @@ -22,7 +21,7 @@ class Fashion200kI2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2017-01-01", "2017-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="Apache-2.0", + license="apache-2.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py index 54a1c24cf1..2648fb8ee1 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py @@ -22,7 +22,7 @@ class Fashion200kT2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2017-01-01", "2017-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="Apache-2.0", + license="apache-2.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py index 45b8e10576..fd78a663cf 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py @@ -22,7 +22,7 @@ class FashionIQIT2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2021-01-01", "2021-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="Apache-2.0", + license="apache-2.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py index 267cf860b9..3a33733e1f 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py @@ -19,11 +19,9 @@ class Flickr30kI2TRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2018-01-01", "2018-12-31"), - form=["written"], - domains=["Web"], + domains=["Web", "Written"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", - socioeconomic_status="medium", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py index 576e6afa50..585fcfc255 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py @@ -19,11 +19,9 @@ class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2018-01-01", "2018-12-31"), - form=["written"], - domains=["Web"], + domains=["Web", "Written"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", - socioeconomic_status="medium", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py index 67b238a470..db61790fb8 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2TRetrieval.py @@ -21,7 +21,7 @@ class GLDv2I2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2017-01-01", "2017-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="Apache-2.0", + license="apache-2.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py index 40323d3636..bf7e273d73 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesI2TRetrieval.py @@ -76,7 +76,7 @@ class HatefulMemesI2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2020-01-01", "2020-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py index fec70177db..89912a1213 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/HatefulMemesT2IRetrieval.py @@ -76,7 +76,7 @@ class HatefulMemesT2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2020-01-01", "2020-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py index 6e34640459..1b8472294c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ImageCoDeT2IRetrieval.py @@ -19,11 +19,9 @@ class ImageCoDeT2IRetrieval(AbsTaskAny2AnyRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2022-05-22", "2022-05-27"), # conference dates - form=["written"], - domains=["Web"], + domains=["Web", "Written"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", - socioeconomic_status="medium", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py index abc71666aa..38caf36f1a 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py @@ -22,7 +22,7 @@ class InfoSeekIT2ITRetrieval(AbsTaskAny2AnyRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py index a856969e75..ac8861f0aa 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py @@ -22,7 +22,7 @@ class InfoSeekIT2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py index b0578c5944..399c1fb792 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/METI2IRetrieval.py @@ -21,7 +21,7 @@ class METI2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2021-12-06", "2021-12-14"), # conference dates domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py index 8652b2e8e0..09cfcdec1f 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py @@ -22,7 +22,7 @@ class MSCOCOI2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2018-01-01", "2018-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py index 4797e98911..f47575b9b4 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py @@ -22,7 +22,7 @@ class MSCOCOT2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2018-01-01", "2018-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py index c9d671d9d6..dfc42881df 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionI2TRetrieval.py @@ -105,7 +105,7 @@ class MemotionI2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2020-01-01", "2020-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py index 331e628f24..dff7746b5a 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MemotionT2IRetrieval.py @@ -104,7 +104,7 @@ class MemotionT2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2020-01-01", "2020-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py index ae0d91a6b5..aed1805aae 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py @@ -21,7 +21,7 @@ class NIGHTSI2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic"], task_subtypes=["Duplicate Image Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py index 9bac08fa34..f355c43ecf 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py @@ -21,7 +21,7 @@ class OVENIT2ITRetrieval(AbsTaskAny2AnyRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py index 0877cfdf33..308283454c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py @@ -21,7 +21,7 @@ class OVENIT2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py index 2fb96a2365..e338eec2d1 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py @@ -21,7 +21,7 @@ class ROxfordEasyI2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2009-01-01", "2010-04-01"), domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], @@ -67,7 +67,7 @@ class ROxfordMediumI2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2009-01-01", "2010-04-01"), domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], @@ -113,7 +113,7 @@ class ROxfordHardI2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2009-01-01", "2010-04-01"), domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py index 61cd189fce..5d08f1ef91 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RP2kI2IRetrieval.py @@ -21,7 +21,7 @@ class RP2kI2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2009-01-01", "2010-04-01"), domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py index e625258053..f29a9849ef 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py @@ -21,7 +21,7 @@ class RParisEasyI2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2009-01-01", "2010-04-01"), domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], @@ -67,7 +67,7 @@ class RParisMediumI2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2009-01-01", "2010-04-01"), domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], @@ -113,7 +113,7 @@ class RParisHardI2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2009-01-01", "2010-04-01"), domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py index 5d754fe0e6..0558f0ce26 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SOPI2IRetrieval.py @@ -21,7 +21,7 @@ class SOPI2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2019-07-17", "2019-07-17"), domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py index 0f7acedab0..a8aac928c4 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRI2TRetrieval.py @@ -81,7 +81,7 @@ class SciMMIRI2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py index 987a00ea6d..41fa6aebc1 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SciMMIRT2IRetrieval.py @@ -81,7 +81,7 @@ class SciMMIRT2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py index 5a4b13ec94..002e1a39e2 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/SketchyI2IRetrieval.py @@ -21,7 +21,7 @@ class SketchyI2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2021-12-06", "2021-12-14"), # conference dates domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py index e98633e899..e8d267eeaa 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/StanfordCarsI2IRetrieval.py @@ -21,7 +21,7 @@ class StanfordCarsI2I(AbsTaskAny2AnyRetrieval): date=("2012-01-01", "2013-04-01"), domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py index fe1c2891db..b85cd1f94b 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/TUBerlinT2IRetrieval.py @@ -22,7 +22,7 @@ class TUBerlinT2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2012-01-01", "2012-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py index 58e1c5d31e..39f07cf945 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VQA2IT2TRetrieval.py @@ -22,7 +22,7 @@ class VQA2IT2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2017-07-01", "2017-07-01"), domains=["Web"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py index fc73789541..8b8b232e1d 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py @@ -108,7 +108,7 @@ class VidoreArxivQARetrieval(AbsTaskAny2AnyRetrieval): date=("2024-01-01", "2024-07-01"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -161,7 +161,7 @@ class VidoreDocVQARetrieval(AbsTaskAny2AnyRetrieval): date=("2024-01-01", "2024-07-01"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -214,7 +214,7 @@ class VidoreInfoVQARetrieval(AbsTaskAny2AnyRetrieval): date=("2024-01-01", "2024-07-01"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -267,7 +267,7 @@ class VidoreTabfquadRetrieval(AbsTaskAny2AnyRetrieval): date=("2024-01-01", "2024-07-01"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -320,7 +320,7 @@ class VidoreTatdqaRetrieval(AbsTaskAny2AnyRetrieval): date=("2024-01-01", "2024-07-01"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -373,7 +373,7 @@ class VidoreShiftProjectRetrieval(AbsTaskAny2AnyRetrieval): date=("2024-01-01", "2024-07-01"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -426,7 +426,7 @@ class VidoreSyntheticDocQAAIRetrieval(AbsTaskAny2AnyRetrieval): date=("2024-01-01", "2024-07-01"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -479,7 +479,7 @@ class VidoreSyntheticDocQAEnergyRetrieval(AbsTaskAny2AnyRetrieval): date=("2024-01-01", "2024-07-01"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -532,7 +532,7 @@ class VidoreSyntheticDocQAGovernmentReportsRetrieval(AbsTaskAny2AnyRetrieval): date=("2024-01-01", "2024-07-01"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -585,7 +585,7 @@ class VidoreSyntheticDocQAHealthcareIndustryRetrieval(AbsTaskAny2AnyRetrieval): date=("2024-01-01", "2024-07-01"), domains=["Academic"], task_subtypes=["Image Text Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py index a36f5ea5fe..ecce9f1e9a 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py @@ -21,7 +21,7 @@ class VisualNewsI2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2020-01-01", "2020-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py index aae9882d52..c700a5ab3c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py @@ -21,7 +21,7 @@ class VisualNewsT2IRetrieval(AbsTaskAny2AnyRetrieval): date=("2020-01-01", "2020-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py index 076f003b2b..96bcac96c3 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VizWizIT2TRetrieval.py @@ -22,7 +22,7 @@ class VizWizIT2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2018-01-01", "2018-01-01"), domains=["Web"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py index fabbf48ed4..583cae54dd 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py @@ -21,7 +21,7 @@ class WebQAT2ITRetrieval(AbsTaskAny2AnyRetrieval): date=("2022-01-01", "2022-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py index a98ee514a9..c431af4394 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py @@ -21,7 +21,7 @@ class WebQAT2TRetrieval(AbsTaskAny2AnyRetrieval): date=("2022-01-01", "2022-12-31"), domains=["Encyclopaedic"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py index ee8b8c4148..5f20e45d25 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py @@ -107,11 +107,9 @@ class WITT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): eval_langs=_LANGUAGES, main_score="ndcg_at_10", date=("2022-01-01", "2022-12-31"), - form=["written"], - domains=["Encyclopaedic"], + domains=["Encyclopaedic", "Written"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", - socioeconomic_status="medium", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py index 507639a4df..98b45006a2 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py @@ -90,11 +90,9 @@ class XFlickr30kCoT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): eval_langs=_LANGUAGES, main_score="ndcg_at_10", date=("2022-01-01", "2022-12-31"), - form=["written"], - domains=["Encyclopaedic"], + domains=["Encyclopaedic", "Written"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", - socioeconomic_status="medium", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py index 7e78db8193..a65d37f324 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py @@ -135,11 +135,9 @@ class XM3600T2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): eval_langs=_LANGUAGES, main_score="ndcg_at_10", date=("2022-01-01", "2022-12-31"), - form=["written"], - domains=["Encyclopaedic"], + domains=["Encyclopaedic", "Written"], task_subtypes=["Image Text Retrieval"], - license="CC BY-SA 4.0", - socioeconomic_status="medium", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py b/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py index bb3c5db181..3f387fdcbf 100644 --- a/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py +++ b/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py @@ -31,7 +31,7 @@ class CVBenchCount(AbsTaskAny2TextMultipleChoice): date=("2024-01-01", "2024-06-24"), domains=["Academic"], task_subtypes=["Question answering"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -91,7 +91,7 @@ class CVBenchRelation(AbsTaskAny2TextMultipleChoice): date=("2024-01-01", "2024-06-24"), domains=["Academic"], task_subtypes=["Question answering"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -153,7 +153,7 @@ class CVBenchDepth(AbsTaskAny2TextMultipleChoice): date=("2024-01-01", "2024-06-24"), domains=["Academic"], task_subtypes=["Question answering"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -213,7 +213,7 @@ class CVBenchDistance(AbsTaskAny2TextMultipleChoice): date=("2024-01-01", "2024-06-24"), domains=["Academic"], task_subtypes=["Question answering"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/Clustering/eng/CIFAR.py b/mteb/tasks/Image/Clustering/eng/CIFAR.py index e9a0429cf9..f20145dbfe 100644 --- a/mteb/tasks/Image/Clustering/eng/CIFAR.py +++ b/mteb/tasks/Image/Clustering/eng/CIFAR.py @@ -24,7 +24,7 @@ class CIFAR10Clustering(AbsTaskImageClustering): ), # Estimated range for the collection of reviews domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], @@ -64,8 +64,7 @@ class CIFAR100Clustering(AbsTaskImageClustering): ), # Estimated range for the collection of reviews domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Clustering/eng/ImageNet.py b/mteb/tasks/Image/Clustering/eng/ImageNet.py index 381b3d9d91..44c9584ebf 100644 --- a/mteb/tasks/Image/Clustering/eng/ImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/ImageNet.py @@ -21,7 +21,7 @@ class ImageNetDog15Clustering(AbsTaskImageClustering): date=("2009-06-20", "2009-06-20"), # Conference date domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], @@ -61,8 +61,7 @@ class ImageNet10Clustering(AbsTaskImageClustering): date=("2009-06-20", "2009-06-20"), # Conference date domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py index 96c557e3ab..95d0ee3246 100644 --- a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py @@ -24,7 +24,7 @@ class TinyImageNet(AbsTaskImageClustering): ), # Estimated range for the collection of reviews domains=["Reviews"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py index cf9a18cc46..4e90a5e30d 100644 --- a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py @@ -24,7 +24,7 @@ class BirdsnapClassification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py index 7560bff77e..c6e7ef6b16 100644 --- a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py +++ b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py @@ -24,7 +24,7 @@ class CIFAR10Classification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], @@ -64,8 +64,7 @@ class CIFAR100Classification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py index 63b0b622ad..76843f50ba 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py @@ -26,7 +26,7 @@ class Caltech101Classification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py index 7fcbd4b209..14427cd530 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py @@ -24,7 +24,7 @@ class Country211Classification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Scene"], task_subtypes=["Scene recognition"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py index 6362785cb7..48a6ca5243 100644 --- a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py @@ -24,7 +24,7 @@ class DTDClassification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Textures recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py index 45638643de..588cce89aa 100644 --- a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py @@ -24,7 +24,7 @@ class EuroSATClassification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Scene recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py index 49323aa4b6..81c2fc5857 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py @@ -24,7 +24,7 @@ class FER2013Classification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Emotion recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py index aac18c7e57..2971faf863 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py @@ -25,7 +25,7 @@ class FGVCAircraftClassification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py index c508486997..a0dee80ad0 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py @@ -24,7 +24,7 @@ class Food101Classification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py index 29c5ccc4c0..4a77c47e98 100644 --- a/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py @@ -24,7 +24,7 @@ class GTSRBClassification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews task_subtypes=["Activity recognition"], domains=["Scene"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py index c8bfd62ce8..ff91119015 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py +++ b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py @@ -24,7 +24,7 @@ class Imagenet1kClassification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Scene"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py index 0956e0d1c3..7e4b81f3f6 100644 --- a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py @@ -24,7 +24,7 @@ class MNISTClassification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py index 9e4a6f6aaa..316107534f 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -24,7 +24,7 @@ class OxfordFlowersClassification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Reviews"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py index 09935a2735..39620326ab 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py @@ -24,7 +24,7 @@ class OxfordPetsClassification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py index 0032266eaa..e2518dc221 100644 --- a/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py @@ -24,7 +24,7 @@ class PatchCamelyonClassification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Medical"], task_subtypes=["Tumor detection"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py index d73abc76b9..b14d31cdc0 100644 --- a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py @@ -24,7 +24,7 @@ class RESISC45Classification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py index fe25c9d3d7..7acd6bb0eb 100644 --- a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py @@ -24,7 +24,7 @@ class STL10Classification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py index f7593a3373..4d0f987564 100644 --- a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py @@ -24,7 +24,7 @@ class SUN397Classification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Scene recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py index 4c049f540d..f9836e4b05 100644 --- a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py @@ -24,7 +24,7 @@ class StanfordCarsClassification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py index 41c923b538..68d2cb74b7 100644 --- a/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py @@ -28,7 +28,7 @@ class UCF101Classification(AbsTaskImageClassification): ), # Estimated range for the collection of reviews domains=["Scene"], task_subtypes=["Activity recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index 057d631784..bd5d0aad6d 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -28,7 +28,7 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py b/mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py index c72ef004bd..ac5f03127e 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py +++ b/mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py @@ -36,7 +36,7 @@ class AROCocoOrder(AbsTaskImageTextPairClassification): ), # Estimated range for the collection of data domains=["Encyclopaedic"], task_subtypes=["Caption Pairing"], - license="MIT", + license="mit", annotations_creators="expert-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py b/mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py index bd8ec152bf..18faadaf23 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py +++ b/mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py @@ -36,7 +36,7 @@ class AROFlickrOrder(AbsTaskImageTextPairClassification): ), # Estimated range for the collection of data domains=["Encyclopaedic"], task_subtypes=["Caption Pairing"], - license="MIT", + license="mit", annotations_creators="expert-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py b/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py index 83755ca489..4f75db410b 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py +++ b/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py @@ -29,7 +29,7 @@ class AROVisualAttribution(AbsTaskImageTextPairClassification): ), # Estimated range for the collection of data domains=["Encyclopaedic"], task_subtypes=["Caption Pairing"], - license="MIT", + license="mit", annotations_creators="expert-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py b/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py index 6d222b90bd..fef938271e 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py +++ b/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py @@ -29,7 +29,7 @@ class AROVisualRelation(AbsTaskImageTextPairClassification): ), # Estimated range for the collection of data domains=["Encyclopaedic"], task_subtypes=["Caption Pairing"], - license="MIT", + license="mit", annotations_creators="expert-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py b/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py index 1db8fd6563..b410cbacd5 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py +++ b/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py @@ -31,7 +31,7 @@ class SugarCrepe(AbsTaskImageTextPairClassification): ), # Estimated range for the collection of data domains=["Encyclopaedic"], task_subtypes=["Caption Pairing"], - license="MIT", + license="mit", annotations_creators="expert-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py index e598b9f4c1..0b8a8bedd7 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py +++ b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py @@ -30,7 +30,7 @@ class Winoground(AbsTaskImageTextPairClassification): ), # Estimated range for the collection of data domains=["Social"], # Getty Images. Could be constructed? task_subtypes=["Caption Pairing"], - license="META Images Reseaerch License", + license="https://huggingface.co/datasets/facebook/winoground/blob/main/license_agreement.txt", annotations_creators="expert-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py index c036b54042..09d550547f 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py @@ -22,7 +22,7 @@ class STS12VisualSTS(AbsTaskVisualSTS): date=("2005-01-01", "2012-12-31"), domains=["Encyclopaedic", "News", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="rendered", diff --git a/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py index cf4c0aa6c4..771e9e0ce8 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py @@ -22,7 +22,7 @@ class STS13VisualSTS(AbsTaskVisualSTS): date=("2012-01-01", "2012-12-31"), domains=["Web", "News", "Non-fiction", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="rendered", diff --git a/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py index 46dce36f80..299e54dca9 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py @@ -23,7 +23,7 @@ class STS14VisualSTS(AbsTaskVisualSTS): date=("2012-01-01", "2012-08-31"), domains=["Blog", "Web", "Spoken"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="rendered", diff --git a/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py index a9aca02c39..1756cdc55c 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py @@ -22,7 +22,7 @@ class STS15VisualSTS(AbsTaskVisualSTS): date=("2008-01-01", "2014-07-28"), domains=["Blog", "News", "Web", "Written", "Spoken"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="rendered", diff --git a/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py index b64e040282..dba6e4af63 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py @@ -22,7 +22,7 @@ class STS16VisualSTS(AbsTaskVisualSTS): date=("2015-10-01", "2015-12-31"), domains=["Blog", "Web", "Spoken"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="rendered", diff --git a/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py b/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py index b72988676a..068fd33b9c 100644 --- a/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py @@ -42,7 +42,7 @@ class STS17MultilingualVisualSTS(AbsTaskVisualSTS, MultilingualTask): date=("2012-01-01", "2017-12-31"), domains=["News", "Social", "Web", "Spoken", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="rendered", diff --git a/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py b/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py index 339be27c37..ce8c047655 100644 --- a/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py @@ -43,7 +43,7 @@ class STSBenchmarkMultilingualVisualSTS(AbsTaskVisualSTS, MultilingualTask): date=("2012-01-01", "2017-12-31"), domains=["News", "Social", "Web", "Spoken", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="rendered", diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py b/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py index ed31e3f89f..2eac4b0214 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py @@ -26,8 +26,7 @@ class BirdsnapClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py b/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py index 81103a0f1d..91b0b159ef 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py @@ -26,8 +26,7 @@ class CIFAR10ZeroShotClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["text", "image"], @@ -73,8 +72,7 @@ class CIFAR100ZeroShotClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py b/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py index fe89f06d42..c96c2727c4 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py @@ -23,8 +23,7 @@ class CLEVR(AbsTaskZeroshotClassification): date=("2016-01-01", "2016-12-20"), domains=["Constructed"], task_subtypes=["Object recognition"], - license="""""", - socioeconomic_status="mixed", + license="not specified", annotations_creators="human-annotated", dialect=[], modalities=["text", "image"], @@ -76,8 +75,7 @@ class CLEVRCount(AbsTaskZeroshotClassification): date=("2016-01-01", "2016-12-20"), domains=["Constructed"], task_subtypes=["Object recognition"], - license="""""", - socioeconomic_status="mixed", + license="not specified", annotations_creators="human-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py b/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py index 00bfdac874..749ac71273 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py @@ -28,8 +28,7 @@ class Caltech101Classification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Country211.py b/mteb/tasks/Image/ZeroshotClassification/eng/Country211.py index 0a60e33003..eb0dd5158b 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Country211.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Country211.py @@ -28,8 +28,7 @@ class Country211Classification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Scene"], task_subtypes=["Scene recognition"], - license="CC BY-SA 4.0", - socioeconomic_status="mixed", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/DTD.py b/mteb/tasks/Image/ZeroshotClassification/eng/DTD.py index 27ef0a6f3d..2d182e0854 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/DTD.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/DTD.py @@ -26,8 +26,7 @@ class DTDClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Textures recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py b/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py index de6fb4c434..85a1b13e5d 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/EuroSAT.py @@ -26,8 +26,7 @@ class EuroSATClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Scene recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py b/mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py index 9cfa0dd3e9..a0a391e235 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/FER2013.py @@ -26,8 +26,7 @@ class FER2013Classification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Emotion recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py b/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py index b973c85607..65af473d3f 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py @@ -27,8 +27,7 @@ class FGVCAircraftClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Food101.py b/mteb/tasks/Image/ZeroshotClassification/eng/Food101.py index fd073ac412..cc64484e65 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Food101.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Food101.py @@ -26,8 +26,7 @@ class Food101Classification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Web"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/GTSRB.py b/mteb/tasks/Image/ZeroshotClassification/eng/GTSRB.py index 5a5d662b16..e08866b6bd 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/GTSRB.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/GTSRB.py @@ -28,8 +28,7 @@ class GTSRBClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews task_subtypes=["Activity recognition"], domains=["Scene"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py b/mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py index 87dc8e277e..53dce7feb1 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Imagenet1k.py @@ -28,8 +28,7 @@ class Imagenet1kClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Scene"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="human-annotated", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py b/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py index c274e72d61..6433104c90 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py @@ -26,8 +26,7 @@ class MNISTClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py b/mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py index 3da580af1b..372d2fa7bf 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/OxfordPets.py @@ -26,8 +26,7 @@ class OxfordPetsClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py b/mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py index 9a49edfedd..196026f971 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py @@ -28,8 +28,7 @@ class PatchCamelyonClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Medical"], task_subtypes=["Tumor detection"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py b/mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py index d6fb98ba6c..e58da7863e 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/RESISC45.py @@ -26,8 +26,7 @@ class RESISC45Classification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py index 8188c7b339..cad88534a2 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py @@ -24,7 +24,6 @@ class RenderedSST2(AbsTaskZeroshotClassification): domains=["Reviews"], task_subtypes=[], license="mit", - socioeconomic_status="mixed", annotations_creators="human-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/STL10.py b/mteb/tasks/Image/ZeroshotClassification/eng/STL10.py index 8b0f42d08d..67357adc88 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/STL10.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/STL10.py @@ -26,8 +26,7 @@ class STL10Classification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Object recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py b/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py index 64252584b8..c28bf146f1 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/SUN397.py @@ -26,8 +26,7 @@ class SUN397Classification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Encyclopaedic"], task_subtypes=["Scene recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/SciMMIR.py b/mteb/tasks/Image/ZeroshotClassification/eng/SciMMIR.py index fddd4dc2a8..abb651612f 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/SciMMIR.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/SciMMIR.py @@ -23,8 +23,7 @@ class SciMMIR(AbsTaskZeroshotClassification): date=("2023-05-01", "2023-10-30"), domains=["Academic"], task_subtypes=["Caption Pairing", "Rendered Texts Understanding"], - license="""""", - socioeconomic_status="mixed", + license="not specified", annotations_creators="human-annotated", dialect=[], modalities=["text", "image"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py b/mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py index c8cc639a4e..d3e01a34ca 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/StanfordCars.py @@ -26,8 +26,7 @@ class StanfordCarsClassification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Scene"], task_subtypes=["Scene recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image", "text"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py b/mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py index b95021184c..b0d5293632 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/UCF101.py @@ -30,8 +30,7 @@ class UCF101Classification(AbsTaskZeroshotClassification): ), # Estimated range for the collection of reviews domains=["Scene"], task_subtypes=["Activity recognition"], - license="Not specified", - socioeconomic_status="mixed", + license="not specified", annotations_creators="derived", dialect=[], modalities=["image"], diff --git a/mteb/tasks/InstructionRetrieval/__init__.py b/mteb/tasks/InstructionRetrieval/__init__.py index f032908014..f5e812247d 100644 --- a/mteb/tasks/InstructionRetrieval/__init__.py +++ b/mteb/tasks/InstructionRetrieval/__init__.py @@ -3,3 +3,4 @@ from .eng.Core17InstructionRetrieval import * from .eng.News21InstructionRetrieval import * from .eng.Robust04InstructionRetrieval import * +from .multilingual.mFollowIR import * diff --git a/mteb/tasks/InstructionRetrieval/eng/Core17InstructionRetrieval.py b/mteb/tasks/InstructionRetrieval/eng/Core17InstructionRetrieval.py index b7b86bae91..9b52f282b2 100644 --- a/mteb/tasks/InstructionRetrieval/eng/Core17InstructionRetrieval.py +++ b/mteb/tasks/InstructionRetrieval/eng/Core17InstructionRetrieval.py @@ -8,7 +8,7 @@ class Core17InstructionRetrieval(AbsTaskInstructionRetrieval): metadata = TaskMetadata( name="Core17InstructionRetrieval", - description="Measuring retrieval instruction following ability on Core17 narratives.", + description="Measuring retrieval instruction following ability on Core17 narratives for the FollowIR benchmark.", reference="https://arxiv.org/abs/2403.15246", dataset={ "path": "jhu-clsp/core17-instructions", @@ -23,7 +23,7 @@ class Core17InstructionRetrieval(AbsTaskInstructionRetrieval): date=("2023-08-01", "2024-04-01"), domains=["News", "Written"], task_subtypes=[], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/InstructionRetrieval/eng/News21InstructionRetrieval.py b/mteb/tasks/InstructionRetrieval/eng/News21InstructionRetrieval.py index ab7d96820c..d693091279 100644 --- a/mteb/tasks/InstructionRetrieval/eng/News21InstructionRetrieval.py +++ b/mteb/tasks/InstructionRetrieval/eng/News21InstructionRetrieval.py @@ -8,7 +8,7 @@ class News21InstructionRetrieval(AbsTaskInstructionRetrieval): metadata = TaskMetadata( name="News21InstructionRetrieval", - description="Measuring retrieval instruction following ability on News21 narratives.", + description="Measuring retrieval instruction following ability on News21 narratives for the FollowIR benchmark.", reference="https://arxiv.org/abs/2403.15246", dataset={ "path": "jhu-clsp/news21-instructions", @@ -23,7 +23,7 @@ class News21InstructionRetrieval(AbsTaskInstructionRetrieval): date=("2023-08-01", "2024-04-01"), domains=["News", "Written"], task_subtypes=[], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/InstructionRetrieval/eng/Robust04InstructionRetrieval.py b/mteb/tasks/InstructionRetrieval/eng/Robust04InstructionRetrieval.py index 658b5f46df..c68dfabc18 100644 --- a/mteb/tasks/InstructionRetrieval/eng/Robust04InstructionRetrieval.py +++ b/mteb/tasks/InstructionRetrieval/eng/Robust04InstructionRetrieval.py @@ -8,7 +8,7 @@ class Robust04InstructionRetrieval(AbsTaskInstructionRetrieval): metadata = TaskMetadata( name="Robust04InstructionRetrieval", - description="Measuring retrieval instruction following ability on Robust04 narratives.", + description="Measuring retrieval instruction following ability on Robust04 narratives for the FollowIR benchmark.", reference="https://arxiv.org/abs/2403.15246", dataset={ "path": "jhu-clsp/robust04-instructions", @@ -23,7 +23,7 @@ class Robust04InstructionRetrieval(AbsTaskInstructionRetrieval): date=("2023-08-01", "2024-04-01"), domains=["News", "Written"], task_subtypes=[], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/InstructionRetrieval/multilingual/__init__.py b/mteb/tasks/InstructionRetrieval/multilingual/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/InstructionRetrieval/multilingual/mFollowIR.py b/mteb/tasks/InstructionRetrieval/multilingual/mFollowIR.py new file mode 100644 index 0000000000..04c1a56e19 --- /dev/null +++ b/mteb/tasks/InstructionRetrieval/multilingual/mFollowIR.py @@ -0,0 +1,368 @@ +from __future__ import annotations + +from collections import defaultdict + +import datasets + +from mteb.abstasks.MultilingualTask import MultilingualTask +from mteb.abstasks.TaskMetadata import TaskMetadata + +from ....abstasks.AbsTaskInstructionRetrieval import AbsTaskInstructionRetrieval + +_LANGUAGES = { + "fas": ["fas-Arab"], + "rus": ["rus-Cyrl"], + "zho": ["zho-Hans"], +} + +_LANGUAGES_CLIR = { + "eng.fas": ["eng-Latn", "fas-Arab"], + "eng.rus": ["eng-Latn", "rus-Cyrl"], + "eng.zho": ["eng-Latn", "zho-Hans"], +} + + +def _build_lang_pair(langs: list[str]) -> str: + """Builds a language pair separated by a dash. + e.g., ['eng-Latn', 'deu-Latn'] -> 'eng-deu'. + """ + return langs[0].split("-")[0] + "-" + langs[1].split("-")[0] + + +def extend_lang_pairs() -> dict[str, list[str]]: + eval_langs = {} + for langs in _LANGUAGES_CLIR.values(): + lang_pair = _build_lang_pair(langs) + eval_langs[lang_pair] = langs + return eval_langs + + +_CLIR_LANGS = extend_lang_pairs() + +EVAL_SPLIT = "test" + + +def load_data( + path: str, + langs: list, + eval_splits: list, + cache_dir: str | None = None, + revision: str | None = None, +): + corpus = {lang: {EVAL_SPLIT: {}} for lang in langs} + queries = {lang: {EVAL_SPLIT: {}} for lang in langs} + og_relevant_docs = {lang: {EVAL_SPLIT: {}} for lang in langs} + changed_relevant_docs = {lang: {EVAL_SPLIT: {}} for lang in langs} + top_ranked = {lang: {EVAL_SPLIT: {}} for lang in langs} + + for lang in langs: + if "-" in lang: + loading_lang = lang.split("-")[1] # don't care about the eng part + else: + loading_lang = lang + + # Load corpus data + corpus_data = datasets.load_dataset( + path, + f"corpus-{loading_lang}", + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + corpus[lang][EVAL_SPLIT] = { + row["_id"]: {"title": row["title"], "text": row["text"]} + for row in corpus_data["corpus"] + } + + # Load queries data + queries_data = datasets.load_dataset( + path, + f"queries-{loading_lang}", + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + queries[lang][EVAL_SPLIT] = { + row["_id"]: { + "text": row["text"], + "instruction_og": row["instruction_og"], + "instruction_changed": row["instruction_changed"], + "keywords": row["keywords"] if "keywords" in row else None, + "short_query": row["short_query"] if "short_query" in row else None, + } + for row in queries_data["queries"] + } + + # Load qrels_og data + qrels_og_data = datasets.load_dataset( + path, + f"qrels_og-{loading_lang}", + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + for row in qrels_og_data[EVAL_SPLIT]: + if row["query-id"] not in og_relevant_docs[lang][EVAL_SPLIT]: + og_relevant_docs[lang][EVAL_SPLIT][row["query-id"]] = { + row["corpus-id"]: int(row["score"]) + } + else: + og_relevant_docs[lang][EVAL_SPLIT][row["query-id"]][ + row["corpus-id"] + ] = int(row["score"]) + + # Load qrels_changed data + qrels_changed_data = datasets.load_dataset( + path, + f"qrels_changed-{loading_lang}", + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + for row in qrels_changed_data[EVAL_SPLIT]: + if row["query-id"] not in changed_relevant_docs[lang][EVAL_SPLIT]: + changed_relevant_docs[lang][EVAL_SPLIT][row["query-id"]] = { + row["corpus-id"]: int(row["score"]) + } + else: + changed_relevant_docs[lang][EVAL_SPLIT][row["query-id"]][ + row["corpus-id"] + ] = int(row["score"]) + + # Load top_ranked data + top_ranked_data = datasets.load_dataset( + path, + f"top_ranked-{loading_lang}", + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + for row in top_ranked_data["top_ranked"]: + if row["qid"] not in top_ranked[lang][EVAL_SPLIT]: + top_ranked[lang][EVAL_SPLIT][row["qid"]] = [row["pid"]] + else: + top_ranked[lang][EVAL_SPLIT][row["qid"]].append(row["pid"]) + + # make og_instructions and changed_instructions from queries and then turn queries into just queries + og_instructions = {lang: {EVAL_SPLIT: defaultdict(dict)} for lang in queries} + changed_instructions = {lang: {EVAL_SPLIT: defaultdict(dict)} for lang in queries} + queries_only = {lang: {EVAL_SPLIT: {}} for lang in queries} + for lang in queries: + for split in queries[lang]: + for qid in queries[lang][split]: + text = queries[lang][split][qid]["text"] + og_instructions[lang][split][text] = queries[lang][split][qid][ + "instruction_og" + ] + changed_instructions[lang][split][text] = queries[lang][split][qid][ + "instruction_changed" + ] + queries_only[lang][split][qid] = text + + queries = queries_only + + return ( + corpus, + queries, + og_instructions, + changed_instructions, + og_relevant_docs, + changed_relevant_docs, + top_ranked, + ) + + +class mFollowIRCrossLingual(MultilingualTask, AbsTaskInstructionRetrieval): + metadata = TaskMetadata( + name="mFollowIRCrossLingualInstructionRetrieval", + description="This tasks measures retrieval instruction following ability on NeuCLIR narratives for the mFollowIR benchmark on the Farsi, Russian, and Chinese languages with English queries/instructions.", + reference="https://neuclir.github.io/", + dataset={ + "path": "jhu-clsp/mFollowIR-cross-lingual-parquet", + "revision": "7a82814a53229d3c8f18b2e18762a1a959dc5ff6", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=[EVAL_SPLIT], + eval_langs=_CLIR_LANGS, + main_score="p-MRR", + date=("2021-08-01", "2022-06-30"), + domains=["News", "Written"], + task_subtypes=[], + license="odc-by", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{weller2024mfollowir, + title={{mFollowIR: a Multilingual Benchmark for Instruction Following in Retrieval}}, + author={Weller, Orion and Chang, Benjamin and Yang, Eugene and Yarmohammadi, Mahsa and Barham, Sam and MacAvaney, Sean and Cohan, Arman and Soldaini, Luca and Van Durme, Benjamin and Lawrie, Dawn}, + journal={arXiv preprint TODO}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"eng-fas": 40 * 2, "eng-rus": 40 * 2, "eng-zho": 43 * 2}, + "test": { + "num_docs": 121635, + "num_queries": 123, + "average_document_length": 2331.0777818884367, + "average_query_length": 81.8780487804878, + "average_instruction_length": 389.9512195121951, + "average_changed_instruction_length": 450.5528455284553, + "average_relevant_docs_per_query": 10.30952380952381, + "average_top_ranked_per_query": 1024.3902439024391, + "hf_subset_descriptive_stats": { + "eng-fas": { + "num_docs": 41189, + "num_queries": 40, + "average_document_length": 3145.4990895627475, + "average_query_length": 80.075, + "average_instruction_length": 396.875, + "average_changed_instruction_length": 463.175, + "average_relevant_docs_per_query": 10.465116279069768, + "average_top_ranked_per_query": 1075, + }, + "eng-rus": { + "num_docs": 39326, + "num_queries": 40, + "average_document_length": 2784.0813456746173, + "average_query_length": 81.875, + "average_instruction_length": 371.125, + "average_changed_instruction_length": 431.8, + "average_relevant_docs_per_query": 9.775, + "average_top_ranked_per_query": 1000, + }, + "eng-zho": { + "num_docs": 41120, + "num_queries": 43, + "average_document_length": 1082.0501215953307, + "average_query_length": 83.55813953488372, + "average_instruction_length": 401.0232558139535, + "average_changed_instruction_length": 456.25581395348837, + "average_relevant_docs_per_query": 10.651162790697674, + "average_top_ranked_per_query": 1000, + }, + }, + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + ( + self.corpus, + self.queries, + self.og_instructions, + self.changed_instructions, + self.og_relevant_docs, + self.changed_relevant_docs, + self.top_ranked, + ) = load_data( + path=self.metadata_dict["dataset"]["path"], + langs=self.metadata.eval_langs, + eval_splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True + + +class mFollowIR(MultilingualTask, AbsTaskInstructionRetrieval): + metadata = TaskMetadata( + name="mFollowIRInstructionRetrieval", + description="This tasks measures retrieval instruction following ability on NeuCLIR narratives for the mFollowIR benchmark on the Farsi, Russian, and Chinese languages.", + reference="https://neuclir.github.io/", + dataset={ + "path": "jhu-clsp/mFollowIR-parquet", + "revision": "2c5cdcb438eff9de6412803768ac7304d4771cdc", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=[EVAL_SPLIT], + eval_langs=_LANGUAGES, + main_score="p-MRR", + date=("2021-08-01", "2022-06-30"), + domains=["News", "Written"], + task_subtypes=[], + license="odc-by", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{weller2024mfollowir, + title={{mFollowIR: a Multilingual Benchmark for Instruction Following in Retrieval}}, + author={Weller, Orion and Chang, Benjamin and Yang, Eugene and Yarmohammadi, Mahsa and Barham, Sam and MacAvaney, Sean and Cohan, Arman and Soldaini, Luca and Van Durme, Benjamin and Lawrie, Dawn}, + journal={arXiv preprint TODO}, + year={2024} +}""", + descriptive_stats={ + "n_samples": {"fas": 40 * 2, "rus": 40 * 2, "zho": 43 * 2}, + "test": { + "num_docs": 121635, + "num_queries": 123, + "average_document_length": 2331.0777818884367, + "average_query_length": 57.113821138211385, + "average_instruction_length": 281.0650406504065, + "average_changed_instruction_length": 326.9430894308943, + "average_relevant_docs_per_query": 10.30952380952381, + "average_top_ranked_per_query": 1024.3902439024391, + "hf_subset_descriptive_stats": { + "fas": { + "num_docs": 41189, + "num_queries": 40, + "average_document_length": 3145.4990895627475, + "average_query_length": 72.65, + "average_instruction_length": 358.925, + "average_changed_instruction_length": 415.325, + "average_relevant_docs_per_query": 10.465116279069768, + "average_top_ranked_per_query": 1075, + }, + "rus": { + "num_docs": 39326, + "num_queries": 40, + "average_document_length": 2784.0813456746173, + "average_query_length": 77.5, + "average_instruction_length": 387, + "average_changed_instruction_length": 458, + "average_relevant_docs_per_query": 9.775, + "average_top_ranked_per_query": 1000, + }, + "zho": { + "num_docs": 41120, + "num_queries": 43, + "average_document_length": 1082.0501215953307, + "average_query_length": 23.697674418604652, + "average_instruction_length": 110.09302325581395, + "average_changed_instruction_length": 122.81395348837209, + "average_relevant_docs_per_query": 10.651162790697674, + "average_top_ranked_per_query": 1000, + }, + }, + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + ( + self.corpus, + self.queries, + self.og_instructions, + self.changed_instructions, + self.og_relevant_docs, + self.changed_relevant_docs, + self.top_ranked, + ) = load_data( + path=self.metadata_dict["dataset"]["path"], + langs=self.metadata.eval_langs, + eval_splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py b/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py index 0db3ec24e1..b5b8d4ec26 100644 --- a/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py +++ b/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py @@ -51,7 +51,7 @@ class MultiEURLEXMultilabelClassification( date=("1958-01-01", "2016-01-01"), domains=["Legal", "Government", "Written"], task_subtypes=["Topic classification"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/MultiLabelClassification/por/BrazilianToxicTweetsClassification.py b/mteb/tasks/MultiLabelClassification/por/BrazilianToxicTweetsClassification.py index 16129f82fe..bbc81a0cb8 100644 --- a/mteb/tasks/MultiLabelClassification/por/BrazilianToxicTweetsClassification.py +++ b/mteb/tasks/MultiLabelClassification/por/BrazilianToxicTweetsClassification.py @@ -31,7 +31,7 @@ class BrazilianToxicTweetsClassification(AbsTaskMultilabelClassification): date=("2019-08-01", "2019-08-16"), domains=["Constructed", "Written"], task_subtypes=["Sentiment/Hate speech"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="expert-annotated", dialect=["brazilian"], sample_creation="found", diff --git a/mteb/tasks/PairClassification/ara/ArEntail.py b/mteb/tasks/PairClassification/ara/ArEntail.py index 7dfeb62df6..588027cb0e 100644 --- a/mteb/tasks/PairClassification/ara/ArEntail.py +++ b/mteb/tasks/PairClassification/ara/ArEntail.py @@ -25,7 +25,7 @@ class ArEntail(AbsTaskPairClassification): ), # best guess based on google searching random samples domains=["News", "Written"], task_subtypes=["Textual Entailment"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/PairClassification/ces/CTKFactsNLI.py b/mteb/tasks/PairClassification/ces/CTKFactsNLI.py index 23834378b0..d2e3296df8 100644 --- a/mteb/tasks/PairClassification/ces/CTKFactsNLI.py +++ b/mteb/tasks/PairClassification/ces/CTKFactsNLI.py @@ -23,7 +23,7 @@ class CTKFactsNLI(AbsTaskPairClassification): date=("2020-09-01", "2021-08-31"), # academic year 2020/2021 domains=["News", "Written"], task_subtypes=["Claim verification"], - license="CC-BY-SA-3.0", + license="cc-by-sa-3.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/PairClassification/eng/SprintDuplicateQuestionsPC.py b/mteb/tasks/PairClassification/eng/SprintDuplicateQuestionsPC.py index 39db2db0ac..44aa50e286 100644 --- a/mteb/tasks/PairClassification/eng/SprintDuplicateQuestionsPC.py +++ b/mteb/tasks/PairClassification/eng/SprintDuplicateQuestionsPC.py @@ -26,7 +26,7 @@ class SprintDuplicateQuestionsPC(AbsTaskPairClassification): ), # not found in the paper or data. This is just a rough guess based on the paper's publication date domains=["Programming", "Written"], task_subtypes=["Duplicate Detection"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/PairClassification/fas/FarsTail.py b/mteb/tasks/PairClassification/fas/FarsTail.py index 8379bc45d4..aa74bc5d39 100644 --- a/mteb/tasks/PairClassification/fas/FarsTail.py +++ b/mteb/tasks/PairClassification/fas/FarsTail.py @@ -24,7 +24,7 @@ class FarsTail(AbsTaskPairClassification): date=("2021-01-01", "2021-07-12"), # best guess domains=["Academic", "Written"], task_subtypes=["Textual Entailment"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/PairClassification/hye/ArmenianParaphrasePC.py b/mteb/tasks/PairClassification/hye/ArmenianParaphrasePC.py index 4a22cbfb9b..d430680e19 100644 --- a/mteb/tasks/PairClassification/hye/ArmenianParaphrasePC.py +++ b/mteb/tasks/PairClassification/hye/ArmenianParaphrasePC.py @@ -22,7 +22,7 @@ class ArmenianParaphrasePC(AbsTaskPairClassification): date=("2021-01-01", "2022-04-06"), domains=["News", "Written"], task_subtypes=[], - license="Apache-2.0", + license="apache-2.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/PairClassification/ind/IndoNLI.py b/mteb/tasks/PairClassification/ind/IndoNLI.py index 377fff40cf..ed62d5ab23 100644 --- a/mteb/tasks/PairClassification/ind/IndoNLI.py +++ b/mteb/tasks/PairClassification/ind/IndoNLI.py @@ -23,7 +23,7 @@ class IndoNLI(AbsTaskPairClassification): date=("2021-01-01", "2021-11-01"), # best guess domains=["Encyclopaedic", "Web", "News", "Written"], task_subtypes=["Textual Entailment"], - license="CC-BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/PairClassification/kor/KlueNLI.py b/mteb/tasks/PairClassification/kor/KlueNLI.py index 19a803f0f4..b21891cb9e 100644 --- a/mteb/tasks/PairClassification/kor/KlueNLI.py +++ b/mteb/tasks/PairClassification/kor/KlueNLI.py @@ -23,7 +23,7 @@ class KlueNLI(AbsTaskPairClassification): date=("2016-01-01", "2020-12-31"), domains=["News", "Encyclopaedic", "Written"], task_subtypes=["Textual Entailment"], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py b/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py index 5220783ea6..f2e5f805f5 100644 --- a/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py +++ b/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py @@ -33,7 +33,7 @@ class PawsXPairClassification(MultilingualTask, AbsTaskPairClassification): date=("2016-01-01", "2018-12-31"), domains=["Web", "Encyclopaedic", "Written"], task_subtypes=["Textual Entailment"], - license="Custom (commercial)", + license="https://huggingface.co/datasets/google-research-datasets/paws-x#licensing-information", annotations_creators="human-annotated", dialect=[], sample_creation="human-translated", diff --git a/mteb/tasks/PairClassification/multilingual/XNLI.py b/mteb/tasks/PairClassification/multilingual/XNLI.py index 5a10fc0c8d..826fabda67 100644 --- a/mteb/tasks/PairClassification/multilingual/XNLI.py +++ b/mteb/tasks/PairClassification/multilingual/XNLI.py @@ -40,7 +40,7 @@ class XNLI(MultilingualTask, AbsTaskPairClassification): date=("2018-01-01", "2018-11-04"), domains=["Non-fiction", "Fiction", "Government", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="expert-annotated", dialect=[], sample_creation="created", @@ -344,7 +344,7 @@ class XNLIV2(MultilingualTask, AbsTaskPairClassification): date=("2018-01-01", "2018-11-04"), domains=["Non-fiction", "Fiction", "Government", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="expert-annotated", dialect=[], sample_creation="machine-translated and verified", diff --git a/mteb/tasks/PairClassification/multilingual/XStance.py b/mteb/tasks/PairClassification/multilingual/XStance.py index a2fb5333a8..e6b60861da 100644 --- a/mteb/tasks/PairClassification/multilingual/XStance.py +++ b/mteb/tasks/PairClassification/multilingual/XStance.py @@ -31,7 +31,7 @@ class XStance(MultilingualTask, AbsTaskPairClassification): date=("2011-01-01", "2020-12-31"), domains=["Social", "Written"], task_subtypes=["Political classification"], - license="cc by-nc 4.0", + license="cc-by-nc-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/PairClassification/pol/PolishPC.py b/mteb/tasks/PairClassification/pol/PolishPC.py index 366f4d71a2..2166cebf1c 100644 --- a/mteb/tasks/PairClassification/pol/PolishPC.py +++ b/mteb/tasks/PairClassification/pol/PolishPC.py @@ -92,7 +92,7 @@ class PpcPC(AbsTaskPairClassification): "News", ], # opensubtitles, CCmatrix task_subtypes=[], - license="GPL-3.0", + license="gpl-3.0", annotations_creators="derived", # mined dialect=[], sample_creation="found", @@ -130,7 +130,7 @@ class CdscePC(AbsTaskPairClassification): date=None, domains=["Written"], task_subtypes=[], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", @@ -176,7 +176,7 @@ class PscPC(AbsTaskPairClassification): date=("1996-01-01", "2003-01-01"), # from the paper domains=["News", "Written"], task_subtypes=[], - license="cc-by-3", + license="cc-by-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/PairClassification/por/Assin2RTE.py b/mteb/tasks/PairClassification/por/Assin2RTE.py index 4afd4ccbc1..41f9aa43e8 100644 --- a/mteb/tasks/PairClassification/por/Assin2RTE.py +++ b/mteb/tasks/PairClassification/por/Assin2RTE.py @@ -22,7 +22,7 @@ class Assin2RTE(AbsTaskPairClassification): date=("2019-01-01", "2019-09-16"), # best guess domains=["Written"], task_subtypes=["Textual Entailment"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/PairClassification/por/SickBrPC.py b/mteb/tasks/PairClassification/por/SickBrPC.py index 377617e7cd..f7bf92e69b 100644 --- a/mteb/tasks/PairClassification/por/SickBrPC.py +++ b/mteb/tasks/PairClassification/por/SickBrPC.py @@ -24,7 +24,7 @@ class SickBrPC(AbsTaskPairClassification): date=("2018-01-01", "2018-09-01"), # rough estimate domains=["Web", "Written"], task_subtypes=["Textual Entailment"], - license="Unknown", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="human-translated and localized", diff --git a/mteb/tasks/PairClassification/zho/CMTEBPairClassification.py b/mteb/tasks/PairClassification/zho/CMTEBPairClassification.py index 13d64a55e2..51d9e22b07 100644 --- a/mteb/tasks/PairClassification/zho/CMTEBPairClassification.py +++ b/mteb/tasks/PairClassification/zho/CMTEBPairClassification.py @@ -54,7 +54,7 @@ class Cmnli(AbsTaskPairClassification): type="PairClassification", category="s2s", modalities=["text"], - eval_splits=["validation", "test"], + eval_splits=["validation"], eval_langs=["cmn-Hans"], main_score="max_accuracy", date=None, diff --git a/mteb/tasks/Reranking/__init__.py b/mteb/tasks/Reranking/__init__.py index f96985d458..a4b302a17f 100644 --- a/mteb/tasks/Reranking/__init__.py +++ b/mteb/tasks/Reranking/__init__.py @@ -8,6 +8,7 @@ from .fra.AlloprofReranking import * from .fra.SyntecReranking import * from .jpn.MMarcoReranking import * +from .multilingual.ESCIReranking import * from .multilingual.MIRACLReranking import * from .multilingual.WikipediaRerankingMultilingual import * from .rus.RuBQReranking import * diff --git a/mteb/tasks/Reranking/eng/WebLINXCandidatesReranking.py b/mteb/tasks/Reranking/eng/WebLINXCandidatesReranking.py index c1d748a01d..4790a9460f 100644 --- a/mteb/tasks/Reranking/eng/WebLINXCandidatesReranking.py +++ b/mteb/tasks/Reranking/eng/WebLINXCandidatesReranking.py @@ -33,7 +33,7 @@ class WebLINXCandidatesReranking(AbsTaskReranking): date=("2023-03-01", "2023-10-30"), domains=["Academic", "Web", "Written"], task_subtypes=["Code retrieval", "Conversational retrieval"], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/Reranking/fra/AlloprofReranking.py b/mteb/tasks/Reranking/fra/AlloprofReranking.py index f84e223500..3e5509c936 100644 --- a/mteb/tasks/Reranking/fra/AlloprofReranking.py +++ b/mteb/tasks/Reranking/fra/AlloprofReranking.py @@ -25,7 +25,7 @@ class AlloprofReranking(AbsTaskReranking): date=("2020-01-01", "2023-04-14"), # supposition domains=["Web", "Academic", "Written"], task_subtypes=None, - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="expert-annotated", dialect=None, sample_creation="found", diff --git a/mteb/tasks/Reranking/fra/SyntecReranking.py b/mteb/tasks/Reranking/fra/SyntecReranking.py index 802e502bcd..3b12625e69 100644 --- a/mteb/tasks/Reranking/fra/SyntecReranking.py +++ b/mteb/tasks/Reranking/fra/SyntecReranking.py @@ -25,7 +25,7 @@ class SyntecReranking(AbsTaskReranking): date=("2022-12-01", "2022-12-02"), domains=["Legal", "Written"], task_subtypes=None, - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="human-annotated", dialect=None, sample_creation="found", diff --git a/mteb/tasks/Reranking/jpn/MMarcoReranking.py b/mteb/tasks/Reranking/jpn/MMarcoReranking.py index ed6ce8ebfc..bcfa5bba05 100644 --- a/mteb/tasks/Reranking/jpn/MMarcoReranking.py +++ b/mteb/tasks/Reranking/jpn/MMarcoReranking.py @@ -22,7 +22,7 @@ class VoyageMMarcoReranking(AbsTaskReranking): date=("2016-12-01", "2023-12-23"), domains=["Academic", "Non-fiction", "Written"], task_subtypes=["Scientific Reranking"], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="derived", dialect=["jpn-Jpan"], sample_creation="found", diff --git a/mteb/tasks/Reranking/multilingual/ESCIReranking.py b/mteb/tasks/Reranking/multilingual/ESCIReranking.py new file mode 100644 index 0000000000..c3597c2fdf --- /dev/null +++ b/mteb/tasks/Reranking/multilingual/ESCIReranking.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import logging + +from mteb.abstasks.AbsTaskReranking import AbsTaskReranking +from mteb.abstasks.MultilingualTask import MultilingualTask +from mteb.abstasks.TaskMetadata import TaskMetadata + +logger = logging.getLogger(__name__) + +_EVAL_SPLIT = "test" +_LANGUAGES = { + "us": ["eng-Latn"], + "es": ["spa-Latn"], + "jp": ["jpn-Jpan"], +} + +_CITATION = """@article{reddy2022shopping, + title={Shopping Queries Dataset: A Large-Scale {ESCI} Benchmark for Improving Product Search}, + author={Chandan K. Reddy and Lluís Màrquez and Fran Valero and Nikhil Rao and Hugo Zaragoza and Sambaran Bandyopadhyay and Arnab Biswas and Anlu Xing and Karthik Subbian}, + year={2022}, + eprint={2206.06588}, + archivePrefix={arXiv} +}""" + + +class ESCIReranking(MultilingualTask, AbsTaskReranking): + metadata = TaskMetadata( + name="ESCIReranking", + description="", + reference="https://github.com/amazon-science/esci-data/", + dataset={ + "path": "mteb/esci", + "revision": "237f74be0503482b4e8bc1b83778c7a87ea93fd8", + }, + type="Reranking", + category="s2p", + modalities=["text"], + eval_splits=[_EVAL_SPLIT], + eval_langs=_LANGUAGES, + main_score="map", + date=("2022-06-14", "2022-06-14"), + domains=["Written"], + task_subtypes=[], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=_CITATION, + descriptive_stats={ + "test": { + "num_samples": 29285, + "num_positive": 29285, + "num_negative": 29285, + "avg_query_len": 19.691890046098685, + "avg_positive_len": 9.268089465596722, + "avg_negative_len": 1.5105002561038074, + "hf_subset_descriptive_stats": { + "us": { + "num_samples": 21296, + "num_positive": 21296, + "num_negative": 21296, + "avg_query_len": 21.440833959429, + "avg_positive_len": 8.892515026296017, + "avg_negative_len": 1.1956705484598047, + }, + "es": { + "num_samples": 3703, + "num_positive": 3703, + "num_negative": 3703, + "avg_query_len": 20.681609505806104, + "avg_positive_len": 10.561706724277613, + "avg_negative_len": 2.749932487172563, + }, + "jp": { + "num_samples": 4286, + "num_positive": 4286, + "num_negative": 4286, + "avg_query_len": 10.146756882874476, + "avg_positive_len": 10.016565562295847, + "avg_negative_len": 2.003966402239851, + }, + }, + } + }, + ) diff --git a/mteb/tasks/Reranking/multilingual/MIRACLReranking.py b/mteb/tasks/Reranking/multilingual/MIRACLReranking.py index 2b132ff933..4d226842bf 100644 --- a/mteb/tasks/Reranking/multilingual/MIRACLReranking.py +++ b/mteb/tasks/Reranking/multilingual/MIRACLReranking.py @@ -7,9 +7,9 @@ from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata -from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from mteb.encoder_interface import Encoder from mteb.evaluation.evaluators import RerankingEvaluator -from mteb.load_results.mteb_results import ScoresDict +from mteb.load_results.task_results import ScoresDict from ....abstasks.AbsTaskReranking import AbsTaskReranking @@ -69,7 +69,7 @@ class MIRACLReranking(MultilingualTask, AbsTaskReranking): date=("2022-06-01", "2023-01-30"), domains=["Encyclopaedic", "Written"], task_subtypes=[], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="created", @@ -82,7 +82,7 @@ class MIRACLReranking(MultilingualTask, AbsTaskReranking): def _evaluate_subset( self, - model: Encoder | EncoderWithQueryCorpusEncode, + model: Encoder, data_split: Dataset, *, encode_kwargs: dict[str, Any] = {}, diff --git a/mteb/tasks/Reranking/zho/CMTEBReranking.py b/mteb/tasks/Reranking/zho/CMTEBReranking.py index cdc8dcdd7d..7aa26c4ce0 100644 --- a/mteb/tasks/Reranking/zho/CMTEBReranking.py +++ b/mteb/tasks/Reranking/zho/CMTEBReranking.py @@ -92,7 +92,7 @@ class CMedQAv1(AbsTaskReranking): date=("2017-01-01", "2017-07-26"), domains=["Medical", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="expert-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/__init__.py b/mteb/tasks/Retrieval/__init__.py index 3975cd9bd3..c29a4383c7 100644 --- a/mteb/tasks/Retrieval/__init__.py +++ b/mteb/tasks/Retrieval/__init__.py @@ -94,6 +94,7 @@ from .fra.SyntecRetrieval import * from .hun.HunSum2 import * from .jpn.JaGovFaqsRetrieval import * +from .jpn.JaqketRetrieval import * from .jpn.JaQuADRetrieval import * from .jpn.NLPJournalAbsIntroRetrieval import * from .jpn.NLPJournalTitleAbsRetrieval import * @@ -107,6 +108,7 @@ from .multilingual.MintakaRetrieval import * from .multilingual.MIRACLRetrieval import * from .multilingual.MLQARetrieval import * +from .multilingual.MrTidyRetrieval import * from .multilingual.MultiLongDocRetrieval import * from .multilingual.NeuCLIR2022Retrieval import * from .multilingual.NeuCLIR2023Retrieval import * @@ -131,6 +133,7 @@ from .pol.TRECCOVIDPLRetrieval import * from .rus.RiaNewsRetrieval import * from .rus.RuBQRetrieval import * +from .slk.SKQuadRetrieval import * from .slk.SlovakSumRetrieval import * from .spa.SpanishPassageRetrievalS2P import * from .spa.SpanishPassageRetrievalS2S import * diff --git a/mteb/tasks/Retrieval/ara/SadeemQuestionRetrieval.py b/mteb/tasks/Retrieval/ara/SadeemQuestionRetrieval.py index 367c50aa27..d6efb88828 100644 --- a/mteb/tasks/Retrieval/ara/SadeemQuestionRetrieval.py +++ b/mteb/tasks/Retrieval/ara/SadeemQuestionRetrieval.py @@ -28,7 +28,7 @@ class SadeemQuestionRetrieval(AbsTaskRetrieval): date=("2024-01-01", "2024-04-01"), domains=["Written", "Written"], task_subtypes=["Article retrieval"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/AppsRetrieval.py b/mteb/tasks/Retrieval/code/AppsRetrieval.py index f0cadf9b56..03b6df10af 100644 --- a/mteb/tasks/Retrieval/code/AppsRetrieval.py +++ b/mteb/tasks/Retrieval/code/AppsRetrieval.py @@ -24,7 +24,7 @@ class AppsRetrieval(AbsTaskRetrieval): date=("2021-05-20", "2021-05-20"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py b/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py index 4a272f4200..170e6c8348 100644 --- a/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py +++ b/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py @@ -92,7 +92,7 @@ class COIRCodeSearchNetRetrieval(MultilingualTask, AbsTaskRetrieval): date=("2019-01-01", "2019-12-31"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py b/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py index 71ef272078..6351cc723e 100644 --- a/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py @@ -42,7 +42,7 @@ class CodeEditSearchRetrieval(MultilingualTask, AbsTaskRetrieval): date=("2011-02-12", "2016-01-01"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/CodeFeedbackMTRetrieval.py b/mteb/tasks/Retrieval/code/CodeFeedbackMTRetrieval.py index 41aad8945f..3f307f12a3 100644 --- a/mteb/tasks/Retrieval/code/CodeFeedbackMTRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeFeedbackMTRetrieval.py @@ -24,7 +24,7 @@ class CodeFeedbackMT(AbsTaskRetrieval): date=("2019-01-01", "2019-12-31"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/CodeFeedbackSTRetrieval.py b/mteb/tasks/Retrieval/code/CodeFeedbackSTRetrieval.py index 6c94c8cba2..caae17bada 100644 --- a/mteb/tasks/Retrieval/code/CodeFeedbackSTRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeFeedbackSTRetrieval.py @@ -24,7 +24,7 @@ class CodeFeedbackST(AbsTaskRetrieval): date=("2019-01-01", "2019-12-31"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py b/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py index bcc3192e5e..7751f5ed2a 100644 --- a/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py @@ -91,7 +91,7 @@ class CodeSearchNetCCRetrieval(MultilingualTask, AbsTaskRetrieval): date=("2019-01-01", "2019-12-31"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py b/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py index 5fc6f7862d..a76ac3b231 100644 --- a/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py @@ -28,7 +28,7 @@ class CodeSearchNetRetrieval(MultilingualTask, AbsTaskRetrieval): date=("2019-01-01", "2019-12-31"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/CodeTransOceanContestRetrieval.py b/mteb/tasks/Retrieval/code/CodeTransOceanContestRetrieval.py index 01b6c27167..b7200b38a9 100644 --- a/mteb/tasks/Retrieval/code/CodeTransOceanContestRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeTransOceanContestRetrieval.py @@ -24,7 +24,7 @@ class CodeTransOceanContestRetrieval(AbsTaskRetrieval): date=("2023-10-08", "2023-10-08"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="Apache-2.0", + license="apache-2.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/CodeTransOceanDLRetrieval.py b/mteb/tasks/Retrieval/code/CodeTransOceanDLRetrieval.py index 97906584e5..3b61e0e9c4 100644 --- a/mteb/tasks/Retrieval/code/CodeTransOceanDLRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeTransOceanDLRetrieval.py @@ -24,7 +24,7 @@ class CodeTransOceanDLRetrieval(AbsTaskRetrieval): date=("2023-10-08", "2023-10-08"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="Apache-2.0", + license="apache-2.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/CosQARetrieval.py b/mteb/tasks/Retrieval/code/CosQARetrieval.py index 8e29f6fc17..c51b266ea5 100644 --- a/mteb/tasks/Retrieval/code/CosQARetrieval.py +++ b/mteb/tasks/Retrieval/code/CosQARetrieval.py @@ -24,7 +24,7 @@ class CosQARetrieval(AbsTaskRetrieval): date=("2021-05-07", "2021-05-07"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/StackOverflowQARetrieval.py b/mteb/tasks/Retrieval/code/StackOverflowQARetrieval.py index 42b373d8a8..bd1d2da5ea 100644 --- a/mteb/tasks/Retrieval/code/StackOverflowQARetrieval.py +++ b/mteb/tasks/Retrieval/code/StackOverflowQARetrieval.py @@ -24,7 +24,7 @@ class StackOverflowQARetrieval(AbsTaskRetrieval): date=("2019-01-01", "2019-12-31"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/code/SyntheticText2SqlRetrieval.py b/mteb/tasks/Retrieval/code/SyntheticText2SqlRetrieval.py index 4ae2d48123..267d02048a 100644 --- a/mteb/tasks/Retrieval/code/SyntheticText2SqlRetrieval.py +++ b/mteb/tasks/Retrieval/code/SyntheticText2SqlRetrieval.py @@ -24,7 +24,7 @@ class SyntheticText2SQLRetrieval(AbsTaskRetrieval): date=("2019-01-01", "2019-12-31"), domains=["Programming", "Written"], task_subtypes=["Code retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py b/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py index 62d29199d2..2468463a13 100644 --- a/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py +++ b/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py @@ -23,7 +23,7 @@ class DanFeverRetrieval(AbsTaskRetrieval): main_score="ndcg_at_10", date=("2020-01-01", "2021-12-31"), # best guess domains=["Encyclopaedic", "Non-fiction", "Spoken"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", @@ -70,9 +70,9 @@ def load_data(self, **kwargs): def dataset_transform(self) -> None: """And transform to a retrieval datset, which have the following attributes - self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document data like title and text - self.queries = Dict[query_id, str] #id => query - self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + self.corpus = dict[doc_id, dict[str, str]] #id => dict with document data like title and text + self.queries = dict[query_id, str] #id => query + self.relevant_docs = dict[query_id, dict[[doc_id, score]] """ self.corpus = {} self.relevant_docs = {} @@ -135,7 +135,7 @@ class DanFever(AbsTaskRetrieval): main_score="ndcg_at_10", date=("2020-01-01", "2021-12-31"), # best guess domains=["Encyclopaedic", "Non-fiction", "Spoken"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", @@ -182,9 +182,9 @@ def load_data(self, **kwargs): def dataset_transform(self) -> None: """And transform to a retrieval datset, which have the following attributes - self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document data like title and text - self.queries = Dict[query_id, str] #id => query - self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + self.corpus = dict[doc_id, dict[str, str]] #id => dict with document data like title and text + self.queries = dict[query_id, str] #id => query + self.relevant_docs = dict[query_id, dict[[doc_id, score]] """ self.corpus = {} self.relevant_docs = {} diff --git a/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py b/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py index 8bea8bc83d..0f81e28618 100644 --- a/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py +++ b/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py @@ -23,7 +23,7 @@ class TV2Nordretrieval(AbsTaskRetrieval): main_score="ndcg_at_10", date=("2020-01-01", "2024-12-31"), # best guess domains=["News", "Non-fiction", "Written"], - license="CC0", + license="cc0-1.0", annotations_creators="derived", dialect=[], sample_creation="found", @@ -81,9 +81,9 @@ def load_data(self, **kwargs): def dataset_transform(self) -> None: """And transform to a retrieval datset, which have the following attributes - self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text - self.queries = Dict[query_id, str] #id => query - self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + self.corpus = dict[doc_id, dict[str, str]] #id => dict with document datas like title and text + self.queries = dict[query_id, str] #id => query + self.relevant_docs = dict[query_id, dict[[doc_id, score]] """ self.corpus = {} self.relevant_docs = {} diff --git a/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py b/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py index 160cff671b..85e9d1b8aa 100644 --- a/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py +++ b/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py @@ -23,7 +23,7 @@ class TwitterHjerneRetrieval(AbsTaskRetrieval): main_score="ndcg_at_10", date=("2006-01-01", "2024-12-31"), # best guess domains=["Social", "Written"], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="derived", dialect=[], sample_creation="found", @@ -60,9 +60,9 @@ def load_data(self, **kwargs): def dataset_transform(self) -> None: """And transform to a retrieval datset, which have the following attributes - self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text - self.queries = Dict[query_id, str] #id => query - self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + self.corpus = dict[doc_id, dict[str, str]] #id => dict with document datas like title and text + self.queries = dict[query_id, str] #id => query + self.relevant_docs = dict[query_id, dict[[doc_id, score]] """ self.corpus = {} self.relevant_docs = {} diff --git a/mteb/tasks/Retrieval/deu/GerDaLIRSmallRetrieval.py b/mteb/tasks/Retrieval/deu/GerDaLIRSmallRetrieval.py index aca31408b2..a60c676c86 100644 --- a/mteb/tasks/Retrieval/deu/GerDaLIRSmallRetrieval.py +++ b/mteb/tasks/Retrieval/deu/GerDaLIRSmallRetrieval.py @@ -23,7 +23,7 @@ class GerDaLIRSmall(AbsTaskRetrieval): date=None, domains=["Legal", "Written"], task_subtypes=["Article retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=None, sample_creation="found", diff --git a/mteb/tasks/Retrieval/deu/LegalQuADRetrieval.py b/mteb/tasks/Retrieval/deu/LegalQuADRetrieval.py index e665570eac..d9bf38380a 100644 --- a/mteb/tasks/Retrieval/deu/LegalQuADRetrieval.py +++ b/mteb/tasks/Retrieval/deu/LegalQuADRetrieval.py @@ -23,7 +23,7 @@ class LegalQuAD(AbsTaskRetrieval): date=None, domains=["Legal", "Written"], task_subtypes=["Question answering"], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="derived", dialect=None, sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/AILACasedocsRetrieval.py b/mteb/tasks/Retrieval/eng/AILACasedocsRetrieval.py index 8a791bbb1a..9af2bf3026 100644 --- a/mteb/tasks/Retrieval/eng/AILACasedocsRetrieval.py +++ b/mteb/tasks/Retrieval/eng/AILACasedocsRetrieval.py @@ -23,7 +23,7 @@ class AILACasedocs(AbsTaskRetrieval): date=None, domains=["Legal", "Written"], task_subtypes=["Article retrieval"], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="derived", dialect=None, sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/AILAStatutesRetrieval.py b/mteb/tasks/Retrieval/eng/AILAStatutesRetrieval.py index 341382f395..db84601a7a 100644 --- a/mteb/tasks/Retrieval/eng/AILAStatutesRetrieval.py +++ b/mteb/tasks/Retrieval/eng/AILAStatutesRetrieval.py @@ -23,7 +23,7 @@ class AILAStatutes(AbsTaskRetrieval): date=None, domains=["Legal", "Written"], task_subtypes=["Article retrieval"], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="derived", dialect=None, sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/ARCChallengeRetrieval.py b/mteb/tasks/Retrieval/eng/ARCChallengeRetrieval.py index 853e77bc11..b999b30ce7 100644 --- a/mteb/tasks/Retrieval/eng/ARCChallengeRetrieval.py +++ b/mteb/tasks/Retrieval/eng/ARCChallengeRetrieval.py @@ -24,7 +24,7 @@ class ARCChallenge(AbsTaskRetrieval): date=("2018-01-01", "2018-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/AlphaNLIRetrieval.py b/mteb/tasks/Retrieval/eng/AlphaNLIRetrieval.py index 4559f7efa7..a7cca69d14 100644 --- a/mteb/tasks/Retrieval/eng/AlphaNLIRetrieval.py +++ b/mteb/tasks/Retrieval/eng/AlphaNLIRetrieval.py @@ -24,7 +24,7 @@ class AlphaNLI(AbsTaskRetrieval): date=("2019-01-01", "2019-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY-NC 4.0", + license="cc-by-nc-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/BrightRetrieval.py b/mteb/tasks/Retrieval/eng/BrightRetrieval.py index 715f3e9472..351afa86bf 100644 --- a/mteb/tasks/Retrieval/eng/BrightRetrieval.py +++ b/mteb/tasks/Retrieval/eng/BrightRetrieval.py @@ -47,11 +47,9 @@ class BrightRetrieval(MultilingualTask, AbsTaskRetrieval): eval_langs=DOMAINS_langs, main_score="ndcg_at_10", date=("2024-03-01", "2024-06-01"), - form=["written"], - domains=["Non-fiction"], + domains=["Non-fiction", "Written"], task_subtypes=["Article retrieval"], - license="CC-BY-4.0", - socioeconomic_status="low", + license="cc-by-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py index ed0e24299a..0e63882677 100644 --- a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py @@ -48,3 +48,48 @@ class ClimateFEVER(AbsTaskRetrieval): }, }, ) + + +class ClimateFEVERHardNegatives(AbsTaskRetrieval): + metadata = TaskMetadata( + name="ClimateFEVERHardNegatives", + description="CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html", + dataset={ + "path": "mteb/ClimateFEVER_test_top_250_only_w_correct-v2", + "revision": "3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=None, + domains=None, + task_subtypes=None, + license=None, + annotations_creators=None, + dialect=None, + sample_creation=None, + bibtex_citation="""@misc{diggelmann2021climatefever, + title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, + author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold}, + year={2021}, + eprint={2012.00614}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +}""", + descriptive_stats={ + "n_samples": {"test": 1000}, + "avg_character_length": { + "test": { + "average_document_length": 1245.4236333727013, + "average_query_length": 121.879, + "num_documents": 47416, + "num_queries": 1000, + "average_relevant_docs_per_query": 3.048, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/eng/DBPediaRetrieval.py b/mteb/tasks/Retrieval/eng/DBPediaRetrieval.py index 94a4a98f4d..24e1a9a499 100644 --- a/mteb/tasks/Retrieval/eng/DBPediaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/DBPediaRetrieval.py @@ -50,3 +50,50 @@ class DBPedia(AbsTaskRetrieval): }, }, ) + + +class DBPediaHardNegatives(AbsTaskRetrieval): + metadata = TaskMetadata( + name="DBPediaHardNegatives", + description="DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://github.com/iai-group/DBpedia-Entity/", + dataset={ + "path": "mteb/DBPedia_test_top_250_only_w_correct-v2", + "revision": "943ec7fdfef3728b2ad1966c5b6479ff9ffd26c9", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2017-01-01", "2017-01-01"), # best guess: based on publication date + domains=["Written", "Encyclopaedic"], + task_subtypes=[], + license="mit", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{Hasibi:2017:DVT, + author = {Hasibi, Faegheh and Nikolaev, Fedor and Xiong, Chenyan and Balog, Krisztian and Bratsberg, Svein Erik and Kotov, Alexander and Callan, Jamie}, + title = {DBpedia-Entity V2: A Test Collection for Entity Search}, + booktitle = {Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval}, + series = {SIGIR '17}, + year = {2017}, + pages = {1265--1268}, + doi = {10.1145/3077136.3080751}, + publisher = {ACM} +}""", + descriptive_stats={ + "n_samples": {"test": 400}, + "avg_character_length": { + "test": { + "average_document_length": 338.58561119129564, + "average_query_length": 34.085, + "num_documents": 90070, + "num_queries": 400, + "average_relevant_docs_per_query": 38.215, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/eng/FEVERRetrieval.py b/mteb/tasks/Retrieval/eng/FEVERRetrieval.py index 8a28b3d0e0..058332c94c 100644 --- a/mteb/tasks/Retrieval/eng/FEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/FEVERRetrieval.py @@ -79,3 +79,65 @@ class FEVER(AbsTaskRetrieval): }, }, ) + + +class FEVERHardNegatives(AbsTaskRetrieval): + ignore_identical_ids = True + + metadata = TaskMetadata( + name="FEVERHardNegatives", + dataset={ + "path": "mteb/FEVER_test_top_250_only_w_correct-v2", + "revision": "080c9ed6267b65029207906e815d44a9240bafca", + }, + description=( + "FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences" + + " extracted from Wikipedia and subsequently verified without knowledge of the sentence they were" + + " derived from. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct." + ), + reference="https://fever.ai/", + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=None, + domains=None, + task_subtypes=None, + license=None, + annotations_creators=None, + dialect=None, + sample_creation=None, + bibtex_citation="""@inproceedings{thorne-etal-2018-fever, + title = "{FEVER}: a Large-scale Dataset for Fact Extraction and {VER}ification", + author = "Thorne, James and + Vlachos, Andreas and + Christodoulopoulos, Christos and + Mittal, Arpit", + editor = "Walker, Marilyn and + Ji, Heng and + Stent, Amanda", + booktitle = "Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)", + month = jun, + year = "2018", + address = "New Orleans, Louisiana", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/N18-1074", + doi = "10.18653/v1/N18-1074", + pages = "809--819", + abstract = "In this paper we introduce a new publicly available dataset for verification against textual sources, FEVER: Fact Extraction and VERification. It consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The claims are classified as Supported, Refuted or NotEnoughInfo by annotators achieving 0.6841 in Fleiss kappa. For the first two classes, the annotators also recorded the sentence(s) forming the necessary evidence for their judgment. To characterize the challenge of the dataset presented, we develop a pipeline approach and compare it to suitably designed oracles. The best accuracy we achieve on labeling a claim accompanied by the correct evidence is 31.87{\%}, while if we ignore the evidence we achieve 50.91{\%}. Thus we believe that FEVER is a challenging testbed that will help stimulate progress on claim verification against textual sources.", +}""", + descriptive_stats={ + "n_samples": {"test": 1000}, + "avg_character_length": { + "test": { + "average_document_length": 695.4370242764114, + "average_query_length": 49.62, + "num_documents": 163698, + "num_queries": 1000, + "average_relevant_docs_per_query": 1.171, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/eng/FeedbackQARetrieval.py b/mteb/tasks/Retrieval/eng/FeedbackQARetrieval.py index 1de7c8d978..852a87857a 100644 --- a/mteb/tasks/Retrieval/eng/FeedbackQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/FeedbackQARetrieval.py @@ -25,7 +25,7 @@ class FeedbackQARetrieval(AbsTaskRetrieval): date=("2020-01-01", "2022-04-01"), domains=["Web", "Government", "Medical", "Written"], task_subtypes=["Question answering"], - license="Apache-2.0", + license="apache-2.0", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/Retrieval/eng/HellaSwagRetrieval.py b/mteb/tasks/Retrieval/eng/HellaSwagRetrieval.py index 4cd6f8cc25..9bef36b76a 100644 --- a/mteb/tasks/Retrieval/eng/HellaSwagRetrieval.py +++ b/mteb/tasks/Retrieval/eng/HellaSwagRetrieval.py @@ -24,7 +24,7 @@ class HellaSwag(AbsTaskRetrieval): date=("2019-01-01", "2019-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py b/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py index be19929c95..a11e8b0d79 100644 --- a/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py @@ -80,3 +80,66 @@ class HotpotQA(AbsTaskRetrieval): }, }, ) + + +class HotpotQAHardNegatives(AbsTaskRetrieval): + metadata = TaskMetadata( + name="HotpotQAHardNegatives", + dataset={ + "path": "mteb/HotpotQA_test_top_250_only_w_correct-v2", + "revision": "617612fa63afcb60e3b134bed8b7216a99707c37", + }, + description=( + "HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong" + + " supervision for supporting facts to enable more explainable question answering systems. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct." + ), + reference="https://hotpotqa.github.io/", + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2018-01-01", "2018-12-31"), # best guess: based on publication date + domains=["Web", "Written"], + task_subtypes=["Question answering"], + license="cc-by-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{yang-etal-2018-hotpotqa, + title = "{H}otpot{QA}: A Dataset for Diverse, Explainable Multi-hop Question Answering", + author = "Yang, Zhilin and + Qi, Peng and + Zhang, Saizheng and + Bengio, Yoshua and + Cohen, William and + Salakhutdinov, Ruslan and + Manning, Christopher D.", + editor = "Riloff, Ellen and + Chiang, David and + Hockenmaier, Julia and + Tsujii, Jun{'}ichi", + booktitle = "Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing", + month = oct # "-" # nov, + year = "2018", + address = "Brussels, Belgium", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/D18-1259", + doi = "10.18653/v1/D18-1259", + pages = "2369--2380", + abstract = "Existing question answering (QA) datasets fail to train QA systems to perform complex reasoning and provide explanations for answers. We introduce HotpotQA, a new dataset with 113k Wikipedia-based question-answer pairs with four key features: (1) the questions require finding and reasoning over multiple supporting documents to answer; (2) the questions are diverse and not constrained to any pre-existing knowledge bases or knowledge schemas; (3) we provide sentence-level supporting facts required for reasoning, allowing QA systems to reason with strong supervision and explain the predictions; (4) we offer a new type of factoid comparison questions to test QA systems{'} ability to extract relevant facts and perform necessary comparison. We show that HotpotQA is challenging for the latest QA systems, and the supporting facts enable models to improve performance and make explainable predictions.", +}""", + descriptive_stats={ + "n_samples": {"test": 1000}, + "avg_character_length": { + "test": { + "average_document_length": 373.558822095461, + "average_query_length": 92.584, + "num_documents": 225621, + "num_queries": 1000, + "average_relevant_docs_per_query": 2.0, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py b/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py index 4fe17228a5..e41c85dd3a 100644 --- a/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py @@ -28,7 +28,7 @@ class LEMBNarrativeQARetrieval(AbsTaskRetrieval): date=("1000-01-01", "2017-12-31"), domains=["Fiction", "Non-fiction", "Written"], task_subtypes=["Article retrieval"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py index fa43550350..a6ea725e0d 100644 --- a/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py @@ -37,7 +37,7 @@ class LEMBNeedleRetrieval(AbsTaskRetrieval): date=("2000-01-01", "2023-12-31"), domains=["Academic", "Blog", "Written"], task_subtypes=["Article retrieval"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py index 0b38895f06..0323d65bd7 100644 --- a/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py @@ -37,7 +37,7 @@ class LEMBPasskeyRetrieval(AbsTaskRetrieval): date=("2000-01-01", "2023-12-31"), domains=["Fiction", "Written"], task_subtypes=["Article retrieval"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py index 9a09eb8079..e7f21f8227 100644 --- a/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py @@ -28,7 +28,7 @@ class LEMBQMSumRetrieval(AbsTaskRetrieval): date=("1950-01-01", "2021-12-31"), domains=["Spoken", "Written"], task_subtypes=["Article retrieval"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py index 3c8478c536..d45c938663 100644 --- a/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py @@ -28,7 +28,7 @@ class LEMBSummScreenFDRetrieval(AbsTaskRetrieval): date=("2000-01-01", "2021-12-31"), domains=["Spoken", "Written"], task_subtypes=["Article retrieval"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py b/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py index 6afae35e5f..c5f815e2e2 100644 --- a/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py @@ -28,7 +28,7 @@ class LEMBWikimQARetrieval(AbsTaskRetrieval): date=("1950-01-01", "2019-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Article retrieval"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/LegalBenchConsumerContractsQARetrieval.py b/mteb/tasks/Retrieval/eng/LegalBenchConsumerContractsQARetrieval.py index ed425e8172..27451e30c6 100644 --- a/mteb/tasks/Retrieval/eng/LegalBenchConsumerContractsQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/LegalBenchConsumerContractsQARetrieval.py @@ -23,7 +23,7 @@ class LegalBenchConsumerContractsQA(AbsTaskRetrieval): date=None, domains=["Legal", "Written"], task_subtypes=["Question answering"], - license="CC BY-NC 4.0", + license="cc-by-nc-4.0", annotations_creators="derived", dialect=None, sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/LegalBenchCorporateLobbyingRetrieval.py b/mteb/tasks/Retrieval/eng/LegalBenchCorporateLobbyingRetrieval.py index 3802a41927..939e4cfecc 100644 --- a/mteb/tasks/Retrieval/eng/LegalBenchCorporateLobbyingRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LegalBenchCorporateLobbyingRetrieval.py @@ -23,7 +23,7 @@ class LegalBenchCorporateLobbying(AbsTaskRetrieval): date=None, domains=["Legal", "Written"], task_subtypes=["Article retrieval"], - license="CC BY 4.0", + license="cc-by-4.0", annotations_creators="derived", dialect=None, sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/LegalSummarizationRetrieval.py b/mteb/tasks/Retrieval/eng/LegalSummarizationRetrieval.py index d08b65f0a4..884cc546fe 100644 --- a/mteb/tasks/Retrieval/eng/LegalSummarizationRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LegalSummarizationRetrieval.py @@ -23,7 +23,7 @@ class LegalSummarization(AbsTaskRetrieval): date=None, domains=["Legal", "Written"], task_subtypes=["Article retrieval"], - license="Apache 2.0", + license="apache-2.0", annotations_creators="derived", dialect=None, sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/LitSearchRetrieval.py b/mteb/tasks/Retrieval/eng/LitSearchRetrieval.py index 239e51ce70..dbaa9a6de7 100644 --- a/mteb/tasks/Retrieval/eng/LitSearchRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LitSearchRetrieval.py @@ -31,7 +31,7 @@ class LitSearchRetrieval(AbsTaskRetrieval): date=("2023-07-10", "2024-05-11"), domains=["Academic", "Non-fiction", "Written"], task_subtypes=["Article retrieval"], - license="MIT", + license="mit", annotations_creators="LM-generated", # generated by GPT-4 dialect=[], sample_creation="found", # queries LLM generated, corpus samples are found (extracted from S2ORC) diff --git a/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py b/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py index a2341323cb..dd9260c260 100644 --- a/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py +++ b/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py @@ -76,3 +76,62 @@ class MSMARCO(AbsTaskRetrieval): }, }, ) + + +class MSMARCOHardNegatives(AbsTaskRetrieval): + ignore_identical_ids = True + + metadata = TaskMetadata( + name="MSMARCOHardNegatives", + dataset={ + "path": "mteb/MSMARCO_test_top_250_only_w_correct-v2", + "revision": "67c0b4f7f15946e0b15cf6cf3b8993d04cb3efc6", + }, + description="MS MARCO is a collection of datasets focused on deep learning in search. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://microsoft.github.io/msmarco/", + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=None, + domains=None, + task_subtypes=None, + license=None, + annotations_creators=None, + dialect=None, + sample_creation=None, + bibtex_citation="""@article{DBLP:journals/corr/NguyenRSGTMD16, + author = {Tri Nguyen and + Mir Rosenberg and + Xia Song and + Jianfeng Gao and + Saurabh Tiwary and + Rangan Majumder and + Li Deng}, + title = {{MS} {MARCO:} {A} Human Generated MAchine Reading COmprehension Dataset}, + journal = {CoRR}, + volume = {abs/1611.09268}, + year = {2016}, + url = {http://arxiv.org/abs/1611.09268}, + archivePrefix = {arXiv}, + eprint = {1611.09268}, + timestamp = {Mon, 13 Aug 2018 16:49:03 +0200}, + biburl = {https://dblp.org/rec/journals/corr/NguyenRSGTMD16.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} +}""", + descriptive_stats={ + "n_samples": {"test": 43}, + "avg_character_length": { + "test": { + "average_document_length": 355.2909668633681, + "average_query_length": 32.74418604651163, + "num_documents": 8812, + "num_queries": 43, + "average_relevant_docs_per_query": 95.3953488372093, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/eng/MedicalQARetrieval.py b/mteb/tasks/Retrieval/eng/MedicalQARetrieval.py index 52231247c3..4a8b4bb3b7 100644 --- a/mteb/tasks/Retrieval/eng/MedicalQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/MedicalQARetrieval.py @@ -22,7 +22,7 @@ class MedicalQARetrieval(AbsTaskRetrieval): date=("2017-01-01", "2019-12-31"), # best guess, domains=["Medical", "Written"], task_subtypes=["Article retrieval"], - license="CC0 1.0 Universal", + license="cc0-1.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/NQRetrieval.py b/mteb/tasks/Retrieval/eng/NQRetrieval.py index 7ab8135f3f..0d11c0a4dc 100644 --- a/mteb/tasks/Retrieval/eng/NQRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NQRetrieval.py @@ -46,3 +46,46 @@ class NQ(AbsTaskRetrieval): }, }, ) + + +class NQHardNegatives(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NQHardNegatives", + dataset={ + "path": "mteb/NQ_test_top_250_only_w_correct-v2", + "revision": "d700fe4f167a5db8e6c9b03e8c26e7eaf66faf97", + }, + description="NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://ai.google.com/research/NaturalQuestions/", + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=None, + domains=None, + task_subtypes=None, + license=None, + annotations_creators=None, + dialect=None, + sample_creation=None, + bibtex_citation="""@article{47761,title = {Natural Questions: a Benchmark for Question Answering Research}, + author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh + and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee + and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le + and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational + Linguistics}}""", + descriptive_stats={ + "n_samples": {"test": 1000}, + "avg_character_length": { + "test": { + "average_document_length": 602.7903551179953, + "average_query_length": 47.878, + "num_documents": 198779, + "num_queries": 1000, + "average_relevant_docs_per_query": 1.213, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/eng/PiqaRetrieval.py b/mteb/tasks/Retrieval/eng/PiqaRetrieval.py index 631f408976..df2ae359b2 100644 --- a/mteb/tasks/Retrieval/eng/PiqaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/PiqaRetrieval.py @@ -24,7 +24,7 @@ class PIQA(AbsTaskRetrieval): date=("2020-01-01", "2020-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="AFL-3.0", + license="afl-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/QuailRetrieval.py b/mteb/tasks/Retrieval/eng/QuailRetrieval.py index 7f99f7abce..35e27da8a4 100644 --- a/mteb/tasks/Retrieval/eng/QuailRetrieval.py +++ b/mteb/tasks/Retrieval/eng/QuailRetrieval.py @@ -24,7 +24,7 @@ class Quail(AbsTaskRetrieval): date=("2020-01-01", "2020-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/QuoraRetrieval.py b/mteb/tasks/Retrieval/eng/QuoraRetrieval.py index 40e25ba4f2..378c1d35f6 100644 --- a/mteb/tasks/Retrieval/eng/QuoraRetrieval.py +++ b/mteb/tasks/Retrieval/eng/QuoraRetrieval.py @@ -59,3 +59,52 @@ class QuoraRetrieval(AbsTaskRetrieval): }, }, ) + + +class QuoraRetrievalHardNegatives(AbsTaskRetrieval): + ignore_identical_ids = True + + metadata = TaskMetadata( + name="QuoraRetrievalHardNegatives", + dataset={ + "path": "mteb/QuoraRetrieval_test_top_250_only_w_correct-v2", + "revision": "907a33577e9506221d3ba20f5a851b7c3f8dc6d3", + }, + description=( + "QuoraRetrieval is based on questions that are marked as duplicates on the Quora platform. Given a" + + " question, find other (duplicate) questions. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct." + ), + reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs", + type="Retrieval", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=None, + domains=None, + task_subtypes=None, + license=None, + annotations_creators=None, + dialect=None, + sample_creation=None, + bibtex_citation="""@misc{quora-question-pairs, + author = {DataCanary, hilfialkaff, Lili Jiang, Meg Risdal, Nikhil Dandekar, tomtung}, + title = {Quora Question Pairs}, + publisher = {Kaggle}, + year = {2017}, + url = {https://kaggle.com/competitions/quora-question-pairs} +}""", + descriptive_stats={ + "n_samples": {"test": 1000}, + "avg_character_length": { + "test": { + "average_document_length": 58.96963812985781, + "average_query_length": 51.228, + "num_documents": 177163, + "num_queries": 1000, + "average_relevant_docs_per_query": 1.641, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/eng/RARbCodeRetrieval.py b/mteb/tasks/Retrieval/eng/RARbCodeRetrieval.py index 094aaeea7a..bf5bb87d14 100644 --- a/mteb/tasks/Retrieval/eng/RARbCodeRetrieval.py +++ b/mteb/tasks/Retrieval/eng/RARbCodeRetrieval.py @@ -24,7 +24,7 @@ class RARbCode(AbsTaskRetrieval): date=("2019-01-01", "2023-12-31"), domains=["Programming", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY-NC-SA 4.0", + license="cc-by-nc-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/RARbMathRetrieval.py b/mteb/tasks/Retrieval/eng/RARbMathRetrieval.py index fc2008f272..a41e6960a6 100644 --- a/mteb/tasks/Retrieval/eng/RARbMathRetrieval.py +++ b/mteb/tasks/Retrieval/eng/RARbMathRetrieval.py @@ -24,7 +24,7 @@ class RARbMath(AbsTaskRetrieval): date=("2021-01-01", "2023-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/SiqaRetrieval.py b/mteb/tasks/Retrieval/eng/SiqaRetrieval.py index c316a56427..a3c30d8021 100644 --- a/mteb/tasks/Retrieval/eng/SiqaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/SiqaRetrieval.py @@ -24,7 +24,7 @@ class SIQA(AbsTaskRetrieval): date=("2019-01-01", "2019-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/SpartQARetrieval.py b/mteb/tasks/Retrieval/eng/SpartQARetrieval.py index e7c13854f1..5e81117121 100644 --- a/mteb/tasks/Retrieval/eng/SpartQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/SpartQARetrieval.py @@ -24,7 +24,7 @@ class SpartQA(AbsTaskRetrieval): date=("2021-01-01", "2021-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/TempReasonL1Retrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL1Retrieval.py index f490f3c670..c057f78c6c 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL1Retrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL1Retrieval.py @@ -24,7 +24,7 @@ class TempReasonL1(AbsTaskRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY-SA 3.0", + license="cc-by-sa-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/TempReasonL2ContextRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL2ContextRetrieval.py index 780a5f1682..3ed662a548 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL2ContextRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL2ContextRetrieval.py @@ -24,7 +24,7 @@ class TempReasonL2Context(AbsTaskRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY-SA 3.0", + license="cc-by-sa-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/TempReasonL2FactRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL2FactRetrieval.py index ca85bc6704..ce62c02f81 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL2FactRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL2FactRetrieval.py @@ -24,7 +24,7 @@ class TempReasonL2Fact(AbsTaskRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY-SA 3.0", + license="cc-by-sa-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/TempReasonL2PureRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL2PureRetrieval.py index 99293b15a4..8b775752c1 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL2PureRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL2PureRetrieval.py @@ -24,7 +24,7 @@ class TempReasonL2Pure(AbsTaskRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY-SA 3.0", + license="cc-by-sa-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/TempReasonL3ContextRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL3ContextRetrieval.py index eba9bb5b49..e1a43b3d92 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL3ContextRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL3ContextRetrieval.py @@ -24,7 +24,7 @@ class TempReasonL3Context(AbsTaskRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY-SA 3.0", + license="cc-by-sa-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/TempReasonL3FactRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL3FactRetrieval.py index 49635deb14..bd9d017e53 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL3FactRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL3FactRetrieval.py @@ -24,7 +24,7 @@ class TempReasonL3Fact(AbsTaskRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY-SA 3.0", + license="cc-by-sa-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/TempReasonL3PureRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL3PureRetrieval.py index 399ec3e371..162a9988aa 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL3PureRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL3PureRetrieval.py @@ -24,7 +24,7 @@ class TempReasonL3Pure(AbsTaskRetrieval): date=("2023-01-01", "2023-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY-SA 3.0", + license="cc-by-sa-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py b/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py index 343173e04a..415bc3045b 100644 --- a/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py @@ -106,3 +106,55 @@ def _load_data_for_split(self, dataset_path, split): } return corpus, queries, qrels + + +class TopiOCQARetrievalHardNegatives(AbsTaskRetrieval): + metadata = TaskMetadata( + name="TopiOCQAHardNegatives", + dataset={ + "path": "mteb/TopiOCQA_validation_top_250_only_w_correct-v2", + "revision": "b4cc09fb8bb3a9e0ce0f94dc69c96397a2a47c18", + "trust_remote_code": True, + }, + reference="https://mcgill-nlp.github.io/topiocqa", + description=( + "TopiOCQA (Human-in-the-loop Attributable Generative Retrieval for Information-seeking Dataset) " + + "is information-seeking conversational dataset with challenging topic switching phenomena. " + + "It consists of conversation histories along with manually labelled relevant/gold passage. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct." + ), + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["validation"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2021-03-01", "2021-07-31"), + domains=["Encyclopaedic", "Written"], + task_subtypes=["Conversational retrieval"], + license="cc-by-nc-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation=""" + @misc{adlakha2022topiocqa, + title={TopiOCQA: Open-domain Conversational Question Answering with Topic Switching}, + author={Vaibhav Adlakha and Shehzaad Dhuliawala and Kaheer Suleman and Harm de Vries and Siva Reddy}, + year={2022}, + eprint={2110.00768}, + archivePrefix={arXiv}, + primaryClass={cs.CL} + } + """, + descriptive_stats={ + "n_samples": {"test": 1000}, + "avg_character_length": { + "validation": { + "average_document_length": 538.7586536643946, + "average_query_length": 12.85, + "num_documents": 89933, + "num_queries": 1000, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/eng/Touche2020Retrieval.py b/mteb/tasks/Retrieval/eng/Touche2020Retrieval.py index 2c9dc8df41..01b955b19d 100644 --- a/mteb/tasks/Retrieval/eng/Touche2020Retrieval.py +++ b/mteb/tasks/Retrieval/eng/Touche2020Retrieval.py @@ -1,11 +1,12 @@ from __future__ import annotations +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata -from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval - class Touche2020(AbsTaskRetrieval): + superseded_by = "Touche2020Retrieval.v3" + metadata = TaskMetadata( name="Touche2020", description="Touché Task 1: Argument Retrieval for Controversial Questions", @@ -20,13 +21,13 @@ class Touche2020(AbsTaskRetrieval): eval_splits=["test"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", - date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + date=("2020-09-23", "2020-09-23"), + domains=["Academic"], + task_subtypes=["Question answering"], + license="cc-by-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", bibtex_citation="""@dataset{potthast_2022_6862281, author = {Potthast, Martin and Gienapp, Lukas and @@ -57,3 +58,44 @@ class Touche2020(AbsTaskRetrieval): }, }, ) + + +class Touche2020v3Retrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="Touche2020Retrieval.v3", + description="Touché Task 1: Argument Retrieval for Controversial Questions", + reference="https://github.com/castorini/touche-error-analysis", + dataset={ + "path": "mteb/webis-touche2020-v3", + "revision": "431886eaecc48f067a3975b70d0949ea2862463c", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2020-09-23", "2020-09-23"), + domains=["Academic"], + task_subtypes=["Question answering"], + license="cc-by-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@INPROCEEDINGS{Thakur_etal_SIGIR2024, + author = "Nandan Thakur and Luiz Bonifacio and Maik {Fr\"{o}be} and Alexander Bondarenko and Ehsan Kamalloo and Martin Potthast and Matthias Hagen and Jimmy Lin", + title = "Systematic Evaluation of Neural Retrieval Models on the {Touch\'{e}} 2020 Argument Retrieval Subset of {BEIR}", + booktitle = "Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval", + year = 2024, + address_ = "Washington, D.C." +}""", + descriptive_stats={ + "test": { + "average_document_length": 2096.391812518931, + "average_query_length": 43.42857142857143, + "num_documents": 303732, + "num_queries": 49, + "average_relevant_docs_per_query": 34.93877551020408, + } + }, + ) diff --git a/mteb/tasks/Retrieval/eng/WinoGrandeRetrieval.py b/mteb/tasks/Retrieval/eng/WinoGrandeRetrieval.py index 7d39ce81a9..13cfe8f727 100644 --- a/mteb/tasks/Retrieval/eng/WinoGrandeRetrieval.py +++ b/mteb/tasks/Retrieval/eng/WinoGrandeRetrieval.py @@ -24,7 +24,7 @@ class WinoGrande(AbsTaskRetrieval): date=("2021-01-01", "2021-12-31"), domains=["Encyclopaedic", "Written"], task_subtypes=["Reasoning as Retrieval"], - license="CC BY", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/est/estqa.py b/mteb/tasks/Retrieval/est/estqa.py index 492a59c601..28efd3a71a 100644 --- a/mteb/tasks/Retrieval/est/estqa.py +++ b/mteb/tasks/Retrieval/est/estqa.py @@ -28,7 +28,7 @@ class EstQA(AbsTaskRetrieval): ), # birth of Estonian Wikipedia to publishing the article domains=["Encyclopaedic", "Written"], task_subtypes=["Question answering"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/fra/SyntecRetrieval.py b/mteb/tasks/Retrieval/fra/SyntecRetrieval.py index c671d64270..4240de38f3 100644 --- a/mteb/tasks/Retrieval/fra/SyntecRetrieval.py +++ b/mteb/tasks/Retrieval/fra/SyntecRetrieval.py @@ -27,7 +27,7 @@ class SyntecRetrieval(AbsTaskRetrieval): date=None, # not specified domains=["Legal", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/Retrieval/hun/HunSum2.py b/mteb/tasks/Retrieval/hun/HunSum2.py index 7af6c0bea0..1ef5808507 100644 --- a/mteb/tasks/Retrieval/hun/HunSum2.py +++ b/mteb/tasks/Retrieval/hun/HunSum2.py @@ -30,7 +30,7 @@ class HunSum2AbstractiveRetrieval(AbsTaskRetrieval): ), domains=["News", "Written"], task_subtypes=["Article retrieval"], - license="CC-BY 4.0", + license="cc-by-4.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/jpn/JaQuADRetrieval.py b/mteb/tasks/Retrieval/jpn/JaQuADRetrieval.py index fc5bc6ea4b..b47cecaa98 100644 --- a/mteb/tasks/Retrieval/jpn/JaQuADRetrieval.py +++ b/mteb/tasks/Retrieval/jpn/JaQuADRetrieval.py @@ -25,7 +25,7 @@ class JaQuADRetrieval(AbsTaskRetrieval): date=("2022-01-01", "2022-12-31"), # approximate guess domains=["Encyclopaedic", "Non-fiction", "Written"], task_subtypes=["Question answering"], - license="CC-BY-SA-3.0", + license="cc-by-sa-3.0", annotations_creators="human-annotated", dialect=None, sample_creation="found", diff --git a/mteb/tasks/Retrieval/jpn/JaqketRetrieval.py b/mteb/tasks/Retrieval/jpn/JaqketRetrieval.py new file mode 100644 index 0000000000..0af7d06772 --- /dev/null +++ b/mteb/tasks/Retrieval/jpn/JaqketRetrieval.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class JaqketRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="JaqketRetrieval", + dataset={ + "path": "mteb/jaqket", + "revision": "3a5b92dad489a61e664c05ed2175bc9220230199", + }, + description="JAQKET (JApanese Questions on Knowledge of EnTities) is a QA dataset that is created based on quiz questions.", + reference="https://github.com/kumapo/JAQKET-dataset", + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["jpn-Jpan"], + main_score="ndcg_at_10", + date=("2023-10-09", "2023-10-09"), + domains=["Encyclopaedic", "Non-fiction", "Written"], + task_subtypes=["Question answering"], + license="cc-by-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@InProceedings{Kurihara_nlp2020, +author = "鈴木正敏 and 鈴木潤 and 松田耕史 and ⻄田京介 and 井之上直也", +title = "JAQKET: クイズを題材にした日本語 QA データセットの構築", +booktitle = "言語処理学会第26回年次大会", +year = "2020", +url = "https://www.anlp.jp/proceedings/annual_meeting/2020/pdf_dir/P2-24.pdf" +note= "in Japanese" +}""", + descriptive_stats={ + "test": { + "average_document_length": 3747.995228882333, + "average_query_length": 50.70611835506519, + "num_documents": 114229, + "num_queries": 997, + "average_relevant_docs_per_query": 1.0, + } + }, + ) diff --git a/mteb/tasks/Retrieval/kat/GeorgianFAQRetrieval.py b/mteb/tasks/Retrieval/kat/GeorgianFAQRetrieval.py index 00991a51e9..82b0392faa 100644 --- a/mteb/tasks/Retrieval/kat/GeorgianFAQRetrieval.py +++ b/mteb/tasks/Retrieval/kat/GeorgianFAQRetrieval.py @@ -30,7 +30,7 @@ class GeorgianFAQRetrieval(AbsTaskRetrieval): reference="https://huggingface.co/datasets/jupyterjazz/georgian-faq", date=("2024-05-02", "2024-05-03"), task_subtypes=["Question answering"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], bibtex_citation="", diff --git a/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py b/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py index e17045b90a..8066b4e2d0 100644 --- a/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py @@ -195,7 +195,7 @@ class BelebeleRetrieval(MultilingualTask, AbsTaskRetrieval): eval_langs=_LANGUAGES_MAPPING, reference="https://arxiv.org/abs/2308.16884", main_score="ndcg_at_10", - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", domains=["Web", "News", "Written"], sample_creation="created", # number of languages * 900 date=("2023-08-31", "2023-08-31"), diff --git a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py index 4783e5c05a..d46c87e7e8 100644 --- a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py +++ b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py @@ -48,7 +48,7 @@ class CrossLingualSemanticDiscriminationWMT19(AbsTaskRetrieval, MultilingualTask date=("2018-01-01", "2023-12-12"), domains=["News", "Written"], task_subtypes=["Cross-Lingual Semantic Discrimination"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="LM-generated and verified", diff --git a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py index dcd6d2fc96..871eb030be 100644 --- a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py +++ b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py @@ -48,7 +48,7 @@ class CrossLingualSemanticDiscriminationWMT21(AbsTaskRetrieval, MultilingualTask date=("2020-01-01", "2023-12-12"), domains=["News", "Written"], task_subtypes=["Cross-Lingual Semantic Discrimination"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="LM-generated and verified", diff --git a/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py b/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py index 61395e2b22..2c6e1c70eb 100644 --- a/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py @@ -43,7 +43,7 @@ class IndicQARetrieval(MultilingualTask, AbsTaskRetrieval): date=("2022-08-01", "2022-12-20"), domains=["Web", "Written"], task_subtypes=[], - license="CC0", + license="cc0-1.0", annotations_creators="human-annotated", dialect=[], sample_creation="machine-translated and verified", diff --git a/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py b/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py index 6c2c1b2519..f9e5239c5f 100644 --- a/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py @@ -115,7 +115,7 @@ class MIRACLRetrieval(MultilingualTask, AbsTaskRetrieval): date=("2022-06-01", "2023-01-30"), domains=["Encyclopaedic", "Written"], task_subtypes=[], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="expert-annotated", dialect=[], sample_creation="created", @@ -281,3 +281,330 @@ def load_data(self, **kwargs): ) self.data_loaded = True + + +def _load_miracl_data_hard_negatives( + path: str, langs: list, splits: str, cache_dir: str = None, revision: str = None +): + corpus = {lang: {split: None for split in splits} for lang in langs} + queries = {lang: {split: None for split in splits} for lang in langs} + relevant_docs = {lang: {split: None for split in splits} for lang in langs} + + split = _EVAL_SPLIT + + for lang in langs: + # subsampled langs: th,en,de,fr,es,ru,ja,fa,ar,fi,ko,id,te,hi,zh + if lang in [ + "th", + "en", + "de", + "fr", + "es", + "ru", + "ja", + "fa", + "ar", + "fi", + "ko", + "id", + "te", + "hi", + "zh", + ]: + # load the hard negatives miracle dataset + # Load corpus data + print(f"Loading data for {lang}") + corpus_identifier = f"corpus-{lang}" + corpus_data = datasets.load_dataset( + path, + corpus_identifier, + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + corpus[lang][split] = {} + for row in corpus_data["corpus"]: + docid = row["_id"] + doc_title = row["title"] + doc_text = row["text"] + corpus[lang][split][docid] = {"title": doc_title, "text": doc_text} + + # Load queries data + queries_identifier = f"queries-{lang}" + queries_data = datasets.load_dataset( + path, + queries_identifier, + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + queries[lang][split] = {} + for row in queries_data["queries"]: + query_id = row["_id"] + query_text = row["text"] + queries[lang][split][query_id] = query_text + + # Load relevant documents data + qrels_identifier = f"{lang}" + qrels_data = datasets.load_dataset( + path, + qrels_identifier, + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + relevant_docs[lang][split] = {} + for row in qrels_data[split]: + query_id = row["query-id"] + doc_id = row["corpus-id"] + score = row["score"] + if query_id not in relevant_docs[lang][split]: + relevant_docs[lang][split][query_id] = {} + relevant_docs[lang][split][query_id][doc_id] = score + + else: + corpus_identifier = f"corpus-{lang}" + corpus_data = datasets.load_dataset( + "miracl/mmteb-miracl", + corpus_identifier, + cache_dir=cache_dir, + trust_remote_code=True, + ) + corpus[lang][split] = {} + for row in corpus_data["corpus"]: + docid = row["docid"] + doc_title = row["title"] + doc_text = row["text"] + corpus[lang][split][docid] = {"title": doc_title, "text": doc_text} + + # Load queries data + queries_identifier = f"queries-{lang}" + queries_data = datasets.load_dataset( + "miracl/mmteb-miracl", + queries_identifier, + cache_dir=cache_dir, + trust_remote_code=True, + ) + queries[lang][split] = {} + for row in queries_data["queries"]: + query_id = row["query_id"] + query_text = row["query"] + queries[lang][split][query_id] = query_text + + # Load relevant documents data + qrels_identifier = f"{lang}" + qrels_data = datasets.load_dataset( + "miracl/mmteb-miracl", + qrels_identifier, + cache_dir=cache_dir, + trust_remote_code=True, + ) + relevant_docs[lang][split] = {} + for row in qrels_data[split]: + query_id = row["query_id"] + doc_id = row["docid"] + score = row["score"] + if query_id not in relevant_docs[lang][split]: + relevant_docs[lang][split][query_id] = {} + relevant_docs[lang][split][query_id][doc_id] = score + + corpus = datasets.DatasetDict(corpus) + queries = datasets.DatasetDict(queries) + relevant_docs = datasets.DatasetDict(relevant_docs) + + return corpus, queries, relevant_docs + + +class MIRACLRetrievalHardNegatives(MultilingualTask, AbsTaskRetrieval): + metadata = TaskMetadata( + name="MIRACLRetrievalHardNegatives", + description="MIRACL (Multilingual Information Retrieval Across a Continuum of Languages) is a multilingual retrieval dataset that focuses on search across 18 different languages. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="http://miracl.ai", + dataset={ + "path": "mteb/miracl-hard-negatives", + "revision": "95c8db7d4a6e9c1d8a60601afd63d553ae20a2eb", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=[_EVAL_SPLIT], + eval_langs=_LANGUAGES, + main_score="ndcg_at_10", + date=("2022-06-01", "2023-01-30"), + domains=["Encyclopaedic", "Written"], + task_subtypes=[], + license="cc-by-sa-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="created", + bibtex_citation="""@article{10.1162/tacl_a_00595, + author = {Zhang, Xinyu and Thakur, Nandan and Ogundepo, Odunayo and Kamalloo, Ehsan and Alfonso-Hermelo, David and Li, Xiaoguang and Liu, Qun and Rezagholizadeh, Mehdi and Lin, Jimmy}, + title = "{MIRACL: A Multilingual Retrieval Dataset Covering 18 Diverse Languages}", + journal = {Transactions of the Association for Computational Linguistics}, + volume = {11}, + pages = {1114-1131}, + year = {2023}, + month = {09}, + abstract = "{MIRACL is a multilingual dataset for ad hoc retrieval across 18 languages that collectively encompass over three billion native speakers around the world. This resource is designed to support monolingual retrieval tasks, where the queries and the corpora are in the same language. In total, we have gathered over 726k high-quality relevance judgments for 78k queries over Wikipedia in these languages, where all annotations have been performed by native speakers hired by our team. MIRACL covers languages that are both typologically close as well as distant from 10 language families and 13 sub-families, associated with varying amounts of publicly available resources. Extensive automatic heuristic verification and manual assessments were performed during the annotation process to control data quality. In total, MIRACL represents an investment of around five person-years of human annotator effort. Our goal is to spur research on improving retrieval across a continuum of languages, thus enhancing information access capabilities for diverse populations around the world, particularly those that have traditionally been underserved. MIRACL is available at http://miracl.ai/.}", + issn = {2307-387X}, + doi = {10.1162/tacl_a_00595}, + url = {https://doi.org/10.1162/tacl\_a\_00595}, + eprint = {https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl\_a\_00595/2157340/tacl\_a\_00595.pdf}, +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "dev": { + "average_document_length": 417.6655323669399, + "average_query_length": 37.46957385337667, + "num_documents": 2449382, + "num_queries": 11076, + "average_relevant_docs_per_query": 2.3643011917659806, + "hf_subset_descriptive_stats": { + "ar": { + "average_document_length": 438.1872433017704, + "average_query_length": 29.584, + "num_documents": 192103, + "num_queries": 1000, + "average_relevant_docs_per_query": 1.982, + }, + "bn": { + "average_document_length": 383.2428136511194, + "average_query_length": 46.98053527980535, + "num_documents": 297265, + "num_queries": 411, + "average_relevant_docs_per_query": 2.099756690997567, + }, + "de": { + "average_document_length": 513.7796484139344, + "average_query_length": 46.0, + "num_documents": 71277, + "num_queries": 305, + "average_relevant_docs_per_query": 2.6590163934426227, + }, + "en": { + "average_document_length": 529.2486406963214, + "average_query_length": 40.247809762202756, + "num_documents": 178768, + "num_queries": 799, + "average_relevant_docs_per_query": 2.911138923654568, + }, + "es": { + "average_document_length": 535.8023645655877, + "average_query_length": 47.373456790123456, + "num_documents": 146750, + "num_queries": 648, + "average_relevant_docs_per_query": 4.609567901234568, + }, + "fa": { + "average_document_length": 411.2648282882721, + "average_query_length": 41.1503164556962, + "num_documents": 133596, + "num_queries": 632, + "average_relevant_docs_per_query": 2.079113924050633, + }, + "fi": { + "average_document_length": 462.9445310289844, + "average_query_length": 38.646, + "num_documents": 194415, + "num_queries": 1000, + "average_relevant_docs_per_query": 1.918, + }, + "fr": { + "average_document_length": 460.40909271865917, + "average_query_length": 43.883381924198254, + "num_documents": 75357, + "num_queries": 343, + "average_relevant_docs_per_query": 2.131195335276968, + }, + "hi": { + "average_document_length": 498.6759426632417, + "average_query_length": 53.34, + "num_documents": 63066, + "num_queries": 350, + "average_relevant_docs_per_query": 2.1485714285714286, + }, + "id": { + "average_document_length": 494.1689807519638, + "average_query_length": 37.958333333333336, + "num_documents": 168173, + "num_queries": 960, + "average_relevant_docs_per_query": 3.216666666666667, + }, + "ja": { + "average_document_length": 206.13654293407583, + "average_query_length": 17.71395348837209, + "num_documents": 185319, + "num_queries": 860, + "average_relevant_docs_per_query": 2.0813953488372094, + }, + "ko": { + "average_document_length": 257.82646155267594, + "average_query_length": 21.624413145539908, + "num_documents": 43293, + "num_queries": 213, + "average_relevant_docs_per_query": 2.568075117370892, + }, + "ru": { + "average_document_length": 476.0820349224605, + "average_query_length": 44.055, + "num_documents": 219114, + "num_queries": 1000, + "average_relevant_docs_per_query": 2.833, + }, + "sw": { + "average_document_length": 228.71348655286377, + "average_query_length": 38.97095435684647, + "num_documents": 131924, + "num_queries": 482, + "average_relevant_docs_per_query": 1.887966804979253, + }, + "te": { + "average_document_length": 601.7099283059209, + "average_query_length": 38.11231884057971, + "num_documents": 101961, + "num_queries": 828, + "average_relevant_docs_per_query": 1.0314009661835748, + }, + "th": { + "average_document_length": 478.8818849711528, + "average_query_length": 42.87585266030014, + "num_documents": 116649, + "num_queries": 733, + "average_relevant_docs_per_query": 1.8321964529331514, + }, + "yo": { + "average_document_length": 159.35250698366738, + "average_query_length": 37.6890756302521, + "num_documents": 49043, + "num_queries": 119, + "average_relevant_docs_per_query": 1.2100840336134453, + }, + "zh": { + "average_document_length": 147.36211243527777, + "average_query_length": 10.867684478371501, + "num_documents": 81309, + "num_queries": 393, + "average_relevant_docs_per_query": 2.5292620865139948, + }, + }, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus, self.queries, self.relevant_docs = ( + _load_miracl_data_hard_negatives( + path=self.metadata_dict["dataset"]["path"], + langs=self.hf_subsets, + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + ) + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py b/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py index 70d9dba09d..2850438c3c 100644 --- a/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py @@ -83,7 +83,7 @@ class MintakaRetrieval(MultilingualTask, AbsTaskRetrieval): date=("2022-01-01", "2022-01-01"), # best guess: based on the date of the paper domains=["Encyclopaedic", "Written"], task_subtypes=["Question answering"], - license="CC-BY-4.0", + license="cc-by-4.0", annotations_creators="derived", # best guess dialect=[], sample_creation="human-translated", diff --git a/mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py b/mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py new file mode 100644 index 0000000000..f7bf5f9dc8 --- /dev/null +++ b/mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import logging + +import datasets + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.MultilingualTask import MultilingualTask +from mteb.abstasks.TaskMetadata import TaskMetadata + +_EVAL_LANGS = { + "bengali": ["ben-Beng"], + "english": ["eng-Latn"], + "finnish": ["fin-Latn"], + "russian": ["rus-Cyrl"], + "korean": ["kor-Kore"], + "japanese": ["jpn-Jpan"], + "telugu": ["tel-Telu"], + "thai": ["tha-Thai"], + "swahili": ["swa-Latn"], + "arabic": ["ara-Arab"], + "indonesian": ["ind-Latn"], +} +_EVAL_SPLIT = "test" + +logger = logging.getLogger(__name__) + + +def _load_data_retrieval( + path: str, langs: list, splits: str, cache_dir: str = None, revision: str = None +): + corpus = {lang: {split: {} for split in splits} for lang in langs} + queries = {lang: {split: {} for split in splits} for lang in langs} + relevant_docs = {lang: {split: {} for split in splits} for lang in langs} + + split = _EVAL_SPLIT + + for lang in langs: + qrels_data = datasets.load_dataset( + path, + name=f"{lang}-qrels", + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + )[split] + + for row in qrels_data: + query_id = row["query-id"] + doc_id = row["corpus-id"] + score = row["score"] + if query_id not in relevant_docs[lang][split]: + relevant_docs[lang][split][query_id] = {} + relevant_docs[lang][split][query_id][doc_id] = score + + corpus_data = datasets.load_dataset( + path, + name=f"{lang}-corpus", + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + )["train"] + + for row in corpus_data: + doc_id = row["_id"] + doc_title = row["title"] + doc_text = row["text"] + corpus[lang][split][doc_id] = {"title": doc_title, "text": doc_text} + + queries_data = datasets.load_dataset( + path, + name=f"{lang}-queries", + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + )[split] + + for row in queries_data: + query_id = row["_id"] + query_text = row["text"] + queries[lang][split][query_id] = query_text + + queries = queries + logger.info("Loaded %d %s Queries.", len(queries), split.upper()) + + return corpus, queries, relevant_docs + + +class MrTidyRetrieval(MultilingualTask, AbsTaskRetrieval): + metadata = TaskMetadata( + name="MrTidyRetrieval", + description="Mr. TyDi is a multi-lingual benchmark dataset built on TyDi, covering eleven typologically diverse languages. It is designed for monolingual retrieval, specifically to evaluate ranking with learned dense representations.", + reference="https://huggingface.co/datasets/castorini/mr-tydi", + dataset={ + "path": "mteb/mrtidy", + "revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=_EVAL_LANGS, + main_score="ndcg_at_10", + date=("2023-11-01", "2024-05-15"), + domains=["Encyclopaedic", "Written"], + task_subtypes=[], + license="cc-by-sa-3.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{mrtydi, + title={{Mr. TyDi}: A Multi-lingual Benchmark for Dense Retrieval}, + author={Xinyu Zhang and Xueguang Ma and Peng Shi and Jimmy Lin}, + year={2021}, + journal={arXiv:2108.08787}, + }""", + descriptive_stats={}, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus, self.queries, self.relevant_docs = _load_data_retrieval( + path=self.metadata_dict["dataset"]["path"], + langs=self.hf_subsets, + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py b/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py index 5aa7746764..893f3b51e0 100644 --- a/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py +++ b/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py @@ -128,3 +128,153 @@ def load_data(self, **kwargs): revision=self.metadata_dict["dataset"]["revision"], ) self.data_loaded = True + + +def load_neuclir_data_hard_negatives( + path: str, + langs: list, + eval_splits: list, + cache_dir: str | None = None, + revision: str | None = None, +): + split = "test" + corpus = {lang: {split: None for split in eval_splits} for lang in langs} + queries = {lang: {split: None for split in eval_splits} for lang in langs} + relevant_docs = {lang: {split: None for split in eval_splits} for lang in langs} + + for lang in langs: + corpus_identifier = f"corpus-{lang}" + corpus_data = datasets.load_dataset( + path, + corpus_identifier, + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + corpus[lang][split] = {} + for row in corpus_data["corpus"]: + docid = row["_id"] + doc_title = row["title"] + doc_text = row["text"] + corpus[lang][split][docid] = {"title": doc_title, "text": doc_text} + + # Load queries data + queries_identifier = f"queries-{lang}" + queries_data = datasets.load_dataset( + path, + queries_identifier, + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + queries[lang][split] = {} + for row in queries_data["queries"]: + query_id = row["_id"] + query_text = row["text"] + queries[lang][split][query_id] = query_text + + # Load relevant documents data + qrels_identifier = f"{lang}" + qrels_data = datasets.load_dataset( + path, + qrels_identifier, + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + relevant_docs[lang][split] = {} + for row in qrels_data[split]: + query_id = row["query-id"] + doc_id = row["corpus-id"] + score = row["score"] + if query_id not in relevant_docs[lang][split]: + relevant_docs[lang][split][query_id] = {} + relevant_docs[lang][split][query_id][doc_id] = score + + corpus = datasets.DatasetDict(corpus) + queries = datasets.DatasetDict(queries) + relevant_docs = datasets.DatasetDict(relevant_docs) + + return corpus, queries, relevant_docs + + +class NeuCLIR2022RetrievalHardNegatives(MultilingualTask, AbsTaskRetrieval): + metadata = TaskMetadata( + name="NeuCLIR2022RetrievalHardNegatives", + description="The task involves identifying and retrieving the documents that are relevant to the queries. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://neuclir.github.io/", + dataset={ + "path": "mteb/neuclir-2022-hard-negatives", + "revision": "35dd709a0d846ae987541cf8ca978562636260f0", + "trust_remote_code": True, + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=_LANGUAGES, + main_score="ndcg_at_20", + date=("2021-08-01", "2022-06-30"), + domains=["News", "Written"], + task_subtypes=[], + license="odc-by", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{lawrie2023overview, + title={Overview of the TREC 2022 NeuCLIR track}, + author={Lawrie, Dawn and MacAvaney, Sean and Mayfield, James and McNamee, Paul and Oard, Douglas W and Soldaini, Luca and Yang, Eugene}, + journal={arXiv preprint arXiv:2304.12367}, + year={2023} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 2066.9453653646488, + "average_query_length": 63.529411764705884, + "num_documents": 27931, + "num_queries": 136, + "average_relevant_docs_per_query": 40.39705882352941, + "hf_subset_descriptive_stats": { + "fas": { + "average_document_length": 2816.847782031074, + "average_query_length": 83.26666666666667, + "num_documents": 8882, + "num_queries": 45, + "average_relevant_docs_per_query": 32.71111111111111, + }, + "rus": { + "average_document_length": 2446.5574277854193, + "average_query_length": 85.56818181818181, + "num_documents": 8724, + "num_queries": 44, + "average_relevant_docs_per_query": 42.93181818181818, + }, + "zho": { + "average_document_length": 1101.0984987893462, + "average_query_length": 24.0, + "num_documents": 10325, + "num_queries": 47, + "average_relevant_docs_per_query": 45.38297872340426, + }, + }, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus, self.queries, self.relevant_docs = ( + load_neuclir_data_hard_negatives( + path=self.metadata_dict["dataset"]["path"], + langs=self.metadata.eval_langs, + eval_splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + ) + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py b/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py index 6ef364e7d9..2cde1a6e28 100644 --- a/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py +++ b/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py @@ -129,3 +129,155 @@ def load_data(self, **kwargs): revision=self.metadata_dict["dataset"]["revision"], ) self.data_loaded = True + + +def load_neuclir_data_hard_negatives( + path: str, + langs: list, + eval_splits: list, + cache_dir: str | None = None, + revision: str | None = None, +): + split = "test" + corpus = {lang: {split: None for split in eval_splits} for lang in langs} + queries = {lang: {split: None for split in eval_splits} for lang in langs} + relevant_docs = {lang: {split: None for split in eval_splits} for lang in langs} + + for lang in langs: + corpus_identifier = f"corpus-{lang}" + corpus_data = datasets.load_dataset( + path, + corpus_identifier, + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + corpus[lang][split] = {} + for row in corpus_data["corpus"]: + docid = row["_id"] + doc_title = row["title"] + doc_text = row["text"] + corpus[lang][split][docid] = {"title": doc_title, "text": doc_text} + + # Load queries data + queries_identifier = f"queries-{lang}" + queries_data = datasets.load_dataset( + path, + queries_identifier, + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + queries[lang][split] = {} + for row in queries_data["queries"]: + query_id = row["_id"] + query_text = row["text"] + queries[lang][split][query_id] = query_text + + # Load relevant documents data + qrels_identifier = f"{lang}" + qrels_data = datasets.load_dataset( + path, + qrels_identifier, + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, + ) + relevant_docs[lang][split] = {} + for row in qrels_data[split]: + query_id = row["query-id"] + doc_id = row["corpus-id"] + score = row["score"] + if query_id not in relevant_docs[lang][split]: + relevant_docs[lang][split][query_id] = {} + relevant_docs[lang][split][query_id][doc_id] = score + + corpus = datasets.DatasetDict(corpus) + queries = datasets.DatasetDict(queries) + relevant_docs = datasets.DatasetDict(relevant_docs) + + return corpus, queries, relevant_docs + + +class NeuCLIR2023RetrievalHardNegatives(MultilingualTask, AbsTaskRetrieval): + metadata = TaskMetadata( + name="NeuCLIR2023RetrievalHardNegatives", + description="The task involves identifying and retrieving the documents that are relevant to the queries. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://neuclir.github.io/", + dataset={ + "path": "mteb/neuclir-2023-hard-negatives", + "revision": "5d47e924e632c333d3f087d945642af93b008d2b", + "trust_remote_code": True, + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=_LANGUAGES, + main_score="ndcg_at_20", + date=("2022-08-01", "2023-06-30"), + domains=["News", "Written"], + task_subtypes=[], + license="odc-by", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@misc{lawrie2024overview, + title={Overview of the TREC 2023 NeuCLIR Track}, + author={Dawn Lawrie and Sean MacAvaney and James Mayfield and Paul McNamee and Douglas W. Oard and Luca Soldaini and Eugene Yang}, + year={2024}, + eprint={2404.08071}, + archivePrefix={arXiv}, + primaryClass={cs.IR} +}""", + descriptive_stats={ + "n_samples": None, + "avg_character_length": { + "test": { + "average_document_length": 2236.175955333482, + "average_query_length": 54.10267857142857, + "num_documents": 49433, + "num_queries": 224, + "average_relevant_docs_per_query": 61.816964285714285, + "hf_subset_descriptive_stats": { + "fas": { + "average_document_length": 2895.869857421016, + "average_query_length": 65.89189189189189, + "num_documents": 15921, + "num_queries": 74, + "average_relevant_docs_per_query": 68.08108108108108, + }, + "rus": { + "average_document_length": 2724.294762109928, + "average_query_length": 74.41333333333333, + "num_documents": 16247, + "num_queries": 75, + "average_relevant_docs_per_query": 63.053333333333335, + }, + "zho": { + "average_document_length": 1168.4984071821605, + "average_query_length": 22.16, + "num_documents": 17265, + "num_queries": 75, + "average_relevant_docs_per_query": 54.4, + }, + }, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus, self.queries, self.relevant_docs = ( + load_neuclir_data_hard_negatives( + path=self.metadata_dict["dataset"]["path"], + langs=self.metadata.eval_langs, + eval_splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + ) + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py b/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py index a85947b8b8..b1526a42a7 100644 --- a/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py @@ -78,7 +78,7 @@ class PublicHealthQARetrieval(MultilingualTask, AbsTaskRetrieval): date=("2020-01-01", "2020-04-15"), domains=["Medical", "Government", "Web", "Written"], task_subtypes=["Question answering"], - license="CC BY-NC-SA 3.0", + license="cc-by-nc-sa-3.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py b/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py index 47b5e9a254..cb5d7d618f 100644 --- a/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py @@ -81,7 +81,7 @@ class XPQARetrieval(AbsTaskRetrieval, MultilingualTask): date=("2022-01-01", "2023-07-31"), # best guess domains=["Reviews", "Written"], task_subtypes=["Question answering"], - license="CDLA-Sharing-1.0", + license="cdla-sharing-1.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py b/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py index d796be51c8..a4772002cf 100644 --- a/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py @@ -43,7 +43,7 @@ class XQuADRetrieval(MultilingualTask, AbsTaskRetrieval): date=("2019-05-21", "2019-11-21"), domains=["Web", "Written"], task_subtypes=["Question answering"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/Retrieval/nob/norquad.py b/mteb/tasks/Retrieval/nob/norquad.py index 7aa95ece08..ce11b4b710 100644 --- a/mteb/tasks/Retrieval/nob/norquad.py +++ b/mteb/tasks/Retrieval/nob/norquad.py @@ -24,7 +24,7 @@ class NorQuadRetrieval(AbsTaskRetrieval): date=("2022-01-01", "2023-12-31"), task_subtypes=["Question answering"], domains=["Encyclopaedic", "Non-fiction", "Written"], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", @@ -71,9 +71,9 @@ def load_data(self, **kwargs): def dataset_transform(self) -> None: """And transform to a retrieval datset, which have the following attributes - self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text - self.queries = Dict[query_id, str] #id => query - self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + self.corpus = dict[doc_id, dict[str, str]] #id => dict with document datas like title and text + self.queries = dict[query_id, str] #id => query + self.relevant_docs = dict[query_id, dict[[doc_id, score]] """ self.corpus = {} self.relevant_docs = {} diff --git a/mteb/tasks/Retrieval/nob/snl_retrieval.py b/mteb/tasks/Retrieval/nob/snl_retrieval.py index 185c9e5441..3c8045fe6d 100644 --- a/mteb/tasks/Retrieval/nob/snl_retrieval.py +++ b/mteb/tasks/Retrieval/nob/snl_retrieval.py @@ -23,7 +23,7 @@ class SNLRetrieval(AbsTaskRetrieval): main_score="ndcg_at_10", date=("2020-01-01", "2024-12-31"), # best guess domains=["Encyclopaedic", "Non-fiction", "Written"], - license="CC-BY-NC", + license="cc-by-nc-4.0", # version assumed (not specified beforehand) annotations_creators="derived", dialect=[], sample_creation="found", @@ -59,9 +59,9 @@ def load_data(self, **kwargs): def dataset_transform(self) -> None: """And transform to a retrieval datset, which have the following attributes - self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text - self.queries = Dict[query_id, str] #id => query - self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + self.corpus = dict[doc_id, dict[str, str]] #id => dict with document datas like title and text + self.queries = dict[query_id, str] #id => query + self.relevant_docs = dict[query_id, dict[[doc_id, score]] """ self.corpus = {} self.relevant_docs = {} diff --git a/mteb/tasks/Retrieval/pol/DBPediaPLRetrieval.py b/mteb/tasks/Retrieval/pol/DBPediaPLRetrieval.py index 9b268229f5..8d01491463 100644 --- a/mteb/tasks/Retrieval/pol/DBPediaPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/DBPediaPLRetrieval.py @@ -51,3 +51,51 @@ class DBPediaPL(AbsTaskRetrieval): }, }, ) + + +class DBPediaPLHardNegatives(AbsTaskRetrieval): + metadata = TaskMetadata( + name="DBPedia-PLHardNegatives", + description="DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://github.com/iai-group/DBpedia-Entity/", + dataset={ + "path": "mteb/DBPedia_PL_test_top_250_only_w_correct-v2", + "revision": "bebc2b5c8f73cd6ba9d2a4664d5f3769e6ad557a", + "trust_remote_code": True, + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["pol-Latn"], + main_score="ndcg_at_10", + date=("2017-01-01", "2017-01-01"), # best guess: based on publication date + domains=["Written", "Encyclopaedic"], + task_subtypes=[], + license="mit", + annotations_creators="derived", + dialect=[], + sample_creation="machine-translated", + bibtex_citation="""@inproceedings{Hasibi:2017:DVT, + author = {Hasibi, Faegheh and Nikolaev, Fedor and Xiong, Chenyan and Balog, Krisztian and Bratsberg, Svein Erik and Kotov, Alexander and Callan, Jamie}, + title = {DBpedia-Entity V2: A Test Collection for Entity Search}, + booktitle = {Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval}, + series = {SIGIR '17}, + year = {2017}, + pages = {1265--1268}, + doi = {10.1145/3077136.3080751}, + publisher = {ACM} +}""", + descriptive_stats={ + "n_samples": {"test": 400}, + "avg_character_length": { + "test": { + "average_document_length": 363.468546000768, + "average_query_length": 35.45, + "num_documents": 88542, + "num_queries": 400, + "average_relevant_docs_per_query": 38.215, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/pol/HotpotQAPLRetrieval.py b/mteb/tasks/Retrieval/pol/HotpotQAPLRetrieval.py index 8ce63add1e..c9bab26a2f 100644 --- a/mteb/tasks/Retrieval/pol/HotpotQAPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/HotpotQAPLRetrieval.py @@ -49,3 +49,49 @@ class HotpotQAPL(AbsTaskRetrieval): }, }, ) + + +class HotpotQAPLHardNegatives(AbsTaskRetrieval): + metadata = TaskMetadata( + name="HotpotQA-PLHardNegatives", + description="HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong supervision for supporting facts to enable more explainable question answering systems. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://hotpotqa.github.io/", + dataset={ + "path": "mteb/HotpotQA_PL_test_top_250_only_w_correct-v2", + "revision": "0642cadffa3205c6b21c9af901fdffcd60d6f31e", + "trust_remote_code": True, + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["pol-Latn"], + main_score="ndcg_at_10", + date=("2018-01-01", "2018-12-31"), # best guess: based on publication date + domains=["Web", "Written"], + task_subtypes=["Question answering"], + license="cc-by-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="machine-translated", + bibtex_citation="""@misc{wojtasik2024beirpl, + title={BEIR-PL: Zero Shot Information Retrieval Benchmark for the Polish Language}, + author={Konrad Wojtasik and Vadim Shishkin and Kacper Wołowiec and Arkadiusz Janz and Maciej Piasecki}, + year={2024}, + eprint={2305.19840}, + archivePrefix={arXiv}, + primaryClass={cs.IR} +}""", + descriptive_stats={ + "n_samples": {"test": 1000}, + "avg_character_length": { + "test": { + "average_document_length": 438.3888210025661, + "average_query_length": 95.161, + "num_documents": 212774, + "num_queries": 1000, + "average_relevant_docs_per_query": 2.0, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/pol/MSMARCOPLRetrieval.py b/mteb/tasks/Retrieval/pol/MSMARCOPLRetrieval.py index 002970541d..a3cd81f620 100644 --- a/mteb/tasks/Retrieval/pol/MSMARCOPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/MSMARCOPLRetrieval.py @@ -51,3 +51,51 @@ class MSMARCOPL(AbsTaskRetrieval): }, }, ) + + +class MSMARCOPLHardNegatives(AbsTaskRetrieval): + ignore_identical_ids = True + + metadata = TaskMetadata( + name="MSMARCO-PLHardNegatives", + description="MS MARCO is a collection of datasets focused on deep learning in search. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://microsoft.github.io/msmarco/", + dataset={ + "path": "mteb/MSMARCO_PL_test_top_250_only_w_correct-v2", + "revision": "b609cb1ec6772bf92b8e014343a7ecfb10eef2d9", + "trust_remote_code": True, + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["pol-Latn"], + main_score="ndcg_at_10", + date=("2016-01-01", "2016-12-30"), # best guess: based on publication date + domains=["Web", "Written"], + task_subtypes=["Question answering"], + license="https://microsoft.github.io/msmarco/", + annotations_creators="derived", + dialect=[], + sample_creation="machine-translated", + bibtex_citation=""""@misc{wojtasik2024beirpl, + title={BEIR-PL: Zero Shot Information Retrieval Benchmark for the Polish Language}, + author={Konrad Wojtasik and Vadim Shishkin and Kacper Wołowiec and Arkadiusz Janz and Maciej Piasecki}, + year={2024}, + eprint={2305.19840}, + archivePrefix={arXiv}, + primaryClass={cs.IR} +}""", + descriptive_stats={ + "n_samples": {"test": 43}, + "avg_character_length": { + "test": { + "average_document_length": 382.3476426537285, + "average_query_length": 33.02325581395349, + "num_documents": 9481, + "num_queries": 43, + "average_relevant_docs_per_query": 95.3953488372093, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/pol/NQPLRetrieval.py b/mteb/tasks/Retrieval/pol/NQPLRetrieval.py index eebd5fa10f..697778fef4 100644 --- a/mteb/tasks/Retrieval/pol/NQPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/NQPLRetrieval.py @@ -49,3 +49,49 @@ class NQPL(AbsTaskRetrieval): }, }, ) + + +class NQPLHardNegatives(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NQ-PLHardNegatives", + description="Natural Questions: A Benchmark for Question Answering Research. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://ai.google.com/research/NaturalQuestions/", + dataset={ + "path": "mteb/NQ_PL_test_top_250_only_w_correct-v2", + "revision": "9a2878a70ea545a8f4df0cdfa1adea27f4f64390", + "trust_remote_code": True, + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["pol-Latn"], + main_score="ndcg_at_10", + date=None, + domains=None, + task_subtypes=None, + license=None, + annotations_creators=None, + dialect=[], + sample_creation="machine-translated", + bibtex_citation="""@misc{wojtasik2024beirpl, + title={BEIR-PL: Zero Shot Information Retrieval Benchmark for the Polish Language}, + author={Konrad Wojtasik and Vadim Shishkin and Kacper Wołowiec and Arkadiusz Janz and Maciej Piasecki}, + year={2024}, + eprint={2305.19840}, + archivePrefix={arXiv}, + primaryClass={cs.IR} +}""", + descriptive_stats={ + "n_samples": {"test": 1000}, + "avg_character_length": { + "test": { + "average_document_length": 610.7449138094336, + "average_query_length": 48.381, + "num_documents": 184765, + "num_queries": 1000, + "average_relevant_docs_per_query": 1.213, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/pol/QuoraPLRetrieval.py b/mteb/tasks/Retrieval/pol/QuoraPLRetrieval.py index 86ea0e2806..17b32f5a0d 100644 --- a/mteb/tasks/Retrieval/pol/QuoraPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/QuoraPLRetrieval.py @@ -56,3 +56,49 @@ class QuoraPLRetrieval(AbsTaskRetrieval): }, }, ) + + +class QuoraPLRetrievalHardNegatives(AbsTaskRetrieval): + metadata = TaskMetadata( + name="Quora-PLHardNegatives", + description="QuoraRetrieval is based on questions that are marked as duplicates on the Quora platform. Given a question, find other (duplicate) questions. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs", + dataset={ + "path": "mteb/Quora_PL_test_top_250_only_w_correct-v2", + "revision": "523ff30f3346cd9c36081c19fc6eaef0a2f8d53d", + "trust_remote_code": True, + }, + type="Retrieval", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["pol-Latn"], + main_score="ndcg_at_10", + date=None, + domains=None, + task_subtypes=None, + license=None, + annotations_creators=None, + dialect=[], + sample_creation="machine-translated", + bibtex_citation=""""@misc{wojtasik2024beirpl, + title={BEIR-PL: Zero Shot Information Retrieval Benchmark for the Polish Language}, + author={Konrad Wojtasik and Vadim Shishkin and Kacper Wołowiec and Arkadiusz Janz and Maciej Piasecki}, + year={2024}, + eprint={2305.19840}, + archivePrefix={arXiv}, + primaryClass={cs.IR} +}""", + descriptive_stats={ + "n_samples": {"test": 1000}, + "avg_character_length": { + "test": { + "average_document_length": 67.77529631287385, + "average_query_length": 53.846, + "num_documents": 172031, + "num_queries": 1000, + "average_relevant_docs_per_query": 1.641, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/pol/TRECCOVIDPLRetrieval.py b/mteb/tasks/Retrieval/pol/TRECCOVIDPLRetrieval.py index 66669356b6..04778c0227 100644 --- a/mteb/tasks/Retrieval/pol/TRECCOVIDPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/TRECCOVIDPLRetrieval.py @@ -27,7 +27,7 @@ class TRECCOVIDPL(AbsTaskRetrieval): ), # approximate date of covid pandemic start and end (best guess) domains=["Academic", "Non-fiction", "Written"], task_subtypes=["Article retrieval"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="machine-translated", diff --git a/mteb/tasks/Retrieval/rus/RiaNewsRetrieval.py b/mteb/tasks/Retrieval/rus/RiaNewsRetrieval.py index 4aa8381217..ffd0c919b2 100644 --- a/mteb/tasks/Retrieval/rus/RiaNewsRetrieval.py +++ b/mteb/tasks/Retrieval/rus/RiaNewsRetrieval.py @@ -48,3 +48,48 @@ class RiaNewsRetrieval(AbsTaskRetrieval): }, }, ) + + +class RiaNewsRetrievalHardNegatives(AbsTaskRetrieval): + ignore_identical_ids = True + + metadata = TaskMetadata( + name="RiaNewsRetrievalHardNegatives", + dataset={ + "path": "mteb/RiaNewsRetrieval_test_top_250_only_w_correct-v2", + "revision": "d42860a6c15f0a2c4485bda10c6e5b641fdfe479", + }, + description="News article retrieval by headline. Based on Rossiya Segodnya dataset. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", + reference="https://arxiv.org/abs/1901.07786", + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["rus-Cyrl"], + main_score="ndcg_at_10", + date=("2010-01-01", "2014-12-31"), + domains=["News", "Written"], + task_subtypes=["Article retrieval"], + license="cc-by-nc-nd-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{gavrilov2018self, + title={Self-Attentive Model for Headline Generation}, + author={Gavrilov, Daniil and Kalaidin, Pavel and Malykh, Valentin}, + booktitle={Proceedings of the 41st European Conference on Information Retrieval}, + year={2019} + }""", + descriptive_stats={ + "n_samples": {"test": 1000}, + "avg_character_length": { + "test": { + "average_document_length": 1225.7253146619116, + "average_query_length": 62.338, + "num_documents": 191237, + "num_queries": 1000, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/mteb/tasks/Retrieval/slk/SKQuadRetrieval.py b/mteb/tasks/Retrieval/slk/SKQuadRetrieval.py new file mode 100644 index 0000000000..2795ef82d3 --- /dev/null +++ b/mteb/tasks/Retrieval/slk/SKQuadRetrieval.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class SKQuadRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="SKQuadRetrieval", + description=( + "Retrieval SK Quad evaluates Slovak search performance using questions and answers " + "derived from the SK-QuAD dataset. It measures relevance with scores assigned to answers " + "based on their relevancy to corresponding questions, which is vital for improving " + "Slovak language search systems." + ), + reference="https://huggingface.co/datasets/TUKE-KEMT/retrieval-skquad", + dataset={ + "path": "TUKE-KEMT/retrieval-skquad", + "revision": "09f81f51dd5b8497da16d02c69c98d5cb5993ef2", + }, + type="Retrieval", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["slk-Latn"], + main_score="ndcg_at_10", + date=("2024-05-30", "2024-06-13"), + domains=["Encyclopaedic"], + task_subtypes=["Question answering"], + license="cc-by-nc-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="", + descriptive_stats={ + "n_samples": {"test": 1134}, + "avg_character_length": { + "test": { + "average_document_length": 1180.5071792496526, + "average_query_length": 53.63403880070547, + "num_documents": 6477, + "num_queries": 1134, + "average_relevant_docs_per_query": 11, + } + }, + }, + ) + + def load_data(self, eval_splits=None, **kwargs): + """Load and preprocess datasets for retrieval task.""" + eval_splits = eval_splits or ["test"] + + # Load datasets + ds_default = load_dataset("TUKE-KEMT/retrieval-skquad", "default") + ds_corpus = load_dataset("TUKE-KEMT/retrieval-skquad", "corpus") + ds_query = load_dataset("TUKE-KEMT/retrieval-skquad", "queries") + + if "test" in eval_splits: + # Corpus, Queries, and Relevance dictionary for 'test' split + self.corpus = { + "test": { + row["_id"]: {"text": row["text"], "title": row["title"]} + for row in ds_corpus["corpus"] + } + } + self.queries = { + "test": {row["_id"]: row["text"] for row in ds_query["queries"]} + } + self.relevant_docs = {"test": {}} + + for row in ds_default["test"]: + self.relevant_docs["test"].setdefault(row["query-id"], {})[ + row["corpus-id"] + ] = int(row["score"]) + + print( + f"Data Loaded:\n- Corpus size: {len(self.corpus['test'])}\n- Query size: {len(self.queries['test'])}\n- Relevance entries: {len(self.relevant_docs['test'])}" + ) diff --git a/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py b/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py index 7fddbfd287..f01cb25db8 100644 --- a/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py +++ b/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py @@ -26,7 +26,7 @@ class SweFaqRetrieval(AbsTaskRetrieval): date=("2000-01-01", "2024-12-31"), # best guess task_subtypes=["Question answering"], domains=["Government", "Non-fiction", "Written"], - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], sample_creation="found", @@ -62,9 +62,9 @@ def load_data(self, **kwargs): def dataset_transform(self) -> None: """And transform to a retrieval datset, which have the following attributes - self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text - self.queries = Dict[query_id, str] #id => query - self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + self.corpus = dict[doc_id, dict[str, str]] #id => dict with document datas like title and text + self.queries = dict[query_id, str] #id => query + self.relevant_docs = dict[query_id, dict[[doc_id, score]] """ self.corpus = {} self.relevant_docs = {} diff --git a/mteb/tasks/Retrieval/swe/SwednRetrieval.py b/mteb/tasks/Retrieval/swe/SwednRetrieval.py index 185d46ec36..381961542c 100644 --- a/mteb/tasks/Retrieval/swe/SwednRetrieval.py +++ b/mteb/tasks/Retrieval/swe/SwednRetrieval.py @@ -25,7 +25,7 @@ class SwednRetrieval(AbsTaskRetrieval): main_score="ndcg_at_10", date=("2000-01-01", "2020-12-31"), domains=["News", "Non-fiction", "Written"], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="derived", dialect=[], task_subtypes=["Article retrieval"], @@ -61,9 +61,9 @@ def load_data(self, **kwargs): def dataset_transform(self) -> None: """And transform to a retrieval datset, which have the following attributes - self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text - self.queries = Dict[query_id, str] #id => query - self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + self.corpus = dict[doc_id, dict[str, str]] #id => dict with document datas like title and text + self.queries = dict[query_id, str] #id => query + self.relevant_docs = dict[query_id, dict[[doc_id, score]] """ self.corpus = {} self.relevant_docs = {} diff --git a/mteb/tasks/Retrieval/tur/TurHistQuad.py b/mteb/tasks/Retrieval/tur/TurHistQuad.py index 6a64391f69..d896e36fa0 100644 --- a/mteb/tasks/Retrieval/tur/TurHistQuad.py +++ b/mteb/tasks/Retrieval/tur/TurHistQuad.py @@ -24,7 +24,7 @@ class TurHistQuadRetrieval(AbsTaskRetrieval): date=("2021-01-01", "2021-10-13"), task_subtypes=["Question answering"], domains=["Encyclopaedic", "Non-fiction", "Academic", "Written"], - license="MIT", + license="mit", annotations_creators="derived", dialect=[], sample_creation="found", @@ -58,9 +58,9 @@ class TurHistQuadRetrieval(AbsTaskRetrieval): def load_data(self, **kwargs) -> None: """And transform to a retrieval datset, which have the following attributes - self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text - self.queries = Dict[query_id, str] #id => query - self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + self.corpus = dict[doc_id, dict[str, str]] #id => dict with document datas like title and text + self.queries = dict[query_id, str] #id => query + self.relevant_docs = dict[query_id, dict[[doc_id, score]] """ if self.data_loaded: return diff --git a/mteb/tasks/Retrieval/zho/LeCaRDv2Retrieval.py b/mteb/tasks/Retrieval/zho/LeCaRDv2Retrieval.py index 82d72de9d4..587f435389 100644 --- a/mteb/tasks/Retrieval/zho/LeCaRDv2Retrieval.py +++ b/mteb/tasks/Retrieval/zho/LeCaRDv2Retrieval.py @@ -23,7 +23,7 @@ class LeCaRDv2(AbsTaskRetrieval): date=None, domains=["Legal", "Written"], task_subtypes=["Article retrieval"], - license="MIT", + license="mit", annotations_creators="derived", dialect=None, sample_creation="found", diff --git a/mteb/tasks/STS/eng/STS12STS.py b/mteb/tasks/STS/eng/STS12STS.py index 875766280b..5cf0b1ccfc 100644 --- a/mteb/tasks/STS/eng/STS12STS.py +++ b/mteb/tasks/STS/eng/STS12STS.py @@ -23,7 +23,7 @@ class STS12STS(AbsTaskSTS): date=("2005-01-01", "2012-12-31"), domains=["Encyclopaedic", "News", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/STS/eng/STS13STS.py b/mteb/tasks/STS/eng/STS13STS.py index d2fbdf03c3..716f42fdef 100644 --- a/mteb/tasks/STS/eng/STS13STS.py +++ b/mteb/tasks/STS/eng/STS13STS.py @@ -23,7 +23,7 @@ class STS13STS(AbsTaskSTS): date=("2012-01-01", "2012-12-31"), domains=["Web", "News", "Non-fiction", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/STS/eng/STS14STS.py b/mteb/tasks/STS/eng/STS14STS.py index b01f9f9470..12bc9a4d18 100644 --- a/mteb/tasks/STS/eng/STS14STS.py +++ b/mteb/tasks/STS/eng/STS14STS.py @@ -23,7 +23,7 @@ class STS14STS(AbsTaskSTS): date=("2012-01-01", "2012-08-31"), domains=["Blog", "Web", "Spoken"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="created", diff --git a/mteb/tasks/STS/eng/STS15STS.py b/mteb/tasks/STS/eng/STS15STS.py index 4e9815e576..a2c84dd3de 100644 --- a/mteb/tasks/STS/eng/STS15STS.py +++ b/mteb/tasks/STS/eng/STS15STS.py @@ -23,7 +23,7 @@ class STS15STS(AbsTaskSTS): date=("2008-01-01", "2014-07-28"), domains=["Blog", "News", "Web", "Written", "Spoken"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/STS/eng/STS16STS.py b/mteb/tasks/STS/eng/STS16STS.py index 4954ace706..ca93d0867d 100644 --- a/mteb/tasks/STS/eng/STS16STS.py +++ b/mteb/tasks/STS/eng/STS16STS.py @@ -23,7 +23,7 @@ class STS16STS(AbsTaskSTS): date=("2015-10-01", "2015-12-31"), domains=["Blog", "Web", "Spoken"], task_subtypes=["Sentiment/Hate speech"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/STS/jpn/JSTS.py b/mteb/tasks/STS/jpn/JSTS.py index 6659becbad..7838b50132 100644 --- a/mteb/tasks/STS/jpn/JSTS.py +++ b/mteb/tasks/STS/jpn/JSTS.py @@ -26,7 +26,7 @@ class JSTS(AbsTaskSTS): date=("2016-01-01", "2022-12-31"), domains=["Web", "Written"], task_subtypes=[], - license="CC BY-SA 4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/STS/kor/KlueSTS.py b/mteb/tasks/STS/kor/KlueSTS.py index c3905f3341..1046e9b87c 100644 --- a/mteb/tasks/STS/kor/KlueSTS.py +++ b/mteb/tasks/STS/kor/KlueSTS.py @@ -24,7 +24,7 @@ class KlueSTS(AbsTaskSTS): date=("2011-01-01", "2021-11-02"), # rough estimate, domains=["Reviews", "News", "Spoken", "Written", "Spoken"], task_subtypes=None, - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/STS/kor/KorSTS.py b/mteb/tasks/STS/kor/KorSTS.py index 002fba3d70..39e3f17264 100644 --- a/mteb/tasks/STS/kor/KorSTS.py +++ b/mteb/tasks/STS/kor/KorSTS.py @@ -23,7 +23,7 @@ class KorSTS(AbsTaskSTS): date=("2012-01-01", "2017-01-01"), # rough approximates domains=["News", "Web"], task_subtypes=None, - license="CC-BY-SA-4.0", + license="cc-by-sa-4.0", annotations_creators=None, dialect=[], sample_creation="machine-translated and localized", diff --git a/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py b/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py index 44a0096e33..463db58232 100644 --- a/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py +++ b/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py @@ -56,7 +56,7 @@ class IndicCrosslingualSTS(AbsTaskSTS, MultilingualTask): "Spoken", ], task_subtypes=[], - license="CC0", + license="cc0-1.0", annotations_creators="expert-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py b/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py index a45e92037a..b3b4cbdb7e 100644 --- a/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py +++ b/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py @@ -38,7 +38,7 @@ class STS17Crosslingual(AbsTaskSTS, MultilingualTask): date=("2014-01-01", "2015-12-31"), domains=["News", "Web", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py b/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py index 3f7f70a1dc..ae394b5512 100644 --- a/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py +++ b/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py @@ -45,7 +45,7 @@ class STS22CrosslingualSTSv2(AbsTaskSTS, MultilingualTask): date=("2020-01-01", "2020-06-11"), domains=["News", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", @@ -111,7 +111,7 @@ class STS22CrosslingualSTS(AbsTaskSTS, MultilingualTask): date=("2020-01-01", "2020-06-11"), domains=["News", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py b/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py index 2d77424f9e..7aacebe342 100644 --- a/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py +++ b/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py @@ -42,7 +42,7 @@ class STSBenchmarkMultilingualSTS(AbsTaskSTS, MultilingualTask): date=("2012-01-01", "2017-12-31"), domains=["News", "Social", "Web", "Spoken", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="machine-translated", diff --git a/mteb/tasks/STS/multilingual/SemRel24STS.py b/mteb/tasks/STS/multilingual/SemRel24STS.py index fa3b3613d3..b990170215 100644 --- a/mteb/tasks/STS/multilingual/SemRel24STS.py +++ b/mteb/tasks/STS/multilingual/SemRel24STS.py @@ -45,7 +45,7 @@ class SemRel24STS(AbsTaskSTS, MultilingualTask): date=("2023-01-01", "2023-12-31"), domains=["Spoken", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="created", diff --git a/mteb/tasks/STS/pol/PolishSTS.py b/mteb/tasks/STS/pol/PolishSTS.py index 321327dcc3..38bc37d50e 100644 --- a/mteb/tasks/STS/pol/PolishSTS.py +++ b/mteb/tasks/STS/pol/PolishSTS.py @@ -22,7 +22,7 @@ class SickrPLSTS(AbsTaskSTS): date=("2018-01-01", "2019-09-01"), # rough estimate domains=["Web", "Written"], task_subtypes=["Textual Entailment"], - license="CC-BY-NC-SA-3.0", + license="cc-by-nc-sa-3.0", annotations_creators="human-annotated", dialect=[], sample_creation="human-translated and localized", @@ -89,7 +89,7 @@ class CdscrSTS(AbsTaskSTS): date=("2016-01-01", "2017-04-01"), # rough estimate domains=["Web", "Written"], task_subtypes=["Textual Entailment"], - license="CC-BY-NC-SA-4.0", + license="cc-by-nc-sa-4.0", annotations_creators="human-annotated", dialect=[], sample_creation="human-translated and localized", diff --git a/mteb/tasks/STS/por/Assin2STS.py b/mteb/tasks/STS/por/Assin2STS.py index 51861a5c2c..9140849547 100644 --- a/mteb/tasks/STS/por/Assin2STS.py +++ b/mteb/tasks/STS/por/Assin2STS.py @@ -22,7 +22,7 @@ class Assin2STS(AbsTaskSTS): date=("2019-01-01", "2019-09-16"), # best guess domains=["Written"], task_subtypes=["Claim verification"], - license="Not specified", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/STS/por/SickBrSTS.py b/mteb/tasks/STS/por/SickBrSTS.py index 7492e7a603..4db82be50b 100644 --- a/mteb/tasks/STS/por/SickBrSTS.py +++ b/mteb/tasks/STS/por/SickBrSTS.py @@ -25,7 +25,7 @@ class SickBrSTS(AbsTaskSTS): date=("2018-01-01", "2018-09-01"), # rough estimate domains=["Web", "Written"], task_subtypes=["Textual Entailment"], - license="Unknown", + license="not specified", annotations_creators="human-annotated", dialect=[], sample_creation="human-translated and localized", diff --git a/mteb/tasks/STS/rus/RUParaPhraserSTS.py b/mteb/tasks/STS/rus/RUParaPhraserSTS.py index 2eb02622c5..b577cbf6d8 100644 --- a/mteb/tasks/STS/rus/RUParaPhraserSTS.py +++ b/mteb/tasks/STS/rus/RUParaPhraserSTS.py @@ -23,7 +23,7 @@ class RUParaPhraserSTS(AbsTaskSTS): date=("2009-01-01", "2019-01-01"), # rough estimate, domains=["News", "Written"], task_subtypes=[], - license="MIT", + license="mit", annotations_creators="human-annotated", dialect=[], sample_creation="found", diff --git a/mteb/tasks/STS/zho/CMTEBSTS.py b/mteb/tasks/STS/zho/CMTEBSTS.py index 3e86ec554b..94bf2d126a 100644 --- a/mteb/tasks/STS/zho/CMTEBSTS.py +++ b/mteb/tasks/STS/zho/CMTEBSTS.py @@ -304,3 +304,10 @@ class QBQTC(AbsTaskSTS): bibtex_citation=None, descriptive_stats={"n_samples": None, "avg_character_length": None}, ) + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 0 + metadata_dict["max_score"] = 2 + return metadata_dict diff --git a/mteb/tasks/SpeedTask/CPUSpeedTask.py b/mteb/tasks/SpeedTask/CPUSpeedTask.py index 028cf444e7..75f223378f 100644 --- a/mteb/tasks/SpeedTask/CPUSpeedTask.py +++ b/mteb/tasks/SpeedTask/CPUSpeedTask.py @@ -19,7 +19,7 @@ class CPUSpeedTask(AbsTaskSpeedTask): date=("2024-06-20", "2024-06-20"), domains=["Fiction", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/SpeedTask/GPUSpeedTask.py b/mteb/tasks/SpeedTask/GPUSpeedTask.py index 6f0b16c309..4c29368e4f 100644 --- a/mteb/tasks/SpeedTask/GPUSpeedTask.py +++ b/mteb/tasks/SpeedTask/GPUSpeedTask.py @@ -20,7 +20,7 @@ class GPUSpeedTask(AbsTaskSpeedTask): date=("2024-06-20", "2024-06-20"), domains=["Fiction", "Written"], task_subtypes=[], - license="Not specified", + license="not specified", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/pyproject.toml b/pyproject.toml index 66c1ce2445..72f47523d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mteb" -version = "1.14.15" +version = "1.16.5" description = "Massive Text Embedding Benchmark" readme = "README.md" authors = [ @@ -23,10 +23,10 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", ] -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "datasets>=2.19.0", - "numpy>=1.0.0,<2.0.0", # note: https://github.com/huggingface/datasets/issues/6980 + "numpy>=1.0.0,<3.0.0", "requests>=2.26.0", "scikit_learn>=1.0.2", "scipy>=0.0.0", @@ -53,9 +53,14 @@ homepage = "https://github.com/embeddings-benchmark/mteb" mteb = "mteb.cli:main" [project.optional-dependencies] -dev = ["ruff>=0.6.0", "pytest", "pytest-xdist", "pytest-coverage"] +dev = ["ruff==0.6.4", # locked so we don't get PRs which fail only due to a lint update +"pytest", "pytest-xdist", "pytest-coverage"] codecarbon = ["codecarbon"] speedtask = ["GPUtil>=1.4.0", "psutil>=5.9.8"] +peft = ["peft>=0.11.0"] +leaderboard = ["gradio>=4.44.0", "gradio_rangeslider>=0.0.6"] +flagembedding = ["FlagEmbedding"] + [tool.coverage.report] @@ -87,7 +92,7 @@ exclude = ["tests", "results"] [tool.ruff] -target-version = "py38" +target-version = "py39" [tool.ruff.lint] @@ -98,10 +103,8 @@ select = [ "D", # formatting for docs "UP", # upgrade to latest syntax if possible "FA", # Future annotations - "C4", # cleaner comprehensions - "ISC", + "C4", # cleaner comprehensions ] -unfixable = ["ISC001"] ignore = ["E501", # line too long diff --git a/scripts/data/run_mteb_english.py b/scripts/data/run_mteb_english.py deleted file mode 100644 index 8b2d50950d..0000000000 --- a/scripts/data/run_mteb_english.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Example script for benchmarking all datasets constituting the MTEB English leaderboard & average scores""" - -from __future__ import annotations - -import logging - -from sentence_transformers import SentenceTransformer - -from mteb import MTEB - -logging.basicConfig(level=logging.INFO) - -logger = logging.getLogger("main") - -TASK_LIST_CLASSIFICATION = [ - "AmazonCounterfactualClassification", - "AmazonPolarityClassification", - "AmazonReviewsClassification", - "Banking77Classification", - "EmotionClassification", - "ImdbClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "MTOPDomainClassification", - "MTOPIntentClassification", - "ToxicConversationsClassification", - "TweetSentimentExtractionClassification", -] - -TASK_LIST_CLUSTERING = [ - "ArxivClusteringP2P", - "ArxivClusteringS2S", - "BiorxivClusteringP2P", - "BiorxivClusteringS2S", - "MedrxivClusteringP2P", - "MedrxivClusteringS2S", - "RedditClustering", - "RedditClusteringP2P", - "StackExchangeClustering", - "StackExchangeClusteringP2P", - "TwentyNewsgroupsClustering", -] - -TASK_LIST_PAIR_CLASSIFICATION = [ - "SprintDuplicateQuestions", - "TwitterSemEval2015", - "TwitterURLCorpus", -] - -TASK_LIST_RERANKING = [ - "AskUbuntuDupQuestions", - "MindSmallReranking", - "SciDocsRR", - "StackOverflowDupQuestions", -] - -TASK_LIST_RETRIEVAL = [ - "ArguAna", - "ClimateFEVER", - "CQADupstackAndroidRetrieval", - "CQADupstackEnglishRetrieval", - "CQADupstackGamingRetrieval", - "CQADupstackGisRetrieval", - "CQADupstackMathematicaRetrieval", - "CQADupstackPhysicsRetrieval", - "CQADupstackProgrammersRetrieval", - "CQADupstackStatsRetrieval", - "CQADupstackTexRetrieval", - "CQADupstackUnixRetrieval", - "CQADupstackWebmastersRetrieval", - "CQADupstackWordpressRetrieval", - "DBPedia", - "FEVER", - "FiQA2018", - "HotpotQA", - "MSMARCO", - "NFCorpus", - "NQ", - "QuoraRetrieval", - "SCIDOCS", - "SciFact", - "Touche2020", - "TRECCOVID", -] - -TASK_LIST_STS = [ - "BIOSSES", - "SICK-R", - "STS12", - "STS13", - "STS14", - "STS15", - "STS16", - "STS17", - "STS22", - "STSBenchmark", - "SummEval", -] - -TASK_LIST = ( - TASK_LIST_CLASSIFICATION - + TASK_LIST_CLUSTERING - + TASK_LIST_PAIR_CLASSIFICATION - + TASK_LIST_RERANKING - + TASK_LIST_RETRIEVAL - + TASK_LIST_STS -) - -model_name = "average_word_embeddings_komninos" -model = SentenceTransformer(model_name) - -for task in TASK_LIST: - logger.info(f"Running task: {task}") - eval_splits = ["dev"] if task == "MSMARCO" else ["test"] - evaluation = MTEB( - tasks=[task], task_langs=["en"] - ) # Remove "en" for running all languages - evaluation.run( - model, output_folder=f"results/{model_name}", eval_splits=eval_splits - ) diff --git a/scripts/mmteb_create_author_list.ipynb b/scripts/mmteb_create_author_list.ipynb index e29a4f0491..3c9d99c2ed 100644 --- a/scripts/mmteb_create_author_list.ipynb +++ b/scripts/mmteb_create_author_list.ipynb @@ -107,9 +107,9 @@ " \n", " \n", " KennethEnevoldsen\n", - " 591\n", + " 593\n", " 85\n", - " 322\n", + " 324\n", " 68\n", " 35\n", " 0\n", @@ -190,7 +190,7 @@ " 0\n", " \n", " \n", - " bakrianoo\n", + " cslizc\n", " 2\n", " 0\n", " 0\n", @@ -202,7 +202,7 @@ " 0\n", " \n", " \n", - " cslizc\n", + " hanhainebula\n", " 2\n", " 0\n", " 0\n", @@ -214,19 +214,19 @@ " 0\n", " \n", " \n", - " hanhainebula\n", + " hongjin-su\n", " 2\n", " 0\n", - " 0\n", " 2\n", " 0\n", " 0\n", " 0\n", " 0\n", " 0\n", + " 0\n", " \n", " \n", - " achibb\n", + " bakrianoo\n", " 2\n", " 0\n", " 0\n", @@ -239,23 +239,23 @@ " \n", " \n", "\n", - "

92 rows × 9 columns

\n", + "

96 rows × 9 columns

\n", "" ], "text/plain": [ " Total Bug fixes Review PR New dataset \\\n", "GitHub \n", - "KennethEnevoldsen 591 85 322 68 \n", + "KennethEnevoldsen 593 85 324 68 \n", "isaac-chung 433 50 194 120 \n", "imenelydiaker 358 24 144 120 \n", "awinml 302 0 2 300 \n", "x-tabdeveloping 239 10 32 144 \n", "... ... ... ... ... \n", "antoniolanza1996 2 2 0 0 \n", - "bakrianoo 2 0 0 2 \n", "cslizc 2 0 0 2 \n", "hanhainebula 2 0 0 2 \n", - "achibb 2 0 0 2 \n", + "hongjin-su 2 0 2 0 \n", + "bakrianoo 2 0 0 2 \n", "\n", " Dataset annotations Paper writing New task Coordination \\\n", "GitHub \n", @@ -266,10 +266,10 @@ "x-tabdeveloping 0 0 12 41 \n", "... ... ... ... ... \n", "antoniolanza1996 0 0 0 0 \n", - "bakrianoo 0 0 0 0 \n", "cslizc 0 0 0 0 \n", "hanhainebula 0 0 0 0 \n", - "achibb 0 0 0 0 \n", + "hongjin-su 0 0 0 0 \n", + "bakrianoo 0 0 0 0 \n", "\n", " Running Models \n", "GitHub \n", @@ -280,12 +280,12 @@ "x-tabdeveloping 0 \n", "... ... \n", "antoniolanza1996 0 \n", - "bakrianoo 0 \n", "cslizc 0 \n", "hanhainebula 0 \n", - "achibb 0 \n", + "hongjin-su 0 \n", + "bakrianoo 0 \n", "\n", - "[92 rows x 9 columns]" + "[96 rows x 9 columns]" ] }, "execution_count": 4, @@ -325,7 +325,7 @@ "\\endfoot\n", "\\bottomrule\n", "\\endlastfoot\n", - "KennethEnevoldsen & 591 & 85 & 322 & 68 & 35 & 0 & 0 & 81 & 0 \\\\\n", + "KennethEnevoldsen & 593 & 85 & 324 & 68 & 35 & 0 & 0 & 81 & 0 \\\\\n", "isaac-chung & 433 & 50 & 194 & 120 & 1 & 12 & 2 & 54 & 0 \\\\\n", "imenelydiaker & 358 & 24 & 144 & 120 & 0 & 0 & 0 & 70 & 0 \\\\\n", "awinml & 302 & 0 & 2 & 300 & 0 & 0 & 0 & 0 & 0 \\\\\n", @@ -333,14 +333,14 @@ "davidstap & 176 & 0 & 0 & 176 & 0 & 0 & 0 & 0 & 0 \\\\\n", "jaygala24 & 149 & 0 & 0 & 149 & 0 & 0 & 0 & 0 & 0 \\\\\n", "wissam-sib & 144 & 4 & 6 & 134 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "Muennighoff & 140 & 0 & 46 & 0 & 0 & 0 & 0 & 70 & 24 \\\\\n", + "Muennighoff & 142 & 0 & 48 & 0 & 0 & 0 & 0 & 70 & 24 \\\\\n", "dokato & 112 & 12 & 6 & 94 & 0 & 0 & 0 & 0 & 0 \\\\\n", "gentaiscool & 110 & 0 & 0 & 110 & 0 & 0 & 0 & 0 & 0 \\\\\n", "jupyterjazz & 108 & 0 & 0 & 108 & 0 & 0 & 0 & 0 & 0 \\\\\n", "SaitejaUtpala & 102 & 0 & 0 & 102 & 0 & 0 & 0 & 0 & 0 \\\\\n", "orionw & 100 & 20 & 20 & 0 & 0 & 0 & 10 & 50 & 0 \\\\\n", - "MathieuCiancone & 88 & 0 & 0 & 88 & 0 & 0 & 0 & 0 & 0 \\\\\n", "schmarion & 88 & 0 & 0 & 88 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "MathieuCiancone & 88 & 0 & 0 & 88 & 0 & 0 & 0 & 0 & 0 \\\\\n", "GabrielSequeira & 88 & 0 & 0 & 88 & 0 & 0 & 0 & 0 & 0 \\\\\n", "digantamisra98 & 71 & 0 & 0 & 71 & 0 & 0 & 0 & 0 & 0 \\\\\n", "vaibhavad & 68 & 8 & 4 & 6 & 0 & 0 & 0 & 50 & 0 \\\\\n", @@ -352,71 +352,75 @@ "asparius & 48 & 0 & 14 & 34 & 0 & 0 & 0 & 0 & 0 \\\\\n", "Akash190104 & 46 & 0 & 0 & 46 & 0 & 0 & 0 & 0 & 0 \\\\\n", "MartinBernstorff & 43 & 13 & 8 & 2 & 0 & 0 & 0 & 20 & 0 \\\\\n", - "akshita-sukhlecha & 40 & 4 & 0 & 36 & 0 & 0 & 0 & 0 & 0 \\\\\n", "staoxiao & 40 & 0 & 0 & 40 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "akshita-sukhlecha & 40 & 4 & 0 & 36 & 0 & 0 & 0 & 0 & 0 \\\\\n", "rafalposwiata & 36 & 0 & 0 & 36 & 0 & 0 & 0 & 0 & 0 \\\\\n", "bp-high & 36 & 0 & 0 & 36 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "KranthiGV & 34 & 0 & 14 & 20 & 0 & 0 & 0 & 0 & 0 \\\\\n", "bjoernpl & 28 & 0 & 0 & 28 & 0 & 0 & 0 & 0 & 0 \\\\\n", "rasdani & 28 & 0 & 0 & 28 & 0 & 0 & 0 & 0 & 0 \\\\\n", "loicmagne & 28 & 28 & 0 & 0 & 0 & 0 & 0 & 0 & 0 \\\\\n", "jphme & 28 & 0 & 0 & 28 & 0 & 0 & 0 & 0 & 0 \\\\\n", "ShawonAshraf & 28 & 0 & 0 & 28 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "kranthigv & 26 & 0 & 6 & 20 & 0 & 0 & 0 & 0 & 0 \\\\\n", "violenil & 26 & 0 & 0 & 26 & 0 & 0 & 0 & 0 & 0 \\\\\n", "dwzhu-pku & 24 & 0 & 0 & 24 & 0 & 0 & 0 & 0 & 0 \\\\\n", "hgissbkh & 23 & 13 & 2 & 0 & 0 & 3 & 5 & 0 & 0 \\\\\n", - "jankounchained & 22 & 8 & 0 & 14 & 0 & 0 & 0 & 0 & 0 \\\\\n", "taeminlee & 22 & 0 & 0 & 22 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "jankounchained & 22 & 8 & 0 & 14 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "tomaarsen & 22 & 0 & 2 & 0 & 0 & 0 & 0 & 20 & 0 \\\\\n", + "kwojtasi & 22 & 0 & 0 & 22 & 0 & 0 & 0 & 0 & 0 \\\\\n", "mrshu & 21 & 0 & 4 & 16 & 1 & 0 & 0 & 0 & 0 \\\\\n", "crystina-z & 21 & 0 & 0 & 21 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "Andrian0s & 20 & 2 & 4 & 14 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "rbroc & 20 & 0 & 0 & 20 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "ManuelFay & 20 & 13 & 0 & 2 & 0 & 0 & 5 & 0 & 0 \\\\\n", "AlexeyVatolin & 20 & 20 & 0 & 0 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "john-b-yang & 20 & 0 & 0 & 0 & 0 & 20 & 0 & 0 & 0 \\\\\n", + "Andrian0s & 20 & 2 & 4 & 14 & 0 & 0 & 0 & 0 & 0 \\\\\n", "mmhamdy & 20 & 0 & 0 & 20 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "manandey & 18 & 0 & 0 & 18 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "ManuelFay & 20 & 13 & 0 & 2 & 0 & 0 & 5 & 0 & 0 \\\\\n", + "rbroc & 20 & 0 & 0 & 20 & 0 & 0 & 0 & 0 & 0 \\\\\n", "thakur-nandan & 18 & 0 & 0 & 18 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "dipam7 & 16 & 0 & 2 & 14 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "sted97 & 16 & 0 & 0 & 16 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "manandey & 18 & 0 & 0 & 18 & 0 & 0 & 0 & 0 & 0 \\\\\n", "PranjalChitale & 16 & 0 & 0 & 16 & 0 & 0 & 0 & 0 & 0 \\\\\n", "Sakshamrzt & 16 & 0 & 4 & 12 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "dipam7 & 16 & 0 & 2 & 14 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "sted97 & 16 & 0 & 0 & 16 & 0 & 0 & 0 & 0 & 0 \\\\\n", "artemsnegirev & 14 & 0 & 0 & 12 & 2 & 0 & 0 & 0 & 0 \\\\\n", "taidnguyen & 14 & 0 & 0 & 14 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "anpalmak2003 & 12 & 0 & 0 & 9 & 3 & 0 & 0 & 0 & 0 \\\\\n", "mariyahendriksen & 12 & 0 & 0 & 0 & 0 & 12 & 0 & 0 & 0 \\\\\n", - "Art3mis07 & 12 & 0 & 0 & 12 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "xhluca & 12 & 4 & 2 & 6 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "jordiclive & 12 & 10 & 0 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", "guenthermi & 12 & 0 & 0 & 12 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "jordiclive & 12 & 10 & 0 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", "slvnwhrl & 12 & 0 & 0 & 12 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "anpalmak2003 & 12 & 0 & 0 & 9 & 3 & 0 & 0 & 0 & 0 \\\\\n", + "xhluca & 12 & 4 & 2 & 6 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "Art3mis07 & 12 & 0 & 0 & 12 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "xiamengzhou & 12 & 0 & 0 & 12 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "swj0419 & 12 & 0 & 0 & 12 & 0 & 0 & 0 & 0 & 0 \\\\\n", "henilp105 & 11 & 2 & 0 & 0 & 9 & 0 & 0 & 0 & 0 \\\\\n", "ab1992ao & 11 & 0 & 0 & 8 & 3 & 0 & 0 & 0 & 0 \\\\\n", "MariyaTikhonova & 11 & 0 & 0 & 7 & 4 & 0 & 0 & 0 & 0 \\\\\n", - "sarahooker & 10 & 0 & 0 & 0 & 0 & 10 & 0 & 0 & 0 \\\\\n", "simon-clematide & 10 & 0 & 0 & 10 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "sarahooker & 10 & 0 & 0 & 0 & 0 & 10 & 0 & 0 & 0 \\\\\n", "ABorghini & 10 & 0 & 0 & 10 & 0 & 0 & 0 & 0 & 0 \\\\\n", "xu3kev & 10 & 0 & 0 & 10 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "Ruqyai & 10 & 0 & 8 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", "malteos & 10 & 0 & 0 & 10 & 0 & 0 & 0 & 0 & 0 \\\\\n", "ljvmiranda921 & 10 & 0 & 0 & 10 & 0 & 0 & 0 & 0 & 0 \\\\\n", "Alenush & 10 & 0 & 0 & 6 & 4 & 0 & 0 & 0 & 0 \\\\\n", - "guangyusong & 10 & 0 & 0 & 10 & 0 & 0 & 0 & 0 & 0 \\\\\n", "HLasse & 10 & 5 & 0 & 0 & 5 & 0 & 0 & 0 & 0 \\\\\n", + "guangyusong & 10 & 0 & 0 & 10 & 0 & 0 & 0 & 0 & 0 \\\\\n", "cassanof & 10 & 1 & 0 & 8 & 0 & 0 & 0 & 0 & 1 \\\\\n", "ZhengLiu101 & 10 & 0 & 0 & 10 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "Ruqyai & 10 & 0 & 8 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "KranthiGV & 8 & 0 & 8 & 0 & 0 & 0 & 0 & 0 & 0 \\\\\n", "izhx & 6 & 0 & 0 & 6 & 0 & 0 & 0 & 0 & 0 \\\\\n", "marcobellagente93 & 6 & 0 & 0 & 6 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "tomaarsen & 2 & 0 & 2 & 0 & 0 & 0 & 0 & 0 & 0 \\\\\n", "MexicanLemonade & 2 & 0 & 0 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", "NouamaneTazi & 2 & 0 & 2 & 0 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "monikernemo & 2 & 0 & 0 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", "PhilipMay & 2 & 0 & 2 & 0 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "monikernemo & 2 & 0 & 0 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "achibb & 2 & 0 & 0 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", "antoniolanza1996 & 2 & 2 & 0 & 0 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "bakrianoo & 2 & 0 & 0 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", "cslizc & 2 & 0 & 0 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", "hanhainebula & 2 & 0 & 0 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", - "achibb & 2 & 0 & 0 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "hongjin-su & 2 & 0 & 2 & 0 & 0 & 0 & 0 & 0 & 0 \\\\\n", + "bakrianoo & 2 & 0 & 0 & 2 & 0 & 0 & 0 & 0 & 0 \\\\\n", "\\end{longtable}\n", "\n" ] @@ -523,7 +527,7 @@ " Enevoldsen\n", " kennethcenevoldsen@gmail.com\n", " ~Kenneth_Enevoldsen1\n", - " Aarhus University, Denmark\n", + " Aarhus University\n", " \n", " \n", " 1\n", @@ -532,7 +536,7 @@ " Kardos\n", " martonkardos@cas.au.dk\n", " ~Márton_Kardos1\n", - " Aarhus University, Denmark\n", + " Aarhus University\n", " \n", " \n", " 2\n", @@ -540,8 +544,8 @@ " Imene\n", " Kerboua\n", " \n", - " \n", - " Esker, Lyon, France && INSA Lyon, LIRIS, Lyon,...\n", + " ~Imene_Kerboua1\n", + " INSA Lyon, LIRIS\n", " \n", " \n", " 3\n", @@ -550,7 +554,7 @@ " Siblini\n", " wissam.siblini92@gmail.com\n", " ~Wissam_Siblini1\n", - " N/A\n", + " Individual Contributor\n", " \n", " \n", " 4\n", @@ -559,7 +563,7 @@ " Sequeira\n", " \n", " \n", - " N/A\n", + " Individual Contributor\n", " \n", " \n", " ...\n", @@ -571,83 +575,83 @@ " ...\n", " \n", " \n", - " 69\n", - " sivareddyg\n", - " Siva\n", - " Reddy\n", - " siva.reddy@mila.quebec\n", - " ~Siva_Reddy1\n", - " McGill University && Mila - Quebec AI Institut...\n", + " 80\n", + " john-b-yang\n", + " John\n", + " Yang\n", + " johnby@stanford.edu\n", + " ~John_Yang3\n", + " Stanford University\n", " \n", " \n", - " 70\n", - " guenthermi\n", - " Michael\n", - " Günther\n", - " michael.guenther@jina.ai\n", - " ~Michael_Günther1\n", - " Jina AI\n", + " 81\n", + " thakur-nandan\n", + " Nandan\n", + " Thakur\n", + " \n", + " ~Nandan_Thakur1\n", + " University of Waterloo\n", " \n", " \n", - " 71\n", - " violenil\n", - " Isabelle\n", - " Mohr\n", - " isabelle.mohr@jina.ai\n", - " ~Isabelle_Mohr1\n", - " Jina AI\n", + " 82\n", + " loicmagne\n", + " Loic\n", + " Magne\n", + " ~Loïc_Magne1\n", + " Individual Contributor\n", + " None\n", " \n", " \n", - " 72\n", - " Muennighoff\n", - " Niklas\n", - " Muennighoff\n", - " n.muennighoff@gmail.com\n", + " 83\n", + " sarahooker\n", + " Sara\n", + " Hooker\n", " \n", - " Contextual AI\n", + " ~Sara_Hooker2\n", + " Cohere For AI\n", " \n", " \n", - " 73\n", - " AlexeyVatolin\n", - " Aleksei\n", - " Vatolin\n", - " vatolinalex@gmail.com\n", - " ~Aleksei_Vatolin1\n", - " FRC CSC RAS\n", + " 84\n", + " kwojtasi\n", + " Konrad\n", + " Wojtasik\n", + " ~Konrad_Wojtasik1\n", + " Wrocław University of Science and Technology\n", + " None\n", " \n", " \n", "\n", - "

74 rows × 6 columns

\n", + "

85 rows × 6 columns

\n", "" ], "text/plain": [ - " GitHub First name Last name Email \\\n", - "0 KennethEnevoldsen Kenneth Enevoldsen kennethcenevoldsen@gmail.com \n", - "1 x-tabdeveloping Márton Kardos martonkardos@cas.au.dk \n", - "2 imenelydiaker Imene Kerboua \n", - "3 wissam-sib Wissam Siblini wissam.siblini92@gmail.com \n", - "4 GabrielSequeira Gabriel Sequeira \n", - ".. ... ... ... ... \n", - "69 sivareddyg Siva Reddy siva.reddy@mila.quebec \n", - "70 guenthermi Michael Günther michael.guenther@jina.ai \n", - "71 violenil Isabelle Mohr isabelle.mohr@jina.ai \n", - "72 Muennighoff Niklas Muennighoff n.muennighoff@gmail.com \n", - "73 AlexeyVatolin Aleksei Vatolin vatolinalex@gmail.com \n", + " GitHub First name Last name Email \\\n", + "0 KennethEnevoldsen Kenneth Enevoldsen kennethcenevoldsen@gmail.com \n", + "1 x-tabdeveloping Márton Kardos martonkardos@cas.au.dk \n", + "2 imenelydiaker Imene Kerboua \n", + "3 wissam-sib Wissam Siblini wissam.siblini92@gmail.com \n", + "4 GabrielSequeira Gabriel Sequeira \n", + ".. ... ... ... ... \n", + "80 john-b-yang John Yang johnby@stanford.edu \n", + "81 thakur-nandan Nandan Thakur \n", + "82 loicmagne Loic Magne ~Loïc_Magne1 \n", + "83 sarahooker Sara Hooker \n", + "84 kwojtasi Konrad Wojtasik ~Konrad_Wojtasik1 \n", "\n", - " User on openreview Affiliations \n", - "0 ~Kenneth_Enevoldsen1 Aarhus University, Denmark \n", - "1 ~Márton_Kardos1 Aarhus University, Denmark \n", - "2 Esker, Lyon, France && INSA Lyon, LIRIS, Lyon,... \n", - "3 ~Wissam_Siblini1 N/A \n", - "4 N/A \n", - ".. ... ... \n", - "69 ~Siva_Reddy1 McGill University && Mila - Quebec AI Institut... \n", - "70 ~Michael_Günther1 Jina AI \n", - "71 ~Isabelle_Mohr1 Jina AI \n", - "72 Contextual AI \n", - "73 ~Aleksei_Vatolin1 FRC CSC RAS \n", + " User on openreview Affiliations \n", + "0 ~Kenneth_Enevoldsen1 Aarhus University \n", + "1 ~Márton_Kardos1 Aarhus University \n", + "2 ~Imene_Kerboua1 INSA Lyon, LIRIS \n", + "3 ~Wissam_Siblini1 Individual Contributor \n", + "4 Individual Contributor \n", + ".. ... ... \n", + "80 ~John_Yang3 Stanford University \n", + "81 ~Nandan_Thakur1 University of Waterloo \n", + "82 Individual Contributor None \n", + "83 ~Sara_Hooker2 Cohere For AI \n", + "84 Wrocław University of Science and Technology None \n", "\n", - "[74 rows x 6 columns]" + "[85 rows x 6 columns]" ] }, "execution_count": 8, @@ -672,52 +676,51 @@ "\\toprule\n", "GitHub & First name & Last name & Affiliations \\\\\n", "\\midrule\n", - "KennethEnevoldsen & Kenneth & Enevoldsen & Aarhus University, Denmark \\\\\n", - "x-tabdeveloping & Márton & Kardos & Aarhus University, Denmark \\\\\n", - "imenelydiaker & Imene & Kerboua & Esker, Lyon, France && INSA Lyon, LIRIS, Lyon, France \\\\\n", - "wissam-sib & Wissam & Siblini & N/A \\\\\n", - "GabrielSequeira & Gabriel & Sequeira & N/A \\\\\n", - "schmarion & Marion & Schaeffer & Wikit, Lyon, France \\\\\n", - "MathieuCiancone & Mathieu & Ciancone & Wikit, Lyon, France \\\\\n", - "MartinBernstorff & Martin & Bernstorff & Aarhus University, Denmark \\\\\n", + "KennethEnevoldsen & Kenneth & Enevoldsen & Aarhus University \\\\\n", + "x-tabdeveloping & Márton & Kardos & Aarhus University \\\\\n", + "imenelydiaker & Imene & Kerboua & INSA Lyon, LIRIS \\\\\n", + "wissam-sib & Wissam & Siblini & Individual Contributor \\\\\n", + "GabrielSequeira & Gabriel & Sequeira & Individual Contributor \\\\\n", + "schmarion & Marion & Schaeffer & Wikit \\\\\n", + "MathieuCiancone & Mathieu & Ciancone & Wikit \\\\\n", + "MartinBernstorff & Martin & Bernstorff & Aarhus University \\\\\n", "staoxiao & Shitao & Xiao & Beijing Academy of Artificial Intelligence \\\\\n", "ZhengLiu101 & Zheng & Liu & Beijing Academy of Artificial Intelligence \\\\\n", - "achibb & Aaron & Chibb & N/A \\\\\n", - "cassanof & Federico & Cassano & Northeastern University, Boston, USA \\\\\n", + "achibb & Aaron & Chibb & Individual Contributor \\\\\n", + "cassanof & Federico & Cassano & Northeastern University && Cursor AI \\\\\n", "taidnguyen & Nguyen & Tai & University of Pennsylvania \\\\\n", "xu3kev & Wen-Ding & Li & Cornell University \\\\\n", - "Rysias & Jonathan & Rystrøm & University of Oxford, UK \\\\\n", + "Rysias & Jonathan & Rystrøm & University of Oxford \\\\\n", "taeminlee & Taemin & Lee & Korea University Human-Inspired AI Research \\\\\n", - "izhx & Xin & Zhang & Harbin Institute of Technology, Shenzhen \\\\\n", + "izhx & Xin & Zhang & Harbin Institute of Technology \\\\\n", "orionw & Orion & Weller & Johns Hopkins University \\\\\n", - "slvnwhrl & Silvan & Wehrli & Robert Koch Institute, Berlin, Germany \\\\\n", - "manandey & Manan & Dey & Salesforce, India \\\\\n", - "isaac-chung & Isaac & Chung & N/A \\\\\n", + "slvnwhrl & Silvan & Wehrli & Robert Koch Institute \\\\\n", + "manandey & Manan & Dey & Salesforce \\\\\n", + "isaac-chung & Isaac & Chung & Individual Contributor \\\\\n", "asparius & Ömer & Çağatan & Koç University,Turkey \\\\\n", - "rafalposwiata & Rafał & Poświata & National Information Processing Institute, Warsaw, Poland \\\\\n", - "rbroc & Roberta & Rocca & Aarhus University, Denmark \\\\\n", - "awinml & Ashwin & Mathur & N/A \\\\\n", + "rafalposwiata & Rafał & Poświata & National Information Processing Institute \\\\\n", + "rbroc & Roberta & Rocca & Aarhus University \\\\\n", + "awinml & Ashwin & Mathur & Individual Contributor \\\\\n", "guangyusong & Guangyu & Song & Tano Labs \\\\\n", - "davidstap & David & Stap & University of Amsterdam. \\\\\n", - "HLasse & Lasse & Hansen & Aarhus University, Denmark \\\\\n", + "davidstap & David & Stap & University of Amsterdam \\\\\n", + "HLasse & Lasse & Hansen & Aarhus University \\\\\n", "jaygala24 & Jay & Gala & MBZUAI \\\\\n", - "digantamisra98 & Diganta & Misra & Mila - Quebec AI Institute \\\\\n", - "PranjalChitale & Pranjal & Chitale & Indian Institute of Technology Madras \\\\\n", - "Akash190104 & Akash & Kundu & Heritage Institute of Technology, Kolkata && Apart Research \\\\\n", + "digantamisra98 & Diganta & Misra & Max Planck Institute for Intelligent Systems && ELLIS Institute Tübingen \\\\\n", + "PranjalChitale & Pranjal & Chitale & Indian Institute of Technology \\\\\n", + "Akash190104 & Akash & Kundu & Heritage Institute of Technology && Apart Research \\\\\n", "dwzhu-pku & Dawei & Zhu & Peking University \\\\\n", "ljvmiranda921 & Lester James & Miranda & Allen Institute for AI \\\\\n", - "Sakshamrzt & Saksham & Thakur & N/A \\\\\n", "Andrian0s & Andrianos & Michail & University of Zurich \\\\\n", "simon-clematide & Simon & Clematide & University of Zurich \\\\\n", "SaitejaUtpala & Saiteja & Utpala & Microsoft Research \\\\\n", "mmhamdy & Mohammed & Hamdy & Cohere For AI Community \\\\\n", "jupyterjazz & Saba & Sturua & Jina AI \\\\\n", "Ruqyai & Ruqiya & Bin Safi & NaN \\\\\n", - "kranthigv & Kranthi Kiran & GV & New York University \\\\\n", + "KranthiGV & Kranthi Kiran & GV & New York University \\\\\n", "shreeya-dhakal & Shreeya & Dhakal & Individual Contributor \\\\\n", "dipam7 & Dipam & Vasani & Individual Contributor \\\\\n", - "Art3mis07 & Gayatri & K & R. V. College of Engineering, Bengaluru \\\\\n", - "jankounchained & Jan & Kostkan & Aarhus University, Denmark \\\\\n", + "Art3mis07 & Gayatri & K & R. V. College of Engineering \\\\\n", + "jankounchained & Jan & Kostkan & Aarhus University \\\\\n", "bp-high & Bhavish & Pahwa & Microsoft Research \\\\\n", "rasdani & Daniel & Auras & ellamind, Germany \\\\\n", "ShawonAshraf & Shawon & Ashraf & ellamind, Germany \\\\\n", @@ -727,7 +730,7 @@ "ManuelFay & Manuel & Faysse & CentraleSupélec && Illuin Technology \\\\\n", "hgissbkh & Hippolyte & Gisserot-Boukhlef & CentraleSupélec && Artefact Research Center \\\\\n", "sted97 & Simone & Tedeschi & Sapienza University of Rome \\\\\n", - "gentaiscool & Genta Indra & Winata & N/A \\\\\n", + "gentaiscool & Genta Indra & Winata & Individual Contributor \\\\\n", "henilp105 & Henil & Panchal & Nirma University \\\\\n", "ABorghini & Alessia & Borghini & Sapienza University of Rome \\\\\n", "jordiclive & Jordan & Clive & Imperial College London \\\\\n", @@ -735,17 +738,29 @@ "mariyahendriksen & Mariya & Hendriksen & University of Amsterdam \\\\\n", "dokato & Dominik & Krzemiński & Cohere For AI Community \\\\\n", "Samoed & Roman & Solomatin & ITMO \\\\\n", - "Alenush & Alena & Fenogenova & SaluteDevices, Russia \\\\\n", - "ab1992ao & Aleksandr & Abramov & SaluteDevices, Russia \\\\\n", - "artemsnegirev & Artem & Snegirev & SaluteDevices, Russia \\\\\n", - "anpalmak2003 & Anna & Maksimova & SaluteDevices, Russia \\\\\n", - "MariyaTikhonova & Maria & Tikhonova & SaluteDevices, HSE University, Russia \\\\\n", - "vaibhavad & Vaibhav & Adlakha & McGill University && Mila - Quebec AI Institute && ServiceNow Research \\\\\n", - "sivareddyg & Siva & Reddy & McGill University && Mila - Quebec AI Institute && ServiceNow Research && Facebook CIFAR AI Chair \\\\\n", + "Alenush & Alena & Fenogenova & SaluteDevices \\\\\n", + "ab1992ao & Aleksandr & Abramov & SaluteDevices \\\\\n", + "artemsnegirev & Artem & Snegirev & SaluteDevices \\\\\n", + "anpalmak2003 & Anna & Maksimova & SaluteDevices \\\\\n", + "MariyaTikhonova & Maria & Tikhonova & SaluteDevices && HSE University \\\\\n", + "vaibhavad & Vaibhav & Adlakha & Mila, McGill University && ServiceNow Research \\\\\n", + "sivareddyg & Siva & Reddy & Mila, McGill University && ServiceNow Research \\\\\n", "guenthermi & Michael & Günther & Jina AI \\\\\n", "violenil & Isabelle & Mohr & Jina AI \\\\\n", - "Muennighoff & Niklas & Muennighoff & Contextual AI \\\\\n", + "akshita-sukhlecha & Akshita & Sukhlecha & Individual Contributor \\\\\n", + "Muennighoff & Niklas & Muennighoff & Stanford University && Contextual AI \\\\\n", "AlexeyVatolin & Aleksei & Vatolin & FRC CSC RAS \\\\\n", + "xhluca & Xing Han & Lù & Mila, McGill University \\\\\n", + "crystina-z & Xinyu & Zhang & University of Waterloo \\\\\n", + "tomaarsen & Tom & Aarsen & Hugging Face \\\\\n", + "mrshu & Marek & Suppa & Comenius University Bratislava && Cisco Systems \\\\\n", + "swj0419 & Weijia & Shi & University of Washington \\\\\n", + "xiamengzhou & Mengzhou & Xia & Princeton University \\\\\n", + "john-b-yang & John & Yang & Stanford University \\\\\n", + "thakur-nandan & Nandan & Thakur & University of Waterloo \\\\\n", + "loicmagne & Loic & Magne & NaN \\\\\n", + "sarahooker & Sara & Hooker & Cohere For AI \\\\\n", + "kwojtasi & Konrad & Wojtasik & NaN \\\\\n", "\\bottomrule\n", "\\end{tabular}\n", "\n" @@ -776,8 +791,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "achibb has less than 10 points\n", - "izhx has less than 10 points\n" + "izhx has less than 10 points\n", + "achibb has less than 10 points\n" ] } ], @@ -799,25 +814,45 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'guangyusong'" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gh" + ] + }, + { + "cell_type": "code", + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "['akshita-sukhlecha', 'loicmagne', 'mrshu', 'crystina-z', 'thakur-nandan', 'xhluca']\n" + "['Sakshamrzt']\n" ] } ], "source": [ - "missing_users = [user for user in df[df[\"Total\"] > 10][\"GitHub\"] if user not in github]\n", + "missing_users = [user for user in df[df[\"Total\"] >= 10][\"GitHub\"] if user not in github]\n", "print(missing_users)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -828,7 +863,7 @@ }, { "cell_type": "code", - "execution_count": 191, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -846,7 +881,7 @@ }, { "cell_type": "code", - "execution_count": 192, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -863,13 +898,23 @@ " if row[\"Affiliations\"]:\n", " affiliations = row[\"Affiliations\"].split(\"&&\")\n", "\n", + " aff_string = \"\"\n", " for aff in affiliations:\n", + " aff = aff.strip()\n", " if \"N/A\" in aff:\n", " continue\n", " if aff not in affiations:\n", " affiations[aff] = aff_id\n", " aff_id += 1\n", - " author_str += f\"\\\\textsuperscript{{{affiations[aff]}}}\"\n", + " aff_string += f\"{affiations[aff]},\"\n", + "\n", + " # remove last comma\n", + " aff_string = aff_string[:-1]\n", + "\n", + " if aff_string:\n", + " author_str += f\"\\\\textsuperscript{{{aff_string}}}\"\n", + " else:\n", + " author_str += \"\"\n", "\n", " # if row[\"Affiliations\"] not in affiations and row[\"Affiliations\"]:\n", " # affiations[row[\"Affiliations\"]] = aff_id\n", @@ -881,7 +926,27 @@ }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "# Move last author to the end\n", + "last_author1 = \"Niklas Muennighoff\"\n", + "last_author_ = [a for a in author_list if last_author1 in a][0]\n", + "last_author2 = \"Siva\"\n", + "last_author__ = [a for a in author_list if last_author2 in a][0]\n", + "# remove from author list\n", + "author_list = [\n", + " a for a in author_list if last_author1 not in a and last_author2 not in a\n", + "]\n", + "\n", + "author_list.append(last_author__)\n", + "author_list.append(last_author_)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -889,7 +954,7 @@ "\n", "latex = \"\"\n", "line_length = 0\n", - "max_line_length = 80\n", + "max_line_length = 85\n", "\n", "for i, author in enumerate(author_list):\n", " _line_length = len(author.split(\"\\\\textsuperscript\")[0])\n", @@ -916,7 +981,7 @@ }, { "cell_type": "code", - "execution_count": 194, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -924,154 +989,185 @@ "output_type": "stream", "text": [ "\\textbf{Kenneth Enevoldsen\\textsuperscript{1}}, \n", - "\\textbf{Isaac Chung}, \n", - "\\textbf{Ashwin Mathur}, \n", + "\\textbf{Isaac Chung\\textsuperscript{2}}, \n", + "\\textbf{Imene Kerboua\\textsuperscript{3}}, \n", "\\\\\n", - "\\textbf{Imene Kerboua\\textsuperscript{2}\\textsuperscript{3}}, \n", + "\\textbf{Ashwin Mathur\\textsuperscript{2}}, \n", "\\textbf{Márton Kardos\\textsuperscript{1}}, \n", "\\textbf{David Stap\\textsuperscript{4}}, \n", "\\textbf{Jay Gala\\textsuperscript{5}}, \n", "\\\\\n", - "\\textbf{Wissam Siblini}, \n", - "\\textbf{Dominik Krzemiński\\textsuperscript{6}}, \n", - "\\textbf{Genta Indra Winata}, \n", + "\\textbf{Wissam Siblini\\textsuperscript{2}}, \n", + "\\textbf{Dominik Krzemiński\\textsuperscript{8}}, \n", + "\\textbf{Genta Indra Winata\\textsuperscript{2}}, \n", "\\\\\n", - "\\textbf{Saba Sturua\\textsuperscript{7}}, \n", - "\\textbf{Saiteja Utpala\\textsuperscript{8}}, \n", - "\\textbf{Gabriel Sequeira}, \n", + "\\textbf{Saba Sturua\\textsuperscript{9}}, \n", + "\\textbf{Saiteja Utpala\\textsuperscript{10}}, \n", + "\\textbf{Orion Weller\\textsuperscript{11}}, \n", + "\\textbf{Mathieu Ciancone\\textsuperscript{12}}, \n", "\\\\\n", - "\\textbf{Marion Schaeffer\\textsuperscript{9}}, \n", - "\\textbf{Mathieu Ciancone\\textsuperscript{9}}, \n", - "\\textbf{Diganta Misra\\textsuperscript{10}}, \n", + "\\textbf{Marion Schaeffer\\textsuperscript{12}}, \n", + "\\textbf{Gabriel Sequeira\\textsuperscript{2}}, \n", + "\\textbf{Diganta Misra\\textsuperscript{13,14}}, \n", "\\\\\n", - "\\textbf{Shreeya Dhakal\\textsuperscript{11}}, \n", - "\\textbf{Jonathan Rystrøm\\textsuperscript{12}}, \n", - "\\textbf{Orion Weller\\textsuperscript{13}}, \n", + "\\textbf{Vaibhav Adlakha\\textsuperscript{15,16}}, \n", + "\\textbf{Shreeya Dhakal\\textsuperscript{2}}, \n", + "\\textbf{Jonathan Rystrøm\\textsuperscript{17}}, \n", "\\\\\n", - "\\textbf{Chenghao Xiao\\textsuperscript{14}}, \n", - "\\textbf{Ömer Çağatan\\textsuperscript{15}}, \n", - "\\textbf{Akash Kundu\\textsuperscript{16}\\textsuperscript{17}}, \n", - "\\textbf{Shitao Xiao\\textsuperscript{18}}, \n", + "\\textbf{Roman Solomatin\\textsuperscript{18}}, \n", + "\\textbf{Chenghao Xiao\\textsuperscript{19}}, \n", + "\\textbf{Ömer Çağatan\\textsuperscript{20}}, \n", + "\\textbf{Akash Kundu\\textsuperscript{21,22}}, \n", "\\\\\n", - "\\textbf{Bhavish Pahwa\\textsuperscript{8}}, \n", - "\\textbf{Rafał Poświata\\textsuperscript{19}}, \n", - "\\textbf{Shawon Ashraf\\textsuperscript{20}}, \n", + "\\textbf{Martin Bernstorff\\textsuperscript{1}}, \n", + "\\textbf{Shitao Xiao\\textsuperscript{23}}, \n", + "\\textbf{Akshita Sukhlecha\\textsuperscript{2}}, \n", "\\\\\n", - "\\textbf{Björn Plüster\\textsuperscript{20}}, \n", - "\\textbf{Jan Philipp Harries\\textsuperscript{20}}, \n", - "\\textbf{Daniel Auras\\textsuperscript{20}}, \n", + "\\textbf{Bhavish Pahwa\\textsuperscript{10}}, \n", + "\\textbf{Rafał Poświata\\textsuperscript{24}}, \n", + "\\textbf{Kranthi Kiran GV\\textsuperscript{25}}, \n", "\\\\\n", - "\\textbf{Kranthi Kiran GV\\textsuperscript{21}}, \n", - "\\textbf{Isabelle Mohr\\textsuperscript{7}}, \n", - "\\textbf{Dawei Zhu\\textsuperscript{22}}, \n", + "\\textbf{Shawon Ashraf\\textsuperscript{26}}, \n", + "\\textbf{Daniel Auras\\textsuperscript{26}}, \n", + "\\textbf{Björn Plüster\\textsuperscript{26}}, \n", "\\\\\n", - "\\textbf{Martin Bernstorff\\textsuperscript{1}}, \n", - "\\textbf{Hippolyte Gisserot-Boukhlef\\textsuperscript{23}\\textsuperscript{24}}, \n", - "\\textbf{Taemin Lee\\textsuperscript{25}}, \n", + "\\textbf{Jan Philipp Harries\\textsuperscript{26}}, \n", + "\\textbf{Loic Magne}, \n", + "\\textbf{Isabelle Mohr\\textsuperscript{9}}, \n", + "\\textbf{Dawei Zhu\\textsuperscript{27}}, \n", "\\\\\n", + "\\textbf{Hippolyte Gisserot-Boukhlef\\textsuperscript{28,29}}, \n", + "\\textbf{Tom Aarsen\\textsuperscript{30}}, \n", "\\textbf{Jan Kostkan\\textsuperscript{1}}, \n", - "\\textbf{Andrianos Michail\\textsuperscript{26}}, \n", - "\\textbf{Manuel Faysse\\textsuperscript{23}\\textsuperscript{27}}, \n", "\\\\\n", - "\\textbf{Mohammed Hamdy\\textsuperscript{6}}, \n", + "\\textbf{Konrad Wojtasik}, \n", + "\\textbf{Taemin Lee\\textsuperscript{31}}, \n", + "\\textbf{Marek Suppa\\textsuperscript{32,33}}, \n", + "\\textbf{Xinyu Zhang\\textsuperscript{34}}, \n", + "\\\\\n", "\\textbf{Roberta Rocca\\textsuperscript{1}}, \n", - "\\textbf{Manan Dey\\textsuperscript{28}}, \n", - "\\textbf{Dipam Vasani\\textsuperscript{11}}, \n", + "\\textbf{Mohammed Hamdy\\textsuperscript{8}}, \n", + "\\textbf{Andrianos Michail\\textsuperscript{35}}, \n", + "\\textbf{John Yang\\textsuperscript{6}}, \n", + "\\\\\n", + "\\textbf{Manuel Faysse\\textsuperscript{28,36}}, \n", + "\\textbf{Aleksei Vatolin\\textsuperscript{37}}, \n", + "\\textbf{Nandan Thakur\\textsuperscript{34}}, \n", + "\\textbf{Manan Dey\\textsuperscript{38}}, \n", "\\\\\n", - "\\textbf{Simone Tedeschi\\textsuperscript{29}}, \n", - "\\textbf{Pranjal Chitale\\textsuperscript{30}}, \n", - "\\textbf{Saksham Thakur}, \n", + "\\textbf{Dipam Vasani\\textsuperscript{2}}, \n", + "\\textbf{Pranjal Chitale\\textsuperscript{39}}, \n", + "\\textbf{Simone Tedeschi\\textsuperscript{40}}, \n", + "\\textbf{Nguyen Tai\\textsuperscript{41}}, \n", "\\\\\n", - "\\textbf{Roman Solomatin\\textsuperscript{31}}, \n", - "\\textbf{Nguyen Tai\\textsuperscript{32}}, \n", - "\\textbf{Artem Snegirev\\textsuperscript{33}}, \n", - "\\textbf{Gayatri K\\textsuperscript{34}}, \n", + "\\textbf{Artem Snegirev\\textsuperscript{42}}, \n", + "\\textbf{Mariya Hendriksen\\textsuperscript{4}}, \n", + "\\textbf{Michael Günther\\textsuperscript{9}}, \n", "\\\\\n", - "\\textbf{Mariya Hendriksen\\textsuperscript{35}}, \n", - "\\textbf{Michael Günther\\textsuperscript{7}}, \n", - "\\textbf{Anna Maksimova\\textsuperscript{33}}, \n", + "\\textbf{Mengzhou Xia\\textsuperscript{43}}, \n", + "\\textbf{Weijia Shi\\textsuperscript{44}}, \n", + "\\textbf{Xing Han Lù\\textsuperscript{15}}, \n", + "\\textbf{Jordan Clive\\textsuperscript{45}}, \n", "\\\\\n", - "\\textbf{Silvan Wehrli\\textsuperscript{36}}, \n", - "\\textbf{Jordan Clive\\textsuperscript{37}}, \n", - "\\textbf{Maria Tikhonova\\textsuperscript{38}}, \n", + "\\textbf{Gayatri K\\textsuperscript{46}}, \n", + "\\textbf{Anna Maksimova\\textsuperscript{42}}, \n", + "\\textbf{Silvan Wehrli\\textsuperscript{47}}, \n", + "\\textbf{Maria Tikhonova\\textsuperscript{42,48}}, \n", "\\\\\n", - "\\textbf{Aleksandr Abramov\\textsuperscript{33}}, \n", - "\\textbf{Henil Panchal\\textsuperscript{39}}, \n", - "\\textbf{Alena Fenogenova\\textsuperscript{33}}, \n", + "\\textbf{Henil Panchal\\textsuperscript{49}}, \n", + "\\textbf{Aleksandr Abramov\\textsuperscript{42}}, \n", + "\\textbf{Malte Ostendorff\\textsuperscript{50}}, \n", "\\\\\n", - "\\textbf{Lester James Miranda\\textsuperscript{40}}, \n", - "\\textbf{Alessia Borghini\\textsuperscript{29}}, \n", - "\\textbf{Zheng Liu\\textsuperscript{18}}, \n", + "\\textbf{Sara Hooker\\textsuperscript{51}}, \n", + "\\textbf{Zheng Liu\\textsuperscript{23}}, \n", + "\\textbf{Simon Clematide\\textsuperscript{35}}, \n", "\\\\\n", + "\\textbf{Lester James Miranda\\textsuperscript{52}}, \n", + "\\textbf{Alena Fenogenova\\textsuperscript{42}}, \n", "\\textbf{Lasse Hansen\\textsuperscript{1}}, \n", - "\\textbf{Simon Clematide\\textsuperscript{26}}, \n", - "\\textbf{Malte Ostendorff\\textsuperscript{41}}, \n", "\\\\\n", - "\\textbf{Guangyu Song}, \n", - "\\textbf{Wen-Ding Li\\textsuperscript{42}}, \n", + "\\textbf{Guangyu Song\\textsuperscript{53}}, \n", "\\textbf{Ruqiya Bin Safi}, \n", + "\\textbf{Wen-Ding Li\\textsuperscript{54}}, \n", "\\\\\n", - "\\textbf{Federico Cassano\\textsuperscript{43}}, \n", + "\\textbf{Alessia Borghini\\textsuperscript{40}}, \n", + "\\textbf{Federico Cassano\\textsuperscript{55,56}}, \n", + "\\textbf{Siva Reddy\\textsuperscript{15,16}}, \n", "\\\\\n", + "\\textbf{Niklas Muennighoff\\textsuperscript{6,7}}, \n", "\\\\\n", - "\\textsuperscript{1}Aarhus University, Denmark, \n", - "\\textsuperscript{2}Esker, Lyon, France , \n", - "\\textsuperscript{3} INSA Lyon, LIRIS, Lyon, France, \n", "\\\\\n", - "\\textsuperscript{4}University of Amsterdam., \n", + "\\textsuperscript{1}Aarhus University, \n", + "\\textsuperscript{2}Individual Contributor, \n", + "\\textsuperscript{3}INSA Lyon, LIRIS, \n", + "\\textsuperscript{4}University of Amsterdam, \n", "\\textsuperscript{5}MBZUAI, \n", - "\\textsuperscript{6}Cohere For AI Community, \n", - "\\textsuperscript{7}Jina AI, \n", - "\\textsuperscript{8}Microsoft Research, \n", "\\\\\n", - "\\textsuperscript{9}Wikit, Lyon, France, \n", - "\\textsuperscript{10}Mila - Quebec AI Institute, \n", - "\\textsuperscript{11}Individual Contributor, \n", + "\\textsuperscript{6}Stanford University, \n", + "\\textsuperscript{7}Contextual AI, \n", + "\\textsuperscript{8}Cohere For AI Community, \n", + "\\textsuperscript{9}Jina AI, \n", + "\\textsuperscript{10}Microsoft Research, \n", "\\\\\n", - "\\textsuperscript{12}University of Oxford, UK, \n", - "\\textsuperscript{13}Johns Hopkins University, \n", - "\\textsuperscript{14}Durham University, \n", + "\\textsuperscript{11}Johns Hopkins University, \n", + "\\textsuperscript{12}Wikit, \n", + "\\textsuperscript{13}Max Planck Institute for Intelligent Systems, \n", "\\\\\n", - "\\textsuperscript{15}Koç University,Turkey, \n", - "\\textsuperscript{16}Heritage Institute of Technology, Kolkata , \n", - "\\textsuperscript{17} Apart Research, \n", + "\\textsuperscript{14}ELLIS Institute Tübingen, \n", + "\\textsuperscript{15}Mila, McGill University, \n", + "\\textsuperscript{16}ServiceNow Research, \n", "\\\\\n", - "\\textsuperscript{18}Beijing Academy of Artificial Intelligence, \n", + "\\textsuperscript{17}University of Oxford, \n", + "\\textsuperscript{18}ITMO, \n", + "\\textsuperscript{19}Durham University, \n", + "\\textsuperscript{20}Koç University,Turkey, \n", "\\\\\n", - "\\textsuperscript{19}National Information Processing Institute, Warsaw, Poland, \n", - "\\textsuperscript{20}ellamind, Germany, \n", + "\\textsuperscript{21}Heritage Institute of Technology, \n", + "\\textsuperscript{22}Apart Research, \n", "\\\\\n", - "\\textsuperscript{21}New York University, \n", - "\\textsuperscript{22}Peking University, \n", - "\\textsuperscript{23}CentraleSupélec , \n", - "\\textsuperscript{24} Artefact Research Center, \n", + "\\textsuperscript{23}Beijing Academy of Artificial Intelligence, \n", + "\\textsuperscript{24}National Information Processing Institute, \n", "\\\\\n", - "\\textsuperscript{25}Korea University Human-Inspired AI Research, \n", - "\\textsuperscript{26}University of Zurich, \n", + "\\textsuperscript{25}New York University, \n", + "\\textsuperscript{26}ellamind, Germany, \n", + "\\textsuperscript{27}Peking University, \n", + "\\textsuperscript{28}CentraleSupélec, \n", "\\\\\n", - "\\textsuperscript{27} Illuin Technology, \n", - "\\textsuperscript{28}Salesforce, India, \n", - "\\textsuperscript{29}Sapienza University of Rome, \n", + "\\textsuperscript{29}Artefact Research Center, \n", + "\\textsuperscript{30}Hugging Face, \n", + "\\textsuperscript{31}Korea University Human-Inspired AI Research, \n", "\\\\\n", - "\\textsuperscript{30}Indian Institute of Technology Madras, \n", - "\\textsuperscript{31}ITMO, \n", - "\\textsuperscript{32}University of Pennsylvania, \n", + "\\textsuperscript{32}Comenius University Bratislava, \n", + "\\textsuperscript{33}Cisco Systems, \n", + "\\textsuperscript{34}University of Waterloo, \n", + "\\textsuperscript{35}University of Zurich, \n", "\\\\\n", - "\\textsuperscript{33}SaluteDevices, Russia, \n", - "\\textsuperscript{34}R. V. College of Engineering, Bengaluru, \n", + "\\textsuperscript{36}Illuin Technology, \n", + "\\textsuperscript{37}FRC CSC RAS, \n", + "\\textsuperscript{38}Salesforce, \n", + "\\textsuperscript{39}Indian Institute of Technology, \n", "\\\\\n", - "\\textsuperscript{35}University of Amsterdam, \n", - "\\textsuperscript{36}Robert Koch Institute, Berlin, Germany, \n", + "\\textsuperscript{40}Sapienza University of Rome, \n", + "\\textsuperscript{41}University of Pennsylvania, \n", + "\\textsuperscript{42}SaluteDevices, \n", "\\\\\n", - "\\textsuperscript{37}Imperial College London, \n", - "\\textsuperscript{38}SaluteDevices, HSE University, Russia, \n", - "\\textsuperscript{39}Nirma University, \n", + "\\textsuperscript{43}Princeton University, \n", + "\\textsuperscript{44}University of Washington, \n", + "\\textsuperscript{45}Imperial College London, \n", "\\\\\n", - "\\textsuperscript{40}Allen Institute for AI, \n", - "\\textsuperscript{41}Occiglot, \n", - "\\textsuperscript{42}Cornell University, \n", + "\\textsuperscript{46}R. V. College of Engineering, \n", + "\\textsuperscript{47}Robert Koch Institute, \n", + "\\textsuperscript{48}HSE University, \n", + "\\textsuperscript{49}Nirma University, \n", "\\\\\n", - "\\textsuperscript{43}Northeastern University, Boston, USA, \n", + "\\textsuperscript{50}Occiglot, \n", + "\\textsuperscript{51}Cohere For AI, \n", + "\\textsuperscript{52}Allen Institute for AI, \n", + "\\textsuperscript{53}Tano Labs, \n", + "\\textsuperscript{54}Cornell University, \n", + "\\\\\n", + "\\textsuperscript{55}Northeastern University, \n", + "\\textsuperscript{56}Cursor AI, \n", "\n" ] } @@ -1082,14 +1178,248 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "@orionw \n", + "@ZhengLiu101 \n", + "@staoxiao \n", + "@xiamengzhou \n", + "@xu3kev \n", + "@swj0419 \n", + "@Art3mis07 \n", + "@sted97 \n", + "@vaibhavad \n", + "@isaac-chung \n", + "@taeminlee \n", + "@Samoed \n", + "@mrshu \n", + "@Muennighoff \n", + "@kwojtasi \n", + "@jankounchained \n", + "@imenelydiaker \n", + "@AlexeyVatolin \n", + "@mariyahendriksen \n", + "@KennethEnevoldsen \n", + "@akshita-sukhlecha \n", + "@awinml \n", + "@jaygala24 \n", + "@digantamisra98 \n", + "@gentaiscool \n", + "@Rysias \n", + "@MartinBernstorff \n", + "@jupyterjazz \n", + "@davidstap \n", + "@Alenush \n", + "@MathieuCiancone \n", + "@xhluca \n", + "@rafalposwiata \n", + "@x-tabdeveloping \n", + "@ab1992ao \n", + "@artemsnegirev \n", + "@jphme \n", + "@slvnwhrl \n", + "@hgissbkh \n", + "@HLasse \n", + "@Ruqyai \n", + "@bp-high \n", + "@ljvmiranda921 \n", + "@violenil \n", + "@malteos \n", + "@rasdani \n", + "@asparius \n", + "@simon-clematide \n", + "@dokato \n", + "@mmhamdy \n", + "@john-b-yang \n", + "@henilp105 \n", + "@dwzhu-pku \n", + "@tomaarsen \n", + "@sarahooker \n", + "@manandey \n", + "@ManuelFay \n", + "@sivareddyg \n", + "@thakur-nandan \n", + "@Akash190104 \n", + "@shreeya-dhakal \n", + "@PranjalChitale \n", + "@schmarion \n", + "@ShawonAshraf \n", + "@loicmagne \n", + "@KranthiGV \n", + "@gowitheflow-1998 \n", + "@dipam7 \n", + "@rbroc \n", + "@ABorghini \n", + "@jordiclive \n", + "@Andrian0s \n", + "@bjoernpl \n", + "@taidnguyen \n", + "@MariyaTikhonova \n", + "@wissam-sib \n", + "@cassanof \n", + "@SaitejaUtpala \n", + "@GabrielSequeira \n", + "@crystina-z \n", + "@guenthermi \n", + "@anpalmak2003 \n", + "@guangyusong \n" + ] + } + ], "source": [ "# authors with 10 points or more\n", "\n", - "{g for g in github if g not in not_10}" + "{g for g in github if g not in not_10}\n", + "\n", + "# to a string @author\n", + "\n", + "for g in github:\n", + " if g in not_10:\n", + " continue\n", + " print(f\"@{g} \")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Kenneth Enevoldsen ~Kenneth_Enevoldsen1\n", + "Isaac Chung ~Isaac_Kwan_Yin_Chung1\n", + "Imene Kerboua ~Imene_Kerboua1\n", + "Ashwin Mathur ~Ashwin_Mathur1\n", + "Márton Kardos ~Márton_Kardos1\n", + "David Stap ~David_Stap\n", + "Jay Gala ~Jay_Gala1\n", + "Wissam Siblini ~Wissam_Siblini1\n", + "Niklas Muennighoff ~Niklas_Muennighoff1\n", + "Dominik Krzemiński ~Dominik_Krzemiński1\n", + "Genta Indra Winata ~Genta_Indra_Winata1\n", + "Saba Sturua ~Saba_Sturua1\n", + "Saiteja Utpala ~Saiteja_Utpala1\n", + "Orion Weller ~Orion_Weller1\n", + "Mathieu Ciancone ~Mathieu_Ciancone1\n", + "Marion Schaeffer ~Marion_Schaeffer1\n", + "Gabriel Sequeira \n", + "Diganta Misra ~Diganta_Misra1\n", + "Vaibhav Adlakha ~Vaibhav_Adlakha1\n", + "Shreeya Dhakal \n", + "Jonathan Rystrøm ~Jonathan_Rystrøm1\n", + "Roman Solomatin ~Roman_Solomatin1\n", + "Siva Reddy ~Siva_Reddy1\n", + "Chenghao Xiao ~Chenghao_Xiao1\n", + "Ömer Çağatan ~Ömer_Veysel_Çağatan1\n", + "Akash Kundu ~Akash_Kundu2\n", + "Martin Bernstorff ~Martin_Bernstorff1\n", + "Shitao Xiao ~Shitao_Xiao1\n", + "Akshita Sukhlecha ~Akshita_Sukhlecha1\n", + "Bhavish Pahwa ~Bhavish_Pahwa1\n", + "Rafał Poświata ~Rafał_Poświata1\n", + "Kranthi Kiran GV ~Kranthi_Kiran_GV1\n", + "Shawon Ashraf ~Shawon_Ashraf1\n", + "Daniel Auras ~Daniel_Auras1\n", + "Björn Plüster ~Björn_Plüster1\n", + "Jan Philipp Harries ~Jan_Philipp_Harries1\n", + "Loic Magne Individual Contributor\n", + "Isabelle Mohr ~Isabelle_Mohr1\n", + "Dawei Zhu ~Dawei_Zhu2\n", + "Hippolyte Gisserot-Boukhlef ~Hippolyte_Gisserot-Boukhlef1\n", + "Tom Aarsen ~Tom_Aarsen1\n", + "Jan Kostkan ~Jan_Kostkan1\n", + "Konrad Wojtasik Wrocław University of Science and Technology\n", + "Taemin Lee ~Taemin_Lee1\n", + "Marek Suppa ~Marek_Suppa1\n", + "Xinyu Zhang ~Crystina_Zhang1\n", + "Roberta Rocca ~Roberta_Rocca1\n", + "Mohammed Hamdy ~Mohammed_Hamdy1\n", + "Andrianos Michail ~Andrianos_Michail1\n", + "John Yang ~John_Yang3\n", + "Manuel Faysse ~Manuel_Faysse1\n", + "Aleksei Vatolin ~Aleksei_Vatolin1\n", + "Nandan Thakur ~Nandan_Thakur1\n", + "Manan Dey ~Manan_Dey2\n", + "Dipam Vasani ~Dipam_Vasani1\n", + "Pranjal Chitale ~Pranjal_A_Chitale1\n", + "Simone Tedeschi ~Simone_Tedeschi1\n", + "Nguyen Tai ~Nguyen_Tai1\n", + "Artem Snegirev ~Artem_Snegirev1\n", + "Mariya Hendriksen ~Mariya_Hendriksen1\n", + "Michael Günther ~Michael_Günther1\n", + "Mengzhou Xia ~Mengzhou_Xia1\n", + "Weijia Shi ~Weijia_Shi1\n", + "Xing Han Lù ~Xing_Han_Lù1\n", + "Jordan Clive ~Jordan_Clive1\n", + "Gayatri K ~Gayatri_K1\n", + "Anna Maksimova ~Anna_Maksimova1\n", + "Silvan Wehrli ~Silvan_Wehrli1\n", + "Maria Tikhonova ~Maria_Tikhonova1\n", + "Henil Panchal ~Henil_Shalin_Panchal1\n", + "Aleksandr Abramov ~Aleksandr_Abramov1\n", + "Malte Ostendorff ~Malte_Ostendorff1\n", + "Sara Hooker ~Sara_Hooker2\n", + "Zheng Liu ~Zheng_Liu4\n", + "Simon Clematide ~Simon_Clematide1\n", + "Lester James Miranda ~Lester_James_Validad_Miranda1\n", + "Alena Fenogenova ~Alena_Fenogenova1\n", + "Lasse Hansen ~Lasse_Hansen2\n", + "Guangyu Song ~Guangyu_Song1\n", + "Ruqiya Bin Safi ~Ruqiya_Bin_Safi1\n", + "Wen-Ding Li ~Wen-Ding_Li1\n", + "Alessia Borghini ~Alessia_Borghini1\n", + "Federico Cassano ~Federico_Cassano1\n" + ] + } + ], + "source": [ + "# get openreview ids for author_df\n", + "\n", + "# filter out authors with less than 10 points\n", + "tt = author_df[author_df[\"GitHub\"].isin({g for g in github if g not in not_10})]\n", + "\n", + "t = tt[[\"First name\", \"Last name\", \"User on openreview\"]]\n", + "\n", + "for row in t.iterrows():\n", + " print(row[1][\"First name\"], row[1][\"Last name\"], row[1][\"User on openreview\"])" ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "First name Federico\n", + "Last name Cassano\n", + "User on openreview ~Federico_Cassano1\n", + "Name: 11, dtype: object" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "row[1]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/scripts/running_model/check_results.py b/scripts/running_model/check_results.py index a4b166e0c8..c410fb5be7 100644 --- a/scripts/running_model/check_results.py +++ b/scripts/running_model/check_results.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Iterable +from collections.abc import Iterable import pandas as pd @@ -13,7 +13,7 @@ def results_to_dataframe( - mteb_results: dict[MODEL, dict[REVISION, list[mteb.MTEBResults]]], + mteb_results: dict[MODEL, dict[REVISION, list[mteb.TaskResult]]], ): data = [] for model_name, revisions in mteb_results.items(): diff --git a/scripts/running_model/check_run.sh b/scripts/running_model/check_run.sh index 6741f5f7c5..d4a9de4d70 100644 --- a/scripts/running_model/check_run.sh +++ b/scripts/running_model/check_run.sh @@ -7,7 +7,7 @@ # pip install codecarbon # ensure latest version of sentnece-transformers is installed: # pip install sentence-transformers --upgrade -# ensure that the the huggingface token is set and accecible using: +# ensure that the the huggingface token is set and accessible using: # huggingface-cli login echo "Running model on a sample set of tasks" # this is to check tasks are running correctly diff --git a/scripts/running_model/create_slurm_jobs.py b/scripts/running_model/create_slurm_jobs.py index 177775144e..606630d9e5 100644 --- a/scripts/running_model/create_slurm_jobs.py +++ b/scripts/running_model/create_slurm_jobs.py @@ -3,8 +3,8 @@ from __future__ import annotations import subprocess +from collections.abc import Iterable from pathlib import Path -from typing import Iterable import mteb diff --git a/scripts/task_selection/mteb_lite_results.csv b/scripts/task_selection/mteb_lite_results.csv index 2a7b4b0e5b..e382c9e3b1 100644 --- a/scripts/task_selection/mteb_lite_results.csv +++ b/scripts/task_selection/mteb_lite_results.csv @@ -1,13 +1,13 @@ -,model,revision,mean,mean (STS),mean (Classification),mean (Reranking),mean (Retrieval),mean (Clustering),mean (PairClassification),mean (weighted by task type),borda_count,Total Evaluation time (hours) -11,intfloat/e5-mistral-7b-instruct,07163b72af1488142a360786df853f237b1a3ca1,0.691,0.84,0.674,0.498,0.573,0.518,0.884,0.665,275.0,1.453 -2,GritLM/GritLM-7B,13f00a0e36500c80ce12870ea513846a066004af,0.686,0.828,0.695,0.496,0.572,0.5,0.873,0.661,265.0,1.707 -7,intfloat/multilingual-e5-large-instruct,baa7be480a7de1539afce709c8f13f833a510e0a,0.678,0.846,0.642,0.487,0.547,0.499,0.862,0.647,256.0,1.127 -3,intfloat/multilingual-e5-large,4dc6d853a804b9c8886ede6dda8a073b7dc08a81,0.643,0.815,0.658,0.447,0.493,0.427,0.847,0.615,190.0,1.211 -9,intfloat/multilingual-e5-base,d13f1b27baf31030b7fd040960d60d909913633f,0.627,0.799,0.639,0.443,0.459,0.427,0.836,0.6,147.0,0.72 -6,sentence-transformers/all-mpnet-base-v2,84f2bcc00d77236f9e89c8a360a00fb1139bf47d,0.595,0.724,0.513,0.484,0.469,0.458,0.83,0.58,143.0,0.741 -4,sentence-transformers/paraphrase-multilingual-mpnet-base-v2,79f2382ceacceacdf38563d7c5d16b9ff8d725d6,0.605,0.801,0.637,0.452,0.342,0.423,0.817,0.579,130.0,0.694 -8,sentence-transformers/all-MiniLM-L12-v2,a05860a77cef7b37e0048a7864658139bc18a854,0.581,0.711,0.523,0.475,0.433,0.438,0.825,0.568,122.0,0.571 -10,sentence-transformers/all-MiniLM-L6-v2,8b3219a92973c328a8e22fadcfa821b5dc75636a,0.579,0.708,0.515,0.471,0.43,0.446,0.824,0.566,106.0,0.521 -0,intfloat/multilingual-e5-small,e4ce9877abf3edfe10b0d82785e83bdcb973e22e,0.611,0.785,0.62,0.432,0.43,0.413,0.827,0.584,102.0,0.565 -5,sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2,bf3bf13ab40c3157080a7ab344c831b9ad18b5eb,0.584,0.778,0.59,0.454,0.328,0.411,0.8,0.56,76.0,0.618 -1,sentence-transformers/LaBSE,e34fab64a3011d2176c99545a93d5cbddc9a91b7,0.533,0.712,0.637,0.413,0.184,0.37,0.789,0.518,36.0,0.675 +,model,revision,mean,mean (Clustering),mean (STS),mean (Classification),mean (Reranking),mean (Retrieval),mean (PairClassification),mean (weighted by task type),borda_count,Total Evaluation time (hours),Total CO2-eq emissions (kg) +11,intfloat/e5-mistral-7b-instruct,07163b72af1488142a360786df853f237b1a3ca1,0.67,0.514,0.836,0.752,0.498,0.548,0.884,0.672,393.0,2.502,2.971 +2,GritLM/GritLM-7B,13f00a0e36500c80ce12870ea513846a066004af,0.664,0.508,0.825,0.77,0.496,0.532,0.873,0.667,384.0,3.111,3.409 +7,intfloat/multilingual-e5-large-instruct,baa7be480a7de1539afce709c8f13f833a510e0a,0.652,0.499,0.843,0.732,0.487,0.51,0.862,0.656,357.0,2.033,1.418 +3,intfloat/multilingual-e5-large,4dc6d853a804b9c8886ede6dda8a073b7dc08a81,0.621,0.428,0.806,0.728,0.447,0.49,0.847,0.624,270.0,2.549,1.563 +6,sentence-transformers/all-mpnet-base-v2,84f2bcc00d77236f9e89c8a360a00fb1139bf47d,0.56,0.466,0.722,0.566,0.484,0.419,0.83,0.581,211.0,1.19,0.688 +9,intfloat/multilingual-e5-base,d13f1b27baf31030b7fd040960d60d909913633f,0.602,0.422,0.791,0.7,0.443,0.461,0.836,0.609,211.0,1.17,0.691 +4,sentence-transformers/paraphrase-multilingual-mpnet-base-v2,79f2382ceacceacdf38563d7c5d16b9ff8d725d6,0.573,0.435,0.798,0.686,0.452,0.341,0.817,0.588,188.0,1.017,0.563 +8,sentence-transformers/all-MiniLM-L12-v2,a05860a77cef7b37e0048a7864658139bc18a854,0.547,0.446,0.707,0.558,0.475,0.407,0.825,0.57,172.0,0.814,0.442 +10,sentence-transformers/all-MiniLM-L6-v2,8b3219a92973c328a8e22fadcfa821b5dc75636a,0.544,0.449,0.704,0.554,0.471,0.398,0.824,0.567,149.0,0.733,0.391 +0,intfloat/multilingual-e5-small,e4ce9877abf3edfe10b0d82785e83bdcb973e22e,0.584,0.408,0.776,0.677,0.432,0.437,0.827,0.593,147.0,0.833,0.459 +5,sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2,bf3bf13ab40c3157080a7ab344c831b9ad18b5eb,0.551,0.417,0.775,0.644,0.454,0.328,0.8,0.57,109.0,0.879,0.469 +1,sentence-transformers/LaBSE,e34fab64a3011d2176c99545a93d5cbddc9a91b7,0.486,0.361,0.702,0.668,0.413,0.168,0.789,0.517,49.0,1.02,0.582 diff --git a/scripts/task_selection/mteb_lite_tasks.csv b/scripts/task_selection/mteb_lite_tasks.csv index 3d6987bf44..25e359e383 100644 --- a/scripts/task_selection/mteb_lite_tasks.csv +++ b/scripts/task_selection/mteb_lite_tasks.csv @@ -1,29 +1,41 @@ ,name,type,languages,domains,license -0,AmazonCounterfactualClassification,Classification,"['deu', 'eng', 'jpn']","['Reviews', 'Written']",CC BY 4.0 +0,AmazonCounterfactualClassification,Classification,"['deu', 'eng', 'jpn']","['Reviews', 'Written']",cc-by-4.0 1,ArguAna,Retrieval,['eng'],"['Medical', 'Written']",cc-by-sa-4.0 -2,ArXivHierarchicalClusteringP2P,Clustering,['eng'],"['Academic', 'Written']",CC0 -3,AskUbuntuDupQuestions,Reranking,['eng'],, -4,BIOSSES,STS,['eng'],, -5,BiorxivClusteringP2P.v2,Clustering,['eng'],"['Academic', 'Written']",https://www.biorxiv.org/content/about-biorxiv -6,CQADupstackGamingRetrieval,Retrieval,['eng'],, -7,FiQA2018,Retrieval,['eng'],, -8,MassiveIntentClassification,Classification,"['afr', 'amh', 'ara', 'aze', 'ben', 'cmo', 'cym', 'dan', 'deu', 'ell', 'eng', 'fas', 'fin', 'fra', 'heb', 'hin', 'hun', 'hye', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kan', 'kat', 'khm', 'kor', 'lav', 'mal', 'mon', 'msa', 'mya', 'nld', 'nob', 'pol', 'por', 'ron', 'rus', 'slv', 'spa', 'sqi', 'swa', 'swe', 'tam', 'tel', 'tgl', 'tha', 'tur', 'urd', 'vie']",['Spoken'],Apache 2.0 -9,MedrxivClusteringP2P.v2,Clustering,['eng'],"['Academic', 'Medical', 'Written']",https://www.medrxiv.org/content/about-medrxiv -10,MindSmallReranking,Reranking,['eng'],"['News', 'Written']",https://github.com/msnews/MIND/blob/master/MSR%20License_Data.pdf -11,SCIDOCS,Retrieval,['eng'],"['Academic', 'Written', 'Non-fiction']",cc-by-sa-4.0 -12,SICK-R,STS,['eng'],, -13,STS12,STS,['eng'],"['Encyclopaedic', 'News', 'Written']",Not specified -14,STS13,STS,['eng'],"['Web', 'News', 'Non-fiction', 'Written']",Not specified -15,STS15,STS,['eng'],"['Blog', 'News', 'Web', 'Written', 'Spoken']",Not specified -16,STS16,STS,['eng'],"['Blog', 'Web', 'Spoken']",Not specified -17,STS17,STS,"['ara', 'deu', 'eng', 'fra', 'ita', 'kor', 'nld', 'spa', 'tur']","['News', 'Web', 'Written']",Not specified -18,STS22.v2,STS,"['ara', 'cmn', 'deu', 'eng', 'fra', 'ita', 'pol', 'rus', 'spa', 'tur']","['News', 'Written']",Not specified -19,STSBenchmark,STS,['eng'],, -20,SprintDuplicateQuestions,PairClassification,['eng'],"['Programming', 'Written']",Not specified -21,StackExchangeClustering.v2,Clustering,['eng'],"['Web', 'Written']",Not specified -22,StackExchangeClusteringP2P.v2,Clustering,['eng'],"['Web', 'Written']",Not specified -23,TRECCOVID,Retrieval,['eng'],, -24,ToxicConversationsClassification,Classification,['eng'],"['Social', 'Written']",CC BY 4.0 -25,TweetSentimentExtractionClassification,Classification,['eng'],"['Social', 'Written']",Not specified -26,TwitterSemEval2015,PairClassification,['eng'],, -27,TwitterURLCorpus,PairClassification,['eng'],, +2,ArXivHierarchicalClusteringP2P,Clustering,['eng'],"['Academic', 'Written']",cc0-1.0 +3,ArXivHierarchicalClusteringS2S,Clustering,['eng'],"['Academic', 'Written']",cc0-1.0 +4,AskUbuntuDupQuestions,Reranking,['eng'],, +5,BIOSSES,STS,['eng'],, +6,Banking77Classification,Classification,['eng'],['Written'],mit +7,BiorxivClusteringP2P.v2,Clustering,['eng'],"['Academic', 'Written']",https://www.biorxiv.org/content/about-biorxiv +8,CQADupstackGamingRetrieval,Retrieval,['eng'],, +9,CQADupstackUnixRetrieval,Retrieval,['eng'],, +10,ClimateFEVERHardNegatives,Retrieval,['eng'],, +11,FEVERHardNegatives,Retrieval,['eng'],, +12,FiQA2018,Retrieval,['eng'],, +13,HotpotQAHardNegatives,Retrieval,['eng'],"['Web', 'Written']",cc-by-sa-4.0 +14,ImdbClassification,Classification,['eng'],"['Reviews', 'Written']",not specified +15,MTOPDomainClassification,Classification,"['deu', 'eng', 'fra', 'hin', 'spa', 'tha']","['Spoken', 'Spoken']",not specified +16,MassiveIntentClassification,Classification,"['afr', 'amh', 'ara', 'aze', 'ben', 'cmo', 'cym', 'dan', 'deu', 'ell', 'eng', 'fas', 'fin', 'fra', 'heb', 'hin', 'hun', 'hye', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kan', 'kat', 'khm', 'kor', 'lav', 'mal', 'mon', 'msa', 'mya', 'nld', 'nob', 'pol', 'por', 'ron', 'rus', 'slv', 'spa', 'sqi', 'swa', 'swe', 'tam', 'tel', 'tgl', 'tha', 'tur', 'urd', 'vie']",['Spoken'],apache-2.0 +17,MassiveScenarioClassification,Classification,"['afr', 'amh', 'ara', 'aze', 'ben', 'cmo', 'cym', 'dan', 'deu', 'ell', 'eng', 'fas', 'fin', 'fra', 'heb', 'hin', 'hun', 'hye', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kan', 'kat', 'khm', 'kor', 'lav', 'mal', 'mon', 'msa', 'mya', 'nld', 'nob', 'pol', 'por', 'ron', 'rus', 'slv', 'spa', 'sqi', 'swa', 'swe', 'tam', 'tel', 'tgl', 'tha', 'tur', 'urd', 'vie']",['Spoken'],apache-2.0 +18,MedrxivClusteringP2P.v2,Clustering,['eng'],"['Academic', 'Medical', 'Written']",https://www.medrxiv.org/content/about-medrxiv +19,MedrxivClusteringS2S.v2,Clustering,['eng'],"['Academic', 'Medical', 'Written']",https://www.medrxiv.org/content/about-medrxiv +20,MindSmallReranking,Reranking,['eng'],"['News', 'Written']",https://github.com/msnews/MIND/blob/master/MSR%20License_Data.pdf +21,SCIDOCS,Retrieval,['eng'],"['Academic', 'Written', 'Non-fiction']",cc-by-sa-4.0 +22,SICK-R,STS,['eng'],, +23,STS12,STS,['eng'],"['Encyclopaedic', 'News', 'Written']",not specified +24,STS13,STS,['eng'],"['Web', 'News', 'Non-fiction', 'Written']",not specified +25,STS14,STS,['eng'],"['Blog', 'Web', 'Spoken']",not specified +26,STS15,STS,['eng'],"['Blog', 'News', 'Web', 'Written', 'Spoken']",not specified +27,STS17,STS,"['ara', 'deu', 'eng', 'fra', 'ita', 'kor', 'nld', 'spa', 'tur']","['News', 'Web', 'Written']",not specified +28,STS22.v2,STS,"['ara', 'cmn', 'deu', 'eng', 'fra', 'ita', 'pol', 'rus', 'spa', 'tur']","['News', 'Written']",not specified +29,STSBenchmark,STS,['eng'],, +30,SprintDuplicateQuestions,PairClassification,['eng'],"['Programming', 'Written']",not specified +31,StackExchangeClustering.v2,Clustering,['eng'],"['Web', 'Written']",not specified +32,StackExchangeClusteringP2P.v2,Clustering,['eng'],"['Web', 'Written']",not specified +33,TRECCOVID,Retrieval,['eng'],, +34,Touche2020,Retrieval,['eng'],, +35,ToxicConversationsClassification,Classification,['eng'],"['Social', 'Written']",cc-by-4.0 +36,TweetSentimentExtractionClassification,Classification,['eng'],"['Social', 'Written']",not specified +37,TwentyNewsgroupsClustering.v2,Clustering,['eng'],"['News', 'Written']",not specified +38,TwitterSemEval2015,PairClassification,['eng'],, +39,TwitterURLCorpus,PairClassification,['eng'],, diff --git a/scripts/task_selection/mult_results.csv b/scripts/task_selection/mult_results.csv index 6395579c02..98edf2b0e1 100644 --- a/scripts/task_selection/mult_results.csv +++ b/scripts/task_selection/mult_results.csv @@ -1,13 +1,13 @@ -,model,revision,mean,mean (BitextMining),mean (PairClassification),mean (Classification),mean (STS),mean (Retrieval),mean (MultilabelClassification),mean (Clustering),mean (Reranking),mean (InstructionRetrieval),mean (wieghted by task type),borda_count,Total Evaluation time (hours) -7,intfloat/multilingual-e5-large-instruct,baa7be480a7de1539afce709c8f13f833a510e0a,0.634,0.801,0.811,0.65,0.767,0.58,0.222,0.515,0.625,-0.004,0.552,1237.0,6.396 -2,GritLM/GritLM-7B,13f00a0e36500c80ce12870ea513846a066004af,0.61,0.705,0.802,0.619,0.732,0.595,0.212,0.504,0.628,0.035,0.537,1114.0,8.63 -11,intfloat/e5-mistral-7b-instruct,07163b72af1488142a360786df853f237b1a3ca1,0.601,0.706,0.813,0.603,0.739,0.553,0.2,0.514,0.631,-0.006,0.528,1087.0,7.516 -3,intfloat/multilingual-e5-large,4dc6d853a804b9c8886ede6dda8a073b7dc08a81,0.587,0.717,0.793,0.599,0.734,0.543,0.213,0.431,0.626,-0.031,0.514,972.0,7.339 -9,intfloat/multilingual-e5-base,d13f1b27baf31030b7fd040960d60d909913633f,0.571,0.694,0.776,0.582,0.712,0.53,0.202,0.428,0.599,-0.027,0.5,802.0,3.738 -4,sentence-transformers/paraphrase-multilingual-mpnet-base-v2,79f2382ceacceacdf38563d7c5d16b9ff8d725d6,0.522,0.521,0.816,0.551,0.695,0.4,0.164,0.412,0.532,-0.011,0.453,693.0,15.838 -0,intfloat/multilingual-e5-small,e4ce9877abf3edfe10b0d82785e83bdcb973e22e,0.555,0.675,0.768,0.565,0.699,0.496,0.191,0.418,0.602,-0.024,0.488,645.0,2.686 -1,sentence-transformers/LaBSE,e34fab64a3011d2176c99545a93d5cbddc9a91b7,0.524,0.763,0.761,0.546,0.652,0.338,0.201,0.394,0.504,-0.03,0.459,586.0,3.382 -5,sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2,bf3bf13ab40c3157080a7ab344c831b9ad18b5eb,0.49,0.445,0.794,0.517,0.664,0.37,0.149,0.396,0.51,-0.013,0.426,471.0,2.626 -6,sentence-transformers/all-mpnet-base-v2,84f2bcc00d77236f9e89c8a360a00fb1139bf47d,0.427,0.212,0.71,0.47,0.571,0.342,0.163,0.411,0.421,-0.031,0.363,397.5,3.967 -8,sentence-transformers/all-MiniLM-L12-v2,a05860a77cef7b37e0048a7864658139bc18a854,0.423,0.229,0.719,0.468,0.566,0.336,0.146,0.368,0.443,-0.008,0.363,353.0,2.56 -10,sentence-transformers/all-MiniLM-L6-v2,8b3219a92973c328a8e22fadcfa821b5dc75636a,0.417,0.201,0.713,0.463,0.556,0.345,0.151,0.383,0.4,-0.028,0.354,288.5,2.316 +,model,revision,mean,mean (BitextMining),mean (PairClassification),mean (Classification),mean (STS),mean (Retrieval),mean (MultilabelClassification),mean (Clustering),mean (Reranking),mean (InstructionRetrieval),mean (weighted by task type),borda_count,Total Evaluation time (hours) +7,intfloat/multilingual-e5-large-instruct,baa7be480a7de1539afce709c8f13f833a510e0a,0.634,0.801,0.812,0.65,0.767,0.58,0.229,0.515,0.63,-0.004,0.553,1244.0,6.884 +2,GritLM/GritLM-7B,13f00a0e36500c80ce12870ea513846a066004af,0.609,0.705,0.802,0.619,0.732,0.591,0.212,0.504,0.628,0.035,0.536,1119.0,10.675 +11,intfloat/e5-mistral-7b-instruct,07163b72af1488142a360786df853f237b1a3ca1,0.602,0.706,0.814,0.603,0.739,0.554,0.222,0.514,0.634,-0.006,0.531,1100.0,9.969 +3,intfloat/multilingual-e5-large,4dc6d853a804b9c8886ede6dda8a073b7dc08a81,0.587,0.717,0.793,0.599,0.734,0.55,0.213,0.431,0.626,-0.031,0.515,980.0,9.206 +9,intfloat/multilingual-e5-base,d13f1b27baf31030b7fd040960d60d909913633f,0.571,0.694,0.776,0.582,0.712,0.536,0.202,0.428,0.599,-0.027,0.5,811.0,4.261 +4,sentence-transformers/paraphrase-multilingual-mpnet-base-v2,79f2382ceacceacdf38563d7c5d16b9ff8d725d6,0.52,0.521,0.816,0.551,0.695,0.393,0.164,0.412,0.532,-0.011,0.452,698.0,16.15 +0,intfloat/multilingual-e5-small,e4ce9877abf3edfe10b0d82785e83bdcb973e22e,0.556,0.675,0.768,0.565,0.699,0.502,0.191,0.418,0.602,-0.024,0.488,654.0,2.893 +1,sentence-transformers/LaBSE,e34fab64a3011d2176c99545a93d5cbddc9a91b7,0.521,0.763,0.761,0.546,0.652,0.329,0.201,0.394,0.504,-0.03,0.458,589.0,3.818 +5,sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2,bf3bf13ab40c3157080a7ab344c831b9ad18b5eb,0.488,0.445,0.794,0.517,0.664,0.362,0.149,0.396,0.51,-0.013,0.425,475.0,2.759 +6,sentence-transformers/all-mpnet-base-v2,84f2bcc00d77236f9e89c8a360a00fb1139bf47d,0.424,0.212,0.71,0.47,0.571,0.328,0.163,0.411,0.421,-0.031,0.362,397.5,4.772 +8,sentence-transformers/all-MiniLM-L12-v2,a05860a77cef7b37e0048a7864658139bc18a854,0.421,0.229,0.719,0.468,0.566,0.324,0.146,0.368,0.443,-0.008,0.362,355.0,2.691 +10,sentence-transformers/all-MiniLM-L6-v2,8b3219a92973c328a8e22fadcfa821b5dc75636a,0.415,0.201,0.713,0.463,0.556,0.331,0.151,0.383,0.4,-0.028,0.352,289.5,2.43 diff --git a/scripts/task_selection/task_selection_eng_lite.ipynb b/scripts/task_selection/task_selection_eng_lite.ipynb index a3883cfe59..6b9e36a7b3 100644 --- a/scripts/task_selection/task_selection_eng_lite.ipynb +++ b/scripts/task_selection/task_selection_eng_lite.ipynb @@ -4,16 +4,22 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Task Selection for MTEB(eng, lite)\n", - "\n", - "This is intended for creating mteb lite.\n" + "# Task Selection for MTEB(eng)\n" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 1, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/au561649/.virtualenvs/mteb/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, { "name": "stdout", "output_type": "stream", @@ -34,13 +40,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Defining the Tasks\n", - "Here we define the tasks for MTEB(eng, v2)" + "## Defining the initial scope\n", + "Here we define the tasks for MTEB(eng)" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -66,7 +72,7 @@ "source": [ "from mteb.benchmarks import MTEB_MAIN_EN\n", "\n", - "tasks = mteb.get_tasks(tasks=MTEB_MAIN_EN.tasks)\n", + "tasks = MTEB_MAIN_EN.tasks\n", "\n", "\n", "# get the updated version of tasks, which uses the new implementation (typically notably faster, but SummEvalSummarization.v2 also contains a notable bug fix: https://github.com/embeddings-benchmark/mteb/issues/1156\n", @@ -87,7 +93,30 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# replace hardegatives retrieval tasks:\n", + "\n", + "hard_retrieval_mapping = {\n", + " \"ClimateFEVER\": \"ClimateFEVERHardNegatives\",\n", + " \"FEVER\": \"FEVERHardNegatives\",\n", + " \"HotpotQA\": \"HotpotQAHardNegatives\",\n", + " \"DBPedia\": \"DBPediaHardNegatives\",\n", + "}\n", + "\n", + "tasks = [\n", + " task\n", + " if task.metadata.name not in hard_retrieval_mapping\n", + " else mteb.get_task(hard_retrieval_mapping[task.metadata.name])\n", + " for task in tasks\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -117,12 +146,12 @@ "CQADupstackUnixRetrieval\n", "CQADupstackWebmastersRetrieval\n", "CQADupstackWordpressRetrieval\n", - "ClimateFEVER\n", - "DBPedia\n", + "ClimateFEVERHardNegatives\n", + "DBPediaHardNegatives\n", "EmotionClassification\n", - "FEVER\n", + "FEVERHardNegatives\n", "FiQA2018\n", - "HotpotQA\n", + "HotpotQAHardNegatives\n", "ImdbClassification\n", "MTOPDomainClassification\n", "MTOPIntentClassification\n", @@ -162,6 +191,7 @@ } ], "source": [ + "# list of tasks\n", "for task in tasks:\n", " print(task.metadata.name)" ] @@ -176,9 +206,17 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 5, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/au561649/.virtualenvs/mteb/lib/python3.8/site-packages/huggingface_hub/file_download.py:1150: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + }, { "name": "stdout", "output_type": "stream", @@ -222,17 +260,25 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Already up to date.\n" + ] + } + ], "source": [ "# load results from mteb/results repository\n", - "mteb_results = mteb.load_results(models=models, tasks=tasks, download_latest=False)" + "mteb_results = mteb.load_results(models=models, tasks=tasks, download_latest=True)" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -243,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -278,9 +324,9 @@ " AskUbuntuDupQuestions\n", " BIOSSES\n", " ...\n", - " StackExchangeClustering.v2\n", " StackExchangeClusteringP2P.v2\n", " StackOverflowDupQuestions\n", + " SummEvalSummarization.v2\n", " TRECCOVID\n", " Touche2020\n", " ToxicConversationsClassification\n", @@ -317,32 +363,8 @@ " \n", " \n", " 0\n", - " GritLM/GritLM-7B\n", - " 0.633\n", - " 0.792\n", - " 0.966\n", - " 0.556\n", - " 0.598\n", - " 0.623\n", - " 0.632\n", - " 0.674\n", - " 0.863\n", - " ...\n", - " 0.681\n", - " 0.438\n", - " 0.559\n", - " 0.743\n", - " 0.278\n", - " 0.688\n", - " 0.663\n", - " 0.573\n", - " 0.811\n", - " 0.874\n", - " \n", - " \n", - " 1\n", " intfloat/e5-mistral-7b-instruct\n", - " 0.629\n", + " 0.626\n", " 0.732\n", " 0.963\n", " 0.520\n", @@ -352,9 +374,9 @@ " 0.670\n", " 0.855\n", " ...\n", - " 0.643\n", " 0.481\n", " 0.549\n", + " NaN\n", " 0.870\n", " 0.263\n", " 0.717\n", @@ -364,9 +386,33 @@ " 0.878\n", " \n", " \n", + " 1\n", + " GritLM/GritLM-7B\n", + " 0.626\n", + " 0.792\n", + " 0.966\n", + " 0.556\n", + " 0.598\n", + " 0.623\n", + " 0.632\n", + " 0.674\n", + " 0.863\n", + " ...\n", + " 0.438\n", + " 0.559\n", + " NaN\n", + " 0.743\n", + " 0.278\n", + " 0.688\n", + " 0.663\n", + " 0.573\n", + " 0.811\n", + " 0.874\n", + " \n", + " \n", " 2\n", " intfloat/multilingual-e5-large-instruct\n", - " 0.615\n", + " 0.611\n", " 0.681\n", " 0.962\n", " 0.508\n", @@ -376,9 +422,9 @@ " 0.644\n", " 0.875\n", " ...\n", - " 0.600\n", " 0.461\n", " 0.525\n", + " NaN\n", " 0.825\n", " 0.274\n", " 0.668\n", @@ -390,7 +436,7 @@ " \n", " 3\n", " intfloat/multilingual-e5-large\n", - " 0.574\n", + " 0.569\n", " 0.762\n", " 0.933\n", " 0.439\n", @@ -400,9 +446,9 @@ " 0.592\n", " 0.825\n", " ...\n", - " 0.464\n", " 0.385\n", " 0.501\n", + " 0.314\n", " 0.712\n", " 0.231\n", " 0.660\n", @@ -414,7 +460,7 @@ " \n", " 4\n", " intfloat/multilingual-e5-base\n", - " 0.557\n", + " 0.556\n", " 0.743\n", " 0.918\n", " 0.425\n", @@ -424,9 +470,9 @@ " 0.593\n", " 0.851\n", " ...\n", - " 0.446\n", " 0.389\n", " 0.497\n", + " NaN\n", " 0.695\n", " 0.215\n", " 0.643\n", @@ -437,32 +483,8 @@ " \n", " \n", " 5\n", - " sentence-transformers/all-mpnet-base-v2\n", - " 0.541\n", - " 0.622\n", - " 0.671\n", - " 0.268\n", - " 0.615\n", - " 0.565\n", - " 0.465\n", - " 0.659\n", - " 0.804\n", - " ...\n", - " 0.448\n", - " 0.403\n", - " 0.520\n", - " 0.513\n", - " 0.199\n", - " 0.611\n", - " 0.550\n", - " 0.501\n", - " 0.739\n", - " 0.851\n", - " \n", - " \n", - " 6\n", " intfloat/multilingual-e5-small\n", - " 0.537\n", + " 0.536\n", " 0.695\n", " 0.886\n", " 0.408\n", @@ -472,9 +494,9 @@ " 0.564\n", " 0.825\n", " ...\n", - " 0.434\n", " 0.376\n", " 0.470\n", + " NaN\n", " 0.726\n", " 0.212\n", " 0.636\n", @@ -484,9 +506,33 @@ " 0.850\n", " \n", " \n", + " 6\n", + " sentence-transformers/all-mpnet-base-v2\n", + " 0.531\n", + " 0.622\n", + " 0.671\n", + " 0.268\n", + " 0.615\n", + " 0.565\n", + " 0.465\n", + " 0.659\n", + " 0.804\n", + " ...\n", + " 0.403\n", + " 0.520\n", + " NaN\n", + " 0.513\n", + " 0.199\n", + " 0.611\n", + " 0.550\n", + " 0.501\n", + " 0.739\n", + " 0.851\n", + " \n", + " \n", " 7\n", " sentence-transformers/paraphrase-multilingual-...\n", - " 0.527\n", + " 0.517\n", " 0.729\n", " 0.764\n", " 0.386\n", @@ -496,9 +542,9 @@ " 0.602\n", " 0.763\n", " ...\n", - " 0.462\n", " 0.382\n", " 0.468\n", + " NaN\n", " 0.379\n", " 0.174\n", " 0.656\n", @@ -510,7 +556,7 @@ " \n", " 8\n", " sentence-transformers/all-MiniLM-L12-v2\n", - " 0.523\n", + " 0.516\n", " 0.624\n", " 0.630\n", " 0.264\n", @@ -520,9 +566,9 @@ " 0.641\n", " 0.836\n", " ...\n", - " 0.447\n", " 0.389\n", " 0.515\n", + " NaN\n", " 0.508\n", " 0.172\n", " 0.633\n", @@ -534,7 +580,7 @@ " \n", " 9\n", " sentence-transformers/all-MiniLM-L6-v2\n", - " 0.520\n", + " 0.512\n", " 0.620\n", " 0.643\n", " 0.265\n", @@ -544,9 +590,9 @@ " 0.635\n", " 0.816\n", " ...\n", - " 0.452\n", " 0.403\n", " 0.508\n", + " NaN\n", " 0.472\n", " 0.169\n", " 0.621\n", @@ -558,7 +604,7 @@ " \n", " 10\n", " sentence-transformers/paraphrase-multilingual-...\n", - " 0.506\n", + " 0.497\n", " 0.683\n", " 0.692\n", " 0.354\n", @@ -568,9 +614,9 @@ " 0.605\n", " 0.742\n", " ...\n", - " 0.435\n", " 0.375\n", " 0.458\n", + " NaN\n", " 0.391\n", " 0.161\n", " 0.601\n", @@ -582,7 +628,7 @@ " \n", " 11\n", " sentence-transformers/LaBSE\n", - " 0.442\n", + " 0.426\n", " 0.754\n", " 0.689\n", " 0.378\n", @@ -592,9 +638,9 @@ " 0.527\n", " 0.787\n", " ...\n", - " 0.302\n", " 0.353\n", " 0.424\n", + " NaN\n", " 0.163\n", " 0.049\n", " 0.632\n", @@ -605,34 +651,34 @@ " \n", " \n", "\n", - "

12 rows × 61 columns

\n", + "

12 rows × 66 columns

\n", "" ], "text/plain": [ " model Average \\\n", "Rank \n", - "0 GritLM/GritLM-7B 0.633 \n", - "1 intfloat/e5-mistral-7b-instruct 0.629 \n", - "2 intfloat/multilingual-e5-large-instruct 0.615 \n", - "3 intfloat/multilingual-e5-large 0.574 \n", - "4 intfloat/multilingual-e5-base 0.557 \n", - "5 sentence-transformers/all-mpnet-base-v2 0.541 \n", - "6 intfloat/multilingual-e5-small 0.537 \n", - "7 sentence-transformers/paraphrase-multilingual-... 0.527 \n", - "8 sentence-transformers/all-MiniLM-L12-v2 0.523 \n", - "9 sentence-transformers/all-MiniLM-L6-v2 0.520 \n", - "10 sentence-transformers/paraphrase-multilingual-... 0.506 \n", - "11 sentence-transformers/LaBSE 0.442 \n", + "0 intfloat/e5-mistral-7b-instruct 0.626 \n", + "1 GritLM/GritLM-7B 0.626 \n", + "2 intfloat/multilingual-e5-large-instruct 0.611 \n", + "3 intfloat/multilingual-e5-large 0.569 \n", + "4 intfloat/multilingual-e5-base 0.556 \n", + "5 intfloat/multilingual-e5-small 0.536 \n", + "6 sentence-transformers/all-mpnet-base-v2 0.531 \n", + "7 sentence-transformers/paraphrase-multilingual-... 0.517 \n", + "8 sentence-transformers/all-MiniLM-L12-v2 0.516 \n", + "9 sentence-transformers/all-MiniLM-L6-v2 0.512 \n", + "10 sentence-transformers/paraphrase-multilingual-... 0.497 \n", + "11 sentence-transformers/LaBSE 0.426 \n", "\n", " AmazonCounterfactualClassification AmazonPolarityClassification \\\n", "Rank \n", - "0 0.792 0.966 \n", - "1 0.732 0.963 \n", + "0 0.732 0.963 \n", + "1 0.792 0.966 \n", "2 0.681 0.962 \n", "3 0.762 0.933 \n", "4 0.743 0.918 \n", - "5 0.622 0.671 \n", - "6 0.695 0.886 \n", + "5 0.695 0.886 \n", + "6 0.622 0.671 \n", "7 0.729 0.764 \n", "8 0.624 0.630 \n", "9 0.620 0.643 \n", @@ -641,13 +687,13 @@ "\n", " AmazonReviewsClassification ArXivHierarchicalClusteringP2P \\\n", "Rank \n", - "0 0.556 0.598 \n", - "1 0.520 0.653 \n", + "0 0.520 0.653 \n", + "1 0.556 0.598 \n", "2 0.508 0.625 \n", "3 0.439 0.556 \n", "4 0.425 0.567 \n", - "5 0.268 0.615 \n", - "6 0.408 0.543 \n", + "5 0.408 0.543 \n", + "6 0.268 0.615 \n", "7 0.386 0.553 \n", "8 0.264 0.574 \n", "9 0.265 0.591 \n", @@ -656,58 +702,58 @@ "\n", " ArXivHierarchicalClusteringS2S ArguAna AskUbuntuDupQuestions BIOSSES \\\n", "Rank \n", - "0 0.623 0.632 0.674 0.863 \n", - "1 0.613 0.617 0.670 0.855 \n", + "0 0.613 0.617 0.670 0.855 \n", + "1 0.623 0.632 0.674 0.863 \n", "2 0.613 0.585 0.644 0.875 \n", "3 0.562 0.544 0.592 0.825 \n", "4 0.561 0.442 0.593 0.851 \n", - "5 0.565 0.465 0.659 0.804 \n", - "6 0.542 0.391 0.564 0.825 \n", + "5 0.542 0.391 0.564 0.825 \n", + "6 0.565 0.465 0.659 0.804 \n", "7 0.552 0.489 0.602 0.763 \n", "8 0.551 0.471 0.641 0.836 \n", "9 0.545 0.502 0.635 0.816 \n", "10 0.522 0.449 0.605 0.742 \n", "11 0.500 0.342 0.527 0.787 \n", "\n", - " ... StackExchangeClustering.v2 StackExchangeClusteringP2P.v2 \\\n", - "Rank ... \n", - "0 ... 0.681 0.438 \n", - "1 ... 0.643 0.481 \n", - "2 ... 0.600 0.461 \n", - "3 ... 0.464 0.385 \n", - "4 ... 0.446 0.389 \n", - "5 ... 0.448 0.403 \n", - "6 ... 0.434 0.376 \n", - "7 ... 0.462 0.382 \n", - "8 ... 0.447 0.389 \n", - "9 ... 0.452 0.403 \n", - "10 ... 0.435 0.375 \n", - "11 ... 0.302 0.353 \n", + " ... StackExchangeClusteringP2P.v2 StackOverflowDupQuestions \\\n", + "Rank ... \n", + "0 ... 0.481 0.549 \n", + "1 ... 0.438 0.559 \n", + "2 ... 0.461 0.525 \n", + "3 ... 0.385 0.501 \n", + "4 ... 0.389 0.497 \n", + "5 ... 0.376 0.470 \n", + "6 ... 0.403 0.520 \n", + "7 ... 0.382 0.468 \n", + "8 ... 0.389 0.515 \n", + "9 ... 0.403 0.508 \n", + "10 ... 0.375 0.458 \n", + "11 ... 0.353 0.424 \n", "\n", - " StackOverflowDupQuestions TRECCOVID Touche2020 \\\n", - "Rank \n", - "0 0.559 0.743 0.278 \n", - "1 0.549 0.870 0.263 \n", - "2 0.525 0.825 0.274 \n", - "3 0.501 0.712 0.231 \n", - "4 0.497 0.695 0.215 \n", - "5 0.520 0.513 0.199 \n", - "6 0.470 0.726 0.212 \n", - "7 0.468 0.379 0.174 \n", - "8 0.515 0.508 0.172 \n", - "9 0.508 0.472 0.169 \n", - "10 0.458 0.391 0.161 \n", - "11 0.424 0.163 0.049 \n", + " SummEvalSummarization.v2 TRECCOVID Touche2020 \\\n", + "Rank \n", + "0 NaN 0.870 0.263 \n", + "1 NaN 0.743 0.278 \n", + "2 NaN 0.825 0.274 \n", + "3 0.314 0.712 0.231 \n", + "4 NaN 0.695 0.215 \n", + "5 NaN 0.726 0.212 \n", + "6 NaN 0.513 0.199 \n", + "7 NaN 0.379 0.174 \n", + "8 NaN 0.508 0.172 \n", + "9 NaN 0.472 0.169 \n", + "10 NaN 0.391 0.161 \n", + "11 NaN 0.163 0.049 \n", "\n", " ToxicConversationsClassification \\\n", "Rank \n", - "0 0.688 \n", - "1 0.717 \n", + "0 0.717 \n", + "1 0.688 \n", "2 0.668 \n", "3 0.660 \n", "4 0.643 \n", - "5 0.611 \n", - "6 0.636 \n", + "5 0.636 \n", + "6 0.611 \n", "7 0.656 \n", "8 0.633 \n", "9 0.621 \n", @@ -716,13 +762,13 @@ "\n", " TweetSentimentExtractionClassification TwentyNewsgroupsClustering.v2 \\\n", "Rank \n", - "0 0.663 0.573 \n", - "1 0.649 0.533 \n", + "0 0.649 0.533 \n", + "1 0.663 0.573 \n", "2 0.592 0.507 \n", "3 0.628 0.392 \n", "4 0.628 0.358 \n", - "5 0.550 0.501 \n", - "6 0.628 0.345 \n", + "5 0.628 0.345 \n", + "6 0.550 0.501 \n", "7 0.590 0.452 \n", "8 0.542 0.470 \n", "9 0.540 0.460 \n", @@ -731,23 +777,23 @@ "\n", " TwitterSemEval2015 TwitterURLCorpus \n", "Rank \n", - "0 0.811 0.874 \n", - "1 0.816 0.878 \n", + "0 0.816 0.878 \n", + "1 0.811 0.874 \n", "2 0.798 0.867 \n", "3 0.753 0.858 \n", "4 0.722 0.855 \n", - "5 0.739 0.851 \n", - "6 0.708 0.850 \n", + "5 0.708 0.850 \n", + "6 0.739 0.851 \n", "7 0.688 0.853 \n", "8 0.700 0.848 \n", "9 0.679 0.847 \n", "10 0.651 0.838 \n", "11 0.628 0.846 \n", "\n", - "[12 rows x 61 columns]" + "[12 rows x 66 columns]" ] }, - "execution_count": 38, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -767,6 +813,53 @@ "_results_df.round(3)" ] }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index(['model', 'Average', 'AmazonCounterfactualClassification',\n", + " 'AmazonPolarityClassification', 'AmazonReviewsClassification',\n", + " 'ArXivHierarchicalClusteringP2P', 'ArXivHierarchicalClusteringS2S',\n", + " 'ArguAna', 'AskUbuntuDupQuestions', 'BIOSSES',\n", + " 'Banking77Classification', 'BiorxivClusteringP2P.v2',\n", + " 'BiorxivClusteringS2S.v2', 'CQADupstackAndroidRetrieval',\n", + " 'CQADupstackEnglishRetrieval', 'CQADupstackGamingRetrieval',\n", + " 'CQADupstackGisRetrieval', 'CQADupstackMathematicaRetrieval',\n", + " 'CQADupstackPhysicsRetrieval', 'CQADupstackProgrammersRetrieval',\n", + " 'CQADupstackStatsRetrieval', 'CQADupstackTexRetrieval',\n", + " 'CQADupstackUnixRetrieval', 'CQADupstackWebmastersRetrieval',\n", + " 'CQADupstackWordpressRetrieval', 'ClimateFEVERHardNegatives',\n", + " 'DBPediaHardNegatives', 'EmotionClassification', 'FEVERHardNegatives',\n", + " 'FiQA2018', 'HotpotQAHardNegatives', 'ImdbClassification',\n", + " 'MTOPDomainClassification', 'MTOPIntentClassification',\n", + " 'MassiveIntentClassification', 'MassiveScenarioClassification',\n", + " 'MedrxivClusteringP2P.v2', 'MedrxivClusteringS2S.v2',\n", + " 'MindSmallReranking', 'NFCorpus', 'RedditClustering.v2',\n", + " 'RedditClusteringP2P.v2', 'SCIDOCS', 'SICK-R', 'STS12', 'STS13',\n", + " 'STS14', 'STS15', 'STS16', 'STS17', 'STS22.v2', 'STSBenchmark',\n", + " 'SciDocsRR', 'SciFact', 'SprintDuplicateQuestions',\n", + " 'StackExchangeClustering.v2', 'StackExchangeClusteringP2P.v2',\n", + " 'StackOverflowDupQuestions', 'SummEvalSummarization.v2', 'TRECCOVID',\n", + " 'Touche2020', 'ToxicConversationsClassification',\n", + " 'TweetSentimentExtractionClassification',\n", + " 'TwentyNewsgroupsClustering.v2', 'TwitterSemEval2015',\n", + " 'TwitterURLCorpus'],\n", + " dtype='object')" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "_results_df.columns" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -776,7 +869,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -787,7 +880,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -813,7 +906,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -850,23 +943,23 @@ " GritLM/GritLM-7B\n", " 66.76\n", " 1.0\n", - " 63.256300\n", - " 1.0\n", + " 62.576234\n", + " 2.0\n", " \n", " \n", " 1\n", " intfloat/e5-mistral-7b-instruct\n", " 66.63\n", " 2.0\n", - " 62.881090\n", - " 2.0\n", + " 62.623036\n", + " 1.0\n", " \n", " \n", " 2\n", " intfloat/multilingual-e5-large-instruct\n", " 64.41\n", " 3.0\n", - " 61.453870\n", + " 61.127328\n", " 3.0\n", " \n", " \n", @@ -874,7 +967,7 @@ " intfloat/multilingual-e5-large\n", " 60.89\n", " 4.0\n", - " 57.440855\n", + " 56.925622\n", " 4.0\n", " \n", " \n", @@ -882,7 +975,7 @@ " intfloat/multilingual-e5-base\n", " 59.11\n", " 5.0\n", - " 55.690921\n", + " 55.578989\n", " 5.0\n", " \n", " \n", @@ -890,23 +983,23 @@ " sentence-transformers/all-mpnet-base-v2\n", " 57.78\n", " 6.0\n", - " 54.122250\n", - " 6.0\n", + " 53.148536\n", + " 7.0\n", " \n", " \n", " 6\n", " intfloat/multilingual-e5-small\n", " 57.04\n", " 7.0\n", - " 53.699672\n", - " 7.0\n", + " 53.556709\n", + " 6.0\n", " \n", " \n", " 7\n", " sentence-transformers/paraphrase-multilingual-...\n", " 54.64\n", " 10.0\n", - " 52.654694\n", + " 51.724491\n", " 8.0\n", " \n", " \n", @@ -914,7 +1007,7 @@ " sentence-transformers/all-MiniLM-L12-v2\n", " 56.53\n", " 8.0\n", - " 52.315120\n", + " 51.633906\n", " 9.0\n", " \n", " \n", @@ -922,7 +1015,7 @@ " sentence-transformers/all-MiniLM-L6-v2\n", " 56.10\n", " 9.0\n", - " 51.958840\n", + " 51.215533\n", " 10.0\n", " \n", " \n", @@ -930,7 +1023,7 @@ " sentence-transformers/paraphrase-multilingual-...\n", " 52.45\n", " 11.0\n", - " 50.629107\n", + " 49.685354\n", " 11.0\n", " \n", " \n", @@ -938,7 +1031,7 @@ " sentence-transformers/LaBSE\n", " 45.21\n", " 12.0\n", - " 44.235938\n", + " 42.554815\n", " 12.0\n", " \n", " \n", @@ -961,21 +1054,21 @@ "11 sentence-transformers/LaBSE 45.21 12.0 \n", "\n", " Average_v2 Rank_v2 \n", - "0 63.256300 1.0 \n", - "1 62.881090 2.0 \n", - "2 61.453870 3.0 \n", - "3 57.440855 4.0 \n", - "4 55.690921 5.0 \n", - "5 54.122250 6.0 \n", - "6 53.699672 7.0 \n", - "7 52.654694 8.0 \n", - "8 52.315120 9.0 \n", - "9 51.958840 10.0 \n", - "10 50.629107 11.0 \n", - "11 44.235938 12.0 " + "0 62.576234 2.0 \n", + "1 62.623036 1.0 \n", + "2 61.127328 3.0 \n", + "3 56.925622 4.0 \n", + "4 55.578989 5.0 \n", + "5 53.148536 7.0 \n", + "6 53.556709 6.0 \n", + "7 51.724491 8.0 \n", + "8 51.633906 9.0 \n", + "9 51.215533 10.0 \n", + "10 49.685354 11.0 \n", + "11 42.554815 12.0 " ] }, - "execution_count": 41, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -991,22 +1084,22 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Text(0.5, 1.0, 'Pearson correlation: 0.99, p-value: 0.0000\\nSpearman correlation: 0.98, p-value: 0.0000')" + "Text(0.5, 1.0, 'Pearson correlation: 0.99, p-value: 0.0000\\nSpearman correlation: 0.97, p-value: 0.0000')" ] }, - "execution_count": 42, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA/gAAANrCAYAAADyKTN6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd1gUV9sG8Ht32V1gaVYw2EWQDopYUFFiIdZYoybYe40VE43GVGNXQOwaDclr7Ik1iYm9d40YFRtgF+mwy+7O9wffjq50pSjev+vyCpw5M3NmdiA8c855jkQQBAFERERERERE9FaTlnQDiIiIiIiIiOj1McAnIiIiIiIiKgUY4BMRERERERGVAgzwiYiIiIiIiEoBBvhEREREREREpQADfCIiIiIiIqJSgAE+ERERERERUSnAAJ+IiIiIiIioFGCAT0RExUoQhJJuAhEREVGpxACfiN5IQUFBcHJyMvrn5uaG5s2bY+bMmUhISCjpJr7TtmzZAicnJ8TExOR7n8TEREyePBmnT58Wy4KCghAUFFQUTcwXrVaLhQsXwt/fH56enujduzcuXLiQ535RUVEYNmwYvL294evrizFjxuD27dtGdTQaDebNmwd/f394eHigc+fO2LlzZxFdSdGJiYmBk5MTtmzZUmJtuHPnDoYNGwYfHx80aNAAM2bMQHJycp77Xbp0CUFBQfD29kaTJk0wf/58aDQaozpPnjzBhAkT0KBBA9SrVw/jx4/Ho0ePjOrk9znZsWMH2rVrBw8PD3zwwQfYunXr6104ERFRAZmUdAOIiHLi4uKCGTNmiN9nZGTg33//xfz58xEZGYlffvkFEomkBFtIBREZGYnt27eja9euYtmLn29JmDVrFjZt2oQJEybA3t4ea9asQb9+/bBt2zZUq1Yt232io6PRq1cvWFpaYvr06ShXrhw2bdqEjz76CJs3b0blypUBAOPGjcP+/fsxYMAANGrUCJcvX8bUqVMRFxdXoi813jaJiYno27cvypcvj1mzZiEuLg5z5sxBTEwMVq1aleN+0dHR6N+/P7y8vLBw4UJERUVhwYIFiI+Px1dffQUgM3AfPHgwkpOT8eWXX0Kr1WLevHkYOHAgtmzZArlcDiB/z8nevXsxceJE9OnTB02bNsVff/2FKVOmQKFQoF27dkV/o4iIiABAICJ6A33yySfCJ598ku220NBQwdHRUTh37lzxNopEmzdvFhwdHYXo6Oh873P8+HHB0dFROH78eBG2LP/u3bsnuLi4CBEREWKZWq0WmjdvLkydOjXH/b7++mvBzc1NuHv3rlim0+mErl27CuPHjxcEQRD+/fdfwdHRUViyZInRvuvXrxe8vLyEhISEQr6aohMdHS04OjoKmzdvLpHzL126VPD09BSePn0qlu3fv19wdHQUTp8+neN+X3zxhdCsWTNBrVaLZREREUKdOnWE2NhYQRAE4ffffxccHR2F69evi3WuX78uODk5Cdu3bxcEIf/PSevWrYWxY8catWHs2LFCq1atXu3CiYiIXgGH6BPRW8fNzQ0AcO/ePbHsr7/+QpcuXeDu7g4/Pz988803SE1NNdrvr7/+Qu/eveHt7Q03NzcEBgYiIiJC3H7ixAk4OTnhf//7H1q0aIG6deviyJEjiIuLw4QJE+Dn5wd3d3d06tQJ27ZtMzr27du3MWbMGPj5+cHLywtBQUE4c+aMuN0wzHn37t0YM2aMOLR72rRpWdr5skePHiE4OBiNGjWCt7c3PvnkE5w7d07crlarERYWhsDAQLi7u6N169ZYvnw59Hq9WCcoKAgTJ07EmDFj4OXlhf79+4ttWrNmDQIDA+Hp6YnNmzcDAK5du4ahQ4eibt26qFu3LkaOHIno6Ohc27lx40Z06dIFXl5e8PDwQKdOnbB7927x3vbp0wcA0KdPH7EH++Uh+vm9lqlTp2L58uVo3rw53N3d0bNnT1y8eDHL/Q4JCcmxvceOHYNWq0WrVq3EMoVCgebNm+PAgQM57nfz5k04ODigSpUqYplUKkX9+vXF/aKiogAALVq0MNq3QYMGSE1NxcmTJ3M8/ssM0yEuXLiAzp07w8PDAx06dMCePXty3e/333+Hk5MTrl27ZlT+119/wcnJCVeuXAEAXL16FaNGjULDhg3h6uqKpk2b4ptvvkF6enq2xw0JCYGTk1OW8pfvt1qtxuzZs+Hv7w83Nzd06NABu3btyvZYuU31OHz4MOrVq4eyZcuKZU2aNIFKpcLBgwdz3c/f3x8KhUIsCwwMhF6vx+HDh8U6NWrUgIODg1jHwcEBtWrVEj/L/DwnMTExuH37tlEdAGjTpg3u3LmTZfoGERFRUWGAT0RvnVu3bgGAGGD9/vvvGDlyJGrWrImwsDCMGjUKv/32G0aMGCEmdNu/fz9GjhwJV1dXLFmyBCEhIahSpQq++uqrLHNpQ0NDERwcjOnTp8Pb2xuTJk1CVFQUZs6ciRUrVsDFxQXBwcE4fvw4AODGjRvo0qULYmJiMG3aNMydOxcSiQR9+/bNEsjNmDED9vb2WLJkCQYOHIhNmzYhPDw8x2tNSUlBr169cOLECUyaNAmhoaFQKpUYMGAAbt++DUEQMGzYMKxcuRLdu3fH0qVLERgYiIULF2YZ/r57926oVCqEh4dj0KBBYnlISAgGDx6M2bNnw8/PD7du3ULPnj3x9OlT/PDDD/j222/FYelPnz7Ntp0RERGYPn06WrZsiWXLlmHu3LlQKBSYOHEiHjx4AFdXV0yfPh0AMH369GyH5hfkWvbu3Yt9+/Zh2rRpmD9/Pp48eYLRo0dDp9MBACpWrIgNGzage/fuOd7bqKgoqFQqVKhQwai8WrVqePToEVJSUrLdr0yZMnj8+DEyMjKMyqOjo5GUlIT4+HiUKVMGgPFLKAC4e/euWLeghg4divfffx+hoaGoUaMGPv3001xfRLRs2RLm5uZZ5v3v2LEDtWvXhouLCx49eoSPP/4YaWlpmDVrFlasWIF27dph/fr1WLduXYHbaCAIAkaOHIn//e9/6N+/P8LDw+Ht7Y1x48YZvRzr3r07NmzYgIoVK+Z4rKioKNSoUcOoTCaToXLlyuLvgpelp6cjNjY2y35ly5aFhYWFuF9UVBSqV6+eZf+qVasa1cnrOTG80Hn5WIbh+zm1k4iIqLBxDj4RvbEEQYBWqxW/T0hIwMmTJ8Vgwc3NDYIgYO7cuWjatCnmzp0r1q1evTr69euHAwcOoHnz5rhx4wY6d+6MqVOninW8vb3RoEEDnDhxAp6enmJ57969ERgYKH5/8uRJjBw5Ei1btgQA+Pr6wsbGRuwZDA0NhUKhwLp162BhYQEAaN68Odq3b4/Zs2dj06ZN4rH8/f0RHBwMAGjUqBGOHDmC/fv3Y8KECdneg61btyI2NhZbt26Fs7MzAKBu3br48MMPcerUKdy5cwdHjx7F/PnzxXm+fn5+MDU1xaJFi9CnTx/Url0bACCXyzFz5kyx3YZe0w8++MBoXvyECRNgZmaGtWvXitfTqFEjtGzZEitXrhTb/6Lo6GgMHDgQI0aMEMvs7e3RpUsXnDlzBu3atRN7SR0cHIx6TA0OHjyY72vRarVYtWqV2L6UlBQEBwcjMjISbm5uUCgU8PLyyvaeGiQlJYn7v0ilUgEAkpOTxa9f1KVLF+zYsQPBwcEYN24cLCwssH37dhw6dAgAkJaWBl9fX1SpUgXffPMNzMzM4O7ujqtXr4ovf/IatZGdoKAgjBw5EgDQtGlTdO7cGWFhYfD398+2vpmZGdq0aYNdu3Zh3LhxADLv0z///CMe59q1a3B2dsaiRYvEe9G4cWMcOXIEJ06cwJAhQwrcTgA4evQoDh06hAULFqBt27Zim9PS0jB37ly0b98eJiYmsLOzg52dXa7HSkpKyvZzUKlUOSbaS0pKAoAcP1/DfklJSdnmWlCpVOILnvw8J4bjvVzvxTpERETFgQE+Eb2xTp06BVdXV6MyqVSKxo0b46uvvoJEIkFUVBQePHiAoUOHGr0MqF+/PiwsLHDkyBE0b95c7LFOSUnBrVu3cPfuXVy6dAkAsmTVNgTSBg0aNEBISAiuXLmCpk2bGgXpQOYLgBYtWhj9cW9iYoJ27dohLCzMqCf45aDTzs4OsbGxOd6DM2fOoHLlykZtMjMzw969ewEAc+bMgYmJidELCQDo2LEjFi1ahJMnT4pBcc2aNY2GK+d0vcePH4evry9MTU3Fe2phYQEfHx8cPXo023ZOmTIFQGZCtJs3b+LOnTs4ceIEgKz3NycnT57M97U4ODgY3W9bW1sAmcF1fgl5LNcnlWY/yM3Pzw9z5szBd999J/aON27cGIMHD0ZISAhMTU2hUCiwatUqfP755+jXrx8AoEKFCpg2bRo+/fRTmJmZ5budBp07dxa/lkgkaNWqFUJCQpCeng65XG50PRKJBDKZDJ06dcLWrVtx8eJFeHh4YN++fdBoNOjYsSOAzKHuTZo0QUZGBm7cuIE7d+7g2rVriIuLg42NTYHbaHDs2DFIJBL4+/sb/VwGBATgt99+w/Xr17M8dznJ7XPKKcnmi1M6ctsvP8fOz3OS1/lyepaIiIgKGwN8Inpjubq6YubMmQAy/9hWKpWoVKmSUWAXHx8PAJg5c6ZY90WG5a7i4uIwY8YM/PXXX5BIJKhWrRp8fHwAZP0D3tzc3Oj7BQsWYOnSpdi9ezf27t1r9JLB3t4eCQkJKF++fJZzly9fHoIgGPXevRzYSaXSXAOI+Ph4lCtXLsftCQkJKFOmDGQymVG5YTixoScTQLa9oEDW642Pj8euXbuyzJcGYDQP+kV3797F9OnTcezYMcjlctSsWRN16tQBkP917wtyLdndRyDvwO5FFhYW2Q7DN3xelpaWOe7bsWNHtGvXDtHR0TAzM4OtrS0WLVoEqVQKKysrAJnDsyMiIvD06VPEx8ejWrVquH//PgRBgLW1db7bafDyMPZy5cpBEAQkJiZiwoQJRtNBfH19sX79ejRo0AC2trbYuXMnPDw8sHPnTvj6+oq95nq9HvPnz0dERARSU1NRqVIleHh4QKlUFrh9L4qPj4cgCKhbt2622x89epTvAD+3z8nwYie7fQDkuJ/hs83t2PmpA2Q+J4a6L9fLqWefiIioqDDAJ6I3lkqlgru7e651DMHU5MmT4evrm2W7IZCaOHEibt68ibVr18Lb2xsKhQJpaWn49ddf82yHpaUlJk2ahEmTJuHmzZvYt28flixZgpkzZ2L58uWwtrbGkydPsuz3+PFjAJlztl9eVzu/LC0ts01AdvbsWVhbW8Pa2hrPnj2DTqczCowN5zPMBS/oORs3boz+/ftn2WZikvV/G3q9HkOGDIFcLsemTZvg7OwMExMT3LhxA9u3b8/3eYviWnJTs2ZNJCcnIy4uzujFxZ07d2Bvbw9TU9Ns94uKisKlS5fw4YcfGs25vnLlCpycnCCTyZCeno69e/eibt26qFKliviS5t9//wWALCNT8iM+Pt7oRdKTJ08gk8lgY2ODmTNnGgWXhpc5UqkUHTp0wI4dOzBs2DAcOXJEXCIOAJYvX461a9di5syZaN26tRioduvWLcd2GHq2X/ycXg5sLS0tYW5unuM8/pyWIMxOjRo1xNwFBjqdDjExMWjdunW2+6hUKtja2uLOnTtG5U+fPkVKSgpq1aolHjsyMjLL/nfv3oWHhweA/D0nhrn+d+7cgYuLi1EdAOL5iIiIihrHjBHRW61mzZooV64cYmJi4O7uLv6ztbXFvHnzxEzhZ86cQevWrdGgQQNxmLohA3duvb6xsbHw9/cXM5bXrFkTgwcPRuPGjcUEavXr18c///xj1FOv0+mwc+dOuLu7ZzssPr98fHwQHR2N69evi2VqtRqjR4/Gpk2b4OvrC61WmyWj+m+//QYAqFevXoHP6evrixs3bsDZ2Vm8n25ubli7di3+/PPPLPWfPXuGW7duoVu3bnB3dxdfArx8f1/umc/uvIV9Lblp3LgxABidT6PRYP/+/fDz88txv+vXryM4OBg3b94Uy27cuIHDhw+LeRrkcjm+/vproxdIWq0WP/30E6pWrQpHR8cCt/evv/4SvxYEAX/88Qfq1asHhUKBmjVrGj3/NWvWFOt26tQJDx48QFhYGGQymVFQfObMGTg4OKBr165icP/w4UNcu3Ytx58LQ2/0gwcPjI7zIl9fX6SmpkIQBKN2Xbt2DWFhYUbD9vPi5+eHU6dOIS4uTiw7fPgwUlNTc/2c/Pz8sH//fqMpInv37oVMJkPDhg0BZE5RiIqKwo0bN8Q6N27cQFRUlHjs/Dwn1apVQ+XKlcWpMwZ//PEHqlevjsqVK+f7eomIiF4He/CJ6K0mk8kwbtw4TJ8+HTKZDC1atEBiYiKWLFmChw8fij2lHh4e+P333+Hq6go7OzucPXsWy5cvh0QiyXXetr29Pezs7PDNN98gOTkZVatWxeXLl3HgwAEMHToUADBq1CgcPHgQffr0EXuyf/rpJ0RHR2PlypWvdX1dunTB+vXrMXz4cIwZMwZlypTBunXrkJGRgd69e6Ny5cpo0KABpk2bhocPH6JOnTo4efIkVqxYgc6dO2ebzC4vI0aMQM+ePTF06FD06tULSqUSGzZswF9//YXFixdnqV+uXDnY29sjIiICdnZ2sLKywqFDh8TeW8P9NQSQ+/fvh7W1tTiE36BZs2aFdi0ajQZXrlzJNYmbvb09OnfujO+//x5qtRrVq1fHmjVrkJiYaLTKwN27dxEXFyfmT/D390fVqlUxceJEjB07FsnJyZg9ezYqV64szreXyWTo3bs3fvzxR9jZ2aFGjRqIiIjA2bNnERYWJk4pSE5Oxo0bN1C1atUcpz8YzJ49G2q1GjVq1MDGjRsRFRWFH3/8Mc974ejoCGdnZ/z888/44IMPjIaLe3h4YMmSJVi+fDm8vLxw584dLFu2DBqNJsefC39/f3z//feYPn06Bg4ciPv37yMsLMxoCoi/vz/q16+PESNGYMSIEahVqxYuXryIxYsXo2nTpuK1PnjwAA8ePICLi0uOL8J69+6Nn376Cf3798eoUaMQHx+POXPmoFmzZkZTAM6fP4+yZcuiatWqAIBBgwZh586dGDRoEPr374/bt29j/vz56NGjB9577z0AQNu2bbF06VIMHjxYTHQ5b948ODo64oMPPgCQ/+dk5MiR+Oyzz2BjY4OAgADs27cPu3fvxoIFC/L8jIiIiAqNQET0Bvrkk0+ETz75JN/1d+7cKXTu3Flwc3MTfH19hWHDhglXr14Vt8fExAhDhw4V6tWrJ9SrV0/o2rWrsH37dmHgwIFC165dBUEQhOPHjwuOjo7C8ePHjY796NEjYcqUKUKTJk0EV1dXoWXLlkJ4eLig0+nEOleuXBEGDRokeHl5Cd7e3kLfvn2FU6dOidujo6MFR0dHYfPmzUbHDg4OFlq0aJHrtT148EAYP3684OPjI9StW1cYMGCAEBkZKW5PTU0VZs2aJTRt2lRwdXUV2rRpI6xcudKofdndz5zaJAiCcPnyZWHgwIGCt7e34OXlJfTo0UP466+/xO2bN28WHB0dhejoaEEQBCEyMlL45JNPBC8vL8HX11fo3bu3cPDgQSEwMFAYM2aMIAiCoNPphPHjxwvu7u5Cu3btsm3Xq17Ly5+d4doWL16c671Vq9XCt99+KzRq1Ejw9PQUevfuLZw/f96oTnBwsODo6GhUdvv2bWHIkCFCvXr1hMaNGwtTpkwRHj58aFRHo9EI8+fPF/z9/QUvLy+hZ8+ewqFDh7Jtd3afgYHhXm/fvl1o27at4O7uLnz00UdZntPcrF69WnB0dBT279+f5fpnzpwp+Pn5CR4eHkKbNm2ExYsXCyEhIYKbm5uQkJCQ7XOydetWoXXr1oKrq6vQsWNH4fDhw+K+BikpKcJ3330nNGvWTHB1dRUCAgKEefPmCenp6WKdxYsXGz1HOfnvv/+Evn37Ch4eHkKjRo2EL774QkhKSjKq4+joKAQHBxuVnTp1Sujevbvg5uYmNG3aVJg7d66g0WiM6ty7d08YOXKk4OXlJdSvX1/49NNPs3yW+XlOBEEQfvnlF6FVq1aCm5ub8MEHHwhbt27N9bqIiIgKm0QQ8pn9iIiIiArdokWL4ODgIC4N+LItW7bgs88+w759+zjUm4iIiHLFOfhEREQl5OHDh9i7dy+8vb1LuilERERUCjDAJyIiKiE2NjYICQkR54QTERERvQ4O0SciIiIiIiIqBdiDT0RERERERFQKMMAnIqI3BgeVEREREb06BvhEVGKuXbuGcePGwc/PD25ubmjSpAk+/fRTXL16taSbRq9hy5YtcHJyQkxMTL73SUxMxOTJk3H69GmxLCgoCEFBQUXRxHzRarVYuHAh/P394enpid69e+PChQt57hcVFYVhw4bB29sbvr6+GDNmDG7fvp3l2MuXL0fr1q3h5eWFTp06YdeuXUV0JUXnxIkTcHJywokTJ0qsDZcuXUJQUBC8vb3RpEkTzJ8/HxqNJs/9Dh8+jK5du8LT0xMBAQFYtWpVlhdMd+7cwbBhw+Dj44MGDRpgxowZSE5ONqqTkpKCmTNnws/PD97e3hg8eDBu3ryZ5Xw//vgjWrVqBQ8PD3Tu3BkHDhx4vQsnIiLKBgN8IioR169fx0cffYT4+HhMmzYNq1evxuTJk3Hv3j306NED58+fL+kmUjGKjIzE9u3bodfrxbIZM2ZgxowZJdamWbNmYe3atRg0aBAWLFgAmUyGfv364c6dOznuEx0djV69euH69euYPn065s6dCwD46KOPjF54hISEYMGCBejYsSPCw8NRr149jBs3Dnv37i3y6ypNoqOj0b9/fyiVSixcuBADBgzAmjVr8M033+S63/nz5zFs2DDUrFkTISEh6NChA+bMmYMVK1aIdRITE9G3b188efIEs2bNwoQJE7Br1y6MHTvW6FgTJkzAnj17MGHCBPzwww94+PAh+vTpg4SEBLHOmjVr8MMPP+DDDz9ESEgIqlSpguHDhxu90CIiIioUAhFRCfjss8+EFi1aCBkZGUblKSkpQtOmTYXBgweXUMvodW3evFlwdHQUoqOj873P8ePHBUdHR+H48eNF2LL8u3fvnuDi4iJERESIZWq1WmjevLkwderUHPf7+uuvBTc3N+Hu3btimU6nE7p27SqMHz9eLPPz8xMmTpxotG+PHj2ETz75pBCvouiV9Of2xRdfCM2aNRPUarVYFhERIdSpU0eIjY3Ncb8BAwYI3bp1MyqbPXu24O3tLaSlpQmCIAhLly4VPD09hadPn4p19u/fLzg6OgqnT58WBEEQzp49Kzg6Ogr79+8X6zx9+lTw8vISlixZIgiCIKSlpQk+Pj7C7NmzxTp6vV7o0aOH0K9fv9e4eiIioqzYg09EJeLJkycQBMGoxxYAzM3N8fnnn+ODDz4Qy4KCgjBlyhQsXboUjRs3Rr169TBixAjExsYa7Xvt2jUMHToUdevWRd26dTFy5EhER0cb1bl69SpGjRqFhg0bwtXVFU2bNsU333yD9PR0sY6TkxNCQ0PRpUsXeHh4IDQ0FFu2bIG7uztOnz6Nrl27wt3dHW3atMHff/+Nmzdvom/fvvD09ESrVq2wc+dOo3OeOnUKAwcORP369eHm5oaAgACEhISI1x4TEwMnJyfs3r0bY8aMEYd2T5s2Dampqbnex0ePHiE4OBiNGjWCt7c3PvnkE5w7d07crlarERYWhsDAQLi7u6N169ZYvny50X0PCgrCxIkTMWbMGHh5eaF///5im9asWYPAwEB4enpi8+bN+b7PL9u4cSO6dOkCLy8veHh4oFOnTti9ezeAzGHeffr0AQD06dNHHJb/8hD9/F7L1KlTsXz5cjRv3hzu7u7o2bMnLl68KNYxXFtISEiO7T127Bi0Wi1atWollikUCjRv3jzXodU3b96Eg4MDqlSpIpZJpVLUr1/faD+NRgMLCwujfW1sbBAfH5/jsbMTEhKCgIAA/PPPP+Ln1KNHjzyHzC9duhRubm5GvcwAsHbtWri6uuLp06cA8n52XzZlyhQEBAQYlRnu95YtW8Sy+Ph4TJ8+HY0bN4a7uzt69OiBY8eOZTmWk5NTrtdx+PBh+Pv7Q6FQiGWBgYHQ6/U4fPhwtvtoNBqcOHHC6LMFgDZt2iAlJQVnzpwRj12vXj2ULVtWrNOkSROoVCocPHhQrGNubo4mTZqIdcqWLWv0eV+4cAGJiYlG55NIJGjVqhVOnDhh9LuHiIjodTHAJ6IS0bx5c9y7dw89e/ZEREQEoqKixPmvgYGB6Ny5s1H9ffv2YcuWLZg2bRpmzpyJyMhIBAUFIS0tDQBw69Yt9OzZE0+fPsUPP/yAb7/9VhwubQhWHj16hI8//hhpaWmYNWsWVqxYgXbt2mH9+vVYt26d0fmWLl2KDh06YPHixWjTpg2AzHnTEyZMQM+ePREeHg4zMzNMnDgRw4YNQ/PmzbF06VJUrFgRwcHBePDgAYDMFwr9+vWDjY0NFixYgPDwcPj4+CA0NFQMcA1mzJgBe3t7LFmyBAMHDsSmTZsQHh6e4z1MSUlBr169cOLECUyaNAmhoaFQKpUYMGAAbt++DUEQMGzYMKxcuRLdu3fH0qVLERgYiIULF2YZ+r57926oVCqEh4dj0KBBYnlISAgGDx6M2bNnw8/PL1/3+WURERGYPn06WrZsiWXLlmHu3LlQKBSYOHEiHjx4AFdXV0yfPh0AMH369GyH5RfkWvbu3Yt9+/Zh2rRpmD9/Pp48eYLRo0dDp9MBACpWrIgNGzage/fuOd7bqKgoqFQqVKhQwai8WrVqePToEVJSUrLdr0yZMnj8+DEyMjKMyqOjo5GUlCQG8H369MG2bdtw8OBBJCcn47fffsOhQ4fQqVOnHNuUk7i4OAQHB6N3795YtGgRTE1NMXDgQERGRua4T4cOHaDVavHHH38Yle/cuRNNmjRBuXLlCvTsFoRarUbfvn2xb98+jBs3DqGhobCzs8OgQYOMgvwRI0Zgw4YNOR4nPT0dsbGxqFGjhlF52bJlYWFhgVu3bmW7X3R0NDIyMlC9enWj8mrVqgGAuF9UVFSWY8tkMlSuXNmoTuXKlSGTyYzqVa1a1agOgGzPp9PpcPfu3RyvkYiIqKBMSroBRPRu6t27Nx4/foxVq1bhq6++ApAZHDVp0gR9+vSBh4eHUf20tDRs2bJF7BmtWbMmOnfujG3btqFXr14IDQ2FmZkZ1q5dK/aMNmrUCC1btsTKlSsRHByMa9euwdnZGYsWLRLrNG7cGEeOHMGJEycwZMgQ8Xw+Pj7o37+/+P2lS5eg1+sxbNgwMTBMTEzEuHHj0LdvX7GupaUlunbtisuXL8POzg5Xr15F48aNMWfOHEilme9U/fz88Pfff+PEiRNo166deA5/f38EBweLbT9y5Aj279+PCRMmZHsPt27ditjYWGzduhXOzs4AgLp16+LDDz/EqVOncOfOHRw9ehTz588Xz+Pn5wdTU1MsWrQIffr0Qe3atQEAcrkcM2fOFHtCDfPFP/jgA3Tt2lU854QJE/K8zy+Ljo7GwIEDMWLECLHM3t4eXbp0wZkzZ9CuXTs4ODgAABwcHMSvX3Tw4MF8X4tWq8WqVavE9qWkpCA4OBiRkZFwc3ODQqGAl5dXtvfUICkpKUsPOwCoVCoAQHJysvj1i7p06YIdO3YgODgY48aNg4WFBbZv345Dhw4ByHyObWxs0K9fP5w/fx6DBw8W9+3atavRy5X8SktLw5dffokPP/wQANCwYUO0bNkSy5cvx4IFC7Ldx97eHvXr18eOHTvE5/nu3bu4ePGiuE9Bnt2C2L59O65evYpff/0Vnp6eAIBmzZohKCgIc+fOFUeKVK1aFVWrVs3xOElJSQCQ4+f0cjK8vPZ78bM11MvuM37x2Lk9J4aXQIa6eZ2PiIioMLAHn4hKzNixY3Ho0CHMmzcP3bp1g4WFBX7//Xf06NEjS4963bp1jYY9u7i4oEqVKjh16hQA4Pjx4/D19YWpqSm0Wi20Wi0sLCzg4+ODo0ePAsgcXvvTTz9BqVTixo0b2LdvH8LDwxEXF5cl67YhYH6Zt7e3+HW5cuUAQAxSgMxh1kBm8A8AH374IVasWIGMjAxcvXoVe/fuxeLFi6HT6bL08r4cdNrZ2eU6RP/MmTOoXLmyUVvNzMywd+9edO/eHSdPnoSJiQkCAwON9uvYsSMA4OTJk2JZzZo1jYY5G7x8H/Jzn182ZcoUTJw4EYmJiTh//jy2b9+OiIgIAMhXtnNDW/N7LQ4ODkbBlK2tLQCIoz3yQ8hjuT5DwPsyPz8/zJkzB0ePHkXLli3RsGFDHDhwQAzkTU1NodFo8PHHH+PKlSuYOXMm1q9fj/Hjx2PXrl15JofLjomJCdq3by9+b2pqimbNmok/GzqdTvystFqtOLy+Y8eOOHXqFB4/fgwgs/fewsJCHGJfkGe3II4dO4YKFSrA1dVVbJNOp0OLFi1w+fLlLNMGcpLTNAEDiUTySvsZPtvcngHDsfNTJ7/nIyIiKgzswSeiEmVtbY327duLAcqVK1cwadIkzJkzBx06dECZMmUAPA/SXlSuXDkxGIiPj8euXbuyXWrMMIdWr9dj/vz5iIiIQGpqKipVqgQPDw8olcos+5ibm2fb3ux668zMzHK8vvT0dHz99dfYvn07tFotKleuDG9vb5iYmGQJDl4+jlQqzTWAiI+PF18yZCchIQFlypTJMnzYMOzc0JMJINueSiDrfcjPfX7Z3bt3MX36dBw7dgxyuRw1a9ZEnTp1AOR/3fuCXEt29xHIO9B6kYWFRbbD8A29rZaWljnu27FjR7Rr1w7R0dEwMzODra0tFi1aBKlUCisrK+zatQtXr17FmjVr0LhxYwCAr68vLCws8NVXX6FHjx5wdHTMd1vLly8PExPj/52XK1dOnA7QqlUro3wVnTt3xqxZsxAYGIivv/4au3fvRp8+fbBz5060adMGpqamAAr27BZEfHw8Hj9+DFdX12y3P378GNbW1nke58URGi9LTk7O8TMylL+838s97bk9A4bfRxYWFnjy5EmWOikpKeJ5Xjzfi9eVn2eJiIiooBjgE1Gxe/jwIbp27YqxY8dmmQft4uKCcePGiYnbDAH+s2fPshznyZMn4hBeS0tLNG7c2GhYvYEh+Fm+fDnWrl2LmTNnonXr1uIf1t26dSvU63vRt99+i71792LhwoVo3LixGDA3atTotY9taWmZ7VrzZ8+ehbW1NaytrfHs2TPodDqjwPjRo0cAIN7bgp4zr/v8Ir1ejyFDhkAul2PTpk1wdnaGiYkJbty4ge3bt+f7vEVxLbmpWbMmkpOTERcXZ/Ti4s6dO7C3txeD4JdFRUXh0qVL+PDDD43mXF+5cgVOTk6QyWS4d+8egMxRKS+qX78+AODGjRsFCvCzS8z35MkT8eVPeHi40UgJw72ytLREQEAAdu/ejYYNG+L69ev44osvxHqv8uxKJBIx14HBy6NQLC0tUb16dXEJwZdVrlw5l6t9TqVSwdbWNsuyhU+fPkVKSgpq1aqV7X5Vq1aFTCbLsp9hLrxhvxo1amSZH6/T6RATE4PWrVuLdQ4fPgy9Xm/UE3/nzh2j4xjKXpx6dOfOHcjlcqORSURERK+L48KIqNgZehx//vlnqNXqLNtv3rwJpVIpJr0CMoejvxjkX758GTExMWKw4evrixs3bsDZ2Rnu7u5wd3eHm5sb1q5diz///FM8hoODA7p27SoG9w8fPsS1a9cK1LtbEGfOnEGDBg3QsmVLMUC6fPky4uLiXvucPj4+iI6OxvXr18UytVqN0aNHY9OmTfD19YVWq8WePXuM9vvtt98AAPXq1SvwOfNzn1/07Nkz3Lp1C926dYO7u7v4EsCQhdxwD17umc/uvIV9Lbkx9Ky/eD6NRoP9+/fDz88vx/2uX7+O4OBg3Lx5Uyy7ceMGDh8+jJYtWwLIfHkAIMsa6GfPngWQ/wDXID09XZzjb/j+4MGD4s+Gk5OT+Fm5u7sbHb9Tp044f/48fvnlF7z33nvw9fUVt73Ks6tSqfDs2TOjn2tDVnoDX19f3L9/H+XKlTNq15EjR7By5co8n4UX+fn5Yf/+/UYvMPbu3QuZTIaGDRtmu49SqYSPjw/+/PNPo5EIe/fuhaWlpRiE+/n54dSpU4iLixPrHD58GKmpqeIz0KRJE6SkpBjd/7i4OJw+fVqs4+3tDXNzc+zdu1esIwgC/vzzT/j6+mY7NYaIiOhVsQefiIqdTCbDl19+iZEjR6Jr1674+OOPUatWLaSlpeHIkSOIiIjA2LFjjYazpqWlYdCgQRg+fDhSUlKwYMECODo6ikP7R4wYgZ49e2Lo0KHo1asXlEolNmzYgL/++guLFy8GAHh4eGDJkiVYvnw5vLy8cOfOHSxbtgwajaZA87MLwsPDA7t378Yvv/yCWrVq4erVqwgPD4dEInntc3bp0gXr16/H8OHDMWbMGJQpUwbr1q1DRkYGevfujcqVK6NBgwaYNm0aHj58iDp16uDkyZNYsWIFOnfunG0yu7zk5z6/qFy5crC3t0dERATs7OxgZWWFQ4cOiTkWDPfA8MJl//79sLa2FofwGzRr1qzQrkWj0eDKlSuws7ODnZ1dtnXs7e3RuXNnfP/991Cr1ahevTrWrFmDxMREo0R4d+/eRVxcnJg/wd/fH1WrVsXEiRMxduxYJCcnY/bs2ahcuTL69esHAAgICICnpycmTZqE0aNHo2bNmrh48SLCw8MREBAgBphxcXG4e/dulpwC2fnss8/w6aefoly5cli1ahVSU1MxfPjwPO9F06ZNYWNjgw0bNmDQoEFG89Zf5dlt0aIF1q9fj6lTp6Jbt264du0a1qxZYxS0d+nSBT/99BP69++PYcOGoVKlSjh69ChWrFiBTz75BHK5PNt7m51BgwZh586dGDRoEPr374/bt29j/vz56NGjB9577z0A2X/ew4cPR//+/TF27Fh07doV586dw6pVq8QkkkBmIlBDO0eNGoX4+HjMmTMHzZo1E0df1K9fH76+vpg0aRImTZoEGxsbhISEwNLSEr169QKQOWVkwIABCAsLg1wuh7e3NzZv3ox///03S64RIiKi18UAn4hKRPPmzfHrr79i1apVWLp0KeLi4qBQKODi4oIFCxaIQ2ANfHx80LBhQ0ydOhVAZpA0efJksferTp06iIiIwIIFCzB58mQIggBHR0eEhYXh/fffBwAMHToUz549w7p16xAWFoZKlSqhU6dOkEgkWLZsGRITE2FlZVWo1zllyhRkZGRg4cKF0Gg0qFy5MoYPH44bN27g77//zjKcuSAsLCzw008/Yfbs2fj666+h1+vh5eWFdevWicN+ly1bhsWLF2Pt2rWIi4tD5cqVMX78+GyH2OdHfu7zy5YsWYJvv/0WU6ZMgUKhgIODA8LDw/Hdd9/h9OnTCAoKQu3atdG+fXtERETg0KFD2LFjh9ExDJ9RYVzLo0eP8NFHH2HUqFEYPXp0jvW++uorWFlZYcWKFUhNTYWrqyvWrFljNLJkyZIl2Lp1K/777z8AmcHcypUr8d1332HChAlQKpVo1qyZmFEfyHzBtXr1aixYsABLlixBQkICqlSpguHDh4svAYDMlx2fffYZ1q1bhwYNGuR6TV9++SW+++47xMXFoW7duvjll1+M2pkTExMTcalIQ8JCg1d5dv38/BAcHIz169dj7969cHV1RWhoKHr27CnWMTc3R0REBObNm4c5c+YgKSkJ9vb2mDBhAgYMGJDjvc1OrVq1sHr1asyePVt8ydWvXz+MGTNGrJPd592oUSOEhIRg8eLFGDlyJGxtbTF58mSj85ctWxbr1q3Dd999h4kTJ0KlUiEwMBCTJ082akNoaChmzZqF2bNnQ6/Xo27duli4cKHRC8qRI0dCJpPh119/xerVq+Hg4IAlS5YU+sgTIiIiifA6mXKIiIpBUFAQAGD9+vUl3BKi4jV58mT07t07x17skJAQhIaG5hoEExER0buDc/CJiIjeQDdu3MCFCxcKlHCPiIiI3m0M8ImIiN5AZcuWxdq1a3NcspGIiIjoZRyiT0RERERERFQKsAefiIiIiIiIqBRggE9ERERERERUCrxzy+SdO3cOgiCI6+wSEREREdG7KSMjAxKJBN7e3iXdFKJC8c714AuCgOJMOyAIAjQaTbGekygnfB7pTcFnkd4kfB7pTcFnsfgVd2xAVNTeuR58Q8+9u7t7sZwvNTUVkZGRcHBwYCZkKnF8HulNwWeR3iR8HulNwWex+F26dKmkm0BUqN65HnwiIiIiIiKi0ogBPhEREREREVEpwACfiIiIiIiIqBRggE9ERERERERUCjDAJyIiIiIiIioFGOATERERERERlQIM8ImIiIiIiIhKAQb4RERERERERKUAA3wiIiIiIiKiUoABPhEREREREVEpwACfiIiIiIiIqBRggE9ERERERERUCjDAJyIiIiIiIioFGOATERERERERlQIM8ImIiIiIiIhKAQb4RERERERERKUAA3wiIiIiIiKiUoABPhEREREREVEpwACfiIiIiIiIqBRggE9ERERERERUCjDAJyIiIiIiIioFGOATERERERERlQIM8ImIiIiIiIhKAQb4RERERERERKUAA3wiIiIiIiKiUoABPhEREREREVEpwACfiIiIiIiIqBRggE9ERERERKWKIAgl3YQiU5qvjV4fA3wiIiIiIipRUVFR+Prrr9GmTRt4enqiXr166NmzJ37++Wdotdpc992yZQucnJwQExMDALh+/Tp69eplVCcgIABTpkzJ9ThTpkyBk5MTmjVrlmMQPXfuXDg5OSEoKCjLtuHDhyM8PNyobM+ePRgyZAiaNm0KNzc3NGnSBGPHjsXFixdzbYtBUFCQ0bk2btyIH374Qfz+xIkTcHJywokTJ3I9jpOTE5ycnDB//vxst+v1ejRt2hROTk7YsmVLjseJiYkRj5Xdv88++wzA88/kxX8eHh744IMPsHTpUuh0unxdPxWcSUk3gIiIiIiI3l27du3CZ599hlq1aqF///6oUaMG0tPTceDAAXz33Xc4dOgQlixZAolEku3+zZs3x4YNG1CxYkUAmUH1uXPnXqktUqkUDx8+xNmzZ1GvXr1s25odjUaD48ePY/To0QAArVaLCRMm4M8//0THjh3xxRdfoEyZMrh37x5+/fVX9OzZE3PnzkXbtm1zbc+MGTOMvg8PD4evr+8rX9uePXswfvz4LNtOnTqFR48e5XmMihUrYsOGDVnKIyIisHv3bnTt2tWoPDQ0FBUqVIAgCEhLS8PZs2exePFipKen49NPP32l66DcMcAnIiIiIqISERUVhc8++wxNmzbFwoULYWLyPDzx9/dHgwYNMGbMGOzevTvHYLhs2bIoW7ZsobSnUqVKEAQBu3fvzhLgnz9/Hg8fPoSjo2OW/U6fPg2VSgVnZ2cAwNKlS7Fnzx4sXrwYbdq0MarboUMHjBw5EjNnzkRAQABMTU1zbI+Dg0MhXFWmunXr4vTp07hy5QpcXFyMtu3cuRPOzs6IjIzM9RgKhQJeXl5GZZcvX8bu3bsxbtw4+Pj4GG1zdnZG5cqVxe8bN26M6Oho/O9//2OAX0Q4RJ+IiIiIiErEypUrIZVKMXPmTKPg3qBNmzb48MMPxe+dnJwQGhqKLl26wMPDA6GhoUZD9ENCQhAaGirWDQkJKXCbAgMD8ccff2QZpr9r1y40btwYNjY2WfY5cOAAmjZtColEgrS0NKxatQqBgYFZgnsgsyf9008/RYMGDfD06VMAmUPaXVxcsHHjRvj5+cHX1xc3btwwGqIfEBCA2NhYbN261WhKQn7Vr18f5cuXx549e4zKtVot/vjjD7Rr165AxwMy8wF89dVXqFWrFvr165evfaysrHIcjUGvjwE+ERERERGViH379qFhw4YoV65cjnV++OEHo977pUuXokOHDtn2jnfv3h3dunUDAGzYsAHdu3cvcJvatm0rDtM30Ov12LNnT45B8IEDB+Dv7w8AOHr0KFJTU9G+ffscz+Hk5ITFixfD3t5eLNPpdFi9ejW+/fZbccrCiwzD3f39/Y2mJOSXTCZDmzZtsgT4x44dg1qtRkBAQIGOB2S+9Lhw4QI+//xzyGSyLNv1ej20Wi20Wi2Sk5Nx8OBBbN++HR9//HGBz0X5wyH6RERERERUrHQ6PZKTk5CQkIDq1atn2f5yYj2JRCIGkD4+Pujfv7+47dKlS+LXdnZ2sLOzA4AsQ8nzy93dHVWqVDEapn/69GnEx8ejZcuW2Lx5s1H96OhoREdHw8/PT/weQJbr0uv10Ov1RmVSqRRS6fM+12HDhqF58+bZtsvFxQUKhQJly5Z95Wtr27YtIiIijIbp79q1C++//z6USmWBj7dq1SrUrVsXDRo0yHZ7q1atspS5u7ujb9++BT4X5Q8DfCIiIiIiKhZ6vYCrt+KgztBB0KZlW+fOnTto3bq1UZm9vT3+/vtvABDnuReltm3bYtu2bZg6dSokEgl27tyJ5s2bw8LCIkvdgwcPwtvbG5aWlgCQJYg3WLRoEZYuXWpUNmrUKDExH1D011avXj3Y2tpiz549cHFxgUajwV9//YU5c+ZkqavT6YymKbz8MuLs2bP4999/ERYWluP5wsPDUaFCBQCAWq3G9evXER4ejp49e2LDhg3Z3k96PQzwiYiIiIioyP178wl2H72Nuw+ToJRLEZegholciSv/3TSqV6lSJWzatEn8PiwsDNeuXRO/Nzc3L/K2tm3bFsuWLcPZs2fh5eWFP/74A19++WW2dQ8cOIBmzZqJ37/33nsAgNjYWNSuXVss7927N1q2bCl+b5hK8KKivjaJRILAwEAxm/6hQ4cglUrh5+eHhw8fGtVt1aoVYmNjxe87d+6MWbNmid/v3bsX1tbW4tSE7Dg6Ohol2fPx8YGjoyN69+6NjRs3Go3EoMLBAJ+IiIiIiIrUvzefIGLvVTx8mgozUxNYmiuQmq6F1XuuOHXyOI5fuIOGntUAZGZqd3d3F/fNLqldUatTpw5q1KiBPXv2ID09HWq1Otuh82q1GidOnMCECRPEMj8/PyiVSuzZs8doH1tbW9ja2hZD63PXtm1b/Pjjj4iMjMSuXbvQunVryOXyLPXCw8Oh0WjE78uUKWO0ff/+/Xj//fez3Tc3hs/29u3bBW885YlJ9oiIiIiIqMjo9QJ2H72NB09SoJRLoTCRQiqVQCmXwaV+O+j1OkybNhXp6eos+6anp4tz2vPrxWHkr6Nt27b4448/sGvXLrRq1SrbOeonTpyAjY0NnJycxDJLS0v0798f27Ztw59//pntsV8ckVAQhXFtXl5esLe3x/bt2/H333/nmDjQyckJ7u7u4r8Xe+Lj4+Nx+/Zt1K1bt8Dnv3jxIoCsOQqocLAHn4iIiIiIisyVW09x7e4zyGTSLMujWZWvDDf/frhyaD06dvoQvXt9BCcnJ2i1Wpw7dw4bN27E06dPMWjQoHyfz8rKCgCwY8cOeHp6okqVKgCAGzduYO3atUZ179+/LwayL2vbti3CwsKwfft2LFmyJNtzHTx40Gh4vsGYMWPw4MEDjB49GoGBgWjVqhUqVqyIx48f459//sHu3btha2uLRo0a5fu6DNd25coVnDx5Eh4eHmL53r17s13Dvnv37lCpVFnKAwMDsW7dOtjY2MDX1xcAsiwLmBvDCwoHB4dc60VGRuLJkycAMnMTREVFISQkBBUqVEDnzp3zfb43iSAIxbbM36uciwE+EREREREVOkEQkJCiwcOnqUhJz8DlnV+jTCVH1G1pPO/a3qEeZBbvoarJDWzatAmxsbEQBAFlypSBpaUl1q9fb9Tb++OPP2LZsmXo0qULbt26hcePHxsdr3Xr1ti+fTumTJmCbt26iXPnL126ZJRx36Bnz57o0qWLUdmDBw8we/Zs1KhRA/Hx8WjcuLG47dq1a2JyuIMHD2Ly5MlZjimTyfDDDz+gffv2WLt2LcaPH5+ljoWFBXx8fHK/iS8ZMGAAvvvuOwwcOBBr1qwRyyMiIrKtHxgYmG2A37ZtW6xatQoffPABkpOT8c033+SYvT87hqDd8DLlRU5OTnj//fcBZCYRNDAxMUGZMmXg7u6Ov//+O8fM+126dMH333+PkJAQhIaG4r///st3uwyCgoIAAOvXry/wvjl58OABpk+fjunTpxuNZigqZ86cwdKlS7FixYoC7ccAn4iIiIiICpUgCIhPViMtXQszUxPIpFK4NB8MU7OsSeQytHqorMpjaL+2cKhiI5YbgrQXg3sbGxtUr14d48ePh62tLb744gtUqFDBaG13W1tboyR9AMQM/C97MeB/MYHc0aNHceDAAezbt88omNNoNNBoNAgJCQEA/PHHH7neh6ZNmyIlJQWHDx/G2rVrYWZmJm4zNTUVv+7SpUuWlwxA1gC1ffv2aN++vVHZywFwXFwc7t69Ky6rl10dNzc3sezEiRPYvn07unbtmu9gum3btmjbtm2O252dnXMc9aDRaHDlypUs5REREdi9eze6du2arzbkZsaMGa99jJcZnonisnHjRkRFRRV4Pwb4RERERERUaF4M7gUAVWwtYVvOHLF6O5ipFFnqJqVloHolK9S0t87z2PHx8fDz88ux97eonT59GiqVqkDL2UVGRsLOzq7Aw/FfVdmyZVG2bNliOderUCgU8PLyMiq7fPkydu/ejXHjxhV4VEN28po6UJoxyR4RERERERUKQRDwLOl5cA8AUqkEgQ2r4/KOr3H+77VIiHuM9T/0xd3/TuH4rnCc2TgZO5ePwfTpXyA1NRVAZu/9yZMncfLkSTg5OeHEiRNiIruwsDA4OTkhJiYmy/nVajXCwsIQGBgId3d3tG7dGsuXLzdam16n02H58uVo3749evfujY8//hg9e/bE8ePHAQBbtmzBZ599BgB4//33MWXKFHHfAwcOoGnTpuK86Pj4eEyfPh2NGzeGu7s7evTogWPHjhm16erVq6+0vv2WLVvg7u6O06dPo2vXrnB3d0ebNm3w999/4+bNm+jbty88PT3RqlUr7Ny502i/F+9PXFwcJkyYAD8/P7i7u6NTp07Ytm0bgMze+z59+gAA+vTpI46aCAoKwsSJEzFmzBh4eXmJy9nFxMRg8uTJaNKkCVxdXdGoUSNMnjwZz549K/D1GQiCgK+++gq1atVCv379smz/66+/0KZNG7i7u6N79+5Z7m92goKCxGsBMqcNREREYOrUqfD19YW3tzfGjh0rTjUAgLt372LYsGFo0KABPD098dFHH4k99jk9EwEBAfjuu+/Qt29feHh4YOrUqVnuv0FAQIDRs6TRaLBw4UK8//778PDwQPv27bF161YAwJQpU7B161bExsbCyckJW7ZsyefdZIBPRERERESFQK8XEJeoRpr6eXBvUKd6WajM5DA3NUGGVgcAiDzyMypUtMPnX/6AoUMGY9OmTQgPDweQOcTaxcUFLi4u2LBhA1xdXbFhwwYAmevHb9iwARUrVjQ6hyAIGDZsGFauXInu3btj6dKlCAwMxMKFC42GbM+dOxdLlizBRx99hKlTp2LYsGGIj4/H2LFjkZaWhubNm2P48OEAgNDQUIwYMULc98CBA+K672q1Gn379sW+ffswbtw4hIaGws7ODoMGDTIKQiMjI5GSkoKePXvC3d0dfn5+mDt3LjIyMvK8p1qtFhMmTEDPnj0RHh4OMzMzTJw4EcOGDUPz5s2xdOlSVKxYEcHBwXjw4EG2x5g0aRKioqIwc+ZMrFixAi4uLggODsbx48fh6uqK6dOnAwCmT59udJ92794NlUqF8PBwDBo0CGlpaejTpw+ioqIwY8YMrFq1Cn369MHOnTuxYMGCPK8lJ7t27cKFCxfw+eefQyaTZdk+depU9OnTByEhIVCpVBg8eHC2uRTysmDBAuj1esyfPx+TJ0/GP//8g++++w5AZgLAoUOHIi0tDbNnz8aSJUtgY2OD4cOH486dO7k+ExEREXB3d8eSJUvQrVu3fLdn4sSJWLNmDbp3745ly5ahSZMmmDJlCnbs2IERI0bA398fFSpUwIYNGwqUH4FD9ImIiIiI6LXo9ALik9KRrtHlWEduIoVz9XJo3dIRJzYDTZs1w5KQhZBKM3vDjxw5gv3792PChAlwcHAQE9kZhnMb/mtnZ5dliDeQmfDu6NGjmD9/vrj0m5+fH0xNTbFo0SL06dMHtWvXxqNHjzBu3DgEBQWJgaKDgwNGjx6N//77D15eXqhatSqAzLnkhjn40dHRiI6Ohp+fHwBg+/btuHr1Kn799Vd4enoCAJo1a4agoCDMnTsXmzdvRlxcHB4+fAidTodJkybhvffew7Fjx7BixQrcv38f8+bNy/W+6vV6DBs2DN27dwcAJCYmYty4cejbt6/Yq25paYmuXbvi8uXLsLOzy3KMkydPYuTIkWjZsiUAwNfXFzY2NlAoFLCwsBCHszs4OBgNbZfL5Zg5c6Y4j98w1eCHH34QVyZo2LAhLly4gJMnT+Z6HblZtWoV6tatm+O0i5kzZyIwMBAA0KhRI7z//vtYsWIFFi9eXKDzODo64vvvvxe/v3jxopi74enTp7h586YYWAOAh4cHQkNDodFoULZs2WyfCQB47733MHHiRPH7W7du5dmWa9euYe/evfj888/Rt29f8dpiY2Nx4sQJtG/fHmXLls12OkNeGOATEREREdEr0+kFxCemIz0ja3Cv1wuIfpiE5NQMaHWZw+TtK2QG7k0b+4rBPZAZuMfGxr5yO06ePAkTExMxGDTo2LEjFi1ahJMnT6J27dpiUB0XF4fIyEjcv38f169fB5A5bDonBw8ehLe3NywtLQEAx44dQ4UKFeDq6gqtVivWa9GiBWbPno2EhASYm5tj9erVqFatmhgU+vr6QqFQYOHChRgxYgRq1KhhNIUAyMw4b+Dt7S1+Xa5cOQAQXygAmYkHgczgPzsNGjRASEgIrly5gqZNm8Lf3x/BwcE5XqdBzZo1xeAeyAxsf/75Z+j1ety+fRt37tzBjRs3cPPmTaPrf5FOpzNafk8qlUIqfT6I/OzZs/j3338RFhaW7f5yuRytW7cWv1cqlWjWrBn++eeffB3/RS8HynZ2dkhLSwMAlC9fHg4ODvjiiy9w+PBhNGnSBM2aNROH5efmVaZfnDlzBgCMrg2AmLzxdTDAJyIiIiKiV6LT6fEsSQ11NsH91dtx2HP8Nh4+TYVOr0dSagYibz/FzdgEADDKKA9kBmcFWYv9ZQkJCShTpkyWYd4VKlQAACQlJQHIzJw/c+ZMXLp0CUqlElWqVEGtWrUA5L4W/IEDB4zWvI+Pj8fjx4/h6uqabf3Hjx/DwcFB7PF/UfPmzbFw4UJcvXoVu3btQmhoqNH2F7PZG0YyvOjle5ebBQsWYOnSpdi9ezf27t0LqVSKxo0b46uvvoK9vX2O+2W3vN6aNWuwdOlSxMfHo3z58nBzc4OZmZl4b1/WqlUro5c2nTt3NlqtYO/evbC2thZ7zV9WpkyZLAF7uXLlxJcZeR3/Rbk9bxKJBKtXr0Z4eDj+/PNPbNu2DXK5HC1btsTMmTNhbZ1zAkhz86wrQ+QlPj5evJbCxgCfiIiIiIgKTKfTIy4xHRqtPsu2q7fjELH3KtLVWqjM5DCRmUACIDVdi51H8h7C/Cqsra3x7Nkz6HQ6oyD/0aNHADKDxeTkZAwaNAhOTk7YuXMnUlJSIJVKERcXh7179+Z4bLVajRMnTmDChAlimaWlJapXr465c+dmu0/lypVx+/ZtHD9+HG3btjVaMz49PR1AZsZ7Hx+fAs2xLihLS0tMmjQJkyZNws2bN7Fv3z4sWbIEM2fOxPLly/N9nN9//x2zZs3CpEmT0KVLFzFT/9ixY3OcEx8eHm40KqJMmTJG2/fv34/3338fcrk82/2TkpIgCIKY1BAAnjx5Ip47r+MXhK2tLb788kvMmDEDV69exZ49e7BixQqUKVOmQMvuGdr68qiMlJQU8WvDsxAXF2c0rSIqKgrx8fGoV6/eK18Hk+wREREREVGB5Bbc6/UC9hy/jXS1FjYWCihMpJBKJJBIAIWJFOqMzOHcefXW5zTUOie+vr7QarXivGqD3377DQBQr1493Lx5E/Hx8ejTpw8cHBzEcxw8ePD/267P9twnTpyAjY2NmMnfcL779++jXLlycHd3F/8dOXIEK1euhEwmw+PHjzFjxowsbdq1axcsLCzg6uoKW1tbo/3d3d0LdN25iY2Nhb+/v3j+mjVrYvDgwWjcuDHu3bsHANkmtsvOmTNnYGVlhUGDBokBdkpKCs6cOZMlmDVwcnIyuq4X567Hx8fj9u3bqFu3bo7nTEtLE1c3MJxv//794nz93I5fEOfOnUPjxo1x8eJFSCQSODs7Y9y4cXB0dBTvU36fR8OIixeTHhoCdwNDAP/3338b7Tt37lx8++23BTrfy9iDT0RERERE+ZZbcA8A0Q+T8PBpKlRmcqOeVwCARAIzRWZv7aNnabmex8rKCufOncOxY8fg4uKS6zBpIDPBXYMGDTBt2jQ8fPgQderUwcmTJ7FixQp07twZDg4OSEpKgoWFBZYuXQoTExNER0fj+PHjYqBlmJNt6GH9888/0axZMxw8eNBoeD4AdOnSBT/99BP69++PYcOGoVKlSjh69ChWrFiBTz75BHK5HPXq1UOjRo0wa9YspKenw8HBAfv378f69esxZcoUo179omBvbw87Ozt88803SE5ORtWqVXH58mUcOHAAQ4cOBQAxp8D+/fthbW2NOnXqZHssDw8P/PLLL5g1axZatGiBR48eYdWqVXjy5Emen012rl27BiD3Nevlcjk+//xzjB8/HhYWFli+fDnS09ONstgXBhcXF5iammLy5MkYPXo0ypcvj6NHjyIyMlJcRvDlZ8IwreNlDRo0gKmpKWbNmoWxY8ciJSUFixcvFnMlAECdOnUQGBiIOXPmID09Hc7Ozjh48CD++ecfcbqGlZUVnjx5ggMHDsDZ2TnLqhE5YYBPRERERET5kldwDwDJqRnQ6fUwkWUfahg6jNPV2SdmM/j4449x+fJlDB48GN9//z06dOiQa32JRIJly5Zh8eLFWLt2LeLi4lC5cmWMHz/eKOP8kiVLMHv2bIwdOxZKpRI1atTATz/9hMGDB+P06dMICAhAgwYN0LhxY8ybNw/Hjh3D7du3MXnyZKPzmZubIyIiAvPmzcOcOXOQlJQEe3t7TJgwAQMGDACQ2QsbGhqK0NBQrF27Fo8fP0bVqlXx9ddfi5nxi1poaCjmz5+PRYsW4dmzZ6hUqRJGjRqFIUOGAABq166N9u3bIyIiAocOHcKOHTuyPU7nzp0RExODzZs34+eff4atrS38/f3Ru3dvfPHFF4iKisox6M2OYQ363F5ylC1bFhMmTMD8+fPx+PFjeHp64qeffkLNmjULcAfyplQqsXr1asybNw/ffvstEhMTUb16dXz11Vfo0qULAGR5JnKa3mBlZYWQkBDMmzcPI0eOhL29PUaNGoVt27YZ1ZszZw5CQ0Px448/4tmzZ6hVqxYWL14srnbQpUsXHDhwACNHjsSYMWPEzysvEuF1Mlm8hQzzQwpz6EtuUlNTERkZCWdn51dKwEBUmPg80puCzyK9Sfg80pviTX8W8xPcA8Cd+4lYuuUilAoZFCZZhxlLJBKozOQY3MkdDlVsiqi1+VPcsQFRUeMcfCIiIiIiylV+g3sAqGJrCdty5khJz8gyz14QBKSka1CxrBlq2hd8WDcR5Y4BPhERERER5aggwT0ASKUSBDasDlOFCeJTNNBk6KEXBGgy9HickI5niRrcjElEanpGEbec6N3DAJ+IiIiIiLJV0ODeoE71svi4TR3YV7CAOkOHhGQ14pPVSFfroNMLuPswCeeuPS6iVhO9u5hkj4iIiIiIsnjV4N6gTvWycKxaBrfuJeC3Qzfx351n4jYrlQLutcoXVlOJ6P8xwCciIiIiIiOvG9wbpKm12HYgCtej48Uy27LmmNq/Pmwsla/ZSiJ6GQN8IiIiIiISFVZw/+hZKsI2XsDDuFSxzLGqDYZ18YBdOdXrNpOIssEAn4iIiIiIABRecH8jJh7hmy8iJe15Ir2GbpXwyQd1IM9m6TwiKhwM8ImIiIiIqNCC+1NXHuDHnVeg1T1fIq9j05r4oHF1SCSS120mEeWCAT4RERER0TuuMIJ7QRCw+9ht/HbwplhmIpOgT1sX+LraFUYziSgPDPCJiIiIiN5hhRHca3V6ROy5imOX7otlKjM5hnfxgEMVm0JoJRHlBwN8IiIiIqJ3VGEE9ynpGVi+5RL+u/t8GbyKZcwwsrsXbMuaF0YziSifGOATEREREb2DtDo9nr1mcP84Pg1hG8/jwdPnmfIdKttgWFcPWJjJC6OZRFQADPCJiIiIiN4xhRHc34xNQPjmC0hKfZ4p39fVDkEfODNTPlEJYYBPRERERPQOKYzg/szVh1i74woyXjhGO78aaN+kBjPlE5UgBvhERERERO+IDK0e8UmvHtwLgoC9x+9g24EosUwmlSCorTMaulUqrGYS0StigE9ERERE9A543eBep9Pj573/4cjFe2KZuakJhnXxgGPVMoXVTCJ6DQzwiYiIiIhKudcN7lPTM7B86yVcvfM8U34FGzOM7O4Ju3KqwmomEb0mBvhERERERKVYhlaHZ4lqZOheLbh/Ep+GsE0XcP9JilhWq7I1hnfxgIW5orCaSUSFgAE+EREREVEppcnQ4VlSOrQ64ZX2v3UvAUs2GWfKr+9iiz5tnSE3kRVWM4mokDDAJyIiIiIqhdQaLZ4lq6F7xeD+7NVHWLPjX6NM+W0bV0eHpjWZKZ/oDcUAn4iIiIiolFFrtHiWpIZOX/DgXhAE/HnyLrb8c0Msk0kl+OQDZzRyZ6Z8ojcZA3wiIiIiolIkXaNF/CsG9zqdHv/78z8cOv9CpnylCYZ28YBTNWbKJ3rTMcAnIiIiIiol0tQZiE/SQC8UPLhPS9di+bZLiLwdJ5aVtzHDKGbKJ3prMMAnIiIiIioFUtMzkJD8asH904Q0hG28gHsvZMqvaW+N4V09YMlM+URvDQb4RERERERvuZQ0DRJTNHiFUfm4fT8RSzZdQGKKRizzca6Ivu1cmCmf6C3DAJ+IiIiI6C2WlKJBctqrBffn/nuE1b8bZ8oPbFQdHZvVhJSZ8oneOgzwiYiIiIjeUkkpGiSlaVDQUfmCIOCv/8+Ub9hVKpXg4zZ14Of5XqG3k4iKBwN8IiIiIqK3UEKyGinpGQUO7nV6PTb8eQ0Hz8WKZWZKEwzt7I461csWciuJqDgxwCciIiIieosIgoCEFA1SXyG4T1NrsXL7Zfx786lYVs7aFCO7e+K98haF3FIiKm4M8ImIiIiI3hKCICA+WY20dC0KOuU+LiEdYZsuIPZxslhW4z0rDO/qASuVsnAbSkQlggE+EREREdFbQK8X8CxJDbWm4MH9nQeJCNtonCm/rlNF9GvvAoWcmfKJSgsG+EREREREJUAQBEjymalep9MjPkmN9Axdgc9z/tpjrP79MjQZzzPlt2lYDR2b1YRMKi3w8YpKQe4HUX4U9zP1JjzDb85PNBERERHRWyogIABTpkzJd/19+/YhODjYqGzPnj0YM2YMfH19MX36dAQFBSEoKAgZWj3iEtMLHNwLgoB9p+5i2ZaLYnAvlUjQvkF5nNy5GI8fPsiyz7Qp4/HTj6sKdJ6CenD/Hrw83LBlyxYAQGJiIiZPnozTp0+LdQzXDgAxMTFwcnIS62/ZsgVOTk6IiYkp0na+jhfb/7o2btwIJyenLP+++uqrYmtDUXr58y0s2f2MFaUlS5Zg1aqi/dnJD/bgExERERG9ptDQUFhY5D9J3dq1a7OU/fDDDyhfvjy+//57VK1aFV988QX0egFxienQ6vRZD5ILnV6PX/+6jgNnnwfBpkoZhnzojruRR3Di2JEs+2RkZODsmZPoO2BIgc71uiIjI7F9+3Z07dpVLJsxY0aO9Zs3b44NGzagYsWKxdG8EhcZGYkaNWpg1qxZRuXly5cvoRYVrooVK2LDhg2oWrVqoR43u5+xorRo0SKMGjWqWM+ZHQb4RERERESvycXF5bWPER8fj4CAAPj4+MDc3Bx6vQCtTl/g4D5drcXK3y7jctTzTPllrUwxqrsn3qtggbuR2e938fxZqMxVcKjt9DqXUSgcHBxy3Fa2bFmULfvuLOcXGRkJd3d3eHl5lXRTioRCoSi111YSOESfiIiIiOg1GYboG4Yb7969G2PGjIG3tzd8fX0xbdo0pKamAsgcOn3y5EmcPHkSTk5OOHHiBJycMoPqLVu2wNvbGzeibiNDpzdaBk+jVmP92hXo27sL2gQ0QlDPD/HLT2uh1z9/AfDkWQpGB3+PHas/x/Vdn+P67ql4eHIp2npK8F4FC+zZ9RtmfzcTANC7ewf88O3znvITx4+gfoPGkEgkOH/2NAKa1MPZ0ycxfvQQBAY0Rs8ubbHz9614+uQxpn8+EW1bNcFHXdpi068/i8fYs+s3BDSphwf37xndn17d2hudSzzniRPo06cPAKBPnz7ikPLchpe/PER/ypQp6NevHzZv3ow2bdrAzc0NnTp1wsGDB432O3fuHD7++GN4eXmhefPm+PHHHzFz5kyEhoaKbTF8Hi96uS3p6emYN28eWrduDTc3N9StWxf9+/dHZGQOb05yce3aNQwdOhR169ZF3bp1MXLkSERHR4vbBUHAf//9B2dn5wIf+2VxcXGYOXMmWrRoATc3N/j6+mLkyJFGUx2CgoIwceJEjBkzBl5eXujfvz8A4NGjRxg3bhx8fX1Rv359TJ8+HQsWLEBAQIDROTZu3Ih27drBzc0NzZs3R0hICHS63KeWZDcFw8XFBRcuXMBHH30Ed3d3tGjRIsvw9x07dqBjx47w8PBAw4YNMXHiRDx8+FC8jpd/xgyf7//+9z+0aNECdevWxZEjR7J91rJ7Fm7evIlRo0aJ92Do0KGIiooCAPHnNzQ0VPy6pDDAJyIiIiIqZDNmzIC9vT2WLFmCgQMHYtOmTQgPDxe3ubi4wMXFBRs2bICrqys2bNgAAGjRogWWrVgNuZkVXkyVLwgCpgaPw/8i1qFt+w/x7Q8L4N+iJVatWIIFc74DANx9kIQxk77EzXM7YV21IewbDIRv6/6wVOow99tpSE9PQ8NGTfFJ34EAgJnfzsEn/QaJ5zhx7DAaNPIzuo5vvvwcjfya4bvZC1GlanUsmPs9xo8Ziho1a+GbWfNRx9kVSxbPQ+SVywW4O8+TkLm6umL69OkAgOnTp+c6ND83ly9fxqpVqzBmzBiEhYVBJpNh9OjRSEhIAABERUWhX79+AID58+dj9OjRWL58Oa5evVrgc02ePBmbN2/GkCFDsHr1anz22We4fv06JkyYAEHI//oGt27dQs+ePfH06VP88MMP+PbbbxEdHY1evXrh6dPM0Rd3795FSkoKLl26hDZt2sDV1RVt2rTBtm3bCtRmQRAwdOhQHDlyBBMnTsSqVaswatQoHDt2LMs93717N1QqFcLDwzFo0CBoNBr07dsXZ8+exeeff47vv/8eV69exerVq432W7ZsGb744gs0atQIS5cuxccff4wVK1bgiy++KFBbAUCv1+PTTz9F27ZtsXz5ctStWxezZ8/GoUOHAABnzpzB5MmT0bp1a6xYsQKfffYZjh8/jgkTJgDI/mfMIDQ0FMHBwZg+fTq8vb3z1Z6HDx/io48+wu3bt/Hll19izpw5ePLkCfr27Yv4+Hjx57dbt27i1yWFQ/SJiIiIiAqZv7+/mOCrUaNGOHLkCPbv348JEybAwcFBnK9vGJps+K9NmfKoWsMJJnKF0fFOHj+KM6dPYNqX3yGgZRsAgE/9hlAqTbFmZTjcGgZi1+lkpCTGobzTByhTww+tfKuicwsHHD5YG19OnYSbN27Axc0d79lXBgDUdqwDu0rvAQDu34vFvdgY1KvfwOi8ge06onvPTwAAZmZmGDm0H+o4u6H/oOEAgFoOjjh04G/8e/kinF3c8rwvJjIJbCyeX5uFhYU4HN/BwSHXofm5SUpKwpYtW8R53Obm5vjkk09w/PhxtGnTBsuWLYOlpSVWrlwJMzMzAEDNmjXRs2fPAp1Ho9EgJSUF06ZNQ9u2bQEAvr6+SE5OxqxZs/DkyRNUqFAhX8cKDQ2FmZkZ1q5dKz4PjRo1QsuWLbFy5UoEBweLowJiYmIwZcoUmJiYYNu2bQgODoZGo0GPHj3yda5Hjx7BzMwMwcHB8PHxAQA0aNAAd+/ezRKQyuVyzJw5EwpF5ue0adMm3Lx5E5s3b4abW+Zn3LBhQ7Rs2VLcJykpCUuWLMFHH32EadOmAQCaNGkCGxsbTJs2Df3790ft2rXz1VYg84XEiBEj0L17dwBAvXr18Oeff2L//v1o2rQpzpw5A1NTUwwZMkRsp42NDS5dugRBELL9GTPo3bs3AgMD890WIHM+v0ajwZo1a8TPt06dOujVqxcuXLgAf39/AICdnV2JTzdggE9ERERE9Ar0egE3YxOQmKJBhlZv1Hv78h/5dnZ2iI2NzfOY6gwt9Nn0Al84dwYymQz+LVoalbdq0xZrVoZjzf92w6ZaY1Sq2xtSiQQdGtmisnUy9u76HceOZA5Vz8jQ5HjeE8eOwNXdExYWlkblru6e4tdlypYDAKNA3traBgCQnJSU57XJpBKUszZDRqosz7oFVbZsWaMkbXZ2dgCAtLQ0AMDx48fRrFkzMbgHAG9v7wIn6lMoFOJQ8YcPH+LWrVu4ffs2/vnnHwCZLwBeptfrjaZRSCQSyGQyHD9+HL6+vjA1NYVWqwWQ+cLDx8cHR48eBQDUr18fS5cuRYMGDWBubg4AaNq0KeLi4rB48WJ0794der3xs2c4/otsbW2xbt06CIKAmJgY3LlzBzdv3sTZs2eztLlmzZpi0Gy4d1WqVBGDe0M7W7RoIQ5hP3fuHNLT0xEQECBeCwBxCP+RI0fg4OCQZbj+y+180Yu96wqFAmXLlhWnudSvXx8LFixA+/bt0aZNG/j7+6NJkyZioJ2bV5nucObMGXh5eRm9vLGzsxM/9zcJA3wiIiIiogK6cP0xNv19HbGPkqHV6ZGQrMb5a48ReTsOAIwCSQCQSqU5Dt8WBAEJKZlBll6ffZ3EpARY29gYBUR6vYB95zOHcus06ZllKfeQfmc35vz+H0xNTVGtRi3Y2tqJ58nJieOH0aChX5ZylbkqS5mpmWmOx8mOVJK5PJ9CLoOJrGhmCL98vw1rkRsC67i4OJQrVy7LftbW1gU+16FDh/Ddd9/h5s2bUKlUqFOnjhh8Z3ePP//8c2zdulX83t7eHn///Tfi4+Oxa9cu7Nq1K8s+hiSC5cqVQ4sWLbJs9/f3x9GjR/HkyROMHz8eJ0+eFLf5+vpi/fr1Wfb57bffMH/+fNy/fx82NjZwdnaGqWnWz1KlMv7Mnz17lu29e7EsPj4eADBkSPYrMDx69AgnT54U8y0YrFu3Dvb29tnu83LbXvwZ8vb2xvLly7F27VqsWbMGy5cvR/ny5TFs2LA8lwY0fFYFER8fj8qVKxd4v5LAAJ+IiIiIqAAuXH+MsE0XkJauhaVKDrlMDokESE7LwPpdBUu0JggC4pPVSEvX5lrPytIaCfHx0Ol0kMlkSNdosWr7vzh76RoAQKYwh6WZgP8OrIZDbUd8uX4jqlarDqlUiuPHDuPg/n05HlujVuP82dMYPGx0gdqeHUNgrdNn9tTKpBJYqRRIS0sVt5UEOzs7PHnyJEt5QkKCGGC+/FLAICUlRQx67969i5EjR6Jly5ZYtmwZqlSpAolEgoiICHF++MtGjRqFjz/+WPze0DtuaWmJxo0bi4nsXmRikhmmnT59GtHR0ejcubPRdrVaDZlMBmtra8ycORMpKSnitpcDdMNxgoODERQUhIEDB8LW1hYAMHv2bJw5cybbdhvY2tri9u3bWcoNeQIAwMrKCgAwd+5cVK9ePUvd8uXLw9LSEps2bTIqr1GjhvhyoKCaNm2Kpk2bIi0tDcePH8e6devwzTffwNPTEx4eHgU61ssjCwwjBQwsLS0RFxeXZb9jx46hcuXKqFKlSsEvoIgwyR4RERERUT7p9QI2/X0daelalLNWQimXQSqVQAIJlHIp0jWZgXpeydYMvZFxiZnBfV6p2Ty860Kn0+HAP3/hWVI65kWcxaWoJ0iMOQsAqOnohh5NyiE5KRFdu/dC9Ro1IZVm/ql/8njmcG+9oP//cxsPiz5/7gysrG1Qs1b+50jnxPz/e/yfPHoEhYkU5axNcT/2bq5BXG7DtAtL/fr1cejQIajVarHsypUrePTokfi9Yc72gwcPxLKEhAQxUzqQmcxPrVZjyJAhqFq1qvhSwBDcZ/e5V65cGe7u7uI/Q5Z1X19f3LhxA87OzuI2Nzc3rF27Fn/++SeAzOHxU6ZMwa1bt8Tj6fV67N27F97e3lAoFKhZs6bR8WvWrJmlDefOnYNer8fo0aPF4F6n04lTAV5+qfEiX19fxMTEGK0SkJ6ebvRCw9PTE3K5HA8fPjRqi4mJCebPn4+YmBhYWFgYbXN3dxfveUH98MMP6Nq1KwRBgJmZGVq0aCHmvLh3L3MFB8PznxcLCwujzxxAlpcePj4+uHDhglGQ//TpUwwaNAgHDhwo0PmKGnvwiYiIiIjy6WZsAmIfJcNSJc+mR1oClakcAPDoWVqux7G0tMS5c+dx9NhR1K5dB5b/3wOakwYN/eBV1wdzZ32NCnVOQK+siLS4m4i78Q+qufhhxsj2yNCkQqVS4ad1qyGTySAzMcHBf/Zh987tAID0tMxh/IZ59ocO/I0Gjfxw4vhh+DZo/Ap3IyvvuvWhVCqxbMlCmH/6KVJTU7B48WLY2Njkei8AYP/+/bC2tkadOnUKpS0vGjZsGHbt2oVBgwZhwIABSExMxKJFiyCVSsXP0cnJCZUqVUJYWBgsLCwgkUiwbNkyo+H/rq6uMDExwZw5czBgwABoNBps2bIF+/fvB5C15zc3I0aMQM+ePTF06FD06tULSqUSGzZswF9//YXFixcDAHr27In//e9/GDZsGMaOHQszMzP8/PPPuHbtGiIiIvJ9LkOP9ldffYWuXbsiISEBERER4ioCqampOQbb7du3x/LlyzFy5EiMHTsWVlZWWLNmDZ4+fYr33stM0limTBkMGjQIixYtQnJyMho0aICHDx9i0aJFkEgkhf6ZNmzYEGvWrMGUKVPQsWNHZGRkYOXKlbCxsUHDhg0BZI4qOHfuHI4dOwYXF5ccj9WiRQv8/fff+P777xEQEIDTp09nWaWgX79+2LZtGwYNGoShQ4dCLpcjPDwcdnZ26NChg3i+s2fP4tSpU/Dx8SmxEStvxmsGIiIiIqK3QGKKBlqdHvIc5pKbmGT+UZ+uznnIvU4voOOHPSCVyfDZxDE4efxInueVSCT4eNgXsKjii/tX9+PeqdVIuncRjVr1woolC6BUyGBhYYmvv58PCAJmfhGMWV9Px6OHD7AwdAXMzVW4dPEcAMC7rg/q+TTAymWhCA9dgJPHj2ZZHu9VWVlZYe68hYCgx6hRI7Fo0SKMHDnSKEHby2rXro327dsjIiICEydOLJR2vKxatWpYtWoV1Go1xowZgwULFmDw4MGwsbER53rLZDIsXrwY5cuXx/jx4/Htt9+iXbt2aN26tdFx5s2bh4cPH2L48OHiEn/r16+HRCLB6dOn892mOnXqICIiAhKJBJMnT8aYMWPw+PFjhIWFiecsX748IiIi4OTkhG+++Qaffvop0tLSsHbtWnh6euZxhucaNGiA6dOn49y5cxg8eDBmzZqF9957D6GhoQCy9li/yMTEBKtWrYKLiwu+/PJLTJ48GbVr10arVq2M5rN/+umnmDJlCv78808MHjwYc+bMQb169fDTTz+JL3EKi7+/P+bOnYvr169j1KhRGD9+PMzMzLBu3TrxZdLHH38MuVyOwYMH4+DBgzkeq2vXrhg8eDB27NiBIUOG4Ny5c+ILFoNKlSrh559/RsWKFTFlyhR89tlnqFSpEn788Ucxj8OwYcNw+fJlDB48GPfv3y/U6y0IiVCQxRpLgUuXLgEA3N3di+V8qampiIyMhLOz8ysldCAqTHwe6U3BZ5HeJHweqSBuRMfju7UnYaqUQSnPOrRcrdEhXaPD5/184VDFJst2nU6PZ0lqqDN0WbalpaUhKioKtWrVypI0bv/ZGGz48z8Y/nKXSICPWjmhed03J/FX5hJ4SigVb94g4WPHjkEul4tLxAFAYmIiGjVqhKCgIEyZMqUEW/dmu379Om7evInWrVsb9Up369YNdnZ24ksCejO8eT99RERERERvqJr21rCvaIHb9xOhsJIaBTyCICApLQPVK1mhpn3W7Oy5Bfc50esFbP7nOvadihbLlAoZBnVyg3ut8q93MYVIKZfB2kIJucmbOUD433//xeLFizF+/Hi4uroiPj4ea9asgbm5OZo0aVLSzXujpaamYuzYsejduzdatWoFnU6HXbt24fLly0U24oJeHQN8IiIiIqJ8kkol6BZQG2GbLuBpohqWZnLITaTI0OqRlJYBc6UJugXUhlRqPP9Wq9PjWWI6NNqck5m9TK3RYfXvl3Hh+vPs7zaWSozq5onKtoU75PlVSSSAuakcVuaKLNf8JjHMl//ll19w//59mJubw9fXF/369XulpfLeJZ6enli4cCFWrVqFbdu2QRAEuLi4YOXKleJ8d3pzMMAnIiIiIioAz9oVMLKbJzb9fR2xj5KRnJYBE5kU1StZoVtAbXjWrmBU/1WC+4RkNcI2XcDdB0liWRVbS4zo5oEylgVbh76oSCUSWKnkUJkpSropeZJKpRgxYgRGjBhhVG6Yvku5CwwMRGBgYEk3g/KBAT4RERERUQF51q4A91rlcTM2AYkpGlipFKhpb52lFztDq0d8UsGC+3tPUrDy96t4lvh8STcPh/IY0NEVpm/I/Ha5TAprC8UbOd+e6F3Gn0giIiIiolcglUqyTaRn8CrB/Z1Havzxx2WoNc/n6Qf4VMl22H9JkOD/59tbKmGSw0oCRFRyGOATERERERWyDK0OzxLVyNDlP7g/eukBdpyKN8qU3+N9R7TwqVJErSwYiQRQmcphpVKU2BrfRJQ7BvhERERERIVIk6HDsyQ1tPkM7vWCgK3/3MCfJ++KZUq5DAM7ucHD4c3IlC+TSmCpUkBlKi/pphBRLhjgExEREREVErVGi2fJauh0Qr7qazJ0WP37vzh/7bFYZq1SYGR3L1S1ezMy5StMpLCyUEIpl5V0U4goDwzwiYiIiIgKgVqjxbMkNXT6/AX3CclqLNl8EXfuJ4pl5a1MMLKbOypVLPngXgJAqZDBxkIJGefbE70VGOATEREREb2mNHUGEpI1+Q7uYx8nI2zjBcQlpotlztVt0LSOHDaWyqJqZr5JJYDKVAFLlZzz7YneIgzwiYiIiIheQ2p6ZnCvF/IX3F+59RTLt11Cuvp5pnz/upXRwa8Kbt+6WVTNzDeZVAJrCwXMlJxvT/S2YYBPRERERPSKUtI0SEzJyHdwf/hCLH7e859YXwKg2/u1EeBTBenp6bnvXAwUJlJYWyih4Hx7orcSA3wiIiIioleQlKJBcpoG+RmVrxcEbD8Qhb3H74hlCrkUAzq4wcuxQhG2Mn8kEsBUYQJrCyVkUg7JJ3pbMcAnIiIiIiqghGQ1UtIzkJ+Oe02GDmt3XMHZ/x6JZVYqBUZ290Q1O6sibGX+SCWAhZkClipFSTeFiF4TA3wiIiIionwSBAEJKRqk5jO4T0xRI3zzRdy69zxTvn0FC4zs7omyVqZF2NL8kckksFZxvj1RacEAn4iIiIgoH/R6AfHJaqSrtcjPjPt7TzIz5T9NeD633rVmOQzq5AYzZcn/Ga4wkcLGUgm5CefbE5UWJf+bhYiIiIjoDafTC4hPSke6Rpd3ZQBXb8dh2dZLSFNrxbJm3vb4qJUjZNKSXVNeIgHMlCawVikh5Xx7olKFAT4RERERUS50Oj2eJamhzshfcH/kwj1E7L0Kvf55pvyuAbXxfv0qJb6mvFQigaW5HBbmnG9PVBoxwCciIiIiykGGVo/4pHRotPo86+oFAb8djMKeY88z5ctNpBjQwRXeThWLspn5YiKTwMZCCaWCIQBRacWfbiIiIiKibGgydIhPUiNDl3dwr8nQ4cedV3DmqnGm/BHdPFG9UslnylfKZbC2UEJuUrLTA4ioaDHAJyIiIiJ6iVqjxbNkNXS6vNPpJaZoEL75glGm/PfKqzCyuyfKWZsVZTPzJJEA5qZyWJkrON+e6B3AAJ+IiIiI6AVp6gwkJGug0+cd3N9/koKwjefx5IVM+c7Vy2LIh+4wMy3ZP7WlEgksVXJYmHG+PdG7ggE+EREREdH/S0nTIDElA/p8LHL/3504LNtyCakvZMpv6vUeerZygkxWskPh5TIprC0UnG9P9I7hTzwREREREYCkFA2S0jTIR2yPoxfvIWLPVaNe/i4tHNDKt2qJZsqX4P/n21sqYVLCLxmIqPgxwCciIiKid5ogCEhM0SAlPSPP4F4QBPx+6CZ2Hb0tlslNpOjf3hV165RspnyJBFCZymGlUpT4cnxEVDIY4BMRERHRO0sQBMQnq5GWrkVeHfcZWh1+3BmJ05EPxTJLczlGdPNEjfesi7aheZBJJbBUKaAylZdoO4ioZDHAJyIiIqJ3kl4v4FmSGukabZ51k1I1CN98ETdjE8SySuVVGNnNE+VtSjZTvsJECisLJZRyWYm2g4hKHgN8IiIiInrn6HR6xCepkZ6hy7Pug6cpCNt4AY/j08SyOtXKYEhnd5iXYI+5BIBSYQIbC0WJJ/UjojcDA3wiIiIieqdodXo8S0yHRqvPs+61u8+wdMtFpKY/7+X383gPvduUbKZ8qQRQmSpgqZJzvj0RiRjgExEREdE7I0OrQ3ySOl/B/fHL97F+V6RRpvzOzWuhdYNqJRpUy6QSWFsoYKbkfHsiMsYAn4iIiIjeCeoMHeKT0qHV5Z5OTxAE7Dh8CzuP3BLLTGRS9O/ggnp1bIu6mblSmEhhbaGEgvPtiSgbDPCJiIiIqNRTa7R4lqQ26o3PToZWj/W7I3Hy3wdimaW5HMO7eqKmfcllypdIAFOFCawtlJBJOSSfiLLHAJ+IiIiISrXU9AwkpGigzyO4T07LwNLNF3EjJl4ssytnjpHdvVChBDPlSyWAhZkClipFibWBiN4ODPCJiIiIqNRKTtUgKVWDPGJ7PIxLRdjG83j07HmmfKeqZTCki3uJri0vk0lgreJ8eyLKHwb4RERERFTqCIKAxBQNUtIzIOQR3F+Pfoalmy8i5YVM+Y3cK+HjwDowKcFM+Uq5DNYWSshNuAQeEeUPA3wiIiIiKlX0egEJKWqkpWuRR2yPk/8+wLpdV4wS73VqVguBjUouU75UIoG5qQmsVUpIOd+eiAqAAT4RERERlRo6vYD4pHSka3S51hMEAbuO3sbvh26KZSYyKfq2c0Z9F7uibmaOFHI5rFQKlLE0LbE2ENHbiwE+EREREZUKWp0ezxLT81zjPkOrR8SeSBy//DxTvspMjuFdPeBQ2aaIW5kzuVwGGws5zE35JzoRvRr+9iAiIiKit16GVodniWpk6HIP7lPSMrB0y0Vcj44Xy2zLmmNkd09ULGNexK3MngSZ8+0VZko8va/Nsz4RUU7eiIwd27ZtQ9u2beHu7o527dph9+7d4raYmBgMHToUdevWRZMmTbBw4ULodLkPuSIiIiKid4c6Q4e4xPQ8g/vHz1Ixe/1po+C+dhUbTA7yKbngXpI5eqCstSlkUgmEvDICEhHlosR78Ldv346pU6fi888/R9OmTbFz506MHz8ednZ2cHNzw8CBA1G9enX873//w927dzF16lRIpVKMGTOmpJtORERERCVMrdHiWZIaujzWwbsRE4/wzReRkpYhljV0s8MnHziXWKZ8mVQCS5WiRJfhI6LSpUQDfEEQsGjRIvTp0wcff/wxAGD48OE4ffo0Tp48idjYWNy7dw+//vorrK2t4ejoiKdPn2L27NkYNmwYFApFSTafiIiIiEpQmjoDCcmaPIP7U1ce4MedxpnyOzStibaNq5dYpnyFiRTWFkoo5LISOT8RlU4lGuDfunULsbGx6NChg1H5qlWrAABffvklXF1dYW1tLW5r2LAhkpOTERkZCU9Pz2JtLxERERG9GVLTM4N7fS5D2gVBwO5jt/HbwRcz5UvQp60LfF1LJlO+RAIo5SawsVRCxiXwiKiQlXiADwCpqakYOHAgrly5gsqVK2P48OEICAjAgwcPYGdn/Mu3YsWKAID79++/coAvCAJSU1Nfr/H5lJaWZvRfopLE55HeFHwW6U3C5/Htk5auQ0KqGvpceu61Oj027ovCycjHYpnK1AQD2tdBTXurEvm8pVIJLM0VMJProU7Pen4+i8VPEIQSG8VBVBRKNMBPTk4GAAQHB2PUqFGYOHEi9u7dixEjRmDNmjVIT0+HlZWV0T5KpRIAoFarX/m8GRkZiIyMfPWGv4Lbt28X6/mIcsPnkd4UfBbpTcLn8c1nYmICjVaCpFQ1MrQ5J11O1+ix+0w8Yp8+n29vrZKhQ31rCOmPERX1OMd9i4qpqRKWZjIkQp9nwmg+i8WL036pNCnRAF8uz0woMnDgQHTu3BkA4OzsjCtXrmDNmjUwNTWFRqMx2scQ2Jubv3qmU7lcDgcHh1fevyDS0tJw+/ZtVK9eHWZmZsVyTqKc8HmkNwWfRXqT8Hl8eySlZiAlLQNlchmW/yQhHb9uj8SjZ8+D+5rvWWFAeyeozIo/mZ1hSL61SgGZLPeeYj6Lxe/GjRsl3QSiQlWiAb6trS0AwNHR0ajcwcEB+/fvh6+vL65du2a07dGjR0b7vgqJRPJaLwhehZmZWbGfkygnfB7pTcFnkd4kfB7fXHq9gPhkNXSCDErTnJPS3YxNQPjmS0hKfR7c+7raIegDZ8hNij9TvkQCmClNYK1SQlqA+fZ8FosPh+dTaVMya4L8P1dXV6hUKly4cMGo/Nq1a6hatSrq16+PK1euiEP5AeD48eNQqVSoU6dOcTeXiIiIiIqZTqfHs8R0pKm1yC1X/pmrD7Hgl7NGwX07vxro396lRIJ7qUQCK5UCZSxNCxTcExG9jhLtwTc1NcWgQYMQFhYGW1tbeHh4YOfOnThy5AjWrl0LLy8vLFy4EJ9++ikmTpyImJgYzJ8/HwMGDOBcGSIiIqJSLkOrQ3ySGhqtPsc6giDgjxN3sHV/lFgmk0oQ1NYZDd0qFUczszCRSWFjoYBSUaJ/ahPRO6jEf+uMGDECZmZmWLBgAR4+fIhatWohJCQEDRo0AACsXLkSM2fORI8ePWBtbY3evXtjxIgRJdxqIiIiIipKao0W8clqo7XrX6bT6fHz3v9w5OI9sczc1ATDunjAsWqZ4mimEQkApVwGa0slTGQlOlCWiN5RJR7gA0D//v3Rv3//bLdVq1YNq1evLuYWEREREVFJSVNnrnGvy2UZvNT0DCzfeglX7zwTyyrYmGFkd0/YlVMVRzONSCSAuakc1ioF53UTUYl5IwJ8IiIiIiIASE7TICklA/rcMuXHpyFs0wXcf5IiltWqbI3hXTxgYV780zgz59vLoTLjFFIiKlkM8ImIiIioxAmCgMQUDVLSM5BLbI9b9xKwZNMFo2R6Ps626NvOGXKTnDPsFxW5TAprzrcnojcEfxMRERERUYkyLIOXnkem/LNXH2HNjn+R8ULSvbaNq6N905qQFvOweM63J6I3EQN8IiIiIioxWp0e8UlqqDN0OdYRBAF/nryLrf/cEF8AyKQSfBxYB4093iuehr5AIgFUpnJYcb49Eb1hGOATERERUYnQZOiQkJz7Mng6nR7/+/M/HDr/QqZ8pQmGdnGHU7WyxdFMIzKpBJYqBVSm8mI/NxFRXhjgExEREVGxS9doEZ+kzjVTflq6Fsu3XULk7TixrLy1KUb18CqRTPkKEymsLJRQyot/rj8RUX4wwCciIiKiYpWSnoHEZE2umfKfJqQhbOMF3HshU35Ne2sM7+oBy2LOlC8BoFSYwMZCARnn2xPRG4wBPhEREREVm6QUDZLTNMil4x637ydiyaYLSEzRiGX16lRE33YuUBRz77lUAqhMFbBUyTnfnojeeAzwiYiIiKjICYKAhBQNUvNYBu/8tcdY9dtlo0z5gY2qoWOzWsWeKV8mlcDaQgEzJefbE9HbgQE+ERERERWp/CyDJwgC9p2Kxua/r4t1pFIJPm5TB36exZ8pX2EihY2lEnITzrcnorcHA3wiIiIiKjK6/18GLz2XZfB0ej02/HkNB8/FimVmShMM7eyOOtWLN1O+RJJ5biuVEjIph+QT0duFAT4RERERFYkMrQ7xSbkvg5em1mLl9sv49+ZTsayctSlGdvfEe+UtiqOZIqlEAgszOSxVxZvEj4iosDDAJyIiIqJCp9ZoEZ+shlaX84T7uMR0hG28gNjHyWJZ9UpWGNHNA1YqZXE0UySTSWBjoYSpgn8eE9Hbi7/BiIiIiKhQpakzkJCsyXWN+zsPMjPlJyQ/z5Rf16ki+rUv/kz5SrkM1hZKyE24BB4Rvd0Y4BMRERFRoUlO0yApJSPXNe4vXM/MlK/JeD50v03DaujkX7yZ8iUSwNxUDitzBaScb09EpQADfCIiIiIqFInJGiSna3JcBk8QBPx9Ohqb9r2QKV8iQa82TmjqZV9s7TSc10olh8qM8+2JqPRggE9EREREr0UQMpfBS1NrcwzudXo9Nv51HfvPxohlpkoZhnzoDpca5YqppZnkMimsLRRQcr49EZUy/K1GRERERK9MpxcQn5QOtUaX4xr36WotVv52GZejnmfKL2uVmSnfvkLxZcqXAFAqZLCxUEIm43x7Iip9GOATERER0SvR/v8a9+pc1rh/lpiOsE0XEPPoeab8apWsMKKrB6wtii9TvkQCqEzlsFIpICnGef5ERMWJAT4RERERFZgmI3ON+wxdzmvc332QhLBNF5CQrBbLvBwrYEAH12LNlC+TSmClUsDcVF5s5yQiKgkM8ImIiIioQNI1WsQnqXNdBu/ijSdYtf2yUe9+K9+q6NzCoVgz5StMpLC2UBb70ntERCWBAT4RERER5VtKegYSkzW5LoP3z+lo/LrvmphwTyqRoGdrRzTzrlxMrcwckm+qMIG1hRIyLoFHRO8IBvhERERElC9JKRokp2mQU8e9Xi9g475r+OfMC5nyFTIM/tAdrjWLL1O+VAJYmClgYS7nfHsieqcwwCciIiKiXAmCgIQUDVLTM3JcBi9do8Xq3/7FxRtPxLIyVkqM6uYF+4rFlylfJpPAWqWAmZLz7Yno3cMAn4iIiIhypNcLeJakhlqjzXEZvPgkNcI2XUD0wySxrKqtJUZ084SNZfFlyleYSGFjqYTchPPtiejdxACfiIiIiLKVn2XwYh5mZsp/lvQ8U75n7fIY0MENSkXxBNoSCWCmNIG1Sgkp59sT0TuMAT4RERERZZGfZfAuRz3Biu2XodY8fwHQsn5VdGnhUGyBtlQigaW5HBbmimI5HxHRm4wBPhEREREZyc8yeAfOxuB/f/4nzsmXSICPWjmhed3iy5RvIpPCxkIBpYJ/0hIRAQzwiYiIiOgFeS2Dp9cL2PzPdew7FS2WKRUyDOrkBvda5YuljRIASrkM1pZKmMikxXJOIqK3AQN8IiIiIgKQuQxeUpomx0z5ao0Oq3+/jAvXX8iUb6nEyG6eqGxrWSxtlEgAlakcVioFl8AjInoJA3wiIiKid5wgCIhPViNNrc0xuE9IzsyUf/fB80z5VWwtMbIYM+XLpJnz7VVmnG9PRJQdBvhERERE7zCdTo/4ZDXUGl2Oy+DFPkpG6KbzeJb4PFO+h0N5DOjoCtNimv8ul0lhbamEUs4l8IiIcsIAn4iIiOgdlaHNzJSv0eacKf/fm0+xYtslpL+QKT/Apwq6BdQulkz5EmTO8bexUELG+fZERLligE9ERET0DlJrtIhPVkOryzlT/sFzMfjfH9fEhHsSCdDjfUe08KlSLG2USgCVqQKWKjnn2xMR5QMDfCIiIqJ3TGp6BhJTNDkug6cXBGz95wb+PHlXLFPKZRjYyQ0eDsWTKV8mlcDaQgEzpbxYzkdEVBowwCciIiJ6hySlaJCcpkFOS9xrMnRY8/u/OHftsVhmbZGZKb+qXfFkyleYSGFjqYTchPPtiYgKggE+ERER0Tsgv5nyl2y+iDv3E8WyyhUtMLKbJ8pYmRZ5GyUSwFRhAmsLJWTFML+fiKi0YYBPREREVMrp9ALik9Jzz5T/OBlhGy8gLjFdLHOrVQ6DOrrBVFn0fzJKJRJYmMlhqeISeEREr4oBPhEREVEplqHVIz4pPddM+VduPcXybZeQrn6eKd+/bmX0aFkbMmnRZ66XySSwsVAW25J7RESlFX+LEhEREZVSmZnyNdDqcg7uD1+Ixc97/nueKR9At/drI8CnSrFkrlfKZbC2UEJuwiXwiIheFwN8IiIiolIoTZ2BhOTcM+VvPxCFvcfviGUKuRQDOrjBy7FCkbdPIgHMlCawVikh5Xx7IqJCwQCfiIiIqJRJTtUgKTX3TPlrd1zB2f8eiWVWKgVGdvdENTurIm+fVCKBpUoOCzPOtyciKkwM8ImIiIhKCUEQkJiiQUp6Ro6Z8hNTNAjffAG37j3PlG9fITNTflnros+UL5dJYW2hgJLz7YmICh1/sxIRERGVAjq9gIRkNdLV2hwz5d97kpkp/2nC80z5LjXKYvCH7jAr4kz5EgBKReZ8exMZ59sTERUFBvhEREREbzmtTo/4JDXUGboc61y9HYdlWy8hTa0Vy5p62aNna8ciz5QvkQAqUzmsVIpiSdxHRPSuYoBPRERE9BZTZ+iQkKRGRi6Z8o9evIef9lyFXv88U36XFg5o6Vu1yANumVQCS5UCKlN5kZ6HiIgY4BMRERG9tfKTKf+3g1HYc+x5pny5iRQDOrjC26likbdPYSKFtYUSCrmsyM9FREQM8ImIiIjeSvnJlP/jzis4c9U4U/6Ibp6oXqloM+Vnzrc3gY2FAjLOtyciKjYM8ImIiIjeIoIgICFFg9RcMuUnpWoQvvkibsYmiGXvlVdhZHdPlLM2K9L2SSWAOefbExGVCAb4RERERG8JnV7ITKanyTlT/oOnKQjdeAFP4tPEMufqZTHkQ3eYmRbtn34yqQRWKgXMOd+eiKhEMMAnIiIiegtkaPWIT0qHRptzMr3/7jzDsi0XkfpCpvwmnu+hV2unIh8qL5dJYW2phJLz7YmISgwDfCIiIqI3nDpDh/gkNbS5ZMo/duk+ftodaZRwr0sLB7Qq4kz5hvXtbSyUnG9PRFTCGOATERERvcHyypQvCAJ+P3QTu47eFsvkJlL0a++CenVsi7RtXN+eiOjNwgCfiIiI6A2VV6b8DK0O63ZF4tSVh2KZpbkcI7p5osZ71kXaNqlEAiuVHCozRZGeh4iI8o8BPhEREdEbJj+Z8pNTNQjfchFRMc8z5duVM8eo7l4ob1O0mfJNZFLYWCigVPBPSSKiNwl/KxMRERG9QfKTKf9hXCpCfz2Pxy9kyq9TrQyGdHYv8gz2SrkMNpZKmHC+PRHRG4cBPhEREdEbIj+Z8q/ffYbwLReRmv48U76fx3vo3aZoM+VLJICZ0gTWKiWkUs63JyJ6EzHAJyIiInoDqDN0SEhSIyOXTPnHL9/H+l3GmfI/9K+FNg2rFWmSO6lEAgszOSxVnG9PRPQmY4BPREREVMLykyl/x+Fb2HnkllhmIsvMlO/jXLSZ8mUyCaxVCpgpi3boPxERvT4G+EREREQlKDlNg6SUDOhzyKaXodVj/e5InPz3gVhmaS7H8K6eqGlftJnyFSZS2FgqITeRFel5iIiocDDAJyIiIioBgiAgMUWDlNwy5adlYOnmi7gREy+W2ZUzx8juXqhQhJnyOd+eiOjtxACfiIiIqJjp9QLik9VIV+eeKT9s43k8evY8U75T1TIY3NkNFkW49rxUIoGluRwW5tmfQxCEIp3vX1BvWnuIiEoS1zchIiIiKkZanR5xielIU2vRs1t7/PDtjCx1rkc/w+x1p4yC+0buleBTOQEhc78xqvv3X3vRs2s7tGnREPNnf4txo4Zg3Kghr9Q2E5kEZa2U2Qb3Dx48wJAhQxAbG5tl2/DhwxEeHv5K58yvmJgYODk5YcuWLQCAxMRETJ48GadPnxbrBAUFISgoKNv6W7ZsgZOTE2JiYoq0na9j0KBB+PrrrwvlWBs3boSTk1OWf1999VWu+714D4no7cMefCIiIqJi8nKm/K++mwtzlcqozsl/H2DdrivQ6p737XdqVguBjaph/OiFWY65eMEPqFy5KqZMnYnyFSpi3g/fZKmTH0q5DNYWSshNsu//OXr0KA4cOJClXKPR4Pjx4xg9evQrnfdVRUZGYvv27ejatatYNmNG1pclBs2bN8eGDRtQsWLF4mheiYuMjESNGjUwa9Yso/Ly5cuXUIuIqDgwwCciIiIqBtllyq/tWEf8WhAE7Dp6G78fuimWmcik6NvOGfVd7HI8bmJCAny6NIRXXZ9XapdEApibymFlrnil+fanT5+GSqWCs7PzK52/MDk4OOS4rWzZsihbtmwxtqZkRUZGwt3dHV5eXiXdFCIqRhyiT0RERFTEktM0iE/Kugxer/8foh8TE4P3m/rg5w3bcO/0elzfPQ1Re79E2aS/4VYjM1P+uFFDcOH8GVw4fwYBTerh/NnTCGhSDwCwbs0KBDSphwf372U5t0atxvq1K9C3dxe0CWiEoJ4f4pef1kKv10MqyVwCz9LMBCtXrkD79u3h4eEBLy8v9OzZE8ePHweQObz9s88+AwC8//77mDJlinj8AwcOoGnTppBIJDhx4gScnJxw7NgxBAUFwcPDA82bN8fGjRvx6NEjjBo1Ct7e3vD398fatWvFY+Q0fD4gIMDoXAYnTpxAnz59AAB9+vQRh5TnNrz85XNMmTIF/fr1w+bNm9GmTRu4ubmhU6dOOHjwoNF+5879H3v3HVdl+T5w/HMGh73c5UgRBBVQHKiYi0wtKVPTUHPlFvfOr19Ly5UrFBBXmkrl11FWaqaVI/coxVwpDswtsuEcOOf8/uDHo0e24qiu9+vVK7ifdT8Dz7me+76v+ze6detG7dq1ad68OZ9//jm9evVS6pV9zgcPHrTY7uG6pKenM3fuXFq1aoW3tzd16tShd+/enD59Otf65ufcuXMMGDCAOnXqUKdOHUJCQoiNjVWWm81mzp49WywvXeLi4pgyZQotWrTA29sbf39/QkJCLO5V9+7dGTNmDMOGDaN27dr07t0bgFu3bjFy5Ej8/f2pX78+kydPZv78+QQGBlocY926dbRt2xZvb2+aN2/OwoULMRqNj113If6NJMAXQgghhHhCzGYzCcl6ElMM+U6Dt/L7PwC4Gb0BKztXagYOoFOXd9n7y1bWfL4MgOGjJ+BezRP3ap6ERa7Ew9OLsMiVALwe1I6wyJWUKGnZ/dpsNvOf8SP5KmoVrwe9xbRZ82nWoiXLl0YQOncGJZyssbfVMWfOHCIiInjnnXdYtmwZH330EfHx8QwfPpy0tDSaN2/OoEGDAAgLC2Pw4MHKMXbt2kWzZs0sjjtq1CgCAwNZvHgxVapU4YMPPqBHjx54eHgQERGBr68vM2bM4MSJE490XWvWrMnkyZMBmDx5cr5d8/Nz8uRJli9fzrBhwwgPD0ej0TB06FASEhIAuHDhAr169QJg3rx5DB06lCVLlnD06NEiH2vcuHFs2LCB/v3789lnn/H+++/z559/Mnr0aMx5TaOQi4sXLxIcHMzdu3eZNWsW06ZNIzY2li5dunD37l0Arly5QkpKCtHR0bRu3ZqaNWvSunVrvvnmmyLV2Ww2M2DAAPbu3cuYMWNYvnw5Q4YMYf/+/Tmu+datW7G3t2fRokX07dsXg8FAz549OXbsGBMnTmTGjBmcOXOGzz77zGK7xYsX89///pdGjRoRGRlJt27dWLp0Kf/973+LVFchRBbpoi+EEEII8QQYTVnBfX6Z8k0mMydj7mJvTATAvkx1Alq/y4AOvjjYWnHm5DEO7NtDv4FDqVzFDXs7BwBqePtY/L9U6bLKzw86dGAfR48cZNKH0wls2RqA+vUbYm9nx9LF4Qzo9x4eHh5KS+uDLc7W1tYMHTqUs2fPUrt2bSpVqgRA9erVqVChAgCxsbHExsbSuHFji+N27NhRacW1s7Ojc+fO+Pr6Mnz4cAC8vLz48ccfOXbsGL6+vkW+tg4ODkp3fHd393y75ucnKSmJjRs3KudmZ2fHu+++y4EDB2jdujWLFy/G0dGRZcuWYWubNS2hm5sbwcHBRTqOwWAgJSWFSZMm8frrrwPg7+9PcnIyM2fO5M6dO5QuXbpQ+woLC8PW1paVK1fi4JD1PDRq1IiWLVuybNkyxo8fr/QKuHr1KhMmTECr1fLNN98wfvx4DAYDnTt3LtSxbt26ha2tLePHj6devawhIA0aNODKlSusXbvWYl0rKyumTJmCTpeVoHH9+vXExMSwYcMGvL29AWjYsCEtW7ZUtklKSlJeLE2aNAmAl19+GRcXFyZNmkTv3r3x8PAoVF2FEFkkwBdCCCGEKGYZmSbik9IxZJpyLDOZzMTeTOLPK/HEJ+ux1WWSnWaveg1vhr3jpyS6K126LDevX3/kehz/7SgajYZmLbKCKpUK7G2seKdTe5YuDufQoUN4eHgwd+5cIKs7dkxMDJcvX+aXX34BsoLTvOzevRs/Pz8cHR0tyv38/JSfS5YsCUCtWrWUMldXVyArwHuWSpQooQT3AOXKZeU6SEvLmr3gwIEDNG3aVAnuIevcypcvX6Tj6HQ6li9fDsDNmze5ePEily5dyvcam0wmTKb7z49KpUKj0XDgwAH8/f2xsbEhMzMTyHrhUa9ePfbt2wdA/fr1iYyMpEGDBtjZ2QHQpEkT4uLiWLBgAZ06dcJkMln0HMje/4PKli3LqlWrMJvNXL16lcuXLxMTE8OxY8dy1NnNzU0J7rOvXcWKFZXgPrueLVq0UIYz/Pbbb6SnpxMYGKicC6B04d+7d68E+EIUkQT4QgghhBDFSG/IJD5Zb5EFP9uZS3H8cOASsTeSSEnP5OGe2c3qVbHIYq9WqzCZc74kKKzEpAScXVzQaDRo1Coc7XXY21hhY5WVST47wI6OjmbKlClER0dja2uLu7s7L774IkC+3cd37dpF06ZNc5Rntyw/6MEg+XnxcJ1Uqqwkg9mBdVxcnPKC4kGPkol+z549TJ8+nZiYGOzt7fHy8lKC79yu8cSJE/n666+V38uXL8/PP/9MfHw8W7ZsYcuWLTm2yU4iWLJkSVq0aJFjebNmzdi3bx937txh1KhRHDp0SFnm7+/P6tWrc2zz7bffMm/ePK5fv46LiwvVq1fHxsYmx3r2D80Gce/evVyv3YNl8fHxAPTvn/u0jrdu3cq1XAiRNwnwhRBCCCGKSWp6BokpOZPpQVZwv+aH0ySmGDBkWAbt9rZWQFYLe3FycnQmIT4ejcqMq5Mt1lZZLbTZgZOrqyvJycn07dsXT09PNm/ejJubG2q1ml27drFt27Y8963X6zl48CCjR49+7Ho+HFhnS0lJeex9P45y5cpx586dHOV3797Fzc0NyL/u2UHvlStXCAkJoWXLlixevJiKFSuiUqmIiopiz549uR57yJAhdOvWTfk9u3Xc0dGRgIAAZQjEg7TarK/2R44cITY2lvbt21ss1+v1aDQanJ2dmTJlisX1fThAz97P+PHj6d69O3369KFs2bIAfPLJJwXmIShbtiyXLl3KUZ6dJwDAyckJgDlz5lC5cuUc68qUfkIUnSTZE0IIIYQoBkkpBhKS9bkG9yaTma37LxKfpM8R3NvoNPD/gX1BudbUmqJ9davlVwej0cjh/buU4B6yWmUB6tatS0xMDPHx8fTo0QN3d3fU6qxjZGeTzw5cs8uzHTx4EBcXFzw9PYtUp9xkt/jfuHFDKbtw4YLSwpubh7uTPwn169dnz5496PV6pezUqVMWGeRzq3tCQgIXLlxQfj958iR6vZ7+/ftTqVIl5aVAdnCfWwt+hQoV8PHxUf7Lvs7+/v6cP3+e6tWrK8u8vb1ZuXIl27dvB7K6x0+YMIGLFy8q+zOZTGzbtg0/Pz90Oh1ubm4W+89+YfGg3377DZPJxNChQ5Xg3mg0KkMBHn6p8SB/f3+uXr1qMUtAenq6xQuNWrVqYWVlxc2bNy3qotVqmTdvXo5ZFYQQBZMWfCGEEEKIx2AymUlI0ZOmz9nlPtu5K/e4cDXBotu+VqNCo1ah0aix02V9JYtLSM/3WA4Ojpw6eYJjRw/h4eGF4/+3gOZGrVLRMrAF3zRowOTJ/+X27Vt4eXlx6NAhli5dSvv27XF3dycpKQkHBwciIyPRarVotVq2bdvG+vXrgfvj0bNbW7dv307Tpk3ZvXt3rt3zH0WDBg2wsbFh5syZDB8+nJSUFBYsWICLi0ue22SP+9+5cyfOzs54eXkVS10eNHDgQLZs2ULfvn157733SExMJDQ0FLVarQTpnp6evPDCC4SHh+Pg4IBKpWLx4sUW3f9r1qyJVqtl9uzZvPfeexgMBjZu3MjOnTsBSE1NLXSdBg8eTHBwMAMGDKBLly5YW1uzdu1aduzYwYIFCwAIDg7mq6++YuDAgQwfPhxbW1u++OILzp07R1RUVKGPlZ0AcerUqXTs2JGEhASioqI4c+aMUu/chmMABAUFsWTJEkJCQhg+fDhOTk6sWLGCu3fvKsM/XF1d6du3L6GhoSQnJ9OgQQNu3rxJaGgoKpXqidxTIf7ppAVfCCGEEOIRZRpN3EtMJzWX8fTZbsen8fnmUxbBvbWVmrIl7JQu+RpN1g/phvzn/n6rY2c0Wi3vjxnGoQN781xPpQJXJ2ucHKxZvHgxwcHBrFy5kv79+/PDDz8watQopk+fDmQFyhEREZjNZoYPH864ceO4du0aa9aswd7eniNHjgBZQXhAQABz585l1qxZ7N69O8f0eI/KyclJmfs8JCSE0NBQQkJCLBK0PczDw4OgoCCioqIYM2ZMsdTjYS+99BLLly9Hr9czbNgw5s+fT79+/ShdurTSpV2j0bBgwQJKlSrFqFGjmDZtGm3btqVVq1YW+5k7dy43b95k0KBByhR/q1evRqVSKde4MLy8vIiKikKlUjFu3DiGDRvG7du3CQ8PV45ZqlQpoqKi8PT05OOPP2bEiBGkpaWxcuVKi2SHBWnQoAGTJ0/mt99+o1+/fsycOZMXX3yRsLAwgHy76Wu1WpYvX06NGjX48MMPGTduHB4eHrz66qtK7gGAESNGMGHCBLZv306/fv2YPXs2devWZc2aNTmSNwohCqYyF2XizX+A6OhoAHx8ck4l8ySkpqZy+vRpqlevbvGPmRDPgjyP4nkhz6J4njzq82jIMBKfpCfDmHc35Zi/Eli04ThJqRlKmZ2NlpJO1koLcNa+TOgzjAzs4MtLL+TdKl8QFWBtpcHJwdoiWZ94NPv378fKykqZIg4gMTGRgIAAxo0bR48ePYr1eP+kfxv//PNPYmJiaNWqlcWz/vbbb1OuXDnlJcGz9rRjAyGeNOmiL4QQQghRRGn6DBKSc0+ml+3omZus/P4UGQ9MlaezUlPCUWcR8JjNZlL0GZQv7UDFso/eYpk9BZ6TveX+xaP7448/WLBgAaNGjaJmzZrEx8ezYsUKHB0dCQoKetbVe66lpqYyfPhwunbtyquvvorRaGTLli2cPHnyifW4EEJIgC+EEEIIUSTJaQaSUjIw5dEJ0mw28+PBy3y9836SNY1axSv1K/LbudskpGZgb22FVqsiMzMruLfVaWnTsDJq9aMF5g9OgSeKT/Z4+S+//JLr169jZ2eHv78/M2bMUKakE7mrVasWn376KcuXL+ebb77BbDZTo0YNli1bRsOGDZ919YT4x5IAXwghhBCiEMxmM4kpBlLSM/Icb280mvhi21n2nrimlNnZaBnYwZdqlVypUaUkPxy4xM27qaTqTWjUasqXdqBNw8p4VX60gFGnVePsYI3O6slnlf+3UavVDB48mMGDBz/rqvwttWnThjZt2jzragjxryIBvhBCCCFEAYwmM/FJevSGTPLqlJ+ansGSr6M5c/meUlbaxZaQTrUoVzIrIZtX5RJUq+RK7M0kklMzcLCzomJZx0dquVepwEanxdnBGs0jtvwLIYT4Z5EAXwghhBAiH5lGU1Zwn5F3hvs78WmErz/O9TspSlnVCs4M6uCLg53OYl21WvVYifQgawo8RzurHPsWQgjx7yYBvhBCCCFEHvQZRhIKyJR/8VoCEestM+XXq16Wnm2rY6Ut/m7zWo0aFwcd1jr5GieEEMKSfDIIIYQQQuSiMJnyj525xYrv/7DIlP96QGWCmrihLuZM9irAWqfB2cEarUamwBNCCJGTBPhCCCGEEA9JSjGQnGYgr9jebDaz/dAVvv7lvDImX61W8W4bLwJ8Xyz2+qhVYCdT4AkhhCiABPhCCCGEEP/PbDYTn6wnTZ+Zb6b8r7afZc/vD2TKt9YyoIMPni8V/9RpGrUKZwcdttYyBZ4QQoj8SYAvhBBCCAGYTGbiEtPRG4x5ZspPS89k6aZoTl2MU8pKOdswpHNtJVN+cdJp1bg4Wj+RsfxCCCH+eSTAF0IIIcS/nkar416SHrU276z0cQnphK3/nWu372fKdyvvzKCOvjgWczZ7lQpsrbU421s/0hR6Qggh/p0kwBdCCCHEv5ohw0RCaiYOGUZs8/hmdPl6IuHrj5OYYlDK6niVoVfbGuisird1Xa1S4WhvhYOtTIEnhBCiaCTAF0IIIcS/VkqagXtJ6aSn6/Nc5/dzt1n+7UmLTPltGr3Em02rFnumfJkCTwghxOOQTw8hhBBC/OuYzWYSUwykpGfkOQ2e2Wzmp8OxbPj5T4tM+d1ae9G4VvFnyrex0uDsKFPgCSGEeHQS4AshhBDiX8VoMpOQrCddn5lnMj2jycTa7efY/dtfSpmttZYB7X3wqly8mfJV2VPg2elkvL0QQojHIgG+EEIIIf41Mo0m4pP06DOMea6Tps9k2aaT/BFzVykr6WxDSKdavFjKoVjrI+PthRBCFCcJ8IUQQgjxr6DPMJKQpCfDaMpznbjEdMLXHeev28lKWeUXnBj8ti9O9tbFWh+tRoWLg7WMtxdCCFFs5BNFCCGEEP94afoMEpINeY63B4i9mczy78+QkHw/U76fZ2l6B9Us1kz5KsBaxtsLIYR4AiTAF0IIIcQ/WlKKgeQ0A/nE9ly8kc72309ieCBTfqsGL/FW8+LNlJ893t7ZXoeqmDPwCyGEEBLgCyGEEOIfyWw2E5+sJ02fiTmP4N5sNrPrt2tsPpKglKlVKrq09qRJ7fLFWh+NWoWjvQ57G6ti3a8QQgiRTQJ8IYQQQvzjGI0m4pP16A3GfDPlr9vxJzuPXVXKbHQa+rf3oUaVksVaH51WjZODNdbF2NVfCCGEeJgE+EIIIYT4R8nINBKfpLfobv+wdH0my749yckL9zPluzrqGNLZj/Kliy9Tvgqw1mlxcdChkfH2QgghnjAJ8IUQQgjxj6E3ZHIvWY/RmPeA+3uJ6YSvP87VW/cz5Zdx1jK4ky/linEaPLUKHGx1ONhZyXh7IYQQT4UE+EIIIYT4R0hJM5CYkoEprwH3wJUbSYSvP05Csl4p86lagoBqWpzti28ueo1GhbO9DltrGW8vhBDi6ZEAXwghhBB/a2azmcQUAynpGXkm0wM4cf4OyzedRJ9hVMpa+lfitQbluXgxptjqY22lwdnBGiutdMkXQgjxdEmAL4QQQoi/LaPJTEKynnR9Zp7J9AB+ORrL/3acU14AqFUqgltVo6lfBdLS0oqlLtlT4DnZ6VCrpUu+EEKIp08CfCGEEEL8LWUaTcQn6S1a5B9mMplZ//Of/HwkVimz0Wno95YPNd2KL1O+Rq3C0c4Ke9vi6+YvhBBCFJUE+EIIIYT429FnGElI0pNhzCdTviGTz779gxPn7yhlro7WhHSqRYUyjsVWF5kCTwghxPNCAnwhhBBC/K2k6TNISDZgNOXdKT8+SU/E+uNcuZmklFUq68jgt2vh4mhdLPVQpsBztEYjXfKFEEI8ByTAF0IIIcTfRlKKgeQ0A/nE9ly9mZUp/17S/Uz5vu6l6POmN9a64mllV2ePt7fXyRR4QgghnhsS4AshhBDiuWc2m4lP1pOmz8w3U/7JC3dYuukkesP9cfmv1K9IxxYexZb4TqNW4WSvw85GpsATQgjxfJEAXwghhBDPNaPRRHyyHr3BmG+m/F3HrvLV9rPKCwCVCt5pWY3mdSsWW12sNGqcHWW8vRBCiOeTBPhCCCGEeG5lZBqJT9JjyMw7mZ7JZGbjL+fZcfiKUmat09C3nTc+VUsVSz1U/79PFwdrNBqZ314IIcTzSQJ8IYQQQjyX9IZM7iXrMRrzbrfXG4x89t1Jjv95P1O+i6M1Q96uRYWyxZMpX6UCexlvL4QQ4m9AAnwhhBBCPHdS0gwkpmRgymfAfUKynvD1x7ly436m/IplHRn8ti+ujjbFUg+1SoWjvRUOMr+9EEKIvwEJ8IUQQgjx3DCbzSSmGEhJz8g3md5ft5IJW/879xLvZ8r3qVqKPu1qYqMrnq83Go0KFwfrYtufEEII8aTJJ5YQQgghngsmU1am/HR9Zr7J9P6IucvSb6JJfyBTfou6Fej0SrViy5Sv06pxcbTGSivJ9IQQQvx9SIAvhBBCiGcu02giPkmPPsOY73p7fv+LL7edVbruq1TQ6ZVqBNYrnkz5Wcn0tLg4WqMpppcFQgghxNMiAb4QQgghnil9hpGEJD0Zxnwy5ZvNfL3zPNsPPpAp30pDn3be+LoXT6Z8tUqFva0k0xNCCPH3JQG+EEIIIZ6ZNH0GCckGjKa8O+UbMoys+O4Pfjt3WylzdrAm5O1aVCpXPJnydVZWONtb4+xgXSz7E0IIIZ4FCfCFEEII8UwkpxpISjWQT2xPQrKeiA0nuHw9USkrX9qBIZ1q4epUPJnyrbQaXByssLWR8fZCCCH+3iTAF0IIIcRTZTabSUgxkFpApvxrt5MJW3ecuMR0pcy7akn6vumNjXXOrzBms7lIXetVZHXz19lac/dGZlFOIVdFPf6T9rzVRwghxJOnftYVEEIIIcS/h9FkJi4xndS0/IP7t996naEjRlsE983qlGdQR99cg/u9v+5i5scfWJT9vGMbwR3b0rpFQ+Z9Mo2RQ/ozckh/ICs5n72tFSWcbdCoVZjzq0wBbty4Qf/+/fnrr79yLBs0aBCLFi165H0XxtWrV/H09GTjxo0AJCYmMm7cOI4cOaKs0717d7p3757r+hs3bsTT05OrV68+0Xo+jgfr/7jWrVuHp6dnjv+mTp2a73YTJkwgMDCwWOoghBBPirTgCyGEEOKpyMg0Ep+kx5CZdzI9gF+P/4VTzS6gzhoPrwLefsWDwHoV82yRXv9VVI6yBfNnUaFCJSb8ZwqlSpdh7qyPAdCoVTjZ67CzsXq8E/p/+/btY9euXTnKDQYDBw4cYOjQocVynMI6ffo0mzZtomPHjkrZBx98kOf6zZs3Z+3atZQpU+ZpVO+ZO336NFWqVGHmzJkW5aVKFU+yRiGEeJYkwBdCCCHEE6c3ZHIvWY/RmHdLuclsZtOuC2w7cBmd44sA6KzUvPeGN7WrlS7yMRMTEqjXoSG169RTylQqKOFkg87qyY+3P3LkCPb29lSvXv2JH6sg7u7ueS4rUaIEJUqUeIq1ebZOnz6Nj48PtWvXftZVEUKIYidd9IUQQgjxRKWkZxCXmH9wb8gwsuybk2w7cBmAmJ9mcPfkOnq+Wp5R77Vh58/b+XDSONq+2oR2r7VgzqyPSEtLA2DkkP4c//0ox38/SuDLdfn92BECX64LwKoVSwl8uS43b1xDrVZhpVUrwb1eryc8PJz27dvTs2dP3nzzTZYsWYLJdL+HgdFoZMmSJQQFBeHr60vt2rUJDg7mwIEDQFb39vfffx+AV155hQkTJijb7tq1iyZNmqBSqTh48CCenp7s37+f7t274+vrS/PmzVm3bh23bt1iyJAh+Pn50axZM1auXKnsI6/u84GBgRbHynbw4EF69OgBQI8ePZRu7fl1cX/4GBMmTKBXr15s2LCB1q1b4+3tTbt27di9e7fFdr/99hvdunWjdu3aNG/enM8//5xevXop9co+54MHD1ps93Bd0tPTmTt3Lq1atcLb25s6derQu3dvTp8+nWt983Pu3DkGDBhAnTp1qFOnDiEhIcTGxirLzWYzZ8+efayXLmvXrqV58+b4+vrSs2dPTp06ZbH88OHD9OnTh/r16+Pt7U1gYCALFy60eK6+//573nzzTXx9fWnYsCFjxozh5s2bFvtZt24dbdu2xdvbm+bNm7Nw4UKMRuMj11sI8e8gAb4QQgghnpjEZAMJyXpM+YxxT0wxMP/LYxw7e0sp02pU+FQtRfnSDgDMnz2dsuVe4KMZc3mna3e2fr+JNZ8vA2D46Am4V/PEvZonYZEr8fD0IixyJQCvB7UjfPFKXqrwIlZatdLF32w2M3DgQJYtW0b79u0ZM2YMr776Kp9++qlFd/Y5c+YQERHBO++8w7Jly/joo4+Ij49n+PDhpKWl0bx5cwYNGgRAWFgYgwcPVrbdtWsXzZo1szjXUaNGERgYyOLFi6lSpQoffPABPXr0wMPDg4iICHx9fZkxYwYnTpx4pOtds2ZNJk+eDMDkyZPz7Zqfn5MnT7J8+XKGDRtGeHg4Go2GoUOHkpCQAMCFCxfo1asXAPPmzWPo0KEsWbKEo0ePFvlY48aNY8OGDfTv35/PPvuM999/nz///JPRo0cXKTfCxYsXCQ4O5u7du8yaNYtp06YRGxtLly5duHv3LgBXrlwhJSWF6OhoWrduTc2aNWndujXffPNNoY5x48YNwsLCGDFiBPPmzSMhIYHu3btz7do1AM6cOUOvXr1wcXFh/vz5LFq0iHr16hEWFsbWrVsBOHr0KOPGjaNVq1YsXbqU999/nwMHDjB69GjlOMuXL+e///0vjRo1IjIykm7durF06VL++9//Fvp6CCH+naSLvhBCCCGKnclkJiFFT5o+M/9M+XeSCV93nLsJ95Pp1ahSgvjD1ljr7nejbxDwMoOGjASgTj1/jhw+yIF9e+g3cCiVq7hhb5f1IqCGt4/F/8uUKUvjRvWwtbYcb79792727dvHvHnzaNGiBadPn6Zz5844OjoSGhqqBN23bt1i5MiRFi3O1tbWDB06lLNnz1K7dm0qVaoEQPXq1alQoQIAsbGxxMbG0rhxY4vjduzYkd69ewNgZ2dH586d8fX1Zfjw4QB4eXnx448/cuzYMXx9fQt5te9zcHBQuuO7u7vn2zU/P0lJSWzcuFE5Nzs7O959910OHDhA69atWbx4MY6OjixbtgxbW1sA3NzcCA4OLtJxDAYDKSkpTJo0iddffx0Af39/kpOTmTlzJnfu3KF06cINzwgLC8PW1paVK1fi4JD1PDRq1IiWLVuybNkyxo8fr/QKuHr1KhMmTECr1fLNN98wfvx4DAYDnTt3zvcYRqOR8PBw5d7UqlWLli1bsnr1asaPH8+ZM2cICAhg9uzZqNVZ7WiNGzfm559/5uDBg7Rt25ajR49iY2ND//790el0ALi4uBAdHY3ZbCY1NZVly5bxzjvvMGnSJABefvllXFxcmDRpEr1798bDw6NI11kI8e8hAb4QQgghilWm0UR8kh59Rv7dic9cimPx19Gk6e9PUedXrTQBPi+y5yuTxYuBmjV9LLYtXbosN69fL7AuttbaHME9wKFDh9BqtbRp0wa9Xq+Uv/nmm4SGhnLo0CE8PDyYO3cuAHFxccTExHD58mV++eUXICs4zcvu3bvx8/PD0dHRotzPz0/5uWTJkkBWkJjN1dUVyAqwn6USJUoowT1AuXLlAJRhEQcOHKBp06ZKcA9Z51a+fPkiHUen07F8+XIAbt68ycWLF7l06VK+19hkMll0d1epVGg0Gg4cOIC/vz82NjZkZmY9Uw4ODtSrV499+/YBUL9+fSIjI2nQoAF2dnYANGnShLi4OBYsWECnTp0wmUwWPQey9w9QsWJFixcvpUuXpnbt2hw+fBiAt956i7feegu9Xs/Fixe5fPkyp0+fxmg0kpGRodRh/vz5BAUF0bp1a5o1a8bLL79Ms2bNSE1N5c8//yQ9PZ3AwEDlPAAlg//evXslwBdC5EkCfCGEEEIUG32GkYQkPRnG/DPl7z1+jahtZzCZ7gdSpV1tuXgtgQt/xZOUmsHpS3e5cDUeAGsbG4vt1WoVJnPux1AB1rqsrzgaTe6jERMSEnB1dVUCN6UO/99anB1gR0dHM2XKFKKjo7G1tcXd3Z0XX8xKAJhf9/Fdu3bRtGnTHOXZLcsPejBIfl48XKfsoQ3ZgXVcXJzyguJBj5KJfs+ePUyfPp2YmBjs7e3x8vJSgu/crvHEiRP5+uuvld/Lly/Pzz//THx8PFu2bGHLli05tslOIliyZElatGiRY3mzZs3Yt28fd+7cYdSoURw6dEhZ5u/vz+rVq/M8v5IlS3L9/182paen89FHH7Fp0yYyMzOpUKECfn5+aLVa5Vz8/PxYsmQJK1euZMWKFSxZsoRSpUoxcOBAOnbsqDx7/fv3z/V63bp1K9dyIYQACfCFEEIIUUzS9BkkJBswmvLPlP/t7hh+2H9JKdNqVNjZWJGuz8Te1gqtRoUKSE3P5JvdF4pUB5UK7G2scLLX5bues7Mz9+7dy5G0LDt4cnV1JTk5mb59++Lp6cnmzZtxc3NDrVaza9cutm3blue+9Xo9Bw8etBhT/ageDqyzpaSkPPa+H0e5cuW4c+dOjvK7d+/i5uYG5F93e3t7IGtMfEhICC1btmTx4sVUrJg1FWJUVBR79uzJ9dhDhgyhW7duyu/Z3dwdHR0JCAhQhkA8SKvN+sp75MgRYmNjad++vcVyvV6PRqPB2dmZKVOmWFzf7LoCSg6CB92+fVt5gTBt2jS2bdvGp59+SkBAgPKiolGjRhbbNGnShCZNmpCWlsaBAwdYtWoVH3/8MV5eXsrx5syZQ+XKlXMcT6bzE0LkR5LsCSGEEOKxJacaiE/S5xvcZ2Qa+ezbkxbBvZO9jvJlHDCbzbg46NBp1ahVKlQq0GnVSjf/gnKtqTVqVCpwdrDG2cFaCS7z4u/vT2ZmJj/88INF+bfffgtA3bp1iYmJIT4+nh49euDu7q6Mqc7OJp8duGaXZzt48CAuLi54enrmX+lCyG7xv3HjhlJ24cIF4uPj89zm4V4JT0L9+vXZs2ePxfCGU6dOWWT7z63uCQkJXLhw/6XNyZMn0ev19O/fn0qVKin3LTu4z60Fv0KFCvj4+Cj/ZV9nf39/zp8/T/Xq1ZVl3t7erFy5ku3btwNZQwsmTJjAxYsXlf2ZTCa2bduGn58fOp0ONzc3i/1nv7CArER+V65cUX6/fv06v/32Gw0aNACyEug1aNCAli1bKsH9yZMniYuLU56XWbNm0bFjR8xmM7a2trRo0YLx48cDcO3aNdzd3bGysuLmzZsW9dBqtcybNy/HjApCCPEgacEXQgghxCMzm80kpBhITc/INwhPSjWwaMMJYv663wL6Yil73mpWlS9/PIu9rVXOoFylwu7/u9rHPZCELzdOjo6c+iOaE78doUaNGjg7O+e7ftOmTWnQoAGTJk1iwIAB2NjY8PPPP7Ny5Urat2+Pu7s7SUlJODg4EBkZiVarRavVsm3bNtavXw/cH4/u5OQEwPbt22natCm7d+/OtXv+o2jQoAE2NjbMnDmT4cOHk5KSwoIFC3Bxcclzm+xx/zt37sTZ2RkvL69iqcuDBg4cyJYtW+jbty/vvfceiYmJhIaGolbfn6nA09OTF154gfDwcBwcHFCpVCxevNii+3/NmjXRarXMnj2b9957D4PBwMaNG9m5cycAqampha7T4MGDCQ4OZsCAAXTp0gVra2vWrl3Ljh07WLBgAQDBwcF89dVXDBw4kOHDh2Nra8sXX3zBuXPniIqKKvAY1tbWDBo0iJEjR2I0GgkNDcXFxYWePXsC4Ovry9atW/nyyy+pWrUqZ86cYdGiRahUKuV5adiwIStWrGDChAm8+eabZGRksGzZMlxcXPD39+fatWv07NmT0NBQkpOTadCgATdv3iQ0NBSVSvVE7qcQ4p9DWvCFEEII8UiMJjNxiemkpuUf3N+4m8KsVUcsgvvqlUsw9t16aNRqjCYTWk3uLe6a/y9PN+SesE8F2Og09OrZAysrK/r165djvvZct/v/YDM4OJioqChmz57Njh07GDVqFNOnTweyAuWIiAjMZjPDhw9n3LhxXLt2jTVr1mBvb8+RI0eArCA8ICCAuXPnMmvWLHbv3p1jerxH5eTkpMx/HhISQmhoKCEhIXh7e+e5jYeHB0FBQURFRTFmzJhiqcfDXnrpJZYvX45er2fYsGHMnz+ffv36Ubp0aaWLuUajYcGCBZQqVYpRo0Yxbdo02rZtS6tWrSz2M3fuXG7evMmgQYOUKf5Wr16NSqVSrnFheHl5ERUVhUqlYty4cQwbNozbt28THh6uHLNUqVJERUXh6enJxx9/zIgRI0hLS2PlypUWyQ7zUqNGDTp16sSHH37IuHHjqFSpEl988YXSRX/ChAm0bNmSTz/9lAEDBrBu3ToGDRpE586d+e233zAajTRr1ow5c+bw559/MmTIEEaNGoWtrS2rVq1SXkyFhIQwYcIEtm/fTr9+/Zg9ezZ169ZlzZo1ORI3CiHEg1Tmokww+g8QHR0NgI+PTwFrFo/U1FROnz5N9erVla5aQjwr8jyK54U8i39/GZlG4pP0GDLzT6Z39vI9Fm88QeoDmfJfrvUiXVp5otGouXw9kciNJ7DWadBpc7Y7GDJM6DOMDOzgy0svOFkse3C8fUFd8vMjz2PR7d+/HysrK+rVq6eUJSYmEhAQwLhx4+jRo8czrN3flzyLT9/Tjg2EeNKki74QQgghikRvyOResh6jMf82gv3R11mz9bTFuPwOLdx51f/+WOuKZR0pW9KOv24nY/VQoG42m0nRZ1C+tAMVy1q2WmrUKhztddjb5JwCTzx5f/zxBwsWLGDUqFHUrFmT+Ph4VqxYgaOjI0FBQc+6ekII8a8lAb4QQgghCi0lPYPEZAOmfDoAms1mvtsTw5Z9l5QyK62aXkE1qOtV1mJdtVpFm4aVidp2hvgUA/bWVmi1KjIzs4J7W52WNg0ro1bfD/ytNGqcHa2xtnryyeRE7rLHy3/55Zdcv34dOzs7/P39mTFjhtJdXQghxNMnAb4QQgghCiUx2UByuiHf8fYZmUZWbTnN4VM3lTJHOysGv12LKi/mnvjOq3IJurX24ocDl7h5N5VUvQmNWk350g60aVgZr8r3A0ZrKw0ujtZo85jfXjwdarWawYMHM3jw4GddFSGEEA+QAF8IIYQQ+TKZzCSk6EnTZ+Yb3CenGojceILzV+8n0ytX0o4hnWpTysU27w3JCvKrVXIl9mYSyakZONhZUbGso0XLvY2VBhcnGzTqRx9vL4QQQvyTSYAvhBBCiDxlGk3EJ+mV+ejzcjMulbB1v3P7XppS5vmSKwPa+2BXyHHyarUqRyI9yMqUb63T4upobRHwCyGEEMKSBPhCCCGEyJUhIytTfoYx/0z5f165R+TGE6Sk38+UH+D7Al1bez12V3oVYGujxcXB+rEy5QshhBD/BhLgCyGEECKHNH0GCckGiwz4uTl48jqrtlhmyn+rWVVaN3zpsQPy4poGTwghhPi3kABfCCGEEBaSUw0kpRrIL7Y3m81s3nuR73+9qJRpNVmZ8utVL5v3hoWkUoGjrQ5He91j70sIIYT4t5AAXwghhBBAVtCekGIgNT2jgEz5JtZsPc3BP24oZQ62WZny3crnnim/KNQqcLTT4WAnwb0QQghRFBLgCyGEEAKjyZyVTM+QSX6d8lPSMojceII/Y+OVsnIl7QjpVJvSBWTKLwyNWoWzgw5b68Il5hNCCCHEfRLgCyGEEP9yGZkm4pPSMWTmn0zv1r1Uwv73O7ceyJRfrZILAzr4Yl/ITPn50WnVODtYo7PSPPa+hBBCiH8jCfCFEEKIfzG9IZP4ZD2ZxvyT6Z2PjWfRxhOkpGUoZY18XqBbm8fPlA//P8e9ozWaYtiXEEII8W8lAb4QQgjxL5WSnkFiigFTAZnyD5+6weebT1m8BHizqRuvNapcLJnyba21ONvLHPdCCCHE45IAXwghhPgXSkw2kJJecKb8rfsu8e2eGKVMq1HRs20N6tco99h1UKvAQTLlCyGEEMVGAnwhhBDiX8RkMpOQoidNn5lvpvxMo4k1W89w4OR1pcze1opBHX1xr+Dy2PWQZHpCCCFE8XsuAvybN2/StGnTHOUzZsygQ4cOTJo0iXXr1lksK1++PD///PPTqqIQQgjxt2c0mohP0pOeYcx3vZS0DBZ/fYJzV+KVsrIl7AjpVIsyrnaPXQ+NRoWrgzXWuufia4gQQgjxj/FcfLKeOXMGa2trduzYYTGWz9HREYCzZ88ycOBA3n33XWWZRiMZdoUQQojCMmQYiU/Sk2HMP1P+7XuphK07zs24VKXMo6ILAzv4Ym/7+K3tVho1rk7WWGnlc1wIIYQobs9FgH/u3DkqV65MmTJlciwzm82cP3+e/v37U7p06WdQOyGEEOLvLU2fQUKyAWMByfQuXI1n0YYTJD+QKb9BzXK8+1p1rLSPn91ep1Xj6mRTLFn3hRBCCJHTcxHgnz17lqpVq+a67MqVK6SmpuLm5vaUayWEEEL8/SWnGkhKzcCU34B74Mjpm6z8/hSZD7Twv/FyFV5vXOWxM+UD2Og0uDjaoJFM+UIIIcQT81wE+OfOncPV1ZVu3bpx8eJFXnrpJQYNGkTTpk05d+4cAKtXr2b37t2o1WqaNm3KyJEjlS78RWU2m0lNTS14xWKQlpZm8X8hniV5HsXzQp7FpyMpNYOUtPyDe7PZzI7Df7Fl/xWlTKNREdzSnXpepUlPT3+sOqhUYGdthY1WjT79+bzf8jyK54U8i0+f2WwulpeYQjwvnnmAn5mZSUxMDO7u7kyYMAEHBwc2b95M//79WbFiBefOnUOtVlOmTBkiIyO5cuUKn3zyCX/++Seff/45anXRu/llZGRw+vTpJ3A2ebt06dJTPZ4Q+ZHnUTwv5Fl8MrRaK1L1JlLSDBhNeY+5N5rM7IxO5HTs/SDe2kpF23ouuFolcuFC4mPVQ6PR4GhnjY2VmWuZmY+1r6dBnkfxvJBn8enS6WSqTvHPoTKbC+iz9xSkpKSg0WiwsbFRyvr27QvAkiVLSEhIwNXVVVl2/PhxOnfuzP/+9z9q1apVpGNFR0djNptxd3cvnsoXIC0tjUuXLlG5cmVsbW2fyjGFyIs8j+J5Ic/ik2M0mklIMZBuyD+gTk3PZMXmM5y/ej+IL+VsQ/921Snt+vj3RK1S4WCnw8H2mbclFEieR/G8kGfx6Tt//jwqlQofH59nXRUhisVz8alrb2+fo8zDw4Nff/0VtVptEdxnLwO4ceNGkQN8AJVKhZ3d40/zUxS2trZP/ZhC5EWeR/G8kGexeOkNmSSl61FprLDNJ+P97fg0wtef5Mbd+8PV3Cs4M7BjLRyKIVO+WgWOdjoc7P5erWLyPIrnhTyLT490zxf/NM88je2ff/5JnTp1OHjwoEX5yZMncXd3Z9y4cfTq1ctiWXR0NMBTa4UXQgghnncp6RncS9KTacy/Y17MXwl8suqwRXDvX7Mcw4PrFFNwr8LJwfpvF9wLIYQQ/wTPvAW/atWquLm5MXXqVKZMmYKrqyv/+9//+P3339mwYQOxsbEMHjyYsLAw3nzzTS5evMjUqVMJCgrKM/O+EEII8W+SlGIgOc1AAbPgcfRMVqb8jMz74/LbNq5C0MvFkylfo1bh7KDD1vrxXxQIIYQQouieeYCvVquJjIxk7ty5jBgxgsTERGrUqMGKFSuoVq0a1apV49NPP2XJkiUsXboUR0dH3njjDUaMGPGsqy6EEEI8UyaTmYQUPWn6TPLLqGM2m/nx4GW+3nlBKdOoVXR/rToNfV4olrpo1CpcHa2x1j3zrxZCCCHEv9Zz8SlcqlQpZsyYkefy1157jddee+0p1kgIIYR4vhmNJuKT9KRnGAtc74sfz7L3+DWlzM5Gy8AOvlSr5JrPloWn1ahwcbTB2kpTLPsTQgghxKN5LgJ8IYQQQhSeIcNIQrIeQ2beU+ABpKVnsvibaM5cilPKSrnYMqRTLcqVzJng9lFYadS4OlljpZXgXgghhHjWJMAXQggh/kbS9BkkJBswFjDg/k58GuHrj3P9TopS5lbemUEdfXEspgR4Oq0aF0cbrLTPPGevEEIIIZAAXwghhPjbSE41kJSagSm/AffAxWsJRKw/TlJqhlJWr3oZeratUWwt7TqtGlcnG7QaCe6FEEKI54UE+EIIIcRzzmw2k5BiIDU9I99kegDHztxixfd/WGTKfy2gMm80cUNdTPM9W1tpcHW0RiPBvRBCCPFckQBfCCGEeI4ZTWbik9LRG4zkF9ubzWa2H7rC17+cV9ZTq1W828aLAN8Xi60+NlYaXJxs0KiL52WBEEIIIYqPBPhCCCHEcyoj00h8UsHJ9IxGE19tP8ue3x/IlG+tpX97H7wqlyi2+tjoNLg4SnAvhBBCPK8kwBdCCCGeQ3pDJveS9RiN+ffJT0vPZOmmaE5dfCBTvrMNIZ1q80Kp4smUD2Cj0+LqaI1agnshhBDiuSUBvhBCCPGcSUkzkJhScDK9uIR0wtb/zrXb9zPlV3nRiUEda+FkXzyZ8lWAjXVWcK8qpjH8QgghhHgyJMAXQgghnhNms5nEFAMphUimd/l6IuHrj5OYYlDK6niVoVfbGuisiidT/uME92az+W/5QuDvWm/xz2Iu6B8AIYTIg6S/FUIIIZ4DRpOZe0l6UtIKDu5/P3ebOVFHLYL7No1eom877+ciuI+IiGD58uXFUo+naePGjcyaNetZV8PC2bNneeutt/D29ub1119/1tV5Yk6dOoWfnx8Gg6HglYEJEyYQGBio/O7p6cnChQufVPWeql9++YV58+YVuF5gYCATJkx4CjV6sgwGA5GRkbRp04batWvTunVrwsLCCv0sCCEsSQu+EEII8YxlZJpISNajzzDmu57ZbOanw7Fs+PlPi0z53Vp70bhW8WXKf9xu+aGhoQwZMqTY6vO0LFu2jIYNGz7ralgIDw/n2rVrhIeHU6JE8SVMfN7s2rWLhg0botMVz9CSv7Ovv/6aRo0aPetqPDUff/wx3377LYMHD8bHx4fo6GjluZ8+ffqzrp4QfzsS4AshhBDPkN6QSXyygUxjAZnyTSb+t+Mcu479pZTZWGsY0N6X6sWYKV/G3D9f7t27R7Vq1WjWrNmzrsoTtXv3btq1a/esqyGesnv37vG///2PMWPG0LdvXwDl5cbcuXMZM2bMP/rFlhBPgnTRF0IIIZ6R1PQM7iXpCwzu0/WZRKw/YRHcl3CyYdy79YoU3J87c5rRwwfyRuumtH21CWOGD+LUyWhlefTx3xg5tD8tmjSgQYMGjB8/nri4+9n5N27cSI0aNTh+/DjvvPMOPj4+tGjRwqI7vqenJwBhYWHKzwDnzp1jwIAB1KlThzp16hASEkJsbKyy/ODBg3h6erJ//37ee+89atWqRePGjZk9ezZG4/2eDQaDgU8//ZRXXnkFX19fgoKC+Prrry3Oc8eOHXTo0AEfHx8aN27Mxx9/TGpqar7XZtiwYVy/fp2vv/4aT09Prl69qpzvunXraNy4Mf7+/pw/fx6j0ciSJUsICgrC19eX2rVrExwczIEDB5T9LVy4kFdffZWdO3fyxhtv4O3tTevWrfnmm28sjvv555/Tpk0bfHx8aNKkCR9++CHJycnKtTx06BCHDx/G09OTjRs3AnDp0iWGDRtG48aNqV27Nt27d+fo0aPKPq9evYqnpycrVqygTZs21KpViw0bNrBw4ULatGnD9u3bCQoKwsfHh3bt2vHbb7/x+++/06lTJ+Wa7t+/3/LZKeT9++qrr2jRogV16tRh7969xMXFMXr0aBo3bqwc7+FrkJCQwPHjx2natClAoa7vo+jevTuTJ08mIiKCJk2aUKtWLfr168edO3fYsGEDr776Kn5+fvTq1YurV69abDdhwgQiIyMJCAigbt26DB48mL/+uv/3WNj7HR8fz+TJkwkICMDHx4fOnTtbXOvXX3+dO3fu8N133ynPYX4yMjL4+OOPqV+/PvXq1cvxNwuwbt06OnToQO3atfH19aVdu3Zs3bpVWW4ymZg/fz6BgYF4e3sTGBjI3LlzycjIUNbR6/V88sknNGvWDG9vb9544w22bNmSb91u3LhB9erVWbNmjUV5XFwcNWvWZOXKlSQnJxMcHGwx3ALAzc0NwOIZE0IUjgT4QgghxDOQlGIgIVmP0ZT/gPt7ienMXnOUP2LuKmUvveDE+B71eLG0Q6GPl5KSzPgxQ3B2duHDj2fz3ynTSU9PY/zoEJKTkzjx+zFGDx+Io4Mdn376KRMnTuTQoUP06NGD9PR0ZT8mk4kRI0bw+uuvs2TJEurUqcMnn3zCnj17AFi7di0Ab7/9tvLzxYsXCQ4O5u7du8yaNYtp06YRGxtLly5duHv3rkU9x4wZQ926dYmMjCQoKIhly5axbt06i+UrVqygU6dOLF68mJdffpkJEybw/fffA/Ddd98REhKCm5sb4eHhDBkyROn+m1/islGjRlGqVCmaNWvG2rVrKVOmDJAVbH722WdMmzaN999/n6pVqzJnzhwiIiJ45513WLZsGR999BHx8fEMHz6ctLQ0ZZ+3b99m6tSp9OjRgyVLllChQgXGjx/PhQsXAPj++++ZPXs23bp1Y/ny5YSEhLBp0yY++ugj5VrWqFGDGjVqsHbtWpo3b8758+fp0KEDV69eZdKkScyZMweVSkXPnj05dOiQxTktXLiQfv368cknn9C4cWMgK+iaOXMmAwcOJDQ0lMTERIYNG8aoUaPo1KkT4eHhmM1mRo4cqdz3oty/sLAwxo8fz+TJk/Hz82Ps2LFcuHCBKVOmsHTpUmrUqMH48eMtgvVff/0VNzc3Xnwxa5hJYa/vo/j+++/Zv38/06ZN4z//+Q/79+/n3XffZdWqVYwfP56pU6dy/Phxpk6darHdTz/9xMaNG5k0aRJTpkzh9OnTdO/evUj3W6/X07NnT3766SdGjhxJWFgY5cqVo2/fvkqQP2/ePFxcXHj55ZctnsO8bN26lT/++IOZM2cyfvx4du7cSb9+/ZSXYlFRUUyePJmWLVuyePFi5syZg06nY8yYMdy4cQOApUuX8uWXXxISEsJnn31Gly5dWL58OYsWLQKyhgaFhITw1Vdf0bt3bxYtWoSfnx8jR47M8QLjQeXKlcPf35/NmzdblP/www+YzWbatm1LxYoV+fDDD5WA/sHrbWVlReXKlfM9fyFETtJFXwghhHiKTCYzCSl60vSZBSbTu3IjkfD1J0hI1itlftVK0/uNmkVOpnf54kUS4uPp0KkL3j61AKj4UmU2b/qa9NRUli8Jp0qVKixevBiNJmvftWrVom3btmzYsIFu3boBWV/2Bw8eTKdOnQCoW7cu27dvZ+fOnTRp0oTatWsDWV/us38OCwvD1taWlStX4uCQ9VKiUaNGtGzZkmXLljF+/Hilnp06dSIkJERZZ8eOHezcuZPg4GDOnTvHtm3bmDhxIj179lTW+euvvzh48CBt27Zlzpw5NGnShDlz5ij7rFy5Mr169WLXrl00b9481+tTuXJlrKysKFGihFLvbAMHDrTY7tatW4wcOZLu3bsrZdbW1gwdOpSzZ88q26elpTFt2jSly3HlypVp0aIFu3btomrVqhw6dIgKFSrQrVs31Go1/v7+2NnZkZCQAEDt2rWV65W9z6lTp6LT6Vi1apWyrHnz5gQFBfHJJ5+wfv16pU6vvfYaHTt2tDiXtLQ0PvjgA6W1/Pz588ydO5dp06bx9ttvA5CamsqwYcO4ePEi1atXL9L969q1K23atFF+P3ToECEhIbRs2RIAf39/XFxcLMba7969W6lPUa7vo8jMzCQsLAxnZ2cAfvzxR/bs2cOOHTuoWLEiAL///jubNm3Kcd02btyorOPm5kb79u355ptv6NKli7JOfvd706ZNnDlzhv/973/UqpX1N9i0aVO6d+/OnDlz2LBhA15eXmi1WlxdXQt1nq6urixfvhw7Ozvl95CQEHbv3k2LFi2IjY2lT58+DB48WNmmfPnydOjQgaNHj9K2bVsOHTqEt7e38qz4+/tja2uLo6MjAPv27WPPnj3Mnz9fSfTYpEkT0tLSmDNnDkFBQWi1uYcU7dq1Y+LEiVy7dk15gbN582YCAgIoXbp0rtts376dr7/+mnfffVe5T0KIwpMAXwghhHhKMo0mEpL0pBeQTA/gxJ+3Wf7tHxaJ915tUIn2zd1RF3JsvMlkJvZmEsmpGWjtyuDi4sp/xo2geeCr1PNvSD3/RgwIGYbGnMkfJ0/Qp08fzGYzmZmZAFSsWJGqVauyd+9eJcAH8PPzU37W6XSUKFEi3y7wBw4cwN/fHxsbG2XfDg4O1KtXj3379lms++C+IetFQfa+s7uht2rVymKd7OzpFy5c4MaNGwwYMEA5DkD9+vVxcHBg7969NG/e3GIZFDwlWfXq1S1+nzt3LpDV1TgmJobLly/zyy+/AOTI/P1gkFauXDkA5XwaNmzI2rVr6dChAy1btqRZs2a88cYb+eY+OHToEC1atFACbQCtVkvbtm0JDw8nJSUlz3pnq1OnjvJzqVKlAJSAE8DFxQWAxMREoGj37+FjNmjQgIULF3Lq1CmaNGlCs2bNLF4ImM1m9uzZY5E1vijXNzdGo9HinqrVatTqrE6rVatWtQgaS5UqhaurqxK4Z59/UlKSxT7r1KljsU6NGjWoWLEihw8fVgJ8yP9+79+/n9KlS1OzZk2LZ7BFixZ88sknJCQkYGVlleN8Hn5eHzyfZs2aKcE9ZGXW12q1HD58mBYtWihZ9hMTE5VrefDgQeD+tWzQoAFz586la9euBAYG0rx5c959911ln/v370elUtGsWTOLugQGBvLtt9/y559/4unpiclkOdRIq9XSqlUrpkyZwpYtW+jbty/Xr1/n6NGjzJ49O8d5QtYLl9GjR1O3bl3Gjh2b6zpCiPxJgC+EEEI8BfoMIwlJejIKGG8P8PORWNb9dE5p4VerVAS3qkZTvwqFPt6ZS3H8cOASN++mYjSZ0KjV1Gk7iuRLO/nlpx/5btMGrK2tCQp6k5CQQZhMJpYuXcrSpUtz7Mva2tridxsbG4vf1Wp1vkFyfHw8W7ZsyXXM7sMJtPLbd3x8PAAlS5bM8zgAU6ZMYcqUKTmW37p1i6tXr/LKK69YlE+ZMgUPD4886/9gAAUQHR3NlClTiI6OxtbWFnd3d6V18uHrYGtra3EuD67z+uuvYzKZ+OKLL4iIiGDhwoWUL1+eMWPG5DklXkJCghKUP6hUqVKYzWZl/H5u9c724MuB3Or5sKLcv4ePOX/+fCIjI9m6dSvbtm1DrVYTEBDA1KlTKV++PNHR0ej1eurWratsU5Trm5tXX33VYnx8+/btmTlzZp7nntd1elDZsmVzlJUsWVLpbZEtv/sdHx/P7du3qVmzZq7HuH37tnKeD3p4/SFDhjB06FCAHK3garUaV1dX5eXMlStXmDx5Mvv378fKygo3Nze8vLws6tW3b1/s7e3ZsGEDc+bMYfbs2Xh4eDBp0iQaNmxIfHw8ZrPZ4sXQg27dusWOHTsICwuzKD979iwODg60bNmSzZs307dvX7Zs2YKtra3So+NBK1euZNasWfj7+xMeHp7j3x0hROFIgC+EEEI8YWn6DBKSDQWOtzeZzKz76Ry/HL2fWMtGp6HfWz7UdMs9qM3NmUtxRG07Q7o+E3tbK7QaLZlGMwnpDti4vcXMAWNRpV1n508/sO5/aylf/gVUKhW9evWibdu2OfaXX/BXGI6OjgQEBNC7d+8cy/Lq2psbJycnIKtlN7t1FLJa7uPj45Xl48aNw9/fP8f2zs7OlClTxqIbO2QFatevXy9UHZKTk+nbty+enp5s3rwZNzc31Go1u3btYtu2bYU+l2xBQUEEBQWRlJTEr7/+ytKlSxk7dix169bNNah0dnbmzp07Ocpv374NZHXRvnXrVpHrkZ/HuX+Ojo6MHTuWsWPHEhMTw08//URERARTpkxhyZIl7N69m4CAAKXlujiu76JFiyxa+l1dXYtwtrm7d+9ejrI7d+5QqVKlQu/D0dGRypUrWwwfeVCFChVytIIDOZ7XB8flZ7/UymY0Grl37x4lS5bEZDLRv39/rKysWL9+PdWrV0er1XL+/HmLIQhqtZpu3brRrVs37t69y65du4iMjGTo0KHs3bsXR0dH7OzsWLVqVa71fumll/Dy8spz+Mubb75J//79uXz5Mps3b6Z169YW/6aYzWamTZvG6tWrCQoKYsaMGTJdohCPQQJ8IYQQ4glKSjGQnGaggNiedEMmyzf9QfSF+8FbCScbQjrVonwRkumZTGZ+OHCJdH0mLg46pbv3vdjfOLPvK2q0HsuhP24xqlt9WjYPYNsPW7l79y41atQgJiYGHx+f+3VKT2fYsGE0a9YMd3f3Qtchu+UyW3b2+ewAA7K+1I8ZM4aXXnopz67kD8tu5f3555/p2rWrUj5nzhxu3rzJunXrKFmyJFevXqVPnz7K8lu3bjFu3DiCg4OpVKmSxTlCVhfq69evK7kH8hMTE0N8fDw9evSwuCa7d+8GyDVAy8uIESPIyMggPDwcR0dHXnvtNaysrAgJCeHWrVu5Bvj169fnl19+ITk5WWmNNhqNbN68GR8fnycSGD3q/fvrr7/o2rUr77//Pm3atMHNzQ03Nzd+//13Ll++DGRdt+x8DlA81/fB2RuKy9GjR7l3757ysuDkyZNcvXpVyRdRGP7+/uzcuZOSJUtatNRHRkZy+vRp5syZg8lkyvH38/Dz+qC9e/eSmZmp3Jdt27aRmZlJgwYNuHfvHhcvXmTixIkW+3j4WgYHB+Pt7c2kSZMoWbIkHTp0ICkpienTp5OcnIy/vz+fffYZZrMZX19fZT8bNmxg+/btTJ8+nbJly+b6vAK8/PLLlCpVilWrVvHHH3/k6Ho/b948Vq9eTe/evRk/frxMzynEY5IAXwghhHgCzGYz8cmFS6Z3LymdiHXHib11v3v1S+UcGfx2LZwditZNNfZmEjfvpmJva2XxRdmlbFXMZhNXDnyOdcZrbPkxieOHd5OUlESrVq0IDAykf//+jB49mjfffFPJHn/8+HGLBF2F4eTkxLFjxzh8+DD16tVj8ODBBAcHM2DAALp06YK1tTVr165lx44dLFiwoND79fLyok2bNsyePZv09HSqV6/O7t27+eWXXwgLC0Oj0TBy5EgmT56MRqOhRYsWJCYmEhERwc2bN/PsGp3N0dGRU6dOcejQIYtA5kFVqlTBwcGByMhItFotWq2Wbdu2Ka2sRcny3rBhQz744ANmzZpF06ZNSUxMJCwsjMqVKyvdqB82ZMgQdu/eTY8ePZTW2TVr1hAbG8uyZcsKfeyieNT7V758ecqVK8fHH39McnIylSpV4uTJk+zatYsBAwYQFxfHyZMnlRwKULzXtzilpaXRt29fBg0aREpKCvPnz6datWoEBQUVeh8dOnRgzZo19O7dm4EDB/LCCy+wb98+li5dyrvvvouVlRUZGRnY2dlx5swZ5Tl8eNjKg27fvs3QoUPp3r07ly5dYt68eTRu3JhGjRqhUqkoX748UVFRlCtXDicnJ/bs2aO0xGdfy/r16/PZZ59RqlQp/Pz8uHnzJitWrMDf358SJUrQrFkz6tevz+DBgxk8eDBVq1blxIkTLFiwgCZNmhQ4T71Go6Ft27asWbOGsmXL0qBBA2XZ6dOnWbp0KT4+PrRp04bjx49bbOvu7p7rsAohRN4kwBdCCCGKmdFoIj5Zj95gpKARw7E3kwhff5z4pPuZ8mt5lOa9N2pirStapnyA5NQMjCYTWo3lR7ydgwst3h5N9N6NHNuxgt92ZOJZrRoLFy6kYcOGACxfvpywsDCGDRuGlZUVNWvWZMWKFUXOWj5w4EAiIiLo168fW7ZswcvLi6ioKObPn8+4ceMwm81Uq1aN8PDwHOPhCzJ79mzCwsL4/PPPuXfvHlWrVmXBggXKmN5OnTphb2/PsmXLWLt2LXZ2dtSpU4c5c+ZYJEnLTffu3Zk7dy59+vRhxYoVua7j6OhIREQEn3zyCcOHD8fe3l6Z67tfv34cOXIkx5zeeQkODiYjI4OvvvqKL774AhsbGxo1asTYsWNzTbYG4OHhwRdffMG8efN4//33UalU+Pr6smrVKurVq1eo4xbV49y/sLAw5s2bR2hoKPfu3eOFF15gyJAh9O/fn++//x4PDw+Llt/ivL7FqV69ejRs2JD//Oc/QFaCuXHjxhWpx4SdnR1RUVHMnTuX2bNnk5SURPny5Rk9ejTvvfeesl5QUBBffPGF8hzmd1+7du1KUlISISEh6HQ63njjDcaOHau83IuIiGDatGlMmDABnU6Hu7s7ixYtYvr06Rw5coTu3bszfPhwdDodGzZsUHqTBAYGMnr0aCCrR86SJUsIDQ1l8eLF3L17l7Jly9K7d+9C92Bo164dn3/+OUFBQRY9FH788UfMZjPR0dG88847ObZbtWqVxQsBIUTBVObCZCv5B4mOjgby7+5UnFJTUzl9+jTVq1cvVBIXIZ4keR7F8+Kf/CxmZBqJT9JjyCy4K3H0+Tss23TSIlN+S/9KdGjujlpdcDdVs9mcozvr5euJRG48gbVOg06b9UVaq1Hj6mSDCjO376WTqs9kYi9/3Cu6FO3kCnH8Z+lR6/NPfh7F48uerm/16tVP/FjyLD59Tzs2EOJJUxe8ihBCCCEKQ2/I5G5ieqGC+1+OxhKx4bgS3Mf8NAPbWz/wdqBHoYL7vb/uYubHH1iU/bxjG+OHdOHw2tGc2hPFkc3zObZlPq5ONkDWkIGEVAPlyzjgVv7R55e+ceMG/fv3t8hUnm3QoEEsWrTokfddGFevXsXT05ONGzcCWVOAjRs3jiNHjijrdO/eXQnMHl5/48aNeHp6cvXq1Zw7f048WP/HtW7dOjw9PXP8N3Xq1GLZ/5Nw8OBBPD09lSndJkyY8Exa7oUQ4u9GuugLIYQQxSAlzUBiSgamAjrGmUxm1v/8Jz8fiVXKrHUaRr7/Mb6e5Qt9vPVfReUoWzB/FhUqVKJrnzHs/iOB8/u/xMpKg8lo4m5COolpGdhZawv9EiEv+/btY9euXTnKDQYDBw4cUKbwelpOnz7Npk2b6Nixo1L2wQcf5Ll+8+bNWbt2rUU28n+y06dPU6VKFWWquGy5TbcnhBDi700CfCGEEOIxmM1mElMMpKRnFJhML92QyWff/sGJ8/cz5bs6WhPydi0qlHV87LokJiRQr0ND3ny9BdW97/HB7+swGExcupGEVqOm8gtOvB3oQS2P0gXv7BEcOXJEGTP9rOWX9b9EiRIFJgb7Jzl9+jQ+Pj5FzqUgsjyNrvlCCFFcpIu+EEII8YhMJjP3kvSkpBUc3Mcn6ZkXdcwiuK9Y1pHxPepToawjXd4OYta0D7hx/RqBL9dl58/b+XDSONq+2oR2r7VgzqyPlKzXI4f05/jvRzn++1ECX67L78eOEPhy1hRyq1YsJfDlupR1zKBiGUfKl3ZgRHAdJvbyZ2KPOvz64/9o06YNPj4+tGrViiVLllhMPWY0GlmyZAlBQUH4+vpSu3ZtgoODOXDgAJDVvf39998H4JVXXmHChAnKtrt27aJJkyaoVCqli/X+/fvp3r07vr6+NG/enHXr1nHr1i2GDBmCn58fzZo1Y+XKlco+8uo+HxgYaHGsbAcPHqRHjx4A9OjRQ+nWnl8X94ePMXnyZKZNm8amTZto3bo13t7etGvXTplOLNtvv/1Gt27dqF27Ns2bN+fzzz+nV69eSr0e7lae7eG6pKenM3fuXFq1aoW3tzd16tShd+/enD59Otf65ufcuXMMGDCAOnXqUKdOHUJCQoiNvd87xGw2c/bs2Ud66RIXF8fo0aNp3LgxPj4+tGvXjm+++UZZvnHjRnx8fDhy5AgdO3bEx8eH1q1b8/PPPxMTE0PPnj2pVasWr776Kps3b7bY9+HDh+nTpw/169fH29ubwMBAFi5cWKRpBoUQQuQkAb4QQgjxCDKNJuIS07OmwStg3au3kpi16jBXbiYpZb7upRjdrQ4ujrlPgzd/9nTKlnuBj2bM5Z2u3dn6/SbWfJ41Ddrw0RNwr+aJezVPwiJX4uHpRVjkSgBeD2rHipVrqFq5AiqVClsbLXW8ylC1gjODBw9i2bJldOrUicjISNq0acOnn35q0Z19zpw5RERE8M4777Bs2TI++ugj4uPjGT58OGlpaTRv3pxBgwYBWRnSH5xCb9euXTRr1sziPEaNGkVgYCCLFy+mSpUqfPDBB/To0QMPDw8iIiLw9fVlxowZnDhxorCX3kLNmjWZPHkykBWo59c1Pz8XL17k888/Z9iwYYSHh6PRaBg6dCgJCQkAXLhwgV69egFZ83YPHTqUJUuWcPTo0SIfa9y4cWzYsIH+/fvz2Wef8f777/Pnn38yevRoipL7+OLFiwQHB3P37l1mzZrFtGnTiI2NpUuXLty9exeAK1eukJKSQnR0NK1bt6ZmzZq0bt3aIlDPy9ixY7lw4QJTpkxh6dKl1KhRg/HjxysvewAyMzMZPXo0wcHBLFq0CFtbW8aMGcPAgQNp3rw5kZGRlClThvHjx3Pjxg0Azpw5Q69evXBxcWH+/PksWrSIevXqERYWxtatW4t2MYUQQliQLvpCCCFEEekzjCQk6ckwFtza+EfMXZZ8E43ecD9TfmC9igWOg28Q8DKDhowEoE49f44cPsiBfXvoN3Aolau4YW+XNTd0DW8fi/9XKP8iAY3q59jf7t272bdvH/PmzaNt27YANG7cGBsbG0JDQ5Wg+9atW4wcOdKixdna2pqhQ4dy9uxZateuTaVKlQCoXr06FSpUACA2NpbY2FgaN25scdyOHTvSu3dvIGuasM6dO+Pr68vw4cOBrCnYfvzxR44dO5bn3PP5cXBwULrju7u759s1Pz+pqamEhobi6emp1PXdd9/lwIEDtG7dmsWLF+Po6MiyZcuwtbUFwM3NjeDg4CIdx2AwkJKSwqRJk3j99dcB8Pf3Jzk5mZkzZ3Lnzh1Kly7cEIqwsDBsbW1ZuXKlMld4o0aNaNmyJcuWLWP8+PFKr4CrV68yYcIEtFot33zzDePHj8dgMNC5c+c893/o0CFCQkKUKQj9/f1xcXGxmBrOZDIxcOBAOnXqBGQlPBw5ciQ9e/ZU7rujoyMdO3bk5MmTlCtXjjNnzhAQEMDs2bOVKdMaN27Mzz//zMGDB5XnUwghRNFJgC+EEEIUQZo+g4RkA0ZTwS2tu45dZe32c0riPZUKOresRou6+c/HDlCzpuWUTaVLl+Xm9et5rp/9rkBnpcl1+aFDh9BqtbRp08ai/M033yQ0NJRDhw7h4eHB3Llzgazu2TExMVy+fJlffvkFyApO87J79278/PxwdLTMJeDn56f8XLJkSQBq1aqllLm6ugKQlJTEs+Tk5ETFivfvS7ly5QCUYREHDhygadOmSnAPWedWvnzhEyMC6HQ6li9fDsDNmze5ePEily5dyvcam0wmi67rKpUKjUbDgQMH8Pf3x8bGhszMTCDrhUe9evXYt28fAPXr1ycyMpIGDRoo0641adKEuLg4FixYQKdOnTCZTBY9B7L336BBAxYuXMipU6do0qQJzZo1Y/z48TnqV9A9dnFxAbKCf4C33nqLt956C71ez8WLF7l8+TKnT5/GaDSSkZFRpOsphBDCkgT4QgghRCElpxpISjVQUGxvMpnZ+Mt5dhy+opRZW2no284bH/dSFuvF3kwiOTWDTKPJYhy/tY2NxT7VahUmc+49BtQqcLTX5bosW0JCAq6urmg0li8AsluLswPs6OhopkyZQnR0NLa2tri7u/Piiy8C5Nt9fNeuXTRt2jRHeXbL8oMeDJKfFw+2SkNWkAsogXVcXJwSvD7oUTLR79mzh+nTpxMTE4O9vT1eXl5K8J3bNZ44cSJff/218nv58uX5+eefiY+PZ8uWLWzZsiXHNtlJBEuWLEmLFi1yLG/WrBn79u3jzp07jBo1ikOHDinL/P39Wb16NfPnzycyMpKtW7eybds21Go1AQEBTJ061eLFRlHvcXp6Oh999BGbNm0iMzOTChUq4Ofnh1arLdIQBSGEEDlJgC+EEEIUwGw2k5BiILUQmfL1BiOfffcHx/+8rZS5/H+m/IoPZMo/cymOHw5c4ubdVIwmE0mpGZy+dJcLV+OLVDe1Cpzsddjb5h/gOzs7c+/ePYxGo0WQf+vWLSCrJT05OZm+ffvi6enJ5s2bcXNzQ61Ws2vXLrZt25b3Oev1HDx4kNGjRxep7rl5OLDOlpKS8tj7fhzlypXjzp07Ocrv3r2Lm5sbkH/d7e3tgawx8dnd3hcvXkzFihVRqVRERUWxZ8+eXI89ZMgQunXrpvye/TLC0dGRgIAApSv8g7TarK94R44cITY2lvbt21ss1+v1aDQanJ2dmTJlisX1za6ro6MjY8eOZezYscTExPDTTz8RERHBlClTWLJkST5XK3/Tpk1j27ZtfPrppwQEBCgvNxo1avTI+xRCCJFFAnwhhBAiH0aTmfgkPXpDwcn0EpL1RKw/zuUb97ubVyzjwOBOtXB1vN8if+ZSHFHbzpCuz8Te1gqtRosKSE3P5JvdFwpVL7VGjclkKlRwD1mtssuWLeOHH36wGOP87bffAlC3bl1iYmKIj4+nR48eFmPZs7PJZweu2eOmsx08eBAXFxdl/PrjyG4NvnHjhjLW/8KFC8THx+e5zcO9Ep6E+vXrs2fPHvR6PdbWWYkRT506xdWrV6lfPyvnwYN1z5aQkMCFCxeU/AInT55Er9fTv39/5fwAJbjPrQW7QoUKSq6DB/n7+3P+/HmqV6+uBPRms5kxY8bw0ksvUb16dQ4cOMDChQupXbs2VapUAbLu47Zt2/Dz80On0ykvKB70119/0bVrV95//33atGmDm5sbbm5u/P7771y+fLnoF/ABR48epUGDBsrY/uzrEhcXJ1n0hRDiMUmAL4QQQuQhI9NEfFI6hsyCg46/bicTvu44cYnpSplP1ZL0aeeNje7+x63JZOaHA5dI12fi4qBTWn1VKtBp1egzspLxFdRTwNHBkVOnojnx+1Fq1KiBs7Nzvus3bdqUBg0aMGnSJG7evImXlxeHDh1i6dKltG/fHnd3d5KSknBwcCAyMhKtVotWq2Xbtm2sX78euD8e3cnJCYDt27fTtGlTdu/enWv3/EfRoEEDbGxsmDlzJsOHDyclJYUFCxYo47hzvRb/P+5/586dODs74+XlVSx1edDAgQPZsmULffv25b333iMxMZHQ0FDUarVyDz09PXnhhRcIDw/HwcEBlUrF4sWLLbqr16xZE61Wy+zZs3nvvfcwGAxs3LiRnTt3AlnJ/gpr8ODBBAcHM2DAALp06YK1tTVr165lx44dLFiwAIDg4GC++uorBg4cyPDhw7G1teWLL77g3LlzREVF5bnv8uXLU65cOT7++GOSk5OpVKkSJ0+eZNeuXQwYMOARruB9vr6+bN26lS+//JKqVaty5swZFi1ahEqlUp4xIYQQj0amyRNCCCFyoTdkEpeYVqjg/tTFu8xec8QiuG9RtwKDOtayCO4BYm8mcfNuKva2VkpgqFCpsPv/9eMS0smLWqWi27vvorOyol+/fjnma89NdrAZHBzMypUr6d+/Pz/88AOjRo1i+vTpQFagHBERgdlsZvjw4YwbN45r166xZs0a7O3tOXLkCJAVhAcEBDB37lxmzZrF7t27c0yP96icnJxYuHAhRqORkJAQQkNDCQkJwdvbO89tPDw8CAoKIioqijFjxhRLPR720ksvsXz5cvR6PcOGDWP+/Pn069eP0qVLK13aNRoNCxYsoFSpUowaNYpp06bRtm1bWrVqZbGfuXPncvPmTQYNGqRM8bd69WpUKpVyjQvDy8uLqKgoVCoV48aNY9iwYdy+fZvw8HDlmKVKlSIqKgpPT08+/vhjRowYQVpaGitXrrRIhJebsLAwmjRpQmhoKO+99x5ffvklQ4YMISQkpKiXz8KECRNo2bIln376KQMGDGDdunUMGjSIzp0789tvv2E0GgveiRBCiFypzP+ybCbR0dEA+Pj4FLBm8UhNTeX06dNUr15dGWMmxLMiz6N4Xjzvz2JqegaJKYXLlL/n97/4cttZi0z5nV6pRmC93DPl/3HhLp9v+QMnex3qhwN8wGQ2k5hioOfrNalZNWdSN7VKhbODDjsbqyKelchLYZ7H/fv3Y2VlRb169ZSyxMREAgICGDduHD169Hha1RX/YM/7v43/RE87NhDiSZMu+kIIIcQDklIMJKcVIlO+2czXO8+z/aBlpvw+b9bE1yPvecwd7KzQqNVkGs3otDkD/MxMMxq1Gge7nAG8Rp0V3NtaS3D/tP3xxx8sWLCAUaNGUbNmTeLj41mxYgWOjo4EBQU96+oJIYQQQBEC/CNHjrBs2TJiY2Px8PCgb9++ObrLnT59miFDhvDTTz8Ve0WFEEKIJ8lkMpOQoidNn1ng+HdDhpEV3//Bb2fvZ8p3dsjKlF+pnGM+W0LFso6ULWnHX7eTsbLXWXTTN5vNpOgzKF/awSLjPmQF966O1ljr5N38s5A9Xv7LL7/k+vXr2NnZ4e/vz4wZM5Qp6YQQQohnrVDfEvbv30+fPn2oWrUqVapUYf/+/ezYsYP//Oc/dOnSRVnPYDBw7dq1J1ZZIYQQ4kkwGk3EJ+lJzyh47G9iip6I9Se4dD1RKatQxoGQt2vh6mSTz5ZZ1GoVbRpWJmrbGeJTDNhbW6HVqsjMzArubXVa2jSsjFp9P/DXalS4ONpgbfXks8WL3KnVagYPHszgwYOfdVWEEEKIPBUqwF+4cKGSDEWtVpOYmMikSZOYOnUqZrOZrl27Pul6CiGEEE9ERqaR+CR9oZLpXbuTTNj/LDPle1ctSd83vbGxLnzLulflEnRr7cUPBy5x824qqXoTGrWa8qUdaNOwMl6V77cI67RqXBytsdJKcC+EEEKI/BXq28i5c+cYPHiwMu+tk5MToaGhjBkzhmnTplGqVCmLDLFCCCHE30G6IZP4ZD1GY8HJ9E5fimPx1ydI199v5W9WpzydW1ZDoy76pDRelUtQrZIrsTeTSE7NwMHOioplHS1a7nVaNa5ONmg1MumNEEIIIQpWqADf1taWlJQUizKVSsWsWbO4ffs2Y8eOpVSpUmg00roghBDi7yElzUBiSoaS/T4/vx7/iy+2ncX0/5n3VMDbr3gQWK9izqnuikCtVvHSC065LrOx0uDiaI1GgnshhBBCFFKhvjXUqVOHiIgIbt++bVGu1WoJDw/nxRdfZMCAAfz6669PpJJCCCFEcTGbzSQk60lIMRQY3Gdnyl+z9YwS3Ous1Azo4Msr9Ss9VnCfHxudBhcnGwnuhRBCCFEkhfrmMHr0aOLi4ggMDGTevHkWyxwdHVmxYgUlSpRg4cKFT6SSQgghRHEwmczcS9KTkpZRqEz5yzadZNuBy0qZk72O0V3rUrta3tPgPS4bnQZXRxs06ifz8kAIIYQQ/1yF6qJfqVIlvvvuO7755hvKli2bY3m5cuXYsGEDCxYsYPv27cVeSSGEEOJxZf5/pnx9oTLlG1i04TgXr93PlP9iaXuGvF2bEs4FZ8p/VNnBvVqCeyGEEEI8gkKn/HVxcaFXr17ExcXlutzBwYGJEycyceLEYqucEEIIURz0GUYSkvRkGAvOlH/9Tgrh637nTsL9TPk1qpSg31s+2BYhU35RSXAvhBBCiMdV5G8qTZs2pUmTJrRr147AwEB0Ot2TqJcQQghRLNL0GSQkGzCaCk6md+ZSHIu/jiZNn6mUNaldnuBWj5Ypv7BsdFpcHa0luBdCCCHEYylygD9mzBi2bNnCiBEjcHR0pE2bNrRr14569eo9ifoJIYQQjyw51UBSqoFCxPbsO3GNNT+csciU36GFOy39n1wyPZDgXgghhBDFp8gBfq9evejVqxexsbF8//33bNmyhXXr1vHiiy/y5ptv8sYbb1C1atUnUVchhBCiUMxmMwkpBlLTC06mZzKb+W5PDFv3XVLKrLRq3nujJn6eZZ5oPSW4F0IIIURxeuT+hhUrVmTQoEF89913fPfddzRv3pylS5cSFBRUnPUTQgghisRoMhOXmE5qITLlZ2Qa+ezbPyyCeyd7HaO71X3iwb2ttQT3QgghhChej5Ut6O7du2zdupWtW7fy22+/4eLiwuuvv15cdRNCCCGKJCPTRHxSOobMgpPpJaUaWLThBDF/JShlL5ayJ6RTLUo62z6xOqoAm/8P7p9k138hhBBC/PsUOcBPSkpi27ZtbN68mcOHD6PRaAgMDCQiIoImTZqg0WieRD2FEEKIfOkNmdxL1mM0Fjzg/sbdFMLXHed2fJpS5lW5BAPe8sHW5sllypfgXgghhBBPUpG/xTRq1AiTyUTdunX58MMPadOmDQ4ODk+ibkIIIUShpKRnkJhiUBLk5efs5Xss/voEqen3M+U3rvUiXVt5otE8uUz5EtwLIYQQ4kkrcoA/dOhQ3njjDV588cUC1z18+DA1a9bEzs7ukSonhBBCFCQx2UBKeuEy5R+Ivs7qractpsxr39ydVg2ebKZ8FWBro8XFQYJ7IYQQQjw5RW6qGDBgQKGCe6PRSI8ePbh48eIjVUwIIYTIj8lk5l5SOsmFCO7N/58pf+XmU0pwb6VV0+8tb1o3fEmCeyGEEEL8Izy5gYZkfaESQgghipvRaCI+SU+aIbPAoDkj08SqLac4fOqmUuZoZ8Xgt2tR5UXnJ1rPvIJ7s9kswb4QT4D8bT3f5P4I8eQ9ucGGQgghxBNgyDASl5jO0qWLWfvl6nzXTU41EPrVMYvgvlxJO8b3qP/Mgvt169Yxa9asJ3rsojp79ixvvfUW3t7e/+jZcE6dOoWfnx8Gg+FZV6XQrl69iqenJxs3bnzWVXnuBAYGMmHCBOX3iIgIli9frvy+cOFCPD09c13/73BdH67/39mNGzfo378/f/31V77rbdy4EU9PT65evfqUavbk/PHHH/Tr14+GDRvSoEED3nvvPf74449nXS3xLyABvhBCiL+NNH0GcYlZ0+CtWLaI9LS0PNe9GZfKrNVHOH/1/jR4ni+5Mq57PUq5PLlp8CD/bvmLFi0iPj7+iR6/qMLDw7l27Rrh4eHP3cuH4rRr1y4aNmyITqd71lURT0BoaChpD/yb0KlTJ9auXZvrumXKlGHt2rU0b978KdXu323fvn3s2rXrWVfjqbl8+TLvvvsu6enpTJs2jRkzZmAwGOjatSsxMTHPunriH+6JdtEXQgghiktymoGklAxMhRj+9WfsPSI3nCDlgUz5Ab4v0LW1F9onmCkf/p5j7u/du0e1atVo1qzZs67KE7V7927atWv3rKshnpJy5cpRrly5XJfpdDpq1679dCsk/jVWr16Nra0tixcvVpKNN2zYkMDAQNasWcPkyZOfcQ3FP5m04AshhChWJ0+epGfPntStWxc/Pz969erF77//riw/cuQIffr0oVevXjRr1ozx48cTFxenLN+4cSM1atTg+PHjvPPOO/j4+NCsWXOWLl2mBPeBL9cFYNWKJcrPABdjzjNo0EAGdW/L8W/e56/Dn2NIuctbzarS/bXqnDx+jMCX63LsyCHGjhzMa68E0PHNViyJWIDRaFT2k5GRwWdLI+jW6U3aBAbwXvfObNv6ncV57t2zk4F93qV1YCM6vtmKsE9nk56Whp2tVZ7BfWBgIH/99Rdff/210g01+3zXrVtH48aN8ff35/z58xiNRpYsWUJQUBC+vr7Url2b4OBgDhw4oOxv4cKFvPrqq+zcuZM33ngDb29vWrduzTfffGNx3M8//5w2bdrg4+NDkyZN+PDDD0lOTgbA09OTQ4cOcfjwYYsuy5cuXWLYsGE0btyY2rVr0717d44eParsM7uL84oVK2jTpg21atViw4YNLFy4kDZt2rB9+3aCgoLw8fGhXbt2/Pbbb/z+++906tQJX19fgoKC2L9/v0U9z507x4ABA6hTpw516tQhJCSE2NhYZfnBgwfx9PTkq6++okWLFtSpU4e9e/cSFxfH6NGjady4sXK8h69BQkICx48fp2nTpgB0796dCRMmEBkZySuvvEKfPn0YOXJkji7EO3bsoGvXrvj5+eHt7U2bNm2IiooqsE6QNRyjQ4cO1K5dG19fX9q1a8fWrVuVbbO7Ix8/fpz27dvj6+vLG2+8wQ8//JDj2bl9+zbDhg3Dz88Pf39//vvf/5KSkmLxbE2fPp2ePXvi6+vLf/7zHwDOnDnDkCFDaNiwITVr1qRJkyZ8/PHHpKenK9vu3buXzp074+fnR/369Rk0aBAXLlzIcR06dOiAj48PjRs35uOPPyY1NTVHPR/m6enJl19+yYQJE6hbty7+/v7K8WfNmqV0X/7Pf/6DXq8H8u4+P2HCBAIDA/M8DkBYWJjyc35d3B8+Rm7/7rRo0cKiyz/ArVu3GDlyJP7+/tSvX5/Jkyczf/58i3p5enqycOFCi+1yq0tuz8f27dvzvZ4Py37+9u/fT/fu3fH19aV58+asW7eOW7duMWTIEPz8/GjWrBkrV67Msd2vv/5Kt27d8PX1pVWrVnzxxRcW+/f09CQqKor//Oc/+Pv74+fnx/Dhw7lz547Fevk9Hxs3buT9998H4JVXXrEYVpGXY8eOKcOGgoKC2LJli8Xyq1evMm7cOF5++WVq1qxJo0aNGDduHPfu3VPWKeizCLI+jyZPnkzXrl3x9/fP8XmUm//+9780btzY4jMDYNq0aTRo0ICMjAzc3Nx47733LGYSs7Ozo1y5cly5cqXA8xficUiAL4QQotgkJyfTt29fXF1dWbhwIfPnzyctLY0+ffqQlJTE4cOH6dWrFzY2NgwbNowxY8Zw6NAhevToYRFwmEwmRowYwWuvvc6c+WHU9KlFZHgohw/uAyAsciUArwe1U36+cvkSg/r14tKVG5St9Q5lfTuRmRbH3WNLaODlZBFwT5s6Cd9afkz75FNeebUNX33xOVu+++b+8in/Yd1Xa3j9jbeY/smn1PdvyKxpH/LT9qzA66cft/Lf90dT6aXKfDR9Lj3f68/2bVv44D+jcbKzyrPlPiwsjNKlS9OsWTPWrl1LmTJlgKyZZz777DOmTZvG+++/T9WqVZkzZw4RERG88847LFu2jI8++oj4+HiGDx9u0Q359u3bTJ06lR49erBkyRIqVKjA+PHjlQDt+++/Z/bs2XTr1o3ly5cTEhLCpk2b+OijjwBYu3YtNWrUoEaNGkqX5fPnz9OhQweuXr3KpEmTmDNnDiqVip49e3Lo0CGLc1q4cCH9+vXjk08+oXHjxkDWeNuZM2cycOBAQkNDSUxMZNiwYYwaNYpOnToRHh6O2Wxm5MiRyn2/ePEiwcHB3L17l1mzZjFt2jRiY2Pp0qULd+/ezXEdx48fz+TJk/Hz82Ps2LFcuHCBKVOmsHTpUmrUqMH48eMtXob8+uuvuLm5WcwE9NNPP7Fx40bGjx9Pnz59OHPmDN27d1eu786dOwkJCaFmzZpERESwcOFCKlasyNSpUzl+/Hi+dYqKimLy5Mm0bNmSxYsXM2fOHHQ6HWPGjOHGjRsW2w4YMIBXXnmFsLAwqlSpwogRI3J0Zw4NDeWFF14gIiKCnj178r///Y+wsDCLdaKiovDx8SEiIoK3336bW7du0a1bN9LS0pg5cyZLly6lbdu2rF69mlWrVgEQGxvL4MGD8fb2ZtGiRUybNo2LFy/Sv39/TCYTAN999x0hISG4ubkRHh7OkCFD+Pbbbxk8eHChEirPnj0bnU5HWFgYb731FqtXr+att97i+vXrzJkzh+7du7N+/XpWr84/p0Z+srviv/3223l2yy9I9r87r7/+OkuWLKFOnTp88skn7NmzBwCDwUDPnj05duwYEydOZMaMGZw5c4bPPvusyMfK6/mYOHFijue9MEaNGkVgYCCLFy+mSpUqfPDBB/To0QMPDw8iIiLw9fVlxowZnDhxwmK7kSNHUqNGDcLDwwkICGDKlCk5gvz58+djMpmYN28e48aN45dffmH69OnK8oKej+bNmzNo0CAg6+9k8ODBBZ7P5MmTee2114iIiMDDw4ORI0eyY8cOANLS0ujRowcXLlzggw8+YPny5fTo0YPNmzczf/58oODPIkD5PNLpdIwaNYqJEyfm+nn0sHbt2nHnzh0OHjyolJlMJrZu3Urbtm2xsrKia9eu9O3b12K7y5cv8+eff+Lh4VHg+QvxOKSLvhBCiGJz/vx57t27R48ePahTpw4Abm5urF27lpSUFObOnUuVKlVYsGAB586do3r16vj7+9O2bVs2bNhAt27dgKxMywMGDuKVVkEYMk1Uq+7Dnl2/sH/fr9RvEEANbx8ASpUuSw1vHzIyTXw4bQ5GNFRu2A+NlQ0OtlaMH9yR/4zoztovVjEwZIRSz7ZvvEX3Xv0AqFPXn727d7J/3x7eeKsjF2POs3vnT4QMG03Hzl2z1qnnz40b1/n92BECW7ZmSeRC/BsEMHHyxwCoVOBetQpDBvVn9+7deY7rrVGjBjqdjhIlSuToHjxw4ECL7bJbCrt3766UWVtbM3ToUM6ePatsn5aWxrRp02jUqBEAlStXpkWLFuzatYuqVaty6NAhKlSoQLdu3VCr1fj7+2NnZ0dCQlZugtq1a+Pg4KD8DDB16lR0Oh2rVq1SljVv3pygoCA++eQT1q9fr9Tptddeo2PHjhbnkpaWxgcffKC0lp8/f565c+cybdo03n77bQBSU1MZNmwYFy9epHr16oSFhWFra8vKlSuVYzZq1IiWLVuybNkyxo8fr+y/a9eutGnTRvn90KFDhISE0LJlSwD8/f1xcXGxGGu/e/dupT4P1nPjxo2ULFmS06dP07hxY7p06cI333xDly5dOH/+PO3bt1dawwH8/Pxo0KABBw8epFatWnnWKTY2lj59+lgEM+XLl6dDhw4cPXqUtm3bKuXdu3cnJCQEgCZNmtC+fXvCw8Mthky0bt1aaQVt1KgRe/futXiBAfDiiy8yZswY5fdff/2V6tWrExoaqlzTgIAA9u7dy8GDB+nfvz8nTpwgPT2dAQMGULZsWSCra/tPP/1Eamoq9vb2zJkzhyZNmjBnzhxl35UrV6ZXr17s2rWrwHHs7u7uTJ06Vbk369atIyMjgzlz5qDVann55ZfZtm0bx44dy3c/+cl+dsuVK/fIXe/NZjODBw+mU6dOANStW5ft27ezc+dOmjRpwrfffktMTAwbNmzA29sbyOp2nf3cFUV+z8fZs2d5+eWXi7S/jh070rt3byCrpbhz5874+voyfPhwALy8vPjxxx85duwYvr6+ynavvvqq8nw3adKEW7duERERQZcuXZQXldWqVWPGjBnKNidOnFB6mZjN5kI9H5UqVQKgevXqVKhQocDzGTp0KH369AGgadOmXLp0iYiICFq2bMmlS5coV64cs2bNomLFikDWfTh+/LjyArKgzyJHR0fl8+j9999Ho9Hg4+NDrVq1cnwePaxu3bqUL1+e77//noCAACCrR8Tt27fzHAKUnp7O+PHj0el0vPvuuwWevxCPQwJ8IYQQj81kMhPzVwKpZhecXVwZOHAgbdq0oUmTJjRu3JixY8eSlpbG8ePH6dOnD2azGaPRSGZmJhUrVqRq1ars3bvX4gtVFffqGDKzWhB1Oh0uLi65JtVLScsgcuMJYi9EY1eyKmqNFWVcrBn8ti+lXezw8fXj6OGDFtvU8Pa1+L1UmTKkp2ftO/rE7wA0aWbZFXjKtNkAXLl8kdu3btK1e2+MmZmoVGBrY0WLpo1xcHBg7969NG/enMzMTIvtNRpNvmPyq1evbvH73LlzAYiLiyMmJobLly/zyy+/AOTIAv9gQJM95ji7e2zDhg1Zu3YtHTp0oGXLljRr1ow33ngj37ocOnSIFi1aKEEhgFarpW3btoSHh1t0DX+43tmyv1QDlCpVCsAiIHZxcQEgMTERgAMHDuDv74+NjY1y7RwcHKhXrx779u2z2PfDx2zQoAELFy7k1KlTNGnSRBn6kc1sNrNnzx7mzZuXo44VK1ZUrpWXlxcVK1bk8OHDdOnSRWmBS0lJ4eLFi1y5coXo6Ggg5z14uE7Z3ZATExOV+5fd4vfwtu3bt1d+VqlUvPrqqyxcuNCiFbFevXoW21SoUMFiyERudXj55Zd5+eWXycjI4Pz581y+fJlz584RFxenXP9atWphbW3N22+/TZs2bWjatCkNGjRQgsALFy5w48YNBgwYYPFM169fv9DPu5+fn0W5q6srNWvWRKu9/zXUxcVFaVl9lh6sa/bLuOzn48CBA1SsWFEJ7iHrGW3RooVFa25h5Pd8PHwtAeXfzAc9eP0erHfJkiUBy783V1dXgBzX+MFnD6BVq1b89NNPXLx4ETc3N4AcL0zKlSun9HKJiYkp1PPxMJPJpPQQye18Hp7No2XLlixcuJCUlBSqV6/OF198gclk4tKlS1y+fJnz588TExOj1MHDw4MSJUrk+lkEWHweAXl+Hj18L9RqNWq1mjfffJMvvviCDz/8EJ1Ox+bNm6lcubLFNc+WnJxMSEgI0dHRhIaGUr58+RzrCFGcHivAT0pK4tatW1SsWBGNRoNGo1GWaTQaZsyYUai3dEIIIf6+jv95m/U//8lft5LJNJpwaxJC3Pmf+P77zaxduxYbGxvatWvHgAEDMJlMLF26lKVLl+bYj7W1NZA1DR6A1sraYrlKrcZstvxCmJKewSerj3AzLhWjIZWka8dJunacP4Ff19xfz8XF1WI7G2sbi9/VKjVmU1ZX48T/b9l2cS2R6/lmLw+dO5PQuTNzLL916xZXr17llVdesSifMWMGHTp0yHWfgMVYTYDo6GimTJlCdHQ0tra2uLu7K93LH+4WbWt7f1YAtVptsc7rr7+OyWTiiy++ULqZly9fnjFjxuQ5JV5CQoISlD+oVKlSmM1mZfx+bvXO9uDLgdzq+bD4+Hi2bNmSY6wtQIkSlvfi4WPOnz+fyMhItm7dyrZt21Cr1QQEBDB16lTKly9PdHQ0er2eunXrWmyX3WL9oJIlSyq9G+Li4vjggw/YsWMHKpWKl156SQm0H74HD9fpypUrTJ48mf3792NlZYWbmxteXl65bps9VOPBOpjNZuXlB+S8dmq1usA6ZHerjoqKIjU1lRdeeAFfX1/lbw2yXhSsWbOGJUuWsH79elatWoWTkxNdu3ZlxIgRyowPU6ZMYcqUKTmuV2Ge99yehbyem2fNxuahfxseuM737t1TgucH5VZWkKI8HwBff/210oMj208//aT8XNS/t2wP/w1kn0v230Bu+3nwmhTm+chNeHh4jiEmZ8+eVX5++N+f7L+J5ORk7O3tWbFiBZGRkcTHx1OqVCm8vb2xtbVVXmDY29sTFRXFokWL2Lp1q8Vn0aRJk0hMTCzU51HNmjUtyocMGcLQoUNp164dixYtYs+ePTRp0oQff/yRnj175tjP9evXGTBgABcvXmT+/PmP1NtDiKJ6pAD/4MGDzJkzh5MnT6JSqVi3bh1Lly6lXLlyFokzHn4rKIQQ4p/l+J+3CV9/nLT0TBztrbDSWJFh9yJWDsG8VL8rrXys+OPYLr788kvKli2LSqWiV69evPLKK1y6dInKlSsrX6htbW1JSjGQmp5R6OPvPX4NJ7esVkuNlS0Vq/owZsQgNA9lyn/wBXRBHBwcAUiIv0fpMve//F65fJHEhATs/3/5wJDhNGzYEAdbK4vtnZ2dKVOmjEU3dqBIL7yzx496enqyefNm3NzcUKvV7Nq1i23bthV6P9mCgoIICgoiKSmJX3/9laVLlzJ27Fjq1q2ba5Dr7OycI4kWZI33h6zWwLy+uD8qR0dHAgIClG7GD3qwZS+vbceOHcvYsWOJiYnhp59+IiIigilTprBkyRJ2795NQEAAVlaW9+rBhFzZ7ty5o3QnHjNmDDExMaxcuRI/Pz90Oh1paWn873//y7c+JpOJ/v37Y2Vlxfr166levTparZbz58+zadOmHOtnBykP1kGj0eDi4vJY13nJkiWsXLmSKVOm0KpVKxwds57d7GES2Xx9fQkLC8NgMHD06FHWrl1LZGQkXl5euLu7AzBu3Dj8/f1zHKM4nveHZbf8P9xiXZikfk9S2bJluXTpUo7y3MbM51f3oj4fAC1atMhxjcuUKVPgvPIFuXfvnvK8w/1zKexLCycnJyD/5yM3nTt3zndox8MvGbP/Jpydnfnuu++YOXMmY8eOpUOHDsoLwOHDhys9bCCrS/7s2bMxGo2cOHGCTZs28eWXX1KpUiWCg4OVz6Ps5IfZzzrcf6mR2zUHqFKlCr6+vmzduhW1Wk1iYiJvvvmmxbpnz56lT58+6PV6PvvsM+rXr5/n+QpRnIqcZG///v306dMHGxsbxowZo7zB8/LyYtWqVaxYsaLYKymEEOL5YzKZWf/zn6SlZ1LS2RprKw03Lx5jT9Q4HLTppGeYOHHdmsmTP8DJyYm7d+9So0YNYmJiqFmzJm5ubtSsWRMPDw8WLFjIL7v3kpRmoOCUXXD41A1ApbT2A7hV80FtuIunpxeeXjXw9KpBNc/qrPtqDb/u/qXQ5+XjWxuAfXt3W5QvWbSQsNA5VHqpMq6uJbhz6waN/Ovg4+ODj48PZcuWZe7cuZw6dQqdTqeUZ/+X3UU2u4U9PzExMcTHx9OjRw/c3d2VbXbvzqrTw11b8zNixAhlfLejoyOvvfYagwcPJjMzM8/gsX79+vzyyy8WLfVGo5HNmzfj4+PzROaRz549oHr16so18/b2ZuXKlflmFv/rr79o1qyZMibYzc2Nfv36ERAQwLVr14Dcx98DHD161CLIP3XqFFevXlXyGRw9epRWrVrRoEED5ZwLcw/u3bvHxYsXefvtt/Hx8VFeUOS1bXbyMMhqvf3xxx+pW7fuY1/no0eP4u7uTseOHZXg/ubNm5w7d06pw8qVK2nRogUGgwGdTkejRo2UBIzXrl3Dzc2NkiVLcvXqVYvnubDP+6PIbo2+efOmUpaRkZEjQdzDCvO39Tj8/f25evUqp0+fVsrS09OVJHzZHBwcLOoOWOQXKOj5yK0F39XVNcc1Lo6/wwefPYAffviB8uXLWwT9+SnM8wE5703ZsmVz9jLhUAAAujtJREFUnM+Ddu7cqfxsMpn44YcfqFWrFjY2Nhw9ehQnJyf69u2rBPcpKSkcPXpUea5/+OEHGjZsyO3bt9FoNPj5+fHhhx/i5OTEtWvXcHBwUD6P3N3dcXd3x8fHBw8PDxYuXKgMl3i4jg++EG3Xrh179uxh8+bNynCfbNevX6d3796oVCq+/PJLCe7FU1XkFvxPP/2UV155hdDQUDIzM5k9O2tM4sCBA0lNTWXdunW5vn0XQgjxzxLzVwJ/3UrG0f5+1njXcu6YzSaObYukks+r/HFbx8gxm0hKSqJVq1YEBgbSv39/Jk6ciLe3N7du3WLNmjWcOHGCzl17U1BCbrMZtuy9yLd7YlBb2ZJ+7zL6excZ2L0tpWwqMXRgLyaOG8Gb7d9Gp9Px3aaN7N2zkw8+/qTQ51XVoxrNWrRkcUQo+vR03D08OXhgL/v37mbKtNlotRoGhQxl5vSPsLXR0aJFCxITE4mIiODmzZs5unQ+zMnJiVOnTnHo0CGLZFcPqlKlCg4ODkRGRqLVatFqtWzbtk1pTSpKS2bDhg354IMPmDVrFk2bNiUxMZGwsDAqV66sdAl+2JAhQ9i9ezc9evRQWhrXrFlDbGwsy5YtK/SxH5ZfxvXBgwcTHBzMgAED6NKlC9bW1qxdu5YdO3awYMGCPLcrX7485cqV4+OPPyY5OZlKlSpx8uRJdu3axYABA4iLi+PkyZM5pi2DrHG4ffv25b333uP8+fNs3LiRatWqERQUBGS1bH/33XfUrFmTcuXKcezYMZYsWYJKpbKYyeBhJUuWpHz58kRFRVGuXDmcnJzYs2ePkrn+4W0/+eQT9Ho9VapUYd26dVy4cIHPP/8832tZGL6+vkRERLBkyRJq167N5cuXWbx4MQaDQalDw4YNmTNnDiEhIbz77rtoNBq++uordLqsZ1uj0TBy5EgmT56MRqMp8vP+KJydnfHz82P16tW89NJLODs7s2rVKtLT0/Pt2u/k5MSxY8c4fPhwjpwFxSEoKIglS5YQEhLC8OHDcXJyYsWKFdy9e9didobmzZuzefNmatWqxUsvvcTGjRu5fPmysryg5yN7usCnYcWKFVhbW1O7dm1+/PFHfvnlFyX/R2EU9vnIbunfvn07TZs2pWrVqvnu99NPP8VoNPLCCy/w5ZdfcvHiRaUR0dfXly+//JKZM2fSokULbt26xfLly7lz547SY6BOnTqYTCZCQkLo378/9vb2bN26VfksgqyZB/r374/RaKRp06bcvn2bzz77jOPHjxcq0//rr7/OzJkz2bJlCx988IHFso8//pi7d+8yZcoUkpOTLabnc3BwsOgtIERxK3KAf/r0aaUl4OEEPY0bNy6WDyQhhBDPv8QUA5lGE1aa+92ebeydqd92OOcObeL0r1FkZhpIq1KVhQsX0rBhQwCWL19OaGgoP/30EzqdDk+vGsyeH6Fkxs/PhavxXNoTA0BJj0Di/vyJ60dWUHlkW8qW8+DT8GV8tiSCGR9Nxmw2U8WtKlNnzKXxy80K2LOliZM/5vPPFrNh3ZckxMdTqXJlPvj4E5o0a4G9jRU9u3eldEkXli1bxtq1a7Gzs6NOnTrMmTPHohUnN++99x7Tp0+nT58+efZ6c3R0JCIigk8++YThw4djb29P9erVWbNmDT179mTFihU5xjznJTg4mIyMDL766iu++OILbGxsaNSoEWPHjs3RZT2bh4cHX3zxBfPmzeP9999HpVLh6+vLqlWrHitwioyMZNq0abku8/LyIioqivnz5zNu3DjMZjPVqlUjPDy8wHMNCwtj3rx5hIaGcu/ePV544QWGDBlC//79+f777/Hw8Mh1KEKNGjW4ceMGo0aNQqVS8cYbbzBx4kSlZXTmzJl89NFHSot25cqVmTJlCt9++y1HjhzJt04RERFMmzaNCRMmoNPpcHd3Z9GiRUyfPp0jR45YzI7w4YcfsnjxYmJjY6lRowafffZZsQSoAwYM4N69eyxfvpz4+HgqV65Mu3btUKlULF68mMTERLy8vIiMjCQ8PJxRo0ZhNBqxs7PDyclJSbI2adIkWrduzbFjx/J83idMmMDXX39N2bJl2bVrV65JHOfMmcPSpUstWp4DAwNz7dqdfe0nTZqEg4MDb7/9NnXr1mXdunX/x959x9d49mEAv85JcrKXrTEjhJBEjNQeqRER1Ko9WiUqSof9KqXV1mhVM0pQW6V2K0btVcSooBQRFEWQeZKzz/P+keaJIwknyYms6/v5vJ+X+1l3znmk5zrPff/ubMe4u7tj/PjxGDt2LMLDwzF69OhstRyGDRuGBw8eZFt+8EVvvfUWevfujW++yV5fw9zcXFxqcsqUKbCzs8Pbb78NJycn3L59W9xv+vTp0Gq1mD9/PszNzREQEIBWrVohKipK3Cfz/pg6dar4xZdEIoGFhQUOHDggftYuCL1ej59//hkbN27E/fv3AQDHjx83eBA3Y8YMbN++HcuWLYOrqyt++OEHdO3aNU/X6d+/P2xtbV/6+/DNN99Eq1at8O233+LUqVOIiIh46Tm//vprfPPNN7h79y7q1auH5cuXi+957969cf/+fWzduhUbN25E5cqV0b59ewwePBifffYZbt26hTp16mDFihVYsmQJ/ve//0GhUIhP5zP/W9SmTRusXLkS8+fPx6JFi2BpaYmGDRti1apVRq3EUK5cObRp0wYnT540WEFDrVaLIxBeDP5AxkiQgiwJSfQqEsGYBUyf07p1a3zyySfo27cvdDodGjZsiK1bt6Jhw4bYsWMHFi1ahBMnThRWfwssc27Oi0OBCkt6ejquXbuGBg0aFNuCMlR28H4kU4q9l4SvVkfDytIMlhbZ57ir1Doo1TrMGOkLt+pOBtvS09Px941YlK9UDeYWrx5mmqbUYNm2S7jxT5LYVsnZGsH9G6NyuddzL0skgK2VBRztLF+9cyHKDDMffvhhkfYjrzJDWU7BqSgMGzYMN2/ehF6vxxdffIGUlBR07979tf5u3LZtG6ZPn46DBw8WalHiH3/8EZcuXcKPP/5o1P7Tpk1DdHQ0Dh06BMC4e27atGnYuXOnWNDxxYKGQMY98ODBA4OAc/XqVdjZ2Rk9JDynY4zp37Bhw3Du3DkIgoAjR46Iq01kSk9PR6tWraBQKHIN+Ddv3kRcXBzkcjlmzJghvm/9+vVDlSpVshWNyxQVFYUpU6agcuXK4muaadasWeKa8Z6enrhw4QKWLl2Kbt26YcEC40ce5SQiIgLff/89Ro0ahZYtW+L27dv44Ycf4OHhgaCgIIwYMQJr167Fm2++WaDrlHSvOxsQFbY8P8F/6623sHjxYtSrVw8eHh4AMr5xfPToEZYuXfrKtVCJiKh0cHVxhEslO9x5mAKZg9TgiZ0gCEhVaFCrqgNcXbIXWVKodEiWa+BQTgfznB8ii54kKRD6y0U8Tsgalu5WzQlj+3plK3BXWCQSwM5KBgc70889p6Kj1WpRv359tG3b1mBedWlz7NixXNfnNqWqVatCEATs2bMnW8C/ePEiHj9+jHr16hm0Z36WzIv8HJN5XGxsLPbu3YuRI0cabDt8+DCsra3FoeQ5SU9Px8SJE8VAfO7cOYSHh+PKlSuYNGlStv2fPXuGJUuWIDIyUlyW8HmJiYn45ZdfMGnSJHFJRm9vb8THx2PTpk2YNm1athUkjJVZIX7AgAH49NNPAQCtWrWCs7MzPv74Y3GYOhGVPnmuRvLpp5+ifPnyBtUvP/nkE/j7+0MikeCTTz4xdR+JiKgYkkol6OdXF9aW5niWooJKrYNeL0Cl1uFZigo2lubo51cXUqlh8E+Wq5CcpoJa8+pq+bfuJ2H+mrMG4f7NhlUwcaBPsQv3V65cwYgRI9C0aVP4+Phg5MiRBvMuz507h6FDh8Lb2xu+vr6YOnUqEhISxO3btm2Dh4cHYmJiMGDAAHh6eqJjx45YuXKluE9mtefQ0FDxzwBw48YNBAUFoUmTJmjSpAmCg4Nx7949cfuZM2fg7u6OU6dO4b333oO3tzdat24tVpjOpFarxVo7Xl5eCAwMxPbt2w1+zgMHDqBPnz7w9PRE69at8eWXX76yJkDmk9vt27fD3d0d9+/fF3/ezZs3o3Xr1mKRPZ1Oh4iICAQGBsLLywuNGzfGwIEDcfr0afF8ISEh6Ny5M44cOYIePXqgUaNG6Nq1K3bs2GFw3TVr1sDf3x+enp5o27YtPv/8c7FwYHR0NFJTU3H27Fn4+PiIQ7fv3LmDCRMmoHXr1mjcuDGGDRtmsN78/fv34e7ujlWrVsHf3x/e3t7YunUrQkJC4O/vj/379yMwMBCenp7o1asX/vzzT1y8eBH9+/cXX9NTp04Z9PP27dtGvX+bNm1Cx44d0aRJE5w8eRIJCQn49NNP0bp1a/F6L74GycnJiImJEYsMGvP6FoS/vz9+//33bPUWdu/ejVatWmULun5+fuIKTJmv7Z49ezBhwgT4+PjA19cXM2fONLjHnj8mL2xsbAwKMr7Yv65du750xQZvb298//334nD32bNn4+7du1ixYoU47Pt5S5cuxYkTJxASEoKOHTtm2y6XyzFw4ED4+fkZtGfO53/+Hnhe5pKP8+fPN2jXarVo0aKFWI+iV69eYj2JTJnTLky9CgYRFR95DviOjo7YvHkz5syZg+bNm6NVq1Zwd3fH5MmTsW3btnx/00hERCWPd92KCO7njVpVHaBU65CYqoJSrUOtqg4Y188b3nUrivvq9QISU1VIU2ig1796dti5a4+x+Oc/IVdkfREQ2KY2RgZ6wMK8cKtlZzI23Gcua+fs7IyQkBAsXrwYCoUCo0aNEkPkyJEjYWVlhe+//x4zZsxAdHQ0hg8fDqVSKZ5Hr9fjo48+QkBAACIiItCkSRMsWLBArNIdGRkJIGOZs8w/3759GwMHDsSzZ88wf/58zJs3D/fu3cOgQYOyLd81adIkNG3aFEuXLkVgYCBWrFiBzZs3G2xftWoV+vfvj2XLlqFNmzaYNm0adu3aBQD47bffEBwcDFdXV4SFhWH8+PHi8OKXzfgLDQ1FxYoV0b59e0RGRopLTel0Ovz000+YN28epk+fjjp16mDRokUIDw/HgAEDsGLFCnzxxRdISkrCxIkTDYrTPXnyBHPnzsXw4cMRERGBatWqYerUqbh16xYAYNeuXVi4cCGGDBkizpveuXOnOJ8+MjISHh4e8PDwwJo1a+Dj44Nbt26hT58+uH//PmbOnIlFixZBIpFgxIgRiI6ONviZQkJCMHr0aCxYsACtW7cGADx69AjffPMNxo4diyVLliAlJQUTJkzAJ598gv79+yMsLAyCIODjjz+GUqlEnz59sHfvXkycONGo9y80NBRTp07FrFmz4OPjg8mTJ+PWrVuYM2cOli9fDg8PD0ydOtUgrJ84cQKurq5iaDT29c2vgIAAPH782KBqfGYV9O7duxt1jtmzZ8PFxQXh4eEYNWoUtmzZYvT0AmP6d/HiRTx69Ehsk8vlOHbsWLYwnBN/f39xfnxUVBQ2bNiANm3a5LjvwIEDsW/fvlyfllevXh2ff/65GLoznTt3Dubm5qhVq1aOx1laWqJr167Ys2ePwb+7kydPIjExEb169YKDgwNmzpyZbSRFZtX8bt264fr162V+eD5RaZTnIfoAIJPJ8M477+Cdd94xdX+IiKiE8a5bEZ51KiDuQTJS0tRwsJXB1cXR4Mm9VqdHUqoKKo3uJWfKIAgC9p66i53HboltZlIJhgc0wJuNqhbKz5ATiQSwt5bB3vbVw/JjY2ORmJiI4cOHo0mTJgAynpRFRkYiLS0N3377LWrXro1ly5bBzCyjXoG3tze6d++OrVu3YsiQIQAyfvZx48ahf//+AICmTZti//79OHLkCNq2bSsWfqpSpYr459DQUFhbW2P16tXi8mItW7ZEp06dsGLFCkydOlXsZ//+/cVw0rJlSxw4cABHjhzBwIEDcePGDezbtw8zZszAiBEjxH0ePHiAM2fOoHv37li0aBHatm2LRYsWieesVasWRo4ciaNHj+Y6Tc/DwwMymQzlypXLVrxq7NixBsfFx8fj448/NihCZ2lpiQ8//BDXr18Xj1coFJg3b564pF2tWrXQsWNHHD16FHXq1EF0dDSqVauGIUOGQCqVwtfXFzY2NkhOTgYANG7cWHy9vLy8cO3aNUREREAmk2Ht2rXitg4dOiAwMBALFiwwWBO7W7du6Nu3r8HPolAoMHv2bPFpeWxsLL799lvMmzdPXHs+PT0dEyZMwO3bt9GgQYM8vX+DBw82KOYVHR2N4OBgdOrUCUBG8S4nJyeDInYvLhFo7OubX56enqhevbrBMP1z584hKSkJnTp1wtatW195jvbt24s/d8uWLXHy5EkcOXJEHGpeEB06dIC1tbXBMP39+/ejfPnyOdYNKIhXVYrPyaFDh3D8+HEMGDAg1zXkgYwl2rZu3Yrz58+LBRmjoqLg6uqa61zymJgYREREoGPHjtmmShBR6ZHngJ9bAREgY41LGxsb1KxZE61bty6UdXKJiKj4kUol2QrpZVJpdEhOVUGje/Xa7VqdHhv3/Y0/Lj0U22ytzDG2jxfq1sj/utp5ZUy41+sF8UsNC5tKKFeuHMaOHQt/f3+0bdsWrVu3xuTJk6FQKBATE4NRo0ZBEARotVoAGU/v6tSpg5MnT4oBHwB8fHzEP2eG4pcNgT99+jR8fX1hZWUlntvOzg7NmjXDH3/8YbDv8+cGMr4oyDx35jD0F582Zi4vd+vWLTx69AhBQUHidQCgefPmsLOzw8mTJ9GhQweDbUDGMlo5VVTP1KBBA4O/Zy7RlZCQgLi4ONy9exeHDx8GkDGF4HnPh9HMommZP0+LFi0QGRmJPn36oFOnTmjfvj169Ojx0r6cO3cOHTt2FIM2kFE5vXv37ggLC0NaWlqu/c6U+QUPAFSoUAFAxpc5mTKHqKekpADI2/v34jXffPNNhISE4OrVq2jbtq1BMAYyvjA6fvw4vvvuO7EtL69vTnQ6ncFTY6lUmm2N84CAAOzYsQP/+9//IJFIEBUVhQ4dOhi8ri/z4pcMVapUwYMHD4w69lWsrKzg5+dnEPCjoqLQrVu3bPeGXq8X11UHMmpOZX5BVxh+//13TJ8+He7u7vjoo48AZLyHz0+jATLuSV9fX7zxxhuIiopCs2bNoFKpcODAAYwZMybHc58/fx5jx45FtWrV8PXXXxfaz0BERS/PAf/XX3/Fo0ePoFarYW5uDicnJyQlJUGr1UIikYi/9N3c3LB27VoO2SciKsMUKg2S5WrojBiSn67UYNn2y7h+N1Fsq+hsjfGvsVI+YFy4j7n5BFsO3cSDeDm0Oj3MzaTw7TkJaXePYs+ePYiMjISVlRV69eqFoKAgseDV8uXLs53L0tKwKr+VlZXB36VS6UuHvyclJWH37t3ZlgUDkO2/wS87d1JSEoCMNbpzuw4AzJkzB3PmzMm2PT4+Hvfv38+2pN3XX3+NPn365Nr/F6vWX758GXPmzMHly5dhbW0NNzc3cXj5i6+DtbW1wc/y/D4BAQFiRffw8HCEhITAxcUFkyZNQkBAQI59SUlJEUP58ypUqABBEMT5+zn1O1NOIfb5fr4oL+/fi9dcvHgxli5dij179mDfvn2QSqVo1aoV5s6dCxcXF1y+fFmcr50pL69vTjp37mwQtnOqOB8QEIBly5bhwoUL4vrqn3/++SvPnenF1+tV/wbyqlu3bhg/fjwePXoES0tLnDp1SgzUz8tcQi6Ti4tLtir4prJ69WrMnz8fzZo1Q1BQkPh7Yfv27Zg+fbrBvpnV+3v06IHNmzdj5syZOHz4MNLT09GjR49s5969ezemTZuGWrVqYcWKFXB2fn1flhLR65fngD9x4kTMnj0b33zzDfz9/cVfugcPHsSsWbMwa9Ys1KlTB5988gm+++47fPnll4XRbyIiKuZS09SQK9QwItvjaZICoZsv4tGz5yvlO2JsHy/Y2by+0WDGhvuwLTFQKLWwt7WAhZkFNDo9EtJtYV2zJyI+mgEoHmLnzp34+eefUblyZUgkEowcOTLHOcgvC3/GsLe3R6tWrQzWts70soJhL8qsHp6QkGCwhNitW7eQlJQkbp8yZUqO65Y7OjqiUqVKBsPYAeRp+bfMWgbu7u7icGOpVIqjR49i3759Rp8nU2BgIAIDA5GamooTJ05g+fLlmDx5Mpo2bYrKlStn29/BwQFPnz7N1v7kyRMAgLOzs8mLkxXk/bO3t8fkyZMxefJkxMXF4eDBgwgPD8ecOXMQERGBY8eOoVWrVrCwyChIaYrX98cffzR40p9TWKxfvz5q166NvXv3QqlUQqVSFatVltq1awdbW1vs3bsXNjY2qFatGho1apRtv/HjxxuMrimMkamCIGDevHlYt24dAgMDMWvWLMTGxorbO3bsmO3fVGYNi169emHZsmU4c+YMdu/ejebNm8PFxcVg35UrV2LhwoXw9fVFWFgY7O3tTf4zEFHxkueAHxISIhYAyiSRSNCpUyc8ffoUS5YswZ49ezB27Nhis9YtERG9PoIgIDlNjXSlBsY8dIt7kIwft8YgNT2rmF5zj8oYHvD6iukBxg/L33LoJhRKLco7WopDehNuX8SV4xvR0H8qth2Nw5zRLeHj44OoqCg8e/YMHh4eiIuLM5gbq1QqMWHCBLRv3x5ubm5G9/PF4dCZ1ecbNGggBkJBEDBp0iTUrFkz16HkL8p8ynvo0CEMHjxYbF+0aBEeP36MzZs3o3z58rh//z5GjRolbo+Pj8eUKVMwcOBA1KhRI9f5vy/2OydxcXFISkrC8OHDDV6TY8eOAYDBcOlX+eijj6DRaMRQ061bN1hYWCA4OBjx8fE5BvymTZvi8OHDkMvl4pN4nU6HqKgoeHp6FkrAy+/79+DBAwwePBjTp0+Hv78/XF1d4erqiosXL+Lu3bsAMl63zHoOgGle3+dXb3iZgIAAbN26Fenp6ejcuXO2kSpFSSaToVOnTti3bx+srKxyLf5XrVq1PH1BlR/fffcd1q1bh3fffRdTp07NVujQ2dk51yfuderUQcOGDREVFYWjR49i5syZBts3bdqEBQsWICAgAPPnz+fUWaIyIs8B/+HDh6hZs2aO21xcXMRhW5UrVxYL2RARUdmg0wtISlVCpdbBmAG1F28+xcbfY6HRZgWL7q1rI7BN7ZfOlTY1iQRwsJG9crRA3INkPIiXw97WwqB/zlXcAEGPuJMroUzshO1V0xFz7hhSU1PRpUsX+Pn5YcyYMfj000/Rs2dPsXr8xYsXMW7cuDz11cHBARcuXMDZs2fRrFkzjBs3DgMHDkRQUBAGDRoES0tLREZG4sCBA/jhhx+MPm/9+vXh7++PhQsXQqlUokGDBjh27BgOHz6M0NBQmJmZ4eOPP8asWbMglUrh5+eHlJQUhIeH4/Hjx2jYsOEr+3316lVER0fDy8srx31q164NOzs7LF26FObm5jA3N8e+ffvEJ5h5qfLeokULzJ49G/Pnz0e7du2QkpKC0NBQ1KpVC/Xr18/xmKCgIJw8eRLDhw/HmDFjYGFhgfXr1+PevXtYsWKF0dfOi/y+fy4uLqhSpYq4JFqNGjVw5coVHD16FEFBQUhISMCVK1fEGgqAaV/fVwkICEBYWBh27tyJ8PBwk503NxcvXsTq1auztbdt2zbHYncBAQEICgqCVCrNFoyNtXXr1myF8KRSKYYPH27U8deuXcPy5cvh6ekJf39/xMTEQKlU4s6dO9BoNGjUqNEr6xb06tUL8+fPh7m5uUEBxidPnuDrr7+Gi4sLhgwZgqtXrxocV6NGDU6jJSql8hzw3dzcsHnzZrRt2zbbti1btqB27doAMtaRzRxCREREpZ9Gq0NSqgpq7aufAgqCgAuxafjj78dim5lUgmHdGqCF5+urlA8AUglgb0S4B4CUNDW0Oj0szCwM2q1sHdG8+0Rcj96J2FMbMfvUWrjXq4eQkBBxfeyVK1ciNDQUEyZMgIWFBRwdHdGvX788Vy0fO3YswsPDMXr0aOzevRv169fHhg0bsHjxYkyZMgWCIKBevXoICwvLNh/+VRYuXIjQ0FCsWbMGiYmJqFOnDn744QexSnv//v1x+fJl7NmzB7/88gtsbGzQpEkTLFq0CNWrV3/pud977z189dVXGDVqFFatWpXjPvb29ggPD8eCBQswceJE2NraokGDBli/fj1Gjx6Nc+fOZVszHACuX78uFpfbsGEDPvzwQwwcOBAajQabNm3Cxo0bYWVlhZYtW2Ly5MnikPUX1alTBxs3bsR3332H6dOnQyKRwMvLC2vXrhUrlZtaXt6/O3fuYOzYsThz5gxkMhlCQ0Px3XffYcmSJUhMTETVqlUxfvx4jBkzBrt27YKtrS0GDRokzhtv1qwZ3n77bcTGxub6+v7+++/Yvn27OAUzpy/aFi1ahOXLl8PX1xfr1q0DgGxD/N3c3FCvXj08efIErVq1yvFn9/PzQ2JiVs0NY+7XYcOG4cGDBzh69KhB+4kTJ3DixIls+zs7O+cY8Fu1agUrKysoFIpXji4QBAE//fQTIiMj8fDhQ3G6Sk5fXJiZmRkd8H///XcIgoDLly9jwIAB2bavXbv2lcvYZa7w0LFjR4Ph90ePHoVSqcSDBw8MphlkelVtDCIquSRCHquWHDlyBMHBwahXrx66dOmC8uXL4+nTpzhw4ACuX7+OH374AVWrVsW7776Lvn37YsqUKYXV93y5fPkyAOQ6hNDU0tPTce3aNTRo0CDXgjxErwvvRyosKrUWiXIVdLpX/ydFp9Nj3Z6rOH0lK9zbWJojqI8X3Gu+3uJPUglgbyuDnbVxQ1dj7yXhq9XRsLI0g6VF9mraKrUOSrUOM0b65rqqQCZ3d3eMHz8eH374YX66XmT8/Pzg6+tbrKbhTZgwAadPn8bChQtRrly5PP83vqT8bvzxxx9x6dIlo9eEnzZtGqKjo8WAb8w9N23aNOzcuVMsUJjT0nF+fn548OCBQcC/evUq7OzsUKNGDaN/nhePMaZ/w4YNw7lz5yAIAo4cOWJQLwLIeC9btWoFhUKRYwHA523btg3Tp08Xi9blZv78+Vi3bh0mTJgAT09PHDt2DD/99BPmzp2bYzA3liAIGD16NC5fvowJEybA1dUVx44dw6pVqzB27NgcC/+R6b3ubEBU2PL8BL9Dhw5YuXIlQkJCEBoaCp1OB3NzczRt2hRr1qxBs2bNcOjQIXTv3p2/mIiIyoA0pQYpcjX0RnxfrFBqEbHjMq7dSRDbKjhZY3x/b1Qpb1uY3cwmr+EeAFxdHOFSyQ53HqZA5iA1eLopCAJSFRrUquoAV5fc168m00tMTES9evXQvn37ou5KoTp27Bh69epV6NepWrUqBEEwWMs+08WLF/H48eNs66h7eHjk+Tr5OSbzuNjYWIOl7jIdPnwY1tbW4lP2grp//z5Wr16Nzz77TKxN0bJlSzx8+BAnTpwoUMC/evUqjh8/ju+//x7dunUDkLGk4t27d7FmzRpMnDjxtU5VIqLSIV/Vi1q0aIENGzbgzz//xPHjx3Hp0iUx3AMZ3+zOmjWLxTyIiEq5FLkayXKVUeH+WbICC9efMwj3taraY+rwZsUq3F+5cgUjRoxA06ZN4ePjg5EjR+LixYsZx0klaFgpHdcPhWDfig/x+08f4+LBVUhNTsKzFBVsLM1RQX8TjRo1RExMDAYMGABPT0907NgRK1euFK+RWagsNDTUoGjZjRs3EBQUhCZNmqBJkyYIDg7GvXv3xO1nzpyBu7s7Tp06hffeew/e3t5o3bo1Fi5caLBWtlqtxvfff4+33noLXl5eCAwMNFjuCwAOHDiAPn36wNPTE61bt8aXX34priGfm8wnt9u3b4e7uzvu37+Pbdu2wcPDA5s3b0br1q3FonE6nQ4REREIDAyEl5cXGjdujIEDB+L06dPi+UJCQtC5c2ccOXIEPXr0QKNGjdC1a1fs2LHD4Lpr1qyBv78/PD090bZtW3z++efiknXu7u6Ijo7G2bNn4e7ujm3btgHIGM4+YcIEtG7dGo0bN8awYcNw/vx58Zz379+Hu7s7Vq1ahd69e2PkyJHYuXMnQkJC4O/vj/379yMwMBCenp7o1asX/vzzT1y8eBH9+/cXX9NTp04Z9NPY92/Tpk3o2LEjmjRpgpMnTyIhIQGffvopWrduLV7vxdcgOTkZMTExaNeuHQAY9foWhL+/vziE/Hm7d+9Gq1at4OTkZNDu5+eHadOmGby2e/bswYQJE+Dj4wNfX1/MnDnT4B57/pi8sLGxQfv27bF3795s23bv3o2uXbvmaQWJlzlw4AAsLS3Rr18/g/bvv//eoL7Biy5cuAB3d3ccPnzYoP3atWtwd3fH/v37AQADBgxAy5YtDfZ54403kJ6ejmfPnpnkZyCisiVfAV+lUuHSpUu4dOkS7ty5g/Pnz+PMmTM4cuQIFi1aZOo+EhFRMaPXC0hMVUKuVBtVKf/OwxTMX3sO/z5NE9vqvmGJcX0awv41LoMHZIR7h1zCfeYyYs7OzggJCcHixYuhUCgwatQopKam4uzZs/jys49Qu1p5tOoxDrWb9cHTB9dxfvf3qFbBCuP6eaN6ZXvo9XpxxZmIiAg0adIECxYswPHjxwEAkZGRAIB+/fqJf759+zYGDhyIZ8+eYf78+Zg3bx7u3buHQYMGZfugP2nSJDRt2hRLly5FYGAgVqxYgc2bNxtsX7VqFfr3749ly5ahTZs2mDZtGnbt2gUA+O233xAcHAxXV1eEhYVh/Pjx+PXXXzFu3LiXrjceGhqKihUron379oiMjBRr7WQWDZw3bx6mT5+OOnXqYNGiRQgPD8eAAQOwYsUKfPHFF0hKSsLEiRMNirk9efIEc+fOxfDhwxEREYFq1aph6tSpuHXrFgBg165dWLhwIYYMGYKVK1ciODgYO3fuxBdffCG+lh4eHvDw8EBkZCQ6dOiA2NhY9OnTB/fv38fMmTOxaNEiSCQSjBgxAtHR0QY/U0hICEaOHIlx48aJ9RIePXqEb775BmPHjsWSJUuQkpKCCRMm4JNPPkH//v0RFhYGQRDw8ccfQ6lU5vn9Cw0NxdSpUzFr1iz4+Phg8uTJuHXrFubMmYPly5fDw8MDU6dONQjrJ06cgKurq7hmvbGvb34FBATg8ePHuHDhgtim1+uxd+/eXKvOv2j27NlwcXFBeHg4Ro0ahS1bthg9vcCY/l28eBGPHj0S2+RyOY4dO4bAwECTXAPICOQ1a9bE2bNn0bt3bzRs2BB+fn7iv9vcNGnSBDVq1EBUVJRB+65du+Dk5IT27dujYcOGmDt3brYvS86fPw9nZ2cWwSOifMnz15tnzpzBxIkTc62Qb2tri0mTJhW4Y0REVDzpdHokpaqg1OhevTOAP6/H46ff/jKolN+5eTXUq6R+rcvgAVnh3jaXYfmxsbFITEzE8OHD0aRJEwCAq6srIiMjkZaWhm+//Ra1a9fGxnWrIJFIEfcgGddvdMOk8cPh4Xgf3nU74NbljOH648aNE5coa9q0Kfbv348jR46gbdu2YmG9KlWqiH8ODQ2FtbU1Vq9eLVbObtmyJTp16oQVK1aIReSAjGJ3wcHB4j4HDhzAkSNHMHDgQNy4cQP79u3DjBkzMGLECHGfBw8e4MyZM+jevTsWLVqEtm3bGnwpX6tWLYwcORJHjx7Ndc1yDw8PyGQylCtXLltxwLFjxxocFx8fj48//hjDhg0T2ywtLfHhhx/i+vXr4vEKhQLz5s0Tn2LWqlULHTt2xNGjR1GnTh1ER0ejWrVqGDJkCKRSKXx9fWFjYyN+DmncuLH4emWec+7cuZDJZFi7dq24rUOHDmJBsufXFe/WrRt69eqFa9euiUvnKRQKzJ49W3xaHhsbi2+//Rbz5s0Tn+Smp6djwoQJuH37Nho0aJCn92/w4MEGFc+jo6MRHBwsFjP09fWFk5OTwUjIY8eOif3Jy+ubX56enqhevbrBMP1z584hKSkJnTp1wtatW195jvbt24s/d8uWLXHy5EkcOXIEn376aYH6BmS8n9bW1gbD9Pfv34/y5cvnWDcgvxISEvD48WNMmjQJ48ePh6urK3bv3o1Zs2YBwEuH6Pfs2RM//fQTlEolrKysIAgCdu/eDX9//1xHuW7cuBFXr17FJ598YtTSkkREL8pzwF+8eDGcnZ3xxRdf4Ndff4VUKkWfPn1w7Ngx/Pzzz1i+fHlh9JOIiIoBtSajUr5GZ1yl/P3R/2D74VhxyTypVIKh/vXhU9dZfEL7uuQW7vV6AXEPkpGSpoaFTSWUK1cOY8eOhb+/P9q2bYvWrVtj8uTJUCgUiImJwahRoyAIAgRBh1pV7VCramMsdauDU6f+wLBhQ8Xz+vj4iH/ODMUvGwJ/+vRp+Pr6wsrKClqtFgBgZ2eHZs2a4Y8//jDY9/lzAxlfFGSeO3MYepcuXQz2yRxOfOvWLTx69AhBQUHidQCgefPmsLOzw8mTJ9GhQweDbUBGdfCXzQd+cb32b7/9FkBGQIqLi8Pdu3fF4cpqtdpg3+fDaGbRtMyfp0WLFoiMjESfPn3QqVMntG/fHj169HhpX6Kjo9GxY0eDJcbMzc3RvXt3hIWFIS0tayRJbuvMZ37BAwAVKlQAkDE/OlPmU9eUlBQAeXv/Xrzmm2++iZCQEFy9ehVt27Y1CMZAxr+l48eP47vvvhPb8vL65kSn0xmM1pBKpdkCZUBAAHbs2IH//e9/kEgkiIqKQocOHV65dFumF79kqFKliricckFZWVnBz8/PIOBHRUWhW7du2e4NvV4PvT7rd5ZEIoGZWfYimTnRaDRITExESEiI+G+qZcuW+PfffxEaGooBAwbkev6ePXsiNDQUhw8fRrdu3XDhwgX8+++/udZRWL9+PRYtWoQWLVpg6NChOe5DRPQqeQ74169fx5dffonOnTsjNTUVmzZtQvv27dG+fXtoNBr8+OOPiIiIKIy+EhFREVKoNEiWq6HTG1EpX6/Hpt9v4PjFrA/z1pbmCOrtifq1ypl0vW1jSCWAg50lbK0Ml0eLufkEWw7dxIN4ObQ6PczNpPDtOQlpd49iz549iIyMhJWVFXr16oWgoCDo9XosX748xy+zX1xqy8rKyrAP/y07lpukpCTs3r0bu3fvzrbtxaG6Lzt3UlISAKB8+fK5XgcA5syZgzlz5mTbHh8fj/v372dbsuxVy2q9WH3+8uXLmDNnDi5fvgxra2u4ubmJw8tffB2sra0Nfpbn9wkICBAruoeHhyMkJAQuLi6YNGkSAgICcuxLcnKyGMqfV6FCBQiCIM7fz6nfmXIKsc/380V5ef9evObixYuxdOlS7NmzB/v27YNUKkWrVq0wd+5cuLi44PLly1CpVAZPpvPy+uakc+fOBmE7p4rzAQEBWLZsGS5cuIDGjRvj999/x+eff/7Kc2d68fV61b+BvOrWrRvGjx+PR48ewdLSEqdOncqxwPOMGTMMalC4uLiIKwu8iq2tLSQSSbYCjm3btsWJEyfw9OlTLFq0KMfz16xZEz4+PuIXD1FRUahRo4bBl0dAxhcQCxYswKpVq9CtWzcMHjyYxfWIKN/yHPD1er04hK1mzZq4efOmuK1r164G3zgTEVHpIFeokZqmMa5SvkqL5Tsu4+rtrGJ65R2tML5/Y1St8HqL6QGAVCKBg50sx3AftiUGCqUW9rYWsDCzgEanR0K6Laxr9kTERzMAxUPs3LkTP//8MypXrgyJRIKRI0fmOAf5ZeHPGPb29mjVqhXefffdbNvyUjAss3p4QkKCwRJit27dQlJSkrh9ypQp8PX1zXa8o6MjKlWqZDCMHcBLlxF7UWYtA3d3d0RFRcHV1RVSqRRHjx7Ntl66MQIDAxEYGIjU1FScOHECy5cvx+TJk9G0aVPxM8mLP8PTp0+ztT958gRAxtro8fHxee7HyxTk/bO3t8fkyZMxefJkxMXF4eDBgwgPD8ecOXMQERGBY8eOoVWrVrCwyLiHTfH6/vjjjwZP+p2dsy9RWb9+fdSuXRt79+6FUqmESqXKdfpGUWjXrh1sbW2xd+9e2NjYoFq1amjUqFG2/caPH2+wFnxeikDXrFkTgiBAo9EYfImXOUrDysrqpefv2bMnvv76a6SmpmLv3r0YNGiQwfnVajU+/fRT/P7773jvvfcwfvx4/P3330b3j4joRXkO+DVq1MD169fRrFkz1K5dGwqFAnFxcXB1dYVWqzUY9kZERCWbIAhISVMjTakxqpheQrISYVti8OBJ1hPS2m844IO+3nCwff0rq0glEjjayWDzQrjX6wVsOXQTCqUW5R0txadlCbcv4srxjWjoPxXbjsZhzuiW4hO4Z8+ewcPDA3FxcQbrJSuVSkyYMAHt27eHm5ub8X17YTh0ZvX5Bg0aiIFQEARMmjQJNWvWzHUo+Ysyn/IeOnRIXNYLyCjK9vjxY2zevBnly5fH/fv3MWrUKHF7fHw8pkyZgoEDB6JGjRq5rgltzLzguLg4JCUlYfjw4QavybFjxwDAYDjzq3z00UfQaDQICwuDvb09unXrBgsLCwQHByM+Pj7HgN+8eXMcPnwYcrlcfBKv0+kQFRUFT0/PQlnlJ7/v34MHDzB48GBMnz4d/v7+cHV1haurKy5evIi7d+8CyHjdMus5AKZ5fZ9fveFlAgICsHXrVqSnp6Nz587ZRqoUJZlMhk6dOmHfvn2wsrLKtfhftWrV8vQF1fPat2+PlStXIioqymC+/aFDh+Du7g47OzvY2dnlev6AgAB89dVXWLJkCZ49e4aePXsabJ8+fTr279+P6dOnY+TIka9cyYKI6FXyHPB79OiBRYsWQRAEDB06FI0aNcIXX3yBYcOGYenSpXn6cENERMWXTi8gKVUFlVoLYwbV3n2UgrDNMUhJy3oq2MS9EkYGekBmYdx8V1PKLdwDQNyDZDyIl8Pe1sJgKKxzFTdA0CPu5EooEzthe9V0xJw7htTUVHTp0gV+fn4YM2YMPv30U/Ts2VOsHh8TE4Nx48blqX8ODg64cOECzp49i2bNmmHcuHEYOHAggoKCMGjQIFhaWiIyMhIHDhzADz/8YPR569evD39/fyxcuBBKpRINGjTAsWPHcPjwYYSGhsLMzAwff/wxZs2aBTMzM3Ts2BEpKSkIDw/H48eP0bBhw1f2++rVq4iOjoaXl1eO+9SuXRt2dnZYunQpzM3NYW5ujn379omjAvIyRaNFixaYPXs25s+fj3bt2iElJQWhoaGoVasW6tevn+Mx48ePx7FjxzB8+HCMGTMGFhYWWL9+Pe7du4cVK1YYfe28yO/75+LigipVquDLL7+EXC5HjRo1cOXKFRw9ehRBQUFISEjAlStXDJZkM+Xr+yoBAQEICwvDzp07ER4ebrLz5ubixYtYvXp1tva2bduiTp06OfYvKCgIUqkUM2fOzNc1t27dCkdHR4M2qVSK4cOH480330THjh3x9ddfQ6FQoG7dutixYwcuXLhg1OuRWTF/48aN8PHxQc2aNcVtBw4cwK5du+Dn54fGjRvj4sWLUCqVuHPnDjQaDZo0acIlp4koz/Ic8N9//30kJiYiJiYGQ4cOxezZszF69GiMGzcOdnZ2Jlv+hIiIio5Gq0dSqhJqrXFPWi/eeIKffrsCtSZr/64taqJX+zqQFsFc0peFewBISVNDq9PDwsxwu5WtI5p3n4jr0TsRe2ojZp9aC/d69RASEiIuobZy5UqEhoZiwoQJsLCwQMOGDbFq1ao8Vy0fO3YswsPDMXr0aOzevRv169fHhg0bsHjxYkyZMgWCIKBevXoICwvLNh/+VRYuXIjQ0FCsWbMGiYmJqFOnDn744QexSnv//v1ha2uLFStWIDIyEjY2NmjSpAkWLVqE6tWrv/Tc7733Hr766iuMGjUKq1atynEfe3t7hIeHY8GCBZg4cSJsbW3RoEEDrF+/HqNHj8a5c+fg5+dn1M8ycOBAaDQabNq0CRs3boSVlRVatmyJyZMni0PWX1S3bl1s3LgR3333HaZPnw6JRAIvLy+sXbsWzZo1M+q6eVWQ9y80NBTfffcdlixZgsTERFStWhXjx4/HmDFjsGvXLtStW9dgpIIpX99XcXNzQ7169fDkyRO0atXKJOd8mRMnTuDEiRPZ2p2dnXMM+K1atYKDgwOqVq2a43Zj5BTUzczMMHz4cADAkiVLEBoailWrViEhIQFubm4IDQ01+jXu1asXDhw4gB49ehi0//777wAyRgPkVBPg4MGD+R55QERll0TIY7WTW7duZfsFKpfLxWH6xlZWLSqXL18GgFyHHppaeno6rl27hgYNGuRayIfodeH9SMZQqbVIkmcE4FcRBAGHzt3DloM3syrlSyQY7O+ONt4uuR6nUCjE/54UdO76i6QSCZzsZbC2zDn8AUDsvSR8tToaVpZmsMxhdIFKrYNSrcOMkb5wq+5k0v5R8cPfjVRc8F58/V53NiAqbHleYHPw4MHYsWOHQZudnR28vLyKfbgnIqKXS1dqkJiqMirc6/R6bNp/A5ufC/dWlmYY/473S8N9YTIm3AOAq4sjXCrZITVdk62qtyAISFVo4FLJDq4ujrmcgYiIiKj4yXPAt7CwyLHSKhERlWypaWoky1VGLYOnVGkRvuUSjl64L7aVc7DClKHN4FE75+XZCpux4R4ApFIJ+vnVhbWlOZ6lqKBS66DXC1CpdXiWooKNpTn6+dWFVMqlqoiIiKjkyPMc/IkTJ2LBggVITU1F/fr1cxw+lLkOKxERFX96vYDkNBUUKq1RlfITU5QI3WxYKb9mVQeM6+sFR7uiqbCdl3CfybtuRQT388aWQzfxIF4OuUIDczMpalV1QD+/uvCuW7EQe0xERERkenkO+J9//jl0Oh0mT56c6z7Xrl0rUKeIiOj10On0SEpVQanRGbX/P49SELblEpLlKrGtcb2KeK9HwyKplA/kL9xn8q5bEZ51KiDuQTJS0tRwsJXB1cWRT+6JiIioRMpzwP/yyy8Lox9ERPSaqTU6JMtVRlfKv3TzCVb8algpv7NvDfTu6FYklfKBgoV78RxSCQvpERERUamQ54Dfu3fvwugHERG9RgqVBslytVHz7QHg0Ll72HzghkGl/IFd6qGdT9Et4WSKcE9ERERUmuQ54AOAWq3Gli1b8Mcff+DJkyf46quvEB0djYYNG8LLy8vUfSQiIhOSp6uRmq6B3ogJ93q9gM0Hb+Dw+axielYyM4x+2xMNXYummB6Q8dTdyY7hnoiIiOh5eQ74CQkJGDFihLjufWxsLJRKJY4cOYJvvvkGq1evho+PT2H0lYiICkAQBCSnqZGu1BhVTE+p1mLlziu4fOuZ2ObsYInx/RrDpVLRLYsqlTDcExEREeUkz8vkLViwAGlpadi9eze2b98urh/8ww8/wNPTEz/88IPJO0lERAWj0wtISFEhXWFcuE9MVeLb9ecNwn2NKvaYNrx50Yd7DssnIiIiylGeA/7hw4cxceJE1KxZE5LniipZWlrivffew19//WXSDhIRUcFotHokJCugVGthzIz7e49TMX/tOdyLz1oGz7tuRXw6uGmRLYMHMNwTERERvUqeh+irVCo4OTnluM3MzAwajaagfSIiIhNRqbVIlKug0xlXTO9y7FOs2HkFqueWzevkWwN9OrgV6dJxUokEjhyWT0RERPRSeX6C7+npiY0bN+a47bfffkOjRo0K3CkiIiq4NKUGCanGh/sj5+8hfGuMGO4lEmBQF3f086tbxOEecLCTwcaK4Z6IiIjoZfL8BH/ixIkYOXIkevXqhfbt20MikWDXrl0ICQnBiRMnsGLFisLoJxER5UGKXI00pRrGrIKn1wvYcugmDp27J7ZZyswwulcjNKpToRB7+WpSCeBgK4Mtwz0RERHRK+X5CX6zZs2watUqWFtbY8WKFRAEAatXr8aTJ0+wbNkytGjRojD6SURERtDrBSSmKiE3Mtyr1Dos3XbJINw721ti8pCmxSLc29vKYGstK9J+EBEREZUUeX6CDwDNmzfHpk2boFQqkZycDDs7O9ja2pq6b0RElAdanR7JqSoon5s//zJJqSqEb4nBP49Txbbqle0R3M8bTvZFV0wPyJgeYG8jgx3DPREREZHR8hzw3377bbz99tsIDAxEhQoVYGVlVRj9IiKiPFBrdEhKVUGj0xu1//34VIRtjkFiqkps83KrgPd6NoSVLF/f/ZqMRAI42MhgZ8NwT0RERJQXeR6i/8Ybb+Dbb79F+/btMWrUKPz2229QKpWF0TciIjKCQqVBQorS6HD/V9wzLFx/3iDc+zWrjrF9vIo83EsZ7omIiIjyLc8BPzw8HH/88QfmzJkDQRAwbdo0tGrVClOnTsUff/wBQTCuWjMRERWcPF2NpFQ1dMZMuAdw9MJ9hG2OgUqdVSl/QKd6eKdTvSKtlA/8N+ee4Z6IiIgo3/L1qMbe3h79+vVDv3798OzZM+zduxd79+7F6NGjUaFCBRw9etTU/SQioucIgoCUNDXSlBoY872qXi9g25FYHIj+R2yztDDDqF6N4OVWtMX0AIZ7IiIiIlMo8FjMZ8+e4enTp0hJSYFOp4Ojo6Mp+kVERLnQ6QUkpaqgUmthzHN7lVqHn377CzE3n4htjnaWCO7njRpV7Auvo0bKrJbPgnpEREREBZOvgH/v3j3s2rULu3fvRmxsLCpUqIDAwEDMnz8f9evXN3UfiYjoPxqtHkmpSqi1xs23T5ZnVMq/+yirUn61SnYI7ucNZ4eiL5IqlUoY7omIiIhMJM8Bv2/fvrh69SqsrKzQuXNnTJs2DS1btoRUmjGdXxAESCRFO4+TiKg0Uqm1SJKroNUZN9/+wRM5wjbHICElqxBqozrl8X7PRrCyLNpiegBgYWEORxtLhnsiIiIiE8nzJzwnJyd888036NKlC6ytrcX2+Ph4/PLLL9i6dSsOHz5s0k4SEZV1aUoNUtOML6Z39fYzROy4DKVKJ7Z1aFIN/TvVhZk0z/VVTU4qlcDBRgZrK7Oi7goRERFRqZHngL9y5UqDvx8/fhybNm3C0aNHodVqUa1aNZN1joiIgBS5GmlKNYzM9jh+8QF+3ncd+v+q70kA9O9UD37NqhdeJ/NAKgEcbS2RIjVumgERERERGSdfYzQTEhKwZcsW/PLLL3jw4AHs7OzQu3dv9OrVC82aNTN1H4mIyiS9XkBymgoKlda4SvmCgO1HYrH/TFalfJmFFKN6NoJ33YqF2FPjSSWAg60MEkELnU736gOIiIiIyGh5CvinT59GZGQkDhw4AJ1Oh6ZNm+LBgwcICwuDr69vYfWRiKjM0en0SEpVQakxLgSrNTqs2vUX/rz+fKV8Gcb180bNKg6F1c08yQz3ttYypKdri7o7RERERKWOUQF/9erViIyMxO3bt1GzZk2MGzcOvXv3ho2NDXx9fVlUj4jIhNQaHZJSVdDojBvCnpKmQviWS7jzMEVsc6loh+D+3ihXDCrlA4BUIoGDnQy2VhZF3RUiIiKiUsuogP/NN9/A3d0da9euNXhSn5qa+pKjiIgorxQqDZLlxhfT+/epHKG/GFbKb+haHqN7FY9K+QDDPREREdHrYlQp5e7du+Pu3bsICgrCuHHjsH//fmi1HF5JRGRK8nQ1klINw73wksn31+4kYMG6cwbhvp2PC8b18yrycJ/Zb6lEAkeGeyKTednvBCIiIqMC/rfffosTJ05gypQpePLkCT788EO0bdsWCxYsgEQi4RB9IqICEAQBSXIVUtLVYuV7AFi3egUif16X4zEnYh4g5JeL4jJ4EgD9/OpiUBf3Il8GL+q37VgaulgM9zbFINxfv34db7/9Nho1aoSAgICi7k6huXr1Knx8fKBWq4u6K0a7f/8+3N3dsW3btqLuSrHj5+eHadOmiX8PDw83WM0oJCQE7u7uOe5fEl7XF/ufk8yfw93dHZGRkTnuk5qaCk9PT7i7u+PMmTMAgG3btsHd3R337983uj8vHmNM/86cOSP278SJEznuc+vWLXGfV/UnICAAS5cuNbrPcrkcfn5+Ob7Pf/31F0aPHo0WLVrgzTffxHvvvYe//vrL6HO/zN27dzFx4kS0adMGTZs2xaBBg3Dq1CmTnJuICsboT4F2dnYYNGgQNm/ejN9++w29evXCoUOHIAgCZsyYgSVLliA2NrYw+0pEVOro9AISUpRIV2iyVcpfteJHKBUKg7bMSvnr9/wN/X9P+i3MpQjq44VOvjWKxReu69esREpKCpzsi0e4B4CwsDD8+++/CAsLw/z584u6O4Xm6NGjaNGiBWQyWVF3hQrBkiVLoHjud0L//v1zDb2VKlVCZGQkOnTo8Jp6V7ikUin27t2b47b9+/dn+1KrQ4cOiIyMRKVKlYy+Rn6OMaZ/u3fvzvP5jJGcnIygoCA8ePAg27a7d+9i6NChUCqVmDdvHr7++muo1WoMHjwYcXFxBbpuYmIihg4diri4OMyYMQOLFy9GhQoV8N577yE6OrpA5yaigsvXY566deti2rRpOHr0KEJCQuDq6orly5ejR48e6Nmzp6n7SERUKmm0OiQkK6BU62DMoFu1RocVO69g3+m7YpuDrQyThjRF43rFYxm8TDILKawti0e4BzI+kNarVw/t27eHp6dnUXen0Bw7dgzt27cv6m7Qa1KlShU0btw4x20ymQyNGzdGuXLlXm+nCkmTJk1w5swZJCQkZNsWFRWFBg0aGLSVK1cOjRs3ztOXXfk55vn+5TaFdffu3dn6V1AHDx5E7969c324tm7dOlhbW2PZsmV466234Ofnh4iICFhbW2P9+vUFuvaOHTuQmJiIFStWICAgAO3atcP333+POnXqGIwwIaKiUaBxnObm5ujcuTOWLl2KI0eO4JNPPuHcfCIq9a5cuYIRI0agadOm8PHxwciRI3Hx4kVx+7lz5zB06FB4e3vD19cXU6dONfhQum3bNnh4eOD4qbMYM2o4uvq1xMC+3RG5ca24j1+bpgCAtasi4NemKVLS1Fj88wWcir6EB9E/IXbvZ7i1bxYk/2yFhT6rev7FC+fg16YpLpyLxuSPx6HbW63Qt2cXRIT/YLDuvEajwU/LwzGkf0/4+7XCe8Pewb49vxn8nCePH8HYUUPR1a8l+vbsgtDvFxo8PczJoH6BePzoIX77dac4HDXz5928eTNat24NX19f3Lp1C3q9HqtWrUJgYCC8vLzQuHFjDBw4EKdPnxbPFxISgs6dO+PIkSPo0aMHGjVqhK5du2LHjh0G112zZg38/f3h6emJtm3b4vPPP4dcLgcAuLu7Izo6GmfPnjUYsnznzh1MmDABrVu3RuPGjTFs2DCcP39ePGfm0OBVq1bB398f3t7e2Lp1K0JCQuDv74/9+/cjMDAQnp6e6NWrF/78809cvHgR/fv3h5eXFwIDA7MNWb1x4waCgoLQpEkTNGnSBMHBwbh37564PXO476ZNm9CxY0c0adIEJ0+eREJCAj799FO0bt1avN6Lr0FycjJiYmLQrl07AMCwYcMwbdo0LF26FK1atULTpk0xbty4bE/7Dhw4gMGDB8PHxweNGjWCv78/NmzY8Mo+AcDmzZvRp08fNG7cGF5eXujVqxf27NkjHps55DkmJga9e/eGl5cXevTokeOTzidPnmDChAnw8fGBr68vPvvsM6SlpYnb/fz88NVXX2HEiBHw8vLC//73PwDA33//jfHjx6NFixZo2LAh2rZtiy+//BJKZVZtipMnT+Kdd96Bj48Pmjdvjg8++AC3bt0yuP65c+cwePBgeHp6onXr1vjyyy+Rnp6erZ8vcnd3x88//4xp06ahadOm8PX1Fa8/f/58cXj0//73P6hUKgC5D5+fNm0a/Pz8cr0OAISGhop/ftkQ8hevkfnvMCYmBgMGDICnpyc6duyYLZDFx8fj448/hq+vL5o3b45Zs2Zh8eLFBv1yd3dHSEiIwXE59eVV90dedO7cGVKpFPv37zdoT0xMxOnTp9G9e3eD9heH20+bNg0jR47E1q1b0bVrVzRq1Ai9evXCsWPHcj0mLwICApCUlGTw+wvIuD/v3LmDbt265fmcuUlJScH48ePRvHlzrFixIsd9XF1d8d5778HGxkZss7GxQZUqVfDPP//keu6lS5eiUaNGSE5ONmhfvXo1GjZsiGfPnqFy5coYOXIkKleuLG43MzNDzZo1X3puIno9TDZRs0KFChg9enShDUMiIioO5HI53n//fTg7OyMkJASLFy+GQqHAqFGjkJqairNnz2LkyJGwsrLC999/jxkzZiA6OhrDhw8XA4dKrYVer8fs/01Fx7e64OuFS+Dp1RjLwpfg7Jk/AAChS1cDAAICe+Hz+T9iwdqzuH7jFu79EQatKg3NuozC5Gmz8Cz+ESaMew+JiYZPtebNnQkvbx/MW/A93ursj00b12D3bzuyts/5HzZvWo+AHm/jqwXfo7lvC8yf9zkO7s8IXgd/34PPpn+KGjVr4YuvvsWI98Zg/77d+GzaJ7kW+ZJKJViy5AdUrFgR7du3NxjqqtPp8NNPP2HevHmYPn06XF1d8fPPPyMiIgIDBgzAihUr8MUXXyApKQkTJ040+CLhyZMnmDt3LoYPH46IiAhUq1YNU6dOFQParl27sHDhQgwZMgQrV65EcHAwdu7ciS+++AIAEBkZCQ8PD3h4eIhDlmNjY9GnTx/cv38fM2fOxKJFiyCRSDBixIhsQ0xDQkIwevRoLFiwAK1btwYAPHr0CN988w3Gjh2LJUuWICUlBRMmTMAnn3yC/v37IywsDIIg4OOPPxbf99u3b2PgwIF49uwZ5s+fj3nz5uHevXsYNGgQnj17ZnDN0NBQTJ06FbNmzYKPjw8mT56MW7duYc6cOVi+fDk8PDwwdepUgzBx4sQJuLq64o033hDbDh48iG3btmHmzJmYM2cOrl27hmHDhomv75EjRxAcHIyGDRsiPDwcISEhqF69OubOnYuYmJiX9mnDhg2YNWsWOnXqhGXLlmHRokWQyWSYNGkSHj16ZHBsUFAQ3nrrLYSGhqJ27dr46KOPcPToUYN9lixZgqpVqyI8PBwjRozAL7/8gtDQUIN9NmzYAE9PT4SHh6Nfv36Ij4/HkCFDoFAo8M0332D58uXo3r071q1bh7VrM74wu3fvHsaNG4dGjRrhxx9/xLx583D79m2MGTMGen3GUpR79uzBd999h9q1ayMsLAzjx4/Hr7/+inHjxhlV1G7hwoWQyWQIDQ3F22+/jXXr1uHtt9/Gw4cPsWjRIgwbNgxbtmzBunU519QwRuZQ/H79+uU6LP9V9Ho9PvroIwQEBCAiIgJNmjTBggULcPz4cQCAWq3GiBEjcOHCBcyYMQNff/01/v77b/z00095vlZe7g9jODg4oHXr1tm+HNq3bx/eeOMNeHl5vfIcV65cwcqVKzFhwgSEhYXBzMwMH374YbYwmx9ubm6oW7dutv5FRUXB19cXFSuabpSVlZUVoqKiMH/+fDg7O+e4z+DBg/H+++8btN29exc3b95E3bp1cz13jx49oNVq8fvvvxu0R0VFoU2bNihfvjwCAgIwadIkg+3Jyck4e/bsS89NRK9H8VhDiYiohIiNjUViYiKGDx+OJk2aAMh4UhIZGYm0tDR8++23qF27NpYtWwYzMzMAgLe3N7p3744tW7agx9v9oVBpIQgChr87GgGBbwMAGnk2xvGjh3HqjxNo/mYreDT6bxi5hQO2RyugUGnx7MZ+SMws8M6YzzA80AdmZlK0aNkCQ97piciNazE2+COxn917vI1hI0cDAJo09cXJY0dw6o/j6PF2X9yOi8WxIwcRPOFT9H1ncMY+zXzx6NHDjBEAnboiYmkIfN9shRmzvhTPWa1aDUz66AOcOXUCLVq1NXhdzKQZBfWqNvaCTCYTh7o+b+zYseJ84PT0dCQmJuLDDz/EsGHDxH0sLS3x4Ycf4vr16+LxCoUC8+bNQ8uWLQEAtWrVQseOHXH06FHUqVMH0dHRqFatGoYMGQKpVApfX1/Y2NiIH9obN24MOzs78c8AMHfuXMhkMqxdu1bc1qFDBwQGBmLBggXYsmWL2Kdu3bqhb9++Bj+LQqHA7NmzxaflsbGx+PbbbzFv3jz069dP/BknTJiA27dvo0GDBggNDYW1tTVWr14tXrNly5bo1KkTVqxYgalTp4rnHzx4MPz9/cW/R0dHIzg4GJ06dQIA+Pr6wsnJyWAo8bFjx8T+PN/Pbdu2oXr16gAy7tXevXtjx44dGDRoEGJjY9G7d2/xaTgA+Pj44M0338SZM2fg7e2da5/u3buHUaNGYdy4cWKbi4sL+vTpg/Pnzxs8UR02bBiCg4MBAG3btkXv3r0RFhZmMJ2ga9eumD59uvi6nDx5MtvT0DfeeMMgWJw4cQINGjTAkiVLxNe0VatWOHnyJM6cOYMxY8bg0qVLUCqVCAoKEp84VqlSBQcPHkR6ejpsbW2xZMkSeHt7Y968eeITz1q1amHkyJE4evToK+exu7m5Ye7cueJ7s3nzZmg0GixatAjm5uZo06YN9u3bhwsXLrz0PC+Tee++bFj+qwiCgHHjxqF///4AgKZNm2L//v04cuQI2rZti19//RVxcXHYunUrGjVqBABo0aKFeN/lRV7uD2N169YNM2bMQEJCgjj1ICoqyujCmampqdi2bRtq1KgBIOOJ9tChQ3H69Gl07do1z/3JqX9r167F559/DnPzjI/Yu3fvxtixYwt87ufJZDK4urrm6RilUompU6dCJpNh6NChue7n4uKC5s2bY9euXeJ98s8//+DSpUtYvHhxjsfo9Xp89tln4hfgRFS0GPCJiIyg1wuIe5CMdMEJjk7OGDt2LPz9/dG2bVu0bt0akydPhkKhQExMDEaNGgVBEMQpS9WrV4erqyuOHD0Bv65vi/PtPRpmPXGSyWRwcnLKVlTvzF+PUL5exnnSn8WiXgNvDAvwAgQ9dFo9bG1s4enlg/Nnzxgc59HI8GlWhUqVoFRmnPvypYsAgLbtDYcCz5m3EADwz93beBL/GIOHvQvdc9OuvBs3ga2tLc6dPYMWrdqK26RSCWxtZLCSvfw/KS/OQR0/fjwaNGiAhIQExMXF4e7duzh8+DAAZCuY9XygqVKlSsbr8d/w6RYtWiAyMhJ9+vRBp06d0L59e/To0eOlBQejo6PRsWNHMRQCGdPOunfvjrCwMIOh4bnNnc38ggfIGMUGwCAQOzk5AcgYTgsAp0+fhq+vL6ysrMR7w87ODs2aNcMff/xhcO4Xr/nmm28iJCQEV69eRdu2bdG+fXuDLwQEQcDx48fx3XffZetjZrgHAA8PD1SvXh1nz57FoEGDxA/jaWlpuH37Nv755x9cvnwZQPb34MU+ZVZrT0lJEd+/zArmLx7bu3dv8c8SiQSdO3dGSEiIwTD6Zs2aGRxTrVo1gykTOfWhTZs2aNOmDTQaDWJjY3H37l3cuHEDCQkJ4uvv7e0NS0tL9OvXD/7+/mjXrh3efPNN8YnvrVu38PjxYwQEBECr1YrvTfPmzWFnZ4eTJ0+iQ4cO2aYgmpmZifeYj4+PQbuzszMaNmwohjwg435ITU1FUXu+r5lfxmX+Wzp9+jSqV68uhnsg4x7t2LGj+N4aKy/3B5BxDz8/jQiAwesHAJ06dcJnn32G/fv3Y8CAAYiPj8e5c+cwa9asHOfmv6hcuXJiuAeyfpe8auqRsQICAvDDDz/g9OnTaNOmDWJiYvD48WN06dIFBw8eNNj3ZfeTqcnlcgQHB+Py5ctYsmQJXFxcAGSMrHp+hIpUKoVUKkXPnj0xe/ZsPHnyBBUrVkRUVBTs7OxynD6i0Wgwbdo07Nu3D7NmzTJqJAURFS4GfCKiV4i5+QRbDt3Eg3g5tDo9XNsGIyH2IHbtikJkZCSsrKzQq1cvBAUFQa/XY/ny5Vi+fHm280jNZQbF9KysrAy2S6RSCIIeekHAr8cyqhxnfviyMJdC0Cpw7eJJdOnYItu5nZwMh2laWRqeWyqRQviv6n7Kf0+2nZxzLr6VuX3Jt99gybffZNv+7OkTPHr4Lwb372HQ/vXXX6NPnz45nhOAwVxQAIiLi8O8efPw119/wdraGm5ubuLw8heHRVtbW2f9LP8tA5i5T0BAAPR6PTZu3CgOM3dxccGkSZNyfbKXnJwshvLnVahQAYIgiPP3c+p3pue/HMipny9KSkrC7t27c5zK9mIhtBevuXjxYixduhR79uzBvn37IJVK0apVK8ydOxcuLi64fPkyVCoVmjZtanDc83NkM5UvX14c3ZCQkIDZs2fjwIEDkEgkqFmzphi0X3wPXuzTP//8g1mzZuHUqVOwsLCAq6sr6tevn+OxL1YlL1++PARBEL/8ALK/dlKp9JV90Ov1+O6777Bhwwakp6ejatWq8PLygqWlpbhPtWrVsH79ekRERGDLli1Yu3YtHBwcMHjwYHz00UdISkoCAKxatQqrVq3K9nrFx8fj/v37eOuttwzan7/fc7oXcrtvitqLv3eef50TExNRvnz5bMfk1PYqebk/AGD79u3iCI5ML4ZiOzs7tGvXDnv37sWAAQOwd+9ecWi8MV9AvHiPZQbqzKkaBVW7dm00aNAAe/fuRZs2bbB79260adMGjo6OBvu96n4ypYcPHyIoKAi3b9/G4sWLDUZjdO7c2aAmR+/evfHNN9/A398fX3zxBfbs2YPhw4cjKioKXbt2zXbvZNYCOHv2LD777DMMGTLE5P0norxjwCcieomYm08QtiUGCqUW9rYWsDCzgMbmDVjYDUTN5oPRxdMCf104ip9//hmVK1eGRCLByJEjxeGnGq0OKeka6PUCLF8I3TnR6wX89OsVnLsWL7Y52MrwQV8vfHLEHk2avol3BmUfXpk5HcAYdnb2AIDkpERUrJQVAP+5exspycmw/W970LiJaNykWfbj7e1RvkJFLFu5Hva2FpCZZ1y7WrVqRvdBLpfjm2++QYMGDRAVFQVXV1dIpVIcPXoU+/btM/o8mQIDAxEYGIjU1FScOHECy5cvx+TJk9G0adMcQ66joyOePn2arf3JkycAAGdnZ8THx2fbXhD29vZo1aoV3n333WzbXnxSmdOxkydPxuTJkxEXF4eDBw8iPDwcc+bMQUREBI4dO4ZWrVrBwsJw5YLExMRs53r69Kn4FHPSpEmIi4vD6tWr4ePjA5lMBoVCgV9++eWl/dHr9RgzZgwsLCywZcsWNGjQAObm5oiNjcXOnTuz7Z+UlGTwhcrTp09hZmYGJyenAr3OERERWL16NebMmYMuXbrA3j7j3s2cJpHJy8sLoaGhUKvVOH/+PCIjI7F06VLUr18fbm5uADKmIAQEBGQLMY6OjqhUqZLBtA0gb/f7izKD5YtPrI0p6leYKleujDt37mRrf7FGBPDyvuf1/gCAjh07ZnuNK1WqlO3+CAgIwOTJk5GQkIDdu3fna6h/YQoICMDKlSsxe/Zs7N27N9tcdQAmv59yc/36dYwaNQoqlQo//fQTmjdvbrD9xx9/NBhNkTmf397eHn5+ftizZw9atGiBmzdv4rPPPjM49tGjR3j33Xdx//59fPfddyYtIkhEBcOAT0SUC71ewJZDN6FQalHe0RISiQQPb53HX8c3ok3/zyDXWOHSQ0vMmTUbUVFRePbsGTw8PBAXFwdPT0+kKdRISdPASanA5zOnoEXLNqhVO/d5k4Ig4K/bCbCzyPxAK4G9jQxThzdDeUdreDduirt34uDmVg9m/wVCQRAwb87/UK16DbjVzbma9os8vRoDAP44eQy9evcX2yN+DMHTJ/EIi1gDZ+dyePTwX7jX9xC3P3v6BF9/MQs93u6LGtVroKWvDyxzGJaf+YT9Ze7cuQO5XI5BgwaJAQuAWNE6L0/UPvroI2g0GoSFhcHe3h7dunWDhYUFgoODER8fn2PAb968OQ4fPgy5XC4+fdXpdIiKioKnp2ehrCPv6+uL2NhYMewAGe/fpEmTULNmzVynAjx48ACDBw/G9OnT4e/vD1dXV7i6uuLixYu4ezdjycRjx46J82Wfd/78eSQmJoof3K9cuYL79++L8+HPnz+PAQMG4M033xSPMeY9SExMxO3btzFjxgyDZQdzO/bAgQMYOHCg+DP//vvvaNq0aYFf5/Pnz8PNzc2gRsLjx49x48YNsV+rV6/GmjVrsG/fPshkMrRs2RKNGjXCnj178O+//6JLly4oV64cnjx5goYNG4pP3uPj4zFlyhQMHDgQNWrUMOnyipn33OPHj8U2jUaDS5cuvfTfjzH/tgrC19cXW7duxbVr18T7UalU4vjx4wbvlZ2dnUHfARjUF8jr/QFkhMvcCsY9r2PHjpDJZFi/fj0uXryIhQsX5u2HLGTdunXDt99+i6VLlyI5OTnbk3ogY2pEYS/X+fDhQ7z77rswMzPDzz//bPB7NlNuKzAAQK9evTBu3Dj8/PPPeOONN+Dr6ytuk8vlGDFiBJ4+fYpVq1Zlm15DREWLAZ+IKBdxD5LxIF4Oe1sL8YmbcxU3CIIeF/YtRQ3PzvjriQwfT9qJ1NRUdOnSBX5+fhgzZgwmTPwYHd7qCp1Oj182rcO1q1cwbMToXK/16FkakuVqWFqokTnY18LSBs5mT3H/9jWU8/bBsJGj8eHYkZgx5SP07N0PMpkMv+3chpPHj2D2lwuM/rnq1K2H9h07YVn4EqiUSrjVdceZ0ydx6uQxzJm3EGZmZnhvzDgsXvgVpGZStGzdDvLUVKxfswJP4uPRoIEHnO0tcwz3QEa166tXryI6OjrX+Zg1a9aEtbU1Vq5cCVtbW5ibm2Pfvn3iU628zIlt0aIFZs+ejfnz56Ndu3ZISUlBaGgoatWqJQ4JftH48eNx7NgxDB8+XHzSuH79ety7dy/XZacKaty4cRg4cCCCgoIwaNAgWFpaIjIyEgcOHMAPP/yQ63EuLi6oUqUKvvzyS8jlctSoUQNXrlzB0aNHERQUhISEBFy5ciXbsmVAxuv4/vvv44MPPkBaWhoWL16MevXqITAwEEDGk+3ffvsNDRs2RJUqVXDhwgVERERAIpG89D0oX748XFxcsGHDBlSpUgUODg44fvy4WLn+xWMXLFgAlUqF2rVrY/Pmzbh16xbWrFmTn5fRgJeXF8LDwxEREYHGjRvj7t27WLZsGdRqtdiHFi1aYNGiRQgODsbQoUNhZmaGTZs2QSaToWPHjjAzM0NwcDC+/PJLzJ8/H126dEFKSgrCw8Px+PFjNGzYsMD9fJGjoyN8fHywbt061KxZE46Ojli7di2USuVLh/Y7ODjgwoULOHv2bKGEqsDAQERERCA4OBgTJ06Eg4MDVq1ahWfPnhmsztChQwdERUXB29sbNWvWxLZt28Qvm4C83x95YWNjg/bt2yMiIgJeXl4GNSYKy+rVq7O1OTg45Dikvnr16vD09MSyZcvQuXPnfE3VuH//PjZs2JBtRE6TJk2MnuP+5Zdf4tmzZ5gzZw7kcrnBUq52dnY5Bv7ntW3bFk5OToiMjMT7779vUB/ghx9+wJ07d/Dhhx/C3Nzc4NwymQweHh45nJGIXhcGfCKiXKSkqaHV6WFhlvUhy8rWEc27T8SN6J24dmIDtFo1FLXrICQkBC1atIBeL2DxDz9iRcSP+HzmFJhbWKCeewMsWhyeVRn/BdfvJmLZ9kvQ6bPmpbb2fgMda4zGhrUrMX3Sh1i1fgvquNXF92Er8FNEOL7+YhYEQUBt1zqY+/W3aN2mfY7nzs2MWV9izU/LsHXzz0hOSkKNWrUw+8sFaNOuIwCge4/esLWxxaaNa7Hr1+2wtrZGI8/GmPn5PHi4u+Ya7gHgvffew1dffYVRo0blOKcZyBgC+umnn2Lbtm2YOHEibG1t0aBBA6xfvx6jR4/GuXPncl0P/EUDBw6ERqPBpk2bsHHjRlhZWaFly5aYPHlytg/ImerWrYuNGzfiu+++w/Tp0yGRSODl5YW1a9cW2tOo+vXrY8OGDVi8eDGmTJkCQRBQr149hIWF5fiU73mhoaH47rvvsGTJEiQmJqJq1aoYP348xowZg127dqFu3bo5jlRo1qwZWrRoIVbJ9/Pzw5QpU8Snsd988w2++OILcUnBWrVqYc6cOfj1119x7ty5l/YpPDwc8+bNw7Rp0yCTyeDm5oYff/wRX331Fc6dO2ewOsLnn3+OZcuW4d69e/Dw8MBPP/1kktc5KCgIiYmJWLt2LcLCwlC1alX06tULEokEy5YtQ0pKCurXr4+lS5ciLCwMn3zyCXQ6HRo1aoSffvpJrETep08fJCcnY//+/di2bRtsbGzQpEkTLFq0qNACZOZrP3PmTNjZ2aFfv35o2rQpNm/enOsxY8eORXh4eKEtS2xubo6VK1di3rx5YiX4nj17wsnJCbdv3xb3mz59OrRaLebPnw9zc3MEBATg008/xcyZM8V98nJ/5FVAQAD27t1rdPX8gvr666+ztdWoUSPXOfMBAQG4fPlyvqcPxMXFYdGiRdnaJ06caFTAV6vVOHLkCABg9uzZ2bb7+vq+csnGzKKj69atQ8+ePQ22ZS6hFxISku2LRRcXFxw6dOiVfSSiwiMRjFngtRTJrA5c2EOjMqWnp4tD3YprwR0qO3g/5k3svSR8tToaVpZmsLTIPsddpdZBqdZhxkhfuFV3glanR1KqCiqNLoez5ez05YdYt+eaQbjv3cENXd6sUWgVlfPLTCp56ZP7vOC9WPgyA1RB1l4vqG3btmH69Ok4ePBgocwxNhXejxlu3ryJuLg4dOnSxeD3T79+/VClShWEhoYWYe/KBt6Lr9/rzgZEhY1P8ImIcuHq4giXSna48zAFMgepwQdeQRCQqtCgVlUHuLo4QqXRITlVBY3OuLnjgiBg14nbiDqZ9VTMwlyKkYEeaFo/+5PYombKcE9ExVN6ejomTpyIwYMHo3PnztDpdNi9ezeuXLmSY7E4IiIqfvhJjYgoF1KpBP386iJsSwyepahgb20BC3MpNFo9UhUa2FiaY0CnelCqtUhJUxs8hX8ZjVaPdXuuIfqvR2KbvY0FPujrDVcXx5ccWTQY7onKBm9vb3z//fdYuXIlduzYAUEQ4OHhgRUrVqBFi+zLcxIRUfHDT2tERC/hXbcigvt5Y8uhm3gQL4dcoYG5mRS1qjpgYOd6qFHZAclyFYzM9pCnq7F02yXE3k8W26qUt8H4/o1RwSn3NdSLipmZBM72VjlOUaDirSiH5mfq06dPoaztTYXH398f/v7+Rd0NIiLKJwZ8IqJX8K5bEZ51KiDuQTJS0tRwsJWhVlUHpCrUSFWoYWwlk8cJ6QjdfBFPErMqSLvXcMaYPp6wtcq5GFxRYrgnIiIiKlkY8ImIjCCVSuBW3QkAoNXpkZyqgjIPxfRu3kvE0q2XkKbUim2tPKtisH99mJsV7trW+cFwT0RERFTyMOATEeVBXovpAcCZvx5h3e6r0OqyHvX3alcH/i1rFrtK+QBg/l+4lzHcExEREZUoDPhEREZSqDRIlhtfTE8QBOw+eRu/nciqlG9ullEpv1mD4lcpH2C4JyIiIirJGPCJiIyQmqaGXKE2upieRqvHhr3XcPpKVqV8O2sLfNDXC3WqORVOJwuI4Z6IiIioZGPAJyJ6Cb1eQHKaCgqV1uhiemkKDZZuu4Sb95LEtsrlbDC+vzcqOtsUTkcLyNxMCmd7S4Z7IiIiohKMAZ+IKBc6nR5JeSym9yQxHaGbY/A4IV1sq1vdCWP7eMHWuvhVygcACzMpnB0sYWHOcE9ERERUkjHgExHlQK3RIVmuglprfDG92PtJWLr1EuQKjdjWolFVDO1WPCvlAwz3RERERKUJAz4R0QvyWkwPAM5efYQ1Udegfa66fs+2rujWqlaxrJQPMNwTERERlTYM+EREz5Gnq5GaroHeyAn3giBgz6k7+PVYnNhmbibB8AAP+DasUljdLLCMcG8FC/PiObKAiIiIiPKuWHyye/z4Mdzd3bP9b9u2bQCAa9euYejQoWjcuDH8/Pywdu3aIu4xEZU2giAgSa5CSrra6HCv1emxbvc1g3Bva22BjwY2KdbhXmbOcE9ERERUGhWLJ/h///03LC0tceDAAYOhrPb29khMTMS7774LPz8/zJkzBxcvXsScOXNga2uLvn37FmGviai0yCymp9LoYOyg/DSlBhHbLuP6P4liWyVnawT3b4zK5YpnpXwgI9w72TPcExEREZVGxSLg37hxA7Vq1UKlSpWybVuzZg0sLCwwd+5cmJubo06dOrh79y4iIiIY8ImowFQaHZJTVdDojC+m9yRJgbDNF/HoWValfLdqThjb1wt2xbRSPsBwT0RERFTaFYtPedevX0edOnVy3Hbu3Dn4+vrC3Dzru4gWLVrgzp07ePr06evqIhGVQulKDRJTlHkK93EPkrFg7VmDcO/bsAomDvQp9uGew/KJiIiISrdi8wTf2dkZQ4YMwe3bt1GzZk188MEHaNeuHR49eoR69eoZ7J/5pP/hw4eoUKFCnq8nCALS09NfvaMJKBQKg/8nKkq8H7PIFVrIFWro81Ap/88bT7Hx95vQ6rKO6fpmNXR9szq0GhW0mpccXIQsLcxgZW0JtUoJdVF35j+8F6k44f1IxQXvxddPEIRiu9oNUX4UecDXarWIi4uDm5sbpk2bBjs7O0RFRWHMmDFYtWoVlEolZDKZwTGWlpYAAJVKla9rajQaXLt2rcB9z4s7d+681usRvUxZvh/NzS2QrtYjTaGGzsgn94Ig4PytdJz+Wy62SSWAn7cD6lbUIC4u7iVHFx2JRAIba0vYWUkR/2/x/PahLN+LVPzwfqTigvfi6/Vi1iAqyYo84Jubm+PMmTMwMzODlZUVAKBRo0a4efMmVq5cCSsrK6jVhs+cMoO9jU3+CllZWFjAzc2tYB03kkKhwJ07d1CrVi1YW1u/lmsS5aas3486nYDkNDVs1VpUNPoYPTYfisOZ58K9jZU53gt0Rx0Xx8LpqIlYyczhaCeDmbT4PZko6/ciFS+8H6m44L34+sXGxhZ1F4hMqsgDPgDY2tpma6tbty5OnDiBKlWqID4+3mBb5t8rV66cr+tJJJJ8fzmQX9bW1q/9mkS5KYv3o0qjQ6pSBYmZBayNnCufrtRgxW+X8ffdrEr5FZ2tMb6YV8oHMubcl3OwgplZ8Z5zXxbvRSq+eD9SccF78fXh8HwqbYr8k9/NmzfRpEkTnDlzxqD9ypUrcHNzQ/PmzXH+/HnodDpx2+nTp1G7dm2UL1/+dXeXiEoghSqjmJ42D8X0niYpsGDdOYNwX6eaI6YOa8ZwT0RERETFUpF/+qtTpw5cXV0xd+5cnDt3Drdu3cLXX3+Nixcv4oMPPkDfvn0hl8vxv//9D7Gxsdi2bRtWr16NoKCgou46EZUAqWlqJKWqoMtDMb3b/yZj/guV8pt7VMZHA31gZ1O85+lZWpgx3BMRERGVUUU+RF8qlWLp0qX49ttv8dFHHyElJQUeHh5YtWqVWD1/xYoVmDdvHnr37o2KFStiypQp6N27dxH3nIiKM0HImG+frtRAMD7b48Lf8Vi16y9otFlP+wNa1UKPtq7FfhifpYUZnO0tGe6JiIiIyqgiD/gAUKFCBXz99de5bvfy8kJkZORr7BERlWQ6vYCkVBVUai2MzfaCIGB/9D/Ydjir2I6ZVIKh3RqgpWfVwumoCTHcExEREVGxCPhERKai0eqRlKqEWmv8fHudTo9N+6/j+MV/xTYbS3ME9fGEe81yhdFNk7K0MIOzg1WxrJZPRERERK8PAz4RlRoqtRaJchV0OuPH5CuUWkTsuIxrdxLEtgqOVhj/TmNUKZ99hY/ixsrCDE4M90REREQEBnwiKiXSFGqkpGmgz8OE+2fJCoRtjsG/T9PENlcXR3zQ1wv2xbyYHsBwT0RERESGGPCJqEQTBAEpaWqk5bGY3p2HKQjfEoOUNLXY1rR+JYzo7gGZhVkh9NS0GO6JiIiI6EUM+ERUYuWnmB4A/Hk9Hj/9Zlgp379lTfRsVwfSYl4pH2C4JyIiIqKcMeATUYmUn2J6giDgwH+V8jO/EJBKJRjStT5ae79ROB01MSuZGZzsGe6JiIiIKDsGfCIqcVRqLZLkKmjzUExPp9cjcv8NHPvzgdhmbWmOoN6eqF+r+FfKBzLCvbO9FaQM90RERESUAwZ8IipR5Ao1UvNYTE+h0mLFziv4K+6Z2Fbe0QrB/b3xRgW7wuimyTHcExEREdGrMOATUYkgCAKS09RIz2MxvYQUJcI2x+DBE7nYVvsNB3zQ1wsOtpaF0FPTY7gnIiIiImMw4BNRsafT6ZEkV0Gl1uWpmN7dRxmV8pPlWZXym7hXwsjAklEpHwCsZOZwtrdkuCciIiKiV2LAJ6JiTa3RIVmuylMxPQCIufkEK3+9ArUm67iuLWqiV/uSUSkfYLgnIiIiorxhwCeiYkuh0iBZroZOb/xze0EQcOjcPWw5eDOrUr5EgsH+7mjj7VI4HS0EDPdERERElFcM+ERULKWmqSFXqJGHbA+dXo9fDtzE0Qv3xTYrSzOMedsTHrXLF0IvTU8CwMoyI9xLSshIAyIiIiIqHhjwiahY0esFJKepoFBp81RMT6nSYsWvV3DlVlal/HIOVhjf3xtvVCwZlfIZ7omIiIioIBjwiajY0Or0SE5VQanR5em4xBQlwrbE4H58VqX8mlUdMK6vFxztSkalfIZ7IiIiIiooBnwiKhZUGh2SU1XQ6PJWTO+fR6kI2xKDZLlKbPOpVxHv9mhYYirlSwBYW5nDyY7hnoiIiIjyjwGfiIpcforpAcCl2KdYufMKVM898e/sWwO9O7qVmEr5DPdEREREZCoM+ERUpPJTTA8ADp27h80Hb4jz9KUSCQZ2qYd2PtVM38lCwnBPRERERKbEgE9ERSK/xfT0egGbD97A4fPPVcqXmWH0255o6FoyKuUDDPdEREREZHoM+ET02uW3mJ5SrcXKnX/h8q2nYpuzgyXG92sMl0olo1I+wHBPRERERIWDAZ+IXiu1RoekfBTTS0xVInzLJdx7nCq21ahij+B+3iWmUj6QEe5trC3gaCtjuCciIiIik2LAJ6LXJr/F9O4/TkXolhgkpWZVyveuWwHv9WgES1nJqJQPABIJYGPFcE9EREREhYMBn4hei/wW07t86ylW7LwClTprOH+n5jXQp6MbpNKSE5Izw71TCRptQEREREQlCwM+ERUqQRCQJM97MT0AOHLhPiL3XxePk0iAAZ3d0aFJyamUD2T029bKokRNJSAiIiKikocBn4gKjU6nR5JcBZVah7xke71ewNbDN3Hw7D2xzVJmhvd7NYJnnQqm72ghYrgnIiIioteFAZ+ICoVao0OyXAW1Nm/F9FRqHX767QpibmZVyneyt8T4ft6oVtne1N0sVAz3RERERPQ6MeATkcnlt5heslyFsC0x+OdRVqX86pXtMa6fF5ztrUzdzULFcE9ERERErxsDPhGZlFyhRmqaBvo8Trh/EC9H6JaLSEzJqpTvWacCRvVqCCtZyfpVxXBPREREREWhZH1qJqJiSxAEpKSpkabU5LmY3l9xz7B8x2Uon6uU37FpNfR/q16JqpQPZIR7OysZHOxkRd0VIiIiIipjGPCJqMB0egFJqco8F9MDgGN/3sem32+IT/wlEqD/W/Xg16y66TtayBjuiYiIiKgoMeATUYFotDokpea9mJ5eELDtcCwORP8jtllamGFUr0bwcitZlfKBjHBvby2DvS3DPREREREVDQZ8Isq3/BbTU2t0+Om3v3DxxhOxzdHOEsH9vFGjSsmqlA8w3BMRERFR8cCAT0T5Ik9XIzU978X0kuUqhG+9hLsPU8S2apXsENzPG84OJatSPsBwT0RERETFBwM+EeWJXi8gOU0FhUqb52J6D57IEbY5BgkpSrGtUZ3yeL9nI1hZlrxfRwz3RERERFSclLxP1ERUZLQ6PZJTVVBqdK/e+QVXbz9DxI7LUKqyjm3fpBre6VQXZlKpKbv5WkgkgIONDHY2DPdEREREVDww4BORUVQaHZJTVdDo8lZMDwCOX3yAn/ddz6qUD6DfW3Xh16w6JJKStQweAEglgD3DPREREREVMwz4RPRKaYqM+fZ5LaanFwTsOHILv5+5K7bJLKQY1bMRvOtWNHU3XwuGeyIiIiIqrhjwiShXer2AlHQ10pWaPM+3V2t0WLXrL/x5/flK+TKM6+eNmlUcTNzT10MqAextZbCzZrgnIiIiouKHAZ+IcqTR6pEsV0GVj/n2KWkq/Lj1Em7/m1Up36WiHYL7e6NcCayUDzDcExEREVHxx4BPRNko1VokyVXQ6fL42B7Av08zKuU/S86qlN/QtTze79UI1iWwUj6QEe4dbGWwZbgnIiIiomKsZH7aJqJCk5qmhlyhRh6n2wMArt1JQMT2y1CotGJbOx8XDOhcr0RWygcY7omIiIio5GDAJyIAgE4vIFmuglKd9/XtAeBkzL/YsO9v6PVZlfL7+tXFW81LZqV8gOGeiIiIiEoWBnwigkarR1KqEmpt3pfA0wsCdh69hX2nsyrlW5hL8V6PhvBxr2TKbr5WUokEDnYy2FpZFHVXiIiIiIiMwoBPVMap/ptvr83HfHu1RofVUVdx4e94sc3BNqNSfq2qJbNSPsBwT0REREQlEwM+URmWplAjJU0DfT7G5KekqfHj1hiDSvlvVLBFcH9vlHe0NmU3XyupRAJHOxlsGO6JiIiIqIRhwCcqo5LlKqTlY317AHj4NA1hmy/i6XOV8j1ql8PoXp6wtiq5v1YY7omIiIioJCu5n8SJKF/MzS2QJFcDEnPkI9vj7zsJWPZCpfy2jd/AwM7uMDMrmZXyAYZ7IiIiIir5GPCJyhCdTkCKUg9bpQbW1nn/5//HpX+xfm9WpXwA6NPRDZ19a5TYSvlARrh3spfB2pLhnoiIiIhKLgZ8ojJCqdYiIVWF9HTlq3d+gV4Q8NvxOOz5447YZmEuxbuBDdGkfsmtlA8w3BMRERFR6cGAT1QGyNPVSE3XQKPV5flYjVaHNVHXcO7aY7HN3sYC4/p5o/Ybjqbs5mvHcE9EREREpQkDPlEpJggCktPUSM9nMb3UdDV+3HoJcQ+SxbaqFWwR3M8bFZxKbqV8ADCTZsy5Z7gnIiIiotKCAZ+olNLp9EhKVUGl0eWrmN6jZ2kI2xyDJ0kKsa1+rXIY83ajEl+IjuGeiIiIiEojBnyiUkil0SE5VQWNTp+v42/8k4il2y4hXZlVKb+11xsY3LVkV8oHMsK9k70lrGT89UdEREREpQs/4RKVMulKDVLS1NDp8/PcHjh95SHW7b5mcHzvDnXQ5c2aJbpSPsBwT0RERESlGz/lEpUSgiAgNU2DNKUa+cn2giBg14nbiDp5W2wzN5Pi3R4eaFq/sgl7WjTMpBI421vCkuGeiIiIiEopftIlKgV0eiFjvr1am6/59hqtHuv2XEP0X4/ENnsbC3zQ1xuuLiW7Uj7AcE9EREREZQM/7RKVcBqtDkmpKqi1+ZtvL1dosHTrJcTeTxLbqpS3QXD/xqhYwivlAwz3RERERFR28BMvUQmmUGmQLM//fPvHCekI23wR8YlZlfLdazhjTB9P2JbwSvkAwz0RERERlS381EtUQqWmqSFX5G++PQDcvJeIpVsvIe25SvktPatiiH99mJfwSvkAwz0RERERlT385EtUwuj1ApLTVFCotBDyGe7P//0Emw7EQqvLOkGvdnXg37LkV8oHGO6JiIiIqGzip1+iEkSr02cU09Po8nW8IAg4e0OOMzcei23mZlKM6N4AzT2qmKqbRcrMTAJneytYWpgVdVeIiIiIiF4rBnyiEkKl1iJJroZWl79iehqtHhv3x+LcjTSxzc7aAmP7esGtmpOJelm0GO6JiIiIqCxjwCcqAdIUaqSkaaDP55j8NIUGS7ddws17SWJb5XI2GN/fGxWdbUzUy6LFcE9EREREZR0DPlExJggCUtLUSFNq8j3f/kliOkI3x+BxQrrYVsfFAeP6NYatdcmvlA9khPty9laQMdwTERERURnGgE9UTOl0eiTJVVCpdchntkfs/SQs3XoJcoVGbHOvZoX33/YoNeHe/L8n9wz3RERERFTWMeATFUMarQ5JqSqotfmbbw8AZ68+wpqoawZz9ru1qA7X8qpSsQwewHBPRERERPQ8BnyiYkah0iBZroYunwvcC4KAPafu4NdjcWKbuZkEwwM84OnqiFu3bpmqq0XK3EwKZ3tLhnsiIiIiov8w4BMVI/J0NVLT819MT6vTY8Pev3Hq8kOxzdbKHB/09YZbdScoFApTdbVIWZhJ4exgCQtzhnsiIiIiokwM+ETFgCAISJKroFBp811ML02pQcT2y7h+N1Fsq+hsjfH9G6NyudJRKR9guCciIiIiyg0DPlER0+n0SEpVQanR5fscT5IUCNt8EY+eZVXKd6vmiLF9vWFXSorpAQz3REREREQvw4BPVIRUGh2SU1XQ6PJfTC/uQTJ+3BqD1PSsSvnNPSpjeIAHLMxLRzE9IDPcW5Wqn4mIiIiIyJQY8ImKSJpSg9S0/BfTA4Dzfz/G6l1XoXmu2n731rUR2KY2JBKJKbpZLDDcExERERG9GgM+0WsmCAJS0tRIU2ryPd9eEATsO30XO45mVcQ3k0owLKABWjSqaqKeFg8ycymc7BnuiYiIiIhehQGf6DXS6fRIkqugUuuQ3+f2Op0eG/ddx8lL/4ptNlbmGNvHC/VqOJumo8UEwz0RERERkfEY8IleE5VGhxS5Cmpt/ufbp/9XKf/v5yvlO1kjuL83qpS3NUU3iw2GeyIiIiKivGHAJ3oNTDHf/mmSAmFbYvDwaZrYVqeaIz7o4wU7G5kpullsyMwz5tybmzHcExEREREZiwGfqBCZYr49ANz+NxnhW3KqlN+g1C0Zx3BPRERERJQ/DPhEhcQU8+0B4MLf8Vi16y+DSvkBrWqhR1vXUlUpH2C4JyIiIiIqCAZ8okKg1uiQXMD59oIgYH/0P9h2OFZsM5NKMLRbA7T0LF2V8oGMcF/OwQpmDPdERERERPnCgE9kYgqVBsnygs231+n02LT/Oo5ffK5SvqU5gvp4wr1mOVN0s1hhuCciIiIiKjgGfCITEQQBqWkapCnVKEC2h0KpRcSOy7h2J0Fsq+BohfHvNC51lfIBwNLCDM72lgz3REREREQFxIBPZAI6vYBkuQpKtbZAxfSeJSsQtjkG/z5XKd/VxREf9PWCfSmrlA8w3BMRERERmRIDPlEBabQ6JKUWbL49ANx5mILwLTFISVOLbU3rV8KI7h6QWZSuSvkAwz0RERERkakx4BMVgFKtRZJcBZ2uIHXygT+vx+On3wwr5fu3rIme7epAWsoq5QP/hXsHK5hJS9/PRkRERERUVBjwifJJrlAjNU0DfQHG5AuCgINn72HroZviUnpSqQRDutZHa+83TNPRYsbKwgxODPdERERERCbHgE+UR4IgIDlNjXSlpkDz7XV6PSL338CxPx+IbdaW5gjq7Yn6tUpfpXyA4Z6IiIiIqDAx4BPlgU6nR5JcBZVah4IMyleqtFi+8wr+insmtpV3tEJwf2+8UcGu4B0thhjuiYiIiIgKFwM+kZHUGh2S5QUvppeQokTY5hg8eCIX22q/4YAP+nrBwdayoN0slhjuiYiIiIgKHwM+kRHSlRqkpKmhK8gC9wDuPsqolJ8sz6qU7+NeEe8GNiyVlfIBwEpmBid7hnsiIiIiosLGgE/0EoIgIOW/+fYFzPaIufkEK3+9ArUmawRA1xY10at96ayUD2SEe2d7K0gZ7omIiIiICh0DPlEuTDXfXhAEHDp3D1sOPlcpXyLBoK7uaNvYxRRdLZYY7omIiIiIXi8GfKIcqDQ6pJhgvr1Or8fmAzdx5MJ9sc3K0gxj3vaER+3yBe1mscVwT0RERET0+jHgE70gTalBqgnm2ytVWqz89Qou38qqlF/OIaNSvkvF0lkpHwCsZOZwtrdkuCciIiIies0Y8In+kznfPq2A69sDQGKKEmFbYnA/PqtSfs0q9hjXzxuOdqWzUj7AcE9EREREVJQY8Ilguvn2APDPo1SEbYlBslwltjWuVxHv9Si9lfIBhnsiIiIioqLGgE9lnqnWtweAS7FPsXLnFag0OrGts28N9O7oVmor5UsAWFlmhHtJKf0ZiYiIiIhKAgZ8KtMUKg2S5QWfbw8Ah8/dwy8Hb4jD+6USCQZ2qYd2PtUKfO7iiuGeiIiIiKj4YMCnMis1TQ25Ql3g9e31egGbD97A4fPPVcqXmWH0255o6Fp6K+Uz3BMRERERFS8M+FTm6PUCktNUUKi0BS6mp1Rr8dOvf+FS7FOxzdnBEuP7NYZLpdJbKV8CwNrKHE52DPdERERERMUFAz6VKRqtHslylcEc+fxKSlUhbEsM7j1OFdtqVLFHcCmvlM9wT0RERERUPDHgU5mhUmuRJFdBqyv4fPv7jzMq5SemZlXK93KrgFE9G8FSVnor5TPcExEREREVXwz4VCbIFWqkpmmgL+iYfABXbj3F8p1XoFJnjQJ4q3l19O1Yt1QvEcdwT0RERERUvDHgU6kmCAKS09RIV2oKPN8eAI5euI9N+6+L55JIgAGd6qFD0+oFP3kxxnBPRERERFT8MeBTqaXT6ZGUmjHfvqDZXq8XsO1wLA6c/Udss5SZ4f1ejeBZp0IBz168SQDYWFvA0VbGcE9EREREVIwx4FOppNLokJyqgkanL/i51Dr89NsVxNzMqpTvZG+J8f28Ua2yfYHPX5xJJICNFcM9EREREVFJwIBPpU6aUoPUNDV0BV3gHkCyPKNS/j+PsirlV69sj3H9vOBsb1Xg8xdnmeHeqRSvCEBEREREVJow4FOpIQgCUtLUSDPRfPsH8XKEbrmIxJSsSvmedSpgVK+GsJKV7n86Eglga2VRqpf7IyIiIiIqbUp3SqEyQ6fTI0muhkqtLfB8ewD4K+4Zlu+4DOVzlfI7Nq2G/m/VK9WV8gGGeyIiIiKikooBn0o8tUaHZLkKam3B59sDwLE/72PT7zfEJfUkEqD/W/Xg16x0V8oHGO6JiIiIiEoyBnwq0RQqDZLlpplvrxf+q5Qf/VylfAszjOrVCF5upbtSPsBwT0RERERU0jHgU4mVmqaGXKGGCbI91BodVv32F/688URsc7SzRHA/b9SoUror5QMM90REREREpQEDPpU4Or2AZLkKSrXWJMX0kuUqhG+9hLsPU8S2apXsENzPG84OpbtSPpAR7u2sZHCwkxV1V4iIiIiIqAAY8KlE0Wh1SEo13Xz7f5/IEbo5BgkpSrGtUZ3yeL9nI1hZlv5/Hgz3RERERESlR+lPMFRqKFQaJKepodOZok4+cO1OApZtvwSlKqtSfvsm1fBOp7owk0pNco3iTCIB7K1lsLdluCciIiIiKg0Y8KlEMOV8ewA4EfMAG/ddh/6/E0oA9HurLvyaVYdEUrqXwQMAqUTCcE9EREREVMow4FOxptcLSE5TQaEyzXx7vSBg59Fb2Hf6rtgms5DivR6N0LhexYJfoAQwNzeDvS3DPRERERFRacOAT8WWRqtHslwFlUb36p2NoNbosDrqKi78HS+2OdrJMK6fN2pWcTDJNYo7qUQCBxtL2Frxnz4RERERUWnDT/lULKnUWiTKVSabb5+SpsaPW2Nw+9+sSvkuFe0Q3N8b5cpApXzgvzn3tjKkmJmmQCERERERERUvDPhU7MgVaqSmaaA3xZh8AP8+lSNscwyeJWdVym/oWh7v92oE6zJQKR/ICPcONjJIoYVOZ5oREUREREREVLyUjXRDJYKp59sDwN93ErBs+2UoVFqxrZ2PCwZ0rlcmKuUDgFQC2NvIYGcjQ3q69tUHEBERERFRicSAT8WCVqdHcqoKShPNtweAkzH/YsO+vw0q5ff1q4u3mpeNSvnAf+HeVgY7axbUIyIiIiIq7Rjwqcip1Foky9XQ6EwzN1wvCPj12C3sPZVVKd/CXIpRPctOpXyA4Z6IiIiIqKxhwKcilaZQIyVdIz5lLyi1Roc1UVdx/rlK+Q62MgT380bNqmWjUj6QEe4dbGWwZbgnIiIiIiozGPCpSAiCgJQ0NdKUGpPNt09NV+PHrZcQ9yBZbHujoi2C+3mjvKO1aS5SAjDcExERERGVTQz49NrpdHokyVVQqXUwUbbHo2dpCN0cg6dJCrHNo3Y5jO7lCesytOY7wz0RERERUdlVdpIPFQsqjQ7JqSqTzbcHgOt3E7Bs22WkP1cpv23jNzCwszvMzMpGpXzgv3BvZwlbK4ui7goRERERERUBBnx6bdKVGqSkqaEz0Xx7APjj0r9Yv/dvgzn8fTq6obNvjTJTKR8ApBIJHOxkDPdERERERGUYAz4Vusz59ulKDUyV7QVBwG/H47D7jztim4W5FO8GNkST+pVMc5ESQiqRwNFOBhuGeyIiIiKiMo0BnwqVTi8gKVUFlVprsvn2Gq0Oa3dfw9mrj8U2exsLjOvnjdpvOJroKiUDwz0REREREWViwKdCo9bokCxXQa013Xx7eboaP267hFv3syrlV62QUSm/glPZqZQPMNwTEREREZEhBnwqFAqVBsly0863f5yQjtBfLuLJc5Xy69d0xpjenmUu5EolEjjZy2BtWbZ+biIiIiIiyh0DPplcapoacoXaZPPtAeDmP4n4cdslpCuzKuW39noDg7uWrUr5AMM9ERERERHljAGfTEanF5AsV0Gp1kIwYbg/c+Uh1u6+ZjAa4O32ddC1Rc0yVSkfYLgnIiIiIqLcMeCTSWi0eiSlKk06314QBOw6cRtRJ2+LbeZmUowM9ECzBpVNdp2SwkyaMeee4Z6IiIiIiHLCgE8FplJrkShXQacz3WN7jVaP9Xuu4cxfj8Q2exsLfNDXG64uZatSPsBwT0REREREr8aATwUiV6iRmqaB3oRj8uUKDZZuvYTY+0liW5XyNgju3xgVy1ilfCAj3DvZW8JKxn+uRERERESUOyYGyhdBEJCSpkaaUmPS+fbxiRmV8uMTsyrlu9dwxpg+nrAtY5XyAYZ7IiIiIiIyHlMD5ZlOp0eSXAWVWgcTZnvE3kvCj9suIU2hEdtaelbFEP/6MC9jlfIBhnsiIiIiIsobJgfKE41Wh6RUlUmL6QFA9F+PsHb3VWifm8ffq10d+Lcse5XygYxw72xvCUuGeyIiIiIiMhLTAxlNodIgWa42WK6uoARBwO4/7uC343Fim7mZFCO6N0Bzjyomu05JwnBPRERERET5wQRBRpGnq5GabtpielqdHuv3/I3TVx6KbbbWFvigrxfcqjmZ7DolCcM9ERERERHlF1MEvZQgCEiSq6BQaU1aTC9NocGy7Zdw458ksa1yORuM7++Nis42prtQCcJwT0REREREBcEkQbnS6fRISlVBpTFtMb0niekI3RyDxwnpYlvd6k4Y28cLttZlr1I+wHBPREREREQFxzRBOVJpdEhOVUGjM20xvdj7SVi69RLkz1XKb9GoCoZ2a1AmK+UDgJmZBM52DPdERERERFQwTBSUTWEU0wOAs1cfYU3UNWif+9KgR1tXBLSqVSYr5QP/hXt7K1hamBV1V4iIiIiIqIQrVo9Mb9++DR8fH2zbtk1smzlzJtzd3Q3+5+fnV4S9LN1S09RISlUVQqX821j5619iuDc3k+DdHg3RvXVthnuGeyIiIiIiMoFi8wRfo9Fg0qRJSE9PN2i/fv06xo4di6FDh4ptZmYMRKam0wtIlqugVJu2mJ5Wp8eGvX/j1OXnKuVbmWNsXy/Ure5suguVMAz3RERERERkasUm4IeEhMDOzs6gTRAExMbGYsyYMahYsWIR9az0U2t0SJaroNaadr59mlKDiG2Xcf2fRLGtorM1xvdvjMrlymalfCBj9IKzvRVkDPdERERERGRCxSLgnz17FpGRkdixYwc6dOggtv/zzz9IT0+Hq6tr0XWulEtTapCaZvr59k+SFAjbfBGPnmWNyHCr5oixfb1hV0Yr5QMM90REREREVHiKPOCnpKRgypQpmDlzJqpWrWqw7caNGwCAdevW4dixY5BKpWjXrh0+/vhj2Nvb5/uagiBkmwpQWBQKhcH/Fyep6RqkKTTQm3JMPoA7D1Ox8rdrkCu0YlsT9woY1MkNZtBC8Vx7WWJhYQZ7Kxm0GhW0mlfvXxiK8/1IZQvvRSpOeD9SccF78fUTBKHM1oOi0qnIA/7nn38OHx8f9OjRI9u2GzduQCqVolKlSli6dCn++ecfLFiwADdv3sSaNWsgleavRqBGo8G1a9cK2vU8uXPnzmu9Xm4kEgnMzM2RptQjTaGGXm/aYfk3/1XiwMVkPL+6XvO6tvB1k+Lu3dsmvVZJYmNlCXsbczx7qIFg4i9U8qO43I9EvBepOOH9SMUF78XXSyaTFXUXiEymSAP+jh07cO7cOfz22285bv/ggw8wePBgODtnFGOrV68eKlasiHfeeQeXL1+Gt7d3vq5rYWEBNze3fPc7LxQKBe7cuYNatWrB2tr6tVzzZdRaPZLlathqdSY9ryAIOHjuAfZdeCy2mUklGNCpDpo3qGTSa5U0MgszONtZwsys6L8dLm73I5VdvBepOOH9SMUF78XXLzY2tqi7QGRSRRrwt27dimfPnhnMuweA2bNnY/fu3VixYoUY7jPVrVsXAPDo0aN8B3yJRAIbm9db5M3a2vq1X/NFcoUaCrUG5hYymJtwGrxOp8fGfddx8tK/YpuNlTnG9vFCvRplt1I+AFiYSeHsYAkL8+I157443I9EAO9FKl54P1JxwXvx9eHwfCptijTgL1q0CEql0qCtS5cumDBhAnr27IkpU6YgPj4eq1evFrdfvnwZAF7bE/jSQBAEJKepka7UmHQJPABIV2oQseMK/r6TILZVdLJGcH9vVClva9qLlTDFNdwTEREREVHpVKQBv3Llyjm2ly9fHpUrV0bXrl0xbtw4hIaGomfPnrh9+zbmzp2LwMBA1KlT5zX3tmTS6fRISlVBpdHB1DO/nyYpELYlBg+fpoltdao54oM+XrCzKdtzmTLCvRUszPNXJ4KIiIiIiCivirzI3su89dZb+P777xEREYHly5fD3t4ePXr0wEcffVTUXSsRVBodklNV0OhMW0gPAG7/m4zwLTFITc8qB9+sQWWM6N6gzD+xlplL4WTPcE9ERERERK9XsQv4169fN/h7t27d0K1btyLqTclVWOvbA8CFv+Oxatdf0GizvjgIaFULgW1dIS3j85gY7omIiIiIqKgUu4BPBSMIAlLS1EgrhPn2giBgf/Q/2H44VhzuL5VKMNS/Plp5vWHai5VADPdERERERFSUGPBLEZ1OjyS5Ciq16efb63R6bNp/HccvPlcp39IcY3p7on6tcia+WskjM8+Yc29uxnBPRERERERFgwG/lCjM+fYKpRbLd17G1dtZlfIrOFohuH9jVK1QtivlAwz3RERERERUPDDglwKFOd8+IVmJ0C0X8e+TrEr5td9wwAd9veFgW7Yr5QMM90REREREVHww4Jdg+Z1vLwgCJEYUw7v7MAVhW2KQkqYW25rUr4SR3T0gs3j9lfKN7ffrwnBPRERERETFCQN+CZXf+fbrVq+AhUyGgYOHv3S/P6/H46ffDCvl+7esiZ7t6hRJpfyo37bjnzu38cGHn7z2a+dEZi7F00f/YPS70xAbG4saNWpg9+7dRd2tQnH16lUMGTIEZ86cwaxZsxAdHY1Dhw4V6Jz379/HW2+9ZdBmbm6O8uXLo0OHDvjoo49QrpxhbYdDhw5hzZo1+Ouvv6BSqVClShV06NABY8eORfny5cX9hg0bhujo6Fyv7e3tjV9++aVA/SciIiIiKo4Y8EsgtUaHpHzOt1+14kcMf3dMrtsFQcDBs/ew9dBNg0r5Q7rWR2vvoquUv37NSjT2aVpk13+ezFyKcg5W+GJ2OP7991+EhYVlC6OlydGjR9GiRQvIZKafkvHBBx+gQ4cOAACVSoXbt28jJCQEsbGx2Lhxo7jf9u3bMX36dAwcOBAjR46EtbU1YmNjERERgcOHD2Pr1q1wdHQU9/fw8MDs2bNzvKatLetGEBEREVHpxIBfwihUGiTLC2e+vU6vR+T+Gzj25wOxzcrSDEG9vdCAlfIBAJYWZnC2t4SZmRSJiYmoV68e2rdvX9TdKlTHjh1Dr169CuXcNWrUQOPGjcW/v/n/9u47vqbzgeP4995sSYgS1BaSIIIYsUVD0Zq1Su1RFE1bs2iroUrVaJoIpUVV+6NGVY2fltZqjURL+ZlpjNoziSCy7u+P1K1LrDYybj7v1yuvV+4Zz3nOzeO8fM/znOfUqiU7OzuNGTNGR48elaenpyRp5syZatGihd59913ztrVr11aNGjXUpk0bLV26VP369TOvc3FxsSgXAAAAyA14eDib2L9/v3r27Knq1avLz89PvXr10p49e8zrIyMj1aXLS6pVs7paNmukye+9o5irV83r/7t2lZoE+Ovg//ZpyIBeahZYR53bt9CSrxaatwmsn9YDvnD+HPPvknQsOkqjhgXpucYNNG9SH52O+FyJ1y/rqbyOGtmthm5diVZg/er6NXKXRrwxSM81rqv2rZtqTvjHSklJMZeTlJSkeXPD1bVjazUPrKs+3Ttp/brvLM7z562bNLBvNzULrKP2rZsq7KMPdfPmzQd+N106tNT5c2e1ft1qBdavrnNnz5jPd81336h966Zq89wzOn4sWikpKfrPogXq072TmgfW1fNN6mnIwN767dcIc3kLPvtE3V5sox2/bFW/ni+q2TO11aPzC/r+v2ssjrv866/U86V2ahZYRx3bNlfojMmyMyTJxsYob29v7dq1SxEREfL29taKFSskScePH1dQUJDq1aunqlWrqnv37tq9e7e5zFOnTsnb21vz589X8+bNVaVKFS1fvlyhoaFq3ry5fvjhB7Vs2VK+vr5q06aNfvvtN+3Zs0cdO3ZU5cqV1bJlS23fvt2inkeOHNGAAQNUrVo1VatWTYMHD9aff/5pXr9z5055e3tr8eLFeu6559S3b1/t2LFDV65c0bBhw1SvXj3z8VauXGlRdmxsrPbu3auGDRs+8G90tw0bNuill16Sn5+fKlWqpObNm+vLL798pH1v98TfOd/CpUuXZEpnoony5ctr9OjRqlSp0mPVDwAAALBGBPxsID4+Xv369VP+/PkVGhqqGTNm6ObNm+rbt6+uXbumnTt3qWfPnrK1d9A74ydrcNAw7fltt4YGDdCtWwnmckypqQp+500907ipJn0YIt/KVfVJeIgidv4iSQqbvUCS9HzLNubf/zx5QkMG9NahP07JvXInFa7cUUk3rujMjlka2MZDRd1dzOVPHP+WKlfx08QpH6nxs821+KvPtfa7lX+vDx6rpYsX6flWbfX+lI9U07+2Ppj4rjb+8F9J0sbv1+nt0cNUslRpTXh/mnr26a8f1q/V228OTTe83Tb+/al6qkAB1apTT2GzF+ipAgUlSakpKVq6eJFGvPm2BgUNVanSZTR3dqi+WDBXLdu00wfTQjVs5Fu6Fher4LdGKSHh7xsJVy5f0sczpqhdxy56f0qIihQtqsnvvaOTJ46l1fWH/2rOrI/Vpl0nTZkWpj59B2j9ujV6//2JkqQlS5aoYsWKqlixopYsWaJGjRopKipK7dq106lTp/TWW29p6tSpMhgM6tmz5z3PhIeGhurll1/WlClTVK9ePUnSuXPnNHnyZA0cOFAhISGKi4tTUFCQhg4dqo4dO2rmzJkymUx64403lJCQ9nc/duyYOnfurMuXL+uDDz7QxIkT9eeff6pLly66fPmyxTHDwsI0dOhQ9erVS1WqVNGIESP0xx9/KDg4WHPnzlXFihU1atQo7dixw7zPtm3b5OHhoaJFH/3xjE2bNmnw4MHy8fFReHi4QkNDVaJECY0fP1579+612DY1NVXJyclKTk5WQkKCDh06pPDwcNWuXVvlypUzb9eoUSOtWbNGgwcP1urVq3X+/Hnzul69eql27doW5ZpMJnO5d/88qK0BAAAAORlD9LOBqKgoXb16VT169FC1atUkSR4eHlqyZIli467pw6lTVaJkKU384CPZ2KTNXl/Bx1e9u3fUutWr1LZ9J0lpoaZH75f1fMu2kqRKvlW1dfNP2v7LNtWsVVcVK/lKkgq6Fzb/Pit8ppJMNipava9s7BwlSbVq19IPn4/W2pVLNHDw6+Z6tmjVVt17vSxJqlbdXz9v2aTtv2xVq7btdSw6Sls2bdTgoGFq3+mltG1q+OvcubPa82ukAps005zZofKvVVdj3nnPXGbx4iU1/PVXtHP7NtWu2yDd78fTq7zs7Ozl5pbfXO/buvboa7Hf5UsX1bf/YLXr0Nm8zM7BQe+OHaHoqCjz/gkJCRo+6m1Vq+GfVo8SJdWlQ0vt+GWbSpYqo9/3/KoiTxdV23ad5ORgp/yNG6hQwXyKjY2VJFWtWlUuLi7m3yVp/Pjxsre318KFC83rGjVqpJYtW2rKlClatmyZuU7PPfec2rdvb3EuN2/e1Lhx48y95VFRUZo2bZomTpyoDh06SJJu3LihoKAgHTt2TBUqVFBYWJicnJy0YMEC8zHr1KmjJk2a6NNPP9WoUaPM5b/00kt69tlndfDgQTk5OWnXrl0aPHiwmjRpIkny9/eXm5ubxbP2W7Zseeze+6ioKL3wwgsaO3aseZmfn59q1aqlnTt3qkqVKublY8eOtdhOktzc3PTFF19YLJswYYJSU1P1/fffa8OGDZLShvc3btxYvXv3VuHChS22j4iIkI+PT7r1CwkJUfPmzR/rnAAAAICcgICfDXh6euqpp57SwIED1bx5czVo0ED16tVT0Gtv6NylWP1v/z516tJdMpmUkpwsSSpatJhKlSqt3ZE7zAFfkir6VDb/bm9vLzc3NyXcZwj83qMXFRmxS04FPGS0sZMpNUXP+pdSmwAPndvjp90ROy22r1ipssXngoUKmXvF9/2+R5LUICDQYpvgiR9Kkk6eOKaLF87rpe69zecgSVWqVpOzs7MiI3aqdt0GFuskyWhj88BX45Xz9LL4PHZcWg97zNWr+vPkcZ069ae2/7xFkpSUlGix7Z03C9wLpQXE2+dTtVoNffftcr3Sr5uaNW2qRo0C1KpVqwfWZdeuXXrmmWfMQVtKmxm+RYsWmjlzpq5fv25eXqFChXTLuH2DR5IKFkwbqXBnIHZzc5MkxcXFSZJ27Nghf39/OTo6Kvmv787FxUU1atTQL7/8YlH23cesVauWQkNDdeDAATVo0EABAQEWNwRMJpO2bt2q6dOn3/ec03P7Wfjr16/r2LFjOnnypPbt2ydJSky0/BsMGTLEPMlecnKyzp49q4ULF6pz58764osvzCHd1dVVH3/8sU6dOqXNmzdr586d2rlzp+bPn68lS5Zo3rx58vPzM5fr4+Oj4ODgdOtXsmTJxzofAAAAIKcg4GeR1FSTok/HKu56ovI622vRokWaPXu21q1bpyVLlsjR0VHPNm+hLt16KzU1VYu//FyLv/z8nnLsHRwtPjs6Wn42GI0yme6ebd+kjREntWzjUSUnXte1M3t17Uza0Omja6Xwv7Zyc8tvWfZdxzIajDL9Ndlf3F89227505+M7/b6kGmTFTJt8j3rL1+6qHNnz+iljq0slo8cM07Nn2+dbpmS5OSUx+Lz4UMH9NG0yTp88H9ydHRUqTJlVbhwkbSzvmtotqOj09/nYkx7WiX1r/N5pnFT2RikVd8u06xZ4QoLC1WxYsU0fPhwPf/88+nWJTY21hzK71SwYEGZTCbFx8ebl+XJk+ee7SRZ3Bz4+xyd0tkyTUxMjNauXZvuK/runtn/7mPOmDHD3ObWr18vo9GounXravz48SpWrJj27dunW7duqXr1x3t7wZUrVzRu3Dht2LBBBoNBpUqVUo0aNSTd+zcoVqyYfH3/vtHi5+engIAANWrUSKGhoZo9e7bF9sWLF1fXrl3VtWtXpaamasOGDXrzzTc1YcIE8zwIUtpM+XeWCwAAAOQGBPwssPfoRS378ahOX4hXckqqbG2MKlbIRd36j9SkSZO0fdevWrXqW636ZpkKFnSXwWBQh04vKfDZe4cVO9wVuh/F/j8ua8fFo5IkGzsnuRTyVP+X+8mjWD6L7W4/DvAoXFxcJUmxMVfNveFSWs99XGysnP9aP2DQa6parca9+7u6qkBBd8361HJodpGnH/3Z7+vX4zVq2BB5lPXUvC+WqmSp0jIajdqxfZu2bNr4yOVIkqOdjTp1fEFdXmyna9euadu2bZo7d65GjBih6tWr3zMkXEqbHO7SpUv3LL948aIkKX/+/Lpw4cJj1eNhXF1dVbduXfXu3fuedba2D/7n7erqqhEjRmjEiBGKjo7Wxo0bFR4eruDgYM2ZM0dbtmxR3bp1ZWdn91h1Gj58uKKjo7VgwQL5+fnJ3t5eN2/efOR3zzs7O8vDw0MnTpyQJK1fv17jxo3Tf/7zH5UpU8a8ndFoVNOmTRUREcF77QEAAAAxyV6m23v0omYu26vjZ+Lk6JD2yrXYM3u1YuYQzf9mp3YfvqiynhX1+rDRcnFx1ZUrV+TpVV4nTx6Xd/mK5p/SZTy04LPZ2vtb5CMfO+FWsmQw6PjZOPOyfIXLKa/tNTUNqGku28u7gpYuXqRtW3565LJ9K1eVJP3y13D42+bMClVYyFSVLFVa+fM/pXNnz1icR8GC7po7K1RRRw7Lzs7OYp13+YrKl89N0t897A9y8sRxxcXGqn3HLipdxsO8z64daUPVU+8ZyZA+RzsbjR83WkGvDpGUFoSfe+45DRo0SMnJyfcN6TVr1tRPP/1k0VOfkpKiNWvWyNfX94m8R97f319RUVGqUKGCfH195evrq0qVKmnBggX64Ycf7rvfmTNnFBAQoP/+N20CRA8PD7388suqW7euzpw5I+mfPX8vSbt371bTpk1Vq1Yt8zlv2ZLWLlJTH/43uHbtmo4dO6ZSpUpJSnuEJSYmRp9/fu8IFintzQVeXl7prgMAAAByE3rwM1FqqknLfjyqmwnJKpDPwfw8d/FS3jryi/TLd2G6cu55dWhSSZt/+kHXr8erYaNA1a3fUKNHvKaJwWPVuOlzSk1J1deLv9DBA/vVvefLj3Tsq3EJmrlsr4y2Tkq4ekI3LkerfMXKemX0MI0e2l9jRr6u1i90kL29vb77doV+3rpJ496b8sjnVtbTSwHPNNEn4SG6lZCgcp7e2rnjZ23/eYuCJ34oGxsb9ek/SDM+fF9GG6Pq1Guo+GvXtOjzT3XxwgV5eaf/TPptLi6uOnrksPb+tlvlK6Y/eVqJkqXk7OysRQvnycbGRja2ttry00atW/OtJCnhZkK6+93J1sYgt7yOqlOntsaNG6cPPvhADRs2VFxcnMLCwlS6dGmVL18+3X2HDBmiLVu2qEePHurfv7/s7Oy0aNEi/fnnn/r0008feux/YtCgQercubMGDBigLl26yMHBQUuWLNGGDRv08ccf33e/okWLqkiRInrvvfcUHx+vkiVLav/+/dq8ebMGDBigK1euaP/+/QoNDb1n3/j4eC1YsCDdMps2barKlSvru+++k4+Pj4oUKaJff/1Vc+bMkcFguOeViCdPnrR4HeSlS5f06aefmt8sIaXdfOjfv78++eQTnTlzRq1bt1aRIkV0+fJlffvtt9q+fbvmz59/Tx3vLPduvr6+jzVCBQAAAMgJCPiZKPp0rE5fiJers5053Ds62CpPvkKq1+YN7dmyTLt/mK9fv0+WR9lyeve9KfKrVlOSNGVamD6fP0fBb42UrZ2dvLwraOqM8HtmlU9P/M0kTV4Yqdj4WyrgGajLRzfqbMQ8vf/aMhUvXkwfzfxU8+aEa9KEd2QymVTGo6zGT5qmevUDHuv8xrzznj6f94mWL/2PYmNiVLJ0aY17b4rqN3xGktSi1QtyzuOsxV8t1OpV38jJyUmVfKtqzDvv6emixR5Ydqcu3RX+8TSNHDZEU2eEp7uNi4urJkyark/CQxT89ijlyeOscp7e+ihsrt4cHqR9v/+muvUf3CPt6GArG6NBnTt3VlJSkhYvXqyvvvpKjo6OqlOnjkaMGHHfIeuenp766quvNH36dI0ePVoGg0GVK1fWwoULzc+gZ7Ty5cvryy+/1IwZMzRy5EiZTCZ5eXlp5syZaty48QP3DQsL0/Tp0xUSEqKrV6/q6aef1pAhQ9S/f3+tXr1anp6e6T6KEBsbq0mTJt2zvE6dOmratKkmT56sCRMmaMKECZKk0qVLKzg4WKtWrVJkpOWIk1mzZmnWrFmS0kZpuLq6ysfHR5999pnFdzZ06FBVqFBBS5cuNd+UyJs3r2rUqKFly5bdc9PlwIEDevHFF+977hEREcqbN+8Dvx8AAAAgpzGYctlLoW/P5p1ZE3DduHFDBw8eVIUKFXToZLw+Wvyr8rs6yGhMC/hP5XXQjYRkxV1PVKrJpLjrier5vI98yhbIkOP/HnVJn327X7eSUszLmviXVLtG5cx1gORob6P8ro5W/53c2R7vN9EfkBloi8hOaI/ILmiLmS+zswHwpNGDn4nyOtvL1saopJRUORj/Hh6ckpL2XHJyskk2RqNc8jzepGb381Pkn/p64xHdvoVjNBjUuamXGvoVz5DyrUVuCfcAAAAArBsBPxN5FMunYoVcdPxsnOzzGi3eqW4ymXT9VpKKubuoRGHXf3Wc1FSTlm48op92nzIvc7S30cttfeXjkTEjA6wF4R4AAACAtWAW/UxkNBrUIdBTTg62uhx3S7cSU5SaalJiskkx1xPlZG+r5rVL/6uwmZCYrNkrfrcI9/nzOmh4t+qE+7sQ7gEAAABYEwJ+Jqvi6a7BHaqo9NN5lZCYotjriUpMSlExdxe91Ky8ypd+6h+XHXPtlqZ9+at+j/r7Xewli7jqzR41VbzQvxsVYG0c7W0J9wAAAACsCkP0s0AVT3f5li2o6NOxun4zSbeSkvV0QZd/FTZPnb+mmcv26uq1W3ccp6D6tKokB3teB3antHDvQLgHAAAAYFUI+FnEaDSoXAk3JSWn6FLMTaX+i3cZ7P/jkuZ+u1+3Ev+eKT+wRgl1CPQkxN6FcA8AAADAWhHwc7jNv57S4h8Om2fKNxikF5t4qVH1EllbsWzIySEt3N85uSEAAAAAWAsCfg6VmmrSip+itCHipHmZg72N+rWpJN+yBbOwZtmPQZIj4R4AAACAlSPg50C3ElM077v92nv078n03FwdNLhDlX/9ij1rQ7gHAAAAkFsQ8HOY2PhbCl+2VyfOXTMvK1HIRYM6VlF+V8csrFn2Y5Dk5GgrNxfCPQAAAADrR8DPQU5fiFfYsj26Gvf3TPm+ZQuob5tKcrTnT3knwj0AAACA3IZUmEP8L/qy5q7cp4Q7Zsp/pnpxdWzsxYzwdyHcAwAAAMiNCPg5wJbfTmnx90eU+tdU+QaD1LGxlwJrMFP+3Qj3AAAAAHIrAn42lmoy6ZufovTDrjtmyrezUd82lVS5HDPl341wDwAAACA3I+BnU4lJKZr/3f/025GL5mX5XNJmyi9ZhJny70a4BwAAAJDbEfCzodj4Wwpf/rtOnI0zLyteyEWDO1RR/rzMlH83wj0AAAAAEPCznTMX4xW2dK+uxCWYl1UqW0D9WleSowN/rrsZJOVxslM+Z3vCPQAAAIBcjcSYjRw8fkWffPO7Em79PVN+QLXi6tTEUzZGYxbWLHsyGKQ8joR7AAAAAJAI+NnGtr2n9dX6w0pN/WumfEkdGnsqsEYJwms6bod7NxeHrK4KAAAAAGQLBPwslppq0opNUfrv9hPmZfZ2RvVpVUlVvdyzsGbZl8EgOTvaKR/hHgAAAADMCPhZ6FZSiqZ/9at+2XfWvCyvs70Gd6yiUkXyZmHNsi/CPQAAAACkj4CfRVJTTQqeu0P7/rhkXlbMPW2m/KfyMVN+egj3AAAAAHB/BPwscuJcnEW4r1jmKb3c1ldOzJSfLsI9AAAAADwYU7NnkaLuLvIq6Saj0aDA6sU1uGMVwv19EO4BAAAA4OFIlFnEwc5GU4Ma6tqNRF2/maS/Js/HXQj3AAAAAPBo6MHPQgaDgV77BzAYJBdHe8I9AAAAADwCAj6ypdvhPq+LfVZXBQAAAAByBLqPke0YDJKrk71cnQn3AAAAAPCo6MFHtkK4BwAAAIB/hoCPbINwDwAAAAD/HAEf2QLhHgAAAAD+HQI+shzhHgAAAAD+PQI+shThHgAAAAAyBrPoI8sYDFLePPZyyUO4BwAAAIB/i4CPLGE0SK6EewAAAADIMAR8ZDqjQXJ1tpeLE+EeAAAAADIKz+AjUxHuAQAAAODJIOAj0xDuAQAAAODJYYg+MoXRIOV1tpcz4R4AAAAAngh68PHEEe4BAAAA4Mkj4OOJItwDAAAAQOZgiD6eGKPBoLwu9nJ2tMvqqgAAAACA1aMHH08E4R4AAAAAMhc9+MhwRoNB+VzslYdwDwAAAACZhh58ZCjCPQAAAABkDQI+MgzhHgAAAACyDkP0kSGMBoPcXO3l5EC4BwAAAICsQA8+/jXCPQAAAABkPQI+/hXCPQAAAABkDwR8/GOEewAAAADIPngGH/+IjTFtQj3CPQAAAABkD/Tg47ER7gEAAAAg+6EHH4/FxmiQm6uDHO1pOgAAAACQndCDj0dGuAcAAACA7IuAj0dCuAcAAACA7I20hoeyMRqU39VBDoR7AAAAAMi26MHHAxHuAQAAACBnIODjvgj3AAAAAJBzEPCRLsI9AAAAAOQsBHzcg3APAAAAADkPAR8WCPcAAAAAkDMR8GFGuAcAAACAnIskB0mSjY1B+V0d5WBnk9VVAQAAAAD8A/Tgg3APAAAAAFaAgJ/LEe4BAAAAwDoQ8HMxwj0AAAAAWA+ewc+lbP8K9/aEewAAAACwCvTg50KEewAAAACwPgT8XIZwDwAAAADWiSH6uYitjVH5XR0I9wAAAABghQj4uYSdjVH58zrIzpZwDwAAAADWiCH6uQDhHgAAAACsHwHfyhHuAQAAACB3IOBbMcI9AAAAAOQePINvpdLCvaPsbLmHAwAAAAC5AenPChHuAQAAACD3oQffytjbGuXmSrgHAAAAgNyGFGhFCPcAAAAAkHuRBK0E4R4AAAAAcjeG6FsBe9u0Z+5tbQj3AAAAAJBbkQhzOMI9AAAAAEAi4OdohHsAAAAAwG0kwxyKcA8AAAAAuBPP4OdA9rZGPZXXUTaEewAAAADAX0iIOQzhHgAAAACQHlJiDkK4BwAAAADcD0P0cwgHOxvld3Ug3AMAAAAA0kVazAEI9wAAAACAhyExZnOEewAAAADAo2CIfjbmYGej/HkdZWM0ZHVVAAAAAADZHAE/m3K0s5Eb4R4AAAAA8IgY950NEe4BAAAAAI+LgJ/NEO4BAAAAAP8EAT8bIdwDAAAAAP4pnsHPDgwGOdoaCfcAAAAAgH+MgJ8N2Nsald/VUUbCPQAAAADgH2KIfhYzGgn3AAAAAIB/jx78LMaQfAAAAABARqAHHwAAAAAAK0DABwAAAADAChDwAQAAAACwAgR8AAAAAACsAAEfAAAAAAArQMAHAAAAAMAKEPABAAAAALACBHwAAAAAAKwAAR8AAAAAACtAwAcAAAAAwAoQ8AEAAAAAsAIEfAAAAAAArAABHwAAAAAAK0DABwAAAADAChDwAQAAAACwAgR8AAAAAACsAAEfAAAAAAArQMAHAAAAAMAKEPABAAAAALACBHwAAAAAAKwAAR8AAAAAACtAwAcAAAAAwAoQ8AEAAAAAsAIEfAAAAAAArAABHwAAAAAAK0DABwAAAADAChDwAQAAAACwAgR8AAAAAACsAAEfAAAAAAArQMAHAAAAAMAKEPABAAAAALACBHwAAAAAAKyAwWQymbK6Epnp119/lclkkr29faYcz2QyKSkpSXZ2djIYDJlyTOB+aI/ILmiLyE5oj8guaIuZLzExUQaDQdWqVcvqqgAZwjarK5DZMvtiaTAYMu1mAvAwtEdkF7RFZCe0R2QXtMXMZzAYuJkCq5LrevABAAAAALBGPIMPAAAAAIAVIOADAAAAAGAFCPgAAAAAAFgBAj4AAAAAAFaAgA8AAAAAgBUg4AMAAAAAYAUI+AAAAAAAWAECPgAAAAAAVoCADwAAAACAFSDgAwAAAABgBQj4AAAAAABYAQI+AAAAAABWgID/BBw7dkx+fn5asWKFedlbb70lb29vi5/AwMAsrCWs2fnz5+9pb97e3uY2efDgQXXr1k1Vq1ZVYGCgFi5cmMU1hjV7WHvk+ojMtHLlSj3//PPy9fVVixYttG7dOvO6U6dOacCAAapWrZrq16+vjz76SCkpKVlYW1izB7XFWbNmpXvdBICHsc3qClibpKQkDR8+XDdu3LBYfvjwYQ0cOFDdunUzL7Oxscns6iGXOHTokBwcHLRhwwYZDAbzcldXV129elW9e/dWYGCggoODtWfPHgUHB8vZ2Vnt27fPwlrDWj2oPUpcH5F5vv32W40dO1ZjxoxRgwYNtGbNGg0dOlRFihRRpUqV1LdvX5UuXVqLFy/WyZMnNXbsWBmNRgUFBWV11WFlHtQW/fz8dPjwYbVp00YjRozI6qoCyGEI+BksNDRULi4uFstMJpOioqLUv39/ubu7Z1HNkJscOXJEpUuXVqFChe5Z9/nnn8vOzk7jx4+Xra2typYtqxMnTmjOnDkEfDwRD2qPXB+RWUwmk0JCQtSjRw917dpVkvTKK68oMjJSu3bt0unTp3XmzBl9/fXXypcvn7y8vHT58mVNmTJFAwcOlL29fRafAazFw9qin5+fjhw5ok6dOnFdBPDYGKKfgSIiIrRkyRJNnjzZYvnJkyd148YNeXh4ZFHNkNscPnxYZcuWTXddZGSk/P39ZWv79/292rVr6/jx47p06VJmVRG5yIPaI9dHZJZjx47p9OnTatWqlcXyzz77TAMGDFBkZKR8fHyUL18+87ratWsrPj5eBw8ezOzqwoo9rC0mJibq+PHjXBcB/CME/AwSFxenkSNH6q233tLTTz9tse7IkSOSpC+++EKBgYFq0qSJxo8fr2vXrmVFVZELHDlyRFeuXFHXrl1Vt25ddenSRVu2bJEknTt3TkWKFLHY/nbP6tmzZzO9rrB+D2qPXB+RWY4dOyZJunHjhvr27as6deqoY8eO+vHHHyVxbUTmeVhbjIqKUkpKitavX69mzZqpUaNGGjFihC5cuJCV1QaQQxDwM8i7774rPz+/e+7GSmn/gTUajSpUqJBmz56tN998U9u2bdOgQYOUmpqaBbWFNUtOTlZ0dLRiY2P16quvas6cOapatar69++v7du3KyEh4Z6hpg4ODpKkW7duZUWVYcUe1h65PiKzxMfHS5JGjRqlli1bat68eapXr54GDRrEtRGZ6mFt8faNTycnJ4WEhGjixImKjo5Wjx49lJCQkJVVB5AD8Ax+Bli5cqUiIyP13Xffpbv+lVde0UsvvaT8+fNLkry8vOTu7q5OnTpp3759qlKlSmZWF1bO1tZWO3fulI2NjRwdHSVJlSpV0tGjR/XZZ5/J0dFRiYmJFvvc/s9rnjx5Mr2+sG4Pa49z5szh+ohMYWdnJ0nq27evXnjhBUlShQoVdODAAc2fP59rIzLNw9rinDlz1LBhQz311FPmfTw9PdWwYUP9+OOPev7557Ok3gByBnrwM8Dy5ct1+fJlNWrUSH5+fvLz85MkjRs3Tv369ZPRaDT/5/U2T09PSWlDAoGM5uzsbA5Tt3l6eur8+fMqUqTIPcP8bn8uXLhwptURuceD2iPXR2SW29c3Ly8vi+XlypXTqVOnuDYi0zysLUqyCPdS2uMibm5uXBcBPBQBPwNMnTpVa9eu1cqVK80/khQUFKSJEydq5MiR6tWrl8U++/btk5R2MQcy0tGjR1WtWjXt3LnTYvn+/ftVrlw51axZU7t377Z4t/OOHTtUpkwZFShQILOrCyv3sPbI9RGZxcfHR87Oztq7d6/F8iNHjqhkyZKqWbOmDhw4YB4+LaVdG52dnVW+fPnMri6s2MPa4owZM9SsWTOZTCbzulOnTunq1atcFwE8FAE/AxQuXFilSpWy+JGkAgUKqHDhwmrWrJm2b9+usLAwnTx5Ups3b9aYMWPUsmXL+84sDfxTZcuWlYeHh8aPH6/IyEj98ccfmjRpkvbs2aNXXnlF7du3V3x8vMaOHauoqCitWLFCCxYs0IABA7K66rBCD2uPXB+RWRwdHdWvXz/NnDlTq1ev1smTJzVr1iz9/PPP6t27t5o0aSJ3d3e9/vrrOnTokDZs2KDp06erT58+vCIPGephbfHZZ5/V6dOn9e677+rYsWOKiIjQq6++qmrVqqlBgwZZXX0A2ZzBdOftQWQYb29vTZo0Se3atZMkrVu3TnPmzFF0dLRcXV3VqlUrvf766+YJfICMdOnSJU2bNk1bt25VXFycKlasqOHDh6tGjRqSpN9//10TJ07UgQMH5O7urj59+qhbt25ZXGtYq4e1R66PyEzz58/XokWLdP78eZUtW1avvvqqmjRpIkk6ceKEgoODFRkZqXz58qlDhw569dVXZTTSH4KM96C2uH37doWEhOjw4cOyt7dX48aNNWrUKIvXOAJAegj4AAAAAABYAW5JAwAAAABgBQj4AAAAAABYAQI+AAAAAABWgIAPAAAAAIAVIOADAAAAAGAFCPgAAAAAAFgBAj4AAAAAAFaAgA8AAAAAgBUg4AMAnphhw4bJ29tb8+bNy+qq5DiTJ09W9+7ds7oaAAAgByHgAwCeiGvXrmnDhg3y8vLSkiVLZDKZsrpKOca8efM0f/78rK4GAADIYQj4AIAnYvXq1ZKksWPH6vjx49qxY0cW1yj7+/PPPzVkyBBNnTpVrq6uWV0dAACQwxDwAQBPxPLly1WnTh3Vrl1bpUqV0uLFi83r+vTpo3bt2t2zz6BBg9S6dWvz58jISHXr1k1VqlSRv7+/Ro0apStXrpjXr1ixQhUrVtTSpUtVr149+fv7KyoqSikpKZozZ45atmypypUrq2rVqurcufM9Nxk2bdqkdu3aqXLlymrWrJlWr16tZ599VqGhoeZtYmJi9M4776hu3bry9fVVp06dtH379sf6LmbPnq1KlSopNjbWYvmCBQvk4+Ojy5cvS5ImTZqkEydO6PPPP1eFChUe6xgAAAAEfABAhjt69Kj27duntm3bSpLatm2rjRs36tKlS5Kk1q1b63//+59OnDhh3icuLk5btmxRmzZtJEkRERHq1auXHB0d9dFHH2nMmDHatWuXevTooYSEBPN+KSkpmjdvniZOnKjRo0erbNmymjp1qsLDw/Xiiy/q008/1YQJExQTE6PXXntNN2/elCTt2LFDgwYN0tNPP63Q0FB17dpV48aN09mzZ81l37p1Sz179tTGjRv1xhtvKCwsTEWKFFG/fv0eK+S3atVKycnJ+v777y2Wr1mzRvXr11eBAgUkSa+//rpWrVqlmjVrPsa3DQAAkIaADwDIcMuXL5ebm5sCAwMlSS+88IJSUlK0bNkySVLTpk2VJ08e8zB+Sfr++++VkpKili1bSpKmTZumMmXK6JNPPtEzzzyjtm3bat68eYqOjtby5cstjjdw4EA1atRIL7zwggwGgy5cuKA33nhD3bt3l7+/v1q1aqWhQ4cqJiZGhw8fliSFhobK09NTYWFhCggIUI8ePTR+/HglJSWZy/3222916NAhhYeHq2PHjgoICFBISIj8/Pw0derUR/4+ihUrppo1a1qc78mTJ/X777+bb2hIkpeXlwwGwyOXCwAAcCcCPgAgQyUlJWnVqlVq0qSJEhISFBcXJ2dnZ1WvXl1ff/21UlNTlSdPHjVp0kRr164177dmzRrVqVNHhQsX1s2bN7V3714FBATIZDIpOTlZycnJKlGihMqWLauff/7Z4ph3D2efNm2aevbsqStXrigyMlLLly/XqlWrJEmJiYlKTEzUb7/9pqZNm1oE6ubNm8vW1tb8efv27XJ3d5ePj4+5DikpKXrmmWe0f//+e4bcP0jr1q0VERGhixcvms/XxcXFfBMEAADg37J9+CYAADy6TZs26fLly1q2bJm5x/5OW7duVUBAgNq0aaNVq1bp0KFDKliwoHbu3Kn3339fUtpw/dTUVM2dO1dz5869pwwHBweLz3ny5LH4vG/fPgUHB2vfvn1ycnJSuXLlVLRoUUmSyWRSTEyMUlJSzEPjb7OxsZGbm5v5c0xMjC5evCgfH590z/XixYvKly/fw78Upd08mDBhgtatW6cePXpozZo1atasmRwdHR9pfwAAgIch4AMAMtTy5ctVokQJTZw40WK5yWTSkCFDtHjxYgUEBKhOnTpyd3fXunXr5O7uLgcHBzVt2lSS5OzsLIPBoF69eqlFixb3HMPJyem+x4+Pj1e/fv3k7e2tNWvWyMPDQ0ajUZs3b9b69eslSQUKFJCdnZ15ToDbUlNTFRMTY/7s6uqq0qVL33c4fvHixR/pO7ldVmBgoNatW6fatWvr6NGjevvttx95fwAAgIch4AMAMszFixe1detW9evXT7Vq1bpnffPmzbVixQqdP39ehQsXVqtWrfTTTz8pb968atKkibkn3sXFRRUrVlR0dLR8fX3N+yckJCgoKEgBAQEqV65cunWIjo5WTEyMevToYbHNli1bJKWFeBsbG1WrVk0bN27UkCFDzNv8+OOPSk5ONn/29/fXpk2bVKBAAfMIACltVvyDBw8+1nP4ktSmTRsNGjRI//nPf1S0aFH5+/s/1v4AAAAPwjP4AIAMs3LlSiUnJ6fb6y6lzaafkpKir7/+WlJa4D18+LB2795tMdmcJA0dOlTbtm3TsGHDtHnzZv3444/m2evvN2ReksqUKSMXFxfNnj1bmzZt0rZt2/T222/rq6++kiTzLPpBQUE6dOiQgoKCtGXLFi1evNjco377ufx27dqpaNGi6t27t7755hvt2LFD06dPV0hIiAoVKiQ7O7vH+n4aNGggNzc3LVmyRK1atWJCPQAAkKEI+ACADLNixQp5enrKy8sr3fXVq1dX8eLFtXTpUqWkpKh8+fLy8vJSgQIFVKdOHYtt69evr88++0znzp1TUFCQRo4cKRsbG82fP19Vq1a9bx1cXV0VHh4uk8mk1157TSNHjtSZM2e0aNEiOTs7KzIyUpJUo0YNhYaG6tixYxo0aJDmz59vDvjOzs6S0p7t//LLL1W9enV9+OGHevnll/X9999r2LBhGj169GN/P7a2tmrRooVSUlLUunXrx94fAADgQQwmk8mU1ZUAACCzbdy4UUWKFLEYDXD06FG1bNlS4eHhaty4cRbWDgAA4PHxDD4AIFfatm2b1q5dq+HDh6tMmTI6f/68Zs2aJQ8PD9WvX/+RyjCZTEpJSXnodjY2NgzHBwAATxw9+ACAXCkhIUEhISFav369Lly4IDc3NzVo0EDDhg1TwYIFH6mMFStWPNJQ/YULF6Y76SAAAEBGIuADAPAPXb16VadOnXrodrcn/gMAAHiSCPgAAAAAAFgBZtEHAAAAAMAKEPABAAAAALACBHwAAAAAAKwAAR8AAAAAACtAwAcAAAAAwAoQ8AEAAAAAsAIEfAAAAAAArMD/ATwD6m9PkdbJAAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA/gAAANrCAYAAADyKTN6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd1gUV9sG8HuX3hEbCjZEEOkWUFFBYkssib0FS+xiiSWi0WhINXYFu0ajIXmNPYkav2hiF+wtYlRUilEssPTdZXfn+4PshKUjKIj377q83nB2ypnZwddnzjnPIxEEQQARERERERERvdakFd0BIiIiIiIiIio7BvhEREREREREVQADfCIiIiIiIqIqgAE+ERERERERURXAAJ+IiIiIiIioCmCAT0RERERERFQFMMAnIiIiIiIiqgIY4BMRERERERFVAQzwiYjolRIEoaK7QERERFQlMcAnokopKCgIzs7OOn/c3NwQEBCA0NBQpKSkVHQX32h79uyBs7MzEhISSrxPamoqZs2ahQsXLohtQUFBCAoKehldLBGVSoUVK1bA398fnp6eGDJkCK5evVrsfjExMRg/fjy8vb3h4+ODKVOm4MGDBzrbKJVKLF26FP7+/vDw8EDv3r1x4MCBl3QlL09CQgKcnZ2xZ8+eCutDbGwsxo8fj5YtW8LX1xcLFixAenp6sftdv34dQUFB8Pb2Rrt27bBs2TIolUqdbZ49e4YZM2bA19cXLVq0wPTp0/HkyROdbUr6nPz666/o3r07PDw88Pbbb2Pv3r1lu3AiIqJS0q/oDhARFaZZs2ZYsGCB+HN2djb++usvLFu2DNHR0fjxxx8hkUgqsIdUGtHR0di/fz/69u0rtuX+fivCwoULsWvXLsyYMQN2dnbYsmULRowYgX379qFBgwYF7hMfH4/BgwfDwsIC8+fPR/Xq1bFr1y4MHDgQu3fvhr29PQBg2rRpOHbsGD744AO0adMGN27cwNy5c5GUlFShLzVeN6mpqRg+fDhq1KiBhQsXIikpCYsXL0ZCQgI2b95c6H7x8fEYOXIkvLy8sGLFCsTExGD58uWQyWT47LPPAOQE7mPGjEF6ejo+/fRTqFQqLF26FKNGjcKePXtgYGAAoGTPyeHDhzFz5kwMGzYM7du3x5EjRzB79mwYGhqie/fuL/9GERERAYBARFQJvf/++8L7779f4Gfh4eGCk5OTcPny5VfbKRLt3r1bcHJyEuLj40u8T2RkpODk5CRERka+xJ6V3D///CM0a9ZMiIiIENsUCoUQEBAgzJ07t9D9Pv/8c8HNzU2Ii4sT29RqtdC3b19h+vTpgiAIwl9//SU4OTkJa9as0dl3+/btgpeXl5CSklLOV/PyxMfHC05OTsLu3bsr5Pzr1q0TPD09hefPn4ttx44dE5ycnIQLFy4Uut8nn3widOjQQVAoFGJbRESE0LRpU+Hhw4eCIAjCL7/8Ijg5OQl37twRt7lz547g7Ows7N+/XxCEkj8nXbp0EaZOnarTh6lTpwqdO3d+sQsnIiJ6AZyiT0SvHTc3NwDAP//8I7YdOXIEffr0gbu7O/z8/PDFF18gMzNTZ78jR45gyJAh8Pb2hpubG7p164aIiAjx86ioKDg7O+N///sfOnbsiObNm+P06dNISkrCjBkz4OfnB3d3d7z77rvYt2+fzrEfPHiAKVOmwM/PD15eXggKCsLFixfFz7XTnA8dOoQpU6aIU7vnzZuXr595PXnyBCEhIWjTpg28vb3x/vvv4/Lly+LnCoUCq1evRrdu3eDu7o4uXbpgw4YN0Gg04jZBQUGYOXMmpkyZAi8vL4wcOVLs05YtW9CtWzd4enpi9+7dAIDbt29j3LhxaN68OZo3b47g4GDEx8cX2c+dO3eiT58+8PLygoeHB959910cOnRIvLfDhg0DAAwbNkwcwc47Rb+k1zJ37lxs2LABAQEBcHd3x6BBg3Dt2rV89zssLKzQ/p49exYqlQqdO3cW2wwNDREQEIDjx48Xut+9e/fg6OiIevXqiW1SqRStWrUS94uJiQEAdOzYUWdfX19fZGZm4ty5c4UePy/tcoirV6+id+/e8PDwQM+ePfHbb78Vud8vv/wCZ2dn3L59W6f9yJEjcHZ2xs2bNwEAt27dwqRJk9C6dWu4urqiffv2+OKLLyCXyws8blhYGJydnfO1573fCoUCixYtgr+/P9zc3NCzZ08cPHiwwGMVtdTj1KlTaNGiBWxsbMS2du3awczMDCdOnChyP39/fxgaGopt3bp1g0ajwalTp8RtGjVqBEdHR3EbR0dHNG7cWPwuS/KcJCQk4MGDBzrbAEDXrl0RGxubb/kGERHRy8IAn4heO/fv3wcAMcD65ZdfEBwcDAcHB6xevRqTJk3Czz//jIkTJ4oJ3Y4dO4bg4GC4urpizZo1CAsLQ7169fDZZ5/lW0sbHh6OkJAQzJ8/H97e3vjoo48QExOD0NBQbNy4Ec2aNUNISAgiIyMBAHfv3kWfPn2QkJCAefPmYcmSJZBIJBg+fHi+QG7BggWws7PDmjVrMGrUKOzatQtr164t9FozMjIwePBgREVF4aOPPkJ4eDiMjIzwwQcf4MGDBxAEAePHj8emTZvQv39/rFu3Dt26dcOKFSvyTX8/dOgQzMzMsHbtWowePVpsDwsLw5gxY7Bo0SL4+fnh/v37GDRoEJ4/f45vvvkGX375pTgt/fnz5wX2MyIiAvPnz0enTp2wfv16LFmyBIaGhpg5cyYeP34MV1dXzJ8/HwAwf/78Aqfml+ZaDh8+jKNHj2LevHlYtmwZnj17hsmTJ0OtVgMAatWqhR07dqB///6F3tuYmBiYmZmhZs2aOu0NGjTAkydPkJGRUeB+1apVw9OnT5Gdna3THh8fj7S0NMhkMlSrVg2A7ksoAIiLixO3La1x48bhrbfeQnh4OBo1aoQPP/ywyBcRnTp1gqmpab51/7/++iuaNGmCZs2a4cmTJxg6dCiysrKwcOFCbNy4Ed27d8f27duxbdu2UvdRSxAEBAcH43//+x9GjhyJtWvXwtvbG9OmTdN5Oda/f3/s2LEDtWrVKvRYMTExaNSokU6bnp4e7O3txb8L8pLL5Xj48GG+/WxsbGBubi7uFxMTg4YNG+bbv379+jrbFPecaF/o5D2Wdvp+Yf0kIiIqb1yDT0SVliAIUKlU4s8pKSk4d+6cGCy4ublBEAQsWbIE7du3x5IlS8RtGzZsiBEjRuD48eMICAjA3bt30bt3b8ydO1fcxtvbG76+voiKioKnp6fYPmTIEHTr1k38+dy5cwgODkanTp0AAD4+PrC2thZHBsPDw2FoaIht27bB3NwcABAQEIAePXpg0aJF2LVrl3gsf39/hISEAADatGmD06dP49ixY5gxY0aB92Dv3r14+PAh9u7dCxcXFwBA8+bN8d577+H8+fOIjY3FmTNnsGzZMnGdr5+fH4yNjbFy5UoMGzYMTZo0AQAYGBggNDRU7Ld21PTtt9/WWRc/Y8YMmJiYYOvWreL1tGnTBp06dcKmTZvE/ucWHx+PUaNGYeLEiWKbnZ0d+vTpg4sXL6J79+7iKKmjo6POiKnWiRMnSnwtKpUKmzdvFvuXkZGBkJAQREdHw83NDYaGhvDy8irwnmqlpaWJ++dmZmYGAEhPTxf/O7c+ffrg119/RUhICKZNmwZzc3Ps378fJ0+eBABkZWXBx8cH9erVwxdffAETExO4u7vj1q1b4suf4mZtFCQoKAjBwcEAgPbt26N3795YvXo1/P39C9zexMQEXbt2xcGDBzFt2jQAOffpzz//FI9z+/ZtuLi4YOXKleK9aNu2LU6fPo2oqCiMHTu21P0EgDNnzuDkyZNYvnw53nnnHbHPWVlZWLJkCXr06AF9fX3Y2trC1ta2yGOlpaUV+D2YmZkVmmgvLS0NAAr9frX7paWlFZhrwczMTHzBU5LnRHu8vNvl3oaIiOhVYIBPRJXW+fPn4erqqtMmlUrRtm1bfPbZZ5BIJIiJicHjx48xbtw4nZcBrVq1grm5OU6fPo2AgABxxDojIwP3799HXFwcrl+/DgD5smprA2ktX19fhIWF4ebNm2jfvr1OkA7kvADo2LGjzj/u9fX10b17d6xevVpnJDhv0Glra4uHDx8Weg8uXrwIe3t7nT6ZmJjg8OHDAIDFixdDX19f54UEAPTq1QsrV67EuXPnxKDYwcFBZ7pyYdcbGRkJHx8fGBsbi/fU3NwcLVu2xJkzZwrs5+zZswHkJES7d+8eYmNjERUVBSD//S3MuXPnSnwtjo6OOve7du3aAHKC65ISiinXJ5UWPMnNz88PixcvxldffSWOjrdt2xZjxoxBWFgYjI2NYWhoiM2bN+Pjjz/GiBEjAAA1a9bEvHnz8OGHH8LExKTE/dTq3bu3+N8SiQSdO3dGWFgY5HI5DAwMdK5HIpFAT08P7777Lvbu3Ytr167Bw8MDR48ehVKpRK9evQDkTHVv164dsrOzcffuXcTGxuL27dtISkqCtbV1qfuodfbsWUgkEvj7++v8XgYGBuLnn3/GnTt38j13hSnqeyosyWbuJR1F7VeSY5fkOSnufIU9S0REROWNAT4RVVqurq4IDQ0FkPOPbSMjI9SpU0cnsJPJZACA0NBQcdvctOWukpKSsGDBAhw5cgQSiQQNGjRAy5YtAeT/B7ypqanOz8uXL8e6detw6NAhHD58WOclg52dHVJSUlCjRo18565RowYEQdAZvcsb2Eml0iIDCJlMhurVqxf6eUpKCqpVqwY9PT2ddu10Yu1IJoACR0GB/Ncrk8lw8ODBfOulAeisg84tLi4O8+fPx9mzZ2FgYAAHBwc0bdoUQMnr3pfmWgq6j0DxgV1u5ubmBU7D135fFhYWhe7bq1cvdO/eHfHx8TAxMUHt2rWxcuVKSKVSWFpaAsiZnh0REYHnz59DJpOhQYMGePToEQRBgJWVVYn7qZV3Gnv16tUhCAJSU1MxY8YMneUgPj4+2L59O3x9fVG7dm0cOHAAHh4eOHDgAHx8fMRRc41Gg2XLliEiIgKZmZmoU6cOPDw8YGRkVOr+5SaTySAIApo3b17g50+ePClxgF/U96R9sVPQPgAK3U/73RZ17JJsA+Q8J9pt825X2Mg+ERHRy8IAn4gqLTMzM7i7uxe5jTaYmjVrFnx8fPJ9rg2kZs6ciXv37mHr1q3w9vaGoaEhsrKy8NNPPxXbDwsLC3z00Uf46KOPcO/ePRw9ehRr1qxBaGgoNmzYACsrKzx79izffk+fPgWQs2Y7b13tkrKwsCgwAdmlS5dgZWUFKysrJCcnQ61W6wTG2vNp14KX9pxt27bFyJEj832mr5///zY0Gg3Gjh0LAwMD7Nq1Cy4uLtDX18fdu3exf//+Ep/3ZVxLURwcHJCeno6kpCSdFxexsbGws7ODsbFxgfvFxMTg+vXreO+993TWXN+8eRPOzs7Q09ODXC7H4cOH0bx5c9SrV098SfPXX38BQL6ZKSUhk8l0XiQ9e/YMenp6sLa2RmhoqE5wqX2ZI5VK0bNnT/z6668YP348Tp8+LZaIA4ANGzZg69atCA0NRZcuXcRAtV+/foX2Qzuynft7yhvYWlhYwNTUtNB1/IWVICxIo0aNxNwFWmq1GgkJCejSpUuB+5iZmaF27dqIjY3VaX/+/DkyMjLQuHFj8djR0dH59o+Li4OHhweAkj0n2rX+sbGxaNasmc42AMTzERERvWycM0ZErzUHBwdUr14dCQkJcHd3F//Url0bS5cuFTOFX7x4EV26dIGvr684TV2bgbuoUd+HDx/C399fzFju4OCAMWPGoG3btmICtVatWuHPP//UGalXq9U4cOAA3N3dC5wWX1ItW7ZEfHw87ty5I7YpFApMnjwZu3btgo+PD1QqVb6M6j///DMAoEWLFqU+p4+PD+7evQsXFxfxfrq5uWHr1q34/fff822fnJyM+/fvo1+/fnB3dxdfAuS9v3lH5gs6b3lfS1Hatm0LADrnUyqVOHbsGPz8/Ard786dOwgJCcG9e/fEtrt37+LUqVNingYDAwN8/vnnOi+QVCoVvv/+e9SvXx9OTk6l7u+RI0fE/xYEAf/3f/+HFi1awNDQEA4ODjrPv4ODg7jtu+++i8ePH2P16tXQ09PTCYovXrwIR0dH9O3bVwzuExMTcfv27UJ/L7Sj0Y8fP9Y5Tm4+Pj7IzMyEIAg6/bp9+zZWr16tM22/OH5+fjh//jySkpLEtlOnTiEzM7PI78nPzw/Hjh3TWSJy+PBh6OnpoXXr1gBylijExMTg7t274jZ3795FTEyMeOySPCcNGjSAvb29uHRG6//+7//QsGFD2Nvbl/h6iYiIyoIj+ET0WtPT08O0adMwf/586OnpoWPHjkhNTcWaNWuQmJgojpR6eHjgl19+gaurK2xtbXHp0iVs2LABEomkyHXbdnZ2sLW1xRdffIH09HTUr18fN27cwPHjxzFu3DgAwKRJk3DixAkMGzZMHMn+/vvvER8fj02bNpXp+vr06YPt27djwoQJmDJlCqpVq4Zt27YhOzsbQ4YMgb29PXx9fTFv3jwkJiaiadOmOHfuHDZu3IjevXsXmMyuOBMnTsSgQYMwbtw4DB48GEZGRtixYweOHDmCVatW5du+evXqsLOzQ0REBGxtbWFpaYmTJ0+Ko7fa+6sNII8dOwYrKytxCr9Whw4dyu1alEolbt68WWQSNzs7O/Tu3Rtff/01FAoFGjZsiC1btiA1NVWnykBcXBySkpLE/An+/v6oX78+Zs6cialTpyI9PR2LFi2Cvb29uN5eT08PQ4YMwXfffQdbW1s0atQIERERuHTpElavXi0uKUhPT8fdu3dRv379Qpc/aC1atAgKhQKNGjXCzp07ERMTg++++67Ye+Hk5AQXFxf88MMPePvtt3Wmi3t4eGDNmjXYsGEDvLy8EBsbi/Xr10OpVBb6e+Hv74+vv/4a8+fPx6hRo/Do0SOsXr1aZwmIv78/WrVqhYkTJ2LixIlo3Lgxrl27hlWrVqF9+/bitT5+/BiPHz9Gs2bNCn0RNmTIEHz//fcYOXIkJk2aBJlMhsWLF6NDhw46SwCuXLkCGxsb1K9fHwAwevRoHDhwAKNHj8bIkSPx4MEDLFu2DAMGDEDdunUBAO+88w7WrVuHMWPGiIkuly5dCicnJ7z99tsASv6cBAcHY86cObC2tkZgYCCOHj2KQ4cOYfny5cV+R0REROVGICKqhN5//33h/fffL/H2Bw4cEHr37i24ubkJPj4+wvjx44Vbt26JnyckJAjjxo0TWrRoIbRo0ULo27evsH//fmHUqFFC3759BUEQhMjISMHJyUmIjIzUOfaTJ0+E2bNnC+3atRNcXV2FTp06CWvXrhXUarW4zc2bN4XRo0cLXl5egre3tzB8+HDh/Pnz4ufx8fGCk5OTsHv3bp1jh4SECB07dizy2h4/fixMnz5daNmypdC8eXPhgw8+EKKjo8XPMzMzhYULFwrt27cXXF1dha5duwqbNm3S6V9B97OwPgmCINy4cUMYNWqU4O3tLXh5eQkDBgwQjhw5In6+e/duwcnJSYiPjxcEQRCio6OF999/X/Dy8hJ8fHyEIUOGCCdOnBC6desmTJkyRRAEQVCr1cL06dMFd3d3oXv37gX260WvJe93p722VatWFXlvFQqF8OWXXwpt2rQRPD09hSFDhghXrlzR2SYkJERwcnLSaXvw4IEwduxYoUWLFkLbtm2F2bNnC4mJiTrbKJVKYdmyZYK/v7/g5eUlDBo0SDh58mSB/S7oO9DS3uv9+/cL77zzjuDu7i4MHDgw33NalG+//VZwcnISjh07lu/6Q0NDBT8/P8HDw0Po2rWrsGrVKiEsLExwc3MTUlJSCnxO9u7dK3Tp0kVwdXUVevXqJZw6dUrcVysjI0P46quvhA4dOgiurq5CYGCgsHTpUkEul4vbrFq1Suc5Kszff/8tDB8+XPDw8BDatGkjfPLJJ0JaWprONk5OTkJISIhO2/nz54X+/fsLbm5uQvv27YUlS5YISqVSZ5t//vlHCA4OFry8vIRWrVoJH374Yb7vsiTPiSAIwo8//ih07txZcHNzE95++21h7969RV4XERFReZMIQgmzHxEREVG5W7lyJRwdHcXSgHnt2bMHc+bMwdGjRznVm4iIiIrENfhEREQVJDExEYcPH4a3t3dFd4WIiIiqAAb4REREFcTa2hphYWHimnAiIiKisuAUfSIiIiIiIqIqgCP4RERERERERFUAA3wiIqo0OKmMiIiI6MUxwCeiCnP79m1MmzYNfn5+cHNzQ7t27fDhhx/i1q1bFd01KoM9e/bA2dkZCQkJJd4nNTUVs2bNwoULF8S2oKAgBAUFvYwulohKpcKKFSvg7+8PT09PDBkyBFevXi12v5iYGIwfPx7e3t7w8fHBlClT8ODBA/Fz7f0p7M/evXtf4lWVr6ioKDg7OyMqKqrC+nD9+nUEBQXB29sb7dq1w7Jly6BUKovd79SpU+jbty88PT0RGBiIzZs353vBFBsbi/Hjx6Nly5bw9fXFggULkJ6errNNRkYGQkND4efnB29vb4wZMwb37t3Ld77vvvsOnTt3hoeHB3r37o3jx4+X7cKJiIgKoF/RHSCiN9OdO3cwcOBAeHl5Yd68eahevToeP36M77//HgMGDMC2bdvg5eVV0d2kVyQ6Ohr79+9H3759xbYFCxZUYI+AhQsXYteuXZgxYwbs7OywZcsWjBgxAvv27UODBg0K3Cc+Ph6DBw+GhYUF5s+fj+rVq2PXrl0YOHAgdu/eDXt7ewQEBGDHjh359p03bx7S09Ph7+//si+tyoiPj8fIkSPh5eWFFStWICYmBsuXL4dMJsNnn31W6H5XrlzB+PHj8fbbb2Pq1Km4ePEiFi9eDLVajbFjxwLIeek0fPhw1KhRAwsXLkRSUhIWL16MhIQEbN68WTzWjBkzcPXqVXz00UcwNzdHeHg4hg0bhgMHDsDKygoAsGXLFixevBjBwcFwc3PD7t27MWHCBGzbtg0tW7Z8uTeJiIjeLAIRUQWYM2eO0LFjRyE7O1unPSMjQ2jfvr0wZsyYCuoZldXu3bsFJycnIT4+vsT7REZGCk5OTkJkZORL7FnJ/fPPP0KzZs2EiIgIsU2hUAgBAQHC3LlzC93v888/F9zc3IS4uDixTa1WC3379hWmT59e6H7fffed0LRpU+HKlSvlcwGvSEV/b5988onQoUMHQaFQiG0RERFC06ZNhYcPHxa63wcffCD069dPp23RokWCt7e3kJWVJQiCIKxbt07w9PQUnj9/Lm5z7NgxwcnJSbhw4YIgCIJw6dIlwcnJSTh27Ji4zfPnzwUvLy9hzZo1giAIQlZWltCyZUth0aJF4jYajUYYMGCAMGLEiDJcPRERUX6cok9EFeLZs2cQBAEajUan3dTUFB9//DHefvttsS0oKAizZ8/GunXr0LZtW7Ro0QITJ07Ew4cPdfa9ffs2xo0bh+bNm6N58+YIDg5GfHy8zja3bt3CpEmT0Lp1a7i6uqJ9+/b44osvIJfLxW2cnZ0RHh6OPn36wMPDA+Hh4dizZw/c3d1x4cIF9O3bF+7u7ujatSv++OMP3Lt3D8OHD4enpyc6d+6MAwcO6Jzz/PnzGDVqFFq1agU3NzcEBgYiLCxMvPaEhAQ4Ozvj0KFDmDJliji1e968ecjMzCzyPj558gQhISFo06YNvL298f777+Py5cvi5wqFAqtXr0a3bt3g7u6OLl26YMOGDTr3PSgoCDNnzsSUKVPg5eWFkSNHin3asmULunXrBk9PT+zevbvE9zmvnTt3ok+fPvDy8oKHhwfeffddHDp0CEDONO9hw4YBAIYNGyZOy887Rb+k1zJ37lxs2LABAQEBcHd3x6BBg3Dt2jVxG+21hYWFFdrfs2fPQqVSoXPnzmKboaEhAgICipxafe/ePTg6OqJevXpim1QqRatWrQrd79mzZ1ixYgUGDx4MT0/PQo9dkLCwMAQGBuLPP/8Uv6cBAwYUO2V+3bp1cHNzQ0pKik771q1b4erqiufPnwMo/tnNa/bs2QgMDNRp097vPXv2iG0ymQzz589H27Zt4e7ujgEDBuDs2bP5juXs7FzkdZw6dQr+/v4wNDQU27p16waNRoNTp04VuI9SqURUVJTOdwsAXbt2RUZGBi5evCgeu0WLFrCxsRG3adeuHczMzHDixAlxG1NTU7Rr107cxsbGRuf7vnr1KlJTU3XOJ5FI0LlzZ0RFRen83UNERFRWDPCJqEIEBATgn3/+waBBgxAREYGYmBhx/Wu3bt3Qu3dvne2PHj2KPXv2YN68eQgNDUV0dDSCgoKQlZUFALh//z4GDRqE58+f45tvvsGXX34pTpfWBitPnjzB0KFDkZWVhYULF2Ljxo3o3r07tm/fjm3btumcb926dejZsydWrVqFrl27AshZkz1jxgwMGjQIa9euhYmJCWbOnInx48cjICAA69atQ61atRASEoLHjx8DyHmhMGLECFhbW2P58uVYu3YtWrZsifDwcDHA1VqwYAHs7OywZs0ajBo1Crt27cLatWsLvYcZGRkYPHgwoqKi8NFHHyE8PBxGRkb44IMP8ODBAwiCgPHjx2PTpk3o378/1q1bh27dumHFihX5pr8fOnQIZmZmWLt2LUaPHi22h4WFYcyYMVi0aBH8/PxKdJ/zioiIwPz589GpUyesX78eS5YsgaGhIWbOnInHjx/D1dUV8+fPBwDMnz+/wKn5pbmWw4cP4+jRo5g3bx6WLVuGZ8+eYfLkyVCr1QCAWrVqYceOHejfv3+h9zYmJgZmZmaoWbOmTnuDBg3w5MkTZGRkFLhftWrV8PTpU2RnZ+u0x8fHIy0tDTKZLN8+q1atglQqxYcfflhof4qSlJSEkJAQDBkyBCtXroSxsTFGjRqF6OjoQvfp2bMnVCoV/u///k+n/cCBA2jXrh2qV69eqme3NBQKBYYPH46jR49i2rRpCA8Ph62tLUaPHq0T5E+cOLHApQxacrkcDx8+RKNGjXTabWxsYG5ujvv37xe4X3x8PLKzs9GwYUOddu2yC+1+MTEx+Y6tp6cHe3t7nW3s7e2hp6ens139+vV1tgFQ4PnUajXi4uIKvUYiIqLS4hp8IqoQQ4YMwdOnT7F582ZxrWy1atXQrl07DBs2DB4eHjrbZ2VlYc+ePeLIqIODA3r37o19+/Zh8ODBCA8Ph4mJCbZu3Qpzc3MAQJs2bdCpUyds2rQJISEhuH37NlxcXLBy5Upxm7Zt2+L06dOIiooS194CQMuWLTFy5Ejx5+vXr0Oj0WD8+PFiYJiamopp06Zh+PDh4rYWFhbo27cvbty4AVtbW9y6dQtt27bF4sWLIZXmvFP18/PDH3/8gaioKHTv3l08h7+/P0JCQsS+nz59GseOHcOMGTMKvId79+7Fw4cPsXfvXri4uAAAmjdvjvfeew/nz59HbGwszpw5g2XLlonn8fPzg7GxMVauXIlhw4ahSZMmAAADAwOEhoaKI6HaBHlvv/22zrr4GTNmFHuf84qPj8eoUaMwceJEsc3Ozg59+vTBxYsX0b17dzg6OgIAHB0dxf/O7cSJEyW+FpVKhc2bN4v9y8jIQEhICKKjo+Hm5gZDQ8Ni8zukpaWJ++dmZmYGAEhPTxf/O7c+ffrg119/RUhICKZNmwZzc3Ps378fJ0+eBJDzHFtbW4vbP3/+HPv27cPIkSNhaWlZZJ8Kk5WVhU8//RTvvfceAKB169bo1KkTNmzYgOXLlxe4j52dHVq1aoVff/1VfJ7j4uJw7do1cZ/SPLulsX//fty6dQs//fSTOGOhQ4cOCAoKwpIlS8SZIvXr10f9+vULPU5aWhoAFPo95U2GV9x+ub9b7XYFfce5j13Uc6J9CaTdtrjzERERlQeO4BNRhZk6dSpOnjyJpUuXol+/fjA3N8cvv/wiJtnLrXnz5jrTnps1a4Z69erh/PnzAIDIyEj4+PjA2NgYKpUKKpUK5ubmaNmyJc6cOQMgZ3rt999/DyMjI9y9exdHjx7F2rVrkZSUlC/rtjZgzsvb21v87+rVqwOAzrRqbfCWmpoKAHjvvfewceNGZGdn49atWzh8+DBWrVoFtVqdb5Q3b9Bpa2tb5BT9ixcvwt7eXqevJiYmOHz4MPr3749z585BX18f3bp109mvV69eAIBz586JbQ4ODjrTnLXy3oeS3Oe8Zs+ejZkzZyI1NRVXrlzB/v37ERERAQAlynau7WtJr8XR0VEnmKpduzYAiLM9SkIoplyfNuDNy8/PD4sXL8aZM2fQqVMntG7dGsePH8eYMWMAAMbGxjrb79y5ExqNBsOHDy9x3/LS19dHjx49xJ+NjY3RoUMH8XdDrVaL35VKpRKn1/fq1Qvnz5/H06dPAeSM3pubm4tT7Evz7JbG2bNnUbNmTbi6uop9UqvV6NixI27cuJFv2UBhClsmoCWRSF5oP+13W9QzoD12SbYp6fmIiIjKA0fwiahCWVlZoUePHmKAcvPmTXz00UdYvHgxevbsiWrVqgH4L0jLrXr16mIwIJPJcPDgQRw8eDDfdto1tBqNBsuWLUNERAQyMzNRp04deHh4wMjIKN8+pqamBfa3oNE6ExOTQq9PLpfj888/x/79+6FSqWBvbw9vb2/o6+vnCw7yHkcqlRYZQMhkMvElQ0FSUlJQrVq1fNOHtdPOtSOZAAocqQTy34eS3Oe84uLiMH/+fJw9exYGBgZwcHBA06ZNAZS87n1prqWg+wgUH2jlZm5uXuA0fO1oq4WFRaH79urVC927d0d8fDxMTExQu3ZtrFy5ElKpNN8o/eHDh+Hn51fovSuJGjVqQF9f9//Oq1evLi4H6Ny5s06+it69e2PhwoXo1q0bPv/8cxw6dEjM+t61a1fxJURpnt3SkMlkePr0KVxdXQv8/OnTp2L2+aLknqGRV3p6eqHfkbY97355R9qLega0fx+Zm5vj2bNn+bbJyMgQz5P7fLmvqyTPEhERUWkxwCeiVy4xMRF9+/bF1KlT862DbtasGaZNmyYmbtMG+MnJyfmO8+zZM3EKr4WFBdq2baszrV5LG/xs2LABW7duRWhoKLp06SL+w7pfv37len25ffnllzh8+DBWrFiBtm3bigFzmzZtynxsCwuLAmvNX7p0CVZWVrCyskJycjLUarVOYPzkyRMAEO9tac9Z3H3OTaPRYOzYsTAwMMCuXbvg4uICfX193L17F/v37y/xeV/GtRTFwcEB6enpSEpK0gm+Y2NjYWdnl28kXismJgbXr1/He++9p7Pm+ubNm3B2dtbpe2JiIm7evFmm0XsABa7rf/bsmfjyZ+3atTozJbT3ysLCAoGBgTh06BBat26NO3fu4JNPPhG3e5FnVyKRiLkOtPLOQrGwsEDDhg2xZMmSAo9hb29fxNX+x8zMDLVr10ZsbKxO+/Pnz5GRkYHGjRsXuF/9+vWhp6eXbz/tWnjtfo0aNcq3Pl6tViMhIQFdunQRtzl16hQ0Go3OSHxsbKzOcbRtuZcexcbGwsDAQGdmEhERUVlxXhgRvXLaEccffvgBCoUi3+f37t2DkZGRTq3xixcv6gT5N27cQEJCghhs+Pj44O7du3BxcYG7uzvc3d3h5uaGrVu34vfffxeP4ejoiL59+4rBfWJiIm7fvl2q0d3SuHjxInx9fdGpUycxQLpx4waSkpLKfM6WLVsiPj4ed+7cEdsUCgUmT56MXbt2wcfHByqVCr/99pvOfj///DMAoEWLFqU+Z0nuc27Jycm4f/8++vXrB3d3d/ElgDYLufYe5B2ZL+i85X0tRWnbti0A6JxPqVTi2LFj8PPzK3S/O3fuICQkBPfu3RPb7t69i1OnTqFTp0462169ehVAzvKTspDL5eIaf+3PJ06cEH83nJ2dxe/K3d1dJ4B+9913ceXKFfz444+oW7cufHx8xM9e5Nk1MzNDcnKyzu+1Niu9lo+PDx49eoTq1avr9Ov06dPYtGlTsc9Cbn5+fjh27JjOC4zDhw9DT08PrVu3LnAfIyMjtGzZEr///rvOTITDhw/DwsJCDML9/Pxw/vx5JCUliducOnUKmZmZ4jPQrl07ZGRk6Nz/pKQkXLhwQdzG29sbpqamOHz4sLiNIAj4/fff4ePjU+DSGCIiohfFEXwieuX09PTw6aefIjg4GH379sXQoUPRuHFjZGVl4fTp04iIiMDUqVN1prNmZWVh9OjRmDBhAjIyMrB8+XI4OTmJU/snTpyIQYMGYdy4cRg8eDCMjIywY8cOHDlyBKtWrQIAeHh4YM2aNdiwYQO8vLwQGxuL9evXQ6lUlmp9dml4eHjg0KFD+PHHH9G4cWPcunULa9euhUQiKfM5+/Tpg+3bt2PChAmYMmUKqlWrhm3btiE7OxtDhgyBvb09fH19MW/ePCQmJqJp06Y4d+4cNm7ciN69exeYzK44JbnPuVWvXh12dnaIiIiAra0tLC0tcfLkSTHHgvYeaF+4HDt2DFZWVuIUfq0OHTqU27UolUrcvHkTtra2sLW1LXAbOzs79O7dG19//TUUCgUaNmyILVu2IDU1VafKQFxcHJKSksT8Cf7+/qhfvz5mzpyJqVOnIj09HYsWLYK9vT1GjBihc47bt2/D0NCw0ERySUlJiIuLy5dToCBz5szBhx9+iOrVq2Pz5s3IzMzEhAkTir0X7du3h7W1NXbs2IHRo0frrFt/kWe3Y8eO2L59O+bOnYt+/frh9u3b2LJli07Q3qdPH3z//fcYOXIkxo8fjzp16uDMmTPYuHEj3n//fRgYGADIf28LMnr0aBw4cACjR4/GyJEj8eDBAyxbtgwDBgxA3bp1ART8fU+YMAEjR47E1KlT0bdvX1y+fBmbN28Wk0gCOYlAtf2cNGkSZDIZFi9ejA4dOogvZVq1agUfHx989NFH+Oijj2BtbY2wsDBYWFhg8ODBAHKWjHzwwQdYvXo1DAwM4O3tjd27d+Ovv/7Kl2uEiIiorBjgE1GFCAgIwE8//YTNmzdj3bp1SEpKgqGhIZo1a4bly5eLU2C1WrZsidatW2Pu3LkAgMDAQMyaNUsc/WratCkiIiKwfPlyzJo1C4IgwMnJCatXr8Zbb70FABg3bhySk5Oxbds2rF69GnXq1MG7774LiUSC9evXIzU19YUzmRdm9uzZyM7OxooVK6BUKmFvb48JEybg7t27+OOPP/JNZy4Nc3NzfP/991i0aBE+//xzaDQaeHl5Ydu2beK03/Xr12PVqlXYunUrkpKSYG9vj+nTpxc4xb4kSnKf81qzZg2+/PJLzJ49G4aGhnB0dMTatWvx1Vdf4cKFCwgKCkKTJk3Qo0cPRERE4OTJk/j11191jqH9jsrjWp48eYKBAwdi0qRJmDx5cqHbffbZZ7C0tMTGjRuRmZkJV1dXbNmyRWdmyZo1a7B37178/fffAHKCuU2bNuGrr77CjBkzYGRkhA4dOogZ9XN79uxZkc/bsWPHMGfOHGzbtg2+vr5FXtOnn36Kr776CklJSWjevDl+/PFHnX4WRl9fXywVqU1YqPUiz66fnx9CQkKwfft2HD58GK6urggPD8egQYPEbUxNTREREYGlS5di8eLFSEtLg52dHWbMmIEPPvhA3C7vvS1I48aN8e2332LRokXiS64RI0ZgypQp4jYFfd9t2rRBWFgYVq1aheDgYNSuXRuzZs3SOb+NjQ22bduGr776CjNnzoSZmRm6deuGWbNm6fQhPDwcCxcuxKJFi6DRaNC8eXOsWLFC5wVlcHAw9PT08NNPP+Hbb7+Fo6Mj1qxZU+4zT4iIiCRCWTLlEBG9AkFBQQCA7du3V3BPiF6tWbNmYciQIYWOYoeFhSE8PLzIIJiIiIjeHFyDT0REVAndvXsXV69ehZOTU0V3hYiIiF4TDPCJiIgqIRsbG2zdurXQko1EREREeXGKPhEREREREVEVwBF8IiIiIiIioiqAAT4RERERERFRFfDGlcm7fPkyBEEQ6+wSEREREdGbKTs7GxKJBN7e3hXdFaJy8caN4AuCgFeZdkAQBCiVyld6TqLC8HmkyoLPIlUmfB6psuCz+Oq96tiA6GV740bwtSP37u7ur+R8mZmZiI6OhqOjIzMhU4Xj80iVBZ9Fqkz4PFJlwWfx1bt+/XpFd4GoXL1xI/hEREREREREVREDfCIiIiIiIqIqgAE+ERERERERURXAAJ+IiIiIiIioCmCAT0RERERERFQFMMAnIiIiIiIiqgIY4BMRERERERFVAQzwiYiIiIiIiKoABvhEREREREREVQADfCIiIiIiIqIqgAE+ERERERERURXAAJ+IiIiIiIioCmCAT0RERERERFQFMMAnIiIiIiIiqgIY4BMRERERERFVAQzwiYiIiIiIiKoABvhEREREREREVQADfCIiIiIiIqIqgAE+ERERERERURXAAJ+IiIiIiIioCmCAT0RERERERFQFMMAnIiIiIiIiqgIY4BMRERERERFVAQzwiYiIiIiIiKoABvhEREREREREVQADfCIiIiIiIqIqgAE+ERERERERURXAAJ+IiIiIiIioCmCAT0RERERERFQFMMAnIiIiIiIiqgIY4BMRERERERFVAQzwiYiIiIiIiKoABvhEREREREREVQADfCIiIiIieqXUak1Fd4GoSmKAT0REREREr0xKugJKlbqiu0FUJTHAJyIiIiKiVyIlXYFMhaqiu0FUZTHAJyIiIiKily4lXYEMeTYEQSjxPqXZ9mWo6PO/Sm/StZbFq7xPL3IuBvhERERERPRSpaQr0LN7Vyz8YkGJ9zl69ChCQkJ02g4cOICOHTvCzc0N8+fPR1BQEIKCgvLtW9Yg7PHjxxg7diwePnyY77MJEyZg7dq1JT5WbGwsnJ2d8/3p0aPHC/WtuGvbs2cPnJ2dkZCQUKLjpaamYtasWbhw4cIL9ScvZ2dnhIWFFfp5QkJCgfdD+2fOnDkAgLCwMDg7O79QHwp7LsqiqGfiZbh48SLGjh1b6v30X0JfiIiIiIiIAACydAUy5dn47KslMDUzK3CbmJgY/PDDDzh16hQeP34Mff2cMKV69epQqVTiz5999hkaNmyIhQsXonbt2vjkk0/w9OlTODs74+jRo7C3t8edO3fwySef4H//+594/MDAQPj4+GDhwoWF9nP27NnYu3cvateujalTp+L48eP5tvnmm2/wxx9/IDExERMmTND5bMKECfDw8NBp/+2337B+/XoAgJ6eHiwsLNC0aVP07NkTbm5uxd47bZC6fft2AMDOnTsRExOD2bNnAwCioqIwbNgwbNu2Db6+vgCAgIAA7NixA7Vq1RKPow2Ux40bh+nTp+ucIzo6Gvv378fx48chk8nw9ddfo0+fPgX2JyEhAW+99Vah/dXuFx0dnS84NzIygp2dHbp3744ff/wRUqnuWHNERAQOHTqEvn37Fn1TSmDBgpK/SCqpM2fOFPhMvCza77q0GOATEREREVG5EwQBKRlKZGZlQwDQxKlpgdsdPHgQc+bMQePGjTFy5Eg0atQIcrkc8+bNQ1xcHCZPnow1a9ZAIpFAJpPBz89PDGYBwNraWieg/e2333D58uUX6rNUKkViYiJiY2ML/Hz//v0AAFNTU512pVKJyMhITJ48GQCgUqkwY8YM/P7773B0dIS1tTXCw8Pxzz//4KeffsL8+fOxZMkSNG1a8D3Ryhuorl27Fj4+PkXuY2NjAxsbmwKv7bfffssX4GvJZLIijwsAtWrVwo4dO/K15w7O9+zZI7aHh4ejZs2aEAQBWVlZuHTpEtasWQOVSoUPP/xQ3O7GjRs4dOgQpk2bhpYtWxbbj+I4OjqW+RivK07RJyIiIiKiciUIQs7I/b/BPQAM7tcD33y5AI8f/QMvDzccOnQIH3zwAaZNmwaVSoWmTZuiV69e8PX1xaZNm/DkyRMIgoA//vhDZ7r26tWrdaagGxgYwMvLC4aGhlAoFDh//jwAwN3dHV26dMGGDRt0prWr1Wps2LABPXr0wJAhQzB06FAMGjQIT58+RZ06dWBtbS2Our/11lviaPmVK1eQlJQES0tLSCQSADlB8fz58+Hn54fMzEx8+umnOHv2LNatW4fffvsNy5cvR506deDt7Y1WrVrh3Xffxfbt2+Hv74/Q0FDI5fJC7+GePXvQu3dvyGQy9O3bF+7u7khMTMSjR49w7949DB8+HKNHjwYAREZG6uyX+/4kJSUByAnwY2Nj0bVrV+zbtw/AfzMA8goKCsLMmTMxZcoUeHl5YeTIkQCAJ0+e4IcffsCkSZMwdOhQTJgwAWvWrMHBgwcLDM5dXFzg5eUFb29vtG3bFpMmTULPnj11ZlcIgoDPPvsMjRs3xogRI/L15ciRI+jatSvc3d3Rv39/nD17ttB7lrv/uafoOzs7IyIiAnPnzoWPjw+8vb0xdepUPHv2TNwmLi4O48ePh6+vLzw9PTFw4EBxxH7Pnj3i0oHcz0RgYCC++uorDB8+HB4eHpg7d26hSyQCAwPF/YCcl0IrVqzAW2+9BQ8PD/To0QN79+4F8N9skocPH8LZ2VnnpUlxGOATEREREVG50Qb3WXIVilotvmDBAjx58gRGRkYYPXo09uzZI65tX7BgAZo1a4ZmzZqhQ4cOsLe3F0eOmzVrhkaNGuHtt9/Gw4cPxSn68fHx6N69O6KiogDkBFA1atTAihUrdEanlyxZgjVr1mDgwIGYO3cuxo8fD5lMhqioKGg0Grz99tsw+3cpQXh4OCZOnAggZ6aBsbGxOFNAoVBg+PDhOHr0KNzc3NC2bVvY2tpi9OjR2LBhA7p164auXbsiOjoaGRkZGDRoENzd3dG+fXtUq1YNrVq1wvPnzwHkBJDNmjXDzp074efnBx8fHyQmJkKpVGLkyJEYNGgQLCwsoFKpEBkZibfffhteXl6YNm0aAGDDhg14/Phxgff5o48+AgB06tQJlpaWMDQ0REhICCIjI+Hq6op58+YBQL6p94cOHYKZmRnWrl2L0aNHIysrC8OGDUNMTAwWLFiAzZs3Y9iwYTh+/DjMzc0LDM4LkvsFifa+Xr16FR9//DH09PTybT937lwMGzYMYWFhMDMzw5gxY3D9+vUSnSu35cuXQ6PRYNmyZZg1axb+/PNPfPXVVwAAjUaDcePGISsrC4sWLcKaNWtgbW2NCRMmIDY2FgEBAeLSi9zPBJAze8Hd3R1r1qxBv379StyfmTNnYsuWLejfvz/Wr1+Pdu3aYfbs2fj1118xceJE+Pv7o2bNmtixYwcCAgJKfFxO0SciIiIionJR0uAeAPz9/XH8+HH4+flh2rRpuHz5Mo4dO4YZM2bA0dER5ubmAICNGzfq7Hfr1i3MmjULjRo1QlhYGDIyMgAA586dQ3x8PHx9fREVFYUdO3bA1tYWe/bswcqVK5GamgogZxR62rRpCAoKEgNFR0dHTJ48GSqVCn379sWPP/4IIGcE2t7eHhqNBgcOHIBCoYClpSWAnOn6t27dwk8//YSQkBB8+OGH6Nq1K3r06IG7d++iR48eSEpKQmJiItRqNT766CPUrVsXZ8+excaNG9G1a1fY2dmJ16VWq/Htt9/iyy+/RHJyMjQaDQCgbt266N+/P1xdXTFs2DCkpaWhb9++CA4OFpciqFQq3LhxA7a2tvnu87lz58RrrF69Ok6ePIkPPvgAhoaGMDc3F2cRdO7cGUePHhX3MzAwQGhoKAwNDQHkrKu3tbXFN998g3r16gGA+ILCxMSkwOBco9FApcopiyiXy3Hp0iXs379fZ9bA5s2b0bx5c51lF7mFhoaiW7duAIA2bdrgrbfewsaNG7Fq1aoCty+Mk5MTvv76a/Hna9eu4bfffhOv4969e2JgDQAeHh4IDw+HUqmEjY0N6tevD+C/Z0Krbt26mDlzpvjz/fv3i+3L7du3cfjwYXz88ccYPny4eG0PHz5EVFQUevToARsbGxgaGsLLy6tU18kAn4iIiIiIykwQBCSnKSBX/BfcazQC4hPTkJ6ZDZVag9wJ4Js2bYqff/4ZDRs2BADY2tqKGcpVKpU4rV6lUkEikYgBZN26dcUp45s3bxYD/KtXr0JfXx8tWrRAVFSUGBj16tULK1euFKdjL126FEDO1PXo6Gg8evQId+7cEa/B3d0dNjY24tR2ALhw4QJkMhm8vLzEhH9nz55FzZo1YWVlhbi4OPj6+kKtVqN+/fq4e/cuatSoAVNTU3z77bdo0KAB6tatC41Gg+bNm0NfXx+rVq3C+PHj0bhxY6jVagDAmDFj0K5dO+jr64vTsrUvOpo1awYjIyOkpaWhS5cuYuCtpX2BkZevry9OnjyJQ4cOITAwEHFxcejZsyeaNWsGAOKMBwMDA539HBwcdM7h4uKCH374ARqNBg8ePEBsbCy++eYbGBkZ5UuYp9W5c+d8be7u7mJQe+nSJfz1119YvXp1gfsbGBigS5cu4s9GRkbo0KED/vzzTwA5L0VyL7+QSqWF9iVvoGxra4usrCwAQI0aNeDo6IhPPvkEp06dQrt27dChQwdxWn5RXFxcit0mr4sXLwKAzrUBKLL6QEkxwCciIiIiojLRaP4duVeoxLZbD5LwW+QDJD7PhFqjQVpmNqIfPMe9hykAkC9AlUqlEAQBsbGxOoGPq6sr7Ozs8McffwDICcYKkpaWhmrVquUL8GrWrAkAyM7OBgBcv34doaGhuH79OoyMjFCvXj00btxYZx8PDw8cO3ZMDB4PHDgAa2trdOzYESdPngSQs/7+6dOn6Nq1KwCgdevWOsdISkqCsbEx/Pz8AORMEV+3bp3ONqtWrYKTkxPCw8MBAHPmzMGcOXPw999/i9sUNDJuYmJS4D0oyPLly9GyZUs8efJEnA0xefJkbNu2DTVr1hSDzbxMTU3F0Xcg5/v57rvvsG7dOshkMlhZWSElJUVnFkJeeV+UeHp64vHjxxg0aBB27NiBw4cPw8rKShw1z6ug77N69eriy4zOnTvrlK3r3bt3oZUS8t4z7fMGABKJBN9++y3Wrl2L33//Hfv27YOBgQE6deqE0NBQWFlZFXqNeRMuloR2yUj16tVLvW9xGOATEREREdEL02j+HblX6gb3EYdvQa5QwczEAPp6+pAAyJSr8POJnNJf6QoJTE1N89UVr1OnDnbt2oX58+cDAGrXro3bt2+Ln+cdadaysLDQmdqu9eTJEwA5LxTS09MxevRoODs748CBA8jIyIBUKkVSUhIOHz4s7qMN8P/66y/UrVsXhw8fRnp6Ovz9/cUA38LCAg0bNoSNjQ1cXFzQu3dvADkj+0uXLoVarcaDBw8QGRmJd955B0OGDEGnTp0AAH///Tfmzp0LExMTDBgwAIIgYPXq1VizZo1OebvyYGFhAQAYPnw4unfvjnnz5uHSpUsIDQ3FwIEDddbD53b16lW4urqKP7ds2RIXLlzARx99hD59+mD9+vXYu3cv3NzccOPGjQKP8c0336BatWriz9WqVUNiYiKGDBmCnTt34tixY3jrrbcK/U7T0tIgCIJOH589eyZWCVi7di2USqXO8V9U7dq18emnn2LBggW4desWfvvtN2zcuBHVqlUrVdk9bV/zPofamSYAxGUeSUlJOssqYmJiIJPJ0KJFixe+DibZIyIiIiKiF5IT3Mt1gnuNRsBvkQ8gV6hgbW4IQ30ppP8GPWqNgNSMnJH0X0/eQ4167jhx4iTS09PF/Q0NDeHu7g5LS0uYm5vD2tq6RH3x9PSESqXC3bt3ddp//vlnADmjpffu3YNMJsOwYcPg6Ogojg6fOHFCZx/tqPTx48cRGRmJzMxM2NjY6NR29/HxwaNHj/DXX39h4MCBcHd3h7u7O7KysiCVSnHkyBE8ffoUCxYswG+//YbatWuL22hH6GvVqoXatWuLa7qdnZ3h7u5eoustiYcPH+qMjjs4OGDWrFkQBAH379/HwYMHCy275+TkhF27dol/6tatC0tLS4wePRo2NjY4duwY/P39cenSpXzBbO7zaa/Z3d0d9vb24vX9/fffePDgAZo3b15o/7OysnQqBGRkZODYsWPien3t/cp9/Bdx+fJltG3bFteuXYNEIoGLiwumTZsGJycn/PPPPwBQ6NT/vLRLKnInPdQG7lraAF47K0VryZIl+PLLL0t1vrw4gk9ERERERKWm1giQpckhV6p12uMT05D4PBNmJgbiaKZcqYZGI0CjESD5N26RSgFLhwAk3L2ESR9+hJrVzHWOY2lpicuXL0MmkxUaQObm4+MDX19fcX32mTNncO7cOWzcuBGmpqawtLREo0aNYG5ujnXr1kFfXx/x8fGIjIwUAy3tebQjrL///jvS09NhZ2eXrwRcnz59sHHjRjx9+hQ3b95EcnIyzpw5g40bN8LV1RU///wz3nrrLbRp0wYLFy6EXC6Ho6Mjjh07hu3btwPIWVNeGoWNthfGzs4Otra2ePz4MW7evIlz586Jo+02Njb4448/8PHHH+PPP/8U1+JrmZqa6rxsaNOmDX7++WcsXLgQrVq1woMHD6BQKPDs2bMip7Dnde3aNQD/TZkvqma9gYEBPv74Y0yfPh3m5ubYsGED5HK5Thb78tCsWTMYGxtj1qxZmDx5MmrUqIEzZ84gOjpaTAiY+5no0KFDvmUdWr6+vjA2NsbChQsxdepUZGRkYNWqVTovqpo2bYpu3bph8eLFkMvlcHFxwYkTJ/Dnn3+KyzUsLS3x7NkzHD9+HC4uLiWe2cEAn4iIiIiISkWtESBLlUOerc73WXpmNtQaDfT1ckINAUBKuhICAKkk5w+QE6zWq98I8jbv43zUjzAxMoBUKsXZs2ehUqlgYmKC58+f4+nTpwgMDCy2TxKJBOvXr8f48eMRGRmJ0aNHo27dupg+fTq+//573L17F7t370avXr3wxx9/IDg4GPr6+qhXrx6+//57DBs2TJzu7evrCy8vL1y5cgW///47atasmW+duKmpKdq1a4eLFy9i8eLFSEtLg52dHWbMmIHhw4dj3rx5mDp1Kt566y20bNkSGzZsQHJyMkxMTCCVSlGjRg20adOmVPfdzMwMT58+xa1bt+Dp6anz2alTp5CamioG0Lt27cKYMWMQHh6Odu3aISoqCidPnkSdOnXQokULXL16FdWqVUPv3r0RGRmJffv2FXnu3r17IyEhAbt378b3338PIGfWxMSJE/HJJ58gJiYmX9AbHR0tJjfUaDSIiYlBWFgYatasKU7/1wbOBbGxscGMGTOwbNkyPH36FJ6envj+++/h4OBQqvtWHCMjI3z77bdYunQpvvzyS6SmpqJhw4b47LPP0KdPHwA5z0Tbtm2xdOlSnD17Fhs2bCjwWJaWlggLC8PSpUsRHBwMOzs7TJo0Kd/9Xbx4McLDw/Hdd98hOTkZjRs3xqpVq8RlHH369MHx48cRHByMKVOmYOzYsSW6FomQO+3gG0BbCqM8p74UJTMzE9HR0XBxcXmhBAxE5YnPI1UWfBapMuHzSJXF6/IsFhXcA0Dso1Ss23MNRoZ6MNSXQpGtwTNZJiQSSU52fQEwMtSDjYURjI30oVCqIUt6ikZGMbh66SwePnwIQRBQr149+Pn5YdCgQWKmfWdnZ0yaNAmTJ08Wz7dnzx7MmTMHR48ehb29PRITExEcHIxbt26hX79++PTTTxEYGJhvrb/WoEGDEBoaitmzZ+PcuXM606Z79uyJp0+f4tSpU2L2/KCgIADA9u3b0aVLF8yaNUsMyvI6efIkdu7ciWvXruHZs2cwMzODi4sLunbtivfee08cxc57DVq5zwUAv/76K7766iukpaVhy5YtUKvVOiXn8jp+/DhsbW3z3bcbN26gb9++GDZsGObOnQsASEhIwFtvvYWvv/5aDGoLc/DgQUybNg0HDx4scCRbez256evro1q1avD19cXUqVPFsnNUvhjgv2Svy1/U9Gbg80iVBZ9Fqkz4PFJl8To8i2qNgORUORSFBPdAzhr8VT9dxsOn6bA2M/w3wM+CRAKoc820t7UxgbmpoZik78NBzdG8afkmmCvOq44NiF42JtkjIiIiIqJiqdWaYoN7AJBKJejWuiGMDfUhy1BCpdFAEHSDe4kkZxQfALJVGujrSWFpZljIEYmopBjgExERERFRkdRqDZLTFMUG91pNG9pgaNemqFvDDBlZKuSeMiyRAE0bVIOhvh4EQUBaVjbsapnDwa7kidqIqGAM8ImIiIiIqFBqtQZJJRi5z8uxnjWszIygyJVlXyIBbG1MYW5iAIVSjeepCpga6aNfYBNIpaXLEE9E+TGLPhERERERFUj177R8par4MnW5ZSlUWL/3Om49SBLbalgZw8rCCJlZ2UjLVEKuVKNhHUv0C2wCzyY1y7vrRG8kBvhERERERJRPtkoDWVrpg/uUdAXCfrqChCfpYptz/WoY38cDRoZ6ePg0HUb6erAwM4SDnRVH7onKEQN8IiIiIiLSocxWQ5amQLa6dMH9o2cZCPvpCpJS5WJbS5faGN69GQz0c1YH17e1QDULI5gYGZRrn4mIAT4REREREeWiUKqQnK6AWl26atp3E2RYs+sqMuUqsa2zT3307ugIqYSj9ESvAgN8IiIiIiICAMiVKsjSFFBrShfcX/77Cb795S9k/zudXwKg31tN8Far+i+hl0RUGAb4RERERESELEU2UtKVpQ7u/7wYj59+vy2WwtPXk2Jkz2Zo0bR2+XeSiIrEAJ+IiIiI6A2XIc9GaroSGqHkwb1GELD/eAwOR8aKbaZG+pjQ1wNN6ld7Gd0komIwwCciIiIieoOlZymRlqFEaQbuVWoNth2Mxrm/Hott1SyNMLm/F+rWNH8JvSSikmCAT0RERET0hkrLUCItS4lSDNzn1Ljfcw23YpPFNrua5pg0wBPVLIxfQi+JqKQY4BMRERERvWEEQUBqhhIZ8uxSBfeyNAXCdxZc497EmKEFUUXjbyERERER0RtEEATI0hXIkqtQmnR6BdW4b9WsNoa981+NeyKqWAzwiYiIiIjeEBqNgOQ0BRTK0gX3rHFP9HpggE9ERERE9AZQawTIUuWQZ6tLtR9r3BO9PhjgExERERFVcSq1Bsmpcij/DdJLijXuiV4vDPCJiIiIiKqwbJUasjRFqYL7Qmvc9/NAk3qscU9UWTHAJyIiIiKqopTZaiSnKaBSlzy4L7TG/QAv1K3BGvdElRkDfCIiIiKiKkihVCE5TQG1puTp9Fjjnuj1xgCfiIiIiKiKyVJkIyVdWargvsAa9w2qYXzv8q1xL5VKoK/HsnpELwMDfCIiIiKiKiRDno3UdCU0QsmD+8Jq3A/v3qxcg3FDfSmsLYxgoK9Xbsckov8wwCciIiIiqiLSM5VIy1SiFAP3uBsvw5rdeWrc+9ZH74Dyq3EvAWBkqAdrC2PoScvnmESUHwN8IiIiIqIqIDVdiXS5EqUYuMelWzk17rVJ+CQA+ndyQmDLeuXWL4kEMDU2gJWZISTl9MKAiArGAJ+IiIiI6DUmCAJSMpTIlGeXKrj/80I8fjqiW+P+g56uaN60Vrn1TSoBLEwNYW5qWG7HJKLCMcAnIiIiInpNaTQCZOkKyBUqlDS21wgC9h67i9+j4sS2l1HjXk8qgZW5IUyMDMrtmERUNAb4RERERESvIbVGgCxNDrlSXeJ9VGoNvjtwE+dvJoptL6PGvYGeFFYWRjAyYDI9oleJAT4RERER0WtGpdZAlqaAIrvkwX2WXIX1e/PXuJ88wAvWFkbl1recTPnGMNBnKTyiV40BPhERERHRayRbpYYsTQGlSlPifZLT5Aj/6SoePv2vxn3TBtUwrhxr3DNTPlHFY4BPRERERPSaUGSrIUuTQ6UueTa9f56lI+ynK0hOVYhtPq62GPaOS7nVuJdIABMjfVibGzFTPlEFYoBPRERERPQaUChVSE5TQF2KIvd34pOxdtc1ZCr+q3HfxbcB3gtoXH417iWAubEhLM2ZKZ+oojHAJyIiIiKqAIIglHi0W65UQVbK4P7irURs+eWmTo37AZ2c0PHfGvelOX9hpBIJLM0MYGZS9uC+PPpDlNurfqYqwzPMzBdERERERGUUGBiI2bNnl3j7o0ePIiQkRKftt99+w5QpU+Dj44P58+cjKCgIQUFByFJkIzm1dMH9nxfisWnfDTG419eTYsx77ujYsh6ePknEnI+mIPHxo3z7zZs9Hd9/t7lE59CTSmBtYVjq4D4hIQHOzs7Ys2cPACA1NRWzZs3ChQsXxG20117Q9nv27IGzszMSEhJKdd5XKXf/y2rnzp1wdnbO9+ezzz57ZX14mfJ+v+WloN+xl2nNmjXYvLlkvzsvE0fwiYiIiIjKKDw8HObmJS8zt3Xr1nxt33zzDWrUqIGvv/4a9evXxyeffAK1RgNZmhIaoWTBfYE17o31MbGvJxzrWQMALl6IQtTZ0/n2zc7OxqWL5zD8g7HFnqc8y+BFR0dj//796Nu3r9i2YMGCQrcPCAjAjh07UKtWrTKf+3UQHR2NRo0aYeHChTrtNWrUqKAela9atWphx44dqF+/frket6DfsZdp5cqVmDRp0is9Z0EY4BMRERERlVGzZs3KfAyZTIbAwEC0bNkSpqamUKs1UGmEEgf3BdW4t7E0xuQBXqhTw6zY/a9duQQzUzM4NnEucrtXUQbP0dGx0M9sbGxgY2Pz0s5d2URHR8Pd3R1eXl4V3ZWXwtDQsMpeW0XgFH0iIiIiojLSTtHXTjc+dOgQpkyZAm9vb/j4+GDevHnIzMwEkDN1+ty5czh37hycnZ0RFRUFZ+ecoHrPnj3w9vbG33fuQ6URgFyxvVKhwPatGzF8SB90DWyDoEHv4cfvt0Kj0SBLrkL4T1dw7q9HSLr7Jx4cX4q7h+bi8u5Z+HLeJFy+dB4A8NvBn7Hoq1AAwJD+PfHNl/+NlEdFnkYr37aQSCS4cukCAtu1wKUL5zB98lh0C2yLQX3ewf8d3A+1Ig3TPsy5Nn9/f52R0sKmzxe2hCEqKgrDhg0DAAwbNkycUl7U9PK855g9ezZGjBiB3bt3o2vXrnBzc8O7776LEydO6Ox3+fJlDB06FF5eXggICMB3332H0NBQhIeHi33Rfh+55e2LXC7H0qVL0aVLF7i5uaF58+YYOXIkoqOjC+xvUW7fvo1x48ahefPmaN68OYKDgxEfHy9+LggC/v77b7i4uJT62HklJSUhNDQUHTt2hJubG3x8fBAcHKzzXQUFBWHmzJmYMmUKvLy8MHLkSADAkydPMG3aNPj4+KBVq1aYP38+li9fjsDAQJ1z7Ny5E927d4ebmxsCAgIQFhYGtVpdZL8KWoLRrFkzXL16FQMHDoS7uzs6duyYb/r7r7/+il69esHDwwOtW7fGzJkzkZiYKF5H3t8x7ff7v//9Dx07dkTz5s1x+vTpAp+1gp6Fe/fuYdKkSeI9GDduHGJiYgBA/P0NDw8X/7uiMMAnIiIiIipnCxYsgJ2dHdasWYNRo0Zh165dWLt2rfhZs2bN0KxZM+zYsQOurq7YsWMHgJxAeN2Gb2FkZqUT3AuCgLkh0/C/iG14p8d7+PKb5fDv2AmbN67BN199jiURF3ErNhnPog/i+Z0jcPLqiK8WrcLMkHlIS01B6LwQyOVZaN2mPd4fPgoAEPrlYrw/YrR4jqizp+Dbxk/nOr749GO08euArxavQIOGjbD4my8xYsRwNGnSBGvWrIGHhwe+/vprXLt27YXuk6urK+bPnw8AmD9/fpFT84ty48YNbN68GVOmTMHq1auhp6eHyZMnIyUlBQAQExODESNGAACWLVuGyZMnY8OGDbh161apzzVr1izs3r0bY8eOxbfffos5c+bgzp07mDFjBoQSzrYAgPv372PQoEF4/vw5vvnmG3z55ZeIj4/H4MGD8fz5cwBAXFwcMjIycP36dXTt2hWurq7o2rUr9u3bV6o+C4KAcePG4fTp05g5cyY2b96MSZMm4ezZs/nu+aFDh2BmZoa1a9di9OjRUCqVGD58OC5duoSPP/4YX3/9NW7duoVvv/1WZ7/169fjk08+QZs2bbBu3ToMHToUGzduxCeffFKqvgKARqPBhx9+iHfeeQcbNmxA8+bNsWjRIpw8eRIAcPHiRcyaNQtdunTBxo0bMWfOHERGRmLGjBkACv4d0woPD0dISAjmz58Pb2/vEvUnMTERAwcOxIMHD/Dpp59i8eLFePbsGYYPHw6ZTCb+/vbr10/874rCKfpEREREROXM399fTPDVpk0bnD59GseOHcOMGTPg6OgortfXTk3W/q+ldXXUd3CGgYFu4rpzkWdw8UIU5n36FQI7dQUAtGzVGkq1FLt+2IwG/o4wsrCFSp6KFgED8NX8D8Ua9wZGRvh07ke4d/cumrm5o66dPQCgiVNT2NapCwB49M9D/PMwAS1a+eqct1v3Xhgw+H2YGxuifp3qGDBgADw8PDB16lQAQNOmTfF///d/uHTpEjw8PEp9n8zNzcXp+I6OjkVOzS9KWloa9uzZI67jNjU1xfvvv4/IyEh07doV69evh4WFBTZt2gQTExMAgIODAwYNGlSq8yiVSmRkZGDevHl45513AAA+Pj5IT0/HwoUL8ezZM9SsWbNExwoPD4eJiQm2bt0qPg9t2rRBp06dsGnTJoSEhIizAhISEjB79mzo6+tj3759CAkJgVKpxIABA0p0ridPnsDExAQhISFo2bIlAMDX1xdxcXH5AlIDAwOEhobC0DDnGdy1axfu3buH3bt3w83NDQDQunVrdOrUSdwnLS0Na9aswcCBAzFv3jwAQLt27WBtbY158+Zh5MiRaNKkSYn6CuS8kJg4cSL69+8PAGjRogV+//13HDt2DO3bt8fFixdhbGyMsWPHiv20trbG9evXIQhCgb9jWkOGDEG3bt1K3BcgZz2/UqnEli1bxO+3adOmGDx4MK5evQp/f38AgK2tbYUvN2CAT0RERET0AjQaAfcepiA1Q4lslUZn9DbvP/JtbW3x8OHDYo+pzFYVOAp89fJF6Onpwb/jf0HVnfhk/JVUBwCQ9fwejCxsMXzCHLwX0BipMhni4x4gISEeZ0/nTFXPzlYWet6os6fh6u4Jc3MLnXZ3dy9YmeVkyq9evToAwNPTU/y8WrVqAHICvIpkY2Ojk6TN1tYWAJCVlQUAiIyMRIcOHcTgHgC8vb1LnajP0NBQnCqemJiI+/fv48GDB/jzzz8B5LwAyEuj0UCj0Yg/SyQS6OnpITIyEj4+PjA2NoZKpQKQ88KjZcuWOHPmDACgVatWWLduHXx9fWFqagoAaN++PZKSkrBq1Sr0798fGo3us6c9fm61a9fGtm3bIAgCEhISEBsbi3v37uHSpUv5+uzg4CAGzdp7V69ePTG41/azY8eO4hT2y5cvQy6XIzAwULwWAOIU/tOnT8PR0THfdP28/cwt9+i6oaEhbGxsxGUurVq1wvLly9GjRw907doV/v7+aNeunRhoF+VFljtcvHgRXl5eOi9vbG1txe+9MmGAT0RERERUSlfvPMWuP+7g4ZN0qNQapKQrcOX2U0Q/SAIAnUASAKRSaaHTtwVBQEpGTpClKaQUXmpaCqysrcWASFvjPluScx5NthwDOzmhrnkagscOx9/Rf8HY2BgNGjVG7dq24nkKExV5Cr6t/fK1165lna8MXt5rqwzy9klbi1wbWCclJYkvKHKzsrIq9blOnjyJr776Cvfu3YOZmRmaNm0qBt8F3eOPP/4Ye/fuFX+2s7PDH3/8AZlMhoMHD+LgwYP59tEmEaxevTo6duyY73N/f3+cOXMGz549w/Tp03Hu3DnxMx8fH2zfvj3fPj///DOWLVuGR48ewdraGi4uLjA2Ns63nZmZbkLG5OTkAu9d7jaZTAYAGDu24AoMT548wblz58R8C1rbtm2DnZ1dgfvk7Vvu3yFvb29s2LABW7duxZYtW7BhwwbUqFED48ePL7Y0oPa7Kg2ZTAZ7e/tS71cRGOATEREREZXC1TtPsXrXVWTJVbAwM4CBngEkEiA9KxvbD5Yu0ZogCJClK5AlVxW5naWFFVJkMqjVahy//A92HrkNAYBakTNyHuDTBD4u1TB0wHA4NG6Cb7fvRP0GDSGVShF59hROHDta6LGVCgWuXLqAMeMni23a6f2G+qUrg5c3sNbKyMgo1XHKm62tLZ49e5avPSUlRQwwi+q7NuiNi4tDcHAwOnXqhPXr16NevXqQSCSIiIgQ14fnNWnSJAwdOlT8WTs6bmFhgbZt24qJ7HLT188J0y5cuID4+Hj07t1b53OFQgE9PT1YWVkhNDRU5/7mDdC1xwkJCUFQUBBGjRqF2rVrAwAWLVqEixcvFthvrdq1a+PBgwf52rV5AgDA0tISALBkyRI0bNgw37Y1atSAhYUFdu3apdPeqFEj8eVAabVv3x7t27dHVlYWIiMjsW3bNnzxxRfw9PQs9XKRvDMLtDMFtCwsLJCUlJRvv7Nnz8Le3h716tUr/QW8JEyyR0RERERUQhqNgF1/3EGWXIXqVjl14KVSCSSQwMhACrkyJ1AvLtmadjRSG9wXl5rNw7s51Go1Fod/j5/+De4BIOvxFQDAO106IC72AVJTUtC3/2A0bOQAqTTnn/rnInOme2sEzb/n1g3ar1y+CEsrazg0zlkjbWSgBwsz3VH7ktKue378+LHYFhMTU2QQV9Q07fLSqlUrnDx5EgqFQmy7efMmnjx5Iv5cUN9TUlLETOlATjI/hUKBsWPHon79+uJLAW1wX9D3bm9vD3d3d/GPNsu6j48P7t69CxcXF/EzNzc3bN26Fb///juAnOnxs2fPxv3798XjaTQaHD58GN7e3jA0NISDg4PO8R0cHPL14fLly9BoNJg8ebIY3KvVanEpQN6XGrn5+PggISFBp0qAXC7XeaHh6ekJAwMDJCYm6vRFX18fy5YtQ0JCAszNzXU+c3d3F+95aX3zzTfo27cvBEGAiYkJOnbsKOa8+OeffwBAfP6LY25urvOdA8j30qNly5a4evWqTpD//PlzjB49GsePHy/V+V42juATEREREZXQvYcpePgkHRZmBmJw9x8JzIwNAABPkrOKPI6FhQUuX76C06fPwLFJU1j8OwJamBat2qB2fRf8vmcdqjt1hpFlXSA9Dk//PoKub/dAw0YOSE9Pg5mZGb7f9i309PSgp6+PE38exaED+wEA8iw5AIjr7E8e/wO+bfwQFXkKPr5tIQFgbKQPa3Mj6EnzXlvJ+Pr6wtjYGAsXLsTUqVORkZGBVatWwdraush7AQDHjh2DlZUVmjZt+kLnLsr48eNx8OBBjB49Gh988AFSU1OxcuVKSKVS8Xt0dnZGnTp1sHr1apibm0MikWD9+vU60/9dXV2hr6+PxYsX44MPPoBSqcSePXtw7NgxAPlHfosyceJEDBo0COPGjcPgwYNhZGSEHTt24MiRI1i1ahUAYNCgQfjf//6H8ePHY+rUqTAxMcEPP/yA27dvIyIiosTn0o5of/bZZ+jbty9SUlIQEREhVhHIzMwsNNju0aMHNmzYgODgYEydOhWWlpbYsmULnj9/jrp1c5I0VqtWDaNHj8bKlSuRnp4OX19fJCYmYuXKlZBIJOX+nbZu3RpbtmzB7Nmz0atXL2RnZ2PTpk2wtrZG69atAeTMKrh8+TLOnj2LZs2aFXqsjh074o8//sDXX3+NwMBAXLhwIV+VghEjRmDfvn0YPXo0xo0bBwMDA6xduxa2trbo2bOneL5Lly7h/PnzaNmyZQF/P7waleM1AxERERHRayA1QwmVWgMDvYL/Ga2vn/OPermi8Cn3Go2AXr0HQKqnh9kzp+Bc5Okiz5klVyF851VYuA6FdYPWkN0/hX/ObwFSojF63CR8NCenzJm5uQU+/3oZIAgI/SQECz+fjyeJj7EifCNMTc1w/dplAIB385Zo0dIXm9aHY234cpyLPIPWbf1gZmKAahZGkL5gcA/kBDna2ufBwcFYuXIlgoODdRK05dWkSRP06NEDERERmDlz5gufuygNGjTA5s2boVAoMGXKFCxfvhxjxoyBtbW1uNZbT08Pq1atQo0aNTB9+nR8+eWX6N69O7p06aJznKVLlyIxMRETJkwQS/xt374dEokEFy5cKHGfmjZtioiICEgkEsyaNQtTpkzB06dPsXr1avGcNWrUQEREBJydnfHFF1/gww8/RFZWFrZu3aqT7LA4vr6+mD9/Pi5fvowxY8Zg4cKFqFu3LsLDwwHkH7HOTV9fH5s3b0azZs3w6aefYtasWWjSpAk6d+6ss579ww8/xOzZs/H7779jzJgxWLx4MVq0aIHvv/9efIlTXvz9/bFkyRLcuXMHkyZNwvTp02FiYoJt27aJL5OGDh0KAwMDjBkzBidOnCj0WH379sWYMWPw66+/YuzYsbh8+bL4gkWrTp06+OGHH1CrVi3Mnj0bc+bMQZ06dfDdd9+JeRzGjx+PGzduYMyYMXj06FG5Xm9pSITSFGusAq5fvw4AcHd3fyXny8zMRHR0NFxcXF4ooQNReeLzSJUFn0WqTPg8UmncjZfhq63nYGykByOD/FPLFUo15Eo1Ph7hA8d61vk+V2sEyNLkkCvV+T7LyspCTEwMGjduLI4aJ6fJEf7TVTx8mi5u17RBNYzr7QET4/KZjCuVSGBhZgBzkxeblv86OHv2LAwMDMQScQCQmpqKNm3aICgoCLNnz67A3lVud+7cwb1799ClSxedUel+/frB1tZWfElAlQOn6BMRERERlZCDnRXsapnjwaNUGFpKdQIeQRCQlpWNhnUs4WCXPzu7Wq1BcpoCiuz8wX1B/nmWjrCfriA59b914z6uthj2jouYBK+s9KQSWJkbwsTIoFyOV1n99ddfWLVqFaZPnw5XV1fIZDJs2bIFpqamaNeuXUV3r1LLzMzE1KlTMWTIEHTu3BlqtRoHDx7EjRs3XtqMC3pxDPCJiIiIiEpIKpWgX2ATrN51Fc9TFbAwMYCBvhTZKg3SsrJhaqSPfoFN8k1zV6k1SE6VQ6kqPJlZbnfik7F21zVk5prq38W3Ad4LaAxpOa3tNdSXwsrcCIYFzESoarTr5X/88Uc8evQIpqam8PHxwYgRI16oVN6bxNPTEytWrMDmzZuxb98+CIKAZs2aYdOmTeJ6d6o8GOATEREREZWCZ5OaCO7niV1/3MHDJ+lIz8qGvp4UDetYol9gE3g2qamzfbZKA1layYP7K3eeIeLwXajUOdtLAAzo5ISOLcuvFJexgR6sLYygV04zASo7qVSKiRMnYuLEiTrt2uW7VLRu3bqhW7duFd0NKgEG+EREREREpeTZpCbcG9fAvYcpSM1QwtLMEA52VvlG7pXZasjSFMhWlyy4v3o/E6f+ShTL4OnrSfFBT1c0b1qrXPotkQAmRvqwMitbMj0iqpwY4BMRERERvQCpVFJgIj0tRbYayWlyqNXF57TWCAJ+PvkAJ/9KE9tMjfUxsa9nkecoDakEMDcxfOEa90RU+THAJyIiIiIqZwqlCslpCqg1xQf32SoNvjtwExeiE8U2G0tjTB7ghTo1zMqlP3pSCSzNDGFqXLWT6RG96RjgExERERGVoyxFNlLSlSUK7rPkKqzbcw1/xyWLbXVrmGLKwOawtjAql/4Y6ElhZWFUYFk/IqpaGOATEREREZWTDHk2UtOV0AjFB/cF1bi3r2GI4H5u5RbcG/2bTK+8yuoRUeXGAJ+IiIiIqBykZymRlqFECQbu8c/TdITt1K1x38K5BnwaS2FsVPZ/okskgLGhPqzNmUyP6E3CAJ+IiIiIqIzSMpRIy1KiBAP3uBOXjLW7dWvcd23dAF186uL+vXtl7otEApgbG8LCzAASCYN7ojcJA3wiIiIiojJISVcgQ55douD+4q1EbPnlL6j+zawvATCwsxMCWtRDVlZWmfsi/TeZnhmT6RG9kRjgExERERG9AEEQkJKhRGYJg/uj5+Ow6+gdsca9gX5OjXtv5/Kpca+vJ4G1uRGMDPlPfKI3FX/7iYiIiIhKSaMRIEtXQK5QobjYXiMI2PPnXRw5Fye2mRrrY2I/TzjaW5dLfwz1pbC2MIaBPpPpEb3JGOATEREREZWCWiNAliaHXKkudtuXXeNeAsDIUB/WFkbQYzI9ojceA3wiIiIiohJSqTWQpSmgyC4+uM+UZ2Pdnmu4HScT2+rVMsekAV6wMi97GTyJBDAzNoClmSGT6RERAAb4REREREQlkq1SQ5amgFKlKXbb5FQ5wnZewT9PM8S2pg1tMK63O0zKoQyeVCKBhZkBzE0My3wsIqo6GOATERERERVDka2GLE0uZr8vyj9P0xH20xUkp/1X497X1RZB77hAX6/sa+T1/k2mZ8xkekSUB/9WICIiIiIqglypgixNAbWm+OD+dlwy1uWpcd+tTQO826FxuUyjz0mmZwQDfb0yH4uIqh4G+EREREREhciUZyMlXQlNCergXYhOxNZf89S47+KMgOb2Ze5HTjI9PVibG0GvHGYBEFHVxACfiIiIiKgAaRlKpGcpUYKB+5da414iAUyNDWDFZHpEVAwG+EREREREeaSkK5Ahz0ZxA/cF1bg3+7fGfeNyqHEvlUhgYWoAc1Mm0yOi4jHAJyIiIiL6lyAIkKUrkKVQFRvcF1bjfspAL9hWL3uNez2pBNYWTKZHRCXHvy2IiIiIiACoNQJkaXIolGoUNyv/Zde4ZzI9InoRDPCJiIiI6I2nUmsgS1NAka0udtuCaty7NLTB2HKocS+VSmFqpA8bS2Mm0yOiUmOAT0RERERvtGyVGrI0BZQqTbHbPnyajvA8Ne5bu9ni/bfLXuNeKpHA0swY1hbMlE9EL4YBPhERERG9sRTZasjS5GJpu6LcjkvG2t3XkPUSatznBPeGSDMoQcp+IqJCMMAnIiIiojeSXKmCLE0BdQnq4OWrcS8BBnV2hn851LjXk0pgZW4IQZ0NlUpV/A5ERIVggE9EREREb5wMeTZS05XQFJcqH8CRc3HY9ccd8WcDfSlG9XKDl1PNMvfDQC8nmZ6hgR4yM7PLfDwierNVisU9+/btwzvvvAN3d3d0794dhw4dEj9LSEjAuHHj0Lx5c7Rr1w4rVqyAWl188hMiIiIiooKkZSiRmq4oNrjXCAJ2Hb2jE9ybGevjw0He5RLcGxvoobqVMQwNmCmfiMpHhY/g79+/H3PnzsXHH3+M9u3b48CBA5g+fTpsbW3h5uaGUaNGoWHDhvjf//6HuLg4zJ07F1KpFFOmTKnorhMRERHRayYlXYEMeXYJa9z/hQvRT8S26lbGmDyg7DXuJRLAxEgfVmZGkErLtnafiCi3Cg3wBUHAypUrMWzYMAwdOhQAMGHCBFy4cAHnzp3Dw4cP8c8//+Cnn36ClZUVnJyc8Pz5cyxatAjjx4+HoaFhRXafiIiIiF4TgiBAlq5AlkJVbHBfYI372haY1N+zzDXupRLA3MQQFmb8dywRlb8KDfDv37+Phw8fomfPnjrtmzdvBgB8+umncHV1hZWVlfhZ69atkZ6ejujoaHh6er7S/hIRERHR60etESBLk0OhVKO4FfeF1bgf19sdxmWucZ+TKd/M2KBMxyEiKkyFB/gAkJmZiVGjRuHmzZuwt7fHhAkTEBgYiMePH8PW1lZnn1q1agEAHj169MIBviAIyMzMLFvnSygrK0vnf4kqEp9Hqiz4LFJlwuexatNoBMjSlZAri89O/+hZBjbsj4YsXSm2tXSpiYFvNYagyUZW1osnwTPQ14OVuSEkmuxCk+nxWXz1BEEoc4lDosqkQgP89PR0AEBISAgmTZqEmTNn4vDhw5g4cSK2bNkCuVwOS0tLnX2MjHKmRSkUihc+b3Z2NqKjo1+84y/gwYMHr/R8REXh80iVBZ9Fqkz4PFYtEokEUj0DpGWpkCVXQihmXn7CMyUOXpBBqfpvuxaOZvB1kCD2wf0y9cPUxAjmxnp4/ji72H4AfBZfNS77paqkQgN8A4Oc6UmjRo1C7969AQAuLi64efMmtmzZAmNjYyiVSp19tIG9qalpmc7r6Oj4wvuXRlZWFh48eICGDRvCxMTklZyTqDB8Hqmy4LNIlQmfx6pJpdYgOU0Jc1Xx1Zcu336GX87fgTpXjfu+AQ7w87AtZs+iSSSAiaE+rMyNUJJBYj6Lr97du3crugtE5apCA/zatWsDAJycnHTaHR0dcezYMfj4+OD27ds6nz158kRn3xchkUjK9ILgRZiYmLzycxIVhs8jVRZ8Fqky4fNYdciVKmRmKaBvYAj9Ypa7v6wa9xIJYGpsACszw1JPAeez+Opwej5VNdKKPLmrqyvMzMxw9epVnfbbt2+jfv36aNWqFW7evClO5QeAyMhImJmZoWnTpq+6u0RERERUyWXKs5GcqoBaU3yN+51Hb+vWuDcxwLTBzcsc3EslgIWJIazNjRhAEtErVaEBvrGxMUaPHo3Vq1fj119/RVxcHNauXYvTp09j5MiR6NSpE2rWrIkPP/wQt27dwpEjR7Bs2TJ88MEHXCtDRERERDrSM5VISVdAU8w692yVBpv338DR8/FiW3UrY3z0fgs42FkVsWfx9KQSWJkbsQweEVWICp2iDwATJ06EiYkJli9fjsTERDRu3BhhYWHw9fUFAGzatAmhoaEYMGAArKysMGTIEEycOLGCe01ERERElUlquhLpcmWJatyv3X0Nd+JlYlt51bjX05OgmrkRjAwr/J/YRPSGqhR/+4wcORIjR44s8LMGDRrg22+/fcU9IiIiIqLXgSAISM1QIkOeXWxwn5QqR/hPV/DPs/9q3DdrZIOx75W9xr2hvhTWFsYw0K/QCbJE9IarFAE+EREREVFpCYIAWboCWXIViis+9/BJOsJ2XoEs7b9Sy63dbBH0tgv09MoWlBsb6sHa3KjMxyEiKisG+ERERET02tFoBCSnKaBQFh/c/x2bjHV7riFLoRLb3m7bEL3aO5QpCZ5EApgY6TOZHhFVGgzwiYiIiOi1otYIkKXKIc8uvsb9hehEbP31L6hy1bgf3MUZHbzty9QHqQQwNzFkMj0iqlQY4BMRERHRa0Ol1iA5VQ6lSlPsti+rxr1UIoGVuSFMjQ3KdBwiovLGAJ+IiIiIXgvZKg1kacUH9xpBwO4/7uiUwTMzMUBwP8+yl8FjpnwiqsT4NxMRERERVXrKbDWS0xRQqYsO7rNVGmz99S9cvPVEbKtuZYzJA7xgW92sTH3IyZRvBAN9vTIdh4joZWGAT0RERESVmkKpQnK6Amp10en0CqpxX7+2BYLLoca9sYEerC2YKZ+IKjcG+ERERERUaWUpspGSroRaU3Rwn5wqR9hLqHGvzZRvZWYEqZSZ8omocmOAT0RERESVUqY8J7jXCEUH9wXXuK+DoLeblmnEnZnyieh1wwCfiIiIiCqd9Cwl0jKUKGbgvsAa9++0bYieZaxxL5VIYGluCDNmyiei1wgDfCIiIiKqVNIylEjLUqKYgfuXVuOemfKJ6HXFv7WIiIiIqFIQBAEpGUpkyrOLDe5fVo17ZsonotcZA3wiIiIiqnAajQBZugJyhQpFxfYvs8Y9M+UT0euOAT4RERERVSi1RoAsVQ55trrI7QqqcV/DyhiTB3qjto3pC5+fmfKJqKpggE9EREREFSZbpYEsTQ6lSlPkdi+rxr1EApgbG8LSnJnyiej1xwCfiIiIiCqEMlsNWZoC2eqig/ukVDnCX0KNe6lEAkszA5iZMLgnoqqBAT4RERERvXIKpQrJ6Qqo1RVT415PTwJrcyMYM1M+EVUh/BuNiIiIiF6pLEU2UtKVUBdT5P7v2CSs3XMNcsV/a/PfbtsQvcpY495QXworcyMYGjBTPhFVLQzwiYiIiOiVSc9SIi0jG5pi6uCdv/kY3x24We417pkpn4iqMgb4RERERPRKpKQrkPGCNe5Hv+sGzyYvXuOemfKJ6E3AAJ+IiIiIXipByKlxn6VQFRncawQBu47ewR8XyrfGvUQCmBkbwNLMsExT+4mIKjsG+ERERET00qg1AmRpciiUahQ1cJ+tUmPrrzfLvcY9M+UT0ZuEAT4RERERvRQqtQayNAUU2eoit8uQZ2PdS6hxryeVwNqCmfKJ6M3Bv+2IiIiIqNxVdI17Az0prC2YKZ+I3iwM8ImIiIioXJW0xn3CkzSE/XQVKen/1bhv414H73crW417o38z5eszUz4RvWEY4BMRERFRucmU59S4L64MXkE17t9p2xA9y1DjXgLA2Egf1ubMlE9EbyYG+ERERERULtIylEjPUkJTTBm88zcfY+uvN6HWlF+Ne2bKJyJigE9EREREZSQIAlIzlMXWuBcEAUfOxWH3n3fFtvKocS+VSGBhZgBzZsonojccA3wiIiIiemEaTU6Ne7lCVWQZvJdV415PKoGVuSFMjAxe+BhERFUFA3wiIiIieiHqf8vgyYspg5etUmPLLzdx6e/yrXHPTPlERLoY4BMRERFRqWWrNJClyaFUFV0GL0OejbW7ruFugkxsq29rgUn9PWFp9uI17pkpn4goPwb4RERERFQqin9r3KuKq3GfIkfYzit4lKvGvatDdYx5zw3Ghi/2z1BmyiciKhwDfCIiIiIqMblSBVmaQsyAX5iCaty3da+DoWWocc9M+URERWOAT0REREQlkiHPRmoJatzfepCEdXvLt8Y9M+UTERWPAT4RERERFaukNe7P/fUY3x3QrXE/pGtTtPeye+FzM1M+EVHJMMAnIiIiokKVpsb97+fisCdPjfsx77nDw7HGC5/fQE8KKwsjGDFTPhFRsRjgExEREVGBBCGnxn2WvJga9xoBO4/exp8XE8Q2cxMDBPf3RKO6L17jnpnyiYhKhwE+EREREeWj1giQpckhVxZf4/7bX/7C5b+fim01rE0weYDXC9e4lwAwMtSHtYUR9Jgpn4ioxBjgExEREZEOtVqD5DQFFNlFB/cZWdlYu1u3xn0DWwsEl6HGPTPlExG9OAb4RERERCTKVmkgS5NDqSq+xv2qny7j8fNMsa2sNe6lEgksTA1gbspM+UREL4IBPhEREREBAJTZasjSFMhWFx3cJySmIWxnnhr3HnUwtOuL17hnpnwiorJjgE9EREREUChVSE5XQK0uQY37Pdd01uaXtcY9M+UTEZUPBvhEREREb7gsRTZS0pVi7frCvIwa98yUT0RUfhjgExEREb3BMuTZSE1XQlNEkftCa9y/6waPJjVf6LzMlE9EVP4Y4BMRERG9odIzlUjLVKKogfuXUeOemfKJiF4OBvhEREREb6DUdCXS5UoUMXD/UmrcSyWAhakhM+UTEb0EDPCJiIiI3iCCICAlQ4lMeXaRwX1OjfuruJuQIraVtcY9M+UTEb1cDPCJiIiI3hAajYCUDAWy5CoUlU7veUoWwn66Uq417vX1pLBmpnwiopeKAT4RERHRG0CtESBLk+uUtytITo37K0hJV4ptZa1xb6gvhbWFMQz0mSmfiOhlYoBPREREVMWp1BrI0hRQZBcd3BdU4767XyP0aNfohZLhMVM+EdGrxQCfiIiIqArLVqkhS1NAqdIUuV3UX4+xLVeNe6lEgiHdnNHO88Vq3EskgKmxAayYKZ+I6JVhgE9ERERURSmUKiSnK6BWF13j/v+i4rD32H817g0NpBjzrjvcHWu80HmlEsDcxBAWZsyUT0T0KjHAJyIiIqqCshTZSElXiiPyBXkZNe6ZKZ+IqOIwwCciIiKqYtKzlEjLyIamiDp4yuycGvdXbv9X476mtQkmD/RCrWovVuNeX08Ca3MjGL1gpn0iIiob/u1LREREVIWkpiuRLlcWW+N+ze6riMlX494Lli84rZ6Z8omIKh4DfCIiIqIqQBAEyNIVyFKoigzuC6px79a4Oka/+2I17nMy5evB2sKYmfKJiCoYA3wiIiKi11xOjXsFFEoViojtEZ+YhvA8Ne79POpiSDdn6ElLP/LOTPlERJULA3wiIiKi15harUFyCWrcRz9IwvpyrHHPTPlERJUPA3wiIiKi11SJa9zfeITvDkZDU0417pkpn4iocmKAT0RERPQaUihVkKUroCq2xn0s9h6LEdvKWuNeX08Ka3NDZsonIqqE+DczERER0WumpDXufzpyG8culV+Ne2bKJyKq3BjgExEREb1GMrKUSH3BGveTBnihtk3pa9znZMrXh7WFETPlExFVYgzwiYiIiF4TaRlKpGcpUcTAfcE17utYIrif5wvVuGemfCKi1wcDfCIiIqJKThAEpGYokSHPfqEa92PedYeRoV6pz8tM+URErxcG+ERERESVmEYjQJaugFzxAjXuPetiSNcXq3HPTPlERK8fBvhERERElZRarYEsTQH5C9S479GuEbr7vViNe309CazNjZgpn4joNcO/tYmIiIgqoWyVBrI0+QvVuB/arSn8POu+0HmZKZ+I6PXFAJ+IiIioklFmqyFLUyBbXXhwX9417nMy5evB2sKYmfKJiF5TDPCJiIiIKhGFUoXkNEWpa9xbmBoguL8XGtaxLPU5JRLAxEgf1uZGzJRPRPQaY4BPREREVElkKbKRkq4sMrgvrMb95IFeqFWt9DXumSmfiKjqYIBPREREVAmkZymRlpENTRF18Mq7xr1UIoGluSHMjJkpn4ioKmCAT0RERFTB0jKUSMtSvtIa93p6ElRjpnwioiqFf6MTERERVRBBEJCSoUSmPLvI4D4+MQ1hP11Bakb51LjPyZRvBAP90r8YICKiyosBPhEREVEF0GgEyNIVkCtUKCK2x837z7Fh73WdGvc92zXCOy9Y497YQA/WFkbQ02MZPCKiqoYBPhEREdErptYIkKXJdYL2gkTeeIRt5VTjXpsp38rMCFKWwSMiqpIY4BMRERG9QtkqDWRpcihVRde4PxwZi33H89S4f88dbg7VS31OiQQwNzaEpXnZM+ULglCpSulVtv4QEVUkzs0iIiIiekWU2Wokp/4X3A/u1wPffLlAZxuNRsD/fr+tE9xbmBpg+pAWSH30FxZ+obv9H0cOY1Df7ujasTWWLfoS0yaNxbRJY8XPpRIJrMyNyhzcP378GGPHjsXDhw/zfTZhwgSsXbu2TMcvTkJCApydnbFnzx4AQGpqKmbNmoULFy6I2wQFBSEoKKjA7ffs2QNnZ2ckJCS81H6WxejRo/H555+Xy7F27twJZ2fnfH8+++yzIvfLfQ+J6PXDEXwiIiKiV0CuVEGWptCpcf/ZV0tgamYm/lxgjftqJpgywAs1q5li5ZcR+Y67avk3sLevj9lzQ1GjZi0s/eYL8TM9PQmszY1gXA6Z8s+cOYPjx4/na1cqlYiMjMTkyZPLfI7SiI6Oxv79+9G3b1+xbcGCBYVuHxAQgB07dqBWrVqvonsVLjo6Go0aNcLChQt12mvUqFFBPSKiV4EBPhEREdFLlinPRkq6Ml+N+yZOTcX/Ts/KxppdV3Hv4X817hvWsURwf09YmBY++p6akoKWfVrDq3lLnXZDfSmszI1gaPByM+VfuHABZmZmcHFxeannKQlHR8dCP7OxsYGNjc0r7E3Fio6Ohru7O7y8vCq6K0T0CnGKPhEREdFLlJahREq6Il9wD/w3Rf/m3/fQq3NrXD1/Ev9c2I47h+bh/u+fQvrwEPQlOYn4pk0ai6tXLuLqlYsIbNcCVy5dQGC7FgCAbVs2IrBdCzx+9A8AQCoFbCyNYWigB4VCgdWrV6Nbt25wd3dHly5dsGHDBmg0/+UAUKvV2LBhA3r06AEPDw94eXlh0KBBiIyMBJAzvX3OnDkAgLfeeguzZ88W9z1+/Djat28PiUSCqKgoODs74+zZswgKCoKHhwcCAgKwc+dOPHnyBJMmTYK3tzf8/f2xdetW8RiFTZ8PDAzUOZdWVFQUhg0bBgAYNmyYOKW8qOnlec8xe/ZsjBgxArt370bXrl3h5uaGd999FydOnNDZ7/Llyxg6dCi8vLwQEBCA7777DiNGjBD7pb3mqKgonf3y9kUul2Pp0qXo0qUL3Nzc0Lx5c4wcORLR0dEF9rcot2/fxrhx49C8eXM0b94cwcHBiI+PFz8XBAF///13ubx0SUpKQmhoKDp27Ag3Nzf4+PggODhY57sKCgrCzJkzMWXKFHh5eWHkyJEAgCdPnmDatGnw8fFBq1atMH/+fCxfvhyBgYE659i5cye6d+8ONzc3BAQEICwsDGp10QkoiahgDPCJiIiIXgJBEJCSrkBalhKaIurgpWdlY/3eawCAxOu7YWBaDV0GTkfQ8BE4fOhnfP/dJgDA1Bmz4ejkDEcnZ4Sv24omzk0Rvm4rAOCdHu8ifN1WVK9eA1KpBPp6UujpSSEIAsaPH49Nmzahf//+WLduHbp164YVK1boTGdfsmQJ1qxZg4EDB2LTpk34/PPPIZPJMHXqVGRlZSEgIAATJkwAAISHh2PixInivsePH4e/v7/ONU2fPh2BgYFYv349GjVqhAULFmDYsGFo0qQJ1qxZAw8PD3z99de4du3aC91bV1dXzJ8/HwAwf/78IqfmF+XGjRvYvHkzpkyZgtWrV0NPTw+TJ09GSkrOLIqYmBiMGDECALBs2TJMnjwZGzZswMWLF0t9rlmzZmH37t0YO3Ysvv32W8yZMwd37tzBjBkzIBTw8qcw9+/fx6BBg/D8+XN88803+PLLLxEfH4/Bgwfj+fPnAIC4uDhkZGTg+vXr6Nq1K1xdXdG1a1fs27evVH0WBAHjxo3D6dOnMXPmTGzevBmTJk3C2bNn893zQ4cOwczMDGvXrsXo0aOhVCoxfPhwXLp0CR9//DG+/vpr3Lp1C99++63OfuvXr8cnn3yCNm3aYN26dRg6dCg2btyITz75pFR9JaIcnKJPREREVM40GgEpGQpkyYuucZ+t0uCve89RzTEbAGBWywUfjJ0s1ri/dOEcIs+cxJjxk9GwkQPMTM0BAM3c3HX+t0bN2nB1d4eZsQH09SRiVvkTJ07gzJkzWLZsGbp37w4A8PPzg7GxMVauXCkG3dqR1twjzkZGRpg8eTL+/vtveHl5oX79+gAAFxcX2NvbAwDi4+MRHx8PPz8/nevq27evOIpramqKAQMGwMPDA1OnTgUANG3aFP/3f/+HS5cuwcPDo9T319zcXJyO7+joWOTU/KKkpaVhz5494rWZmpri/fffR2RkJLp27Yr169fDwsICmzZtgomJCQDAwcEBgwYNKtV5lEolMjIyMG/ePLzzzjsAAB8fH6Snp2PhwoV49uwZatasWaJjhYeHw8TEBFu3boW5ec7z0KZNG3Tq1AmbNm1CSEiIOCsgISEBs2fPhr6+Pvbt24eQkBAolUoMGDCgROd68uQJTExMEBISgpYtc5aA+Pr6Ii4uDjt27NDZ1sDAAKGhoTA0zFlOsmvXLty7dw+7d++Gm5sbAKB169bo1KmTuE9aWpr4YmnevHkAgHbt2sHa2hrz5s3DyJEj0aRJkxL1lYhyMMAnIiIiKkfF1bjXaATEJ6bhQnQiUjOUMDX+7xWAfzsfdG/nIP5cs2ZtJD56VOw5JRLA0swQ5iaGOiXjzp07B319fXTr1k1n+169emHlypU4d+4cmjRpgqVLlwLImY597949xMbG4s8//wSQE5wW5sSJE/D29oaFhYVOu7e3t/jf1avnlPXz9PQU26pVqwYgJ8CrSDY2NmJwDwC2trYAgKysLABAZGQkOnToIAb3QM612dnZleo8hoaG2Lx5MwAgMTER9+/fx4MHD4q8xxqNRmcZhUQigZ6eHiIjI+Hj4wNjY2OoVCoAOS88WrZsiTNnzgAAWrVqhXXr1sHX1xempqYAgPbt2yMpKQmrVq1C//79odFodGYOaI+fW+3atbFt2zYIgoCEhATExsbi3r17uHTpUr4+Ozg4iMG99t7Vq1dPDO61/ezYsaO4nOHy5cuQy+UIDAwUrwWAOIX/9OnTDPCJSokBPhEREVE5Uak1OmXw8rr1IAmHzt5H7KM0nRcAeno5qyabNa6ts71UKoFGKPhYuRkb6sHcJH8ivpSUFFSrVi1f4KYdLdYG2NevX0doaCiuX78OExMTODo6om7dugBQ5PTx48ePo0OHDvnatSPLueUOkiuLvH3SvhzRBtZJSUniC4rcXiQT/cmTJ/HVV1/h3r17MDMzQ9OmTcXgu6B7/PHHH2Pv3r3iz3Z2dvjjjz8gk8lw8OBBHDx4MN8+2iSC1atXR8eOHfN97u/vjzNnzuDZs2eYPn06zp07J37m4+OD7du359vn559/xrJly/Do0SNYW1vDxcUFxsbG+bYzy1UNAgCSk5MLvHe522QyGQBg7Nix+bYDcmYQEFHpMMAnIiIiKgfZKjWSUxXIVhce3H//WzRS0pXIzvMCwMz4xf5Jpv/viwED/YIz5VtZWSE5ORlqtVonyNcGTtWqVUN6ejpGjx4NZ2dnHDhwAA4ODpBKpTh+/DgOHz5c6LkVCgWioqIwY8aMF+p7bnkDa62MjIwyH7ssbG1t8ezZs3ztz58/h4NDzkyLovquDXrj4uIQHByMTp06Yf369ahXrx4kEgkiIiJw8uTJAs89adIkDB06VPxZOzpuYWGBtm3biksgctPXz3mOLly4gPj4ePTu3Vvnc4VCAT09PVhZWSE0NFTn/uYN0LXHCQkJQVBQEEaNGoXatXNeQC1atKjYPAS1a9fGgwcP8rVr8wQAgKWlJYCcHBANGzbMty1L+hGVHpPsEREREZWRQqnC81R5ocG9RiPg4Jn7kKUpdIJ7CQATIz2o/x3BLS7XmlTvv3+6GepLYWOZfyQ1Nx8fH6hUKvz222867T///DMAoEWLFrh37x5kMhmGDRsGR0dHSKU559Bmk9cGrtp2raioKFhbW8PZ2bnoTpeAdsT/8ePHYltMTIw4wluQvLMSXoZWrVrh5MmTUCgUYtvNmzd1MsgX1PeUlBTExMSIP9+4cQMKhQJjx45F/fr1xZcC2uC+oBF8e3t7uLu7i3+099nHxwd3796Fi4uL+Jmbmxu2bt2K33//HUDO9PjZs2fj/v374vE0Gg0OHz4Mb29vGBoawsHBQef42hcWuV2+fBkajQaTJ08Wg3u1Wi0uBcj7UiM3Hx8fJPw/e/cdF3X9B3D8dXdw7OUktzhAFAQHKG4yNaVcabj3xL0r0zRnbhnutFz5c5SVmmm5cms5c6PmHigb7oC73x/EV0+GgDiq9/Px8KF85+e+95W79/fz/nzeN2+aVAlITEw0eaBRuXJlzM3NuXfvnklbzMzMmD17drqqCkKI55MefCGEEEKIF5CgS61xn5LFVPkXrj/mys0ok2205mo0GhVqtRprbepXskdRiVmey9bWjj/PnOLMyeNUr1oZczPrLLevW7cuvr6+jB07lnv37uHm5saRI0dYsmQJLVu2pGzZssTExGBra8vChQsxMzPDzMyM7du3s2HDhtTX9/d49LTe1h07dlC3bl327t2bYXp+bvj6+mJpacm0adMYPHgwcXFxzJ8/H0dHx0z3SRv3v3v3bhwcHHBzc8uTtjytb9++bN26lZ49e9K9e3eio6OZN28earVaCdJdXV156623CA0NxdbWFpVKxaJFi0zS/ytWrIiZmRkzZsyge/fu6PV6Nm3axO7duwGIj4/Pdpv69+9PYGAgffr0oV27dlhYWLBu3Tp27tzJ/PnzAQgMDOSbb76hb9++DB48GCsrK9asWcPFixdZvXp1ts+VNgHixIkTad26NVFRUaxevZrz588r7c5oOAZAQEAAixcvJigoiMGDB2Nvb8/y5cuJiIhQhn84OTnRs2dP5s2bR2xsLL6+vty7d4958+ahUqleynsqxL+d9OALIYQQQuRSXIKeyJisg/uHkQl8tfVPk22stBoKOT0JADWa1GAxs4n50rRs3RZzc3OGDwli/28Zp3Y/LS3YDAwMZMWKFfTu3ZuffvqJYcOGMWXKFCA1UA4LC8NoNDJ48GBGjRrF7du3WbVqFTY2Nhw7dgxIDcL9/PyYNWsW06dPZ+/evenK4+WWvb29Uvs8KCiIefPmERQUZDJB27PKlStHQEAAq1evZsSIEXnSjmeVLFmSZcuWodPpGDRoEHPmzKFXr14ULFhQSWnXaDTMnz+fAgUKMGzYMCZPnkyzZs1o1KiRyXFmzZrFvXv36Nevn1Lib+XKlahUKuUaZ4ebmxurV69GpVIxatQoBg0axIMHDwgNDVXOWaBAAVavXo2rqyuTJk1iyJAhJCQksGLFCpPJDp/H19eXcePG8ccff9CrVy+mTZtGkSJFCAkJAcgyTd/MzIxly5bh7u7OZ599xqhRoyhXrhzvvPOOMvcAwJAhQxgzZgw7duygV69ezJgxg6pVq7Jq1ap0kzcKIZ5PZcxJ4c1/gdOnTwPg4eHxSs4XHx/PuXPnqFChgskvMyFeB7kfxZtC7kXxJsnt/Rgdqyc2UZ9lWv1fd2MIWX+C6LgnM47bWpnhZGdhMtu9PsmALimFvq08KfmWfYbHUqvA1kqLnU36yfTEy3Hw4EHMzc2VEnEA0dHR+Pn5MWrUKDp37pyn5/s3/W68dOkS4eHhNGrUyORe/+CDD3B2dlYeErxurzo2EOJlkxR9IYQQQogcMBqNRMbqSNAlZxnc/3k1gkXfnkb3VK+81lyNo61pKTuj0UicLomiBW0pXjjjHku1SoW9rRYbS/M8ex3i+c6ePcv8+fMZNmwYFStWJDIykuXLl2NnZ0dAQMDrbt4bLT4+nsGDB9O+fXveeecdUlJS2Lp1K2fOnHlpGRdCCAnwhRBCCCGyLbXGvQ6dPpmsUiAPnbnD11vPYfg7LV+tUvF29eKcuPSAqPgkbCzMMTNTkZycGtxbac1oUqMUarUq3bE0GhVOthZYaOVr26uWNl5+7dq13LlzB2tra3x8fJg6dapSkk5krHLlysydO5dly5bx3XffYTQacXd3Z+nSpdSoUeN1N0+Ify35pBBCCCGEyIbkFENqcJ+U+Th5o9HI9kPX+W7PkxnULcw19GpRiUplClDRJT8/HbrGvYh44nUGNGo1RQva0qRGKdxKpQ8YtWZqHO0sMi2DJ14utVpN//796d+//+tuyj9SkyZNaNKkyetuhhD/KRLgCyGEEEI8hz4pJbXEXSZl8CC1FN66nRfY8/stZZmdtTkD2ngp4+rdSuWjfAknbtyLITY+CVtrc4oXtsuw597SXIOjnQUajcyJLIQQInskwBdCCCGEyIJOn8zjWB0pKZkn5euTUlj2/VlOXnqgLCvkZMXAD70p6Ghlsq1arcp0Ij0AlQqsLMxwsLHIMPAXQgghMiMBvhBCCCFEJrJT4z42IYmwDScJvxWlLCtdxJ7+H1TGzjpnM96rVGBrqcXeVmbKF0IIkXMS4AshhBBCZCA2QU9MnJ4sYnseRiYQ/L8T3HsUryzzLFuAns0roTXP2bh5tUqFvY05NlYS3AshhMgdCfCFEEIIIZ6RvRr30YSsP2lS476OVxECG7miUeds3LxGrcLRzgJLmSlfCCHEC5BPESGEEEKIv71Ijfv367jwrl8pkxr32WGuSZ0pP6c9/kIIIcSzJMAXQgghhACMRngUnY0a96fv8PU20xr3Hd91w8+zSI7PafH3TPlmMlO+EEKIPCABvhBCCCH+8zRm5jyO0aHSmGe6jdFo5KeD19m817TGfe+WHlR0yZ+j86kAC61Zahk8mSlfCCFEHpEAXwghhBD/aSkpRmISUrDVJ2NllXGAbzAY+WbHBfb+8UyN+7ZelHTOvORdRlQqsLY0x8FGm+N0fiGEECIrEuALIYQQ4j9Lp0/mUYyO+ARdptvkpMb986hUYGNpjoOtRa7bLIQQQmRGAnwhhBBC/Cel1bhPSk7JdJvMatwHfVAZ2xzWuFerwM5am+P9hBBCiOySAF8IIYQQ/zlxCXqi45IwZDFVfp7XuLfVYmOZ+Rh/IYQQ4kVJgC+EEEKI/5SYOD2xCXoMOa5xX5TARuVzVePewVaLlYUE90IIIV4uCfCFEEII8Z9gNBqJitMTn5iUZY37s+ERLP7umRr3dV14t2bOa9xr1Cqc7Cyw0MpXLiGEEC+ffNoIIYQQ4l/PYDASGasjUZd1jfuDp++wMo9q3Jtr1DjaWeQ4nV8IIYTILQnwhRBCCPGvlpJiIDJGR2JS5pPpGY1Gdhy5ydaDfynLclvjHkBrpsbRzhJzs5yl8wshhBAvQgJ8IYQQQvxrJSWnEBmjQ59syHQbg8HInjMxnLl+X1mW2xr3AJbmGhztLNBoJLgXQgjxakmAL4QQQoh/JZ0+mcexOlJSMk/K1yelsHzLBc5cT1CW5brGPWBlaYaDjQVqdc7G6gshhBB5QQJ8IYQQQvzrxCcmER2nJyWLqfJj4/WEbjjJ1dvRyrLc1rhXqcDG0hx7G22OJ+ITQggh8ooE+EIIIYT4V4mN1xMTn3UZvAeRCYQ8U+O+kosTvVtWzkWNe7C10mJnk7OHAkIIIURekwBfCCGEEP8KRqOR6Dg9cc8pg3f9bjQh/ztBTHySsqxiCSu6NnPLRXCfWuPe2lJq3AshhHj9JMAXQgghxD9edsvgnQ2PYPG3p9E9NaN+05olKJ0vEU0Ox81r1Coc7SywlBr3Qggh3hDyiSSEEEKIf7TslMEDOHDqNqt+Op+uxr13OSeuXLmSo3NqNCqcbC2wkOBeCCHEG0Q+lYQQQgjxj5WdMnhGo5FtB6/x/d5wZdnTNe4TEhIy3TcjZho1TnYWOU7nF0IIIV42CfCFEEII8Y/0bBk8o9GYbgb7FIOBb36+yL4Tt5RlL1LjXmumxtHOEnOz9DXuMzr/6/SmtUcIIcTLl/7TSQghhBDiNfP392fMmDGZro9PTOJRzJPgfv9ve5g2abzJNj9v30bzZk34ckpn7p3ayI0DC7l7dDGjOlfPdXDvZJ8+uL979y69e/fm1q1b6fbp168fCxYsyPG5cuLmzZu4urqyadMmAKKjoxk1ahTHjh1TtunUqROdOnXKcPtNmzbh6urKzZs3X2o7X8TT7X9R69evx9XVNd2fiRMnZrnfmDFj8Pf3z5M2CCHEyyI9+EIIIYR444SEhGBra5vhupg4PbEJpmXwNnyz2mSb2Hg9s76YisYqP0V9PsDM0p6oC99RtKAtBR2tctweS3MNjnYWaDTp+0YOHDjAnj170i3X6/UcOnSIgQMH5vh8L+LcuXNs3ryZ1q1bK8vGjx+f6fb169dn3bp1FCpU6FU077U7d+4cpUuXZtq0aSbLCxQo8JpaJIQQeUcCfCGEEEK8cdzd3dMtMxqNRMXpiX9OGbwHkQkEr/uDJF0cdiVqYl2gDJ5lC3Dh8S+oc5GybqnV4GhnmeNZ9o8dO4aNjQ0VKlTI8TnzWtmyZTNdly9fPvLly/cKW/N6nTt3Dg8PD7y8vF53U4QQIs9Jir4QQggh3jhpKfpp6eRbtm6jb/8B1K9dg/ebNGDm9M+VyfGGDujNyRPHOXniOP61q/LpzLX8tjK11/zRpZ1c/HEUzWvmTxfc63U6Vq5YQp/u7Rk2oCe9ugSydtUKDIbUCftUgNZMxYZvVtL8/ffw9PTEy8uLwMBADh06BKSmt3/00UcAvP322ybDCvbs2UOdOnVQqVQcPnwYV1dXDh48SKdOnfD09KR+/fqsX7+e+/fvM2DAALy9valXrx4rVqxQjpFZ+nxmQxgOHz5M586dAejcubOS1p5Vivuz5xgzZgxdu3Zl48aNNG7cmEqVKtG8eXP27t1rst8ff/xBhw4d8PLyon79+nz11Vd07dpVaVfaaz58+LDJfs+2JTExkVmzZtGoUSMqVapElSpV6NatG+fOncuwvVm5ePEiffr0oUqVKlSpUoWgoCBu3LihrDcajVy4cOGFHrqsW7eO+vXr4+npSZcuXfjzzz9N1h89epQePXpQvXp1KlWqhL+/P8HBwcp9BfDjjz/y/vvv4+npSY0aNRgxYgT37t0zOc769etp1qwZlSpVon79+gQHB5OSknWlCCGEkABfCCGEEG+88ePHUbCQM59PncWH7Tux7cfNrPpqKQCDh4+hbHlXipUsS5l6g0ixKEzxWkEAeFR/m+CFyylY0DT93Gg08snooXyz+msavRtA7/5DqF23AcuWhDFnxhRUgJWlGV8uDmHBgjA+/PBDli5dyueff05kZCSDBw8mISGB+vXr069fPyB1WEH//v2Vc+zZs4d69eqZnHfYsGH4+/uzaNEiSpcuzfjx4+ncuTPlypUjLCwMT09Ppk6dyqlTp3J1nSpWrMi4ceMAGDduXJap+Vk5c+YMy5YtY9CgQYSGhqLRaBg4cCBRUVEAXLlyha5duwIwe/ZsBg4cyOLFizl+/HiOzzVq1Cg2btxI7969+fLLL/noo4+4dOkSw4cPx5hVqsYzrl69SmBgIBEREUyfPp3Jkydz48YN2rVrR0REBAB//fUXcXFxnD59msaNG1OxYkUaN27Md999l61z3L17l5CQEIYMGcLs2bOJioqiU6dO3L59G4Dz58/TtWtXHB0dmTNnDgsWLKBatWqEhISwbds2AI4fP86oUaNo1KgRS5Ys4aOPPuLQoUMMHz5cOc+yZcv49NNPqVmzJgsXLqRDhw4sWbKETz/9NNvXQwjx3yQp+kIIIYR4YyX9Xf7Ot2Zt+g0YCkCVaj4cO3qYQwf20avvQEqVdiHZaE5EbBLF7YoBYJO/FADelcpQsZJnuuMeOXSA48cOM/azKdSsVZcrV65Q5r3m2NjasnzpAjp07EiVyhW5f/8+Q4cONelxtrCwYODAgVy4cAEvLy9KlCgBQIUKFShWLPX8N27c4MaNG9SqVcvkvK1bt6Zbt24AWFtb07ZtWzw9PRk8eDAAbm5u/Pzzz/z+++94eqZv9/PY2toq6fhly5bNMjU/KzExMWzatEl5bdbW1nTs2JFDhw7RuHFjFi1ahJ2dHUuXLsXKKnVOAxcXFwIDA3N0Hr1eT1xcHGPHjqVp06YA+Pj4EBsby7Rp03j48CEFCxbM1rFCQkKwsrJixYoVyvwNNWvWpGHDhixdupTRo0crWQE3b95kzJgxmJmZ8d133zF69Gj0ej1t27bN8hwpKSmEhoYq703lypVp2LAhK1euZPTo0Zw/fx4/Pz9mzJiBWp3aj1arVi1+/fVXDh8+TLNmzTh+/DiWlpb07t0brVYLgKOjI6dPn8ZoNBIfH8/SpUv58MMPGTt2LAC1a9fG0dGRsWPH0q1bN8qVK5ej6yyE+O+QAF8IIYQQbwSDwUj4rSii4/QkJRtITjYQGasDoGJFD5NtCxYszL07d1Jr3B+4xr2IeGWdhVZD7xYeDPw+83Od/OM4Go2Geg0aotfrleWNmjRl+dIFnD97kqpelZg1axYAjx49Ijw8nOvXr7Nr1y4Ak/2etXfvXry9vbGzszNZ7u3trfw7f/78QGqQmMbJyQlIDbBfp3z58inBPYCzszOAMizi0KFD1K1bVwnuIfW1FS1aNEfn0Wq1LFu2DIB79+5x9epVrl27luU1NhgMJunuKpUKjUbDoUOH8PHxwdLSkuTkZCD1gUe1atU4cOAAANWrV2fhwoX4+vpibW0NQJ06dXj06BHz58+nTZs2GAwGk8yBtOMDFC9e3OTBS8GCBfHy8uLo0aMAtGjRghYtWqDT6bh69SrXr1/n3LlzpKSkkJSUpLRhzpw5BAQE0LhxY+rVq0ft2rWpV68e8fHxXLp0icTERPz9/ZXXASgz+O/fv18CfCFEpiTAF0IIIcRrd/LSAzb8eolb92OxtNAQE6/nTPhDLv31GAALS0uT7dVqFQajgTXbz7PvxG1lub2NlgFtKlPiOWXwomOicHB0VAI3ALVKRbEiqYFsWoB9+vRpJkyYwOnTp7GysqJs2bIUKVIEIMv08T179lC3bt10yzOqDPB0kPymeLZNqr/nL0gLrB89eqQ8oHhabmai37dvH1OmTCE8PBwbGxvc3NyU4Duja/zxxx/z7bffKj8XLVqUX3/9lcjISLZu3crWrVvT7ZM2iWD+/Plp0KBBuvX16tXjwIEDPHz4kGHDhnHkyBFlnY+PDytXrsz09eXPn587d+4AqfMJfP7552zevJnk5GSKFSuGt7c3ZmZmymvx9vZm8eLFrFixguXLl7N48WIKFChA3759ad26tXLv9e7dO8Prdf/+/QyXCyEESIAvhBBCiNfs5KUHhG44SUJiMkUKWmNtkRoMRcfp+W7vlQz3STEYiYnTmwT3WnM1ozpVo0A2yuDZ2zkQFRmpTFpmZqbBzkZLYlzqGHMnJydiY2Pp2bNn6iR/W7bg4uKCWq1mz549bN++PdNj63Q6Dh8+bDKmOreeDazTxMXFvfCxX4SzszMPHz5MtzwiIgIXFxcg67bb2NgAqWPig4KCaNiwIYsWLaJ48eKoVCpWr17Nvn37Mjz3gAED6NChg/JzWpq7nZ0dfn5+yhCIp5mZpX7lPXbsGDdu3KBly5Ym63U6HRqNBgcHByZMmGByfdPaCihzEDztwYMHygOEyZMns337dubOnYufn5/yoKJmzZom+9SpU4c6deqQkJDAoUOH+Prrr5k0aRJubm7K+WbOnEmpUqXSnU/K+QkhsiKT7AkhhBDitTEYjGz49RKJumTKFLXDxsKcqNjUtGytmRpdUmoA/nRHbky8nj+vRqBPfhI4WlmaU7ywXbaCewBP7yqkpKSwZ9dONGoVDjYW2Fia8f33qXn9VatWJTw8nMjISDp37kzZsmWVMdVps8mnBa5py9McPnwYR0dHXF1dc3FFTKX1+N+9e1dZduXKFSIjIzPd5+mshJelevXq7Nu3D51Opyz7888/TWb7z6jtUVFRXLny5KHNmTNn0Ol09O7dmxIlSigPBdKC+4x68IsVK4aHh4fyJ+06+/j4cPnyZSpUqKCsq1SpEitWrGDHjh1A6tCCMWPGcPXqVeV4BoOB7du34+3tjVarxcXFxeT4aQ8sIHUiv7/++kv5+c6dO/zxxx/4+voCqRPo+fr60rBhQyW4P3PmDI8ePVLul+nTp9O6dWuMRiNWVlY0aNCA0aNHA3D79m3Kli2Lubk59+7dM2mHmZkZs2fPTldRQQghniY9+EIIIYR4bcJvRXH3YRwlnG1BpSYiOpHklL8Dd5UKa23qV5VHUYnAkxr3sfFJyjEqlyvA7aginP/zNL8fP0K5cm7Y2Wedou9boxZeVaox64tJRD2+j72tDbt+2c6KFSto2bIlZcuWJSYmBltbWxYuXIiZmRlmZmZs376dDRs2AE/Go9v/fa4dO3ZQt25d9u7dm2F6fm74+vpiaWnJtGnTGDx4MHFxccyfPx9HR8dM90kb9797924cHBxwc3PLk7Y8rW/fvmzdupWePXvSvXt3oqOjmTdvHmq1WgnSXV1deeuttwgNDcXW1haVSsWiRYtM0v8rVqyImZkZM2bMoHv37uj1ejZt2sTu3bsBiI+Pz+j0Gerfvz+BgYH06dOHdu3aYWFhwbp169i5cyfz588HIDAwkG+++Ya+ffsyePBgrKysWLNmDRcvXmT16tXPPYeFhQX9+vVj6NChpKSkMG/ePBwdHenSpQsAnp6ebNu2jbVr11KmTBnOnz/PggULUKlUyv1So0YNli9fzpgxY3j//fdJSkpi6dKlODo64uPjw+3bt+nSpQvz5s0jNjYWX19f7t27x7x581CpVC/l/RRC/HtID74QQgghXpuYeD32tlowqoiISngS3P9No0kNFhP1KVy/G80XXx/l/uMEZX1d76L0aelJqzYfojEz46MRgzhyaP9zz6tSqZgxaz5t27Rl3do1TJ8+nZ07dzJs2DCmTJkCpAbKYWFhGI1GBg8ezKhRo7h9+zarVq3CxsaGY8eOAalBuJ+fH7NmzWL69Ons3bs3XXm83LK3t1fqnwcFBTFv3jyCgoKoVKlSpvuUK1eOgIAAVq9ezYgRI/KkHc8qWbIky5YtQ6fTMWjQIObMmUOvXr0oWLCgkmKu0WiYP38+BQoUYNiwYUyePJlmzZrRqFEjk+PMmjWLe/fu0a9fP6XE38qVK1GpVMo1zg43NzdWr16NSqVi1KhRDBo0iAcPHhAaGqqcs0CBAqxevRpXV1cmTZrEkCFDSEhIYMWKFSaTHWbG3d2dNm3a8NlnnzFq1ChKlCjBmjVrlBT9MWPG0LBhQ+bOnUufPn1Yv349/fr1o23btvzxxx+kpKRQr149Zs6cyaVLlxgwYADDhg3DysqKr7/+GgcHBwCCgoIYM2YMO3bsoFevXsyYMYOqVauyatWqdBM3CiHE01TGnBQY/Rc4ffo0AB4eHs/ZMm/Ex8dz7tw5KlSooKRqCfG6yP0o3hRyL4o0F/96pIy/NzdL3++gTzKgS0rhHd+SbN5zRUnZB2hetwxNapZUeoxzwtJcg4OdBWYatdyPuXDw4EHMzc2pVq2asiw6Oho/Pz9GjRpF586dX2Pr/rnkXnz1XnVsIMTLJin6QgghhHgtYhP02Fhp0ZpreBCZgKON1iRYNxqNxOmSsLE0Z8MvlzD83SehVqvo9G4Fanq8leNzqlRgZWGGg40FanXOHwyIVGfPnmX+/PkMGzaMihUrEhkZyfLly7GzsyMgIOB1N08IIf6zJMAXQgghxCsXFasjLjF1HH2TGqVYvf08kXF6bCzMMTNTkZxsJDZRj9EAd5+pcd+npQfupdOXaHsetQpsLLWpQwLEC0kbL7927Vru3LmDtbU1Pj4+TJ06VUlXF0II8epJgC+EEEKIV8ZgMBIVpyNBl6zMjO9WKh8dGrvx06Fr3IuIJ15nQK1SYaZWExmvV/bNbo37jKhVKuxttdhYmufVS/lPU6vV9O/fn/79+7/upgghhHiKBPhCCCGEeCVSDEYiYxJJ1KekW+dWKh/lSzhx414Mj6MT2Xn0Ly7ffFJzvHA+awa29cp2GbynadQqHGy1WFlIcC+EEOLfTQJ8IYQQQrx0SckGImMSTWrXP0utVpHPwZJvdlzg6u1oZblLUQf6t/bE1jrnqfUajQonWwsstPKVRwghxL+ffNoJIYQQ4qXSJaUQFaMjKSXz4B7gweN45v/vBA+eKoNXuVwBerxfCa25JsfnNdeocbSzyNW+QgghxD+RBPhCCCGEeGkS9clExuhIMWRdlff6nWhC1p8gJj5JWVbXuyiB77jmarZ7rZkaRzvLDEvvCSGEEP9WEuALIYQQ4qWIS9ATHZeklLfLzJkrD1ny3Zk8rXHvaGeBRiPBvRBCiP8WCfCFEEIIkedi4vTEJuh5Tsc9B07dZtW283lS4x7AUmuWGtxLjXshhBD/QRLgCyGEECLPGI1GouL0xCcmkVXHvdFoZOuBa/ywL1xZ9iI17lWAlaUZjrYWuer1F0IIIf4NJMAXQgghRJ5ILYOnQ6dPJquO+xSDgbXbL/DbydvKshepca9SgbWlOQ42WgnuhRBC/Ke9EQH+vXv3qFu3brrlU6dOpVWrVowdO5b169ebrCtatCi//vrrq2qiEEIIIbKQnGJIDe6T0te4f5pOn8LSzWc4feWhsuxFatyrVGBjaY6DrUWO9xVCCCH+bd6IAP/8+fNYWFiwc+dOkyfvdnZ2AFy4cIG+ffvSsWNHZZ1GIyVvhBBCiDeBPimFyGyUwYuJ1xO24WSe1bhXqcDOSoudTc73FUIIIf6N3ogA/+LFi5QqVYpChQqlW2c0Grl8+TK9e/emYMGCr6F1QgghhMiMTp/M41gdKSlZz6aX1zXu1Sqws9FiayXBvRBCCJHmjQjwL1y4QJkyZTJc99dffxEfH4+Li8srbpUQQgghshKXmER0rP65ZfCu3Ykm9Jka9/WqFOXDhrmrca9WqXCw1WJtaZ7jfYUQQoh/szciwL948SJOTk506NCBq1evUrJkSfr160fdunW5ePEiACtXrmTv3r2o1Wrq1q3L0KFDlRT+nDIajcTHx+flS8hUQkKCyd9CvE5yP4o3hdyL/3xxicnExOsxPKcO3p9XH/PV1gvok5+k7zfzK8Hb1Yqi0yXm+LxmZmocbCzAkET8Uw8MXoTcj+JNIffiq2c0GmVyTvGv8toD/OTkZMLDwylbtixjxozB1taWLVu20Lt3b5YvX87FixdRq9UUKlSIhQsX8tdff/HFF19w6dIlvvrqK9RqdY7PmZSUxLlz517Cq8nctWvXXun5hMiK3I/iTSH34j+PmZkZiUkQG68nOSXrCfX+/CuBXaejlXJ5ahX4V7andD4d4eHhWe6bEWtLC2ytzHhsTMZgyHq8f27I/SjeFHIvvlparQz1Ef8eKqPxOXl1r0BcXBwajQZLS0tlWc+ePQFYvHgxUVFRODk5KetOnjxJ27Zt+d///kflypVzdK7Tp09jNBopW7Zs3jT+ORISErh27RqlSpXCyirnswMLkZfkfhRvCrkX/5mMRoiK05Oge36N+5+P3OSnQzeUZRbmaro1c8O1pGOuzm1hrsHR1gKNJu972uR+FG8KuRdfvcuXL6NSqfDw8HjdTREiT7z2HnwAGxubdMvKlSvHb7/9hlqtNgnu09YB3L17N8cBPoBKpcLa2jp3jc0lKyurV35OITIj96N4U8i9+M+RnGLgcXQiRpUZlpaZf33IvMa9FyWccze0ztJcg6OdBRpNzrP2ckLuR/GmkHvx1ZH0fPFv83I/KbPh0qVLVKlShcOHD5ssP3PmDGXLlmXUqFF07drVZN3p06cBXlkvvBBCCPFfpk9K4VFUosk4+ozo9Cks3HjaJLgvnM+aUZ2q5Sq4VwHWlmY42Vu+9OBeCCGE+Dd47T34ZcqUwcXFhYkTJzJhwgScnJz43//+x4kTJ9i4cSM3btygf//+hISE8P7773P16lUmTpxIQEBApjPvCyGEECJvJOiSiIrVk/KcyfRi4vWErj/JtTvP1Lj/oDK2Vjmf7V6lAhtLc+xttNLDJoQQQmTTaw/w1Wo1CxcuZNasWQwZMoTo6Gjc3d1Zvnw55cuXp3z58sydO5fFixezZMkS7OzseO+99xgyZMjrbroQQgjxrxaXoCc6Lum5ZfAyqnHvVb4g3d+rmPsa99ZabK1l4ishhBAiJ157gA9QoEABpk6dmun6d999l3ffffcVtkgIIYT4b4uO1RObqM9yMj3IrMZ9MT5sWD6XNe5Tx+zbWElwL4QQQuTUGxHgCyGEEOLNYDQaiYzVkaBLfm5wf+bKQxZ/dxp90pOx+S3qlaFxjZK5SqtXq1TY22qxscx5Sr8QQgghJMAXQgghxN9SDEYiYxJJ1Gdd3x5g/8nbrP7pvJK+r1ar6Ny0AjUqvZWrc6tVKhxstVhLcC+EEELkmgT4QgghhCAp2UBkzPNnyjcajWzdf5UffruqLLPQaujT0gP30vlzdW61WoWjrRYrCwnuhRBCiBchAb4QQgjxH6fTJxMZqyM5Jeuc/JdR416jTu25l+BeCCGEeHES4AshhBD/Ydktg6fTp7B082lOX4lQlhXOZ83Atl4UcLTK1bk1ahWOdhZYauXriBBCCJEX5BNVCCGE+I+KjdcTE//8MnjRcXpCN5zkeh7VuIfU4N7JzgILCe6FEEKIPCOfqkIIIcR/jNFoJDpOT1xi0nNnys+oxn3lcgXp8X7uatwDmGvUONpZ5Hp/IYQQQmRMAnwhhBDiP8RgSC2Dl6hL5jmxfSY17ovyYUPXXNW4B9CaqXG0s8TcTJ2r/YUQQgiROQnwhRBCiP+IlBQDkTE6EpOeXwbv9OWHLNlsWuO+Zf0yNPLNXY17AAtzDU52Fmg0EtwLIYQQL4ME+EIIIcR/QFJyCpExuueWwYOMa9x3aVoB31zWuAew1GpwtLNEk8uefyGEEEI8nwT4QgghxL+cTp/M41gdKc8pg2c0Gvnxt6ts2f+kxr2lVkPvF6hxrwKsLM1wtLXIdc+/EEIIIbJHAnwhhBDiXyw+MYnouOeXwUsxGFjz0wX2nzKtcT+wrRfFC+euxr1KBTaW5tjbaCW4F0IIIV4BCfCFEEKIf6mYOD2xCXqeE9uj06ewZPNpzjxV4945vzUD2uS+xr1KBXZWWuxstLnaXwghhBA5JwG+EEII8S9jNBqJitMTn40yeBnVuC9TzIH+rStjk8sa92oV2NlosbWS4F4IIYR4lSTAF0IIIf5FUgxGImN06PTPL4N3/3E8wetO8CDySY177/IF6fZe7mvcq9UqHGy0WFvm7uGAEEIIIXJPAnwhhBDiXyL57zJ4umyUwbt6O4qwDSdNatzXr1KMtg3L57rGvUatwtHOAkutfL0QQgghXgf5BBZCCCH+BfRJqWXwklKeXwYv4xr3ZWnkWyLXk+GZaVQ42llikcuefyGEEEK8OAnwhRBCiH+4RH0ykTG6586UD+lr3GvUKjo3c8e3onOuz681U+NoZ4G5mQT3QgghxOskAb4QQgjxDxaXmER0rF4J2DNjNBrZsv8qP/5mWuO+TytPKpTKl+vza83UONlbYqZR5/oYQgghhMgbEuALIYQQ/1DZLYOXYjCwZvsF9p98UuPewVbLgDa5r3EPYGmuwdHOAo0E90IIIcQbQQJ8IYQQ4h/GaDQSGasjQZf83DJ4mdW4H9jWi/wOuatxD2CpNcPJziLXE/IJIYQQIu9JgC+EEEL8g6SWwUtEp095bhm8l1HjXgVYWZrhaGuR6wn5hBBCCPFySIAvhBBC/EMkpxh4HJ2IPvn5M+W/jBr3KhVYW5rjYKOV4F4IIYR4A0mAL4QQQvwD5KQM3suoca9SgY2lOQ62FrnaXwghhBAvnwT4QgghxBsuJ2XwTl1+yNI8rnGvUoGdlRY7G22u9hdCCCHEqyEBvhBCCPEGy6oMntFoNAna9524xZrt55WJ9/Kixr1aBfY2Wmys0gf3z57/dXvT2iOEEEK8alLXRgghhHgD+Pv7M2bMGJNlMXF6omN1GQb3+3/bw7RJ44HUwPaHfeEsWPYNV3ZO4dLWj3h4ZhPJF1fyzYJxuW6TWqXCwdYiXXB/9+5devfuza1bt9Lt069fPxYsWJDrc2bHzZs3cXV1ZdOmTQBER0czatQojh07pmzTqVMnOnXqlOH2mzZtwtXVlZs3b77Udr6Ip9v/otavX4+rq2u6PxMnTsyT478Mhw8fxtXVlcOHDwMwZswY/P39X3OrhBDizSc9+EIIIcQbICQkBFtbWyA1YI+K0xOfmJRpGbwN36wGICXFwOrt5zlw6g73z3yH1qYAZWp0oHurmqxZNifX7dGoVTjaWWCpTf9V4cCBA+zZsyfdcr1ez6FDhxg4cGCuz5sb586dY/PmzbRu3VpZNn78+Ey3r1+/PuvWraNQoUKvonmv3blz5yhdujTTpk0zWV6gQIHX1CIhhBAviwT4QgghxBvA3d0dSCuDp0OnT35uGTyD0UjYxlOcDU+tcW9IiqdQyYpMGtGO/A5WrMllWzQaFU62FlhkENxn5dixY9jY2FChQoVcnjnvlC1bNtN1+fLlI1++fK+wNa/XuXPn8PDwwMvL63U3RQghxEsmKfpCCCHEG8Df359Ro0fz5/kr+PlUZtevO/hs7CiavVOH5u82YOb0z0lISC15N3RAb06eOM7pk7/z7fwexD+8wsUfRwHw18lttGlWm7t3bqc7h16nY+WKJXRp34rG/jXpFNiCtatWYDA8mZBPjZFv/7eK1q1a4OnpiZeXF4GBgRw6dAhITW//6KOPAHj77bdNhhXs2bOHOnXqoFKplBTrgwcP0qlTJzw9Palfvz7r16/n/v37DBgwAG9vb+rVq8eKFSuUY2SWPp/REAZITeXu3LkzAJ07d1bS2rNKcX/2HOPGjWPy5Mls3ryZxo0bU6lSJZo3b87evXtN9vvjjz/o0KEDXl5e1K9fn6+++oquXbsq7Xo2rTzNs21JTExk1qxZNGrUiEqVKlGlShW6devGuXPnMmxvVi5evEifPn2oUqUKVapUISgoiBs3bijrjUYjFy5cyNVDl0ePHjF8+HBq1aqFh4cHzZs357vvvlPWb9q0CQ8PD44dO0br1q3x8PCgcePG/Prrr4SHh9OlSxcqV67MO++8w5YtW0yOffToUXr06EH16tWpVKkS/v7+BAcHm9yLQgghck4CfCGEEOINYDQa0elTlBr3c2ZMobDzW3w+dRYftu/Eth83s+qrpQB07DkYG6diWNgXoXitICwcivJ2u7EANA1oTsjCFeTLXyDd8T8ZPZRvVn9N04AWTJ4+h3oNGrJsSRhzZkwBQGumZsWyMBYtXMCHH37I0qVL+fzzz4mMjGTw4MEkJCRQv359+vXrB6QOK+jfv79yjj179lCvXj2T8w4bNgx/f38WLVpE6dKlGT9+PJ07d6ZcuXKEhYXh6enJ1KlTOXXqVK6uW8WKFRk3LnWegXHjxmWZmp+Vq1ev8tVXXzFo0CBCQ0PRaDQMHDiQqKgoAK5cuULXrl0BmD17NgMHDmTx4sUcP348x+caNWoUGzdupHfv3nz55Zd89NFHXLp0ieHDh2PMbExGJm0ODAwkIiKC6dOnM3nyZG7cuEG7du2IiEjN6vjrr7+Ii4vj9OnTNG7cmIoVK9K4cWOTQD0zI0eO5MqVK0yYMIElS5bg7u7O6NGjlYc9AMnJyQwfPpzAwEAWLFiAlZUVI0aMoG/fvtSvX5+FCxdSqFAhRo8ezd27dwE4f/48Xbt2xdHRkTlz5rBgwQKqVatGSEgI27Zty9nFFEIIYUJS9IUQQojXTKdPxmAwmgR3vn616TdgKABVqvlw7OhhDh3YR8P3O7N2TwQGlRa1GVg5laRB1WK0ebs8v6ydRIGChXGv5JHuHEcOHeD4scOM/WwK/g0bA1Cteg0sLCxZvnQBH7brQFWvijx8cJ+hQ4ea9DhbWFgwcOBALly4gJeXFyVKlACgQoUKFCtWDIAbN25w48YNatWqZXLe1q1b061bNwCsra1p27Ytnp6eDB48GAA3Nzd+/vlnfv/9dzw9PXN87WxtbZV0/LJly2aZmp+V+Ph45s2bh6urq9LWjh07cujQIRo3bsyiRYuws7Nj6dKlWFlZAeDi4kJgYGCOzqPX64mLi2Ps2LE0bdoUAB8fH2JjY5k2bRoPHz6kYMGC2TpWSEgIVlZWrFixQpm/oWbNmjRs2JClS5cyevRoJSvg5s2bjBkzBjMzM7777jtGjx6NXq+nbdu2mR7/yJEjBAUF0bBhQ6Wdjo6OaLVPJl00GAz07duXNm3aAKkTHg4dOpQuXboo77udnR2tW7fmzJkzODs7c/78efz8/JgxYwZqdWpfU61atfj11185fPgwzZo1y8klFUII8RQJ8IUQQojXxGAwcumvx9yJiCMpxWAyoV7FiqZBesGChblx4yaz1/xOUvKTNOZWDcryjs/za9yf/OM4Go2Geg0amix/p3FTli9dwKVzJ/Gt6sGsWbOA1PTs8PBwrl+/zq5du4DU4DQze/fuxdvbGzs7O5Pl3t7eyr/z588PQOXKlZVlTk5OAMTExGTZ/pfN3t6e4sWLKz87O6eWFkwbFnHo0CHq1q2rBPeQ+tqKFi2ao/NotVqWLVsGwL1797h69SrXrl3L8hobDAaT1HWVSoVGo+HQoUP4+PhgaWlJcnIykPrAo1q1ahw4cACA6tWrs3DhQnx9fbG2tgagTp06PHr0iPnz59OmTRsMBoPJw6W04/v6+hIcHMyff/5JnTp1qFevHqNHj07Xvue9x46OjkBq8A/QokULWrRogU6n4+rVq1y/fp1z586RkpJCUlJSjq6nEEIIUxLgCyGEEK/B2fCH/HTwGlduRhEVpyMmPolz1yK4cjMSAAtLS5PtH0QmEBWrI//fwb1KBYXz29DIt2S2zhcdE4WDoyMajcZk+VuFU2eSj42NBeD06dNMmDCB06dPY2VlRdmyZSlSpAhAlunje/bsoW7duumWp/UsP+3pIPlN8XSvNKA8MEkLrB89eqQEr0/LzUz0+/btY8qUKYSHh2NjY4Obm5sSfGd0jT/++GO+/fZb5eeiRYvy66+/EhkZydatW9m6dWu6fdImEcyfPz8NGjRIt75evXocOHCAhw8fMmzYMI4cOaKs8/HxYeXKlcyZM4eFCxeybds2tm/fjlqtxs/Pj4kTJ5o82Mjpe5yYmMjnn3/O5s2bSU5OplixYnh7e2NmZpajIQpCCCHSkwBfCCGEeMXOXHnImp/Pcy8iHrVahb2NFhUQn5jMd3uvmGybVuM+/FaUssxSq6FoQVusLc2zfU57OweiIiNJSUlRgnxLcw2x0Y+B1J702NhYevbsiaurK1u2bMHFxQW1Ws2ePXvYvn17psfW6XQcPnyY4cOH5+AqZOzZwDpNXFzcCx/7RTg7O/Pw4cN0yyMiInBxcQGybruNjQ2QOiY+Le190aJFFC9eHJVKxerVq9m3b1+G5x4wYAAdOnRQfk57GGFnZ4efn5+SCv80M7PUr3jHjh3jxo0btGzZ0mS9TqdDo9Hg4ODAhAkTTK5vWlvt7OwYOXIkI0eOJDw8nF9++YWwsDAmTJjA4sWLs7haWZs8eTLbt29n7ty5+Pn5KQ83atasmetjCiGESCWT7AkhhBCvUHKyga0HrnLnQRwW5mq0ZmrUKhUqVeokd7qkFACMxtQa9yu3nmPrgWvK/g62WkZ0qIqtjUWOzuvpXYWUlBT27NoJpD4kcLK35McfvgegatWqhIeHExkZSefOnSlbtqwyPjptNvm0wDVteZrDhw/j6OiojF9/EWm9wWkTskHqBHeRkZGZ7vNsVsLLUL16dfbt24dOp1OW/fnnnyaz/WfU9qioKK5cefLQ5syZM+h0Onr37k2JEk+GVqQF9xn1YBcrVgwPDw/lT9p19vHx4fLly1SoUEFZV6lSJVasWMGOHTuA1KEFY8aM4erVq8rxDAYD27dvx9vbG61Wi4uLi8nxXVxcuHXrFvXq1eOnn34CUucb6NWrF35+fty+nb5CQ04cP34cX19fGjZsqAT3Z86c4dGjRzKLvhBCvCDpwRdCCCFekZQUA2fDI7hw/TFmZur04+ZVKqz/rj1//1G8SY17AI1axehO1cnnYImtrR1/njnF78ePUK6cG3b29lme27dGLbyqVGPW9ElEPn6It2cljh07ypIlS2jZsiVly5YlJiYGW1tbFi5ciJmZGWZmZmzfvp0NGzYAT8aj2/99rh07dlC3bl327t2bYXp+bvj6+mJpacm0adMYPHgwcXFxzJ8/XxnHnZG0cf+7d+/GwcEBNze3PGnL0/r27cvWrVvp2bMn3bt3Jzo6mnnz5qFWP3kfXV1deeuttwgNDcXW1haVSsWiRYtM0tUrVqyImZkZM2bMoHv37uj1ejZt2sTu3buB1Mn+sqt///4EBgbSp08f2rVrh4WFBevWrWPnzp3Mnz8fgMDAQL755hv69u3L4MGDsbKyYs2aNVy8eJHVq1dneuyiRYvi7OzMpEmTiI2NpUSJEpw5c4Y9e/bQp0+fXFzBJzw9Pdm2bRtr166lTJkynD9/ngULFqBSqZR7TAghRO5ID74QQgjxCiQlG3gUncj9x/Ek6pMx02Q8KZ7m7+UHTt82Ce7trLU42GrJ55A6Nr9F67ZozMz4aMQgjhza/9zzq1Qqpn4xlxatPmD9N6vp27cPP/30E8OGDWPKlNQyeXZ2doSFhWE0Ghk8eDCjRo3i9u3brFq1ChsbG44dOwakBuF+fn7MmjWL6dOns3fv3nTl8XLL3t6e4OBgUlJSCAoKYt68eQQFBVGpUqVM9ylXrhwBAQGsXr2aESNG5Ek7nlWyZEmWLVuGTqdj0KBBzJkzh169elGwYEElpV2j0TB//nwKFCjAsGHDmDx5Ms2aNaNRo0Ymx5k1axb37t2jX79+Som/lStXolKplGucHW5ubqxevRqVSsWoUaMYNGgQDx48IDQ0VDlngQIFWL16Na6urkyaNIkhQ4aQkJDAihUrTCbCy0hISAh16tRh3rx5dO/enbVr1zJgwACCgoJyevlMjBkzhoYNGzJ37lz69OnD+vXr6devH23btuWPP/4gJSXlhY4vhBD/ZSrjf2w2k9OnTwPg4ZG+hNDLEB8fz7lz56hQoYKShibE6yL3o3hT/NfuRV1SCpExiSSnGLl+J5qFm05hodWgNUv/nD0+MZlH0YkYnvp0ruJaiG7vuWNulvtUdBVgaWGGk53Fc2fczwtGo/GVnCcvPH0/WllZZdjugwcPYm5uTrVq1ZRl0dHR+Pn5MWrUKDp37vwqmyz+pdLuRTc3N+XBkXi5XnVsIMTLJj34QgghxEuUoEvicXRqcA9QvLAdhfNbE5eYlG68daI+mYgo0+C+QdVi9Gxe6cWCexVYW5m/suA+LCxMKQX3T7Jp0yamT5+e4bqzZ8/SvXt3VqxYwdGjR9mxYwd9+/bFzs6OgICAl9amCxcu0KJFCypVqkTTpk1f2nletz///BNvb+8sSzE+bcyYMfj7+ys/u7q6Ehwc/LKa90rt2rWL2bNnP3c7f39/xowZ8wpa9HLp9XoWLlxIkyZN8PLyonHjxoSEhGT7XhBCmJIx+EIIIcRLEpegJzouCcNTgbxaraJJjVKs3n6eyDg9NhbmmJmpiI1PIjLW9AttdmvcZ0WlAjsrLXY22udvnEfmzZvHgAEDXtn58srSpUupUaNGhuvSxsuvXbuWO3fuYG1tjY+PD1OnTlVK0r0MoaGh3L59m9DQ0Jd6ntdtz5491KhRI125wv+ib7/99j9VUWDSpEl8//339O/fHw8PD06fPq3c92nDh4QQ2ScBvhBCCPESxMTpiU3Qm/TGp3ErlY8Ojd346dA17kXEExmbTKL+ybhjjVpFl2bu+FR0fqE2qFUq7GzMsbWSoOlFqdVq+vfvT//+/V/peR8/fkz58uXzbI6DN9XevXtp3rz5626GeMUeP37M//73P0aMGEHPnj2BJ+USZ82axYgRI/7VD7aEeBkkRV8IIYTIQ0ajkchYHTGZBPdp3ErlY2AbL9xL5zcJ7i0tNAxs6/XCwb1GrcLRTmsS3J85c4YuXbpQtWpVvL296dq1KydOnFDWHzt2jI4dO1K5cmV8fHwYPXo0jx49UtZv2rQJd3d3Tp48yYcffoiHhwcNGjQwScdPK+EWEhJiUjbv4sWL9OnThypVqlClShWCgoK4ceOGsv7w4cO4urpy8OBBunfvTuXKlalVqxYzZswwmXRNr9czd+5c3n77bTw9PQkICODbb781ee07d+6kVatWeHh4UKtWLSZNmvTc2ekHDRrEnTt3+Pbbb3F1deXmzZvK612/fj21atVSytKlpKSwePFiAgIC8PT0xMvLi8DAQA4dOqQcLzg4mHfeeYfdu3fz3nvvUalSJRo3bsx3331nct6vvvqKJk2a4OHhQZ06dfjss8+IjY1VruWRI0c4evQorq6ubNq0CYBr164xaNAgatWqhZeXF506deL48ePKMW/evImrqyvLly+nSZMmVK5cmY0bNxIcHEyTJk3YsWMHAQEBeHh40Lx5c/744w9OnDhBmzZtlGt68OBBk3Zm9/375ptvaNCgAVWqVGH//v08evSI4cOHU6tWLeV8z16DqKgoTp48qVRiyM71zY1OnToxbtw4wsLCqFOnDpUrV6ZXr148fPiQjRs38s477yj/L54uf9ipUyfGjBnDwoUL8fPzo2rVqvTv359bt24p22T3/Y6MjGTcuHH4+fnh4eFB27ZtTa5106ZNefjwIT/88INyH2YlKSmJSZMmUb16dapVq5bu/yzA+vXradWqFV5eXnh6etK8eXO2bdumrDcYDMyZMwd/f38qVaqEv78/s2bNIikpSdlGp9PxxRdfUK9ePSpVqsR7773H1q1bs2zb3bt3qVChAqtWrTJZ/ujRIypWrMiKFSuIjY0lMDDQZLgFpJZlBEzuMSFE9kiAL4QQQuSRFIORR9E64hOSeN4UtikpBlb/dJ79p57UFHewtWBEh6q4lXqxHiuNRoWTnQVWFubKstjYWHr27ImTkxPBwcHMmTOHhIQEevToQUxMDEePHqVr165YWloyd+5cPv74Y44cOULnzp1JTExUjmMwGBgyZAhNmzZl8eLFVKlShS+++EKp475u3ToAPvjgA+XfV69eJTAwkIiICKZPn87kyZO5ceMG7dq1IyIigqeNGDGCqlWrsnDhQgICAli6dCnr1683Wb98+XLatGnDokWLqF27NmPGjOHHH38E4IcffiAoKAgXFxdCQ0MZMGCAkv6b1bzCw4YNo0CBAtSrV49169ZRqFAhIDXY/PLLL5k8eTIfffQRZcqUYebMmYSFhfHhhx+ydOlSPv/8cyIjIxk8eLBJmbcHDx4wceJEOnfuzOLFiylWrBijR4/mypUrAPz444/MmDGDDh06sGzZMoKCgti8eTOff/65ci3d3d1xd3dn3bp11K9fn8uXL9OqVStu3rzJ2LFjmTlzJiqVii5dunDkyBGT1xQcHEyvXr344osvqFWrFpAadE2bNo2+ffsyb948oqOjGTRoEMOGDaNNmzaEhoZiNBoZOnSo8r7n5P0LCQlh9OjRjBs3Dm9vb0aOHMmVK1eYMGECS5Yswd3dndGjR5sE67/99hsuLi4UKVIEINvXNzd+/PFHDh48yOTJk/nkk084ePAgHTt25Ouvv2b06NFMnDiRkydPMnHiRJP9fvnlFzZt2sTYsWOZMGEC586do1OnTjl6v3U6HV26dOGXX35h6NChhISE4OzsTM+ePZUgf/bs2Tg6OlK7dm2T+zAz27Zt4+zZs0ybNo3Ro0eze/duevXqpTwUW716NePGjaNhw4YsWrSImTNnotVqGTFiBHfv3gVgyZIlrF27lqCgIL788kvatWvHsmXLWLBgAZD60DIoKIhvvvmGbt26sWDBAry9vRk6dGi6BxhPc3Z2xsfHhy1btpgs/+mnnzAajTRr1ozixYvz2WefKQH909fb3NycUqVKZfn6hRDpSYq+EEIIkQdSUgw8jtGhS3p+ia9EfTJLvjtjUgbPOb81g9p6K2Xwcstco8bRzgKtuemkfJcvX+bx48d07tyZKlWqAKm9ZOvWrSMuLo5Zs2ZRunRpFi1ahEaTum/lypVp1qwZGzdupEOHDkDql/3+/fvTpk0bAKpWrcqOHTvYvXs3derUwcvLK/X1ODsr/w4JCcHKyooVK1Zga2sLpKbhNmzYkKVLlzJ69GilnW3atFHKsNWsWZOdO3eye/duAgMDuXjxItu3b+fjjz+mS5cuyja3bt3i8OHDNGvWjJkzZ1KnTh1mzpypHLNUqVJ07dqVPXv2UL9+/QyvW6lSpTA3NydfvnxKu9P07dvXZL/79+8zdOhQOnXqpCyzsLBg4MCBXLhwQdk/ISGByZMnKynHpUqVokGDBuzZs4cyZcpw5MgRihUrRocOHVCr1fj4+GBtbU1UVBQAXl5eyvVKO+bEiRPRarV8/fXXyrr69esTEBDAF198wYYNG5Q2vfvuu7Ru3drktSQkJDB+/Hilt/zy5cvMmjWLyZMn88EHHwCpM7kPGjSIq1evUqFChRy9f+3bt6dJkybKz0eOHCEoKIiGDRsC4OPjg6Ojo8lY+7179yrtycn1zY3k5GRCQkJwcHAA4Oeff2bfvn3s3LmT4sWLA3DixAk2b96c7rpt2rRJ2cbFxYWWLVvy3Xff0a5dO2WbrN7vzZs3c/78ef73v/8pJQrr1q1Lp06dmDlzJhs3bsTNzQ0zMzOcnJyy9TqdnJxYtmyZUo3EycmJoKAg9u7dS4MGDbhx4wY9evQwGVpStGhRWrVqxfHjx2nWrBlHjhyhUqVKyr3i4+ODlZUVdnZ2ABw4cIB9+/YxZ84cZaLHOnXqkJCQwMyZMwkICMDMLOOQonnz5nz88cfcvn1beYCzZcsW/Pz8KFiwYIb77Nixg2+//ZaOHTsq75MQIvskwBdCCCFeUFJyCpExOvTJhuduGx2nI3T9Sa7fjVGWlS3mSL/WnthYmWex5/NpzdQ42lli/nf5PYPBSPitKKLj9JhbFyJfvnz07duXJk2aUKdOHWrVqsXIkSNJSEjg5MmT9OjRA6PRSHJyMgDFixenTJky7N+/XwnwAby9vZ+cU6slX758WabAHzp0CB8fHywtLZVj29raUq1aNQ4cOGCy7dPHhtQHBWnHTktDf7quPKDMnn7lyhXu3r1Lnz59lPMAVK9eHVtbW/bv30/9+vVN1gFZ9uwDVKhQweTnWbNmAampxuHh4Vy/fp1du3YBpJv5++kgzdk5ddhF2uupUaMG69ato1WrVjRs2JB69erx3nvvZTmp4pEjR2jQoIESaAOYmZnRrFkzQkNDiYuLy7TdadIe8AAUKFAAQAk4ARwdHYHUMoCQs/fv2XP6+voSHBzMn3/+SZ06dahXr57JAwGj0ci+fftMZo3PyfXNSEpKisl7qlarUatT/0+UKVPGJGgsUKAATk5OSuCe9vpjYp78/4TUa/b0Nu7u7hQvXpyjR48qAT5k/X4fPHiQggULUrFiRZN7sEGDBnzxxRdERUVhbp7+d8Cz9+vTr6devXompUb9/f0xMzPj6NGjNGjQQJllPzo6WrmWhw8fBp5cS19fX2bNmkX79u3x9/enfv36dOzYUTnmwYMHUalU1KtXz6Qt/v7+fP/991y6dAlXV1cMBtPff2ZmZjRq1IgJEyawdetWevbsyZ07dzh+/DgzZsxI9zoh9YHL8OHDqVq1KiNHjsxwGyFE1iTAF0IIIV6ATp/M41gdKSnPyckH7j2KJ/h/J3gY+SStNy9q3EP64P7kpQds+PUSt+7HkpxiwEyjxuf9EcRd38O2bdtYt24dlpaWNG/enD59+mAwGFiyZAlLlixJd2wLCwuTny0tTbMM1Gp1lkFyZGQkW7duzXDM7rMTaGV17MjISADy58+f6XkAJkyYwIQJE9Ktv3//Pjdv3uTtt982WT5hwgTKlSuXafufDqAgtW72hAkTOH36NFZWVpQtW1bpnXz2OlhZWZm8lqe3adq0KQaDgTVr1hAWFkZwcDBFixZlxIgRmZbEi4qKUoLypxUoUACj0aiM38+o3WmefjiQUTuflZP379lzzpkzh4ULF7Jt2za2b9+OWq3Gz8+PiRMnUrRoUU6fPo1Op6Nq1arKPjm5vhl55513TMbHt2zZkmnTpmX62jO7Tk8rXLhwumX58+dXsi3SZPV+R0ZG8uDBAypWrJjhOR48eKC8zqc9u/2AAQMYOHAgQLpecLVajZOTk/Jw5q+//mLcuHEcPHgQc3NzXFxccHNzM2lXz549sbGxYePGjcycOZMZM2ZQrlw5xo4dS40aNYiMjMRoNJo8GHra/fv32blzJyEhISbLL1y4gK2tLQ0bNmTLli307NmTrVu3YmVlpWR0PG3FihVMnz4dHx8fQkND0/3eEUJkjwT4QgghRC7FJyYRHacnJavZ9P529XYUoetPEpvwZOKqBlWL0ebt8qjVL1abXmumxsneEjPNk+A+dMNJEhKTsbMxx1xjTlKKgUfxNliVfJ/FQz6GhDts3ryZtWvXUrhwYVQqFV27dqVZs2bpjp9V8JcddnZ2+Pn50a1bt3TrMkvtzYi9vT2Q2rOb1jsKqT33kZGRyvpRo0bh4+OTbn8HBwcKFSpkksYOqYHanTt3stWGtLkMXF1d2bJlCy4uLqjVavbs2cP27duz/VrSBAQEEBAQQExMDL/99htLlixh5MiRVK1aNcOg0sHBgYcPH6Zb/uDBAyA1Rfv+/fs5bkdWXuT9s7OzY+TIkYwcOZLw8HB++eUXwsLCmDBhAosXL2bv3r34+fkpPdd5cX0XLFhg0tPv5OSUg1ebscePH6db9vDhQ0qUKJHtY9jZ2VGqVCmT4SNPK1asWLpecCDd/fr0uPy0h1ppUlJSePz4Mfnz58dgMNC7d2/Mzc3ZsGEDFSpUwMzMjMuXL5sMQVCr1XTo0IEOHToQERHBnj17WLhwIQMHDmT//v3Y2dlhbW3N119/nWG7S5YsiZubW6bDX95//3169+7N9evX2bJlC40bNzb5nWI0Gpk8eTIrV64kICCAqVOnSrlEIV6ABPhCCCFELmRVBu9Zpy49YMnmMyQ9lcKfFzXuIX1wbzAY2fDrJRISk8nvYKEc/9HVE5zZt4aKTUazaU84E3rVxNvbmy1bthAREYG7uzvh4eF4eHgox05MTGTQoEHUq1ePsmXLZrtNaT2XadJmn08LMCD1S/2IESMoWbJkpqnkz0rr5f31119p3769snzmzJncu3eP9evXkz9/fm7evEmPHj2U9ffv32fUqFEEBgZSokQJk9cIqSnUd+7cUeYeyEp4eDiRkZF07tzZ5Jrs3bsXIMMALTNDhgwhKSmJ0NBQ7OzsePfddzE3NycoKIj79+9nGOBXr16dXbt2ERsbq/RGp6SksGXLFjw8PF5KYJTb9+/WrVu0b9+ejz76iCZNmuDi4oKLiwsnTpzg+vXrQOp1S5vPAfLm+j5dvSGvHD9+nMePHysPC86cOcPNmzeV+SKyw8fHh927d5M/f36TnvqFCxdy7tw5Zs6cicFgSPf/59n79Wn79+8nOTlZeV+2b99OcnIyvr6+PH78mKtXr/Lxxx+bHOPZaxkYGEilSpUYO3Ys+fPnp1WrVsTExDBlyhRiY2Px8fHhyy+/xGg04unpqRxn48aN7NixgylTplC4cOEM71eA2rVrU6BAAb7++mvOnj2bLvV+9uzZrFy5km7dujF69OgX/p0oxH+dBPhCCCFEDhiNRqLi9MQnPn+mfIB9J26xZvt5Zdu8qnEPqcF9PntLNJonAUH4rShu3Y/Fzsbc5Iuyk3NZMBoI37+MxMcN+fateE4e20tMTAyNGjXC39+f3r17M3z4cN5//31l9viTJ0/muPa7vb09v//+O0ePHqVatWr079+fwMBA+vTpQ7t27bCwsGDdunXs3LmT+fPnZ/u4bm5uNGnShBkzZpCYmEiFChXYu3cvu3btIiQkBI1Gw9ChQxk3bhwajYYGDRoQHR1NWFgY9+7dyzQ1Oo2dnR1//vknR44cMQlknla6dGlsbW1ZuHAhZmZmmJmZsX37dqWXNSezvNeoUYPx48czffp06tatS3R0NCEhIZQqVUpJo37WgAED2Lt3L507d1Z6Z1etWsWNGzdYunRpts+dE7l9/4oWLYqzszOTJk0iNjaWEiVKcObMGfbs2UOfPn149OgRZ86cUeZQgLy9vnkpISGBnj170q9fP+Li4pgzZw7ly5cnICAg28do1aoVq1atolu3bvTt25e33nqLAwcOsGTJEjp27Ii5uTlJSUlYW1tz/vx55T58dtjK0x48eMDAgQPp1KkT165dY/bs2dSqVYuaNWuiUqkoWrQoq1evxtnZGXt7e/bt26f0xKddy+rVq/Pll19SoEABvL29uXfvHsuXL8fHx4d8+fJRr149qlevTv/+/enfvz9lypTh1KlTzJ8/nzp16jy3Tr1Go6FZs2asWrWKwoUL4+vrq6w7d+4cS5YswcPDgyZNmnDy5EmTfcuWLZvhsAohROYkwBdCCCGyKcVgJDJGh06fzPNie6PRyA/7wtl64JqyzNJCQ9+Wni9cBg/AwlyDk52FSXAPEB2nJznFgLnGdLIuSxsHqjcbzIUjm7l8cA3jD36Na/nyBAcHU6NGDQCWLVtGSEgIgwYNwtzcnIoVK7J8+fIcz1ret29fwsLC6NWrF1u3bsXNzY3Vq1czZ84cRo0ahdFopHz58oSGhqYbD/88M2bMICQkhK+++orHjx9TpkwZ5s+fr4zpbdOmDTY2NixdupR169ZhbW1NlSpVmDlzpskkaRnp1KkTs2bNokePHixfvjzDbezs7AgLC+OLL75g8ODB2NjYKLW+e/XqxbFjx9LV9M5MYGAgSUlJfPPNN6xZswZLS0tq1qzJyJEjM5xsDaBcuXKsWbOG2bNn89FHH6FSqfD09OTrr7+mWrVq2TpvTr3I+xcSEsLs2bOZN28ejx8/5q233mLAgAH07t2bH3/8kXLlypn0/Obl9c1L1apVo0aNGnzyySdA6gRzo0aNylHGhLW1NatXr2bWrFnMmDGDmJgYihYtyvDhw+nevbuyXUBAAGvWrFHuw6ze1/bt2xMTE0NQUBBarZb33nuPkSNHKg/3wsLCmDx5MmPGjEGr1VK2bFkWLFjAlClTOHbsGJ06dWLw4MFotVo2btyoZJP4+/szfPhwIDUjZ/HixcybN49FixYRERFB4cKF6datW7YzGJo3b85XX31FQECASYbCzz//jNFo5PTp03z44Yfp9vv6669NHggIIZ5PZczObCX/IqdPnwayTnfKS/Hx8Zw7d44KFSpkaxIXIV4muR/Fm+KfeC8mpxhSg/tslMFLq3F/4PSTcd0OthYMbFuZYoXsXrgtmQX3AJdvRDJlxREsLTRYmKdPOdfpU0jUp/BxVx/KFnd84bb8G/wT70fx6qSV61u5cuVLP5fci6/eq44NhHjZ0n8zyMSxY8fo27cvzZo1Y8iQIZw5cybdNufOncvxk3ghhBDiTadPSuFRVGK2a9yHbTxlEty/VcCG0Z2q5Ulwb2muwemZtPynuRR1oGghW2Lik9LNOG40GolJSKJoIVtcikp9aSGEEOLfJlsB/sGDB+ncuTO3bt2idOnSHDx4kMDAQNauXWuynV6v5/bt2y+loUIIIcTrkKhP5lF0Ikkp2atxP3vN75wNj1CWlS3myIiOVcnnkPk42uyyNNfgaG+JJotZ99VqFR/4l8PKwoyIaB06fQoGgxGdPoWIaB3WFmZ84F/uhWfuF0IIIcSbJ1tj8IODg2nYsCFz585FrVYTHR3N2LFjmThxIkaj0WQmWyGEEOLfIi5BT3RcEoZsjGa79yie4HV/8DAqUVmWVzXuIXvBfZrK5QoS9EFlNvx6iVv3Y4lNSMJMo6bUW/Z84F+OyuUKPvcYQohUryI1Xwgh8kq2AvyLFy/Sv39/ZVIMe3t75s2bx4gRI5g8eTIFChSgUaNGL7WhQgghxKsUHasnNlGfrZnyw29FEbrhJHHP1rhvWB51HpR8stRqcLKzzFGve+VyBfEoU4DwW1FEx+mxt9HiUtRBeu6FEEKIf7FsBfhWVlbExcWZLFOpVEyfPp0HDx4wcuRIChQokK36sUIIIcSbzGAwEhWnI0GXnK3gPqMa960blKVhHtS4h9wF92nUapVMpCeEEEL8h2RrDH6VKlUICwvjwYMHJsvNzMwIDQ2lSJEi9OnTh99+++2lNFIIIYR4FVJSDDyOTiQ+MXvB/b4Tt1iw6ZQS3GvUKnq8X5F3fEvmUXBvluvgXgghhBD/PdkK8IcPH86jR4/w9/dn9uzZJuvs7OxYvnw5+fLlIzg4+KU0UgghhHjZkpINPIpOJDEbM+UbjUa+33uF1T+dVx4EWFpoGNjWi+ruznnSHisLM5zsLCS4F0IIIUS2ZStFv0SJEvzwww989913FC5cON16Z2dnNm7cyPz589mxY0eeN1IIIYR4mXT6ZCJjdSSnPL/bPiXFwKqfznPwqTJ4jnYWDGiTNzXuVYDl38F9XmQBCCGEEOK/I1sBPoCjoyNdu3bl0aNHGa63tbXl448/5uOPP86zxgkhhBAvW4IuiahYPSmG5wf3ifpkFn97mj+vPvksfKuADQPbeOVJGTwVYGVphqOtBPdCCCGEyLlsB/hp6tatS506dWjevDn+/v5otdqX0S4hhBDipYuN1xMTrycbsT1RsTpC15/kr3sxyrJyxR3p29oTG0vzF26LBPdCCCGEeFHZGoP/tBEjRhAREcGQIUOoVasWn376KceOHXsZbRNCCCFeCqPRSFSsjuhsBvf3HsUzY+Uxk+C+ilshBn3oJcG9EEIIId4YOe7B79q1K127duXGjRv8+OOPbN26lfXr11OkSBHef/993nvvPcqUKfMy2iqEEEK8MIPBSGSsjkRdMtmI7TOsce9frTgfvF0uT2rcqwBrK3McbLQS3AshhBDiheS4Bz9N8eLF6devHz/88AM//PAD9evXZ8mSJQQEBORl+4QQQog8k1YGLyGbwf3JSw+Ys/Z3k+D+A/9ytG1YPm+Ce5UE90IIIYTIOznuwX9aREQE27ZtY9u2bfzxxx84OjrStGnTvGqbEEIIkWeSklOIjNGh/7tm/fPs/eMma3++oJTB06hVdA1wz7MyeCoV2Fia42BrkSfHE0IIIYTIcYAfExPD9u3b2bJlC0ePHkWj0eDv709YWBh16tRBo9G8jHYKIYQQuabTJ/M4VkdKNsrgGY1Gvt8XzrYD15RllhYa+rXyxLVkvjxpjwT3QgghhHgZchzg16xZE4PBQNWqVfnss89o0qQJtra2L6NtQgghxAuLT0wiOi57ZfBSUgys3HaeQ2dMa9wPbONF0UJ581mnAqwluBdCCCHES5DjAH/gwIG89957FClS5LnbHj16lIoVK2JtbZ2rxgkhhBAvIiZOT2xC9mbKT9Qls/g70xr3RQrYMKCtF/nsX7zGPTyZLd/BRkrMCiGEECLv5XiSvT59+mQruE9JSaFz585cvXo1Vw0TQgghcstoTJ0pPyabwX1UrI7Za343Ce7LFXdkRMeqeR7cSyk8IYQQQrwsLzTJ3vMYjdmZo1gIIYTIOykGI5ExOnT67M2UfzcijuD/nSAiKlFZVtWtEF0DKmJulutiMyYkuBdCCCHEq/BSA3whhBDiVUpOMaQG90kp2do+/FYUoetPEJeYrCx7u3pxWvvnTY17SA3uLS0kuBdCCCHEyycBvhBCiH8FfVJqGbyklOyVwTtx8QHLvj9D0lNl8z7wL0dDnxJ52i5LCzOc7CS4F0IIIcTLJwG+EEKIf7xEfTKRMbpszZQP6Wvcm2lUdGmWdzXu01hqpedeCCGEEK+OBPhCCCH+0eISk4iO1WPIxrwvGdW4t7Iwo28rT1xLOuVpuyzNNTjZWaBWS3AvhBBCiFdDAnwhhBD/WDkpg/cqatyn0ZqpcbS3lOBeCCGEEK+UBPhCCCH+cdLK4CXokslOwZZXUeM+jdZMjZO9JRoJ7oUQQgjxikmAL4QQ4h8ltQxeIjp9SrbK4EXF6ghdf5K/7sUoy8oVd6Rfa0+sLc3ztG3mGjWOdpaYafKmvJ4QQgghRE68UIAfExPD/fv3KV68OBqNBo1Go6zTaDRMnTqVYsWKvXAjhRBCCMh5GbxXUeM+jUajwtHOIs+PK4QQQgiRXbkK8A8fPszMmTM5c+YMKpWK9evXs2TJEpydnRkzZoyyXcuWLfOsoUIIIf7bcloGL/xWFKEbThKXkKQsa1i9BK38y+ZZjfs0GrUKJ1sLtOaa528shBBCCPGS5Lib4eDBg/To0QNLS0tGjBiB8e/Bj25ubnz99dcsX748zxsphBDivy1Rn8yj6MQc1bifs/Z3k+D+A/9yfPB2uZcS3DvaWWChlVFvQgghhHi9chzgz507l7fffpuVK1fSpUsXJcDv27cvPXv2ZP369XneSCGEEP9dcYlJPI7OuMa9MYMZ9vb+cZNF354iKTn1YYCZRkXP5pVo6FMiz9umUatwsNVimcPgPqN2CyFenPzferPJ+yPEy5fjAP/cuXO0bt0aANUzvSC1atXi1q1bedMyIYQQ/3kxcXqiY3UZ1rhfuWIp69auVH42Go18t+cKa7ZfUGbWt7IwY2Bbb6pVKJznbUvrubeyyNlEfevXr2f69Ol53p4XceHCBVq0aEGlSpVo2rTp627OS/Pnn3/i7e2NXq9/3U3Jtps3b+Lq6sqmTZted1PeOP7+/iZDQ8PCwli2bJnyc3BwMK6urhlu/0+4rs+2/5/s7t279O7d+7lxwqZNm3B1deXmzZuvqGUvz9mzZ+nVqxc1atTA19eX7t27c/bs2dfdLPEfkOMA387OjgcPHmS47s6dO9jZ2b1wo4QQQvy3pZXBi8mixv3ypQtITEgAUmvcf7XlT346eE1Z72hnwYiOVXEt6ZTn7dOoVTjZWeS45x5gwYIFREZG5nmbXkRoaCi3b98mNDT0jXv4kJf27NlDjRo10Gq1r7sp4iWYN28eCX//TgBo06YN69aty3DbQoUKsW7dOurXr/+KWvffduDAAfbs2fO6m/HKXL9+nY4dO5KYmMjkyZOZOnUqer2e9u3bEx4e/rqbJ/7lcvzN5O2332bOnDmUL18ed3d3ILUn/+7duyxcuFB+UQohhHghBkNqcJ+oS85WGbwMa9wXtGFgGy+c8rjGPTwJ7l/nmHuj0Zgui+5FPH78mPLly1OvXr08O2ZG8rrdObV3716aN2/+2s4vXi1nZ2ecnZ0zXKfVavHy8nq1DRL/GStXrsTKyopFixZhbW0NQI0aNfD392fVqlWMGzfuNbdQ/JvluAd/+PDh5M+fn7Zt2yrB/LBhw2jSpAkqlYphw4bldRuFEEL8g5w5c4YuXbpQtWpVvL296dq1KydOnFDWHzt2jB49etC1a1fq1avH6NGjefQoNThPSTGwas06atWowp9nTzOgT1ca+9cksHUz1q35WjmGf+2qAHy9fDFN3/ZVgntd9F0iT6/k8LqRdPqgEZ9+NJzbt56kep74/Rj+tavy+7EjjBzan3ff9qP1+41YHDaflJQnpfeSkpL4ckkYHdq8TxN/P7p3asv2bT+YBPc7d+6kVatWeHh4UKtWLSZNmkR8fHyW18bf359bt27x7bffKmmomzZtwt3dnfXr11OrVi18fHy4fPkyKSkpLF68mICAADw9PfHy8iIwMJCPPvpISUMODg7mnXfeYffu3bz33ntUqlSJxo0b891335mc96uvvqJJkyZ4eHhQp04dPvvsM2JjYwFwdXXlyJEjHD161CRl+dq1awwaNIhatWrh5eVFp06dOH78uHLMtBTn5cuX06RJEypXrszGjRsJDg6mSZMm7Nixg4CAADw8PGjevDmzZs1i+PDhtGnTBk9PTwICAjh48KBJOy9evEifPn2oUqUKVapUISgoiBs3bijrDx8+jKurK9988w0NGjSgSpUq7N+/n0ePHjF8+HBq1aqlnO/ZaxAVFcXJkyepW7cuAJ06daJfv37Url0bb29vOnTowNChQ9OlEO/cuZP27dvj7e1NpUqVaNKkCatXr35umyB1OEarVq3w8vLC09OT5s2bs23bNmXftHTkkydP0rJlSzw9PXnvvff46aef0t07Dx48YNCgQXh7e+Pj48Onn35KXFycyb01ZcoUunTpgqenJ5988gkA58+fp3Pnzri6uuLu7k6dOnWYNGkSiYlPSkfu37+ftm3b4u3tTfXq1alXrx516tRR1ru6ujJ06NAs7/cxY8bg6upK3bp1TcZZu7q6snbtWsaMGUOlSpWUbRITE5k+fToVKlSgUqVKfPLJJ+h0OpN769n0+TFjxuDv758uNd/V1VVJZQ8JCVH+/XSKe6dOnbh165bSi/zsOTZt2oSbmxuurq7UqFEDDw8PGjRoYJLyD3D//n3atGmDq6srVatWZdy4ccyZMwd/f3+T9gQHB5vs17Nnz3Tp9uvXr6dFixZUrFgRNzc33NzceOedd9i0aRNJSUlkR9r9d/DgQTp16oSnpyf169dn/fr13L17l+bNmyvH9vX1ZcqUKcTGxir7/fbbb3To0AFPT08aNWrEmjVrTI7v6urK6tWr+eSTT/Dx8cHb25vBgwfz8OFDk+2y+n24adMmPvroIyC1o/Dp9y4zv//+uzJsKCAggK1bt5qsv3nzJqNGjaJ27dpUrFiRmjVrMmrUKB4/fqxs87zPIkj9PBo3bhzt27fHx8fH5PMoM59++im1atUy+cwAmDx5Mr6+viQlJeHi4kL37t2V4B7A2toaZ2dn/vrrr+e+fiFeRI4DfAcHB9avX8+ECROoXr06fn5+uLq6MnLkSDZt2kS+fPleRjuFEEL8A8TGxtKzZ0+cnJwIDg5mzpw5JCQk0KNHD2JiYjh69Chdu3bF0tKSQYMGMWLECI4cOULnzp2JiY1XZso3GgxMGDeGBm83YuqMeXh4erEobB5HDx8AIGThCgAKlalB8VpBAOhjH3DrUBiOVimM+WQCI8Z8yp3btxjUvzuPH5t+YZs8cSyelb2Z/MVc3n6nCd+s+YqtP3z3ZP2ET1j/zSqavteCKV/MpbpPDaZP/oyD+3ZioTXjhx9+ICgoCBcXF0JDQxkwYADff/89/fv3z3ISqZCQEAoWLEi9evVYt24dhQoVAiAlJYUvv/ySyZMn89FHH1GmTBlmzpxJWFgYH374IUuXLuXzzz8nMjKSTZs2ER0drRzzwYMHTJw4kc6dO7N48WKKFSvG6NGjuXLlCgA//vgjM2bMoEOHDixbtoygoCA2b97M559/DsC6detwd3fH3d1dSVm+fPkyrVq14ubNm4wdO5aZM2eiUqno0qULR44cMXlNwcHB9OrViy+++IJatWoBqeNtp02bRt++fZk3bx7R0dEsXbqUXbt20aZNG0JDQzEajQwdOlQJNK9evUpgYCARERFMnz6dyZMnc+PGDdq1a0dERES66zh69GjGjRuHt7c3I0eO5MqVK0yYMIElS5bg7u7O6NGjOXTokLLPb7/9houLC0WKFFGW7d27l4iICLp168YHH3zA+fPn6dSpk5LmvXv3boKCgqhYsSJhYWEEBwdTvHhxJk6cyMmTJ7Ns0+rVqxk3bhwNGzZk0aJFzJw5E61Wy4gRI7h7967Jvn369OHtt98mJCSE0qVLM2TIkHTpzPPmzeOtt94iLCyMLl268L///Y+QkBCTbVavXo2HhwdhYWF88MEH3L9/nw4dOnD37l0qV67M0qVLadasGStXruTrr1MfmN24cYP+/ftTqVIlFixYwOTJk4mNjSUiIgKD4UnViq1btz73fler1dy7d4/ff//dpF0zZsxAq9Xi4OAAwL1792jRogV37txh7NixtGvXjg0bNrBy5UqyIyQkhP79+5ss++CDD5S/M0vLB3j06FG66w+g1+uV11KsWDEWL15MlSpV+OKLL9i3b5+yTZcuXbh27RoAo0aN4vz583z55ZdZtnfLli3KQ580afeHRqNBrVYTEBBAqVKlePToEZs3b2by5MlZX4RnDBs2DH9/fxYtWkTp0qUZP348LVq04MKFCzRt2pQqVaoQGRnJxo0bGThwoPJahw4diru7O6Ghofj5+TFhwoR0Qf6cOXMwGAzMnj2bUaNGsWvXLqZMmaKsf97vw/r169OvXz8g4/cuI+PGjePdd98lLCyMcuXKMXToUHbu3AlAQkICnTt35sqVK4wfP55ly5bRuXNntmzZwpw5c4DnfxYByueRVqtl2LBhfPzxx8rn0dMPwJ7VvHlzHj58yOHDh5VlBoOBbdu20axZM8zNzWnfvj09e/Y02e/69etcunSJcuXKPff1C/EicpVfqNVqadu2LW3bts3r9gghhPgHu3z5Mo8fP6Zz585UqVIFABcXF9atW0dcXByzZs2idOnSzJ8/n4sXL1KhQgV8fHxo1qwZq9as4/2WbYDUVO7O3XrRNKAFAJU8vNi3ZxcHD/xGdV8/LJ1SZ8RPVtvi6FQSAPNHB7CztWHWvAXY2NgCUKWaDx3avs+6NV/TN2iI0s5m77WgU9deqdtU9WH/3t0cPLCP91q05mr4Zfbu/oWgQcNp3bY9ANV9fXkUcZ/fjx+jRfP3mTlzJnXq1GHmzJnKMUuVKkXXrl3Zs2dPpsPV3N3d0Wq15MuXL116cN++fU32u3//PkOHDqVTp07KMgsLCwYOHGgS8CYkJDB58mRq1qyptKNBgwbs2bOHMmXKcOTIEYoVK0aHDh1Qq9X4+PhgbW1NVFQUAF5eXtja2ir/Bpg4cSJarZavv/5aWVe/fn0CAgL44osv2LBhg3L+d999V5l89+k2jR8/Xuktv3z5MrNmzcLV1VX57hAfH8+gQYO4evUqFSpUICQkBCsrK1asWKGcs2bNmjRs2JClS5cyevRo5fjt27enSZMmys9HjhwhKCiIhg0bAuDj44Ojo6PJWPu9e/cq7UmTkpKCh4cHgwYN4ty5c7Rq1Yp27drx3Xff0a5dOy5fvkzLli2V3nAAb29vfH19OXz4MJUrV860TTdu3KBHjx4mwUzRokVp1aoVx48fp1mzZsryTp06ERSU+qCqTp06tGzZktDQUJMhE40bN1Z6QWvWrMn+/ftNHmAAFClShBEjRig///bbb1SoUAG9Xk+rVq3w8/PDz8+P/fv3c/jwYXr37s2pU6dITEykT58+FC6cOhnlhg0bOH78OPHx8djY2ABQokSJ597vb731FkajkW3btlG1alVl27Jly9KqVSs2btxIuXLluHLlCklJScycORMzs9SvoocOHUr3YCAzaUNEn5aWiu/s7Jxp6r25uTkGg4GffvpJuVfSnDt3DgB7e3vKli1LzZo1qVq1Kjt27GD37t3UqVOH77//nvDwcIKCgggNDaVWrVo0a9Ys3bHSREREMG/ePNatW4eFhYWSoQCp90eHDh1YtWoVI0aMoGfPnpw9e5ZWrVrh6+vL5s2b6dGjR7auB0Dr1q3p1q0bkNpT3LZtW2JjY2nXrh3jx4/n8ePH1KhRg7fffpvNmzfTqFEjAN555x3l/q5Tpw73798nLCyMdu3aKcNpypcvz9SpU5VznTp1SskyMRqN2fp9WKJE6u/sChUqUKxYsee+noEDByqvv27duly7do2wsDAaNmzItWvXcHZ2Zvr06RQvXhxITX8/efKk8gDyeZ9FdnZ2yufRRx99hEajwcPDg8qVK9OsWTM2btxIhw4dMmxb1apVKVq0KD/++CN+fn5AaibFgwcPMh0ClJiYyOjRo9FqtXTs2PG5r1+IF5HjAP/Zp8VPU6vVWFtbU7JkSWrVqiWT2AghxH+EwWAk/FYU8UZHHByd6Nu3L02aNKFOnTrUqlWLkSNHkpCQwMmTJ+nRowdGo5GUlBSSk5MpUMiZEiVLc/TIQSXAB3Cv6Kn8W6vV4ujoSGJCAicuPmDZ92eUdSrgg7fLEfb5eby8q2JpYUlKcjIANtY2eHh6c/zok54WAPdKniY/FyhUiMTE1F7b06dOAFCnXmrKrUajwsnOkrDQ1M+/K1eucPfuXfr06UPy3+eB1C/VGo2GAQMGYG5ujqenJ4MGDVKCwD/++IN58+Zx+/ZtfvzxR4xGo0nQeuvWLdzd3Vm7di1Tpkzhzz//5Pfffyc6OpqaNWty/fp1Pv74YyA1ANuwYQMDBgwAwNbWlj59+nD06FHleHfu3AFSv/iuW7eOChUq0KJFC8LDw7l48SK2trbcvXvXZGidXq8nLCyM7du3AxAYGEiPHj1o2bIlZmZmNGvWjJCQEFq0aMHly5cB+Ouvv4iPjzdJRQWUL9WAkub8xx9/4Orqyi+//MLp06cB+P777+nZsycRERHUrl0bc3NzFi5cyA8//MCNGzdISkpi7dq11KtXTwk4Ll++rAQms2bNIikpiTlz5rBz504+/PBDZejHV199xccff8ytW7dISkqiQYMGxMbGYmtrqwQCp06dwtvbmz59+tC3b1+cnZ1ZuHAhISEhxMXF4eHhwf79+3FwcOCvv/7iwIHULJLDhw+zadMmJaX/8uXLNGnShOHDhzNv3jyuX7+Oi4sLNWrU4Pbt2yxevFjZ9s8//zQJ8L28vEzev8KFC3Pu3DmTXsStW7fi6+vLokWLiIqKwtPTk9u3bzN8+HAOHTrEw4cPsbOz47vvvqNFixYA1K5dGw8PD2rWrMmQIUP45ZdfOH/+PDdv3uTGjRt4enqiUqlQqVS8//77vP/++9StWxcHBwccHBywtbVVMkFKly5tcr9Xr14dW1tb9u/fT/369TEYDBiNRho3bszWrVv5+OOPUatTE0W9vb3ZunUrfn5+JCYmYmZmRsWKFTEzM8Pf3195IPPw4UNcXV2V8cnjx49n2rRpNGrUSLn3AWWfnFKpVBQuXDjDAP/UqVMAWFo+mbcj7WFcWqr5oUOHKF68uEmAamtrS4MGDUx6c9MsXLiQ3377jeDgYEJDQzl//ryybsyYMdy4cYP4+HiKFCnC5s2blWPY29sDKA/hAOV3JoBOp6Nu3bomw2W9vb1JTk6mdu3aJssCAgIAcHJKnWzUysoKSH2ACNCyZUuTNjdq1IhffvmFq1ev4uLiApDugYmzs7OS5RIeHp7h78Nn749nGQwGkwwRQHnYA6Sr5tGwYUOCg4OJi4ujQoUKrFmzBoPBwLVr17h+/TqXL18mPDxcaUO5cuXIly9fhp9FgMnnEaB8HhUvXpwyZcqwf/9+OnToYPKaIDXWUavVvP/++6xZs4bPPvsMrVbLli1bKFWqlMlDvzSxsbEEBQVx+vRp5s2bR9GiRdNtI0ReynGA//3333P37l30ej1mZmY4OjoSGRlJcnIyKpVKSfkpW7YsX3/9taTsCyHEv9zJSw/Y8Oslbt2PJTnFgEudIB5d/oUff9zCunXrsLS0pHnz5vTp0weDwcCSJUtYsmRJuuNoLSxMfn76izaASq3mzsNYFn17SimDp1ar6NG8EtUqFGZaVCS7fvmZXb/8nO7Yjo6mM+lbWpgeW61SY/x7uv7ov79UOzrlU4J7C3ONsm3aDPgTJkxgwoQJ6c7l7e2t9Po+3cOq0Wjw8/PDycmJ0qVLK6mgaT30Wq0Wg8HAkCFD6Nq1K82bN2fu3LmEhISwePFiXF1dlZ7jevXq0b9/fyV1uEuXLri4uDB9+nSSk5MZMmQIGzZsoG/fvjRt2pSLFy+yYMECZVx6gQIFqFixIkuXLlV6wABGjBihpIa/++67FCpUiDFjxmBubk5AQAC3b98GUlOYO3fuzEcffcTp06fp378/y5cvN5lAL60XHlJ7qIODg/H19WXYsGHK0ASAbdu2MXnyZPr168e+ffvw9DR9+AKpAc3gwYOZMWMGkNoTmzY0oV+/fjg5OTFlyhROnz7N6dOnUavVlCtXjvDwcEaPHo2ZmRlTp07l0KFDfP7550yfPh13d3du3bpF0aJFGT16NAkJCVy5coX79+9jbW3NpEmT0Ol0TJs2je7du6NWqylVqhQVKlQAUgP8CRMmcOfOHYKDgzE3N1eGJgwdOpS4uDgmTZqkBBCFCxfG19eXgwcPsnbtWgYOHKi8voEDB5q8f2m9pdevX1d60CG1k2Xs2LEkJiby22+/8fvvvytDE8aPH4+1tTWjR4/G2dmZGjVqYDAYGDNmDAaDgW7duvHWW29hYWFBQkICxYsXZ/Lkydy7d4/Zs2fz4MED1q9fz9dff425uTkWFhap1Sz+vt/37NlDxYoV07039+/f5+bNm2zevBmA5cuXAzB37lzl4ZGNjQ0bNmxg2LBhbNy4ESDdA6GnpaVZd+rUCQcHB+bMmaMEqC+qWLFiHD161KQiVGxsLBcvXgTSl4BWq9XKd9vHjx+TP3/+dMfMaBmkPiAbNWoU5ubmhIaGmqz766+/GDduHAcPHuT777/HxcUFNzc3IHW4Stp37DTffvutksGR5scff1SyPNKC6cePH/POO+/w7bff0rJlS5NMCkCZwT3tIUVa1sazr+XphwtpDwUyuibP+32Y9iDhWaGhoek6DS9cuKD8u0CBAunaZTQaiY2NxcbGhuXLl7Nw4UIiIyMpUKAAlSpVwsrKSkm/t7GxYfXq1SxYsIBt27aZfBaNHTuW6OjoLD+PLP7+PHr2nh8wYAADBw6kefPmLFiwgH379lGnTh1+/vlnunTpku44d+7coU+fPly9epU5c+Zkmu0hRF7KcYA/ePBg5YlqkyZNlP/kv/zyC+PGjWPcuHGUKVOGYcOGMXv2bCZNmvQy2i2EEOINcPLSA0I3nCQhMRk7G3PMNeYkWRfB3DaQktXb08jDnLO/72Ht2rUULlwYlUpF165defvttwm/dhOHfAXQalO/SFlYZD7jvdFoJD4xmau3o3B+EhviW9FZqXFva2dHlaq+tG2XPv1Ro9GkW5YZW9vUcq8xMZEUKVRSCe6vXLlCZGSk0rs2atQopRfx4sWLfPzxx0yaNAlfX1+cnZ1xcHBgx44dNG3alNmzZ5OcnMyiRYt45513KFGiBJMnT6ZZs2YcO3bM5HX279+fd999l7fffpvy5ctz8uRJ2rRpw7hx49izZw+HDx/GyckJLy8vJcB/NrV9yJAhJCcnK6ntNWvWZMGCBfTo0QMPDw+WLFnC/v37eeutt9i9ezeQmjJ/5MgRPv74YxYtWoSFhQVjxozh1q1bHD58mGbNmvHzz6kPT2bPnq18cW/fvj3Lli3LcmhC2rh3BweHdL2B77//PvXr18fBwQE/Pz+ioqIoV66cSQ/30aNHmT59usmEe88OTXB3d6dBgwb07NkTR0dH5s6di7m5OR06dCAsLIx69erxzjvvKIFL2rWytbXF09OTc+fOsXjxYiA1Jfjdd9+le/fuWFhY8NZbb5E/f342btzI5cuX2bJlC+XKlaN169YmPbdpQxNq165N06ZNsbe35+HDh3z++ee0bduWy5cv06xZM+Li4rh69aqyn4WFhcn7d/bsWZYsWcKmTZtMhmg8PQzg0KFD6HQ6mjRpQsOGDZkyZQpVqlShYcOGSgbl4sWL2bVrF7Vr12bOnDnY2dkxfPhwdDodzs7Oyv2bNvRj+fLlJCcn88knn3Dr1i1++uknypYtC4Cfn1+GEyk7ODhQqFAh6tevz9mzZ1mwYAFBQUEmk53dvn2byMhIGjZsqAT4WfH19WXnzp2ULl2aNm3asH//fnbv3k3p0qWfu+/zODs7Y2VlZTLHwY4dO7CxsTEJajNSuHBhZfz9056dIwJSe4TLlCmj/Px0T7DBYKB3796Ym5uzYcMGKlSogJmZGZcvX2bz5s1cvHiRwMBAk4ecDRo0MBkac+bMGT777DPlwQSkjvV3cXFJN5nf044fP06DBg2UAP/x48dK+vzTryWzhxbPyuj34dPS5l141tPZBxmJiooyCfIfPnyIRqPBwcGBH374gWnTpjFy5EhatWqldCYOHjxYyQyC1JT8GTNmkJKSwqlTp9i8eTNr166lRIkSBAYGKp9Hadcr7V6HJw81nr7mgPJwsnTp0nh6erJt2zbUajXR0dG8//77JtteuHCBHj16oNPp+PLLL6levXqmr1eIvJTjSfaCg4MZMmQITZs2VVKvVCoVDRs2ZNCgQcybN49y5crRt2/f/1S9SyGE+K8xGIxs+PUSCYnJ5HewwMJcw72rv7Nv9ShszRJJTDJw6o4F48aNx97enoiICNzd3QkPD6d46fLkL1yMsuXcKFXahRXLFnLyj2MZnif57xr3CbonX5Cd7CxQqdXkd3jSs1TZqyrXr4VTtmx5XN3ccXVzp7xrBdZ/s4rf9u7K9uvy8PQC4MTRgyY99zNnzmTy5Mm4uLiQP39+bty4iZVjcZLMC1PeoyaOjo6MHz+e8ePHs2fPHmrVqsUXX3yBr68vly5dwt/fXykTZzAYlFTQtFT3NN7e3oSHhxMZGUnXrl3Jnz+/kg67d+9egHQT+fn4+GBpaUlycrISSBQpUoQDBw4wZMgQ5s6dC6QGTu+++y79+/cnOTkZR0dHJf04bVb9Ro0aUb16dXbt2kVsbCzBwcF8/vnnXL58mdjYWIoUKYJarVbShUuXLq30HgJK2m1aW7KadBCgZMmSymu4fPkyixYtYsyYMRQtWhSdTsfnn3/O+vXrlWM+zcvLi1u3blGvXj1ldmwrKyt69eqFm5sbCQkJtGrViv/973+UKVOG9957zyRgjomJMZmt/NChQxgMBmUG+ePHj9O4cWNat27N2bNniYuLU1L7MwuAqlSpwuPHj7l69So1atQAUt9TePL+ASYTJRYpUkR5/5KSktizZw+Ojo7pxtinZQ+ksbCwIDg4mEGDBhEXF4dOp2P06NHK8Ihjx46hVqvp2bMndnapD65GjRrFo0eP0Ol0HDt2jGHDhilpy0ajkZo1ayrtvX37tpKmHR0djYeHB+7u7lSoUIH8+fMzc+ZMzpw5g1arxcnJCa1Wi4eHBy1atGDPnj3Ke3/p0iXq169vktWRlbT237t3D0gNyuPj45U0+sykfS/Nikajwd/f3+Q76pYtWzLMHEkbdmAwGEhOTqZatWrcvHlTyWSB1LHVaQ/a0tja2iptT/P0Q4C0++ODDz7Aw8NDSU1P60l2dnZmyJAhyvbJycnY2dlRoUIFKlSogIeHB4GBgRQpUkS5R/R6PTt37sx0DHhaBQx7e3uT8fRpE9el+emnnyhatKhJ0J+VtN+HN2/exMPDQ/lTuHBhZs2axZ9//gmkf28KFy5ssr2Hh4fJ+rQHj4Ayb0LlypWxtLTk+PHj2Nvb07NnTyW4j4uL4/jx48rvn59++okaNWrw4MEDNBoN3t7efPbZZ9jb23P79m1sbW2Vz6OyZctStmxZPDw8KFeuHMHBwcqDu2fb+HTGQ/Pmzdm3bx9btmyhSpUqJtlQd+7coVu3bqhUKtauXSvBvXilctyDf+fOHeXD+FlFixZVxpcVLlz4uU9ChRBC/HOF34ri1v1Y7GzMlbRWJ+eyGI0Gft++kBIe73D2gZahIzYTExNDo0aNqN+gAX379GH8p5/gWsGDe3du8t2m/3HuzzN06tIr3TkSdcks+vY0566lr3HfbYsdZ0+f5OSJ3/Gs7E2nrr0Y2LcrH48awvstP0Cr1fLD5k3s37eb8ZO+yPbrKu/mSqPGjZkzeybJSToqVKjA3r172bVrFyEhIWg0Gj5o35PFIV+w+/dbOLzljjFFRzIWoIrl1KlTHDhwIMuhCU+nND874ZSlpSWFCxfG1taWhQsXotfruXv3Lp9++qnSm/RsoLt169Z0ZaSuXbtGvnz5lEm2IDULITExkZCQEEqVKoWNjY0ShKUdM3/+/AwYMIC9e/fSuXNnpacxLCwMSA36nk5bHTt2LPAkVTttu7Rtng4mMpKWCtu/f38CAwNp3749kZGRXL9+HbVajcFgUHr9n31YYGVlRdGiRXF2dlZm9b558yZffvklFy5coFGjRty7d4979+6xYMECvv/+e0aMGKGM7zUYDFy4cIFdu3Zx+fJloqOjcXJyUsYte3p68sMPP1CvXj2MRiNhYWGsWrUqw7aksbW1xdbWlqJFiyplAM+dO8e3336rzFz/rDNnzmSY/v5sUPRsWnu+fPlo1qwZ27ZtIzIykm3bthETE8PEiROV65KSksIff/yBWq3m+vXrBAcHo9PpOHnyJD179qRYsWLK5G8nTpxAr9crwWCDBg2U7JczZ84wadIktm7dahKs2tvbpxsv3bRpUxYtWqRMmnflyhV69+6d4WvPiJOTE97e3qxcuZKSJUty//59Hjx4gLW1dZap/fb29vz+++8cPXqUatWqZbrdu+++q8xfERcXx8GDB+nbt2+6QP3jjz/m9u3bbN68mc2bN1OkSBHKlCmjzPZ/8OBBfvjhByIiIkyqM9SvX58tW7ZQuXJlSpYsyaZNm0xKGubPn5+iRYuyevVqnJ2dsbe3Z8GCBUpQ6e/vr/y/gPRp4r/88gvFihXjvffeU2a8P3HiBPHx8bz33nvp7s2tW7cqpelatGhhMtxh+fLlWFhY4OXlxc8//8yuXbuYNWtWptfuWRqNhqFDhyoVARo0aEB0dDRhYWHcu3dPaXtaT/+OHTuoW7euSXZDRubOnUtKSgpvvfUWa9eu5erVq8rwD09PT9auXcu0adNo0KAB9+/fZ9myZTx8+FDJGKhSpQoGg4GgoCB69+6NjY2N8v8jbYLBYcOG0bt3b1JSUqhbty4PHjzgyy+/5OTJk9ma6b9p06ZMmzaNrVu3Kr9j00yaNImIiAgmTJhAbGysSXk+W1tbk2wBIfJajgP8smXLsn79SZ5ltgAAo45JREFUepP6qGk2bNigpE9du3bNZIydEEKIf5foOD3JKQbMNebKMksbB6o3G8zFI5s599tqkpP1JJQuQ3BwMNWq+xAZo2P6rBCWL1vI/n0haLVayrtVYOacMNwrmfbgxMTpWfbT79y4F6Mss7fRMqJDVawtzenQuTsrVyzloxEDWb5qA2XKlmNu6FK+XBzG1M/HYTQaKe1SholTZ1Grdj2yQ6NRkc/OklkzZxISEsJXX33F48ePKVOmDPPnz6dhw4acvPSA8zElcKnZhfsXf+Xe5f1ozCywKVAar3o9GNm9MSTcUdJBnx6a0KxZM3777TeWL19OXFwc48ePJyIiQulhT2NnZ0dYWBhffPEFERERHD9+HLVazapVq2jfvr1JDyJAs2bNlBm0IbVUWJs2bejQoQMVKlTg0qVLrFq1itmzZ2NjY0PNmjUZOXKkST3qtEDu0aNHlCtXjjVr1jB79mxlIsC0NNa0VNz79+/Tv39/goKCaNCggZKq3aZNG9avX688jChWrBi7dj0/g8LNzY0lS5bQtWtXDAYDlpaWuLq60qdPH8zMzLIMEENCQpg9ezabNm1i8+bNFCtWjAEDBtC7d29+/PFH4uPjCQoKYsmSJYwcOVIZl2xnZ4e9vT0TJkwgJSUFrVaLn5+fkuI+bdo0Pv/8c7Zs2QKkBnRDhw5l6tSpGaZqPy0sLIwhQ4YQERHBxIkTcXV1ZcGCBXz66afKBIhpvLy8ePjwIffu3cPFxYV27dpRqVIlk0nHMqJWqxk5ciQjR46kTp06FCxYkN9//50JEyawePFiChYsSPHixVm9ejULFy5UOl+KFSvGvXv32L17N46OjoSFhTFv3jxCQ0NRqVRYW1tToEABpfceUmfx//3334mKisLW1hY3NzfatWuX4Yz1bm5ulC5dWplpPTk5Oct07IykXfuxY8diNBoxNzenS5cuSjZHRvr27UtYWBi9evVK98DraXXr1sXa2pq4uDjOnj1LsWLFMpz4bMCAAfz2229UrFiRAQMGoNVqsbe3JygoiMjISKZNm0aLFi1wdHQ0GXLx0UcfkZyczPTp0zEzM6Np06ZUrFiRP/74Q9kmLCyMyZMnM3r0aFJSUtDr9cr/q7SJDdNklibevHlzFi1aBKTOCVG9enWKFi3KzZs3lW2XLVvGjBkz8PHx4fDhwyZzOkDqQ4xvv/2WRYsW4eLiwvz582ncuHGm1y4jbdq0wcbGhqVLl7Ju3Tqsra2pUqUKM2fOVHq1fX198fPzY9asWRw8eFAZDpOZqVOnMm3aNK5fv0758uVZsmSJMgSgZcuW3Lx5k40bN7JmzRoKFy5MvXr1aN++PZ9++ilXrlyhTJkyLF26lHnz5vHJJ5+QkJCg9M6nZdbUrl2bZcuWMX36dGbOnImFhQUVK1Zk+fLlmVZieFq+fPmoXbs2+/fvN6mgodfrlQyEZwN/SM1Wym5JSCFyxZhDu3btMrq7uxtbtGhhDAsLM65bt84YGhpqbNmypdHd3d24c+dO49mzZ40+Pj7G6dOn5/TwL92pU6eMp06demXni4uLMx47dswYFxf3ys4pRGbkfhR56dJfj43dJm439pu+0zhk9q50f/pN22nsNnG78fKNx0adPtl4LyLOePN+jPHm/Rjjpev3jT/9etB46fp9ZdnTf46du2vs/NlPxoBh3yl/PltywHj1dlSG2+fFnzsRsUadPjnL15ySYjCOXbjf2HHcNuPgWb8qr/XDoJnGip5VjG1HrTOOXbjfmJJiMBqNRmO1atWMEyZMMLZs2dLYq1cvk2MlJCQYe/XqZVy1apXRaDQaN27caCxfvrzxxo0bJts1aNDAOHr0aOVnNzc34/z585WfBw4caHzvvfeMSUlJyjKDwWAcNmyYcd68eUaj0Wg8dOiQsXz58sZDhw6ZHLtjx47Gjh07Go1Go/HcuXPG8uXLG1evXm2yTd++fY0tW7Y0JicnG2vWrGn87LPPTNbfu3fP2KVLF+O2bduyvHZvv/22yevI6PWePHnSWL58eeOOHTtM9p04caKxfPnyxgMHDhiNRqNx/vz5xvLly6c7R/ny5ZVrM3jwYGP//v1N1u/YscNYvnx546lTp4wdO3Y0Vq1a1dixY0fld2NQUJDR19fXGBMTo+yTnJxsbNSokbF169ZGo9FovHHjhrF8+fLGjRs3mhw7ozZl9Bqffi/S1jdp0iTH79//2bvv8KbKBQzgb9I2TUdKy1CQXTpooYtR9pRRSpmCLEEUGVKWyhAuigVRRIbQwZYhIGU52IKylSWWIQiUvVdn2maf+0fogXTRdKbl/T3PfS6cJCdf0kPNm/Od771z547QsmXLTO/7yJEjhc6dOwuCIAi9e/cWNm7caPb7O2nSJKFNmzZZvq/ZyfiYBQsWCC1bthSmTJkiTJw4Udz+4jEnCKbHd3bvbcZ9Z/w3kZvxZXzeiRMnCn379hUGDx4svs9Z7ftFly9fFnbv3i1s3rzZ5Of61ltvCaGhoTk+f8bXkG7OnDmCh4eH8PXXXwsGg8Hs/0736NFDmDJliuDr62vysxYEQfjxxx8FDw8PYdy4cYJarTa5LbvfCa+ios4GRIXN7DP4rVu3xooVKxAeHo6IiAjo9XpYW1ujfv36WL16NRo0aIA//vgDnTt3Nrl+iIiIShfXymVQ+TVH3LifBJmT1GT1aUEQkJymhUc1Z7xRwQFxSSroDTlfi53u6p0ERG0+gxTV82no7QKroWcbN0gzrHBdUNLP3Mtscl6ML6vLEgDjpQkQDLh2dAVU8e3wU6VUnDl1SJwO2rZtWwwbNgyffPIJunbtCr1eb9ZU0BdlnIacPrV9+PDh6NevH2xtbREdHY19+/Zh4cKFud5v7dq1ERQUhG+//RYqlSrLSxNyMxU3p3FfuHABJ06cyPJ6Z+D59fyLFy+GtbU1rK2tsWfPHvEMZvpaBLnRuHFjTJs2Dd988w1atmyJpKQk8dKE9NXKMxo+fDiOHj1qcmnC2rVrcfv2bSxfvjzXz22ue/fumf3zS5+C/+WXX0KpVKJatWo4f/48Dh48iOHDhyMuLg7nz59HeHi4+JiCfH9fJjg4GJGRkfjll1/EyzYKU0xMDFatWpVpe4sWLbKcDh4cHIzhw4dDKpWKl5m8TGpqKsaOHYtGjRoBME4jv3nzJs6dOwcfHx+sWrUKUqkUgwYNytX+Ll68iGXLlsHHxwdBQUE4c+YMVCoVbty4Aa1Wi7p167503YJu3bqJMwVePIv8+PFjfP3116hcuTIGDBggXguf7sU1IIiodDE74APG/2g2btwYGo0GiYmJKFeunMl1Ym3btkXbtm0LbJBERGR5pFIJerV1R+TmM3iapIbCzgY21lJodQYkp2lRvowcXZq7IiFZA8NLFlpL98+lR/h+27/Q6owLJaV33L/ZMHcLPuVFehXey8I9kPVlCcDzSxMunfgFsX+tx7S/1sDTw8NkOuiKFSsQERGBMWPGwMbGxqypoC/KOA25du3aWLduHebPn4+JEydCEAR4eHggMjISb775pln7/vbbb7O9NAHI3VTc7Lz//vv46quvMGTIEPFa2oxevDRh7NixcHBwgJeXF9auXYuhQ4fi1KlTuf580bdvX2i1WmzYsAHr16+HXC4XL02wsbHJ8jG1atUSL02YPHkyJBIJfH19sWbNmhyv6c6v+fPnY/369Wb//NIvTViwYAHi4+NRqVIlk0sT3N3dTRYFK8j392Xc3Nzg4eGBx48fo2nTpgWyz5wcOXIER44cybTdxcUly4DftGlTODk5oVKlSi+9Hjydn58fvvvuO7Gucdu2beJt6dfCW1lZ5Trg//bbbxAEAefOnUOfPn0y3b5mzRrxy4TshISEYPbs2WjTpo24kCJgrDVUqVS4e/cuBgwYkOlxQ4dmXvOEiEoHiSDk8lPXC9RqNS5dugSNRiMu5GEwGJCWloZTp05h/PjxBT7QgpJen5Fxtc7CkpqaiosXL8LLyyvHhWGIigKPRyoMZ648xuY/ruDuIyV0egOsraTwrO6C9oHVUKm8A7I6cZ/eOV6rVi2xjujA37cRvfcy0u9ubSXB4JA6Yg1eYbCSSuDiZNpzn5PY2wn4atUJyG2tsnyMWqOHSqPHlMGBcKvq/NL9Cc9W1S9pSuq4s8PfjWQpeCwWvaLOBkSFzeyavOPHj6NVq1bo06cPBg4ciEGDBmHQoEEYPHgwPvzwQ2zYsKEwxklERBbKz70CwoY2wZTBgRjXtx6mvh+Iod18UDGbcJ+RIAj46UAsNrwQ7u1srTGmT0Dhh3uFba7DPfD8soTkVG2mlarTL0uo/JojXCtn3f38oqioKKxYscLscRe3TZs24ZtvvinuYZi4dOkSunfvjrp162Za0b00uXDhAgICAqDRaHJ1/08//dTkjLynp6fJlP3sHuPp6YmWLVtm2xQwZ84ceHp6mlQOtm3b1mTRxtzI+JjcjG/gwIHw9PRE3759s73PRx99BE9Pz5eOZ+vWrfD09DRZlC4rgiBgxYoV6NChA3x8fNCxY0esW7cux8fkllKpxDfffIN27drB398fvXv3xt69e8W6NyIic5k9RX/+/PlwcXHBjBkz8Ouvv0IqlaJnz544dOgQfvzxR7ECiIiIXh1SqQRuVZ2h1xuQkKxGmkb38gfB2HG/avsFHP/3gbjNRWGLUW/7o3KF3HVm54UY7mXm/WfwZZcl2Ntao1dbd0ilLz+7vWDBArGqqyRZtGiRuJq1pYiMjMS9e/cQGRkp9mKXRgcPHkTjxo3FVf4Li1QqxcOHD3H69GmxceBFWa1QHxERkeue+/w8Jn18MTExePDgASpWrGhyW2pqaq5aG8wxe/Zs/PDDDxgzZgx8fHxw6NAhTJ8+HdbW1llOrc8tQRAwbtw4nDt3DmPGjIGrqysOHTqElStXQi6Xcy0rIsoTswP+pUuX8OWXX6J9+/ZITk7Ghg0b0KpVK7Rq1QparRaLFi16afUFERGVPlqdHgnJamh0uTvzpNEasOzXi7h8K1Hclt5x7+IkL6xh5jncp/Nzr4DQXn7iZQnKNC2sraSoUckJvdq6w8+9QgGPmF4mPj4eHh4eaNUqd3WIJdWhQ4fQrVu3Qn+eSpUqQRAE7Nq1K1PAj4mJwcOHD+Hh4WGy3dvb2+znyctj0h8XGxuL3bt3Y/DgwSa37d+/H3Z2dmLven7duXMHq1atwmeffYb+/fsDAJo0aYL79+/jyJEj+Qr4Fy5cwOHDh/Hdd9+hU6dOAIzX+d+8eROrV6/G2LFjS9WlMERUNMyeom8wGMQFW6pXr44rV66It3Xs2DHTKp1ERFT6qTU6PE1S5TrcJyo12PpXvEm496jmjPED6hd6uHd+Sbg/f/483n33XdSvXx8BAQEYPHgwYmJixNtPnTqFb8M+wq9RH+LML/+D7MEuhHZ3R9jQJvBzr4CtW7fC29sbZ86cQZ8+feDj44M2bdqYTMdP75SPiIgQ/wwAly9fxvDhw1GvXj3Uq1cPoaGhuH37tnj78ePH4enpib/++gvvv/8+/Pz80KxZM3z77bfQ6/Xi/TQaDb777ju8+eab8PX1RUhICH766SeT17lv3z707NkTPj4+aNasGb788kukpqbm+P61bdsWd+/exU8//SRObU5/vZs2bUKzZs0QGBiI2NhY6PV6LF26FCEhIfD19YW/vz/69u2LY8eOifsLDw9H+/btceDAAXTp0gV169ZFx44d8fPPP5s87+rVqxEUFAQfHx+0aNECX3zxBZRKpfhenjhxAidPnoSnpye2bt0KALhx4wbGjBmDZs2awd/fHwMHDsTff/8t7vPOnTvw9PTEypUr0aNHDwwePBi//PILwsPDERQUhL179yIkJAQ+Pj7o1q0b/vnnH8TExKB3797ie/rXX3+ZjDO3P78NGzagTZs2qFevHo4ePYq4uDh88sknaNasmfh8Gd+DxMREnDlzBi1btgSAXL2/+REUFCQuAveinTt3omnTpnB2djbZ/uJ0+/T3dteuXRgzZgwCAgIQGBiIqVOnmhxjeZnWDwD29vZo1aoVdu/enem2nTt3omPHjrC2ztsXeBnt27cPtra26NWrl8n27777LsfLCU6fPg1PT89MswkuXrwIT09P7N27FwDQp08fNGnSxOQ+b7zxBlJTU/H06dMCeQ1E9GoxO+BXq1YNly5dAmCsW0lLS8O1a9cAADqdDikpKQU7QiIismgpKi3iktTQ63O3Zuv9JylYsPEsniQ9n8bfwOs1jH47APbyrFc3Lwjp4V6eQ7hXKpX44IMP4OLigvDwcMyfPx9paWkYMmQIkpOTcfLkSQwePBhyuRzfffcdPpv6P1y9dBbTJo+GRqMW92MwGDBu3DgEBwdj6dKlqFevHmbPno3Dhw8DAKKjowEAvXr1Ev98/fp19O3bF0+fPsU333yDmTNn4vbt2+jXr1+mD/rjx49H/fr1sXjxYoSEhGD58uXYtGmTye0rV65E7969sWTJEjRv3hyffvoptm/fDsC4+ndoaChcXV0RGRmJUaNG4ddff8XIkSOzve4aMH4hUaFCBbRq1QrR0dF47bXXAECs/Zs5cyYmT56MWrVqYc6cOYiKikKfPn2wfPlyzJgxAwkJCRg7dqxJHdvjx48xffp0DBo0CEuXLkWVKlUwadIkXL16FQCwfft2fPvttxgwYABWrFiB0NBQ/PLLL5gxY4b4Xnp7e8Pb2xvR0dFo3bo1YmNj0bNnT9y5cwdTp07FnDlzIJFI8O677+LEiRMmryk8PByDBw/GyJEjxcaDBw8eYNasWRgxYgQWLFiApKQkjBkzBh9//DF69+6NyMhICIKAjz76CCqVyuyfX0REBCZNmoTPP/8cAQEBmDBhAq5evYqwsDAsW7YM3t7emDRpkklYP3LkCFxdXfHGG28AQK7f37wKDg4Wp+mnMxgM2L17Nzp37pyrfUybNg2VK1dGVFQUhgwZgs2bN2PRokX5Hlv6+NKn6adTKpU4dOgQQkJCCuQ5AGMgr169Ok6ePIkePXqgTp06aNu2rfjvNjv16tVDtWrVsGPHDpPt27dvh7OzM1q1aoU6depg+vTpmb4s+fvvv+Hi4lKqLzchosJj9tebXbp0wZw5cyAIAt555x3UrVsXM2bMwMCBA7F48WK4ubkVxjiJiMgCJadooEzT5GoxPQCIfdZxn1qEHfdA7qflx8bGIj4+HoMGDUK9evUAAK6uroiOjkZKSgrmzp2LmjVrYsmSJbCyMi7O5+fnh86dO2PLli1iHZUgCBg5ciR69+4NAKhfvz727t2LAwcOoEWLFmI1XsWKFcU/R0REwM7ODqtWrRKvS27SpAnatWuH5cuXY9KkSeI4e/fujdDQUPE++/btw4EDB9C3b19cvnwZe/bswZQpU/Duu++K97l79y6OHz+Ozp07Y86cOWjRogXmzJkj7rNGjRoYPHgwDh48iNatW2f5/nh7e0Mmk6Fs2bKZ6v1GjBhh8rhHjx7ho48+MlmIzdbWFqNHj8alS5fEx6elpWHmzJniWcwaNWqgTZs2OHjwIGrVqoUTJ06gSpUqGDBgAKRSKQIDA2Fvb4/EROPsD39/f/H9St/n9OnTIZPJsGbNGvG21q1bi5Vi6b3vANCpUyd069YNFy9eFGcopqWlYdq0aeLZ8tjYWMydOxczZ84Uz+SmpqZizJgxuH79Ory8vMz6+fXv39+ks/zEiRMIDQ0V6wgDAwPh7Oxscq39oUOHxPGY8/7mlY+PD6pWrWoyTf/UqVNISEhAu3btsGXLlpfuo1WrVuLrbtKkCY4ePYoDBw7gk08+ydfYAOPP087OzmSa/t69e1GuXLks1w3Iq7i4ODx8+BDjx4/HqFGj4Orqip07d+Lzzz8HgByn6Hft2hXff/89VCoV5HI5BEHAzp07ERQUlO06CuvXr8eFCxfw8ccfm1RQExHlltkB/4MPPkB8fDzOnDmDd955B9OmTcPQoUMxcuRIODo6Ftg3s0REZLkEQUBiigapKi1yW7aaseMeALq3rIGgprnroM6rl4V7g0HAtbuJSErRwMb+NZQtWxYjRoxAUFAQWrRogWbNmmHChAlIS0vDmTNnMGTIEAiCAJ3O+CVF1apVUatWLRw9etSkbzogIED8c3oozmkK/LFjxxAYGAi5XC7u29HREQ0aNMCff/5pct8X9w0YvyhI33f6NPQOHTqY3Cd9OvHVq1fx4MEDDB8+XHweAGjYsCEcHR1x9OhRtG7d2uQ2wNjvndP1wF5eXiZ/nzt3LgBjQLp27Rpu3rwpTlfOuAr8i2E0fdG09NfTuHFjREdHo2fPnmjXrh1atWqFLl265DiWEydOoE2bNiYLuFlbW6Nz586IjIw0mW2Ycdzp0r/gAYDy5csDMH6Zky79rGtSUhIA835+GZ+zUaNGCA8Px4ULF9CiRQuTYAwY/70dPnwY8+bNE7eZ8/5mRa/Xm8zWkEqlmQJlcHAwfv75Z/zvf/+DRCLBjh070Lp161wvjJfxS4aKFSvi7t27uXrsy8jlcrRt29Yk4O/YsQOdOnXKdGwYDAaTVeklEon4Bd3LaLVaxMfHIzw8XPw31aRJE9y7dw8RERHo06dPtvvv2rUrIiIisH//fnTq1AmnT5/GvXv3sl1HYe3atZgzZw4aN26Md955x5y3g4hIZHbAv379usl/dHx8fLBv3z5cu3YNrq6ueVoNlYiISg69QUBCshpqjQ65zPbY//dtbHyhBs/KSoJ2fk5oFfBGYQ3T+DwvCfdnrjwWF8vT6Q2wtpIisOt4pNw8iF27diE6OhpyuRzdunXD8OHDYTAYsGzZsiwbY2xtbU3+LpebriUglUpznP6ekJCAnTt3ZrlCecapujntOyEhAQBQrly5bJ8HAMLCwhAWFpbp9kePHuHOnTt48803TbZ//fXX6NmzZ7bjz9jZfe7cOYSFheHcuXOws7ODm5ubOL084/tgZ2dn8lpevE9wcDAMBgPWr1+PqKgohIeHo3Llyhg/fny2lXiJiYliKH9R+fLlIQiCeP1+VuNOl9XnmRfHmZE5P7+Mzzl//nwsXrwYu3btwp49eyCVStG0aVNMnz4dlStXxrlz56BWq03OTJvz/malffv2JmG7R48emDVrlsl9goODsWTJEpw+fRr+/v747bff8MUXX7x03+kyvl8v+zdgrk6dOmHUqFF48OABbG1t8ddff2W58vyUKVNM1qCoXLky/vjjj1w9h4ODAyQSSaYFHFu0aIEjR47gyZMnmDNnTpb7r169OgICAsQvHnbs2IFq1aqZfHkEGL+AmD17NlauXIlOnTqhf//+XFyPiPLM7IDfv39/TJ48Gd27dxe3OTo6wtfXtyDHRUREFkirMyAhOfeL6RkEAb8cvIo9x26K2+xtrfFeiCck6ieFNUwAxi8RXBTybHvuz1x5jMjNZ5Cm0kHhYAMbKxto9QbEpTrArnpXLB03BUi7j19++QU//vgjXn/9dUgkEgwePDjLa5BzCn+5oVAo0LRpU7z33nuZbjNnwbD01cPj4uJMKsSuXr2KhIQE8faJEydmWXdXpkwZvPbaaybT2AGgSpUquR5D+loGnp6e2LFjB1xdXSGVSnHw4EHs2bMn1/tJFxISgpCQECQnJ+PIkSNYtmwZJkyYgPr164vT6jO+hidPMh9fjx8/BgC4uLjg0aNHZo8jJ/n5+SkUCkyYMAETJkzAtWvX8PvvvyMqKgphYWFYunQpDh06hKZNm8LGxrhGRUG8v4sWLTI50+/i4pLpPrVr10bNmjWxe/duqFQqqNXqbC/fKA4tW7aEg4MDdu/eDXt7e1SpUgV169bNdL9Ro0aZzK4xp2awevXqEAQBWq3W5Eu89Fkacrk8x/137doVX3/9NZKTk7F7927069fPZP8ajQaffPIJfvvtN7z//vsYNWoU/vvvv1yPj4goI7MDvo2NTZb/ESAiotJNrdUjMVkNrT534V6nN+CHnRczddyPftsfLo5WuHq18AL+y8K9wSBg8x9XkKbSoVwZW/FsWdz1GJw/vB51giZh68FrCBvaRDwD9/TpU3h7e+PatWvw8fER96VSqTBmzBi0atXKrHVoMk6HTl993svLSwyEgiBg/PjxqF69erZTyTNKP8v7xx9/iLVegHFRtocPH2LTpk0oV64c7ty5gyFDhoi3P3r0CBMnTkTfvn1RrVo1k9eY07izcu3aNSQkJGDQoEEm78mhQ4cAwGQ688uMGzcOWq0WkZGRUCgU6NSpE2xsbBAaGopHjx5lGfAbNmyI/fv3Q6lUimfi9Xo9duzYAR8fn0Lpkc/rz+/u3bviyZOgoCC4urrC1dUVMTExuHnT+MXYoUOHxPUcgIJ5f19sb8hJcHAwtmzZgtTUVLRv3z7TTJXiJJPJ0K5dO+zZswdyuTzbxf+qVKli1hdUL2rVqhVWrFiBHTt2mFxv/8cff8DT0xOOjo5wdHTMdv/BwcH46quvsGDBAjx9+hRdu3Y1uX3y5MnYu3cvJk+ejMGDB7+0yYKI6GXMDvhjx47F7NmzkZycjNq1a2c5tS19ihgREZUOaWotEpUa6HO5ml6aWoclP53DfzfixG2VKzhiVG8/uDjJC2SV7+xYPwv3smzCPQBcu5uIu4+UUDjYmEyFdanoBggGXDu6Aqr4dvipUirOnDqE5ORkdOjQAW3btsWwYcPwySefoGvXruLq8WfOnMHIkSPNGqeTkxNOnz6NkydPokGDBhg5ciT69u2L4cOHo1+/frC1tUV0dDT27duHhQsX5nq/tWvXRlBQEL799luoVCp4eXnh0KFD2L9/PyIiImBlZYWPPvoIn3/+OaysrNCmTRskJSUhKioKDx8+RJ06dV467gsXLuDEiRPZzt6rWbMmHB0dsXjxYlhbW8Pa2hp79uwRZwWY8/Nv3Lgxpk2bhm+++QYtW7ZEUlISIiIiUKNGDdSuXTvLx4waNQqHDh3CoEGDMGzYMNjY2GDt2rW4ffs2li9fnuvnNkdef36VK1dGxYoV8eWXX0KpVKJatWo4f/48Dh48iOHDhyMuLg7nz583qWQryPf3ZYKDgxEZGYlffvkFUVFRBbbf7MTExGDVqlWZtrdo0QK1amVeryM4OBjDhw+HVCrF1KlT8/ScW7ZsQZkyZUy2SaVSDBo0CI0aNUKbNm3w9ddfIy0tDe7u7vj5559x+vTpXL0f6Svmr1+/HgEBAahevbp42759+7B9+3a0bdsW/v7+iImJgUqlwo0bN6DValGvXr1C+TKKiEo3swP+F198Ab1ejwkTJmR7n4sXL+ZrUEREZDlS0jRIStHCkMtrZxOS1YjYFIM7j55f5+xZzQUjevrCTl4w3dTZsbGSwsXJFjbWOS+glZSigU5vgI2VaS2f3KEMGnYei0snfkHsX+sx7a818PTwQHh4uFihtmLFCkRERGDMmDGwsbFBnTp1sHLlSrNXLR8xYgSioqIwdOhQ7Ny5E7Vr18a6deswf/58TJw4EYIgwMPDA5GRkZmuh3+Zb7/9FhEREVi9ejXi4+NRq1YtLFy4UFylvXfv3nBwcMDy5csRHR0Ne3t71KtXD3PmzEHVqlVz3Pf777+Pr776CkOGDMHKlSuzvI9CoUBUVBRmz56NsWPHwsHBAV5eXli7di2GDh2KU6dOoW3btrl6LX379oVWq8WGDRuwfv16yOVyNGnSBBMmTBCnrGfk7u6O9evXY968eZg8eTIkEgl8fX2xZs0aNGjQIFfPa678/PwiIiIwb948LFiwAPHx8ahUqRJGjRqFYcOGYfv27XB3dzeZqVCQ7+/LuLm5wcPDA48fP0bTpk0LZJ85OXLkCI4cOZJpu4uLS5YBv2nTpnByckKlSpWyvD03sgrqVlZWGDRoEABgwYIFiIiIwMqVKxEXFwc3NzdERETk+j3u1q0b9u3bhy5duphs/+233wAYZwNktSbA77//nueZB0T06pIIZq528uIiItnp0aNHngdU2M6dOwcA2U49LGipqam4ePEivLy8sl3Ih6io8HgkcyWnaJCcpsn1Svn3n6QgfGMM4pJU4rYGXq/j3c7esLF+PrU7LS0NV69eRa1atfJ97Xq63IZ7AIi9nYCvVp2A3NYqy2n8ao0eKo0eUwYHwq2qc4GMjywXfzeSpeCxWPSKOhsQFTazT6VYcngnIqKCIQgCEpRqpKl1uQ73WXXctw+shh6F3HEPADJrKZwVcpMvEXLiWrkMKr/miBv3kyBzkppM0xcEAclpWtSo5ATXymVy2AsRERGRZcndJ6EMNBoN1q9fj1GjRqFPnz64evUqfvzxR5w9e7agx0dEREVMbxAQl6RGmir34f6fS4+wYMM/YriXAOj9pjveautuceEeAKRSCXq1dYedrTWeJqmh1uhhMAhQa/R4mqSGva01erV1h1TKqioiIiIqOcwO+HFxcXjrrbcwc+ZM3Lx5E2fPnoVKpcKBAwcwcOBA/PPPP4UxTiIiKgI6vQHxSSqozOy4X/rTOWifVedZW0nxQfe6eLNhtcIb6DMyaylcnMwL9+n83CsgtJcfalRygkqjR3yyGiqNHjUqOWFkLz/4uVcohBETERERFR6zp+jPnj0bKSkp2LlzJypXriz2jS5cuBBDhgzBwoULs110h4iILJdWp0d8Uu5r8LLruP/wLV+4Vyv8OtX0cG9tlafJaACMId+nVnlcu5uIpBQNnBxkcK1chmfuiYiIqEQyO+Dv378fU6ZMQfXq1aHX68Xttra2eP/99/Hpp58W6ACJiKjwqTU6xCvV0Otzd94+p477Nyo4FtYwRTJrKco6yWGVj3CfTiqVcCE9IiIiKhXMDvhqtRrOzs5Z3mZlZQWtVpvfMRERURFKVWmRlFJwHfeFzdbGCi4K2wIJ90RERESlidmfjnx8fLB+/fosb9u2bZs4ZZ+IiCyfMlWDRKU61+E+IVmNuev+Ngn3ntVcMH5A/aIL9wV05p6IiIiotDH7DP7YsWMxePBgdOvWDa1atYJEIsH27dsRHh6OI0eOYPny5YUxTiIiKkCCICApRYMUlbbAO+4Li9zGCs5Ocljx+ngiIiKiLJn9iaxBgwZYuXIl7OzssHz5cgiCgFWrVuHx48dYsmQJGjduXBjjJCKiAmIwGDvuU9JyH+5jbyfg27WnTMJ9+0bV8H7XOkUT7mUM90REREQvY/YZfABo2LAhNmzYAJVKhcTERDg6OsLBwaGgx0ZERAVMrzcgIVkNlVb/8js/88+lR1jx67/QPVtdXwKg15vuRVKDBxjDvYtCzpXtiYiIiF7C7IDfvXt3dO/eHSEhIShfvjzk8sK/5pKIiPJPqzMgIVkFjS53NXiAseN+497LSD/Rb20lxXtdvFG/9uuFM8gM5DJruChsGe6JiIiIcsHseZVvvPEG5s6di1atWmHIkCHYtm0bVCrVyx9IRETFRq3RIS4pLdfh3iAI2Lo/FtEvhHt7W2uM7eNfJOFeAoZ7IiIiInOZHfCjoqLw559/IiwsDIIg4NNPP0XTpk0xadIk/PnnnxBye0EnEREViTS1FvHJaujM6Lhftf0Cfjt+U9zm4mSL8e/Uh3s1l8IapkgCQG5rjbJODPdERERE5sjTNfgKhQK9evVCr1698PTpU+zevRu7d+/G0KFDUb58eRw8eLCgx0lERHmgTNUgOVWDXLbgGTvut57FfzfjxW2VKzhi1Nt+cFEU/iVZEgB2cms4O9pCImG4JyIiIjJHngL+i54+fYonT54gKSkJer0eZcqUKYhxERFRPuSlBi8hWY2ITTG480gpbvOs5oIRPX1hJ8/3fy5eiuGeiIiIKH/y9Int9u3b2L59O3bu3InY2FiUL18eISEh+Oabb1C7du2CHiMREZkhvQZPpdYhtxdNZdVx39D7dQwKLpqOe4Z7IiIiovwzO+C/9dZbuHDhAuRyOdq3b49PP/0UTZo0gVRq/AAoCAI/nBERFRPdsxo8tRk1eLG3ExC15QxSVTpxW/tG1dCjtRukRfD7XCIB7OU2KOMg438/iIiIiPLB7IDv7OyMWbNmoUOHDrCzsxO3P3r0CBs3bsSWLVuwf//+Ah0kERG9nFanR0Ky2qwavKw67nu380DbBlULaZSmJBIJHOQ2KONoWyTPR0RERFSamR3wV6xYYfL3w4cPY8OGDTh48CB0Oh2qVKlSYIMjIqLcUWt0iFeqoc/lSvlA8XfcW1tZwdGe4Z6IiIiooOTpGvy4uDhs3rwZGzduxN27d+Ho6IgePXqgW7duaNCgQUGPkYiIcpCq0iIpRQN9LpfKNwgCfj5w1aQGz97WGh/28oV71cKvwQMAqUQChYMtFHY2RfJ8RERERK8CswL+sWPHEB0djX379kGv16N+/fq4e/cuIiMjERgYWFhjJCKibJhbg6fTG7Bm50Wc+PeBuM3FyRaj3/bHG+UdC2mUpiQSQOEgQ7J17mcbEBEREdHL5Srgr1q1CtHR0bh+/TqqV6+OkSNHokePHrC3t0dgYCAXRSIiKmJ5qcFLU+mw5KfMHfej3/aHs6JopslLJIDCTgYriQ46ne7lDyAiIiKiXMtVwJ81axY8PT2xZs0akzP1ycnJhTYwIiLKWl5q8BKS1QjfGIO7j5933Neu7oLhPYqm4x54Hu4VDjKkpjLcExERERW0XJUbd+7cGTdv3sTw4cMxcuRI7N27l2deiIiKgV5vQHySCmlmhPt7T5SY/cMpk3AfWKciRr3tX2ThXioBnOyN4Z6IiIiICkeuPtnNnTsXSqUS27Ztw9atWzF69Gi4uLigXbt2kEgknKJPRFQE8lKDd+V2PBZtPotU9fMvZTs0qo7urWsVScc9YAz3CgcZHO0Y7omIiIgKU65P3Tg6OqJfv37o168frly5gi1btmDbtm0QBAFTpkxB586d0blzZ7i5uRXmeImIXklqjQ4JSjV0ZtTgnf7vEb7fZtpx/3Y7D7Qpoo574NmZewcZHBjuiYiIiApdrqboZ+Tu7o5PP/0UBw8eRHh4OFxdXbFs2TJ06dIFXbt2LegxEhG90tLUWsQnmxfu95+6jWU/nxPDvbWVFEO7+xRxuJfAydGW4Z6IiIioiOTr4ktra2u0b98e7du3x5MnT/DTTz/hp59+KqixERG98pRpGiSnaGHI5VL5WXbcy60x8i0/uFV1LqRRZiaVSFDGUQZ7OXvuiYiIiIpKga2uVL58eQwdOhRDhw4tqF0SEb3SkpQaKFWaXNfg6fQGrN5xAScvPBS3lXWSY/Tb/qhU3qGQRpmZVCKBs0IGO1uGeyIiIqKiVDTLJxMRUa4JgrEGL02ty1fHfZXXHDGqd9F13AOAVCqBsyPDPREREVFxYMAnIrIgeoOAhGQVVBp9rh+Tbcd9T1/Y2Rbdr3krqXFaPsM9ERERUfHI0yJ7Be3hw4fw9PTM9L+tW7cCAC5evIh33nkH/v7+aNu2LdasWVPMIyYiKnhanQFxiWlmhfscO+6LONw7K2wZ7omIiIiKkUWcwf/vv/9ga2uLffv2QfJCL7NCoUB8fDzee+89tG3bFmFhYYiJiUFYWBgcHBzw1ltvFeOoiYgKjkZr7LjX6vPXcd+xcXV0a1V0HfeAMdy7KGxhK7OI/6QQERERvbIs4tPY5cuXUaNGDbz22muZblu9ejVsbGwwffp0WFtbo1atWrh58yaWLl3KgE9EpYJKo0NCshp6Q+YL7gVBMPniM12WHfftPdCmftHV4AHZh/vsxk1E+cN/W0RElBOLCPiXLl1CrVq1srzt1KlTCAwMhLX186E2btwYS5YswZMnT1C+fHmzn08QBKSmpuZ5vOZIS0sz+X+i4sTj0fKo1HokpmQd7jesWwUbGxneeru/yfZDMffx88HrSH+EtZUEA4M84OtWrkh/ttbWUjg62kKv0yBVpxG3b926FdevX8cnn3yS7WOL+li8cuUKPvvsM1y7dg1VqlQRLwErbf777z+8//77OHjwIGxsSsblEvfu3UPnzp0RFhaGrl27FssYLPV3Y3BwMBo0aIDp06cDAJYtWwaZTIZ3330XALB48WIsWbIE//zzT6b7W8L7+jIZx5+V9NcBAFOnTs3y5E5ycjLatWsHjUaDZcuWoUGDBvj1118xbdo07NixA2+88UauxpPxMbkZ36lTp8QGqaioKDRp0iTTfa5fv46ePXsCwEvH06lTJ3h4eGDWrFm5GnNKSgrefvttDB8+PNPP+eLFi4iMjMS///4LQRDg5eWFMWPGwMvLK1f7zsmtW7cQERGBf/75ByqVCm5ubhgxYgQaNWqU730XNX5pRqWNRQT8y5cvw8XFBQMGDMD169dRvXp1fPjhh2jZsiUePHgADw8Pk/unn+m/f/9+ngK+VqvFxYsXC2TsuXXjxo0ifT6inPB4LH7W1tbQ6CRITtVAq9NleZ8fVi1HUOfuuHr1KgDjh5A/Lyrxz7XnX1Da2kjQuaEzHCQJuHo1oSiGDgCQy23hZGeNuAdaCBmW+l+0aBG8vb1z9Xu2qI7F7777Dnfu3MG4ceNQpkyZIv9vQFH5+eef4e3tjdjY2OIeSq49fvwYgDHIFffPxdJ+N2q1WiQmJorvS1RUFHr27Cn+vW7duggLCxP//uL9tVotwsLC8Prrrxf7+5qd9J99TuNLv49EIhGP74wOHjwIjcb4JePNmzfh4OCA119/HWFhYXjy5AkSExNzNZ6Mj8nN+G7evCmOb9OmTXB2ds50n82bN4t/jo2NzXE8umf/PcjNsahUKjFv3jzcu3cv07+fBw8eYMqUKahZsyaGDBkCwPjlwuDBg/HVV1/l+kuPrCQnJ+PTTz+Fo6Mj+vfvDzs7O/zxxx/48MMPMXXq1AL5AqGoyWSy4h4CUYEp9oCv0+lw7do1uLm5ib8sduzYgWHDhmHlypVQqVSZ/tHZ2horn9RqdZ6e08bGBm5ubvkee26kpaXhxo0bqFGjBuzs7IrkOYmyw+PRciSnapGSpoXLS3rwypZ1Qa1ataDTGfDj3liTcO+ikGF4d2+8Xta+sIdrwsbaCi4KGaytsl6n1cbGBmXKlMnxQ15RH4sGgwGenp7o37//y+9cgs2ePRudO3cuUR+wy5QpAwB44403im3clvq7Mat/SxUqVMj2fcp4f19f3yIZZ15VqFABAHL8uacfH/7+/jh79iwqVqwIFxcXk/tERETA09MTly5dQvXq1QvsOMrN+FJSUsTx/fPPP3B3dzeZdQoA//zzjzg+Nze3HMN1+mNfdiweOHAAs2fPFmekZvz38+uvv8Le3h4rVqwQ9/PWW28hODgYJ0+exKeffprTS8/RDz/8gOTkZGzYsEE86da7d2/06dMHBw4cEGcrlBQl6QtRolwRLIBSqRTS0tJMtg0ZMkQYMmSIEBISIsyePdvktitXrggeHh7Cv//+a/ZznT17Vjh79my+xmuOlJQU4dSpU0JKSkqRPSdRdng8Foxz584JgwYNEurVqyf4+/sL7777rvDPP/+It588eVIYMGCA4OvrKzRs2FCYOHGi8PTpU0EQBEGnNwir124Qant5CXv3/yl07/GWUKduXaF5i5bC3O8ihTuPkoU7j5IFDw8Pk/99suCgEPLxz0L7DxYLAc27Cl51fAU/P3/hvSHDhBP/XBQft333fsHDw0P4defvQr8BAwUfHx+hUeMmwudfzBRu3k8Q73f9bpwwfeY3QqtWbYS6dX2EDh07CStWrxdvv/MoWYjesk0I6dJNqFO3rtCocRPh0ymfC9fvPBHUGl22702bNm1Mxn379m1hy5YtgpeXl7Bx40ahadOmQsOGDYWzZ88KJ06cECIiIoTOnTsLPj4+gp+fn9CnTx/hr7/+Eve3cOFCoV27dsL+/fuFkJAQoU6dOkKHDh2En376yeR5V61aJXTs2FGoW7eu0Lx5c2HatGlCcnKyIAhCpvdyy5YtgiAIwvXr14XRo0cLTZs2Ffz8/IR33nlHOHXqlLjP27dvCx4eHsL3338vdOzYUfD19RU2b94sLFy4UOjYsaPw22+/CZ07dxbq1q0rdO3aVTh9+rTwzz//CL169RJ8fHyEzp07C3/++afJOC9duiQMGzZMCAgIEAICAoSRI0cKt27dEm8/duyY4OHhIfz4449C69athYCAAOHIkSPC06dPhY8//lho2rSp+HwZ34OEhATBy8tLuHv3riAIgvDOO+8IkyZNEhYtWiQ0adJEqFevnvDhhx8Kd+7cMXnc3r17hX79+gn+/v5CnTp1hI4dOwpr16596ZgEQRA2btwo9OjRQ/Dz8xN8fHyErl27Cjt37hQfu2XLFsHDw0OIiYkRunfvLvj4+AghISHCrl27Mr3PixcvFkaPHi34+/sLDRs2FKZOnSoolUqTY2vmzJnCoEGDBB8fH2HKlCmCIAjCxYsXhdDQUKFRo0aCt7e30Lx5c2HGjBkmnyuOHDki9O7dW/D39xcaNGggjBgxQoiNjRVvT0lJERYvXix069ZNqFu3rtC0aVNhxowZufpd6eHhIaxfv16YNGmSUK9ePaFhw4bi88+aNUto1KiREBgYKEyZMkVQqVQmrzn9WEw3adIkoU2bNiavedKkSeLzvPg/QTD++0j/c8b7Z3yO9H+HMTExwttvvy3UrVtXaN26tbB8+XKTMTx8+FAYN26c0LBhQ6FBgwbCZ599JsybN89kXB4eHsLChQtNHpdxLILw8uMjq8dklP46Vq5cKdSpU0fYsGGDye1xcXGCt7e3sHTpUsHDw0M4duyY+HrTfwelv7fvvvuusHnzZqFDhw5CnTp1hK5duwoHDx4U95XxMbkZX/q/j7Vr1woeHh7C4cOHTW6/ePGiULt2bWHx4sUm+85Oq1athA8++CDHYy8xMVGoXbu2MHHiROHs2bNZHkvr1q0Tli1blumx3bp1E4YMGZLtvhctWiTUqVNHSEhIMNm+cuVKwdvbW3jy5ImwY8cO4dtvv8302JEjRwpBQUE5vj5LVNTZgKiwWURNnoODA+Ryuck2d3d3PHz4EBUrVsSjR49Mbkv/++uvv15kYyQiAoxTIj/44AO4uLggPDwc8+fPR1paGoYMGYLk5GScPHkSgwcPhlwux3fffYcpU6bgxIkTGDRoEJQpqYhPUkGr00MwGBD2+ado82YHfP3tAvj4+mNJ1AKcPP4nACBi8SoAwJsdQ1Cv8wRcuhkPjfIxbv8ZCRuo8OnUMEyY/Dnu37uLMSPfR3x8nMk4Z06fCl+/AMyc/R3ebB+EDetXY+e2n5/fHvY/bNqwFsFduuOr2d+hYWBjfDPzC/y+dzcA4PffduGzyZ+gWvUamPHVXLz7/jDs3bMT06Z8DBvr7P/TERERgQoVKqBVq1aIjo4Wz+7o9Xp8//33mDlzJiZPngxXV1f8+OOPWLp0Kfr06YPly5djxowZSEhIwNixY02uhX78+DGmT5+OQYMGYenSpahSpQomTZokXrqwfft2fPvttxgwYABWrFiB0NBQ/PLLL5gxYwYAIDo6Gt7e3vD29kZ0dDRat26N2NhY9OzZE3fu3MHUqVMxZ84cSCQSvPvuuzhx4oTJawoPD8fQoUMxe/ZsNGvWDIBx+uusWbMwYsQILFiwAElJSRgzZgw+/vhj9O7dG5GRkRAEAR999BFUKhUA43W4ffv2xdOnT/HNN99g5syZuH37Nvr164enT59meh8nTZqEzz//HAEBAZgwYQKuXr2KsLAwLFu2DN7e3pg0aRKOHTsmPubIkSNwdXU1OTv4+++/Y+vWrZg6dao4lXvgwIHi+3vgwAGEhoaiTp06iIqKQnh4OKpWrYrp06fjzJkzOY5p3bp1+Pzzz9GuXTssWbIEc+bMgUwmw/jx4/HgwQOTxw4fPhxvvvkmIiIiULNmTYwbNw4HDx40uc+CBQtQqVIlREVF4d1338XGjRsRERFhcp9169bBx8cHUVFR6NWrFx49eoQBAwYgLS0Ns2bNwrJly9C5c2f88MMPYqXu7du3MXLkSNStWxeLFi3CzJkzcf36dQwbNgwGg3GRyl27dmHevHmoWbMmIiMjMWrUKPz6668YOXJkpktQsvLtt99CJpMhIiIC3bt3xw8//IDu3bvj/v37mDNnDgYOHIjNmzfjhx9+eOm+shMdHQ0A6NWrl/hncxkMBowbNw7BwcFYunQp6tWrh9mzZ+Pw4cMAAI1Gg3fffRenT5/GlClT8PXXX+O///7D999/b/ZzmXN85IaTkxOaNWuG3bt3m2zfs2cP3njjjVzNVjh//jxWrFiBMWPGIDIyElZWVhg9enSup/DnxM3NDe7u7pnGt2PHDgQGBoqzAQqCXC7Hjh078M0332SazZCuf//++OCDD0y23bx5E1euXIG7u3u2++7SpQt0Oh1+++03k+07duxA8+bNUa5cOQQHB2P8+PEmtycmJuLkyZM57puIikaxT9G/cuUK+vTpg0WLFpkszHH+/Hm4ubnBy8sLGzZsgF6vh5WVFQDg2LFjqFmzJsqVK1dcwyaiV1RsbCzi4+MxaNAg1KtXDwDg6uqK6OhopKSkYO7cuahZsyaWLFki/s7y8/ND586dsXZ9NLp07w3AeD39oPeGIjikOwCgro8/Dh/cj7/+PIKGjZrCu64PAOC/uzo41DB+MHx6eS9ktnKsXLUSZZycAAD1GgRiwNtdEb1+DUaEjhPH2blLdwwcbFz4qV79QBw9dAB//XkYXbq/hevXYnHowO8IHfOJuIBfvQaBePDgPmJOn0Lbdh2xdHE4Ahs1xZTPvwQA2FhJUdfLDR8MGYKDBw+idevWWb4/3t7ekMlkKFu2LPz9/U1uGzFihPi41NRUxMfHY/To0Rg4cKB4H1tbW4wePRqXLl0SH5+WloaZM2eKi1fVqFEDbdq0wcGDB1GrVi2cOHECVapUwYABAyCVShEYGAh7e3vxQ7u/vz8cHR3FPwPA9OnTIZPJsGbNGvG21q1bIyQkBLNnzza5ZrZTp06ZFvZKS0vDtGnT0LJlS/G4mDt3LmbOnIlevXqJr3HMmDG4fv06vLy8EBERATs7O6xatUp8ziZNmqBdu3ZYvnw5Jk2aJO6/f//+CAoKEv9+4sQJhIaGol27dgCAwMBAODs7m1zCdujQIXE8L45z69atqFrV2K7g6uqKHj164Oeff0a/fv0QGxuLHj164H//+5/4mICAADRq1AjHjx+Hn59ftmO6ffs2hgwZgpEjR4rbKleujJ49e+Lvv/8WF0YDgIEDByI0NBQA0KJFC/To0QORkZFo1aqVeJ+OHTti8uTJ4vty9OhRky8wAOM05BeDxZEjR+Dl5YUFCxaI72nTpk1x9OhRHD9+HMOGDcPZs2ehUqkwfPhw8cRAxYoV8fvvvyM1NRUODg5YsGAB/Pz8MHPmTNjbGy95qVGjBgYPHpzj8Z7Ozc1NXAgvMDAQmzZtglarxZw5c2BtbY3mzZtjz549OH36dI77yUn6sVuxYsVM/7ZySxAEjBw5Er17G38P1a9fH3v37sWBAwfQokUL/Prrr7h27Rq2bNmCunXrAjAubJx+3JnDnOMjtzp16oQpU6YgLi4OZcuWBWAMnsHBwbl6fHJyMrZu3Ypq1aoBAOzt7fHOO+/g2LFj6Nixo9njyWp8a9aswRdffCFOtd+5cydGjBiR732/SCaTwdXV1azHqFQqTJo0CTKZDO+8806296tcuTIaNmyI7du3i8fJrVu3cPbsWcyfPz/LxxgMBnz22WfiF+BEVLyKPeDXqlULrq6umD59OsLCwuDi4oKNGzciJiYGW7ZsQbly5bB8+XL873//wwcffICzZ89i1apVCAsLK+6hE9ErxGAQcO1uIlIFZ5RxdsGIESMQFBSEFi1aoFmzZpgwYQLS0tJw5swZDBkyBIIgiIslvfZ6JVSrXhMnjv8lBnwA8K7z/IyTTCaDs7MzVM/OrF65FQ8ASFPr4PDsPvqk62jcKBCO9vbQP9u3g70DfHwD8PfJ4ybj9a5rejar/GuvQaUy7vvc2RgAQItWbU3uEzbzWwDArZvX8fjRQ/Qf+B70Oh1srKVQ2MvQpHFjODo64ujRo2jdurX4+tJZWVnluBJxxutYR40aBS8vL8TFxeHatWu4efMm9u/fDwDiglnpXgw0FStWBADx2tPGjRsjOjoaPXv2RLt27dCqVSt06dIlx7GcOHECbdq0EUMhYLz2tXPnzoiMjBSvq81q3OnSv+ABIC74+mIgTl9sKykpCYDxy+nAwEDI5XLxvXN0dESDBg3w559/muw743M2atQI4eHhuHDhAlq0aIFWrVqZfCEgCAIOHz6MefPmZRpjergHjF/AVK1aFSdPnkS/fv3ED+MpKSm4fv06bt26hXPnzgHI/DPIOKb0a3iTkpLEn9/x48ezfGyPHj3EP0skErRv3x7h4eHi7AYAaNCggcljqlSpgr///jvHMTRv3hzNmzeHVqtFbGwsbt68icuXLyMuLk58//38/GBra4tevXohKCgILVu2RKNGjcQzvlevXsXDhw8RHBwMnU4n/mwaNmyY6+M9ICDAZLuLiwvq1Kljci22s7MzkpOTUdxeHGv6l3Hp/5aOHTuGqlWriuEeMB6jbdq0EX+2uWXO8QEYj2G9Xm+yLeO17O3atcNnn32GvXv3ok+fPnj06BFOnTqFzz//HHFxprOYslK2bFkx3APPf5cUVHtCcHAwFi5ciGPHjqF58+Y4c+YMHj58iA4dOuD33383ua+5vz/zQ6lUIjQ0FOfOncOCBQtQuXJlAMaZVS/OUJFKpZBKpejatSumTZuGx48fo0KFCtixYwccHR3Rtm3bTPvWarX49NNPsWfPHnz++ecWv+4D0aug2AO+VCrF4sWLMXfuXIwbNw5JSUnw9vbGypUrxdXzly9fjpkzZ6JHjx6oUKECJk6caPJhgYioMJ258hib/7iCu4+U0OkNcG0RirjY37F9+w5ER0dDLpejW7duGD58OAwGA5YtW4Zly5Zl2o/s2QKh6TJemiSRSiEIBvz930Os3Pbv8+0wdtx/tUuJA3/sxYE/9mbat7Oz6TRNua3pvqUSKYRnVXxJz85sO7uUzfL1pt++YO4sLJibuarp0aNHuHPnDt58802T7V9//XWOiyulnxlNd+3aNcycORP//vsv7OzsTBafyjgt+sXFpqRSqcl9goODYTAYsH79enGaeeXKlTF+/Phsz+wlJiZm2cJSvnx5CIIApVKZ7bjTvfjlQFbjzCghIQE7d+7Ezp07M92WfjYyu+ecP38+Fi9ejF27dmHPnj2QSqVo2rQppk+fjsqVK+PcuXNQq9WoX7++yeOyupStXLly4uyGuLg4TJs2Dfv27YNEIkH16tXFoJ3xZ5BxTLdu3cLnn3+Ov/76CzY2NnB1dUXt2rWzfGz6pRovjkEQBPHLDyDzeyeVSl86BoPBgHnz5mHdunVITU1FpUqV4OvrKy7GCxi/KFi7di2WLl2KzZs3Y82aNXByckL//v0xbtw4JCQkAABWrlyJlStXZnq/cnO8Z3UsZHfcFLeMv3defJ/j4+OznB2ZlxmT5hwfAPDTTz+JMzjSZQzFjo6OaNmyJXbv3o0+ffpg9+7d4tT43HwBkfEYSw/U6Zdq5FfNmjXh5eWF3bt3o3nz5ti5cyeaN28uLhSYLi+/P/Pq/v37GD58OK5fv4758+ebzMZo37497t69K/69R48emDVrFoKCgjBjxgzs2rULgwYNwo4dO9CxY8dMx05SUhJGjRqFkydP4rPPPsOAAQMKfPxEZL5iD/iA8QPV119/ne3tvr6+eb7ejIgoP85ceYzIzWeQptJB4WADGysbaO3fgI1jX1Rv2B8dfGzw7+mD+PHHH/H6669DIpFg8ODB6Ny5M9JUOqSotGJfvW2G0J2V+09SsPzn8+JjpFIJhvXwQYDna1ioUKBe/UZ4u1/m6ZXplwPkhqOjAgCQmBCPCq89D4C3bl5HUmIiHJ7dPnL0R2jVoimspKZnlcqUKYPXXnvNZBo7YAxSuaVUKjFr1ix4eXlhx44dcHV1hVQqxcGDB7Fnz55c7yddSEgIQkJCkJycjCNHjmDZsmWYMGEC6tevn2XILVOmDJ48eZJpe3otlouLS6b1X/JLoVCgadOmeO+99zLdlvFMZVaPnTBhAiZMmIBr167h999/R1RUFMLCwrB06VIcOnQITZs2hY2Njcnj4uPjM+3ryZMn4lnM8ePH49q1a1i1ahUCAgIgk8mQlpaGjRs35jgeg8GAYcOGwcbGBps3b4aXlxesra0RGxuLX375JdP9ExISTL5QefLkCaysrODs7Jyv93np0qXirL4OHTpAoTAeu+mXSaTz9fVFREQENBoN/v77b0RHR2Px4sWoXbu22KrTv39/BAcHZwoxBXG8Z5QeLDOesU4/k15cXn/99Szr2TKuEQHkPHZzjw8AaNOmTab3+LXXXst0fAQHB2PChAmIi4vDzp078zTVvzAFBwdjxYoVmDZtGnbv3p3pWnUABX48ZefSpUsYMmQI1Go1vv/+ezRs2NDk9kWLFpnMpki/nl+hUKBt27bYtWsXGjdujCtXruCzzz4zeeyDBw/w3nvv4c6dO5g3bx46depU4OMnoryxiIBPRGSJDAYBm/+4gjSVDuXK2EIikeD+1b/x7+H1aN77Myi1cpy9b4uwz6dhx44dePr0Kby9vXHt2jVUq+lhDPcCoFar8MXUiWjcpDlq1Mz6ukmDICBVpcWN+0moKOZRCRrXrYQAT+PZTz//+rh54xrc3Dxg9SwQCoKAmWH/Q5Wq1eDm7pmr1+Xj6w8A+PPoIXTr8fySgaWLwvHk8SNELl0NF5eyiHvyAP5+z6dbPnr0CBMnTkTfvn1RrVo1+Pj4ZLn/9DPsOblx4waUSiX69etnUlt66NAh4/thxhm1cePGQavVIjIyEgqFAp06dYKNjQ1CQ0Px6NGjLAN+w4YNsX//fiiVSvHsq16vx44dO+Dj41MonciBgYGIjY0Vww5g/PmNHz8+x2qvu3fvon///pg8eTKCgoLg6uoKV1dXxMTEiB3chw4dEq+XfdHff/+N+Ph48YP7+fPncefOHfF6+L///ht9+vQxWQMnNz+D+Ph4XL9+HVOmTDE5DrJ77L59+9C3b1/xNf/222+oX79+vt/nv//+G25ubiZrJDx8+BCXL18Wx7Vq1SqsXr0ae/bsgUwmQ5MmTVC3bl3s2rUL9+7dQ4cOHVC2bFk8fvwYderUEc+85/Z4z4v0Y+7hw4fiNq1Wi7Nnz+b47yc3/7byIzAwEFu2bMHFixfF41GlUuHw4cMmPytHR0eTsQMwWV/A3OMDMIbL7BaMe1GbNm0gk8mwdu1axMTE4NtvvzXvRRayTp06Ye7cuVi8eDESExMznakHjJdGFOTxlJX79+/jvffeg5WVFX788ccs66E9PbP/b0a3bt0wcuRI/Pjjj3jjjTcQGBgo3qZUKvHuu+/iyZMnWLlyZabLa4ioeDHgExFl49rdRNx9pITCwUY84+ZS0Q2CYMDpPYtRzac9/n0sw0fjf0FycjI6dOiANm3aYPjw4Zj86US82aETDHoDNm74ARcvnMfAd4dm+TxanQGrd1xAmloP+2czfcs6yeGoUOD+zUs4E3Mavn4BGDh4KEaPGIwpE8eha49ekMlk2PbLVhw9fADTvpyd69dVy90Drdq0w5KoBVCrVHBz98TxY0fx19FDCJv5LezlMnz00Th88cUXsLG2Rps2bZCUlISoqCg8fPgQderUyXH/Tk5OuHDhAk6cOJHt9ZjVq1eHnZ0dVqxYAQcHB1hbW2PPnj3iWS1zrolt3Lgxpk2bhm+++QYtW7ZEUlISIiIiUKNGDXFKcEajRo3CoUOHMGjQIPFM49q1a3H79m0sX748189tjpEjR6Jv374YPnw4+vXrB1tbW0RHR2Pfvn1YuHBhto+rXLkyKlasiC+//BJKpRLVqlXD+fPncfDgQQwfPhxxcXE4f/48wsPDMz02LS0NH3zwAT788EOkpKRg/vz58PDwQEhICADjme1t27ahTp06qFixIk6fPo2lS5dCIpHk+DMoV64cKleujHXr1qFixYpwcnLC4cOHxZXrMz529uzZUKvVqFmzJjZt2oSrV69i9erVeXkbTfj6+iIqKgpLly6Fv78/bt68iSVLlkCj0YhjaNy4MebMmYPQ0FC88847sLKywoYNGyCTydCmTRtYWVkhNDQUX375Jb755ht06NDBrOM9L8qUKYOAgAD88MMPqF69OsqUKYM1a9ZApVLlOLXfyckJp0+fxsmTJwslVIWEhGDp0qUIDQ3F2LFj4eTkhJUrV+Lp06cm7QytW7fGjh074Ofnh+rVq2Pr1q3il02A+ceHOezt7dGqVSssXboUvr6+JmtMFJZVq1Zl2ubk5JTllPqqVavCx8cHS5YsQfv27fN0qcadO3ewbt26TDNy6tWrl+tr3L/88ks8ffoUYWFhUCqViImJEW9zdHTMMvC/qEWLFnB2dkZ0dDQ++OADk/UBFi5ciBs3bmD06NGwtrY22bdMJoO3t3euxkhEhYMBn4goG0kpGuj0BthYPf+QJXcog4adx+LyiV9w8cg66HQapNWshfDwcDRsGIiEZDW+mRuB1SuXImzqRFjb2MDD0wtz5keJK+O/KE2lw+KtZ3Hp1vOp1FVec8So3v7YW3YIfli1HJPHj8bKtZtRy80d30Uux/dLo/D1jM8hCAJqutbC9K/nolnzVpn2nZMpn3+J1d8vwZZNPyIxIQHVatTAtC9n480328FFYYs+ffpAoVBg+fLliI6Ohr29PerVq4c5c+a89AP1+++/j6+++gpDhgzJ8ppmwDgF9JNPPsHWrVsxduxYODg4wMvLC2vXrsXQoUNx6tSpLBd0ykrfvn2h1WqxYcMGrF+/HnK5HE2aNMGECRMyfUBO5+7ujvXr12PevHmYPHkyJBIJfH19sWbNmkI7G1W7dm2sW7cO8+fPx8SJEyEIAjw8PBAZGZnlWb4XRUREYN68eViwYAHi4+NRqVIljBo1CsOGDcP27dvh7u6e5UyFBg0aoHHjxuIq+W3btsXEiRPFs7GzZs3CjBkzxErBGjVqICwsDL/++itOnTqV45iioqIwc+ZMfPrpp5DJZHBzc8OiRYvw1Vdf4dSpUybtCF988QWWLFmC27dvw9vbG99//32BvM/Dhw9HfHw81qxZg8jISFSqVAndunWDRCLBkiVLkJSUhNq1a2Px4sWIjIzExx9/DL1ej7p16+L7778XVyLv2bMnEhMTsXfvXmzdutWs4z2v0t/7qVOnwtHREb169UL9+vWxadOmbB8zYsQIREVFYejQoVmu5ZBf1tbWWLFiBWbOnCmuBN+1a1c4Ozvj+vXr4v0mT54MnU6Hb775BtbW1ggODsYnn3yCqVOnivcx5/gwV3BwMHbv3p3r1fPzK6vLSKtVq5btNfPBwcE4d+5cni8fuHbtGubMmZNp+9ixY3MV8DUaDQ4cOAAAmDZtWqbbAwMDX1rZmL7o6A8//ICuXbua3JZeoRceHp7pi8XKlSvjjz/+eOkYiajwSITcFLyWIumrAxf21Kh0qamp4lQ3S11wh14dPB7NE3s7AV+tOgG5rRVsbTJf467W6KHS6DFlcCCqV3JCQrIKGl3up5bHJ6sQsfEM7j5+vqBb7RplMbyHD+xsi/77V7mNFZyd5JmuuS8MPBYLX3qAyk/3en5t3boVkydPxu+//14o1xgXFB6PRleuXMG1a9fQoUMHkzO2vXr1QsWKFREREVGMo3s18FgsekWdDYgKG8/gExFlw7VyGVR+zRE37idB5iQ1+cArCAKS07Rwr+qMKq85Ii5JBZ0+9+H+3mMlwjfFID5JLW5rVKciBgZ7wdqqcK+zzUpRhnsiskypqakYO3Ys+vfvj/bt20Ov12Pnzp04f/58lovFERGR5WHAJyLKhlQqQa+27ojcfAZPk9RQ2NnAxloKrc6A5DQtyips0b1lLSQo1dAbcj8Z6sqteCzachap6uc9yB0bV0f3VrUKrQc5J3KZFZwVDPdErzo/Pz989913WLFiBX7++WcIggBvb28sX74cjRs3Lu7hERFRLjDgExHlwM+9AkJ7+WHzH1dw95ESyjQtrK2k8KpRFsFNa6CCi71Z4T69416nNz5GAqBPew+0rl/4C0VlRS6zgotCDinDfalTnFPz0/Xs2bNQur2p8AQFBSEoKKi4h0FERHnEgE9E9BJ+7hXgU6s8rt1NRFKKBi4KW5QtI0eKSguDGcuY/H7yFjb/fkXsuLexluL9LnXEGryiJpdZw0Vhy3BPREREVEow4BMR5YJUKoFbVWcIgoAEpVrsuM8NgyBg6/5Y7DtxS9zmILfGh7384FbFuXAGnAMJALmtMdwXxyUBRERERFQ4GPCJiHJJbxCQkKyGWqNDbs/bp3fcn7r4UNxW1kmOMX38UbGcQ+EMNAcM90RERESlFwM+EVEu6PQGY7jX6nP9mFSVFou3nsXlWwnitqqvOWLU2/4o42hbCKPMmQSAndwazo4M90RERESlEQM+EdFLaLR6JCrV5nXcJ6kQscm0496rRlkMK6aOe4Z7IiIiotKPAZ+IKAdqjQ7xSjX0+twvpnfvsRLhG2MQn2wZHfcSCWAvt0EZBxnDPREREVEpxoBPRJSNVJUWiSkaGMyowbt8Kx6LLajjXiIBHOQ2xXJJABEREREVLQZ8IqIsJKdooEzTwIxsj1MXH2LV9gwd9x080bpelcIZ5Esw3BMRERG9WhjwiYheIAgCklI0ZtXgAZbXcc9wT0RERPTqYcAnInrGYDB23KvUua/By67jfmQvP9Qqho57wBjuHeUyODnKiuX5iYiIiKh4MOATEQHQP6vBU5lRg2dpHfeAMdwr7GRQODDcExEREb1qGPCJ6JWn1emRkGxeDZ6lddwDDPdERERErzoGfCJ6pak1OiQo1eLCeLkRn6RC+KYY3HucIm4rzo57AJBKAIW9DI72DPdEREREryoGfCJ6ZaWptUhUaqA3Y6n8u4+ViLCgjnvgWbh3kMHRjuGeiIiI6FXGgE9EryRlmgbJKebV4F2+FY9FW84i7YWO+6Am1dGtZfF03AMM90RERET0HAM+Eb1yEpVqs2vwLK3jHjCGeycHGRwY7omIiIgIDPhE9AoxGAQkpqiRptaZ3XG/6fcr4t+Lu+MeAKQSCZwcZXCQ2xTbGIiIiIjIsjDgE9ErQW8QkJCsgkqT+xo8gyBg6x+x2HfScjruAWO4L+Mogz3DPRERERG9gAGfiEo9rc6AhGSVWTV4xo77f3Hq4iNxW7kycox+u/g67gFjuHdWyGBny3BPRERERKYY8ImoVFNr9UhMVkOrL9kd9wBgJTWeuWe4JyIiIqKsMOATUamVlxq8rDruvWuWxbDuPpAXU8c9wHBPRERERC/HgE9EpVJKmgZJKVoYzFhNL6uO+8Z1K+KdTsXXcQ8Yw72zwhZyGX9lExEREVH2+GmRiEqd5BQNktM0Zq2Un3XHfQ10a+labB33gDHcuyhsYctwT0REREQvwU+MRFRqCIKABKX5NXiZOu4lQN/2nmhVjB33AMM9EREREZmHnxqJqFQw1uCpodboYEa2x74Tt7D5D9OO+yFd68Lfo0LBD9IMVlYSuDgy3BMRERFR7vGTIxGVeDq9wRjutfnsuLezQWgvP7hWLlMYw8w1KysJXBRy2NpYFes4iIiIiKhkYcAnohJNo9UjwcwaPEvtuAcY7omIiIgo7xjwiajEUmt0iFeqodfnflJ+qkqLRVvO4srtBHFb1dcVGNXbr1g77gFjuC+rkEPGcE9EREREecCAT0QlUqpKi8QUDQxmdNzHJakQsTEG955YVsc9AFg/O3PPcE9EREREecWAT0QlTnKKBso0DczI9rj7WInwjTFIyNBxP7CTF6yKseMeYLgnIiIiooLBgE9EJYYgCEhK0SBFpTWrBu/SzXgs3mracd+paQ10bVG8HfcAYG0lhYvCluGeiIiIiPKNAZ+ISgSDwdhxr1KbV4NnqR33gDHcl3WyhY01wz0RERER5R8DPhFZPP2zGjyVGTV4gOV23AMM90RERERU8BjwiciiaXXGGjyNLvc1eAZBwObfr+CPU7fFbZbScQ88n5bPcE9EREREBYkBn4gsllqjQ4JSLU6vzw2tTo9V2y/g7/8sr+MeSF9Qj9fcExEREVHBY8AnIouUptYiUamB3oyl8lNUWiy20I57gKvlExEREVHhYsAnIoujTNMgOUULgxlL5cclqRC+MQb3LbDjHmC4JyIiIqLCV/yfeomIXpCoVJtdg3fnUTLCN55BotLyOu4BhnsiIiIiKhoM+ERkEQwGAYkpaqSpdWZ23Mdh0dazUKmfr7Af1KQGurUs/o57ALBiuCciIiKiIsKAT0TFTm8QkJCsgkpjXg3eyQsPsGr7BfE6fYkE6NfBEy0Dir/jHjCG+7IM90RERERURBjwiahYaXUGJCSrzKrBEwQB+07cwpb9seI2S+q4B3jmnoiIiIiKHgM+ERUbtVaPxGQ1tPrS03EPPA/3tgz3RERERFSEGPCJqFjkpQZPq9Nj5fYLOP1Cx335MnKM7hOA18vaF8YwzcZwT0RERETFhQGfiIpcXmrwsuq4r/a6AqEW0nEPMNwTERERUfFiwCeiIiMIApJSNGbX4Fl6xz3AcE9ERERExc8yPhkTUamX1xq8rDrum/hUwjtBtS2i4x4ArKQSuDjaMtwTERERUbFiwCeiQqfXG5CQrIZKa14N3n834rD4J9OO++CmNdClhWV03APPwr3CFrYy/jolIiIiouLFT6REVKi0Oj0SktVm1eABlt9xDzDcExEREZFl4adSIio0ao0O8Uo19Prcz8kXBAF7T9zC1gwd9x90qws/d8vouAcY7omIiIjI8vCTKREVilSVFkkp5tXgGQwCNv1+Gfv/viNus7SOe4DhnoiIiIgsEz+dElGBS07RQJmmgRnZ3thxv+0CTl+y3I57gOGeiIiIiCwXP6ESUYHJaw1eikqLRZvPIvZOgritWkUFRvX2g5ODZXTcAwz3RERERGTZ+CmViAqE3iAgIVkNtUYHM7I94hJVCN9k2nFfx7UchnavC7kFBWmGeyIiIiKydPykSkT5pntWg6c2swavJHTcAwz3RERERFQy8NMqEeWLRmuswdPqzavBu3QzDou2WnbHPcBwT0REREQlBz+xElGe5aUGDygZHfcAwz0RERERlSz81EpEeZKi0iJJqYHBjNX0suu4H9qtLnwtqOMeMIZ7Z4Z7IiIiIipB+MmViMyWlxq8rDruHe1sMNLCOu6B5+Hekhb5IyIiIiJ6GX56JaJcEwQBiSkapJpZg5dlx72zHUa/7W9RHfcAwz0RERERlVz8BEtEuZLXGryS0nEPMNwTERERUcnGT7FE9FJ5rcErKR33gDHcl3GUWdy4iIiIiIhyi59kiShHea3By6rjvqlPJQywsI574Hm4t7O1Ke6hEBERERHlGQM+EWVLpdEhIVkt1tnl1n834rD4J8vvuAcAKcM9EREREZUSDPhElKWUNA2SUrRm1eABWXfc9+9YGy38KxfGMPNFKpHAmeGeiIiIiEoJBnwiyiRJqYFSpTFrpfyS1HEPPAv3CoZ7IiIiIio9GPCJSCQIAhKUaqSpdWaF++w67kN7+6HmG5bVcQ8w3BMRERFR6cSAT0QA0mvwVFBpzFspX6vT4/tt/+KfS4/FbZbacQ8w3BMRERFR6cWAT0TQ6gxISFZBozNvpfyUNC0WbTHtuK9eUYFQC+y4BxjuiYiIiKh0Y8AnesWpNTokKNXQ6c1bTC8uUYWFG//Bg6ep4jZL7bgHGO6JiIiIqPSzvE/hRFRk0tRaJCo1Ztfg3XmYjPBNGTrufSthQEfL67gHGO6JiIiI6NXAgE/0ilKmapCcqoGZ2d7Ycb/1rMm1+p2b1URI85oW13EPGMM9e+6JiIiI6FXAgE/0irG2tkZyqhZ6GMxaKR8ATvz7AKt3lIyOe+B5uLeXM9wTERERUenHgE/0ChEEIFUjQJmmgVxuZ8bjsum47+4DX7fyhTHUfGO4JyIiIqJXDQM+0StCpzcgPlmN5BQ1KpTijnuA4Z6IiIiIXk0M+ESvALVWj8RkNVQaHQQz5uWXtI57gOGeiIiIiF5dDPhEpVxeV8ovaR33gDHcOzHcExEREdErigGfqBRLTtFAmWb+SvlPE9MQvjGmxHTcA4BUAjg5yuDAcE9EREREryjL/KRORPkiCAISUzRIVWnNXinf2HEfg0SlRtxmyR33QHq4t2W4JyIiIqJXGgM+USmjNwhISFZBrdHDzGxf4jruAYZ7IiIiIqJ0DPhEpYhWZ0BCsgoancHsxx7/9wHWlKCOe+BZuHfgtHwiIiIiIoABn6jUUGt0SFCqodObd95eEAT8dvwWfjpQcjrugRfCvZ2suIdCRERERGQRGPCJSoFUlRZJKeavlF8SO+4BhnsiIiIioqww4BOVcHldKV+j1WPltn/xz+WS03EPMNwTEREREWWHAZ+ohMrPSvkpKi1W7fgXsXcSxW2W3nEPMNwTEREREeWEAZ+oBMrPSvlJqXps2nQeD+PSxG11a5XDB90st+MeYLgnIiIiInoZy/00T0RZys9K+Xcfp2Dz0Tikqp8/tpnvG+gf5AkrqWV23APGcK9guCciIiIiyhEDPlEJotbqkZishlZvfrj/70YcFm09D7Xm+WMtveMeeB7uHRnuiYiIiIhyxIBPVEKkqbVIVJq/Uj6QueNeKgH6B9VGcz/L7bgHGO6JiIiIiMzBgE9UAqSkaZCUooXBzNX0jB33N/HTgaviNmsrYHBwbTSow3BPRERERFSaMOATWbgkpQZKlcbslfINBgEb913GgdPPO+4d7KwRXM8JdVzLFvAoCxbDPRERERGR+RjwiSyUIAhIUKqRptaZHe41Wj2+3/YvYl7ouK/gbIeh3Woj6em9Ah5pweJq+UREREREecOAT2SB9HoDEpTqPNXgpaRpEbXlDK5m6rj3h41Uj6SnBTvWgiSVSODkKIOD3Ka4h0JEREREVOIw4BNZGK1Oj4RkdZ5q8J4mpiF8YwwePE0Vt73YcZ+WllaQQy1QUokEZRxlsGe4JyIiIiLKEwZ8Igui0uiQoFRDrzd/pfzbD5MRsSkGiUqNuK0kdNwDxnDvrJDBzpbhnoiIiIgorxjwiSxEXlfKB4CLN+KwZOtZqDR6cVtJ6LgHAKlUAmdHhnsiIiIiovxiwCcqZoIgIClFgxSV1uzF9ADg+Pn7WL3zIgxix70E/YM8Lb7jHgCspMZp+Qz3RERERET5x4BPVIz0BgGJSjVUap3Zi+ll1XEvs5FiaDcf+LiVL9iBFgIrqQTOClvIZfw1RERERERUEPjJmqiY6PQGJCSrodbqX37nDLLquFfY2yC0tz9qVHIqyGEWCiupBC4KW9gy3BMRERERFRh+uiYqBhqtcaV8rd78lfKz67gf3ccfr7nYF+QwCwXDPRERERFR4eAnbKIilqbWIlGpgd5g/gX3WXbcV3JCaC8/ODnICnKYhcLKSgIXR4Z7IiIiIqLCwE/ZREVImaZBch5Xys+q496nVjl80M0HtjKrghxmobCyksBFIYetjeWPlYiIiIioJGLAJyoC+V0p//bDZIRvjEFSyvOO++Z+b6BfR8vvuAcA62fhXsZwT0RERERUaBjwiQqZwSAgMUWNNLUuT+H+wvWnWPrTOZOO+y7NayK4meV33AOAtZUULgpbhnsiIiIiokLGgE9UiPTPVspX5WGlfAA4dv4+1mTouB8QVBvN/N4oyGEWGhsrKVycbGFjzXBPRERERFTYGPCJColWZ1wpX6Mzf6V8QRCw59hN/HwwQ8d9dx/41LL8jnsgPdzLYWNt+ZcQEBERERGVBgz4RIVArdEhQamGTm/+nHyDQUD0vks4ePquuK0kddwDgMxaCmcFwz0RERERUVFiwCcqYPmpwcuy497FDmPe9keFEtBxDxjDvYuTHNZWDPdEREREREWJAZ+oAClTNUhO1SAP2R7KNC2iNp/BtbvPO+5rVHJCaG8/KOwtv+MeYLgnIiIiIipODPhEBSC/NXhPEowd9w/jXuy4L48PutUtER33gDHcl3WSw4rhnoiIiIioWDDgE+WTWIOn0iEP2R63HiQjYlPJ7bgHAFsbK7gobBnuiYiIiIiKEQM+UT7ktwbvwvWnWPLTOahf7Lhv4YrgpjVKRMc9wHBPRERERGQpGPCJ8kirMyAhWZWnGjyg5HfcA4DcxgrOTnJYSUvGlxFERERERKUZAz5RHuSnBi+7jvth3X1Qt4R03AMM90REREREloYBn8hM+anBKw0d9wAgl1nBRSGHlOGeiIiIiMhiMOATmUGZpkFyihaGPCyVr9HqseLXf3HmSsntuAcAucwaLgpbhnsiIiIiIgvDgE+US4lKdZ5r8EpDxz3AcE9EREREZMkY8IleQqzBU+te2Y57CQC5rTHcl5TV/YmIiIiIXjUM+EQ5yG8N3q0HSYjYdMak476F/xvo26HkdNxLANjJreHsyHBPRERERGTJGPCJspHfGrysOu67tnBFpxLUcc9wT0RERERUcjDgE2UhPzV4AHDs3H2s2WXacf9Op9po6ltyOu4Z7omIiIiIShYGfKIM8lODJwgCdv91E78cet5xb2tjhaHd65aojnuJBLCX26CMg4zhnoiIiIiohGDAJ3qBMlWD5FQN8pDtYTAI2LD3Eg79Y9pxP6q3P6qXoI57iQRwkNugjKNtcQ+FiIiIiIjMwIBPBOOZ96QUTZ5r8LLquH/NxQ6j+wSggrNdAY60cDHcExERERGVXAz49MozGAQkKNVQqXXIyxX3WXXc13zDCSN7layOe4Z7IiIiIqKSjQGfXmm6ZzV46jzW4GXVce/rZuy4l9mUjI57wBjuHeUyODmWnC8kiIiIiIjIFAM+vbI0Wj0Sleo81+CVho57wBjuFXYyKBwY7omIiIiISjIGfHolqZ7V4OnzWIP377WnWPpzye64BxjuiYiIiIhKEwZ8euWkqLRIUmpgyMtqegD+OncfP5TwjnsAkEoAhb0MjiVonQAiIiIiIsoeAz69UpJTNFCm5a0Gr7R03AOAVCqBwkEGRzuGeyIiIiKi0oIBn14JgmBcKT9NrctTDV5p6bgHABsba5RxsGW4JyIiIiIqZRjwqdTTGwQkJKug1ujzVINn7Lg/jzNXnojbSmLHPQBYSSVwcpDBzrbkrPBPRERERES5w4BPpZpWZ0BCsirPK+UrUzWI2nK2xHfcA8a1Aso42iLhcd4qAYmIiIiIyLIx4FOppX62Ur4ujyvlZ9Vx71OrPIZ2L1kd94Ax3DsrZBD0WhgMefuyg4iIiIiILBsDPpVKaWotEpUa6POymh5KT8c98Dzc29naIDVVW9zDISIiIiKiQmJRSeX69esICAjA1q1bxW1Tp06Fp6enyf/atm1bjKMkS5ecokFCsjrP4f7fa08xd/1pk3DftYUr+nesXeLCvZX0ebgnIiIiIqLSzWLO4Gu1WowfPx6pqakm2y9duoQRI0bgnXfeEbdZWZWs6dFUNAwGAYkpeV8pHyg9HfdAeri3hVxmMf/MiYiIiIioEFnMJ//w8HA4OjqabBMEAbGxsRg2bBgqVKhQTCOjkkCvNyAhWQ2VNm8LyGXXcT+shw/quJYrqGEWGSupBC4KW9gy3BMRERERvTIs4tP/yZMnER0djZ9//hmtW7cWt9+6dQupqalwdXUtvsGRxVNr9UhMVkOrz9vicXqDAdF7L2fuuH/bH9UrlqyOe4DhnoiIiIjoVVXsCSApKQkTJ07E1KlTUalSJZPbLl++DAD44YcfcOjQIUilUrRs2RIfffQRFApFnp9TEIRMlwIUlrS0NJP/p4Kl0hiQlKqGLo81eBqtHj/svozz1+LFbRWc5RjW3Rvly9iUuJ+btbUUjo620Os0SNVpMt3O45EsBY9FsiQ8HslS8FgseoIgQCKRFPcwiApMsQf8L774AgEBAejSpUum2y5fvgypVIrXXnsNixcvxq1btzB79mxcuXIFq1evhjSPC55ptVpcvHgxv0M3y40bN4r0+Uo7a2traHQSJKdpoNXq8rSPNI0B208k4GHC85XlX3e2QUigAolP7iLxSUGNtmjI5bZwsrNG3AMthJcsQsDjkSwFj0WyJDweyVLwWCxaMpmsuIdAVGCKNeD//PPPOHXqFLZt25bl7R9++CH69+8PFxcXAICHhwcqVKiAt99+G+fOnYOfn1+entfGxgZubm55Hrc50tLScOPGDdSoUQN2dnZF8pyvgqRULVLStHDJ42p6TxJViP75Ah6/EO7rurpgYJBHieu4BwAbayu4KGSwtsr5Sy8ej2QpeCySJeHxSJaCx2LRi42NLe4hEBWoYg34W7ZswdOnT02uuweAadOmYefOnVi+fLkY7tO5u7sDAB48eJDngC+RSGBvb5+nx+aVnZ1dkT9naaQ3CEhIVsMgWEEuz1sQv/kgCREbzyH5hU74Fv6V0beDR4mrwQMAaysJXBRys76Y4PFIloLHIlkSHo9kKXgsFh1Oz6fSplgD/pw5c6BSqUy2dejQAWPGjEHXrl0xceJEPHr0CKtWrRJvP3fuHAAU2Rl4shy6Zyvlq/O4Uj5g7Lhf+tM5k310bemKTk1qlMhf8DZWUrg42cLGuuTNOiAiIiIiooJVrAH/9ddfz3J7uXLl8Prrr6Njx44YOXIkIiIi0LVrV1y/fh3Tp09HSEgIatWqVcSjpeKk0eqRkI+V8gHgz7P3sHb3f6Wi4x5guCciIiIiIlPFvsheTt5880189913WLp0KZYtWwaFQoEuXbpg3LhxxT00KkJqjQ7xyWroDXm73l4QBOz66wZ+PXRN3FaSO+4BQGYthbNCDhvrkndJARERERERFQ6LC/iXLl0y+XunTp3QqVOnYhoNFbdUlRaJSg0MeVxMT28wYMNvl3E45nnHvZODDKG9/Upkxz3AcE9ERERERFmzuIBPlC45RQNlmgZ5PHEPjVaP5b+cx9nY5313r5e1x+i3/VHeuWSuTCuzlsLFSf7S1fKJiIiIiOjVw4BPFkcQBCSmaJCq0iKPJ+6hTNUgcvMZXL+XJG6r+YYTQnv5wdG+ZHadMtwTEREREVFOGPDJohgMAhKUaqjUOuQx2+NxQhrCo//Bo/g0cZuvW3l80K1uiey4B4zhvqyTHFYM90RERERElA0GfLIYer0B8fmswbv1IAkRm84gKUUjbmsZUBl92pfMjnvAuCCgi8KW4Z6IiIiIiHLEgE8WQasz1uBpdHmvwcuq475by1oIalK9RHbcAwz3RERERESUewz4VOzUGh3ilWro9XmdlJ9Fx71UgneCSm7HPfAs3DvJYSUtmV9OEBERERFR0WLAp2KVqtIiKUXDjvsMGO6JiIiIiMhcDPhUbPJbg1caO+4BQG5jBWeGeyIiIiIiMhMDPhW5gqjBK40d9wDDPRERERER5R0DPhUpvUFAQrIaak3ea/CSUzWIKmUd9wDDPRERERER5Q8DPhUZrc6AhGRVvlbKL40d9wDDPRERERER5R8DPhUJtVaPxGQ1tPq8h/ubD5IQsTEGyalacVtJ77gHGO6JiIiIiKhgMOBToUtTa5GozPtK+UDp7LgHGO6JiIiIiKjgMOBTocrvSvnAs477Xf/BIDzvuB/YyQtNfCoV0CiLB8M9EREREREVJAZ8KhQFsVK+IAjY+ecNbDv8Qse9zArDe/jAu2bJ7bgHGO6JiIiIiKjgMeBTgdPrDUhQqqHW6PO8Ur7eYMCPey7hyJl74jYnBxlG9fZDtRLccQ8w3BMRERERUeFgwKcCpdXpkZCsztdK+WqNseP+3NXS1XEPMNwTEREREVHhYcCnAqPW6BCvVEOvz/sF91l13LtWLoORb/mW6I57AJDLrOCsYLgnIiIiIqLCwYBPBSIlTYOkFK24EF5ePI5PxcKNMXj8Qse9n3sFDOlap0R33AMM90REREREVPgY8CnfkpQaKFWaPC+mBwA37ychYpNpx32repXRp50npCU8FMtlVnBRyEv86yAiIiIiIsvGgE95JggCEpRqpKl1+Qr3568+wbKfz5t03HdvVQsdG5fsjnuA4Z6IiIiIiIoOAz7lid4gICFZla+V8oGsO+4HdfJC4xLecQ8w3BMRERERUdFiwCezaXUGJCSr8rVSfmnuuAcY7omIiIiIqOgx4JNZ1FpjDZ5On/dwrzcYsOG3Szgck7Hj3h/VKioKYpjFiuGeiIiIiIiKAwM+5VqaWotEpQZ6Q94n5ZfmjnuA4Z6IiIiIiIoPAz7lijJNg+R81uAlp2oQuekMbtzP0HHfyw+OdjYFMcxixXBPRERERETFiQGfciQIApJSNEhRafO1Un5p7rgHALnMGi4KW4Z7IiIiIiIqNgz4lC2DQUBiihppKl2+Vsq/cT8JkaW04x5guCciIiIiIsvAgE9Z0usNSEhWQ/VCN31enLv6BMt+PgeN9vmifKWl414CQG5rDPcl/bUQEREREVHJx4BPmWh1xpXy81ODBwBHz9zDut0ZOu6DvdC4bsnvuGe4JyIiIiIiS8OATybUGh0SlGro9HmflC8IAnYevY5tR66L20pTx70EgJ3cGs6ODPdERERERGQ5GPBJlKrSIiklfzV4eoMBP+65hCNnSmfHPcM9ERERERFZKgZ8AgAkp2igTNMgH9m+1HfcM9wTEREREZElY8B/xQmCgMQUDVLzWYNX2jvuJRLAXm6DMg4yhnsiIiIiIrJIDPivML1BQEKyGmpN/mrwSnvHvUQCOMhtUMbRtriHQkRERERElC0G/FeUVmdAQrIq3yvl37yfhIhS3HHPcE9ERERERCUFA/4rSK011uDp9PkL96W54x4whntHuQxOjrLiHgoREREREdFLMeC/YtLUWiQq87dSPpBNx30nLzT2Kfkd94Ax3CvsZFA4MNwTEREREVHJwID/ClGmapCcmr+V8kt7xz3AcE9ERERERCUTA/4rQBAEJKVokJLPlfJLe8c9AEglgMJeBkd7hnsiIiIiIipZGPBLuYJaKb+0d9wDz8K9gwyOdgz3RERERERU8jDgl2JanQGJSjXUWn2+9lPaO+4BY7h3cpDBgeGeiIiIiIhKKAb8Ukqt1SMxWQ1tPlfKL+0d9wAglUjg5CiDg7x0fFlBRERERESvJgb8UqigVsq/cT8JkaW44x4whvsyjjLYM9wTEREREVEJx4BfyhTESvkAcC72CZb9Uno77gGGeyIiIiIiKl0Y8EuJglopH8im4z7YC43rlo6Oe8AY7p0VMtjZMtwTEREREVHpwIBfChTUSvmCIGDH0evY/kLHvVxmhWGlqOMeAKykxjP3DPdERERERFSaMOCXcDq9wRju87lSvt5gwPo9l3A0Q8f96Lf9UfX10tFxDxjDvbPCFnIZD30iIiIiIipdmHJKsIJaKd/YcX8O564+FbdVLGePUb1LT8c9YAz3Lgpb2DLcExERERFRKcSkU0IV1Er5SSkaRG4+g5svdNzXqlIGH75VejruAYZ7IiIiIiIq/Zh2SiBlmgbJKVpxEby8ehSfivDoGDxOeN5xH+BRAe91KT0d9wBgZSWBiyPDPRERERERlW5MPCVIQa6Uf/1eIqI2nzHpuG9drwrebudRajrugWfhXiGHbSn6woKIiIiIiCgrDPglhN4gIFGphkqdv5Xygaw77nu0dkOHRtVKTcc9AFg/C/elaTYCERERERFRdhjwS4CCWikfAI6cuYv1uy+J0/utpBIM6uyNRnUq5nvflsTaSgoXhS3DPRERERERvTIY8C1cQa2Un13H/fCevvCqUTa/w7QoNlZSuDjZwsaa4Z6IiIiIiF4dDPgWrKBWytcbDFi/+xKOnjV23AuCAGeFLUb1Llkd94IgvPQSAmO4l8PGWlpEoyIiIiIiIrIMDPgWqqBWyldr9Fj2yzmcf9Zx//TK73BytMPEkeNRrkzJ6bjfse0n3LpxHR+O/jjb+8ispXBWFF24v3TpEiZNmoTY2FhUq1YNO3fuLJLnLWoXLlzAgAEDcPz4cXz++ec4ceIE/vjjj3zt886dO3jzzTdNtllbW6NcuXJo3bo1xo0bh7JlTWeW/PHHH1i9ejX+/fdfqNVqVKxYEa1bt8aIESNQrlw58X4DBw7EiRMnsn1uPz8/bNy4MV/jJyIiIiKyRAz4FqYgV8rPquP+6aU96DBwSIkK9wCwdvUK+AfUz/Z2mbXxzL21VdGduY+MjMS9e/cQGRmZKYyWJgcPHkTjxo0hk8kKfN8ffvghWrduDQBQq9W4fv06wsPDERsbi/Xr14v3++mnnzB58mT07dsXgwcPhp2dHWJjY7F06VLs378fW7ZsQZkyZcT7e3t7Y9q0aVk+p4ODQ4G/DiIiIiIiS8CAb0EMBgEJBbRSfnYd95eBUndtenGEewCIj4+Hh4cHWrVqVaTPW9QOHTqEbt26Fcq+q1WrBn9/f/HvjRo1go2NDaZMmYIrV67A3d0dgPHLlM6dO+OLL74Q79u4cWM0aNAA3bp1w6ZNm/DBBx+Itzk6Oprsl4iIiIjoVcALlS3EmbPn8M7AQWjXuimC27fA+LEf4sL5c+LtZ8/8g3GjhqLTm03RrVMbzPrycyTEx4u37975K9q1CsTFf8/hg/cGol+3Nji+5TPEXT0IAGhTvwqi570HAFizcinaNn9+Nvz6tVhMmTgWIR1aIqRDS3w2+RPcu3tHvD3m9Cm0bV4fp0+dwISPRqLTm03xVtcOWBq1EHr985X9tVotvl8WhQG9uyKobVO8P/Bt7Nm1zeR1Hj18ACOGvIOObZvgra4dEPHdt0hLS0NO+vUKwcMH97Fn13a0bV4fD+7fE1/vnh0/o1vn9mjapDFiY2Oh1+uxdOlShISEwNfXF/7+/ujbty+OHTsm7i88PBzt27fHgQMH0KVLF9StWxcdO3bEzz//bPK8q1evRlBQEHx8fNCiRQt88cUXUCqVAABPT0+cOHECJ0+ehKenJ7Zu3QoAuHHjBsaMGYNmzZrB398fAwcOxN9//y3u886dO/D09MTKlSsRFBQEPz8/bNmyBeHh4QgKCsLevXsREhICHx8fdOvWDf/88w9iYmLQu3dv+Pr6IiQkBH/99ZfJOC9fvozhw4ejXr16qFevHkJDQ3H79m3x9uPHj8PT0xMbNmxAp06dMGTIEBw7dgxxcXH45JNP0KxZM/H5Mr4HiYmJOHPmDFq2bJnjzyijffv2oX///ggICEDdunURFBSEdevW5eqx6WfiX1xv4cmTJxCymNJSu3ZtTJ48GXXr1jVrfEREREREpREDvgWIi0/E0KEfQOFUBl98+S0+C/sKKlUaJn0SCqUyGWdiTmP82BGQ28rx+fRZCB3zCWL++RsfjxkOtVol7kcwGDB1ykSkyt1ROfB92LnUwJOLO+BfKRlvt/NAxOJVAIDgkG7in2/fuonRI95HfHw8Jv3vC4z/9DPcv3cXY0a+j/j4OJNxzpw+Fb5+AZg5+zu82T4IG9avxs5tPz+/Pex/2LRhLYK7dMdXs79Dw8DG+GbmF/h9724AwO+/7cJnkz9Bteo1MOOruXj3/WHYu2cnPvv04yzDW7rpX81B2XLl0KhJM0QsXoWy5coDAAx6PTZFr8NXX83E5MmTUatWLcyZMwdRUVHo06cPli9fjhkzZiAhIQFjx441+SLh8ePHmD59OgYNGoSlS5eiSpUqmDRpEq5evQoA2L59O7799lsMGDAAK1asQGhoKH755RfMmDEDABAdHQ1vb294e3sjOjoarVu3RmxsLHr27Ik7d+5g6tSpmDNnDiQSCd59991M14SHh4dj6NChmD17Npo1awYAePDgAWbNmoURI0ZgwYIFSEpKwpgxY/Dxxx+jd+/eiIyMhCAI+Oijj6BSGX/u169fR9++ffH06VN88803mDlzJm7fvo1+/frh6dOnJs8ZERGBjz/+GIMHD4afnx8mTJiAq1evIiwsDMuWLYO3tzcmTZpk8mXIkSNH4OrqijfeeCPbn09GBw4cQGhoKOrUqYOoqCiEh4ejatWqmD59Os6cOWNyX4PBAJ1OB51OB5VKhf/++w9RUVFo3Lgx3NzcxPu1bt0aO3bsQGhoKLZv346HDx+Ktw0ePBiNGzc22a8gCOJ+M/4vp2ONiIiIiKgk4xT9YqbTG/DP2QtITEhAz979UNfHDwBQtXoN7PjlJ6SlpmL5kghUrVYdM2d/Bysr4/R6rzo+eG9gb+za/iu6v/U2AGOokVVpCaeqgQAAh3I1cfXxBWjiLkMikcC7rg8AoHyF18U/r1m5FLZyOeZ8FwUHB0cAQL0GgRjwdldEr1+DEaHjxLF27tIdAwcPNd6nfiCOHjqAv/48jC7d38L1a7E4dOB3hI75BG+93V/cz4MH940zANp1xNLF4Qhs1BRTPv9S3GeVKtUwftyHOP7XETRu2iLL98jdozZsbGRwdnYRx50+HX/EiBHiNdwA8OjRI3z00UcYOHCguM3W1hajR4/GpUuXxGnbaWlpmDlzJpo0aQIAqFGjBtq0aYODBw+iVq1aOHHiBKpUqYIBAwZAKpUiMDAQ9vb2SExMBAD4+/vD0dFR/DMATJ8+HTKZDGvWrBFva926NUJCQjB79mxs3rxZHFOnTp3w1ltvmbzOtLQ0TJs2TTxbHhsbi7lz52LmzJno1asXACA1NRVjxozB9evX4eXlhYiICNjZ2WHVqlXiczZp0gTt2rXD8uXLMWnSJHH//fv3R/v27XHx4kXY2dnhxIkTCA0NRbt27QAAgYGBcHZ2NrnW/tChQ2afvY+NjUWPHj3wv//9T9wWEBCARo0a4fjx4/Dz8xO3/+9//zO5HwA4Ozvjhx9+MNk2Y8YMGAwG/Pbbb9i3bx8A4/T+N998E++99x5ef/11k/ufPHkSderUyXJ8CxYsQFBQkFmviYiIiIioJGDAL2aCIKBGzVpwdnbB/yaOQ+u27dEgsDEaBDbBsJFjoFKl4eK/5/B2v4GAIECv0wEA3nijMqpXr4G/Tx1Dt569EXP5MQBA7lzd+P8yK4zoGYAvTrpAlcMU+NN/n4R/QH3IbeXivh3sHeDjG4C/Tx43ua93XV+Tv5d/7TWoVMZ9nzsbAwBo0aqtyX3CZn4LALh18zoeP3qI/gPfE58HAPz868HBwQGnTh5H46YtTG4DAKmVVaZqPLmNFRzsbAAAXl5eJrfNnTsXABAXF4dr167h5s2b2L9/PwBAo9GY3PfFa7QrVqwIwBigAeP13dHR0ejZsyfatWuHVq1aoUuXLjnW9J04cQJt2rQRgzZgXBm+c+fOiIyMREpKirg947jT1atXT/xz+fLGmQovBmJnZ2cAQFKSceHEY8eOITAwEHK5HLpn752joyMaNGiAP//802TfGZ+zUaNGCA8Px4ULF9CiRQu0atXK5AsBQRBw+PBhzJs3L9vXnJX0a+FTUlJw/fp13Lp1C+fOGS83yfgzGDVqlPgFjU6nw/3797FmzRr07dsXP/zwgxjSFQoFFi5ciDt37uDgwYM4fvw4jh8/jpUrVyI6Ohrff/89AgICxP3WqVMHYWFhWY6vWrVqZr0eIiIiIqKSggG/mBgMAq7dTURKmhZqrR7zI5Zh3Q/fY//vv2HbL1tga2uLDkEh6DfwPRgMBmxYtxob1q3OtB+ZzBY/7LqIM1eeAACkVjYo4yjD6N7+qPK6AhKpFIJgyHYcSYkJ2P/7b9j/+2+ZbnN2djH5u9xWbvJ3qUQKwSA824/xzLazS9aryaffvmDuLCyYOyvT7U+fPMaD+/fQv3cXk+0Tp0xDUHDX52OQWcFZIYf0WdC2t7c3uf+5c+cQFhaGc+fOwc7ODm5ubuL08oxTs+3snjcJSKVSk/sEBwfDYDBg/fr14jTzypUrY/z48QgODs7yNSYmJoqh/EXly5eHIAji9ftZjTvdi18OZDXOjBISErBz584sK/oyruyf8Tnnz5+PxYsXY9euXdizZw+kUimaNm2K6dOno3Llyjh37hzUajXq18++vSArcXFxmDZtGvbt2weJRILq1aujQYMGADL/DCpXrgwfHx/x7wEBAWjVqhVat26N8PBwLF682OT+6bMqBgwYAIPBgH379uHTTz/FjBkzxHUQAONK+S/ul4iIiIjoVcCAXwzOXHmMzX9cwd1HSjgrZFCp9XCwt0HPgR9h0pQv8N/Ff7F39w78+vNmlK9QARKJBL3e7o+27U2nFau1emw7cht/nr0vbqvgYocJAxvkugbPUaFAvfqN8Ha/dzLdln45QK7246gAACQmxKPCa8+nS9+6eR1JiYlweHb78JFj4V+vQZbjKFe+AhYtN52aXbHS82u/raQSuCjkkEqzPouuVCrxwQcfwNPTEzt27ICrqyukUikOHjyIPXv25Pq1pAsJCUFISAiSk5Nx5MgRLFu2DBMmTED9+vUzTQkHjIvDPXnyJNP2x4+NsytcXFzw6NEjs8eRE4VCgaZNm+K9997LdJu1dc7/vBUKBSZMmIAJEybg2rVr+P333xEVFYWwsDAsXboUhw4dQtOmTWFjY2PWmMaPH49r165h1apVCAgIgEwmQ1paWq675x0cHODq6oqbN28CAPbs2YNp06bhxx9/RM2aNcX7SaVSdOjQASdPnmSvPRERERERGPCL3JkrjxG5+QzSVDooHGxQxkGGu7F/4eChdbj/YAre6x6IOnV9UaeuL/7YtwdxcXFw96iNW7duwLO2t7ifx3FJCA0dDWkZNzjXaCpuH9bdJ8dwn36mOp2ff33cvHENbm4esHoWCAVBwMyw/6FK1Wpwc/fM1evy8fUHAPx59BC69egtbl+6KBxPHj9C5NLVcHEpiwf375m8jqdPHuPrGZ+jS/e3ULlyVZPbXmRlZQWZjVW24R4Arl27hoSEBAwaNMhkgbZDhw4BMC7ollvjxo2DVqtFZGQkFAoFOnXqBBsbG4SGhuLRo0dZBvyGDRti//79UCqV4pl4vV6PHTt2wMfHp1B65AMDAxEbGwsvLy8x0AuCgPHjx6N69erZXgpw7949DBkyBJMnT0ZQUBBcXV3h6uqKmJgYMVgfOnQIvXv3zvLxOfn777/Rp08fNGrUSNxmzs8gOTkZ169fFy9XcHd3R0JCAlavXm1Sk5fuxo0b8PDwMHucRERERESlDQN+ETIYBGz+4wrSVDqUK2MLiUQCqVSC1yu74bwg4MIfS7Ey9S7ebu+Dg/v3IiVFiZat26Jp85aYPGEsZob9D2926IT4pDQsWbIcyU9uoEo1Ywd79YoKPDwD2MtzPtvq6KjAv+fO4EzMafj6BWDg4KEYPWIwpkwch649ekEmk2HbL1tx9PABTPtydq5fWy13D7Rq0w5LohZArVLBzd0Tx48dxV9HDyFs5rewsrLC+8NGYv63X0FqJUWTZi2hTE7G2tXL8fjRI3h4Zh1EJQDkttZwLuOEixcv4sSJE/D19c3yvjVr1oSjoyMWL14Ma2trWFtbY8+ePeLidi+r43tR48aNMW3aNHzzzTdo2bIlkpKSEBERgRo1aqB27dpZPmbUqFE4dOgQBg0ahGHDhsHGxgZr167F7du3sXz58lw/tzlGjhyJvn37Yvjw4ejXrx9sbW0RHR2Nffv2YeHChdk+7o033kDFihXx5ZdfQqlUolq1ajh//jwOHjyI4cOHIy4uDufPn0d4eHimxyqVSqxatSrLfXbo0AG+vr7Ytm0b6tSpg4oVK+L06dNYunQpJBJJpp/BrVu3EBMTI/79yZMnWL58uTgbAwBcXV0xbNgwLFmyBPfu3UPXrl1RsWJFPH36FL/88gv++usvrFy5MtMYX9xvRj4+PmbNUCEiIiIiKgkY8IvQtbuJuPtICYWDjclibXIHZwQEjULsyV/xz75ViNmrg2stN3zx5WwE1GsIAJg9NwKrVy7FF/+bCL0ggcypCqo0Hgo7l+poU78KnDQpOJGLWegDBr2PH1Ytx+Txo7Fy7WbUcnPHd5HL8f3SKHw943MIgoCarrUw/eu5aNa8lVmvb8rnX2L190uwZdOPSExIQLUaNTDty9lo3rINAKBzlx5wsHfAhvVrsP3Xn2BnZ4e6Pv6Y8vmXqPRG5Uz7kwCwk1vD2dEWQ4YMwVdffYUhQ4ZkCnPpFAoFoqKiMHv2bIwdOxYODg7w8vLC2rVrMXToUJw6dQpt27bN8rEZ9e3bF1qtFhs2bMD69eshl8vRpEkTTJgwIdsp6+7u7li/fj3mzZuHyZMnQyKRwNfXF2vWrBGvQS9otWvXxrp16zB//nxMnDgRgiDAw8MDkZGRePPNN3N8bEREBObNm4cFCxYgPj4elSpVwqhRozBs2DBs374d7u7uWc5USExMxNdff51pe5MmTdChQwfMmjULM2bMECsFa9SogbCwMPz66684deqUyWMWLVqERYsWATDOLlEoFKhTpw5WrFhh8p59/PHH8PLywqZNm8QvJZycnNCgQQNs3rw505cuFy5cQJ8+fbJ97SdPnoSTk1OO7w8RERERUUkjEV6xUuj01byLagGu1NRUXLx4EV5eXvjvlhLfbTgNF4WtONW8rJMtklM0SFHpYBAEJKVo8G5wHdSpVS7Tvs7GPsHyX85Bo30+zblnGze0D6yW4+ruJdGL4b60vbbi9OLxmN1Cf0RFgcciWRIej2QpeCwWvaLOBkSFjWfwi5CTgwzWVlJo9QbYSjNPD9bpBFhJpXC0z3yG+HDMXazf8x/Sv46xkkowqLM3GtWpWNjDLnIM90REREREROZjwC9CrpXLoPJrjrhxPwkyJ6lJeBUEASlqLSpXcETV1xUm27cdvoadf94Qtxk77n1Ru0bWlXQlmURiXEegjIOM4Z6IiIiIiMgM0pffhQqKVCpBr7busLO1xtMkNdQaPQwGARqdgIQUDexk1ghqXEOcvq/XG/DDzosm4b6MowzjB9QvteHeQW7DM/dERERERER5wIBfxPzcKyC0lx9qVHKCSqNHYooGGq0elSs4on/H2mJwV2l0iNpyFn+ee95xX7GcPSYObIAqL5zhLy3Sw30ZR9viHgoREREREVGJxCn6xcDPvQJ8apXHtbuJSEnTQq3VoVJ5R/HMfVKKGpGbzuDmg2TxMW5VyuDDt/zgYJdzDV5JJJEAjnIZnBwLvieeiIiIiIjoVcGAX0ykUgncqjpDq9PjSUIaDM8Wz3sYl4rwjTF4kvC8LzzAswLe71IHNtalr7dbIgEUdjIoHBjuiYiIiIiI8oMB34Jcv5eIyE1noEzTitva1K+C3m96iGf3SxOGeyIiIiIiooLDgG8hzsY+wbKfz0GrK/0d9wAglQCODPdEREREREQFhgHfAhz65y7WZei4f7ezNwJLYcc9YAz3CgcZHO0Y7omIiIiIiAoKA34xEgQBP/52CZt+vyJuK80d94Ax3Ds5yODAcE9ERERERFSgGPCL0aKtZ7HLpOPeFqN7+5XKGjyA4Z6IiIiIiKgwMeAXk9sPk03CfcVy9hjzdgDKlpEX36AKkVQiQRlHGezlpa/mj4iIiIiIyBIw4BcTF4UtXBS2iE9Ww72qM0b09C2VHfeAMdw7K2Swsy2dr4+IiIiIiMgSMOAXE0d7GRZ+0ga3HiahXBk5gNK3Uj7AcE9ERERERFRUGPCLkbPCFg52ZfEkIQ0GobhHU/CspMZp+Qz3REREREREhY8BnwqFlVQCZ4Ut5DIeYkREREREREWB6YsKnJVUAheFLWwZ7omIiIiIiIqMtLgHQKULwz0REREREVHxYAqjAmNlJYGLI8M9ERERERFRcWASowJhZSWBi0IOWxur4h4KERERERHRK4kBn/LN+lm4lzHcExERERERFRsGfMoXayspXBS2DPdERERERETFjAGf8szGSgoXJ1vYWDPcExERERERFTcGfMoTmbUUzgo5bKxZxEBERERERGQJmM7IbAz3RERERERElodn8MksMmspXJzksLZiuCciIiIiIrIkTGmUawz3RERERERElotn8ClXbG2s4KKwhRXDPRERERERkUViWqOXYrgnIiIiIiKyfDyDTzmS21jB2UkOK6mkuIdCREREREREOeApWcoWwz0REREREVHJwTP4lCW5zArOCoZ7IiIiIiKikoIBnzKRy6zhorCFlOGeiIiIiIioxOAUfTLBcE9ERERERFQy8Qw+AQAkAOS2xnAvkTDcExERERERlTQM+AQJADu5NZwdGe6JiIiIiIhKKk7Rf8Ux3BMREREREZUODPivMIZ7IiIiIiKi0oNT9F9REglg///27j065jv/4/hrZnKfhNiIhG3rEqJ1q1BZWhpNU3pKELb2QlO3LbKkisYWXaQny9mNbnNisWldaru7aJMNFQ4H69aDmu621UXIJmGpO6k6RJIxvz/yM7tTl6AxM/nm+Tgnx5nPd77zfWe8z+fk9f185zsBvmpo9SPcAwAAAIABEPDrIZNJsgb4qmGwv6dLAQAAAADUEi7Rr2cI9wAAAABgTKzg1yMmkxQc4KcGwX6eLgUAAAAAUMsI+PWEySSFBPopxEq4BwAAAAAj4hL9eoBwDwAAAADGxwq+wZlNUkiQn4KDCPcAAAAAYGQEfAMzm6QQq5+CAwn3AAAAAGB0BHyDMpukBlY/WQn3AAAAAFAvEPANyGwyqUGwn6wBvp4uBQAAAADgJgR8gzGbTGoY7Kcgwj0AAAAA1CsEfAMxm0wKDfFToD/hHgAAAADqG74mzyAI9wAAAABQv7GCbwAWc/Vl+YR7AAAAAKi/CPh1nMVsUmiIvwL8+K8EAAAAgPqMVFiHWcwmNQrxlz/hHgAAAADqPT6DX0cR7gEAAAAA/4t0WAdZLCY1CibcAwAAAAD+i4RYx1gsJjUKCZC/r8XTpQAAAAAAvAgBvw7x+f9w70e4BwAAAAB8BwG/jvC1mNWogb98fQj3AAAAAICbcZO9OoBwDwAAAACoCSv4Xs7Px6zQkAD5+nAuBgAAAABwe6RGL0a4BwAAAADcLVbwvZSfj1mNGgTIx0K4BwAAAADUjPTohQj3AAAAAIB7xQq+l/H3tahRiL8shHsAAAAAwD0gRXoRwj0AAAAA4H6xgu8NTCYF+JgV2iBAFrPJ09UAAAAAAOogloq9gJ+FcA8AAAAA+H5Ywfcws7n6hnpmwj0AAAAA4Hsg4HsYq/YAAAAAgNrAJfoAAAAAABgAAR8AAAAAAAPwqoBfUlKimJgY5eXlOccOHjyo4cOHq3PnzoqPj9eKFSs8WCEAAAAAAN7JawJ+ZWWlpk6dqitXrjjHLl68qJEjR+qRRx5Rbm6ufvnLXyozM1O5ubkerBQAAAAAAO/jNTfZy87OVnBwsMvY6tWr5evrq/T0dPn4+CgqKkpHjx5VTk6OhgwZ4qFKAQAAAADwPl6xgr9v3z6tWrVK8+bNcxm32WyKjY2Vj89/z0N0795dpaWlOnfunLvLBAAAAADAa3l8Bf/SpUtKS0vTzJkz1bRpU5dtp06dUnR0tMtYkyZNJEknT55U48aN7+uYDofD5aMAD9LVq1dd/gU8iX6Et6AX4U3oR3gLetH9HA6HTCa+thrG4fGAP3v2bMXExCgxMfGmbeXl5fLz83MZ8/f3lyRdu3btvo9ZWVmpgwcP3vf+96O0tNStxwPuhH6Et6AX4U3oR3gLetG9vps3gLrMowE/Pz9fNptNH3/88S23BwQEqKKiwmXsRrAPCgq67+P6+vqqdevW973/vbh69apKS0vVokULBQYGuuWYwO3Qj/AW9CK8Cf0Ib0Evul9RUZGnSwBqlUcDfm5urs6fP6/evXu7jM+aNUvr169XZGSkzpw547LtxuOIiIj7Pq7JZPpeJwjuR2BgoNuPCdwO/QhvQS/Cm9CP8Bb0ovtweT6MxqMBPzMzU+Xl5S5jffr0UWpqqgYMGKA1a9Zo5cqVstvtslgskqQ9e/aoZcuWCgsL80TJAAAAAAB4JY/eRT8iIkLNmzd3+ZGksLAwRUREaMiQIbp8+bJmzJihoqIi5eXlafny5Ro7dqwnywYAAAAAwOt4xdfk3U5YWJjee+89lZSUKCkpSQsWLFBaWpqSkpI8XRoAAAAAAF7F43fR/67CwkKXx506ddKqVas8VA0AAAAAAHWDV6/gAwAAAACAu0PABwAAAADAAAj4AAAAAAAYAAEfAAAAAAADIOADAAAAAGAABHwAAAAAAAyAgA8AAAAAgAEQ8AEAAAAAMAACPgAAAAAABkDABwAAAADAAAj4AAAAAAAYAAEfAAAAAAADIOADAAAAAGAABHwAAAAAAAyAgA8AAAAAgAEQ8AEAAAAAMAACPgAAAAAABkDABwAAAADAAEwOh8Ph6SLc6R//+IccDof8/PzccjyHw6HKykr5+vrKZDK55ZjA7dCP8Bb0IrwJ/QhvQS+6X0VFhUwmk7p06eLpUoBa4ePpAtzN3ZOlyWRy28kEoCb0I7wFvQhvQj/CW9CL7mcymTiZAkOpdyv4AAAAAAAYEZ/BBwAAAADAAAj4AAAAAAAYAAEfAAAAAAADIOADAAAAAGAABHwAAAAAAAyAgA8AAAAAgAEQ8AEAAAAAMAACPgAAAAAABkDABwAAAADAAAj4AAAAAAAYAAEfAAAAAAADIOADAAAAAGAABPwHoKSkRDExMcrLy3OOzZw5U23btnX5iY+P92CVMLLTp0/f1G9t27Z19uTBgwc1fPhwde7cWfHx8VqxYoWHK4aR1dSPzI9wp/z8fL3wwgvq2LGj+vXrpw0bNji3HT9+XGPHjlWXLl3Us2dPvfPOO7Lb7R6sFkZ2p15ctGjRLedNAKiJj6cLMJrKykpNnTpVV65ccRkvLCzUuHHjNHz4cOeYxWJxd3moJw4dOiR/f39t3rxZJpPJOR4SEqKLFy9q5MiRio+P15w5c/T5559rzpw5slqtGjJkiAerhlHdqR8l5ke4z5o1azRjxgxNnz5dvXr1UkFBgSZPnqzIyEh16NBBo0ePVosWLbRy5UodO3ZMM2bMkNlsVmpqqqdLh8HcqRdjYmJUWFiogQMH6vXXX/d0qQDqGAJ+LcvOzlZwcLDLmMPhUFFRkV555RWFh4d7qDLUJ4cPH1aLFi3UpEmTm7a9//778vX1VXp6unx8fBQVFaWjR48qJyeHgI8H4k79yPwId3E4HMrKylJycrKGDRsmSRo/frxsNps+/fRTnThxQl9//bVWr16thg0bKjo6WufPn9dvf/tbjRs3Tn5+fh7+DWAUNfViTEyMDh8+rKFDhzIvArhnXKJfi/bt26dVq1Zp3rx5LuPHjh3TlStX1KpVKw9VhvqmsLBQUVFRt9xms9kUGxsrH5//nt/r3r27SktLde7cOXeViHrkTv3I/Ah3KSkp0YkTJ5SYmOgyvmTJEo0dO1Y2m03t27dXw4YNndu6d++uy5cv6+DBg+4uFwZWUy9WVFSotLSUeRHAfSHg15JLly4pLS1NM2fOVNOmTV22HT58WJL0pz/9SfHx8UpISFB6erq+/fZbT5SKeuDw4cO6cOGChg0bpieffFI/+9nPtGPHDknSqVOnFBkZ6fL8GyurJ0+edHutML479SPzI9ylpKREknTlyhWNHj1aPXr00IsvvqitW7dKYm6E+9TUi0VFRbLb7dq4caP69u2r3r176/XXX9eZM2c8WTaAOoKAX0tmz56tmJiYm87GStV/wJrNZjVp0kSLFy/Wr371K+3atUspKSm6fv26B6qFkVVVVam4uFjffPONJk6cqJycHHXu3FmvvPKKdu/erfLy8psuNfX395ckXbt2zRMlw8Bq6kfmR7jL5cuXJUnTpk1T//79tXTpUj311FNKSUlhboRb1dSLN058BgYGKisrSxkZGSouLlZycrLKy8s9WTqAOoDP4NeC/Px82Ww2ffzxx7fcPn78eP385z9Xo0aNJEnR0dEKDw/X0KFDtX//fj3++OPuLBcG5+Pjo71798pisSggIECS1KFDBx05ckRLlixRQECAKioqXPa58cdrUFCQ2+uFsdXUjzk5OcyPcAtfX19J0ujRo5WUlCRJeuyxx3TgwAEtW7aMuRFuU1Mv5uTk6Omnn9YPfvAD5z5t2rTR008/ra1bt+qFF17wSN0A6gZW8GtBbm6uzp8/r969eysmJkYxMTGSpFmzZmnMmDEym83OP15vaNOmjaTqSwKB2ma1Wp1h6oY2bdro9OnTioyMvOkyvxuPIyIi3FYj6o879SPzI9zlxvwWHR3tMt66dWsdP36cuRFuU1MvSnIJ91L1x0VCQ0OZFwHUiIBfCzIzM7V+/Xrl5+c7fyQpNTVVGRkZSktL04gRI1z22b9/v6TqyRyoTUeOHFGXLl20d+9el/GvvvpKrVu3Vrdu3fTZZ5+5fLfznj171LJlS4WFhbm7XBhcTf3I/Ah3ad++vaxWq7744guX8cOHD+uRRx5Rt27ddODAAefl01L13Gi1WvXoo4+6u1wYWE29+Pvf/159+/aVw+Fwbjt+/LguXrzIvAigRgT8WhAREaHmzZu7/EhSWFiYIiIi1LdvX+3evVsLFizQsWPHtH37dk2fPl39+/e/7Z2lgfsVFRWlVq1aKT09XTabTf/+9781d+5cff755xo/fryGDBmiy5cva8aMGSoqKlJeXp6WL1+usWPHerp0GFBN/cj8CHcJCAjQmDFj9Ic//EHr1q3TsWPHtGjRIn3yyScaOXKkEhISFB4erkmTJunQoUPavHmz3n77bY0aNYqvyEOtqqkXn3vuOZ04cUKzZ89WSUmJ9u3bp4kTJ6pLly7q1auXp8sH4OVMjv89PYha07ZtW82dO1eDBw+WJG3YsEE5OTkqLi5WSEiIEhMTNWnSJOcNfIDadO7cOc2fP187d+7UpUuX1K5dO02dOlVPPPGEJOnLL79URkaGDhw4oPDwcI0aNUrDhw/3cNUwqpr6kfkR7rRs2TJ98MEHOn36tKKiojRx4kQlJCRIko4ePao5c+bIZrOpYcOG+vGPf6yJEyfKbGY9BLXvTr24e/duZWVlqbCwUH5+fnr22Wc1bdo0l69xBIBbIeADAAAAAGAAnJIGAAAAAMAACPgAAAAAABgAAR8AAAAAAAMg4AMAAAAAYAAEfAAAAAAADICADwAAAACAARDwAQAAAAAwAAI+AAAAAAAGQMAHADwwU6ZMUdu2bbV06VJPl1LnzJs3Ty+99JKnywAAAHUIAR8A8EB8++232rx5s6Kjo7Vq1So5HA5Pl1RnLF26VMuWLfN0GQAAoI4h4AMAHoh169ZJkmbMmKHS0lLt2bPHwxV5v//85z+aMGGCMjMzFRIS4ulyAABAHUPABwA8ELm5uerRo4e6d++u5s2ba+XKlc5to0aN0uDBg2/aJyUlRQMGDHA+ttlsGj58uB5//HHFxsZq2rRpunDhgnN7Xl6e2rVrpw8//FBPPfWUYmNjVVRUJLvdrpycHPXv31+dOnVS586d9dOf/vSmkwzbtm3T4MGD1alTJ/Xt21fr1q3Tc889p+zsbOdzysrK9Otf/1pPPvmkOnbsqKFDh2r37t339F4sXrxYHTp00DfffOMyvnz5crVv317nz5+XJM2dO1dHjx7V+++/r8cee+yejgEAAEDABwDUuiNHjmj//v0aNGiQJGnQoEHasmWLzp07J0kaMGCA/vWvf+no0aPOfS5duqQdO3Zo4MCBkqR9+/ZpxIgRCggI0DvvvKPp06fr008/VXJyssrLy5372e12LV26VBkZGXrjjTcUFRWlzMxMLVy4UD/5yU/03nvv6a233lJZWZleffVVXb16VZK0Z88epaSkqGnTpsrOztawYcM0a9YsnTx50vna165d08svv6wtW7botdde04IFCxQZGakxY8bcU8hPTExUVVWVNm3a5DJeUFCgnj17KiwsTJI0adIkrV27Vt26dbuHdxsAAKAaAR8AUOtyc3MVGhqq+Ph4SVJSUpLsdrs++ugjSVKfPn0UFBTkvIxfkjZt2iS73a7+/ftLkubPn6+WLVvqj3/8o5555hkNGjRIS5cuVXFxsXJzc12ON27cOPXu3VtJSUkymUw6c+aMXnvtNb300kuKjY1VYmKiJk+erLKyMhUWFkqSsrOz1aZNGy1YsEBxcXFKTk5Wenq6Kisrna+7Zs0aHTp0SAsXLtSLL76ouLg4ZWVlKSYmRpmZmXf9fvzwhz9Ut27dXH7fY8eO6csvv3Se0JCk6OhomUymu35dAACA/0XABwDUqsrKSq1du1YJCQkqLy/XpUuXZLVa1bVrV61evVrXr19XUFCQEhIStH79eud+BQUF6tGjhyIiInT16lV98cUXiouLk8PhUFVVlaqqqvTwww8rKipKn3zyicsxv3s5+/z58/Xyyy/rwoULstlsys3N1dq1ayVJFRUVqqio0D//+U/16dPHJVA///zz8vHxcT7evXu3wsPD1b59e2cNdrtdzzzzjL766qubLrm/kwEDBmjfvn06e/as8/cNDg52ngQBAAD4vnxqfgoAAHdv27ZtOn/+vD766CPniv3/2rlzp+Li4jRw4ECtXbtWhw4dUuPGjbV371795je/kVR9uf7169f17rvv6t13373pNfz9/V0eBwUFuTzev3+/5syZo/379yswMFCtW7dWs2bNJEkOh0NlZWWy2+3OS+NvsFgsCg0NdT4uKyvT2bNn1b59+1v+rmfPnlXDhg1rflNUffLgrbfe0oYNG5ScnKyCggL17dtXAQEBd7U/AABATQj4AIBalZubq4cfflgZGRku4w6HQxMmTNDKlSsVFxenHj16KDw8XBs2bFB4eLj8/f3Vp08fSZLVapXJZNKIESPUr1+/m44RGBh42+NfvnxZY8aMUdu2bVVQUKBWrVrJbDZr+/bt2rhxoyQpLCxMvr6+znsC3HD9+nWVlZU5H4eEhKhFixa3vRz/oYceuqv35MZrxcfHa8OGDerevbuOHDmiN9988673BwAAqAkBHwBQa86ePaudO3dqzJgx+tGPfnTT9ueff155eXk6ffq0IiIilJiYqL///e9q0KCBEhISnCvxwcHBateunYqLi9WxY0fn/uXl5UpNTVVcXJxat259yxqKi4tVVlam5ORkl+fs2LFDUnWIt1gs6tKli7Zs2aIJEyY4n7N161ZVVVU5H8fGxmrbtm0KCwtzXgEgVd8V/+DBg/f0OXxJGjhwoFJSUvTXv/5VzZo1U2xs7D3tDwAAcCd8Bh8AUGvy8/NVVVV1y1V3qfpu+na7XatXr5ZUHXgLCwv12WefudxsTpImT56sXbt2acqUKdq+fbu2bt3qvHv97S6Zl6SWLVsqODhYixcv1rZt27Rr1y69+eab+stf/iJJzrvop6am6tChQ0pNTdWOHTu0cuVK54r6jc/lDx48WM2aNdPIkSP1t7/9TXv27NHbb7+trKwsNWnSRL6+vvf0/vTq1UuhoaFatWqVEhMTuaEeAACoVQR8AECtycvLU5s2bRQdHX3L7V27dtVDDz2kDz/8UHa7XY8++qiio6MVFhamHj16uDy3Z8+eWrJkiU6dOqXU1FSlpaXJYrFo2bJl6ty5821rCAkJ0cKFC+VwOPTqq68qLS1NX3/9tT744ANZrVbZbDZJ0hNPPKHs7GyVlJQoJSVFy5YtcwZ8q9Uqqfqz/X/+85/VtWtX/e53v9MvfvELbdq0SVOmTNEbb7xxz++Pj4+P+vXrJ7vdrgEDBtzz/gAAAHdicjgcDk8XAQCAu23ZskWRkZEuVwMcOXJE/fv318KFC/Xss896sDoAAIB7x2fwAQD10q5du7R+/XpNnTpVLVu21OnTp7Vo0SK1atVKPXv2vKvXcDgcstvtNT7PYrFwOT4AAHjgWMEHANRL5eXlysrK0saNG3XmzBmFhoaqV69emjJliho3bnxXr5GXl3dXl+qvWLHiljcdBAAAqE0EfAAA7tPFixd1/PjxGp9348Z/AAAADxIBHwAAAAAAA+Au+gAAAAAAGAABHwAAAAAAAyDgAwAAAABgAAR8AAAAAAAMgIAPAAAAAIABEPABAAAAADAAAj4AAAAAAAbwf2bjLS1Ju7i8AAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -1042,133 +1135,244 @@ "metadata": {}, "source": [ "# Task selection \n", - "Here we do task selection for construction of MTEB(eng, v2, lite)" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [], - "source": [ - "tasks_not_in_index = [\n", - " \"ClimateFEVER\",\n", - " \"DBPedia\",\n", - " \"FEVER\",\n", - " \"HotpotQA\", # tasks which we will downsample (so will be included after the task selection)\n", - " \"SummEvalSummarization.v2\",\n", - "] # only summ task" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [], - "source": [ - "def is_candidate_valid_removal(current_tasks: list[str], task_to_remove: str) -> bool:\n", - " \"\"\"Determine if target task should be removed.\n", - " This checks that all task types are present in the current tasks\n", - " \"\"\"\n", - " # check if removing task removes a unique task type - if so, don't remove\n", - " _current_tasks = current_tasks.copy()\n", - " if task_to_remove in _current_tasks:\n", - " _current_tasks.remove(task_to_remove)\n", - " task = mteb.get_task(task_to_remove)\n", - " ctasks = mteb.get_tasks(tasks=_current_tasks)\n", - "\n", - " # don't remove a unique task type\n", - " task_types = {t.metadata.type for t in ctasks}\n", - " if task.metadata.type not in task_types:\n", - " return False\n", - "\n", - " return True" + "Here we do task selection for construction of MTEB(eng)" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 14, "metadata": {}, "outputs": [ { - "name": "stderr", - "output_type": "stream", - "text": [ - "Task: AmazonCounterfactualClassification: 0%| | 0/59 [00:00\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DBPediaHardNegativesSummEvalSummarization.v2
Rank
00.46571NaN
10.40407NaN
2NaNNaN
30.424750.314073
40.42578NaN
50.40379NaN
60.35720NaN
70.30667NaN
80.36958NaN
90.35697NaN
100.27419NaN
110.21176NaN
\n", + "" + ], + "text/plain": [ + " DBPediaHardNegatives SummEvalSummarization.v2\n", + "Rank \n", + "0 0.46571 NaN\n", + "1 0.40407 NaN\n", + "2 NaN NaN\n", + "3 0.42475 0.314073\n", + "4 0.42578 NaN\n", + "5 0.40379 NaN\n", + "6 0.35720 NaN\n", + "7 0.30667 NaN\n", + "8 0.36958 NaN\n", + "9 0.35697 NaN\n", + "10 0.27419 NaN\n", + "11 0.21176 NaN" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# results_df\n", + "\n", + "# columns with nan values\n", + "\n", + "_results_df[_results_df.columns[_results_df.isna().any()].tolist()]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "tasks_not_in_index = [\n", + " \"SummEvalSummarization.v2\",\n", + " \"DBPediaHardNegatives\", # remove them until we have results\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "def is_candidate_valid_removal(current_tasks: list[str], task_to_remove: str) -> bool:\n", + " \"\"\"Determine if target task should be removed.\n", + " This checks that all task types are present in the current tasks\n", + " \"\"\"\n", + " # check if removing task removes a unique task type - if so, don't remove\n", + " _current_tasks = current_tasks.copy()\n", + " if task_to_remove in _current_tasks:\n", + " _current_tasks.remove(task_to_remove)\n", + " task = mteb.get_task(task_to_remove)\n", + " ctasks = mteb.get_tasks(tasks=_current_tasks)\n", + "\n", + " # don't remove a unique task type\n", + " task_types = {t.metadata.type for t in ctasks}\n", + " if task.metadata.type not in task_types:\n", + " return False\n", + "\n", + " return True" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Task: TwitterURLCorpus: 100%|██████████| 62/62 [00:01<00:00, 59.78it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 61/61 [00:00<00:00, 67.21it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 60/60 [00:00<00:00, 70.91it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 59/59 [00:00<00:00, 71.41it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 58/58 [00:00<00:00, 69.97it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 57/57 [00:00<00:00, 65.43it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 56/56 [00:00<00:00, 56.90it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 55/55 [00:00<00:00, 66.02it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 54/54 [00:00<00:00, 65.71it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 53/53 [00:00<00:00, 76.50it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 52/52 [00:00<00:00, 66.05it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 51/51 [00:00<00:00, 61.66it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 50/50 [00:00<00:00, 67.87it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 49/49 [00:00<00:00, 70.84it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 48/48 [00:00<00:00, 73.11it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 47/47 [00:00<00:00, 75.61it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 46/46 [00:00<00:00, 56.74it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 45/45 [00:00<00:00, 73.59it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 44/44 [00:00<00:00, 77.61it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 43/43 [00:00<00:00, 75.50it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 42/42 [00:00<00:00, 76.87it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 41/41 [00:00<00:00, 78.52it/s] \n", + "Task: TwitterURLCorpus: 100%|██████████| 40/40 [00:00<00:00, 78.11it/s] \n" + ] + } + ], + "source": [ + "from sklearn.linear_model import LinearRegression\n", + "\n", + "# remove tasks one by one\n", + "tasks_to_select_from = [\n", + " t.metadata.name for t in tasks if t.metadata.name not in tasks_not_in_index\n", + "]\n", + "\n", + "tasks_removed = []\n", + "predicability_scores = []\n", + "\n", + "while tasks_to_select_from:\n", + " most_pred_tasks = task_selection.most_predictable_task(\n", + " results_df[tasks_to_select_from],\n", + " sklearn_estimator=LinearRegression(),\n", + " metrics=[\n", + " task_selection.spearman,\n", + " task_selection.pearson,\n", + " task_selection.mse_with_zscore,\n", + " ],\n", + " )\n", + "\n", + " # reverse the list to get the least predictable task\n", + " most_pred_tasks.reverse()\n", + "\n", + " while most_pred_tasks:\n", + " most_pred_task = most_pred_tasks.pop()\n", + " most_pred_task_name = list(most_pred_task.keys())[0]\n", + "\n", + " # if the task is too hard to predict, skip it (this essentially stops the loop)\n", + " if (\n", " most_pred_task[most_pred_task_name][\"mse_with_zscore\"] > 0.2\n", - " or most_pred_task[most_pred_task_name][\"spearman\"] < 0.9\n", + " or most_pred_task[most_pred_task_name][\"spearman\"] < 0.95\n", " ):\n", " continue\n", "\n", @@ -1184,12 +1388,12 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 18, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1oAAAL9CAYAAAAyzBd8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3hb5fXA8e/VlmzLK3HsDMfZey/CCgQS9ix70wKlQNmUwq+FQoGwdwsdjLBXGqBNgBACAQLZe5NhxzOO99CW7u8PWcpyEsu+GrbP53l4msrSe0+cRNa573nPUVRVVRFCCCGEEEIIoRldvAMQQgghhBBCiI5GEi0hhBBCCCGE0JgkWkIIIYQQQgihMUm0hBBCCCGEEEJjkmgJIYQQQgghhMYk0RJCCCGEEEIIjUmiJYQQQgghhBAak0RLCCGEEEIIITRmiHcA7cGqVatQVRWj0RjvUIQQQgghhBBx5PV6URSFMWPGHPZ5sqPVAqqqoqpqvMMAgrF4PJ6oxBPNtaO9vsQen/Ul9visL7HHZ32JPT7rS+yxXzva60vs8VlfYtdOS3MD2dFqgdBO1ogRI+IcCTgcDjZt2kT//v2x2WztZu1ory+xx2d9iT0+60vs8VlfYo/P+hJ77NeO9voSe3zWl9i1s27duhY9T3a0hBBCCCGEEEJjkmgJIYQQQgghhMYk0RJCCCGEEEIIjUmiJYQQQgghhBAaS6hE6x//+AdXXnnlYZ9TXV3NXXfdxYQJE5g4cSIPPfQQTqdzv+d88cUXnH766YwcOZJzzz2Xn3/+OZphCyGEEEIIIcR+EibRevfdd3n++eeP+Lxbb72VgoIC3nzzTV544QUWLlzIX/7yl/DXFy9ezD333MMll1zC7NmzmTx5MjfccAPbt2+PXvBCCCGEEEIIsY+4J1q7d+/mxhtv5OmnnyYvL++wz121ahVLly7liSeeYNiwYUyePJmHH36Yzz77jN27dwPwr3/9i5NPPpmrrrqKfv36ce+99zJs2DBmzpwZg9+NEEIIIYQQQiRAorVhwwaMRiOff/45o0aNOuxzly9fTteuXenXr1/4sYkTJ6IoCitWrCAQCLBy5UomT5683+smTZrEsmXLohK/EEIIIYQQQhwo7gOLp06dytSpU1v03N27d5OTk7PfYyaTibS0NEpLS6mrq8PhcJCdnb3fc7KysigrK9MsZiGEEEIIIYQ4nLgnWpFwOp2YTKaDHjebzbjdblwuF8BBzwl9vS1UVcXhcLRpDS2EGn8c2AAk0deO9voSe3zWl9jjs77EHp/1Jfb4rC+xx37taK8vscdnfYldO6qqoijKEZ/XrhIti8WCx+M56HG3243NZsNsNgMc9By3243Vam3Ttb1eL5s2bWrTGlrKz89vl2tHe32JPT7rS+zxWV9ij8/6Ent81pfYY792tNeX2OOzvsSujeY2fw7UrhKt7Oxs5s+fv99jHo+HmpoasrKySEtLw2azUV5evt9zysvL6datW5uubTQa6d+/f5vW0ILT6SQ/P5+8vLw2J4+xXDva60vs8VlfYo/P+hJ7fNaX2OOzvsQe+7Wjvb7EHp/1JXbtbNu2rUXPa1eJ1oQJE3j66acpKCigd+/eACxduhSAcePGoSgKY8eOZenSpVx44YXh1y1ZsoTx48e36dqKomCz2dq0hpasVmvU4onm2tFeX2KPz/oSe3zWl9jjs77EHp/1JfbYrx3t9SX2+KwvsbddS8oGIQG6Dh6O3+9nz5494bNXo0aNYuzYsdxxxx2sXbuWxYsX88ADD3DuueeGd6yuvfZa5syZwxtvvMH27dt58skn2bRpE1dffXU8fytCCCGEEEKITiShE63S0lKOPfZY5s6dCwSzx5dffpmePXty9dVXc/vtt3P88cfvN7D42GOP5bHHHuP999/nvPPOY/Hixbz66qv7tYQXQgghhBBCiGhKqNLBxx9/fL//37NnT7Zs2bLfY5mZmbz44ouHXefcc8/l3HPP1To8IYQQQgghhGiRhN7REkIIIYQQQoj2SBItIeIs4Haga6iMdxhCCCGEEEJDCVU6KERnVDPnZey71uPp0Q1b/1HxDkcIIYQQQmhAdrSEiCO/qxHPrvUoqopjzTfxDkcIIYQQQmhEEi0h4shVsB5UNfjrX5bidzXGOSIhhBBCCKEFSbSEiCPnzrV7/4/fS8P6H+IXjBBCCCFEKwU8TvB74x1GQpFES4g4cuYHEy1vRm8A6lfPj2c4QgghhBAR8zvr2fPv20lZ+h6qGoh3OAlDEi0h4sRXV4G3sgQUBcfw00FvwLN7J+7SHfEOTQghhBCixTx7ClHdDgy1xXh2bYh3OAlDEi0h4iRUNmjs1peALR1L//EA1EtTDCGEEEK0I35HbfjXjlXz4hhJYpFESwCgqiruwk2YilbLlm+MOPPXAWDKHQaAddgUABrWf0/A645bXEIIIYQQkfA37E203DvX4K0ui2M0iUMSrU5OVQM0bl5CyZv3Uf3JYyStn4tz3XfxDqvDU1U1vKNlyh3e9L9DMaRmEXA7aNy8OJ7hCSGEEEK02L47WqBSt+LLuMWSSCTR6qRUv4/6td9S9M872D3rSdwlv4S/5lj9NWpTy3ERHd49hfgba1AMJkw5/QFQFB0po6YCUL9aygeFEEII0T4EGoOJls+eA0D9mgUEPK54hpQQJNHqZAJeN7XL5rDr7zez578v460oQjHbSDv6fLpc8ySq3oivsghX4aZ4h9qhhboNWnKHohiM4cdTRp0Iig7Xrg14q0riFZ4QQgghRIv5GmsA8PQYjj41i4CrkYb138c3qAQgiVYn4Xc2UP3jJ+x6+UYq572Ov64CfVIaGSdeQe9bXiXjxMsxpOfgyQmeF5It3+gKlQ1a+4zc73GDvQvWvqOA4N0gIYQQQohEF3DUBf/XnIxt9MkA1C7/otNXSEmi1cH56qup/GYmu17+LdUL3yfgqMOQlkWXU6+n181/J+3o89BZksLPd+eOA6Bx82J89dXxCrtDU/0+nAXB1qfWvJEHfd3e9AZVv+Zb1IA/prElEm9VCeX/vh1z/tJ4hyKEEEKIw/A3lQ6qJhvWocejGM149+zC1clbvRviHYCIDm9VKTWLP6N+7bfg9wFgysolbfL5JA09GkWnb/Z1fns3jN0H4i3ZSv3q+aQfd2Esw+4U3CW/oHpd6Gx2TN1643TuX8NsGzAOnc2Ov7EGx7aVJA2cEKdI46txy1IC9ZWYyqSMVQghhEhk/qbSwYApCZ0lieQRU6hfOY/aZXOx9h4e3+DiSHa0Ohh32U52z36WwldvpX7V1+D3Ye45iG4X3UeP654lefhxh0yyQmyjTgKgbtW8Tr2jEi2OUNlg3ggU5eB/goreSMrIEwCoXz0/lqElFM/ufAAUV0N8AxGiiXvnGkzF6+IdhhBCJBTV5yXgdgR/bQpWSaWOPw0Ax9Zl+Gr3xC22eJMdrQ5AVVVchRup+Wk2zu2rwo9b+40h7ejzseYOjWg9S/8JNCS9h7++isatS0kePFnrkDu18PmsZsoGQ1JGnUTt4s9xbFuJr74KQ0pGrMJLGO7yfAB07nqZ7SbiTvX7qJnzEkleN54ho7ANGBPvkIQQIiH4m85nodOjGi0AmLrmYskbgSt/HXUrvyLjxCviGGH8yI5WO6aqKo2/LKfkrf+j9O0HgkmWoiNp6DH0+M3T5Fzyp4iTLADFYCQltKu14iutw+7UAm4H7uKtwMGNMPZl6tITc8/BoAaoX/tdjKJLHAGfB29FMQCKGkB1yq6WiC9vdRlq0yDxxlXyviiEECGh81k6awooSvjx0K5W3ar5BJrePzsbSbTao0AA5+afKP73nez+aAbuoi2gN5AyZhq9bnyRbufdiTm7T5suYR87PdhmPH8dnooijQIXzl0bQQ1gSM/GmJZ12OfaRweT3fo133S6rj3ePUWwzy6Wv1Eas4j48uwpDP/avW053tryOEYjhBCJI3Q+S2dL3e9x24DxGFK7EnDW07hxURwiiz9JtNoZ54YfsP/wKrVfvIKnfBeKyULqUeeQe/MrdD39RowZOZpcx5DaFduA8YC0eteSMz94vuNwZYMhSUOORjFZ8VWXdbquPZ6mssGQQENNXOIQIsS7T6KFqlK3/Iv4BSOEEAnE72ja0bLZ93tc0emxjzsVgNplczvdTWOQRKtd8VaXUTvvn+idNSjWFNKnXEruLf8g86SronKGJ/SPo37tdwQ8Ts3X74wONT+rOTqTheRhxwJQv/qbqMaVaNxNjTBC/A2yoyXiy7NnFwDejN4A1K+aL++LQgjBvqWD9oO+ljLqJBSDCc/uncEKrE5GmmG0I3p7Fyr6nEJRlZPG3GMxNCbB99qV9Xm8Xioq6lhfuh2T0QiqkZHmLljdFXz74UeUZ01q0/o+v48sq1ejaNsfX0M13j27AGW/VqclFY2sy3cwaNDBd3pSRp1E/aqvady8GP/036C3Jscw4vgJ7WgpRjOq101ASgdFnHkqgjtarj6TsATc+GvKqF/7XfgMghBCdFbhRCvp4ERLb0shedix1K9ZQO3yuVh6DY51eHEliVY7Ul7j5q8rugX/z85onpuqC/+q2NyH85MqMO/4nvdWpwLKoV/WAhajwsABDvrYbG2Msf0JlQ2asvugt6UAUFbZyJ/+sZRGl4+AcTu/Pnv/nS5z9/6YsnLxlO+iYcMPneJDnaqqeHYXAGDqNQz3jpX4pXRQxJHq8+KtKgXAn5yFbcx06r99i7plc7CPO6XZMQ1CCNFZhEsHm9nRArCPP536NQto3Ly403VSlkSrHemWYeOKUweweXsJ6enpGAza/vH5fD6qq6v3W9vgz8a3bTXdDTVcNlpHta13q9dfv62CwvIGnn1/LU/ddjwWU+f663dg2aDL42PGm8todAUHSs9euJOhfbty1PC95+wURSFl1ElUfv0G9au/6RSJlr++koCrAXR6TLnBREt2tGIn4HXTsPRzdPou8Q4lYXirSiDgRzFZUS0pWPuPovGnT/BWleLcvgpb/3HxDlEIIeImvKNlaz7RMmf3wdJrCK7CTdStnEfGlEtiGV5cda5Puu2coiicdUwe/TOcDBkyBJvGu0IOh4NNmzYdtPaeOVOoXz2fqcnb6Hbe2a1ev7C0irtfWkR+WT2vzFrL7ZeMQVHatkPWXqiqut/8LFVVeWXWWnaU1GJPMtIny8ianQ6ee38lz90+he5d95YIJg+fQuWCt4P1zaU7MOf0jddvIyZC57OMmT3Qp3YF5IxWLNUu+S8Niz7G2rU/jDs63uEkhFDHQUNmD1AUdCYLKWNOpnbx59QunSOJlhCiUwslWnpbKriaf459/Gm4CjdRv+pr0o/9FYreGMMI40fqHcQRhZpiBLd8W/+BNzPVwoXHZqAosGB5IV/+nK9RhInPW1WCv74SRW/E0mswX/6cz4LlhegUuO2ikZw9KZ1BuWk4XD4ee3MpLrcv/Fq9LYWkQcHzcfVrOn5TDE9TomXuloc+OR2AgCRaMdO46ScAjFW7UP2+Izy7c9ibaPUMP2YffxooOpw714QbZQghRGd0pB0tgKRBk9CnZOBvrKFh08+xCi3uJNESR2TO7oO55yAI+KlfPb9Na/XpZuGy6QMA+Oen69hcUKVFiAkvtJtl7jWYrSWN/PPT4Hmtq88YyvC+Geh1CndcMpL0FDMFZfW89PHq/dqgpjTN1GpY/32HH/oXSrRMWb3RJTUlWo461IA/jlF1Dt6qUjzlwfNxit+Dd/fOOEeUGEKNMPZNtIypWSQNmghA7dI5cYlLCCHiTVXVQ7Z335eiN2AfewoAdcvmxiS2RCCJlmiR1HGh6d7z2vyB96xjenP0yBx8fpXHZy6jpr5jJw6wN9FScoby+Mxl+PwqR4/M4bwT+oefk55i5t6rJqDXKXy/qpj//rgj/DVr3ggMqVkE3A4aNy+OefyxFOo4aOrWB50tBVVRAFUaYsTAgX+3PEWb4hRJYvE27VgZMnvs93jqxDOB4A0Qv6PuoNcJIURHF3A1QNPnwkM1wwixj5kGegPukl9wlWyLRXhxJ4mWaJGkwUehs9nx11fRuHVpm9ZSFIXbLh5Dz6xkKmtdPPXOcvz+gEaRJh414MdVsB6ADzboqax10TMrmdsuPviM2rC+mfz6rGEAvP75BjbsqARAUXSkjJoKdOyZWgGPC29VGRAsHVQUHao5eF7NV985dj/jqXFzsJzD0DXY9Maza2M8w0kIAZ8Hb/VuYP8dLQBzz8GYsvuh+jzUrWrbbr8QQrRH4bJBsw3FcPhzV/qkVJKHHgNA3fLOsasliZZoEcVgxD76ZADqVnzV5vVsFiP3XzMRq1nP2m0VvP1Fx71z7i7dTsDtwKu38G2+DqtZz/3XTMRmaf4N6azj+nL86B74AypPvLWMqrrgydKUUSeCosO1a0OwC1oHFDzroqJPSkOflApAwBxshe9vkEQrmry15bhLtwMKKVMuB8BTshXV13ln3wF4K4pBDaCzJKNr+jsZoigKqRPPAKBu+Rdypk0I0emEygb1B7w/Hop9/OkANGxcFE7SOjJJtESL2cdOD37Qz1+Hp6Ltc7x6dUvh1ovHADDr220sWtsxk4dQ2eBGZxYqOm69eAy9uqUc8vmKonDLRaPJzU6hut7Nk28vx+cPYLB3wdp3FAD1axbEJPZYC5/P6pYXfixgCe1oSUOMaGrcvAQAS+5QTD0HEzAlgd+Lq2RrnCOLr1CjC1PXXs12SU0eejT6pDT8DVXhHUEhhOgs/I3BsmmdrWWJlqV7f8zdB4DfR92qr6MZWkKQREu0mCG1K7YBwTbGdSu+1GTNY0f1CJ9TeuGDlRTurtdk3URSs3UVAFu8wTNZx47qcYRXgNVsaNr1MrBhRyVv/G8DQHhXsX7Ntx2yOUSziZbsaMVE6HxW0uCjUBQFX0awfNCZvz6eYcWdt6kRhrFrr2a/ruiNwQ6EQO2S/+3XxEYIITo6f2MN0PIdLSD8nlm34qsOXwnQqkSroKCADz74gKeffpoHH3yQ559/no8//pjS0lKt4xMJxt7UFKN+7XcEPE5N1rz69CGM6NcFp9vPjJlLcbg6TqlSQ3093tKmHYGcoVx9+pAWv7ZH12TuuHQsAJ9/v4PvVxVhGzAueFausQbHtpXRCDmu3E2NMMxZeeHH5IxW9Pnqq3AXbQYIjxLwZgYTrdD5ws4q1Nrd1KX5RAuCB7wVvRF36TbcxZ17B1AI0bmEZ2hFkGglDzkafVJqsBJgy5JohZYQIkq05s+fz4UXXsipp57KX/7yFz766CO+++47Zs6cyZ///GemTp3KxRdfzIIFHbOsSYC1zwiMGd1RPU4a1n2vyZp6vY57rhxHZqqFwt0NvPjh6g5xV1hVVT5+by4GAtSoydxw9Uno9ZHd2zhqeA4XnhRsh//iR6vZVe4kZeQJAG1utZ9oVDUQbi1u6tY7/HjAIjta0Rb6QWfuMRCDPRMgvKPlKtra4UcKHE64dDAr95DP0Selkjz8OABql/4vJnEJIUQiCJ/RamHpIATP/aeMmQYEz7d2ZC361FdcXMyVV17Jn/70J4YMGcLrr7/O8uXLWbp0KQsXLmTVqlUsXbqUv/3tbwwZMoQ//OEPXHrppRQWFkY7fhFjiqLDPi44B6F2xZeaJUTpKRb+eNUEDHqFRWtL+HThdk3WjadPF27HVxjcDUjpP4oMu7VV61x+6hBGD+iK2+PnsTeXYhg8BQDHtpUdapfHV1OO6nGh6I0Y92mjHSodlDNa0bNv2WBIwJaOLjkDAj5cTbtdnU3A48JXUw4cfkcLwD4h2BSjcfNifHUVUY9NCCESwd4drbSIXmcfewro9LgKN+Eu67gzG1uUaF1xxRVMmzaN77//nocffpjJkyeTnJy833PsdjtTp07lL3/5Cz/88APTpk3jyiuvjErQIr6SR56IYjTj3bMLV6F23QIH52Vw3TkjAHhzzkbWbWu/H1bWbavgzTkbGWgMtirPGTGh1WvpdQp3XzGOrulWSioaeemr3Zh7DgY1QP3a7zSKOP5C57OMXXuh6PThxwNNpYOyoxUd/sZaXE1t3PdNtFAUTL2GAuDqpOe0vE1Nf3Q2+xHLYszd8rDkjQA1QG0Hv0MrhBAhrSkdBDCkZIR/5nTkXa0WJVqzZ8/mqquuwmQytWhRq9XKr3/9a2bPnt2m4ERi0luSSB4WLJPRqilGyOlH53HiuJ4EAipPvr2cylptzoHFUmWtkyffXo5VddLLEEwOLL1HtGnN1GRz046fjiUbythiCs7aql/zDaraMWaQuUONMPY5nwWgNnUdDDgbOnUJW7Q0bl0GagBTdl+Mad32+5qpV/BMobOTntPyNDXCMHU9dNngvlKbdrXqV80n4HFFLS4hhEgUgXB798MPK25OaqjV+4Yf8Ds6XjM0aGGilZaWdsivud3uQ5aPpaentyookfjs404FmspkNCzpUhSFmy4YRZ/udmoa3MyYuQyvr/0kEl5fgBkzl1HT4OaYrOCbjymrN4bktDavPTA3nRvPDyZsr6wwoRos+KrLwrsR7Z0n1Ahjn/NZAKrBAobgTR5/g5QPai3Ukny/3awm5qYdLXfJNgLu9nfTo632be3eErYB4zCkZxNwNdCwbmE0QxNCiITga2XpIIC55yBM3fqg+jzUr/lG48gSQ6u6Du7YsYPbb7+diRMnMmbMGDZu3MhDDz3E22+/rXV8IkGZs/tg7jkIAn7NmzJYTAbuu3oiSVYjWwqqee3z9nM3/bXP17OloJokq5Gz+wV3X6x9Rmq2/ilH5TFtYi4u1chyVx4A9as7xpuTZ3eoEUbe/l9QlPAbuCRa2vI7G3DmrwOaT7T09i4Y0rqBGsBV2DES+ki0pOPgvhRFR+qE4B3a2mVzOsxusxBCNCfg86C6HUBkzTBCFEUJv2fWrfiyQ46tiTjR2rRpExdccAEbNmzgrLPOCu9m6fV6HnvsMSkX7ERSm1q9162ap/k/jpwuSdx1WbC1+ZxFO1mwPPEbqyxYXsicRcEDnXddNhalLPjB1JqnXaIFcOP5I+nfM5XvG/sC0LD5Z/zOBk2vEWt+VyO+2qamAweUDgLokoO74x2p+UcicPyyHAJ+jF1zMWU2P9/NmhfcRe2M87S8ew4/Q6s5KSOnophteCuLce5YE63QhBAi7gKO4LBidAZ0lqRWrZE09Bh01hR8tXuCP5M6mIgTrSeeeILhw4fzxRdfcN9994UTrT/96U9ccMEFvPXWW5oHKRJT0uCjgjOd6qto3LpU8/UnDM3m0umDAPjbJ2vYWVKr+TW0srOklr99EvxQdcm0QYzOCXbRQ6fHktvy2VktYTLq+ePVE6k2Z1PiSwOfl4YNP2h6jVgLlQ0a7F3QW5MP+rpeEq2oaK7b4IGsvYcDne+cVsDtCHcPbGnpIIDObMU+aioAtUvnRCU2IYRIBP6GGiB4PktRlFatoTOaSRl9EkCHbCQUcaK1evVqrrnmGgwGw0Hf1NNPP538/HytYhMJTjEYsY8+GQhO946GS6YNYtzgLDzeYGvzBocnKtdpiwaHh8feXIrH62fs4CwumT4I5861AFh6DERnal1b98PplmHj7ismsNgTnLFV/FP7fnM6ZNlgE11SMNGSzoPaCbidOHesBiD5MImWpSnR8pTtbPc7p5HwNHUc1Ceno7emRPRa+4TTQdHh3LEqvI4QQnQ0rZmh1Rz7uFNA0eHKXxc+G9tRRJxomc1mXK7muynV1NS0uDOh6BjsY6fv/ccRhQ8UOp3CXZePo1uGjbJKB8+8t5JAIHGGGQcCKs+8t5KySgdZGTbuvnwcep2CMz+YaGl5PutAYwdlkXfsKfhUHZb6Yratbr9lSp5DdBwM0Tc1E/HLLC3NOLatQPV7MWbkYDxMVz1DSnrTXDMV164NsQswziJthLEvY1o3bAODIx1ql8mulhCiY2pta/cDGVOz9r5ndrBdrYgTrWOOOYYXX3yRsrKy8GOKotDY2Mjrr7/O0UcfrWmAIrEZUrtiGzAO0L7Ve0iKzcR9V0/AZNCxfNNuPvpma1Su0xoffbOV5Zt2YzLouP/qCaTYTKhqINxgIJqJFsD5p4ymyDoQgFX/+4S6xsTb8WuJUOngkXa0fLKjpZl9ywaPVPLRGcsH957Pallr9wOFWr03rP0Ov7Njti0WQnRu/qYzWm1NtABSxwfP/TesW4jf1djm9RJFxInWPffcg8Ph4NRTT+Xyyy9HURQef/xxTj31VEpLS7nzzjujEadIYPamphj1a78j4IlOC+h+PdO46YJRALz31WaWb9odletEYvmm3bz31WYAfverUfTrmQaApyyfgLMBxWTFnNM/qjHodAqjz/gVAEPUX3ju7Z/xJ9COX0uoAX+4u9uBrd1DQme0/HJGSxMBrxvH9pUAJA2efMTnWzphQ4xIOw4eyJI7dG/b4lXadmYVQohE4G+sAbRJtCy9h2PsmovqdVO/ZkGb10sUESdaOTk5fPbZZ1x99dWoqkpubi4Oh4MzzzyT//znP/Tq1bofSqL9svYZgTEjB9XjpGHd91G7zkkTcjltch6qCs+8u4Kyyvjd8SirbOSZd1egqnDa5DxOnrj3rne4bLD3MBS9IeqxpA8aDcldsOm8ULCC95uSv/bCW1WK6vOgGC0Y0rObfc7eroPVh5zbJ1rOuX01qteNITULU3bfIz7f2js4INu7Z1e4VKSjCydarSgdhKa2xRODu1q1y79A9fs0i00IIRJBuHSwjWe0oOk9s2lXq27Flx1mPEar5milp6dz9dVX8+GHH/LVV1/x2muv8atf/YqsrCyt4xPtgKLowgOMa1d8GdUPwtefO5xBuek0OL3MmLkMtzf2MxfcXj8zZi6jwellYG4a1587fL+vhxphRLtsMERRdKSPDTYlOcq8jQ/nb2XphrIjvCpx7D2flYuiNP+WpGu6W6Z6XahR2jXtTPYOKZ7Uok5RepsdU1Zwt7EzlA/6nQ3hxiumLj1bvU7y0GPRJ6Xir6+kccsSrcITQoiEEG6GocGOFkDy8OPRWZLwVZfh3L5KkzXjLeJEq76+nuuuu47LL788/NiaNWs488wzufXWWw/ZKEN0bMkjT0QxmPDu2YWrcFPUrmM06Pnj1RNITTaxo7iWV2atiekOh6qqvDJrDTuKa7EnmfjjVRMxGvThrwd8nvDvX+v5WYeTMupEQGGAcTdddXU8+94KSiraR4e48PmsQzTCANAZLejMNkBavLeV6vPSuG0FcPi27geydKJzWt6K4G6W3t6l1bNhoKkz69imm1DS6l0I0cH4G0KJVpom6+lMFlJC4zGWdYymGBEnWk8//TSbNm3i97//ffixo446ipdeeomVK1fy0ksvaRqgaB/0liSShx8PRK8pRkiXNCv3XDEenQLfLCvky8UFUb3evuYvL+abZYXoFPjDlePpmr5/63Z30RZUnwd9cjrGNtwJj5TB3gVrv9EAnJ5VRKPLx4w3l+FyJ365krtpR+tQ57NC9CkZgJzTaivnzrWobgf65AzMPQa2+HWhwcWuTnBOq63ns/aVMnY66A24i7fgKk6cRj5CCNFWWrV331ewQkoJjseoLNFs3XiJONFasGAB9957L6effnr4MZPJxLRp07jzzjuZO3eupgGK9iNUPti4eTG+KLfhHjWgK1efMRSAf85eyy+F0T83UlTh5o05wfNPV50+lFEDuh70nH3LBls7vK+1QjPNxhl+ISPFSH5pHS9/HNsdv9Y40gytEENToiWdB9umIdxtcNIhSzWbY8kdCooOb1VJh99VbOv5rH0ZktNIHnYsIK3ehRAdh6oGNO06GGJMz8bWfywAdSva/65WxIlWQ0MDqanNf0O7du1KVVXH/gEsDs2c3Qdzz0EQ8FO/Ovpdts47oT+TR+Tg86s898EaGl3RO69V1+jhox+r8PtVJo/I4fwTm+8mGE60Ylg2GGIbMA6dzY7qqOWPU83odAoLVxXx1ZLCmMfSUn5H3d6zMF2PsKOVHNrRkllaraX6fTi2LgUiKxuE4K61ObsP0PHLB9syQ6s5oVbvjZt+xldXqcmaQggRTwFnIwSCn7v0SXZN17Y3NcWoX/MtAXf7PpcdcUu0wYMHM2vWLKZMmXLQ1z799FMGDRqkSWCifbKPO5U9RVuoWzWPtGPOR9Hpj/yiVlIUhdsvGcOusnqK9zTw/veV7KrdgdFk1PxaS9eXUufw072LjdsvGdPsbpXf2YC7dDuwt8wqlhS9kZSRJ1C7+HPSy5Zy7ZkX8drn63nri60cPSSZTbu1/954PV727Klr9doptdsYDLjMmXz8w8HT4Pddv0+FSg6waeN2djm2aBI7HieDByf2jp+WnAUbCLga0NnsWHoNifj1lt7DcZdux5W/jpSmUuGOKHRGq7UztA5kzu6LJXcYrl0bqFvxJRknXn7kFwkhRAILlQ3qLEkoem0/W1j7jsKY0R1vVQn1674LdyNsjyJOtG688UZuvPFGzj//fKZNm0ZmZiZVVVV8++23rFu3jldeeSUacYp2InnwZCq/fgN/fRWNW5eS3IIZPW1hsxi5/5oJ3PnC9xRVePjwm+1Ru5bRoHDXpaOwWZp/Q3EVbABUjF16YrBnRi2Ow0kZdRK1iz/HsW0lZ5x6A1t39eCH1cX8sKEeNkRzaGpdq151omUDg22wuT6Jd744XFv6Oo4zO7kgCUoLi3lnk3Yt7M3Ju7jw5MiTjvYoPKR44MRW3QSx5o2gdvFnHXpHy++oC7csNnXpodm6qRPPCCZaq74m7dgL0BnNmq0thBCxpmVr9wMpig77+NOonPcadcu/CB9NaY8iTrSmTJnC3//+d1566SVefPFFVFVFURSGDBnC3//+92Z3ukTnoRiM2EefTM1P/6FuxVdRT7QAcrPtPHDtOGYv2EBqahoGg/azqwIBP73TPPTMSj7kc8Lzs+JQNhhi6tITc8/BuIs207BuIbdedDbdMy38kl9Gerr23xufz0d1dU2r1x5ZugrqwZbTh1MyDy4d3Hf9Hk4HlC6ltz3AKcMPX2bYEtV1TpZuLOedr35hcJ+ujOjXpc1rJjI14MexNdhiPGlI6/5dWnoNBp0eX0053ppyjGkdb6RH6HyWIS0Lncl6hGe3nG3AeAxpWfhqymlY/z32MdM0W1sIIWJNy2HFzUkZeQJV372Lt7I4eCwje0BUrhNtrfrUdeKJJ3LiiSfidrupqakhJSUFm82mdWyinbKPnU7Nz5/iyl+Hp6KoTXNoWqp/z1TOnJDOkCFDovJ30eFwsGnT4dvWx3p+1qHYR5/EnqLN1K/5hrSjz+X8E/qyaZM7Kt+b0PeltWsX/estPPVw3NTJnDpw9GHX11XbKXnzE7qa3dxy4cHPjVRjYyOPvv4T6/IdPPnWcp6/cwqZqdp9sE40rqLN+Btr0VmSsfYefuQXNENnsmLO6R/soFewHmPaVI2jjL/w+SwNOg7uS9HpsY8/nar5b1K79H+kjD455g1zhBBCK+EdrSglWjqzjZSRJ1K3/Avqls/FfuZtUblOtLVqYDFAbW0t1dXV+P1+ampqKCkpCf8nOjdDaldsA8YB0W/1nih8tXvwVpWAosOaOzSusSQNmYxisuKrLsO1a2NcYzkc1e/FU1EMgOkIrd0BDMnpAPjqqzWZGK8oCmdNTCO3WzI1DW6eeGs5Xl/HmETfnMZNwbJB28DxKPrW72xa85rmaeWv0ySuROPdEzqfpW2iBWAfNRXFZMFbURS+MSOEEO1RNFq7HyjUFMPxywp8NeVRu040RZxoFRQUcMkll3DUUUdx4oknctJJJx30nxChetr6td8R8LTvjjEtEfrQae4+oE0DTrWgM1lJHnoMAPWrv4lrLIfjqSiGgA+dJQmD/eBW+QfSNyVaBHwEHNqcNzMZdNx16SiSLAY25Vfx+n875tkjVQ3QuKXpfNagyLoNHsi6z+DiRB8d0BqeCu1aux9IZ0naO4xz6f80X18IIWLF3xhq7Z4WtWuYMntg7TsKUHGsjX4362iI+LbmX//6V/Lz87nlllvIzs5Gp2v1ppjowKx9RmLMyMFbVUrDuu+xjzsl3iFF1d6ywdh3G2xOyuiTqV89n8bNi7Edd1m8w2mWp2lQsSmrd4tKqBS9AX1SKv7GWnwN1ZqVK2Rn2rjzsnH89fUl/O/HnQzMTefEcdp/yI4nd8k2/PVVKCZr0w+t1jP3HAR6A/76KrxVpZgyu2sUZfypqrpPa3dtOg4eKHX86dQt+wLn9pV4Kks61PdPCNF57D2jpW1r9wPZx5+Oc8canOsXwvGtK3uPp4gTrWXLlvHoo49y5plnRiMe0UEoig77uFOp/PoNald8ScrY6fEOKWpUVd3bCCPO57NCzN37Y+yai3fPLpybfwJz9M/JRcpTng+AKSuvxa/RJ2fgb6zFX18FRxhwHImJw7K5+OSBfDh/Ky9/vIa8HDt9ukevHCLWGjf/DDTNWjOY2rSWzmjG0mMQrl0bcBWs71CJgr+xhoCzARQdxkztOg7uy5iRg23AeBy/LKNu2Ry6nHp9VK4jWsbvrKdh2Rcoppx4hyJEuxJu7x6lM1ohtn5jMKR1w1ezG1PJehieGJ+zWiri7ajk5ORDDiwWYl/JI05AMZjw7tmFq/DwjSTaM++eXfgba1GMZiw9BsY7HCB4/sg+OljG61y/MM7RNC+8o9WC81kh+vA5Le0Ho196ymDGDsrC4/Uz481lNDi9ml8jHlRVDZ/PinRI8aF01HNa3n07Dkax/XrqxOAA4/q13+J3NkTtOuLIKr78Fw0/fohl+6J4hyJEuxLN9u77CjYSCh5HMRcsb3cl6xEnWueccw7vvvtuu/uNitjTW5NJbhpq2pGbYjiaygYtvYZqPrSvLZKHTwG9Ad+eAvS1ZfEOZz+qquIuLwDAHMGOliElAwB/g/aJll6ncNfl48hKt1Ja2ciz760gEGj/73Oesp34astRDCZsfcdosqal6ZyWa9eGDvWzINplgyGW3sMxZfVG9bqpX5O45yg7Om9NOY2bgru9hqqDB6YLIQ5tb9fBtKhfK2XkVBSDCUPDHryl26J+PS1FXDpotVpZsWIF06ZNY8SIEVgslv2+rigKjz32mGYBivbNPu7UvWeFjrkk3uFERaK0dT+Q3pZC0qBJNG5chKloNXBivEMK8zdUE3DUBUu0Img6oG9KtHz11VGJy55k4r5rJvKHl35g2cbdfPzNVi6eNigq14qVUNmgtd8YdCbLEZ7dMpYeA1AMJvyNtXgrCqOemMRKaIZWNBph7EtRFOwTzqBizt+pWzaX1IlntmqAtGib2mVzoKmDqb6+nIDbATKqRogjCnjdqE2NzqLV3n1femsytnGn0bD0fyiGxLmh3RIR72jNnj2blJQUAoEAa9asYcmSJQf9J0SIObsP5h6DIODHuf67eIejOdXvDbdQT7RECyBlZDC5Mu3eoklLdK2EygaNmd0jKtEKtXiPxo5WSP+eadz0q+Cf5btfbWbl5vbZUhaaygabug0mt3JIcXMUvRFLryEAOPM7TqfGaHYcPFDy8OPQ2ez46ipo3LI06tcT+/M7G6hf1dTFTG9EQW13d8qFiJeAI9hxEL0BnTk2NyeSJ/+Kmun3YIygCiYRRLyjtWDBgmjEITow+/hT2VO8Bce6BXB0+5zsfSiu4l9QvS50NjumrMS7q2/tPRzFaEHnacS3Ox/6JkbHnnAjjAgbWuzd0YpeogVw8sTebC6o5qvFBTz97nKeu+MEumW0vzvd3opCvJUloDdg6z9O07UtvYfj3LkGZ8F6Uiecruna8aCqaviMVix26HQGU3C4+4+fULdsjqaJsDiy+lXzUL0uTFm56DJzcW36EU/xFhiqzTlGIToy3z7ns2I1eF1RFFDaX6dzzSPesWOH1kuKdi558GR0NjuBhmqM5VvjHY6mwmWDeSNQEvANQDEYMYXO0+xcHd9g9uFu2tGK5HwW7HNGK8qJFsBvzxvBgF5p1Du8zJi5FLfXH/Vrai08pLjPKM3vOoYaYrgKNiTUbmlr+eurgqVjig5jRmw6KdrHngo6A67CTbhLZDclVlSfl9plcwFInXQOph7B8mBPScf6+SREtATC57OkOd6RRPzJsKamhr/85S+cfvrpnHzyyeEhxVOnTuWoo47ijDPOiEacoh1TDEbso08GwLr1O3xVpXGOSDuJ1ta9OeY+owFwJ1CitbfjYF5Er9MnNyVajbWogegmPkaDnj9ePQF7kontRbW8Omttu2v80LgleD5Lq26D+zLn9EMxWQm4GsJ/nu1ZqBGGMSMnZmcADCnpJA89Gmg6LyRiomHDD/gbqtGnZJA87BhMTd1ivaXbUf0do9uoENEUau0e7Y6DHUHEidaMGTP45JNP6N27N3q9npSUFEaMGIHX66Wuro6HH344GnGKds4+/nR0SWnoHVVUvvcADRvbfyvdgNuBu/gXINETreCAWt/unVFrIhGJgNeNtynZjmSGFjQNRlR0gIq/oUbz2A6UlW7jD1eMR6fA/GW7mLekIOrX1Iq3qgRP+S7Q6bENnKD5+opOjzV3KNAxzmnF8nzWvlInBmdSNmz8CX9D/P99dnSqqlKz5HMAUiecgaI3ok/PIWC0gt+Lu1SqcoQ4kvCw4mRJtI4k4kTrhx9+4Pe//z2vvPIKF198MdnZ2Tz//PN8+eWXDBo0iG3bpPxBHMyQkk7m5X/Fm5GL6nVRPvtZKr76N6qv/d49dBZsADWAIT0bY2pWvMM5JH1SGj57cBinY/vKOEfT1NlNDaCz2dEnp0X0WkXRRXWWVnNGDezKFacFGz+8+p91bN3VPj4MN24Olg1a84ajt6ZE5RrhNu8FHSDRKg8mWsYYd1A05/QLNhYJ+HCslVbv0ebcvgrvnkIUkxX7mGlA8OyHLz2YYHfkmY9CaCVWM7Q6gogTrbq6OsaMCc5i6devH+vXB3/AJiUl8etf/5rvvvtO0wBFx6FPSqNh/GUkTTwbgLrlX1Dy9p/x1rbPrm6hYa2JvJsV4u3aDwDHthVxjmRv2aC5W16rDtFGc5bWoVwwdQBHDc/G5w8wY+YyahvcMbt2azWEhhQPit7h/vDg4l0bo17KGW3eOO1oAdibBhg71i4Avy/m1+9MQrtZ9jEno7MkhR/fm2htjktcQrQn/qaug3JG68giTrTS09Opr68HIC8vj8rKSmpqagDo1q0bu3fv1jRA0cHodKQccyHZF92PzpKMu+QXiv99T0IkAJFqD+ezQrxZwW6Pzp1r4r6LGO442MoWrdGepdUcRVG4/ZKx9OiaREWNk6feWY4/gYcZe2vK8ZRtB0VH0qBJUbuOqVseOksyqseJu3R71K4Tbaqq7i0d7BL7RCtp4EQMqV1RnfVYdizqEM1FEpG7dAeu/HWg6EidsP95cl96TwBcRZvl+y/EEfilGUaLRZxoTZ48mVdffZXi4mJyc3NJTU1l9uzZAHz77bekp6drHqToeGwDxtHjN09hzulPwNVA2YePUfXtu+3mrrivvrqpFbSCtXditEw/HL89G50tFdXjwtk09yte9jbC6N2q18dillZzkqxG7rtmImaTnjW/VPDul4lbYhSanWXpNSSqPwgVRYel9zCgfZ/T8tXtQfW4QGfAmJET8+srOj2pk4I7/dbti6j66FE8FUUxj6Ojq23azUoediyG1K77fc1vzwaDiYCzHm9FcTzCE6LdCJ/RktLBI4o40brtttuorKzk3nvvRVEUfvvb3/LEE08wadIk3nzzTX71q19FI07RARnTsuh+1SPYx50KQM1P/6H0vYfxtYMD4aHdLHNO36idf9GUooSbYsRz91BVVTzlwYYSbd/Rim2iBdA7285tFwVLpz/+5hd+Xtd8B82A20Hdd++gry6MZXhhofNZ0eg2eCBrBzin5Q2dz8rsjqKPeLykJuzjT8M+9WpUvQlvyVaK/n0X1T9+giqlhJrw1paHmzClTjrr4Cfo9JhygiXWck5LiMOTHa2WizjR6tGjB3PnzuVPf/oTANdeey1PPfUUp59+Oo899hi33HKL5kGKjksxGOly6vVknXcnismCq2A9xf++G+euDfEO7bDaU9lgSKjNu2Pbiri1KffV7gnOKtIZMHXp0ao14nFGa1/HjenBOccHP5A99/5Kivc0HPSchvU/4Fj1FSnL3sddGNsdRF9dJe6iLQBRLRsMCc/TKtzUbltjx6vj4L4URcE26mRqj70eU95I8PuoXvg+xa//QWZsaaBu6RxQA1jzRmDO7tvsc4zdg/O0XEVyTkuIQ1HVwN4zWrKjdUStmrBqsVgYPHhw+P+fddZZPPjgg5x33nkRrxUIBHjxxRc57rjjGD16NNdffz2FhYe+C5yfn88NN9zA+PHjOf7443nxxRfx+fa/4/fWW28xbdo0Ro8ezfnnn8/ChQsjjkvEVvLQY+hx7RMYu/bC31hD6Tt/oean2QlZK6+q6j6DittPomXqPRx0BnzVZXirSuISQ7hssEtPFH3rZhWFZmnFs1X9NWcOZVjfTJxuH4+9uRSne//3oND3Vwn4qPns2ZgesG/csgQAc49BGOyZUb+esUsv9EmpqD4PrqZxB+2NZ0/8zmcdSLWmkn7u3XQ95zZ01hQ85QUUv3kfld+8RcCb+E1YEpHf1Ujd6vkApB51ziGfF5qnJTtaQhxawNkATZ/N9En2OEeT+FqVaH311VfMmDGD++6776D/7r///ojW+vvf/857773HX//6Vz744AMCgQDXXXcdHo/noOfW1tZy+eWX43Q6mTlzJs8++yxffPEFDzzwQPg5//nPf3juuee46667+O9//8uUKVO4+eab2bxZ7lAlOlOXnvS45nGSR0wBNUDVt++w+6PH8TsP3jGIJ29lMf76KhSDCXOvwUd+QYLQmaxYewfnHjl+iU/5YLgRRoSDivdlSInPGa39YtDruPfK8WTYzewqq+elj1bvt0vorQ42BQoYLKheN6UfPoorRrsSofNZsSgbhOBOjCU3eE6rvZYPhoYVm2Lc2v1QFEUhZfjx9PrtCyQNOxbUALWLP6PoX3eGu52Klqtf9TWqx4Wxay7WvqMP+TxjTn9QdPhqyvHVVcYuQCHakVDZoM6S3Oobpp1JxInW008/zW233casWbNYsmRJs/+1lMfj4fXXX+fWW2/lhBNOYPDgwTz33HOUlZUxb968g54/e/ZsHA4HL7zwAsOGDWP8+PE88sgjzJo1i6Ki4MHh+fPnc+yxx3LqqafSq1cvbrvtNmw2Gz///HOkv1URBzqTha5n/Z4up9+Iojfi2LaC4tfuSajSmdBulqXXYHQGU5yjiYyt/zggfue03G1shAGgTwnu0gScDXG9w59ut3DvVRPQ6xR+WF3M5z/sHXTqrQ6e3XKMOANjz8Gobgdl7/81/PuPFn9jLa5dwbvxsUq0AKx5I4D22RBDVQN4mxpPGONYOtgcfVIq3c69g24X3Yc+JQNfdRml7/6FPXNewe9qjHd47YLq91K7dA4AaUedfdiREjqTNXwTSMoHhWie3xE6nyW7WS0RcaI1e/ZsLrvsMpYvX86CBQsO+u+bb1o+cHHz5s00NjYyefLk8GN2u52hQ4eybNmyg55fUFBA3759ycjICD82dGjwDv3y5csByMzMZNmyZWzevBlVVZk7dy719fWMGDEi0t+qiBNFUbCPmUb3ax7DkNYNX205xW/9H3Urvozb2aJ9hc9ntaOywZBQouUq3EQgDh/UQo0wzN36tHoNndmG0pTg+uPcOGVon0x+c3bwjNLr/93Ahh2VqGoAX01wNpw/uSvp59yFuccgAq4GSt97KFymFg2NW5eCGsCU3Q9jWuyGaIcHFxdvaXflbb6aclSfB0VvxJjeLd7hNCtpwHh6/fYF7GNPAaB+9XyK/nE7jVuWxjmyxNew4Uf8DVXokzNIHnbsEZ9vaapSkPJBIZq3txFGWnwDaScibq/kdruZPn26JhcvKysDICdn/3a6WVlZ4a8d+Hh5eTl+vx+9Xg9AcXGwDWtlZXCb//e//z3btm3jnHPOQa/XEwgE+Mtf/sL48ePbFKuqqjgcjjatoQWn07nf/7aXtVu1vj2bjEsfonbev3BvX0HFl/+iIX899pN+jc5kadvaEQqt62hsCN+1V3IGavJ3Iqbfd2sq+vQc/NWl1GxeimVg25olRBJ7wO3EVx38d+1PyWrR9+5Q6+uS0vDXltNYUYrJ3Lq7alp936eO7caGHXtYtLaMx2cuZcbVg1B9HlB0BKypuP0qqefcSdUnj+Mr30nJOw+ScdGfMKRnt/qah4q9bkOwq5qp39g2/d2M9HujWlLRJacTaKimdvtazE2lhFqsHalI13cVBc+V6TO643QdPkmMd+y2KVdg6Dee2q9fw19Txu5PnsAyYCIpJ151xO5f8Y49Hmurqkr1T58CYB09DafbCxzcsGXf9XVZwUYZjoKNCf/+3hn/TBNh/c4eu7N6DwCqOemgfyOJHruWVFU97A55SMSJ1vTp05k/fz5HHdX2spTQN8tk2r/8ymw2U1tbe9DzTzvtNP7+978zY8YM7rzzThwOB4888ggGgwGvN/jmuWvXLgKBAE8++SQDBgxg3rx5PProo/To0YPjjjuu1bF6vV42bUqcO1z5+fntcu1Wrd9/OmZDGtatC3Bt/pnGwq00jDmfQHLXg54a7diL1i7G7nESMFrZVumAKu3+TsTq+25N7YWlupTdqxbi8Guz9d+S2PXVhdiBgDmFLfmRzQg6cP1knRkjsGvLerx1bWuYosX3fcpgHb/sMlJe4+GtD7/jEsBvsYNOH15fGX4Oycvew1BfTvkHD1M/8UoCtrQ2XXff2BWvk9RdG1CAYl0GAQ3eryL53tjsPTA3VFOy6ntcjUculkiU9xnL9lVYgUZDMnta+D2Le+wTrsSy7Ucs+Ytx/bIUR/5anINPxtN9BBzhB3/cY4/h2oY9O0ipLELVmygy90A9wp9vfn4+iktHGuDdU8CmtavAaDnsa1oqkb4vibS+xB6f9duytqVwB1ag1u2n7BD/phI1dq0dmL80J+JE6/777+fCCy/kyiuvZOTIkVit1v2+rigKN998c4vWsliCb2Aejyf8awjumh24LkBeXh4vvPACDzzwAO+++y42my28g5WSkoLD4eDmm2/mvvvu45xzgp2Fhg4dSnFxMU8//XSbEi2j0Uj//v1b/XqtOJ1O8vPzycvLa/Z7lKhrt3n9oUPxjJpMzZy/QWMlaYtnYj/511iHHBPT2Lv6a3ATbGndfeih79q3Zu1Yfd/dySrV+UuxVueTO2gQiq5VPXGaXftwHGuKqQOs3fvSfciQNq1fs7MHrupCctJsJLVwrbbE3hL/193Bfa8uIVBXBclgzgzu1O+7vn9Af6o+fgyqSshY/TEZF/0pPBesrbE7N/xArRrAkNmTQeOPXCIV6fpH4gjsoa5kPXZnOX0O82eSaO8zNfnf4gIy+w6l9xH+LiVU7MNH4i0/ndqv/42vvICkdf8jva4A+0nXHjSMN+Fij9HaVRs/wwMkjZpKzsgxLV5/z6os/LXl9LXrMee17v2lrbHHe+1ory+xx2d9LdauLf4ZJ9ClRx7JB7xnJnrsWtq2rWW9AyJOtN5++2127tzJzp07mz1HFUmiFSoZLC8vJzd3b7en8vJyBg0a1Oxrpk6dytSpUykvLyctLQ2fz8fjjz9Or1692L59OzU1NQedxxo9ejRff/11S3+LzVIUBZvN1qY1tGS1WqMWTzTXbsv6tgFjSLn+Gco/ex7nzrXUfvkqgfIdZE67FrC2ae2WCpQG5xOl9Buj+XVi9X239h9NjdlGwFmPvrYES1NLYy3WPpzG6mDLc2tOv4h/nweu70zrigvQuRva/D3T6vve12bjzkvHsuqD4PtipZp68Po2G9YrHqLk7T/jqy6j5j+Pk3PFX8OdFNsSe93OlQCkDD1Gs79HkXxvjAPGUjcPvLt3YtEr6MyH/0GYKO8zVVXBxiVJ3Vv+9zJRYidvKPZfP0ntkv9S/f2HeArWUfn2/WSccCn28aeh6PStX7uVEuVnk7tsB55dG0DRkXn0ORhb8Lrwe2TvoTSsLUct34FtqDZNZRLl+5Jo60vs8Vm/LWvXuYOdoC2pXQ65RqLGrqWWlA1CKxKtd955h7POOos//vGPZGa2bUbL4MGDSU5OZsmSJeFEq66ujo0bN3LFFVcc9Pzly5fzwgsv8MYbb5CVFTzoPXfuXKxWK2PHjg2XIm7ZsoV+/fqFX7dlyxby8vLaFKtIDPqkVLIv+RPVP3xMzY+fUL9yHu6SbdhPb1ly3yY+D96S4HmO9jSo+ECK3oCt7ygaN/2M45cVmiRaLRGeodWG1u4hoVla/jjO0mrOpOE50B2ohu+3+/ArdWzdsxPjAeUFxj7XMNDxKlSVsuVf9/PLsBvwGZNbfB2vx0P5nr1r63wuRm5bjQ5YVJeDa0Eb51mpfrIs/oheYkzLwpCWha+mHFfhJmz9x7YthlZwe/0sXFVCEi2LXQ348VQGy1jjOay4LRS9gbSjz8M2aBIVc1/BtWsjlV+/QcPGRXQ946Z2+/tqq9ol/wUgaejRGFMjawxj6TmEhrXfxXT+nRDthb+xaVjxEc6FiqCIEy2Hw8EFF1zQ5iQLgrWNV1xxBU8//TQZGRn06NGDp556iuzsbKZPn47f76eqqoqUlBQsFgt9+/Zly5YtPPHEE1x11VVs2bKFRx55hN/+9rckJyeTnJzMmWeeyWOPPYbZbGbgwIF8++23zJo1i2eeeabN8YrEoOj0ZEy5BEvPQZR/9gKesh1UvvtnjENPQx3c/E6oFgzVhRDwY0jt2qZGBonA1n9cMNHatoKMEy6N+vXUgH/vrKKs1rd2DwntAPnq4zdL61B6mB14gN3eZNatrQPqmn1epu5EbrV/RZqznIzFf+Pl+uk4VXOEVwuuPda0k9HJPnb77fzj20qg7d+XLnYDw4b6iOTGobX3cOprFuAsWB/zREtVVZ59bwU/rS1lSC8r4w9dKRbmrS4Dvw/FaMYQwy6N0WDK7E7OFQ9Rv2o+ld+8hbt4K0X/vpv0Y39F2tHnxTu8mPLVVdCw4UcA0iYdekDxoYQ6D7pLfkH1eVEMMitIiJC97d3T4htIOxFxonX00UezZMkSJk1qW7eykFtvvRWfz8ef/vQnXC4XEyZM4LXXXsNoNFJUVMRJJ53EjBkzOP/888nIyODVV1/l8ccf58wzz6Rr167ccsstXHPNNeH1Hn30UV555RUef/xxKioq6NOnD88++yynnHKKJvGKxGHrN4ae1z3N7v88i7t4C8mrZrFny9c09h2Ntc8orH1GtbokqznGynwg2Na9pVvGicrWbyyg4Nm9E19dJQZ722+cHI63ejeq141iMGHMyDnyC44gdK4pnkOLD8VXExxWPGrsYPR7IC0tDYO++bfalZ5sji55i55Uc2/2Dyzufjk+3ZGTLZ/fR01NTXjtcWVLoRHqModz8sC2J7IrNpdRUefhldkb+L9rj2rx33dL7+HUr1kQl8HFs7/bzk9rg2WAW4qc1NS7j1he4m1qtW/M7ImitP6sYqJQFB32sdOx9R9HxRf/wLFtBdXff0jDpp9JOfnX8Q4vZmqXzgE1gCVvBOacvhG/3pjZA53NTsBRh7tsB5ae0buBJ0R742+sAWSOVktFnGidffbZ/PnPf6agoIAxY8aQnHxwucu5557b4vX0ej333HMP99xzz0Ff69mzJ1u2bNnvsbFjx/LRRx8dcj2LxcIdd9zBHXfc0eIYRPtlsHeh+5UPsfvrt2hc/TUBRx0N67+nYf33AJiycrH2GY217ygsvYagM0a6Y7DPtUKJVjsuGwzRJ6Vi7t4fd8kvOLatwD5Wm5ENh+IpzwfA1DW32XMjkTI0lQ766qtb3GI1FvzOegKuYP36GaeMo/+2nQwZMuSwH/g95QMpeecB0p0lnOv9LzmX/gmd6QiNRRwONm3axJAhQ7AY9RQ8txMVOOH88zilFR8sD7R6cwkPvraMJRvKmf3dds4/sWWNgKxN87TcZTvxuxrRW5LaHEtLrN22h5lzNgCQZDHQ6PKxcHUpl55y+Bste3dZO1Z5ncGeSbeL7qNx4yIq5r2Gd88uqj54iKSsgdSW9MBpTUIxmFGMZnRGE4qx6dcGM4rJjM5obvq6Kfjr0H8GY8InpAFXI3Wrgmey0yad3ao1FEXB0nMwjq1LcRVukkRLiCYBrxvV4wJAb5PSwZaIONG67bbbAJgzZw5z5sw56OuKokSUaAnRVoreiP34SynOHEG/VD2B4s04d67BXboDT/kuPOW7qF3yOYreiCV3KNY+I7H2HY0pq3eLP6AHHHUY6oM7Fda8jjH82jZgfFOitTL6iZaG57MA9MnBD9Cq14XqcaKY438wFoI7dxAsqdC1sC20KSuXnMseoPTdv+Au2kzZR4+TffH9Lb4p4Ny+CtXrxpCahSm79YOg9zUwN41Tx6Yxd3kNM+dsoH+vVEb2P7iT3YEM9kyMGd3xVpXg2rWRpIETNInncCpqnDz59nICKkwd34sBPVP4x6cb+XZFMZdMH3LYf+Oh4dGmLh0r0YLgz+LkYcdi7TMyeGZr/feYdm/BuXsLbZlCoxj2ScyMJhSjBZ3RjGqyos8aCbStS19b1a36GtXjxNilJ9Z+LagfPQRLryFNidZmmKxhgEK0Y6GyQUVvTJifu4ku4kTrm2++iUYcQrSdTo+p5xBsA8fBiZfjd9TjzF+Lc8caHDtW46+vxLlzDc6da2DB2+iT0pqSrqYyw+RD3/12F24EwNA1t8McALX1H0v1wvdx5q8l4POgMxx5HkRrhRMtDc5nAehMFnRmGwG3A199FaYEecMPDWSOtDzSnN2X7Ev+ROl7D+MqWM/uT54i+8J7W3Q2pGHzzwAkDW55iV9LTBiQRIPPyverS3ny7eU8f8cJdEk7cktdS95wvFUlOPPXRT3R8vr8PD5zGbUNHvp2T+WmC0bR2NDIG//bRGmlg/U7KhnRr8shX++paEq0OnDDCL3NTtY5t2EachxFa38mKz0VvaKiet2oXnfwDrXXg+rb59de1z6/dqP69w75VX2e4EBuZ/1BLUdSCjbg7dMP+gyN7W8yFJvfS+2y4A3gtKPOadO/h9A5LVfRZlQ1kPA7eULEgr8hmGjpklITppIk0UWcaD3wwANcd911TJ4st3hEYtPbUkgeegzJQ49BVVW8lcXBRGvHGpwF6/E31hxQZti7KekajaXX4P12FDy7gmVJplxtZmclAlO3PuiTM/A3VOEq2ICtDXd/j8RdXtB0zTzN1tSnZBBwO/DXV0GXnpqt2xbepkTLkN4t4tdaegwk55L/o/T9v+LcsYrds5+h2/l3oxzifBeA6vPi+GUFEEy0tKQoCtedNYTC8kZ2ltTx+FvLmHHTMRgNhy/9tPYeTv3KeTE5p/Wvz9azZVc1SVYj910zAbNRj99sYHhvGyu3NzJvScEhEy3V78VbGRw5YOqa2+xzOhJTz8G461WSj1DK2hw14A8mWF5PUwLmDv+v6nUT8LmpXvxfvMVbqJr9JNarHsEUh3+TDRsX4a+vQp+cTvKw1s/NhODND8VgIuCsx1tR3KGTcSFaKtwIQ8oGWyziWzQrV66ULFa0O4qiYOrSk9QJZ5B98f3k3TmTnCseIu3o8zFl9wMUPOUF1C7+nLL3H6bg2Wsofe9hahZ/hnt3Pp5dwQ+N5tzh8f2NaEhRlHBnOMcvy6N2Hb+zHn9dBQBmjXa0AAxNDTF8CdQQI5RoGdNa15XS0msI2Rf+EUVvxLF1GeWfvYAaOHSrcveu9ageJ/rkDMw9BrTqmodjNum57+qJJFmNbCmo5t+fHTl5Cp3T8pQX4G+s1TymkG+W7eKLn/JRFLj78nFkZ+49Dza2X/DXP60pocHhafb13qpSCPhRTFb09kPveolgp1edyYo+KRVjWhamrr2wdO+PtfcwbP3Hkjx4Munn3IXPnoPqrKf03YfwNjWFiRVVVald/DkAqRNOb3OnQEVvwNw0+sJVuKnN8QnREYTe06URRstFnGgdd9xxfP7553i93iM/WYgEpRiMWHsPJ+PEy+n5myfpfcfrZJ13JymjpqJPyUT1eXDuXEPVN29R/O+78NdVoCo6jDGaORUrtgHjAXBsW4mqqlG5hqdpN8uQloVOw+YIiThLK1w62Ib2/9Y+I+l2wR9AZ6Bx00/s+d/fUNVAs891/xIcjhwsG4xOaVNOlyTuvnwcAHN/ymfB8l2Hfb4+KRVj0w6Rc9fGqMS0o7iWv3+yBoBLpw1i/JD9dxB7ZBrJ7ZaMxxdg4cqiZtfYez6rp9w81IDObKVh/MUYMnrgb6ii9N2HYjp+wblzDZ7yAhSjhZQx2pw53bd8UAixb6KVFt9A2pGISwfNZjOff/45X3zxBf369TuoBEFRFGbOnKlZgELEgt5mP7jMcMdqHDvW4Nq1AdXrxtulX4sbHLQX1rwRKHojvtpyvBWFUSmh2ns+K0/TdRNxllaoGYYhPZvmU6OWsfUfS7fz72T3rKdpWLcQRW+ky+k37p8QBPy4tq8EtC8bPND4Id24dPog3p+3hb99vIa8nFT69jh06Yg1bzjePbtw5a8jeYi2ZeYNDg+PvbkUjy/A+CHduHjawR3hFEVh6vgevDlnC18tKeD0Y/oclEyFE61OUDYYK6rJRvqv7qX640fx1eym9L2H6H7lX9Hbon/3O7SblTL6JPTWlg//PhxLr2BjD9nREiJo7wwtKR1sqYhvgZaVlTFmzBiGDx+O1WpFVdX9/gsE2vLxQoj4C5cZTjyTnEv+j7w7Z5J52cM0jmpdq+BEpjNZsDSVejm2rYzKNdwadxwMCe9oJUjpYMDrDsfSlh2tkKRBk8g693ZQdNSvnk/lvNf323U0VBWguhvRJ6WG77xH0yXTBjFucBYeX4AZM5cesiQP9pYPOjU+pxUIqDzz3kp2VznolmHjzsvGotM1vxt13MgcjAYdO0vq2F50cAljqLW7Uc7eaEqfnE7O5Q+iT8nAW1FE6fuPEHA1RvWa7t35wSZHio7UiWdqtq6lxyBQdPhqyvHVVWq2rhDtVaBRzmhFKuIdrbfffjsacQiRsBSDEWO3PlDVMe9q2vqPxbljFY5flpM2+VzN1w/taJk139HaO0srEfiadrN0Zhs6azI429JEOyh56DGoPi97/vsydcvnohhNZJx4BQCmsmA5k23gRE1mkx2JTqdw1+XjuOO5hZRVOnjmvZX8+deTmk10LLnDAAVvZTG++qrwn1Vbffj1FpZv2o3JoOO+qyeQYjt0p8xkm5GjR3Rn4aoivlpSQP9eaft93dsJOg7GizGtGzmXPUjJ23/GU7adso9mkH3pn9s0x/BwapcEd7OShkzGmJal2bo6sxVTtzw8ZTtwFW0meegxmq0tRHskO1qRa3VR//bt23n//ff55z//yccff8yOHTu0jEsIESO2AcHzN66iLfid9Zqurfp9e1tod9OuEQYEuw5C4uxo7e04mK3pmZ+UkSfQ5bQbAKj9+VOqf/gINRDAWL4VgKTBsesAm2Izcd/VEzAZdCzftJsP529t9nl6a3J4pperYIMm116+aTfvfx0cYH/zhaPo1zPtiK+ZflSwLHDhyiJcbl/48YDPg7cq+OfVEWdoJQJTl57kXPpndGYbrsJN7P7kqf3axGvFV1dJw4YfgdYPKD6c8DktKR8UAn9jDSCJViQiTrRUVeWBBx7gzDPP5KGHHuLZZ5/lz3/+M2eccQb3339/NGIUQkSRMa0bxi49QQ3g3LFa07W9lSXg96GYrBg0vNMMhOee+eqrD9ksIpa8NW1vhHEo9rHTyZz+awBqfviI2rl/Q+dxoJiTsPaO7ciBfj3TuOmCUQC8P28zyzc1311Oy/LBsspGnn53BaoKpx2dx9TxLTtXNbxvF3Iyk3C6ffy4piT8uLeyBNQAOktSOGEX2jNn9yX74v9DMZpx7lhF+afPH7aLZmvULvsfBPxYeg/D3L2/pmvDvue0pCGGEP7GOkBKByMRcaL173//m1mzZnHrrbfyzTffsHbtWubPn88tt9zC559/zptvvhmFMIUQ0bRv90EtucvzgeCMMq274ulDA6YDPgIObXfiWiNUOhiNRAsgdcIZZEy9EgDXL0sBsPQbe9g5W9Fy0oRcTpuch6rC0++uoKzy4DM41rymRCt/XZuu5fL4eOzNpTQ6vQzKTef6c1o+YkGnU5g2KZiUzVtSEH7c29QIw9ill3QcjDJLr8F0u+Be0Bto3LyYPXNe0ezGSMDVSN3KrwFIm3SOJmseyNIzmGh5dudH/ayZEIlMVQP4HU2JlnQdbLGIP/l88sknXHfddfzud7+jR48emEwmevbsyc0338x1113HRx99FI04hRBRFJ6ntX2lpnecw+ezNG6EAcE5N6HyBV9D/M9ptWVYcUulTT6X9OMuDv9/84AJUbvWkVx/7nAG5abT6PQy481luL37/72x9Bra1EhgN97a8lZdQ1VVXpm1lp0ldaQmm/jj1ROOODD5QCdNyEWnU9iUX0Xh7mBCHmqEIeezYsPWdxTdzr0TFB0Na7+l8us3NBknUbf6G1SPE2OXnlj7R2fguiElHUN6NqDiKm6+VFaIziDgqIemmyR6W0qco2k/Ik60SktLOeqo5lsJT5o0iaKi5meWCCESl6XnYHSWJALOBtzFv2i2rmefHa1o2DtLK/7ntLwazNBqibTjLiTlhCtw5Y7HnDcyqtc6HKNBzx+vnkBqsokdJcG5Vvt+eNaZreFSLld+68oHv/w5nwXLC9Ep8Icrx9MlzRrxGhl2CxOa5myFdrXC5waltXvMJA2eRNezbgagbtlcqhd+0Kb1VL+P2qX/AyB10tlRmyMHck5LCNjbCENnTY5LJUV7FfE7U48ePdiyZUuzX9u8eTMZGVLvLkR7o+j0WPuOBsCxbblm63p2Bz/Yat3aPUSfnBiztNSAH1/tHiD6iZaiKCSNOQXn0Okx6TZ4OF3SrNxzxXh0CixYXsiXP+fv9/W957Qib4ixuaCKf34aLDu8+oyhjOzftdVxTj8qmOgvWF6I1+ffZ4aW7GjFUsqIE8g85XoAahZ9Qs3Pn7Z6rYaNi/DXV6JPSiNl+PEaRdi8UPmgnNMSnZkMK26diBOtM888k5deeokvvvgifPdSVVXmzp3Lyy+/zOmnn655kEKI6Nt7TmuFJuv5GmqaOhQpUds5MCRI50Ff7R4I+FH0xk7XXGHUgK5cfcZQAP756Tq2FOz9s7Dsc04rklKxmno3j89chs+vcvTIHM47oW1NDsYNyiLDbqGu0cPStbv2nqeTjoMxlzr+VDJOvByAqgVvU7dyXsRrqKoaHlBsn3A6isGoaYwHsuQGEy13yS+oPu07JwrRHvhlhlarRJxoXX/99YwcOZI77riDESNGcNxxxzFixAjuuusuhg8fzm233RaNOIUQUWbrOwYUHZ7yXeHdmbYIlQ0aM3PQmSxtXq85+gSZpeVt+uBuSO8W1RKmRHXeCf2ZPCIHn1/l8ZnLqKl3A8GSVHQG/PWV+JpKK4/E7w/w1DvLqax10TMrmdsuHtPmhhV6vY6TJwaT/WU/rwZUdNYUaVEcJ2lHn0/a0ecDUPHFP2lY/0NEr3fmr8VTno9itGAfOz0aIe7HmNEdnc2O6vPgLpNRNqJz2jtDyx7nSNqXFn0iWLp0Kc6m4Zsmk4k33niDf/zjH1xzzTWccMIJXHPNNbz66qvMnDkTszk6AwmFENGlt6Vg7jEQ0GZXK9QIw6TxoOJ9hVq8x31Hq7oUCLbK74wUReH2S8bQo2syFbUunnpnOX5/AJ3RjKVn8O9US7sPvv3FJtZuq8Bq1nP/NROxWbTZrZjWlGjVF+8EguezpONg/KSfcBn2cacCKuWfv0jjlqUtfm1oNytl9FT01ugfylcUJXjTADmnJTov2dFqnRYlWjfddBMbN24E4KqrrmL79u1MmTKFu+++m7/+9a/cfffdTJkyJaqBCiGiz9Y/OLy48RcNE60onc+CfXe04pto7d3Riu75rERmsxi5/5oJWEx61m6r4O0vgh9ILRHM01q0toRZ324D4NaLx9Crm3YforMzkxg1oAvddDWAnM+KN0VRyDzlNySPmAJqgN2zn8G5c+0RX+fenR+c96foSJ14ZvQDbSLztERnJ2e0WqdFiVYgEODnn3+muLiYpUuXkp+fT0lJySH/E0K0T0kDgomWq2A9Aa+7TWuFZmiZo7mjlZIYXQdj1XEw0eVm27ntkmCb7VnfbmPR2pLwPC1XwYbDntMq3F3PCx8E57idd0J/jh3VQ/P4TpmUR46+BgBDZk/N1xeRURQdXc+8GdugSeD3UfbxE7iKmm+2FVK75L8AJA0+KqY7yOFEq2hTQgxIFyLW9iZasqMViRb1Z5w+fTovv/wyf/vb3wC45ZZbDvv8TZtka12I9sjYNRe9vQv+ugqc+etIamqQEamAz4O3ohiI8o5WqL17Yy1qwB+3Lny+Gkm0Qo4d1YMtU6r5dOF2XvhgFU/fMhnFYMLfWIO3ogiSMg96jcPlZcbMpTjdfkb068LVpw+JSmxHjchm7Zzgh4V8Vwqjo3IVEQlFp6fbuXdQ9vEMnDvWUPbho+Rc/hDm7D4HPddfX0XDhuB5rtSjojOg+FDM2X1QDCYCzga8FcWyIyo6nfAZLSkdjEiLEq1HH32UU089lerqau677z5uvPFGcnNl/ogQHY2iKCT1H0fdyq9wbFvR6kTLu6cI1AA6a3JUu/Dpk+yg6EAN4G+owWA/+EN8tKmqKqWDB7jmjKFsK6ph/fZKHn9nFfd3H4hn13qcBesxDt2/zFxVVV78cDWFuxvITLVwz5Xj0Ouj01BEH/CQrjQAMH+rn9HHReUyIkKKwUi3X/2B0vf/irtoM6XvP0z3qx7BlLn/rqZj9TwI+LHkDsPSvW2dKCOOUW/A3GMgroL1uAo3SaIlOp3wjlayJFqRaFGipdfrOeGEEwB47rnnOPPMM+nXr1804xJCxIktlGj9sgL1VLVVDQP2DirOi2rDAUXRoU9OD3a1q6+KS6Llb6hB9bpB0WFMa/2sp45Er9fxhyvHc/uzCync3cAKWwYjCDbEODDR+nThdhatLcGgV/jjVRNIT4lOh0oAz54iAOoCFn7cXMtv6lyk26N3PdFyOpOFnIvvp+SdB/Hs3knpuw/R/epHMKZmBZ/gc+NYuwCA1KPOjkuMll6Dg4lW0eaYdDsUIpFIM4zWifi2YWNjIxs2RD58UgjRPljyhgdLveor8ZQXtGoNdwwaYYTEe5ZWqGzQYO+Coo/uPJ/2JD3Fwn1XT8CgV5hfmASAa9eG/c63rNtWwZtzgo2WrjtnBIPzojuDzFsRHFRca+yKP6CyYHlhVK8nIqOzJJFz6Z8xZvbAX19J6bsP4WsIjm4wF65G9TgxZvbA1n9sXOKz9ArOi3PtkuMRonMJeFyoXhcgZ7QiFXGilZqaisUidwCF6Kh0RjPWvBFA69u8e8KNMHprFdYhxXuW1t5GGJ2ztfvhDM7L4Lqzh7PLl4lLNRBwNuDbE0xuqupcPPn2cgIBlRPH9eT0o/OiHo9nzy4AUroHrzVvSUFEg5RF9OmTUsm57EEMaVn4qssofe9h/I21WAqWAZA66ey4zaqz9BgIig5fbTm+usq4xCBEPPgddQAoBhOKyRrnaNqXFpUO7uu3v/0tjzzyCDt37mTw4MHYbLaDnjNhwgRNghNCxIet/zgc21bg2LaC9GN+FdFrVVXFszu4ExaTHa3QLK36+HzwCSVacj6reacf04fNu6rZsaUbQ03F1Gxbi8/ej2c/WEtNg5s+3e3cdMGomMy08jQleb0HD8a62UdJRSMbdlQyvF+XqF9btJzBnknOZQ9S8taf8O7ZReU7/4fOVYfOlkryiOPjFpfObMXULQ9P2Q5cRZtJHnpM3GIRIpb8jTUA6G12mT8YoYgTrQcffBAIntUC9vuGq2rwPId0HRSifQuV5riLtuJ31KG3tXwSvL++koCrAXR6TF2if2BcnxI8lxUqMYo1ae1+eIqicPMFo/jg2d7gL2bb8qX8kJrJL4WNJFmN3Hf1RCymiH8UtUoo0UrKyeP4MS6+WlzAvCUFkmglIGN6NjmXPUDJ2w8QaOp2Zhs9DZ3BFNe4LL0GBxOtwk2SaIlOQ1q7t17EP93eeuutaMQhhEgghtSumLJ64ykvwLF9FSkjWj6QPHQ+y5jZA8UQ/TNLhpSmHa14ndFq6jgoidahWUwGpp1zCu7//ES2t5hV2+oBHXddNpacLkkxicHvagzvehq79mL6JA9fLS5g0ZoSbjhvJMnW+J6vc3l8fL+6hGRFZjSFmLrmknPpnyl550H8KthGTo13SFh6DaVu2Vw5pyU6lXBrdxlWHLGIE62JEydGIw4hRIKx9R8XTLS2rYgo0fI0JVrmGJQNwt5ZWvE+oyWlg4fXfdAQthutWL1OeuqrmHDcRCYMjd33LNQIQ5+Sgd6SxIBeNvJy7OSX1rFwRSFnHNs3ZrEcKBBQefqdFSzZUMbIPBtjR8UtlIRjzulHl2ue4petW+luTYl3OFh6DgbAU15AwNWIzhKbGwVCxFNoR0snHQcj1qoTpVVVVTz11FOcd955HHvssWzevJmXX36Z+fPnax2fECJObAPGAeDcsRrV72vx68Kt3WOUaIV3tOpjv6PldzUScNYDYEyTZhiHo+j0JPcZDsAZeTVccGJsE5tQ2WBo/pGiKEybFJwH+VWcm2LM+vYXlmwIJuwbdjlocHjjFksi0ielolqS4x0GEHy/Cd5UUXEVb413OELExN7SwZYfIxBBESdahYWFnH322Xz00Ud069aNyspK/H4/O3fu5NZbb+W7776LQphCiFgzdx+AzppCwNWIq2hLi18XboSRlRelyPYXOqMVcDUQ8Lpjcs0QX9Nulj4pFZ1ZOjEdibV3MNHqpy9Dp4vtgepQx0FT19zwYyeO64XRoGNnSR3bi2pjGk/Iqi3lvPNFsAzNZjHgD8APa0vjEotoGUuv4K6Wq1DKB0XnIKWDrRdxovXEE0+QmZnJN998w8svvxy+C/jMM88wdepUXn31Vc2DFELEnqLTY+s3Bmh5m/eAx4W3KvghMValgzqzDaXpgLw/xg0xwmWDaVI22BKhsQGG6sKIdkm14G3a0TLu06AlxWZi8ogcINjqPdbKqxw89c4KAipMm5jLxSf1A2DB8mJpO5/ALL2GAJJoic5DmmG0XsSJ1s8//8xNN92E3X5wi8eLL76YX375RbPghBDxZesfLB9saaIV3DVQ0SelxewNWVGU8Cyt2CdaTY0wMiTRaglj117oktJQ/F4ca76O6bUPLB0MOeWo4Ky3hauKcLljl/x5vH5mvLWMeoeH/j1TufH8kRw7KgeDHnbtbuCXwpqYxSIiE0q03CXbUH1S5ik6vnCiJWe0ItaqM1oGQ/M9NDwej/TXF6IDsfYdDYoOb0VRePfmcEKNMGJ1PiskNEvLF+NzWqHSQaPsaLWIouhInhycy9bw839i9ufld9SH58AcOHJgeN8u5GQm4XD5WLS2JCbxAPzz03VsK6whxRZscW8y6km2GhnaKzibMh47bKJljBnd0dnsqD4P7rId8Q5HiKgLOGRHq7UiTrTGjx/PP/7xDxwOR/gxRVEIBAK8//77jB07VtMAhRDxo7cmh88jOLatPOLz45VohXa0Yp1oeWtCHQelEUZLWYcfjy+1O6rHRdU3sRkX4qkIns8ypHY96CydTrdPU4zFsUlu5i0p4KvFBSgK3H3FeLIybOGvje0X7GL3/aoinDHcYRMtpyhKuPuglA+Kjk4N+PE7gk2fJNGKXMSJ1l133cX27duZPn06f/jDH1AUhddee43zzz+fFStWcMcdd0QjTiFEnERSPuhu6jhojlEjjBBDuHQwxolWlQwrjpSi6HAMPQVQaNjwA86C9VG/ZnPns/Y1dXwvdDqFTflVFO6uj2osvxRW8+p/1gJw+amDGTsoa7+v984ykZ1hxen28+Pq4qjGIlpPzmmJziLgbAA1ON9Pb5Oug5GKONEaOHAgs2bNYtKkSSxZsgS9Xs9PP/1Ebm4uH3zwAUOGDIlGnEKIOAklWs6C9QQ8zkM+T1UDeMqbOg526x2T2EJCs7T8MZylFfB5wi3lJdGKjD81B9uo4PDZii//FfXGGIc6nxWSmWplwpDgrmQ0S/ZqG9zMmLkMry/ApGHZXDh14EHPURSFqeN6RD0W0TbhRKtoM6oqQ6ZFxxWeoWVNQdHp4xxN+9OqM1p5eXk888wz/Pjjj6xfv56ff/6ZF198kUGDBmkdnxAizoxdemJIywK/D+fOdYd8nq+mHNXjQtEbMWb2iGGEe2dpxbJ00FdTDqgoJis6ucsXseSjL0Rns+OtKKJ22ZyoXmtvopV7yOdMnxS8ObBgeSFen/YfnP0BlaffXcGeaic5XZK4/dKxh2xxP2VMd3Q6hc0F1RSU1Wkei2g7c3YfFIOJgLMBb4XsPIqOyy/ns9qkVYmW2+3mww8/5K677uK6667j3nvv5dNPP8Xnk3pyIToaRVFaVD4YOp9l7Nor5ne99HEoHQw1BzGmZ0sToFbQWZLInHolANXff4SvrjJq1/JUHH5HC2Dc4Cwy7BbqGj0s3XDkxi+Reu+rzazeugezSc/910wk2Wo85HPTUsxMHBrcYft6yS7NYxFtp+gNmHsEdySlfFB0ZKFGQpJotU7EiVZJSQlnnHEGDz74IGvXrqWhoYEVK1bwxz/+kQsuuIDa2vgMfRRCRM++idah5vu4Q40wYnw+C8CQHGqGUR2z+UPhjoPSCKPVkkeegLnHIFSvi8pvZkblGv7GWgKOOkDB2KXnIZ+n1+s4eWJwx0vrkr0l60v5aP5WAH5/4Wjyco68A3rKUXlAaIfNr2k8QhvhwcVFm+MciRDRI63d2ybiROvRRx8lEAgwe/Zsvv76az744APmz5/PJ598Qm1tLU8++WQ04hRCxJGl9zAUoxl/QzWesp3NPscTaoSRnRe7wJrom9q7q14X6mHOkWkpPKxYzme1mqLo6HLqdaDoaNy4COfOtZpfIzjbLdgZUmc0H/a505oSrVVbyymvchz2uS1VsqeBZ98Pduw867i+TBl76GRvX2MGZZGZaqHe4WHxeu132ETbWXoNBcC1S3a0RMe1d1hxWnwDaaciTrSWLFnC3XfffVDTi+HDh3P77bfzzTffaBacECIx6AwmrH1GAocuH/TsbmqEEYcdLZ3Jgs4cbJEdq3Na+5YOitYzZ/fFPu4UACq++jeqX9sBsOHzWYfoOLiv7MwkRg3ogqrC10vbXrLncvt47M2lOFw+huRlcO2Zw1r8Wr1OidoOm9CGpcdAUHT4asujWvoqRDztTbRkR6s1Ik60zGYzen3z5y+Sk5NjVrYjhIitw53TCrgd+GrLATBlxbbjYEj4nFaMEi1f9W5AEi0tpE+5NNgYo7KY2qXaNsY4UsfBA4WaYsxfWoA/0PqfZ6qq8vLHaygoqyc9xcy9V43HaIjsR+60ib1RFFi9dQ9llY2tjkVEh85sDc8MlPJB0VGFm2FI06dWiTjRuuqqq3j22WcpLt6/y05tbS2vvvoqV111lWbBCSESh61fcBi5u2Qbvoaa/b7ma/owa7B3QW9NjnVowWuHhhbHoCGGGvDjrQkmljKsuO30liQyTwr+7Kj+4WN8dRWare2tOHLHwX0dNTyHFJuRiloXq7aUt/q6//txJwtXFaHTKdx71QQyU61HftEBumXYGDWgKwDzNdhhE9oLn9OShhiig5LSwbYxRPqC/Px8qqurOfXUUxk3bhzdunWjurqaFStW4HQ6sVgsLFmyBAh2K5s5MzoHnIUQsWWwZ2Lq1gfP7p04t68kpWkOEoB3T2h+Vl6coovtLC1fXSUEfKA3YEjJjPr1OoPkESdQv/obXIWbqJz/Jt3Ov7vNa6qqGj6jZWzhjpbJqOfEcb34/IcdzFtSwPghkSfSG3ZU8trnwUHMvz5rGMP6tv7vyPRJvVm9dQ/zl+3i0umD0Otb1SxYRIml11Dqls2Vc1qiw5L27m0T8Tt2UVERgwYNYvTo0fj9fkpKSnA6nQwdOpRx44KlRaqqoqoqgYAM8ROiI7ENCJUPrtzvcV9F8MNsPM5nhcRylla442Balgxw1IiiKGSe0tQYY9PPOHasbvOa/oZqAq5GUHQYM7u3+HWh8sGlG8qorndFdM2qOhdPvLUMf0DluNE9OPu4vhG9/kBHDc8mxWaistbFyjbssInosPQM7mh5yguCf9eE6GDkjFbbRLyj9fbbb0cjDiFEO2DrP46aHz/BsWP1fk0LvE27BgmxoxWD0sFwx8E0OZ+lJXO3POzjT6Nu2Rwqv3oN6/XPohgOPW/qSELns4zp2egMpha/rneOnUG909lSUM23yws5/8QBLXqdzx/gybeXU13vJjc7hd9fNLrNM9aMBj0nTejFpwu3M29JAROGyt+5RGJISceQno2vugxX8VZs/cbEOyQhNBPwuFC9bkDau7eW1CAIIVrM3L0/Opsd1ePEVdh0+DsQwFdRFPx6t/g0woB9zmjFoHTQWyONMKIl4/iL0Sel4a0qoWbJf9u0VqRlg/sK7WrNW1LQ4iZPb/xvAxt2VGI1G7j/molYzRHfy2xWqO380o27qa6LbIdNRF+8zml5KwqxbvwKv4ZnGoXYV6hsUDGYUEyWOEfTPkmiJYRoMUXR7e0++MtyAHSOKvB7UYyWuM6UCncdjMWOVlUpIMOKo0FnSSKjqTFGzY8f46vd0+q1vBF2HNzXcaN7YDXrKd7TyIYdR27d/f2qIj7/fgcAd1w6hh5dtWsKk5ttZ0heBoGAyvxl0hQj0Vh6BcfdxDLR8taUUz3rCSy7VuBYuyBm1xWdy75lg23dne+sJNESQkRkb5v34DktfX1wd8eUlYuixO8txZAcOqNVjapG93zo3tbuOVG9TmeVPPx4LL2GoPo8VHz9RqvX8UTYcXBfVrOB40YHhwsfaY5VQVkdL360GoALpg5g8oiWnwdrqemTgr+Hr5fskjEqCSaUaLlLtqH6tJ0D1xy/s4GyDx4h0LTb4K+XHS0RHeFES8oGW00SLSFERGx9R4FOj7eqBF91Gfq60PysvLjGpW9KtAj4CDjqo3YdVVXx1jSd0ZIdrahQFIUup14Pig7HliU4tq+KeI1gx8HW72jB3uRm0ZoSGpzNf4BudHqZ8eZS3B4/owd05YrThrTqWkdyzKgeWM0GSisbWb9dhuMmEmNG92BJtc+Du2xHVK+l+rzs/uRJvJXF0LTDEKvZgaLzkUYYbSeJlhAiIjqzDUvuUADcO1djqA8mWvE8nwWg6A3hHwa+huid0wo46lA9LkDBmCaJVrSYsnqTOuF0ACq++nfEOwWB+kpUjxN0eowZrdt5HJibTu/sFDy+AAtXFh18jYDK8x+spHhPI13SrNx9xTj0uuiU11jNBo4f0wM48g6biC1FUcLdB6NZPqiqAcr/9zKuXRtQzDbsJwZLbANRfL8TnZu/sQaQRKstWnRSd9myZREtOmHChFYFI4RoH2z9x+HKX4d7x+q9pYNx7DgYok/OwN9Yi7++EqIUj7c6eD5Lb89sU0c8cWTpx19Mw8ZF+KrLqFn8GenHXtDi1/oqiwEwZuSg6Fv356QoCtOP6s2/Pl3PvCUFnDhm/8R61re/sHh9GQa9jvuunkBqsrlV12mpU47qzVeLC1i0toTfnjeCZFvLOymK6LLkDsGxdWkw0Zp8blSuUf3d+zRu+BF0err96m58ljQgeC5VVVU5QyM0JzO02q5FidaVV155yH/AoVrxfb++aZMM7hOiI7P1H0fV/DfxFG1CpwYABVPX+O5oQVP54O6dUe08GGrtLh0Ho09ntpF58tWUf/o8NYtmkTz8eIxpWS16ra8yuAPV2rLBkBPH9eLN/21kR3EtO0rqwo+v3lrOO18Ef9bdeP4IBuamt+k6LdG/Zxp9utvZWVLHdyuLOPPYts3oEtqx9GxqiFG0GVUNaH5etW7lPGp++g8AXU+/EVufUTTWN/199PsIOOrkw7DQ3N7SwbT4BtKOteid4K233mLmzJnMnDmTGTNmYDAYuPjii3nzzTeZO3cu7733Hr/5zW9ISkrimWeeiXbMQog4M2V2D5ZjNTWd0KdloTNb4xzV3hbv0ew86K2W1u6xlDT0WCy9h6H6PFR+/XqLX+cNJ1qRN8LYV4rNxOQRwdLDBcuDu2QVNU6eemcFATXYej3UCj7aFEUJX+urxS1vOy+iz5zdB8VgIuBswFtRrOnajm0rqfjyXwCkHXcRKaOmAsFy6YApCQBfvZzbE9qTZhht16JEa+LEieH/Zs2axTXXXMNf/vIXJk2aRN++fRk7dix3330311xzDW+80foOUUKI9sPa1H0QwNClbR9mtaKPwSwtX3hHS85nxYKiKHQ55TrQ6XFsXYbjlxUtel24dLCNO1qwd6bWj2vLcLgDPPPBWuoaPfTrmcpvzx8Z05KtE8b2xGjQkV9axy+FNTG7rjg8RW/A3GMgoO05LXfZDnb/5xlQAySPPIH04y7a7+sBix0AX50kWkJ7odJBXZI9zpG0XxHvba9du5bJkyc3+7UxY8awdevWNgclhEh8SfskWsY27hpoJdTi3R/Fu7uhHa14zgzrbExdc0mdeAYAFfNeI+DzHP4Fqoq/KdEydWl7ojWiXxeyM2043T5e/7qcHcV1pNiM3Hf1RMxGfZvXj0SyzcQxI4Pt46UpRmLRep6Wr3YPZR8+hup1YckbQdfTbzwoqQ9YUoLPlURLREFoR8sgpYOtFnGilZ2dzQ8//NDs17788ktycxPjA5cQIrosuUPCk+INCZJohXe0otiFK9QMQ0oHYyv92IvQJ2fgq9lN7c+fHva5OmcNqs8DekOrOw7ut55OYdrE4K5WRZ0PRYG7Lx9Ptwxbm9dujdAO2/erinC6fXGJQRxsb6K1uc1r+V2NlH74KP6Gaoxdc8n+1T3NNnUJJVrRvLkkOic14A+PStFJ6WCrRZxoXXvttbzxxhvcdddd/O9//2PRokV89tln/O53v+OTTz7hpptuikacQogEo+iNpEy5Anf3EZjzRsY7HGCfM1pRmisTcDsJOIIH0CXRii2d2UrmtGsAqPlpdrgpSXP0DcEBrqbMHig6bXacTprQC11T6/YLp/Zj7OCWNeWIhuH9MsnpkoTT7WfRGm3PA4nWs/QYCIoOX215m3aYVL+X3bOewrunEH1yOjmX/B86S1Kzzw2XDkqiJTTmd9QDKqCgt6XEO5x2q0VdB/d1ySWX4PP5eOWVV5gzZ0748ZycHJ5++mlOO+00TQMUQiQu2/ApOPRZKPqI30qiQp/clGg11qIG/Jp9yA4JfbjX2ezozPHZzejMkoYcjWXV17jy11H59RtkX3Rfs8/TNewBtDmfFZKZauWm84exedsuzju+j2brtkaoKcbMORuZt2QXJ0+Mf8dPEbwZYOqWh6dsB66izSQPPSbiNVRVZc+cV3Hlr0MxWci++H4M9i6Hfr6UDoooCYTOZ9lSNP9Z2pm0qv/oFVdcwaJFi5g7dy7vv/8+X375Jd9++y1nnHGG1vEJIUSL6ZPsoNMDKv6GGs3X99Y0NcKQQcVxsbcxhgHHL8tp3Nr8jMfwjpbGJa3Hjcrh2KH28M5WPJ00PrjDtim/il1ldUd+gYiJcPngro2ten319x/SsO47UHR0O/9uzNmHb+EvpYMiWva2dpeywbZo9aCH2tpadu7cyebNm7Hb7ezYsUNazQoh4kpRdMFZWoAvCuWDviqZoRVvpi49SZ10JgCV814n4HUf9Bx9/Z6m52q3o5Vo0u0WJg4NJvxfL90V52hESFvOadWvWUDNjx8D0OW0G7D1G3PE1+zbdVA+gwktSWt3bbQq0XrllVeYMmUKN998Mw8//DClpaXMmDGDCy+8kLo6ubMmhIifcOfBKMzSCpUOSsfB+Eo/9gL0KZn4asup+Wn2fl9TAwH0jcG7+20dVpzoQk0xvllWiNfnj3M0AsDSczAAnvICAq7GFr/OsWMNe+a+CkDa0edjHzOtRa8LmIM7WqrPQ8DVEGG0QhxaqLW77Gi1TcSJ1jvvvMNLL73Etddey0cffRS+g3LFFVdQWFjICy+8oHmQQgjRUtGcpeWtkWHFiUBnspI57VoAan/+FG9Vafhr/tpylIAP9EYMafFrWBELYwdlkZlqod7hYfH6QzcHEbFjSElvuhGj4ipu2bgb9+58ds96CgJ+kocdR/oJl7X8gnoDOquc0xLak9JBbUScaL399tvccMMN3HbbbQwbNiz8+JQpU7j99ttZsGCBpgEKIUQkojlLa++wYkm04i1p8FFY+4xC9XupmPda+Kafr7IIAIOGHQcTlV6v4+QJwXNoMlMrcURyTstXV0nZh4+iepxYcofR9cybIx6ArQt1W5VES2jI31gDSOlgW0WcaJWUlDBx4sRmv9a3b18qKiraHJQQQrSWPiUT0H6Wlurz4qsNvr8Z0qUZRrwpikLmKb8BnQHn9lU4mhpj7JtodQYnTwwmWqu37qGssuWlaiJ6LL2C5YOuosOf0wq4HZR9+Cj++iqMXXrS7YI/oBgOnpV1JKFuq9LiXWhJdrS0EXGilZOTw6pVq5r92vr168nJaftwSCGEaC1DSnTOaHlrywEVxWhBn5Sm6dqidUyZPUg76mwAKr8ONsbwVQbnShkye8YztJjJzkxi9MCuAMxfJk0xEkFoR8tdsg3V5232Oarfx+7/PI2nvAB9UhrZF/8femtyq66nCyVasqMlNORvmhkpP+/aJuJE64ILLuDVV1/ltddeIz8/HwCHw8FXX33FP/7xD8477zytYxRCiBbbe3dX2x2tvWWD3SIu7RHRk3bMr9Dbu+Cr3UPNolmdLtGCvU0x5i/dhT8gnefizZjRHZ3Njurz4C3PP+jrqqpS8cU/ce5Yg2I0k33RfRjbcJ5w77lUSbSEdsKlg7Kj1SYRJ1rXX3895513Hk8//TRnnhlssXvVVVdx++23c8IJJ/Db3/5W8yCFEKKlwjtaGrd391YHG2FIx8HEojNZ6NLUGKNm8Wf4qkqAzlM6CHDU8GxSbCYqa12s3Lw73uF0eoqi7O0+WLzloK/XLJpF/ZpvQNGRde4dmLv3b9P1woPaJdESGlFVdZ/27vY4R9O+GSJ9gaIoPPzww1x77bUsWbKEmpoaUlJSmDBhAgMHDoxGjEII0WKhM1oBVwMBrxud0azJut59drREYrENmoS17xicO4Jl7areiN6eGeeoYsdo0DN1fC8++34785YUMGGo3AyIN0vuEBxbl+It2QoD+4Ufr1+3kOqF7wOQOf03JA2c0OZr6UKzA6V0UGhE9bpQfR5ASgfbKuJEa+bMmZx11ln06dOHPn36RCMmIYRoNZ3ZhmIwofo8+Buq0Wm0A7W3dFDOoSYaRVHocsqvKfznHeD34U/uiqK0akxkuzVtUi6ffb+dpRt3U13nIt1uiXdInZqlZ/CclqfkFxhwKgDO/HXs+d/fAUg96hxSx5+qybXCDYCahhZLabNoq9BulmI0ozPJe0lbRPyT6KmnnuL444/nhhtuYO7cubjd7mjEJYQQraIoSvjMgl/DzoN7hxXLjlYiMmZ0J21y8IywL63znM8K6Z1tZ3DvdAIBlW+WF8Y7nE7PnN0neMPH1YCusQJvRRG7P3kSAj6ShhxNxtQrNLuWvmlHS/W6UN0OzdYVndfeskE5n9VWESdaP/zwA/fffz8Oh4O77rqLo48+mvvuu48lS5ZEIz4hhIhYaJaWT6NzWqoawFdTDsgMrUSWfvzFZFz4fzgHHB/vUOLilKOCTTHmLSkIzxUT8aHoDZh7BI9TmMo2U/3p0wTcDsw9B9P17N9ruuOqGM3omjoWSvmg0IK0dtdOxP/S09PTueyyy3jnnXdYsGABN910E1u3buWaa67hhBNO4JlnnolGnEII0WJ7u3Bpk2j566tQ/V7Q6THYu2iyptCeoiiYeg4GgyneocTFMaN6YDUbKK1oZP0O+cAdb6E279ZtPxCor8SYkUP2hX9EF4W/n4ZQ+aA0xBAa8Dsk0dJKm26p5OTk8Jvf/IbnnnuOyy+/nD179vDvf/9bq9iEEKJVDOHSQW0SLW9VKQDGtCwUnV6TNYXQmtVs4PgxwW6L85YUxDkaEUq0AHTWFLIv+RN6W0pUrrXvOS0h2kpKB7XT6kSrrKyM119/nfPPP59TTjmFL7/8kiuuuIJZs2ZFtE4gEODFF1/kuOOOY/To0Vx//fUUFh66vjw/P58bbriB8ePHc/zxx/Piiy/i8/n2e87ChQs5//zzGTFiBCeffDLvvvtuq36PQoj2aW+7Y23OaIVbu6dJ2aBIbKGZWj+tKaHB4YlzNJ2bpcdAFLMNVWcg7Zw7o1p2bLDLjpbQjpQOaifiroPvvvsuc+fOZdWqVZhMJk466SRuv/12jj32WHS6yPO2v//977z33ns8/vjjZGdn89RTT3Hdddfx3//+F5Np/+312tpaLr/8cvr27cvMmTNxOp38+c9/pqysjMceewyApUuX8rvf/Y4bb7yR559/niVLlvDggw+Snp7O6aefHnF8Qoj2JzRLS6vSQV+NtHYX7cOAXmnk5djJL63ju5VFnHls33iH1GnpzFYyL/sr27dvJyenbbOyjiRUOuiXHS2hASkd1E7EmdGjjz6K0Wjk0UcfZdGiRTzzzDMcf/zxrUqyPB4Pr7/+OrfeeisnnHACgwcP5rnnnqOsrIx58+Yd9PzZs2fjcDh44YUXGDZsGOPHj+eRRx5h1qxZFBUVAfDSSy9x8sknc+utt5Kbm8uFF17Iueeey/LlyyOOTwjRPum1Lh0MdxyUHS2R2BRFCe9qfbVYmmLEmyEti4AtLfrXkR0toSF/Yw0giZYWIt7Ruvnmmzn11FPp16/fkZ98BJs3b6axsZHJkyeHH7Pb7QwdOpRly5Zx5pln7vf8goIC+vbtS0ZGRvixoUOHArB8+XIyMzNZvnw5L7744n6vC+12CSE6B0NyqBlGtSZzZUKlg9JxULQHJ4zryRv/20B+aR3bimoY0Cs93iGJKNNLoiU0JGe0tBNxovWvf/2L4cOHa5JolZUF7xLn5Ow/ADQrKyv8tQMfLy8vx+/3o9cHD6QXFxcDUFlZSUFBAYFAAL1ez6233sqyZcvIysriiiuu4MILL2xTrKqq4nDEfz6F0+nc73/by9rRXl9ij8/6iRp7QG8GgnNlGmsq0ZltrV5bVdVwMwy/Na3F7wOd8fse77WjvX57iV0PTBqaxY9ry5i7aDvXnz203cQe67WjvX6s1jYakwDw1VZo9lmlvX5for1+Z4jd15RoefRm1Aj+PiVC7LHS0pu4ESda/fr1Y+fOnUyZMqVVge0r9M068CyW2Wymtrb2oOefdtpp/P3vf2fGjBnceeedOBwOHnnkEQwGA16vl4aGBgAeeOABbrjhBn73u9+xZMkSHnroIYA2JVter5dNmza1+vVay8/Pb5drR3t9iT0+6ydi7KkGMzqfm1/WriCQfOiW7EdaW/E4SPME36t+Ka2E8rqI4uhs3/dEWDva67eH2Pt19fEj8P2qEib2UTEZdJqtfTid/fset7V9btIB1eNk07rVYDBru36UyJ9pfNY/7NqBAGnOehRge9Fu1D0N2q7fRtH+vkfiwPylOREnWieeeCLPPvssP/zwA4MGDcJm2/9OsaIo3HzzzS1ay2KxAMGzWqFfA7jdbqxW60HPz8vL44UXXuCBBx7g3XffxWaz8fvf/55t27aRkpKC0WgE4JxzzuGqq64CYMiQIRQUFPDmm2+2KdEyGo307x/dw6wt4XQ6yc/PJy8vr9nvUaKuHe31Jfb4rJ/IsVcs7YKvqpi8bhmYc4cc9PWWru0p3UYVoEtOZ8jwkTGJPZ5rR3t9iT026w8erPLVqkWUVTmp9qYzqX96u4k9lmtHe/1Yrr37Bxuq28GA7l0xZPbQfH0tyZ9pfNZvydr+xhr2ACgKg0aORYmgB0O8Y4+lbdu2teh5ESdaL7/8MgCLFi1i0aJFB309kkQrVDJYXl5Obm5u+PHy8nIGDRrU7GumTp3K1KlTKS8vJy0tDZ/Px+OPP06vXr3Izg6enxg4cOB+r+nfvz//+c9/WhTToSiKclBSGU9WqzVq8URz7WivL7HHZ/1EjN2YmomvqhiDt/Gwrz3S2gFncHfdlJHTqt9jZ/u+J8La0V6/vcQ+/ag83pq7ie9WlXLC2O6arn0o8n2P39oGeybePQ4MXoem12qv35dor99RY3fXlwOgt9lJSk7WfP22ivb3vaVaevY74laBmzdvPux/kZTXDR48mOTkZJYsWRJ+rK6ujo0bNzJhwoSDnr98+XKuvPJKfD4fWVlZmEwm5s2bh9VqZezYsXTr1o3c3FzWrFmz3+u2bt26XyInhOj49s7SalvnQW9107BiaYQh2pmTJuSi0ylsyq+iqDzy8h/RvoRbvEtDDNEGodbuOmmEoYlWDywGqK+vZ/v27Xg8Hvx+f8SvN5lMXHHFFTz99NN88803bN68mTvuuIPs7GymT5+O3+9nz549uFwuAPr27cuWLVt44oknKCwsZP78+TzyyCP89re/Jbkp677lllv48MMPeffddyksLOSDDz5g1qxZ/OY3v2nLb1UI0c7snaXVtqHF4WHFkmiJdibDbmHCkODstwUriuMcjYi2UKLlk1laog1kWLG2Ii4dBFiyZAlPP/0069evR1EUPv74Y/71r3+RnZ3NH//4x4jWuvXWW/H5fPzpT3/C5XIxYcIEXnvtNYxGI0VFRZx00knMmDGD888/n4yMDF599VUef/xxzjzzTLp27cott9zCNddcE17vnHPOAeAf//gHM2bMoEePHjz44IOce+65rfmtCiHaqfCOVhtnaYVmaMmOlmiPTjmqN0s2lPH96lLG9MqKdzgiigz2YNMfafEu2kISLW1FnGj9/PPPXH/99YwZM4a7776bp59+GgiWAb744ot069aNa6+9tsXr6fV67rnnHu65556DvtazZ0+2bNmy32Njx47lo48+Ouya55xzTjjhEkJ0ToaUvbO02sIXSrTSurU5JiFibeygLDJTLVTWuthS7GTE8HhHJKJFb296z5MdLdEGAYfM0NJSxInW888/z0knncQLL7yAz+fjqaeeAuDGG2/E4XDw8ccfR5RoCSFENOhTQme0Wv+hI+Bx4m+sAaR0ULRPer2Okyfk8uH8rSzaWE9KWiGmpg69WtIpAdIMAc3XFS2394xWRZwjEe2Zr0F2tLQUcaK1adOmcFfBAztuHHPMMcycOVObyIQQog0MyU1ntBpqUNUAihL5kdTQ+SydNRm9tXXdl4SIt5MnBhOtkiovr/9vc9Suk5dlZvhQSbbixWAPndFqW7m0iB+/o46aL/+NOWCGIQePJYmF8I6WJFqaiDjRSklJYc+ePc1+rbS0lJSUlDYHJYQQbaVvSrQI+Ag46lv1Q8PXlGgZ02Q3S7Rf2ZlJXHf2EH5anY89xY7eoNf8Gis27Sa/3M3787dxw3mjNV9fHFloRyvgaiDgcaEzWY7wCpFIPOUFlH30OL7acqyKDnXaZUDs25iHz2hJ6aAmIk60TjrpJJ577jkGDhzI0KFDgeDOVllZGa+++ionnHCC1jEKIUTEFL0BfVIq/sZafA3VrUq0vDXB81mGdDmfJdq3aRN60jO5niFDhkRlBs2CpTt57sO1/PfHAob1y+KYkd01v4Y4PMVsQzFZUD0ufPVVmDLlz6C9aNy6jPLPnkf1BLtsK2oAT9kOklLHxTyWULm87GhpI+JamrvuuovMzEwuuuiicFJ15513cuqpp6IoCnfeeafWMQohRKvsnaXVunNaezsO5mgWkxAd0VHDu3H0kGB57QsfrKRwd32cI+p8FEWRWVrtjKqqVC/6D7s/fgLV48KSNwJT7xEAeEu2xiUev6MOkERLKxHvaKWmpvLxxx/z6aefsnjxYmpqakhJSeHKK6/k/PPPx2q1RiNOIYSImD45HXbvbHXnwXDHQdnREuKIThqVSo3LyMad1Tz25lKeue14bBbtG2+IQzPYM/FWFkvnwXYg4HVTMecVGjb8AIB93KlkTruWisX/xVOwDk88Ei2PC9XnAaR0UCutmqNlMpm46KKLuOiii7SORwghNBNq8d7aWVqhHS3pOCjEkel1CrddNIL7X11KUXkDL364mnuvGn9Q4ywRPfoUmaXVHvjqq9j98RO4S7eBTk+XU67DPnY6AKbuAwHwlm5rdSOn1gqVDSpGi5zx00ir/vRmz57NwoULAdiyZQtnnXUWY8eO5f7778fj8WgaoBBCtJa+DbO0VL8PX22wTbIMKxaiZdKSzfzxqgkY9AqL1pbw6cLt8Q6pUzGEZ2lJi/dE5S7ZRvHr9+Iu3YbOmkzOZQ+EkywAQ9dcVL0R1e3Au6coprH5wx0H7TG9bkcWcaL1+uuvc//997Nx40YAHnzwQaqrq7nwwguZP38+L774ouZBCiFEa4RavLfmvIKvdg+oARSDaW8HQyHEEQ3Oy+C6c4LnTN6cs5F12+RDf6yEz2hJ6WBCatjwIyVv/xl/QxXGrr3oce0TWHvvP0Vc0enxpQYbmbiKojeOoTnhjoNJaTG9bkcWcaL18ccfc9111/G73/2OoqIiVq9ezU033cR9993HXXfdxZw5c6IRpxBCRCz0ocPXEPmO1r5lg1L6JERkTj86j6njexEIqDzx9jIqapzxDqlTCM/SqpdZWolEVQNUffce5Z8+h+rzYOs/jh5XP3bIaglfek8AXEVbYhmmtHaPgogTraKiIo4//ngAFi5ciKIoTJ06FYC+fftSWSl3UYQQiUGfEtrRivxDh1caYQjRaoqi8LtfjaRPdzu1DR4ef2sZXp8MM442fejmkpzRShgBj5PdnzxFzaJZAKROPpduF96LznzoMQu+tFCiFa8dLUm0tBJxopWRkUFFRbAMYOHChfTt25fs7GBGvmXLFrp06aJthEII0Urh9u6NtagBf0Sv3dtxUM5nCdEaFpOB+66eSJLVyJaCav792bp4h9ThhXa0Ao46Aj45Mx9v3ppySmb+H46tS1H0RrqefSuZU69E0R1+aLgvrQeg4Ksuw9dQE5NYYd8zWpJoaSXiROvEE0/kmWee4YEHHuD777/n7LPPBuCNN97g+eef5+STT9Y8SCGEaA19kh10ekDFH+EPK2/1bgAMaZJoCdFaOV2SuOuysQDM/SmfBct3xTmijk1nSUYxmIDW7eQL7Th3baT4jXvxlBegT0oj58qHSRkxpWUvNlowdAnuarljuKslO1raizjRuu+++zj66KNZtmwZl1xyCb/+9a8B+OCDD5gyZQq333671jEKIUSrKIou3Mgi0jML3upSAIwZkmgJ0RYThmZz6fRBAPzt4zXsKK6Nc0Qdl6IoGOxNLd6lIUbc1K2eT+m7DxFw1GHK7kuPXz+JpcfAiNYwdh8AxLZ8UM5oaS/iOVpms5mHH374oMc///xzzGazJkEJIYRWDMnp+OsqIpqlpaoBfDXlgJQOCqGFS6YNYuuualZsLmfGzKU8d/sUkm2meIfVIentmXirSuScVhyoAT+V82dStyzYGC5pyNF0PesWdMbIPx+bug/EuXYBrsIYJlpSOqi5Vg0sdjgczJ49m+XLl1NXV0dGRgZHHXUUZ511FiaTvHEKIRJHa2Zp+eurUX0eUHThu8NCiNbT6RTuunwcdzy3kLJKB8+8t5I//3oSOp109NTa3hbv0lY/lvyuRspnP4NzxxoA0qdcStoxv2p119rQjpa7bCcBr7tVyVqkpHRQexEnWoWFhVx99dWUlJTQq1cvMjMzyc/P57///S9vvfUWb775JunpnXPmjN/vx+v1RvUabrc7/L86nbbTwqO5drTXj3bsfn9kjRRE4mjNLC1vTVNr99SuKPpW3Y8SQhwgxWbivqsn8IeXfmD5pt18OH9ruKRQaMcQurkkpYMx46ksYfdHM/BWlaAYzWSdfStJg49q05p6e1f0yen4G6pxl27DmjtMo2ibp/p9BJz1wWtL6aBmIv4E8fjjj6MoCp9++imDBw8OP75mzRp+//vfM2PGDJ588klNg0x0qqpSVlZGTU1N1K8VCAQwGAyUlJRonlBEc+1orx+L2CH4Zy3aF30rZmn5mhphyPksIbTVr2caN10wiuc/WMX78zYzoFca44fICAUt7Z2lJYlWLDh2rKZ89rMEXI0Y7F3oduEfMWf3afO6iqJg6TmIxs2LcRVuiXqi5XfUN11Yh86aHNVrdSYRJ1o//fQTjz766H5JFvD/7N13eBTV1wfw72xL74WEBAgBJAkQIIQIUqWJCAgoL72LBQEFAQEBfyjSBJQiAkoRBVGQIlUEFBGVphKkBEJIIL33Tbbd94/NbhJS2DKzm3I+z8Oj7E7OnITZyZ69956Ltm3bYvbs2Vi2bBlvydUWuiLL29sb9vb2gm5uqlarUVxcDBsbG4jF1bcHrUmxhY4vZGzGGPLz85GUlISMjAw4ODjwGp8IS6LbS8uINVrKzJJGGNRxkBDe9e7YGFFxWTj5ZyzW7rmGT2b1gI8H3Vf5ov9wKZe6DgqJMYacy8eQceYrgGlg4x8En5fn8TrtzsY/CAV3/rJI50H9+ix75ye2nyeGM7rQsre3h1QqrfQ5d3d3Qd6g12RqtVpfZHl4eFjkfABga2srSLEiVGyh4wudu0wmg1KpRFZWFtRqdb27zmsz3V5axqzRUmaXtHanRhiECGLqkNaISchB1MMsrNh1BatndoONlO6rfNCNaBkzXZoYSaNG7pkdkP/3KwDAMbQXvJ5/FZyk8vfHprL11w5qFCVEgTENOI7/GTs6peuznAU7R31k9L/YmDFjsH79eqSmppZ7PD8/H1u3bsXIkSN5S6420K3JsreveodvUjfoumoKvQ6P8Es/omVEe/fSzYppShMhQpBKxJg/oSNcHGWISczB5gPXaWo2T/TNMAqywdT0+4pvGnkeHK/s1RZZnAjufSbCa+A03ossALDxaQpOIoNGng9lRiLv8ctSF2QDoPVZfDNoRGv8+PHl/v7gwQP07dsXYWFh8PT0RE5ODq5duwaNRoOGDRsKkmhNJ+R0QVIz0L9x7aSbRqMpyje4c5Nus2Jq7U6IcDxd7TB3bDiWbP0D564+QlCAO57vHGDttGo9kb0zOLEUTK2EKi8LUldva6dUp+T/eRDSrEfgZHZoMOwd2DdrL9i5OLEENg1boOjhTRQ9ugNZySbGQiht7e4q2DnqI4NGtBhj5f6EhYUhNDQUKpUKycnJkMvlCAkJQevWrZGSkiJ0zoQQYjCRjT04iXbbCbUBDTHU8jxoivIBABJXGtEiREhtW3hh/IAQAMC2Q5GIiqN1RebiOA5imj4oGEXyfQCAc5/JghZZOrb+2s6cQm9crJs6KKLW7rwyaETr66+/FjoPQggRBMdxEDu5Q5WVDFVe5hNHqXSjWWJHN4hktpZIkZB6bdizzRH1MAt/3kjCyq+u4JNZPeHqJPyeQXWZxMlDe8+jvbR4xZgGqowEAIDUO8Ai57RtpF2nJXRDDP0aLZo6yCuTN4i5f/8+Ll++jLy8PLi5uaFDhw4IDAzkMzdCCOGFxNENqqxkg0a0Stdn0bRBQiyB4zi8PbI9HibnISEtHx9/cxUfvNoZYrFwC//rOn2Ld9pLi1eqrBRApQATSSB2scyUTBs/7YiWMjMJ6oIcwTYTps2KhWF0ocUYw/vvv4/9+/eXW7jKcRyGDh2K5cuX85ogIYSYS6zbwNOAhhjKkkJLQo0wCLEYe1spFk7siHfW/4bI6HR8ffI2Jg4Udt+guqz0nkeFFp8UaQ8BAGpHT3AC7NlZGbGdI6RejaBMe4Si+DtwaPm0IOfRFFKhJQSjr5Ivv/wSP/zwA2bOnImzZ88iMjISZ86cwfTp0/Hjjz9i165dAqRJLOXixYsYPnw42rZti86dO2P+/PnIycnBpUuX0LJlS5w+fRp9+vRBu3btMHHiRNy/f1//tYwxfPHFF+jduzfatm2LF198ET/++GO5+L/88gtGjBiBdu3aoU2bNhg2bBguXLigf37cuHFYvHgxhg8fjvDwcPz444+YP38+5s2bh2XLliE8PBwRERHYsGED7t+/j9GjRyM0NBRDhgzBjRs39HHu3r2L1157DR07dkTr1q3Ru3dv7NixQ//8xo0bMXHiRGzbtg3du3dHmzZtMHbs2HLfD6k7JCVvOgzZS6u0EYavoDkRQspr7OOMt0Zq17z88Es0LkYK22WtLpM40YiWEBRpjwAAakcvi57X1k/4dVoqmjooCKMLrQMHDuCVV17BG2+8AT8/P8hkMvj7++PNN9/EK6+8gu+//16IPGsdxhiKilX8/1GoUKRQa/9bzXGmtMnNysrCO++8g2HDhuHEiRPYtGkTrly5gtWrV+uPWblyJRYvXozvvvsOEokE48ePR16edjfxTz75BN9++y0WL16Mo0ePYvz48fjf//6HPXv2AABu3ryJuXPn4oUXXsDRo0fx/fffw93dHfPmzYNCodCfY//+/Rg/fjz27t2Lbt26AQBOnDgBsViMgwcPYuLEifjss8/w+uuvY8qUKdi/fz9sbGywYsUKAIBcLsfkyZPh6uqKffv24dixY+jfvz9WrVqF27dv689z9epVXLt2Ddu2bcPevXuRkZGBpUuXGv+PTWq80r20nlxoqbJKNiumES1CLK5rWz8M6dEMALB+3z94lJJn5Yxqp9K9tKi5CJ/0I1pOFi60StZpFcVHCRKfMQaNrtBypEKLT0ZPHUxKSkKnTp0qfe7pp58uN2pQXzHG8O6m33E71no3uOAAd6ya3tWoluTJyclQKBTw9fWFn58f/Pz8sGXLFqjVauTkaF+A7777Lnr06AEAWLNmDXr27Injx49j8ODB2LVrF9atW4eePXsCABo3boyEhARs374dY8aMgUgkwrvvvotx48bpN/wdP348pk6dioyMDPj6akcQgoODMWjQoHK5ubq64t1334VIJMLEiROxfv16DBgwAL179wYADB06tFyhNX78eIwZMwYODg4AgJkzZ+LLL79EVFQUgoODAQAqlQqrV6+Gi4v2pjJy5Eh8/PHHRv+sSc1XupfWk9do6Ua0JK60RosQa5j4Qgii47Px3/0MrPjqMj6c2tHaKdU6NKIljNKpgxYutEo2Li5Oug+NSgFRSSddvjCFXL/nGo1o8cvoQsvPzw9RUVHo3Llzhefu3LkDd3d3XhIjlhccHIz+/ftj2rRp8PLyQpcuXdCzZ0/07dsX165dA6AtpnVcXV3RtGlT3L17F9HR0SguLsY777wDUZl5yyqVCgqFAkVFRQgODoatrS2+/PJLPHjwAHFxcbhzRzsMrlar9V/TpEmTCrn5+/vr4+o2h27UqJH+eRsbG/1Gwu7u7hg9ejSOHTuGW7du4eHDh/rzaDQa/dd4enrqiywAcHJyos2I6yixgVMHNcpi/THUDIMQ6xCLRZg3LhxvrzuPRyn52HLoFp4LNbl3V70kdvYEoN3SgqlV4MT08zMXUyv1mwZbekRL4uYDsYML1AU5UCTF6Ee4+KLbrJiT2Rq01yQxnNGvvIEDB2Ljxo1o0KAB+vfvD47jwBjDyZMnsWnTJowYMUKIPGsVjuOwanpXFCvUTz7YSGqNGkVFxbC1tYFYJK7yOBuZ2KQNdpcvX44ZM2bg999/xx9//IG5c+eiQ4cOmDZtGgBAIil/yajVaohEIv1UxU8//bTS7pMymQyXL1/G1KlT0aNHD4SHh2PQoEGQy+V48803yx1ra1uxpbZUWnHHdVEVC1HT0tIwYsQIuLu7o1evXujatSvatGmjH4krmxOpHyT6qYNZYIxV+dpQZWtHs0S2DhDZOVosP0JIeW5OtlgwoSMWbP4df91MgVrhgISCeEHu2wqFAslJ+YjP5z++QqGASKFAMK9Rn0zs4AyIJIBGBXV+FiQuli0MrC0mMRfpufx+cKrMSAQ0anA29mA2TrzGfhKO42DjH4TCqEsoir8jQKGVC4A2KxaC0YXW1KlTcfXqVcyaNQtz586Fm5sbsrKyoFarERERgbfeekuIPGsdjuNga8P/J0hqNQdoVLCVSfTT7/hy/fp1HD16FIsWLULz5s0xceJE/Pjjj5g7d66+gL5x44Z+NDMzMxNxcXGYNGkSAgMDIZFIkJiYiGeffVYfc/fu3YiOjsYHH3yAXbt2ITw8HOvXr9fnrtujzZQ1ZVU5duwYsrOz8dNPP+kLtKioKN7PQ2oPsaN26iBTFoEVF4Kzdaj0OGVmScdBVx+TPqgghPAnKMAdrwxujS2HbuDKvQJcuXf7yV9klmxBokolHJ4JV6FkMoZFcJwIEid3qHJSocrLqFeFVnq2HEu+uAJbKdClI3+/83XTBiUe/oAVfj/Y6gqtR3eAipPKzEJ7aAnH6EpAJpNh586dOH/+PK5cuYKcnBy4uLigY8eOFUYMSO3i6OiI77//HnZ2dhgxYgSKi4tx4sQJBAQEwM1N+0Z16dKl+PDDD+Hk5IRVq1bBy8sL/fv3h52dHUaOHIn169fD0dERYWFhuHTpEj7++GO89tprAAAfHx+cOXMG165dQ8OGDXHp0iWsX78eAMo1wzCXj48P5HI5Tp06hQ4dOiAmJka/fovP85DaQySzhcjGHpriQqjysyCrqtDK1u2hRY0wCKkJBnRpCoVSiT/+jYWTkzMkEn4/YAQAlUqNvLxcQeJHRqehQK7CnbgsdHFz5jX2k0icPbSFVj1bp/Xv3VQoVRooVUBmbhEcHSu/3xtLkVqm0LICfUOMhKhqZ2aYQq1v7W7Za7Q+MHnIpUePHlRY1THNmjXDmjVr8OWXX+Lbb7+FSCRCp06d8MUXXyApSduJbcSIEZg3bx6ys7PRqVMn7N69G3Z2dgCABQsWwM3NDevXr0dqaip8fX0xc+ZMvPLKKwCAGTNmICUlRT8NsXnz5li+fDnmzp2LGzduoFmzZrx8H/3798fNmzexcuVK5Ofnw8/PD8OHD8fZs2dx48YNjBo1ipfzkNpF7OQOTXGhtguXZ+W/KFX61u60PouQmoDjODz3dCM0ds5HcHCwfo0unwoLC3H79m1B4n+y9yrOXUvAjfuZ6NKu4vpjIdXXvbSu30vX/39CeiEaN/TgJa6utbukit8fQrPxaQpOLIWmMBfKzETIPPx4i61bo0UjWvyj1ZGknO7du6Nfv34VpiXqCq2+ffti0qRJlX6tRCLB9OnTMX369Eqfd3V1xZo1a2Bra1sufr9+/fT/r5tKWNbKlSsrPKabCqgzdOhQPP/88wC0v5jnzJmDOXPmlDumbN4zZszAjBkzyj0/bNgwDBs2rNLcSe0ncXKHMj0eqmoaYpRuVkyFFiHEfK0D3XHuWgL+i7F8F2J9i/d6NKLFGMP1e2n6vyemFfAWWzd1UOrpD1hh1wFOLIVNw+YoenQbRY/u8Fxo6Ua0XHmLSbQss601IYRYmW4vrer2ldEVWjSiRQjhQ6tA7bT7uOR85OQXW/Tc+hbv9WhEKz41H1l5pT/nhHR+Ci2NQq5vlmStqYMAYOOv3bi4mOeNi0unDtKIFt+o0CKE1Au6vbRUVeylxTRqqHK0n4RSoUUI4YOrow28XbSTh27cT3/C0fySlLR4V+XWn02LdaNZuuVLfI1oKdLiAWhHfER2lu04WJZuPy2+Ny4uHdGiQotvVGgRgzz99NOIioqCv7/1PskhxBz6Ea0qpg6qctK0rXvFUohLijJCCDFXUx/tliVl1w5ZgrgejmjpCq0OLbVdFhN5GtHSTRuUeTfmJZ6pdIWWMiMB6kL+5i9SoSUcKrQIIfWCxKl0L63KKEsaYUjcGoDj6NZICOFHYAPtBrBl1w5Zgn6NVl4mmIb/fT1rGrWG4cZ9bVH53NONAABZeQoUyM3fT0upW5/lZd1CS2zvBGnJ2qwiHqcP6qcOUjMM3vH6buL+/fsYP348nyEJIYQXug5c6io+3VXR+ixCiACaeNuA44Ck9AKkZhVa7LxiBxeAEwFMA3V+tsXOay0xCdkokCthbytBq6ZucLTTvsVNSMs3O7au46DMyoUWUHb6ID+FFlOroJFrf0Y0osU/XgsthUKB+Ph4PkMSQggvJCWbFqvys8GYpsLz1HGQECIEW5kIzfy0b2AjLTh9kBOJ61WLd93UzDbNPCEWi+DpLAUAxKeaP8VOP3WwJhRaJftpFfO0TktdmKv9H04EkZ0jLzFJKV4LreDgYJw7d47PkIQQwgtxSaEFjQqaSua26zsOutJmxYQQfrUJ1BY816MtPH2wHq3T0k3NDG2hbQLi6axtQhKfat6IlrowD+p87ZRzmZf116nblIxoFSdGg6nMnxapX59l70zT5gVAP1FCSL3AiSX6aRGq/IrrtFTZNHWQECKM1s20hVbkvTQwxix2XolzyZTpOr6XllKlxq0H2kZHbZtrG2HwVWjpRrMkrt4QyezMisUHqbsvRPbOYGolipNjzI6n36yYpg0KwqRC69ChQzh//jwA4M6dOxg0aBDCwsKwcOFCKBQKXhMkhBC+lO6lVf5NB2NM3wxD6k6FFiGEX081coFUIkJmbrHZb/yNUV9GtO7EZUGhVMPVyQaNfbTt1/maOliTpg0CAMdxsPXT7qfFxzqt0j20XM2ORSoyutDasWMHFi5ciFu3bgEA/ve//yErKwvDhw/HmTNnsGHDBt6TJIQQPlTVeVCdnw2mLAY4ESQuXtZIjRBSh8mkYgQHlI5qWYpYv5dW3S609NMGm3uCK9lESzeilZReAJW64rpcQ9W0QgsoXadV9IiHQqtAu0aLRrSEYXShtX//frzyyit44403EB8fj3///RfTpk3DggUL8M477+D48eNC5EkIIWbTrdN6fC8t3bRBibMnOLHU4nkRQuq+ti20H+Jcj7ZcQ4yyLd7rMl2TEd3PGACc7cWwkYqgUjOkZJre7VGp6zho5T20ytJ1HixOiDJ7Kmppa3dns/MiFRldaMXHx6N79+4AgPPnz4PjOPTq1QsAEBgYiIyMuv2pCSGk9hJXMaKlb4ThRo0wCCHCaFvSpCEyOh1qjWXWaemnDuZadrNkSyosUuLuQ+09PbS5p/5xEcfB19MBABCfYtr0QcYYFKlxAGrWiJbMNxAQS6AuyNFvTWIqWqMlLKMLLXd3d6Sna1+w58+fR2BgIHx8tGsaoqKi4OnpWd2XE0KI1ehavD++Rqu0tbuvxXMihNQPzf1dYW8rQYFciZiEbIucUzeipcrLrHRbi7rgZkwG1BqGBu728PFwKPecn67QMnFdnDovE5riQkAkhtSjodm58kUkkcHGtxkA89dp6bsO0hotQRhdaD377LNYu3YtlixZgt9++w2DBw8GAOzcuROffvop+vTpw3uStRFjDBpFkSB/mLL4yceYMJQcEhKC7777DiNHjkSbNm0waNAgnD17ttwxv/zyC4YNG4bQ0FD07dsXn376abkGKHfv3sVrr72Gjh07onXr1ujduzd27Nihf37Lli0YP348Zs2ahbCwMHz44YdQq9X4+OOP0aNHD7Ru3Rr9+/fHt99+W+68hw8fxuDBgxEaGopevXph8+bNUKu1O93Hx8cjJCQEZ8+exYgRI9C6dWv06tUL3333ndE/A1K36T/dfazroErXCINGtAghAhGLRWgdWDKqZaH9tMQOrtpNizVq/RvquiYyuuK0QZ2GXvYATC+0dOuzpO6+NW5auX7jYjPXaenXaNnTiJYQJMZ+wYIFC/DRRx/hypUrGDlyJCZPngwA2LdvH3r06IG3336b7xxrHcYYEne/x9tmcqaw8Q9Cw/HL9ItCDbVx40bMnj0bq1atwsGDBzF9+nTs2bMHYWFh+O233/D2229jwYIFeOaZZ/Dw4UN8+OGHePDgAdavXw+5XI7JkyejS5cu2LdvH8RiMfbv349Vq1ahc+fOeOqppwAAV69exfjx43HkyBGo1Wrs3bsXp06dwieffIIGDRrgl19+wf/+9z+0aNEC4eHh2LVrF9auXYv58+ejS5cuuH79Oj744ANkZWXhvffe0+e+du1aLFq0CEFBQdi5cyf+97//4ZlnnkGjRo14/dmS2kvspBvRKr9eoXTqIHUcJIQIp20LT1y+lYzr99LwUq8Wgp9Pu62FK9T5mVDnZuhH9esSXSMM3dTMsnQjWo9M7DxYExth6Nj6ByEHR3gY0coGAIho6qAgjC60JBIJPvjggwqP//jjj7CxseElqbrBuAKnphg0aBBGjx4NsViMOXPm4PLly/jmm28QFhaGLVu24P/+7/8wcuRIAEDjxo2xdOlSTJgwAfHx8bC3t8f48eMxZswYODhob24zZ87El19+iaioKH2hpXvcyUnbgnXPnj2wt7eHv78/vL29MXbsWAQGBqJp06ZgjOGLL77A2LFjMWbMGABAQEAAsrOz8fHHH2PmzJn6mGPGjEGvXr0gFosxa9Ys7NmzB9evX6dCi+jp27sX5IBp1PrHS6cOUqFFCBGObtTl5oNMKFVqSCViwc8pcfaAOj8TqrwM2KC54OezpJz8YjxI1I7ItGlesdBq6FU6dZAxZvSHz/pCy7uJmZnyz9Zf2+JdmR4PtTwPYjsno2MwxvTNMCRUaAnC6EKra9eueOGFF/Diiy+iTZs2+sepyCrFcRwajl+mbRfNM7VajeLiYtjY2EAsrvoGzUltjL6hAEB4eHi5v7dv3x4XL14EANy6dQuRkZE4cOCA/nndFMX79++jR48eGD16NI4dO4Zbt27h4cOHuHNH+0mLRlM6N9zDw0NfZAHaAunMmTPo0aMHgoOD0aVLF7zwwgvw8PBARkYG0tPT0aFDh3J5RUREQKlUIiYmBh4e2ulgTZs21T+vi69Umr9rOqk7xA7OgEgMaNTQlEyj0RQXQiPXftopdaWpg4QQ4TT2cYKrow2y84txJy4LbZoJv65d4uyB4sR7dbLFu27aYBMfJ7g52VZ43tfDHhwHFMiVyM4vrvSY6ihSSzoOetW8D2zFDi6QujeEMjMRxfF3Yd+iw5O/6DGsuBBQqwAAIuo6KAijC62BAwfi1KlT2LNnD5o0aYIhQ4Zg0KBB8PPzEyK/WovjOHAy417QhmBqNTgNIJLZQlRNoWUqiaT8JaFWqyESaZfyaTQavPLKKxg6dGiFr/Py8kJaWhpGjBgBd3d39OrVC127dkWbNm3Qo0ePcsc+XpQHBATg9OnTuHz5Mi5evIhff/0VX3zxBVasWIFu3bpVmqeucCubr0wmq3CcuW1PSd3CcSKIHd2gzk2HukC7TkudnQpAu5ZBZGNnzfQIIXUcx3EIbe6J3/5NQOS9dIsUWuI6vGlxdeuzAO3+ZQ3c7ZGcUYj41HyjCi2mUUOZriu0at7UQQCw8W8JZWYiiuLvmFRoqUo+cORkdhBJacBECEY3w3jvvffw22+/YceOHQgPD8fOnTvRt29fjB07Fvv370dennk7cBPr0m1ErfPPP/+gVatWAIAWLVrgwYMHaNKkif5PcnIyVq9ejYKCAhw7dgzZ2dn49ttvMW3aNPTt2xc5OdoXcXUFz+7du3H69Gl06dIF8+bNw9GjR9G5c2ecOHECnp6e8PT0xLVr18p9zdWrVyGVStG4cc28+ZGaS7dGQVPSEEOVrW2EIaFGGIQQCwjV7adloY2L9Xtp1cERrdL1WVVvNO/vrZ3hYmxDDFV2CphKAU4ig8TV2/QkBaRviGHiOi2Nbg8tmjYoGKMLLUD7iUznzp2xbNky/P7779i8eTN8fX2xdOnSKkcgSO2wd+9eHDt2DA8ePMCqVasQFRWFCRMmAACmTp2Kn376CZs2bcKDBw/w559/YsGCBcjLy4OXlxd8fHwgl8tx6tQpJCYm4vfff8fs2bMBoFxnwsdlZmbigw8+wNmzZ5GQkIALFy7g9u3baN++PQBgypQp+Oabb7B3717ExcXh6NGj2LRpE0aMGFFuCiIhhtDtpaUuKbTUObqOg7Q+ixAiPF3ThrsPsyAvVgl+PkkdHdFKzSpEUnoBRCIOrZt5VHmcv7cjACDeyIYYummDUs9G4ETCr6UzhW2jko2LE6PB1MZfS6Wt3anQEorRUwfLUqlU+P3333Hy5En89ttvAIDOnTvzkhixjpdeegm7d+/G3bt3ERQUhO3btyMoSPtC7t+/Pz755BNs3boVW7ZsgaurK3r16oU5c+bon7958yZWrlyJ/Px8+Pn5Yfjw4Th79ixu3LiB//u//6v0nNOnT4dSqcSyZcuQlpYGLy8vjBo1Cq+99hoAYPLkyZDJZPjqq6+wfPly+Pj4YOrUqZgyZYplfiikTtGPaBVkAzb+UOdopw5KXanQIoQIz8fDAQ3c7ZGSWYibMRkIDxZ2NF2/l1YdG9GKLBnNauHvCnvbqluvlxZaxo1olTbCqLkzZ6QeDSGyc4RGno/i5Aew9TOuk6V+s2Jq7S4Yowstxhj++usvHD9+HD///DNycnIQGhqKmTNnYsCAAXBzq3utQ+uTwMBAzJ8/v8pGG88//zyef/75Sp/jOA5z5szRF146kyZNAqBd7/X6669X2AJAIpFg7ty5mDt3bpV5jR07FmPHjq30OX9/f9y6dQtFRUXlHo+Ksl57fVJz6dYraPKzAPfSNVoSdyq0CCGW0baFF05fisP1e2mCF1pl12iZ0nmvprpesj4rtJK27mWZOnWwJrd21+E4EWz9WqIw+hqK4u+YUGiV7KFFmxULxuhCq1u3bsjIyEDDhg0xevRovPjiiwgICBAgNUII4Z9Et5dWySd5ujVaNHWQEGIpoc09cfpSnEU2Ltbd86BWQVOYWyemiTHG9CNa1a3PAkpHtNKyClGkUMFWZthb39JCq+Z1HCzLtlEQCqOvoTj+DvD0IKO+Vq1fo0UdB4VidKHVq1cvDB48uEIbcEIIqQ10e2lp8jO1bzxK1mpRa3dCiKXoRmFiEnOQk18MF0fhOr5xYql20+KCbKjyMupEoRWfmo/M3GJIJSIEB7hXe6yzgwxO9lLkFSqRlF6Apg2f/P0zlRLKjEQANXtECwBsdA0xHt0xesRSv0aLpg4KxuhmGB988AEVWXXUrVu3MHjwYGunQYig9CNa+dkQybMBMG1rW9pDhBBiIW5Otmjio53SduO+BUa16tg6LV23weAAd8ik1Teq4DiudPpgimHTBxUZCQDTQGTroG+gVFPZ+DYDRBJtIV0yQ8NQ+jVajq78J0YAmNh1kBBCaivdegVWXABxvvaXtdTNp86sWyCE1A66KW+WmD6ou++p60jnwSftn/U4YzsPKtNK98+q6b8bRFIb2PgGAjC+zbt+6iCNaAmGCi1CSL0isrEHJ9Fubi3JKmnfS+uzCCEWFtpcO33QEvtp1aURLbWGlSm0DNvw2djOg7WhEUZZtv4tAQBFj4xrAlbaDIMKLaFQocWT6jbkJXUD/RvXDRzH6aeCSDK1hRZtVkwIsbTWzTwh4oDE9AKkZckFPVdd2ksrJiEbBXIl7G0laO7vatDXGNt5UJEaBwCQ1ppCy/iNi5laCU2R9udBI1rCoULLTFKpdu+GwsJCK2dChFZcXAyg9N+c1F66vbTEedRxkBBiHQ52UrRopL0XCT2qJa5DI1rXS6ZatmnmCbHYsLex+hGttHxoNE/+0FShmzroXbM7DuroGmIo0x5BXVRg0NfoRrMgEkNk5yBUavWeQV0HExMTjQrasGFDk5KpjcRiMVxdXZGaqt2Lx97eXtD5vGq1Wv+Gv6q9rmpibKHjCxmbMYb8/Hykp6fD1dVVkJ8NsSzdiJbulUqFFiHEGkJbeCLqYRauR6ehT4RwoyeSOrRGS1eU6qZeGqKBuz0kYg4KpRrp2XJ4u9tXeaymWA5VyUb2Ms/aMaIlcXSFxM0HqqxkFCfchX2z9k/8mtL1Wc7gOBp3EYpBhVavXr2MKh5u375tckK1kY+P9k2artgSkkajgUqlgkQigUjE7wtDyNhCx7dE7kqlEh4eHrzHJpYneayLFBVahBBraNvcC/vP3kPkvXRBNxMuu0arNm9arFSpcetBJgDDG2EAgFgsgq+nIx6l5CE+Nb/aQkuRrh3NEju6QWzvZF7CFmTrH4T8rGQUPbpjWKFFrd0twqBCa/ny5foXZU5ODtasWYPOnTvj+eefh5eXF7Kzs3Hu3Dn8+uuvmD9/vqAJ10Qcx8HX1xfe3t5QKpWCnksulyMmJgaNGzeGnZ1drYktdHyhc1cqlbh7926t/eVEytPtpaX9i6TGt+8lhNRNQU3dIZWIkJlbhPjUfDRqIMwbe909jqkU0BTlQ2xXewqIsu7EZUGhVMPVyQaNfYz7Hvy9tYXWo9Q8hAV5V3lcbWuEoWPr3xL5N341eJ2WvtBypEJLSAYVWsOGDdP//5tvvokhQ4Zg2bJl5Y4ZNGgQPvroI5w8eRIjRozgN8taQiwWCz6tTKPRAABsbGxga2tba2ILHd9SuZO6QbeXFgCInb3AiWg6KCHE8mykYgQHuCMyOh2R99IEK7REEhlE9s7QFOZClZtRawutstMGjf3g09DOg6Xrs2pZodVIu06rOPEemFoFTlz9W3xq7W4ZRs+xunjxIp5//vlKn+vZsyf++ecfs5MihBAhlR3BkrhW/ckmIYQITTcF7nq0sPtp1YV1Wro9x0KbGz5tUEdXxD5pLy1lScfB2jaiJfX0h8jWAUxZDEVK7BOP149oUWt3QRldaLm5uSEyMrLS5/766y80aEBtkgkhNZukzNRBsQvdswgh1hNashfUjeh0qA3oiGeq2r6XVmGREncfZgEwfP+ssowd0aotrd11OE4EG7+S/bQMmD5YWmi5CplWvWfQ1MGyhg8fjs8++wxFRUXo2bMn3NzckJ6ejlOnTuHbb7/FwoULhciTEEJ4I3YsM3WQRrQIIVbUwt8V9rYS5MuVeJCQg+aNXAU5j34vrVpaaN16kAm1hqGBuz18PIxvR+7npS20svOKkV+ogKO9rMIx6oIcqAuyAQAyT3+z8rUGW/+WkN//G0Xxd+ASMbDaY0ubYThbIrV6y+hC64033kBeXh62b9+Obdu2AdC2v7a1tcVbb72FMWPG8J4kIYTwSSSzBWdjD1ZcSCNahBCrEotFaB3oicu3knH9XppghZZ+L61aOnVQtz7LmG6DZdnbSuHhYouMnCLEp+UjqEnFJki6joMS1wYQyfhf7y003TqtokdRT+wuqV+jRVMHBWV0ocVxHN59911MmzYN//77L3JycuDm5ob27dvD3r7qdpmEEFKT2AS0ReGD65D5NrN2KoSQeq5ti9JC66VeLQQ5R21fo1VaaBk/bVDH39tRW2ilVFFopZZ0HKxljTB0bBq2AERiqPMzocpNg9Sl6hkbupE7mjooLJM3HHJwcICXlxecnZ3Rtm1bKBQKPvMihBBBuTz/BnJ6zoSolnbfIoTUHbpRmpsPMqFUCdPltjav0crJL8aDxFwAQBsjNip+nL939Q0xamtrdx2R1AY2DZoCAIofRVV5HGMM6gLtz5NGtIRlUqF15MgR9OzZE0OHDsXrr7+OuLg4zJ8/HzNmzDC64NJoNNiwYQO6deuGdu3aYerUqXj06FGVx8fGxuLVV19FeHg4unfvjg0bNkClUlV6bGZmJrp27YqNGzcalRMhpO7jOA4QYHNrQggxVmMfJ7g62kChVCMqLlOQczy+aXFtcuO+tttgEx8nuDmZPqXvSQ0xanuhBQA2uumD1TTEYMWFgEb73llEa7QEZfS7jBMnTuDdd99Fp06dsG7dOv3+Qn379sX58+exefNmo+Jt3rwZe/fuxYcffoh9+/ZBo9HglVdeqbRgy8nJwZgxYyCXy/HVV19h3bp1OHnyJJYsWVJp7EWLFiEtLc3Yb5EQQgghxGI4jkNoyUjN9XvCtHkXl0wdZMoi7RvtWkT3MzF1fZZOdYUWY6x0D61aXGjZ+uvWaVVdaGkKtaNZnI09RJKKTUEIf4wutLZs2YKRI0di9erV6Nevn/7xl156CTNmzMDx48cNjqVQKLBjxw7MnDkTPXv2RFBQED755BMkJyfj9OnTFY4/dOgQCgsLsX79erRq1Qrh4eFYtmwZfvjhB8THx5c79rvvvkNsbCy8vMx7URJCCCGECC1Ut5/WPWE+IBZJbSCy0xYatW36oLmNMHR0UweTMwqgUpefoqnOy9AWoCIxpB6+Zp3HmnSFliLtITRVFNS6QktC0wYFZ3Sh9eDBA/Tt27fS59q2bYuUlBSDY925cwcFBQXo3Lmz/jFnZ2eEhITgypUrFY6Pi4tDYGAg3N1LFzCGhIQAAK5evVouxzVr1uDjjz+GTEaVOiGEEEJqNl2Th7sPsyAvrnxJhLn0Ld5rUUOM1KxCJKUXQMQBrQI9zIrl4WILW5kYag1DUnpBued0jTCkHg3BiaVmnceaJE5ukLh6A0yDooS7lR6jkWsLLZE9FVpCM7rroIeHB+7fv48uXbpUeO7+/fvw8DD8RZCcnAwA8PUt/8mBt7e3/rnHH09NTYVarYZYLAYAJCQkAAAyMrQ3DaVSiXfeeQdTpkxBq1atDM7lSRhjKCy0/lC7XC4v99/aElvo+JS7deJT7taJT7lbJz7lbp349SV3ZzsOXq62SMsuwt+3E9H+qeqbPpiSN+fgCiAOhelJgG9L3uMbypjYV28mAgAC/VzAMSUKC5VmxW/oaY+YxDzEPMqAh5NY/3hB4n0AgNjdr9r3e7XhepT4NIcqOxV5MTcAn6cqxC7KLpmeauvI63vbmnLNWMKT2ufrGF1oDRgwABs2bIC3tzd69OgBQDu3+L///sPmzZsxcGD1G6SVpfthPT7qZGNjg5ycnArHP//889i8eTNWrFiB2bNno7CwEMuWLYNEIoFSqX3hbdiwATY2Npg6daqx31q1lEolbt++zWtMc8TGxtbK2ELHp9ytE59yt058yt068Sl368SvD7n7e4iQlg2cv3IPtmrDphAak7e9koMNgJQHUSiSGLaHoLV/Lhf/0TYH8XXRGP0+rLL4DjI1AOCfWw/gyJWO7Nnf/w82ALI0Nkgy4Dw1+XqUiZzgACA7+l/Eu4VUeD4r6SHsAOQWq5EswHtba18zlmLIrDmjC623334bd+/exdtvvw1RSceucePGobCwEOHh4XjrrbcMjmVrq+0co1Ao9P8PAMXFxbCzs6twfEBAANavX48lS5Zgz549sLe3x4wZMxAdHQ0nJydcvnwZ3377LQ4dOqQf8eKLVCpF8+bNeY1pCrlcjtjYWAQEBFT6M6qpsYWOT7lbJz7lbp34lLt14lPu1olfn3LPVCbhn/v/ISmbQ3BwMK+xASA/Jwr58f/C3U4MFwHiG8rQ2IwxPDr6GwCgZ8RTCA6suPeVsfGDU2JwI/Y+VCKHcj/j9L/3QAXAN7g9bJtV/bOpDdej0tMBGbd+giw3CUEtnwInEpeL7WIjhgKAe8MmaPKE68DSuVsjtimio6MNOs7oQksmk+HLL7/ExYsX8ddffyE7OxtOTk6IiIhAjx49DBpG09FNGUxNTUXjxqUdXlJTU9GyZeVD2r169UKvXr2QmpoKV1dXqFQqrFy5Eo0aNdI3yxg8eLD+eLlcjq1bt+LUqVNGNep4HMdxNWpDZjs7O8HyETK20PEpd+vEp9ytE59yt058yt068etD7uGt/ID9/yE2OQ9KjRgujja8xQYAtYcP8gGgMMfgr7Hmz+VRSh6y8hSQSkRo29IXNlLjPkSvLH6gnzuA+0jOKNI/xzRqqDO0UxSd/FtAasD3W5OvR9aoBTJt7MGKCyHJT4ONT2C55zmFdrqgraunIN9DTXgtCc3QesfoQuvw4cPo0aMHunTpUmGdVlpaGg4fPmzwtL2goCA4Ojri0qVL+kIrNzcXt27dwtixYyscf/XqVaxfvx47d+6Et7d2t+sTJ07Azs4OYWFhaNWqFV5//fVyXzNu3Dj069cPkyZNMvZbJYQQQgixGDcnWzT2ccLD5Dz8dz8DXdo25DW+2Ll2NcOILOk2GBzgbnSRVZXSFu95+nU2yqwUMLUSnNRG20iiluNEYtj6PQV5zL8oenSnQqGlKdQuzxE7uFohu/rF6K6DCxYsqHJD4du3b2PDhg0Gx5LJZBg7dizWrFmDs2fP4s6dO5g1axZ8fHzQr18/qNVqpKWloaioCAAQGBiIqKgorFq1Co8ePcKZM2ewbNkyvPbaa3B0dISHhweaNGlS7o9EIoGLiwv8/PyM/VYJIYQQQiyqrYBt3nVdB9W1pL379Wh+9s8qq6GXA0QcUFCkQlZeMQBAqduo2LMROK5ubGSv30+rko2LdV0HxQ60WbHQDBrRevXVV3H/vrYbC2MMb775ZqULwDIyMspNATTEzJkzoVKpsGjRIhQVFaFjx47Yvn07pFIp4uPj0bt3b6xYsQLDhg2Du7s7tmzZgpUrV2LgwIHw8vLC9OnTMXHiRKPOSQghhBBSE7Vt7omjF2IELbQ0xYXQFMshsrH+WpeqqDUMkfpCq/oOjMaQSsRo4OGApPQCxKfmwd3ZtrS1ey3eqPhxto10hVZUhed0+2iJqb274AwqtF5//XXs378fgHbT4JCQkHJ7WQGASCSCs7Mzhg0bZlQCYrEYc+fOxdy5cys85+/vj6io8hdIWFgYvv/+e4Pjnzt3zqh8CCGEEEKspXUzT4g4IDG9AGlZcni58VcMiWzsILKxh6a4EKq8DMhs/HmLzbeYhGwUyJWwt5Wgub8rr7H9vR1LCq18hDb3gkI3ouXdiNfzWJNNwxYAJ4I6Nx2q3HRInEuKVY1KuzEzaOqgJRhUaIWFhSEsLEz/92nTpqFRo7pzMRJCCCGE1AQOdlK0aOSGqIdZiIxOQ++O/I6yiJ09oEkrhCo3AzLPmltoRd7Tjma1DvSEWMzvdD5/bydcuZWC+NR8ACgttOrQiJZIZgtZg6ZQJN9H0aM7cGzVFUBpIwyIxBDZOlgxw/rB6Ct3xYoVuHHjBpYsWaJ/7O+//8bLL79Mo0eEEEIIIWYKLZkqJ+g6rRreEEP3vfM5bVBH3xAjJQ8alQLKzCQAdavQAgBbf20H77LrtES60Sx7F6M6hRPTGF1oHT58GLNnz0Z2drb+MVdXV/16qTNnzvCZHyGEEEJIvdK2ua4hRjoYY7zG1hVaqhrcEEOpUuPmA+1GxXw2wtDRF1pp+VBmJAJMA5GtI8SObryfy5r067QelRZanKIAACB2oPVZlmB0obV9+3ZMmjSpXHfBwMBAfP7555gwYQI2b97Ma4KEEEIIIfVJUFN3SCUiZOYW6ae38UW3Vqcmt3i/E5cFhVINV0cbNPZx4j2+v7c2ZlqWHAWJDwAAMu/GdW6ER9d5UJEaB02xHAAgokLLoowutB4+fIgePXpU+lz37t0RExNjdlKEEEIIIfWVjVSM4ABt0zFd5z2+iJ21cWvyiJZufVZoc09Bih9nBxmcHbTds7Meartq17VpgwAgcfbQFtZMg+LEewBK12hRoWUZRhdaXl5eiIyMrPS5O3fuwM2tbg27EkIIIYRYmlDrtGrDGi3d9xwqwLRBHd30waKUOACAzKtuNnmzaVR+Py1RccmIFrV2twijC62BAwfi888/xzfffIOUlBQolUqkpKRg37592LhxIwYPHixEnoQQQggh9YZubdKN6HSoNfyt05I41+w1WvJiFe4+zAIgTCMMHd30QS4nEUDd2kOrrMc3LqY1WpZlUHv3st58803ExMRg2bJl+Oijj/SPM8bQv39/zJgxg9cECSGEEELqmxb+rrC3lSBfrsSDhBw0b+TKS1z9psVF+dAoiiCS2fISly83YzKg1jA0cLeHj4dw7cf9vR1hAwVsFdkA6ubUQaBsoXUXzhoNRDR10KKMLrSkUik2bNiAu3fv4tq1a8jJyYGTkxM6dOiAoKAgIXIkhBBCCKlXxGIRWgd64vKtZFy/l8ZbocXZ2IOT2YIpiqDKy4TMoyEvcflS2tZduGmDgLbQ8hHnAADETu4Q2zkKej5rkXk3BiezA1PIocp4VDqiRVMHLcLoQkvnqaeewlNPPcVnLoQQQgghpERoC22hFRmdjpd6teAlJsdxkDh5QJmRoF2nVcMKrbKNMITk7+2EhhLtFEWpZ90czQIATiSGrd9TkD+4DmXivdI1Wg6u1k2snjCo0Bo/fjzef/99NGvWDOPHj6/2WI7j8NVXX/GSHCGEEEJIfaUb1bn5IANKlQZSidFL6yslcfaEMiOhxq3TyskvRkyidpQpVMD1WQDg7W6PhhLtuVTOvoKey9ps/FtC/uA6FAl3qeughRn0ii27WR5jrNo/Go1GsGQJIYQQQuqLJj5OcHW0QbFCjai4TN7iinWbFtewzoM37mtHs5r4OMHNSdi1Y2IRhwDbXABAtkTYos7adOu0iuNugGPa9+lie2drplRvGDSi9fXXX1f6/4QQQgghRBgcxyG0uSd++zcB1++lo3UzfgoCScleWuoaNqJ1vWTaoNDrs3S8uSyAAUkqV4Ra5IzWYev3FMCJwIq0m19zNvbgJFIrZ1U/8DMGTQghhBBCeKfbSyoymr/9tHSdB1W5/G6GbK5I3f5ZAq/PAgB1QQ5sNYXQMOB+gXDdDWsCkY0dZN5NSv9Oo1kWY9CIVq9evYzamfvs2bMmJ0QIIYQQQrR0e0lFxWVBXqyCnY3Jfcz09Htp5fE3HdFcaVlyJKYXQMSBt5G76ijSHgIAMjROeJheLPj5rM3WvyUUKQ8AACI7KrQsxaBXa0REhL7Q0mg0OH78OJycnNCjRw94eXkhOzsbFy9eRGZmJkaMGCFowoQQQggh9YWPhwO83e2RmlmImzEZCA9uYHbMmrhGS9fWvUUjNzjYCT+tTVdoJaldEZ+aJ/j5rM22URByr50CAIiotbvFGFRorVy5Uv//a9asQWhoKLZv3w47Ozv940qlEm+88QYKCwv5z5IQQgghpJ5q29wTP19+iOv30ngptHQjWprCXGhUCogkMrNjmut6ydRIobsN6ihSSwutHLkCuQUKODtY/+cgFF1DDICmDlqS0Wu09u/fj6lTp5YrsgDtRsbjxo3DiRMneEuOEEIIIaS+a6tfp8XPmiqRrSO4kuJKXQOmDzLG9PtntW1umUYYirRHAIB8G28AQEJqvkXOay0SFy+IHLVNUKjQshyTmmHk5ORU+nhiYiJsbGzMSogQQgghhJTSNYd4kJiD3AKF2fE4joPEWRuzJuylFZ+aj8zcIkglIgQ1dRf8fIwx/dRBkbt/SQ51f/qgTeNWAACJq/mjosQwRhdavXr1wpo1a3Dx4kX9Y4wx/Pzzz/j0008xYMAAXhMkhBBCCKnP3Jxt0djHCYwBN3ga1RI715x1Wrpug8EB7rCRigU/nzo3HUwhB0QSOPk2AqAt9uo6px6jkR82HLZPPW3tVOoNo1vXLFiwANHR0ZgyZQpkMhlcXFyQlZUFtVqNLl26YO7cuULkSQghhBBSb7Vt4YWHyXm4fi8NXdo2NDuersV7TdhL63q0ZffP0q3Pknk2hF8DVwAP60WhJbJ1hNK7BTix+Z0riWGM/kk7Ozvj+++/x/nz53H16lXk5ubCzc0NnTp1QufOnYXIkRBCCCGkXmvb3BNHL8Tou/OZS+KknaJn7b201BqmX3vW1lKNMEqmDUq9GsPf2xFA/Zg6SCzPpJKW4zj07NkTPXv2RHFxMaRSKUQi2vuYEEIIIUQIrZp5QsQBiekFSMuSw8HMJfGSGjJ18EFCDgrkStjbStDc39Ui59QVWrIyhVZyZiGUKjWkEuGnLpL6w6TqKCYmBm+//TYiIiLQvn173L59G0uXLsXXX3/Nd36EEEIIIfWeo50UzRu5AgAio80f1dLvpZVr3a6DuhG61oGeEIst86G9ruOgzKsx3J1tYWcjgUbDkJReYJHzk/rD6Cv69u3bePnll3Hz5k0MHDgQjDEAgFgsxvLly3Ho0CHekySEEEIIqe90a5j4mD6oG9FSW3lES/e9WGraINOooUyPBwDIvBqB47gy0wfr/jotYllGF1qrVq1C69atcfLkSSxcuFBfaC1atAgvv/wydu/ezXuShBBCCCH1nW6Pqev30vXvv0yla++uLsgGUyvNzs0USpUGNx9oR9Qs1QhDmZkEplaCk9pC4qrdQ4sKLSIUowutf//9FxMnToREIgHHceWeGzBgAGJjY/nKjRBCCCGElAhq6g6pRITM3CIkpReaFUtk5wROLAUAqPKy+EjPaPfic6BQquHqaIPGPk4WOWfptMFG4Djt22B/b+25qSEG4ZvRhZaNjQ2KiooqfS47OxsymczspAghhBBCSHk2UjGCA7TdAm/EmLe2iuM4/V5a1po++N997fcQ2tyzwof3QilthNFI/xiNaBGhGF1odenSBRs2bEBycrL+MY7jUFBQgB07duCZZ57hNUFCCCGEEKIVWrKW6T8zCy2gdC8tlZX20tJ9D6EWmjYIAMoyrd11yhZa5k7JJKQsowutuXPnorCwEP3798eYMWPAcRxWrlyJ/v37IykpCbNnzxYiT0IIIYSQek+3lunmg0xoNOau09IVWpbfS6tYqUF0fA4AyzXCAMq3dtfx9XSASMRBXqxCZm7ls7YIMYXRhZavry+OHDmCCRMmgDGGxo0bo7CwEAMHDsTBgwfRqFGjJwchhBBCCCFGa+HvCjsbCQrkKiRnm9fEQqzbtNgKUwcfphVDrWFo4G4PHw8Hi5xTo1JAmamdkSXzLi20pBIxfNztAdD0QcIvozcs3rx5M5577jnMmjVLiHwIIYQQQkgVxGIRWjfzwJVbKXiQXGxWLGtOHYwpyT20ueVGs5Tp8QDTQGTnBLGDa7nnGjVwQmJ6AeJT8izWAZHUfUaPaG3duhXx8fFC5EIIIYQQQp5AVwjEpJg3za10Ly3Lb1r8IEVbaFmyqCk7bfDx5hvUEIMIwehCq3nz5njw4IEQuRBCCCGEkCfQFScPUxVQqTQmx9HtpWXpEa3cAgWSs7TTHkMtuj6rtLX746jQIkIweurgs88+i3Xr1uHChQto2bIl7O3tyz3PcRzefPNN3hIkhBBCCCGlmvg4wdlBitwCJY78Hgv/Bi4mxREVF6MRAFV+Fn65EguIxPrnFMUKJCQWIL04CTIbfrfuuRenLewaeTvCzcmW19jVUaRWbIShQ3tpESEYXWht2rQJAHDx4kVcvHixwvNUaBFCCCGECIfjOLRq6o4//0vB92fvmx4HDGvcRJBwGuz4/g9kayprSiHcZsatA90Ei10ZXWv3so0wdPxKRrTSc4pQWKSEva3UormRusnoQuvOnTtC5EEIIYQQQgw0rEdTZGblwMbWHiKx+MlfUAV5piOcNLmIaCJFqrR0vZRGrUZ+QQEcHRzMil8ZjVoNpUKOAc804TVutecsKtC3sZdWMqLlZC+Dq6MNsvOLkZhWgOaNXC2WG6m7jC60yoqNjUVOTg48PDzg7+/PV06EEEIIIaQajX2cMKqHJ4KDgyss4zBG4u4TKHqUi0m9GsIx5Bn944WFhbh9+7bZ8Suji+3tZsdr3Ooo0rXrs8ROHhDbVt5O3s/bEdn5xYhPzaNCi/DCpEJr37592Lx5M9LS0vSPNWzYEO+88w4GDBjAW3KEEEIIIUQ41txLy5KqW5+l4+/tiJsxGdQQg/DG6ELr22+/xdKlS9G7d2/069cPHh4eSE9Px6lTp/DOO+9AJpOhT58+QuRKCCGEEEJ4pG/xboW9tCxJ33HQu2LHQZ3ShhhUaBF+GF1o7dq1C6NHj8aSJUvKPT5kyBAsWbIEn332GRVahBBCCCG1gL7Fe10f0UozbEQLoM6DhD9G76OVnJyM3r17V/rcc889h5iYGLOTIoQQQgghwpM4aUe0VLmW37TYkkoLraobcOgKrYS0Aqg1zCJ5kbrN6EKrTZs2uHDhQqXP/fPPP2jZsqXZSRFCCCGEEOGJdYVWHR7RUuVnQ1OYC4CD1NOvyuO83Owhk4igUmuQmllouQRJnWX01ME33ngDs2fPRkFBAV588UU0aNAAWVlZOHv2LHbu3ImFCxfiypUr+uM7duzIa8KEEEIIIYQf+jVaeZlgGjU4Eb+t3GsC3f5ZUncfiKQ2VR4nFnFo6OWI2KRcxKfmwdez8u6EhBjK6EJrypQpAID9+/fjwIED+scZ0w6xLl26VP93juNw+/ZtPvIkhBBCCCE8Ezu4AJwIYBqo87P1hVddops2WNn+WY/z99YVWvnoGCJ0ZqSuM7rQ2r17txB5EEIIIYQQC+NEYoid3KHOTYcqL6OOFlolHQe9qu44qEOdBwmfjC60IiIihMiDEEIIIYRYgcTJQ19o1UWGdBzUoc6DhE8GNcMYO3Ys7ty5Y1TgGzduYNSoUSYlRQghhBBCLEPirN20uC7upcWYprTQ8q6646BOowY0okX4Y9CI1rhx4zBlyhSEhoZi0KBBePbZZ2FnZ1fhuPz8fFy4cAHfffcdbt++jffff5/3hAkhhBBCCH/q8l5aqpx0MEURIJZA6ubzxOMbemkbYOQWKJCTXwwXx6qbZxDyJAYVWs899xw6duyIzZs347333oNKpULz5s3h7+8POzs75ObmIjk5Gffu3YNEIsHw4cOxZs0aeHp6Cp0/IYQQQggxg77Fex0c0dKPZnn4gRM/+W2vrUwCbzc7pGbJEZ+aT4UWMYvBa7Tc3d2xaNEiTJs2DadPn8alS5fw6NEj5OXlwc3NDc2aNcP48ePx7LPPws3NTcicCSGEEEIIT8q2eK9rlEasz9Lx93bSF1qtAutecxBiOUY3w3B3d8fIkSMxcuRIIfIhhBBCCCEWJKnTI1rajoOGtHbX8fd2xN9RqdQQg5jNoGYYhBBCCCGkbtKNaKnyMsGYxsrZ8EuRGgcAkHkbV2gB1BCDmI8KLUIIIYSQekzs4KrdtFijgrogx9rp8IapVVBkJAAwfuogACRQoUXMRIUWIYQQQkg9xokl2mILdavFuzo7BVCrwMlsIXExvEGbbkQrJbMACqVaqPRIPUCFFiGEEEJIPVc6fbDuFFqqjHgAgMyzETjO8Le8rk42cLCVQMOApPQCodIj9QAVWoQQQggh9Zy+0KpDI1rK9JJCy4hpgwDAcZx++iCt0yLmMLrrIADk5eXhr7/+QmFhIRhjFZ4fMmSIuXkRQgghhBAL0e+lVRdHtIxohKHj5+2IqIdZ1HmQmMXoQuvChQuYOXMmioqKKi2yOI6jQosQQgghpBapi3tpqUwc0QKo8yDhh9GF1tq1axEYGIgFCxagQYMGEIlo9iEhhBBCSG1WupdWupUz4YlaqW2GAeP20NIpnTpII1rEdEYXWvfv38fmzZsRHh4uRD6EEEIIIcTC6toaLXF+BgAGkb0zxA4uRn992REtxhg4juM5Q1IfGD0c1bBhQ+Tn0zAqIYQQQkhdUXaNVmVLQ2obcX4aAEDm1cikIsnX0wFiEYcihRoZOUV8p0fqCaMLrddeew2fffYZ4uPjhciHEEIIIYRYmMTJDQAHqFXQFOYKei6mLIY09R40cuGm5ZUWWsZPGwQAiVgEHw8HADR9kJjO6KmDR48eRUpKCvr27Qt3d3fY2tqWe57jOJw5c4a3BAkhhBBCiLA4sRRiBxeoC7K1nQedfQQ5j0ZRhMwfVsIxKRqp/x5EXkAoHFt1gcNTERDZOvB2HnFeKgDTCy1AO30wIS0f8an5aPeUN1+pkXrE6ELLx8cHPj7CvPgIIYQQQoh1SJw9tIVWbgY4AQotplYi5YePoUyKBuNE4DRqyGP+gTzmH6SJJbBv1h6OIV1g3yIcIpmdWefSj2iZ0Npdp1EDJ1y6mWzRzoMJaQXIl6stdj4iLKMLrRUrVgiRByGEEEIIsSKxkweQdB/qvAzTNlqtBtOokfrjRshj/gUkMuR1GInAoNZQP7iG/FsXoUyPR+HdKyi8ewWcRAb7Fh3gGNIVds3aQyS1MepcmqICiIq00/1kno1MzlnXEONRimWmDl69nYIPtv8FGymH1QGBaGpvb5HzEuGY/Dr67bffcPnyZeTm5sLNzQ3h4eHo1q0bn7kRQgghhBALKdt5kM9CizGGjJ+2o+DWRUAkgdugt5All0Li7gtn//+Da9fhUKY9RP6ti8i/dRGqrGQU3P4TBbf/BCezhcNTEdqiKzAUnFj6xPOpMhIAACInD7OmI1pyL63kjAKs3XMNjAFFCoZ130Zizds9YCMVC35uIhyjX0cKhQLTpk3D77//DrFYDDc3N2RlZWHbtm3o1KkTtm7dCplMJkSuhBBCCCFEIJIynQf5lHV+H3L//gkAB+8XZ0IUEArcvq1/nuM4yLybwN27Cdx6jIIiOQb5t35H/q0/oM5NR/5/vyH/v98gsnWEQ8un4RDSBXYBrcGJKi9CVBnahm0SDz+z8vYr2UsrM7cIhUVK2Ns+ucgzRbFSjRW7riBfrkRgQyckZxQgNjkPmw9cx9sj21Nr+VrM6K6DGzduxLVr17B69WpERkbi999/x/Xr17FixQr8+++/+Pzzz4XIkxBCCCGECEgswF5aOZePIfviAQCAZ/9X4BjSpdrjOY6DjW8zePSegMbTP0fDCR/BueMAiB1coSnKR971s0j+9gPErX8F6Se3Qf7wJhjTlIuhK7SkHv5m5e5oJ4Wbk3baolCjWowxbD5wHTGJOXBxlGHO6HYY3tUdHAecu/oIp/6MFeS8xDKMHtE6duwYpk+fjsGDB5cGkUgwZMgQZGRk4Ntvv8Vbb73Fa5KEEEIIIURYuhEtNU8jWnmRvyLj550AALceo+Dcob9RX89xItj6B8HWPwgefSai6NFt5N+8iII7f0JTmIvcv39C7t8/QezkDofgZ+AY0gU2DVtAmf5I+/2YsT5Lx9/bCVl5xYhPzcdTjd3Mjve4U3/F4dzVRxBxwNyx4fBwsUXTBrYY3a8F9vx0D9sO30BTPxcENXHn/dxEeEYXWpmZmQgJCan0uZCQEKSkpJidFCGEEEIIsayya7TM3bS44O4VpB37DADgHDEQrl1eMiseJxLDrklr2DVpDc/npkAeewP5ty6iMOoS1HmZyL18DLmXj0Hi4g11Qbb2+/E0b0QL0K7TunE/XZC9tKLiMrHtUCQAYPyAELRt4YXCwkIAwKAuTfAgKR9/RCZh5VdX8OmsnnB1Mq4pCLE+o6cONm7cGNeuXav0uStXrsDX19fspAghhBBCiGWJnbSjJkylACsuMDmOPO4mUg+uBZgGjm16wqPPBF7XGXElreC9B01Hk7d3oMHw+XBs1Q2c1BaqnFRt/uAgcW9o9rmEaoiRnVeMlV9dgUrN0LmNL4Y927zc8xzH4a0R7eHv7YiMnCJ8/M1VqNWaKqKRmsroEa2RI0di5cqVsLW1xQsvvABPT0+kp6fj2LFj+OKLLzB9+nQh8iSEEEIIIQISSWQQ2TtDU5gLdV6mSTGKk2OQ/P0KMLUS9i3C4fXCG+A4oz/XNxgnkcLhqY5weKojNMpiFEb/jdxbfyKTcwAnMb85m39JQww+Cy21WoOPv7mK9Jwi+Hk5Vtnwwt5WioUTI/DO+vOIjE7H7hO3MWlQK97yIMIzutAaNWoUbt26hTVr1mDt2rX6xxljGDp0KF599VVeEySEEEIIIZYhcfKAojAXmvxMAMZNVVNkJCLp2w/BFHLYNg6B99DZ4MR878hVNZHUBo7BnSFq0hZJZboamkM3opWUns/biNLXJ28jMjodtjIxFk7sWG03w0YNnDBzRHus2n0VB3+NxlNN3NAl1PyROmIZRl/9IpEIH330ESZNmqTfR8vFxQURERFo1qyZSUloNBps2rQJ+/fvR15eHjp27IglS5agUaPKFzHGxsZi+fLl+Pvvv2Fvb4+XX34Z06ZNg0Si/XaKiorw2Wef4fjx48jKykLTpk3x5ptvonfv3iblRwghhBBSH0icPaBIeaAd0ZIavhxElZuB5L1LoSnMhaxBU/gMn2/0RsM1kaerHWRSMRRKNVIyC+HqYN7o3B+Rifjhl2gAwFsj26Oxj/MTv6ZrWz/c7ZmNQ79GY/2+v9G4gRMaNXAyKw9iGSZfLc2bN8fo0aPx+uuvY9SoUSYXWQCwefNm7N27Fx9++CH27dsHjUaDV155BQqFosKxOTk5GDNmDORyOb766iusW7cOJ0+exJIlS/THLFu2DEePHsX777+Pw4cPo0+fPpg+fTouXbpkco6EEEIIIXWdvvNgvuFTB9WFuUj69gOoctMhdfeF76jFZm0UXJOIRBz8vfhZp/UoJQ+f7vsHADCkRzN0bWv4Pl8TBgSjTTNPyIvVWL7rMgqLlGblQizDoEKrd+/euHPnDgCgV69e6N27d5V/+vTpY1QCCoUCO3bswMyZM9GzZ08EBQXhk08+QXJyMk6fPl3h+EOHDqGwsBDr169Hq1atEB4ejmXLluGHH35AfHw85HI5Dh8+jNmzZ6NHjx5o0qQJpk2bhoiICPzwww9G5UYIIYQQUp/o9tLSGLhGS1MsR/K+j6BMj4fYyR0+o5dA7OAiZIoWV9oQw/TOg/JiFVZ8dRnyYhVaN/PAxBcq7+BdFbFYhLnjOsDDxRbxqfnY8N2/ZneGJMIzaOpgREQEHBwc9P/PZ+eYO3fuoKCgAJ07d9Y/5uzsjJCQEFy5cgUDBw4sd3xcXBwCAwPh7l66n4Cu3fzVq1fRv39/bNmyBa1bty73dSKRCLm5ubzlTQghhBBS1xgzosVUSqQcWIXipGiI7BzhO2oJpC7eQqdoceZ2HmSMYcN3/+BRSj7cnW0xb1w4xGLjJ5W5Odli/viOWLD5d1yMTMTh8/cxtGfzJ38hsRqDCq0VK1bo/3/lypXVHqtWq41KIDk5GQAqtIX39vbWP/f446mpqVCr1RCLxQCAhIQEAEBGRgZsbW3RtWvXcl8TGRmJv/76C4sWLTIqt7IYY/q9DaxJLpeX+29tiS10fMrdOvEpd+vEp9ytE59yt058yt2ysVU22qJClZtRbXym0SD7+CYUx94AJ7WB65A5UDl4QGXAe6Xa9m/q5artXhiXnGNS7GMX4/D79USIxRxmjWgDG7GmyveUT4rf2NsW459viR3H7mDX8Vvw97JFq6aGbWZc237uloptCsaYQQNPHDNy3LF379747LPPEBQUVOG5yMhITJ061ai1UEeOHMG8efNw+/ZtiESl1f28efOQmpqKXbt2lTs+NjYWgwYNwogRIzB79mwUFhZizpw5uHLlCmbMmIHXX3+93PExMTGYMGECGjZsiG+++QZSadWdXapy48aNSteLEUIIIYTUJaKCTLhc2AImliK7zxygsjeTjMH+5gnYxF8H48TI7/B/UHk2tXyyFpKcpcCWk6mwlXF496WGRs3sik0pxlfn0sAYMCDcFRFPOZqdD2MMh/7MQmRsIextRHjteW+42FuuuyPRkslkaNOmTbXHGPSvcuzYMahUKgDa0aPTp0/r12yV9eeff0KpNG5xnq2tLQDtWi3d/wNAcXEx7OzsKhwfEBCA9evXY8mSJdizZw/s7e0xY8YMREdHw8mpfAeWv//+G9OmTYOPjw+2bNliUpGlI5VK0by59Ydn5XI5YmNjERAQUOnPp6bGFjo+5W6d+JS7deJT7taJT7lbJz7lbtnYTFmMlAsAp1YCqmIEtAiqED/vwncoiL8OcBzcXngTvi061ojchYrfTKnG1lPnUKRg8PD2R2ZagkGxM3OL8MmRS2AM6NbWF+MHt3pikWZo7s2aq7H4i8uIS87HsWty/G9yOCSS6qcj1rafu6VimyI6Otqg4wwqtG7cuIGvvvoKgHan6s2bN1d57KRJkww6sY5uymBqaioaN26sfzw1NRUtW7as9Gt69eqFXr16ITU1Fa6urlCpVFi5cmW5dvCnT5/GnDlz0LZtW2zevLlCEWYsjuNgb29vVgw+2dnZCZaPkLGFjk+5Wyc+5W6d+JS7deJT7taJT7lbKrY9RHaO0MjzISrKqxA/+8/DKLh6DADg+fzrcG7bw+Qz1ZZ/U3sA3m72SMksRGa+2qDYSpUG67+/ipwCBQJ8nTFzZBhsZYaPOj0pvr098N6kTpj16Xnce5SDPT/fxxsvteUltrlq1/VuGkNHNQ36F3/nnXcwfvx4MMbQp08fbNq0CcHBweWOEYvFcHR0hKOjcUOiQUFBcHR0xKVLl/SFVm5uLm7duoWxY8dWOP7q1atYv349du7cCW9v7YLLEydOwM7ODmFhYQCAc+fOYdasWejduzfWrFkDmcz8ncEJIYQQQuoDiZMHFPJ8iIrKNxHL/fcMMs99DQBw7zUOzu2N6zRdm/l7OyIlsxAJaQXwM+Ct7o4f/8OduCw42EqwcGKEUUWWoXw9HfDO6DB8sP0STvwRi5ZN3NArvPGTv5BYjEH/6jKZDH5+2l7/Z8+ehbe3t1nT8B6PPXbsWKxZswbu7u7w8/PDxx9/DB8fH/Tr1w9qtRqZmZlwcnKCra0tAgMDERUVhVWrVmH8+PGIiorCsmXL8Nprr8HR0RE5OTl499130apVK7z33nvIycnRn0sqlcLV1ZWXvAkhhBBC6iKxkweQGgdRUWk784I7fyH9xFYAgEunF+HaeYiVsrMOf28nXLuTisT0Jxdav1x7hGMXHwAAZo/pAF9P4fYU6xjig1H9WuLb01H4bP91BPi6INCvbrXXr82MLq/9/PwQGRmJS5cuQaFQ6Hv467ryXbt2Dd9//71RMWfOnAmVSoVFixahqKgIHTt2xPbt2yGVShEfH4/evXtjxYoVGDZsGNzd3bFlyxasXLkSAwcOhJeXF6ZPn46JEycCAH777Tfk5ubi+vXr6N69e7nzRERE4Ouvvzb2WyaEEEIIqTckJXtp6Ua05A8ikXL4E4Bp4NS2N9x7jbNmelaha/GekFaAjgFVrxF6kJiDTfuvAwBG9m2JiBAfwXMb2bcl7j7MwrU7qVjx1WV88nYPONrTbK6awOhCa8+ePVi2bFmlm6SJRKIKrdUNIRaLMXfuXMydO7fCc/7+/oiKiir3WFhYWJXF3KBBgzBo0CCjcyCEEEIIIaV7aYmK86BMjkHmD6sAtQr2LZ+G54DXeN1PtbbQFVqJ6YUAKi+08uVKrNh1BQqlGmFB3hjZr/JeA3wTiTi8M6YDZn1yHskZhVi7928snvw0RKL69+9U0xi9W9o333yD7t2749KlS5g8eTL+7//+D//++y/Wr18PGxsbDB48WIg8CSGEEEKIBehGtCTZCcg89DGYogh2AW3QYMgscCKxlbOzDn9vbVO1tGw5lKqKgw0aDcO6vdeQlFEAb3d7zBnTAWILFjpO9jIsmNARMokIV2+n4Lszdy12blI1owut+Ph4jB49Gi4uLmjdujWuXbsGW1tbPPfcc3j11Vexe/duIfIkhBBCCCEWIC4ptMT56WBF+bDxbY4GL78LTsLP+vzayMVRBkc7KRgDMvIqbmW0/+xdXLmVAplEhAUTOsLJClP3mvm7YtrL2s6D356+g6u3UyyeAynP6EJLKpXq97tq0qQJ4uLi9HtndejQAbGxsbwmSAghhBBCLEc3dRAAxO4N4TPyPYhsrL93kTVxHKefPpieqyr33LU7Kdjzk3Z/2Tdeaovm/q6WTk+vd8fGeP6ZADAGrNlzDckZBVbLhZhQaAUHB+OXX34BADRt2hQajQbXr2sX/SUnJ/ObHSGEEEIIsSipWwOInb2gtneD+7B3IbZ3tnZKNYJu+mDZQis5owBrvrkGxoD+nQPQJ8L67dWnvtgaLRu7oaBkzVixUm3tlOotowutSZMmYdeuXVi4cCHs7e3Ru3dvzJs3DytXrsSqVavQoUMHIfIkhBBCCCEWwIml8Jy4GrldX4XYyd3a6dQYpSNa2plcxUo1Vu6+gny5Ek81dsWrQ1pbMz09qUSM+RM6wsVRhpjEHGw+cL3SJnZEeEYXWn369MGWLVvQrFkzAMAHH3yAgIAA7Nu3D4GBgVi8eDHvSRJCCCGEEMvhxBKgnja+qErZqYOMMWz5IRL343Pg7CDD/PERkEpqzs/L09UO88aFQ8QB564+wqk/Y62dUr1k0jbVPXv2RM+ePQEAbm5u2LFjB585EUIIIYQQUqP4N9BOHczIVeHM1QScufIQIg6YNzYcXm41bw1baHMvTHghBDuP3cK2wzfg62Fj7ZTqHYMKrStXrhgVtGPHjiYlQwghhBBCSE3UwN0eYjEHpZphxzFt84txA0LQ9ikvK2dWtaE9myPqYRb+iEzCJ/siMbkPTQW1JIMKrXHjxlXYnI4xpn9M9/+6/96+fZv/TAkhhBBCCLESiVgEH3d7JKQVQKNh6NzGFy8929zaaVWL4zi8NaI9HibnIT41H/t/z4BamgIbG/5HtxSKYkio8UY5BhVatDcWIYQQQgip7/y8HJCQVgBfD3u8PbJ9hYGImsjeVoqFEyMw+9PziEtV4JPvIgU7l4OtCB83bQ57e3vBzlGbGFRoRUREVPlccXExZDJZrbjQCCGEEEIIMdXgrgEoLMjDlCHtYG9bezZwbtTACXPHtMPek/9BamMHsZj/xh1J6fnIzC3Gun2RWD2jW41qDmItJjXDiImJwYYNG/DHH38gPz8f+/fvx4EDBxAYGIhx48bxnSMhhBBCCCFW16KRC17u4oGGng7WTsVorQPdMa6XF4KDgwUZcXoQn455m/5AdHwOvjjyH6a91Jb3c9Q2Rrd3v337Nl5++WXcvHkTgwYN0vflF4vFWL58OQ4dOsR7koQQQgghhJCaq4G7PYY94w6OA07+EYuzVx5aOyWrM7rQWrVqFVq3bo2TJ09iwYIF+kJr0aJFePnll2k9FyGEEEIIIfXQU352eKlnIABg84HriEnIsXJG1mV0ofXvv/9i4sSJkEgkFdZlDRgwALGxsXzlRgghhBBCCKlFXuoZiPDgBlCoNFi+6zLyCxXWTslqjC60bGxsUFRUVOlz2dnZkMlkZidFCCGEEEIIqX1EIg6zR4ehgbs9UjILsXbv39BomLXTsgqjC60uXbpgw4YNSE5O1j/GcRwKCgqwY8cOPPPMM7wmSAghhBBCCKk9nOxlWDgxAjKJCFdvp+C7n6OsnZJVGF1ozZ07F4WFhejfvz/GjBkDjuOwcuVK9O/fH0lJSZg9e7YQeRJCCCGEEEJqiUA/F7w5XNt58Nufo3D1doqVM7I8owstX19fHDlyBBMmTABjDI0bN0ZhYSEGDhyIgwcPolGjRkLkSQghhBBCCKlFeoU3xvPPBIAxYM2ea0jOKLB2ShZl9D5amzdvxnPPPYdZs2YJkQ8hhBBCCCGkjpj6YhvEJOQgKi4Ly3ddxuoZ3WArM2kr31rH6BGtrVu3Ij4+XohcCCGEEEIIIXWIVCLC/PEd4eIow4PEXHz+Q6R+e6i6zuhCq3nz5njw4IEQuRBCCCGEEELqGE9XO8wbFw4RB5y7+gin/oy1dkoWYfS43bPPPot169bhwoULaNmyJezt7cs9z3Ec3nzzTd4SJIQQQgghhNRuoc29MOGFEOw8dgvbDt9AUz8XBDVxt3ZagjK60Nq0aRMA4OLFi7h48WKF56nQIoQQQgghhDxuaM/miHqYhT8ik7Dyqyv4dFZPuDrZWDstwRhdaN26dQsikdEzDgkhhBBCCCH1GMdxeGtEezxMzkN8aj4+/uYqPni1M8TiullbGP1dDR48GL/88osQuRBCCCGEEELqMHtbKRZOjICdjRiR0enYfeK2tVMSjNGFVlJSEuzs7ITIhRBCCCGEEFLHNWrghLdGhAEADv4ajYuRiVbOSBhGF1qDBg3Crl27kJqaKkQ+hBBCCCGEkDquS9uGGNqzOQBg/b6/8Sglz8oZ8c/oNVqxsbG4evUqevToAVdX10q7Dp45c4a3BAkhhBBCCCF1z4QBwYh+lI0b99OxfNdlrH2rO+xtpdZOizdGF1q+vr4YNGiQELkQQgghhBBC6gmxWIS54zpg1ifnEZ+ajw3f/Yt3x4eD4zhrp8YLowutFStWCJEHIYQQQgghpJ5xc7LF/PEdsWDz77gYmYjD5+/rpxTWdkYXWjq//fYbLl++jNzcXLi5uSE8PBzdunXjMzdCCCGEEEJIHRcU4I5XXmyDLQcjsev4LTT3d0Wb5p7WTstsRhdaCoUC06ZNw++//w6xWAw3NzdkZWVh27Zt6NSpE7Zu3QqZTCZEroQQQgghhJA6aMAzAbj7MAvnrj7C6q+v4pNZPeDpWrs7nRvddXDjxo24du0aVq9ejcjISPz++++4fv06VqxYgX///Reff/65EHkSQgghhBBC6iiO4/DGS6Fo2tAZ2fnFWLn7CpQqjbXTMovRhdaxY8cwffp0DB48GGKxGAAgkUgwZMgQTJ8+HUePHuU9SUIIIYQQQkjdZiuTYMGECDjYSREVl4XtP/5n7ZTMYnShlZmZiZCQkEqfCwkJQUpKitlJEUIIIYQQQuofX08HzBnTAQBw/OIDnLv6yMoZmc7oQqtx48a4du1apc9duXIFvr6+ZidFCCGEEEIIqZ/CgxtgVL+WAIDP9v+L2KTauZmx0YXWyJEjsXXrVnz55ZdISkqCUqlEUlISvvjiC3zxxRd46aWXhMiTEEIIIYQQUk+M7NsSHYK8oVBpsO7b65Arat96LaO7Do4aNQq3bt3CmjVrsHbtWv3jjDEMHToUr776Kq8JEkIIIYQQQuoXkYjDO2O0mxmnZBbi4B8M7dowa6dlFKMLLZFIhI8++giTJ0/G5cuXkZOTAxcXF0RERKBZs2ZC5EgIIYQQQgipZ5zsZVgwoSPmbryAe4lFiLyfgWfaOlg7LYMZXWgVFRXB1tYWzZo10xdWt2/fpiKLEEIIIYQQwqtm/q6YPTIUR89HIbChs7XTMYrBa7SioqLw0ksvYefOneUez83NxUsvvYQXX3wRDx484D1BQgghhBBCSP0V1tIL/9fNA84OMmunYhSDCq34+HiMHz8e6enpaNq0abnnpFIp5s2bh+zsbIwePZrauxNCCCGEEELqPYMKrW3btsHV1RWHDh1C//79yz1nZ2eHiRMn4sCBA7CxscHWrVsFSZQQQgghhBBCaguDCq0///wTr7zyCtzd3as8xsvLC5MnT8bFixd5S44QQgghhBBCaiODCq3U1FQEBAQ88binnnoKycnJ5uZECCGEEEIIIbWaQYWWu7s7UlNTn3hcVlYWXFxczE6KEEIIIYQQQmozgwqtjh074uDBg0887vDhwwgJCTE7KUIIIYQQQgipzQwqtMaNG4dLly5h5cqVKC4urvC8QqHA6tWr8dtvv2HMmDG8J0kIIYQQQgghtYlBGxa3adMGCxYswPLly3HkyBF07twZ/v7+UKvVSExMxKVLl5CVlYW33noL3bp1EzpnQgghhBBCCKnRDCq0AGDMmDEICgrC9u3bcfbsWf3IloODA7p27YrJkyejbdu2giVKCCGEEEIIIbWFwYUWAHTo0AEdOnQAAGRmZkIikcDZ2VmQxAghhBBCCCGktjKq0Cqruj21CCGEEEIIIaQ+M6gZBiGEEEIIIYQQw1GhRQghhBBCCCE8o0KLEEIIIYQQQnhGhRYhhBBCCCGE8IwKLUIIIYQQQgjhGRVahBBCCCGEEMIzKrQIIYQQQgghhGdUaBFCCCGEEEIIz6jQIoQQQgghhBCeUaFFCCGEEEIIITyjQosQQgghhBBCeEaFFiGEEEIIIYTwjAotQgghhBBCCOEZFVqEEEIIIYQQwjMqtAghhBBCCCGEZ1RoEUIIIYQQQgjPqNAihBBCCCGEEJ5RoUUIIYQQQgghPKNCixBCCCGEEEJ4RoUWIYQQQgghhPDM6oWWRqPBhg0b0K1bN7Rr1w5Tp07Fo0ePqjw+NjYWr776KsLDw9G9e3ds2LABKpWq3DF79uxB7969ERoaitGjR+PWrVtCfxuEEEIIIYQQomf1Qmvz5s3Yu3cvPvzwQ+zbtw8ajQavvPIKFApFhWNzcnIwZswYyOVyfPXVV1i3bh1OnjyJJUuW6I85dOgQVq9ejbfeegsHDx6Ev78/Jk2ahMzMTEt+W4QQQgghhJB6zKqFlkKhwI4dOzBz5kz07NkTQUFB+OSTT5CcnIzTp09XOP7QoUMoLCzE+vXr0apVK4SHh2PZsmX44YcfEB8fDwDYsmULxo4di8GDB6N58+ZYvnw57OzssH//fkt/e4QQQgghhJB6yqqF1p07d1BQUIDOnTvrH3N2dkZISAiuXLlS4fi4uDgEBgbC3d1d/1hISAgA4OrVq8jIyEBsbGy5eBKJBOHh4ZXGI4QQQgghhBAhSKx58uTkZACAr69vuce9vb31zz3+eGpqKtRqNcRiMQAgISEBAJCRkVFtvDt37piXLGNQFxVV+hQnEkEkk+n/XtVx2oM5iG1sTDu2uBjqoiIwhQLqoiKoRaJqjwVjBsfVfX+VxRbb2hoW97FjNQoFmEZT7nstG7+6Yx8nsrEBx3HaY5VKMLW63PNlYzM7u2qPLRdXJgNX8r1Wd6y6qAiszPf9xLhSKbiSa/RJx5b9vjUqFdhjaw6rjGvEsUyjqXjNlMFJJBBJtLcDplZDo1RWGbfcsRpN5dej7lixGCKp1LC4ZY/VaKDR/XtWEr+yYw2Kyxg0xcUAKl6P1R1badwnvO7LxtdIJLzeI3SxNQoFYG9f+rgJr/vKPP59G/O6f9Kx5c7zhNe9qfcIplJVe70/6X5S7tiy9wiVqtrr3dD7CVD5PaKq693U133ZYyuLXe4eYczrvpJjy13vMpnh9xMD7hH6673M923U6/4Jx5Y95xNf90beIx7/Pqo71px7RFXXujnvDTTFxVVf78bERcV7RJXvZWDc676qYyu93o153Vdz7OOxjXrdG3CsLn659wbG3E+quUeoi4pMfs9hyD1CR3c/q+5Yk+4RT3ovY+p7A1PeRzAGlFwj1bFqoSWXywEAsjI3FwCwsbFBTk5OheOff/55bN68GStWrMDs2bNRWFiIZcuWQSKRQKlUVhuvuJqboSGKUlPx1+KllT7n3K4tmr07R//36xOnQFNc+T+YY3AQWix5T//3G69Ogyovr9Jj7QObouVHH+j/fnPGLCjS0wEAkY8da+vnh+A1K/V/vz1nPopKitDHyTw90WrjJ/q/R723BIUxD/R/Lxtb4uSENts26/9+74OPkH+78qJVZCND213b9X+/v2oNcv+9XuE4Xfz2336tf+zBpxuQfanqUcfQnV/ob6hxn29F5m+/V3pcJIDWWz+D1NkZAPBoxy6k/3y2yrghG9bBxssLAJCw51ukHjtR5bGy16fqr7GkAweR/MOhKo99atlSODQLBACkHD2OxL37qjy20bx3AFsbyOVypJ3+GfE7d1d5bODcd+AS1g4AkHH+Nzzc8kWVxwa8NR1unZ6GXC6H5k4UIpetrPLYxq9PhUeP7gCAnL//RczHa6s81n/SeHj166vN4XokilevrXA96jQcPRINBr0AACi4H4O7i96vMq7PS0Ph+/IwAID8UTzuzFugf+7x+N4DB8BvzCgAQHFaGm7NnF1lXM++vdFo8kQAgDI3F/+99ma558vGdu/eFU3eeA2A9hdS5KSpVcZ1fbojmr49U//3f0aNq/S4SAh3j4gKaILgFcv0fy97j3icMfcIiYcHJG++pr/eH79HlDvWyHvEU59vAqC9/9/f8Fml9wgdU+4RcrkcyuMnEXl9dZXHmnqPeLTnWxSfOl3l9R60egXsGvkDMO8e8Xj85osXwikkGADMvkeUja27RwBA1l+XELt+U5VxDb1HRKL8PSLv1m1Ef7i8yrjG3CPiBw9E41EjAFS8RzzOmHuE67M9gW7PaK+dSu4RZRl7j/B5Tfu8XC7HP5NfrfJYU+4RutfnzRmzoM7Pr/TY6t5HPK7CPWLhYigSkyq93p/0PqKs6u4Rj8c29H2EzpPuEWXjG/o+AjDsHqGLbcz7CGPuEVmLF4Ired0/6X2EMfcI6cj/gzxQe+8x9H0EYNg9wj6iIwAg/cpVxFdzrCn3CLlcDpaUXO17mereRzzO3PcRshlvwLZBgyq/RseqhZZtyQWvUCj0/w8AxcXFsLOzq3B8QEAA1q9fjyVLlmDPnj2wt7fHjBkzEB0dDScnp3LxyqoqHl/y8/Nx+/Zt/d81mqo/1SkoLCx3rEpd9ScJ8qKicscqqqn4ixXF5Y4tVlRdWCqUyvLHVvOpg0qtKn9sYWGVx2o0rHy+Vdz4dcodm1v5G0mdqKgocCUFtKKSIryse3fvgnNwAAAos7KqPTY6Ohqikl86yoyMao8FtF0vAUCZllb9cQ8eQFTyb6BKTa322OTkZIgDmiA2NhaqSkZyy3r06BES7bSfIqoSk6o9NiEhAcllfsbVSUxMQmrJsepqun7q8k3XHfuEfFNTU5FZcqwmIbHaY9PS0pCtOza1+p9vRkYGcnXHZmdXe2xWVhbyS45lBQXVHpudk4NC3bHVfLoFALm5eeWu4eoIdY8oVigEuUeoVCpIUHq983mP0MWMjY0V7B7xJCbfI55w74mJiYEoX5snn/eIuLg4iEs+OBXqHqGuoujWMfkeERdX7bHG3COys7NRIMA9Ii8vF1Jor0m+7xFFZa736phzj1Brqh7pMOd9hLKaY+l9RClj3kcYc49ISkxEiu51z+M9Aii9Hvm+R0hK4qakPPl9jyn3iCepCe8jHscxZuAcDwFERkZi+PDh+Pnnn9G4cWP946NGjULLli3xv//9r8qvTU1NhaurK1QqFcLCwrBt2za0atUKzzzzDHbt2lVundacOXOQk5ODL76oumqvzo0bN8A0GjRt1KjS540Z8jfnWE1xMeRyOeLi4tCkSZNyxSnHcRA9PuRfxT9thWNLhqiLiooqjc3HMD6ACvH5nDpYNra9i4vp04KqGEIvKipCXGIimjZtCjs7uycPtxsYFwCK1WrEPXyIgIAA2Eql5abHVIhbdhhfpTLoWLlcjgcxMWji51fu37XcsRKJftj/ScP4ZY8tLChAbHR0hWtG5/FphtUOzVdybFXXpFFxq5jqU1lsPqcOlo1vZ2/P6z1CHzsgAA4uLvrHjXrdV3NsUVERHiYlISAgQHu98zjFT8G0xVZAQABsxGLepw7K5XI8iI5Gk0aNqr7eTZw6WJiXh9iYmCqvd2Ne95UdW9X1XvZYQ1/3jx9bWWxjpgU96R5R7h7s6Fh6rAmv+8fpYgcEBsLeyYm3uPr4CgUeJiRo78G2trxOHSxWq/XXu6yaaUamvDeQy+WIjY1FY1/fKq/1yt5HGHqPKMjOrvR6rOxYY+8R8sLCKmPz8Z6j0uudp6mDj8c293X/OP313rw57EuKPWPiVnePKCoqQlxCApoGBsLOzs64+4kB94gipRKxsbFo0qgRbCRVj+WUu58Y+Fo25L2Mue85qjy2ktf9g0ePwIlEaNOmTZVfB1h5RCsoKAiOjo64dOmSvtDKzc3FrVu3MHbs2ArHX716FevXr8fOnTvh7e0NADhx4gTs7OwQFhYGR0dHNG3aFJcuXdIXWiqVClevXsXo0aPNypUTieBUpglHtcqsmeD7WJGNDbikJDi4usK+uq81IQdxYeGTY5vxvVUb35i4lTAodx7i29nZ8R5fXPLpnhCxdTiRSLCfDSeTCf5z5zV+yS8ug2KXHGsQoa73So7Vx3ZxKR+bp3uPuLAQKHu983hPKzT1ejfiWE4ioevdgrGFjq+Lbe/kJFjuSEgovSaNed0/4ViRBa53o37mRv78DP43Nfa9jEwm2HsZoG68luwdHATLXaj3G1zJ9W7v6ChMfAHfyxjFwQHcE0b4dKxaaMlkMowdOxZr1qyBu7s7/Pz88PHHH8PHxwf9+vWDWq1GZmamflpgYGAgoqKisGrVKowfPx5RUVFYtmwZXnvtNTg6OgIAJk+ejI8++ghNmjRBmzZtsG3bNhQVFeHll1+25rdKCCGEEEIIqUesWmgBwMyZM6FSqbBo0SIUFRWhY8eO2L59O6RSKeLj49G7d2+sWLECw4YNg7u7O7Zs2YKVK1di4MCB8PLywvTp0zFx4kR9vP/7v/9DXl4ePv30U2RnZ6N169bYuXNnuZbwhBBCCCGEECIkqxdaYrEYc+fOxdy5cys85+/vj6ioqHKPhYWF4fvvv6825pQpUzBlyhRe8ySEEEIIIYQQQ1l1w2JCCCGEEEIIqYuo0CKEEEIIIYQQnlGhRQghhBBCCCE8o0KLEEIIIYQQQnhGhRYhhBBCCCGE8IwKLUIIIYQQQgjhGRVahBBCCCGEEMIzKrQIIYQQQgghhGdUaBFCCCGEEEIIz6jQIoQQQgghhBCeUaFFCCGEEEIIITyjQosQQgghhBBCeEaFFiGEEEIIIYTwjAotQgghhBBCCOEZxxhj1k6ipvv777/BGINMJrN2KmCMQalUQiqVguO4WhNb6PiUu3XiU+7WiU+5Wyc+5W6d+JS75WMLHZ9yt058yp0/CoUCHMchLCys2uOo0DLAP//8A8YYpFKptVMhhBBCCCGEWJFSqQTHcWjfvn21x1GhRQghhBBCCCE8ozVahBBCCCGEEMIzKrQIIYQQQgghhGdUaBFCCCGEEEIIz6jQIoQQQgghhBCeUaFFCCGEEEIIITyjQosQQgghhBBCeEaFFiGEEEIIIYTwjAotQgghhBBCCOEZFVqEEEIIIYQQwjMqtAghhBBCCCGEZ1RoEUIIIYQQQgjPqNAihBBCCCGEEJ5RoUUIIYQQQgghPJNYOwFSd+Xn50MikcDW1rbCc6mpqVi6dCk+++wzk2KnpaXh008/xd9//w2lUgnGWLnnz549a3TMxMREg49t2LCh0fEJIcJITEyEs7MzHB0d8ddff+H06dMICwvDwIEDrZ1anSXEPZhYX3Z2NoqLi2Fvbw8nJydeYx8+fBgDBgyATCYr93hhYSG+//57TJw4kdfz1XRZWVk4fvw4XnzxRTg5OUGtVuPTTz/Fr7/+Ck9PT7z++ut4+umnrZ0mMRMVWjXUggULDD52xYoVJp0jKysLv/zyC3Jzc9GlSxe0aNGi3POFhYXYsWMHpk+fblTczMxMLFiwAL/99hs4jkO/fv2wcuVKfcH13XffYc2aNVAqlSblDQCLFy/Gf//9hxdeeIG3Xwa9evUCx3HVHsMYA8dxuH37ttHxN23aZPCxxv7MLUWpVKKgoACurq4VntNoNEhOTja6CDXk565j6pu32NhYHD16FDk5OejevTu6d+9e7vn8/Hx89NFHRr+WDh8+bPCxQ4YMMSq20ITO3VI/m59//hmzZs3C1q1b0ahRI7zyyito1KgRDh48iJycHIwZM8bomEK+Vi1xvVuCEPfg8ePHG3zs7t27eTknnxQKBf755x/k5uYiPDwcbm5u5Z4vLi7GyZMnjb7ehf7d8ddff+HLL7/EtWvXUFRUpH/c3t4eYWFhmDp1KiIiIoyOC2jfD+hiLliwAC1atKjwc7l16xbWrVvHe6H16quvYtmyZfD29uY1Lh8ePXqEUaNGIS8vD926dYOTkxOWL1+OvXv3ol+/fnB2dsZrr72G7du3o0OHDtZO94nkcjnu3r1b6YcuHTt2tFJWNQPHHv+JkBph3LhxBh/79ddfGx0/OjoaEyZMQGFhIQDtL4CJEydi3rx5+mPS09PRrVs3o4uKefPm4ZdffsHEiRMhk8mwe/duDBw4EDNnzsSsWbPw66+/Ijw8HMuWLUNAQIDRuQNAu3bt8OWXXyI8PNykr6/M5cuXDT7WlF86vXr1Mug4juPMfoOlUCiwY8cOPP/882jSpAnee+89nDhxAmFhYVizQQLtugABAABJREFUZk2FX3RPUlxcjA8++AA//vgjVCoV2rRpg/fffx+tWrXSH2Pq9bJx40aD33ia8ibi2rVrmDJlCry9vcFxHB4+fIh+/frh448/1n+yamruQUFBBh1nanFeFt8jCELnbqmfzdChQ9G9e3e89dZb+Pzzz3HkyBGcOnUKp06dwsaNG3Hy5EmjYwr5WhX6egeAI0eO4MCBA/oPFl5//XU4Ojrqn8/MzMTw4cPNus8IcQ+2xAeMOhqNBkePHq3y9WRs/KSkJEydOhXR0dEAADs7O8yZM6dcoW/qfUbI6/HEiROYN28ennvuOXTq1Ane3t6wsbFBcXExUlNT8ddff+Gnn37C2rVr8dxzzxkVG9B+4DJ//nxwHKf/oPJxjDH06NEDW7duNSl+Vd5//3289dZbcHd3B2D+h118XjPvvvsuEhMTsXnzZjg5OSE7Oxtdu3ZFjx499DN9tmzZgqtXr+LLL780K29A2BHos2fPYv78+cjPz68Ql4/ffVevXq0y75r6oXRZNKJVQ5lSPBlj5cqV6NChA9asWQORSITdu3dj3bp1yM7OxvLly82K/ccff+D999/XT9vp0KED3n77bTx8+BCXLl3CkiVLMHr0aLPOYW9vDw8PD7NiPM7UT+wMde7cOUHjl7VmzRocOXIE3bp1w2+//YZDhw5h5syZ+PXXX7F69Wqj30SsX78eFy9exLJly8BxHHbt2oXRo0dj06ZN6Natm/44Uz63mTFjhtFfY4y1a9fipZdewuLFiwEAP/30ExYuXIhp06Zhy5YtkEhMvw3euXOHrzSfiO8RBKFzt9TP5v79+9i0aRNEIhEuXryIHj16QCQSoV27dkhISDApppCvVaGv9/3792Pp0qV48cUXIRKJsGfPHvz888/YuXOnfrRZo9EYNVW6MkLcg80tnoyxfPly7NmzB0FBQeWKUFOtXLkSbm5u+PXXX8FxHLZt24Zly5YhMzPT7H9zIa/HTZs2Yfbs2Zg8eXKlzw8fPhytWrXChg0bTCq0hgwZAj8/P2g0GkyYMAEbNmyAi4uL/nmO42Bvb4+nnnrKpPyXLl2qHzGr7PfP6tWr9ecxt9Di85r5448/8Mknn+jv5X/88QdUKlW5HLt27YodO3aYdR4dIUagddasWYPOnTtj2rRpvMf+7LPPsHHjRv3U8LI4jqsVhRYYqRWUSiVLTk5mCQkJLCEhgcXHx7OYmBh25MgRk+J17NiRRUdHl3vs9OnTLCQkhK1atYoxxlhaWhoLCgoyOnarVq1YfHx8ucdCQkLYs88+W+Gcplq5ciVbuHAhU6lUvMR7XFFREfvqq6/Ye++9x+bPn6//M3v2bNavXz9BzskYY8XFxezq1atmx+nWrRv7/fffGWOMLVmyhE2YMIExxtiNGzdYp06djI737LPPst9++03/d6VSyd5++23Wtm1bfb6mXi+Pu337Njty5Ag7dOgQO3ToEDt48CDbt28fe++990yKFxYWxmJjY8s9du3aNdauXTs2e/ZsXnOvSlJSktkx2rZty65cucJDNsbhI3ch43fp0oXdvn2b5ebmslatWrFz584xxhj7888/Wbdu3fhIsVJ8vVb5vt4HDhzIvvvuO/3fExIS2IABA1ivXr1YamoqY4yf613oezBjjGVkZLCrV6+yy5cvs8uXL7NLly6xCxcusM2bN5sdOyIigh08eJCHLLU6d+7Mbt68We6x3bt3s5YtW7Ldu3czxoS9z5h6PbZt25bFxMRUe8z9+/dZaGioqanpXbp0iSkUCpaZmal/7Pr162ZdQzExMWzYsGFs4sSJLDk5udxz7dq1Yw8fPjQ59uP4vGZatWrFEhMT9X9funQpCw4OZtnZ2frH4uPjefm5Mybs74/WrVuzuLg4QWJ36dKFbdmyRZDYlkIjWrXA77//jnfffReZmZkVnrO1tcXgwYONjimTyVBcXFzusb59+2LRokVYunQpvL29TV5IrlKpKjTAkMlkWLJkCZo1a2ZSzMdlZ2fj2LFj+PXXX9GoUaMKi2vNnb+/bNkyHD58GCEhIbhx4wbat2+PuLg4ZGRk8DKP/L///sPixYtx9+5daDSaCs+bO9SenZ2t/1lfvHgRI0aMAAC4urqWm4NvqKysLDRp0kT/d4lEgjVr1uC1117DG2+8gb1791a6bstYO3fuxKpVqwBAP9VE9/+mTlFydHRERkZGufzDwsLw8ccfY+bMmfD09MTUqVPNzv3Ro0dYtWoV7t69C7VaDUD7CatCoUBmZiZu3bplVnwhRhB0hM5dyPg9evTAkiVL4ODgACcnJ3Tp0gV//PEH/ve//6Fnz55m5Q0I+1oV4nqPj49H586d9X9v2LAhvvrqK4wcORKvvPIK9uzZY3K+ZQl9D/7xxx+xaNEiKBSKCtPO/Pz88MYbb5gVX6FQ8Lp2RK1Ww8bGptxj48aNQ0ZGBlasWAFPT09ezsf39disWTMcPXoUM2fOrPKYQ4cOmTzNvywfHx8MHDgQvXv31i9TePXVV+Hp6YkvvvgCvr6+Rsds2rQpvvvuO2zYsAEvvvgilixZggEDBpida2X4vGbc3d2Rmpqq/57/+OMPBAcHlxvtu337Njw9PXk5n5C/PwICApCcnIzGjRvzHjsvL6/WNzWi9u61wLp16xASEoKtW7fC1tYWmzZtwsKFC+Ho6IiPP/7YpJgRERFYuXIl0tLSyj0+atQoTJkyBatWrcK2bdv4SF+PryJLZ+DAgejevTuaNm0KPz+/cn/MdfbsWaxYsQLfffcd/Pz88OGHH+KXX35B7969zWriobNixQqIxWIsWrQIUqkUixcvxoQJEyCRSLBu3Tqz4zdu3Bg3btzAzZs3ER8fr5/ed+bMGfj7+xsdr1mzZjh16lS5x8RiMdavXw9fX1+88soriImJMTvvPXv2YOrUqbh+/Trc3Nxw/vx5HDlyBM2aNUPv3r1NitmjRw8sXboU//77b7l/uz59+mDhwoX46quv8MEHH5id+wcffICoqCg899xzSElJwQsvvIBWrVohPT0d//vf/8yO/+KLL+LLL7/UFyp8Ejp3IeMvXrwYYWFhsLe3x+effw6ZTIZr166hXbt2ePfdd83OXcjXqhDXu7e3N27evFnuMU9PT2zbtg3Jycl44403TPqwpTJC3oO3bNmCF154AcePH4eTkxMOHDiAzz77DN7e3rxMv+zWrRvOnz9vdhyddu3aYdOmTVAoFOUef/vttzFgwADMmzcPR48eNfs8fF+Pc+bMwfbt2zFixAh88skn2LdvHw4ePIh9+/Zhw4YNGDt2LHbt2oW5c+eanfvy5cvRpEkTTJo0Sf/YiRMn4Ovra9a0UYlEgtmzZ2Pjxo1Ys2YN5syZg7y8PLPzfRyf10y3bt3w+eefIz8/Hz/++CNiY2Px/PPP658vLCzE5s2b0aVLF17OJ+Tvjzlz5uDDDz/EuXPnEBsbi8TExHJ/zBEWFoZ//vmHp0ytxJrDacQwbdq0Ybdv32aMMTZq1Cj2xx9/MMYYO3DgABs5cqRJMZOSktjAgQNZUFBQuSlhOqtXr2YtW7Y0aZpDUFAQy8jIKPcY30P4QmvVqhVLSEhgjDH2+uuvs6NHjzLGGIuMjGS9evUyO367du3Y9evXGWOMvfzyy/oh/Z07d7KJEyeaHf/QoUOsdevWLDQ0VD9tcNOmTSw4OJgdOnTI6Hhnz55lISEhbPLkyezOnTvlnktNTWX9+/dnbdq0MXtaTKtWrfRTECZNmsR+/vlnxhhjFy5cYC+88IJJMbOzs9mkSZNYUFAQO3/+fIXn9+zZw1q1amV27mFhYeyvv/5ijDH24osv6v99161bx6ZNm2ZWbMYYmz9/PmvdujV75pln2IgRI9i4cePK/anJuQsdX0hCvlaFuN6/+OILFhERwbZu3VphKtXVq1dZ+/btWd++fQWdKsuH1q1b66eajx07Vv/a/emnn9jQoUPNjr99+3bWtm1b9uabb7J169axjRs3lvtjrHv37rEuXbqwDh06sEuXLpV7TjfV2tTfqWUJcT1GR0ezxYsXswEDBrCwsDAWEhLC2rdvzwYMGMCWLFnC7t27Z1bOOh06dGD379+v8Pjdu3dZx44deTlHTk4Omz17NuvevTtr1aoVr+87+LxmUlJSWL9+/VhQUBBr2bIle+mll1hxcTFjjLG9e/eyLl26sK5du1Z4DZtKyN8fISEhrGXLlvrrW/eHj+v9wIEDLCIigq1YsYJ99913+inWuj+1AU0drAXEYrF+gWGTJk1w9+5ddO7cGZ06ddJPOzGWj48PfvjhB1y7dq3SRahz585F9+7dcfz4caNjM8bw5ptvQiqV6h8rLi7GnDlzKkytMGd6SVJSEvbs2YO7d+9CIpGgRYsWGDFiBC97XLm7uyMjIwMNGzZEQEAA7t69CwBwc3NDenq62fE1Gg28vLwAlP6bhoeHo3fv3iZ1XnrckCFDEBQUhPj4eH0r8zZt2mD79u3lphUZqlevXvjqq6/w3XffVVhw7OXlhe+++w7Lly83qcNbWfb29vpP3Bo3bozo6Gj06dMHzZo1M7mxgYuLC3bs2IGHDx/qu0+VNXr0aHTu3BmnT582K3eFQqGfOtG0aVNERUUhNDQUQ4YMMaqLaHWEmkIhdO5CxxeyK5WQr1UhrvcpU6ZAqVRiz549CA0NRYMGDfTPdejQAbt27cKcOXPMyltHyHuwTCbTT0ds0qQJ7t27h+7du6N169aIi4szO/4333wDd3d33Lp1q8LUVVMW2Tdv3hzHjh3Dzz//XGHWgEQiwSeffIJu3bqZ9Du1LCGux2bNmvEyqv8kEokEubm5FR6Xy+UmNVKqjLOzM9auXYvDhw/j4MGDFd5zmIPPa8bb2xtHjx7FxYsXIRKJ8Mwzz+jfM0kkEgwcOBCTJk0q9/o1l1C/P3bu3ClIXAB47733AAC7du2q8BwfDU4sgQqtWqBFixY4d+4cxo0bh8DAQFy7dg0TJkxAcnKyWXFlMhk6d+6M9evXY9iwYWjUqFG5559++mmTNssbOnRohcf4mEpSVlRUFMaOHQtbW1uEhoZCo9Hg4MGD2LNnD7799tsKe4IZq3v37li6dClWrFiBDh06YPny5ejbty9OnDgBHx8fs/Nv0qQJrl27hoEDByIwMBA3btwAoJ2P/PjUE1MFBQWVa7H9+N5RxgoPD0d4eDgePXpU4TlnZ2esXLnS7I6VYWFh2LZtG5YsWYKQkBAcOHAAr776Kq5duwYHBwezYjdu3Bjz5s3DsGHD0KlTp3LPNW3aFK+99ppZ8f38/HD37l34+vqiadOm+rUSGo0GBQUFZsUGhO3IJnTuQsYXuiuVkK9VIa53juPwxhtvVLmGKTQ0FCdPnkRkZKQ5qQt+D27dujX279+P2bNn46mnnsL58+cxZcoUREdHl/sQz1RCdPJzdXXF8OHDceDAATz//PMV/g2HDRuGYcOGmXUOIa5HofYZfFz37t2xbNkyrFu3Tv/By6NHj7BixYpy3Wv5MGTIEN7fhPN9zchkMjz77LMVHh8+fDiv5wGE/f1RtmNzZmYmJBIJnJ2deYltyc6+QqF9tGqBM2fOYObMmViyZAl69Oih3+8iKioKbdu2xYYNG8yK36dPHyQkJCAsLAzDhg1D//79zX5TK7QpU6bAzs4Oa9eu1X9ipRs1UygUZn/SnJubi/nz56NLly4YPXo0Xn31VVy4cAESiQSrVq3CCy+8YFb8/fv3Y9myZfjoo48QHByMF198EcOHD8fff/8NT09PbN++3az4T9r405yRxKCgIHTo0EGQa+Xu3buYPHkyJk6ciFGjRmHQoEHIzc2FXC7HlClTMHv2bLPijx8/HlevXoWPjw+GDBmCoUOHVviAwVRbt27Fjh07sHr1ari5uWH8+PGYPn06Ll68CLlcjn379pl9DqFGEITOXcj4Xbt2xbhx48wulKsi5GtV6Ou9V69evF/nOkLfg69evYpXXnkFM2fOxNChQ9G/f394enoiKSkJAwYMwLJly8z+HhhjuHDhQrnXU6dOnSAWi82KGxYWBsYY+vbtW+kHO+bg+3oUcp/Bx2VkZGDSpEm4d++e/o14bm4uWrVqhc8//1w/UleT8XnNXLlyBT/88IO+wP2///u/cnFycnIwY8YM3jbnFnIEevfu3di2bRsyMjIAaNeFTpkyhbdNqO/fv4+7d+9CKpWiWbNmaNq0KS9xLYEKrVri5s2bEIvFCAoKwuXLl7Fz5074+vpi5syZvHR7+/vvv3Hs2DGcPHkSRUVF6Nu3L4YOHWrSNDNDyOVyfPLJJ1i4cKFJX9++fXvs27cPLVu2LPf4nTt3MHbsWFy9epWPNPUYY/oOQHztMv/zzz/Dzc0N4eHhOHr0qL7r0uLFi01qWFHW4xt/qlQqxMXF4e7du5gwYQJmzZplcuwrV67g6NGj+Omnn6BQKHh/M1FUVITCwkK4u7sjPT0dx44dg4+PD/r3789L/JSUFBw9ehRHjx5FVFQUOnTogKFDh1b6CbQxGGP46quvEBAQgJ49e2Lbtm3YunUrfH198fHHHyM4ONisvCsbQfjvv/8gl8vNHkEQOnch47dt2xYnTpzgfdS8LCFfq0Je75s2bcKxY8cQFxfH+wdplrgHp6SkQKFQoFGjRoiOjsa+ffvg6+uLcePGVehyaKzs7GxMmTIFN2/ehJOTExhjyM/PR6tWrbBz506zPpEvKirCzz//jGPHjuHixYvw9vbGiy++WOnMEVPweT2OHj0awcHBFfYZbN++vX6fQb4KLUDbnfGPP/7AvXv3IJFI0Lx5c3Tu3NngTbwfd+XKFYOPNbdjIJ/XzLlz5zB9+nRERERAJBLhr7/+Qtu2bbFlyxZ950E+f+5C/v7Yt28fPvroI4wePRodO3YEYwxXrlzBvn37sGTJErz88ssmxy4uLsY777yDM2fO6B/jOA7PPvssPv30U7PvAxZh6UVhxHj79+9n+fn5FjmXUqlkv/zyC5szZw5r3749e/bZZ42OIZfL2dKlS1lERATr0qULW716NVOr1frnL1y4wHr16sWCg4NNzrNLly7sxo0bFR6PjIxkHTp0MDmuzrPPPsvWr18vWAMPXUMTS9u0aRNbtGgRL7EUCgU7ffo0mzFjBgsNDdX/zMwxd+5c9ueff/KSnyHu3r3L1q1bxzp06MDat29vViyhm71MnjyZvfnmm6yoqEj/WFFREZs+fTp79dVXzYotdO5Cxp84caK+WY0QhHytWup6v379Ovvwww9Zly5dWLt27di8efPMPq/Q9+BPP/1U0Otm/vz5bMCAAfpGU4xp9zQbOHAgW7JkCW/nycjIYN988w0bNWoUCwkJYWPGjDErHt/XY03YZ5AxVu6+ZoyuXbuWa75Q1R8+8ufzmhk6dGi5/eCuX7/OunXrxoYOHcry8vIYY/z+3IX8/fHcc8+xr7/+usLj33zzDRs4cKBZsVesWMG6devGzpw5w3Jzc1l2djb7+eefWY8ePdiaNWvMim0pVGjVAu3bt2ft2rWzyC/llJQUtmPHDjZixAgWEhJiUhejDz74gLVq1YotWLCAvf/++6x9+/Zs8+bNTK1Ws6VLl7KgoCDWr18/dvnyZZPzfOedd9i4ceNYVlaW/rGMjAw2btw4Nn36dJPj6mzcuJE999xzLCgoiI0ePZodOHCA12I3KChI8GKuMo8ePeLlTVBZGRkZbPv27fpuVeYYN24cCw4OtsjP5t9//2Uffvgh69q1KwsLCzN5g1idli1bCnKt6LRr165Cx0fGtL/ozf03FTp3IeML3ZVKyNeqJa93xhhTq9Xsm2++YWFhYWa/gRP6Hty7d2/B7r+MMfb0009X6A7IGGN//fUXe+aZZ3g7T3FxMfvpp5/Y7NmzWbt27cze8J7v67F79+7s2rVrFR7/+eefWXBwMFu+fDlvb/gzMzPZ8uXL2cSJE/Xd7saOHctGjBhh8j0sMzOTDR06lL300kvs0aNHLD4+vso/5uLzmqmsE/P9+/dZp06d2IQJE5hCoeC10BLy90ebNm0q3bA4Li6OtWnTxqzYXbp00W9CX9a5c+dYjx49zIptKVRo1QJyuZz9+OOP7NVXX2WtWrVizz77LK+f9uXl5bEDBw6wCRMmsJCQENanTx/22Wefldu13Bg9e/Zke/fu1f/9l19+YX369GGLFy9mwcHBbPXq1fo2pqZKSkpiPXr0YO3atWNDhgxhQ4YMYe3atWM9e/Zkjx49Mit2WUJ8EswYYwkJCezzzz9nL7zwgv7NhCVGLn/88UcWERFhdpyCggJ2+PBhNmXKFNaqVSvWr18/s66ZspKTk9kXX3zBBg8erH+DztfPJiYmhq1fv17f3nr8+PHs8OHDTC6Xmx378uXLbNGiRSwiIkL/wQifnz4LOYIgdO5Cxhf6U2yhX6tCXu86iYmJbOvWrWzgwIEsODiYTZgwgR05csSsmJa4B1+7do0tXbqUderUiffrpqpW49HR0Sw0NNTs+H/++SdbuHAhCw8P13+Qo2vFbg6+r8fFixezwYMHs3/++YcpFIpyz3399desZcuWbMaMGby8lt566y0WERHBZs2axYKDg9mcOXPY0KFDWcuWLdnWrVtNjpucnMyefvpp9v3335udY3X4vGZ69erFLly4UOHxa9eusdDQUPb222+z5ORk3gotIX9/PPfcc+z48eMVHj927JhJs6LKateuXYURV8YYe/DggdlFnKXQGq1aJjMzEydPnsTx48dx/fp1tG/fHt98841ZMUNDQyGVStGvXz+89NJLCA8PNytemzZtcOLECf1cdLVajTZt2sDFxQWffvqpSZ0MK1NQUIAjR47g3r17YIyhZcuWGDRoUIXOY3zQaDT49ttvsW7dOhQWFvIyZ1rn1q1bOHr0KE6dOoXs7Gz069fP5Lb9OpU1w8jPz0dUVBRGjx6tb5lqilmzZuHXX38Fx3Ho378/hg0bZvY1U5V79+7h2LFj2LNnDzQaDf7++2+z4gUFBcHf31/fIECIdT1KpRK//vorjh49ivPnz8PDwwNDhgzBzJkzzYo7Z84cpKamYsOGDfp1mZmZmXj77bfh4uKCjRs31tjcLRVfaEK8Vsvi+3rft28fjh07hr///ht+fn76656Pxe+A5e7BKpUKv//+O44fP46zZ8/C1dXV7A5wEyZMwFNPPVXhXrhs2TL8999/ZjVo6datGzIyMhAeHo5hw4bhueeeg52dnVn5VoaP6zEnJwezZs3Cn3/+ia1bt1boOLh3714sX74carXa7N97Tz/9NFatWoWePXvihRdewNq1axEUFITFixejuLgYq1evNjn2gQMHcObMGWzZssWsHKvD5zWzdu1aHD9+HO+88w66detWbn3X6dOnMXv2bLRu3RrXr1/n5f2GkL8/du3ahc8//xxvvfUWwsLCAGibrGzYsAHjx4/Hm2++aXLsESNGoFevXhWaHX3++ec4deoUjhw5YnJsS6FCq5ZRKBT49ddf8dNPP+HcuXPw9vbGTz/9ZFbMQ4cOoX///rz9IggKCsLFixfh4eGhf6x9+/b48MMPBdvHQShJSUn6xgn3799HREQEhg0bhsGDB/N6nps3b+LU/7N35uFUrt//f5Oc6jSPnD5FGmxHxopKk6E+zaVJKSVDs0ojaTAl0aAQEtFMhQaaTh00KdIpSokUKRSlUKZ9//5w7edrt1H7GfbJ5+d1Xa6r/ditfXv2/dzTWuu9Ll/GiRMnICUlxTiR/HsxDABo3rw5NDU1MXnyZMjI0K/sYGZmhunTp3O2eBDw6NEjSnSjrKwM48aNY6w0dv/+fSEpWi4pKipCVFQUfH198e3bNzx58oSRvby8PMyePRvFxcVQVFQEUCPL3L59exw9epSxKENt2G67JOxLSpWK7WdVABf9XUtLizoMYSoE8G9SUFCA6OhoXLlyBSkpKdDR0WFcu+fhw4eYP38+eDye0OLw2bNnOHToECNxHx8fH0ydOpXVZ7I+2OqP2dnZ6NChA1WzszZZWVm4evUqY2XP/v3749q1a5CXl4eNjQ0MDQ0xdepUPH/+HIsWLUJcXBwj+1zDZp8pLy+Hi4sLzp07h4CAAAwdOlTo9zdu3MDGjRtRUlLCykaLy/mDz+djx44dOHnyJKqrq0EIgYyMDGbPno1NmzZBWlqatu3Y2FgsW7YMY8aMEbrn165dw+7duzFu3DjatiVF00arkZCQkIALFy7g6tWr4PP5GDduHKZOncqZJ4EJ9W20zp07R9XOoIOhoSHOnDmDDh06wMDAoEGVouvXr9P+HID7k2Cgpn6IYBP3+vVr6OrqYurUqfjvf/+LFi1asPY5jYmsrCxcuHABFy9eRE5ODrWxbSz3pKysDNeuXcOFCxeQkJCA7t27Y8qUKTA2Noa8vDxj+1x6ELhuO1f2JaFKxdWzynV/LysrQ6tWrRjbESDJMbikpARXrlzBhQsXkJiYiD/++APGxsas9UcAePz4MYKDg4Wep4ULF0JdXZ0V+1zBVX8sKirCu3fvUF5ejlatWqFr1651Fnmni6GhIXbu3ImBAwdi9+7dqKqqwsaNG5GdnY3Jkyfjn3/+Ye2zuILtPvPt2zdISUnVWVj58+fPiI+PZ+2AmmsPdElJCV6+fAkAUFJSYs3utWvXEBgYiPT0dKrdVlZWGDNmDCv2uaZpo9UIkFQYwvcUFRVh5syZYk+YPB4Pd+7cERqgtbS0cP78eUbStj4+PlTtFh8fnwbfy7RIKdcnwbNmzUJKSopQGBubmzgAyM3NRXh4OJ4/f45mzZpBVVUVs2bNQufOnVn9HAFlZWUIDg5mdO8lEdpXF2zI6EoypJJtuG47l/bd3d0RExODbdu2QUdHB3w+H4mJiXB1dcWkSZOwdu1aRva5fFb/rf5eWVmJf/75R+yxTZJjMNsh7b8CbNRF4qI/RkZG4uDBg3j16hWAmnIMQM2BhaCY+5QpUxh9BgDs3LkTf/31F9zd3VFRUQFbW1s4Ojrir7/+wosXL2iHgVVVVeHmzZsYPHgwtTY6deoUYmNj0blzZyxcuBC9e/dm3H5JUFRUhKSkJHTq1AkDBgz4t5tTL4mJidDS0oKMjMwPJfYbszedDejHDzUhMUxMTCQWhlAbWVlZ2mFWrq6uQic0lZWV8PT0FKnfIk618toTt66uLjQ1NdG8eXOh95SXlyM2NpZWm2tz+/ZtVk+Cv6d3795Yv349ZwNQcnIyLCws0KFDB/Tv3x/V1dUICwtDaGgojh07xqhmRn2UlZXBx8eH0QLryJEjEgvtq02rVq0YxZEDNZu1bdu2sXoQIikPAhdtl5T9ixcvwsXFBfr6+tQ1IyMjNGvWDE5OTow3Wlw+q/9Wfy8uLsb8+fPFPliQ5Bjs5OTEakg7UBNS7eDggNatW9cZXl0bceYmcWB6ts12fzx69Ch27doFc3NzDB48GF27doWsrCwqKipQUFCAu3fvYtu2bSgtLYWpqSmjz7K1tUVVVRXevn2LSZMmYcyYMVi9ejXatm2Lffv20bJZWFgIMzMzZGVl4eLFi+jduzcOHDgAb29vqKmpoaKiArNmzcKpU6dozXtc9hlfX18cOXIE4eHhUFBQQHJyMhYtWoSSkhIAwJAhQ+Dn50fbS8nl/GFmZkZFLpmZmUFKSqrOvi0lJSX2OCPJAx1J0OTRakQkJiYiMzMTEydORF5eHhQVFRnl2nCFmZnZT7/36NGjtD5DRUUFt2/fFglrePLkCebMmYPHjx/TslubuLg4BAUF4eXLlwgLC0NERAR69uzJysmegLdv3yIzMxODBg1CaWmpULglE0xMTKCkpAQXFxeqj1RWVsLe3h4fPnxASEiIWPYqKirg4eGBCxcuoHnz5hg/fjzWrFnDSThfQUEBwsPD8fLlSzg4OCAxMRH9+vWDkpIS65/FBRUVFXjz5g169uwJQojIQlQcJD3hsNl2SdnX0tJCVFQUFBQUhK6/evUKkydPZmUsALh7Vv+N/l5ZWYmHDx8y2uRJYgz+9u0bLl++jMzMTFhaWiI9PR19+/ZFhw4daNkzMzODr68v2rZt+8N5iu7cJCnY6o+GhoZYsWIFjI2N631PREQEDhw4IBSeyxafPn1C69ataa9lnJ2dkZSUBC8vLygpKaG0tBR6enpQU1OjvkN3d3e8ffsW+/fvF9s+V30mLCwMrq6uMDc3x+LFi9G6dWuMHTsWJSUlCAkJQZs2bWBjY4Nhw4bRFgvicv7Izc2FvLw8pKWlkZub2+B7xfXUGxgY4OzZs9QGsT6kpKQYhyhLBMkJHDZBly9fvhATExNKrjg7O5ssXryYjB8/nuTl5f3bzZMYhw8fFipMKPj39z+zZs1i/Fm3bt0iqqqqZOPGjURNTY1kZ2eTPXv2EBUVFVZq81RUVJDVq1cTZWVloqKiQrKzs8ny5cuJubk5VayQCerq6nXK0L548YJoamqKbc/d3Z1oaGiQLVu2ULXRmNadqotXr14RHR0dYmBgQFRVVUl2djaxsbEhmpqa5J9//mFsPzc3t86ft2/fkvfv3wsV1qaDp6cnUVdXp75TW1tbsmnTJhHZZDrcv3+/Tjvfvn0jly9fZmyfy7ZzaX/WrFnE399f5PqBAwfI5MmTGdkmhNtnlev+zjaSHIPfv39PDAwMiIaGBnXfFy9eTPT09EhGRgYLf039FBQUsGrv48ePpLKykhVbbPdHTU3NOueK2jCRvF+5ciUrc1p96Ovrk1u3blGvr127RpSVlcn58+epaw8fPiSDBw/mrA2EiN9npk+fTo4dO0a9fvz4MVFWVhYay27cuMG47poALucPOzu7Or/jjx8/kqVLlzKy3RBM52tJ0bTRagQ4OTkRExMTkp2dTRW5y8jIIFOnTqUqtzOhoQlTTU2NjB49mvj4+BA+n8/4swoLC8mlS5do1VmprKwkkZGR5OzZs0RZWZmEhoaSiIgI6icyMpJcuXKFFBcXM26niYkJOXz4MCFEuLDgwYMHGVc6J4QQLy8vMnbsWJKQkEDZT0hIIPr6+mTbtm2M7U+cOLHOuhaxsbFk3LhxYtvT19cXsvf3338TTU1NVvpEbZYsWUI2bdpE+Hw+dV8qKyvJmjVryLx58xjbb6iv83g8oq6uTuzt7WnVeQsNDSXDhg0jZ8+eJRoaGiQ7O5tER0cTHR0dsmfPHsZt5/F4pLCwUOR6amoq43oiXLedS/t///03UVFRIatWrSKhoaEkNDSUrFy5kqioqJCYmBjGbefyWeWyvz9//lyoH8fHxxMXFxdy4MCBOvvRzyDJMXjt2rVk8eLFpKysjLo3nz59Iubm5mTx4sWM7df3POXk5NA6jCKkpnCtjY0Nyc/PJ4QQkp+fT0xMTAiPxyOamprE19eXUZsJYb8/zps3j2zevLneRSufzyd2dnbExMSEVnt5PB758OGD0LWJEyeyUnOREEJUVVVJbm4u9drd3Z3weDzy/v176lpubi7p378/489is89oamqSrKws6vXBgwcJj8cjaWlp1LXs7GxW2k0I+/NHUlISVRSex+ORo0ePihSL37VrF+1nSYCBgYFQUXQBeXl5rNQElQS/XtxZEyL8/fff2L17t5CQRO/evbF161bGeSUAsGnTJuzevRtz5syhEo4fPnyI48ePY86cOWjXrh2OHDkCWVlZWFtbi2U7PT0dNjY2cHV1hbKyMiZPnowPHz5AVlYWBw8eFEsOVUZGBlOnTgVQ4zKeMGECK4pidfH8+fM6a3qMHTv2hy74nyE6OhqOjo5CNcV0dXWxfft2bNiwAY6OjozsW1lZwdXVFQUFBdDV1YWMjAxSUlKwd+9emJqaCiWv/kysf0FBASWtCgB6enr49u0b3r9/j65duzJqa22Sk5Nx/PhxoVhyGRkZLFu2DLNmzWJs383NDTt37sSKFSuovzs5ORne3t6YN28eevbsCR8fH3h7e4ud2xMWFoatW7di9OjRcHFxAQCMHz8ezZs3x44dO2Brayt2e0NCQqi6OIQQ6Onp1fk+pippXLRdUvZHjRqFffv2ITAwELGxsZQqlZeXFyuqVFw+q1z099LSUixbtgz379+nclZOnz6NrVu3olu3bvjtt99w4sQJhIWFiS2iIMkxOCEhAQcPHhTK0WrXrh02btxYZ53An+HMmTM4f/48gJrnafny5SLhqwUFBUI1jX6We/fuwcLCAmpqatQ1e3t7PH36FJs3b0abNm3g4eEBOTk5TJs2jVb7Afb746ZNm2BhYYH4+HgMGjQI8vLyQjlaSUlJKCkpQVBQEK32kjqyU968eYOqqipa9r6nbdu2KC4upvpyQkIClJSUhESfsrKyaKsnctlnaj/3SUlJaNeuHXg8HnWttLSUUY4il/OHlJQU7OzsqH/XVYqiVatWsLS0FNt2TEwMbt68CaAmRNHZ2VlElTE3N7fBnLNfiaaNViOgqKgIXbp0Ebnetm1blJWVMbYfHR2NTZs2wcTEhLpmZGQEJSUlnDlzBidPnkTfvn3h4eEh9kZr586dUFBQgJKSEi5evIiqqirExcXh1KlT8PLyol0U0tjYGEVFRcjKygKfzwdQM5BUVFQgJSUFS5cupWVXQJs2bVBQUCAiR5+RkYF27doxsg0A+fn5dUrdy8vLo7i4mLH9jRs3AqiJTf+e2knHP5uoWlVVJTS5NG/eHC1atEB5eTnjttaGz+dT32dtSktL0axZM8b2Dx8+jG3btmH8+PHUNR6Phy5dusDHxwfnzp1D586dsWnTJrE3Wm/evIGKiorIdR6Ph/fv39Nq77x589C+fXvw+Xxs2rQJ9vb2QnVupKSk0KpVK0Y1fwBu2i5J+6NHj8bo0aMZ26kLLp9VLvp7QEAAcnJycPDgQfTq1QsVFRXw9PQEj8dDWFgYZGVlsWHDBnh7ezMSfOB6DC4tLa1XkIjuIt3IyAgPHjygXsvJyYnkmfbr14/aTIrDwYMHMXPmTGqjk5OTg9u3b2PevHmYO3cuAKC6uhrHjx9ntNFiuz+qqKjg0qVLCAsLw4MHD5Ceno5v377ht99+g7y8PExMTDBjxgxWZd7ZZPDgwTh27Bi2b9+OxMREpKWlCfU9Pp+PwMBA2qqVXPWZfv36ITk5GQoKCvj8+TPu3bsHQ0NDofdcunQJ/fr1o9VugNv5Q1tbG8+ePQNQM47funWLNUVjLS0tnDp1itqkv337Vmj9IWg3m8XiuaRpo9UIUFNTw6VLl7Bo0SKh68ePH8eff/7J2H5aWlqdD9rAgQPh5OQEAPjzzz/x7t07sW0/fPgQp0+fRqdOnXDz5k2MHDkS3bp1w7Rp0xgVnDx//jw2b96MyspKADUTvOB0o3v37own+UmTJsHNzQ1ubm6QkpJCaWkp4uPj4eLiIrRIp0vv3r1x9+5dzJw5U+h6dHQ0+vTpw9j+tWvXGBUJ/LcYNmwYAgIC4OnpSV379OkTPD09GW8mAOD169d1PjN9+/ZFVlYWAEBRURGFhYVi2+7evTslu1yb+Ph42mUNJOVB4KLtXNqXpEgIl88qF/39ypUr2LRpE4YPHw6gpkj358+fsX79eqrvzJw5k7GXkusxeNCgQTh58qSQ0ltlZSX8/PyEvOvi0L59e6HNpUBNjg1SUlKwfv166vWdO3cgJSUldACgrq4OZ2dnRp/DRX9s374942LE/xY2NjYwNTWFjo4OSktL0b17dyxcuBBAjWckICAAb968QXh4OC37XPWZuXPnYtu2bUhLS8PDhw9RUVGBBQsWAKjZTF+4cAFBQUHYvn077c+Q1Pwh2HCxhby8PFUCwczMDD4+PqwccP9bNG20GgFr1qyBhYUFHj9+jKqqKvj5+SEzMxNPnjyh7c6vzX/+8x/8/fffMDc3F7r+999/Q05ODkBN1Xg6J1rS0tKQlZVFVVUV7t+/jy1btgCoOa1koljn7++PCRMmwMrKCnPmzEFwcDAKCgrg5OQEGxsb2nYFrF69Gnl5edQgZWxsDEIIRo0axXiBAtRMDra2tsjIyEB1dTUiIyORlZWFK1euYO/evYzt29rawtXVVSgMgQlSUlIibnou3PZ2dnaYP38+hg0bhvLycixduhS5ublo3759nd45cenTpw/Onj0r4q06e/YspVqXlpaGbt26iW3b0tISTk5OeP/+PQghuHv3LsLCwnD06FEqxIIJXHoQuG472/YjIiIwd+5ctGzZEhEREfW+T0pKivFGi8tnlYv+/u7dOyHvYWJiIqSkpDB06FDqWvfu3Rl747gegzdu3Ii5c+fi/v37qKyshKOjI16+fIkvX77g2LFjjO3X580TPE/i1jD6+vWrUPhYYmIiZGVloaWlRV1r1qwZ43GTi/6YmJiIs2fPori4GCNGjMCsWbOEPKpM6n/VNXewSa9evXDx4kVcvnyZ2kwIvofc3Fz07NkTO3fuZKWOFpt9ZvLkyaioqMDJkychLS2NvXv3UiF8AQEBCA8Ph7W1NWsqx1zOH+Xl5QgLC0N6ejqqq6up6xUVFUhNTcWVK1do225IyTEvL49ao/7KNMm7NxKePXuG4OBgPH36FHw+H3379oWFhQU0NDQY275w4QLs7OwwduxYaGlpgc/n49GjR7h8+TKcnZ2hra0NS0tLGBkZYdOmTWLZtrKygpycHDp27Ijg4GDEx8ejsrISW7ZsgbS0NPz9/Wm1WU1NDVFRUejduzfMzMxgbW2NESNG4OrVq/D3929w8SUOr1+/RlpaGvh8Pvr168eKt0lAfHw8AgIChL5Ta2tr/Pe//2VsW1dXF6dPn64zxIQOPB5PZLKsfYJdGyZFf4GaBcvFixep+963b19MmTKFlVPEW7duYcmSJejfv79QX09NTYWPjw+6deuG+fPnY+HChVi2bJnY9sPCwuDn54e8vDwAQMeOHWFtbU2dsDLhRx4EptLLXLZdEvbrgs/ns+LZ5fJZZbu/6+rqCtXKmz17Nj58+CDUP5KSkrB69WrcunWLdrslMQbn5+fj5MmTQvfG1NSUlZqST548webNm5Genl5n+Ka449i4ceNgb2+PESNGoLq6GsOHD4eKiorQYeipU6dw4sQJKueHLmz2xxs3bmDFihXQ0dGBtLQ0EhISoKGhAX9/f8qLwKSgO4/Hg5aWllDoV1JSEtTU1ETybuhs5N6+fQt5eXmJ5Ouw3WfqIz8/H7KysrTLGNQFl/PHli1bEBUVhT///BMpKSnQ0tLC69evUVhYCHNzcyqVgQ45OTnYuXOn0CZOsEEsKirC06dPaduWFE0brSYA1HivgoOD8eTJE8jIyEBZWRmLFi3C8OHDkZiYiFu3bmHFihVi1715/fo1bG1tkZOTA1tbW5iamsLFxQWxsbEIDAykXStmwIABiIqKQo8ePbB582b06tULlpaWVCHE2jHVP0vtAfvt27cNvlfcJHJJExgYiPj4eFhaWqJnz54i3kNx2x8ZGfnT722oHsuvQFpaGkJCQvD06VPIyMiAx+PBwsICffv2RUpKCtLS0hgLbxQVFYEQwlqtJaBGPEJDQ6NOD8KaNWtYO/nkou1c2jc0NMTZs2fRvn17oev5+fmYPHky7t27x8rnNBaWLFmCfv36Yc2aNVTdxfnz5wuF4K1evRqVlZXw9fWl/TlcjMGSZO7cuSgvL8f06dOxY8cO2NnZITs7G8ePH4eHhwfGjRsnlj0fHx/ExMRg7dq1uHXrFk6ePIk9e/ZQoeaPHz/GsmXLMGfOHFZErNhi2rRpGD16NOXRePz4MVasWIHOnTvjyJEjaN26NaONlp2d3U9vgujkDKqoqODWrVucjVe1YbvPSBIu54+hQ4di06ZNmDhxIkaPHo2AgAD06NEDtra2kJOTw+bNm2nbtra2xqtXrzB27FgcPnwYFhYWyMrKwrVr1+Ds7MyKSBbXNIUO/qJIuoK9vr4+9PX16/zdoEGDaFehV1BQEDnZtLa2xqZNmxiJG/Tv3x+nT5/GmjVr0K9fP8TFxcHS0hIZGRm0i6AaGhpSA3Z9VdQFp0B0JhxJ5pV4eXmhurqaChsSQLf9XG6euKxeXxcqKir1JtGqqakJqYb9iKioKIwfPx6ysrKIiopq8L10Euxrk5OTA29vb/Tu3RvKysooKiqCgYEBqqqq4O/vL/ZEyXXbubTPtSoVl88q1/192bJlmD9/PuLj45Gbm4u2bdvCwsICQI0i2+HDh3H79m3G4XdcjMHz58+Hj48PVRy2oXtDx/tRm6dPnyI0NBTq6uqIiIhAv379YGpqCjk5OYSHh4u9aF60aBGys7OxYsUKSEtLw8zMjNpkubu7IyQkBDo6OrCyshK7rVz2x6ysLEycOJF6ra6ujpCQEMydOxcrVqxAYGCg2O2tDRsh3w0hSV8B231GQF0RIwKaN28OOTk5TJkyBcuWLaM9prE9f9Tm8+fPVN5knz598PTpUygpKWHx4sVYvXo1o41WcnIyDhw4AF1dXdy8eRNGRkZQV1fH3r17ERcX17TRaoI+b968oVzTb9684fzzkpKSkJycjMrKSpGBi8mi/9u3b3BycoKioiKVbGtqago9PT1s2bKFdmKmjY0NrKys0L59exgbG8PX1xcTJkzAu3fvaItVhIaGUqESTCfxupBkXgkToZH64CqO39jYmPK4MVHj+hn4fD4uXLhQb18X99DCzs4Ow4cPR6dOnRrMNZKSkmK80ZKVlaWeFwUFBbx48QIjRoxA//798fr1a7Htcd12Lu1zrUrF5bPKdX9XV1fH6dOnERERAWlpaZiYmFA5hzdv3sT79+/h5+cHTU1NRp/DxRjcvXt3KtSTjfDAhuDz+ZSar4KCAtLT0zFw4EAYGhoiICBAbHuysrLw8PDA1q1bISUlhd9//536nYGBAXR0dKCvr09rocxlf+zYsSNev34tJEqjpKQEX19fLFy4EBs2bGCUp/m/FCnCdp8RwGWJHQFszx+16dixIwoLC/HHH39AUVER6enpAIAOHTrgw4cPjGxXVFRQKRC9evXC8+fPoa6ujqlTp8LMzIyRbUnRtNH6RamdALhq1Sqoq6tzVq/E19cX3t7eaNu2rUheANNFv7u7O5KSkoQ8Ivb29vD09MTevXtpx+4OHDgQV65cQUVFBTp06IATJ07g5MmTkJeXp/3w6ejoUP8+cuQIbG1tWUmgFXDjxg3q34cPH6bEF7ig9t9SVFQEGRkZWnU+BHwfx+/i4oLz588LxfFXVlYK1ef6WWr3r5YtW2LixIm0xCh+Bjc3Nxw/fhw8Ho+VnK/aaktJSUmsKZjVBdseBK7bzqV9rlWpuHxWJdHf+/btW+fYWlsVjylcjMG1Dzr09PQwbNgwkZBQtlBQUMCDBw8wceJEKCkpISUlBQDw5csXVFRU0LZbVz+vPR7Tgcv+OH78eGzduhVr167F8OHDqXlCW1sbnp6eWLNmDS3FYQFcR4oAQHBw8E/Vm2J6gMlVn+GyxI4ALjzQAkaMGAEnJyfs2LEDAwYMgJubG0aPHo2YmBjGYhXdu3dHeno65OXl0atXL6qP8Pl8lJaWMrItMTguiNwECwwePJikpqZyZl9PT4/4+/tzZjs5OVnk+v3798nw4cMZ2b537x65ffs29drd3Z0kJiYysilgwIABJCcnhxVbdTFs2DDy6NEjzuwTQkhoaCjR09MjPB6P8Hg8MmzYMHL48GFatoyNjcmBAweo148ePSLDhw8nxsbG5MuXL4QQQt6/f094PB6jNg8YMIC8evWKkY2G0NHRIREREZzYNjIy4vQ5TUxMJBoaGiQoKIgUFRURHR0dMn78eKKlpUUcHBwY2ea67Vzbr01hYSG5dOkSa88vl88qV/29qKiInD17lhw+fJikp6eL/L60tJR4e3sz/hwux+BBgwaRFy9esGKrLsLDw4m6ujq5cOECycjIIKqqqsTR0ZFMnjyZWFhYiG2vurqa+Pr6En19faKtrU1Wr15NcnNzhd7DxhjJdn/89u0bcXBwIP379xf6LgVcv36dDBw4kHa77927RyorK6l/N/RDB2VlZTJy5Eiir6/f4I+BgQEt+7Vhu88IUFNTq3McePXqFVFTUyOEEPL27Vvq33Tgcv4oLi4mS5cuJceOHSN8Pp9YWVkRZWVloqqqSi5evMjItr+/P9HR0SGxsbHk0aNHRENDgwQGBhJzc3NiYmLCyLakaNpoNQLGjx9P7t69y5l9dXV18ubNG05sa2pqkoyMDJHrGRkZRF1dnbbdixcvElVVVXLo0CHqmo2NDVFVVSXXrl2jbVeAq6srWbVqFUlPTyfl5eWM7X2Pvr4+efLkCet2BZw8eZL079+fuLm5kWvXrpGrV6+S7du3EzU1NXL69Gmx7WlqapLs7Gyha5mZmWTw4MFkwYIFpKKigpVFhIWFBQkODmZkoyE0NTU520Dr6enV2dfZJC8vj/oeMjIyiIuLCzl06BDjPsp127m0//z5czJmzBhy//59UlxcTPT09IiysjJRU1NjZdzk8lnlor+/ePGCDB06lGhqahJNTU2ioqJCdu7cKfQeNp5VrsfgmTNnkujoaMZ2GuLatWvUxvD8+fNk0qRJZNGiRbTGiICAADJgwADi6+tLDhw4QEaMGEF0dXWF+s779++JsrIyozZz1R+/fv1Kvn37VufviouLyYULF1j5nKysLJKSkkK9DgkJIVlZWbTtKSsrkw8fPrDQsp+DzT4jYNy4cXUegh4+fJiMHj2aEEJIQkICGTlyJO3PIIS7+eN7+Hw+SU5OJvn5+azYOnz4MPn7778JITXPmba2NpkwYQJ5+vQpY/uSoCl0sBEwYsQILF68GCNHjoSCgoJI0jdTd7i2tjYePnyI7t27M7JTF5qamjh06BC2b99Oxd4TQhAaGiqW6MD3+Pv7w87ODvPmzaOu7d+/H0eOHIG3tzeMjIwYtTsuLg7Z2dn11n9gKuNqbGwMKysrTJkyBQoKCiKqgEzzeUJCQrBx40ah+zN69GgoKCggNDQUM2bMEMse13H8Alq3bg0PDw/4+/tDUVGRFfnf2gwfPhxxcXGYO3cuIzt1MX/+fNjY2GDu3Ll1Kj3SFZSpzevXr1FVVYUePXqgd+/eaN68OTQ0NBiHFXPddi7t79y5EwoKClBSUsLFixdRVVWFuLg4nDp1Cl5eXjh16hSjtnP5rHLR393d3TFgwADs2rUL0tLSOHLkCPbs2YNPnz7Bzc2Ndlu/h+sxmMfjYd26dTh06FCd94YNEShdXV18/PgRQE2R+t9++w06Ojq0whXPnj0LV1dXjB07FgAwb948LFu2DAsXLsSJEyeoMHSmMuRc9cfadhITEynFyry8PCgqKgoJZtDlzp07WLp0KczNzdG/f38ANWFzXl5eCAwMpPKTxEESsu61YbPPCFi6dCns7OwoafTvS+xkZWXB3t4eY8aMYdR2ruaP73PxpaSksHbtWsa5+EDN97tgwQJ8+vQJQI3ojK6uLvr3789IUE2SNMm7NwIMDAzq/Z2UlBRjJbazZ8/Cw8MDxsbGUFJSEnkomCwkHj9+jAULFqBDhw7UwPrkyRN8+vQJwcHBtOuAaWho4OLFi0ILf6BGWWfixIl49OgR7TYDP5YzZ6rC11AhYSax6gLU1dVx8eJFkTpa2dnZmDhxIh4/fiyWvd27dyM6Olokjh8Arl69ijVr1qB///549OgRo7ZzrbAZHByM/fv3Y9iwYdREUxsmhxZcf6fR0dHYuHEjbG1tYWlpCQBYuXIlbty4AS8vL0YLW67bzqV9bW1tnD59Gr1798bSpUvRtm1b7Ny5Ezk5OZg0aRL++ecf2rYBbtvORX/X0dHByZMnhfJLr127htWrV2PBggXYsGEDI7luAVyPwT/K82qokOnP8OTJE1hYWGDatGlUPpuBgQEqKysRFBSEfv36iWVPS0sL58+fF7ofZWVlmDdvHj59+oSTJ0+iWbNmjO87l/2xpKQEVlZW+OeffyAlJYWrV69i+/btyM7OxuHDhxnnEk6fPh3Dhg2Dra2t0PU9e/bg/v37tA5FeDwebt++LRF5d7b7TG24KrEjgMv5w9HREbdv38b27dupfMRr167B09MThoaGjOpoZWdnw9raGoaGhtiwYQMAYPDgwejcuTMCAwMhLy9P27bE+Jc9ak0wpLq6mrENZWXlen+YhpcQQkhOTg7ZtWsXWbx4MVm+fDnZs2cPY5fyf//7X3L27FmR6+fOnSOGhoaMbBNCSGRkZJ3u9NLSUtp5TpLkv//9b51hNxcvXiT6+vpi2xPE8auqqnISxy8gNze3zj5dWVnJSl4ClzH8b968afCHKRMnTiRHjx4VuR4aGkomT57MyDbXbefS/oABA0h2djaprKwk2traJDIykhBCSFpaGtHV1WXcdi7hor/r6enVGVp24sQJoqysTA4fPsxK6CDXY3BDFBQUMLYxb948YmdnJzTOV1ZWEjs7O7Jw4UKx7U2cOJGEh4eLXC8oKCCjRo0iEydOJM+ePWNlTuUKJycnYmJiQrKzs6lw8YyMDDJ16lSyZs0axvY1NDREQtAJIdTn0cHb25uUlZWJXC8sLCRXrlwhSUlJtOzWBdt9RpJwOX9wmYu/ePFiYm1tLfTMFxYWEisrK2JjY8PItqRo2mg1AgwMDMjHjx9Frufl5REdHR3JN+gXICQkhGhqapK9e/eSGzdukBs3bpB9+/YRbW1toZwBcSgsLCS5ubkkNzeX8Hg8kpqaSr0W/Pz111+MElIFmJmZkc+fP4tc//DhA5kyZQpj+4cPHyY6Ojrk+PHjJC0tjaSlpZFjx44RHR0d4uPjQ9su13H8PB6PFBYWilzPyspilNMnCezs7ChhkNp8/PiRLF26lLF9dXX1ehcpTO8N123n0r6lpSVxcHAgu3fvJqqqqqSwsJDk5eURa2trsnjxYka2CeH2WeWiv9va2hIzM7M6NyMeHh6Ex+OR7du3M17wczEG16a+e5OTk0N7UV6buvJOCSHk5cuXRFtbW2x7Z86cIaqqqmTLli0iwgYZGRlET0+P6OjoML7vXPbHUaNGkQcPHhBChO9PcnIyGTJkCCPbhNQcdF29elXk+vXr1xktyH18fIiOjg513x88eEAGDBhAHRibm5uTr1+/0rYvgO0+U5vExEQSEBBAfHx8iLe3t9APG3A5f3CVi09IzUFaZmamyPX09HQyaNAgRrYlRVOO1i8K1wU5Jcn169cREBCA9PR0yMjIoE+fPrC0tMTo0aNp21ywYAEqKipw5MgR+Pv7AwC6du0KW1tboZwBcYiPj6eq2BNC6sxjIoRg5MiRtOzHxcVRcrD379+Hn58fWrVqJfSe169fIzc3l5b92syfPx+5ublwc3NDdXU1CCGQkZHB7NmzsXTpUlo28/LycO3aNcjKymLkyJEisq1t27alFcd//PhxBAcHA6i5v9OnT6fy+QR8/vyZdo0VLuu4PHjwADk5OQBqCvSqqqqKyDtnZmbi7t274jW6DuTl5ZGYmCgSqvXw4UOqtos4cN12Sd2bLVu2wNbWFjk5Odi0aRM6duwIFxcXZGZm0i62yuWzynV/37BhA6ytrTFixAgcPHgQw4cPp34nkHcPCgpiPHdwMQafOXMG58+fB1Bzb5YvXy4SKlVQUMCoVIWA33//HTk5OSLPU0FBAa2ckunTp6N58+Y4deoUioqKhCTYe/fujZMnT2LTpk1ISkoS27ak5o6ioqI6x5K2bduirKyMkW0AmDJlChwdHfHp0ycqbSAlJQVeXl60UxTCwsLg7+8Pc3NzKnxw06ZNaNGiBU6dOoU2bdrAxsYGBw8exMqVKxm1n+0+I4DLEjsC2J4/asNVLj4AyMjI4PPnzyLXv379KtFi1UxoytH6RXn37h02btwIQggSExOhqalZZ0HOOXPm0Fr4q6ioULUtGqpKDjATfrh69SpWrVoFQ0NDDBo0iPp7/v77b3h7e8PQ0JC2bQEfP35E8+bNWanTk5iYCD6fjwULFsDb21uoNo/gnvfr149WnHRGRgYWL14MQgjevXuHbt26CS2wBPbnz5+PmTNnMv5bgJqY+5cvXwKoEa+ge4+SkpJgZWWFb9++AQBatWpF5Tox5evXrwgKCgIhhBLXqF3sE6iZ4MaMGUNLsOVn+jqhWcclOTkZpqamAEBt0L+nVatWsLCwYDxZhoaGwsvLCwsWLBBapISGhmLZsmVU3P2v0nZJ3pvvKSoqQrt27WgnS3P5rHLd34GaIp8PHjxA37590blzZ5Hf37t3D9HR0XB2dqZl/3vYGoM/ffpEFZmOjIzEuHHjRMQefv/9d0ydOpXK+aWLi4sLbt26BUdHR6irqwOoeZ6cnZ0xcOBA1u7N9xQUFKBr165i/R9JzR3z5s3DiBEjsGjRIqGcM2dnZzx79gwnTpygbRsAqqqq4OrqirNnz6KyshJSUlJo1qwZzMzMsGbNGlrz6owZM2BsbEwJHKWkpGDmzJmwtbXF4sWLAdTkP7m7u9crbvWzcNVnhg0bBjMzM6q9XMD2/FEbrnLxgZqDo5cvX2LPnj1UznlOTg42bNgAeXl57Nmzh7ZtSdG00WoEcFGQMzIyEhMmTICsrCynwg9Tp06FkZGRyELKx8cHcXFxOH369E/bSkxMhJaWFmRkZH5YGJepStr9+/ehra0NGRlunL4GBgY4c+YMOnbsyLrtR48eQVlZWWiBcvXqVXTt2hWampq0bJqZmaF169ZwcnJCs2bN4OzsjMzMTFy8eJGlVtfg4+MDS0vLnyo++bPU/i7v37/f4HuZFBbl8Xi4detWnQtbtggMDMSRI0fw/v17ADUehEWLFtH2IAjguu1c2//27RsuX76MzMxMWFpaIj09HX379kWHDh0Y2+byWeWivzfEp0+f0Lp1a9rjmiTHYHt7ezg4OHBWBLysrAyrVq3CzZs3hQ5fRo8eDTc3N9qfm56eDkVFRcrDcfPmTcTFxaFTp04wMTFh3I+47I/JycmwsLDAsGHDEBcXh0mTJiEzMxNPnjxBUFAQdHV1Wfmc0tJSZGZmIj4+HoMHD6alNihAS0sLkZGRUFRUBFAzRu7ZsweRkZGUcEhOTg7Gjx9PeQXpwlWf0dDQQExMDCfKz7Xhav4Aau5xeHg4Xrx4ARkZGfTu3Rtz584V+1DhewoLC7Fw4UK8ePGC8mR//vwZqqqq8PPzY+yNkwgSDFNsgiG5ubkkPj6efP36ldW6ES4uLuT169es2atNfYX46OQg1K6XIRDq4ErAg5CaZHo7OztiYmJC8vLyyLFjx0hCQgIrtgWUl5eTzMxMUllZSSoqKhjb27ZtG+HxeCL1g6ysrAiPxyNubm607GprawsVD83LyyM8Hq/OvBumfP36lURGRpJdu3aRjx8/knv37pGioiJWbC9fvpzzWleEEE5qr9WmqKiIk3tPCPdtZ9v++/fviYGBAdHQ0CAqKiokOzubLF68mPXaXWw/qwK46O8JCQnExsaGEh3Kz88nJiYmhMfjEU1NTeLr60vLrqTHYEJqEupPnjxJvnz5Ql68eEEVv2WLzMxMcunSJfLXX38xqudUUlJC5s+fT3g8HtXvwsPDCY/HIyNHjiRjxowhw4YNEyliTBeu+mNaWhpZv349mTBhAhk3bhxZuXIl+eeffxjZ/D6HKjk5mcqh4vF4jHKoNDU1hdYYixYtEhHBSUtLYzWfh60+I8Dc3Jy1OmU/A5fzBxdUVVWR+Ph4EhQUREJDQ8nt27cJn8//t5v10zTlaDUCKisrsWHDBly6dAnS0tK4cuUKdu7cidLSUnh7ezM+8YuMjIS5uTk7jf2Orl274vXr10Lx6kBNPHmbNm3EsnXmzBnqRIOppP2PSE1NxZw5c6CpqYnU1FRUVFQgLS0NO3bsgK+vL+08rdrs2rULR48eRWVlJa5cuYK9e/eiZcuWcHR0pBVCcfr0aZw7dw47duwQOU0OCAjAuXPn4OjoCBUVFbHj4cvKyoTqhHTr1g3NmzdHcXExqyfOHz58gImJCQoLC1FRUYFZs2YhODgYqampCA0NFZKspkNCQoJIriObnDx5EoGBgcjLy8OVK1dw6NAhdOvWDcuWLaNlT5IeBLbbLin77u7u6Nu3Ly5evIihQ4cCqKmttXr1anh6elL5Q0xg+1kVwEV/v3fvHiwsLIRyI+zt7fH06VNs3rwZbdq0gYeHB+Tk5DBt2jSxbEtyDP5ealxPTw+7du1iTWpcgJKSEpSUlBjbCQgIQE5ODg4ePIhevXqhoqICnp6e4PF4CAsLg6ysLDZs2ABvb2/GZSq46o9AjffZw8ODkY3a1JVDZW9vz1oOVb9+/ZCcnAwFBQV8/vwZ9+7dE0lJuHTpEiPp9e9hq88ImDhxIlxcXJCamspqiR0u54/aHmeuy7IIyiLUzjdtVPzbO70mfoyXlxcZO3YsSUhIoFRvEhISiL6+Ptm2bRtj+7a2tmTHjh2cnHDs3buX6Ovrk9jYWPLlyxfy5csXEhsbS/T19cn27dvFsqWlpUXevXtHCKlRXiouLma9vQLmz59P9uzZQwgRVhpyc3Mj06dPZ2w/NDSUDBs2jJw9e5aSvI2OjiY6OjrU54qLsbExOXHiRIPv8ff3JzNnzhTbdu2TbAH1KTAxYe3atWTx4sWkrKyMsv/p0ydibm7OioKcq6srWbVqFUlPT2fds3L+/HkyYMAAsn//fkrhKTQ0lKipqZGgoCBaNiXlQeCi7ZKyX1vOvHafZOsUm4tnVQAX/d3CwkJoXsjOzibKysrExcWFuhYREUFrHJPkGMyF1DiPxxN5nur7EZcxY8aQa9euUa9v3rxJlJWVhSTf79+/T/T09Gi1XQCX/ZHP55OzZ88SJycnYm9vT+zs7IR+6DB9+nRy7Ngx6vXjx4+JsrIy8ff3p67duHGDjBkzhpb9c+fOEU1NTbJ9+3YyY8YMoqKiQpVGyMvLI4GBgURVVZVERUXRss9lnxHAVYkdLuePefPmUc//vHnzGvwRFwMDA8qrLyi9Ut9PY6DJo9UIiI6OhqOjo1B8tK6uLrZv344NGzbA0dGRkf33798jJiYGoaGh6NSpk8iJP5OTy6VLlyI9PZ2qFg7UCA+MGjUKa9asEctW8+bNcfr0aejq6uL+/fu4f/9+vXlrTE/3nzx5Uud9nTt3LsLDwxnZBmpO+bZu3YrRo0fDxcUFADB+/Hg0b94cO3bsECno+DO8evUKenp6Db7HyMiIthKbJEhISMDBgweFclbatWuHjRs3Yv78+Yztx8XFITs7u96kaCbCL8HBwXBwcICxsTGlKjd//ny0atUKgYGBsLCwENumpDwIXLRdUvZLS0tFFNgEVFVV0bYrgItnVQAX/T0lJYVSFwSAO3fuQEpKSkjlVV1dnVbiviTH4L///hu7d+8WUknr3bs3tm7diuXLl9Oy6erqSkVSMD1l/553795BRUWFep2YmAgpKSnKywoA3bt3R3FxMaPP4bI/7ty5EyEhIVBWVmZF2RGoURatPS8lJCRASkpKKCqkT58+P1SErY/JkyejoqICJ0+ehLS0NPbu3UsJVQQEBCA8PBzW1taYMmUKLftc9hkBz54948Qul/OHmpoaJXC0c+dOyMnJiain0sXY2JjKMRfX6/4r0rTRagTk5+dTaiu1kZeXZzxoAzWbNraSXL/nt99+w4EDB5CZmYn09HQQQqCsrEwrJMbc3Bz79++Hr69vg5KndNTjvqd58+YoKSkRuf7u3TtWEtffvHkjNCkL4PF4VKKquMjKylKqgA1BV4ktODhY6G+vqqrCkSNHRBZaTBTkuF4005W2/xmysrLqTOrW1dWlrUY1f/58xMTEQE5ODvb29vDx8WFtAVQbLtouKfuDBg3CyZMnhcJXKisr4efnB21tbUa2AW6eVQFc9PevX78K9ZHExETIyspCS0uLutasWTNa8u6SHIO5kBrft28fhg0bhm7duiE3N5dVIZKWLVsKtevevXvo3r27kMDB27dvGYtacdkfo6Ki4ObmxvritnZfS0pKQrt27SihCqDmOWDyPcyYMaPOciyLFy+GjY0NI1EcLvsM13A5fxw7dgympqZo164dDA0Ncfv2bdYEWuLj42FiYoKWLVviP//5D8aPH89IPv/fpmmj1Qjo3bs37t69KyLbGh0djT59+jC2z7a0cm0qKyvx+vVrfPnyBf369YOioiLthf7SpUthZmaG4uJiGBoa4vTp05woLwE1nh8vLy/s3buXupaZmYnt27dj1KhRjO13794dKSkp+M9//iN0PT4+XqTOxc+iqqqK2NjYBmPRr1+/Tiu2/I8//sClS5eErnXp0kXklIxpzQ+uF81MFDR/ROfOnZGVlVVnnRK6ykuS8iBw0XZJ2d+4cSPmzp2L+/fvo7KyEo6Ojnj58iW+fPmCY8eOMbINcPOsCuCiv//nP/9BRkYG/vjjD1RXV+POnTsYOHCg0EIlISFB5O/5GSQ5BqupqeHSpUtYtGiR0PXjx4/jzz//pGWzuLgYz58/R7du3eDr6wtTU1PWFs1aWlq4cOEC1qxZg8zMTDx69EjEK3ns2DFGUtcAt/2xvLyc9UPXfyOHSgAbeXxc9RlJlNjhcv7o3r07VqxYARUVFRBC4OrqWm/+s7iewGfPnqGgoABdunSBvb09RowYwdk4IwmaNlqNABsbG9ja2iIjIwPV1dWIjIxEVlYWlQTLBs+ePUNoaCiysrKwb98+/PXXX+jbty9tueu8vDzs2bMH165dE/KytGzZEuPGjcOqVatoLbBat26N1q1b48iRI1BRUeFMfn3jxo2wsrLC4MGDwefzMW3aNJSUlIDH42HDhg2M7VtaWsLJyQnv378HIQR3795FWFgYjh49Cjs7O1o2TU1NsW7dOvTt2xf6+voiv79x4wYOHDhAK9T0xo0btNokLlwvmoGa8MGgoCC8fPkSYWFhiIiIQM+ePWmHlggwMTGBs7MztWh++fIlbt26RdUuoYOkPAhctF1S9nv37o1z587h5MmT6Nq1K/h8PsaNGwdTU1Nam4nv4eJZFcBFf58wYQLc3d1RWVmJW7duoaioCNOnT6d+//jxY/j4+GDOnDm07EtqDF6zZg0sLCzw+PFjVFVVwc/PT0hqnA4jR47EokWLqLpuDYVai/s8LVu2DPPnz0d8fDxyc3PRtm1bKiQ2ISEBhw8fxu3btxmPY1z2x2HDhuHvv/9mRe5bwNy5c7Ft2zakpaXh4cOHqKiooJ75/Px8XLhwAUFBQdi+fTtrn8kmXPUZNzc3zkMSuZw/PD09ERAQgNzcXEhJSeHt27eMhVgEaGtrY86cOejcuXO9Rd0FcC3KwwZNdbQaCfHx8QgICMDTp0/B5/PRt29fWFtb47///S9j27UV9h4+fIhLly4hICAAUVFRtBT2cnNzYWJiAhkZGUybNg39+vVD27Zt8eXLF6SmpuLcuXMAalTyxDlxkqTKjYC7d+9S97xfv34YPnw4a3HIYWFh8PPzQ15eHgCgY8eOsLa2xsKFC2nb3LlzJw4fPgwVFRVoa2ujbdu2+PTpE5KTk5Geng4TExPGOX1cU1BQgJMnTwr1dbYWzbdv38aSJUswYcIExMTEIDo6GmfOnEFgYCDc3NxoqzsJ2LNnD0JDQ1FeXg6gpqr97NmzYW9vT9uTW1JS8lMeBKY1WLhouyTtcwkXz6oAtvt7RUUFNm/ejAsXLkBaWhqmpqZwcHAAUKPQGBISAh0dHQQGBoqtwCnpMfjZs2cIDg4WujcWFha0vUKVlZW4efMmiouLYW9vj02bNtWrfkvH+/3ixQtERERAWloaJiYmVMi/p6cn7t69C1tbW1aU09jsjz4+PtS/CwoKEBERgf/+979QUFAQmevoRiucOXOGyqGysrKi1i3Ozs5UDtWqVato2eYarvsMUJMHNn/+/DpTRJgiifnDwMAAZ8+eZaVuIVDjRTx//jyKi4vh4+NTZ1F3AVxGZLFF00arkVNWVlZvjP/PYm5uDg0NDdja2gpVg9+xYwcePHiAM2fOiGVv/fr1yM7ORlBQUJ3S3yUlJVi8eDFUVVWxadOmn7ZrZmYGX19ftG3bFmZmZg2+9+jRo2K1WRxCQ0NZOeUXUFRUBEIIJX1LCKGVPyEgLi4OJ0+eRGpqKoqLi9GxY0doaWlh1qxZQonZdGkozKF58+aQk5PDlClTsGzZMkZ/x/c8efIEqqqqjGzMnj0bY8eOhbm5uVBfDwwMxPnz53HhwgXG7fz69SsyMjJACIGSkhJr8vdcF9EGuGs7l/YldegCsP+sNgTT/l5SUgIpKSmhBcr9+/dRUlICfX19Wu3+VcbgvLw8yMnJMbIh6WLRXMBGfzQwMPip90lJSbHuPcjPz4esrCxrC3Su4arPDBgwAOfOnWPlMLE+JDF/cAHXhcslQdNG6xfm69evSEhIgIyMDAYNGkSpsAiIjY2Fk5MT/v77b0afM3DgQJw+fRq9evUSWnxmZ2djypQpePjwoVj2hg0bhl27dmHw4MH1vufWrVvYunWrxELSfpagoCBER0ejefPmmDJlCkxNTanfvXjxAps3b8bjx48ZhWmlp6dDRkamzlypZ8+ewcHBAWfPnqVtn2uOHDmC3bt3Y86cOZTAwcOHD3H8+HHMmTMH7dq1w5EjR7Bw4UJYW1v/lM3Hjx/j0qVLkJGRwYQJE4QSpcvLy+Hl5YWjR48iNTWVUdu1tLRw7tw59OzZU6iv5+TkYOLEiXj06BEtuyUlJZCRkRF5RoGaU2InJyf4+vqKbVcSHgSu2i4p+98v+Kurq5GdnY2SkhJMmDCBUUgSV8+qpPp7Y+Tdu3e4fv06ZGRkYGhoKCKIcezYMezduxcPHjwQ23ZUVBSVWB8VFdXge5l4t+urWSQlJUUdRtHJH2rsc0djRBJ9Zs2aNejatStWrFjB6oaCy/mDyxwzSdaPlASNa2v7/xFpaWmwsrKiTqy6d++Oo0eP4o8//kBxcTGcnZ0RHR3NuIArwL7C3qdPn37oAu/duzcKCgrEtl2bt2/fom3btmjdujUSEhJw9epVaGtrY+LEibTs7du3D35+ftDV1cVvv/0GNzc3SEtLY/bs2QgKCoKXlxdatWpFe0Gbk5ODZcuWISMjA0CNxHJAQADat2+PyspKeHt7Izg4mLYqlYeHB1asWMHYw/kjoqOjsWnTJpiYmFDXjIyMoKSkRIWI9O3bFx4eHj+10YqJicG6desgKysLGRkZHD58GIcPH8agQYPw8OFDbNiwATk5OawoYbVp0wYFBQUi/TMjI4PWfS8qKoK9vT3i4+MhJSWFMWPGwN3dndpUhIWFYdeuXaisrKTV3jdv3oDP51P/ZhOu2861fQF1eU4IIXBxcak33ORHcPmsSrK/m5mZ1bkAqr3gnzJlCu3FCttj8N27d7F06VIqr3f37t04duwYlJWVkZOTgw0bNuDhw4cNHuI1hJ2dHYYPH45OnTo1mM8kJSXFaKNlbm5OPbeCs+zvvwcdHR14e3v/lAoc13OHAEIIfH190blzZ8yePRtATX6lvr4+lixZwsh2Y0USfYarEjtczh+1c8zc3NxY9eybmZnh9u3b6NSpEzWG1eUTYkPdVCJIolhXE+KzcOFCMmnSJJKYmEgePXpE5s2bR1auXElevnxJRo0aRVRVVYmXlxcrRVc3b95MLCwsSHFxsVBhyEmTJhF7e3ux7dVV3PZ73r9/z6gQ39WrV4mqqiq5desWef36NVFVVSVjx44lGhoaQsURxWH06NHE19eXeh0ZGUnGjx9P9u/fT5SVlcmqVatIYWEh7TYvW7aMjBo1ikRGRpLo6GgyceJEsmnTJvLhwwdibGxMlJWVyfr168nHjx9p2a9dWFGAtbU1yc/Pp93mulBTUyOvXr0Suf7q1SuipqZGCCHk7du31L9/xNSpU4mNjQ0pLy8nlZWVxMXFhZiampK//vqLqKqqEkNDQ3Lnzh1W2u7h4UGMjY1JWloa0dLSImlpaSQuLo7o6+uTHTt2iG1v/fr1ZODAgcTHx4ccPHiQDBs2jLi7u5OysjKyePFioqysTObOnUuysrJYaT+bcN32f/vevH79mgwePJjW/+XyWZVkf9++fTvh8XjE2NiYbN++nWzfvp3MmjWLKCsrk+XLlxMLCwuiqqpK/vrrL7FtczEGm5iYkHnz5pG3b9+SDx8+kJUrVxJLS0uSnJxMBgwYQAYNGkROnz5Ny7YkOX/+PNHX1yfXr18nnz9/Jp8/fyaxsbHEyMiIBAcHkwcPHpCZM2eSLVu2/JQ9rucOAXv37iW6urrk6tWr1LWQkBCiq6tL/Pz8GNluon68vb0b/Pn/jTdv3hA+n0/9u6GfxkDTRusXZdCgQeTu3bvU6+zsbKKlpUUmTpxIJk6cSNLS0lj7rC9fvhATExOioqJClJWVycCBAwmPxyNTp06lNXArKyv/cEPCdKM1depUsmfPHlJdXU18fHzI6NGjSXV1NYmOjiZjx46lZVNdXZ28fPmSel1eXk54PB4ZMGAAiYyMpN1WAYMHDyY3btygXj979ozo6OgQU1NToqenR2JjYxnZr2uDK9g4s8m4cePI4cOHRa4fPnyYjB49mhBCSEJCAhk5cuRP2dPU1CRPnz6lXhcXFxNVVVUyePBgYm9vT0pLS9loNiGEkIqKCrJmzRqirKxMlJWVCY/HI8rKymTx4sXk69evYtvT09MjFy5coF4/ePCADB8+nCxbtoxoamqS48ePs9Z2QgjJzc0lX758IYQQcvfuXeLk5CT0+eLAddslfW++Jz4+nmhra9P6v1w+q5Ls7zY2NsTFxUXkuru7O7G1tSWE1Dy3M2bMENs2F2OwtrY2efjwIfX6/fv3RF1dnejr65OFCxeSvLw8WnZ/hsLCQnLp0iWSk5PD2JaRkRGJj48XuX779m1qjExOTiZ6eno/ZY/ruUPAyJEjyc2bN0Wux8bGEgMDA1Y+438JNvuMJGBz/qhNeXk58fPzow5gN23aRDQ1NYmFhQUpKipibL82FRUV5PHjx6SkpIRVu1zSFDr4i1JSUoJevXpRr3v06IHKykp06tQJ/v7+deY70KV169Y4deoUqwp7DdVUAEApj9ElMzMTPj4+kJaWxu3btzFy5EhIS0tDU1MTubm5tGyWl5cLhXHIysqiRYsWWLNmDWM1OgD4/PmzUKFJZWVllJaWoqysDOfOnaMSmn91li5dCjs7O6SkpEBLSwt8Ph+PHj3C5cuX4ezsjKysLNjb22PMmDE/Ze/r169CeRht27alclc2b97MatubN2+O3bt3Y+XKlUhLS6P6ep8+feoMTfgRnz59EioEq62tjcLCQqSlpeHMmTOshPYKuHbtGmxtbREQEIAePXrAysoKPXr0QEREBIqLizF37txfqu2Sujd15R6Ulpbi9u3bIvV6fhYun1VJ9vebN28iIiJC5LqJiQmlkGZoaIh9+/aJbZuLMbisrExI/axz584AAA0NDezevZs1xVegJt/JxsYGrq6uUFZWxuTJk/HhwwfIysri4MGDtMMTgZpQsD/++EPkerdu3SilwG7duuHLly8/ZU9Sc8enT5/qVJ9TVFRkXAz5fwEu+wzbJXa+h+35oza7du3CuXPnMHz4cMTHxyMyMhIrV65EbGwsPDw8GAkSvXv3Dg4ODli9ejX69euHGTNmUKH+ISEhdRbv/tVgb9RqglX4fL6IOoyMjAxWr17N6iYLqKke/vnzZwwZMgSWlpawtrbGyJEj8fHjR1objEGDBuH9+/d48+ZNvT/v37+nhBToIJCL//LlCx4/fkyp6WVnZ6N9+/a07dYFG0p9QE2S/vd1Jpo3bw47O7tGs8kCgEmTJsHHxwcFBQXYs2cP9W9/f39Mnz4dHz58wKRJk7B+/fqftvl9fLeUlJRQDhhbGBoa4tOnT1BQUMDYsWMxfvx49OnTB/n5+bQmyaqqKpHnUVZWFlu3bmV1kwUABw4cgKWlJYYMGYILFy7gjz/+QHR0NNzc3GjV5uG67ZK6N3WNL6WlpTA3N8fWrVtp2eT6WZVUf2/dujVevnwpcj0jI4PKvy0tLaU1p3AxBhNCRDZT0tLSWLRoEaubLKCmFIaCggKUlJRw8eJFVFVVIS4uDpaWlvDy8mJku3///jh06BCqq6upa9XV1Th06BAlfHL//v2fVpmT1NzB4/Hq3JifO3cOffr0Ye1zGitc9ZnU1FTMnDkTb968QWpqKioqKpCWlgYLCwvExcWx0na254/aXL58GXv27IGqqiquX78OHR0dLFmyBJs3b0ZsbCwj2zt27MCXL1/QsWNHXLp0Cbm5uThx4gRGjx4NT09PRrYlRZNHq5HB1qAaFxeHlJQUADUKL/7+/iIiCq9fv6Z1MsmlrK+AkSNHYuvWrfj999/Rpk0b6Onp4c6dO3B0dMSoUaNY/Syua/zUdfJJF66kpr9HX1+/zqLIQM1Gmw0lILYOFGJiYnDz5k0ANTXenJ2dRbytgqKLbMH2JgvgxoNQF1y0nUv7khhvBLD5rH4P2wdoADBt2jRs2bIFRUVF0NDQoLzP+/fvx5QpU/Dx40d4eHjQel4lOQZzIe388OFDnD59Gp06dcLNmzcxcuRIdOvWDdOmTcPhw4cZ2bazs4O5uTnu3buH/v37g8/n4+nTp/j8+TMOHTqE5ORkbN68mbEHk+3+uHz5cixevBhJSUnQ1NQEAKSkpOCff/5hpAz6vwJXfWbXrl2wsLCgSuwANVFBv//+O7y9vcWuZVoXXM4fnz59osb127dvU4dG7du3p4Rt6JKQkIDQ0FD85z//wa5duzBixAhoa2ujQ4cOrAgGSYKmjdYvipSUVJ2nnmzRvXt3ODs7U+FSMTExQieGUlJSaNWqFTZs2CC27bdv30JeXp6qFt4QdCeKLVu2wMvLCzk5OfDz84OsrCwePHgATU1NbNy4kZZNAAgODhZSWqyqqsKRI0dE1JzoFMmr6ztlm+9DNisrK+Hp6Smivsa0tlBSUhKSk5NRWVkpEnJH5948fPhQ6B4TQvD48WMqzEYAnQWhlpYWTp06RbXz+wr2gr6+c+dOsW1L4jsV8L0HQaDoSNeDwHXbJXVvsrKykJycjMLCQrRr1w4aGhqU1yA2NhYtWrQQ21vJddu57O+1WbVqFSoqKrB9+3aUl5eDEIIWLVrAzMwMq1atQmxsLMrKyuDq6iq2ba7G4Ly8PJHQ8vz8fJEDL6abDGlpacjKyqKqqgr379/Hli1bAND38NWmf//+iImJwalTp/D06VPIyMhg2rRpMDU1RceOHZGZmYmAgADo6en9lD1JPUvDhw/H8ePHcezYMdy6dQsyMjLo3bs3zpw5I1SC4P9XuOozqamp2LZtm8j1uXPnIjw8nLbd2rA9f9SmZ8+eSElJQWFhId68eUMV5f7rr78Y1warrKxEu3btQAjB3bt3sWbNGgB1R339qjTV0fpFqasuAamnGCFTeUsDAwOcOXOm3orh4vIz9RUEf8uvJM3JdeFGHo8HeXl5oQ3t27dv0a1bN5FFBB37PyogWhsmXgBfX19Klvj7k2Y690bQR340FLHRX8zMzODj48NYBlkAj8eDlpaW0MYtKSkJampqIl6zI0eOMPosBwcHvHjxAr///juePXuGuLg4JCUlwdHREYMHD4azs/Mv1Xau7RcUFMDBwQG3bt0S6jtSUlLQ1taGi4sLlixZgg0bNsDIyEjstnP1rEqyvwv49u0bMjMz0axZMygqKnLiPWODn5n32Jo7rKysICcnh44dOyI4OBjx8fGorKzEli1bIC0tDX9/f0b22YTruaOJn4OrPjNkyBAcPHgQampqQvUd7927B1tbW9y5c4dx29meP2oTFRVF3QMtLS2EhITA19cXvr6+cHNzY5TjPnfuXOjq6qJLly5wdnbG33//jY4dO2Lr1q3Izc2VaEQDXRrHdvD/Q5h6HMTh+6LBlZWVePbsGZSUlGjVoQkNDaUWskwXlw1x4cIFDBo0CHJycjhw4ABiYmKgra0NBweHBoU46oPr4sl0PD3iIKkB5+TJk7C1tcXixYtZsSfJhcH396ioqAj3799H//79aZ28CQQFalNXMjkbsO1B4LrtXNovKSnBggULwOfzsXPnTgwZMgQdOnRAcXEx7t27B19fX0yePBkaGhpib7IAbp9VSS+Ev379ivT0dMr7LAgZB5h7zNgeg7mcL75ny5YtsLW1RU5ODjZt2oSOHTvCxcUFmZmZCAwMZGS7rKwMISEh9Xr9xf07uZ47BHBZxPl/Aa76jJGREby8vLB3717qWmZmJrZv385aGC5XHmigpl/weDy8efMGI0aMAACoqakhKCgIQ4YMYWR748aNWLJkCT5+/Ahra2vIycnB0dER169fx6FDhxjZlhRNHq0mRFRdpk+fjszMTNZUXV69eoWSkhL0798fQM1GbOTIkVBUVKRt88CBA/D390dISAgIIZg7dy5mzpyJ+/fvY8SIEXBwcGDU5n+T+jyXP2L+/Pk/9T4pKSmEhoaKbV+AhoYGYmJiONtQcMn3qlHjx49nTTWqCcnh4+OD6OhonD59us78nYcPH2LOnDlYuHAh40VEY+b69euws7NDSUmJyGKfqVfof3EMLioqQrt27Rjn5a5btw7Xr1+Hnp4eVdS1NpI8SBWH+sIDf/vtN8jJyeHKlSsSbtGvDxt9pqSkBFZWVnj8+DH4fD7atGmDkpIS8Hg8HD58mHWBL65helj/PXw+HyUlJZQqdFZWFjp06NBo7kuTR6uRkJycDEVFRXTs2BFRUVG4dOkStLW1sWjRIsax29+rurx79w4nTpxAREQEPD09ERwcTNv2nTt3sHTpUpibm1MbrejoaHh5eSEwMJC28uDZs2exc+dOaGtrw83NDZqamnBxcUFSUhJsbW0ZT/KFhYXw9PREamoqvn37JrJIYXoq7erqCjs7O5EY45ycHKxfvx6nTp0S2+aPNj5JSUnIyckRkrCng7a2Nh4+fMjJRquiogLBwcEYN24cFBQU4ODgQJ2S79q1Cx06dGBkvz7VqFOnTsHLy4vWfa/Nt2/fqDj+zMxMxMbGQktLC9ra2ozsCmDbgyCptoeEhMDc3Fzk+ufPn7Ft2zahk9yf4dKlS1i5cmW9Igm+vr7Q1tbG33//TWuj9aPc0towzRW6fv060tPThRTqKioqkJKSwliUYdeuXRgyZAiWLVtW54KfCVyPwQC3/R2o+Z4FIdAJCQm4evUqtLW1MXHiREZ2//77b+zZs6dewSCmcNXuZ8+eCb2urq7Gq1ev4OjoyIkqZmOEi3vPRYmduuDqeeJagr2iooIKd+ZiXuUcCdXraoIBJ0+eJDwej9y5c4ekpaURHo9HFi5cSAYPHsxK1fBBgwZRBTRXrVpFVq5cSQgh5OXLl0RTU5OR7WnTppE9e/aIXN+9ezcxMTGhbbd///7k3bt3hBBCJkyYQN2HN2/eEHV1ddp2BSxbtowMHTqUuLq6clKpXVdXl0ydOlWoQHJYWBjR1NQkxsbGjO3X5suXL8TBwYEoKysTCwsL6r7R5cyZM0RHR4fs2LGDhIWFkcjISKEfJmzfvp3o6OiQ1NRUEhcXR1RUVIifnx8xMTEhdnZ2jGwTQoiWlhbJyMgghBCyZMkSsmHDBkJITUFwDQ0NRrbv379PBg0aRO7cuUPy8/OJlpYWGTBgAPnzzz9JTEwM06YTX19foqamRh48eECSkpKIsrIy2bx5MxkzZgxxdXX9pduupqZGFi1aJFTI/ObNm2T48OE/Xdi6NhoaGg0WCd29ezd5+vQp7e9UUMz6Z36Y4OnpSZSVlYmenh7h8Xhk5MiRRFVVlfB4PLJ27VpGtgmpGSdfv37N2E59trkcg7ns74QQcvXqVaKqqkpu3bpFXr9+TVRVVcnYsWOJhoYGOXbsGCPbAwcO5Oy+c9nu+njy5AkxMjLixHZjgqt7b2ZmRoqLi0Wuf/jwgUyZMoVBi/8PLp8nGxsbMmPGDJKTk0MiIiKIlpYWefDgAXFwcCALFy5kZJvruUkSNG20GgFjx46lHuLdu3eTSZMmEUIIiY+PJ/r6+ozta2pqktzcXMLn84mOjg45deoUIYSQjIwMMnDgQEa2NTQ0SHZ2tsj17OxsRps4IyMjcufOHfLq1SuirKxMkpOTCSGEREVFkTFjxtC2K0BTU5Pcv3+fsZ36yM/PJxYWFkRTU5McOXKELF68mKipqRE/Pz9SVVXF2ufcvn2b6Ovrk4EDB5Lw8HBWbCorK9f7w3ThOXz4cHLr1i1CCCFbt24lCxYsIIQQkpKSQgYPHsy06WTAgAEkOzubVFZWEm1tbWpjmJaWRnR1dRnZnj17NrGzsyNfvnwhQUFBZPjw4eTbt2/k+PHjrEyWBgYG1MSyfft26qAiMTGRDBs2jJFtrtv+7NkzMnHiRKKnp0euX79OnJycCI/HI5s3byZfvnwR297QoUNJWlpag+9JS0sjenp6tNp779496ic0NJTo6OiQkJAQ8ujRI/L06VMSFhZGhg8fTk6ePEnLvoARI0aQ0NBQ6t+5ubnk48ePxNTUlHh5eTGyTQghEydOJPfu3WNspy64HoO57O+EEDJ16lSyZ88eUl1dTXx8fMjo0aNJdXU1iY6OJmPHjmVke9OmTWTv3r2M21gXXLa7PtLS0ljZPDd22Lz3sbGx1MEtj8cjO3fuFDnQXbduHeM1mAAunycuD+u5npskQVPoYCPgzZs3lCLe7du3qWTD3r1748OHD4zt//nnnzhz5gy6dOmCz58/Y+TIkaioqEBgYCBjSdeOHTvi2bNn6NGjh9D1Fy9eMAplmT17NlavXg1ZWVkoKytDS0sLx48fh4eHB1auXMmozUBNTZsuXbowtlMfXbt2RVBQEDZt2oTt27dDRkYGISEhjIo416asrAzu7u4IDw+Hnp4eXF1dIS8vz4rt78NL2ITLehwAoKmpiYCAAHTs2BHl5eUYMWIE8vPzsWfPHqpuDF2ePn0KDw8PtG7dGrdu3cKoUaPw22+/YeTIkXB3d2fc9oKCAqrGyp07dzB27FgAgLy8PD5//vxLt11ZWRlnz56Fvb09li9fjmbNmsHf3592fRg1NTVcunSpwfHp4sWL0NDQoGVfR0eH+rebmxtcXV0xevRo6pqKigq6dOkCDw8PzJ49m9ZnADUhyoKxXVlZGY8fP8bYsWOp0LtVq1bRtg3U5Aq5uLjA1tYWSkpKkJWVFfo9k7BHrsdgLvs7wG1dodqqdL169RK570xytLhsd11iGCUlJQgPD4e6ujoj2/8LsHnvuSyxUxdcPk9cSrBzPTdJgqaNViOgU6dOKCgogIyMDNLS0rBu3ToANQvezp07M7bPparLlClT4OjoiE+fPlGLnpSUFHh5eTFSMLK0tESvXr2Qk5ODyZMnA6ipE7FlyxbMmDGDUZuBGsW0oKAguLi4MLZVF6WlpfDw8EBUVBSmTp2K58+fY+XKldiyZQvGjRvHyPbdu3fh4OCA4uJiODs7Y9asWSy1mnu4rMcBcKs01rJlS1RUVKC8vBwPHjzAzJkzAQAfPnxgJT9GTk4OWVlZKC8vR0ZGBlWDJykpCXJycr902/l8PkJCQvDXX39h4MCByMnJwc6dO9G+fXtam6H58+dj0aJFlKDJ90RERCA0NJRxjhNQk3jdp08fkes9e/bEu3fvGNlu27YtysrKKHsZGRkAajZA+fn5jGwDwLJly1BdXY1ly5axLpHO9RjMZX8HuK0r9M8//1D9uqCggGlTheCy3XZ2diLXZGRkoKWlBUdHR0a2/xdg89736dOHyvVmu8ROXXD5PHF5WM/13CQJmjZajYAJEyZg3bp1aNmyJeTk5KCjo4OYmBi4uLiwMqGpq6vj1q1bQqouCxYswOrVqxkP3MuXL8fHjx/h7OyMyspKSElJoVmzZjAzM2N86vl93atJkyYxsldbta+qqgrJycmIi4tDz549RRJSmcoQjx8/HlVVVdi/fz+MjIxQVVWFffv2Ye3atYiJiYG3t7fYNsvKyuDh4YGwsDAMGTIE27dvZ82L9TO10QQwWbxZWVlhzZo1kJaWxuDBg8Hj8YTqcTBFQUEBERERQteWL1+OTZs2MVYa09XVhaenJ9q1awdpaWkMHz4caWlpcHV1ha6uLiPbALceBK7bPmPGDLx8+RJr1qzBggUL8PnzZzg6OsLU1BRmZmZ1LvAaYujQobCxscHatWsRGBiIAQMGoE2bNiguLkZiYiJevnyJ9evXs+IhVlZWxpEjR7B161aq31dVVSEgIABqamqMbOvq6mLXrl1wcXGBhoYGAgICYGpqiitXrrCy6AoODua00C3bY3BtuPaYjRw5Elu3bsXvv/+ONm3aQE9PD3fu3IGjoyNjSW0uS21w2W4uoxX+F+Dq3rNdYqcuuHyeuDys53pukgRN8u6NAD6fj+PHjyMnJwdz586FgoICjh49iqKiItjY2LCqSsMVpaWlyMzMRHx8PAYPHsx4AVReXo6wsLA61bpSU1NpydDa29v/9HuZyvMuWbIE27dvR6dOnYSuP3jwAHZ2drh27ZrYNg0NDfH27Vv06NGDOmGuD3HrskRGRmLChAmQlZVFZGRkg++tq36SODx79oyqxyErK4v4+Hg0b96ccT0O4MdqckxCqYqKiuDo6Ijs7GysWLECRkZGcHd3pzy4bISi3rhxg/IgdOjQARcuXEB5eTnjAxeu2z5t2jR4enpSYaECLl68CBcXF9y7d4+W3QcPHuDYsWN48OABPn78iI4dO2LgwIEwMzNjHAoqICkpCZaWlujSpQv+/PNP8Pl8pKam4uvXrwgNDWV0Yvvu3TssXboUU6dOhampKWbPno2nT59CSkoKdnZ2WLBgASt/AxdwMQZ/D1f9HahR2RTUFbK2toampia8vb2Rk5ODrVu31qtoWR+JiYnQ0tKCjIwMEhMT632flJQUo/mvoXZv27aNlYV5ZmYm0tPT0bx5c/Tu3Ru9evVibPN/Abb7jACuS+wI4PJ54kqCvaioCNu2bUNOTg5n8yrXNG20GglfvnzBx48f0bNnTwDA1atXoaOjw0odAS48FL6+vjhy5AjCw8OhoKCAhw8fwtraGiUlJZCSksLgwYPh5+dHSXaKy5YtWxAVFYU///wTKSkp0NLSwuvXr1FYWAhzc/NGXTunrKwMrVq1Evv/fX+6XB9SUlKM5OldXV0xf/58qi82Jrj0xl28eBHDhg0TeiYrKipE8jN+Rbhue1VVVb2x+vn5+ejWrRsrn8MVOTk5CA8Px4sXLwDUeHjnzJmDrl27smK/vLwcv/32G75+/Ypbt26hW7dutHNi5s+fDx8fH7Rt2xZmZmYN9ncmnvl/YwwuKiriNLyKCTweD7dv36a8/vXBNGQzMTERmpqaaN68udD18vJyxMbG4r///S9t2+Xl5Vi7di3++usvofbq6+vDy8urUYxljZGVK1fi3bt32Lt3LxITE+Hi4oJDhw4hIiICb9++ZVRi599EUKZiwIABrNttTH2xKXSwEfDkyRNYWFhg2rRp1OTl7u6OyspKBAUFoV+/fozsu7m5CU3GVVVVePXqFaKiomglYoaFhcHf3x/m5uaUx8be3h4tWrTAqVOn0KZNG9jY2ODgwYO0XdbXr1/Hjh07MHHiRIwePRouLi7o0aMHbG1tUVlZScvm93BZuwwA4uLicOjQIWRlZSEsLAwRERHo2bMnpkyZQsve9+EHXBEZGVlnTSS6GBoa4syZM+jQoQMMDAwavLdM65d9v7Csrq5GVlYWQkJCxA5f+x5nZ2ecOHFCaLPC5mTApQeBi7b7+PjA0tISLVu2hL+/f73vk5KSwvLlyxl9Vm5uLsLDw/H8+XM0a9YMqqqqmDVrFis5rADQo0cPrF27lhVbteHz+Thw4AA6d+6M2bNno2XLlggMDISBgQHtjVb37t2pKAc28hrrg+sx+PPnz/D09MS8efPQp08fWFlZISEhAYqKijh48KCIwBIdnj17hvT0dPD5fAA1uWuCxaGrq6tYtq5fv05tAq9du8ZZpMn8+fNx+/ZtkQ1nRkYG1q9fz2ijtXfvXjx+/Bi+vr7Q0dEBn89HYmIiXF1d4e3tzckz0Nhgs88ISEhIQGhoKP7zn/9g165dGD58OLS1tdGhQwdMmzaNlXZzOX+kpqZiy5YtQvelNkwOFoCaA5asrKw67/nSpUsZ2ZYETRutRoC7uzsMDAxga2tLXbt69Sq2bNkCd3d3xqcd9T3I/fv3x+nTp8Ve+J8+fRp2dnaYO3cugBrxi1evXsHW1pZKKl+6dCnc3d1pb7Q+f/5MFavr06cPnj59CiUlJSxevBirV6/G5s2badkVcOrUKTg5OSE4OBgdOnSAvb09hgwZgpCQEFRWVoodevc9t2/fxooVKzBhwgQ8evQIfD4fVVVVsLe3ByGEkVAI14wcORLHjh3DihUraIdK1MbY2JjybBobG3OaU1JbTU7AkCFD0KNHD3h7e/+0V7AuFBUVkZ6eXqdwAhu4uro26EFgAhdtj4iIwNy5c9GyZUuRvLjaMN1oJScnw8LCAh06dED//v1RXV2NsLAwhIaG4tixY+jbt6/YNmt7hWrnbtYFE6/Q/v37cerUKSHRnQkTJsDPzw9ATYixuNQOa9bT0xPxVLIF12Pwjh07kJSUBHNzc1y7dg1JSUnw8PBATEwMPDw8aOWx1ubw4cPYuXMngJo+KAjuoRvaV7uAu62tLVxdXRkLAQgICQmh2koIoYQMvoepMqAglLd2oWUjIyM0a9YMTk5O/99vtNjuMwK4VO0TwOX8sWPHDjRr1gybN2/Gjh07YGdnh+zsbCoHjAnnz5/H5s2bUVFRQd1zwRqhe/fuTRutJtghNTUVbm5uQifMMjIyWLRoESuxtfWhrq5O65Q/MzNTaCJISEiAlJSUkJRznz59fpgv0xAdO3ZEYWEh/vjjD2qRCAAdOnRgRfI+NDQUmzdvxpAhQ7Bnzx707dsXwcHBuHnzJrZt28Z4oyU4HTQ3N6dOkmxtbdG6dWsEBQX90hut9+/fIyYmBqGhoejUqZNIRXlxvU6176WNjQ0rbRQXRUVFxongPB4P69atw6FDh6CoqChyX5jm9XHpQeCi7bU9rFx6W3fu3Ilx48bBxcWFWpRUVlbC3t4e27dvR0hIiNg2a3uFai+g2SYqKgq7du3CsGHDqGsLFiyAoqIinJ2daW20alOXp5ItuB6D4+Li4Ovri969eyMwMBB6enqYNGkSlJWVqUM8Jhw/fhzW1tZYsWIF9PX1ERkZiU+fPmHt2rUwNDRkZDsnJ4dW+Hd9zJs3D+3btwefz8emTZtgb28vpLgmkAIfPHgwo88pLS2FkpKSyPVevXqhqKiIke3/BbjqM1yq9gngcv54+vQpQkNDoa6ujoiICPTr1w+mpqaQk5NDeHg4IyVlf39/TJgwAVZWVpgzZw6Cg4NRUFAAJyenf229IC5NG61GwO+//46cnByRUImCggLO4lRLS0tx7Ngx2qE3tb0SSUlJaNeundCAUVpaipYtW9Ju34gRI+Dk5IQdO3ZgwIABcHNzw+jRoxETE8OK9C/XtcueP39e50nP2LFj4ePjw9g+l+jq6nKi9pOZmUmJJRw8eBAVFRXU79TV1anvgAl1be5LSkoQEBDAOMwqKyuLikV///49I1t1waUHgeu2C/jw4QMqKyvxfWowExGSZ8+eYceOHUInv82bN8eSJUsoKWBxqb2xZLpBbohPnz7VuZFTVFRk5Xvg0svK9RhcVlZGqabevn2bktJu0aKFUOgTXfLy8jBz5kz89ttv4PF4SElJgZGREezs7ODu7s7olN/KygoODg6wtLREz549RXKRxe3vMjIy1OGblJQUJUzENv369cPly5exePFioeuXLl1qEsQAd32GS9U+AVzOH3w+nxKlUFBQQHp6OgYOHAhDQ0MEBAQwandOTg68vb3Ru3dvKCsro6ioCAYGBqiqqoK/vz/tVAtJ0rTRagT897//hZOTExwdHanQgJSUFDg7OwsV0aRLfQIBUlJScHJyEttev379kJycDAUFBXz+/Bn37t0TOe25dOkSo9yyDRs2wM7ODvfv34epqSnCwsIwc+ZMyMjIUK59JnBdu6xNmzYoKCgQEZTIyMhAu3btGNvnEqbevLrYunUrTp8+jcuXL0NBQQF+fn5o27YtmjVrhrKyMlRVVeHy5cuM731dOWCEELRq1Qqenp6MbHMp6Qxw60Hguu3Jycmwt7dHdna20HU26jn17NmTkkKuTW5uLmvlDd69e4fjx48jPT0dMjIy6Nu3L0xMTBhtEIGasTciIkIkJOvcuXOsbI649LJyPQb37t0bsbGxkJeXx/v376mDlvDwcBH1Sjq0atWK2rAJapgZGRmhd+/ejAv/enl5obq6GomJiazXLzM2NmY1Z6V26Y6lS5di2bJlSEtLoxblDx48wLVr17B7927abf5fgas+w2WJHQFczh8KCgp48OABJk6cCCUlJaSkpACoEXGrfWBKB1lZWepQQUFBAS9evMCIESPQv39/vH79mpFtSdG00WoErF27FtnZ2Vi4cKHQoD169GhWqoZ/L4YB1JwIa2ho0Eo4njt3LrZt24a0tDQ8fPgQFRUVlExxfn4+Lly4gKCgIGzfvp12m9u2bYsDBw5Qrw8ePIi0tDR07tyZFSUwrmuXTZo0CW5ubtS9Ly0tRXx8PFxcXOoswPqr8ezZM4SGhiIrKwv79u3DX3/9hb59+9aZA/UjBBuswMBAKCgoUNePHTuGHj16oLi4GJMmTcLJkycZhwqEhobW2df79evHiizyt2/fcPnyZbx8+RIWFhZIT09H37590aFDB8a2ufYgcNl2V1dXdOnSBRs2bGC9yKSVlRVcXV1RUFAAXV1dyMjIICUlBXv37oWpqamQ1PagQYPEtv/8+XPMmzcPLVq0gLq6Ovh8PiIiInD8+HGcPHmSVg6YgOXLl2Px4sVISkqi5OhTUlLwzz//wNfXl7ZdAVx6Krkeg1euXAkbGxtUVlZi4sSJUFRUxI4dO3D8+HFW7o22tjYOHjyIrVu3UqFbixYtwoMHDxiPBWwUyq4PQc6KINyLac5Kbe/yqFGjsG/fPgQGBiI2NhaEECgrK8PLywtjxoxh749opHDZZ6SlpalNFgDWPYhczh9mZmZwcHAAUOMYmDJlClq0aIHk5GTGZTYEWgFr1qxBv379EBcXB0tLS2RkZIgob/6qNMm7NyJevnwpVNtCUVHx325SvZw5cwYnT56EtLQ0rKysKCUkZ2dnhIeHw9raGqtWrRLLpjg5XUxPmhuqXbZixQpaxW3t7e3h4OCA1q1bo7KyEnZ2doiOjgbwf4m1gonu+5PnX4nU1FTMmTMHmpqaePjwIS5duoSAgABERUXB19dXKBfvZ5g9ezYmTZoklHehra2Nc+fOURv90NBQnDt3rkFRhX+bDx8+wMTEBIWFhaioqMCVK1ewfft2pKamIjQ0lPEp/OfPn2FnZwc9PT2Ymppi0aJFuHnzJuVBmDBhwi/bdjU1NURFRbHiifien81hoOtJECgn7t69m3ouy8vLsW7dOlRUVDAOjXn48CGOHj2KFy9eQEZGBr1794aVlRVruRlsIskxGAA+fvyI/Px86l48fvwYv//+Oyv9KD09HRYWFjA3N8ecOXMwadIkfP78GV+/foWlpSUlSMAUtqWox48fDw0NjTpzVtasWSN2KFVtWfomGoarPsNl2REBXM4fAPDXX3+hffv2GDhwIC5cuIDAwEDIy8tjy5YtjMLyk5KSYGVlhZUrV8LY2Bhjx45F586d8e7dO4wfP5620qMkadpo/X+KOHlAbIaK5efnQ1ZWltYp+Y8GI4Cd0AyuqB2iISA7OxtPnz4Fn89Hv379OFOsYxNzc3NoaGjA1tYWWlpaOH/+PHr06IEdO3bgwYMHOHPmjFj2BgwYgLNnzwodHNS2C9Tkb82YMQMPHz4Uu72SKkS9bt06lJSUYO/evRg6dCjOnz+Ptm3bYvXq1fjtt98alDinAyGENQ8C120fM2YMdu3axVgVrS7ECdmhI2yhpaWFU6dOQVlZWej6s2fPMG/ePCQlJYltU5Kw6an8N8bgt2/fIjMzE4MGDUJpaSmrG4Jv376hrKwMHTt2xIcPH3DhwgXIy8tj7NixjG2fPHkSgYGByMvLw5UrV3Do0CF069YNy5YtY2S39qGFmZkZrK2tMWLECFy9ehX+/v5iH0bxeDxs3rz5pxRkf2WRJknBRZ+JiIhosMQOF3lIbM4fXJOfn4+Kigr06NEDGRkZOHXqFOTl5WFmZtYo6mk1hQ7+otRelHNx2vGzg7GUlBSrGy0mhUmZyCj/DLU9Tj9anNNZkNd1ptGzZ89GV/g3NTUV27ZtE7k+d+5chIeHi22Pz+eLhABcuXJFKB9LVlaWlhcRqBE2kQQJCQk4ePCgkMhLu3btsHHjxh9KhNfHjzwI7du3R1VVFd6+fcvIg8BF22uzdOlSuLm5wcnJCUpKSqyGfHCpCgjUiBHVpcpFV6mrdn2xHx14MR17v/dUzpw5E8HBwbQ9lVyPwbWpqKjAxo0bcenSJUhLS+PKlSvYuXMnSktL4e3tzUppiRYtWlBCFZ07d8bChQsZ2wSACxcuYPfu3ViwYAElZtC7d2/s2rULLVq0gIWFBW3bXOSs/IxXQEpKqmmjBW76DNsldgRwOX9I8rC+9rqxT58+jEtHSJqmjdYvipubG5XLUFcOFVMkVdyWTRrK/2EjPOPNmzdUcjFXi3Mua0RJiubNm6OkpETk+rt372gpScrLyyM9PV1owfz9CVtqairt8AOuhR4ElJaW1ivpXFVVRcvmjwo4A+x4ELhoe238/Pzw9u3behdqTNrOZZ0rABg8eDA8PDywf/9+KjG9qKgInp6eGDJkiNj2xKkvxnSB4u7ujr59++LixYsYOnQogBo5/NWrV8PT01NsTyXXY3Bt/Pz8qFxQgcy9mZkZ7O3tsWvXLjg6Oopt82eeJwFMiqMHBwfDwcEBxsbGVJ3L+fPno1WrVggMDGS00eIiZ6UpdLB+JNVn6oJuiR0BXM4fXB7Wi3O4J8nDH7o0bbR+UYyNjal/s1UZvC7y8vLQuXNnIWnkpKQk9OjRg5H3iSsyMzOxY8cObN26VcgTtHHjRnz+/Bnbtm2j7SGqvSDnanFeX6HJ7/kVQx8FGBkZwcvLC3v37qWuZWZmYvv27Rg1apTY9vT19REQEIARI0bU6bWqqqpCUFAQY4XNR48eQVlZWUhq+erVq+jatSvjhF2gRmjh5MmTQt7QyspK+Pn5UQpe4iKpSYSLtteGy6KS33u0qqqq8Pr1a6Snp1MiPExYu3Yt5syZA319fSq89dWrV2jfvj3c3NzEtvez9cUEhz5M4MJTyeUYXJvo6Gg4OjoKlZLQ1dXF9u3bsWHDBlobLa4LogvIysqqs4Ctrq4unJ2dGdm2sbGBlZUV2rdvD2NjY/j6+mLChAl49+4drXpF/wuHf1wiqT7zPUxL7ADczh9cHtZzHaUgaZo2Wo2AiooKBAcHY9y4cVBQUICDgwNiYmKgra2NXbt20VYFCwwMxP79+xESEkIpUwE1xXQfPHiADRs2sBI2xBbZ2dmYO3cuOnbsiPLycqHfjRw5EocOHcLs2bNx+vRpWg+qJBK9vy802RjZuHEjrKysMHjwYPD5fEybNg0lJSXg8Xi0VDAtLCxw7tw5zJ07F+vXrxfqi48fP8auXbtQVFTEaNHs6OiIsLAwHD58WKio5+nTp3Hr1i3Mnz9frFyuuti4cSPmzp2L+/fvo7KyEo6Ojnj58iW+fPmCY8eO0bIpKQ8CF22vTe2DI7apL4zX19cXeXl5jO3Ly8sjOjoa586dw4sXL0AIwaxZszBp0iTG4WuGhoY4e/asiIRzfn4+Jk+ejHv37jGyz7ankusxuDb5+fl1btjk5eVRXFxMy6akCpx27twZWVlZIqq9Dx8+ZJwPM3DgQFy5cgUVFRXo0KEDjh8/TuWs0Jmvm9L0G0YSfYbtEjsCJDF/cHFYz2Xtwn+DJjGMRoCbmxvOnTuH4OBgFBYWYsmSJVi5ciViY2PRq1cvWp3y6tWrsLW1xbJly7BgwQKhBcOXL18QGhoKPz8/+Pr60vJScMHGjRvx4cMH+Pn51TlAfP36FRYWFlBSUqIlHc91ovf/mrrT3bt3hYQ8hg8fDmlpaVq2Xrx4gXXr1uH58+do0aIF2rVrh+LiYpSXl6NPnz7w8vKirTR2+vRpuLm5Ydu2bZg0aZKQ14zP5+PcuXNwdHSEk5MT4xyEgoICnDhxAmlpaeDz+ejbty9MTU0ZqS7V50GwtbVl1YPAdtujoqLqvC4jI4N27dpBVVUVHTt2ZNDihnnz5g2mTp3KWKxixowZcHV1ZU0FMCYmBjdv3gQAREZGYvz48SIqo7m5uUhPT0dCQgKjz1q0aBF69eoFe3t7SmBGTk4O69atQ2lpqdjFULkeg2szbdo0zJkzBzNnzhQSx/H19cX169dpK5B+/foVR48exYwZM4T6n4+PD2RlZbFgwQLGqq+BgYEICwuDvb091q1bBy8vL7x9+xZeXl5YsGABI0GM2rnEtfn06RM2bdokJLnPxF4T/wfXfeZ7MQyAWYmd2nA5f9R3WL9gwQJWDutjYmIwYsQIob556tQptGnTBuPHj2803tgmj1Yj4PLly9izZw9UVVWxbds26OjoYMmSJRg2bBisra1p2QwJCYGNjQ0V+16bNm3aYMWKFSgrK0NQUNAvs9G6d+8e9uzZU+8pTMuWLbFixQps2bKFln2uw7Qay6DwI+bPnw8fHx8MGTJEKEelsLAQlpaW9S6wG6Jv376IiopCQkICEhMT8eHDB3To0AEDBgzAsGHDaG/ggBr1rw0bNtS5iZKWloaxsTG1yWC60eratStWr17NyEZtJOVBEAg0fN/2kpISbN++naqRIg4/yi2QlpbGnDlzaD+vP+Lhw4e0BVRqk5OTU69XiA4CFUPBGefbt2+FcmukpKTQqlUrVor+su2p5HoMro2NjQ1sbW2RkZGB6upqREZGIisrC1euXBEKWxaHkpISmJub49mzZxg4cKDQovnLly84deoU/v77bwQFBTH6zq2trfHlyxesWbMG5eXlWLx4MWRkZDB79uw659wf8eDBA+Tk5ACoOcBQVVUV2RhlZmbi7t27Ytv+X/MesI0k+gxX6SFczh9Xr16Fl5cXli1bJqLI6uPjg9DQUOzcuRM9e/YUew1ZWVkJGxsbxMXFISQkRCh8+MGDB7hw4QKuXLmCPXv2CHnSflWaPFqNAHV1dVy9ehVycnIwMjKCiYkJrK2t8ebNG0yaNImW5PWgQYMQFhYGJSWlet/z7NkzLFiwgHH4CltoaGggJiamwQHhzZs3mDhxIv755x/JNewnacwerbi4OKrau6+vLxYuXCgyqbx+/RqxsbFCBWLFpaFwhoSEBKGwv59FW1sbUVFRDZ7aZWZmwsTEhJH3o6ioCIGBgXjx4gUqKipEfk9nI8+lByEzMxNFRUUAajbP3t7eaNeundB70tPT4eHhgUePHond9voghODTp09ISkqCs7MzLC0tYW5uTtteXSemJSUleP78OUxNTWltEmsTGBiI+Ph4WFpaomfPnkI5fgCzelFmZmbw8fERue9swqanUtJjcHx8PAICAijPed++fWFtbU3VZRQXLy8vXLp0CYcOHarTU5CZmQkrKyvMmDEDy5cvZ9p8fP36FRkZGSCEQElJCa1bt8b79+/RpUsXsewkJyfD1NQUwP/VXPyeVq1awcLCglWV4Ca46zOSUO3jcv4wNTXFiBEjGjw48PDwQEpKitg570FBQQgODsa+ffvqzHW8d+8eVq5ciRUrVsDMzEws2/8Gv/5WsAn07NkTKSkpKCwsxJs3bzB8+HAANQXimIQk/WiP3bJlS1RXV9O2zzZycnJ49epVg5P8q1evGCWPCjAzM6s3Zrp58+aQk5PDlClTMGjQoJ+2+ezZM+rfXGwouKR79+5wdnam+kxMTIyQl0lwCk8nR6s2S5cuhb+/v9AJf1lZGdzd3XH69GlaIZuysrL49u3bD9/H1PuxYcMGpKSkYOjQoSKLcbpw6UHIycnBkiVLqH5e32Q+ffp0sW03hJSUFDp06IDRo0ejtLQUQUFBjDZaf/zxR51hN/PmzcPkyZMZtrZmoVVdXY3ExEShz2FD7bGhBUheXh7k5ORo2wbY91RKcgwOCgrCxIkTcfz4cca2BFy6dAkbNmyoNxyrd+/eWL16NQ4ePMhoo6WiooLbt2+jY8eOUFNTo67TPRzV1tam5o/GfGDXGOGqz0iixA6X88eLFy9+WBZg8uTJOHv2rNi2o6KiYGdnV+cmC6gRlVm1ahXCwsKaNlpNsIOVlRXWrFkDaWlpDB48GDweD76+vvD19aWlegUA/fr1w7179xrMe7l9+/YvVePJ0NAQfn5+0NXVrdNdXFVVhYCAAErGmAkqKio4evQoVFRUqIf90aNHePToEYyMjPDu3TssXLgQ+/btg6Ghodj2ly5dKnLKxHRDwSV9+vShpGsNDAxw5swZTvJrcnNzsXz5cvj6+qJ58+a4c+cONm/ejNLSUtoV4FVVVREbG4t+/frV+57r16836N39GR48eICAgIAGE5DF5ePHjz9MKFZQUKA8U+IwatQo3LhxA3w+H0ZGRjh9+rTQdyrYPH8v1MAmGhoajEspuLu7s9Saujl8+DBntnNycrBz506kp6dTh1qEEFRUVKCoqAhPnz4V22ZtT6Wvry94PF6dnsrw8HCxN1qSHIP9/PxgZGTE2E5t8vLyfphrp62tLVYRbAFnzpzB+fPnAdR8h8uXLxeRWy8oKEDbtm3Ftl2b2gd2TXAPV31GEiV2uJw/AO4O69+8eQMtLa0G3zN06FB4enqKbfvfoGmj1QiYOnUqVFRUkJOTgxEjRgCoqQ4fFBREq44LUJPg7eHhAW1t7ToHkbS0NOzbt49xFXs2sbKywtSpUzFv3jwsWrQIWlpaaNeuHT59+oTk5GQEBgYiJyeHlYVXXl4e5s6dK1IYb+fOncjPz4ePjw9CQkLg7+9Pa6OVm5uLFStWsLqhkBTfTxCVlZV49uwZlJSU8PvvvzOyffToUSxcuBArVqxAly5dcObMGYwfPx4ODg60T3BNTU2xbt069O3bF/r6+iK/v3HjBg4cOEBLLro23bp1Y/z3fw/XHgRB2Nv169fr9AxxjZSUFKMYez6fj4cPH+LZs2coKSlBmzZtoKqqCg0NDdbayObG+XucnZ3x6tUrjB07FocPH4aFhQWysrJw7do12jLgXHoqJTkGa2ho4MaNG6wVEQZqCrQWFhY2+Dx9/PiRljKskZERHjx4QL2Wk5MT8Wz369ePdh7oy5cvcfr0aVhbW6Njx44oKSnB1q1bERsbi86dO2P58uW0C9s2UT9c9hmA2xI7XM4fXB7W//777/jy5UuD7/n27Rutup3/Bk0brUaAp6cnpk+fLnS6J9hw0cXY2BixsbGYMWMGRo0aBW1tbbRt25aaMOPj4zFs2LBfyi3bsWNHhIaGYv369Vi2bJlIGI+WlhaOHDnCSg2Gmzdv1unaNzExoaSqDQ0NsW/fPlr2udhQSIp3797BwcEBq1evRr9+/TB9+nRkZmaiXbt2CAkJgYqKCm3bXbp0wZEjR7Bw4ULcvHkTPj4+jE+1DQ0NMXv2bCxduhQqKioifT09PR0mJiaMhTDWr18PJycn2NraokePHiICHnRyeSTlQejatSsCAgJYLyHxI65evdqgp7Ehbt68CScnJ+Tm5gqdrEpJSaFHjx5wcnKifRAljtQ/EzGB5ORkHDhwALq6urh58yaMjIygrq6OvXv3Ii4uDrNmzRLbJpeeSkmOwa1bt4aHhwf8/f2hqKgooupGJ+dRV1cX4eHhUFdXr/c9YWFhQuF+P0v79u2F+gKbSn5paWkwNTVF+/btMW/ePADA1q1bcfnyZZibm6NNmzZwdnZGmzZtYGBgwMpnNlEDl32G6xI7XM4fXB7Wa2pqIjo6usG1xIULF0REOH5VmjZajYCkpCQEBwdDTU0N06dPx4QJE1gZwPft20fV4Pjrr78A1EzC/fv3x7Zt2zBjxoxfTimvV69eOHPmDJ48eYKUlBR8/vwZHTp0gLa2Nm3577po3bo1Xr58iV69egldz8jIoE5RSktLaeficLGhkBQ7duzAly9f0LFjR1y6dAnv3r3DiRMnEBERAU9PTwQHB4tlry6VwqlTp2Lv3r04e/YsSkpKhK7TYePGjRg8eDBOnjyJK1euoLi4GB07doSWlhY2btzISqgTIQSZmZmwsLAQuU43l0dSHgRPT0+cO3cOw4cPR3x8PCIjI6kSEh4eHrQ2E/WJovD5fHz58gVJSUk4fvw4du/eLbbt+/fvY8mSJRgxYgRcXV3Rr18/tG3bFl++fEFqaiqOHz8Oa2trnDp1Cv379xfbfu1wRkIIkpKS0LlzZ/z555+QkZHBs2fPkJ+fT8ubXZuKigrqxLdXr154/vw51NXVMXXqVEaHXFx6KiU1Brdq1Yrx4cf3WFhYYNasWWjTpg2WLFkiFFL56dMn+Pv749y5cwgJCWH0OTt27MC3b9+oXNzMzEzExsZCS0uLVgHwAwcOYPjw4ZTKWn5+Pi5duoSpU6dSebFt27ZFcHBw00aLZbjqM1yq9gngcv7g8rDe3Nwc5ubmkJOTg6mpqdChJSEER48eRUhICLy9vcW2/W/QpDrYSMjKykJUVBQuXLiAwsJCGBkZwdjYGHp6eqxMohUVFSguLkb79u2F4soFi8TGQFFREWt5Q3v37sXp06dha2sLDQ0N8Pl8PHr0CPv378fkyZOxaNEirF27Fq1bt8b+/ft/ymZdG4qPHz9i79690NPTE1LSYnuBwSY6OjoIDQ2FiooKVq9eDUII9u3bh6ysLEybNk3sRO+frU/EVHiAawwMDKCiooJZs2bVGdJANwQtKysL69evR2pqap0eBBcXF/Tp04d2u4EaD/mOHTugp6eHbdu24fXr1wgJCUFqaiqsra1pyUYL6tLVN8UoKSlh0aJFtPq6hYUFunXr1uAG0MHBAaWlpfDy8hLbfm127dqFd+/eYceOHVROZXV1NbZu3QopKSlGob5jx46Fvb09Ro4cif379+PTp0/YunUrMjMzMWPGDFqKsrWprKxEUFAQJ57KEydOYNKkSY2uAPvVq1dhb2+P8vJy9OrVi1ocvnr1Cq1atcK2bdswceJERp+RmJiI5cuXY9++fejduzfGjh0LaWlpfP36Fbt27cK4cePEsjdkyBAcPHiQ8ppERkZi06ZN8Pf3x8iRIwHU5G6ZmpoiOTmZUdubEIWLPsOlal9tuJ4/BIf1L168APB/h/WzZs1idFh/9OhR7Ny5E23btoW6ujp1zx89eoTS0lKsWrWKdnkjSdO00WqE3L9/H5cvX0ZkZCTatWuHadOmwcTEhHY8r6GhIc6ePSsSSpKfn4/Jkyf/MvLutfn8+TM8PT0xb9489OnTB1ZWVkhISICioiIOHjzIuMgfn8+Hp6cnTp48ifLychBC0KJFC5iZmWHVqlWIjY3FoUOHsGfPnp8OCftf2VBoaWkhOjoa8vLyGDx4MNasWQMTExNkZmZi9uzZjOTdJcHbt2/Rtm1btG7dGgkJCbh69Sq0tbUZL640NTVx4cIFxn2vPp48eYLU1FQUFxez7kHgooREfcnhgoLFTJQZdXV1ERwcDFVV1Xrf8/jxYyxZsgR37tyh/TlATSmMU6dOidzrly9fYsaMGYwWtgEBAQgODoaHhwc6dOiA+fPnY8WKFbh9+za+fv2KU6dOMWo7F8XuBYwcORKfPn2CgYEBpk+fztqhX21yc3MRHh6O58+fo1mzZlBVVcWsWbMYqxp++PAB58+fx5MnT/Dp0yfKuz1u3DhWwmTnzJkDRUVFODg4IDw8HCEhIbh27RrOnj2L8PBwsWsN9u/fH9euXYO8vDyAmkOEc+fO4f79+1SZjZycHEyaNOmXLG3yvwDbfUbSJXa4nD+A+g/rmfDs2TOcPn1a5J5PmzaN1bZzTVPoYCPj8ePHuHr1KiVIMGjQICQmJiIoKAguLi4/LWccExODmzdvAqiZzJydnUVi4HNzc39Zb9aOHTuQlJQEc3NzXLt2DUlJSfDw8EBMTAw8PDwYu5SlpaWxceNGrFq1CpmZmWjWrBkUFRWpxaGRkZHYoX7/K2pRf/75J86cOYMuXbrg8+fPGDlyJCoqKhAYGPjTm8mGiIqKwm+//Uad+q5atQpGRkaYNGkSY9vXrl2Dra0tAgIC0KNHD1hZWaFHjx6IiIhAcXEx5s6dS9u2rq4uHj58yNlGS1VVtcGNBRO4KCFRO0+H7XIGX758+eFiu1u3biguLhbLbl00b94cb9++FZnYMzMzGRcyXrRoEX777TcQQqCuro5ly5bBz88P8vLy8PDwYGQb4KbYvYDY2Fjcvn0bUVFRWLFiBdq1a4cpU6bA2NhYJOSaDsnJybCwsECHDh3Qv39/VFdXIywsDKGhoTh69Cjt3D4A6Ny5M9TU1GBmZiayKCwvL0dsbCztWl0A8PTpU3h4eKB169a4desWRo0ahd9++w0jR46kFabVrVs3vHnzhtpo3blzBxoaGkL9759//mFcDqCJ+uncubNIWDhTJFlip/b8UVJSwtpmSICsrCykpKTw4cMHkb+Lbq1BHo/HWUF7SdK00WoEvHv3DufOncO5c+eQlZUFDQ0NLFu2DOPHj6dytby9veHm5vbTGy0tLS2cOnWKeiDevn0r9OAJEqZ37tzJ/h/EAnFxcfD19UXv3r0RGBgIPT09TJo0CcrKyowWy7X5+vUr0tPTUVlZCUIIVbAXgFj1s+qDyw0Fl2zcuBFLlizBx48fYW1tDTk5OTg6OuL69es4dOgQI9tHjhzBrl27hAZXOTk5bNmyBV+/fqUlDlCbAwcOwNLSEkOGDIGfnx/++OMPREdH4/Lly/D29mbUdwYOHIht27YhNjYWPXv2FEk+ZlpItKysDCEhIUhOTqb6ZG3oiAMI4KKERG3YLmfA5/N/qFbYrFkz8Pl8Wu2tzcSJEynxl/79+4PP5yM5ORne3t5UEVm6SElJCdUQW7RoERYtWsSwxf/Hp0+fqA3i7du3YWJiAqBGuOFnass1hJSUFIYNG4Zhw4ahtLQUV69exZUrVyiV3JkzZ2LChAm0PZc7d+7EuHHj4OLiQn3XlZWVsLe3h5ubG+M8qvnz51O1rmqTkZGB9evXM9potWzZEhUVFSgvL8eDBw8wc+ZMADVeETqhlqNHj8bu3bvh6OiIuLg4vHv3DosXL6Z+n5+fD29vb8Y5g02IYm5ujiVLlrBe2/LfKLETGhqKw4cPIz8/H1JSUvjPf/6DZcuWMU5VSE5Ohr29PbKzs4Wus1FrEKhZAx8/fhzp6emQkZFB3759MWvWLFZEdyRB00arEWBgYICOHTti8uTJ8PHxqfPB/PPPP6GoqPjTNuXl5amFmZmZGXx8fERqrfzKlJWVUad7t2/fpk5nW7RowcoJ0PXr12FnZ4eSkhKRBS0bAwfXGwouUVdXx61bt1BSUkLVhFmwYAFWr17NuObS0aNH4e7ujvHjx1PX7O3t0b9/f/j6+jK+L5mZmfDx8YG0tDRu376NkSNHQlpaGpqamrRq59Tm5MmT6NChA/755x+R8B0mRScFbN26FdevX4eenh66dOnCyNb3TJ06FTweD2/evGGthERt2C5nICUlJTFv+7p16/Dt2zds27YNVVVVIITgt99+w7x58xgVtRUQGxuL9PR0VFRUiPyOaZ/hqtj993z9+hXFxcX4/PkzqqqqIC0tDT8/P+zZswe7du2i1YeePXuGHTt2CG2omzdvjiVLllAbF3EJCQmhDg8JIdDT06vzfQ0pzP0Murq68PT0RLt27SAtLY3hw4cjLS0Nrq6u0NXVFdve8uXLsWTJEmpBbGBgQG2a/fz8cODAASgoKGDp0qWM2t2EKAkJCUhKSsKyZcuwaNEiRuUoaiPpEjve3t4IDg7G/PnzqbzzpKQkODo6oqSkhFKzpIOrqyu6dOmCDRs2sJ6z+fz5c8ybNw8tWrSAuro6+Hw+IiIicPz4cZw8eRJ9+/Zl9fO4oClHqxFw/fp1jBo1Cs2aNfu3m/LLMH36dMycORPy8vJYvHgxLl++DEVFRezatQv37t3D6dOnGdkfN24c+vbti2XLltU5cDA9SRk9ejRsbW2FNhRAjWSpr68vLl++zMh+Y0VDQwMXLlwQOcXLycnBhAkT8PjxY0b2hw0bhkOHDqF79+4YMmQIvL29oa+vj4SEBGzYsAHx8fGM7HPJgAEDsGvXrjprgXFJeXm5SFixuLx//x4LFy5E9+7dWSlnwOPxIC8vLyKhXxs+n4+8vDzW8h1LS0uRlZUFKSkp9OrVi3HYIFBTR+vEiRPo1KmTyD2WkpKiioTTJSoqClu2bIG0tDS0tLQQEhIi5KlkcpJdXl6Oq1ev4ty5c0hISECnTp0wdepUTJs2DQoKCgAAJycn3LhxA3FxcWLbnzRpEpYuXSoyRsbFxWHnzp2IiYkR22ZVVRUuXrwIPp+PTZs2YdOmTULjuyCSY/DgwYwKCxcVFWHbtm3IycnBihUrYGRkBHd3d6SkpMDLy4v2QcmLFy8gLS0tdNh67do15OXlYdq0aazX8WuiZqzZvHkz9u3bh27dumHt2rWsjcGrVq2i1nf1qfYdOHCgwXHuZxk+fDg2btwokot8+vRp+Pn5MSqgrKamhqioKE7ypiwtLdGyZUvs3r2bGiPLy8uxbt06VFRUICAggPXPZJsmj1YjwNDQEF+/fkVGRgYlzFAbOmFsKioquHXrFjp16kSpg9XHryjMsHLlStjY2KCyshITJ06EoqIiduzYgePHj8PX15ex/Tdv3iAgIIBVt31tCgoK6pSd1tTUxNu3bzn5TLbgsr8oKirixo0bQuFUQM3iio0T+JEjR2Lr1q34/fff0aZNG+jp6eHOnTtwdHSkLaH7PR8+fKjTO0E3Tl3A9wssNvn48SP8/f2Rnp5OeYQJIaisrERGRgaSkpIY2We7nAFTT4+4CMZfQcjmkydPqN8xCSOOjo6Go6MjZs+ezUYzReDSUzlkyBBUVVVh1KhR8PX1xfDhw0UWhEOGDKG9WbSysoKrqysKCgqoOkApKSnYu3cvTE1NhUR3fvY7kJGRoTaXUlJSmDBhQr25g0zo2LGjSJ7wmjVrGH9W3759RcaW0aNHU/+mk+/YxI8ZN24cjIyM4OnpiaVLl4LH42HOnDkYO3Yso0ggSZbY+fLlS505vgMGDEBhYSEj2/Ly8igtLWVkoz6Sk5Nx6tQpoYOo3377DcuXL2fkhZMkTR6tRkBcXBxWr16Nb9++sRbGFhkZSU0yERERDT7MggK9vxofP35Efn4+5XZ//Pgxfv/9d1YWo5MmTcKWLVtoS3L/CEHS+PcbimPHjuHEiRO0Tmslxff9paqqCq9evUJUVBQ2bNiAKVOm0LZ9/vx52NnZYcKECdDQ0AAApKSkIDo6Gi4uLoz74rdv3+Dl5YWcnBxYW1tDU1MT3t7eyMnJwbZt2xidCMfFxcHe3h4fP34Uus5WnLqDgwO6dOmC1atXM7JTF6tXr8bdu3ehp6eHy5cvY8KECcjMzMTTp0+xZs0aWnlDkipnkJ+fT1tx9WfgMoxYT08Px44dY0U8QtIcOXIEkyZNqlNxTdDnq6qqaIdaSUKlNTc3F48eParzYIRJn/yR8iqTzbmlpSWr+Y5NNAyPx8Pt27cpz/vz588REhKCy5cvo6qqCioqKuDxeGjfvj3WrFlD+3O4UO2rzdq1a9GpUyds2rRJ6Pru3bvx6tUrRgJikZGRCAsLg5OTE5SUlFht/7Bhw+Dv7y9yMJ2SkoKFCxcyPgSUBE0brUaAwGOzfPnyOsMZGktCIJtYWFjA0NAQBgYGVK4Wm8TFxWHXrl2wtbWFkpKSyEkkU+8E1xuKf4NLly7h9OnTYhcs/p6YmBgcOXIEz58/R/PmzdG7d28sWrRI4iFz4jJmzBgoKSnB1NS0TgEAppv23bt3IzQ0FH369EGvXr1E+iQTqW5dXV3s3LkTo0aNwoQJE7B7925K8am8vJyWAp6kyhnweDyoqqrC0NAQRkZGjNTo6oLLMGJfX19kZ2fDxcWFE8/K/PnzG/w9EwEVrsuCiJMzSec7CA8Ph5OTU505vWz0ye9ryAnyCqWlpZGamkrb9tixY9GzZ8868x03bNiA6dOn07bdhCi1o39qU15ejlu3biEpKQlpaWn48OEDLl68+C+1sm7s7e2pf5eWluLatWtQUVHBgAED0KxZMzx58gTJycmYOXMmHB0daX/OmDFj8Pbt23rz45k8S+vWrUNBQQH2799PjTVFRUVYvXo12rVr1yiKFjdttBoBampquHjxIhX3zgY+Pj4//V5Jh+n8DO7u7oiLi8OrV6+goqICAwMDGBkZsSIvDtRIoQoGje+L/LFV56qxbijqIzc3F+PHj8ejR4/+7aY0yD///IOjR48iPT2dqs1jbm7OOKlWU1MTERERDdZFYYKZmVmDv2dS1LJ2nR4bGxsYGhpi6tSpeP78ORYtWkQrx0ZSPH/+HHFxcYiLi6Mkrg0NDWFoaIhBgwYxzm9QU1NDdHQ0J2HEL1++xJw5c1BWVoYuXbqIRBYwzdGqvdACarzPr1+/Rnp6OhYsWABbW1ux7NUuCxIZGYnx48fXWRYkPT0dCQkJjNrONQYGBtDX14etrS2l3ssW328Sq6urkZWVhX379mHdunUYOnQobdts5zs20TDfe7TYtPuzYYF01xs/mjMESElJMTp0iYyMbPD3TA6O8/LyMHv2bBQXF1OCb69evUL79u1x9OhRVkV9uKIpR6sRoKioiPfv37O60YqIiBB6/e7dOzRv3hw9evSAjIwMsrOzUVlZif79+/+SGy07OzvY2dkhOzsbsbGxiIuLw8GDB9G5c2cYGhrCwcGBkf3Dhw+z1NL6GT9+vEiid2OltLQUx44dY1xIFKg5rcrKyqKkuQkhqKioQEpKCmNVrRs3bmDFihVQU1ODnp4eqqur8fDhQ0ybNg2HDx/GwIEDadsePHgwnjx5wtlGi8lG6kd069YNubm5kJeXh6KiIp4/fw6gRqaajVpUAHflDJSVlaGsrIxFixbh8+fPuHnzJq5evQoLCwu0adOG8YJfUVEReXl5nGy01q9fj7Zt22L69OmsiGt8T31eTl9fX+Tl5Yltj+uyID/ywNWGycIQ+L8NC9ubLKBuD1vPnj3RunVrODo64sKFC7Rts53v2ETD7Nixg3UlPaCmmDjXyqlczhm14TICR05ODtHR0Th37hxevHgBQghmzZqFSZMmcfLsckHTRusXpbYgwuzZs7FlyxY4ODhAQUFBRH2QThhbbYWZkJAQxMbGYvfu3dSpzefPn7FhwwbWw3DYpmfPnpgwYQI6deqE9u3b49KlSzh27BjjjRZXuVm14XJDwSX1ncRJSUnBycmJke3z589j8+bNqKiooEJvBJ/VvXt3xvdl7969sLS0xNq1a4Wu79y5E56enggLC6Nt29HRETNmzMDNmzfRo0cPkXvExoFFaWkpzp8/L1RPpHY9PbqMGTMG9vb2cHd3x9ChQ2FrawsNDQ389ddfrBzwcF3OgBCC1NRUJCQk4N69e3jw4AGaN2/OSoHndevWwcXFhZMw4hcvXuDMmTMSH2enTJmCqVOnwsXFRaz/x3VZkNoblPLycsTExEBFRQWampqQkZFBamoqHj9+TFvevTYqKirIyMiQ6Il4hw4d8Pr1a7H/X135jlOnTsXevXtx9uxZlJSUCF1vgj2GDBnCSVjvtGnTWLf5PeIIa4k7jtnb28PBwQGtW7cW8ZzXRkpKinEtxt9//51xzcJ/k6bQwV+U7xezgq+JizC2oUOHIjg4WCTsLj09HWZmZoxj7bng6tWruHfvHu7du4fMzEx07doVQ4YMweDBgzF06FB07dpVbJtLlizBrl27hBatt27dwqBBg6jwmKKiIhgZGSE5OZlR+3+0oRAoEP2K1CWe0rx5c2hoaKBHjx6MbI8fPx4aGhqwsrLCnDlzEBwcjIKCAjg5OWHNmjWMhDaAmvo458+fF6k59+rVK0yZMoVR2OPWrVsRHh6ODh06oGXLlkK/Y0Oq++3bt5g3bx4KCwvRq1cv8Pl8vH79Gp06dcKJEycgJydH23ZFRQU8PT2hrq6OSZMmUX9LmzZtsH//fsYKdVyWM1iyZAkePHiAb9++QVVVFbq6uhg6dCi0tLRYWSBxGUZsbGwMBwcHRp5UOly4cAGurq6/5NguwN7eHu3atYOdnZ3QdS8vL2RmZjLOzbh69Src3d2xcOHCOjfQTAQr6hLDKCkpQWhoKL58+YKzZ8+KZU9S+Y5NiKKiooKpU6fC2dmZE5EKAdevXxdSfQVAHbzSjbDhMjzRzMwMvr6+aNu2Leth7YaGhjhz5gw6dOgAAwODBv8GpvOqJGjyaP2iMA2LEIfKykqUlZWJXC8sLJRYUVBxWblyJaSlpWFoaAh3d/c6pdLFJS4uDuXl5UIbrZUrV+LcuXPUBoLP59d5r8TF398fEyZMqHNDYWNjw9g+l3B5EpeTkwNvb2/07t0bysrKKCoqgoGBAaqqquDv7894o6WiooK7d++KbLRSU1MZ52hdvHgRO3bs4CyMwt3dHXJycggPD6dCND98+IDVq1fD09MTu3fvpm1bVlZWyAvs7OyMNWvWoHXr1qwU6OSynMHDhw9RUlKCkSNHQl9fH0OGDGE1zI/LMGJra2ts2rQJlpaW6Nmzp8i9ZrLYB+oOxSspKcHz589pnRBLsizI5cuX68z9mDp1Kitem5UrVwIAtm/fLvI7phsWMzMzETEMoOYQzdPTU2x7z549o92WJphB/h97Zx4P5fr//9fY2hctn7Ry0qJSljZLkuKEKUWLFilJe4nTIaWVFGkjpUghhUoionRKllKoSEo57ZuECtVY7t8ffuZrmqHMfd/DnO7n43Eej9wz531fM8Y11+u63u/XmyBw7do17gmwuro65ffw9PSEv78/unTpgo8fP6Jbt24oLCxEVVUV2Gy20HHrriMfPnwIHx8fLF++HGpqapCWlkZ2djYOHDggVFPkuuKJ6hRFU1NTrqGUKE7+6IYRWs2U2tS10tJSSEtLk24Y2hDjx4/Hxo0bsWnTJigrK4MgCGRkZMDFxYXUHzmdHDhwADdu3EBKSgpmzZqFYcOGQVNTE5qamtw0k8Yi6HBX0DUqxCfdgoJqRGWeIiMjw91ZlpeXx+PHjzF27FgoKysLlXLzIyYmJvD09MS///7L05snMDAQs2bN4knRaexirlWrVrR8CdeSmpqKgIAAnjq4Ll26wMHBATY2NqTjP378GI8fP6bc6hqgtz/azZs3kZOTg9TUVMTGxmL79u3o0qULNDQ0oKmpSboGrL404pycHISGhpJKM661g968eTPfY1ScTvTo0UPg6bOFhQVMTEwaHc/NzY1br0LG5fJXaN++PR48eMC3KZKenk6JMQGdO+GCYktLSwuVaSEIuuodGfhhsVg4ceIE9u/fDwsLC+jr62P58uWUGW8BNSfM69evh6WlJXR1dXHy5Em0bt0aK1asIJUlUnducnNzg6urK0/ftUGDBqFr167w8PAg3cuvsrISHz9+5OnDWHsi19i5pu4aolWrVpg0aRKtLTzohhFazZTaGqnr16+DxWJh3LhxcHFxQadOnSi/18aNG2Fra4v58+dzv5QJgoChoSEcHR0pvx8V6OvrcwuA3759i9TUVNy4cQP+/v6QkJDAnTt3mniEDUO3oKCaH81T6oPFYpESWsrKyjh9+jTs7e0xYMAAJCYmwtraGk+ePKEkbaO2JiU4OJhvF87f35/7bxaL1WhxMWfOHHh7e8PFxYUvdZAKJCUlBcZt0aKFQHHUGHx9fbFv3z6BjwnzXvyItbU11q1bh5ycHIHtDMhQ2+BTWVkZixcvxsePH+Hr64uwsDCcO3eO0sXnt2/fEB0djbCwMOTk5KBFixakxk932svOnTspjVf3tDYnJweWlpa0NXU3NzfHpk2bkJ+fD2VlZVRXVyMzMxMhISH4+++/ScevrQfjcDh49eoV+vTpA4IgKJln6Gy5Qne9IwMvBEGgY8eO8PLyQnJyMvbu3QtTU1MMGzYMJiYm0NDQIN278+PHjxg/fjyAGnOfrKwsGBoaws7ODhs2bICtrS3p1/H06VP069eP73qfPn3w9u1bUrGTk5Ph6OiIoqIivsdatmwp1KZOLYcOHRJ7sxdGaDVTPDw8kJWVBVtbW0hISCA4OBhbtmyBl5cX5fdq27Ytjh49in///Rd5eXmQkJDAoEGDSNfbiIL379/jxo0bSE1NRWpqKiQkJKCtrd3Uw/opdAsKqqlrnkInq1atwqJFi9CxY0eYmprCx8cHbDYbb9++5e7ekoHOFJz09HTcvn0bcXFx6Ny5M9+pKtlFtbq6Og4ePAgPDw/uZ6SiogK+vr6kT9KCgoKwfPlyLFmyhJbTcxMTE0hJSSEoKAgJCQncdgbe3t6k2xkQBIHs7GykpKQgNTUVd+/eRYcOHWBiYkJZq4S8vDyEhYUhKioKpaWlkJWVxYoVK0gXaNOx2P9Zs9y6kElNPHfuHN8JJZUsX74ckpKSOHHiBHx8fADUmHE4ODhQUhhPEAR2796N4OBgVFRUID4+Hnv37kWrVq2wZcuWRv8OROWYGBwcjJ07d/LUOzo5OUFZWRk+Pj6M0KKRMWPGYMyYMbh9+zYiIyNx4MABuLq6QlpaGu3bt0dycrJQcdu3b88tSejTpw+ePHkCoOZE+v3795SMfeDAgQgKCsKmTZu4G+qVlZU4fPgwhg4dSir2nj17MHjwYMybNw+2trbw9PTEmzdv4OXlRfrkW0VFBf/88w+srKxIxWlKGKHVTLl+/Trc3d2ho6MDoMZW18rKCpWVlZTUTAiib9++tFlTU8327duRmpqKf//9F//73/+gp6eHnTt30uYQRDV0Cwq6ePfuHbp06cLzGUxPT0fv3r0pOdofMWIE4uPjweFwICsri5CQEISGhqJ79+6NWsg0BcOHD8fw4cNpi7927VrMmjULBgYG3Hqn7OxsrrU+GSoqKjBlyhRaU5TpamcwevRofPnyBf3794eenh7+/vtvDBs2jHRcDoeDixcvIjQ0FHfv3oWEhAQ0NDRw48YNHD9+nBKnQKoX+0D99UE/QjY1UVdXFydOnMDKlStpsVm+cOECZs2ahSVLlqC4uBgsFouvOTIZgoODcf78eWzevBnbtm0DUJMpsXXrVnTp0qXRPcboPMWqC531jgz8CCoVGDlyJEaOHAmCIPDkyRPk5OSgsLBQ6HuMHj0anp6ecHFxgYqKCg4fPow5c+YgPj6esiwmBwcHWFtbIykpCYMHD0Z1dTXu37+Pr1+/IjAwkFTsJ0+ewM3NDUpKShg0aBBat26NefPmoXXr1jh69CipE6m2bdvCw8MDvr6+UFBQ4PuOEqWfgbAwQquZUlRUxPNFrqamhqqqKm6hJFlE0SyPTjIyMmBkZITx48dj8ODBlMW9c+cOj10xQRDIysri9pyhqqeQOAoKPz8/eHl54fjx4zyCwtvbGxkZGXBwcCA99lrL2NrPeL9+/eDs7IySkhLY2tri4MGDpOLT6WBEd785RUVFREZG4uTJk9x+IpMnT8bs2bNJL/KmTJmC8PBwSlKy6oOudgarV6+Gnp4epQvdnTt34ty5c/j8+TPU1dXh7OwMQ0NDdO7cGUOGDCHdBLkWqhf7gOhcuD58+IDY2FgEBgaic+fOfAsgsuPYtm0bTp48iQ4dOkBWVpZULEGEhYVh06ZNMDAw4KZ/GhsbQ1paGjt27Gj0e093zVotdNY7MvDT0IYFi8VC//79SRspOTg4YNmyZbh48SLmzJmDY8eOQUtLCywWi891U1hGjBiBCxcuIDw8HI8fPwZQkwo8e/Zs0rWDkpKS3NpNeXl55OXlcV2ghe2pV0vr1q3FvmUBI7SaKT+eXElKSlJSi1FL3WZ5r1+/hp+fH8zNzXncaEJCQpptP6famqE3b94gKSkJI0eORFlZGeki6VWrVvFNrD/2XKLCDINuQUE1ly5dwr59+7B8+XIMHDiQ57EDBw4gMDAQ7u7u6NOnD8aNG9eo2BkZGXj58iWAmiLvIUOG8O2Q5+fn48aNG6ReA1DzxVL391dZWYlnz54hKSmJ60JGhocPHyIvL0+gmHB1dSUdv2fPnrSIoUWLFmHKlCmIjY1Fr169+D7jZHcN6eyPZmFhgbdv38LT05PbX6xfv34wNzcXWnwdP34cffv2hZub20/FORmoXuwDDZ+sfPz4kRIjCaBmF3706NGUxBKEgoIC8vLyBNaVUMGrV68waNAgvutKSkr48OGD0HFjY2MxduxYnjksNDQU7dq1g7GxMenPEp31jgz8iMLxsXv37oiMjMT3798hIyODkJAQJCcno1u3bpScztfSu3dvvvUMFfTv3x///PMP5s2bh759+yIjIwPz588Xqin6j4hqA4NOGKH1m1LXMtPCwgIbN27E9OnTudf09fWhqKiIwMBAWFtbN8UQG6SiogIODg64ePEiJCQkEB8fD3d3d5SVlcHb21uoVBa6d4JFKSio5vjx41i1ahWWLl3K91i7du2wcuVKlJeX4+jRo40WWnV37VgslkBB0rp1a0o+h/VZ54eGhiI1NRXz588XOvaxY8e4u3d1U7dYLJbQfZIsLS1x4MABbq+ShhZpZMRQ7fs/bNgwtG7dWug49UFnO4NHjx7BwsICLVu2xLBhw1BdXY1z587h5MmTOHXqlFC7zdu2bUNERARWrFiBDh064M8//4SxsTHlwoKuxf6tW7ewdetW7Nu3j+f1Ozs749mzZ9ixYwdUVVWFjg/Qf4KrpKSEtWvXwt/fX2DKENkFWM+ePZGdnc13CnT9+nWh6pMrKiqwatUqJCYm4vjx4zyflYyMDERHRyM+Ph579uwhlf5PZ70jQ8PUV//IYrEgLS0NOTk5oTKOqqurcfDgQXTp0gWzZs1Cq1at4Ofnh/Hjx1MmtIqKiuDn51evsyyZ74/Fixdj9erVkJaWxqRJk+Dt7Y3Fixfj0aNH0NDQIDNsAPRlQ4gKRmg1Y969e4fv37/zXHv//j0kJSV5rjW2o/ePZGVlCewlMmzYMG5RZnPj4MGDePjwIQIDA7mL/3nz5sHJyQmenp7YsmVLo2PW3QnmcDj11nrdvHlTqJ1yUQoKqnn8+PFPT2RMTEwa3YgTqDF5qN01VFJSQnJyMo+FuSjQ0dEhneIQEhICGxsbrFy5Enp6ejh37hxKSkrw119/YcKECULF7NmzJzdNrWfPnrSdrNy5cwdBQUHcHXKqobOdgYeHB0aPHo3du3dzF+Pfv3/H2rVr4enpicOHDzc65syZMzFz5kzk5+cjIiICUVFROHPmDDp37ozq6mq8fPmSkpMWqhf7APDgwQMsXrwYqqqq3F40tVhaWsLX1xdWVlYIDw8nnfJE5wnu06dPuSnKZERnfVhbW2Pr1q348OEDCILAjRs3EBYWhuDgYKHStYKCgpCdnY3g4GC+jZVdu3Zh+vTpWL16NU6dOvXTBq8/g656R4aGWbBgAc9nHeDPcBk1ahS8vb3Rvn37X47r5eWF0NBQnhNJNpuNQ4cOAYDADc7G4uDggOzsbGhpafHNC2TR19fH6dOnISkpie7du8Pf3x/Hjh3DhAkTSGeK0JkNITIIhmbJwIEDCSUlJZ7/frxW+zNZJk2aRPj4+PBd37lzJ2FmZkY6Ph0YGBgQKSkpBEEQhKqqKvHixQuCIAgiNTWVGDNmDOn4CxcuJL5//85zraysjNi4cSMl7/nAgQOJDx8+kI4jKkaMGEE8efKkwec8e/aMGD58OOX3/vjxI+Uxf+To0aOEtrY2qRhDhgwhnj9/ThAEQVhZWRGXL18mCIIgkpKSCDabTXqMr1+/JqqqqviuV1RUEPfu3SMVe+LEiUROTg6pGA2hrq7O/RvdsGED4e/vTxBEzWtSV1cnFVtVVZV4+PAh3/Xc3FzKPo9VVVXEP//8Q6xYsYIYMmQIoaSkRMyaNYuIiYkhFffMmTPE6NGjicDAQEJFRYUICwsjdu3aRQwbNow4efKkUDFXrFhB2Nvb1/t4dXU1sWTJEmL16tXCDpsgCIIICAggBg4cyP0eqvtvCwsLUrFFRWhoKKGrq8sdu6amJhEQECBUrEmTJhFRUVENPickJIQwMTERKn5dPn78SKSnpxO3bt0ibt26RaSlpRFJSUnEwYMHScdmqJ+oqChCT0+PuHLlCvH582fi8+fPxLVr1wh9fX0iICCAyMjIIGbMmEFs3LixUXF1dXWJpKQkvuvXrl0jxo8fT8nYVVVVibS0NEpiiRIjIyNi3bp1xJMnT4iRI0cS2dnZxJUrV4ixY8cSkZGRTT28X4I50WqmiNJJZfXq1Vi9ejVSU1MxdOhQVFdX486dO8jNzYWfn5/IxtEY3r9/L7B/S/fu3SkxrHj9+jVWrlwJHx8fSEtLIzU1Fc7OzigrK6Ok1kZQ3ndRUREtfdKoYMCAAUhLS2uwX0hKSgrpnjpfvnyBh4cHLCws0K9fP1hbWyMtLQ0KCgo4cuQI6ZYDguptysrK8OnTJ9IpbK1bt+Y2a6y16K1NwX39+jWp2AAwYcIEpKSk8H1GXr16hXnz5uHevXtCx167di02bdqENWvWoE+fPnypTWRPzelsZ9CmTRtUVFTwXRd0TVgkJCSgp6cHPT09FBUVISoqCmfPnsVff/1F6mRh2rRpqKysxKFDh/Dt2zds2rQJnTp1wpo1azB79myhYt69e7fBUzwWi4VFixZxmyULCx0nuD9SVlaGqKgobu1d//79YWxsTInL4Zs3bzBjxgyYm5ujqKgIBEGgc+fOqKysRFZWVqNTtl69egU1NbUGn6OlpYVdu3aRGfZ/Y4dfTPHy8sLWrVu5btBAjfumtLQ0tmzZAisrKzg5OWHVqlVcc5tfoaSkRGCWjIKCAmWnud26dUObNm0oifUjdKYl0pkNISoYodVMqdvR+2dpbGQxMDBASEgITpw4we0DMWjQIGzbto3S7udUoqioiBs3bmDGjBk812NiYihJ6QkODoaVlRVWrlyJrl274syZMzA2NsaGDRsoKSanW1BQzfTp0+Hh4QF1dXWBn4nc3Fzs378fy5cvJ3UfNzc3pKenY8GCBbh8+TIyMjLg4eGB2NhYeHh4wNvbm1T8H80wAEBaWhqqqqqk62/U1dVx5MgRbNq0CYMHD8aZM2ewePFiZGRkCP0FFxISgoCAAAA1qSrTpk3jc7z7/PkzaSFka2uLqqoqLFq0iOf9qV3EkXUepbOdgYaGBjw8PODl5cW1/y4qKsKuXbugqalJKjbAP/926tQJCxYswIIFC3Ds2DHS8c3NzfkW+2QoKyv7adqSnJwcPn/+TOo+7969w4wZM9CiRQsoKSkhOzsb+vr6WLduHXbu3Em6x9abN29gYWGBjx8/4o8//kB1dTXCw8Ph6+uLkydPQk5OjlT8uhsXdTcvhN24aNOmDb58+dLgc759+0a6mTmd9Y4MDfPhwweBc223bt24xg/dunX76efgR5SUlBAREcFnVHH+/HnKzGD+/vtvbN26FXZ2dujduzff9wiZ7xA60xJlZGS486+8vDweP36MsWPHQllZGc+fP6f0XnTBCC0xYNmyZTh06BDPl315eTl27tyJ06dPU2K/rq6uTrrpqShZtWoV7Ozs8OTJE1RVVeHcuXN4+vQptw8NWbp27YqgoCBYWVkhKSkJBw4coLQ7Od2CgmpMTU1x7do1TJ8+HePGjYO6ujrat2+PkpISZGZm4vr16xgzZgzp2oPExET4+PhAUVERfn5+0NbWxuTJkzFw4EDMnTuX9OugcyFib2+PhQsXIiQkBLNnz4avry9GjRqFr1+/Cl13Z2ZmhuLiYhAEAR8fHxgaGvKJtjZt2uDPP/8kNXYqBEND0NnOoLa/mJ6eHhQUFAAAz549Q8eOHeHm5kZ67PXNv+7u7ggPDyfdSPPly5fIycnBt2/f+B4Txta4V69eePToUYObNQ8fPiTdJoTuE9ydO3dCTk4O4eHh3JrNwsJCrFmzBrt27cLu3bsbHZPOjQtVVVXExMQINDepJTo6ms+1tbH8F3b4xRVlZWX4+/vD1dWVWytfVVUFf39/7gbkrVu3Gm2zv2LFCixZsgTp6elck5rs7GzcvXuX26ybLARBID8/HwsXLuS7TnYzLSMjA4cPH+Y5IKAKOrMhRAUjtMQAutPYgJoF7tGjR/Hvv/8iLCwMERER6NOnT7OdtPX09ODl5YXDhw9DUlISR48eRf/+/bF3715MnDhRqJiRkZF816ZOnYq9e/fi7NmzKC0t5blOBroFBR3s37+fu0BOSEgAUJOGpKysjM2bN2P69OmkzRrKy8vRvXt3ADWpiDY2NgCAli1bchd1ZLl//z6OHj3KYwU+f/580u5OPXv2REJCAsrLy9GmTRuEh4fjwoULkJOTg6GhoVAxW7VqxXV3Y7FYsLa2Jr0jLoi6X5ClpaWQlpamtHkxne0M5OTkEBMTg/Pnz3P7i82cOROTJ0+mJMWMzvk3IiICzs7O3AL7urBYLKHmGUNDQ3h5eUFDQ0Pg6y8rK4OXlxfGjx8vzJC50HGCW5fU1FQEBATwGON06dIFDg4O3HmhsdC5cVF7yiknJ4c5c+bwCDiCIBAcHIzjx4+T3kT7L+zwiyvr1q3DggULkJaWBmVlZVRXV+PBgwf4/Pkz/P39kZmZCWdnZzg7Ozcqro6ODkJCQhAcHIzk5GRISUlBUVERZ86coSyryM3NDRoaGpg5cybl3yF0piU2lA0hLoYwLIL4Sft4hibnw4cPsLKyQs+ePWlJY0tJScHSpUvBZrMRGxuLmJgYnDlzBn5+fnBzcxP7ZnG/yq9OaFSkUqmqqiIuLg5ycnLQ0dGBjY0NLC0t8eLFC0ydOhWZmZmk4tMNh8PBp0+f0LFjR55dpbr1AsIwbdo0zJgxA927d8eSJUsQFxcHBQUFeHp6Ii0tDadPnyY17lu3bmHhwoUYMGAARowYgerqamRmZiIvLw+BgYE8jZgbi4GBAfbt24chQ4aQGmND0FmzEhgYiGPHjuH9+/dgsVjo1asXli9fLvTff912BrVCS1A7gxMnTuDOnTtkh08bdM6/+vr60NHRgZ2dXaNcyhri69evMDMzQ0VFBebPnw81NTV06NABxcXFyMzMRFBQEKSkpBAWFkaqEXBeXh4WLlyIBQsWYPbs2Zg8eTI+f/7MPcElWwM2evRonDhxgs8Z8dGjR5g1axbpz8yBAwco37gIDg6Gu7s72rdvj2HDhnFP/e/du4eysjLY2toKLRJrmT9/PlRUVGBvb4+goCDuJun169fh4OBASTkBQ/0UFBQgNDQUDx48gJSUFJSUlDBnzhx06tQJ+fn5ePfuHbS1tZt6mHyoqqoiOjqalrKEK1eu4PDhw7SkJQI1NfkcDge9e/fGkydPuNkQ8+bNq7espjnBCC0xoaioCFZWVnj8+DG8vLwoTWObNWsWDA0NsWDBAqipqSEqKgq9e/eGn58foqKiEB0dTdm9yFD3i/HAgQMNPpfuHi9koVtQ0MmECRNw9uxZbj1MLe/fv4eJiQnS0tKEjp2YmIhVq1ahoqICbDYbnp6e2LFjB0JCQuDj4wNdXV1SY589ezYGDBiArVu38lzfunUrnjx5guDgYKFjjxkzBoGBgQ0ahpBBUM3K8+fP0blzZ9I1K97e3ggICIClpSVUVFRQXV2N9PR0hIaGYu3atbCwsGh0zMzMTMyZMwcAb1+xurRu3RoLFy5s9N/rz3qK1YUKYyG65t+hQ4ciNjaW8sVPcXExtmzZgoSEBJ7TMklJSRgaGsLR0RFdu3YlfZ9v376hvLwcnTp1QmFhIaKjo9G9e3ehT3DrsmzZMrRs2RIeHh7czZza/omfP3/G0aNHSd/j69evyMvLQ0VFBd/nc+TIkULFfPjwIU6fPo2cnByUlJSgU6dOUFNTg5mZGSVzQ3p6OhYtWoTVq1fD1NQUhoaG6NKlC7feUVCrFobmh6jXM0uWLAGbzYaJiQnpWD+SkJAAR0dHlJeX81ynqsZX3GGEVjNFUBpbcXEx9u7dC21tbZ70OLInTmpqajh//jz69OnDI7RevnyJSZMmkXIzo5Lx48fj7NmzkJWVbTDthcViUdJ8ODIyEi1atOAW69va2kJfXx+TJ08mHZtuQUE1sbGxSEpKAgCcO3cOxsbGfKllr1+/Rl5eHukd1eLiYrx//557wpiVlYU2bdpQskhRUVFBREQEX6z8/HxMnz6d1C75kSNHEBkZiblz56JPnz58RcHCLtxqWb16NQoLC+Hl5cVXs9KtWzehalZq0dHRgaOjIyZNmsRz/fTp0zh06BD++ecfUmOnuj+ak5PTLz9XmMa2opp/Z86ciRUrVtD29/7x40fk5ubi06dP6NSpE5SVldGuXTta7kU1+fn5mDVrFtq0aQNlZWUANXUrZWVlOHHiBOmUqitXrmDdunUoLS3lE1lkF4dlZWW0pVIB9e/wW1paik3dijhSXl6O48ePIzMzU6A4b8ymjqjXM35+fjh48CD09PQEOsuSEXPjx4/HoEGD6k1LJFO79eHDB+zbt6/e95yK94ZuGKHVTBFlGtvYsWOxZ88ejBgxgkdoXb16FZs3b8b169dJxRdHgoKC4OnpiY0bN3KdDXfs2IGwsDCsX78eM2fOJH0POgUF1bx9+xaOjo4gCAK3b9+Gqqoqzxc6i8VC69atMXv2bEoWjc+ePUNeXh4kJCQwZMgQbt0WWcaNGwdPT0++hqK3b9/GypUrSZ3GNfQ3S8Xf6YgRIxAQEMBXS5aVlQUbGxtSY1dVVcW5c+fwxx9/8Fz/999/YWpqSstmC5l2BnZ2dti4cSM6deok8PNIFjrn39u3b3P/XdsoetWqVZCXl+drRk9WnO/fvx9mZmaUnZgJao9QH1QsgF6/fo2TJ09ya+8GDhyI2bNnC9Uw/keMjIzQv39/LF++XKD4JHMPNTU1/PnnnzA1NYWGhgaZYfJRXxpuSUkJ1q9fT6rekaFh1q5diytXrkBbW1vgZ0aYTR1RQaeYozMtcenSpbh//z7YbLbA97y5Zy8BjBlGs0VQnyW6mDx5Mtzc3ODm5gYWi4WysjJcv34dLi4uzbbYcO7cuTAzM4ORkRFat25Nefzg4GDs3LmT5/U7OTlBWVkZPj4+lAgtWVlZfPr0CZcuXaJcUFBN9+7dubt18+bNw4EDB9ChQwfK71NaWgp7e3skJSVxd65YLBaMjY2xY8cO0vnYenp6cHFxwZ49e7iC9smTJ3B1dSVtDkD3zpqkpKTA3cIWLVoI7F3SGCZMmIBTp05h/fr1PNfPnTuHsWPHkooNUN/O4MqVK1i9ejU6deoES0tLgf3FyEDn/Fub9lh3j3Pz5s18z6NCnEdHR8PX1xfq6uowMzMTaP7QGAS1R6CSpUuXYsyYMdDU1ISioiJ69uyJv//+m5Z7vXr1CocPHybd+08QmzdvRnR0NKytrdGtWzdMnToVpqamQi9E69Y7RkZGYsiQIQLrHW/cuEF67Az1c/XqVezZswd6enq0xC8qKsL379/5Tm3I1jgBIJ2V0BCjR4/GnTt3aBFaN2/ehL+/P9/mqDjBnGiJCXSmsVVUVGDdunWIiYkB8H/1FOPGjcP+/fspdR+jCicnJ1y+fBlVVVUwMDCAmZkZpTuHKioqiI6O5vsSfvnyJdhsNrKyskjFp1tQiCtOTk5IT0/Hpk2boKamxjWrcHFxgYGBAdatW0cq/qdPn2BlZYXc3Fzu7tiXL1+gpKSEgIAAUuYAdEN1zUrd9LuysjJcvnwZgwYNwvDhwyEpKYmcnBxkZmZixowZ2LJlC6mx1/5efX198fjxY/z9999wc3NDbGwspKSkGu3ENm3aNLx//x5//PEHbt++DTU1tXpPtKio0aJy/m2M9TkVJzeZmZmIjo5GXFwcvn37BgMDA5iamlLSY4xqrKyscPfuXXz79g3dunWDlpYWxowZAy0tLb6aULJMnjwZGzdupMWSupbCwkJcuHAB0dHRePDgAVfwTps2rVFx6Kx3ZPh1Ro4cibNnz1IuzrOysrBmzRq8ffuW5zrZGqdfbZ3BYrEQGBgo1D0AetMStbS0EBISwpdtIU4wQksMEEUaGwA8f/4cubm5qK6uxoABA9CjRw/s27ePb5e7ucDhcHDlyhVER0fj+vXr+N///kd657CWKVOmwNTUlK/p5okTJ3Dy5EnExsaSik+3oKCaQYMGITk5GZ07d4aSklKDu9pkduFHjhyJgwcP8qVMpaamYu3atUhNTRU6NlAjKFq1aoWkpCSedKQxY8bwOSU1lvrSqlgsFqSlpSEnJ4cpU6YIXdNDdc3Kr/Y8Y7FYpMWKlpYWfHx8oKamhnXr1qGkpAS+vr7Iy8vD3LlzedLpfoVXr17hxIkT+PTpE86dOwcjI6N6G2WSTeehc/4VZRpYZWUlkpOTERMTgytXrqBjx46kd7npaJVQVVWFnJwcpKenIz09HRkZGdzNEG1tbWhra2P48OGkU0UTExPh6ekJOzs79O3bl29zi4pThFoqKioQHh6OPXv2oLy8nNQcSXW9I8Ovs2HDBnTt2hVr1qyhNO60adMgIyMDa2trge6jwm4G0F3LWgudaYnu7u74/Pkztm3bxpdaLS4wQksMMDAwgJ2dHV8aX3R0NHx8fBAXF9fomN+/f4e7uztiYmIgLS2NKVOm4K+//uIuNpOTk7F582a8ffsWDx48oOR10Entgsvb25v0FxkAREVFYd26dWCz2VBRUQFQs6iNiYmBi4sLTE1NScWnW1BQzblz58BmsyEjI4OIiIgGhRaZ90ZDQwPHjx/nEw2PHz/G3LlzcevWLaFjA/RasPv4+MDHxwcTJkzgpjncuXMHly5dgpmZGSQkJBAdHY3169dzF+yNhc6aFTqhs53B2LFjceHCBcrs0X+E6vm3qWzvCwoKEBMTg/j4eGRnZ2PUqFGkGlXT2SrhR/Ly8rjC69q1ayAIgvR7M2TIEG5vvrrzGZVOaenp6dzTxKqqKkycOBFmZmaUvjcAuXpHhl9n9+7dCAwMRL9+/fDHH3/wiXNhxcqwYcMQERGBfv36UTHM/xROTk7c+b1379587zkVGQt0w9RoiQEFBQXcHey6qKqq4s2bN0LF9PDwQHh4OExMTCAjI4NTp06hbdu2WLJkCVxdXXHq1Cn06dOH1HGyKPj+/TuuXLmCqKgopKSkoHv37li0aBHpuCYmJpCSkkJQUBASEhIgLS0NRUVFeHt7U5KfLSkpKbCws2vXrqisrCQdn2rqiiczMzPa7mNpaQkXFxfs37+fu2NbWlqKffv2/XIaREN8/fq13pMPsmRmZsLW1hZLlizhXps/fz6OHj2Kmzdvws/PD+rq6jh69KjQQovOmpVv374hLi4O//77LxYuXIi8vDz079+fknRKRUVFXLt2Dd27d8eHDx+4dV/h4eGkzV+qq6vx7Nkz0g2n64Pq+ZfFYnFPrFkslsCmx61bt4a1tXXjB/sDpaWliI+PR3R0NG7fvo0ePXrA1NQUe/fuJV0PunfvXkybNk1gq4R9+/aRapVQS0VFBdLT03Hz5k3cvn0bubm5kJaWpiTtkYzI/Bm7d+9GTEwM3r17h5EjR8LJyQmGhoaUzD1U1zsy/Dp3797lbrwWFBRQFrdbt2749u0bZfGaAoIgkJSUxNPjUUNDg5JTqB/dcMUN5kRLDKAjjU1PTw+LFy/G7NmzAQDXrl3D9u3boampiTNnzsDKygq2trbNtlYoKSkJFy5cQEJCAgiCgKGhIczMzMSmYPLgwYNISUnhExSOjo4YNGhQs8uz/1mfj7qQGfucOXOQnZ0NCQkJKCgoQEpKCs+ePUNZWRl69OjBs/MsTDoCnRbsqqqqOH/+POTl5Xmuv3jxApMnT8a9e/fw5s0bGBkZCeXiV1RUBD8/Pzx+/Fig+QWZnb3CwkKYm5vj48eP4HA4iI+Px/bt23H//n1KeoPR2c5g/PjxOHDgAAYPHkxqjPVBZxox3Wlgw4YNg7S0NP78809MmzaN0vmRrlYJz549Q1JSEpKTk3Hr1i1wOBwMHToUY8aMwZgxYzBs2DDSab50o6+vz01jp/q0mep6R4am59y5cwgPD4ebmxsUFBRoNZyhg5KSElhbWyMnJwft2rUDQRAoLS3FkCFDcOzYMdqyDcQF5kRLDLC2tsa6deuQk5MjMI1NGAoLCzFmzBjuzzo6Onj9+jUuX76MY8eOYfTo0ZSMnS4WL16MkSNHYuPGjZg4caJANzayFBUV4enTp9yGnwRBgMPhIDs7G8uWLSMVOzk5GdnZ2ZgwYQKfoMjNzcW5c+e4z20OfSIiIiJ4fn779i2kpaXRu3dvSElJ4cWLF6ioqICysjLpwlctLS2yw62XPXv2AIDAvxuy6UKdO3dGZmYmn9DKyMjgngp9+PBB6D5GDg4OyM7OhpaWFuWncjt37kT//v1x4cIF7vvv7u6ONWvWYNeuXfD19SUVX1dXF4mJiTztDNhsNmbOnElaxJmammLRokWYMmUK5OXl+d4bsn0G6Zh/a6HbXXbr1q0wNDSkZX6UlZVFcXEx3/WioiKhN+j09fXx+vVr9OzZE1paWnB3d4empiZlvb8E9UerDzKfm4SEBKH/35+RmJgIHx8fKCoqws/PD9ra2pg8eTIGDhyIuXPn0nbf35Vasx0pKakGa0lZLFajNjJ+rHUmCKJel+fm3vDX3d0d3759Q2RkJHd+f/jwIf7++2/s3r2b79S7sRQUFCA8PBxPnz7F+vXrcfv2bQwYMAB9+/alYvi0wwgtMYCONLaKigoeW3RJSUm0aNECGzZsaPYiCwAuX76MXr160RY/KioKzs7O4HA4XJen2kmxZ8+epIUW3YKCauoWzR8/fhzXrl3D7t270blzZwDA58+f4eDggAEDBpC6D90neXSK1nnz5mHbtm149uwZVFVVUV1djXv37iE4OBjLli3D27dvsWXLFujo6AgVPyMjA4cPH6bFJe3mzZs4cuQIz4K8Q4cOcHR0pCRlE6CvnYGPjw8AwalgLBaLtNCiM434+/fvCAsLQ15eHrdeCKgx+rl//z7i4+MbHbPuwrBXr164f/9+vc8lc4JLR6uEV69eoXv37mCz2RgzZgz3dVDFr5oMCfO5sbS0xIEDB9C+ffuf/s2QOX0uLy/n/t2kpKTAxsYGANCyZUuezxADNcybNw8pKSno3LmzwNYMtTR2o662nc5/gatXr8LLy4untlpJSQnOzs6wt7cnJbSeP3+OmTNnom3btnj//j3WrFmD2NhYODk54fjx49zNr+YMI7TEBGNjY5H0tKKrzoFqevXqhYcPHyIwMBBPnz7F/v37kZCQgH79+lEiFH19fcFms7Fo0SLMnj0bAQEBKCgowNatW7Fq1SrS8ZtbamBjOHLkCAICArgiCwDat28Pe3t7zJs3D/b29kLH/tmOM9lFM52mEQsWLIC0tDSOHj2Kw4cPA6hxLnN0dMSsWbOQlJSEvn37NsoJqi7dunUj1QOpIcrKyurtR0dFzSCd7QxE0XOQrvnX1dUVkZGRGDx4MLKzs6Gmpobnz5/j48ePfKmKvwpdC8MfWbNmDaysrDBp0iS+VgkODg5CxYyNjeWmDQYGBkJCQgKjR4/mpg3+eFrcWOj8rPTs2ZOb0vhjmjOV0FnvyMDPlStXuEYjVG7U/Vjr/OXLFxQXF3Ot4y9duoRRo0ZR3taADiorKwWmP3fp0gWlpaWkYu/cuRP6+vpwdXWFuro6gJrMFEdHR3h6elJSC0o3jNASE+hIY6vPilocuH//PmbPng1VVVXcv38fHA4Hubm52LFjB+maD6CmX5a3tzcUFRUxcOBAFBUVYfz48aisrISvry+mTJlCKj7dgoJOKioqUF5eznf948ePpD8/9e04t2jRAnJycqTfF7ot2OfOnYu5c+eipKQEUlJSPG5yOjo6Qp9mAcDff/+NrVu3ws7ODr179+arUyFjRz1y5EicOnWKRwRWVFTg0KFD3C83Mmzfvh1Pnz7FkSNH+NoZ7Nmzh5J2Bm/evEF+fj5GjhyJsrIyno0AstCVRnzlyhXs2LEDkyZNgoGBAVxcXNC7d2/Y2dmhoqJC6Jh0LAx/pEOHDjhz5gylrRL69u2Lvn37Yv78+eBwOLh16xaSk5Nx6tQpuLi4oFevXtDW1oaOjg709fUpfkXkqOs4t3Pnznqf9/XrV1L3Wb16NbfecdKkSVBQUOCpd2Sglrqbc9XV1QLNRiorK3Hw4EGsXr1aqHvk5ORg4cKFMDMzg6OjI4CazxCHw8GxY8fQv39/4QYvIoYMGYJTp05hw4YNPNdPnTqFQYMGkYqdmZmJkJAQnu9tKSkpLF++nLLWRnTDCC0xgK40NldXV55mxBUVFdi1axffrjnZPjR0sGvXLixcuBB2dnZQU1MDUPN62rRpA29vb9JCS0ZGhrvLLi8vj8ePH2Ps2LFQVlbG8+fPSY+fbkFBJ+PHj8fGjRuxadMmKCsrgyAIZGRkwMXFBWw2m1TsH3ecq6qq8OzZM2zZsgXm5uakYgM1/Up+ZsG+detWVFRUCOUM+PbtW4SEhPA4L82cOZOSkzSCIJCfn4+FCxfyXSd7OuHo6Mi1z6+oqMCWLVvw77//4suXLzhx4gTZoSMhIYGvncG4ceMgIyODtWvXkhJaHA4Hjo6OuHjxIiQkJBAfHw93d3eUlZXB29ubzzq9sdCZRvz582eukO3Xrx8ePHiAvn37YsmSJVizZg2cnZ0bHbPuZ+3jx4/1ZimcPHmS2wS3MXz9+hU3btyAjIwM1NXVoaurS3q+FYSMjAz3JAuo2fwKDAxEREQEwsPDhfq8czgceHh4IDo6GtLS0jA2Noa9vT3lNY/Lly/Hzp07+UwA7ty5A0dHR1y6dEno2HTWOzI0zPz583HixAmeTa0HDx7A0dERz58/F1po7dy5E+PHj4ednR332qVLl7Bx40bs2LEDAQEBpMdOJ2vWrIGlpSXu3r3Lnc8yMjLw8OFD+Pv7k4pdXV3N3eCqS1lZmdj01Wre1j0MAP4vjS0mJgbt2rXDmTNn4OPjg//9739Cp7GNHDkSHz58wKtXr7j/qampobi4mOfaq1evKH411JCTkyNQjMydOxf5+fmk4ysrK+P06dMAgAEDBnD7Wj158oR0o0ygRlDU/S8nJwcxMTEYNmwYJamJdLJx40bIyclh/vz5GDFiBEaOHIklS5ZAWVmZuxtHFZKSklBUVISTkxP2799POl6tBbu3tzfmz5+P+fPnY9++ffjrr7/w/v17bNu2DZs3bxaqhuLRo0cwMTHB+fPnIS0tDYIgEBERgSlTpuDx48ekx+7m5gYNDQ34+voiMDCQ+19QUBDpNgyKioo4f/48dHV1oa2tDQkJCRgZGfEUN5OBznYGhw4d4qYR124czZs3D8+fP4enpyep2AA9828tnTp1wsePHwEACgoKyMvLA1BTz1ZYWEh67HPmzIGfnx/PtcLCQtjY2GD79u2NjpeXlwcDAwMsX74cixYtgpGREXJyckiPUxCFhYW4fPky3N3dYW5uDmNjY5w/fx4aGhpYv369UDH37t2LM2fOYOLEidDX18eZM2cE2uuTJT8/H5MnT+aaJ1RWVmLv3r2YO3cuJQX8srKyaNmyJS5duoSEhAR07dqVEVkiQFlZGfPmzcO7d+9QUVGBvXv3YubMmWjXrh2PgVVjuX//PpYvX86TQi0lJYXFixcL5VAratTU1BASEoKePXsiOTkZSUlJ6N27N06ePAkNDQ1SsceMGYPDhw/ziK2SkhLs2rWLdGyRQTA0e5SVlYknT54QBEEQFhYWRGJiIkEQBBEfH0+Ympo25dCaDA0NDSIrK4sgCIJQVVUlXrx4QRAEQdy8eZPQ1NQkHf/27duEiooKcfToUaKoqIgYNWoUYWxsTKipqRHr168nHb8+cnJyCH19fdriU0l+fj5x8eJFIj4+nvv+00Vubi4xbNgw0nFUVFSIZ8+e8V1//vw5N/7r16+FutfChQuJFStWEN++feNe+/btG7Fy5Upi8eLFwg/6/6OiokLp+7xkyRIiODiYO7fQiY+PDzFnzhziw4cP3Gtfvnwhli9fTnh7e5OKbWBgQKSkpBAEwTsXpKamEmPGjCEVmyDonX83bNhATJs2jcjLyyMuX75M6OnpEVlZWcTOnTuJP//8k/TYQ0NDCVVVVcLKyoooKCgg4uLiiFGjRhFsNpu4d+9eo+PZ2NgQ5ubmRGZmJnHv3j3CwsKCmDFjBulx1nLq1CnCwcGB0NfXJ5SUlIihQ4cSlpaWxKFDh4i7d+8SVVVVpOLr6ekRMTEx3J+vXr1KqKqqEtXV1WSHzkN5eTmxceNGYvDgwcSOHTsIU1NTYvTo0URUVBTp2F++fCFsbGwIJSUlYuDAgcTAgQMJJSUlwt7envj+/TsFo2eoj8rKSsLOzo7Q19cnJk2aRAwfPpw4deoU6bja2trcOawuN2/eJDQ0NEjHF2fevXtH/Pnnn4SmpiYxaNAggs1mE6qqqoSuri7x6tWrph7eL8GkDooBdKexiSP6+vrYt28f9u7dy72Wn5+P7du3Y9y4caTjjxgxAvHx8eBwOJCVlUVISAhCQ0PRvXt3ylzYBCEhIUFpI0Q6qa2noBJBtWulpaUIDw+nxKiFTgv2zMxMhIaG8qTjtmjRAitWrICFhQW5gQMYPXo07ty5Q1lD0u/fv2P37t349u0bunXrBi0tLYwZMwZaWlqUF2DT2c7g/fv33ALyunTv3h2fPn0iPXY6518HBwesW7cOt27dwpw5cxAWFoYZM2ZASkoK7u7upMdubm6O0aNHw9HREX/++ScqKiqwZMkSLF26VKiT+bt37+LYsWMYMmQIgJo2CcbGxvj69SslFvLbtm3D4MGDYWhoCE1NTQwfPpzn74ksBQUFPDWH2tra+PbtGz58+ID//e9/lN2nVatW2LZtG6qqqnD8+HFISUnh+PHjlPQxE0W9I4NgJCUl4enpiXXr1uHChQsIDQ2l5Htp4sSJ2Lp1K7Zs2cKNl52djW3btsHAwIB0fDp59uwZLl68iIcPH6K0tBTt2rXD4MGDYWxsTIkzdLdu3RAZGYmYmBg8ePAA1dXVmD17NqZMmUI6LVxUMEJLDKhNY7O3t8eAAQOQmJgIa2trytLYxBFHR0csWrQIGhoaqK6uhpmZGUpLS0k5XtXFyckJGzZsQLdu3QDU1E84OzujpKQEtra2OHjwIKn4dAsKqvmx50dDkKkVErRIkJKSgpqaGrZs2SJ03FrotGBv06aNQAMDYU0NfmTEiBHYvHkzrl27hj59+vDZXjfWyfLYsWOoqqpCTk4O0tPTkZ6ejq1bt3Kd47S1taGtrY3hw4eTnmfobGegqKiIGzdu8NXUxcTEoF+/fqTj0zn/tm/fnmcuOXLkCHJzc9GlSxfKFv7FxcUoLy9HixYtwOFw8ObNG3z//l2osZeWlvKMS0FBARISEiguLqZEaN28eZNb18ThcOp1o7x586ZQaUOVlZU8r1taWhotW7bE9+/fhRtwPbx9+xabNm3CjRs3sGLFCty/fx/W1tZYtWoVrK2tSZkG0VnvyMCPIJdYFosFKSkp/PXXXzziWdh69r/++gsvXryAlZUVz2fDwMCAkvUMXezbtw/+/v7cVhLt2rVDQUEBEhIS4OXlhaVLl1LisJydnY0ePXpg+vTpAGr6dj18+JDSBux0wggtMWDVqlVYtGgROnbsCFNTU/j4+IDNZuPt27cwMjJq6uE1CW3btkVoaChu3LjB3eUYMGAAdHR0hHa8ysjIwMuXLwHUCKEhQ4bw7Zjk5+fjxo0bpMdPt6Cgmro9P16/fg0/Pz+Ym5tDTU0N0tLSyM7ORkhICOn+YnRbddNpwa6hoQEPDw94eXlxT4SKioqwa9cuaGpqkh77qVOnICsri7t37+Lu3bs8j7FYLKG+0CQlJTFs2DAMGzaMa7KRl5fHFV7Lly8HQRC4c+cOqbHT2c5g1apVsLOzw5MnT1BVVYVz587h6dOniI+P5znxJhOfyvn3zZs3DT7esWNHVFZW4s2bN6ScJIEag6BTp05h/PjxCAwMxIsXL+Do6AgjIyNs3bq10f2uqqur+eZXaWlpyvo31TWPWLZsGQ4dOsQjtsrLy7Fz506cPn26WTdxZbPZ6NatG0JDQ6GsrAwACA0Nhbu7O+Lj47n1v8JAZ70jAz/11anX9m+ioo6dIAj4+fnh6dOnXCMlRUVFKCgokI5NF+Hh4Th69CjWrl2LmTNn8rQHKS8vR1hYGPbt24d+/frB0NBQ6PvExMTA0dERdnZ23M26169fY8GCBdi3b1+zcx8VBIsgBDTYYGh2vH//HhwOB71798aTJ0940th+11Ot+ggMDMT8+fMb/f9lZmZyXbjq6z3TunVrLFy4UKz7YJHFwsICU6dO5e4u1RIVFYXAwECcPXuWVPyvX7/iy5cvaNeuHSW75PUhyIKdDG/fvsXs2bPx6dMn7hfks2fP0LFjRwQHBwuVRlFSUvLTND4Oh4Nr167hzz//FGLU/0dFRQXS09Nx8+ZN3L59G7m5uZCWloampiZpIxK62xlcv34dhw8f5m669O/fHzY2Npg4cSKpuLVQOf/+aHdM1HEx/PEaWTExfPhwrF+/HtOmTeNe+/r1K3bs2CGUWFFSUuL26KpFTU0NUVFRlKW01mJoaIg+ffrAx8cH0tLSSE1NhbOzM8rKyuDg4MDzmn6VQYMGISUlhWt/DwDq6uo4f/48peN3dXXF33//zZf2+Pz5czg6OiI0NFTo2AcPHkRKSgr279/P7V1UWloKR0dHDBo06Lf+bhJXDAwMsG/fPm5KrjhgZmYGQ0NDLF68uN7n+Pn54fr166R6XU2ePBnm5uZ86fdBQUE4e/Yszp8/L3RsUcEILTGgNo3txwVhSUkJ1q9fTzqNTZw4evQoYmJiIC0tjSlTpvDYEz9+/BjOzs7IysoivUBRUlJCcnKywCZ8VCEqQUE1w4YNQ3R0NF+d07NnzzBlyhShXJJKS0u5v9vaU0WgpibGxMQEVlZWlL1HmZmZUFBQQKdOnRAZGYmLFy9CXV0dixcvJt0HrKysDOfPn+fpKzR58mShxdygQYOQnJzMs7B1dHSEg4MD91phYSF0dHSE+sw/e/aM2yD21q1b4HA4GDp0KNdWe9iwYUKfENelPufC2nYG8fHxQsd++fIl5Yv8ulA9/44YMQJlZWUYMWIE2Gx2g3WOo0aNEmrMtdy+fRuqqqoCxWBiYmKjbdmVlJTg7OzM815s3rwZtra2POIFIC+eP3z4ACsrK/Ts2RNdu3bFmTNnYGxsjA0bNgjdI01QCrQgoQuQS4FeuXIl7OzsBDoBCjoVbAxz5sxBdnY2JCQk+Oodf2yUTGcftd+Vr1+/Ii8vDxUVFTybsSwWS+hUtjFjxiAwMFCsnCPV1NRw5syZBsf85MkTmJubIyMjQ+j7qKio4MKFC3xz/MuXLzFp0iSxcGVkUgebKaJMYxMX9u/fj0OHDmH06NFo0aIF3NzcICEhgVmzZuHo0aPYt28fWrduTUnfL0EpbEVFRXyLicYiSkFBF/Ly8oiJicHy5ct5roeFhQlVE1NcXAwLCwu8ffsWBgYGMDc3R/v27fHlyxfk5OTgyJEjuHjxIk6ePCmUSUVdQkNDsXXrVgQEBEBWVhZOTk7Q1NTE8ePHUVFRQWo3ePr06XB1dRWqN1F9CNoHu3TpElauXMmz2BRmv0xfXx+vX79Gz549oaWlBXd3d2hqapJ+jwVBZ380AwMDDB8+HGZmZjAyMuJJYREWOuff1NRUJCUlITY2Fh4eHujTpw+MjY3BZrMp6bdWl9WrV8Pf31/gTrmwva8E2aF7eHjw/MxisUgLra5duyIoKAhWVlZISkrCgQMHSKcJiaon5M2bN+s18SC7cUFnvSNDw1y5cgXr1q1DaWkp35xL5gTa0tISq1atwty5c9GnTx++vm516/GaC9++fePrE/cjHTp0QHl5Oan7dO/eHbdv3+YTWnfu3EHXrl1JxRYVjNBqprBYLG4dD4vFEvjl1rp1a1hbW4t6aE1GTEwMVq9ezV3gR0ZGws/PDx8+fICPjw8MDQ2xadMm0mIIAL58+QIPDw9YWFigX79+sLa2RlpaGhQUFHDkyBGhdtBFKSjoZPXq1Vi9ejVSU1MxdOhQVFdX486dO8jNzeXr2fMr7N+/H9XV1YiJiUH37t35Hn/37h1sbGwQEBAAW1tbUmMPDAyEs7MzNDU1sWfPHvTv3x8BAQFISkrC5s2bSQmtly9fUrLIFwZhTuJevXqF7t27g81mY8yYMVBTU+Mz2KCLuv3RbG1tMWnSJKFjBQcHIzo6Gh4eHnB1dYWBgQFMTU1J1cXROf/KyMhgwoQJmDBhAr5//46rV68iNjYWhw4dwsCBA2FsbAwjIyNKFhGdOnXCly9fSMephe4aSkEpplOnTsXevXtx9uxZlJaW8lxvLKampiRG17j7eHp6YsWKFZCXl6/X1EMYmNTApsPT0xOamppYvnw5pd/Re/bsAVDj4vkjVKQQ0wFBED/dNCCbIQIAs2fPhouLC168eMGti8vOzkZgYCDfZm9zhUkdFANEkcYmDqioqCAyMhJ//PEHgJraFBUVFbRp0wbOzs6kd1Dr4uTkhPT0dPj6+uLx48f4+++/4ebmhtjYWEhJScHb27vRMbds2YK0tDQEBAQ0KCj09fVJCwq6yczMxIkTJ7iNeAcNGoSFCxcK1dxWT08PmzZtgp6eXr3PSUhIwK5du0ilmAHA0KFDcenSJXTv3h3Tpk2DpqYm1q5dizdv3sDQ0BBZWVlCx67NR7e2tha4KymMscGv1MQImzr477//ctMGb9++DQkJCYwePZqbNvhjaigdPHz4EObm5pSkf1RUVODatWuIjo5GYmIiOnfujKlTp2L16tWk4opq/i0rK8PVq1dx8eJF3LhxA0OHDiXdiNrd3R0nT56Erq4u5OXl+U5Zmtui/VfnD7KLz1evXiE8PBwZGRncTIXhw4djxowZlKSh/vnnn3jx4kW9C00yY6e73pGhfoYOHYqYmBiB7STI8Pr16wYfp/qkmwqUlJRw9uxZblsUQXz8+BEzZ84kLRT9/PwQFBSEDx8+AAD+97//YfHixZS0TREFzImWGEBXGpu48f37d56jahkZGbRs2RL29vaUf7kkJibCx8cHioqK8PPzg7a2NiZPnoyBAwdi7ty5QsfctGmTQJEFAHJycrC1tcWuXbuavdBSV1fn6UdDhsLCQgwYMKDB5ygpKf3Ure1X6Ny5MwoKCiAlJYXc3FysXbsWQM3fGNmF9L59+1BVVYXbt2/zLLCoMjagmto+aPPnzweHw8GtW7eQnJyMU6dOwcXFBb169YK2tjZ0dHRIp2yJop2BtLQ0N40wMjISPj4+OHz4MGmhJar5t7KyEt+/fweHwwGHw+FJLRaW+Ph4dO7cGffv38f9+/d5HhPWqbKWDx8+YN++fcjMzOSrVwGEqw+i+8QMqMmMcHZ2BovFgpqaGoYMGYLPnz8jLCwMwcHBcHFxIXXCCoC0+2pD1GffXlvvyAgt+lBQUMC7d+8oF1oNCSmqWw9QyY+GWD9SX/1jY7GxsYGNjQ2Ki4shLS0tNv2zamGElhhARxrbfwk68tXLy8u5giglJQU2NjYAgJYtWwptZSxKQUE3iYmJOHr0KP7991+EhYUhIiICffr0wZQpUxodq6Kigu/050datmxJiXUxm83G2rVr0apVK8jJyWHUqFGIjY2Fi4vLT780fsaxY8dIj08QVHxR/QwZGRnuSRZQkwYZGBiIiIgIhIeHkxaJdLczKC8vx+XLlxEdHY2bN2+iZ8+esLa2piRVjM75t6SkBJcvX0ZcXBzS0tLQuXNnTJw4EcuXL4eamhrpsf/zzz+kY9THxo0bcf/+fbDZbNpSnSMjI9GiRQuujb6trS309fUxefJkoeI9ePAA69atw7x582Bra8tzwsfhcODr64v169ejX79+Qp3O10JniiKd9Y4MDbN27Vq4uLjAzs4Offv25UsJFbYdQ3FxMXx9fZGXl8ddXxAEgYqKCjx58gTp6emkx041QUFBtN+jtLQUrVu35qYoFhYWcrMLJk6cSGlKLp0wQksMcHNzQ3p6OhYsWIDLly8jIyMDHh4e3GJqYdLY/ktISkpSHlNRURHXrl1D9+7d8eHDB4wdOxZATe8IYZ2BRCko6CQlJQUrV64Em83G3bt3UV1djcrKSjg5OYEgiGa9o/rXX39BTk4OL1++xNy5cyEpKYmPHz9i1qxZWLVqldBxS0tLoaKiUm8BPBlcXV154lZUVGDXrl1o06YNAGp2PAsLC3Hnzh1kZmYiMzMTDx48QMuWLaGhoUFJDzA6Tyrs7Oxw7do1sFgsGBoa4vjx45Q2sqR6/i0uLkZCQgJXXMnKymLixIlYtmwZLQ04CYJAUlIStz9P//79oaGhQXrevHnzJvz9/WlrGhoUFARPT09s3LiRe01OTg4bN27E169fMXPmzEbHPHr0KCZOnCiwCayMjAxWr16NgoIC+Pv7w9PTs1GxDxw48MvPpTJlk8p6R4aGWb58OaqqqrB8+XJKsxa2bt2KGzduQFtbG3FxcWCz2cjPz8eDBw9gb29P1fAphawjakNUVlZi06ZNOH/+PC5cuIA//vgDV65cga2tLaSkpCApKYkjR44gODj4p+1PmgOM0BID6EhjE1cCAgJ4XPkqKysRFBSEDh068DyP7BfZ6tWrsWrVKlRUVGDSpElQUFDAjh07EBISAh8fH1KxxR1vb2/89ddfWLBgAbdmys7ODm3btsXRo0eFElo//l5/hKxzUS0xMTGYOnUqzw78vHnzhI73+fNnODg44Pr162CxWBg3bhxcXFwoSysbOXIkNy+9FjU1NRQXF6O4uJh7TZjFbmhoKFdcvXr1CtLS0lBTU4Oenh7Wr1+PoUOHUmLtXgtd7QwKCwuxefNmTJw4kRbHTqrnXx0dHbBYLGhpacHV1RUjRozgvs8/nmaTbVhcUlICa2tr5OTkoF27diAIAqWlpRgyZAiOHTv2U9ewhmjdurXQNuu/QnBwMHbu3AljY2PuNScnJygrK8PHx0cooZWenv7TnnDm5uZYsWJFo2NHRETw/Pz27VtIS0ujd+/ekJKSwosXL1BRUQFlZWVaauMkJCRQUFBAeVyG/4OurIUbN27A3d0d48aNw6NHj2BtbQ0lJSVs3LgRT548oeWeZOFwOPDw8EB0dDSkpaVhbGwMe3v7n24m/woBAQG4cuUKNm7ciB49eqCqqgpbt25Fjx49EBYWhtatW2PlypXw8fHBhg0bKHg19MIILTGAjjQ2caRHjx64ePEiz7WuXbvy1QKQrT0AaqyPExMT8f79e24KCZvNxsyZM0n1uhCVoKCTR48e8dk5AzUNRhuzq1uLoN+rIOqrbWsMbm5ucHJywogRI7jOb2QWsx4eHsjKyoKtrS0kJCQQHByMLVu2wMvLi/RYAZBq9Pgztm3bhsGDB8PQ0BCampoYPnw45SdyomhnEBwcjJcvXyIvLw+dOnVCr169KE23pHr+rT2xTkxMxPXr1wU+h6q6Pnd3d3z79g2RkZHceezhw4f4+++/sXv3bmzdulXo2FOmTIG/vz+2bdtGS1ZBQUEBlJWV+a6rqqoKnV5dVFSEbt26Nficrl274vPnz42OXTdN8/jx47h27Rp2797NFaO1mzI/Sx//GaKod2QQDF2nOGVlZRg4cCCAmtrZhw8fQklJCRYWFg02BG5K9u7dizNnzsDExAQSEhI4c+YMysvLBTq0Npbo6Gg4OTlxN21v376NgoICrF+/nmu+YWVlBWdnZ0ZoMVADHWls4gid9QaCkJWVxadPn3Dp0iVISEhgyJAhpBb7ohQUdNKuXTsUFBTwFQQ/efKE72TxVxDl7zU1NRX37t1DYmIizp07Bzc3NwwcOJArugYPHtyoeNevX4e7uzt0dHQA1Jw2WVlZobKyUmRW6cJy8+ZN7okGh8OpN9/95s2b0NDQaHR8utsZEAQBf39/nDhxgmcnv2vXrrCwsICNjQ0lgovq+VcUtQ21XL16FV5eXjz1RrVNh+3t7UkJrZKSEly4cAHXrl1D7969+T4/ZF+ngoIC/vnnHyxYsIDnemJiInr16iVUzK5du+Lp06cNzrH5+fmQk5MTKn4tR44cQUBAAM+JX/v27WFvb4958+aRSgeju96RoX6+f/+OsLAwnloqoGb+vH//vtCuuN26dcPr16/RvXt3KCgo4NGjRwCAVq1a4dOnT5SMnWri4+Ph5ubGPXEeN24c7Ozs4OLiQnreffnyJYYPH879+datW2CxWNwaYqBms+7HbI/mSvNeCTAAYNLYmoLS0lLY29sjKSmJ66bFYrFgbGyMHTt2CFWEKWqhSBeTJ0+Gm5sb3NzcwGKxUFZWhuvXr8PFxYUnzYcqPn36xDVl+BWh2hAsFguqqqpQVVWFra0t3rx5A29vbxw6dAgHDx5s9AlCUVERzw61mpoaqqqq8PHjx5/unDc1ddPGli1bhkOHDvF8rsvLy7Fz506cPn1aqJMVuvujrVy5EtevX8eUKVOgqanJ3RhJS0vDoUOHcO/ePUrmR6rn37q74j8TuGSprKwU6KbZpUsXnp5UwkJnPZC1tTXWrVuHnJwcnv45MTExAvsN/Qq6uro4dOgQNDU1BS4Gq6ur4evrS9pls6KiQmB2wsePH0kvQkXhzMggGFdXV0RGRmLw4MHIzs6Gmpoanj9/jo8fP/JtCDSGP//8E05OTti5cye0tLRgZ2cHFRUVJCQkiKTNhjAUFBTwOA9ra2vj27dv+PDhA/73v/+Rii0pKYmKigruz7dv30bXrl3Rt29f7rWPHz+Kj/sgwSAWFBUVEbm5udyf7927Rzx58qQJR9S0FBQUEOvXrycMDQ2JCRMmEOPHj+f5jyzr1q0j9PX1ievXrxNfvnwhPn36RFy9epUYP348sWPHDgpeAT8lJSVEQEAAYWhoSEt8quBwOIS9vT0xcOBAYuDAgYSSkhIxcOBAYsmSJcS3b98ou096ejqxdu1aYtiwYcTAgQMJU1NT0jGrq6uJ7Oxswt/fn1i0aBGhpqZGDB06lLC0tCR8fX0bHW/gwIFEYWEhzzVVVVXixYsXpMcqSiZOnEjY2NgQHA6HIAiCSElJIfT09IhRo0YRZ86cESrmuHHjiH/++afB51y+fJn4888/Gx37zJkzxIgRI4gHDx4IfDwvL48YPXo0cfbs2UbHFgRd8+/ChQuJ79+/81wrKysjNm7cSCgpKZGOb2lpSbi6uvJdd3FxIczNzUnHp5uYmBjC3NycUFVVJUaOHEnMmjXrp5+phnj37h2hoaFBLFmyhOf3SRAEkZubS1hbWxPjxo0jioqKSI3bwcGBMDY2Jm7evEmUlpYSX758Ia5du0ZMmDCB2LZtG6nYBEEQ5eXlxPv374ny8nLSsRh+HU1NTSI6OpogCILQ19cn8vPzCQ6HQ6xYsYJwcXEROu73798JV081DPkAAFfFSURBVFdXIioqiiAIgti4cSMxcOBAYuTIkURqaiolY6caOr/75s2bRxw7dowgiJq/2SFDhhDOzs48z9m8eTOxcOFC0vcSBUzDYjHi2bNnyMvLoySNTdxZunRpg9bCZGu0Ro4ciYMHD2LkyJE811NTU7F27VqkpqaSil+XjIwMhIaG4tKlS/j+/TsGDx7MV1jdHHn+/Dlyc3NRXV2NAQMGoEePHti3bx/Wr18vdMzS0lJERkYiLCyMWwQ8ZswY2NjYUJIfP2LECJSXl0NVVRUjRoyAhoYGqdqkX2koLA58+PABVlZW6NmzJ7p27YozZ87A2NgYGzZsENrwYOjQoYiLi2uwR8yrV69gZGSE7OzsRsWeNWsWjIyMMH/+/HqfExISgpiYGJw8ebJRseuDjvnX0NAQffr0gY+PD6SlpZGamgpnZ2eUlZXBwcEB06ZNIxX/zp07sLS0hJKSEnf3OSMjAw8fPoS/v79QKaF1efv2LUJCQngcDc3NzUmbeNBJbm4ubG1t8fLlS7Rq1QodOnRAaWkpSktL0b9/f+zdu5d0Sn5paSlsbW2RkpLCPcEiCAKGhobw8PAQKiNCFPWODA2jrKyMS5cuoUePHli2bBnYbDYmTZqE7OxsrFmzRqjecQBw4cIF6Ojo8KTel5SUoG3bts02BZ3O776kpCQsW7YMurq6yM3NxcePHxEREQFFRUU8fPgQYWFhCA0Nha+vL3R1dcm+FNppnr9BBh7oSGMTd+i2FpaUlBQo4Lp27UqJ/TrdgoJqvn//Dnd3d8TExEBaWhpTpkzBX3/9xU1rSE5OxpIlS/D27VuhhFZWVhbCwsIQGxuLr1+/on///rC1tYW3tzccHR3Rr18/Sl7H6NGjcevWLTx79gzdu3fH69evIS8v36AY+Bnv3r3js1h///49n0FAc158du3aFUFBQbCyskJSUhIOHDhASfoUXe0M8vPzuXVx9aGjo4N9+/Y1OvaP0Dn/BgcHw8rKCitXrqRM4NZFTU0NISEhCAgIQHJyMgiCwMCBA7Fp0ybSxgmPHj2ChYUFWrZsiWHDhqG6uhoREREICQnBqVOn0L9/f9LjLyoqwtOnT1FdXQ2gRqxwOBxkZ2cL3RR40KBBuHjxIq5du4a7d+/i06dPkJWVxfDhwzFmzBhKnDZrHVifPn2KvLw8sFgsDBo0SOgFKN31jgy/RqdOnfDx40f06NEDCgoKyMvLA1BTz11YWCh03G3btuHkyZM8Qqu525azWCy+NFiqTIh0dHRw+PBhhIWFYdiwYZg/fz538+PMmTO4ePEitm3bJhYiCwCYEy0xwMnJCenp6di0aRPU1NRQXV2NzMxMuLi4wMDAoN5O8f9ltLS0EBISgj/++IOW+AcPHkRKSgr279/PrXEoLS2Fo6MjBg0aJPSJmSBBwWaz4e3tjfPnz1MmKKjGxcUFYWFhMDExgYyMDKKiomBjY4MlS5bA1dUVp06dQp8+feDq6sp3CvgzzMzMkJubC0VFRRgYGMDY2Ji7SBsyZAjl70t1dTWysrKQkpKCGzduICsrC//73/+gqanZ6NoPJSUlvi8X4v87xv34M1kHOaoR5F5WXFyMvXv3QltbGxMnTuReF8ayX9CO548UFhZCR0en0e+Nmpoazp8/z2fIUpcXL15g+vTpuHXrVqNi/wjd829RURGsrKzw+PFjeHl5kRa4osLa2hqtWrXC7t27uSfC379/x9q1a8HhcHD48GFS8aOiouDs7AwOhwMWi8Xzd9WzZ08kJCQ0OqaTkxM2bNggstqO27dvIz8/H5MmTcK7d++goKAg1AnFli1bkJaWhoCAgAbrHfX19YWqd2T4NZydnfHw4UPs2LEDz58/h5ubG/bv34/Y2Fj8888/QpthzJw5EwsWLKClxpkufuW7rxYqv/u+fv2Kli1bUuosSzeM0BIDRJnGJi64u7vj8+fPtFkLz5kzB9nZ2ZCQkOB+OT579gxlZWXo0aMHzx/5r6YLiFpQUImenh4WL16M2bNnAwCuXbuG7du3Q1NTE2fOnIGVlRVsbW2F2t1XUlJC3759YWZmBm1tbQwaNIj7GJ3vC4fDQVpaGq5cuYKIiAhUV1fj/v37jYrRmEV8czuprOtE1xDCikQlJSXuYrw+ysvLcezYsUbHnzFjBkxNTTFnzpx6n3Py5EnExMQgJCSkUbF/hOr5l26BW1dMODk5NfjcHTt2NDp+LWpqaggNDeXaUtfy8OFDWFhYID09XejYAGBsbAwVFRUsWrQIs2fPRkBAAAoKCrB161bY29tjypQpjY45aNAgJCcn09r/C6jZlLO2tsa9e/fAYrFw6dIlbN++HS9evMCxY8cabZSjp6eHTZs2QU9Pr97nJCQkYNeuXUIv9hl+zufPn7Fu3Tpoa2tjzpw5WLx4MZKSkiAlJQV3d3ew2Wyh4m7atAlnzpyBkpISFBQU+FLZyfyd0sW5c+d++bmmpqak70fVpkVTIB6j/M2hO41NHKHbWlhLSwtaWlqkYvzIgwcP0LdvX0ydOhXa2tqUpNaIisLCQh5rVR0dHbx+/RqXL1/GsWPHMHr0aKFjx8fHIyIiAsHBwdi9ezf69OkDY2NjWnb3cnNzkZKSgpSUFGRmZkJGRgZjxoyBq6urUGkIonSQoxq63cvobGdgamqKAwcOQFtbW6ArV35+Pg4cOABHR8dGx/4Rquffhk7Arl69iqtXrwKoEbjCCK1Xr15xU+1evXrV6P//V2nTpg2PM1gtgq4Jw8uXL+Ht7Q1FRUUMHDgQRUVFGD9+PCorK+Hr6yuU0BLVvvKePXvAYrFw+fJlmJiYAAD+/vtvrF27Fh4eHti9e3ej4hUWFv60/5aSkpLQ/cUYfo327dvj4MGDAGocPS0sLKCqqoqZM2eia9euQsd9+vQp185cXCzLqRBPv0JpaSkWLVqEu3fvgsViQVtbG56enkJvWjQFjNASAywtLeHi4sKXxrZv3z5YWlo28eiaDjqthcmaaQhClIKCaioqKtC6dWvuz5KSkmjRogU2bNhASmQBNcXcdnZ2sLW1RVJSEs6ePQt/f3/4+voCAGJjY2FlZUVJ7YGpqSl69uwJPT09bj0cVbtidFiki5LIyEi0aNECRkZGAABbW1vo6+tj8uTJQsWjs53BrFmzcO3aNUybNg1mZmZQV1dHx44dUVpaitu3b+P06dPQ1dUVajH+I1TPv3QL3LpNruv+u6KiAqmpqSAIApqamqSbU2toaMDDwwNeXl7cepKioiLs2rULmpqapGIDgIyMDPdvSV5eHo8fP8bYsWOhrKyM58+fCx1XFClHV69exe7du3lqshQVFbFp0yasWLGi0fHorHdk+DknT57kGlTNnDkTbDYbc+fOxaNHj0AQBE6fPo3jx49DQUFBqPh0Nqanm1evXiE8PBwZGRkoKipCp06dMHz4cMyYMYMSQ6g9e/YAAGWbFk0BkzooBtCRxsbQMILSe+oizE5zLdXV1VxBcfXqVe6X47JlyygTFFRTn8PQz+pkhKWkpARRUVE4d+4ccnNz0apVK5iYmJBqsAoAeXl5P90ZFha6HeToJCgoCJ6enti4cSNmzJgBoCZdJSwsDOvXr8fMmTMpvyfZ/mi1JxsnTpxASUkJ93rXrl2xYMECLFy4kJJFNd3zL9UCFxC8MLSwsOCKvG7duiEwMJBUj553795h1qxZ+PTpE3eB+ezZM3Ts2BHBwcFCNxWuZf78+VBRUYG9vT2CgoKQmJiIo0eP4vr163BwcBDqlFhQXUl9kNkYUVFRwYULF9C7d28eJ7b8/HxMmzYNd+/ebVQ8OusdGRrm6NGjOHDgACZPnoxWrVpxf6/l5eXYsmULqqursWPHDvTq1Qv79+8X+j7fvn1DXFwc/v33XyxcuBB5eXno378/ZGVlKXw11BITEwNnZ2ewWCyoqalBVlYWnz9/xr1798DhcODi4kJ6Q1xPTw+7d++Guro6z9/SnTt3sGLFCrEonWFOtMQAOtLY/gvQaS1cX3pPixYtICcnR0poSUhIQFdXF7q6ujyC4uDBgzh27BglgoIOBC1Q6Nod7tixIywtLWFpaYkHDx4gIiICFy5cIP2+DBgwgLtg+/fffxEWFoaIiAj06dOH9OkH3Q5ydBIcHIydO3fynK46OTlBWVkZPj4+lAotQe0MhEFKSgorV67EihUr8PTpU657XJ8+fShxjquFzvm3rsCtRU5ODhs3bsTXr1+Fet9/XBju378fERERqKqqQkhICHdhuHv3bnh5eQk9djk5OcTExOD8+fN4/PgxCILAzJkzMXnyZErMJlatWoVFixahY8eOMDU1hY+PD9hsNt6+fcsVpcLg5ORE+2bW0KFDcfHiRSxevJjnekhIiNCf94CAgJ/WOzJQT3h4OLZv386dG9lsNmbOnAlfX19uywQnJydSJiSFhYUwNzfHx48fweFwMGPGDAQEBOD+/fsIDAwk3W6ADh48eIB169Zh3rx5sLW15Tkh53A48PX1xfr169GvX79frgcWRFFRkcC0zPbt24vNZ5450WIQSwRZC9+/fx9fv36lzFq4LlVVVXj27Bm2bNkCc3NzWtIW6wqK5lbTo6SkBGNjY57JNDo6GuPHj0ebNm14nku2cLeqqgrFxcVo164dz/3evn0LWVnZn6bQNERKSgqWLFmCSZMmITY2FjExMThz5gz8/Pzg5uZGSkAD4usgp6KigujoaL7TyZcvX4LNZiMrK4tUfDrbGWRnZ2Po0KF81798+QJXV1e4u7uTik8nBgYGsLOz40sfjo6Oho+PD+Li4hodc+LEibC1teXGzMrK4i4Mx40bBwBIT0/n9nkSlvT0dKirq/OJ2vLycgQEBFCSfv3+/XtwOBz07t0bT548QWhoKLp37w5LS0tIS0s3Ot6vnAxRQWZmJhYuXIgxY8YgMTERkydPRn5+PnJycnD06NFGp1uPHz/+l59LZ8ru74igfoDKysqIiopC3759AdTUVY0bNw45OTlC3WPt2rUoLS3F3r17oaWlhaioKLRv3x5r1qxBixYtuGn0zYm//voLLBYLnp6e9T7H2dkZ3759a/A5P8PCwgJjx47F4sWLeU60tm3bhocPH1LWJ5FOmBMtMYDONDZxxcPDA6NHjxZoLezp6UnaWvhHJCUloaioyN25okJo/SgoBg8ejMGDB8Pa2hrfvn0jJSioZuTIkXxFumpqaiguLkZxcTEl94iNjUVQUBCysrK4ResDBgzAvHnzMH36dGzfvh0qKiqwsbER+h7e3t5Yu3YtFixYwHXnsrOz4/a9aezfkqC/zalTp2Lv3r04e/YsSktLea43VxQUFPDPP/9gwYIFPNcTExNJpYCJoj/aokWLcPz4cR63yn/++QebNm1CVVUV6fh0zr8FBQVQVlbmu66qqiq0scGbN2+goqLC/XnYsGGQkpLiEdHy8vI86ZbCYGFhAW1tbezfv5/nBKu8vBw+Pj6khVate2JtsXu/fv3g7OyMkpIS2Nrack0JGoOoLKHV1dURFhaGo0ePQl5eHnfv3kX//v2xfv16nt/Nr8KIp6ZDUH2ctLQ0j9BnsVhcAxphuHnzJo4cOcJzYtmhQwc4Ojo22zr89PT0n6ZKmpubC1WTWBd7e3ssXLgQWVlZqKysxKFDh3g2LcQBRmiJAXSmsYkrmZmZCA0N5TnxaNGiBVasWAELCwva7ishIYGCggJSMUQhKKiG7mJdV1dXnDhxApqamlizZg1kZWVRUlKCtLQ0bNy4EZGRkcjPz8f27dtJ3efRo0fw8PDgu25oaIgDBw40Oh7dDnKiwtraGuvWrUNOTg53IZidnY2YmJhG9xarpW47g9oeMbUnzd7e3pSN3cTEBAsWLEBgYCC6d+8OFxcXXLhwAZMmTRKqefaP0Dn/0iFwRbEwrOXNmzcwNzfH4cOHSddkATVppS9fvgRQI3CHDBnCl4aYn5+PGzduCBWfzgQeQfNHnz59eARuUlISkpOTSS8+f4RsvSND01JWVsZjNlWX5mpwUlRU9FPHv65du+Lz58+k7qOuro7Q0FAEBARQsmnRFDBCSwz40aXqxzS23xG6rYUF7WKXlpYiPDwcw4YNEzquqASFOHH58mWEhYXh8OHDfBbrNjY2SEhIwMqVK7FkyRJ06NCB1L3atWuHgoICvhS5J0+eCBWbbgc5UWFiYgIpKSkEBQUhISEB0tLSUFRUhLe3d4O9expCVO0MNmzYAGlpaSxYsABSUlKQkZGBn58fdHR0KIlP5/xLh8AVFSwWC0eOHIG7uztmzJgBb29vjBgxgnTMWmHLYrHg6urK95zWrVvD2tpaqPgrV67EqVOnfvm5jeHAgQOQkJCAnJxcg89jsViUCS2q6h0ZGubH+rjKykoEBQVxvzPI1gqNHDkSp06d4ul7V1FRgUOHDnHrwJobXbt2xdOnTxtsz5Gfn//Tv4efcePGDWhqagrcIBUXGKElhtCRxiZu0G0tLGgXW0pKCmpqatiyZYtQMUUpKMSJkJAQLFmypN4+Vnfv3kXbtm2RkZFB+l6TJ0+Gm5sb3NzcwGKxUFZWhuvXr8PFxYUSm306HOREBdWtBkTZzsDBwQEyMjI4cuQITp06RetOJ5XzLx0CF6B/YQjUnA61bt0aBw4cgIeHB6ysrLBt2zZuHZgwqKurc4WtkpISkpOTuZb6VNAYMdRYoTVz5kxcvnwZQI1hApvNJmUCUB901jsy8COoH2DXrl35HEaF6QdYi6OjI+bOnYtbt26hoqICW7Zswb///osvX77gxIkTQselE11dXRw6dAiampoCU3Krq6vh6+tLuk554cKF6N69O6ZOnQpTU1NKLONFDWOGIcY8fPgQ5ubmuHfvXlMPReTQbS1MBwsWLMCIESPq/QL39PREaGgolJSUmu3kSgcaGhoIDg6u98SDzWZj2bJlcHFxQVpaGql7VVRUYN26dYiJiQFQs6AiCALjxo3D/v37SfUWagqLdCopKirC06dPuSllBEGAw+EgOzsby5YtEzouHe0M6qtbuHv3Llq3bs1j4U+2eXl9NNf5V1TGCYMGDUJycjLXWOL06dPYunUr5s6di6CgIFpsxmv79AjLpk2bcPnyZbRq1YoWMVRVVYWbN28iNjYWCQkJ6Ny5M/c+wvZYqkVQvSObzYa3tzfOnz9PSb0jQ9Px/v17nDp1Crm5uaiurkb//v0xZ86cZrmWAWrGO3XqVKioqGDNmjU8f0cPHz6Ep6cn8vPzERERQcqi/s2bN4iKisKFCxeQn58PdXV1mJqawsjIiM+Iq7nCCC0xoKE0tg4dOoh1szsylJWV8VgLDxw4kDJrYQD4+vUrvnz5gnbt2jVoq/uriFJQiBPq6uqIjo7mcXWqC0EQeP36NUxMTJCZmUnJPZ8/f879QhswYAAlixQ6HORERVRUFJydncHhcLjis3aXsmfPnkhISKDkPlT1R6ubYvMzyLpg0j3/0iVw6UaQg19qaipsbW1RWlpKWmh9+fIFHh4esLCwQL9+/WBtbY20tDQoKCjgyJEjQu9s0ymG6lJRUYHk5GRcvHgRV65c4Z7ostnsRrcgqVvvaGBgwFPvOGTIEEZoMTQJubm5sLW1xcuXL9GqVSt06NABpaWlKC0tRf/+/bF3715KrekfPHiA6OhoxMXFoaSkBH/++WezdpWthRFaYoCgHbe6aWzNsccC3dBlLVxaWoqjR48iJiaGW5QN1Lh0mZiYwMrKSmjR1RSCQhwwNTXFwoULG0yvO3/+PAIDA7lNWJsjdFuk04mxsTFUVFSwaNEizJ49GwEBASgoKMDWrVthb29PuseYIKhuZ8DhcCAjIwOgZrf1Z4Xavwqd86+oBC4d3Lp1C+rq6pCS4q1AyM/Px8WLFylxHUxPT4evry8eP36Mv//+G25uboiNjYWUlBQlhipUiqGG4HA4OHv2LHbv3o2ysrJGi1AlJSX07dsXZmZm0NbW5nHYZISWeGJpaYkDBw6gffv2P3UWpOtUngqqqqpw7do13L17l9vLcPjw4RgzZgyl/QxrycnJQVxcHE6ePAkWi4X09HTK70E1TI2WGPBfKbinEjqshYuLi2FhYYG3b9/CwMAA5ubmaN++Pb58+YKcnBwcOXIEFy9exMmTJ4VKd5KXl0dmZma9QovFYiEjI4PSXVVxgM1mw8vLCzo6Otx6u7p8+PABXl5emD9/vlDxx48f/0u2ziwWi9TCli6LdFHw8uVLeHt7Q1FREQMHDkRRURHGjx+PyspK+Pr6UiK06GpnUFRUhDVr1kBNTQ12dnYAasS7kpIS9u7dS7rekc7519fXF2w2W6DAXbVqFW33pYJRo0ahsrIS79+/59roEwQBCQkJvs0GYUhMTISPjw8UFRXh5+cHbW1tTJ48GQMHDsTcuXNJxwdq3Bj19PSgp6fHI4b27NlDSepjQUEBLl26hLi4OGRkZEBeXh7z5s1rdBxR1jsyiIaePXtyhUh9a4LmTG37hbZt22LChAmYMGECbfd6+fIloqOjER0djefPn2P06NHYtGkTJk6cSNs9qYQRWmIC1Wls/wWothbev38/qqurERMTI7Cw9d27d7CxsUFAQIBQXeDpFhTiiqWlJS5evIipU6fC2toa6urqaN++PUpKSnD79m2urauwiytTU9MGhVZUVBRevHhB+stOnB3kZGRkuKdB8vLyePz4McaOHQtlZWU8f/6cVGy62xls374dX79+BZvN5l7z8/PDli1b4O7uDjc3N1LjB+ibf0UhcOkiOTkZjo6OKCoq4nusZcuWMDExIRW/vLycOw+npKRwPx8tW7akpD9aLVSJoVrev3+P+Ph4xMXF4c6dO+jduzeMjIzg7OwsdD2YvLw87OzsYGtry6139Pf35zayjY2NFbrekaFpqJvSTDa9uSmIjIzE2rVrKSvVqI+ZM2ciOzsbvXr14hpiUHnaLAoYodWMoTONTdyhw1o4MTERmzZtqtc9SE5ODra2tti1a5dQQotuQSGuyMjIICgoCO7u7ti1axePRb+MjAzMzMzg4OAASUlJoeLXdzLw9u1bODs748WLFzA3N4eDg4NQ8Wuhy0FOFCgrK+P06dOwt7fHgAEDkJiYCGtrazx58oSn/1JjEUU7g+TkZAQGBvIYYAwZMgSbN2/G4sWLhY4rivmXToFLN3v27MHgwYMxb9482NrawtPTE2/evIGXlxclC0dFRUVcu3YN3bt3x4cPHzB27FgAQHh4OOl0eTrE0PHjxxEfH4979+6hR48eMDIywoYNGzBkyBBSY62LhIQEdHV1oaury1PvePDgQRw7dkyoekeGpqExDcmbo7AQVdWRoqIi/v77b4wcOVIk96MDpkarmfJjGtuAAQN40tiuXLmC3r17C53GJu7ULcT28PBAcHAw11pYS0tLqLSPoUOHIi4ursGTjVevXsHIyAjZ2dlCjbusrAzu7u6IjIysV1D8ruL569evqKqqQlZWFoqLi9GpUyfIyspCQUFB6LSy+jh9+jQ8PDzQrl07uLq6QktLi9L44kZ6ejoWLVqE1atXw9TUFIaGhujSpQvevn0LIyMjoYTQ5cuXYW9vjwMHDgi07q/bzqA25U8YRowYgeDgYJ66FaCmOfWsWbNw586dRscU1fw7f/58qKiowN7eHkFBQUhMTMTRo0dx/fp1ODg4UFK7RhfDhg1DeHg4lJSUMGfOHKxatQqampo4e/Yszpw588v9quojMTERq1atQkVFBdhsNjw9PbFjxw6EhITAx8en3nYQDSFIDBkaGlIihpSUlCAtLQ0tLS0MHTq0weeSrV/7EarrHRnoR0lJ6acp7bU1m3Q4eJJFSUkJqamppFxAfxeYE61mCt1pbOJO3QnKwcEBf/zxBzZu3EjqNKiiouKnC/qWLVuS6tTepk0bODk5wcHBQSSCQly4cOECtm/fDj8/Px7RY21tjfv378PV1RUGBgak7/P+/Xts2LABycnJmDFjBhwdHSlNfRBXB7kRI0YgPj4eHA4HsrKyCAkJQWhoKLp37/7TQu36EFV/tJEjR2LPnj3Yu3cv93dZWlqK/fv3C70LKqr5d9WqVVi0aBE6duwIU1NT+Pj4gM1mcwVuc0ZSUpIrMuXl5ZGXlwdNTU1oaGhQ4gSmq6uLxMREvH//nnvKxGazMXPmTKFPtHbu3AlpaWno6OhwxdDVq1dx9epVvuc2VgzVnjo8fvwYjx8/rvd5wvTo+hG66h0ZRMePBhcEQWDx4sVwdXWlzMiHbrS1tX/peY0VinVbR/xMkDZHEfojjNBqptCdxibu/HgQO2PGDPTs2bPZvxeiEhTiRFpaGhwcHDB16lS+L5j169fD398fa9asQXBwMNTV1YW+z9mzZ7Fjxw60a9cO/v7+GDNmDNmh8/AzB7nmLLRqC5tr3/9+/frB2dkZJSUlsLW1xcGDBxsd8+HDh9iwYUO9j1+9ehVbtmwhXb/m5OSEuXPnYuzYsXw99fz9/YWKKar5lw6BKyr69++Pf/75B/PmzUPfvn2RkZGB+fPn4927d5TdQ1ZWFp8+fcKlS5cgISGBIUOGkGoMS6cYItOT7Fehu96RQXQIai4tISEBVVVVsWnK6+TkREtGlZubGzeum5vbL5lZNWcYodVMKSws5Kk5EISSklKj8nz/SwQFBfG5iWlpaSE0NJSvi3tjCAgIaDB1r7y8XOjYohIU4saRI0dgYWGB9evX8z2mqKjIrfc4dOgQ/Pz8Gh3//fv32LhxI5KSkjBt2jQ4OTnR0uhQ3BzkMjIyuLVHkZGRGDJkCN/pXn5+Pm7cuCFUfA6Hg9atW9f7+IULF/D69WueFFph6NOnD2JjYxETE4PHjx9DSkoKs2fPxuTJk4Xe2RfV/EuHwBUVixcvxurVqyEtLY1JkybB29sbixcvxqNHj6ChoUE6fmlpKezt7ZGUlMQVFSwWC8bGxtixYwe3tq0xiEIM0YUo6h0ZGBoDm83m6aNHFaamptx/m5mZUR5f1DBCq5kiijQ2cYYOa+EePXr8kkgTdkeVbkEhrjx48ACOjo4NPmfOnDlYunSpUPEnTZqE0tJS9O7dG1VVVXB1da33uWSK+MXNQY7FYmHdunXcfwt6X1q3bg1ra2uh4ouynUG7du0wa9Ys0nFqoXP+pVvgigp9fX2cPn0akpKS6N69O/z9/XHs2DFMmDABq1evJh1/+/btePr0KY4cOQI1NTVUV1cjMzMTLi4u2LNnD/ez+ztw+fJlhIWF4fDhw3ypuDY2Njz1jmTbGTAw/AqiOmXicDgICAiAkZER5OXlsWHDBsTGxkJdXR2enp6QlZUVyTjIwAgtBrGEDmthunc76RYU4sr3799/uqjt2LEjvn79KlT8ui5ir169EirGryBuDnLq6urcHlFKSkpITk5Gly5dKIsvqnYG379/R1hYGPLy8nhsvzkcDu7fv4/4+HhS8amGboErSuqaSIwaNUpgOpSwJCQk4ODBgzx1duPGjYOMjAzWrl37WwktUdU7MjD8KqLy0fP09MT58+eho6OD69ev49y5c1i9ejWuXbsGDw8PsbDGZ4RWM4bONDZxh25rYUF8+vQJERERCA8PFyo9kW5BIa788ccfuHPnToMnkQ2djPyM4OBgYYfWKOiySBcFgpryFhUVkXKUElU7A1dXV0RGRmLw4MHIzs6Gmpoanj9/jo8fP/I1j24MdM2/dAtcUVFUVAQ/Pz88fvwYHA6H7/Efi/0bS12zjbp07dr1t8vkEFW9I0PTIk61SCtXrvxlZ1Ey5i9xcXHYs2cPt2XHqFGjsHTpUowZM0ZsahEZodVMoTuNTdx58uQJ3NzcoKSkhEGDBqF169aYN28eWrdujaNHj0JfX5+ye2VkZCA0NBSXLl3C9+/fMXjwYKHi0C0oxBUTExPs378fGhoaAt2W3r9/j/3792PatGmk7zVjxgxMmzYNkyZNorzRojg7yH358gUeHh6wsLBAv379YG1tjbS0NCgoKODIkSNCFWfT3R+tlitXrmDHjh2YNGkSDAwM4OLigt69e8POzk7o+i9Rzb90CFxR4eDggOzsbGhpadHicmdpaQkXFxfs37+fK0RLS0uxb9++Zm8UQjWiqndkEB2C3PQIgsCff/7J99zm6Kx34MABSEhIQE5OrsHnkXXZLCkp4bqMpqSkwNzcHEDNpvS3b9+EjitKGKHVTBHnol1RQLe1cGlpKSIjIxEWFoYnT54AAHcHRdj0GFEKCnHCwsIC8fHxmDRpEqZNmwY1NTXuyUdmZibOnTsHBQUFSlKpNDQ04Ovrix07dmDChAkwMzODtrY2JTuJ4uwg5+bmhvT0dCxYsACXL19GRkYGPDw8EBsbCw8PD3h7ewsVVxTtDD5//sw1j+nXrx8ePHiAvn37YsmSJVizZg2cnZ0bHVNU8y8dAldUZGRk4PDhw5SmC9YlOTkZ2dnZmDBhAhQUFCAlJYVnz56hrKwMubm5OHfuHPe5V65coWUMzQVR1jsyiAZxd9ObOXMmLl++DKAmTZzNZgvd7Lsh+vTpg+zsbHz8+BGvXr2Cjo4OgJrU4l69elF+PzpghJaYQjaNTdyhy1o4KysLYWFhiI2NxdevX9G/f3/Y2trC29sbjo6O6Nevn9CxRSkoxAlJSUkcP34c+/btw9mzZ3H8+HHuY126dMHcuXOxbNkyShblf/31F+zt7ZGamorIyEisWrUK7du3x9SpUzF16lT88ccfQscWZwe5xMRE+Pj4QFFREX5+ftDW1sbkyZMxcOBAUql9omhn0KlTJ3z8+BE9evSAgoIC8vLyANRYgxcWFpKKXR9Uzb90CVxR0K1bN1rcO2vR0tL67RuJ1yKqekcG0VHXTa+srIzWvyU62LZtGzZv3oybN28iNjYW8+fPR+fOnbmiiyrRv2jRItjb20NCQgIaGhpQUlKCj48PfHx84ObmRsk96IZFiKqijYESBKWxRURENPWwRE5CQgJWr16NTZs2QVdXFxMnToSGhgYePXoEFRUVeHl5NTqmmZkZcnNzoaioCAMDAxgbG6N///4Aaoq+z58/T0poATUpILWC4tOnT9zrXbp0wbRp0ygTFOJKZWUlXr58iU+fPqFTp07o3bs3rbt+X79+RXBwMA4ePIjv379DXV0d8+fPF5i+IYi6DnK1QkuQg9yJEydw584dysdPFaqqqoiLi4OcnBx0dHRgY2MDS0tLvHjxAlOnTkVmZmajY6alpcHKygpTp06FnZ0dunbtyn0sPz8f/v7+iIqKIt3OwNnZGQ8fPsSOHTvw/PlzuLm5Yf/+/YiNjcU///xDqRkG1fOvlpYWfHx8oKamhnXr1qGkpAS+vr7Iy8vD3Llzcfv2bcrGTjVXrlzB4cOHYWdnh969e0NCQoLn8dqeVQzk4XA4mD17Nj5+/NhgvWNQUBDpVFwG0aOmpoY///wTpqamlLRGaAoqKiqQnJyMixcv4sqVK+jTpw+MjY3BZrNJzwUPHz7Eq1evMHbsWMjIyOD69euQlpaGpqYmRaOnF0ZoiQF0pLH9F8jJyYGkpCSUlJRw69YtHDt2DN27d8fq1asF7vr9DCUlJfTt25ebTjZo0CDuY1QJrVpELSgYeCkoKEBUVBSioqKQl5cHdXV1mJqa4t27dwgODsaUKVMaLD6vJTMzE3PmzAEAbpPiH2ndujUWLlxIKk+dbqZNm4YZM2age/fuWLJkCeLi4qCgoABPT0+kpaXh9OnTjY5pbW0NRUVFge0ManFyckJhYSGpdgafP3/GunXroK2tjTlz5mDx4sVISkqClJQU3N3dwWazhY4N0Dv/0iFwRUVCQgIcHR35TEFqG3WTrSuJjIxs8PGpU6eSii9ulJWVwd3dHZGRkfXWOzZk3sLQfImMjER0dDRu3ryJbt26YerUqTA1NW3WqcMNweFwcPbsWezevZub6ksVFRUVePjwIfr27Ss2p4CM0GrGCEpjY7PZ8Pb2pnTRz1DD8+fPERERgcjISBQUFHB3ZIyNjTF16lTmPf8PcP78eZw/fx5paWno1KkTpk6dimnTpvGkOZw5cwbbt29v9AmUODvIJSYmYtWqVaioqACbzYanpyd27NiBkJAQ+Pj41Gsr3RCampoIDAxssPFvdnY2li5dipSUFDLD54EgCOTm5qJLly5gsVg8J2mNQRTzLx0CV1SMHz8egwYNwsyZMwUu8MmK0PrqPVq0aAE5OblmZ9svCr5+/Yqqqira6h0ZmpbCwkJcuHAB0dHRePDgAdTV1WFmZiY2ddsFBQW4dOkS4uLikJGRAXl5eRgZGcHW1lbomG/fvsWGDRuwZs0aDBgwANOnT8eTJ0/QoUMHHD9+nGdDvLnCCK1miijS2MQZOq2Fq6urkZSUhLNnz+Lq1atcK+Fly5bByspKoOUwg3igrKwMPT09TJs2DWPHjuVLdwJqTqmuXr2Kv/76i/T9xMVBDgCKi4vx/v177gI3KysLbdq04To+NRZ1dXVERkY26LL58uVLTJkyhdTJzaBBg5CSksL3Pr969QqTJ08WKmVTVPMvHQJXVKiqqiI6Olpku+5VVVV49uwZtmzZAnNzc0yaNEkk920u1K13VFZW5l6nst6RoXlQUVGB8PBw7NmzB+Xl5c3SdbCW9+/fIz4+HnFxcbhz5w569+4NIyMjGBkZUWKOsXr1arx9+xZ79+7F7du34eLiAn9/f0RERODNmzcICAig4FXQC2OG0Uypdc6aOnUqtLW1uV/yDDXQaS0sISEBXV1d6OrqoqSkBFFRUTh37hwOHjyIY8eOwcTEBFu3bqX0ngyiwdPTExMmTODra/X9+3dcu3YNEydOhLq6ulA1Q+LsIAfUmEd8+vQJly5dgoSEBIYMGULKvpzOdgZnzpxBVFQUgJoTrBUrVvD9TgsKCtC+fftGxwZEN//q6uoiMTGRR+Cy2WzMnDlTaIErKkaPHs1dWIkCSUlJKCoqwsnJCba2tr+V0EpLS4ODgwOmTp3K51i7fv16+Pv7Y82aNaTrHRmalvT0dERHRyMuLg5VVVUwNDTkMc1oThw/fhzx8fG4d+8eevToASMjI2zYsIGniTkV3Lx5E4GBgejVqxc8PT2ho6MDdXV1yMrKNtv35kcYodVMiY+PR0REBIKDg7F7926eNDYG+q2Fa+nYsSMsLS1haWmJBw8eICIiAhcuXGCElphiZ2cn8PTjyZMn+PvvvzFx4kShY4uzg1xpaSns7e2RlJTErTNjsVgwNjbGjh07ICMj0+iYdLYz0NfXR0ZGBvdnOTk5vg2XAQMGCF3HI8r5l2qBKypGjBiBzZs349q1a+jTpw+kpHiXE3TVJEpISKCgoICW2M2VI0eOwMLCQmC9o6KiInbs2AEAOHToEKl6R4amYffu3YiJicG7d+8wcuRIODk5wdDQsFmng+7cuRPS0tLQ0dHB0KFDAdQ0zr569Srfc8nMBRUVFejQoQMIgsCNGzdgb28PoCbz6Mc5p7nCpA42c5g0NsEYGhpi9+7dlO+e1KWqqgrFxcVo164dWrRowb3+9u1byMrKNutJkOH/OH78OLe3Wm2hviCGDRuGsLAwoe8jzg5yTk5OSE9Px6ZNm6Cmpobq6mpkZmbCxcUFBgYGWLduXaNjVlVVYd68eXj8+HGD7QxOnDhB6m+prttjUVER0tPT0blzZwwfPlzomLXQPf/SIXBFxfjx4+t97Nu3b0hNTSUVX5AZRmlpKcLDw9GhQwcEBweTii9ONFW9I4No0NfX5xpgCHPC3xQ09PdfFxaLRarP3dy5czF69Gh07doV27Ztw9WrV9GpUyds2rQJr1+/Fot5gBFaYkTdNLbc3Fy0atXqt01jo9NaODY2FkFBQcjKyuIufgYMGIB58+Zh+vTpWLlyJVRUVGBjY0PqNTCIhsrKSly4cAHV1dVYv3491q9fz7NAZrFYaN26NTQ0NIRONQPE20Fu5MiROHjwIEaOHMlzPTU1FWvXrhV60UxnO4ODBw8iMDAQ4eHhkJeXx507d2BjY4OysjIANc2pDx06RNmGCB3zLx0Ctyl5/PgxQkNDER0djVu3bpGKJai+Q0pKCmpqatiyZUuzT62kElHVOzIwNDeysrKwdOlSFBcXw8bGBvb29tiyZQtiYmLg7+8PFRWVph7iT2GElphSN43t5s2bTT0ckUOXtbCrqytOnDgBTU1NaGpqQlZWFiUlJUhLS0NKSgqGDx+O/Px8xMXFoUOHDlS8FAYRcu7cObDZbFpOCsTZQU5DQwPHjx/nW9w+fvwYc+fOJb1oprqdQVhYGFxdXbFgwQIsWbIEbdu2haGhIUpLS3H8+HG0a9cOq1atwpgxY7B69WpSYxcEVfMvXQJXlHA4HMTFxSE0NBR37twBi8WCvr6+UL0MGQQzbdo0WFpaYsqUKfU+5/z58/D390d0dLQIR8YgLJaWljhw4ADat28PS0vLBp9Lxtzrv0B1dTVKS0u5G6FPnz6FrKysUG18mgLxSHBk4EtjGzx4MAYPHgxra2t8+/btt0tjc3Nzg4aGRr3WwsJw+fJlhIWF4fDhw3xuXzY2NkhISMDKlSuxZMkSRmSJKaampnj9+jXu3bsn0K2STG+e1atXcx3kJk2aBAUFBR4HueaMpaUlXFxcsH//fq49fWlpKfbt2/fTRcCvICUlhT/++IN0nFpOnz6NdevWYe7cuQBq0qaePXsGOzs7rhvgsmXLsHPnTkqEFl3zr6SkpMD0w65du3LTFJsrz58/R2hoKM6dO4eSkhKwWCyYmZlh6dKllBlkfP36FV++fEG7du1+6x5RdNY7MjQNPXv25Gbi9OjRg+mj2QASEhI82SZ//PEHOBwOMjIyKEkRpxvmRKuZw6SxCYYOa+EFCxZgxIgR9RZuenp6IjQ0FEpKSjhx4gRl92UQHeHh4di6dSuqqqr4HqOiySrVFumiYs6cOcjOzoaEhAQUFBQgJSWFZ8+eoaysjG8RQCbfnirU1NS4NV4A4Ofnhz179uDcuXPc9/7ly5cwNjZGdna20Pehe/49ePAgUlJS+ASuo6MjBg0a1OyaXFdVVeHSpUsICwtDWloaJCUlMWbMGLDZbDg5OSEyMpK07X1paSmOHj2KmJgYvHz5kntdXl4eJiYmsLKy+u1El6jqHRmaH58+ffqtN3bv37+PjRs3Ii8vD9XV1XyPN2fr+1oYodWMYdLY6mfJkiVgs9kwMTGhLKaGhgaCg4PrtXJms9lYtmwZXFxckJaWRtl9GUTH+PHjoaenBzs7O7Rt25aWezx79gx5eXli5SB34MCBX35uc1j8q6mpITIyEvLy8gBq5oN79+7xpPE9fPgQlpaWQqc9imL+FTeBO2bMGHz58gUaGhowMDCAgYEB97VT0V+suLgYFhYWePv2LQwMDDBgwAC0b98eX758QU5ODq5cuYLevXvj5MmTv50RFJ31jgyi56+//oKLiwtat25d73Nu3rwJR0dHJCYminBkzYu5c+fi+/fvmDZtGnbs2IF169bhxYsXCAkJgYeHB4yMjJp6iD+FSR1spjBpbA1Dh7Uwh8NpcNK7cOECXr9+jYqKikbHZmgefPjwAVZWVrSILHF2kGsO4qkxDBgwAJmZmZCXl8fnz5+RlpaGCRMm8Dzn4sWLDbq0NYSo5l8tLS1oaWkJ/f+Lmi9fvqBz587o0aMHOnbsSPnJ0v79+1FdXY2YmBiBGxTv3r2DjY0NAgICYGtrS+m9mzsyMjJwcHCAvb09pfWODE3DvXv3YGpqin379mHQoEE8j1VWVmLv3r0ICAjA4MGDm2iEzYMHDx4gMDAQw4YNQ0REBAYMGIA5c+ZATk4O4eHhjNBiEJ6QkBAsWbKE70u+lrt376Jt27Y8vWR+J06dOgVZWVncvXsXd+/e5Xns27dvQi0c5eXlG2ygymKxkJGRwU1XYhA/Bg0ahCdPnqBXr16Ux96+fTuePn2KI0eO8DnI7dmzp1k7yAmy0q4Lmdo1Opg7dy42b96M3Nxc3LlzBxwOB/PnzwdQU68SHR2No0ePYvv27ULFF9X8K24CNyUlBbGxsTh79ixOnTqFNm3aYMKECTA2NqZksZ+YmIhNmzbVewosJycHW1tb7Nq167cTWrVQXe/I0DRERkZi/fr1MDc3h4ODAywsLADUGD389ddfePToEZYsWSJ2cwTVVFdXo2vXrgBq1mh5eXkYMWIEJkyYgMOHDzfx6H4NRmg1Ux4+fIgNGzbU+/jVq1exZcsWuLi4iHBUzYd//vmH71pda2FhYLPZ8PLygo6OjkA3mw8fPsDLy4u7oGMQPxYtWoRt27bh5cuX6Nu3L98p04/ub40hISGBz0Fu3LhxkJGRwdq1a5u10KpvbC1atICcnFyzE1omJibgcDg4deoUJCQksHfvXgwbNgwAcPjwYYSHh8PGxqZBl7aGENX8K24Ct23btpg5cyZmzpyJ/Px8nDlzBtHR0YiKigKLxcLx48dhY2PDTelsLIWFhT89hVRSUsKbN2+Eis/A0Fxo27YtvLy8EBwcDA8PD9y8eROjR4/G7t278b///Q8hISFQVVVt6mE2OfLy8sjIyMCkSZPQt29fbs3tly9fBBpaNUcYodVMYdLYfo36rIWFwdLSEhcvXsTUqVNhbW0NdXV1bsHx7du3ERAQAHl5ea7TGYP4UetAJ+ikg6wZhjg7yD18+JDn56qqKjx79gxbtmyBubl5E42qYaZPn47p06fzXV+yZAlWrVoFWVlZoWOLav4VN4FbF0VFRTg6OmLt2rW4du0azp07h8jISEREREBLSwv+/v6NjllRUfHTOqOWLVs2+78nBoZfZd68eRg0aBAWLFiAK1euQFNTEz4+Pr+d4Ut9zJs3D+vXrwcATJw4EVOmTEHLli2RmZkpFj20AEZoNVuYNLaGocNaWEZGBkFBQXB3d8euXbt4FlEyMjIwMzODg4MDJCUlqXoZDCKGTkMBui3SRYmkpCQUFRXh5OQEW1tbTJo0qamH9MsIsr9uLKKaf8VR4P6IpKQkJkyYgAkTJqCoqAjnz59HREREUw+LgUEsSExMhLOzMzp06IDhw4fj8uXL8PLygr29PaSlpZt6eE3OjBkzuD2zFBUVsXPnThw5cgQ9evTAxo0bm3p4vwTjOthM8ff3R1hYGE6fPl1vGtusWbMwf/58sVvECYsorIWBmt4tVVVVyMrKQnFxMTp16gRZWVkoKCgwrk7/ETgcDl69eoU+ffqAIAhKvtDEzUHuV3j48CHMzc1x7969ph6KSGnq+ffBgwewtbXF5cuXKY/dnFFSUoK1tXWDu/nl5eU4duyYWNg6MzDUB4fDwc6dO3Hy5EmMGzcObm5u6NSpEyIjI7Ft2zYoKChg9+7dv2U9nri54P4MRmg1UzgcDmbPno2PHz82mMYWFBT025yw0G0tDNSkBG3fvh1+fn5QVlbmXre2tsb9+/fh6uoKAwMDUvdgaDoIgsDu3bsRHByMiooKxMfHY+/evWjVqhW2bNlCSnCJ85eDoFqh0tJShIeHo0OHDggODhb9oJqQpp5/f1eBO378+F9+rqA6XQYGccHY2Bhv3ryBg4MD5syZw/PY06dPYWdnh+fPn2P9+vWYMWNGE42yaVBSUoKEhATk5OQafB6LxRKLTUtGaDVjysrK4O7ujsjIyHrT2H6nPF4VFRV07twZurq60NbWxtixY7lmBlQIrbS0NFhZWWHq1Kmws7PjOt0AQH5+Pvz9/REVFYXg4GCoq6uTfj0MoicoKAh+fn6ws7PDtm3bEB0djezsbGzduhWzZs2CnZ1dUw+xSaht8lsXKSkpqKmpYcuWLc2+4TIdiGL+ZQQuA8PviampKTw9PeudW2tPvE6dOvXbnd5u2rQJly9fRqtWrcBms8FmswV+R4kLjNBq5jBpbP9HaWkp11r43r17PNbCK1euJJ06aG1tDUVFRW7hpSCcnJxQWFgIPz8/oe/D0HSw2WysWbMGBgYGUFNTQ1RUFHr37o3Lly9jx44dpHbJxc1BjuHn0D3/MgK3cXz69AkREREIDw/HxYsXm3o4DAxCU1FRwc2gSE9Ph4qKisCMioSEBKENvsSZqqoq3Lx5E7GxsUhISEDnzp25okvcvAkYodWMYdLY6qeutXBhYSFYLBamTZtGylpYU1MTgYGBDdoLZ2dnY+nSpUhJSRF26AxNiIqKCmJiYtCrVy8eofXy5UsYGxtzrWOFob4dt1oHufj4eKFji4KvX7/iy5cvaNeu3W91Ul4fzPzbfMjIyEBoaCguXbqE79+/Y/DgwYzhBsN/Bk1NTfj7+2PIkCFNPZRmSUVFBZKTk3Hx4kVcuXIFffr0gbGxMdhsNnr06NHUw/spjOtgMyUtLQ0ODg6YOnUqn4vW+vXr4e/vjzVr1vy2aWx0WAt///79p7vUHTt2xNevX4UdNkMT07NnT2RnZ/M1LL5+/brQbpW1iKODXGlpKY4ePYqYmBi8fPmSe11eXh4mJiawsrL6LUWXKOdfRuAKprS0FJGRkQgLC8OTJ08A1NTp2tjYYNSoUU08OgYG6ujUqRO+fPnS1MNotkhLS0NPTw96enrgcDg4e/Ysdu/ejT179ohFWiVzotVMYdLYGk9da2FhmhZPmzYNlpaWDTY5PX/+PPz9/YVuiszQtJw9exa7du3C8uXLsWfPHqxfvx4vXrxAcHAw1q1bh9mzZ1N+z+bqIFdcXAwLCwu8ffsWBgYGGDBgANq3b48vX74gJycHV65cQe/evXHy5EmB/cH+y9A9/zICt36ysrIQFhaG2NhYfP36Ff379webzYa3tzclhkcMDM0Nd3d3nDx5Erq6upCXl0eLFi14Hm9u5klNQUFBAS5duoS4uDhkZGRAXl4eRkZGsLW1beqh/RRGaDVTmDQ20RMYGIjAwECcOnVKYC+e9+/fY/bs2Zg2bRpWrFjRBCNkoIKwsDAcOnQI7969A1Czm2hjYwMrKyta7tdcHeS2bNmCtLQ0BAQEoHv37nyPv3v3DjY2NtDX1xeLLzMqoXP+ZQRu/ZiZmSE3NxeKioowMDCAsbEx+vfvD4A6Z1kGhuZGQ26b3759Q2pqqghH03x4//494uPjERcXhzt37qB3794wMjKCkZGRWJljMKmDzRQmjU30WFhYID4+HpMmTcK0adOgpqbGtXTOzMzEuXPnoKCgAGtr66YeKoOQvH//Hubm5jA3N0dRUREIgkDnzp0pid2Qg9ywYcMouQeVJCYmYtOmTQJFFgDIycnB1tYWu3bt+u2EFp3z7/79+1FdXY2YmJgGBW5AQMBv974/ePAAffv2xdSpU6Gtrc0VWQwM/2UEmTA9fvwYoaGhv2X2zPHjxxEfH4979+6hR48eMDIywoYNG8S2ho0RWs2UP/74A3fu3EGfPn3qfU5mZiZ69uwpwlH9t5GUlMTx48exb98+nD17FsePH+c+1qVLF8ydOxfLli377dwe/0vo6upiyJAh0NfXx4QJExo8sWgs69at47tW10GuuVFYWPjT16+kpIQ3b96IaETNBzrnX0bg1k98fDwiIiIQHByM3bt3c4vejY2Nm3poDAy0w+FwEBcXh9DQUNy5cwcsFuu3dBzcuXMnpKWloaOjg6FDhwIArl69iqtXr/I9VxzSKpnUwWYKk8bWtFRWVuLly5f49OkTOnXqhN69e4PFYjX1sBhI8ujRIyQmJiIxMRF3796FnJwcJkyYgAkTJmDkyJGQkJBo6iGKDCUlJaSkpDR4oldYWAgdHR2xKDimEjrn36FDhyIuLq5Bkfbq1SsYGRmRcsEUZ6qrq5GUlISzZ8/i6tWrqKysBAAsW7YMVlZWv11KJcN/m+fPnyM0NBTnzp1DSUkJWCwWzMzMsHTpUtImTeLIrzYuZxoWM5CiqqoK8+bNw+PHjxtMYztx4gRzwsLAIASfP39GUlISLl26hISEBLRr1w43b94kFVOcHOQYoVU/dM6/zPveOEpKShAVFYVz584hNzcXrVq1gomJCbZu3drUQ2NgEJqqqipcunQJYWFhSEtLg6SkJMaMGQM2mw0nJyfSfUEZmg+M0GrGcDgcbhrbp0+fuNe7dOmCadOmMWlsDAxCQBAE7t+/j5s3byItLQ0ZGRkgCALDhw/H0aNHGx1PXB3klJSUYG1t3eDYysvLcezYsd9ywU/X/MsILeF58OABIiIicOHCBdKbIgwMTcmYMWPw5csXaGhowMDAAAYGBujQoQMAxvjlvwYjtMQAJo2NgYEali5dioyMDHz79g1DhgzB6NGjoaWlBTU1NcjIyDQ6njg7yP1qegYguFj7d4Hq+ZcRuL9OVVUViouL0a5dOx7L67dv30JWVpbZaGQQW1RUVNC5c2fo6upCW1sbY8eO5X4HMULrvwVjhiEGSElJ4Y8//mjqYTAwiD137txBaWkpdHV1oaenB01NzQYND36GODvI/c7iqTFQPf/26NEDFy9e/Onz6jPL+B2IjY1FUFAQsrKyULsXPGDAAMybNw/Tp0/H9u3boaKiAhsbmyYeKQODcKSkpCA2NhZnz57FqVOn0KZNG0yYMAHGxsbMRvp/DOZEi4GB4beBIAjk5OQgNTUVKSkpuHPnDrp06QINDQ1oampi8uTJjYqnp6eHTZs2QU9Pr97nJCQkYNeuXYiPjyc7fJHy6dMnREREIDw8/JeEAQMDFbi6uuLEiRPQ1NSEpqYmZGVlUVJSgrS0NKSkpGD48OHIz89HXFwcN9WKgUGcyc/Px5kzZxAdHY3CwkKwWCxMmzYNNjY2kJeXb+rhMZCEEVoMDAy/LR8/foSvry/CwsJQUVHR6FSt/6KDXEZGBkJDQ3Hp0iV8//4dgwcPRkRERFMP67fhdxa4ly9fhr29PQ4cOABdXV2+xxMSErBy5UosWbIEdnZ2TTBCBgb6qKqqwrVr13Du3Dlcu3YN1dXV0NLSgr+/f1MPjYEETOogAwPDbwNBEMjOzkZKSgpSU1Nx9+5ddOjQASYmJg2eStVHRUXFT+tEWrZsybWnbq6UlpYiMjISYWFhePLkCYCaYm0bGxuMGjWqiUf3eyBI4P5uhISEYMmSJQJFFgDcvXsXbdu2RUZGhohHxsBAP5KSktx2I0VFRTh//jyzyfUfgDnRYmBg+E8zaNAgJCcno3Pnzhg1ahS+fPmC/v37Q09PDxMmTMCwYcOEji3uDnJZWVkICwtDbGwsvn79iv79+4PNZsPb25spxhYBjMDlRUNDA8HBwejfv7/Ax9lsNpYtWwYXFxekpaWJeHQMDAwMjYc50WJgYPhPU3cvafXq1dDT02sw1a+xBAQE/NRBrjliZmaG3NxcKCoqYsGCBTA2NuYucL29vZt4dP9tBAlcW1tbeHt7w9HR8bcVuBwOB61bt6738QsXLuD169eoqKgQ4agYGBgYhIcRWgwMDL8NFhYWlMYTZwe5Bw8eoG/fvpg6dSq0tbXrPUVgoBZG4NaPvLw8MjMz690IYbFYyMjIgIKCgmgHxsDAwCAkjNBiYGD4z3Px4kW0bdv2p8+bOnVqo+KKs0V6fHw8IiIiEBwcjN27d6NPnz4wNjaGsbFxUw/tPw0jcOuHzWbDy8sLOjo66NixI9/jHz58gJeXF+bPny/6wTEwMDAIAVOjxcDA8J9GSUnpl57HYrEor6MSBwe56upqJCUl4ezZs7h69SrXuGPZsmWwsrJqdo2WxZ3nz58jIiICkZGRKCgo4BG4U6dO/a1r4zgcDmbPno2PHz/C2toa6urqaN++PUpKSnD79m0EBARAXl4eQUFBkJSUbOrhMjAwMPwURmgxMDD8p/kVwwqqEVeL9JKSEkRFReHcuXPIzc1Fq1atYGJigq1btzb10P5zMAJXMGVlZXB3d0dkZCRPLZaMjAzMzMzg4ODQYE0kAwMDQ3OCEVoMDAz/aeq6DtLJf81B7sGDB4iIiMCFCxdw8+bNph7OfxpG4PLy9etXVFVVISsrC8XFxejUqRNkZWWhoKDw03YKDAwMDM0JRmgxMDD8p6H7ROu/YpFeVVWF4uJitGvXDi1atOBef/v2LWRlZZkFroj43QXuhQsXsH37dvj5+f2/9u49qOo6/+P4E7mMmSk3BzdKMW9cveSKYAgrUysTrSJQUdy0Yyu6NoY5BISXLXDXBTZhCS+whCBjXshM8rLW7GihgyW6eFlnzG3L3VpWUbDdRQjo94fj+e1ZwfJ4Okfh9ZhxxvP9fPl+Xx7/evH9ft9f/P39jdsNBgMnT54kOzubxx57zIYJRUS+v362DiAi8kOaPXu2SXGwpOjoaJ5++mn+9Kc/MWfOHHbt2sWuXbtISUn5Qc73Q9i9ezdxcXEEBAQwbdo0JkyYwKxZs9i+fTsAOTk5VFZW2jhl79XZ2cnFixdpa2sDwNfXl6ysLHbs2MHVq1dtnM666urqSEtLY/r06Xh4eJisZWZmEh4ezosvvkh9fb2NEoqI3Bpd0RIRMZO3tzcPPfQQ0dHRPPLII/j4+BjX/Pz87vgrWtnZ2WzatIng4GCCg4NxcXGhubmZuro6amtrmTRpEufOnWPv3r0MHjzY1nF7ld27d1NRUUFDQ4PxXW9jxowhMTGR2NhYFi1axPjx43n++edtnNR6DAYDI0eOJDMzs8d9MjIyuHjxIiUlJVZMJiJiHhUtEREz3c0T5Pbv38+SJUsoKioiLCzshvX333+fRYsWMX/+fFJTU22QsPdSwe1ecHAwGzduZMyYMT3uc+LECVJSUqitrbViMhER8+g9WiIiZho+fDipqaksXrzYOEGutLSUdevWAdeuWtypE+SqqqqYP39+tyUL4Pjx4wwcOJCjR49aOVnvtn//frZs2cL69etv+O6ff/55k4Lbl0oWQFtb23c+C+js7Exra6uVEomI3B49oyUicpv69etHWFgYhYWFfPjhh2RkZODt7U1xcTGhoaGsWLHC1hFvcObMGWbMmNHj+h//+EdWrlzJ2bNnrZiq91PB7dmIESM4duzYTfepr6/H09PTSolERG6PipaIiAU5OzuTlJTEjh07ePvtt4mJiWHfvn22jnWD9vZ2BgwY0ON6TU0NEyZMMHmXkdw+FdyezZw5k4KCAhobG7tdb2xspKCggIiICCsnExExj24dFBGxkP8dke7r64uvry8Gg4GrV6/eUSPShw8fftOrA3Z2dhw9ehQvLy/rBuvlvk/B/fvf/94nC25CQgL79u3jiSeeICYmhokTJzJo0CCam5upr69nx44deHl5YTAYbB1VROR7UdESEblN3zVBLicn546bIBcZGUlhYSHTpk3D2dn5hvULFy5QWFhIcnKy9cP1Yiq4PbO3t6e8vJw1a9ZQXV1NeXm5cc3d3Z34+HgWLFhwR/3CQkTkZjR1UETkNtytE+Ta29t55plnaGpqwmAw8PDDDxuvHnz88ceUlZUxfPhwKioqsLe3t3XcXqO0tJQtW7awbdu2HgtuXFwcycnJJCUlWT/gHaKjo4Pz58/T0tKCq6srDz74IHZ2draOJSJyS1S0RETMdLePSP/3v//N6tWreeedd0xuVXNyciI6Opq0tDTuueceGybsfVRwRUT6DhUtEREzzZkzhx//+McsWrSo2/W8vDzeeustvL292bRpk5XTfT+tra10dnbS0NDA5cuXcXV1xcXFBS8vL92i9QNRwRUR6RtUtEREzBQUFERlZSWjR4/udj0yMpIFCxbw2muvUVdXZ+V0362mpoacnBxKSkrw9/c3bjcYDJw8eZLs7Gwee+wxGybsvVRwRUR6P413FxEx0908Ir2uro60tDSmT5+Oh4eHyVpmZibh4eG8+OKL1NfX2yhh71VTU0N4eDh//etfmTp1KpGRkQQHB5Obm0tYWBj79++3dUQREbEAFS0RETNdnyDXkzt5gtyGDRtISEhg1apVDBkyxGRt5MiR/OpXv2LmzJmsXbvWRgl7JxVcEZG+Q0VLRMRM10ekNzc3d7t+fUR6VFSUVXN9H6dPnyY2Nvam+zz77LOcPn3aSon6BhVcEZG+Q0VLRMRMSUlJDBo0iKioKCorKzl16hTnz5/nxIkTlJWVMXv2bIYOHUp8fLyto96gra3tO58FcnZ2prW11UqJ+gYVXBGRvkMvLBYRMZOTkxMVFRWsXr2a3NzcHifI3YljukeMGMGxY8cYNmxYj/vc7MW6Yh4VXBGRvkNFS0TkNtx7771kZGSQlpZ2V02QmzlzJgUFBQQFBd3wrBBAY2MjBQUFxMTE2CBd76WCKyLSd6hoiYjchv8ekT516lTj9jt9RHpCQgL79u3jiSeeICYmhokTJxpfnFtfX8+OHTvw8vLCYDDYOmqvooIrItJ36D1aIiJmqqurY+7cuURFRZGammoy3ODcuXOUlpby7rvvUllZycMPP2zDpN1rb29nzZo1VFdX09LSYtzu7u5OTEwMCxYsuGOvyN2tOjs7SUxM5OzZszctuJs2bdJ3LyJyl1PREhExk8FgYOTIkWRmZva4T0ZGBhcvXqSkpMSKyW5NR0cH58+fp6WlBVdXVx588EHs7OxsHavXUsEVEekbVLRERMwUHBzMxo0bGTNmTI/7nDhxgpSUFGpra62YTO4GKrgiIr2bntESETGTJsjJ7XBwcGDEiBG2jiEiIj8QvUdLRMRM1yfI3YwmyImIiPRNKloiIma6PkGusbGx2/XrE+QiIiKsnExERERsTc9oiYiYSRPkREREpCcqWiIit0ET5ERERKQ7KloiIhagCXIiIiLy31S0RERERERELEzDMERERERERCxMRUtERERERMTCVLREREREREQsTEVLREQsJj09nbFjx970T2Ji4m2f529/+xtjx47l7bfftkDqvi09PZ3w8HBbxxAR6XUcbB1ARER6j4ULFxIXF2f8XFxczOnTpykqKjJuGzhwoC2iiYiIWJWKloiIWMywYcMYNmyY8bOrqytOTk5MmDDBdqFERERsQLcOioiI1W3bto3o6GgmTJjAuHHjmDVrFnv27DGud3V18frrrxMeHo6/vz/h4eHk5+fzzTffdHu8b7/9loyMDMaNG8dHH30EwBdffEFKSgpTpkxh/PjxPP300xw4cOCmucLDw1m1ahXJycmMGzeOV155BYDm5maWL1/O1KlTCQgI4KmnnuLw4cMmPzt27Fg2b95Meno6kyZNIjAwkOzsbK5evcrq1asJCgpiypQpvPLKK7S1tRl/rq2tjTfeeIOIiAgCAgL46U9/yoYNG+jq6gJg3bp1+Pv7m7wQG6C8vBw/Pz+ampoA+PLLL1myZAmBgYGMHz+e5ORkTp8+bfIzLS0tZGRkEBgYyOTJk8nNzTWeR0RELEtFS0RErKqqqorly5fz6KOPsn79evLy8nBycmLp0qX84x//AKCkpITNmzfzi1/8grKyMp555hl+//vfs3bt2m6PmZ2dTU1NDUVFRYSEhNDV1cX8+fNpbW3lN7/5DcXFxTg7O7NgwQI+//zz78wXEBBAcXExsbGxtLW1kZyczAcffEBqaipFRUUMHTqUefPm3VC2cnNzcXJyoqioiKioKCorK4mKiuKrr74iLy+PxMREtm/fTmVlJXCtIKakpFBaWsqTTz7JunXriIiIYM2aNaxYsQKAn/3sZ3R0dPCHP/zB5FzvvfceISEhuLm5cenSJeLi4jh16hTLli0jPz+frq4u4uPjOXfuHHCtvM6bN48DBw7w8ssv8+tf/5r6+np279596/+JIiLynXTroIiIWNX58+cxGAwsXLjQuM3T05Po6GiOHj1KZGQkR44cwd/fn5iYGAACAwO55557uO+++244Xn5+Plu2bKGoqIjQ0FAAmpqa+Mtf/sLChQsJCwsDYNy4cRQVFdHe3n7TfPfffz9Lly41ft66dStnzpxh69atjB8/HoDQ0FASExPJy8ujurrauO+oUaN49dVXjZm3bdvGN998Q15eHg4ODoSEhLBv3z7q6+sBOHjwIIcOHeK3v/0tkZGRADzyyCP079+fgoICkpKSGD16NJMnT6ampoYnn3wSuHa1rqGhgddffx2AjRs30tzczObNm/H09DRmfPzxxykoKKCwsJCDBw/S0NBASUmJ8XsKDg7WIAwRkR+IrmiJiIhVpaens3TpUq5cucLx48fZuXMnVVVVAMYSNGXKFGpra3n22WcpLS3l008/JSEhgVmzZpkcq6qqig0bNhAZGclPfvIT43Z3d3dGjRrFsmXLePnll9m1axddXV1kZGQwevTom+bz8fEx+Xz48GGGDBmCn58fHR0ddHR00NnZyfTp0zl58qTJLX0TJ040/t3e3h4XFxf8/PxwcPj/32s6Ozvz9ddfA3DkyBEcHByIiIgwOefMmTON69c/f/zxx1y4cAG4djVr4MCBxpJ0+PBhfHx88PDwMGbs168foaGhHDp0CIBPPvkER0dHpk2bZjzPgAEDjEVUREQsS1e0RETEqr744guWL1/O4cOHcXR05KGHHsLb2xu4disdwLx587j33nuprq4mLy+P3NxcRo8eTVZWFkFBQcZjnTlzhpCQEGpqakhOTsbX1xcAOzs7ysrKWLt2Lfv37+edd97B0dGRRx99lF/+8pcMHjy4x3wDBgww+dzc3MyFCxfw8/Prdv8LFy4Yj9fdRMX/Pd5/a2lpwcXFBXt7e5PtQ4YMATAWsoiICF577TX27NlDUlIS7733HjNmzKB///7GjJ9//nmPGVtbW2lpacHZ2Rk7O7tuzyUiIpaloiUiIlbT1dXFz3/+cxwdHdm+fTs+Pj44ODjw6aefsnPnTuN+/fr1Iz4+nvj4eJqamjhw4ADr1q3jhRdeoLa21rjf4sWLSUpKIjIykqysLLZt22YsLR4eHqxcuZIVK1Zw5swZ9u7dS0lJCS4uLsbnn76P++67Dy8vL/Ly8rpdf+CBB8z8NmDw4MFcvnyZzs5Ok7L1z3/+EwAXFxdjhvDwcPbs2UNQUBBnz55l2bJlJhkDAwNJS0vr9jxOTk64uLh0e67m5maz84uISM9066CIiFjN5cuX+eyzz4iNjSUgIMB4S93BgwcBjBPw4uLiyM7OBsDNzY3o6Gji4+O5cuUK//rXv4zHc3d3p3///ixfvpxTp07x5ptvAnDs2DGmTp1KQ0MDdnZ2+Pj4kJqaypgxY/jyyy9vKXNgYCBfffUVbm5uBAQEGP/U1tZSWlp6w9WoWz12R0cHe/fuNdn+7rvvAjBp0iTjtlmzZnH8+HE2b97M/fffT2BgoMlxPvvsM0aMGGGScefOnWzfvh17e3uCg4Pp6Ojg/fffN/5ce3u7SXEVERHL0RUtERGxGjc3Nzw9PamqqmLo0KEMGjSIDz/8kIqKCuDaLW4AkydPpqysDHd3dyZOnEhjYyNvvvkmgYGBuLq68p///MfkuGFhYURERPC73/2OGTNm4OvrS//+/UlLS+OFF17A3d2dQ4cO8ec//5mkpKRbyhwdHc2mTZuYO3cuKSkp/OhHP+LQoUOUlJSQkJCAo6Oj2d9HaGgoU6ZMISsri8bGRry9vTly5AglJSXMnj2bUaNGGfedNm0azs7ObNmyhXnz5pncAjhnzhx27tzJnDlzeO6553BxcWH37t1s3bqVjIwM4Nrgi5CQELKysmhqasLT05OKigouXbqEm5ub2f8GERHpnoqWiIhYVXFxMTk5OaSnp+Pk5MSoUaNYu3Ytq1at4pNPPiExMZHFixfj5OREdXU1b7zxhvHWuZdeeqnH42ZmZvLRRx+xbNkyysvLKSsrIz8/n5ycHK5cuYKXlxevvvoq0dHRt5R3wIABVFVVkZ+fT25uLl9//TWenp689NJLPPfcc7f1XdjZ2bF+/XoKCwspLy/n0qVLPPDAAyxZsoS5c+ea7Ovg4EBkZCSVlZXGYRnXeXh48NZbb5Gfn8/KlStpa2vDy8uLnJwcYmNjjfsVFRWRl5dHYWEhbW1tPP744zz11FN88MEHt/XvEBGRG9l9e/3JYxEREREREbEIPaMlIiIiIiJiYSpaIiIiIiIiFqaiJSIiIiIiYmEqWiIiIiIiIhamoiUiIiIiImJhKloiIiIiIiIWpqIlIiIiIiJiYSpaIiIiIiIiFqaiJSIiIiIiYmEqWiIiIiIiIhamoiUiIiIiImJhKloiIiIiIiIW9n8ekpVluPb5RwAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1oAAAMCCAYAAABwf7cEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3yT5f7/8dednXTvsssue5UlynQAggriZBy3HgcOEI/n59HjxomgX8Vz3CJ6RMWBgAoqKntvirSU1ZbumZ37/v2RJoCspk2apL2ejwcPsUnu+92Epvnc13V9LklRFAVBEARBEARBEATBb1TBDiAIgiAIgiAIgtDYiEJLEARBEARBEATBz0ShJQiCIAiCIAiC4Gei0BIEQRAEQRAEQfAzUWgJgiAIgiAIgiD4mSi0BEEQBEEQBEEQ/EwUWoIgCIIgCIIgCH4mCi1BEARBEARBEAQ/E4WWIAiCIAiCIAiCn4VUofX2228zderUc96ntLSUGTNm0L9/fwYMGMCTTz6JxWI55T7Lli1j7Nix9OzZk6uuuoq1a9cGMrYgCIIgCIIgCMIpQqbQ+uSTT3jttdfOe7/p06dz6NAhPvjgA+bOncuqVav497//7b193bp1PPzww1x//fUsXryYwYMHc8cdd5CVlRW48IIgCIIgCIIgCCeRFEVRghng+PHjPPHEE6xfv57U1FQSExP5+OOPz3jfrVu3cv3117N06VLat28PwB9//MFtt93GqlWrSElJ4dZbbyUqKuqUou3666+nU6dOPPXUUw3xLQmCIAiCIAiC0MQFfURr9+7daLVavv32W3r16nXO+27atImkpCRvkQUwYMAAJEli8+bNyLLMli1bGDx48CmPGzhwIBs3bgxIfkEQBEEQBEEQhL/SBDvAyJEjGTlyZK3ue/z4cZo1a3bK13Q6HbGxseTl5VFRUYHZbCY1NfWU+yQnJ5Ofn1/njFu3bkVRFLRabZ2PIQiCIAiCIAhC+HM4HEiSRJ8+fc55v6AXWr6wWCzodLrTvq7X67HZbFitVoDT7uO5va4URUFRFOx2e52PIQiCIAiCIAhC0xFWhZbBYDhjsWOz2TCZTOj1eoDT7mOz2TAajXU+r1arRVEUOnToUOdj+IvFYiEnJ4e0tLR6fU/hniFUcoRCBpEj9DKESo5QyBAqOUIhQ6jkCIUMIkfoZQiVHKGQIVRyhEKGUMkRChlOduDAASRJOu/9wqrQSk1NZcWKFad8zW63U1ZWRnJyMrGxsZhMJgoKCk65T0FBASkpKfU6tyRJmEymeh3Dn4xGY9DzhEKGUMkRChlEjtDLECo5QiFDqOQIhQyhkiMUMogcoZchVHKEQoZQyREKGUIlRyhkAGpVZEEINMPwRf/+/cnPz+fQoUPer23YsAGAfv36IUkSffv29X7NY/369WRkZDRoVkEQBEEQBEEQmq6QLrRcLheFhYXetVe9evWib9++PPjgg+zYsYN169bx+OOPc9VVV3lHrG6++Wa+//573n//fbKysnjxxRfZu3cvf/vb34L5rQiCIAiCIAiC0ISEdKGVl5fHhRdeyNKlSwH3MN0bb7xBy5Yt+dvf/sYDDzzA0KFDT9mw+MILL+S5557j008/ZcKECaxbt4758+ef0hJeEARBEARBEAQhkEJqjdbs2bNP+f+WLVuSmZl5ytcSEhKYN2/eOY9z1VVXcdVVV/k7niAIgiAIgiDgcrlwOBwBPYenY7bNZkOlCt7YSCjkaMgMWq0WtVrtl2OFVKElCIIgCIIgCKFKURTy8/MpKysL+LlkWUaj0ZCbmxvUQisUcjR0htjYWFJTU2vd9OJsRKElCIIgCIIgCLXgKbKSk5MxmUz1/iB+Li6XC5vNhl6v99sIS7jmaKgMiqJgNpu9HcybNWtWr+OJQksQBEEQBEEQzsPlcnmLrISEhAY5H7j3kQ12oRXsHA2ZwbNPV0FBAcnJyfU6X0g3wxAEQRAEQRCEUOBZkxUK+zgJgeV5jeu7Dk8UWoIgCIIgCIJQS4GcLiiEBn+9xqLQEgRBEARBEARB8DNRaAmCIAiCIAiCIPiZKLQEQRAEQRAEQRD8TBRagiAIgiAIgiAIfiYKLUEQBEEQBEEQBD8T+2gJgiAIgiAIQh0pioLN7vL7cV2yC6vdBSonapVy1vvpdWqfu+StWrWKuXPnkpWVhclkYtiwYTz66KPs27ePadOm8frrr/Piiy9SVFREr169mDlzJl27dgXc3+8777zDZ599RlFREWlpadx6661cccUV3uOvWLGCt99+mz///BOXy0XHjh158MEHueiiiwCYOnUqaWlp7Nu3j4MHD/L444+zZs0aZFkmOjqar7/+GpVKxZQpU7j88st57LHH2L17N23atOGZZ56hV69eAOzfv59XXnmFLVu2YLFYSElJYfLkydxyyy0AvP7662zevJkLLriABQsWUFpaSq9evXjyySdp3769T89ZXYhCSxAEQRAEQRDqQFEUHnnjD/bmlAQtQ5e0eF6498JaF1slJSXce++9/OMf/2D48OHk5+cza9YsXnzxRW+xNHv2bJ544glSU1N58cUXufPOO1m6dCmxsbHMmTOHJUuW8Pjjj9OuXTs2btzIv//9byorK5k8eTK7du3ivvvu45FHHmHUqFFUVVXxyiuvMGvWLFatWoVOpwNg0aJFvPTSS3Tu3JmkpCTWrFnD0qVLmTx5Ml999RVLlixh7ty5fPfdd8yaNYukpCSeeeYZnnzySb766issFgu33HILQ4YM4bPPPkOtVrNo0SJeeOEFBg8eTJcuXQDYtGkTer2e//znPzgcDmbNmsWTTz7JRx99FJgX5CRi6qAg1IMiy6gr8lFczmBHEQRBEARBOK/jx49jt9tp3rw5LVq0oF+/fsyfP5+pU6d67/PII48wbNgwOnfuzIsvvkh1dTVLly7FbDbzwQcf8M9//pPhw4fTunVrrr76am666SbeffddANRqNf/617+46aabaNWqFV26dGHatGmUlJRQXFzsPUeXLl0YP348nTp1Ii4uDoDY2FgeeeQRWrduzU033QTA2LFjGTlyJB07dmTChAns378fAIvFwrRp03j88cdp3749aWlpTJ8+HYDMzEzveZxOJy+++CLp6en06NGD66+/ni1btgT0OfYQI1qCUA/V678met1iinZ9i+uia4nqMQxJLX6sBEEQBKEpkCSJF+69MHBTB602DAY9apX6rPfzdepgly5dGDduHHfddRdJSUkMGTKE4cOHc8kll7B582YABg4c6L1/bGwsaWlp/Pnnnxw4cACbzcaMGTNQqU6M1zidTux2O1arlS5duhATE8N//vMfsrOzOXToEPv27XN/T64Tz1ObNm1Oy9ayZUvvcU0mEwCtWrU68b3q9TgcDgDi4+O58cYbWbJkCXv27OHw4cPe88iy7H1MYmIiMTEx3v+PioryHiPQxCdCQagHS+Y6AFwVRRR9/yZla74i7sJJRHYfinSON0VBEARBEBoHSZIw6P3/kdrlkkB2YtBpUKv9+5nilVde4Z577uG3335jzZo1PPzww/Tr14+7774bAI3m1O/H5XKhUqlQFPdasddee4127dqddlydTseGDRu49dZbGT58OP369WP8+PFYLBbuueeeU+5rMBhOe7xWqz3taycXdCcrLCzkuuuuIz4+npEjR3LhhRfSo0cPhg0bdlqmYBFTBwWhjhxlx3GV5qFIEpFDrkFlisZZmk/hd29w9O37qdz1G4rs/ytcgiAIgiAIdbV9+3aee+452rVrx0033cR//vMfnnvuOdatW+ed2rdz507v/UtLSzly5Ahdu3alXbt2aDQacnNzadOmjffPqlWrePfdd1GpVLz33nsMHDiQ119/nZtuuokhQ4aQl5cH4C3U/GHJkiWUlZXx6aefcvfdd3PJJZdQXl7u9/PUhxjREoQ6Mh/YCoAztiWRA64g8YKrqNi8nLK1X+MoyaPwm7mU/fEFcRddS0TXC5AkcV1DEARBEITgioyMZOHChWi1Wq699lpsNhtLly4lLS3Nu1bqySef5OmnnyYqKorZs2eTmJjIZZddRmRkJNdffz1z584lMjKSvn37sn79el566SXuvPNOAJo1a8aKFSvYtGkTqamprF+/nrlz5wJgt9v99n2kpqZisVhYvnw5/fr1Izs7m+eff97v56kPUWgJQh1ZsmsKrUR3e1CVzkDs4KuI7nsZ5ZuWUb7uGxzFxyj4eg7a1V8Qd9F1RKQPFAWXIAiCIAhB0759e15//XXeeOMNFi5ciEqlYtCgQfz3v//1jjxdd911zJo1i7KyMgYOHMjbb7+N0WgE4NFHHyUuLo65c+dSUFBAs2bNmD59OrfddhsA06dPp6ioiLvuuguADh068Nxzz/Hwww+zc+dOv7VVHz16NLt372b27NlUVVXRokULrrnmGlauXMnOnTu54YYb/HKe+hCFliDUgeJ0YMnZBYAj6dQ3DJXeSNyQicRkjKZ8w/eUb/gOR+ERCr56GV1yG+Iuug5T5wE+73khCIIgCILgDyNGjGDEiBGnfd1TaF1yySXcfPPNgHt9ltVq9d5Ho9Fw7733cu+9957x2HFxcbz++uunff3SSy/1/v3jjz8+7fbZs2ef9jVP90BPE40JEyYwadIkwL02bubMmcycOfOUx3hyA9x3333cd999p9w+ceJEJk6ceMbs/iYKLUGoA8uRPSgOKypTDK6o5DPeR6U3EXfRNUT3H0v5hiWUb1iCveAQx798EV1KW+KGXoepY4YouARBEARBEBohMYdJEOrAkrUNAH1aTzhPoaQ2RBA/9Dpa3/MmsUMmIekM2I8f5Pii2eS+/wjmA5tDZtGmIAiCIAiC4B9iREsQ6sCc5d7oTpfWC+Tz3LmG2hhF/PAbiBlwOeXrv6V84zJseVnk/+859M07Ejfseoxte4kRLkEQBEEQgmLgwIGnbPYr1I8Y0RIEHznLC3EUHQVJhb5NN58frzZFEz9iCq3veZOYQVcgaXTYcv8k/9Onyf3oMSwHd4gRLkEQBEEQhDAnCi1B8JE5y91tUN+iIypDZJ2Po46IIWHU32h1z5vEDBjnLriO7iNv4ZPkLXgcy6Hd/oosCIIgCIIgNDBRaAmCjzyFlql9X78cTxMZR8IlN9Pq7v8jOmMsqDVYD+8hb8Hj5C54AuuRvX45jyAIgiAIgtBwRKEl+ExRFHCGxkZwDU1xObDk7ADA1L6PX4+tiYon8bJbaX33m0T3Gw0qDdZDu8j96DHyFj6F9aiYMy0IgiAIghAuRKEl+Kzq98+IXfEK9qP7gh2lwVmP7EOxW1FHxKBLbRuQc2iiE0gcfTut736DqD6XgEqN5eB2cj/8J3mfPoP12J8BOa8gCIIgCILgP6LQEnzirCimeusPSCiYd/4a7DgNzpy9DQBju95IUmB/fDQxSSSNvYtWf3+dqF6jQFJhyd5K7gf/IP9/z2HLyw7o+QVBEARBEIS6E+3dBZ+Urf8WZPfu3LaDW1FcDiS1NsipGo6lpq27v6cNnos2NoWkcXcTO2QipX8somrnb5gPbMZ8YDOmTv2Ju+g6iE5psDyCIAiCIAjC+YlCS6g1V3U5lVt/AkCR1GAzY8nZ1aBFRzA5K4qxFxwGJIxtezf4+bVxqSSPv4/YC66m7I9FVO36HfP+jZj3b0TfoT+q1N5AlwbPJQiCIAiCIJxOTB0Uaq1841IUhw1NclvsLXsCUJ25PsipGo63rXvzDqhNUUHLoUtoTvKV99PyzteI6DoEkLAd2Ej02g9wFB4KWi5BEARBEAThBFFoCbUi28xUbFoKQOSA8dhTOgNg3r8RpWYqYWNnyfZvW/f60iW2JGXCQ7S841W0zTshuRyUfvsaLnNFsKMJgiAIgiA0eaLQEmqlYvMPyDYz2oQW6Dv0wxnfBklvwlVdhu3Y/mDHCzjF5cR80N3W3RhiUyV1Sa2Ju/IhXKY45Ioijn/1CorLGexYgiAIgtAkKIqCbLcG5I/isJ3/Poric+bOnTvzySefcO2119KjRw/Gjx/PypUrT7nPL7/8wsSJE+nTpw9XXHEF8+bNw24/sb3P/v37ufPOO+nfvz/du3dn1KhRvPfee97bX3/9daZMmcKDDz5I3759efrpp3G5XLz00ksMGzaM7t27M3r0aD799NNTzvv1119zxRVX0LNnT0aOHMmbb76Jy+W+qH/s2DE6d+7MDz/8wDXXXEP37t0ZOXIk//vf/3x+DhqCWKMlnJfssFG+4TsAYi+Y6O62p1Kjb9cH697VVO9bh6FV414bZD2WiWIzozJGoW/WLthxTqMyRFDVZxKxGz7GemgXxSs/JPHSW4MdSxAEQRAaNUVRyP3o/2EL4l6X+pbpNJ/2DJIk+fS4l19+mZkzZzJ79my++uor7r33Xj755BP69u3Lb7/9xgMPPMCjjz7KwIEDycrK4qWXXiInJ4e5c+disVi45ZZbGDJkCJ999hlqtZpFixbxwgsvMHjwYLp0cX8u3LhxI9OmTeObb77B5XKxcOFCli9fzpw5c0hJSeGXX37h3//+Nx07diQjI4MPPviAV155hX/84x8MGTKE7du389RTT1FSUsKDDz7ozf7888/zr3/9i06dOvH+++/z73//mwsuuIBWrVr59bmtLzGiJZxX5fafcVWXo4lJIrLbhd6vGzpkAO51WnW5mhJOLDXrs0zteiOp1EFOc2ZyVBIxo+8EoGLjUiq3/xzkRIIgCILQFPhW4ISKiRMnMnnyZNq1a8fMmTPp0aMHCxYsAGD+/Plce+21XH/99bRu3ZrBgwfzxBNPsHz5co4ePYrFYmHatGk8/vjjtG/fnrS0NKZPnw5AZuapRef06dNp1aoVaWlpHD58GJPJRMuWLWnRogVTpkzh/fffp23btiiKwn//+1+mTJnC5MmTSUtL48orr2T69Ol8+umnVFZWeo950003MWrUKFq1asWDDz6ILMts37694Z68WhIjWsI5KS4n5Wu/BiBm0FVIag3gHjbWt+mBpNXjLC/Enn8wJEd6/MWctQ0IvWmDf2XokEHsRddS9vvnFC57G21iSwwtOgU7liAIgiA0SpIk0XzaMygOm9+P7XK5sNls6PV61OqzX+SVtHqfR7MABg4ceMr/9+nTh9WrVwOwZ88eduzYwRdffHHaxfSsrCyGDRvGjTfeyJIlS9izZw+HDx9m3759AMiy7L1vQkICUVEnGohNnjyZFStWMGzYMLp06cKQIUO4/PLLSUhIoLi4mKKiIvr163fK+QYMGIDT6SQnJ4fmzZsD0L59e+/tnuM7HA6fn4NAE4WWcE5Vu3/HWVGEOiKWqF4jTrlN0uoxtuuNOXM91ZnrGm2h5awsxX78ICBhatc72HHOK+6ia7Afz8G8fwPHv3iJFre8iCYqLtixBEEQBKFRkiQJSWfw+3EVlwtJBpXOgOochVZdaTSnlgEulwuVyj3ZTZZlbrvtNiZMmHBawZeUlERhYSHXXXcd8fHxjBw5kgsvvJAePXowbNiwU45pMJz6vKSlpfHjjz+yYcMGVq9eza+//sp///tfnn/+eS666KIz5vQUbifn1el0p90vFGdXiUIrzBzKr2TvEQtVSgE6vT6wJ1NkEn5ZhAYoaz2M3H0lANhtNo4cdWeIjkgnhvUU7VhNZsKIcx/Pz07OEcjnwnB4LTGAI7YVG7LNgPmUDPl5Vjp3Dp0fbklSkXzFdI598A8cRUc5/uWLNJ/yFJKm6WwsLQiCIAjCue3cuZORI0d6/3/r1q1069YNgI4dO3Lw4EHatGmDy+XCarWyY8cOFixYwL///W+WLFlCWVkZP/zwA1qt+/OFZ8rguQqejz76iISEBC6//HKGDBnCrFmzuPnmm1m6dCkTJkwgMTGRzZs3c/HFF3sfs2nTJrRaLS1btjylGUc4EIVWGCkoNTPr/9bV/F9xwM/XS3uIW6KOY5Z1PL3aiG31hr/coxijZOeZWBWaynze+egnCuSYgOc6XWCfi79F/EFfPazMi2XZB399DtwKLZnce21otH0HUOmNpF7zD469/wi2Y/spWv5fEi//e52mFgiCIAiC0Ph8+OGHtGvXju7du/P555+TmZnJs88+C8Dtt9/OAw88wBtvvMGYMWM4dOgQTz/9NK1atSIpKYnU1FQsFgvLly+nX79+ZGdn8/zzzwOcsxgqKSnh//7v/zAYDKSnp5Odnc3evXuZNm0aALfeeitz5syhVatWDBkyhB07dvDGG29wzTXXEBUVRXFx4D//+pMotMJIfLSBi3o1I+dYMUaTEXUgmzIoCuOrl4MMewx9aJeW6r3JJbuwmC3eDLnVrWjtOsTFKYVs1bcNXKa/+GuOQJAUmW6V+QBYEtLpook/NYPLxf4j5fyw/gj9uqQyuEfzgOSoC218M5KvepD8/z1H5faV6FLbEpMxJtixAk62WZCsVcGOIQiCIAgh7frrr+eDDz5g//79pKen8+6775Keng7A6NGjmTNnDm+//Tbz588nJiaGESNGMGvWLO/tu3fvZvbs2VRVVdGiRQuuueYaVq5cyc6dO7nhhhvOeM57770Xh8PBM888Q2FhIUlJSdxwww3ceae7mdctt9yCTqfjww8/5LnnniM1NZXbb7+dm266KSTXYJ2PKLTCiEat4t5J3dm7dy9dunTBZDIF7FzmrK3kf1aApNVz9T13ca0p+sRtZvMpGSq2WilaOp+h8ce54ZYzz68NSMa/5AgE65F95H5kQ2WI5KH7J53WcdBsNjN34VrW7K1i3v+20b5lLMlxgXtdfGVq34f4EZMp+fljin96H11Sa4xtugU7VsDY8rMp+vRpYmwW5PR0CODPiCAIgiCEsw4dOngLpzMZM2YMY8aM8U4dNBgM3qYckiQxc+ZMZs6cecpjbr75Zu/f77vvPu67775TbtdoNDz88MM8/PDDZz3vlClTmDJlyilfc7lcOBwOWrRocVpXQzi902GoEO3dhTMqW/MVANF9LkF9UpF1JhGdBgAStrwsnOWFDZCu4ZiztgBgbNfrrG3dR/aMoX2LaKosDl75ZDMul3zG+wVLzKArieh2Icgujn/1Mo7ygmBHCgjrkX3kLXgC2VyB5HLgKDoa7EiCIAiCIDRhotASTmM9shfr4T2g0hAz8Irz3l8dEYOhtXtjuurM9YGO16A8bd1N52jrrlFLTL+2B0a9hj0HS/jsp/0NlK52JEki6fK70aW0RTZXcHzRi8gBaEMbTOasreQtfBLZdqJRiauRFpSCIDQ+ss2CZLcEO4YgCH4mCi3hNKWr3aNZUT2Ho4lOqNVjIjq792JoTIWWs6oMe34WAMbztHVPjTdxz6ReAHy+IpOdB4oCHc8nKq2e1GseQWWKxn78IIVL/i8k26DWRdXeteR/PhvFacfYrg+Gru5NtUWhJQhCOHCUFVD04Syif5+PbKk8/wMEwQ8yMzOZOHFisGM0enUqtA4dOsRnn33Gyy+/zBNPPMFrr73GokWLyMvL83c+oYHZ8g9iydoCkorYwVfV+nGmzgMA9/QtV3V5gNI1LEv2NgB0KW3RRJ5/H6phfVtycf/WyAq8/MlmyqtCa9RIE5NEytUzQaWmes9qytd9E+xI9VaxbSUFi18F2UlElwtIvfYRtImtAHCVHQ9yOkEQhHOTrdXkf/4ccnUZKocF865VwY4kCIIf+dQMY8WKFbz99tvs2rULRVGIjo7GaDRSUVGBxWJBkiR69uzJnXfeeUpffiF8eNZmRXS9AG18s1o/ThuTjC61Pfb8LKr3byS6z8Xnf1CIM2dvBc49bfCv7pzQg705JRwrdDfHeOyWASHVUt3YuhsJl9xC8Q//peTnBeiS2/j0/YWSsvXfUbLiAwCiel9M4pg7kFRq1DHJALga2XpBQRAaF0V2cXzxKzgKj4BKDbIL8/YVKBddfdY1wUJoaCwzQoSz89drXKsRrWPHjjF16lQee+wxunTpwnvvvcemTZvYsGEDq1atYuvWrWzYsIH/+7//o0uXLsyaNYsbbriBI0eO+CWk0DDsxblU710LQOzgCT4/PiLdM31w3XnuGfoU2eUd0TJ1qP3+WAa9hllTM9CoVWzYk893f2QHKGHdRfe7jKjeFwMKBYtfxVGSG+xIPlEUhZJVn3mLrJiBV5A49i7vBxNPoeUUUwcFQQhRiqJQ9MM7WLK3I2n1xE/6J7LWiFxZjPnPTcGOJ5yFZ2Nes9l8nnsK4c7zGnte87qq1YjWlClTuPnmm3n33XfR6XRnvE90dDQjR45k5MiRPPLII3z66adMnTqVX3/9tV4BhYZTvnYxoGDqmIE+Jc3nx0d0HkjprwuxHNyJbK1GZYjwe8aGYsvLQrZUodKb0Lfo5NNj27WI4dYruvH24p28/90eurVNoH3L2MAErQNJkki87DbshUewHcskf9ELtLjpeVT60G+FrigyxT+9T8XGpQDEDbuB2CFXnzJq6Cm0FGsVLms16jD+dygIQuNUsfF7Krf8CEgkX3k/UotO2Fv2xnBwLeWblnnXPQuhRa1WExsbS0GB+0KeyWQK6KwVl8uFzWbznjtYQiFHQ2VQFAWz2UxBQQGxsbH1PletCq3FixcTGxtb64MajUZuueUWJkzwfVRECA5nRRGVO91zw2OHXF2nY+gSW6JNbImj6CjmA1uI7N5we2r5m/lATVv3tmdv634ulw9py7b9hazfnc+LH2/itYeGY9SHzrZ1kkZLytUPc+y9WTiKjlLwzTxSrpmFJIVufxxFdlH4/ZtU7fgVgIRLbyWm/9jT7qfSGZB1Eajs1ThLj6Nu1q6BkwqCIJxd9f6NFP/0AQDxo6YR0XkgZrMZW+u+GHLWYc3Zib3wCLqkVsENKpxRamoqgLfYCiRZlnE6nWg0GlSq4P1+DoUcDZ0hNjbW+1rXR60++Z2ryLLZbOh0ujNW9HFx528gIISGsnXfguzC0KY7Bh9HcE4W0WkAZUVHqc5cF9aFlmfaoLGO65ckSWL6dX24/5VfyC2qZv5XO3jwhtpPQWwImqg4UibNIu/jf2H+cyOlv31O/LDrgx3rjBSng+Nfz8GcuR4kFUnj7yGqx/Cz3l82xaKyV+Moy0cvCi1BEEKELT+bgq9fAxSi+lxCzMDx3ttkYwz6dn2xZW2mYtMyEsfcEbScwtlJkkSzZs1ITk7G4XAE9FwWi4Xs7Gxat26N0WgM6LlCPUdDZtBqtX4bNavTJfbs7GzmzZvHmjVrqKqqYtGiRXzxxRe0a9eOqVOn+iWY0HBc1eVUbv0JgNgh9Wv1GZE+iLI1X2HO2orssKHS6v0RsUG5qsux5R4AfGuE8VfRETpmTO7H/3trNT9vOkKfTkkM7xdaVygNLTqSOPZOCr97g7I/FqFPSSMifVCwY51Ctls4/sWLWA7uALWGlAkPnXdajcsUh6bsGM5S0XlQEITQ4KwoJv/z51EcVoxte5J42W2nXaQ29b4EW9ZmKneuIn7E5LCegt/YqdXqgE+jk2UZAL1ej8FgCOi5Qj1HKGSoC5/H3vbu3cukSZPYvXs348eP93blUKvVPPfccyxevNjvIYXAKt/4PYrTjr5Ze4xpPet1LF1qOzTRiSgOG5bs7X5K2LDMB7cDCrrkNDRR8fU6Vvf2iVx3SWcA3vxyO7lFVX5I6F9RPUcQ3f9yAAq+fR17weEgJzrBZakib+FTWA7uQNIaaHbd/6vV2gXZGAuAozQ/wAkFQRDOT7ZbyP/8eVyVJWgTW5I8cSaS+vRr3bpWXdEmtkRxWKnc8UsQkgqC4E8+F1ovvPAC3bt3Z9myZTz66KPeQuuxxx5j0qRJfPTRR34PKQSObK2mYtMyAGIvuLreizolScJUMyISrpsXW7Lcbd2N7Xv75XjXXdyJbu0SsNhcvPTxJhxO2S/H9aeEi/+GIa0HisNK/qLZuEJg00xnVSl5Cx7Hdmw/KkMEzSY/gbFt7S4EyCb3tGWH2EtLEIQgU2QXBV/PxX78ICpTNKnX/fOsTXokSSImYwwA5ZuWoSih9/tCEITa87nQ2rZtGzfddBMajea0D+Vjx44lJyfHX9mEBlCx5QdkmxltYktMnfv75ZieEQfznxtRXE6/HLOhKIqM2dPWvb1/1lSp1SpmTu5HlEnLgaPlfLR0j1+O60+SSk3KhIfQxCTjLDtOweI5KLIraHkc5QXkfvQY9oJDqCNiaT71aZ/WDrpMsQBi6qAgCEFX8vMCzH9uRFJrSb3mEbSxKee8f2SPYUh6E87S/LCdGSIIgpvPhZZer8dqtZ7xtrKysrO2fz8XWZaZN28eF110Eb179+b2228/5x5cOTk53HHHHWRkZDB06FDmzZuH03niA73dbufVV19l5MiR9O3bl7vuuotDhw75nKuxkx02ytZ/B0DsBRP81nHO0LIz6ogYZGs1lkO7/XLMhmLLy0Y2VyDpjBhadvbbcRNjjUy/zr3e6+tVWWzaG3oFgNoUTco1jyBp9VgObqfk5wVByWEvOkruh4/hLM1HE5NE82nPoEtu49MxZKN7RMtZUYTiCuxiZUEQhLOp2PIj5eu/BSBp/D0YWqaf9zEqnZGoniMAKK/ZykIQhPDk8yfrIUOGMG/ePPLzT6x9kCSJ6upq3nvvPS644AKfQ7z55pssXLiQp59+ms8++wxZlrntttuw2+2n3be8vJzJkydjsVj48MMPefXVV1m2bBmPP/649z7PPPMMn376KTNnzmTRokWkpKRw4403UlJS4nO2xqxy20pkcwWamGQiu17ot+NKKjWmju7RsXDbvNiS5Wnr3vOM8+frY1D3Zowb0haAOZ9uobjc4tfj+4M+JY2k8fcCUL7+Wyp3/dag57flZZP78b9wVRajTWhB82nPoo1v5vNxFH0EkkYHioyzvDAASQVBEM7NnL2douX/BSBu6PVEdqt9J96YjNGAeyq7oyQvIPkEQQg8nwuthx9+GLPZzOjRo5k8eTKSJDF79mxGjx5NXl4eDz30kE/Hs9vtvPfee0yfPp3hw4eTnp7OnDlzyM/P58cffzzt/osXL8ZsNjN37ly6detGRkYGzzzzDF9++SVHjx6lvLyczz//nIceeoixY8fSvn17nnjiCSIjI1m4cKGv326jpbgclK37BoDYwVf6vajwdK4zZ24Iqznm5qxtQP26DZ7LzeO70bZ5NBXVdl5duAWXrATkPPUR2eUCYi9wd58s+v4tbHlZDXJey+E95H7yBLK5Al1qO5pPfRpNdELdDiZJ3o2LHWL6YFA5K4ooWvAY+oPhddFFEOrDXniE41+9DIpMZPehxF44yafHa+Ob12wvolCxeXlgQgqCEHA+f7pu1qwZ33zzDR988AHr1q2jdevWmM1mxo0bx80330xycrJPx9u3bx/V1dUMHjzY+7Xo6Gi6du3Kxo0bGTdu3Cn3P3ToEO3atSM+/kQ3uK5duwKwadMm2rVrh6IoZGRkeG9XqVSkp6ezYcMGX79dL89O0cFmsVhO+W9dmXf/hquiCJUpBnXHQT59b7XJoCS3R9IZcVWXUZ61A13zuu/NVZ8cvpCtVdhy9wMgNe9Sq+ekLhmmT+rOP95ax44DRXz2wx4mDGtbt8D1zHEu+gFXos/LxnZwG3mfzybhxqdQR8QELIctZwel380Fpx1ti87EXvkQNkkLdfiZ8547KgGKj2IuOALN/DcN1Ncc/npNwjVD+arPcRYeQm+uavLPRajkCIUMjTmHy1xOyWfPoNjMaJt3ImLETec99pkyGHqMxJK1lYrtP6PvfyUqXeBbWofCaxIKGUIlRyhkCJUcoZDhZIqi1KqBXJ2GMeLi4vjb3/7Ggw8+CLin8xUWFvpcZAHeKYjNmp06PSg5OfmU6Yknf72goACXy+Xdv+DYsWMAFBcXewu23NxcOnbs6H3csWPHzrq2rDYcDgd79+6t8+P9rV5NRxSZ6NVfogaqWvWj+M+6jVicL4MpoR36vN3krv8RS3rgGiv4qwGLNm83kYqCKzKJ/UePA7UfCfE1w+i+MXyzvpT/rTyASSqndZJ/9hvzazOadiOJLjgMVSXkLppNVf/JoKrdniG+5NDm7yVi+zdIiowjqT2lXa+gILv+ayqrFD0GoCB7HxZdi3ofr65CoUFQsDJI9mpidq1CAiS7uUk/F38VCjlCIQM0shwuB1EbFqKpKMJliqMsfSwFfx6oWwZFR7QpDrW5lIO/fIm9dcNteh8Kr0koZIDQyBEKGSA0coRCBo/a9KXwudCqrKzkwQcf5NixYyxb5m4Lvn37du644w4uvfRSXnzxRZ82EvNUpn8Nq9frKS8vP+3+Y8aM4c033+T555/noYcewmw288wzz6DRaHA4HKSkpDBo0CBeeuklWrVqRatWrfj000/Zu3cvLVu29PXb9dJqtXTo0KHOj/cXi8VCTk4OaWlpdd4Z27p/A2XVJUh6E2kXX49K59txapvBqqmibMluIkqyaZN+T71bx9c1R22VHf4NKxDVuT8tunQJaIb0dIUi8y5W78znu42VvHB3NyKM2jom9/9z4eFsmUrxp0+gLT1K8/yNxIy62a85zLt+pWL716AoGDoNJGX0XfWexurJENuqHdZDG4hRO0mr5evpT4F6TcIpQ9W6xVTJ7kZFKoeFNq1aYoqMavAcEPznIpRyhEKGxphDUWTKl76JtfwYkt5EyjWP0qKWa0zPlqHaOpbKVZ8Qc3wnCZfe6Pffo7XN0ZBCIUOo5AiFDKGSIxQynOzAgdpdQPH5E83LL7/M3r17+X//7/95vzZo0CBef/11nnzySV5//XUefvjhWh/PU5TZ7fZTCjSbzXbGJzItLY25c+fy+OOP88knn2Aymbjvvvs4cOAAUVHuX+Avvvgi//jHPxg7dixqtZqhQ4dy9dVXs3t33TvgSZKEyWSq8+P9zWg01imPoiiUbFoCQEz/y4mMreMamFpkMHQZSPlyHa6KQjSVBehT6z9Fri45akNRZAoP7QIgJn0ARh+PV5cM06/vS1bur+QXm3lnSSb/mNa/3r9E/fFcnMLUHs1VD5L/+fNYdvxMRIuORPe91C85ytZ9S8XKDwGI6nMJiaNvR6rliFltGBNbYgWUysKg/uz6/TUJkwyyw0bB9hWnfE2Ps0k+F6GaIxQyNKYcJas+xbp/PajUpE6ahbFl+3pn0GdcRtWaL3AWH0NVmI0xrUed89UnRzCEQoZQyREKGUIlRyhkAGr9ec3nZhg///wzjzzyCGPHjvV+TafTcckll/DQQw+xdKlvrUg9UwYLCgpO+XpBQQEpKWfea2LkyJH88ccfrFq1irVr13LttddSVFREq1atAEhJSeH9999n06ZNrFmzhrfeeovy8nJat27tU7bGyJK9Dfvxg0haAzH9Lw/ouVQ6A8Z2vYHQ7z5oz8/BVV2GpDNgaHX+9rv+YDJoeXhKBmqVxJodeSxfF5pbEJg69iNu+A0AFP3wLtYj++p1PEVRKPn1U0pqiqyYQVeSOOZOvxZZAOpYTzOMAu/G6kLDqdq5qqaraRIqo/simGw+fZaCIDQGlTt+peyPLwBIGnuX3woitSGCyB7DAPcGxoIghBefC62qqipiYs68KD4pKcnnFurp6elERkayfv1679cqKirYs2cP/fufvoHupk2bmDp1Kk6nk+TkZHQ6HT/++CNGo5G+ffuiKAp33HEHq1atIjIykpiYGKqqqlizZg1Dhgzx7ZtthMpWfwlAdN9LUJsCP4UnIt29eXF15vrz3DO4zJ627mk9kNR1n8Lnq06t45g21t3M5Z2vd3Ior6LBzu2L2AsmEtFlMMhOjn/5Es6K4jodR1Fkin98l7LV7g8kccMnEz9yakCmw6ijEgEJxWHFVS0+4DckRXZ59w6KGTAOVaR7XzPZHJr/vgWhPiyHd1P4/VuAe0/KqF4j/Xr8mH5jADDv3yi2qxCEMONzoZWens6XX355xtu+/vprOnf2rbuXTqdjypQpvPzyy6xcuZJ9+/bx4IMPkpqayqWXXorL5aKwsNDbyKJdu3ZkZmbywgsvcOTIEVasWMEzzzzDnXfeSWRkJJIkERsby8svv8yuXbvYv38/d999NykpKVxxxRW+fruNiuXwHqxH9oJaQ8yA8Q1yTlOHDFCpcRQewV6c2yDnrAtL9jYATO0C09b9XK4a1p6+nZOxO2VeXLAJq915/gc1MEmSSBp3L7rkNriqyzj+xQvIDptPx1BkF4XfvUFFzVXZhMtuJ27IxICtOZA0Wm97eGfZ6Y11hMAx/7kJR0keKkMEUb1HoTK5L86JQktobBwleRz/4kWQnUSkDyJu+I1+P4cuuTWGNt1BkanY8oPfjy8IQuD4XGjddddd/PTTT0ycOJG33nqLzz//nPnz53PdddexbNky7rnnHp9DTJ8+nUmTJvHYY49xww03oFareffdd9FqteTl5XHhhRd6pyTGx8czf/58tm/fzrhx45g9ezb33nsvd911l/d4//rXv+jevTu33norU6ZMISkpiQ8++KBW3UEas7I1XwEQ1XNE3fcn8pHaGIkxrTsA5hAd1XJZq7EezQTA2KHhCy2VSuLBG/oSF6XncH4l735b97WEgaTSGUi55hFUxihseVkULXu71lPyZKed41++TNXOVSCpSLpiundDzkDSxKUCYi+thla2zj2aFd33MlQ6IypjNCCmDgqNi8tSSf7/nkO2VKFv1oGkK6YjST5/rKqVmAz3co2KrSuQnfaAnEMQBP/zuRnGsGHDePPNN3n99deZN2+et498ly5dePPNNxk2bJjPIdRqNQ8//PAZm2i0bNmSzMzMU77Wt29fPv/887MeLyoqiueff97nHI2ZLT8bS9ZWkFTEDr6qQc8d0XkQluztVGeuJ/aCCQ167tqwHNwOiow2sSXaGN+3KPCH2Cg9D93Yl8f/s5bla3Po1TGRC3sFryX52WhjU0iZOIO8hU9RtXMVupS2xA489+iobLdwfNELWHJ2Iqm1JE+cQUSn06cFByqv9dAunKLQajDWo5nYju4DlYbomg+Hqgh3oeUShZbQSCguB8e/fAlHSS6a6ERSrv0HKq1/tuk4E1OnDNTRibgqiqje/YffpycKghAYdbr0MmLECL766iu2bdvGqlWr2Lx5M1999RXDhw/3czzBXzyjWZFdh6CtucrfUEyd+gMSttw/cVYUNei5a8OStRUAU03jjmDp3SmZq0e493574/NtHC8J/gbZZ2JM60HCxX8DoGTlR5gPbj/rfV2WSvIWPuUusrQGUq//fw1WZMFJI1plotBqKGXrvgEgsvtQNFHutVlqz9TBajF1UAh/iqJQuPQ/WA/tRtIZSb3un2hq1iEGiqRSE9PvMsDdFEM0+BGE8FDnMe7y8nJKS0txuVyUlZWRm5vr/SOEFnvxMar3urv+BWNESRMZ5+3kV525ocHPfy6KomDO2gaAsX3DbQZ5NpNHp9O5TRzVVicvL9iE0yUHO9IZRfe/nMiew0GRKfjqVRylp6+BclaVkrfgcWzH9qMyRNJs8hMN1prYQxvn7lx6pnyC/zlK8jDX/IzHDjox0nlijZYY0RLCX/naxVTt+BkkFSkTHkKX3KZBzhvV+2IktRZ7fja2Y/sb5JyCINSPz4XWoUOHuP766xk0aBAjRoxg1KhRp/0RQkvZmq8BBVPH/g32C+GvTJ0HAKHX5t1ecAhXVQmSVo+xdddgx0GjVjFzcj9MBg37DpWy8If6tVIPFEmSSBxzJ/rmHZGtVeQvegHZbvHe7iwvJPejx7AXHEYdGUfzqU9jaNGpwXNqY92FllMUWg2ifP13gIKxfV90SSe201CZatZoWcSIlhDeqvaupeSXTwBIuPQWTB0a7gKd2hRNRLcLASjf5NtWOoIgBIfPa7SefvppcnJyuPfee0lNTUWlCszCT8E/nOWFVO1aBUDskIlByxHReSAlKz7EengvLnMF6poPXsFm8bR1b9MdSdNwbd3PJTUhgvuu7c0LH23ii5//pFeHJHp1Sgp2rNOoNDpSrn6YY+/NwlF4mIJvXydqzN2oqooo+fwt5KpSNLHJNLvxiQafrurhmTroqi5DdtgCuoaiqXNVl1O54xcAYgdfecpt3kJLtNkXwpj12J8UfjsPgOj+Y4nJGNPgGWIyxlK14xeq967FOeom7/RcQRBCk8+F1saNG3n22WcZN25cIPIIfla27luQXRjSegRlRMFDG5uCLqUt9uMHqd6/kejeoTHyeWLaYMN3GzyXC3u1YNugQn5Yd4hXFm5m3owRxEaFXpGgiU4gZdIschc8jjlzPYrWSFTmemSHBW1iS5rd8HiDdbg8E7UxEpUhAtlajbP0OLpksWl5oFRs+QHFaUeX2h5D626n3OadOmipRFHkgHVmE4RAcZQXcHzRbBSnHVOHfiRcfFNQcuibtUPfsjO2o5lUbv2JuKHXBiWHIAi14/NvO88mwELoc1WXU7ltBQBxFwRvNMsjIn0QEDpt3mWbGetR99Q8U4gVWgC3XdmdVilRlFbaeO2zLchyaC5+NrTsTOLo2wGw7PoVlcOCJqUtzac+HdQiy0MT62nxLqYPBorssFFesz9a7KArTtsbTWWs2RxdkZHNlQ0dTxDqRbaZyf/f87iqy9AltyH5qgeRVOqg5fGMpFVs+QHF5QhaDkEQzs/nQuvKK6/kk08+ER1vwkD5hiUoTjv65h0xNHATgjOJ6DwQAPPB7ci24HfUsxzcAbILbXzzoE1tOxeDTsOsqRnoNCo27yvg29+zgh3prKJ7X0x0zS9/R1xr4q9+NGSmh3obYojOgwFTtXMVsrkCTUwSEV0Gn3a7pNYga42A+wKQIIQLRXZx/KtXcRQeRh0RS+p1/0SlNwY1U0T6INQRsbiqy6jeFxoXLgVBODOfpw4ajUY2b97MJZdcQo8ePTAYDKfcLkkSzz33nN8CCnXjslZTvnk5ALEXTDztCnMwaBNbok1ojqM4F/OBLUTWLOoNFnNNW/dQmzZ4srRm0dx2ZXfe/HIHH36/h27tEujYKjTn5CdceivaLhdx4HhF0D+InMxTaImGGIGhKDLl690bFMcMGHfWK/2KPgIcFrGXlhA2FEWh+Mf3sGRvRdLoSL32UTTRicGOhaTWEt33Mkp//x/lm5YG/XepIAhn5/OI1uLFi4mKikKWZbZv38769etP+yMEX8Xm5Sg2M9qkVpg6ZQQ7DuAuwj2jWsHuPuhu616zf1YIF1oAowencUHPZjhdCi99vBmzNTSnikiShDaxFYRYg5wTUwfFiFYgmPdvxFGSh8oQQdQ51l7KOhPgbkwiCOGgYtNSKjYvBySSr3wAffMOwY7kFdXnElCpsR3NxJaXHew4giCchc8jWj///HMgcgh+JDtslG9YAnhGs0Lng6+p8yDK1izGfGArstOOSqMLSg5H4RFclcVIGh2GEGjrfi6SJHHfNb3580gZecXVvPXlDh66sW9IjFKGA++Ilpg6GBBl69yjWdF9L0OlO/tIpqKLAMTUQSE8VP+5ieKfPgAgfuQUItIHBjfQX2ii4ojoMpjq3X9QvmkZyePvCXYkQRDOwO+fwLOzxZWVYKvctsK9XiI2mciuQ4Id5xT6Zu1RRyWgOKxYsrcHLYe5pq27oU23sGj5HWnSMXNyP1QqiV+3HOXnTUeCHSlsaLxrtApQZFeQ0zQu1qOZ2I7uA5WG6Iyx57yvrPcUWmUNkEwQ6s6Wf5CCxXNAkYnqfTExg648/4OCIKbmZ6569++4zGKPOkEIRT6PaJWVlfHaa6+xYcMG7Ha7tymGoiiYzWbKy8vZu3ev34MKtaO4HN4rzLGDrgpqZ6QzkSSJiPSBVGxcSnXmeiI69Q9KDnP2NiD0pw2erGvbBG68rDMLlu1j/lc76NwmjpbJUcGOFfI0UQmg0oDsxFVZgiYm9PYkC1dl674BILL70PPu5+MZ0XJWiREtIXQ5K0vI//w5FIcVQ1oPEkffHrKzB/QtOqFLbYc9P5vKbSuIDYHuwoIgnMrnEa3nn3+eL774gjZt2qBWq4mKiqJHjx44HA4qKip46qmnApFTqKWqXb/jqihCHRFLZK8RwY5zRhGda9q8/7kRxeVs8PPLNgvWw+6LAeFUaAFMGtmJnh0SsdpdvPTxZhxOMUJzPpJKjTbWXVyJFu/+4yjJw5y5AYDYQePPe3/PiJYsmmEIIUp2WMn//HlclSVoE1qQMnEmktrn69ENRpKkE63eN/8gRuwFIQT5XGj9/vvv3Hfffbz11ltcd911pKam8tprr7F8+XI6d+7MgQMHApFTqAVFdlG2ZjEAMYOuCNr6p/MxtEpHZYpGtlRhPbynwc9vydkJshNNXCra+OYNfv76UKskHrqxL9EROrJzy3l/ScM/f+FINMTwP/c6UAVj+77oks6/EfSJNVplgQ0mCHWhyJQvewt7fjYqUzSp1/0TtTEy2KnOK6LrEFTGKJwVRZj3bwp2HEEQ/sLnQquiooI+fdyjAO3bt2fXrl0AREREcMstt/Drr7/6NaBQe9WZ63GU5KIyRBLd59JgxzkrSaUmoqN7ymB1EDYvDpdug2eTEGPkgevd2b/7PZsNu8UozfmIhhj+5TJXULnd3RgpdtAVtXqM7Jk6KJphCCHImPkLtqwtoNaQOumRkNxb8UxUWj3RfS4GoHzT0iCnEQThr3wutOLi4qisrAQgLS2N4uJiysrKAEhJSeH4cfFBJhgURaFs9VcARPcfG1L7GJ2Jp4NTdeYGFEVusPMqioIlu6bQaheehRZA/66pXDm0PQCvfbaV4nJrkBOFNm9DDDF10C8qNi9HcdrRpbbH0KZ7rR6jnNQMQ2x4L4QS846fMeS4L/olj7sXQ6v0ICfyTXTfy0BSYT20C3vh4WDHEQThJD5PPh48eDDz588nPT2d1q1bExMTw+LFi7n55pv55ZdfiIsLzc1UGztL1lbsxw8iaQ3eTkShzJjWE0lnxFVVgi33AIYWnRrkvI7iYzjLC5HUWgxptfuAGKr+dnkXdmUXkXW0nDe+2MWkQaZgRwpZWjF10G9kh43yTcsA92hWbRsFeEa0cDmRbWbUhohARRSaEEVRUJx2FLsV2W5BtltRav5bm78rDivWo5kARA6aQGT3i4L8HflOE5OEqVN/zJnrKd+0jKQxdwY7kiAINXwutO6//36mTp3KI488woIFC7jzzjt54YUXmD9/PhUVFdxzj9jLIRjK1tSMZvW9FLUp9DvRSRotpo79qN79B9X71jVYoXWirXvXsGjrfi5ajZpZUzJ4YM6v7Mkp5RvJRk75IbTa4K3NU2QnifrQW5Atpg76T9XOVe7tI2KSiOgyuPYPVGuQdEYUuwVXdbkotARkSyXqsmPYDrlQJAXZYakpmE4umtx/P/H1kwomh/t2/DArwtasGymDJvjhuwqOmIwxmDPXU7VzFfEjpoifL0EIET4XWi1atGDp0qXk5OQAcPPNN5OYmMiWLVvo2bMnEyaE7xtVuLIc3oP1yF5Qa4gZeP7uX6EiovMgd6GVuZ74kVMbpIWuJWsbAKb2fQN+robQPCmSuyb2Ys6nW9h+0Mz2g/uDHYnWSTp692i46aC1oYl1F1qytQqXpSosFrmHIkWRKV/v3j4iZsA4n7ePUJmicdkt7oYYCeHViEbwL5e5gsIPZhFtraLUT8eUdAZUWoP7vzojKt1f/+7+78l/l3QGnGojWeXOkG3jXhuGNt3RJrXCUXiEqh2/EDNgXLAjCYJAHQotAIPBQHr6iTnM48ePZ/z48PmA39h41mZF9RyJJio+yGlqz9S+N5JGh7M0H3vBIfQpaQE9n2y3Yjm8GwBju94BPVdDGpnRitLyajbvOUJMdDQaTfDaEa/fncfhQjtf/nqQm8b3CFqOv1LpDKgjYnFVl+EsOy4KrToy79+IoyQPlSGCqF6jfH68yhSDq+w4LtEQo8kr37AExVqFotGjjU1GbTC5Cx+tAZXe6HvBpNUjST4vOwfAbDZDRXjv/ylJEjH9xlC0/D+Ub1pGdP+xdX4+BEHwnzp9Ivvhhx/YsmULFRWn70QuSRLPPfdcvYMJtWPLy3Y3d5BUxA4Ozd3rz0alM2Js1wvz/o1UZ64PeKFlObQLXE40McloE1oE9FwNbczg1qTFVtOlSxdMpuCt1fppXRbzFu3iq1XZ9OvajB7tE4OW5a80cSm4qstwlOajb9Y+2HHCkmcz9Oi+l9ap4Y7KFA2IFu9Nncta7V3nV91jHB2GXxXU963GIrLHUEp+WYCzNB9L1jZMHRrHzA1BCGc+F1ovv/wy77zzDpGRkURHR592ezgPvYcjz9qsyG4Xhk072pNFdB6Ief9GzJnriB96XUDPZTmprbv4dxoYQ3o247fN2WzLNvPKJ5uZN2ME0RGhsZ+bNi4V29FM0RCjjqxHM7Ed3QcqDdEZl9fpGCpTDCAKraauYtMyFJsZTUILHMkNsz63KVDpjET1Gkn5hiWUb1oqCi1BCAE+F1qLFy/mxhtv5PHHHw9EHsEH9qKjVO9bB0Ds4PBcG2fqmAGSCnvBYRwleWjjmwXkPIqieBthGMN0/6xwMTYjloIKyC0yM/ezrTx2y4CQKGw9nQdFQ4y6KVv3DQCR3Yeiiapbd1m1d0RLTB1sqmS7tWaza4gYcAWFSvDfGxqT6H6jKd/wPZasrThKctHGi7WQghBMPk/gtdlsXHpp6G6G25SUrf0aUDB16o8uuXWw49SJ2hiFsabNeiA3L3aU5OEsKwC1xns+ITB0GhX3X9sTrUbFhj35fPdHdrAjAaCJSwbEXlp14SjJw5y5AYDYQXVfj6sShVaTV7H1J2RLJZq4VAydBgY7TqOjjW/mvZhYvvmHIKcRBMHnQuvSSy9lxYoVgcgi+MBVUUTVrt8AiL1gYpDT1E9EZ8/mxYErtCye0axWXVDpQnsz58YgrVkUt4zvBsD73+0h62hZcAOBd2qtUxRaPnOPQCgY2/dFl1T3izonpg6KQqspUpwOymvW+cUOnuBz10qhdmL6u/fSrNz+M7LdEuQ0gtC0+Tx18J///CfXXHMNU6dOpWfPnhiNp35olSRJ7KXVAKo3LwXZhTGtR4PtQRUopk4DYfk72I7tx1lRjCY6we/nMNe0dTc2krbu4eDyIW3Ztr+Q9bvzeWnBJuY8OByjPngdET0t3p0VxSguB5JaG7Qs4cRlrqBy+8+Ae4Pi+hBrtJq2yh2/4KoqQR2VQFTPYVhsjmBHapSM7XqhjW+GoySPqp2riO43OtiRhDDmKDtO4SdPYYxuAV26BDtO2PH5U8/HH3/MwYMHOXjwIBs3bjztdlFoBZ5kq8K881cAYodcHdQs/qCJikPfshO2o5lU799ATMYYvx5fdtiw1rR1N7Xv7ddjC2cnSRLTr+vD/a/8wrHCat5evIMHrg9eoauOiEXSGlAcVhxlhejEPk61UrF5OYrTji61HYY29Zt2q4oQI1pNleJyUrZmMQCxg6+qudAhCq1AkCQV0f1GU/zT+5RvWkZU38tCYp2sEH4URaFo2X9xleWjLzuOq7IYRIdQn/g8dXDBggWMHz+e1atXs2/fvtP+7N0b3ntRhAN9zkZwOdA371jvDz6hIpDTB62HdqM47aijE9EmtvL78YWzi47QMWNyP1QSrNx4hF83HwlaFkmS0Nas0xINMWpHdti8bbhjB11Z7w9rnjVaisOKbLfWO58QPqp2/4GzvAB1RAxRvX3fg03wTVTPEUhaA46io1gP7Qp2HCFMVe9b595CCJBQsOz+LciJwo/PhZbZbGbSpEkkJPh/epdwfrK1GsPhzYB7NKuxXKXyFFrWQ7txmSv9emyzaOseVN3bJ3L9JZ0BePPL7eQWVQUti2f6oGiIUTtVO1chmyvQRCcS0WVwvY8naQ1IGne7f5dZjGo1FYoie7ciiRkwHpVWH+REjZ/KEEFUj2EA3oslguAL2Wam+Mf3ANCktAXAvGsViuwKZqyw43OhdcEFF7B+feCaFgjnZt6+AsllR5PQElPHfsGO4zfauFR0yWmgyJj/PH1Kan14rsaYRFv3oLn2ks50a5eAxebipQWbcTjloOQQDTFqT1Fkyte7GxfEDBzvl8YFkiShFtMHm5zqfetxFB9DZYggut9lwY7TZETXTMM379+Io7wgyGmEcFOy6jNcVSVo4lKJn/gIstaAXFmM5eCOYEcLKz6v0briiiv417/+xaFDh+jTpw+RkZGn3eeqq67yRzbhL2SHjeqt7natEQPGI0k+18khLSJ9IPaCHKoz1xPVa6RfjukozcdRkgcqNca0Hn45puA7tUpi5uR+TH/lFw4cKeOjpXu49YqGn/aqqdlLyyGmDp6Xef8mHCV57ivjvfw31UsdEYuzvBBXVZnfjimELkVRKFv9JQDR/S9HpRfrOxqKLqkVhrQeWHN2UrH5BxJGTg12JCFM2PKyqagZCU0cfTsYIrA3747h0CYqtv4kLlz7wOdC6/777wfg+++/5/vvvz/tdkmSRKEVIK7qMhRLJS5TfKPcfySi8yBKf/sfluztyDYLKn3927CbD7jbuhtapYtf8EGWGGtk+nV9ePb9DXy9KoteHZPI6JLSoBm0cZ6pg6LQOh/PBsXRfS/1y8+ix4kRrTK/HVMIXZYDW7AfP4ikM3jbjgsNJyZjDNacnVRuW0ncRdeKaZvCeSmyi6Jlb4MiE9F1CKZ2vTGbzdha9sZwaBPmPzfhrCpDExkb7KhhwedCa+XKlYHIIdSCNjaF+OufIDu/pFHuP6JNauVtSWvO2kJk1yH1PqYlexsAJtHWPSQM6t6McRe2ZckfB3ntsy3MmzGC+GhDg53fU2g5y46jKIpYs3cW1qOZ2I7uA5WG6Az/fjhWR8QCYupgU6AoCqWe0ay+l6E2RgU5UdNj6piBJjoRZ0UR1XtW+222iNB4VWz5CVveASS9iYSLb/Z+XY5KRpvaHkd+FlU7fyV28FVByxhOfJ579vjjj3P48GFatGhx1j9C4OiadUAxNM5fVpIkYfJj90HZaceSsxMQ67NCyc3jutG2eTTlVXZeXbgZWVYa7NyamCSQVCgOmxhROQfP2qzI7kPRRMX79djqms6DohlG42c9tAvbsUwktZaYgeODHadJklRq7z5a5RuXoigN934rhB9nVSklv34CQPzwG9FExZ1yu7HHcAAqt60Q/5ZqyedCa8uWLeIqsBAwnu6D5gObkZ32eh3LeniPu617VDzapNb+iCf4gU6rZtbUDPQ6Ndv/LOLLX/5ssHNLaq13Q2ynmD54Ro6SPKr3uS90xA7y/4djdc10E1HoNn6etVlRvUehiYw7z72FQInqfTGSRof9+EFsxzKDHUcIYcU/vY9iM6Nv1p7ovpeedruh0yAknQFHSZ53f1Lh3HwutC666CK+/fZbHA6x0aDgf/rmHVBHxaPYrVgP7qzXsbxt3duJtu6hpmVyFHdNcDcnWbB8H/tyShrs3Jo4T0MM0XnwTMo3LAEUjO37oAvABQoxdbBpsB7b755RoFKLKUZBpjZFEdntQsA9qiUIZ2LO3kb1ntUgqUgcc9cZl6iodAYiu10EQOXWFQ0dMSz5vEZLr9fz7bffsmzZMtq3b4/pLztES5LEhx9+6LeAQtMiSSoiOg+kYtMyqjPX1auFvaWm0DJ2ENMGQ9Go/q3Zur+Q37Ye46UFm5g7YwSRRm3Az6uNTcHKTtEQ4wxc5goqt/8MuDcoDgTRDKNpKPvjCwCiegxzT9kVgio6YwyV23+met86nJUlfp8SLIQ32WGjaPl/Afe/FX2zdme9b3Tvi6nc+hPV+9bhslSKtZfn4fOIVn5+Pn369KF79+4YjUYURTnljywHZ38cofHwTB+s3r+xzhvjOcoKcBQfA0mFMa2nP+MJfiJJEvdM6kVqgomCUgtvfL6tQeZ8extiiL20TlOxeTmK044utR2GNoFpvy9GtBo/2/EczAc2g6Qi9oIJwY4jAPrUduhbpoPsomLrT8GOI4SYsjWLcZbmo46MJ37Y9ee8r65Ze3TJaSguB1W7fmughOHL5xGtjz/+OBA5BMHL0LorKmMUsqUS6+E9ddr/ypJV09a9ZWfUhgh/RxT8xGTQ8vCUDGa9/jurd+Tyw7pDjB6cFtBznpg6KEa0TiY7bJTX7JsSO+iKgE23VZvcI1qytRrF5UBSB34UU2hYnrVZEV0Go41vHuQ0gkdM/7EUHN1H5ZYfiRsyUfzsnYOzopiiT5/GENceunQJdpyAshcfo2ztYgASLr3lvFvhSJJEVJ+LKf7hHSq2riA6Y6xYnnEOdd7xNisri08//ZT//Oc/LFq0iOzsbH/mEpowSaUmolN/oO7dBz3rs4yirXvI69Q6jmljuwLw3693cii/IqDn08Z6RrREoXWyqp2rkM0VaKITiehyQcDOozJGQM3cf1d1YF9roeHZi49RvXctAHFDrg5yGuFkEZ0Hoo6Mw1VdRvW+dcGOE9LMWVtxFh3B+OevWPb8Eew4AaMoinvKoMuJsX0fItIH1epxkd2HIml0OAoPY8ttuIZW4cjnQktRFB5//HHGjRvHk08+yauvvsq//vUvLr/8cv75z38GIqPQBJ3c5l1RfJuOqjgdWHJ2uY8j2rqHhauGtadv52TsTpkXP96E1e4M2Lk8I1qu6jJkuyVg5wkniiJ7W7rHDBwf0H36JEnlHdVqauu0FEWhcs0XaHMbb7eusjVfAQqmTv3RJbcJdhzhJJJa4+0kV75xWZDThDZnRZH37+Ur3sOaeyCIaQKnavfvWHN2Iml0JF52W61HptSGCO8FOdEU49x8LrTeeecdvvzyS6ZPn87KlSvZsWMHK1as4N577+Xbb7/lgw8+CEBMoakxtu2JpDPgqizBlpvl02OtR/aiOKyoI2LRpaQFJqDgVyqVxAM39CE2Ss/h/Ere/TZwH0TVhghUhkgAnGUFATtPODHv34SjJA+V3kRUr1EBP19TbYhhz8+mev03ROz4BvvRvcGO43eOsgKqdrrXbMQOmRTkNMKZRPW5BFQabMcyseX59ru1KXFWFAOgqLXgcnB80Qs4K0uDnMq/XJYqSlZ8AEDshZPQ1lyErK3oPhcDULVnNbJNXLQ8G58LrS+++ILbbruNv//977Ro0QKdTkfLli255557uO222/j8888DkVNoYlQaHaYO7o6D1Zm+TXE4MW1QtHUPJ3FRBmbc2BdJguVrc1i9Izdg5/I0xHCIhhgAlK37BoDofpeh0hsDfr6m2hDDWe6+Si4BZcvfxmWpCm4gPytf+zUoMsa2vTA07xDsOMIZaCLjiOwyGMC7JlM4navS/bNq6TQCTXwLXFUlHP/yJRRn49naqOTXT3BVl6NNbEnsoCt8fry+ZTrahBYoDitVu38PQMLGwedCKy8vj0GDzjyHc+DAgRw9erTeoQQBTuo+uG+dT93ozNk1+2eJaYNhp3enZK4e0RGA1/+3leMl5oCcx9sQQ6zTwno0E9vRfaDSEJ0xtkHOeWJEq4kVWlUn9ouTK4spWvZ2g3TabAjOytITWwOItVkhLbq/++e8evcfTe5nsLY8I1quyERir3wQlSEC27FMipb/t1H8zFqP7adyi7v7ZOLoO+rUGEWSJKJ6u0e1KreJ6YNn43Oh1aJFCzIzz7yz+L59+4iPF3szCP5hat8XSa3FWZqPo/BIrR7jrChy31dSYWzbK8AJhUCYPDqdzm3iqLY6eXnBJpwu/28Z4W2IIToPetdmRXYf2mB76zTVqYOuSneh5YxtCZKK6r1rqNq5Ksip/KN8/bcoLgeGVl0wtO4a7DjCOeibd0SX2h7F5aBi28pgxwk5iqJ4Cy3ZEI0mNoXkCQ+BpKJy+0oqwnwkUJFdFC19G1CI7DkcY5tudT5WVI9h7qmoeVnY8g/6L2Qj4nOhNW7cOF5//XWWLVvmreoVRWHp0qW88cYbjB3bMFdEhcZPpTdibOculmo7fdB8wN3WXd+iI2pjZMCyCYGjUauYObkfJoOGfYdKWfjDPv+fQ0wdBMBRkkf1Pndnz9hB4xvsvE126mDNiJYjqQORgycCUPTDf8P+36HLXEHFlh8A92iWmLId2iRJIqb/GAAqtvxQ5/0qGyvZWo3isLr/bnBvxmtq15v4UVMBKP7pfSw5O4OWr77KN36PvSAHlTGShJHT6nUsdUQMEZ0HAGJU62x8LrRuv/12evbsyYMPPkiPHj246KKL6NGjBzNmzKB79+7cf//9gcgpNFEnpg/Wrs27Z32WSbR1D2upCRHcd21vAL74+U+2/1no1+N7Fv029RGt8g1LAAVj+z7oklo32Hmb7oiWezG9bIgiov94DK26oNitFHwzN6w/7JZv+B7FYUOX2h5ju97BjiPUQkTXIahM0bgqijDv3xjsOCHF03FQMkTCSVPqYgaMJ7LHMFBkjn/1cljuxeisKKJ01f8AiB851fteXB+e6YNVu35DdtjqfbzGplaF1oYNG7BY3B1FdDod77//Pm+//TY33XQTw4cP56abbmL+/Pl8+OGH6PX6gAYWmhZTxwyQVNgLcs571VdxObxXmcT6rPB3Ya8WXDaoDYoCr3yymbJK/72Be6YOOsoKw/oDbn24zBUn1tQMurJBz+0Z0XI20REtWR+JpFKRdOV0JL0J27H9lP7xRZDT1Y1sraZi01IA9ya4YjQrLKg0OqJrPiCX17x+gpur0j1tUB2VcMrXJUkiccyd6Jt1QLZUcXzR7LDbIqTox/dQHFb0LdOJ6jXSL8c0tu2BJjYZ2Wb27qEnnFCrQuvuu+9mz549AEybNo2srCyGDRvGzJkzefrpp5k5cybDhg0LaFChaVKbojHUzB+uztxwzvtaj2ai2C2oI2LQpbZtiHhCgN12ZXdapURRWmnjtc+2IMv+WYSsjooHtQZkp3cuflNTsXk5itOOLrUdhjbdG/TcnquosrlpFVqeNVpyzfYC2phkksbcAUDZH19gPer/abKBVr75B2SbGW1iS0w1U4iE8BDd7zKQVFgP7cZecDjYcUKG53eC+gxrVlVaPSmTZqGOiMVecJiCb1/3ea/PYKnevxFz5npQqUkacweS5POktjOSJJV3WxAxffB0tXqWZVlm7dq1HDt2jA0bNpCTk0Nubu5Z/wiCP3mnD55nnZa3rXu73n57AxGCy6DTMGtqBjqNis37Cvj2d//s+yKp1GhjkoGmOX1Qdti8rZ1jB13R4KMQ3qmD5somM6IoO2zI1mr33/VR3q9HdruIyO5DQZEp+GYusi0wnTYDQbZbKd/wHeBZmyXed8OJJjrRu75GtHo/wTN18EyFFoAmOoGUSbNArcGcuZ6yMBiNlu1Win98F3BvSu/vzcSjeo10F+1H9mIvEt3HT1ard8VLL72UN954g4svdg8z33vvvYwaNeqsfwTBnyI6uX8R2I5mnnPDQEuWaOveGKU1i+bWK90jLh9+v4cDR8r8ctym3BCjaucqZHOF+4NWlwsa/PxqUzQggSLjMlc2+PmDwVXlfu+SNDrQnDrFPvGy29DEJOMsK6Doh3eDEa9OKretcP87ik0hsuuQYMcR6iA6w90Uo2rXKlw1FwKaOs+Iliry7F1YDS07kzTmTgBKf/sf1Zm1W0ceLKV/LMJZXogmOpG4C6/x+/E1UfHevU8rRSfLU2hqc6dnn32W0aNHU1payqOPPspdd91F69YNt3BaaNo00QnoW3TCdmw/5v3rie43+rT7OCuKsRccAiSMbXs3eEYhsMYMTmPb/kLW7szjxQWbeO3BYZgMvu/7cTJtXCoWml6hpSgy5evdoxAxA8cjqdQNnkFSqVGZopDNFbiqy9BExjZ4hobmrJk2qIqMg7+MIKoMESRfeT+5H/+Lqp2/Ymrfh8huFwYhZe0pTgdla90bXcdeMCEo/46E+jO07oY2qTWOwsNUbv8ZXQ9xsdx5ljVafxXVayS24wep2LiUgm/n0eJvz6NLDr3PxvaCw973/ITLbkOlMwTkPFG9R2H+cyOVO38lfviNSJr6/Y5uLGpVaKnVaoYPHw7AnDlzGDduHO3btw9kLkE4RUTngdiO7ac688yFlmeTYn3zDqhNUafdLoQ3SZKYfm1vDhwtI6+omre+3MFDN/at15Q3TRPdS8u8fxOOklxUepN3Xn0wqCNiawqtprFOy7M+S1XTCOSvDK3SiR1yNWV/LKJo2dsYWnZGE5PUgAl9U7njF1xVJaij4onqMTzYcYQ6kiSJmIwxFC17m4rNy0noPiLYkYLOdfLUwfMMuCeM+hv2wiNYc3aSv2g2LW55AbUxdD6DKIpM0fL/gOzC1GkAEZ36B+xcpg59UUfG46oqoXr/BjHKXcPnCdXV1dXs3r3bbwFkWWbevHlcdNFF9O7dm9tvv50jR86+OW1OTg533HEHGRkZDB06lHnz5uF0Ok+5z0cffcQll1xC7969mThxIqtWNY4NIZsyzzxyy6HduCynv/NZRFv3Ri/SpGPm5H6oVBK/bjnKz5tqt4n12XhavDtKm1ah5dmgOLrfZaj0xqDl0HjXaTWNQsvTcVB9julIcRddg75FJ2SbmYJv54Xs+jVFdlG2djHg7lgprlyHt8juQ1EZInCW5mPP2RHsOEF18mbF55o66CGpNaRMmIEmNhln2XEKFr8aUj+3ldt/wXpkL5LWQOJltwb0XJJK7e1kKJpinOBzoRUTE4PB4L9hxzfffJOFCxfy9NNP89lnnyHLMrfddht2u/20+5aXlzN58mQsFgsffvghr776KsuWLePxxx/33uerr75izpw5zJgxg++++45hw4Zxzz33sG9f+HVzEk7Qxjd3D8nLLsx/bj7lNsXlxHzQ/cvBKNZnNWpd2yZw46WdAZj/1Q6OFVbV+VjauKY3omU9th/rkb2g0hCdEdzN5VVNbC8tzx5aqnNMk5RUapKvvB9JZ8B6eI93al6oqdr9O86yAlSmaKL6XBLsOEI9qXQG7wfk6m0/BTlNcMmWShSn+/PnuS6KnExtiiL1mn8gaQ1YDu6geOVHgYxYay5zBSU/u7PEDbsOTXRiwM8Z1XsUIGE5uCMs9xkLBJ8LrTvvvJNnnnmGt99+m1WrVrFx48bT/tSW3W7nvffeY/r06QwfPpz09HTmzJlDfn4+P/7442n3X7x4MWazmblz59KtWzcyMjJ45pln+PLLLzl61N3lZMWKFVx44YWMHj2aVq1acf/992MymVi7VvT2D3emzoOA07sPOvKyUGxmVKZo9M3FlNbGbtKoTvRon4jV7uLFjzfhcNbt6qFn6qBsrT7jKGljVL7O/cE9svtFaM7SUauhePbSaipTB73rPiLiznk/bVwqiZe6rzyX/vYZ1twDAc/mC0WRKVtTM5o1cDwqrdg7szFwT8mXsOfsQFXdNLe8gJNau0fE+DRSq0tuQ/IV9wFQsWGJd4/CYCpe+TGypQpdchox/S9vkHNqY5MxtusJiKYYHj4XWk888QQFBQXMmTOHO++8k2nTpnn/TJ06lWnTptX6WPv27aO6uprBgwd7vxYdHU3Xrl3PWLAdOnSIdu3aER9/4gNC165dAdi0aRMACQkJbNy4kX379qEoCkuXLqWyspIePXr4+q0KIcbT5t2Svf2UTQJtOdsBMLXtJdoLNwFqlcSMyX2JjtCRfaycD5bsqdNxVFo96kj3h96mMH3QUZJH9T53Z6zYQVcEOc1JUwebyohWlWdE69yFFkBkzxFEdBkMsovCb14LqU1RqzPX4yg6isoQccb1skJ40salYurgnnqvP7ItuGGC6ERrd99HfyLSBxF70bUAFC57G+ux/X7N5gvL4d1U7fgZkEgce2eDNquJqtkIu3L7LyE1jTJYatUM42QffeS/IdH8fHe3r2bNmp3y9eTkZO9tf/16QUEBLpcLtdr9j+bYsWMAFBe7r0Lcd999HDhwgCuvvBK1Wo0sy/z73/8mIyOjXlkVRcFsDv7+JhaL5ZT/NqUMSmQS6phkXOUFlO1Zj9LKXTxbDm4DQN2qW4O/RqHwejTFHEYt/H1CV15YsI1vf88mvXU0/dKTfM6gik7CVVVK9fHDyLHN/ZoxFF6TkzNUrF0MKOjSeuKMSMTZgD8rZ3ounFoTAPaKkgb5uQ326+GouVLu1JrAdv4cEcOnYTmSiaMkj+PL3iHmEv+tr6jrc6EoCiW/LwLA2OsSrC6gHq9dsF+TUMoRChk0bfvAgc2oKwua7HNhKc4DQIqIqVMOfb/L0edmYcvaTP6iF0i48SnvBb065alDBsXlpOj7+QAYe4xAjmtZ7/dYX3JILbujMkbhqiqhdPdaDH5aOx8KPyMnUxSlVg25fC60Bgzw387vnidLp9Od8nW9Xk95+enTScaMGcObb77J888/z0MPPYTZbOaZZ55Bo9HgcDgAOHz4MLIs8+KLL9KxY0d+/PFHnn32WVq0aMFFF11U56wOh4O9e/fW+fH+lpOTE+wIQclgjG+HobyAgi0rqXZFIdmqkIuOoACH7HqUIL1GofB6QNPKYQQGdY5kXWYVr3+xg7+PSSHadOKqXW0ymNCjB/L/3IVVjglIzlB4TQ7t30PMzlVIQHFSd46HwM+JpricKMBcUtCgeYLyeigKsRXFSEBuaRWY4mqVQ9NlNJEbF2LZ9SuFmngcqel+jeXrc6EpzCKq4BCKWsuxiDS/vd+Gws8IhEaOYGbQlJmJAlTWiib7XBhy/sQIlDsk8mvO73OOtsOJPn4IqorI+3w2lQOmgNrnj9un8CWDIWsNxpJcZJ2JvORe5Prx/bW2OYwpXTHkrKdg7XdU2/3bdCkU/m16/LV+OZM6vfIlJSW8++67rFmzhsLCQt555x1WrFhBenq6d1Pj2vA01bDb7ac02LDZbBiNp78waWlpzJ07l8cff5xPPvkEk8nkHcGKiorCbDZzzz338Oijj3LllVcC7qmFx44d4+WXX65XoaXVaunQoUOdH+8vFouFnJwc0tLSzvgcNfYM9hgNJQfXoS/OJrFVC/LWunez16a0Jb1X/UYt6yIUXo+mnKNDR5n8/2wgJ6+S5dus/Ovmfths1lpnqCrfR1XuTuJ1CjFduvg1Wyi8Jp4MKRXZ2GQnmuQ0OgwZXa+2+PXJcfJz4YjTU7wZdLKNLn5+7muboaHI1moKZHd33FadunHoaG4tc3ShUqmgetMSovb9SGL/YbVeoH8udXkuFEWhZPsiHEBE70to1qtfUHIEQijkCIUMzrJ4ijYuRGUpp02bNphMpqDkCOZzUZazCiuQ2LoD6rS0OudwtmpG8cIn0JTn0uzYGmIuvaNO77u+PhfOsgKKfloNQNzIaTTv4r/RJJ9ypMRSlLMeXVEWLVqlBO19K5AOHKjd+lmfC60jR45www03YLPZ6NevH/v27cPlcnHw4EHefPNN3nzzTe+eW+fjmTJYUFBwygbIBQUFdO7c+YyPGTlyJCNHjqSgoIDY2FicTiezZ8+mVatWZGVlUVZWdtp6rN69e/PTT/XrpCNJUtDedM7EaDQGPU8wMhjb96A8Mg5XVSnqwoNoi7IAiOzYL6jPRyi8Hk01x6N/G8ADc35lT04pS9Ye5YohrWqdwZXUkipAqSoOWN6gvyYuB47dvwAQf8FVREREBC3Kyc+FMzGVYkA2V2I0Ghus+AvG62GvaS6gMkRiiooBcmudw3jxFI4d3YM9P5uqn94h9cbH/bYW1ZfnwnJoF468P5HUWhKHTETjx+cw6D8jIZQjmBkUXUuKAEl2YsDZJJ+LMnMZAKbEZqhqPszXKYepLZqrZ5D/6TNY9/xBRIuOxAwYV+dctcmgKAr53y4AlwNDm+7E973Y7++rtX4uTO2patUF65G9OPevI+rCSQ2fIcBq+9z6/G79wgsvkJCQwMqVK3njjTdQFAWAV155hZEjRzJ//vxaHys9PZ3IyEjWr1/v/VpFRQV79uyhf//TN1XbtGkTU6dOxel0kpycjE6n48cff8RoNNK3b19SU9374mRmZp7yuMzMTNLS0nz9VoUQJEkqb1MM6/71aIoPAmL/rKaseVIkd03sBcCnP+xjb05prR/bFPbS0uXuQrZUoolOJCJ98Pkf0EDUppqpmrIT2Vr3Nv3hwFmzWbE6yve1GpJaS/JVDyBpdFhydlK+fom/49VK2eovAXf7Zk0dvg8h9EkaLaqan0tXZdPsPHiiGUZCvY9latuLhIv/BkDxig8xH9xe72OeS3XmOveeomoNiWPqNoLmT1F9appibFuJoshBzRJMPhdaa9eu5e677yY6Ovq0F/G6667jzz//rPWxdDodU6ZM4eWXX2blypXs27ePBx98kNTUVC699FJcLheFhYVYrVYA2rVrR2ZmJi+88AJHjhxhxYoVPPPMM9x5551ERkaSlJTEuHHjeO6551i5ciVHjhzho48+4ssvv+Suu+7y9VsVQpS3++De1agcViR9BPrmwZ/WKQTPyIxWjOjXElmB17/YhdlWuzd1T6HlqihGcToCGTEoFEXGcNB9IStm4Hikeq4T8CdJo0VlcI+uNfYW766azYrr2lJfl9CChEtuBqDkl0+w5R/0W7basB7bj+XgDlCpiRl8ZYOeW2hY6pq9llw1BUdToiiKdxsGf+05Fd3/ciJ7jgBFpuCrV3GU5PnluH8l2ywU//AeALGDJ6BLaBGQ8/giIn2weyPs8gL3+0cTVaffuhrNmR9mt9t9rqCnT5+O0+nksccew2q10r9/f9599120Wi1Hjx5l1KhRPP/880ycOJH4+Hjmz5/P7NmzGTduHElJSdx7773cdNNN3uM9++yzvPXWW8yePZuioiLatm3Lq6++ymWXXVaXb1UIQYbWXVEZI5Et7qvg+jbdG7R1qRCa7prYk32HSskrqmbhr0XsOLbvrO9VXorCCEmLRnGw4IvfMev9t6Gj0+mkpKSMddm1yBEg8WW76WcuwaHSs+hwCq5jwflld7bn4gLZQATVfL18C6URJUHJ0BDSivbQEThQLLHj+311y6Gk0CuqM8mVmWR+PJv17e5AVtV+n5+TOZ1OqirKadXGQW1m4HhGsyK7D0Ubk1yncwrhQR2diCM/q0mOaMnmCnA5AQlNVDwOm73ex5QkicQxd+AoOoot90/yF82mxU2zUen9u8ao5LfPcFWVoIlLJXbIRL8eu65UWj2R3YdSsWkZlVtXYGrXO9iRgsLn3zYZGRm8/fbbDB48GL3evVGhJEnIssynn35K376+TeFSq9U8/PDDPPzww6fd1rJly9OmAfbt25fPP//8rMczGAw8+OCDPPjggz7lEMKHpNZg6tifqh3udSf6tF5BTiSEApNBy6wpGcx8/TeOFts5WnykVo/rGh1JC00pu7btZY+jZQCSBW9a3D1Rf4AWVlV34Lu1uUHLccKpz0XbKDUdtLBz90G22ZWgZGgIV5uO0dEAu/NdLD/o+Xfpe45fpF48EnOYGHsR2m1f8qV5YL1yVTl28uQdQ1Cpzn6B1HY8B/OfmwCJ2AtC4wOcEDiqmilzTXFEyzttMDK2ZvS//oUWgEqjI2XSLI69NwtH0VEKvp1LyqRZfltracvPpmLjUgASR9+OSnP+TngNJar3xVRsWkb1/o24qstRRwSmu28o87nQmjFjBjfccAOXXnopAwcORJIk3n33XbKysjh06BALFy4MRE5BOEVE54HeQkuX1jPIaYRQ0aFVLP+c1pdVG/aTmJiIVnv+K/6GrGZQXsolXUz0SO7ktywOh4OioqJa5/A3SXbSYXshKBDdawTXRaY2eAaPsz0XsdmboayAYelRdPbjc+9LhobQOWsTlEOHTmlMjG1brxxHK4zEHPiQoYZMkrr1pzTG95bvNrudJatz2PZnMd/+nsVVw84+9bpszVcARHS9AF2Cf/eaE0KPd+pgExzR8hRaGj+sz/orTVQ8KZNmkfvxvzDv30jpb58TP+z6eh9XkV0ULX0bFJmIrkNCbtRIn5KGvlkHbHkHqNz5K7GDmt7UY58LrU6dOvHll1/y+uuvs379etRqNWvWrKF///688MILZ+0WKAj+ZGrXG32HDMqd6iZ5hUQ4u+7t4lHbYujSpUOtOhMVr+hA+fo99G4OiZf4r8242Wxm715HrXP4my33AMe2uZC1RsaPHRDUboNney6KlreiYvNu+rYxED88sC3eg/l6HHvfga0cBg/swgWtOtQzRxeKfyqmfMMSuuV9S8txI9BExvp0BLPZjMNSxvcby/jw+z10b5dIh1anH8NenEv1njUAYjSriVB7R7SaYqHl/p7V0f4vtAAMLTqRNPYuCr97g7I/FqFLaUNkPRsUVW79CVveASS9iYSLb/ZTUv+K6nOxu9DatoKYgVcEvUlHQ6vTuGVaWhqvvPIKf/zxB7t27WLt2rXMmzdPFFlCg5E0WuLG348lfVSwowhhThuXAoCzND/ISfzLmutuTOSKaRayv9jUEbFA42+G4axyd8L0x14yAHEjJqNLboNsrqBwyYnuv77I6BDBwK7JOF0KL368CbP19GYwZWsWAwqmjhnoU9LqH1wIeZ4iQ65sglMH/dwI40yieo7wtnkv/PZ1bMdz6nwsZ1UpJb98AkD88BtDthtoZNcLkbQGHMW5WI/sCXacBlenQstms/G///2PGTNmcNttt/HII4/w9ddf43Q6/Z1PEAQhoDSx7kLLUda4Wrzbct2bKTpjQne6l2c02lVdFtwgAaQoMq6aQquuXQf/SqXRnWj5nrWVik3LfD6GJEnccVVXkuKM5BVX89aXO04p2BzlBVTtWgVA7JCr/ZJbCH3qKHeRIVsqkR22IKdpWN6pgwEstADiR03D2LYnisPG8UUv4DJX1Ok4xSs+QLaZ0TdrT3TfS/2c0n9UeiOR3S4E3K3emxqfC63c3Fwuv/xynnjiCXbs2EFVVRWbN2/mH//4B5MmTaK8vHFfmRQEoXHxtHh3lh6v08hAqLLl7gfAGRsOhVbj/b3hqq4A2QVIqH2c4ncuuqTWxI+cCkDJyo+wFx72+RiRRi0PT85ApZL4dctRft50ooFM+dpvQHZhbNsTQ4vArp8TQoekN6Go3c0UnOWFQU7TsDzTJTUBmjroIanUJE94CE1sCs7yAo5/9QqKy7eBCnP2dqp3/wGSisQxd4Z85+Wo3u49tar3rsVladz7Jv6Vz4XWs88+iyzLLF68mJ9++onPPvuMFStW8MUXX1BeXs6LL74YiJyCIAgBoYlJAkmF4rTjqioLdhy/cFmrcRS7uwy6YpoFOc3ZNYWpg549tNQRMX7/MBSdMQZj+74oLgcFX7+G7PS9S1qXtvHceJl72v/8r3ZwrLAKZ2Wp98qzGM1qWiRJQjZGAydGeJoKZwMVWgBqYxSp1/4DSWfAemgXxSs+rPVjZaedouX/ASA6YzT6Zu0DFdNv9M07oEtujeK0U7Xrt2DHaVA+F1rr169n5syZdOly6sLl7t2788ADD7ByZdMbFhQEIXxJao13qoizrHGs07LluacNqqOTUHTBa4JxPidPHWxMo4knc1bWFFp+mjZ4MkmSSBp3DypTNPaCQ5TWrNfw1aSRnejZIRGr3cWLH22idN03KC4H+padMbTu5ufUQqiTje6fy6Y0oqUosvdnNdBTBz10Sa1JvmI6ABWbllKxbUWtHle2ZjHO0nzUkfHED7shkBH9RpIkonpfAkDlthWN9v3+THwutPR6PWr1ma/KRUZGNqknTxCExsHTEMPRSBpi2I65G2FoQ/xKp2dES3HaUezW4IYJEJfnw1sACi0ATWQsyePuBaB8wxLM2dt8PoZaJfHQjX2JjtCRn3ec0k3LAYgbcnXINlIRAsdl8BRaTWdEy1VVDrITJBXqyIZrKhHReSBxQ68DoGjZf7Ee3XfO+9uLc71bLiRcegsqfcN3tK2ryO5DkTQ67AWHvGuImwKfC61p06bx6quvcuzYsVO+Xl5ezvz585k2bZrfwgmCIDQETc06LUdp42iI4fklpk1tF+Qk56bSGZC0BqDxNsTwd8fBMzF17Ed0v9GAu5NZXRbXJ8QYefCGvgwz7EMjO3DGtMLYvq+/owph4MTUwaYzonXKZsUNvN4p9sJJRKQPAtnJ8S9e8k5h/CtFUShe/h9wOTG27+N+TBhRGyO9mStrOXrXGPi8j1ZOTg6lpaWMHj2afv36kZKSQmlpKZs3b8ZisWAwGFi/fj3gHir88MPazzsVBEEIBm1N50FnI+g8qCgKtprW7tqU9lDhCnKic1NHROMss+Iyl6OND931ZHV1YkQrsFfJ40dNw3JoF46ioxR+/yYpkx7xeTSqb7sooiL2gwyfF3bi7xVWEmKMAUoshCq5KY5oNUBr97ORJBVJ4+/FUZKLveAwx794gWZTn0al1Z9yv+rdf2DJ2Ymk0ZF42W1hOdoc1ftiqnb9RtWeP0i45CZUusb//uLziNbRo0fp3LkzvXv3xuVykZubi8VioWvXrvTr1w9w/6JXFAVZlv0eWBAEwd9OjGiF/9RBV0WRe3RIUqENg72PvA0xqhpnQ4xArtE6mUqrJ/nKB0Clwbx/I5Vbf/L5GBWbl6OVrZRIcWyoas7Ln2zGJYvlAE2Nd0SrCa3ROtHaPfCNMM5EpTOScs0/UBkjseVlUbR0/ilLcWRrNcUr3gfcI2CebrnhxtC6K9r45ih2K1W7Vwc7ToPweUTr448/DkQOQRCEoGlMI1rWmmmDuuQ2SBpdkNOcX2PfS8u7h1YApw566FPbEj9iMiUrP6T4p/cxtO6KLrFlrR4rO2yUrf8OgJRh12D43sGurGI+X7GfGy7tHMjYQojxNsOoLEaRXSHfOtwfvB0Ho4JTaIH791DKxJnkLXyKql2/oUtJQ9ezpoHE6kW4qsvRJrYkdtAVQctYX+6mGKMo+fljKretILrPxcGOFHB12rBYEAShMfE0w3BVlyPbLEFOUz+e/bP0LToGOUntNPYW786aKUmBHtHyiBk4DmNaDxSnnYJv5qK4HLV6XOXWn5DNFWhik2k5+GL+fnUvAD77cR+7s8+8ZkRonBR9JKjUILsazZYX5+P9OQ3C1MGTGdN6kHDJzQCU/LwAW84O1GXHsOz4GYDE0XcgqbXBjFhvUT1HgEqDLfdPbMdzgh0n4EShJQhCk6cyRKAyRgHgCPNRLU8jDEPzcCm0Gu+IluJyINc0pghU18G/cq/3uA+VMRJ7fjYlqz4772MUp4Oydd8AEDt4ApJKzYh+rRiZ0QpZgZc/2Uyl2fc9uoQwdVLnvaYyffDE1MHgFlrg3h8vqtdIUGTKlv4fETuXAAqRPYdjbBP+2y2oI2KI6NQfwLtfX2MmCi1BEAROmj4Yxp0HFdmFLS8LcG8QGQ7UpppCy9z4RrS8owEqjbeQbwia6ASSxt4NQPnab7Ac2nXO+1fu/BVXZQnqqHj31eYad03sSfPECIrKLMz731axfUsToqqZQtdUOg825GbF5yNJEomj70DfohOKzYy6uhjJEEnCyMbT1Tuq9ygAqnatQnbYgpwmsEShJQiCAGg8e2mF8abF9sIjKA4bks6INqFFsOPUijoyFmicUwedJ3UcbOgOYRHpA4nqfTGgUPDNPFyWqjPeT5FdlK1ZDEDsoCuRNCemJRn1Gh6emoFGLbFuVz5LVx9siOhCCPBMoWsKI1qK7DqpO2jwCy0ASaMl5epZqGpGFqMuus47+t8YGNv1QhOThGytpnrfumDHCShRaAmCIIC3i1M4j2h5pg3qm7UPmwXsjXnqoLOqYToOnk3CJTehjW+Gq7KYomXzzzgiVbVnNc6y46hM0TWF2ak6tIzl5nHu6Urvfrebg7mNryAWTnei0Gr8Ld5dVWWgyKBSey/8hAJNVBwJ1z9BZb9rMXYbFuw4fiVJKqJ6uUe1GvueWrXqOrhx40afDtq/f/86hREEQQgWTWz4j2h59s8yhEkjDDjRDMPZCEe0XJUN13HwTFQ6I0lXPkDuh/+keu9aqjr8esrUQEWRKVv9JQAxA8ah0hnOeJzxF7Vj25+FbNxznBc/3sScB4Zh0PvctFgII+qakR1HExjR8jTC0ETGhdwFKnVUAs6kDmG5Z9b5RPUaSenvn2M9vAd7cS66hObBjhQQtXqnnDp16llfZM8VspNv37t3rx+iCYIgNBytdy+tcB7Rchda+mbhV2gpNjOy044qDFrS15bLO6IV2M2Kz8XQvANxQ6+n9NdPKPrhHQytuoDevU+S7cBmHEVHUelNxPQbfdZjSJLE/df1Yforv3K0oIr/fL2T6df1aahvQQgC74hWReMf0fJ8j8HuONjUaKITMLXvg/nAZiq3rSBhVONZg3ayWk0d/Oijj/jwww/58MMPef7559FoNFx33XV88MEHLF26lIULF3LrrbcSERHBK6+8EujMgiAIfudp8e4sL0SRXUFO4zvZbsVeeAQIn0YYACq9CdTua35yIxvV8q7RCtKIlkfs4CsxtO6KYrdS8PVrKC4nKApVG74FIDpjLCpDxDmPEROpZ8bkvkgS/LThML9tPdoQ0YUg8YxoOcsLG30TlGBvVtyUeaYrV+74pdZbUYSbWo1oDRgwwPv3qVOnctNNNzFjxoxT7tO3b18MBgPvv/8+Y8eO9W9KQRCEAFNHxSOptSguB86KIm8XwnBhy88CRUYdFR9WHxgkSUJtisFVWYyzuhxNTFKwI/mNZ4F9sNZoeUgqNclXTOfofx/ClvsnVeu/QePU4yzIQdLqiRlwea2O07NDEtde3In//bSfNxZtp1PrOFITzl2gCeHJU2gpdguyzYz6PIV4OHOFUMfBpsbUsR/qyDhcVaVU799EZJfBwY7kdz43w9ixYweDB5/5iejTpw/79++vdyhBEISGJkkqNLHJADhKw2+dlrcRRpjsn3WyE5sWlwU1h785q2rWaAW50ALQxCSROPYuAKo3fINp748ARPe9FLUputbHueGSznRtG4/F5uSlBZtwuuSA5BWCS9LqUdX8u2jsnQdPtHYXUwcbmqRSe9eNVm77KchpAsPnQis1NZXff//9jLctX76c1q1b1zuUIAhCMGjCeC8tbyOMMJo26NFYOw96pg56Nn8NtsiuQ4jsMRwUBbW5FNQaYgZe4dMx1GoVMyb3I9KoZf/hMhYsE2uyGytNtHt0ufEXWjVTB6NEoRUMnj21LNk7cJQVBDmN//lcaN188828//77zJgxgyVLlrB69Wq++eYb/v73v/PFF19w9913ByKnIAhCwHkbYpSFYaF1rKYRRliPaDWeNVqy3YJiMwOhszcPQOJlt6Ku+QBt7DasTqNtyXEmpl/XG4AvfznAln2N78ORAJqYprGXlmdESy2mDgaFNi4VY1oPQKFy+8pgx/E7n/uzXn/99TidTt566y2+//5779ebNWvGyy+/zJgxY/waUBAEoaF4G2KE2YiWs6q05qqshL5Z+2DH8Zk6wj1FqTEVWq6aaYOSzoBKbwxymhNUehNxEx7myO9fkzzkmjofZ3CP5oy9II2la3KY8+kW5s0YTlz0mdvDC+HJs17SWdF4Cy3F5fT+rIo1WsET1ecSLDk7qdz+M3EXXRtybfbro04bYUyZMoUpU6aQnZ1NeXk5cXFxpKWl+TmaIAhCw/LupRVma7Q867O0SS3dXfzCTGNcoxUqHQfPRBPfDGun4eftNHg+t1zRnT0HS8jJq+DVT7fw5O2DUaka334/TZXWU2g14k2L3UWWAiqNdwqz0PAiOg1AZYzCVVmCOWsrER0zgh3Jb3yeOuhRXl7OwYMH2bdvH9HR0WRnZzf6FqCCIDRuJ08dDKf3s3DcP+tkmkY4ddCzWXGwOw4Gkl6rZtbUDHRaNdv2F7L41wPBjiT4kac5RGOeOniiEUY8klTnj8RCPUkaLVE9hwNQuW1FcMP4WZ3+Vb311lsMGzaMe+65h6eeeoq8vDyef/55rrnmGioqKvydURAEoUF4ug4qNjOypSrIaWrPM6IVjo0woHE2w3DWbFYcCh0HA6lVShR3XNUDgI+X7SXzUEmQEwn+cmLqYOMd0TrRCENMGww2z55a5j8346y5UNUY+FxoLViwgNdff52bb76Zzz//3HvVd8qUKRw5coS5c+f6PaQgCEJDUGn1qGumeoXL9EFFkU+MaLUIzxGtxtgMI9Q6DgbSpQNbc1HvFrhkhRcXbKba0jg3Hm1qPCNarqpSFGfjfE2dlaK1e6jQJbZE3zIdFJnKHT8HO47f+Fxoffzxx9xxxx3cf//9dOvWzfv1YcOG8cADD/Dzz43nyREEoenxNsQoC49Cy1GSh2wzI2l06JLCc3sNz4iWbKlEkV1BTuMfns2KG/uIFrg3nb5nUi9S4k0UlJj5vy+2h9XUW+HMVKZoJI0OOFGQNDaeES3RcTA0RPdxj2pVbluBojSOPfp8LrRyc3MZMGDAGW9r164dRUWNd4hZEITGTxPnaYgRHp0HPaNZutR2SOo69TcKOpUxEmrWR7iqG8f0c08ns8a8RutkEUYtD0/ph1ol8fu2Y/y04XCwIwn1JElSo2/x7l2jJaYOhoSILheg0ptwlhVgydkZ7Dh+4XOh1axZM7Zu3XrG23bt2kWzZs3qHUoQBCFYtLE1DTHCpdA6Fr4bFXtIKjVqk6fFe1lww/iJd0pSCHYdDJTObeKZMqYLAG8v3snh/MZRNDdl3nVajbTQcnnWaImpgyFBpdUT2X0oAJXbGseeWj4XWpMmTWL+/Pm8++675OTkAGA2m/nhhx94++23mTBhgr8zCoIgNBhP50FnmGxa7GmEEY4bFZ+sMTXEUBSlSXQdPJOJwzvQu1MSdoeLlxZsxuZoHFNBmypNdONu8X6i66AotEKFpylGdeZ6XObwv1jjc6F1++23M2HCBF5++WXGjRsHwLRp03jggQcYPnw4d955p99DCoIgNJQTUwdDf42W7LRjO54DhG8jDI/G1BBDtlahuNzNAzRNoBnGyVQqiYdu6EtspJ6cvAre+3ZXsCMJ9eAZ0XI0whEtxeXwvt+IzYpDhz61LbrU9uByUrnz12DHqTefJ/RLksRTTz3FzTffzPr16ykrKyMqKor+/fvTqVOnQGQUBEFoMNqaTYtdlSXITjuqmsXgoch+PAdkJypTNJqY5GDHqRfviJY5/Astz2iWyhiFpNEGOU3Di4s28OCNfXniP2tZuiaH3p2SGNyjebBjCXXgWaPlqmh8hZa7M6iCpNaiqpm6LISG6D4XU7Qsi8qtK4gZMB5JCt+N0H0utD788EPGjx9P27Ztadu2bSAyCYIgBI3KFI2kM6DYrTjLCtAltgx2pLPyThts1iGsfxFB45o66F2fFdW0RrNO1rdzMhOHd+CrXw8w93/baN8yluQ4U7BjCT7yTB1sjCNanmmD6uiEsH//bGwiu11I8YoPcBQfw3Y0E0Or9GBHqjOfpw6+9NJLDB06lDvuuIOlS5dis9kCkUsQBCEoJEnyNsRwhnhDDE/HQUOYTxuExjV10NtxMLJpT0eaMqYLHVvFUm1x8PKCzbhcjaNdc1PimTroqihuNO22PVze9VlN++c0FKn0JiK7DgGgYttPQU5TPz4XWr///jv//Oc/MZvNzJgxgwsuuIBHH32U9evXByKfIAhCg/Ou0wrxvbS8GxWHeSMMaGwjWp49tJruiBaAVqNi1tQMTAYNe3NK+PSnzGBHEnykiYoHSXXKeqbGwik6DoY0b1OMPWtwWauDnKbufC604uLiuPHGG1mwYAE///wzd999N/v37+emm25i+PDhvPLKK4HIKQiC0GC0YbCXlstSiaMkDwB9GLd29/COaFWF/4c5z2bF6ibU2v1sUhMiuGdSLwA+X7GfHQca3xS0xkxSa1DXNHRpbJ0HvYWW2EMrJOlbdEKb1ArFaadq1+/BjlNnPhdaJ2vWrBm33norc+bMYfLkyRQWFvLOO+/4K5sgCEJQnJg6GLojWp71WZq4VNTGqCCnqT+1qfE0w3BWeUa0RKEFMLRPSy4Z0BpFgVc+2UJ5lVhyEE68mxY3soYY3rWUYupgSJIkieiaUa3KbStQFCXIieqmzoVWfn4+7733HhMnTuSyyy5j+fLlTJkyhS+//NKf+QRBEBqcpmYvLUcI76XlKbQMjWDaIJw8dbA87NeCNNU9tM7ljqt60DI5kpIKK3P/tzVsPzQ1RY110+ITzTDE1MFQFdljGJJai/34QZwFOcGOUyc+dx385JNPWLp0KVu3bkWn0zFq1CgeeOABLrzwQlSqeg2QCYIghATP1EFnWQGKIiNJoffe5l2f1QgaYcCJQgtFRrZUoQ7jdsveNVpNbA+tczHoNcyamsGMub+xcc9xvvs9m4szmgU7llALnjVMjXbqoCi0QpbaGEVE+iCqdv+Oeeev0GJQsCP5zOdPD88++yxarZZnn32W1atX88orrzB06FBRZAmC0GhoohPdC8Cddu/oRChRFAVrI2qEAe61ICpjJBDeDTEU2eXNrxZrP07RtnkMt47vBsD7S/aQnVsR5ERCbWgb4YiW7LQjm93//sQardAW1XsUANbMNeC0BzmN73yuju655x7+9a9/MWHCBCIiIgKRSRAEIagktca7LiEUpw86ywvdHxJUGnQpacGO4zeNocW7q7oCFBkkFeqI8B2VC5SxQ9oyqHsqTpfMvM93YnOE9zTRpsA7dbCi8YxoeRrWSBqd9wKPEJoMbbqjiUtFsVvR5e8Ndhyf+Tx18L///S/du3enffv2gcgjCIIQErRxqTjLCtwNMVp3DXacU3inDaa0QaXRBTmN/6hNMTg4Gt6FVs0Ce3VELJJKHeQ0oUeSJKZf14cDR34hr9jMez85aLF9Kyp18J4rlaTQL02iS9AShDbPpsUNNaK1amsu2/eW0alT4Irwk6cNis2KQ5unKUbJLwvQH9kGXB/sSD7xudBq3749Bw8eZNiwYYHIIwiCEBI0sanAjpBs8d6Y9s86WWPYS8tZs1mx6Dh4dlEmHTOnZPDom39wvMzB8bLgj5T8eVjNgD5OTKZgJwk9ntF92VqFbLOg0hsDdq5t+wt4a/FuFAUGZRYxPCMwo00nCi0xbTAcRPYcQcmqT9GUH8NRdARadw52pFrzudAaMWIEr776Kr///judO3fG9Jd3JUmSuOeee/wWUBAEIRhONMQIvULLeqyxFlqxQJhPHfTsodXENys+n27tEnjq9v5s3H6A5s2aodMFZ2RWAT79YR9F5Vbe/W4vs6YNECMcf6HSm1AZIpCt1TgrCtEltQ7Iecoqbby6cAuehpS/bctleEZaQM51ouOgKLTCgSYyFkOngVj3rfGurQsXPhdab7zxBgCrV69m9erVp90uCi1BEBoDjXfT4tDaS0txObHnZwONY6PikzWKES1vx0ExonU+nVrF4qqKoEuXFqddtG1IidFa/v3uRv7YkU/GpiOM6h+YQiKcaaKTsFurcZYXBaTQkmWFOZ9tobTSRmKMgaJyK1v2F1FRbSc6wv9FuKum0NJEiY6D4SLm4lsoiEsntXW3YEfxic+F1r59+wKRQxAEIaR4Ni0OtWYY9sIjKE47Kr0JbULzYMfxq0YxolXlGdEShVa4SG8Ty4ge0fy8o4L5X+2gc5s4WiaH/ybg/qSJScRekBOwdVrf/JbFln0F6DQq/jG1Dy9/son8Uge/bzvG5UPa+v18Yupg+JG0elwxqcGO4bN69WSvrKwkKysLu92Oy+XyVyZBEISg80wdlM0VyDZzkNOccGJ9VoeQ3N+rPhrHiJZYoxWOLuwaRbe2cVjtLl76eDMOp/hMc7ITnQf9X2jtP1zKh9/vAeD2q3rQKiWSXm3dI5y/bD7i9/PBiamDotASAq1Ov6XXr1/PNddcw4ABAxg/fjx//vknM2bMYPbs2f7OJwiCEBQqvQlVzaa5odQQw1toNWtc0wbh5EKrEYxoic2Kw4pKJXHvpO5ER+jIzi3n/SV7gh0ppARq02Kz1cFLCzbhkhWG9GrOZYPaANCjjQmVSiLzUCnHCqv8ek4AZ6XYrFhoGD4XWmvXruXWW2/FYDAwc+ZMlJpVi+np6Xz00Ue8//77fg8pCIIQDNrY0GuIYc09AIC+RacgJ/G/k6cOen63hBvvGi0xohV24qMNPHB9HwC++z2bDbtDa31mMHlGtBx+nDqoKApvfrGD/GIzyXFG7r2mt7cRSaRRTa8O7tGmXzb5d1RLdtiQLe7iTS0KLSHAfC60XnvtNUaNGsXHH3/M3/72N+8vw7vuuovbbruNRYsW+T2kIAhCMIRaQwzZZsFR6P7Q0ZhHtBSXAyWEpmvWluJ0IFsqAbFGK1z175rKlUPd+4S+9tlWisosQU4UGgKxafHKjUdYtfUoKpXEw1MyiDRqT7l9aO9mAPyy5Siy7L8LL55pg5LOgEov+vkLgeVzobV3716uvvpqgNNaoA4ZMoRjx475J5kgCEKQeRtihMjUQVt+FqCgjk5E0wjbh6u0eiSde48eZxhOH/TsoSWptagMgdn/Rwi8v13ehfYtY6g023ll4WZcfvyQH648mxa7KktQXM56H+/I8UrmL94BwJTR6aSnnX5hIiM9CZNBQ0GJmT0Hi+t9Tg+XpxFGVIJo5S8EnM+FVlRUFIWFZx46zsvLIyrKt049siwzb948LrroInr37s3tt9/OkSNnHybOycnhjjvuICMjg6FDhzJv3jycTvcP/dGjR+ncufMZ/6Snp/uUSxAEIdT20rLV7J9laGT7Z50snBtinLyHlvgAF760GjWzpmRg1KvZlVXM5yv2BztS0KkjY0CtAUXGWbMOsa7sDhcvLdiEze6iV8dErh5x5vcznVbNkJ7uzqo/+3H6oLPS0whDTBsUAs/nQmvUqFHMmTOHnTt3er8mSRL5+fnMnz+f4cOH+3S8N998k4ULF/L000/z2WefIcsyt912G3a7/bT7lpeXM3nyZCwWCx9++CGvvvoqy5Yt4/HHHwegWbNm/PHHH6f8WbhwIXq9nrvvvtvXb1UQhCYu1KYOWk/qONhYeQstcziOaHnWZ4lOZuGueVIkd03sBcBnP+5jd7b/RlTCkSSpTmqIUb91Wu8v2c3B3ApiInU8dGM/VKqzX5QYmdEKgNU7crE5/NMJUnQcFBqSz4XWjBkzSEhI4Nprr/UWVQ899BCjR49GkiQeeuihWh/Lbrfz3nvvMX36dIYPH056ejpz5swhPz+fH3/88bT7L168GLPZzNy5c+nWrRsZGRk888wzfPn/2bvv8KaqNw7g35vVPYFSdikIZUMZguwyBKysnyBbGYIgQ1BUFFG0skQ2CChLRaaAbFAUEBRoQXZbZBTaQumeaZt1fn+kuW3pIGmTe2/b9/M8PkJym3zbkty895zznl9+QVRUFORyOapUqcL/V6lSJSxcuBCtWrXCtGnTLP1WCSEVnGnqoC4lzirTZUorm2+EUZ5HtNwBAPr0sldo8SNa1HGwXAhoUwsBbWrBwIClP4UgTV3wAnBFwq/TKkXnwYs3n+DwuQcAgHeH+cPT1b7Y4xvXrQQvDweos3S4dNM6F7xM68yoEQYRgsWFlpubG/bs2YP58+ejbdu2eOmll9CwYUPMnj0b+/btg6en+QuAw8LCkJGRgQ4dOvC3ubq6onHjxggODi5w/MOHD+Hr65vvORo3bgwACAkJKXD8nj17cOfOHcyfP5+mcRBCLCZ38QAnVxqny1hxEXhJ6NISoU9LADgZ7Lx9Rc1iS2V56qAujTYrLm/eHtwc1Ss7IT4lCyt3/ltmu2FaQ2lHtOKTM7Fy178AgIFd66FNo6rP/RqZjEP31sZRrT+stKeWLs8aLUJsTVGSL1KpVBg6dCiGDh1aqiePiTFenahWrVq+2728vPj7nr09NjYWer0ecrkcAPjmGwkJ+Yf1NRoNVq9ejWHDhsHHx6dUOQFjG1K1WvwuWJmZmfn+X1EzSCWHFDJQDttmkLlVgT7xMdJjHsLOzlW0HFkPbgIAFJVqIEvHAN3z34+k8PuwNIdB5QQAyE5JsOp7rhA/i+xk4wdQZudcZHYp/E6kkKEs5Zg2pCnmbryEi7dicOD0Hbz8Yi3BMwiluBzM0R0AkJXwxOLXpsHAsOTHEKSptfCt4Yoh3XzMfo20b1IZu36/gyvhsXgcmwR3ZzuLnvtZ2pwROX0xr9PCcohBChmkkkMKGfJijJk1iFOiQmv//v3w9PRE165dER4ejvfffx/R0dHo06cPPv/8c6hUKrMex/TDevZ4Ozs7pKQUnDbSt29frFu3DgsXLsSsWbOgVqsRFBQEhUIBrVab79ijR48iJSUFEyZMKMm3WIBWq0VoaKhVHssaIiIixI4giQyANHJIIQNAOWyRwUnuCBWAqLDr0Kgtf8u0Vg77O8FwAJBhXwlxFr4XSeH3AZiXwy5VDUcAKU+j8NgG77m2/Fk4x0ZDCeBJSia0z8kuhd+JFDIAZSNHzxauOHElBduOhkGlT4S3h3mfc6yZQUiF5VCla+AEIDXmkcWvzdM3UhEakQqVgsMr/o74779wizLUqKRCdIIG+367hg5+ljVde5ZbchxkACLikmHIfP73IYXfiRQyANLIIYUMJubUOxZ/ati8eTO+/vprTJ8+HV27dsVnn32GpKQkDBkyhC/A3n//fbMey97eODdXo9HwfwaA7OxsODg4FDjex8cHK1euxLx587B9+3Y4Ojpi2rRpuHv3boFuh/v370ePHj3g5eVl6bdYKKVSifr1xV+AnpmZiYiICPj4+BT6M6ooGaSSQwoZKIdtM6TG+EIddxdeDjK4NGokWo7EWwegAVDFzx+OZuaQwu/D0hxZinQk3z4BJzlDbQt+3tbMUFJxF7OhB1CrQRPY1So8uxR+J1LIUNZy+PkxxGVcxZXweBwMzsDCyU1hr5ILmkEIxeXIdjIg6eYR2BuyUMuC1+btiCScuRkFAJg4sAk6tahW7PGFZXg51RmbD4fhTowB4waV/H3BoMlC7PEsAMALLdpCpir6Zy2F34kUMkglhxQy5HX37l2zjrO40NqzZw8mTJiAyZMnIyoqClevXsW8efMwYsQI+Pr6Yv369WYXWqYpg7GxsahduzZ/e2xsLBo2bFjo1wQEBCAgIACxsbFwd3eHTqfDokWLUKtW7lB+cnIygoODsXr1aku/vSJxHAdHR+lsbOfg4CB6HilkkEoOKWSgHLbJoPWqCTUAlp5QosezRg7GDNA+NS4gd/FpAjsLH08Kvw9zc8g8vZAMgGWl2SSzLX8Whpx1Zc5VqkH5nOeQwu9EChnKUo5ZI9pg+jen8Tg+Az+duIvpr7cSPINQCsuh9KqJJACGtAQ4ODiYNW0qNUODNXtvgjGgR9taeLlDvRJl6NGuLn44Fo4Hj9MQl6JDnWrmT+POS6M2rqPk7Bzh7G7eGi0p/E6kkEEqOaSQASi4l3BRLG6GERUVhS5dugAAzpw5A47jEBAQAADw9fUtsFaqOH5+fnB2dsbFixf521JTU3H79m20bdu2wPEhISEYPXo0dDodvLy8oFKpcPLkSTg4OMDf358/7t9/jQtW27dvb+m3Rwgh+SjcTXtpxYqWQRsfDabJBKe0g6qK9deHSElZbYZhyM4E0xivlFPXwfLJzdkO7430B8cBv116hLP/RokdSVDynHboTJsNQ2bac49njGHVrn+RkJKFGlWcMWlQ8xI/t6uTim+e8WcpmmJQa3ciNIsLLU9PT8THGxcSnjlzBr6+vvD2NrZADg8PR+XK5rfLVKlUGDVqFJYuXYpTp04hLCwMM2fOhLe3N3r37g29Xo+4uDhkZRlPXr6+vggPD8fixYsRGRmJ33//HUFBQZg0aRKcnZ35x719+zZq1aoFJycnS789QgjJR+lhfH/TJsWI1nEs27R/lrcvOJn1pitJkam9O9NkwaDNFjeMBUyboHJ2jsVORyJlW/P6VTC0RwMAwJo91xCTkCFyIuHIFCr+9WlOi/cj5x/g4q0YKOQyfDC6DRzsStQWgGfaU+v0lSjoDSV7L87tOEit3YkwLC60unfvjm+++Qbz5s3D2bNn0b9/fwDAli1bsGLFCvTs2dOix5s+fTpee+01zJ07F8OHD4dcLsemTZugVCrx5MkTdOrUCUePHgVgLPLWr1+Pa9euITAwEIsWLcLUqVPx9ttv53vMuLg4uLu7W/qtEUJIAQp34zpPpsmEQZ0qSoaKsH+WCadyMLbUR9ka1dKnJwEAFNTavdwb3rshGvl4IjNbh69/CoFWZxA7kmBy99IqvsX7/egUbDp4CwAw9tXG8K3hVurnbtOoKpwdlEhIycKNuyVrMa+nES0iMIsvL8yZMwdfffUVgoODMWzYMIwbNw4AsHPnTnTt2hXvvvuuRY8nl8sxe/ZszJ49u8B9NWvWRHh4/s40/v7+2L17d7GP+fnnn1uUgRBCiiJTqCB38YQ+LRHa5Kf81DYhZZlGtKpXgEKL4yB3coMuNR76jBQo3Z+/144UmPbQUtC0wXJPLpfh/VGtMeOb07jzKBk/HQvF2FebiB1LEAq3ysh+/B90qUUXOlnZOiz5MQQ6vQHtGnvj1U7W2fdPqZCjc6saOPZ3BP68HIWWDSxvdsaPaFGhRQRicaFlZ2eHL774osDtBw8ehJ1d6fY2IIQQKVJ6eEOflghd0lOgRgNBn9ugzYYm9iEAwK66+J1PhSB3cucLrbLCNKJFmxVXDF4ejpj+ekss2BqMfafvosULVeDvZ50ux1KmcH3+iNaG/TcQHZeOSm72mDGsldlNA8wR0LoWjv0dgb+vP8bbg5tbPB3RNMXXtPkyIbZm8dRBAFCr1di+fTtmzpyJ8ePHY/bs2Th8+DA0Go218xFCiOhMDTG0yU8Ff27N0wjAoIfcyZ3/kFPelcWGGKYPcNQIo+Lo0Kw6+r7kAwBYvuMKklKzxA0kANPUQW0RhdbpK1H4PfgRZBzw3sjWcHWy7n5jDet4oFplJ2Rp9PjnxhOLv940oiWnES0iEIsLrcjISAQGBuLLL7/EzZs3kZGRgStXruCTTz7BkCFDkJSUZIuchBAimrwNMYTGN8KoXt+qV4alzLTgvkyNaKWZ1mjRB7iKZHz/pvCp5ork9Gws+/kKDCVs0lBWmEaC9KkFm2E8ic/Aur3XAABDezZEs3rWHzXiOI5vilGS7oN810F6nRKBWFxoLVq0CBzH4cCBA/jtt9+wc+dOnDp1Crt27UJSUhIWLlxoi5yEECIapUdOi3cRCq2KtD7LpEyOaKUb12jJXWhEqyKxU8rxweg2UCnluPpfHPadNm8T07KqqBEtrc6AJT+FIDNbhya+lTCsl+2mWHfzrwkAuPZfHBJSMs3+OkNWBpjGeDxNHSRCsbjQ+vvvv/Hee+/Bz88v3+0tWrTArFmz8Mcff1gtHCGESIGYUwezoytyoVWWRrRMzTBojVZFU6uqCyYObAYA+OlYKMIfJoqcyHZMhZZBnZpv+4Ufj4XibmQynB2UeG9Ea8jlJVqZYhbvSk5o4lsJjAGnL5u/l5lpeq/M3hkylb2t4hGSj8WvBEdHRyiVykLv8/T0hFxevvd4IYRUPKapg/q0REH3dtKrU6HLKe4qSiMMoOxNHWSMQZdG7d0rst4v1kanFtWhNzAs+eky0jO1YkeyCZm9E7icIsW03uly2FPszxnJmzGsFap42H4fue6tjdMH/7gcafb+hrRZMRGDxYXWyJEjsXLlSsTGxua7PT09HRs2bMCwYcOsFo4QQqRA5uACzs4RAKBLjn3O0dZj2j9LWak65PYVZwP2sjZ10JCZBhh0AAC5s7u4YYgoOI7D1CEt4eXpiNhENdbuuSraBue2xHFcnr204pGYmoXlO64AAF7pWBftm1YTJEfHFtWhVMjwKCYN96PNuyDDN8Kg9VlEQGb1xRwzZky+vz948AC9evWCv78/KleujJSUFFy+fBkGgwHVq1e3SVBCCBELx3FQuleF5ukDaJOfQlWlliDPWxHXZwFlb0TLtIeWzNGV32yZVDxODkp8MKo1PlxzDueuPUbLBo/wcvs6YseyOoVrZWjjIqFNjsXykxlISdfAp5orxgm4l5izgxIvNvHGuWuP8eflKNSr6f7cr8kd0aL1WUQ4Zo1oMcby/efv74/mzZtDp9MhJiYGmZmZaNy4MZo2bYqnT4Vfw0AIIbYmRkOM7ApbaBlHtAxZ6WB66U/B4tdn0ZXyCq9hHU+M6tsIALDxwA08ikkVOZH1mUa0bl4Px9X/4mCnym0IIqTuOd0Hz1yJgl5veO7xNHWQiMGsEa0ff/zR1jkIIUTS+IYYScJcTGKM8VMHK1qhJXNwBjgZwAzQZ6RK/oMR33GQ9tAiAAZ3q49r/8Xh6p04fP3TZSyd0QV2AhchtmTazy/qwSMAtTBpYDPUquoieA7/hl5wc1YhOT0b/96JQ5tGVYs9Xp9mnDpII1pESCVuC3Pv3j3s2LEDGzduxJ49e3D//n1r5iKEEEkxNcTQCdR5UJf81Lj2R66AnVf5m35UHI6TlanOg7kjWtQIgwAyGYdZw/3h7myHiCep2HTwptiRrErvYLyg4M6lo0urGujZrrYoORRyGbq2MrZ6/zPk+XtqmdZoSf3CDSlfzBrRyosxhs8++wx79uzJt9CT4zgMGjQICxYssGpAQgiRAoWHaURLmKmD/LTBqnXBKSreuh+5kzv06UlloiGGqeOgnAotksPD1R4zR/jjs43/4NjfEWj5QhW81Lzsr2FnjOHA5SQEAKisVOOd11qIupF699a1cPCv+7hw8wkyMrVwcij8vZIxBl1qzgURKrSIgCwe0fr+++/xyy+/YPr06Th16hSuX7+O33//HVOnTsXBgwexdetWG8QkhBBxKXOmDuqSY8HY89cDlFZWBdw/K6+y1HlQn27aQ4umDpJc/g29MLibcVuGVbuvIjZJLXKi0jt58SHO3MkCALhzajiobLdfljnq1XRDraou0OgMOH/9cZHHGbIywLTG3NR1kAjJ4lfI3r17MWHCBEyePBk1atSASqVCzZo18c4772DChAnYvXu3LXISQoioFG5VAJkcTK+FPmcEw5Zy12dVnP2z8ipLUwdNXQdpRIs8a1TfRnihljsyMrVY+tNls5o2SNXDmFRsPHATKQYHMMjAMT306cmiZuI4Dt1b50wfvFz09EHTtEGZoytkSjtBshEClKDQevLkCdq3b1/ofS+++CKioszfpZsQQsoKTibnu23Zevog0+ugiTGue7WvUcFHtNTSL7RojRYpilIhwwej28DRXoHQiETsOBkudqQSydbqseTHEGi0erRq6A2Fm3FUSJcaJ3IyoJt/LXAccPNeAp4mFj5qqDd1HKTRLCIwiwutGjVqIDy88DeKsLAweHrSiYYQUj7lTh+0bUMMTexDML0WMntnKDyE2QBUasrKXlrMoOczyp3p/EcK8q7khHdeawEA2H3qDq7fFb84sdSmX2/iUUwa3F3s8O7wVlDm2bRYbFU8HNC8vrGT4OkrhY9q6dKotTsRh8WFVmBgIFavXo1jx47xzTAYYzh69CjWrFmDfv36WT0kIYRIgVANMXL3z6ov6kJzMZWVNVrGqVMM4GSQO7mKHYdIVJdWNdGrXW0wBnyz/QpS0rPFjmS289cf49g/EQCAWcP94eFiz7dI16VIo2js3tq4p9afIZH5GrWZ5HYcpNbuRFgWF1pvvfUWmjdvjpkzZ6JZs2bo3LkzmjVrhvfeew9NmzbFjBkzbJGTEEJEp3QXptDKylNoVVS5I1rJouZ4Hn59lrMHOE7cxgBE2iYObIaaXs5ITM3Cip3/FloQSE1sohqrd18FAPyve320augFIHfTYqkUWh2aVYOdSo7ouAzceVRwDS1tVkzEYnF7d5VKhS1btuDMmTMIDg5GSkoK3Nzc0LZtW3Tt2tUWGQkhRBJy99KKtenzVNSNivOSO5aNZhh8x0Fan0Wew95OgQ9Gt8F7K88iJPQpDv51HwO61BM7VpH0egOWbr+MjEwtGtb2wKi+jfj7+BGtVPGnDgKAo70SHZpVw+nLUfjzchQa1sn/ejTlpI6DRGgWF1omXbt2pcKKEFKhKHIKLVuOaBmyMqCNjwYA2FfkQss0oqVOAzPowcnk4gYqAu2hRSxRt7obxr/aBOv338DWw7fQxLcS6td0FztWoX4+GY7QiEQ42ivw/qjWUMhzR2z5xkASGdECjNMHT1+Owtl/ozC+f1MoFbl59fwaLZo6SIRF8xwIIcRMpqmDhsw0GLIybPIc2U/uAWBQuHnx65QqIuN6Jw5gBhgy08WOUyT+AxztoUXM1K9jXbRv6g2dnuHrH0OgztKKHamAm/cTsefUHQDA1CEt4V3JKd/9/NRBiYxoAUCLF6rA09UOaWotQkJzGxYZNyumqYNEHFRoEUKImWR2DpA5GhseaG3UeTDLNG2wgrZ1N+FkcsgcXQBIe52WLt00okUf4Ih5OI7D9NdbobKbPR7HZ2DD/htiR8onI0uPNXtvgDGg94t10LlljQLHmEaGWLYaehtddLKUXMahq39OU4w8e2oZMtPAdBoA1N6dCI8KLUIIsYCSnz5om0Irmxph8EwjejoJF1q5a7RoRIuYz8VRhfdHtYGMA/4IicTZq4/FjgQAMBgYDlxIQlKaBrWqOuOtgU0LPU6msucvOkmlIQYABLQxFlrBt2OQpjYWV/z6LCc3cAqlaNlIxUSFFiGEWMCWe2kxxvhCqyKvzzIpCw0xcrsO0hotYpkmvpUwrLcfAOD7Q2FISBV/CuGxC4/w3+OsnI2W28JeVfRSfqm1eAcAn2qu8K3uBp2e4dxV41pX07RBuQutzyLCK3EzDEIIqYhsuZeWPi0R+vQkgJNB5e1r9ccva3L30pJuoaXPaYZBXQdJSQzt2QDX78bh5r0EfP9bHPZf/BucTLxr4I/jjOshx/RtAJ9qxe8Lp3CrAk3MfUmt0wKA7m1q4v7BFPwREom+L9Wl9VlEVFYttO7du4f58+fjhx9+sObDEkKIZPAjWjYotLIeGxefq7zqQKa0s/rjlzVS30vLoM2GIcv4wZS6DpKSkMs4vDeiNaZ/8yfS1FpExYm/3qlRLQf0alvzucdJcUQLALq2qokth24h7GESHselwz7NtFkxFVpEeFYttDQaDaKioqz5kIQQIil8i3cbTB3M3T+L1mcBeQstaY5o6XMaYXAKFWR2jiKnIWVVZXcHLJ3aAX9fvo3atevA3l68iyxajQa69CfgOO65x0pt02ITD1d7tGzohSthsfjzchR6ZVNrdyIeqxZajRo1wh9//GHNhySEEEnhNy1OiQfT68DJrfc2mtsIg9ZnAXmnDiaLG6QI/PosF0+zPpgSUhR3FzvUrWqPRr6ecHQUr2hXq9UIDTVvtF7hJq1Ni/MKaF0LV8Ji8cflSHSrTSNaRDzUDIMQQiwgd3YHp1ABzGDVDxjMoM/ZQ4saYZhIfY2WaUSL1meRikjpKs0RLQB4sak3HOwUiE1UIzMxFgCNaBFxlKjQ2r9/P86cOQMACAsLw6uvvgp/f398/PHH0Gg0Vg1ICCFSwnEyKNy9AFi3IYY2PhpMkwVOZQ9l5YL71lREUl+jpcvZrFhOmxWTCsg0dVCfngSmE79jYl72KgU6Nq8ODgwsg/a6I+KxuNDavHkzPv74Y9y+fRsA8PnnnyMpKQlDhgzB77//jlWrVlk9JCGESAk/fdCKhVaWadpgtXrgZHKrPW5ZpuBHtFLBGBM5TUG5HQfpAxypeGSOrsbRfeRedJCSgDa14MRlQc70ADgaeSaisLjQ2rNnDyZMmIDJkycjKioKV69exZQpUzBnzhy89957OHLkiC1yEkKIZChyOg9asyEGrc8qSJZTaMGggyFL/G5sz9Klm9Zo0YgWqXg4jstdpyXB6YNNfCuhrrseAKC3c7HqelpCzGVxoRUVFYUuXboAAM6cOQOO4xAQEAAA8PX1RUKC9K5qEEKINZlGtLRJ1iy0jB0HaX1WLlmebn5SnD7Ij2jRZsWkglJIeJ2WTMbhJV9jB8ckg5PIaUhFZXGh5enpifh44wLwM2fOwNfXF97exg8d4eHhqFyZFhsSQso3a++lZdBmQxP7EACNaD1Lyi3e+TVaNKJFKii+xbsEOw8CQFNv48fc6AwlktOyRU5DKiKLC63u3bvjm2++wbx583D27Fn0798fALBlyxasWLECPXv2tHpIQgiREoVH7tRBa6wd0sTcB5gBcmdPakH8DKm2eGeM5ek6SL8zUjFJddNiExcYpxwnGZxw9irt80qEZ3GhNWfOHLz00ksIDg7GsGHDMG7cOADAzp070bVrV7z77rvWzkgIIZJi7DrIgWmyYFCnlvrx+EYYtFFxAVJt8c6y1WBa4xVy6jpIKiqpblpsYhppSzY44s+QSJHTkIrI4pWBCoUCX3zxRYHbDx48CDs78XYzJ4QQocgUKshdPKFPS4A2+SlfDJRUdjQ1wiiKVKcOmjYrltk7Qaakcx+pmKS8aTEA6FON03tTmDPuRqXgUUwqanu7ipyKVCQWj2h16tQJQUFBuHHjRr7bqcgihFQkuQ0xSr9Oi2+EUYMKrWdJdepgbsdBaoRBKq7cEa14MGYQOU1BpgKwWu2aAIA/L9P0QSIsiwutwMBAnDhxAkOHDkWfPn2wfv16REdH2yIbIYRIltLDOg0x9Bkp0KXEAuBg5+1rhWTli1RHtKjjICHI2ZuKA9Nroc8o/TRqa2LMwI88t2jVEABw+nIkDAbp7clHyi+LC61PPvkEZ8+exebNm9GmTRts2bIFvXr1wqhRo7Bnzx6kpaXZIichhEiKtfbSMq3PUlauAZk9tSB+ltxRoiNaabSHFiGcXMm/BqS2TkufngIY9AAnQ+tWL8DJQYn4lCzcuCfNaY6kfLK40AKMm9R16NABQUFBOHfuHNatW4dq1aph/vz56Ny5s7UzEkKI5JimDupKuZcWbVRcPLlzTqGlltiIVs7UQRrRIhWdVFu8m/LInT2gUqnQuWUNAMAf1BSDCKhEhZaJTqfDuXPncPToUZw9exYA0KFDB6sEI4QQKVNYaY2WqdCyp46DhZLq1MHcES0qtEjFJtUW77o0Y6Fl2jIjoHUtAMDf1x8jK1snWi5SsVjcdZAxhgsXLuDIkSP47bffkJKSgubNm2P69Ono168fPDxoGgUhpPwzbVqsT0+CQZtdos5zjDG+EYZd9QZWzVdemJphMG02DJpMyFQOIicy0qfRiBYhQN4RLWkVWqaOg6ZCy8/HA9UqOeFJQgYu3HyCbjmFFyG2ZHGh1blzZyQkJKB69eoYMWIEBgwYAB8fHxtEI4QQ6ZI5OENm5whDthq65KdQValt8WPokp7AkJUBTq6Eysvyr68IZCoHcEo7MG029Bkpkim0dDmbFdOIFqnoFK7S3EtLxxdaxhE3juPQvXVN/HwyHH+ERFKhRQRhcaEVEBCA/v37o02bNrbIQwghZQLHcVC4V4Xm6QNok0pWaGXl7J+l8vYFJ7f47bjCkDu6QZcSC31GMr82TkyMGaDPKbQUVGiRCk6Zp8W7lPBrtFwq8bd1b1MLP58Mx7X/4pCQkolKbtK4cEPKL4vXaH3xxRdUZBFCCPI0xChh50F+2iDtn1Ws3L20pLFOy6BOM3YzA1fqzaoJKetyNy2W9ogWAHhXckIjH08YGHDmCm1NRGyvVM0wCCGkIlPk7KVV0oYY1AjDPFJriKFLM36Akzu50UgkqfBMa7QMmekwaDJFTpPL9Do1rdEyCWhjnDL452XqPkhsjwotQggpIVNDDG0JWrwznRbZTx8AoNbuz5M7opUsbpAcps2KaX0WIYDMzhEyO0cA0pk+yAz63IY1eUa0AKBTi+pQyGWIeJKKB4+lcfGGlF9UaBFCSAnlTh20fEQrO/YhoNdB5uDCb35MCie5ES1+Dy3qsksIkKfzoEQaYujTkwFmAGTyAtN7nR1VeLGJ8b2b9tQitkaFFiGElBC/l1ZyLJhBb9HXZkffAQDYVa8PjuOsnq08kdqIFu2hRUh+Utu0mJ826OwBTiYvcL9p+uDpK1HQ6w2CZiMVi1mTyx8/fmzRg1avXr1EYQghpCxRuFYCZHJAr4M+PanAFJXiZD8xNsKwp/2znktqzTD4KUlUaBECQHqbFvMdB4t4T/b384KrkwrJadm4+l8cWvvRrAJiG2YVWgEBARZdcQ0NDS1xIEIIKSs4mRwKtyrQJcVAmxRjWaGV0wjDjhphPJfUpg6aWrvLabNiQgBIcEQrJ8ezjTBMFHIZurSqgcPnHuCPkEgqtIjNmFVoLViwgC+0UlJSsHTpUnTo0AF9+/ZFlSpVkJycjD/++AOnT5/GRx99ZNPAhBAiJUqPqnyh5VCnqVlfo89MhzbBOFOAGmE8n1SnDipcaI0WIYD01mjltnYvvNACgO6ta+HwuQe4cOMJ1FlaONorhYpHKhCz1mgNHjwYgwYNwqBBgxAcHIyBAwfiu+++w+DBg9G5c2e8+uqrWL58OV577TUcO3bMogAGgwGrVq1C586d0bJlS7z11luIjCx6cWJERAQmTpyINm3aoEuXLli1ahV0Ol2+Y86cOYPBgwejWbNm6NmzJ7Zv325RJkIIMZfSPachhgWdB7Of3ANgXOMld3SxSa7yxDSiZchWw6DTiBsGgD7dtEar6A9xhFQkpkJLK5FCS1/IHlrPeqGWO2p6OUOjM+Dv65YtkSHEXBY3wzh//jz69u1b6H3dunXDv//+a9HjrVu3Dj///DO+/PJL7Ny5EwaDARMmTIBGU/BkmpKSgpEjRyIzMxPbtm3DsmXLcOzYMcybN48/5tKlS5g8eTK6deuGI0eOYNKkSfjqq69w9OhRy75RQggxQ25DDAsKLZo2aBGZvRMgM07AMIg8fZDpdfwURuo6SIiRqaDRpyVa3BjIFvipgy5FF1ocx/FNMf4IiRIkF6l4LC60PDw8cP369ULvu3DhAqpWNX+eq0ajwebNmzF9+nR069YNfn5+WL58OWJiYnDy5MkCx+/fvx9qtRorV65EkyZN0KZNGwQFBeGXX35BVJTxRbJ69Wr07NkT06dPR+3atTFkyBAMHDgQISEhln6rhBDyXKa9tCwa0eI3KqZpg+bgOA5yJ1cA4q/TMq3PgkwBGY1GEgIAkDu7Gy+GMAPf8U9M5kwdBICu/jUBADfuxSM2UW3zXKTisXhL+yFDhmDt2rXIyspCt27d4OHhgfj4eBw/fhw7duzAxx9/bPZjhYWFISMjAx06dOBvc3V1RePGjREcHIzAwMB8xz98+BC+vr7w9MxdgNy4cWMAQEhICCpVqoSQkBCsWrUq39ctWLDA0m+TEELMovDI2bTYzL20GGPIfmzsOEjrs8wnd3KHPi1R9EJLl1NoKVw8wHG0QwohAMBxMihcK0GX/BS6lHgo3bxEy8JyusACgPw5hZaXhyOa16+M63fjcfpKFIb2pC6wxLosLrQmT56MtLQ0bNq0CRs3bgRg/OBgb2+PGTNmYOTIkWY/VkyM8YNJtWrV8t3u5eXF3/fs7bGxsdDr9ZDLjfsiREdHAwASEhLw8OFDGAwGyOVyTJ8+HcHBwfDy8sKoUaMwZMgQS7/VfBhjUKvFv9qRmZmZ7/8VNYNUckghA+UQN4PBzjjSYshMR3pinHGaWzE59KnxxqYOMjn0rlVt+r4ihd+H1XLYOwMA1EmxQAl+Ztb6WWTFPwEAcI5uJfrdSeF3IoUMlEN6GUqbQ+biCSQ/hTouGqyyjygZAON7LMAAmRzZnBKa57xOOzariut343Eq+CFe6VCDb/4mhd+JFDJIJYcUMuTFGDOrI7vFhRbHcfjwww8xZcoUXL16FSkpKfDw8ECrVq3g6Oho0WOZflgqlSrf7XZ2dkhJKXjVsm/fvli3bh0WLlyIWbNmQa1WIygoCAqFAlqtFunp6QCAefPmYeLEiZg8eTIuXryI+fPnA0Cpii2tViuptvURERFiR5BEBkAaOaSQAaAcYmVwUzlCplHj7tVL0Lt5F5tDGRMKZwA65yoI+++eIPmk8PsASpfDUQvYAXgacRdZ8pJfLS/tz8LuYSgcAWQY5IgtxTlBCr8TKWQAKIfUMgAly+FoUMIOQMy9UGSh9FsflPRnIU+KhCsAvZ0LwsLCn3u8u8IAhZzD43g1Tp2/jhqV8n8mlcLvRAoZAGnkkEIGk2frl8JYXGiZODk5oUqVKmCMoUWLFtBoNBYXWvb29gCMa7VMfwaA7OxsODg4FDjex8cHK1euxLx587B9+3Y4Ojpi2rRpuHv3LlxcXKBUGltzDhgwAGPGjAEANGrUCA8fPsTWrVtLVWgplUrUry/+wvXMzExERETAx8en0J9RRckglRxSyEA5xM+QcK06tE/uoranI+wbNCo2R2rcVagBuPg0Qc1GjWyaSwq/D2vlSIu/hozo66jkbAfXEvzcrPWzSEu4jgwA7tXqoLaIOUpDChkoh/QylDZHWtJtZERfh6e9DG6leG8r7c8iMzwZKQDsPauihpk5Xgw34Pz1GDxKtkPPTn5WyWENUsgglRxSyJDX3bt3zTquRIXWr7/+im+++QZxcXHgOA579uzB6tWroVQq8c0335hV4QG5UwZjY2NRu3Zt/vbY2Fg0bNiw0K8JCAhAQEAAYmNj4e7uDp1Oh0WLFqFWrVrw9jZeSW7QIP8c2/r162Pfvn0l+VZ5HMdZXEjakoODg+h5pJBBKjmkkIFyiJchvZKx0JKpkwo857M5kmMfAACc6zQSLJ8Ufh+lzaFxr4wMAJwmo1TfS2l/FulZxpkT9h5eouawBilkoBzSy1DSHLrK1ZEBABnJVvkeSvqz0GSlAQDs3M1/jfZ60Qfnr8fg7xtPMWlwSygVuesvpfA7kUIGqeSQQgYAZk0bBErQdfDo0aP48MMP0b59eyxbtgwGgwEA0KtXL5w5cwbr1q0z+7H8/Pzg7OyMixcv8relpqbi9u3baNu2bYHjQ0JCMHr0aOh0Onh5eUGlUuHkyZNwcHCAv78/qlatitq1a+PatWv5vu7OnTv5CjlCCLEmRU7nQe1zOg8ygx7ZMfcBUCMMS5n20hK7GUbuHlrU2p2QvBRuxlbqulRx99IydT18XiOMvFq+UAUeLnZIU2twJcz8DrKEPI/Fhdb69esxbNgwLFmyBL179+Zv/9///odp06bhyJEjZj+WSqXCqFGjsHTpUpw6dQphYWGYOXMmvL290bt3b+j1esTFxSErKwsA4Ovri/DwcCxevBiRkZH4/fffERQUhEmTJsHZ2bhQeurUqdi1axe2b9+OyMhI7Ny5E7/88gvGjx9v6bdKCCFmUZq5l5YmLhJMmw3OzhHKStWFiFZuyJ3cAMDYSEREujRjoaWgzYoJyUeZs2mxLiUOjDHRcujM2Kz4WXK5jG/1/sflSJvkIhWTxYXWgwcP0KtXr0Lva9GiBZ4+texKwPTp0/Haa69h7ty5GD58OORyOTZt2gSlUoknT56gU6dO/GbDnp6eWL9+Pa5du4bAwEAsWrQIU6dOxdtvv80/3oABA7BgwQJs374dffv2xZYtW/DZZ59h4MCBln6rhBBiFqWHeXtp8ftnVatHrcEtpJDKiFZOoSWnzYoJyUeeU9gwbTYMmemi5dDzmxVbdjHEtHnxpVtPka7WWD0XqZgsXqNVqVIl3Lt3Dx07dixw371791CpkmX/sOVyOWbPno3Zs2cXuK9mzZoID8/fMcbf3x+7d+8u9jEHDBiAAQMGWJSDEEJKSuFuHNHSpcaD6bXg5MpCj6P9s0pO5mgc0TKo08AMenAyueAZDJosGLKNraIVLqXvqkZIeSJTqIz73WUkQ5cSB7lIG3qXZEQLAOpWd4NPNVdEPEnFX9ceo2sL8fYCI+WHxZdU+/Xrh1WrVuH48ePQaIwVP8dxuHnzJtatW4c+ffpYPSQhhEiZ3NkdnEIFMAN0KUWvT8h+fAcAFVolIXd0ATgZAAa9OlWUDKb1WZzSHpxK/K5XhEiNqbgRa50W02v56cUKC9ZomXRvbRzV+jOEpg8S67C40Hr33XfRsmVLvPvuu2jdujUAYPTo0RgyZAh8fHwwY8YMq4ckhBAp4zgOCo/iG2IYNJnQxEUBoEKrJDiZnL9CLtb0QV1aEgDjaJa5HacIqUgUedZpicG0hpKTKyFzdLX467v614CMA0IjEhGTaLvN5EnFYfHUQZVKhe+//x7nz5/HhQsXkJycDBcXF7Rr1w5du3alkw8hpEJSuntDGxdZZKGVHXMfYAbIXSpBQR3rSkTu5AZ9RopoDTGo4yAhxeMLrZx1UkIzPa/ctVKJPo9WcnNAywZeuBIei7+uPkHTatZOSCoaiwutAwcOoGvXrujYsWOBdVpxcXE4cOAA3nrrLasFJISQssA0oqVLjin0/uzonEYYNWg0q6SMLd4fiVZo8R0HnWl9FiGF4Vu8izSipU/NeY2WYNqgSffWNflCq4k3vdZJ6Vg8dXDOnDmIjCx87mpoaChWrVpV6lCEEFLWKJ+zlxY1wig9uaOpxbs4Uwf5joPUCIOQQilcTVMHxR3RsrQRRl7tm1aDg50cT5MyERlP3QdJ6Zg1ojVx4kTcu3cPAMAYwzvvvAOVSlXguISEBNoYmBBSIZn20ipqRCsrp7W7XfX6gmUqb3L30hJpjVZ67hotQkhBYm9arCtha/e87O0UeKl5dZwKjsTW3+Pw0+lTAMRaFsNgMDDIZI9FzABwYGhS2wF+fuLtj1ZWmVVovf3229izZw8AYP/+/WjcuDE8PfOfaGQyGVxdXTF48GDrpySEEInLbYYRW2CzTl1aknFvF04Gu2r1xIhXLshF3kuL9tAipHimNVr6jBQYtNmQKe0Eff7c1u6l21C830t1cfpyFPQGBo3WYI1opaMXv8C5ci8Dp0Ki0b9rA7GjlClmFVr+/v7w9/fn/z5lyhTUqlXLZqEIIaSsUbp5AeDAtFnGQkCWO+qf/cQ4bVBZuSZk1Ba8xHJHtJJFeX5+jVYprpYTUp7J7J3BKe3BtFnQpSZAVam6oM+vSzMWWvJSTB0EgAa1PfDdR11x/VYY6tevDwd7cd63M7MycffuXVEzAMDJC/ex69Q9bD0ajhYNvVHH2/KOjhWVxc0wFi5ciKNHj+K7777DF198AQC4cuUKFixYgClTpiAgIMDqIQkhROo4hRIK10rQpcYbpw965k6jzo427p9lT+uzSkXMES3GGPQ5Uwep6yAhheM4Dgq3ytDGR0GXEid8oWWFNVomTg5KuDspUMXdAY6OjqV+vJJQq4F4kTMAwMAudRFyOwr3nmRjyY8hWPZuV9gphd80viyyuBnGgQMHMGvWLCQnJ/O3ubu7o0qVKpg6dSp+//13a+YjhJAyo6i9tEwjWrQ+q3TEHNEyZKWD6YwL42nqICFF4xtiCLxOy6DTwJCzmXlppw6S/GQyDoM6eMLdWYVHMWnY9OtNsSOVGRYXWps2bcLYsWPzdRf09fXFt99+izfeeAPr1q2zakBCCCkrlO45DTHyFFqMGZBFHQetIu+IFmPCrpvQ52xWLHNwgUxRsBkUIcRIrE2L9TnrsziFCjJ7Z0GfuyJwtpfjndeaguOAY/9E4Pz1x2JHKhMsLrQePXqErl27Fnpfly5dcP/+/VKHIoSQsogf0crTeVCfFAOWrQanUEHlRV1ZS0PulLMugBlgyMwQ9Ll16ab1WTSaRUhxxNq02LQ+S+FauUSbFZPna16vEv7X3XjBcPXuq4hNVIucSPosLrSqVKmC69evF3pfWFgYPDzoJEQIqZhMLd7zTh3UxhgvPtlVqwdORnPaS4OTK/kr1UJPH8ztOEit3QkpjlibFueuz6Jpg7Y0so8fGtbxQEamFku3X4ZeL4GujBJmcaEVGBiIb7/9Fj/99BOePn0KrVaLp0+fYufOnVi9ejX69+9vi5yEECJ5pk2LdUm5I1raGJo2aE1irdPK7ThIhRYhxVGKNHXQ1Nq9tB0HSfEUchneH9kajvYKhEYk4ueT4WJHkjSLuw6+8847uH//PoKCgvDVV1/xtzPG0KdPH0ybNs2qAQkhpKwwTR3UZyTDoM0CAGhMI1rUCMMq5E5u0CZEC955kO84SCNahBTL1PFPl5oIZtALNpJvjc2KiXm8Kzlh2tCWWPxDCPacuoPm9SujxQtVxI4lSRYXWkqlEqtWrcKdO3dw+fJlpKSkwMXFBa1bt4afn58tMhJCSJkgd3CBzN4JhqwM6FPiAL0OurhHAGhEy1rEH9Gi6fGEFEfu4glwMsCggz49WbCpfHorbVZMzNOpRQ1cbR+HExceYtnPl7Hqve5wcxZ2g+qywOJCy6RBgwZo0IB2hyaEkLwU7t7QxNyDPiUW8rQkwKCH3MmNXyBOSkesvbT4NVp0tZyQYnEyORQunsY9BVPjBSt8dKm5zTCIMCYMaIrbDxIR+TQNK3b+i0/HvQiZjBqR5GVWoTVmzBh89tlnqFevHsaMGVPssRzHYdu2bVYJRwghZY3Sw4svtBTJxjUKdtXqUxcsKxGr0OK7DtIeWoQ8l8KtirHQSokDajYU5Dl1adQMQ2j2KgU+GN0G7604g5DQpzj4130M7FpP7FiSYlYzDMZYvj8X95/BQN1HCCEVl6nzoD45FvIU4z4jNG3QesSYOsgMeujTjc8np2YYhDyX0C3eDdpsGDLTAVAzDKH5VHPFhAFNAQDbjtzC3chkcQNJjFkjWj/++GOhfyaEEJKfwtR5MCUWClOhVYMKLWuRO5oKLeFGtPQZqQAzAJyML/QIIUXjG2II1HnQNG2QU9lDZucoyHOSXH06+ODqf3H4+/oTLPkpBCtmdoWjvVLsWJJgcXt3QgghRTONaOliIyBXGzvV2VWjjoPWInd2ByDsiJY+Z9qg3Mmd9kIjxAwKgVu86/k9tGizYjFwHIdpQ1qiiocDnsRn4Nt9he+3WxGZNaIVEBBg0T/cU6dOlTgQIYSUZaYW7wZ1KgBA7uENuYOzmJHKFX7qoDoVjDFBPlRRx0FCLJM7dVCoES1q7S42Z0cV3h/ZGnPWncfpy1Fo1aAKAtrUFjuW6MwqtNq1a8efzAwGA44cOQIXFxd07doVVapUQXJyMs6fP4/ExES8/vrrNg1MCCFSpnCpBMgUgEEHAFBWpYXB1mRqhsF0GjBNJjgBpgnxHQdpDy1CzGIqtLQpwqzR4i+GUCMMUTWuWwkjXm6In46F4dtfrqNhHU/UqFKxLzSaVWgtWrSI//PSpUvRvHlzbNq0CQ4ODvztWq0WkydPhlqttn5KQggpIziZHEr3KtAmPgEAKL19RU5UvsiUduBU9mCaLOgzkgVZj8F3HKRGGISYxbRGi2WrYcjKgMzeyabPZxrRokYY4nstoAGu/xeP63fjseTHECyd3hlKRcWdcm3xGq09e/bgrbfeyldkAcaNjEePHo2jR49aLRwhhJRFCndv/s/KajSiZW1CN8TQpxnX2lHHQULMI1PZQ+bgAgDQCrBOi586SCNaopPLOMwa4Q9XJxXuR6dg6+HbYkcSVYmaYaSkFH5ye/z4MezsaFdoQkjFpsxZp8U4OZSVaY66tZmmD+oEaoih46cO0hotQszFdx4UoMU7v1kxrdGShEpuDnh3WCsAwMG/7uPSrRiRE4nH4kIrICAAS5cuxfnz5/nbGGP47bffsGLFCvTr18+qAQkhpKwxNcTQu1YFp6AWt9ZmaohhEGpEi6YOEmIxITsP6tNyCi2aOigZbRt7Y0AX44yOFTv/RUJKpsiJxGHWGq285syZg7t372L8+PFQqVRwc3NDUlIS9Ho9OnbsiNmzZ9siJyGElBnOjTsjPfwS0is3ETtKuZQ7oiVMoaVLN04dpEKLEPMp3IQZ0TJoMmHIyjA+JxVakvLGK41w83487kWlYOn2ywh6uyPksorVft/iQsvV1RW7d+/GmTNnEBISgtTUVHh4eKB9+/bo0KGDLTISQkiZonDxgOdrH+NpaKjYUcolvsW7AFMHmU6b26qfug4SYjahRrRM0wZldo6Q2Tk852giJKVCjg9GtcG7y0/j5r0E7Dl1B8N6NRQ7lqAsLrQA48Zk3bp1Q7du3ZCdnQ2lUgmZjPY+JoQQYnu5hZbtR7RMo1mcXAkZ7YdGiNlyCy3bjmjldhyk9VlSVL2KM94e3ALLd1zBjhNhaFavMpr4VpzfVYmqo/v37+Pdd99Fu3bt0KpVK4SGhmL+/Pn48ccfrZ2PEEIIycc0dVCIES3T+iy5i4cgmyMTUl4oXIUd0VK40LRBqQpoUwvdW9eEgQFLt19GmlojdiTBWFxohYaG4rXXXsOtW7cQGBgIxhgAQC6XY8GCBdi/f7/VQxJCCCEmgo5omVq707RBQiyizBnR0qcngem1NnsevanQohEtSXt7cHNUr+yE+ORMrNr1L18/lHcWF1qLFy9G06ZNcezYMXz88cf8D2ru3Ll47bXX8MMPP1g9JCGEEGKSO6Jl+0KL72ZGjTAIsYjM0RWcQgWA8aNOtpC7hxaNaEmZo70Ss0e3gULO4cLNGBz9O0LsSIKwuNC6evUq3nzzTSgUigLTKPr164eIiAhrZSOEEEIKUOSMaDFNJgzabJs+l2mNFm1WTIhlOI7L3UvLhtMHdWm0WXFZUb+mO94MNHbj3XTwJh48FqZzrJgsLrTs7OyQlZVV6H3JyclQqVSlDkUIIYQUhbNzBOTGXk62HtXS52xWrKDNigmxmBAt3k2jZdQMo2zo39kXbRpVhVZnwNc/hSArWyd2JJuyuNDq2LEjVq1ahZiY3F2eOY5DRkYGNm/ejJdeesmqAQkhhJC8OI4TrCGGLs3UDINGtAixlBANMfhmGDR1sEzgOA7vDmsFT1d7RD5Nx8YDN8SOZFMWF1qzZ8+GWq1Gnz59MHLkSHAch0WLFqFPnz548uQJZs2aZYuchBBCCE8hUEMMU9dBWqNFiOX4ES0btXg3ZGWAaTKNz+VCI1plhZuzHd4b6Q+OA3679Ah//RstdiSbsbjQqlatGn799Ve88cYbYIyhdu3aUKvVCAwMxL59+1CrVi1b5CSEEEJ4QjXEoK6DhJQcv5dWqm1GtPjNiu2dIVPZ2+Q5iG00r18FQ3s0AACs2XsVMQkZIieyDYs3LF63bh1efvllzJw50xZ5CCGEkOfKbfGebLPnMGRn5rlaTmu0CLGUrTct1qVRa/eybHjvhrh+Nx6hEYn4+qcQLJ7aGQp5ibb4lSyLv5sNGzYgKirKFlkIIYQQswixl5YuZ9ogZ+cImcrBZs9DSHnFdx1MjbfJvknU2r1sk8tleH9Uazg5KHHnUTJ+OhYqdiSrs7jQql+/Ph48eGCLLIQQQohZhGiGQR0HCSkd40gTB6bT2OSiiKnQktP6rDLLy8MRM15vCQD45c+7uBIeK24gK7N46mD37t2xbNky/PXXX2jYsCEcHR3z3c9xHN555x2rBSSEEEKexY9oqW04okUdBwkpFU6uhNzZA/r0ROhS46Fwdrfq4+d2HKRCqyzr0Kw6+r7kg2N/R2D5jitY9V43eLiUjzV3Fhdaa9asAQCcP38e58+fL3A/FVqEEEJsTYhmGPqczYqp4yAhJadwq2IstFLigOr1rfrY+jRq7V5ejO/fFKEPEhHxJBXLf76Cz9/qAJmMEztWqVlcaIWFhdkiByGEEGI2IZph8CNaNHWQkBJTuFVGdnS4TToP5q7RohGtss5OKcfsUa0xc8VZ/HsnDvtP38X/Al4QO1aplaq1R0REBK5du0bNMQghhAhK7mgstAyZ6WB6nU2eg/bQIqT0cjsPWrfQYozR1MFypra3KyYObAYA+PFYKMIfJoqcqPRKVGjt3LkTXbp0Qd++fTFs2DD06tULPXr0wNGjR62djxBCCClA5ugCcMZTmK2mD9IaLUJKj+88aOUW74asDDBtNgBqhlGe9H6xNjq1qA69geHrny4jI1MrdqRSsXjq4I4dOzB//nz06NEDvXv3RqVKlRAfH4/jx4/jvffeg0qlQs+ePW2RlRBCCAEAcJwMckdX6DOSoVen2OSKtj5ns2IFbVZMSInZakTLNG1Q5ugKmdLOqo9NxMNxHKYOaYk7kcl4mqjG2r3XMHtUa7FjlZjFI1pbt27FiBEjsHbtWgwYMACdOnXCwIEDsX79egwZMgRr1661RU5CCCEkH1s2xGCM8ftoyWmzYkJKTGkqtFKtO6KlN00bpNGscsfJQYnZo1pDLuPw19Vo/HbpkdiRSsziQismJgY9evQo9L6XX34Z9+/fL3UoQggh5HnkzrZriGHITANy1n7RPlqElJxp6qAhMw0GTZbVHpcaYZRvfnU8MapvIwDAhv03EBWbLnKikrG40GrWrBn++uuvQu/7999/0bBhw1KHIoQQQp7H1BDDJhuh5qzPkjm6gpMrrf74hFQUMnsnyOyMe65ac/qgjlq7l3uDu9VHywZVoNHqsXL3DWj1TOxIFrN4jdbkyZMxa9YsZGRkYMCAAahatSqSkpJw6tQpbNmyBR9//DGCg4P549u2bWvVwIQQQgiQd+pgstUfm99Di9ZnEVJqCrfK0MQ+gi41HqoqtazymNRxsPyTyTjMGu6P6d+cxqOn6Th5haF5U7FTWcbiQmv8+PEAgD179mDv3r387YwZq8z58+fzf+c4DqGhodbISQghhOSTu5eWLUa0jB/iaH0WIaWncK1iLLSsOaKVM3VQTiNa5ZqHqz1mDvfHZ9/9g+D/MnA3KgXNGziKHctsFhdaP/zwgy1yEEIIIRaxZTMMvuMgLbQnpNRs0XmQX6NFr9Fyz9/PC8N71cfhcw9gr5KLHcciFhda7dq1s0UOQgghxCK5I1rJVn9svuMgNcIgpNQUVu48yBiDPmcdJU0drBgGdqmLhlWyUNPLWewoFjGrGcaoUaMQFhZm0QPfuHEDw4cPf+5xBoMBq1atQufOndGyZUu89dZbiIyMLPL4iIgITJw4EW3atEGXLl2watUq6HQ6/n69Xo/mzZujYcOG+f5bvXq1RfkJIYRImy2nDvIf4mizYkJKLXfTYuuMaBky08B0GuNj04gWkTCzRrRGjx6N8ePHo3nz5nj11VfRvXt3ODg4FDguPT0df/31F3bt2oXQ0FB89tlnz33sdevW4eeff8aiRYvg7e2Nr7/+GhMmTMChQ4egUqnyHZuSkoKRI0fC19cX27ZtQ2ZmJj799FPExMRgwYIFAIyFWHZ2Nn799VdUqpT74nN0LDvzOQkhhDwfP3VQnQpm0IOTWW9KiS5n6qCcCi1CSs3aUwf59VlO7uAU1BWUSJdZhdbLL7+Mtm3bYt26dfjkk0+g0+lQv3591KxZEw4ODkhNTUVMTAz+++8/KBQKDBkyBEuXLkXlysUvUNRoNNi8eTPef/99dOvWDQCwfPlydO7cGSdPnkRgYGC+4/fv3w+1Wo2VK1fC09N48gsKCsKIESMwZcoU1KxZE+Hh4XB2doafn18JfhyEEELKCrmjq/EPzABDZjo/wmUN+pypg9R1kJDS4wuttESrXBQxdRyU02gWkTiz12h5enpi7ty5mDJlCk6ePImLFy8iMjISaWlp8PDwQL169TBmzBh0794dHh7mzWkPCwtDRkYGOnTowN/m6uqKxo0bIzg4uECh9fDhQ/j6+vJFFgA0btwYABASEsIXWvXq1TP32zIfY9BnFb7RHieTQZZn9K2o44wHc5Db2ZXs2Oxs6LOywDQa6LOyoJfJij0WrIj9Biw5FoDc3t68DM8ca9BowAwGsx73ecfK7OzAcZzxWK0WTK8vMkdhxxb5uCoVuJyvLcmxRWZQKsHJ5eY9bt5jdTqwPFNhzT22sBx5j2V6PQxabZGPyykUkCkUpT722RycXA6ZUmne4+Y91mCAQaMp0bHFZmAMhuxs8x73ecc+53WfN4dBoRDsPSLvaznfz0IuF+w94tlj8+ZAntkFVnmPUDnDkJkOXXoyX2gV9prLm4E5OBT7HsEMemhTkwEGyPIUb9Z4PzHlyPu9CPEeUWiGPM8p1HtEgRyWZLDSe8Sz8j6nkO8R+TI8k0+o94hnj82XwZrvEc7ugEwBGHTQJsSYXSAV9brXxMfAoM+/PqsknyOKO5Z/XK222J9xaT9HFHmsMnekzqDTFZ/Byu8RhR3LDIZCP/eZ2OI9Aij4Wi42g43eIwp93TNW4DVTGIubYXh6emLYsGEYNmyYpV9aQExMDACgWrVq+W738vLi73v29tjYWOj1eshzfvHR0dEAgIQE49WNO3fuQKfTYfz48QgLC0PVqlXxxhtvYMCAAaXKmhUbiwufzi/0PteWLVDvw/f5v197czwM2YX/wpwb+eGFeZ/wf78xcQp0aWmFHuvoWxcNv/qC//utaTOhiTcOl19/5lj7GjXQaOki/u+h73+ErJyfzbNUlSujyerl/N/DP5kH9f0HhR6rcHFBs43r+L//98VXSA8NKzSDzE6FFls38X+/t3gpUq9eK/RxAaDVjh/5Pz9YsQrJF4OLPLb5lu/4D10Pv92AxLPn+PuezdF0w1ooXY1XuiM3b0X8b6eKfNzGq5bBrorxSlv09h2IPXy0yGP9liyEQ62aAIAne/ch5pf9RWZoEDQfTvV8AQBPDx3B4593Fvm49T/9GC6Njbufx538DVFbiu7s6Tv7Pbj5twQAJJw5i0frv8t3f94cPjOmwqP9iwCApAsXEbFyTZGPW/vtt1CpaxcAQMqVq7j/9TdFHltz7BhU6d0LAJB2OxR3v1xQ4BhTjuojhqHqq68AADLu3ceduUVPJ/b+3yBUe20wACAzMgphH8wp8livwH6oMdK4BjQ7Lg63p88qMkPlXj1Qa9ybAABtaipuTnqnyMf17NIJdSZPAmD88HJ97FtFHuv+YlvUfXc6//d/h48u9LjrEOc94tkMYrxHPOuGSoWW22zzHuHxNAp6F+Nr+dn3iLyuw7L3CM/EVNgrjFPlS/Me8aykTz8Gl/O6F/I9Iq/YyRNRtUtnYx6B3yNMFD0DkJlzcVTo9wgT9+7dgM4vITMzU5T3CABwat4UGNgfmZmZAIR/jwCM7xE+Xxp//pmZmQj9dL5V3yPkLp7Qp8Ti1oKlUN9/VOixMjsVGny7hs9wb9XaYt8jnFu7Qa1WAyjd54hnNd2wFrqcD9YPt/yA5D9PF3mstT5HPKtB0HzIqhs/Hz8+eBhxe34p8lhbvUeYPkdkZmbCEBaO60GLijzWVu8Rps8RmZmZYE9iis1gq/eIwj5HqKZNhn3VqkV+jYnFhZY1md5Qnl2LZWdnh5SUgoub+/bti3Xr1mHhwoWYNWsW1Go1goKCoFAooM2phv/77z8YDAZMnz4d3t7eOHPmDObMmQOtVovXXnvNJt9Henp6vv3CDIair+pkqNX5jtXpi76SkJmVle9YTTEVf7YmO9+x2Zqir7JptNr8xxZzlUSn1+U/NucNrTAGA8ufNz29yGMB5D82tfCThEl4eDi4nH8nmkL+beT135074JycAADapKRij7179y5kOScdbU6xXpT79+9Dlm7MqY0rfp55xIMHkOX8DnSxscUe+/DhQ8hzLoroCrnAkFdkZCQeOxivtOkePyn22OjoaMTk/Iz1RZwsTR4/foJY07HFNKMBjBdI4k3HPnxY7LGxsbFIzDnWEP242GPj4uKQbDo2tvifb0JCAlJNxyYnF3tsUlIS0nOOZRkZxR6bnJICtenYYq5uAUBqaprZ+wTSe4QRY7Z7j4i6Gw6d1vghyprvEffu3YcswTiN0JrvEU8eP8ZT0+tepPeIuLg4/vUp1nsEYFxbDYj3HpGWlgplTg6x3iMy1ZlQIc/PQqT3CNPzR0REWP09wlluDyWAbHXRP2ODgeXL8Lz3iHi1DtE5OWz1OSItLbXYY4X4HJH0nPcpIT5HPI8QnyOeRwqfI57FMVbMWK+NnThxAtOnT8e1a9dgn2eKyIwZM6DRaPDtt98W+Jo//vgD8+bNQ0JCAhwdHTFt2jR8//33mDx5MkaOHImsrCzo9Xo45bxAAOCzzz7DpUuXcOzYsRLlvHHjBpjBgLq1Ct/N3JLpAaU51pCdjczMTDx8+BB16tTJ9zPjOC7/UHd2Nor61RY41sLpO5lqdaEZCjvWllMHs7KyCs1R4qmDzxtCL+TYIjOU8HGZTgeDucP4eY4tLIclUwdlCgU4M4fx8x37zHD7sznyTQ943tC8lY4tkEGkqYN5czg4Ogr2HpH3dZ8vg4ODYO8Rzx6bN4eTu7vVHhcAUk5sQNZ/IXDtPhzOrfsZjy3kdZ83g6ObW7HvEdkPriL5yFooq/qg8sj5pXqPeJYph0/9+nDMOU8J8R5RaIZ69eDobGyVLNR7RIEcUVGoW68eHBwcBHuPKJBDo8Gj6Gj4+PjA3t5elKmDeTM4ODgI9h6R71iOQ7bBgIiICPj4+MBOJrPqe0TyiQ3Iun0Oji8OhpN/3yKP1TCWm0EuL/RxE/ctgfbxf/AInAIHvw5mZbB06mBWVhYiIiJQu0YN2D8zIJDv2FJ+jiju2KxsY/Fbp2ZN2OWZSljc41rjPeLZYzMzM/Hg/n3UqVGjwOc+/lgbvEcAua9lczLY7DNHIa/7B5GR4GQyNGvWrMivA0Qe0TJNGYyNjUXt2rX522NjY9GwYcNCvyYgIAABAQGIjY2Fu7s7dDodFi1ahFo5RVBhP/wGDRrg4MGDpcrKyWRwybM2rFiWdDi08FiZnR24J0/g5O5efCdFW2ZQqcTNkEOuVpuXw4akkIFyWJghz0WY57Lk2Geeq9gcNnx9Si1DsTmskCG7khc09wG5NrPYf3OW/Ns03NdAJgfsKlWFk7N192wx5XB0chL9NeLo7CyJ9wsHBwfRcyA6OjeHQO8RxWYQ8PWZlyxnRNri34kZx2Z5eiMLgFyTXuznKbUZGZKyUyGTA05VqsPedIyV/w2ZijInV1fR/n2aiidHFxfRu2dzMpnonzGkkAEA4OQE7jmzAEzM2kfLVvz8/ODs7IyLFy/yt6WmpuL27dto27ZtgeNDQkIwevRo6HQ6eHl5QaVS4eTJk3BwcIC/vz9SU1PRrl077Nu3L9/X3bhxAy+88ILNvx9CCCHC4lu8ZxQ/vccSuR0HabNiQqzFWi3eGTNAl5bTdZA2KyYSJ+qIlkqlwqhRo7B06VJ4enqiRo0a+Prrr+Ht7Y3evXtDr9cjMTERLi4usLe3h6+vL8LDw7F48WKMGTMG4eHhCAoKwqRJk+Ccc9Wxffv2WL58OSpVqoQ6derg5MmTOHjwIDZs2CDmt0oIIcQGcjctTrbaY9IeWoRYn8I1p9BKLV2hpc9IBfQ6ABxtv0Akr0SFVlpaGi5cuAC1Wl3o/N2BAwea/VjTp0+HTqfD3LlzkZWVhbZt22LTpk1QKpWIiopCjx49sHDhQgwePBienp5Yv349Fi1ahMDAQFSpUgVTp07Fm2++yT/eggULsHr1anz22WdISEhAvXr1sGrVKnTu3Lkk3yohhBAJkzuaCq3iF7dbQp9uvFquoEKLEKvJHdGKB2OMn5pnKb1pNMvZHZxc1PECQp7L4n+hf/31F6ZPn46srKxCiyyO4ywqtORyOWbPno3Zs2cXuM+0L1Ze/v7+2L17d5GP5+zsjDlz5mDOnKJbOhJCCCkfcqcOJlvtMfkRLbpaTojVmPa8YtosGLLSIXdwKdHj6FLjcx6vstWyEWIrFhda33zzDXx9fTFnzhxUrVoVsiI2DSOEEEJsTe6cM6KlTinVVfK8+DVaLrRGixBrkSntIHdygz4jBbqUuFIUWjkjzrQ+i5QBFhda9+7dw7p169CmTRtb5CGEEELMZhrRgl4HQ7YacnsLusAVwqDNhiHTuHcPjWgRYl0K1yp8oWXn7VuixzCNaMldqNAi0mfxcFT16tWR/pwN5AghhBAhyBQqcHbGVr/WmD6oTzdOG+QUKshKWbQRQvJTuBmn+5mKpZIwdRykqYOkLLC40Jo0aRLWrl2LqKgoW+QhhBBCLCJ3dAVg3UJL7uJplWmIhJBc1mjxrqepg6QMsXjq4KFDh/D06VP06tULnp6eBTYI5jgOv//+u9UCEkIIIcWRO7lDlxRjlc6DujTaQ4sQWzGNQulSSjGiRc0wSBlicaHl7e0Nb29vW2QhhBBCLMbvpZWeXOrHMhVatIcWIdZX2hEtZtDnXgyhES1SBlhcaC1cuNAWOQghhJASUZhavKtLP6KV23GQCi1CrC130+KSjWjpM1IBgx7gZJDTqDMpA0q809vZs2dx6dIlpKamwsPDA23atKFNgQkhhAgudy8t600dpI6DhFifaURLn5EMg04DmUJl0dfzHQedPcDJ5FbPR4i1WVxoaTQaTJkyBefOnYNcLoeHhweSkpKwceNGtG/fHhs2bIBKZdkLhxBCCCkpuZMVm2HkbFZMI1qEWJ/MwRmc0h5MmwV9ajxkntUt+npdGq3PImWLxV0HV69ejcuXL2PJkiW4fv06zp07h2vXrmHhwoW4evUqvv32W1vkJIQQQgplzREt09RBOW1WTIjVcRzHt3jXlmCdVm7HQboQQsoGiwutw4cPY+rUqejfvz/kcuOwrUKhwMCBAzF16lQcOnTI6iEJIYSQouQWWsmlehzGGHSmES2aOkiITfDrtEpQaOlSaQ8tUrZYXGglJiaicePGhd7XuHFjPH36tNShCCGEEHPxXQczUkv1OCxbDabNMj4mTR0kxCb4TYtL0OKdWruTssbiQqt27dq4fPlyofcFBwejWrVqpQ5FCCGEmMs0osW0WTBoskr8OLqczYpl9k6QKe2sEY0Q8gy+xXtqyUe05C7U2p2UDRY3wxg2bBgWLVoEe3t7vPLKK6hcuTLi4+Nx+PBhfPfdd5g6daotchJCCCGF4lT24BQqMJ0G+oxkyFQl2+tRz3ccpPVZhNhKaTYtzh3RokKLlA0WF1rDhw/H7du3sXTpUnzzzTf87YwxDBo0CBMnTrRqQEIIIaQ4HMdB7uQGXUoc9BkpUHqUrNDiN0Klq+WE2ExJNy1mBj30OaPONHWQlBUWF1oymQxfffUVxo4dy++j5ebmhnbt2qFevXq2yEgIIYQUS+7knlNoJZf4MajjICG2x6/RSk0AYwZwnHmrWPTpyQAzADI5vy6TEKkr8YbF9evXR/369a2ZhRBCCCmR3IYYJW/xTh0HCbE9hUslgJMBBh306clm71nHTxukzYpJGWJWodWjRw+sXbsWfn5+CAgIAMdxRR7LcRx+//13qwUkhBBCnscae2np0nIW2tMaLUJshpPJoXDxhC41HrrUePMLLdPrk6YNkjLErEKrXbt2cHJy4v9cXKFFCCGECC13RCu5xI/Br/+gNVqE2JTCrYqx0EqJA2o0MOtrqBEGKYvMKrQWLlzI/3nRokXFHqvX60uXiBBCCLGQNaYO8l0HaY0WITalcKsCRIZa1BCDNismZZHF+2j16NEDYWFhhd53/fp1vPTSS6UORQghhFgid+pgcom+njEDv4+WuVOZCCElk9vi3fxCS08jWqQMMmtE6/Dhw9DpdACA6OhonDx5stBi659//oFWq7VuQkIIIeQ5+BEtdclGtAzqNMCgB8DxRRshxDZK0uKdH9FyoREtUnaYVWjduHED27ZtA2BsdrFu3boijx07dqx1khFCCCFmKm0zDNMeWnInN3DyEjfkJYSYgR/RSjV/0+LcqYM0okXKDrPOJu+99x7GjBkDxhh69uyJNWvWoFGjRvmOkcvlcHZ2hrOzs02CEkIIIUUxjWgZsjLAdFpwCqVFX8+vz6KOg4TYnKUjWkyv45vVUNdBUpaYVWipVCrUqFEDAHDq1Cl4eXlBqbTsJEYIIYTYiszeCZDJAYMeenWKxQvmdTmbFdP6LEJsz7RpsSFbDUNWhvH1Wwzj65MBMgXkTq4CJCTEOiyeH1GjRg1cv34dFy9ehEajAWMMAMAYg1qtxuXLl7F7926rByWEEEKKwnEyyB3doE9PNG6CamGhpc/ZrFhOhRYhNidTOUDm4AxDZjp0qfFQPafQ0qfmXAhx9QTHWdzHjRDRWFxobd++HUFBQXyBlZdMJkOnTp2sEowQQgixhNwpp9AqQUMMfkTLmQotQoSgcK0CTWY6dCnxUHnVKfbY3D20aNogKVssvizw008/oUuXLrh48SLGjRuHoUOH4urVq1i5ciXs7OzQv39/W+QkhBBCilWahhi5e2hRoUWIEEzTB7VmrNOiQouUVRYXWlFRURgxYgTc3NzQtGlTXL58Gfb29nj55ZcxceJE/PDDD7bISQghhBRL7mzatDjZ4q81dR1U0GbFhAiCb4iRak6hZew4SBdCSFljcaGlVCphb28PAKhTpw4ePnzI753VunVrREREWDUgIYQQYg65o7HQ0pVkRMvU0YymDhIiCIWr+Z0HdWmm1u40okXKFosLrUaNGuHPP/8EANStWxcGgwHXrl0DAMTExFg3HSGEEGKm3KmDyRZ9HdPr+OmG1HWQEGHktnh//l5aepo6SMooi5thjB07FlOnTkVqaioWLFiAHj164IMPPkDv3r1x6NAhtG7d2hY5CSGEkGLxe2lZOKJlLMyMraNlji7WD0YIKSB302Lzpw7SZsWkrLF4RKtnz55Yv3496tWrBwD44osv4OPjg507d8LX1xeffvqp1UMSQgghz2Ma0bJ06iC/PsvZnVpHEyIQ04iWPi0JTK8t8jim0/Kj1AoXKrRI2WLxiBYAdOvWDd26dQMAeHh4YPPmzdbMRAghhFjMNKJl6dRB6jhIiPDkTq7g5EowvRa6tEQo3asWepxp6wVOoYLMkTYrJmWLWYVWcHCwRQ/atm3bEoUhhBBCSoqfOqhOAzPowcnkZn1dbsdBKrQIEQrHyaBwqwxt4hPoUuKKLrRy1mfJXTzBcZyQEQkpNbMKrdGjRxf4x80Y428z/dn0/9DQUOsnJYQQQoohd3QFwAFg0KtToXA2r1U7dRwkRBwKtyp8oVUUWp9FyjKzCi3aG4sQQojUcTI5ZI4uMKhToc9IMbvQoj20CBEH3xCjmM6D1HGQlGVmFVrt2rUr8r7s7GyoVCoaziWEECI6uZM7X2iZS59Oa7QIEUNui3czRrSoEQYpg0rUXun+/ft499130a5dO7Rq1Qq3b9/G/Pnz8eOPP1o7HyGEEGI2RQkaYuR2HaRCixAh5bZ4L3pEK3fqII1okbLH4kIrNDQUr732Gm7duoVXX30VjDEAgFwux4IFC7B//36rhySEEELMIeMLLQtGtKjrICGiMG9EK6cZBq3RImWQxe3dFy9ejKZNm/It3bdv3w4AmDt3LrKzs/HDDz9g0KBB1k1JCCGEmMG0l5a5I1oGTRYM2WoA1HWQEKHxhVZqfL4ma3np0mhEi5RdFo9oXb16FW+++SYUCkWBF0S/fv0QERFhrWyEEEKIRRQWjmiZOg5ySntwKgeb5SKEFGRcd8WB6TQwqFML3G/Iczt1HSRlkcWFlp2dHbKysgq9Lzk5GSqVqtShCCGEkJLIHdEyr9AybYaqcPGgpk6ECIxTKCF3dgdQ+PRBfc76LE5pB5m9s5DRCLEKiwutjh07YtWqVYiJieFv4zgOGRkZ2Lx5M1566SWrBiSEEELMJbewGQatzyJEXKbpg9rUgoWWaX2WwqUSXQghZZLFa7Rmz56N119/HX369IGfnx84jsOiRYvw4MEDMMawbNkyW+QkhBBCnkvuaNnUQV2aceogdRwkRBwKtyrIjr5T6IhW7vosmjZIyiaLR7SqVauGX3/9FW+88QYYY6hduzbUajUCAwOxb98+1KpVyxY5CSGEkOcyTUPSq1PAmOG5x+fuoUWbFRMihuI2LTa1dpdTIwxSRlk8orVu3Tq8/PLLmDlzpi3yEEIIISVmGtGCQQ9DZgbkji7FHs/voUWboRIiiuJavPNTB2lEi5RRFo9obdiwAVFRUbbIQgghhJQKp1BCZu8EwDiq9Tz8Gi1nGtEiRAzFbVpsaoZBF0JIWWVxoVW/fn08ePDAFlkIIYSQUrOkIYYup7077aFFiDiKH9GiPbRI2Wbx1MHu3btj2bJl+Ouvv9CwYUM4Ojrmu5/jOLzzzjtWC0gIIYRYQu7oBm3C4+c2xGCM0YgWISIzFVqGzDQYNPm3D9KlmaYOUqFFyiaLC601a9YAAM6fP4/z588XuJ8KLUIIIWLK3UsrudjjDFkZYDqN8WtoRIsQUcjtncDZOYJlq43TBx2Nr0WmzYYhMx0ArdEiZZfFhdbt27chk1k845AQQggRBD91MD252ONMHQdlDs6QKVS2jkUIKYLCtTK0cY+M0wdzCi3TaDOnsgdn51jclxMiWRZXTP3798eff/5piyyEEEJIqfEjWurUYo/T8dMGaTSLEDEpC1mnZboQonCtTJsVkzLL4kLryZMncHBwsEUWQgghpNTMbYah51u7U6FFiJgKa4ihp82KSTlgcaH16quvYuvWrYiNjbVKAIPBgFWrVqFz585o2bIl3nrrLURGRhZ5fEREBCZOnIg2bdqgS5cuWLVqFXQ6XaHHJiYmolOnTli9erVVshJCCJG+3EKr+GYYpo6DNKJFiLgKa/FuoD3uSDlg8RqtiIgIhISEoGvXrnB3dy+06+Dvv/9u9uOtW7cOP//8MxYtWgRvb298/fXXmDBhAg4dOgSVKv+c+ZSUFIwcORK+vr7Ytm0bMjMz8emnnyImJgYLFiwo8Nhz585FXFzBdqGEEELKL3ObYeSOaFHHQULEVNyIlpw6DpIyzOJCq1q1anj11Vet8uQajQabN2/G+++/j27dugEAli9fjs6dO+PkyZMIDAzMd/z+/fuhVquxcuVKeHoar0AGBQVhxIgRmDJlCmrWrMkfu2vXLkRERKBKlSpWyUoIIaRsyDuixRgrcn2HjqYOEiIJCreCI1q5a7RoRIuUXRYXWgsXLrTak4eFhSEjIwMdOnTgb3N1dUXjxo0RHBxcoNB6+PAhfH19+SILABo3bgwACAkJ4QutBw8eYOnSpdi6dSumTZtmlayMMajVaqs8VmlkZmbm+39FzSCVHFLIQDmkl0EqOaSQQYwcBplxNgTTaZCRkgiZyqHQDNqcD3U6lZNg7+9S+J1IIQPlkF4GMXPoVc4AjBsUqzPSc/5sfH3q7VxE+fwlhd+JFDJIJYcUMuRV3EW8vCwutEzOnj2LS5cuITU1FR4eHmjTpg06d+5s0WPExMQAMI6S5eXl5cXf9+ztsbGx0Ov1kMvlAIDo6GgAQEKCcYhZq9Xivffew/jx49GkSROLv6+iaLVahIaGWu3xSisiIkLsCJLIAEgjhxQyAJRDahkAaeSQQgZA2BzuciU4vRb/Xb8Cg1Puxbm8GdyS4yAD8CguGXqNsO/vUvidSCEDQDmklgEQIQczwJ2TgWMGPAq/BTi4QpeaABmAiNgkGNTiff6Swu9EChkAaeSQQgaTZ5c4FcbiQkuj0WDKlCk4d+4c5HI5PDw8kJSUhI0bN6J9+/bYsGGDWU8M5Falzx5vZ2eHlJSCi5j79u2LdevWYeHChZg1axbUajWCgoKgUCig1WoBAKtWrYKdnR3eeustS7+1YimVStSvX9+qj1kSmZmZiIiIgI+Pj2jdH6WQQSo5pJCBckgvg1RySCGDWDni/naHPjUOdatVgapGgwIZmMGApycyAAD1mvpD7izMOi0p/E6kkIFySC+D2Dni/qkEfWocarg7ICojGzJdNgDghRZtIVMJ/zORwu9EChmkkkMKGfK6e/euWcdZXGitXr0aly9fxpIlS/DKK69ALpdDp9Ph8OHDmD9/Pr799lvMmDHDrMeyt7cHYCzeTH8GgOzs7EJ/iD4+Pli5ciXmzZuH7du3w9HREdOmTcPdu3fh4uKCS5cuYceOHdi/fz8/4mUtHMcVaPwhJgcHB9HzSCGDVHJIIQPlkF4GqeSQQgahcyhcPKBPjYNSn5XvOU0ZdOlJADMAnAzOlb3Byax7zngeKfxOpJCBckgvg1g5lO5e0KfGQaFJhyxLDwCQ2TnC2V3cNVpS+J1IIYNUckghAwCz93azuNA6fPgwpk6div79++c+iEKBgQMHIiEhATt27DC70DJNGYyNjUXt2rX522NjY9GwYcNCvyYgIAABAQGIjY2Fu7s7dDodFi1ahFq1avHNMvJmy8zMxIYNG3D8+HEcOXLE0m+XEEJIGfS8vbT0aUn8cUIXWYSQgkydB/Wp8ZBlG1+T1HGQlHUWF1qJiYl8A4pnNW7cGE+fPjX7sfz8/ODs7IyLFy/yhVZqaipu376NUaNGFTg+JCQEK1euxJYtW+Dl5QUAOHr0KBwcHODv748mTZrg7bffzvc1o0ePRu/evTF27FizcxFCCCnbclu8pxZ6v860GSp1HCREEkydB/WpCZAx4ywn2kOLlHUWF1q1a9fG5cuX83UKNAkODi7Q2KI4KpUKo0aNwtKlS+Hp6YkaNWrg66+/hre3N3r37g29Xo/ExES4uLjA3t4evr6+CA8Px+LFizFmzBiEh4cjKCgIkyZNgrOzM5ydnVGpUv4XpUKhgJubG2rUqGHpt0oIIaSMeu6IFm1WTIikKFxzRrTS4iFTuOXcRoUWKdssLrSGDRuGRYsWwd7eHq+88goqV66M+Ph4HD58GN999x2mTp1q0eNNnz4dOp0Oc+fORVZWFtq2bYtNmzZBqVQiKioKPXr0wMKFCzF48GB4enpi/fr1WLRoEQIDA1GlShVMnToVb775pqXfBiGEkHJM7mj8oKYrotAy7aElp82KCZEE09RBQ2oCZI7G9S8KmjpIyjiLC63hw4fj9u3bWLp0Kb755hv+dsYYBg0ahIkTJ1r0eHK5HLNnz8bs2bML3FezZk2Eh4fnu83f3x+7d+82+/H/+OMPi/IQQggp++TO7gCMmxYXRm/arJhGtAiRBH7qYFoCZJyxGzWNaJGyzuJCSyaT4auvvsK4ceNw6dIlpKSkwM3NDe3atUO9evVskZEQQgixyPOmDurSTSNaVGgRIgWm0SumzYI8PRYAIKdCi5RxFhdaWVlZsLe3R7169fjCKjQ0lIosQgghksE3w1AX3gzD1HWQmmEQIg0ypR1kjq4wqFMhyzbucUdTB0lZJzP3wPDwcPzvf//Dli1b8t2empqK//3vfxgwYAAePHhg9YCEEEKIpUyFFstWw6DTFLifH9ESaKNiQsjzKXPWaZlQoUXKOrMKraioKIwZMwbx8fGoW7duvvuUSiU++OADJCcnY8SIERa1dyeEEEJsQWbnCMiNkzaenT7IdFoYcka6qH00IdKRd98szt4ZMqWdiGkIKT2zCq2NGzfC3d0d+/fvR58+ffLd5+DggDfffBN79+6FnZ0dNmzYYJOghBBCiLk4juM7D+rT8zfE0GUYpw1CroDMwVnoaISQIuQd0aLRZlIemFVo/fPPP5gwYQI8PYuey16lShWMGzcO58+ft1o4QgghpKRyNy1Oznd73o6DHMcJnIoQUhRF3kKLRptJOWBWoRUbGwsfH5/nHtegQQPExMSUNhMhhBBSanznQfUzI1o5jTCo4yAh0mLatBgAZPT6JOWAWYWWp6cnYmNjn3tcUlIS3NzcSh2KEEIIKa3cEa38hZY+pxGGgjYrJkRS8o1o0R53pBwwq9Bq27Yt9u3b99zjDhw4gMaNG5c6FCGEEFJacidXAAWnDurSTB0H6YMcIVJi2rQYoKmDpHwwq9AaPXo0Ll68iEWLFiE7O7vA/RqNBkuWLMHZs2cxcuRIq4ckhBBCLFXkiJZpjRZNTSJEUmQOLuAUKuOf6fVJygGzNixu1qwZ5syZgwULFuDXX39Fhw4dULNmTej1ejx+/BgXL15EUlISZsyYgc6dO9s6MyGEEPJciiKaYejSaY0WIVLEcRzsG7ZH+v3rUHrVff4XECJxZhVaADBy5Ej4+flh06ZNOHXqFD+y5eTkhE6dOmHcuHFo0aKFzYISQgghluCbYRQ1okXtowmRHLfeb+Hx7duoYecgdhRCSs3sQgsAWrdujdatWwMAEhMToVAo4OrqapNghBBCSGkUNXWQX6NFa0AIkSbadoGUExYVWnkVt6cWIYQQIjbTiJYhMw1MrzP+WZMJpskEQCNahBBCbMusZhiEEEJIWSNzcAY442nOkJlm/H/Oei1O5QAZTU0ihBBiQ1RoEUIIKZc4mRxyR+P0dkPOpsX6nEYYtIcWIYQQW6NCixBCSLnFTx9Upxr/z3ccpPVZhBBCbIsKLUIIIeWWqSEGX2jlTB2k9VmEEEJsjQotQggh5VbuiFb+qYO0hxYhhBBbo0KLEEJIucXvpZVTaBn4NVpUaBFCCLEtKrQIIYSUW/zUwQzj1EF9ztRBuTMVWoQQQmyLCi1CCCHl1rNTBw3pxs2KqesgIYQQW6NCixBCSLnFj2hlpgKM5Y5o0dRBQgghNkaFFiGEkHJL7pgzopWRAk6bCeh1AACFE41oEUIIsS0qtAghhJRb/NTBzDTIstIAADJHV3AKpZixCCGEVABUaBFCCCm3TIUWmAHy9DgAgIIaYRBCCBEAFVqEEELKLU6ugMzBGQAgT4kx/p8aYRBCCBEAFVqEEELKNVNDDHnaUwA0okUIIUQYVGgRQggp10wNMeSpxkKLOg4SQggRAhVahBBCyjXTOi2ZLgsAoKBCixBCiACo0CKEEFKumaYO8n93pjVahBBCbI8KLUIIIeUa33kwB41oEUIIEQIVWoQQQsq1AiNaVGgRQggRABVahBBCyrV8I1qcDHJHV/HCEEIIqTCo0CKEEFKu5S20ZE5u4GRyEdMQQgipKKjQIoQQUq7lnTpIjTAIIYQIhQotQggh5Vr+ES0qtAghhAiDCi1CCCHlmkxpB05lDwCQO7uLG4YQQkiFQYUWIYSQck/mYGyAIXOmjoOEEEKEQYUWIYSQck+Ws05L9kyrd0IIIcRWqNAihBBS7jm16QeNVwPY1/MXOwohhJAKQiF2AEIIIcTW7Ou1RobGETJ7Z7GjEEIIqSBoRIsQQgghhBBCrIwKLUIIIYQQQgixMiq0CCGEEEIIIcTKqNAihBBCCCGEECujQosQQgghhBBCrIwKLUIIIYQQQgixMiq0CCGEEEIIIcTKqNAihBBCCCGEECujQosQQgghhBBCrIwKLUIIIYQQQgixMiq0CCGEEEIIIcTKRC+0DAYDVq1ahc6dO6Nly5Z46623EBkZWeTxERERmDhxItq0aYMuXbpg1apV0Ol0/P2ZmZn48ssv0alTJ7Ro0QIjR47E1atXBfhOCCGEEEIIIcRI9EJr3bp1+Pnnn/Hll19i586dMBgMmDBhAjQaTYFjU1JSMHLkSGRmZmLbtm1YtmwZjh07hnnz5vHHzJ07F+fOncOyZctw8OBBNGjQAGPHjsXTp0+F/LYIIYQQQgghFZiohZZGo8HmzZsxffp0dOvWDX5+fli+fDliYmJw8uTJAsfv378farUaK1euRJMmTdCmTRsEBQXhl19+QVRUFPR6PVQqFT7//HO0a9cOderUwaxZs6BWq3HlyhURvkNCCCGEEEJIRSRqoRUWFoaMjAx06NCBv83V1RWNGzdGcHBwgeMfPnwIX19feHp68rc1btwYABASEgK5XI6FCxfyj5eeno6NGzfCyckJLVu2tO03QwghhBBCCCE5FGI+eUxMDACgWrVq+W738vLi73v29tjYWOj1esjlcgBAdHQ0ACAhISHfsevXr8fy5cvBcRy++uqrAs9hKcYY1Gp1qR7DGjIzM/P9v6JmkEoOKWSgHNLLIJUcUsgglRxSyCCVHFLIQDmkl0EqOaSQQSo5pJBBKjmkkCEvxhg4jnvucRxjjAmQp1C//vorPvjgA4SGhkImyx1c++CDDxAbG4utW7fmOz4iIgKvvvoqXn/9dX5K4Pvvv4/g4GBMmzYNb7/9Nn/sw4cPkZGRgaNHj2LTpk1Yt24dunfvXqKcN27cKHTNGCGEEEIIIaTiUalUaNasWbHHiDqiZW9vD8C4Vsv0ZwDIzs6Gg4NDgeN9fHywcuVKzJs3D9u3b4ejoyOmTZuGu3fvwsXFJd+xderUAWCcWhgaGootW7aUuNACAKVSifr165f4660lMzMTERER8PHxKfRnVFEySCWHFDJQDullkEoOKWSQSg4pZJBKDilkoBzSyyCVHFLIIJUcUsgglRxSyJDX3bt3zTpO1ELLNJ0vNjYWtWvX5m+PjY1Fw4YNC/2agIAABAQEIDY2Fu7u7tDpdFi0aBFq1aqFjIwM/PXXX2jfvj3c3d35r2nQoAH++OOPEufUarVgjOHevXslfgxrMQ1ARkdHmzVkWV4zSCWHFDJQDullkEoOKWSQSg4pZJBKDilkoBzSyyCVHFLIIJUcUsgglRxSyJCXVqs1K4eozTD8/Pzg7OyMixcv8relpqbi9u3baNu2bYHjQ0JCMHr0aOh0Onh5eUGlUuHkyZNwcHCAv78/DAYDZs2ahePHj+f7uuvXr5dqNIrjOEn8UgFjFpVKJWoeKWSQSg4pZKAc0ssglRxSyCCVHFLIIJUcUshAOaSXQSo5pJBBKjmkkEEqOaSQIS9zawNRR7RUKhVGjRqFpUuXwtPTEzVq1MDXX38Nb29v9O7dG3q9HomJiXBxcYG9vT18fX0RHh6OxYsXY8yYMQgPD0dQUBAmTZoEZ2dnAMDQoUOxcuVKeHt7o3bt2ti5cyeuXbuGnTt3ljhnq1atrPUtE0IIIYQQQioAUZthAIBer8eyZcuwb98+ZGVloW3btpg3bx5q1qyJqKgo9OjRAwsXLsTgwYMBAFeuXMGiRYsQHh6OKlWqYNSoUXjzzTf5x9NoNFi7di0OHjyI+Ph4NGnSBO+//z7atGkj0ndICCGEEEIIqWhEL7QIIYQQQgghpLwRdY0WIYQQQgghhJRHVGgRQgghhBBCiJVRoUUIIYQQQgghVkaFFiGEEEIIIYRYGRVahBBCCCGEEGJlVGgRQgghhBBCiJVRoUUIIYQQQgghVkaFFiGEEEIIIYRYGRVahBBCCCGEEGJlVGgRQgghhBBCiJVRoUUIIYQQQgghVqYQOwApXlxcHFasWIErV65Aq9WCMZbv/lOnTomUjBBCSFlx4MAB9OvXDyqVKt/tarUau3fvxptvvmnzDHQ+I4RUNFRoSdynn36Kmzdv4pVXXoGLi4ugzz1nzhyzj124cKENkxhptVpkZGTA3d29wH0GgwExMTGoXr26TZ57zZo1Zh87depUm2Qg+R04cMDsYwcOHGizHER6rl27hosXL2LixIkAgAsXLmDr1q2IiopC7dq1MW7cOLRp00aQLOnp6VAoFLC3ty9wX2xsLObPn4+1a9fa5LkTExORlZUFwPh+/sILL8DDwyPfMbdv38ayZcsEKbTEPJ9JhRTOJY8fPzb7WFudUwvz+PFjuLq6wtnZGRcuXMDJkyfh7++PwMBAwTKIRSrnszFjxph97A8//GCTDFJ4jVgTx569pEQkpWXLlvj+++8F+1CQ1+jRo80+9scff7RZjuzsbHzxxRc4ePAgdDodmjVrhs8++wxNmjThj4mPj0fnzp0RGhpqkwwBAQFmHcdxnE2vygYEBIDjOLOOtfXV4aSkJPz5559ITU1Fx44d8cILL+S7X61WY/PmzTZ7I/Tz8zPrOI7jbPbv4llSuGJvMBhw6NChIjMIcVEEADQaDTZv3oy+ffuiTp06+OSTT3D06FH4+/tj6dKlBT7wW8vx48cxa9YsvPTSS/j+++/x559/YsqUKejSpQvq16+PO3fu4O+//8aaNWvQvXt3m2QAjEXOnDlzcPbsWXAch969e2PRokV8wbVr1y4sXboUWq0WV69etUmGAwcO4KOPPgLHcWCMFfrewRhD165dsWHDBptkyEvM85nJr7/+ir179yIlJQVdunTB22+/DWdnZ/7+xMREDBkyxGavVSmcS/z8/J57HjH9exHqvfO3337DzJkzsWHDBtSqVQv9+vVDrVq18OTJE8yePRsjR460yfNK5ZwqlfOZFC6wS+E1Yk1UaEncSy+9hO3bt6Nu3bpiRxHNkiVLcPToUcycORMcx2Hr1q24d+8e1qxZg86dOwMwFlqdOnVCWFiYyGlta/Xq1WafFGx5pefu3bt44403oFarARiL4TfffBMffPABf4yti18pevvtt4u9Yi/E1begoCBs374dfn5++T5AmtjyokheCxYswK+//orNmzcjISEBb7/9NqZPn47Tp0+jbt26NjtJBwYGIjAwEG+//TYAYOjQoejYsSNmzJjBH/Ptt9/i5MmT2L9/v00yAMAHH3yAP//8E2+++SZUKhV++OEHBAYGYvr06Zg5cyZOnz6NNm3aICgoCD4+PjbLERwcDIPBgDfeeAOrV6+Gm5sbfx/HcXB0dESDBg2gVCptlsFE7PPZnj17MH/+fAwYMAAymQyHDx+Gl5cXtmzZwo/cVIT3rUuXLpl9bLt27WyYJNegQYPQpUsXzJgxA99++y1+/fVXHD9+HMePH8fq1atx7NgxmzyvVM6ppBxjRNIWLVrEPv74Y6bT6cSOwrRaLYuJiWHR0dEsOjqaRUVFsfv377Nff/3Vps/bvXt3dvbs2Xw53n33XdaiRQsWEhLCGGMsLi6O+fn52TTH82RnZ/N5yrvx48ezadOmsezsbKbVatmmTZtYkyZN2Jw5c/hjpPA7YYyxJ0+eCPZcLVq0YMHBwYI9X2HatWvH9u3bJ2oGxhjr3LkzO3fuHGOMsXnz5rE33niDMcbYjRs3WPv27W32vM2aNWOPHj3i/96hQwcWGhqa75hHjx6xZs2a2SwDY4x17NiRHTp0iP/75cuXWefOndmUKVNYy5Yt2fbt2236/M+6ePEi02g0LDExkb/t2rVrgp5bxD6fBQYGsl27dvF/j46OZv369WMBAQEsNjaWMSaN962KdC4xadasGYuKimKMMTZ8+HAWFBTEGDP+jmz9Wi1LhDyfMcZYQkICCwkJYZcuXWKXLl1iFy9eZH/99Rdbt26doDmeVZZeI7RGS+KSk5Nx+PBhnD59GrVq1SqwkNlWc2Sfde7cOXz44YdITEwscJ+9vT369+9vs+dOSkpCnTp1+L8rFAosXboUkyZNwuTJk/Hzzz8Xum7LVm7evIlPP/0Ud+7cgcFgKHC/kFdCw8LC8uVgjEGj0eDGjRsICgqy2fNev34dO3bs4P89jhs3DrVq1cK7774Ld3f3fCNbQoiMjMTixYtx584d6PV6ALk/i8TERNy+fVuQHI6OjqhUqZIgz1UUjUaDtm3bipoBML531atXDwBw/vx5vP766wAAd3d3ft2QLdSqVQvnz5/HsGHDAACNGjVCWFhYvqk5169fR9WqVW2WATB+/61ateL/7u/vj4SEBISGhmLv3r38z0Yo3t7eCAwMRI8ePfjX58SJE1G5cmV89913qFatms0ziH0+i4qKQocOHfi/V69eHdu2bcOwYcMwYcIEbN++3abP/ywpnEuys7Oxa9eufO+dgPF95ObNmzhx4oTNMwCAq6sr0tLSkJaWhuvXr+Ott94CADx69EjQ87tY59S8pHI+O3jwIObOnQuNRlNg+nGNGjUwefJkm2eQwmuktKjQKgOksBB02bJlaNy4MUaPHo0ZM2Zg6dKlePz4MVatWmXzNR/16tXD8ePH+YXtACCXy7Fy5UqMGDECEyZMwJIlS2yaIa+FCxdCLpdj7ty5WLhwIT766CM8evQI27dvFzTHli1bsHjxYgDg3wRNf7b1GgiVSoXs7Ox8t/Xq1Qtz587F/Pnz4eXlJei/2y+++AIRERHo06cPtmzZgnHjxuHBgwf47bff8MUXXwiWY8CAAfj+++/xxRdfQC6XC/a8eXXu3Blnzpyx2ZoGc9WuXRs3btxAQkICoqKi+Gm+v//+O2rWrGmz533rrbcwd+5cREVFITAwEFOmTMFHH32E7OxsvPDCC7h27RrWrl1r82lAOp2uQAMMlUqFefPmCV5kAcapnHXq1MHYsWP5244ePYoPP/wQCxcuxKpVqwTJIeb5zMvLC7du3UKtWrX42ypXroyNGzdi+PDhmDx5DlaYIwABAABJREFUsmBrGAFpnEuCgoJw4MABNG7cGDdu3ECrVq3w8OFDJCQkCNIgxaRr166YN28enJyc4OLigo4dO+Lvv//G559/jm7dugmSQcxzal5SOZ+tX78er7zyCiZMmIDhw4dj8+bNfAOfadOmCZJBCq+RUhNxNI2UIc2aNeOn3wwfPpz9/fffjDHG9u7dy4YNG2bT5z516hRr3LgxGzduHAsLC8t3X2xsLOvTpw9r1qyZYNM9WrZsya5du8YYY+y1117jp4pt2bKFvfnmm4JkYIyxHj16sKVLl7KsrCzWoUMHFhMTw8LCwtgrr7zCtmzZYtPnnjlzJhs9ejQ/3SavJUuWMD8/P/bVV18J9jvx9/dnFy5cYIwxNmDAAP73s2zZMjZlyhRBMjDG2EcffcSaNm3KXnrpJfb666+z0aNH5/tPCJs2bWItWrRg77zzDlu2bBlbvXp1vv+Esn//fta0aVPWvHlzftrgmjVrWKNGjdj+/ftt+twHDhxgAQEBrGHDhszPz481bNiQ/8/f31+QaS8NGzZk8fHx+W5r2bJlvmmNQmrdujW7d+9egdvv3LnD2rZtK0Ii4X333XesXbt2bMOGDSwmJibffSEhIaxVq1asV69eFepc0qFDB36Ka8+ePdm9e/eYRqNh77zzDvvyyy8FycAYY5mZmWzhwoVsypQp7N9//2WMMbZq1So2e/Zslp6eLkgGMc+peUnlfNa0aVN29+5dxhhjo0aNYmfOnGGMMXbixAk2aNAgQTJI4TVSWjSiVQY8efIE27dvx507d6BQKPDCCy/g9ddfF7Ttqlwu5xf316lTB3fu3EGHDh3Qvn17/gqQrQQEBGDbtm3YtWtXgQ5qVapUwa5du7BgwQKbLZZ9lsFgQJUqVQDk/izatGmDHj16CNK5yyQmJgZDhgyBnZ0d/Pz8cOPGDfTs2RMfffQRFi1aZNOrkR988AHeeustdOnSBRs3buRHKwBg9uzZAIBNmzaZvci4tDQaDWrXrg0AqFu3LsLDw9G8eXMMHDjQou6Z1iD2CPRPP/0ET09P3L59u8AUE47jBFvQPXDgQPj5+SEqKgpdunQBADRr1gybNm3KN33LFgYMGIABAwbgwYMHePDgAd9i3dvbG02aNIGdnZ1Nnx8w/qyF+vdvDoVCgdTU1AK3Z2ZmFnhftSUxz2fjx4+HVqvF9u3b0bx583zTR1u3bo2tW7fi/ffft3kOEymcS1JTU+Hv7w8AqF+/Pm7fvg1fX19MmjQJ7777LubOnStIDnt7e3z00Uf5bhNq1MREzHNqXlI5n6lUKn56b506dfDff/+hS5cuaNq0KR4+fChIBim8RkqLCi2JCw8Px6hRo2Bvb4/mzZvDYDBg37592L59O3bs2FGgpbatvPDCC/jjjz8wevRo+Pr64vLly3jjjTcQExMjyPO3adMGbdq0QWRkZIH7XF1dsWjRIixYsECQLHXq1MHly5cRGBgIX19f3LhxAwCQlpYGjUYjSAbAuB7INH+7du3auHv3Lnr27Il69eohOjraps/t7e2NX375BZcvX0aDBg0K3D979mx06dIFR44csWkOkxo1auDOnTuoVq0a6taty8/bNhgMyMjIECQDIFzr9OL88ccfYkfg+fn55VsbZSq4hFK3bl3ROtwxxvDOO+/k6+aXnZ2N999/v0ChJ8Ra2y5duiAoKAjLli3jP8RFRkZi4cKF+S6U2JLY5zOO4zB58uQi15Y0b94cx44dw/Xr122aw0QK5xJPT08kJCSgevXq8PHxwZ07dwAAHh4eiI+PFySDSUhISJHbUghxgUjMc2peUjmfNW3aFHv27MGsWbPQoEEDnDlzBuPHj8fdu3cF6VIKSOM1UlpUaEnckiVL8OKLL+Kbb77hT86mk/XSpUsFq+gnTpyI6dOnQ6lUIjAwEKtXr8bEiRMRHh6O9u3bC5IBMK4Dat26NQYPHow+ffrAycmJv08mkwmSYfTo0fjkk08AAC+//DIGDBgAe3t7XLlyBS1bthQkA2BcXL9x40bMmzcPjRs3xt69ezFx4kRcvnw538/FVlQqFTp06ICVK1di8ODB+dY9AMCLL76IF1980eY5AGNr4A8++ABLlixBt27dMGbMGFSvXh3nz59Hw4YNBclgIoURaMYY/vrrr3wZ2rdvL+i6sedtfClUIx+xDBo0qMBtNWrUECGJ0YcffoixY8fi5ZdfhqurKwDjaEaTJk0s2junNKRyPgOMMyUGDhyIQYMG5Xvvksvl+ZqY2JIUziVdunTB/PnzsXDhQrRu3RoLFixAr169cPToUXh7ewuSAQDWrl2L1atX8xsW5yXUSLzY51QTqZzPpk2bhgkTJsDd3R2DBg3C2rVr8corr+DJkyfo16+fIBmk8BopLdpHS+JatWqFnTt3FnhxhYWFYdSoUQgJCREsy61btyCXy+Hn54dLly5hy5YtqFatGqZPny5YV6Dg4GAcOnQIJ06cgEajQa9evTB48GBBiz3AuLmih4cH2rRpg0OHDvFduz799FObLvTP686dOxg3bhzefPNNDB8+HK+++ipSU1ORmZmJ8ePHY9asWYLk6NmzJ6Kjo+Hv719oASwExhi2bdsGHx8fdOvWDRs3bsSGDRtQrVo1fP3112jUqJEgOQq7Yn/z5k1kZmYKNgKdnJyM8ePH49atW3BxcQFjDOnp6WjSpAm2bNnCf8i2tWc/vOt0Ojx8+BB37tzBG2+8gZkzZ9rkeYODg80+VgrdGYWk1+vx999/47///oNCoUD9+vXRoUMHwaY4Sul8tmbNGhw+fBgPHz4U9b1L7HNJamoqPvroI3Ts2BEjRozAxIkT8ddff0GhUGDx4sV45ZVXbJ4BADp16oTRo0dj0qRJgjxfYaRyTpXK+QwAnj59Co1Gg1q1auHu3bvYuXMnqlWrhtGjRxfoGmorYr9GSosKLYnr1KkT1q9fj6ZNm+a7/caNGxg7dqxgJ6a9e/eib9++gp+EiqLVanH69GkcOnQIZ86cQaVKlTBw4EBMnz7d5s/9zz//2HyNibmysrKgVqvh6emJ+Ph4HD58GN7e3ujTp4+gOa5cuYLDhw/j2LFjyMrKQq9evTBo0CDBfk6RkZEFRtTEMH78eDg4OBR6xV6j0QhyxX7OnDm4fv06vvnmG37aXlhYGGbPng1/f3/Mnz/f5hmKs3btWsTExODLL7+0yeN37tyZn/JU3OmN4zjRWwNnZmZi+fLl+Pjjj0XNkZ2dLci6Namcz/K6fv06Dh48iOPHjyMjIwO9e/fGoEGDBLl4J6VziQljDKGhoahcuTK8vLwEe94WLVrg6NGjoo76AtI4p0rlfFbUbBUhSfE1YikqtCTu/fffR2xsLFatWsWPGiUmJuLdd9+Fm5sbVq9eLUgOf39/MMZEG0EqSmJiIg4cOIC1a9ciKysLt27dsvlzNmrUCNWqVSt02omQPvjgA0n9LgDjqMW5c+dw5MgRnDp1Cu7u7oKsGfLz8ytySqmQpHDFvn379li1ahXatWuX7/aLFy9i1qxZOH/+vM0zFCcqKgoDBw602c8iKSkJ48ePh0wmw4oVK4odrbHlh7qsrCwsWbIER44cgVKpxIABA/Dee+/xU5zPnTuHzz77DE+ePBFkX5ykpCSsX7++wN48Wq0Wd+/eFeTfplTOZ4UxGAzYsWMHli1bBrVaLUgRLoVzSVFTKIU2duxY/O9//xO1mZBUzqlSOZ9JYbaKFF4jpUVrtCTu/fffx7Bhw9C9e3f4+PgAACIiIuDu7i5Y8wcA+Pvvv/Hbb7/h8OHDmDBhAry8vDBgwABRrnao1Wr89ttvOHToEC5cuIAaNWpg/Pjxha6JsIVTp07h4MGDOHz4ML799lv4+/tj0KBBgo/4xcTEYNy4cfD29pbMm1BiYiIePHiAyMhIZGdn59to2pZ+/PFHHDx4EEuWLEFQUJDgI2omTk5O0Gq1BW4v7DZb0el0qFy5coHbK1eujPT0dMFyFOXff/+16VoxDw8PfPvttxgwYAD++ecfDBkyxGbPVZyvv/4au3fvRv/+/aFSqbBjxw44Oztj0qRJCAoKwo4dO1C7dm1s27ZNkDzz58/HP//8g44dO+L48eN45ZVXcO/ePdy+fVuwKVFSOZ/l9eTJExw6dAiHDh3CvXv30K5dOwwePFiQ55bCuWTw4MH5nl+sD9SBgYH48ssvcfPmTfj6+haYljZw4ECbZ5DKOVUq57Pff/+dn62ydOlSUbJI4TVSWjSiVQZkZGTg119/xX///QfGGBo2bIhXX321wIJRoSQmJuLYsWM4cuQIrl27hlatWuGnn34S5LlnzpyJ06dPg+M49OnTB4MHDxZ0I8Fn3b59G4cOHcLx48eRnJyM3r1727zdfV5Pnz7lPySEh4ejdevWgr8Jpaen48SJEzh06BCCg4NRvXp1DBo0CIMGDUK1atUEyWAi5pRSQBpX7N944w00aNCAX0BsEhQUhJs3b2Lnzp02zwAU3gwjPT0d4eHhGDFiRIF81rZ37178/vvvWL9+vU2fpyjdu3fHxIkTMXz4cADA6dOn8dVXX6FDhw7Yu3cvxo4dixkzZgi2zuHFF1/E4sWL0a1bN7zyyiv81NJPP/0U2dnZgm3+KZXz2c6dO3H48GFcuXIFNWrU4D9YC9m0Ji+xzyViTqEEkK876bOEnOYrhXOqidjns7zEmq2Sl9ivkZKiQotYTKPR4PTp0zhx4gT++OMPeHl54cSJE4I89+jRo/G///0PL7/8MhwcHAR5zue5desWjh8/jp9//hkcx4myzgAA/vvvPxw+fBjbt2+HwWDAlStXBHne5s2bQ6lUonfv3vjf//4nauFrIsaUUsB4RXTYsGFISUkpcMX+xx9/FGTh7r///osxY8bAz8+P3x/n8uXLCAsLw/fffy/YB6fCOtkplUq0bNkS/fv3h0JRvidUNGvWDEePHuWviOv1ejRr1gxubm5YsWKFYB05TZo2bYrffvsN1apVw7Rp09CjRw8MHDgQ4eHhmDhxIs6cOSNoHrG1atWKv1gnlaYoUjiXiDGFUqrEOqc+S6zzWV6xsbE4cuQITpw4gRs3bqBdu3bYsmWL4Dmk8BqxVPk+05VRPXr0wN69e+Hh4YGAgIBi1xicOnVKsFwXLlzAoUOHcPLkSRgMBvTt2xffffedoB+sf/zxR8GeqziRkZH8Va+HDx/ixRdfxLx58/Dyyy+LkufatWt8N0bGmGCtVwHjlKQ+ffqIXviKPaUUMO4vduTIkXxX7IcOHSroFftWrVph+/bt2Lx5M86dO8ePGsybNw/NmzcXJAMgjT3FxKTVauHo6Mj/XS6Xw87ODp988ongRRYAVK1aFdHR0ahWrRp8fHwQHh4OAHBwcEBKSorNnleq57Pz58/n+/2IRSrnEjGnUOZ179493LlzB0qlEvXq1RNtHzwxz6mANM5nRc1WWb58uaCzVaTyGikpKrQkaNCgQbC3twcAUd7oCtO5c2ckJCSgTZs2+OSTTyQ1ogQY35Q2b94syF4bQ4cOxY0bN1CzZk1Rp5s8ePAAhw4dwuHDhxEZGYl27drh/fffx8svv8z/+xGCkG/8RXl2SunWrVtFG1lzcnLCiBEjRHluk+bNm2PFihWiZgCA6Oho7N69G+Hh4ZDL5WjSpAmGDh1a6Boya9LpdPjrr7/Qvn17/n1q586dOH36NCpXroyxY8eiXr16Ns1QFCGL3bx69+6NOXPmYNGiRXjppZcwc+ZMtGjRAr///rtN11JK8XwGoMgiS6vV4urVq4KMcknhXCKVKZTZ2dl477338Pvvv/O3cRyH7t27Y8WKFYJMsZXKOVUq57OXXnqJn62ybds2UTJI4TVSWlRoSVDeYuHFF19Ey5YtC+zCnZ2djdOnTwuW6fXXX8fAgQMlu2eBWq3GmjVrBCm06tWrh9mzZ4s+3aRv37753nzEbov7rMTERAwZMkSQq9Tx8fH47LPPRLkAIIUr9nPmzMEnn3wCZ2fn524+K9RI05UrVzBu3Dh4eHigadOm0Ov12LVrF7Zt24affvrJZnuKJSQkYPTo0Xjw4AEOHz6MevXqYd26dVi9ejWaNWsGjUaDoUOHYufOnTbf16ywfwtC7Vn1rJkzZ0Kn0+Hx48d49dVX0bt3b7z77rtwdXXFypUrbfa8UjyfFSclJQVjxowRZMqcFM4lixcvRp8+fTBjxgxRcyxfvhzXr1/H2rVr0a5dOxgMBgQHByMoKAirV6/Ge++9Z/MMUjmnink+y0sKs1Wk8BopLVqjJXGNGjXC+fPn4enpme/2W7duYfjw4bh+/bqgeYKDg3Hv3j0EBgYiJiYGPj4+Nl9rodFosGTJEhw6dAhKpRL9+vXDrFmzBL3CVJjHjx/j3r17aNu2LTIyMlCpUiVBn//SpUsFWnhLSXp6Or766itBp5BpNBpERUWhdu3aYIwV+EBnC2vWrOH3z1qzZk2xx9rqQsDo0aOxdu1auLq6YvTo0cUeK9T029dffx2+vr748ssv+fcIrVaLOXPmID4+Hlu3brXJ837xxRcICQnBihUr4Ovri4yMDHTs2BHNmjXjv/dFixbh8ePHWLVqlU0yAMbF/f369cu3P9WhQ4cQEBBQYFG9WNMsk5OT4ezsLNh6Oamdzwqj1Wrx77//CvreKua5RK1WS2IKZadOnfDll1+ie/fu+W7/888/MX/+fEEKcamdU8U4nz0rKysLx48fx7179zB+/HjcuXMHL7zwAjw8PATNIfbnrdKgES0J2rp1K99JhTGGjh07FnqckFNQ0tPTMWHCBFy9ehUcx6Fjx45YunQpIiMjsXnzZlStWtVmz718+XLs3bsX/fv3h0wmw969e6FWqxEUFGSz5yyOVqvFBx98gGPHjkEmk+HEiRNYvHgxMjIysHr1asHW4rRr1w6xsbHYvXs37t+/j08++QTBwcFo0KABfH19BclQHGdnZ0E/QC5duhQ//vgjtFotTpw4geXLl8PBwQGff/65TU9QUrhin7d4Kq6QiouLs1mGZ4WFhWHhwoX5PsQrlUq8/fbbNm25fvr0aXz55Zf8a+Cff/5BVlYWhg4dyh/Tp08fTJ482WYZAKBt27YFft6tWrVCUlISkpKSbPrcJjNmzMBXX31V5HuSqTOmLUnxfFYcpVIp2IdtKZxLHB0dcebMGWzatAn379/Hrl27sG/fPtSuXRsDBgyw+fObZGRkFHreqlu3LhITEwXJIKVzqljns7zi4+Px+uuvIyEhgZ8JsHnzZty8eRPbtm0TZPq1FF4jpcaI5Gi1WrZ//372yy+/sIYNG7Jt27axffv28f/t37+fnThxgqWkpAiWaf78+ez1119njx49Yi1btmSPHj1id+/eZQMHDmSzZs2y6XN3796dHTlyhP/7n3/+yVq2bMkMBoNNn7coK1asYH369GEXLlzgfxYXLlxg3bt3Z5999plgOSIiIli7du1YQEAAa9KkCXv06BGbNm0aa9myJbt69apgOaRg27ZtrFOnTuyXX35hLVq0YI8ePWJHjhxh7dq1Y8uWLRMsh5+fH0tISChw+82bN1mzZs1EzRAZGclatmwpSAbGGAsMDMz3ujU5ffo069u3r82et0mTJiw6Opr/+6JFi5ifnx+Li4vjb4uOjmZNmza1WQap8PPzY/Hx8fluCwwMZI8fPxYsgxTPZ+Hh4Sw7O5v/+9mzZ9mXX37J1q1bV+hrx1akcC45d+4ca9KkCfvwww9Zs2bN2KNHj9iyZctYo0aN2P79+wXJwBhjQ4cOZevXry9w+7p161j//v0FySCVc6pUzmfvvfcemzRpElOr1fy/z+TkZPbmm2+ySZMmCZJBCq+R0qJCS+L27duX74Qglm7durHLly8zxhj/j50xxq5cucI6dOhg0+du0qQJe/LkCf93jUbD/Pz82NOnT236vEXp1asXO3/+PGMs/8/i77//Zp06dRIsx9tvv80+/vhjZjAY+BxarZbNmjWLjRo1SrAcDRs2ZH5+foX+16xZM9arVy+2Zs0amxbG/fr1YydPnmSM5f+dnDx5knXv3t1mz8sYY1u2bOG/3+J+FkOHDrVZhj179rDRo0ez0aNHs4YNG7Jhw4bxfzf99/LLL7MuXbrYLMOzDhw4wDp06MC2bNnCbt++ze7cucN++eUX1qlTJ7Zu3Tp26dIl/j9r6tChA7t9+zb/94EDB7J+/frlO+bcuXOC/iyelZCQwI4dO8YiIyNt+jwNGzYsUGjlfX0ITezzWXp6OhszZgzz8/Njd+/eZYwxtnv3bubn58e6du3KevfuzTp16pSvULclKZxLXn/9dbZly5YCGTZu3MgCAwMFycCY8QJqo0aN2IwZM9i2bdvYtm3b2PTp01mjRo3Y0aNHBckglXOqmOezvDp27Mhu3bpVIEdoaChr27atIBmk8BopLZo6KHGDBg1CYmIiHjx4AIPBAMA4/UKj0eDGjRs2n/5ikpiYiCpVqhS43dXVFWq12qbPrdPp8g2VK5VK2NvbIzs726bPW5SnT5+idu3aBW6vVq2aTdskP+vKlSvYvn17vsX1CoUCU6ZMyTdNytY+/vhjfPPNNxg+fDjflejff//F9u3bMXz4cLi5ueGHH36ASqXCW2+9ZZMMUVFRaNSoUYHb/fz8bD5dbtSoUXB3d4fBYMDHH3+MOXPmwMXFhb+f4zg4OjradP+qnj174vLly/zfvb29C6xhbNCgAQYOHGizDM/68MMPARjXQz0rb/MFa29G2r59e/z000/46quvEBwcjNDQ0HzvkwaDQfBtKe7cuYNp06YhKCgIDRs2RP/+/REfHw+VSoWNGzcKtreZ2MQ+n23YsAGRkZHYuHEj6tatC41Gg6+//hp+fn7YtWsXVCoVPvjgA6xevVqQac9SOJeEh4cXull1nz59nrvm1Jq6deuGlStX4rvvvsPp06f5bSlWrFiB3r17C5JBKudUMc9neWVkZBS5fk+n0wmSQQqvkdKiQkviDh48iLlz50Kr1QIwnpRMbwI1atQQrNBq1qwZjh07hokTJ+a7ffv27WjcuLEgGaSiXr16+OeffwqsMzly5Ajq168vWA6DwcB/WMkrIyMDcrlcsBxHjhzBxx9/jNdff52/rWfPnvD19cXevXuxY8cOvPDCC1iyZInNCq0aNWrwLWDzOnv2LL9hrK0oFAq+gOE4Dq+88oogrYjzcnd3z/fB0NSBUEy//fYbZDKZ4M87bdo0jBgxAu3atUNGRgZq1KiBsWPHAgCOHj2KDRs2ICoqCrt37xYs0+LFi1GnTh34+vri8OHD0Ol0OHPmDHbu3IkVK1Zg586dgmURk9jnsxMnTuDjjz9G586dARibH6SmpmL27Nn8a3bIkCGYOXOmTXOYSOFc4uLigtjY2AIfZu/evQs3NzdBMpj06tULvXr1EvQ585LKOVXM81lebdu2xY4dO/J1stVqtfj222/h7+8vSAYpvEZKiwotiVu/fj1eeeUVTJgwAcOHD8fmzZsRGxuL+fPnY9q0aYLlmDVrFsaNG4fr169Dp9Ph22+/xb1793Dr1i1s2rTJps/NcVyBlshitUgGjB/kZs6cibt370Kv12P//v148OABv2BVKJ06dcKGDRvw9ddf87clJyfj66+/FvQKeWhoaKHP16ZNG8yfPx8A0LhxYzx58sRmGcaPH4/58+cjLi4OjDH8888/2LVrF3788Ud89NFHNnveZ4l9xR4ouoOdKUPr1q1tngEwthMPCgqCn5+fIM9nUrduXRw+fBjHjx/nC19XV1cAxn29ateujcWLFwu6j9a///6LPXv2oFKlSvjrr7/QtWtXVK1aFYMHD8aWLVts9ryFvXeKSezz2ZMnT/KNFAQHB4PjOLz00kv8bTVq1BDsSrkUziWvvvoqFixYgAULFoDjOGRkZODs2bP48ssvbb5JrxQ6tuYllXOqVM5nH374IUaOHIlLly5Bq9Xi888/x/37/2fvzMNqXL///94p85ipOBTRoDSSyNTgHDKVKVOhwRzCoSRUGoWOSoWiiIjKkCHDESJNDqGkhFQylKhovH9/9Ov5tu0Mn8/pufdzPme/rst16d77st7a+3nWs9a97rWe4dOnTzhy5AgVDVy4Rv4uovbuHGfw4MGIiYmBnJwczMzMYG1tjdGjRyMuLg6BgYGIioqipiUzMxMhISF4/Pgx6urqMHDgQFhYWEBNTY1Vu4qKigIPC40zoY2hMfsEqM8sBQUF8f0urK2tqU4qLyoqgrm5OT59+oQPHz6gf//+yM/PR+fOnXHkyBFqM0CMjIwwa9YsLFy4kG/90KFDOHr0KOLi4nD37l1s3LiR1c57x48fR0BAAF6/fg0AkJSUhLW1NbObQYMfZewbD+Nki0ePHmHz5s3IyspqMjtL6xoZNmwYIiMjmyz7YJOCggJIS0tzKsAYMmQIoqOjIS0tjWHDhsHR0RHGxsbIzMzEwoULkZiYyIpdRUVFaGho8JVep6SkYPDgwXxt5wEgLCyMFQ2NEbY/GzZsGN8Mt9mzZ+Pdu3d812VKSgrWrFmDW7dusaqlAWH7kurqatjZ2SE2NhZAfXBOCGFK+b7+njQn+vr6OHXqFDOD8FvweDwq8xi54lMBbvgzoP53cuzYMWRkZDDfz7lz51KdqSrsa+TvIgq0OI6WlhZiYmLQp08fbN68Gf369YOlpSUzdLLxuYz/VaKjo3/6vSYmJiwq4R6fP3/GuXPn+G6CU6dOpVo2dvbsWdjZ2WH8+PHQ0NBAXV0d7t+/j4sXL8LZ2RmampqwtLSEoaEhNm3axLqe4uJiEEKEMmfDyMgIampqTWbs165dS6Vd8rx581BZWYnp06fD3d0ddnZ2ePnyJcLDw+Hl5YUJEyawrgEA9u/fjxs3bsDS0hJ9+/YVODPWq1cvVuwqKSnh1q1bnJqzYmVlBSkpKUhKSiIkJAQ3btxAdXU1HB0dISYmhsDAQFbs2tnZ/XTASeNMkrD92dKlSyEvL4+1a9cy8yDNzc35SqPWrFmD6upq+Pv7s6qFa7x48YLxI/Ly8pwqy6qrq6NWhswFn9oYYfozEc2DqHSQ46ioqCAyMhJr166FvLw84uPjYWlpiezsbNZnKdjb2zNnPRo7oqZg00lzIXjiWolDA23atGF1JtHPMHnyZLRv3x4hISHYtWsXxMXFoaCggMDAQIwaNQrJycmYPHlys/9eYmJiYGRkhJYtWyImJua776XVBCIvLw++vr6Qk5ODgoICiouLoa+vj5qaGgQGBlIJtB4/fozQ0FCoqqoiKioK8vLymDt3LqSkpHDixAlqgZaPjw9qa2uZ8qwGGnb52NpZ42Lu0NHREba2tsjLy8OmTZsgKSkJFxcX5OTkYP/+/azZbaoRiTARpj8DgOXLl8Pc3Bw3btxAfn4+OnbsCAsLCwBAYmIiDh48iISEBFbLorjgSxrv+hYUFACobzLVeJZZwzpbCZGvMTAwwKlTpwRmuxUVFWHKlCm4e/cuFR3C8qlc8Wfm5ubw8/NDx44dYWZm9t1EDVu74Fy4RpoTUaDFcWxsbGBlZYXOnTvDxMQE/v7+mDhxIgoLC1mvn3716hVTevTq1StWbf2I5ORknDp1CqWlpRg9ejRmzZrFdzi1tLQUNjY2rF34UVFRmDdvHtq0afPd8hYej8fqhW9gYICTJ08ypRbfuwnSKLVoQE9PD3p6ek2+NnToUAwdOrTZbdrZ2WHUqFHo2rXrd+vWeTwetUCrZcuWzKF6GRkZPH36FKNHj4aKigpevHhBRUNdXR3TIVRGRgZZWVkYMmQIDAwMEBQUREUDAFbPHv3TkJGREbhvWFtbY9OmTawesm/qgfpb0HigFqY/A+qHIkdGRiIqKgpiYmIwNTVFz549AQA3b97E27dvERAQAHV1ddY0cMGXGBgYMLu+3/IjbCdEgPrmNDdv3gRQf37S2dlZoFQxPz+f1TJgrvhUrviz3r17M7uHNMsDG8OFa6Q5EZUO/gMoKipCVVUV+vTpg5ycHBw7dgzS0tIwMzOj1t0sJSUFqqqq1LupAcC1a9ewcuVKaGtrQ0xMDImJiVBTU0NgYCDTFendu3cYNWoUlfMnL168gIyMDOt2moKrmZ6UlBSkpaWhurpaYEeBho6ysjKhd9kDgAULFkBNTQ1r165FWFgY4uPjERwcjBs3bmDDhg2sncVpzJQpU7B48WJMmjQJe/fuRV5eHtzd3fH48WOYmZkJpdy4uLgY4uLiTFMKNlFUVGSukR9B6xr58uULnJycICsriyVLlgCoP5+iq6sLR0dH1u6rjcsomzrrCtB5oG4MF/wZVxCWL0lKSoKmpibExcWRlJT03fdqa2uzpqOwsBAbN24EIQTJyclQV1fn29lsGI0xZ84cjBkzhhUNXPSpXPFn586dw8iRIwV2GWkizOet5kK0o/UP4MWLF6ipqUGfPn0gJycHCQkJqKmpUXVKNjY2OHDgAJSVlanZbMDPzw82NjZMx7YHDx5g5cqVWLRoEcLCwqjfkObPnw9/f3++MgtaNL7Rt2nTBpMmTWIyssLC398fvr6+6Nixo8BnQSvjZGJiAh8fH6F8Pxsj7Iw9AJiZmcHBwQEA8Ntvv2Hq1Klo3bo10tLSWM3UN0VYWBj27duH9+/fAwC6desGS0tLgcYpzU1sbOwPz3TQzIZ6eHggJSWFrwza3t4eO3bswO7du5mZY81NaGgok4yi0eziZxC2PyspKcGff/6Jjx8/QldXl2mM0UBFRQVCQkKofDeE5UsaB09hYWGwtbWl2oWzAWlpaeZ7aWZmBj8/P+ot5bnoU7niz5ydnXH06FGhBlrCfN5qNqiNRhbxX3Hu3DmirKxMDhw4wKzZ2NgQZWVlcvnyZWo6jIyMyJ07d6jZa0zjaeAN5OTkEB0dHbJgwQJSVVVF3r59SxQVFano0dPTY6alCxMtLS3y/PlzYcsgurq6JDAwUOgasrOzhaqhgdevXzPf1+zsbOLi4kIOHDhAKisrqWm4fPkySU5OJoQQcubMGTJ58mSyePFikpeXR03DsWPHiIqKCnFzcyOXL18mcXFxxNXVlQwePJhERkayZldBQYG8e/eOtX//v0FXV5ekpaUJrCclJZFRo0ZR05Gbm0vS09OZnw8dOkRyc3Op2Re2P3v69CkZMWIEUVdXJ+rq6kRJSYl4enryveff5ku0tLSo3hd+lvfv35MLFy5Q1cYln8oFfzZz5kwSGxsrVA1cuEb+LqIdLY4TGBgIOzs7zJ8/n1nbs2cPwsLC4OvrC0NDQyo6Ro8ejSVLlmDMmDGQkZERqKNmM/snKSmJFy9e8A3q69+/P/z9/bFo0SJs2LCB+qwkKysrTJ06FTIyMgLd1GidB1JTU8O1a9eot3v9mk+fPmHSpElC1WBubg4bGxvMmzevyQ53bJwR+xbCztgD9W2sS0pKANQ3K2nVqhW0tbWpZiYPHTqEjRs38t27xo0bBxkZGYSGhmLGjBms2OVSW/cGysvLmyyblJSUpDaz6fbt21i2bBkWLlwIFRUVAPU7fz4+Pti/fz+GDBnCugZh+zMPDw9oaWnB29sbYmJiCAsLw65du/Dhwwe4ubmxarspuOBLTExM4O3tjRUrVkBGRkZo5ZtZWVmwsbHB9u3boaCggClTpuDdu3do2bIl9u3bR2WOFVd8Klf8maKiItavX48DBw5AVlZW4LmPRqdSLlwjfxfRGS2Oo6amhnPnzglMA8/Ly8OkSZNw//59KjqEOeNi586diI2Nxbp16zBq1Ci+B5a4uDisXbsWKioquH//PpVzBt8bwErzrMPq1asRFxeHjh07NnkTpFUqtGjRIkyfPl2owRZXPpPY2Fhs3LgRtra2sLS0BACsWrUK165dg4+PD5XEyKNHj2BhYYFp06YxJWn6+vqorq5GcHAw5OXlWdcA1DceOHfunMAcrZcvX2LSpEl48OABK3YVFRWRkJDAqXbIixYtgpSUFFxdXZmSRkIItm7dimfPnlEZ/jl9+nSMHDkStra2fOu7du1CUlISIiIiWNcgbH+mra2NY8eO8ZXJXb58GWvWrMGCBQuwYcMGqud9uXDf+vXXX/Hy5ctvJiho3TstLS3RokULuLu749KlS9izZw9Onz6NiIgI3Llzh8r3kys+lQvfC6C+nPN7HD58mHUNXPld/B1EO1ocR1paGsnJyQKO6d69e0xnMRpcu3btm681NRS1OVm5ciVKSkqwceNG7Nu3DyNGjGBe+/XXX7Fnzx7Wzjg0RWZmJjVb36Nt27acyOZMmjQJLi4uePjwIfr37y+QEaWhkWaHxe8h7Iw9UJ+119fX53ugjouLg6OjIzw8PBASEsK6BqC+i93Dhw8FAq309HR069aNNbsrV65E27ZtBdaLi4uRkpKCrl27QktLizX7TWFra4sFCxbg7t27zG7So0eP8OHDB2qfR05ODnx8fATWZ86cSeWBCRC+P2vZsiUqKyv51saNG4fNmzfDyckJPXr0oJow4oIvaTj7LGzu3buHyMhIdO3aFTdv3sSYMWPQs2dPTJs2jVoHU674VK74s+/dF96+fUtFAxeukb+NkEsXRfyAQ4cOEXV1dbJ7925y7do1cu3aNfLHH38QTU1Nvjp3ttHX1yclJSUC669fvyba2tpUNHz+/Jl8+fKlyddKS0vJ2bNnqegwMzMjHz9+FFh/9+4dmTp1KhUNhBCSn59PamtrBdarq6vJ/fv3qelQUFD45h9aZx3s7OzIp0+fBNZLSkrIsmXLqGgghBBVVVWB84SEEPLy5UuiqqpKRUNTZxoJIeTZs2dEU1OTigZCCDl48CDR1tYm4eHhJCMjg2RkZJAjR44QbW1t4ufnx6ptPz8/oq2tzZy3SE1NJVpaWsz3cuHCheTz58+saviavLw84u3tTZYsWUJWrFhBdu3aRYqKiqjZ19PTI3FxcQLrV69epXZOTNj+zNbWlpiZmZE3b94IvObl5UUUFRWJq6srtfsWF3xJdHR0k+dHy8vLycGDB6loIKT+fNTLly9JdXU10dTUJNHR0YQQQjIyMsiwYcOoaOCKT+WKP1NUVCTv378XWM/LyyPq6upUNHDhGvm7iHa0OM6CBQtQVVWFsLAwBAYGAgB69OgBW1tbvqw5G3BhxkUDr1+/xuXLl9GyZUuMGTMGUlJSfK937NiR1UxkfHw80tPTAdS3xg0ICBDImr948QL5+fmsafgaAwMDJCQkQFJSkm/91atXMDMzo1ZWKqyMU2pqKvLy8gDUD3tUVlYW6HqYk5ODO3fuUNMk7Iw9ALRr1w55eXkCGt68eUP1/IW5uTny8/Ph5uaG2tpaEEIgLi6O2bNns5pFP378OAIDA7Fw4UKmfHDTpk1o3bo1IiIi0KFDB9jY2GDfvn1YtWoVazq+5pdffsG6deuo2fuaqVOnYtu2bfjw4QPU1NQA1O8u+vj4UMviC9OfAcCGDRtgbW2N0aNHY9++fRg1ahTz2u+//w4ACA4OZtWnccGXFBcX48uXLwDqu18OHDgQXbp04XtPRkYGdu3axXqH0AbU1dURFBQESUlJVFZWYvTo0SgqKsKuXbuodUsVpk/lij87efIkzpw5A6C+vHnFihUCw8TfvHnD6qgOLlwjzYko0PoHYG1tDWtra5SUlEBCQoJaO3MNDQ1EREQwc5EKCgqanHHh6enJqo6UlBRYWVkxjqFt27bYs2cPRo4cyardxvTu3RvOzs7MzJnz58/ztY9u+F1s2LCBVR3h4eFMqREhBNOnTxdoY/3x40cqw0eFDY/HY5qg8Hg8bN++XeA9bdu2Zc5K0WDOnDlwcXHBy5cv+R5mQ0NDsXz5cioafvvtNzg5OWHbtm1MS9z09HQ4Oztj3LhxVDQAgJiYGBwcHLB69Wo8e/YMQH0TG7bvX5GRkbCzs8O8efMA1P/fnz9/DltbWwwYMABAfbmUh4cH1UDr6tWrCAoKQlZWFsTFxTFgwABYWlpS+0xWrFiBkpISODs7o7q6GjweDy1atICZmRnV34Ow/BkASElJ4dSpU0hNTRVo6w7UB1ujR49GbGwsaxq44Etu3LgBOzs78Hg8EEKabExDCGFtdlVTODo6wtbWFnl5edi0aRMkJSXh4uKCnJwc7N+/nzW7XPGpXPFnhoaGfLMWpaSkBBpQyMvLsz40WdjXSHMiaobBQZKTk6GhoQFxcXEkJyd/9720us8Ia8ZFg+327dvDyckJLVq0gLOzM3JycnDu3DnqWoD6xgInT54UyHrR4PPnzwgODgYhhOm62K5dO773tGvXDr/++it69+7Nmo6fGYTaAK1D5bdu3WL17M/Psn//foSFhTE17D169MDixYupZOyB+jlAq1evxs2bN/k+l3HjxsHNzY3Kg+39+/ehoKDA56Dj4uLQo0cP1rPTGhoaiI6OhqysLID6z2PXrl2Ijo5mDlbn5eXByMiIyZqyTVxcHFavXg0DAwMMHTqUGdD6559/wtfXFwYGBlR0APUdEHNycnDjxg3o6Oiw3m2Qi/6sKT58+ID27dtDXJxe/lmYviQ5ORl1dXVYsGABfH19+Xx7w4OsvLy8wG4GTYqLi9GpUye0aNGCNRtc8amN4Yo/s7e3h4ODg1CHJwvzGmkuRIEWB2ncNavhIbapj0kYHVcKCgqQk5ODoUOHory8nEpnLy0tLRw/fpzJRhcVFWHs2LFITk4W6g2gqqoKr169Qt++fUEIoe6QGk+0p010dDQmTpyIli1bIjo6+rvvbTyklQZVVVVCa1HcGGFk7Bvz7NkzZGVlQUJCAnJyckzgwTbbtm3D8ePHcfDgQb6WzNbW1rh16xbMzc1hb2/Pmn0NDQ3ExMRARkYGALBkyRLcv38fiYmJzHsyMzNhbm6OpKQk1nQ0xtjYGIaGhgJjMPz8/BAfH4/IyEjWbPv7+yMsLAwnTpyAjIwM7t27B2tra5SVlYHH40FHRwcBAQECWevmgmv+7O7duwgPD8fmzZvRo0cPvHnzBqtWrcL9+/fRunVrWFtbU9t9bkCYviQpKQmamppUA8xv8eXLF1y8eBE5OTmwtLREVlZWk2WNbCFMn/otuODPkpOTkZOTg0mTJuH169eQlZWl/n0R9vPW34LmgTARP0d6ejqpqqoihBDy6tWr7/6hRVVVFVmzZg1RUFAgSkpK5OXLl2TFihVk4cKFTR7abE4UFRXJ27dv+dYGDx5M9f//NTt27CCqqqrM78LW1pZs2rSJ+dxo8fnzZxIdHU28vb1JSUkJuXv3LikuLqaqwcXFhbx48YKqzaY4evQo0dPTYz6TLVu2EH9/f9btJiUlkerqaubv3/vzv86JEyeIuro6iY6OJjU1NXyv1dbWkqioKKKqqsocdGeDWbNmkaioKEJIfZMcNTU1snbtWr737Nq1i8ybN481DV8zePDgJgeh5ubmstokJSIigqioqBBvb2/mPv3bb78RXV1d8vTpU/L69Wsyc+ZM8scff7CmgUv+LDExkQwaNIiYmpoyjUgsLCzI4MGDyZEjR8jp06eJrq4uOXXqFOtaGuCCL8nIyCB2dnbE1NSUvH79mhw5coQkJiZSs09I/aBofX19oqamxvwulixZQn14Lxd8KiHC82eN+fTpEzE1NWUaWzV8JhMmTCCvX7+mpoML18jfQRRocRANDQ1SWFhICKnvuFJaWipkRYT4+PiQ8ePHk8TERKarWWJiItHT0yNbt25l1baCggJ59+4d39q3OqvRIDQ0lIwcOZKcOnWKqKmpkZcvX5LY2Fiira1Ndu3aRU0HVxyTpqYmycvLo2avKc6cOUO0tLTInj17mM5/oaGhZPDgwSQ4OJhV242/nw0OiXYHRkVFRQEN3/rDJiYmJuTo0aPffU9gYCCZOXMmaxpOnz5N1NXViaurK5kxYwZRUlJiOoa9fv2a7N+/nygrK5OYmBjWNHyNgYEBiY+PF1i/fv060dXVZc3u9OnTyZEjR5ifHzx4QBQUFEhgYCCzdu3aNfLrr7+ypoFL/szCwoLPX718+ZIoKCgQFxcXZi0qKopMnz6dih4u+JL09HSioqJC5s+fT5SVlcnLly+Jg4MDUVZWJtevX6eigRBC1q1bR5YsWUIqKioY//7hwweycOFCsmTJEioauOJThenPGuPk5ERMTU3Jy5cvmc8kOzubGBsbCySv2IIL18jfRRRocRBtbW2yZ88ecvfuXaKgoEAuX74s9Az5uHHjSEJCAiGEP8i5ffs2GTlyJKu2uRZoGRkZMW2SG+uIi4sjenp61HRwwTERUt8y2d3dnfWdze9hbGzM7GI0/kwiIyNZfYgkhBsZ+5MnTzItmqOior77h000NDR+uLuZnZ1NtLS0WNURGRlJpk2bRmbMmEEuXrzIrDs5ORFlZWXi4+PDqv2v2b17N9HT0yPXr18nnz59Ip8+fSLXr18nenp6xNXVlTW76urqJDc3l/l53759RFFRkWRkZDBrL1++JCoqKqxp4JI/Gzp0KN//PSIigigqKvLt3mRnZ1NrXc0FX2Jubs48sDbW4ObmRi3gJIQQXV1d8ujRIwEdGRkZZOjQoVQ0cMWnCtOfNWbs2LEkNTVVQEdaWhoZPnw4FQ1cuEb+LsIvyhUhwMKFC7Fnzx74+/uDx+MJ1PU3QPOMVlFRkcDgUaC+nXVpaSnr9kNCQvjqpmtqahAWFibQnONbv6vm5NWrV1BSUhJYV1RUpDbEDwASExOxb98+vt9Lp06dsHHjRpibm1PT8fbtW5w/fx6hoaHo2rWrwAgAGsMXc3NzmzzUP2zYMDg7O7Nq29zcHOfPn4eUlBTs7e3h5+fHauvbpvjjjz8wcuRI9OzZE/n5+UI7Z9CyZUumO+j3YPNwOwDMmDGjyU5qS5YsgY2NDbUzHw0sW7YMWVlZWLJkCdOghBCCsWPHYu3atazabtwQJSUlBZ06dWKaggD1zTHY/K5wyZ99/vyZ79pMTk5Gy5YtoaGhway1aNGCysgSgBu+5NGjR9i2bZvA+rx583DixAkqGoD672FTg8aBen9PA674VGH6s8YUFxc3OZakY8eOqKiooKKBC9fI30UUaHGQZcuWwczMDKWlpTAwMEBkZKTQO67Iycnhzp07mDlzJt96bGws06SCLXr16oULFy7wrXXv3l3gAf57Trw56d27N9LT0/HLL7/wrd+4cUNgdhGbcMExAfU3/2HDhlGz1xTdunVDbm5uk/OrevTowaptCQkJREZGYtiwYUhKSkJSUtI3u3Oy1VWttLQUT548Qc+ePeHv74+5c+cKJdBSVlbG9evXIS8v/833XL16Ff3796eo6v/o2bOnUOy2atUKe/fuRU5ODrKyskAIgYKCAuTk5Fi1Ky8vj7S0NMjIyODjx4+4e/euQIfDCxcufPfz+rtwyZ/98ssvyM7ORq9evVBbW4vbt29jyJAhfM0GEhMTBe7tbMEFXyIhIYGysjKB9cLCQqr3kKFDh+LYsWN8jXKqq6sREBAATU1NKhq44lOF6c8aM3jwYFy4cAGLFy/mWw8PD8egQYOoaODCNfJ3EQVaHKV9+/Zo3749wsLCoKSkJPSOQDY2NrC1tUV2djZqa2sRHR2N3NxcXLp0Cbt372bV9rVr11j99/9TLC0t4eTkhLdv34IQgjt37uD48eM4fPgwMweDBlxwTACdXcQfYWpqCmdnZ+Z38ezZM9y6dQs+Pj5YsGABq7a5kLEfM2YMFi9ezHR009XV/eZ72dw1mDt3LtavX4+BAwdCT09P4PVr165h7969TWbQ/5eprq7Gixcv8OnTJ8jLy0NWVpb1XT2gfldi69atyMjIwL1791BVVcVcD0VFRTh79iyCg4Ph6urKqg6u+LOJEyfCw8MD1dXVuHXrFoqLizF9+nTm9QcPHsDPzw9z5syhoocLvsTQ0BA+Pj58fjwnJweurq4YO3YsFQ0AsHHjRsybNw9JSUmorq7Gtm3b8OzZM3z69AlHjhyhooErPlWY/qwxa9euhYWFBR48eICamhoEBAQgJycHjx49QnBwMBUNXLhG/i6i9u4cpPHsgh+1QXZ3d6ekqj6DEBQUhMePH6Ourg4DBw6EtbU1fvvtN2oauMLx48cREBCA169fAwAkJSVhbW2NRYsWUdOQk5ODefPmQVpaGk+fPsWwYcP4HFPj8iC2yczMRGhoKHJzc/HHH3/gypUrGDhwILS1talp2LVrF0JDQ1FZWQkAEBcXx+zZs2Fvb8/6Q21ZWdlPZezZmsNSXV2NmzdvorS0FPb29ti0aRM6dOjQ5HvZbrfv6emJgwcPQklJCZqamujYsSM+fPiAtLQ0ZGVlwdTU9F8TaL1+/Rq7du3C5cuX+Uoq27RpgwkTJmD16tWsZ6hPnjyJY8eOQUxMDFZWVsz92tnZGSdOnIC1tTVWr17Nmn0u+bOqqips3rwZZ8+ehZiYGObOnQsHBwcAgIeHBw4dOgRtbW3s379foASaLYTtS8rKymBlZYUHDx6grq4OHTp0QFlZGRQVFXHw4EF07tyZig6gPvg/duwYMjIymGeMuXPnUtth5JJPFaY/a0xmZiZCQkL4nvssLCygpqZGTYOwr5G/iyjQ4iBmZmbw9/dHx44dYWZm9t33Hj58mJKqb1NRUfHN7fbm5nvDcSUkJCAlJYWpU6di+fLlVOrsi4uLQQhh5omR/z/JnBZv3rzBsWPH+G6CNB0TADx8+BBz5syBuro67t27hwsXLiAoKAgxMTHw9/fHmDFjqGn5/PkzsrOzQQhB//79qc+w4sJMGi7MgomPj8exY8fw8OFDlJaWQlJSEhoaGpg1axZGjBghNF00yc/Ph6mpKcTFxTFt2jTIy8ujY8eO+PTpEx4+fIjTp08DACIjI4VS0lhUVISWLVuyfl6Ni/6sYYZY48G0SUlJKCsrg56eHtV7eAPC9iV37txh/Ii8vDxGjRoFMTExava5Ahd8agPC9mff4/Xr15CSkqJqU9jXyH+LKNAS8V0+f/6MxMREiIuLY+jQoQJDLa9fvw4nJyf8+eefVPSEhYVh586dmDNnDnNY9N69ewgPD8ecOXPQqVMnhIWFYdGiRbC2tm52+1lZWRAXF2/yjElmZiYcHBxw6tSpZrf7n/Lo0SMoKytTsbVw4UKoqanB1tYWGhoaOHPmDPr06QN3d3ekpqbi5MmTrNovKyuDuLh4kwNX37x5AycnJ/j7+7NmnwsZ+5iYGBgZGaFly5aIiYn57nuNjY1Z0SCCn99//x0vX75EcHBwkw9IZWVlWLJkCZSVlbFp0yYhKBQhTP4JviQ0NJRaqZqwdzt/BC2fKmx/BtSfz7t69SrExcVhYGAg0BDjyJEj2L17N1JTU1nV8U+4Rn4G0RmtfwAFBQXo2LEj2rdvj8TERMTFxUFTUxOTJk1i1W5GRgasrKyYLELv3r1x+PBh9OrVC6WlpXB2dkZsbCzrB7obExsbi02bNsHU1JRZMzQ0RP/+/ZkSmYEDB8LLy6tZA628vDwsX74c2dnZAABVVVUEBQWhc+fOqK6uhq+vL0JCQr7ZBKE5efDgAS5cuABxcXFMnDiRr5yhsrISPj4+OHz4MB4+fMi6FqB+R2vr1q0C62x3rSouLoa9vT1u3LgBHo+HX3/9FR4eHoyDOn78OLy9vVFdXc2aBqC+K1JdXR3zd2FgZ2eHUaNGoWvXrt+tW+fxeKwGWl5eXli5ciW1HW4uc+fOHXh7e38zC92+fXssW7YMW7Zs+VcFWsLyZ19jZmbWZDacx+PxVUc0dwMbrviS4OBgxMbGQkJCAlOnTsXcuXOZ154+fYrNmzfjwYMH1AKtr++dtbW1ePnyJcrKyjBx4kRWbXPBp3LFn925cwfLli1jSp137tyJI0eOQEFBAXl5ediwYQPu3bsHHR0d1jRw5RppNqg2kxfxHxMXF0eUlZXJrVu3yIsXL4iysjIZP348UVNT4xtEyQaLFi0ikydPJsnJyeT+/ftk/vz5ZNWqVeTZs2dk7NixzDyahvk9NBg8eDB5/vy5wPrz58/J4MGDCSGEFBQUMH9vLpYvX07Gjh1LoqOjSWxsLJk0aRLZtGkTeffuHTExMSEKCgrk999/JyUlJc1q92tiY2OJkpISUVNTI1paWkRZWZmZP5OWlkYMDQ2JgoICsbe3Z1VHY3R0dMiDBw8IIfxzLhITE1mdtfH777+TIUOGED8/P7Jv3z4ycuRI4uHhQSoqKsiSJUuIgoICmTdvHt8cIRHs0nhwcgPW1takqKhISIqEh7KyMsnPz//uewoKCoiysjIlRcJHmP7sa1xdXYmioiIxMTEhrq6uxNXVlcyaNYsoKCiQFStWEAsLC6KsrEyuXLnSrHa54Et8fHyIgoICMTc3J9bW1kRZWZkcO3aMEELIgQMHiIqKCtHW1ibR0dGsafgZ6urqiJOTE/H29mbNBld8Klf8mampKZk/fz4pKCgg7969I6tWrSKWlpYkLS2NaGlpkaFDh5LIyEhWNXDhGmlORIEWxzE2Nia7du0itbW1xM/Pj4wbN47U1taS2NhYMn78eFZtDx06lNy5c4f5+eXLl0RDQ4NMmjSJTJo0iW/wIy0mTJhADh48KLB+8OBBMm7cOEJI/QP+mDFjmtWujo4OuXbtGvNzZmYm0dbWJnPnziW6urrk+vXrzWrvWxgbGxMbGxtSWVlJqquriYuLC5k7dy65cuUKUVZWJgYGBuT27dtUtDSwefNmYmFhQUpLS/mmx0+ePJlV56Srq0vOnj3L/JyamkpGjRpFli9fTtTV1Ul4eDhrtr9Hfn4+M7z5zp07xMnJiU8nbd6/f08uXLhA8vLyWLfFteHiwqSp38XXvH37ligqKlJSJHyE6c++xsbGhri4uAise3h4EFtbW0JIvV+ZMWNGs9rlgi8ZN24c8ff3Z36Ojo4mRkZGZM+ePURBQYGsXr2avH//nnUdP8OLFy+Ijo4Oa/8+V3wqV/yZpqYmuXfvHvPz27dviaqqKtHT0yOLFi0ir1+/Zl0DF66R5kQUaHGcwYMHk1evXhFCCJkzZw7Zvn07IaT+Ya65d22+RklJSeCiUlFRIQsWLCCfP39m1fa3OHPmDBk0aBBZu3YtOXz4MAkNDSVr164lgwYNIidPniTPnj0jenp6xNXVtVntDho0iBQWFvKtKSsrE2Nj4x8+TDUn6urq5PHjx8zPpaWlRFlZmejo6BB7e3tSXl5OTUsDnz59IqampkRJSYkoKCiQIUOGEEVFRWJsbMxqxklZWZm5NhoYNGgQ0dPTI9nZ2azZ/R5cyNg/efKE/PrrryQpKYmUlpYSXV1doqCgQAYPHsyXOGEDUaD1fygoKPzwYfXfFmgJ0599jbq6Onn27JnAem5uLlFXVyeE1CcXG/7eXHDBl6iqqvL93ysrK4mioiLR0tIS+i7W19y4cYNoamqy9u9zxadyxZ8pKiqSN2/e8K2pqqqSNWvWkNraWioauHCNNCeiM1ocp6FL1adPn/DgwQPm3NHLly9Zb7taV1cn0D1NXFwca9asafKgJg0mT56M9u3bIyQkBLt27YK4uDgUFBQQGBiIUaNGITk5GZMnT2722U61tbWQkJDgW5OQkICdnR3TAYcGnz9/5juY2rFjR6aufPPmzdR0NKZ9+/aIiIig3rWqpqZG4HvYsmVLbNmyheq5wcbs3bsXlpaWGD58OAICAtCrVy/Exsbi4sWL8PX1xbx581jX4OnpCRkZGfTv3x/nzp1DTU0N4uPjERERAR8fH0RERLCuQUQ927dv/26b8IbWzf8WhOnPvqZ9+/Z49uwZ+vXrx7eenZ3NdOwsLy9vdl/HBV9SWVmJjh07Mj+3bNkSrVu3xtq1a4XWLKepZhjl5eVISEgQGLLdnHDFp3LFnxFCBPy2mJgYFi9eTK0LJReukeZEFGhxnDFjxmDLli1o164dOnToAF1dXdy+fRvbtm2jOkywMcL+ouvp6TU5DBWoHzjY3IeXv0evXr2o2Wrg6wPcPB6PrzkIbczNzeHn54fhw4dj+PDhzPr79+9haWn5wy54zY2wgiygfg6Ln58fxMTEkJCQgDFjxkBMTAzq6urIz8+nouHevXuIjIxE165dcfPmTYwZMwY9e/bEtGnTcPDgQdbt/xPa7dJg6NChePv27Q/f19A99d8Al/zZtGnT4OjoiOLiYqipqaGurg7379/Hnj17MHXqVJSUlMDLy4uaPxGGL/kaYY5eaKqRUMuWLbFw4ULW5yVxzac2Rpj+rDFcaC3PhWvkv0EUaHEcR0dH+Pj4IC8vDwEBAWjZsiVSU1Ohrq6OjRs3smqbx+M1eQMSNikpKUhLS0N1dTXIV9MJmnsnq4GmfhdcgvYOY3x8PNLT0wEAycnJCAwMFOg09+LFC1aDCy5+JlzI2IuJiaFly5aoqalBUlISHB0dAbCTnW+Kr3dxqqursWPHDr6ZRYDw2zWzDRdmHHINYfqzr1m9ejWqqqrg6uqKyspKEELQunVrmJmZYfXq1bh+/ToqKiqwffv2ZrXLxftWAzQH4X4N164X2j6VS9+L169fC+y2FxUVCXw/2Ap8uPS7aA5Ec7REfJOmhgOTbwyIy8jIoKLJ398fvr6+THvgxvB4PFy9epUVu4qKipCWlubbOi8oKEDPnj0Fbj5saWjQ4efnx9fW1NraGq6urujRowffe9nMxGZnZ2PJkiUghKCwsBA9e/bk+93weDy0bdsW5ubmmDlzJisaFBUVoaGhwVdikJKSgsGDBwuUa4WFhbGi4WscHBzw9OlTtGvXDpmZmYiPj0dKSgq2bdsGHR0dODs7s67BysoKUlJSkJSUREhICG7cuIHq6mo4OjpCTEwMgYGBrNn+0UDaxnDtwaq5KSgogLS0NHg8HgoKCr773n9qpvZ/gS9fviAnJwctWrSArKws6w/YXPAlioqKAkPNg4KCMHv2bIGW2WwlL78mNzcXaWlpeP/+PTp16gQ1NTWmzfr169fRunVrVlqKc8WncsWf/cxzX8PPbD33ceEaaU5EgdY/gLNnz2Lo0KGQkpLC3r17cf78eWhqasLBweG79f9/l+jo6J9+r4mJCWs6GjNy5EiYmZlhyZIlVOw14Ofn99PvZdMxNdwEf3TZsnkT/Bp9fX2cPHkSkpKSVOw18KMBl42htXvy5csXJmNvbW0NdXV1+Pr6Ii8vD1u3bhXY1WGDFy9ewNbWFnl5ebC1tcXcuXPh4uKC69evY//+/U0OfxTR/CgpKeHWrVvo2rVrkw8vAPsPLFxEWP6sKT5//oysrKwmqyPYeqjmgi/R19f/qfexmbxs4M2bN3BwcMCtW7f4PgMejwdNTU24uLhg6dKl2LBhAwwNDZvdPld8Klf8WVJS0k+/V1tbmxUNXLhGmhNRoMVx9u7di8DAQBw6dAiEEMybNw8zZ85EUlISRo8eDQcHB2FLpIqamhrOnz+P3r17C1tKk3xrx6+5+E9K8YT1O6qurkZmZib69+9PJbAQ8WOKi4vRqVMn1kuDzM3Nf+p9PB4PoaGhrGoRNklJSdDU1IS4uPgPH17YemDhGlzyZ1evXoWdnR3KysoEHrL/bcGvsCgrK8PMmTNRV1eHFStWYPjw4ejSpQtKS0tx9+5d+Pv748WLF1BTU0N4eDgrGv4JPlXEPxvRGS2Oc+rUKXh6ekJTUxNubm5QV1eHi4sLUlJSYGtrS9UxpaWlQVZWFpKSkoiJicGFCxegqamJxYsXU6un1dTUxL1794R6w9u+fTvs7OwEOjLm5eXh999/Z7WrGxdv9IWFhXBwcMCaNWsgLy+P6dOnIycnB506dcKhQ4egpKRERceXL1+Y80k5OTm4fv06NDQ0oKmpScV+A1zI2BcUFDDltYmJiYiLi4OmpiYmTZrEqt0ffT9TUlKQl5fH1/Hsf5XGwZO2tjaeP3+OsrIyqKioAABCQ0MxZswYyMrKCkkhfbjkz7y9vTF8+HAsX74cHTp0oGb3R2WkjflfLyk9dOgQgPrvReOjAF27doWRkRGkpaUxZ84cqKqqsqaBiz4VEPmzxgjLnzUXokCL47x58wYaGhoAgNu3b2P8+PEAAGlpaXz8+JGajoiICDg5OSEkJARdunSBvb09hg8fjkOHDqG6upra9u2kSZPg4uKChw8fon///mjZsiXf6zRa0547dw6pqanYtWsX0xr4xIkTcHd3F2gVzCZVVVUICQnBhAkTICMjAwcHB+Ym6O3tjS5dulDR4e7ujk+fPkFSUhIXLlxAYWEhjh49iqioKOzYsQMhISGsa0hOTsaKFSvwxx9/QE5ODjNnzoSYmBg+f/4Mb29vTJgwgXUNAH/GPj8/H3v27MHMmTNx9+5deHt7U3mQvHz5MmxtbREUFIQ+ffrAysoKffr0QVRUFEpLS1ltMf+tkpaysjJ4eHggLy8Purq6cHV1ZU0DF7l9+zaWLVuGhQsXMoFWbGwsfHx8sH///n9N50Gu+DOgvstdUFAQ+vbtS9Wuvr7+TycmaeyqvX//Hjt27MDDhw/x5csXgd09NksHL1y4gFWrVn2zo52/vz80NTXx559/UmuWcvXqVWRlZaG2tpZZq6qqQnp6OpWurYDInzVGmP6s2aA4s0vEf4GhoSG5ffs2ef78OVFQUCBpaWmEEEJiYmLIr7/+Sk3H+PHjmYGrO3fuJJMnTyaE1A8T1NPTo6ZDQUHhm39oDf4sKioiFhYWRF1dnYSFhZElS5aQwYMHk4CAAFJTU0NFAyGEuLq6Em1tbfLw4UMSHx9PlJSUSEBAADE1NSV2dnbUdAwdOpQZ+Lh69WqyatUqQgghz549a/Zhn99i9uzZxM7Ojnz69IkEBweTUaNGkS9fvpDw8HAydepUKhoIIURfX5+cP3+eEFL/+ZiamhJCCElOTiYjR46kosHY2Jjs2rWL1NbWEj8/PzJu3DhSW1tLYmNjyfjx46loaExCQgLR09MjQ4YMISdOnKBunwtMmzaN7Nq1S2B9586dzHfk3wBX/BkhhEyaNIncvXuXqk1CCLl79y7zJzQ0lGhra5NDhw6R+/fvk8ePH5Pjx4+TUaNGkWPHjlHRs3z5cjJixAiyfft24uvrK/CHTdTU1EheXt43X9+5cyd5/PgxUVNTY1VHAzt27CAKCgpEV1eXKCoqkjFjxhBlZWWiqKhI1q1bR0UDISJ/1hiu+bP/BlGgxXEOHDhAtLW1yciRI8mUKVMIIYQcOXKEqKqqkgMHDlDToaKiQgoKCggh9Q8NO3bsIIQQkp+fTwYPHkxNB5ewt7cnCgoKRFlZmSQnJ1O3P2rUKHLr1i1CCCFbtmwhCxYsIIQQkp6eTnR0dKjpUFdXJ/n5+aSuro5oa2uTiIgIQggh2dnZZMiQIVQ0qKqqkpcvXxJCCFm0aBFxdHQkhBDy6tUrqt9PFRUVZqL9xIkTmQeVV69eEVVVVSoaBg8eTF69ekUIIWTOnDlk+/bthBD612p5eTlxdHQkCgoKxMLCgrl//BtRU1Njvp+NefnyJbVkBBfgij8jhJDr16+TSZMmkatXr5Lc3FySn5/P94cGU6dOJXFxcQLr165do/YQqa6uTpKSkqjY+poRI0aQjIyM774nIyOD6OrqUtEzevRoEhoayvw9Pz+flJSUkLlz5xIfHx8qGggR+bPGcMWf/R1EpYMcx9LSEv369UNeXh6mTJkCoH5Wj6OjI2bMmEFNR9euXfHmzRuIi4sjIyMD69evBwBkZmaiW7du1HRwgfLycnh5eSEmJgbGxsZ48uQJVq1aBUdHR2pb+gDw4cMHZphhQkICM2Cxc+fO+PLlCzUdgwYNwsmTJ9G9e3d8/PgRY8aMQVVVFfbv38+052WbNm3aoKqqCpWVlUhNTWVayr97947q+QspKSnk5uaisrIS2dnZ0NXVBVB/NklKSoqKBi7M8rpz5w4cHBxQWloKZ2dnzJo1i4pdriIpKYnMzEz06dOHb/3p06dUv5/Chiv+DACWL1+O2tpaLF++nGrr6sbk5uZiwIABAut9+/ZFYWEh6/aB+nlR3bt3p2LrawYPHowLFy5810+cO3cOampqVPS8f/+e6ciooKCABw8eYPz48cz5wdWrV1PRIfJn/wcX/NnfRRRo/QP4uhXr5MmTqWuYOHEi1q9fjzZt2kBKSgra2to4f/48XFxcWHeQP9MmuQEaztHIyAg1NTXYs2cPDA0NUVNTgz/++APr1q3D+fPn4evry7oGoN4Zp6en4/3793j16hVGjRoFALhy5Qp++eUXKhoAYOPGjVi6dClKSkpgbW0NKSkpbNu2DVevXsWBAweoaBg2bBh27NiBTp06QUxMDKNGjUJGRga2b9+OYcOGUdEAALNnz8aaNWvQsmVLKCgoQENDA+Hh4fDy8sKqVauoaBgzZgy2bNmCdu3aoUOHDtDV1cXt27exbds2jB07llXbFRUV8PLywvHjxzF8+HC4urpCWlqaVZv/BKZOnYpt27bhw4cPzENjeno6fHx8qJwr5RJc8GcAEBISIvShqAoKCggLC8OWLVsYLTU1NQgKCsLgwYOpaDAxMUFwcDBcXFyo2GuMubk5Fi9eDAUFBRgZGQm8HhUVhdDQUGpnozp27IiKigoA9f41OzsbQH1TkqKiIioaAJE/a4ww/VlzIWrvznEqKytx/PjxJg9nPnz4EJcuXaKio66uDuHh4cjLy8O8efMgIyODw4cPo7i4GDY2NnyD5Zqb6OhoTJw4ES1btvzhbC8a87yWLl0KV1dXdO3alW89NTUVdnZ2uHz5MusaACAmJoYZQquhoYFDhw7B398f/v7+cHNzo/oAV1dXh7KyMqabXG5uLrp06UIt41RcXIxt27bh5cuXWLlyJQwNDeHh4cE8zNLM2F67do3J2Hfp0gVnz55FZWUltYz992Z5bdmy5ZsHz5sDAwMDFBQUoE+fPsyOxbf4J8w/aS5qamqwfft2nDp1CtXV1eDxeGjRogXMzMywdu1aviGl/8twxZ9xhZSUFFhaWqJ79+4YNGgQ6urq8PDhQ3z+/BmhoaGsVQQ0HsNQU1ODtLQ09OjRA3379hXw5WwPew8KCoKPjw8UFRWhpaWFDh06oLS0FMnJyXj27Bl+//13LFy4kFUNDdja2qK8vBwuLi5ISkpCUFAQwsLCcPr0aRw+fBjXrl2jokPkz/4PLsym/LuIAi2O4+joiJiYGAwaNAjp6enQ0NDAixcv8P79eyxcuJBaJx4A+PTpE0pKSpguTXFxcdDW1qa6fbt9+3aYm5tT7xT1s1RUVKBt27bU7GVmZuLVq1cYPXo0WrZsiRs3bkBCQgLDhw+npoELnDt3DiNHjuT7LlZVVQl0pRTBLlwahMpFysvLkZOTgxs3bkBHR+df022wAWH7M3Nzc/j5+aFjx44wMzP77o4W2wFGA3l5eThx4gSePn0KoL6CY86cOejRowdrNrkyHLeB1NRUHDlyBKmpqSgpKYGkpCSGDBkCMzMzqKurs26/gcLCQixbtgzGxsaYO3cuZs+ejcePH4PH48HOzg4LFiygooPL/qy4uBiSkpLU7CUnJ0NdXV0gGVVZWYnr16/jt99+o6blv0UUaHGcESNGYNOmTZg0aRLGjRvHtLi0tbWFlJQUNm/eTEXHo0ePYGFhgWnTpjHOUF9fH9XV1QgODoa8vDwVHVpaWjh9+jTV0rimiI+Px4EDB5Cbm4vjx48jKioKffv2xdSpU4WqSxhwoZxTW1sbR48ebfK8A024krHPzMxEVlYW6urqANSfO2loUbx9+3YqGkTUt6cOCwvDiRMnICMjg3v37sHa2hplZWXg8XjQ0dFBQEAAWrduLWypVBC2P7O3t4eDgwPat2//w2CDRoAhgptUVlaiVatW+Pz5M27duoWePXuyOsvra7jizz5+/IgdO3Zg/vz5GDBgAKysrJCYmAhZWVns27dP4MwpGygpKSEhIUEguHv06BHmzJmDBw8esK7h7yI6o8VxPn78yAyoGzBgAB4/foz+/ftjyZIlWLNmDbVAy8PDA/r6+rC1tWXW4uLi4OjoCA8PDyqzkoD6et0jR45g5cqVrJZAfY+EhASsXLkSEydOxP3791FXV4eamhrY29uDEMJqyZ6BgQFOnjyJLl26/HAeC60dAzc3Nz4dNTU1eP78OWJiYrBhwwYqGmRlZZGVlSV0x7R9+/bvZuxpcPDgQXh6egKo3zlqyKXxeLx/3Q6KMDl+/DgCAwOxcOFCpszY3t4erVu3RkREBDp06AAbGxvs27eP2nkHYSNsf9Y4eNLV1RXYNaBB4121xiV8TUFrVy0tLQ2ysrKQlJRETEwMLly4AE1NTSxevJjqObb8/HycOHECT548QYsWLaCsrIxZs2ZRa7hVV1eHvXv3olu3bpg9ezbatGmD/fv3Q19fn2qgxRV/5u7ujpSUFCxcuBCXL19GSkoKvLy8cP78eXh5ebF2Hv3QoUOMDyOEME04vobmZ/J3EAVaHEdSUhLv379Hr169mIsPALp06YJ3795R0/Hw4UO4ubnxbV2Li4tj8eLFVLtFvX37FufPn0doaCi6du0qMJmcRnDh6+uLdevWYeHChcwOha2tLdq3b4/g4GBWAy0TExMm+21iYiL0w9wAMG3atCbXVVRUEBkZSWWXT1FREevXr8eBAwcgKysr8L2glZ2+evUq3N3dmYy9i4sLk7Gvrq6moiE8PBzW1tZYuXIl9PT0EB0djQ8fPmDdunUwMDCgokEEEBkZCTs7O2agZnp6Op4/fw5bW1vmAWrZsmXw8PD41wRaXPFnAODs7IyjR49SD7R69+7NnIPq3bs3VdtNERERAScnJ4SEhKBLly6wt7fH8OHDcejQIVRXV1M7S5mWlgYLCwt06dIFKioqqK2txfHjxxEaGoojR45g4MCBrGvYs2cPIiIi+BqDTJw4EQEBAQDqz2fTgCv+LD4+Hv7+/pCTk8P+/fuhq6uLyZMnQ0FBgdVBwfPnz0fnzp1RV1eHTZs2wd7enq/bIo/HQ9u2baGjo8OahuZEFGhxnNGjR8PJyQnu7u7Q0tKCm5sbxo0bh/Pnz1NrrwkA7dq1Q15ensBW8Zs3b6jWDQ8bNoxq152mePLkCby8vATWx48fDz8/P1ZtN3Z6NjY2rNr6u6iqqsLOzo6KrdzcXGhpaQGoD8aFhbAz9gDw+vVrzJw5E61atYKioiLS09NhaGgIOzs7eHh4UNtZ+7eTk5PDl4lNTEwEj8fDmDFjmLUBAwagoKBAGPKEAlf8GSC8XYPGD8lcKE8MDQ3F5s2bMXz4cOzatQsDBw5ESEgIbt68ia1bt1ILtDw9PTFhwgS4uLhAXLz+0bS6uhr29vZwdXXFoUOHWNcQExMDb29vjBw5kllbsGABZGVl4ezsTC3Q4oo/q6ioYLrGJiQkMK3VW7duzVca39yIi4szCWsej8c0Q/unIgq0OM6GDRtgZ2eHpKQkzJ07F8ePH8fMmTMhLi7ObK3S4LfffoOTkxO2bdvGbNemp6fD2dkZ48aNo6aDC53KOnTogDdv3gg05MjOzkanTp2oaMjJyWFmaO3btw9VVVXMa6qqqhg9ejQVHd+ivLwcR44coVbycfjwYSp2fgQXMvZt27ZlnGBDi2JDQ0PIyckhPz+figYR9TTecU5JSUGnTp34OsmVl5ejTZs2wpAmFLjizwDu7BoUFhYiPDwcWVlZEBcXx8CBA2FqaopevXpRsf/q1SumiU1CQgLjO+Tk5KjuMmZmZsLd3Z0JsgBAQkICS5cuZeZIsc2HDx+a3GWUlZWlGvBwxZ/Jycnh+vXrkJaWxtu3b5nvxokTJ5jnD7YxMTFBcXExcnNzmzxzvGzZMio6/g6iQIvjdOzYEXv37mV+3rdvHzIyMtCtWzdWuxJ9zbp16/Dy5UssWrSI7+Fh3Lhx1M7hNJCZmYnQ0FDk5ubijz/+wJUrVzBw4EBoa2tTsT958mS4ubkxZ5PKy8tx48YNuLi4NDkLpLnZsmULIiMjcfHiRcjIyCAgIAAdO3ZEixYtUFFRgZqaGly8eJFakPOtZhg8Hg9OTk5UNAD1bWAvXryIZ8+ewcLCAllZWRg4cCC6dOlCTQMXMvaamprYt28ftmzZwgyTXrx4MVJTU/8RrXD/V5CXl0daWhpkZGTw8eNH3L17V6B088KFC9QaCXEBrvgzgBu7Bk+ePMH8+fPRunVrqKqqoq6uDlFRUQgPD8exY8eolMt17doVb968gbi4ODIyMrB+/XoA9X6Wlg8B6pNCmZmZ6N+/P996fn4+tVl8ioqKiIqKwrp16/jWT58+TX3nkwv+bNWqVbCxsUF1dTUmTZoEWVlZuLu7Izw8HP7+/lQ0nDlzBps3b2ZK7xsGigP1pbf/hEBL1HWQg/wnpSS0sl4NPHv2DFlZWZCQkICcnBxkZWWp2n/48CHmzJkDdXV13Lt3DxcuXEBQUBBiYmLg7+/PV5bTnDTuVlVdXQ07OzvExsYC+L+GA2PHjsUff/whkBltTiIjI7Fjxw7s2rWLKW/Q0NDAmTNn0KdPH5SWlmLy5MmYOXMmtdLCqKgogUBLQkICampqVLoSAcC7d+9gamqK9+/fo6qqCpcuXYKrqysePnyI0NBQatm3jx8/ws7ODrq6upg7dy4WL16MmzdvMhn7iRMnsq4hKysLFhYWWLhwIebMmYPJkyfj48eP+Pz5MywtLbF27VrWNYiof0DYunUrZs6ciXv37uHRo0eIiIiAqqoqioqKcPbsWfj4+MDV1fV/ulspl/2ZsLG0tESbNm2wc+dOxm9UVlZi/fr1qKqqQlBQEOsaduzYgbi4OLRp0wZlZWW4fPkyLl26BBcXF8yYMUMg6GCL06dPw9PTE4sXL8awYcMgLi6O9PR07N69G3PnzuVr5DN06FBWNNy8eRNLliyBmpoa01Y+PT0df/31F6vPF1/DFX8GACUlJSgqKmJ24h88eIB27dpR02BkZAQ1NTVYWVlhzpw5CAkJwZs3b+Dk5IS1a9f+I+6dokCLg/yoXTbwf1E9jdbZXGLhwoVQU1ODra0tX4Dh7u6O1NRUnDx5khW7SkpKuHXrFt+Q4pcvX+Lx48eoq6uDvLw8lYzX7NmzMXnyZL6DqJqamjh9+jQT1ISGhuL06dOIiopiXQ9XWL9+PcrKyrB7926MGDECZ86cQceOHbFmzRq0atUKgYGBQtFFCBFKxv7Lly+oqKiApKQk3r17h7Nnz0JaWhrjx4+npkEEcPLkSRw7dgxiYmKwsrJiZr44OzvjxIkTsLa2xurVq4Wskl247M+EvWugoaGBiIgIKCgo8K1nZmZi/vz5SElJYV1DXV0dwsPDkZeXh3nz5kFGRgaHDx9GcXExVq5ciRYtWrCuAcBPD2dm+3ty7949HD58GE+fPoW4uDjk5ORgZWXF2vDopuCaPysoKEBOTg6GDh2K8vJyvucgthk8eDBiYmIgJycHMzMzWFtbY/To0YiLi0NgYOA/4jlHVDrIQWi1dP0RjYMLLsxKAup3tLZu3SqwPm/ePJw4cYI1u03lI/r27Ut9cPLTp08FWp1+rW3kyJHw8fFhVcd/0vSDxrm6xMRE7Nu3j++8S6dOnbBx48YftlD+u/woY9+5c2fU1NSgoKCAWsa+devWTHfKbt26YdGiRVTsiuBnxowZTXZlXbJkCWxsbKiWAQkLrvizr/l612DmzJkICQmhumvQrl27JruR0upQCgBiYmIwMzPjW/v6ZxpwZYC5hoYGNDQ0hKpBmP6sMVVVVdi4cSMuXLgAMTExXLp0CZ6enigvL4evry+VETstW7ZkGmHIyMjg6dOnGD16NFRUVPDixQvW7TcHokCLg3zvrBHN6eBubm5MS82vZyUJCwkJCZSVlQmsFxYWsn6onAv//7q6OoEJ6ZcuXeKrpW/ZsiXrWcifzSLxeDwqgVZ5eTnatm3b5Gs1NTWs2v7RPDOA/Yz9z2hogCsPNP9mevbsKWwJ1OCKP/saDw8PDBw4EOfOncOIESMA1He+W7NmDXbs2EFl10BHRwdeXl7Ys2cP02a+uLgYO3bswPDhw1mzy8XBzcJqde/n58eUcP4ogUirGZcw/VljAgICmDPxDR0XzczMYG9vD29vb2zbto11DQ1jYtauXQt5eXnEx8fD0tIS2dnZAs9CXEUUaHGUnJwcuLu7Y8uWLXy7Jhs3bsTHjx+xdetW1ndTTExMmL9/a1YSbQwNDeHj44Pdu3czazk5OXB1dcXYsWNZtf2toXlfw+bunrS0NLKysvic0tclaQ8fPsQvv/zCmgYAuHbtGqv//n/K0KFDcezYMb6HhurqagQEBDDt1tmCCxl7rsxUEyGiKbjgz76GC7sG69atw5w5c6Cnp8ecd37+/Dk6d+4MNzc31uy+evWK6eD26tUr1uz8JwhreHNUVBTmzZuHNm3afDeBSCtpCAjXnzUmNjYW27Zt4xupM2zYMLi6umLDhg1UAi0bGxtYWVmhc+fOMDExgb+/PyZOnIjCwkJMmDCBdfvNgSjQ4iAvX77EvHnzICkpicrKSr7XxowZgwMHDmD27NmIjIyklgWqqqpCSEgIJkyYABkZGTg4OOD8+fPQ1NSEt7c3tRKYjRs3wsrKCjo6Oqirq8O0adNQVlYGRUVF1rsffj00Txjo6ekhKCgIo0ePbnLXqqamBsHBwdRa7r9+/RrdunXja8mbkpKCPn36UM3ab9y4EfPmzUNSUhKqq6uxbds2PHv2DJ8+fcKRI0dYtc2FjD3XZ6qJ+PfCRX8GcGPXQFpaGrGxsTh9+jSePn0KQghmzZqFyZMns1qW1bh9OFdaiX/92dfU1ODFixfIysrCggULWLPbOGn4vQRiQ2BKA2H6s8YUFRU1mQCRlpZGaWkpFQ1DhgzBpUuXUFVVhS5duiA8PBwRERGQlpamWkb5tyAiOMeGDRuIhYUFqaysbPL1iooKMnv2bLJp0yZqmlxdXYm2tjZ5+PAhiY+PJ0pKSiQgIICYmpoSOzs7ajoauH37Njlw4ADZt28fuX79OqmtrWXVnoKCAnn37h2rNn6Gd+/eEV1dXWJqakpSUlL4Xrt//z4xMzMjenp65NOnT6xr2bdvH1FRURHQYW5uTpSVlUloaCjrGhpTVFREdu/eTRYvXkysrKyIp6cnycvLo2I7OzubWFpakhcvXvCtr1mzhlhYWAiss0FFRQUJCgoi79+/51v39fUlQUFB5MuXL6xrECHia7jozwghxNramri5uRFCCFFXVycvX74kVVVVZNWqVcTS0pKKhunTp5OMjAwqthqTn5//03+EjZ+fH9m8eTMVW/r6+qSkpERg/fXr10RbW5uKhgaE6c8aMDExISdOnCCE/N81Qkj9Z2JiYkJFg52dXZPPMyUlJWTZsmVUNPxdRDtaHOTu3bvYtWvXNzPhbdq0wcqVK+Ho6EhN08WLF7Fr1y4oKytj69at0NbWxtKlSzFy5EhmWjgNzM3N4efnh+HDh/PVsL9//x6WlpaIiYlhxS5XyrK6du2KgwcPYv369Zg3bx5at26NTp06obS0FJWVlRgwYAD279/P+iHVuLg4+Pj4YPny5QIds/z8/BAaGgpPT0/07duX9ZLOBpuWlpZYs2YN33pZWRlcXV3h4ODAmm0uZOzLysqwcOFCZGZmYsiQIZCUlGRe+/TpEyIiIvDnn38iODj4m1l8ESLYgIv+DODGrkFeXp5QrkcunCv9WaZOnQpjY2O4uLiw8u+fP38eN2/eBFA/s8vZ2VlgREt+fj71Z4AePXoI+DPa2NjYwNbWFtnZ2aitrUV0dDRyc3Nx6dIlvuMbzU1qairy8vIAADExMVBWVhZ4psnJycGdO3dY09CciAItDlJSUvLDsisZGRkUFxdTUlQ/Mb2hC1NCQgJMTU0B1HdU+/LlC6u24+PjkZ6eDgBITk5GYGCggHN68eIF8vPzWdNAODQFYeDAgYiJiUFiYiKSk5Px7t07dOnSBVpaWhg5ciTExMRY13Do0CHY2NgwB2Qb06FDB6xcuRIVFRUIDg5mLdDKyclhrgF/f38oKiqiU6dOfO/JysrCiRMnWA20/P39oaysjICAAIGHSWNjY/z222+wsLDA3r174erqyoqGAwcO4NOnT7hw4YLA7DJ7e3vMmjULVlZWOHjwIFasWMGKBhEimoKL/gwA5OTkcObMGRw9ehQ9evRAXV0dJkyYgLlz57J+xrUBKysrODg4wNLSEn379mU6hTbAVpdSLpwr/Vnu3bvHanOnhhb7DT6+oKCAr8kCj8dD27Zt4enpyZqGrykuLsb+/fvx9OlTVFVVCbxO6/PT09PDnj17EBQUhBYtWiA4OBgDBw7E7t27mVEVbMDj8WBnZ8f8ffv27QLvadu2LSwtLVnT0JyIAi0OIiUlhefPn383+/38+XPqU9vT09Px/v17vHr1CqNGjQIAXLlyhXWn1Lt3bzg7OzM3wvPnz/MFEw03QjbPaGVmZjJ//965m8TEROjo6LCmowEejwctLa1vdqZiW8fTp0+bvPk1ZsqUKTh16hRrGvLy8rB06VIm0/itg8rTp09nTQPAjYz9hQsXsGHDhm8OiJaTk8OaNWuwb98+UaAlgipc9GeAcHfBG/Dx8UFtbS2Sk5P5dkzY3k363rlSYdHUeZuysjI8efIEc+fOZc2utLQ0E7iYmZnBz89PIGFHmw0bNiA9PR0jRowQCL5pEhwcjEmTJiE8PJyqXU1NTeaZS1FREQkJCVRndzU3okCLgxgYGCAgIICZjv41NTU1CAoKYlrS0sDKygpr166FmJgYdHR0oKioCH9/f/j7+7PaHQkABgwYwLSk1tfXx8mTJ/lKo2izbNkygd2LiooKeHh4IDIyklqpxbJlyxAYGMiXfaOp40e7fG3atEFtbS1r9seOHYtr166hrq4OhoaGiIyM5PteNATgDW2T2YILGfvXr1//cKCmpqYmq7u+IkQ0BZf8GVd2wRs4ePAg6zZ+hJmZWZNlcTweDxISEpCSksLUqVMxdOhQVnX06tVLQIeEhATmz5+PKVOmsGq7ge81Bnn9+jWkpKSo6EhNTUVQUJDQA+KAgAAYGhoKVUPjJPc/FVGgxUGsrKxgbGyM+fPnY/HixdDQ0ECnTp3w4cMHpKWlYf/+/cjLy4OHhwc1TcbGxlBSUkJeXh5Gjx4NoH5id3BwMKvzPr7m665A1dXVyMzMRP/+/dGuXTsqGvLz87Fy5Ur4+/tDQkICt2/fxubNm1FeXv7DXZ7m1rFixQqh6JCXl8fdu3e/O9QzISGB9ZbNDaU1V69ebdJR04ALGfvOnTvj/fv339VQUlIi9K6ZIv59cMmfcWUXvAFhP0gDgJKSEg4fPgwlJSUMGTIEAHD//n3cv38fhoaGKCwsxKJFi/DHH3/AwMCANR00n2e+RV5eHjw9PZGVlcUkCQkhqKqqQnFxMR4/fkxFR8+ePak9z3wPNTU1XLt2TSgD7589e4bIyEhYW1tDUlISZWVl2LJlC65fv45u3bphxYoVmDp1KnVd/w08wqXDJyIYcnNz8fvvv+Phw4cCJQUaGhpwcXHBgAEDqOnZsWMHpk+fjv79+1Oz2RSFhYVwcHDAmjVrIC8vj+nTpyMnJwedOnXCoUOHoKSkxLqGt2/fYtGiRejduze6d++OkydPwsjICA4ODlS3t4WpIzo6Gl5eXjh48GCTOykZGRlYuHAhli9fzmpr3gaqq6sRHBwslPEDXl5eePDgAQ4dOvTNjP2iRYvQr18/ODs7s6Jhw4YNaNmy5XcDbAcHB5SUlGDv3r2saBAh4ltwyZ8VFBQIdRf8RwOCG0NjWPCqVavQo0cPbN68mW/d09MTRUVF2LVrFw4dOoTY2FhERkayoqGurg737t1DZmYmysrK0KFDBygrK0NNTY0Ve9/C2toaz58/x/jx43Hw4EFYWFggNzcXly9fhrOzM2bNmkVFx9WrVxEUFARbW1v06dNH4Nw1W2f3vmb16tWIi4tDx44dISsrK9AkhK2zYhkZGZg7dy46d+6MI0eOoHfv3li7di0uXryIhQsXokOHDjhw4AB27NgBfX19VjQ0J6JAi+M8evQI6enp+PjxI7p06QJNTc3v7iKwhampKR48eIDBgwdj+vTpmDhxIuud7Zpi1apVKCwsxO7du5GcnAwXFxccOHAAUVFRKCgoQEhICBUdxcXFWLRoEZ4+fYo9e/YIbXtdmDpWr16Nq1evYuzYsdDU1ETHjh2ZLPWNGzcwcuRI7N27l0pzDjc3N5w+fRohISF4//49li5dilWrVuH69evo168fqw8sxcXFMDY2Rq9evb6bsWez62BmZiZmzZqFefPmYenSpXzlUB8+fEBgYCCOHDmCQ4cOMVlrESJowxV/BtRXBAhjF9zMzIz5OyEEKSkp6NatGwYNGgRxcXFkZmaiqKgIBgYG2LNnD+t6NDQ0EBUVhX79+vGtP3/+HCYmJrh37x7y8vIwZcoU3Lt3r9nt37x5E05OTsjPz+crR+fxeOjTpw+cnJyoVc1oaWlh7969GDZsGIyNjeHs7AxVVVXs3r0b2dnZ8Pf3p6LjypUr2LhxIyoqKvjWaXeC/FFSgC2/amNjAx6Ph127dkFcXBxFRUUYO3YsjI2NGZvh4eG4cOEC1bli/y2i0kGOo6ysDGVlZb614uJi6meUjh8/jtzcXMTExCAoKAhubm4wNDSEiYkJdHV1qTmrxMREhIaG4pdffoG3tzdGjRoFTU1NdOnSBdOmTWPNblNt442NjbF7926cOnUKZWVlfOv/6zoA4I8//mCGB165cgVAvXNUUVHB1q1bMWPGDGrfC2GOH5CUlERoaCh+//13LF++vMmMfVhYGKvDWBUVFeHt7Q17e3scPnwY/fr1YwLf58+fo23btvDw8BAFWSKEyv379zF58mROlLD26NEDQUFB1HfBG58D8vb2Rs+ePeHu7s6c+a2trcWWLVuo3Tvbt2+PZ8+eCQRa2dnZaNOmDYD64c5sNGVISkrC0qVLMXr0aGzfvh3y8vLo2LEjPn36hIcPHyI8PBzW1taIiIiAiopKs9v/mqqqKqbcvV+/fnjy5AlUVVVhbGzMFyCzjZubG3R0dDBr1izmMxAGNHZUmyIlJQX79u1jKkRu374NABg/fjzzHi0tLezcuVMo+v5TRDtaHOfjx4/YsWMH5s+fjwEDBsDKygqJiYmQlZXFvn37vtlljG2SkpJw8eJFREdHo1OnTpg2bRpMTU1/2BTg76KhoYHY2FhIS0tDR0cHa9euhampKXJycjB79mwkJyezYvdHjQYaYDvbxBUdX1NVVYXS0lJ07tyZrzlHQwaObVRVVREXFwcpKSkYGhrC1NQU1tbWePXqFSZPnsxKJrYpHj16hIcPH6K0tFQoGft3797hzJkzePToET58+ABJSUloaGhgwoQJrJZPihDxM4wZMwYfPnyAvr4+pk+fTjVJ9zXC3AVvYOjQoYiIiBC4Rzx79gwzZsxAWloa6xp2796NyMhI2NraQk1NDXV1dbh//z727NmDKVOmYPHixVi3bh3at2/f7DtsFhYWTKD5LRwcHFBeXg4fH59mtd0U48ePh729PcaMGYM9e/bgw4cP2LJlC3JycjBjxgxqfkRdXR1nz54V2vNdY/Lz83HixAk8efIELVq0gLKyMmbNmsXqmWMVFRVcvnwZ0tLSAOq/A6dPn0ZSUhIz2icvLw+TJ0/GX3/9xZqO5kK0o8Vx3N3dkZKSgoULF+Ly5ctISUmBl5cXzp8/Dy8vL/j6+lLX9ODBA8TFxTGNKYYOHYrk5GQEBwfDxcWF1Q5BgwYNwsmTJ9G9e3d8/PgRY8aMQVVVFfbv3//TQch/A1c633BFR2MMDAxw6tQpdO/enW+9qKgIU6ZMwd27d1nXIMzxA41pageaJt26dcPgwYNhZmbGF/ACQGVlJa5fv87q/BMRIr7H9evXkZCQgJiYGKxcuRKdOnXC1KlTYWJiIrCjwjbC3AVvQEJCAgUFBQKBVk5ODrVBxqtXr0ZVVRVcXV1RWVkJQghat24NMzMzrF69GtevX0dFRQUrDZYePXqEdevWffc9pqamTc5rZAMTExNs2LABXl5eGDt2LMzNzdGrVy8kJCRAQUGBigYAGDZsGO7duyf0QCstLQ0WFhbo0qULVFRUUFtbi+PHjyM0NBSHDx+GvLw8K3Z79uyJV69eMYHW7du3oaamxndN/PXXX9S6QP5dRIEWx4mPj4e/vz/k5OSwf/9+6OrqYvLkyVBQUMC8efOo6SgsLMTp06dx+vRp5ObmQk1NDcuXL4eRkRFzVsvX1xdubm6sBlobN27E0qVLUVJSAmtra0hJSWHbtm24evUqDhw4wJrdr4mJiUGrVq0wYcIEAPXOytDQEJMnT6amQZg6zp8/j5s3bwKoz3g5OzsLHJTNz8+nlq0W5viBxlRUVODQoUNIS0tDdXW1QAt8GoMmzc3NkZCQIFBenJ2djd9//10UaIkQGjweDyNHjsTIkSNRXl6OuLg4XLp0ielqO3PmTEycOJHK7KAPHz4wAU5CQgJMTU0B1Hfw/PLlC+v2AWDSpElMcycVFRXU1dUhLS0Nvr6+rM6OaoyYmBg2btyI1atXIycnBy1atICsrCzzGRgaGrJ29vfTp08/3Bnp2bMnSktLWbH/NYsXL0arVq1ACIGqqiqWL1+OgIAASEtLw8vLi4oGABgyZAi2bt2K69evo2/fvgJNlr7VLbO58fT0xIQJE+Di4sJoqK6uhr29Pdzc3HDo0CFW7I4bNw47d+7Etm3bEB8fj8LCQixZsoR5vaioCL6+vqx2wWxORIEWx6moqGCi+oSEBCbT1rp1a1ZnFH2Nvr4+JCUlMWXKFPj5+TVZDjVo0CDIysqyqkNVVRW3bt1CWVkZOnbsCABYsGAB1qxZw/q8pAbCwsLg7e3NN4BWSkoKjo6O+Pz5M7XORMLUoaGhgYiICCaQKCgo4NtBaeje5enpyZqGxhgbG0NRURGvXr0S6viBLVu24OrVq9DV1RXY4WOTQ4cOMb9rQgh0dXWbfJ+qqio1TSJEfI/Pnz+jtLQUHz9+RE1NDcTExBAQEIBdu3bB29ub9euWC7vg69evx5cvX7B161bU1NSAEIJWrVph/vz5VAeLf/78GVlZWUxyKD09nXmNzflZdXV1TXZqbUyLFi1QV1fHmobG8Hg8LFy4kPl58eLFWLx4MRXbjTl27Bi6dOmCv/76S6A0jsfjUQu0MjMz4e7uzvcZSUhIYOnSpZg5cyZrdlesWIGlS5cy58z19fWZREhAQAD27t0LGRkZLFu2jDUNzYnojBbHmT59OmbOnAlpaWksWbIEFy9ehKysLLy9vXH37l3W2q1+TUN3uRYtWlCxx2XGjRsHW1tbGBkZ8a2fPXsW/v7+uHjx4r9Kh5mZGfz8/AQGf3KFyspKgd02ttDS0oK3tzf09PSo2GugpqYG586dQ11dHTZt2oRNmzbxNRxoCHx1dHSYBIUIEbSprKxEXFwcTp8+jcTERHTt2hXGxsaYNm0aZGRkAABOTk64du0a4uPjWdUSExMDR0dHiImJQUNDA4cOHeLbBWe7mVBjysvLkZubCx6Ph379+lErGwTqfbudnR3KysoEduDZPuurpKTU5O57Y969e4dRo0ZRO3N8/fp1ZGVloaqqSuA1WgEOV5g8eTKWLVsm8IwRHx8PT09PnD9/nlX7T58+hZiYGF9i//Lly3j9+jWmTZvGiVljP4NoR4vjrFq1CjY2NqiursakSZMgKysLd3d3hIeHU2s1CtSfw/n8+TOys7OZOu7GsD01vgFFRcXvlqPRuBm/efOmyQ5I6urqKCgoYN0+13Q07qIlLEpKShAYGCgwaLK6uhrZ2dlISUmhouNrp0ALcXFx5sGQx+Nh4sSJTBczESK4wvDhw1FTU4OxY8fC398fo0aNEhj/MHz4cFy9epV1LVzZBW/wqw27SY8ePWJeo+FXG3YPly9fTr0bJCEE06dP/+4IEFq7WQDg7OyMo0ePomvXrgLJOZo7SQ28e/euyYCP1hwtKysrbN++HW/evMGwYcMgLi6O9PR07N69G3PnzuVrPsbGd3XgwIEC//9x48Yxf09MTISOjk6z221uRDta/wBKSkpQVFTENHt48OAB2rVrR/WBLj4+HmvWrMGXL1+oZ70aExUVxRdo1dTU4Pnz54iJicGGDRuoTApvOLzduMQAAI4cOYKjR4+ynuXhgg4lJSXcunULXbt25UTwu2bNGty5cwe6urq4ePEiJk6ciJycHDx+/Bhr166lVv7h4OCA7t27Y82aNVTsfYv8/Hzcv3+/SSdNM1MvQkRjwsLCMHny5CY7YDZ0KK2pqflhOdn/CsLcTWpg8ODBiI2NZdqa08TPz++n30sjyBk2bBhsbW0xe/Zs1m19j/j4eNjb26OkpIRvnfYcLS50Oba0tERAQABf4rCiogIeHh6IjIyk2l35v+XfcTf7B2NhYQEDAwO+6dfCOGexY8cO6OrqYsWKFUItPfrWrCwVFRVERkZSCbQsLS1hZ2eHR48eMZPr09PTERsbCxcXF9btc0GHm5sbk/10c3MTWovmBu7cuQNPT0+MHTsWT548gaWlJRQVFeHo6Ijs7GxqOiQlJRESEoIbN26gX79+ArtKNFpGnzhxAk5OTk2e4eTxeKJAS4TQCA0NbbJZUuMOpbSCLHNz8+++TqNxjTB3kxqQlZXF69evhRJoNQ6eioqKWB8P8yPExcUxbNgwoWoAAFdXV6iqqmLu3LlUGsN8Cxo7yz8iPz8fK1euhL+/PyQkJHD79m1s3rwZ5eXlrHTCZAPRjhbH8fDwQHx8PJ4/fw4lJSXo6+vD0NCQ1VbmTTF48GCcO3eOqaPnGvn5+TAyMsL9+/ep2Dt//jzCwsLw5MkTSEhIQE5ODosXL6Z+NocrOoRN47kbNjY2MDAwgLGxMZ48eYLFixezft6jgR8NtaRRZqmvrw89PT3Y2toyHUFFiBAWjTuURkdHw8jIqMkOpVlZWUhMTKSmy97enu/nmpoavHjxAllZWViwYAFsbW1Z1yDM3aQG4uPj4e3tDVtbW/Tv318gOUSrTE1RURHKysowMDCAoaEha63Dv4e/vz9evnwJFxcXoZZeq6urIyoqCv379xeaBq7w9u1bLFq0CL1790b37t1x8uRJGBkZwcHBAV27dhW2vJ9CtKPFcezs7GBnZ4eXL1/i+vXriI+Px759+9CtWzcYGBjAwcGBig5ZWVm8ffuWk4FWeXk5jhw5wuoAva8xMjISOCAqDISlg2slHz179kR+fj6kpaUhKyuLJ0+eAADatGlDrTUwwI3zag2OSRRkieACXOtQ2sC3dpf9/f3x+vVrKhqEuZvUwPLly1FbW4vly5fzVSbQLlM7ffo04uPjmZE2UlJSMDAwgIGBAYYOHfrdc1zNxYQJEzBnzhxoaWmhe/fuApUatHZ4dHR08OjRI6EEWj/a6W0MjV3f7t27IywsDIsWLcLNmzfh5+fH2rgBthDtaP2DeP/+PRITE3Ht2jVcuHABhBBWb4KNGyr8+eefOHLkCBwcHCAjIyPQfZBm1qupMjUejwcnJydWW442pri4GLm5ucxBXUIIqqqqkJ6eTrXlqLB0NC5lBernrElISKBPnz4QFxfHy5cvUV1dDRUVFURERLCmowFPT09cuXIFHh4eqKqqgq2tLbZt24YrV67g6dOnOH36NOsaGigvL8eZM2eQlZUFcXFxDBw4kG/eHNvMmjULy5cvx9ixY6nYEyHiZ+F6h1IAePXqFYyNjak00OHCblJSUtJ3X9fW1mZdw9d8/PgRN2/eRFxcHK5cuYIOHTpQ2e2cPn06Pn78iHHjxjXZ+ZFWM4zXr19jxowZGDFiBPr06SPwzMOmjsY7vZWVlTh//jyUlJSgrq4OcXFxPHz4EA8ePMDMmTOxbds2VjTExMQIrJWUlGD37t3Q1dXlmwX5TyiFFwVaHCcuLg53797F3bt3kZOTgx49emD48OHQ0dHBiBEj0KNHD9Zsfx3UNHxVhJn1+roZBlA/10FNTY3aFPUzZ85g8+bNqKqqAo/HY34HANC7d29cuXLlX6Xj0KFDuH79Onbu3Mls5X/8+BEbNmyAvLw81q5dy7qGqqoq7NixA6qqqpg8eTK2bNmCEydOoEOHDtizZw+1LmIFBQWYP38+3r9/j379+qGurg4vXrxA165dcfToUSqT7OPi4uDh4YFFixY1+fBGq0OoCBH/RM6ePYvt27fj7t27rNtSVlZmzlIK069yBUIIHj58iMTERNy9exepqakghEBLSwvBwcGs21dVVcXJkyeFUrbYmAb/1aVLF7Rp04bvNR6PR21nzd7eHp06dYKdnR3fuo+PD3JycuDr68uKXS404WhORIEWx1FUVISYmBgMDAywZMmSJtt5s8WPMl2NEUbWS1gYGRlBTU0NVlZWmDNnDkJCQvDmzRs4OTlh7dq1VBpycEnHiBEjEBISInBzzMrKgpmZGZUHlqb48OED2rdvT7WD2apVq/Du3Tvs2bOHKWV99+4d1qxZg549e2Lnzp2sa/iek/qnOCYR/ztwrUNpA02VSJWVleHJkyeYO3culbJ8Ye0mLV26FN7e3ny77Ldu3cLQoUOZ83PFxcUwNDREWloaKxqa0pSamoovX75AWVkZw4YNw4gRI6ChoUHtvJSJiQkcHBwwZMgQKva+haamJhwdHWFiYiJUHRoaGoiOjoasrCzf+vPnz2FsbCwwTFlE04jOaHEcPz8/3LlzBwkJCZg9ezZUVVUxfPhwDB8+nNnKZYuGm3xZWRkkJCSoDX39Gq6dB8rLy4Ovry/k5OSgoKCA4uJi6Ovro6amBoGBgdQCHK7oqK6uRkVFhcD6+/fvqXYjfPr0KZ4+fSrUlua3b99GSEgI33nBbt26YcOGDbC2tqaigQudokSIaKBxh1IaXTd/ll69ejVZHTF//vwmOyOywbcCqUePHiEiIoK1QCs+Ph6VlZV8gdaqVatw+vRppjKkrq6uyfs6W9y7dw9lZWUYM2YM9PT0MHz4cOpn16ytrbFp0yZYWlqib9++As9XtKoB2rRpA01NTSq2vkfHjh3x+PFjgUArJSWFaiOKmJgYtGrVChMmTAAArF69GoaGhpg8eTI1DX8HUaDFcQwNDZmDf4WFhbh9+zbu3LmDAwcOQExMDPfu3WPNdkP5140bN8Dj8TB27Fi4uLh8d4o7G0RFRf3U+2gNFGzZsiWTYZORkcHTp08xevRoqKio4MWLF6zb55oOfX19ODo6YsuWLVBRUQEhBKmpqXBxccHEiROpaAgMDISPj0+Tr9Fsad6iRQuBUg8AaNWqVZMBIBv07t0bQH055atXr9C3b18QQvgaEIgQQYvGWflHjx7B3NxcqM0fGvDw8BC2BD6+fPmCs2fP4vjx43j06BFatWrF2piOpgqZmlqjmShLTEzEo0ePcPv2bZw/fx6urq7o1q0bdHR0MHz4cCoP1Q1l7lu3bhV4jWY1wNy5c+Hr6wsXF5cm/QktTE1NsWXLFuTk5EBFRQV1dXVIS0tDeHg4fv/9dyoawsLC4O3tDUdHR2ZNSkoKjo6O+Pz5M2bNmkVFx99BFGj9QygqKsKdO3dw+/Zt3L59G2JiYtDV1WXVppeXFx48eIDVq1dDTEwMhw8fxrZt27Bnzx5W7X7NtWvXqNr7EQ0zu9auXQt5eXnEx8fD0tIS2dnZVB9muaLD0dERq1evxoIFCxjHTAjB+PHjsXHjRioawsLCsHz5cixZskRoO69AfcnH3r174eXlxXwG1dXVCAwMpJahJIRg586dOHz4MKqrq3Hp0iXs3r0bbdq0wbZt20QBlwihER0dLTBgnSbJyck//V5auxdZWVk4fvw4zpw5g7KyMnTp0gUrVqzA3LlzqdjnCjweDyoqKlBRUcHixYvx/v17BAYG4vjx44iOjqYSaHGlGiAlJQXJycm4ePEiunbtKrCzRkvn8uXL0aJFCxw5cgT+/v4AAGlpaWzYsIHa9/Pw4cPw8PDg665sb28PFRUV+Pv7iwItEX8fV1dX3L59G8+ePUOPHj2gp6cHDw8PDB8+nPW65Rs3bsDT0xOjRo0CUF+vu2jRItTU1FA999LA69ev0a1bNz7bKSkp6NOnD9VBhzY2NrCyskLnzp1hYmICf39/TJw4EYWFhczW9r9JR/v27REcHIxnz54hKysLYmJiUFJSotacBKgPZqZOnSrUIAsA1q9fj9mzZ2PcuHHMecr09HRmBAENDh8+jNOnT2Pr1q1wdnYGUL8z7uTkhG7dulGZDyRCRFOMGTMGR44cwcqVK4UyfsDMzIxpHPQ92N69qKqqwoULFxAREYG//voLYmJi0NHRwZ07d3Do0CGhN2MQBoQQpKenIyEhAbdv38Zff/2FTp06YcqUKdTmQnKlGkBLSwtaWlpUbTbFuXPnMHv2bCxZsgQlJSXg8Xjo3LkzVQ1v3rxpsjeBuro6X2dsLiNqhsFxpk2bBn19fejr62PQoEFUbauoqODq1atMEFNbW4vBgwfjzz//pD7Bff/+/dizZw8OHTrEdwNasGABUlNTsWHDhv9o/sPfpaioCFVVVejTpw+ys7MREREBaWlpmJubU70pc0WHsNm+fTtatWpFrZzhe+Tn5+Po0aN4+vQpCCFQUFDAnDlzGCfONhMnTsSaNWswbtw4aGho4MyZM+jTpw8uX74Md3d3zu0Qi/j3YGZmhuTkZPB4PHTt2lUgMcJ2pj4/P/+n38vW9erh4YHo6Gh8/PgRmpqaMDIywvjx49G1a1coKyvj9OnTGDBgACu2G1BUVERCQgLfOZvG9wqgvonPqFGjqJXLaWtr49OnTxg4cCD09PRgYGAAVVVVKrYbEFUD8KOtrY2jR4+y/n38HlOnToWJiYnATviRI0dw9OhRnD9/XjjC/gNEO1ocp+F8UkFBAW7evImhQ4eivLycykHEr3euWrRoQfWsSQNxcXHw8fHB8uXLoaCgwPean58fQkND4enpib59+1KZHWRvbw8HBwcm2BwwYAA2b96MDx8+YPXq1di7dy/rGoSt40fdwxpDw1FbWVlh6tSpOH/+PH755RcBbTQGKzbQu3dvoQZ8r169gpKSksC6oqIi3r59KwRFIkTUM2zYMAwbNkxo9r8XPL1//56KXz106BD69+8PNzc36OvrUz0H1Zh79+7xzTMjhODBgwfMsGaag96B+mYcenp61BJSTcGlaoDMzExkZWU1OSNz+/btVDTIysoiKytLqIGWpaUl7Ozs8OjRI6ipqQGorxKJjY1l7QxjcyMKtDhOdXU1NmzYgAsXLkBMTAyXLl2Cp6cnysvL4evrK5TyC9ocOnQINjY2WLp0qcBrHTp0wMqVK1FRUYHg4GDWAq3U1FTk5eUBqO+Ao6ysLPC7z8nJwZ07d1ixzzUdbm5uzANCfn4+9u/fD1NTU2hoaEBCQgLp6ekIDw+nNry5Yc6Hqqpqk4Mm2cTc3Bx+fn7o2LEjU5r0LWgEfL1790Z6ejp++eUXvvUbN25QLecUIeJraA18/R5JSUlwcnKCj48PBg4cyKxv3rwZz58/h7u7O9TV1Vmz7+zsjKioKKxYsQKdOnXCr7/+CiMjI+oBqI2NjUAJ5bp16/h+phkEzp8/H4WFhfD29maGvQ8YMACmpqbUgq/jx49jy5YtGDduHPMQb2RkBAkJCbi7u1MLtA4ePAhPT08A4Ct15fF4VFvPKyoqYv369Thw4ABkZWUFdqBpdBGdMmUKxMXFERYWhitXrkBCQgJycnLw9fWlVlL6dxEFWhxn7969yMzMRGhoKBNomJmZwd7eHt7e3qxN5m7g9evXqKys5FsrKipCixYt+NbYnGD/9OnTH2ZwpkyZglOnTrGmgcfjMQ/zPB6vST1t27aFpaUlaxq4pGPatGnM3+fPnw9HR0fMmDGDWTM0NIScnBxCQ0NZ1wLUZ2fDwsKYjBdNevfuDTExMebvwspQN2BpaQknJye8ffsWhBDcuXMHx48fx+HDhwUGT4oQQRthZuofP36MxYsXQ11dHa1bt+Z7zdzcHIGBgVi0aBFOnDjBF4Q1J7NmzcKsWbOQk5ODqKgonDlzBidPnkTXrl1RV1eHvLw81ncQuNL0oTFPnjzB/Pnz0bp1a6iqqqKurg7R0dE4evQojh07xtrn0RiuVAOEh4fD2toaK1euhJ6eHqKjo/HhwwesW7cOBgYG1HTk5uYyRzWEWQ1hZGTE1wzjHwcRwWnGjRtHEhISCCGEqKurk5cvXxJCCLl9+zYZOXIkq7YVFBSIoqIi35+v1xp+ZpMhQ4aQ7Ozs777n+fPnREtLi1UdDSgoKJC3b99SsfVP0DF48GDy/PlzgfXc3FyiqqpKRcNvv/1GHj16RMXW98jPzye1tbUC69XV1eT+/fvUdERERJAxY8YQBQUFoqCgQIYPH05CQkKo2RchoilCQkKY72SD/2j4+/z581m3v2LFCrJ27dpvvl5XV0eWLFlCVq1axbqWBmpra8m1a9fIihUriLKyMlFUVCSzZ88msbGxVOxXVlZ+87U7d+5Q0UAIIRYWFmTFihXky5cvzNqXL1/IypUryeLFi6lomDBhAjl//jwhhP9568iRI2TChAlUNBBCiLKyMnnx4gUhhJBFixaRy5cvE0IIuXnzJpk4cSI1HVzh/fv3JCUlhSQlJZGkpCRy9+5dcvPmTbJ3715hS/spRDtaHKeoqKjJmSPS0tKs11DTPNfyPeTl5XH37l3Iycl98z0JCQnUZrNkZmYKrBUXF1OfL8YVHTIyMoiNjcXy5cv51o8fP06ttnv9+vXYsmUL1qxZ0+SgSTZ3XBtjYGCAhIQEgc/g1atXMDMzw/3791nXUFBQgJkzZ8LU1BTFxcUghKBr166oqanBgwcPqB8wFyGiAWFn6v/66y8EBQV983UejwcrKytmnhINxMTEoKenBz09PRQXF+PMmTM4deoU1q1bRyWLv2zZMgQEBPB1Ma6oqICHhwciIyOpNcNIS0tDREQEX3laq1atsGLFCsyfP5+KBq5UA7Rt2xa1tbUAgL59+yI7O5upEvlPGro0B+Xl5Thz5gxTzjlw4EAYGRlRO7Zy5swZbN68GVVVVUwZZUPVSO/evakdT/g7iAItjiMnJ4c7d+5g5syZfOuxsbGsP8Q2nkpfVVX1zXbyiYmJrOqYMWMGvLy8oKmpCUVFRYHXMzIy8Mcffwg86LPFp0+f4OXlhfnz52PAgAGwtLTE3bt3ISsri3379lE7B8MVHatWrcKqVatw+/ZtDB48GHV1dbh37x4yMjKwf/9+KhpWr16N2tpaWFlZ8ZXuNdyU2XxYCA8PR0hICGNv+vTpTClhAx8/fhRKsNc44KMZ7IkQ0RSvX7/GzJkz0apVKygqKiI9PR2Ghoaws7ODh4cH6zO2ysvL0bFjx+++R0pKCh8/fmRVRwNf+1VJSUksXLgQCxcuxMGDB6loyM/Px8qVK+Hv7w8JCQncvn0bmzdvRnl5ObWmCwDQrl07VFdXC6w3tcYW06dPR01NDQICAvDlyxds2bIFkpKSWLNmDebMmUNNh6amJvbt24ctW7Zg0KBBOHnyJBYvXozU1FS0a9eOmo6CggLMnz8f79+/R79+/VBXV4cTJ04gMDAQR48ehZSUFOsaAgMDMXHiRFhZWWHOnDkICQnBmzdv4OTkBBsbG9btNweiQIvj2NjYwNbWFtnZ2aitrUV0dDRyc3OZtqO0EGbWy8TEBNevX8eMGTMwduxYaGpqomPHjvjw4QPS0tJw48YNjBw5EmZmZqxpaIybmxtSUlKwcOFCXL58GampqfDy8sL58+fh5eUFX1/ff5WOcePGITw8HEeOHMGtW7cAAEpKSnB2dm4yMGYDWg8lTTFt2jSUlJSAEAJ/f3+MHz9ewBm2a9cOv/76K2sauBbsiRDRFMLO1P/yyy948uTJd5NQmZmZ1MaXfMuvenp64sSJE1i0aBHrGg4fPoxFixZh5cqV6N69O06ePAkjIyM4ODhQ6cLYgI6ODry8vLBnzx5mVlNxcTF27NiB4cOHU9NhamoqUA1Am7Vr18LCwgLh4eGYM2cOAgMDoa2tjc+fP1M589yAh4cHpKSkcOLECXTr1g1Afdv/NWvWYMeOHdi5cyfrGvLy8uDr6ws5OTkoKCiguLgY+vr6qKmpQWBgIKZOncq6hr+LaI7WP4AbN24gKCgIjx8/Rl1dHQYOHAhra2v89ttv1DSMHz8effv2bTLrtWHDBkyfPp11DeHh4YiIiMDTp08B/N8k+VmzZmHGjBnUmhCMGDEC/v7+0NDQgJ2dHT58+IDAwEBkZWVh3rx5SE5O/lfp4BplZWWQkJAQyvBiPz8/WFpaok2bNlTtfv78GcHBwUywt2jRom8Ge8Jsnyzi383SpUvRpUsXbNmyBWfPnsXJkycRERGB8+fPw8PDg0nUsIW/vz8uXbqEo0ePNln6VF5ejjlz5kBXVxcbN25kVQvADb8K1Ac0ixYtwtOnT7Fnzx4YGhpSsduY169fY/bs2SgtLYWsrCwA4Pnz5+jcuTMOHz4s0EWVLfLy8vDo0SN8+fJF4DVjY2MqGsrLy9GiRQtUVFRAUlIS7969w7lz5yAlJYXx48dT0QAAQ4YMQUhIiEC5+YMHD2BtbY27d++yrkFLSwsxMTHo06cPNm/ejH79+sHS0hIFBQWYPHkyUlNTWdfwdxHtaP0DGD16NEaPHi1UDVzIes2bNw/z5s1DVVUVSktL0blzZ74Bgo1rd9mkoqIC0tLSAOrPhllbWwMAWrduzWRracAVHQAQHx+P4OBgPHv2DMePH0dUVBT69u1LNdsUGhqKgwcPoqioCDweD7/88guWL19OzTkC9e2ry8vLcezYMao17W3atGFaZ/N4PKEEeyJE/AhhZ+otLCxw7tw5GBsbY8GCBdDQ0ECnTp1QUlKCtLQ0hIWFQVxcHIsXL2ZdCyA8vxoTEyOwZmxsjN27d+PUqVMoKyvjW6eBlJQUYmNjcfr0aWbY+6xZszB58mRq54GioqKwefNmpiNmY3g8HrXfhbGxMXx8fKCsrAwA6NatG+tltU3RokWLJv0IzXmqKioqiIyMxNq1ayEvL4/4+HhYWloiOzv7HzNAWhRocZDGWXE/P7/vvpfWXJLu3bsjLCwMixYtws2bN+Hn5yeUrJeBgQFOnTqF7t27860XFRVhypQpVDIscnJyuH79OqSlpfH27VsmCD5x4sR3G3b8r+pISEjAypUrMXHiRPz111+oq6tDTU0N7O3tQQih4px8fX0REhICc3NzqKmpoa6uDikpKdi2bRvKysqoHabmQk37ypUr8fnzZ9y/fx/V1dUCs3KGDh3KugYRIppCXl4eV65cQUVFBdq1a4cTJ07g7NmzkJaWppKpb9OmDY4ePYpt27bBw8OD74G6RYsWGD9+PDZu3IguXbqwrgUQnl/9XmOHP//8E3/++ScAusEFUL/rPnfuXGr2vmbv3r0wNTWFra3tD8/yscnnz58Fxg8IA01NTezduxdeXl5MUFNdXY3AwEBoampS0WBjYwMrKyt07twZJiYm8Pf3x8SJE1FYWIgJEyZQ0fB3EZUOchB9fX2cOnUKXbp0gb6+/jffx+PxWJ2H0VTWq6SkBLt374auri5f6SKbN+Pz58/j5s2bAIDo6GgYGRkJlIXl5+cjKyuL9cYcQP3ujY2NDaqrqzFx4kR4e3vD3d0d4eHh8Pf3x5gxY1jXwCUds2fPxvjx47Fw4UJoaGjgzJkz6NOnD/bv348zZ87g7NmzrGsYNWoUNm7ciEmTJvGtR0ZGIiAgANeuXWNdA1DfGOTdu3fYs2ePQE17z549qdS0X716FXZ2digrKxMIsthuDCJCxD+F9+/fIyMjA6WlpZCUlISKigo6dOjAul2u+FUu8KMB742h0QV58ODBOH/+vNAHu+/btw8xMTGYN28e+vbtKxB00UqW5eTkYPbs2WjXrh1UVFQAAOnp6SgvL8eRI0eoncEuKipCVVUV+vTpg+zsbEREREBaWhrm5ub/iF0tUaAl4pv87EXE9sNbYWEhNm7cCEIIkpOToa6uzndx8Xg8tG3bFnPmzKEWXJSUlKCoqIj5HT148ADt2rWjupPEFR0aGho4ffo0+vbtyxdo5eXlYdKkSVS63KmrqyM6Ohr9+vXjW3/27BlMTEyoddrjQk37hAkTMHDgQCxfvrzJB0fRGS0RNNHX1//ph2mag3T/+OMPTJs2jfpDNVf8amNiYmLQqlUrZodg9erVMDQ0xOTJk1m1a29v/9PvdXd3Z1FJPbNmzcKKFSuoPUd8i+99R2gny/Lz83H06FGmnFNBQQFz5syh5kfs7e3h4OAgUD764cMHbNq0CXv37qWi4+8gKh3kOPPmzcO0adMwYcIEtG3blqrtpuY0CQNpaWkmm2VmZgY/Pz906tRJqJq6dOmC0tJSxMXFQUxMDMrKysx5qX+bjg4dOuDNmzcCc8yys7OpfU4GBgY4duwYNm3axLceHR1N9XwjF2raX716haCgIGpz5USI+B4mJibUGhX9J5w9e5YpgZo2bVqT3ULZgCt+tYGwsDB4e3vD0dGRWZOSkoKjoyM+f/6MWbNmsWb7y5cvcHR0hKSkZJNJVBo0bhplaGgIBwcH2NjYQEZGBi1atOB7L62dJJoJh69ZunQpRo4cieHDh0NOTg69e/fG77//TlVDamoq8vLyANQnAZSVlQUCrZycHNy5c4eqrv8W0Y4Wx7G3t8fly5dRW1uLcePGYdq0adDR0RGKFmFlvbhGWVkZ1q5di5s3bzKlWTweD0ZGRnB3d//mvLH/VR07duzAnTt34Obmhrlz5+Lo0aN48+YNtm3bhl9//ZW1QY+Ns6Hl5eW4fPkylJSUoKWlhRYtWuDRo0dIS0vDzJkzsW3bNlY0fM2yZcvQunVrgZr2DRs24OPHjwgODmZdw+TJk+Ho6Mg3B0+ECBGCpKWl4ezZs7h48SK+fPmCcePGwcTEhGo7cWH71XHjxsHW1lZgOPLZs2fh7++PixcvsmZbVVUVp0+fRr9+/aCkpNTksHe2UVRUZAbhfo9/S9n1okWL8Ndff+HLly/o2bMnRowYgZEjR2LEiBFM2322SUtLY87qfeuzadu2LSwsLKj1Kfg7iAKtfwBVVVW4evUqzp49ixs3bqBHjx4wNjaGiYkJtbKHxlmvhuHJ7u7uOH78ODZt2sRq1ktJSQm3bt1C165dmZvit6BxI7S3t0dKSgq2bNkCDQ0N1NXVIS0tDS4uLhg3bhy1CfJc0VFdXQ07OzvExsYC+L8b49ixY/HHH3+w1mb9Z+em8Xg8KvX9ADdq2uPj4+Ht7Q1bW1v0799fIOAWzdISIUwePnyI4OBgpivngAEDsGDBAoFyW5rU1NTg1q1biI2NxdWrV9G5c2cq5zqF6VcbUFNTw9mzZwV2wPPy8jBx4kQ8ePCANdvTp09HUVER+vXrh+TkZGhoaHxzR4ute/h/Mr+NVrnct8pteTweJCQkICUlhalTp7J2hq+2thaPHj1CSkoKUlJSkJqaik+fPkFRURG6urrQ1dWFlpYWld1HRUVF3Lp1iznz/E9EFGj9wygtLUV0dDR8fX1RUVFBLcMizKxXdHQ0Jk6ciJYtWyIqKuq7gZaJiQlrOhoYOnQo9u7dK1BGcPv2baxfvx63b99mXQOXdDTw4sULZGRkoK6uDvLy8ujVqxd8fHwEyvn+1xF2TbuysjLT3r/xtdIw/uDfkJUVwU2SkpJgYWEBeXl5DBkyhEkOZWVlITQ0FFpaWkLR9ebNG8TGxuLSpUtIT0+HtrY2lSHowvSrDUydOhUmJiYC7cOPHDmCo0eP4vz586zZfvXqFY4cOcI810yYMOGb3fZonNHiynkgf39/+Pv7w8DAAEOGDAEA3Lt3D3FxcZg2bRrExMRw9uxZbNq0iQnQ2SYrK4sJvK5fvw5CCO7du0fF9tcUFxdT3/n8O4jOaP1DqKysxNWrV3HmzBkkJCRAWloaVlZW1Oy/efOGydA3Rl1dHQUFBazabhw8TZs2jVVbP0OLFi2abDLQvXt31NTU/Ct0VFZWwtPTE7GxsZCQkMDUqVOxbt06yMjIAABu3bqFJUuWoLCwkFqg9eXLF1y8eBHPnj2DhYUFsrKyMHDgQGqtmhsQRk17Y2g8IIoQ8d+we/duTJ8+HU5OTnzrTk5O8PHxweHDh6lpKSsrw6VLl3D27FkkJyejV69eMDExwe7du6mdcxWmX23A0tISdnZ2ePToEdTU1ADU78LHxsbCxcWFVdu//PILU3mRkJAAJycn6m3VuXgeKC0tDatXr8aSJUuYtQULFiA4OBiJiYnYv38/NDU1ERwczHqgVV1djZSUFCQmJiI5ORkZGRmQkJCgVl776dMneHl5Yf78+RgwYAAsLS1x9+5dyMrKYt++fULvEPkziAItjnPz5k2cO3cOV65cASEE48ePx8GDB5ksBy1kZWVx7do1gaxXfHw86xPbfzRLrDE06nXNzc3h4uKCP/74g9nOLisrg4+PD8zNzVm3zwUdXl5eOHHiBKZMmYKWLVvi2LFjaN++PZYsWYLt27fj2LFj6Nu3L0JDQ1nV0cC7d+9gamqK9+/fo6qqCjNnzkRISAgePnyI0NBQal0Yi4uLsX//fjx9+rTJ5hc0ShhFZ7NEcJXHjx9j+/btAuvz58/HjBkzqGoZMWIEJCQk8OuvvyI0NJS6TwWE61cbmDJlCsTFxREWFoYrV65AQkICcnJy8PX1hZ6eHhUNAFBXV4fnz59TLyHl8XhMsMfj8Zr8frZt25bKQO0GUlNTsWXLFoH1cePGYc+ePQDq7/Nbt25lxf7z589x8+ZN3Lp1C0lJSaiqqsLgwYMxcuRIbNiwAaqqqhATE2PF9te4ubkhJSUFCxcuxOXLl5GamgovLy+cP38eXl5e8PX1paLj7yAKtDjO4sWLMXToUDg6OuK3335rsqMZDYSZ9YqKiuL7ubCwEBISEujTpw/ExcXx8uVLVFdXQ0VFhUqgdevWLaSnp8PAwACysrIQFxfH8+fPUV5ejoyMDERHRzPvZbN7kDB1XLt2DQ4ODpgzZw4AYOzYsXB1dUVhYSFOnjwJCwsLrF69mlpDDg8PDwwcOBDnzp3DiBEjAACenp5Ys2YNduzYgcDAQCo6NmzYgPT0dIwYMYLqwMmmZvN8i//12TwiuEuXLl1QUlIisF5cXEztXtGAk5MTxo8fLzSfCgjXrzbGyMhIoHyRNi1btoS4OP1HUk1NTaYTJFfOA3Xt2hVpaWlMhUgDqampTIXG27dvWZn7ZmhoiPz8fPTu3RsjRoyAp6cnhg8fTmXGXFPEx8fD398fcnJy2L9/P3R1dTF58mQoKChg3rx5QtH0nyIKtDjO5cuXqWW2vocws16NDyUfOnQI169fx86dO9G1a1cAwMePH7FhwwbIy8uzqqOBESNGMA/zwkSYOt69e4eRI0cyP48aNQr5+fm4fPkyDh48iGHDhlHVk5iYiH379vE9NHXq1AkbN26kusuYmpqKoKAg6rtKP9v4hMfjiQItEUJDT08PLi4u2LVrF7PLnJ2dje3bt0NfX591+w0NF8TFxfHLL7/g4cOH33wvjVbeXNlNKi4uRm5uLurq6gDUn+esqqpCeno6li1bRkWDiYkJrKysMHXqVMjIyAgkqmjct7jSet/MzAzOzs54/vw51NXVUVdXh/v37+Pw4cNYtmwZCgsLsW3bNowaNarZbb969QrS0tKYOHEiRo4cyVwvwqKiooIp5U1ISIC1tTUAoHXr1sxZZK4jaobxDyAzMxOhoaHIzc3FH3/8gStXrmDAgAHUH2a5wIgRIxASEiLQvS0rKwtmZmZUBsKKqM/8JSQkMMEuUD+42MXFBZMmTaKuR0NDA9HR0ZCVleUbmpyZmYm5c+ciLS2Nio7x48dj586dUFZWpmJPhIh/EqWlpVi0aBEyMjKYDHlDN7OQkBDWz1M2vm99r633v6lpzJkzZ7B582ZUVVUxv4+GJjq9e/fGlStXqOjgwpDeyspKHD9+HFlZWXwP8VVVVXj48CEuXbrEuoYGwsPDERwczJzV69WrFxYvXozZs2fj5s2biImJwdatW5v9TNuzZ8+YssHk5GSIiYlh2LBhGDlyJEaOHCmwy8Y206dPx8yZMyEtLY0lS5bg4sWLkJWVhbe3N+7evYvIyEiqev4bRDtaHOfhw4eYM2cO1NXV8fDhQ1RVVSEjIwPu7u7w9/enOsGcC1mv6upqVFRUCKy/f/+e2lDMH5Vp0dox4IqOxgirRfPQoUNx7Ngxvtla1dXVCAgIgKamJjUdv//+O5ycnGBra4s+ffoI1LGLWquL+DfTqVMnnDx5Ejdv3uTryjly5EgqZz6uXr3KdCsT5lDYxgjbrwYGBmLixImwsrLCnDlzEBISgjdv3sDJyQk2Njas22+AC7tJ27dvR0xMDAYNGoT09HRoaGjgxYsXeP/+vcA5OraZN28e5s2bhw8fPkBcXJyvQceoUaNY2c0CgP79+6N///5YsGABqqqqkJSUhFu3buHYsWNwcXHBL7/8Al1dXYwaNQqGhoasaGjMqlWrYGNjg+rqakyaNAmysrJwd3dHeHg4/P39WbffHIh2tDjOggULoK6uDltbW75Mvbu7O1JTU3Hy5EkqOriS9dq4cSMePnyILVu2QEVFBYQQpKamwsXFBWPGjOGbbs8W38q8tWrVClJSUtSyXsLUoaioiNu3b/O1WG38/aRNTk4O5s2bB2lpaTx9+hTDhg3Ds2fP8OnTJ2rzqwDgypUr2Lhxo0AygO3W6lVVVfDy8sLZs2chISEBIyMjrF27luo5MREivsXnz59x584dtGzZEpqammjbtq2wJeHBgwffTAwdPXqUGZjKJlzwq4MHD0ZMTAzk5ORgZmYGa2trjB49GnFxcQgMDBQ4I802BQUFyMnJwdChQ1FeXs5XNcE2I0aMwKZNmzBp0iSMGzcOQUFB6NOnD2xtbSElJYXNmzdT01JYWIjw8HBm3tzAgQMxa9YsamNCmiIvLw+hoaGIiorC58+fqe36lpSUoKioiPHjDx48QLt27ag1ufq7iHa0OM6jR4+wbds2gfV58+bhxIkT1HRwJevl6OiI1atXY8GCBYxDaujGuHHjRioavs681dbW4vnz59i2bRtMTU2paOCCju3bt/MNI66ursaOHTvQrl07vvfRmH8iJyeH06dP49ixY+jRowfq6uowYcIEzJ07l+oZRzc3N+jo6GDWrFlUD9nv3r0bJ0+exJQpUyAmJoaTJ0+ioqKiyQ5aIkTQJCsrCxYWFnj37h0AoGfPnti7d6/Qy2vnzp2L1atXM2c+gPqzp/b29rh9+zaVQIsLfrVly5ZMIxIZGRk8ffoUo0ePhoqKCl68eEFFA1CfLNq4cSMuXLgAMTExXLp0CZ6enigvL4evr69Ay3U2+PjxI1MBMWDAADx+/Bj9+/fHkiVLsGbNGmqB1pMnTzB//ny0bt0aqqqqqKurQ1RUFMLDw3Hs2DEMHDiQio53797h3r17SEtLQ1paGh4/fozWrVtDR0eHWnt3oL6JTmlpKeLi4iAmJgZlZWVqIxiaA1GgxXEkJCRQVlYmsF5YWEj1QS4vLw++vr6Qk5ODgoICiouLoa+vj5qaGgQGBmLq1KlUdLRv3x7BwcF49uwZsrKyICYmBiUlJaHOUmjRogXk5ORgb2+P1atXC+WMEm0dQ4cOxdu3b/nWNDQ0UFJS0mRXMTZYunQpRo4cieHDh0NOTg49e/bEmjVrqNj+FsXFxbCzs6P+fbx06RLc3NyYzmFjx46Fra0tXFxcqJXUihDRFN7e3vjll1/g6+uLFi1aYMeOHXBycqKaKGwKR0dHeHh44M6dO/D09ERaWhq2bNmC7t2749ixY1Q0cMGvqqioIDIyEmvXroW8vDzi4+NhaWmJ7OxsSEhIsG6/gYCAAOY8+tKlSwHUN4Wwt7eHt7d3kwnn5kZSUhLv379Hr169ICsri6ysLAD1D/oNiQIaeHl5YdiwYdi5cyeTzKysrMT69evh7e2NoKAg1mxHREQwwdWrV68gISEBDQ0N6OnpYdOmTRg8eDC11u5A/ciatWvX4ubNm8x5Sh6PByMjI7i7u1PvVvrfIAq0OI6hoSF8fHywe/duZi0nJweurq4YO3YsNR1cyXo10FBHzCXExMTw5s0bYcugooPmYNFvUVlZiZ07d+LLly/o2bMnRowYgZEjR2LEiBHo3LmzUDQNGzYM9+7dox5ovXnzhu8smq6uLr58+YK3b9+iR48eVLWIENGYv/76CwcPHmR2sFxcXGBkZITPnz8LtbW6qakphg0bho0bN+LXX39FdXU1lixZgqVLl1ILMLjgV21sbGBlZYXOnTvDxMQE/v7+mDhxIgoLCzFhwgQqGgAgNjYW27Zt42vyNWzYMLi6umLDhg1UAq3Ro0fDyckJ7u7u0NLSgpubG8aNG4fz589DSkqKdfsNpKWlISIigq9ipFWrVlixYgXmz5/Pqm1nZ2cMGjQI48ePx/Dhw6GlpcWngzaurq7Izc3Fvn37oKGhgbq6OqSlpTHdS3+2464wEQVaHGfjxo2wsrKCjo4O6urqMG3aNJSVlUFRUREbNmygpkOYWa+G7lA/A42a4aaaUJSVleHEiRNUm0FwRYewOHjwIGpra/Ho0SOkpKQgJSUFTk5OTBczXV1d6OrqQktLi9qD05AhQ7B161Zcv34dffv2FWiLy9act5qaGr7/o4SEBFq3bo3KykpW7IkQ8bOUlZXxBfuysrIQExNDSUmJUAMtoP7sR0VFBVq1aoWqqioUFBSgsrKS2v2CC7tJQ4YMwaVLl1BVVYUuXbogPDwcERERkJaWpjoao6ioCH379hVYl5aWRmlpKRUNGzZsgJ2dHZKSkjB37lwcP34cM2fOhLi4ODw9PaloAIB27dqhurpaYL2pteYmMTGR6WRYVVX1zR2jxMRE6OjosK7nypUr2Lt3L9+4hbFjx6Jly5ZYv369KNAS8fdp3749IiIicOfOHTx+/Bh1dXWQl5fHqFGjqG7fCjPr5ebmxgRa+fn52L9/P0xNTaGhoQEJCQmkp6cjPDycWufDpi5scXFxaGhoUMm6cU2HMGnRogVUVVWhqqoKCwsLAPVnQhoCr+XLl4MQgnv37lHRc+zYMXTp0gV//fUX/vrrL77XeDwelYHaIkRwibq6OgFfJSEhIfQZONu3b8exY8egr6+P0NBQvHz5Ehs3bsSECRPg5OREZa4XF3aT7O3t4eDggJ49ewKoP5u0efNmfPjwAatXr8bevXup6JCTk8OdO3cwc+ZMvvXY2FgMGDCAioaOHTvy/X/37duHjIwMdOvWjWplgI6ODry8vLBnzx6mOqO4uBg7duxg/WxU43bxy5YtQ0BAAF+wVVFRAQ8PD0RGRlJJbLdo0aLJYcndu3dHTU0N6/abA1Gg9Q9h+PDhAhdYaGgoFixYQMW+MLNe06ZNY/4+f/58ODo6YsaMGcyaoaEh5OTkEBoaCktLS1a1ANxoQwtwRwdXqK6uRkpKChITE5GcnIyMjAxISEiw7pg+fPjAOMPGw7UbU1VVhevXr7OmgcfjCez6is5miRDxbaKjo+Hs7Izp06cDqD+fExMTA3d3d6xYsYLKQ6Sw/Gpqairy8vIA1FdGKCsrCzSbyMnJwZ07d1jT8DU2NjawtbVFdnY2amtrER0djdzcXFy6dInv6ERz0zCn6lt07twZNTU1KCgooDaeY926dZgzZw709PQgKysLAHj+/Dk6d+4MNzc3KhqA+sT2ypUr4e/vDwmJ/9fevcflfP//A39cnSSiotvQVCQitDJ0EGP6SBeVMsdK7dJiQ5iVEssqlsPo4FjSQVR0QAcUCZlQLJQtfcbaZk4dVFpHvz/8ur5dOrDPeh/ieb/ddrut9/ut13NMr+v5ej9fz5csLl++DC8vL1RXV7PWaMnBwQE+Pj4ICAhA3759Abx6S75z505W37j+G9TenacOHDiAlJQUyMrKwsrKSqIDUlFREby8vJCfn89ae83mVa/XfxiXl5fD09OTtVWv0aNH4+TJk60Ozbt//z6srKzw008/sRJHTU0NKisroaioyGn5C1/i4Mr9+/fFhytevXoVdXV1GDVqlPhwxdGjRzP+5nf48OG4dOmSRBtid3d3uLm5ia89ffoUpqamjP19bau8tmWr6Jbel4NYCT/o6OjAy8tLYu749ttv4erqKnE8BMDu2X/Xrl3DRx991GaJXlZWFitnVHI1r+bl5Yk/U7R3aLOCggI+//xzVt/CX7hwAfv27RNX72hra8PZ2RnTpk1jbMzhw4dLfN3Wz02mj+doS3V1NY4fPy5x3tzMmTNZ6b7Y7MmTJ3BycoKamhpUVVVx7NgxWFhYYN26day13V+wYAFu3boFKSkpaGpqQkZGBvfv30d1dTUGDBgg8WfFl7PxXkdvtHgoICAAe/bswfjx49GtWzds2rQJUlJSmDdvHg4cOICdO3dCQUGB8bbZfFz10tDQQEpKCr788kuJ67GxsYyXF1RVVYkT4Obfl+aYLC0t4eTkxEqyw5c4uDZ16lT88ccfUFNTg7GxMfz9/WFkZNRmmQGT2vqQcubMGSxbtkxiMmJyTYuNFvqE/K/aWv3esmWLxNcCgYDVRGvFihUIDQ1ts808k0kWH+ZVAwMDcUWEjo4OLl26JH5bwJWSkhJMnDgREydOZHXcHj16oLq6Gh9//DGEQiEvmmzNnj0bvr6+rBwx0BFVVVVERkbCyckJFy9eRHBwMCuHFLdkbGwMY2NjVsfsbJRo8VBKSgpWrFghTiaSkpIQEhKCJ0+eYNeuXTA3N8eGDRtarQZ2NoFAIN4HJBAI2pwsFRQUWCnXa7ZixQqsWLECly9fxqhRo9DU1IQbN26gsLAQISEhjI1bVlYGOzs7PHz4EGZmZpg7dy569eqFyspK3LlzB/v370daWhoOHz7M6Ad9vsTBB7///jv69+8PoVCICRMmQF9fv1XzCT5hspRv1qxZjH1vQv4NvpY4q6iooLKykvVx+TavtvXnU1payvjni9eZmZlhzJgxsLGxwfTp01k71Pry5cu4ePEiUlNTsWXLFqirq8PCwgJCoZCzw4FLSko4O9S7rSZb1tbW2LFjB+Lj4yWOG2JjYeRd2NdMpYM8pKenh6SkJAwaNAjAq/0denp66NGjB7y8vFhd9WvGl1Uv4FXZw6FDh1BUVATg1av/zz//XHxqOBO8vb2Rk5ODsLCwNg/K++uvv+Ds7IypU6fC1dX1nY+DD/773/+KywavXbsGKSkpjB8/Xlw2+Hp5KVN0dHSQnZ0t8fZKX18fJ06cELd5Z7p0sNnvv/+OuLg45Obmij8sjRkzBp999hmnZ80Rwjf+/v44fPgwJk2aBA0NjVYtrNn4gMeHebWyshJbtmyBnZ0dhgwZApFIhJycHGhqamL//v2s/dy4du0aTp48Kd6zZmZmhlmzZrF6MG5tbS0yMzORmpqKixcvYtiwYbCwsMD06dOhqqrKWhwhISG4cOECRCIR1NXVIS8vL3Gfyb1ib/s5iq1SyrYSv5a4+Dz8T1GixUPtfXD75ptvOH+V3BIXq15cmTx5MjZs2IDJkye3+0xGRga2bt2K06dPv/Nx8E1dXR2uXr2KS5cu4dKlS7h37x4+/PBDmJiYwNTUlNFyB74kWikpKfDy8oJAIIC+vj6UlZXx/Plz/PTTT6irq4OPjw9nh2kTArza87Fz507k5eWhvr6+VTktm3ssOuoqKBAIONvvwfa86uHhgevXr2Pv3r0oKirCN998g02bNiE1NRUyMjIICgpiLRbgVVOj8+fP4+TJk8jKykKfPn1gbW2NFStWsBpHdXU1MjMzkZaWhh9//BGjRo1CREQEK2Pr6uqKu3K2rITgYq8Y19pL/Lp164Z+/fp1ic85/K2zIa1wWafKl1Uv4NVG5QMHDuC///0vYmNjkZCQAHV1dVhZWTE25tOnTzF06NAOn9HR0XljB6N3JQ6+kZOTE7/JAl6VXkRERCAhIQFxcXGMT0xcd/grKCjA2rVrYW9vD1dXV4nV+bq6Ouzduxeenp4YMmQIo29+CenI+vXrcfv2bQiFQs5Lm9vrEMomPsyrWVlZ2LVrF7S0tBASEgITExPMnDkTw4YNw8KFCxkf/3WysrLiMsKkpCTs2rUL+/btYz3RamhoQG1tLerq6lBXVyexH5ppBw8eZG2sN0lKSkK3bt3Exw24urpi6tSpmDlzJivjv17a2tjYiPv378Pb2xtz585lJYZ/ixKtLkRaWpqzsTdt2oTr16/D0dER6enpyM3NxZYtW8R1zWytemVnZ2PZsmUQCoW4efMmmpqa0NDQAA8PD7x8+ZKx18j19fWtXt+/Tl5envFzHfgSB988ffoUN27cQF5eHvLy8lBQUAB5eXkYGhqyUnri6+srkdzU19dj69at6NGjBwAwfnDwgQMHMG3atDYPMZeTk8OKFSvw+PFjhIaGYtu2bYzGQkh7rly5gtDQUHz88cdchwLg1RuCixcv4pdffoGMjAy0tbVhaGjI2lzLh3n1xYsX4jL07OxsODs7A3g1j7B91tmLFy+Qnp6OkydP4sqVK1BTU4NIJGJtD2p5eTnS09Nx6tQp5OTkoE+fPpg2bRq+/PJL6OvrsxJDVVUV9PT0WpWyciEyMhLbtm3D+vXrxdf69euH9evXo6amBnPmzGE9JmlpaWhpacHDwwOurq5dokqDEi2eCgsLk+gc19DQgMjISPTu3VviObY2CvJl1SsoKAhff/01HB0dxa+MV61ahZ49e+LAgQNdol6XdI6YmBhxcvX7779DVlYW+vr6mDx5Mjw9PTFq1ChWDvUeO3Ysnjx5InFNX18fZWVlKCsrE19j8sPl9evXERAQ0OEzc+fOxVdffcVYDIS8iYKCAmttod+kvLwcIpEId+7cgaKiIl6+fImqqiro6uri4MGDEge3MoUP86qWlhbOnz+P/v3748mTJ+Kuf3FxcdDS0mIlBuDVPH7+/HkIBAKYm5sjPDyclYS8rKwMGRkZ4uRKWVkZ06ZNw9KlS1ldEHj+/Dnc3Nxw4cIFCAQCfPLJJ/Dx8eF0e0ZUVBS+//57WFhYiK95eHhg5MiR2LVrFyeJVjMpKSk8fvyYs/H/CUq0eGjAgAFIS0uTuKaqqtqqZlwgELCWaPFl1evnn39u1RYYAMzNzREcHMzo2K8nv6978eIFo+PzLQ6ufffddxgxYgTMzc1hZGSEMWPGcLIKGBUVxfqYrystLcUHH3zQ4TOqqqp4/vw5SxER0pqVlRVCQ0Px3XffcVqhAbxqhvH3338jKSlJXE579+5dfPPNN9i+fTs2btzIeAx8mFdXrFiB5cuXo76+HjNmzICmpiY2b96M6Oho7Nq1i5UYgFdVCd9++y2mTZvG6vEkpqamEAgEMDY2hq+vLz7++GPxAt3rJfhMNqHYsmUL8vPz4erqCikpKURFRcHb2xuBgYGMjfkmjx8/xsiRI1td/+ijj1jbntBWM4yqqirExcVh9OjRrMTwb1GixUN8qB1/HV9WvRQVFfH48WOoq6tLXL93716rt32dqa3kty1tdQJ8F+PggytXrohXnevq6iAnJ9fuc4aGhmyGxjpVVVX8+uuvHf65FxcXo1+/fixGRYik8vJyJCcn4/z58xg4cGCrv7ORkZGsxZKZmYnAwECJPYvNhyuvXr2alUSLD/PqpEmTkJWVhUePHol/L4RCIebMmcPq3B4VFYWSkhL88ssvUFFRwYcffsjK3tfmMvusrCxcuHChzWfYaEJx4cIF+Pv7w9TUFMCrqggnJyc0NDRwdmyJpqYmzp07B0dHR4nrWVlZ+PDDD1mJofkohJZkZGSgr68Pb29vVmL4tyjRIm+FL6teM2fOxKZNm7Bp0yYIBAJUV1fjwoUL8PHxkXi93dn4kvzyJQ4+aFnas3TpUuzZs0fig9uLFy/w/fff4+jRo+98l6ZJkyZhz549MDIyavPDSVNTE/bu3cv6YZOEvI4veyoaGhrabKvet29fibOCmMSXeVVZWRkVFRU4c+YMpKSkoKury9pi3cuXLxEaGopDhw5JlIKpqqrCzs4Ozs7OjCZcbCb3HSktLZVodKWvr4/GxkY8e/bsjdUKTBGJRFi7di3u3LkDPT09AMCtW7eQkpICHx8fVmLg6zl8/wS1d+c5PrXDLSsrk1j1ys/PR48ePVhd9aqvr8fatWuRkpIC4FX55MuXL/HJJ58gICCAsw2kFRUV4g53b/PG6V2Pg23m5uZQV1fHrl27ICsri8uXL8PLywvV1dVwc3ODra0t1yEy6tGjR7C2toaenh5WrlwpsUp/9+5dbNu2DcXFxUhISICysjKHkRLCD4sWLcLQoUOxbt06ieu+vr64ffs2YmJiWImD63m1qqoKq1evxsWLF8WfLwQCASwsLLB58+Z2KwU6y1dffYULFy7AysoKRkZG4qQvJycHx48fh7GxMWtJJ5eVEW9zTAgXUlNTERkZiZ9//hmysrLQ0tLCF1980eERM52tpqYGlZWVUFRUZLWstLNQosVzS5Ys6bAdLtunZt+/fx+//PIL66ter3vw4AEKCwvR1NSEoUOHYsCAAdi5cyc8PT1ZjSM3NxcxMTE4c+YMamtrMWLECCQkJLAaA5/i4MqTJ0/g5OQENTU1qKqq4tixY7CwsMC6det4s/meaYWFhXB1dUVJSQm6d++O3r17o6qqClVVVdDW1saOHTtYXRQhpC0PHz5EdHS0RKe/uXPnMrr/pS03btyAg4MDdHR0YGBgAODVz9G7d+8iNDSU1XJjLufV5nO0NmzYAH19fTQ1NSEvLw8+Pj4wMzNrs3Srs8THx+P7779HZGQkhg8f3up+UVER7O3t4ebmBhsbG8biaCYSiTirjOBrosWVqqoqHDhwACkpKRKt9TU0NGBpaQknJ6cuk3RRosVzH330ES/a4XK56lVbWwt/f3+kpKRAVlYWVlZW+Prrr8UbVi9duoRvv/0WDx8+REFBAWNxNKuqqkJSUhJiY2Nx7949AMCECRPg7OyMcePGMT4+3+Lgi9LSUjg5OaGoqAiBgYHvZZlcY2Mjzp8/j5s3b6KiogLKysoYM2YMJkyYwEoHRkI68vPPP8POzg7y8vIYPXo0mpqacPv2bdTU1ODIkSPQ1tZmNZ78/HyEhYWhqKgIL1++xLBhw+Dk5MTaJnuu3yYBr7qm7t69G2PHjpW4fvnyZaxZswaXL19mbOx58+Zh+vTpWLRoUbvPREdHIyUlBYcPH2YsjmZcVkbo6OggPj5eouJAKBQiJCSk1SIEm4sSpaWl+PXXX9HU1ATgValnXV0dbt26haVLlzIyZllZGezs7PDw4UOYmZlh6NCh6NWrFyorK3Hnzh2cPXsWAwcOxOHDhzk/j+9tUKLFc8bGxoiOjsagQYM4jYPLVS8fHx/ExsbC0tIScnJyOHHiBJydneHi4gJfX18cOXIE6urq8PX1bTVZdKb8/HzExsYiNTUVNTU10NbWhlAoRFBQEI4fP44hQ4YwNjYf4+BaW92IysrKsGPHDpiYmGDatGni6+96238PDw+sW7cOPXv25DoUQtolEonQvXt3bN++XVzmXVtbizVr1qCurg779u3jOEJ2cTmvNjM0NER4eHirg8yLioqwcOFCXL16lbGxx44di9jYWAwePLjdZ3777TfY2tri2rVrjMXRjMvKCB0dnVZ70ZqbcLz+NVt7jk+cOAEvLy/U1dWJt2k0x6OmpoaMjAxGxvX29kZOTg7CwsLafLv7119/wdnZGVOnToWrqysjMXQmSrR4zt/fH8+fP+e8HS6Xq16TJ0/GF198gfnz5wMAzp8/Dz8/PxgZGeHYsWNwcnKCq6sro6t/NjY2KCwshJaWFszMzGBhYSFefdXV1WUtweFLHHzw+geD9rA5MXFl+PDhuHTp0ntTJkm6Jn19fcTExGDYsGES1+/evQs7Oztcv36d0fFbLkh4eHh0+OzmzZsZjQXgdl5ttnv3bmRnZyMgIEDcHKSqqgru7u4YPnw4o9sT9PX1cfz48VZdhFv67bffMHv2bEYTvpa4qoz4J/99bFWsWFhYQE9PD4sXL8b8+fMRFhaGx48fY+PGjVi9ejWsrKwYGXfy5MnYsGFDh/vAMjIysHXrVvF5qnxGXQd5ji/tcKWlpdt8Rauqqipuj8qUp0+fYsKECeKvTU1N8ccffyA9PR0HDx7E+PHjGR0fAAoKCjB48GBYW1vDxMSE9RIXvsXBB+9CN6LOQutlpCvo0aMH6uvrW11v6xoTfv/9d3EJ1O+//87KmB3hcl5tdunSJdy6dQuffvopNDU1ISMjg/v376O6uhqFhYVITEwUP9vZzbeGDBmCS5cuYcGCBR3Gx+Q811ZlhLW1NXbs2IH4+HiJDpRMVka0TJ7e1JSDLSUlJQgKCoKWlhaGDRuG0tJSTJkyBQ0NDdi7dy9jidbTp08lOjC2RUdHh7WzvP4tSrS6AD60w3VwcICPj0+rVa+dO3fCwcGB0bHr6+uhoKAg/lpaWhrdunXDunXrWEmyAOD06dNISEhAVFQUtm/fDnV1dVhYWDDaUp7PcfBRUlISunXrhunTpwMAXF1dMXXqVMycOZPjyNjBxpkzhPwbhoaG2LJlCwIDA6GkpATg1RuErVu3wsjIiPHxWx4u3vLf6+vrcfnyZbx8+RJGRkasda/lcl5tZmxsDGNjY1bGet2sWbMQHBwMExMTaGhotLpfXFyM4OBguLu7MxZDR+WZmZmZyMzMBPDq5ytbJeh8Oa5ETk5OHIOGhgaKioowceJEjBw5Eg8ePGBs3Pr6esjLy3f4jLy8PGuLEf8WlQ6St7JgwQLcunULUlJSrVa9BgwYIPEhr7NXvdrrxvOmkgMmNDU14eLFi4iPj0dmZqb4L/rSpUvh5OTE2sZMvsTBF5GRkdi2bRvWr1+Pzz77DMCr0p/Y2Fh4enpizpw5HEfIrLbq+9vzrpdREv7666+/MG/ePFRUVEBTUxPAq457SkpKiIqKYuUQ1MOHD4s7ss6ZMwdCoRB2dnbiN+QffPABIiIi2vzg39m4nFf5oKmpCUuWLEFeXh5sbGxgYGAAJSUlVFVV4dq1azh69CgmTZqEHTt2cB0qq/hyXMmiRYugp6eH1atXIzIyEllZWThw4AAuXLgANzc3xt6utfWZ73VPnz6Fqalpl5jPKNHqAvjQDjc4OPitn+3smm4dHR1cvnwZKioq4mt8aHtaXl6OEydOIDExEYWFhejevTssLS2xcePG9zIOLpmZmWHVqlWt3uydPHkSu3btwqlTpziKjB06Ojrw9PR8qwR71qxZLERESNuqq6tx/PhxiU5/M2fOZKWRy4EDBxAcHIyZM2eie/fuSE5OxsCBA/HixQt4e3ujqakJmzdvhpqaGgIDAxmPh8t5tVlbpXMtMf0Wp7kM7dChQygvLxdfV1VVhaOjIz7//HNW39bzoTKCL8eVXL9+HYsXL8aKFSswa9YsmJubo2/fvnj48CGmT58OPz8/RsbV0dERN85pz4sXL3Dw4EFKtMi/x7d2uFzQ0dGBhYWFRDnHyZMnMWXKFPTo0UPiWTY2MLeloKAACQkJSE5OZrWGmq9xsE1PTw8nT55s9YazpKQEQqEQ+fn5HEXGjrdZASSEa9evX4eBgUGrowZevHiBsLAwxs+FnDZtGlxdXcULMvn5+ZgzZw727t2LTz75RByjq6srsrOzGY2FL9prKtStWzf069ePtWYDL1++xK+//io+lkJdXZ31Iyn4VBnBl+NKHj16hLq6OgwcOBD37t1DTEwM+vfvDwcHB8jKyjIy5pQpU9762XPnzjESQ2eiRIvn+NIOl8tVL3t7+7d+tmXdPZMaGxtRVlYGRUVFiQTw4cOHUFZWfmN98bsWB9esrKwwa9YsODo6Slw/dOgQDh8+jNTUVG4CYwl1HSRdgY6ODkxMTBAQECDxBoutMqBRo0bh1KlTUFNTE18bOXIkTpw4IW4x/uTJE3zyySe4c+cOo7EA3L9NaktjYyPu378Pb29vzJ07l7U94rdu3cKoUaNaXa+srISvry/8/f0Zj4HLygg+HlfS3rEh5eXl8PT0xO7du1mJo6ujZhg8l5eXh5iYGIkP0d26dcNXX30FOzs71uJob8No86oXk3/x2Uqe3kZqaioiIyORn58v7vQ2dOhQ2NvbY/bs2fDz84Oenh6cnZ3fizj4QiQSYe3atbhz5w709PQAvJq4U1JS4OPjw3F0zKP1MtJV/Pnnn5g7dy727dvHyp6sltraZC8rKyuxMi8QCMSdCZnG5bzaHmlpaWhpacHDwwOurq6sJVqLFy9GeHg4hg8fLr527tw5bNiwAY2NjazE8PjxY4wcObLV9Y8++ojxDnd8acqRm5uLkpISAK+SP11d3VaJVnFxMX788UfGYniTiooKJCQkIC4uDmlpaZzF8bYo0eI5rtvhNnu9lfbrq17vA19fXxw6dAhGRkZYuXIllJWVUV5ejpycHKxfvx5JSUkoLi5mrG6Zb3HwiaWlJWRkZBAZGYmMjAzIyspCS0sLQUFBHZ7F8a5YtmwZjhw58tbPEsIFgUCA/fv3w9/fH5999hmCgoLw8ccfcx0WZ/g8r0pJSeHx48esjWdpaQlHR0dERESgf//+8PHxQXJyMmbMmAFPT09WYtDU1MS5c+daVUZkZWUxvijAl+NKBAKBOOkTCATw9fVt9YyCggJEIhHboSE3NxcxMTE4c+YMamtrMWLECNZj+F9QosVzXLfDbQ9Xq15cSU9PR2xsLPbt24dJkyZJ3HN2dkZGRgaWLVsGFxcX9O7d+52Pg4/e5zb3wcHBkJKSQr9+/Tp8TiAQUKJFOPPy5UsoKCggODgYW7ZsgZOTE7777jvx/ig2hIWFSWyyb2hoQGRkpPjn5YsXL1iL5XVczKttlaxVVVUhLi4Oo0ePZnz8ZuvWrYOsrCwcHR0hIyMDOTk5hISEwNTUlLUY+FQZwVVTDgMDA3HSp6Ojg0uXLomPHuBCVVUVkpKSEBsbi3v37gEAJkyYAGdnZ9YObv63KNHiuTVr1mDevHmYPHlyq3a4mzZt4jY4sL/qxZXo6Gi4uLi0Sm6a3bx5Ez179kRubu57EQcflZaW4tdffxWX/bx8+RJ1dXW4desWli5dynF0zJozZw7S09MBAEKhEEKhsN1N7oRwpWX3ODc3NwwaNAjr16/HwoULWRl/wIABrUqNVFVVW7VO79+/PyvxtIfNebWtkjUZGRno6+vD29ublRiaubm5QU5ODvv378eRI0fEyQ5b+FIZ0bIpR7N+/fph/fr1qKmpYa0pR1tv2UpLSyU6QDMlPz8fsbGxSE1NRU1NDbS1teHq6oqgoCC4u7tjyJAhjMfQWagZRhfAZTvcZh2tevXu3ZtX+6iYYGhoiKioqHa7PAqFQixduhQ+Pj7Iycl55+PgmxMnTsDLywt1dXUQCAR4+fKl+EOdmpoaMjIyOI6QeY2Njbhy5QpSU1ORkZGBPn36iJOu5kUaQrjUVnfMy5cvw9XVFVVVVV2iVXNnet/n1fYOZb558yYUFBQwdOhQ8bXIyEi2wuIcX44rqaysxJYtW2BnZ4chQ4ZAJBIhJycHmpqa2L9/P2PH69jY2KCwsBBaWlowMzODhYWF+DOPrq4ujh8/3qUSLXqjxXPN7XAXLFggcf3FixcIDg5mrQyIT6teXKirq4OCgkK795OTk/HHH38wvneOL3Hwzd69eyEUCrF48WLMnz8fYWFhePz4MTZu3Ijly5dzHR4rpKWlYWJiAhMTE3h7e+PSpUtIS0uDra0t1NXVYWFhAaFQyOr5e4S01LJEr5mxsTFiYmK6xKb2zsaXebWmpgaVlZVQVFTs8Oyiztay++PbXGcDHyojuGzK0dKmTZtw/fp1ODo6Ij09Hbm5udiyZQtSU1OxZcsWBAUFMTJuQUEBBg8eDGtra5iYmHT5Y4wo0eI5Ozu7NtvhvnjxArt27WIt0eLLRk2uaGhoIC8vr90JQCAQIDc3l/E3B3yJg29KSkoQFBQELS0tDBs2DKWlpZgyZYr4MEwrKyuuQ2SVrKwsJk+ejMmTJ6Ourg7x8fHYvn07fvjhh/furQHhj3HjxqGhoQGPHj0Sd5J7+fIlpKSkWp2B9z7gcl6tqqrCgQMHkJKSIu4yB7yaYywtLeHk5MR40tXWuZd1dXWQk5MD8OoMpw8++IDRGFp6U2UEW4kWl005Xh9v165d0NLSQkhICExMTDBz5kwMGzaM0XLf06dPIyEhAVFRUdi+fbt4obCr7sGmRKsL4LIdbktcrXrxgVAoRGBgIExNTcVNSVp68uQJAgMDsWjRovciDr6Rk5MTT84aGhooKirCxIkTMXLkSDx48IDj6Ljx+PFjnDlzBqdOnUJubi40NDT+0Zl0hHS2S5cuwd3dHaWlpa3uycvLw9LSkoOouMXFvFpWVgY7Ozs8fPgQZmZmmDt3Lnr16oXKykrcuXMH+/fvR1paGg4fPgxFRUVWYiotLcXKlSuhr6+PVatWAQBmzZoFHR0d7Nixg5XmTnypjOBLU44XL16I9ytmZ2eLj4uRl5dntOW+hoYGVq1aBVdXV1y8eBHx8fEIDQ3F3r17Abw63sbJyYm1/zf/LUq0eI7rdrh8WPXiAwcHB6SlpcHa2hoikQgGBgbo1asXysvLce3aNYSFhUFDQ4PxTd18iYNvRo4ciaNHj2L16tUYOnQosrKyIBKJcO/ePcZOr+ejR48e4fTp0zh16hRu3LiBgQMHYvr06fDy8qLmGIRzP/zwA0aMGAF7e3u4urpi27Zt+PPPPxEYGNjm2413FdfzakBAAJqampCSktJm44+//voLzs7OCAsLg6urK2NxtOTn54eamhoIhULxtZCQEHh7e8Pf35+V5l98qYzgS1MOLS0tnD9/Hv3798eTJ08wceJEAEBcXBy0tLQYH19KSgqTJk3CpEmTUF5ejhMnTiAxMRG7d+/GwYMHYWlpiY0bNzIex79FzTB4ruXm4S1btiAqKkrcDtfY2JjRMqDXV72GDh0qsep19uxZDBw4kNVVLy5VV1fD398fSUlJEnug5OTkYGNjAzc3N1aSTr7EwSfXr1/H4sWLsWLFCsyaNQvm5ubo27cvHj58iOnTp7/zZ4qFh4fj9OnT+OmnnzBgwABMnz4d5ubm0NXV5To0QsRGjx6NuLg46OjoYMGCBVi+fDmMjIwQHx+PY8eOvfVZcF0ZH+bVyZMnY8OGDR1+aM/IyMDWrVtx+vRpRmJ43fjx4xEREdFqQej27dv44osvcPnyZcZjGDNmDJKSkjBw4EB4eXlh0KBBEIlE+PPPPzFz5sz3rptvVlYWli9fjvr6egiFQmzbtg2bN29GdHQ0du3a1W73Y6YVFBQgISEBycnJuHLlCicx/BP0RovnuGyHy8dVLy716NEDHh4ecHNzQ35+PsrKyqCiogJlZWVoampCXl7+vYqDTz7++GOcPn0adXV1UFZWRnR0NGJiYtC/f/92O1u9S77//nvIysrC1NQUo0aNAgBkZmYiMzOz1bN0jhbhirS0tDh50NDQwC+//AIjIyMYGhrC39+f4+jYwYd59enTpxId/dqio6PDauOFxsZGtLXuLysri5qaGlZi4FNlBB+ackyaNAlZWVl49OiROAEWCoWYM2cOK2+0mjU2NqKsrAyKioro1q0bRowYgREjRkAkEuHvv//m/WceSrR47vUfPJ999hnU1NRYSWyysrKwYcOGds8U6devH1xdXbF169b3ItFKTk6Gn58fQkJCYGxsLL4uEolw+/Zt+Pr6wszM7L2Jg088PDywbt068cbpIUOGwMvLC+Xl5XB1dcXu3bs5jpBZzZ0Ei4qKUFRU1O5zdGAx4ZK2tjbOnTsHe3t7DB48GLm5uVi0aBH++usvrkNjDR/m1fr6+jd+OJWXl0dDQwMj47dl7Nix+OGHH7Bjxw5x46+qqioEBARg7NixrMSwfPlyLF68GEpKSpg1axZ27doFoVAoroxgC1+acgCAsrIyKioqcObMGUhJSUFXV5e1c+ZSU1MRGRmJ/Px88WfhoUOHwt7eHrNnz4afnx/09PTEe8f4ihItnuOyHS4fV724kpOTAzc3N1hbW7fqguTp6YnQ0FCsXLkSUVFRMDAweOfj4IPc3Fzx/oakpCTo6uq2OluuuLgYP/74IxfhsercuXNch0DIG33xxRdYsWIFZGVlMWPGDAQFBeGLL77Azz//DENDQ67DYwXNq23z8PDAwoULMXHiRHHX3Pv370NJSQmhoaGsxMCXygi+NOWoqqrC6tWrcfHiRXGiIxAIYGFhgc2bN4sbUDHB19cXhw4dgpGREVauXAllZWWUl5cjJycH69evR1JSEoqLi7vEtgBKtHiOy3a4fFz14sr+/fthZ2cHT0/PVve0tLTEG7n37NmDkJCQdz4OPhAIBOJzaAQCAXx9fVs9o6CgAJFIxHZohJA2TJ06FUePHoW0tDT69++P0NBQHDx4EJ9++ilWrFjBdXis4Mu8GhYW1uFe3hcvXjA6/uvU1dWRmpqKlJQUFBUVQUZGBvPnz8fMmTNZKw3jS2UEX5py+Pn54ddff8X+/fuhr6+PpqYm5OXlwcfHBz/88EOb58B1hvT0dMTGxmLfvn2t9oE5OzsjIyMDy5Ytg4uLCyvdKP8tSrR4jtrh8kNBQQHc3d07fGbBggVYsmTJexEHHxgYGIjPodHR0cGlS5fQt29fjqMihHSkZYOWcePGYdy4cRxG834aMGDAW1XEsFUi1kxRURHz5s1jdUw+Vkbw5biSjIwM7N69W6J085NPPoGcnBzWrFnDWKIVHR0NFxeXdptt3Lx5Ez179uwyzUko0eI5rtvh8m3Viyu1tbVvXFVTUlJifNMuX+Lgm7YO/iwtLYWKigoH0RBC2lJaWoqQkBAUFRWhrq6u1f3IyEgOomIf1/MqH0uNa2trERsbi19++UXijKa6ujrcvn2bse6HfKyM4EtTjpbNa1pSVVVl9I3r3bt3sW7dunbvZ2Zmwtvbm9Uzxf4NSrR47t69e9i0aRN0dHQwfPhwKCgowN7eHgoKCjhw4ACmTp3K2Nh8XfXiwqBBg3Djxo0OyzXz8vKgpqb2XsTBN5WVldiyZQvs7OwwZMgQiEQi5OTkQFNTE/v378fAgQO5DpGQ956bmxtu3boFY2Nj3ncKY0pXmFcrKiqQkJCAuLg4xveCN/P19UVSUhJGjBiBW7duQV9fHw8ePMCzZ8/g6OjI2Lh8rIzgS1MOBwcH+Pj4ICAgQPx7UlVVhZ07dzK6Z62urg4KCgrt3k9OTsYff/whcbwNn1GixXNctsPl46oXVywtLREQEABDQ8NWTSiAVwfFBgQEwNbW9r2Ig282bdqE69evw9HREenp6cjNzcWWLVuQmpqKLVu2ICgoiOsQCXnv5ebmYt++fe91uSCf59Xc3FzExMTgzJkzqK2txYgRI1gb++zZs9i8eTNmzJgBMzMz+Pj4YODAgVi1ahVrH6j5UhnBl6Ycly5dwq1bt/Dpp59CU1MTMjIyuH//Pqqrq1FYWIjExETxs2fPnu20cTU0NDpcMBYIBMjNzRU3TeE7SrR4js/tcLlY9eKKnZ0dTp8+jRkzZsDW1hb6+vro1asXysvLkZeXh8TERGhqajJeXsCXOPgmKysLu3btgpaWFkJCQmBiYoKZM2di2LBhrJw5Rwh5sw8++AA9evTgOgxeY3teraqqQlJSEmJjY3Hv3j0AwIQJE+Ds7MxqQvz8+XNxp9whQ4agoKAAgwcPhouLC1auXAkvLy/GY+BLZQRfmnIYGxtLHCHDFqFQiMDAQJiamkJJSanV/SdPniAwMBCLFi1iPbb/BSVaPMfHdrhcrnpxRVpaGuHh4di5cyfi4+MRHh4uvte3b18sXLgQS5cuZbwchi9x8M2LFy/EpTbZ2dniczXk5eUl6v0JIdz55ptvsHHjRqxatQoDBw6ElJSUxP3m8+DeR2zPq/n5+YiNjUVqaipqamqgra0NV1dXBAUFwd3dHUOGDGF0/NepqKjg2bNnGDBgADQ1NfHLL78AeHWO09OnT1mJgcvKCD425eDqzEUHBwekpaXB2toaIpEIBgYG4gXla9euISwsDBoaGl1mEVXwsq2juAmv3LlzB9LS0tDR0cHVq1dx8OBB9O/fHytWrGgz22cCX1a9+KChoQElJSWoqKiAiooKBg4cKD5M8H2Mgw9sbW3x2WefoX///nBxccGpU6egqamJbdu2IScnB0ePHuU6RELeexkZGXB3d2/V7KH5QNbCwkKOIuMGV/OqjY0NCgsLoaWlBTMzM1hYWEBbWxvAq66Qx48fZz3R8vLywt27d7F582Y8ePAAmzZtQkBAAFJTU3Hu3DnGmmG0ZGxsjF27dkFfXx9r165FeXk59u7di19++QULFy7EtWvXGBs7Ly8PCxYsAADxIcWvU1BQwOeff85aApSUlNThfWtra8bGrq6uhr+/P5KSkiRKR+Xk5GBjYwM3N7cOG8rwCSVapENtrXoJhUIEBQVx8sOYkLZkZWVh+fLlqK+vh1AoxLZt27B582ZER0dj165d7baJJYSwZ8qUKRg+fDjmzJnT5oek92XRjut5VUdHB4MHD4aNjQ1MTEwwfPhw8T2uEq3nz59j7dq1MDExwYIFC/DFF1/g4sWLkJGRgb+/P4RCIeMxfPTRRzh16hT69esHU1NTODs7w8HBAb/99husra2Rl5fHeAwAf5py6OjotHm9W7du6NevH+PJb01NDRobG5Gfn4+ysjKoqKhAWVkZmpqaXapqhxItnuOyHS4fV70IaU9ZWRkePXoknhzy8/PRo0cPaGlpcRwZIQR49UH25MmT73UXUD7Mqw8ePEBCQgKSkpLw+PFjqKurw8LCAhYWFrC2tubF3P7y5UsUFhaib9++EAgEUFVVZXxMPldG8OG4ksbGRty/fx/e3t6YO3cuZsyYwdhYycnJ8PPzQ0hICEaOHCm+LhKJcPv2bfj6+sLMzIyx8TuT1JsfIVxyc3NDQkICFBUVoaam1uofJhUUFGDQoEGwtrbGf/7zH/FkQAgfKSsrQ15eHmfOnEFGRgZUVVUpySKER8aPH48bN25wHQan+DCvamhoYNWqVcjMzMTevXsxbNgwhIaGwtLSEk1NTUhNTUVlZSWrMQ0fPhylpaXirwUCAUaMGIG6ujr85z//YSWGFStWYNOmTViyZAlmzJgBTU1NbN68GeHh4azuV6qsrMT69evx888/o7GxEY6OjjAxMcH06dPF+7i4IC0tDS0tLXh4eCAgIICxcXJycuDm5obJkye36q7s6emJKVOmYOXKlay9Yfy36I0Wz+nr63PWDrcrrHoRArza67B69WpcvHhRXNsuEAhgYWGBzZs3Q05OjuMICSEhISHYvXs3Jk+eDHV1dcjISPbj4mrzPZv4Oq+Wl5fjxIkTSExMRGFhIbp37w5LS0ts3LiRsTGPHTuGEydOAACuXr0KfX39VgfyPn78GDU1NcjKymIsjpb4UBnh4eGB69evY+/evSgqKsI333yDTZs2ITU1FTIyMpwfV3L37l3MnTsXP/30EyPfXyQSQUtLC56enu0+4+HhgadPnyIkJISRGDoTJVo8Z25uju3bt0NXV5ezGJqamnDx4kXEx8cjMzNTfCL40qVL4eTk1ObJ4YSwqXli2rBhA/T19dHU1IS8vDz4+PjAzMwMa9eu5TpEQt57U6ZMaffe33//jcuXL7MYDbf4PK8WFBQgISEBycnJuHLlCmPjlJeXi88DTUxMxPTp01vtvenRowesra0lyseYdv/+ffzyyy+QkpKCrq4u64dHc9mUo6W2mmFUVVUhLi4OvXv3RlRUFCPjGhkZISIiAkOHDm33mVu3bmHJkiXIzs5mJIbORIkWz509exb79u3jTTtcLla9CHmTsWPHYvfu3Rg7dqzE9cuXL2PNmjXv1Qc4QrqSoqIixMTE4OTJk7h69SrX4XCC63m1sbERZWVlUFRURLdu3cTXHz58KC7JZlrz2VE9e/ZEaWkprl+/jj59+mDMmDGMj92ML5URfGrK8ToZGRno6+vD29ubsbd8BgYGSEpKgrq6ervPlJSUwMrKqkuUD9I5Wjz38uVLFBcX4/PPP291nYt2uEpKSnBwcICDg4PEqhclWoRL0tLSba4Aq6qqileKCSH8UFdXh1OnTiEmJgY3btyAQCDA1KlTuQ6LM1zNq6mpqYiMjER+fr44sRg6dCjs7e0xe/Zs+Pn5QU9PT3wuIRN2796NiIgIxMXFoWfPnrhx4wacnZ1RXV0NADA0NMSePXtYSfb8/Pzw66+/Yv/+/a0qI3744QfWKiO0tLRw/vx59O/fH0+ePMHEiRMBAHFxcayWMN69e5e1sVoaNGgQbty40WGilZeXx3ifgs5Cb7R4jk/tcPmw6kVIW3bv3o3s7GwEBASIW+JWVVXB3d0dw4cPfy/2fhDCdw8ePEBMTAwSExNRXl4OgUAAGxsbLFmy5L3tRMjVvOrr64tDhw7ByMgIRkZGUFZWRnl5OXJycpCdnY0xY8aguLgYp06dQu/evRmJITY2Fr6+vnB0dISLiwt69uwJc3NzVFVVITw8HIqKili+fDkmTJiAFStWMBJDS3ypjODTcSU1NTWorKyEoqIia+dWRUREICIiAkeOHGnVDAMAHj16hPnz58PW1hZfffUVKzH9G5Ro8Rwf2uG+adVr2bJljK96EdKRBQsW4NatW5CSkoKmpiZkZGRw//59VFdXY8CAARIHOZ89e5bDSAl5vzQ2NuLMmTOIjY1FTk4OpKWlMWHCBAiFQnh4eCApKem9bKrE5byanp6O1atXIzg4uM0P7RkZGVi2bBlcXFywatWqTh+/2ezZszFr1iwsXLgQwKt9N5999hlWrVoFFxcXAEBmZia+//57Vg4sNjQ0RHh4eKuSuaKiIixcuJDV0lYum3JUVVXhwIEDSElJkehyqKGhAUtLSzg5OTGadDU2NsLe3h5FRUWwtbWFvr4+evXqhfLycuTl5SExMRGampo4dOhQl1jgp9JBnmtuh8tVotVy1WvlypUSq17r169HUlISiouL4efnx0l8hACvNg8bGxtzHQYh5DWTJk1CZWUlDA0Nxc1pmt+QvK9NarieV6Ojo+Hi4tLum5GbN2+iZ8+eyM3NZWT8ZsXFxTAxMRF/feXKFQgEAom4hgwZgj///JPROJo5ODjAx8enVWXEzp074eDgwEoMzZSVlVFRUYEzZ86w2pSjrKwMdnZ2ePjwIczMzDB37lz06tULlZWVuHPnDvbv34+0tDQcPnyYsYYt0tLSCA8Px86dOxEfH4/w8HDxvb59+2LhwoVYunRpl0iyAEq0eO/jjz/Gt99+i/Pnz7PeDjc9PR2xsbHYt29fqx/Izs7OEqteTJUWEPI2qDSQEH6qrKxEnz59MGDAACgpKbFWfsRXfJhX7969i3Xr1rV7PzMzE97e3vDx8WFk/JZaVhtcv34dvXv3lnijVF1dzdr/M5cuXcKtW7fw6aeftqqMKCwsRGJiovhZJisjuGzKERAQgKamJqSkpLSZ2P31119wdnZGWFgYXF1dGYtDTk4Obm5uWL16NUpKSlBRUQEVFRUMHDhQ4v+ZroASLZ47cuQIlJWVcfPmTdy8eVPi3t9//83oB0y+rHoR8iZttaFtydrampU4CCGSsrOzkZqaivj4eBw5cgQ9evTAp59+CgsLiy73gakz8GFeraurg4KCQrv3k5OT8ccff6C+vp6xGIBXpZJ5eXnQ0NDA8+fPkZOTg08//VTimbS0tA7bfHcmvlRGcNmUIysrCxs2bGj37Vm/fv3g6uqKrVu3MppoNZORkcGgQYMYH4dJlGjx3Llz51pda9kOl0l8WvUipCPtTTzdunVDv379KNEihCM9e/bEnDlzMGfOHBQXF+PYsWM4efIkTpw4AYFAgPDwcDg7O0NDQ4PrUFnBh3lVQ0Ojw65tAoEAubm50NTUZCwGAFi4cCG+/fZbFBYW4saNG6irq8OiRYsAvGp4cPLkSRw4cIC1rQl8qYzIyMho1ZTjk08+gZycHNasWcNoovX06dM3JrY6OjqslXO+CyjR6iK4aIfLl1UvQt7k9Ta0jY2NuH//Pry9vTF37lyOoiKEtKSlpQV3d3esWbMG58+fR2JiIpKSkpCQkABjY2OEhoZyHSLj+DCvCoVCBAYGwtTUFEpKSq3uP3nyBIGBgeKkhymWlpaoq6vDkSNHICUlhR07dmD06NEAgH379iEuLg7Ozs6wsrJiNI5mfKmM4PK4kvr6+jfufZKXl6djU/4BSrR4jst2uHxZ9SLkn5KWloaWlhY8PDzg6uqKGTNmcB0SIeT/k5aWxqeffopPP/0UpaWlOH78OBISErgOixV8mFcdHByQlpYGa2triEQiGBgYiLu6Xbt2DWFhYdDQ0BB3A2TS7NmzMXv27FbXXVxcsHz5cigrKzMeQzO+VEbwqSkH+fco0eKhN7XDdXJyYqULIV9WvQj5X0lJSeHx48dch0EIaYeKigqcnJzg5OTEdSis4MO8Kicnh8jISPj7+2Pr1q0Sb8/k5ORgY2MDNzc3SEtLMxbDm7R1fhLT+FIZwXVTjrCwsA4bkLx48aLTx3yX0TlaPDRhwgRxO1wzMzOJdri6uro4fvw4K+eO1NXVYf78+Xj27FmHq16RkZGc/kAmpK2Sj6qqKsTFxaF3796IiopiPyhCCHkNn+bVmpoaNDY2Ij8/H2VlZVBRUYGysjI0NTW7TOtsNhQUFMDV1RXp6emsjBccHPzWz3b2vrIpU6a89bNt9RAgrdEbLR7iSzvcrrDqRQjQdsmHjIwM9PX14e3tzX5AhBDSBr7Mq8nJyfDz80NISIhEpz2RSITbt2/D19cXZmZmjMbQVbBdGcFlUw5KnjofvdHioaqqKnE73J9++kmiHe6yZcuQlJTEyhutZrTqRQghhHQeLufVnJwcODk5wdraGqtWrYKqqqr4XnFxMUJDQ3HixAlERUXBwMCA0Vj4hC+VEXxpytGWiooKJCQkIC4uDmlpaZzF0ZVQosVzLdvhPn36FAKBALa2tqy1w2256jVy5EjxdVr1InxTU1ODyspKKCoqvveHohJC+IvreVUkEkFLSwuenp7tPuPh4YGnT58iJCSEsTj4puVByc1aVkZoaWlxFgfwf005Tp8+zUocLeXm5iImJgZnzpxBbW0tRowY8d40sPm3KNHqIhobG8XtcM+fP4+mpibG2+HSqhfhu6qqKhw4cAApKSkoKSkRX9fQ0IClpSWcnJwo6SKE8AYf5lUjIyNERER0eF7SrVu3sGTJEmRnZzMSA3l7rzflYKuLblVVFZKSkhAbG4t79+4BeNVDwNnZGePGjWMlhncBJVpdUMt2uEweWkyrXoTPysrKYGdnh4cPH8LMzAxDhw5Fr169UFlZiTt37uDs2bMYOHAgDh8+3OaZJIQQwjY+zKsGBgZISkqCurp6u8+UlJTAysoKeXl5jMTAV3yujGCrKUd+fj5iY2ORmpqKmpoaaGtrQygUIigoiLVmbO8SaobRBbHVDregoADu7u4dPrNgwQIsWbKE0TgIaUtAQACampqQkpKC/v37t7r/119/wdnZGWFhYXB1deUgQkIIkcSHeXXQoEG4ceNGh4lWR2d9vWu6SmUEG005bGxsUFhYCC0tLTg6OsLCwgLa2toAgKCgIEbHfldRokXaVVtb+8ZNuUpKSqipqWEpIkL+T1ZWFjZs2NBmkgUA/fr1g6urK7Zu3UqJFiGEF/gwr1paWiIgIACGhoZtnlf16NEjBAQEwNbWlrEY+OL1yoi5c+dKVEbs378faWlprFZGdNSUY/To0YyOXVBQgMGDB8Pa2homJibiJIv87yjRIu2iVS/CZ0+fPu1wjwHwalPxn3/+yVJEhBDSMT7Mq3Z2djh9+jRmzJgBW1tb6Ovri8/yysvLQ2JiIjQ1NSESiRiLgS/4WBnB5XElp0+fRkJCAqKiorB9+3aoq6vDwsICFhYWjI77LqM9WqRdERERiIiIwJEjR9pd9Zo/fz5sbW3x1VdfcRAheZ/p6OggOzsbffr0afeZp0+fwtTUFIWFhSxGRgghbePLvFpXV4edO3ciPj4eFRUV4ut9+/aFra0tli5d+l4c3zJ58mRs2LABkydPbveZjIwMbN26lZNuf1xpamrCxYsXER8fj8zMTDQ0NAAAli5dCicnJ9r3/A9QokXa1djYCHt7exQVFXW46nXo0KH34gcy4RdKtAghXQ3f5tWGhgaUlJSgoqICKioqGDhwIAQCAePj8sWoUaNw6tSpDt8g/v7775g+fTpu3brFWlx8aspRXl6OEydOIDExEYWFhejevTssLS2xceNGTuPqKijRIh2iVS/CVzo6OhCJRB1OQi9evMDBgwcp0SKE8AbNq/zBpwW7rtCUo6CgAAkJCUhOTsaVK1c4jaWroESLvJX3fdWL8M+UKVPe+tlz584xGAkhhPxzNK9yjy+JFl+PK2lsbERZWRkUFRXRrVs38fWHDx9CWVmZFgTeAjXDIG9FRkYGgwYN4joMQsQoeSKEdGU0r/JDWFjYGysjmMa3phypqamIjIxEfn4+mt/HDB06FPb29pg9ezb8/Pygp6cHZ2dnxmPp6uiNFiHknVRRUYGEhATExcUhLS2N63AIIYTwDF8qI/jUlMPX1xeHDh2CkZERjIyMoKysjPLycuTk5CA7OxtjxoxBcXExTp06hd69ezMay7uA3mgRQt4pubm5iImJwZkzZ1BbW4sRI0ZwHRIhhBAe4ktlBF+OK0lPT0dsbCz27duHSZMmSdxzdnZGRkYGli1bBhcXF0qy3hIlWoSQLq+qqgpJSUmIjY3FvXv3AAATJkyAs7Mzxo0bx3F0hBBCuiK2KiPq6+vfuN9JXl5e3GadKdHR0XBxcWmVZDW7efMmevbsidzcXEbjeJdIcR0AIYT8r/Lz87Fu3TqYmprC19cXAODq6gopKSm4u7tTkkUIIeQfy83NxTfffIOJEyfC39+f825/bLl79y6mTZvW7v3MzEx4e3ujqKiIxai6NnqjRQjpkmxsbFBYWAgtLS04OjrCwsIC2traAICgoCCOoyOEENKVcF0ZwYemHHV1dVBQUGj3fnJyMv744w/U19czHsu7ghItQkiXVFBQgMGDB8Pa2homJibiJIsQQgh5W/n5+YiNjUVqaipqamqgra0NV1dXBAUFwd3dHUOGDGE8hgEDBrxVaWJbHQk7k4aGBvLy8to9wFkgECA3NxeampqMxvEuoUSLENIlnT59GgkJCYiKisL27duhrq4OCwsLWFhYcB0aIYSQLoAvlRF8acohFAoRGBgIU1NTKCkptbr/5MkTBAYGYtGiRewH10VRe3dCSJfW1NSEixcvIj4+HpmZmeLNwkuXLoWTkxOrhzsSQgjpOnR0dDB48GDY2NjAxMQEw4cPF9/T1dXF8ePHWXmj9SZsNeWoq6vD/Pnz8ezZM4hEIhgYGKBXr14oLy/HtWvXEBYWBg0NDURGRkJaWpqxON4llGgRQt4Z5eXlOHHiBBITE1FYWIju3bvD0tISGzdu5Do0QgghPPPgwQMkJCQgKSkJjx8/lqiMsLa25jzRauu4koSEBEbHrK6uhr+/P5KSkiT2YsnJycHGxgZubm7vTXOQzkCJFiHknVRQUICEhAQkJyfjypUrXIdDCCGEp/hUGcF1Uw4AqKmpQWNjI/Lz81FWVgYVFRUoKytDU1PzjW3oiSRKtAgh74TGxkaUlZVBUVER3bp1E19/+PAhlJWVaXIghBDyRlxVRrTVlEMoFCIoKIjVN2vJycnw8/NDSEgIRo4cKb4uEolw+/Zt+Pr6wszMjJVY3gWUaBFCurTU1FRERkYiPz8fzT/Ohg4dCnt7e8yePRvLli2Dnp4enJ2dOY6UEEJIV8JWZUTLphxmZmYSTTnY3CuWk5MDJycnWFtbY9WqVVBVVRXfKy4uRmhoKE6cOIGoqCgYGBgwHs+7gBItQkiX5evri0OHDsHIyAhGRkZQVlZGeXk5cnJykJ2djTFjxqC4uBinTp1C7969uQ6XEEIIj3FVGcGXphwikQhaWlrw9PRs9xkPDw88ffoUISEhjMfzLqD27oSQLik9PR2xsbHYt28fJk2aJHHP2dkZGRkZWLZsGVxcXCjJIoQQ0q43VUb4+fkxWhnBl+NKCgoK4O7u3uEzCxYswJIlS1iKqOujN1qEkC7J0dERH3/8MZYtW9bm/W3btiEmJgY6Ojo4dOgQy9ERQgjpCvhUGcF1Uw4DAwMkJSVBXV293WdKSkpgZWWFvLw8RmN5V9AbLUJIl3T37l2sW7eu3fuZmZnw9vaGj48Pi1ERQgjpKvhWGSElJYVJkyZh0qRJEk05du/ejYMHDzLelGPQoEG4ceNGh4lWXl4e1NTUGIvhXSPFdQCEEPK/qKurg4KCQrv3k5OT8dFHH0mcA0IIIYQ0i46OhouLS6skq9nNmzfRs2dP5ObmshwZoKSkBAcHByQmJiIhIQG2trY4ffo0o2NaWloiICAAjx49avP+o0ePEBAQAHNzc0bjeJfQGy1CSJekoaHR4cqaQCBAbm4uNDU12Q2MEEJIl8DXyojXm3KMGDECI0aMgEgkwt9//81YUw47OzucPn0aM2bMgK2tLfT19dGrVy+Ul5cjLy8PiYmJ0NTUhEgkYmT8dxElWoSQLkkoFCIwMBCmpqZQUlJqdf/JkycIDAzEokWL2A+OEEII771NZcQff/zBWmUE1005pKWlER4ejp07dyI+Ph7h4eHie3379sXChQuxdOlSOpfyH6BmGISQLqmurg7z58/Hs2fPIBKJYGBgIF55u3btGsLCwqChoYHIyEhIS0tzHS4hhBCemTVrFj7//HPMnDmz3WeOHz+OiIgIJCQkMBoLn5pyAEBDQwNKSkpQUVEBFRUVDBw4EAKBgPFx3zWUaBFCuqzq6mr4+/sjKSlJYsVRTk4ONjY2cHNzQ/fu3TmMkBBCCF+FhoYiNjYWR48ebbcyYt68eVi0aBEcHBwYiyM9PR2rV69GcHBwm/vFWjblWLVqFWNxkM5HiRYhpEurqalBY2Mj8vPzUVZWBhUVFSgrK0NTU5PKGwghhLSLL5URdFzJu4sSLUJIl5WcnAw/Pz+EhIRg5MiR4usikQi3b9+Gr68vzMzMOIyQEEIIn/GhMsLQ0BBRUVHQ1tZu875QKMTSpUvh4+ODnJwcRmMhnYsSLUJIl5STkwMnJydYW1tj1apVUFVVFd8rLi5GaGgoTpw4gaioKBgYGHAYKSGEED7jujLCwMAAJ0+ebLeL7suXL/HHH3/A0tKSDgruYugcLUJIl7R//37Y2dlh06ZNEkkWAGhpaWHz5s2wtLTEnj17OIqQEEII3yUnJ2PKlCm4f/8+jI2NIRQKYWRkhK1bt2LSpElIT09nPIbm40raQ8eVdF2UaBFCuqSCggLMnj27w2cWLFiAgoICliIihBDSleTk5MDNzQ2TJ0/GBx98IHHP09MTU6ZMwcqVKxl/i9R8XEl5eXmb95uPK7G2tmY0DtL5qHSQENIlGRgYICkpCerq6u0+U1JSAisrKyq1IIQQ0opIJIKWlhY8PT3bfcbDwwNPnz5FSEgIY3HwpSkH6Xx0YDEhpEsaNGgQbty40WGilZeX127NOyGEkPdbQUEB3N3dO3xmwYIFWLJkCaNxyMnJITIyEv7+/ti6dWu7TTkoyep6KNEihHRJlpaWCAgIgKGhYauSDwB49OgRAgICYGtry0F0hBBC+K62tvaNzS6UlJRQU1PDeCw9evSAh4cH3Nzc6LiSdwglWoSQLsnOzg6nT5/GjBkzYGtrC319fXGpRV5eHhITE6GpqQmRSMR1qIQQQniIT5URLY8rMTY2Fl+n40q6NtqjRQjpsurq6rBz507Ex8ejoqJCfL1v376wtbXF0qVLaRWQEEJImyIiIhAREYEjR460Wxkxf/582Nra4quvvmIsDjqu5N1FiRYhpMtraGhASUkJKioqoKKigoEDB0IgEHAdFiGEEB5rbGyEvb09ioqKOqyMOHToEKOLdnxpykE6HyVahBBCCCHkvcSHyggjIyNERERg6NCh7T5z69YtLFmyBNnZ2YzGQjoX7dEihBBCCCHvJTk5Obi5uWH16tWcVUbwqSkH6VyUaBFCCCGEkPeajIwMBg0axMnYfGrKQTqXFNcBEEIIIYQQ8r5qPq7k0aNHbd5vPq7E3Nyc5cjIv0V7tAghhBBCCOEIX5pykM5HiRYhhBBCCCEc4kNTDtL5KNEihBBCCCGEB+i4kncLJVqEEEIIIYQQ0smoGQYhhBBCCCGEdDJKtAghhBBCCCGkk1GiRQghhBBCCCGdjBItQgghhBBCCOlklGgRQgjpNGvXrsWwYcM6/Mfe3v5fj/P7779j2LBhSEhI6ISo329r167FlClTuA6DEELeOTJcB0AIIeTd8eWXX2LevHnir3fv3o2CggIEBweLr/Xs2ZOL0AghhBBWUaJFCCGk06irq0NdXV38tYqKCuTk5PDRRx9xFxQhhBDCASodJIQQwrqjR4/CxsYGH330EUaPHg0rKyukpaWJ7zc1NWHHjh2YMmUKRo4ciSlTpmD79u2or69v8/u9fPkSHh4eGD16NC5dugQA+O2337BkyRKMHz8eenp6mDt3LrKysjqMa8qUKdi0aRMWLVqE0aNHY926dQCA8vJybNiwAcbGxhg1ahTmzJmDH3/8UeLXDhs2DEeOHMHatWsxZswYjBs3Dr6+vvj777/h7+8PQ0NDjB8/HuvWrUNtba3419XW1mLXrl0wNzfHqFGj8J///Af79+9HU1MTAGDv3r0YOXIkKioqJMYLDw+Hrq4unj17BgD4888/sXr1aowbNw56enpYtGgRCgoKJH5NRUUFPDw8MG7cOIwdOxZbt24Vj0MIIaRzUaJFCCGEVdHR0diwYQOmTp2Kffv2Ydu2bZCTk8OaNWvw119/AQBCQkJw5MgRfPXVVwgLC8P8+fNx4MAB7Nmzp83v6evri+TkZAQHB2PChAloamqCi4sLampqsGXLFuzevRtKSkpYunQpHjx48Mb4Ro0ahd27d2P27Nmora3FokWLcPbsWaxatQrBwcHo168fFi9e3CrZ2rp1K+Tk5BAcHAxra2tERUXB2toaDx8+xLZt22Bvb49jx44hKioKwKsEccmSJQgNDcVnn32GvXv3wtzcHDt37sS3334LAJg5cyYaGhpw5swZibFSUlIwYcIE9OnTB6WlpZg3bx7u3LmD9evXY/v27WhqasLChQtRXFwM4FXyunjxYmRlZcHd3R3ff/898vLykJqa+s//EAkhhLwRlQ4SQghhVUlJCUQiEb788kvxNTU1NdjY2CA3NxdCoRBXr17FyJEjYWtrCwAYN24cunfvDkVFxVbfb/v27YiNjUVwcDAmTpwIAHj27Bn++9//4ssvv8SkSZMAAKNHj0ZwcDDq6uo6jG/AgAFYs2aN+Ou4uDjcvXsXcXFx0NPTAwBMnDgR9vb22LZtG+Lj48XPDhkyBN9995045qNHj6K+vh7btm2DjIwMJkyYgNOnTyMvLw8AcOHCBVy+fBk//PADhEIhAMDExATy8vIICAiAg4MDtLW1MXbsWCQnJ+Ozzz4D8OptXX5+Pnbs2AEAiIiIQHl5OY4cOQI1NTVxjBYWFggICEBgYCAuXLiA/Px8hISEiH+fjIyMqBEGIYQwhN5oEUIIYdXatWuxZs0aPH/+HDdv3sTx48cRHR0NAOIkaPz48cjOzsaCBQsQGhqKe/fuwc7ODlZWVhLfKzo6Gvv374dQKMQnn3wivt63b18MGTIE69evh7u7O06ePImmpiZ4eHhAW1u7w/iGDx8u8fWPP/4IVVVV6OrqoqGhAQ0NDWhsbMTkyZNx+/ZtiZI+fX198b9LS0tDWVkZurq6kJH5v3VNJSUlVFZWAgCuXr0KGRkZmJubS4xpaWkpvt/89bVr1/DkyRMAr95m9ezZU5wk/fjjjxg+fDg++OADcYxSUlKYOHEiLl++DAC4fv06ZGVlYWpqKh5HQUFBnIgSQgjpXPRGixBCCKt+++03bNiwAT/++CNkZWUxePBg6OjoAHhVSgcAixcvRo8ePRAfH49t27Zh69at0NbWhpeXFwwNDcXf6+7du5gwYQKSk5OxaNEijBgxAgAgEAgQFhaGPXv2ID09HUlJSZCVlcXUqVOxceNG9O7du934FBQUJL4uLy/HkydPoKur2+bzT548EX+/tjoqvv79WqqoqICysjKkpaUlrquqqgKAOCEzNzeHj48P0tLS4ODggJSUFEybNg3y8vLiGB88eNBujDU1NaioqICSkhIEAkGbYxFCCOlclGgRQghhTVNTE7744gvIysri2LFjGD58OGRkZHDv3j0cP35c/JyUlBQWLlyIhQsX4tmzZ8jKysLevXuxfPlyZGdni59zdXWFg4MDhEIhvLy8cPToUXHS8sEHH8Db2xvffvst7t69i1OnTiEkJATKysri/U9vQ1FREZqamti2bVub9z/88MP/8XcD6N27N8rKytDY2CiRbD1+/BgAoKysLI5hypQpSEtLg6GhIYqKirB+/XqJGMeNGwc3N7c2x5GTk4OysnKbY5WXl//P8RNCCGkflQ4SQghhTVlZGX799VfMnj0bo0aNEpfUXbhwAQDEHfDmzZsHX19fAECfPn1gY2ODhQsX4vnz56iqqhJ/v759+0JeXh4bNmzAnTt3cPDgQQDAjRs3YGxsjPz8fAgEAgwfPhyrVq3C0KFD8eeff/6jmMeNG4eHDx+iT58+GDVqlPif7OxshIaGtnob9U+/d0NDA06dOiVx/cSJEwCAMWPGiK9ZWVnh5s2bOHLkCAYMGIBx48ZJfJ9ff/0VgwYNkojx+PHjOHbsGKSlpWFkZISGhgZkZGSIf11dXZ1E4koIIaTz0BstQgghrOnTpw/U1NQQHR2Nfv36oVevXrh48SIiIyMBvCpxA4CxY8ciLCwMffv2hb6+Ph49eoSDBw9i3LhxUFFRwYsXLyS+76RJk2Bubo6goCBMmzYNI0aMgLy8PNzc3LB8+XL07dsXly9fRmFhIRwcHP5RzDY2Njh06BCcnJywZMkS9O/fH5cvX0ZISAjs7OwgKyv7P/9+TJw4EePHj4eXlxcePXoEHR0dXL16FSEhIZg1axaGDBkiftbU1BRKSkqIjY3F4sWLJUoAHR0dcfz4cTg6OuLzzz+HsrIyUlNTERcXBw8PDwCvGl9MmDABXl5eePbsGdTU1BAZGYnS0lL06dPnf/5vIIQQ0jZKtAghhLBq9+7d8PPzw9q1ayEnJ4chQ4Zgz5492LRpE65fvw57e3u4urpCTk4O8fHx2LVrl7h07uuvv273+3p6euLSpUtYv349wsPDERYWhu3bt8PPzw/Pnz+HpqYmvvvuO9jY2PyjeBUUFBAdHY3t27dj69atqKyshJqaGr7++mt8/vnn/+r3QiAQYN++fQgMDER4eDhKS0vx4YcfYvXq1XBycpJ4VkZGBkKhEFFRUeJmGc0++OADxMTEYPv27fD29kZtbS00NTXh5+eH2bNni58LDg7Gtm3bEBgYiNraWlhYWGDOnDk4e/bsv/rvIIQQ0prgZfPOY0IIIYQQQgghnYL2aBFCCCGEEEJIJ6NEixBCCI+7WdEAAABySURBVCGEEEI6GSVahBBCCCGEENLJKNEihBBCCCGEkE5GiRYhhBBCCCGEdDJKtAghhBBCCCGkk1GiRQghhBBCCCGdjBItQgghhBBCCOlklGgRQgghhBBCSCejRIsQQgghhBBCOhklWoQQQgghhBDSyf4fZRFotbDYQfQAAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -1212,8 +1416,8 @@ "plt.ylabel(\"Correlation (predicted vs. observed performance)\")\n", "plt.legend()\n", "\n", - "# add vline for 0.9 spearman\n", - "plt.axhline(y=0.9, color=\"r\", linestyle=\"--\")\n", + "# add vline for 0.95 spearman\n", + "plt.axhline(y=0.95, color=\"r\", linestyle=\"--\")\n", "\n", "# add task names to the x-axis\n", "plt.xticks(range(len(tasks_removed)), tasks_removed, rotation=90)\n", @@ -1222,12 +1426,12 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 19, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2MAAAL9CAYAAACmKr6OAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3hTZfsH8O9JmjTpTncZZUPZQ4oiIgiKCxX3eHGDvIoyVByvyM/xKgiIAgoKgijoq+JGEVFRVFT2EijIKKu7Tbqyk/P7Iz1pS1dOmtHx/VwXlzY5uXM3TU/Pned57kcQRVGEDOeddx7mzZuHUaNG1bjvl19+wfTp07F79245IYmIiIiIiFodhdwH9O/fH++++y5sNlu12+12O95//33069fPZ8kRERERERG1VILckbE9e/bgzjvvhE6nw8UXX4y4uDgUFRVhy5YtKCwsxOrVq1mQERERERERNUB2MQYABw8exFtvvYVdu3bBYDAgMjISgwcPxkMPPYSePXv6I08iIiIiIqIWxatijIiIiIiIiBonxJsHiaKIQ4cOwWg0orZaLj09vdGJERERERERtWSyi7F9+/Zh6tSpyMnJAQB3MSYIAkRRhCAIOHTokG+zJCIiIiIiamFkF2OzZ89GSEgIZs+ejeTkZCgUshsyEhERERERtXqyi7EDBw5gwYIFuPTSS/2RDxERERERUasguxiLi4uDUqn0Ry7N1u7duyGKIlQqVbBTISIiIiKiILLZbBAEAQMHDmzwWNlzDO+44w68/fbbMBqNXiXXEomiWGsjk2AQRRFWq9Vv+fgzPnMPTnzmHpz4zD048Zl7cOIz9+DEZ+7Bic/cAx87EPHl5uJpHrJHxk6ePIljx45h2LBh6NatGzQaTbX7BUHAe++9JzdssyaNiPXt2zfImQBGoxGHDh1C165dERYW1qziM/fgxGfuwYnP3IMTn7kHJz5zD0585h6c+Mw98LEDEV+O/fv3e3ysV8VYWlqa++tzq76mUI0SERERERE1dbKLsdWrV/sjDyIiIiIiolbFp33pjUYjfv31V1+GJCIiIiIiapFkj4ydPXsWzz33HLZt2war1VrrMdz0mYiIiIiIqH5ebfq8a9cu3Hzzzdi1axe0Wi0GDBiALVu24MiRI1i8eLE/8iQiIiIiImpRZE9T3L59O6ZPn46ZM2fihhtuQGhoKGbMmIHPPvsM6enp+Omnn/yRJxERERERUYsiuxgrLy9Hjx49AACdO3fGwYMHAQBKpRJ33HEH/vrrL99mSERERERE1ALJLsYSExNRUFAAAOjQoQOKi4uRn58PAIiJiUFhYaFvMyQiIiIiImqBZBdjI0aMwOuvv47du3ejbdu2SE5OxsqVK1FWVobPPvsMSUlJ/siTiIiIiIioRZFdjE2ZMgVRUVFYuHAhAGD69Ol47733kJ6ejnXr1uHee+/1eZJEREREREQtjexuijqdDmvXrkVeXh4A4Nprr0WbNm2wZ88e9OvXD0OGDPF5kkRERERERC2N15s+JyYmuv9/8ODBmDBhgteFmNPpxKJFizB8+HAMGDAAEydOxOnTp+s8/p9//sEDDzyA888/H0OHDsWUKVOQlZVV7ZgPPvgAo0ePRr9+/XDHHXe4G41Izpw5g0mTJmHQoEG46KKL8Prrr8PhcHiVPxERERERkVyyR8asVivWrFmDXbt2oaSkpMb9giDgvffekxVzyZIl+PDDDzFnzhwkJydj3rx5mDBhAtatWwe1Wl3tWL1ej3vvvReDBg3C6tWrYbVaMWfOHEyYMAFffPEFQkND8cUXX2Du3Ll48cUX0atXLyxbtgz33nsvvvvuO8TGxsJms+H+++9Hx44d8dFHH+HUqVN45plnoFAoMGXKFLkvCRERERERkWyyR8ZeeOEFzJ07FydPnoQoijX+OZ1OWfGsVitWrlyJKVOmYOTIkUhLS8Nrr72GnJwcbNy4scbxP/74I4xGI+bOnYvu3bujT58+mDdvHo4dO4Zdu3YBAN566y2MHz8e1157Lbp27YqXX34ZWq0Wa9euBQB8//33yMrKcse49NJL8eijj+K9996D1WqV+5IQERERERHJJntk7IcffsAjjzyCyZMn+ySBjIwMlJeXY+jQoe7boqKi0KtXL2zfvh1jx46tdvzQoUOxZMkSaDQa920KhaumLCkpQWFhITIzM6vFCwkJweDBg7F9+3ZMmjQJO3bsQO/evREdHe0+5oILLkBZWRkOHTqE/v37++R7IyIiIiIiqovsYkyhUGDgwIE+SyAnJwcAkJKSUu32xMRE931VtWvXDu3atat227Jly6DRaJCeno7s7Ow642VkZLifMzk5ucb9AJCdne1dMSaKcJjNtd4lKBRQVJluWddxroMFKENDvTvWYoHDbIZotcJhNsOhUNR7LETR47jSsbXFV1YpjOuNe86xTqsVYpWR1HNj13fsuRShoRAEwXWszQaxlvV/UnyxSn51HeuOq1ZDqPg+6zvWYTbLi6tSQVAqPT5WItrt9b4nqsW12yHa7R4dKzqdNd8zVQghIVCEuE4XosMBp81WZ9xzj63zPQlAUCrd31+Dcase63TCWRGztti1HetRXFGE02Jx33du/PqOrRG3gd/7qrGdISE+P0c4zGaI57ye3v7en+vc71vO773cc0Rd78najvX0HCH9HtUV25PzifvYc84R9b3fPT2fALWfI+qK7e3v/bnHnhu/2jlC5u/9ucdWe7+r1Z6fTzw8RzjM5mqvp6zf+waOrfqcDf7eN/IcUec5uBHXEXW9H2s7Vu45wifXHKj9HFFXbDm/93KuOWT93jdwbNXYokbj+e+9nGuOKt+LrPNJA+eIqnFkXUd4cI6o+hz1vYdlXXOc87tc3/vd62sDb64jRBGoeI80RHYxNm7cOHz66ae44IIL3CNSjWEymQCgxtqw0NBQFBcXN/j41atXY82aNZg5cyZiY2Nx/PjxOuNZKk6KZrMZUVFRNe4H4D5GLnNeHv569vla74sa0B9dnnzc/fXee+6H01L7DzWiZxq6zXrG/fX+Bx6CvbS01mPDOndCj5decH994JHpsFZsyL3vnGM1bdui5/w57q8PPf4UzGfP1hpXHR+P3otfc399+JlZMB4/Ue0YKX5IZCT6Llvivv2fF15C2aGMWuMqQtXov2qF++tjr8xHyZ69NY6TYg/832r3bSdeXwTD1u21xgWAfu8ud590Ty59G0W//l7nsWWvvwohyfULcnrlKhT88FOdx/ZatAChCQkAgLMf/A9536yv81j1vye638/Zn36OnM++qPPY7v99HuFdOgMActd9i6wPP6rz2K7P/gchnTq64m7YiNwP/lfnsZ1nPIboQQMAAIWbf8Wpt5bXeWzHqQ9Dd8H5MJlMcGYcxr7/zqnz2NR/T0TciIsBAMW79uD4vFfrPLbdvXchYcxlAIDSg4dw9MWXAdR8TwJAmztuQ9I1VwMAyo8dx5GZ/1dn3OQbr0fKTTcAAEynzyDjiafd950bO3HsVWj7r9sBAJb8fByc8midceMvG432990DALCVlODvSTVH/aX4sRdfhA4PTgLg+mO4796JdcaNOT8dnaZVrkHdffudtR63D/47RwhtUmD6b+V5qeo54lxyzhEhcXEImTzJ/X6v7RzhPtaLc4QU9+irr6F839+1Hgt4d44wmUywffsd9u2dW+exfd5+E6qKvxHeniNqe7+nzZ0NbXvXh4mNOUecG7vrs/9BZK+eAID8jT/gzLvv1xnXk3OEFF86RwCA/q+tyFz4Rp1xPT1H7EPd54jayDlHKC++CKYuXQDUPEecS845IuaSkcDwC13vnTrOERJvzhHS+72+Y709R5hMJlgWvYl9RlOtx9Z3HXGu+s4R574nPbmOkDR0jqga29PrCIkn5wgpvpzrCE/PEfsg7zpCzjlCff89MHX2/DrC03NEu6kPA9FRMJlMHl9HAJ6dI8KGpAMACrbvwJl6jvX2HKHPOAzLnPm1nn+B+q8jztXY6wj1Iw9C4+Hey7KLsWnTpmHcuHG4/PLL0bt3b2i12mr3C4KAl1+u+0U7lzTd0Gq1Vpt6aLFYasSuShRFLFy4EEuXLsWDDz6IO++8s0a8qqrG02g0td4PAGFhYR7n7ilp+qPE6az706Fyo7HasXZH3Z9ImMzmasda6/nkwGK1VDvWYq276LTabNWPrefTC7vDXv1Yo7HOY51OsXq+ZWV1Hgug+rEltV9sSg4fPgyhogC3NlDEnz59CkJRIQDAptfXe+zRo0ehqPjDZCssrPdYAMjMzHQdm59f/3EnTkBR8TOwV2wTUZeTJ09CWfHhSmFR/TmcPn0aWVrXBwv2rOx6jz179ixyqrzG9cnKykZexbGOejqdAq6R5wLp2JMn6z02Ly8PRRXHOs9m1Xtsfn4+DNKxefW/voWFhSiRjjUY6j1Wr9ejrOJYsby83mMNxcUwSsc2sL60pKS02nu4Pv46RwCV70nAd+cIu92OkCqx/XWOMNVxASnx9hzRkH+OHIEQHg7At+eI48ePQ1HmytNf5wh7LTNKqvL2HOGoozCXNIVzBFD5nvTlOaK0tASqitjN8RxRH15HVJJzHdEUzhFA5fvdl+eI3Nw8KKOjkJmZ6fNzREhFvrm59efr9Tkiq+lfR9RGEMV6xo1r8dJLL2H16tXQarXQ6XQ1AwoCfvqp7k8Rz7Vv3z7cfPPN+OGHH5Camuq+/fbbb0ePHj3w3HPP1XiMzWbD008/jW+++QZPPfUU7rnnHvd9hYWFuPDCC7Fq1apq68Yef/xxFBcXY/ny5Xjuuedw5MgRfPjhh+77T548iTFjxmDt2rXo16+fx/kDwP79+yE6nejUvn2t98uZXtCYY50WC0wmE06ePIkOHTpUK24FQYCiypQBp8WCun70NY6tMrRvNptrxPfVlIFzY/t6mqIUv2O3bu6iW9aUgXqG681mM05mZaFTp07QarUND+17GFc61myxIDMzEx3atUNolWmLNY6tOmXAbofTg+kFJpMJJ44fR4e2bau9Z6odGxLinmLQ0JSBasc6nTCWlNT6ngTOmYrQ0DSAWo6t7f0oO24904rOje/LaYpVY2vDwnx+jjCbzTh56hQ6devm/iBK1u99PceazWacys5Gx44dXe/3Rvze13asyWRCZmYmUtu2haaeAsqbc4TJZMKJo0fRoX37ut/v3k5TtNthKiur8/0u9/f+3GPrer9XPdbT3/vajj03vpwpSA2dI6rGDouIqHaOkPt7Xxuz2YyTZ86gU5cu0Gq1PosLAGarFafOnkXHjh2h0Wh8Pk3R/X5PSanzPenttYHJZMKJI0dqfT/WFlfuOcLjaw4vzhF1vd99cc2xaVcW3vs2AyP7RuH2K/q43u8+nKZY7f0eFdWo3/vamM1mnDx7Fp06d5Z9zdHQOcJit+Pk6dOu97tK5fn5xINzhNlmc13LtG+P0JC6x4POvY7w9HfZWF6OzKNH636/N/Kao85ja/m9P3H6NASFAn379q3zcRLZI2Nff/017rnnHjzxxBM+maaYlpaGiIgIbN261V2MlZSU4ODBgxg/fnytj3niiSfwww8/4NVXX8XVV19d7b64uDh06tQJW7dudRdjdrsdO3bswB133AEASE9Px5dffomysjJEREQAAP766y+Eh4cjLS3Nq+9DUCgQGRvr2cFyRt9kHqsIDYWQnY3wmJj6R/m8zEFpNNYfvxHfW72xfTBiKcUPCwvz+QioFFur1fpldFU6iYZFRvotfoPvmUbE9ug96YUG34/eqPi006P4VY5tkL/e73UcW+t70kfnHqXRCFSN7adzWnhUlOc/VxlxhZAQv73fFSEhzev9HqD4gcrdH+dgpdEInD1bGVvO772MY2W9NnLe72q157Gb0DWHR+8ZL889Ww4dgE2hwkmDyPd7LbEB+CW2UBE7LCLCL68LIPP97i/h4RAaGCmsSnY15XA4cMkll/ikEANca7vGjx+P+fPn46effkJGRgamT5+O5ORkjBkzBg6HA/n5+TBXfAr0+eefY/369Zg+fTqGDBmC/Px89z/pmPvuuw/vvvsuvvjiCxw9ehT/+c9/YDabcdNNNwEALr30UiQkJGDatGnIyMjAjz/+iAULFuC+++6rsdaMiIiIiFqG4jILDp8scv1/ed2jX0SBIntk7LLLLsN3332H888/32dJTJkyBXa7HTNnzoTZbEZ6ejpWrFgBlUqFM2fOYPTo0Zg9ezZuuOEGfPPNNwCAuXPnYu7c6guwpWNuueUWlJaW4vXXX4fBYECfPn3w7rvvIrZi5Co0NBTvvPMOnn/+edxyyy2Ijo7GHXfcgYceeshn3xMRERERNS07M/IgLbkzlNvrnJZJFCiyi7H+/ftj/vz5yMjIwMCBAxF+zhC8IAiy9yBTKpWYMWMGZsyYUeO+du3a4fDhw+6vV65c6VHM+++/H/fff3+d93fo0MHjWERERETU/G07WNnAwmITUW62y5p5SuRrsosxqaHGnj17sGfPnhr3e1OMERERERH5k83uxK4MVyc/QXBtBVVgMCMxLjrImVFrJrsYO3jwoM/WixERERERBcKB4wUwWeyIiQxFbKQax7NKka+vfwsNIn+TXVVde+21+Pnnn/2RCxERERGRX2w/mAsASO+ZhASda8uPPAOLMQou2cVYdkUrTSIiIiKi5kAURfd6sfReyUiMcV3LFhjq3rONKBBkF2PXXHMNVq1ahbwGdvsmIiIiImoKTueWIqfQCFWIAgO7J7hHxvI5MkZBJnvNWGZmJnbs2IERI0YgppZN1QRBwI8//uizBImIiIiIGmNbxRTFfl3joQkNQUKMBgCQr+fIGAWX7GIsJSUF11xzjT9yISIiIiLyuW0HKqcoAkB8DEfGqGmQXYzNnj3bH3kQEREREflccZkFh08WAQDSeyUBgHtkrNxsR7nJhnCtKmj5UesmuxiT/Prrr9i2bRtKSkqg0+kwePBgDB8+3Je5ERERERE1ys6MPDhFoFObKCTqXMtrtKEh0IYqYLI4kac3opOWe41RcMguxqxWKx566CH8/vvvUCqV0Ol00Ov1WLZsGS644AK8/fbbUKvV/siViIiIiEgWqYvikIopipKYMKWrGCsyolMbFmMUHLK7KS5evBg7d+7E3LlzsW/fPvz+++/Yu3cvZs+ejT179mDp0qX+yJOIiIiISBab3YldGa4O4EN6n1OMRbjGJPK48TMFkexi7JtvvsHDDz+Ma6+9FkqlEgAQEhKCcePG4eGHH8a6det8niQRERERkVwHjhfAZLEjJjIUXdvFVLsvJtx1HZunNwYhMyIX2cVYUVERevXqVet9vXr1Qm5ubqOTIiIiIiJqLKmlfXrPJCgUQrX7YsJdI2O5RSzGKHhkF2OpqanYuXNnrfdt374dKSkpjU6KiIiIiKgxRFGs0dK+qmiOjFETILuBx2233YY5c+ZAo9Hg6quvRnx8PAoKCvDNN99g+fLlePjhh/2RJxERERGRx07nliK3yAhViAIDuyfUuN89TbGIa8YoeGQXY7fffjsOHjyI+fPn49VXX3XfLooirr/+ejzwwAM+TZCIiIiISC5pimK/rvHQhNa85JWmKZYarTCabQjTcK8xCjyPirGvvvoKF198MXQ6HRQKBV566SXcd9992LZtG4qLixEdHY0hQ4agS5cu/s6XiIiIiKhB9U1RBACNWoFwTQjKzXbk603okMJijALPo2Lsueeew4oVK6DT6TB69Gi8+eabSEtLY/FFRERERE1OcZkFh08WAQDSeyXVeVxCjBblOaXI0xvRISUqUOkRuXlUjKnVanz11Vew2+04e/Ys9uzZg9LS0jqPT09P91mCRERERERy7MzIg1MEOrWJQqIurM7jEnQaZOaUIo8dFSlIPCrGbrrpJqxYsQKffPIJBEHA888/D8C1TkwiCAJEUYQgCDh06JB/siUiIiIiasC2g64pikPqmKIoSYjRAgByufEzBYlHxdiMGTMwbtw46PV63HXXXZg1axa6du3q79yIiIiIiGSx2Z3YlZEHoP4pigAQH6MBAI6MUdB43E2xW7duAIA2bdpg9OjRSEqq/81NRERERBRoB44XwGSxIyYyFN3a6+o9NrFiZIx7jVGwyN70uaioCPv27fNHLkREREREjSK1tE/vmQSFQqj32AQdizEKLtnFWEpKCsrKyvyRCxERERGR10RRbLClfVXSNMXiMivMFrtfcyOqjexNn2+99Va89NJL2L17N3r06IHw8PAax4wbN84XuREREREReex0bilyi4xQhSgwoHtCg8eHa0IQpgmB0WxHvsGE9kmRAciSqJLsYmzOnDkAgE8++aTW+wVBYDFGRERERAEnTVHs2zUe2tCGL3MFQUCiLgyZ2SXILTKyGKOAk12M/fTTT/7Ig4iIiIioUaQpig21tK8qKdZVjHHdGAWD7GKsbdu21b62WCxQq9UQhPoXSBIRERER+UtxmQWHTxYBaLilfVXuJh5sb09BILsYA4Djx49j0aJF+OOPP1BWVoa1a9fi008/RefOnXHnnXf6OkciIiIionrtzMiFUwQ6tYlCoi7M48clxbqOzePGzxQEsrspHjp0CDfddBMOHDiAa665BqIoAgCUSiVefvllfPHFFz5PkoiIiIioPtJ6MTlTFAG4CzeOjFEwyB4Ze+WVV9CnTx+sXLkSAPDBBx8AAGbOnAmLxYL3338f119/vW+zJCIiIiKqg83uxK6MPADypigClcVYLteMURDIHhnbs2cP7rnnHoSEhNRYJ3bVVVchMzPTV7kRERERETXowPECmCx2xESGolt7nazHJlZMUzSUWmCxOfyRHlGdZBdjoaGhMJvNtd5nMBigVqsbnRQRERERkaekKYrpPZOgUMhrKhcZpoI2VAkAyOfoGAWY7GJs2LBhWLRoEXJycty3CYKA8vJyrFy5EhdeeKFPEyQiIiIiqosoiu6W9uky14sBlXuNAUBeEZt4UGDJXjM2Y8YM3HrrrbjiiiuQlpYGQRAwZ84cnDhxAqIoYsGCBf7Ik4iIiIiohtO5pcgtMkIVosCA7glexUjQheFkTin3GqOAkz0ylpKSgq+++gp33303RFFEamoqjEYjxo4di88//xzt27f3R55ERERE1EQ4HE4cOFEEu0MMdiruKYp9u8ZDG+rVrk1V2tuzGKPA8uodq9PpMH36dF/nQkRERETNwC+7zuD1j3ajR1sN+vQObkEmTVGU29K+KndHRba3pwDzqhgrKCjA+++/j23btqG4uBhxcXEYOnQo7rzzTkRFRfk6RyIiIiJqQjKzSwAAh8+ase1gHi5J7xSUPIrLLDh8sgiA/Jb2VSXGagFwrzEKPNnTFDMyMnDVVVfhvffeg0ajQa9evaBUKvH222/jmmuuQVZWlj/yJCIiIqImwlBqcf//u99moNxkC0oeOzNy4RSBTm2i3KNb3nA38NC37gYexWVWlJnZ3j+QZI+MzZkzBykpKXjnnXeQkFC5SDI3NxcTJkzAK6+8goULF/o0SSIiIiJqOvSl5ir/b8X76w/iwRv7BzyPbQdc68UaM0URqFwzVlRihs3ugCpE2ejcmhu7w4kn3vwTNrsdA/v1CnY6rYbskbG9e/diypQp1QoxAEhKSsLDDz+MP/74w2fJEREREVHTU1TiGhm7qFckAOC7PzORkVkU0Bxsdid2Hc4D0LgpigAQFa5GqFraa6x1jo4VFpthKLOi3OxEmTE4I52tkexiTKfTobS0tNb7HA4HNBpNo5MiIiIioqbLUDEy1q9jGC4ekAJRBN5Yuwd2hzNgORw4XgCTxY6YyFB0a69rVCzXXmMV68ZaaUfFAkNlEVputgcxk9ZFdjE2efJkzJ8/H7t27ap2+/Hjx7Fw4UI8/PDDsuI5nU4sWrQIw4cPx4ABAzBx4kScPn3ao8dNmDABixcvrnZ7jx496vwnrWfbuXNnrfdv3bpVVu5ERERErY3N7kBpxchJhFaBO6/ojsgwNU7mlOKLX44GLA+ppX16zyQoFEKj41V2VGydI2P5VYuxIK0BbI1krxn78ssvYbFY8K9//Qvt2rVDUlIS9Ho9MjMz4XQ6sWzZMixbtgyA61OGH3/8sd54S5YswYcffog5c+YgOTkZ8+bNw4QJE7Bu3Tqo1epaH2O1WjFr1iz89ttv6N+/+vzk33//vdrXxcXFGD9+PEaMGIE2bdoAAA4fPozU1FR8+OGH1Y6Njo6W9VoQERERtTb6iuYdSqUArVqBqHA17r+2N17/aDc+2ngYF/Vvi5T4cL/mIIqiu6V9eiPXi0kSW/leY/lVvm+OjAWO7GKsXbt2aNeuXbXb2rdvj379+sl+cqvVipUrV+Lxxx/HyJEjAQCvvfYahg8fjo0bN2Ls2LE1HrNr1y7MmjULZrO51jb6565le/HFF6HT6fDiiy+6bzty5Ai6du1a41giIiIiqp/USTEmIhSC4BqRGjW4PTbtOI19Rwuw5LO9eOGBoe77/OFUbilyi4xQhSgwoLtvrufcHRVbaXv7qtMUjWaOjAWK7GJs9uzZPnvyjIwMlJeXY+jQoe7boqKi0KtXL2zfvr3WYmzz5s0YPnw4Jk+ejGuvvbbe+L///js2btyINWvWVBtlO3z4MM477zyffR9ERERErYW+xLVeLCai8tpKEARMvqk/Hp7/M/YcycfmXWcw8rz2fsthe8UUxb5d46EN9Wrb3BqSdK17ZKzAUNkhkyNjgeObd6+XcnJcw8spKSnVbk9MTHTfd67p06d7HH/BggUYPXo0Bg8eXO32f/75BzqdDjfccANyc3PRvXt3TJ8+3avRPSIiIqLWpMg9MlZ9OUmbhAjceml3rNmQgXe+/hvn9UxCZFjtS04aS5qi2NiW9lW19o2fqzXwMLEYC5SgFmMmk+uHfu7asNDQUBQXFzcq9vbt23HgwIFq0xMBIDs7G6WlpTAajZg5cyaUSiXWrFmD8ePH4/PPP0fXrl29ej5RFGE0Bv+XV3pNpf82p/jMPTjxmXtw4jP34MRn7sGJz9yDE99fsfMKXV21I7UhNeJfeX5b/LLzNM7kl2P5F3vx7+t7e/Uc9eVeUm5FxklXG/2+naK8uv6qLX6k1jWtsrDEjJKSMoSEyO5zV2dsX/JX/HxD5etYXGby+XVtc31dvCGKosfTdINajElt8K1Wa7WW+BaLBVqttlGxv/jiC/Tr1w+9e1c/CaSkpGD79u3QarVQqVQAgL59++LgwYNYvXo1nn/+ea+ez2az4dChQ43K2ZcyMzObbXzmHpz4zD048Zl7cOIz9+DEZ+7Bie/r2Jmn9QAA0V4OILpG/Mv6h+HdH8vx864sdNDZ0DEp1PvnqiX3PcfLIYpAUowK+dmZyM/2Ony1+KIoIkQJ2B3A1t0HEBvRuMvk5vSesdqd7g6ZAJCbb/DbdW1zel0ao65GhOcKajEmTU/My8tDamqq+/a8vDz06NHD67hOpxObNm3CQw89VOv95zb+UCgU6NKlC3Jzc71+TpVK5fWomi+ZTCZkZmaiY8eOjS5oAx2fuQcnPnMPTnzmHpz4zD048Zl7cOL7K/Y3u/YAKEeHtokALDXi9+wJnDIcxE87zmLj3nLMndwPKpmjTPXl/t3evQCAC/u3Q8+e3l171RU/UadHVoER0XFt0bNzrE9j+4o/4mfllwPIcn+tVGnRs2dPn8SWNMfXxVtHj3q+xUNQi7G0tDRERERg69at7mKspKQEBw8exPjx472Oe/ToUej1elx44YU17vv1118xdepUfP3112jf3rWw1G63IyMjA2PGjPH6OQVBQFhYmNeP9zWtVuvXfPwZn7kHJz5zD0585h6c+Mw9OPGZe3Di+zp2qdG1nigxNgKApdb4E67rh52HC5BVYMS3f57BHZenefVc58a22Z3Yd9Q1RXFY/3aN/r7OjZ8cF4GsAiOKyx0+j+1rvoxfZimr9rXF5mw278dAx/eEnE6iHhVjX375pawExo0b59FxarUa48ePx/z58xEbG4u2bdti3rx5SE5OxpgxY+BwOFBUVITIyMhq0xgbcvDgQahUKnTu3LnGfYMGDYJOp8OTTz6J//znP1CpVFi2bBkMBgPuuecej5+DiIiIqDXSl7q67kVHquEoq/2YiDA1HriuL+au2YG1P/2D4QPaon1SZKOf+8DxApgsdsREhqJbe12j450rQVfRxEMf/HVHgSQ17xAEQBTZTTGQPCrGnnrqqWpfS9WeKIo1bgM8L8YAYMqUKbDb7Zg5cybMZjPS09OxYsUKqFQqnDlzBqNHj8bs2bNxww03eBwzPz8f0dHRUChqDolHRERg1apVmD9/Pu6//35YLBacd955WLNmDeLj4z1+DiIiIqLWRhRFFJVU7jNWWEcxBgAXDWiDn3YkYmdGHt78dC9efnAYFIrG7T22raKlfXrPpEbHqk1SK934Ob+irX1KXBiyCowoN3GfsUDxqBj76aef3P9/6NAhzJgxAw899BCuvPJKJCYmQq/XY9OmTVi8eLHsfciUSiVmzJiBGTNm1LivXbt2OHz4cJ2P3bRpU623T5w4ERMnTqzzcampqVi0aJGsPImIiIhau3KTDXaHE4CrtX1hPccKgoAHb+yPyfM24cDxQvy4/RTGnN/B6+cWRdHd0j7dhy3tq5I2fs5tZe3tpZGx1KRIVzHGkbGA8Wg1Zdu2bd3/lixZgoceeggTJ05Eu3btoFarkZSUhNtvvx2TJk3CvHnz/J0zEREREQWBvmKPsXCtCmqVssHjk2LDcMcY13qxd9cdgKHi8d44lVuK3CIjQpQKDOie4HWc+rTWkTF3MZYcAQAwmu3VZsCR/8jeQOHYsWPo1atXrfd17twZZ86caXRSRERERNT0SOvFdJGet6u/7uLO6NwmGmUmG9756m+vn1saFevXLR7aUP/0oJPWjBUWm+GoGAFsDaQ9xlKTXMWYwynCYnUEM6VWQ3Yx1rFjR6xbt67W+z7++GN079690UkRERERUdMjrReLjfK8sZpSqcDkm/tDIQCbd5/Brow8r557e8V6sSF+mqIIALpIDUKUCjidIgqKzX57nqZEFEX3yFjb+HBIS/HKzVw3FgiyP1aYPHkypk6diszMTFxyySXQ6XQoKCjAxo0bcfToUSxfvtwfeRIRERFRkBkqRsZiZIyMAUD3VB2uvqgz1v12HEs+24s3ZlwCjdrzy9DiMgsOn3S1tE/vlSTrueVQKAQk6rTIKihHXpHRPW2xJSs322GyuEbB4qI10KgVMFqcKDPZEBcd3P26WgPZI2NjxozBm2++CZvNhtdffx2zZs3CG2+8gfDwcKxatQpDhw71R55EREREFGT6ipExXaTnI2OS8VekIT5ag9wiIz7aWHeDttrszMiFUwQ6tYlyN9nwFyl+a1k3Jo2KRYapEKpWQqNylQdlRo6MBYJXE25HjRqFUaNGwWKxoLi4GDExMVCr1b7OjYiIiIiakKKKkbHYKHkjYwAQplFh0g398NK72/DF5mMYMagdOrWJ9uix2w5UtLT34xRFSaLUxKOVdFSUirGEGNf3rVG75ilymmJgyB4Zkxw7dgwff/wxVq9eDb1ejx07dqCsrJ7NJoiIiIioWTNIe4x5MTIGABf0ScHQvilwOkW8uXYvHM6GO/bZ7E7sOuxaZzbEj1MUJYmxrql5ua1kZCy/ohiLj3F93xq1qzzgXmOBIXtkzOl0YtasWfjss88giiIEQcAVV1yBJUuW4NSpU1izZg2Sk/3/qQURERERBZY33RTPNen6vthzJB+HT+mx4Y8TuPqizvUef+B4AUwWO2IiQ9Gtvc7r5/VUkjRNscjk9+dqCvIris74GFeBzWIssGSPjC1ZsgTr1q3Df//7X2zZssW9B8GMGTPgdDrx2muv+TxJIiIiIgo+b7opnisuWou7r+oJAHhv/SEUFtdf9Gyr6KKY3jMJCqnVnx8ltNI1Y+6RMRWLsUCSXYx99tlnmDJlCm688UbExMS4b+/ZsyemTJmCLVu2+DI/IiIiImoCbHYnSo1WAPK7KZ7rigs7oUeqDiaLHW9/sb/O40RRdO8vFoj1YkDlxs8FBlOr2GuswOAa7UxwT1N0FbxlLMYCQnYxVlBQgJ49e9Z6X1JSEkpKShqdFBERERE1LcVlrlExpUJAZFjjGrcpFQIm39wfSoWAP/dnY+vf2bUedyavHLlFRoQoFRjQPaFRz+kpXZQGIUoBDqeIwpKWv9dYjZExTlMMKNnFWIcOHbB58+Za79u2bRs6dOjQ6KSIiIiIqGkpKqncY8wX0wU7tYnGuBFdAABvfb4Pxlq69+08nA8A6NctHtpQr5qAy6ZUCO7CJF/fsteNiaKIgoppotL0TG1FMcaRscCQXYzdfffdeP/99/HCCy/gjz/+gCAIOHnyJFauXImVK1fijjvu8EeeRERERBREhtKKPcYasV7sXLeN6YGk2DAUFJvxwYaMGvfvOlwAABgSoCmKEmmvsdwW3t6+uMwKm90JQXBt+AxwzVigyf6I4eabb0ZRURGWLl2K//3vfxBFEY8++ihUKhUmTJiA22+/3R95EhEREVEQ+aKT4rk06hA8dFN//N+yP/HN78cx8rx27o6J5WYHjpw2AADSA9DSvipp3VhLb+KRb3B9f7rIUIQoFbCC+4wFmlfjvZMmTcK//vUv7N69GwaDAVFRUejfv3+1hh5ERERE1HJInRR1Xu4xVpdBPRIxYmA7bN59Bm98shcLpl0MAPgnywxRBDq1iXKPVAVKa9n4+dz1YkDlyFiZkcVYIMguxl5++WWMGzcOvXr1wvDhw/2RExERERE1Me6RsSjfjYxJJlzXBzszcnE8qxhf/3Yclw9pgyNnXc8XqC6KVSXqXMVJyx8Zq6UYYwOPgJK9Zuzrr7/GjTfeiKuvvhrLli1Ddnbt3W+IiIiIqOVwrxnz8cgY4GoKcu81vQEAH3yfgayCchzNdhVjQwI8RRGoXDPW0jd+ltra11aMGc02OJ1iUPJqTWQXY1u2bMFbb72F3r174+2338bo0aNx11134fPPP0dZWZk/ciQiIiKiIJO6KfpyzVhVlw1JRe/OcbBYHXj5vV2w2kVER6jda8gCSZqmmG8wtuiCRJqmmBBTOQ1UKsacImC22oOSV2siuxhTKpUYMWIE5s6diz///BMLFixATEwMXnjhBVx00UV49NFH/ZEnEREREQWRvmJkLNaH3RSrEgQBk2/qjxClAvkVIzaDusf7pI2+XHFRGigUAuwO0T09syWqLMYqR8ZUSgGqELa3DxTZxVhVarUal19+OW6//XaMHTsWNpsNGzdu9FVuRERERNQEiKIIQ5V9xvylfVIkbh7dzf31eT0Cs9HzuZRKhXvqXktub59fsSYuPqZ6gR2mcbWV4Lox//N697ydO3di/fr12LBhA4qKitC7d288+eSTGDt2rC/zIyIiIqIgKzfbYbU7Afh2n7Ha3Dy6G3YcykGRoRx9u8b59bnqk6QLQ16REXlFRvTqFLw8/MXhcLqnnlZdMwYA4ZoQFJdZOTIWALKLsblz52LDhg3Izs5GSkoKbrrpJlx77bXo0qWLP/IjIiIioiDTV1y0h2tCEKpS+vW5VCFKvDAhHRkZh6BR+/e56pPg7qjYMpt4FJVY4BQBpUJATOS5I2MqABwZCwTZxdgnn3yCyy+/HNdddx2GDBnij5yIiIiIqAmROimee9HuLwqFAEEI/Fqxqlr6xs/SerG4aA2U56zLi9BymmKgyC7GrrjiCtx8883o37+/P/IhIiIioibGn3uMNVVSe/uWumbM3byjlg21uWYscGQ38Fi3bh3Ky8v9kQsRERERNUFFJRWdFAM0MtYUuEfGWmgx5t7wOVpb477wimmKXDPmf7KLsYEDB+Kvv/7yRy5ERERE1AQZKkbGYlrRyJi0ZizfYGqRe43lG2rvpAhwZCyQZE9T7NGjB1auXInvv/8eaWlpCAurPrQpCAJefvllnyVIRERERMEl7TGma0UjY/ExWigEwGZ3wlBm8dv+asFS2x5jkvCKYowjY/4nuxj74YcfkJiYCJvNhv3799e4P9iLLYmIiIjIt6QW6LGtaGQsRKlAXIwW+XoT8oqMLbYYO7etPQCEa9lNMVBkF2ObNm3yRx5ERERE1EQFuptiU5GoC3MVY3oj0jrGBjsdnyow1L7HGFA5MlZuZjHmb7LXjEmcTicyMjLw66+/oqysDAaDwYdpEREREVFT4e6mGNl6RsYAILFi3VhL66hosztgKHMV2LV2U6wYGSszshjzN9kjYwDw1Vdf4dVXX0VeXh4EQcCnn36KxYsXQ6VS4dVXX4VarfZ1nkREREQUBHaHE8VlVgBocVP1GpLo3musZW38LI2KqVVKRIapatzPkbHAkT0ytn79ejz55JO44IIL8Nprr0EUXd1lLrvsMmzevBlLlizxeZJEREREFBzFFSMoCoWAyLDW9YF7kq5ltreXOikmxGhq7ffAboqBI3tk7K233sJtt92G5557Dg6Hw337jTfeiKKiInzyySeYNm2aL3MkIiIioiDRV+wxFhMRCoWidTVqkzZ+ztO3rGKsvuYdABBRMU3RaLbD4RShbGU/90CSPTJ24sQJXHbZZbXe179/f+Tm5jY6KSIiIiJqGopKW18nRUlilY2fpdlgLUF+A8VYWGjleI2JUxX9SnYxFhcXh2PHjtV637FjxxAXF9fopIiIiIioaXCPjLWyToqAq1gRBMBasddYS1FfJ0UACAlRIFStBMC9xvxNdjF21VVXYdGiRdiwYQOsVtdiTkEQ8Pfff2PJkiW44oorfJ4kEREREQWHoZV2UgQAVYjC3bQkvwU18ajc8LlmJ0VJuKaioyKLMb+SvWZs2rRpOHLkCKZNmwaFwlXL3XnnnTAajRg8eDCmTp3q8ySJiIiIKDikDZ91rayToiRRF4bCYjNyi4zonqoLdjo+UVmM1T4yBrg2fi4qMbOJh5/JLsbUajXeeecdbNmyBX/99RcMBgMiIyMxZMgQjBgxotaOLERERETUPOkrNnyObYUjYwCQFBuGQ5lFLaqjYuWasboLbKmJB4sx//JqnzEAGDZsGIYNGwYAsNvtKCsrYyFGRERE1MIYKoqxmNY6MlbRxCO3hXRUNJpt7gKrrjVjgGtkDGAx5m+y14zZ7Xa88cYbWLduHQBg69atGDZsGIYOHYq7774bxcXFPk+SiIiIiILDPU2xlY6MJepcBUtLWTMmTVEM14QgTFNzw2eJe2SM3RT9SnYxtmjRIixduhQlJSUAgP/+97+IiYnB008/jVOnTuHVV1/1eZJEREREFHiiKFZOU2ytI2MVe43ltpBpig11UpRII2NlRhZj/iS7GPv222/x6KOP4l//+heOHTuGf/75Bw8++CDuuusuTJ8+HZs2bfJHnkREREQUYCaLHVabA4Br0+fWKCm2cuPnlrDXmLReLEFXdydFgNMUA0V2MZaXl4f+/fsDAH755RcoFApcfPHFAIDk5GSUlpbKiud0OrFo0SIMHz4cAwYMwMSJE3H69GmPHjdhwgQsXry4xn1jxoxBjx49qv176qmn3Pfr9Xo89thjSE9Px5AhQ/D888/DZGoZQ89EREREviKNimlDQ6AJ9brVQLMmjSBZrA6UlFuDnE3jFTSw4bNEmqZYxmmKfiX7tyoxMRFnzpzB4MGDsWnTJvTs2ROxsbEAgN27dyM5OVlWvCVLluDDDz/EnDlzkJycjHnz5mHChAlYt24d1Gp1rY+xWq2YNWsWfvvtN3dhKDEajTh9+jTefvtt9O7d2327RlM5tD5lyhSYTCasWrUKJSUleOaZZ2A0GvHKK6/Iyp2IiIioJZPWi8VGtc5RMQBQq5SIjQpFUYkFeXojopv5CGGBB50UAY6MBYrskbGxY8di9uzZuP/++7Fz507ceOONAICXXnoJixcvxjXXXONxLKvVipUrV2LKlCkYOXIk0tLS8NprryEnJwcbN26s9TG7du3CDTfcgB07diAqKqrG/UePHoXT6cTAgQORkJDg/hcZGQnAVTBu27YNr7zyCnr37o2hQ4fihRdewFdffYXc3Fy5LwcRERFRi2UoqeikGNk614tJpHVjeUXNfyZVvsG19q2+PcYAFmOBIrsYmzZtGu677z4IgoDHHnsMd9xxBwBg//79uO+++/DQQw95HCsjIwPl5eUYOnSo+7aoqCj06tUL27dvr/UxmzdvxvDhw/Hll1+6C6yqDh8+jPj4eERHR9f6+B07diAhIQFdunRx3zZkyBAIgoCdO3d6nDsRERFRS6cvbd2dFCWJVdaNNXceT1Os6LRYxmLMr2RPUxQEAZMmTcKkSZOq3f7RRx/JfvKcnBwAQEpKSrXbExMT3feda/r06fXGPHz4MMLCwjBlyhTs2rULOp0ON954I+666y4oFArk5ubWeD61Wo2YmBhkZ2fL/h6IiIiIWqrKaYocGQPQ7Dd+FkUR+TK7KXJkzL+8WomZk5OD999/Hzt27EBxcTHi4uJwwQUX4M4774ROp/M4jtQ049y1YaGhoV7vV/bPP/+gpKQEl19+OSZPnoydO3di3rx5KC4uxtSpU2EymWpdixYaGgqLxeLVcwKuN7fRGPxfUOk19VdDEn/GZ+7Bic/cgxOfuQcnPnMPTnzmHpz4vohdYCgHAIRrFDWuc5p67r6MHxPhumTOKiht8HqvqeVeVanR6u6OGaaqee1aNbZScHWOLDPZfHKN25RfF18TRRGCIHh0rOxi7NChQ7jzzjtht9sxYMAAtG/fHvn5+Vi+fDk+++wzrFmzBu3bt/coltRUw2q1VmuwYbFYoNXWX63XZfny5bBYLO4pjD169EBZWRmWLl2KRx55BBqNBlZrzU44FosFYWH1t/isj81mw6FDh7x+vK9lZmY22/jMPTjxmXtw4jP34MRn7sGJz9yDE78xsc/kFAEAjKWFOHTI7PP4DWkqr7u5YoTwTI7B4+u9ppJ7Vdl61zVwuEaBY0eP1BvbaHECcHWR/PvAQSgVnhUXDWmKr4s/1NWI8Fyyi7FXXnkF7du3x/LlyxEfH+++PTs7GxMmTMDs2bOxZMkSj2JJ0wXz8vKQmprqvj0vLw89evSQmxoA1zd+7jffvXt3GI1GFBcXIzk5GT/++GO1+61WKwwGAxITE716TgBQqVTo2rWr14/3FZPJhMzMTHTs2NHrgjZY8Zl7cOIz9+DEZ+7Bic/cgxOfuQcnvi9i234yALCgV/eO6Nktvtp9TT13X8aPii/Hml8KUGoWkZaWVu+oR1PLvSpjRj6APCTFhqNnz571xlarQ4HPsgAAqR27Iircs+LCH3k3hfhyHD161ONjZRdju3fvxoIFC6oVYoCrsJoyZUq1/bwakpaWhoiICGzdutVdjJWUlODgwYMYP3683NQgiiIuu+wyjBs3Dg8//LD79v379yMhIQE6nQ7p6emYP38+Tp48iQ4dOgAAtm3bBgA477zzZD+nRBCERo2s+ZpWq/VrPv6Mz9yDE5+5Byc+cw9OfOYenPjMPTjxGxO7pNy1XiglIbrOGE01d1/GT23jamBisjjgFFSIDGu4MGkquVdVYnRNUUyMDa/3sVJsbWgITBY7nAjx2ffSFF8XX/N0iiLgRTEWGxuL8vLyWu9TKpUIDw/3OJZarcb48eMxf/58xMbGom3btpg3bx6Sk5MxZswYOBwOFBUVITIysto0xroIgoDLLrsMK1asQOfOndGnTx/8+eefeOedd/DMM88AAPr3749BgwZh+vTpeO6552A0GjFr1iyMGzcOSUlJHudORERE1JI5HE4Ul0ut7Vt3N8VQlRIxkaEwlFqQW2T0qBhrivI97KQoCdeqYLLY2VHRj2QXYw8++CBeffVVdOnSpdqmyqdPn8bChQvxwAMPyIo3ZcoU2O12zJw5E2azGenp6VixYgVUKhXOnDmD0aNHY/bs2bjhhhs8ivfYY48hIiICCxYsQE5ODtq1a4dnnnkGt9xyCwBXwfbGG2/g+eefx913343Q0FBcccUVePrpp2XlTURERNSSGcosEEVAoRAQFd66izEASNKFwVBqQb7eiK7tYoKdjlcKKjopNrTHmCRCq0KBwcSOin7kUTE2atSoasNtBQUFuOmmm9C+fXvEx8ejuLgYJ06cgFqtxvfff4+77rrL4wSUSiVmzJiBGTNm1LivXbt2OHz4cJ2P3bRpU43bQkJCMHnyZEyePLnOx8XFxWHRokUe50hERETU2uhLK0bFItQ+a97QnCXotDh8So/cZrzxc0GxK/eEGM+m8bnb25tZjPmLR8WYtClyffr16+eThIiIiIgo+AxSMRbZuvcYkyS1gI2f5U5TjOBeY37nUTE2Z84cf+dBRERERE0IN3yuLlEqxprpxs8Op4iiYvlrxgCgzMhizF+82vQZAH799Vds27YNJSUl0Ol0GDx4MIYPH+7L3IiIiIgoSPSlrmJM18qbd0gSdc17ZMxQaobdIUIhALFRnv1MOU3R/2QXY1arFQ899BB+//13KJVK6HQ66PV6LFu2DBdccAHefvttjzc5IyIiIqKmyVDCTopVJepco0nNdWSsoGKKYmyUBkqlwqPHhGsqRsY4TdFvPPtJVLF48WLs3LkTc+fOxb59+/D7779j7969mD17Nvbs2YOlS5f6I08iIiIiCqCiUk5TrEoaGSs3N89W71InRU+nKAJARBjXjPmb7GLsm2++wcMPP4xrr70WSqUSgKuDobTR8rp163yeJBEREREFlr5iZEzHBh4AAE1oCKIjXLO/8pvhVEWpeUeCzvMNkaWRMRZj/iO7GCsqKkKvXr1qva9Xr17Izc1tdFJEREREFFyV3RQ5TVEiFTK5zXCqYoHMTopAlQYeLMb8RnYxlpqaip07d9Z63/bt25GSktLopIiIiIgouNwNPDxs9tAaJOmab0fFymLM85FOtrb3P9kNPG677TbMmTMHGo0GV199NeLj41FQUIBvvvkGy5cvx8MPP+yPPImIiIgoQIxmG8xWBwBOU6xKam+f2yynKbpyTvBiZIzFmP/ILsZuv/12HDx4EPPnz8err77qvl0URVx//fV44IEHfJogEREREQWWNEVRG6qENtTrnZBaHKmjYr7eFORM5PNmmiJHxvxP9m+XQqHASy+9hPvuuw/btm1DcXExoqOjMWTIEHTp0sUfORIRERFRAOnd68U4KlaVe2SsmU1TtNmd7p+pN2vGrHYnrDYH1CqlX/Jrzbz+qKNLly4svoiIiIhaoKIStrWvTXNdM1ZUYoYoAqoQBaLDPV8DqA0NgSAAouja+JnFmO/JbuBBRERERC2b1LyDnRSrS6iYplhmssFobj5T99xTFKO1UCgEjx+nUAgIkzZ+Njaf77c5YTFGRERERNVIa8Z0LMaqCdOoEFmxEXJeM1o3lu/FejGJu4lHMyo+mxMWY0RERERUDacp1k1aN9acpipKm1TLaWsvYRMP/2IxRkRERETV6DkyVqdEad1YM2pv700nRQmLMf9iMUZERERE1RhK2E2xLlIx1pw6KhYYXCOdcvYYk3CvMf/yaTF26NAhjB492pchiYiIiCjAiko5TbEuibGugqY5jowlVBSScoRLDTxYjPmFT4sxtVqNNm3a+DIkEREREQWQwymipIzTFOvSHNvb+6SBB4sxv/BpMdalSxesXr3alyGJiIiIKIBKyixwioBCAKIiWIydy93Ao5l0UzRb7Sg1WgF4uWYsjCNj/uSTYsxut8NgMPgiFBEREREFkdRJMToiFEoZe1K1FtKasZJyK0wWe5CzaZg0RVEbqkS4JkT246VpihwZ8w/ZxZjdbscbb7yBdevWAQC2bt2KYcOGYejQobj77rtRXFzs8ySJiIiIKDAqOylyvVhtwrUq99S95rBurGonRUGQX1xzmqJ/yS7GFi1ahKVLl6KkpAQA8N///hcxMTF4+umncerUKbz66qs+T5KIiIiIAsNQ0bwjJopTFOsirRvLbwZTFd3FWLT8KYpAZWt7TlP0D9nF2LfffotHH30U//rXv3Ds2DH8888/ePDBB3HXXXdh+vTp2LRpkz/yJCIiIqIAKKpoax/LkbE6JehchU1zaG+fL7W196KTIsCRMX+TXYzl5eWhf//+AIBffvkFCoUCF198MQAgOTkZpaWlvs2QiIiIiAJGXzEypuPIWJ2SYptPR8XGbPgMVNn02cxizB9kF2OJiYk4c+YMAGDTpk3o2bMnYmNjAQC7d+9GcnKybzMkIiIiooCR1ozFsK19naSOirnNaM1YQox3I51VR8ZEUfRZXuQiuxgbO3YsZs+ejfvvvx87d+7EjTfeCAB46aWXsHjxYlxzzTU+T5KIiIiIAsNQUYxxw+e6JbrXjDX9Yizf4MrR25ExqRizO0RYbA6f5UUusvtbTps2DWFhYdi+fTsee+wx3HHHHQCA/fv347777sNDDz3k8ySJiIiIKDCk1vbspli3xIo1Y3lFTbuBhyiKjZ6mqFEroVAIcDpFlJts0Kjlt8enusl+NfV6PSZNmoRJkyZVu/2jjz7yWVJEREREFBxSN0UdpynWSVozZiizwGy1N9kCpdxsh8niGs3ythgTBAHhGhVKjVaUmWyI87IrI9VO9jTFiy++GA8++CA2bNgAq9Xqj5yIiIiIKAhMlsqLdx2nKdYpXKtCWMUGyk25vb00KhYZpm5UwRjBjop+I7sYe/zxx1FYWIhp06Zh2LBhePbZZ7Fjxw5/5EZEREREASR1UtSoldCGNs3RnqZAEAT3urGmvPFzZfOOxo1mhYexGPMX2b9l99xzD+655x6cPn0a33zzDdavX4+1a9eiTZs2uPbaa3HNNdegS5cu/siViIiIiPxIX7HHGNeLNSxRF4bM7JIm3d4+v5HrxSQRGhZj/iJ7ZEzSvn17PPjgg1i3bh3WrVuHkSNHYvny5Rg7dqwv8yMiIiKiAJE6KXKPsYYlxjb9jZ+lbo/xXra1l0gdFctYjPlco8afCwsL8d133+G7777D7t27ERMTg6uuuspXuRERERFRALGTouekJh7NYc1YY0fGwrlmzG9kF2OlpaX4/vvv8e2332L79u1QKpUYNWoUlixZguHDh0OpVPojTyIiIiLyMz07KXosQdf0N34uMLh+no1eM8aRMb+RXYwNHToUTqcT5513Hp577jlcccUViIiI8EduRERERBRA0jTFGE5TbFCS1MCjCU9TdDfwqMjVW+ym6D+yi7FHHnkE11xzDdq0aeOPfIiIiIgoSKRpirGcptigxIppivpSC6w2B9SqpjU7TBRFFBT7eJqimcWYr8kuxs7d7JmIiIiIWga9u4EHi7GGRIapoFErYbY6kG8woW1C05opVlxmhc3uhCAAcdE+auBhZDHma153UyQiIiKilsVQsWYshmvGGiQIgnt0rCl2VMw3uHLSRYYiRNm4S/4Ijoz5DYsxIiIiIoLDKbrXjMVyZMwj0sbP+U2wiYevOikCQDj3GfMbFmNEREREhJJyC5wiIAhAdLg62Ok0C4m6prvXmK82fAaAiDAWY/7CYoyIiIiI3KNi0eGhUDZyWltrIe01llfU9PYaq2xr37hOikD1fcZEUWx0PKrkUQOP7du3ywqanp4u63in04k33ngDa9euRWlpKdLT0zFr1iy0b9++wcc98MAD6N+/Px555JFqt69cuRJr165Fbm4u2rZti3vuuQc333yz+5ilS5fi9ddfrxHz8OHDsnInIiIiagncGz6zrb3HpDVjeS19mmJFMeYUAZPFjrCKaYvUeB4VY3feeScEQaj1Pqk6rnr/oUOHZCWxZMkSfPjhh5gzZw6Sk5Mxb948TJgwAevWrYNaXfswudVqxaxZs/Dbb7+hf//+1e57++23sXLlSjz//PPo06cP/vzzTzz33HNQqVQYN24cAFfRdd1112HGjBmyciUiIiJqifQlFZ0U2dbeY9KasaZcjDV2w2cAUIcoEKJUwO5wosxkYzHmQx4VY++//777/7OysvDss8/ixhtvxJVXXomEhAQYDAZs2rQJH330EV544QVZCVitVqxcuRKPP/44Ro4cCQB47bXXMHz4cGzcuBFjx46t8Zhdu3Zh1qxZMJvNiIqKqnH///73P9x333246qqrAACpqanYu3cv1q5d6y7Gjhw5gltuuQUJCQmy8iUiIiJqifTspCibVIwVlZhhszugCmk6e41JTUXiYxpfXAuCgAitCoYyi2vdmK7RIamCR8XYkCFD3P9/55134p577sFjjz1W7ZhBgwZBo9Hg3XffdRdBnsjIyEB5eTmGDh3qvi0qKgq9evXC9u3bay3GNm/ejOHDh2Py5Mm49tprq93ndDrxyiuvoFOnTtVuVygUKCkpAeAqADMzM9G5c2eP8yQiIiJqyfTspChbdIQaapUSVptrr7E28U1jrzGHw+meduqLaYqAa6qiuxgjn5G9OnPfvn3VCqeqBg4ciCNHjsiKl5OTAwBISUmpdntiYqL7vnNNnz4dTz75JCIiar7hFQoFhg4diuTkZPdtWVlZ+Pbbb3HRRRcBAI4ePQqHw4Hvv/8el19+OUaOHIkZM2YgLy9PVu5ERERELYVeWjPGkTGPCYKApFhXsZPfhJp4FJW4OmMqFQJifDTtNELLjor+4NHIWFXJycn47bffcOGFF9a4b8OGDUhNTZUVz2RyvXHPXRsWGhqK4uJiuenVUFBQgIkTJyIuLg4PPvggALgLRq1Wi4ULF6KwsBALFizAXXfdhS+//BIajfw3rSiKMBqDP19Yej2l/zan+Mw9OPGZe3DiM/fgxGfuwYnP3IMTX27swmLXdUxYqODRNU1Tyj2Y8eOiQnE6twyncw3o1i68SeR+JtcAAIiNCoXF7Hke9cXWqF1jOEXF5V5d8zaF1yVQRFGss9/GuWQXY/feey+ee+455OXl4ZJLLoFOp0NBQQE2bNiAX375BQsWLJAVTyp8rFZrtSLIYrFAq23csOrx48fxwAMPwOFw4P3333evLxs3bhwuvvhixMbGuo/t1q0bLr74YmzatEnWNEuJzWaT3bjEnzIzM5ttfOYenPjMPTjxmXtw4jP34MRn7sGJ72nsvMIyAEBxUQ4OHdL7PL43msPrHiK6RhQP/XMaKWElPo1dn/ri/33SVSxpVU6vrk9ri223umIezzyDRI33AybN4WfqC3U1ITyX7GLstttug91ux9KlS/Htt9+6b09JScH8+fNx5ZVXyoonTU/My8urNqqWl5eHHj16yE3PbefOnXjwwQeRlJSEd955B0lJSdXur1qIAa5pkTExMXVOjWyISqVC165dvc7XV0wmEzIzM9GxY8dGF7OBjs/cgxOfuQcnPnMPTnzmHpz4zD048eXGNn6WDQDo37sb2sSH+zy+HM3pdT+SfwI7jh6FGBKBnj17NoncjxZmAihC+5RY9OzZ0yexU/45iAOnziIiOg49e3bxS96N4e/4chw9etTjY2UXYwAwfvx4jB8/HsePH0dxcTF0Oh06duzoTSikpaUhIiICW7dudRdjJSUlOHjwIMaPH+9VzH379mHChAno1asXli5dWqPj4muvvYYNGzZgw4YN7iHEM2fOQK/Xe11QCYKAsLDGb6rnK1qt1q/5+DM+cw9OfOYenPjMPTjxmXtw4jP34MT3JLbZYofJ4gAAtEmMkdW6PNi5Bzt+u6QYAEBRqbVarGDmbii3AwCS4yK8yqG22DGRrq+tDjTq+2oOP9PG8nSKIuBFAw9JcXExTpw4gYyMDERFReH48eNe7citVqsxfvx4zJ8/Hz/99BMyMjIwffp0JCcnY8yYMXA4HMjPz4fZbPYont1ux+OPP464uDjMmTMHFosF+fn5yM/PR1FREQDgsssuw9mzZ/Hcc8/hxIkT2L59Ox555BEMGjQIw4cPl/09EFHLI4oiftx+BllF1mCnQkTkd4YyVyfFULUS2lCvPqtvtRIqGnjkFgW/d4DEl3uMScLZwMMvvPptW7p0Kd5++22YzWYIgoB+/frh9ddfh16vx8qVK2vd+6s+U6ZMgd1ux8yZM2E2m5Geno4VK1ZApVLhzJkzGD16NGbPno0bbrihwVj79u3DyZMnAQCXXnpptfvatm2LTZs2oU+fPli+fDkWLlyIG264AWq1GqNHj8aTTz4pq5Ilopbr8Ck9ln99CEkxKoweFuxsiIj8q6hKJ0VeC8mTJO01VmyC3eEMcjYuUjHmq7b2AIsxf5FdjK1ZswaLFy/GpEmTcMkll+CWW24B4Jq6+MQTT2DhwoV49tlnZcVUKpWYMWMGZsyYUeO+du3a4fDhw3U+dtOmTdW+HjRoUL3HS4YOHVpni34iopxC1yecBSU2OJ3yR/2JiJoTaY8xnY/aoLcmMZGhUIcoYLU7UWAwIUob/GK2wOAqrhN0vpuuF1ExdbWMxZhPyZ6muHr1ajzwwAOYOnUqevfu7b59xIgRmDZtWo3iiIioOZL223E4Kz8xJiJqqQzSyFgU9xiTSxAEJOhcI1B5+uBPVbTZHe5ppxwZa/pkF2NZWVkYMmRIrfd17twZBQUFjU6KiCjYpE+JASBXH/w9S4iI/KmII2ONklgxApXXBNaNSaNiapUSkWGeN2JpSEQYizF/kF2MpaSkYPfu3bXe9/fff7tb1RMRNWf6KqNhuUUsxoioZdNXWTNG8iXGuoqxpvD3It/gKggTYjQ+Xf/HkTH/kL1m7KabbsLixYuh0WgwcuRIAIDRaMT333+Pt99+G/fee6+vcyQiCriiasVY8D/pJCLyJ/easSiOjHkjqaIYawrTFP3RvAMAwivWjBktdjidIhSK4K+NawlkF2MTJ07EmTNnMH/+fMyfPx8AcNdddwEArrnmGkyaNMm3GRIRBYG+lCNjRNR6SOc8jox5R2qU0RSKsXx/FWNaV9kgioDRbENEmNqn8Vsr2cWYIAh44YUXcN999+Gvv/6CwWBAZGQk0tPT0b17d3/kSEQUcEUlXDNGRK2HvoRrxhojqQmuGUuI8e3Gx6oQJdQqJaw2B8pMLMZ8RXYx9sYbb+Dmm29Gx44d0bFjx2r3nTlzBitXrsSsWbN8lR8RUcBZbY5qc+I5TZGIWjKnU3R332M3Re8kVmz8XFBshiPIe435a5oiAERoQ1B0zt9IahzZDTzefPNN5Obm1nrf3r17sXbt2kYnRUQUTNLaCWXFfPhykx1lRmswUyIi8puSciucThGCAERHsBjzhi5SgxClAk6n6O5MGSxSMZbgh2IsXOsaDSs3sxjzFY9Gxm677Tbs3bsXACCKIm699dY6j+3bt69vMiMiChKpq1hsVCiMZivKzU7kFBrRlVMyiKgFktaLRYWrEaKU/Tk9AVAoXHuNZReUI19vQjBbW+RXrFuLj/H9lNMIdlT0OY+Ksf/+97/YsGEDRFHEm2++iRtvvBHJycnVjlEoFIiKisKYMWP8kigRUaAUVWnxHBriRLnZiuzCcnRtHxPcxIiI/EDPPcZ8IkkX5irGDGYkBumlNJptKDfbAfhnmqLU3r7MyGLMVzwqxrp27YqHH34YgKuBx80334ykpCT3/Xa7HSEhspefERE1SdKFSXREKDRKG84UWJFTWB7krIiI/IN7jPmGtNdYvsGExOQGDvYTaYpiuCYEYRrfbfgskdrbc5qi78gei3744Yfx1Vdf4YEHHnDftnPnTlx00UVYs2aNT5MjIgqGygsTNXQRSgBATiGbeBBRy8Q9xnwjUecaico3mBs40n/cnRR1vu2kKJHa25dxmqLPyC7GVq5ciddff71aJ8XU1FRcccUVmDNnDht4EFGzJ01TjIkMhS7C9YeHI2NE1FJxjzHfcI+MBXE7FH/tMSaR2tlzzZjvyJ5b+NFHH2HatGnVRsZSUlIwc+ZMxMfHY9WqVbj55pt9miQRUSBVrp8IhU3hOk1msxgjohbKUMKRMV9IrBiNco1OhQclB3+2tQeqTFNkMeYzskfGcnNz6+yY2L9/f5w5c6bRSRERBZN7ZCxC7R4ZKzCYYLMHd+8YIiJ/KOLImE+4i7FiM5xOMSg55Bv810kRqNLAg8WYz8guxtq2bYs///yz1vu2b99eo8siEVFzYyitnKYYoVEgVKWAKAJ5eq4bI6KWR1/Cboq+EButgVIhwOEUUWpyBCUHf+4xBrC1vT/InqZ4yy23YN68ebDZbLj00ksRFxeHoqIi/Pzzz3j33Xfx2GOP+SNPIqKAcDhFGKpMU8wpFpCoC8PpvDJkF5SjbUJEkDMkIvIt6QMoXRRHxhpDWbHXWE6hEYby4BZjfpumWNHAg8WY78guxu655x7k5uZi9erVWLVqlft2pVKJu+++G/fee68v8yMiCqiSMgucIqAQgOhwNXIAJMVqcTqvDLlcN0ZELYzF5nDvS8WRscZL1IVVFGP2gD+3KIruTo4JMf7pphihZQMPX/Nqc7Ann3wSDz30EHbv3o3i4mJERUWhX79+0Ol0vs6PiCigpPVi0RGhUCgEAK5iDACy2d6eiFoYaSsPdYgCYRruGdtY0rqxYIyMlRptsNpcz+vvNWPcZ8x3vP6ti4yMxMUXX+zLXIiIgq6yk2LlH7KkinbFbG9PRC2NocoeY4IgBDmb5k9qbx+MkTFpimJMRChUIUq/PIdUjJksDjgcTiiVsttP0Dk8KsZGjx6NN998E2lpaRg1alS9v6yCIODHH3/0WYJERIHk3vC5ytoJaWSMxRgRtTRFJeyk6EvS34viIIyM5ev920kRAMKrjJ6WmWyIjuD7prE8KsaGDBmC8PBw9//zkxMiaqmkFs+xVfbbSdRVFGNFRoiiyHMgEbUY+lLuMeZLCdI0xbLAF2P+bt4BAEqlAtpQJUwWB8rNLMZ8waNibPbs2e7/nzNnjt+SISIKNn0tm58mxmihEACL1QF9qaVaoUZE1JzpuceYTyVVFGPFRnvA9xrLD0AxBgDhWjVMFhObePgIJ3oSEVVR24VJSIjC/ceNUxWJqCWp7QMo8l5ctAZqlQIOJ3A8qySgz13g506KEu415lsejYylpaXJmpZz6NAhrxMiIgqmui5MkuPCkac3IaewHL06xQUjNSIin+PImG8plQoM6ZmI3/flYNPOs+jXPSVgz11Q7N8NnyXujoqmwDcpaYk8KsYmT57sLsYsFgveffdddOzYEZdffjkSEhJgMBiwadMmHDlyBA8++KBfEyYi8idpMXtsZM1ibN/RAmQXsL09EbUcXDPme6MGt8Xv+3Lwx/4c/PsGOzShgdkyIGDTFDWuYqzMZPXr87QWHr07HnnkEff//+c//8HIkSOxePHiaqNl//73vzFjxgwcOHDA91kSEQWAKIq1dlMEgOS4ivb2RZymSEQth57dFH2uV0cddBFK6Msc2LIvC6PTU/3+nA6niMKArRlzlQ+cpugbsteMfffdd7j11ltrnbZ43XXX4bfffvNJYkREgVZutsNqdwKofZoiAOQUsBgjopbB6RQr9xmL5MiYrwiCgIFdXH8zNm49GZDnNJSa4XCKUAhAbJR/C+uIMDUAV2t7ajzZxVh4eDhOnTpV630HDx5EdHR0o5MiIgoG6RPicE0IQlXVN8xMkYqxQk5TJKKWodRohaOi418MR8Z8akCncAgCcPBEEc7klfr9+aS29rFRGr9vxCxNU+TImG/I/mldffXVWLBgAT755BPk5eXBZrMhJycHq1atwptvvombbrrJH3kSEfmdeyF7LWsnkuNdxZihzAKjmX+AiKj5k9aLRYWrEeLnC/jWJipMiYHd4wEAP26rfRDDl9ydFHX+7aQIsIGHr8leUfjYY48hOzsbs2bNqjZVURRF3HLLLZg8ebJPE2xpHA4HbDb/XchZLBb3fxUK359Y/RmfuQcn/rmxVSoVlEplA49qmYoqOinWto9YhFaFyDAVSo025BYZ0akNZwEQUfPG9WL+NWpQW+w6XICfdpzG+Ct7+rXgDVTzDgCIqFgzxgYeviG7GFOr1Vi0aBH++ecf7NixAyUlJdDpdLjggguQmur/BYrNlSiKyMnJgcFg8OvzOJ1OhISEICsryy9FgT/jM/fgxK8tdkxMDJKTk2VtadESVF6Y1L52IikuHKVGA3IKy1mMEVGzp+d6Mb8a2CMeMZGhMJRasP1gLob29V+b+4IAFmPh3GfMp7zutdmtWzckJycjLy8P7du3b7WfpHtKKsQSExMRFhbmt4tch8MBi8WC0NBQv/xM/BmfuQcnftXYCoUCRqMReXl5AICUlMDtj9IUVLZ4rv1T4pS4cBw9bWB7eyJqEQyltXePJd8IUSow6rz2+PyXo/hh20m/FmP5BtffpfgY/xfWEVpXA49yTtn3Ca+Ksa1bt2L+/Pn4+++/IQgC1q5di+XLlyM5ORlPPfWUr3Ns9hwOh7sQi4vz72axDocDAKDRaPxWFPgrPnMPTvxzY2u1rk/V8vLykJiY2Ko+aGloZIzt7YmoJZGmZnNkzH8uHZKKz385ip2HclFYbEJctH9GrqSRMX9v+AxwZMzXZM93+vPPP3H//fdDo9Hg8ccfhyi6uvCkpaXh/fffx7vvvuvzJJs7aY1YWJj/F1US+YL0XvXn+samyL3hcx2fErO9PRG1JHqOjPld+6RI9OoUC6cIbNpx2m/PE4xpimVs4OETsoux119/HaNHj8bq1atx9913u4uxf//735gwYQLWrl3r8yRbita2/oaar9b6Xq2vmyLA9vZE1LJwj7HAuGxIBwDAD1tPwVmxlYAv2exO9zT7hJjAdVO02hyw2R1+f76WTnYxdujQIdx4440Aal6wDRs2DGfPnvVNZkREAVY5Zaf+kbE8vREOhzNgeRER+YM0G4AjY/51Uf820IaGILuwHAeOF/o8flGJGaIIqEIUiI5Q+zz+ucJCQyCVANz4ufFkF2ORkZHIz8+v9b7s7GxERkY2Oikif/m///s/3H333e6vd+7ciR07dgAAzpw5gx49emDr1q3BSo+CyGpzuOe/19baHgBiozUIUSrgcIruNsJERM0VuykGhiY0BBcPbAsA2LjtpM/ju6coRmsDMrNFoRAQFupqO8F1Y40nuxgbPXo0XnvtNezfv999myAIyMnJwVtvvYWRI0f6Mj8in3r88cexcOFC99d33HEHTp3y/2aM1PRJFyWqEIV7Csa5lAoBSbEVTTwKuW6MiJqvqh9A1TU1m3xnzPmuqYp/7M3y+WhSIPcYk4SHVXRUZDHWaLKLscceewxxcXG45ZZb3IXXo48+iiuuuAKCIODRRx/1dY5EPhMZGYmYmJhgp0FNkLuTYpSm3k8WU+K5boyImr9qH0BpvN7piDzUrX0MOiRHwmp34tfdZ3waO18fuLb2kgiN1FGRTTwaS3YxFh0djbVr1+L5559Heno6LrzwQvTo0QMzZszA559/jtjYWH/k2SKJogizxe7bf1Y7zFaH678NHCs1X5GjV69e+OyzzzB+/Hj07dsXV155JXbt2oWPP/4YI0eOxKBBgzBt2jSYza4LW4fDgXnz5mHEiBHo06cPrrjiCvzvf/+rFvOzzz7DlVdeiYEDB+KGG27A6tWr4XR6th5n9erVSE9Pd7dndzqdOP/88zFp0iT3MYcPH0aPHj2QnZ1dbZpijx49AABPP/10tS0Z9u7di5tvvhl9+vTB6NGj8dlnn3n8+vznP/9Bjx49avwbNWoUAMBkMuGZZ57BsGHD0LdvX4wbNw4bN250P14URbz33nu4/PLL0a9fP1x99dX45ptv3PdnZ2fj8ccfx7BhwzBgwADcf//9yMjIcN//1FNPYcqUKbjvvvswaNAgLF++HADw888/44YbbkC/fv1w2WWX4fXXX4fVavX4+2oN3Gsn6lgvJknmyBgRtQDuhkWRoa22aVMgCYKAyypGxzZu9e1UxUB2UpRUdlTktURjyf4o5Nlnn8VNN92EW265Bbfccos/cmoVRFHEk2/8jkOZRUHLoWfHWLzy8EWyT8JvvvkmXnrpJXTu3BlPPfUU/v3vf6NPnz5YtmwZTpw4gcceewxr167FnXfeiQ8//BAbNmzAa6+9hqSkJPz888947rnn0K1bNwwePBgff/wxFixYgFmzZqF3797Yt28fXnnlFeTl5eGJJ55oMJdLLrkE//3vf/H333+jf//+OHDgAIqLi7Fjxw44HA4olUps3rwZvXv3rrGB8e+//46LLroI//nPf3DDDTeguLgYAPDee+/hxRdfRNeuXbFy5UrMnDkTgwcPRocOHRrM5+mnn8bjjz/u/nrnzp149NFH8cgjjwAAFi5ciMOHD2PZsmWIiorC2rVrMX36dHz//fdo164d3nnnHbz55pt45plncP7552Pz5s144oknEBsbi65du+Jf//oXUlNTsXTpUqjVaixevBjjx4/HV199hbZtXfPRv//+e8yYMQPPPvssNBoNfv31V0ybNg1PP/00LrzwQpw6dQovvvgiTpw4UW3KZmsnfUpc13oxSXLFyFg2izEiasb0UsMiTlEMmJGD2mHVNwdw7Ewxjp8tRue20T6JW2BwFdYJusBtoRSu5ZoxX5E9Mvb111+jvNx3FyFOpxOLFi3C8OHDMWDAAEycOBGnTze8D4PT6cSECROwePHiGvd99913uOqqq9CvXz+MGzcOf/75Z7X79Xo9HnvsMaSnp2PIkCF4/vnnYTJxMb6nrrvuOlxyySXo3LkzrrvuOhQXF2PWrFno3r07Lr/8cvTs2RP//PMPAODUqVMICwtDu3bt0LZtW4wfPx7vvvsuOnXqBABYsmQJHnzwQVx99dVo3749Ro8ejWnTpmHNmjWwWCwN5tKuXTt0794dv//+OwDgjz/+wIgRI2CxWHDgwAEAwC+//OIemaoqISEBgGvqYtXGM5MnT8aoUaOQmpqK6dOnw+l0umM1JDIyEgkJCUhISIDJZML//d//4b777sP111/vfj3Cw8PRvn17tG/fHlOnTsVbb72F6Oho96jYXXfdhZtvvhmpqam48847MX36dNjtdqxfvx4GgwELFy5Ev379kJaWhldffRUajQYffPCBO4fo6GhMmDABnTp1QkpKCt566y3ccsstuO2225CamoqLLroIzz//PDZs2IAzZ3w7VaI503s4Msb29kTUElQdGaPAiI4IxQV9XB8M/+DD0bFAbvgsqRwZYzHWWLJHxgYOHIitW7fiwgsv9EkCS5YswYcffog5c+YgOTkZ8+bNw4QJE7Bu3Tqo1bW357RarZg1axZ+++039O/fv9p9f/31F2bMmIEnnngCw4YNw6effooHHngAX375Jbp06QIAmDJlCkwmE1atWoWSkhI888wzMBqNeOWVV3zyPXlCEAS88vBFsFh9uz+Dw+mA2WyBRhMKpUJZ77GhaqVXUxPat2/v/n+t1vWLn5qa6r5No9G4p8D961//wo8//ogRI0agZ8+eGDZsGK6++mrExcWhqKgIOTk5WLBgARYuXOieNimKIiwWC86cOeP+mdVn1KhR+OOPPzB58mRs2bIFV155JfR6Pf766y906NABe/bswaxZszz+/qRCEXAVNgA8KgyrKi4uxgMPPID09HQ89thj7tsnTpyIf//73xg6dCj69euHYcOG4ZprrkFkZCSKioqQn59f4z09ceJEOBwObNy4ER06dKg2FVij0aBfv344cuSI+7ZzR/AOHjyIffv24dNPP3XfJr3Wx44dqzFi2FpVbvjcwMhYXOU0RVEUOb2HiJol98gYOykG1GXnd8Dve7Pw864zuOea3ghV1X+t5olgNPCI0LKBh6/ILsZ69OiBFStWYMOGDUhLS0NYWPUhUUEQ8PLLL3sUy2q1YuXKlXj88cfdzUBee+01DB8+HBs3bsTYsWNrPGbXrl2YNWsWzGYzoqKiaty/fPlyXHrppbjrrrsAAE8++SR2796N9957Dy+88AJ2796Nbdu2Yf369e4L/RdeeAETJkzAo48+iqSkJDkvR6MIggBNqG8XzTocAuC0Q6MOgVLZ+F/w2oSE1MxZoah9kLVjx47YuHEjtm3bhi1btuCXX37B8uXLMXv2bAwfPhwA3NPnHA4HLBYLQkNDoVQqPS4SRo0ahRUrVqCgoAC7d+/GCy+8gJycHGzduhVt2rRBcnIy0tLS3OvKGlLb9yJnfZ3NZsPDDz8MrVaLuXPnVrtYHzhwIDZv3owtW7bgzz//xJdffomlS5finXfeQZ8+feqNW1cOTqez2s9Eo9HUuH/ChAnu0bmqpNFBqtLiuYFiLKliZMxotqOk3IroCH6qTETNT0Ob3JN/DOiWgASdFvl6E/7cn42Rg9o1Kp7Zakep0fUBeDDWjJWb2cCjsWRPU/zhhx+QmJgIm82G/fv3Y+vWrTX+eSojIwPl5eUYOnSo+7aoqCj06tUL27dvr/UxmzdvxvDhw/Hll1/W2NPM6XRi165d1eIBwPnnn++Ot2PHDiQkJFQbcRkyZAgEQcDOnTs9zp088/7772Pjxo0YNmwYnnjiCaxbtw5Dhw7F+vXrERcXh9jYWJw+fRodOnRAhw4dkJqaioMHD+L111/3+Dn69euH6OhovPXWW4iLi0PHjh0xdOhQ7Ny5Exs3bqx1iqI/zZo1CydOnMDSpUtrfFixaNEi7Ny5E6NHj8bMmTPx/fffo3379vj+++8RGRmJxMTEattGAK6R3FdeeQXdunXDyZMnUVhYuWGkxWLB33//ja5du9aZT7du3XDixAn3a9yhQwfk5ORg7ty5Pp1y3Nx52sAjVKV0j57lFnGqIhE1T/oGNrkn/1AoBFya7ppN5IupitIURW2oMqBdMaU1Y2VGNvBoLNk/tU2bNvnsyXNycgCgxghIYmKi+75zTZ8+vc54JSUlMBqNSE5OrjNebm5ujedTq9WIiYlBdna27O+B6ldUVIQ333wTGo0GaWlpOH78OA4dOoS77roLgiBg4sSJeO2119CmTRsMGzYMf//9N1544QVceumldU5TPZcgCBg5ciQ+/vhj92jqoEGDIIoifvjhB6xcubLOx4aFheHYsWPQ6/U++X6XLVuG9evX4+2334ZKpaq2QbpUeH799dd48cUXkZqair179yIrKwsDBw4EADzwwANYsGABOnXqhIEDB+KXX37BTz/9hBUrVqBjx4549913MW3aNMyYMQNqtRpvvvkmjEYjbr311jpzmjhxIqZNm4Y33ngDV199NXJycvDMM8+gXbt2SEhI8HjEsKUzyPiUOCU+HEUlZmQXlKN7qs7fqRER+RzXjAXPpemp+OiHw9h3tAA5heVIrphx4Y2qnRQDOW0+QhoZ4zTFRvO6hC4vL8eePXtQXFyMuLg49O/fv8b0qIZITTPOvegODQ11d7aTQ2qnXls8ac2PyWSq9SK/6jHeEEURRmPtn5JbLBY4nU44HA6/X/hWXXflz+eS4kst6Ks+lyiK7vsffPBBWK1WvPjiiygoKEB8fDxuu+02TJgwAQ6HA3fffTfUajXWrFmD2bNnIz4+HjfffDMeeeQRWfmPGDECn376KYYMGeLuojho0CD8/fffGDRoEBwOR7VpflLse+65BytXrsSxY8fwn//8BwDcP6uqarvt3NcDAD755BOYzWZ3+/yqfvjhB8ycORPz5s3DjBkzYDAY0LZtWzz66KMYO3YsHA4Hbr/9dphMJixcuBD5+fno2LEjXn31VQwePBgWiwWrVq3CvHnzcM899wBwFZ0ffPAB2rRp4/4ez/3ZX3bZZXj11VexbNkyd7OQSy65BI899li116Xq46Sfrclk8nibgbpIv+f+aJLjq9hOp+iepqhVVf4u1xU/Ptp1DjmVY4DR6N12Hv58Xfwdn7kHJz5zD078lpq7NBtAq0ad1y+Nid9YLfV1B4AIDdC3Sxz2HS3Ed1uO4dZL657d0lD8s3mu6+XYyFCvf451xa5PiOC6Nig1Wjx+3ub8M5VLzppyQZS52ZQoiliwYAHee++9avsUabVaTJ48GRMmTPA41vfff48pU6Zg79691Qq5qVOnwmq1YunSpfU+ftSoUbj++uvdbcP1ej0uuOACLFu2DCNGjHAf98EHH2DBggXYuXMnXnzxRezbtw9r166tFmvo0KGYNGmS+yJXjv379ze4Z1NISAjat2+P0FB+AkVNn8ViwenTp2G3t4654KUmB179IhuCADx7a1soFPWfQDfvL8HP+0swoHMYxl3AvRWJqHkRRREvfnwWTicw/bpkRIdz0+dA+/ukEZ9uKUKkVonp1yU3+HenLr/sL8Ev+0swqEs4rj0/cDM1TuZZ8O6P+YiLDMEj1yQ3/IBWSK1Wo2/fvg0eJ/u3b+nSpVixYgXGjx+PMWPGIC4uDoWFhe69pKKiojzef0yaLpiXl1etG19eXp57Q145YmJiEBYWhry8vGq35+XluRtzJCcn48cff6x2v9VqhcFgQGJiouznlKhUqjrX7VgsFmRlZSE0NFT26KFcUifC0FD/bOLoz/jMPTjx64odEhKC1NTURn+AYDKZkJmZiY4dO7q7b/qKr2KfyCoBkI2ocDV69+7VYPxCazZ+3v83LE41evbsGdTcgxGfuQcnPnMPTvyWmHup0Qqn8ywAYPCA3ggJkd1CoN74vtASX/equnZz4vvdv6LUaINVlYCB3T1vqFU1vuLwCQAl6NIhCT17NtyB2he5A0C4rhT4MR92p8Ljv4PN+Wcq19GjRz0+VnYxtnbtWkyaNAlTp05139apUycMHjwYYWFhePfddz0uxtLS0hAREYGtW7e6i7GSkhIcPHgQ48ePl5saBEHAoEGDsG3bNtx8883u27du3YrBgwcDANLT0zF//nycPHnS3QJ827ZtAIDzzjtP9nNWfe5zmzVIFAoFFAoFlEql3zocSqRpZoIg+OW5/Bn/3Njr16/HM888U+9j7r33XkyZMsWr+HJde+21de6BJw0w//nnn345AQTydQcApVIJhUIBrVbrsw8QtFptnb8jwY5tspUCAOKiao9zbvzUNq5PH/P05kZ/T/58Xfwdn7kHJz5zD078lpR7folr1kNkmBpRURE+j+9LLel1P9cl57XH178dx+Y9uRg2oEOdx9UX31DmWrPVJiHKp99HQ7nH6Vwf3pab7dBq5a1Xa84/U0/JeT1kF2N6vb7OouX888/H+++/73EstVqN8ePHY/78+YiNjUXbtm0xb948JCcnY8yYMXA4HCgqKkJkZKTHF4T33nsvHnjgAfTq1QsXX3wxPvvsMxw6dAgvvfQSAKB///4YNGgQpk+fjueeew5GoxGzZs3CuHHjAtrWnho2YsQIfPnll/UeU9v2Bv7y1ltvwWarfaGq1JZfpVIFLB/yHfeGz1GejQJKGz8XFpthsTl8sk8MEVGgGKROih6e88g/Lju/A77+7Ti2HciBodSCGC+aqeQbXOu1AtnWHqjspmh3OGGxOaBRc6qrt2S/chdccAG+/vprXHTRRTXu27x5s+zRpSlTpsBut2PmzJkwm81IT0/HihUroFKpcObMGYwePRqzZ8/GDTfc4FG8iy66CC+//DKWLFmC1157DV27dsVbb73lbmUvCALeeOMNPP/887j77rsRGhqKK664Ak8//bSsvMn/wsPDER7ufYchX2vTpk2d9zkcDpjNZm4A3EwVlXq24bMkKlwNbWgITBY78oqMaJ8U2fCDiIiaiCJ2UmwSOqZEoXtqDI6cMuDnnadx/Uh5jTxEUazWTTGQtKEhUCgEOJ0iyk02FmONIPuVu/baa/H888/j/vvvx7XXXoukpCTo9Xr8+OOP2LBhA6ZOnVptNGPcuHH1xlMqlZgxYwZmzJhR47527drh8OHDdT62rjb748aNq/d54+LisGjRonrzIqLWQ9pvx9NPJQVBQHJcGE5klSC7sJzFGBE1K+49xrjhc9BdNqQDjpwyYOPWkxg3oousD3WNZjtMFtdSg0AXY4IgIFyjQqnRinKTDXHRwV2j1ZzJLsYeffRRAMCWLVuwZcuWGvcvWLDA/f+CIDRYjLUmMhtXEgVNa3uvSi2ePR0ZA4DkuHCcyCpBTgE3ziai5qVyjzEWY8F28cC2eOfrv3EmrwwZmXr07OR5h97CYtfPMTJMHZSRqQitVIy1js7L/iL7J/fTTz/5I48WTVpHZDQag97dhcgT0p4hrWUNnKFU/qfE0rqxnKLG7+tCRBRI7pExTlMMujCNCsP6tcGmHafxw7aT8oqxig8SEwI8KiaR1o2Vm7nxc2N4VIzZbDb3RVnbtm09Dl5Xs4PWRqlUIiYmxt1yPywszG9ri6RGEtLzNqf4zD048avGVigUMBqNyMvLQ0xMjN+7fzYV7pExGZ8SJ8e7irFsjowRUTOj55qxJmXM+R2wacdp/LbnLCZc1wdhGs8+CC0odv3tDvQURUm41pVnmbH+vXapfh4VY9dccw2eeOIJjBo1yuPA0r5j33//vdfJtSTJya4N8c7dA83XnE4n7HY7QkJCoFB4t29IsOIz9+DEry12TEyM+z3b0omiKLubIgAkx7ra5uYUshgjouZF78VsAPKfXp1i0TYhAmfzy/DbnixcfoFnbe4LDa6/XfExwfk5SsVYuYmDL43hUTE2d+5cPPXUU1i4cCHGjh2LMWPGuPfoquqff/7B5s2bsXbtWjidTsydO9fnCTdXgiAgJSUFiYmJfh0xNJlMOH78OFJTU/22oZ6/4jP34MQ/N7ZKpWo1I2KAa48Uq90JQOY0xYqRsdwiI5xOEQoFO2kSUfPg/gCKI2NNgiAIuGxIKlZ9exA/bDvpcTFWUCwVY8EZGYvQqgEAZZym2CgeFWP9+vXDl19+iQ8++ACrVq3CggULEBUVhbZt20Kr1aKkpAS5ubkoLS1FbGwsJkyYgDvuuAOhofwlP5e/N352Ol0XlaGhoT7brDdQ8Zl7cOL7O/emTrooCdeEyNovLCFGC4VCgM3uRFGJOWh/DImI5LDZHSirGMngyFjTMWpwe7z/3SEcPqnHqZwSpCY3vI+qe82YLjgbHFeOjLGBR2N43MBDrVbj3nvvxfjx4/HXX39h69atOH36NMrKypCcnIxLLrkEw4YNw+DBg1vVp+pE1Ly5107IvChRKhVI1GmRU2hEdmE5izEiahakKYohSgUitK2jSVNzoIvSYEivJPz1dw5+2HYK91/bp8HHSN0Ug97Ag9MUG0V2N0WVSoXhw4dj+PDh/siHiCigiiq6islpay9JjgtHTqERuYXl6Nsl3tepERH5XNU1sv5qJkbeuez8Dvjr7xxs2nEad13VC6qQuteIi6Lo/vsVtGmKFY1Gykxs4NEYvu80QETUjEgXJp5u+FyV1N4+u5Dt7YmoeXA37+B6sSbnvB6JiI0KRUm5FdsO5NR7bLnFCZvdCUEA4qLZwKM5YzFGRK2aNxs+S5LjKjoqsr09ETUTlcUY14s1NUqlAqPTUwEAG7edrPfY4nIHAFdRHaIMzuV8RJirgQeLscZhMUZErZqhERcmye6Nn1mMEVHzUDlNkcVYU3TpEFcxtvtwHvL1pjqPKzG6irFgrlcO17CBhy+wGCOiVq1yZMyLaYrujZ85TZGImgdOU2za2sRHoG+XeIgi8NOOU3UeV2x0FUAJMcHppAhUNvAo48hYo8guxp599lns3bvXH7kQEQWcu5uiFyNjSRUbP5carZymQUTNAkfGmr7LzneNjv2w7RScTrHWY5rEyJi0ZsxsgyjWnic1THYx9vXXX6O8nFNyiKhlkLpR6bwYGQvTqBAd4Zozn13I8yIRNX2VH0BxZKypurBfG4RrQpBXZMS+o/m1HiOtGWsKxZjTKcJk4VRFb8kuxgYOHIitW7f6IxciooCy2hzuES1vGngAlevGctlRkYiaAU5TbPpCVUpcPKgdAOCHrbVPVZRGxoK1xxjgyjNE6doegevGvCd7n7EePXpgxYoV2LBhA9LS0hAWVn2uqiAIePnll32WIBGRv0gXJaoQhfsTPrlS4sJx+KSeI2NE1OSJogi9ezYApyk2ZWPO74Dv/sjEn39no9RoRWRF50JJsXuaYvB+joIgIEKrhqHMgnKzDQkIXmHYnMkuxn744QckJibCZrNh//79Ne7nBoJE1FxUXTvh7bkrSWpvz2KMiJq4MpMNdocTAEfGmrqu7WLQuU00jmcV45edZ3DN8M7u+xwOJ0pNFSNjuuA18ABcTTwMZRaum24E2cXYpk2b/JEHEVHASZ0UG3NRIm38zGKsefjkp6P4a38eXuzYFWHBvYYhCjjpA6gIrQqqEGWQs6GGXHZ+Kt7+Yj82bj2JsRd1cn9oqC+zQhQBpVJATERwi2ppVkmZ0RrUPJozr1vbl5SU4Ndff8W3336LP//8E2VlZb7Mi4jI76Rpit6uFwMq14xlc81Yk7ftYA4+++UEzhZa8ffxomCnQxRw7vVinKLYLIwc1A6qEAUys0tw9IzBfXthccWWLJGhUCiCOyPNvdeYmSNj3pI9MgYAy5Ytw5IlS2A2m923qdVqTJo0CZMnT/ZZckRE/qT3wchYcsU0xQK9ETa7E6oQbt/YFJUarXjjkz3ur1k8U2vE5h3NS0SYGkP7puDX3Wfxw9ZT6NZeB6CyGIuLDn5R7R4Z4zRFr8m+avjss8+wYMECjB07Fu+//z7Wr1+P9957D2PHjsUbb7yBL774wh95EhH5XOWGz97/QYuN0kCtUsIpAvkGXuA3VW9/vt99IQoAOSzGqBWq/AAq+Bfx5Jkx53cAAGzefQZmq6tjYVMqxiIqGouwm6L3ZI+MrVq1Crfffjv+7//+z31b586dcf7550Oj0eD999/H9ddf79Mkiaj52nU4H6XFNvQMdiK1kC7OYxpxYSIIApLjwnAqpxQ5BUa0iY/wVXrkI3/sy8Lm3WegEIDLL0jFd3+eQk4RizFqfSqnKXJkrLno2yUeSbFhyC0y4o99WRg1OBUFFcVYfBMoxsI1rlKCDTy8J3tk7OTJk7j00ktrvW/06NE4fvx4o5MiopbhyCk9XlmzB5/+XhjsVGpVOTLWuAuT5Fhp3RibeDQ1xWUWLPlsLwDgxlHdcFG/ZAAcGaPWqXLD5+BfxJNnFAoBlw1JBQBsrNhzrCmNjFVOU2QDD2/JLsaSkpKQlZVV631nzpxBRAQ/FSYil/1HCwAA+SV22O3OIGdTk6G0srV9YyTHs719UySKIpZ+tg/FZVZ0TInC7WN6uNf46UutMFk4rYaCK6/IiP+u2ol/skwBeb7K7Tw4MtacjE5PhUIADhwvRFZ+mbsYawojYxEVxRhHxrwnuxgbNWoUFi5ciH379lW7fe/evVi8eDFGjRrls+SIqHk7lOnqWCeKQJ4+MBcbnnI4RRh80E0RYHv7puq3PWexZV8WlAoB024bCFWIEhFaFbShrj99/HlRsG34KxP7jxXhy7/0MAfgwwF3B1mOjDUr8TFaDEpLAgD8sO0UCotdP8emNDLGNWPek12MPfLII9DpdLj11ltx6aWX4rbbbnP/NyYmBo899pg/8iSiZkYURWScrGwfnt3E1uiUlFngFAFBAKLD1Y2KlewuxprW99ia6UvMeOtz14eGt17aHV3axbjvi41wrXHIymcxRsF15JQeAFBuduLbP075/fn0JRXrZDky1uxIUxV/3HYKxeWuKYFx0cH/OUZopQYeHBnzluxiLCIiAp9++imeffZZ9O3bF2FhYejbty+effZZfPrpp4iJifFDmkTU3GQXlqO4rHIOeVMrVKT1YtERoVAqG9eOXpr6llNYDlEUG50bNY4oinjz070oNdrQuW00br60e7X74yIrirEC7o9JweNwijhyyuD+et2WTBSXWep+QCPZ7E6UVmzMyzVjzU96r2RER6hhqHiPhCgF9xTBYArXus6nZdxnzGuyuynef//9mDBhAu644w7ccccd/siJiFqAQyeqb6rb1IoxX07XSYoNgyAAZqsDhjILL3SC7Oedp7H1QA5ClAKm3z4IIecU27EVxVh2AUfGKHjO5JbCZLEjVK2ELlyBHL0Nn/x4BBPH9fXL80nTskOUAiLDgn8RT/KoQhS45Lz2+HLzMQBAdJgSghDcDZ+BqtMUWYx5S/bHwbt27WoSP3wiatqk9WLSFMCcoqa1ZsyXC9lVIUrEx2gBADkFTavobG0Ki01Y9sV+AMDtY9LQMSWqxjGx7pExFmMUPIcrpih2aRuFywZEAwDW/3HCb2sZpU6KMZEaXsc1U9KeYwAQHa4MYiaVpGLMaLbB6eTMEG/ILsaGDx+Or7/+GjYbK2AiqltGRTF2Uf+m2Uq8qLTxGz5XJbW3zyniBX6wiKKIRZ/sQbnZju6pMbjxkq61HueeppjPaYoUPIdPuoqxbu2i0SVFg75dYmF3iPjg+wy/PF/lhs/BX2dE3mmfFImeHWMBAFFhTaMYk6ZKiiJgZIdar8iephgaGoqvv/4a3333Hbp06YKwsLBq9wuCgPfee89nCRJR81NmsuFUbikA4OIBbfDtH6eQbzDBZndCFdK49Vm+4l7I7qMLk+S4MOw/BuRwtCVoNm49hV0ZeVCFKDDttkF1rgWURsb0pRaYLHZoQ2X/KSRqNKl5R7d20QAKccdl3fD0sa3YvOsMrh/RFZ3bRvv0+dwbPnMadbP2r8vTsPDjXeidGtbwwQGgClFCrVLCanOg3GRrEuvYmhvZV0U5OTkYOHAg+vTpA61WC1EUq/1zOpveXkJEFFhHTuohiq6W7x2SI6AKESCKQG4TGjWq3PDZNxcmKfHc+DmY8oqMWPH13wCA8Vf0RPukyDqP1aoV7jUzXDdGwWA023AypwQA0LW9q+jq3DYKwwe0hSgC760/6PPndBdj7KTYrPXvnoA3HhuObm2aTlEdUdHEg+vGvCP748DnnnsOXbp08UcuRNRCSOvF0jrqIAgCYiNCkGuwIbugHO0S675IDiSD+8LEx9MUm9h0zNbA6RSx6JPdMFns6NkxFteNaPhvVHJcGEqNxcgqKPP5CARRQ46eMUAUgQSdFrrIUORU3D7+yjT8sS8LuzLysO9oPvp1TfDZc1ZOU2w6F/HUMoRrVSgqsbAY85LskbE77rgDX375pR9SIaKW4lBmIQCgZ6c4AFVbiTedUQj3yJiPLkyS4yvb21NgffdnJvb+UwC1Solptw2EUtFwc4LkWNfPi3uNUTBI68W6p+qq3d4mPgKXX+Bq0vDetwd9ulWG1MCDI2Pka+Ea10yDMpO1gSOpNrKLMZVKBZ1O1/CBRNQqORxO91oIaaFxU2slLoqiT7spAq4pmYBrKpCZi5gDJqewHO9+cwAAcPfVPdEmIcKjx6VU7A3XVN6T1LpIxVhah5rXU7dd1gMatRJHThnwx/5snz0n14yRv7C9fePInqY4depUzJ07F6WlpUhLS6vRwAMA2rRp45PkiKj5OZlTCpPFgTBNCNonRcJiNlW2Em8i3evKzXZY7a71rb6aphgRpka4VoVykw05RcZaW6qTbzmdIl7/aDcsVgf6donH2GGdPX5scpxrKwJu/EyBJoqiu639uSNjgOucdN2ILvj4hyNYvf4gzu+dXGOvPG/4+gMoIkmE1rWFTZmJH0R6w6s1Yw6HAzNmzKjzmEOHDjUqKSJqvqT1Yj1Sde7pYu6RsSYyhU+6KAnXhCBU5bv2wClxYTh6phg5heUsxgJg3e/HceB4ITRqJabcOgAKD6YnSpIrRsaa0tRZah3y9SYYSi1QKgR0aRcDh81S45gbRnbFd39k4mx+OX7YdgpXDu3YqOcURZEjY+Q34Wzg0Siyi7H//ve//siDiFoIaX8xaYoiULlmLK/I2CTa21eunfDtRUlyXLi7GCP/Optfhve/dXWcu++a3kiumCbqKWnNmKHUAqPZhjAN2zFTYEhTFDu1jUaoSgljLdevYRoVbr2sO5Z/+Tc+2piBSwa1g6YRWzAYzXbYpNkA3GeMfMw9TdHMYswbsn+zr7/+en/kQUQtxEGpGOtUWYxFaBQIVSthsTqQpzeirYfrevylqMQ/nxBLBQHXIfmXwyni9f/tgtXuxIBuCbjCi1GDcK0KUeFqlJRbkV1Qji7tYnyeJ1FtpCmKPWqZoljVlUM74qtfjyOvyIivfzuOWy7t7vVzGspcjRXCtSqofTgbgAio3Pi5zMgGHt7w6uNpq9WKDz/8EA8//DBuvfVWHDt2DP/73/+wb98+X+dHRM1IYbEJeUVGKITqayEEQUBybMUanSawbsxfayekYiyniO3t/enLX44i46QeYZoQPHLrAAiC59MTq2pTsTccpypSIB0+WTGVu5bmHVWpQpS484o0AMBnP/+DknLvL3QrpyhyVIx8r7KBB9eMeUN2MVZUVIQbb7wRL730Ek6ePIl9+/bBbDbjl19+wZ133ondu3f7I0+iJm37oTys+CEPBcXmYKcSVBkV0286pETVmPYlTQtrCqNGvt7wWZIitbdvAt9jS3UqpwRrNmQAACZc2weJuppNpDwldV5sCu9Jah1sdieOnS0G0PDIGABcPLAdOrWJgtFsx9qfjnj9vMVlrmLM1+c8IqCygQenKXpHdjE2d+5clJeXY/369fjiiy/ce2AsWrQIffv2xaJFi3yeJFFTJooiPtz4D07nW/HbHt+1IW6OMtybPcfWuK8pNUww+Gkhu7Txc57eCIfTd/sDkYvD4cRrH+2G3eHE4J5JuHRIaqPipbhHxoI/Wkutw4msYtjsTkSGqdzvv/ooFALuvroXAOCb308gz8tRd2maYgxHxsgP2MCjcWQXYz///DOmTp2KDh06VJsaEhoaivvuuw8HDhyQFc/pdGLRokUYPnw4BgwYgIkTJ+L06dN1Hq/X6/HYY48hPT0dQ4YMwfPPPw+TyeS+v0ePHnX+y8rKAgDs3Lmz1vu3bt0q89UgAk7lliKrwPUHsrVPTztUS/MOSXIT2tepcmTMtxcmcTFahCgF2B0iCg2mhh9Asny66R8cPW1AuFaFh2/u7/X0RIl7miI3fqYAqbrZs6fv30E9EtGvazzsDic++D7Dq+dlJ0XyJ2maYhmLMa/IbuBhsVgQExNT631KpRI2m7wfxJIlS/Dhhx9izpw5SE5Oxrx58zBhwgSsW7cOarW6xvFTpkyByWTCqlWrUFJSgmeeeQZGoxGvvPIKAOD333+vdnxxcTHGjx+PESNGuPc/O3z4MFJTU/Hhhx9WOzY6OlpW7kQA8MfeLPf/Zxe03mLMYnPg2BkDgPqLsaYwCuHupujjCxOlQkBSbBjO5pcju7AcibHeT6Gj6k5kFeOjHw4DACZd3xdx0dpGx2wTXzFNkd0vKUCOSM07OtQ8R9ZFEFyjY48t/BU/7zyN60d2lb11hqHUNTLm6w+giICqa8bYwMMbskfG+vbtW6OIkaxbtw59+vTxOJbVasXKlSsxZcoUjBw5EmlpaXjttdeQk5ODjRs31jh+9+7d2LZtG1555RX07t0bQ4cOxQsvvICvvvoKubm5AICEhIRq/xYtWgSdTocXX3zRHefIkSPo2rVrjWNrK/6IGrJlX5VirBVf1B09bYDdIUIXGYqkWooQqYGH1N4+mNzdFP1wYeJu4tGK3wu+ZrM78dr/dsHuEHFBn2SMHNTOJ3GlaWJSe3sif5NGxjxZL1ZV91QdhvVrA1EE3qvY0kEOac1YDEfGyA/CK9aImywOOBzB/fveHMkuxqZOnYotW7bguuuuw8KFCyEIAr755hv8+9//xoYNGzB58mSPY2VkZKC8vBxDhw513xYVFYVevXph+/btNY7fsWMHEhIS0KVLF/dtQ4YMgSAI2LlzZ43jf//9d2zcuBEvvvhitULr8OHD1WIQeet0bilO5pS6NzcuKbe12tauVdeL1Tb9RhcZilC1Ek7RtaYqWKw2h3teuz8Ws1cWY613lNTXPv7xME5klSAyTI2Hbmr89ERJuFaF6AjX34amsJaRWrbiMov7A7vuqTGyH3/nVT2hUAjYcSgXfx8rkPVYfcXIGLspkj9II2MAUG5mR0W5ZBdjgwcPxrvvvgutVot33nkHoihi1apVyM/Px9tvv40LLrjA41g5OTkAgJSUlGq3JyYmuu+rKjc3t8axarUaMTExyM6u2ThhwYIFGD16NAYPHlzt9n/++QfHjx/HDTfcgGHDhuHee+9lW37yyh/7XaNifbvEIlLr+nVqrRd19a0XA1xTbVKawD5c0toJVYii2h8QX3HvNcaRMZ84etqAtT/9AwB48MZ+Pp9a6p6q2Ep/bylwpCmKbRMiEBEmfyZO24QIjDm/AwBg1bcH3Q3UPGFgN0XyoxClAtpQ1/51bOIhn1fbuaenp+Ojjz6C2WxGcXExIiIiEB7ecFegc0mNN86dHhgaGori4uJaj69tKmFoaCgsFku127Zv344DBw5Um54IANnZ2SgtLYXRaMTMmTOhVCqxZs0ajB8/Hp9//jm6du0q+/sAXB31jMbgfxIuvaZVm5o0l/jNMfffd58FAAzqroOhuAylJgtOnClCu3jffvrY1F93URRxKLMQANApOaza70LV+Ek6DTKzS5CZpUevDpGNyLpmbE9l5xkAADER6gYf50382AjXH6Ss/NJ6zwnN8f0eiNhV45aUluPVD/fC6RQxtE8Szuuua/R59tzcE3WhOJQJnMzS47zu8qaOeRLfl/ieCU58X8U+cCwPANC1XWSd58iGjBueip93nMLhk3r8uvMk0nsl1nu8yWSCwymi1Oi6QA4Ncfr0WqU5vO7BiN8acw8LDYHJ4kChvhTRYbXPXmjOr4tcoih6PIvDq2JMotFooNF4/ymL9Fir1VotjsVigVZbc3G2RqOB1VpzCpjFYkFYWPU1Kl988QX69euH3r17V7s9JSUF27dvh1arhUrl+lS8b9++OHjwIFavXo3nn3/eq+/FZrPh0KFDXj3WHzIzM5tt/OaSe2GpHZk5pRAEIF5TjrioEGTmWbD/8EnEhxp88hznaqqve2GpHSXlNigVgKUkC4cO1RypzszMRAhcJ8hDR8+gU4zvRiLk5H7wtCuHUKXT499ZOfHLDK6Lnuz8Mo/iN5f3e6BjA8B73+zHmbxyhGsUuKi7wqfnWCl3hcP1Psw4no1DSZZ6HuFdfH/geyY48Rsbe3dGPgAgIsRU63vZ0/hDuofjtwOlWPXtAWjFAvc0+bqUmR0AAIUAnD55FAofTfOtqim/7sGM35pyVwqu99nBI8dgKam/NmjOr4scnvaiaFQx1ljSlMO8vDykplbuF5OXl4cePXrUOD45ORk//vhjtdusVisMBgMSEys/HXI6ndi0aRMeeuihWp83Kqp6FyKFQoEuXbq4m4B4Q6VSeT2q5ksmkwmZmZno2LFjrQVtU47f3HL/6tcTAIC+nWPRq0cX7D6+HwBgE8LQs2fPRsevqqm/7pt3ZwHIQdd2Mejbp1ed8fsYi7Dl4EFYnBqfvEbe5H6q5DSAQqQkxjSYgzfxO1sdWLo+F2abiPYduiIirPapkM3t/R6o2FL833YcxpZDpQCAB6/vi8ENjADIiV01d709Bz/v2w+TQxW092RTiO3v+K09d6dTRM7nruUXwwf3QKc2ldchcuN36GTDnhNbUFBiQ64pCqMH193QxmQyYctO12bRushQ9O7Vq85jvdHUX/dgxW+NucduKUNesQHxCW3Qs2eST2N7yt/x5Th69KjHxwa1GEtLS0NERAS2bt3qLsZKSkpw8OBBjB8/vsbx6enpmD9/Pk6ePIkOHVzzprdt2wYAOO+889zHHT16FHq9HhdeeGGNGL/++iumTp2Kr7/+Gu3btwcA2O12ZGRkYMyYMV5/L4Ig1BidCyatVuvXfPwZv7nkvi3DtYB6+MD20Gq1iIt0/Trl6s3N8rVpTOxjWa529b07x9cZQ6vVokMb1zSwPB+/RnJyLze7Oj0l6MI8foyc+GFhrvbRRSUWGIxOJMbX/7jm8n4PZGyrzYEv/9JDFIGR57XDiMEdff4cUu4d27rWOOYVBe892ZRi+zt+a839dG4pjGY71Col0jolQqmsuWTf0/hhYcAtl/bAiq//xme/nMBlF3SGRl335Zw0MqaLbpqvTTBj+zt+a8o9Mty1PMPmVDT4uOb8unhKTqMp2Q08fEmtVmP8+PGYP38+fvrpJ2RkZGD69OlITk7GmDFj4HA4kJ+fD7PZtSdQ//79MWjQIEyfPh379u3DX3/9hVmzZmHcuHFISqqswg8ePAiVSoXOnTvXeM5BgwZBp9PhySefxN9//43Dhw/jySefhMFgwD333BOob52audwiI46eNkAhABf0cY3wSsVYVn6ZrIXVLUHVTor1kTbZzdUbYQ9S+1tpw2edHxeyJ8WyvX1jfPTjURSW2KGLVGPSuL5+fS7pPWkoY3t78h+ppX239jG1FmJyXT2sIxJ1WhQWm/HN7yfqPbbMVFGMsZMi+VGEe68xnkflCmoxBrg2cb7pppswc+ZM3H777VAqlVixYgVUKhWys7Nx0UUXYf369QBcVeYbb7yBdu3a4e6778a0adNw8cUX47nnnqsWMz8/H9HR0VAoan57ERERWLVqFeLj43H//ffj1ltvhcFgwJo1axAfHx+Ib5lagD8q9hbr0yUeMRV/4HQRIRAE1z4bUse+1qDMZMOpXNd0srSO9TdAiI3SuNrbO0XkFQWn4Y30s/F1V76qpP2r2N5evszsEqz/8xQA4IHrennVdU6OMI0KMRGu3+GsfBbP5B+HT3m3v1hdVCFK/OuKNADAp5v+QWk9W6qUmVwffPnznEfk3viZH2rJ5tE0xS+//FJW0HHjxnl8rFKpxIwZMzBjxowa97Vr1w6HDx+udltcXBwWLVpUb8yJEydi4sSJdd6fmpraYAyi+mzZ6yrGLuzXxn1biFJAok6L3CITzuaXtZoWwkdOuqaTpcSFN/jHXmpvn5ldgqyCcrRJiAhQlpWkkbFYP2z4LOHGz97b908+RBHokhKKQT0SAvKcKfHhMJRZkF1Qjq7tYwLynNS6HKkYGevewTfFGACMGNQeX/xyDJnZJfj0p39w7zW9az3OPU3Rj+c8IqkYa617rTaGR8XYU089Ve1raR5k1alYVedGyinGiJqbPL0Rh0/pIQjA0L7V971LiQtDbpEJWfll6NuldYy0Hqxoad/QqJgkJV4qxsoA1L7I158Mpf6fppgc55qrzr3G5Due5drWpF2cf0fEqkqJD8ehzKKK9ySRb5ktdmRmu97XaT4sxpQKAXdd1RMvrNiKdb8fx9iLOiNBV7NpQal7mmLr+ICQgqNymiI3fZbLo2mKP/30k/vfG2+8AY1Gg0cffRQ//vgj9u3bh82bN2PWrFnQ6XR46623/J0zUVD9ud/Vtr1Xp7gao1/S9LSzrWi6k7RerGenOI+Ol9boZAfhNXI4RRhK/b/5aUocpyl668TZEgBAsi5wxVibBNfPq7Vu2E7+dfSMAU4RiIvWIC7atx3eBvdMQu/OcbDZnfjfxoxajykzS9MUOTJG/hOu4TRFb3lUjLVt29b9b8mSJXjooYcwceJEtGvXDmq1GklJSbj99tsxadIkzJs3z985EwVV5RTFlBr3pVSMiGTlt45P2B0OJ45UrIXo2UDzDklKvGtqYjAufEvKLHCKgCAA0eH+u9iXpikWFptgszv89jwtjc3uxKlcVzGWElv7lgD+0KbiPZnNYoz8QGre0cOHo2ISQRBwz1hXu/qftp/CyZySGsdIDTxay9R5Cg5pGxc28JBPdgOPY8eOoVcd+1R07twZZ86caXRSRE1VYbEJhypGgi7s26bG/SkVbczPtpJi7GROKUwWB8I0IWifFOnRY6RRiGBc+ErrxaIjQn3S0awu0RFqaEOVEEWOjslxOrcUdoeIcE0IosOUAXteaUSb0xTJH3zdvONcaR1iMbRvCpwisHp99c2kRVF0j4zFcGSM/Mi9ZozFmGyyr0Y6duyIdevW1Xrfxx9/jO7duzc6KaKmSpqimNZBh/iYmtNN2lSMjOUUlsMRpNbtgSQVpj1SdVAqPNtTI5jt7aVOirF+XjshCIK7vX1ukLpGNkfHz7rW1XRIiZS1R0tjSe/J4jIrP9UlnxJFEYdPVpwnO3g2e8Abd17ZEwoB2HogBwdPFLpvN1nssDtc6/v9uU6WyD1NkedQ2WRv+jx58mRMnToVmZmZuOSSS6DT6VBQUICNGzfi6NGjWL58uT/yJGoStlS0tB/Wv+aoGOCaBqIOUcBqdyJPb3J/4t5SudeLeThFEah4jVRKWG0O5BUZA9pRUe/eY8z/nxBLjUo49c1zUvOOjsmRAAJXqIdpVIiJDIWhlB0VybcKDGYUlVigUAjo0i7ab8/TPikSl53fAd//dRKrvjmIVx6+CIIgwFDq6mwXpglBqCpwo83U+nBkzHuyR8bGjBmDN998EzabDa+//jpmzZqFN954A+Hh4Vi1ahWGDh3qjzyJgk5fasaB465PHGuboggACoXQqqY8HXQ37/C8GBMEwT0SEeh1Y0VSJ8UAdBVje3v5pJGxjimeTXn1pTat6PeWAkdaU9sxJQoatezPv2W5fUwPqEMUOJRZhO0HcwG4NjMHgJiIwDXEodZJ6qZotTm4Vlomr84Mo0aNwqhRo2CxWFBcXIyYmBio1fxFp5btr/3ZEEWgW/sYJMaG1Xlcm4QInMwpxdn8MpyXFvjW7YFSWGxCXpERCgHoLnMtRLDa2+tLKjZ8DsDIWLJ7yiqnKXpCFEWcyKosxoz64oA+f0p8OA6eKGJHRfKpjJOVU7n9LS5ai2sv7oJPN/2DVd8exHk9k6CvGBmTNjYn8pcwjQqCAIiiq719TCRHYj3l9Qr2Y8eO4eOPP8bq1auh1+uxY8cOlJXxE0VqudxTFPvVPiomaVsx7S6rhbe3z6joENYhJQphGnmd74LV3r5yw+fAjYxxrzHP5BYZYTTbEaJUoG1C4Kf3Sh0VW0snVAoMaWTMH50Ua3PjqG6I0KpwOrcUP+84VTkyFskPzMm/FAoBYaGuMR62t5dH9siY0+nErFmz8Nlnn0EURQiCgCuuuAJLlizBqVOnsGbNGiQnJ/sjV6KgKS6zYP+xiimKDRZj0l5jLfuiTlovliZjvZjE3d4+wIWKtMdYIBayS3uN5RaWw+kUofCwwUlrVbV5R4gfO13WJZhdPqllsjucOHr6/9k77/Aoqi6Mv7ubTe+9QAIJJYWSBAggvSMgBBQRpArqZ0NAUEFBQQRUigJKURARFBQL0osISoeETkghpPdedrP1fn9sZkhIAltmdjfJ/T1PHsjs5OzJZHZmzj3nvKcEgPGCMXsbMcYPaofvD97B7qP30C3EAwDNjFGMg52NGJVVSirioSM63/G++eYbHDhwAMuXL8e5c+dAiEalZ8GCBVCr1Vi3bh3nTlIopubi7Ryo1QSBfk5PFOXw9WgeK+xxeoh3MLAPvibKjBlj+KmHiw2EQgHkSjWKq3vVKA3DiHcE+vIncvA4mOCZlilSuCIluwxypRp2NmI282oMRvVuDXdnGxSUVuFUTCYA2jNGMQ6siIeEBmO6oHMw9ttvv2H27Nl49tln4ezszG4PCQnB7Nmzce7cOS79o1DMgvNaligCD8sU80ukkCmaZhOrTKHC/YwSAHoGYyaQtyeEsGqKxihTtBAJ4VE9/oD2jT2ZB5maYbWtTRWMVZ+TZZVyqgbWyEhML8bsdWdxNcm8FsDYYc/+LkbNjFuKRXhxWHsAgExBZ4xRjAcTjNHMmG7oHIwVFBQgJCSk3te8vLxQVlZ3+juF0pgpl8hxIzEfQMOS9jVxtLOEnY1YM/C3ia6yJ6WXQKkicHGwgtdjxEwagpG3V6sJ8ow0h6uySgm5UvNgYqx5O0y2hZa+PZnkzBIAQKCfaYIxW2sxmzHNpoqKjQaVSo0Nv1xHbpEUZ26VQ60mpnaJhekX01XgiAsGdPWHv/dDVVIajFGMAaOoWEF7xnRC52AsICAAZ86cqfe1y5cvIyAgwGCnKBRz4tLtHKjUBK18HNms1+MQCARNvm+sZr+YPsN5TSFvz2TF7Iw4b8eLUVQsosHY4yitkKGgVPP3ae3raDI/2LEUTVx8pylx+HwKHmRpFoHLpSrEVWejzIGHw56NH4yJhAJMGxHKfk/LFCnGgGbG9EPnYGzatGnYuXMnli1bhvPnz0MgECA1NRXbt2/H9u3bMWnSJD78pFBMxpMGPdcH0zfWVIMxQ/rFGIw9j43p2zJWVgx4mBnLKaBlio8jpfph2sfNTmdlTi5hFRVpJrNRUFxehd1H4wAAbtXjKs7eyDGlSyzlEjkyq4N6U2TGAKBbqBcGdPFFkI8VWphAoZTS/KDBmH7orKY4fvx4FBUVYdOmTfj5559BCMG8efMgFosxa9YsTJw4kQ8/KRSTUClV4HpCHgDt+sUYmrK8PSGEnZ1jSDDGytsb6cG3iJkxZoSBzwze7nTwszYw4h2t/UyXFQNqKio2zUWUpsYPh+6iskqJoBZOeGFwED7dEYtLd3KhUKogtjDtjCOmRNHX3Q6OdqbJSgkEAvwvOgxxcXEQmUChlNL8sK9eTKN9t7qhczBWXl6OV199FS+++CKuXbuGkpISODo6onPnzrUEPSiUpsDluzlQqghaejmgpZfDk3+gGj/3ppsZyy6oRGmFHGILIYJa6N/f42PkLARTpmiMgc8MPnTWmFYwsvam6hdj8DFy6SxFf+6lFOHvK+kAgNfGdYKfmxUcbIQolypxNS4PPTv6mNS/hOpyyXYmKFGkUEwFzYzph85LJSNGjMDhw4dhb2+PPn364JlnnkG/fv1oIEZpkpy7ob2KYk18PIxbgmdMmBLFNi2cDVp9Nn5mzHhKigze1T1jZZVySGhDc4OYWtae4eHgZxqMmTMqNcGm328CAIZE+aN9gCuEQgE6BGg+b2euZZjSPQDAverMWLCJShQpFFNgb0uDMX3QORiTy+VwcaEXF0rTR1KlQGx8dYmiDv1iwMNAo7Si6clkc9EvBjwsCcstMo68PTvw2YhlirbWYrZEicrb149MoUJGnmbRwlwyY+USOSokcpP6QmmYoxdSkJxZCjsbMaaNfChS0bGVJhi7cifHpIsfhBCaGaM0S+ysaTCmDzoHY1OnTsWXX36Ja9euQSqV8uEThWIWXI3LhUKphp+HHQK8tS9RBDQP4a7V5XBNbfhzTSVFQ3BxqCFvX8x/oFJkgjJFgJYqPonU7DKo1QSOdpZGzVrWh42VxcPPLS1VNEtKK2T48YhGtGPK0yFwsn/4efZxEcPX3Q5ypRoXbmWbykVkFVSiQqqApYUQrXxMu8BAoRgTdugzDcZ0QudgbP/+/bhz5w4mTZqEyMhIhISE1PoKDQ19shEKpRHAqCg+1clXL/l2X1bEo+kEYxVSBdJyywEAwa0MW/EVCmvI2xuhLIxRU3Q1YmYMeChvn0uDsXp5kPWwX0yfzxnXGLuXkaIbPxy6i0qpAoF+Thjes1Wt1wQCAXp38gYAnI41XakiI2kf1MIZYgsqnEFpPtCeMf3QWcBj9OjRfPhBoZgVVTJNEzige78Yg5+HPW7fL2TljZsCCanFIEST7eGi3M/H3Q4p2WVG6Rtj1RRNlhmjZYr1wYp3mLhfjMHX3Q53kgvpoG4z5F5qEU5cTgOgEe0QCesG7706e+OXU/dxMzEfxWVVRh1lwRBfXaJoivliFIopqZkZI4SYxQJbY0DnYOzNN9/kww8Kxay4ei8XcoUK3m62evexPBQDaDqZsbsphQCAkNaGlSgy+Bpp1phcoWJX6oxdCufNzhqjD/f1wQzsbW3ifjEGY8+/o2iHSk2wuVq0Y1C3lg2WSXu72qJ9gAviU4vx3/VMjO4bZEw3AQDxaTQYozRP7KuDMaVKDblSDSuxaUdMNBZ0DsYAQCaTIT4+HnK5HIQQAIBarYZUKsXVq1cxf/58Tp2kUIxNTRVFfVd2/KoFKjKb0EMdV/1iDD5GUlQsrhbvEFsI2ZU7Y8EoKuYU0WDsUVRqwpYpBplJMMYsomQ3oYx2U+D4xRTczyiFnbUFpo8Me+y+/SJaID61GKdjM4wejFXJlewQc1MNe6ZQTIWNlQWEAkBNNKWKNBjTDp2DsUuXLuHtt99GaWlpva/b2dnRYIzSqKmSK3E1LheApl9MX2r2jDWFdL1KpWYHmRqqpMjga6T+HHbGmIOV0f8OTMCZVyyFUqWGBR2+ypJTWIkquQqWYhH7eTE1vk14LEVjpbRChp2HNaIdk58OgbPD40uNe4f74ru/biMxvQRZ+RVGPbfuZ5RCpSZwdbSCh7ON0d6XQjEHBAIB7GzEKJcoUClVmFyUqbGg81PBunXr4OLigvXr12Pw4MEYOnQoNm/ejEmTJkEgEODbb7/lw08KxWhci89DlVwFDxcbtG3prLcdbzc7CAWAVKZiMzONmdScckhlKthaW+g0APtxsIEKz/L2D5UUjX9jcHGwhqWFEGo1QX4xVaCtCdMv1srHod7+H1PA9PiVSxQop/L2ZsGPR+JQIVWgta8jnn5EtKM+XBysEd7WAwBwxshCHsyCVTt/l0a/AEeh6APbNyahIh7aonMwFh8fjzfffBNDhgzBgAEDkJ2djX79+mHx4sV47rnnsGnTJj78pFCMxrkbGklkQ0oUAU1JnJdrdaliE+gbY+aLtfd34ezB2dVRI2+v4lnevtgEA58ZhEIBq6iYQxUVa8EEY63NRLwDAKxryNtTEQ/Tk5BWjOOXUgEA/xvXCSItM8v9IlsA0AyAZtopjMFD8Q5uqgcolMYGq6howll/jQ2dgzG1Wg0vLy8AQEBAABITE9nXhg0bhrt373LnHYViZOQKFS7fzQGgv4piTdiSp6YQjD3gZthzTYRCAXyqAxU+H3yL2YHPxlVSZGBFPGgwVotkM+sXY/BpguI7jRGVmmDT7zdBCDCwa0uEtnbT+md7dPCGpViEzPxKJGWU8OfkI7DiHbRfjNJMYQY/01lj2qNzMObv74/4+HgAQOvWrSGVSpGcnAwAUCqVqKykDxuUxsv1hHxIZUq4OVlz0nztV92r0BTk7eOqZ+dwpaTI8LC3jr9jZMoyRYDK2zfEAyYzZmbBmK+RhGUoj+fEpVQkpZfA1toC00fpNsPU1lqM7mGamWNnYjP5cK8OhaVSFJRIIRQAbQwocadQGjP2tnTWmK7oHIw988wzWL16NXbt2gVXV1d06NABn3zyCU6dOoWvv/4abdq04cNPCsUo1Bz0LOSgFO/hUOPGvcJeWCpFXpEEQgH3CmEPAxVjZMZME4zRzFhdisuqUFwug0AAtPJ2NLU7tXgob0//XqairFKOnYc1lTYvDgvW67PbL8IPAPDf9Qyo1PyXKjL9Yv7ejrCx0kusmkJp9DCZMRqMaY/OwdisWbPwwgsv4MaNGwCAjz76CHFxcXj99deRnJyMd999l3MnKRRjoFCqcen2w34xLmCzPo1cme1edR9EgI8jbK25lYY3RilnEdszZqoyRdoz9ihMiaKfhz2szezBtal8bhszPx6JQ7lEgVY+jhjZq7VeNiKDvWBvI0ZRmQy3kwo49rAudNgzhVJ78DNFO3S+AwqFQrz33nvs9x07dsTJkyeRnJyMwMBA2NubhzwxhaIrNxLzUVmlhIuDFWdztJgyxeyCSqjUxGwU43SF6/liNTHGrLGSctOWKdbMjDWFMQdcwIh3BJqReAcDLVM0LYnpxTh2MQWAbqIdjyK2EKJXZ18cu5iK07EZ6NzOg0Mv60L7xSiUh4OfaWZMezgZeGNvb49OnTrRQIzSqDlfXaLYs6MPZ0GTu7MNxBZCKFUE+TyqBfINH+IdDMyssdwiCVQ8yNur1AQlJhbw8HK1haB6zEFZJZVLB4AH1YNxza1fDKDy9qZErSbYXC3a0b9LC4QFai/aUR/9q1UVz9/Kglyh4sLFelGp1EhMLwFAM2OU5o0dDcZ0RufM2MCBA5+4qvv333/r7RCFYgqUKjUuMiWKnbkpUQQ0aoG+7nZIzSlHZn4FmyFpTMgUKtzPLAHATzDm6qiZwyVXqpFXLGUzZVxRViGDmgACAeBsb5pgzFIsgpujNQpKq5BdWAknnv0ghOBqXB7kFUpe38cQkqvPKXPMjGnk7a1RVFaFrPwKKlNuRE5cTkNCWglsrCwwY1SYwfZCW7vB3dkGBSVSXInL5awE/VFSc8ohk2vmMLbw5GYOI4XSGKGZMd3ROTMWFRVV5yssLAxqtRpSqRTPP/88H35SKLxyK6kA5RIFnOwtEaaDfLI2+LKKio2z/yQpvQRKFYGLgxW8XG05ty8UCmoIJnB/jJh+MSd7K73LnbjAu/p3zDFC6dsvJxPwxU83sO9cIe/vpQ9SmZIVx2jtZ17iHQxMLyMtVTQe5RI5fjikEe2YNCyYk7mAQqGAFfLgcwA0U6LYrqULJ+JPFEpjhe0Zo3PGtEbnzNiqVavq3a5QKPD6669DKpUa7BSFYmwYFcUeHXw4f2D3M4J0O5/U7Bfjq9fJpzp7yMeDL6Ok6GoiJUUGHzc73L5fyLu8/eU7Odh97B4AILNQgbJKOWxtuQ+iDSE1uwyEaARVTKVw+SR83e1x+34hVVQ0IhrRDjn8vR0wqrd+oh310S+yBX77JwlX7uaiQqpgV+65JL569ActUaQ0d2iZou5w9tQpFosxdepU7Nu3jyuTFIpRUNUsUeShhMWveoW9sWbG4lL46xdjYPrG+HjwLa7OjDmbSEmRwcsIiorpueVY81MMSHVZJgDcTi7i7f305T4zX8wMSxQZ2GxtI11EaWwkZZTg6IUUABrRDgsOF8Va+TjC39sBSpWa7Q3mGkbWvh0NxijNHDYzJqHBmLZwmgIoLS2lQ58pjY47DwpRWiGHg60YHdu4c27/4VDjxheMEUJwL5X/YIxPRcWiaiVFc8iMAfwFY5VSBT79/jIkVUqEBbphWPeWAMwzGHtQLWsfaIbiHQysomJh4/vcNjZqinb0jfBDxyBur8MCgYAV8uCjVLFCqkB6ruY8oUqKlOYO2zNWpQAh/M/3awroXKb4559/1tmmUqmQk5ODXbt2oWvXrlz4RaEYjXM3HpYocrkay8CUKeaXSCFTqGAlFnH+HnyRXVCJ0go5xBZCBLXg78GZz1ljxWXVSoomzozxOfhZrSZY81MMMvMr4O5kjfendsOd+7k4ejEdt+6bXzDGytqbczDWyMuLGxOnrqYhPrUYNlYivPSM4aId9dE3ogV2Ho7DrfsFKCyVws3JhjPbidVZMW83W97FeSgUc4fJjKnVBFVyFR2ArgU6H6H333+/wdciIiKwePFigxyiUIyJSk1w4ZamRPEpnlS2HO0sYWcjRqVUgZyCSgT4mKdgQX0wJYptWjhDbMFfEOnjVlvensu+vYcDn02bGWOCsaIyGark3Koc/nw8Hlfu5kJsIcSiGVFwdrBCSIAzhAIgr1iKnMJKs1HyVKnUSM3WyNqbo5IiAzOou0Kq6btztLM0sUdNkwqJHN8f1Ih2TBwazGmQVBMvV1uEtHJFXEoR/rueieh+bTiz/XC+GFXdpFCsxCJYiARQqggqpQoajGmBzkeoPtl6gUAAe3t7ODo2nodMCgXQiFMUl8tgZyNG57b8DAQVCDTy9onpJcjMr2iUwRifJYoA4ObEn7w9O2PMxMGYg60YdtYWqKxSIrdIAg9Hbm5QF25lYc+JeADAm+M7o21LTZmUtZUFWrhbIi1fjhuJ+WYTjGXkV0CuVMPGSmQ2PtWHtaUF3JysUVhahayCCjja0QdtPth19B7KKuVo6eWAZ/oE8vpe/SJbIC6lCKdjM7gNxlKZfjFnzmxSKI0VgUAAOxsxSivkqJQq4O7MzwJLU0Ln5Wc/P786X76+vjQQozRKGBXF7mHeEFvwJ3vOKio2MmW2mkqKfCIUCljpd67l7ZnMmKkGPjMIBALO5e1Tc8qw7udYAMDovoEY2NW/1uuB3poA9HpCPifvxwUPqksUW/k4mb0EOCMsQ+Xt+SE5sxRHzj8AAPxvXEdeysRr0ruzL0RCAe5nlCI9t5wTm4QQNhgLpvPoKBQAgJ11tYgHVVTUCp2XZhcuXKj1vgKBACtWrHjsPmq1Ghs3bsSvv/6K8vJydOvWDUuWLEHLli3r3b+4uBjLly/Hv//+C4FAgJEjR+Ldd9+Fjc3DyHvo0KFITU2t9XNjx45lZfm1sUFp+qjVhFXW4nLQc300RhGPCqkCadUPLMGt+G9K93W3QxrH8vaEEFZN0dRlioCmVPF+RilyiiToGGhYiV6FRI5Pv78MqUyFTm3c8VI9A3IDva1w+hZwI7EAajUxi+AnOau6RNGM+8UYfNztcOt+Ae0b4wFGtENNgD7hfujUhp/KhJo42Vshor0nrsbl4sy1DEweHmKwzZxCCcolcliIhGjtSxelKRSAytvris7BWE5ODu7evYvS0lL4+fnBy8sLJSUlSE1NBSEE3t7e7L7azCT65ptv8NNPP2HVqlXw9vbGF198gVmzZuHAgQOwtKxboz979mxIpVLs2LEDZWVl+OCDDyCRSPDZZ58BACQSCdLT07FlyxaEhT18OLG2ttbaBqV5EJ9ajMLSKthYWSCiHb8PAo1R3j4+tQiEaFQAjTELig95+8oqJeRKNQDTlykCgHf10GxDM2MqNcEXu2OQXVAJTxcbvDula719dn5ulrC2FKFcIkdKdplZBEDJmSUAzFvWnsGXx2HkzZ1/YtIRl1IEa0v+RDvqo19kC00wFpuBF4cFGzw7kZkvFtTCide+WgqlMcEoKtLMmHboHIyNGDECiYmJ+OmnnxAZGcluT05OxmuvvYZJkyZh2rRpWtmSy+XYvn075s+fj/79+wMA1q1bhz59+uD48eMYNWpUrf2vXbuGy5cv4/DhwwgKCgIALFu2DLNmzcK8efPg5eWFpKQkqNVqREREwMmp7s1eGxuU5kHtEkV+b6KNUZmN7RdrbZzSGz7k7ZmsmJ21hVmoWLK/o4GKiruPxiH2Xh4sxSIsmh7VoIKbSChAaGsXxMYX4HpCvsmDMUIIkjM1mbEgMwgMnwSj8knLFLmlQqrAjmrRjheGtDdqT0mPMG9YWYqQUyhBQlox2htYWvhQvINK2lMoDDQzphs6F2hv3rwZ8+fPrxWIAUBgYCDmzJmDbdu2aW3r3r17qKysRM+ePdltjo6OCA0NxZUrV+rsf/XqVXh4eLBBFABERUVBIBAgJiYGABAfHw93d/d6AzFtbVCaPoQQNhjjS0WxJswKe0mFrNGsFBmrX4yBD3n74uoZY84mnjHG8FDeXqK3jf+uZ+LXvxMBALOfD0dQC+fH7t8hUPP3u5Fo+r6xwtIqlEvkEAoF8Pd2MLU7T6RmtpbOy+GOn47dQ0mFDC087TG6b9CTf4BDrK0s0CPMBwBwmoOZY0y/WHs67JlCYbGrMWuM8mR0DsaKiooaDHSEQiHKy7Vvis3JyQEA+Pj41Nru6enJvlaT3NzcOvtaWlrC2dkZ2dkaefL4+HjY2tpi9uzZ6N27N5555hns2LEDarVaaxuUpk9iegkKSqSwthQhMtiT9/eztRbDtXrOVWPoG1Op1EioXvHlW0mR4VF5ey4oqp4xZg79YsDDYCy3SAK1WveH+wdZpfhq7zUAwLj+bdCvepDt4+gY5AZAM9xcoVTp/J5cklw97Lmlpz0szSBT+SS8quXtK6vl7SmGk5JdjkNnkwEAr47tyKtwUkP076L53Jy9nmXQtUauULEDzNvRzBiFwsKWKUpoMKYNOpcpdu7cGRs3bkR4eDhcXB5efPLy8rBhwwb07t1ba1tSqRQA6vSGWVlZobS0tN796+sjs7KygkymeehKTExEWVkZhg0bhjfeeAMxMTH44osvUFpairffflsrG/pACIFEov9qN1cwx5T5tzHZN6bvp2PSAQAR7dyhUshg6PVCG9+9XW1RVCbDg8witHDXTdnP2Mc9JbscUplmWKO7g8igc1tb323EBGILIRRKNVKzi9j+KkNs5xZqSuIcbS30+h24Pu52lgQikQBKlRpZeSU62S6XyLF8+2XI5Cp0auOG8QNaPfZ3Yuy6OwjhZG+J0go5bsRnI5SDslN9j0v8gwIAgL+XvVa+m8N1xs3RCoVlMjzILES7ls6c29eFxn59J4Rg2193oSZAjzAvtGvx+PNAV/s1/30c7VrYwcFWjJIKGS7dzkB4W3e9bCeklUCpInCys4SDNfT+Xeg5Y3zbfNtv7r4za22lFdJan4vGfFx0hRCidU+qXkOfJ0+ejIEDByIiIgIuLi4oLCzEtWvX4ObmhkWLFmltixHVkMvltQQ2ZDJZvcqG1tbWkMvrrk7KZDLY2moe3L799lvIZDI4OGhKYNq3b4+Kigps2rQJb731llY29EGhUCAuLk7vn+ealJSURmufb98fPHiA/65pMq8tnLn9uz3Od2uR5ry7dS8V7pYlnNs3lJq2Lydosne+LiLEx9/j3H5DONsJkV+qxpVr8Wjjq302qyHb91NKAABqRYVBf2cuj7uTrQhF5UrcvJeC1l7WWtlWqQl2ny5AXrEMLvYiDO9spfXfJTU1Ff5uItyqAP65lABBFXe9Wroel1uJhQAAG6FEq7+HOVxnHGyAwjIg9lYSVBXaz0UzB9/Nzf7NFAkSMsogFgnQs42Al3umtr4H+1niSqICh/+Lh5VSuxLeR21fuKepBPJ2FuLePcOvk/ScMb5tvu03V9/LSzXPEDl5RfV+zhvzcdGF+pI/9aFzMBYcHIxDhw5hx44diImJQWZmJlxcXPDSSy9h2rRpcHZ21toWUy6Yl5cHf/+H83Hy8vLQvn37Ovt7e3vj5MmTtbbJ5XKUlJTA01NTamZpaVnnl2/Xrh0kEglKS0u1sqEPYrEYbdpwN0RSX6RSKVJSUtCqVStepPr5tG8s3wU2HiipzISlWIhRA8JhbWl4uZQ2vicVpiD2fiIUAluEhOgmqWzs437i9i0AQESIH0JCDOvp0MX3VtdkyC/Nh9jODSEh/o/dVxvbJ2/fAlCBwAAfhIS04tV3bWl5WYqi8kIIrZwBVGlle9fRBCTnyGAlFmLRtG5a9VvV9L2X1BG3Uu8ip0yo87n3JNu6HJeiI2cBAN3D2yAksOEMnTldZ4ISCFJyMyG0ckZIyJOv8ebkuznZLywux/HfLwMAnhsYhB5dW3NqX1ffBbYluJJ4BfGZMgQGtYPVY+4DDdk+dvMmgFJEhLRASIj+vw89Z4xvm2/7zd33IkUOcKUEIrFNrXtOYz4uupKUlKT1vjoHYwDg5eWF9957T58frUVwcDDs7e1x6dIlNhgrKyvD3bt3MXny5Dr7d+vWDatXr0ZqaioCAgIAAJcvay7uXbp0ASEEQ4YMQXR0NN588032527dugUPDw+4uLg80Ya+CAQCgzJrXGNjY8OrP3za59v360klAICuIV5wdeZWROBxvrfy1ZT15hZX6f37Geu4J2Zoyvs6t/Pi7P208b2FlxOuxOWjsEyh0/s2ZLtMogQAeLk5GPR7cHnc/TwdcCOpEMUVSgQ4P9n26dgMHDinmZs4d2IXBAfqpvhqY2ODbmF+2PzHXSRlloEIxGxztaHoclwqpQrkFmtKR0ICPWFr++QVQ3O4zvh7OwHIRH6pnJNzkgvM4fpOCIFMrkJllQKSKiUkVQpUVikhrVJWb9Nsr6xSQCJVIiW7FJVVavi42WL84BDeesW0PTYRwTbwdLVFXpEEt1PK0CfcT2fb96uVQTu08eTk79HUzxlztM23/ebqu6uzpopAKlfXa6MxHxdt0WVshk7BWGFhIQQCAVxdNSuacrkcv/76K+7fv4/27dtj7NixWqfkAE0Wa/LkyVi9ejVcXV3h5+eHL774At7e3hg6dChUKhWKiorg4OAAa2trdO7cGZGRkZg7dy4+/vhjSCQSLFmyBNHR0awk/ZAhQ7Bt2zYEBgaiQ4cOuHDhAr777jt88MEHAKCVDUrThRCCi7dzAQC9jKCiWJOag591qSU2NoWlUuQVSSAUGL8p/eFcJ26kxBk1RVczUVMEHsrb5xZJgRaPD4qSMkqwoVqwY/ygtnoPJ/d0sYWvux2yCipx+34BunfwefIPcQwjdODhYgMHLQIxc8GHh/l35ohaTXAtIR83EypwL+8BFCpUB1magEoqU6JS+jDIksiUeonQzBgVbBLRjkcRCAToF+GHX/9OxJnYDK2CsZoUl1Uhr1gKgQBoq0UvIYXSnLCjc8Z0QutgbOXKldi9ezfmzp2LmTNnQq1WY8aMGYiNjYWjoyN++eUX/PLLL9i1a5dOqcHZs2dDqVTiww8/RFVVFbp164Zt27ZBLBYjIyMDgwYNwsqVKzFu3DgIBAJs3LgRS5cuxbRp02BlZYXhw4dj4cKFrL133nkH9vb2WLt2LXJyctCiRQt88MEHeP755wFAKxuUpktuiQI5RVKILYToGmLc4NvbzRZCASCVqVBcLjMbhb9HuVct1Rzg4whba24yKNrycNYYN4qTjJqii6Nugil84uVaHYwVSwE0fHxLK2RYseMy5Eo1uoZ44cXhhpUXdm7ngayCStxIMk0wxigpBjaCYc81YRYIss18EcUQVCo1vtxzrYbUe4nWPysUaNRibW3EsLWygJ2NGLbWFrC1EsPWxoLdZiEkUEkL0LmNGy+/gz70i2yBX/9ORMy9XJRL5DotEjDzxfy9HIx+naRQzB07azpnTBe0Csb27duHnTt3YurUqRg8eDAA4LfffkNMTAwmTZqExYsXIzc3F5MnT8a3336L2bNna+2ASCTCggULsGDBgjqvtWjRAvHx8bW2ubm5Yf369Q3/QhYWeOONN/DGG280uM+TbFCaLnfTNWVSke09jX4DFVuI4Olqi5xCCbLyK8w3GDPyfLGaMHOdcgo18vYikf4r6HKFir0RmNOxfpgZkwBwrHcfpUqNz3+8ivxiKXzd7fDOi10gEhoWBIS39cCR8ym4nmCaeWMPqku6WjeyYMy7+u9VWaVEWaW8wQHbjRWFUoXPf7yKi7dzIBIK0NbXCt6ernC0t4addXVgZS2GnbUYNtYWsKv+3tbaAnbWYlhZirQKUCUSCeLizGusR4C3I1r7OuJBVhnO3cjC8J6ttP5ZZvQHlbSnUOrCSNtLqhRQqwmEBt6/mjpaB2OTJ0+ulT3666+/IBaLMXfuXAgEAnh7e2PGjBnYvXu3TsEYhWJM7qZpgjF9y70MxdfDHjmFEmTmV6JD0OPllE1F3ANNMGas+WI1cXOyhqWFEHKlGvklUnYulz4Ul2uyYmILIWc9UlzASPZXSpWQyuufcfT9gTu4mVQAGysRPpgRxd7YDKFjG3cIBEB6bjkKS6VwczJuc3NyZnVmzK/+ANRcsRKL4O5sg4ISKbILKptUMFYlV2LF95dxLSEfYgsh5r7QCbbqAoSEhJi838JY9ItogQdZd3E6NkOnYOzhsGfjXycpFHOHuecSAkhlSrO6B5sjWi07JyUl1ZofJpPJcO3aNXTq1ImVkAc0MvKZmZnce0mhcEBGXgUKypQQiQSICvU2iQ9+NfrGzBGZQoX7mSUATBOMCYUCNhORlW9Yj05xmaZfzMXByqxKy6ytLODsoHmgL65Q1nn97ytp+Os/zVDcuRO7wN+bm+DFwdYSQS2cAQA3kwo4saktCqUaabmazFign7NR35sLHvYymufnVh8kVQp8/O1FXEvIh7WlCB/N7IEu7T1M7ZbR6RuhGQB9J7kQ+cXazSZSqQkS05lgjGbGKJRHsRSLYFndG0pLFZ+MVsGYQqGoNQfsxo0bUCqViIqKqrWfVCqFWEyjX4p5cvGORrijc5CbyVZp/Kof6jLNNBhLStcMMXVxsIKXFkOX+cDHjZu+sSImGDOjEkUG5ncsKq8djCWkFePrfTcAABOHtkfPjtz2doW31TxsG7tUMSOvHEoVgZ21BTxdTCs3rA8+HC0QmAtllXJ8sPk87iQXws7aAsteeQqd2zW/QAzQCMqEBWr62P69lvGEvTWk55ZDKlPBxkqEll7cKvJSKE0FKuKhPVoFYy1atKill//vv/9CIBCgV69etfa7dOkS/Px0UySiUIwBIQSX7uQBALqHmU41k1VUNNMV9pr9YqbKJj08RtxkxsypX4zB200T6NbMjBWXV2HFjstQKNXoHuaNF4bUnbVoKJ3bakpjbyTmgxDdlfD05X6GpkSxtZ+TWWUptYXpZcxuAoqKxWVVWPTNWSSll8DRzhKfvtYLIa2bd6ld/0hNduyMlsEYU6LYtqWLwb2cFEpThQnGaGbsyWgVjA0fPhxbtmzBhQsXcO7cOfzyyy9o2bIlunbtyu5z8+ZN/PTTTxg4cCBvzlIo+pCQVowPN59HWm4FhAKga7DpVoCZMsXsgkqo9JCF5pu4FNP1izH4cCRvz/SMMSWB5gSTGSuuUAHQlPGt+uEKCkur0MLTHvMmRfLS8BzS2g1iCyEKS6uMmp1lZO0D/RqXeAeDr0fTKFPMK5bgva/PIjWnHK6OVlj5ei+2dLU506uzLyxEAjzIKkNqTtkT949P1VwnaYkihdIwNDOmPVoJeMyaNQtXrlzBjBkzAAC2trZYuXIl+/qMGTNw6dIlBAUFYdasWfx4SqHoSGZ+BX48HIdzN7MAABYiAQZ1doS9relKad2dbSC2EEKhVCO/WGKQQAXXEEJwL9X0wZgvR/L2RWacGfNiyhSrM2Pf7b+Fuw+KYGttgQ9mRPGm9GklFiG0tStuJBbgRkI+Wngap8SqscraM9RcIGis8vZZ+RX4cMt55BdL4elig+X/68X+Xs0dB1tLdAn2wqU7OTgTm4GpI0Ifuz9VUqRQnow9zYxpjVbBmI2NDXbu3ImrV6+ioKAAUVFR7OBnAHB2dsbLL7+MmTNnws6OXtwppqWorAo/H4/H8UupUKsJBAJgQJeWGNcvAAXZKSb1TSgUwNfdDqk55cjMrzCrYCynSILSCjnEFkIEtTDdQ3NN6XdD5O2ZzJiLGQ18ZmAzY+VK/H01A4fPp0AgAOa/2IX3AKlzWw/cSCzA9cR8jOwdyOt7AZog/0Fm486M+bjZQSDQDEFujPL2qdllWLzlPIrLZfDzsMfy/z0Fd+fG17vHJ/0iWmiCsWuZmPJ0SIMBt6RKgbTccgBAexqMUSgNwpYpVtFg7EloPfQZQK2yxJqsW7eOE2coFEOolCrw++kk7P/3PmRyTflX1xAvTBsZilY+jpBIJCjINrGT0PREMcFYl2DT9a89SkKa5oG5TQtniC1EJvPD3alG9tAAefuHmTHze3D2dtf0jJVKVNh+8B4A4MXhwehmBJXPzm09AMThVlIBVGrCe89LbpEElVVKWIgERsvEcY1ltbx9frEUWfmNS94+Mb0YH229gHKJAq18HPHJq0+ZZemuqekW5gUbKxHyiiS4l1LcYB9dYnoJCAE8XWzMUhyIQjEX2DJFCQ3GnoROwRiFYo4olCocOpeCX04moFwiB6Cp5Z8+MtQsZ3k9lLc3LzGA+LQSAKYtUQSq5e3d7JCeW46sgkq9g7GScvNVU3S2t4KVpQgyuQpKFcFTnXzw/KB2RnnvoBbOsLMRo1KqwP2MEt5LrZh+MX9vR4gt9B/ibWp83Ow0wVhBRaMRvLiTXIil312EVKZEO39nfPxyTzjYWpraLbPE2tICPTr44J+YDJyOTW/wb0zni1Eo2mFPM2Na03jvjJRmj0pNcOpqGv636m9s++s2yiVytPC0x6Lp3fDFW33MMhADHvZEmZu8PZMZCzZxMAbU6BvT8xip1AQlbJmi+WUBBAIBvKol3lt42mHOC5FG60MSCQXo1Ebz2TCGxH1yZvV8sUbaL8bg69G4FBWvxedhydYLkMqU6Bjkjk9efYoGYk+gf2RLAMDZG1lQquofyP4wGKMlihTK47Czpj1j2kIzY5RGByEEV+NysfNwHFKyNQ96ro7WmDQsGIO7tdS7x8hY+Jrh4GepXI2Man+CW5n+IYMVTCjU78G3rEIGNQEEAk0WyhwZ+VQAjp5PwpxJ4bCxMu6luHNbD1y4lY0bifl4fjC/GbnkTEbWnpvh1abClyOVT2Nw4VY2Pv/xKpQqNboEe2Lh9ChYiU1XetxY6NzWHc72ViipkOF6Qj66htQuIyeEsOIdtF+MQnk8jFgaDcaeDA3GKI2Ke6lF2HHwLu4kFwLQ1CQ/N7AtRvVuDWvLxnE6M2WK+SVSyBUqWJrBQ1JGgRyEaEqxzEHwwtfAUk6mX8zJ3spsg/P+kb7wsimFtwmGa4dXD/i9+6AIVXIlr58dRkkxyM+Zt/cwBg8VFc1nEaU+TsdmYN3PsVCrNeWv81/s2qjLQ42JSCRE73BfHDz7AKdjMuoEY/klVSipkMFCJGi0YjQUirGg0vba0zieXinNnvTccvx4JA4XbmkUOMQWQjzTOxDPDWrb6EpvnOwtYWdtgcoqJbILKxHgbfqMQXqBpqTPXHphfN0YeXv9gjFGSdHVDAJLc8TX3Q7uTtYoKK1C3IMiRLT35OV9yirlKCiRAgBa+Zj+PDeEhyMXzFfe/tjFFHy97wYIAQZ2bYnZz4eb7WKEudI/sgUOnn2Ai3eyUSVTwrpG1joxvTrL6+tkFotoFIo5Q8sUtUevYOzcuXP4559/IJVKoVbXrqsWCARYsWIFJ85RKIWlUvx8PB4nLqVCTQChABjUzR8ThwbDw6VxSjMLBAL4etgjMb0EWfkV5hGM5WuET8yhXwwAfDwYeftKveTti6szY85mqKRoDggEAnRu54G/r6TjRmI+b8EYI97h7WbLrpI2VrxryNuXVsjNTpHwzzP3se2v2wCAEU+1wqtjO/EyOLyp087fBT5udsgurMTFOznoH9mCfS0xQ3M+034xCuXJ0MyY9ugcjG3fvh2ff/45rKys4OrqWmd10BxXCymNjwqJHPtOJeLAf8mQKzUBf/cwb0wdEQJ/MwheDMWvOhjLNANFRZVKjcxCTTBmaiVFBkPl7YuqlRRpZqxhwts+DMb4gu0Xa+TiHcAj8vYFFWYTjBFCsPdkAnYf1YxIeHZAG0wbGUrvxXoiEAjQN9IPe08k4ExsRq1gLIkJxmi/GIXyROjQZ+3RORjbtWsXnnnmGXz66aewtGxc5WEU80ehIjhwNgV//pvCrqaEtnbFtJGhCG3tZmLvuMOcRDzS8yohVxLYWFmgpZd5zIEyVN6+uKxaSZFmxhqkU1tN39j9zFKUS+S8lPs+7Bdr/MEYoClVZGaNmcP1iBCCHQfv4vfTSQCAycOD8fzgdjQQM5B+ES2w90QCrsXnobRCBrEQUKoIHmRpBKOorD2F8mSYzJhUptSrwqU5ofORKSgowHPPPUcDMQrnJKSXYMOBHOw6logKqQL+3g5Y/FJ3rHqjt1k8+HCJn4f5yNsz88XatnTifQCwLtTs0dGVhwOfaWasIVwdreHv7QBCgJtJBby8xwNWSbGpBGPV8vZ6qnxyiVpNsOm3m2wgNmtMB0wY0p4GYhzQ0ssBQS2coFITnLuZBQDIKVZAqSJwtLOEt5vxRXcolMZGzdJ0iUxpQk/MH52DsdDQUCQmJvLhC6UZQwjBd3/FoUyigpuTNd6eEIH17wxAVJh3k3y4MFQtkEvuMXNz/M3rgdkQ9bqHM8ZoMPY4wquzYzd4mDcmU6iQnqf52zX2GWMMvtWLKKbOaKtUany5JxZHLqRAIADeHB+OMX2DTOpTU6NfhKY88XRMBgAgo1BzTWnn79Ik70kUCtdYiISwttQI3dBSxcejc5niokWLMGfOHNja2qJz586wsakrouDr68uJc5Tmw90HRUjNqYCFSIDPXu8OL3dnU7vEK0zWp6RChgqpgq2tNjZKlRo3EjVjAsLMREmRgYvMGC1TfDyd23rgr/+ScZ2HvrG0nDKo1ZpMgptT0wiKfdxMP2tMqSL48pdbuHw3D0KhAPMmRqJfjb4mCjf0jfDD9wfvIC6lCHnFUmQWaPpqqXgHhaI9djZiVMlVqJAogKZV4MQpOgdjEydOhFqtxqJFixpcHYqLizPYMUrz4sDZZABAp1a2jU6qXh9srcVwdbRCUZkMWfkVaGeihvBbSQWorFLCzlqI9v7OJvGhIZiSMF2zh4QQVk2Rlik+ng5BbhAKBcguqERekQSeHM48S87U9NcE+jo1mUwCk9E2lby9TK7Cnn8LkJQtg4VIiPemdkWPDj5G9aG54OZkg45B7riZVIBzN3OQUS1yRMU7KBTtsbMRo7C0imbGnoDOwdjy5cv58IPSjCkokbLzw6La6SbU0Jjxcbc3eTB24bbmuLf3szE7Gexa8vZqonU/W2WVklXgdKHB2GOxtRajvb8L4lKKcCMxH0O6B3Bmm5G1byr9YoBGol8g0DSkl1TIjF4Gu2HfbSRly2AlFuKDGd15G0lA0dAvsgVuJhXg76sZKK5QAQDa0mCMQtEapuqnoooGY49D52Bs7NixfPhBacYcuZACtZogpJUzvF2aflaMwc/DHneSC00mb69WE1yqDsZCWppf0FJL3r5YorWiIpMVs7O2gBUdzPpEOrV1R1xKEa5zHIwxsvaBvo1/FAWD2EIED2cb5FUrKhozGHuQVYorcXkQCICFUyNpIGYEnurki02/3UR+ieaa4udhZ7KScgqlMWJH5e21Qq+hz7m5uYiJiYFcLme3qdVqSKVSXL16FevWrePMQUrTRq5Q4djFFADA8O7+AIpM6o8x8TOxGEBCWjGKymSwsbJAay/zC8b0lbcvrp4x5kzFO7QivK0H9p5IwM3EAs5K79RqwmbGAptQZgzQlM/mFUuRXVCJsEDjNUHs//c+ACC0pQ1CWtHsjDGwtxGjW6gXW7nRtmXTOpcpFL6hwZh26ByMHT16FPPnz4dSqWRv2jVv4IGBgdx6SGnSnL2RidIKOdydrNEtxAMJCc0nGGMVFfVQC+SC89UPGJHt3GEhMq8SRQZfd00wll1QCbTX7meKqmeM0X4x7Wgf4AorSxFKKmRIzSlHKx/DM1k5hZWokqtgaSGEX/V53lTw8bDD9cR8o35ui8uqcCY2EwDQI7hpHU9zp19kCzYYa9OCBmMUii7YW1eXKdJg7LHoLG2/efNmhIWF4ffff8e4ceMwZswYHDp0CAsWLIBIJMKiRYv48JPSBCGE4MDZBwCAp59q3ewGAjIPqZn5GjEAY0IIwUWmTy/UfMud9JG3L6ZKijohthCyGZ7rHEnc368uUQzwcWxyn2tfd+MrKh46/wBKlRrtWjqhpTs9r41JtxAv2Nlo1q2DzUzkiEIxd2hmTDt0vks+ePAAL7/8MkJDQ9G9e3fcu3cPQUFBeOmllzB16lRs3ryZDz8pTZD4tGIkpZdAbCHEsB7c9ao0FrzdbCFkxACq52IZi9SccmQXVkJsIUR4W/PVm9VH3p4OfNYddt4YRxL3TbVEEagx+NlIwZhMocKR8ykAgBFPNb/rpKmxFIuwaGokJvRxQ0svmpWkUHTB3pYGY9qgczAmFArh5KS5wQYEBCA5ORlqtUa5rG/fvkhKSuLWQ0qT5eB/mqxYn3A/ONk3v9VesYWIlRLPNHLf2IWbWQCAiHaesLbSq3XUKOgjb/9w4HPzO6f0JbydJhi7fb8ASpXaYHuseEcTDMZ82AWCCqNktE/HpKOsUg5PFxtEhXjw/n6UurRp4YSQlnVnqlIolMdjR8sUtULnYCwwMBCxsbHs/+VyOe7duwcAKCsrqyXqQaE0RHFZFc7d1PRAPNO7+fYZ+tYoVTQmjKR9z47mPaOIefBl5O214eHAZ5oZ05YAb0c42lmiSq5CfGqxwfbYzJhv0wvGHma0VbxntAkhrHDHM30Cm1zJJ4VCadoYu0yxpEKGyiqVUd6LS3S+sr/wwgv46quvsG7dOjg4OKBHjx5YuHAhfvzxR6xZswZhYWF8+ElpYhy9kAKliiCklSvatHQ2tTsmg+kbM6aiYk5hJR5klUEoFCAqzNto76sP7s4aeXuliiC/WKLVzzBqiq5UTVFrhEIBOleXKt40sFSxuLwKRWUyCASanrGmhthCBHcXTUab776x2Pg8pOdWwMbKAkOiaIkihUJpXDDBmDEyY+USOeZvuIDvjufx/l5co3MwNn78eHzwwQdsBmzZsmWQyWT49NNPoVQq8cEHH3DuJKVpoVCqceRCCgBgVO/WpnXGxPhVZ36MWabIKIN1CHSDo515z3XTyNtrHny17dFh1BSpgIduMMHYdQODsQeZZQA0JaY2ZlwCawi+NUoV+WT/GU1WbEh3f/ahhkKhUBoLxsyMnbqajnKJAlbixldBoNed8sUXX2T/7+/vjyNHjqC4uBiurq6cOUZpupy/mYXichlcHa3wVCdfU7tjUkwhb88EY+Zeosjg626P9NwKZBVUIuIJ8vYyhYq96NMyRd3o3NYdABCfWgypTKl3IJXchMU7GHzd7XA9IZ/XzFhqdhmuJeRDKGjepdwUCqXxwgxJr6ziNxgjhOBo9SJ/lzbazSQ1J/Retrx//z7OnTuHvLw8TJkyBenp6bC0tIS9PVUbojyeg2eTAQDDe7aGRTPvgWCCsewCTU+USMjvvK/isircS9XMcuvRoXEEYz46KCoyPTxiCyF7E6Boh7ebHbzdbJFTKMGd5EJ0DfHSyw4j3tHat+mVKDL46CEsoytMr1iPjj5aDzynUCgUc4K5D8vkKiiUhotDNcSd5EJk5FXAylKEjq1seXsfvtA5GFOr1ViyZAl+++03dtjz008/jW+++QapqanYvXs3vL3Nuw+FYjqS0ktwL7UYFiIBhjdDOftHYXqiFEo18oslvD90XbyTA0KAdv7OcHduHOpgvjrMGmNnjDlYsYPoKdrTua0HcgpTcT0h3+BgLMjPmUPPzAtfD91HLuhCSbkMp2MzAADRfdvw8h4UCoXCNzbWDxdFJVUK8FVBePRCKgCgV0dvWIsb371f58PyzTff4MCBA1i+fDnOnTvHSvsuWLAAhBCsW7eOcycpTYcD1Vmx3p39aBkZAJFQwGZ+jNE3xkjaN5asGKBbZowqKRpGZwPnjVXJlGzQ3Nqv6WbGai4Q8CFvf+T8AyiUarTzd0ZwKxfO7VMoFIoxEAkFsLXW5H34EvEorZDhXPWzzZBuLXh5D77RORj77bffMHv2bDz77LNwdnZmt4eEhGD27Nk4d+4cl/5RmhClFTL8d10jZ9/chTtq4sfK2/MbjFVIFbiZVAAAjapXj5k1llMoeaK8fTEd+GwQndpo+sZSsstYVUpdSMkpAyGazKRLE1az9HK1g1AAVMlVKOZY3l6uUOHQec0Mxui+bWiGl0KhNGr4FvH4+0o6lCo12rRwQmAjXQTUORgrKChASEhIva95eXmhrKzMYKcoTZNjF1OhUKrRtqUz2gdQsRcGVpmN51ljV+/mQKUmaOnlwAaAjQE3ZxtYiIRQqtQoKJE+dl/mwdiZDnzWCyd7K1Z442Zigc4/z/aLNWHxDkDTk+jhopvKp7acic1AaYUc7s42eKpT48lgUygUSn3wOfiZEIJjF1MAAMN7tuLcvrHQORgLCAjAmTNn6n3t8uXLCAigfUCUuqhUahypXu0dRZXBamGszNj5RqaiyKAp5aye6/SEY1REM2MGY0ip4sN+saYdjAE1ShU5/NwSQvAnM+S5Nx3yTKFQGj/2tvxlxm4mFSCroBI2VhboG9E4SxQBPYKxadOmYefOnVi2bBnOnz8PgUCA1NRUbN++Hdu3b8ekSZP48JPSyLl4OwcFpVVwtrdCn/DGUyJnDBhFxUweZbKr5ErExmsGITa2YAwAfNyqVScLH3+MmMxYUy6R45vwGvPGdO2HepDFKCk2/WDMh+0b4+5zez0hH2k55bC2FGEoFTiiUChNACYzxkcwxsjZ949s0ajnWurs+fjx41FUVIRNmzbh559/BiEE8+bNg1gsxqxZszBx4kQ+/KQ0chjhjmE9AiC2EJnYG/OCyYzlF0sgV6hgKeb++FyLz4dMroKni02jzFow6nVPkhJ/mBmjZYr6EtraFRYiIfKLpcgurGR79p6ESqVGSpamTL0pzxhjqDmWgiuYrNiQ7gF0NAOFQmkSMD1jXJcpFpdX4eJtTcVPYy5RBPScM/bqq6/ixRdfRGxsLEpLS+Ho6IjOnTvXEvSgUBgeZJXiTnIhREIBnn6qlandMTuc7C1hZ22ByiolsgsrEeDNfQMqc8Hq0dGnUQoCaKuoWFLOSNvTzJi+WFtZILiVC27fL8SNhHytg7HM/ArIlWpYW4rg0wzmYukyckEb0nLKEHsvDwI65JlCoTQh7HkS8Dh5OQ1KFUF7f5dGvwCod07P3t4effv25dIXShPl4FlNr1jPjj5wc2ocs62MiUAggK+HPRLTS5CVX8F5MKZUqXH5Tg4AoGcjkrSviTYPvio1YYc+u9DMmEGEt/XQBGOJBXj6Ke2UT5Ors2KtfZ0g5Hl4uTlQc4GAmblpCH/9p6ke6NHBh7VNoVAojR0+1BTVaoLjlzSzxYb3bPwl3VoFY1OnTtXaoEAgwA8//KC3Q5SmRblEzg4vpcIdDeNXHYxl8qCoePt+ASqkCjjZWyKktRvn9o2BzyPy9qJ6HvbLKmRQE0AgAJztaTBmCJ3beWDX0Xu4mZTf4PF+FFZJ0bdxSgvryqPy9oaIxpRWyPDP1XQAwJi+QVy5SKFQKCaHj8zY9cR85BRKYGdtgd7hfpzZNRVaCXhcvnwZV65cQUlJCQghj/1Sq9U6O6FWq7F+/Xr06dMH4eHhePnll5Gent7g/sXFxXjnnXfQrVs3REVFYenSpZBKpbXsfffddxg2bBjCw8MxcuRI/Prrr7VsbNq0Ce3bt6/zReGWE5dSIVeoEOjrhNDWVM6+IZj+Ey6V2RguVKsodg/z0eqh2hxx10LenukXc7K3oip0BtK2hTNsrS1QLlHgQXWQ9SSY/Rp7uYi2iC2E8HTVTuXzSRy5kAK5Uo02LZ3pdZJCoTQp2J6xKu6CMUa4Y0CXlrC2bLzCHQxa/QazZs3CkSNHkJycjF69emHkyJEYPHgwbG1tOXHim2++wU8//YRVq1bB29sbX3zxBWbNmoUDBw7A0tKyzv6zZ8+GVCrFjh07UFZWhg8++AASiQSfffYZAGDLli3Yvn07li5dig4dOuDChQv4+OOPIRaLER0dDQCIj4/HmDFjsGDBAk5+B0pdVGqCQ+dTAGiGPDfGXiVjwZThcS1vr1YTXLxdXaLYCFUUGURCAbzdbJGRV4Hsggp4uda99jxUUqRZMUMRiYToGOSOS3dycCMxH21aOj92f0IIkrOaVzAGaAaS5xRKkFVQiQ5B7nrZUChVOHROU8o9pm8QvU5SKJQmBVumKOEmGCsqq8Kl6taLxi7cwaDV8vH8+fPx999/Y/fu3WjVqhXWrFmDp556CnPmzMHJkychl8v1dkAul2P79u2YPXs2+vfvj+DgYKxbtw45OTk4fvx4nf2vXbuGy5cv47PPPkNYWBh69uyJZcuWYf/+/cjNzQUA/Pzzz3jppZcwYsQI+Pv7Y8KECRgzZkyt7FhCQgJCQ0Ph4eFR64vCHVfu5iCvSAIHW0v0jWy88x+MgR+bGeO2TDEhvRhFZVWwsbJA57b6PSyaC4yQRENS4sXVmTEXOmOMEzrXkLh/EkVlVSirlEMoFMCfBwEac8WHg1ljZ2IzUVIug5uTNXp3pmM/KBRK04JrNcUTl1OhVhOEtHJFgE/TuN/oVMvTuXNnLFy4EKdPn8bWrVvh7OyMJUuW4KmnnsLChQtx9uxZncsU7927h8rKSvTs2ZPd5ujoiNDQUFy5cqXO/levXoWHhweCgh7W1UdFRUEgECAmJgZqtRqfffYZxo4dW/sXFQpRVqZpMJfL5UhJSUFgIO1h4pOD1XL2Q7v7w4oHufamBCPdXlIh47Su+mJ1iWK3EK9GP1LgSYqKRdVKiq5USZETmOD9bnIh5ArVY/e9X12i2MLTvll91pmM9pPm3zUEIQT7q+XsR/UOhAUtr6VQKE0MtmeMgzJFlZrg2EVGuKOVwfbMBb2u/AKBAFFRUfj4449x9uxZrF+/HiKRCK+99hr69Omjk62cHE2q0cendgmVp6cn+1pNcnNz6+xraWkJZ2dnZGdnQygUomfPnvD29mZfz8rKwqFDh9C7d28AQFJSElQqFY4dO4Zhw4ahf//+WLBgAfLy8nTyndIwaTlluJFYAKEAGKGlGltzxtZazJbXcSWVTQjB+VsPJe0bO0+aNVZcRpUUuaSllwNcHa0gV6pxL7Xosfs2t34xBl8DM9o3EwuQkl0GK0sRhtMhzxQKpQnCpZritfg85BdLYW8jRq8mVElgcNfbjRs3cObMGZw9exYKhQKurro1HzPCG4/2hllZWaG0tG7juFQqrbePzMrKCjKZrM72goICvPzyy3Bzc8Nrr70GQFOiCAA2Njb46quvUFhYiLVr12Lq1Kn4888/YW2t+8o6IQQSiUTnn+Ma5njWFDQxhf0/TycCALoEe8LeGlodG3Px3VS2vd1sUFwuw4OMIvi51Q4o9LGflluB7IJKiC2ECA1waPBv0FiOu6u95nKVmV/O/i41becXax6I7a2FnH0Wzf2c4dt+WGtX/HcjG1fvZKON70O59UdtJ6ZpgrUWHjacHPvGctyd7TTrmdkFFaisrIRAINDJ/u//aO5F/SN8IYQSEonysfs3hnPGFLb5tk99N4196rtp7HNtW0g01zWFUo3S8kqDbB86q6kk6BvuA5VChkfb0Pg+7rqgy8gTvYKxa9eu4ciRIzh+/DhycnLQunVrPPvssxgxYkSt8kFtYAIfuVxeKwiSyWSwsak7k8ra2rreHjWZTFZHUCQ5ORmvvPIKVCoVdu7cCUdHTW1pdHQ0+vbtWytwbNu2Lfr27YtTp05hxIgROv0OAKBQKBAXF6fzz/FFSkqKyexL5WqcjtVkZEJ91TofF1P6bkrbNiLNVeXmvVS4WZYYbP/Mreq5T56WSElOfOL+5n7cKyo0F/ScwkrcuXO31iyrlJQUZOeVaPYrzUdcHLe9d+Z6zvBt381GU/p5+U4GOreou6rJ2E5IKwQACOXFiIvj7iZo7sddpSYQCACZQo3LsXfgaPuwRPNJ9vNLFYhNKAAAtPOQ63SdNOdzxpS2+bZPfTeNfeq7aexzZVtNNNdJQoD4xAdwsBHpZbtUokRMvKaHubWr7LHXTL6Pu7bUlzyqD62DsUcDsJYtW2LMmDF4+umnERwcrLejTMlhXl4e/P392e15eXn1Ss17e3vj5MmTtbbJ5XKUlJTA09OT3RYTE4PXXnsNXl5e+O677+Dl5VXrZx7N4Hl6esLZ2bne0khtEIvFaNOmjV4/yyVSqRQpKSlo1apVvcGsMewfOp8KhYqgpac9nu4XrvXKgDn4bkrbiYUpiL2fCKXADiEhIQbb33HqIgBgYFQgQkIansPRWI67Wk3w9aFcKFUEnn6t4eFsU8u2TKUJCDoEByE4wNmsfDe2ba7se/pW4c+L/yGrSAH/Vm3YcpOatonAAsUVmlmCfbuHwcFWu5sP374by7anSxFyi6RwcPVFSGtXre2f+0vzINE12AO9u3cyie/GtE99N4196rtp7FPfa2NrnYtKqRJuHj6QV+TpZfvXU/dBCBDaygV9e9R/zeT7uOtCUlKS1vtqFYz179+f7dUaOXIkRowYgbCwML0drElwcDDs7e1x6dIlNhgrKyvD3bt3MXny5Dr7d+vWDatXr0ZqaioCAjQ19pcvXwYAdOnSBQBw8+ZNzJo1C6Ghodi0aRObEWNYt24djh49iqNHj7KBQkZGBoqLi/UOqAQCAWdS/1xgY2PDqz8N2VerCU5czgQAPNM3CHZ2dnX20dc2V/Bp3xDbrXxdAAC5xdIGbWhrP6ewEik55RAKgN4R/rC1fXIfVWM47t5udsjIq0BxhQoBvg9tWVtbo6Ra2t7X04nz38Nczxm+7dva2qKFpz0y8iqQlCWpMx7BxsYGD3I0mTB3Zxt4uTsb6m4d++Z+3P08HJBbJEVhuaqWvcfZL6uU48x1TfXAuIHtdPbDnM8ZU9rm2z713TT2qe+msc+lbXsbS1RKlVARkV62VSo1TsdmAQBG9Ap84s/yfdy1QZcxJVoFYzk5ORAKNbXxTBDzuDd/NHP1OCwtLTF58mSsXr0arq6u8PPzwxdffAFvb28MHToUKpUKRUVFcHBwgLW1NTp37ozIyEjMnTsXH3/8MSQSCZYsWYLo6Gh4eXlBqVRi/vz5cHNzw6pVqyCTyZCfr0lrikQiuLq6YsiQIdi2bRs+/vhjTJ8+HQUFBVixYgUiIyN1FiCh1CbmXi6yCythZyPGACpnrxOMvH1mfqVOtcb1cfG25kEvLNAdTvZNR9DC192+etZYJcLbPdwuqVJCrtQouTrTOWOc0rmtBzLyKnAjMb/eWXXJjHiHb/MS72DwdbdDbLymb0xbjl5IgVyhQqCfEzoEuvHoHYVCoZgee1sxcouAyiol9NE7vhqXi4LSKjjaWeKpTo1fkOxRtArGHpWJ55rZs2dDqVTiww8/RFVVFbp164Zt27ZBLBYjIyMDgwYNwsqVKzFu3DgIBAJs3LgRS5cuxbRp02BlZYXhw4dj4cKFADRZsdRUjezl4MGDa72Pn58fTp06hQ4dOuDbb7/FV199hXHjxsHS0hKDBg3Ce++9RwduGsjBs5rhpUOi/GFt1finohsTbzdbCAWAVKZESbnMoHlZF6pVFBvzoOf6YOc6PSJvzwx8trW2gLUlPe+4pHNbDxw69wA3Gpg3xgRjrf2axrwXXfHxqP+cbAiFUo1D5zRjP+iQZwqF0hyws9aUuEuqFKj+r04crZazH9TNv9GP6akPrZ5aVq5cyasTIpEICxYswIIFC+q81qJFC8THx9fa5ubmhvXr19drKzIyss7+9dGzZ89as80ohpORV47Y+DwIBMDIXlTOXlfEFiJ4utoip1CCzPwKvYOx4vIqxKVo1O16dGhawRgjb//orLGSCo2ojwudMcY5Hdu4QygAMvIqUFAihbtz7Tr85CxNMBbUzGTtGZhh5A3Nv3uU/65noqhMBldHK/QJb7iXk0KhUJoKrLx9lRKuOgZjeUUSxNzLBYAmOwKETpikcMahc5qsWNcQL3i76d4rRnn4YJep59wiALh0OweEAG1aOsPDxbQNrFzj48ZkIWqXhDH9Yq4GZBMp9WNvI0abls4AUCc7plSqkZZTDgBo3YzLFAFNZowQ8th9CSHYf0YjzTyyVyDEFvQWTKFQmj7s4Gfp48d31MfxS6kgBOjUxp2d7djUoHcCCidIqhT4+0o6AGBU70ATe9N4eTjYWP/Bzxeq+8WeamIlisDDIbvZBRKo1A8ffIsr6MBnPunc1gMAcP2RYCwjvxJKlRq21hbwcjUfASNj4ulqC6FQALlChaKyqsfue/t+IZKzSmEpFmF4z1bGcZBCoVBMzMPMmG6Dn5UqNU5c1pQoPv1UK67dMhtoMEbhhFNX0yGVKeHnYY/w6gc3iu48FPHQLxirlCpws/qBuamVKAIaxT4LkRBKlRqFJQ/nWZWU0zJFPglvp/lM30zMr5X9Sa2RFWuuvU8WIiEbiGY9IaP9Z3VWbFDXlnC0M3wEAIVCoTQGmMyYpEq3zNjlOzkoKpPB2d4K3cOa3jMNAw3GKAajVhNWuGNU79a1hvFSdIPJ/DxahqctV+I0c7haetmjpZcDl66ZBSKhAN5u1Q++NY7RwzJFmhnjg+AAV1haCFFUJkN6bjm7PSVb8//m2i/G0JCwTE0y8ytwJU4zx3J0X1o9QKFQmg92epYpHr2QAgAYHOXfpMu6m+5vRjEa1xPzkZlfARsrCwzs2tLU7jRq/Boow9OWi9Uqik0xK8bAPPjWFExgBTxozxgvWIpFCK2WYK9ZqpjSzPvFGHzZc7LhRZS//tUMLO0a4oUWnk1voYRCoVAaws7moZqituQUVuJaguZ+M6yJCncwaKWmeOXKFZ2MduvWTS9nKI2Tg2c1Ms2DurWErT6apRQWd2cbiC2EUCjVyC+W6CSEIlOocLVacaipSdrXRCNyklsrC8FI27vSMkXeCG/rgesJ+biZWIDBXXxACEFqdWYssJlnxhjhnYYyY+USOf6+qumpje4XZDS/KBQKxRxggrEKHcoUj1XL2Ue082jyonBaBWNTpkxpsB+A6R+o+XpcXBwHrlEaAzmFlbgapwkAqHCH4YiEAvi42yEtpxxZ+ZU6XYCux+dBJlfB3dkGbVo48+ekiak3M1ZOBTz4hhHxuJlUAJVKjZJKFSqrlLAQCZpkSawusGWKDfR6Hr2QAplchVY+jujUxt2YrlEoFIrJYeeMSbXLjCmUapy8nAagaQt3MGgVjO3cuZP9f1ZWFhYvXoxnn30WTz/9NDw8PFBSUoJTp05hz549WLZsGW/OUsyPQ+cegBAgsr0nW2JHMQw/D3uk5ZQjM78CkcGeWv/c+RqDnpuymMJDKXHNg69CSVBZvdpGyxT5o7WfExxsxSiXKHA/sww5xZqbqr+XY5Ou5dcGdv5doQTqR8qLFUo121Mb3Y8OeaZQKM0Pe9uHc8a04dKdbJRUaOYxdgv15tM1s0CrYCwqKor9/5QpUzB9+nS88847tfaJjIyEtbU1vv/+e4wYMYJbLylmSZVMiRPVKxejetMhz1zh+4RV9vpQqdS4clcjDtCUSxSBmpkxzYNvRZUKACC2ELKKTRTuEQkF6NTGA+duZuFWchFyizV9eq39HE3smenxcrGFqFrenimZZTh3IxNFZVVwdrBC3wg65JlCoTQ/aqopPmkeIwAcOZ8CABgSFQALUdNf7NP5N7x58yZ69uxZ72sRERFISEgw2CmK/iRnluHXs4U4fCHtiTNvDOWf2AxUShXwcbNDl2AvXt+rOaGPvP3t5EKUSxRwtLNEaGs3vlwzCzxcbGEhEmjk7cuqUCHVBGMuDlY068AzndtqSuxu3y9iM2OBzVy8AwBEIiE8q+Xtswsl7HZCCP78VyNnP6pXa4gtRCbxj0KhUEwJU6aoUhMoVI8PxrLyK3AzqQACATC0e9MW7mDQORjz9vbGf//9V+9rR48ehb+/v8FOUfQnNiEfd9Kk+OFwPGYsO4YPNp3DsYupqJDIOX0fQggr3DGiF5Wz5xJG3j7zMTLZj3KhukSxe5g3RE38byESCuDlqsmO5RRKUF6lBkBLFI1B5+p5Y/HpJcgo1FxTmrt4BwOT0c4tehiM3UkuxP2MUlhaCOmQZwqF0myxshSxzyZSufqx+x6tFu7oEuzFLnI1dbQqU6zJjBkz8PHHHyMvLw8DBgyAi4sLCgoKcPToUZw+fRpr167lw0+Klozp0xoVpUVIyiNITC/FzaQC3EwqwObfbyCyvRf6Rvihe5g3rK10/tPX4s6DYqTllMPKUoTBUTQA5xKm/yS/WAK5QgVL8eNX09Vqgou3H/aLNQd8PeyQmV+BnEJJrcwYhV983Ozg6WKDvGIpKqtXN5u7rD2Dr4c9Yu7lIbtQAt9q3Z391VmxAV1bwsmenp8UCqV5IhAIYGcjRlmlHFXyhjNjCqUKf1+pFu5oRgtYOj+Rv/DCC1Aqldi0aRMOHTrEbvfx8cHq1avx9NNPc+ogRTfEFkJ0b2+P6dEhKJMS/Hc9E/9ey0RKdhku383B5bs5sLIUoXuoN/pG+CEy2FOv0pmjFzUyzQO7tKR9OhzjbG8FW2sLSKqUyC6sRID343tykjJKUFhaBRsrEat419Rh+sZyiqQPgzGaGeMdgUCAzm092F5RTxcbVrK4ucNkxnIKJYC/FXKKJLh0R9PHOaYvlbOnUCjNm4fBWMOZsfM3s1FWKYe7kzW66CBg1tjRKz0yefJkTJ48GcnJySgtLYWLiwtatWrFsWsUQ/F2s8P4Qe0wflA7pOaU4d9rmfj3WgZyCiX493om/r2eCTsbMZ7q6IN+ES3QoY27ViVuJZVKXL2XBwAYSYU7OEcgEMDXwx5J6SXIyq94YjB2/mYWAKBriPcTs2hNBWauU06hBCqFJhhzpcGYUagZjLXyad6S9jVhFwgKpQCscORCGggBugR7NnvpfwqFQmEW7qsUDQdjRy6kAND0iomagXAHg961aqWlpXjw4AHy8vIwbNgwJCcno3Xr1rSB3kwJ8HbElKcdMXl4MBLTS3DmWgbOXs9EUZkMJy6n4cTlNLg4WKF3uB/6hvuhfYBLg3/LK4mVIATo1Mb9iYECRT/83DXBWGb+4/vGCCFsv1jPDs2jRBGo+eArgY24umeMDnw2Cp3aPpyT1cqbBhkM7AJBkQQSmSP+idXMX6RZMQqFQnk4+LmhzFh6bjnuJBdCKACGNBPhDga9grFNmzZhy5YtqKqqgkAgQKdOnfDll1+iuLgY27dvh6MjfUA3VwQCAdr5u6CdvwteeqYD7iQX4N9rmTh/MwvF5TIc+C8ZB/5LhqerLfqG+6FfZAu08nn495QrVIhN0gQIdMgzf/h5aCdvn5ZbjqyCSliIhOgS0nxS+qxYQrEUrvaabCAd+GwcXBysEejrgOSscrRtSfvFGDxdbCASCqBQqnHmVhlkchUCvB0Q3q55lA5TKBTK43gYjNXfM3b0YgoAoFuoN9ydbYzlllmgcw5w165d2LBhA2bMmIFffvmFnRcwefJkpKen46uvvuLcSQo/MHOD3hwfjh8+Go4lM7ujX0QLWFuKkFckwb5TiXhr9T9444tT2HsiHtkFlTh3MwdSuRoeztaICmv6g/hMBaOomPUERcWL1Vmx8HYesLVuPr07Hs42sBBpHnzzSzUS6640M2Y0Zj/fEc/1ckXHIFdTu2I2iERCeFUrf11O1CyijOlLhzxTKBQKUKNMsZ7MmEyhwqkrGi2C5qg8q3Nm7Mcff8Qrr7yCt99+GyqVit3er18/zJkzB1u3bsXixYs5dZLCP2ILIbqFeqNbqDeq5EpcuZOLM9cyEHMvD2k55dh19B52Hb0HS7Emfh/avWWTl1A3JdrOGrvQzFQUGTQPvhpFRXX1IhvNjBkPHzc7dAiwpYHGI/h62COrQFPG7WRniX6RLUztEoVCoZgFzKyx+nrGzt3IQoVUAU8XG0S0bz5VPgw6B2NZWVmIioqq97XAwEAUFBQY7BTFtFhbWqBPhB/6RPihQqrAhZtZ+Pd6Jm4m5kOuUEMsEmBgpJ+p3WzSMPL2JeUyVEoVqO+RN7dIgvsZpRAKNPPFmhs+7nZssCoQaFQoKRRTwpTPAsCQqBbNRlCHQqFQnoS9bcOZsaOMcEePgGa50K9zMObj44Nr167hqaeeqvPa7du34ePTvFbomzr2NmIM6R6AId0DUFxehbPX0qCUFLAfKgo/2FqL4eJgheJyGbIKKuDnVjfQYGaLhQa6NcsZRr4edkCc5v+OdpbNSnmJYp4wwjIiITA0qqWJvaFQKBTzge0ZU9TuGUvNLkNcShFEQgGGRDUv4Q4GnZ9ennvuOWzevBnbtm1DSkoKAEAikeDYsWPYsmULxo4dy7WPFDPBxcEag7q2gL9H83vwNwW+bKli/X1jzVFFsSa+bg+zEM72lib0hELR0C3UG26OVujbwRFO9JykUCgUFqZMUfpIZozJikWFeTfbETU6Z8ZefvllZGRkYPXq1Vi9ejUAYOrUqQCAZ555Bq+++iq3HlIozRQ/D3vcSS7UKCoG1xZKKCmX4e6DQgBAj2bWL8bgUx2sAoCLA10goJgeL1dbfLOgL+Li4kztCoVCoZgV9UnbV8mV+Cem+Qp3MOgcjAkEAixbtgwvvfQSLl68iJKSEjg4OKBbt25o164dHz5SKM0SRt6+PhGPS3dyQAjQpoUTPF1sje2aWVCzP8eZBmMUCoVCoZgt9akpnr2eicoqJbzdbBHetvmOAdE5GLty5Qo6duyIVq1aoVWrVrVey83Nxa+//oo333yTK/8olGaLT/UQ2fpmjV24lQUA6NnR16g+mRMezjYQiQRQqQhcmmHPHIVCoVAojYWHPWMPg7GjF1IBAMN6tIKwGQp3MOjcMzZlyhRMnDgRWVlZdV7LycnB119/zYljFEpz52FmrJKd5wcAlVIFbiRqVEubm6R9TUQiITyrB0M6OdD+HAqFQqFQzBUmMyZTEKjVBMmZpYhPK4aFSIDB3fxN7J1p0Ut+LDc3F+PGjcPFixe59odCoVTj424HgQCQypQorZCz26/G5UKpUsPPwx4tvRxM6KHpCQ5wBgC09nE0rSMUCoVCoVAahMmMEQJUyVWscEePDj7NvtVAr2BszZo16Nq1K2bNmoUdO3Zw7BKFQgEAsYWI7QfLKpSw25lBz091ar5ZMYZZz4Rg9mhvNiijUCgUCoVifliKRRBbaMKOwtIqnI7NANC8hTsY9ArG7OzssHHjRsycOROrVq3Cu+++C7lcDoGg+dZ7Uih84FetGJhdoJG3lytUiInLBaBZTWruWFgI4Wqvc+srhUKhUCgUI2Nrrblfn7iSAalMCT8PO3Rq425ir0yPQVNS586dizVr1uD48eOYOHEiSktLufKLQqGgerAxgJxCKQDg1v0iVMlVcHeyRtuWzib0jEKhUCgUCkV7mFlj/8RmAtAId9BEjoHBGACMHDkSO3fuRH5+Pt5++20ufKJQKNWwmbFCTWbsclweAM1sMXoBo1AoFAqF0liws9FkxuQKNSxEQgzs2tLEHpkHOgdj3bp1g52dXa1tnTp1wq+//orAwEDOHKNQKIAvW6YogUpNEHMvH0DzVlGkUCgUCoXS+LCzfthW0KuTL5zoWBoAeswZ+/HHH+vd7uXlhZ9//hl5eXkGO0WhUDQwmbGcIglS8mxQLlHAwdYSYa3dTOwZhUKhUCgUivbY1gjGhvcMMKEn5oVWwdiff/6Jfv36wcXFBX/++ecT9/fz8zPULwqFAsDd2QZiCyEUSjUu3dMMf+4e5g2RyOAKYwqFQqFQKBSjwfSM+XnYISyQLiozaBWMvf/++/jll1/g4uKC999//7H7CgQCREdHc+EbhdLsEQkF8HG3Q1pOORKyqgAAPamkPYVCoVAolEZGh0BXnIrJwHMDAmnfew20Csb+/vtveHh4sP+nUCjGw7c6GAMAa0sRwtt6mNgjCoVCoVAoFN3o0cELDgI/hIV5m9oVs0KrYKxm2SEtQaRQjAvTNwYA4W3dYSkWmdAbCoVCoVAoFP0QCmlG7FG0CsYWLlyotUGBQIAVK1bo7RCFQqmNb41gLCrU04SeUCgUCoVCoVC4RKtg7NKlS1obpDWgFAq3MJkxkRCIaEcn1VMoFAqFQqE0FbQKxk6dOsW3HxQKpQGCW7liSLcWsERFLVlYCoVCoVAoFErjhlN9bIlEgn///ZdLkxRKs0ckFGDW6BB0a2v/5J0pFAqFQqFQKI0GnZfZMzMz8fHHH+Py5cuQy+X17hMXF2ewYxQKhUKhUCgUCoXSlNE5GFu5ciViY2Mxfvx4xMbGwsbGBuHh4Th37hwSEhKwYcMGPvykUCgUCoVCoVAolCaFzmWKV65cwdy5c/Hhhx9i3LhxsLKywoIFC/Dbb7+hW7duOs8hU6vVWL9+Pfr06YPw8HC8/PLLSE9Pb3D/4uJivPPOO+jWrRuioqKwdOlSSKXSWvscOXIEI0aMQKdOnRAdHY0LFy7obINCoVAoFAqFQqFQ+ETnYKyyshLt27cHAAQGBuLu3bsAAJFIhEmTJuHixYs62fvmm2/w008/4ZNPPsGePXugVqsxa9asBksgZ8+ejdTUVOzYsQNfffUVzpw5g48//ph9/eLFi1iwYAFeeOEF/PHHH+jZsydeeeUV3L9/X2sbFAqFQqFQKBQKhcI3Ogdjnp6eKCgoAAAEBASgtLQU+fn5AABnZ2cUFhZqbUsul2P79u2YPXs2+vfvj+DgYKxbtw45OTk4fvx4nf2vXbuGy5cv47PPPkNYWBh69uyJZcuWYf/+/cjNzQUAfPvttxg8eDCmTp2KoKAgvPfeewgLC8MPP/ygtQ0KhUKhUCgUCoVC4Rudg7F+/frhyy+/xLVr1+Dn5wdvb29s374dFRUV+O233+Dl5aW1rXv37qGyshI9e/Zktzk6OiI0NBRXrlyps//Vq1fh4eGBoKAgdltUVBQEAgFiYmKgVqsRGxtbyx4AdO/enbX3JBsUCoVCoVAoFAqFYgx0DsZmz54NR0dHfPXVVwCAuXPn4ocffkC3bt1w4MABzJgxQ2tbOTk5AAAfH59a2z09PdnXapKbm1tnX0tLSzg7OyM7OxtlZWWQSCTw9vZu0N6TbFAoFAqFQqFQKBSKMdBZTdHFxQW//vor8vLyAACjR4+Gr68vrl+/jk6dOiEqKkprW4xohqWlZa3tVlZWKC0trXf/R/dl9pfJZKiqqmrQnkwm08qGvhBCIJFI9P55rmCOKV+CJHzap76bxj713TT2qe+msU99N4196rtp7FPfTWOf+m5828awrwuEEAgEAq321TkYY/D09GT/37VrV3Tt2lVnG9bW1gA0vWPM/wFAJpPBxsam3v3rE/aQyWSwtbWFlZUVa+/R1xl7T7KhLwqFwqzmq6WkpDRa+9R309invpvGPvXdNPap76axT303jX3qu2nsU9+Nb9sY9rWlvuRPfegcjMnlcuzatQuxsbEoKyur87pAIGDFMp4EUy6Yl5cHf39/dnteXh6r2FgTb29vnDx5so4/JSUl8PT0hLOzM2xtbdmsXU17TC/bk2zoi1gsRps2bfT+ea6QSqVISUlBq1at6g1ozdk+9d009qnvprFPfTeNfeq7aexT301jn/puGvvUd+PbNoZ9XUhKStJ6X52DsWXLlmHfvn1o27YtnJ2d67xOCNHaVnBwMOzt7XHp0iU2GCsrK8Pdu3cxefLkOvt369YNq1evRmpqKgICAgAAly9fBgB06dIFAoEAkZGRuHz5MsaPH8/+3KVLl9jM3ZNs6ItAIDAos8Y1NjY2vPrDp33qu2nsU99NY5/6bhr71HfT2Ke+m8Y+9d009qnvxrdtDPvaoG2JIqBHMHbixAm89dZbeOONN3T90TpYWlpi8uTJWL16NVxdXeHn54cvvvgC3t7eGDp0KFQqFYqKiuDg4ABra2t07twZkZGRmDt3Lj7++GNIJBIsWbIE0dHRbOZrxowZeOWVVxAaGoq+ffvit99+Q1xcHD799FMA0MoGhUKhUCgUCoVCofCNzmqKQqEQERERnDkwe/ZsPPfcc/jwww8xceJEiEQibNu2DWKxGNnZ2ejduzcOHz4MQBNlbty4ES1atMC0adMwZ84c9O3bt9bA5t69e2PFihX4+eefMXbsWFy8eBGbN29mpey1sUGhUCgUCoVCoVAofKNzZiw6Ohr79u1Djx49IBTqHMvVQSQSYcGCBViwYEGd11q0aIH4+Pha29zc3LB+/fon+hgdHd3g69rYoFAoFAqFQqFQKBQ+0TkYmzNnDqKjozFs2DCEhYXVaZATCARYsWIFZw5SKBQKhUKhUCgUSlNE52Bs9erVePDgAWxsbHDz5s06r+vSsEahUCgUCoVCoVAozRWdg7G//voL06dPx7vvvstJmSKFQqFQKBQKhUKhNEd0jqZUKhUGDBhAAzEKhUKhUCgUCoVCMQAB0WUwGICFCxfCysqKqg/WIDY2FoQQrSdt8wkhBAqFAmKxmJeSUT7tU99NY5/6bhr71HfT2Ke+m8Y+9d009qnvprFPfTe+bWPY1wW5XM7OP34SOgdje/bswerVq9GmTRtERETAzs6utkGBgJMZZI2Ja9eugRACsVhsalcoFAqFQqFQKBSKCVEoFBAIBFqNA9M5GAsODn68QYEAcXFxupikUCgUCoVCoVAolGaHzsGYWq2m/WIUCoVCoVAoFAqFYiA6R1WjR4/GP//8w4cvFAqFQqFQKBQKhdJs0DkYy87OrjPomUKhUCgUCoVCoVAouqFzMPbMM89gx44dyMvL48MfCoVCoVAoFAqFQmkW6Dz0OSUlBVevXkW/fv3g7OwMW1vbWq8LBAKcPHmSMwcpFAqFQqFQKBQKpSmiczDm4+ODZ555hg9fKBQKhUKhUCgUCqXZoLOaIoVCoVAoFAqFQqFQDEfnzBjDv//+i8uXL6OsrAwuLi7o2rUr+vTpw6VvFAqFQqFQKBQKhdJk0TkzJpfL8frrr+Ps2bMQiURwcXFBcXEx1Go1evTogS1btsDS0pIvfykUCoVCoVAoFAqlSaCzmuKGDRsQExODzz//HDdv3sTZs2dx48YNrFy5EtevX8emTZv48JNCoVAoFAqFQqFQmhQ6Z8YGDBiAyZMnY+bMmXVe27ZtG37++WeqpkihUCgUCoVCoVAoT0DnzFhRURFCQ0PrfS00NBS5ubkGO0WhUCgUCoVCoVAoTR2dgzF/f3/ExMTU+9qVK1fg4+NjsFMUCoVCoVAoFAqF0tTRWU3xhRdewKpVq2BtbY2RI0fC3d0dBQUFOHjwIL799lu8+eabfPhJaYJUVFTAwsIC1tbWdV7Ly8vD0qVL8fXXX+ttPz8/H19++SViY2OhUCjwaEXu33//rbPNrKwsrff19fXV2T6FQuGHrKwsODo6wt7eHhcvXsTx48cRGRmJUaNGmdq1Jgkf11+KeVBSUgKZTAZbW1s4ODhwavvPP//EiBEj6gjBSSQS/PLLL5g+fTqn72fuFBcX49ChQxgzZgwcHBygUqnw5Zdf4vTp03B3d8f//vc/dO/e3dRuUgxE52Bs4sSJuHv3LlavXo01a9aw2wkhGDt2LF555RVOHaTUz8KFC7Xed+XKlXq9R3FxMf755x+UlZWhV69eaNu2ba3XJRIJtm/frnMAXlRUhIULF+Lff/+FQCDA0KFD2QAfAPbu3YvVq1dDoVDo5TfD4sWLcfv2bYwcOZKzG8bAgQMhEAgeuw8hBAKBAHFxcTrb37hxo9b7muvCh0KhQGVlJZydneu8plarkZOTo3Ogqs1xZ9DnIS8lJQUHDhxAaWkp+vbti759+9Z6vaKiAp9++qlen6U///xT632jo6N1ts8nfPpuzONy4sQJzJ07F1u2bEHLli0xa9YstGzZEr///jtKS0vx4osv6mSP788p3+e7MeDj+jt16lSt9925cycn78k1crkc165dQ1lZGbp27QoXF5dar8tkMhw5ckTnc57vc/LixYv47rvvEBMTg6qqKna7ra0tIiMj8fLLLyMqKkpnu4DmmYCxuXDhQrRt27bOcbl79y7Wrl3LaTD2yiuvYPny5fD09OTMJpekp6dj4sSJKC8vR58+feDg4IAVK1bgp59+wtChQ+Ho6IhXX30V27ZtQ5cuXUzt7hORSqVISEiod3GmW7duJvLKPNB76HNSUhI7Z8zJyQlRUVEICgri2j9KA0yZMkXrfX/88Ued7SclJWHatGmQSCQANDeI6dOn491332X3KSgoQJ8+fXQOOt599138888/mD59OiwtLbFz506MGjUKs2fPxty5c3H69Gl07doVy5cvR6tWrXT2nSE8PBzfffcdunbtqreNR7l8+bLW++pzYxo4cKBW+wkEAoMewuRyObZv346nn34aAQEB+OCDD3D48GFERkZi9erVdW6E2iCTybBs2TL89ddfUCqV6NixIz766COEhYWx++h7zmzYsEHrh1NdHzRiYmIwc+ZMeHp6QiAQIC0tDUOHDsUXX3zBrs7q6zcABAcHa7WfvgF8TbjORvDpuzGPy9ixY9G3b1+8/fbb2LRpE/bv34+jR4/i6NGj2LBhA44cOaKTPb4/p3ye7wz79+/Hvn372AWI//3vf7C3t2dfLyoqwvjx4/W+zvBx/TXGIiSDWq3GgQMHGvws6WM/OzsbL7/8MpKSkgAANjY2mD9/fq3FAH2vNXyek4cPH8a7776LYcOGoUePHvD09ISVlRVkMhny8vJw8eJFHDt2DGvWrMGwYcN0sg1oFmbef/99CAQCdjHzUQgh6NevH7Zs2aKz7Yb46KOP8Pbbb8PV1RWA4Ys+XJ8z7733HrKysvDNN9/AwcEBJSUl6N27N/r1f3KNIAABAABJREFU68dWDW3evBlXr17Fd999Z5DvfGey//77b7z//vuoqKioY5uLa/zVq1cb9N1cF65rovfQ5zZt2qBNmzZc+kLRAX0CLF1YtWoVunTpgtWrV0MoFGLnzp1Yu3YtSkpKsGLFCoNsnz9/Hh999BFbHtSlSxfMmTMHaWlpuHTpEpYsWYJJkyYZ/DvY2trCzc3NYDs10XflT1tOnTrFq32G1atXY//+/ejTpw/+/fdf/PHHH5g9ezZOnz6Nzz//XK8Hja+++grnzp3D8uXLIRAIsGPHDkyaNAkbN26sNRBen/Wft956S+ef0ZY1a9bg2WefxeLFiwEAx44dw6JFi/D6669j8+bNsLDQ+zIJALh37x4XbmoF19kIPn035nG5f/8+Nm7cCKFQiHPnzqFfv34QCoUIDw9HZmamzvb4/pzyeb4DwK+//oqlS5dizJgxEAqF2L17N06cOIHvv/+ezVqr1WqdyrIfhY/rr6EBli6sWLECu3fvRnBwcK0g1RBWrVoFFxcXnD59GgKBAFu3bsXy5ctRVFRk8N+cz3Ny48aNmDdvHl566aV6Xx8/fjzCwsKwfv16vYKx6Oho+Pn5Qa1WY9q0aVi/fj2cnJzY1wUCAWxtbdGuXTudbS9dupTNutV37/n888/Z9zA0GOP6nDl//jzWrVvHXsvPnz8PpVJZy8/evXtj+/btBr8XH5nsmqxevRo9e/bE66+/zrn9r7/+Ghs2bGDL0GsiEAiaTjCmy2qUQCAw+GGdojtKpRKFhYVQqVQANBcduVyOW7duYfTo0Trbu3nzJn7++Wc2M/DSSy+hZcuWmDNnDpydnWtlyHSlpKQEERER7PeRkZEoLCxEXFwc9u3bx1mGdcyYMfjuu++wbNkyiEQiTmzWRCaTYe/evUhISGCPO6DJOt2+fRvHjh3j/D0Z+7du3TKoLOHo0aNYu3YtwsLC8NFHHyEqKgr/+9//0Lt3b7z88st62/zkk0/YwGvEiBFYsGAB3nrrrVplFNqu+D+Oe/fuISEhAWq1GkDt83358uU62YqPj6/1kDds2DB4eHhg5syZeO+992qVY/NJTk4OvL29DbLBlBJxmY3QBi5859O2o6MjysvLUV5ejps3b7LneFpaWr3ltFzAxeeUgcvzHdCU8C1ZsgTPP/88AOC1117Dyy+/jClTpmDPnj3w8PAw2Ge+r7+AJnv34MGDeo/La6+9ZpDtAwcOYMWKFRg7diwXrgLQiJx999137Pm8ePFitGrVCp9++imcnZ11qnjRB33PyaysLAwYMOCx+/Tv3x9fffWV3r4xZWo7d+5EREQEKioq2AqNmzdvIjg4WK/z6Pfff8f8+fPh6OiIVatWwcvLi30tIiICf/31F1q2bKm33zXh+pwpLi6Gn58f+/3Vq1chFAprLQq7uLhAJpMZ/F583zsyMjKwZcsW+Pv7c277559/xty5c/Hqq69ybttYaBWMXbp06Yn7FBcXQyqV0mDMBJw9exbvvfceioqK6rxmbW2tVzBmaWlZ5wM+ZMgQfPjhh1i6dCk8PT31bnxXKpV1RDssLS2xZMkSTktdS0pKcPDgQZw+fRotW7as0xBsaE/B8uXL8eeffyI0NBS3bt1CREQEUlNTUVhYyEld++3bt7F48eJaD2E1MSStX1JSwh7rc+fOYcKECQAAZ2fnWv0AulBcXIyAgAD2ewsLC6xevRqvvvoqXnvtNfz000+cPPh+//33+OyzzwCALWth/q/PjcTe3h6FhYW1fI+MjMQXX3yB2bNnw93dXe8A9VHS09Px2Wef1QrgmQfIoqIi3L171yD7fGQjGPj0ne/j0q9fPyxZsgR2dnZwcHBAr169cP78eXz88cfo37+/Qbb5/JwC3J/vgObBqGfPnuz3vr6++OGHH/DCCy9g1qxZ2L17t0E+A/xff//66y98+OGHkMvldcrb/Pz8DA7G5HI5530sKpUKVlZWtbZNmTIFhYWFWLlyJdzd3Tl5T67PyaCgIBw4cACzZ89ucJ8//vjDoLYCBm9vb4waNQqDBg1iF31feeUVuLu749tvv9VZsbt169bYu3cv1q9fjzFjxmDJkiUYMWKEwX7WB9fnjKurK/Ly8tjf+fz58wgJCamVNYyLi4O7u7vB78XnvQMAWrVqhZycHF6CsfLy8kYvxKSVtP2pU6ca/Dp+/Diio6OhUCjg7u6ODRs28O0z5RHWrl2L0NBQbNmyBdbW1ti4cSMWLVoEe3t7fPHFF3rZjIqKwqpVq5Cfn19r+8SJEzFz5kx89tln2Lp1Kxfus/DRczhq1Cj07dsXrVu3hp+fX60vQ/n777+xcuVK7N27F35+fvjkk0/wzz//YNCgQQaLjwCakhyRSIQPP/wQYrEYixcvxrRp02BhYYG1a9caZNvf3x+3bt3CnTt3kJGRwWazTp48iRYtWuhlMygoCEePHq21TSQS4auvvoKPjw9mzZqF5ORkg/wGgN27d+Pll1/GjRs34OLigjNnzmD//v0ICgrCoEGDdLbXr18/LF26FNevX6/1dxs8eDAWLVqEH374AcuWLTPYbwBYtmwZ4uPjMWzYMOTm5mLkyJEICwtDQUEBPv74Y4PtM9mImplaruDTd76Py+LFixEZGQlbW1ts2rQJlpaWiImJQXh4ON577z2DbPP5OQW4P98BwNPTE3fu3Km1zd3dHVu3bkVOTg5ee+01vRdlasLn9Xfz5s0YOXIkDh06BAcHB+zbtw9ff/01PD09OSnz7NOnD86cOWOwnZqEh4dj48aNkMvltbbPmTMHI0aMwLvvvosDBw4Y/D5cn5Pz58/Htm3bMGHCBKxbtw579uzB77//jj179mD9+vWYPHkyduzYgQULFhjs+4oVKxAQEIAZM2aw2w4fPgwfHx+9y1QtLCwwb948bNiwAatXr8b8+fNRXl5usK+PwvU506dPH2zatAkVFRX466+/kJKSgqeffpp9XSKR4JtvvkGvXr0Mfi8+7x2A5hz65JNPcOrUKaSkpCArK6vWlyFERkbi2rVrHHlqIogB3L17l4wZM4YEBweTd955h5SUlBhijqInHTt2JHFxcYQQQiZOnEjOnz9PCCFk37595IUXXtDLZnZ2Nhk1ahQJDg4m//77b53XP//8c9K+fXsSHByss+3g4GBSWFhYa1t4eDhJS0vTy1dTERYWRjIzMwkhhPzvf/8jBw4cIIQQcvPmTTJw4ECD7YeHh5MbN24QQgh57rnnyJUrVwghhHz//fdk+vTpBtn+448/SIcOHUinTp3ItGnTCCGEbNy4kYSEhJA//vhDL5t///03CQ0NJS+99BK5d+9erdfy8vLI8OHDSceOHfU6Z2oSFhZGUlNTCSGEzJgxg5w4cYIQQsh///1HRo4cqbO9kpISMmPGDBIcHEzOnDlT5/Xdu3eTsLAwg/0mhJDIyEhy8eJFQgghY8aMYf++a9euJa+//rrB9t9//33SoUMH8tRTT5EJEyaQKVOm1PoyV9/5Pi58wufnlBDuz3dCCPn2229JVFQU2bJlC8nJyan12tWrV0lERAQZMmQIJ+c8X3To0IEkJSURQgiZPHky+9k9duwYGTt2rMH2t23bRjp37kzeeOMNsnbtWrJhw4ZaX/qQmJhIevXqRbp06UIuXbpU6zWFQkHmzJmj9321Jnyck0lJSWTx4sVkxIgRJDIykoSGhpKIiAgyYsQIsmTJEpKYmGiQzwxdunQh9+/fr7M9ISGBdOvWzWD7paWlZN68eaRv374kLCyM0+cOrs+Z3NxcMnToUBIcHEzat29Pnn32WSKTyQghhPz000+kV69epHfv3nU+w/rA572DEEJCQ0NJ+/bt2fOb+eLifN+3bx+JiooiK1euJHv37iV//PFHra/GgF6d6UqlEl9//TW+/fZbODs7Y+PGjXqv0FEMRyQSsQ2RAQEBSEhIQM+ePdGjRw+2vEVXvL298dtvvyEmJqbeptkFCxagb9++OHTokM62CSF44403IBaL2W0ymQzz58+vU8JhaClLdnY2du/ejYSEBFhYWKBt27aYMGECJzPAXF1dUVhYCF9fX7Rq1QoJCQkANDXcBQUFBttXq9Vs7wbzd+3atSsGDRqks6LUo0RHRyM4OBgZGRmsjHvHjh2xbdu2WuVLujBw4ED88MMP2Lt3b51GaQ8PD+zduxcrVqzQWbnuUWxtbdnVO39/fyQlJWHw4MEICgrSS4zByckJ27dvR1paGquqVZNJkyahZ8+eOH78uEF+A5oyFqZMo3Xr1oiPj0enTp0QHR3NWb8IX+UafPpujOPCl9oWn59TgPvzHQBmzpwJhUKB3bt3o1OnTrX6aLp06YIdO3Zg/vz5BvvO5/XX0tKSLX0MCAhAYmIi+vbtiw4dOiA1NdVg+7t27YKrqyvu3r1bp0xWX1GANm3a4ODBgzhx4kSdCgQLCwusW7cOffr00eu+WhM+zsmgoCDOKgQeh4WFBcrKyupsl0qleok/PYqjoyPWrFmDP//8E7///nudZw5D4Pqc8fT0xIEDB3Du3DkIhUI89dRT7HOThYUFRo0ahRkzZtT6/BoCn6V+33//PW+2P/jgAwDAjh076rzGhTCLMdA5GLt79y4WLlyI+Ph4jB49Gh9++CEcHR358I2iJW3btsWpU6cwZcoUBAYGIiYmBtOmTUNOTo5Bdi0tLdGzZ0989dVXGDduXJ0m1+7du+s1bLC+5lYuylYeJT4+HpMnT4a1tTU6deoEtVqN33//Hbt378bPP/9cZ26arvTt2xdLly7FypUr0aVLF6xYsQJDhgzB4cOHOREzCAgIQExMDEaNGoXAwEDcunULgKY++tEyF30IDg6uJS/+6GwtfejatSu6du2K9PT0Oq8xDdSG9pRGRkZi69atWLJkCUJDQ7Fv3z688soriImJgZ2dnd52/f398e6772LcuHHo0aNHrddat27NSXOwn58fEhIS4OPjg9atW7O9G2q1GpWVlQbb51Ntjk/f+T4ufKpt8f055eN8FwgEeO211xrsq+rUqROOHDmCmzdv6u0339ffDh064Ndff8W8efPQrl07nDlzBjNnzkRSUlKthT594Uud0NnZGePHj8e+ffvw9NNP1/kbjhs3DuPGjTPoPfg4J/mcxViTvn37Yvny5Vi7di27QJOeno6VK1fWUuU1lOjoaM4f0vk4ZywtLesVTxk/fjyn78O3UmlN0ZGioiJYWFhwFjsYU5mXL7SeM6ZUKrFx40Z89913cHFxwbJly56orkMxDidPnsTs2bOxZMkS9OvXj50FEh8fj86dO2P9+vUG2R88eDAyMzMRGRmJcePGYfjw4QY99BqLmTNnwsbGBmvWrGFXv5gMnFwuN3jVuqysDO+//z569eqFSZMm4ZVXXsF///0HCwsLfPbZZxg5cqRB9n/99VcsX74cn376KUJCQjBmzBiMHz8esbGxcHd3x7Zt2/S2/aThqYZmJIODg9GlSxdezpeEhAS89NJLmD59OiZOnIhnnnkGZWVlkEqlmDlzJubNm6e37alTp+Lq1avw9vZGdHQ0xo4dy5nSFgBs2bIF27dvx+effw4XFxdMnToVb775Js6dOwepVIo9e/YY/B58ZSP49J3v49K7d29MmTKFF7UtPj+nAL/nO6DJaPNxrvN9/b169SpmzZqF2bNnY+zYsRg+fDjc3d2RnZ2NESNG6KUy+SiEEPz333+1Pks9evTgRB0yMjIShBAMGTKk3gUgQ+D6nOR7FmNNCgsLMWPGDCQmJrIP62VlZQgLC8OmTZs4UfrkE67PmStXruC3335jg+Dnn3++lq3S0lK89dZbnAw55zOTDWieK7Zu3YrCwkIAmj7VmTNncjbI+/79+0hISIBYLEZQUBBat27NiV1joFUwdufOHbz//vtISkpCdHQ0Fi1axMscAor+3LlzByKRCMHBwbh8+TK+//57+Pj4YPbs2Zwo2MXGxuLgwYM4cuQIqqqqMGTIEIwdO1bvkrYnIZVKsW7dOixatEhvGxEREdizZw/at29fa/u9e/cwefJkXL161VA3a0EIYZWNPD09ObF54sQJuLi4oGvXrjhw4ACrJrV48WK9hTaAuuMqlEolUlNTkZCQgGnTpmHu3LkG+X3lyhUcOHAAx44dg1wu5/yBo6qqChKJBK6urigoKMDBgwfh7e2N4cOHG2w7NzcXBw4cwIEDBxAfH48uXbpg7Nix9a5i6wohBD/88ANatWqF/v37Y+vWrdiyZQt8fHzwxRdfICQkxCD79WUjbt++DalUanA2gk/f+T4unTt3xuHDh3nJwAP8fU4Z+DzfN27ciIMHDyI1NZXTBTdjXH9zc3Mhl8vRsmVLJCUlYc+ePfDx8cGUKVPqqDfqSklJCWbOnIk7d+7AwcEBhBBUVFQgLCwM33//vcGr+lVVVThx4gQOHjyIc+fOwdPTE2PGjKm3CkUfuDwnJ02ahJCQkDqzGCMiIthZjFwFY4BGdfL8+fNITEyEhYUF2rRpg549e+o1FuXKlSta72uoEiLX58ypU6fw5ptvIioqCkKhEBcvXkTnzp2xefNmVlGRq+PO570DAPbs2YNPP/0UkyZNQrdu3UAIwZUrV7Bnzx4sWbIEzz33nN62ZTIZ3nnnHZw8eZLdJhAIMGDAAHz55ZcGXwuMgjaNZaGhoSQ4OJh069atTlPfo19Tp07ltquN8kR+/fVXUlFRYZT3UigU5J9//iHz588nERERZMCAATrbkEqlZOnSpSQqKor06tWLfP7550SlUrGv//fff2TgwIEkJCTEIF979epFbt26VWf7zZs3SZcuXQyyTQghAwYMIF999RVvwiOMEIsx2bhxI/nwww85syeXy8nx48fJW2+9RTp16sQeM0NYsGABuXDhAkcePp6EhASydu1a0qVLFxIREWGwPb5Fal566SXyxhtvkKqqKnZbVVUVefPNN8krr7xikG0+fef7uEyfPp0V2OEavj+nxjrfb9y4QT755BPSq1cvEh4eTt59912D3pfv6++XX37J63nz/vvvkxEjRrDiWIQQEhcXR0aNGkWWLFnC6XsVFhaSXbt2kYkTJ5LQ0FDy4osvGmSP63MyMjKSpKSk1NoWExNDwsPDybx58wghhOTn5/Mu+FLzuqYtvXv3riUW0dAXF75zfc6MHTuWfPPNN+z3N27cIH369CFjx44l5eXlhBDujjuf9w5CCBk2bBj58ccf62zftWsXGTVqlEG2V65cSfr06UNOnjxJysrKSElJCTlx4gTp168fWb16tUG2jYVWmTFdG6h//PFHvYNDiu7wWe7wKHl5eTh06BCOHTuGW7duISoqSufGzE8++QR79+7F6NGjYWlpib/++gsvv/wyXn31VSxfvhw///wz/P39sXz5coNWqubPn4+8vDysX7+ezQ4WFRVhzpw5cHJyMngMA18rygwhISHw8fHhpYSoITIyMhAdHc151rCoqAh//vknvv76a1RVVdWR1dYFvksJGW7cuMFm9yQSCZ5++mmDS5/4LN8E+M1G8Ok738flt99+w+eff46xY8ciMDCwzkqpIb0jfH9OjXW+M6jVavz8889Yu3YtJBKJ3ivufF9/+S6f79GjB9avX1+r1wXQzF2dN28ezp07x9l7yeVynD59GseOHcOpU6fg6emJY8eO6W2P63OyX79+WLduHSIjI2ttZ1okpkyZgpdffpmTDE1xcTE2b95cZ+agQqFAUlKSztew4uJizJw5E0KhEF9++eVjs2uGZs65PmfqG0qdnJyMF198Ee3bt8e3336L0tJSTo4735nsTp064eDBg3XmjKWlpWHUqFEG9af27t0bn3zySZ3WqX/++QdLly7F6dOn9bZtNEwbC1K4QCqVkr/++ou88sorJCwsjAwYMIDTVcPy8nKyb98+Mm3aNBIaGkoGDx5Mvv76a5KVlaWXvf79+5OffvqJ/f6ff/4hgwcPJosXLyYhISHk888/Z+VbDSE7O5v069ePhIeHk+joaBIdHU3Cw8NJ//79SXp6usH2GbheUWbIzMwkmzZtIiNHjiTBwcFk0qRJvGdB//rrLxIVFcWJrcrKSvLnn3+SmTNnkrCwMDJ06FCDzpua5OTkkG+//ZaMHj2atG/fnrNjk5ycTL766itW1nvq1Knkzz//JFKp1GCfCSHk8uXL5MMPPyRRUVEkPDycLFiwgNNVbD6zEXz6zvdx4XNF3BifU77O95pkZWWRLVu2kFGjRpGQkBAybdo0sn//fr3tGeP6GxMTQ5YuXUp69OjB+XnTkMR6UlIS6dSpEyfvceHCBbJo0SLStWtXEhkZST744ANWht4QuD4nFy9eTEaPHk2uXbtG5HJ5rdd+/PFH0r59e/LWW29xkqF5++23SVRUFJk7dy4JCQkh8+fPJ2PHjiXt27cnW7Zs0ctmTk4O6d69O/nll18M9u9xcH3ODBw4kPz33391tsfExJBOnTqROXPmkJycHE6OO9+Z7GHDhpFDhw7V2X7w4EG9KqxqEh4eXidzSwghDx48IB07djTItrHQWsCD0jgoKirCkSNHcOjQIdy4cQMRERHYtWuXQTY7deoEsViMoUOH4tlnn0XXrl0NstexY0ccPnyYXe1RqVTo2LEjnJyc8OWXX+ql0NgQlZWV2L9/PxITE0EIQfv27fHMM8/UUVTjAq5WlOvj7t27OHDgAI4ePYqSkhIMHTpU77EFQP0CHhUVFYiPj8ekSZNYqVh9mTt3Lk6fPg2BQIDhw4dj3LhxBp83DZGYmIiDBw9i9+7dUKvViI2N1dtWcHAwWrRowa4o89VjpFAocPr0aRw4cABnzpyBm5sboqOjMXv2bIPs8p2N4NN3vm0bA64/p/XB5fkOaHo5Dh48iNjYWPj5+bHnPhdN+8a6/iqVSpw9exaHDh3C33//DWdnZ4OV7aZNm4Z27drVuRYuX74ct2/fNlhUpk+fPigsLETXrl0xbtw4DBs2DDY2NgbZrA8uzsnS0lLMnTsXFy5cwJYtW+ooKf70009YsWIFVCqVwfe97t2747PPPkP//v0xcuRIrFmzBsHBwVi8eDFkMhk+//xzvezu27cPJ0+exObNmw3y73Fwfc6sWbMGhw4dwjvvvIM+ffrU6jk7fvw45s2bhw4dOuDGjRsGH3e+7x07duzApk2b8Pbbb7MZ1piYGKxfvx5Tp07FG2+8obftCRMmYODAgXUEmjZt2oSjR49i//79BvluDGgw1sTgutwBAP744w8MHz6csxtFcHAwzp07Bzc3N3ZbREQEPvnkE17nXPBFdnY2K/hw//59REVFYdy4cRg9ejSn73Pnzh0cPXoUP/30EwQCgUFlA48KeACAWCxGeHg4Ro8eDQsLvUYQskyZMgXPPvssbw8YDFyXEl6+fLlOiQmfcFm+CQA5OTl44YUXUFpailatWgHQSFI7Ozvjxx9/5ERMgoFr341h2xhqW1x+Th+Fj9LZiIgIdsHEUAEDU8FF+fyjXLt2DVOnTkVwcHCth8d79+7hu+++M7gdYOPGjYiOjub0M9kQXJ2TaWlpcHFxqVfA7cGDBzh+/LjBiqUdOnTAiRMn4OPjg7feeguDBg1CdHQ04uPj8corr+DMmTMG2ecTrs8ZmUyGTz75BPv378eWLVvw1FNP1Xr91KlTeO+991BRUWFwMMb3vUOtVmPlypX4+eefoVKpQAiBhYUFXnjhBSxatAhCoVBv26dPn8brr7+OoUOH1jruJ06cwJo1a/D0008b5LsxoMFYE+HixYs4cOAAjh8/DrVajaeffhrR0dG8ZSMMoaFgbP/+/XXqiXVl0KBB2LdvH1xcXDBw4MDH1of//fffBr0XnyvKDOnp6Wygl5qaiu7duyM6OhrDhg2DtbU1Z+/TmHjw4AEOHDiAgwcPIj09nQ1+G8sxkUgkOHHiBA4cOICLFy/Cz88PY8aMwdixY+Hj42OwfT6zEXz6zqdtvtW2+Pyc8n2+SyQS2NraGmwHMO71t6KiAseOHcOBAwdw5coV+Pr6YuzYsZx9jgDg5s2b2L59e63P0owZM9CpUydO7PMJX+dkUVERsrOzIZPJYGtrC09PT7i6unLm96BBg/DZZ5+ha9euWLNmDZRKJd577z2kpaVh9OjRuH79OmfvxQd8nDNVVVUQCAT1DqguKyvDv//+y8lCtjEy2RUVFUhOTgYABAYGcmb7xIkT+Pbbb5GQkMD6PmvWLAwdOpQT+3xDg7EmgLHKHR6lqKgI48eP1/mmGhwcjPPnz9e6gNfXqKoPGzduZOfbbNy48bH7GjLoFeB/Rfn555/HrVu3apXNcRnoZWZm4pdffkF8fDxEIhHCwsLw/PPPw93dnbP3eBSJRILt27cbdOyNVUpYE67kg41Zvsk1fPrO93FZtWoVDh8+jI8++ghRUVFQq9W4cuUKli9fjmeeeQbvvPOO3rb5/pya4nwHNCWj169f1+naZszrL9fl8+YCF3Oj+Dgn//jjD2zduhUpKSkANKIagGZRo3Xr1nj11VcxZswYg94DAD777DOcPHkSq1atglwux9y5c/Hxxx/j5MmTSExM1KvkTKlU4r///kOPHj3YZ6M9e/bg9OnTcHd3x4wZMxAUFGSw78aiqKgIV69ehZubG7p06WJqdxrkypUriIiIgIWFxRNHDDTWrDxXGFaLRDELJkyYYLRyh5pYWlrqXdK1fPnyWqs8CoUCX3zxRR01LF2nwte8wXfv3h3h4eEQi8W19pHJZJyo65w7d46zFeX6CAoKwoIFC3i5SMXGxuKll16Ci4sLOnToAJVKhb179+KHH37Arl27DJ4p0hASiQQbN2406EFs586dRi0lBABbW1uDatoZCgoK8NFHH3G6YGKsbAQfvhvDNgAcPHiwjtrW4MGDIRKJsHTpUoOCMT4/p4BpzndAExRMnTpVpwUIY15/ly5dymn5PKAp3/7ggw9gb29fbyl3TXS9N+mCoWvkXJ+TP/74I1avXo3p06ejR48e8PT0hKWlJeRyOfLy8nDhwgV89NFHqKysxKRJkwx6r7lz50KpVCIrKwvPPPMMhg4dijlz5sDR0RFfffWVzvYKCwsxZcoUPHjwAAcPHkRQUBC++eYbbNiwAR07doRcLsfzzz+PPXv26HXf4/uc+frrr7Fz50788ssvCAgIQGxsLF555RVUVFQAAHr27IlNmzbple3k+94xZcoUtgpqypQpEAgE9Z7bAoFA54VOYy78GAOaGWtCXLlyBffv38eoUaOQk5ODVq1aGdz7wwe6jEowZExCSEgIzp07V6eE4s6dO5g4caJBUqoMZ86cwbZt25CcnIy9e/fi999/h7+/PycrhAxZWVm4f/8+unXrhsrKylrlnfoyYcIEBAYG4pNPPmHPEYVCgYULF6KgoAA7duzQ2aZcLsfnn3+OAwcOQCwWY8SIEZg3bx4vpYN5eXn45ZdfkJycjA8++ABXrlxBu3btEBgYyPl78YFcLkdGRgb8/f1BCKnzwKoLxr4pcem7sWxHRETgzz//REBAQK3tKSkpGD16NCfXAj4+pwymON8VCgWuXbumdyBojOtvVVUVjh49ivv372PmzJlISEhA27Zt4eLiope9KVOm4Ouvv4ajo+MT71ONYYQPV+fkoEGD8Oabb2Ls2LEN7vP777/jm2++qVUKzBUlJSWwt7fX63lm2bJluHr1Kr788ksEBgaisrISvXr1QseOHdm/4apVq5CVlYX169frbJ/Pc2bv3r1Yvnw5pk+fjldffRX29vYYPnw4KioqsGPHDjg4OOCtt95C79699RI54vvekZmZCR8fHwiFQmRmZj52X10z/gMHDsRvv/3GBpINIRAIDC6JNgrGE26k8EV5eTmZMGECK9OclpZGXn31VTJixAiSk5NjaveMyvfff19rwCPz/0e/nn/+eYPf6+zZsyQsLIy89957pGPHjiQtLY2sXbuWhISEkD/++MNg+3K5nMyZM4e0b9+ehISEkLS0NPLGG2+Q6dOnswMf9aVTp071SvAmJiaS8PBwvWyuWrWKdO7cmSxevJh89NFHJCIignzwwQcG+VkfKSkpJCoqigwcOJCEhYWRtLQ08tZbb5Hw8HBy/fp1g2xnZmbW+5WVlUXy8/NrDSfXly+++IJ06tSJ/ZvOnTuXLFq0qI5ktD5cvny5XjtVVVXk6NGjBtvn03c+bT///PNk8+bNdbZ/8803ZPTo0QbZ5vNzSgi/5zvXGPP6m5+fTwYOHEg6d+7MHvdXX32V9OrViyQlJXHw2zRMXl4e5zaLi4uJQqHgxBbX52R4eHi994uaGCL5P3v2bE4+K/UxYMAAcvbsWfb7EydOkPbt25O//vqL3Xbt2jXSo0cPXt6fQZ9z5tlnnyW7du1iv7958yZp3759rWvZqVOnyNChQw32j+97x/vvv1/v37i4uJi89tprBttvCC7u2caABmNNgKVLl5IJEyaQtLQ0Eh4eTtLS0khSUhKJjo4m8+bNM9j+426qHTt2JEOGDCEbN24karXa4PcqLCwkR44c0XsOjUKhIH/88Qf57bffSPv27ckPP/xAfv/9d/brjz/+IMeOHSOlpaUG+zphwgTy/fffE0IIe9wJIWTr1q0GT5QnhJAvv/ySDB8+nFy8eJG1f/HiRTJgwADy0UcfGWR71KhR9c78OH36NHn66af1sjlgwIBaNv/55x8SHh7OyXlRk//9739k0aJFRK1Ws8dFoVCQefPmkcmTJxtk+3HnenBwMOnUqRNZuHCh3nPwfvjhB9K7d2/y22+/kc6dO5O0tDRy6NAhEhUVRdauXWuQ74QQEhwcTAoLC+tsv337tsHzVvj0ne/j8s8//5CQkBDy9ttvkx9++IH88MMPZPbs2SQkJIQcPnzYINt8fk4J4fd8j4+Pr3Uu//vvv+STTz4h33zzTb3n0ZMw5vX3nXfeIa+++iqRSCTscSkpKSHTp08nr776qsH2G/ospaen671gRQghFy9eJG+99RbJzc0lhBCSm5tLJkyYQIKDg0l4eDj5+uuv9bbNwPU5OXnyZPLhhx82+GCrVqvJ+++/TyZMmKCXv8HBwaSgoKDWtlGjRnEykzIsLIxkZmay369atYoEBweT/Px8dltmZibp0KGDwe/F9TkTHh5OHjx4wH6/detWEhwcTOLi4thtaWlpvPpuyL3j6tWr5I8//iB//PEHCQ4OJj/++CP7PfO1evVqgz5PhGjmsRUXF9fZnpOTw9ncVL4xvxo2is78888/WLNmTS3xi6CgICxZsoSTPpdFixZhzZo1mDhxItskfe3aNezevRsTJ06Ek5MTdu7cCUtLS7z88ss62U5ISMBbb72F5cuXo3379hg9ejQKCgpgaWmJrVu36iwFa2FhgejoaACa9PTIkSMNVkpriPj4+HpnngwfPvyJKX9tOHToED7++ONac9e6d++OTz/9FO+++y4+/vhjvW3PmjULy5cvR15eHrp37w4LCwvcunUL69atw6RJk2o122rbd5CXl8fKygJAr169UFVVhfz8fHh6eurt66PExsZi9+7dterbLSws8Prrr+P55583yPaKFSvw2Wef4c0332R/79jYWGzYsAGTJ0+Gv78/Nm7ciA0bNujVZ7R3714sWbIEQ4YMwSeffAIAGDFiBMRiMVauXIm5c+fqbHPHjh3s3CBCCHr16lXvfoYqwPHhuzFsA0D//v3x1Vdf4dtvv8Xp06dZta0vv/zSYLUtPj+nAD/ne2VlJV5//XVcvnyZ7aP59ddfsWTJEnh5ecHKygo//fQT9u7dq5PwgzGvvxcvXsTWrVtr9Yw5OTnhvffeq3eOojbs27cPf/31FwDNZ+mNN96oUyqbl5dXa96TLly6dAkvvfQSOnbsyG5buHAh7t69iw8//BAODg74/PPP4e3tjXHjxun1HgD35+SiRYvw0ksv4d9//0W3bt3g4+NTq2fs6tWrqKiowLZt2/Tyl9TTLZORkQGlUqmXvZo4OjqitLSUPY8vXryIwMDAWkJVDx480FsRku9zpubn/urVq3ByckJwcDC7rbKyUu++Sb7vHQKBAO+//z77//rGcNja2mLmzJk62z58+DD+++8/AJpyyGXLltVRm8zMzHxsH5w5QYOxJkBRURE8PDzqbHd0dIREIjHY/qFDh7Bo0SJMmDCB3TZ48GAEBgZi3759+Pnnn9G2bVt8/vnnOgdjn332GQICAhAYGIiDBw9CqVTizJkz2LNnD7788kuDBmuOHTsWRUVFePDgAdRqNQDNBUcul+PWrVt47bXX9LYNAA4ODsjLy6sjx5+UlAQnJyeDbANAbm5uvVL/Pj4+KC0tNcj2e++9B0BTK/8oNZukdWmsVSqVtW5CYrEY1tbWkMlkBvn6KGq1mv171qSyshIikcgg299//z0++ugjjBgxgt0WHBwMDw8PbNy4Efv374e7uzsWLVqkVzCWkZGBkJCQOtuDg4ORn5+vl8+TJ0+Gs7Mz1Go1Fi1ahIULF9aaAyQQCGBra2vwXCQ+fDeGbYYhQ4ZgyJAhnNiqCZ+fU4Cf833Lli1IT0/H1q1b0bp1a8jlcnzxxRcIDg7G3r17YWlpiXfffRcbNmzQW6iC7+tvZWVlgwJK+j7EDx48GDExMez33t7edXpe27VrxwacurJ161aMHz+eDYbS09Nx7tw5TJ48GS+++CIAQKVSYffu3QYFY1yfkyEhIThy5Aj27t2LmJgYJCQkoKqqClZWVvDx8cGECRPw3HPPcSpxzxU9evTArl278Omnn+LKlSuIi4urde6p1Wp8++23eqtx8nnOtGvXDrGxsQgICEBZWRkuXbqEQYMG1drnyJEjaNeunV6+833viIyMxL179wBoruVnz57lTK05IiICe/bsYQP5rKysWs8fjO+6Djg3FTQYawJ07NgRR44cwSuvvFJr++7duxEaGmqw/bi4uHo/jF27dsXSpUvxf/bOPJ6q/fv/ryM0z3XT7VNpdFwyVVQqGeo2aNCcUjI0aNQg0mBKQqUQEtEkGmjSdOuiSYVuUSRSJFQUURnfvz88zv46zqHO2Xufe+vn+Xh4PNjnWOdte+/33uu91notAPjjjz+Ql5cnsu1Hjx7h1KlT6Ny5M27dugUdHR1069YN06dPp9208/z589iyZQsqKysB1D4I8HZJevToQfthYPLkyXB1dYWrqys4HA7KysoQFxcHZ2dnvod5cenXrx/u3buHWbNm8R2/dOkS+vfvT8v29evXaTVZ/DcZOXIkAgIC4OHhQR379OkTPDw8aDscr1+/FnrNDBgwAFlZWQAAeXl5FBYWimW/R48elOR0XeLi4sRu6yCpaAQbY2fTtqSETdi8TgF25vvVq1exefNmjBo1CkBts/OSkhJs3LiRmjuzZs2iFZFke/0dOnQowsLC+BTsKisr4efnxxehF4UOHTrwOZ88lTymSE5OxsaNG6mf7969Cw6Hw7dJoKKiAicnJ1qfw8ac7NChA+2Gzv8Gq1atgrGxMTQ1NVFWVoYePXpg8eLFAGqjKwEBAXjz5g0iIiLEss/mnJk/fz62b9+O1NRUPHr0CBUVFVi0aBGAWof7woULCAoKwo4dO8SyL8lINs8pY4ru3btT7R9MTEzg4+PDyCb4v0WTM/YLsG7dOpiZmeHJkyeoqqqCn58fMjMz8fTpU7HTBuryv//9D3///TdMTU35jv/999+Qk5MDAGRnZ4u1KyYlJQVZWVlUVVXhwYMH2Lp1K4DaXU+6Knz+/v6YNGkSLCwsMG/ePAQHB+Pdu3dwdHTEqlWraNkGgLVr1yI/P59azIyMjEAIwZgxY2inVQG1NxFra2tkZGSguroakZGRyMrKwtWrV7F3715atq2treHi4sKX7kAXDocjkBLARoqAra0tFi5ciJEjR6K8vBzLly9Hbm4uOnToIDTSJwr9+/fHmTNnBKJeZ86coZT4UlNT0a1bN7Hsm5ubw9HREe/fvwchBPfu3UN4eDiOHj1KpXPQgc1oBJtjZ8P22bNnMX/+fLRs2RJnz55t8H0cDoeWM8bmdQqwM9/z8vL4IpEPHz4Eh8PBiBEjqGM9evSgFdlje/3dtGkT5s+fjwcPHqCyshIODg54+fIlPn/+jGPHjtG231BEkHctidPf6evXr3zpag8fPoSsrCzU1dWpY82aNaO9brIxJx8+fIgzZ86guLgYo0ePxuzZs/kis3T6owm7dzBFnz59cPHiRVy5coVyOHj/g9zcXPTq1Qu7du1ipM8Y03NmypQpqKioQFhYGKSkpLB3714qZTAgIAARERGwtLRkRL2Z7Uh2eXk5wsPDkZ6ejurqaup4RUUFUlJScPXqVbFtN6ZSmZ+fTz2n/pdpkrb/RUhLS0NwcDCePXuGmpoaDBgwAGZmZlBVVaVt+8KFC7C1tcX48eOhrq6OmpoaPH78GFeuXIGTkxM0NDRgbm4OAwMDbN68WSTbFhYWkJOTQ6dOnRAcHIy4uDhUVlZi69atkJKSgr+/v9jjHjRoEKKiotCvXz+YmJjA0tISo0ePxrVr1+Dv79/oA5oovH79GqmpqaipqcHAgQMZ2Q3nERcXh4CAAL7/q6WlJf78809adrW0tHDq1CmhqSziwuVyBW6odXfD60K3efLXr19x8eJF6rwPGDAAU6dOpb0jefv2bSxbtgzKysp8cz0lJQU+Pj7o1q0bFi5ciMWLF8PKykqszwgPD4efnx/y8/MBAJ06dYKlpSW1W0uH70Uj6MpOszl2Nm03Rk1NDe0oMVvXKQ+m57uWlhZfP8G5c+fiw4cPfPMjISEBa9euxe3bt8X6DEmsvwUFBQgLC+M7L8bGxoz03Hz69Cm2bNmC9PR0oWmi4qxhEyZMgJ2dHUaPHo3q6mqMGjUKioqKfJumJ0+exIkTJ6g6JHFhck7evHkTK1euhKamJqSkpBAfHw9VVVX4+/tT0YgPHz5g1KhRYp0XLpcLdXV1vjSzhIQEDBo0SKAOSFRn7+3bt+jevbtEaofYmDMNUVBQAFlZWbHbONSH7XvH1q1bERUVhT/++APJyclQV1fH69evUVhYCFNTU6p0QhxycnKwa9cuPkeP50gWFRXh2bNntMYuCZqcsSZ+iL///hvBwcF4+vQppKWloaCggCVLlmDUqFF4+PAhbt++jZUrV4rcF+j169ewtrZGTk4OrK2tYWxsDGdnZ8TExCAwMJBWH53BgwcjKioKPXv2xJYtW9CnTx+Ym5tTzSTr5nn/KHUX9rdv3zb6XlEK3yVNYGAg4uLiYG5ujl69eglEIcUZe2Rk5A+/t7F+Nf82qampCAkJwbNnzyAtLQ0ulwszMzMMGDAAycnJSE1NpS0UAtTWehJCGO1HNXHiRKiqqgqNRqxbt46x/ndsjJ1N2/r6+jhz5gw6dOjAd7ygoABTpkzB/fv3Gfusn4Fly5Zh4MCBWLduHdWbcuHChXwpf2vXrkVlZSV8fX3F+gw21l9JMn/+fJSXl2PGjBnYuXMnbG1tkZ2djePHj8Pd3R0TJkwQ2aaPjw+io6Oxfv163L59G2FhYdizZw+V1v7kyRNYWVlh3rx5jIhvMcX06dMxduxYKjry5MkTrFy5El26dMGRI0fQpk0bWs6Yra3tDztLotYwKioq4vbt26ysVfVhY85ICrbvHSNGjMDmzZthaGiIsWPHIiAgAD179oS1tTXk5OSwZcsWsW1bWlri1atXGD9+PA4fPgwzMzNkZWXh+vXrcHJyYuR+zTZNaYo/KWx3fa+Prq4udHV1hb42dOjQH1bcq0/v3r0FdkgtLS2xefNm2mIMysrKOHXqFNatW4eBAwciNjYW5ubmyMjIELuZrL6+PrWwN9SxnrejJM5NSVJ1Ll5eXqiurqbSk3jQGTubDpa+vj5Onz5NNXhs7MZNt8GjoqJig0W/gwYN4lNC+xGioqIwceJEyMrKIioqqtH3iisMwCMnJwfe3t7o168fFBQUUFRUBD09PVRVVcHf31/kGyqbY2f7vLCptsX2dcr2fLeyssLChQsRFxeH3NxctGvXDmZmZgBq1eYOHz6MO3fu0Er3Y2P9XbhwIXx8fKgGu42dF3HS5ery7NkzhIaGQkVFBWfPnsXAgQNhbGwMOTk5REREiPVgvWTJEmRnZ2PlypWQkpKCiYkJ5Yi5ubkhJCQEmpqasLCwENk2m3MyKysLhoaG1M8qKioICQnB/PnzsXLlSgQGBoo83rrQTS9vDEnGG9iYMzyEZZ7wkJGRgZycHKZOnQorKyux1jSm7x31KSkpoWo5+/fvj2fPnqFv375YunQp1q5dS8sZS0pKwoEDB6ClpYVbt27BwMAAKioq2Lt3L2JjY5ucsSbY482bN1QY/M2bN6x/XkJCApKSklBZWSmwuNFxCr59+wZHR0fIy8tTxcHGxsbQ1tbG1q1baRWTrlq1ChYWFujQoQOMjIzg6+uLSZMmIS8vT2yBjdDQUCotg+7NXhiSqnOhK47SEGzVFRgZGVHROzoqY9+jpqYGFy5caHCui7OxYWtri1GjRqFz586N1j9xOBzazpisrCx1zfTu3RsvXrzA6NGjoaysjNevX4tsj82xs31e2FTbYvs6ZXu+q6io4NSpUzh79iykpKQwZ84cqg7y1q1beP/+Pfz8/KCmpib2Z7Cx/vbo0YNKKWUiFbExampqKJXi3r17Iz09HUOGDIG+vj4CAgLEsikrKwt3d3ds27YNHA4HrVu3pl7T09ODpqYmdHV1xXqYZnNOdurUCa9fv+YT0+nbty98fX2xePFi2NjY0Kob/VUyTtiYMzzYbDEEMH/vqE+nTp1QWFiI33//HfLy8khPTwcAdOzYER8+fKBlu6Kigiq56NOnD54/fw4VFRVMmzYNJiYmtMcuCZqcsZ+UugWLa9asgYqKCmsqOL6+vvD29ka7du0EahToOgVubm5ISEjgi6rY2dnBw8MDe/fupZVHPGTIEFy9ehUVFRXo2LEjTpw4gbCwMHTv3l3sC1RTU5P6/siRI7C2tmak8JfHzZs3qe8PHz5MiUYwTd2/o6ioCNLS0mL3QeFRv67A2dkZ58+f56srqKys5Oth9qPUnWMtW7aEoaGh2CIajeHq6orjx4+Dy+UypohVV0UqISGBUXW2+jAdjWBz7GyfFzbVtti+TiUx3wcMGCB0fa2r9kcHNtbfupsh2traGDlypEDqKVP07t0biYmJMDQ0RN++fZGcnAwA+Pz5MyoqKmjZFjbX667J4sDmnJw4cSK2bduG9evXY9SoUdS9QkNDAx4eHli3bp1Yaso82M44CQ4O/qFeXHSeZQB25wybLYYAdiLZdRk9ejQcHR2xc+dODB48GK6urhg7diyio6NpC2z06NED6enp6N69O/r06UPNkZqaGpSVldEeu0Rgual0ExJg2LBhJCUlhTX72traxN/fnzXbSUlJAscfPHhARo0aRdv+/fv3yZ07d6if3dzcyMOHD2nbJYSQwYMHk5ycHEZsCWPkyJHk8ePHrNkPDQ0l2trahMvlEi6XS0aOHEkOHz4stj0jIyNy4MAB6ufHjx+TUaNGESMjI/L582dCCCHv378nXC6X1rgHDx5MXr16RctGQ2hqapKzZ8+yYpsQQgwMDFi9Vh8+fEhUVVVJUFAQKSoqIpqammTixIlEXV2d2Nvb07LN5tjZPi/1KSwsJJcvX2bk+mX7OmVrvhcVFZEzZ86Qw4cPk/T0dIHXy8rKiLe3N63PYHP9HTp0KHnx4gUjtoQRERFBVFRUyIULF0hGRgZRUlIiDg4OZMqUKcTMzEwsm9XV1cTX15fo6uoSDQ0NsnbtWpKbm8v3HibWSKbn5Ldv34i9vT1RVlbm+3/yuHHjBhkyZIjY475//z6prKykvm/sS1QUFBSIjo4O0dXVbfRLT09PrLHXhY05w2PQoEFC14FXr16RQYMGEUIIefv2LfW9qLB57yCEkOLiYrJ8+XJy7NgxUlNTQywsLIiCggJRUlIiFy9epGXb39+faGpqkpiYGPL48WOiqqpKAgMDiampKZkzZw7tsUuCJmfsF2DixInk3r17rNlXUVEhb968YcW2mpoaycjIEDiekZFBVFRUaNm+ePEiUVJSIocOHaKOrVq1iigpKZHr16/Tsk0IIS4uLmTNmjUkPT2dlJeX07ZXH11dXfL06VPG7RJCSFhYGFFWViaurq7k+vXr5Nq1a2THjh1k0KBB5NSpU2LZVFNTI9nZ2XzHMjMzybBhw8iiRYtIRUUFIw8aZmZmJDg4mJaNhlBTU2PVwdbW1hY635kkPz+f+j9kZGQQZ2dncujQIdpzlM2xs31enj9/TsaNG0cePHhAiouLiba2NlFQUCCDBg2ivXayeZ0Sws58f/HiBRkxYgRRU1MjampqRFFRkezatYvvPXSvVbbX31mzZpFLly7RttMY169fp5zH8+fPk8mTJ5MlS5aIvUYEBASQwYMHE19fX3LgwAEyevRooqWlxTd/3r9/TxQUFGiNm605+fXrV/Lt2zehrxUXF5MLFy4w8jlZWVkkOTmZ+jkkJIRkZWWJZUtBQYF8+PCBkXH9CEzPGR4TJkwQull6+PBhMnbsWEIIIfHx8URHR0fsz2Dr3iGMmpoakpSURAoKChixdfjwYfL3338TQmqvMw0NDTJp0iTy7Nkz2vYlQVOa4i/A6NGjsXTpUujo6KB3794CRep0Q+8aGhp49OgRevToQcuOMNTU1HDo0CHs2LGDqgUghCA0NFRkoYT6+Pv7w9bWFgsWLKCO7d+/H0eOHIG3tzcMDAxo2Y+NjUV2dnaD/THoytgaGRnBwsICU6dORe/evQUUD+nUF4WEhGDTpk1852bs2LHo3bs3QkNDMXPmTJFtsl1XwKNNmzZwd3eHv78/5OXlaUsf12XUqFGIjY3F/Pnz6Q5TKAsXLsSqVaswf/58oSqW4grh1OX169eoqqpCz5490a9fP8jIyEBVVZV2GjObY2f7vOzatQu9e/dG3759cfHiRVRVVSE2NhYnT56El5cXTp48KbZtNq9TgJ357ubmhsGDB8PT0xNSUlI4cuQI9uzZg0+fPsHV1ZXWeHmwvf5yuVxs2LABhw4dEnpemBCu0tLSwsePHwEAkydPRvPmzaGpqSl2auSZM2fg4uKC8ePHAwAWLFgAKysrLF68GCdOnKBS3unKsLM1J+vaefjwIaXEmZ+fD3l5eT6RD3G5e/culi9fDlNTUygrKwOoTdHz8vJCYGAgVS/1o0hC0r4uTM8ZHsuXL4etrS0lC1+/xVBWVhbs7Owwbtw4sT+DrXsHIKgPwOFwsH79ekb0ATgcDhYtWoRPnz4BqBXK0dLSgrKyMm0hOEnRJG3/C6Cnp9fgaxwOh7a63JkzZ+Du7g4jIyP07dtX4KKh87Dx5MkTLFq0CB07dqQW3qdPn+LTp08IDg6m1SdNVVUVFy9e5HMOgFrVIENDQzx+/Fhs28D3pdzpqgs21pBZ3Nx5HioqKrh48aJAn7Hs7GwYGhriyZMnItvcvXs3Ll26JFBXAADXrl3DunXroKysjMePH9MaO5vqocHBwdi/fz9GjhxJ3YzqQndjg83/KVD70LJp0yZYW1vD3NwcALB69WrcvHkTXl5etB6A2Rw72+dFQ0MDp06dQr9+/bB8+XK0a9cOu3btQk5ODiZPnox//vlHbNtsj52N+a6pqYmwsDC+etfr169j7dq1WLRoEWxsbGhJlQPsr7/fqztrrBHsj/D06VOYmZlh+vTpVG2dnp4eKisrERQUhIEDB4psU11dHefPn+c7J1++fMGCBQvw6dMnhIWFoVmzZrTOO8DunCwtLYWFhQX++ecfcDgcXLt2DTt27EB2djYOHz5Mu7ZxxowZGDlyJKytrfmO79mzBw8ePBB544TL5eLOnTsSkbZnY87Uha0WQwC79w4AcHBwwJ07d7Bjxw6qPvL69evw8PCAvr4+LX2A7OxsWFpaQl9fHzY2NgCAYcOGoUuXLggMDET37t1pjV0i/MuRuSZYprq6mrYNBQWFBr/oppwRQkhOTg7x9PQkS5cuJStWrCB79uxhJHT9559/kjNnzggcP3fuHNHX16dtPzIyUmj4vqysjFbtlST4888/hab4XLx4kejq6oplk1dXoKSkxEpdAY/c3Fyh87qyspJ2nQTbNQVv3rxp9IsuhoaG5OjRowLHQ0NDyZQpU2jZZnPsbJ+XwYMHk+zsbFJZWUk0NDRIZGQkIYSQ1NRUoqWlRds+m7Ax37W1tYWmsZ04cYIoKCiQw4cP005TZHv9bYx3797RtrFgwQJia2vLt8ZXVlYSW1tbsnjxYrFsGhoakoiICIHj7969I2PGjCGGhoYkLS2NkfsqWzg6OpI5c+aQ7OxsKjU9IyODTJs2jaxbt462fVVVVYF0d0II9Xmi4u3tTb58+SJwvLCwkFy9epUkJCSINU5hsDFnJAWb9w5C2NUHWLp0KbG0tOS77gsLC4mFhQVZtWoVLduSoskZ+wXQ09MjHz9+FDien59PNDU1JT+g/wghISFETU2N7N27l9y8eZPcvHmT7Nu3j2hoaPDVMYhCYWEhyc3NJbm5uYTL5ZKUlBTqZ97XX3/9JXYRbV1MTExISUmJwPEPHz6QqVOn0rJ9+PBhoqmpSY4fP05SU1NJamoqOXbsGNHU1CQ+Pj60bLNdV8DlcklhYaHA8aysLNp1hmxja2tLiZnU5ePHj2T58uW07auoqDT4IEP33LA5drbPi7m5ObG3tye7d+8mSkpKpLCwkOTn5xNLS0uydOlSWrbZvE4JYWe+W1tbExMTE6FOi7u7O+FyuWTHjh20nAI21t+6NHRecnJyxHpor4+wGlhCCHn58iXR0NAQy+bp06eJkpIS2bp1q4AYQ0ZGBtHW1iaampq0nTE25+SYMWNIYmIiIYT/HCUlJZHhw4fTsk1I7YbYtWvXBI7fuHFD7Id2Hx8foqmpSZ3zxMREMnjwYGpT2dTUlHz9+pXWuAlhZ87U5eHDhyQgIID4+PgQb29vvi+6sHnvIIRdfYDBgweTzMxMgePp6elk6NChtGxLiqaasZ8UNhuaSpobN24gICAA6enpkJaWRv/+/WFubo6xY8fSsrto0SJUVFTgyJEj8Pf3BwD89ttvsLa25qtjEIW4uDjY2tqCw+GAECK0tooQAh0dHbHsx8bGUnK4Dx48gJ+fH1q1asX3ntevXyM3N1cs+zwWLlyI3NxcuLq6orq6GoQQSEtLY+7cuVi+fLnYdvPz83H9+nXIyspCR0dHQLK2Xbt2YtUVHD9+HMHBwQBqz++MGTOoGkMeJSUlYvWgYbvHTWJiInJycgDUNjpWUlISkLbOzMzEvXv3RLZdn+7du+Phw4cCqWGPHj2i+t+IAptjl+R52bp1K6ytrZGTk4PNmzejU6dOcHZ2RmZmplgNa9m+Ttmc7wBgY2MDS0tLjB49GgcPHsSoUaOo13jS9kFBQbTuH2ysv6dPn8b58+cB1J6XFStWCKRkvXv3jnabDgBo3bo1cnJyBK6ld+/eiV3fMmPGDMjIyODkyZMoKirik5/v168fwsLCsHnzZiQkJIhsW1L3jqKiIqFrSbt27fDlyxdatgFg6tSpcHBwwKdPn6gyheTkZHh5eYlVEhEeHg5/f3+YmppSqYqbN29GixYtcPLkSbRt2xarVq3CwYMHsXr1alpjZ2PO8GCzxRDA/L2jPmzqA0hLS6OkpETg+NevXyXa9JsOTTVjPyl5eXnYtGkTCCF4+PAh1NTUhDY0nTdvnliOgaKiItX3o7HO7wA9oYpr165hzZo10NfXx9ChQ6m/5++//4a3tzf09fXFtl2Xjx8/QkZGhpFeRg8fPkRNTQ0WLVoEb29vvt5FvPM+cOBAsfK2MzIysHTpUhBCkJeXh27duvE9hPHsL1y4ELNmzaL9t5SWluLly5cAasU26JyfhIQEWFhY4Nu3bwCAVq1aUfVXdPn69SuCgoJACKEEQeo2TAVqb4Tjxo0TWWjmR+Y6odHjJikpCcbGxgBAOfH1adWqFczMzGjfUENDQ+Hl5YVFixbxPciEhobCysqKqgX4L4xdkudFGEVFRWjfvr1YBd5sX6dsznceFRUVSExMxIABA9ClSxeB1+/fv49Lly7ByclJLPt1YWr9/fTpE9WkOzIyEhMmTBAQp2jdujWmTZtG1SCLi7OzM27fvg0HBweoqKgAqL2WnJycMGTIEEbOS0O8e/cOv/32m0i/I6l7x4IFCzB69GgsWbKErwbOyckJaWlpOHHihNi2AaCqqgouLi44c+YMKisrweFw0KxZM5iYmGDdunUi31dnzpwJIyMjSpQpOTkZs2bNgrW1NZYuXQqgthbLzc2tQTGuH4XNOTNy5EiYmJhQY2Yapu8d9WFTH8DGxgYvX77Enj17qDr4nJwc2NjYoHv37tizZw+tsUuCJmfsF4DphqZA7Y1u0qRJkJWVZVWoYtq0aTAwMBB42PLx8UFsbCxOnTolkr2HDx9CXV0d0tLS320uTFeh7cGDB9DQ0IC0NDsBZj09PZw+fRqdOnVi1O7jx4+hoKDA9xBz7do1/Pbbb1BTUxPbromJCdq0aQNHR0c0a9YMTk5OyMzMxMWLFxkY9f/h4+MDc3PzH2ri+SPU/T8+ePCg0ffSbczK5XJx+/ZtoQ+/TBEYGIgjR47g/fv3AGqjEUuWLBE7GsGDzbFL4rx8+/YNV65cQWZmJszNzZGeno4BAwagY8eOtOyydZ3yYHq+f49Pnz6hTZs2Yq1rklx/7ezsYG9vz1oT9S9fvmDNmjW4desW3wbN2LFj4erqSutz09PTIS8vT0VLbt26hdjYWHTu3Blz5syhPZfYnJNJSUkwMzPDyJEjERsbi8mTJyMzMxNPnz5FUFAQtLS0GPmcsrIyZGZmIi4uDsOGDRNZRZGHuro6IiMjIS8vD6B2fdyzZw8iIyMpoZOcnBxMnDiRiiyKC5tzRlVVFdHR0ayoWvNg697BIycnBxEREXjx4gWkpaXRr18/zJ8/X+SNh/oUFhZi8eLFePHiBRUVLykpgZKSEvz8/BiJ7LGOBFMim2CZ3NxcEhcXR75+/cpoXw1nZ2fy+vVrxuzVpaFGhuLWQ9TtKcITGGFLeISQWgEAW1tbMmfOHJKfn0+OHTtG4uPjGbHNo7y8nGRmZpLKykpSUVFBy9b27dsJl8sV6K1kYWFBuFwucXV1Fdu2hoYGXxPW/Px8wuVyhdYC0eXr168kMjKSeHp6ko8fP5L79++ToqIi2nZXrFjBeh8wHmz0bqlLUVERK+eeEHbHzobt9+/fEz09PaKqqkoUFRVJdnY2Wbp0KaP9zZi8TuvDxnyPj48nq1atosSSCgoKyJw5cwiXyyVqamrE19dXZJuSXn8JqRUACAsLI58/fyYvXrygmgczRWZmJrl8+TL566+/xO51xaO0tJQsXLiQcLlcat5FREQQLpdLdHR0yLhx48jIkSMFGkGLC1tzMjU1lWzcuJFMmjSJTJgwgaxevZr8888/tGzWr+tKSkqi6rq4XK7YdV1qamp8zxhLliwREO1JTU1ltLaIyTnDw9TUlLE+bt+DzXsHW1RVVZG4uDgSFBREQkNDyZ07d0hNTc2/Pawfpqlm7BegsrISNjY2uHz5MqSkpHD16lXs2rULZWVl8Pb2pr1zGBkZCVNTU2YGW4/ffvsNr1+/5sudB2pz29u2bSuyvdOnT1M7I3Ql/b9HSkoK5s2bBzU1NaSkpKCiogKpqanYuXMnfH19xa4bq4unpyeOHj2KyspKXL16FXv37kXLli3h4OAgcrrGqVOncO7cOezcuVNgVzogIADnzp2Dg4MDFBUVxcrN//LlC18vlW7dukFGRgbFxcWM7l5/+PABc+bMQWFhISoqKjB79mwEBwcjJSUFoaGhfHLdohIfHy9Qe8k0YWFhCAwMRH5+Pq5evYpDhw6hW7dusLKyEsueJKMRTI9dUrbd3NwwYMAAXLx4ESNGjABQ23ts7dq18PDwoGqaxIXJ67Q+bMz3+/fvw8zMjK9Ww87ODs+ePcOWLVvQtm1buLu7Q05ODtOnT/9hu5Jcf+tLrGtra8PT05MxiXUeffv2Rd++fRmxFRAQgJycHBw8eBB9+vRBRUUFPDw8wOVyER4eDllZWdjY2MDb25t2nzQ25ySXy4W7uzstG3URVtdlZ2fHSF3XwIEDkZSUhN69e6OkpAT3798XKH+4fPkybdn5ujA5Z3gYGhrC2dkZKSkpjLUYYvveUTd6zWZLGgBUS4i69a8/Ff+2N9gEfby8vMj48eNJfHw8peYTHx9PdHV1yfbt22nbt7a2Jjt37mRlp2Tv3r1EV1eXxMTEkM+fP5PPnz+TmJgYoqurS3bs2CGyPXV1dZKXl0cIqVWUKi4uZnrIFAsXLiR79uwhhPCrKLm6upIZM2bQth8aGkpGjhxJzpw5Q8n9Xrp0iWhqalKfKwpGRkbkxIkTjb7H39+fzJo1S6zx1t0V59GQuhQd1q9fT5YuXUq+fPlC2f/06RMxNTWlrYzn4uJC1qxZQ9LT01mJ0Jw/f54MHjyY7N+/n1KvCg0NJYMGDSJBQUFi2ZRUNIKNsUvCNiH8Uu515yQTO+JMX6f1YWO+m5mZ8d0bsrOziYKCAnF2dqaOnT17VuR1TJLrLxsS61wuV+BaauhLHMaNG0euX79O/Xzr1i2ioKDAJ3f/4MEDoq2tLZZ9HmzOyZqaGnLmzBni6OhI7OzsiK2tLd+XOMyYMYMcO3aM+vnJkydEQUGB+Pv7U8du3rxJxo0bJ7Ltc+fOETU1NbJjxw4yc+ZMoqioSLWEyM/PJ4GBgURJSYlERUWJNXa25wwPNloMsX3vWLBgAbUGLFiwoNEvUdHT06OyA3itZxr6+hloioz9Aly6dAkODg58udpaWlrYsWMHbGxs4ODgQMv++/fvER0djdDQUHTu3FkgckBnB3T58uVIT0+nOrIDtWIJY8aMwbp160S2JyMjg1OnTkFLSwsPHjzAgwcPGqyloxslePr0qdBzO3/+fERERNCyDdTuFm7btg1jx46Fs7MzAGDixImQkZHBzp07BZpifo9Xr15BW1u70fcYGBiIpS4nSeLj43Hw4EG+Gpr27dtj06ZNWLhwIS3bsbGxyM7ObrCQm24D3+DgYNjb28PIyIhSy1u4cCFatWqFwMBAmJmZiWxTUtEINsYuCdtAbf1JfWU5HlVVVbRsM32d1oeN+Z6cnEypJgLA3bt3weFw+BRsVVRURBYckOT6+/fff2P37t186m/9+vXDtm3bsGLFCrFsuri4UBkZdHfqhZGXlwdFRUXq54cPH4LD4VDRWgDo0aMHiouLaX0Om3Ny165dCAkJgYKCAiOqlUCtamrde1N8fDw4HA5fdkn//v2/q3YrjClTpqCiogJhYWGQkpLC3r17KXGNgIAAREREwNLSElOnThVr7GzPGR5paWmM22T73jFo0CBKmGnXrl2Qk5MTUIUVFyMjI6ruXZTo/X+VJmfsF6CgoIBSkKlL9+7daS/qQK1jx1RRbn2aN2+OAwcOIDMzE+np6SCEQEFBQexUM1NTU+zfvx++vr6Nyr2Kq4xXFxkZGZSWlgocz8vLY6TY/s2bN3w3bh5cLpcqsBUFWVlZSumwMcRRl+MRHBzM97dXVVXhyJEjAg9kdNTx2HywpiPr/yNkZWUJLUTX0tISW2lr4cKFiI6OhpycHOzs7ODj48PYQ1Jd2Bi7JGwDtQ/+YWFhfKkylZWV8PPzg4aGBi3bTF+n9WFjvn/9+pVvjjx8+BCysrJQV1enjjVr1kxkaXtJrr9sSKzv27cPI0eORLdu3ZCbm8u4cErLli35xnb//n306NGDT5Th7du3tMW42JyTUVFRcHV1ZfwBuO5cS0hIQPv27SmBDaD2OhD3fzFz5kyhbWiWLl2KVatW0RLxYXvOsAnb945jx47B2NgY7du3h76+Pu7cucOYqExcXBzmzJmDli1b4n//+x8mTpxIu33Av0mTM/YL0K9fP9y7d09ArvbSpUvo378/bftsyErzqKysxOvXr/H582cMHDgQ8vLytJyB5cuXw8TEBMXFxdDX18epU6dYUzkzMDCAl5cX9u7dSx3LzMzEjh07MGbMGNr2e/TogeTkZPzvf//jOx4XFyfQC+RHUFJSQkxMTKO58Tdu3BA71/3333/H5cuX+Y517dpVYMeNbk8UNh+s6SiD/ghdunRBVlaW0F4u4ipKSSoawcbYJWEbADZt2oT58+fjwYMHqKyshIODA16+fInPnz/j2LFjtGwzfZ3Wh435/r///Q8ZGRn4/fffUV1djbt372LIkCF8DzPx8fECf9P3kOT6O2jQIFy+fBlLlizhO378+HH88ccfYtksLi7G8+fP0a1bN/j6+sLY2JjRB2t1dXVcuHAB69atQ2ZmJh4/fiwQ3Tx27BgtmW+A3TlZXl7O+Obsv1HXBYCRukI25wzbLYbYvnf06NEDK1euhKKiIgghcHFxabAmW9SoYlpaGt69e4euXbvCzs4Oo0ePZm2tkQRNztgvwKpVq2BtbY2MjAxUV1cjMjISWVlZVNEuE6SlpSE0NBRZWVnYt28f/vrrLwwYMEBsqe/8/Hzs2bMH169f54vWtGzZEhMmTMCaNWvEfghr06YN2rRpgyNHjkBRUZE16flNmzbBwsICw4YNQ01NDaZPn47S0lJwuVzY2NjQtm9ubg5HR0e8f/8ehBDcu3cP4eHhOHr0KGxtbUW2Z2xsjA0bNmDAgAHQ1dUVeP3mzZs4cOCA2GmtN2/eFOv3RIXNB2ugNlUxKCgIL1++RHh4OM6ePYtevXqJncZSlzlz5sDJyYl6sH758iVu375N9XcRB0lFI9gYuyRsA7UbVufOnUNYWBh+++031NTUYMKECTA2NhbZ4agP09dpfdiY75MmTYKbmxsqKytx+/ZtFBUVYcaMGdTrT548gY+PD+bNmyeybUmtv+vWrYOZmRmePHmCqqoq+Pn58Umsi4OOjg6WLFlC9b1rLK1bnGvJysoKCxcuRFxcHHJzc9GuXTsqBTc+Ph6HDx/GnTt3aK9jbM7JkSNH4u+//2ZM7hyoTe3fvn07UlNT8ejRI1RUVFDXfUFBAS5cuICgoCDs2LGDsc9kCjbnjKurK6spkGzfOzw8PBAQEIDc3FxwOBy8ffuWtngMDw0NDcybNw9dunQBIQQzZsxoMAWSbTEhJmjqM/aLEBcXh4CAADx79gw1NTUYMGAALC0t8eeff9K2XVc18NGjR7h8+TICAgIQFRUllmpgbm4u5syZA2lpaUyfPh0DBw5Eu3bt8PnzZ6SkpODcuXMAatX/RN25kqR6D4979+5R533gwIEYNWoUY3nR4eHh8PPzQ35+PgCgU6dOsLS0xOLFi8Wyt2vXLhw+fBiKiorQ0NBAu3bt8OnTJyQlJSE9PR1z5syhXWMoCd69e4ewsDC++c7Eg/WdO3ewbNkyTJo0CdHR0bh06RJOnz6NwMBAuLq6iqVYVZ89e/YgNDQU5eXlAABpaWnMnTsXdnZ2YkeFS0tLfygaQbdHDRtjl4RttmH6Oq0P0/O9oqICW7ZswYULFyAlJQVjY2PY29sDqFWeDAkJgaamJgIDA0VSF5X0+puWlobg4GC+82JmZiZ2ZKmyshK3bt1CcXEx7OzssHnz5gZVfcWNor948QJnz56FlJQU5syZQ5UYeHh44N69e7C2tmZEEY7JOenj40N9/+7dO5w9exZ//vknevfuLXCvEzfr4fTp01Rdl4WFBfXs4uTkRNV1rVmzRizbbCKJOQPU1qYtXLhQaEkKHSR179DT08OZM2do93XkUVxcjPPnz6O4uBg+Pj5YvHgxWrduLfS9bGZ3MUWTM/aL8+XLlwbrDX4UU1NTqKqqwtraGurq6jh//jx69uyJnTt3IjExEadPnxbJ3saNG5GdnY2goCChkuelpaVYunQplJSUsHnzZpFsm5iYwNfXF+3atYOJiUmj7z169KhItkUhNDSUkR19HkVFRSCEULK/hBCR6zl4xMbGIiwsDCkpKSguLkanTp2grq6O2bNn8xWS06GxlAoZGRnIyclh6tSpsLKyEvvvEMbTp0+hpKQk9u/PnTsX48ePh6mpKd9cDwwMxPnz53HhwgVGxvn161dkZGSAEIK+ffsyJv3PdiNygL2xs2lbUhszTF6nPwLd+V5aWgoOh8P3EPPgwQOUlpZCV1dX5LH/V9bf/Px8yMnJ0bIh6WbbbMHEnNTT0/uh93E4HMajEAUFBZCVlWXsIZ5N2JwzgwcPxrlz52hvODaEJO4dbMF2A3hJ0OSM/cR8/foV8fHxkJaWxtChQyllGR4xMTFwdHTE33//TetzhgwZglOnTqFPnz58D6jZ2dmYOnUqHj16JJK9kSNHwtPTE8OGDWvwPbdv38a2bdsklvomCkFBQbh06RJkZGQwdepUGBsbU6+9ePECW7ZswZMnT2ilhKWnp0NaWlpo/VZaWhrs7e1x5swZse2zzZEjR7B7927MmzePEmZ49OgRjh8/jnnz5qF9+/Y4cuQIFi9eDEtLyx+y+eTJE1y+fBnS0tKYNGkSX3F3eXk5vLy8cPToUaSkpIg9bnV1dZw7dw69evXim+s5OTkwNDTE48ePxbZdWloKaWlpgesUqN1tdnR0hK+vr8h2JRGNYGvsbNvmUd8xqK6uRnZ2NkpLSzFp0iSx05/YvE4lMd9/RvLy8nDjxg1IS0tDX19fQMTj2LFj2Lt3LxITE0W2HRUVRQkBREVFNfpeulHyhvo6cTgcasNKnJqmn/3e8bMhqTmzbt06/Pbbb1i5ciWjm1Rs3jvYrHmTZH9NSfDzucBNAKiduBYWFtSuV48ePXD06FH8/vvvKC4uhpOTEy5dukSrAS4PplUDP3369N1Qe79+/fDu3TuRbdfn7du3aNeuHdq0aYP4+Hhcu3YNGhoaMDQ0FMvevn374OfnBy0tLTRv3hyurq6QkpLC3LlzERQUBC8vL7Rq1Ursh96cnBxYWVkhIyMDQK28dEBAADp06IDKykp4e3sjODhYLLUtd3d3rFy5knak9Ee4dOkSNm/ejDlz5lDHDAwM0LdvXyodZcCAAXB3d/8hZyw6OhobNmyArKwspKWlcfjwYRw+fBhDhw7Fo0ePYGNjg5ycHNoKX23btsW7d+8E5mdGRobYCmdFRUWws7NDXFwcOBwOxo0bBzc3N8r5CA8Ph6enJyorK8Wy/+bNG9TU1FDfMwmbY2f7vNRFWBSGEAJnZ+cGU1sag83rFJDcfAdqHVVhD0l1nYKpU6eK9UDD9Pp77949LF++nKoz3r17N44dOwYFBQXk5OTAxsYGjx49anSjrzFsbW0xatQodO7cudHaKg6HQ9sZMzU1pa5b3p54/f+DpqYmvL29f0jhju05yYMQAl9fX3Tp0gVz584FUFvzqauri2XLltGy/TMiqTnDRoshNu8dAH/Nm6urK6NZAiYmJrhz5w46d+5MrWHCYktMKLdKBEk0M2uCeRYvXkwmT55MHj58SB4/fkwWLFhAVq9eTV6+fEnGjBlDlJSUiJeXFyONa7ds2ULMzMxIcXExX3PNyZMnEzs7O5HtCWsOXJ/379/TbpJ47do1oqSkRG7fvk1ev35NlJSUyPjx44mqqipfg0lRGDt2LPH19aV+joyMJBMnTiT79+8nCgoKZM2aNaSwsFDsMVtZWZExY8aQyMhIcunSJWJoaEg2b95MPnz4QIyMjIiCggLZuHEj+fjxo8i26zan5GFpaUkKCgrEHm9DDBo0iLx69Urg+KtXr8igQYMIIYS8ffuW+v57TJs2jaxatYqUl5eTyspK4uzsTIyNjclff/1FlJSUiL6+Prl79y7tcbu7uxMjIyOSmppK1NXVSWpqKomNjSW6urpk586dYtncuHEjGTJkCPHx8SEHDx4kI0eOJG5ubuTLly9k6dKlREFBgcyfP59kZWXRHj/TsDn2/8J5ef36NRk2bJjIv8fmdUqI5OY7IYTs2LGDcLlcYmRkRHbs2EF27NhBZs+eTRQUFMiKFSuImZkZUVJSIn/99ZdIdtlYf+fMmUMWLFhA3r59Sz58+EBWr15NzM3NSVJSEhk8eDAZOnQoOXXqlFi2Jc358+eJrq4uuXHjBikpKSElJSUkJiaGGBgYkODgYJKYmEhmzZpFtm7d+kP22J6TPPbu3Uu0tLTItWvXqGMhISFES0uL+Pn50bLdRMN4e3s3+vX/G2/evCE1NTXU9419/Qw0OWM/KUOHDiX37t2jfs7Ozibq6urE0NCQGBoaktTUVMY+6/Pnz2TOnDlEUVGRKCgokCFDhhAul0umTZsm1sKuoKDwXYeFCWds2rRpZM+ePaS6upr4+PiQsWPHkurqanLp0iUyfvx4sWyqqKiQly9fUj+Xl5cTLpdLBg8eTCIjI2mNlxBChg0bRm7evEn9nJaWRjQ1NYmxsTHR1tYmMTExYtsW5gTznGummTBhAjl8+LDA8cOHD5OxY8cSQgiJj48nOjo6P2RPTU2NPHv2jPq5uLiYKCkpkWHDhhE7OztSVlbGxLBJRUUFWbduHVFQUCAKCgqEy+USBQUFsnTpUvL161exbGpra5MLFy5QPycmJpJRo0YRKysroqamRo4fP87I2Hnk5uaSz58/E0IIuXfvHnF0dOT7fFFgc+ySPi/CiIuLIxoaGiL/HpvXKSGSm++EELJq1Sri7OwscNzNzY1YW1sTQmqv25kzZ4pkl431V0NDgzx69Ij6+f3790RFRYXo6uqSxYsXk/z8fLHs/giFhYXk8uXLJCcnhxF7BgYGJC4uTuD4nTt3qDUyKSmJaGtr/5A9tuckDx0dHXLr1i2B4zExMURPT4+Rz/hVYHrOsA2T9476lJeXEz8/P2qTdvPmzURNTY2YmZmRoqIiRj6DR0VFBXny5AkpLS1l1C6bNKUp/qSUlpaiT58+1M89e/ZEZWUlOnfuDH9/f6H1F+LSpk0bnDx5klHVwMb6TQCgFNXokJmZCR8fH0hJSeHOnTvQ0dGBlJQU1NTUkJubK5bN8vJyvpQRWVlZtGjRAuvWrWNEaa+kpISvWaeCggLKysrw5csXnDt3jirC/q+zfPly2NraIjk5Gerq6qipqcHjx49x5coVODk5ISsrC3Z2dhg3btwP2fv69StfbUi7du2oWpotW7YwNm4ZGRns3r0bq1evRmpqKjXX+/fvLzQF4kf49OkTXzNdDQ0NFBYWIjU1FadPn2YklZjH9evXYW1tjYCAAPTs2RMWFhbo2bMnzp49i+LiYsyfP/8/M3ZJnhdh9RBlZWW4c+eOQD+jH4Ht61RS8x0Abt26hbNnzwocnzNnDqX+pq+vj3379olkl43198uXL3yqbl26dAEAqKqqYvfu3Yyp2AK1tVerVq2Ci4sLFBQUMGXKFHz48AGysrI4ePCg2KmQPN6/f4/ff/9d4Hi3bt0oBcRu3brh8+fPP2RPUveOT58+CVXWk5eXZ6TJ+c8M23OG6RZDdWH63lEfT09PnDt3DqNGjUJcXBwiIyOxevVqxMTEwN3dnZaIUl5eHuzt7bF27VoMHDgQM2fOpEoLQkJChDZA/6/B3MrVhESpqakRUL2RlpbG2rVrGXXEgNou7SUlJRg+fDjMzc1haWkJHR0dfPz4USwHZOjQoXj//j3evHnT4Nf79+8p4Qdx4cnlf/78GU+ePKGUArOzs9GhQwdatuvDlAphdXW1QB8OGRkZ2Nra/jSOGABMnjwZPj4+ePfuHfbs2UN97+/vjxkzZuDDhw+YPHkyNm7c+MM26+ebczgcvpo0JtDX18enT5/Qu3dvjB8/HhMnTkT//v1RUFAg9o20qqpK4JqUlZXFtm3bGHU4AODAgQMwNzfH8OHDceHCBfz++++4dOkSXF1dxepdxObYJXlehK0xZWVlMDU1xbZt20S2J4nrVBLzHajdbHv58qXA8YyMDKomuKysTOT7ChvrLyFEwOGSkpLCkiVLGHXEgNo2IL1790bfvn1x8eJFVFVVITY2Fubm5vDy8qJtX1lZGYcOHUJ1dTV1rLq6GocOHaLEWh48ePDD6nmSundwuVyhzvu5c+fQv39/xj7nZ4TNOZOSkoJZs2bhzZs3SElJQUVFBVJTU2FmZobY2FjaY2f63lGfK1euYM+ePVBSUsKNGzegqamJZcuWYcuWLYiJiaFle+fOnfj8+TM6deqEy5cvIzc3FydOnMDYsWPh4eFBe+ySoCky9ovB1KIbGxuL5ORkALWqNf7+/gLCD69fvxZrh5NNSeO66OjoYNu2bWjdujXatm0LbW1t3L17Fw4ODhgzZgyjn8V2HyRhO6jiwKbMdn10dXWFNpcGah1yJhSOmNh4iI6Oxq1btwDU9sBzcnISiNrymlYyCdMOB8BONEIYbIydTduSWnOYuk4bgumNNgCYPn06tm7diqKiIqiqqlJR7P3792Pq1Kn4+PEj3N3dRb5eJbn+siFp/ejRI5w6dQqdO3fGrVu3oKOjg27dumH69Ok4fPgwbfu2trYwNTXF/fv3oaysjJqaGjx79gwlJSU4dOgQkpKSsGXLFtqRUKbn5IoVK7B06VIkJCRATU0NAJCcnIx//vmHturpzw6bc8bT0xNmZmZUiyGgNsOodevW8Pb2Frnfa33Yvnd8+vSJWtvv3LlDbSx16NCBEuQRl/j4eISGhuJ///sfPD09MXr0aGhoaKBjx46MiBxJgiZn7CeFw+EI3Tllih49esDJyYlKzYqOjubbeeRwOGjVqhVsbGxEtv327Vt0796d6sjeGHRuJFu3boWXlxdycnLg5+cHWVlZJCYmQk1NDZs2bRLbbnBwMJ+KZFVVFY4cOSKgUiVOo0Fh/1cmqZ8eWllZCQ8PDwFFOSb6LiUkJCApKQmVlZUCKX7inJtHjx7xnWNCCJ48eUKl9PAQ9aFRXV0dJ0+epMb49u1bvh1m3lzftWuXyGPm/b6knOD60QieUqW40Qg2xy7J85KVlYWkpCQUFhaiffv2UFVVpaIPMTExaNGihUiRT0mMna35Xp81a9agoqICO3bsQHl5OQghaNGiBUxMTLBmzRrExMTgy5cvcHFxEckuW+tvfn6+QBp7QUGBwIYYXSdESkoKsrKyqKqqwoMHD7B161YA4kUJhaGsrIzo6GicPHkSz549g7S0NKZPnw5jY2N06tQJmZmZCAgIgLa29g/Zk9T1NGrUKBw/fhzHjh3D7du3IS0tjX79+uH06dN87Rf+f4TNOZOSkoLt27cLHJ8/fz4iIiJo2QaYv3fUp1evXkhOTkZhYSHevHlDNTb/66+/aPdOq6ysRPv27UEIwb1797Bu3ToAwjPI/qs09Rn7SRHWs4E00MyRrqynnp4eTp8+3WBndlH5kd4TvL/lvyZJynbzSy6Xi+7du/M5vm/fvkW3bt0EHjZEtf+9Jqx1oRtJ8PX1pSSZ6+9ai3NuePPke8sV3TljYmICHx8f2vLPdeFyuVBXV+dz8BISEjBo0CCBCNyRI0dofZa9vT1evHiB1q1bIy0tDbGxsUhISICDgwOGDRsGJyen/8zYJXFe3r17B3t7e9y+fZtv7nA4HGhoaMDZ2RnLli2DjY0NDAwMRBo7W9cpz74k5ntdvn37hszMTDRr1gzy8vKsROHo8iP3PabuHRYWFpCTk0OnTp0QHByMuLg4VFZWYuvWrZCSkoK/vz8t+0zD9pxs4vuwOWeGDx+OgwcPYtCgQXw9MO/fvw9ra2vcvXuX1tiZvnfUJyoqijoP6urqCAkJga+vL3x9feHq6kqr5n7+/PnQ0tJC165d4eTkhL///hudOnXCtm3bkJubK7HMCDr8HC5jEwIwEbn4Ueo3Xq6srERaWhr69u0rVo+e0NBQ6mGX7sPn97hw4QKGDh0KOTk5HDhwANHR0dDQ0IC9vX2jAiINwXYTanEiRj+KJBeksLAwWFtbY+nSpYzYk9TDQ/1zVFRUhAcPHkBZWVns3TueCEJdhBXAMwHT0Qg2x872eSktLcWiRYtQU1ODXbt2Yfjw4ejYsSOKi4tx//59+Pr6YsqUKVBVVRXJEQPYvU4ByT8sf/36Fenp6VQUm5eiDtCLvDG9/rJ9v6jL1q1bYW1tjZycHGzevBmdOnWCs7MzMjMzERgYSNv+ly9fEBIS0mD2gKh/K9tzkgfbzbB/ZticMwYGBvDy8sLevXupY5mZmdixYwcjab9sRbJ5TJs2DVwuF2/evMHo0aMBAIMGDUJQUBCGDx9Oy/amTZuwbNkyfPz4EZaWlpCTk4ODgwNu3LiBQ4cO0R67JGiKjDXxXeor1cyYMQOZmZmMKdW8evUKpaWlUFZWBlDrrOno6EBeXp6W3QMHDsDf3x8hISEghGD+/PmYNWsWHjx4gNGjR8Pe3p6W/X+ThqKgjbFw4cIfeh+Hw0FoaKg4w6JQVVVFdHQ0a04HW9RXw5o4cSKjalhNSA4fHx9cunQJp06dElpT9OjRI8ybNw+LFy9m5GHjZ+XGjRuwtbVFaWmpgENAJ8L0K66/RUVFaN++PSM1whs2bMCNGzegra1NNcatiyQ3XEWhoVTE5s2bQ05ODlevXpXwiP7bMDVnSktLYWFhgSdPnqCmpgZt27ZFaWkpuFwuDh8+zLgoGdvQ3dSvT01NDUpLSym166ysLHTs2PGnOS9NkbFfhKSkJMjLy6NTp06IiorC5cuXoaGhgSVLltDOI6+vVJOXl4cTJ07g7Nmz8PDwQHBwsNi27969i+XLl8PU1JRyxi5dugQvLy8EBgbSUlQ8c+YMdu3aBQ0NDbi6ukJNTQ3Ozs5ISEiAtbU17YeBwsJCeHh4ICUlBd++fRN4kKG7u+3i4gJbW1uBnOecnBxs3LgRJ0+eFMne9xyjhIQE5OTk8En3i4uGhgYePXrEijNWUVGB4OBgTJgwAb1794a9vT214+7p6YmOHTuKbbshNayTJ0/Cy8tL5HMujG/fvlG1BZmZmYiJiYG6ujo0NDRo2waYj0ZIYuwhISEwNTUVOF5SUoLt27fz7Qb/KJcvX8bq1asbFHfw9fWFhoYG/v77b5Gdse/VutaFCQGFGzduID09nU95r6KiAsnJyYwIAwwfPhxWVlZCnQJxYXv9Bdid60Dt/5mXah0fH49r165BQ0MDhoaGtG3//fff2LNnT4MiR3Rha+xpaWl8P1dXV+PVq1dwcHBgRe3zZ4Ot885Gi6H6sHk9sS0/X1FRQaVWs3FfZR0J9TNrgkXCwsIIl8sld+/eJampqYTL5ZLFixeTYcOGMdKZfejQoVQD0jVr1pDVq1cTQgh5+fIlUVNTo2V7+vTpZM+ePQLHd+/eTebMmUPLtrKyMsnLyyOEEDJp0iTqXLx584aoqKjQsk0IIVZWVmTEiBHExcWFeHt7C3zRRUtLi0ybNo2vyXR4eDhRU1MjRkZGtO3z+Pz5M7G3tycKCgrEzMyMOmd0OH36NNHU1CQ7d+4k4eHhJDIyku+LDjt27CCampokJSWFxMbGEkVFReLn50fmzJlDbG1tadlWV1cnGRkZhBBCli1bRmxsbAghtU3VVVVVadkmhJAHDx6QoUOHkrt375KCggKirq5OBg8eTP744w8SHR1N276vry8ZNGgQSUxMJAkJCURBQYFs2bKFjBs3jri4uPxnxz5o0CCyZMkSvmbwt27dIqNGjfrhxuD1UVVVbbTZ6u7du8mzZ8/E+r/yGoL/yBddPDw8iIKCAtHW1iZcLpfo6OgQJSUlwuVyyfr162nbV1ZWJq9fv6ZtR5hdNtdfNuc6IYRcu3aNKCkpkdu3b5PXr18TJSUlMn78eKKqqkqOHTtG2/6QIUNYOe+EsD92YTx9+pQYGBiwYvtngc3zbmJiQoqLiwWOf/jwgUydOpWWbULYv55WrVpFZs6cSXJycsjZs2eJuro6SUxMJPb29mTx4sW0bLN9X5UETc7YL8D48eOpC3337t1k8uTJhBBC4uLiiK6uLm37ampqJDc3l9TU1BBNTU1y8uRJQgghGRkZZMiQIbRsq6qqkuzsbIHj2dnZtB09AwMDcvfuXfLq1SuioKBAkpKSCCGEREVFkXHjxtGyTUjteXnw4AFtOw1RUFBAzMzMiJqaGjly5AhZunQpGTRoEPHz8yNVVVWMfMadO3eIrq4uGTJkCImIiGDEJiG1D6sNfdF9QB01ahS5ffs2IYSQbdu2kUWLFhFCCElOTibDhg2jZXvw4MEkOzubVFZWEg0NDcpxTE1NJVpaWrRsE0LI3Llzia2tLfn8+TMJCgoio0aNIt++fSPHjx9n5Iaqp6dH3Xx27NhBbWg8fPiQjBw5kpZtNseelpZGDA0Niba2Nrlx4wZxdHQkXC6XbNmyhXz+/FksmyNGjCCpqamNvic1NZVoa2uLbPv+/fvUV2hoKNHU1CQhISHk8ePH5NmzZyQ8PJyMGjWKhIWFiTX2uowePZqEhoZS3+fm5pKPHz8SY2Nj4uXlRdu+oaEhuX//Pm079WF7/WVzrhNCyLRp08iePXtIdXU18fHxIWPHjiXV1dXk0qVLZPz48bTtb968mezdu5e2HWGwPXZhpKamMuJk/8wwfd5jYmKozV0ul0t27dolsOm7YcMG2s9hhLB/PbG5qc/2fVUSNKUp/gK8efOGUvm7c+cOVRzZr18/fPjwgbb9P/74A6dPn0bXrl1RUlICHR0dVFRUIDAwkLaUbadOnZCWloaePXvyHX/x4gXtlJm5c+di7dq1kJWVhYKCAtTV1XH8+HG4u7tj9erVtGwDtT1/unbtSttOQ/z2228ICgrC5s2bsWPHDkhLSyMkJIR2M2ygtnjczc0NERER0NbWhouLC7p3787AqGupn8rCJGz2K1FTU0NAQAA6deqE8vJyjB49GgUFBdizZw/VU4cOz549g7u7O9q0aYPbt29jzJgxaN68OXR0dODm5kbb/rt376geNHfv3sX48eMBAN27d0dJScl/duwKCgo4c+YM7OzssGLFCjRr1gz+/v60eucMGjQIly9fbnSNunjxIlRVVUW2rampSX3v6uoKFxcXjB07ljqmqKiIrl27wt3dHXPnzhXZfl0KCwup9V1BQQFPnjzB+PHjqVS/NWvW0LK/YcMGODs7w9raGn379oWsrCzf6+KmWbK9/rI51wH2+y7VVdzr06ePwHmnUzPG5tiFCXiUlpYiIiICKioqtGz/7DB93tlsMVQftq8nNuXn2b6vSoImZ+wXoHPnznj37h2kpaWRmpqKDRs2AKh9IO7SpQtt+2wq1UydOhUODg749OkT9VCUnJwMLy8v2qpM5ubm6NOnD3JycjBlyhQAtb00tm7dipkzZ9KyDdSqwQUFBcHZ2Zm2LWGUlZXB3d0dUVFRmDZtGp4/f47Vq1dj69atmDBhgth27927B3t7exQXF8PJyQmzZ89mcNTsw2a/ErYV1Fq2bImKigqUl5cjMTERs2bNAgB8+PCBkXodOTk5ZGVloby8HBkZGVSPooSEBMjJyf1nx15TU4OQkBD89ddfGDJkCHJycrBr1y506NBBLGcJqBWsWbJkCSXEUp+zZ88iNDSUds1VVlYW+vfvL3C8V69eyMvLo2UbqF2zvnz5QtnMyMgAUOskFRQU0LZvZWWF6upqWFlZMSoRz/b6y+ZcB9jvu/TPP/9Qc/vdu3e07dWFzbHb2toKHJOWloa6ujocHBxo2f7ZYfq89+/fn6o9Z7rFUH3Yvp7Y3NRn+74qCZqcsV+ASZMmYcOGDWjZsiXk5OSgqamJ6OhoODs7M3LTU1FRwe3bt/mUahYtWoS1a9fSXthXrFiBjx8/wsnJCZWVleBwOGjWrBlMTEwY2T2t3xds8uTJtOzVVSSsqqpCUlISYmNj0atXL4EiWroyzBMnTkRVVRX2798PAwMDVFVVYd++fVi/fj2io6Ph7e0tkr0vX77A3d0d4eHhGD58OHbs2MFoNOxH+sfxoNMDyMLCAuvWrYOUlBSGDRsGLpfL16+EDr1798bZs2f5jq1YsQKbN29mREFNS0sLHh4eaN++PaSkpDBq1CikpqbCxcUFWlpatO2zGY1gc+wzZ87Ey5cvsW7dOixatAglJSVwcHCAsbExTExMhD4Afo8RI0Zg1apVWL9+PQIDAzF48GC0bdsWxcXFePjwIV6+fImNGzfSjjQrKCjgyJEj2LZtGzXnq6qqEBAQgEGDBtGyDdSed09PTzg7O0NVVRUBAQEwNjbG1atXGXkwCw4OZq1ZMNPrb13Yjrzp6Ohg27ZtaN26Ndq2bQttbW3cvXsXDg4OjEiJs9lqhM2xs5n18LPD5nlnusVQfdi+ntjc1Gf7vioJmqTtfwFqampw/Phx5OTkYP78+ejduzeOHj2KoqIirFq1ijGlHTYpKytDZmYm4uLiMGzYMEZS8crLyxEeHi5UhSwlJUUsCV47O7sffi9daeJly5Zhx44d6Ny5M9/xxMRE2Nra4vr16yLZ09fXx9u3b9GzZ09qp7ohxOlZExkZiUmTJkFWVhaRkZGNvldYjylRSEtLo/qVyMrKIi4uDjIyMrT7lXxPJY+uMl5RUREcHByQnZ2NlStXwsDAAG5ublQ0mIm015s3b1LRiI4dO+LChQsoLy+nvTHD5tinT58ODw8PKv2Ux8WLF+Hs7Iz79++LbTsxMRHHjh1DYmIiPn78iE6dOmHIkCEwMTFhJPU0ISEB5ubm6Nq1K/744w/U1NQgJSUFX79+RWhoKO1d37y8PCxfvhzTpk2DsbEx5s6di2fPnoHD4cDW1haLFi2i/TewARvrb33YmutArXIor++SpaUl1NTU4O3tjZycHGzbtq1Blc7GePjwIdTV1SEtLY2HDx82+D4Oh0PrHtjY2Ldv387Iw3tmZibS09MhIyODfv36oU+fPrRt/uywMWd4sN1iCGD3egLYk58vKirC9u3bkZOTw9p9lW2anLFfhM+fP+Pjx4/o1asXAODatWvQ1NRkJJ2CjSiHr68vjhw5goiICPTu3RuPHj2CpaUlSktLweFwMGzYMPj5+VFSpeKwdetWREVF4Y8//kBycjLU1dXx+vVrFBYWwtTU9KfuLfTlyxe0atVKpN+pv0vdEBwOhxFZ/oULF1Lz8WeBzYgeUOtcjBw5ku+6rKioEKgX+S/C5tirqqoarBsoKChAt27daH8Gm+Tk5CAiIgIvXrwAUBslnjdvHn777TfGPqO8vBzNmzfH169fcfv2bXTr1k3sGp2FCxfCx8cH7dq1g4mJSaNzXtwI/7+x/hYVFbGWxsUEXC4Xd+7cobIHGoJOeihQ6/SpqalBRkaG73h5eTliYmLw559/im27vLwc69evx19//cU3Xl1dXXh5ef0Ua9nPyOrVq5GXl4e9e/fi4cOHcHZ2xqFDh3D27Fm8ffuWVouhfxNei47BgwczbvdnmotNaYq/AE+fPoWZmRmmT59O3eDc3NxQWVmJoKAgDBw4kJZ9V1dXvpt1VVUVXr16haioKLEKR8PDw+Hv7w9TU1Mq6mNnZ4cWLVrg5MmTaNu2LVatWoWDBw/SCo/fuHEDO3fuhKGhIcaOHQtnZ2f07NkT1tbWqKysFNtuXdjs7wYAsbGxOHToELKyshAeHo6zZ8+iV69emDp1qsi26qc5sElkZKTQvlHioq+vj9OnT6Njx47Q09Nr9NzScSTrP3hWV1cjKysLISEhYqXK1cfJyQknTpzgc2iYvGGwGY1geuw+Pj4wNzdHy5Yt4e/v3+D7OBwOVqxYIfbn8MjNzUVERASeP3+OZs2aQUlJCbNnz2akrrZnz55Yv349bTvCqKmpwYEDB9ClSxfMnTsXLVu2RGBgIPT09MR2xnr06EFlTNCts2wIttffkpISeHh4YMGCBejfvz8sLCwQHx8PeXl5HDx4UEAUShzS0tKQnp6OmpoaALV1dLyHRxcXF5Ht3bhxg3IWr1+/zlrWysKFC3Hnzh0BxzQjIwMbN26k5Yzt3bsXT548ga+vLzQ1NVFTU4OHDx/CxcUF3t7erF0HPwtMzxke8fHxCA0Nxf/+9z94enpi1KhR0NDQQMeOHTF9+nTa42Y7kp2SkoKtW7fynZu60N3oLCoqQlZWltDzvnz5clq2JUGTM/YL4ObmBj09PVhbW1PHrl27hq1bt8LNzY32jklDF7qysjJOnTolsmNw6tQp2NraYv78+QBqBTtevXoFa2trqhB++fLlcHNzo+WMlZSUUA3/+vfvj2fPnqFv375YunQp1q5diy1btohtGwBOnjwJR0dHBAcHo2PHjrCzs8Pw4cMREhKCyspKsVL96nLnzh2sXLkSkyZNwuPHj1FTU4OqqirY2dmBEEJb4IRNdHR0cOzYMaxcuZJWagYPIyMjKkpqZGTEWo1LXZU8HsOHD0fPnj3h7e39w9HFhpCXl0d6erpQwQcmcHFxaTQaQQemx3727FnMnz8fLVu2FKjTqwsTzlhSUhLMzMzQsWNHKCsro7q6GuHh4QgNDcWxY8cwYMAAkezVjSzVrSMVBt3a0f379+PkyZN8QkGTJk2Cn58fgNp0ZlGpm0Ktra0tEPFkArbX3507dyIhIQGmpqa4fv06EhIS4O7ujujoaLi7u4tcU1ufw4cPY9euXQBq5yAviYhOCmGPHj2o762treHi4kI7jZVHSEgINV5CCCXAUB+6ioe81OG6zaoNDAzQrFkzODo6/n/tjLExZ3iwqUYIsHvvAGqv12bNmmHLli3YuXMnbG1tkZ2dTdWl0eH8+fPYsmULKioqqPPOe0bo0aNHkzPWhGRISUmBq6sr3y61tLQ0lixZwliurzBUVFTEihZkZmby3Sji4+PB4XD4ZKz79+//3fqd79GpUycUFhbi999/px4kAaBjx46MSP6HhoZiy5YtGD58OPbs2YMBAwYgODgYt27dwvbt22k7Y7xdRlNTU2pXytraGm3atEFQUNB/2hl7//49oqOjERoais6dO6N58+Z8r4savap7LletWsXIGEVBXl6ekcJ1LpeLDRs24NChQ5CXlxc4L3TrDNmMRjA99rqRWrajtrt27cKECRPg7OxMPbhUVlbCzs4OO3bsQEhIiEj26kaW6j5gs0FUVBQ8PT0xcuRI6tiiRYsgLy8PJycnsZyxugiLeDIB2+tvbGwsfH190a9fPwQGBkJbWxuTJ0+GgoICtdFHh+PHj8PS0hIrV66Erq4uIiMj8enTJ6xfvx76+vq07efk5Iicat4YCxYsQIcOHVBTU4PNmzfDzs6OT0mOJ4M+bNgwWp9TVlaGvn37Chzv06cPioqKaNn+2WFzzrCpRgiwH8l+9uwZQkNDoaKigrNnz2LgwIEwNjaGnJwcIiIiaClE+/v7Y9KkSbCwsMC8efMQHByMd+/ewdHR8V95XhCHJmfsF6B169bIyckRSMt49+4dazmzZWVlOHbsmNgpPnUjGwkJCWjfvj3fglJWVoaWLVvSGuPo0aPh6OiInTt3YvDgwXB1dcXYsWMRHR3NiFQr2/3dnj9/LnTHaPz48fDx8aFtn020tLRYUTHKzMykRB4OHjyIiooK6jUVFRXqfyAuwjYASktLERAQwEg6V1ZWFpUb//79e9r26sNmNILtsQO1UsSVlZWoX8pMVzglLS0NO3fu5NtBlpGRwbJlyygZZFGo63jSdaC/x6dPn4Q6fPLy8oz8H9iK1rK9/n758oVSg71z5w4lI96iRQu+NCtxyc/Px6xZs9C8eXNwuVwkJyfDwMAAtra2cHNzox0tsLCwgL29PczNzdGrVy+B+mhR57y0tDS1QcfhcCgxJaYZOHAgrly5gqVLl/Idv3z58v/3Ih5szhk21QgB9iPZNTU1lJBG7969kZ6ejiFDhkBfXx8BAQG0bOfk5MDb2xv9+vWDgoICioqKoKenh6qqKvj7+4tV1iFpmpyxX4A///wTjo6OcHBwoFIQkpOT4eTkxNeIVFwaEjXgcDhwdHQU2d7AgQORlJSE3r17o6SkBPfv3xfYNbp8+TLtWjcbGxvY2triwYMHMDY2Rnh4OGbNmgVpaWkqlYAObPd3a9u2Ld69eycggpGRkYH27dvTts8mdKOCwti2bRtOnTqFK1euoHfv3vDz80O7du3QrFkzfPnyBVVVVbhy5Qqtcy+sHo0QglatWsHDw4Pun8CqnDXAbjSCzbEnJSXBzs4O2dnZfMfp9rri0atXL0oGui65ubmMtHfIy8vD8ePHkZ6eDmlpaQwYMABz5syh7UQCtevv2bNnBdK/zp07x4gDxVa0lu31t1+/foiJiUH37t3x/v17aiMmIiJCQJVTHFq1akU5dbz+bgYGBujXrx8jTZ+9vLxQXV2Nhw8fMtrfDahN5WayhqZu25Lly5fDysoKqamp1MN7YmIirl+/jt27d4s95l8BNucMmy2GAPYj2b1790ZiYiIMDQ3Rt29fJCcnA6gVn6u7qSoOsrKy1MZD79698eLFC4wePRrKysp4/fo17bFLgiZn7Bdg/fr1yM7OxuLFi/kW9bFjxzLSmb2+gAdQu6usqqoqVpH0/PnzsX37dqSmpuLRo0eoqKig5JkLCgpw4cIFBAUFYceOHbTG3a5dOxw4cID6+eDBg0hNTUWXLl0YUTlju7/b5MmT4erqSp3/srIyxMXFwdnZWWgT2/8aaWlpCA0NRVZWFvbt24e//voLAwYMEFqX9T14TlhgYCB69+5NHT927Bh69uyJ4uJiTJ48GWFhYbTSEkJDQ4XO9YEDBzIiBw3Uyh9fuXIFL1++hJmZGdLT0zFgwAB07NiRtm22oxFsjd3FxQVdu3aFjY0NK006LSws4OLignfv3kFLSwvS0tJITk7G3r17YWxszCczPnToUJFsP3/+HAsWLECLFi2goqKCmpoanD17FsePH0dYWJjI9Wj1WbFiBZYuXYqEhARKij85ORn//PMPfH19adkG2It4sr3+rl69GqtWrUJlZSUMDQ0hLy+PnTt34vjx44ycFw0NDRw8eBDbtm2jUsSWLFmCxMRERtYCus3GG4NXQ8NLL6NbQ1M3Uj1mzBjs27cPgYGBiImJASEECgoK8PLywrhx45j7I35C2J4zUlJSlCMGgNFIJNv3DhMTE9jb2wOoDSBMnToVLVq0QFJSEu0WIzz9gnXr1mHgwIGIjY2Fubk5MjIyBBRF/6s0Sdv/Qrx8+ZKv74e8vPy/PaQGOX36NMLCwiAlJQULCwtK3cnJyQkRERGwtLTEmjVrRLYrSp0Z3V3rxvq7rVy5UqwmwXZ2drC3t0ebNm1QWVkJW1tbXLp0CcD/FQTzbob1d7D/S6SkpGDevHlQU1PDo0ePcPnyZQQEBCAqKgq+vr589YE/wty5czF58mS+WhANDQ2cO3eO2hAIDQ3FuXPnGhWD+Lf58OED5syZg8LCQlRUVODq1avYsWMHUlJSEBoaSntHv6SkBLa2ttDW1oaxsTGWLFmCW7duUdGISZMm/SfHPmjQIERFRTES0RDGj9ZUiBOR4ClC7t69m7omy8vLsWHDBlRUVNBOwQGAR48e4ejRo3jx4gWkpaXRr18/WFhYMCb+wBSSXH8B4OPHjygoKKDOw5MnT9C6dWtG5lF6ejrMzMxgamqKefPmYfLkySgpKcHXr19hbm5OCSgwAdMy3BMnToSqqqrQGpp169aJnLZVV5K/iYZhc86w3XaFzXsHj7/++gsdOnTAkCFDcOHCBQQGBqJ79+7YunUrrTKAhIQEWFhYYPXq1TAyMsL48ePRpUsX5OXlYeLEibRULCVFkzPWhFBEqUliMiWtoKAAsrKyYu+0f2/BAphLfWKDuukgPLKzs/Hs2TPU1NRg4MCBrCnxMYmpqSlUVVVhbW0NdXV1nD9/Hj179sTOnTuRmJiI06dPi2Rv8ODBOHPmDN8GQ127QG092cyZM/Ho0SORbEuykfeGDRtQWlqKvXv3YsSIETh//jzatWuHtWvXonnz5o1KvIsDIYSxaASbYx83bhw8PT1pK701hCgpQqIKcqirq+PkyZNQUFDgO56WloYFCxYgISFBJHv/BkxFPP+N9fft27fIzMzE0KFDUVZWxqjD8O3bN3z58gWdOnXChw8fcOHCBXTv3h3jx49nxH5YWBgCAwORn5+Pq1ev4tChQ+jWrRusrKxo2a27uWFiYgJLS0uMHj0a165dg7+/v8gbVlwuF1u2bPkhZdz/srCUJGBrzpw9e7bRFkNM10Uxee+QBAUFBaioqEDPnj2RkZGBkydPonv37jAxMfkp+o01pSn+pNR9aGdjx+RHF2sOh8OoM0a3uStdGenvUTdy9b2HeHEe3IXtjfTq1euna56ckpKC7du3CxyfP38+IiIiRLZXU1MjkG5w9epVvvowWVlZsaKRb968Efl3xCU+Ph4HDx7kE6dp3749Nm3a9F2J9Ib4XjSiQ4cOqKqqwtu3b2lFI9gYO4/ly5fD1dUVjo6O6Nu3L+OpJWwqHrZu3Vqo2hgdBbK6Pdi+tzFGd/2tH/GcNWsWgoODxYp4sr3+1qWiogKbNm3C5cuXISUlhatXr2LXrl0oKyuDt7c3Iy01WrRoQQlrdOnSBYsXL6Ztk8eFCxewe/duLFq0iBJg6NevHzw9PdGiRQuYmZmJbZuNGpofiS5wOJz/750xtuYM0y2GAPbvHZLc1K/77Ni/f3/agiOSpskZ+0lxdXWlaiuE1XTRRZINgpmksXokJlJB3rx5QxVEs/UQz1YPLUkiIyOD0tJSgeN5eXliqWR2794d6enpfA/V9XfrUlJSxEp1YFtUoy5lZWUNyllXVVWJZfN7TbABZqIRbIydh5+fH96+fdvggxzdKAqbvcCGDRsGd3d37N+/nyqkLyoqgoeHB4YPHy6WTVF6sNF9iHFzc8OAAQNw8eJFjBgxAkBtK4C1a9fCw8NDpIgn2+tvXfz8/Ki6VJ68v4mJCezs7ODp6QkHBweRbf7ItcSDTnN5AAgODoa9vT2MjIyoXqALFy5Eq1atEBgYSMsZY6OGpilNUTiSnDPCELfFEMD+vYPNTX1RNgAluUkkLk3O2E+KkZER9T0T3dcbIj8/H126dOGThE5ISEDPnj1pR7HYIjMzEzt37sS2bdv4IkqbNm1CSUkJtm/fLnakqe6DO1sP8Q0166zPfzHNkoeBgQG8vLywd+9e6lhmZiZ27NiBMWPGiGxPV1cXAQEBGD16tNDoV1VVFYKCgmiphz5+/BgKCgp8EtPXrl3Db7/9RrvAmMfQoUMRFhbGF1WtrKyEn58fpUwmKpK60bAxdh5sN+WsHxmrqqrC69evkZ6eTokHicv69esxb9486OrqUmm0r169QocOHeDq6iqWzR/twcbbGKID0xFPNtffuly6dAkODg58LTS0tLSwY8cO2NjYiOWMsdlQvj5ZWVlCGwFraWnBycmJlu1Vq1bBwsICHTp0gJGREXx9fTFp0iTk5eWJ1c/pV9ggZAtJzpn60G0xxPa9g81Nfbb7O0qaJmfsF6CiogLBwcGYMGECevfuDXt7e0RHR0NDQwOenp5i118FBgZi//79CAkJodS2gNpmxImJibCxsaGdnsQ02dnZmD9/Pjp16oTy8nK+13R0dHDo0CHMnTsXp06dEutilkSBev1mnT8jmzZtgoWFBYYNG4aamhpMnz4dpaWl4HK5Yil8mpmZ4dy5c5g/fz42btzINx+fPHkCT09PFBUVif1g7eDggPDwcBw+fJivKeqpU6dw+/ZtLFy4UKTasobYtGkT5s+fjwcPHqCyshIODg54+fIlPn/+jGPHjollU1LRCDbGzqPu5hIbNJQy7Ovri/z8fFq2u3fvjkuXLuHcuXN48eIFCCGYPXs2Jk+ezEiqnL6+Ps6cOSMgX11QUIApU6bg/v37tOwzGfFke/2tS0FBgVCnrnv37iguLhbLpiQbxHbp0gVZWVkCisSPHj2iXaMzZMgQXL16FRUVFejYsSOOHz9O1dCIc89ukhZoGEnNGaZbDAGSu3ewsanPdn9HSdMk4PEL4OrqinPnziE4OBiFhYVYtmwZVq9ejZiYGPTp00esSXvt2jVYW1vDysoKixYt4nuo+Pz5M0JDQ+Hn5wdfX1+xIh1ssWnTJnz48AF+fn5CF5KvX7/CzMwMffv2FUs6n+0C9V9NterevXt84iOjRo2ClJSUWLZevHiBDRs24Pnz52jRogXat2+P4uJilJeXo3///vDy8hJLRe3UqVNwdXXF9u3bMXnyZL7IW01NDc6dOwcHBwc4OjoyUg/x7t07nDhxAqmpqaipqcGAAQNgbGxMS02qoWiEtbU1o9EIJsceFRUl9Li0tDTat28PJSUldOrUieaIG+fNmzeYNm0aLZGNmTNnwsXFhVFlw+joaNy6dQsAEBkZiYkTJwqop+bm5iI9PR3x8fG0PmvJkiXo06cP7OzsKFEcOTk5bNiwAWVlZSI1lGV7/a3L9OnTMW/ePMyaNYtPzMfX1xc3btwQW1X169evOHr0KGbOnMk3/3x8fCArK4tFixYxomQbGBiI8PBw2NnZYcOGDfDy8sLbt2/h5eWFRYsW0RLxqFvbXJdPnz5h8+bNfC0H6NhrohZJzJn6Ah4AvRZDPNi+dzS0qb9o0SJGNvWjo6MxevRovrl58uRJtG3bFhMnTvxporpNkbFfgCtXrmDPnj1QUlLC9u3boampiWXLlmHkyJGwtLQUy2ZISAhWrVpF5eLXpW3btli5ciW+fPmCoKCg/5Qzdv/+fezZs6fBHZ2WLVti5cqV2Lp1q1j22Q7r/ywLx/dYuHAhfHx8MHz4cL66mcLCQpibmzf4IN4YAwYMQFRUFOLj4/Hw4UN8+PABHTt2xODBgzFy5EixnbywsDDY2NgIdbSkpKRgZGREOSFMOGO//fYb1q5dS9sOD0lFI3iiEvXHXlpaih07dlA9ZH6U79U5SElJYd68eWJfqz/Co0ePxBJ9qUtOTk6DkSVx4Sk08vZK3759y1frw+Fw0KpVK0aaJzMZ8WR7/a3LqlWrYG1tjYyMDFRXVyMyMhJZWVm4evUqX3q0KJSWlsLU1BRpaWkYMmQI34P158+fcfLkSfz9998ICgqi/T+3tLTE58+fsW7dOpSXl2Pp0qWQlpbG3Llzhd53v0diYiJycnIA1G50KCkpCThPmZmZuHfvnsi2f7UoBJNIas6wUY7C9r3j2rVr8PLygpWVlYDarI+PD0JDQ7Fr1y706tVL5OfIyspKrFq1CrGxsQgJCeFLV05MTMSFCxdw9epV7Nmzhy8i91+lKTL2C6CiooJr165BTk4OBgYGmDNnDiwtLfHmzRtMnjxZZKlvoLY+JDw8HH379m3wPWlpaVi0aBHtNBkmUVVVRXR0dKMLx5s3b2BoaIh//vlHcgP7QX7myFhsbCySk5MB1KZ/LV68WODm8/r1a8TExPA12RWVxtIn4uPj+dIMfwQNDQ1ERUU1uvuXmZmJOXPm0JYpLyoqQmBgIF68eIGKigqB18Vx9tmMRmRmZqKoqAhArYPt7e2N9u3b870nPT0d7u7uePz4schjFwYhBJ8+fUJCQgKcnJxgbm4OU1NTWjaF7byWlpbi+fPnMDY2FtmRrEtgYCDi4uJgbm6OXr168dUcAvT7aZmYmMDHx0fgvDMJUxFPSa+/cXFxCAgIoKLvAwYMgKWlJdW3UlS8vLxw+fJlHDp0SGi0ITMzExYWFpg5cyZWrFhBd/gAaq/PjIwMEELQt29ftGnTBu/fv0fXrl1FspOUlARjY2MA/9eTsj6tWrWCmZkZowrI/7/D5pxhW42Q7Ui2sbExRo8e3ejmgru7O5KTk0WuwQ8KCkJwcDD27dsntPby/v37WL16NVauXAkTExORxy5p/vvuYhPfpVevXkhOTkZhYSHevHmDUaNGAahtsEcn9el7fnrLli1RXV0ttn02kJOTw6tXrxp9GHj16pXYBa91MTExaTCHW0ZGBnJycpg6dSqGDh36wzbT0tKo75l2OtimR48ecHJyouZNdHQ0X7SKt5svTs1YXZYvXw5/f3++SMGXL1/g5uaGU6dOiZweKisri2/fvn33fXQjKABgY2OD5ORkjBgxQuChXVzYjEbk5ORg2bJl1Dxv6IY/Y8YMkW03BIfDQceOHTF27FiUlZUhKCiItjP2+++/C03xWbBgAaZMmULLtpeXF6qrq/Hw4UO+z2Cqn1ZjDyn5+fmQk5OjZZ/JiKck19+goCAYGhri+PHjtG3xuHz5MmxsbBpM++rXrx/Wrl2LgwcP0nbGFBUVcefOHXTq1AmDBg2ijou7iaqhoUHdP37mTb2fDTbnDNsthtiOZL948eK7LRGmTJmCM2fOiGw7KioKtra2Qh0xoFYIZ82aNQgPD29yxpqQDBYWFli3bh2kpKQwbNgwcLlc+Pr6wtfXV2w1r4EDB+L+/fuN1uDcuXPnP9f/Sl9fH35+ftDS0hIamq6qqkJAQAAl4UwHRUVFHD16FIqKitSC8PjxYzx+/BgGBgbIy8vD4sWLsW/fPujr64tsf/ny5QI7VnScDrbp378/Jd2rp6eH06dPs1Lzk5ubixUrVsDX1xcyMjK4e/cutmzZgrKysh/qhVMfJSUlxMTEYODAgQ2+58aNG41GiX+UxMREBAQENFo4LSofP378bhF07969qQiXKIwZMwY3b95ETU0NDAwMcOrUKb7/Kc/Bri8uwRSqqqqMtJBwc3NjYDTCOXz4MGu2gVqHeNeuXUhPT6c2vwghqKioQFFREZ49eyayzboRT19fX3C5XKERz4iICJGcMUmuv35+fjAwMKBtpy75+fnfrf3T0NAQqYl4XU6fPo3z588DqP0frlixQkBq/t27d2jXrp1Y9nnU3dRrgl3YnDNstxhi897Bg61N/Tdv3kBdXb3R94wYMQIeHh4i2/43aHLGfgGmTZsGRUVF5OTkYPTo0QCAQYMGISgoSOw+NzNnzoS7uzs0NDSELjSpqanYt28frSJjNrCwsMC0adOwYMECLFmyBOrq6mjfvj0+ffqEpKQkBAYGIicnh5GHs/z8fMyfP1+gueCuXbtQUFAAHx8fhISEwN/fXyxnLDc3FytXrmTM6ZAk9W8ilZWVSEtLQ9++fdG6dWtato8ePYrFixdj5cqV6Nq1K06fPo2JEyfC3t5erJ1gY2NjbNiwAQMGDICurq7A6zdv3sSBAwfEksquT7du3Wj//fVhOxrBS7O7ceOG0AgTm3A4HNr5/jU1NXj06BHS0tJQWlqKtm3bQklJCaqqqoyMkUnHWhhOTk549eoVxo8fj8OHD8PMzAxZWVm4fv262BLobEU8Jbn+qqqq4ubNm4w2Yu7QoQMKCwsbvZY+fvwottqtgYEBEhMTqZ/l5OQEIuQDBw4Uuzb15cuXOHXqFCwtLdGpUyeUlpZi27ZtiImJQZcuXbBixQqxmgM30TBszxmAvRZDbN872NzUb926NT5//tzoe759+yZWX9N/gyZn7BfAw8MDM2bM4Nsl5Dll4mJkZISYmBjMnDkTY8aMgYaGBtq1a0fdVOPi4jBy5Mj/XPi3U6dOCA0NxcaNG2FlZSWQNqSuro4jR44w0qPi1q1bQtMI5syZQ0l16+vrY9++fWLZZ9rpkCR5eXmwt7fH2rVrMXDgQMyYMQOZmZlo3749QkJCoKioKLbtrl274siRI1i8eDFu3boFHx8fWjvk+vr6mDt3LpYvXw5FRUWBuZ6eno45c+YwIt6xceNGODo6wtraGj179hQQHRGnvkhS0YjffvsNAQEBjLfQaIxr1641GrH8Hrdu3YKjoyNyc3P5dmg5HA569uwJR0dHsTasRGlzQFf8ICkpCQcOHICWlhZu3boFAwMDqKioYO/evYiNjcXs2bNFtslWxFOS62+bNm3g7u4Of39/yMvLC6jViVN/qaWlhYiICKioqDT4nvDwcL60QlHo0KED33xgUqEwNTUVxsbG6NChAxYsWAAA2LZtG65cuQJTU1O0bdsWTk5OaNu2LfT09Bj5zCbYnzNsthhi+97B5qa+mpoaLl261OizxIULFwSEQ/6rNDljvwAJCQkIDg7GoEGDMGPGDEyaNImRBX7fvn1Uf5K//voLQO1NWllZGdu3b8fMmTP/k+p/ffr0wenTp/H06VMkJyejpKQEHTt2hIaGhljS5w3Rpk0bvHz5En369OE7npGRQe3GlJWViV0bxLTTIUl27tyJz58/o1OnTrh8+TLy8vJw4sQJnD17Fh4eHggODhbJnjD1xWnTpmHv3r04c+YMSktL+Y6LyqZNmzBs2DCEhYXh6tWrKC4uRqdOnaCuro5NmzYxklYF1D6QZmZmwszMTOC4uPVFkopGeHh44Ny5cxg1ahTi4uIQGRlJtdBwd3cX2eloSMSlpqYGnz9/RkJCAo4fP47du3eLNd4HDx5g2bJlGD16NFxcXDBw4EC0a9cOnz9/RkpKCo4fPw5LS0ucPHkSysrKItmumzpJCEFCQgK6dOmCP/74A9LS0khLS0NBQYFYEfH6VFRUUDvHffr0wfPnz6GiooJp06bR2gxjK+IpqfW3VatWjGyQ1MXMzAyzZ89G27ZtsWzZMr7UzU+fPsHf3x/nzp1DSEgI7c/auXMnvn37RtUGZ2ZmIiYmBurq6mI1UT9w4ABGjRpFqccVFBTg8uXLmDZtGlWn265dOwQHBzc5YwzC5pxhU40QYP/eweamvqmpKUxNTSEnJwdjY2O+jU1CCI4ePYqQkBB4e3uLNXZJ06Sm+IuQlZWFqKgoXLhwAYWFhTAwMICRkRG0tbUZuclWVFSguLgYHTp04Mtx5z1E/iwUFRUxVse0d+9enDp1CtbW1lBVVUVNTQ0eP36M/fv3Y8qUKViyZAnWr1+PNm3aYP/+/T9kU5jT8fHjR+zduxfa2tp8KmFMP4gwiaamJkJDQ6GoqIi1a9eCEIJ9+/YhKysL06dPF7k4/Ud7ODEhmMAmenp6UFRUxOzZs4WmT4ib8paVlYWNGzciJSVFaDTC2dkZ/fv3F3vcQG20fefOndDW1sb27dvx+vVrhISEICUlBZaWliJLZvN69jV0C+rbty+WLFki9jw3MzNDt27dGnUS7e3tUVZWBi8vL7E+AwA8PT2Rl5eHnTt3UvWd1dXV2LZtGzgcDu2U4vHjx8POzg46OjrYv38/Pn36hG3btiEzMxMzZ84USy23LpWVlQgKCmI84nnixAlMnjz5p2tgf+3aNdjZ2aG8vBx9+vShHh5fvXqFVq1aYfv27TA0NKT9OQ8fPsSKFSuwb98+9OvXD+PHj4eUlBS+fv0KT09PTJgwQSR7w4cPx8GDB6kITGRkJDZv3gx/f3/o6OgAqK0lMzY2RlJSEu3xN/F/sDVn2FQj5CGJewdvU//FixcA/m9Tf/bs2bQ29Y8ePYpdu3ahXbt2UFFRoc7748ePUVZWhjVr1ojd3knSNDljvyAPHjzAlStXEBkZifbt22P69OmYM2eO2LnF+vr6OHPmjEDKSkFBAaZMmfKfkravS0lJCTw8PLBgwQL0798fFhYWiI+Ph7y8PA4ePEirUSJQu4Pv4eGBsLAwlJeXgxCCFi1awMTEBGvWrEFMTAwOHTqEPXv2/HD62a/idKirq+PSpUvo3r07hg0bhnXr1mHOnDnIzMzE3LlzaUnbs83bt2/Rrl07tGnTBvHx8bh27Ro0NDQYeQBTU1PDhQsXaM+9hnj69ClSUlJQXFzMeDSC6RYaDRW085o+01Wb1NLSQnBwMJSUlBp8z5MnT7Bs2TLcvXtX7M8ZOnQoTp48KXCeX758iZkzZ9J+8A0ICEBwcDDc3d3RsWNHLFy4ECtXrsSdO3fw9etXnDx5kpZ9V1dXnDt3DsHBwSgsLMSyZcuoiGefPn3ETrPU0dHBp0+foKenhxkzZjC2MViX3NxcRERE4Pnz52jWrBmUlJQwe/Zs2mqNHz58wPnz5/H06VN8+vSJipJPmDCBsXTcefPmQV5eHvb29oiIiEBISAiuX7+OM2fOICIiQuRejMrKyrh+/Tq6d+8OoHaj4dy5c3jw4AHVYiQnJweTJ0/+T7Z1+dlhY85IssUQm/cOHg1t6tMhLS0Np06dEjjv06dPZ3z8bNKUpviL8eTJE1y7do0SUBg6dCgePnyIoKAgODs7/7CMc3R0NG7dugWg9obn5OQkkJOfm5v7n46K7dy5EwkJCTA1NcX169eRkJAAd3d3REdHw93dnXb4WkpKCps2bcKaNWuQmZmJZs2aQV5ennqINDAwEDmt8FdRwfrjjz9w+vRpdO3aFSUlJdDR0UFFRQUCAwN/2OFsjKioKDRv3pzaPV6zZg0MDAwwefJkWnavX78Oa2trBAQEoGfPnrCwsEDPnj1x9uxZFBcXY/78+bTsa2lp4dGjR6w5Y0pKSo06H3RguoVG3bohNto4fP78+bsP5d26dUNxcbHItusiIyODt2/fCtz4MzMzGWkGvWTJEjRv3hyEEKioqMDKygp+fn7o3r073N3dadu/cuUK9uzZAyUlJWzfvh2amppYtmwZRo4cSWtXOSYmBnfu3EFUVBRWrlyJ9u3bY+rUqTAyMhJI7RaHpKQkmJmZoWPHjlBWVkZ1dTXCw8MRGhqKo0eP0qo17NKlCwYNGgQTExOBh8by8nLExMSI3cuMx7Nnz+Du7o42bdrg9u3bGDNmDJo3bw4dHR2x0sK6deuGN2/eUM7Y3bt3oaqqyjcH//nnH9qtEJoQTpcuXQTSz5lAUi2G6t47SktLGXOW6iIrKwsOh4MPHz4I/F3i9mPkcrmMNJH/t2lyxn4B8vLycO7cOZw7dw5ZWVlQVVWFlZUVJk6cSNWOeXt7w9XV9YedMXV1dZw8eZK6YN6+fct3cfIKvHft2sX8H8QQsbGx8PX1Rb9+/RAYGAhtbW1MnjwZCgoKtB+qeXz9+hXp6emorKwEIYRqegxApP5iDcGW08E2mzZtwrJly/Dx40dYWlpCTk4ODg4OuHHjBg4dOkTL9pEjR+Dp6cm3AMvJyWHr1q34+vWrWIIGPA4cOABzc3MMHz4cfn5++P3333Hp0iVcuXIF3t7etOfNkCFDsH37dsTExKBXr14CRdN0m7F++fIFISEhSEpKouZkXcQRNeDBRgsNHmy0caipqfmuEmOzZs1QU1Mjsu26GBoaUmI1ysrKqKmpQVJSEry9vakmvHTgcDh8fdaWLFmCJUuW0LbL49OnT5QjeefOHcyZMwdArdjEj/TfawgOh4ORI0di5MiRKCsrw7Vr13D16lVK/XfWrFmYNGmS2BHQXbt2YcKECXB2dqb+z5WVlbCzs4Orqyvtuq6FCxdSfcDqkpGRgY0bN9J2xlq2bImKigqUl5cjMTERs2bNAlAbYREntXPs2LHYvXs3HBwcEBsbi7y8PCxdupR6vaCgAN7e3ozUMTbxf5iammLZsmWs9P2UdIuh0NBQHD58GAUFBeBwOPjf//4HKysrRkoikpKSYGdnh+zsbL7jTPVjzMvLw/Hjx5Geng5paWkMGDAAs2fPZkQsSBI0OWO/AHp6eujUqROmTJkCHx8foRfuH3/8AXl5+R+22b17d+rBzcTEBD4+PgJ9aP7rfPnyhdolvHPnDrXL26JFC0Z2km7cuAFbW1uUlpYKPPQysbiw6XSwjYqKCm7fvo3S0lKqZ86iRYuwdu1a2j2pjh49Cjc3N0ycOJE6ZmdnB2VlZfj6+tI6L5mZmfDx8YGUlBTu3LkDHR0dSElJQU1NTezeQnUJCwtDx44d8c8//wikConbuLMu27Ztw40bN6CtrY2uXbvSslWfadOmgcvl4s2bN4y10ODBRhsHDocjkcj9hg0b8O3bN2zfvh1VVVUghKB58+ZYsGAB7cbAPGJiYpCeno6KigqB1+jOGaYjnsL4+vUriouLUVJSgqqqKkhJScHPzw979uyBp6enWPMnLS0NO3fu5HO4ZWRksGzZMsqxEZWQkBBqg5EQAm1tbaHva0w570fR0tKCh4cH2rdvDykpKYwaNQqpqalwcXGBlpaWyPZWrFiBZcuWUQ/Oenp6lGPt5+eHAwcOoHfv3li+fDntsTfxf8THxyMhIQFWVlZYsmQJ7VYcdZFkiyFvb28EBwdj4cKFVA18QkICHBwcUFpaSil0iouLiwu6du0KGxsbxutInz9/jgULFqBFixZQUVFBTU0Nzp49i+PHjyMsLAwDBgxg9PPYoKlm7Bfgxo0bGDNmDJo1a/ZvD+U/xYwZMzBr1ix0794dS5cuxZUrVyAvLw9PT0/cv38fp06domV/woQJGDBgAKysrIQuLnR3ZMaOHQtra2s+pwOolWv19fXFlStXaNn/WVFVVcWFCxcEdgNzcnIwadIkPHnyRGzbI0eOxKFDh9CjRw8MHz4c3t7e0NXVRXx8PGxsbBAXF0d3+KwyePBgeHp6Cu2Xxibl5eUCacyi8P79eyxevBg9evRgrI0Dl8tF9+7dBdoH1KWmpgb5+fmM1F+WlZUhKysLHA4Hffr0YSRFEajtM3bixAl07txZ4BxzOByq0bq4REVFYevWrZCSkoK6ujpCQkL4Ip7i7oqXl5fj2rVrOHfuHOLj49G5c2dMmzYN06dPR+/evQEAjo6OuHnzJmJjY0W2P3nyZCxfvlxgfYyNjcWuXbsQHR0tss2qqipcvHgRNTU12Lx5MzZv3sy3tvMyQoYNG0a7MXNRURG2b9+OnJwcrFy5EgYGBnBzc0NycjK8vLzE3kx58eIFpKSk+DZlr1+/jvz8fEyfPp3xPof/v8PlcrFlyxbs27cP3bp1w/r16xldf9esWUM94zWkRnjgwIFG17kfYdSoUdi0aZNAbfSpU6fg5+dHuwH1oEGDEBUVxUodl7m5OVq2bIndu3dTa2R5eTk2bNiAiooKBAQEMP6ZTNMUGfsF0NfXx9evX5GRkUEJSdRFnHQ5RUVF3L59G507d6ZUzxrivyoksXr1aqxatQqVlZUwNDSEvLw8du7ciePHj8PX15e2/Tdv3iAgIICxFIH6vHv3TqjktpqaGt6+fcvKZzIFm3NGXl4eN2/e5EvdAmofwuju5Ovo6GDbtm1o3bo12rZtC21tbdy9excODg5iSQc3xIcPH4RGOcTNm+dR/yGMST5+/Ah/f3+kp6dTkWVCCCorK5GRkYGEhASxbbPRxoFuxEgUeOsvLzX06dOn1Gt005UvXboEBwcHzJ07l+4whcJWxHP48OGoqqrCmDFj4Ovri1GjRgk8MA4fPlxsZ9LCwgIuLi549+4d1ScpOTkZe/fuhbGxMZ9I0I/+D6SlpSnnk8PhYNKkSQ3WMtKlU6dOAnXL69ato/15AwYMEFhbxo4dS30vbg1mEw0zYcIEGBgYwMPDA8uXLweXy8W8efMwfvx42hlFkmox9PnzZ6H1xoMHD0ZhYSFt+927d0dZWRltO8JISkrCyZMn+TarmjdvjhUrVtCO6EmKpsjYL0BsbCzWrl2Lb9++MZYuFxkZSd2Izp492+jFzmtw/F/k48ePKCgooEL8T548QevWrRl5YJ08eTK2bt0qthz59+AVu9d3Oo4dO4YTJ06ItfMrKerPmaqqKrx69QpRUVGwsbHB1KlTxbZ9/vx52NraYtKkSVBVVQUAJCcn49KlS3B2dqY1H799+wYvLy/k5OTA0tISampq8Pb2Rk5ODrZv3057Vzk2NhZ2dnb4+PEj33Gm8ubt7e3RtWtXrF27lpYdYaxduxb37t2DtrY2rly5gkmTJiEzMxPPnj3DunXrRK5jkmQbh4KCArHVZL8H2+nK2traOHbsGCOiF5LkyJEjmDx5slAlOd58r6qqEjutSxLKs7m5uXj8+LHQjRO6c/J7irJ0nHhzc3PGazCbEA6Xy8WdO3eoCP7z588REhKCK1euoKqqCoqKiuByuejQoQPWrVtH67PYUCPksX79enTu3BmbN2/mO7579268evWKtuBZZGQkwsPD4ejoiL59+zI6/pEjR8Lf319g8zo5ORmLFy+mtVEoKZqcsV8AXtRnxYoVQlMnfpYCRqYxMzODvr4+9PT0qNoxJomNjYWnpyesra3Rt29fgR1NulEONp2Of4vLly/j1KlTIjd9rk90dDSOHDmC58+fQ0ZGBv369cOSJUsknp4nKuPGjUPfvn1hbGwsVLiArmO/e/duhIaGon///ujTp4/AnBRXphyorXHZtWsXxowZg0mTJmH37t2UklV5ebnIyn6SbOPA5XKhpKQEfX19GBgY0FLaqw/b6cq+vr7Izs6Gs7MzK1GahQsXNvq6uKIvbLdEEaWGU5z/QUREBBwdHYXWFzM1J+v32ePVOUpJSSElJUVs2+PHj0evXr2E1mDa2NhgxowZtMbexP9RN4uoLuXl5bh9+zYSEhKQmpqKDx8+4OLFi//SKIVjZ2dHfV9WVobr169DUVERgwcPRrNmzfD06VMkJSVh1qxZcHBwoPVZ48aNw9u3bxus16dzPW3YsAHv3r3D/v37qfWmqKgIa9euRfv27X+Kxs9NztgvwKBBg3Dx4kUqD58JfHx8fvi9kkwHEgU3NzfExsbi1atXUFRUhJ6eHgwMDBiRVgdqpWB5C0v9ZolM9QH7WZ2OhsjNzcXEiRPx+PHjf3soDfLPP//g6NGjSE9Pp3oXmZqaMlIErKamhrNnzzbaN4YOJiYmjb4ubmNQgL+P0apVq6Cvr49p06bh+fPnWLJkiVh1P5Li+fPniI2NRWxsLCXvra+vD319fQwdOpRWvcWgQYNw6dIl1tKVX758iXnz5uHLly/o2rWrQJYC3Zqxug9kQG0U+/Xr10hPT8eiRYtgbW39w7bqtkSJjIzExIkThbZESU9PR3x8PK1xs42enh50dXVhbW1NqRIzSX1nsrq6GllZWdi3bx82bNiAESNGiG2bjRrMJoRTPzLGtO0fTUEU53nje/cLHhwOh5YSL1C7HjQGnc3l/Px8zJ07F8XFxZRQ3atXr9ChQwccPXqUMSEiNmmqGfsFkJeXx/v37xl1xs6ePcv3c15eHmRkZNCzZ09IS0sjOzsblZWVUFZW/s86Y7a2trC1tUV2djZiYmIQGxuLgwcPokuXLtDX14e9vT0t+4cPH2ZopA0zceJEgQL1n5WysjIcO3aMdkNWoHbXKysri5IlJ4SgoqICycnJtNTCbt68iZUrV2LQoEHQ1tZGdXU1Hj16hOnTp+Pw4cMYMmQIrXEPGzYMT58+Zc0Zo+NsfY9u3bohNzcX3bt3h7y8PJ4/fw6gVqKbbq8ugN02DgoKClBQUMCSJUtQUlKCW7du4dq1azAzM0Pbtm1pOQby8vLIz89nzRnbuHEj2rVrhxkzZjAmClKXhqKlvr6+yM/PF8kW2y1RvhfFqwvdh0eeQ8OGIwYIj9b16tULbdq0gYODAy5cuCC2bTZqMJsQzs6dOxlXB+Th6urKqiIsm/eL+rCZySMnJ4dLly7h3LlzePHiBQghmD17NiZPnsza9cs0Tc7YT0pdAYe5c+di69atsLe3R+/evQVUFcVJl6urnBMSEoKYmBjs3r2b2v0pKSmBjY0No+k+bNGrVy9MmjQJnTt3RocOHXD58mUcO3aMtjPGVq1YXdhyOtimoR09DocDR0dHWrbPnz+PLVu2oKKigkrz4X1Wjx49aJ2XvXv3wtzcHOvXr+c7vmvXLnh4eCA8PJzW2B0cHDBz5kzcunULPXv2FDhHTGxslJWV4fz583z9Vur2HBSXcePGwc7ODm5ubhgxYgSsra2hqqqKv/76i/ZGkCTaOBBCkJKSgvj4eNy/fx+JiYmQkZGh3SR7w4YNcHZ2Zi1d+cWLFzh9+rTE19qpU6di2rRpcHZ2/uHfYbslSl0Hpry8HNHR0VBUVISamhqkpaWRkpKCJ0+eiC1tXxdFRUVkZGRIfFe9Y8eOeP36tci/J6wGc9q0adi7dy/OnDmD0tJSvuNNMMPw4cNZE3mZPn06K3Z5iCIEJs46ZmdnB3t7e7Rp00YgAl8XDodDu1dl69atGenr+G/RlKb4k1L/YZf3b2QjXW7EiBEIDg4WSO9LT0+HiYkJ7dx/trh27Rru37+P+/fvIzMzE7/99huGDx+OYcOGYcSIEfjtt99Etrls2TJ4enryPdjevn0bQ4cOpdJxioqKYGBggKSkJFrj/57TwVNW+i8iTPRFRkYGqqqq6NmzJy3bEydOhKqqKiwsLDBv3jwEBwfj3bt3cHR0xLp162iJg6ioqOD8+fMCPflevXqFqVOn0k6v3LZtGyIiItCxY0e0bNmS7zUmZMrfvn2LBQsWoLCwEH369EFNTQ1ev36Nzp0748SJE5CTkxPbdkVFBTw8PKCiooLJkydTf0vbtm2xf/9+Wsp7bLdxWLZsGRITE/Ht2zcoKSlBS0sLI0aMgLq6Ou0HKbbTlY2MjGBvb087KisqFy5cgIuLy392fbezs0P79u1ha2vLd9zLywuZmZm060SuXbsGNzc3LF68WKiTTVclU5iAR2lpKUJDQ/H582ecOXNGJHuSrMFs4v9QVFTEtGnT4OTkxLioRn1u3LjBp2YLgNqcFSdTRxJpkL6+vmjXrh3jKfT6+vo4ffo0OnbsCD09vUb/Drr3VUnQFBn7SaGbgiEKlZWV+PLli8DxwsJCiTRVFZfVq1dDSkoK+vr6cHNzEyoTLyqxsbEoLy/nc8ZWr16Nc+fOUU5GTU2N0PMlKv7+/pg0aZJQp2PVqlW07bMJmzt6OTk58Pb2Rr9+/aCgoICioiLo6emhqqoK/v7+tJwxRUVF3Lt3T8AZS0lJYaRm7OLFi9i5cydrKRtubm6Qk5NDREQElQ764cMHrF27Fh4eHti9e7fYtmVlZfmiyU5OTli3bh3atGlDu9Ep220cHj16hNLSUujo6EBXVxfDhw9nLK2Q7XRlS0tLbN68Gebm5ujVq5fAuabrFAhL/SstLcXz589F3mmWZEuUK1euCK1DmTZtGiORn9WrVwMAduzYIfAaEw6NiYmJgIAHULvR5uHhIbK9tLQ0WuNpQjwIIYiJiaGiyBoaGqx8jqenJw4dOoQuXbqgsLAQ3bp1w4cPH1BdXY1JkyaJZbPuc2RaWhp8fX1hZWUFdXV1yMjIIDk5GT4+PmI3la7rYDGdEmlkZESJYLEdQZQETc7YTwovRa60tBQyMjK0Gq5+Dz09PWzduhXbtm2DsrIyCCFITEyEs7Oz2IuAJPDx8cG9e/dw584dzJ07FyoqKhg+fDiGDx9OpbWIirBAsrBjTDipbDodbCAp0RdZWVlql7p379548eIFRo8eDWVlZbHSe+oyZcoUeHp64uXLl3y9i0JDQzF37ly+VCBxHvhatmzJ2s0aAO7evYvg4GC+urwuXbrAxsYGlpaWtO2/ePECL168YFzqm83ecUBtb6WnT5/i7t27iI6Oxo4dO9ClSxcMGzYMw4cPp1WX1lC68tOnT3Hy5Ena6cw8Oezt27cLvMaEU/D7778LjWIvWLAAU6ZMEcmWq6srVT9DR7nzR2jXrh2ePXsmsHGSkJDAiJgC27vpwuzLyMiIlbEhDDZrMJv4PzgcDo4dO4Z9+/ZhwYIFMDAwgJWVFWNCYTwuXLiAzZs3Y+HChdDR0cGJEyfQqlUrrFixQuxsk7prk6urK1xcXPh60ikqKqJr165wd3dnpM9hVVUVCgsL+fpU8iJ7oq41dZ8hWrZsCUNDQ9bal0iCJmfsJ4VXsxUXFwcOh4MxY8bA2dkZnTp1Yvyztm7dijVr1mDRokXUTZsQgvHjx2PTpk2Mfx5TGBgYUEXLeXl5uHv3Lu7du4dDhw5BSkoKjx49+pdH2DhsOh1sUF/0pSE4HA4tZ0xZWRmnTp3CunXrMHDgQMTGxsLc3BwZGRm000R49TFHjx4V2Mk7dOgQ9T2HwxHL+TA2Noa3tzecnZ0F0hSZoFmzZkLtNm/eXKgDJQr+/v7w8vIS+pq454OHubk5bG1t8fTpU6FtHOjCa5KqrKyMJUuWoLCwEP7+/ggPD0dkZCRjD6jfvn3DhQsXEB4ejqdPn6J58+a0x8+2U+Dm5saYrboR36dPn2LhwoWsCZvMmTMH27ZtQ2ZmJpSVlVFTU4OkpCQcP34cGzdupG2fV59WUVGBN2/eoFevXiCEMJaKxmbLGUnUYDZRCyEEHTp0wP79+3H79m3s3bsXRkZGUFFRwZQpUzBs2DBG+poWFhZCT08PQK0g0ZMnTzB+/HhYW1vD3t4ea9asoWU/KysL/fv3Fzjeq1cv5OXl0bIN1JZzbNq0CUVFRQKvtWjRQmRnrC5+fn4/vUBNkzP2k+Lu7o4nT55gzZo1kJKSwtGjR+Hg4ID9+/cz/llt2rRBUFAQXr58ifT0dEhJSUFRUZF27Y+kKCgowL1793D37l3cvXsXUlJS0NbW/reH9V3YdDrYoK7oC5usWrUKFhYW6NChA4yMjODr64tJkyYhLy+P2gUWF7ZTfRISEvDw4UNcuXIFnTt3FojO0n3w1tDQwIEDB+Du7k7NkcrKSvj7+9OOyB05cgRWVlZYunQp45H4KVOmQFpaGkeOHMFff/1FtXHw9vZmpI0DIQTJycm4c+cO7t69i3/++Qft27fHlClTGLGfnp6O8PBwnD9/HqWlpejYsSNWrFjBSEE5G07B9xoO10XcNMjIyEiBSCeTWFlZoVmzZjh27Bh8fX0B1AqI2NjYMHLeCSHYvXs3jh49isrKSly9ehV79+5Fy5Yt4eDgINb5l5Qa5NGjR+Hm5sZXg2lnZwdlZWX4+vo2OWMsMXLkSIwcORIPHz5EVFQUfHx84OLiAhkZGbRr1w63b98W23a7du2o8odevXohIyMDQG1ku6CggPbYFRQUcOTIEWzbto3adK+qqkJAQAAGDRpE2/6ePXvwxx9/wMTEBGvWrIGnpyfevn2L/fv3046iq6qq4ubNm1i8eDHtcf5bNDljPylxcXHYtWsXRo0aBaBWUnjx4sWoqqqiXb/REH379mVNkpsNduzYgbt37+Lly5f47bffoKurCzc3N1bVj5iETaeDTfLz89GlSxe+eZiQkICePXsykkYwZMgQXL16FRUVFejYsSOOHz+OkydPonv37iI97PwbDB48GIMHD2bN/oYNGzB37lyMHTuWqsFKTk6m2grQobKyElOnTmUtJZrNNg5aWlr4/PkzBgwYAF1dXWzcuBEqKiq0bFZUVODy5cs4efIk/vnnH0hJSWHYsGG4d+8eQkJCGFM/ZMMpaKheqT500iB1dHRw7NgxrFy5khV56YsXL2Lu3LlYunQpPn78CA6HI9Bgmg5Hjx7FuXPnsH37djg5OQGozbZwdHREly5dROq/xoPNaFhd2K7BbOL/EFaSMHToUAwdOhSEEGRkZODp06f48OEDrc/R0tKCp6cnnJ2doaqqioCAABgbG+Pq1auMZETZ2NjA3Nwct27dwh9//IGamhqkpKTg69evCA0NpW0/IyMDrq6u4HK5UFRURKtWrWBiYoJWrVohKCiIVmSrTZs2cHd3h7+/P+Tl5QXuUZLUWBCXJmfsJ6WoqIjvZq+uro7q6mqqsJMubKvsSILExERMmDABenp6+OOPPxiz++jRIz65ZkIInjx5QvXkYaLnEvBzOh2BgYHYv38/QkJC+JwOb29vJCYmwsbGhvbYeXK5vHnev39/bNmyBZ8+fcKaNWtw4MABsW2zrcrEdk++fv36ISoqCidOnKD6rUyePBnz5s2j/SA4depUREREMJICJgw22zisXr0aurq6jD0Mu7m5ITIyEiUlJdDQ0MCWLVswfvx4dO7cGUpKSrSaSNeHDadAEupi79+/R3R0NEJDQ9G5c2eBByS6Y3BycsKJEyfQvn17dOzYkZYtYYSHh2Pbtm0YO3YslWo6ceJEyMjIYOfOnWKdd7br6HiwXYPZxP/R2IYGh8PBgAEDGBF/srGxwfLly3H58mUYGxvj8OHDGDFiBDgcjoCiqDgMGTIEFy9eREREBF68eAGgNu143rx5jNQxNmvWjKon7d27N9LT0yl1a3H7DvJo1arVT9+uockZ+0mpHwFr1qwZI3UhPOo2G8zNzUVgYCDmzJnDp7Jz/Pjx/3SvK14N09u3b3Hr1i0MHToUZWVltIu7V61aJbAA1+9LxYSAB5tOBxtcu3YNXl5esLKygoKCAt9rPj4+CA0Nxa5du9CrVy+MGTNGJNuJiYnIyckBUFuYrqSkJLDbnpmZiXv37tH6G4yMjPj+d1VVVXj16hVu3bpFqavRJS0tDenp6UKdDhcXF9r2e/TowYrDZGFhgalTpyI6Ohr/+9//BOY4nd1HNnvHAcCCBQuQl5cHT09Pqv9a//79MWfOHLEctJCQEPTt2xeurq7fdeDpwoZT0NjfXFhYyIgAhpaWFrS0tGjbaQh5eXmkp6cLrXNhgjdv3kBRUVHgOJfLxfv372nZjo6OxujRo/nWsJMnT6Jt27aYOHEi7fnEdg1mE/+HpFQsu3fvjqioKJSXl0NWVhbHjx/H7du30a1bN9pRfh49e/YUeJZhigEDBuDmzZswMTFB3759kZiYiEWLFoncWF4YktrkYJMmZ6wJodSVCl2wYAG2bt2KmTNnUscMDAzQr18/hIaGwtzc/N8Y4neprKyEjY0NLl++DCkpKVy9ehW7du1CWVkZvL29xUqdYXtHWVJOBxuEhIRg1apVWLZsmcBrbdu2xcqVK/HlyxcEBQWJ7IzV3f3jcDhCnZZWrVrRnosNtQw4efIk7t69i0WLFtGyf/jwYWoXsG6aGIfDEbuP1MKFC+Hj40P1cmnsQY6Ow8Q7/yoqKmjVqpXYdoTBdhuH58+fY8GCBWjRogVUVFRQU1ODyMhInDhxAmFhYSLvXDs5OeHs2bNYsWIF2rdvj3HjxmHixImsOB9sOQUPHjyAo6MjvLy8+P7+LVu24NWrV9i5cyfU1NTEts92FJjL5WLDhg04dOiQ0NQkug9oPXr0QHJyskAkKS4uTux66crKSqxatQqxsbEICQnhmy+JiYm4cOECrl69ij179tAqN2C7BrMJ4TRUi8nhcCAjIwM5OTmxM5dqampw4MABdOnSBXPnzkXLli0RGBgIPT09RpyxoqIiBAYGNqiWSzfVb8mSJVi9ejVkZGRgaGgIb29vLFmyBM+fP8ewYcNo2QbYzayQBE3O2E9Mfn4+ysvL+Y4VFBSgWbNmfMfE6ZxelydPngjttaKiokIVkf4XOXDgANLS0hAaGko5CCYmJrCzs4OnpyccHBxEtll3R7mioqLB2rP4+Hixdtwl5XSwwYsXL74b2ZkyZYrIzUyBWmEK3g4kl8vF7du3+eTb2WbUqFG0UykA4Pjx47C0tMTKlSuhq6uLyMhIfPr0CevXr4e+vr5YNnv06EGlxfXo0YO1KM2jR49w5MgRaqedSdhu4+Du7g4tLS3s3r2bemgvLy/Hhg0b4OnpiYCAAJHszZ49G7Nnz0ZmZibOnj2L8+fP4/Tp0+jcuTNqamqQk5PDWMSGDafg2bNnWLJkCdTU1KhePTwWLlwIf39/LF68GBEREbRSrNiMAmdlZVGp0HQjVcIwNzeHo6Mj3r9/D0II7t27h/DwcBw9elTstLAjR44gOTkZR48eFdh88fDwwMyZM7F69WqEhYV9t0nu92CzBrMJ4ZiamvLNdUAwS0ZTUxPe3t5o166dSLb379+PkydP8kU2J02aBD8/PwAQugkqCjY2NkhOTsaIESME1gQmMDAwwKlTp9CsWTN0794dhw4dwuHDh6Gvr08764TtzAqJQJr4KVFQUCBcLpfvq/4x3s90MTQ0JL6+vgLH3dzcyPTp02nbZ4uxY8eSO3fuEEIIUVNTI9nZ2YQQQu7evUtGjhxJ276ZmRkpLy/nO1ZWVka2bt3KyHlXUFAg79+/p21HUgwZMoRkZGQ0+p5Xr16RwYMHM/7ZhYWFjNusS1BQENHW1qZtR0lJibx+/ZoQQsjixYvJ9evXCSGE3Lp1i0yaNIm2/dzcXFJdXS1wvLKykjx+/JiW7T///JM8ffqUlo2G0NDQoK5Pe3t7cujQIUJI7d+joaFB276amhpJS0sTOJ6amsrIfKyuriY3b94kK1asIEpKSoTL5ZK5c+eSS5cu0bZ9+vRpoqWlRUJDQ4mqqioJDw8nHh4eREVFhZw4cUIsmytWrCDr1q1r8PWamhqydOlSsnr1anGHTYKDg4mCggJ1H6r7/YIFC8S2K0lOnjxJdHR0qLEPHz6cBAcHi23P0NCQnD9/vtH3HD9+nEyZMkXsz+BRWFhIEhISyIMHD8iDBw/I/fv3ya1bt8iBAwdo225COOfPnye6urrkxo0bpKSkhJSUlPw/9s48Lsb1//+vabMvWT7CocgSosXWIgkd1VhakCWRZCd1nDZZS5QsIVuJSlRItCjlEEJUKOREx76nQknTcv/+6Nd8G03R3HNPTa7n43Eej9M9t/d9NdU11+u63u/Xm7p8+TI1fvx4KiAggEpLS6OmTZtGrV27tt6xdXV1qatXr9a4fvnyZWrs2LG0x66qqkqlpKTQjtMQGBoaUk5OTtSTJ0+o4cOHU5mZmdTFixep0aNHU5GRkQ09vF+CnIyJKaJ0h1m5ciVWrlyJ69evY/DgwaioqMCdO3eQlZUFPz8/kY2jvrx//55vj5uuXbsKxWTj9evXWL58OXx9fSEtLY3r16/D1dUVRUVFQqn94ZeLnpeXx0gvOWHQr18/pKSk1NlTJTk5mXbfoa9fv8LLywsWFhbo06cPrK2tkZKSAgUFBRw6dIhWywV+9T9FRUX4/PmzUNLlWrZsyW14WWVPXJXy+/r1a9rxx40bh+Tk5Bq/I69evcKcOXNw7949gWOvXr0a69atw6pVq9CzZ88aaVR0TuCZbuPQqlUrlJaW1rjO75ogSEhIQE9PD3p6esjLy8O5c+dw+vRp/PXXX7RPJ8zMzFBWVob9+/fj+/fvWLduHTp06IBVq1Zh5syZAsW8e/dunaeBLBYLCxYs4DacFgQmToF/pKioCOfOnePWAfbt2xdGRkZCcW988+YNpk2bBnNzc+Tl5YGiKHTs2BFlZWXIyMgQKDXs1atXUFNTq/MeLS0tbNu2TdBhA2giJwViyO7du7Fx40auyzVQ6SoqLS2NDRs2wMrKCs7OzlixYgXXjOdXKSgo4Jtto6CgIJST4S5duqBVq1a049QGk2mQTGdWiAIixsSU6p3Tf5YuRxd9fX2EhITg2LFj3D4ZAwYMwKZNm4TeZV6YKCoq4saNG5g2bRrP9ZiYGKGkEAUHB8PKygrLly9H586dcerUKRgZGWHNmjVCKYBnUnQwwdSpU+Hl5QV1dXW+vxdZWVnw8fHB0qVLaT3Hw8MDqampmDdvHhISEpCWlgYvLy/ExsbCy8sLe/bsETj2jwYeACAtLQ1VVVWh1AOpq6vj0KFDWLduHQYOHIhTp05h4cKFSEtLE/iDMCQkBAEBAQAqU2PMzMxquPl9+fKFdrqyra0tysvLsWDBAp73qGqhR8dVlek2DhoaGvDy8sLu3bu59ud5eXnYtm0bNDU1acX+cf7t0KED5s2bh3nz5uHIkSO0Yldhbm5eQxTQoaio6KdpUnJycvjy5YvAz3j37h2mTZuGZs2aQUlJCZmZmRg/fjycnJywdetW2j3I3rx5AwsLC3z69Am9evVCRUUFwsPDceDAARw/fhxycnK04lff2Ki+uUFnY6NVq1b4+vVrnfd8//6ddkN4pmswCfz5+PEj33m2S5cuXKOKLl26/PR3gB9KSkqIiIioYbBx9uxZoaxn/v77b2zcuBF2dnbo0aNHjc8Qup8fTKZBysjIcOdgeXl5PH78GKNHj4aysjKeP38u1GcxBRFjTYAlS5Zg//79PAuCb9++YevWrTh58qRQrOfV1dVpN40VNStWrICdnR2ePHmC8vJynDlzBk+fPuX26aFL586dERQUBCsrK1y9ehV79+4Vahd4JkUHE5iYmODy5cuYOnUqxowZA3V1dbRt2xYFBQVIT0/HlStXMGrUKNq1EElJSfD19YWioiL8/Pygra2NSZMmoX///pg9ezat2EwvVOzt7TF//nyEhIRg5syZOHDgAEaMGIHi4mKB6wBNTU2Rn58PiqLg6+sLAwODGsKuVatW+PPPP2mNXVjCgh9Mt3Go6r+mp6cHBQUFAMCzZ8/Qvn17eHh40Ipd2/zr6emJ8PBwoTQiffnyJR48eIDv37/XeE0QS+c//vgD//77b50bOo8ePaLVJoXpU+CtW7dCTk4O4eHh3PrR3NxcrFq1Ctu2bcP27dvrHZPpjQ1VVVXExMTwNWSpIioqqoYbbX1pCicF4oiysjL8/f3h7u7Ord0vLy+Hv78/d4Py1q1bArUXWLZsGRYtWoTU1FSusU5mZibu3r3LbXpOB4qikJOTg/nz59e4TnezDag0qDl48CDPQYKwYDqzQhQQMdYEYDpdDqhcAB8+fBj//fcfwsLCEBERgZ49ezbqSV1PTw+7d+/GwYMHISkpicOHD6Nv377YuXMnJkyYIFDMyMjIGteMjY2xc+dOnD59GoWFhTzX6cCk6GAKHx8f7kI6MTERQGXKk7KyMtavX4+pU6fSNpj49u0bunbtCqAy7dHGxgYA0Lx5c+7ijw7379/H4cOHeSzQ586dKxTHqu7duyMxMRHfvn1Dq1atEB4ejujoaMjJycHAwECgmC1atOA617FYLFhbW9PeWedH9Q/RwsJCSEtLC60BNNNtHOTk5BATE4OzZ89y+69Nnz4dkyZNop3SxvT8GxERAVdXV64xQHVYLJZA84yBgQF2794NDQ0Nvt9/UVERdu/ejbFjxwoyZADMnAJX5/r16wgICOAx8unUqRMcHBy4c0J9YXpjo+rEVE5ODrNmzeIRehRFITg4GEePHqW90dYUTgrEEScnJ8ybNw8pKSlQVlZGRUUFHj58iC9fvsDf3x/p6elwdXWFq6trvWPr6OggJCQEwcHBuHbtGqSkpKCoqIhTp04JJUPJw8MDGhoamD59OiOfH0ymQdaVWSEuJjYsiqqjYx1BLPj48SOsrKzQvXt3RtLlkpOTsXjxYrDZbMTGxiImJganTp2Cn58fPDw8xL7ZXn341UlPGDtJqqqqiIuLg5ycHHR0dGBjYwNLS0u8ePECxsbGSE9PpxWfaTgcDj5//oz27dvz7E5Vr18QBDMzM0ybNg1du3bFokWLEBcXBwUFBXh7eyMlJQUnT54UOPatW7cwf/589OvXD8OGDUNFRQXS09ORnZ2NwMBAnkbWgqCvr49du3Zh0KBBtOLUBZN1NIGBgThy5Ajev38PFouFP/74A0uXLhVoDqjexqFKjPFr43Ds2DHcuXOH9tiZgun5d/z48dDR0YGdnV29Hdhqo7i4GKampigtLcXcuXOhpqaGdu3aIT8/H+np6QgKCoKUlBTCwsIEbqicnZ2N+fPnY968eZg5cyYmTZqEL1++cE+B6dSjAZV9zI4dO1bD7fHff//FjBkzaP/O7N27l5GNjeDgYHh6eqJt27YYMmQIN3vg3r17KCoqgq2trcBisoq5c+dCRUUF9vb2CAoK4m6mXrlyBQ4ODkIpXyDw58OHDwgNDcXDhw8hJSUFJSUlzJo1Cx06dEBOTg7evXsHbW3thh5mDVRVVREVFcVY+cPFixdx8OBBxtIg379/Dw6Hgx49euDJkyfczIo5c+bUWsbTmCBirImQl5cHKysrPH78GLt37xZqutyMGTNgYGCAefPmQU1NDefOnUOPHj3g5+eHc+fOISoqSmjPokv1D9C9e/fWeS/TfXDowqToYJpx48bh9OnT3PqcKt6/f4/JkycjJSVF4NhJSUlYsWIFSktLwWaz4e3tjS1btiAkJAS+vr7Q1dUVOPbMmTPRr18/bNy4kef6xo0b8eTJEwQHBwscGwBGjRqFwMDAOk1O6MCvjub58+fo2LEj7TqaPXv2ICAgAJaWllBRUUFFRQVSU1MRGhqK1atXw8LCol7x0tPTMWvWLAC8Pdeq07JlS8yfP1+gv9Wf9VyrDl1DJCbn38GDByM2Nlboi6T8/Hxs2LABiYmJPKdukpKSMDAwgKOjIzp37kzrGd+/f8e3b9/QoUMH5ObmIioqCl27dhX4FLg6S5YsQfPmzeHl5cXd7KnqLfnlyxccPnyY9jOKi4uRnZ2N0tLSGr+fw4cPFzjuo0ePcPLkSTx48AAFBQXo0KED1NTUYGpqKpS5ITU1FQsWLMDKlSthYmICAwMDdOrUiVuDya9VDaHxIcr1zKJFi8BmszF58mRacWojMTERjo6O+PbtG891YaVBijtEjIkp/NLl8vPzsXPnTmhra/Ok4dE9uVJTU8PZs2fRs2dPHjH28uVLTJw4kZZDm7AZO3YsTp8+DVlZ2TpTbFgsllAaOEdGRqJZs2ZckwFbW1uMHz8ekyZNoh2bSdHBBLGxsbh69SoA4MyZMzAyMqqRxvb69WtkZ2fT3pnNz8/H+/fvuSeVGRkZaNWqFe2FjIqKCiIiImrEycnJwdSpU2nvth86dAiRkZGYPXs2evbsWaOQmc4CD6h0Ps3NzcXu3btr1NF06dJFoDqaKnR0dODo6IiJEyfyXD958iT279+Pf/75R+DYTPSOc3Z2/uV769sgWJTz7/Tp07Fs2TLG/t4/ffqErKwsfP78GR06dICysjLatGnDyLOESU5ODmbMmIFWrVpBWVkZQGUNTVFREY4dO0Y7devixYtwcnJCYWFhDSEmjMVjUVERo+51tZ0UWFpaik0djbjx7ds3HD16FOnp6XwFfH03fUS5nvHz88O+ffugp6fH1y2XrtgbO3YsBgwYUGsaJJ1aso8fP2LXrl21vu/CWOsxDRFjYooo0+VGjx6NHTt2YNiwYTxi7NKlS1i/fj2uXLlCK764EhQUBG9vb6xdu5br2LhlyxaEhYXBxcUF06dPp/0MpkQHE7x9+xaOjo6gKAq3b9+Gqqoqz4c+i8VCy5YtMXPmTKEsLJ89e4bs7GxISEhg0KBB3DoyOowZMwbe3t41GrLevn0by5cvp3WiB9T9dyuMv9Vhw4YhICCgRn1bRkYGbGxsaI1fVVUVZ86cQa9evXiu//fffzAxMRH6pgzdNg52dnZYu3YtOnTowPf3kQ5Mz7+3b9/m/n9Vs+0VK1ZAXl6eawxQBV0B7+PjA1NTU6GcvPFrDVEbwlggvX79GsePH+fWAfbv3x8zZ87kawFeXwwNDdG3b18sXbqUrzil+ww1NTX8+eefMDExgYaGBq1YP1Jb2m9BQQFcXFxo12AS+LN69WpcvHgR2trafH9n6rvpI0qYFntMpkEuXrwY9+/fB5vN5vu+N/YsKIAYeIgt/HpQMcWkSZPg4eEBDw8PsFgsFBUV4cqVK3Bzc2vUxZGzZ8+GqakpDA0N0bJlS6HHDw4OxtatW3neA2dnZygrK8PX11coYkxWVhafP3/GhQsXhCo6mKBr167cnb85c+Zg7969aNeundCfU1hYCHt7e1y9epW7A8ZisWBkZIQtW7bQyg/X09ODm5sbduzYwRW8T548gbu7Oy0zgyqY3qGTlJTku+vYrFkzvr1d6sO4ceNw4sQJuLi48Fw/c+YMRo8eTSs2E20cLl68iJUrV6JDhw6wtLTk239NUJief6tSLKvvla5fv77GfcIQ8FFRUThw4ADU1dVhamrK17TiV+HXGkKYLF68GKNGjYKmpiYUFRXRvXt3/P3334w869WrVzh48CDtvoi1sX79ekRFRcHa2hpdunSBsbExTExMBF6sVq/BjIyMxKBBg/jWYN64cYP22An8uXTpEnbs2AE9PT3GnpGXl4eSkpIapz90a67oZDb8CiNHjsSdO3cYEWM3b96Ev79/jU1UcYKcjDURmEyXKy0thZOTE2JiYgD8X33HmDFj4OPjIzRHNWHj7OyMhIQElJeXQ19fH6ampkLdgVRRUUFUVFSND+uXL1+CzWYjIyODVnwmRYc44+zsjNTUVKxbtw5qampckw03Nzfo6+vDyclJ4NifP3+GlZUVsrKyuDtsX79+hZKSEgICAgQ2MxAVwq6jqZ7qV1RUhISEBAwYMABDhw6FpKQkHjx4gPT0dEybNg0bNmwQeNxVP9MDBw7g8ePH+Pvvv+Hh4YHY2FhISUkJ5C5nZmaG9+/fo1evXrh9+zbU1NRqPRmjWzMm7Pm3PtbvwjgFSk9PR1RUFOLi4vD9+3fo6+vDxMSEdg82YWNlZYW7d+/i+/fv6NKlC7S0tDBq1ChoaWnVqE+ly6RJk7B27VpGrLirk5ubi+joaERFReHhw4dcUWxmZlavOEzXYBJ+zvDhw3H69GlGBHxGRgZWrVqFt2/f8lynU3P1q21DWCwWAgMD6x2/OkymQWppaSEkJKRG1oY4QcRYE0AU6XIA8Pz5c2RlZaGiogL9+vVDt27dsGvXrho75Y0JDoeDixcvIioqCleuXMH//vc/2juQVUyZMgUmJiY1mpceO3YMx48fR2xsLK34TIoOJhgwYACuXbuGjh07QklJqc4dcjq7+cOHD8e+fftqpGddv34dq1evxvXr1wWOXVRUhBYtWuDq1as8qU+jRo2q4f4kCLWlcbFYLEhLS0NOTg5TpkwRuM5I2HU0v9oTjsVi0RI0Wlpa8PX1hZqaGpycnFBQUIADBw4gOzsbs2fP5knb+1VevXqFY8eO4fPnzzhz5gwMDQ1rbTZKJ32I6flXlClnZWVluHbtGmJiYnDx4kW0b9+e1o45E20iysvL8eDBA6SmpiI1NRVpaWncDRNtbW1oa2tj6NChtFNSk5KS4O3tDTs7O/Tu3bvG5hfdk4gfKS0tRXh4OHbs2IFv377RmiOZqMEk/Jw1a9agc+fOWLVqldBjm5mZQUZGBtbW1nxdVQXZNGCyrvZHmEyD9PT0xJcvX7Bp06YaadziAhFjTQB9fX3Y2dnVSBmMioqCr68v4uLi6h2zpKQEnp6eiImJgbS0NKZMmYK//vqLuyC9du0a1q9fj7dv3+Lhw4dC+T6YpmpRtmfPHtofdgBw7tw5ODk5gc1mQ0VFBUDlwjcmJgZubm4wMTGhFZ9J0cEEZ86cAZvNhoyMDCIiIuoUY3TeGw0NDRw9erSGsHj8+DFmz56NW7duCRybaet5X19f+Pr6Yty4cdyUijt37uDChQswNTWFhIQEoqKi4OLiwl3Y1xcm62iYguk2DqNHj0Z0dLTQrOGrw8T821C2/x8+fEBMTAzi4+ORmZmJESNGCNzsm+k2EdXJzs7mirPLly+Doija78ugQYO4fQurz2XCdn9LTU3lnkqWl5djwoQJMDU1Fer7A9CvwST8nO3btyMwMBB9+vRBr169agh4OoJmyJAhiIiIQJ8+fegOs8nh7OzMnd979OhR432nm/kgCkjNWBPgw4cP3F3w6qiqquLNmzcCxfTy8kJ4eDgmT54MGRkZnDhxAq1bt8aiRYvg7u6OEydOoGfPnrSPrkVBSUkJLl68iHPnziE5ORldu3bFggULaMedPHkypKSkEBQUhMTEREhLS0NRURF79uwRSs64pKQk32LUzp07o6ysjHZ8YVNdYJmamjL2HEtLS7i5ucHHx4e781tYWIhdu3b9ctpFbRQXF9d6eiIM0tPTYWtri0WLFnGvzZ07F4cPH8bNmzfh5+cHdXV1HD58WGAxxmQdzffv3xEXF4f//vsP8+fPR3Z2Nvr27Us7fVNRURGXL19G165d8fHjR24NWnh4uFDMaioqKvDs2TOhNO7+ESbmXxaLxT35ZrFYfJtHt2zZEtbW1gLFr05hYSHi4+MRFRWF27dvo1u3bjAxMcHOnTtp1afu3LkTZmZmfNtE7Nq1i3abCKDyNCk1NRU3b97E7du3kZWVBWlpaaGkVwoqQn+V7du3IyYmBu/evcPw4cPh7OwMAwMDocw/TNRgEn7O3bt3uRuzHz58EGrsLl264Pv370KNKWooisLVq1d5emBqaGgI5TTrR5dfcYOcjDUBmEiX09PTw8KFCzFz5kwAwOXLl7F582Zoamri1KlTsLKygq2tbaOuW7p69Sqio6ORmJgIiqJgYGAAU1NTsSny3LdvH5KTk2uIDkdHRwwYMKDR5f3/rA9KdeiMfdasWcjMzISEhAQUFBQgJSWFZ8+eoaioCN26dePZxa5v6gPT1vOqqqo4e/Ys5OXlea6/ePECkyZNwr179/DmzRsYGhoK5E6Yl5cHPz8/PH78mK9hB50dwtzcXJibm+PTp0/gcDiIj4/H5s2bcf/+fdq905hu4zB27Fjs3bsXAwcOpBWHH0ynKzOdcjZkyBBIS0vjzz//hJmZmdDmR6baRDx79gxXr17FtWvXcOvWLXA4HAwePBijRo3CqFGjMGTIEKGkFDPN+PHjuSnzwj61ZqIGk9CwnDlzBuHh4fDw8ICCggKjRjlMUFBQAGtrazx48ABt2rQBRVEoLCzEoEGDcOTIEUayFsQJcjLWBLC2toaTkxMePHjAN11OEHJzczFq1Cju1zo6Onj9+jUSEhJw5MgRjBw5UihjZ5KFCxdi+PDhWLt2LSZMmMDXZY4ueXl5ePr0KbdpKkVR4HA4yMzMxJIlS2jFvnbtGjIzMzFu3LgaoiMrKwtnzpzh3tsY+mhERETwfP327VtIS0ujR48ekJKSwosXL1BaWgplZWXaxbpaWlp0h8uXHTt2AADfvxthpCZ17NgR6enpNcRYWloa93Tp48ePAvd5cnBwQGZmJrS0tIR+wrd161b07dsX0dHR3Pff09MTq1atwrZt23DgwAGBY+vq6iIpKYmnjQObzcb06dOFcjJmYmKCBQsWYMqUKZCXl6/x3tDpBcbE/Fsdpp0bN27cCAMDA6HPj7KyssjPz69xPS8vT+BNvPHjx+P169fo3r07tLS04OnpCU1NTaH1RePXP6426PaPS0xMpPXv6yIpKQm+vr5QVFSEn58ftLW1MWnSJPTv3x+zZ89m7Lm/I1XmQFJSUnXWtrJYrHpvdPxYe01RVK0O1o29abKnpye+f/+OyMhI7hz/6NEj/P3339i+fXuNE/T68uHDB4SHh+Pp06dwcXHB7du30a9fP/Tu3VsYw2ccIsaaAEyky5WWlvLYwUtKSqJZs2ZYs2aNWAgxAEhISMAff/zBWPxz587B1dUVHA6H615VNXF2796dthhjUnQwQfVC/6NHj+Ly5cvYvn07OnbsCAD48uULHBwc0K9fP1rPYfJEkGlRO2fOHGzatAnPnj2DqqoqKioqcO/ePQQHB2PJkiV4+/YtNmzYAB0dHYHip6Wl4eDBg4w4wN28eROHDh3iWbS3a9cOjo6OtNNDAWbbOPj6+gLgn3rGYrFoLayZTlcuKSlBWFgYsrOzuTVMQKU50f379xEfH1/vmNUXkH/88Qfu379f672CngYz0Sbi1atX6Nq1K9hsNkaNGsX9HoTFr5oiCfo7Y2lpib1796Jt27Y//Zuhc4r97ds37t9OcnIybGxsAADNmzfn+R0i0GfOnDlITk5Gx44d+bakqEKQzbyqdkJNgUuXLmH37t08td5KSkpwdXWFvb09LTH2/PlzTJ8+Ha1bt8b79++xatUqxMbGwtnZGUePHuVukjVmiBhrIhgZGYmk5xcTNRdM8ccff+DRo0cIDAzE06dP4ePjg8TERPTp00cogvLAgQNgs9lYsGABZs6ciYCAAHz48AEbN27EihUraMdvbGmI9eHQoUMICAjgCjEAaNu2Lezt7TFnzhzY29sLHPtnu9d0FtZMm1zMmzcP0tLSOHz4MA4ePAig0pXN0dERM2bMwNWrV9G7d+96uVxVp0uXLgL3iPoZRUVFtfbro1vDyHQbB6ZPl5icf93d3REZGYmBAwciMzMTampqeP78OT59+lQjNfJXYXIBWcWqVatgZWWFiRMn1mgT4eDgIFDM2NhYbopiYGAgJCQkMHLkSG6K4o8nzvWF6d+T7t27c1Mof0ypFiZM12AS/o+LFy9yjVGEvZn3Y+31169fkZ+fz7XOv3DhAkaMGCH0tg5MUFZWxjfVulOnTigsLKQVe+vWrRg/fjzc3d2hrq4OoDLLxdHREd7e3kKpT2UaIsaaCEyky9VmwS0u3L9/HzNnzoSqqiru378PDoeDrKwsbNmyRSh1KC9fvsSePXugqKiI/v37Iy8vD2PHjkVZWRkOHDiAKVOm0IrPpOhgmtLSUnz79q3G9U+fPtH+Hapt97pZs2aQk5Oj9b4wbT0PVDYjnz17NgoKCiAlJcXjkqejoyPwqRgA/P3339i4cSPs7OzQo0ePGrUzdOy4hw8fjhMnTvAIxdLSUuzfv5/7ASgomzdvxtOnT3Ho0KEabRx27NghtDYOb968QU5ODoYPH46ioiKezQI6MJmufPHiRWzZsgUTJ06Evr4+3Nzc0KNHD9jZ2aG0tFTgmEwtIKto164dTp06JdQ2Eb1790bv3r0xd+5ccDgc3Lp1C9euXcOJEyfg5uaGP/74A9ra2tDR0cH48eOF/B3Rp7qb3tatW2u9r7i4mNZzVq5cya3BnDhxIhQUFHhqMAnCo/oGXkVFBV9zlLKyMuzbtw8rV64U+DkPHjzA/PnzYWpqCkdHRwCVv0McDgdHjhxB3759BY4tCgYNGoQTJ05gzZo1PNdPnDiBAQMG0Iqdnp6OkJAQns9uKSkpLF26VGitnZiGiLEmAFPpcu7u7jwNnUtLS7Ft27YaO+90+08wxbZt2zB//nzY2dlBTU0NQOX31KpVK+zZs4e2GJORkeHu2MvLy+Px48cYPXo0lJWV8fz5c9rjZ1J0MM3YsWOxdu1arFu3DsrKyqAoCmlpaXBzcwObzaYV+8fd6/Lycjx79gwbNmyAubk5rdhmZmY/tZ7fuHEjSktLBXY7fPv2LUJCQngcpaZPny6UUzmKopCTk4P58+fXuE635s3R0ZHbOqC0tBQbNmzAf//9h69fv+LYsWO0xp2YmFijjcOYMWMgIyOD1atX0xZjHA4Hjo6OOH/+PCQkJBAfHw9PT08UFRVhz549NWzj6wPT6cpfvnzhit0+ffrg4cOH6N27NxYtWoRVq1bB1dW13jGr/659+vSp1oyH48ePcxsJ/yrFxcW4ceMGZGRkoK6uDl1dXdpzLT9kZGS4J2JA5eZYYGAgIiIiEB4eLtDvOofDgZeXF6KioiAtLQ0jIyPY29sz4rC6dOlSbN26tYZxwZ07d+Do6IgLFy4IHJvpGkwCf+bOnYtjx47xbHo9fPgQjo6OeP78OS0xtnXrVowdOxZ2dnbcaxcuXMDatWuxZcsWBAQE0Bo706xatQqWlpa4e/cudz5LS0vDo0eP4O/vTyt2RUUFdyOsOkVFRWLTd6zxWw4RfkpVulxMTAzatGmDU6dOwdfXF//73/8ETpcbPnw4Pn78iFevXnH/U1NTQ35+Ps+1V69eCfm7ER4PHjzgK1hmz56NnJwc2vGVlZVx8uRJAEC/fv24fb+ePHlCu+EoUCk6qv/34MEDxMTEYMiQIUJJg2SStWvXQk5ODnPnzsWwYcMwfPhwLFq0CMrKytxdPWEhKSkJRUVFODs7w8fHh1asKuv5PXv2YO7cuZg7dy527dqFv/76C+/fv8emTZuwfv16ges5/v33X0yePBlnz56FtLQ0KIpCREQEpkyZgsePH9MaO1BZY6ChoYEDBw4gMDCQ+19QUBDtNhSKioo4e/YsdHV1oa2tDQkJCRgaGvIUZAsK020c9u/fz01ZrtpgmjNnDp4/fw5vb29asZmYf6vToUMHfPr0CQCgoKCA7OxsAJU1drm5ubTjz5o1C35+fjzXcnNzYWNjg82bN9crVnZ2NvT19bF06VIsWLAAhoaGePDgAe0x8iM3NxcJCQnw9PSEubk5jIyMcPbsWWhoaMDFxUWgmDt37sSpU6cwYcIEjB8/HqdOneLbVkAY5OTkYNKkSVzTh7KyMuzcuROzZ88WiumArKwsmjdvjgsXLiAxMRGdO3cmQoxhlJWVMWfOHLx79w6lpaXYuXMnpk+fjjZt2vAYbgnC/fv3sXTpUp6UbSkpKSxcuFAg511Ro6amhpCQEHTv3h3Xrl3D1atX0aNHDxw/fhwaGhq0Yo8aNQoHDx7kEWQFBQXYtm0b7dgigyKIPcrKytSTJ08oiqIoCwsLKikpiaIoioqPj6dMTEwacmgNioaGBpWRkUFRFEWpqqpSL168oCiKom7evElpamrSjn/79m1KRUWFOnz4MJWXl0eNGDGCMjIyotTU1CgXFxfa8WvjwYMH1Pjx4xmLL0xycnKo8+fPU/Hx8dz3nymysrKoIUOG0IqhoqJCPXv2rMb158+fc2O/fv1a4OfMnz+fWrZsGfX9+3fute/fv1PLly+nFi5cKNigq6GioiLU93nRokVUcHAwd35hCl9fX2rWrFnUx48fude+fv1KLV26lNqzZw/t+Pr6+lRycjJFUbxzwfXr16lRo0bRis30/LtmzRrKzMyMys7OphISEig9PT0qIyOD2rp1K/Xnn3/Sjh8aGkqpqqpSVlZW1IcPH6i4uDhqxIgRFJvNpu7du1evWDY2NpS5uTmVnp5O3bt3j7KwsKCmTZtGe4xVnDhxgnJwcKDGjx9PKSkpUYMHD6YsLS2p/fv3U3fv3qXKy8tpxdfT06NiYmK4X1+6dIlSVVWlKioq6A69Bt++faPWrl1LDRw4kNqyZQtlYmJCjRw5kjp37hzt2F+/fqVsbGwoJSUlqn///lT//v0pJSUlyt7eniopKRHC6An8KCsro+zs7Kjx48dTEydOpIYOHUqdOHFCKLG1tbW5c1h1bt68SWloaAjlGeLKu3fvqD///JPS1NSkBgwYQLHZbEpVVZXS1dWlXr161dDD+yVImmITgOl0OXFl/Pjx2LVrF3bu3Mm9lpOTg82bN2PMmDG04w8bNgzx8fHgcDiQlZVFSEgIQkND0bVrV6G4y9WGhISE0BtKMkVVjYcw4VdLV1hYiPDwcNoGM0xbz6enpyM0NJQn/bdZs2ZYtmwZLCwsBB/4/2fkyJG4c+eO0Jq6lpSUYPv27fj+/Tu6dOkCLS0tjBo1ClpaWkItGme6jcP79++5Re/V6dq1Kz5//kxr7EzPvw4ODnBycsKtW7cwa9YshIWFYdq0aZCSkoKnpyft+Obm5hg5ciQcHR3x559/orS0FIsWLcLixYvrfcJ/9+5dHDlyBIMGDQJQ2SLCyMgIxcXFQrHO37RpEwYOHAgDAwNoampi6NChPH9LdPnw4QNP/aO2tja+f/+Ojx8/4n//+5/QngMALVq0wKZNm1BeXo6jR49CSkoKR48eFUqfN1HVYBJ4kZSUhLe3N5ycnBAdHY3Q0FChmZ5NmDABGzduxIYNG7gxMzMzsWnTJujr6wvlGUzx7NkznD9/Ho8ePUJhYSHatGmDgQMHwsjISCiO1126dEFkZCRiYmLw8OFDVFRUYObMmZgyZQqtFHRRQsRYE6AqXc7e3h79+vVDUlISrK2thZYuJ644OjpiwYIF0NDQQEVFBUxNTVFYWEjLzas6zs7OWLNmDbp06QKgsp7D1dUVBQUFsLW1xb59+2jFZ1J0MMGPPVHqgk7tEr+FhJSUFNTU1LBhwwaB4wLMW8+3atWKr+mCoEYMPzJs2DCsX78ely9fRs+ePWvYftfXofPIkSMoLy/HgwcPkJqaitTUVGzcuJHriqetrQ1tbW0MHTqU1lzDdBsHRUVF3Lhxo0adX0xMDPr06UMrNtPzb9u2bXnmkkOHDiErKwudOnUSmkDIz8/Ht2/f0KxZM3A4HLx58wYlJSX1Hn9hYSHPmBQUFCAhIYH8/HyhiLGbN29ya6w4HE6tLps3b94UKD2prKyM53uWlpZG8+bNUVJSItiA6+Dt27dYt24dbty4gWXLluH+/fuwtrbGihUrYG1tTcvoiOkaTML/wc/5lsViQUpKCn/99RePuKZTX//XX3/hxYsXsLKy4vnd0NfXF8p6hil27doFf39/bhuNNm3a4MOHD0hMTMTu3buxePFioThHZ2Zmolu3bpg6dSqAyr5mjx49EloTe6YhYqwJsGLFCixYsADt27eHiYkJfH19wWaz8fbtWxgaGjb08BqM1q1bIzQ0FDdu3ODulvTr1w86OjoCu3mlpaXh5cuXACrF0qBBg2rsvOTk5ODGjRu0x8+k6GCC6j1RXr9+DT8/P5ibm0NNTQ3S0tLIzMxESEgIbUMDJu2nmbae19DQgJeXF3bv3s09WcrLy8O2bdugqalJe/wnTpyArKws7t69i7t37/K8xmKxBPrQk5SUxJAhQzBkyBCuMUh2djZXnC1duhQUReHOnTsCj5vpNg4rVqyAnZ0dnjx5gvLycpw5cwZPnz5FfHw8z8m5oLGFPf++efOmztfbt2+PsrIyvHnzhpZDJlBpanTixAmMHTsWgYGBePHiBRwdHWFoaIiNGzfWqydYRUVFjblVWlpaaL2tqptdLFmyBPv37+cRZN++fcPWrVtx8uTJRt8El81mo0uXLggNDYWysjIAIDQ0FJ6enoiPj+fWIwsC0zWYhP+jtrr5qt5WwqqrpygKfn5+ePr0Kdf8SVFREQoKCkKJzwTh4eE4fPgwVq9ejenTp/O0Rvn27RvCwsKwa9cu9OnTBwYGBgI/JyYmBo6OjrCzs+Nu6r1+/Rrz5s3Drl27GqWz6o+wKIpPcxGC2PH+/XtwOBz06NEDT5484UmX+51Px2ojMDAQc+fOrfe/S09P57qL1dabp2XLlpg/f75Y9wmji4WFBYyNjbm7VFWcO3cOgYGBOH36NK34xcXF+Pr1K9q0aSOUHXd+8LOep8vbt28xc+ZMfP78mfsh+uzZM7Rv3x7BwcECpWwUFBT8NGWQw+Hg8uXL+PPPPwUY9f9RWlqK1NRU3Lx5E7dv30ZWVhakpaWhqalJyzxFFG0crly5goMHD3I3Zvr27QsbGxtMmDCBdmxhz78/Wj1T1Rwaf7xGV3QMHToULi4uMDMz414rLi7Gli1b6i1qlJSUuP3LqlBTU8O5c+eEljpbhYGBAXr27AlfX19IS0vj+vXrcHV1RVFRERwcHHi+n19lwIABSE5O5tr+A4C6ujrOnj0r9PG7u7vj77//rpFm+fz5czg6OiI0NFTg2Pv27UNycjJ8fHy4vZ0KCwvh6OiIAQMG/NafTeKKvr4+du3axU0BFgdMTU1hYGCAhQsX1nqPn58frly5QqsX2KRJk2Bubl4j1T8oKAinT5/G2bNnBY4tKogYawJUpcv9uGgsKCiAi4sL7XQ5cePw4cOIiYmBtLQ0pkyZwmPN/PjxY7i6uiIjI4P2IkZJSQnXrl3j28hQWIhCdDDBkCFDEBUVVaP26tmzZ5gyZYpA7k+FhYXcn23V6SRQWaczefJkWFlZCeU9Sk9Ph4KCAjp06IDIyEicP38e6urqWLhwoVD67BUVFeHs2bM8vZcmTZoksOgbMGAArl27xrMAdnR0hIODA/dabm4udHR0BPqdf/bsGbfR7q1bt8DhcDB48GCurfiQIUMEPmmuojY3xqo2DvHx8bTiv3z5UuiL6SqYmH+HDRuGoqIiDBs2DGw2u866yxEjRtQ7fnVu374NVVVVvqIxKSmpXrb0SkpKcHV15Xkv1q9fD1tbWx6BA9AX2B8/foSVlRW6d++Ozp0749SpUzAyMsKaNWsE7h/HL9WanxAG6KVaA5WnwXZ2dnwdDvmdMNaHWbNmITMzExISEjVqMH9sNs1Un7nfleLiYmRnZ6O0tJRns5bFYtFKmRs1ahQCAwPFyhFTTU0Np06dqnPMT548gbm5OdLS0gR+joqKCqKjo2vM8S9fvsTEiRPFwm2SpCmKKaJMlxMnfHx8sH//fowcORLNmjWDh4cHJCQkMGPGDBw+fBi7du1Cy5YthdIbjV+6XF5eXo1FR30RlehgEnl5ecTExGDp0qU818PCwgSq0cnPz4eFhQXevn0LfX19mJubo23btvj69SsePHiAQ4cO4fz58zh+/LjA5hpAZZrQxo0bERAQAFlZWTg7O0NTUxNHjx5FaWkp7R3lqVOnwt3dvd69m+qC337ahQsXsHz5cp5FqSD7buPHj8fr16/RvXt3aGlpwdPTE5qamrTeY34w2TsOqNxVHjp0KExNTWFoaMiTLiMITM+/169fx9WrVxEbGwsvLy/07NkTRkZGYLPZQulHV52VK1fC39+f7467IP3B+FnBe3l58XzNYrFoi7HOnTsjKCgIVlZWuHr1Kvbu3Us7HUmUPTNv3rxZq/kI3c0NpmswCfy5ePEinJycUFhYWGO+pXuKbWlpiRUrVmD27Nno2bNnjd531esDGwvfv3+v0UfvR9q1a4dv377Rek7Xrl1x+/btGmLszp076Ny5M63YooKIMTGFxWJxa4pYLBbfD8CWLVvC2tpa1ENrUGJiYrBy5UquCIiMjISfnx8+fvwIX19fGBgYYN26dbQFEwB8/foVXl5esLCwQJ8+fWBtbY2UlBQoKCjg0KFDAu3Ei0p0MM3KlSuxcuVKXL9+HYMHD0ZFRQXu3LmDrKysGj2NfgUfHx9UVFQgJiYGXbt2rfH6u3fvYGNjg4CAANja2go87sDAQLi6ukJTUxM7duxA3759ERAQgKtXr2L9+vW0xdjLly9pCwFBEeRU79WrV+jatSvYbDZGjRoFNTW1GqYgTFC9d5ytrS0mTpxIK15wcDCioqLg5eUFd3d36Ovrw8TEROA6PabnXxkZGYwbNw7jxo1DSUkJLl26hNjYWOzfvx/9+/eHkZERDA0NhbLQ6NChA75+/Uo7DsBsPSfAP53V2NgYO3fuxOnTp1FYWMhzvb6YmJjQGF39n+Xt7Y1ly5ZBXl6+VjMSQSBpiA2Dt7c3NDU1sXTpUqF/Pu/YsQNApUPpjwgjXZkJKIr66caCMLJNZs6cCTc3N7x48YJbq5eZmYnAwMAaG8KNFZKm2AQQRbqcuKCiooLIyEj06tULQGWtjIqKClq1agVXV1eh1J5U4ezsjNTUVBw4cACPHz/G33//DQ8PD8TGxkJKSgp79uypd8wNGzYgJSUFAQEBdYqO8ePH0xIdoiA9PR3Hjh3jNjMeMGAA5s+fL1CDYD09Paxbtw56enq13pOYmIht27bRSmkbPHgwLly4gK5du8LMzAyamppYvXo13rx5AwMDA2RkZAgcG/i//Hhra2u+u5uCmDH8Sp2OoGmK//33HzdF8fbt25CQkMDIkSO5KYo/pqEKm0ePHsHc3FxoaSalpaW4fPkyoqKikJSUhI4dO8LY2BgrV64UOKYo59+ioiJcunQJ58+fx40bNzB48GDazbw9PT1x/Phx6OrqQl5evsZpTWNa2P/q3EF3cfrq1SuEh4cjLS2Nm+0wdOhQTJs2TWjprn/++SdevHhR62KUzvhFUYNJqMngwYMRExPDt40GXV6/fl3n68I+MRcGSkpKOH36NLctDD8+ffqE6dOn0xaTfn5+CAoKwsePHwEA//vf/7Bw4UKhtIwRBeRkrAnAVLqcOFJSUsJzLC4jI4PmzZvD3t5e6B9ASUlJ8PX1haKiIvz8/KCtrY1Jkyahf//+mD17tsAx161bx1eIAYCcnBxsbW2xbdu2Ri/G1NXVeXr20CE3Nxf9+vWr8x4lJaWfutD9jI4dO+LDhw+QkpJCVlYWVq9eDaDyb0wYi+1du3ahvLwct2/f5lmECcuMQdhU9YmbO3cuOBwObt26hWvXruHEiRNwc3PDH3/8AW1tbejo6NBKERNVGwdpaWluymJkZCR8fX1x8OBBWmJMlPNvWVkZSkpKwOFwwOFweNKYBSU+Ph4dO3bE/fv3cf/+fZ7XBHXgBCprunbt2oX09PQa9TOAYLVKTJ+8AZXZFa6urmCxWFBTU8OgQYPw5csXhIWFITg4GG5ubrRPagHQdpWti9qs66tqMIkYYwYFBQW8e/eOETFWl9hiovWCsPjRxOtHaqvJrC82NjawsbFBfn4+pKWlxaa/WBVEjDUBmEiXa2owkT//7ds3rmhKTk6GjY0NAKB58+YCWzmLSnSIgqSkJBw+fBj//fcfwsLCEBERgZ49e2LKlCn1jlVaWlrjFOlHmjdvTtu2mc1mY/Xq1WjRogXk5OQwYsQIxMbGws3N7acfKr/CkSNHaMfghzA+zH6GjIwM90QMqEy5DAwMREREBMLDwxtt77gqvn37hoSEBERFReHmzZvo3r07rK2taaemMT3/FhQUICEhAXFxcUhJSUHHjh0xYcIELF26FGpqarRiA8A///xDOwY/1q5di/v374PNZjOWUh0ZGYlmzZpxWwjY2tpi/PjxmDRpkkDxHj58CCcnJ8yZMwe2trY8p4QcDgcHDhyAi4sL+vTpI9AJf3WYTIlkugaTwJ/Vq1fDzc0NdnZ26N27d43UUzptKPLz83HgwAFkZ2dz1xcURaG0tBRPnjxBamoqrbEzQVBQEOPPKCwsRMuWLbnpkLm5udxMhQkTJgg1/ZdJiBhrAnh4eCA1NRXz5s1DQkIC0tLS4OXlxS3+FiRdrqkhKSkp9JiKioq4fPkyunbtio8fP2L06NEAKntrCOp4JCrRwTTJyclYvnw52Gw27t69i4qKCpSVlcHZ2RkURTXandm//voLcnJyePnyJWbPng1JSUl8+vQJM2bMwIoVK2jFLiwshIqKSq1F+3Rwd3fniVtaWopt27ahVatWAISzc5qbm4s7d+4gPT0d6enpePjwIZo3bw4NDQ3aPdKYPvGws7PD5cuXwWKxYGBggKNHjwqtGSgT829+fj4SExO5AkxWVhYTJkzAkiVLGGliSlEUrl69yu1f1LdvX2hoaNCaN2/evAl/f3/Gmq4GBQXB29sba9eu5V6Tk5PD2rVrUVxcjOnTp9c75uHDhzFhwgS+TXRlZGSwcuVKfPjwAf7+/vD29q53/L179/7yvcJMDxV2DSaBP0uXLkV5eTmWLl0q9MyHjRs34saNG9DW1kZcXBzYbDZycnLw8OFD2NvbC2P4Qoeu02tdlJWVYd26dTh79iyio6PRq1cvXLx4Eba2tpCSkoKkpCQOHTqE4ODgn7Z+aQwQMdYEYCJdTpwJCAjgcRssKytDUFAQ2rVrx3Mf3Q+7lStXYsWKFSgtLcXEiROhoKCALVu2ICQkBL6+vrRiizt79uzBX3/9hXnz5nFruOzs7NC6dWscPnxYIDH248/1R+g6MgGVKUrGxsY8O/lz5syhFfPLly9wcHDAlStXwGKxMGbMGLi5uQktjW348OHcPPkq1NTUkJ+fj/z8fO41QRbFoaGhXAH26tUrSEtLQ01NDXp6enBxccHgwYNpO79VwWQbh9zcXKxfvx4TJkwQemwm5l8dHR2wWCxoaWnB3d0dw4YN477PP56K0236XFBQAGtrazx48ABt2rQBRVEoLCzEoEGDcOTIkZ+6odVGy5YtBbaY/xWCg4OxdetWGBkZca85OztDWVkZvr6+Aomx1NTUn/bLMzc3x7Jly+odGwAiIiJ4vn779i2kpaXRo0cPSElJ4cWLFygtLYWysjIjtXoSEhL48OGD0OMSKmEq8wEAbty4AU9PT4wZMwb//vsvrK2toaSkhLVr1+LJkyeMPZcOHA4HXl5eiIqKgrS0NIyMjGBvb//TDedfISAgABcvXsTatWvRrVs3lJeXY+PGjejWrRvCwsLQsmVLLF++HL6+vlizZo0QvhtmIWKsCcBEupy40q1bN5w/f57nWufOnWvUJ9CphahCV1cXSUlJeP/+PTdlhc1mY/r06bR6gYhCdDDNv//+W8POGqhs1Fqf3eEq+P1c+VFbrd2v4uHhAWdnZwwbNozrZkd3sevl5YWMjAzY2tpCQkICwcHB2LBhA3bv3k0rbhV0mmX+jE2bNmHgwIEwMDCApqYmhg4dKtSTPVG1cQgODsbLly+RnZ2NDh064I8//hBaaicT82/VyXdSUhKuXLnC9x5h1Rl6enri+/fviIyM5M5jjx49wt9//43t27dj48aNAsWdMmUK/P39sWnTJkYyEz58+ABlZeUa11VVVQVO487Ly0OXLl3qvKdz58748uWLQPGrp4QePXoUly9fxvbt27mitWrj5mep6j9DVDWYBF6YPAkqKipC//79AVTW8j569AhKSkqwsLCos6lyQ7Jz506cOnUKkydPhoSEBE6dOoVv377xdZ+tL1FRUXB2duZu7N6+fRsfPnyAi4sL1zDEysoKrq6uRIwRRAMT6XLiClP1D7UhKyuLz58/48KFC5CQkMCgQYNoCQJRiQ6madOmDT58+FCjkPnJkyc1Tih/BVH9XK9fv4579+4hKSkJZ86cgYeHB/r3788VZgMHDqx3zCtXrsDT0xM6OjoAKk+trKysUFZWJhKbeDrcvHmTezLC4XBqzb+/efMmNDQ06hVbFG0cKIqCv78/jh07xnMi0LlzZ1hYWMDGxoa2KGNi/hVFrUUVly5dwu7du3lqoKqaN9vb2wssxgoKChAdHY3Lly+jR48eNX536H6PCgoK+OeffzBv3jye60lJSfjjjz8Eitm5c2c8ffq0zvk1JycHcnJyAsWvzqFDhxAQEMBzeti2bVvY29tjzpw5tFLPRFGDSahJSUkJwsLCeOq6gMq58/79+7Scfrt06YLXr1+ja9euUFBQwL///gsAaNGiBT5//kx77EwQHx8PDw8P7un1mDFjYGdnBzc3N9rz7suXLzF06FDu17du3QKLxeLWNAOVm3o/Zo00Vhr3SoDwS5B0uYahsLAQ9vb2uHr1KtcpjMViwcjICFu2bBGocFTUYpIpJk2aBA8PD3h4eIDFYqGoqAhXrlyBm5sbT1qRsPj8+TPXSOJXxGxtsFgsqKqqQlVVFba2tnjz5g327NmD/fv3Y9++fQKdQuTl5fHsdKupqaG8vByfPn366S58Q1M9RW3JkiXYv38/z+/1t2/fsHXrVpw8ebLe740oesctX74cV65cwZQpU6CpqcndPElJScH+/ftx79492nMkE/Nv9R32n4lgupSVlfF1Cu3UqRNP3y5BYLI2ydraGk5OTnjw4AFPb6GYmBi+vZh+BV1dXezfvx+ampp8F4sVFRU4cOAA7ebSQGVdJ78sh0+fPtFeqIrCdZJQE3d3d0RGRmLgwIHIzMyEmpoanj9/jk+fPtXYNKgvf/75J5ydnbF161ZoaWnBzs4OKioqSExMZLzFiKB8+PCBx1FZW1sb379/x8ePH/G///2PVmxJSUmUlpZyv759+zY6d+6M3r17c699+vRJfFwVKUKTIC8vj8rKyuJ+fe/ePerJkycNOKKG58OHD5SLiwtlYGBAjRs3jho7dizPf3RxcnKixo8fT125coX6+vUr9fnzZ+rSpUvU2LFjqS1btgjhO6hJQUEBFRAQQBkYGDASX1hwOBzK3t6e6t+/P9W/f39KSUmJ6t+/P7Vo0SLq+/fvQntOamoqtXr1amrIkCFU//79KRMTE1rxKioqqMzMTMrf359asGABpaamRg0ePJiytLSkDhw4IFDM/v37U7m5uTzXVFVVqRcvXtAaq6iZMGECZWNjQ3E4HIqiKCo5OZnS09OjRowYQZ06dare8caMGUP9888/dd6TkJBA/fnnnwKN99SpU9SwYcOohw8f8n09OzubGjlyJHX69GmB4leHyfl3/vz5VElJCc+1oqIiau3atZSSkhLt+JaWlpS7u3uN625ubpS5uTnt+EwSExNDmZubU6qqqtTw4cOpGTNm/PR3qi7evXtHaWhoUIsWLeL5eVIURWVlZVHW1tbUmDFjqLy8PLpDpxwcHCgjIyPq5s2bVGFhIfX161fq8uXL1Lhx46hNmzbRjv/t2zfq/fv31Ldv32jHIvwampqaVAT4bioAAFh0SURBVFRUFEVRFDV+/HgqJyeH4nA41LJlyyg3NzdasUtKSih3d3fq3LlzFEVR1Nq1a6n+/ftTw4cPp65fv0577EzA5GffnDlzqCNHjlAUVfl3O2jQIMrV1ZXnnvXr11Pz58+n/SxRQJo+NyGePXuG7OxsoaTLNQUWL15cp7Uy3Zqx4cOHY9++fRg+fDjP9evXr2P16tW4fv06rfjVSUtLQ2hoKC5cuICSkhIMHDiwRjF4Y+T58+fIyspCRUUF+vXrh27dumHXrl1wcXEROGZhYSEiIyMRFhbGLVweNWoUbGxsaOfsDxs2DN++fYOqqiqGDRsGDQ0N2nVSv9KUWRz4+PEjrKys0L17d3Tu3BmnTp2CkZER1qxZI5BRw+DBgxEXF1dn/5xXr17B0NAQmZmZ9Y4/Y8YMGBoaYu7cubXeExISgpiYGBw/frze8X+EqfnXwMAAPXv2hK+vL6SlpXH9+nW4urqiqKgIDg4OMDMzoxX/zp07sLS0hJKSEncXOy0tDY8ePYK/v3+900+r8/btW4SEhPC4NJqbm9Ouw2SSrKws2Nra4uXLl2jRogXatWuHwsJCFBYWom/fvti5c6dQ0v8LCwtha2uL5ORk7kkYRVEwMDCAl5eXQJkVoqrBJPBHWVkZFy5cQLdu3bBkyRKw2WxMnDgRmZmZWLVqlUC99aqIjo6Gjo4OT5p/QUEBWrdu3WjT3Zn87Lt69SqWLFkCXV1dZGVl4dOnT4iIiICioiIePXqEsLAwhIaG4sCBA9DV1aX7rTBO4/wJEuoFE+lyTQGmrZUlJSX5irzOnTsLxXqeSdHBBCUlJfD09ERMTAykpaUxZcoU/PXXX9wUimvXrmHRokV4+/atQGIsIyMDYWFhiI2NRXFxMfr27QtbW1vs2bMHjo6O6NOnD+3vYeTIkbh16xaePXuGrl274vXr15CXl69TMPwK7969q2Ev//79+xrGBo15kdq5c2cEBQXBysoKV69exd69e2mlazHdxiEnJ4dbp1cbOjo62LVrl0Dxq2B6/g0ODoaVlRWWL18uFBH8I2pqaggJCUFAQACuXbsGiqLQv39/rFu3jpbZw7///gsLCws0b94cQ4YMQUVFBSIiIhASEoITJ06gb9++tMeel5eHp0+foqKiAkClmOFwOMjMzBS4qfKAAQNw/vx5XL58GXfv3sXnz58hKyuLoUOHYtSoUUJzD61yln369Cmys7PBYrEwYMAAgRepoqjBJNRNhw4d8OnTJ3Tr1g0KCgrIzs4GUFlbnpubSyv2pk2bcPz4cR4x1tgt21ksVo2UW2EZJ+no6ODgwYMICwvDkCFDMHfuXO4myalTp3D+/Hls2rRJLIQYAJCTsSaAs7MzUlNTsW7dOqipqaGiogLp6elwc3ODvr4+32Le3wEtLS2EhISgV69ejMTft28fkpOT4ePjw625KCwshKOjIwYMGCDwyRs/0cFms7Fnzx6cPXtWKKKDCdzc3BAWFobJkydDRkYG586dg42NDRYtWgR3d3ecOHECPXv2hLu7e43TxJ9hamqKrKwsKCoqQl9fH0ZGRtzF3KBBg4T6vlRUVCAjIwPJycm4ceMGMjIy8L///Q+ampoC1aIoKSnV+ACi/r8T3o9f03XGEzb8XNny8/Oxc+dOaGtrY8KECdzr9W1XwG/X9Edyc3Oho6Mj0PuipqaGs2fP1jCRqc6LFy8wdepU3Lp1q97xqxDF/JuXlwcrKys8fvwYu3fvFkrNEtNYW1ujRYsW2L59O/dkuaSkBKtXrwaHw8HBgwdpxT937hxcXV3B4XDAYrF4/qa6d++OxMTEesd0dnbGmjVrRFpncvv2beTk5GDixIl49+4dFBQUBDrp2LBhA1JSUhAQEFBnDeb48eMFrsEk1I2rqysePXqELVu24Pnz5/Dw8ICPjw9iY2Pxzz//0DLwmD59OubNm8dIzTVT/MpnXxXC/OwrLi5G8+bNhSb8RAERY00AUabLiROenp748uULY9bKs2bNQmZmJiQkJLgfoM+ePUNRURG6devGMxH8anqCKEWHsNHT08PChQsxc+ZMAMDly5exefNmaGpq4tSpU7CysoKtra1AJwVKSkro3bs3TE1Noa2tjQEDBnBfY+p94XA4SElJwcWLFxEREYGKigrcv3+/3nHqs9BvbCee1R326kIQIamkpMRdsNfGt2/fcOTIEYE+qKdNmwYTExPMmjWr1nuOHz+OmJgYhISE1Dt+FUzMv0yKYIBXdDg7O9d575YtW+odH6gUw6GhoVw77ioePXoECwsLpKamChS3CiMjI6ioqGDBggWYOXMmAgIC8OHDB2zcuBH29vaYMmVKvWMOGDAA165dY7Q/WhWFhYWwtrbGvXv3wGKxcOHCBWzevBkvXrzAkSNH6m3uo6enh3Xr1kFPT6/WexITE7Ft2zZaooBQO1++fIGTkxO0tbUxa9YsLFy4EFevXoWUlBQ8PT3BZrMFjr1u3TqcOnUKSkpKUFBQqJE6L+jfKZOcOXPml+81MTGh/TxhbWw0BOIxSkKdMJ0uJ64wba2spaUFLS0tWjF+5OHDh+jduzeMjY2hra0tlFQeUZGbm8tjK6ujo4PXr18jISEBR44cwciRIwWOHR8fj4iICAQHB2P79u3o2bMnjIyMhL5LmJWVheTkZCQnJyM9PR0yMjIYNWoU3N3dBU53EKUznrBh0pWN6TYOJiYm2Lt3L7S1tfm6jeXk5GDv3r1wdHQUKH4VTMy/dZ2mXbp0CZcuXQJQKYIFEWOvXr3ipva9evVKoDH+jFatWvG4nVXB75ogvHz5Env27IGioiL69++PvLw8jB07FmVlZThw4IBAYkyUe9M7duwAi8VCQkICJk+eDAD4+++/sXr1anh5eWH79u31ipebm/vT/mRKSkoC92Aj/Jy2bdti3759ACpdSi0sLKCqqorp06ejc+fOtGI/ffqUa+UuLnbtwhBYv0JhYSEWLFiAu3fvgsViQVtbG97e3gJvbDQERIw1ASwtLeHm5lYjXW7Xrl2wtLRs4NE1LExaK9M1AOGHqEQHE5SWlqJly5bcryUlJdGsWTOsWbOGlhADKgvQ7ezsYGtri6tXr+L06dPw9/fHgQMHAACxsbGwsrKiXQthYmKC7t27Q09Pj1ubJ8ydNWHbw4uayMhINGvWDIaGhgAAW1tbjB8/HpMmTap3LKbbOMyYMQOXL1+GmZkZTE1Noa6ujvbt26OwsBC3b9/GyZMnoaurK9CivTpMzL9MW5NXbxRe/f9LS0tx/fp1UBQFTU1NWsY1Ghoa8PLywu7du7m1LXl5edi2bRs0NTUFjluFjIwM9+9IXl4ejx8/xujRo6GsrIznz58LHFdUqU2XLl3C9u3beWrEFBUVsW7dOixbtqze8ZiuwSTUzvHjx7mGWtOnTwebzcbs2bPx77//gqIonDx5EkePHoWCgoLAz6j+dypuvHr1CuHh4UhLS0NeXh46dOiAoUOHYtq0aUIxsdqxYwcACG1joyEgaYpNACbS5Qg/h18qUXUE2bGuoqKigis6Ll26xP0AXbJkiVBEBxPU5pz0s7odQSkoKMC5c+dw5swZZGVloUWLFpg8ebLATWoBIDs7+6e7y3Rg2hmPSYKCguDt7Y21a9di2rRpACpTY8LCwuDi4oLp06cL9XnC6B1XdUpy7NgxFBQUcK937twZ8+bNw/z582kvvkUx/wpTBFfBbwFpYWHBFYJdunRBYGCgwD2M3r17hxkzZuDz58/cReizZ8/Qvn17BAcHC9yYuYq5c+dCRUUF9vb2CAoKQlJSEg4fPowrV67AwcFBoJNmfjUutUF340RFRQXR0dHo0aMHj8NcTk4OzMzMcPfu3XrFY7oGk8Cfw4cPY+/evZg0aRJatGjB/Zl++/YNGzZsQEVFBbZs2YI//vgDPj4+tJ71/ft3xMXF4b///sP8+fORnZ2Nvn37QlZWVkjfjfCJiYmBq6srWCwW1NTUICsriy9fvuDevXvgcDhwc3OjvWmup6eH7du3Q11dnedv6c6dO1i2bJlYlOqQk7EmABPpck0FJq2Va0slatasGeTk5GiJMQkJCejq6kJXV5dHdOzbtw9HjhyhLTqYgt9Chqmd5vbt28PS0hKWlpZ4+PAhIiIiEB0dTet96devH3dR999//yEsLAwRERHo2bMn7RMUgHlnPCYJDg7G1q1beU5pnZ2doaysDF9fX6GJMX5tHARFSkoKy5cvx7Jly/D06VOuM17Pnj2F5orH9PxbXQRXIScnh7Vr16K4uFig9/3HBaSPjw8iIiJQXl6OkJAQ7gJy+/bt2L17t0DjlpOTQ0xMDM6ePYvHjx+DoihMnz4dkyZNEopBxooVK7BgwQK0b98eJiYm8PX1BZvNxtu3b7miVRCcnZ1Fstk1ePBgnD9/HgsXLuS5HhISIvDvfEBAwE9rMAnCJTw8HJs3b+bOi2w2G9OnT8eBAwe4rSKcnZ1pm6bk5ubC3Nwcnz59AofDwbRp0xAQEID79+8jMDBQKO0WhM3Dhw/h5OSEOXPmwNbWluekncPh4MCBA3BxcUGfPn1+uT6ZH3l5eXzTQNu2bSs2v/PkZIzQZOFnrXz//n0UFxcLzVq5OuXl5Xj27Bk2bNgAc3NzRlIkq4uOxlZjpKSkBCMjI54JNyoqCmPHjkWrVq147qVbbFxeXo78/Hy0adOG53lv376FrKzsT9N1aiM5ORmLFi3CxIkTERsbi5iYGJw6dQp+fn7w8PCgJbCrEEdnPKByJz8qKqrGKefLly/BZrORkZEhcGym2zhkZmZi8ODBNa5//foV7u7u8PT0pP0MptDX14ednV2NVOWoqCj4+voiLi6u3jEnTJgAW1tbbsyMjAzuAnLMmDEAgNTUVG4fLEFITU2Furp6DdH77ds3BAQECCXN+/379+BwOOjRoweePHmC0NBQdO3aFZaWlpCWlq53vF85XRIW6enpmD9/PkaNGoWkpCRMmjQJOTk5ePDgAQ4fPlzv1O6xY8f+8r1Mpwj/TvDrl6isrIxz586hd+/eACprvMaMGYMHDx4I/JzVq1ejsLAQO3fuhJaWFs6dO4e2bdti1apVaNasGTdlvzHx119/gcViwdvbu9Z7XF1d8f379zrv+RkWFhYYPXo0Fi5cyHMytmnTJjx69EgofSSZhpyMNQGYTJcTZ7y8vDBy5Ei+1sre3t60rZV/RFJSEoqKitxdMGGIsR9Fx8CBAzFw4EBYW1vj+/fvAosOJhg+fHiNwmI1NTXk5+cjPz9fKM+IjY1FUFAQMjIyuMX2/fr1w5w5czB16lRs3rwZKioqsLGxESj+nj17sHr1asybN4/rOGZnZ8ftCSTI3xK/v09jY2Ps3LkTp0+fRmFhIc/1xoqCggL++ecfzJs3j+d6UlKSwClnougdBwALFizA0aNHeVw4//nnH6xbtw7l5eW0YjM9/3748AHKyso1rquqqgpsxvDmzRuoqKhwvx4yZAikpKR4hLa8vDxPamd9sbCwgLa2Nnx8fHhOwr59+wZfX1/aYqzKEbKqOL9Pnz5wdXVFQUEBbG1tuUYK9UGUVtjq6uoICwvD4cOHIS8vj7t376Jv375wcXHh+dn8KkRgNQz8avWkpaV5NgNYLBbXMEdQbt68iUOHDvGcfLZr1w6Ojo6N1hsgNTX1p6mZ5ubmAtVIVsfe3h7z589HRkYGysrKsH//fp6NDXGAiLEmAJPpcuJMeno6QkNDeU5OmjVrhmXLlsHCwoKx50pISODDhw+0YjAtOpiA6QJjd3d3HDt2DJqamli1ahVkZWVRUFCAlJQUrF27FpGRkcjJycHmzZsFfsa///4LLy+vGtcNDAywd+9egWIy7YwnKqytreHk5IQHDx5wF4uZmZmIiYkRqP9a9TYOVf1zqk6r9+zZI9SxT548GfPmzUNgYCC6du0KNzc3REdHY+LEiQI1IK8O0/MvEyJYVAvIN2/ewNzcHAcPHqRdIwZUprC+fPkSQKUIHjRoUI2Ux5ycHNy4cUOg+EwnCvGbQ3r27Mkjgq9evYpr167RXqD+iDBqMAkNR1FREY9BVnUaqylLXl7eT50MO3fujC9fvtB6jrq6OkJDQxEQECCUjY2GgIixJsCPzls/psv9rjBtrcxvR7ywsBDh4eEYMmSIwHFFITrEjYSEBISFheHgwYM1LOZtbGyQmJiI5cuXY9GiRWjXrp3Az2nTpg0+fPhQIxXvyZMnAsdl2hlPVEyePBlSUlIICgpCYmIipKWloaioiD179tTZ26g2RNnGYc2aNZCWlsa8efMgJSUFGRkZ+Pn5QUdHh3ZspudfYYtgUcFisXDo0CF4enpi2rRp2LNnD4YNG0Y7ZpX4ZbFYcHd3r3FPy5YtYW1tLVD85cuX48SJE798b33Zu3cvJCQkICcnV+d9LBZLaGJMmDWYBP78WKtXVlaGoKAg7meGMOqWhg8fjhMnTvD0BSwtLcX+/fu5tWmNjc6dO+Pp06d1tifJycn56d/Dz7hx4wY0NTX5bqSKC0SMNUGYSJcTR5i2Vua3Iy4lJQU1NTVs2LBBoJiiEh3iRkhICBYtWlRrr6+7d++idevWSEtLo/WcSZMmwcPDAx4eHmCxWCgqKsKVK1fg5uYmtPYCTDjjiQphtlkQdRsHBwcHyMjI4NChQzhx4gRjO6bCnn+FLYKrYHoBSVEUWrZsib1798LLywtWVlbYtGkTtyZNENTV1bniV0lJCdeuXeO2ExAG9RFLgoix6dOnIyEhAUCl0QObzaZlXFAbTNdgEv4Pfv0SO3fuXMM5VdB+iVU4Ojpi9uzZuHXrFkpLS7Fhwwb8999/+Pr1K44dO0YrNlPo6upi//790NTU5JsCXFFRgQMHDtCum54/fz66du0KY2NjmJiYCMUuX9QQA48mzKNHj2Bubo579+419FAaBKatlZlg3rx5GDZsWK0f9N7e3ggNDYWSklKjnYCZQENDA8HBwbWenrDZbCxZsgRubm5ISUkR+DmlpaVwcnJCTEwMgMpFF0VRGDNmDHx8fGj1XQJEbw8vbPLy8vD06VNu+hpFUeBwOMjMzMSSJUsEislUG4fa6iju3r2Lli1b8rQwoNsAnh+Nef4VhdnDgAEDcO3aNa4ZxsmTJ7Fx40bMnj0bQUFBjNirV/UwEpR169YhISEBLVq0YEwslZeX4+bNm4iNjUViYiI6duzIfRadPlQA/xpMNpuNPXv24OzZs0KrwSQ0DO/fv8eJEyeQlZWFiooK9O3bF7NmzWqUaxmgcrzGxsZQUVHBqlWreP6WHj16BG9vb+Tk5CAiIoKWPf+bN29w7tw5REdHIycnB+rq6jAxMYGhoWEN87DGChFjTYC60uXatWsn1s0C6VJUVMRjrdy/f3+hWSsDQHFxMb5+/Yo2bdrUaSn8q4hKdIgb6urqiIqK4nGsqg5FUXj9+jUmT56M9PR02s97/vw59wOvX79+QlvEMOGMJyrOnTsHV1dXcDgcrkit2u3s3r07EhMTaT9DmL3jqqfz/Aw67p6imH+ZEMFMw8+Z8Pr167C1tUVhYSFtMfb161d4eXnBwsICffr0gbW1NVJSUqCgoIBDhw4JvDvOpFj6kdLSUly7dg3nz5/HxYsXuafDbDa73u1Xqtdg6uvr89RgDho0iIgxQoOQlZUFW1tbvHz5Ei1atEC7du1QWFiIwsJC9O3bFzt37hSqLf/Dhw8RFRWFuLg4FBQU4M8//2zUbrlVEDHWBOC3c1c9Xa4x9p8QBUxZKxcWFuLw4cOIiYnhFpMDle5jkydPhpWVlcDCTNSiQ1wwMTHB/Pnz60zlO3v2LAIDA7mNbBsjTNrDM42RkRFUVFSwYMECzJw5EwEBAfjw4QM2btwIe3t7ofRhqw4TbRw4HA5kZGQAVO7a/qy4/Fdgev4VhQhmglu3bkFdXR1SUrzVEDk5OTh//rxQ3BRTU1Nx4MABPH78GH///Tc8PDwQGxsLKSkpoZjACFMs/QwOh4PTp09j+/btKCoqqrdYVVJSQu/evWFqagptbW0e51AixsQTS0tL7N27F23btv2pYyITp/vCory8HJcvX8bdu3e5vR6HDh2KUaNGCa3fY3UePHiAuLg4HD9+HCwWC6mpqUJ/hrAhNWNNgKZiECBsmLBWzs/Ph4WFBd6+fQt9fX2Ym5ujbdu2+Pr1Kx48eIBDhw7h/PnzOH78uEDpVfLy8khPT69VjLFYLKSlpQl9h7axw2azsXv3bujo6HDr/6rz8eNH7N69G3Pnzq137LFjx/6SpTWLxaK98GXCGU9UvHz5Env27IGioiL69++PvLw8jB07FmVlZThw4ABtMcZkG4e8vDysWrUKampqsLOzA1Ap8JWUlLBz505a9ZdMz78HDhwAm83mK4JXrFjB6LPpMGLECJSVleH9+/fc9gEURUFCQqLGZoQgJCUlwdfXF4qKivDz84O2tjYmTZqE/v37Y/bs2bTjA5UOk3p6etDT0+MRSzt27BBamuWHDx9w4cIFxMXFIS0tDfLy8pgzZ06944i6BpPAPN27d+eKldrWBI2ZqvYTrVu3xrhx4zBu3DjGnvXy5UtERUUhKioKz58/x8iRI7Fu3TpMmDCBsWcKEyLGmgjCTpdrKgjbWtnHxwcVFRWIiYnhW5D77t072NjYICAgALa2tvWOz6ToEGcsLS1x/vx5GBsbw9raGurq6mjbti0KCgpw+/ZtrqWtIIswExOTOsXYuXPn8OLFC6F8GIqrMx4AyMjIcE+V5OXl8fjxY4wePRrKysp4/vy5wHFF0cZh8+bNKC4uBpvN5l7z8/PDhg0b4OnpCQ8PD4FjA8zOv0yLYKa4du0aHB0dkZeXV+O15s2bY/LkybTif/v2jTsHJycnc38/mjdvTrt3XHWEJZaq8/79e8THxyMuLg537txBjx49YGhoCFdXV4Fr1OTl5WFnZwdbW1tuDaa/vz+3GXBsbCytGkyC6KmePk0nlbqhiIyMxOrVq4VWFlIb06dPR2ZmJv744w+uiYewT66ZhogxMYbJdLmmABPWyklJSVi3bl2tzkhycnKwtbXFtm3bBBJjTIoOcUZGRgZBQUHw9PTEtm3beNoTyMjIwNTUFA4ODpCUlKx37NpOF96+fQtXV1e8ePEC5ubmcHBwEHj8VTDljCcKlJWVcfLkSdjb26Nfv35ISkqCtbU1njx5wtOfqj6Iqo3DtWvXEBgYyGPaMWjQIKxfvx4LFy4UKKao5l+mRDDT7NixAwMHDsScOXNga2sLb29vvHnzBrt37xbKwlJRURGXL19G165d8fHjR4wePRoAEB4eTjs1lAmxBABHjx5FfHw87t27h27dusHQ0BBr1qzBoEGDaI23OhISEtDV1YWuri5PDea+fftw5MgRgWswCaKnPk3dG6P4EFUVlKKiIv7++28MHz5cJM9jAlIzJqb8mC7Xr18/nnS5ixcvokePHgKnyzUFqheQe3l5ITg4mGutrKWlJVCayeDBgxEXF1fnKcmrV69gaGiIzMxMgcZdVFQET09PREZG1io6fleRXVxcjPLycmRkZCA/Px8dOnSArKwsFBQUaKWx/cjJkyfh5eWFNm3awN3dHVpaWkKLLa6kpqZiwYIFWLlyJUxMTGBgYIBOnTrh7du3MDQ0rLdgSkhIgL29Pfbu3cu3ZUH1Ng5VqYWCMmzYMAQHB/PU0QCVTb5nzJiBO3fu1CueKOffuXPnQkVFBfb29ggKCkJSUhIOHz6MK1euwMHBQWj1dMJmyJAhCA8Ph5KSEmbNmoUVK1ZAU1MTp0+fxqlTp365n1dtJCUlYcWKFSgtLQWbzYa3tze2bNmCkJAQ+Pr61toGoy74iSUDAwOhiSUlJSVIS0tDS0sLgwcPrvNeujV1P8JEDSaBWZSUlH6aQl9VQ8qEOyldlJSUcP36dVoOp78L5GRMTGE6Xa4pUH0Sc3BwQK9evbB27Vpap0qlpaU/XfQ3b96ca88tCK1atYKzszMcHBwYFx3iRHR0NDZv3gw/Pz8ecWRtbY379+/D3d0d+vr6tJ7x/v17rFmzBteuXcO0adPg6Ogo9BQLcXTGAyoFTXx8PDgcDmRlZRESEoLQ0FB07dr1p8Xl/BBV7zigsmHqjh07sHPnTu7Ps7CwED4+PgLtpopy/l2xYgUWLFiA9u3bw8TEBL6+vmCz2VwR3FiRlJTkClF5eXlkZ2dDU1MTGhoaQnE309XVRVJSEt6/f889rWKz2Zg+fbrAJ2Nbt26FtLQ0dHR0uGLp0qVLuHTpUo17BRFLVacXjx8/xuPHj2u9T9A+ZtVhsgaTIBp+NOWgKAoLFy6Eu7u7UMyHRIG2tvYv3VdfMVm9dcbPRGtjFKo/QsSYmMJ0ulxT4MdD32nTpqF79+6N/v0QhegQN1JSUuDg4ABjY+MaH0IuLi7w9/fHqlWrEBwcDHV1dYGecfr0aWzZsgVt2rSBv78/Ro0aJYyh8/AzZ7zGLMaqirGr3v8+ffrA1dUVBQUFsLW1xb59++oV79GjR1izZk2tr1+6dAkbNmwQSi2ds7MzZs+ejdGjR9foOejv71/veKKcf4UtgkVF37598c8//2DOnDno3bs30tLSMHfuXLx7905oz5CVlcXnz59x4cIFSEhIYNCgQbSa6zItlgTt2VYfRFGDSRAN/Bp0S0hIQFVVVWwaGzs7OzOSneXh4cGN6+Hh8UsmXI0ZIsbElNzcXJ76B34oKSnVK+e4qREUFFTDJU1LSwuhoaE4f/68wHEDAgLqTBP89u2bwLFFITrEkUOHDsHCwgIuLi41XlNUVOTWoOzfvx9+fn71iv3+/XusXbsWV69ehZmZGZydnRlrFCluznhpaWnceqjIyEgMGjSoxklhTk4Obty4Ue/YHA4HLVu2rPX16OhovH79midVV1B69uyJ2NhYxMTE4PHjx5CSksLMmTMxadIkgU4IRDn/ClsEi4qFCxdi5cqVkJaWxsSJE7Fnzx4sXLgQ//77LzQ0NGjHLywshL29Pa5evcoVHSwWC0ZGRtiyZQu3zq4+iEIsMYmoajAJhF+FzWbz9BoUFiYmJtz/NzU1FXp8UUPEmJgiinQ5cYcJa+Vu3br9kpATdHeWSdEhzjx8+BCOjo513jNr1iwsXry43rEnTpyIwsJC9OjRA+Xl5XB3d6/1XrrGA+LmjMdiseDk5MT9f37vTcuWLWFtbV3v2KJu49CmTRvMmDFDKLGYnn+ZFMGiYvz48Th58iQkJSXRtWtX+Pv748iRIxg3bhxWrlxJO/7mzZvx9OlTHDp0CGpqaqioqEB6ejrc3NywY8cO7u/t70JCQgLCwsJw8ODBGqm/NjY2PDWYdFo5EAi/iqhOqzgcDgICAmBoaAh5eXmsWbMGsbGxUFdXh7e3N2RlZUUyDjoQMUZosjBhrcz0zimTokOcKSkp+enit3379iguLq537OruaK9evar3v68P4uaMp66uzu2jpaSkhGvXrqFTp05CiS3KNg4lJSUICwtDdnY2j+05h8PB/fv3ER8fT/sZwoRJESxKqhtfjBgxgm/alaAkJiZi3759PDV/Y8aMgYyMDFavXv3biTFR1mASCL+CqPwBvb29cfbsWejo6ODKlSs4c+YMVq5cicuXL8PLy0ss2gIQMSbGMJku1xRg2lqZH58/f0ZERATCw8MFSoVkUnSIM7169cKdO3fqPNGs65SlLoKDg+kMrV4wYQ8vKvg1N87LyxPYKUuUbRzc3d0RGRmJgQMHIjMzE2pqanj+/Dk+ffpUowH3r8Lk/MukCBYVeXl58PPzw+PHj8HhcGq8/qM5QX2pbhBSnc6dO/+WGSGirMEkNBziVBu1fPnyX3ZNpWNYExcXhx07dnDblYwYMQKLFy/GqFGjxKY2kogxMYXpdLmmwJMnT+Dh4QElJSUMGDAALVu2xJw5c9CyZUscPnwY48ePF9qz0tLSEBoaigsXLqCkpAQDBw4UKA6TokOcmTx5Mnx8fKChocHXRer9+/fw8fGBmZkZredMmzYNZmZmmDhxIiONKsXVGQ8Avn79Ci8vL1hYWKBPnz6wtrZGSkoKFBQUcOjQoXoXlDPZO+5HLl68iC1btmDixInQ19eHm5sbevToATs7O4Fq0kQ5/wpbBIsKBwcHZGZmQktLixHnPktLS7i5ucHHx4crVAsLC7Fr165GbWzCFKKswSSIBn4ugRRF4c8//6xxb2N0DNy7dy8kJCQgJydX53103UMLCgq4DqrJyckwNzcHULlx/f37d4HjihIixsQUcS80FgVMWysXFhYiMjISYWFhePLkCQBwd2IETccRlegQNywsLBAfH4+JEyfCzMwMampq3FOU9PR0nDlzBgoKCrTTtjQ0NHDgwAFs2bIF48aNg6mpKbS1tYW2GymuznhApWNVamoq5s2bh4SEBKSlpcHLywuxsbHw8vLCnj176h1TVG0cvnz5wjW86dOnDx4+fIjevXtj0aJFWLVqFVxdXesVT5Tzr7BFsKhIS0vDwYMHhZqaWJ1r164hMzMT48aNg4KCAqSkpPDs2TMUFRUhKysLZ86c4d578eJFRsbQmBB1DSaBecTdJXD69OlISEgAUJmWzmazaTVNr42ePXsiMzMTnz59wqtXr6CjowOgMpX5jz/+EPrzmICIsSYK3XS5pgBT1soZGRkICwtDbGwsiouL0bdvX9ja2mLPnj1wdHREnz59BI4tKtEhbkhKSuLo0aPYtWsXTp8+jaNHj3Jf69SpE2bPno0lS5bQXrz/9ddfsLe3x/Xr1xEZGYkVK1agbdu2MDY2hrGxMXr16kUrvrg64wGVdu6+vr5QVFSEn58ftLW1MWnSJPTv31/gVEJRtXHo0KEDPn36hG7dukFBQQHZ2dkAKq3Rc3Nzacf/EWHOv0yIYFHQpUsXxlxJgUpnXNKM/f8QZQ0mQTRUdwksKipi9O+JCTZt2oT169fj5s2biI2Nxdy5c9GxY0euMBPWxsCCBQtgb28PCQkJaGhoQElJCb6+vvD19YWHh4dQnsE0LEpUFXYEkcAvXS4iIqKhh9UgJCYmYuXKlVi3bh10dXUxYcIEaGho4N9//4WKigp2795d75impqbIysqCoqIi9PX1YWRkhL59+wKoLFY/e/YsLTEGVKabVImOz58/c6936tQJZmZmQhEd4kxZWRlevnyJz58/o0OHDujRowdju4fFxcUIDg7Gvn37UFJSAnV1dcydO5dvmkhtVHfGqxJj/Jzxjh07hjt37gh1/MJEVVUVcXFxkJOTg46ODmxsbGBpaYkXL17A2NgY6enp9YqXkpICKysrGBsbw87ODp07d+a+lpOTA39/f5w7d04obRxcXV3x6NEjbNmyBc+fP4eHhwd8fHwQGxuLf/75R2gGHkzMv1paWvD19YWamhqcnJxQUFCAAwcOIDs7G7Nnz8bt27eFMnZhc/HiRRw8eBB2dnbo0aMHJCQkeF6v6ulFEA4cDgczZ87Ep0+f6qzBDAoKEkrqL0G0qKmp4c8//4SJiYlQWkM0BKWlpbh27RrOnz+PixcvomfPnjAyMgKbzaY9Hzx69AivXr3C6NGjISMjgytXrkBaWhqamppCGj2zEDHWBGAiXa6p8ODBA0hKSkJJSQm3bt3CkSNH0LVrV6xcuZLv7uHPUFJSQu/evbnpawMGDOC+JiwxVoUoRQeBlw8fPuDcuXM4d+4csrOzoa6uDhMTE7x79w7BwcGYMmVKncXy1UlPT8esWbMAgNvo+UdatmyJ+fPn08qbZxozMzNMmzYNXbt2xaJFixAXFwcFBQV4e3sjJSUFJ0+erFc8a2trKCoq8m3jUIWzszNyc3Npt3H48uULnJycoK2tjVmzZmHhwoW4evUqpKSk4OnpCTabLXBspudfYYtgUZGYmAhHR8caRiZVjc7p1rhERkbW+bqxsTGt+OJIUVERPD09ERkZWWsNZl2mM4TGS2RkJKKionDz5k106dIFxsbGMDExabRpyj+Dw+Hg9OnT2L59Oze1WFiUlpbi0aNH6N27t9icJhIxJsbwS5djs9nYs2ePUEUB4f94/vw5IiIiEBkZiQ8fPnB3doyMjGBsbEzedzHn7NmzOHv2LFJSUtChQwcYGxvDzMyMJ53i1KlT2Lx5s0CnWOLqjAdUpimuWLECpaWlYLPZ8Pb2xpYtWxASEgJfX99aLbVrQ1NTE4GBgXU2T87MzMTixYuRnJxMd/g8UBSFrKwsdOrUCSwWi+dU7lcR1fwrbBEsKsaOHYsBAwZg+vTpfAUAXaFaW+1Js2bNICcn1+jaFYiK4uJilJeXM1qDSWg4cnNzER0djaioKDx8+BDq6uowNTUVmzryDx8+4MKFC4iLi0NaWhrk5eVhaGgIW1tbgWO+ffsWa9aswapVq9CvXz9MnToVT548Qbt27XD06FGeTfPGChFjYooo0uXEHSatlSsqKnD16lWcPn0aly5d4lopL1myBFZWVnwtlwmNH2VlZejp6cHMzAyjR4+ukVoFVJ50Xbp0CX/99ZdQnikOznhV5Ofn4/3799yFcEZGBlq1asV1sqoP6urqiIyMrNM59OXLl5gyZQrt058BAwYgOTm5xvv86tUrTJo0qd7CWpTzr7BFsKhQVVVFVFSUyHbuy8vL8ezZM2zYsAHm5uaYOHGiSJ7bmKheg6msrMy9LuwaTELDU1paivDwcOzYsQPfvn1rlG6KVbx//x7x8fGIi4vDnTt30KNHDxgaGsLQ0FAohh4rV67E27dvsXPnTty+fRtubm7w9/dHREQE3rx5g4CAACF8F8xCDDzElCo3MGNjY2hra3MXAoT/g0lrZQkJCejq6kJXVxcFBQU4d+4czpw5g3379uHIkSOYPHkyNm7cKNRnEpjH29sb48aNq9Hzq6SkBJcvX8aECROgrq4ucA2TuDrjVSErK4vPnz/jwoULkJCQwKBBgwS2b2e6jcOpU6dw7tw5AJUnYcuWLavxc/3w4QPatm1b79iinH91dXWRlJTEI4LZbDamT58ukAgWFSNHjuQuvESBpKQkFBUV4ezsDFtb299OjKWkpMDBwQHGxsY1nHhdXFzg7++PVatWCaUGk9BwpKamIioqCnFxcSgvL4eBgQGP0Udj4ujRo4iPj8e9e/fQrVs3GBoaYs2aNTzN4IXBzZs3ERgYiD/++APe3t7Q0dGBuro6ZGVlG+178yNEjIkp8fHxiIiIQHBwMLZv386TLkeohGlr5Srat28PS0tLWFpa4uHDh4iIiEB0dDQRY2KInZ0d3xOUJ0+e4O+//8aECRNoxRdXZzygsjbK3t4eV69e5da9sVgsGBkZYcuWLZCRkalXPKbbOIwfPx5paWncr+Xk5GpsyvTr10+g2iJRz7/CFMGiYtiwYVi/fj0uX76Mnj17QkqKd7nBVH2khIQEPnz4wEjsxsyhQ4dgYWHBtwZTUVERW7ZsAQDs37+fdg0mQfRs374dMTExePfuHYYPHw5nZ2cYGBg06tTTrVu3QlpaGjo6Ohg8eDCAyubjly5dqnEvnfmgtLQU7dq1A0VRuHHjBuzt7QFUZjD9OO80VkiaophD0uVqx8DAANu3bxf6Lkx1ysvLkZ+fjzZt2qBZs2bc62/fvoWsrGyjnigJlRw9epTbd67KXIAfQ4YMQVhYGK1niaszHlBpppGamop169ZBTU0NFRUVSE9Ph5ubG/T19eHk5FSveOXl5ZgzZw4eP35cZxuHY8eO0f47qu5imZeXh9TUVHTs2BFDhw6lFVcU86+wRbCoGDt2bK2vff/+HdevX6cVn5+BR2FhIcLDw9GuXTsEBwfTii9uNGQNJoF5xo8fzzXtEDRbQNTUNQdUh8Vi0eoFOHv2bIwcORKdO3fGpk2bcOnSJXTo0AHr1q3D69evxWIuIGKsCVE9XS4rKwstWrT4rdPlmLRWjo2NRVBQEDIyMrgLpH79+mHOnDmYOnUqli9fDhUVFdjY2ND6HgjMU1ZWhujoaFRUVMDFxQUuLi48i2gWi4WWLVtCQ0NDoJS26oirMx4ADB8+HPv27cPw4cN5rl+/fh2rV68WaHHNdBuHffv2ITAwEOHh4ZCXl8edO3dgY2ODoqIiAJVNvvfv3y+UTROm5l9hi+CG5PHjxwgNDUVUVBRu3bpFKxa/WhMpKSmoqalhw4YNjTqFkwlEWYNJIDQmMjIysHjxYuTn58PGxgb29vbYsGEDYmJi4O/vDxUVlYYe4k8hYqyJUj1d7ubNmw09nAaBKWtld3d3HDt2DJqamtDU1ISsrCwKCgqQkpKC5ORkDB06FDk5OYiLi0O7du2E8a0QRMSZM2fAZrMZO20QV2c8oFK4HD16tMYi+PHjx5g9ezatxTUTbRzCwsLg7u6OefPmYdGiRWjdujUMDAxQWFiIo0ePok2bNlixYgVGjRqFlStX0nrWjwhz/mVCBIsSDoeDuLg4hIaG4s6dO2CxWBg/frxAfR4JtWNmZgZLS0tMmTKl1nvOnj0Lf39/REVFiXBkBEGxtLTE3r170bZtW1haWtZ5Lx1DsqZARUUFCgsLuRumT58+haysrEAtjBoC8UimJPyUH9PlBg4ciIEDB8La2hrfv3//LdPlPDw8oKGhUau1siAkJCQgLCwMBw8erOFiZmNjg8TERCxfvhyLFi0iQkwMMTExwevXr3Hv3j2+Dpx0exetXLmS64w3ceJEKCgo8DjjNWYsLS3h5uYGHx8frjV/YWEhdu3a9dOFws+QkpJCr169hDFMLidPnoSTkxNmz54NoDJF69mzZ7Czs+M6HS5ZsgRbt26lLcaYnH8lJSX5pjt27tyZmxbZGHn+/DlCQ0Nx5swZFBQUgMViwdTUFIsXLxaaqUdxcTG+fv2KNm3a/Pb9s5iuwSSInu7du3Mzerp160b6jNaBhIQET+ZKr169wOFwkJaWRjsdXRSQkzExh6TL1Q4T1srz5s3DsGHDai029fb2RmhoKJSUlHDs2DGhPZcgGsLDw7Fx40aUl5fXeE0YjWoB4drDi5JZs2YhMzMTEhISUFBQgJSUFJ49e4aioqIaCwU6+f/CQk1NjVt3BgB+fn7YsWMHzpw5w33vX758CSMjI2RmZgr0DFHMv/v27UNycnINEezo6IgBAwY0qkbh5eXluHDhAsLCwpCSkgJJSUmMGjUKbDYbzs7OiIyMpG35X1hYiMOHDyMmJgYvX77kXpeXl8fkyZNhZWX1WwozUdZgEhoXnz9//q03f+/fv4+1a9ciOzsbFRUVNV5vzLb/VRAxJsaQdLm6WbRoEdhsNiZPniy0mBoaGggODq7VyprNZmPJkiVwc3NDSkqK0J5LEA1jx46Fnp4e7Ozs0Lp1a8ae8+zZM2RnZ4uNMx4A7N2795fvbQwCQU1NDZGRkZCXlwdQOR/cu3ePJ23w0aNHsLS0FCjFUlTzrziJ4FGjRuHr16/Q0NCAvr4+9PX1ud+7MPqv5efnw8LCAm/fvoW+vj769euHtm3b4uvXr3jw4AEuXryIHj164Pjx47+leRXTNZgE0fLXX3/Bzc0NLVu2rPWemzdvwtHREUlJSSIcWeNi9uzZKCkpgZmZGbZs2QInJye8ePECISEh8PLygqGhYUMP8aeQNEUxhaTL/RwmrJU5HE6dE2N0dDRev36N0tLSescmNDwfP36ElZUVY0JMXJ3xgMYhsOpDv379kJ6eDnl5eXz58gUpKSkYN24czz3nz5+v032uNkQ5/2ppaUFLS4tWDFHx9etXdOzYEd26dUP79u2FfkLl4+ODiooKxMTE8N3AePfuHWxsbBAQEABbW1uhPlsckJGRgYODA+zt7YVeg0kQPffu3YOJiQl27dqFAQMG8LxWVlaGnTt3IiAgAAMHDmygETYOHj58iMDAQAwZMgQRERHo168fZs2aBTk5OYSHhxMxRmCOkJAQLFq0qMZCoIq7d++idevWPH12fjdOnDgBWVlZ3L17F3fv3uV57fv37wItLuXl5etsRMtisZCWlsZNjSKIFwMGDMCTJ0/wxx9/MBJ/8+bNePr0KQ4dOlTDGW/Hjh2N2hmPn5V4dejW0wmb2bNnY/369cjKysKdO3fA4XAwd+5cAJX1M1FRUTh8+DA2b95c79iinH/FSQQnJycjNjYWp0+fxokTJ9CqVSuMGzcORkZGQhEDSUlJWLduXa0nyXJycrC1tcW2bdt+SzFWBRM1mATRExkZCRcXF5ibm8PBwQEWFhYAKs0p/vrrL/z7779YtGiRWM0RTFBRUYHOnTsDqFyjZWdnY9iwYRg3bhwOHjzYwKP7NYgYE1MePXqENWvW1Pr6pUuXsGHDBri5uYlwVI2Lf/75p8a16tbKgsBms7F7927o6Ojwden5+PEjdu/ezV30EcSLBQsWYNOmTXj58iV69+5d46TqR0e7+pKYmFjDGW/MmDGQkZHB6tWrG7UYq21szZo1g5ycXKMTY5MnTwaHw8GJEycgISGBnTt3YsiQIQCAgwcPIjw8HDY2NnW6z9WGKOdfcRLBrVu3xvTp0zF9+nTk5OTg1KlTiIqKwrlz58BisXD06FHY2NhwU0frS25u7k9PMpWUlPDmzRuB4hMIjYnWrVtj9+7dCA4OhpeXF27evImRI0di+/bt+N///oeQkBCoqqo29DAbHHl5eaSlpWHixIno3bs3twb469evfI24GiNEjIkpJF3u16nNWlkQLC0tcf78eRgbG8Pa2hrq6urcIunbt28jICAA8vLyXAc3gnhR5arH77REGAYe4uqMB1QKkOqUl5fj2bNn2LBhA8zNzRtoVHUzdepUTJ06tcb1RYsWYcWKFZCVlRUorijnX3ETwVUoKirC0dERq1evxuXLl3HmzBlERkYiIiICWlpa8Pf3r3fM0tLSn9Y8NW/evNH/LREI9WHOnDkYMGAA5s2bh4sXL0JTUxO+vr6/pVENP+bMmQMXFxcAwIQJEzBlyhQ0b94c6enpYtFjDCBiTGwh6XI/hwlrZRkZGQQFBcHT0xPbtm3jWWzJyMjA1NQUDg4OkJSUFNa3QRAhTBsgMGkPL2okJSWhqKgIZ2dn2NraYuLEiQ09pF+Gn/V3fRDl/CuOIrg6kpKSGDduHMaNG4e8vDycPXsWERERDT0sAkFsSEpKgqurK9q1a4ehQ4ciISEBu3fvhr29PaSlpRt6eA3OtGnTuD3FFBUVsXXrVhw6dAjdunXD2rVrG3p4vwRxUxRT/P39ERYWhpMnT9aaLjdjxgzMnTtX7BZ5dBCFtTJQ2d+mvLwcGRkZyM/PR4cOHSArKwsFBQXiVtUE4HA4ePXqFXr27AmKooT2gSdOzni/yqNHj2Bubo579+419FBERmOYfx8+fAhbW1skJCQwEr8xoqSkBGtr6zpPBL59+4YjR46IhZ01gVAXHA4HW7duxfHjxzFmzBh4eHigQ4cOiIyMxKZNm6CgoIDt27f/lvWB4ubu+zOIGBNTOBwOZs6ciU+fPtWZLhcUFPRbndIwba0MVKYgbd68GX5+flBWVuZet7a2xv379+Hu7g59fX1azyA0DBRFYfv27QgODkZpaSni4+Oxc+dOtGjRAhs2bKAtysT5A4Rf7VJhYSHCw8PRrl07BAcHi35QDURjmH9/RxE8duzYX76XX80wgSBOGBkZ4c2bN3BwcMCsWbN4Xnv69Cns7Ozw/PlzuLi4YNq0aQ00yoZBSUkJEhISkJOTq/M+FoslFhubRIyJMUVFRfD09ERkZGSt6XK/W06xiooKOnbsCF1dXWhra2P06NFcEwZhiLGUlBRYWVnB2NgYdnZ2XAcfAMjJyYG/vz/OnTuH4OBgqKur0/5+CKIlKCgIfn5+sLOzw6ZNmxAVFYXMzExs3LgRM2bMgJ2dXUMPscGoapRcHSkpKaipqWHDhg2Nvmm1sBHV/EtEMIHwe2JiYgJvb+9a59aqk7MTJ078difB69atQ0JCAlq0aAE2mw02m833M0pcIGJMzCHpcrwUFhZyrZXv3bvHY628fPly2mmK1tbWUFRU5BaL8sPZ2Rm5ubnw8/MT+DmEhoHNZmPVqlXQ19eHmpoazp07hx49eiAhIQFbtmyhvdsuTs54hJ8jivmXiOBf5/Pnz4iIiEB4eDjOnz/f0MMhEGhRWlrKzcZITU2FiooK3+yMxMREgU3JxJny8nLcvHkTsbGxSExMRMeOHbnCTNz8EogYE2NIulzdVLdWzs3NBYvFgpmZGS1rZU1NTQQGBtZpr5yZmYnFixcjOTlZ0KETGggVFRXExMTgjz/+4BFjL1++hJGREdcyV1Bq27mrcsaLj4+nFZ9piouL8fXrV7Rp0+a3O3X/ETL/Nh7S0tIQGhqKCxcuoKSkBAMHDiQmIYQmhaamJvz9/TFo0KCGHkqjpLS0FNeuXcP58+dx8eJF9OzZE0ZGRmCz2ejWrVtDD++nEDdFMSUlJQUODg4wNjau4Qzm4uICf39/rFq16rdOl2PCWrmkpOSnO97t27dHcXGxoMMmNCDdu3dHZmZmjabPV65cEdiBszri6IxXWFiIw4cPIyYmBi9fvuRel5eXx+TJk2FlZfXbCTNRz79EBNeksLAQkZGRCAsLw5MnTwBU1gzb2NhgxIgRDTw6AkG4dOjQAV+/fm3oYTRapKWloaenBz09PXA4HJw+fRrbt2/Hjh07xCKFk5yMiSkkXU4wqlsrC9L42czMDJaWlnU2ij179iz8/f0FbixNaDhOnz6Nbdu2YenSpdixYwdcXFzw4sULBAcHw8nJCTNnzmTkuY3VGS8/Px8WFhZ4+/Yt9PX10a9fP7Rt2xZfv37FgwcPcPHiRfTo0QPHjx/n2z+tqSKK+ZeIYP5kZGQgLCwMsbGxKC4uRt++fcFms7Fnzx6hGDQRCI0RT09PHD9+HLq6upCXl0ezZs14Xm9shk8NwYcPH3DhwgXExcUhLS0N8vLyMDQ0hK2tbUMP7acQMSamkHS5hiEwMBCBgYE4ceIE315F79+/x8yZM2FmZoZly5Y1wAgJdAkLC8P+/fvx7t07AJU7kjY2NrCysmLsmY3VGW/Dhg1ISUlBQEAAunbtWuP1d+/ewcbGBuPHjxeLDzxhwfT8S0Qwf0xNTZGVlQVFRUXo6+vDyMgIffv2BSA8t1wCoTFSl5Po9+/fcf36dRGOpvHw/v17xMfHIy4uDnfu3EGPHj1gaGgIQ0NDsTL0IGmKYgpJl2sYLCwsEB8fj4kTJ8LMzAxqampcS+v09HScOXMGCgoKsLa2buihEgTg/fv3MDc3h7m5OfLy8kBRFDp27Ci0+HU54w0ZMkRozxEWSUlJWLduHV8hBgBycnKwtbXFtm3bfisxxvT86+Pjg4qKCsTExNQpggMCAn6r9/3hw4fo3bs3jI2Noa2tzRViBEJTh5951OPHjxEaGvpbZuEcPXoU8fHxuHfvHrp16wZDQ0OsWbNGbGvqiBgTU3r16oU7d+6gZ8+etd6Tnp6O7t27i3BUTR9JSUkcPXoUu3btwunTp3H06FHua506dcLs2bOxZMmS39LJsimgq6uLQYMGYfz48Rg3blydJx+C4OTkVONadWe8xkZubu5P3wMlJSW8efNGRCNqHDA9/xIRzJ/4+HhEREQgODgY27dv5xbpGxkZNfTQCASRwOFwEBcXh9DQUNy5cwcsFuu3dFLcunUrpKWloaOjg8GDBwMALl26hEuXLtW4VxxSOEmaophC0uUanrKyMrx8+RKfP39Ghw4d0KNHD7BYrIYeFoEG//77L5KSkpCUlIS7d+9CTk4O48aNw7hx4zB8+HBISEg09BBFipKSEpKTk+s8HczNzYWOjo5YFEkLC6bn38GDByMuLq5OMffq1SsYGhrSdvgURyoqKnD16lWcPn0aly5dQllZGQBgyZIlsLKy+q1SNwm/B8+fP0doaCjOnDmDgoICsFgsmJqaYvHixUIxlxI3frUBPGn6TGCU8vJyzJkzB48fP64zXe7YsWPklIZAEIAvX77g6tWruHDhAhITE9GmTRvcvHmTdlxxcsYjYow/TM+/5H3/dQoKCnDu3DmcOXMGWVlZaNGiBSZPnoyNGzc29NAIBFqUl5fjwoULCAsLQ0pKCiQlJTFq1Ciw2Ww4OzvT7ptKaDwQMSbGcDgcbrrc58+fudc7deoEMzMzki5HIAgARVG4f/8+bt68iZSUFKSlpYGiKAwdOhSHDx8WKKa4OuMpKSnB2tq6zrF9+/YNR44c+e1EAZPzLxFjgvHw4UNEREQgOjpaKBsnBEJDMmrUKHz9+hUaGhrQ19eHvr4+2rVrB4AY1jQ1iBhrApB0OQJBOCxevBhpaWn4/v07Bg0ahJEjR0JLSwtqamqQkZERKKY4O+P9aioIwL/A/HeAifmXiOBfo7y8HPn5+WjTpg2P1ffbt28hKytLNiMJYo2Kigo6duwIXV1daGtrY/To0dzPISLGmhbEwKMJICUlhV69ejX0MAgEsefOnTsoLCyErq4u9PT0oKmpWadJw68gzs54v6vAqg9MzL/dunXD+fPnf3pfbQYfTZ3Y2FgEBQUhIyMDVfvJ/fr1w5w5czB16lRs3rwZKioqsLGxaeCREgiCk5ycjNjYWJw+fRonTpxAq1atMG7cOBgZGZEN9yYGORkjEAiE/w9FUXjw4AGuX7+O5ORk3LlzB506dYKGhgY0NTUxadKkesfU09PDunXroKenV+s9iYmJ2LZtG+Lj4+kMX+R8/vwZERERCA8P/yXxQCDQxd3dHceOHYOmpiY0NTUhKyuLgoICpKSkIDk5GUOHDkVOTg7i4uK4KV0EgriTk5ODU6dOISoqCrm5uWCxWDAzM4ONjQ3k5eUbengEmhAxRiAQCLXw6dMnHDhwAGFhYSgtLRUoJawpOuOlpaUhNDQUFy5cQElJCQYOHIiIiIiGHtZvw+8qghMSEmBvb4+9e/dCV1e3xuuJiYlYvnw5Fi1aBDs7uwYYIYHALOXl5bh8+TLOnDmDy5cvo6KiAlpaWvD392/ooRFoQNIUCQQC4f9DURQyMzORnJyM69ev4+7du2jXrh0mT55c58lWXZSWlv60dqV58+Zce+7GSmFhISIjIxEWFoYnT54AqCwwt7GxwYgRIxp4dL8H/ETw70RISAgWLVrEV4gBwN27d9G6dWukpaWJeGQEgmiQlJTktlvJy8vD2bNnyUZYE4CcjBEIhN+aAQMG4Nq1a+jYsSNGjBiBr1+/om/fvtDT08O4ceMwZMgQWvHF3RkvIyMDYWFhiI2NRXFxMfr27Qs2m409e/aQAnIRQETw/6GhoYHg4GD07duX7+tsNhtLliyBm5sbUlJSRDw6AoFAEAxyMkYgEH5rqu9HrVy5Enp6enWmFApCQEDAT53xGiOmpqbIysqCoqIi5s2bByMjI+5CeM+ePQ08uqYNPxFsa2uLPXv2wNHR8bcUwRwOBy1btqz19ejoaLx+/RqlpaUiHBWBQCDQg4gxAoFA+P9YWFgIPaY4O+M9fPgQvXv3hrGxMbS1tWs9kSAIFyKC+SMvL4/09PRaN0tYLBbS0tKgoKAg2oERCAQCDYgYIxAIvz3nz59H69atf3qfsbFxvWOLsz18fHw8IiIiEBwcjO3bt6Nnz54wMjKCkZFRQw+tSUNEMH/YbDZ2794NHR0dtG/fvsbrHz9+xO7duzF37lzRD45AIBAEhNSMEQiE3xolJaVfuo/FYjFS0yUOzngVFRW4evUqTp8+jUuXLnHNRpYsWQIrK6tG16xa3Hn+/DkiIiIQGRmJDx8+8IhgY2Pj37ZWj8PhYObMmfj06ROsra2hrq6Otm3boqCgALdv30ZAQADk5eURFBQESUnJhh4ugUAg/BJEjBEIhN+aXzHYYAJxtYcvKCjAuXPncObMGWRlZaFFixaYPHkyNm7c2NBDa3IQEVyToqIieHp6IjIykqc2TEZGBqampnBwcKizPpNAIBAaG0SMEQiE35rqbopM09Sc8R4+fIiIiAhER0fj5s2bDT2cJg0Rwf9HcXExysvLkZGRgfz8fHTo0AGysrJQUFD4aRsJAoFAaGwQMUYgEH5rRHEy1lTs4cvLy5Gfn482bdqgWbNm3Otv376FrKwsWQiLiN9ZBEdHR2Pz5s3w8/ODsrIy97q1tTXu378Pd3d36OvrN+AICQQCoX5INPQACAQCoSExMTHhERbCxtTUFObm5rh37x7mzZuHqKgoREVFYfHixYw9U9jExsZixowZGDx4MHR0dKCqqoopU6bg1KlTAIDNmzcjODi4gUfZdCkvL0dubi5KSkoAAAMHDoSrqyvOnDmD79+/N/DoREdKSgocHBygp6eHLl268Lzm4uKCsWPHYtWqVUhPT2+gERIIBEL9ISdjBAKBwCBKSkro3bs3TE1Noa2tjQEDBnBfGzRoUKM/GXN3d8exY8egqakJTU1NyMrKoqCgACkpKUhOTsbQoUORk5ODuLg4tGvXrqGH26SIjY1FUFAQMjIyuP3w+vXrhzlz5mDq1KlYvnw5VFRUYGNj08AjFQ3W1tZQVFSEi4tLrfc4OzsjNzcXfn5+IhwZgUAgCA4RYwQCgcAg4uyMl5CQAHt7e+zduxe6uro1Xk9MTMTy5cuxaNEi2NnZNcAImy5EBNdEU1MTgYGB6NevX633ZGZmYvHixUhOThbhyAgEAkFwSJ8xAoFAYBB5eXnY2dnB1taW64zn7++PAwcOAKg8/WisznghISFYtGgRXyEGAHfv3kXr1q2RlpYm4pE1bRISEhAWFoaDBw/WeO9tbGx4RPDvIsQAoKSk5Kd1ie3bt0dxcbGIRkQgEAj0ITVjBAKBIAIkJCSgq6uL3bt34+rVq3B2doaSkhL27duH0aNHY/369Q09xBo8evQIEyZMqPX1S5cuYcOGDXj8+LEIR9X0ISKYP7169cKdO3fqvCc9PR3du3cX0YgIBAKBPkSMEQgEgohp3749LC0tcebMGURERMDMzAzx8fENPawacDgctGzZstbXo6OjoaqqytPviUAfIoL5M3nyZPj4+OD9+/d8X3///j18fHxgYGAg4pERCASC4JA0RQKBQBAhP9rDDxw4EAMHDoS1tTW+f//eqOzh5eXl6zxpYLFYSEtLg4KCgmgH1sT5FRH8+vXr304EW1hYID4+HhMnToSZmRnU1NTQtm1bFBQUID09HWfOnIGCggKsra0beqgEAoHwyxAxRiAQCCLgZ854mzdvbnTOeGw2G7t374aOjg7at29f4/WPHz9i9+7dmDt3rugH14QhIpg/kpKSOHr0KHbt2oXTp0/j6NGj3Nc6deqE2bNnY8mSJY1qQ4NAIBB+BnFTJBAIBIYRV2c8DoeDmTNn4tOnT7C2toa6ujr3JOL27dsICAiAvLw8goKCICkp2dDDbTL4+/sjLCwMJ0+erFUEz5gxA3PnzoWlpaXoB9gIKCsrw8uXL/H582d06NABPXr0AIvFauhhEQgEQr0hYoxAIBAYRNzt4YuKiuDp6YnIyEietDgZGRmYmprCwcEBLVq0aMARNj2ICCYQCITfByLGCAQCgUHmzZuHYcOGYfny5Xxf9/b2RmhoKJSUlHDs2DERj+7XKC4uRnl5OTIyMpCfn48OHTpAVlYWCgoKJCWMIYgIJhAIhN8DIsYIBAKBQTQ0NBAcHIy+ffvyfZ3NZmPJkiVwc3NDSkqKiEf3c6Kjo7F582b4+flBWVmZe93a2hr379+Hu7s79PX1G3CETRciggkEAqHpQ6ztCQQCgUHE2R4+JSUFDg4O0NPTQ5cuXXhec3FxwdixY7Fq1Sqkp6c30AibLtHR0Rg7diyePXsGLS0tsNlsaGpqYtu2bdDV1UVCQkJDD5FAIBAIQoCIMQKBQGCQKme82mjMzniHDh2ChYUFPDw80LlzZ57XFBUVsWXLFkyePBn79+9voBE2TYgIJhAIhN8HIsYIBAKBQars4QsKCvi+XmUPb2xsLNJx/QoPHz7E1KlT67xn1qxZePjwoYhG9HtARDCBQCD8PhAxRiAQCAxiaWmJtm3bwtjYGMHBwXjw4AFevnyJzMxMBAQEwMTEBHJycpg9e3ZDD7UGJSUlP61Nat++PYqLi0U0ot8DIoIJBALh94E0fSYQCAQGkZGRQVBQEDw9PbFt27ZanfEao0V5r169cOfOHfTs2bPWe+pqTkwQDCKCCQQC4feBiDECgUBgmFatWsHZ2RkODg5i5Yw3efJk+Pj4QENDo0btEgC8f/8ePj4+MDMza4DRNV2ICCYQCITfByLGCAQCgWGq28NraWlxrzd2e3gLCwvEx8dj4sSJMDMzg5qaGrf5cHp6Os6cOQMFBQVYW1s39FCbFEQEEwgEwu8D6TNGIBAIDJKSkgIrKysYGxvDzs6Ox5AhJycH/v7+OHfuHIKDg6Gurt6AI+UPh8PBrl27cPr0aXz+/Jl7vVOnTjAzM8OSJUsa7cmeuFJeXo45c+bg8ePHdYrgY8eOkfeeQCAQxBwixggEAoFBrK2toaioCBcXl1rvcXZ2Rm5uLvz8/EQ4svpRVlaGly9f4vPnz+jQoQN69OgBFovV0MNqshARTCAQCL8HRIwRCAQCg2hqaiIwMBD9+vWr9Z7MzEwsXrwYycnJIhwZQRwgIphAIBCaNqRmjEAgEBiEOOMR6CAlJYVevXo19DAIBAKBwBCkzxiBQCAwSJUzXl0QZzwCgUAgEH5PiBgjEAgEBqlyxnv//j3f16uc8QwMDEQ8MgKBQCAQCA0NqRkjEAgEBiHOeAQCgUAgEGqDiDECgUBgGOKMRyAQCAQCgR9EjBEIBIKIIM54BAKBQCAQqkPEGIFAIBAIBAKBQCA0AMTAg0AgEAgEAoFAIBAaACLGCAQCgUAgEAgEAqEBIGKMQCAQCAQCgUAgEBoAIsYIBAKBIFKcnJzQv3//Ov+bM2cO7ee8evUK/fv3R0REhBBG/Xvj5OSEsWPHNvQwCAQCockh1dADIBAIBMLvxdKlSzFjxgzu1/v27cPDhw+xd+9e7rXWrVs3xNAIBAKBQBApRIwRCAQCQaT07NkTPXv25H7doUMHyMjIQFVVteEGRSAQCARCA0DSFAkEAoHQKDl58iRMTU2hqqqKIUOGYMqUKTh//jz39YqKCuzcuRNjx46FsrIyxo4di+3bt6O0tJRvPIqi4OzsjCFDhuDatWsAgBcvXmDx4sUYOXIkVFRUYG5ujqSkpDrHNXbsWHh4eGDu3LkYMmQI1qxZAwAoKCjAunXroKWlhcGDB2P69Om4ceMGz7/t378/Tpw4AScnJwwdOhQjRoyAu7s7vn//Dk9PT2hoaGDkyJFYs2YNSkpKuP+upKQEvr6+MDAwwODBg/Hnn3/i0KFDqKioAAAcOHAAysrKPE3FAeDo0aMYNGgQPn36BAB48+YN7O3tMWLECKioqGDu3Ll4+PAhz7/5/PkznJ2dMWLECAwfPhzbtm3jPodAIBAIwoWIMQKBQCA0OkJCQrBu3TqMHz8eBw8ehLe3N2RkZLB69Wq8e/cOAODn54cTJ05g2bJlCAgIwMyZM3H48GHs37+fb0x3d3dER0dj7969GDVqFCoqKrBo0SIUFxfDy8sL+/btQ/v27bFkyRI8f/78p+MbPHgw9u3bh6lTp6KkpARz587FxYsXYWdnh71790JOTg4LFiyoIci2bdsGGRkZ7N27F8bGxggODoaxsTHevn0Lb29vzJkzB6dOnUJwcDCAShG5ePFi+Pv7Y9q0aThw4AAMDAywa9curF+/HgAwadIklJWV4cKFCzzPiomJwahRo9CxY0fk5eVhxowZePDgAdauXYvt27ejoqICs2fPRk5ODoBKgbtgwQIkJSXB0dERW7duRXp6OmJjY+v/QyQQCATCTyFpigQCgUBodLx8+RLW1tZYunQp91r37t1hamqKtLQ0sNls3Lp1C8rKyjAzMwMAjBgxAi1atECbNm1qxNu+fTvCwsKwd+9ejB49GgDw6dMn/Pfff1i6dCl0dXUBAEOGDMHevXvB4XDqHF+3bt2wevVq7tfh4eF49OgRwsPDoaKiAgAYPXo05syZA29vb5w+fZp7b58+fbBp0ybumE+ePInS0lJ4e3tDSkoKo0aNQnx8PNLT0wEAV65cwfXr17Fjxw6w2WwAgLa2Npo3bw4fHx9YWlqib9++GD58OKKjozFt2jQAlad+GRkZ2LlzJwAgMDAQBQUFOHHiBLp3784do5GREXx8fLB7925cuXIFGRkZ8PPz475PmpqaxLyDQCAQGIKcjBEIBAKh0eHk5ITVq1fjy5cvuHv3Ls6ePYuQkBAA4AqlkSNHIjk5GbNmzYK/vz+ePHkCCwsLTJkyhSdWSEgIDh06BDabjTFjxnCvd+rUCX369MHatWvh6OiIqKgoVFRUwNnZGX379q1zfAMGDOD5+saNG+jcuTMGDRqEsrIylJWVoby8HHp6erh//z5P+qCamhr3/yUlJSErK4tBgwZBSur/9kfbt2+Pr1+/AgBu3boFKSkpGBgY8Dxz8uTJ3Nervr59+zY+fvwIoPJUrHXr1lwhdePGDQwYMABdunThjlFCQgKjR4/G9evXAQCpqamQlpaGjo4O9zktW7bkilUCgUAgCBdyMkYgEAiERseLFy+wbt063LhxA9LS0ujduzeUlJQAVKbtAcCCBQvQqlUrnD59Gt7e3ti2bRv69u0LV1dXaGhocGM9evQIo0aNQnR0NObOnYuBAwcCAFgsFgICArB//34kJCQgMjIS0tLSGD9+PDZu3Ih27drVOr6WLVvyfF1QUICPHz9i0KBBfO//+PEjNx4/p8gf41Xn8+fPkJWVhaSkJM/1zp07AwBXtBkYGMDNzQ3nz5+HpaUlYmJiMGHCBDRv3pw7xufPn9c6xuLiYnz+/Bnt27cHi8Xi+ywCgUAgCBcixggEAoHQqKj4f+3cS0iUaxzH8a+lgwRdphmwGhcTSGTgIqLpQhdoU+QiGiSCKVGJaCNSglSMJdG0ydnYnQkVS7rZYlqUi1p00UUERRAFBe4KAzUsAsM6ZxENDBUcPYczLb4fmM37PvO8//fZ/Xj/z/PtG3v37qWkpIS+vj4qKyspLi7mzZs3ZLPZ3LgZM2aQSCRIJBKMjIxw//59zp8/T2NjIwMDA7lxTU1N1NbWUl1dTTKZ5MaNG7lgU1ZWRltbG0ePHuXVq1f09/eTyWQIBoO5/Vj/xOzZs4lGo7S3t//yfnl5+TRXA+bOncvY2Bhfv37NC2Tv378HIBgM5mrYtGkTd+7cYfXq1bx+/ZrW1ta8GmOxGC0tLb98TiAQIBgM/vJZHz58mHb9kqTfs01RkvRHGRsbY2hoiJqaGqqqqnLtew8ePADIney3c+dOjh8/DkAoFCIej5NIJBgfH+fTp0+5+cLhMKWlpRw5coQXL17Q1dUFwNOnT1m7di3Pnz+nqKiIyspK9u/fz5IlS3j79u2Uao7FYrx7945QKERVVVXuNzAwwMWLF3/6qjXVuScnJ+nv78+7fuvWLQBWrFiRu7Zt2zaePXvGlStXWLRoEbFYLG+eoaEhFi9enFdjNpulr6+PmTNnsmbNGiYnJ7l7927uf1++fMkLt5Kk/45fxiRJf5RQKEQkEqG3t5cFCxYwZ84cHj58SE9PD/C9nQ5g5cqVdHZ2Eg6HWb58OcPDw3R1dRGLxZg/fz6fP3/Om3fjxo1s2bKFU6dOsXnzZpYtW0ZpaSktLS00NjYSDocZHBzk5cuX1NbWTqnmeDzO5cuXqa+vZ9++fSxcuJDBwUEymQy7du2ipKRk2uuxYcMGVq1aRTKZZHh4mKVLl/L48WMymQzbt2+noqIiN3b9+vXMmzePa9eusWfPnrx2w7q6OrLZLHV1dTQ0NBAMBrl9+zbXr1/n0KFDwPfDOtatW0cymWRkZIRIJEJPTw+jo6OEQqFpv4Mk6dcMY5KkP87Zs2dJpVIcPHiQQCBARUUF586d48SJEzx58oTdu3fT1NREIBDg5s2bnDlzJtem19zc/Nt5Dx8+zKNHj2htbaW7u5vOzk7S6TSpVIrx8XGi0SjHjh0jHo9Pqd5Zs2bR29tLOp3m5MmTfPz4kUgkQnNzMw0NDf9qLYqKirhw4QIdHR10d3czOjpKeXk5Bw4coL6+Pm9scXEx1dXVXLp0KXfAxw9lZWVcvXqVdDpNW1sbExMTRKNRUqkUNTU1uXGnT5+mvb2djo4OJiYm2Lp1Kzt27ODevXv/6j0kST8r+uvHTmhJkiRJ0v/GPWOSJEmSVACGMUmSJEkqAMOYJEmSJBWAYUySJEmSCsAwJkmSJEkFYBiTJEmSpAIwjEmSJElSARjGJEmSJKkADGOSJEmSVACGMUmSJEkqAMOYJEmSJBWAYUySJEmSCuBvT7Fv49L8oKcAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2MAAAL9CAYAAACmKr6OAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdeXhTZdoG8PskTbq3SfeNRYpSQESQooyCCoqO2zCKG4MgiDoCgqi4zCifOwgIiggqgqjoqIwrroziMjKyr0rZigVK9yVJ2+zL90d60pa2kNNmOUnv33XNNZKcnNwkbThP3vd9XsHlcrkgwXnnnYcFCxZg5MiRre778ccfMWvWLOzcuVPKKYmIiIiIiLochdQHDBw4EG+++SZsNluL2+12O95++22cc845PgtHREREREQUrgSpI2O7du3CbbfdBq1WixEjRiA5ORk1NTXYuHEjqqur8c4777AgIyIiIiIiOg3JxRgA7Nu3D6+++ip27NgBnU6H+Ph4DBkyBFOnTkXfvn39kZOIiIiIiCisdKgYIyIiIiIios6J6MiDXC4XCgoKYDQa0VYtl5+f3+lgRERERERE4UxyMbZnzx7MnDkTZWVlAOApxgRBgMvlgiAIKCgo8G1KIiIiIiKiMCO5GJs7dy4iIiIwd+5cZGRkQKGQ3JCRiIiIiIioy5NcjP3+++9YtGgRLrvsMn/kISIiIiIi6hIkF2PJyclQKpX+yBKydu7cCZfLBZVKFewoREREREQURDabDYIgYNCgQac9VvIcw3HjxuG1116D0WjsULhw5HK52mxkEgwulwtWqzXoeZhDXhnkkkMOGeSSQw4Z5JJDDhnkkkMOGZhDfhnkkkMOGeSSQw4Z5JJDDhnklEPM4m0OySNjR48eRWFhIS688EKceeaZiIqKanG/IAh46623pJ42pIkjYgMGDAhyEsBoNKKgoAC9e/dGTEwMc8gghxwyyCWHHDLIJYccMsglhxwyyCWHHDIwh/wyyCWHHDLIJYccMsglhxwyyCkHAOzdu9frYztUjOXl5Xn+fHLVJ4dqlIiIiIiISO4kF2PvvPOOP3IQERERERF1KT7tS280GvHzzz/78pRERERERERhSfLI2IkTJ/DEE09gy5YtsFqtbR7DTZ+JiIiIiIhOrUObPu/YsQM33ngjduzYgejoaJx77rnYuHEjDh48iJdfftkfOYmIiIiIiMKK5GmKW7duxaxZs/DYY4/h+uuvR2RkJGbPno2PPvoI+fn5+P777/2Rk4iIiIiIKKxILsYaGhrQp08fAECvXr2wb98+AIBSqcS4ceOwadMm3yYkIiIiIiIKQ5KLsbS0NFRVVQEAevToAb1ej8rKSgCARqNBdXW1bxMSERERERGFIcnF2MUXX4wXX3wRO3fuRHZ2NjIyMrBq1SrU19fjo48+Qnp6uj9yEhERERERhRXJxdiMGTOQkJCAl156CQAwa9YsvPXWW8jPz8e6deswadIkn4ckIiIiIiIKN5K7KWq1WqxduxYVFRUAgOuuuw5ZWVnYtWsXzjnnHAwdOlRyCKfTiaVLl2Lt2rWoq6tDfn4+5syZg27durV5/KFDh7BgwQLs3r0bCoUC+fn5eOSRR5CVleU55t1338WqVatQWVmJs88+G4899hj69evnub+4uBhPP/00tm7dipiYGIwdOxb33nsvlEql5PxERERERERSdXjT57S0NM9/DxkyBFOmTOlQIQYAy5Ytw3vvvYenn34a77//PpxOJ6ZMmdLmPma1tbWYNGkSoqKi8M4772DFihWoqanBlClTYLFYAACffPIJ5s+fj5kzZ+Ljjz9GTk4OJk2ahJqaGgCAzWbDHXfcAQB4//338cQTT+Bf//oXXnnllQ7lJyIiIiIikkryyJjVasWaNWuwY8cOGAyGVvcLgoC33npL0vlWrVqFBx98EJdccgkAYPHixRg+fDjWr1+Pa665psXx3333HYxGI+bPn4+oqCgAwIIFC3DJJZdgx44dGDZsGF599VWMHz8e1113HQDgueeew2WXXYa1a9fi7rvvxrfffouSkhJ8+OGHSExMxFlnnYXq6mrMnz8ff//736FWq6W+LERERERERJJILsaeeuop/Pvf/8aZZ54JjUbT6n6XyyXpfPv370dDQwOGDRvmuS0hIQH9+vXD1q1bWxVjw4YNw7JlyzyFGAAoFO4BPoPBgOrqahQVFbU4X0REBIYMGYKtW7fi7rvvxrZt29C/f38kJiZ6jrngggtQX1+PgoICDBw4UNLfAQDgcsFhNrd5l6BQQNGswGvvOPfBApSRkR071mKBw2yGy2qFw2yGQ6E45bFo772SciwAZbP3Qjy2vRzNj3VarXA5nV6d93THKiIjIQiC+1ibDS6Hw52njRztHdvmedVqCI2P6+ixbWZQqSA0Tok97XmbH2u3w2W3d+jYk3M0P9blcMBps7V7XiEiAoqIiE4fe3IGQamEQqXy7rzNj3U64Wxj5NzbY5vncKpUTce6XHA2jrCf9rynO/Y0v/ctMkREBOwz4uTfZTGH02IBYmJOeayU8zbX1mdEWxkcZnOLDIH4jGgrR/N/xwLxGdFmhmZ/l0B9RrTK0SxDoD4jTuZsli+QnxEtMjR7zkB+Rpx8bHOB/IxofmyLDAH8jGjvWKfV2vJa5zTHdvYzovmxnvPabKd8jX39GeE59qR/w1td97VzrK8+I1plcDpPmcFfnxEn/y63eQ3czrG++oxo8/fe5Wr1O9MeycXYf/7zH9x7772YNm2a1Ie2qaysDACQmZnZ4va0tDTPfc3l5OQgJyenxW2vv/46oqKikJ+fj9LS0nbPt3//fs9zZmRktLofAEpLSztUjJkrKrDp8SfbvC/h3IHIffhBz593334HnJa239S4vnk4c84/PX/ee9dU2Ovq2jw2ptcZ6PPsU54//37vLFgbtx3Yc9KxUdnZ6LtwnufPBQ8+AvOJE22eV52Sgv4vL/b8+cA/58B45I82j42Ij8eA15d5/nzoqWdRX7Df8+fmORSRagxcvdLz58LnF8Kwa3eb5wWAQf96x/Pff7y4BLrNW9s99pw3V3g+dI8ufw01P//S4v7mOc5+7RWoEhIAAMdXrUbVf9rfqLzfkkWITE0FAJx491+o+OKrdo/Nmz8X0d3cP5ul//4YZR990m6Gs555ErG5vQAA5eu+RMl777d73t6P/wPx/foCACrX/wfFb77d7rG9Zj+AxMHnAgCqf/oZx15d0eoYMUfPmdOhveB8AEDtps0oemlpu+ft/vc7kXzxCACAfscuHFnwQrvH5kyagNTRlwMA6vYV4PDTz7WbIWvcLUi/9moAQEPhERx87P/aPW/GDX9F5tjrAQCm48XY/9Cj7R6bds1VyP7brQAAS2Ul9s24v83j9gBIuXwUuk2+HQBgMxjw293tf7YljbgIPe65G4D7AmfPpDvbPVZzfj7OuG+G5887b72t3QzB+Iw4WUFWJvq/ML/pzwH6jGhur1qNc98KzmdEc/UvvgAh3f2PaCA/I5qrffwfEBp/7wP9GSFSjf0rTL3cn1OB/owQpd54A9C3D0wmU1A+IwBAc+klwPA/wWQyBeUzAnB/RmTNcD+vyWTC7numB/wzIio7Gz2f/j9PhoLHnwz4Z4R4HWEymQAAh19YjIY9v7V5LOC/z4izX3sF9saL76Nvvg3dDz+2e6y/PiPE6wiTyQTH5q3Y8+zz7R7rr88I8TrCZDLBuf8A9jwzr91j/fUZ0fw6onb/AVjmLWx1DSzy12dEW9cR6nvvQZSXHeYlF2MKhQKDBg2S+rB2ib9QJ08NjIyMhF6vP+3j33nnHaxZswaPPfYYkpKScOTIkXbPJ64pM5vNSGi8GG9+PwDPMb4kjriJnM72v/FpMBpbHGt3tP+NhMlsbnGs9RTfHFislhbHWqzt/z2tNlvLY0/xjY/dYW95rNHY7rFOp6tl3vr6do8F0PJYQ9v/kIgOHDgAofE9t57m5+bQwYMQYmMBALba2lMee/jwYSga/2GynWYPvSNHjkBR785pa9x7rz1Ff/wBReN7YG9shtOeo0ePQtn45Yq9jS8omjt+/DhKot0/y/aS0lMee+LECZQ1vsaOdv5BFZWUlKJCPPb48VMeW1ZWhirx2KNHT3lsRUUFahqPdZ4oOeWxlZWV0InHVpz69a2uroZBPFanO+WxtbW1qG881tXQcMpjdXo9jOKxp/iWDAAMhroWP8OnIofPCNvJv/dB+IxwueTxGXH8+DEINe7f92B9RpSWlKBc/L0P0mcEABQVFQEI3mdEbW0tIhpzBOszoq7OAFVjhmB+RojvRVFRUdCuI5pnkMN1hMloavdYIDDXEXV1rZfsNBeI64jTCcR1xOkE5DqiRP7XEW0RXBLnFT7//PMoLy/HwoULPdMDO+Pbb7/FjBkzsHv37hZTD2fOnAmr1Yrly5e3+TiXy4WXXnoJy5cvxz333IP77rsPAPDbb7/hhhtuwFdffYXc3FzP8fPnz8cvv/yCzz//HH//+98RFRWFF1980XO/yWTCueeei6VLl+Lyyy+X9HfYu3cvXE4nzmin+6PUqQgdPdZpscBkMuHo0aPo0aNHi9dTEISWw+oWS7tTSlsd24GpQmazuc0cgZ6m2FaODk9TPN1wfTvHtpmhg+d12e0tpu20Orb5lIGTjj05h5RpioqICAheThlocexJQ/snZ2gxFeF00wB8eGzzHNGxsUGZptgiQ0xMwD4jTv69b54jttnUc39/RniVIcDTFMUcPc88EzGN0yUD8RnRZobevRHTeLEXqM+IVjlOnMAZvXohOjo6YJ8RJ7PYbDhaXIyePXsiKjIyYJ8RzZmtVhw7ccKdISoqaNMULQ4HioqK0LNnT6hPMf3J158RnmMFARan05MhUqEI2GfEyceaTCYUFRWhe3Y2ok6xzt+f0xTNZrNXGXz9GXHysSaTCX8UFqJHTk6L6632zuurz4jmx5pMJvxx5Ah6ZGe3n8FPnxHNf5eNDQ0oOny41bVnW8f69Jqjjd/7P44fh6BQYMCAAe0+TiR5ZOy+++7DmDFjcMUVV6B///6Ijo5uGUgQ8Nxz7Q8nnkycTlhRUYHu3bt7bq+oqECfPn3afIzNZsOjjz6KL774Ao8++ihuv/32Ns/XvBirqKjwbEidkZGBgwcPtjin2Kq/o5tWCwoF4pOSvDu42ZoIXx+riIyEUFqKWI3GczER6AwAoDQaT5/DXxma8SqHn8khg1xyyCHDaXM0XgB7RcqxJz3PKTP4+ffTqxzhnKEdYo6YmJig/47ExMbK4nckOjo66DkABDWH0mgETpxoyhCgz4iTKTr6WvjwWDlkAIDj5fUwW52ITUjwPoePf37Ewk1SBj8RlMqg/7sqKBRBzwAAglod/ByxsRBOM5ugOcnF2MKFC/HHH38gOjoae/a0npUpeLlYTZSXl4e4uDhs3rzZU4wZDAbs27cP48ePb/MxDz30EP7zn//ghRdewNVXX93ivuTkZJxxxhnYvHmzp4mH3W7Htm3bMG7cOABAfn4+Pv30U9TX1yMuLg4AsGnTJsTGxiIvL09SfiIiIiIKjKOlBjy49Ff0zozEoA70WyOSG8nF2Oeff47bb78dDz30kE+mKarVaowfPx4LFy5EUlISsrOzsWDBAmRkZGD06NFwOByoqalBfHw8oqKi8PHHH+Orr77CQw89hKFDh6Ky2Zxa8ZjJkyfj2WefRY8ePTBgwAC8/vrrMJvNGDt2LADgsssuw4svvoj77rsPDz74IIqLi7Fo0SJMnjyZbe2JiIiIZKrwhHs9V7mu/elzRKFEcjHmcDhw6aWX+qQQE82YMQN2ux2PPfYYzGYz8vPzsXLlSqhUKhQXF2PUqFGYO3curr/+enzxxRcA3GvA5s+f3+I84jE33XQT6urq8OKLL0Kn0+Hss8/Gm2++iaTGaYSRkZF444038OSTT+Kmm25CYmIixo0bh6lTp/rs70REREREvlWpc0+VbDA7TtnIhChUSC7GLr/8cnz99dc4//zzfRZCqVRi9uzZmD17dqv7cnJycODAAc+fV61a5dU577jjDtxxxx3t3t+jRw+vz0VEREREwVdZ6+6g6HQBDWYbGlebEIUsycXYwIEDsXDhQuzfvx+DBg1C7EmLUwVB8NkeZEREREREokpdUzt7XZ0V6SlBDEPkA5KLsSeeeAIAsGvXLuzatavV/SzGiIiIiMgfqpoXY/W+3xuWKNAkF2P79u3z6XoxIiIiIqLTcblcqKxt2hRaV3/qjbWJQoHkquq6667DDz/84I8sRERERERtajDbYbI0bcbMkTEKB5KLsdLGTSCJiIiIiAKl+agYAOg5MkZhQHIxdu2112L16tWoqKjwRx4iIiIiolaaN+8A3A08iEKd5DVjRUVF2LZtGy6++GJoNBrExMS0uF8QBHz33Xc+C0hEREREJLa1FwTA5QL0nKZIYUByMZaZmYlrr73WH1mIiIiIiNokTlPMSonFicoGNvCgsCC5GJs7d64/chARERERtUucptg7J6GxGOPIGIU+ycWY6Oeff8aWLVtgMBig1WoxZMgQDB8+3JfZiIiIiIgANE1TzM1OxE87S1FntMHhcEKp5JZLFLokF2NWqxVTp07FL7/8AqVSCa1Wi9raWrz++uu44IIL8Nprr0GtVvsjKxERERF1UeLI2BmZ8U3rxhqsSEqICnIyoo6T/FXCyy+/jO3bt2P+/PnYs2cPfvnlF+zevRtz587Frl27sHz5cn/kJCIiIqIuyuFwokbvLsbStNGIjXRfwtYazMGMRdRpkouxL774AtOnT8d1110HpVIJAIiIiMCYMWMwffp0rFu3zuchiYiIiKjrqjaY4XQBEUoBCbFqxEW7r0Fr67hujEKb5GKspqYG/fr1a/O+fv36oby8vNOhiIiIiIhE4nqxFE00FAoBcVHuS1hdHUfGKLRJLsa6d++O7du3t3nf1q1bkZmZ2elQREREREQicb1Yqsa9v21sFEfGKDxIbuBxyy23YN68eYiKisLVV1+NlJQUVFVV4YsvvsCKFSswffp0f+QkIiIioi5K3GMsVRsNAJ5pijoWYxTiJBdjt956K/bt24eFCxfihRde8Nzucrnw17/+FXfddZdPAxIRERFR19Y0MtZYjDVOU+TIGIU6r4qxzz77DCNGjIBWq4VCocCzzz6LyZMnY8uWLdDr9UhMTMTQoUORm5vr77xERERE1MWIa8Y8I2OeaYpcM0ahzati7IknnsDKlSuh1WoxatQovPLKK8jLy2PxRURERER+V3XSmrG4aLG1PUfGKLR5VYyp1Wp89tlnsNvtOHHiBHbt2oW6urp2j8/Pz/dZQCIiIiLq2lqtGWscGdPVsxij0OZVMTZ27FisXLkSH374IQRBwJNPPgnAvU5MJAgCXC4XBEFAQUGBf9ISERERUZfSYLKhwWwH4G5t73JYPQ08Gkw2WG0OqFXKYEYk6jCvirHZs2djzJgxqK2txYQJEzBnzhz07t3b39mIiIiIqIsTpyjGx6gQHRkBo9GKKJWACKUAu8MFXZ0FaUkxQU5J1DFed1M888wzAQBZWVkYNWoU0tPT/RaKiIiIiAhovccY4J6RpYmLRJXejNo6M4sxClmSN32uqanBnj17/JGFiIiIiKiFk9eLiTTxagBsb0+hTXIxlpmZifr6en9kISIiIiJq4eQ9xkSJse5ijBs/UyiTvOnzzTffjGeffRY7d+5Enz59EBsb2+qYMWPG+CIbEREREXVxJ+8xJtLERwLgyBiFNsnF2Lx58wAAH374YZv3C4LAYoyIiIiIfKKtNWMAoIkTpyly42cKXZKLse+//94fOYiIiIiIWmlvzVhinHtkjNMUKZRJLsays7Nb/NlisUCtVkMQBJ+FIiIiIiJyOF2o1rtHvlpNUxRHxgwcGaPQJbkYA4AjR45gyZIl+N///of6+nqsXbsW//73v9GrVy/cdtttvs5IRERERF2Qrs4Mh9MFpUKAJj6qxX2ekbF6joxR6JLcTbGgoABjx47F77//jmuvvRYulwsAoFQq8dxzz+GTTz7xeUgiIiIi6nrE5h3JmmgoFS1nYTVvbS9ejxKFGsnF2PPPP4+zzz4bX3/9NR599FHPD/9jjz2GsWPH4u233/Z5SCIiIiLqejydFE9qaw8AmsbW9harAyaLPaC5iHxFcjG2a9cu3H777YiIiGi1Tuyqq65CUVGRr7IRERERURdWqWu7eQcAREVGIDpSCYBNPCh0SS7GIiMjYTa3vVBSp9NBrVZ3OhQRERER0alGxgB41pFxrzEKVZKLsQsvvBBLlixBWVmZ5zZBENDQ0IBVq1bhT3/6k08DEhEREVHX5NljTBvT5v0atrenECe5m+Ls2bNx880348orr0ReXh4EQcC8efPwxx9/wOVyYdGiRf7ISURERERdzOlGxrQJ7mKMGz9TqJI8MpaZmYnPPvsMEydOhMvlQvfu3WE0GnHNNdfg448/Rrdu3fyRk4iIiIi6mFOtGQMALacpUojr0D5jWq0Ws2bN8nUWIiIiIiIAgMliR53RBuAUI2PxjSNj3PiZQlSHirGqqiq8/fbb2LJlC/R6PZKTkzFs2DDcdtttSEhI8HVGIiIiIupiqhrXi8VGRSAmStXmMWzgQaFO8jTF/fv346qrrsJbb72FqKgo9OvXD0qlEq+99hquvfZalJSU+CMnEREREXUhnvVi7TTvAJpGxnT1LMYoNEkeGZs3bx4yMzPxxhtvIDU11XN7eXk5pkyZgueffx4vvfSST0MSERERUdcirhdLaWeKIgBoxGKM0xQpREkeGdu9ezdmzJjRohADgPT0dEyfPh3/+9//fBaOiIiIiLqmppGx9osxsYGHrt4Cp9MVkFxEviS5GNNqtairq2vzPofDgaioqE6HIiIiIqKuzbPH2ClHxtQAALvDhXqTLSC5iHxJ8jTFadOmYeHChejevTsGDx7suf3IkSN46aWXMH36dEnnczqdWLp0KdauXYu6ujrk5+djzpw5p22R73Q6cdddd2HgwIG49957Pbf36dOn3cf88MMPyMrKwvbt2zFu3LhW97/99ts4//zzJeUnIiIiIt/zZs2YKkKJ+BgV6ow21NaZkRCrDlQ8Ip+QXIx9+umnsFgs+Nvf/oacnBykp6ejtrYWRUVFcDqdeP311/H6668DAARBwHfffXfK8y1btgzvvfce5s2bh4yMDCxYsABTpkzBunXroFa3/QtltVoxZ84c/Pe//8XAgQNb3PfLL7+0+LNer8f48eNx8cUXIysrCwBw4MABdO/eHe+9916LYxMTEyW9FkRERETkH549xk4xMga4143VGW3Q1VnQIyMQyYh8R3IxlpOTg5ycnBa3devWDeecc47kJ7darVi1ahUefPBBXHLJJQCAxYsXY/jw4Vi/fj2uueaaVo/ZsWMH5syZA7PZ3GYb/ZPXsj399NPQarV4+umnPbcdPHgQvXv3bnUsEREREQWf0+nytLY/1ZoxwL1u7Hh5PdvbU0iSXIzNnTvXZ0++f/9+NDQ0YNiwYZ7bEhIS0K9fP2zdurXNYuynn37C8OHDMW3aNFx33XWnPP8vv/yC9evXY82aNS1G2Q4cOIDzzjvPZ38PIiIiIvIdXb0FdocLCgFITjh1PwJPR8U6dlSk0NOhTZ99paysDACQmZnZ4va0tDTPfSebNWuW1+dftGgRRo0ahSFDhrS4/dChQ9Bqtbj++utRXl6Os846C7NmzerQ6J7I5XLBaDR2+PG+YjKZWvw/cwQ/hxwyyCWHHDLIJYccMsglhxwyyCWHHDIwh/wyyCVHIDMcL9UDALQJUbBYWhZZJ+eIj1YCACpq6gN2LSaH90MuOeSQQU45AHddIAiCV8cGtRgTX6yT14ZFRkZCr9d36txbt27F77//3mJ6IgCUlpairq4ORqMRjz32GJRKJdasWYPx48fj448/Ru/evTv0fDabDQUFBZ3K7EtFRUXBjgCAOeSWAZBHDjlkAOSRQw4ZAHnkkEMGQB455JABYA65ZQDkkSMQGX4/5i6qYtTOdq+vxBwWo7vL99HiShQUOPyera0MwSaHHHLIAMgnR3u9L04W1GJMbINvtVpbtMS3WCyIjj71/ODT+eSTT3DOOeegf//+LW7PzMzE1q1bER0dDZVKBQAYMGAA9u3bh3feeQdPPvlkh55PpVJ1uJDzJZPJhKKiIvTs2bPTryFzhE8GueSQQwa55JBDBrnkkEMGueSQQwbmkF8GueQIZIbDNUUAatA9Iwl9+/Y9ZY4Kcwm+26UHIqJbHesvcng/5JJDDhnklAMADh8+7PWxQS3GxOmJFRUV6N69u+f2ioqKU7aoPx2n04kNGzZg6tSpbd5/cuMPhUKB3NxclJeXd/g5BUFATEz7rVcDLTo6WhZ5mENeGeSSQw4Z5JJDDhnkkkMOGeSSQw4ZmEN+GeSSIxAZ9A3uEa6MlLh2n0vMkZ7svq4zNNgC/trI4f2QSw45ZJBLDm+nKAId2PTZl/Ly8hAXF4fNmzd7bjMYDNi3bx/y8/M7fN7Dhw+jtrYWf/rTn1rd9/PPP2PQoEE4fvy45za73Y79+/fLYmSLiIiIqKurrG1sa3+KPcZE2gSxgQe7KVLo8Wpk7NNPP5V00jFjxnh1nFqtxvjx47Fw4UIkJSUhOzsbCxYsQEZGBkaPHg2Hw4GamhrEx8e3mMZ4Ovv27YNKpUKvXr1a3Td48GBotVo8/PDD+Mc//gGVSoXXX38dOp0Ot99+u9fPQURERET+UellW3ugqZuivsECh8MJpTKoYw1EknhVjD3yyCMt/iwOvblcrla3Ad4XYwAwY8YM2O12PPbYYzCbzcjPz8fKlSuhUqlQXFyMUaNGYe7cubj++uu9PmdlZSUSExOhULT+ZYyLi8Pq1auxcOFC3HHHHbBYLDjvvPOwZs0apKSkeP0cREREROQflbWNxdhpNnwGgITYSCgEwOkC9A1WJJ2mFT6RnHhVjH3//fee/y4oKMDs2bMxdepU/PnPf0ZaWhpqa2uxYcMGvPzyy5L3IVMqlZg9ezZmz57d6r6cnBwcOHCg3cdu2LChzdvvvPNO3Hnnne0+rnv37liyZImknERERETkf2arHYYGKwDvpikqFQIS4iKhq7NAV2dhMUYhxatiLDs72/Pf9957L6ZOndqi2ElPT8ett94Kq9WKBQsW4OKLL/Z9UiIiIiIKe9V6975i0ZERiI3yrtecNt5djNXWmQEk+jEdkW9JnlRbWFiIfv36tXlfr169UFxc3OlQRERERNQ1NTXviPa6K5023j0aVmtgEw8KLZKLsZ49e2LdunVt3vfBBx/grLPO6nQoIiIiIuqapKwXE4lNPNwjY0ShQ/I+Y9OmTcPMmTNRVFSESy+9FFqtFlVVVVi/fj0OHz6MFStW+CMnEREREXUBTZ0Uvd8rShvP9vYUmiQXY6NHj8Yrr7yCV155BS+++CJcLhcUCgUGDRqE1atXY8iQIf7ISURERERdQMdGxtzTFFmMUaiRXIwBwMiRIzFy5EhYLBbo9XpoNBqo1WpfZyMiIiKiLqZS17RmzFtazzRFFmMUWjpUjAHuRh4bN25EZWUlxo8fj+PHjyMvLw9xcXG+zEdEREREXUhHRsa0CVwzRqFJcjHmdDoxZ84cfPTRR3C5XBAEAVdeeSWWLVuGY8eOYc2aNcjIyPBHViIiIiIKYy6Xq4Nrxhq7KXJkjEKM5G6Ky5Ytw7p16/DMM89g48aNcLlcAIDZs2fD6XRi8eLFPg9JREREROFPX2+Fze6EIADJid5v3ixOU2ww2WC1OfwVj8jnJBdjH330EWbMmIEbbrgBGo3Gc3vfvn0xY8YMbNy40Zf5iIiIiKiLENeLJSVEIULp/WVqbLTKc7yunqNjFDokF2NVVVXo27dvm/elp6fDYDB0OhQRERERdT3ierEUCevFAEAQBM9eY+yoSKFEcjHWo0cP/PTTT23et2XLFvTo0aPToYiIiIio6/GsF5NYjAHNOioa2MSDQofkBh4TJ07EnDlzYLPZcOmll0IQBBw9ehSbN2/GqlWr8Mgjj/gjJxERERGFOU8nRQnNO0Rs4kGhSHIxduONN6KmpgbLly/Hv/71L7hcLtx///1QqVSYMmUKbr31Vn/kJCIiIqIw59ljrCMjYwnca4xCT4f2Gbv77rvxt7/9DTt37oROp0NCQgIGDhzYoqEHEREREZEUTSNj0osxTZy4ZozTFCl0SC7GnnvuOYwZMwb9+vXD8OHD/ZGJiIiIiLogn6wZ48gYhRDJDTw+//xz3HDDDbj66qvx+uuvo7S01B+5iIiIiKgLsdocnk6IHVkzpklwrxljN0UKJZKLsY0bN+LVV19F//798dprr2HUqFGYMGECPv74Y9TX1/sjIxERERGFuSq9e1QsUq1EfIxK8uObRsY4TZFCh+RiTKlU4uKLL8b8+fPx66+/YtGiRdBoNHjqqadw0UUX4f777/dHTiIiIiIKY571YppoCIIg+fHspkihqEMNPERqtRpXXHEFEhMTkZCQgE8++QTr16/3VTYiIiIi6iKaF2MdIW76bLE6YLLYER3ZqctcooDo8E/p9u3b8dVXX+Gbb75BTU0N+vfvj4cffhjXXHONL/MRERERURfgad7RgfViABAdGYEotRJmqwO1dWZER8b5Mh6RX0guxubPn49vvvkGpaWlyMzMxNixY3HdddchNzfXH/mIiIiIqAuorG3cY6wDbe1F2vgolFY3oNZgQVYKizGSP8nF2IcffogrrrgCf/nLXzB06FB/ZCIiIiKiLqYzbe1FmvhIlFY3sKMihQzJxdiVV16JG2+8EQMHDvRHHiIiIiLqgjqz4bNIm8COihRaJHdTXLduHRoaGvyRhYiIiIi6IJfL1WxkrGNrxgBAE+cuxjgyRqFCcjE2aNAgbNq0yR9ZiIiIiKgLMjRYYbU5AAApmqgOn0ebwPb2FFokT1Ps06cPVq1ahW+//RZ5eXmIiWn57YUgCHjuued8FpCIiIiIwps4KqaNj4QqQtnh83DjZwo1koux//znP0hLS4PNZsPevXtb3d+RTfqIiIiIqOuq0nV+vRjAjZ8p9EguxjZs2OCPHERERETURTVt+Nzx9WJA08bPOgNHxig0SF4zJnI6ndi/fz9+/vln1NfXQ6fT+TAWEREREXUVlT4aGfMUY/UWuFyuTuci8jfJI2MA8Nlnn+GFF15ARUUFBEHAv//9b7z88stQqVR44YUXoFarfZ2TiIiIiMKUZ8PnTuwxBjStGbM7XKg32RAfw2tSkjfJI2NfffUVHn74YVxwwQVYvHix51uHyy+/HD/99BOWLVvm85BEREREFL58NTKmilAiLloFAKjlVEUKAZKLsVdffRW33HIL5s+fj9GjR3tuv+GGG3Dvvffiyy+/9GlAIiIiIgpvvlozBjTf+JlNPEj+JBdjf/zxBy6//PI27xs4cCDKy8s7HYqIiIiIugab3elpRd/ZkTGAHRUptEguxpKTk1FYWNjmfYWFhUhOTu50KCIiIiLqGqr1JrhcgDpCgYTYzq/x0sQ1NvFgMUYhQHIxdtVVV2HJkiX45ptvYLVaAbj3Fvvtt9+wbNkyXHnllT4PSUREREThqfl6MV/sV6tJEIsxrhkj+ZPcTfG+++7DwYMHcd9990GhcNdyt912G4xGI4YMGYKZM2f6PCQRERERhSdfrhcDOE2RQovkYkytVuONN97Axo0bsWnTJuh0OsTHx2Po0KG4+OKLffKNBhERERF1DZU6d1v7lE62tReJ7e3ZTZFCQYf2GQOACy+8EBdeeCEAwG63o76+noUYEREREUniGRnzQfMOgCNjFFokrxmz2+1YunQp1q1bBwDYvHkzLrzwQgwbNgwTJ06EXq/3eUgiIiIiCk+eNWM+GhnTNI6M6epZjJH8SS7GlixZguXLl8NgMAAAnnnmGWg0Gjz66KM4duwYXnjhBZ+HJCIiIqLw5PuRMXcxZqi3wOF0+eScRP4iuRj78ssvcf/99+Nvf/sbCgsLcejQIdxzzz2YMGECZs2ahQ0bNvgjJxERERGFGZfLharGNWOpWt808EiIi4RCAJwud0FGJGeSi7GKigoMHDgQAPDjjz9CoVBgxIgRAICMjAzU1dX5NiERERERhaUGkw0miwOA7xp4KBUCEhr3GuO6MZI7ycVYWloaiouLAQAbNmxA3759kZSUBADYuXMnMjIyJJ3P6XRiyZIlGD58OM4991zceeedOH78uFePmzJlCl5++eVW940ePRp9+vRp8b9HHnnEc39tbS0eeOAB5OfnY+jQoXjyySdhMpkk5SYiIiKizhHXiyXGqRGpUvrsvJ6OitxrjGROcjfFa665BnPnzsW6deuwfft2zJkzBwDw7LPP4l//+hf+/ve/SzrfsmXL8N5772HevHnIyMjAggULMGXKFKxbtw5qddu7sFutVsyZMwf//e9/PaN0IqPRiOPHj+O1115D//79PbdHRUV5/nvGjBkwmUxYvXo1DAYD/vnPf8JoNOL555+XlJ2IiIiIOq5pjzHfjIqJNHHixs8cGSN5kzwydt9992Hy5MkQBAEPPPAAxo0bBwDYu3cvJk+ejKlTp3p9LqvVilWrVmHGjBm45JJLkJeXh8WLF6OsrAzr169v8zE7duzA9ddfj23btiEhIaHV/YcPH4bT6cSgQYOQmprq+V98fDwA9+jdli1b8Pzzz6N///4YNmwYnnrqKXz22WcoLy+X+nIQERERUQdV1vp2vZhIm8D29hQaJBdjgiDg7rvvxhtvvIE777zTc/v777+P+++/HwqF96fcv38/GhoaMGzYMM9tCQkJ6NevH7Zu3drmY3766ScMHz4cn376qafAau7AgQNISUlBYmJim4/ftm0bUlNTkZub67lt6NChEAQB27dv9zo7EREREXWOr9vaizhNkUJFhzZ9Lisrw9tvv41t27ZBr9cjOTkZF1xwAW677TZotVpJ5wGAzMzMFrenpaV57jvZrFmzTnnOAwcOICYmBjNmzMCOHTug1Wpxww03YMKECVAoFCgvL2/1fGq1GhqNBqWlpV5nJyIiIqLO8XVbe5GmceNnnYEjYyRvkouxgoIC3HbbbbDb7Tj33HPRrVs3VFZWYsWKFfjoo4+wZs0adOvWzatziU0zTl4bFhkZ2eHNow8dOgSDwYArrrgC06ZNw/bt27FgwQLo9XrMnDkTJpOpzbVokZGRsFg6/gvrcrlgNBo7/HhfEV/TYDckYQ55ZZBLDjlkkEsOOWSQSw45ZJBLDjlkYA75ZZBLDn9kKKupBwAkRCu9vo7yJkese2AMVXqjX67P5PB+yCWHHDLIKQfgrgsEQfDqWMnF2PPPP49u3bphxYoVSElJ8dxeWlqKKVOmYO7cuVi2bJlX5xKbalit1hYNNiwWC6KjO/YNyYoVK2CxWDxTGPv06YP6+nosX74c9957L6KiomC1Wls9zmKxICam4/OVbTYbCgoKOvx4XysqKgp2BADMIbcMgDxyyCEDII8ccsgAyCOHHDIA8sghhwwAc8gtAyCPHL7MUFblLsbqastQUFDjsxy6avf0xIrqOr9en8nh/QDkkUMOGQD55GivEeHJJBdjO3fuxKJFi1oUYoB7quGMGTNatJA/HXG6YEVFBbp37+65vaKiAn369JEaDYD7L37yX/6ss86C0WiEXq9HRkYGvvvuuxb3W61W6HQ6pKWldeg5AUClUqF3794dfryvmEwmFBUVoWfPnh0uaJkj/DLIJYccMsglhxwyyCWHHDLIJYccMjCH/DLIJYevM9gdTtSZ3Nsl5Z+bB03jOi9f5IhPrgc2VMFkA/r27dvprB3JEAhyyCGHDHLKAbgbCnpLcjGWlJSEhoaGNu9TKpWIjY31+lx5eXmIi4vD5s2bPcWYwWDAvn37MH78eKnR4HK5cPnll2PMmDGYPn265/a9e/ciNTUVWq0W+fn5WLhwIY4ePYoePXoAALZs2QIAOO+88yQ/p0gQhE6NrPladHS0LPIwh7wyyCWHHDLIJYccMsglhxwyyCWHHDIwh/wyyCWHrzJU1BjhcgERSgUyUjVQKLyb1uVNjsw09yVug8kOlToSqgjf7WHmbYZAkkMOOWSQSw5vpygCHSjG7rnnHrzwwgvIzc1tsY/X8ePH8dJLL+Guu+7y+lxqtRrjx4/HwoULkZSUhOzsbCxYsAAZGRkYPXo0HA4HampqEB8f32IaY3sEQcDll1+OlStXolevXjj77LPx66+/4o033sA///lPAMDAgQMxePBgzJo1C0888QSMRiPmzJmDMWPGID09XerLQUREREQd0LyTotRC7HTiolWIUAqwO1yorbMgzcet84l8xatibOTIkS0qvKqqKowdOxbdunVDSkoK9Ho9/vjjD6jVanz77beYMGGC1wFmzJgBu92Oxx57DGazGfn5+Vi5ciVUKhWKi4sxatQozJ07F9dff71X53vggQcQFxeHRYsWoaysDDk5OfjnP/+Jm266CYC7YFu6dCmefPJJTJw4EZGRkbjyyivx6KOPep2ZiIiIiDqnaY8x308pEwQBmvgoVOlM0LEYIxnzqhgT9+E6lXPOOadDAZRKJWbPno3Zs2e3ui8nJwcHDhxo97EbNmxodVtERASmTZuGadOmtfu45ORkLFmypEN5iYiIiKjzxJGxFB/vMSbSxEd6ijEiufKqGJs3b56/cxAREQXVjgOVqKy0wA9r/YmoDZ5pin4YGQO48TOFhg5t+gwAP//8M7Zs2QKDwQCtVoshQ4Zg+PDhvsxGREQUELUGM+a/uwuREQIuv8gV7DhEXYJnw2eNf6YQahs3fq7lyBjJmORizGq1YurUqfjll1+gVCqh1WpRW1uL119/HRdccAFee+01r/vqExERyUFJVQNcLsBsc6GmzoK4OO87AxNRx1QFamTMwJExki+F1Ae8/PLL2L59O+bPn489e/bgl19+we7duzF37lzs2rULy5cv90dOIiIiv6lobCQAAGXVxlMcSUS+4mng4ac1Y2IxpqvnyBjJl+Ri7IsvvsD06dNx3XXXQal079kQERHh2dtr3bp1Pg9JRETkTxU1TQVYeQ2LMSJ/azDZ0GC2A/BfMaYRpykaWIyRfEkuxmpqatCvX7827+vXrx/Ky8s7HYqIiCiQKhrXrgBAKUfGiPxOnKIYH6NGVGSHWxickkYcGeOaMZIxycVY9+7dsX379jbv27p1KzIzMzsdioiIKJBaTlM0neJIIvIFf3dSBABtArspkvxJ/irilltuwbx58xAVFYWrr74aKSkpqKqqwhdffIEVK1Zg+vTp/shJRETkN5W1nKZIFEj+Xi8GNHVTNFsdMFnsiPbTCBxRZ0j+qbz11luxb98+LFy4EC+88ILndpfLhb/+9a+46667fBqQiIjIn5xOV4tpimU1RrhcLgiCEMRUROEtECNj0ZERiFIrYbY6oKuzsBgjWZL8U6lQKPDss89i8uTJ2LJlC/R6PRITEzF06FDk5ub6IyMREZHf6OstsNmdEGsvq82JGoMZyYn+u0gk6uqa9hjz7++ZJj4SZdVG1NaZkZnCLStIfjr8FUFubi6LLyIiCnnierGk+Eg4nXbU1jtQUtXAYozIjzwjY37a8FmkjY9qLMbYxIPkSXIDDyIionAiTlFM0UQjKd79HWVpVUMwIxGFPc+aMT9OUwSadVTkxs8kUyzGiIioS2tqJBCFpDgWY0T+5nC6UKV3F0f+LsbEjZ85MkZyxZWMRETUpYndE1M10TBGuEfJSqrqgxmJKKzVGsxwOl1QKgTPxsz+ok1wn19Xz2KM5InFGBERdWlN0xSjUC9wZIzI38TmHcmaaCgV/u1aqolrHBkzsBgjeWIxRkREXZo4TTFFE4VIZ1Mxxvb2RP5RqfP/HmOipmmKXDNG8uTTNWMFBQUYNWqUL09JRETkNy5X0x5jaZpoaGIjIAjw7EtERL7naWvv5/ViQNM0Ra4ZI7nyaTGmVquRlZXly1MSERH5Tb3JBpPFDsA9MhahFJCS6L54K+FURSK/aGpr7/9izNNNsc4Cl8vl9+cjksqnxVhubi7eeecdX56SiIjIbyoam3do4iKhVikBABnJ7n2PStnEg8gvmkbG/LvHGNA0TdHucKLBZPP78xFJ5ZNizG63Q6fT+eJUREREAVPRxnSpjCT3BSJHxoj8I5BrxlQRSsRGqwBwqiLJk+RizG63Y+nSpVi3bh0AYPPmzbjwwgsxbNgwTJw4EXq93uchiYiI/EFs3pGW1PQNfdPIGIsxIn8I5JoxgE08SN4kF2NLlizB8uXLYTAYAADPPPMMNBoNHn30URw7dgwvvPCCz0MSERH5Q7lYjDWbLpWR5L5ALK1mMUbka0azDfWN0wUDMTIGANrGvczY3p7kSHIx9uWXX+L+++/H3/72NxQWFuLQoUO45557MGHCBMyaNQsbNmzwR04iIiKfE7+hT2s+TbFxZKyksoEL/ol8rKqxeUdstAoxUaqAPGfTyBiLMZIfycVYRUUFBg4cCAD48ccfoVAoMGLECABARkYG6urqfJuQiIjITyraGBlL00ZDEACTxQ59vTVY0YjCUiA7KYo0CWJHRU5TJPmRXIylpaWhuLgYALBhwwb07dsXSUlJAICdO3ciIyPDtwmJiIj8pKKm9doVtUqJlMYLRa4bI/KtQK8XA9zdUgGOjJE8SS7GrrnmGsydOxd33HEHtm/fjhtuuAEA8Oyzz+Lll1/Gtdde6/OQREREvmay2FFndI98pZ3UYjszORYAUFrN9vZEvhSMkTFxzRg3cic5klyM3XfffZg8eTIEQcADDzyAcePGAQD27t2LyZMnY+rUqT4PSURE5GviFMXYaJWn9bUoM8VdjJVUcmSMyJfEDqaB2GNMpE1gN0WSrwipD6itrcXdd9+Nu+++u8Xt77//vs9CERER+VtbzTtEWY3FGKcpEvlWlc5dEAVjZIzTFEmOJI+MjRgxAvfccw+++eYbWK1c2ExERKGpreYdosyUOABACdvbE/mUZ8PnAK4ZE7spGuotcDjZIZXkRXIx9uCDD6K6uhr33XcfLrzwQjz++OPYtm2bP7IRERH5TUVN6w2fRZ6Rscp6trcn8hGn0+VpbZ+qCdw0xYRYNQQBcLoAQwNHx0heJBdjt99+Oz788EP85z//weTJk7Fr1y6MHz8eI0eOxIsvvojCwkJ/5CQiIvKpilNMU0xv3GuswWyHoYGzQIh8QVdvgd3hgkIhIKlxHVcgKJUKJMaK7e1ZjJG8SC7GRN26dcM999yDdevWYd26dbjkkkuwYsUKXHPNNb7MR0RE5BcVp2gkEKWOQHKie51JKacqEvmE2LwjOTEKSmWHL0E7RCNu/GxgMUbyIrmBR3PV1dX4+uuv8fXXX2Pnzp3QaDS46qqrfJWNiIjIbyo9a8baXruSmRKLar0ZpVUNyOuRFMhoRGEpGG3tRdr4SBSVsqMiyY/kYqyurg7ffvstvvzyS2zduhVKpRIjR47EsmXLMHz4cCiVSn/kJCIi8hmb3YGaxm/I22rgAQBZKXH4rbCaHRWJfMSz4XMA14uJtAnca4zkSXIxNmzYMDidTpx33nl44okncOWVVyIuLs4f2YiIiPxCvCiMVCuREKtu85hMtrcn8inPyFgAOymKNHHiXmMsxkheJBdj9957L6699lpkZWX5Iw8REZHfVTSboigIQpvHeDZ+rqoPWC6icNa04XMQpily42eSKcnF2MmbPRMREYUasZNiW807RNz4mci3xJGxlCCsGdPEc5oiyVNgW9kQERHJwKk2fBZlJruLsTqjDXVGtrcn6qymNWPBaeABcGSM5IfFGBERdTmVp9hjTBQVGeHZC4mjY0SdY7Y27dl3qhFpfxGLMY6MkdywGCMioi6nvOb0I2MAkJniblBVwmKMqFOqGqcoRkdGIDaqUzsrdYg4TbHOaIPN7gj48xO1h8UYERF1OZVeTFMEmqYqcmSMqHM8UxRP0TTHn+KiVYhQup9XV8dpxyQfXn01sXXrVkknzc/P71AYIiIif3M4nKjSu9eNpCWdeu1KVqpYjLGjIlFnBHPDZwBQKARo4iJRpTejts4clI6ORG3xqhi77bbb2v0Ww+VyAUCL+wsKCnwQjYiIyPeqDWY4nS5EKAVoG6cutYd7jRH5RqUXHUz9TZMQhSq9mevGSFa8Ksbefvttz3+XlJTg8ccfxw033IA///nPSE1NhU6nw4YNG/D+++/jqaee8ltYIiKizqpoXC+WqomBQnHq6VLiNEWuGSPqnEqd+HsXvBGppo6KLMZIPrwqxoYOHer579tuuw233347HnjggRbHDB48GFFRUXjzzTdx1VVXSQrhdDqxdOlSrF27FnV1dcjPz8ecOXPQrVu30z7urrvuwsCBA3Hvvfe2uH3VqlVYu3YtysvLkZ2djdtvvx033nij55jly5fjxRdfbHXOAwcOSMpOREShpaLZ2pXTEUfGDA1W1JtsiItW+TUbUbiqlPB75y+aOLGjItvbk3xIbuCxZ88eDBs2rM37Bg0ahIMHD0oOsWzZMrz33nt4+umn8f7778PpdGLKlCmwWttfYGm1WvGPf/wD//3vf1vd99prr+G1117DzJkz8fnnn2PChAl44okn8Omnn3qOOXDgAP7yl7/gl19+afE/IiIKb9427wCAmCiV5wKujKNjRB0W7DVjAKBNcE9L5sgYyYnkYiwjI6PNAggAvvnmG3Tv3l3S+axWK1atWoUZM2bgkksuQV5eHhYvXoyysjKsX7++zcfs2LED119/PbZt24aEhIRW9//rX//C5MmTcdVVV6F79+64+eab8Ze//AVr1671HHPw4EH069cPqampLf5HREThrcKLPcaa47oxos5xOl2e1vbBXDPGjZ9JjiRv9DBp0iQ88cQTqKiowKWXXgqtVouqqip88803+PHHH7Fo0SJJ59u/fz8aGhpajLYlJCSgX79+2Lp1K6655ppWj/npp58wfPhwTJs2Ddddd12L+5xOJ55//nmcccYZLW5XKBQwGAwA3AVgUVERevXqJSkrERGFvgpxZCzJu4vCzJRYFBTVoIQdFYk6RN9ggc3uhCAAyYmnbprjT2LDnloDR8ZIPiQXY7fccgvsdjuWL1+OL7/80nN7ZmYmFi5ciD//+c+SzldWVuZ5fHNpaWme+042a9asds+nUChaTaMsKSnBl19+iVtuuQUAcPjwYTgcDnz77bd49tlnYbFYkJ+fj9mzZyMtLU1SfpHL5YLRaOzQY33JZDK1+H/mCH4OOWSQSw45ZJBLDjlkkEuOQGcor3aPcCXEKFp8breXIyVRDQA4Xq73++e8HN4P5pBfBrnk6GiG46V6AO6RKavFjM7u8tXRHNFqdwfwWoOp07/Lcng/5JJDDhnklANw1wXe7qfXoS3Qx48fj/Hjx+PIkSPQ6/XQarXo2bNnR07lecHUanWL2yMjI6HX6zt0zuaqqqpw5513Ijk5Gffccw8AeNa1RUdH46WXXkJ1dTUWLVqECRMm4NNPP0VUlPRvbWw2m6xa+hcVFQU7AgDmkFsGQB455JABkEcOOWQA5JEjEBmcLpdnZExfdQIF5vLT5nCY3McfKa4O2Oe8HN4PgDnklgGQRw6pGfYdc/8OxahdPv0dkpqj2mADANQYzD7LIYf3A5BHDjlkAOST4+Tapj0dKsYAQK/X448//kBFRQWuuOIKHDlyBGeccYbkXdXFwsdqtbYogiwWC6KjO7fI88iRI7jrrrvgcDjw9ttve9aXjRkzBiNGjEBSUpLn2DPPPBMjRozAhg0bJHeDBACVSoXevXt3Kq8vmEwmFBUVoWfPnp1+/ZgjfDLIJYccMsglhxwyyCVHIDPo6ixwOE9AEIChg/sjQtm0dLq9HJEJBnz0v80wmIC+ffv6NZ8c3g/mkF8GueToaIYjtUcB1KBbhtYnv0MdzWGy2IEvymG1u3BGrzMRFdnhy2BZvB9yySGHDHLKAbhn4XmrQz+Fy5cvx2uvvQaz2QxBEHDOOefgxRdfRG1tLVatWtVmU432iNMTKyoqWjT/qKioQJ8+fToSDwCwfft23HPPPUhPT8cbb7yB9PT0Fvc3L8QA97RIjUbT7tTI0xEEATExwVuUerLo6GhZ5GEOeWWQSw45ZJBLDjlkkEuOQGQ4XuleuJ+cGI2E+DivcvTMcbez19dbAYUKMVH+b28vh/eDOeSXQS45pGbQNdgBAJkp8T7NLjVHdLQLkWolLFYHLA4lknyQRQ7vh1xyyCGDXHJIGZyS3E1xzZo1ePnllzFp0iR8+OGHcLnc82/Hjx+P48eP46WXXpJ0vry8PMTFxWHz5s2e2wwGA/bt24f8/Hyp8QC42+9PmTIFZ555Jt59991WhdjixYtxxRVXeLIDQHFxMWpra2UxukVERP5RUSOtkyIAxEWrkBDrnm7CjopE0slhjzHAfYHMjookN5KLsXfeeQd33XUXZs6cif79+3tuv/jii3Hfffdhw4YNks6nVqsxfvx4LFy4EN9//z3279+PWbNmISMjA6NHj4bD4UBlZSXMZu9+aex2Ox588EEkJydj3rx5sFgsqKysRGVlJWpqagAAl19+OU6cOIEnnngCf/zxB7Zu3Yp7770XgwcPxvDhwyXlJyKi0FEhYY+x5jzt7atZjBFJJYc9xkRiR0Ud9xojmZA8TbGkpARDhw5t875evXqhqqpKcogZM2bAbrfjscceg9lsRn5+PlauXAmVSoXi4mKMGjUKc+fOxfXXX3/ac+3ZswdHjx4FAFx22WUt7svOzsaGDRtw9tlnY8WKFXjppZdw/fXXQ61WY9SoUXj44Yclr3kjIqLQIRZjUr+hz0qJxYGjtRwZI+oAOewxJtJ4RsZYjJE8SC7GMjMzsXPnTvzpT39qdd9vv/3WqkW9N5RKJWbPno3Zs2e3ui8nJwcHDhxo97Enj8QNHjz4lMeLhg0b1qoFPhFRoJitDlTobPBvOwg6WdOGz1JHxtzry1iMEUljtTk8o1DBnqYINC/GOE2R5EFyMTZ27Fi8/PLLiIqKwiWXXAIAMBqN+Pbbb/Haa69h0qRJvs5IRBR2ln/8Ozb9Xo6MbB0G5QX/2+KuQuqGzyJxmmIJizEiSar07i9AotRKxEX7v/nN6XCaIsmN5GLszjvvRHFxMRYuXIiFCxcCACZMmAAAuPbaa3H33Xf7NiERURg6VOzeR/G3IzUYlJcV5DRdg8vlQqVnzZj0aYoAUFpV7/NcROGsefMOOSwF8TTwMLAYI3mQXIwJgoCnnnoKkydPxqZNm6DT6RAfH4/8/HycddZZ/shIRBRWLDYHqvXuKTJ/lBiCnKbrqDfZYLI4AEhfuyIWYzUGC8wWe6f2JyLqSjzFmEYeMwDEYkxXz2mKJA+S/zVZunQpbrzxRvTs2RM9e/ZscV9xcTFWrVqFOXPm+CofEVHYKW/Wke+PkrogJulaKmrco2KauEhEqpSSHhsXo0Z8jAp1RhtKqxtwRlaiPyIShR1PJ0UZrBcD2MCD5Edya/tXXnkF5eXlbd63e/durF27ttOhiIjCWfMmEFV6M/T1vCgIBE/zjqSOXRRy3RiRdOLUYDm0tQea1ozVGiwt9pslChavRsZuueUW7N69G4B7zv3NN9/c7rEDBgzwTTIiojB18sV8YbEeg/PSgpSm62hqa9+x6VKZyXE4eEzHjopEEsh1ZMzucKLBZENcjDrIiair86oYe+aZZ/DNN9/A5XLhlVdewQ033ICMjIwWxygUCiQkJGD06NF+CUpEFC5OvpgvPKFjMRYAHd3wWeTZ+JnFGJHX5LZmTK1SIjZahQaTDbV1FhZjFHReFWO9e/fG9OnTAbgbeNx4441IT0/33G+32xERwcXMRETeEC/mUxMjUKm343CxLriBuohKzx5jHfuGPiuVxRiRFC6XyzMyliKTaYqAu4lHg8kGXZ0F3dLjgx2HujjJa8amT5+Ozz77DHfddZfntu3bt+Oiiy7CmjVrfBqOiCgclTQ28Ojf3f1NcWFjm3vyL9+NjLG9PZE3DA1WWG3uDqYpmqggp2nCjZ9JTiQXY6tWrcKLL77YopNi9+7dceWVV2LevHls4EFEdAo2uwNVjUVB/+7ub4rLa4yoM1qDGatLELspSt3wWZSZ7C7GqvRmmK12n+UiClfiqJg2PhKqCGkdTP3J08SDHRVJBiQXY++//z7uu+8+/OMf//DclpmZicceewzTp0/H6tWrfZmPiCislFUb4XQBUWolUhIikN44Ze4IR8f8ymSxo85oA9DxaYoJsWrERrmn5JdXG32WjShcNd/wWU6aNn7myBgFn+RirLy8vN2OiQMHDkRxcXGnQxERhavSximKGUkxEAQBZ2QlAADXjfmZOEUxNlqFmChVh84hCAIyU+MAsL09kTcqdWJbe3k07xBxrzGSE8nFWHZ2Nn799dc279u6dWurLotERNREbP6Qnuz+prhXlnvxOIsx/+ps8w5RVjKbeBB5S74jY+5pijru8UgyILkF4k033YQFCxbAZrPhsssuQ3JyMmpqavDDDz/gzTffxAMPPOCPnEREYaGk0t38ISMpBoDdMzJWeILTFP2pvKZzzTtETRs/s4kH0el49hiTUSdFoGlkTGdgMUbBJ7kYu/3221FeXo533nmnxfowpVKJiRMnYtKkSb7MR0QUVsQRlYzkGAAGnNE4MlZa1YAGkw2x0R2bQkenVlnbueYdIu41RuS9KtmOjLGbIslHhzYHe/jhhzF16lTs3LkTer0eCQkJOOecc6DVan2dj4gorHjWjCXHACYD4mPUSNNGo6LWhCMn9BjQOyXICcNTha+mKaa414yJ7yMRtU+ua8a0Ce5pivp6CxxOF5QKIciJqCvr8E7N8fHxGDFihC+zEBGFNZvd6WmvnpEUjfIT7ttzczSoqDXhcLGOxZifiA08Un00TbFKZ4LV5oBaJZ923URyYrM7UNM4DVBuI2OJsWoIAuB0AYYGi2cNGVEweFWMjRo1Cq+88gry8vIwcuRICEL73yAIgoDvvvvOZwGJiMJFRa27rb1apYQ2PhLljbf3ztHg172lbOLhR+I0xfROFmOJcWpER0bAZLGjrLoB3TMSfBGPKOxU691TANURCiTEqoOcpiWlUoHE2Ejo6i3Q1bEYo+DyqhgbOnQoYmNjPf99qmKMiIjaJq4zykqJbfE5mpuTCAAo5F5jfmG1+e4bekEQkJkSiyMn9CitYjFG1J7mnRTleN2oiXcXY7V1FpwR7DDUpXlVjM2dO9fz3/PmzfNbGCKicCZ24BOnuolyszWe+41mW4f3waK2VTV2dItUK33yDb2nGOO6MaJ2yXW9mEgTHwmUAjo28aAgk7zPGBERdUzzkbHmNPGRSEmMgssF/FFiCEa0sCauF0vz0Tf0WZ729izGiNoj1z3GRJ6OimxvT0Hm1chYXl6epH/ACgoKOhyIiChciRfvJ4+MAe4mHlX6Mhwu1qF/r+RARwtrFZ6LQt98Qy8WY6WVLMaI2iPXPcZE4jqx2joWYxRcXhVj06ZN8xRjFosFb775Jnr27IkrrrgCqamp0Ol02LBhAw4ePIh77rnHr4GJiEJV6SmKsd7dNNj8exmbePiB2MGys807RJmN7e1LOE2RqF2yHxlLaNz4mcUYBZlXxdi9997r+e9//OMfuOSSS/Dyyy+3GC37+9//jtmzZ+P333/3fUoiohBndzS1tXfvVeVqcX9uNpt4+EtTW3vfXBR62tvXGmGzO6CKYHt7opPJfs1YHDd+JnmQvGbs66+/xs0339zmtMW//OUv+O9//+uTYERE4aSy1gSH0wV1hAJJCa3bKPfO0QAATlTUwWyxBzhdeGva8Nk3F4Xa+EhEqZVwuoDyxgKbiJq4XC75j4xxmiLJhORiLDY2FseOHWvzvn379iExMbHToYiIwo3YSTEjJRYKResvs7QJUUhKiISTTTx8rtLTwMM3xZjY3h5omnpKRE0aTDaYrQ4AQLJM14xpPNMUOTJGwSW5GLv66quxaNEifPjhh6ioqIDNZkNZWRlWr16NV155BWPHjvVHTiKikOZZL5bcer2YKLdxdIzrxnzH4XCiqnHz2bQk310UZrKjIlG7xOYdmrhIRKrkOY1XHBmrM9pgszuDnIa6Mq/WjDX3wAMPoLS0FHPmzGkxVdHlcuGmm27CtGnTfBqQiCgceNrap8a1e0zvHA227itH4QldgFKFv2q9GU6nCxFKhefiyxfEopojY0StiVMUU2Q6RREA4qJViFAKsDtc0NdbkCLTETwKf5KLMbVajSVLluDQoUPYtm0bDAYDtFotLrjgAnTv3t0fGYmIQt6p2tqL2MTD9zzNOzTRbU4P7SixqGYxRtRaZbPfO7lSKAQkxkWiWm9GbZ2ZxRgFjeRiTHTmmWciIyMDFRUV6NatG5RKeQ5DExHJQWnjmrGsU0xT7N1NAwA4Vl4Hi80h2+k9oaTCT00EuGaMqH2ePcZkPDIGuJvxuIsxNvGg4JG8ZgwANm/ejBtvvBFDhw7Ftddei0OHDuGBBx7AvHnzfJ2PiCjkORxOT9e9zNT2i7GkhCho4iLhdLpQVMLRMV/wdfMOkbjxc3mtEXYH15sQNefppCjTtvYijdhR0cBijIJHcjH266+/4o477kBUVBQefPBBuFzuvXLy8vLw9ttv48033/R5SCKiUFapM8HucEEVoUBKYvvfFAuCgNwc91TFw5yq6BNiEZyW5NuLQm18FNQqJZxOl2f/OCJyC6WRMYAdFSm4JBdjL774IkaNGoV33nkHEydO9BRjf//73zFlyhSsXbvW5yGJiEKZOJUtIznmtOuWxI6Kheyo6BOVnj3GfHtRqFAIntExdlQkaikU1owB7i1FAEDHaYoURJKLsYKCAtxwww0A0Grj5wsvvBAnTpzwTTIiojDhad6R3H4nRVHvHDbx8KUKP01TBLhujKgtdocTNQb3SJPcizFNnHtkjGvGKJgkF2Px8fGorKxs877S0lLEx8d3OhQRUThpamvf/noxkTgydrTMAKvN4c9YYc/pdPl1upTY3l7c0JuIgBq9GU4XEKFUILGx2JErbYJYjHGaIgWP5GJs1KhRWLx4Mfbu3eu5TRAElJWV4dVXX8Ull1ziy3xERCGv1Iu29qJUTTTiY9RwOF04Wmbwd7Swpqu3wGZ3QiHAL22rOTJG1JrnCxAfbyfhD+LegxwZo2CSXIw98MADSE5Oxk033eQpvO6//35ceeWVEAQB999/v68zEhGFNHHkJPMUbe1FgiB4piqyiUfniFMUkxKjEaHsUPPgU2IxRtSaZ72YzJt3AGzgQfIgeZ+xxMRErF27Fp9++ik2bdoEnU6H+Ph43Hbbbbj++usRHS3/Xz4iokBxOF0oq25sa+/FyBjgnqq482Alm3h0UmWNf5p3iLJS3GsAy2uMcDicUPqh4CMKNeLIWChsoqxpLMZMFgfMFjuiIju8/S5Rh0n+qXv88ccxduxY3HTTTbjpppv8kYmIKGxU60ywO5yIUApI9bKJRG92VPQJfzbvAIDkxCioIhSw2Z2oqDV5XWwThbNKP2207g/RkRFQq5Sw2hzQ1VuQwWKMgkDy13iff/45Gho4JYOIyBviFLb0pFgovVw/Ie41VlRaB5udGwp3lKcY8/EeYyKFQkBGMqcqEjXXtGZM3hs+A+5p4eJURW78TMEiuRgbNGgQNm/e7I8sRERhx7NeTMKoSXpSDOKiVbA7nDjGJh4dVuGnPcaay/KsG2NHRSIgtNaMAU3rxthRkYJF8nhsnz59sHLlSnzzzTfIy8tDTEzLbz4EQcBzzz3ns4BERKFM3GMsS0IxJggCcnMSsftQFQ4X6z3t7kmaCs9Fof++oReL7JJqjowRAS27KYYCceNndlSkYJFcjP3nP/9BWloabDZbi/b2opM3giYi6spKO1CMAUButga7D1Wh8IQOQA/fBwtzLpfL8w19YEbGWIwRNZhsMJrtAEKnGNN4OiqyGKPgkFyMbdiwwacBnE4nli5dirVr16Kurg75+fmYM2cOunXrdtrH3XXXXRg4cCDuvffeFvd9/fXXePnll1FcXIxevXrh4YcfxrBhwzz319bW4plnnsHPP/8MQRBw9dVX46GHHmInSCLyudJqcY+xOEmPYxOPzqk32WCyuDfNDsjIWCWLMSJxVCw+Rh0ynQm1cZymSMHV4T68DQ0N2LhxI7766its3rwZZnPHfoiXLVuG9957D08//TTef/99OJ1OTJkyBVartd3HWK1W/OMf/8B///vfVvdt2rQJs2fPxi233IJPPvkEw4YNw1133YXCwkLPMTNmzMDRo0exevVqvPTSS/jpp5/wxBNPdCg/EVF7nE4XyiRs+Nxcbjd3E48/SgywO9jEQ6ryGveomCY+EpEqpd+eJ9PT3r4BDqfLb89DFApCbb0YAGgapylyZIyCRXIx5nK58MILL2DYsGG44447cP/992PixIkYNmwY3njjDUnnslqtWLVqFWbMmIFLLrkEeXl5WLx4McrKyrB+/fo2H7Njxw5cf/312LZtGxISElrdv2LFClx22WWYMGECcnNz8fDDD6N///546623AAA7d+7Eli1b8Pzzz6N///4YNmwYnnrqKXz22WcoLy+X+nIQEbWrWm+G1e6EUiFIniqXkRSLmKgI2OxOHC+v81PC8BWIKYqAey+lCKUCdocLVY2jAkRdVaitFwPYwIOCT3Ixtnz5cqxcuRK33HIL1qxZg6+//hpr1qzBDTfcgMWLF+PDDz/0+lz79+9HQ0NDiymECQkJ6NevH7Zu3drmY3766ScMHz4cn376KeLj41vc53Q6sWPHjhbnA4Dzzz/fc75t27YhNTUVubm5nvuHDh0KQRCwfft2r7MTEZ1OabW7w156UozkDYEVCgG52RoAnKrYERWevY78215bqRCQkex+DnZUpK4ulPYYEzUVYxwZo+CQPKF37dq1uPvuuzFz5kzPbWeccQaGDBmCmJgYvPnmm15vBl1WVgYAyMzMbHF7Wlqa576TzZo1q93zGQwGGI1GZGRktHu+8vLyVs+nVquh0WhQWlrqVe62uFwuGI3GDj/eV0wmU4v/Z47g55BDBrnkkEOGQOY4WlILAEjTRrX6fPAmQ/f0GOwtBPYXVeNPZ6f6JaMc3hN/ZCipcG8JkBSn8vqzuaM50rRRKK6oR1FJLc7KkbY20FcZfI055JVBLjlOl6G0yj2Kr4mN8Os1kS9fiyiVe3qxrs6ChoYGrxvRyeH9kEsOOWSQUw7AXRd4+7MkuRirra3Feeed1+Z9559/Pt5++22vzyW+WGq1usXtkZGR0Ov1UqN51q21dT6LxeJ5zpPvP/mYjrDZbCgoKOjw432tqKgo2BEAMIfcMgDyyCGHDID/c/x2UAcAUAuWdj8fTpUhEu6Lmd8Ly1FQ4N/1SHJ4T3yZ4cjxKgCAw6KX/NksNYca7n979h0qRrd434yOyeH9AJhDbhkAeeRoL8Px0hoAgLm+GgUF/r8g9sVrYXO4P1ttdid27tmHaLW0WQxyeD8AeeSQQwZAPjnaqjfaIrkYu+CCC/D555/joosuanXfTz/91G6h1paoKPeiSavV6vlvALBYLB3qbBgZGek5X3PNzxcVFdVmcxCLxdJqzzQpVCoVevfu3eHH+4rJZEJRURF69uwZ1O6QzCGvDHLJIYcMgczx5c7dAOrRr3c2+vbtLjlDYkoDPv7f/1Cht6NPnzwoFL7fOkQO74k/Mph/2ATAjLPzeqJvH+9GFTua45jhODYd2A8botG3b98OJu5cBl9jDnllkEuO02Uwfun+EmRg/1yc1U0TtBxSxXxWDqPZjvSsnshO9a7ZkhzeD7nkkEMGOeUAgMOHD3t9rORi7LrrrsOTTz6JO+64A9dddx3S09NRW1uL7777Dt988w1mzpyJTz/91HP8mDFj2j2XOF2woqIC3bs3XahUVFSgT58+UqNBo9EgJiYGFRUVLW6vqKhAeno6ACAjIwPfffddi/utVit0Oh3S0tIkP6dIEIROFXO+Fh0dLYs8zCGvDHLJIYcMgchRqXOPmPTI0rb7PKfK0KtbNKIjlTBZHKiud6BHRuumRb4ih/fElxmq9e7XvltG+6+9r3L0yNICAMprzT7LL4f3gznkl0EuOdrK4HA4UdO47sr9e+f/i2FfvRba+CgYzfUw26Rfy8nh/ZBLDjlkkEsOKfsuSy7G7r//fgDAxo0bsXHjxlb3L1q0qEWQUxVjeXl5iIuLw+bNmz3FmMFgwL59+zB+/Hip0SAIAgYPHowtW7bgxhtv9Ny+efNmDBkyBACQn5+PhQsX4ujRo+jRw72R6pYtWwBA0qgeEdGpuFwulHRww2eRQiGgV7YGvx+pRmGxzq/FWDgxWeyoM9oA+L+bItD0/pZVN8DpdPllBJNI7moMFjidLkQoBWjjo07/ABnRJkTiRGU9OypSUEguxr7//nufPblarcb48eOxcOFCJCUlITs7GwsWLEBGRgZGjx4Nh8OBmpoaxMfHt5jGeCqTJk3CXXfdhX79+mHEiBH46KOPUFBQgGeffRYAMHDgQAwePBizZs3CE088AaPRiDlz5mDMmDGe0TMios6qMZhhtTmgUAid6uiXm53YWIzpMXKIDwOGsYrGtvax0SrERKn8/nypmmgoFQJsdieq9Cak+bmDI5EcVercv3fJidEh94WEWDxyrzEKBq+KMZvNBpXK/Q9adna21ye32WynPWbGjBmw2+147LHHYDabkZ+fj5UrV0KlUqG4uBijRo3C3Llzcf3113v1nBdddBGee+45LFu2DIsXL0bv3r3x6quvelrZC4KApUuX4sknn8TEiRMRGRmJK6+8Eo8++qjXfy8iotMRR8XStNFQRUjeRcQjN0cDADjM9vZeq2jc8Dk9QEWRUqlAelIMSqoaUFrVwGKMuiRxn71Qamsv0rC9PQWRV8XYtddei4ceeggjR470+sTffPMNFi9ejG+//faUxymVSsyePRuzZ89udV9OTg4OHDjQ7mM3bNjQ5u1jxow55fTI5ORkLFmy5JS5iIg6o9QzRbFzrc575yQCAI6c0HMKnJcqgrDXUVZqnKcYG3imf7YhIJIzzx5jIbThs4gbP1MweVWMzZ8/H4888gheeuklXHPNNRg9erRnvVVzhw4dwk8//YS1a9fC6XRi/vz5Pg9MRBQKxGIss4PrxUTZafGIVCthtjpworIe3dLjT/+gLq6ycZpiWlLgRqjE91kcESXqaio9I2OhNzLMjZ8pmLwqxs455xx8+umnePfdd7F69WosWrQICQkJyM7ORnR0NAwGA8rLy1FXV4ekpCRMmTIF48aN87SaJyLqakqq3PtNdbYYUyoE9MpKREFRDQpP6FmMeUEcGQtE8w5RZrL7fS6t8s0+Y0ShJpRHxjTimjEDizEKPK8beKjVakyaNAnjx4/Hpk2bsHnzZhw/fhz19fXIyMjApZdeigsvvBBDhgyBUqn0Z2YiItnz1cgY4G7iUVBUg8JiHS4ZnNPp84U7cc1YINduie9zKUfGqIsSG3iE4poxcWRMV89pihR4krspqlQqDB8+HMOHD/dHHiKikOdyuZqtGfNBMcYmHpKI3RQDWYxlNW4UW1pt5No+6pJCe2RMLMascDhdUPL3lwKo4y2+iIioTbo6C8xWBxQCkO6DdUu5JzXxoPZZbQ7Puo9AfkOfpo2BQiE0Pj+/XaeuxWi2od7k7qCdEoLFWGJcJAQBcDpdqGuwBjsOdTEsxoiIfExs4pCijYEqovPTtrunx0MdoYDRbEdZNafBnYrYXjtSrURCrDpgzxuhVHha6ZdU8j2irkX8vQvU3n6+FqFUeD4v+GUKBRqLMSIiHxObOGQld36KIuDex+qMLPfoWGGx3ifnDFflnvVi0RCEwE41YkdF6qo8nRRDcFRMJG78zI6KFGgsxoiIfEy8GM9M9U0xBgC9Gqcqct3YqTV1Ugx8e+2sFHZUpK6pMgh7+/maJq5x3RiLMQowycXY448/jt27d/sjCxFRWPBl8w5Rbzbx8EplEJp3iDwdFTmVlLqYcBgZ0ySIxRinKVJgSS7GPv/8czQ08B8aIqL2eEbGfDRNEXC3tweAwhN6uFxs4tEesZNiML6h90xT5Jox6mIqPb93obfhs4jTFClYJBdjgwYNwubNm/2RhYgo5DVva++LPcZE3TMSEKFUoMFk86yLotaCOU2x+cgYC2bqSsJhZEzca6yWGz9TgEneZ6xPnz5YuXIlvvnmG+Tl5SEmpuU/eIIg4LnnnvNZQCKiUKKvt8JksUMQgAwfjoypIhTomZWAw8d1KCzW+/Tc4UQcGfPFlgJSpSfFQiEAFqu7vX5SQlTAMxAFQzisGfMUY5ymSAEmuRj7z3/+g7S0NNhsNuzdu7fV/YHuXkVEJCfiqFiKJhpqVefb2jeXm52Iw8d1OFysw4UDs3x67nDgcDhRrXdfSAXjolAVoUCqNgblNUaUVjWwGKMuweF0oVovjoyF7jTFpo2fOTJGgSW5GNuwYYM/chARhYWSxk56vlwvJuqdo8G3OMomHu2o1pvhdLoQoVR41n8EWmZKLMprjCiprEf/XslByUAUSLo6M+wOFxQKAUmNTTBCkWfNGKcpUoBJLsZEBoMBu3btQl1dHZKSkjBgwADExcX5MhsRUcjxx3oxUW5O015jLpeLMxFO4mneoYmGQhGc1yYzJRa7DlayoyJ1GeJ6seTEKCiVobtjkjgyVme0wmZ3QhURun8XCi0dKsZef/11LFu2DGZz07xatVqNu+++G9OmTfNZOCKiUNPU1t73X071zExAhFJAndGKyloT0oKwLkrOxGIsLSl461ayuPEzdTGe9WIh3LwDAOJj1FAqBDicLujrLUgJ8b8PhQ7JxdhHH32ERYsWYezYsbjuuuuQkpKCyspKfPbZZ1i6dCmysrLw17/+1R9ZiYhkr6TafyNjqgglumck4MgJPQpP6FiMnSSYnRRFYhFeymKMuoimYiy0P48UCgGa+EhU682orTOzGKOAkVyMrV69Grfeeiv+7//+z3Nbr169cP755yMqKgpvv/02izEi6pJcLhdKK91rxny54XNzudmJOHJCj8PFegwbwCYezVXUBH+vI097+6oGTiWlLqFSF7y9/XxNLMZ03GuMAkjyhNijR4/isssua/O+UaNG4ciRI50ORUQUigwNVjSY7QCA9GT/FAS9u2kAAIVs4tFKpWdkLHgXhelJMRAEwGSxsysbdQnh0NZexI2fKRgkF2Pp6ekoKSlp877i4mI28SCiLkts2pCSGIUodYf7I51SbnbLJh7UpFxcMxbEkTG1SumZ3sSpitQVhMOGzyLuNUbBILkYGzlyJF566SXs2bOnxe27d+/Gyy+/jJEjR/osHBFRKGnqpOi/L6V6ZiVCoRCgq7d49tQiwOl0NY2MBXktXVazqYpE4a5pZCy014wBzfYaY3t7CiDJX93ee++9+N///oebb74Z2dnZSElJQVVVFU6cOIHc3Fw88MAD/shJRCR7JZX+a94hilQp0T09HkWlBhQW67jIvJGu3gK7wwmF4G6xHUyZKXHYfaiKxRiFPbPFjjqjFUC4jIxxmiIFnuRiLC4uDv/+97/x0UcfYevWrdDr9RgwYAAmT56M66+/HlFRwf1HkIgoWJra2vuvGAPc+40VlRpwuFiP88/O9OtzhQqxrX1SYjQigrzXkbjhN9vbU7gTpyjGREUgNloV5DSd5xkZ43pPCiDJxdgdd9yBKVOmYNy4cRg3bpw/MhERhaTSancnRX+OjAFA7xwNvt96HIUndH59nlBSWRP85h2ipo6K9UFOQuRf4bReDGi2ZszAKeAUOJK/PtyxYwdb9RIRtaFpzZifR8ayNQDYUbE5T/MOGey9lnVSe3uicBVO68UAQJvAaYoUeJKLseHDh+Pzzz+HzWbzRx4iopBUZ7Sizuj+XBSnqfnLGVkJUAhAjcGCGn6DC6BpmmIwOymKMhqLsQazHYYGa5DTEPlPVZiOjJksdpgt9iCnoa5C8jTFyMhIfP755/j666+Rm5uLmJiW//AJgoC33nrLZwGJiEKBOCqWlBCFqEj/tLUXRUVGIDstHsfL61BYrENSvwy/Pl8okMMeY6JIlRIpiVGo0ptRWtWAxLjIYEci8otw2vAZAKIjI6BWKWG1OaCrtyDDz5/lREAHRsbKysowaNAgnH322YiOjobL5WrxP6fT6Y+cRESyVhKgKYqi3jnu/cYOF+sD8nxyJ46MyWW6lLi9AZt4UDjzTFMMk5ExQRCamnhwqiIFiOSS/4knnkBubq4/shARhaxAdVIU9c7R4IftxVw3BsDlcqGiRpymKI+LwsyUWOwtZHt7Cm+eBh4y+RLEF7TxkaioMXLjZwoYySNj48aNw6effuqHKEREoaukKjCdFEW5ORoAbOIBAHVGG8xWBwD5XBRy42cKd06nK+zWjAHNOipyZIwCRHIxplKpoNVq/ZGFiChkNY2MxQXk+c7ISoAgAFV6c5efTiNOUdTERyJSpQxyGjexKC9he3sKU/oGC2x290brSUHeaN2XPBs/G7r25yoFjuRpijNnzsT8+fNRV1eHvLy8Vg08ACArK8sn4YiIQkWg2tqLYqJUyEqJw4nKehSe0OG8vPSAPK8cVdbKa4oi0HyvMY6MUXgS14slJUQFfaN1X2oaGeM0RQqMDq0ZczgcmD17drvHFBQUdCoUEVEoqTfZPC3MM5IDN02ud44GJyrrcbi4axdjFTLc60jc3qDeZEOd0Yr4GHWQExH5lrheLCWMpigCYAMPCjjJxdgzzzzjjxxERCGrtHEqmiY+EjFRqoA9b+9uifhpZzEKu3hHRbF5R7qMirGoyAgkJUShxuBubx/fncUYhZdw2/BZpGmcpshijAJFcjH217/+1R85iIhCVqA7KYpyszUA2MSjQobTFAH3VMUagxklVQ04qzvXWlN48ewxFmYjY9oETlOkwOrQJF+r1Yr33nsP06dPx80334zCwkL861//wp49e3ydj4hI9gK9XkzUK9u911hFrckzTbIr8kxTTJLXN/SejoqVbOJB4adpZCzMijGxgUedBS6XK8hpqCuQXIzV1NTghhtuwLPPPoujR49iz549MJvN+PHHH3Hbbbdh586d/shJRCRbgd7wWRQbrfI8Z1ceHWtq4CGvYszTUbGaTTwo/FSGYVt7oGnNmM3uRIPZHuQ01BVILsbmz5+PhoYGfPXVV/jkk0883xosWbIEAwYMwJIlS3wekohIzjzTFJMD09a+ud6N+40d7qLFmNFsQ53RBkCe0xQBdlSk8FQVpmvGIlVKxES5V/HoOFWRAkByMfbDDz9g5syZ6NGjBwRB8NweGRmJyZMn4/fff/dpQCIiufNMU0wN7MgYAPTOcU9VLDzRNZt4iFOl4qJVAW2e4g1xzzkWYxRurDYHdPXuBhfhNk0R4MbPFFiSizGLxQKNRtPmfUqlEjabrbOZiIhChtFs81yUiO3MA6mrN/GokOkURaBpmwNDgxX1Jv7bSOGj2uAeMYpSKxEXLa8vQXzB01GRGz9TAEguxgYMGID33nuvzfvWrVuHs88+u9OhiIhChTjqkRinRmwQLkpyG0fGyqqNqDd2vSYeFTJuIhATpfKsPxG3PyAKB1U6dzGWqo1uMUsqXHDjZwokycXYzJkzsXHjRvzlL3/BSy+9BEEQ8MUXX+Dvf/87vvnmG0ybNs0fOYmIZMnTvCMIo2IAEBejRnpjF8GuOFVR3GMsTWadFEXizwWnKlI4qdI3FmMaef7edZY2oamjIpG/SS7GhgwZgjfffBPR0dF444034HK5sHr1alRWVuK1117DBRdc4I+cRESy5GnekRr45h0isYlHV5yqKOdpigCQlcpijMJP85GxcKSJc4+MceNnCgTJmz4DQH5+Pt5//32YzWbo9XrExcUhNjY43woTEQVTsPYYay43JxEb95SgsLjrjYyJDTzk1klR5Glvz2KMwki1Z2RMnr93ncVpihRIHSrGRFFRUYiKiupUAKfTiaVLl2Lt2rWoq6tDfn4+5syZg27durV5fG1tLZ555hn8/PPPEAQBV199NR566CFER7s/EPr06dPuc/3www/IysrC9u3bMW7cuFb3v/322zj//PM79fchoq6ltDq40xQBILcLt7eX/chYMjsqUvjxTFOU6ZcgncVpihRInSrGfGHZsmV47733MG/ePGRkZGDBggWYMmUK1q1bB7Va3er4GTNmwGQyYfXq1TAYDPjnP/8Jo9GI559/HgDwyy+/tDher9dj/PjxuPjii5GVlQUAOHDgALp3796qEUliYqKf/pZEFK5KKt2NGYI6Mpbt/uwqqWqA0WyTXYt3f7HaHJ6LJbleFHKvMQpHnmmKYbpmTGy8w33GKBAkrxnzJavVilWrVmHGjBm45JJLkJeXh8WLF6OsrAzr169vdfzOnTuxZcsWPP/88+jfvz+GDRuGp556Cp999hnKy8sBAKmpqS3+t2TJEmi1Wjz99NOe8xw8eBC9e/dudWxbxR8RUXtMFrunGMgKYjGWGBfpKUa6UhOPSp17imKUWomEWHl+fovFmK7eAqOZ7e0p9LlcrvAfGROLsXornE5XkNNQuAtqMbZ//340NDRg2LBhntsSEhLQr18/bN26tdXx27ZtQ2pqKnJzcz23DR06FIIgYPv27a2O/+WXX7B+/Xo8/fTTLQqtAwcOtDgHEVFHlDVOUYyPUSMuJrjFQFMTj65TjImdFFO1MbJtrx0brUJinPtng+vGKBwYLU7Y7E4IApCc2LmlKnKV2NjAw+l0oa4LbhlCgRXUaYplZWUAgMzMzBa3p6Wlee5rrry8vNWxarUaGo0GpaWlrY5ftGgRRo0ahSFDhrS4/dChQ9Bqtbj++utRXl6Os846C7NmzcI555zT4b+Ly+WC0Wjs8ON9xWQytfh/5gh+DjlkkEsOOWTwZY6iEzUAgPSkaMm//75+LbqnxeBXAAeKqmDMzzzt8f7K0REdzVBcrgMAJCeoffL566/XIl0bDX29FUdP1CAr6dRFuxzeD+aQXwa55DCZTNAbHQAATZwaNqsFtiDUKoF4LeJjVKgz2lBaoYNKER+UDN6QQw45ZJBTDsBdF3j7JaFXxdinn34qKcCYMWO8Ok58sU6eHhgZGQm9vvW3uyaTqc2phJGRkbBYWi6y3Lp1K37//fcW0xMBoLS0FHV1dTAajXjsscegVCqxZs0ajB8/Hh9//DF69+7tVfaT2Ww2FBQUdOix/lBUVBTsCACYQ24ZAHnkkEMGoPM5dhcYAADRER3//ffVaxHhcH+e7i+q6lAWObwnUjPsP+z+dyLCZfbp56+vX4sopXt64p4DR6GJqA1Kho5iDnllAIKfQ9/gLsZi1Aj6dY8/X4tolQt1APbsO4yG2vZHAIP9fojkkEMOGQD55PB2+ZNXxdgjjzzS4s9ipedyuVrdBnhfjImdGK1Wa4uujBaLxdMd8eTjrdbWX8FYLBbExLRcRPrJJ5/gnHPOQf/+/VvcnpmZia1btyI6OhoqlXuR+4ABA7Bv3z688847ePLJJ73KfjKVStXhQs6XTCYTioqK0LNnzzZfQ+bomhnkkkMOGXyZ46cDvwMwoM8ZGejbV9rUZ1+/FpndLHj3x59RXWfHGb3ORFSkdxMf5PCedDTD97//BqAOZ56Rib59zwhajtPJKzuCPUWFcCrj0Ldv36BkkIo55JVBLjlMJhM27d8LAOiWoT3tz7M/c/j7tUjfbESFvgYJSeno2zcrKBm8IYcccsggpxwAcPjwYa+P9epf6++//97z3wUFBZg9ezamTp2KP//5z0hLS0NtbS02bNiAl19+GXPnzvX6ycUphxUVFejevbvn9oqKijZb1GdkZOC7775rcZvVaoVOp0NaWprnNqfTiQ0bNmDq1KltPm9CQkKLPysUCuTm5nqagHSEIAitCsJgio6OlkUe5pBXBrnkkEMGX+So1LlH5Htkajp8Hl+9FjExMUhOjEK13ozSWhv690o4/YP8kKMzpGaoqXN/OZeTnujT7L5+LbpnagC4f168Pa8c3g/mkF8GOeQQpylmpMQF/fXw52uRrIkBUAOjxXXK5wj2+yGnHHLIIJccUtYxe9XAIzs72/O/ZcuWYerUqbjzzjuRk5MDtVqN9PR03Hrrrbj77ruxYMECr588Ly8PcXFx2Lx5s+c2g8GAffv2IT8/v9Xx+fn5KCsrw9GjRz23bdmyBQBw3nnneW47fPgwamtr8ac//anVOX7++WcMGjQIx48f99xmt9uxf/9+WYxsEVHoKKkM/obPzXmaeJzQBTVHoMh9jzGRZ+Pnxm0QiEKZWIyFaydFkSZO3PiZe42Rf0nuplhYWIh+/fq1eV+vXr1QXFzs9bnUajXGjx+PhQsX4vvvv8f+/fsxa9YsZGRkYPTo0XA4HKisrITZ7G6hOnDgQAwePBizZs3Cnj17sGnTJsyZMwdjxoxBenq657z79u2DSqVCr169Wj3n4MGDodVq8fDDD+O3337DgQMH8PDDD0On0+H222+X9mIQUZdltthRY3B/NmWmxAU5jZu431hX6KjocDhRHSLttcVtD2rrLDBZ7EFOQ9Q54pqxcN1jTKSNdy+f0bEYIz+TXIz17NkT69ata/O+Dz74AGeddZak882YMQNjx47FY489hltvvRVKpRIrV66ESqVCaWkpLrroInz11VcA3EN+S5cuRU5ODiZOnIj77rsPI0aMwBNPPNHinJWVlUhMTIRC0fqvFxcXh9WrVyMlJQV33HEHbr75Zuh0OqxZswYpKSmSshNR11XW2FY9Llolmz2ucrtpAACHi3VBzREIVXoznE4XIpQKz0WTXMXFqBEf416jLG6HQBSq9Eb3Fwpy/xKks7QJ4sgYN34m/5Lc2n7atGmYOXMmioqKcOmll0Kr1aKqqgrr16/H4cOHsWLFCknnUyqVmD17NmbPnt3qvpycHBw4cKDFbcnJyViyZMkpz3nnnXfizjvvbPf+7t27n/YcRESnUlrlnnImlymKQNM0xeLyOpitdkSpg7p7iV+JUxRTtdFQKOS5x1hzWSlxOHCsFiVVDTgjKzHYcYg6xGZ3ot7kBACkasK8GIvnNEUKDMn/Uo8ePRqvvPIKXnnlFbz44otwuVxQKBQYNGgQVq9e3WpPLyKicCS39WIAkJQQBW18JGrrLCgqMSCvZ1KwI/lNpWe9WGhcEGamxLqLMa4boxAmTs1WqxSymRHgL+KIe62BxRj5V4e+Nh05ciRGjhwJi8UCvV4PjUbjdS99IqJwUFotv2IMAHJzNNhWUI7CYl1YF2MVte591eTevEMk/pyUVnGaIoWuqsZ1mimJUZK6xYUiTePIWJ3RCrvDiQil5JU9RF7p8E9WYWEhPvjgA7zzzjuora3Ftm3bUF/Pb/yIqGsQL6qzZNK8Q5Sb454CdzjMm3hU1IjTFEOsGOOaMQphVbqmYizcxceoPVOg9fUcHSP/kTwy5nQ6MWfOHHz00UdwuVwQBAFXXnklli1bhmPHjmHNmjXIyMjwR1YiItko8RRj8hoZE9eNhXsTD3HNWHpSaExTzOLIGIUBcWQsuQsUYwqFAE1cJGoMZtQaLEhODI3PGgo9kkfGli1bhnXr1uGZZ57Bxo0b4XK5AACzZ8+G0+nE4sWLfR6SiEhOLDYHqnTuaXJym6YoFmPHyutgtTmCG8aPxGmKoTMy5h5BrdabYbayvT2FpuouVIwB7KhIgSG5GPvoo48wY8YM3HDDDdBoNJ7b+/btixkzZmDjxo2+zEdEJDtie/KYqAjZLWJPToxCYpwaTqcLRaWGYMfxC6fThcoQWzMWH6NCbLTY3t4Y5DREHeOZpqjpIsWY2MSDHRXJjyQXY1VVVejbt2+b96Wnp8NgCM9//ImIRKXNpijKbRG7IAjIbRwdKwzTqYq6egvsDicUQuh8Qy8IQrMmHlxfTaHH5XLheIX7ZzdUuph2libOPTLGjZ/JnyQXYz169MBPP/3U5n1btmxBjx49Oh2KiEjOxGIsU2bNO0S52eHdxENs3pGUGB1SHc64boxC2dGyOtQYLIhQAmfmdI298jhNkQJBcgOPiRMnYs6cObDZbLj00kshCAKOHj2KzZs3Y9WqVXjkkUf8kZOISDZKquTZ1l4krhsrPKELag5/aWreERpTFEXiz0sJizEKQdsLygEAPdOioFYpg5wmMDTc+JkCQHIxduONN6KmpgbLly/Hv/71L7hcLtx///1QqVSYMmUKbr31Vn/kJCKSDXGaWWayvIuxo6UG2OwOqCLC68KpqXlHaE2V4sgYhbLt+ysAAGdmhcbUYF8Q14xxmiL5k+RirK6uDnfffTf+9re/YefOndDpdEhISMDAgQNbNPQgIgpXnjVjqfIsxlK10YiPUaHOaMPR0jr07qYJdiSfEkfGQqV5hygz2T2tlSNjFGqMZhv2/VENAOjdpYqxxpExA6cpkv9Inmx/1VVX4auvvkJcXByGDx+Oa6+9FhdffDELMSLqEmx2Bypl2tZe1KKJRxhOVRTXjIVaEwHx56VKZ4IljLcdoPCz+1AlHE4XMpKikRwv+Xv8kCVOU9Rx02fyI8nFmNVqhVar9UcWIiLZK6s2wuUCoiOVnk5bchTOTTwqQqytvSgxTo2YKPeFbHk1R8codIhTFM89KyXISQJLnKZoNNu5PyD5jeRibMKECXjxxRexc+dOmEwmf2QiIpItTyfF5DjZtbVvTpyaGG7t7V0uFyrFaYoh1sCjeXt7TlWkUOFyuTzNO849s2sVYzFREVBHuC+VuW6M/EXyWPNnn32GkpISjBs3rs37BUHAvn37Oh2MiEiOPJ0UZbpeTCQ28SgqNcDucIZUC/hTqTPaYLa6p/ilakJrmiLgbvpSWKxnEw8KGcfK6lClN0MdoUD/M7QoPFwZ7EgBIwgCNAlRqKgxQldnQYZMmzZRaJNcjF133XX+yEFEFBLETopZMl0vJkpPikFstAoNJhuOldWhV3Z47AskNu/QxEeGZHvtTHZUpBCzfb97VOzs3ikh+TvXWdr4SFTUGLnXGPmN5GJs+vTp/shBRBQSPCNjMv+GVBAE5GYnYs/hKhQW68KnGAvR5h2irMaNwlmMUagQ14udl5cW5CTBIa4N5jRF8pcOtcSxWCw4cOAArFYrXC4XAMDpdMJkMmHbtm148MEHfRqSiEgumtraxwU5yenl5miw53AVDhfrcPn5PYIdxydCtXmHqGnNWH2QkxCdXvOW9kPy0oOcJji0Ce4mHtz4mfxFcjG2efNmzJw5E3p92x26YmNjWYwRUViy2Z2e5hFybWvfXO8c92hY4Ynw6ahYGaJ7jInE6a2VOlNYbshN4WX3oSrYHS5kJsciKzUORqMx2JECzrPXGIsx8hPJK7oXL14MrVaLJUuW4LLLLsPo0aPx6quvYty4cRAEAStWrPBHTiKioKuoNcLpAiLVSs8/0HIm7jX2R4kBDoczuGF8pGnD59CcpqiJj0R0pBIul3ubBCI5E9eLddUpigA3fib/k1yMHThwANOnT8fll1+OSy+9FKWlpbj44ovx+OOPY+zYsVi+fLk/chIRBV1JpXtqWWZyrKzb2osyk2MRHRkBq82B4orwmBZXUeOeppgaYm3tRYIgIDO5cd0Y9xojGXO5XE3rxfp2zSmKAKBp3GuMa8bIXyQXY06nE+np7l/KHj164NChQ577rrjiCra1J6Kw1bReTP5TFAFAoRCQmyNu/qwLbhgfEUfG0kN0miLQbN1YJYsxkq9j5XWo0pmgilDg7NzkYMcJGs/IWD2LMfIPycVY9+7dceDAAQDAGWecAZPJhCNHjgAA7HY7Ghr4jwsRhafSEOmk2FxutgZAeBRjRrMN9SYbACA1RKcpAs3b24fHaCWFp+0F7lGxAbkpiFJ3qN9bWNA0FmM6g9nTtI7IlyQXY9deey0WLlyINWvWICkpCWeffTaefvppbNiwAa+88gp69+7tj5xEREFX0jitLDNF/p0URZ4mHsWh38SjsrGTYly0CjFRqiCn6TjuNUahgOvF3MRizGp3wmi2BzkNhSPJxdiUKVNwyy23YPfu3QCA//u//0NBQQGmTp2KI0eO4KGHHvJ5SCIiOShtnFYm9w2fmxObeBwp0cPhDO1vdStCvJOiSPz54ZoxkqvmLe278noxAIhSRyAmyj0yyI2fyR8kjzsrFAo8/PDDnj8PGDAA3333HY4cOYJevXohLi50vjEmIvKW3eFEeWMxECprxgD3fmhRaiXMVgdKKuvRLT0+2JE6TNzwOZSnKAJNI2MVNUbY7E6oIiR/L0rkV3sOu1vaZyTHhNSXT/6ijY+E0WxHbZ0FOWmh+xlK8uSTfwHi4uJwzjnnsBAjorBVUWuE0+mCWqWEtrG7VihQKgT0yg6PJh7ihs/pIdpJUZSUEAW1Sgmnq2m0j0hOPF0U89JDonOsv7GjIvmT5JGxkSNHnvYX8/vvv+9wICIiOWpq3hEDhSK0Lk5yczTY90cNDhfrcOl53YIdp8PEwiU1xKcpCoKArJRYFJUaUFrVgOxUfpFJ8uFuac/1Ys1pPBs/c5oi+Z7kYmzo0KGtirGGhgbs3bsXFosFEydO9Fk4IiK5ENuQZ4bglJ1waeIhNvAI1Q2fm8tsLMZKquoBdO01OSQvxRX1qKx1t7Qf0Dsl2HFkQWxvz5Ex8gfJxdi8efPavN1ms2Hq1KkwmUydDkVEJDdis4WsEOqkKBLb2x85oYfT6Qq5kT1ReZg08ACaNfFgR0WSGXFU7OxeyV26pX1z4tT0WgOLMfI9n60aVqlUmDBhAv7973/76pRERLLhmaYYgiNjOWlxUKuUMFnsIdvBz2pzeL6VTgvxNWNAs42fWYyRzIj7i3X1LorNaTlNkfzIpy2c9Ho9N30morAkbtAbisWYUqlAr6wEAMDh47rghumgSp171kWUWon4mNDdY0wk573GnE4X3lt/COt36rjJbRdjstjx2xF3S/vBfbheTOTZ+LmeI2Pke5LHnz/99NNWtzkcDpSVlWHNmjUYMmSIL3IREcmGw+FEWbV7ilwoFmOAu4nH/qO1OFysw8WDc4IdR7KmtvYxYdHdLTPZPd21osYIu8OJCKV82tu//dU+fPbfIgDABfsqcGn+GcENRAGz93AV7A4n0pJikJMWelOy/YXTFEPDnsJqnKiwoG/fYCeRRnIx9sgjj7R736BBg/D44493KhARkdxU6kxwOF1QRSiQkhiazSPEJh5HToRmE4+mDZ9D8/U/WXJiFNQRCljtTlTWmmRT5H+76Sg++uGw58/vf1eIEYN7QCmjYpH8Z1uzLorh8KWHr2gTmkbGQnndbTg7eKwWz67egZhIBf58cbDTSCO5GGurbb0gCIiLi0NCQoJPQhERyYm4ricjOTZk/xHOzdEAAAqL3VPPQu1CS9xjLBzWiwGAQiEgIyUWx8rqUFJVL4tibPfBSiz/aDcA4JoLe+D7bcdRUtWAH7Yfx2VDewQ5Hfmbu6W9e73YkDyuF2suMc5djDmdLtQZrZ4/kzy4XC688dlvAIAzs0JnH1CR5GIsOzvbHzmIiGRLXNeTJYML5o7qlh4PVYQCDWY7yqqNsrj4l6IijDopijKT3cWYHNaNHS+vw9y3tsDhdOHiQTkYf8WZsBp1WL9Tj3e/PYARg3KgVimDHZP8qLiiHhU1RkQoFTiHLe1biFAqkBCrhqHBito6C4sxmfl55wkUFNUgUq3EqIGJwY4jmeRi7NFHH/X6WEEQ8Nxzz0l9CiIiWSkJ4eYdogilAmdkJeDgMR0OF+tC7u8STnuMieTSxENXZ8GTb2xCg9mOvj2TMOPmc2G3WZB/Zhy2F5pRpTPh61+L8JcRuUHNSf4ljoqd3SsZUZFsaX8yTXwkDA1W6OrMQCZngsmF2WrH6i/3AQDGDO+JhJjQW9cn+betrKwM+/btg16vR3Z2NtLT06HT6XD06FG4XC5kZGR4jg21aTBERG0Jh5ExwL3f2MFjOhQW6zD83NCa5RCOI2NZqe4GCcFsb2+1OfDsm5tRXmNERnIM/jlpKNQqJew2QBUh4IZLe+H1zwrw4XcHcfnQ7oiJCv1OltQ2cX+x8/qyi2JbtPGROFZWh1pu/Cwrn/xYiCqdCWnaaFxzYQ8UHj4Y7EiSSS7GrrrqKhw6dAjvvfceBg8e7Ln9yJEjuOeeezBu3DhMnDjRpyGJiIIplPcYa65p3VhoNfGwO5yobmxtnxpGI2NZyeLIWH1Qnt/lcuGl93di/9FaxEarMOeOC1pNv7pkUBa+/N8xnKhswGc/FeLWK/KCkpX8y2yx47dCd0v787herE3sqCg/VToTPvrhEADg9mv6h+xUasntkV599VU8+OCDLQoxAOjVqxfuu+8+rFy50mfhiIiCzeF0NWtrH9qtnnMbOyoeLg6t/aOq9WY4Xe6pluIFUTgQi/vyGiMcDmfAn//db/fj510noFQIeHRiPrqlx7c6RqlUYPyf3X2iP/npMPTcZyks7SlsbGmvjWZL+3ZouPGz7Lz15T5YrA7075WMiwZmBTtOh0kuxmpqapCY2PbiOIVCgbq6uk6HIiKSiyqdybMPVIomtEdlemQkIEIpoN5k83QnDAXiFMVUbXTIdrNsS7ImGhFKBewOl2dT60DZsO0YPviPezrP9BsHYuCZqe0e+6cBWcjNSYTJ4sDa7w8FKiIF0PYCsaV9OpeYtEP8IkjHaYqysP9oDX7cUQxBAKb85eyQ/rmVXIwNHDgQS5cuRW1tbYvbKyoq8PLLL+Oiiy7yWTgiomATp5BlJMdAGeKFgCpCgR6NC88PF+uCG0aCyjDbY0ykVAjISHavgQtkE4/fCqvw8oe7AABjR5552rb1CoWACVf1AwB8ufEPT3FM4aF5S/vz8rherD3iyBiLseBzOl1Y8eleAMBl+d3Ru3EKfqiSXIw98sgjOHr0KEaOHInJkyfjgQcewO23347LL78cRqMR//jHP/yRk4goKMJlvZiod7P9xkJFeY3YSTF8mneIshqnvpZWB6YYK6msx3Ort8DucOHCc7JwW+MUxNMZdFYqBuSmwO5w4v31B/yckgLpRGU9ysWW9qcYIe3qtJymKBs/7ijGwWM6REdGeP0ZJmeSi7G8vDx8+eWXuOWWW1BfX4/ffvsNZrMZkydPxscff4zMzExJ53M6nViyZAmGDx+Oc889F3feeSeOHz/e7vG1tbV44IEHkJ+fj6FDh+LJJ5+EydRyesfo0aPRp0+fFv975JFHJJ2DiAho6nQXLsVYKDbx8IyMhcmGz82JP1cllf4vxuqMVjz5xibUGW04q7sGs8YN9nrapyAImHC1+6Ln+63HcLycSxLChTgq1r9XEqLZ0r5d2oTGBh4cGQsqk8WOtxpb2d902Vme9yWUdei3Lj09HQ8//LBPAixbtgzvvfce5s2bh4yMDCxYsABTpkzBunXroFarWx0/Y8YMmEwmrF69GgaDAf/85z9hNBrx/PPPAwCMRiOOHz+O1157Df379/c8LioqyutzEBGJPG3tk8OkGMsOvSYeFWE6TREI3F5jNrsTz63egpKqBqRqo/HYpPMRKbHzWF6PJJzfPwObfy/Dmm8K8OjEoX5KS4HUfL0YtU8cGTM0WGEPQsMdcvtowyHUGMzISI7BX0b0CnYcn5A0MlZdXY2amhrPn61WK95991089dRT+OCDD2C1WiU9udVqxapVqzBjxgxccsklyMvLw+LFi1FWVob169e3On7nzp3YsmULnn/+efTv3x/Dhg3DU089hc8++wzl5e4Pk8OHD8PpdGLQoEFITU31/C8+Pt7rcxARiTwjY6nh0WGsZ2YClAoBhgYrqnShMd1GbDaSGobTFD3FWLX/2tu7XC4sXbsLvxVWIzoyAv93xwUd/jb5tqv6QhCA/+0pxcFjtad/AMma2WrHb0fElvZcL3Yq8TFqz0gyu4oGR0WNEZ/8eBgAMPna/lBFhGYr+5N5XYzNnTsXF198MT755BMA7umFkyZNwjPPPIMvv/wSTz/9NG699VZJ0/3279+PhoYGDBs2zHNbQkIC+vXrh61bt7Y6ftu2bUhNTUVubq7ntqFDh0IQBGzfvh0AcODAAaSkpLTb8dGbcxARAe5FwmXV4bHhs0itUqJ7hvvLqVBo4uF0ulBZG85rxsSRMSMcTv+MVH74/UFs2HYcCoWARybke5q4dESPjARcel43AMA7XxX4KiIFyd7DVbDZnUjVRre5tQE1USgEaOLcM7Y4VTE4Vn+5D1a7EwNyU3DB2dKWRcmZV9MU//3vf+Ptt9/GhAkTcNlllwEAPvroI2zfvh3jxo3D448/jvLycowfPx4rVqzAjBkzvHrysrIyAGi1ziwtLc1zX3Pl5eWtjlWr1dBoNCgtLQXgLsZiYmIwY8YM7NixA1qtFjfccAMmTJgAhULh1Tk6wuVywWgMfocpsRgO9ho4ueQ4XlYLl8sV1BxyeS3kkEMOGaTkqNKbYbM7oVQIiFX79nc8mK9Fz4w4/FFiwIGiSpyVpQ5aDtGpXosagxl2hxOCAESrnH79nA3GexKrdkGpFGB3OFFcVgNxz2VfZfjf3jKs+Xo/AGDS1X2Q1z3Oq9fwVK/FX0f0wE87i7HrUCU27z2OAbnJPskqNUegyCGDv3Js/q0EADCwd5JX5w3n18IbCbFq1BgsKK/SQxsdF5QMJ5PDexKIDPuP1uK/u05AEIDxV/Ru87nk8FqIXC6X1+32vS7Gxo8fj0cffdRz2+effw6VSoVZs2ZBEARkZGRg0qRJePfdd70uxsQX6+S1YZGRkdDrWy8uN5lMba4ji4yMhMXi/pbi0KFDMBgMuOKKKzBt2jRs374dCxYsgF6vx8yZM706R0fYbDYUFMjnW8KioqJgRwAQ3BxbD9Xjy606XH5uIgQheDlEfE/klQE4fY4jZe5pfImxShw86J8OcsF4LaIV7ovxPQdLMSDLFrQcJ2srw/FK9+dyQrQSh/z0HniTw580MUpU19mxZed+9MqI8lmG45UWrP6+EgBwQZ84dIuvl/zvVHs5zsuNxZaD9Vj1+V7ceUWa3/f4kevPZzD4KofL5cKW391ffKdEmyX9bITba+GtCLg/L/cdPIooR2xQMrRHDjn8lcHpcmHFt+5GM4NzY2HSFaNAF/gcUrVVb7TFq2Ls8OHDuOeeezx/tlgs2LlzJwYOHOhZiwUAffr0wYkTJ7wOKTbVsFqtLRpsWCwWREe3XqgdFRXV5ro0i8WCmBj39JUVK1bAYrF4cvXp0wf19fVYvnw57r33Xq/O0REqlQq9e/fu8ON9xWQyoaioCD179mzzNewqOZxOF5Z9vREAsPlgPW7989mI7cT72xnBfi3klEMOGaTkOFFfDKAKPTI16NvXt+1zg/laKON0+GrbVlQYnOjRoweOHj0q25+LGlsZgEpkpsb7/D2QksOfum8zo7quCuq4VPTsmeyTDBW1Jiz6bDMcTmBIXipm3jpQ0obZp3stMrtZsHvxRpTU2FDnSsb5/fzT/EEOnxlyyOCPHCVVDaitPwGlUsCVI87xqpNiuL4W3srZ/zsOl5YgJj4ZPXtmdOnXIpAZfthxAqU1JxAdGYG7bxiCxLi2ixw5vBaiw4cPe32sV8WYzWZrUSzt3r0bdrsdQ4e27KRkMpmgUqm8fnJxumBFRQW6d+/uub2iogJ9+vRpdXxGRga+++67FrdZrVbodDqkpbkXnqrV6laV6FlnnQWj0Qi9Xu/VOTpCEIROFXO+Fh0dLYs8wcqxraDcs+jfYHTgjzIzhp6dEvAczXX190RuGbzJUW1wfwuak57gt7zBeC369oqEQiFAX2+F2a4IWo6TtZVBb3QAADJS4gKWL9CvRbf0BOw8WIVqg81zAdGZDPUmG+a/+ysMDTb0yk7EQxOGdrhleXs5YmJiMGZELj747iDW/nAEI87r6ddN0eX68xnKOfYVuZdm9D8jGclaaesIw+218Faq1j0aVm92+OR31ZfkkMMfGYxmGz74rhAAcOvoPshM0wQlh1RSZgt41cAjJyenRYX3888/QxAEXHjhhS2O27x5M7Kzs71+8ry8PMTFxWHz5s2e2wwGA/bt24f8/PxWx+fn56OsrAxHjx713LZlyxYAwHnnnQeXy4XLLrsMS5cubfG4vXv3IjU1FVqt9rTnoPDw9f+KAACqCPeP+M+7SoKYhkKVuBFvZpi0tRdFqpToluZe7/BHibz3i6qoEdvaB/9ix1+a9hrrfEdFu8OJ59/aiuPl9UhOjMKcO873295Rf72kN+JjVDheXo8ftrW/PyjJk7i/GFvae0/TuKhTJ5MGHg6HE+u3HMfh0tDojNsRa78/hNo6C7JSYnHNReHRyv5kXhVjV155JV577TX8+uuv2LhxIz788EN069YNQ4YM8RyzZ88evPfeexg5cqTXT65WqzF+/HgsXLgQ33//Pfbv349Zs2YhIyMDo0ePhsPhQGVlJcxm9w/ZwIEDMXjwYMyaNQt79uzBpk2bMGfOHIwZMwbp6ekQBAGXX345Vq5cia+++grHjh3DBx98gDfeeMOzju1056DQV1FrxLYC9zz4O67JAwBs+r0CJos9mLEoBIkXx+Gy4XNz4ubPR0oMwQ1yGk17jIV/MSYW/x3lcrnw6sd7sOtQJaLUSjw++XwkJ/pvqk5stApjR54FAHhv/X7Y7A6/PRf5ltlqx97CKgDAeX3Z0t5b2nj5bPxcb7LhyTc2YeW6/Xj3xyrsPFgV7Eg+V1bdgE9/co+K3XHd2Z4v2MONV3+rKVOmoGfPnpg0aRLuuOMO2O12zJ0713P/pEmTcMsttyA7OxtTpkyRFGDGjBkYO3YsHnvsMdx6661QKpVYuXIlVCoVSktLcdFFF+Grr74C4B7yW7p0KXJycjBx4kTcd999GDFiBJ544gnP+R544AFMmTIFixYtwlVXXYU333wT//znP3HTTTd5fQ4Kbd9uOgqnCzindwouGZwFbZwSFqsDv+7teLdM6nqcThdKq92FQFZq+BVjvRuLsT9kX4yJbe3Db8NnkViMlVU1wNmJ9vaf/FiIbzcdhSAAs8cP8RTc/nT1RWcgOTEKlbUmz4wEkr/fCqthszuRoolGd7a095omQRwZC+5IVGlVA2Yv+Rk7D7ob9LhcwEsf7sHRUnl/nkv15he/w+5w4tyzUpHvp3WpcuDV3IXo6Gi8/fbb2LZtG6qqqjB06FAkJSV57tdoNLjzzjtxxx13IDZW2kWLUqnE7NmzMXv27Fb35eTk4MCBlt2zkpOTsWTJknbPFxERgWnTpmHatGntHnO6c1DostmdWL/ZPQX1qj+dAUEQMPCMWPy414Afth3HyCHdgpyQQkVtnRlWmwMKhRCWozK5Oe69GP8orQPQsQ2A/c3lcqFSHBlLCr/3QJSujYFSIcBqd3b4G/df95Zg9Ze/A3B/gzy0f4YvI7YrUqXEraP7YOna3fjgu4O4bGh3xER5v3acgmP7/nIA7o2e/d0JM5xo493FWDBHxn4rrMJzq7eizmhFSmIUZt1yDlZ8shtFFRY8tXITFs4c4RnBC2V7C6vwvz2lUAjAlL+cHdY/p5LG+4YMGYIrr7yyRSEGAIsXL8asWbOQkNDxjSSJfGHT3lLo6ixISojE+We7L0YGnuG+iNt9uNKzeSzR6ZRUuaeMpWtjEKEMv6kRvbISIQhAjcGCepM8p5cZGqwwW93ZUjXhOzKmVCo8xaY4GivF4eM6LHx3B1wu4Ko/9cR1wwO7ruKy/O7ISomFocGKz34+EtDnpo7herGOEYsco9kOqy3wn5vfbTmKx1/7H+qMVpzZTYMX7rsYvXMScdPwZGQmx6Ci1oRnV22BJQjZfMnhdGHFp3sBAFcO64keGeFdX4TfFQZ1aV/9+gcA4PLze3guoLVxEejbUwOXC/hxBxeZk3dKKhubd4ThejEAiIqMQE5jE4+SmtbbfciB+OWJNj4SapUyyGn8S/w5K6+RVoxV1prw9KpNsNocGJyXhrvGDAj4N8hKpQLj/+zeduCTHw9DXx/89TTUvpKqepRWNUCpEDDwzOB2GQ41MVERnnVLuvrAfW46nC6sWvc7XvpgF+wOFy4amIXnpl6IpAR3cRgTqcDDt52LuGgVDhyrxUvv7+zUlOdg+27LUfxRYkBstArjrsgLdhy/YzFGYeNYmQG/FVZDoRBw5QU9W9w34twsAMCGbcfhcoXuBxQFTmmVu3lHVpgWY0BTE4/SWltwg7SjKzTvEGU1duwsq/Z+9N5otuGplZtQY7CgR0Y8Hr5tCJRBGsW98Jws9MpOhMlix783HApKBvLO9gL3qFi/M5I5pVQiQRA8UxV1AfrSwWSxY+7qLfjkR3dX81su74PZ44cgSt1ypVFmciz+MWkoIpQC/rvrBN5bvz8g+XytwWTDO1+7NyAfN7oPEhs7WIYzFmMUNr7+tQgAMLRfOlJOmtJ0Qf90qCMUKK6ox6HjusCHo5DjaWsfxsWY2MSjVKYjY2LzjtQwbt4h8jTx8HJkzOFwYsGa7SgqNUATH4k5d1wQ1AtrhULAxKv6AQC+3PgHp4TLWPP1YiSdOFVRX+f/z82KWiMeXvpfbP69DKoIBR7823n425V57W7gPiA3BdPGngsA+OA/B/HD9tCbDfTBdwehr7ciJy0OV114RrDjBASLMQoLZosdGxr3ufnzn1r/8sZEReCCAe5NxjdwPxzyQmlV1ynGjpRZUKWX3z41XWpkLNU9ZbTMyzVjb3z+G7YVlEOtcrewl0ODk0F9UnF2bjJsdif+FaLfyoc7i82BvYfFlvZcL9YRmgCNjB04WoMHXvoZf5S4v3B5buqFuHhwzmkfd9nQ7hg78kwAwJIPduH3I9V+zelLJZX1WPffplb24bheuy1d429JYe+nnSdgNNuRmRyLc89MbfOYUUO6AwB+3lkMm90ZyHgUYlwuV5coxvJ6JuGs7omw2l147dN9spvC27Thc9cZGSuvMZ72fVj33yP44hf3+tj7xw3GWd21fs/nDUFoGh37fusxHC+X94biXdFvhVWw2p1IToxCj/9n77zDori6MP4uvUtRRBFBVFhEOmLvsUJUNHaxl2jU2Gs0tthjr1Gxx16x94ahCAoKWECl994X2Pv9se5+IKAgO7MD3N/z8Cgzs3NfdnfmzLn3FANa0v5H0PmSp8VkztiTl1FYvMcTaZn5MGmghb9/7wS+se73X/gFtz4WaGfdAIVFQvx12Ediz7iOu0cQCosIHPj6cKxFkwUVKm3/NZ6ennj48CFyc3MhFJZ8qOXxeFi7dq1UxFEoFYEQgptfCnf0bmtS7vK9jVk96GopIyUjHy9C4tDWqiGbMinViNTMfOQJiiDHA+rr1lxnTF6Oh6mulpi/6zkCQ5Nxxzscvb7Kt5Ql4lA3Lqz6MI2+jhrkeEB+gRCZueVPFvkGx+HgFVGVsTHOLdDemlv3Mb6JLlpbGsA7KA4nb73FojGtZC2JUoziVRRrcqlwJtH+ksOUzoAzRgjBv7ff4fRdUVsnpxYGmDvSvtIhyHJyPMwebo+EVE+ERqZh1SEvbJrZCRqq3M0RfPU+Ad5BcZCX42FCv5aylsMqlV4Zc3d3x4QJE3Du3Dk8f/4c3t7epX4oFDb5EJmGsKh0KCrI4SenxuUeJy/HQxd7UZ8xGqpI+RbiWcR6OmqSylk1lYZ11dHdRtRz7NDVN5LVKC5Qm8IUFRXkUO/L35mSVVjmMZ9i0rHpxAsICdDDqTEGdW3GpsQK49bHAjwe4BkYgw+RqbKWQymGXwjNF6sqOlrMhCnmFxRh0wk/iSPm2qUZloxz+uFcUBUlBSwb3xp166ggKiEL64/6oLCIm1FBRUVCHLzyBgDg3L4JjGpZI/JKP2WcOHECP//8M3x9ffHw4UM8ePCgxM/9+/eZ0EmhlMuN56JVsQ42DaGlrvTNY8VNn1+ExNPyy5RyEVdSrMkhisVpbaYBvrE2cvOLsOPsS06EK+bkFSArV1TlsTYU8AD+X7kzJbO0M5acnotVB72Qm18E62Z1Me0XG86ubBg30EKXL7ktx26EyFgNRUxsUjZivpS0tzUrO5yf8n3+X01ReitjKRl5WLLnGZ6+ioa8HA8zhthi/M+WkC8n0qei6GqpYPnENlBRkkfAhyTsuxjIifv719z2Dkd4XCY01RQxvKe5rOWwTqWdsaSkJPzyyy9QUvr2Qy+FwgaZOQI8fRkNAOhbRuGOrzFuoIWmjeqgsIjg6atopuVRqinihs81uax9ceS+hCsqKYoM9q0vlUllibiSooaqYq0pv92gHGcsL78Qa9y9kZSeh0b6Glg8phXnE9tH9OJDQZ6HV+8TEfAhUdZyKPh/FUWLJrq15ppiAnE1xbRM6UzofoxOx9xtj/E+Ig2aaopY/Ws79GxtLJVzA0CThnUw380Rcjzgtlc4rjwJk9q5pUFWjgAnbooK/ozsxYeGWu3zLyp9N2/RogU+fKA9RCjc4L5vJASFQjRpqAVz44olsXdzEK2O3aehipRyiJEU79CQsRL2MNBTwxhnUeNed48gxCXLNuFbEqJYC/LFxIi/b8WdMaGQ4O9//RAalQ4tdSUsn9CmWjysGOipS/o9HrvBveIwtZHi+WKUH0e72MpYVb/XXm9isXDXU8lEy+bfO8GqqfQbcTu1MMD4L3lY7h5B8H4TK/UxfpRTd98hM0eAxgaa6N3WRNZyZEKlnbElS5bA3d0dFy9eRFhYGGJiYkr9UChsIBQS3PwSotinXZMKh+x0smsEeTkeQiPTEBGXwaRESjUltpatjIlxaW8KS1M95AmKsPPsKwiFsnuATqxFlRTFSMIUi+WMHbkeDK83cVCQl8PScU7VKnR2SA8zqCjJ431EGrw49PBXGxEUFCFQXNKe5otVCbEzVlAoRH7hj90jCSG48OAD1h7xQZ6gCLZm9bBpZic0ZHACsF9HU/RpawJCgE0n/RAWlcbYWBUlMj4T179Uhp3Yr6XMmtbLmkr/1cOHD0dsbCyWLFkCFxcXdO/evdQPhcIGgaGJiEnKhqqygiQ/oSJoaypLZgYf+kUxJY9STRGVta9dOWNi5OR4mDXMDspK8ggMTZJMdsgCcZhibSjeIaZ4mCIhBLf++4xLj0IBALOG2aFFEz1Zyqs0Opoq6NepKQDg+M23KJKhc1/beROWDEFBEfTqqMCkgZas5VRrVJQUoKosKkaelVtU6dcXFBZh+5mXOHI9GIQAfduZYMXENoxXOuTxeJjsagVbs3rIFxRhtbs3ktNl25zd3SMIRUICpxYGsDOvvZMElS5tv2bNGiZ0UCiV5sbzzwCArg6NJDfGitLN0Qg+wXF46BeJUX0sqpwkS6k5pGXlIze/CDyeKHSvtmGgp45xzi2w79JrHL4eDAeL+jDQY98pjf8SplivFjljBnpq4PEAQSHB04BY7L0UDECUf1WRZq9cZGCXZrj5/BMi4zPxyC8S3VuVX/GWwhzifDF7c33OFn6pTuhoKiM3vxBZeZWrTpielY91R30R9DEZcnI8TO7fEs4dTBlSWRoFeTksHN0KC3Y+QWR8Fla7e2P9tA5QqeQzlDTwexuPFyHxUJDnYUI/S9bH5xKVfvddXV2Z0EGhVIrk9Fx4B8UBqFjhjq9xsqwPdVVFJKfn4XVoImzNau+MDKUkkrL22qpQVJCXsRrZ0KddE3gGxuJ1WBK2nX6JtVPbl9u/jykSU2tfmKKigjzq1lFBYloe9lwMAiFAF4dGGNbDTNbSfhh1VUX80q05Dl8Lxsnbb9HJzrDWXleyROyMOdSiRrpMoqOlgpik7EqtjEXEZWC1uzfiknOgpqKAhW6tYC+DkFENVUUsn9AGc7c/QVhUOv7+1w+Lxzixeo8vLFbK3qWDKRrWqz352WXxQ8GZ8fHxuHHjBi5fviz5uXjxIk6ePInZs2dLWyOFUoo7XuEQCgksTfVg/AMhF4oK8uhkawiAFvKglCRWUryjdoUoFkdOjoeZQ22hoiSPoI/JuOb5kXUNCbWo4XNx6n/5ewkBLE31MHOIbbVfyXDuYApdLRUkpubiJgcqddY24pKzEZ34paR9c1rSXhqI88ay8irmjPm/TcD8nU8Rl5wDAz01bJrRUSaOmBgDPXUsHecEBXk5eL2Jw7EbwayOf+P5J0QlZKGOhhKG9ah9pey/ptIrY7du3cK8efNQWFgoMRCEEMn/TU3ZW26l1E6KioS47R0OAOhThco73VoZ4eZ/n/Hf61jk5BXQUr8UALWzkmJZGOipY/zPlthzIRBHr4fA0aI+o8nlxREUFEnKRtemnDEAMKynjjcfU2Cgq4olY51qxCqSsqI8hvc0x+7zATh77z1+atWY3m9ZRNzomW+iC3WG85JqCzoaImcsuwJhiteefcSBy68h/DLBsnhMK9T58npZ0qKJHn4fZoe/T/rhwsNQNKynIdWS+uWRkS3AqduixtajelvQ7yR+YGVs3759sLS0xMWLFzFw4ED0798f169fx/z58yEvL48lS5YwoZNCkeATHIfk9DzU0VBCO+sGP3we88Y6aFhXHfmCIjwPpJW+KCJqayXFsujd1gQ2zetCUFCE7adfslZdMTFNtCqmoiQPTbXaZahd2hujnYUGlo51+G4T++rET06N0aCuOtKzBLj6lP2V1trMC0lJexqOLy20tb6sjH0jTLGwSIi9FwKw/5LIEeveygirp7TlhCMmpot9I0mT5T3nAxAYynxPwFO33yIrtwAmDbTQgwXnrzpQaWfs06dPmDRpElq0aIHWrVvj7du3aNq0KcaPH4/Ro0dj3759TOikUCSIC3f0cDKu0qwxj8dDt1ainmMP/WioItMQQrDlVAD+vhSLuC9ly7lIba2kWBY8Hg8zh9hBVVkewZ9S4PGMnYfo+JT/F++o7iF6lUVfRxU97bRrXK6cgrwc3HqL+thdfBiK9CzpNMylfJviJe0dab6Y1BA3fi6vgEdWbgFWHvTCjeefweMB41xa4Pehdpxc6R7e0xyd7AxRJCRYe8QXUQmZjI0VHpeBG19ClScNaEmLp32h0s6YnJwc6tSpAwAwNjbGx48fIRSKvoydOnVCaGiodBVSKMWISczCq/eJ4PEgleaAXe1FzlhgaBISOOwg1ATu+0bCOzgBmblFOHAlhJNNYEVl7WnOWHH0ddUw/mdRs9Bj14MRnZjF+Jji4h31a1m+WE2nvU1DmDasg9z8Qpx/8EHWcmoFbz6KStrratGS9tJER7P8lbGYpCzM3/EEr94nQkVJHkvGOmFg1+acnVji8Xj4fagd+MY6yM4twKqD3sjIFkh9HEIIDl15A6GQoK1VA1g3o/mLYirtjJmamsLf31/yf4FAgLdv3wIAMjIyIBBI/wOkUMSIk78d+PWl8qCmr6sG62aibvcP/enqGFOkZebD3eON5Pc3H1PwgIOFUzKyBcjOEzXclUU5d67Sq40xbM3qQVAoxLZT/oz3ixIX76hXw1aHajtycjyMdhatjl33/ITEVNn2OKoNSKoo8mlJe2ny/5Wxks7Y67AkzNv+BFEJWairrYoN0zuiTcsfT6dgCyVFeSwd1xr6umqITc7G2iM+KCisfA+1b+EbEo+X7xOhIC+H8T/X7lL2X1NpZ2zYsGHYvn07tm7dCk1NTbRp0waLFy/G8ePH8ffff8PSkr7BFGbILyjCfd8IAECfdiZSO29XB9Hq2APfSE6u1tQEDl55g8ycApgYaKKrtWh29tDVN5IiDVxBvCpWV1sVyorcCyeRFeJwRTUVBbwNT8XVJ2GMjpcgKWtPV8ZqGvbm+rA01UNBoRCn776TtZwaj1+IOF+MhihKE3E1xew8oSSX9q53OJbvf47MnAKYNdbGlt87wdSwjixlVgptTWUsn9AaaioKCPqYjF3nAqT2TFRQKMShL6Xs+3cypZOdX1FpZ2zw4MFYunSpZAVs1apVyM/Px19//YXCwkIsXbpU6iIpFADwDIhGZk4B9HVUpWpY2lk3gLKSPGKSsvEuIlVq56WIeBESj8cvoyDHA6YMaIEOLTRhbKCBzJwCSZ8RrhBDi3eUSz0dVUzoJwpXPH4zBJHxzOUViEOGa1reFEXk2I/p2wIAcM8nnNH8lNqOqKR9FuTkeLAxoyFh0kRchENIgIwcAdw9grDj7CsUFhF0tDXE2mkdoKOlImOVlcfYQAsL3VpBTo6HBy8ipRZOfN3zI2KSsqGtqYwhP1XfvolM8UN9xkaOHImFCxcCABo3boybN2/i+fPnuH//PszNab8ACjOIC3f0amMi1aRPNRVFtLUShRE88OVe6Fx1Jje/EHsvBAAA+nVqClNDLcjL8TC5fwvI8YDHL6Pw4kvZZS5A88W+TQ+nxrDn66OgUIjtp18yFq5YW3uM1RYsmujCqYUBhAQ4ceutrOXUWPy+VFG0MNGFBi0fLlUUFeQklV43nHiFS49E9RKG9zTH/FEO1Tqywp6vjymuVgCAYzdC4BkQU6XzpWfl4/Qd0Sr46D4WtK1FGfyQMwYAYWFhOHbsGDZv3oyEhAR8/PgRWVnMJ3ZTaidhUWl4F54KBXkeerRuLPXzd3cUhSo+eRUt9Tjp2sy/t98iITUX+jqqGNGLL9nerFEd/NyxKQBg74UA5OYXykpiCWLElRRpCEWZ8Hg8zBhsC3UVBbyLSMXlR9Iv2FRYJERK+hdnjIYp1ljc+lqAxwM8A2IQGpkmazk1En9a0p5R6miIWk98jM6AkoIc5o9ywIhe/BqRm9e3XRP06yjqG7zlXz+8r0LU0Ilbb5GdV4imjeqgeyvpP7/VBCrtjAmFQvzxxx9wcXHB2rVrcejQISQlJWHPnj3o378/4uLimNBJqeWIC3e0tWooSZyVJlbN6kGvjgqycwvgE8ydlZrqzIfI/+cWTfvFBqrKJXvMj+zNh76OKhJSc3GSI7Pjkh5j9agzVh51tVUxsb9o1vTErbeIiMuQ6vmT0/MgJKJS6Noc6sdDkS4mDbTQ2b4RAODYjWAZq6l5FBQWSXpG2ZtTZ4wJdL+EIdbRUMLaae3Rya6RjBVJl/H9WsLRoj4EhUKsdveW5PJWhk8x6bjj9RkAMKm/FeRoKfsyqbQztmfPHnh4eGDNmjXw9PSUJPfNnz8fhBBs3bpV6iIptZvs3AI88o8CIN3CHcWRl+OVKORBqRqFRULsOhsAIQE62zUqM8dPVVkB036xAQB4PA2r0sybtPh/mKKGjJVwm+6tjOBoUR+FRUJsO/0SRUVl99r5ERIkPcZUqeGu4YzsxYeCPA8v3yey0my2NhH0MRl5giLoaCpXqyIS1YlBXUzRqrk61k5xgrmxrqzlSB15OR7mj3KASQMtpGXmY/Uhb+TkFVT49YQQHLzyBkICdLBpCEtTPQbVVm8q7YxduHABM2fOxKBBg6CtrS3ZbmFhgZkzZ8LT01Oa+igUPPSLRL6gCEb1NdGSwYu5q4NoVsvvbTznqvxVN648DsPHmHRoqiliYv+W5R7nwK+PznaNICTAzrOvUCjFh/rKkpkjQFauyNAY0Fylb8Lj8TB9sA3UVRXxITINF6UYrvj/Soq0eEdNx0BPHb3amAAAjl3nZu/B6oo4X8yelrRnDL6xNpxb6aCuds29V6mpKGLZhNbQ1lTG59gMbDrhV+FcYa83cQgMTYKighzGutBK69+i0s5YUlISLCwsytxXv359ZGRIN2SFUjly8wvxISZPqjPVsoQQIinc0aetCaNGpbGBFpoZaaNISPDkZRRj49R0YpOy8e+XZN3xP7eUlAAuj4n9W0JTTRGfYzMkSdCyIOZLM2NdLRWofBVSSSmNXh1VTB4gClf89/ZbhMdK594vKd5B88VqBUN/MoOykjzeRaTCO4imOUiL//cXoyXtKVVDX0cNy8a3hpKCHF6ExMP96verIBcUFuGwRxAAwLVLM6n0ha3JVNoZMzY2xuPHj8vc5+PjA2Nj4yqLovw41zzDcfJREg56cCMHp6oEfUxGZHwmlJXk0e1LkQ0mERfyeOBHQxV/BEII9pwPgKCgCDbN66J7q+9/ZtqaypKS6afvvJMU0WAbmi9Webo6NIJTCwMUFhFsO+0vlZXNRPHKGDXetQIdLRVJoYBjN0IYbyheG0hIyUFkfBbkeIAdLWlPkQJmjXUwZ4QDAODq04+48fzTN4+/+uQjYpOzoauljF+6NWdDYrWm0s7YmDFjcOzYMaxatQrPnz8Hj8dDeHg43N3d4e7ujhEjRjChk1JBWpqK4pYf+EXj6ctoGaupOje/rIp1sW8EdRZK83a0NYSCPA9hUelSm+mvTTz0i8SrD4lQUpDDtF9sKryS2c3RCLbN60FQKMRuKTaarAySfDFaSbHC8Hg8/DbYBhqqigiNSscFKfSkoWGKtY+BXZtDQ1URkfGZeOxPJ8KqinhVzNxYFxpqSjJWQ6kptLdpiNF9RZFx+y+9llTr/JrUzDycufceADDGuUWp4l2U0vxQ0+dZs2bh4sWLmDx5MgghmDNnDrZu3Yrx48dj+PDhTOikVBALEx10tNQEAOw6/wpxydkyVvTjpGbm4flrUX+LPm1NWBmzjoYyHC1EYR0PXtCHgsqQnpWPg1dEYQnDepqjYSWKYPB4PEz7xQZKivIIDE3Cfd8IpmSWS0wy7TH2I+hqqWDKQGsAwOm77/ApJr1K50tIEYUp1qNhirUGDVVFyez5yVtvaXuRKiLOF3OwoFUUKdLll27N0c3RCEIhwYbjvggvo5ru8RshyM0vhFljbXSxZz6iqSbwQ33GpkyZgmfPnmH//v3YtGkT9u/fj6dPn+L333+Xtj7KD9DFSgtmjesgJ68Qm0/4ybQoQlW45xOBwiIC88Y6aNpIm7VxxeGQj/wja0zuHRscvPoGmTkCNGmoBdcuzSr9+gZ11TGyl6hp/KGrQUjNzJO2xG8Sm/glTJFWUqw0ne0M0abll3DFUy9/+J4jFBIkpomcsfrUGatVOHdoAl0tFSSk5uLWf+GyllNtKSgsQsAHUWVKmi9GkTai4k22sDTVQ05eIVYd8i5R8OxTTAbufZlMpaXsK84PN33W0NBAp06d8PPPP6Nz584lKitSZIu8HA8zB1tBXVUR7yJSceJmiKwlVZoiIcGtL73FmCpnXx6OFgbQVFNESkY+Aj4ksTp2dcX/bQIe+UVBjgdMH2wLBfkfu7X079QUpoZ1kJVbgIOXv58kLE1iaM7YDyNe2dRUU8LHmHSc+xKiUlnSsvJRWCSEnBwPenWk30+Qwl1UlBQwrKdoMubMvXecaQRf3Qj+mII8QRG0NZVh2pCWtKdIH0UFOSwZ64QGddWRkJKDvw57Q1BQBEIIjt58B/KlpQ3fpOaV+2eKCgVyjh49usIn5PF4OHr06A8LokiHetqqmDHEFuuP+uLCw1DYNK8Hu2rU+NH/bTwSUnOhoaqIDraGrI6tqCCHTnaNcN3zE+6/iIA9v/q8b7IgL78Quy8EAABcOprCrLHOD59LXl4OMwbbYu72x3jyKhpdHY0kYaNMkpUjQGaOAICo3Dal8uhoquDXgVbYdMIPZ+69R+uWDSrd3ygpTbQaqldHBfI/6NBTqi89nBrj0qNQxCZl4+qTMAztYS5rSdWOF1/yxezN9emqBIUxtNSVsHxCa8zb8RRvw1Ox71IwGtYRIORzGpQU5THGuYWsJVYrKmTtfHx84Ovri7S0NBBCvvkjFNKwLq7Q3rqhJNdqyyl/1sO+qoK4nP1PTo2hrCjP+vjiUEWv17GVanJYGzl5+y0SUnJQT0cVo3qX3faiMjQz0ka/Tk0BALvPB7Dy/sd+yRfT0VSmycZVoKOtIdpZN0CRkGDrKX8UFFbOHohDFGlZ+9qJgrwcRvXmAwAuPgpFRrZAxoqqH+J8MUcaokhhmEb6mlgythXk5XjwfB2Hi89TAAC/dG2GerQAU6WokDM2ceJENGzYEB8/foSamhoGDx6M/fv34/jx42X+ULjDhP4tYWygibTMfGw79RLCalA2OC45W1INiq3CHV/T3EgbjfQ1ICgU4llAjEw0VAdCI9Nw9UkYAGDaIBupOTIje/Ghr6uGpLRcnLjFfJuGmERavEMa8Hg8TB1oAy11JXyOzcCZe+8q9frELytj1JDXXjrYGMK0oSjn+bwUqnNWlSIhQXqWAAVF3LedCak5iIzPhBwPsDWnJe0pzGPdrB6m/WIDACgSAnpaynDtWvmc8dpOhZyxefPm4f79+zh58iRMTEzw999/o127dpg1axbu3bsHgYDOXnEVZUV5zHdzhJKiPPzfJeDy4zBZS/out73CQQhg27weGtaTTTEFHo8nWR2jVRXLpqhIiJ3nXkFIgE62hlINJ1RRVsBvX27w1559xLvwFKmduyzEK2O0eEfV0dZUxtRBouqK5+5/QGhUWoVfKw5TpMU7ai9ycjy4fSmffe3ZRySnSzeio7BIiJSMPHyKSUfA+0Q8eRkFj6cfceJWCPacD8C6oz5YtPsZpm28j5HLb8J1wVVM3vAYf1+MQWBYslS1SBvxqpi5sS40aUl7Ckv0bG2MQV1MIScHjHPhQ0WJRpdUlkq9YzY2NrCxscGiRYvg6+uLGzduYPny5RAIBOjRowecnZ3Rrl07yMnRWH8uYWyghUn9W2L3+QAcuxGMlk31qpTXwyQFhUW46yOqpMV24Y6v6epghOM3QxD0MRlxydk0l+grrjz5iI/R6dBQVcTEAS2lfn57c310cWiER35R2HUuAFtnd/7hwiDfQ9JjjK6MSYUONobwtInBs4AYbDvlj62zO0NR4fvhxuIwRVrWvnbjwNeHpakegj4m48Kjj+hoVn7uU0FhETKyBUjLzEd6tgAZWaJ/07PykfHl3/QsATKyRf9m5f5Y2HNeAcGG4y8xf5Q82lk3/NE/jVH8QkQRJQ40z5nCMkO6N4WlQT4saTuFH+KH3FcejwcnJyc4OTlh+fLl8PLywo0bNzB16lRoaWnB09NT2jopVaRXG2O8+pAIz4AYbDrxAtvndIGaCvNNlCvL88BYpGcJoKulgtaWBjLVUldbFTbN6uHVh0Q89IvC8J40mVxMXHI2Tt4WhQ+O/9kSOprMVL6b2K8l/EIS8Dk2AxcfhmLIT2aMjEOdMenz60BrvA5LQnhcJk7ffQ+3Pt/PJxSHKdKGz7UbHo+H0X0tsHDXMzz0jwEp0MTLyA/IyRciPUuA9Ox8ZHz5Nyev8lUX5XiAproStNSVUUdDCXXUlaH15d/iv2triP4lRQVYf+Q/hETmYsMxX0wfbIserY0Z+Mt/nIJCIQJDaUl7iuygBWN+nCqvJQYEBODx48d49uwZCgoKoKtLS1lyEXFviA8RqYhLzsGe84GYO9IePB63Lp6bX8rZ92pjzIlqal0djUTO2ItIDOthxrn3SxYQQrD7fAAEBUWwblYXPzk1ZmysOhrKmDSgJbb864/Td9+hvU1DGDIQuhqTlAWAOmPSpI6GMqYOssH6o744/+ADWlsafHNFnhCCJHEBD126MlbbadFED61a1IdvcDweBGQAKN1cVoycHA911JVQR0MZWl/+raOuBC2Nr50t0T4NNSXIV+LBMSdHiF/a6+LZe4KH/jHYcfYVsvMKMKAzd3Jjgj8lIze/CNoaypWuYkqhUGTLDzljL1++xM2bN3Hnzh3ExcWhSZMmGDRoEPr27YumTZtKWyNFSmioKmL+KEcs3P0Mj19GwdasHqMP0pUlPDYDQR+TISfHQ6823Jh1bGfVAHsvyCM2ORshn1PQoomerCXJnEf+UXj1PhGKCnL47Rcbxh3ULvaiUEX/dwnYfS4Af01tJ9Uxs3MLkJ4lynttSJ0xqdLeuiE62RniyctobDv9Ettmd4ZSOdVRc/KFyC8QVV+sp01XxijAFFdrKMgFIiszA40a1oOetppk9UqyqqWhDHUVRcZn5eXleJgywALaWmq49CgUh64GITOnAKN68zkxSSfOF7Pn05L2FEp1o8LO2NcOmJGREfr3748+ffqAz+czqZEiRfgmuhjZi4/jN0Ow71Ig+CY6aKSvKWtZAP6/Ktba0gB6dbjxMKairIB21g3x4EUkHryIrPXOWHpWPg5eETVjHt7TnJUCKzweD1MHWWP65od4HZaEuz4R6CnFECFx8Q5tDWVOhu5Wd6a4WiMwNAmR8Zk4dedduf1n0nOKAIjaC5TnsFFqF/V11TBrqDVCQkJgYWEONTXZrpjyeDyMc2kBTTVFHLsRgrP33iMrR4AprtYyd4DEFYhpvhiFUv2oUBxYly5dMGLECNy7dw/Ozs64cOEC7t69i9mzZ0vFERMKhdixYwc6duwIW1tbTJo0CZGR5VewS01Nxdy5c9GqVSs4OTlh5cqVyM3NLXG+gwcPolevXrC1tYWzszPOnTtX4hx79+6Fubl5qZ/awKBuzWHdrC7yBUXYdNwPgoIiWUtCbn6hpGphXxkX7vgacVXFZ6+ikc+B90qWHLr6BhnZApg00IJrF/ZCdAz01DGyl+he4+4RhNQM6VVYo/lizKKlriSpjHnx4YdyK2OmZYuuLdpjjMJleDweBnc3w7RB1uDxRD0x//7XD4VFsuuxmpiai4g4UUl7O3PqjFEo1Y0KrYzFxcVJKiTeunULt27dKvdYHo+He/fuVUrEnj178O+//2L9+vUwMDDApk2bMHHiRHh4eEBJqXR51pkzZyI3NxdHjhxBRkYGli5dipycHGzYsAEAsH//fri7u2PlypVo2bIl/vvvP6xYsQKKiooYMGAAAODdu3fo378/5s+fXymtNQF5OR7mjLDHzL8f4WNMOg5fC8IUV2uZanrsH4Xc/EI0rKsO62bc6o9i1bQu6mqrIiktFz5BcehoayhrSTLB/10CHvpFgccDpg+2YayyYXn062iKJy+jEBqVjn8uv8bC0a2kcl6aL8Y8bVo2EIWb+kdh2+mX2D6nS6nVr7QsUSEG2mOMUh3o064J1FUVseVffzx5GY2cvEIsHO0ok7Le4lUxs8Y6tKQ9hVINqdBdw9XVlTEBAoEA7u7umDdvHrp06QIA2Lp1Kzp27Ig7d+7AxcWlxPEvX76Ej48Pbty4IclPW7VqFSZOnIg5c+agfv36OHXqFMaPH4++ffsCABo3boyAgACcO3dO4oy9f/8eQ4YMQb163HrwZwu9OqqYPdweKw964dqzT7BtXg+tWzaQiRZCCG4+/wxAVM5e1uEeXyMnx0NXh0Y4d/8DHryIrJXOWJ6gEHvOBwAAXDqYwtyY/UI98vJymD7YFnO2P8GzgBh0DYqDkxQqbopXxmi+GLNMdrVCwIdERCVk4eSttxj3s2WJ/el0ZYxSzehk1whqKopYd9QXL0LiseKAF5aNbw11VXbDnSUhilLs9UihUNijQs7YunXrGBPw9u1bZGdno23btpJtWlpaaNGiBXx9fUs5Yy9evEC9evVKFApxcnICj8eDn58fevfujQ0bNqBJkyYlXicnJ4eMDFE1JoFAgM+fP8PU1JSxv6s64GhRH/07NcWVJ2HYfuYldjTSRl0ZJM6/i0jFx5h0KCnIoXsr7hQUKU43RyOcu/8B/u8SkJqRBx0tZkq5c5VTt98hPiUHdbVVMaq37HJEmzbSxoBOTXHxUSj2XghAy6Z6Vc7zomGK7KCppoTpg22x2t0blx6Hoq1VA/BN/u/Up+WIVsZoJUVKdcLRoj5WTW6L1Ye8EPQxGUv2eGLF5DaMtfv4moJCIQI+iEva0xBFCqU6IvPa4XFxcQCABg1Krsro6+tL9hUnPj6+1LFKSkrQ1tZGbGws5OTk0LZtWxgY/H/GPCYmBtevX0eHDh0AAKGhoSgqKsLt27fRq1cvdOnSBfPnz0dCQoK0/zzOM8bZAs0a1UFmTgH+/tcPRULCuoYbnp8AAB1sDTkbYtFIXxPmjXUgFBI8fhklazmsEhaVhstPwgAAUwdZy7zIxfBe5jDQU0NSeh6O3wip8vmoM8YeTpYG6OZoBEKAbaf9S+Rg/n9ljIYpUqoXlqZ6WDutA7Q1lPExJh2Ldj1DQkoOK2OHfP5/SfumhtqsjEmhUKQL+8HNXyEuvPF1bpiysjLS09PLPL6sPDJlZWXk5+eX2p6UlIRJkyZBT08PU6dOBSAKUQQAVVVVbN++HcnJydiyZQtGjx6Ny5cvQ0Wl8jNahBDk5LBz8/0W4vezeEGT7zHjl5ZYuMcLb8KScfLmG/zStertCSqqIzNHgGcBMQCAbvYNpP4e/sj7UR4dbOrjXUQq7vmEo2erhjLRUBV+REdRkRDbz/hDKCRo27I+WppoVekzktZ7MeFnPv464o/rzz+hdYu6MGus/UM6UtIykZopum9oq8mxeg1X5+9FVRjZsylevktAdGI2Dl8NxOg+5sjNzZXkjGmqsvs5FIcLnwkXNFAdlddgoKOIFRMd8NcRf8QkZWPBzidYOtYBhvWkN8lTlg6v19EAAKumusjLY/494sLnwRUdXNDAFR1c0MAlHYDIL6ho2wuZO2Nix0cgEJRwgvLz86GqWnqGVEVFBQKBoNT2/Pz8UmVvP378iMmTJ6OoqAjHjh2DlpYWAGDAgAHo1KlTiQbVzZs3R6dOnfDgwQNJrlllKCgoQEhI1WfppcXnz58rdXwfBy1c+i8V5x9+hIZcJoz1lVnR4RmSiYJCIQx0FFGQGY2QkBipjFtZHRVBT0kIeTkgPC4LD58HwECncqt40tAgDSqj43lIJj7FZEJFkYd2zeWk9h2v6nuhCMCmiRoCPuVgx9mXmNK7PhTkK59r+Cr4IwBATVkOEZ9Dq6TpR6mO34uq0sdeA/8+zsf15xHQV8tFfR1F5BWIVuVT4sORnSLboA0ufCZc0ABQHZXVMKqLNo4/SEJSRj7+2O+FkV3qwlBPuhEfxXV4vxHli+mr57P6DMKFzwPghg4uaAC4oYMLGgDu6Chr8agsZO6MiUMOExIS0Ljx//OFEhISyiw1b2BgUKpao0AgQFpaGvT1/x8v7efnh6lTp6J+/fo4ePAg6tcvmdha3BEDRGGR2traZYZGVgRFRUU0a8Zeqe/yyM3NxefPn2FiYlKmM1seFhZAcu4bPHkVi6u+mdg4zRIaaj8ejlYRHUIhwd5bngCAfp2aoUWLRj88XlV0VAbHkEJ4BycgMl0FXdtVrBWCtDX8KJXVkZCai8fnngMQhbM62Ve9cIk034tGxgLM2fEciekFeJ+kgkFdKp4DKtahoKILIAGG+pqwsLCokp7KUl2/F9LAwgKIyQzCI/8Y3PTPwqSfzQDEQF1VAbbWlt99PVNw4TPhggaqo2oaWvAFWHf8JT5GZ+D4w2QsGGULyyZVL3r0tY7k9DwkpIkq3PbuZAUtdebD/LnweXBFBxc0cEUHFzRwSQcgSomqKDJ3xvh8PjQ0NODt7S1xxjIyMhAcHIxRo0aVOr5Vq1bYvHkzwsPDYWwsavzq4+MDAHBwcAAABAYGYuLEiWjRogX27t0rWRETs3XrVkmJfvESYlRUFFJTU3/YoeLxeDJvSFkcVVXVSuuZPsQeoVGPEJOUjQMeb7FkrFOFl1h/RIf/uwTEp+RCTUUBPVqbQkWZua/jj7wfZdGjtQm8gxPg+ToekwbYQL4S5d2lpaGqVEQHIQSHTwQgv0CIlk314NyhWZW/C5XV8D3U1NQwaYA1/j7ph4uPPqGro3GlG5infAmNM9TXlNlnU52+F9JkykBbvAlLQVxKLo7eEuUk1tOune8FVzVQHT+mQU1NDeumdcBfh30QGJqEdcdeYqGbo9QqFot1PA0UFe4wM9KBQT1tqZy7shpkDRd0cEEDV3RwQQNXdFTmmalCT5K+vr6V+qkMSkpKGDVqFDZv3oz79+/j7du3mD17NgwMDNCzZ08UFRUhMTEReXmiJq82Njawt7fH7NmzERgYCC8vLyxfvhwDBgxA/fr1UVhYiHnz5kFPTw/r169Hfn4+EhMTkZiYiJQUUbPRHj16IDo6GitWrMCnT5/g6+uLGTNmwN7eHh07dqyU/pqEqrIC5rs5QkGeB683cbjxpdw8U9x8Lirc0c3BiFFHTJo4WNSHlroS0jLz8fJ9oqzlMMZj/yj4v0uAooKonLw0HTFp0tnOEPZ8fRQWCbHrXACElSxAE5csyk1qqEeLd7CNhqoiZgyxAwBExIt6vdXTrl1VSik1FzUVRfw5sQ1aWxqgoFCItUd98eBFpFTHkJS0p1UUKZRqTYWegN3c3Mp9GCNE9PBTfH9l45ZnzpyJwsJC/PHHH8jLy0OrVq1w6NAhKCoqIioqCt27d8e6deswcOBA8Hg87Nq1CytXrsSYMWOgrKyM3r17Y/HixQBEq2Lh4eEAgJ9++qnEOIaGhnjw4AFatmyJAwcOYPv27Rg4cCCUlJTQvXt3LFy4kLMPnWzRrJE2xrpY4uCVNzh09Q1aNNFFk4Z1pD6OuIEyAPRuZyL18zOFgrwcOts3gsfTj3jwIhKONbCvS3pWPg5ceQMAGNrDDIb1NGSsqHx4PB6mDbLBb5seIOhjMu54h6N3W5MKvz7uS8WzBhz+G2sy9nx99GxtjDveonu2LFprUChMoaQoj8VjWmHH2Vd48CISW0/5IytXgH4dq14kq7CoWEn7GmiHKJTaRIWcsWPHjkn+HxMTg2XLlmHQoEHo06cP6tWrh7S0NDx48ACnT5/GqlWrKi1CXl4e8+fPx/z580vta9SoEd69e1dim56eHnbs2FHmuezt7UsdXxZt27Yt0duM8n/6dTTFq/eJeBESj00nXmDL752lvnJ12yscQiIqCWxsoPX9F3CIbo5G8Hj6EV5vYpGVWwANlht8Mo27RxAysgUwNtDEwC7NZS3nu9TXVcOo3hY4dPUNjlwLgpOlAXQr2AcuLllUcYk2fJYdE/pZwu9tPJLT82CgS50xSs1CXl4Ovw+1g4aqIq4+/YgDl98gO6cAw3qaV2nyN+RzCnLyCqGlroRmjbSlJ5hCobBOhcIUnZycJD8XLlzA2LFjsWLFCrRu3Rqmpqawt7fHvHnzMHbsWBw+fJhpzRSG4fF4mDXMDrpayoiMz8I/l19L9fyFRULc8f4MAHBu1+TbB3OQpoZ10NhAEwWFQngGRMtajlR59T4BD15EgscDpg+xhaKCzFsRVoifO5qimZE2svMKsf9SYIVeIygUSsra0x5jskNNRRGL3ezQoYUmOtlWvGUEhVJdkJPjYWL/lhjZmw8A+PfOOxy48qbSYdXF8QsRhSjam+tDTq52R/RQKNWdSj9pBQYGlruiZGdnJ+nhRane1NFQxtyRDuDxgLs+EXj6UnpOh3dQHFIy8qGtqYw2VtJJaGYTHo+H7o5GAID7vtLNAZAleYJC7D4fAEDkJPONq179iy3k5XiYOcQWcnI8PA+Mhdeb2O++JiVTVLxDQ1WRs83GawtG9TXwk20dqKlUj9xRCqWy8Hg8DOthjimuVgAAj6cfse20PwqLhD90Pr+3CQBovhiFUhOotDNmYGCAp0+flrnv1q1bJcrTU6o31s3qYXB3MwDArvOvEJecLZXzigt39HBqXG1WXr6ms30jyPFEoSKxSdJ5X2TN6TvvEJecg7p1VODWl90y79KgScM6GNhFVA1138VA5OQVfPP4lKwiAEBDKTZlpVAolG/h0sEUc0fYQ06Oh4d+UVh/1BeCgqJKnSMlIw+fYzPA4wF25tQZo1CqO5V+Eh43bhwOHz6MuXPn4tq1a/D09MSVK1cwdepUnD9/HtOmTWNCJ0VGjOhpDgsTXeTkFWLzCb8fnsUTE5WQiYAPSaK+KG1MpCNSBujVUYWtmcgISrtCliz4GJ2OS49F5cV/HWgNNZXqmQc3rKc5GtRVR3J6Ho5eD/7mseKVsQZ6tHgHhUJhjy4ORlg61glKCnLwDorDigNe3508Ks6rD8kAgOZG2qijocyUTAqFwhKVdsaGDRuGP/74A15eXpg3bx4mTJiAhQsX4u3bt9i8eTP69OnDhE6KjJCXl8O8kQ5QV1XEu4hUnLhZuUqZX3Pzv88AAEeL+tDXlX0viqrQ9Uuo4gO/yCrF/suaIiHBznOvIBQStLduKLVeOLJAWVEev/1iA0D0XQv5lFLusRJnjOaLUSgUlnGyNMCKyW2hqqyA12FJWLrXE+lZ+RV67av3SQAABz6tokih1AR+KEZs1KhR8PT0xI0bN3Dq1CncunULDx8+hLOzs7T1UTiAvq4aZg6xBQBceBiKl+8Sfug8eYJCSY5V32pYuONr2rQ0gKqyAhJSchD8KVnWcn4Yj6cfERqZBnUVBUz+ks9QnbFpXg8/tWoMQoCd516hoLDsECCxM0bDFCkUiiywaloXa6e1h5a6EkKj0rFo9zMkpuZ+8zVFQoLAMNEkE80Xo1BqBj+csJOeno5Pnz7h7du30NLSwsePHyU9xyg1j3bWDdHnS/+mLaf8kZqZV+lzPHsVjezcAujrqtWIOHcVJQV0sBFVf6uuoYrxKTk4cUu02jnuZ8sKl4TnOuP7WUJbQxmR8Zk4/yC0zGNSsujKGIVCkS3NGmlj/W8dUFdbFVEJWVi4+ymiE7PKPT4ySYDc/EJoqimhmZEOi0opFApT/JAztnfvXnTu3Bm//fYbVq1ahdjYWKxbtw6DBw9GRkaGtDVSOMKE/i1hbKCJtMx8bDv1stKheTeefwYA9G5jDPkaUopXHKr4LCAGeYJCGaupHIQQ7LkQgHxBESxN9dDDyVjWkqSGppoSJg1oCQA4e+89IuMzS+wXFBQhI0e0YtZAjzpjFApFdhjV18SG6R1gWE8diam5WLjrKcKi0so8NjRGNBFqb65fY+wohVLbqbQzduLECezcuRPjxo3D2bNnJatho0aNQmRkJLZv3y51kRRuoKwoj/lujlBSlIf/uwRc/lLwoSKERqbhQ2QaFOR5Neqh37KJHvR11ZCbXwjvN3GyllMpnryMhv/bBCjIy+G3X2xqXK+ajraGcLSoj8IiIXaefVVi8iA+RRQKpKaiAC11WtaeQqHIFn0dNaz/rSNMDesgPUuAJXs9EfSxdPj7hy/OmINF9Y8uoVAoIirtjB0/fhyTJ0/G77//DktLS8n2zp07Y9asWXjw4IFUBVK4hbGBFib1F604HLsRjPcRqRV63Y0v5ezbWTeEtmbNqf4kJ8dDV4dGAKpXqGJGtgAHroiaeQ/tYQaj+poyViR9eDwepg6yhoqSPEI+p+C212fJvrjkHACAgZ4aeLya5YRSKJTqibamMtZObQ9LUz3k5BVi+f7nePGluTMgKmkfn1YAHk+0MkahUGoGlXbGYmJi4OTkVOY+U1NTJCUlVVkUhdv0amOM9jYNUSQk2HTixXdL8mblFuDxl6bRNaFwx9d0+xKq+Op9ApLTv518zRXcPd4gPUsAo/qaGNS1uazlMIa+jpqkZ9qR68GSzycu5YszpqsqM20UCoXyNeqqilg5uS1atagPQaEQa9y98dg/CgAQ8KWkvWlDLVrSnkKpQVTaGWvQoAFevnxZ5r43b96gQYPqWxabUjF4PB6mD7aFvo4q4pJzsPt8wDeLtzx4EQFBQREaG2iiRRNdFpWyQ8O6GrAw0YWQAI/9o2Ut57u8DkvGfd9I8HjAjMG21bbxdkVxbm8K88Y6yMkrxP5LotXA/ztj1bu9AoVCqXkoK8pjyVgndLFvhCIhwd//+uHG8094+UE02W3bvK6MFVIoFGlS6aewX375Bfv27cOhQ4fw+fNnAEBOTg5u376N/fv3w9XVVdoaKRxEQ1UR80c5Qk6Ohycvo3HfN6LM4wghuPWlt1jftiY1NiRMvDr24EUEp6uKFhQSHLgqqp7Yp60JLGqgc/w18nI8TB9iC3k5Hv57HYv/XscgLlm0QmagR50xCoXCPRTk5TB7uD2c2zcBIcDeC4F4EZIIALA105OxOgqFIk0q7YxNmjQJrq6u2Lx5M1xcXAAAo0ePxqxZs9ClSxdMmTJF6iIp3IRvoouRvfgAgH2XXpeqWAcAwZ9TERmfBRUleUnlwZpIB1tDKCrIITwuEx+j02Utp1wev8lAfEou9OqoYIxzC1nLYQ2TBloY2LUZAGDfxUBEJohKR1NnjEKhcBU5OR6muFphaA8zAKIeY6rKcmhmWEfGyigUijRRqOwLeDweVq1ahfHjx8PLywtpaWnQ1NREq1atYGZmxoRGCocZ1K05Aj4kIjA0CZtOvMDmmZ2gpCgv2X/XRxTr3sXBCGoqirKSyTgaqopobWmAZwExePAiEk0bactaUik+x2bCM0TkMP860LpGfx5lMayHOTwDYhCTlC3ZRsMUKRQKl+HxeBjV2wKaako4dPUNrIzValzlWwqltlPplTFfX1/k5eXBxMQEw4YNw6+//oqRI0fCzMwM8fHx2LVrFxM6KRxFXo6HOSPsoaWuhE8xGTh8LUiyLzO3CD7BCQCAvu1MZKSQPcShio9fRqGwSChjNSUpLBLinyvBIARwaqGPNi1rX26nkqI8pg+2/f/vCjxoqdcuh5RCoVRP+ndqikOLu6CPA10Vo1BqGpVeGXNzc4OFhQV2796Nhg0bltgXFxeH3bt3Y/r06VITSOE+enVUMXu4PVYe9MK1Z59g27werEzrwD8sG0VCAr6xDpo0rPkGxN5cH9oaykjLyof/2wQ4WRrIWhJikrJwzycC930jkJKRD2VFHsY5m8talsywalYXPZwa465PBPS0FGpsDiOFQql5qKsq1pp7VlFREQoKyq/UnJ+fL/lXTk42Rai4oIErOriggW0dioqKkJeX//6BFaDSzhgAxMfHY+DAgdi2bRvatGkjFSGU6o2jRX0M6NwUlx+HYfuZl1g3tTX8QkXhYH1qYDn7spCXl0Nn+0a48iQMD15EyswZyxMU4nlgLO76hONN2P+bhmqqKaKvgxZ0tVRkoosrTOzfEoryQF3VHFlLoVAoFEoxCCGIi4tDWlraN48TCoVQUFBATEyMzB7+uaCBKzq4oEEWOrS1tWFgYFDlSZIfcsb+/vtvnDx5EhMnTsS8efMwduzYKomg1AxG922BN2FJCI1Kx/IDvsjIKYKmmiI62DT8/otrCN1bGeHKkzB4B8UhM0cATTUlVsYlhCA0Kg13vSPw+GUUcvIKAQA8HmBnro+eTsawaqKFDx/esaKHy6ipKGJMX3OEhITIWgqFQqFQiiF2xPT19aGmplbuQ25RURHy8/OhrKwstdWJysIFDVzRwQUNbOoghCAnJwcJCaJUnKq29fohZ0xdXR27du3C1q1bsX79egQHB2PNmjW1ZvmcUjaKCnKY7+aIWVseISVDtFTcxa5hiYIeNZ0mDevApIEWPsdm4NmraMZXBTOyBXjkH4m73hH4HJsh2a6vq4YeTo3R3bEx6umIGhvn5NCVIAqFQqFwk6KiIokjpqf37fL9RUVFAAAVFRWZOiCy1sAVHVzQwLYOVVXRs1VCQgL09fWrNN4POWNiZs+eDTMzMyxduhRhYWGYNWtWVU5HqQE0rKuBqYNssOVffwDAT60ayVgR+3RzNIK7RxDuv4hkxBkTCgkCPiTirk8E/nsdKykWoqggh7ZWDUSrYM3q0opbFAqFQqk2iHPE1NRolVtK9UD8XS0oKJCdMwYAzs7OMDIywvTp0/H7779X9XSUGkBXByNk5+QhKSGuVvZx6mLfCEeuBeFdeCqiE7Ogoy6duOWE1Bzc94nAPd8IJKTmSrabNqyDHq0bo7N9I9bCIikUCoVCYQIaZUWpLkjru1ppZ6xVq1ZQV1cvsc3a2hrnzp3Db7/9hqCgoHJeSalNdHMwREhIxvcPrIHoaKnAzlwffm8T8OBFJAZ1Nv7hcxUUFsHrTRzueofj1YdEECLarq6igM72jdCjtTGacbCnGYVCoVAoFArl+1TaGTt+/HiZ2+vXr49Tp05JktkolNpMd8fG8HubgId+kXDt2LjSr/8cm4G73uF46BeFzByBZLv1l9Lsba0bQrkW5eJRKBQKhUJhnkWLFiE6OlryvO/n5wdCCBwdHREVFYXu3bvj2LFjaN26tYyV1hwq5IxdvnwZnTt3ho6ODi5fvvzd4w0NDauqi0Kp1ji1NIC6igISU3MREp5aoe7qOXkFePwyGne9w/EhMk2yXa+OCrq3aoyfWjVGg7rq5Z+AQqFQKBQKpQosXbpUUggDAEaMGIF169bB0dFRhqpqNhVyxhYtWoSzZ89CR0cHixYt+uaxPB4PAwYMkIY2CqXaoqwojw62hrjtFY7HL2PR1aJsd4wQguBPKbjjHY5nATEQFIhugPJyPDhZGqBna2PYmetDnhbjoFAoFAqFwjCampqyllDrqJAzdv/+fdSrV0/yfwqF8n26Ohjhtlc4vIPi0b55/RL7UjPycP9FJO75hCM6MVuy3ai+Bno4GaOrgxG0NZXZlkyhUCgUCqUKmJubY9WqVbhy5Qpev36NRo0a4a+//sKHDx+wd+9eZGRkoFOnTli/fj1UVFRQVFSELVu24Nq1a0hOTkajRo0wZswYDB8+XHLOCxcu4ODBg4iOjoahoSGGDRsGNze3CjU2Pn78OHbs2AEvLy/Iy8tDKBSibdu2sLW1xf79+wEA7969Q79+/fDo0SNs375dEqZobm4OAFi8eDF8fHwwffp0AEBAQAA2b96MkJAQ1K9fH7/++iv69u1bofdn0aJFuHTpUqnthoaGePDgAXJzc7FmzRo8evQIGRkZaNq0KaZNm4aePXsCEE1iHzt2DP/++y9iY2NhZGSEqVOnwsXFBYCoV92ePXvg5eWF7OxsODg4YP78+eDz+ZLxc3JykJWVhVevXmHq1KmYNGkSHj58iJ07dyI0NBT169eHs7Mzpk2bBiUl5gujVcgZKx52SEMQKZSK0aKJLgz01BCXnIOQyFy0bCGE95tY3PWJgG9IPIRCUTUOFSV5dLQ1RM/WxjA31qGVpCgUCoVC+QIhBPmCohLbioRFyBMUAXKFkJcjjI2trCT/QzZ569atWLt2LUxMTLBo0SL8+uuvaNmyJf755x98+vQJc+fOxblz5+Dm5oZ///0Xt27dwtatW1G/fn08fPgQK1asQPPmzeHo6IgzZ85gy5YtWL58OaytrREcHIzVq1cjPj4eCxYs+K6Wrl27Ys2aNXjz5g1sbGwQFBSE9PR0vHjxAkVFRZCXl8fjx49haWlZqnnxs2fP0KFDByxZsgQDBw5Eeno6AODo0aNYvXo1mjVrBnd3dyxfvhwtW7aEmZnZd/UsXboUc+fOlfzu5+eHOXPmYMaMGQCA7du34927d/jnn3+gpaWFc+fOYfbs2bh9+zYaNWqEgwcPYvfu3Vi6dClat26Nx48fY8GCBahbty5atGiB8ePHw8jICHv37oWSkhJ27tyJUaNG4cqVKxIf5vbt25g/fz6WLVsGFRUVPHnyBLNmzcLixYvRrl07REREYPXq1fj06RO2b99e4c/9R6mQM7Z48eIKn5DH42Ht2rU/LIhCqSnweDx0czDCv3fe4WFgBh6+foq0rP8X47Aw0UUPp8boYGsIVeUqd5mgUCgUCqVGQQjBwl3PEPI5RSbjW5joYsP0DpV2yAYNGoRu3boBAPr3749Vq1Zh+fLlMDExgZmZGQ4ePIgPHz4AACIiIqCmpoZGjRpBX18fo0aNgqmpKZo0EfUp3bNnD6ZOnQpnZ2cAgJGREbKysrBy5Ur8/vvvUFD49vNDo0aNYGZmhmfPnsHGxgbPnz9H586d4enpiaCgIFhbW+PRo0cSvcURR8VpampCU1NT4oz99ttvkuNnz56NU6dO4e3btxVyxsTnEv/tf/75J8aPHw9XV1fJNnV1dRgZGUFLSwu///47WrVqhTp16oAQgqNHj2L06NEYPHgwAMDNzQ15eXkoLCyEh4cH0tLScP78eYn2v//+Gz/99BNOnjwpcV7r1KmDiRMnSjTNnTsXQ4YMwbBhwwAAjRs3xsqVKzFmzBhERUWhUSNme+ZW6AnQ29u7wieks/oUyv/p6ihyxtKyiwAUoY6GEro6GKFna2MY1adx2RQKhUKh1DSMjf/f0kZVVRWA6AFfjIqKCgQC0eTsyJEjce/ePXTu3BkWFhZo3749nJ2doaenh5SUFMTFxWHLli0lVmiEQiHy8/MRFRUFExOT7+rp1q0bnj9/jt9++w2enp7o06cPUlNT4eXlBWNjY7x69QrLly+v8N8ndhQBkWMDAPn5+RV+PQCkp6dj8uTJaNWqVYmVskmTJuHXX39F27ZtYW1tjfbt2+Pnn3+GpqYmUlJSkJiYCBsbmxLnmjRpEgDg7t27aNy4MXR1dSX7VFRUYG1tjffv30u2Ff98ACA4OBiBgYE4f/68ZBv50ksoLCyMG87YgwcPGBVBodRUDPTUMeynZgh8H43e7Zqjg50xFBWk0wSaQqFQKJSaDI/Hw4bpHcoOU8zLh4qKMuTlmGvz8qNhimWtVpWX32ViYoI7d+7Ax8cHnp6eePToEQ4cOIB169ahY8eOACAJn/uar8MKy6Nbt244dOgQkpKS8PLlS6xatQpxcXHw9vZGw4YNYWBgIMmpqghl/S1i56UiFBQUYPr06VBVVcXGjRtLvMd2dnZ4/PgxPD098d9//+Hy5cvYu3cvDh48iJYtW37zvOVpEAqFJT4TFRWVUvsnTpwoWZ0rjniFjUmk+lSYk5ODJ0+eSPOUFEq1x7VzEwzpoIfWlvWpI0ahUCgUSiXg8XhQUVYo+aOkABUledG/X++T4g8b0V7Hjh3DnTt30L59eyxYsAAeHh5o27Ytbty4AT09Pejq6iIyMhLGxsaSn6CgIGzbtq3CY1hbW6NOnTrYt28f9PT0YGJigrZt28LPzw937twpM0SRSZYvX45Pnz5h7969UFNTK7Fvx44d8PPzQ/fu3fHHH3/g9u3bMDIywu3bt6GpqQl9fX28fv26xGtmzpyJdevWwdzcHBEREUhOTpbsy8/Px5s3b9CsWbNy9TRv3hyfPn0q8R7HxcVh48aNyM7OLvd10qLSiSrR0dFYsWIFfHx8JEusXxMSElJlYRQKhUKhUCgUSk0mJSUFu3fvhoqKCvh8Pj5+/IiQkBCMHj0aPB4PkyZNwtatW9GwYUN06tQJ7969w4oVK9C9e3coKSmV6AlWHjweD126dMGZM2ckVQft7e1BCMHdu3fh7u5e7mvV1NQQFhaG1NRUqfy9+/fvx40bN7B//34oKioiMTFRsk/seF69ehWrV69G48aNERAQgJiYGNjZ2QEAJk+ejC1btqBJkyaws7PDo0ePcP/+fRw+fBhmZmbYv38/5syZgwULFkBJSQm7d+9GTk4Ohg4dWq6mSZMmYdasWdi1axecnZ0RFxeHpUuXolGjRqysjFXaGVu3bh38/f0xePBg+Pv7Q1VVFba2tvD09MT79++xc+dOJnRSKBQKhUKhUCg1iunTp6OgoABr1qxBYmIi6tWrh+HDh2PKlCkAgPHjx0NZWRnHjx/H+vXrUbduXQwZMgQzZ86s1Dhdu3bF+fPn0bp1awCAkpISHBwc8ObNG7Rq1arc140fPx4HDx5EWFgY/vjjjx//Q79w5swZ5OXlYcyYMaX23b9/H3/++Sc2bNiA+fPnIy0tDYaGhpg3bx769+8PABg1ahTy8vKwfft2JCYmwsTEBFu3boWTkxOKiopw4MABbN++HWPHjgUAODg44NSpUzAyMipXU+/evbF161bs378f+/btg7a2Nrp164Z58+ZV+e+tCDxSmSBPAK1bt8aMGTMwatQonDhxAg8ePIC7uzuKioowfvx4GBoa1rpqiuLlUisrKxkrEYWKhoSEwMLCotTSL9UhGx1c0MAVHVzQwBUdXNDAFR1c0MAVHVzQQHVwTwNXdDCpIS8vD58+fUKTJk1K5fR8TVFREfLy8qCiogJ5eeZyxriugSs6uKBBFjq+9Z2tjG9Q6QSW7OxsSRM4U1NTBAcHAwDk5eUxYsQIeHl5VfaUFAqFQqFQKBQKhVLrqHSYor6+PpKSkgCISkOmp6dLllW1tbVLJM1RKBQKhUKhUCgU6XPjxg0sWbJE8ntZBUfGjRtX6ZDGH8XV1RWRkZHfPMbb2xtKSkqs6KkuVNoZ69y5M7Zt2wYDAwPY2dnBwMAA7u7u+O2333DhwgXUr1+fCZ0UCoVCoVAoFArlC507d8bFixeRn58PZWXlMkPztLS0WNOze/duCIXCbx6jqKjIkprqQ6WdsZkzZ+LNmzfYvn07jhw5gtmzZ2PRokU4cuQIAFSqaRyFQqFQKBQKhUKpPOrq6jA2NuZEvhYANGzYUOYaqiOVdsZ0dHRw7tw5JCQkAAD69euHhg0b4tWrV7C2toaTk5PURVIoFAqFQqFQKBRKTaPSzpgYfX19yf8dHR3h6OgoFUEUCoVCoVAolNpJJYt8UygyQ1rf1Uo7YwKBACdOnIC/vz8yMjJK7efxeDh69KhUxFEoFAqFQqFQaj7iXKKcnByoqqrKWA2F8n1ycnIAVD0PrtLO2KpVq3D+/Hk0b94c2trapfbTGQ0KhUKhUCgUSmWQl5eHtra2JA1GTU2tzOqAgKifVH5+vuR1soALGriigwsa2NRBCEFOTg4SEhKgra1d5bEq7YzdvXsXM2bMwG+//ValgSkUCoVCoVAoFDEGBgYAIHHIykMoFKKwsBAKCgqQk6t0y1ypwAUNXNHBBQ2y0KGtrS35zlaFSjtjcnJysLOzq/LAYoRCIXbt2oVz584hMzMTrVq1wvLly2FkZFTm8ampqVizZg2ePHkCHo8HZ2dnLFiwoMSS9s2bN7Fz505ERUXB1NQUCxcuRNu2bSt1DgqFQqFQKBQKe/B4PDRo0AD6+vooKCgo97jc3Fx8/PgRjRs3ltmzGxc0cEUHFzSwrUNRUVFqq2+VdsYGDBiA8+fPo02bNlLxOvfs2YN///0X69evh4GBATZt2oSJEyfCw8OjzKZwM2fORG5uLo4cOYKMjAwsXboUOTk52LBhAwDAy8sL8+fPx4IFC9C+fXucP38ekydPxuXLl9G0adMKnYNCoVAoFAqFIhvk5eW/+aAr7mWlrKwMFRUVtmRxTgNXdHBBA5d0VJZKO2OzZs3CgAED0KtXL1haWpbyPHk8HtauXVuhcwkEAri7u2PevHno0qULAGDr1q3o2LEj7ty5AxcXlxLHv3z5Ej4+Prhx44bEsVq1ahUmTpyIOXPmoH79+jhw4AB++uknjB49GgCwcOFCvHz5EkePHsWqVasqdA4KhUKhUCgUCoVCYZpKL21t3rwZnz59QlJSEgIDA+Ht7V3qp6K8ffsW2dnZJUIItbS00KJFC/j6+pY6/sWLF6hXr57EiQIAJycn8Hg8+Pn5QSgUwt/fv8T5AKB169aS833vHBQKhUKhUCgUCoXCBpVeGbt69SrGjh2LBQsWVDlMMS4uDgDQoEGDEtv19fUl+4oTHx9f6lglJSVoa2sjNjYWGRkZyMnJKZVMV/x83zvHjyKurCJrcnNzS/xLdcheBxc0cEUHFzRwRQcXNHBFBxc0cEUHFzRQHdzTwBUdXNDAFR1c0MAVHVzQwCUdgMgvKK8a6NdU2hkrKipC165dpZIvJn6zvs4NU1ZWRnp6epnHl5VHpqysjPz8fOTl5ZV7PnGpy++d40cpKChASEjID79e2nz+/FnWEgBQHVzTAHBDBxc0ANzQwQUNADd0cEEDwA0dXNAAUB1c0wBwQwcXNADc0MEFDQA3dHBBA8AdHWX5G2VRaWesR48euHnzJlq3bl1pUV8jTq4TCAQlEu3y8/PLrIKioqICgUBQant+fj7U1NSgrKwsOd/X+8Xn+945fgRxxZ+qNn2TBoQQSVnPinrkVEfN18AVHVzQwBUdXNDAFR1c0MAVHVzQQHVwTwNXdHBBA1d0cEEDV3RwQQOXdAAi3+BbFUGLU2lnzMbGBps3b8bbt29hZ2cHdXX1Evt5PF6Fe5CJwwUTEhLQuHFjyfaEhASYm5uXOt7AwAD37t0rsU0gECAtLQ36+vrQ1taGmppaqf4UCQkJksIc3zvHjyD+wGX9wYs1VNQTpzpqjwau6OCCBq7o4IIGrujgggau6OCCBqqDexq4ooMLGriigwsauKKDCxq4pAMQaWEsTHHFihUAgFevXuHVq1dlDl5RZ4zP50NDQwPe3t4SZywjIwPBwcEYNWpUqeNbtWqFzZs3Izw8HMbGxgAAHx8fAICDgwN4PB7s7e3h4+ODwYMHS17n7e0NR0fHCp3jR5Bm3zUKhUKhUCgUCoVSO6i0MxYcHCy1rtZKSkoYNWoUNm/eDF1dXRgaGmLTpk0wMDBAz549UVRUhJSUFGhqakJFRQU2Njawt7fH7NmzsWLFCuTk5GD58uUYMGCAZOVr3LhxmDx5Mlq0aIFOnTrhwoULCAkJwV9//QUAFToHhUKhUCgUCoVCoTANjxBCKvMCFxcXzJ07F127dpWKgKKiImzZsgUXL15EXl4eWrVqheXLl6NRo0aIiopC9+7dsW7dOgwcOBAAkJycjJUrV+Lp06dQVlZG7969sXjxYkm+GABcvnwZe/bsQVxcHJo1a4b58+eXKHdfkXNQKBQKhUKhUCgUCpNU2hlzcHDA7t270aZNG6Y0USgUCoVCoVAoFEqNp9Lxhj///DOOHDlSqkgGhUKhUCgUCoVCoVAqTqVzxj5//owXL16gc+fOkuqFxeHxeKWqFVIoFAqFQqFQKBQKpSSVdsYaNGiAn3/+mQktFAqFQqFQKBQKhVJrqHTOGIVCoVAoFAqFQqFQqk6lV8bEPHnyBD4+PsjIyICOjg4cHR3RsWNHaWqjUCgUCoVCoVAolBpLpVfGBAIBpk2bhmfPnkFeXh46OjpITU2FUChEmzZtsH//fs50v6ZQKBQKhUKhUCgUrlLpaoo7d+6En58fNm7ciMDAQDx79gwBAQFYt24dXr16hb179zKhk0KhUCgUCoVCoVBqFJVeGevatStGjRqFCRMmlNp36NAhnDp1ilZTpFAoFAqFQqFQKJTvUOmVsZSUFLRo0aLMfS1atEB8fHyVRVEoFAqFQqFQKBRKTafSzljjxo3h5+dX5j5fX180aNCgyqIoFAqFQqFQKBQKpaZT6WqKw4YNw/r166GiogJnZ2fUrVsXSUlJuHbtGg4cOIDp06czoZNSDomJidi2bRv8/f1RUFCAr6NO79+/LyNlFAqFQvkeWVlZUFBQgIqKSql9CQkJWLlyJXbv3s2KlsuXL6Nv376linDl5OTg7NmzGDt2LOMaqE3jBjExMRU+tmHDhgwqKUlMTAy0tLSgoaEBLy8v3LlzB/b29nBxcWFNA4UibSrtjA0fPhzBwcHYvHkz/v77b8l2QghcXV0xefJkqQqkfJtly5bhzZs3cHZ2hqamJuvjL168uMLHrlu3jkElIgoKCpCdnQ1tbe1S+4RCIeLi4hgzHLt27arwsUxNWnTr1g08Hq9CxzL9UJOamoqHDx8iIyMD7du3R/PmzUvsz8nJgbu7O2PvxeXLlyt87IABAxjRQOEuAQEB8Pb2ltgsLy8vHDlyBFFRUWjcuDHGjx8PR0dHxsZPSUnB4sWL8eTJE/B4PPTs2VMy0QkAZ86cwebNm1FQUMCYBrGOvLw8AKL7efPmzaGjo1PimODgYGzZsoUVZ0zWNk3WcMGOABWzJYQQ8Hg8hISEMKajOHfv3sXs2bOxf/9+GBkZYeLEiTAyMsLFixeRnp6OkSNHMjIuV+wqF2za6NGjK3zssWPHGNEAcOc6kRY/3PQ5NDRU0mesTp06cHJyQtOmTaWtj/IdbG1tcfDgQUYfGr6Fm5tbhY89fvw4Yzry8/OxatUqXL16FYWFhbCyssKff/4JS0tLyTFJSUno2LEjY4ajW7duFTqOx+MxdsPeuXNnhY0Gkzeo0NBQjBkzBjk5OQBEn8/YsWOxYMECyTFMfx58Pr9Cx7H5MAFwY+ZfKBTCw8OjXA1sTJwAolYp7u7u6NOnD4yNjbF06VLcuHED9vb22Lx5cymnQFrcunULc+bMQbt27XDw4EE8fPgQ06ZNQ6dOndCsWTO8f/8ez58/x65du9C1a1dGNCxYsAAPHz7E2LFjoaSkhGPHjsHFxQUzZ87E7Nmz8ejRIzg6OmLNmjUwMTFhRAMgesBbtGgReDye5OH6awgh6Ny5M/bv38+YDjGytmlXrlzB+fPnkZ6ejk6dOuHXX3+FhoaGZH9KSgoGDx7M2HXKBTsCAD4+PhU+1snJiTEdxXF1dUWnTp3w+++/Y+/evbhy5Qpu3bqFW7duYefOnbh58yYj43LFrnLBpnFlAp4r14m0+GFnjMIN2rVrh5MnT6JJkyayliJTNm7ciBs3bmD27Nng8Xg4cuQIwsLCsGvXLkkz8qSkJHTo0AFv376Vsdqaz8SJE6GmpobNmzdDTk4Ox44dw5YtW9CvXz+sXbsWAPPOGFf59ddfvznzz8Ys3po1a3Dy5Enw+fwSD5pimJw4Kc7atWtx5coVuLu7Izk5Gb/++itmzpyJR48eoUmTJowZcxcXF7i4uODXX38FAAwZMgTt27fH77//Ljlm7969uHPnDi5dusSIhg4dOmDRokWS8Cp/f3/MmjULVlZWeP78OebPn48RI0YwMvbX+Pr6QigUYsyYMdi5cyfq1Kkj2cfj8aCmpgYzMzMoKioyrkWWNu3cuXNYuXIl+vfvDzk5OVy7dg36+vo4fPiwJKKitt63uIC1tTVu3rwJQ0NDjBgxApaWlli6dCliYmLQu3dvBAYGyloihfJDVChMsTKeMI/HkzxsUZinf//+OHjwIFatWgV5eXlZy0FhYSGSk5NRVFQEQDSjKhAI8Pr1a/Tr14+xcW/duoXVq1dLHK++ffti/vz5mDFjBg4dOgQHBwcAqPDsFlOI3wuxHqZ5+/Yt3r9/D6FQCKDk57FmzRrGxg0MDMSpU6ckuSfjx4+HkZERZs2aBW1t7RIrZLImLi4OBgYGrI3n5eUl05l/APDw8MDatWvh6uoqMw2A6LrdsmULLC0t8eeff8LJyQm//vorOnTogEmTJjE2bkREBJydnSW/R0VFoVevXiWOcXFxYbRvZlpaGuzs7CS/29vbIzk5GSEhITh//jyrkSatWrUCIAorsrOzQ1ZWlmRVMjAwEHw+nzX7IkubduzYMSxfvhxDhgwBAEydOhWTJk2Cm5sbTp8+jXr16rGqpzzYtiP5+fk4c+YM3r9/L7HtYh1v3rzB7du3WdGhpaWFzMxMZGZmIjAwUHKPiIiIKDM1gSlkZVcrCps2LSUlBZ8+fSrzvZg6dSorGsqD7eukKlTIGfP29v7uMampqcjNzaXOGMukpaXh2rVrePToEYyMjEolXjMZs/s1z549w8KFC5GSklJqn4qKCqPOWGpqKoyNjSW/KygoYPPmzZgyZQqmTp2Kf//9l9Wb9Zs3b7Bs2bISN+zisDGrevjwYWzYsAEAJCFI4v8z7QgoKSkhPz+/xLYePXrgjz/+wMqVK6Gvr89qwnVkZCQ2bNhQ4mFCbDRSUlIQHBzMmhY1NTXo6emxNl5ZCAQCyQO4LElLS5M4HZ6enhg6dCgAQFtbW5LHxARGRkbw9PTEsGHDAAAWFhZ4+/ZtiTCgwMBA1K9fnzENhYWFpYp2KCkpYfny5TIL+TcwMICLiwu6d+8umTCZPHky6tatiwMHDrBSLVmWNi0qKgpt27aV/N6wYUMcPXoUw4YNw8SJE3Hy5EnGxi4LLtgRQLSSfvnyZbRo0QKvX7+GnZ0dwsPDkZyczEoeoZjOnTtj+fLlUFdXh6amJtq3b4/nz59jxYoV6NKlCysaZGlXi8MFm3b16lX88ccfEAgEpcKcDQ0NWXPGuHKdVIUKlbZ/8OBBuT937tzBgAEDUFBQgLp162Lnzp1Ma6Z8hYuLCzp16oQmTZrA0NCwxA+bbNmyBS1atMD+/fuhoqKCXbt2YcmSJdDQ0MCmTZsYHbtp06a4detWiW3y8vLYvn07GjRogIkTJ+Ljx4+MaijOunXrIC8vjz/++AOKiopYtmwZxowZAwUFBWzZsoUVDSdPnsSkSZMQEBAAHR0dPH78GFeuXEHTpk3RvXt3Rsd2cnLC+vXrkZiYWGL78OHDMWHCBGzYsAH//PMPoxqKs2rVKrx79w69evVCfHw8nJ2dYWlpiaSkJKxYsYI1HcD/Z/6LzzCzTceOHfH48WOZjS+mcePGeP36NYKCghAVFSVZ2b537x4aNWrE2LiTJk3CmjVrsHnzZrx9+xbTpk3D7t27cebMGfj7++Pw4cP4888/GSsI8C1kmXu9du1aGBsbY9y4cZJtN27cQIMGDVjLIwRkZ9P09fURFBRUYlvdunXxzz//IC4uDlOnTmV0kuBruGBHAFEe67p163DmzBkYGhpi9erVePjwIbp37854gZniLFu2DPb29lBTU8PevXuhpKQEPz8/2NraYuHChaxokKVdLQ4XbNq+ffvg7OyM69evQ1NTE+fPn8fu3buhr6+PGTNmsKIB4M51UiVIFQgODib9+/cnfD6fzJ07l6SlpVXldJRqjpWVFQkJCSGEEDJ8+HDy/PlzQggh58+fJ8OGDWN07Pv375MWLVqQ8ePHk7dv35bYl5CQQHr37k2srKwIn89nVIcYW1tbEhAQQAgh5JdffiG+vr6EEEIOHz5Mxo4dy4oGS0tLEh4eTgghZNy4ceTu3buEEEKePn1KnJ2dGR07NjaWuLi4ED6fT548eVJq/8aNG4m5uTlrn4e9vT3x8vIihBDSv39/yWezZcsWMm3aNFY0iFm0aBFp2bIladeuHRk6dChxc3Mr8cMGhw4dIjY2NuS3334jW7ZsITt37izxwxaXLl0iLVu2JNbW1mTMmDGEEEJ27dpFLCwsyKVLlxgd+/Lly6Rbt26S76G5ubnkx97enuzZs4fR8fl8PklOTi6xzdbWlkRERDA67rdwcHAgYWFhpba/f/+etGrVSgaK2OXAgQPEycmJ7N+/n8TFxZXY9+LFC2JnZ0d69OhRq+wIISJbEh0dTQgh5NdffyUeHh6EEEICAwNJt27dWNPBBWRpV4vDBZvWsmVLEhoaSgghZNSoUeTx48eEEEJu375NXF1dWdFACHeuk6pQ6dL2gCi8Yvfu3Thw4AC0tbWxa9cuVmcEKCWJjY3FyZMn8f79eygoKKB58+YYOnQoq70/ANFKlLgggbGxMd6/f4+2bduiTZs2kmV9pujWrRuOHj2KM2fOlKoMV69ePZw5cwZr165lrNrS1wiFQkl+gfi9cHR0RPfu3VmpSAaIwuHEqy+NGzdGaGgofvrpJzRt2hTR0dGMjm1gYIALFy7Az88PZmZmpfbPnz8fnTp1wvXr1xnVIUYgEKBx48YAgCZNmuDdu3ewtrbGgAEDKlURVFrIuifOiRMnoKuri+Dg4FLhLDwej7VSwAMGDACfz0dUVBQ6deoEALCyssKhQ4dKhIsxQf/+/dG/f398+vQJnz59kvT7MjAwgKWlJZSVlRkdnxCC3377rURRjPz8fMybN6/U2GyFmysoKCAjI6PU9tzc3FL3VSaRlU2bMGECCgoKcPLkSVhbW5cIU3VwcMCRI0cwb948RjUUhwt2BAB0dXWRnJyMhg0bwsTEBO/fvwcA6OjoICkpiTUdAPDixYtyq8Cycd+SpV0tDhdsmpKSkiSM2NjYGB8+fECnTp3QsmVLhIeHs6IB4M51UhUq7YwFBwdj8eLFePfuHfr164c//vgDWlpaTGijVIB3795h1KhRUFFRgbW1NYRCIS5evIiTJ0/i1KlTpXo7MUnz5s3x4MEDuLm5wdTUFH5+fhgzZgzi4uJYGd/R0RGOjo6IjIwstU9LSwvr169nLZ/R2NgYfn5+cHFxgampKV6/fg0AyMzMhEAgYEWDvb09/vnnHyxfvhwtWrTA+fPnMXnyZPj5+UFdXZ3x8ZWUlNC2bVts374dAwcOhJGRUYn9rVu3RuvWrRnXAYji19+/f48GDRqgSZMmkhhyoVCI7OxsVjSIYTPcqzwePHggawkS+Hx+iVwtsVPGFk2aNJFJ5b6yiqewHVr+NZ06dcKaNWuwZcsWyYNeZGQk1q1bJwkhZRpZ2jQej4epU6eWm+sirubHVtU+LtgRQPS9WLlyJdatWwcHBwesXbsWPXr0wI0bN1gtfrR7927s3LlT0vS5OGxNIsnarorhgk1r2bIlzp07hzlz5sDMzAyPHz/GhAkTEBoaykrlVTFcuU6qQoWdscLCQuzatQsHDx6Ejo4O9u7dy1j/FUrF2bhxI1q3bo2///5bMpsqnl3dvHkzq7MCkydPxsyZM6GoqAgXFxfs3LkTkydPxrt379CmTRvWdPTo0QMODg4YOHAgevfuXeIGKSdXoTTJKuPm5oalS5cCAHr16oX+/ftDRUUF/v7+sLW1ZUXDnDlzMH78eJw8eRLDhw/Hvn374OTkhNzcXEyYMIEVDYCoct++fftgb29f5mfCBq6urliwYAE2btyILl26YPTo0WjYsCE8PT1hbm7OqhaAG6vZhBA8ffq0hIY2bdqwWsHuew1E2SxAxDZccMq/ZuHChRg3bhx69eolmWTNyMiApaVlpaoqVwWu2LRu3bphwIABcHV1LTGRJC8vX6IKJpNwwY4Aop54ixYtgo+PD0aMGIEzZ85g8ODBUFBQYDzqpTinTp3C7NmzMWXKFNbG/Bqu2FUu2LQZM2Zg4sSJ0NbWhqurK3bv3g1nZ2fExsaib9++rGgAuHOdVIUK9RkLCgrCokWLEBoaigEDBmDJkiVl9sehsI+dnR1Onz5d6uJ7+/YtRo0ahRcvXrCqJygoCPLy8uDz+fDx8cHhw4fRoEEDzJw5k7Vqhr6+vvDw8MDt27chEAjQo0cPDBw4kFWHEADu3r0LHR0dODo6wsPDQ1KNbNmyZYwWJyhOXl4ecnJyoKuri6SkJFy7dg0GBgbo3bs3K+OL8ff3x7Vr13Dz5k3k5eWhR48ecHV1ZTwUTQwhBEePHoWJiQm6dOmCf/75B/v370eDBg2wadMmWFhYsKIDKHvm/82bN8jNzWVtNTstLQ0TJkxAUFAQNDU1QQhBVlYWLC0tcfjwYdaiHb5+wC8sLER4eDjev3+PMWPGYPbs2YyM6+vrW+FjZVl1Mjc3F1u3bsWSJUtYG7OoqAjPnz/Hhw8foKCggGbNmqFt27astQXhik3btWsXrl27hvDwcJlOJHHBjnwNIQQhISGoW7cu9PX1WRvXxsYGN27ckPkKMhfsKldsWnx8PAQCAYyMjBAaGorTp0+jQYMGcHNzK1UJlUm4eJ1Uhgo5Y5aWlhAKhdDU1PxuB3Aej4ejR49KTSDl23To0AH79u1Dy5YtS2x//fo1xo0bx6ozdv78efTp04d1Y1UeBQUFePToETw8PPD48WPo6elhwIABmDlzJuNj//fff6w5GuWxYMECmTih36KwsBDPnj3D9evXcf/+fWhra7MSMhcZGVkqTFJWTJgwAaqqqmXO/AsEAlZm/hcvXozAwED8/fffknv627dvMX/+fNjb22PlypWMa/gWu3fvRlxcHFavXs3I+Tt27CjJdfmWCeTxeIyVRc7Ly8PGjRtx/fp1KCoqon///pg7d65k9f7Zs2f4888/ERsby2rrhfLIz89nPI8O4JZNA0QtDq5evYpbt24hOzsbPXv2hKurKyv3VS7YEaD8VUK2GTduHAYNGiTTnFuu2FUu2LTyUhDYhivXSVWokDNW2WTA48eP/7AgSuWYN28eEhISsGPHDsnKU0pKCmbNmoU6deqw2mrA3t4ehBCZrUSVR0pKCi5fvozdu3cjLy+vVOliJrCwsECDBg1kasBGjx6NFy9ewMDAgBOGFAASEhJw/fp13L59G69fv4aTkxMOHz7M+Lh8Pr/c0FW24cLMf5s2bbBjxw44OTmV2O7t7Y05c+bA09OTcQ3fIioqCgMGDGDsvUhNTcWECRMgJyeHbdu2fXPVh6lZ+NWrV+PMmTPo168flJSUcPXqVUyaNAlTpkzBmjVrcOrUKTRu3Bhr1qxhbXUuNTUV+/btK9W7qKCgAKGhoax8N7lk04ojFApx6tQpbNmyBTk5Oaz0LuKCHQG4s0p44cIFbNy4Ea6urjA1NS218jJgwADGNXDFrnLBpv3000+Ijo6W6XcC4M51UhUq5IxRuEtcXByGDRuG9PR0mJiYAAA+f/4MbW1tHD9+nNXl2by8PNy9exfXrl2Dp6cn9PX10b9/f5nMnOTk5ODu3bvw8PCAl5cXDA0N0b9/f7i6urLSuDQmJgZXr17FtWvXEBYWBnt7e7i6urK+chgfHw8PDw94eHjg3bt3cHBwYF1HVlYWbt++DQ8PD/j6+qJhw4ZwdXVl7bMARGFpV69exZ07dyShq2yGSRaHCzP/jo6OOHv2LExNTUtsDwsLw8CBAxEQEMC4hm/h4eGBNWvWwNvbm7Ex4uPjJatRgwcPZmyc8ujatSsmT56M4cOHAwAePXqEv/76C23btsX58+cxbtw4/P7776yG+syaNQv//fcf2rdvj1u3bsHZ2RlhYWEIDg7GnDlzMHnyZMY1cMmmAaL8TvE9NCwsDE5OThg4cCD69evH+NhcsSNiZLlKCOCbkVlMrmJ/DRfsKldsmqxTEADuXSc/AnXGagDZ2dm4cuUKPnz4AEIIzM3N8fPPP5eqNsQmKSkpuHnzJq5fv46AgADY2dnhxIkTrIw9e/ZsPHr0CDweD71798bAgQPh6OjIythlERwcDA8PD9y6dQtpaWno2bMnq0nPYj58+IBr167h5MmTEAqF8Pf3Z2Vca2trKCoqomfPnhg0aJBMPwtZhq6K4cLM/5gxY2BmZiZJehazZs0avHnzBqdPn2ZcA1B2AY+srCy8e/cOI0aMKKVP2pw/fx737t3Dvn37GB2nLKysrHDjxg3JRFVRURGsrKxQp04dbNu2jbVKo8Vp3bo1NmzYgC5dusDZ2VkSxrps2TLk5+dj48aNrOjggk07ffo0rl27Bn9/fxgaGkpm3dluGSOGK3YEkM0qIVeRlV0VwwWbBsguBeFruHSdVIYf6jNG4Rbq6uoYMWKErGWUQENDA/Xq1UODBg0QEhKCxMRE1sZOSkrCn3/+iV69ekFVVZW1ccujRYsWIIRAQUEB//77L+7fv8+6hoCAAElRE0IIq5WOVq5cid69e3Pis1BUVJRU2xSHru7fv591Z2zYsGHo2rVrqZl/tlovzJo1C6NHj8arV69gb28PAPDz88Pbt29x8OBBVjQAZYcAKioqYtSoUaysPPzyyy/45ZdfGB+nLAoKCqCmpib5XV5eHsrKyli6dKlMHDFA5ASJw2dNTU3x9u1b8Pl8jBo1ipVVMTFcsGkbNmxA79698fvvv8u0iIsYLtiR8lYJ2SYsLAzv37+HoqIimjZtKpPWFIBs7aoYLtg0QDSh+OnTJ0RGRiI/Px/Gxsasji+GC9fJj0BXxqoh3bt3x/nz56Gjo4Nu3bp9M9+B7S+il5cXPDw8cOfOHQiFQvTp0wcDBgyQ6WqIrIiMjJQYrvDwcLRu3RoDBgxAr169oKKiwvj4nz59goeHB65du4bIyEiJ4WRrfK4h69DV4nBh5j8wMBDu7u4lNIwbNw7W1tasaajN8Pl8eHp6Qk9PT7LNzs4OV65ckfT4Ypvu3btjw4YNcHR0xN9//43CwkIsXLgQERER6NevH169esXYuFyzaTk5OSWcZVkhazsCcGeVMD8/H3PnzsW9e/ck23g8Hrp27Ypt27axEtLLJbsqa5vGhRQEMVy4TqoCXRmrhri6ukq+XLKYlSqPjh07Ijk5GY6Ojli6dClnVqbE5OTkwN3dnZXGkEOGDMHr16/RqFEjmRmuPn36lBhf1uWAvyYlJQWDBw9m5eHq69DVI0eOyHSCiJEvXAAA8UVJREFUgAsz/9bW1ti2bZtMNQBAdHQ0zp49i3fv3kFeXh6WlpYYMmQI6taty+i4hYWFePr0Kdq0aSO5T50+fRqPHj1C3bp1MW7cODRt2pRRDWU5HWyVkC+Lnj17YvHixVi/fj3atWuH2bNnw8bGBvfu3WN0ppuLNq08R6ygoACvXr1iZbWMC3YE4M4q4datWxEYGIjdu3fDyckJQqEQvr6+WLNmDXbu3Im5c+cyroErdpULNq1du3aSFISjR4/KzKZy5TqpCnRlrJrj6+sLW1vbUt3O8/Pz8ejRI/Tq1Ys1Lbt27cKAAQM429MhKSkJHTp0wNu3bxkfa/HixRg4cKBMDZePj0+panlcIisrC3/99RcrzW/d3NwwaNAgmU0QcGHmf/HixVi6dCk0NDS+28CXrYbE/v7+GD9+PHR0dNCyZUsUFRUhKCgIeXl5OHHiBGM915KTk+Hm5oZPnz7h2rVraNq0Kfbs2YOdO3fCysoKGhoaCAgIwOnTpxnTwOfz0bdv3xLl4j08PNCtW7dSSedsfR4CgQCbNm2CtbU1fv75Zyxfvhxnz56FlpYWtm/fzkpSPpdsWlkkJSWhY8eOrORJccGOANxZJezQoQNWr16Nrl27ltj+8OFDrFy5Eo8ePWJcA1fsqqxtGgBcunSJEykIXLlOqgJ1xqo5FhYW8PT0hK6ubontQUFBGD58OAIDA1nX5Ovri7CwMLi4uCAuLg4mJiZQUGB2EVYgEGDjxo3w8PCAoqIi+vbtizlz5sh8eTomJgZhYWFo1aoVsrOzS4QksUFCQgLOnj2Ljx8/YunSpfD19YWZmVmpKnq1BYFAgKioKDRu3BiEkFIPfEyxa9cuSX+xXbt2ffNYplZu3dzcsHv3bmhpaX23XQlb7UmGDh0KU1NTrF69WnKPKCgowOLFi5GUlIQjR44wMu6qVavw4sULbNu2DaampsjOzkb79u1hZWUl+dvXr1+PmJgY7NixgxENlWkZI8t2MWlpadDQ0GD8Hi6GizatOAUFBXj58iWrD+SytiMA8PjxYxw6dAgfP37EmTNncPHiRTRu3Bj9+/dnTYOdnR0uX75capX28+fP6NevH2vfDS7ZVVnZNDF5eXm4desWwsLCMGHCBLx//x7NmzeHjo4OqzoAblwnPwyhVDsOHz5M+Hw+4fP5xNzcXPL/r3+GDBnCqq7MzEwydOhQiaaIiAgyZcoU0rdvXxIXF8fo2OvXryc2NjZk2bJl5M8//yR2dnZk6dKljI75LQQCAZk1axYxNzcnFhYWJCIigvz2229k7NixJDMzkxUNnz9/Jk5OTqRbt27E0tKSREREkBkzZhBbW1vy6tUrVjRwiU2bNhFra2vJ5zF79myyZMkSIhAIWNXh4+NT5ph5eXnk1q1brGopi4SEBNbGsra2JmFhYaW2f/jwgdja2jI2bteuXcmzZ88kv9+9e5eYm5uTq1evSra9fPmStGnThjENXGHmzJms3ZPKg6s2TdZwwY4QQsizZ8+IpaUlWbhwIbGysiIRERFky5YtxMLCgly6dIk1HUOGDCH79u0rtX3Pnj2kX79+rGjgkl2VtU1LTEwk3bp1IzY2NhINU6ZMIe3btyehoaGsaCCEO9dJVaDOWDWkoKCAXLp0iVy4cIGYm5uTo0ePkosXL0p+Ll26RG7fvk3S09NZ1bVy5UoydOhQEhERQWxtbUlERAQJDQ0lAwYMIHPmzGF07K5du5Lr169Lfn/48CGxtbUlQqGQ0XHLY9u2baR3797Ey8tL8l54eXmRrl27kj///JMVDb/++itZsmQJEQqFEg0FBQVkzpw5ZNSoUaxoIIR88+HKysqK9OjRg+zatYvRz+ro0aOkQ4cO5MKFC8TGxoZERESQ69evEycnJ7JlyxbGxi0LPp9PkpOTS21/8+YNsbKykqmGyMhIRp2gr3FxcSlx3Yp59OgR6dOnD2PjWlpakujoaMnv69evJ3w+nyQmJkq2RUdHk5YtWzKm4VskJyeTmzdvksjISMbH4vP5JCkpqcQ2FxcXEhMTw/jYYrhm0969e0fy8/Mlvz958oSsXr2a7Nmzp8zrhim4YEcIIWTo0KHk8OHDhBAi0UEIIf/88w9xcXFhTcfDhw+JhYUF+f3338nRo0fJ0aNHycyZM4mFhQW5ceMGKxq4Yle5YNPmzp1LpkyZQnJyciTvRVpaGhk7diyZMmUKKxoI4c51UhWoM1bNuXjxYgmjIUu6dOlC/Pz8CCElb9j+/v6kbdu2jI5taWlJYmNjJb8LBALC5/NJfHw8o+OWR48ePYinpychpOR78fz5c9KhQwdWNDg5OZEPHz6U0hAaGkrs7e1Z0UCIyGhYW1uTdevWkbt375K7d++SjRs3EhsbG7J+/Xqyd+9e0rZtW/LPP/8wpqFv377kzp07hJCS78WdO3dI165dGRtXDBdm/s+dO0fc3NyIm5sbMTc3J8OGDZP8Lv7p1asX6dSpE2Mavuby5cukbdu25PDhwyQ4OJi8f/+eXLhwgXTo0IHs2bOH+Pj4SH6kSdu2bUlwcLDk9wEDBpC+ffuWOObZs2esvRfv3r0jPXv2JD4+PiQ9PZ20b9+emJubEysrK/Lff/8xOra5uXkpZ6z4NcI2srRpWVlZZPTo0YTP50tm9s+ePUv4fD7p3Lkz6dmzJ+nQoUMJR55JuGBHxGOHh4eX0hEREUGsra1Z00GI6J49ePBgYmNjQ6ytrcngwYPJ7du3WRufK3ZV1jaNEELat29PgoKCSmkICQkhrVq1YkUDIdy5TqoCraZYzXF1dZX0dxAKhQAAQggEAgFev36NqVOnsqYlJSUF9erVK7VdS0sLOTk5jI5dWFhYIlZaUVERKioqyM/PZ3Tc8oiPjy+zPHWDBg2Qnp7OigahUCj5ThQnOzsb8vLyrGgAgOvXr2PJkiUYOnSoZNtPP/0EU1NTnD9/HqdOnULz5s2xceNGTJo0iRENUVFRsLCwKLWdz+ez0gNv1KhR0NbWhlAoxJIlS7B48WJoampK9vN4PKipqaFNmzaMafjpp5/g5+cn+d3AwKBUTqWZmRkGDBjAmIavWbhwIQBRftbXbN++XfJ/Ho8n1YIJbdq0wYkTJ/DXX3/B19cXISEhJe6VQqEQBw4cYK062IYNG2BsbAxTU1Ncu3YNhYWFePz4MU6fPo1t27ax1oSbC8jSpu3fvx+RkZH4559/0KRJE0lBEz6fjzNnzkBJSQkLFizAzp07WSmqwgU7AgCamppISEgopSU0NBR16tRhTQcA9OjRAz169GB1zOJwxa7K2qYBor+5vMIuhYWFrGgAuHOdVAXqjFVzrl69ij/++AMFBQUAREZLXKnN0NCQVWfMysoKN2/eLNUc9OTJk2jRogVrOrhA06ZN8d9//2Hw4MEltl+/fh3NmjVjRUOHDh2wf/9+bNq0SbItLS0NmzZtYvSh/2tCQkLKHM/R0RErV64EIGrUGBsby5gGQ0NDSenb4jx58gRGRkaMjStGQUFB4uTweDw4Ozuz0hOnONra2iUeIMWVFWXJ3bt3IScnx/q4M2bMwIgRI+Dk5ITs7GwYGhpi3LhxAIAbN25g//79iIqKwtmzZ1nR8/LlS5w7dw56enp4+vQpOnfujPr162PgwIE4fPgwKxq4gixt2u3bt7FkyRJ07NgRgKhyXkZGBubPny+5XgcPHozZs2czpqE4XLAjAPDzzz9j7dq1WLt2LXg8HrKzs/HkyROsXr2a8UbHXCh+VByu2FVZ2zQAaNWqFU6dOlWiOm9BQQH27t0Le3t7VjQA3LlOqgJ1xqo5+/btg7OzMyZOnIjhw4fD3d0dCQkJWLlyJWbMmMGqljlz5mD8+PEIDAxEYWEh9u7di7CwMAQFBeHQoUOMjs3j8UqVC5dlv54ZM2Zg9uzZCA0NRVFRES5duoRPnz7h9u3b2Lp1KysaFi1ahNGjR6NDhw7Iz8/H1KlTER0dDW1t7TJXIpiiUaNGePjwIcaOHVti+8OHD2FgYAAAiIiIKFU9TZpMmDABK1euRGJiIggh+O+//3DmzBkcP34cixYtYmzcsuDCanZ5s/piDQ4ODoxrAES9ctasWQM+n8/KeGKaNGmCa9eu4datWxLnWEtLC4Co71njxo2xYcMGxvuMiZGTk4OSkhIKCwvh4+ODZcuWARDNPDNdEbase6cskaVNi42NLbHa4OvrCx6Ph3bt2km2GRoasjbbzgU7AgCzZs1CXFycZELJ1dUVhBB06dKFccf04sWLGDlyJFRVVXHx4sVyj+PxeKw4Y1yxq1ywaQsXLsTIkSPh4+ODgoICrFixAh8/fkRmZiZOnDjBigaAO9dJVaCl7as5VlZWuHz5Mpo2bQo3NzdMmjQJnTp1wp07d7Bv375v3ryY4O3bt3B3d0dwcDCEQiGaN2+O8ePHw8bGhtFx+Xx+qQeK4jOqxWGjPwwgmqHav39/ifdi0qRJrPbJyc3NxbVr1xASEiLR0L9/f1ZXRDw8PLBo0SL07t0bdnZ2EAqFCAgIwK1bt7Bq1SrY29tjwoQJ+Omnn7BkyRLGdJw5cwZ79+5FXFwcAEBXVxeTJk2SrIiwxfdm/u/du8e4hqCgIPzxxx94//59mSE3bF0jrVu3xrlz58oMMWGSmJgYNGjQgDNOyMSJE2FgYABdXV24u7vjyZMnKCgowLJlyyAnJ4d9+/YxNjafz4ednV2JMO8XL17AysqqRA80ADh27BhjOsTI0qa1bt26RH+7YcOGISkpqcQ1+eLFC8yaNQvPnj1jTEdxuGBHxISHh0tsiZmZGadWHYRCIWur7FywqwA3bFp8fDxOnTpV4r0YMWIE6/1muXSd/AjUGavmODg44PLlyzAyMsIff/yBJk2aYMKECYiJicHPP/9cIkekJnPp0qUKH+vq6sqgEkpZPHz4EO7u7ggKCoKCggLMzc0xefJkdOzYEb6+vnj27BmmT5/OSo+UlJQUEEJk1oOkb9++sLGxKXPmf86cOaz07Rk5ciTy8/MxaNAgrFu3DosWLUJERAROnjyJjRs3ok+fPoxrAIADBw7gyZMnmDBhAho3blxqFahhw4aMjGthYYFnz55xpg9NeHg4Zs+ejcjISMyePRsjRozA6tWr8ejRIxw4cIDR/kWLFi2qsFPKRp6ULG3ar7/+CjMzM8yZM0fSK3P06NElwrBmzZqFgoIC7N69mzEdXKD4hEVMTMw3j2XqOv2a7t2748KFC9DW1i6xPT4+Hv369YO3tzcrOriGrG0aperQMMVqTsuWLXHu3DnMmTMHZmZmePz4MSZMmIDQ0FBWHmwXL14syT0pbrDKgklDzgUHiwux7d27d8f58+eho6ODbt26ffMh6/79+4xoKIuuXbuia9euZe5r1aoVWrVqJfUxL1++jL59+0JJSQmXL1/+5rFsFq2IjIzEzp070bRpU5ibmyMlJQXdunVDYWEh9u3bx4ozFhwcjKNHj8La2hoXL16EmZkZRowYAQMDA5w9e5Y1Z2zbtm0oKiqShIOJEa8WMrVCx7U5SGNj41IrPpMmTcKSJUsYLwrAZmhVRZClTZs2bRpGjx6NJ0+eIDo6GlpaWhg/fjwAwMvLC4cPH4anpyejIVhcsCOAyJaIJyzKsyVMX6eAKIfz6dOnAEQhxKtWrSq1YhsdHc3oKjdX7CoXbNro0aOxa9cuaGlpwc3N7ZvvBZMr6Vy5TqQFdcaqOTNmzMDEiROhra0NV1dX7N69G87OzoiNjWU8sRYQVfQRhzlFRUUxPt638PX1xYULF5Ceno5OnTphyJAhJR5k0tPTMWPGDMZuEFyIbXd1dZWsLgwcOJCRMX6EFy9ewN/fHwUFBaUehJl6LxYtWoSOHTtCT0/vmzH0PB6PVWdMSUlJUgzA2NgYHz58QKdOndCyZUuEh4ezokEoFEoqnxobG+P9+/dwdHRE9+7dsX//flY0AKh1xSnKIy8vDytXroSJiQmmTJkCABgxYgTat2+PZcuWMVrshWsrILK0adbW1jh37hwuXrwIOTk5DB06FPXr1wcAPH36FImJidi7dy9sbW0Z08AFOwIAR48elVRKZCM8tTzs7Oxw+vRpid2IiYkp4ZSLK9Fu2LCBMQ1csatcsGmGhoaScFC2QxGLw5XrRFrQMMUaQHx8PAQCAYyMjBAWFoZTp06hQYMGcHNzY7Vi24sXL2Btbc16lTgAePDgAaZPnw4nJyfIycnBy8sLNjY22Ldvn8SgJCUloWPHjqzkw4SHh8PY2Jjxcb7FoUOH4OLiInmYkBW7d+/Gzp07oaWlVSqmnsfjsbJCl5WVJfPKgWLGjBkDGxsbzJkzB8eOHcPjx49x6NAhPHnyBAsWLICXlxfjGvr164fJkyfDxcUFe/bsQWRkJNatW4fg4GC4ubnJJLw5JSUFCgoKkkIaTMLn8yWzqt+DDUO+YsUKeHp64q+//oKTkxMAUaXJTZs2oXv37pIWAExQPGSzrNxbgJ0VkOJwxabJGi7YEUB0DcyePZu1gjbl4ebmhl27drFeTr84XLGrXLBp165dQ4cOHUqFjbINV66TqkBXxmoA4eHhKCwshJGREZo2bQpFRUXY2NiwbrRmzJiBgwcPwtLSktVxAdGS9YwZMySV6AIDAzF9+nSMGzcOx44dY/2mNWrUKOzevRvW1tasjlucvXv34qeffpLZ+GJOnTqF2bNnS2b8ZYGrqyu2bdsmk+/m18h6NRsQPdQsXboUANCrVy/0798fKioq8Pf3Z3TWvyyOHTuGf/75B8nJyQCAunXrYsKECaWqb0qb69evfzfhn61Z1Xv37mHnzp2ws7OTbOvRowe0tbUxd+5cRp0xrqyAFEeWNi01NRUPHz5ERkYG2rdvLynmISYnJwfu7u6sfC+4YEcAUYjm16GBsuD48eMlfk9JSYGPjw9atmzJ2ioNV+wqF2zaqlWr8O+//8rcGePKdVIV2G/wQpEq169fx/jx40vMWEZHR2Ps2LGsVGUrjq6uLjIzM1kdU8ynT5/g4uIi+d3a2hpHjhxBbGwspk+fLqlcxxaKiopQUJDtXIeNjQ0ePHggUw0AkJmZWeKzkQW5ubmMlwivKI6Ojrh9+zZ69OgBHR0d/Pvvv2jbti1+++03LF++nBUNgwcPxt9//w0DAwM0bdoU69atg5+fHwwMDCS939jg9OnT2LRpE5ydnbFz507s2LEDffr0wZYtW3D+/HlGx75w4QIePHjwzR+28iqzs7PLXBHU1dVlvIy6k5OT5F7l5OQEfX19qKmpwcnJCU5OTggJCYG+vr5kxY5pZGnTQkND4eLigtWrV2P79u3o378/Nm7cWOKYnJwc1op3cMGOAKIH/82bN+PDhw8QCAQy0/H+/Xv06tULvr6+yMjIQL9+/TBr1iz07duXlYgCgDt2lQs2zcTEBO/fv5epBoA710lVqN7qKdi3bx8WLVqEUaNGSbbt2LEDx44dw86dO1mdwenUqROmTJmCzp07w9jYuNRMGpMzibq6uggPDy/R7NDU1BS7d+/GuHHjsGDBAlb7Sbm6umLixIno378/jI2NS9002chR0tDQwMaNG7Fv3z6YmJjIpEw1ANjb2+Ply5cwNDRkZbyyGD16NGbMmIGRI0eWWbWPiQIi34ILq9mtW7dGamoqAFFTV2VlZTg5ObE6y3nkyBEsXLiwxP2rR48eMDY2xtGjR/HLL78wMi5XStqLsbW1xcGDB/HXX39JVusIITh69CisrKxY0/H8+XNMnToVY8eORcuWLQGInKNt27bhwIEDcHR0ZFyDLG3a+vXr4eDggM2bN0NOTg7Hjh3Dli1bkJaWhrVr1zI2bnlwwY4AwOPHjxEREYHbt2+XuZ+t8NUNGzbA2NgYpqamuHbtGgoLC/H48WOcPn0a27Ztw+nTpxnXwBW7ygWbxufzMW/ePBw8eLDM94KN6qsAd66TqkBzxqo5NjY2uHbtWqmO65GRkXBxcUFAQABrWrp161buPqZzg/7++29cv34dc+fORceOHUvMMt+5cwdz5sxBy5YtERAQwIrh+FYTW7ZyL2RZ3bI4Fy5cwMaNG+Hq6gpTU9NSDgcbN0oufB5irl+/joULF2L27NmYMGECAGDmzJl48OABtm3bxsoESlBQEMaPH4+BAwdKQuC6deuGgoICHDp0CGZmZoxrAEQr2NeuXSvVZywiIgIuLi4IDAxkZFw+nw9PT0/OlIIODAzEmDFjoKOjI3GCgoKCkJaWBnd3d8b7NIoZNGgQOnToUKqR75YtW+Dj48PKw64sbZqTkxNOnTpVIjfq7t27mDVrFsaMGYMFCxawmnvMlfvW91rHsFXN2N7eHufOnUPTpk0xdepUaGlpYcOGDYiMjMTPP/+MV69eMa6BK3aVC98NNze3b+7/OqyUKbjwXlQVujJWzWnQoAF8fX1LGa6XL19KqqWxxbeW7stqLCtNpk+fjtTUVCxcuBD//PMP2rVrJ9nXs2dP7Nixg9G8i695+/Yta2OVx4wZM2BgYFAqL6awsBDBwcGs6RDnJh05cqTUPrYqGbJZxv97cGE1e/369ejWrVuJh+47d+5g2bJlWL9+Pdzd3RnXAIiq871586aUM/b69WvUrVuXsXGnT58ONTW1UttTUlLw4sUL6OnpwcHBgbHxv8ba2hoeHh44c+YMPnz4AAUFBbi4uGDkyJHQ19dnTUdYWBi2bdtWavvgwYNZe7CSpU1TUlJCfn5+iW09evTAH3/8gZUrV0JfX5/VkGsu2BFAdJ8Wl1QvTk5ODs6ePcuaDjk5OSgpKaGwsBA+Pj5YtmwZAFGYL1she1yxq1ywad+6JyQmJrKmgyvXSZUglGrNkSNHiK2tLdm6dSt58OABefDgAdm+fTuxt7cnBw8eZFVLt27dSGpqaqntcXFxxMnJiRUNubm5JC8vr8x96enpxMPDgxUdbm5uJCMjo9T2pKQk0r9/f1Y08Pl8kpycXGr7p0+fiLW1NSsauMKiRYtIZmZmqe2pqalk6tSprGqxtrYmERERpbZHRESw9rnY2tqWqeHjx4/E3t6eFQ2EEHL48GHi5ORETp48SUJCQkhISAg5ceIEcXJyIrt27WJ07F27dhEnJyfy+fNnQgghfn5+xMHBgZibmxNzc3MyduxYkpuby6gGrtG1a1dy586dUtvv379POnbsyIoGWdq02bNnEzc3N5KQkFBq38aNGwmfzyd//fUX4fP5jOoQI0s7kpycTKKjo0l0dDTh8/nkzZs3kt/FP/fu3SNWVlaM6ijOhAkTyNKlS8nff/9NLC0tSXJyMomLiyOTJk0iU6ZMYUUDV+wqF2xaee9FZGQksbW1ZUUDIdx43qoqdGWsmjNmzBgIBAIcO3YM+/btAwDo6+tj9uzZJWbemYILDRnFxMXF4e7du1BSUkLnzp1hYGBQYr+Wlhajs5qPHz/G69evAQA+Pj7Yu3dvqdn38PBwREdHM6bh5MmTklUNQggGDRpUagYvIyODlX5BssbPzw+RkZEARM0yLS0tS1XVDAsLw3///ceqLi6sZqurqyMyMrKUhoSEBFbz1kaPHo3o6GisXbsWRUVFIIRAQUEBw4YNk1RGZYIzZ85g3759GDt2rCRUccmSJVBRUcHp06ehqamJGTNm4J9//sHMmTMZ01Gc+/fvY//+/Xj//j0UFBTQrFkzTJgwAT169GBlfADo378/VqxYgbS0NElo5OvXr7Ft2zbW8i5kadMWLFiASZMmoVOnTvjnn3/QsWNHyb758+cDEJU2Z9KeccGOAMCTJ0+waNEi8Hg8EELKzN8khKBz586M6ijOsmXLMHv2bERGRmLJkiXQ1dXF6tWrERYWhgMHDjA2LlfsKhds2vnz53H16lUAovfit99+K9WMPSEhgfEWJVy5TqQFzRmrQaSmpkJRUZHVMu6xsbFYuHAhCCHw9fWFra1tmQ0Zhw8fzuhN+8WLF5g4cSLy8vIAAGpqatixYwc6dOjA2JhfExoaiilTpoAQgtjYWNSvX7/EDVv8XowePRqDBw9mRENubi4OHToEQoikeIm6unqJY9TV1dGzZ09GC2pUpHeRGKbiuf39/TFixAgAkDxQfI2amhrGjx/PalPIo0ePYtu2bZJ+Y4Dogffo0aOYNm2aJI+MSVavXo1nz55hxYoVknLAr1+/xqpVq+Do6IhVq1YxrqE4WVlZ+PjxIwBR4R2m72G//PILXF1dMXLkSACiv33w4MElWjA8fPgQ69evL7dogTS5c+cOfv/9d3Tv3h2tWrWS3E8fPnyInTt3onv37oxrAEShVmvWrMGFCxdQUFAAHo8HeXl5uLm5Yc6cOaUeuphGFjZNIBDAz88PzZs3LzNU1tvbG9evX2fsGuGCHRHj6+sLoVCIMWPGYOfOnSX6e4l1mJmZsf69KE5KSgrq1KkDeXl5xsbgil3lgk1LS0uTNNi+dOkS+vTpUypEVF1dHQMGDJDkvzIBl64TaUCdsWqIr68v7OzsoKCgAF9f328ey2aVOFk2ZHRzc4OGhgZWrlwJeXl5rFq1CmFhYbh27RrrWgBRMYTz589DV1dXJuMDot5rFW1sK20uXboEZ2dnKCkpcSL5m8/n49mzZ4zmIVWGAwcO4NixY5K4en19fUyePJmV1WxAlOvx+++/4+nTpyUc5R49emDt2rWsPPwGBATA3Ny8hCG/c+cO9PX1Ge91Zmdnh0uXLsHExASA6PPYsmULLl26JEkGj4yMRN++fSWzr0wyYMAA/PTTT6UeoHbt2oXHjx/j3LlzjGsoTnZ2NsLCwvDkyRO0adOG8SqKXLVpxUlLS4OGhgarJbS5YEcA0cqDvb09J8qH5+Xl4datWwgLC8OECRPw/v17NG/eHDo6OqyML0u7Whwu2LTFixdj6dKlMm8+zZXrpEqwHBZJkQLm5uYkKSlJ8n8+ny/Jcyj+w1Zc+9dER0eTJ0+ekNzcXIlOprG3tycfPnyQ/B4XF0f4fH6ZMdVskp+fT8LCwkhBQQERCASsj5+bm0suXbpENm/eTFJTU4m3tzdJSUlhVcPq1atJeHg4q2N+i/z8fFlLkJCSkiLT72hYWBi5efMmuXfvHvn06RNr4/7555+Ez+eT//77r8T2iRMnEj6fT9auXcvo+La2tpJcMUIImTx5MmndunWJY0JCQkirVq0Y1SHGysqqhB4xbOWhfJ0/5+/vL8mf4/P5jOfPccmmeXl5kRkzZpD4+HhCCCHx8fFk6NChhM/nE1tbW7J7927GNXyNrO0IIaLrYdGiRWTo0KEkLi6OnDhxgnh5ebGqITExkXTr1o3Y2NgQCwsLEhERQaZMmULat29PQkNDWdPBBbtaHFnbNB8fH3Lq1CmSmZlJPnz4QAoKCmSigwvXyY8i+2kOSqU5f/68JB6XCxV1xBQUFGDBggW4efMm5OTkcPv2bWzYsAHZ2dnYuXMno7MnOTk5Jfoj1a9fH4qKikhPT5fZrM3mzZtx/PhxFBQU4Pbt29i6dStUVVWxYsUKVsI6kpKSMHToUCQnJ0MgEGDIkCFwd3fHmzdvcPTo0RLlm5nk0qVLGDt2LCtjfYtTp07hwIEDiIuLw+3bt3Hw4EHUr18f06ZNY3xsrs78m5qawtTUlLXxAODcuXO4cuUK1q1bV+pv3b9/P65cuYIVK1bAwsKCsTwlMzMz+Pv7w9jYGBkZGfD29i4VCnjz5k3WSvzr6+sjPDwcxsbGJbaHh4dDU1OT0bHLyp9bvHgxq/lzXLFp3t7eGD9+fInebosXL0ZwcDD++OMPaGpqYuPGjTAwMMDAgQNZ0SRrOwIAb968wfDhw2Fra4s3b95AIBAgJCQE69atw+7du1nLG1u/fj2aN2+Oa9euSSomb9iwAbNmzcKmTZskOYZMwhW7CsjWpgGi8PKJEyfi1atX4PF4aN++PTZv3oyIiAgcPnwY9evXZ0UHwI3rpErI2hukVB47OzsSGxtLCBFVkUlPT5exIhHbtm0jvXv3Jl5eXpJqbV5eXqRr167kzz//ZHTs4jOrYsqrGMcGR48eJR06dCAXLlwgNjY2JCIigly/fp04OTmRLVu2sKJh7ty5ZMqUKSQnJ0fyXqSlpZGxY8eyVnmKEFGFsnXr1sl0Bejq1avEwcGB7NixQ1LN8OjRo8TKyoocOnSI8fG5MPPP5/NLaSjvh0lcXV3Jv//++81j9u3bRwYPHsyYhitXrhBbW1vy119/kV9++YVYWFiQgIAAQohoVf3AgQPE0tKSXL58mTENxdm6dSvp2rUrefToEcnMzCSZmZnk0aNHpGvXruSvv/5idOxBgwaREydOSH4PDAwk5ubmZN++fZJtDx48ID179mRMA1ds2vjx40vYqoiICGJubk5Wr14t2Xbx4kUyaNAgVvRwwY4QQsjo0aMl4xW3q2vXrmXtvSCEkPbt25OgoKBSOthcxeaKXZW1TSOEkJUrV5KhQ4eSiIgIyXsRGhpKBgwYQObMmfM/9s48rqb9+/+v02SeuWRIJEVpMlTKELm3QZR5KDSZSYaURGSoZGpQhqISComUmUIJDcjYoEgSKlHRuH9/9Dv72+kU7ue232d373k+Hh6Pzj7nYS0556y91nut1yLiA0Wx53PyTxAmY82QESNGUJ6entT9+/cpOTk56tq1a9SDBw8a/EOSCRMmUHFxcRRF8X5RxsfHU9ra2ozaZlsyZmBgQEtE1/Xj6tWrlI6ODhEf2BC4KIqiTE1N6Zt/LS0taty4cTx/SGBsbEyFh4dTFMX7uzh9+jSjN5lcUlNT6baJd+/e/fQPU5w5c4ZuZwkPD//pHyZRVVX9ZdtqRkYGNXToUEb9OH36NDVlyhRq2rRp1OXLl+nrW7ZsoRQUFKh9+/Yxar8uP378oJYsWcKTJMvJyVGLFi1iXF5fRUWFp0X10KFDlLy8PPXixQv62tu3bylFRUXGfGBLTBs+fDjPv/vUqVOUvLw8TzteRkYGMdluNsQRiqKooUOHUq9fv+bz482bN0QlzOu+V+vHNFVVVSI+sCWuCjqmURRFjR07lkpKSuLzITk5mdLU1CTiA0Wx53PyTxC2KTZDFixYAE9PT/j4+IDD4TSqmkN683h+fj7f8lagVsq7uLiYcfsBAQE8Q7VVVVUICgriExQhoZz37t07DBo0iO+6vLw8sWWIpaWlDS62BWp/N6RQV1eHuro6MXsNkZWV1aAIgbq6OhHlwHnz5iE6Oho9evSAg4MDvL29GZf+rc/+/fuhra2N7t27Izc3V2BD6BISErTq6c9gUh0NqFVUbEiue9GiRVixYgUxQQAAaNGiBQ4cOIDMzEykpaWBoijIyckRa3mqK+KSmJiIDh060EImQO13CZPvFbbEtO/fv/N8Lh8+fAgJCQmoqqrS10RFRYmsagHYEUcAQFxcHCUlJXzX8/LyiH6HDB8+HCdPnoSDgwN9rbKyEr6+vlBTUyPiA1viqqBjGlCrZNnQOpb27dujrKyMiA8Aez4n/wRhMtYMWbJkCczMzFBcXIzx48fj9OnTrFCRkZGRwb179/hkRKOiojBgwABGbffs2ROXLl3iudatWze++YOfBfqmpFevXkhNTUXv3r15rt++fZtvtxNTsCFwAWSS31/RtWtXZGVlNbjb648//mDcvri4OE6fPg11dXU8ePAADx48aFR1lKmZseLiYrx69Qrdu3eHj48P5syZI5BkTEFBATExMT+dx7px4wbxWTYuJOccuFRWVuLNmzf49u0bBg4cCGlpacaTUS5smJ9jS0zr3bs3MjIy0LNnT1RXVyM+Ph7Dhg3j2b2XkJDA973OFGyIIwCgq6uLffv2Ye/evfS1zMxMbN++HWPHjiXmx/r16zF37lw8ePAAlZWVcHZ2xuvXr/Ht2zccP36ciA9siauCjmkAMGTIEFy6dAkLFy7kuR4SEoLBgwcT8QFgz+fknyBMxpopbdu2Rdu2bREUFIRBgwaxQnJ2xYoVsLW1RUZGBqqrq3Hu3DlkZWXRw5RMcvPmTUb//r+LpaUltmzZgk+fPoGiKNy7dw+hoaEIDg6Gvb09ER/YELi4vHz5EoGBgcjKysL+/ftx/fp1yMrKYsSIEUTsz5w5E1u3bqUD6OvXr3H37l163xfTsKHyP2bMGCxcuJDeT6OlpdXoa5k8fZgzZw7Wrl0LWVlZ6Ojo8D1/8+ZNHDhwAM7Ozoz5wBY+fPiAPXv24Nq1azynha1atYK+vj5sbGwYv7GaO3cuNm/ejBcvXiAlJQUVFRX0ZyI/Px+RkZHw9/fH9u3bGfWDDTHN0NAQrq6uqKysxN27d1FYWIipU6fSzz958gTe3t6YPXs2EX/YEEeA2lhiZWUFDQ0N1NTUYMqUKSgpKYG8vDzs7OyI+SEjI4Pz58/j5MmT+OOPP1BTUwN9fX3MmTOHWILMlrgq6JgGAKtXr4aFhQWePHmCqqoq+Pr6IjMzE8+ePYO/vz8RHwD2fE7+CcI9Y82Qursd6lZnGmLnzp2EvKrl9u3bOHjwIJ4/f46amhrIysrC2toaf/31F1E/2EBoaCh8fX3x4cMHAEDnzp1hbW0Nc3NzYj58/PgRJ0+e5Pn/IBm4AF4lrpSUFFy6dAkHDx5EREQEUSWuPXv2IDAwEOXl5QAAMTExzJo1Cw4ODkROIUpKSn6r8s/U0tDKykrcuXMHxcXFcHBwwIYNGxpV6mN695ubmxuOHj2KQYMGQU1NDe3bt8eXL1+QnJyMtLQ0zJw581+fjOXm5mLmzJkQExPDlClTMHDgQLRv3x7fvn3D06dPcf78eQC16pNMn9adOXMGJ0+ehIiICKysrOjv661btyIsLAzW1tawsbFhzD5bYlpFRQU2btyIyMhIiIiIYM6cOXB0dARQq+R37NgxjBgxAocPH0aLFi0Y86MubIgjXO7du0fHkoEDB2LUqFE8i3b/K7AhrgKCj2lAbaE1ICCA53dhYWEBZWVlIva5sOlz8r8gTMaaIWZmZvDx8UH79u1hZmb209cGBwcT8urnlJWVNdpn3dTIy8s32tMvLi6OHj16YPLkyVi6dCmR3v/CwkJQFEXLRlMURWzmoDGePXsGBQUFIrYWLFgAZWVl2NraQlVVFRcuXECfPn2wc+dOJCUl4cyZM0T8AGpnQjIyMkBRFPr37y+QtQdsWKDKhsWlsbGxOHnyJJ4+fYri4mJ07twZqqqqmDFjBi1b/W9m3bp1ePv2Lfz9/Rt8H5aUlGDRokVQUFDAhg0bBOBh7cmYhIQE4/NzbItpJSUl4HA4aNOmDX3twYMHKCkpgY6OjkC+v9kYRwAgMDCQ2EkM24rP9SEZV7mwIaY1xIcPH9CjRw/idtn6OfkVwmRMyD/m+/fvSEhIgJiYGIYPH46WLVvyPB8TE4MtW7bg1q1bRPwJCgrC7t27MXv2bHrANSUlBSEhIZg9ezY6dOiAoKAgmJubw9rausntp6WlQUxMrMGZl5cvX8LR0RFnz55tcrtcnjx5gkuXLkFMTAyGhoY8g/jl5eXYt28fgoOD8fTpU8Z8qMuwYcNw+vRp9OvXjycZe/v2LSZPnoyUlBRG7ZeUlEBMTIzvfQnUVji3bNkCHx8fRn1gQ+U/IiICBgYGkJCQQERExE9fy9R+LyH/h7a2Njw8PKChodHoa+7evYtNmzaxrg1bCPMIOo4AgL+/P6KioiAuLo7Jkydjzpw59HPp6enYuHEjnjx5QkworH6iXl1djbdv36KkpASGhoaMttKyKa4KOqbl5eXhxo0bEBMTw/jx4/lEPI4fP469e/ciKSmJMR+4sOFz0hQIftBIyD/m/fv3aN++Pdq2bYuEhARcvXoVampqmDhxIuO2X7x4ASsrK7oa0atXLwQHB6Nnz54oLi7G1q1bERUVRXQRYlRUFDZs2ICZM2fS13R1ddG/f3+6HUdWVhbu7u5Nmozl5ORg6dKlyMjIAAAoKSnh4MGD6NixIyorK+Hl5YWAgIBGhRuagujoaKxduxYSEhIQExPD0aNHcfToUQwfPhwpKSmws7NDTk4OsYWlgOCUuAoLC+Hg4IDbt2+Dw+Hgzz//hKurKx3AQkND4eHhgcrKSsZ84PLu3TvU1NTQPwsCe3t7jBo1Cl26dPlpHz2Hw2E0GXN3d8fy5cuJnZSzlS9fvjSoPlsXGRkZfPz4kZBH7EGQMY2LmZlZgxV1DofD02HR1II7bIgjQK36qq+vL9TV1dGiRQvs2LEDIiIimDVrFvz9/bFv3z60bt2a6GlUQ6eiFEXBxcWF5xSzqWFLXGVDTLt37x6WLFlCz7ju3r0bx48fh5ycHHJycmBnZ4eUlJSfFpmaArZ8TpoMokL6Qpqcq1evUgoKCtTdu3epN2/eUAoKCpSenh6lrKzMs8iTKczNzSkjIyPq4cOH1OPHjylTU1Nq5cqV1OvXr6mxY8fS+3q4+41IMGTIECo7O5vvenZ2NjVkyBCKoijq/fv39M9NxdKlS6mxY8dS586do6KioqiJEydSGzZsoD5//kyZmJhQcnJy1Lp166iioqImtVsXY2NjasWKFVR5eTlVWVlJubi4UHPmzKGuX79OKSgoUOPHj6fi4+MZs98QGzdupCwsLKji4mKexZBGRkaUg4MDY3bXrVtHDRs2jPL29qYOHTpEaWtrU66urlRZWRm1aNEiSk5Ojpo7dy7PjiUhzFN3+TQXa2trKj8/X0AeCYaGdiPW59OnT4wv4WYbgo5pXLZv307Jy8tTJiYm1Pbt26nt27dTM2bMoOTk5Khly5ZRFhYWlIKCAnX9+vUmtcuGOEJRtXtDfXx86Mfnzp2jDAwMKE9PT0pOTo6ysbGhCgoKGPXhd3nz5g2loaHB2N/PlrjKhpg2c+ZMytTUlHr//j31+fNnauXKlZSlpSWVnJxMDR06lBo+fDh1+vRpxuxzYcvnpKkQJmPNHGNjY2rPnj1UdXU15e3tTU2YMIGqrq6moqKiKD09PcbtDx8+nLp37x79+O3bt5Sqqio1ceJEauLEiTwLNEmhr69PHT16lO/60aNHqQkTJlAURVEJCQnUmDFjmtSuhoYGdfPmTfrxy5cvqREjRlBz5syhtLS0qJiYmCa11xAqKirU8+fP6cfFxcWUgoICpaGhQTk4OFClpaWM+1Cfb9++UTNnzqQGDRpEycnJUcOGDaPk5eUpY2NjRr8otbS0qMjISPpxUlISNWrUKGrp0qWUiooKFRISwpjtX5Gbm0t9+/aNoiiKunfvHrVlyxYeX0lTUFBAXbp0icrJyWHcFtsWtAsKOTm5X97M/heTMUHHNC4rVqygXFxc+K67urpStra2FEXVxpRp06Y1qV02xBGKoiglJSV60TNFUVR5eTklLy9PDR06lDp37hwRH36X27dvU2pqaoz9/WyJq2yIaWpqalRKSgr9+NOnT5SSkhKlo6NDmZubUx8+fGDcB4piz+ekqRC2KTZzMjMz4e3tDREREcTFxWHMmDEQERGBiooKcnNzGbdfUlKCfv360Y/79OmDyspKdOnSBX5+fg32NDPNkiVLYG9vj9TUVKiqqqKmpgaPHz/G5cuXsXXrVmRlZcHBwQF//vlnk9r9+vUrz+JBOTk5lJaWoqysDOfPn6cHSpnk+/fvPP3b7du3p3vcN27cyLj9hmjbti1OnTpFXInry5cvPMta1dTUUFBQgBcvXuDMmTNEW2frcu3aNdja2uLgwYPo06cPrKys0KdPH4SHh6O4uBhz585l3Ie0tDSsWLEC27Ztg5ycHCZNmoTPnz9DQkIChw4dYrzFREgt27Zt+6kqH1cl7b+EoGMalzt37iA8PJzv+syZM2m10fHjx2P//v1NapcNcQSofe/VXYAtISGBli1bYvXq1QKbKW1o3ra0tBRxcXF8u/GaErbEVTbEtLKyMh7F365duwIAlJWVsXv3bmLqmmz5nDQVwmSsmcOVQv727RuePHlCz0C9ffsWHTt2ZNx+TU0NnyqcmJgYVq1aJZBEDACMjIzQtm1bBAQEYM+ePRATE4OcnBz8/PwwatQoPHz4EEZGRk2+jLi6uhri4uI818TFxWFvb0/0i6H+nAOHw+GZnyPNvHnz4O3tDU1NTWhqatLXCwoKYGlp+Usxif+VqqoqvveghIQENm3aJLBEDAAOHDgAS0tLaGpqwtfXFz179kRUVBQuX74MLy8vIsmYm5sb+vbti/79++PixYuoqqpCbGwsTp06hX379uHUqVOM+/BfZ/jw4fj06dMvX8cVIfqvIOiYxqVt27Z4/fo1T7ERADIyMuhZ19LS0iaPc2yJI40hSKXThuZtJSQksGDBAsYlzNkQV9kQ0yiK4ku4REREsHDhQqJrDtj+Ofm7CJOxZs6YMWOwadMmtGnTBu3atYOWlhbi4+Ph7OyMsWPHCswvQX8YdHR0GlwoC9TeBDX10PXP6NmzJzFbjUE6MY6NjUVqaioA4OHDh/Dz8+MTbHjz5g3RSjcXQSZiADsq/ykpKTh9+jS6dOmCO3fuYMyYMejevTumTJmCo0ePMm6/OUgNMw1b1o6wDbbEtClTpsDJyQmFhYVQVlamOyw8PT0xefJkFBUVwd3dnVgsYUMcAUBsf1VDsO0zI6iCc30EHdMAsEZSny2fk7+LMBlr5jg5OWHfvn3IycmBr68vJCQkkJSUBBUVFaxfv55x+xwOp8GKkaBJTExEcnIyKisrQdXb3tDUJ2JcGvpdCIKUlBQeBSGKovDkyRN6GSIXJm8ievXqha1bt9K/++joaJ6qGYfDQevWrWFnZ8eYD2z5/6gPGyr/IiIikJCQQFVVFR48eAAnJycAzFT6G6J+e15lZSV27drFp4gm6L1BTPL+/XtISkqCw+Hg/fv3P31tc73B+F8QdEzjYmNjg4qKCmzfvh3l5eWgKAotW7aEmZkZbGxsEBMTg7KyMmzbtq1J7bLpeysgIIBH8baqqgpBQUF8CnVMxdT6ZGVlITk5GQUFBejQoQOUlZVpifmYmBi0bNmSsRZrNsRVtrw3Pnz4wNdCnZ+fz5eoM/m9xZbfRVMh3DMm5B/R0IJlqpEle6R2kfj4+MDLy4uWRq4Lh8PBjRs3GLErLy8PSUlJnqTj/fv36N69O9+XFJM+cDgcvgS0PhwOh9j/x7hx43DmzBl07tyZiD0u8vLyUFVV5WllSExMxJAhQ/jmdIKCgoj55ejoiPT0dLRp0wYvX75EbGwsEhMT4ezsDA0NDWzdupVxH6ysrNCjRw907twZAQEBuH37NiorK+Hk5AQRERH4+fkxZvtXS33rwrZKeFMyaNAg3L17F126dGl0UT33u5TUZ1UIPz9+/EBmZiZERUUhLS3NeLGCDXEEqP3e/h2YjKlcPn78CEdHR9y9e5cntnE4HKipqcHFxQWLFy+GnZ0ddHV1m9w+W+IqG2La79zzkfjeYsvnpKkQnoz9C4iMjMTw4cPRo0cPHDhwANHR0VBTU4Ojo+NPh8ObAjZWrk+ePAlbW1ssWrSIqF1S1cGfwcYvnfoLaysrK/Hy5Uv079+f0d0w3CH7utQdPBYUbKj8Ozk5wdbWFjk5OdiwYQM6d+4MFxcXZGZm4vDhw4za/jcnWH+HwMBAutJOshjQHBBkTKvL9+/fkZaWRndYcFuvAeZOQNgQRwD+721BUVJSgvnz56OmpgZubm7Q1NREp06dUFxcjPv378PHxweTJk2CsrIyI4kYwJ64yoaYxpbvKrZ8TpoK4clYM+fAgQPw8/PDsWPHQFEU5s6di+nTp+PBgwcYPXo0HB0dBe0icZSVlREdHc2KG++GaOzk8N9KXl4eHB0dsWrVKgwcOBBTp05FZmYmOnTogGPHjvEoIgkRHIWFhejQoQPjMyHz5s37rddxOBwEBgYy6gubyM7ORklJCRQVFQHUJmtjxoyBtLS0YB0jDFti2o0bN2Bvb4+SkhK+ExHhaSU5vL29ERUVhdOnTzc4l5SSkoLZs2fD3NycaBurECFNifBkrJlz9uxZuLm5QU1NDTt27ICKigpcXFyQmJgIW1tb4slYcnIypKWl0blzZ0RERODSpUtQU1PDwoULiSUgampqSElJEWgytm3bNtjb2/MpTebk5GDdunVE1OoqKioQEBAAfX199O3bF46OjnSF2cPDA506dWLcB6D29PTbt2/o3LkzLl26hLy8PJw4cQLh4eHYtWsXAgICiPjx48cPelYqMzMTMTExUFVVhZqaGhH7dWFD5f/9+/d0K29CQgKuXr0KNTU1TJw4kVG7v/pcJiYmIicnh0dW+99OfHw8lixZggULFtDJWFRUFPbt24fDhw//pxQV2RLTPDw8oKmpiaVLl6Jdu3ZEbAL45fxgXUjNEhYUFGDXrl14+vQpfvz4wZecMnlydOnSJaxcubJRgQgfHx+oqanh1q1bxJKxGzduIC0tDdXV1fS1iooKpKamEhFA4sKGmMaGWAYILp41FcJkrJnz8eNHeu9EfHw89PT0AACSkpL4+vUrUV9OnTqFLVu2ICAgAJ06dYKDgwM0NTVx7NgxVFZWEjtWnjhxIlxcXPD06VP0798fEhISPM+T2JFy8eJFJCUlYc+ePbQ0clhYGHbu3MknlcwUHh4eOH/+PEaNGoXbt2/j3LlzWLlyJWJiYuDu7k6sxTQhIQGBgYHo3bs3PDw8MGrUKKipqaFTp06YMmUKER8ePnyIZcuWYf/+/ZCRkcH06dMhIiKC79+/w8PDA/r6+kT8AHgr/7m5ufD09MT06dNx//59eHh4ELnZFOSus8bedyUlJXB1dUVOTg60tLSwfft2xnxgG7t378aCBQtga2tLXwsLC8OePXvg4eHxn1o1wJaY9u7dOxw8eBBSUlLEbAK1s1q/W7gkdTq3adMmPHr0CAYGBnziHUyTm5uLIUOGNPr84MGDoa+vj9mzZxPxx8PDA0eOHEHXrl1RUFCA7t274/Pnz6iuroahoSERHwB2xDQ2xDKAHbs7/zEEF0wLYQBdXV0qPj6eys7OpuTk5Kjk5GSKoigqIiKC+vPPP4n6oqenRx0/fpyiKIravXs3ZWRkRFEURd2+fZvS0dEh5oecnFyjf+Tl5Yn4kJ+fT1lYWFAqKipUUFAQtWjRImrIkCGUr68vVVVVRcSHUaNGUXfv3qUoiqI2bdpEzZ8/n6IoikpNTaU0NDSI+EBRFKWiokLl5uZSNTU11IgRI6hTp05RFEVRGRkZ1LBhw4j4MGvWLMre3p769u0b5e/vT40aNYr68eMHFRISQk2ePJmID1zGjRtHRUdHUxRFUdu3b6dmzpxJURRFPXz4kNLW1ibig7GxMbVnzx6qurqa8vb2piZMmEBVV1dTUVFRlJ6eHhEf6hIXF0fp6OhQw4YNo8LCwojbFzTKysrU27dv+a6/ffuWUlFREYBHgoMtMW3ixInU/fv3idnjcv/+ffpPYGAgNWLECOrYsWPU48ePqefPn1OhoaHUqFGjqJMnTxLzSUVFhXrw4AExe3UZOXIk9eLFi5++5sWLF5SWlhYRf0aPHk0FBgbSP+fm5lJFRUXUnDlzqH379hHxgaLYEdPYEMsoin3x7H9BeDLWzJk1axZWrVoFCQkJyMnJQVVVFSEhIXB3d8fKlSuJ+vLu3TtagSkuLg6jR48GULsD4/Pnz8T8ePnyJTFbjfHHH3/A398fGzZswPbt2yEmJoZjx44RbTf68uULvX8kLi6OXlDZsWNH/Pjxg5gfgwcPxpkzZ9CtWzd8/foVY8aMQUVFBQ4fPkzLEjPN8+fP4e7ujrZt2+Lu3bsYO3YsWrRogTFjxsDV1ZWID1zYUPlnw64zACgrK4OrqyvCwsKgpaWFbdu2QVJSkph9ttC5c2e8fPkSffr04bmenp5OtEWODbAlpq1duxYuLi6wtbVtsMOCqRbBESNG0D/v2LED27Ztw4QJE+hrgwYNQrdu3eDu7o5Zs2Yx4kN9WrZsiW7duhGxVZ8hQ4bg0qVLP40VFy9ehLKyMhF/CgoK6PscOTk5PHnyBHp6enQLrY2NDRE/2BDT2BDLAPbEs3+CMBlr5lhaWqJfv37IycnBpEmTANTuMXJycsK0adOI+tKlSxd8/PgRYmJiePHiBdauXQugNjnq2rUrUV8ETWlpKdzd3REREQFjY2O8evUKK1euhJOTE7GWOCkpKaSmpqKgoADv3r3DqFGjAADXr19H7969ifgAAOvXr8fixYtRVFQEa2tr9OjRA87Ozrhx4waOHDlCxIdWrVqhoqIC5eXlSEpKwvTp0wEAnz9/Jn6z26NHD2RlZaG8vBwZGRnQ0tICUDsr1aNHDyI+sGHX2b179+Do6Iji4mJs3boVM2bMIGKXjUyePBnOzs748uULfVOZmpqKffv2EWmrZhNsiWlLly5FdXU1li5dSly2m0tWVhYGDBjAd11KSgp5eXmM2+diYmICf39/uLi4ELPJZd68eVi4cCHk5ORgYGDA93x4eDgCAwOJzWq1b98eZWVlAGr/HzIyMgDUJuf5+flEfADYEdPYEMsAdsSzf4owGfsXUH8fiJGRkUD8MDQ0xNq1a9GqVSv06NEDI0aMQHR0NFxcXBgPor+zs4cLiSBqYGCAqqoqeHp6QldXF1VVVdi/fz/WrFmD6OhoeHl5Me6DlZUVVq9eDREREWhoaEBeXh4+Pj7w8fHBjh07GLfPRUlJCXfv3kVJSQktyjB//nysWrWK2Beluro6du3ahQ4dOkBERASjRo3CixcvsG3bNqirqxPxgQsbKv9jxozBpk2b0KZNG7Rr1w5aWlqIj4+Hs7Mzxo4dy6jtsrIyuLu7IzQ0FJqamti+fft/8jSsLsuWLUNRURG2bt2KyspKcDgciIqKwszMjHiHAxtgQ0wLCAgQuOqtnJwcgoKCsGnTJtqXqqoqHDx48KdzVE1BXdXTqqoqJCcnIzY2FlJSUjy7nQBm5c5HjhyJFStWYM2aNTh8+DCGDh2Kdu3aobi4GA8fPsTr16+xbt06Yl0n6urq8PDwgIuLC5SVlXHw4EHMmTMHV65cIbpLkw0xjQ2xDBBsPGsqhNL2zZzy8nKEhoY2qOzz9OlTXLlyhZgvNTU1CAkJQU5ODubOnYu+ffsiODgYhYWFWLFiBd8XeFNy7tw5GBoaQkJCAufOnfvpaxva1dHULF68GNu3b0eXLl14riclJcHe3h7Xrl1j3Aeg9lTy3bt3GD16NCQkJHD79m2Ii4tDU1OTiH22UFhYCGdnZ7x9+xbLly+Hrq4uXF1d6dMH0i04N2/epCv/nTp1QmRkJMrLy4lV/n/8+EHvOrO2toaKigq8vLyQk5ODTZs2Napc1hSMHz8e79+/R58+feiTj8b4t+2S+RWlpaXIzMzE7du3oaGh8Z9SUeTCppgmaBITE2FpaYlu3bph8ODBqKmpwdOnT/H9+3cEBgYy2ubt4ODw268lIQaVlJSE48ePIykpCUVFRejcuTOGDRsGMzMzqKioMG6fS15eHpYsWQJjY2PMmTMHs2bNwvPnz8HhcGBvb4/58+cT8YMtMU3QsQz4eTzbvHkzo/tMmwphMtbMcXJyQkREBAYPHozU1FSoqqrizZs3KCgowIIFC4jv3fj27RuKiopoBaqrV69ixIgRRI+Kt23bhnnz5hFXwfpdysrK0Lp1a0G7QQw2nFRevHgR2traPO/DiooKvjkQIcxT/9SjMTgcDmuWrTKFj48PgoKCEBYWhr59+yIlJQXW1tYoKSkBh8OBhoYGfH190bJlS0G7SgxBxrR58+bB29sb7du3h5mZ2U+/t0gtv83JyUFYWBjS09MB1HaBzJ49G3/88QcR+0Iapry8HC1atMD3799x9+5ddO/eHUpKSsTsszWmFRYWEj0hBGqVJVVUVCAuLs5zvby8HDExMfjrr7+I+vO/IGxTbObcuHEDO3fuxMSJEzFhwgS4uLigT58+sLW1RWVlJVFfnj17BgsLC0yZMoUOmK6urqisrIS/vz8GDhxIxI9z585hwYIFRGz9jNjYWBw5cgRZWVkIDQ1FeHg4pKSkMHnyZMZsjh8/HmfOnEGnTp1+KZFM6kZ3x44dPH5UVVUhOzsbERERsLOzI+LD1q1bceLECZ7AJaigxZbK/8uXL5GWloaamhoAtbMw3F0527ZtY8zuzZs3Gfu7mxOhoaHw8/PDggUL6BN0BwcHtGzZEqdOnUK7du2wYsUKHDp06D/VqijImNarVy+6g4PkXO3P6NOnD9asWSNoN1ixQxSolboPCwvDq1evICoqCgUFBcyYMYPYXHpNTQ0OHDiArl27YtasWWjVqhUOHz6McePGEU3G2BDTvn79il27dsHU1BQDBgyAlZUVEhISIC0tjUOHDvEJEjHFvHnzEBcXx5cEZmRkYN26dcJkTAjzfP36lV7wN2DAADx//hz9+/fHokWLsGrVKmzcuJGYL66urhg3bhzPrpyrV6/CyckJrq6uxJb7jhkzBsePH8fy5csZbbf6GXFxcVi+fDkMDQ3x+PFj1NTUoKqqCg4ODqAoirGhfBMTE7qKbmJiIvCZBwCN7hJTVFTE6dOnGU1OuUhLSyMtLa3BYXjSbNu27aeVfxIcPXoUbm5uAGpPoLgNEhwO5z/ZGicITp8+DXt7e3oHTmpqKrKzs2Fra0u/T5csWQJXV9f/VDImyJhWt91OS0uL7+SBBHVP5+rObTUEqdM5tuwQTU5OhoWFBTp16gRFRUVUV1cjNDQUgYGBOH78OGRlZRn3wdPTE6dOneIRMzE0NISvry+A2hEFErAhpu3cuROJiYlYsGABrl27hsTERLi7uyM6Ohru7u6MzsYfO3aMjmEURdHiIfUhmSD/E4TJWDOnc+fOKCgoQM+ePekPJwB06tSJqJw8ADx9+hQ7duzgqc6IiYlh4cKFRPuHP336hOjoaAQGBqJLly58W+BJnAh5eXlhzZo1WLBgAX3SYWtri7Zt28Lf35+xZKxuUFyxYgUjNpoKJSUl2NvbE7ElLy+PtWvX4siRI5CWluZ7T5BagA2w4zQ7JCQE1tbWWL58OXR0dHDu3Dl8+fIFa9aswfjx44n48F8nMzOT5wYiISEBHA4HY8aMoa8NGDAA79+/F4R7AoMtMa2hkwcS1D2d69WrF1HbjREYGIiNGzdCU1MTe/bsgaysLAICAnDnzh1s3ryZWDLm5uYGfX19uLi4QEys9va1srISDg4O2L59O44dO8a4DxEREfDw8IC2tjZ9bf78+ZCWlsbWrVuJJWNsiGmxsbHw8fGBjIwMDh8+DC0tLRgZGUFOTo7xRcumpqbo2LEjampqsGHDBjg4OPCoSHI4HLRu3RoaGhqM+tFUCJOxZs7o0aOxZcsW7Ny5E0OHDsWOHTswYcIEREdHE5UWBYA2bdogJyeH72j648ePRI/P1dXViSvk1efVq1dwd3fnu66npwdvb2/G7WdmZtI7xg4dOoSKigr6OSUlJXoHnKAoLS3F8ePHibWWZGVlYejQoQBqk3VBwobT7A8fPmD69Olo0aIF5OXlkZqaCl1dXdjb28PV1ZUVbb7/BeqeXCcmJqJDhw48ogylpaVo1aqVIFwTGGyJaYI6eah7E02ySPQz2LRDdOfOnXQiBgDi4uJYvHgxLe3ONF++fGkwSZaWliYaW9gQ08rKymgl3Li4OFpSvmXLljwt+EwgJiZGF7U5HA4t4NZcESZjzRw7OzvY29vjwYMHmDNnDkJDQzF9+nSIiYnRR7ik+Ouvv7BlyxY4OzvTR8OpqanYunUrz9JKpmGDAlu7du3w8eNHPhGRjIwMdOjQgVHbmzZtwunTp3H58mX07dsXvr6+aN++PURFRVFWVoaqqipcvnyZWCLUmIAHh8PBli1biPgQHBxMxM7vwIbKf+vWrelgyd2Vo6urCxkZmWazJLO5M3DgQCQnJ6Nv3774+vUr7t+/z3cqeenSJWKztmyBLTGNDScPQK16X0hICNLS0iAmJgZZWVnMnDmTsaXTDcGWHaJSUlJ4+fIl+vfvz3M9NzeX2HoMeXl5hIeH883xnT9/nmjizoaYJiMjg5iYGEhKSuLTp090kh4WFkYXg0lgYmKCwsJCZGVlNTgDvWTJEmK+/K8Ik7FmTvv27XHgwAH68aFDh/DixQt07dqVuNrSmjVr8PbtW5ibm/PcfE+YMIGYUAOXly9fIjAwEFlZWdi/fz+uX78OWVlZjBgxgoh9IyMj7NixgxavKC0txe3bt+Hi4tLg4sqmgpuEHT58GH379qWvHz9+HH369EFxcTGMjIxw8uRJYm2M9QU8gNpqprKyMrEBX6BW/vby5ct4/fo1LCwskJaWBllZWXTq1ImYDwA7Kv9qamo4dOgQNm3ahMGDB+PMmTNYuHAhkpKSmoUM8L+BuXPnYvPmzXjx4gVSUlJQUVFBy2Ln5+cjMjIS/v7+2L59u4A9JQtbYhobTh5evXoFU1NTtGzZEkpKSqipqUF4eDhCQkJw8uRJIjNSgGB3iNbFysoK27Ztw8ePH6Gurg4xMTGkpqZi7969mDNnDh4+fEi/dvjw4Yz4sGzZMixatAiJiYm0pH5qaioePXoEHx8fRmw2hqBj2sqVK7FixQpUVlZi4sSJkJaWxs6dOxESEkL0d3HhwgVs3LiRbvPnLmYHalt9m0MyJpS2b4b8nRkCktUzLq9fv0ZaWhrExcUhIyMDaWlpovafPn2K2bNnQ0VFBSkpKbh06RIOHjyIiIgI+Pj48MxkNCUODg5wdHRE27ZtUVlZCXt7e0RFRQH4P5GEsWPHYv/+/XxV1qZi1qxZMDIy4unXVlNTw/nz5+nEJzAwEOfPn0d4eDgjPrCRz58/Y+bMmSgoKEBFRQWuXLmC7du34+nTpwgMDCRaxfv69Svs7e2hpaWFOXPmYOHChbhz5w5d+Tc0NGTch7S0NFhYWGDBggWYPXs2jIyM8PXrV3z//h2WlpZYvXo14z4IAc6cOYOTJ09CREQEVlZWtOrX1q1bERYWBmtra9jY2AjYS+Zhe0wTFJaWlmjVqhV2795Nx4zy8nKsXbsWFRUVOHjwIBE/frZDdPny5RAVFSXix+/uVeNwOIyuTElJSUFwcDDS09MhJiYGGRkZWFlZMbr3rT5siWlFRUXIz8+n/+1PnjxBmzZtiMZUAwMDKCsrw8rKCrNnz0ZAQAA+fvyILVu2YPXq1UREwv4pwmSsGfKrvU3A/1UGSOxwYhsLFiyAsrIybG1toaqqigsXLqBPnz7YuXMnkpKScObMGUbsDho0CHfv3uVZ9Pz27Vs8f/4cNTU1GDhwIONtDEOHDsXZs2d5EuC6vwOgdp5s2rRpSElJYcyPvzMXR6KtdO3atSgpKcHevXsxcuRIXLhwAe3bt8eqVavQokUL+Pn5Me5DY1AUJZDK/48fP1BWVobOnTvj8+fPiIyMhKSkJPT09Ij5IKRh8vPzISEhQfzUVlCwNaYJ+uRBVVUVp06dgpycHM/1ly9fwtTUFImJiUT8YAt/p4WaLeInTMGmmPb+/XtkZmZi+PDhKC0t5bkHIsGQIUMQEREBGRkZmJmZwdraGqNHj8bVq1fh5+fXLArPwjbFZggpOdvfoW4CwoblvkDtydjmzZv5rs+dOxdhYWGM2W2oriElJUV0+XRNTQ3f4sMrV67w9PVLSEgwXsn83S8/DodDJBlLSEjAoUOHeMQQOnTogPXr1/9SProp+FXlv2PHjqiqqsL79++JVf5btmxJr0Ho2rUrzM3NidgV8mu6d+8uaBeIwqaYxqX+ycP06dMREBBA9OShTZs2DSqsklBdrdvp4eDg8NPXkpqfE1SC5e3tTZ9S/qrQSGpmXdAxDajdjbl+/XpcunQJIiIiuHLlCtzc3FBaWgovLy9iq4UkJCRo8Y6+ffsiPT0do0ePhqKiIt68eUPEh3+KMBlrhvxs7on0BvYdO3bQcqINzQYJAnFxcZSUlPBdz8vLY1yZTND/fklJSaSlpfEErfqnLU+fPmV8oSnbFvuWlpaidevWDT5XVVXFuP1fLeAGmK/8/44PXEgtBBciBGBXTOPi6uoKWVlZXLx4ESNHjgRQK62+atUq7Nq1i8jJg4aGBtzd3eHp6UlL7BcWFmLXrl3Q1NRk1Pa7d+9oMYR3794xaut3EdTetfDwcMydOxetWrX6aaGRVHEREHxMAwBfX196Pp8r6W9mZgYHBwd4eHjA2dmZiB/cnaWrV6/GwIEDERsbC0tLS2RkZPAVp9mKMBlrpmRmZmLnzp3YtGkTz8nL+vXr8fXrV2zevJnIiYyJiQn9c2PLfUmjq6uLffv2Ye/evfS1zMxMbN++HWPHjmXUdmOLB+vD1A23jo4ODh48iNGjRzd4+lVVVQV/f39i6pYfPnxA165deaSIExMT0adPH6LV/+HDh+PkyZM8Fd7Kykr4+vrSMvNMwobKP1uWgAsR0hBsiWlc2HDysGbNGsyePRs6Ojp063l2djY6duyIHTt2MGq7rlofG5T7AP6TsaqqKrx58wZpaWm0+A0T1C0u/qzQyE1eSSDomAYAUVFRcHZ25lklpK6uju3bt8POzo5YMrZixQpYWVmhY8eOMDExgY+PDwwNDZGXlwd9fX0iPvxThMlYM+Tt27eYO3cuOnfujPLycp7nxowZgyNHjmDWrFk4ffo00WP9iooKBAQEQF9fH3379oWjoyOio6OhpqYGDw8PYn3269evh5WVFTQ0NFBTU4MpU6agpKQE8vLyjKs61l88SBoLCwucP38ec+fOxbp162g1MKB2sNbDwwOFhYWMBi4uhw8fhqenJ44dO8bjh5eXF5KSkmBnZ0fspmb9+vWYO3cuHjx4gMrKSjg7O+P169f49u0bjh8/zrh9NlT+2b4EXMh/FzbGNDacPEhKSiIqKgrnz59Heno6KIrCjBkzYGRkxHgLGBtFVRprh/Tx8cGHDx+I+DB+/HicPXuWbxl4fn4+Jk2ahPv37xPxQ9AxDaj9NzdUIJGUlERxcTERHwBg2LBhuHLlCioqKtCpUyeEhITg1KlTkJSUJHaP8U8RCng0Q9avX4/Pnz/D19e3wZu479+/w8LCAv379ycqi7xjxw6cP38eAQEBKCgowOLFi7Fy5UrExMSgX79+xBdY3rt3j0c8Y9SoURAREWHMnry8POLi4ogPr9YnPT0da9euxatXr9CyZUt06NABxcXFKC8vx4ABA7Bv3z7G5x2uXr0KW1tbLF26FPPnz+e5cfj27RsCAwPh6+sLHx8fxk8ruXz8+BEnTpzAixcvUFNTA1lZWcyZM4fxlk0ujVX+bW1tiVX+v3//juDgYEybNg2dO3emr3t7e0NCQgLz589nTOlTiJDGYGNMW7hwIfr16wcHBwdaBKlHjx5Yu3YtSktLceTIEcZ9mDZtGrZt20ZUpY8LW0VVGuLdu3cwNjZmTNAkOjoad+7cAQCcO3cOBgYGfN+Tubm5SEtLQ0JCAiM+NISgY9qUKVMwe/ZsTJ8+nUcozMfHBzdu3CAmnFF3vrEuX758wYYNG3hWZbAV4clYM+T+/fvYs2dPo9X0Vq1aYfny5XByciLq1+XLl7Fnzx4oKChg8+bNGDFiBBYvXgxtbW16MzsJ5s2bB29vb2hqavL01RcUFMDS0hIRERGM2GVLC5isrCwiIiKQkJCAhw8f4vPnz+jUqROGDh0KbW1tRhNSLseOHcOKFSvoPvK6tGvXDsuXL0dZWRn8/f2JJGPcAexVq1bxXC8pKcH27dvh6OjIqH02VP5LSkqwYMECvHz5EsOGDeNJxr59+4ZTp07h1q1b8Pf3b/REQIgQJmBjTGPDyUNOTo7APotsaK3+XVJSUhgVpeKqWnLPLt6/f88zi8ThcNC6dWuiS8mB2nnw+jGNJCtWrICtrS0yMjJQXV2Nc+fOISsrC1euXOEZE2GCpKQk5OTkAAAiIiKgoKDAl4xlZmbi3r17jPrRVAiTsWZIUVHRL+dt+vbti8LCQkIe1fLlyxf6xCUuLg4zZ84EUKsU9+PHD0Ztx8bGIjU1FQDw8OFD+Pn58QWxN2/e/C1p3L8Lmw6ZORwOhg4d2uiQd0JCAjQ0NBizn56ejm3btv30NZMmTcLZs2cZ8yEzM5P+DPj4+EBeXh4dOnTgeU1aWhrCwsIYT8Z8fHygoKDQYOXf2NgYf/31FywsLHDgwAHGKv9HjhzBt2/fcOnSJb5l2w4ODpgxYwasrKxw9OhRLFu2jBEfhAhpCDbGNBkZGVy4cAEnTpzAH3/8gZqaGujr6xM9ebCysoKjoyMsLS0hJSVFq59yYbI98Get1YKioZazkpISvHr1CnPmzGHMrqSkJJ2cmpmZwdvbmy+WkKawsBCHDx9Geno6Kioq+J4nkUzr6OjA09MTBw8ehKioKPz9/SErK4u9e/fSOxOZgsPhwN7env65ofuN1q1bw9LSklE/mgphMtYM6dGjB7Kzs39aQc/OzuaRMyeBlJQUUlNTUVBQgHfv3mHUqFEAgOvXrzMevHr16oWtW7fSCVF0dDTPCRC3csXkzNjLly/pn382B8R0IsRlyZIl8PPz46nglZWVwdXVFadPn2a8teRXyWmrVq1QXV3NmP2cnBwsXryYPrFsTOVq6tSpjPnAhQ2V/0uXLsHOzo4vEeMiIyODVatW4dChQ8JkTAhR2BjTBH2aDgD79u1DdXU1Hj58yNN5Qbo90MzMrMHODw6HA3FxcfTo0QOTJ0/G8OHDGfWjZ8+efH6Ii4vD1NQUkyZNYtQ2l5+JmXz48AE9evQg4oednR1SU1MxcuRIviSdFP7+/pg4cSJCQkKI21ZTU6PvudgyIvJPECZjzZDx48fD19cX6urqPCp1XKqqqnDw4EFajpcUVlZWWL16NURERKChoQF5eXn4+PjAx8eHceWnAQMG0HLc48aNw5kzZ3jasEizZMkSvlMQkokQUNvDvmzZMvj4+EBcXBzx8fHYuHEjSktLf3lq9U8ZOHAg7t+//9PZtLi4OEZnpMaOHYubN2+ipqYGurq6OH36NM97gpug1x/EZgI2VP4/fPjwy9kTNTU1Rk+PhQhpCLbENDadpgPA0aNHGbfxOwwaNAjBwcEYNGgQhg0bBgB4/PgxHj9+DF1dXeTl5cHc3Bz79+/H+PHjGfPD1dWVsb/7d8nJyYGbmxvS0tLoYiJFUaioqEBhYSGeP39OxI+kpCQcPHhQoCeYvr6+0NXVFZh9LnUL4c0VYTLWDLGysoKxsTFMTU2xcOFCqKqqokOHDvjy5QuSk5Nx+PBh5OTkEP/iMjY2xqBBg5CTk4PRo0cDqN2M7u/vz/hOlLrUl56trKzEy5cv0b9/f7Rp04aID7m5uVi+fLlAEiEuwcHBMDc3x/Lly9GtWzecOXMGBgYGcHR0ZLyCNG3aNLi7u0NNTa3BBODFixfYv38/li5dyqgf3DaeGzduNFhVJQUbKv8dO3ZEQUHBT30oKioSqBqokP8mbIlpbDpNB9jTKvjhwwfMnTsXGzdu5Lnu5uaG/Px8eHt749ixY/Dz82MsGaupqUFKSgpevnyJkpIStGvXDgoKClBWVmbEXmNs3boV2dnZ0NPTw9GjR2FhYYGsrCxcu3YNW7duJeZH9+7did3PNIaysjJu3rwJc3Nzgdh//fo1Tp8+DWtra3Tu3BklJSXYtGkTYmJi0LVrVyxbtgyTJ08WiG9/F6GaYjMlKysL69atw9OnT/naF1RVVeHi4oIBAwYQ9WnXrl2YOnUq+vfvT9RuffLy8uDo6IhVq1Zh4MCBmDp1KjIzM9GhQwccO3YMgwYNYtyHT58+wdzcHL169SKeCNWlsLAQ5ubmSE9Ph6enJ9Eqlo2NDW7cuIGxY8dCTU0N7du3p2+ubt++DW1tbRw4cICIoEhlZSX8/f0FtnbB3d0dT548wbFjxxqt/Jubm6Nfv36MBXQ7OztISEj8tBjg6OiIoqKiZqE+JeTfBVti2vv37wV6ml53b9SvIKVQrKqqivDwcPTr14/nenZ2NkxMTJCSkoKcnBxMmjQJKSkpTW7/zp072LJlC3Jzc3na3zkcDvr06YMtW7YQK/gOHToUBw4cgLq6OoyNjbF161YoKSlh7969yMjIgI+PDxE/bty4gYMHD8LW1hZ9+vThi6Mk1g3Y2Njg6tWraN++PaSlpfkUJpmcW3vx4gXmzJmDjh074vjx4+jVqxdWr16Ny5cvY8GCBWjXrh2OHDmCXbt2Ydy4cYz50VQIT8aaKf369cOZM2fw7NkzpKam4uvXr+jUqRPU1NQYly1vjMTERAQEBGDIkCGYOnUqDA0NGd+F0hA7d+7Et2/f0LlzZ1y6dAl5eXk4ceIEwsPDsWvXLgQEBDDuQ7du3RAUFARzc3PcuXMH3t7eRBKhhpQijY2NsXfvXpw9exYlJSU815lk//799L6P69evA6gNnoqKiti8eTOmTZtG7KRq165dOH/+PEaNGoXbt2/j3Llz9NoFd3d3xm9q2FD5t7CwwIwZM9CuXTssXryYp/3qy5cv8PPzw/nz53Hs2DHGfBAipDHYEtMEfZr+7t07+meKopCYmIiuXbti8ODBEBMTw8uXL5Gfn89oO2B92rZti9evX/MlYxkZGfRS7NLSUkZmlx48eIDFixdj9OjR2LZtGwYOHIj27dvj27dvePr0KUJCQmBtbY1Tp05BUVGxye3Xp6Kigm6v79evH169egUlJSUYGxvDzMyMcftcKIpCZmYmLCws+K6Tmids3bo14/cRjXHgwAGMGjUKe/bsgZiYGPLz83Hp0iUYGxvT2gDt27dHQEBAs0jGhCdj/0IKCwsFNi+VlZWFiIgIREZGoqCgALq6ujAxMYGWlhaxoDZixAgEBgZi0KBBWLVqFSiKwv79+5GVlYUpU6YwUrkDGk6EioqKsHfvXmhpafGoCzH1Bfa7+2hI74apqKhAcXExOnbsyCMowg0cTDN69Gjs3LkTWlpa2Lx5M968eYNjx47h6dOnsLa2JiJ/y4bK/9WrV+Hg4IDy8nL069ePPq3Mzs5G69atsXnzZkycOJFRH4QI+RknTpyAkZGRwNtlBX2aDgAeHh7Iy8vDzp076fnj6upqbNq0qVEFOSbYu3cvTp8+DVtbWygrK6OmpgaPHz+Gp6cnJk2ahIULF2LNmjVo27YtPD09m9S2hYUFunfv/tOCmaOjI0pLS7Fv374mtd0Qenp6cHBwwJgxY+Dp6YkvX75g06ZNyMzMxLRp0xi7v6jPuHHjMGjQIMyYMYNOiOvClhZXptDU1MShQ4cwZMgQALX73zZs2AA/Pz+MGTMGQO0s2Zw5c5CcnCxIV38L4clYM+fr16/YtWsXTE1NMWDAAFhZWSEhIQHS0tI4dOhQo8ppTNGvXz/Y2trC1tYWDx48wOXLl7FixQp06NABU6ZMwcyZM38pZPBPqaysRIcOHUBRFO7du4fVq1cDqO05b6hFrKngyqw2xK1bt3Dr1i0AtYkQU8kYGwdZx48fj7Nnz6Jbt2481/Pz8zFp0iTcv3+fcR8EuXaBS93K/9OnT1FcXEy88v/nn39CTU0NFy5cwLNnz/DlyxcMHjwYc+fOhb6+PpEbTCFCfsbBgwfh5uaGcePGYerUqUQLeXUR9Gk6AISGhuLUqVM8QlCioqKwtLSkF0KTwMbGBhUVFdi+fTvKy8tBURRatmwJMzMz2NjYICYmBmVlZYz48+zZM6xZs+anr5k5c2aDOy2ZwMTEBHZ2dnB3d8fYsWMxb9489OzZE3FxcZCTkyPiA1BbdLe3tyd+j1ef3NxchIWF4dWrVxAVFYWCggJmzJjBuPLpt2/feGwkJiZCVFSUR9GzTZs2qKmpYdSPpkKYjDVzdu7cicTERCxYsADXrl1DYmIi3N3dER0dDXd3d3h5eQnErydPnuDq1au0mMbw4cPx8OFD+Pv7w8XFhVEZ2sGDB+PMmTPo1q0bvn79ijFjxqCiogKHDx/+7ZOj/wU2JkIRERFo0aIF9PX1AdQGVV1dXRgZGTFqNzo6Gnfu3AFQ+2W9detWvn7y3NxcYjdZgly7UB8FBQUoKCgQtVmXrl27YsiQITAzM+M5pQSA8vJyxMTEML4jRoiQxoiJiUFcXBwiIiKwfPlydOjQAZMnT4aJiQlfmxyTXL58GXv27IGCggI2b96MESNGYPHixdDW1oa1tTURH8TFxfH+/Xu+Yk1mZibRZdAiIiJYv349bGxskJmZCVFRUUhLS9Ntibq6uoy14de/6W6I7t27o7i4mBH79Vm4cCFatGgBiqKgpKSEpUuXwtfXF5KSknB3dyfiAwCoq6sjJSVFoMlYcnIyLCws0KlTJygqKqK6uhqhoaEIDAxEcHAwBg4cyJjt7t274927d5CUlAQAxMfHQ1lZmedz8ejRI2KrBv4pwmSsmRMbGwsfHx/IyMjg8OHD0NLSgpGREeTk5DB37lyivuTl5eH8+fM4f/48srKyoKysjKVLl8LAwICeHfPy8sKOHTsYTcbWr1+PxYsXo6ioCNbW1ujRowecnZ1x48YNHDlyhDG79RFUIsQlKCgIHh4ePLurevToAScnJ3z//h0zZsxgzLaqqipOnTpFD1u/f/+e58afOwjv5ubGmA91EeTahfqUlZXh2LFjSE5ORmVlJd8+NhLLOufNm4e4uDi+duaMjAysW7dOmIwJERgcDgfa2trQ1tZGaWkprl69iitXrtBqvdOnT4ehoSHju5XYcJo+ceJEWoxKUVERNTU1SE5OhpeXF6NLjhvi+/fvSEtLo7+zUlNT6eeY3C/2Ox0toqKixE5AOBwOFixYQD9euHAhFi5cSMR2XYYNG4bNmzcjJiYGUlJSfL+jxpRAmxI3Nzfo6+vDxcWFtl9ZWQkHBwfs2LGD0fnjCRMmYPfu3XB2dkZsbCzy8vKwaNEi+vn8/Hx4eXkRna38JwiTsWZOWVkZXRmIi4ujK3YtW7ZkdKFuQ4wbNw6dO3fGpEmT4O3t3WDr1eDBgyEtLc2oH0pKSrh79y5KSkrQvn17AMD8+fOxatUqIjulAMEmQlyCg4Ph6uoKAwMD+pqDgwMUFRXh4+PDqA+SkpJ0UmFmZgZvb2++fT0kMTY2hry8PN69eyfQtQsAsGnTJty4cQNaWlp8rZtMcuzYMTr5pSgKWlpaDb5OSUmJmE9ChPyM79+/o7i4GF+/fkVVVRVERETg6+uLPXv2wMPDg9HPLhtO09euXYsfP35g8+bNqKqqAkVRaNGiBUxNTYkuZr9x4wbs7e1RUlLCVzxiev6Yw+EIbCVJY8TExCAtLQ0VFRV8z5FIggDg5MmT6NSpEx49eoRHjx7xPMfhcIj48fLlS+zcuZMnERQXF8fixYsxffp0Rm0vW7YMixcvpkc+xo0bRxdMfH19ceDAAfTt2xdLlixh1I+mQijg0cyZOnUqpk+fDklJSSxatAiXL1+GtLQ0PDw8cP/+fZw+fZqYL1wZc1FRUWI22cqECRNga2vLkwgBQGRkJHx8fHD58mXGfVBWVkZkZCTfYuWcnBwYGhriyZMnjPvQHCgvL+droWSSoUOHwsPDAzo6OsRsArXy+RcvXkRNTQ02bNiADRs28IgkcE8rNTQ06CKGECGkKS8vx9WrV3H+/HkkJCSgS5cuMDY2xpQpU9C3b18AwJYtW3Dz5k3ExsYy5kdERAScnJwgIiICVVVVHDt2jOc0naSKXGlpKbKyssDhcNCvXz+iLYoAoK+vD1lZWSxdurRBYZWf7S78p8jLy0NSUvKnK1Bqamrw4cMHIqJUW7duxYkTJ9ClSxe+uMHhcHDjxg3GfWALRkZGWLJkCd99TmxsLNzc3BAdHc24D+np6RAREeEp/l+7dg0fPnzAlClTBL6L7XcRnow1c1auXIkVK1agsrISEydOhLS0NHbu3ImQkBBi+y64jB8/Ht+/f0dGRgY95FsXJlsZ6iIvL//TShqJL+yPHz82KLOroqKC9+/fM24fAKSlpXHz5k2elgqg9ouS6cruoEGDcPfuXXTp0oUV/x9FRUXw8/NDWloafWJMURQqKyuRkZGBxMRExn3gUj9wkEJMTIy+geRwODA0NOQRBhAihA1oamqiqqoKY8eOhY+PD0aNGsV3I66pqcn4TS9bTtO5MZXbHvjs2TP6OVIx9d27dzh48CBfYY8EpE6afpeoqCg4Oztj1qxZgnYFAPD58+cGT+hI7BmzsrLCtm3b8PHjR6irq0NMTAypqanYu3cv5syZg4cPH9KvZeq9Kisry/fvnzBhAv1zQkICNDQ0GLHdlAhPxv4FFBUVIT8/nxanePLkCdq0aUP8hi82NharVq3Cjx8/iLcy1CU8PJzn5r+qqgrZ2dmIiIiAnZ0dkY3s3IHz+onQ8ePHceLECSIVowsXLsDe3h6GhoZQVlYGAKSmpiIqKgouLi4wMTFhzPa5c+fom/36/x/1YdIPLqtWrcK9e/egpaWFy5cvw9DQEJmZmXj+/DlWr15NtOff0dER3bp1w6pVq4jZbIjc3Fw8fvy4wUAuqN0xQoQEBQXByMioQWVP7iqMqqoqRpVx2YIg2wPrYmRkBCcnJ4HLpefn5zOuxvwrtLS0cPz4caJiMg0RGxsLBwcHFBUV8VwnuWeMLat0LC0t4evry1NcLCsrg6urK06fPk10jc//ijAZa+ZYWFhg/PjxGDduHD07Jii4J3PLli1rsM2JyVaG3+HSpUs4ffo0kaXPgkyE6hIdHY2goCC8evUK4uLikJGRwcKFC4m3yAkadXV1uLm5YezYsTA0NMTu3bshLy8PJycnlJeXE1XB2r17NwIDAzFgwAD069eP73SKhGR2WFgYtmzZ0uBcKekddEKE1IW7CqP+fC/JVRhArcjNzyAhtCPI9sC6xMbGwsPDA7a2tujfvz/fdxaJUxig9uZfQUEB48ePh66uLqNqfY3h4+ODt2/fwsXFRaCdBX/++Sf69++POXPmNChmQyJxzs3N/e3XMvle1dPTg5SUFHx8fCAuLo74+Hhs3LgRpaWlsLOzw9SpUxmz3VQIk7FmjqurK2JjY5GdnY1BgwZh3Lhx0NXVZVTCvTGGDBmCixcv0n39bCM3NxcGBgZ4/PgxEXv/5UTI29v7t19Log1FUVER165dg6SkJFasWIHx48fD2NgYr169wsKFCxmdPamPmZnZT58PDg5m3Idx48ZBR0cHtra2tNKpECGCou4qjHPnzsHAwKDBVRhpaWlISEgg4pODgwPP46qqKrx58wZpaWmYP38+bG1tGfdhyJAhiIqKEkh7YF0UFBTowk39hfUkizevXr1CbGwsYmNjadny8ePHY/z48Rg+fPhP58qaitevX2P27NkoKytDt27d+Lo+SM2MqaioIDw8HP379ydij818+vQJ5ubm6NWrF7p164YzZ87AwMAAjo6O6NKli6Dd+y3+/ef8/3Ls7e1hb2+Pt2/fIiYmBrGxsTh06BC6du2K8ePHw9HRkZgv0tLS+PTpEyuTsdLSUhw/fpzxRYR1MTAw4BtsJU1hYSGysrJo2V+KolBRUYHU1FRGVYbCw8N5Hufl5UFcXBx9+vSBmJgY3r59i8rKSigqKhJJxrp3747c3FxISkpCWloar169AgC0atWK2H4aLiSSrV/BDV7CREwIG2DbKgyg8RNqHx8ffPjwgYgP0tLS+PDhg8CTsaNHjwrUPhc5OTnIyclh4cKF+Pr1K+7cuYOrV6/CwsIC7dq1I5Kor1u3Du3bt8fUqVOJC6nURUNDA8+ePSOejP3qxLguJE6PAaBbt24ICgqCubk57ty5A29vb8b23jGF8GTsX0RBQQESEhJw8+ZNXLp0CRRFMV6xqitGcevWLRw/fhyOjo7o27cvn6oiyVaGhmaUOBwOtmzZwrjkKhdBJUJcLly4gI0bN6KiogIcDoeuYgK1LQPXr19n3AegVlI9JiYGu3fvpqtUX79+hZ2dHQYOHIjVq1cz7oObmxuuX78OV1dXVFRUwNbWFs7Ozrh+/TrS09Nx/vx5xn2oS2lpKS5cuIC0tDSIiYlBVlaWZx8f08yYMQNLly7F2LFjidgTIuR3YcMqjJ/x7t07GBsbExH9YUt7IJugKApPnz5FQkIC7t+/j6SkJFAUhaFDh8Lf359x+0pKSjhz5oxAWiTr8uHDB0ybNg0jR45Enz59+O55mCpy1j0xLi8vR3R0NAYNGgQVFRWIiYnh6dOnePLkCaZPnw5nZ2dGfABq1U7rU1RUhL1790JLS4tnV2ZzmIEWJmPNnKtXr+L+/fu4f/8+MjMz8ccff0BTUxMaGhoYOXIk/vjjD0bt1098uG8nQbYyNCQYIS4uDmVlZWLb6tmQCBkYGEBZWRlWVlaYPXs2AgIC8PHjR2zZsgWrV68mImQCACNHjkRAQABf62xaWhrMzMyIzIBUVFRg165dUFJSgpGRETZt2oSwsDC0a9cOnp6eRNXR3r9/D1NTUxQUFKBfv36oqanBmzdv0KVLF5w4cQI9evRg3IerV6/C1dUV5ubmDd7kkVJpEyKkuREZGYlt27YR+d4SZHvg4sWL4eHhwVMgunv3LoYPH063kBYWFkJXVxfJycmM+VHfp6SkJPz48QMKCgpQV1fHyJEjoaqqSmx+y8TEBI6Ojhg2bBgRe43BjWGdOnVCq1ateJ4jJbHv4OCADh06wN7enuf6vn37kJmZCS8vL8Zss0U8pKkQJmPNHHl5eYiIiGD8+PFYtGhRg3LqTPLgwYPffq2glZhIwoZEaMiQIYiIiICMjAzMzMxgbW2N0aNH4+rVq/Dz8+NrJWSK4cOH4+DBg1BTU+O5fu/ePdja2hKbAanPly9f0LZtW+KqbCtXrsTnz5/h6elJt81+/vwZq1atQvfu3bF7927GffhZIGsuwUvIvwe2rcIAGm7HKikpwatXrzBnzhwiIwC/iq9MxtS6/ydc1NTUcP78ebqo+fnzZ2hra+Ply5eM+VEXdXV1fP36FWPGjIGOjg40NTWJt3BGR0dj3759sLS0hJSUFF/8IFXIUlNTg5OTEzExsIZQVVXFuXPnIC0tzXM9OzsbxsbGfMuohTSOcGasmePt7Y179+4hLi4Os2bNgpKSEjQ1NaGpqUkfGzMJNxiUlJRAXFyc6PLcurBNMCInJwdeXl6QkZGBnJwcCgsLMW7cOFRVVcHPz49IMiYhIUFXC/v27Yv09HSMHj0aioqKePPmDeP2uYwbNw5OTk7YtGkTFBUVQVEUkpKS4OLiAkNDQ2J+pKenIz09XeBS7vHx8QgICOCZX+zatSvs7OxgbW1NxIf/0mJSIexnx44dtFogCTXR36Fnz54NdliYmppi0qRJRHxoLNl69uwZTp06xWgy1lCdvqFrP0ucm5qEhAQ8e/YM8fHxiI6Oxvbt29G1a1doaGhAU1MTRkZGjPvAbavfvHkz33MkC1mtWrXiK3CSpn379nj+/DlfMpaYmEhcOCMiIgItWrSAvr4+AMDGxga6urpE3hNNgTAZa+bo6urSg4p5eXmIj4/HvXv3cOTIEYiIiCAlJYVR+9zZn9u3b4PD4WDs2LFwcXFB586dGbVbn9895eFwOESSMTYkQoqKijh9+jRWr16NgQMHIjY2FpaWlsjIyOAZjmcaJycn2NjYYP78+XTgpigKenp6WL9+PREf/Pz8sG/fvgaf43A4RJMxUVFRvrYSAGjRokWDiSITcGWGKyoq8O7dO0hJSYGiKKLvCyFCuNSt7j979gzz5s0TuGiFq6urQO3X58ePH4iMjERoaCiePXuGFi1awMXFRdBuEYXD4UBRURGKiopYuHAhCgoK4Ofnh9DQUJw7d47IjTdbCllz5syBl5cXXFxcGownJJg5cyY2bdqEzMxMKCoqoqamBsnJyQgJCcG6deuI+REUFAQPDw84OTnR13r06AEnJyd8//4dM2bMIObL/4owGfuXkJ+fj3v37iE+Ph7x8fEQERGBlpYW43bd3d3x5MkT2NjYQEREBMHBwXB2doanpyfjtuty8+ZNovZ+BRsSoRUrVsDKygodO3aEiYkJfHx8YGhoiLy8PLp6RIK2bdvC398fr1+/RlpaGkRERDBo0CBi83tA7Zf10qVLsWjRIoGd3nJRU1PDgQMH4O7uTr8XKisr4efnR6zSSVEUdu/ejeDgYFRWVuLKlSvYu3cvWrVqBWdnZ2FSJkRgnDt3DgsWLBCI7YcPH/72a0m1o6WlpSE0NBQXLlxASUkJOnXqhGXLlmHOnDlE7LMJiqKQmpqKuLg4xMfH49GjR+jQoQMmTZpEbGUMWwpZiYmJePjwIS5fvowuXbrwdUGRSBqXLl0KUVFRHD9+HD4+PgAASUlJ2NnZEX1/BgcHw9XVlUe92sHBAYqKivDx8REmY0KYZ/v27YiPj8fr16/xxx9/QEdHB66urtDU1CQy0Hr79m24ublh1KhRAGp7iM3NzVFVVUV8FgeoVRjq2rUrj+3ExET06dMH3bt3J+YHGxKhYcOG4cqVK6ioqECnTp0QEhKCU6dOQVJS8m/J0zYV/fv3F9hOlMrKSkyePFngiRgArF27FrNmzcKECRPoGc/U1FR6/QIJgoODcf78eWzevBlbt24FUHvKvmXLFnTt2pXIDiUhQhpizJgxOH78OJYvX0589YKZmRktuPQzmG5Hq6iowKVLl3Dq1Ck8evQIIiIi0NDQwL1793Ds2DGBK/kJCnV1dXz79g2ysrLQ0dHBunXroKSkRNQHthSyhg4diqFDhxKx1RgXL17ErFmzsGjRIhQVFYHD4fAtayfBx48fG9RLUFFR4VH8ZjPCZKyZk5SUBH19fYwbNw6DBw8mbr+wsJAnMKiqqqK6uhoFBQVEkx8AOHz4MDw9PXHs2DGeLykvLy8kJSXBzs6OWBLChkTIwcEBjo6O9P/DgAEDsHHjRnz58gU2NjY4cOAAY7Z/NYRfFxI99pMnT0ZYWBjR1onGkJGRQUREBE6cOIH09HRQFAUjIyPMnj2brroyTWhoKDZt2oQJEybQrU4GBgYQFxfHzp07hcmYEIHx6dMnREdHIzAwEF26dOEroDBZ8WdDC5qrqyvOnTuHr1+/Qk1NDRs3boSenh66dOkCBQUFIouNuaSkpPCsGKAoCk+ePKH3rJHe0bhy5Uro6OgQ+55sCLYUskiMW/yKrVu34sSJE+jQoQM6deokMD+kpaVx8+ZNvhP12NhY9O7dWzBO/U2EyVgzhzsr9f79e9y5cwfDhw9HaWkpseHJ+idgoqKiRGdfuFy9ehX79u3D0qVLIScnx/Oct7c3AgMD4ebmBikpKSK7lQSVCCUlJSEnJwdA7UCrgoICX3U5MzMT9+7dY8Q+lx07dtDJWG5uLg4fPoyZM2dCVVUV4uLiSE1NRUhICJF9awBgZWWFyZMnIzo6Gr179+ZLFEktp+TSq1cvgSaG7969w6BBg/iuy8vL49OnTwLwSIiQWtTV1aGuri4Q2z+7yS8oKCASV48dO4b+/ftjx44dGDduHFGBjPqsWLGC75RwzZo1PI9J+mdqaoq8vDx4eHjQOxoHDBiAmTNn/icLWS9fvkRaWlqDu0y3bdvGuH1paWmkpaVhwIABjNv6GZaWlrC3t8ezZ8+grKwMoLbbJCoqqtnMVQqTsWZOZWUl7OzscOnSJYiIiODKlStwc3NDaWkpvLy8iLd5CIpjx45hxYoVWLx4Md9z7dq1w/Lly1FWVgZ/f3/GkjE2JEIcDofe+cHhcBr8Qm7dujUsLS0Z8wEApkyZQv9samoKJycnTJs2jb6mq6sLGRkZBAYGMu4LAPp3oqSkhNatWzNurz7z5s2Dt7c32rdvT7dCNQaJxLBXr15ITU3lqxrevn2b6CyfECH1EXTF/8GDB9iyZQv27dsHWVlZ+vrGjRuRnZ2NnTt3QkVFhTH7W7duRXh4OJYtW4YOHTrgzz//hIGBAfEElQ2nhPV59eoVTE1N0bJlSygpKaGmpgbnzp3DiRMncPLkSZ7/L6ZgSyHr6NGjcHNzAwCe1loOh0NsB5q8vDzWrl2LI0eOQFpamu8Um5Qy6qRJkyAmJoagoCBcv34d4uLikJGRgZeXF7FZwn+KMBlr5hw4cAAvX75EYGAgnYiYmZnBwcEBHh4ejG5A5/LhwweUl5fzXMvPz4eoqCjPtZ49ezLmQ3p6+i8rQZMmTcLZs2cZ84ENiZCamhq980VeXh53797lkVAXBE+ePMH27dv5rispKSEjI4OIDykpKQgKCqKrZqTp1asX3V7Uq1cvgVa7gdpK4pYtW/Dp0ydQFIV79+4hNDQUwcHBfAs8hQghjaAq/s+fP8fChQuhoqKCli1b8jw3b948+Pn5wdzcHGFhYYzd+M+YMQMzZsxAZmYmwsPDceHCBZw5cwZdunRBTU0NcnJyiJxE1D1pqqioaHQGPSEhgdiplLu7O9TV1bF79276xr+8vBxr166Fh4cHDh48yLgPbClkhYSEwNraGsuXL4eOjg7OnTuHL1++YM2aNRg/fjwRH7KysuiREEF3VBgYGPAIeDQ7KCHNmgkTJlBxcXEURVGUiooK9fbtW4qiKCo+Pp7S1tZm3L6cnBwlLy/P86f+Ne5jJhk2bBiVkZHx09dkZ2dTQ4cOZdQPLnJyctSnT5+I2Po7FBQUELc5ceJEysfHh++6q6srNWXKFCI+/PXXX9SzZ8+I2PoVubm5VHV1Nd/1yspK6vHjx8T8OHXqFDVmzBhKTk6OkpOTozQ1NamAgABi9oUIaYiAgAD6PcmNH9yfTU1NGbW9bNkyavXq1Y0+X1NTQy1atIhauXIlo37Upbq6mrp58ya1bNkySkFBgZKXl6dmzZpFRUVFEfPBwsKCKi8v57lWWlpKOTk5MR7b66KiokK9fPmS7/qLFy+IxfYzZ85Q6urqVGBgIKWsrEyFhoZSu3btopSUlKgTJ04Q8YGiKEpBQYF68+YNRVEUZW5uTl27do2iKIq6c+cOZWhoSMwPtlBQUEAlJiZSDx48oB48eEDdv3+funPnDnXgwAFBu/ZbCE/Gmjn5+fkN7mORlJQkMlxLetamMQYOHIj79+9DRkam0dfExcUR213DPZ2qS2FhIdH9a9++fYO7uztMTU0xYMAAWFpa4v79+5CWlsahQ4eIVfFWrlyJlStXIj4+HkOGDEFNTQ1SUlLw4sULHD58mIgPa9euxaZNm7Bq1SpISUnxKX0yeWpbn/HjxyMuLo7vvfDu3TuYmZnh8ePHjPvw/v17TJ8+HTNnzkRhYSEoikKXLl1QVVWFJ0+eEFcoEyKEiyAr/o8ePfrp6QqHw4GVlRW9+JcEIiIi0NHRgY6ODgoLC3HhwgWcPXsWa9asIXYSkJubi+XLl8PHxwfi4uKIj4/Hxo0bUVpaSmQ2iUubNm1QWVnJd72ha0wxdepUVFVVwdfXFz9+/MCmTZvQuXNnrFq1CrNnzybmR+vWrVFdXQ0AkJKSQkZGBt3+n5ubS8yP0tJSXLhwgZ7hk5WVhYGBAdERmQsXLmDjxo2oqKigWza53Se9evUiNpv+jxB0Nijkn2FiYkKFhYVRFMV7Mubt7U2ZmJgQ9aV+5awu9+7dY9R2eHg4paGhQb148aLB558/f06NGDGCOnbsGKN+cPn69Su1ceNG6uXLl1RVVRU1f/58Sl5entLT06P/j5jG3t6e0tXVpTIyMqhLly5RioqK1IULF6jFixdTy5cvJ+IDl6SkJMrW1paaOHEiNXHiRGrdunWN/l8xweDBg3kq7CRPbSmKoo4fP06NGzeOGjduHCUnJ0eNHTuWfsz9M2zYMEpPT49xXyiKouTl5Rs8Jc3KyqKUlJSI+CBESEMIsuJfN4Y2Rk5ODqWsrMyoH1x+FlNJnmJ//PiRMjQ0pBYuXEg5OjpScnJylK2tLfX582diPlAURa1Zs4YyMzOjioqK6GsFBQWUmZkZ8ZjGtU36d8Bl0aJFlL29PVVWVkaFhoZS06dPp6qrq6nIyEhKS0uLiA+5ubmUjo4OpaSkRE2ePJkyMjKilJSUKB0dHSovL4+IDxRFUfr6+pS9vT2VkZFBDR8+nEpNTaVu3LhBjR49moqIiCDmxz9BeDLWzFmxYgVsbW2RkZGB6upqnDt3DllZWfTuC5IsWbIEvr6+PL3lZWVlcHV1xenTpxmVMDcxMUFMTAymTZuGsWPHQk1NDe3bt8eXL1+QnJyM27dvQ1tbG2ZmZoz5UJcdO3YgMTERCxYswLVr15CUlAR3d3dER0fD3d0dXl5ejPsQGxsLHx8fyMjI4PDhw9DS0oKRkRHk5OQwd+5cxu3XRU1NjdhC44Y4evSowGwDtYImRUVFoCgKPj4+0NPTQ5s2bXhe06ZNG/z555+M+RASEoKAgAAAtTM4U6dO5ZPJ/vr1K9FTQiFC6iPIin/v3r3x6tWrn3YNvHz5ktjalsZiqpubG8LCwmBubk7Ej27duiEoKAjm5ua4c+cOvL29oaurS8R2Xbg7GnV0dCAtLQ0AyM7ORseOHbFjxw5ifuTk5ODZs2f48eMH33PGxsZEfFi9ejUsLCwQEhKC2bNnw8/PDyNGjMD379+JiGIBtWsYevTogbCwMHo2/fPnz1i1ahV27dqF3bt3E/EjJycHXl5ekJGRgZycHAoLCzFu3DhUVVXBz88PkydPJuLHP0GYjDVzdHR04OnpiYMHD0JUVBT+/v6QlZXF3r178ddffxH1RdCtDPv376f3eV2/fh1AbVuJoqIiNm/ejGnTphETTmBDIlRWVgZJSUkAtS2a1tbWAICWLVvSNzukiI2Nhb+/P16/fo3Q0FCEh4dDSkqK2JfkiBEj6J9LSkogLi5OdAF0q1ataJU4DocDS0tLtGrViph9gB0JoRAhv0JNTQ2HDh3Cpk2bMHjwYJw5cwYLFy5EUlIS3/u1qdHT04Onpyc0NDQabLMqLS2Fp6cnxo0bx6gfXAQZUyMiIviuGRsbY+/evTh79ixKSkp4rpOgR48eiIqKwvnz5+kdjTNmzICRkRGxtrjw8HBs3LiRFpepC4fDIfa76NWrF65fv46ysjK0adMGYWFhuHjxInr06AE9PT0iPsTHxyMgIIBHJKxr166ws7Oj7zdIICEhQRcs+vbti/T0dIwePRqKiop48+YNMT/+CcJk7F/A6NGjMXr0aEG7geDgYJibm2P58uXo1q0bzpw5AwMDAzg6OhLbezZ37lzMnTsXFRUVKC4uRseOHSEuLk4/T9XpJWYSNiRCMjIyiImJgaSkJD59+kS/R8LCwn46W9fUxMXFYfny5TA0NMSjR49QU1ODqqoqODg4gKIoYsErMDAQR48eRX5+PjgcDnr37o2lS5cSs89l+fLlKC0txcmTJ4n22bMhIRQi5FcIsuJvYWGBixcvwtjYGPPnz4eqqio6dOiAoqIiJCcnIygoCGJiYli4cCGjfnARZEz9marqrVu3cOvWLQBkExCgtmA0Z84cYvbqc+DAAcycORO2trZo3769wPwwNjbGvn37oKCgAKA2Caq/9JhpREVFG4whpHfNKioq4vTp01i9ejUGDhyI2NhYWFpaIiMjg+f+j81wKKreRj8hrMfb25u+kfL29v7pa0nvbCksLIS5uTnS09Ph6ekpkFaG8ePH4+zZs+jYsSPP9fz8fEyaNAn3799n3IepU6di+vTpkJSUxKJFi3D58mVIS0vDw8MD9+/fx+nTpxn3ITY2FitWrEBlZSUMDQ3h4eGBnTt3IiQkBD4+PhgzZgzjPgDArFmzoKenhwULFkBVVRUXLlxAnz59cPjwYVy4cAGRkZGM++Dl5YWAgADMmzcPysrKqKmpQWJiIk6dOoW1a9fC1NSUcR+4vH//HqampigoKEC/fv1QU1ODN2/eoEuXLjhx4gR69OhBxI/v378jLS0NlZWVfItdhw8fTsQHIUIa4sePHygrK0Pnzp3x+fNnREZGQlJSkkjFv6ioCM7Ozrh+/TrP6YeoqCj09PSwfv16dOvWjXE/uLAhpgqSX+1lrAsJQbEhQ4YgOjpa4PsYtbW1ERgYSLSwWp8lS5agZcuWcHd3p5Me7u7br1+/wt/fn4gfiYmJsLKywsqVK2FiYgI9PT107doVeXl50NfXb3C1DtsQJmPNkHHjxuHs2bPo1KnTT9slOBwO44sbG2plKCoqwt69e6GlpcXTKslk9Sw6Ohp37twBAJw7dw4GBgZ8bWi5ublIS0tDQkICY35wYUsiVFRUhPz8fMjLywOo3fnVpk0bol/gqqqqOH/+PKSkpHiSsZycHEycOJGIeuCoUaOwfv16TJw4kef66dOn4evri5s3bzLuA5eVK1fi8+fP8PT05Ouz7969O5E++xs3bsDe3h4lJSV8iRiHw2F0vlOIkOZAQUEBXrx4geLiYnTu3BmKiopo164d43bZElMb8qtFixbQ19cHANjY2EBXVxdGRkaM2nVwcPjt15JYMjxjxgwsW7aMWAxvjEOHDiEiIgJz586FlJQU3148EgW1zMxMzJo1C23atIGioiIAIDU1FaWlpTh+/Dh930GC/Px8VFRUoE+fPsjIyMCpU6cgKSmJefPmNYvTMWEyJuQf8bsfNqZv8PLy8rB+/XpQFIWHDx9CRUWF5wPI4XDQunVrzJ49+z+VCAG1A85paWkQERGBgoIC3T5JitGjR2PPnj0YNmwYTzJ269YtbN68Gbdv32bcBxUVFZw7dw79+vXjuf769WuYmJgQSQi5DBs2DAEBAXzy8U+ePIG1tTWRk1t9fX3Iyspi6dKlDd5gklriKkQIUFtg/N3TD6YLjFz279+PKVOmED8BYUtMrUtQUBA8PDzg5OSE6dOnA6hNfEJDQ7FhwwbMmDGDMdu2trZwcnJC586dG4ztJHj48CH9c0pKCoKCgrBixQr07dsXoqKiPK8l1VXws/cJyfdGbm4uTpw4Qc/wycnJYfbs2URjiIODAxwdHfna/L98+YINGzbgwIEDxHz5XxHOjDVz5s6diylTpkBfXx+tW7cmbr+hfVqCQFJSkm5RMDMzg7e3Nzp06CBQnzp16oTi4mJcvXpVIIlQSUkJVq9ejTt37tCnHxwOBwYGBti5cyePQheTGBkZYceOHdixYwc4HA5KS0tx+/ZtuLi4ENuTM378eJw8eRIbNmzguX7u3Dni85Zs6LN/9+4dDh48SGzvnhAhP8PExISYuNLvEhkZCT8/P6ipqWHKlCkNCt4wAVtial2Cg4Ph6urK833t4OAARUVF+Pj4MJqM3bhxAytXrkTnzp0xb968Bnc0Mg23VbLu2cXmzZv5XkcyCSJVlKjP4sWLoa2tDU1NTcjIyKBXr15Yt24dcT+SkpKQk5MDoPbUVkFBgS8Zy8zMxL1794j79r8gPBlr5jg4OODatWuorq7GhAkTMGXKFGhoaAjMH0G1MrANNiRCDg4OSExMxKZNm6CqqoqamhokJyfDxcUFEyZM+OmAdlNSWVkJe3t7REVFAQAd1MaOHYv9+/czpmpYt72ltLQU165dw6BBgzB06FCIiori2bNnSE5OxvTp0+Hs7MyIDw3Bhj57IyMjODk58ahMChEihJfk5GRERkbi8uXL+PHjByZMmAATExNoamoS84ENMVVZWRmRkZF8xZucnBwYGhriyZMnjNmeOnUq8vPz0a9fPzx8+BCqqqqNnowxNTP2d1Yq/Nu7CszNzfHo0SP8+PED3bt3x8iRI6GtrY2RI0fyzekzSXJyMi3kUj9R5tK6dWtYWFgQ1074XxAmY/8CKioqcOPGDURGRuL27dv4448/YGxsDBMTE6ItFoJsZRg0aBDu3r2LLl26QF5e/qdVVhKVKzYkQsOHD8eBAwf42ibi4+Oxdu1axMfHM+5DXd68eYMXL16gpqYGAwcORM+ePbFv3z6+06qm4nd3ynE4HCKD31zY0GcfGxsLDw8P2Nraon///nzFAeGuMSGC5OnTp/D396fVRgcMGID58+fztfaSoqqqCnfv3kVUVBRu3LiBjh07EpkzFWRMrcvkyZNhYmLCp9Z3/PhxnDhxAtHR0YzZfvfuHY4fP47i4mKcO3cO+vr6fPNRXEjMjLGlJa6x1l4OhwNxcXH06NEDkydPZmSusLq6Gs+ePUNiYiISExORlJSEb9++QV5eHlpaWtDS0sLQoUOJtZPKy8vj7t27PBL7zQ1hMvYvg/uF5eXlhbKyMqKD+BMmTICtrS1f61lkZCR8fHxw+fJlxmyfO3cOhoaGkJCQQHh4+E+TMRMTE8b84MKGREhDQwPHjh3ju7lPT0/H3Llz8eDBA8Zsl5eXw83NDVFRURAXF8fkyZOxZs0aesnw3bt3sXnzZuTl5eH58+eM+cFWBN1nr6CgQK9YqPtZ4a5+EAp4CBEUDx48gIWFBQYOHIhhw4bRhay0tDQEBgZi6NChxH36+PEjoqKicOXKFaSmpmLEiBFEFskLMqbW5cKFC7C3t4ehoSGUlZUB1BaQoqKi4OLiQiSmArXzxxcvXiQuKV+3Ja6xZCwzMxPHjx9HSkoKEZ98fHzg4+OD8ePHY9iwYQBq59muXr2KKVOmQEREBJGRkdiwYQOdyDNJWloanZzFxMSAoihiv4uGKCwsJN7O+k8Qzoz9SygvL8eNGzdw4cIFxMXFQVJSElZWVkR9+PjxI13pr4uKigrev3/PqO26wWDKlCmM2vodREVFGxRG6NatG6qqqoj4MG/ePLi4uGD//v10xaikpAT79u3DvHnzGLXt7u6OsLAwTJo0CRISEjh58iTatm2LRYsWYdu2bTh58iSkpKQQGBjIqB91+fHjBy5fvozXr1/DwsICaWlpkJWVRadOnYj5wEVQffZcSNxIChHyv7B3715MnToVW7Zs4bm+ZcsW7Nu3D8HBwUT8KCkpwZUrVxAZGYmHDx+iZ8+eMDExwd69e4nN/goyptZl0qRJEBMTQ1BQEK5fvw5xcXHIyMjAy8sLOjo6xPyoqalBdnY28RNSDodDd7NwOJwGF263bt2a8T14dUlOToaNjQ0WLVpEX5s/fz78/f2RkJCAw4cPQ01NDf7+/owmY5WVlUhMTERCQgIePnyIFy9eQFxcnGgr77dv3+Du7g5TU1MMGDAAlpaWuH//PqSlpXHo0CGBryH4HYTJWDPnzp07uHjxIq5fvw6KoqCnp4ejR4/SlRKSSEtL4+bNm3ytDLGxsejduzejtn+1b60uJPqHBZkIcbl79y5SU1Mxfvx4SEtLQ0xMDNnZ2SgtLcWLFy9w7tw5+rVNPQx88+ZNODo6Yvbs2QCAsWPHYvv27cjLy8OZM2dgYWEBGxsbYiIinz9/xsyZM1FQUICKigpMnz4dAQEBePr0KfFdLYWFhTh8+DDS09MbFOwg0TIpnBUTwlaeP3/e4M2uqakppk2bRsyPkSNHQlxcHH/++ScCAwP/czG1PgYGBsQElxpDQkICYmLkb1vV1NRoYRW2tMQlJSVh06ZNfNcnTJgAT09PALXf8w0JjfxTsrOzcefOHdy9excPHjxARUUFhgwZAm1tbdjZ2UFJSYnugiHBjh07kJiYiAULFuDatWtISkqCu7s7oqOj4e7uDi8vL2K+/K8Ik7FmzsKFCzF8+HA4OTnhr7/+alCljRSWlpawt7fHs2fPGmxlYJLw8HCex3l5eRAXF0efPn0gJiaGt2/forKyEoqKikSSMUEmQlxGjhyJkSNHMvJ3/4rPnz9DW1ubfjxq1Cjk5ubi2rVrOHr0KNTV1Yn64+rqCllZWVy8eJH+nbi5uWHVqlXYtWsX/Pz8iPliZ2eH1NRUjBw5stHZByZoaH9RY5DcXyRESF06deqEoqIivuuFhYXEijdA7Umcnp7efzam1qewsBBZWVn0ImyKolBRUYHU1FQsWbKEiA8mJiawsrLC5MmT0bdvX77vTxLfW2xRu+zSpQuSk5PRt29fnutJSUl0t8enT5+afDeerq4ucnNz0atXL4wcORJubm7Q1NQksoOvMWJjY+Hj4wMZGRkcPnwYWlpaMDIygpycHObOnSswv/4OwmSsmXPt2jXiFbLGEGQrQ91h6mPHjiEmJga7d+9Gly5dAABfv36FnZ0dBg4cyKgfXASZCHERpIJQZWUlz6oFUVFRtGjRAo6OjsQTMQBISEjAoUOHeG6sOnTogPXr1xM7qeSSlJSEgwcPEj+d+l3RGA6HI0zGhAgMHR0duLi4YM+ePfSJdUZGBrZt24Zx48Yxapur1icmJobevXvj6dOnjb6WxD4ptrQHXrhwARs3bkRFRQWtXMedNe3VqxexZMzHxwdAw23WpL63ysvLERoairS0NHruFqgVUnv69CmuXLnCuA9ArUDV1q1bkZ2dDRUVFdTU1ODx48cIDg7GkiVLkJeXB2dnZ4waNapJ7b579w6SkpIwNDSEtrY2/XkRJGVlZXTrcFxcHKytrQEALVu25Pk/YjPCZKyZ07t3b7x8+RKBgYHIysrC/v37cf36dQwYMEAgN71saGU4dOgQAgIC6EQMANq3b4/Vq1fDzMwMq1evZtwHNkip/uokRBA33IJSQystLW10Dx+pGT4u3bt3J7KvqD5sqegKEfIzVq1aBXNzc0ycOJGutnOV2uzs7Bi1bWZmhri4OHTp0qXB3VJcSIrcsCGm+vn5wdDQEFZWVpg9ezYCAgLw8eNHbNmyBStWrCDmBxu+w7Zt24aIiAgMHjwYqampUFVVxZs3b1BQUMDXTsokCxYsgLi4OPz9/XHw4EEAtSq469evx6xZs3Dnzh3079+fZ8VLUxAdHU23KAYGBkJERATq6urQ1taGtrY230kdCWRkZBATEwNJSUl8+vSJ3h0aFhZGdAThnyBMxpo5T58+xezZs6GiooKnT5+ioqICL168wM6dO+Hj44MxY8YQ9YcNrQyVlZUoKyvju15QUEBssSgbEqHGTkJatGiBHj16MO5DY7K7gmD48OE4efIkT2CqrKyEr68v1NTUiPqybt06bNmyBba2tujTpw9fb71QVl7If5kOHTrgzJkzuHPnDo/aqLa2NuNzKDdu3KAV2AS1VLc+bIipOTk58PLygoyMDOTk5FBYWIhx48ahqqoKfn5+mDx5MhE/uLx//x6ZmZkYPnw4SktLeQqvTHPjxg3s3LkTEydOxIQJE+Di4oI+ffrA1tYWlZWVxPwAgLlz52Lu3Ln48uULxMTEeBQeR40a1eSnYgDQv39/9O/fH/Pnz0dFRQUePHiAu3fv4uTJk3BxcUHv3r2hpaWFUaNGQVdXt8ntN8TKlSuxYsUKVFZWYuLEiZCWlsbOnTsREhJCn6ayHaG0fTNn/vz5UFFRga2tLVRVVXHhwgX06dMHO3fuRFJSEs6cOUPMl1+1Mly/fp2IH+vXr8fTp0+xadMmKCoqgqIoJCUlwcXFBWPGjIGTkxPjPjS2K4qbCJFqZahLdXU1srOz4ezsjJkzZ2LixImM2ZKXl4eBgQHPQufIyEiMGzeO71SIxG6YzMxMzJ07F5KSkkhPT4e6ujpev36Nb9++EdvtxeX69etYv349X8GAaVn5iooKuLu7IzIyEuLi4jAwMMDq1auJzq0JEdIY379/x7179yAhIQE1NbVGT7JJ8eTJk0ZP8k+cOEEvnGUStsTUoUOHIiIiAn369MHGjRvRr18/WFpa4v379zAyMkJSUhIRPyoqKrB+/XpcunQJIiIiuHLlCtzc3FBaWgovLy8+uXkmUFRUxNWrV9GzZ08sWbIEhoaGmDhxIlJTU7Fq1SqiSXxeXh5CQkLofXyysrKYMWOGwBZP5+TkIDAwEOHh4fj+/TvRFSlFRUXIz8+nY/mTJ0/Qpk0b4cmYEDI8e/YMzs7OfNfnzp2LsLAwor6wpZXByckJNjY2mD9/Ph24uEqT69evJ+JD/XaK+omQIBAVFYWMjAwcHBxgY2PDaDI2fPhwfPr0ieeaqqoqioqKGhzOZxoZGRmcP38eJ0+exB9//IGamhro6+tjzpw5xGcud+zYAQ0NDcyYMYOoOMDevXtx5swZTJo0CSIiIjhz5gzKysoaVK4TIoQkaWlpsLCwwOfPnwHUtvIeOHAACgoKAvNpzpw5sLGxoedPgFphIgcHB8THxxNJxtgSUxUVFXH69GmsXr0aAwcORGxsLCwtLZGRkUFssS8A+Pr60mMZixcvBlDbWurg4AAPD48G74Wams6dO6OgoAA9e/aEtLQ00tLSANQKz3DfvyR49eoVTE1N0bJlSygpKaGmpgbh4eEICQnByZMnISsry7gPnz9/RkpKCpKTk5GcnIznz5+jZcuW0NDQICptD9T+/ouLi3H16lWIiIhAQUGB2AqKJoES0qzR0NCgnjx5QlEURamoqFBv376lKIqiEhISKE1NTaK+KCoqUhkZGRRFUZSpqSkVGxtLURRFXblyhTIxMSHqC0VRVGZmJnXp0iXqypUr9O9F0Dx79ozS1dUVqA8vXryglJSUBOoDCRYtWkQFBwfT70m2oKysLJD3o46ODhUVFUU/vnXrFqWiokLV1NQQ90WIkLpYW1tTM2fOpJKTk6nHjx9Tpqam1PTp0wXq06lTpygVFRXK3Nyc+vjxI3X58mVqxIgRlKGhIfX48WMiPrAlpj58+JBSVlam/P39qcLCQmrEiBGUgYEBpaqqSm3YsIGYHxMmTKDi4uIoiuK934mPj6e0tbWJ+ODo6EhNnTqVSktLo65du0bp6OhQT548oVxdXak///yTiA8URVEWFhbUsmXLqB8/ftDXfvz4QS1fvpxauHAhY3ZPnjxJ2dnZUbq6upS8vDw1ZMgQat68eZSvry/16NEjqrq6mjHbjfHt2zfK2tqakpeXp+Tk5Cg5OTlKXl6eWr16NVVeXk7cn/8F4clYM0dXVxf79u3D3r176WuZmZnYvn07xo4dS9QXCQkJWnq4b9++SE9Px+jRo6GoqIg3b94Q9QX4v95mNiEiIoKPHz8SsdXQ3FpJSQnCwsIEJqRBkvLycuzevRs/fvxA9+7dMXLkSGhra2PkyJHo2LGjwPxSV1dHSkoK8UWUHz9+5JmP09LSwo8fP/Dp0yf88ccfRH0RIqQujx49wtGjR+mTMBcXFxgYGOD79+8Ck5afOXMm1NXVsX79evz555+orKzEokWLsHjxYmKnQWyJqcOGDcOVK1dQUVGBTp06ISQkBKdOnYKkpCRRNdr8/HxISUnxXZeUlERxcTERH+zs7GBvb48HDx5gzpw5CA0NxfTp0yEmJgY3NzciPgC1S59PnTrFMwrQokULLFu2DKampozZ3bp1KwYPHgw9PT1oampi6NChPD4Igu3btyMrKwuHDh2CqqoqampqkJycTKuy/q6SsCARJmPNnPXr18PKygoaGhqoqanBlClTUFJSQkR5qj6CbGWQl5f/bXEIEn3MbEiEGvoCEhMTg6qqKpF2DkFz9OhRVFdX49mzZ0hMTERiYiK2bNlCK7NpaWlBS0sLQ4cOJdpqM2zYMGzevBkxMTGQkpLikwVmSomzqqqK598pLi6Oli1bory8nBF7QoT8LiUlJTwFAWlpaYiIiKCoqEige76KiopQVlaGFi1aoKKiAu/fv0d5eTmx7wu2tAc6ODjA0dER3bt3BwAMGDAAGzduxJcvX2BjY4MDBw4Q8UNGRgb37t3D9OnTea5HRUVhwIABRHxo3749z7/30KFDePHiBbp27Uq0qNWmTZsGBUOYFhFJSEhA+/btAdTO8DW2+y8hIQEaGhqM+sLl+vXrOHDgAM+6ibFjx0JCQgJr164VJmNCmKdt27Y4deoU7t27h+fPn6OmpgYDBw7EqFGjiG5AB4AVK1bAysoKHTt2hImJCXx8fGBoaIi8vDzo6+szanvHjh10Mpabm4vDhw9j5syZUFVVhbi4OFJTUxESEkJMfYoNiRAbZIAFjaioKJSUlKCkpAQLCwsAtfMp3ORs6dKloCgKKSkpxHw6efIkOnXqhEePHuHRo0c8z3E4HFasRRAihCQ1NTV88UpcXFygO4K2bduGkydPYty4cQgMDMTbt2+xfv166OvrY8uWLYzvPAMEG1OTkpKQk5MDoLa4qKCgwCeQkZmZiXv37jHqR11WrFgBW1tbZGRkoLq6GufOnUNWVhauXLnC0x3U1Lx///6nz3fs2BFVVVV4//49MTVcDQ0NuLu7w9PTk+70KCwsxK5duxid1+ImYgCwZMkS+Pr68iRkZWVlcHV1xenTp4kJeIiKija4dLpbt27EV9f8rwjVFP/FBAYGYv78+URt5ufno6KiAn369EFGRgZPKwOpSp6pqSmMjY0xbdo0nusXLlxAYGAgzp49S8QPNvD9+3d8+/YN7dq1E2iFmQ1UVlYiMTERCQkJePjwIV68eAFxcXFoampi//79jNr+8uXLL1sjKyoqEBMTgz///JMRHwYNGoS4uDhauhsA1NTUcP78eeItk0KE1EVeXp7e78WlrjqwIBg6dCg2bNiAqVOn0te+f/+OnTt3Er3RFFRMTU5OpkVKGtu31rp1a1hYWBAtIN2+fRsHDx6ki8+ysrKwtrbGX3/9xZjNQYMG8Tym6qha1r9G6n2Rl5eH2bNno7i4GNLS0gCA7OxsdOzYEcHBwUSEqfT09CAlJQUfHx+Ii4sjPj4eGzduRGlpKezs7Hg+O0xy4MABxMXFYf/+/ejatSuA2tP29evXY9CgQc2iwClMxpop/v7+iIqKgri4OCZPnsyj7JSeno6NGzfiyZMnRKVFua0M9atnX758wYYNG4i1MigpKSEyMpJv+WB2djYmT56Mx48fE/FDUIlQSUkJ/f7gVjaB2pmDSZMmwdzc/D+TmGVnZ9MLKh88eICKigoMGTKEXlCppKRE5AR50KBBuHv3Ls/N5vr162FnZ0df+/z5M0aNGsXYZ7ahVt6GbioAMq28QoRwkZeXx8aNG3lix+bNm2FjY8NTPADILat/+PAhVFRUGkx4YmNjiezwZEtMlZeXx927d+kbXUGRk5MjkOR82LBhKC0txbBhw2BoaPjTWfQRI0YQ86u0tBTnz5/n2cdnZGREROIfAD59+gRzc3P06tUL3bp1w5kzZ2BgYABHR0eiu9/mzJmD1NRUiIiIQFpaGmJiYsjOzkZpaSl69uzJE+PYsj+wPsI2xWbI/v374evrC3V1dbRo0QI7duyAiIgIZs2aBX9/f+zbtw+tW7cmsr+Jja0Mffv2RVRUFJYuXcpzPTQ0lPG+ckEnQkVFRTA1NUVeXh4mTJiAmTNnon379vj27RuePXuGQ4cO4dKlSzhx4kSDx/r/JnR1dZGbm4tevXph5MiRcHNzg6ampkD+3Q3VvK5evYrly5fzBC0ma2Mkvg+ECPlfaWjFgru7O89jDodDLBlbuXIljhw50qC8PpOJGBtjakMt74WFhXyJMtNMmDABQ4cOxZQpU6Cvr09sF118fDzu3LmD6OhouLu7Q0pKCgYGBjA0NBTYTq9p06Zh27ZtRFYsNEa3bt0QFBQEc3Nz3LlzB97e3sQWPddl5MiRGDlyJHG7TYkwGWuGREVFYeXKlXSyERERgcOHD+PTp0/w8fGBnp4eNm3aROSLksPh0PNRHA6nwYDaunVrWFpaMu4Ll5UrV2LlypWIj4/HkCFDUFNTg5SUFLx48QKHDx9mzC4bEqH9+/ejpqYGUVFRDe7Y+PDhA6ytrREQEAAbGxtGfGAL7969g6SkJAwNDaGtrQ1VVVU+sQy28bsiNP8LJiYmjP3dQoT8E9g439q5c2d8+/aNuF02xtRv377B3d0dpqamGDBgACwtLXH//n1IS0vj0KFDxE6rgoODERkZCXd3d2zbtg0TJkyAiYkJ4zutJCQkMH78eIwfPx7l5eW4desWoqOj4evrCzk5ORgYGEBfXx/dunVj1I+65OTkCGQxekPiZMbGxti7dy/Onj2LkpISnuskaA5tiL9C2KbYDFFWVkZERAT69esHoHbWRFlZGW3atMHGjRuJfQDqw5ZWBqC23/348eNIT08HUNsmZmFhQW9nZwJnZ2fcv38fAQEBP02EdHV1GUuEdHR0sGnTJujo6DT6muvXr2PXrl24cuUKIz6whdevX9Mtig8fPoSIiAjU1dXpFsX6baxM8jszMUy3KXJ59+4dwsLCkJSURFe3hw4diunTpwtnx4QI+f+4ubnhxIkTGDNmDPr27csn303iBpAtMdXBwQGJiYnw8/NDeno61q1bhx07diA6OhpiYmLw8vIi6k9lZSViYmIQGRmJ2NhYdOnSBcbGxli5ciVRP0pLS3Hr1i1cunQJ9+7dw5AhQxAYGEjE9uHDh3H79m1YWlpCSkoKLVu25HmeKSGR372HIjk/11CCWBdB3RP/HYTJWDOksRu7devWCfTIuiEE0cogKNiQCA0ZMgSXL1/+aevEu3fvoK+vj9TUVEZ8YCMVFRV48OAB7t69i7t37yIjIwO9e/eGlpYWRo0axXhrBVuSsaioKGzcuBEcDgeqqqro1KkTvn79isePH6OiogIuLi6YOHEiY/aFCPkVnz59wr59+5CcnIzKykq+1l1SMx8/U0vkcDgCmz0RREwdOXIkfHx8oKqqCnt7e3z58gV+fn5IS0vD3Llz8fDhQ6L+cCksLERERAR8fHzw48cPPHv2jKj94uJiXL9+HZcvX8a9e/fwxx9/4ObNm0RsKygo0GqjdTsqSAuJsIHGEsQWLVqgR48ezaLwzO6eHSF/C0H3zLKllQGoHbD29/fH69evERoaivDwcEhJSWHy5MmM2fz8+TMGDhz409fIy8v/Uib3n1BZWclXIatPy5Ytm43ca1MhISFBn4gBtS0egYGBCA8PR1hYGJHAxWQL4u/w/Plz2Nvbw8zMDDY2NjyV/oqKCvj5+WHDhg0YMGAAoyfIQoT8DCcnJzx9+hSGhoYCnWsldVP9M9gSU8vKyuhuj7i4OFhbWwOojSWk1w+UlZXh2rVriIyMREJCAnr16gVLS0tibdhfvnzBtWvXcPnyZdy/fx9dunTBX3/9haVLl0JVVZWID0DtHk02EBERgRYtWtCrFmxsbKCrqwsjIyNiPtRvc66urkZ2djacnZ0xc+ZMYn78E4TJ2L8IUVFRgdrfsWMHEhMTsWDBAly7dg1JSUlwd3enh15JtTLExcVh+fLlMDQ0xKNHj1BTU4Oqqio4ODiAoijGjqyFiRB7+fz5M1JSUpCcnIzk5GQ8f/4cLVu2hIaGBuPzBly2bdvGkwBVVlZi165daNOmDQAwvnzZ398ff/31V4PL4CUkJLBy5Up8/PgRR44cgYeHB6O+CBHSGAkJCThy5AiGDRsmaFdAURTu3LmDtLQ0iImJQVZWFhoaGsRiLVtiqoyMDGJiYiApKYlPnz5h9OjRAICwsDDIyMgQ8QEAbG1tERMTAw6HAz09PRw7dozI+6SoqIg+Abt//z46deqEv/76C0uWLBHI+7SkpATKysp8rbOkCQoKgoeHB5ycnOhrPXr0gJOTE75//44ZM2YIxC9RUVHIyMjAwcEBNjY2zaLbQ5iMNVMCAgJ4VPmqqqoQFBSEDh068LyO5GBjbGwsfHx8ICMjg8OHD0NLSwtGRkaQk5PD3Llzifnh5eWFNWvWYMGCBfTxtK2tLdq2bQt/f/9m0T/8T6j/3qhPWVkZQW8Ex6lTp+gE7N27dxAXF4eqqip0dHSwYcMGDBkyhNhi9OHDh+PTp08811RVVVFUVISioiL6GpOBPTEx8Zf71GbOnIlly5Yx5oMQIb+idevWRGWxG+PLly+wtLTEs2fP0K5dO1AUhZKSEigoKODo0aM8y2+Zgi0xdeXKlVixYgUqKysxceJESEtLY+fOnQgJCYGPjw8xPz5//ozNmzfjr7/+IrqeZdSoUeBwOBg5ciS2bduGYcOG0bGjfqcLk0ufv379Cjs7O9y+fRscDgdjx46Fi4uLwEZBgoOD4erqCgMDA/qag4MDFBUV4ePjI7BkjIuIiAg+fvwoUB9+F2Ey1gzp2bMnLl26xHOtW7dufD3sHA6HaDLGllaGV69e8UkiA7ULCr29vRm1LehEqKH3RkM0JDDyb2Pr1q0YPHgw9PT0oKmpiaFDhwqskhgcHCwQu3UpLCxE9+7df/qabt264evXr4Q8EiKEn8mTJ+PIkSPYunWrQLs93Nzc8OPHD0RERNBtuy9fvsS6deuwe/dubNmyhXEf2BJTx4wZg9jYWOTn59O/C0NDQ8yYMYPoyVhwcDBycnKQlpaGzp07o3fv3kTav7ndLLGxsbh9+3aDryExq+Xu7o4nT57AxsYGIiIiCA4OhrOzMzw9PRmz+TM+fvwIRUVFvusqKiqMjmPUpyEBj5KSEoSFhUFJSYmYH/8EYTLWDGFDL3tDsKWVoV27dvj48SOkpKR4rmdkZPCdHDYlbEiE2PreEAQJCQl09bqiogISEhKNvk5DQ4OkawKhW7duyMrK+un7LzMzEz169CDolRAhvHz58gUXL15ETEwM+vTpw/e5DQoKIuLHrVu34OnpyTM/yV1OvXr1aiLJGFtiKgB06tQJxcXFuHr1KkRERKCgoECsqEdRFI4cOYLjx4/znHR069YNpqamsLa2ZjQpI/We+xW3b9+Gm5sbRo0aBaC2u8Lc3BxVVVUCWdsiLS2NmzdvYsGCBTzXY2Nj0bt3b2J+cFdB1EVMTAyqqqpwdnYm5sc/QZiMCWky2NLKYGRkhB07dmDHjh3gcDgoLS3F7du34eLiwnOc3tQ0h0SouLiYFq34ncSxOVO3jWjJkiXw9fXlubErKyuDq6srTp8+/Z9QnhozZgx8fX2hqanZ4I1LTU0N/Pz8BLK0U4iQurBhxqOqqqpBSfmuXbvy7FJiErbE1JKSEqxevRp37tyh1S05HA4MDAywc+fORgtdTcXy5ctx+/ZtTJ48GZqamnRieP/+ffj6+uLx48eM/j5GjBhB//yrwh6TFBYW8oiEqaqqorq6GgUFBb/semACS0tL2Nvb49mzZ1BWVgYApKamIioqCi4uLsT8YOOewr+LUNq+mcMWGWAuRUVFPK0MT548QZs2bYhW8SorK2Fvb4+oqCgAtUGDoiiMHTsW+/fvF1irmiAToaSkJJw6dQpXr15FeXk5Bg8ejPDwcKI+CBI9PT1ISUnBx8cH4uLiiI+Px8aNG1FaWgo7OztMnTpV0C4yTn5+PoyNjaGsrIxVq1bxVPxfvnwJDw8PZGZmIjw8HJ06dRKgp0KECJ758+dj4MCBcHR05Lm+bds2PH36FKdOnSLiBxtiKnfP2KZNm6CqqoqamhokJyfDxcUFEyZMaPBkoqk4e/YsXF1dERQUhEGDBvE9n56eDjMzM9jZ2WHKlCmM+cHF0tJSYIW931mRQpro6GgEBQXh1atXEBcXh4yMDBYuXPjTFT9M8P37d3z79g3t2rUjOk/YVAiTsWbO4sWLfyoDLIjN5NnZ2UhLSyPeylCfN2/e4MWLF6ipqcHAgQPRs2dP7Nu3Dxs2bCDqh6ASoZKSEkRERCA0NBQZGRkAAG1tbVhbW/NU+v4LfPr0Cebm5ujVqxe6deuGM2fOwMDAAI6OjqwQCyDFixcvYGNjg5ycHLRq1QodOnRASUkJSkpKICsri7179xJvfxIipD55eXkICQnhUTGcOXMmo+II9UlJScG8efMgLy8PNTU1ALXf5S9fvsSRI0eItjYLOqYOHz4cBw4cwPDhw3mux8fHY+3atYiPj2fM9qxZs6Cvr4/58+c3+pqQkBBERUXhxIkTjPnBRZCFPTYmY4KkpKQE/v7+iIqKQk5ODn29b9++mDRpEszNzZtNYiZMxpo5KioqrJEBFmQrQ3l5Odzc3BAVFQVxcXFMnjwZa9asoRWP7t69i82bNyMvLw/Pnz9nzA8ugkyEnjx5gtDQUERHR+P79++QlZWFoaEhvLy8cP78eQwYMIBR+2ylsLAQ5ubmSE9Ph6en53+2Ha+6uhoxMTF49OgRiouL0alTJwwdOhTa2trE1CWFCGmMV69ewdTUFC1btoSSkhJqamrw9OlTfP/+HSdPnoSsrCwxX548eYKAgACkp6eDoijIycnB3NycmCiAoNsDuWhoaODYsWN8+wfT09Mxd+5cPHjwgDHbw4cPR2hoKPr379/oa96+fYupU6cSWT4tyMKevLw8zp49y9O5YGhoiMOHD/MVKkgVLgoLC5GVlYWamhoAtfN9FRUVSE1NxZIlSxizW1RUBFNTU+Tl5WHChAkYOHAg2rdvj2/fvuHZs2e4ceMG+vTpgxMnTgh0X+HvIkzGmjkjR45ESEgI+vXrJ2hXBNrK4OLigtDQUEyaNAkSEhK4cOECrK2tsWjRImzbtg0nT56ElJQUtm3bxlfda0oEnQhNmTIFL168gIyMDCZMmAADAwP65kVBQeE/lYw1pLBUVFSEvXv3QktLC3/99Rd9/d++7gCo/Xw6Ojqibdu2gnZFiJBGsbS0RKtWrbB79266pby8vBxr165FRUUFDh48KGAPySHImFqXAwcOIC4uDvv376fn6EpKSrB+/XoMGjSI0Q4cVVVVnD9/nk+Qqy5v377FtGnTGE0K6yKowp68vDzfvC9XxbH+YxJz0BcuXMDGjRtRUVFBj4NwfenVqxeuX7/OmG1nZ2fcv38fAQEBDZ4Uf/jwAdbW1tDV1YWNjQ1jfjQVwmSsmePm5oavX78KXAYYEGwrg46ODhYuXIjZs2cDAGJiYrB9+3ZoamrizJkzMDc3h42NDaOVRDYkQvLy8ujfvz+mTJkCLS0tnh77/1oyVr+K2xikApegGTRoEO7evfufassU0vxQVVXFqVOnICcnx3P95cuXMDU1RWJiImO26xYsHBwcfvranTt3MuYHF0HG1LrMmTMHqampEBERgbS0NMTExJCdnY3S0lL07NmTJxlo6jn16dOnw8TEBHPmzGn0NSdOnEBUVBRCQkKa1DYXthT2/k6ySWIUwcDAAMrKyrCyssLs2bMREBCAjx8/YsuWLVi9ejUmT57MmG0dHR1s2rTpp7Np169fx65du+h9s2xGqKbYzGGLDDBQu/W8oePgbt260Xs6mOLz58/Q1tamH48aNQq5ubm4du0ajh49CnV1dUbtA8Dz58/Rv39/GBsbQ0tLi2g7DZcrV64gPDwcwcHB2L17N6SkpGBgYMCoiiRb+TcoLDUlwrqbkOZAmzZtUFlZyXe9oWtNzbt37+h2q3fv3jFu71cIMqbWZeTIkRg5ciQxe3UxMTGBt7c3tLS00LdvX77nMzMz4e3tjfXr1zPmw89OIG/duoVbt24BqC3sMZmMsUXVkUtOTg68vLwgIyMDOTk5FBYWYty4caiqqoKfnx+jydjnz595lCUbQl5enui+s3+CMBn7F8AGGWAAmDdvHlxcXPhaGfbt24d58+YxaruyshKtW7emH4uKiqJFixZwdHQkkogB7EiE+vbtC1tbW9jY2ODOnTs4e/Ysjhw5Aj8/PwC1ykfm5ubNooe6qYmIiECLFi2gr68PALCxsYGuri6MjIwE7Bk5SCxIFSLkn6ChoQF3d3d4enqiY8eOAGrbwnbt2gVNTU1Gbdddzl7358rKSsTHx4OiKGhqahJT5BVkTK2LIITAuMyaNQsxMTGYOnUqpkyZAjU1NXTs2BElJSV4+PAhTp8+jTFjxjB648/Gwh4b1rVISEjQ9vv27Yv09HSMHj0aioqKePPmDaO2Kysr0bJly5++pmXLlkSLFv8EYZuikCZDkK0MjakM/arXnAlqamroROjWrVv0l8GSJUsEkgh9+fIFFy5cwLlz5/DixQu0atUKkyZNIrK0lC0EBQXBw8MDTk5OmD59OoDaNqPQ0FBs2LABM2bMELCHzNPQvEFj/BfaNoWwkw8fPmDWrFkoLi6GtLQ0gFo1wY4dOyI4OJjxZbInTpyg1W5nzJgBQ0NDmJqa0jfk3bt3R2BgYIOnNE2NIGNqXRpq06sL0zO33JOW48eP48uXL/T1bt26YcGCBbCwsCBaaGJDYY8N61rmz58PZWVlrF69GkFBQYiNjYW/vz9u374NOzs7Rk/oGrrnq8/nz58xatSoZhHPhMnYvwA2yAADgLe392+/tqkrbfLy8oiPj0fnzp3pa2yQfGVbIvT8+XOEh4fj4sWLxFoZ2MCECRNga2vLd0oZGRkJHx8fXL58WUCekUNeXh4bNmz4rWKAiYkJAY+ECGmY0tJSnD9/nkfF0MjIiHHxGX9/f3h7e8PIyAitWrXCxYsX0adPH5SVlcHZ2Rk1NTXYuXMnevXqBU9PT0Z9AQQbU+vS2PxtixYt0KNHD2IzORRFISsri1aBlZKSIq4Ay5bCHhvWtSQmJsLKygorV66EiYkJ9PT00LVrV+Tl5UFfXx/bt29nzLa8vDwt9tMYZWVlOHr0qDAZE8I8bJIBFiTy8vIwMDDgaR+JjIzEuHHj0KZNG57Xkhi8bghBJELV1dX4f+3de1zO9/8/8MfViTl2JTfRJ13kEDmVaR0Q0XfWZUk5d96lpS1LPj6RY1YxYlM5VkJpKusgKclhRiwUK8oHfccallO1opXK7w/f3j+XDmx6H64877fbbrf1fr/1eq659bqez/fr9XyVlZWha9eucj+b+/fvQywWv/E1f3sxcuRIHD58uMlb0pKSEkilUuTn5/MUGXfeppJICN8uXboEIyOjJh+ynz17hqioKFaTjo8//hje3t5M0SY/Px+zZs3Czp07MWHCBCY+b29vZGdnsxaH0NXX1+P27dvw9/fH7NmzOdsqUVBQgOHDhze5XllZicDAQGzYsIH1GIRU2BPCcS2lpaWora2Fjo4Obt26hbi4OPTu3RvOzs5QVVVlbVxLS8u3fvbkyZOsxdFWKBlTcEJqA8znUgYnJ6e3fvbVvQBs4jMRSk9PR3R0NPLz85nGDYMGDYKTkxNmzJgBLy8vjBw5Eu7u7qzFICTTpk3D9OnT4erqKnd9//79+P7775Gens5PYByibopEEejr68Pc3BwhISFyb8K4WHI0fPhwHD16FNra2sy1YcOGITU1lTnn6uHDh5gwYQKuXbvGWhyN+F4e+CaFhYXw9vZGVlYWJ+N99NFH2Lt3r1yX4JMnT2L16tWor6/H+fPnWY+Bz8KeULo6NmrpuJTy8nIsX74c27dvZz2G9oIaeCi4vLw8xMXFyX3Y79ChA7788ks4OjpyGktLHYcalzKw+cuBqwTrbbwpEQoKCmI1EQoMDMT+/fthamqKRYsWQSwWo7y8HDk5OVi1ahVSUlJQXFzM6hICoZHJZFi2bBmuXbuGkSNHAnhZZT1y5AgCAgJ4jo4bVHcjiuLevXuYPXs2du3axfoesVc11xRAVVVVrsIvEomYjots43NOfRtKSkp48OABZ+PZ2NjA1dUV+/btQ+/evREQEIC0tDRMnToVy5cv5yQGiUSCkydPNinsnT59mvW/q0Lo6pibm4uSkhIAL5NDAwODJslYcXExJ4lxayoqKpCUlISEhARkZGTwGsvboGRMwfHZBvh1r3ccen0pw/uA70QoKysL8fHx2LVrFywsLOTuubu74/jx4/Dy8oKHhwe6d+/OSgxCZGNjAxUVFURHR+P48eNQVVWFnp4ewsLCWj2npD3x8vLCgQMH3vpZQvggEokQHh6ODRs2YObMmQgLC8OHH37Id1i8EMqc2twbmaqqKiQkJGDEiBGcxbFixQqoqqrC1dUVKioqUFNTQ0REBMaNG8dZDHwW9oTQ1VEkEjFJoUgkQmBgYJNnOnXqBJlMxnVoAF4mi3FxcTh27BhqamowdOhQXuL4u2iZooJbsmQJHjx40KQN8KJFi9C9e3eEhYXxGyC4X8rAl6ysLCxevBhbt25tkggBkEuEfHx8WInB1dUVH374YYsfpjdt2oS4uDjo6+tj//79rMRAhElfXx9KSkrQ0tJq9TmRSMRqZzZCWvPq3saNGzciJiYGX3/9NSZMmAAzMzNWlyk21xRg165dmDNnDlO8EkJTAK7n1OYaeKioqMDQ0BD+/v7Q09PjJI5GW7ZsQXh4OA4cOMAkRFxqXP3y3//+lynsff7555wX9vju6qivr4+zZ88yxy7wpaqqCikpKYiPj8etW7cAAGPHjoW7uzsnh1+3BXozpuCWLFmCOXPmYOLEiU3aAK9bt47f4P4P10sZ+BIbGwsPD49mEzEAuHLlCrp06YLc3FzWYrh+/TpWrFjR4v1Tp07B39//vVma96onT57g119/ZZYYvXjxArW1tSgoKICnpyfP0bFv1qxZzIc3qVQKqVTaYpc0QvjyaotyX19f9OvXD6tWrYKDgwPrY/fp06fJkqaePXs2KU707t2b9Vhaw/WcyucbmZbOU1NRUYGHh4fcwb/R0dGcxMT1+aHNebWrYyMtLS2sWrUK1dXVnHR1bO7vxZMnT+S6WrMpPz8f8fHxSE9PR3V1NQYOHAhvb2+EhYVh6dKlGDBgACdxtAVKxhSclpYWjhw5ItcGeNasWZy0AX6dUJYy8EUIiVBtba3c4devS0tLw927d3lZxsqn1NRUrFy5ErW1tRCJRHjx4gXzoU9bW/u9SMa+/vprrFmzBj///DPS09Ph4uKCHj16MIlZYzGHED69vlhn5syZ0NbWhre3N+tjC63rmpDm1OrqalRWVqJr166tthNva682U3mb61wQQmEvJiYG33zzjVxS6Ofnh2HDhmHbtm2cJGOVlZXYuHEjHB0dMWDAAMhkMuTk5EAikSA8PJzVY4Xs7OxQVFQEPT09uLq6wtramukeLoQVYX8XJWMKrrEN8Lx58+SuP3v2DFu3buV070dzm0tfXcrQ3gkhEdLV1UVeXl6LE5VIJEJubu5798F7586dkEqlmD9/PubOnYuoqCg8ePAAa9euxcKFC/kOjzPKysowNzeHubk5/P39cfbsWWRkZMDe3h59+/aFtbU1pFIp52cUEtIoOjq6yX5WMzMzxMXFKcRG/LbE95xaVVWF3bt348iRI0zTBuDlPGNjYwM3NzfWE7PmjqKpra2FmpoagJet1Xv16sVqDK8SSmHvwYMHGDZsWJPro0aNwr179ziJYd26dbh06RJcXV2RlZWF3NxcbNy4Eenp6di4cSOrSVFhYSH69+8PW1tbmJubK/wxTpSMKThHR8dm2wA/e/YM27Zt4zQZE8LmUj4JIRGSSqUIDQ3FuHHjmD2Er3r48CFCQ0Ph4uLCWgxCVFJSgrCwMOjp6WHw4MF48uQJLC0tUVdXh507d2LatGl8h8g5VVVVTJw4ERMnTkRtbS0SExOxefNmfPvttwpxSCZpn4yNjVFXV4fS0lLU19cDePnmQUlJqUk78faOzzm1rKwMjo6OuH//PqysrDB79mx069YNlZWVuHbtGsLDw5GRkYHvv//+rQ6SbwuN++ENDQ2ZfdfTp0+Hvr4+vvvuO06aUgmlsMdnV8dXx9q2bRv09PQQEREBc3NzfPrppxg8eDDry4ozMzORlJSEmJgYbN68mSkm8r189J+iZKwd4KsNcHP4WsogBEJIhJydnZGRkQFbW1vIZDIYGRmhW7duKC8vx8WLFxEVFQVdXV1O9l8IiZqaGlNJ1dXVxc2bNzF+/HgMGzYMd+7c4Tk6/jx48ADHjh3D0aNHkZubC11d3b91Zh8hbe3s2bNYunQpnjx50uRex44dYWNjw0NU/OFrTg0JCUFDQwOOHDnS7B65P/74A+7u7oiKiuJkCSkABAUFobq6GlKplLkWEREBf39/bNiwgZN98kIp7AnhuJZnz54xfzeys7OZ43o6duzIFFLYoqurCx8fH3h7e+PMmTNITExEZGQkdu7cCeBlkxU3NzfOCgXvipIxBSeENsBCWMogBEJIhNTU1BAdHY0NGzYgODhYbkmkmpoa7Ozs4OvrC2VlZdZiEKJhw4bh4MGDWLx4MQYNGoTTp09DJpPh1q1bcmcIvQ9KS0uRmZmJo0eP4vLly9DR0cEnn3yClStXUkMPwrtvv/0WQ4cOhZOTE7y9vbFp0ybcu3cPoaGhzS5Za4+EMKeePn0aq1evbrFZiZaWFry9vREcHMxZMnb27Fns27dPrmmHgYEB1qxZg88//5yTGIRS2BPCcS16enr48ccf0bt3bzx8+BDjx48HACQkJHDWYVNJSQkWFhawsLBAeXk5UlNTkZycjO3bt2PPnj2wsbHB2rVrOYnlXVAypuBevHiBTp06YevWrdi4cSPc3NyYNsBcEOJSBr4IJRHq3Lkz/Pz84Ovri/z8fJSVlUFDQwNisRgSiaTJoabvg4ULF2L+/PlQV1fH9OnTsW3bNkilUty/f59pC9ze7d27F5mZmfjll1/Qp08ffPLJJ1ixYgUMDAz4Do0Qxq1bt7Bu3Tro6+tjyJAh6NSpE5ycnNCpUyfs3r0bkydP5jtEVgllTn306JFc0tMcfX19zvYnAS/PWWvuNCZVVVVUV1dzEoOQCnt8L8v76quvsHDhQjx//hxTp06FRCLB+vXrERsbi23btnEej7q6OpydneHs7IzCwkIkJSUhLS1NIZIxOmdMwQ0ZMgRnz55Fjx49AAAHDx7E2rVr4eDggOjoaNb3fvj7+yMnJwdRUVGtLmWYPHkyZ9UzvlVXV6O+vp63RCgtLQ1BQUGIiIiQ2+Ark8lw9epVBAYGwsrKivU4hKa0tBS1tbXQ0dHBrVu3EBcXh969e8PZ2fm9eDumr68PVVVVmJmZYfjw4a0+S4c+E74YGhoiLS0N2tra8PPzg76+PlxcXHD37l1Mnz4dFy5c4DtEVgllTn31vLeWPHr0COPGjeNsj6mnpyfq6urw3XffMXvkq6qq4Ovri7q6OoSHh7Mew6VLlzB//nx89dVXmD59OqZMmQJNTU2msBcUFMR6DI2E0NWxrKwMpaWlzKqK/Px8dO7cmfOz5+rr61FWVoauXbuiQ4cOzPX79+9DLBYLvghNyZiCa+4X5rlz5+Dt7Y2qqirWf0lOnDgRq1evbvW1+PHjxxEcHIzMzExWYxECvhOhnJwcuLm5wdbWFj4+PujZsydzr7i4GJGRkUhNTUVMTAyMjIxYi0No/Pz8sGLFiibHPZSXl2P58uXYvn07T5Fxx9LS8q2eo0OfCZ8aj2ZxcnJCREQECgoKEBoaitzcXHh6erb7ZEwoc6oQk7HffvsNDg4OePr0aZNzVSMjI9G/f39O4hBCYe9NXR2PHz/OSRzAy/8HN27cgJKSEgwMDDg9h6/xAO78/HzmremgQYPg5OSEGTNmwMvLCyNHjmT2swkVLVNUcHy3ARbiUga+5OTkwNfXF7a2tk1a7S5fvhyRkZFYtGgRq4lQeHg4HB0dsXz58ib39PT0mD0XO3bsQEREBCsxCEVubi6z3yIlJQUGBgZNkrHi4mKcP3+ej/A4J7QzlAhpzueff46vvvoKqqqqmDp1KsLCwvD555/jv//9L0xMTPgOj3VCmlOjoqJa3Zv27Nkz1mN4Vd++fZGeno4jR47g5s2bUFFRwdy5c/Hpp59y9uajsbDXOMcPGDAAK1euRHl5Oby9vTkr7Amhq2NVVRUWL16MM2fOMImQSCSCtbU11q9fz+ytY0tgYCD2798PU1NTLFq0CGKxGOXl5cjJycGqVauQkpKC4uJiTt9W/lOUjCk4vtsAP3/+/I2/BDt27Ii6ujrWY+GbEBKhwsJCLF26tNVn5s2bhwULFrAyvpCIRCLmnB6RSITAwMAmz3Tq1AkymYzr0AghLZg8eTIOHjwIZWVl9O7dG5GRkdizZw8mTZqEr776iu/wWCeUObVPnz5vVdDl8i0IAHTt2hVz5szhdEwhFvaE0NUxKCgIv/76K8LDw2FoaIiGhgbk5eUhICAA3377bbPn5LWVrKwsxMfHY9euXbCwsJC75+7ujuPHj8PLywseHh6cHHnwrigZU3DUBlg4hJAI1dTUvHEiV1dX52yzM5+MjIyYc3r09fVx9uxZaGpq8hwVIeRNXm0qY2xsDGNjYx6jeT8J8U16TU0N4uPjcePGDbnW6bW1tbh69SpryzaFWNgTQlfH48ePY/v27RgzZgxzbcKECVBTU8OSJUtYTcZiY2Ph4eHRJBFrdOXKFXTp0gW5ubmsxdCWKBlTcEJoAyy0pQx8EUIi1K9fP1y+fLnVt6KtHUzdXjV3eOqTJ0+goaHBQzSEkJY8efIEERERuHnzJmpra5vcj46O5iEqbinCnFpRUYGkpCQkJCRwsiUCeLksLSUlBUOHDkVBQQEMDQ1x584dPH78uMnhx21JiIU9IXR1VFZWbrajZ8+ePVl/c3v9+nWsWLGixfunTp2Cv78/Z2euvStKxhQc322AhbqUgQ9CSIRsbGwQEhICExOTJvvWgJcbj0NCQmBvb89aDEJUWVmJjRs3wtHREQMGDIBMJkNOTg4kEgnCw8Oho6PDd4iEEAC+vr4oKCiAmZmZ4DugsUHoc2pubi7i4uJw7Ngx1NTUYOjQoZyNfeLECaxfvx5Tp06FlZUVAgICoKOjAx8fH7mjZNgklMKeEI5rcXZ2RkBAAEJCQpjktKqqClu2bIGzszOrY9fW1qJTp04t3k9LS8Pdu3c5+3vxrigZU3CvViZ0dXVx48YNmJqawsTEBBs2bGB9fCEuZeCLEBIhR0dHZGZmYurUqbC3t4ehoSFz8HReXh6Sk5MhkUjeu31S69atw6VLl+Dq6oqsrCzk5uZi48aNSE9Px8aNGxEWFsZ3iIQQvPywv2vXrvd2aaIQ59SqqiqkpKQgPj4et27dAgCMHTsW7u7unP5/+vPPP5nmVwMGDEBhYSH69+8PDw8PLFq0CCtXrmQ9BqEU9j788ENkZmaitrYWYrEYsbGxcl0duXD27FkUFBRg0qRJkEgkUFFRwe3bt/H06VMUFRUhOTmZebatO/Tq6uq2WtwWiUTIzc1lum4KHSVjCm7gwIE4efIknJyc0L9/f+Tm5sLFxQV//PEH36EB4GcpA1+EkAgpKytj79692LJlCxITE7F3717mnqamJhwcHODp6fneVZxPnz6Nbdu2QU9PDxERETA3N8enn36KwYMHw8HBge/wCCH/p1evXujcuTPfYQgWl3Nqfn4+4uPjkZ6ejurqagwcOBDe3t4ICwvD0qVLMWDAAFbHf52GhgYeP36MPn36QCKR4MaNGwAAsViMR48ecRKDUAp7QujqaGZmBjMzM9bHaY5UKkVoaCjGjRsHdXX1JvcfPnyI0NBQuLi4cB/cP0DJmIITahtgPpcy8EUoiZCamhp8fX2xePFilJSUoKKiAhoaGtDR0WHOIXnfPHv2jFnWk52dzZw50rFjR7mN4IQQfv3nP//B2rVr4ePjAx0dHSgpKcnd79OnD0+R8YvrOdXOzg5FRUXQ09ODq6srrK2tMXDgQADgbSXB+PHjsXbtWqxfvx6jR4/GunXrYGVlhfT0dGhpaXESA5+FPaF1dfTy8uJknOY4OzsjIyMDtra2kMlkMDIyYorfFy9eRFRUFHR1dRWm2ErJmIITUhtgoSxl4JOQEiEVFRX069eP0zGFSk9PDz/++CN69+6Nhw8fYvz48QCAhIQE6Onp8RwdIaTRixcvUFxcjM8++6zJdZFIxNkBw0LA55zauATQ1tYW5ubmTCLGJ19fXyxbtgwXLlzAvHnzEB8fj5kzZ0JFRYWTbRkAv4U9oXV1TElJafW+ra0ta2OrqakhOjoaGzZsQHBwsNzeMDU1NdjZ2cHX1xfKysqsxdCWRC8aT2oj5B9qbimDVCpFWFgYDh06xPlSBkJed/r0aSxcuBDPnz+HVCrFpk2bsH79esTGxmLbtm0ttsclhHDL0tISQ4YMwaxZs5rtKPg+FPaEMKfeuXMHSUlJSElJwYMHD9C3b19YW1vD2toatra2gpjbX7x4gaKiImhqakIkEqFnz56sj2lvb4+ZM2eid+/e8PDwwNGjRyGRSLBp0ybk5OTg4MGDrMcACKOro76+frPXO3ToAC0tLdaOGnhVdXU16uvrkZ+fj7KyMmhoaEAsFkMikSjUdgxKxhQc322AX13KYGVlJbeUwcDAQBC/sAkBgLKyMpSWljITSH5+Pjp37kxvxggRkFGjRuHw4cPvbYdToc2pDQ0NOHPmDBITE3Hq1CmmZbmnpyfc3NyabW3OliFDhiA7O7tJ58Lff/8dn376KS5fvsx6DEIu7PF9XEt9fT1u374Nf39/zJ49G1OnTmV1vLS0NAQFBSEiIgLDhg1jrstkMly9ehWBgYGwsrJiNYa2QsmYgps/f36rbYDZPmtMX18f/fv3h52dHczNzTFkyBDmHiVjRGhu376NGzduQElJCQYGBu/FkQuEKBIPDw9IpVLY2NjwHQovhDynlpeXIzU1FcnJySgqKsIHH3wAGxsbrF27lrUxf/jhB6SmpgIALly4AENDwybnaD148ADV1dU4ffo0a3G8SgiFPaF0dWxOYWEhvL29kZWVxdoYOTk5cHNzg62tLXx8fOTeihYXFyMyMhKpqamIiYlhOnAKGSVjCs7Q0JDXNsCKsJSBkKqqKixevBhnzpxB4688kUgEa2trrF+/HmpqajxHSAgBgIiICGzfvh0TJ05E3759oaIiv7Wdz6YBXFCUObWwsBBJSUlIS0vDzz//zNo45eXlzH6w5ORkfPLJJ00Kz507d4atra3c2xG28V3Y8/Pzw6VLl7Bz507cvHkT//nPf7Bu3Tqkp6dDRUWF1+Narl+/jtmzZ+OXX35hbQyZTAY9PT0sX768xWf8/Pzw6NEjREREsBZHW6FkTMFNmTIFmzdvhoGBAa9xCGkpAyGva5y4Vq9eDUNDQzQ0NCAvLw8BAQGwsrJiNkUTQvhlaWnZ4r2//voL586d4zAa/ghtTq2vr0dZWRm6du2KDh06MNfv378PsVjMyf6cxnbuXbp0wZMnT3Dp0iX06NEDo0ePZn3sRkIp7JmZmWHbtm0wNDTEsmXLUF5ejp07d+LGjRtwcHDAxYsXWY+huQYeVVVVSEhIQPfu3RETE8Pa2Kampti3bx8GDRrU4jMFBQVYsGABsrOzWYujrVAypuBOnDiBXbt2CaoNMB9LGQhpzZgxY7B9+3aMGTNG7vq5c+ewZMmS9+YDHiGK6ObNm4iLi8Phw4dx4cIFvsPhHJ9zanp6OqKjo5Gfn88kH4MGDYKTkxNmzJgBLy8vjBw5kukqyIbt27dj3759SEhIgK6uLi5fvgx3d3c8ffoUAGBiYoIdO3ZwlhAKobA3atQoHD16FFpaWhg3bhzc3d3h7OyM3377Dba2tsjLy2M9huYaeKioqMDQ0BD+/v6sLts0MjJCSkoK+vbt2+IzJSUlmDZtGic/i3dFre0VnBDbAKurq8PZ2RnOzs5ySxkoGSN8UVZWbraS3LNnT6biTAgRjtraWhw9ehRxcXG4fPkyRCIRJk+ezHdYvOBrTg0MDMT+/fthamqKRYsWQSwWo7y8HDk5OVi1ahVSUlJQXFyMoKAg1mKIj4/Hjh074Orqih49egB4mRB17NgRcXFx6Nq1KxYuXIjw8HBOjvM5fvx4k8LehAkToKamhiVLlnCWjAnhuJbr169zMk5z+vXrh8uXL7eajOXl5UFbW5vDqP45SsYU3Lp162BiYtJiG2Cuvb6UYejQoRg6dChkMhn++usvhWo1StoPZ2dnBAQEICQkhGkFXFVVhS1btsDZ2Znn6Aghje7cuYO4uDgkJyejvLwcIpEIdnZ2WLBgwXvZYZGvOTUrKwvx8fHYtWtXkw6B7u7uOH78OLy8vODh4YHu3buzEgMAHDx4EMuWLWMO7y0oKMDt27fh4+PD7J3z9PTEN998w0kyJpTC3ldffcV0dZw6dSokEolcV0euVFdXo7KyEl27duX0M6iNjQ1CQkJgYmKCXr16NblfWlqKkJAQ2NvbcxbTu6BkTME9efIEy5Yt432SetNShqCgINaXMhDSkrNnz6KgoACTJk2CRCKBiooKbt++jadPn6KoqAjJycnMsydOnOAxUkLeP/X19Th27Bji4+ORk5MDZWVljB07FlKpFH5+fnBzc+N9juMa33NqbGwsPDw8WmzVfuXKFXTp0gW5ubmsjN+ouLgY5ubmzNc///wzRCKRXFwDBgzAvXv3WI2jkVAKexYWFjh9+rRcV0epVIpZs2ax/masqqoKu3fvxpEjR1BSUsJc19XVhY2NDdzc3FhPzBwdHZGZmYmpU6fC3t4ehoaG6NatG8rLy5GXl4fk5GRIJBLODsB+V7RnTMEJoQ3wq0sZTE1N5ZYyZGdnY/To0SguLsbRo0dZraAR0pKtW7e+9bPtvVsbIUIzduxYVFZWwsTEBFZWVrCysmLmCr7bufNBCHOqiYkJYmJimDPOXieVSuHp6YmAgADk5OSwEgPwsmN0SkoKdHV1Abz8zPPLL7/IdXC8fv06nJ2dOdlPOG/ePBQUFEBJSalJYa9Pnz4QiUTMs1wU9rju6lhWVgZHR0fcv38fVlZWGDRoELp164bKykpcu3YNJ06cgI6ODr7//nvWm8zU1tZiy5YtSExMREVFBXNdU1MT9vb28PT0VJjVWPRmTMF9+OGHWLNmDX788Ude2gALZSkDIa2hBIsQ4aqsrESPHj3Qp08fqKurC2LJPV+EMqfW1taiU6dOLd5PS0vD3bt38fz5c9ZiAF6+DczLy4Ouri7+/PNP5OTkYNKkSXLPZGRktNpVry2ZmZnBzMyMk7Faw1dXx5CQEDQ0NODIkSPNJn5//PEH3N3dERUVBW9vb1ZiaKSmpgZfX18sXrwYJSUlqKiogIaGBnR0dOSSYkVAyZiCO3DgAMRiMa5cuYIrV67I3fvrr79Y/xAqlKUMhLSmuRa8r7K1teUkDkJIU9nZ2UhPT0diYiIOHDiAzp07Y9KkSbC2tla4D1XvSihzqq6ubqsNEEQiEXJzcyGRSFiNw8HBAWvWrEFRUREuX76M2tpauLi4AHi5L+jw4cPYvXs3q01EXiWUwl5QUBB+/fVXhIeHN+nq+O2337LWSOT06dNYvXp1i2/gtLS04O3tjeDgYNaTsUYqKiro168fJ2OxhZIxBXfy5Mkm115tA8y269evY8WKFS3eP3XqFPz9/REQEMB6LIS0pKWJqUOHDtDS0qJkjBAedenSBbNmzcKsWbNQXFyMH374AYcPH0ZqaipEIhH27t0Ld3d3ZqlaeyaUOVUqlSI0NBTjxo2Durp6k/sPHz5EaGgokxixxcbGBrW1tThw4ACUlJTw3XffYcSIEQCAXbt2ISEhAe7u7pg2bRqrcTQSSmGPr66Ojx49euNbSH19fc728LUXlIy1E3y1ARbKUgZCWvN6C976+nrcvn0b/v7+mD17Nk9REUJep6enh6VLl2LJkiX48ccfkZycjJSUFCQlJcHMzAyRkZF8h8gqocypzs7OyMjIgK2tLWQyGYyMjJgGCRcvXkRUVBR0dXWZLodsmjFjBmbMmNHkuoeHBxYuXAixWMx6DI2EUtjjq6vj8+fP37gPq2PHjnRkzN9EyZiC47sNsFCWMhDydygrK0NPTw9+fn7w9vbG1KlT+Q6JEPIKZWVlTJo0CZMmTcKTJ09w6NAhJCUl8R0W64Qyp6qpqSE6OhobNmxAcHCwXPKnpqYGOzs7+Pr6QllZmdU4WtNcS3O2CaWwJ5SujqRtUDdFBfSmNsApKSmcdZ6KjIxEfHw8Dh482OJShjlz5sDFxYV+QRDBuX79OmbPno1ffvmF71AIIURwc2p1dTXq6+uRn5+PsrIyaGhoQCwWQyKRKEynOi4UFhbC29sbWVlZnIzHV1dHfX19yGSyVpvsPHv2DHv27EFRUVGbjdve0ZsxBWRhYcG0AQ4ICJBrA8zV6e+NhLSUgZCWNLfOv6qqCgkJCcz+A0II4ZuQ5tS0tDQEBQUhIiJCroOgTCbD1atXERgYCCsrK9bjUARKSkp48OABZ+Px1dWxT58+yMjIeONzbLfYb2/ozZgCGjlyJHr06AELCwuYm5tj/PjxTBtTPs5kefr0KTZs2ICUlJQWlzK8z62KCf8aD8V8lYqKCgwNDeHv78/6IZmEEPK2hDCn5uTkwM3NDba2tvDx8UHPnj2Ze8XFxYiMjERqaipiYmJgZGTEaixC0lphr3v37oiJieE+KKLwKBlTQFVVVUwb4F9++UWuDbCXlxenyxQb0VIGQgghpG3wPafKZDLo6elh+fLlLT7j5+eHR48eISIigvV4hEIohT2hdHV8XUVFBZKSkpCQkPBWb9DIS5SMKbhX2wA/evQIIpEI9vb2nLYBfnUpw7Bhw5jrtJSBCEl1dTUqKyvRtWtXelNLCBEsIcyppqam2LdvX6ttzAsKCrBgwQJkZ2ezGgtpqrmkEPj/XR0zMzM5jSc3NxdxcXE4duwYampqMHTo0Pei4U5boWSsnaivr2faAP/4449oaGjgpA0wLWUgQlZVVYXdu3fjyJEjKCkpYa7r6urCxsYGbm5ulJgRQgRDKHOqkZERUlJS0Ldv3xafKSkpwbRp05CXl8daHEIkxMLe610duegQXFVVhZSUFMTHx+PWrVsAgLFjx8Ld3R3Gxsasj9+eUDLWDr3aBpjtg59pKQMRqrKyMjg6OuL+/fuwsrLCoEGD0K1bN1RWVuLatWs4ceIEdHR08P333zd7XgshhHBNKHOqvb09nJ2dWz1M+dChQ4iMjGT9c4YQKEphj4uujvn5+YiPj0d6ejqqq6sxcOBASKVShIWFcd6zoL2gbortkIaGBtzc3ODm5sb6WIWFhVi6dGmrz8ybNw8LFixgPRZCXhUSEoKGhgYcOXKk2c5Of/zxB9zd3REVFQVvb28eIiSEEHlCmVNtbGwQEhICExOTZs/zKi0tRUhICOzt7VmNQwheL+zNnj1brrAXHh6OjIwMQRT22O7qaGdnh6KiIujp6cHV1RXW1tYYOHAgACAsLIy1cds7SsbIO6mpqXnjZmJ1dXVUV1dzFBEhL50+fRqrV69uscWulpYWvL29ERwcTMkYIUQQhDKnOjo6IjMzE1OnToW9vT0MDQ2ZFvt5eXlITk6GRCKBTCZjNQ4hEGJhj6/jWgoLC9G/f3/Y2trC3NycScTIu6FkjLyTfv364fLly62uK8/Ly4O2tjaHURECPHr0qNXN58DLTdD37t3jKCJCCGmdUOZUZWVl7N27F1u2bEFiYiL27t3L3NPU1ISDgwM8PT3fi27JQizsNXem7KtdHdmSmZmJpKQkxMTEYPPmzejbty+sra1hbW3N2pjvA9ozRt7Jvn37sG/fPhw4cKDFpQxz586Fvb09vvzySx4iJO8rfX19ZGdno0ePHi0+8+jRI4wbNw5FRUUcRkYIIc0T4pxaV1eHkpISVFRUQENDAzo6OhCJRJyMLQTDhw/H0aNHW02Af//9d3zyyScoKCjgMDL+NDQ04MyZM0hMTMSpU6dQV1cHAPD09ISbmxvvyzUVDSVj5J3U19fDyckJN2/ebHUpw/79+9+LChoRDkrGCCGKhuZU4RHqXCKUro7l5eVITU1FcnIyioqK8MEHH8DGxgZr167lLSZFQ8kYeWe1tbXMUoaKigrmuqamJuzt7d+bpQxEWPT19SGTyVqdpJ49e4Y9e/ZQMkYIEQyaU4VFSMmY0Ls6FhYWIikpCWlpafj55595i0PRUDJG2sz7vpSBCIulpeVbP3vy5EkWIyGEkL+P5lRhEEphT4jHtdTX16OsrAxdu3ZFhw4dmOv379+HWCymosFbomSMEEIIIYSQZgilsOfv74+cnBxERUW12tVx8uTJrDcSSU9PR3R0NPLz89GYRgwaNAhOTk6YMWMGvLy8MHLkSLi7u7MaR3tByRgh5L1UUVGBpKQkJCQkICMjg+9wCCGEkBZNnDgRq1evxsSJE1t85vjx4wgODkZmZiZrcQQGBmL//v0wNTWFqakpxGIxysvLkZOTg+zsbIwePRrFxcU4evQounfvzloc7Qm1tieEvFdyc3MRFxeHY8eOoaamBkOHDuU7JEIIIQqKq8KeEI5rycrKQnx8PHbt2gULCwu5e+7u7jh+/Di8vLzg4eFBidjfQMkYIaTdq6qqQkpKCuLj43Hr1i0AwNixY+Hu7g5jY2OeoyOEEKJouC7sPX/+/I17sDp27Mi0mWdDbGwsPDw8miRija5cuYIuXbogNzeXtRjaI0rGCCHtVn5+PuLj45Geno7q6moMHDgQ3t7eCAsLw9KlSzFgwAC+QySEEKIg3vfC3vXr17FixYoW7586dQr+/v4ICAjgMCrFR8kYIaRdsrOzQ1FREfT09ODq6gpra2sMHDgQABAWFsZzdIQQQhSFUAp7UVFRb+zqyKba2lp06tSpxftpaWm4e/cunj9/zmoc7Q0lY4SQdqmwsBD9+/eHra0tzM3NmUSMEEIIeVtCKez16dPnrfakNddpsa3o6uoiLy8P2trazd4XiUTIzc2FRCJhLYb2iJIxQki7lJmZiaSkJMTExGDz5s3o27cvrK2tYW1tzXdohBBCFIRQCntCOA9TKpUiNDQU48aNg7q6epP7Dx8+RGhoKFxcXLgPToFRa3tCSLvW0NCAM2fOIDExEadOnWI2N3t6esLNzY2zwzEJIYQonjt37iApKQkpKSl48OCBXGHP1tYWhw4dEsT+Yy66OtbW1mLu3Ll4/PgxZDIZjIyM0K1bN5SXl+PixYuIioqCrq4uoqOjoayszEoM7RElY4SQ90Z5eTlSU1ORnJyMoqIifPDBB7CxscHatWv5Do0QQoiACbWw11xXx6SkJNbGe/r0KTZs2ICUlBS5vWFqamqws7ODr69vq/vaSFOUjBFC3kuFhYVISkpCWloafv75Z77DIYQQoiD4Luzx3dWxuroa9fX1yM/PR1lZGTQ0NCAWiyGRSN7Yfp80RckYIeS9UF9fj7KyMnTt2hUdOnRgrt+/fx9isZgmEEIIIX8bl4W95ro6SqVShIWFcbZcMi0tDUFBQYiIiMCwYcOY6zKZDFevXkVgYCCsrKxYj6M9oWSMENKupaenIzo6Gvn5+Wj8dTdo0CA4OTlhxowZ8PLywsiRI+Hu7s5zpIQQQoSOr8Leq10drays5Lo6GhgYcJKM5eTkwM3NDba2tvDx8UHPnj2Ze8XFxYiMjERqaipiYmJgZGTEaiztCSVjhJB2KzAwEPv374epqSlMTU0hFotRXl6OnJwcZGdnY/To0SguLsbRo0fRvXt3vsMlhBAiUHwX9vT19dG/f3/Y2dnB3NwcQ4YMYe5xlYzJZDLo6elh+fLlLT7j5+eHR48eISIigtVY2hNKxggh7VJWVhYWL16MrVu3wsLCosn948ePw8vLCx4eHvDx8eEhQkIIIYpACIU9IXR1NDU1xb59+zBo0KAWnykoKMCCBQuQnZ3NaiztCZ0zRghpl2JjY+Hh4dFsIgYAV65cQZcuXZCbm8txZIQQQhRFVlYW4uPjsWvXribzibu7u1xhj80VFrq6uvDx8YG3tzfT1TEyMhI7d+4E8PLNHdtdHWtqat64DFNdXR3V1dWsxdAeKfEdACGEsOH69ev4+OOPW7x/6tQp+Pv74+bNmxxGRQghRJEIrbCnpKQECwsLhIaG4syZM/Dz84O+vj62b9+O8ePHY82aNayN3a9fP1y+fLnVZ/Ly8qCtrc1aDO0RJWOEkHaptrYWnTp1avF+WloaRo0aJXdOCiGEEPIqIRf21NXV4ezsjOTkZCQlJcHe3h6ZmZmsjWdjY4OQkBCUlpY2e7+0tBQhISGYMmUKazG0R7RMkRDSLunq6rZaoROJRMjNzYVEIuE2MEIIIQrjbQp7d+/e5byw93pXx6FDh2Lo0KGQyWT466+/WOnq6OjoiMzMTEydOhX29vYwNDREt27dUF5ejry8PCQnJ0MikUAmk7X52O0ZJWOEkHZJKpUiNDQU48aNg7q6epP7Dx8+RGhoKFxcXLgPjhBCiEIQWmHvTV0dg4KCWOvqqKysjL1792LLli1ITEzE3r17mXuamppwcHCAp6cnndv5N1E3RUJIu1RbW4u5c+fi8ePHkMlkMDIyYip4Fy9eRFRUFHR1dREdHQ1lZWW+wyWEECJAkZGRiI+Px8GDB1ss7M2ZMwcuLi5wdnZmNRYhdHVsVFdXh5KSElRUVEBDQwM6OjoQiUSsjtleUTJGCGm3nj59ig0bNiAlJUVuCYmamhrs7Ozg6+uLDz74gMcICSGECJlQCnt0XEv7RckYIaRdq66uRn19PfLz81FWVgYNDQ2IxWJIJBJaSkEIIeSNhFDYc3V1xYcffggvL69m72/atAlxcXHQ19fH/v37WY2FtC1Kxggh7VZaWhqCgoIQERGBYcOGMddlMhmuXr2KwMBAWFlZ8RghIYQQRcB3Yc/ExAQxMTEYOHBgs/elUik8PT0REBCAnJwc1uMhbYda2xNC2qWcnBz4+vpi4sSJ6NWrl9y95cuXw9LSEosWLUJeXh5PERJCCFEEaWlpsLS0xO3bt2FmZgapVApTU1MEBwfDwsICWVlZrMdAx7W0X5SMEULapfDwcDg6OmLdunXo2bOn3D09PT2sX78eNjY22LFjB08REkIIETqhFPYauzq2hI5rUVyUjBFC2qXCwkLMmDGj1WfmzZuHwsJCjiIihBCiaIRS2Gs8rqW8vLzZ+43Htdja2rIaB2l7lIwRQtqlmpqaN67jV1dXR3V1NUcREUIIUTRCKew5OzujW7dusLW1RUxMDK5du4aSkhIUFBQgKioK06dPh5aWFhwcHFiNg7Q9OvSZENIu9evXD5cvX0bfvn1bfKa1gzwJIYQQoRT21NTUEB0djQ0bNiA4OLjFro50bqbioWSMENIu2djYICQkBCYmJk3W+QNAaWkpQkJCYG9vz0N0hBBCFIGQCnudO3eGn58ffH196biWdoSSMUJIu+To6IjMzExMnToV9vb2MDQ0ZA7qzMvLQ3JyMiQSCWQyGd+hEkIIESghFfZePa7FzMyMuU7HtSg2OmeMENJu1dbWYsuWLUhMTERFRQVzXVNTE/b29vD09KRKIiGEkBbV19fDyckJN2/ebLWwt3//flbnk5ycHLi5ucHW1hY+Pj5yzUSKi4sRGRmJ1NRUxMTEwMjIiLU4SNujZIwQ0u7V1dWhpKQEFRUV0NDQgI6ODkQiEd9hEUIIUQBCKOzJZDLo6elh+fLlLT7j5+eHR48eISIigtVYSNuiZIwQQgghhJA34LOwZ2pqin379mHQoEEtPlNQUIAFCxYgOzubk5hI26A9Y4QQQgghhLyBiooK+vXrx8vYQunqSNoenTNGCCGEEEKIgDV2dWwNHdeimCgZI4QQQgghRMAauzqWlpY2e7+xq+OUKVM4joy8K9ozRgghhBBCiIAJpasjaXuUjBFCCCGEECJwQujqSNoeJWOEEEIIIYQoCDqupX2hZIwQQgghhBBCeEANPAghhBBCCCGEB5SMEUIIIYQQQggPKBkjhBBCCCGEEB5QMkYIIYRTy5Ytw+DBg1v9x8nJ6Z3H+f333zF48GAkJSW1QdTvt2XLlsHS0pLvMAghpN1R4TsAQggh75cvvvgCc+bMYb7evn07CgsLsXXrVuZaly5d+AiNEEII4RQlY4QQQjjVt29f9O3bl/laQ0MDampqGDVqFH9BEUIIITygZYqEEEIE6eDBg7Czs8OoUaMwYsQITJs2DRkZGcz9hoYGfPfdd7C0tMSwYcNgaWmJzZs34/nz581+vxcvXsDPzw8jRozA2bNnAQC//fYbFixYgI8++ggjR47E7Nmzcfr06VbjsrS0xLp16+Di4oIRI0ZgxYoVAIDy8nKsXr0aZmZmGD58OGbNmoXz58/L/dnBgwfjwIEDWLZsGUaPHg1jY2MEBgbir7/+woYNG2BiYoKPPvoIK1asQE1NDfPnampqsG3bNkyZMgXDhw/H//zP/yA8PBwNDQ0AgJ07d2LYsGFyB8ECwN69e2FgYIDHjx8DAO7du4fFixfD2NgYI0eOhIuLCwoLC+X+TEVFBfz8/GBsbIwxY8YgODiYGYcQQkjbomSMEEKI4MTGxmL16tWYPHkydu3ahU2bNkFNTQ1LlizBH3/8AQCIiIjAgQMH8OWXXyIqKgpz587F7t27sWPHjma/Z2BgINLS0rB161aMHTsWDQ0N8PDwQHV1NTZu3Ijt27dDXV0dnp6euHPnzhvjGz58OLZv344ZM2agpqYGLi4uOHHiBHx8fLB161ZoaWlh/vz5TRKy4OBgqKmpYevWrbC1tUVMTAxsbW1x//59bNq0CU5OTvjhhx8QExMD4GUSuWDBAkRGRmLmzJnYuXMnpkyZgi1btmDNmjUAgE8//RR1dXU4duyY3FhHjhzB2LFj0aNHDzx58gRz5szBtWvXsGrVKmzevBkNDQ1wcHBAcXExgJcJ7vz583H69GksXboU33zzDfLy8pCenv73/ycSQgh5I1qmSAghRHBKSkogk8nwxRdfMNe0tbVhZ2eH3NxcSKVSXLhwAcOGDYO9vT0AwNjYGB988AG6du3a5Ptt3rwZ8fHx2Lp1K8aPHw8AePz4Mf73f/8XX3zxBSwsLAAAI0aMwNatW1FbW9tqfH369MGSJUuYrxMSEnD9+nUkJCRg5MiRAIDx48fDyckJmzZtQmJiIvPsgAED8PXXXzMxHzx4EM+fP8emTZugoqKCsWPHIjMzE3l5eQCAn376CefOncO3334LqVQKADA3N0fHjh0REhICZ2dnDBw4EGPGjEFaWhpmzpwJ4OVbv/z8fHz33XcAgH379qG8vBwHDhyAtrY2E6O1tTVCQkIQGhqKn376Cfn5+YiIiGB+TqamptS8gxBCWEJvxgghhAjOsmXLsGTJEvz555+4cuUKDh06hNjYWABgEqWPPvoI2dnZmDdvHiIjI3Hr1i04Ojpi2rRpct8rNjYW4eHhkEqlmDBhAnNdU1MTAwYMwKpVq7B06VIcPnwYDQ0N8PPzw8CBA1uNb8iQIXJfnz9/Hj179oSBgQHq6upQV1eH+vp6TJw4EVevXpVbPmhoaMj8u7KyMsRiMQwMDKCi8v/ro+rq6qisrAQAXLhwASoqKpgyZYrcmDY2Nsz9xq8vXryIhw8fAnj5VqxLly5MInX+/HkMGTIEvXr1YmJUUlLC+PHjce7cOQDApUuXoKqqinHjxjHjdOrUiUlWCSGEtC16M0YIIURwfvvtN6xevRrnz5+Hqqoq+vfvD319fQAvl+0BwPz589G5c2ckJiZi06ZNCA4OxsCBA7Fy5UqYmJgw3+v69esYO3Ys0tLS4OLigqFDhwIARCIRoqKisGPHDmRlZSElJQWqqqqYPHky1q5di+7du7cYX6dOneS+Li8vx8OHD2FgYNDs8w8fPmS+X3OdIl//fq+qqKiAWCyGsrKy3PWePXsCAJO0TZkyBQEBAcjIyICzszOOHDmCjz/+GB07dmRivHPnTosxVldXo6KiAurq6hCJRM2ORQghpG1RMkYIIURQGhoa8Pnnn0NVVRU//PADhgwZAhUVFdy6dQuHDh1inlNSUoKDgwMcHBzw+PFjnD59Gjt37sTChQuRnZ3NPOft7Q1nZ2dIpVKsXLkSBw8eZBKbXr16wd/fH2vWrMH169dx9OhRREREQCwWM/ux3kbXrl0hkUiwadOmZu//61//+oc/DaB79+4oKytDfX29XEL24MEDAIBYLGZisLS0REZGBkxMTHDz5k2sWrVKLkZjY2P4+vo2O46amhrEYnGzY5WXl//j+AkhhLSMlikSQggRlLKyMvz666+YMWMGhg8fzizf++mnnwCA6ew3Z84cBAYGAgB69OgBOzs7ODg44M8//0RVVRXz/TQ1NdGxY0esXr0a165dw549ewAAly9fhpmZGfLz8yESiTBkyBD4+Phg0KBBuHfv3t+K2djYGPfv30ePHj0wfPhw5p/s7GxERkY2eav1d793XV0djh49Knc9NTUVADB69Gjm2rRp03DlyhUcOHAAffr0gbGxsdz3+fXXX9GvXz+5GA8dOoQffvgBysrKMDU1RV1dHY4fP878udraWrnklhBCSNuhN2OEEEIEpUePHtDW1kZsbCy0tLTQrVs3nDlzBtHR0QBeLqcDgDFjxiAqKgqampowNDREaWkp9uzZA2NjY2hoaODZs2dy39fCwgJTpkxBWFgYPv74YwwdOhQdO3aEr68vFi5cCE1NTZw7dw5FRUVwdnb+WzHb2dlh//79cHNzw4IFC9C7d2+cO3cOERERcHR0hKqq6j/+eYwfPx4fffQRVq5cidLSUujr6+PChQuIiIjA9OnTMWDAAObZcePGQV1dHfHx8Zg/f77cckNXV1ccOnQIrq6u+OyzzyAWi5Geno6EhAT4+fkBeNmsY+zYsVi5ciUeP34MbW1tREdH48mTJ+jRo8c//m8ghBDSPErGCCGECM727dsRFBSEZcuWQU1NDQMGDMCOHTuwbt06XLp0CU5OTvD29oaamhoSExOxbds2Zpnev//97xa/7/Lly3H27FmsWrUKe/fuRVRUFDZv3oygoCD8+eefkEgk+Prrr2FnZ/e34u3UqRNiY2OxefNmBAcHo7KyEtra2vj3v/+Nzz777J1+FiKRCLt27UJoaCj27t2LJ0+e4F//+hcWL14MNzc3uWdVVFQglUoRExPDNPho1KtXL8TFxWHz5s3w9/dHTU0NJBIJgoKCMGPGDOa5rVu3YtOmTQgNDUVNTQ2sra0xa9YsnDhx4p3+OwghhDQletG4E5oQQgghhBBCCGdozxghhBBCCCGE8ICSMUIIIYQQQgjhASVjhBBCCCGEEMIDSsYIIYQQQgghhAeUjBFCCCGEEEIIDygZI4QQQgghhBAeUDJGCCGEEEIIITygZIwQQgghhBBCeEDJGCGEEEIIIYTwgJIxQgghhBBCCOEBJWOEEEIIIYQQwgNKxgghhBBCCCGEB/8POk3ISHSuiyYAAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -1256,15 +1460,6 @@ "plt.show()" ] }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO: add the tasks in tasks_not_in_index back in" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -1274,7 +1469,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -1286,7 +1481,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -1320,26 +1515,26 @@ " \n", " \n", " 0\n", - " GritLM/GritLM-7B\n", + " intfloat/e5-mistral-7b-instruct\n", " 0.63\n", " 1.0\n", - " 0.69\n", - " 2.0\n", + " 0.67\n", + " 1.0\n", " \n", " \n", " 1\n", - " intfloat/e5-mistral-7b-instruct\n", + " GritLM/GritLM-7B\n", " 0.63\n", " 2.0\n", - " 0.69\n", - " 1.0\n", + " 0.66\n", + " 2.0\n", " \n", " \n", " 2\n", " intfloat/multilingual-e5-large-instruct\n", " 0.61\n", " 3.0\n", - " 0.68\n", + " 0.65\n", " 3.0\n", " \n", " \n", @@ -1347,7 +1542,7 @@ " intfloat/multilingual-e5-large\n", " 0.57\n", " 4.0\n", - " 0.64\n", + " 0.62\n", " 4.0\n", " \n", " \n", @@ -1355,31 +1550,31 @@ " intfloat/multilingual-e5-base\n", " 0.56\n", " 5.0\n", - " 0.63\n", + " 0.60\n", " 5.0\n", " \n", " \n", " 5\n", - " sentence-transformers/all-mpnet-base-v2\n", + " intfloat/multilingual-e5-small\n", " 0.54\n", " 6.0\n", - " 0.60\n", - " 8.0\n", + " 0.58\n", + " 6.0\n", " \n", " \n", " 6\n", - " intfloat/multilingual-e5-small\n", - " 0.54\n", + " sentence-transformers/all-mpnet-base-v2\n", + " 0.53\n", " 7.0\n", - " 0.61\n", - " 6.0\n", + " 0.56\n", + " 8.0\n", " \n", " \n", " 7\n", " sentence-transformers/paraphrase-multilingual-...\n", - " 0.53\n", + " 0.52\n", " 8.0\n", - " 0.60\n", + " 0.57\n", " 7.0\n", " \n", " \n", @@ -1387,31 +1582,31 @@ " sentence-transformers/all-MiniLM-L12-v2\n", " 0.52\n", " 9.0\n", - " 0.58\n", + " 0.55\n", " 10.0\n", " \n", " \n", " 9\n", " sentence-transformers/all-MiniLM-L6-v2\n", - " 0.52\n", + " 0.51\n", " 10.0\n", - " 0.58\n", + " 0.54\n", " 11.0\n", " \n", " \n", " 10\n", " sentence-transformers/paraphrase-multilingual-...\n", - " 0.51\n", + " 0.50\n", " 11.0\n", - " 0.58\n", + " 0.55\n", " 9.0\n", " \n", " \n", " 11\n", " sentence-transformers/LaBSE\n", - " 0.44\n", + " 0.43\n", " 12.0\n", - " 0.53\n", + " 0.49\n", " 12.0\n", " \n", " \n", @@ -1420,35 +1615,35 @@ ], "text/plain": [ " model Average_v2_full \\\n", - "0 GritLM/GritLM-7B 0.63 \n", - "1 intfloat/e5-mistral-7b-instruct 0.63 \n", + "0 intfloat/e5-mistral-7b-instruct 0.63 \n", + "1 GritLM/GritLM-7B 0.63 \n", "2 intfloat/multilingual-e5-large-instruct 0.61 \n", "3 intfloat/multilingual-e5-large 0.57 \n", "4 intfloat/multilingual-e5-base 0.56 \n", - "5 sentence-transformers/all-mpnet-base-v2 0.54 \n", - "6 intfloat/multilingual-e5-small 0.54 \n", - "7 sentence-transformers/paraphrase-multilingual-... 0.53 \n", + "5 intfloat/multilingual-e5-small 0.54 \n", + "6 sentence-transformers/all-mpnet-base-v2 0.53 \n", + "7 sentence-transformers/paraphrase-multilingual-... 0.52 \n", "8 sentence-transformers/all-MiniLM-L12-v2 0.52 \n", - "9 sentence-transformers/all-MiniLM-L6-v2 0.52 \n", - "10 sentence-transformers/paraphrase-multilingual-... 0.51 \n", - "11 sentence-transformers/LaBSE 0.44 \n", + "9 sentence-transformers/all-MiniLM-L6-v2 0.51 \n", + "10 sentence-transformers/paraphrase-multilingual-... 0.50 \n", + "11 sentence-transformers/LaBSE 0.43 \n", "\n", " Rank_v2_full Average_v2_lite Rank_v2_lite \n", - "0 1.0 0.69 2.0 \n", - "1 2.0 0.69 1.0 \n", - "2 3.0 0.68 3.0 \n", - "3 4.0 0.64 4.0 \n", - "4 5.0 0.63 5.0 \n", - "5 6.0 0.60 8.0 \n", - "6 7.0 0.61 6.0 \n", - "7 8.0 0.60 7.0 \n", - "8 9.0 0.58 10.0 \n", - "9 10.0 0.58 11.0 \n", - "10 11.0 0.58 9.0 \n", - "11 12.0 0.53 12.0 " + "0 1.0 0.67 1.0 \n", + "1 2.0 0.66 2.0 \n", + "2 3.0 0.65 3.0 \n", + "3 4.0 0.62 4.0 \n", + "4 5.0 0.60 5.0 \n", + "5 6.0 0.58 6.0 \n", + "6 7.0 0.56 8.0 \n", + "7 8.0 0.57 7.0 \n", + "8 9.0 0.55 10.0 \n", + "9 10.0 0.54 11.0 \n", + "10 11.0 0.55 9.0 \n", + "11 12.0 0.49 12.0 " ] }, - "execution_count": 50, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -1469,12 +1664,12 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 22, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABAQAAANrCAYAAADRYkruAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd1RUx9sH8O/uwtKrBRTsKNJBERSsxELUxJ6oiS32HmMj0VhjYqKxgWAjGhOTn7FEk9iiMXbFGksECwiiUUTpZXfZ3fv+wcvVpQlKUfl+zvHozt6ZO3d24XifO/OMRBAEAURERERERERUpUgruwNEREREREREVPEYECAiIiIiIiKqghgQICIiIiIiIqqCGBAgIiIiIiIiqoIYECAiIiIiIiKqghgQICIiIiIiIqqCGBAgIiIiIiIiqoIYECAiIiIiIiKqghgQICKiKkEQhMruAhEREdErhQEBIqJnDBo0CI6Ojjp/XF1d0b59e8yfPx+pqamV3cUqbefOnXB0dMS9e/dKXCctLQ0zZszA+fPnxbJBgwZh0KBB5dHFElGr1VixYgXatWsHDw8PDBw4EJcvX35uvejoaIwZMwZeXl7w8fHBpEmTEBsbW6DtdevWoXPnzvD09ESPHj2wd+/ecrqS8hMREQFHR0dERERUWh+uXr2KQYMGwcvLC61bt8ayZcugUqmeW+/EiRPo06cPPDw8EBAQgPDw8AIBqbi4OIwZMwbe3t7w9fXF3LlzkZGRUV6XQkREVCi9yu4AEdGrxtnZGXPnzhVf5+Tk4N9//8WyZcsQGRmJn3/+GRKJpBJ7SKURGRmJ3bt3o0+fPmLZs59vZVi8eDG2b9+OqVOnws7ODhs3bsTQoUOxa9cu1KtXr9A68fHxGDBgAMzMzDBnzhxUq1YN27dvx/vvv48dO3bA3t4eABAcHIx169Zh/PjxaN68OQ4ePIgpU6ZAJpOhS5cuFXmZr7X4+HgMGzYMnp6eWLFiBaKjo7F8+XKkpKRgwYIFRdb7559/MGbMGLz99tuYPHkyLly4gCVLlkCj0WDUqFEAcoNUQ4YMQfXq1bF48WIkJSVhyZIluHfvHsLDwyvqEomIiBgQICLKz9TUFJ6enjplLVq0QGZmJlatWoXLly8XeJ9eLw4ODpV27gcPHuDnn3/GrFmzMHDgQABA69at0aVLF6xfvx5ffPFFofW+//57ZGdnY8eOHahTp45Y77333sPy5cvx7bffAgB27NiB7t27Y8KECQCAVq1a4d9//8WPP/7IgEAprF+/HiYmJggNDYVcLke7du1gaGiIhQsXYsyYMahdu3ah9YKDg+Hk5IQlS5YAANq2bQu1Wo01a9Zg8ODBMDQ0xM8//4yUlBTs3LkT1tbWAAAbGxuMGjUKFy5cQPPmzSvsOomIqGrjkgEiohJydXUFAPz3339i2aFDh9C7d2+4ubnB398fX3zxBbKysnTqHTp0CAMHDoSXlxdcXV0RGBiILVu2iO/nTY3+3//+hw4dOqBZs2Y4efIkkpKSMHXqVPj7+8PNzQ09evTArl27dNqOjY3FpEmT4O/vD09PTwwaNAgXLlwQ37937x4cHR2xb98+TJo0SZxqPnv27AL9zO/Ro0eYOXMmWrVqBS8vL3z44Ye4dOmS+L5SqcTq1asRGBgINzc3dO7cGevWrYNWqxWPGTRoEKZNm4ZJkybB09MTw4YNE/u0ceNGBAYGwsPDAzt27AAA3Lx5E6NHj0azZs3QrFkzjB8/HvHx8cX2c9u2bejduzc8PT3h7u6OHj16YN++feLYDh48GAAwePBgcZlA/iUDJb2WWbNmYd26dWjfvj3c3NzQv39/XLlypcB4BwcHF9nf06dPQ61Wo1OnTmKZXC5H+/btcfTo0SLrxcTEwMHBQQwGAIBUKkWLFi106qlUKpiamurUtbS0REpKSpFtFyY4OBgBAQH4+++/xc/pvffee+4U/jVr1sDV1bXA8ppNmzbBxcUFT548AQCcO3cOw4cPR4sWLeDq6oqAgAAEBwfrjPmzgoKCEBAQoFOWN947d+4Uy1JSUjBnzhz4+fnBzc0N7733Hk6fPl2gLUdHx2Kv48SJE2jXrh3kcrlYFhgYCK1WixMnThRaR6VSISIiQuezBYAuXbogMzNT/Nk8ceIEmjdvLgYDgNzgjomJCY4dO1Zsv4iIiMoSAwJERCV0584dABBvyH7//XeMHz8eDRs2xOrVqzFhwgT89ttvGDdunLhe+MiRIxg/fjxcXFwQGhqK4OBg1KlTBwsWLCiwZjwkJAQzZ87EnDlz4OXlhenTpyM6Ohrz58/H+vXr4ezsjJkzZ+LMmTMAgNu3b6N37964d+8eZs+ejaVLl0IikWDIkCE4e/asTttz586FnZ0dQkNDMXz4cGzfvh1hYWFFXmtmZiYGDBiAiIgITJ8+HSEhITAwMMBHH32E2NhYCIKAMWPGYMOGDejXrx/WrFmDwMBArFixosB0/H379sHExARhYWEYMWKEWB4cHIyRI0fim2++gb+/P+7cuYP+/fvjyZMn+Prrr7Fo0SJxmnzeTWR+W7ZswZw5c9CxY0esXbsWS5cuhVwux7Rp0/Dw4UO4uLhgzpw5AIA5c+YUulSgNNdy4MAB/PXXX5g9ezaWLVuGx48fY+LEidBoNACAmjVrYuvWrejXr1+RYxsdHQ0TExPUqFFDp7xevXp49OgRMjMzC61nZWWFxMRE5OTk6JTHx8cjPT1dvOEfPHgwdu3ahWPHjiEjIwO//fYbjh8/jh49ehTZp6IkJSVh5syZGDhwIFauXAlDQ0MMHz4ckZGRRdZ55513oFar8eeff+qU79mzB61bt0a1atUQFRWFoUOHwtLSEsuXL0dYWBi8vb0REhIiBnNehFKpxJAhQ/DXX39hypQpCAkJga2tLUaMGKETFBg3bhy2bt1aZDsKhQL3799HgwYNdMqtra1hamoq/i7ILz4+Hjk5Oahfv75Oed4ykLx60dHRBdqWyWSwt7cvsm0iIqLywCUDRET5CIIAtVotvk5NTcXZs2cRFhYmPuUXBAFLly5FmzZtsHTpUvHY+vXrY+jQoTh69Cjat2+P27dvo1evXpg1a5Z4jJeXF3x9fREREQEPDw+xfODAgQgMDBRfnz17FuPHj0fHjh0BAD4+PrC0tBSfWIaEhEAul2Pz5s3iE+H27duje/fu+Oabb7B9+3axrXbt2mHmzJkAcqeQnzx5EkeOHMHUqVMLHYNff/0V9+/fx6+//gonJycAQLNmzdCzZ0+cO3cOcXFxOHXqFJYtW4Zu3boBAPz9/WFoaIiVK1di8ODBaNy4MQBAX18f8+fPF/udlxDw7bff1lnXP3XqVBgZGWHTpk3i9bRq1QodO3bEhg0bxP4/Kz4+HsOHD8e4cePEMjs7O/Tu3RsXLlxAt27dxOUBDg4OhS4VOHbsWImvRa1WIzw8XOxfZmYmZs6cicjISLi6ukIulz93OUl6enqBJ/gAYGJiAgDIyMgQ//2s3r17448//sDMmTMxZcoUmJqaYvfu3Th+/DgAIDs7G5aWlhg6dCj++ecfjBw5Uqzbp08fnWBMSWVnZ2PevHno2bMnAKBly5bo2LEj1q1bh+XLlxdax87ODi1atMAff/whBkbu3r2LK1euiHWioqLg5+eHJUuWQCrNfTbh7++Pw4cPIyIiQvwcSmv37t2IiorCL7/8Iv5stW3bFoMGDcLSpUvFmSh169ZF3bp1i2wnPT0dAIr8nIpK/ldUvWc/27zjCvuMi2ubiIioPDAgQESUz7lz5+Di4qJTJpVK4efnhwULFkAikSA6OhoPHz7E6NGjdYIHLVq0gKmpKU6ePIn27duLN2GZmZm4c+cO7t69i6tXrwJAgWzleTfeeXx9fREcHIzr16+jTZs2Ojf1QG7AoEOHDjo3H3p6eujWrRtWr16t86Q5/02qra0t7t+/X+QYXLhwAfb29jp9MjIywoEDBwAAS5YsgZ6enk4AAwDeffddrFy5EmfPnhVvohs2bKgz7bqo6z1z5gx8fHxgaGgojqmpqSm8vb1x6tSpQvsZFBQEIDdJW0xMDOLi4sQp7SXJBg/kjmNJr8XBwUFnvG1sbADk3jiX1PO2P8y7Qc7P398fS5YswZdffok9e/YAAPz8/DBy5EgEBwfD0NAQKpUKH3zwARITEzF//nw0bNgQly5dQlhYGIyNjTF79uwS9xPI/T51795dfG1oaIi2bduK09o1Go3O9UilUkilUrz77ruYO3cuEhMTUaNGDezZswempqbilP+ePXuiZ8+eUCqVuHPnDuLi4hAZGQmNRlNgBkRpnD59GjVq1ICLi4vOz2WHDh3wzTffIDU1FRYWFs9tp6hlC3mKSir6vHp5n21x3wEmLCUioorEgAARUT4uLi6YP38+gNz/nBsYGKBWrVo6N4J507Pnz58vHvusR48eAcidcj137lwcOnQIEokE9erVg7e3N4CCNwXGxsY6r5cvX441a9Zg3759OHDggE5Qws7ODqmpqahevXqBc1evXh2CIOg8aTQyMtI5RiqVFntTkpKSgmrVqhX5fmpqKqysrCCTyXTK86bB5z0pBVDok1Cg4PWmpKRg7969hW6R9+xa62fdvXsXc+bMwenTp6Gvr4+GDRuiadOmAJ5/4/0i11LYOALPvxF8lqmpaaHLAvI+LzMzsyLrvvvuu+jWrRvi4+NhZGQEGxsbrFy5ElKpFObm5ti7dy+ioqKwceNG+Pn5AcidWWJqaooFCxbgvffeQ5MmTUrc1+rVq0NPT/e/CtWqVRO//506ddIJLPXq1QuLFy9GYGAgFi5ciH379mHw4MHYs2cPunTpAkNDQwC5U/IXLlyI3bt3Q61Ww97eHl5eXtDT0yvx51aYlJQUJCYmFgjo5UlMTCxRQODZGSD5ZWRkFPkZ5ZXnr5f32ea1W9x3IC/IREREVBEYECAiysfExARubm7FHmNubg4AmDFjBnx8fAq8n3fTMW3aNMTExGDTpk3w8vKCXC5HdnY2fvnll+f2w8zMDNOnT8f06dMRExODv/76C6GhoZg/fz7WrVsHCwsLPH78uEC9xMREALlrzvMCE6VlZmYmTu1/1sWLF2FhYQELCwskJydDo9Ho3Ejnnc/KyuqFzunn54dhw4YVeC//TSmQexM+atQo6OvrY/v27XBycoKenh5u376N3bt3l/i85XEtxWnYsCEyMjKQlJSkE+iIi4uDnZ2deNOcX3R0NK5evYqePXvqrFG/fv06HB0dIZPJxISXzZo106nbokULALl5J0oTECgsEeHjx4/FYFFYWJjOTIy8sTIzM0NAQAD27duHli1b4tatW/j888/F4xYtWoQDBw5gxYoV8PPzE4NDrVq1KrIvEolEzNWQJ39iTDMzM9SvX19nGc+z8rZmfB4TExPY2NggLi5Op/zJkyfIzMxEo0aNCq1Xt25dyGSyAvXu3r0LAGK9Bg0aiGV5NBoN7t27h86dO5eoj0RERGWBSQWJiF5Aw4YNUa1aNdy7dw9ubm7iHxsbG3z77be4fv06gNyp9507d4avr684bT5vunVxT5Xv37+Pdu3aYf/+/eL5Ro4cCT8/P/Gmr0WLFvj77791ZgJoNBrs2bMHbm5uhU7TLylvb2/Ex8fj1q1bYplSqcTEiROxfft2+Pj4QK1Wi/3L89tvvwHAC22b5uPjg9u3b8PJyUkcT1dXV2zatAkHDx4scHxycjLu3LmDvn37ws3NTQwa5B/f/E/+CztvWV9LcfKe3D97PpVKhSNHjsDf37/Ierdu3cLMmTMRExMjlt2+fRsnTpwQ80w0bNgQAHD+/HmduhcvXgRQ8hviPAqFQsxRkPf62LFj4o27o6Ojzvf/2fZ79OiBf/75Bz///DNq166tEzi7cOECfH190bFjRzEYcO3aNSQlJRX5c2FiYoLk5GQolUqddp7l4+ODBw8eoFq1ajr9OnnyJDZs2PDc78Kz/P39ceTIEZ2Ax4EDByCTydCyZctC6xgYGMDb2xsHDx7Umelw4MABmJmZwd3dXWz73LlzSEpKEo85ceIEsrKyiv0OEBERlTXOECAiegEymQxTpkzBnDlzIJPJ0KFDB6SlpSE0NBQJCQnilGV3d3f8/vvvcHFxga2tLS5evIh169ZBIpEUu+7czs4Otra2+OKLL5CRkYG6devi2rVrOHr0KEaPHg0AmDBhAo4dO4bBgweLT8p//PFHxMfHY8OGDS91fb1798YPP/yAsWPHYtKkSbCyssLmzZuRk5ODgQMHwt7eHr6+vpg9ezYSEhLQtGlTnD17FuvXr0evXr0KTd73POPGjUP//v0xevRoDBgwAAYGBti6dSsOHTqEVatWFTi+WrVqsLOzw5YtW2Brawtzc3McP34cmzdvBvB0XX/eNO4jR47AwsJCXFKQp23btmV2LSqVCtevX4etrS1sbW0LPcbOzg69evXCV199BaVSifr162Pjxo1IS0vTSfx39+5dJCUlifkf2rVrh7p162LatGmYPHkyMjIy8M0338De3h5Dhw4FAAQEBMDDwwPTp0/HxIkT0bBhQ1y5cgVhYWEICAgQb0iTkpJw9+7dAjkRCvPpp5/i448/RrVq1RAeHo6srCyMHTv2uWPRpk0bWFpaYuvWrRgxYoTO2nh3d3fs27cPP//8Mxo1aoSoqCiEhYUV+3PRoUMH/PDDD5g1axb69u2LmzdvYuPGjTo3+b1798aPP/6IYcOGYcyYMahVqxZOnTqF9evX48MPP4S+vn6hY1uYESNGYM+ePRgxYgSGDRuG2NhYLFu2DO+99x5q164NoPDPe+zYsRg2bBgmT56MPn364NKlSwgPDxeTZgK5CUTz+jlhwgSkpKRgyZIlaNu2bYHZHUREROVKICIi0Ycffih8+OGHJT5+z549Qq9evQRXV1fBx8dHGDNmjBAVFSW+f+/ePWH06NFC8+bNhebNmwt9+vQRdu/eLQwfPlzo06ePIAiCcObMGaFJkybCmTNndNp+9OiREBQUJLRu3VpwcXEROnbsKISFhQkajUY85vr168KIESMET09PwcvLSxgyZIhw7tw58f34+HihSZMmwo4dO3TanjlzptChQ4dir+3hw4fCJ598Inh7ewvNmjUTPvroIyEyMlJ8PysrS1i8eLHQpk0bwcXFRejSpYuwYcMGnf4VNp5F9UkQBOHatWvC8OHDBS8vL8HT01N47733hEOHDonv79ixQ2jSpIkQHx8vCIIgREZGCh9++KHg6ekp+Pj4CAMHDhSOHTsmBAYGCpMmTRIEQRA0Go3wySefCG5ubkK3bt0K7deLXkv+zy7v2latWlXs2CqVSmHRokVCq1atBA8PD2HgwIHCP//8o3PMzJkzhSZNmuiUxcbGCqNGjRKaN28u+Pn5CUFBQUJCQoLOMenp6cKCBQsEf39/wdXVVXj77beFtWvXCkqlssA45v/OPWvVqlVCkyZNhIMHDwodOnQQPDw8hGHDhul8B55n4cKFQpMmTYRbt27plCcnJwuffPKJ4OPjI3h6egrdu3cXvv/+e+Hzzz8X/P39BbVaXejPRXh4uNC+fXvB1dVVeP/994Vr164Jrq6uOt+lx48fC59++qnQqlUrwdXVVejSpYuwfv16nc+ysLEtzLlz54R+/foJrq6uQps2bYSlS5cKKpVKfL+oz/vPP/8UunfvLri4uAgBAQFCeHh4gbZv3LghDBkyRHB3dxdatWolfP7550J6evrzB5WIiKgMSQThJbL3EBER0WtpxowZGDhwYJFPyYODgxESEoIbN25UbMeIiIiowjCHABERURVz+/ZtXL58uVQJBomIiOjNw4AAERFRFWNtbY1NmzYV2PqRiIiIqhYuGSAiIiIiIiKqgjhDgIiIiIiIiKgKYkCAiIgIACfMERERUVXDgAARVUk3b97ElClT4O/vD1dXV7Ru3Roff/wxoqKiKrtr9BJ27twJR0dH3Lt3r8R10tLSMGPGDJw/f14sGzRoEAYNGlQeXSwRtVqNFStWoF27dvDw8MDAgQNx+fLl59aLjo7GmDFj4OXlBR8fH0yaNAmxsbE6x8TFxcHR0bHAn+7du5fT1ZSPF/msy9qJEyfQp08feHh4ICAgAOHh4SUKLP3xxx/o1q0b3N3d8fbbb+PXX38tcMzVq1cxaNAgeHl5oXXr1li2bBlUKlV5XAYREVVhepXdASKiinbr1i28//778PT0xOzZs1GtWjU8fPgQP/74I9577z1s3ry5yK3Y6M0TGRmJ3bt3o0+fPmLZ3LlzK7FHwOLFi7F9+3ZMnToVdnZ22LhxI4YOHYpdu3ahXr16hdaJj4/HgAEDYGZmhjlz5qBatWrYvn073n//fezYsQP29vYAcq8XADZt2gQjIyOxvqGhYflf2Bvkn3/+wZgxY/D2229j8uTJuHDhApYsWQKNRoNRo0YVWe/AgQOYNm0aBg8ejDZt2uDQoUMICgqCXC5Ht27dAOR+lsOGDYOnpydWrFiB6OhoLF++HCkpKViwYEFFXSIREVUBDAgQUZWzceNGWFlZYf369dDTe/prsGPHjggMDERoaCjWrVtXiT2kyubg4FBp537w4AF+/vlnzJo1CwMHDgQAtG7dGl26dMH69evxxRdfFFrv+++/R3Z2Nnbs2IE6deqI9d577z0sX74c3377LYDcgICtrS1atWpVMRf0hgoODoaTkxOWLFkCAGjbti3UajXWrFmDwYMHFxlgWbZsGQIDA/HZZ58BANq0aYPU1FSsXLlSDAisX78eJiYmCA0NhVwuR7t27WBoaIiFCxdizJgxqF27dsVcJBERvfG4ZICIqpzHjx9DEARotVqdcmNjY3z22Wd4++23xbJBgwYhKCgIa9asgZ+fH5o3b45x48bh/v37OnVv3ryJ0aNHo1mzZmjWrBnGjx+P+Ph4nWOioqIwYcIEtGzZEi4uLmjTpg2++OILKBQK8RhHR0eEhISgd+/ecHd3R0hICHbu3Ak3NzecP38effr0gZubG7p06YLDhw8jJiYGQ4YMgYeHBzp16oQ9e/bonPPcuXMYPnw4WrRoAVdXVwQEBCA4OFi89nv37sHR0RH79u3DpEmTxKnms2fPRlZWVrHj+OjRI8ycOROtWrWCl5cXPvzwQ1y6dEl8X6lUYvXq1QgMDISbmxs6d+6MdevW6Yz7oEGDMG3aNEyaNAmenp4YNmyY2KeNGzciMDAQHh4e2LFjR4nHOb9t27ahd+/e8PT0hLu7O3r06IF9+/YBACIiIjB48GAAwODBg8VlAvmXDJT0WmbNmoV169ahffv2cHNzQ//+/XHlyhXxmLxrCw4OLrK/p0+fhlqtRqdOncQyuVyO9u3b4+jRo0XWi4mJgYODgxgMAACpVIoWLVro1IuKioKTk1OxY1YSQUFBGDRoELZv344OHTrAy8sLQ4YMee6ym88//xz+/v7QaDQ65YsWLYKvry9ycnIAAIcOHcLAgQPh5eUFV1dXBAYGYsuWLUW2W9gyj4iICDg6OiIiIkIs+++///DJJ5/Ax8cHHh4eGDJkCK5fv16grYCAgCLPpVKpEBERofMZAUCXLl2QmZmJCxcuFFrv3r17iI2NLbReXFycuLzjxIkTaNeuHeRyuXhMYGAgtFotTpw4UWS/iIiISosBASKqctq3b4///vsP/fv3x5YtWxAdHS2u+w0MDESvXr10jv/rr7+wc+dOzJ49G/Pnz0dkZCQGDRqE7OxsAMCdO3fQv39/PHnyBF9//TUWLVokTt9+8uQJgNyb5w8++ADZ2dlYvHgx1q9fj27duuGHH37A5s2bdc63Zs0avPPOO1i1ahW6dOkCIHdN+dSpU9G/f3+EhYXByMgI06ZNw5gxY9C+fXusWbMGNWvWxMyZM/Hw4UMAuTd+Q4cOhaWlJZYvX46wsDB4e3sjJCREvCHOM3fuXNjZ2SE0NBTDhw/H9u3bERYWVuQYZmZmYsCAAYiIiMD06dMREhICAwMDfPTRR4iNjYUgCBgzZgw2bNiAfv36Yc2aNQgMDMSKFSsKTMfft28fTExMEBYWhhEjRojlwcHBGDlyJL755hv4+/uXaJzz27JlC+bMmYOOHTti7dq1WLp0KeRyOaZNm4aHDx/CxcUFc+bMAQDMmTOn0KUCpbmWAwcO4K+//sLs2bOxbNkyPH78GBMnThRvfmvWrImtW7eiX79+RY5tdHQ0TExMUKNGDZ3yevXq4dGjR8jMzCy0npWVFRITE8Ub6jzx8fFIT09HSkoKgNwZApmZmejfvz/c3Nzg7++PpUuXFqhXEpGRkVi+fDkmTJiAJUuWIDk5GR9++CEePXpUZJ0ePXrg8ePHOjfpWq0W+/btQ7du3aCvr48jR45g/PjxcHFxQWhoKIKDg1GnTh0sWLCgRLkUipKUlIT+/fvj33//xeeff45vv/0WWq0WH3zwAaKjo8Xj5s6di5CQkCLbiY+PR05ODurXr69Tnrec486dO4XWyztHcfUUCgXu37+PBg0a6BxjbW0NU1PTItsmIiJ6EVwyQERVzsCBA5GYmIjw8HBxPa6VlRVat26NwYMHw93dXef47Oxs7Ny5U3zy2rBhQ/Tq1Qu7du3CgAEDEBISAiMjI2zatAmmpqYAgFatWqFjx47YsGEDZs6ciZs3b8LJyQkrV64Uj/Hz88PJkycRERGhs+bY29sbw4YNE19fvXoVWq0WY8aMEW8k09LSMGXKFAwZMkQ81szMDH369MG1a9dga2uLqKgo+Pn5YcmSJZBKc+O//v7+OHz4MCIiIsTpyQDQrl07zJw5U+z7yZMnceTIEUydOrXQMfz1119x//59/Prrr+LT5mbNmqFnz544d+4c4uLicOrUKSxbtkw8j7+/PwwNDbFy5UoMHjwYjRs3BgDo6+tj/vz54tPQvCRxb7/9ts66/qlTpz53nPOLj4/H8OHDMW7cOLHMzs4OvXv3xoULF9CtWzdxeYCDg0OhSwWOHTtW4mtRq9UIDw8X+5eZmYmZM2ciMjISrq6ukMvlz81PkZ6eLtZ/lomJCQAgIyND/PezevfujT/++AMzZ87ElClTYGpqit27d+P48eMAcr/HWq0WCQkJ0Gg0mD59OmrXro3Tp09j/fr1ePDggbisoKTS09OxZs0aeHt7AwDc3d3RsWNHbN68GdOmTSu0TvPmzWFnZ4c//vgDfn5+AHKf5CcmJqJHjx4AgNu3b6NXr16YNWuWWM/Lywu+vr6IiIiAh4dHqfqZ5/vvv0dKSgp+/vln2NnZAcid6t+1a1esXLkSq1atAvD8JSPp6ekAUOBzevYzKkxeeXH1imo777ii2iYiInoRDAgQUZU0efJkDB06FMePH8fp06cRERGB33//HX/88Qc+++wzcRo5kHuj++w0bGdnZ9SpUwfnzp3DgAEDcObMGfj4+MDQ0BBqtRpA7n/mvb29cerUKQC5a7lbt26NnJwc3L59G3Fxcbh58yaSkpJgaWmp07eipnN7eXmJ/65WrRoA6NwY5bWTlpYGAOjZsyd69uwJpVKJO3fuIC4uDpGRkdBoNAWeBue/SbW1tS2wLOJZFy5cgL29vU5fjYyMcODAAQDAkiVLoKenh8DAQJ167777LlauXImzZ8+KN9ENGzbUmRpd1DiUZJzzCwoKEsckJiYGcXFx4pPpkmZsP3v2bImvxcHBQedGzsbGBgDE2SQl8bws9XnBnfz8/f2xZMkSfPnll+LSET8/P4wcORLBwcEwNDSEkZERvvvuO9SrV09MMujj4wO5XI4VK1Zg3LhxaNSoUYn7am9vLwYDgNwZEF5eXjh37hwAiJ/Ts32XSqV499138dNPP2HevHmQy+XYs2cP6tevL36f82aKZGZm4s6dO7h79y6uXr0KoOSfW2FOnz4NJycn2NjYiH2TSqVo27YtfvvttxK3k3+5UX5FfUYlqfe8YyQSSfGdIyIiKgUGBIioyrKwsED37t3F7dauX7+O6dOnY8mSJXjnnXdgZWUF4OlN3bOqVauG1NRUAEBKSgr27t2LvXv3FjjO2toaQO6NwLJly7BlyxZkZWWhVq1acHd3h4GBQYE6xsbGhfa3sCeGz2aJz0+hUGDhwoXYvXs31Go17O3t4eXlBT09vQI3nfnbkUqlxd6YpqSkiEGJwqSmpsLKygoymUynPG8afN5TUACFPu0GCo5DScY5v7t372LOnDk4ffo09PX10bBhQzRt2hTA82+8X+RaChtH4Pk3gs8yNTUtdFlA3pNhMzOzIuu+++676NatG+Lj42FkZAQbGxusXLkSUqkU5ubmkMlk8Pf3L1Cvffv2WLFiBaKiokoVECjqZ+Pff/8FALi4uOi8N2HCBEycOBE9evRAWFgYjh8/jjZt2uDPP//EkCFDxOOSkpIwd+5cHDp0CBKJBPXq1RMDDyX93AqTkpKCuLi4Av3Kk52dXezPVJ68zyD/51TUDIDS1Ht2dkl+GRkZxX7+REREpcWAABFVKQkJCejTpw8mT55cYB23s7MzpkyZIiaqywsIJCcnF2jn8ePHqFu3LoDc/+T7+fnpTPPPk7eLwbp167Bp0ybMnz8fnTt3Fv9T37dv3zK9vmctWrQIBw4cwIoVK+Dn5yfeYJdFdnkzM7NC93+/ePEiLCwsYGFhgeTkZGg0Gp0b6by15XljW9pzPm+cn6XVajFq1Cjo6+tj+/btcHJygp6eHm7fvo3du3eX+LzlcS3FadiwITIyMpCUlKQT6IiLi4OdnV2R2eujo6Nx9epV9OzZU2eN+vXr1+Ho6AiZTIbY2FicOXMGXbt2hbm5uXhMXmLLogIrRSnqZyMvWLR9+3ad92rWrAkAaNCgAdzd3bFv3z5IpVKkpaXh3XffFY+bNm0aYmJisGnTJnh5eUEulyM7Oxu//PJLsf3Jn6gwf2JMMzMz+Pj4YMaMGYXWL2ymSmHq1q0LmUyGuLg4nfK7d+8CQJFBlby8AHFxcXB2dhbL89pp1KgRTExMYGNjU6DtJ0+eIDMzs1QBGyIioudhUkEiqlKqV68OPT09/PTTT1AqlQXej4mJgYGBgc5e7xcuXNC58bl27Rru3bsn3lj7+Pjg9u3bcHJygpubG9zc3ODq6opNmzbh4MGDYhsODg7o06ePGAxISEjAzZs3S/X0uDQuXLgAX19fdOzYUQwGXLt2DUlJSS99Tm9vb8THx+PWrVtimVKpxMSJE7F9+3b4+PhArVZj//79OvXypmU3b9681OcsyTg/Kzk5GXfu3EHfvn3h5uYmBg2OHTsG4OlT+/xP/gs7b1lfS3Hy1tU/ez6VSoUjR44U+nQ/z61btzBz5kzExMSIZbdv38aJEyfQsWNHAEBiYiLmzp1b4Fr27t0LU1PTIp+cFyU2NlYnGV9CQgIuXbok/mzkfU55f56dUdCjRw8cP34ce/bsKbAs58KFC+jcuTN8fX3Fm/T8n1t+pqamYkLNZ9t5lo+PD+7cuYMGDRro9Gv37t3Yvn37c78LeQwMDODt7Y2DBw/qzFg4cOAAzMzMCuQhyZO3VCNvaU2eP//8E/Xr1xeXcfj7++PIkSM6yyMOHDgAmUyGli1blqiPREREJcEZAkRUpchkMsybNw/jx49Hnz598MEHH6BRo0bIzs7GyZMnsWXLFkyePBkWFhZinezsbIwYMQJjx45FZmYmli9fjiZNmohLDcaNG4f+/ftj9OjRGDBgAAwMDLB161YcOnRITFLm7u6O0NBQrFu3Dp6enoiLi8PatWuhUqlKtb68NPKewP78889o1KgRoqKiEBYWBolE8tLn7N27N3744QeMHTsWkyZNgpWVFTZv3oycnBwMHDgQ9vb28PX1xezZs5GQkICmTZvi7NmzWL9+PXr16vXcpG2FKck4P6tatWqws7PDli1bYGtrC3Nzcxw/flzc1SFvDPICNEeOHIGFhYW4pCBP27Zty+xaVCoVrl+/DltbW9ja2hZ6jJ2dHXr16oWvvvoKSqUS9evXx8aNG5GWlqazC8Pdu3eRlJQk5n9o164d6tati2nTpmHy5MnIyMjAN998A3t7ewwdOhRAbvCiVatWWLx4MRQKBRwcHHDkyBH88MMPCAoKEmcNPHz4EA8fPoSzs3OxT83zdmCYMmUKZDIZQkJCYGFhUWD7v8J07doVixcvxt69ewvs1uDu7o7ff/8dLi4usLW1xcWLF7Fu3bpiv7sdOnTA4cOH8dVXXyEgIADnz5/Hrl27dI4ZOnQodu/ejaFDh+Kjjz6ClZUV9u7di19++QWffvqpeNzt27ehUql0nuLnN3bsWAwbNgyTJ09Gnz59cOnSJYSHh4vJL4HcKf63b99G3bp1xdkX48ePx6effgpLS0sEBATgr7/+wr59+7B8+XKx7REjRmDPnj0YMWIEhg0bhtjYWCxbtgzvvfceateu/dyxJSIiKikGBIioymnfvj1++eUXhIeHY82aNUhKSoJcLoezszOWL1+Ozp076xzv7e2Nli1bihnPAwICMGPGDPFGqWnTptiyZQuWL1+OGTNmQBAENGnSBKtXr8Zbb70FABg9ejSSk5OxefNmrF69GrVq1UKPHj0gkUiwdu1apKWl6UzhLgtBQUHIycnBihUroFKpYG9vj7Fjx+L27ds4fPhwgenVpWFqaooff/wR33zzDRYuXAitVgtPT09s3rxZfNK7du1arFq1Cps2bUJSUhLs7e3xySefFDrlvyRKMs75hYaGYtGiRQgKCoJcLoeDgwPCwsLw5Zdf4vz58xg0aBAaN26M7t27Y8uWLTh+/Dj++OMPnTbyPqOyuJZHjx7h/fffF9fSF2XBggUwNzfH+vXrkZWVBRcXF2zcuFFn5kpoaCh+/fVX3LhxA0Bu/oINGzbgyy+/xNSpU2FgYIC2bduKOw4AuTkNQkJCEBISgk2bNiExMRF169bFwoULdZbQbNu2DSEhIfjrr7/Ep9aFqV27Nj766CN8+eWXyM7Ohp+fH8LCwgokyiyMtbU1WrdujZMnTxZI2Lh48WIsXLgQCxcuBJC7Td/8+fPx22+/4fz584W216dPH9y9exe//vor/ve//6FFixZYtWoVBgwYIB5jY2OD//3vf/j2228xb948MeCyaNEineU78+fPx/3793H48OEi+9+qVSsEBwdj1apVGD9+PGxsbDBjxgx89NFH4jH//vsvBg8ejK+++gq9e/cGkBtMU6lU+O6777Bjxw7UqVMHX3/9Nbp27SrWa9SoEb777jt88803YsBt6NChmDRp0nPHlYiIqDQkwstk5yEiesPlPen84YcfKrknRBXrgw8+wIoVK8TkifkFBQXh7Nmzxd40ExER0auNOQSIiIhIR0REBLKzs1G9evXK7goRERGVIwYEiIiISIe9vT3Cw8O55z0REdEbjksGiIiIiIiIiKogzhAgIiIiIiIiqoIYECAiIiIiIiKqgrjtYBm5dOkSBEGAvr5+ZXeFiIiIiIgqUU5ODiQSCby8vCq7K0TF4gyBMiIIAqpqOgZBEKBSqars9VcGjnnF4nhXPI55xeJ4VzyOecXjmFesqj7eVfnegF4vnCFQRvJmBri5uVVyTypeVlYWIiMj4eDgAGNj48ruTpXAMa9YHO+KxzGvWBzviscxr3gc84pV1cf76tWrld0FohLhDAEiIiIiIiKiKogBASIiIiIiIqIqiAEBIiIiIiIioiqIAQEiIiIiIiKiKogBASIiIiIiIqIqiAEBIiIiIiIioiqIAQEiIiIiIiKiKogBASIiIiIiIqIqiAEBIiIiIiIioiqIAQEiIiIiIiKiKogBASIiIiIiIqIqiAEBIiIiIiIioiqIAQEiIiIiIiKiKogBASIiIiIiIqIqiAEBIiIiIiIioiqIAQEiIiIiIiKiKuiVCAhotVqsWrUKbdq0gaenJ0aOHIn4+PhCjw0ODoajo2Ohfz799FPxuNOnT6N3797w8PBAYGAg9uzZo9OOUqnE/Pnz0apVK3h5eWHq1KlISkoq1+skIiIiIiIielW8EgGB0NBQ/PTTT1i4cCH+97//QavVYsSIEVCpVAWO/eijj3DixAmdP8OHD4exsTGGDh0KAIiOjsbo0aPRpk0b7Ny5E/369cOMGTNw+vRpsZ158+bhxIkTCA4Oxvfff4+YmBhMmjSpoi6ZiIiIiIiIqFLpVXYHVCoVvvvuO0ybNg3t27cHACxfvhxt2rTBn3/+ie7du+scb2JiAhMTE/H19evXsXnzZixcuBCOjo4AgO+//x6Ojo6YMmUKAKBRo0a4fv06NmzYgFatWiEhIQG7du3CmjVr4O3tDQBYtmwZAgMDcenSJXh5eVXAlRMRERERERFVnkoPCERFRSEzMxOtWrUSy8zNzeHs7Ixz584VCAjkt2DBAnh7e6NXr15i2fnz59GxY0ed41q2bIlFixZBEARcuHBBLMvToEED2NjY4Ny5cy8cEBAEAVlZWS9U93WWnZ2t8zeVP455xeJ4VzyOecXieFc8jnnF45hXrKo+3oIgQCKRVHY3iJ6r0gMCDx8+BADUqlVLp7xmzZrie0X5+++/cenSJezatatAm7a2tgXay87ORnJyMhISEmBlZQUDA4NSn7M4OTk5iIyMfOH6r7vY2NjK7kKVwzGvWBzviscxr1gc74rHMa94HPOKVZXHWy6XV3YXiJ6r0gMCeVHD/D8wBgYGSE1NLbbuxo0b0aFDBzg5OemUKxSKAu3lvVapVMjOzi70B9TAwABKpbLU15BHX18fDg4OL1z/dZWdnY3Y2FjUr18fRkZGld2dKoFjXrE43hWPY16xON4Vj2Ne8TjmFauqj/ft27cruwtEJVLpAQFDQ0MAuTfqef8GcncBKO6Xx3///YeIiAisW7euwHsGBgYFEhLmvTYyMoKhoWGhCQufd87nkUgkMDY2fuH6rzsjI6Mqff2VgWNesTjeFY9jXrE43hWPY17xOOYVq6qON5cL0Oui0ncZyFsq8OjRI53yR48ewcbGpsh6hw4dgrW1Nfz9/Qtts7D2jI2NYWZmBltbW6SkpBQICjzvnERERERERERvikoPCDRt2hSmpqaIiIgQy9LS0nD9+nW0aNGiyHrnz5+Hj48P9PQKTnLw9vbG2bNndcrOnDmDZs2aQSqVonnz5tBqtWJyQQC4c+cOEhISij0nERERERER0Zui0gMCcrkcH374IZYuXYq//voLUVFRmDJlCmxtbdG5c2doNBokJiZCoVDo1Lt+/TqaNm1aaJuDBg3ClStXsHTpUkRHR+O7777D/v37MWLECACAjY0NunXrhtmzZyMiIgJXrlzBJ598Ah8fH3h6epb3JRMRERERERFVukoPCADApEmT0LdvX8yePRsDBgyATCZDeHg49PX18eDBA7Ru3Rp79+7VqZOYmAhLS8tC22vcuDFCQ0Nx9OhR9OzZE9u2bcOSJUt0tjZcuHAhWrVqhQkTJmD48OFo2LAhVq1aVZ6XSURERERERPTKqPSkggAgk8kwffp0TJ8+vcB79vb2uHHjRoHyy5cvF9tm27Zt0bZt2yLfNzY2xhdffIEvvvii9B0mIiIiIiIies29EjMEiIiIiIiIiKhiMSBAREREREREVAUxIEBERERERERUBTEgQERERERERFQFMSBAREREREREVAUxIEBERERERK89jUZb2V0geu0wIEBERERERK81ZY4GqZmqyu4G0WuHAQEiIiIiInptKXM0SE5XQNAKld0VotcOAwJERERERPRaUqrUSE5TQKMRwHAAUekxIEBERERERK8dpUqN5HQlNJwZQPTCGBAgIiIiIqLXCoMBRGVDr7I7QEREREREVFIKlRopDAYQlQnOECAiIiIiotfC84IBglC5QYLKPn9FqkrX+jIqcpxe5FwMCBARERER0SsnICAAQUFB4utsZU6xwYDjx45g5syZOmV79uxBhw4d4Orqijlz5mDQoEEYNGhQmff14cOHGDVqFO7fv1/gvbFjxyIsLKzEbcXFxcHR0bHAn+7du5dll0U7d+6Eo6Mj7t27V6Lj09LSMGPGDJw/f75Mzu/o6Ijg4OAi3793716h45H359NPPwUABAcHw9HR8YX6UB7fi+K+E+XhwoULGDVqVKnrcckAERERERG9ckJCQmBqagogLxiggraYJ6Bbf/4B+noynbIFCxagfv36WLx4MWxsbPD555+XS19PnTqFo0ePFihXqVQ4c+YMJk6cWOK2IiMjAQCbNm2CkZGRWG5oaPjyHS1E+/btsXXrVtSsWbPE/du9ezf69OlTLv3Jr2bNmti6dWuB8i1btmDfvn1l0o+5c+e+dBv5FfWdKC/btm1DdHR0qesxIEBERERERK8cZ2dnACULBhQlJSUF/v7+8PX1Levulcj58+dhYmICJyenEteJjIyEra0tWrVqVY49e8ra2hrW1tYVcq4XIZfL4enpqVN27do17Nu3D1OmTIG3t/dLn8PBweGl23hdcckAERERERG9cgICAjBt+gxE3oxFe/9mOHL4IObNnoFundqgx9sdsPTrhcjOzgYATJkwCpcuXsDZs2fh6OiIiIgIcfr46tWri5wSr1QqsXr1agQGBsLNzQ2dO3fGunXroNVqxWM0Gg3WrVuH7t27w93dHZ6enujfvz/OnDkDIHfKfd609bfeektnmcPRo0fRpk0bSCQSALkBijlz5sDPzw9ubm547733cPr0aZ0+RUVFlSqAkGfnzp1wc3PD+fPn0adPH7i5uaFLly44fPgwYmJiMGTIEHh4eKBTp07Ys2ePTr1nxycpKQlTp06Fv78/3Nzc0KNHD+zatQsAEBERgcGDBwMABg8eLE6zHzRoEKZNm4ZJkybB09MTw4YNA5A73X/GjBlo3bo1XFxc0KpVK8yYMQPJycmlvr48giBgwYIFaNSoEYYOHVrg/UOHDqFLly5wc3NDv379CoxvYfIvGXB0dMSWLVswa9Ys+Pj4wMvLC5MnT8bjx4/FY+7evYsxY8bA19cXHh4eeP/998UZAUV9JwICAvDll19iyJAhcHd3x6xZs4pcspF/yYxKpcKKFSvw1ltvwd3dHd27d8evv/4KAAgKCsKvv/6K+/fvw9HRETt37izhaDIgQEREREREryCtIECVoxUTpS1f8iVsbGth4Vff4v2Bg7Dvj9348fsNAIDJU4PQxLEpnJ2dsXXrVri4uIjTzPv27VvolHhBEDBmzBhs2LAB/fr1w5o1axAYGIgVK1boTCFfunQpQkND8f7772PDhg1YuHAhUlJSMHnyZGRnZ6N9+/YYO3YsgNxlDuPGjRPrHj16FO3atQOQG3wYMmQI/vrrL0yZMgUhISGwtbXFiBEjdG5aIyMjkZmZif79+8PNzQ3+/v5YunQpcnJynjtmarUaU6dORf/+/REWFgYjIyNMmzYNY8aMQfv27bFmzRrUrFkTM2fOxMOHDwttY/r06YiOjsb8+fOxfv16ODs7Y+bMmThz5gxcXFwwZ84cAMCcOXN0xmnfvn0wMTFBWFgYRowYgezsbAwePBjR0dGYO3cuwsPDMXjwYOzZswfLly9/7rUUZe/evbh8+TI+++wzyGSyAu/PmjULgwcPRnBwMExMTDBy5EhcvXq11OdZvnw5tFotli1bhhkzZuDvv//Gl19+CQDQarUYPXo0srOz8c033yA0NBSWlpYYO3Ys4uLiiv1ObNmyBW5ubggNDUXfvn1L3J9p06Zh48aN6NevH9auXYvWrVsjKCgIf/zxB8aNG4d27dqhRo0a2Lp1K9q3b1/idrlkgIiIiIiIXimZ2SpotQKAp8sEfP1aY+yEKQCAZt4+OH8uAmdOHcfIMRNRv0FDmJiYQF9PJk4vz/vb1ta2wJRzADh27BhOnTqFZcuWoVu3bgAAf39/GBoaYuXKlRg8eDAaN26MR48eYcqUKTpPkA0MDDBx4kTcuHEDnp6eqFu3LgDAyckJ9vb2uHr1KhISEhAfHw9/f38AwO7duxEVFYVffvkFHh4eAIC2bdti0KBBWLp0KXbs2IGkpCQkJCRAo9Fg+vTpqF27Nk6fPo3169fjwYMH+Pbbb4sdN61WizFjxqBfv34AchMATpkyBUOGDBGf2puZmaFPnz64du0abG1tC7Rx9uxZjB8/Hh07dgQA+Pj4wNLSEnK5HKampuL0egcHB52p9vr6+pg/fz7kcjmAp0sfvv76a9SpUwcA0LJlS1y+fBlnz54t9jqKEx4ejmbNmhW5DGT+/PkIDAwEALRq1QpvvfUW1q9fj1WrVpXqPE2aNMFXX30lvr5y5Qr2798PAHjy5AliYmLEG3EAcHd3R0hICFQqFaytrQt8J/LUrl0b06ZNE1/fuXPnuX25efMmDhw4gM8++wxDhgwRr+3+/fuIiIhA9+7dYW1tXejyiudhQICIiIiIiCqdVisg9kEaktIU/x8M0OXi4qbzukYNGyQ8ePDC5zt79iz09PTEm8c87777LlauXImzZ8+icePG4k14UlISYmJiEBcXh7///htA7jTuoly8eBFeXl4wMzMDAJw+fRo1atSAi4sL1Gq1eFyHDh3wzTffIDU1FcbGxvjuu+9Qr1498SbSx8cHcrkcK1aswLhx49CgQQOdJQ0AoKf39LbOy8tL/He1atUAQAxAAIClpSWA3GBBYXx9fREcHIzr16+jTZs2aNeuXYHdGwrTsGFDMRgA5N4I//TTT9BqtYiNjUVcXBxu376NmJgYnet/lkaj0dk6TyqVQip9Oqn94sWL+Pfff7F69epC6+vr66Nz587iawMDA7Rt21b8vJ7X/rPy31jb2tqKS1SqV68OBwcHfP755zhx4gRat26Ntm3bissEivMiy0EuXLgAADrXBqDY3RlKigEBIiIiIiKqVJdvJWLPyTtITVciJUOBbKUGaZkqpGYoxWMM8mXZl0ol0Ara/E2VWGpqKqysrApMO69RowYAID09HQBw9epVzJ8/H1evXoWRkREcHBxQu3ZtAMXv+37p0iV06NBBfJ2SkoLExES4uLgUenxiYiIcHBzEGQXPat++PVasWIGoqCjs3bsXISEhOu/fuHFD/HfezgzPena3gudZvnw51qxZg3379uHAgQOQSqXw8/PDggULYGdnV2Q9ExOTAmUbN27EmjVrkJKSgurVq8PV1RVGRkbi2ObXqVMnnW36evXqhcWLF4uvDxw4AAsLC/GpfH5WVlYFbvCrVasmBj+e1/6z8o+ZVCoVP2+JRILvvvsOYWFhOHjwIHbt2gV9fX107NgR8+fPh4WFRaFtAoCxsXGR7xUlJSVFvJayxoAAERERERFVmsu3EvH9nuuQSAC1RgupVAIDuQwajRYx91MRfS+lXM5rYWGB5ORkaDQanaDAo0ePAOTeXGZkZGDEiBFwdHTEnj170LBhQ0ilUhw9ehQHDhwosm2VSoVr165h3rx5YpmZmRnq16+PpUuXFlrH3t4esbGxOHPmDLp27Qpzc3PxPYVCASB3RwBvb+9SrREvLTMzM0yfPh3Tp09HTEwM/vrrL4SGhmL+/PlYt25didv5/fffsXjxYkyfPh29e/cWdzKYPHlykWv6w8LCdGZdWFlZ6bx/5MgRvPXWW9DX1y+0fnp6OgRBEJM4AsDjx4/Fcz+v/dKwsbHBvHnzMHfuXERFRWH//v1Yv349rKysSrWNYV5f88/6yMzMFP+d911ISkrSWeYRHR2NlJQUNG/e/IWvg0kFiYiIiIioUmi1AvaeugOJBIAgQK3WQiqRQK4nhVQqgUYr4OilgrsDFEYqLZhgrjg+Pj5Qq9XiuvA8v/32GwCgefPmiImJQUpKCgYPHgwHBwfx6fOxY8f+v//a/z+37m3Vv//+CzMzM3Gng7zzPXjwANWqVYObm5v45+TJk9iwYQNkMhkSExMxd+7cAn3au3cvTE1N4eLiAhsbG536bm66Sylexv3799GuXTvx/A0bNsTIkSPh5+eH//77DwAKTeRXmAsXLsDc3BwjRowQb8gzMzNx4cKFAje/eRwdHXWu69m19ykpKYiNjUWzZs2KPGd2dra4+0Pe+Y4cOSLmGyiu/dK4dOkS/Pz8cOXKFUgkEjg5OWHKlClo0qSJOE5FLUXIL29Gx7NJHvNu9PPk3fAfPnxYp+7SpUuxaNGiUp0vP84QICIiIiKiShH3IA0paUrkqLXQaAreJOrJpHicnF2itkxNzXD93ys4ffo0nJ2di522DeQm9PP19cXs2bORkJCApk2b4uzZs1i/fj169eoFBwcHpKenw9TUFGvWrIGenh709PRw4MABbN++HQDENeV5T3APHjyItm3bivkDntW7d2/8+OOPGDZsGMaMGYNatWrh1KlTWL9+PT788EPo6+ujefPmaNWqFRYvXgyFQgEHBwccOXIEP/zwA4KCgnRmDZQHOzs72Nra4osvvkBGRgbq1q2La9eu4ejRoxg9ejQAiDkRjhw5AgsLCzRt2rTQttzd3fHzzz9j8eLF6NChAx49eoTw8HA8fvz4uZ9NYW7evAkAOokM89PX18dnn32GTz75BKampli3bh0UCoVOlv+y4OzsDENDQ8yYMQMTJ05E9erVcerUKURGRorbMub/TjRq1KjQtnx9fWFoaIjFixdj8uTJyMzMxKpVq8RcDwDQtGlTBAYGYsmSJVAoFHBycsKxY8fw999/i8tHzM3N8fjxYxw9ehROTk4FdtUoCgMCRERERERUKRJTsvEkLRv6elJIn5nmnUciATTFrNN/Vp9+/XHrZiRGjhyJr776Cu+8806xx0skEqxduxarVq3Cpk2bkJSUBHt7e3zyySc6GflDQ0PxzTffYPLkyTAxMYGTkxN+/PFHjBw5EufPn0dAQAB8fX3h5+eHb7/9FqdPn8aNGzfEG8M8xsbG2LJlC7799lssWbIE6enpsLOzw9SpU/HRRx8ByH3KGxISgpCQEGzatAmJiYmoW7cuFi5cKO4cUN5CQkKwbNkyrFy5EsnJyahVqxYmTJiAUaNGAQAaN26M7t27Y8uWLTh+/Dj++OOPQtvp1asX7t27hx07duCnn36CjY0N2rVrh4EDB+Lzzz9HdHR0kTfJhXn8+DEAFBsUsba2xtSpU7Fs2TIkJibCw8MDP/74Ixo2bFiKEXg+AwMDfPfdd/j222+xaNEipKWloX79+liwYAF69+4NAAW+E0UttzA3N0dwcDC+/fZbjB8/HnZ2dpgwYQJ27dqlc9ySJUsQEhKC77//HsnJyWjUqBFWrVol7gbRu3dvHD16FOPHj8ekSZPEz+t5JEJxmTCoxPLWwZTllJ3XRVZWFiIjI+Hk5PRCSTKo9DjmFYvjXfE45hWL413xOOYVj2NesZ433oIgIDVThajYJIRuvwwDuQxyvYJTnlU5WihzNBjT2x31ahX/dNxAX4bqliVPnleeqvK9Ab1emEOAiIiIiIgqjCAISMlQIis7B3Y1TGFTzRiZipwCGfsFQUCmMgc21YxRx8asknpL9GZjQICIiIiIiCqEIAhITlciW6GGgNytAwNb1oehXA8pmSqocrTQCgJUOVqkZKpgJNdDYMv6kEoLLicgopfHgAAREREREZU7rVZAUpoS2crcYECepvWt8UGXprCrYQpljgZpmSooczSwq2GKgV2aoml960rrM9GbjkkFiYiIiIioXGm0AlLSFFDkaAp9v2l9azSpa4X4hHRkZOXA1FgfdWzMODOAqJwxIEBEREREROVGo9EiOV0JZRHBgDxSqeS5iQOLcis+GfcSMvBOm4awMDV4oTaIqiIGBIiIiIiIqMxJJBJoNAKS0hRQqbXlcg5BELDvVCx+Ox4DAIh7mIZZw3zL5VxEbyIGBIiIiIiIqMxJZfpIzlBCpicvl/bVGi1+3BeFM9cePFNWfjuqC4IAieTNXMLwJl8bFY9JBYmIiIiIqEypNVqkZ6mhes4ygReVqcjBqq2XdIIBttbGGN3LTXwdHR2NhQsXokuXLvDw8EDz5s3Rv39//PTTT1Cr1cW2v3PnTjg6OuLevXsAgFu3bmHAgAE6xwQEBCAoKKjYdoKCguDo6Ii2bdsW2FYxz9KlS+Ho6IhBgwYVeG/s2LEICwvTKdu/fz9GjRqFNm3awNXVFa1bt8bkyZNx5cqVYvuSZ9CgQTrn2rZtG77++mvxdUREBBwdHREREVFsO46OjnB0dMSyZcsKfV+r1aJNmzZwdHTEzp07i2zn3r17YluF/fn0008BPP1Mnv3j7u6Ot99+G2vWrIFGUz7ftTcdZwgQEREREVGZUeZokJyuQpZCWS7tJ6ZkI+SXf5CQlCWWNa5jiUnve8G2mgkAYO/evfj000/RqFEjDBs2DA0aNIBCocDRo0fx5Zdf4vjx4wgNDS3yqXj79u2xdetW1KxZE0DuTfilS5deqL9SqRQJCQm4ePEimjdvXuD9vXv3FlpPpVLhzJkzmDhxIgBArVZj6tSpOHjwIN599118/vnnsLKywn///YdffvkF/fv3x9KlS9G1a9di+zN37lyd12FhYfDx8Xnha9u/fz8++eSTAu+dO3cOjx49em4bNWvWxNatWwuUb9myBfv27UOfPn10ykNCQlCjRg0IgoDs7GxcvHgRq1atgkKhwMcff/xC11GVMSBARERERERlQqlSIzldiRx1+TytjbmfirAdl5GelSOW+brY4sO3nWBqpA8gd2bAp59+ijZt2mDFihXQ03t6y9OuXTv4+vpi0qRJ2LdvX5E3z9bW1rC2LpvtDmvVqpWb62DfvgIBgX/++QcJCQlo0qRJgXrnz5+HiYkJnJycAABr1qzB/v37sWrVKnTp0kXn2HfeeQfjx4/H/PnzERAQAENDwyL74+DgUAZXlatZs2Y4f/48rl+/DmdnZ5339uzZAycnJ0RGRhbbhlwuh6enp07ZtWvXsG/fPkyZMgXe3t467zk5OcHe3l587efnh/j4ePzvf/9jQOAFcMkAERERERG9NMX/BwM02vJZx38+MgHLfrqoEwzo3roBhnZ3hr7e09uaDRs2QCqVYv78+TrBgDxdunRBz549xdeOjo4ICQlB79694e7ujpCQEJ0lA8HBwQgJCRGPDQ4OLnXfAwMD8eeffxZYNrB37174+fnB0tKyQJ2jR4+iTZs2kEgkyM7ORnh4OAIDAwsEA4DcJ/Uff/wxfH198eTJEwC5U+ydnZ2xbds2+Pv7w8fHB7dv39ZZMhAQEID79+/j119/1VkiUVItWrRA9erVsX//fp1ytVqNP//8E926dStVe0BuPoMFCxagUaNGGDp0aInqmJubMwfCC2JAgIiIiIiIXkq2Mgcp5RQMEAQB+0/HYsPua1BrcncrkEklGNbdGd1bNyxwI/jXX3+hZcuWqFatWpFtfv311zqzA9asWYN33nmn0Kfv/fr1Q9++fQEAW7duRb9+/Up9DV27dhWXDeTRarXYv39/kTfNR48eRbt27QAAp06dQlZWFrp3717kORwdHbFq1SrY2dmJZRqNBt999x0WLVokLqF4Vt70+3bt2ukskSgpmUyGLl26FAgInD59GkqlEgEBAaVqD8gNkly+fBmfffYZZDJZgfe1Wi3UajXUajUyMjJw7Ngx7N69Gx988EGpz0VcMkBERERERC8hS5GD1AwVtEUkzXsZGo0WPx24gZNX/hPLTAz1MKa3OxrXtSpwfGpqKlJTU1G/fv0C7+VPJCiRSMQbTm9vbwwbNkx87+rVq+K/bW1tYWtrCwAFpraXlJubG+rUqaOzbOD8+fNISUlBx44dsWPHDp3j4+PjER8fD39/f/E1gALXpdVqodXqbukolUohlT597jtmzBi0b9++0H45OztDLpfD2tr6ha+ta9eu2LJli86ygb179+Ktt96CgYFBqdsLDw9Hs2bN4Otb+PaRnTp1KlDm5uaGIUOGlPpcxIAAERERERG9oExFDtLKKRiQrVBj7a6riIpNEstqWBlhQj9P2Fgbi2VarYD4hHQAgLFeToF2ACAuLg6dO3fWKbOzs8Phw4cBQFynX566du2KXbt2YdasWZBIJNizZw/at28PU1PTAsceO3YMXl5eMDMzA4ACN/15Vq5ciTVr1uiUTZgwQUxECJT/tTVv3hw2NjbYv38/nJ2doVKpcOjQISxZsqTAsRqNRmfZRP7gxcWLF/Hvv/9i9erVRZ4vLCwMNWrUAAAolUrcunULYWFh6N+/P7Zu3VroeFLRGBAgIiIiIqJSy8xWIS1ThfJIGfA4JRurt1/Gg8eZYlkjewuM7e0OU2O5WBYVm4T9Z2KR8CQL1haG0GgE6Okb4PqNGJ32atWqhe3bt4uvV69ejZs3b4qvjY2NUd66du2KtWvX4uLFi/D09MSff/6JefPmFXrs0aNH0bZtW/F17dq1AQD3799H48aNxfKBAweiY8eO4uu8pQ3PKu9rk0gkCAwMFHcbOH78OKRSKfz9/ZGQkKBzbKdOnXD//n3xda9evbB48WLx9YEDB2BhYSEulShMkyZNdJIKent7o0mTJhg4cCC2bdumM9ODno8BASIiIiIiKpWMbBXSyykYEPsgDaHbLyMtUyWWeTvZYEg3J+jrPV1THhWbhC0HoqBQqmFipA8zI30oVGqY13bBubNncOZyHFp61AOQm8nezc1NrFtYEr/y1rRpUzRo0AD79++HQqGAUqksdCq/UqlEREQEpk6dKpb5+/vDwMAA+/fv16ljY2MDGxubCuh98bp27Yrvv/8ekZGR2Lt3Lzp37gx9ff0Cx4WFhUGlevq5WlnpLvs4cuQI3nrrrULrFifvs42NjS1956s4JhUkIiIiIqISy8gqv2DApRuP8O2WCzrBgK5+9fHRuy46wQCtVsD+M7FQKNWwNJVDrieFVCqBXF8G5xbdoNVqMHv2LCgUygLnUCgU4pr8knp2WvvL6Nq1K/7880/s3bsXnTp1KnSNfUREBCwtLeHo6CiWmZmZYdiwYdi1axcOHjxYaNvPzngojbK4Nk9PT9jZ2WH37t04fPhwkYkSHR0d4ebmJv559kl/SkoKYmNj0axZs1Kf/8qVKwAK5lig5+MMASIiIiIiKpGMLBXSslQo65QBgiDg0Nm72Pn3beQ1LZVK8GFgU/i51y5wfHxCOhKeZMHESL/ALgPm1e3h2m4orh//Ae/26ImBA96Ho6Mj1Go1Ll26hO3bt+Px48cYMWJEiftnbm4OAPjjjz/g4eGBOnXqAABu376NTZs2FXr8s1P783Tt2hWrV6/G7t27ERoaWui5jh07prNcIM+kSZPw8OFDTJw4EYGBgejUqRNq1qyJxMRE/P3339i3bx9sbGzQqlWrEl9XXl+vX7+Os2fPwt3dXSw/cOAAIiMjCxzfr18/mJiYFCgPDAzE5s2bYWlpCR8fn1L1AXga0HBwcCj2uMjISDx+/BhAbm6F6OhoBAcHo0aNGujVq1epz1vVMSBARERERETPlZ6pQnp22QcDNFotth68iWOXnq4tNzLQw+hebmha37rQOhlZOdBotdCTFX47Y+fQHDLT2qirdxvbt2/H/fv3IQgC6tSpg65du6J///6leprcuXNn7N69G0FBQejbt6+49v/q1as6OxLk6d+/f6EBAQcHBzRp0gSJiYnw8/Mr9FzHjh3DjBkzCpTLZDJ8/fXX6N69O7Zt24YlS5bg8ePHMDExgZOTE2bNmoWePXvCyMioxNcFAB999BG+/PJLDB8+HBs3bhTLt2zZUujxgYGBhQYEunbtivDwcLz99tsvNOsg7yY/L/hSlAkTJoj/1tPTg5WVFXx9fTF58uRKWQryupMIQjmkBK2C8n4RPLs2qarIyspCZGQknJycKiQhC3HMKxrHu+JxzCsWx7viccwrHsf85ZQ2GJCdnY3o6Gg0atSo2BvUbKUaG3Zfw78xT8SyahaGGN/PA7WrF50tPu5BGtbsvAIDuQxyvdybz+qWRlBrtMhSqKFUaaBQafDZUB841LEsWafLUFW+N6DXC2cIEBERERFRkcprZkBSmgKrt13G/cQMsaxBbXOM7eMBcxN5MTWBOjZmsKlmjPuJGdA3kessGxAEAenZOahfyxwN7SzKttNEbxgmFSQiIiIiokKlZZRPMODuwzR8vfmcTjCgmWNNTBnQ7LnBACA3v0Bgy/owlOshJVMFVY4WWq0AlUqDJ2lKGBvooW9AY0ilkue2RVSVcYYAEREREREVkJqhRKYip8yDAVduJWLDb9egytGKZV1a1kOPdo0glZT8Br5pfWt80KUp9p+JRcKTLKRn50CjEVC/ljn6BjSGR+MaZdtxojcQAwJERERERKSjvIIBh8/HY9uhm093EpBIMKCLI9p42r1Qe03rW6NJXSvEJ6QDAGyrmaB+LXPODCAqIQYEiIiIiIhIVB7BAI1Wi22HbuHIxXtimaGBDKN6usG5QbWXalsqlaBeLXMY6MtQ3bJ0GfaJqjoGBIiIiIiICACQkqFEVhkHAxQqNcJ3X8PV6Kc7CVib5+4kYFej6J0EiKj8MSBARERERFTFCYKA1ExVmQcDUjKU+O6Pq+KUfgCoZ2uGcX09YGFqUHYnIqIXwoAAEREREVEVJggCUjKUyFaoUZYpAx6n5eCHI1eRmqESyzwa18BH77jAQC4rwzMR0YtiQICIiIiIqIoqr2DA9TvJ2HEyGTmap612bFEXvTs4MOEf0SuEAQEiIiIioipIq80NBiiUZRsMOHLxHrYevCEuPZBIgPc7OaJ9M/syPAsRlQUGBIiIiIiIqhitVkByugIKlaZM29z5920cOndXLDPQl2JETze4NapeZuchorLDgAARERERURWi0QpISVNAkVN2wQClSoPvfv8Xl28limUmhlKM7e0Gh7oMBhC9qhgQICIiIiKqIjQaLZLTlVCWYTAgNUOJ0O2XEffw6U4CdjVM0MndCHY1TMrsPERU9hgQICIiIiKqAtQaLZLTFFCptWXW5v3EDKzedhlJaQqxzK1RNXzQ2QH34mPL7DzPI5VIYGjAnQuISosBASIiIiKiN1yOWouU9LINBly/8wTrdl2FQvl0tkGH5vbo91YTKJWKYmqWLZlMAktTAxjKeWtDVFr8qSEiIiIieoPlqDVITlMiR1N2wYDj/9zHzwduQPv/WwlIAPTr2AQB3nXK7BwlIdeTwtLMEPp60go9L9GbggEBIiIiIqI3VHHBAEEQIJFIStWeVhCw+2g0DpyJE8vk+lIMf9cVHo1rlKqtFzl/HokEMDLQg4WJAaTSF2ujLPtDVJiK/k69yPkYSiMiIiIies0FBAQgKChIp0yVo0FSmqLQYMDJE0ex+Iu5OmWHDx1A/z7d0KVDSyz7ZhGmTBiFKRNG6bS3Ydc1nWCAhakcUz9oXqpgQOKjBHw6fRISHj4o8N7soE/w4/fhxdaXSiQwM5LDyszwhYIB9+7dg6OjI3bu3AkASEtLw4wZM3D+/HnxmEGDBmHQoEGFHr9z5044Ojri3r17pT53RXm2/y9r27ZtcHR0LPBnwYIFFdaH8pT/8y0rf/31F2bOnFmmbRYnNDQU4eHF/+wUhjMEiIiIiIhecyEhITA1NRVfK1VqJGcoodEIhR6//X9bCpStWv417O3rImjWfFSvURPffv2F+F5apgphOy7jzn9pYpldDVOM7+sBawvDUvX1wvkIRJw+WaA8JycHFy+cxZCPRhVSK1d55AuIjIzE7t270adPH7Fs7ty5RR7fvn17bN26FTVr1iyzPrzKIiMj0aBBAyxevFinvHr1N2M7yZo1a2Lr1q2oW7dumba7adOmMm3veVauXIkJEyaUuh4DAkRERERErzlnZ2fx30qVGsnpSmi0hQcDipKWmgrv3i3h2cxbp/zB40yEbPsHT1KfJgp0bmCNkT3dYGRQdrcTV/65CBNjEzg0diz0/YrMF+Dg4FDke9bW1rC2ti73PrwqIiMj4ebmBk9Pz8ruSrmQy+Vv7LWVBJcMEBERERG95vKWDETfiYO7mwv+OvQn5s2egW6d2qDH2x2w9OuFyM7OBgBMmTAKl/+5gMv/XEBA6+b45+J5BLRuDgDYvHE9Alo3x8MH/wEAshQ5+OaH83iSqoBWk4MnNw8h4fRy7Fk7DqOG9MXPP26CVvt0SYJGo8HPP27CuJGDMHXiCPR+pyMmjBmGSxfPAQD27/0N33w5HwAwsN87+HrR0yfxEWdOooWvHyQSidini+fP4pOJo/D2W37o16srdv26A48ePcKECRPg5eWFdu3a6TyJLWo6f2FLKgAgIiICgwcPBgAMHjxYnOJe3HT3/OcICgrC0KFDsWPHDnTp0gWurq6YNm0aLl26pFPv0qVL+OCDD+Dp6Yn27dvj+++/x9ChQ8V+RUREwNHRERERETr18vdFoVDg22+/RefOneHq6opmzZph2LBhiIyMLLS/xbl58yZGjx6NZs2aoVmzZhg/fjzi4+PF9wVBwI0bN+Dk5FTqtvNLSkrC/Pnz0aFDB7i6usLHxwfjx4/X+awGDRqEadOmYdKkSfD09MSwYcMAAI8ePcKUKVPg4+ODFi1aYM6cOVi+fDkCAgJ0zrFt2zZ069YNrq6uaN++PYKDg6HRaFCcwpaEODs74/Lly3j//ffh5uaGDh06FJiO/8cff+Ddd9+Fu7s7WrZsiWnTpiEhIUG8jrNnz+Ls2bPiZ5r3+f7vf/9Dhw4d0KxZM5w8ebLQ71ph34WYmBhMmDBBHIPRo0cjOjoaAODomBtECwkJEf9dUgwIEBERERG9AdQaLdIyVQCA5Uu+hI1tLSz86lu8P3AQ9v2xGz9+vwEAMHlqEByaOMKhiSNC1mxCY8emCFmzCQDQtXsPhKzZBOtq1ZGaqcS9xAxkK9UQBAH/nduE9Lhj6P/+e1j09XK069AR4etDsXzJl2If1q8Jxg+b1iOwWw+MmTgNk6bMQHpaKubPngmFIhstW7XBh0OGAwDmL1qCD4eOEOtGnD4B31b+Otf0xbzP0L5DB6xduxYNGjTA3LlzMXjwYDRu3BihoaFwd3fHV199hStXrrzQmLm4uGDOnDkAgDlz5hS7VKA4165dQ3h4OCZNmoTVq1dDKpViyZIlSE1NBQBER0dj6NChAIBly5Zh4sSJWLduHS5cuFDqc82YMQM7duzAqFGj8N133+HTTz/FrVu3MHXqVAhCyWeF3LlzB/3798eTJ0/w9ddfY9GiRYiPj8eAAQPw5MkTAMDdu3eRmZmJq1evokuXLnBxcUGXLl2wa9euUvVZEASMHj0aJ0+exLRp0xAeHo4JEybg9OnTBcZ83759MDExQVhYGEaMGAGVSoUhQ4bg4sWL+Oyzz/DVV18hKioK3333nU69tWvX4vPPP0erVq2wZs0afPDBB1i/fj0+//zzUvUVALRaLT7++GN07doV69atQ7NmzfDNN9/g+PHjAIALFy5gxowZ6Ny5M9avX49PP/0UZ86cwdSpUwHkLjlxdnaGs7Mztm7dChcXF7HtkJAQzJw5E3PmzIGXl1eJ+pOQkID3338fsbGxmDdvHpYsWYLHjx9jyJAhSElJwdatWwEAffv2Ff9dUlwyQERERET0mtMKAlQ5Wmj/f5mAr19rjJ0wBQDQzNsH589F4Myp4xg5ZiLqN2gIE+PcfAPOrm46f1evYYOmLq74/XgMEp5kie0rk24i6/EtzJ73JQI6dgEAeLdoCQMDQ2zcEIbe/QagQcNGePI4EcNHjUfXd3ohOjoajRo1gompGebNmo6Y27fh7OqG2nb2AIDGTZrCtlZtAMCD/+7jv/v30LyFr8519erdG2NHjwQAGBsb47333oO7uzsmT54MAGjatCn+/PNPXLx4Ee7u7qUeN1NTU3F5gIODQ7FLBYqTnp6OnTt3iuvQHz58iDlz5uDMmTPo0qUL1q5dCzMzM2zYsAFGRkYAgIYNG6J///6lOo9KpUJmZiZmz56Nrl27AgB8fHyQkZGBxYsX4/Hjx6hRo2QJHkNCQmBkZIRNmzaJ+SdatWqFjh07YsOGDZg5c6Y46+DevXsICgqCnp4edu3ahZkzZ0KlUuG9994r0bkePXoEIyMjzJw5E97euUtSfH19cffu3QI3sPr6+pg/fz7kcjkAYPv27YiJicGOHTvg6uoKAGjZsiU6duwo1klPT0doaCjef/99zJ49GwDQunVrWFpaYvbs2Rg2bBgaN25cor4CuQGMcePGoV+/fgCA5s2b4+DBgzhy5AjatGmDCxcuwNDQEKNGjRL7aWlpiatXr0IQBDg4OIhjmn85wsCBAxEYGFjivgC5+QhUKhU2btwofr5NmzbFgAEDcPnyZbRr1w4AYGtrW+rlDwwIEBERERG9xjKzVf8fCHj6dNjFxU3nmBo1bJDwoGBW//y0Wi2+++1fnI9MEMvMjPXRUJqJ+zIZ2nXoqHN8py5dsXFDGK78cwENGjbCrLmLAAAPH/yH6Ns3EXMrCufPnQEA5OSoijxvxOmTcHHzgKmpGQCIeQJaeDcXj6lWrRoAwMPDQyyzsrICkHtDWJmsra11ktLl9TVvmcaZM2fQtm1bMRgAAF5eXrCzsyvVeeRyuTh1PSEhAXfu3EFsbCz+/vtvALkBg/y0Wq3Osg6JRAKZTIYzZ87Ax8cHhoaGUKvVAHIDJN7e3jh16hQAoEWLFlizZg18fX1hbGwMAGjTpg2SkpKwatUq9OvXD1qtVmdmQl77z7KxscHmzZshCALu3buHuLg4xMTE4OLFiwX63LBhQ/EmO2/s6tSpIwYD8vrZoUMHcUr9pUuXoFAoEBAQIF4LAHFJwcmTJ+Hg4FBg+UD+fj7r2af3crkc1tbWyMrKEsdl+fLl6N69O7p06YJ27dqhdevW4o15cV5k+cWFCxfg6empE+yxtbUVP/eXwYAAEREREdFrSKsVcCMuGQ+fZEKt0eLZ2eIGhrqZ/6VSCbRCwe0H8ztz7SGktZ4GA+T6Mswc3AKb1h6FhaVlgRsoa+vcG9+MjAwAwI2o61jx7WLciPwXcrkc9Rs0EmcBFDedPeLMCfi29IdEAhgZ6MHcJPeG8NmdE/I8e1P9qsjfp7y94PNuxJOSksQgwbNeJFP/8ePH8eWXXyImJgYmJiZo2rSpeLNe2Bh/9tln+PXXX8XXdnZ2OHz4MFJSUrB3717s3bu3QJ28pInVqlVDhw4dCrzfrl07nDp1Co8fP8Ynn3yCs2fPiu/5+Pjghx9+KFDnt99+w7Jly/DgwQNYWlrCyckJhoYFd6gwMTHReZ2cnFzo2D1blpKSAgAYNarwHSoePXqEs2fPivki8mzevLnIoEz+vkmlUnF8vby8sG7dOmzatAkbN27EunXrUL16dYwZM+a5Wy3mfValkZKSAnt7+1LXKwkGBIiIiIiIXjP/xjzG/lOxuHUvBelZKqRn5SAy9gmi76W8UHsJSblPPpPSFKheK7fM2FAPtaqboLqlEczNLJCakgKNRqMTFHjy5DEAwMLCEpmZGZg5dQIaNmqMsA0/QKlSo3Hjxrj8zwUcO/JXkedWKZX45+J5jB47CWZGcpiZyMUb6tLKfyOeJzMz84XaKyu2trZ4/PhxgfInT56gYcOGAIrve95N8t27dzF+/Hh07NgRa9euRZ06dSCRSLBlyxZxfXt+EyZMwAcffCC+znv6bmZmBj8/PzFx37P09HJvE8+fP4/4+Hj06tVL532lUgmZTAYLCwvMnz9fZ3zz39DntTNz5kwMGjQIw4cPh42NDQDgm2++eW4eBRsbG8TGxhYoz8tzAADm5uYAgKVLl6J+/foFjq1evTrMzMywfft2nfIGDRqIwYTSatOmDdq0aYPs7GycOXMGmzdvxhdffAEPD49SL1/JP3MhbyZCHjMzMyQlJRWod/r0adjb26NOnTqlv4D/x6SCRERERESvkX9jHmPLgShcj02CRivA3EQOCYAshRq7jkWXqA2p7OltwK27yfh68zmd9/3da8Oupilk0tzj3L2aQaPR4Ojfh3SOO/Rn7tNlV3dP3I2LRVpqKvr0G4C69RpA+v91z57JnX6eN0NBKtWdZfDPpQswt7BEcy9XmJnI8TLyZhQ8fPhQLIuOji72pq+4aeNlpUWLFjh+/DiUSqVYdv36dZ0M+4X1PTU1VcwkD+QmL1QqlRg1ahTq1q0rBhHyggGFzRCwt7eHm5ub+CcvC72Pjw9u374NJycn8T1XV1ds2rQJBw8eBJA7XT8oKAh37twR29NqtThw4AC8vLwgl8vRsGFDnfbzAhzPunTpErRaLSZOnCgGAzQajbg0IX8Q5Fk+Pj64d++ezi4KCoVCJwDi4eEBfX19JCQk6PRFT08Py5Ytw71792BqaqrznpubW6EzUEri66+/Rp8+fSAIAoyMjNChQwfMnDkTAPDff7k7dOR9/5/H1NRU5zMHUCBI4u3tjcuXL+sEBZ48eYIRI0bg6NGjpTpffpwhQERERET0mtBotNh76g4ePs6EoVwm3hBKJIBcTwplTu6Txuclmzc1NcP1a1fw0/b9OBalBvSeTnnv2a4RurSsh4t7nz6l923pD89m3vj26y/wOPERGjk0weV/LuDnHzehy9vdUb9BQ2RkpMPExAQ/bv4Oao0Gjx49wp7ftuPg/j0AAEW2Qjw3ABw/ehi+rfxx/uwptGvbFobyl7818fX1haGhIRYvXozJkycjMzMTq1atgqWlZZF1zMxy+3PkyBFYWFigadOmL92P/MaMGYO9e/dixIgR+Oijj5CWloaVK1dCKpWKn6GjoyNq1aqF1atXw9TUFBKJBGvXrtVZjuDi4gI9PT0sWbIEH330EVQqFXbu3IkjR44AKPhkuTjjxo1D//79MXr0aAwYMAAGBgbYunUrDh06hFWrVgEA+vfvj//9738YM2YMJk+eDCMjI/z000+4efMmtmzZUuJz5T0xX7BgAfr06YPU1FRs2bIFUVFRYr+Lujnv3r071q1bh/Hjx2Py5MkwNzfHxo0b8eTJE9SunbscxcrKCiNGjMDKlSuRkZEBX19fJCQkYOXKlZBIJGX+mbZs2RIbN25EUFAQ3n33XeTk5GDDhg2wtLREy5YtAeTOWrh06RJOnz4NZ2fnItvq0KEDDh8+jK+++goBAQE4f/58gV0chg4dil27dmHEiBEYPXo09PX1ERYWBltbW7zzzjvi+S5evIhz587B29u7xLNsOEOAiIiIiOg1IAgCrt9Jwo24FMhk0oL/4ZdIYPz/N9VJqYpi2+rR+z2oNED4qjlIS4gSy5s1rYnAVvULtC2RSPDlNyvQvUdvbP/lJ3w2YzKO/n0II0ZPwPRPc7eNMzU1w8KvlgGCgMULP8ePG9ch8VECVoSsh7GxCa5euQQA8GrmjebevtiwNgTrQlfi7JlT6NCh/UuNTR5zc3Nx7/nx48dj5cqVGD9+vE5CuvwaN26M7t27Y8uWLZg2bVqZ9CO/evXqITw8HEqlEpMmTcLy5csxcuRI1KhRQ5xiL5PJsGrVKlSvXh2ffPIJFi1ahG7duqFz58467Xz77bdISEjA2LFjxS0Tf/jhB0gkEpw/f77EfWratCm2bNkCiUSCGTNmYNKkSUhMTMTq1avFc1avXh1btmyBo6MjvvjiC3z88cfIzs7Gpk2bdJI7Po+vry/mzJmDS5cuYeTIkVi8eDFq166NkJAQAAWfiD9LT08P4eHhcHZ2xrx58zBjxgw0btwYnTp10lmP//HHHyMoKAgHDx7EyJEjsWTJEjRv3hw//vijGPQpK+3atcPSpUtx69YtTJgwAZ988gmMjIywefNmMfj0wQcfQF9fHyNHjsSxY8eKbKtPnz4YOXIk/vjjD4waNQqXLl0SAzJ5atWqhZ9++gk1a9ZEUFAQPv30U9SqVQvff/89LCwsAOQGna5du4aRI0fiQQkSiOaRCKXZrJKKdPXqVQCAm5vbc45882RlZSEyMhJOTk4vlCSDSo9jXrE43hWPY16xON4Vj2Ne8V73MddqBSSnK3HpxiNs/OMazE3kkBbyBFArCEjLVGFIVxe4NCqYiA0ActRa/LgvEhH/Pp2mbGqkj3F9PdDQzqJM+pudnS1uO1hYEkCpRAIzY32YGr/cEoFXVf57g9OnT0NfX1/ccg8A0tLS4OfnhxkzZhRIdkdP3bp1CzExMejcubNOoKpv376wtbUVgwr0YrhkgIiIiIjoFabRCkhJU0CRo4GxoR5kUinUGgFyvYIBAbVagEwqhamxfqFtZWbnYM3OK7gVnyKW2VgbY0I/D9SwqphAiUwmgaWpQZksEXhd/Pvvv1i1ahU++eQTuLi4ICUlBRs3boSZmRm6d+9e2d17pWVlZWHy5MkYOHAgOnXqBI1Gg7179+LatWvlNqOjKqk6P4VERERERK8ZjUaL5HSlmBugjo0ZbKoZ435iBvTzZeMXBAGZyhzY1TBFHZuCU6QfJWdh9bbL4o4CANCkriVG93aHiWHhAYSyZqAvg4WpAfT1qtbK5bz1/j///DMePHgAY2Nj+Pj44KuvvhK3+KPCeXh4YMWKFQgPD8euXbsgCAKcnZ2xYcMGcb0+vTgGBIiIiIiIXkFqjRbJaQqo1E8zsEulEgS2rI8tB6KQkqmCiYE+9PQkUKtzgwFGcj0EtqwPqVR39sDteylYs+MKMrJzxLKWrrXw4dtNoScr/5tziQQwMtCDhYlBgb5VBVKpFOPGjcO4ceMquyuvpcDAQAQGBlZ2N95IDAgQEREREb1ictQapKQrdYIBeZrWt8YHXZpi/5lYJDzJQpZSC5lUCrsapghsWR9N6+s+cT53/SG+3xMJteZpW++0aYiufgWTB5aHNz1fANHrrNIDAlqtFiEhIdi2bRvS09PRokULzJkzB3Xq1Cn0+JycHKxatQq7du1Ceno6XF1dMWvWLDg5OQEAAgICcP/+/ULr/vjjj2jRogUSEhLQtm3bAu9/9dVX6N27d9ldHBERERFRKalycoMBOZqi92ZvWt8aTepaIT4hHRlZOTA11kcdGzOdp++CIGDf6Vj8dixGLNOTSTCoqzN8XWzL9Rry6OvJYGVetfIFEL1OKv0nMzQ0FD/99BMWL14MW1tbLFmyBCNGjMDvv/8OubxgFHHevHk4cuSIuFXFypUrMXLkSOzbtw9mZmbYvn07NBqNeLxKpcJHH30EW1tbeHl5AQCioqJgYGCAQ4cO6URFy3o7CiIiIiKi0lCq1EjOUEKjef5GYFKpBPVqmRf6nlqjxU/7o3Dq6tPtx0wM9TCmjzsa17Eqs/4WRQLA1NgQ1mYMBhC9yir1p1OlUuG7777DtGnT0L59ewDA8uXL0aZNG/z5558FMm7Gx8djx44dWLNmDdq0aQMA+OKLL9CzZ09cu3YNrVq1KpCU4+uvv0ZaWhp+/vln6OnlXu7NmzdRv3591KxZs/wvkoiIiIioBBQqNVLSldBoX25X8CxFDtb+ehU34pLFshpWRpjQzxM21uW/k4BEApgY6cPUSAqZrOrlCyB6nVRqQCAqKgqZmZlo1aqVWGZubg5nZ2ecO3euQEDg5MmTMDMz05nub25ujsOHDxfa/u3bt7F582Z8+eWXOoGCGzduoFGjRmV8NbnTsrKysp5/4BsmOztb528qfxzzisXxrngc84rF8a54HPOK96qPuUKlRWqG4qWDAU9SFVj/WyQSkp5eZ4PaZvioe1OYGknK/fplUgnMjOWQQgt1Ts4rO97lTRCECsnPQPSyKjUg8PDhQwBArVq1dMpr1qwpvvesO3fuoE6dOvjzzz+xbt06JCQkwNnZGUFBQYXe4K9atQpNmjRBjx49dMpv3rwJKysrfPDBB7hz5w7q1auHsWPHFppXoDRycnIQGRn5Um28zmJjYyu7C1UOx7xicbwrHse8YnG8Kx7HvOK9amMuk8mgFqRIz1RBlaN+qbYeJudgz7lkZKueBhWa1DbEWx5GSPjvLhJetrPPYWhoADMjGVIEDbTa3PwHr9p4V6TClj8TvWoqNSCQFzHM/8NiYGCA1NTUAsdnZGQgLi4OoaGhmDFjBszNzREWFoaBAwdi7969qFatmnhsfHw8Dh48iJUrV+q0oVarERMTAwcHBwQFBcHU1BR79uzBqFGjsHHjRp3ZCqWlr68PBweHF67/usrOzkZsbCzq168PIyOjyu5OlcAxr1gc74rHMa9YHO+KxzGveK/qmGcq1EjPUsGy2svNDLh86wl2n7mFnGdyD3T2sUdgyzrl/qRaAsBArgcLE7m4ROBVHe+Kcvv27cruAlGJVGpAwNDQEEBuLoG8fwOAUqks9BeHnp4eMjIysHz5cnFGwPLly9GuXTv8+uuvGDFihHjsb7/9hmrVqqFjx44F2oiIiIBMJhPP6erqilu3biE8PPylAgISiQTGxuW/LutVZWRkVKWvvzJwzCsWx7viccwrFse74nHMK96rNOZpGSrkaLUwMDB8/sFFEAQBB8/exc6/n96AyqQSDHrbCS3dahVTs2xIJICRgR4sTAx0djjI8yqNd0XicgF6XUgr8+R5SwUePXqkU/7o0SPY2NgUON7W1hZ6eno6ywMMDQ1Rp04d3Lt3T+fYQ4cOoVu3bpBKC16iiYmJTgACABo3boyEhPKeSEVEREREVZ0gCEjNUCJDoYLwEhMDNBotfjoQpRMMMDbQw6T3vSokGCCVSGBmJIeVmWGhwQAievVVakCgadOmMDU1RUREhFiWlpaG69evo0WLFgWOb9GiBdRqNa5evSqWKRQKxMfHo169emJZRkYGIiMj4efnV6CNW7duoVmzZjrnBIBr165Vyen+RERERFRxBEFASoYSmdk5LxUMyFaoEbLtMo7/859YVt3CEDMGe8OxXvlvKyiTSmBpJoeZCdfJE73OKnXJgFwux4cffoilS5fC2toadnZ2WLJkCWxtbdG5c2doNBokJSXBzMwMhoaG8Pb2hp+fH2bOnIkFCxbA0tISq1atgkwm00kcGBUVBUEQ0LRp0wLnbNSoERo2bIgFCxZg/vz5sLKywi+//IJ//vkHO3bsqMjLJyIiIqIqRKsVkJyuhFKlxstkDEhKVSBk+z/4LzFTLGtoZ4GxfdxhZlz+N+hyPSksTA0g15eV+7mIqHxV6gwBAJg0aRL69u2L2bNnY8CAAZDJZAgPD4e+vj4ePHiA1q1bY+/eveLxwcHB8PHxwYQJE9C3b19kZGRg8+bNOtsK5i1BsLS0LHA+qVSKNWvWwN3dHR9//DF69eqFy5cvY+PGjWjSpEm5Xy8RERERVT0arYDkNAUULxkMiHuQhsWbz+kEA5o3rYmP+3uVezBAAsBQLoO1uSGDAURviEqdIQDkbrUyffp0TJ8+vcB79vb2uHHjhk6Zqakp5s2bh3nz5hXZZteuXdG1a9ci369evTq++uqrF+4zEREREVFJaTTa3JkBOZqXauefm4kI/+0actRasSywVT2827YRpOW9k4AEMDbUh4WJnAnziN4glR4QICIiIiJ6U+WotUhJV0D1zE18aQmCgL/OxWPH4Vvi7AKpVIIPujSFv0ftsuloMaQSCcyM9WFaAcsRiKhiMSBARERERFQOctQaJKcpkaN58WCARqvFL4du4ujF+2KZoYEMo3u5w6m+dTE1y4ZMKoGFqRxGBvrlfi4iqngMCBARERERlTFljgYp6QqoNS+eMUChVGPDb9dwLfqJWGZtbogJ/TxQu4ZpWXSzWEweSPTmY0CAiIiIiKgMKVRqpKQrodG+eDAgOU2B1dsv496jDLGsXi1zjO/rDnMTg7LoZpEkAAzkMliaGkAmq/Qc5ERUjhgQICIiIiIqI9nKHKSkq6AVXjwYcPdhOlZvv4zUDKVY5tmkBj56x6Xcn9YzeSBR1cKAABERERFRGcjMViEtM+elggFXbz/Ght3XdHYk6ORTF706OJT7TgJSiQRmJvowNWLyQKKqggEBIiIiIqKXlJ6pQka2Ci+xSgBHLsRj66GbyIsnSCUS9O/cBG297Mumk8WQySSwNDWAoZy3B0RVCX/iiYiIiIheQmqGEpmKHLzoxACtVsD2w7dw+Hy8WGYol2FkTze4NKxWRr0smlxPCkszA+jrMXkgUVXDgAARERER0QsQBAEpGUpkK9UvHAxQqjQI/+0artx+LJZZmRtgfF8P2Nc0K6OeFo7JA4mIAQEiIiIiolLSanODAQqlGi+6SiAlXYnQ7ZdxNyFdLKtrY4ZxfT1gaVbOOwlIABNDfZgzeSBRlcaAABERERFRKWi0AlLSFVCoNM8/uAj3HuXuJJCc9nQnAXeH6hj+risM5OU7dZ/JA4koDwMCREREREQlpNFokZyu1NkFoLT+jXmC9buu6gQUArzroG9AY0il5fu0nskDiehZ/E1ARERERFQCOWotUtIVUKm1L9zGsUv38L8/b4pbE0okwHsdm6BD8zpl1c0iMXkgEeXHgAARERER0XPkqDVITlMiR/NiwQCtIODXv2/j4Nm7YpmBvgwjerjCzaF6WXWzUGLyQDNDyMp5BgIRvV4YECAiIiIiKoYyR4OUdAXUmhdLH6jK0WDj7//i0s1EsczC1AAT+nmgjk057yTA5IFEVAwGBIiIiIiIiqBUqZGcroRG+2LBgLRMJUK3X0HsgzSxzL6mKcb39YCVuWFZdbNQTB5IRM/DgAARERERUSGylTlIzVC9cDDgv8cZCPnlMpLSFGKZa6NqGPGuKwwNyve/4UweSEQlwd8QRERERET5ZGarkJaZIyb/K62o2CSs/fUqspVqsaxdMzu817EJZFJpWXWzUEweSEQlxYAAEREREdEz0jNVyMhW4QUnBuDk5f+w5UAUtP/fgARAn4DGeKtFnXJdx5+bPFAPlmYGTB5IRCXCgAARERER0f9Ly1AhQ6HCi0wM0AoCfjsWjf2n48QyfT0phr/rCs8mNcqwlwUxeSARvQgGBIiIiIioyhMEAamZKmQpcl4oGJCj1uD7PddxPvKRWGZuIsf4vh6oV8u8DHtakFQigbmJPkyYPJCISokBASIiIiKq0gRBQEqGEtkKNV5klUB6lgphO64g5n6qWFa7ugnG9/NANQujsutoIWQyCaxMDWDA5IFE9AL4m4OIiIiIqiytVkByuhIKlfr5Bxfi4ZNMhGy7jMcp2WKZU31rjOrpBiPD8v2vdm7yQEPo65VvkkIienMxIEBEREREVZJGKyAlTQFFjuaF6t+8m4w1O68gS/E0mNDaozYGdHaETFZ+N+lMHkhEZYUBASIiIiKqctQaLZLTFFCptS9U/8y1B/hhbyQ0z2xF0LuDAzr51C3fnQSYPJCIyhADAkRERERUpeSotUhJf7FggCAI+OPEHew5eUcs09eTYmh3ZzRvalOW3SyAyQOJqKwxIEBEREREVYJEIoFao0VGmgI5mtIHA3LUWvywLxJn/30olpkZ62NsHw80tLMoy64WoCeTwJLJA4mojPE3ChERERFVCRKpHpLTVdDTL/0T9ozsHKzZcQW376WIZbbVjDGhnyeqW5bvTgJMHkhE5YUBASIiIiJ646lytEjLzIGZWgM9/dLVTUjKwupt/+BR8tOdBBzrWWFULzeYGJaysVKQADA00IOFKZMHElH5YECAiIiIiN5oWYocJKcroFDllLrurfhkrNlxBZnP7CTg51YLAwObQq88dxJg8kAiqgAMCBARERHRGys9U4WMbJXObgAldfbfh9i89zrUmqd1e7RthMBW9cr1Jl0qlcDcRF6usw+IiAAGBIiIiIjoDSQIAtIyVchU5EAoZSxAEATsPRWL34/HiGV6MimGdHNCC2fbMu6pLiYPJKKKxN80RERERPRG0WoFpGYqka1Qo7TzAtQaLX7cF4Uz1x6IZSZG+hjbxx0O9pZl2s/8DPRlsDA1YPJAIqowDAgQERER0RtDoxWQkq6AQqUpdd1MRQ7W7ryCm3dTxLKaVkaY8J4naloZl2EvdUkkgKGcyQOJqOIxIEBEREREbwS1RovkNAVUam2p6yamZCPkl3+QkJQlljWuY4nRvd1halR+a/mlEsDEUA4zE30mDySiCseAABERERG99nLUGqSkK18oGBBzPxWh2y8jI/vpLgS+Lrb48G2ncp2+L5NKYMbkgURUiRgQICIiIqLXmjJHg5R0hc5uACV1ISoBG3+/DrXmaSChe+sG6ObfoFyf2OvJpLA0lTN5IBFVKv4GIiIiIqLXllKlRnK6stTbCgqCgANn4rDraLRYJpNKMLirE3xda5V1N3UweSARvSoYECAiIiKi11K2MgepGapSBwM0Gi1+OnADJ6/8J5aZGOphTG93NK5rVdbdFEkkgJGBHixMDCBl8kAiegUwIEBEREREr51MRQ7SMlTQCqULBmQr1Vi7OwpRsUliWQ3L3J0EbKzLbyeBvOSB5qbycjsHEVFpMSBARERERK+V9EwVMrJVKOXEAKRlabDtl6tISMoWyxrZW2Bsb3eYGpffjbpMKoG5iRzGTB5IRK8YBgSIiIiI6LWRmqFEpiIHpZwYgLiH6dh+MglZyqfJA72dbDCkmxP09WRl3Mun9GVSWJgZwEC//M5BRPSiGBAgIiIioleeIAhIyVAiW6kudTDg0o1H+O73f5HzzJaEXf3qo3ubhpCW404CBvoyWJoZQE/G5IFE9GpiQICIiIiIXmkarYCUdAWUKg1KEwsQBAGHzt7Fzr9vi/WkUgk+DGwKP/fa5dFVAEweSESvDwYEiIiIiOiVpdZokZKuhDJHU6p6Gq0WWw/exLFL98UyA30Jhr/jDPcmtmXdTZFUApgayWFmwuSBRPTqY0CAiIiIiF5JOWoNUtKVUD0z1b8kspVqbNh9Df/GPBHLrM0NEOhlisZ1LMq6myKZVAILUzmMDJg8kIheDwwIEBEREdErR6lSIzlDCY2mdAkDktIUWL3tMu4nZohlDWqbY1i3Jnj0IL6suymS60lhYWoAOZMHEtFrhAEBIiIiInqlZCtzkJqhgqaU+wrGPUxD6PbLSM1QiWXNHGtiaHdnaNQqPCrrjgKQADCQy2BpagAZkwcS0WuGAQEiIiIiemVkZKuQnpkDbSm3Erh8KxHhv12DKufp8oIuLeuhR7tGkEokyFaXdU9zkwcaG+rDwkQOSTnuVkBEVF4YECAiIiKiSicIAtIyVchU5JR6W8HD5+Ox7dDNpzsJSCQY0MURbTztyryfeWRSCcyM9WFixOSBRPT6YkCAiIiIiCqVVisgNVOJbIW6VNsKarRabDt0C0cu3hPLDOUyjOrlBucG1cq+o/9PXyaFhakcBnL+V5qIXm/8LUZERERElUbz/9sKKkq5raBCpUb47mu4Gv10JwErcwNM6OcJuxqmZd1NAMwXQERvHgYEiIiIiKhS5Ki1SElXlHpbweR0BUK3X0F8QrpYVtfWDOP7esDC1KCsuwkAkP5/vgBz5gsgojcIAwJEREREVOGUKjVSMpRQl3JbwXsJ6QjZfhkp6UqxzKNxdXz0jisM5OWz5Z9MKoGFqRxGBvrl0j4RUWVhQICIiIiIKtSLbit4Lfox1u++BqXq6fKCji3qoncHB0il5fPUXq4nhaWZAfT1yifYQERUmRgQICIiIqIKk6XIDQaUdlvBoxfv4X8Hb4g7EEgkwPudHNG+mX059DK3fUO5HixMDSArp2ADEVFlY0CAiIiIiCpERpYK6VkqlGZigFYrYOfft3Ho3F2xzEAuw4gernBrVL0cepmbL8DUSA5TY33mCyCiNxoDAkRERERU7tIyVMhQqFCaiQFKlQbf/f4vLt9KFMsszQwwoa8H7G3MyqGXzBdARFULAwJEREREVG4EQUBqpgpZipxSBQNSM5QI3X4ZcQ+f7iRQp6YpxvXzgJWZYTn0NDdfgIWpAeT6zBdARFUDAwJEREREVC60WgEpGUoolGqUJmPA/cQMrN52GUlpCrHMrVE1DO/hCkN52f/3VQLAQK4HSzPmCyCiqoUBASIiIiIqcxqtgJR0BRTP7AhQEtfvPMG6XVehUD6t176ZPd7r2KRcdhKQSAATQ32Ym8iZL4CIqhwGBIiIiIioTKk1WqSkK6HMKV0w4Pg/9/HzgRviDgQSAP06NkGAd51y6GVuvgAzEzlMDJkvgIiqJgYEiIiIiKjM5Kg1SElXQqXWlriOVhCw+2g0DpyJE8vk+lIMf9cVHo1rlEc3oSeTwtJUDoNyWIJARPS64G9AIiIiIioTyhwNUtIVUGtKnjFAlaPBpj+u4+KNR2KZuYkc4/t5oJ6teZn3UQLAQF8GSzMD6MmkZd4+EdHrhAEBIiIiInppCpUaKelKaLQlDwakZaoQtuMy7vyXJpbZ1TDF+L4esLYo+50EZDIZTIz0YW1uWC75CIiIXjcMCBARERHRS8lU5CAtQyWu/S+JB48zEbLtHzxJfbqTgHMDa4zs6QYjg7L/L6pUKoGFiQHMTeQMBhAR/T8GBIiIiIjohaVnqpCRrUIpJgYgKjYJa3+9imylWixr42mH/p2bQCYt+2n8MqkElqaGSHtc8rwGRERVAQMCRERERFRqgiAgLVOFTEUOSjExAKeu/Icf90dBq326k0CvDg7o5FO3XLb9k+tJYWFqAHWOEhpN6XY9ICJ60zEgQERERESlotUKSMlQQqFUo6SxAK0g4PfjMdh3KlYs09eT4qN3XODlWLPM+ygBYCCXwdLUADKZFOqcMj8FEdFrjwEBIiIiIioxjUaLlHQlFDklf9qeo9bg+z2ROB+ZIJaZGetjXF8PNKhtUeZ9lEgAY0N9WJjIy2XWARHRm4IBASIiIiIqkRy1BinpSqjUJV+Ln5GlQtjOK4i+lyqW1apugvF9PVDd0qjM+yiVSGBmog9TI3mZt01E9KZhQICIiIiInkupUiMlQwm1puQJAxKSshDyyz9ITMkWy5rWs8KoXm4wNtQv8z7KZBJYmhrAUM7/4hIRlQR/WxIRERFRsbKVOUjNUEFTiq0Ebt1NRtjOK8hSPN1JwN+9NgZ2cYRMVvY7Ccj1pLA0M4C+nqzM2yYielMxIEBERERERcrIViE9MwfaUmwlEHHtAX7YF6kzm6Bnu0bo0rJema/pz00eqAdLMwPIpMwXQERUGgwIEBEREVGhUjOUpdpWUBAE7Dl5B3+cuCOW6cmkGNrdGd5ONmXeP4kEMDHUhzmTBxIRvRAGBIiIiIhIh1YrIDVTiWylusTBgBy1Fj/ui0TEvw/FMlOj3J0EGtqV/U4CUokE5ib6MGHyQCKiF8aAABERERGJNBotUjKUUKhKvq1gZnYO1uy8glvxKWKZjbUxJvTzQA0r4zLvo0wmgZWpAQyYPJCI6KXwtygRERERAch9yp+SrijVtoKJyVkI2XYZCUlZYlmTupYY3dsdJuWwkwCTBxIRlR0GBIiIiOj/2Lvv8KiqrYHDv+npCaEF6TVAGqGE3iICAoo0P5SqdIigdL0IgtKUIhBCL4KAXIodrgUVEOkd6T2hhJqeKZmZ74+YA0MoSZgEkPU+Tx6SPaes2Rkyc9bZe20hMFmsxCUas7Ws4JmYOOasO0RSqkVpqxVYhM4vV0Tr5JUEVICLQYu3hxQPFEIIZ3H+mi85YLPZmDlzJvXr16dKlSr06tWL6OjoB25vsViYOnWqsn3nzp05duyYwzZvvfUW/v7+Dl9dunRRHjeZTIwdO5batWsTGhrKkCFDuHXrVq49RyGEEEKIp1WqycLthOwlA/Yci2X6qv0OyYBX65ehW8tKTk8GqFXg4aonn6wkIIQQTvVUJASioqJYuXIlH3/8MV999RU2m42ePXtiNpvvu/1HH33E+vXrmTBhAuvWrcPX15devXqRmJiobHPixAk++ugj/vzzT+Vr1qxZDsfIaPviiy84e/YsAwcOzPXnKoQQQgjxNElKMROXaMJqy1oywG63s/Gv8yz89ghp1vSpBVqNirdfCaBF3dJOr/avUavw9jDg5SErCQghhLM98YSA2Wxm8eLFDBw4kEaNGlGxYkWmT5/O1atX+fnnnzNtHx0dzbp16xg/fjz169enbNmyfPLJJ+j1eo4cOQLAzZs3uXnzJiEhIRQsWFD58vHxASA2NpZvvvmGUaNGUb16dYKDg5k2bRq7d+9m//79efn0hRBCCCGeCLvdTlySiYQUM1nMBZBmtbF8wzG+3XJGaXN30fJux6qEBfg5PUadRk0+LxfccqEWgRBCiKcgIXD8+HGSk5OpXbu20ubl5UXlypXZvXt3pu23bduGp6cnDRo0cNj+t99+U45x4sQJVCoVpUuXvu859+7dC0CtWrWUttKlS1O4cOH7nlMIIYQQ4t/EarNzK8FESqoly8sKphgtzPrvAf46fEVpK5jPleFda1CuuI/TY3TRafD1dsGgk+KBQgiRW554UcGrV9PXqi1SpIhDe6FChZTH7nbu3DmKFy/Ozz//zPz584mNjaVy5cqMHDmSsmXLAnDy5Ek8PT0ZN24c27Ztw83NjebNm9O/f3/0ej2xsbHky5cPg8GQpXNmld1uJyUl5dEb/sukpqY6/Ctyn/R53pL+znvS53lL+jvvPck+t1rtxCebMZrTsrzPzXgjC747RuytO/GWfsGTHq0q4u6qcurzUKtUuLnqcNGpMZuM3H8CafbJ6zxvPe/9bbfbZYqLeCY88YRAxh8JvV7v0G4wGIiPj8+0fVJSEhcuXCAqKorhw4fj5eXFnDlzePPNN9mwYQP58+fn5MmTmEwmgoODeeuttzh27Biffvoply9f5tNPPyU1NTXT+TLOaTKZcvxcLBZLpuKGz5Pz588/6RCeO9LneUv6O+9Jn+ct6e+8l5d9rlarQaUhMdWK0WTK8siAq7fN/Lg7jlTznR0qFHXhxWBXrl6+6NQY9Totnm56EjV2LqVlPWGRHfI6z1vPc3/f73pDiKfNE08IuLi4AOm1BDK+h/RVAFxdXTNtr9VqSUpKYvr06cqIgOnTp9OwYUO+/vprevbsybhx4xgxYgTe3t4AVKhQAZ1Ox3vvvcfw4cNxcXG5b8HCB50zq3Q6HeXKlcvx/s+q1NRUzp8/T6lSpR6r/0TWSZ/nLenvvCd9nrekv/Pek+hzo9lKQrIZz38KAWbFwVM3+XbHKSx3rT7QrGYxmtUs7vS7n1qtGm93AwZd7sxoldd53nre+/v06dNPOgQhsuSJJwQypgpcu3aNEiVKKO3Xrl3D398/0/Z+fn5otVolGQDpSYXixYsTExMDpCcNMpIBGcqXLw+kT1Hw8/MjLi4Os9nskLm7du0ahQsXzvFzUalUuLm55Xj/Z52rq+tz/fyfBOnzvCX9nfekz/OW9Hfey6s+T0w2Y0qzodMbyEp5Prvdzi+7LrL+9zsXNRq1ii4vV6JWUJGH7Jkzeq0aH08DOm3u1wuQ13neel77W6YLiGfFEy8qWLFiRTw8PNi5c6fSlpCQwNGjR6lRo0am7WvUqEFaWhqHDx9W2oxGI9HR0ZQsWRKALl268P777zvsd/jwYXQ6HaVKlaJatWrYbDaluCCk1yaIjY297zmFEEIIIZ5Fdrud+CQTialZX0nAarWx8qfjDskANxctgzqGOj0ZoAJc9Bp8vV3zJBkghBDC0RMfIaDX6+ncuTNTpkzB19eXokWL8tlnn+Hn50fTpk2xWq3cunULT09PXFxcqF69OnXq1GHEiBGMGzcOHx8fZs6ciUajoXXr1gA0a9aMCRMmEBwcTL169Th8+DCffvopPXr0wMPDAw8PD1q2bMmoUaOYMGECrq6ujBkzhrCwMKpUqfJkO0QIIYQQwglstvRlBY2mNLKYCyDVmMaCbw9z9Nwtpa2AjysRHULwy+/u1PhUKnB30eHlrpe7qUII8YQ88YQAwMCBA0lLS2PUqFEYjUZq1KjBokWL0Ol0xMTE8OKLLzJx4kTatm0LwKxZs5gyZQoREREYjUaqVq3KsmXL8PX1BaBz586oVCqWL1/OhAkTKFiwIN27d6d3797KOT/++GMmTJhAREQEAA0aNGDUqFF5/+SFEEIIIZwszWojLtGEyWLN8j634o1Erj3A5evJSluZot70axeMp5tzi6OpVSq83HW4u0rRNSGEeJJUdntWa8yKh8mYwhAUFPSEI8l7KSkpHDt2jEqVKj2Xc8SeBOnzvCX9nfekz/OW9Hfey80+N1usxCeZMKdlvXjghasJzF5zkITkO0WXq1cqRLeWlZ0+lF+rUeHjYcCgz9v7UvI6z1vPe38/z9cG4tnyVIwQEEIIIYQQj89kTuN2kgmrNev3ew6cvM7i749gttxJIDSvXYpXG5RB7eSh/OnFA13QaZ94GSshhBBIQkAIIYQQ4l8h2WghIcmMLYuDP+12O7/tiWbtplNKjQG1WkWnZhWpG/KCU2NTqcBFr8Xbw4BGLfUChBDiaSEJASGEEEKIZ1xCkpkko5msTgS12mz899dTbN4Xo7S5GrT0aRNExVK+To1NrQJ3Fz2e7jopHiiEEE8ZSQgIIYQQQjyjbDY78ckmUk1pWU4GGE1pLPzuCEfO3FTafL1ciOgQwgsFPZwan0atwstdj5uLzqnHFUII4RySEBBCCCGEeAZZ/1lJwJiNlQRuJxiZvfYgMdeSlLZSRbzo3z4YL3eDU+PTadR4exow6JxblFAIIYTzSEJACCGEEOIZY0mzEpeYvZUELl5NZPbag8QnmZS20AoFeeuVAPROvmh30Wnw9jSg1UjxQCGEeJpJQkAIIYQQ4hmSk5UEDp++wcJvj2C6azTBS2ElaNO4nFNXElCp0msReLsbUEvxQCGEeOpJQkAIIYQQ4hlgt9tJMVpISLZkeSUBgN/3RvPfX08qNQbUKhUdm1agQWixbJ//YUUB1SrwcNXj6a7P1nFz6lHxCCGEeDQZxyWEEEIIkcdatGjB3Llzs7z9pk2bGDxkGPHJd5YV/O3Xn+jYriXNGtdi2qfjeS+iN+9F9Fb2sdns/PfXk6z+5U4ywEWvYUCHkGwlA65fi+X9YQOJvXol02OjRg7myy8WoVGr8PE05EoyICYmBn9/f9avXw9AQkICw4cPZ8+ePco2Xbp0oUuXLvfdfv369fj7+3P58mWnx+Ysd8f/uNasWYO/v3+mr3HjxuVZDEKIZ4eMEBBCCCGEyGPTpk3j6tWrWdrWZrOzYOFibHa7w0oCM6dPplixEoz8z1gKFCzE1MmfKI+ZzFYWfXeEQ6dvKG35PA0M6BBCsUKe2Yp1756d7Ny+LVO7xWJh395d9OzVF18vF6fXIXiQY8eO8e2339KuXTulbcyYMQ/cvlGjRqxevZoCBQoQHx+fFyE+UceOHaN06dJMmjTJob1AgQJPKCIhxNNMEgJCCCGEEHmsYsWK2LMw7D9jJQGbLfO2CfHxVG9biypVqzu0xyWaiFp7kIuxiUpbicKe9G8fgo+n81YSOHxgHx7uHtSsHoJW+2RXEihXrtwDH/P19cXX15eUlJQ8jOjJOXbsGEFBQVSpUuVJhyKEeAbIlAEhhBBCiDyWMWXg8uXL+Pv7s3HjRgYOHEhoaChhYWGMGjWKuPhEbiUY6denBwcP7OXggb2E16vGgX17CK9XDYBlSxYQXq8aV6+kD4c3WdKYvHw3F2MTsVkt3Dz5K1e2TWPzskG80+v/WPXlUmy2OysTWK1WVn25lLe7vE7z8Dq0aFKXiL5vsX/fbgD+t+E7Pp0wFoA3O7zC5PHpd+JVKti7ZzsNGtRHq9Wwc+dO/P392b59O126dCE4OJhGjRqxZs0arl27RkREBKGhoTRs2JClS5cq588Yzh8TE+PQP+Hh4YwcOTJTv+3cuZOuXbsC0LVrV2WI+8OGu987ZWD06NF0796ddevW0axZMwIDA2ndujVbtmxx2G///v106tSJKlWq0KhRI7744gu6d++uxJXxnHfu3Omw372xGI1Gpk6dStOmTQkMDKRq1aq89dZbHDt27L7xPszJkyfp06cPVatWpWrVqgwYMIDo6GjlcbvdzokTJ6hUqVK2j32vW7duMXbsWBo3bkxgYCBhYWEMGDDA4XfVpUsXhg4dysCBA6lSpQpvvfUWANeuXWPEiBH06tWLBg0aMHr0aKZPn054eLjDOdasWUPLli0JDAykUaNGzJo1C6s168toCiEenyQEhBBCCCGesDFjxlC0aFGioqLo0aMHa9euZeas2ZjTbAwaMpJyFfwpV8GfyLlLKe9fkci5SwFo0ao1kXOX4pu/AMlGC9GxSdxOMGG327m8eykJ57fwRsf/Y/yn02nYuAmLFkQx/bMJynkXzJ3F8qULaNW6LZOnzmLI8FEkJsQzdtQIjMZUatWuT+duPQAYO/4zOnfviVoFXm56dvz1Jw0bNnR4HoMHDyY8PJx58+ZRunRpxowZQ9euXSlfvjxRUVEEBwczceJEDh06lKN+CggIYPTo0UD6hf3Dpgo8zJEjR1i0aBEDBw5k9uzZaDQa3nnnHWVKwZkzZ+jevTuQPr3jnXfeYf78+ezduzfb5xo+fDjr1q2jd+/eLF68mPfff59Tp04xZMiQLI0SyXDu3Dk6duzIzZs3mTx5MuPHjyc6Opo33niDmzdvAnDx4kWSk5M5fPgwzZo1IyAggGbNmvHNN99kK2a73U6fPn3Ytm0bQ4cOZdGiRURERLB9+/ZMfb5x40bc3d2ZM2cOPXv2xGw2061bNw4ePEjXrl0ZO3Ysx48fZ/HixQ77zZs3jw8//JDatWszd+5cOnXqxIIFC/jwww+zFasQ4vHIlAEhhBBCiCesYcOGjBgxAoCgKtX4/Y8t/LVtCz36RFCqdBnc3TwAqBwY5PBvgYKFqRwYxJb9MVy6ngT/XF+mXD9Byo1TjPpoAuFNmgFQvUYtDAYXliycQ9sOb1C6TFlu3rhOj94DaNu+oxKLzmDgo/8M4+zp01QODOKFoukFCMtXqEjRokXx9tBz49pVoqOjqVu3rsPzaNeunXKX2M3Njddff53g4GAGDRoEpE+V+Pnnn9m3bx/BwcHZ7icPDw9lekC5cuUeOlXgYRITE1m/fj0lSpRQYu3cuTM7duygWbNmzJs3D09PTxYuXIirqysAZcqUoWPHjg87bCZms5nk5GRGjRpFixYtAAgLCyMpKYlJkyZx48YNChYsmKVjRUZG4urqytKlS/HwSH891K5dmyZNmrBw4UJGjBihjDqIiYlh5MiRaLVavvnmG0aMGIHZbOb111/P0rmuXbuGq6srI0aMoHr19CkpNWvW5OLFi6xevdphW51Ox9ixY9Hr0wtKrl27lrNnz7JixQpUKhWVKlWiYcOGNGnSRNknMTGRqKgo/u///o9Ro0YBUK9ePXx8fBg1ahRvvfUW5cuXz1KsQojHIwkBIYQQQog8YLPZOXspnoRkM2lWu8Pd4SpVqmC320lINpNstFCgYGGuXslc1f9eduys++0Uv+y6qCQDDDoNxQokclWjoWHjJg7bv9SsBUsWzuHQgb2ULlOW/4wZD0Dc7dtEXzxPTEw027elD523WMwO+2o1aqV44JYtWwgNDcXT07FAYWhoqPJ9/vz5AQgJCVHa8uXLB6RfED5Jvr6+SjIAwM/PD4DU1FQAduzYQYMGDZRkAKQ/t6JFi2brPHq9nkWLFgEQGxvLuXPnOH/+PL///juQnjC4l81mc5jWoVKp0Gg07Nixg7CwMFxcXEhLSwPSEyTVq1fnr7/+AqBGjRrMnTuXmjVr4ubmBkD9+vW5desWM2fOpEOHDthsNofXXsbx71a4cGGWLVuG3W4nJiaGCxcucPbsWfbt25cp5jJlyijJgIy+K168OJUrV1YSFB4eHjRu3FiZXrF//36MRiPh4eHKcwGUKQXbtm2ThIAQeUQSAkIIIYQQuezgqeus/e0Ul64lkWa1kZBs5sxVEycuxgFgcHHhVoIJkzkNO6BWq7DZbQ89JsC+Y9ewXL+o/KzVqBnSqRr//WIr3j4+mS70fH3TL9KTkpIAOHH8KJ9PncSJY3/j4uJCydJlKVw4/eL43uHs+TwNykoCmzdvpkGDBpniybhzfbe7L6qfFvfGpFKpAJQL8Vu3bikJjbvlpFL/1q1bmTBhAmfPnsXd3Z2KFSsqF+v3mzLwwQcf8PXXXys/Fy1alN9++424uDg2bNjAhg0bMu3j6+sLpCdhGjdunOnxhg0b8tdff3Hjxg0GDx7Mrl27lMfCwsJYvnx5pn2+++47pk2bxpUrV/Dx8aFSpUq4uLhk2s7d3d3h59u3b9+37+5ui4uLA6B3796ZtoP0EQpCiLwhCQEhhBBCiFx08NR1Zq89SKoxDU93HTqNDpUKjGYbazadASA5xYLRnPaII90Rn2QC4MrNZDKuUQ16DUULelDCzxMvT2/i4+KwWq0OSYGbN9OXIfT29iE5OYkRQyIoU7Y8i5evoUTJUqjVanZs/5Mtf2wC0osHZiQB1Or0i2aTycTOnTsZMmTI43UMmS/EMyQnJz/2sR+Hn58fN27cyNR+8+ZNypQpAzw89oyL5IsXLzJgwACaNGnCvHnzKF68OCqVihUrVrB169b7njsiIoJOnTopP2fcfff09KROnTrKlIy7abXpH+n37NlDdHQ0bdq0cXjcZDKh0Wjw9vZm7NixDv177wV9xnFGjBhBly5d6NGjB4ULFwbg008/fWQdhcKFC3P+/PlM7Rl1DgC8vLwAmDJlCqVKlcq0rSyRKETekaKCQgghhBC5xGazs/a3U6Qa08jvbcCg0ygX1gadGhdD+sW2xfrw0QBqzZ2PbJevJzF52R6HxwPL5qd4YU+0/2wXHFoVq9XK5t9/ddju15/T7y4HBlfh4oXzJMTH067DG5QqXQa1On3fXTv+yogeT1c9nm6OSxXu3LkTHx8f/P39s9ET95cxouDq1atK25kzZ5Q7yPdz76iH3FCjRg22bt2KyWRS2o4ePepQYf9+scfHx3PmzBnl5yNHjmAymejduzclSpRQkggZyYD7jRAoVqwYQUFByldGP4eFhXH69GkqVaqkPBYYGMjSpUv55ZdfgPTh+iNHjuTcuXPK8Ww2Gz/99BOhoaHo9XrKlCnjcPyMBMfd9u/fj81m45133lGSAVarVZmacG8S5G5hYWHExMRw4sQJpc1oNDokQEJCQtDpdMTGxjrEotVqmTZtWqZVJ4QQuUdGCAghhBBC5JKzl+K5dC0JT3edcjEIoFap0Os0qP6Z+H8r3vjQ43h4eHL0yCG++XETm46YsdjvzNluWLUorzepwNDNd45fs1ZdqlStztTJn3Dj+jXKlqvAwQN7WfXlUpq93IpSpcuQlJSIu7s7Xy5bjEajQaPVsuX3TWz88dv0GO1WPN31yt3cX375hQYNGrBly5b7ThfIiZo1a+Li4sKkSZMYNGgQycnJzJw5Ex8fnwfuk1G34I8//sDb25uKFSs6JZa79e3blw0bNtCzZ0/efvttEhISmDFjBmq1Wvk9+vv7U6RIEWbPno2HhwcqlYp58+Y5TEcICAhAq9Xy2Wef8fbbb2M2m1m/fj1//PEHACkpKVmOqX///nTs2JE+ffrwxhtvYDAYWL16Nb/++iszZ84EoGPHjnz11Vf07duXQYMG4erqysqVKzl58iQrVqzI8rkyCj6OGzeOdu3aER8fz4oVKzh+/LgS9/2mhwC0atWK+fPn895779GmTRuuX7/OypUruXnzJi+88AKQXkuiZ8+ezJgxg6SkJGrWrElsbCwzZsxApVLlyu9UCHF/MkJACCGEECKXpBcQtKG76w6/p5selQosaVZSjOlrrhvND197/bV2r5NmUzFr8vvcjPlbaa9cOj8dX/JHo3b8SKdSqZjw6ee0at2Wtf9dyQfDB7H591/p2SeCYe+nLxvn4eHJxxOngd3O2A9HMOnj0VyLvcqsqEW4u7tz+NB+IP2ivU6dOkydOpXJkyezZcuWTMsN5pSXl5ey9vyAAQOYMWMGAwYMIDAw8IH7lC9fnlatWrFixQqGDh3qlDjuVbJkSRYtWoTJZGLgwIFMnz6dXr16UbBgQWWIvUajYebMmRQoUIDBgwczfvx4WrZsSdOmTR2OM3XqVGJjY+nXr5+yZOLy5ctRqVTs2bPnvue/n4oVKyqV+4cPH87AgQO5fv06s2fPVs5ZoEABVqxYgb+/P5988gnvvvsuqampLF261KG446PUrFmT0aNHs3//fnr16sWkSZN44YUXiIyMBHjotAGtVsuiRYuoVKkSixcvZtSoUZQvX56XXnpJqZ0A8O677zJy5Eh++eUXevXqxWeffUa1atX48ssvMxWrFELkHpU9Owugigc6fPgwAEFBQU84kryXkpLCsWPHqFSpksMfepF7pM/zlvR33pM+z1vS37nndHQcE5buwsWgwUWvwctNj1qt4naikfjEFFRqLZY0O33bBlOyiNd9j2Gz2/l28xl+2nFBadNp1fR4NZAqFbK2ZF1WGXQavD0M6LT/vntG2Xmdb9++HZ1Opyy5B5CQkECdOnUYPnw4Xbt2ze1wn1mnTp3i7Nmz1KtXj+PHjyv93b59e/z8/JSkwr/d83xtIJ4tMmVACCGEECKXlCnqTdFCHkRfS6RwPlfsqLiVYMRksWK320k1WSlWyIPihe9/R9RssfLFj0fZe/xO1XUvdz3924dQ6gEJhJxQAS4GLd4eBjRq1SO3/7f7+++/mTlzJoMHDyYgIIC4uDiWLFmCp6cnrVq1etLhPdVSUlIYNGgQr7/+OuXKlSMuLo5NmzZx5MiRXBvRIYTIOUkICCGEEELkErVaRYcXy/PVLye4mWjGbEpDpU6/0E822nB31dK8Viml0ODdElPMzFl3iLOX4pW2Fwq4M6BDCPm9nbeUn0oF7i46vNz1DnUOnmcZ8/1XrVrFlStXcHNzIywsjIkTJypL/In7CwkJ4fPPP2fBggV89913qFQqKleuzMKFC6lVq9aTDk8IcQ9JCAghhBBC5KLyxX1oUbs03209Q1xCGlabDbVKRX4vLa82KEfFUpkvMK/eTCZyzUFuxKUqbZVK+dL7tSBcXZz38U2tUuHlrsPdVf/ojZ8jarWa/v37079//ycdyjOpefPmNGjQQKYiCfEMkISAEEIIIUQuSUoxk5hioXRRb955PZTo2ESSUizoNDZMiVcpV9w70z4nLtxm3vpDpJjSlLZ6IS/wRlN/NBrnze3XaFTk8zBg0MvHQSGEeF7JO4AQQgghhJPZ7Xbik82kGC1klG9Wq1VK4cDU1FTOJGUenr/j8BWWbzyG1Xan5nPbxuV4KayEU4fz67VqfDwN6LQapx1TCCHEs0cSAkIIIYQQTmS12YlLNGIyW8nqUk52u50f/jzHj9vOKW06rZrurSpTrWJhp8WmAgx6DT6eLlI8UAghhCQEhBBCCCGcxZJmJS7RhDnNlo19bCzbcJTdR2OVNk83Hf3ahVCmaOYpBTmlUoGbiw5vKR4ohBDiH5IQEEIIIYRwApM5jdtJJqzWrI4LgKRUC1+s+5vTMXdWEvDL70ZEhyoU8HHeSgJqlQpPNx0eblI8UAghxB2SEBBCCCGEeEzJqWYSki3Y7FlPBsQlpfHV6sPciDcqbf4l89G7TRDuLjqnxaZRq/D20ONqcN4xhRBC/DtIQkAIIYQQIofsdjsJyWaS7yoemBVnLiWwZtstTJY7O9UJKsKbzSuideJKAnqtGm8PA3qdFA8UQgiRmSQEhBBCCCFywGazE59sItWYluXigQC7/r7KFxuOOkwtaN2gLM1rl3Tq3H4XnQYfT4NTlyoUQgjx7yIJASGEEEKIbLJabcQlmjBarFnex263s+Gv83y/9azSptWo6NayMjUq+zktNpUKXA1avN0NqGUlASGEEA8hCQEhhBBCiGywpNmISzRmayWBNKuNLzceZ8eRK0qbi05Fr9cCCCjrvGUF1SrwcNXj6S7FA4UQQjyaJASEEEIIIbLIZE4jLslEWjZWEkg2Wpi3/hAnL8YpbQV9XGgW6k6ZF7ycFpsUDxRCCJFdkhAQQgghhMiCVJOF+CQzVlvWkwHX41KZveYAV2+mKG3li/vQrUV5rl666LTY9Fo1Pp4GdFopHiiEECLrJCEghBBCCPEISalmErO5rODZS/HMWXeQxBSL0lYzwI/OL1cizWJySlwqFbjotXh7GNBIvQAhhBDZJAkBIYQQQogHyOmygnuPx7L0h6NY7qoz0KpeaVrWLY1KpSLN8pCds0jqBQghhHhckhAQQgghhLgPq81OXKIJkznrywra7XZ+3nmBr/84o7Rp1Cq6tqhEzcAiTotN6gUIIYRwBkkICCGEEELcw5JmJS7RlK2VBKxWGyt/PsG2g5eVNjcXLf3aBlO+RD6nxSb1AoQQQjiLJASEEEIIIe5i/GclAWs2VhJINaYx75vDHD9/S2kr6ONKxOtVKOzr5pS4pF6AEEIIZ5OEgBBCCCHEP5JSzCSmZK944I24VGavPciVG8lKW9li3vRrG4yHm3Pm96tVKjxcdVIvQAghhFNJQkAIIYQQzz273U58spmUbBYPPH8lgai1B0lINitt1SsVplvLSk4b0q/RqPDxMOCil49tQgghnEveWYQQQgjxXLNabenFAy3WLBcPBNh/4hqLv//bYSWBl+uU4pX6ZVCrnDOk36DT4O1hQKdVO+V4QgghxN0kISCEEEKI55bZYiU+KXvFA+12O7/uusj6308rCQS1WkXn5hWpE/yCU+JSqcDNRYeXmx611AsQQgiRSyQhIIQQQojnUqrJQnySGast6+MCrDYbq385yZb9l5Q2V4OWPm2CqFjK1ylxqVUqvNx1uLtKvQAhhBC5SxICQgghhHjuJCabSUo1k41cAKmmNBZ+e4S/z95U2vJ7uzCgQwgvFPBwSlwatQofT6kXIIQQIm/Iu40QQgghnhs2m534ZBOpprRsFQ+8lWBk9pqDXLqepLSVfsGLfu2C8XI3OCU2nUaNt6cBg845xQiFEEKIR5GEgBBCCCGeCxnFA40Wa7b2u3A1fSWB+KQ7KwlU9S9E91aV0Tvp4l2vVePj6SLFA4UQQuQpSQgIIYQQ4l/PkmYlLjF7xQMBDp66zqLvjmC23NmvWa2StG5Y1ikrCahU4KLX4OPpgkaKBwohhMhjkhAQQgghxL+ayZzG7SQTVmt2FhWE3/ZEs+bXk3dWElCpeKOZP/WrFHVKXBqNGncXHb5eLqictEyhEEIIkR2SEBBCCCHEv1ZOVhKw2eys2XSS3/fGKG0uBg29Xwuicun8TolLrVbh5e6Cl7tekgFCCCGeGEkICCGEEOJfKSnFTGJK9lYSMJrTWPTt3xw+c0Np8/VKX0mgaEEnriTgYSDhRvamLwghhBDOJgkBIYQQQvyr2O12EpLNJBst2VpJIC7RxOy1B4mOTVTaSvp50r99CN4ezl1JwGoxYbVmr7ihEEII4WySEBBCCCHEv4bNZicuyYTRlEZ2KgbExCYye+1BbiealLaQ8gV5+5UADPrHX0lABRj0Gnw8DGg0alIsj31IIYQQ4rFJQkAIIYQQ/wo5XVbwyJkbLPj2CCbznf2a1ChB28blUDuh8r9KBe4uOqkXIIQQ4qkjCQEhhBBCPPNyuqzg5n0xrP7lJLZ/5haoVPB/L/nTqGoxp8SlVqnwctfh7qp3yvGEEEIIZ5KEgBBCCCGeaTlZVtBms7P+99P8uvui0mbQa+jZOpCgsgWcEpdWk1480KCXj1tCCCGeTvIOJYQQQohnVrLRQkKSWbnDnxUms5XF3//NwVPXlTYfTwMR7UMoVtjTKXEZdBq8PQzotGqnHE8IIYTIDZIQEEIIIcQzKSHJTLIxe8sKxieZiFp7kAtX76wkULyQB/07hJDP0+WxY1IBLgYt3h4GNE6oPyCEEELkJkkICCGEEOKZYrPZiU82kWpKy9aygpeuJzF7zUFuJRiVtqCy+enROhAXJwzrV6vA3UWPp7tOigcKIYR4JkhCQAghhBDPjJyuJHD03E3mf3MYo+nOfo2rFaPDixWcspKARq3C012Pu4vusY8lhBBC5BVJCAghhBDimWC2WIlPyv5KAlsPXGLVTyccVhLo8GIFwqsXd0pcWo0aHw+9FA8UQgjxzJF3LiGEEEI89VJNFuKTzFizUTDAZrfz7eYz/LTjgtKm16np+WogweULOiUug06Dj6cBrUaKBwohhHj2SEJACCGEEE+1pFQzCUlmslEuALPFytIfjrLvxDWlzdtDz4D2IZTw88p2DHa7PVNdABe9Bh9PlydSPPB+8QghhBDZJelsIYQQQuSp8PBwRo4c+cjt7HY7cUkmNv7vFyZ8PNrhsd9+/YmO7VrSrHEtpn06nvcievNeRG8AEpLNTF+1zyEZULSgByO61sh2MuD6tVjeHzaQ2KtXlDYV4OaiZdTIwcyfNzdbx8uumJgY/P39Wb9+PQAJCQkMHz6cPXv2KNt06dKFLl263Hf79evX4+/vT0xMTK7G+Tjujv9xrVmzBn9//0xf48aNe+h+I0eOJDw83CkxCCHEs0RGCAghhBAiT0VGRuLh4fHQbaw2O3GJRkxmK2u+WpHp8ZnTJ1OsWAlG/mcsBQoWYurkTwC4fCN9JYGb8XdWEggok5+erQNxNWT/Y8/ePTvZuX2b8rNKBe4uOlx0sGPHDt55551sH/NxHDt2jG+//ZZ27dopbWPGjHng9o0aNWL16tUUKlQoL8J74o4dO0bp0qWZNGmSQ3uBAgWeUERCCPF0k4SAEEIIIfJU5cqVH/q4Jc1KXOLDiwcmxMdTvW0tqlStrrSlGC18tnwvqaY0pa1BaFH+76UKaNSPPyhSrQIPVz2e7nr++usv3N3dqVSp0mMf93GVK1fugY/5+vri6+ubh9E8WceOHSMoKIgqVapkeiwlJSXvAxJCiKecTBkQQgghRJ7KmDKQMbx948aNDBw4kNDQUGrUqMGI9/9DfGIyAO9F9Obggb0cPLCX8HrVOLBvD+H1qgGwbMkCwutV4+qVy8Qnm4i5nqQkA+xWC4XT9vHTFx/QokldunR8jVVfLsVmu5NksFqtrPpyKW93eZ3m4XVo0aQuEX3fYv++3QD8b8N3fDphLABvdniFqZPH4emuB2Dz5s3Ur18flUrFzp078ff3Z/v27XTp0oXg4GAaNWrEmjVruHbtGhEREYSGhtKwYUOWLl2qnH/z5s2EhoZmGs7/oCkVO3fupGvXrgB07dpVGWb/sCH3904ZGDlyJN27d2fdunU0a9aMwMBAWrduzZYtWxz2279/P506daJKlSo0atSIL774gu7duytxZTznnTt3Oux3byxGo5GpU6fStGlTAgMDqVq1Km+99RbHjh27b7wPc/LkSfr06UPVqlWpWrUqAwYMIDo6Wnncbrdz4sSJx0rSrF69mkaNGhEcHEy3bt04evSow+O7d++mR48e1KhRg8DAQMLDw5k1a5bD6+qHH37g1VdfJTg4mFq1ajF06FBiY2MdjrNmzRpatmxJYGAgjRo1YtasWVit2VtKUwghnEESAkIIIYR4osaMGUPRokWZOn0G//dGVzZ8/w1ffrEQgEFDRlKugj/lKvgTOXcp5f0rEjl3KQAtWrVm5twlbDuWQOzNFDKqDmo1Kriwlj2bv6NFq9cYP3k6DRs3YdGCKKZ/NkE574K5s1i+dAGtWrdl8tRZDBk+isSEeMaOGoHRmEqt2vXp3K0HANM+n8HAdyKUfTdv3kzDhg0dnsfgwYMJDw9n3rx5lC5dmjFjxtC1a1fKly9PVFQUwcHBTJw4kUOHDuWonwICAhg9Or2WwujRox86VeBhjhw5wqJFixg4cCCzZ89Go9HwzjvvEB8fD8CZM2fo3r07ANOmTeOdd95h/vz57N27N9vnGj58OOvWraN3794sXryY999/n1OnTjFkyBDs9qyXiTx37hwdO3bk5s2bTJ48mfHjxxMdHc0bb7zBzZs3Abh48SLJyckcPnyYZs2aERAQQLNmzfjmm2+ydI6rV68SGRnJu+++y7Rp04iPj6dLly5cvnwZgOPHj9O9e3d8fHyYPn06c+bMoXr16kRGRrJx40YA9u7dy/Dhw2natCkLFizg/fffZ8eOHQwZMkQ5z7x58/jwww+pXbs2c+fOpVOnTixYsIAPP/wwy/0hhBDOIlMGhBBCCPFENWzYkL4D3iXZaME/oCq7d+9gx19b6dX3HUqVLoO7W3q9gcqBQQ7/+uYvyI4zavYcu6Qcy9NNR/2yRj7/dj+jPppAeJNmAFSvUQuDwYUlC+fQtsMblC5Tlps3rtOj9wDatu+o7K8zGPjoP8M4e/o0lQODKF68OAAhQYEUK1YMgOjoaKKjo6lbt67D82jXrh1vvfUWAG5ubrz++usEBwczaNAgACpWrMjPP//Mvn37HjrM/0E8PDyU/cqVK5ejYwAkJiayfv16SpQoocTauXNnduzYQbNmzZg3bx6enp4sXLgQV1dXAMqUKUPHjh0fdthMzGYzycnJjBo1ihYtWgAQFhZGUlISkyZN4saNGxQsmLXlHyMjI3F1dWXp0qVK/YnatWvTpEkTFi5cyIgRI5RRBzExMYwcORKtVss333zDiBEjMJvNtGrV6qHnsFqtzJ49m+DgYABCQkJo0qQJy5cvZ8SIERw/fpw6derw2Wefof5nCkrdunX57bff2LlzJy1btmTv3r24uLjQu3dv9Pr00SQ+Pj4cPnwYu91OUlISUVFR/N///R+jRo0CoF69evj4+DBq1Cjeeustypcvn61+FkKIxyEJASGEEELkOpvNztlL8SQkm7Gk2RzuDleoGEhyqkVZVrBgwcLEXrly/wPdZfuRq2iK3BmKrddpGNG1ButXLkCj0dCwcROH7V9q1oIlC+dw6MBeSpcpy3/GjAcg7vZtoi+eJyYmmu3b0ofOWyxmdBo17i66TOfdsmULoaGheHp6OrSHhoYq3+fPnx9Iv6jMkC9fPiD9gvxJ8vX1VZIBAH5+fgCkpqYC6cUSGzRooCQDIP25FS1aNFvn0ev1LFq0CIDY2FjOnTvH+fPn+f3334H0hMG9bDabw/B7lUqFRqNhx44dhIWF4eLiQlpa+rQQDw8Pqlevzl9//QVAjRo1mDt3LjVr1sTNzQ2A+vXrc+vWLWbOnEnLli2x2WykpaUpx8g4PkDx4sWVZABAwYIFqVKlCrt3p08hee2113jttdcwmUycO3eOCxcucOzYMaxWKxaLRYlh+vTptGrVimbNmtGwYUPq1aunjCbZv38/RqOR8PBwJQZAWeFg27ZtkhAQQuQpSQgIIYQQIlcdPHWdtb+d4tK1JNKsNuKTTBw8dZ0TF28DoNbquHvwuFqtwmZ/cEHBqzfT6wvcTjBSoEh6m5uLliIF3Cng40pCYjzePj7KhV4GX9/0i/SkpCQAThw/yudTJ3Hi2N+4uLhQsnRZChdOvzjWqFXk83JBo8k8u3Lz5s00aNAgU/v9Vk64+6L6aXFvTCqVCkC5EL9165aS0LhbTir1b926lQkTJnD27Fnc3d2pWLGicrF+vykDH3zwAV9//bXyc9GiRfntt9+Ii4tjw4YNbNiwIdM+GUUT8+fPT+PGjTM93rBhQ/766y9u3rzJ+PHjHeoXhIWFsXz58gc+v/z583Pln+SU0Wjk448/5ttvvyUtLY1ixYoRGhqKVqtVnktoaCjz589n6dKlLFmyhPnz51OgQAH69u1Lly5diIuLA6B379737a9r167dt10IIXKLJASEEEIIkWsOnrrO7LUHSTWm4emuQ6fRoVapsKTZWPfbqWwf7+TF28xd7zgHv27wC+yN8UBF+oWtl6c38XFxWK1Wh6TAzZs3APD29iE5OYkRQyIoU7Y8i5evoUTJUqjVanZs/5Mtf2zCy12PTps5GWAymdi5c6fDnPCcuvdCPENycvJjH/tx+Pn5cePGjUztN2/epEyZMsDDY3d3dwfS5/QPGDCAJk2aMG/ePIoXL45KpWLFihVs3br1vueOiIigU6dOys8Zw+49PT2pU6eOMiXjblpt+sfZPXv2EB0dTZs2bRweN5lMaDQavL296dGjB35+fri4uAAosQJKDYW7Xb9+XUk4jB8/np9++onPP/+cOnXqKImN2rVrO+xTv3596tevT2pqKjt27GDZsmV88sknhISE4OXlBcCUKVMoVapUpvPJ8ohCiLwmRQWFEEIIkStsNjtrfztFqjGN/N4GDDoNHq461GoVajXE3k5fBu5RteXU/9yl33HkCjO+2k+K8c5Q69calqXzyxWVZABAcGhVrFYrm3//1eE4v/6cfnc5MLgKFy+cJyE+nnYd3qBU6TLKnPB9u7b/s3V6UOp7livcuXMnPj4++Pv7Z68z7iPjTv3Vq1eVtjNnzih3ke/n3lEPuaFGjRps3boVk8mktB09etRhNYSM0RB3xx4fH8+ZM2eUn48cOYLJZKJ3796UKFFCSSJkJAPuN0KgWLFiBAUFKV8Z/RwWFsbp06epVKmS8lhgYCBLly7ll19+AdKnOowcOZJz584px7PZbPz000+Ehoai0+l44YUXCAgIUI6RkeCA9MKFFy9eVH6+cuUK+/fvp2bNmkB6wcCaNWvSpEkTJRlw5MgRbt26pSRGJk+eTLt27bDb7bi6utK4cWNGjBgBwOXLlwkJCUGn0xEbG+vwPLVaLdOmTcu04oQQQuQ2GSEghBBCiFxx9lI8l64l4emuQ6VS4emmR6tVYbPbsaTZcdOnfwy5FW986HE8PDzYt+8AUUu/w+D1Ahp9+sVY1YqFaF67VKbta9aqS5Wq1Zk6+RNuXL9G2XIVOHhgL6u+XEqzl1tRqnQZkpIScXd358tli9FoNGi1WrZt/Z3vv00frp4xnz7jju4vv/xCgwYN2LJly32nC+RE5cqVcXFxYdKkSQwaNIjk5GRmzpyJj4/PA/fJqFvwxx9/4O3tTcWKFZ0Sy9369u3Lhg0b6NmzJ2+//TYJCQnMmDEDtVqtXNT7+/tTpEgRZs+ejYeHByqVinnz5jlMRwgICECr1fLZZ5/x9ttvYzabWb9+PX/88QcAKSkpWY6pf//+dOzYkT59+vDGG29gMBhYvXo1v/76KzNnzgSgY8eOfPXVV/Tt25dBgwbh6urKypUrOXnyJCtWrHjkOQwGA/369eO9997DarUyY8YMfHx86NatGwDBwcFs3LiRVatWUbZsWY4fP86cOXNQqVTK66VWrVosWbKEkSNH8uqrr2KxWFi4cCE+Pj7UqlULHx8fevbsyYwZM0hKSqJmzZrExsYyY8YMVCpVrvw+hRDiYSQhIIQQQohckZBsJs1qQ6/V4e2hR61SEZdoUu4MazTpF5dG84PXX7ek2XApUhNT2n4u7VpE4ZDXeaFcGAAvFMg8Zx/Sh7NP+PRzliycy9r/riQ+7jZ+RV6gZ58IOnTsDICHhycfT5zGvKgZjP1wBO7u7lSuXJkvv/ySXr16sWfPHsLDw6lZsyZ16tRh6tSpbN++nfPnzzN8+HCn9I+7uztTpkwhMjKSAQMGULRoUSIiIh66TF758uVp1aqVMuz+hx9+cEosdytZsiSLFi3i008/ZeDAgeTPn58+ffowZ84cZYi9RqNh5syZTJgwgcGDB1OgQAG6devG2bNnlTv0JUuWZOrUqURGRtKvXz+8vb2pUqUKy5cvp0uXLuzZsyfLIy0qVqzIihUrmD59OsOHD8dut1OhQgVmz57Niy++CKQPt1+xYgVTp07lk08+ITk5maCgIJYuXUpISMgjExCVK1emWbNmfPTRRyQmJlK7dm0++OADZcrAyJEjsVgsfP7555jNZooVK0a/fv04ffo0v/32G1arlYYNGzJlyhQWL15MREQEKpWKatWqsWzZMiXR8+6771KwYEFWrlzJwoUL8fb2pnbt2gwePDhToUohhMhtKnt2FoEVD3T48GEAgoKCnnAkeS8lJYVjx45RqVIlZQidyF3S53lL+jvvSZ/nrdzq79PRcXy6fDeFfd1Qq9OTAWnWO3POzRYbJouVvm2DKVnEK9P+yakW5q4/xKnoOKWtsK8bEa9XoaCPc4r1qVXg6abHw03vlONl1dP8Gt++fTs6nY7q1asrbQkJCdSpU4fhw4fTtWvXJxhdzj3Nff5v9Lz39/N8bSCeLTJCQAghhBC5okxRb/xL+nLhagI2q81hJQG73U6yyULRgh4UL5z5rui12ynMXnOQ2Ft37upWKOFDn7bB910KMCc0ahXeHnpcDc453r/F33//zcyZMxk8eDABAQHExcWxZMkSPD09adWq1ZMOTwghhBNJQkAIIYQQucJoTiMswI+j526Sak7D3aBDq1WRlpaeDHDVa2leqxRqtcphv9MxccxZd4jkVIvSViuwCJ1froj2PssA5oROo8bbM73QoXCUMd9/1apVXLlyBTc3N8LCwpg4caIyfF4IIcS/w1ORELDZbERGRrJmzRoSExOpUaMGo0ePpnjx4vfd3mKxMHPmTL755hsSExMJDAzkP//5D5UqVVK2WbduHUuXLiU6OppChQrRvn17evTooVTn/e677xg2bFimY2/atIlixYrlzhMVQgghngN2u53EZAvJRjNlinrzZrOK/G/HeWJvppBisqFRqyla0IPmtUpRsZTjBebuo1f54sdjDlMLXq1fhpfrlFIK2j0ug06Dj6fBacmFfxu1Wk3//v3p37//kw5FCCFELnsqEgJRUVGsXLmSSZMm4efnx2effUbPnj35/vvvlfVn7/bRRx/xxx9/MGnSJF544QVmzJhBr1692LhxI56ennz33XeMGTOGDz/8kNq1a3PkyBE+/PBDzGYzERERAJw4cYKwsDCmTZvmcGzJfAshhBA5Z7XZiU8yYTSlKVMEKpbypUKJfETHJpKUYsHDTUfxwp4OIwPsdjsbt5/nuy1nlTatRkXXFpUJC/BzSmwqwMWgxdvDgEbtnOSCEEII8Sx74gkBs9nM4sWLGTp0KI0aNQJg+vTp1K9fn59//jnTXLXo6GjWrVvH3LlzqV+/PgCffPIJr732GkeOHKF27dqsWrWK1157jf/7v/8DoESJEpw7d441a9YoCYGTJ0/i7+9PwYIF8+7JCiGEEP9iljQrcYkmzGm2TI+p1ar7Fg4ESLPaWPG/42w/fEVpc3fR0rddMOWL53NKbCoVuLvo8HLXO22kgRBCCPGse+IJgePHj5OcnEzt2rWVNi8vLypXrszu3bszJQS2bduGp6enwxrAXl5e/Pbbb8rPQ4cOzXSnX61WEx8fr/x84sQJwsPDnf10hBBCiOeS0ZxGXJIJqzV7ixelGC3M+/owJy7cVtoK5nMlokMVCvs6pzK5SgWerno83fN2JQEhhBDiaffEEwJXr14FoEiRIg7thQoVUh6727lz5yhevDg///wz8+fPJzY2lsqVKzNy5EjKli0LQLVq1Rz2SUxMZNWqVcqIgvj4eGJjY9mzZw8rV67k9u3bBAcHM2zYMEqXLp3j52K32x+5xu2/UWpqqsO/IvdJn+ct6e+8J32etx63v1OMaSSmmLHaspcMuBlvZMF3x4i9dee8pV/wpEeriri7qpzy+1erVXi56dGo0khJSXvs4zmLvMbznvR53nre+9tut8toJPFMeOIJgYw/EvfWCjAYDA539DMkJSVx4cIFoqKiGD58OF5eXsyZM4c333yTDRs2kD9/foftk5OT6d+/PyaTieHDhwNw6tQpIP0/6sSJEzEajcoxvv/+ewoUKJCj52KxWDh27FiO9v03OH/+/JMO4bkjfZ63pL/znvR53spuf2u0WoxmSE41k2a1Zmvfq7fN/Lg7jlTznSRChaIuvBjsytXLF7N1rAfR63R4uutIUNmwZjO+vCKv8bwnfZ63nuf+vl8tNCGeNk88IeDi4gKk1xLI+B7AZDLh6uqaaXutVktSUhLTp09XRgRMnz6dhg0b8vXXX9OzZ09l2+vXr9OnTx9iYmJYtGiRsnpA9erV2b59O/ny5VMyd5GRkTRq1Ij169fTu3fvHD0XnU5HuXLlcrTvsyw1NZXz589TqlSp+/7OhPNJn+ct6e+8J32et3LS3zabnfhkM0ZzGgWyNzCAA6du8O2O01juml7QNKwYzWsVd9odNa1GjbeHAYPu6VxJQF7jeU/6PG897/19+vTpJx2CEFnyxBMCGVMFrl27RokSJZT2a9eu4e/vn2l7Pz8/tFqtkgyA9KRC8eLFiYmJUdrOnDlDz549sdlsrFixgvLlyzsc594aA66urhQrVozY2NgcPxeVSoWbm3PmOz6LXF1dn+vn/yRIn+ct6e+8J32et7La3yaLlaREE6h1uLjosnx8u93OL7susv73Ox+UNWoVXV6uRK2gIg/ZM3s0ahX5PA0Y9E/8Y84jyWs870mf563ntb9luoB4VjzxtHnFihXx8PBg586dSltCQgJHjx6lRo0ambavUaMGaWlpHD58WGkzGo1ER0dTsmRJIH0lgm7duuHq6spXX32VKRmwevVqatas6TDfPykpifPnzz+Xd/iFEEKIrEoxWridYMRizbySwMNYrTZW/nTcIRngZtAy8P9CnZoM0GrU5PNyeSaSAUIIIcST9sQTAnq9ns6dOzNlyhQ2bdrE8ePHee+99/Dz86Np06ZYrVauX7+O0WgE0of716lThxEjRrBnzx5Onz7N8OHD0Wg0tG7dGoAPPvgAs9nMtGnT0Gq1XL9+XfkCaNCgATabjeHDh3Pq1CkOHz7MO++8g6+vL23btn1ifSGEEEI8zRKTzcQnmbJdPDDVmMbstQfZeuCy0lbAx5XhXavjX9I5ywoC6LVqfL0MGHQapx1TCCGE+Dd7KtLnAwcOJC0tjVGjRmE0GqlRowaLFi1Cp9MRExPDiy++yMSJE5WL9VmzZjFlyhQiIiIwGo1UrVqVZcuW4evrS2xsLLt27QJQEgR3O3HiBEWKFGHp0qVMnTqVN954A7vdTt26dVm2bBkGgyFPn7sQQgjxtLPZ7MQlmTCa07Bns17ArXgjkWsPcPl6stJWpqg3/doF4+nmvIJbBp2GfJ4GNJonfq9DCCGEeGY8FQkBjUbDsGHDGDZsWKbHihUrxokTJxzaPDw8+Oijj/joo48ybV+4cOFM299PQEAAixcvznHMQgghxPMgzWojLtGEyZL9Kv0XriYwe81BEpLNSlu1ioXo1rIyeifexXfRa/DxdEGjljm7QgghRHY8FQkBIYQQQjx9TBYr8YmmbNcLADhw8jqLvz+C2XJn3+a1S/Jqg7KonVRsSwW4umjx8TBIAS8hhBAiByQhIIQQQohMUk0W4pPM2a4XYLfb+W1PNGs3nSJjT7VaRadmFakb8oLT4lOpwN1Fh5e7XpIBQgghRA5JQkAIIYQQDhKTzSSlmslmLgCrzcZ/fz3F5n13lgF2NWjp0yaIiqV8H7Jn9qhV4OGqx9PdeTUIhBBCiOeRJASEEEIIobidaCTVlP3igUZTGgu/O8KRMzeVNl8vFyI6hPBCQQ+nxadWqfBy1+HuKskAIYQQ4nFJQkAIIYQQaLQ6bieaQK3L9r63E4zMXnuQmGtJSlupIl70bx+Ml7vzVu/RqFV4e+hxNWQ/RiGEEEJkJgkBIYQQ4jmXZrWRmGrFw5SGq2v2LrYvXk1k9tqDxCeZlLbQCgV565UAp64koFGr8PE04KKXjy5CCCGEs8i7qhBCCPEcSzVZuJVoIiXV9OiN73H49A0WfnvEYUnCl8JK0KZxOaetJACg1ajw8XTB4MQEgxBCCCEkISCEEEI8t5JSzCSmWEhLy/6ygr/vjea/v55Uag2oVSo6Nq1Ag9BiTo1Rp1GTz8uATivJACGEEMLZJCEghBBCPGfsdjvxyWZSjJZsFw+02eys/e0Uv+2JVtpc9Bp6vRZEQJn8To1Tr1WTz8sFrUbt1OMKIYQQIp0kBIQQQojniNVqIy7JhMlsJZu5AExmK4u+O8Kh0zeUtnyeBgZ0CKFYIU+nxumi0+DjaUAjyQAhhBAi10hCQAghhHhOWNKsxCWaMOdgikBcoomotQe5GJuotJUo7En/9iH4eDpvJQEVYNBryedpQK12Xh0CIYQQQmQmCQEhhBDiOWA0pxGXZMJqze64AIi5lr6SwO2EO4UHg8sVoMergRj0zpvbr1KBm4sOb3c9KicWJRRCCCHE/UlCQAghhPiXS0o1k5hswZbdggHA32dvsuCbwxjNd1YSCK9enPbh5Z16B1+lAg8XPV4eeqcdUwghhBAPJwkBIYQQ4l/KbreTkGwmOQfFAwG27I/hq59PKokElQpeb1KBxtWKOzVOtUqFp7sOD1dJBgghhBB5SRICQgghxL+Q1WYnLtGEyZyW7eKBNrudr38/zS+7LiptBp2Gnq0DCSpXwKlxatQqvD30uBp0Tj2uEEIIIR5NEgJCCCHEv4wlzUZcojFHxQPNFitLvv+b/SevK23eHgYGtA+hhJ9zVxLQaFTk8zBg0MvHESGEEOJJkHdgIYQQ4l/E9E/xwLQcFA9MSDaz+McjXLiSoLQVK+TBgPYh5PNycWaY6LVqfDwN6LTOK0oohBBCiOyRhIAQQgjxL5FstJCYbMZqy34y4GZiGis2H+Z24p2VBALL5qfnq4G4GJz7ccGg05DP04BGo3bqcYUQQgiRPZIQEEIIIf4FEpLMJBvN5CAXwImLcazbdgtz2p2dG1YtyutNKqBRO++iXQW4GLT4eBicukKBEEIIIXJGEgJCCCHEM8xmsxOfbCLVlJajlQS2HbzMip+OY/snk6AC2r9YnvDqxVGpnLusoLuLDi93vVOPK4QQQoick4SAEEII8YxKs9rSVxKwWLO9r81u57stZ/jf9gtKm06rpsergVSpUNCZYaJWgaebHg83WVZQCCGEeJpIQkAIIYR4BpksVuITTVisOVtJ4Isfj7L3+DWlzc2gpk+bQPxLOTcZIMsKCiGEEE8vSQgIIYQQz5hUk4X4pJwVD0xMMTNn3SHOXopX2orkd+OlEDdKFPZwZpiyrKAQQgjxlMvxO/SZM2fYtm0b165do0uXLkRHR1OxYkU8PJz7YUIIIYQQdyQmm0lKzVnxwKs3k4lcc5AbcalKW+XSvnRuVo7LMRcesmf26TTpywrqdbKsoBBCCPG0ynZCwGazMXr0aNatW4fdbkelUvHyyy8TFRXFxYsX+fLLL/Hz88uNWIUQQojn1uMWDzxx4Tbzvj5EijFNaasX8gJvNPXHbDY9ZM/s02vV+Hi6oNPKsoJCCCHE0yzb79RRUVF8//33fPLJJ2zbtg37P59Khg0bhs1mY/r06U4PUgghhHiepVlt3EowkmLMWTJgx+ErzFy93yEZ0LZxOTo1r4hG49yLdoNOg6+XJAOEEEKIZ0G2363XrVvHwIEDadeuHT4+Pkp7pUqVGDhwINu2bXNmfEIIIcRzzWSxcivemKOVBOx2O99vPcvSH48q9QZ0WjW9Xgukac2STl/+z0WvIZ+Xi9OTDEIIIYTIHdmeMnDjxg0qVap038cKFy5MQkLCYwclhBBCCEgxWkhIzlnxQEuajWUbjrL7aKzS5ummo1+7EMoU9XZmmKgAVxct3u4G1GrnJhmEEEIIkXuyncIvWbIkmzdvvu9ju3btomTJko8dlBBCCPG8S0w2E59kylEyICnVwoyv9jkkA/zyuzGiaw3nJwNU4Oaqw8dDkgFCCCHEsybbIwS6devG6NGjsVgsNG7cGJVKxYULF9i5cyeLFy9m5MiRuRGnEEII8Vx43OKBsbdSmL3mANdu31lJwL9kPnq3CcLdRefESNOTAR4uerw89E49rhBCCCHyRrYTAh06dODWrVvMmTOHVatWYbfbGTx4MDqdjp49e/LGG2/kRpxCCCHEv16a1UZcoilH9QIATkXfZu66QyTfVTywTlAR3mxeEa2T5/WrVSo83XV4uEoyQAghhHhWZTshANCnTx86derE/v37iYuLw8vLi5CQEIcig0IIIYTIOpPFSnyiCYvVlqP9d/19lWUbjpJmvTOsoHWDsjSv7fzigRq1Cm8PPa4G5444EEIIIUTeyvbtgvfff5/o6Gg8PDyoX78+r7zyCg0bNsTHx4ezZ8/St2/f3IhTCCGE+Fey2+2kmizcTjDmKBlgt9v5cds5Fn//t5IM0GrU9Hg1gJfrlHpkMsCezXkJGo2KfJ6GXEsGZDceIYQQQuRclhICly9fVr6++eYbTp486dCW8bVlyxb++uuv3I5ZCCGEeCqFh4dnq5bOpk2bGDxkGHGJd4oH/vbrT3Rs15JmjWsx7dPxvBfRm/ciet93/zSrjS9+PMb3W88qbe6uOt59I5Qalf0eeu7r12J5f9hAYq9eyfTYqJGD+fKLRZnadRo1vp4uGPQ5GmDoICYmBn9/f9avXw9AQkICw4cPZ8+ePco2Xbp0oUuXLvfdfv369fj7+xMTE/PYseSWu+N/XGvWrMHf3z/T17hx45xy/Nywc+dO/P392blzJwAjR44kPDz8CUclhBDibll6Rx87dixbtmxRfo6IiLjvdna7nbp16zonMiGEEOIZExkZiYeHR5a2tdnsLFi4GJvdzt0LCcycPplixUow8j9jKVCwEFMnf3Lf/ZONFuatP8TJi3FKW2FfNwZ0CKFQPrdHnn/vnp3s3L4tU7vFYmHf3l10e9sxCaHXqsnn5eL0WgQZjh07xrfffku7du2UtjFjxjxw+0aNGrF69WoKFSqUK/E8bY4dO0bp0qWZNGmSQ3uBAgWeUERCCCH+DbKUEBg3bhx//fUXdrudDz74gH79+lGiRAmHbdRqNV5eXtSsWTNXAhVCCCGedpUrV87SdtZ/igfa7rOkYEJ8PNXb1qJK1eoP3P96XCqR/z1A7K0Upa18cR/6tg3G3fXxhvIfOrAPdzd3ypX3V9pcdBp8vFzQ5PGyguXKlXvgY76+vvj6+uZhNE/WsWPHCAoKokqVKk86FCGEEP8iWUrzFy5cmDZt2tC2bVsmTpxI165dadOmjcNX69atady4MW5uj74rIYQQQvwbZUwZyBjevnHjRgYOHEhoaChhYWGMGjWKuPhEbsYb6denBwcP7OXggb2E16vGgX17CK9XDYBlSxYQXq8aV69cznSO42ev8e6I8exYN5ZTGz7g3G+TcU/cQ0SHECUZYLVaWfXlUt7u8jrNw+vQokldIvq+xf59uwH434bv+HTCWADe7PAK0z4drxx/545t1KhZB5VKpcR0/O/9dO/WleDgYBo1asSaNWu4du0aERERhIaG0rBhQ5YuXaoc40HD+R80pWLnzp107doVgK5duyrD7B825P7ec4wcOZLu3buzbt06mjVrRmBgIK1bt3YY4Qiwf/9+OnXqRJUqVWjUqBFffPEF3bt3V+K6d5h7hntjMRqNTJ06laZNmxIYGEjVqlV56623OHbs2H3jfZiTJ0/Sp08fqlatStWqVRkwYADR0dHK43a7nRMnTlCpUqVsH/vWrVsMGTKEunXrEhQUROvWrfnmm2+Ux9evX09QUBB79uyhXbt2BAUF0axZM3777TfOnj1Lt27dCAkJ4aWXXuLHH390OPbu3bvp0aMHNWrUIDAwkPDwcGbNmoXNlrPCmEIIIfJelhICu3fvJjk5GYBixYpx+vRpdu/e/cAvIYQQQqQPeS9atChRUVH06NGDtWvXMnPWbCxWG4OGjKRcBX/KVfAncu5SyvtXJHLuUgBatGpN5Nyl+OZ3HA6+++hVhg0ZyNXjm/AuEcYLNboTGtaAg3+uI3L6naHkC+bOYvnSBbRq3ZbJU2cxZPgoEhPiGTtqBEZjKrVq16dztx4AjB3/GW907q7su3P7n9SsXRcVYNBrABg6dAjh4eHMmzeP0qVLM2bMGLp27Ur58uWJiooiODiYiRMncujQoRz1U0BAAKNHjwZg9OjRD50q8DBHjhxh0aJFDBw4kNmzZ6PRaHjnnXeIj48H4MyZM3Tvnv5cp02bxjvvvMP8+fPZu3dvts81fPhw1q1bR+/evVm8eDHvv/8+p06dYsiQIdkqjHjhwgU6duzIzZs3mTx5MuPHjyc6Opo33niDmzdvAnDx4kWSk5M5fPgwzZo1IyAggGbNmjlc2D/IsGHDOHPmDGPHjmXBggVUrlyZESNGsGPHDmWbtLQ0hgwZQseOHZkzZw6urq4MHTqUvn370qhRI+bOnUuhQoUYMWIEV69eBeD48eN0794dHx8fpk+fzpw5c6hevTqRkZFs3Lgxe50phBDiicnSlIEuXbrw3//+l+DgYLp06YJKpcr0ZpfRplKpcpQdF0IIIf5tGjZsyIgRIwAIDK7G739s4a9tW+jRJ4JSpcvg7pZeb6ByYJDDvwUKFla+T2fnf9vPs2LNjyRfP4Vf6JvkKx5K1xaVqBlYhOVL/ViycA5tO7xB6TJluXnjOj16D6Bt+47KEXQGAx/9ZxhnT5+mcmAQLxQtBkD5ChXx9snHmTNnuHrlMpcvxVC9Rk1cXbR4/DPioF27drz11lsAuLm58frrrxMcHMygQYMAqFixIj///DP79u0jODg42/3k4eGhTA8oV67cQ6cKPExiYiLr169XpjW6ubnRuXNnduzYQbNmzZg3bx6enp4sXLgQV1dXAMqUKUPHjh0fdthMzGYzycnJjBo1ihYtWgAQFhZGUlISkyZN4saNGxQsWDBLx5o3bx6urq4sXbpUqT9Ru3ZtmjRpwsKFCxkxYoTyuSomJoaRI0ei1Wr55ptvGDFiBGazmddff/2Bx9+1axcDBgygSZMmSpw+Pj7o9XplG5vNRt++fenQoQOQXuDxvffeo1u3bsrv3dPTk3bt2nHkyBH8/Pw4fvw4derU4bPPPkOtTr+/VLduXX777Td27txJy5Yts9OlQgghnpAsJQSWLVtG2bJlle+FEEII8WhVqlTBZrMTn2wi1ZRGgYKFuXolc1X/h7Fj59qtFL7ZfIaUm2dBpaZwmar0b1eF8iXyAfBSsxYsWTiHQwf2UrpMWf4zJn0KQNzt20RfPE9MTDTbt6UPnbdYzA88155d2wkICqFggXz4eBiUJQtDQ0OVbfLnzw9ASEiI0pYvX3ociYmJ2Xpuzubr6+tQ48jPL32lhdTUVAB27NhBgwYNlGQApD+3okWLZus8er2eRYvSV2GIjY3l3LlznD9/nt9//x1ITxjcy2azOQylt1qtQPoFe82aNXFxcSEtLQ1IT5BUr15dWbmpRo0azJ07l5o1aypTM+vXr8+tW7eYOXMmHTp0wGazOdysUalUaDQaatasyaxZszh69Cj169d3SFLd7VG/Yx8fHyA9WQDw2muv8dprr2EymTh37hwXLlzg2LFjWK1WLBZLdrpTCCHEE5SlhEBYWNh9vxdCCCGedzabnbOX4klINmNJc7wo0xtcuJ1gxGhJv/hTq1XY7FmfX51itHDpWhIpxjS8AKs5BZ3Bg5HdalLY907NHl/f9Au4pKQkAE4cP8rnUydx4tjfuLi4ULJ0WQoXTr84fthw9j27dlC/fgOHZABw35UT7r6oflrcG1PGc8i4EL9165ZysXu3nFTq37p1KxMmTODs2bO4u7tTsWJF5WL9fn38wQcf8PXXXys/FylShKlTpxIfH8+GDRvYsGFDpn0yiibmz5+fxo0bZ3q8YcOG/PXXX9y4cYPBgweza9cu5bGwsDCWL1/O9OnTmTt3Lhs3buSnn35CrVZTp04dxo0b55AIye7v2Gg08vHHH/Ptt9+SlpZGsWLFCA0NRavVZmvKhBBCiCcrSwmByMjILB9QpVIxYMCAHAckhBBCPCsOnrrO2t9OcelaEmlWG/FJJg6cvM6JC7cASE61KMmA7LoRl8rstQdJMaYpbfnz+3DxcjIFvA0O2968eQMAb28fkpOTGDEkgjJly7N4+RpKlCyFWq1mx/Y/2fLHpgeez26zcujgPj4c9b5DMiAn7r0Qz5BRj+hJ8fPz48aNG5nab968SZkyZYCHx+7u7g6kz+nPGIY/b948ihcvjkqlYsWKFWzduvW+546IiKBTp07KzzabjbS0NDw8PKhbt64yNP9uWm36x7Q9e/YQHR1NmzZtHB43mUxoNBq8vb0ZO3asQ/9mxOrp6cmwYcMYNmwYZ8+eZdOmTURFRTF27Fjmz5//8A57iPHjx/PTTz/x+eefU6dOHSUZUrt27RwfUwghRN6ThIAQQgiRAwdPXWf22oOkGtPwdNeh0+hQqcBotvL91nMAWB9RbV2tUWOzZt4mLtHIp8v3kJB8Z+h59UqFqVTvZT488DObf/+V8CbNlMd+/Tn97nJgcBUuXjhPQnw87Tq8QanSZZRtdu1IH36eMUJBrdbciUOlIubCaXx8fPD3v7PcYE5l3G2+evWqMoT/zJkzxMXFPXAfjUbzwMecpUaNGmzduhWTyYTBkJ5UOXr0KDExMdSoUQNwjD1DfHw8Z86cUeojHDlyBJPJRO/evR2mKGQkA+53h7xYsWIUK1ZM+TklJYVjx45RrVo1Tp8+TaVKlZQEgN1uZ+jQoZQsWZJKlSqxY8cOZs2aRZUqVShdujSQnlD46aefCA0NRa/XKwmNu126dIk333yT999/n+bNm1OmTBnKlCnDgQMHuHDhQs47Eti7dy81a9ZUahNk9MutW7dklQEhhHiGZCkhcPz48Rwd/PLlyxQqVEh5gxNCCCH+DWw2O2t/O0WqMY383unD61WkTwlwM2i4lZA+Z/1RI6c9PDw5euQQ+/buonz5inh6eQGw/fAV8pW/kwzw9Xbh7VcDUAFVqlZn6uRPuHH9GmXLVeDggb2s+nIpzV5uRanSZUhKSsTd3Z0vly1Go9Gg0WrZ8vsmNv74LQDGVKNyboCtW36jXp06HD64j7p16zqlfzLmxE+aNIlBgwaRnJzMzJkzlXno9+PpmR7PH3/8gbe3NxUrVnRKLHfr27cvGzZsoGfPnrz99tskJCQwY8YM1Gq1MjLA39+fIkWKMHv2bDw8PFCpVErhvwwBAQFotVo+++wz3n77bcxmM+vXr+ePP/4A0i/2s6p37950796dPn368MYbb2AwGFi9ejW//vorM2fOBKBjx4589dVX9O3bl0GDBuHq6srKlSs5efIkK1aseOCxixYtip+fH5988glJSUmUKFGCI0eOsHnzZvr06ZODHrwjODiYjRs3smrVKsqWLcvx48eZM2cOKpVKqdkghBDi6ZelZQdzwmq18uKLL3LixIncOoUQQgjxRJy9FM+la0l4uutQqVSo1Sq8PfSoALPFpiQCbsUbH3qc19q9jkar5f2hA9m5Yxu/7Ey/a2u1pR9ArVZROL8bBbxdUatUqFQqJnz6Oa1at2Xtf1fywfBBbP79V3r2iWDY++lL9Xl4ePLxxGlgtzP2wxFM+ng012Kv8nnkAtzc3Dl8aD8AoVWrU616TRbOjWTO7Bns37+fevXqOaV/vLy8mDVrFlarlQEDBjBjxgwGDBhAYGDgA/cpX748rVq1YsWKFQwdOtQpcdyrZMmSLFq0CJPJxMCBA5k+fTq9evWiYMGCyhB7jUbDzJkzKVCgAIMHD2b8+PG0bNmSpk2bOhxn6tSpxMbG0q9fP2XJxOXLl6NSqdizZ0+WY6pQoQIrVqxApVIxfPhwBg4cyPXr15k9e7ZyzgIFCrBixQr8/f355JNPePfdd0lNTWXp0qUOhf/uJzIykvr16zNjxgzefvttVq1aRURExGOP5hw5ciRNmjTh888/p0+fPqxZs4Z+/frx+uuvs3//fqVoohBCiKebyp5LlV+sVisBAQGsW7eOgICA3DjFU+Xw4cMABAUFPWLLf5+MYY+VKlVS5hCK3CV9nrekv/Pe097n+45f4/Ov9pHP04BBp8HTXUea1U5copE0qx2b3U5CspluLQIIKJu5iN29rDYbq385yZb9l5Q2V4OWPm2CqFjKN1eeg0oFnq56PN31T31/O8v27dvR6XRUr15daUtISKBOnToMHz6crl275lksz0ufP02kz/PW897fz/O1gXi2yFh+IYQQIpu83PVoNWo0GhVe7npSzWnEJ5mUkQFpaXY0ajUebrpHHivVlMbCb4/w99mbSlt+bxciOlShSAH3XIlfrQJPdz0ervpHb/wv8vfffzNz5kwGDx5MQEAAcXFxLFmyBE9PT1q1avWkwxNCCCHynCQEhBBCiGwqU9Sb8iV8uBGfSmKKmaTUO+uu2+12kk0Wihb0oHhhz4ce51aCkdlrDnLpepLSVvoFL/q1C8HLPXcu1p/XZACgzPdftWoVV65cwc3NjbCwMCZOnKgs8SeEEEI8TyQhIIQQQmSTSgVNw0qy5Me/uZJgxN2gQ6tVkZaWngxw1WtpXqsUavWDl++7eDWB2WsPEp90p3hgVf9CdG9VGb0udyruq1Xpoxvcn8NkAIBaraZ///7079//SYcihBBCPBUkISCEEEJkg81mJy7JRJEC7rRpWI7/7ThP7M0UUkw2NGo1RQt60LxWqYfO/T906joLvzuC2XJnebZmtUrSumFZ1KoHJxEeh1qVXvjQzeXR0xiEEEII8XyQhIAQQgiRRWlWG3GJJkyW9ArqFUv5UqFEPqJjE0lKseDhpqN4Yc+Hjgz4bU80a349SUZFX7VKxRvN/KlfpWiuxa35ZxUEV4MkA4QQQghxhyQEhBBCiCwwWazEJ5qwWG0O7Wq1ipJFvB65v81mZ82mk/y+N0ZpczFo6P1aEJVLP3olgpzSqFXpqyHo5S1fCCGEEI7k04EQQgjxCKkmC/FJZqy2nK3UazSnsejbvzl85obS5uvlwoAOIRQt6OGsMDPRalT4eLpgyKWaBEIIIYR4tuVaQkCtVhMREUGhQoVy6xRCCCFErktMNpOUaiaHuQBuJxqJWnuI6NhEpa2knyf924fg7WFwUpSZ6TRq8nkZ0GklGSCEEEKI+1NndcM9e/bQt29fWrZsybvvvsuRI0cybXPs2DFefPFFAFQqFRERERQsWNB50QohhBB5xGazczvRSOJjJANiYhOZvGyPQzIgpHwBBr9ZLVeTAXqtmnxeLpIMEEIIIcRDZSkhsH37drp27cqlS5coXbo027dvp2PHjqxatcphO7PZzOXLl3MlUCGEECKvWNJs3EowkmJMw5bDbMDhMzf4bMVe4hJNSluTGiXo0yYYgz73LtQNOg2+Xi5oNbmzWoEQzzu7PYcZQpEn5PcjRPZkKSEwa9YsmjRpwrfffktkZCS//PIL4eHhjBs3jpUrV+Z2jEIIIUSeMZnTuJWQisliZfnShaxetTzbx/hjXwxRaw9iMqevRqBSQcem/rR/sfxDVyB4XC46Dfm8XFi/fh2TJ0/OtfPkxIkTJ3jttdcIDAykRYsWTzqcXHP06FFCQ0Mxm81POpQsi4mJwd/fn/Xr1z/pUJ464eHhjBw5Uvk5KiqKRYsWKT/PmjULf3//+27/LPTrvfE/y65evUrv3r25dOnSQ7dbv349/v7+xMTEPHS7Z8Hff/9Nr169qFWrFjVr1uTtt9/m77//ftJhiWdMlhICJ0+e5PXXX0etTt/cy8uLGTNm0KJFC8aPH8/PP/+cq0EKIYQQeSE51cytBBNp1vQ7TEsWzsGYmprl/TNWEvjq5xNk3KQy6DX0bx9Co6rFciNkAFSAi15LPi8XNGoVc+bMIS4uLtfOlxOzZ8/m8uXLzJ49+6lLVjjT5s2bqVWrFnq9/kmHInLBjBkzSL3rb0KHDh1YvXr1fbctVKgQq1evplGjRnkU3fPtr7/+YvPmzU86jDxz4cIFOnfujNFoZPz48UycOBGz2cybb77J2bNnn3R44hmSpaKCrq6uJCcnO7SpVComT57M9evXGTZsGAUKFECjkbmKQgghnj12u52EZDPJRgs5HW1qMltZ/P0RDp66s5KAj6eBAe1DKF7Y00mRZqYCXF20+HgYUKme3mkCt2/fpkKFCjRs2PBJh5KrtmzZQuvWrZ90GCKP+Pn54efnd9/H9Ho9VapUyduAxHNj+fLluLq6Mm/ePNzc3ACoVasW4eHhfPnll4wePfoJRyieFVkaIVC1alWioqK4fv26Q7tWq2X27Nm88MIL9OnThz///DNXghRCCCHuduTIEbp160a1atUIDQ2le/fuHDhwQHl8z549dO7cmZCQEMLCwhgxYgS3bt1SHl+/fj2VK1fm4MGDvP766wQHB/NKi6Z8tWKZsk14vWoALFsyX/ke4NzZ03wwfBCtmjagVdMGfPj+EE6cOsvUlXs5eOoGKTfOcPKH4binxWA6/iW932xGu1ebMj9qJlarVTmOxWJh8YIoOnV4lebhdXi7y+v8tPF7h+e5besf9O3RmWbhtWn3alMiP//M4e6kSgXurjqHZEB4eDiXLl3i66+/VobFZjzfNWvWULduXcLCwjh9+jRWq5X58+fTvn17unXrRu3atenYsSM7duxQzjFr1ixeeukl/vjjD1555RUCAwNp1qwZ33zzjUOsX3zxBc2bNycoKIj69evz0UcfkZSUBIC/vz+7du1i9+7dDkOoz58/z8CBA6lbty5VqlShS5cu7N27VzlmxpDrJUuW0Lx5c0JCQli3bh2zZs2iefPm/PLLL7Rq1YqgoCBat27N/v37OXDgAB06dCA4OJhWrVqxfft2hzhPnjxJnz59qFq1KlWrVmXAgAFER0crj+/cuRN/f3+++uorGjduTNWqVdm2bRu3bt1iyJAh1K1bVznfvX0QHx/PwYMHadCgAQBdunRh5MiRzJ07lzp16lCtWjX69++faUjzr7/+yptvvkloaCiBgYE0b96cFStWPDImgDVr1tC2bVuqVKlCcHAwrVu3ZuPGjcq+GcOjDx48SJs2bdJf66+8wv/+9z/udf36dQYOHEhoaChhYWF8+OGHDjeEwsPDmTBhAt26dSM4OJj//Oc/ABw/fpyIiAhq1apFQEAA9evX55NPPsFoNCr7btu2jddff53Q0FBq1KhBv379OHPmTKZ+aNu2LUFBQdStW5dPPvmElJSUTHHey9/fn1WrVjFy5EiqVatGWFiYcv7Jkycrw6nHjh2rTOV40HD+kSNHEh4e/sDzAERGRirfP2zI/b3nuPvvzv/93/8RFBRE48aNHaYgAFy7do333nuPsLAwatSowejRo5k+fbpDXP7+/syaNcthv/vF8qjXR1ZkvP62b99Oly5dCA4OplGjRqxZs4Zr164RERFBaGgoDRs2ZOnSpcp+e/bs4c0332T79u106tSJ4OBgmjZtmmm6sb+/PytWrOA///kPYWFhhIaGMmjQIG7cuOGw3cNeH+vXr+f9998H4MUXX3SY5vEg+/btU6YxtWrVig0bNjg8HhMTw/Dhw6lXrx4BAQHUrl2b4cOHc/v2bWWbR70XZfTDw96P7ufDDz+kbt26Du8ZAOPHj6dmzZpYLBbKlCnD22+/rSQDANzc3PDz8+PixYuPfP5CZMhSQmDIkCHcunWL8PBwpk2b5vCYp6cnS5YswdfXN9MfJiGEEMLZkpKS6NmzJ/ny5WPWrFlMnz6d1NRUevToQWJiIrt376Z79+64uLjw+eef88EHH7Br1y66du3qcIFis9kYNGgQDcObMuGzGQQGV2Fe1Ax27/wLgMi5SwFo0aq18n30xQu80/dtbt++zYj/fMTQkR9y8WI0A/v14OyFKw5xnt/5JVWrVWP8p5/z4kvN+WrlF2z4/hvl8fFj/8Oar76kxSuvMeHTz6kRVovJ4z9i0y/pF2qbft7Ih+8PoUTJUnw8YSrd3u7NLz9t4MORg7Hb7ahU4OGix/uekQGRkZEULFiQhg0bsnr1amX5X6vVyuLFixk/fjzvv/8+ZcuWZcqUKURFRdGuXTtGjhzJ6NGjiYuLY9CgQQ6Jh+vXrzNu3Di6du3K/PnzKVasGCNGjFAu6H744Qc+++wzOnXqxKJFixgwYADffvstH3/8MQCrV6+mcuXKVK5cWRlCffr0adq2bUtMTAyjRo1iypQpqFQqunXrxq5duxz6ctasWfTq1YtPP/2UunXrAunzhSdNmkTfvn2ZMWMGCQkJDBw4kMGDB9OhQwdmz56N3W7nvffeU37v586do2PHjty8eZPJkyczfvx4oqOjeeONN7h586bDOSMjIxkxYgSjR48mNDSUYcOGcebMGcaOHcuCBQuoXLkyI0aMcEie/Pnnn5QpU4YXXnhBadu0aRPr169n1KhRjB07lmPHjtGlSxelf7du3cqAAQMICAggKiqKWbNmUbx4ccaNG8fBgwcfGtOKFSsYPXo0TZo0Yd68eUyZMgW9Xs/QoUO5evWqw759+vThxRdfJDIyktKlS/Puu+9mGl49Y8YMihQpQlRUFN26deO///0vkZGRDtusWLGCoKAgoqKiaN++PdeuXaNTp06kpqYyadIkFixYQMuWLVm+fDnLlqUn2KKjo+nfvz+BgYHMmTOH8ePHc+7cOXr37o3NZgPg+++/Z8CAAZQpU4bZs2cTERHBd999R//+/bNUJO6zzz5Dr9cTGRnJa6+9xvLly3nttde4cuUKU6ZMoUuXLnzzzTf89NNPjzzWg2RMDWjfvv0Dpwk8is1m491336VFixbMnz+fqlWr8umnn7J161YgvUB3t27d2LdvHx988AETJ07k+PHjLF68ONvnys7rIysGDx5MeHg48+bNo3Tp0owZM4auXbtSvnx5oqKiCA4OZuLEiRw6dMhhvxEjRlC5cmVmz55NnTp1GDt2bKakwPTp07HZbEybNo3hw4fz+++/M2HCBOXxR70+GjVqRL9+/YD0/yf9+/d/5PMZPXo0L7/8MlFRUZQvX5733nuPX3/9FYDU1FS6du3KmTNnGDNmDIsWLaJr1678+OOPTJ8+HXj0exGk1xTJyvvRvVq3bs2NGzfYuXOn0maz2di4cSMtW7ZEp9Px5ptv0rNnT4f9Lly4wKlTpyhfvvwjn78QGbI0ZaBEiRJ8//33fPPNNxQuXDjT435+fqxbt46ZM2fyyy+/OD1IIYQQIsPp06e5ffs2Xbt2pWrVqgCUKVOG1atXk5yczNSpUyldujTz5s1TprKFhITQsmVL1q1bR6dOnYD0aQJduveiecv04d2BQVXYuvl3tv/1JzVq1qFyYBAABQoWVr5ftmQ+BhcXpnwehbu7B0fP3cQzqDuXLo3n9pk/KFi5FVUqFCRmB7R8tQ1duvcCoGq1MLZt+YPtf23lldface7sabb8sYkBA4fQ7vU307epHsbVq1c4sG8P4U2aMX/uLMJq1uGD0Z8oz71YsRIMfbcfu3b8SdMmL+LhlnmeeuXKldHr9fj6+mYarty3b1+H+cwZdyI7dOjAsWPHqFSpEp6enrzzzjucOHFC2T81NZXx48dTu3ZtAEqVKkXjxo3ZvHkzZcuWZdeuXRQrVoxOnTqhVqsJCwvDzc2N+Ph4AKpUqYKHh4fyPcC4cePQ6/UsW7ZMeaxRo0a0atWKTz/9lLVr1ypxvvzyy7Rr187huaSmpjJmzBjlbvzp06eZOnUq48ePp3379gCkpKQwcOBAzp07R6VKlYiMjMTV1ZWlS5cq56xduzZNmjRh4cKFjBgxQjn+m2++SfPmzZWfd+3axYABA2jSpAkAYWFh+Pj4ONQK2LJlixLP3XGuX7+e4sWLA+mv1TZt2vDDDz8QGBjI2bNnadOmjXK3HSA0NJSaNWuyc+dOQkJCHhhTdHQ0PXr0cLj4KVq0KG3btmXv3r20bNlSae/SpQsDBgwAoH79+rRp04bZs2c7TOFo1qyZcpe1du3abNu2zSHhAfDCCy8wdOhQ5ec///yTSpUqMWPGDKVP69Spw7Zt29i5cye9e/fm0KFDGI1G+vTpo3yO9PPzY9OmTaSkpODu7s6UKVOoX78+U6ZMUY5dqlQpunfvzubNmx85D79cuXKMGzcOSP/drFmzBovFwpQpU9BqtdSrV4+NGzdy8uTJhx7nYTJeu35+fjmeCmC32+nfvz8dOnQAoFq1avzyyy/88ccf1K9fn++++46zZ8+ybt06AgMDgfRh4Bmvu+zIzusjK9q1a8dbb70FpN+JzhhdNWjQIAAqVqzIzz//zL59+wgODlb2Cw8PV17f9evX59q1a0RFRfHGG28oycwKFSowceJEZZ9Dhw4po1jsdnuWXh8lSpQAoFKlShQr9uiaLe+88w49evQAoEGDBpw/f56oqCiaNGnC+fPn8fPzY/Lkycr/3Vq1anHw4EElYfmo9yJIT8pk5f3oXtWqVaNo0aL88MMP1KlTB0gfqXH9+vUHTkkyGo2MGDECvV5P586dH/n8hciQpYQAgI+PD927d3/gEBcPDw8++OADPvjgA6cFJ4QQQmSw2eycvRRPit0Hb5989O3bl+bNm1O/fn3q1q3LsGHDSE1N5eDBg/To0QO73U5aWhoAxYsXp2zZsmzbto1OnTphNKe3V6wcpBxfr9fj4+Pz0CKC+/bupkpoNVwMLvyx9yKrfz6Bza7F1bc0KTdO8XqTCviqY/lhGVQODHbYt0ChQhiN6cc+fOgAAPUbOg5NHjv+MwAuXjjH9WuxvNnlLaz/PAeAkCpVcXf34ND+PbR99WXl+WXQaDQPrSNQqVIlh5+nTp0KpA+NPX78OCdPnlSGot9bJf/uC6CMOdMZw3Vr1arF6tWradu2LU2aNKFhw4a88sorD41l165dNG7cWLmIhPSpiC1btmT27NkOQ9XvjTtDxodwgAIFCgA4XED7+PgAkJCQAMCOHTsICwvDxcVF6TsPDw+qV6/OX3/95XDse89Zs2ZNZs2axdGjR6lfvz4NGzZ0SCDY7Xa2bt2aaSRl1apVlQsKSE/YFC9enL179xIYGEi3bt1wc3MjOTmZc+fOcfHiRQ4fPgxk/h3cG1PGsOiEhATOnj3LhQsXlDuK9+7bpk0b5XuVSsVLL73ErFmzHO5SVq9e3WGfYsWKOUzhuF8M9erVo169elgsFk6fPs2FCxc4efIkt27dUvo/JCQEg8FA+/btad68OQ0aNKBmzZrKReOZM2e4evUqffr0cXhN16hRAw8PD7Zt20ajRo0e+noPDQ11aM+XLx8BAQFotXc+6np7e2dpCkJuuzvWjORdRlw7duygePHiSjIA0l+jjRs3drhbnBXZeX1A+mv43iHqd/ff3XHnz58fcPz/li9fPgDl7niGV155xeHnpk2bsmnTJs6dO0eZMmUAMiVY/Pz8lFE0Z8+ezdLr4142m00ZgXK/53PvaidNmjRh1qxZJCcnU6lSJVauXInNZuP8+fNcuHCB06dPc/bsWSWG8uXL4+vre9/3Ikj/u3rq1KlHvh/d+7pWq9Wo1WpeffVVVq5cyUcffYRer+fHH3+kVKlSDn2eISkpiQEDBnD48GFmzJhB0aJFM20jxINkOSGQoUGDBtSvX5/WrVsTHh4uVXSFEELkuoOnrrP2t1NcupZEmtVGmfoDuHV6Ez/88COrV6/GxcWF1q1b06dPH2w2GwsWLGDBggWZjmMwGIhLMmE0pX8Ac3FxcXhcpVZjt9sy7ZchIT6O3zf9zO+bMq+u4+HpTXj14hzYF5t+bIPjsdUqNXab/Z/jpN8598nn+4DzpD8+Y+okZkydlOnx27duEBMTw4svvujQPnHiRNq2bfvA+O+eawpw+PBhxo4dy+HDhzEYDJQrV065s3bvMG1XV9c7z+WfVYcytmnRogU2m42VK1cqw96LFi3K0KFDH7jEYHx8vHIRf7cCBQpgt9uV+gP3izvD3cmE+8V5r7i4ODZs2JBprjCAr6/j7+Lec06fPp25c+eyceNGfvrpJ9RqNXXq1GHcuHEULVqUw4cPYzKZqFatmsN+9xtZmT9/fiVJcfv2bUaMGMGvv/6KSqWiZMmSyoX5vb+De2O6ePEio0ePZvv27eh0OsqUKUPFihXvu2/G1JG7Y7Db7UockLnv1Gr1I2PIGOa9YsUKUlJSKFKkCMHBwRgMBmWbYsWK8eWXXzJ//nzWrl3LsmXL8PLy4s033+Tdd99VVsQYO3YsY8eOzdRf165de+Tr/X6vhQe9bp60e//u3N3Pt2/fVi6273a/tkfJzusD4Ouvv1ZGiGTYtGmT8n12/79luN9rD1BGEN3vOHf3SVZeH/cze/bsTFNeTpw4oXx/79+fjP8TSUlJuLu7s2TJEubOnUtcXBwFChQgMDAQV1dXJeHh7u7OihUrmDNnDhs3bnR4Lxo1ahRJSUmPfD8CCAgIcGiPiIjgnXfeoXXr1syZM4etW7dSv359fv75Z7p165bpOFeuXKFPnz6cO3eO6dOn52g0iXi+ZTshMHToUDZs2MC7776Lp6cnzZs3p3Xr1pmyykIIIYQzHDx1ndlrD5JqTMPTXYdOo8Pi9gI6j46UrPEmTYN0/L1vM6tWraJw4cKoVCq6d++eaTiszWbHYlWTnGohhwsJ4OHhibefPxQMU9rcXXV0eLECxQp7Zes4APFxtylY6M4F48UL50iIj8f9n8f79B9Elarp769qtQovNx06rQZvb28KFSrkMKweyNIw2QwZ81/9/f1Zu3YtRqORgIAAdu/enaN51q1ataJVq1YkJiby559/smDBAoYNG0a1atXue1Hs7e2dqWgYoBQwzpcv3wM/6OeUp6cnderUUYY93+3uO4cP2nfYsGEMGzaMs2fPsmnTJqKiohg7dizz589ny5Yt1KlTB51O57Df3QXIMty4cUO5y/fBBx9w4cIFli5dSmhoKHq9ntTUVP773/8+NB6bzUbv3r3R6XSsXbuWSpUqodVqOX36NN9++22m7TMuau6OQaPR4OPj81j9PH/+fJYuXcrYsWNp2rQpnp7pr92MaRsZgoODiYyMxGw2s3fvXlavXs3cuXOpWLEi5cqVA2D48OGEhYVlOoczXu/3yhhZcO8d8Sc9gqBw4cKcP38+U/u9NS7g4bFn9/UB0Lhx40x9XKhQoUxFMLPr3iVQM55LVpMcXl7pf1sf9vq4n9dff/2hU03uTUpm/J/w9vbm+++/Z9KkSQwbNoy2bdsqCcNBgwYpI3ggfYrAZ599htVq5dChQ3z77besWrWKEiVKEBwc/MD3I7iTBLlfnwOULl2a4OBgNm7ciFqtJiEhgVdffdVh2xMnTtCjRw9MJhOLFy+mRo0aD3y+QjxIlooK3q179+7897//5ZdffuHtt9/mwIEDdO7cmfDwcD7//PNMFWOFEEKInLLZ7Kz97RSpxjTyexsw6DTEntvH1hXD8dAaMVpsHLpiYPToMXh5eXHz5k0qV67M2bNnCQoKUr5KlCrDtM9nsGv3rkef9C4Zd8IBEpLNuPiW5urlixi8XsDFpzhly1di3KC2/Pnr1/y55fcsHzcouAoAf23b4tA+f84sImdMoUTJUuTL58vVK5fxr1iZwIBA6oRVpVjRF5g6dSpHjx5Fr9c7PMegoCBlyO7dcT/I2bNniYuLo2vXrpQtW1bZZ8uW9JjuHWr7MO+++64yP93T05OXX36Z/v37k5aW9sCLzRo1avD77787jASwWq38+OOPBAUF5coIxIzVFSpVqqT0WWBgIEuXLn1oDaRLly7RsGFDZU5zmTJl6NWrF3Xq1OHy5cvA/esHAOzduzdTVfKYmBjlwubAgQM0bdqUmjVrKs85K7+D27dvc+7cOdq3b09QUJCS0HjQvhnF0iD97vDPP/9MtWrVHruf9+7dS7ly5WjXrp2SDIiNjeXkyZNKDEuXLqVx48aYzWb0ej21a9dWCk5evnyZMmXKkD9/fmJiYhxez4ULF87S6z0nMu52x8bGKm0WiyVTQbx7ZeX/1uMICwsjJiaGY8eOKW1Go1EpOpjBw8PDIXZIr5qfIbuvD0hPwt3bx874f/j7745/G//3v/9RtGhRZc7/o2Tl9QGZfzeFCxfO9Hzu9scffyjf22w2/ve//xESEoKLiwt79+7Fy8uLnj17KsmA5ORk9u7dq/Td//73P2rVqsX169fRaDSEhoby0Ucf4eXlxeXLl3F1daV06dKZ3o/Kly/PrFmzlOkb98Z4dwK1devWbN26lR9//DHT9KMrV67w1ltvoVKpWLVqlSQDRI5le4RAhuLFi9OvXz/69evHqVOnWLVqFQsWLGDevHkOf8SEEEKInDp/JZFL15LwdNcpd/Ty+ZXDbrex76e5lAh6ib+v63lv6LckJibStGlTwsPD6d27N0OGDOHVV18lJdXMkiWLOXr0CJ279nzEGR15eHjy9+GD/LZ5G7/+bceleCMsFyO5tHsJQWEv0di/AjM+/ZBtW/9gzCefZvm4ZctXoGHjJsyLmoHJaKRceX927tjG9m1bGDv+MzQaDW/37s/0zyag02po2rQJKclJREVFERsbm2mI6b28vLw4evQou3btcijudbfSpUvj4eHB3LlzsVqtXL58mbVr1ypL6aU+pJbCvWrVqsWYMWOYPHkyDRo0ICEhgcjISEqVKqUMUb5XREQEW7ZsoWvXrsqdzC+//JLo6GgWLlyY5XNnR//+/enYsSN9+vThjTfewGAwsHr1an799Vdmzpz5wP2KFi2Kn58fn3zyCUlJSZQoUYIjR46wefNm+vTpw61btzhy5Mh9V1tKTU2lZ8+e9OvXj+TkZKZPn06FChV4+eWXOX36NAEBAXz//fcEBATg5+fHvn37mD9/PiqV6qG/g/z581O0aFFWrFiBn58fXl5ebN26Vansf+++n376KSaTidKlS7NmzRrOnDnDF198kcOevCM4OJioqCjmz59PlSpVuHDhAvPmzcNsNisx1KpViylTpjBgwAA6d+6MRqPhq6++Qq/X07hxYzQaDe+99x6jR49Go9HQuHFjEhISsvx6zwlvb29CQ0NZvnw5JUuWxNvbm2XLlmE0Gh861cDLy4t9+/axe/fuXBkd26pVK+bPn8+AAQMYNGgQXl5eLFmyhJs3bzqsXtGoUSN+/PFHQkJCKFmyJOvXr+fChQvK49l9feSmL7/8Eg8PD6pUqcLPP//M77//rtQvyYqsvj4yRhL88ssvNGjQgLJlyz70uJ9//jlWq5UiRYqwatUqzp07x5IlS4D01/WqVauYNGkSjRs35tq1ayxatIgbN24oIxKqVq2KzWZjwIAB9O7dG3d3dzZu3Ki8F0F6IdCJEycq70cZq70cPHgwSyshtGjRgkmTJrFhwwbGjBnj8Ngnn3zCzZs3GTt2LElJSQ7LHXp4eCgjb4R4lBwnBCB9yM/GjRvZuHEj+/fvx8fH54FzBYUQQojsSkgxk2a1odPcGYbt4u5NjZaDOLnrW479uYK0NDOppcsya9YsatWqBcCiRYuIjIzknXcGotVpqeBfiSnTo5TVArKqU9e3Wbp4Afv3D6VkoyEYvIpQvE4/bJf/4PDvSzn0m53SZcoybuJU6tZr+OgD3uWD0Z/wxeJ5rFuzivi4OEqUKsWYTz6lXoPGALR8pQ3eXp6sXrmM775dj5ubG1WrVmXKlCkOd4nu5+2332bChAn06NFD+YB7L09PT6Kiovj0008ZPnw4er2ewMBAvvzyS3r16sWePXseuB77vTp27IjFYuGrr75i5cqVuLi4ULt2bYYNG5ZpCH2G8uXLs3LlSqZNm8b777+PSqUiODiYZcuW5do0xIoVK7JixQqmT5/O8OHDsdvtVKhQgdmzZ2ean36vyMhIpk2bxowZM7h9+zZFihQhIiKC3r1788MPP1C+fPn7To2oXr06tWrVUqqsh4eHM3z4cKVfxo0bx5QpU5Q75qVKlWLs2LF899137Nmz56ExRUVFMX78eEaOHIler6dcuXLMmTOHCRMmsGfPHrp06aJs+9FHHzFv3jyio6OpXLkyixcvdko/9+nTh9u3b7Ns2TJmz55NkSJFaN26NSqVinnz5pGQkEDFihWZO3cus2fPZvDgwVitVgIDA1m8eLFSVK5Dhw64u7uzcOFCVq9ena3Xe05NmjSJjz/+mFGjRuHh4UH79u2pVq0aa9aseeA+ffv2JSoqil69et23FsXj0mq1LFq0iPHjx/PRRx+h1Wp59dVX8fHx4dy5c8p277//PmlpaUyePBmtVkuLFi0YMmQIo0aNUrbJzusjNw0dOpQff/yRefPmUaZMGWbOnEmzZs2ydYysvD5q1qxJnTp1mDp1Ktu3b2f+/PkPPebEiROZNGkSFy5coEKFCixYsEAZudOmmiXPYwAA5zlJREFUTRtiYmJYt24dK1eupHDhwjRs2JA333yTDz/8kDNnzlC2bFkWLlzIjBkz+M9//kNqaqpy979WrVocPnyYKlWqKO9HAwcORKfTERAQwJIlS7K0UoWvry/16tVj27ZtDiuMmM1mZYTDvYkCSB9psnz58iz2rnjeqexZWdz1LomJifz000/8+OOP7N69G41GQ3h4OK1bt6Z+/frKkhrPm4z5RPcOR3oepKSkKMtVPa0FfP5tpM/zlvR33svoc4NXUaZ9dQgXgwaDLvP7i8lsxWi28kH3MMoV91HarTY78UkmjOY0svcu5+ivQ5f58n/Hsf1TDFAFtG1cjiZhJR5aQf9xqQCDXoOPpwsade6dJ4O8xnNHxgXX/T6Y51Wfr1+/nvfff59NmzY91pz7f4Nn4XV+6tQpzp49S9OmTR3+xrRv3x4/P79MRfKeZlu2bKFXr14sWLDgvtNp/u2e52sD8WzJ9giB2rVrY7PZqFatGh999BHNmze/b9VRIYQQ4nGVKuJJ0UIenL+SgN5L7fAB2W63k5hqoVQRL8oUvVNUypJmIz7JhMlivd8hs8Rut/P91rNs+Ou80qbTqnn7lQBC/Qs9eEcnUAGuLlp8PAy5mnQQQjx9UlJSGDRoEG+++SYvvfQSVquVDRs2cOTIEYYOHfqkwxNC/AtlOyHwzjvv8MorrzjMY3qQ3bt3ExAQ8NRmYYUQQjzd1GoV7cPLM3vtQW4mmPB01aHTqrGk2UhMteBm0NI+vDzqf+6im8xpxCWZSLPmfFiAJc3Ksg3H2H30TsEuTzcd/duHUPqF+1ezdhaVCtxddHi56yUZIMRzKCQkhM8//5xFixbxzTffYLfbqVy5MgsXLlSmRAkhhDNlOyHQp0+fLG1ntVrp2rUra9euzZViMEIIIZ4PIeULMqB9CGt/O8Wla0kkpVrQatSUKuJF+/DyhJQvCEByqpmEZAu2x5gjkJRiZs76Q5yJubM+dpEC7gxoH0IBn0evt/041CrwcNXj6e786vriyXga5vC2bduWtm3bPukwRDY0b97cYb74s6p69eqsXLmSSpUqPelQhBAP8VhFBR8lm+UJhBBCiPsKKV+QoLIFOHspnoRkM17uesoU9UatVmG320lINpNstDxWvYDYWylErjnA9dt3qm9XLOVL79cCcXO5f2E8Z1GrVHi563B3lWSAEEIIIfJOriYEhBBCCGdRq1UOhQMBrFYbcUkmTGYrj5OCPnXxNnPXHyLZmKa01Q1+gTeb+aPR5O664xq1Cm8PPa6G3E06CCGEEELcSxICQgghnklmi5X4JBPmNNtjHWfn31dZvuGoQ92BNo3K0rRmyVyfx69Rq8jnacCgl7djIYQQQuQ9+QQihBDimZNqshCfZMZqy/m4ALvdzo/bzvHDn3fW9tZq1Lz1SmWqVcy8nryz6TRqfDwN6O+znKIQQgghRF6QhIAQQohnSmKymaRUM4+RC8CSZuPLjcfY+fdVpc3DNX0lgbuXMMwteq0aH08XdNrcnY4ghBBCCPEwkhAQQgjxTLDZ7MQnm0g1pT1W8cDkVAtz1x/iVHSc0uaX340BHapQMJdXEgAw6DTk8zTkem0CIYQQQohHkYSAEEKIp16a1UZ8ogmjxfpYx7l+O4XINQeJvZWitFUo4UOftsG45/JKAgAueg0+ni5o1Llbm0AIIYQQIiskISCEEOKpZjKnEZ9kxmJ9vOKBp2PimLvuEEmpFqWtVmAROr9cEW0u361XAa4uWnw8DLleqFAIIYQQIqse6xNQYmIiZ86cwWw2Y7U63rXRaDRMnDiRYsWKPfI4NpuNmTNnUr9+fapUqUKvXr2Ijo5+4PYWi4WpU6cq23fu3Jljx445bLN9+3batm1LSEgIzZs358cff3R43GQyMXbsWGrXrk1oaChDhgzh1q3/Z+++42s83weOf07GyV42sVcEScSIvVI0JShFbVrzJ2bNtkopagTVDHsWpWYRpbZSxKjUKmLVqJm9zknOOb8/8s3DkSAhiXW9X6++Gvd5xn3u84jzXM99X1dEFt69EEKInGRqakqSRkdkrOaVgwEnLtzjh5//MgoGtKpfmh4tXHM+GKACGytzCQYIIYQQ4o3zUt+Cjh07Rvv27fHy8qJly5ZcvnyZ4cOHM3XqVKPt2rRpg4PDi5MzBQcHs3r1ar777jvWrFmDXq+nd+/eaLXaDLf/9ttv2bhxI1OmTGHDhg3kyZOHPn36EBsbC8CVK1fo168f9evXZ+PGjbRv355Ro0Zx5MgRo2McOnSIgIAAli9fztWrVxk8ePDLDIcQQogcoNWZEBWveeVKAtv/vMaiX8+S8r+ggpmpis9bVqJ53VI5foOuUoGdlRoHCQYIIYQQ4g2U5YDAkSNH6NWrF5aWlowYMQLD/zI7VahQgRUrVrB06dIsHU+r1bJkyRIGDx5Mo0aNqFChArNnz+bu3bv8/vvv6ba/efMmGzZsYPLkydSvX58yZcowadIk1Go1Z8+eBWD58uW4uLgwbNgwypQpQ69evfDx8WHRokUA3Lt3j82bNzN27FiqV6+Ou7s7s2bN4vjx4/z1119ZHRIhhBDZSKc3EBWnJSY+Cf0rBANSdHp+2n6BLQevKm02lmYM6eiJV6VC2dHV5zJRgb2NGjsbdY6fSwghhBDiZWQ5IPDDDz/wwQcf8NNPP9GjRw8lINC/f3969+7NunXrsnS8f/75h/j4eGrXrq202dvbU7FiRY4fP55u+8OHD2NnZ0eDBg2Mtt+7d69yjBMnThgdD6BWrVqcPHkSg8HAyZMnlbY0pUqVomDBghmeUwghRO5ITtEREZ1IQlIyuldYJpCQlEzAL6f588x/Slt+JytGda9BuWJO2dHV5zJRqXCwtcDWSoIBQgghhHhzZTmp4IULF/Dz8wNIN/2xbt26LF++PEvHu3s3tQZ04cKFjdoLFCigvPaka9euUaxYMX7//XcWLFjAvXv3qFixImPGjKFMmTLKMQsVKpTueImJiURGRnLv3j2cnJywsLDI1Dkzy2AwkJCQ8OIN3zGJiYlG/xc5T8Y8d8l45w5Nsp7oeA0pKXo0SZrUtv/9PyseRSexcMsF7kU8/rxKFbGjl28FbKxUOf45mpma4GhrAfpkEhKSX7zDG0Cu8dwnY577ZMxz1/s+3gaDQZaKibdClgMCdnZ2PHjwIMPX/vvvP+zs7LJ0vLRfEmq18VMUCwsLoqOj020fFxfHjRs3CA4OZtSoUdjb2zN37lw6d+7M9u3byZs3L0lJSemOl/ZnrVZLYmJiutfTzqnRZP3LZ5rk5OR0yQ3fJ9evX3/dXXjvyJjnLhnvnGFmZoY2RUVsopbk5BSj127dvpWlY92N1BJyPIpE7eOlBuWdLfnA3Yq7d/7Nlv4+j6WlBfZWpkTe16HXv1oixNdBrvHcJ2Oe+2TMc9f7PN4Z3W8I8abJckDggw8+YPbs2ZQvX56KFSsCqTMF7t69y7x582jUqFGWjmdpaQmk3qin/QypVQCsrKzSd9jMjLi4OGbPnq3MCJg9ezYNGzZk06ZN9O7dGwsLi3QJCdP+bGVlhaWlZYYJC591zswyNzenbNmyL73/2yoxMZHr169TsmTJVxo/kXky5rlLxjvnGAwQm6AlPikFJ8Pjm3hNkoZbt29R1LkoFpYWzznCY6cvP+TXo+Ek6x4f58OaRfmwZrFceUqjNjfF0Vad41ULcoJc47lPxjz3yZjnrvd9vMPDw193F4TIlCwHBIYPH05YWBgdOnQgX758AHzxxRfcvXuXwoUL88UXX2TpeGlLBe7fv0/x4sWV9vv37+Pi4pJu+0KFCmFmZqYEAyA1qFCsWDFu3bqlHPP+/ftG+92/fx9ra2vs7OwoVKgQUVFRaLVao8jd/fv3KViwYJb6/ySVSoW1tfVL7/+2s7Kyeq/f/+sgY567ZLyzV4pOT1SsBh1mWFpm/M+RhaXFC79IGgwGfj/2L5v2P/7yZWqiottHrtRyK/ycPbOP2swERztLzM3evmDAk+Qaz30y5rlPxjx3va/jLcsFxNsiy99cHBwcWLduHRMmTKBGjRrUqVMHFxcXRo4cycaNG8mTJ0+WjlehQgVsbW05duyY0hYTE8P58+epUaNGuu1r1KhBSkoKZ86cUdqSkpK4efMmJUqUAKB69eqEhoYa7Xf06FGqVq2KiYkJ1apVQ6/XK8kFITU3wb179zI8pxBCiOyl0abwKDoJTbLulY6j0+lZvfMfo2CAtYUZgz/1zLVggIW5KXns3/5ggBBCCCHeP1meIQCp62E6dOhAhw4dXrkDarWarl274u/vT548eXB2dmbGjBkUKlSIZs2aodPpiIiIwM7ODktLS6pXr06dOnUYPXo0EydOxNHRkR9//BFTU1Nat24NQLdu3WjTpg3+/v60adOGAwcOsGPHDqXsYMGCBWnRogVjx45lypQpWFlZMX78eLy8vKhSpcorvychhBDPFp+UTGy8Ft0rlBQESExKYcHmM1y4HqG05XO0YmB7DwrltXnVbmaKpdoURztLTE3kSZAQQggh3j5ZDggEBgY+8zUTExOsra0pUaIEdevWzXQijcGDB5OSksLYsWNJSkqiRo0aLF68GHNzc27dusUHH3zA999/T9u2bQEICAjA39+fgQMHkpSURNWqVVmxYoUyO6FcuXIEBwczY8YMli9fTtGiRZkxY4ZRKcLvvvuOKVOmMHDgQAAaNGjA2LFjszocQgghMslgMBATryU+KRnDq8UCiIhOInD9ae48iFfaSjs78H+fuGNnnfNJnFSApYUZjrYWmLylwYC3NQP229pv8W6R61AI8a7IckBgy5Yt3L17F61Wi5mZGY6OjkRFRZGSkoJKpcLwv295ZcuWNbpJfx5TU1NGjhzJyJEj071WtGhRLl68aNRma2vLt99+y7fffvvMYzZo0IAGDRo883Vra2smTZrEpEmTXtg/IYQQr0anNxAVq0GjTeEVYwHc+C+GoPVhxMQ/Tg5brUIBevpWxNzM9BWP/mIqFVj9Lxjwtt4QBAcHo1ar6d279+vuSpasW7eOK1euMGbMmNfdFcXFixcZPXo04eHhFC9enO3bt7/uLuWI8+fP06VLF44dO5apBz5jxowhNDSUvXv3AuDi4sLAgQMZNGhQTnc1x2X2OvT29sbLy4upU6fmUs9yhlarZcmSJWzevJm7d+9SsGBBWrZsSd++fSWLvhDvgCwveBwyZAhqtZpZs2bx999/c+jQIc6cOUNgYCBOTk788MMPbN26FZVKxaxZs3Kiz0IIId4iySk6IqITScqGYMDpSw+YufqkUTDAp3YJerWunGvBABtLc5zsLN/aYADAnDlz3sra4HPnziUqKup1d8NIUFAQd+7cISgoiGnTpr3u7uSYAwcOUKtWLbkB5M28DnPSpEmTmDdvHm3btmXu3Ll88sknLFy48LkP5oQQb48szxAICAhg6NChNG/eXGlTqVQ0adKEhw8fMmfOHH777Tf69+//1kdEhRBCvJpETTLR8Vp0ulcLBRgMBvYcv8mGvZeVoIKJiYouH1agrkeRV+9oJpiowNZKjZ2N3BCJxyIjIylfvjwNGzZ83V3JUQcPHlRyNYn3R2RkJL/88gsjRoxQZhSlLcGdOXMmI0aMyHJCcSHEmyXLMwT+++8/JZv/05ydnbl9+zaQmrgvOjr61XonhBDirRUbryUq9tWDATq9njW7LrH+iWCApYUpgzpUycVggAp7m5wPBpw9e5YePXpQrVo1PD096dmzJ6dPn1ZeP3HiBF27dsXDwwMvLy9Gjx5NRMTjpIobN26kYsWKhIWF8emnn+Lm5kbjxo1ZvHixsk1aSd/AwECj8r43b95k8ODBVK1alapVq+Ln58fNmzeV148dO4aLiwtHjhzh888/x8PDg7p16zJjxgx0usfVIrRaLT/88AMffPAB7u7u+Pr6smnTJqP3uXv3btq2bYubmxt169Zl0qRJJCQkPHdsvL29uX37Nps2bcLFxYVbt24p73fdunXUrVsXLy8vwsPD0el0LFiwAF9fX9zd3alSpQodO3bk6NGjyvECAgJo2rQp+/fvp2XLllSuXJkPP/yQzZs3G513+fLl+Pj44ObmRv369fn222+Ji4tTxjI0NJTjx4/j4uLCxo0bAbh+/TqDBw+mbt26VKlShW7duhlVNrp16xaenp6EhITQpk0bPDw82LBhAwEBAfj4+LBr1y58fX1xc3OjdevW/PXXX5w+fZr27dsrY3rkyBGjfl66dIl+/fq98PNbs2YNjRs3pmrVqhw+fJiIiAiGDx9O3bp1lfM9PQbR0dGEhYUpSzEzM74vo1u3bowbN47g4GDq16+Ph4cHffr04eHDh2zYsIGmTZsqfy/SSk2n7TdmzBjmzZtHnTp1qFatGgMGDFC+k0Lq592qVSv++usvOnTo8MzPOyoqinHjxlGnTh3c3Nzo0KGD0VhndB0+T3JyMpMmTaJGjRpUr1493d9ZSF2C0LZtW6pUqYK7uzutW7fmt99+U17X6/XMnj0bb29vKleujLe3NzNnziQ5OVnZRqPRMH36dBo2bEjlypVp2bLlC5ev3L17F1dXV1auXGnUHhERQaVKlVi2bBlxcXF07NgRb29vo21Kly4NYHSNCSHeTlkOCJQtW5Z169Zl+Nr69espVaoUkPqPYYECBV6td0IIId46er2ByNgkYhO16F8xe2CSVsfcDX9z4NTjL9157C0Z1bU6riVz56mUiUqFg60aG6ucDQbExcXRu3dvnJycCAgIYPbs2SQmJtKrVy9iY2M5fvw4PXv2xNLSkh9++IGvvvqK0NBQunfvTlJSknIcvV6vzORbsGABVatWZfr06fzxxx8ArF27FoB27dopP9+4cYNvv/2WiIgIpk2bxuTJk7l58yadOnXi0aNHRv0cMWIE1apVY968efj6+rJo0SKj7wUjRoxg6dKltG/fnvnz51OvXj3GjBnDtm3bANi6dSt+fn6ULl2aoKAgBg4cyJYtWxgwYICShygjgYGB5M+fn4YNG7J27VrlO4ZOp2PJkiVMnjyZL7/8kjJlyuDv709wcDCffvopixYt4rvvviMqKoohQ4YYLZV48OABEydOpHv37ixYsICiRYsyevRorly5AsC2bduYMWMGXbp0YfHixfj5+fHrr7/y3XffKWNZsWJFKlasyNq1a2nUqBHh4eG0bduWW7duMXbsWPz9/VGpVPTo0SNdSeSNGzfSs2dPpk+fTt26dYHUm7SpU6fSv39/5syZQ0xMDIMHD+aLL76gffv2BAUFYTAYGDZsmPK5X7t2jY4dO/Lo0aMXfn6BgYGMHj2acePG4enpyciRI7ly5QoTJkxg4cKFVKxYkdGjRxvd3B86dIjSpUtTpEhqAC6z4/sytm3bxpEjR5g8eTJff/01R44coWvXrqxYsUKpMBUWFsbEiRON9tuzZw8bN25k7NixTJgwgQsXLtCtW7d0n/eyZcvo1KlThp+3RqOhR48e7Nmzh2HDhhEYGEihQoXo3bu3EhR41nX4LL/99hvnzp1j6tSpjB49mv3799OnTx8liLZq1SrGjRtHkyZNmD9/Pv7+/qjVakaMGMHdu3cBWLhwIT///DN+fn4sWbKETp06sXjxYubOnQukzqDy8/NjzZo1fPbZZ8ydOxdPT0+GDRuWLuDxpEKFCuHl5UVISIhR+44dOzAYDLRo0YJixYrx7bffKgGAJ8fb3NyckiVLPvf9CyHefFleMjBo0CD8/Pxo06YNzZo1I2/evDx8+JDdu3dz8eJFfvzxR86fP8+MGTP45JNPcqLPQggh3lApOn1q8sBk3Ys3foG4RB0B685w5+HjJ8clC9szoJ079jYWr3z8zDBRqXC0U2NlYZ7j5woPDycyMpLu3btTtWpVIPUp3Nq1a4mPj2fmzJmUKlWK+fPnY2qami/Bw8ODFi1asGHDBrp06QKk3hwMGDCA9u3bA1CtWjV27drF/v37qV+/vlJet1ChQsrP8+fPR61WM2/ePOUGp3bt2jRp0oRFixYxevRopZ/t27fHz89P2Wb37t3s37+fjh07cunSJXbu3MlXX31Fjx49lG1u377NsWPHaNGiBf7+/tSvXx9/f3/lmCVLlqRnz54cOHCARo0aZTg+FStWRK1WkydPnnQlgvv372+03/379xk2bBjdunVT2iwsLBg0aBAXL15U9k9MTGTy5MnKFOiSJUvSuHFjDhw4QJkyZQgNDaVo0aJ06dIFExMTvLy8sLa2VmZAVqlSBVtbW+VngIkTJ6JWq1mxYoXyWqNGjfD19WX69OmsX79e6VPNmjVp3bo11tbWSltiYiLjx49XnsaHh4czc+ZMJk+eTLt27QBISEhg8ODBXLt2DVdXVwIDA7GysmLZsmXKOZ/1+XXu3BkfHx/lz6Ghofj5+dGkSRMAvLy8cHR0NMoVcPDgQaNEzZkd35eRkpJCYGAgDg4OAPz+++/88ccf7N69m2LFigFw+vRpfv31V6P9EhMT2bhxo7JN6dKladOmDZs3b6ZTp04AJCUl8cUXX9CmTRusra3Tfd6//vor//zzD7/88gseHh5AapLqbt264e/vz4YNG557HWbEycmJxYsXK5+xk5MTfn5+HDx4kMaNG3Pz5k169erFgAEDlH2cnZ1p27YtJ0+epEWLFoSGhlK5cmXle7WXlxdWVlbY2dkB8Oeff/LHH38we/ZsZUlv/fr1SUxMxN/fH19fX8zMMv7K37p1a7766ivu3LmjBHxCQkKoU6cO+fPnz3CfXbt2sWnTJrp27ap8TkKIt1eWAwKNGjVi8eLFBAQEEBgYiE6nw8zMjGrVqrF8+XKqV6/O3r17adGiBUOHDs2BLgshhHgTabQpRMdpSdbpX/lYt+7Hse5QBPGax8fyLJ+fz1pWQm2e88kDAUxNUmcG5HQwQK83cPV2NAkGRxwcnejfvz8+Pj7Ur1+funXrMnLkSBITEwkLC6NXr14YDAZSUlIAKFasGGXKlOHw4cNKQADA09NT+Tnt5uV5U/JDQ0NxdXXF0tJSObatrS3Vq1fnzz//NNr2yWNDamAh7dhp0+KbNWtmtE1AQAAAV65c4e7du/Tr1085D0CNGjWwtbXl8OHDNGrUyOg1SK1G9Lwkjq6urkZ/njlzJpA69fnq1avcuHGDffv2AalLGp705E1doUKFAJT3U6tWLdauXUvbtm1p0qQJDRs2pGXLls/tS2hoKI0bN1ZuzAHMzMxo0aIFQUFBxMc/LpX5rCWYaQEhgHz58gEoN6gAjo6OAMTExABw9OhRvLy8MvX5PT1WNWvWJCAggPPnz1O/fn0aNmxoFEAwGAz88ccfRomiszK+GdHpdEazQUxMTDAxSZ20WqZMGaObzHz58uHk5KTc6Ke9/9jYWKNjVq1a1WibihUrUqxYMY4fP64EBCC1NHWapz/vI0eOkD9/fipVqmR0DTZu3Jjp06cTHR2d4Q3w09frk++nYcOGRgEfb29vzMzMOH78OI0bN1YqFcTExChjeezYMeDxWNasWZOZM2fSuXNnvL29adSoEV27dlWOeeTIEVQqFQ0bNjTqi7e3N1u2bOHy5cu4uLig1xv/bjYzM6NZs2ZMmDCB7du307t3b/777z9OnjzJjBkz0r1PSA3QDB8+nGrVqmVYHUwI8fbJckAAUv+BrFWrFlqtlujoaPLmzav84oPUX0BPrzUSQgjx7opPSiY2XotO/6p1BODv8Ics+vUs2uTHX16behWnTeOymORSZn9TExWOdhZYql/qn8lMC7v8gPV7L3P7fhwpOj2l6/sREb6HbdtCWLt2LZaWlrRu3Zp+/fqh1+tZuHAhCxcuTHccCwvjGROWlpZGfzYxMXnudPzo6GiOHj1KjRo10r32dMKw5x07LfN63rx5MzxP2usTJkxgwoQJ6V6/f/8+t27d4oMPPjBq//7772nbtu0z+//kDRfAmTNnmDBhAmfOnMHKyoqyZcsqTz+fHgcrKyuj9/LkNs2bN0ev17N69WqCg4MJCAjA2dmZESNGGCVXflJ0dLRyE/+kfPnyYTAYlPwDkH4s0zwZTMion0+Liopi+/btGa4Zf/rze3qsZs+ezbx58/jtt9/YuXMnJiYm1KlTh4kTJ+Ls7MyZM2fQaDRUq1ZN2Scr45uRpk2bGq3vb9OmjZKIOqP3/nSfM1KwYMF0bXnz5k2Xz+rJvytPf95RUVE8ePCASpUqZXiOBw8eZBgQeHr7J8srPv2U3cTEBCcnJyWY8++//zJu3DiOHDmCubk5pUuXpkKFCkb96t27NzY2NmzYsAF/f39mzJhBuXLlGDt2LLVq1SIqKgqDwWAUSHrS/fv32b17N4GBgUbtFy9exNbWliZNmhASEkLv3r3Zvn07VlZWyoyRJy1btoxp06bh5eVFUFBQut87Qoi300t909FoNFy8eBGtVovBYOD69evo9XoSExM5ceIEI0aMyO5+CiGEeAMZDAZi4rXEJyXziukCANh34ia/7LmkHMtEBR2budDAs+irHzyTTE1UONlZYJELwYCg9WEkJqVgZ2OOuak5ydZFMLftSIkanWnmZs65Uwf4+eefKViwICqVip49e9KiRYt0x3rezWJm2Nra4urqip+fX7qb1GdNNc6Ivb09kPrkOO3pK6TODIiKilJeHzVqFF5eXun2d3BwoECBAkbT6gGKFs3855+Wi8HFxYWQkBBKly6NiYkJBw4cYOfOnZk+ThpfX198fX2JjY3l0KFDLFy4kJEjR1KtWrUMb0IdHBx4+PBhuvYHDx4AqVPG79+/n+V+PI+dnR116tThs88+S/faiz4/Ozs7Ro4cyciRI7l69Sp79uwhODiYCRMmsGDBAg4ePEidOnUwN0+dKZMd4zt37lyjmQROTk5ZeLcZi4yMTNf28OFDihcvnulj2NnZUbJkSaPlLE961nX49PX6ZF6Bp8sT6nQ6IiMjyZs3L3q9nr59+2Jubs769etxdXXFzMyM8PBwoyURJiYmdOnShS5duvDo0SMOHDjAvHnzGDRoEIcPH8bOzg5ra2tWrFiRYf9KlChBhQoVnrkcp1WrVvTt25cbN24QEhLChx9+aPQ7xWAwMHnyZH766Sd8fX35/vvvpfykEO+QLH/bOXbsGEOGDHlmBQEbGxsJCAghxHtAp9MTFadBo9XxqrEAvd7A+r2X2XviccZqczMVn7WoQFXX3KkkAGBqqsLJNueDAWnvNzEphbwOFqhUKv67cpJzf6ymXvtviEu25O//LJgwbjwhISE8evSIihUrcvXqVdzc3JTjJCUlMXjwYBo2bEjZsmUzff4nZ/VBap6BtGnFaTftBoOBESNGUKJEiXTTzJ8l7Sny3r176dy5s9Lu7+/PvXv3WLduHXnz5uXWrVv06tVLef3+/fuMGjWKjh07Urx4caP3+Lx+Z+Tq1atERUXRvXt3ozE5ePAgQLpp088zdOhQkpOTCQoKws7Ojo8++ghzc3P8/Py4f/9+hgGBGjVqsG/fPuLi4pSn3TqdjpCQENzc3HLkRiqtukLaDSVk7vO7ffs2nTt35ssvv8THx4fSpUtTunRpTp8+zY0bN4DUcUvLRwHZM75PVrfILidPniQyMlIJLpw9e5Zbt24p+S4yw8vLi/3795M3b15lxgPAvHnzuHDhghIoePo6fNb1CnD48GFSUlKUz2Xnzp2kpKRQs2ZNIiMjuXbtGl999ZXRMZ4ey44dO1K5cmXGjh1L3rx5adu2LbGxsUyZMoW4uDi8vLxYsmQJBoMBd3d35TgbNmxg165dTJkyhYIFC2Z4vQLUq1ePfPnysWLFCs6dO5duKcCsWbP46aef+Oyzzxg9evRzl8wIId4+Wf7GM3v2bJycnPjuu+/YsmULJiYmtG3bloMHD/Lzzz9nOJVRCCHEu0WbrCM6ToM25dXzBWi0OhZvOcvf4Y+fqjraqvGpaotryVd/cphZpqYqnOwssciFHAVXb0dz+34cdjbmypdrp0JlMRj0nNo5j+JuTTn3QM2wEb8SGxtLs2bN8Pb2pm/fvgwfPpxWrVop2fXDwsKMEpJlhr29PadOneL48eNUr16dvn370r17d4YMGUKXLl2wsLBg7dq17N69mx9//DHTx61QoQI+Pj7MmDGDpKQkXF1dOXjwIPv27SMwMBBTU1OGDRvGuHHjMDU1pXHjxsTExBAcHMy9e/eeOVX7yX6fP3+e0NBQoxufJ5UqVQpbW1vmzZuHmZkZZmZm7Ny5U3mKm5Us+LVq1WL8+PFMmzaNBg0aEBMTQ2BgICVLllSmdT9t4MCBHDx4kO7duytPf1euXMnNmzdZtGhRps+dFQMGDKBjx47069ePTp06Zfrzc3Z2plChQkyaNIm4uDiKFy/O2bNnOXDgAP369SMiIoKzZ88qOSAge8c3OyUmJtK7d2/+7//+j/j4eGbPnk358uXx9fXN9DHatm3LypUr+eyzz+jfvz+FCxfmzz//ZOHChXTt2lWZJfH0dfispR+QOjNk0KBBdOvWjevXrzNr1izq1q1L7dq1UalUODs7s2rVKgoVKoS9vT1//PGH8qQ/bSxr1KjBkiVLyJcvH56enty7d4+lS5fi5eVFnjx5aNiwITVq1GDAgAEMGDCAMmXK8Pfff/Pjjz9Sv379dMtGnmZqakqLFi1YuXIlBQsWpGbNmsprFy5cYOHChbi5ueHj40NYWJjRvmXLls1wmYcQ4u2R5YDAxYsXmTRpEk2bNiU2NpY1a9bQsGFDGjZsSHJyMnPnzmXBggU50VchhBBvgISkZGKyKV9AVKyG4PVh/HvvcYKw4gXt+NzXhYf3cq++tdn/ggG5lbAwJl5Lik6PuenjhIWWNg7UaDGES6G/cuHQKlJStCSWKkNAQAC1atUCYPHixQQGBjJ48GDMzc2pVKkSS5cuzXJW9/79+xMcHEyfPn3Yvn075cuXZ9y4cYSEhDBq1CgMBgPly5cnKCgo3Xr+F5kxYwaBgYEsX76cyMhIypQpw48//qisSW7fvj02NjYsWrSItWvXYm1tTdWqVfH39zdKCpeRzz//nClTptCrVy+WLl2a4TZ2dnYEBwczffp0hgwZgo2NjVJrvU+fPpw4cSLTeY46duxIcnIya9asYfXq1VhaWlK7dm1Gjhyp3Bw+rVy5cqxevZpZs2bx5ZdfolKpcHd3Z8WKFVSvXj1T582qChUqsGrVKmbPnp3lzy8wMJBZs2YxZ84cIiMjKVy4MAMHDqRv375s27aNcuXKGT1Zzs7xzU7Vq1enVq1afP3110BqPqtRo0ZlaUaGtbU1q1atYubMmcyYMYPY2FicnZ0ZPnw4n3/+ubLd09fh8z7Xzp07Exsbi5+fH2q1mpYtWzJy5EglEBgcHMzkyZMZM2YMarWasmXLMnfuXKZMmcKJEyfo1q0bQ4YMQa1Ws2HDBmW2ire3N8OHDwdSZywsWLCAOXPmMH/+fB49ekTBggX57LPPMj1DonXr1ixfvhxfX1+jGRC///47BoOBM2fO8Omnn6bbb8WKFUYBBCHE20dlyEz2lyd4eHiwePFiqlevzsmTJ+nduzd//fUXkDotavTo0Rw6dChHOvsmO3PmDPD8aWPvqoSEBC5cuICrq2umEv+IVydjnrtkvFMZDAZi45OJT9KSDbEAbt+PI3D9aSJjNEqbe9l89GpVGb1Oy5UrVyhTpswrr49/kdwOBgCE34xiyrJQLC1MM5yRoNHqSNLq+KqnF2WLOeZ4f+Qaz30y5tknrfzhTz/99NztZMxz1/s+3u/zvYF4u7x4Md5TihcvzsWLF4HUaWOJiYlcvXoVSC278mQ5HSGEEO8Gnd5AZKyGuMTsCQacu/qIGStPGAUDvKsXo39bdyzUuXdj/jqCAQClnR1wLmBLbEJyuqzsBoOB2MRknAvYUtpZanwLIYQQIudkOSDQsmVL/P39WblyJXny5KFy5cp899137N27l6CgoCwlNRJCCPHmS07RERGdSKIm5ZWTBwIc/OsWQevCSNLqAFCp4NMm5enQpDwmJrmXrOp1BQMATExUtPMuh5WFGY9iUhMz6vUGNFodj2I0WFuY0c67XK6OhxBCCCHeP1nOIdC7d28iIyMJCwuja9eujB8/nj59+jBgwABsbW2ZO3duTvRTCCHEa5CoSSY6LnvyBegNBjbtC2dX6L9Km4W5Kb1aV8a9bPq67TnpdQYD0niUy49fOw/W773M7ftxxCUmY2ZqQsnC9rTzLodHufwvPogQ4oVLBYQQQjxblgMC165dY/To0cqf3dzc2L17N1evXqV06dKSaVQIId4RsfHabFsioE3WsXTrOf669EBpc7C1wK+dB8UL2b36CbLgTQgGpPEolx+3Mvm4ejuamHgt9jZqSjs7yMwAIYQQQuSKLC8Z6Ny5M5s3bzZqs7W1xd3dXYIBQgjxDtDrDUTGJhGbTcGA6DgNM1efMgoGFC1gy5ju1d/rYEAaExMVZYs5UrVCAcoWc5RggBBCCCFyTZZnCJibm+PklHt1oYUQQuSe5BQ90XEaNMm6bDnenQdxBK4LIyImSWmrXCYvvVtVxtIiy/8EvRJzUxOc7C0wN3tzggFCCCGEEK9Tlr+NDRkyhOnTpxMbG0uFChUyLCNSpEiRbOmcEEKI3KPRphAZp0Gny47UgXDhegTzN/1NkuZxcKFhVWc6NCmPqUmWJ6i9ErWZCY52lpib5e55hRBCCCHeZFkOCHz77bfodDpGjhz5zG0uXLjwSp0SQgiRu+IStcTGJ6M3ZE8w4HDYHVbt/Af9/9YcqIB2H5TDu3oxVKrcnRJvYW6Kk50FpqYSDBBCCCGEeFKWAwKTJk3KiX4IIYR4DfR6AzEJWhKSksmOWIDeYODXA1fYefSG0mZuZkKvVpWpUj73s+ZbmpviaG+JqazLF0IIIYRIJ8sBgTZt2uREP4QQQuSyFJ2e6FgNSdmUL0CbrGNZyHlO/XNfabO3UePXzoMShe2z5RyZpQIs1GY42VlIkj4hhBBCiGd4qYxOWq2W9evX8+eff/LgwQOmTJlCaGgolSpVwt3dPbv7KIQQIptptClEx2lJ1umz5XixCVrmbvibq7ejlbYi+W3wa+dBXgerbDlHZqkAK0szHG0tcn15ghBCCCHE2yTLAYGIiAh69OjB1atXKV26NOHh4SQlJbF//36mTp3KsmXL8PT0zIm+CiGEyAbxiVpiEpKV9f2v6u6jeALXhfEwKlFpq1gqD31au2FlmbuVBFQqsLY0x8FGLcEAIYQQQogXyHKGpenTpxMfH8/27dvZtGkThv8tOv3xxx9xc3Pjxx9/zPZOCiGEeHUGg4HoOA3R8dpsCwZcvBHJ9BUnjIIB9TyK4NfO47UEA2wt1TIzQAghhBAik7IcENi3bx9DhgyhRIkSRl+4LCws+Pzzzzl37ly2dlAIIcSr0+n0RMQkEZ+YPckDAY6e+Y8f1/5FgiZFaWvbuCxdfCrkekZ/ExXYW6uxt1Xn6nmFEEIIId5mWX58o9FocHR0zPA1U1NTkpOTX7VPQgghspE2WUd0nAZtSvbkCzAYDGw7dI2Qw9eUNnMzE3r6VqRahYLZco6sMFGpcLBVY21pnuvnFkIIIYR4m2X5EY6bmxurV6/O8LWtW7dSuXLlV+6UEEKI7JGoSSYiJinbggHJKXqWbD1nFAywszZnWKeqryUYYGqiwsneQoIBQgghhBAvIcszBIYMGULPnj1p3bo1DRs2RKVSsW3bNgICAjh06BCLFi3KiX4KIYTIoth4LXGJWrIpXQBxicnM2/A34beilLZCea0Z2L4K+Rxzt5IAgJmpCkdbCyzUuZurQAghhBDiXZHlGQLVq1dn6dKlWFlZsWjRIgwGA8uWLePBgwfMnz+fWrVq5UQ/hRBCZJJObyAiJonYbAwG3ItIYPqK40bBAJcSTozqVv21BAPUZibksbeUYIAQQgghxCt4qW9SNWrUYM2aNSQlJREdHY2trS02NjbZ3TchhBBZlJyiJyo2+5YIAITfjGLuxr+JT3ycI6aOW2E6+1TALJeTB0JqMMDJ3vK1nFsIIYQQ4l2S5YDAxx9/zMcff4yvry/58uXD0tIyJ/olhBAiizTaFCLjNOh02TQtADh+/i7LQ86T8sQxWzcog0/tEq+ltF/azIDcrmIghBBCCPEuyvI3qiJFijBz5kwaNmxIr1692Lp1K0lJSTnRNyGEEJkUl6glIib7ggEGg4Hth6+xeMs5JRhgZmpC79aV+ahOydcSDLAwN5VggBBCCCFENsryt6rg4GD+/PNPJkyYgMFgYMyYMdSpU4fRo0fz559/YsiuAtdCCCFeyGAwEB2nISZeiz6bfv+m6PQsD7nAlj+uKm22VuYM7eRJddfcryQAYGluipMEA4QQQgghstVL5RCws7OjXbt2tGvXjkePHrFjxw527NhBnz59yJcvHwcOHMjufgohhHiKTqcnKk6DRqsju0Kx8UnJzN/4N5f+jVLaCuaxZmB7D/I7WWfTWTJPBViqTXG0s8TUJPdnJQghhBBCvMteOT3zo0ePePjwITExMeh0OhwcHLKjX0IIIZ5Dm6wjOk6TrckDH0QlEvjLae5FJCht5Yo50r+tOzZW5tl2nswyMVFhZWmOk50lJhIMEEIIIYTIdi8VELh58ybbtm1j+/bthIeHky9fPnx9fZk2bRoVKlTI7j4KIYR4QqImmeg4LbrsqikIXL0dTfD6MOKeqCRQs1Ihun7kirlZ7k/TV6nA1toCBxu1BAOEEEIIIXJIlgMCn3zyCefPn8fS0pKmTZsyZswYateujYlJ6hdGg8HwWpJNCSHE+yA2XktcopZsjAVw4sI9lm07T4ru8WwD33qlaFG31Gv5fa4CbCzNsVarkH9OhBBCCCFyTpYDAo6OjkydOpVmzZphZWWltN+/f59ffvmFDRs2sG/fvmztpBBCvO/0egPR8RoSNSlkR+7AtASwO4/eYPOBK0q7qYmK7s1dqVm58Kuf5CWoVGBlYYaFqQm3U1LSvS5BZyFyhvzdEkKI91OW54EuXryY1q1bK8GAP/74Az8/P7y9vQkMDMTU1DTbOymEEO+zFJ2eiJgkEpKyJxjw07JFrFm1gpW//WMUDLC2NGNIR8/XGgyw+V/OgIysW7eOadOm5XKvnu/ixYt8/PHHVK5cmebNm7/u7uSY8+fP4+npiVarfd1dybRbt27h4uLCxo0bX3dX3jje3t6MGTNG+XNwcDCLFy9W/hwQEICLi0uG278N4/p0/zOS9j5cXFxYu3ZthtvExsbi5uaGi4sLx44dA2Djxo24uLhw69atTPfn6X0y079jx44p/Tt06FCG21y5ckXZ5kX9efozf5G4uDi8vb0z/JzPnTtHnz59qFWrFjVr1uTzzz/n3LlzmT7289y4cYMhQ4ZQr149qlWrRqdOnThy5Ei2HFsIkbGXWhgaERHBggULaNKkCX379iU0NJQ2bdqwcuVKdu/end19FEKI95ZGm0JEdBKaZF22HXPporkcOHmNw3/fUdryO1oxunsNyhd3yrbzZEVaMMDB1uKZ28ydO5eoqKjc61QmBAUFcefOHYKCgt64YEV2OnDgALVq1UKtVr/urogcMGfOHBITE5U/t2/f/pk3yQUKFGDt2rU0atQol3qXs0xMTNixY0eGr+3atStdEKxRo0asXbuWAgUKZPocL7NPZvq3ffv2LB8vM6Kjo+nXrx+3b99O99qNGzfo2rUrSUlJTJ48me+//x6tVkvnzp25evVqBkfLvMjISLp27crVq1f56quvmD17Nvny5ePzzz8nNDT0lY4thHi2LC0ZOHr0KGvXrmX37t3odDqqVavG7du3CQoKwsvLK6f6KIQQ76WEpGRi4rM3eeDDqETl//kKpraVKerA/7V1x9b69dzsqVRgZ6XGzubtu9mMjIykfPnyNGzY8HV3JUcdPHiQ1q1bv+5uiFxSqFAhChUqlOFrarWaKlWq5G6HclDVqlU5duwYERER5MmTx+i1kJAQXF1duXDhgtKWJ0+edNu9yMvs82T/du3axbfffouZmfHX9u3bt6fr36vas2cPkydPJj4+PsPXf/rpJ6ysrJg/fz7W1qmlaGvVqoW3tzcrV65k3LhxL33uzZs3ExkZyfr16ylYMPUfqLp169K6dWsWL14s9xpC5JBMzRBYtmwZH330ET179uT8+fMMGDCAvXv3EhQUJGvOhBAim5w9e5YePXpQrVo1PD096dXrc878/bfy+t9hfzF0YB8++qAOrT9qzNRJ44iKjFRe37F9C00aenHh3BkG9uvJh9616fhJC9auXgHA9f9i6OBbD4CIy7u5tG0U1V0LMrSjJw/u/stXo4bg26wBvs0a8M2Xw7lz+/EU1NOnTuBdrxqnToQyctgAPvqgDp+0asaC4B/R6R7PXkhOTmbJwmC6tG+Fj3cdPu/WgZ2/bTV6n4f/2E//Xl350Ls2n7RqxvzAWZiq0ucLeJK3tze3b99m06ZNyvTYjRs3UrFiRdatW0fdunXx8vIiPDwcnU7HggUL8PX1xd3dnSpVqtCxY0eOHj2qHC8gIICmTZuyf/9+WrZsSeXKlfnwww/ZvHmz0XmXL1+Oj48Pbm5u1K9fn2+//Za4uDgAXFxcCA0N5fjx40ZTqK9fv87gwYOpW7cuVapUoVu3bpw8eVI5ZtpU5aVLl+Lj44OHhwcbNmwgICAAHx8fdu3aha+vL25ubrRu3Zq//vqL06dP0759e9zd3fH19U03hfbSpUv069ePqlWrUrVqVfz8/Lh586byetr04zVr1tC4cWOqVq3K4cOHiYiIYPjw4TRp0oQePXrw6aefphuD6OhowsLCaNCgAQDdunVjzJgxzJs3jzp16lCtWjUGDBiQ7mni7t276dy5M56enlSuXBkfHx9WrVr1wj5B6vKQtm3bUqVKFdzd3WndujW//fabsm/aFOywsDDatGmDu7s7LVu2zPBJ6oMHDxg8eDCenp54eXnxzTffGN3seHt7M2XKFHr06IG7uztff/01AP/88w8DBw6kVq1aVKpUifr16zNp0iSSkpKUfQ8fPkyHDh3w9PSkRo0a/N///R9XrlwxOv/u3btp27Ytbm5u1K1bl0mTJpGQkMCLuLi48PPPPzNmzBiqVauGl5eXcv5p06Yp07W//vprNBoN8Ozp/GPGjMHb2/uZ5wEIDAxUfn7elPanz5H29zAsLIxPP/0UNzc3GjdubLQEAVJzTQ0bNgwvLy9q1KjBuHHjmD17tlG/XFxcCAgIMNovo7686PrIiqZNm2JiYsKuXbuM2iMjIzl69CgtWrQwan96+v+YMWPo2bMnGzZs4MMPP6Ry5cq0bt2agwcPPnOfrGjevDlRUVFGv78g9fq8fv06H330UZaP+SwxMTEMHDiQGjVqsGjRogy3KV26NJ9//rkSDACwtramUKFC/Pvvv8889rx586hcuTLR0dFG7cuWLaNSpUo8evSIggUL0rNnTyUYAGBqakqJEiWee2whxKvJVEBg6tSpqNVqVqxYwc6dO/m///s/ChUqJIEAIYTIJnFxcfTu3RtHRycmT53J2AlTSEpMZPRwP+LiYgk7fYoRQ/pjaWHJuIlT8Rs8nNN/neSLwf3QaB7foBj0eiaMG0PjD5rx/Yw5uLlXYX7wHH7e8BszV52kWF0/AOyL1aBD/0l83qoSd+/cYlD/z4mMjGT0198yYsw3/HfnNoMHfE5kZIRRPydPHIu7hyeTp//AB019WLN6Odu3bn78+oSvWbdmJc1bfsyU6T9Qw6sW0yZ/y55dqTdqe37/jW++HE7xEiWZ9P1M+v/fAH7bvo0BAwYoiQ4zEhgYSP78+WnYsKHR1FudTseSJUuYPHkyX375JWXKlMHf35/g4GA+/fRTFi1axHfffUdUVBRDhgwxmhb94MEDJk6cSPfu3VmwYAFFixZl9OjRyg3dtm3bmDFjBl26dGHx4sX4+fnx66+/8t133wGwdu1aKlasSMWKFZUp1OHh4bRt25Zbt24xduxY/P39UalU9OjRI92U14CAAPr06cP06dOpW7cuAHfv3mXq1Kn079+fOXPmEBMTw+DBg/niiy9o3769EogfNmyYcmN67do1OnbsyKNHj5g2bRqTJ0/m5s2bdOrUiUePHqUbx9GjRzNu3Dg8PT0ZOXIkV65c4euvv2bUqFFUqFCB0aNHG918HDp0iNKlS1OkSBGlbc+ePWzcuJGxY8cyYcIELly4QLdu3ZTx3b9/P35+flSqVIng4GACAgIoVqwYEydOJCws7Ll9WrVqFePGjaNJkybMnz8ff39/1Go1I0aM4O7du0b79uvXjw8++IDAwEBKlSrF0KFDOXDggNE2c+bMoXDhwgQHB9OjRw9++eUXAgMDjbZZtWoVbm5uBAcH065dO+7fv0+XLl1ITExk6tSpLFy4kBYtWvDTTz+xYkVqgO3mzZsMGDCAypUrM3fuXCZPnsy1a9fo27cven1qxY6tW7fi5+dH6dKlCQoKYuDAgWzZsuWF13uaGTNmoFarCQwM5OOPP+ann37i448/5r///sPf359u3bqxfv16fvrppxce61nSlga0a9fumcsEXkSv1zN06FCaN2/OggULqFq1KtOnT+ePP/4AQKvV0qNHD06dOsVXX33F999/zz///MOSJUuyfK6sXB+ZYW9vT926ddMFk3bu3EmRIkVwd3d/4THOnj3L4sWLGTx4MEFBQZiamjJo0KB0N78vo2zZspQrVy5d/0JCQvDy8iJ//vyvfI40lpaWhISEMG3aNJycMl4+1rlzZ3r37m3UduPGDS5fvky5cuWeeeyWLVuSkpLC77//btQeEhJCvXr1yJs3L82bN2fEiBFGr0dHR3P8+PHnHlsI8WoytWSgRYsW7Nmzh379+lG7dm3atGlD48aNc7pvQgjx3ggPDycyMpKPP/mU8q5uABQrXpKQXzeRmJDAovmBFCtegsnTf1CSt7pWcuOzbu35bdsWPv6kA5CaKbz7Z31o7vsxAJUqe3Bg317WbdpOgcofY+VUAoBqbmXp3zX1ydKKpQuwsLTE/4dgbGxsAaha3YsuHVqxdvUK+vsNVfrZouXHdOvZJ3Wbal4cPrifI3/+QcuPP+Ha1XAO7t+D3+DhfNKhs3Kcu3f/S51h0ORDFswLwKtmHcaOn4S9jRobKzUu5crQs2dPDhw48Mx1yRUrVkStVpMnT55005X79+9vtF/ak8hu3bopbRYWFgwaNIiLFy8q+ycmJjJ58mRq164NQMmSJWncuDEHDhygTJkyhIaGUrRoUbp06YKJiQleXl5YW1srX/KrVKmCra2t8jPAxIkTlQB62muNGjXC19eX6dOns379eqVPH330EZ988onRe0lMTGT8+PHK0/jw8HBmzpzJ5MmTadeuHQAJCQkMHjyYa9eu4erqSmBgIFZWVixbtkw5Z+3atWnSpAmLFi1i9OjRyvE7d+6Mj4+P8ufQ0FD8/Pxo3LgxFy5coG3btuTLl88oV8DBgweV/jzZz40bN1KsWDEg9alhmzZt2Lx5M506dSI8PJw2bdooT9sBPD09qVmzJseOHcPDw+OZfbp58ya9evViwIABSpuzszNt27bl5MmTRk9su3Xrhp9fapCrfv36tGnThqCgIKMlHB9++CFffvmlMi6HDx9O97S1SJEiRjcihw4dwtXVlTlz5ihjWqdOHQ4fPsyxY8fo27cvf//9N0lJSfTr1095olmoUCH27NlDQkICNjY2+Pv7U79+ffz9/ZVjlyxZkp49e3Lo0CHy5cvH85QtW5aJEycC4OXlxbp160hOTsbf3x8zMzPq1avHzp07OXXq1HOP8zxp126hQoVeeimAwWBgwIABtG/fHoBq1aqxa9cu9u/fT/369dmyZQtXr15lw4YNVK5cGUidZt6kSZMsnysr10dmffTRR3z11VdGywZCQkIynSg0NjaWjRs3Urx4cSD1iXnXrl05evQoH374YZb7k1H/VqxYYbRsYPv27fTv3/+Vj/0ktVpN6dKls7RPUlISo0ePRq1W07Vr12du5+zsTI0aNdi2bZtynfz777/8/fffzJ49O8N99Ho933zzjRIwF0LkjEwFBGbOnElcXBxbt25l48aNDBo0CCcnJ5o0aYJKpZKZAkII8ZL0egPX7kSTpHLE3sGRkV8MppF3U6p71aK6V236DhhMUlIiF86doUOnbmAwoPtfOb4iRZwpUaIkJ08cVQICABUrpT7R0un1bNh/Dcys0etSE2NZWaT+2i9W0E7Z/tTJ41TxrIalhaVybBtrG9zcPTl5/JhRfytWNn5alq9AAZKSUp8Kn/n7NAD1GxpPTZ4weQYA/964xoP79+ja/TOsLUywMDchJSWFGjVqYGtry+HDh2nUqBEpKSmkpKSg0+lISUl54dI0V1dXoz/PnDkTSE2Ae/XqVW7cuKGUw306QdiTN0Bpa6bTpnPXqlWLtWvX0rZtW5o0aULDhg1p2bLlc/sSGhpK48aNlZtIADMzM1q0aEFQUJDRVPWn+52matWqys9pN4xP3kA7OjoCqdN7ITW/j5eXF5aWlqT87/OztbWlevXq/Pnnn0bHfvqcNWvWJCAggDNnzlCqVCny5ctnFEAwGAz88ccfzJo1K10f04IBkBqwKVasGMePH6dTp07Kl/f4+HiuXbvGv//+y5kzZ4D0n8HTfUrLhB4TE6N8fmkZ3p/et02bNsrPKpWKpk2bEhAQYDStv3r16kb7FC1a1GgJR0Z9qFevHvXq1SM5OZnw8HBu3LjBpUuXiIiIUMbfw8MDCwsL2rVrh4+PDw0aNKBmzZrKE+UrV65w9+5d+vXrp3wugHK9Hz16FF9fX+V6T2NqaqpcY56enkbtTk5OVKpUyWgtuaOjI7GxsbxuT/Y1LXiX9nfp6NGjFCtWTAkGQOo12rhxY+WzzaysXB+Qeg2nLWtK+73ytCZNmvDNN9+wa9cuPv30U+7fv8+JEycYN24cERER6bZ/Wp48eZRgADz+XfLkjKRX0bx5c3788UeOHj1KvXr1CAsL4969ezRr1ow9e/YYbZvyVLnWJ6+n7BYXF4efnx9nzpxhzpw5ODs7A6kztwwGgzLeaTNmWrVqxfjx43nw4AH58+cnJCQEW1vbDJezJCcnM2bMGHbu3Mm4ceMyNVNDCPFyMp1U0NbWlk6dOtGpUycuX77Mhg0b2Lp1KwaDga+++ooWLVrQokULypYtm5P9FUKId0bY5QeEHL5GdKyGuEQtJesNICJ8L7t37WTrrxuwsLCgmY8vnbp9hl6vZ82q5axZtTzdcdQWxmX6LC0tSdSksOjXs5y7+ig1a5/BQF4HSwa2r0KXDcb7x0RHsW/P7+zbYzyVE8DR0XjaqOVT5zJRmWD4X9LDmP89OXd0yjh5Vtrrs/2nMtt/arrX79+/z61bt/jggw+M2r///nvatm2b4TEBo7WsAGfOnGHChAmcOXMGKysrypYtq0x3f3qadloJXUjN5v3kNs2bN0ev17N69Wpl2ruzszMjRox45pPD6OjoDJ/65suXD4PBoOQfyKjfaZ4MJmTUz6dFRUWxffv2DDOOP53I7Olzzp49m3nz5hESEsLu3btZvHgxderUYeLEiTg7O3PmzBk0Gg3VqlUz2u/JNb5p8ubNq8yeiIiIYPz48ezevRuVSkWJEiWUG/OnP4On+/Tvv/8ybtw4jhw5grm5OaVLl6ZChQoZ7vt01va8efNiMBiUYAmkHzsTE5MX9kGv1zNr1ixWrVpFQkIChQsXxt3dHQuLx1UwihYtysqVK1mwYAHr169nxYoV2Nvb07lzZ4YOHapUxJgwYQITJkxIN14PHjzgwYMHdO7c2aj9yes9o2vhWdfN62Zp+dTvhifGOTIykrx586bbJ6O2F8nK9QGwadMmZYZImpCQEKP+2tra0qBBA3bs2MGnn37Kjh07lKn6mQlYPH2Npd2Ap90Iv6pSpUrh6urKjh07qFevHtu3b6devXo4ODgYbfcyvz9f1n///Ue/fv24du0as2fPNprt0bRpU6OcIi1btsTf3x8fHx++++47fvvtN7p3705ISAgffvhhumsnLZfB8ePH+eabb+jSpUu2918I8ViWqgykKVeuHGPGjGHEiBHs27ePDRs2sHDhQubNm0e5cuXYsmVLdvdTCCHeKWGXH/DT9vMYDJCi16M3GHDMVwRz208pXr0TdcrCpbA/2LJ5Pfny50elUtGuQ2e8m/qkO5bFUzfpUXEaFm4/ye0Hj28+ba3NGd29OvY26cv62drZUbVaTTp0Sj/dM215QmbY2qbOOoiOiiR/gcc3jP/euEZMdDR2dqmvjxo1KsNs0Q4ODhQoUID169eTlJTE9evXKVmyZJYCzWlTS11cXAgJCaF06dKYmJhw4MABdu7cmenjpPH19cXX15fY2FgOHTrEwoULGTlyJNWqVcvwptjBwYGHDx+ma3/w4AEATk5O3L9/P8v9eB47Ozvq1KnDZ599lu61p7OSZ7TvyJEj8fPzY8+ePdy8eZOFCxcyYcIEFixYwMGDB6lTpw7m5uZG+0U+kcwyzcOHD5WnpCNGjODq1assW7YMT09P1Go1iYmJ/PLLL8/tj16vp2/fvpibm7N+/XpcXV0xMzMjPDycX3/9Nd32UVFRRgGYhw8fYmpqiqOj4yuN84IFC1i2bBkTJkygWbNmyrWbtmwjjbu7O4GBgWi1Wk6ePMnatWuZN28eFSpUUK7bZ13varWaqKgoVq5caXRDVLRo0Zfud9qN6NNPwTOTxDAnFSxYkOvXr6drfzrHBTy/71m9PgAaN26sLNVJ+72SP3/+dLMqmjdvzsiRI4mIiGD79u0vtfQgJzVv3pzFixczfvx4duzYkW6tPaD8/nzSq1xPz3Lx4kV69eqFRqNhyZIl1KhRw+j1uXPnotVqlfFOmz1iZ2eHt7c3v/32G7Vq1eLy5ct88803RvvevXuXzz77jFu3bjFr1qxsTZoohMhYppIKPouZmRlNmzZl3rx57N+/ny+++CLdVCUhhBDG9HoDu47dQK9P/YKbkqLnwfW/OLJ2DFaqRDQpei49smbwF2OwtbUjIiKCcuUr8O+/13GpUFH5r2Sp0ixbPI+wv04YHX/ehjCjYIDa3ISKpfIqwYC0J+FpPKpU48b1q5QtW145dnkXV9atWcmhg/sy/b7c3KsA8Ofhg0btC+YGEDjHn8qVXMibNy+3bt3Czc1N+a9gwYLMnDmT8+fPo1arcXNzo1KlSpQuXZpKlSopya2e7ndGrl69SlRUFN27d6ds2bLKPmkZv7PyxG7o0KHK+nQ7Ozs++ugjBgwYQEpKyjNvNmvUqMG+ffuMZgLodDpCQkJwc3MzWpufXdKqK7i6uipjWrlyZZYtW5Yuc/qTbt++TcOGDZVkZUWKFKFnz57UqVOHO3fuABnnDwA4efKkUVDg7Nmz3Lp1S8nHcPLkSZo1a0bNmjWV95yZzyAyMpJr167Rrl073NzclIDGs/bdvXu38rPBYOD333+nWrVqrzzOJ0+epGzZsnzyySdKMODevXtcunRJ6cOyZcto3LgxWq0WtVpN7dq1lYSTd+7coXTp0s+93v/55x/MzMyoVKmS0evPSuaWGWkzCu7du6e0JScn8/cT1Uoykpm/W6/Cy8uLW7duGZXHS0pKUpIOprG1tTXqO2CUHyGr1wekBuHSxjbt98rTAS5IDRyo1WpWrlzJ6dOn37iAwEcffURUVBTz5s0jOjo63UwAQPn9mV3XU0b+++8/PvvsM1QqFT///HO6YACkVot4cryfTEjaunVrTp8+zc8//0yRIkWMgmVxcXH06NGD+/fvs3TpUgkGCJFLXmqGQEby5ctHnz596NOnT3YdUggh3knXbkfz38N4ErXJmPzviZ5jwTIY9Hr+3rOAopWacum+mu8m/kp8fBwNGnlTp14Dvhw5hMkTvuaDZh+h1+n5Zc1PXDh/lm49Un/v3ryX+sQrNiEZ8//NKm5WswTrTqgxNXm8htTW1o5zZ8IIO30Kdw9PuvXsw6D+Pflq1FBatWmHWq1m668bOfzHfsZPmp7p91WmXHkaNm7C/OA5aJKSKFvOhWNHD3Pk8EH8Z/2AvY0lw4YNY9y4cZiamtK4cWNiYmIIDg7m3r17VKpU6bnHt7e35/z584SGhj5zPWmpUqWwtbVl3rx5mJmZYWZmxs6dO5WnZllZ01urVi3Gjx/PtGnTaNCgATExMQQGBlKyZEllivLTBg4cyMGDB+nevbvyJHPlypXcvHnzmWW8XtWAAQPo2LEj/fr1o1OnTlhYWLB27Vp2797Njz/++Mz9nJ2dKVSoEJMmTWLAgAHo9XqOHz/OgQMH6NevHxEREZw9ezZdGThIHcfevXvzf//3f8THxzN79mzKly+Pr68vkPrkfOvWrVSqVIlChQpx6tQpFixYgEqleu5nkDdvXpydnVm1ahWFChXC3t6eP/74Q8ns//S+06dPR6PRUKpUKdatW8eVK1dYvjz9spqscnd3Jzg4mAULFlClShVu3LjB/Pnz0Wq1Sh9q1aqFv78/fn5+dO3aFVNTU9asWYNaraZx48aYmpo+93r/+uuvjZY2ZAcHBwc8PT356aefKFGiBA4ODqxYsYKkpKTnLjWwt7fn1KlTHD9+PF3Ohezg6+vLggUL8PPzY8iQIdjb27N06VIePXpkdLPYqFEjQkJC8PDwoESJEmzcuJEbN24or2f1+sgKa2trGjZsyIIFC3B3dzfKkZFTli1blq7N3t4+wyn+xYoVw83Njfnz59O0adOXWjoSHh6e4TmrVq2a6TX6kyZN4tGjR0yYMIG4uDhOnz6tvGZra/vCGV3169fH0dGRtWvX0rt3b6P8Bj/++CPXr19n0KBBmJmZGR1brVZTsWLFTPVRCJE12RYQEEII8WJJ2hRuP4zjXmQ8dtaPn2JaWDvg6TOQKye3cunIanQpWhKKl+LbSdPxrJr6BGb6zECWL13AhLGjMDM3p7yLK/6zg6lY2Y29J26y78TjGtcmKhWdPnShfhVn1gUZ96FL98/5adkivhwxiKUr11OmbDl+CFrEkgXBfP/dOAwGA6VKl2Hi9zOpW68hWfHVuEksXzKfDet+JjoqihIlSzHdfxa+zVOXOrRv3x4bGxsWLVrE2rVrsba2pmrVqvj7+7/wC/jnn3/OlClT6NWrF0uXLs1wGzs7O4KDg5k+fTpDhgzBxsYGV1dXVq5cSZ8+fThx4sQz67E/rWPHjiQnJ7NmzRpWr16NpaUltWvXZuTIkRk+YYTUJXWrV69m1qxZfPnll6hUKtzd3VmxYsUr3Wg9r0RdhQoVWLVqFbNnz2bUqFEYDAbKly9PUFBQhk8RnxQYGMisWbMIDg4mMjKSwoULM3DgQPr27cu2bdsoV65chksjqlevTq1atZQqAt7e3owaNUp5Mj916lS+++475Yl5yZIlmTBhAlu2bOHEiRPpjvek4OBgJk+ezJgxY1Cr1ZQtW5a5c+cyZcoUTpw4YVQ94ttvv2X+/PncvHmTihUrsmTJkmy5oe3Xrx+RkZGsWLGCoKAgChcuTOvWrVGpVMyfP5+YmBgqVKjAvHnzCAoK4osvvkCn01G5cmWWLFmiZGp/3vXu7Oyc7QEBeDz2Y8eOxdbWlnbt2lGtWjXWrVv3zH369+9PcHAwffr0yTAXxasyMzNj8eLFTJ48WcmU36pVKxwdHbl27Zqy3ZdffklKSgrTpk3DzMyM5s2bM3z4cMaOHatsk5XrI6uaN2/Ojh07Ml1d4FV9//336dqKFy/+zDX/zZs358yZMy89e+HMmTNKcs8nDRkyJFMBAa1Wy/79+wEYP358ute9vLxeWAIzLcnqTz/9RKtWrYxeSytJGBAQkC4Q6ezszN69e1/YRyFE1qkMmSmEK14o7Resm5vba+5J7ktISODChQu4urq+scmO3jUy5rkru8Y7LkFLbEIy1+5EM2/j31ioTVGbpZ+qq03Wo0nW0b+tOyUK2z/3mDq9nnW7L7P/1ONggKXalL5t3KhYKusJu7KTiUqFg60aa8uMb56fJyeu8eDgYNRq9VtXvirtyXdadvWckJXx7tatGwkJCeh0OsLDwylevHiO3EQ+z8aNG/nyyy/Zs2dPjqyRTnP+/Hm6dOnCsWPHMrUMYcyYMYSGhio3Li4uLgwcOJBBgwal2zZtzH/++We2bt1KwYIFOXDgQIYZ4f39/Vm4cKHRDZe3tzdeXl5MnZo+QeezPL3P8/qXplu3boSGhuLp6cmaNWsy3GbYsGFs376dNm3aZNify5cvc/XqVeLi4vjqq6+Uz61du3YUKlSIwMDAZ57/p59+YunSpeluBrVaLUuWLGHz5s3cvXuXggUL0rJlS/r27fvMzyqz17ler2ft2rWsXr2aW7dukSdPHj744AMGDx6cYaJHkbH3/bvK+3xvIN4uMkNACCFymMFgIDpeS0JSMgZDasm/gnmtuf0gDnMbtdENgMFgIF6TjHN+W6PSgBlJ0qaw+NeznLnyODFXHntL/Np74Jz/9X5pNVGpcLRTY2WR9WBATpkzZw4DBw583d3Isrlz52aYlO51un37Nnq9nqCgoHSVDN4lBw4coFatWjmS++FJJiYm3Lt3j1OnTqWr6ABkGHAJDAzM8s3py+yT1r/Tp09z9+5dpaRemoSEBKWs57MkJCQwZMgQatasCcCJEycIDg7m7NmzGSbHSxMSEsLUqVMznKUyadIktmzZwoABA3Bzc+PMmTMEBQVx584dpkyZkuX3+KRFixbxww8/0KtXL2rXrs21a9f48ccfuXz5MkuWLJFy20KId4oEBIQQIgfpdHqiYjVoknWkTccyMVHhU6skq3b+Q1S8FhsLc8zMVKSkpAYDrNRm+NQqiYnJs790RsYmEbwujJv3HyevK17IDr92HjjYpq8kkJtMTVJnBrxJwQCRvVJSUqhQoQING2ZtScnb5uDBg7Ru3TrHz1O4cGEMBgO//fZbuoDA6dOnuXfvHuXLlzdqf5n11C+7BrtixYqEh4ezY8cOevbsafTavn37sLKywt7+2bOZPDw8+OGHH5gxYwaQutSjUqVKLFq0iFq1aqXb/tGjR8yZM4e1a9fi6OiY7vXIyEh++eUXRowYocz4SUtoOXPmTEaMGPHSgSq9Xs/ChQv59NNPGT58OAB16tTBycmJYcOGcfbsWXniK4R4p+RsWlkhhHiPaZJ1PIpOIumJYECaCiXz0OXDCjjnt0WTrCMmXosmWYdzfls6f1iBCiWf/WX25r1Ypq04YRQM8CiXj+Gdq70RwQAnO4ssBQPOnj1Ljx49qFatGp6envTs2dMoK/qJEyfo2rUrHh4eeHl5MXr0aCIiIpTXN27cSMWKFQkLC+PTTz/Fzc2Nxo0bs3jxYmUbFxcXIPUJadrPAJcuXaJfv35UrVqVqlWr4ufnx82bN5XXjx07houLC0eOHOHzzz/Hw8ODunXrMmPGDKPyaFqtlh9++IEPPvgAd3d3fH192bRpk9H73L17N23btsXNzY26desyadKkF5aD8/b25vbt22zatAkXFxdu3bqlvN9169ZRt25dpcqATqdjwYIF+Pr64u7uTpUqVejYsSNHjx5VjhcQEEDTpk3Zv38/LVu2pHLlynz44Yds27bN6LzLly/Hx8cHNzc36tevz7fffqtUTggNDSU2Npbjx4/j4uLCxo0bAbh+/TqDBw+mbt26VKlShW7dunHy5EnlmLdu3cLFxYWlS5fi4+ODh4cHGzZsICAgAB8fH3bt2oWvry9ubm60bt2av/76i9OnT9O+fXtlTI8cOQJA27ZtuXjxIgkJCZn6/NasWUPjxo2pWrUqhw8fJiIiguHDh1O3bl3lfJs3bzYag+joaMLCwpQqC5kZ31fh4+PD77//ni5fxPbt26lTp066G2Nvb29lGUna2P72228MHjwYT09PvLy8GDt2rNE19uQ+WZGWcC+tIsXT/fvwww9fWOLSx8dHqdqxbds2Vq1aRb169TLcdt68eRw6dIiAgAAaN26c7vW4uDg6duyYLh9IWu6GJ6+BJ2k0Gnr16sWsWbOM2lNSUqhVqxaTJk0iLi6O1q1bKwkyM3tsIYR4W0lAQAghckBCUjKRMUkk655dYq1CyTwM7uBJ/7bu9Gheif5t3RncwfO5wYAzVx7iv+okUbEapa1JjeL0a+OOhdo0W99DVqUFAyzUmZ98FhcXR+/evXFyciIgIIDZs2eTmJiIn58fCQkJnDx5kp49e2JpackPP/zAV199RWhoKN27dycpKUk5jl6vZ+jQoTRv3pwFCxZQtWpVpk+frpQ1W7t2LZBaRz7t52vXrtGxY0cePXrEtGnTmDx5Mjdv3qRTp07p6qOPGDGCatWqMW/ePHx9fVm0aJFRkrYRI0awdOlS2rdvz/z586lXrx5jxoxRbrS3bt2Kn58fpUuXJigoiIEDByrTnZ+XyicwMJD8+fPTsGFD1q5dS4ECBYDUm9MlS5YwefJkvvzyS8qUKYO/vz/BwcF8+umnLFq0iO+++46oqCiGDBlilH39wYMHTJw4ke7du7NgwQKKFi3KN998w+3bt4HUm7UZM2bQpUsXFi9ejJ+fH7/++quSIHDt2rVUrFiRihUrsnbtWho1akR4eDht27bl1q1bjB07Fn9/f1QqFT169CA0NNToPQUEBNCnTx+mT59O3bp1gdTa41OnTqV///7MmTOHmJgYBg8ezBdffEH79u0JCgrCYDAwbNgw5XPPyucXGBjI6NGjGTduHJ6enowcOZIrV64wYcIEFi5cSMWKFRk9erTRzf2hQ4eMSqZldnxfVvPmzZVlA2n0ej07duzIdBK58ePH4+zsTHBwML169WL9+vXMnTv3lfuW1r+0ZQNp4uLiOHjwYLqb51fVsWNHdu7cSbNmzTJ8vVixYnz77bfKTXqaPXv2YG5uTsmSJTPcz8LCgpo1a7Jr1y6jv3eHDx8mMjKS1q1bY29vz9ixY9PN1EgrcfmiLPpCCPG2kSUDQgiRjQwGAzH/yxegz0TKVhMT1QsTB6bZf+oWa3ddJO17rEoFnzZ1oVHVnEuqllmmpiqc7CyxMM9aUCI8PJzIyEi6d+9O1apVgdQncatWrSIxMZFFixZRqlQp5s+fj6lp6rE9PDxo0aIF69evp2vXrkDquA8YMID27dsDUK1aNXbt2sX+/fupX78+VapUAaBQoULKz4GBgVhZWbFs2TJlXXXt2rVp0qQJixYtYvTo0Uo/27dvrzzdrF27Nrt372b//v107NiRS5cusXPnTr766it69OihbHP79m2OHTtGixYt8Pf3p379+vj7+2MwGFCpVJQsWZKePXty4MABGjVqlOH4VKxYEbVaTZ48eZR+p+nfv7/Rfvfv32fYsGFGWdYtLCwYNGgQFy9eVPZPTExk8uTJyhTrkiVL0rhxY06fPk2TJk0IDQ2laNGidOnSBRMTE7y8vLC2tiY6OhqAKlWqKOOVdsyJEyeiVqtZsWKF8lqjRo3w9fVl+vTpStlHSK2n/sknnxi9l8TERMaPH688jQ8PD2fmzJlMnjyZdu3aAanr0AcPHsy1a9dwdXXN0ufXuXNnfHx8lD+Hhobi5+dHkyZNgNTs6I6Ojka5Ag4ePKj0Jyvj+7Lc3NwoVqyY0bKBEydOEBUVRZMmTdiwYcMLj9GwYUPlfdeuXZvDhw+zf/9+Zer7q2jUqBFWVlZGywZ27dpF3rx5M8x78CrKlCmT5X127drFpk2b6Nq1Kw4ODs/crl69euzfv5+TJ08qFSlCQkIoXbr0M5cChIWFsWDBAho3bpxu6YYQQrztZIaAEEJkE53eQESMhvjEzAUDMkuvN7BuzyXW/P44GGChNmVAO483IhhgZmpCniwGA/R6A+E3o0gwOOLg6ET//v0ZN24cu3btIl++fAwZMgRbW1vOnDlDw4YNMRgMpKSkkJKSQrFixXB0dOTnn382Oqanp6fyc9pN9POm5B89ehQvLy8sLS2VY9va2lK9enX+/PPPZx4bUgMLacdOmxb/9NPMgIAAvvvuO65evcrdu3fx9vZmzZo1fP/996SkpFCjRg1sbW05fPgwgNKHtP9eVATI1dXV6M8zZ86kR48eREREcOLECTZs2MCWLVuA1CUNT3ry5jWt9N3PP/9M27ZtqVWrFteuXaNt27YEBgZy5swZWrZs+dxybqGhoTRu3NgoYV1aebGzZ88SHx//zH6nSQsIAeTLlw9IDf6kSZsyn9bfrHx+rq6unD9/Hk9PT7RaLTVr1iQgIIDBgwezbt06Hj58yOjRo5U+pK3n//XXX5VjbNu2jZiYmOeO78mTJ7l9+zYNGjRI9/npdDpSUlKYPn06Li4uRtUu7t69y5gxY2jevLnRsoGQkBAaNWqUYSLAu3fvGi3JAIiKijL685PXKaTODtm0aRMdO3ZMd7w0w4YNw8XFJd3SAktLS7y9vZVlAxs3bmTMmDHUq1cvXZI9vV6vfCbJycksXLiQZs2a8c033wCkW57xsuLi4pg2bRp169Zl4MCBWFpaUqxYMfR6vdHvjLT/IPVaKFSoECEhIUDqMoLdu3c/M1fEyZMn6d27N0WLFs2wTKAQQrztZIaAEEJkg+QUHVGxGrQpz14i8DI0Wh1Ltp4l7PJDpc3RzoKB7Two+oIqBLnBzNSEPPYWmJtlPhgQdvkB6/de5vb9OFJ0ekrX9yMifA/btoWwdu1aLC0tadGiBQ0aNFASfC1cuDDdcdJmDKSxtLQ0+rOJiclzb6qjoqLYvn17hhncn05I9rxjp92E5c2bcZnHtNcnTJigtC1fvlz5+f79+9y6dYsPPvjAaL/vv//+mfXIgXRlvM6cOcOECRM4c+YMVlZWlC1bVpnu/vQ4WFlZKT8HBwcDUKtWLQYMGECNGjXQ6/WsXr2a4OBgAgICcHZ2ZsSIEc+szx4dHa3cxD8pX758GAwGJf9ARv1Ok9FN75P9fFpWPj9ra2ujigGzZ89m3rx5/Pbbb+zcuRMTExPq1KnDxIkTcXZ25syZM+h0unTVBX755RdlZsLzxvfpqf8ATZs2VZZlQGrOhSf7O2DAABISEpg/fz6nTp2iSpUq/P7773z77bcZvv88efIY5cMAMDc3zt3xrL8DT0/9T/OiigEfffQRAwcO5O7du0qQ5+l1/ABfffVVuhwaw4cPJzo6mkWLFhEQEED+/Pn59NNPn3muFzEYDAwdOpQTJ06QlJSEq6srtWrVYurUqcTFxVG4cGG+/PJLo31CQkJQqVQ0b96czZs3M3bsWPbt20dCQgItW7ZMd47t27czZswYSpYsyaJFi3Bycnrp/gohxJtKAgJCCPGKEjXJRMdp0WXntAAgOk5D0Pow/r0bq7QVK2DLgPYeONlZPmfP3GFuaoLTSwQDgtaHkZiUgp2NOeam5iRbF8HctiMlanSmmZs5504d4Oeff0alUqFSqejZs2e6NdTt2rVTpnu/LDs7O+rUqcNnn32W7rUXJUh7Ulp29YiICKOSbFeuXCEqKkp5fdSoUSxdupRKlSoZlT90cHCgQIECRtPqAYoWzfzsj7RcDC4uLsr0ZxMTEw4cOMDOnTufu29kZCSQ+jS5UqVKAPj6+uLr60tsbCyHDh1i4cKFjBw5kmrVqmVYAs7BwYGHDx+ma3/w4AEATk5O3L9/P9PvJzOy+vk9WTHAzs6OkSNHMnLkSK5evcqePXsIDg5mwoQJLFiwgIMHD1KgQAHlyXdaQMPc3PyF42tqakrBggX57bffjNrnzp3L2bNn+eabb3B2djaa1q5WqylevDgApUqVYseOHSQlJaHRaJ65nEStVr9UCUFHR0eSkpIyTBD4oooBDRo0wMbGhh07dvDPP/8AZDiFfuDAgXTp0oX79+8zcOBA+vbtS9++fZUElI0aNeLQoUOvFBA4d+6ckiPE19eX77//HrVarSw12rNnT7q/U/nz5yc6OpoWLVqwZMkSjh07xvbt26lRowbOzs5G2y5evJgZM2bg5eVFUFAQdnavPwArhBA5QQICQgjxCmLjtcQlarN1iQDA7ftxBK4/TWTM4+SBbmXy0qt1ZSyzkLQvpzwrGHD27FlmzJjB2bNn0ev1eHh4MHToUKpUqYJeb2DuihBO7fuF+Mh/MTU1xy6PM7ERd6jfYRxxyZb8uvs4p3b/grW1Nbt27QJg9erV5M+fn169egGPKwasXr2a1atXK9N4r127xnfffcfx48eB1CnaT05VV6lUBAYGUr16dRYuXEh0dDQ7duygYMGCjBgxAlNTUwwGA1988QX379/n7t273Lt3D4A//vhDqaEOqTfRt2/fxs3NTXni/XRJNn9/f+7du8e6devImzcvQUFBxMfHs3//fvbv388vv/zCl19+ybVr15g4cSI//PADycnJrF69mlKlSrF48WK2bNnCnTt32LJlC9evX2fo0KHK8ZcvX87+/fv5+uuvmTx5MlFRUVy7do2zZ88qic8OHjwIpAYjIiIilCfIcXFx2NraGj1h3rhxIx4eHhw9epTo6Gjs7Ow4efIk8fHxFC1alJSUFO7fv0/BggXRaDSEhYWxdOlS1q5dS2RkJDt27CBfvnzs2rWL4cOH88MPPxAeHo6lpSXnzp1TggPffPMNS5Ys4euvv1byGEBqxYeZM2dy/PhxZXr3nTt3lMDIhQsXANi7dy9jxowhJiaGsLAwChYsyLRp0zh69CgxMTFYWlpSo0YNZeYDQHx8PGFhYcycOZPbt2/TqVMnvLy8+Oeff/j3338xMTFBrVYTHh6ujFvBggWVIMbVq1eB1CUMTyaVSxtfvd54ZpCPj48yJT2Ni4sLGzZsoG7duiQlJRlVqUhbMjB16lTq1avHihUruHTpEg4ODtSqVQtzc3OjoEHaPk8vGcgMMzOzF1YMSHtfT1Or1TRp0oSdO3cqSzcyUrRoUYoWLcqyZcuwsLBQ8m+k+eabb54b8Dp16hSdOnVi3rx5Ru0XLlzg448/JjAwkL179wKp+SHGjRunBG9Kly5NQkICer0+XU6AtOUTpUuXplKlSoSEhHDgwAHGjh1rtN2aNWuYPn06zZs3Z9q0aelmigghxLtEcggIIcRL0OsNRMYmEZsDwYDz1x4xY+UJo2BAo6pF+b9PPN7oYMCzKgb06tWL2NhYtvy2n/3rp6O2sMCzaV9c63QgPuYBydoETu4IJun+OW7fvI5OpyMhIQEvLy8GDx6MVqtl+vTpBAcHs3fvXmUNeuPGjZWKAQCDBw82yjifkpLCgQMHlIzzaTfuQ4YMoVq1asoU/iVLljBx4kT++OMPBg0axPbt2wkLC6N9+/Z88cUXACxcuNCoYsDly5exsrIiKCiIoUOHYm5uzrRp01i8eDFHjhxh2rRp7Nu3jwEDBmBqasqwYcNISEjAysoKd3d3hg4dyldffcWdO3fQ6/XPrRhQokQJChQowN27d5XxSPNkxQBLS0sSEhIYPXo069at45tvvmHVqlVA6pPdxYsXU6NGDYB0FQMg9altvXr1KF26NH/++ScnTpygQ4cOdOnShVu3bgEQG/t4tgo8rhgwevRo9Ho9mzZt4vbt24wfPx6VSoWJiQk2NjYMHjyYyZMnA9CtW7d0FQMAo4oBaUkHhwwZkq5iwObNmxk9ejSDBg3i4cOHtGzZktOnTyslJ2NiYtizZ49RxYAzZ84oFQOcnZ3R6/Vs3boVFxcXhg8fTpMmTYiOjubhw4fcuXOHs2fPGs2EKFWqFJCa5G///v0cOnSIb775htWrVwOkqzKQVjHgSZmtGJBWZu/o0aNUrlxZqRjw4MED7ty589x9MyutYgCkLh9YtmwZ8+fPZ9++fZiamhIXF5fus35y37/++osrV6688DwXLlygRIkSHD9+nDZt2vD1118DqUkoly1bZvTfihUrlP2qVq1K8eLF0wVVtm3bhqOjIwULFmTTpk1KyciwsDBOnz7N6dOn2bx5M05OTumWjTytdevWbNq0CYPBYJRw8sGDB3z//fc4OzvTpUsXzp8/rxz79OnTRiVPhRDiXfD6v1kKIcRbJjlFT3ScBk2y7sUbZ9Efp2/z886L6P+37lcFtG9SHu/qxbL9XC/jecsEnlUxYO3atcTHx7N4QSCWdgWo7jMQ0//t71iwNAfXjiclWcOFQ6tI1qbeIHbo0IGPP/4YV1dXKleuTN++fQkMDMTa2lqZ1l6pUiWqVKmiPL21sLAwyjg/bdo0Hj58qGScb9WqFT///DPx8fG0adOGIkWKUKlSJTp37sy6devYsmULxYqljvPIkSPp0aMHx44dA6B69epGFQMcHBwoU6aMkoXe2dmZPn36sGDBAhITEylTpgw//vijsqyhffv22NjYMGrUKM6ePcu///5L1apVadGiBXPmzHluxQAnJyemTJlCdHQ0KSkpRmu/n6wYUL58eSZNmsSlS5eYMmUKVatWxdvbm71792Jra4uXlxfHjh1j//79VK5cGTCuGJAnTx7y5MnD1atXsba2xsHBgSVLlmBpaUmDBg04f/48s2bNok6dOsr5n6wYULNmTQYPHsyDBw+Ii4ujfPnyfPvtt5w6dYqZM2cyYsQI/P39KV++PJ6enkrFgDRPVgxIuxnVaDTpKgZ88MEHyg1cgwYN+OSTT4iPj2fp0qWUL1+ewMBATp06hVqtJjk5GUjNEv9kxYAqVapw7949QkND2blzJ4ULF8bX15dt27axYcMGypUrZ5S/IG26uMFgYMiQIdjY2ODq6srKlSvp06cPJ06cMFpLn1Yx4Mma9ZmtGFCiRAkAJdeBmZkZtWvXZuHChemSBr6stIoBCQkJHDp0iEOHDimvrVy5Eni8lORpderUwd7eHisrqwzzEDwpIiKCe/fuMWLECAYOHMidO3dYtGgRBw4c4MCBA0bbmpqa0r17d+XPrVq1YsmSJcrfIYPBwPbt2/Hx8WHfvn0YDAbOnDmT4bKDTp06YWLy/GdeaRUwGjdubLQc4MCBAyQlJXH79m26dOmSbr8X5fYQQoi3jQQEhBAiCzTaFCLjNOh02TstQG8wsHn/FX4/dkNpU5ub0KtVZTzK5c/Wc72sZwUD9HoDV29HG1UM8PHxoX79+tStW5eRI0eSmJhI+KXzFHb1RqvTYfG/7+rW9vmwdSqMjX1+Krcdx40Lhwk/sooOHTooT8IbNGhAoUKFqF27trI84Mmp7m3btmXmzJlGGechtW74wIEDlYzzH330ET///DPBwcFKMrhKlSpRuXJlTE1NWbFiBT///DPffvutUjGgZs2aXLx4UTnXlStXuHv3LuPHj6dDhw7KuerUqYOtrS2tWrXi66+/VtrT/m9qakrz5s3x9/fHy8uLqVOnAihrqjOqGACP8xIMHz6cffv2sWvXLurUqcPQoUMJCAgAHlcMqFmzJr/++iuurq706tWLgQMHsn37dvbs2cORI0cIDAykUaNGDBw4MF1W+GrVqinl/UJDQ/Hx8UmXUT0gIEBZ9jBr1iw++OADo367urrSqlUrAgMD+fPPP5VAw7///guk3oT26dMHQAm0xMTEMGjQINasWWP0+bVq1YpWrVoZfX5p52ratKlyzkqVKlGvXj2OHDlC/fr1qV+/Ph4eHkbb/PPPP9StW5e+ffsqbYGBgcr4Xr16lRs3bijJ9GrWrMmgQYPSZdkH+PTTTxk0aBA6nU5J1hcaGqrcfFarVk1ZCpCWuC4tX8SzKgZMnDgxXQJCgDFjxhjlQvD29jZaIlCoUCGl3F/a9Punr6O06yxN2rWSVjHg9u3brFmzBoDevXtTrlw5Ro8ejbe3t7I8Yfny5UrVAEhNVBgaGsrGjRuNkvalTeF/UnJyMpGRkQQEBCh/p0aOHEmvXr24dOkSf/zxB3q9XllykZKSgkqlwtTUVLmWvL29mTFjBidPnuTOnTu0bt2aqlWrMmTIEKNzrVy5kkmTJuHj48P48ePT9eVpefPm5dy5c+na27Vrp/xdEEKI94EEBIQQIhPMzMxITNKh1WuUp/fZRZusY+m2c/x18YHS5mCrZkA7D0oUyji5V257VjAgMxUDWrduTb9+/dDr9dw+t5vb53anO76pqTmxick42acmS7SysjKaGv8mVgx4smpAmtddMSDtxjRtm+bNm7+TFQOelJmKARqNRrl5hqyNb0aerhjQpk2bdDffzZs3z3TFgIw8PV4v+juQVU9WDLCwsODIkSNGOSrSPF0xwNnZOcOb/4zY2NigUqlo2LChUXv9+vU5dOgQDx8+xN/fP8PjlyhRAk9PT0JCQvjoo48ICQmhePHiRuUpIXUpxvTp01m6dCm+vr5MmzYtXcBLCCHEs0lAQAghXsBggKRkiI7XYGGZvdn9Y+K1zN0QxrU7jxN0Oee3xa+9B3nsX38lAQC1mQmOdpaYmxlPwc1KxYCCBQuiUqlo0boDd1JKkKTVYWNphpmpCSk6PUkpJlhbmFHRtSDHn58UP0Ovo2KAl5dXuv1fd8WAjLyLFQOe3vdFFQPq1KljlFDxVcd37ty5RgGrjMrRVahQIdMVA16HJysGWFtbU7RoUWUpyZPSKgakyUqCvRIlSmAwGEhOTsbCwkJpT5ttYGlp+dzjt2rViu+//57Y2Fh27NhBp06djI6v1WoZPnw4v//+O59//jmjRo2SYIAQQmSRBASEEOI5UnR6ouI0xMQnkTebZwbceRhH0LowHkU/TqxWsVQe+nzshpXFm/Hr+VnBAL3ewPq9l0lMSiGvgwUqlYr/rpzk3B+rqdf+G+KSLfn7PwsmjBtPSEgIjx49omLFisRG3mXMyEHKrAKtJonwY0spVaEaA9q14MqZZ2cuf9LT64O9vLwIDw/H1dVVuYE0GAyMGDGCEiVKpJtK/SxpT5H37t1L586dlfanKwbcunVLqXoAqTMDRo0aRceOHSlevHi67ObP6ndGrl69SlRUFN27d89URvvnGTp0KMnJyUrZtI8++ghzc3P8/PyUigFPq1GjBvv27VMqEUBqxYaQkBDc3NxyJOP6y35+t2/fpnPnznz55Zf4+PhQunRpSpcuzenTp7lxI3X5zcGDB2nfvr2yT3aM75NLVp6nefPmbNiwgYSEBJo2bWp0U/y6PVkxwNLS8pnJDtMqBryMhg0bsnjxYkJCQozW+u/duxcXFxdsbW2xtbV95vGbN2/OlClTmDNnDo8ePaJVq1ZGr3/55Zfs2rWLL7/80qjChxBCiMx7M75xCiHEG0ijTSE6TkuiJgV9NpcS+Od6BPM3nSFRk6K01a/iTMdm5THNxE1jblCbmeBkb4mZafr+XL0dze37cdjZmCtP5JwKlcVg0HNq5zyKuzXl3AM1w0b8SmxsLM2aNcPb25u+fftiZzcVX9+W3HHS8MualcQ9CGfkzHF4lMvPlTOZ65u9vT2nTp3i+PHjVK9enQEDBtCxY0f69etHp06dsLCwYO3atezevZsff/wx0++5QoUK+Pj4MGPGDJKSknB1deXgwYPs27ePwMBApWLAuHHjMDU1pXHjxsTExBAcHMy9e/eUhIfP6/f58+cJDQ3F3d09w21KlSqFra0t8+bNw8zMDDMzM3bu3KnMOng6o/3z1KpVi/HjxzNt2jQaNGhATEwMgYGBlCxZkgoVKmS4z8CBAzl48CDdu3enb9++mJubs3LlSm7evMmiRYsyfe6seNnPz9nZmUKFCjFp0iTi4uIoXrw4Z8+e5cCBA/Tr14+IiAjOnj2r5FuA7B3fF2nevDlBQUH8+uuvRmUQc0paxYCn1a9fnzJlymTYv379+mFiYpKu9F5mbdiwAQcHB6M2ExMTunfvTs2aNWncuDHff/89iYmJlCtXjs2bN3Pq1KlMjYejoyMNGzZk9erVeHp6KgkXITVHyLZt2/D29qZKlSpK1YQ0FStWlHKBQgiRCRIQEEKIDMQnaomJT872fAEAf/59h5U7/lGCDCqgbeOyNPEq/sZMd31eMABSlzqk6PSYm5orbZY2DtRoMYRLob9y4dAqUlK0JJYqQ0BAALVq1QJg8eLFBAYGMnToEMzNzalUqRLLli2lalXPLPWvf//+BAcH06dPH7Zv306FChVYtWoVs2fPZtSoURgMBsqXL09QUFC69fwvMmPGDAIDA1m+fDmRkZHPrBiwaNEi1q5di7W1NVWrVsXf31+pUvAsn3/+OVOmTKFXr14sXbo0w23s7OwIDg5m+vTpmcpo/zwdO3YkOTmZNWvWsHr1aiwtLalduzYjR45UptA/rVy5cqxevZpZs2bx5ZdfolKpcHd3Z8WKFVSvXj1T582qV/n8AgMDmTVrFnPmzCEyMpLChQszcOBA+vbty7Zt2yhXrpzRTIjsHN8XKVu2LOXLl+fBgwdG1RlyytMVA9I4OTllGBBIqxhQuHDhDF/PjIxu7J+sGDBnzhwCAwNZunQpERERlC1bVkkWmBmtW7dm9+7dtGzZ0qj9999/B1JnG2SU02DPnj0vPbNBCCHeJypDdmaoeY+dOZP6WOtZ00TfZQkJCVy4cAFXV9dnJpgS2UvGPOcYDAZi4rXEJyWT9tsxMTGRK1euUKZMmecmRnsRvcHA1j+u8tuf15U2czMTPm9ZCU+XAq/Y8+yjNjMhj70lps8IBgCE34xiyrJQLC1MsTBPX4JQo9WRpNXxVU8vyhZzzHIf5BrPXTLeuU/GPPfJmOeu93283+d7A/F2kRkCQgjxP7r/5QvQaHVkd6Q0OUXH8pALnLhwT2mzt1Hzf5+4U6qIw3P2zF0W5qY42Vk8NxgAUNrZAecCtlz/Lwa1vYnRzAaDwUBsYjIlC9tT2vnNeW9CCCGEEMLYm7FQVQghXjNtso6ImCSSciAYEJeg5Yc1fxkFA4rks2F09+pvZTAAwMRERTvvclhZmPEoJjWIotcb0Gh1PIrRYG1hRjvvcpiYvBlLIIQQQgghRHoyQ0AI8d5L1CQTHadFl82JAwHuRSQQ+MtpHkQ9TlRWoWQe+n3shpXlm/Mr2NLcFEd7S0yzcAPvUS4/fu08lIoBcYnJmJmaULKwPe28y+FRLn8O9lgIIYQQQryqN+fbqBBCvAax8VriErXkQCyAy/9GMnfj3yQkPa4kUNe9CJ0/dMnUU/jc8jLBgDQe5fLjViYfV29HExOvxd5GTWlnB5kZIIQQQgjxFpCAgBDivaTXG4iO15CoSSEnUqseO/sfK7ZfMJp10KZRGZrVLPHGVBIAsFSb4mRn+Uo38CYmqpdKHCiEEEIIIV4vCQgIId47ySl6ouM0aJJ12X5sg8FAyOFrbDt0TWkzMzXhs5YVqVah4HP2zH2WajOc7Czkab4QQgghxHtKAgJCiPeKRptCVJyGFF32TwtITtGz8rcLHDt3V2mzszbn/z7xeOOy7UswQAghhBBCSEBACPHeiE/UEhOfjD4H1gjEJyYzb+PfXL4ZpbQVymuNX/sq5He0yvbzvSwVYGmRGgx4k5YuCCGEEEKI3CcBASHEO89gMBATryU+KTlH8gXcj0wgaF0Y9yISlDaX4k70beuGjaV59p/wJUkwQAghhBBCPEkCAkKId5pObyAqVoNGm0IOxAIIvxXF3A1/E5+YrLTVditMF58KmL1BlQRUgJWlGY62EgwQQgghhBCp3oiAgF6vJzAwkHXr1hEbG0uNGjUYN24cxYoVy3D7LVu2MHLkyHTte/bsoWjRori4uDzzXPv27aNIkSKcPHmSzp07p3t9xYoV1KxZ8+XfjBDijZGcoiMqVoM2RZ8jxz9+/i7LQy6Qont8/Fb1S/NRnZJv1E23BAOEEEIIIURG3oiAQHBwMKtXr2bq1KkUKlSIGTNm0Lt3b7Zu3YparU63/cWLF/Hy8mLWrFlG7Xny5AHg0KFDRu3R0dF07dqVhg0bUqRIEeUYxYsXZ/Xq1UbbOji8WYm/hBAvJ1GTTHSc1qjsX3YxGAz8duQ6Ww5eVdrMTFV0b14Rr0qFsv18r0KlAmtLcxxs1BIMEEIIIYQQRl57QECr1bJkyRJGjBhBo0aNAJg9ezb169fn999/x9fXN90+ly5dwsXFhfz582d4zKfbv/vuO5ycnPjuu++MjlG2bNlnHkMI8faKS9ASm5AzyQNTdHpW7fiHI2f+U9psrMzp39aNcsWcsv18r0KlAhtLcxxsLV53V4QQQgghxBvotQcE/vnnH+Lj46ldu7bSZm9vT8WKFTl+/HiGAYGLFy/i7e2dqeMfOnSI33//nZUrVxrNNrh48SLVqlV79TfwBIPBQEJCwos3fMckJiYa/V/kPBnzZ4tNSCY+MXuDAZokDQBR0fGs3n2Oy7eildfyO1rSp5Ur+Z0s36jPQ6VSYWtljrmJyVv5e0mu8dwl4537ZMxzn4x57nrfx9tgMMjMPPFWeO0Bgbt3U+t1Fy5c2Ki9QIECymtPio6O5t69e5w4cYLVq1cTGRmJu7s7I0eOpFSpUum2nzVrFh988AHVq1c3ar98+TJOTk60bduWe/fuUb58eYYNG4a7u/tLv5fk5GQuXLjw0vu/7a5fv/66u/DekTFPpVKpMDE1I0GjJz5Ri16f/TkDYhJ0rFp/lsg4ndJWOI85zavbERNxh5iIbD/lSzMzNcXO2oI4cwO3UlJed3deiVzjuUvGO/fJmOc+GfPc9T6Pd0ZLn4V407z2gEBa1PDpvzAWFhZER0en2/7y5ctAatTt+++/Jykpiblz59K5c2e2bt1Kvnz5lG2PHz/OuXPnjJYKAPz333/ExsaSkJDA2LFjMTU1ZeXKlXTt2pWNGzdStmzZl3ov5ubmL73v2ywxMZHr169TsmRJrKzenHrr7zIZc2MpOj3RcVpsk3Uv3vglXL7xiHWHLpGofTzroJpLPjo2KYuZ2ZtTSQDARKXC1lqNrdVr//X+SuQaz10y3rlPxjz3yZjnrvd9vMPDw193F4TIlNf+jdHS0hJIzSWQ9jOARqPJ8JdH9erVOXLkCE5OTso0nMDAQBo1asTGjRvp27evsu2mTZtwd3enUqVKRscoXLgwx48fx8rKCnPz1Brhbm5unD9/np9++okJEya81HtRqVRYW1u/1L7vAisrq/f6/b8OMuapyQMTErWYmKnJiXvgk//cY9nWyyTrHgcDWtQthW+9Um/cVMB3MWeAXOO5S8Y798mY5z4Z89z1vo73m/YdQYhnee2PttKWCty/f9+o/f79+xQsWDDDffLkyWP0l8zKyoqiRYty7949pU2v17N3715atmyZ4THs7e2VYACAiYkJZcqUMTqGEOLNFpegJSo25yoJ7Dx6g4Wbz5L8v7KCpiYqeraoSMv6pd+4f+jfxWCAEEIIIYTIWa89IFChQgVsbW05duyY0hYTE8P58+epUaNGuu3Xrl1LzZo1jZJkxcXFcf36daPp+uHh4URGRlKnTp10xzh48CCenp7cvHlTaUtJSeGff/55L6f8C/G2MRgMRMVpiEnQ5kglAZ1Oz+qd/7Bp/+PpfhbmKvq3qUgtt8LP2fP1kGCAEEIIIYR4Ga89IKBWq+natSv+/v7s2bOHf/75h2HDhlGoUCGaNWuGTqfjwYMHJCUlAdCgQQP0ej2jRo3i8uXLnDlzhkGDBpEnTx7atm2rHPf8+fOYm5tTunTpdOesWrUqTk5OjB49mrNnz3Lx4kVGjx5NVFQUPXv2zK23LoR4CTqdnojoJBISk8mBWACJSSkErgvjj9N3lLa8Dha0q5uHskUdsv+Er0iCAUIIIYQQ4mW99oAAwODBg2nXrh1jx46lU6dOmJqasnjxYszNzfnvv/+oV68e27dvB1KXGCxbtoyEhAQ6depEz549sbOzY8WKFVhYPP5C/ODBAxwcHDAxSf8WbW1tWbZsGfny5aNXr158+umnREVFsXLlSqOkhEKIN4smWcej6CSSknXkQCyAiOgkZqw6wYXrj0sGlHZ2YGgHd5xsX3vKlXQkGCCEEEIIIV7FG/EN19TUlJEjRzJy5Mh0rxUtWpSLFy8atVWqVIklS5Y895h9+vShT58+z3y9ePHi/Pjjjy/XYSFErktISiYmPmfyBQDc+C+GoPVhxMRrlbZqFQrQo0VFdCla3rTsIhIMEEIIIYQQr+qNCAgIIcSzGAwGYuOTiU/SkkOxAE5fesDiLWdJTtErbT61S9CqQRlMVCoSU3LmvC9LpQJrCQYIIYQQQohXJAEBIcQbS6c3EB2nIUmbkiP5AgwGA3uO32TD3svKEgQTExVdPqxAXY8i2X/CbCAzA4QQQgghRHaRgIAQ4o2UnKIjKlaD9omn9tlJp9fzy+7LHDh1S2mztDClXxt3XEvmyZFzviqVCmwt1djbql93V4QQQgghxDtAAgJCiDdOkjaFqDgNOl3OrBFI0qSwaMtZzl55pLTlsbdkYHsPiuS3zZFzviqVCuys1NjZSDBACCGEEEJkDwkICCHeKHEJWmITktHnxBoBIDImiaD1Ydy6H6e0lSxsz4B27tjbvJnT8E1UYGetxtZaggFCCCGEECL7SEBACPFG0OsNxCRoSUhKzpF8AQD/3o0laH0Y0XEapc2zfH4+a1kJtblpzpz0FZmowN5GjY2VBAOEEEIIIUT2koCAEOK1S9HpiY7VkJSsy7FznAl/yKJfz6J54hxNvYrTpnFZTFSqHDvvqzBRqbC3VWNjaf66uyKEEEIIId5BEhAQQrxWmmQd0bEaknU5kzwQYN/Jm/yy+5Iy88BEpaJjs/I08CyaY+d8VSYqFY52aqwsJBgghBBCCCFyhgQEhBCvTUJSMjHxWnT6nFkjoNcbWL/3MntP3FTaLNWm9PnYjUql8+bIObODiQrsbSUYIIQQQgghcpYEBIQQuc5gMBATn5ovIIdiAWi0OhZvOcvf4Q+VNic7C/zae1C0gF3OnDQbqFRgZyPLBIQQQgghRM6TgIAQIlfp9AaiYjVotCnkUCyAqFgNwevD+PderNJWvKAdA9p54Gj3ZlYSgMelBW0lgaAQQgghhMgFEhAQQuSa5BQdUbEatCk5ly/g1v3USgKRMY8rCbiXzUevVpWxUL+ZlQQgNRhgY2mOnY0EA4QQQgghRO6QgIAQIlckapKJjsu5fAEA564+YuHmMyRpH1cS8K5ejHbe5TAxeTMrCQCoAGtLcxxs39zZC0IIIYQQ4t0jAQEhRI6LjdcSl6jNsXwBAAf/usWa3y+h/18pAZUKOjQpT+NqxXLupNnEytIMB5kZIIQQQgghcpkEBIQQOUavNxAdryFRk6KU/Mv2cxgMbNwXzu7Qf5U2C3NTereujFvZfDlz0mxkqTbD0dYClerNncEghBBCCCHeTRIQEELkiBSdPjV5YLLuxRu/JG2yjiVbz3H60gOlzcHWgoHtPShW8M2tJJDG0twUJzsJBgghhBBCiNdDAgJCiGyn0aYQFaclRZdzyQOj4zQEb/ibG//FKG1FC9ji184DJ3vLHDtvdrEwN8XR3vKNzm0ghBBCCCHebRIQEEJkq/hELTEJyehzMGHAnQdxBK4LIyImSWmrXCYvvVtVxtLizf+1pjYzwcnOAlMJBgghhBBCiNfozf/mLIR4KxgMBmLitcQnJedYvgCAC9cjmL/pb5I0j5ciNKzqTIcm5TE1Mcm5E2cTtZkJTvaWmJq++X0VQgghhBDvNgkICCFemU6nJypOg0arIwdjARwOu8Oqnf8osw9UwCfe5figRrG3Yh2+2swERztLzCQYIIQQQggh3gASEBBCvBJtso7oOA3alJzLF6A3GNhy8Ao7jtxQ2szNTOjVqjJVyufPsfNmJ3NTExztLDA3k2CAEEIIIYR4M0hAQAjx0hI1yUTHadHlYL4AbbKO5SHnOfnPfaXN3kaNXzsPShS2z7HzZiczU9X/ggGmr7srQgghhBBCKCQgIIR4KbHxWuISteRgLIDYBC1zN/zN1dvRSluRfDb4tfcgr4NVzp04G5maqnC0s0RtLsEAIYQQQgjxZpGAgBAiS/R6A9HxGhI1KTmaPPDuo3gC14XxMCpRaXMtmYe+H7thZfl2/OoyNVXhZGeJhQQDhBBCCCHEG+jt+FYthHgjJKfoiY7ToEnWvXjjV3DxRiTzN/1NQlKK0lbPowidmrm8Ndn5JRgghBBCCCHedBIQEEJkikabQlSchhRdTtYRgKNn/uOn3y4Y5SVo06gszWoWfysqCYAEA4QQQgghxNtBAgJCiBeKT9QSE5+MPgfXCBgMBrYdukbI4WtKm7mZCT19K1KtQsEcO292k2CAEEIIIYR4W0hAQAjxTAaDgZh4LfFJyTmaLyA5Rc+K7ec5fv6e0mZnbc7/feJBaWeHnDtxNpNggBBCCCGEeJtIQEAIkSGd3kBUbBIarY6cXCQQl5jMvA1hhN96XEmgUF5rBravQj7Ht6OSAEgwQAghhBBCvH0kICCESCc5RUdUrAZtij5Hz3MvIoGgdae5H/m4koBLCSf6tnHDxtI8R8+dnSQYIIQQQggh3kYSEBBCGEnUJBMdpzVK6pcTwm9GMXdDGPFPVBKo41aYzj4VMHtLKgmABAOEEEIIIcTbSwICQghFXIKW2IScTR4IEHruLiu2nzeqWNC6QRl8apd4ayoJgAQDhBBCCCHE200CAkIIDAYD0fFaEnI4eaDBYGD7n9fZ+sdVpc3M1IQeLVypUbFQzp04B0gwQAghhBBCvO0kICDEe06n0xMVp8nx5IEpOj0rf7vA0bN3lTYbK3P+7xN3yhZ1zMEzZz8JBgghhBBCiHeBBASEeI/lVvLA+KRk5m/8m0v/RiltBfNY49fegwJO1jl67uxmaiLBACGEEEII8W6QgIAQ76kkbQpRcRp0upzNF/AgKpHAX05zLyJBaStXzJH+bd2xsXp7KgkAmJmZ4GQvwQAhhBBCCPFukICAEO+huEQtsfE5nzzwyq0o5m74m7jEZKWtZqVCdP3IFXOzt6eSAIClhZo8dhYSDBBCCCGEEO8MCQgI8R4xGAzExGuJz+HkgQAnLtxj2bbzpOgeL0doWa8UzeuWeqsqCUDqzAB7a7O3qhyiEEIIIYQQLyIBASHeEzq9gahYDRptSo4mDzQYDOw8eoPNB64obWamKrp95ErNyoVz8Mw5w9REha2tBRF3U153V4QQQgghhMhWEhAQ4j2QW8kDdTo9q3de5PDfd5Q2G0sz+rd1p1xxpxw9d05ITSBogS5FiyGnp1QIIYQQQgiRyyQgIMQ7LlGTTHScFp0+Z29oE5KSWbDpDP/ciFTa8jtZMbB9FQrmebsqCcDjYICF2oyEFO3r7o4QQgghhBDZTgICQrzD4hK0xCbkfPLAh1GJBK0P47+H8UpbmaIO/F9bd2yt1Tl67pzwZDBACCGEEEKId5V82xXiHaTXG4iO15CoScnx5IHX7kQTvD6M2ITHlQSquxakRwtXzM3evoz8EgwQQgghhBDvC/nGK8Q7JkWnJzpWQ1KyLsfP9dfF+yzZeo7kJ3ITNK9TEt/6pTF5yyoJgAQDhBBCCCHE+0W+9QrxDtEk64iO1ZCsy9nkgQaDgV2h/7JpX7hSscDEREVXnwrUcS+So+fOKaamKpxsJRgghBBCCCHeH/LNV4h3REJSMjHxOZ88UKfXs+b3S/xx+rbSZmVhRr82blQomSdHz51TTE1VONlZYmH+9i1xEEIIIYQQ4mVJQECIt5zBYCA2Ppn4JC05HAsgUZPCol/Pcu7qI6Utr4MlA9tXoXA+m5w9eQ6RYIAQQgghhHhfSUBAiLeYwQCRsRqStDmfPDAiJomgdWHcfhCntJUqYs//feKBvc3bV0kAUoMBeewsUUswQAghhBBCvIckICDEW8rUzJzIWA0qU/McP9eNuzEErw8jOk6rtFV1KUBP34pv7c202f9mBryt/RdCCCGEEOJVSUBAiLeQNkVPdIIOW20KVlY5GxD4+/IDFm05izb5caLCD2uVoHXDMm9lJQGQYIAQQgghhBAgAQEh3jpxiVoiY5JIStLk+Ln2nrjJut2XHlcSUKno9KEL9as45/i5c4qZqQlOdhYSDBBCCCGEEO89CQgI8ZYwGAzExGuJT0rOlUoC63ZfZv+pW0qbpdqUvm3c+P/27jw8prN/A/g9M5klqwQhiF0SEllEFrFvRa2R0qJqK9ra+iuii26qfasLStJQe7WU2lqKlxctXSxBaykiQZCSxJI9mZnMzPn9keYwEktIziQz9+e6XG2esz3nazLt3POc5/FtXKNCr12RlAo53FzUUNoxDCAiIiIiYiBAVAUYjSZk5uqg0xtRwXMHQqs3YPmPp3Hqwp2VBKq7aDBxcCDquTtV8NUrDsMAIiIiIiJzDASIKjl9oRFZuTroDaaH7/yEMnK0iNtwAlfT76wk0NDDGRMGBaKak7rCr19RVHZyuDproLSTW7orRERERESVBgMBokqsQFeIrFx9hT8iAABX03Lw5cYTyMy5MzdBoFdNjOnXEmpV1f1WXWUnh5uLBnYKhgFERERERHdjIEBUSeXk6ZFboIcEWQBOXbiJZT+ehk5vFNu6hzZAVJdmkMur5koCAMMAIiIiIqIHYSBAVMmYTAKy8nQo0BkgSBAG/HI8Bev/lyBeSyYDnnvKB52DPSv+4hVIrVTAzVkNBcMAIiIiIqJSMRAgqkQMRhMyc3TQFRofvvMTMpkEbPo5EXvjr4ptapUCYwe0hH/TmhV+/YrEMICIiIiI6OEYCBBVEjq9AVm5ehQaK37yQJ3eiBXbTuNE4k2xzdVZjUmDAuFZ27nCr1+R1EoF3Fw0UFThRx2IiIiIiKTAQICoEsjTFiInT5rJA7NydYjbeAKXU3PEtvq1nTFhUADcnDUVfv2KJI4MYBhARERERPRQDASILEgQBOTkFSJPK83kgf/cyMWXG07gdrZWbPNvWhMvDvCDRlW13w74mAARERERUdlU7U8ARFWY0SQgK1cHrc4ACbIAnLl0C0t+OAWt7s78BF1ae2JwN+8qvZIA8O9qAgwDiIiIiIjKhIEAkQUUGozIzNFBb6j4+QIA4Ne//sF3uxJg+ncpAZkMGNzNG11D6kty/YqkspOjuouGYQARERERURkxECCSmFZvQGauDkZjxY8LMAkCftx/AbsOXRbbVEo5xvZviQAv9wq/fkVjGEBERERE9PgYCBBJKLdAj5y8QvGb+oqkLzRi1U9ncDwhXWyr5qTCxEGBaODhUuHXr2gqO3nRagIMA4iIiIiIHgsDASIJCIKArDw98rWFkCALQHaeHos2ncCla9liWz13J0wcHIjqLlV7JQHgThhgxzCAiIiIiOixMRAgqmBGowmZuTro9EZJJg+8drNoJYFbWXdWEvBrUgNjB7SEvbrq/8rzMQEiIiIiovJR9T8dEFVi+kIjsnKlmzzwXPJtfLXlFAp0BrGtY6t6eO4pbyjkVf8DNJcWJCIiIiIqPwwEiCpIga4QWbl6GE1SjAsA/jh5Dd/+9xxM/15PBuCZrl7oFlofMlnVXlYQADRKBVxdNFBU8SUSiYiIiIgqCwYCRBUgJ0+P3AI9pMgCTIKAbb9exM4/ksU2pZ0cY/r5oZVPrYrvgAQ0KgVcnRkGEBERERGVJwYCROXIZBKQladDgc4gyeSBhQYT1mz9G0fPpoltLo4qTBgUiEZ1qv5KAgCgUdnBzVkNOcMAIiIiIqJyxUCAqJwYjCZk5uigKzRKcr0CvQlxm/9G8vUcsa1OTUdMHBSImq72kvShIskAaNR2cHViGEBEREREVBEYCBCVA53egMxcPQxGaSYPTM8owMbfbiMr/0740LxRdYyPbAkHjVKSPlQkGQB7TVEYYA3zHxARERERVUYMBIieUJ62EDl50k0emHglA4s2nUK+7k4Y0C6gLob19LGK2fcZBhARERERSYOBANFjEgQB2Xl65GkLJZkvAAAOn76O1TvOmoUPAzs3RY/whlbx4VkmAxw0SlRzVFnF/RARERERVWYMBIgeg9EkFM0XoDdAiixAEAT89NslbP/9ktimkAPP9/RG28D6EvSg4slkgKNGiWpOakt3hYiIiIjIJjAQICqjQoMRmTk66A3SzBdQaDDh251ncfjvVLHN0d4OvYKd0cq7piR9qGgyGeCkUcHFSWXprhARERER2QwGAkRloNUbkJmrg9EozTMCeQWFWLz5JBKvZoptHjUc8GK/5si6+Y8kfahoMhngbK+CsyPDACIiIiIiKTEQIHpEuQV65OQVwiTRhAHpGfn4csMJpN3OF9t8GrhhfJQ/5IIBWTcl6UaFkssAJ4YBREREREQWUSmmJDeZTFi4cCE6dOiAoKAgjBs3DlevXr3v/lu3boWPj0+JPykpKeI+PXr0KLH9jTfeELdnZGRg2rRpCA0NRVhYGGbNmoWCgoIKvU+qmgRBQGauDtl5esnCgKSUTHyy+qhZGBDhXweTnwuCoxUsKwgUhQHOjgwDiIiIiIgspVKMEIiLi8PatWsxZ84ceHh44LPPPsPYsWOxbds2qFQlPywkJCQgLCwM8+bNM2uvXr06ACA/Px9Xr17FV199BT8/P3G7RqMR/33KlCkoKCjAqlWrkJ2djZkzZyI/Px+ffPJJBd0lVUVGo6lo8sBCoySTBwJA/JlUfL39LAzGO3MU9O/YBE9HNLKamfflMhlcHJVwtGcYQERERERkKRYPBPR6PVasWIHp06ejc+fOAID58+ejQ4cO2L17N/r27VvimPPnz8PHxwfu7u6lnjMpKQkmkwmtWrVCtWrVSmz/888/ceTIEezYsQNNmzYFAHzwwQcYO3Yspk6ditq1a5ffDVKVpSs0IitHh0KjNJMHCoKAnQeTsfXARbHNTiHDiN6+CPPzkKQPUpDLZKjmpIKDlYx0ICIiIiKqqiz+yMC5c+eQl5eHiIgIsc3FxQW+vr6Ij48v9ZiEhATxg/z9ttesWbPUMAAAjh49Cnd3d7NzhIWFQSaT4dixY495J2RNCnSFyMjWShYGGIwmrN5x1iwMcLRX4v+GBFtdGODqzDCAiIiIiKgysPgIgdTUoqXU6tSpY9Zeq1YtcdvdsrKykJaWhqNHj2Lt2rXIyMhAQEAAoqOj0bhxYwBFgYCDgwOmTJmC48ePw83NDc888wxGjBgBuVyOtLS0EtdTqVRwdXXF9evXH/teBEFAfn7+w3e0MsVzL1jLHAx5WgNy8vUwmaR5SKBAZ8DKnxKQmJIltrm7ajCufwu4u6lLratOqzP7Z1Vgp5CjmpMagrEQ+fmFlu5OmVjba7wqYM2lxXpLjzWXHmsuLVuvtyAIVvOoJ1k3iwcCxW8S984VoFarkZWVVWL/xMREAEW/ZB9//DG0Wi0WLVqEYcOGYdu2bahZsyYSExORnZ2Nnj17YuLEiTh27Bg+++wzZGVl4dVXX0VBQUGpcxOo1WrodI//AauwsBBnz5597OOruuTkZEt34YnYKZUo0AnILdDBKNHIgOx8I7YdyUBGrlFsq1tdiadDnJF9+xqybz/4+JR/Uh68QyWhUSnh7KhERroRJpM0ta0IVf01XhWx5tJivaXHmkuPNZeWLde7tM8bRJWNxQOB4on+9Hq92aR/Op0O9vb2JfYPCQnBwYMH4ebmJqZusbGx6Ny5MzZv3ozx48dj6dKl0Ol0cHZ2BgD4+PggNzcXixYtwuTJk6HRaKDX60ucW6fTwcHB4bHvRalUolmzZo99fFVVUFCA5ORkNGrUqNS/s6rAaBSQlaeHTm9ATYmueTk1B5v3nkVuwZ0woLVPTQzp3gx2dg9+mken1SHlnxR41vOEWqOu6K4+ETs7OVyd1FA95J4qM2t4jVc1rLm0WG/psebSY82lZev1TkpKsnQXiB6JxQOB4qH76enpaNCggdienp4OHx+fUo8pXk2gmL29PTw9PZGWlgagKI27N5Hz9vZGfn4+srKy4OHhgT179pht1+v1yMzMRK1atR77XmQy2RMFClWdvb19lbx/nd6AHK0OMoUSGntpnm0/fi4dK3/6G4WGO9+W92nXGH3bNy7T8DK1Rl2p/yOrUMjg5qSGWmXxt5pyUVVf41UZay4t1lt6rLn0WHNp2Wq9+bgAVRUW/8quefPmcHJywuHDh8W27OxsnDlzBqGhoSX2X79+PcLDw82e1c/NzUVycjKaNWsGQRDQvXt3xMbGmh136tQpuLu7w83NDaGhoUhNTcXly5fF7UeOHAEAtG7durxvkSqxvAI9bmfrYDBKM1+AIAjYffgylvxwSgwDFHIZRvXxRb8OTazqPx52ChncnDVWEwYQEREREVkbiwcCKpUKw4cPx+eff469e/fi3LlzeO211+Dh4YEePXrAaDTixo0b0Gq1AICOHTvCZDJhxowZSExMxKlTpzB58mRUr14dUVFRkMlkeOqpp7B8+XLs2LEDV65cwfr167Fs2TJMmTIFABAYGIjg4GC89tprOHnyJA4dOoR3330XkZGRXHLQRgiCgKxcHbLy9DAJ0oQBRqMJa3edw+af7wwhc1DbYcpzrdDGv84Djqx6lAp5URigVFi6K0REREREdB+V4qu7KVOmwGAw4O2334ZWq0VoaCiWL18OpVKJlJQUdOvWDR9//DGioqJQp04drFq1CnPnzsXQoUMhCALatWuH1atXQ60uepZ62rRpcHJywrx585CamgpPT0/MnDkTzz77LICiITyxsbGYNWsWRo4cCbVajV69euHNN9+0ZBlIIkaTgMwcLXR6I6SJAoACrQFLfjiFs8l3Zgms6WqPSYMD4VHDUaJeSEOpkMPNRQ2lHcMAIiIiIqLKrFIEAgqFAtHR0YiOji6xzdPTEwkJCWZtfn5+WLFixX3PZ2dnh4kTJ2LixIn33adGjRpYuHDh43eaqqRCgxGZOTroDdLNdH8rqwBfbjiBazfzxLYm9arhlWcC4OxgXbPPquzkcHXWQFmFJxAkIiIiIrIVlSIQIJJCga4QWbl6GE1SjQsALl/PxpcbTyA7786qFq2b18LIPr5QWdlwepWdHG4uGtgpGAYQEREREVUFDATIJuTm65GTXyjZfAEA8Nf5G1i+9bTZSgK9Ihqif8emkFvR5IFAURhQ3UUDBcMAIiIiIqIqg4EAWTWTSUBWng4FOgOkygIEQcDe+KvYtC9RnKNALpfh+Z7N0S6wrjSdkJBaqYCbs5phABERERFRFcNAgKyWwWhCVo4O2kKjZNc0mkz4fs957D/+j9imUSvw0sAAtGhUXbJ+SEWjVMDVRQOF3LpGPBARERER2QIGAmSVdIVGZOXoUGiUbvJArc6ApT+ext8Xb4lt1V00mDQ4EHXdnSTrh1Q0KgVcnRkGEBERERFVVQwEyOrkawuRnSft5IEZ2Vp8ufEEUtJzxbZGdVwwYVAAXBzVkvVDKhqVHdyc1ZAzDCAiIiIiqrIYCJDVEAQBOXmFyNPqIWEWgCupOfhy4wlk5erEtlbe7hjdz8/qVhKQAdCo7eDqxDCAiIiIiKiqYyBAVsFoEpCVq4NWL93kgQBwMukmlv94Grq75il4KrwBBnZuZnUrCcgA2GuKwgCZld0bEREREZEtYiBAVV6hwYTMHC30BunmCwCAn49exfd7z4sBhFwmw5Ae3ujYylPSfkiBYQARERERkfVhIEBVmk5vQEauDkajdMMCTCYBG/clYt/Rq2KbRqXAuEh/+DWpIVk/pCKTAQ4aJao5qhgGEBERERFZEQYCVGXlFuiRk1cIk4TPCOj0Rizfehonk26KbW7OakwcHAjPWs6S9UMqMhngqFGimpP1TYxIRERERGTrGAhQlSMIArLz9MjTFko6X0Bmjg5xG0/gSlqO2NagtjMmDAqEq7P1fWCWyQAnjQouTipLd4WIiIiIiCoAAwGqUoxGEzJzddDpjZAwC8A/6bmI3fgXMrLvrCQQ0KwmXuzfEmqVda0kABSFAc72Kjg7MgwgIiIiIrJWDASoytAXGpGVq5N88sC/L97C0h9OQau/s5JA15D6GNTVyyqX3pPLAGcHFZwcGAYQEREREVkzBgJUJRToCpGVq4fRJOW4AODAnylYt/u8OE+BTAY8280bXULqS9oPqchlgLOjCk72DAOIiIiIiKwdAwGq9HLy9Mgt0EPKLMAkCNjycxL+d+SK2KZWKvDigJYIaFZTuo5ISC4DXBxVcGQYQERERERkExgIUKVlMgnIytOhQGeQdPJAfaERK7f9jT/P3xDbqjmpMXFQIBp4WN9KAgDDACIiIiIiW8RAgColg9GEzBwddIXGh+9cjrJydYjbdBKXr2eLbZ61nDBxUCDcXDSS9kUqDAOIiIiIiGwTAwGqdHR6A7Jy9Sg0Sjt54LUbuYjdcAK3s7ViW8umNTC2f0to1Nb5q8IwgIiIiIjIdlnnpxyqsvK0hcjJk37ywLPJt/HVlpPQ6u6MSOgUXA/PdveGQi6XtC9SKZ5AkGEAEREREZFtYiBAlYIgCMjO0yNPWyjpfAEA8PuJa1iz6xxM/4YQMgCDunmha0h9yGTWt6wgwNUEiIiIiIiIgQBVAkaTUDRfgN4AKbMAkyDgx/0XsOvQZbFNpZRjTL+WCPJ2l7An0mIYQEREREREAAMBsrBCgxGZOTroDdLOF6AvNOLr7Wdw7Fy62ObiqMLEQYFoWMdF0r5ISSYDnB0YBhAREREREQMBsiCt3oDMXB2MRmmfEcjJ12PRppO4+E+W2FbX3RGTBgWhejXrXEkA+DcMsFfByYFhABERERERMRAgC8nN1yMnvxAmiScMSL2Vh9gNJ3Azs0Bs821cHeMG+MNeY72/DsVhgLMjwwAiIiIiIipivZ+AqFIymQRk5elQoDNIPnlgwuUMfLX5JPJ1BrGtQ1BdDHnKBwqFda4kADAMICIiIiKi0jEQIMkYjCZk5eigLTQ+fOdydujUdXyz86zZcoZRXZrhqbAGVruSAFAUBjhqlAwDiIiIiIioBAYCJAldoRFZOToUGqWdPFAQBPz02yVs//2S2Ka0k2NUX1+0bl5b0r5IrTgMqOaktnRXiIiIiIioEmIgQBUuX1uI7Dy92bfzUig0mLB6xxnEn0kT25wdlJgwKBCN61aTtC9SYxhAREREREQPw0CAKowgCMjJK0SeVg+JswDk5uuxePNJJKXcWUnAo4YDJg0OQk1Xe2k7IzEZAAeGAURERERE9BAMBKhCGE0CsnJ10OoMkDgLQNrtfMRu+As3Mu6sJODT0A3jB/rDUaOUuDfSkgGw19ihGucMICIiIiKih2AgQOWu0GBEZo4OeoO08wUAQOLVDCzedBJ52jsrCbT1r4NhvZrDzopXEgDuhAGuTmqrniiRiIiIiIjKBwMBKldavQGZuToYjVKPCwCO/J2K1TvOwHDXtQd0bIpeEQ2t/gMywwAiIiIiIiorBgJULuzs7JCvNaDQJMAkSBsGCIKAHX8kY9uvF+/0RyHDyD6+CPX1kLQvlsAwgIiIiIiIHgcDAXpiggAU6IHsPD3UGo2k1zYYTfh25zkcOn1dbHO0V+KVZwLQzNNV0r5YgkwGaNQMA4iIiIiIqOwYCNATMRhNyMzVISdfi5oSjwzI0xbiq80ncf5KpthWu7oDJg4ORC03B0n7YglyuRwOaiXcnBkGEBERERFR2TEQoMem0xuQlatHgc4Ak8TrCt7ILEDs938h7Xa+2OZV3xUvRwXA0d66VxIAikYGODmoUM1JxTCAiIiIiIgeCwMBeix52kLk5OlhlDgIAICL/2QhbuMJ5BYUim3hfh4Y/nQLKO2seyUBoGjOAAe1EvYqBgFERERERPT4GAhQmQiCgOw8PfK0hZD4CQEAwNGzaVj10xkYjHeWNOzbvjH6tGtsE9+UF08gqFbIcc1geOj+RERERERE98NAgB6Z0SQgM0cHnd4AqbMAQRCw69Bl/LD/gtimkMswoncLhLesI3FvLOPu1QQKCgos3R0iIiIiIqriGAjQI9EXGpGVq4PeYHr4zuXMaDRh7a4E/H7ymtjmoLHDy1EB8G7gJnl/LEEmAxw0SlRz5JwBRERERERUPhgI0EMV6AqRlWuZ+QLytYVYsuUUzl3OENvcXe0x6dkg1K5u/SsJAEVhgKNGiWpOakt3hYiIiIiIrAgDAbovQRCQk1eIPK0eFsgCcDOzAF9uPIHrN/PEtqae1fBKVACcHFTSd8gCGAYQEREREVFFYSBApSrrfAFCOc8weOla0UoCOfl3VhIIaVEbI/u0gNJOUW7XEQSh0g7BZxhAREREREQViYEAlVDW+QLWrVmF7OwcNGs2uVyu/2dCOlZs+xuFd13/6baN0K9DE8jL8cP79m1bcCX5El6ZPLXczvmkLl5IxJwP38Pl5Ivw9KyP//53p6W7VCHOnDmD559/HocPH8a7776LI0eOYN++fU90zpSUFHTr1s2szc7ODjVq1EDnzp3xf//3f6hevbrZ9n379uHrr7/G33//DZ1OBw8PD3Tu3Bkvv/wyatSoIe43duxYHDt27L7XDgwMxPfff/9E/SciIiIikhoDATLzOPMFfLNqGXr1iXziawuCgD1HrmDzz0niqAS5XIbhvZqjbUDdJz7/vb79ejmCWrUu9/M+idUrlyIt9To+m/sFPOvWtnR3Ksz+/fvRpk0bqFTl/+jHK6+8gs6dOwMAdDodLl26hJiYGCQlJWHt2rXiflu2bMGbb76JIUOGYNSoUbC3t0dSUhKWLFmCn3/+GZs2bUK1atXE/Rs1aoRZs2ZBo9GUuKajo2O53wcRERERUUVjIECinDw9cgssM1+A0WTC+v+dx4E//xHb7NV2eGmgP5o3qv6AI61LdlYmvLy88HTP7pbuSoU6cOAABgwYUCHnbtCgAYKCgsSfw8PDoVQq8dZbbyExMRFeXl4AgC+//BJ9+vTB+++/L+7bpk0bhISEYMCAAdiwYQPGjh0rbrO3t0dAQAAcHGxjMksiIiIisn5yS3eAKs7p06cxcuRItG7dGq1atcKoUaPw119/iduPHj2K4cOHIzAwEKGhoXjrrTdw+/ad2fz/u2MruncKw9m/T2HSS6PQs2sEhjzTB+vXrhb36dq+6Bv2/27/AX2eai+2X7qYhLdmvIq+PTqib4+OeOfNabj2T4q4/a/jR9G1fWscP3oE0159Bb26tsPKT1/CjbM7IAgm1KimwYwXQtC0njNWLI3D84P7o1fXthjzwrPYtXOb2X3+/usvePnF4ejZNQLP9O+B2C8+Q0FBwQNrM3RQX6SlXseunT+ha/vWSL1+Tbzf7du24Jn+PTDg6S5IvnQRRqMR3327CmNeeBa9urZF7+7tMOnl0fjzeLx4vlXLv8Lw5wbg0B+/YuzI59CzSxuMGDIQu/+73ey6m75fi5HDotCzawQGR/bCF59/jLy8XLGWf/15DMePH4OPjw82b94MAEhOTsaUKVPQrl07BAUF4YUXXjD7e0xJSYGPjw9WrlyJXr16ITAwEJs2bUJMTAx69eqF//3vf+jbty/8/f0xYMAA/Pnnn/jrr78wePBgBAQEoG/fvjh48KBZP8+fP4+XXnoJwcHBCA4OxsSJE3H16lVx++HDh+Hj44N169ahS5cuCA4Oxu+//47bt29j2rRpaNeunXi9H374wezcWVlZOHHiBDp27PjAv6N77dmzB8OGDUOrVq3QsmVL9OrVC2vWrHmkY4u/6b97voibN2+WOvdF8+bN8eabb6Jly5Zl6h8RERERUVXDQMBK5ebmYuzYsXBzc0NMTAzmz5+PgoICvPjii8jJyUF8fDxGjRoFtVqDDz76FBMmT8Nffx7D1CkvQafTiucRTCbMevcNdOnWAx9/tgD+AUH4Km4B4g//AQCIXbwKANCmXUfMXbAYAHD1ymVMfnkMMjIy8PrM9zH9jXdw/do/mDJhDDIybpv188NZM5FpqoU6IaPgXLcVMi78AmXWSbw+IhR1ajrio1kzsWHdt+jdLxL/+fQLhIa1wScfvY+9//svAGDv7p14581paNCwEWb/Zy5GjhmP/+3agXfemPrAiQ4/+M/nqF6jBsIj2iF28SpUr1ETAGAyGrFh3beIfuMdTJgyFQ0bNcbSxTH4ZtVS9B0QhU/mxmDajLeRk52FWW+/Dq32TvBw+9ZNLJz/KaIGD8V/Pl0Aj7p1MefDd3Hl8qWivv7vv1iyaCEGRD2LT+fGYsSocfjfrh2Imf8p5DJg1epv4evrC19fX6xfvx6dO3dGUlISoqKikJKSgrfffhuff/45ZDIZxo0bh7Nnz5rdU0xMDMaNG4dPP/0U7dq1AwCkpqZizpw5ePnll7FgwQJkZ2djypQpmDp1KgYPHowvv/wSgiDgtddeg1Zb9Pd+6dIlDBkyBLdu3cInn3yCjz76CFevXsXQoUNx69Yts2vGxsbi9ddfx7vvvotWrVohOjoaFy5cwKxZs7B06VL4+vri9ddfx6FDh8RjfvvtNzRp0gR16z76YyC//PILJk6cCD8/P8TFxSEmJgb169fHBx98gBMnTpjtazKZYDAYYDAYoNVqce7cOcTFxaFNmzZo1qyZuF/nzp2xfft2TJw4ET/99BPS0tLEbaNGjUKbNm3MzisIgnjee/+U96SaRERERERS4CMDViopKQkZGRkYMWIEgoODAQBNmjTB+vXrkZeXh7lz56Jhw0aYPWc+BBR9a9rCzx+jXxiMnT9tReQzzwIo+hA0YvQ49O4bCQBo6R+EX/f/jIN//IbQ8LbwbekPAHB1rY7mvkXfqK5euQRqjQaffxEHR0cnAEBwSBief7Y/1q9djZcn/p/YT/s6IVDW6wglAIeazVB4+xycjVfg4qjCpYtJOPDLXkycMg3PPDtMPE9q6vWiEQbde2LJ4hiEhbfFW+9+KJ7T07MBpv/fKzh88De0aduh1Pp4eTeHUqmCq6ubeA/Fnh/xotlxt27ewIvjJyJq0BCxTalW4/2Z0biYlCQer9VqMf31dxAcElbUj/oNMHRQXxz64zc0aNgYJ/86Do86dREZ9SzkcjkCW7WGvYM9crNz4OKkRkR4KJyciupVPOT9gw8+gEqlwurVq8VtnTt3Rp8+fbBmzRpERUWJfXr66afxzDPPmN1LQUEB3nvvPfHb+KSkJMydOxcfffQRBg0aBADIz8/HlClTcOnSJbRo0QKxsbGwt7fHqlWrxGtGRESge/fuWLZsGV5//XXx/MOGDUOvXr3En48cOYKJEyeie/eiRx7CwsLg6upqNlfAgQMHyjw6ICkpCQMHDsTMmTPFtlatWiE8PByHDx9GYGCg2D5z5kyz/QDA1dUV33zzjVnb7NmzYTKZsHv3buzZswdA0eMG3bp1w+jRo1G7tvkcDufOnUNoaGip/VuwYIFZHYiIiIiIqgIGAlbGZBJw8Z8s5AuuqObqhpdffhm9evVChw4d0K5dO0RHR6OgoAAnTpzAkGEjYDAYxWPr1q2Hhg0b4djRQ2IgAAC+fgHiv6tUKri6ukL7gCH5x4/FI6hVa2jUGhgNBgCAo4Mj/ANa4Vj8YQDAhX8yAQAyx3ricT3bNMQvVz3FEQqnTv4FAOjQqavZ+Wd99BkA4MrlS7iRnoZhL4wWrwMAgUHBcHR0xNH4w2jTtoPZNgCQKxQPXGqwmZe32c8z3/sIAJCZkYGrV5KRknIVB38/AAAoLNSb7Xt3uOBeq+gDZfEogqDgEGz7cRNefnE42nfsjPCI9niqR2+4OqvhoFGW2pcjR46gS5cu4gdzoGjm/J49e2LJkiXIz88X21u0aFHqOYoDIQCoWbNoJMTdH6BdXV0BANnZ2QCAQ4cOISwsDBqNBoZ/a+fk5ISQkBD88ccfZue+95rh4eGIiYnBmTNn0KFDB3Tq1MksQBAEAb/++ivmzZtXal/vp/hZ/ry8PFy6dAlXrlzBqVOnAAB6vfnfwaRJk8RJBQ0GA65fv47Vq1djyJAh+Oabb+Dn5wcAcHZ2xsKFC5GSkoL9+/fj8OHDOHz4MFauXIn169djxYoVaNWqlXjeRo0a4cMPPyx1UsEGDRqU6X6IiIiIiCoDBgJW5ETiDWzcl4h/0nNhMJrQpMNE3E7ai59+2o7169dDo9FgwIABGPbCaJhMJqz9dhXWfruqxHlUavMPPPd+AJLJ5RCE+y9JmJ2ViZ/37sbPe3eX2Obq6oZ9R69i64GLAAC5QgW5TIahPX3QIageDmyUQ/h3VsPsrKyiY9xKn1SwePuCuXOwYO6cEttv3byB1OvXMGxwP7P2GW+9h169+9+3//b25pPGJZw7gy/mzkHC2b+h0WjQsHFT1K7tAQAlhoprNPbiv8vlRU/kmP69ny7desBkMmHrlg1YvWoZVi3/CnXr1kN09HT07t271L5kZWWJH+LvVqNGDQiCgNzcXNjZFf0a32+yu7vDhDv3aF/KnkUyMzOxY8cO7Nixo8S2e5ftu/ea8+fPx+LFi7Fz507s2rULcrkcbdu2xQcffIB69erh1KlT0Ol0aN26bKs73L59G++99x727NkDmUyGhg0bIiQkBEDJv4N69erB3/9OMNOqVSt06tQJnTt3RkxMDBYvXmy2v6enJ55//nk8//zzMJlM2LNnD9544w3Mnj1bnMcBKKqZn58fJxUkIiIiIqvBQMBKnEi8gS83nkCB1gBnRyWUCiUKHepC6TQEDUOHoYe/EqeP7ce6dd/BwdkNMpkMg54dhq5PlRzmrFaX/Aa0LJycnRHcOhzPDh1u1m4yCdh3LAXf7zmP4s9wKpUck54NhG/jGiXP4+QMAMjKzBC/bQeKRgZkZ2XB8d/tL014FUHBIaX2o0ZNdyxaZj5U3KPOoz+7npeXi9enTUKTpl5Y8c0GNGjYCHK5HIcO/oYDv+x95PMU6/ZUL3R7qhcK8nPx98mj+HrlCkRHR6N169YlhqgDRZPh3bx5s0R7cZurqytyc3PL3I8HcXZ2Rtu2bTF69OgS24rDhwcdGx0djejoaFy8eBF79+5FXFwcZs2ahSVLluDAgQNo27YtlMrSR0Tcz/Tp03Hx4kWsWrUKrVq1gkqlQkFBAb7//vtHOt7R0RFNmjTB5cuXAQC7du3Ce++9h++++w6NGzcW95PL5ejRowfi4+Mf+dxERERERFUVJxW0AiaTgI37ElGgNaBGNTXUSgXSLh3Hr2tmwMlOC6MAXMlxwcRXo+Hk5Izbt2/Dy7s5rlxJhk9zX/FPo8ZNsGr5Ypz482iZrl/8TXixwKDWuJx8Ec2aeYvnbtDYC3M+/xI/79tntu+Qp3xKDQMAwD8gCADwx7/D84stWRSD2AWfo0HDRnBzq47U69fM7qNmTXcsXRSDpPMJUCqVZtt8mvuiWjXXUvtdmiuXk5GdlYVnBg9Fo8ZNxGOOHCoaOm96wEiJe33w7ht4581pUMhl8KxTEwP69cWECRNgMBiQnp5e6jGhoaH4+eefzT70G41G7Nq1C02aNDF7Nr+8hIWFISkpCS1atIC/vz/8/f3RsmVLrFq1Cv/73//ue9w///yDTp064b//LZrwsUmTJhg3bhzatm2La9euAXi8+QMA4NixY+jRowfCw8PFez5woOh1YTI9/O8gJycHly5dQsOGDQEAXl5eyMzMxNdff13q/snJyfD29i51GxERERGRteAIAStw8Z8s/JOeC2dHpfhsvJtHMwiCCX/u/gq+Yb3x13ElEv5Yi7y8XHTs3BVt23fEm9Gv4qNZM9Gtx9MwGU34ft03OHvmNF4YOa5M13d0dMKlC4k4ffIvhIS1wQujxmHyy6Pw1oz/Q/+Bg6AzyvDV0tW4cfkE6rQuGjVQu7oDUgDUrHb/oetNvbzRqUt3fBW3ADqtFs28fHD40O84+PsBzProMygUCowZPwHzP/sP5Ao5Itp1RG5ODr79ehlupKfD26f0Z+qLOTk5I/F8Ak78eQzNff1K3ad+g4ZwdHTEt6tXQKFQQGFnhwM/78XO7T8CALQF2lKPK02r4FDM//w/WP7VQnTt0hnZ2dmIjY1Fo0aN0Lx581KPmTRpEg4cOIARI0Zg/PjxUCqV+Pbbb5GSkoIZM2Y88rXLYsKECRgyZAheeuklDB06FGq1GuvXr8eePXuwcOHC+x5Xr149eHh44MMPP0Rubi4aNGiA06dPY//+/XjppZdw+/ZtnD59GjExMSWOzc3NxapVq0q0161bFz169EBAQAC2bdsGPz8/eHh44Pjx41iyZAlkMlmJJSavXLlitizjzZs3sWzZMnHlDaAorBg/fjy++uorXLt2Df3794eHhwdu3bqFH3/8EQcPHsTKlSvNzltQUICTJ0+WOocAAPj7+0OhUNy3PkRERERElQ0DASuQnaeHwWiCUnFnGLbGsRo6Rk3DmYNbcGT3ShgMetRv0Bjvf/gpWgUXzZT+6dxYfL1yCWa9PQN2SiW8fVrg8/lxJWbdf5jnho3Amm9W4N2Z07Hq241o2swLX3y5DCuWxOGjD96BvtAIlVNt1A0ZAScPPwR6uSOkvjOOlXxEvYS33v0QX6/4Cps2fIeszEw0aNQI7334Kdp37AIA6NNvIBwdHLFu7Wr8tHUL7O3t0dI/CG+9+yHq1K33wHM/O/QFxC2cixnTJuHz+XGl7uPk5IzZH8/DV3ELMOud1+Hg4IhmXj74InYp3pg+BadO/om27R/tG+/IqEFQ2wEbN36P79evg0ajQUREBKKjo+87hN7Lywtr167FvHnz8Oabb0ImkyEgIABLly697wfTJ9W8eXOsWbMG8+fPx4wZMyAIAry9vfHll1+iW7duDzw2NjYW8+bNw4IFC5CRkYE6depg0qRJGD9+PH766Sd4eXmV+mhEVlYWPv744xLtERER6NGjB+bMmYPZs2dj9uzZAIom+Js1axa2bt2Ko0fNR7QsWrQIixYtAlA0CsTZ2Rl+fn5Yvny5OO8AAEydOhUtWrTAhg0bxBDDxcUFISEh2LhxY4mQJjk5GSNHjrzvvcfHx8PFxeWB9SEiIiIiqkxkAhfQLhfFM57fPZmZVJKuZuI/q45Ao1ZArVRABsDZUQmFQo7sXD2ycvXQFRrxclQAGtYp/w8sBQUFuHDhApo2bWo2Wd2pCzex7MfT0OnvrGTQPawBojo3g1x+/1n+rZFCIYObswZqZfl8g5yfn4+zZ8+iRYsWnOROAqy39FhzabHe0mPNpceaS8vW623JzwZEZcERAlagSb1qqFfLCcnXs6FykUOjtoNCIcftLC30hUbk6QpRz90J9Ws7S9anX46nYP3/EsTJA2Uy4LmnfNA52FOyPlQWdv+GAapyCgOIiIiIiIjKAycVtAJyuQyDunrBXm2HW9k66AtNMBkF5BUYkJmnh73KDr3aNJLkW3mTScDGvYlYt/tOGKBWKTBhUKBNhgFKhZxhABERERERVUocIWAlAr3cMXFQIDbuS0RGthbIB3SFRtRzd0KvNo3QvFH1h5/kCekKjVi14yROJN5ZJs/VWY1JgwLhKeHohMpCqZDDzUUNpR3DACIiIiIiqnwYCFiRQC93+DetieRr2UjPyIdJEFC/trMkIwPytEbEbjyNlPQ8sa1+bWdMGBQAN+eKmfyuMlPZyeHqrIHSjoNwiIiIiIiocmIgYGXkchmaeFaDq4saBTqDJNe8fjMPG3+/jZyCO+vB+zetiRcH+EGjsr2XmMpODjcXDewUDAOIiIiIiKjysr1Pa1Suzly6hSVbTkOrvxMGdGnticHdvG1uJQGAYQAREREREVUdDATosf361z/4blcCTP/OHigDMKibF7qFNrBsxyxErVTAzVkNBcMAIiIiIiKqAhgIUJmZBAE//HIBuw9fFtvsFMDI3s0R6lfPgj2zHIYBRERERERU1TAQoDLRFxqx6qczOJ6QLra5OCrRq5UzWjap+JUMKiONUgFXFw0UNviIBBERERERVV0MBOiRZefpsWjTCVy6li221XN3wov9fHA7PcWCPbMcjUoBV2eGAUREREREVPUwEKBHcu1mLr7ccAK3srRim2/j6hgX6Q+YCnE7/QEHWymNSgE3Z41NTp5IRERERERVHwMBeqhzybfx1ZZTZssYdgiqhyE9vKGQy1FQUGjB3lmGRmUHN2c1wwAiIiIiIqqyGAjQA/1x8hq+/e85mEx3VhKI6tIM3cMaQCazvQ/DMgAadVEYYIv3T0RERERE1oOBAJXKJAjY9utF7PwjWWxT2skxuq8fgpvXslzHLEgGwF5jB1cnhgFERERERFT1MRCgEgoNRny9/SyOnk0T25wdlJgwKBCN61azYM8sh2EAERERERFZGwYCZCYnX49Fm07i4j9ZYludmo6YOCgQNV3tLdgzy5HJAAeNEtUcVQwDiIiIiIjIajAQIFHqrTx8ueEEbmQWiG3NG1XH+MiWcNAoLdgzy5HJAEeNEtWc1JbuChERERERUbliIEAAgMQrGVi0+STytXdWEmgXUBfDevpAoZBbsGeWwzCAiIiIiIisGQMBwqHT1/HNjrMw/ruSAABEdmqKnm0a2uwQeZkMcNKo4OKksnRXiIiIiIiIKgQDARsmCAJ++u0Stv9+SWyzU8gxqq8vQlrUtmDPLEsmA5ztVXB2ZBhARERERETWi4GAjSo0mPDtzrM4/Heq2OZkX7SSQJN6trmSAMAwgIiIiIiIbAcDARuUV1CIxZtPIvFqptjmUcMBEwcHwd1GVxIAALkMcHZQwcmBYQAREREREVk/BgI2Jj0jH19uOIG02/lim08DN4yP8oejja4kAPwbBjiq4GTPMICIiIiIiGxDpZg+3mQyYeHChejQoQOCgoIwbtw4XL169b77b926FT4+PiX+pKSkiOdbtmwZevbsiaCgIPTp0wcbNmwwO8eiRYtKPYc1S0rJxCerj5qFARH+dTD5uSAbDwNkcHFSMwwgIiIiIiKbUilGCMTFxWHt2rWYM2cOPDw88Nlnn2Hs2LHYtm0bVKqSH9ISEhIQFhaGefPmmbVXr14dAPDVV19hxYoVmDVrFlq2bImDBw/i/fffh1KpRGRkpHiOAQMGIDo6usLvrzKIP5OKr7efhcFoEtv6d2yCpyMa2exKAkBRGFDNSQUHGw5EiIiIiIjINlk8ENDr9VixYgWmT5+Ozp07AwDmz5+PDh06YPfu3ejbt2+JY86fPw8fHx+4u7uXes7vvvsOY8aMQe/evQEADRo0wIkTJ7BhwwYxEDh//jyeffbZ+57DWgiCgJ0Hk7H1wEWxzU4hw4jevgjz87BgzyxPLpPB1VkFezXDACIiIiIisj0Wf2Tg3LlzyMvLQ0REhNjm4uICX19fxMfHl3pMQkICmjZtWuo2k8mETz75BAMHDjRrl8vlyM7OBlAUQiQnJ6NJkybldBeVk8FowuodZ83CAEd7Jf5vSLDNhwEKOcMAIiIiIiKybRYfIZCaWrTsXZ06dczaa9WqJW67W1ZWFtLS0nD06FGsXbsWGRkZCAgIQHR0NBo3bgy5XG4WLgDAtWvXsH37dgwZMgQAkJSUBKPRiF27duGjjz6CTqdDaGgooqOjUatWrce+F0EQkJ+f//AdJXA7Mw9fbjqNpJRssc3dVYNx/VvA3U2NgoKCcruWTqsz+2dlZ6eQw9FJDcFYiPz8Qkt357EU//2V598j3R/rLT3WXFqst/RYc+mx5tKy9XoLgmDTj+VS1WHxQKD4TeLeuQLUajWysrJK7J+YmAig6Jfs448/hlarxaJFizBs2DBs27YNNWvWNNv/5s2bGDduHGrUqIFXXnkFQNHjAgBgb2+PBQsW4NatW5g3bx5GjBiBH374ARqN5rHupbCwEGfPnn2sY8tTVr4Jq/el41a2QWyrW12Jp0OckX37GrJvV8x1U/5JqZgTlyONSgkXRyUy0o0wmUwPP6CSS05OtnQXbArrLT3WXFqst/RYc+mx5tKy5XqXNhcaUWVj8UCg+MO3Xq83+yCu0+lgb29fYv+QkBAcPHgQbm5uYuoWGxuLzp07Y/PmzRg/fry478WLFzF+/HgYjUasXr0aLi4uAIDIyEh07NhRnIQQALy8vNCxY0fs27dPnHugrJRKJZo1a/ZYx5aXK6k5mPfjMWTn3QkDWvvUxJDuzWBnVzFPiOi0OqT8kwLPep5Qa9QVco3yYGcnh5uTGsoKqoOUCgoKkJycjEaNGpX6e0Lli/WWHmsuLdZbeqy59Fhzadl6vZOSkizdBaJHYvFAoPhRgfT0dDRo0EBsT09Pv+8ygHd/kAeKvun39PREWlqa2Hbs2DG88sorqF27NpYtW4batWs/8By1atWCq6trqY8pPCqZTAYHB4fHPr48bPzlFLLz7gyD79OuMfq2byzJkCW1Rl1p3/AVChncnDVQKxWW7kq5sre3t/hrzpaw3tJjzaXFekuPNZceay4tW603HxegqsLiX5U2b94cTk5OOHz4sNiWnZ2NM2fOIDQ0tMT+69evR3h4uNmz+rm5uUhOTha/nT958iTGjh0LLy8vrFmzpkQYMH/+fPTs2ROCIIhtKSkpyMjIsPg3/E+qrrsTgKJJ80b18UW/Dk1s/g3JzkrDACIiIiIioidh8UBApVJh+PDh+Pzzz7F3716cO3cOr732Gjw8PNCjRw8YjUbcuHEDWq0WANCxY0eYTCbMmDEDiYmJOHXqFCZPnozq1asjKioKBoMB06dPR40aNTBnzhzodDrcuHEDN27cwO3bRQ/PP/XUU/jnn3/w/vvv49KlS4iPj8fkyZMRHByMDh06WLIcT2xE7xaY/VIEPpvSAW386zz8ACtnp5AzDCAiIiIiIiqFxR8ZAIApU6bAYDDg7bffhlarRWhoKJYvXw6lUomUlBR069YNH3/8MaKiolCnTh2sWrUKc+fOxdChQyEIAtq1a4fVq1dDrVbj+PHjuHz5MgCge/fuZtepV68e9u3bh5YtW2Lp0qVYsGABoqKioFKp0K1bN7z++utV/tt0O4UcQd61cDtbiwKd4eEHWDGlQg43FzWUdgwDiIiIiIiI7lUpAgGFQoHo6GhER0eX2Obp6YmEhASzNj8/P6xYsaLUcwUHB5fYvzQREREllick66Gyk8PVWWMVEwgSERERERFVBH5aIqvDMICIiIiIiOjhKsUIAaLyorKTw81FAzsFwwAiIiIiIqIHYSBAVkNlJ0d1Fw0UDAOIiIiIiIgeioEAWQW1UgE3ZzXDACIiIiIiokfEQICqPI1SAVcXDRTyqr1CBBERERERkZT4dSpVaQwDiIiIiIiIHg9HCFCVpVEp4OasgZxhABERERERUZkxEKAqSaOyg5uzmmEAERERERHRY2IgQFWKDIBGXRQGyGQMA4iIiIiIiB4XAwGqMmQA7DV2cHViGEBERERERPSkOKkgVQkMA4iIiIiIiMoXRwhQpSeTAQ4aJao5qhgGEBERERERlRMGAlSpyWSAo0aJak5qS3eFiIiIiIjIqvCRAaq0GAYQERERERFVHAYCVCkxDCAiIiIiIqpYfGSAKh2ZDHDSqODipLJ0V4iIiIiIiKwWRwhQpcIwgIiIiIiISBocIUCVhkwGONur4OzIMICIiIiIiKiicYQAVQoMA4iIiIiIiKTFQIAsjmEAERERERGR9BgIkEUxDCAiIiIiIrIMBgJkMQwDiIiIiIiILIeBAFkEwwAiIiIiIiLLYiBAkmMYQEREREREZHlcdpAkJZMBLg4qODkwDCAiIiIiIrIkjhAgyTAMICIiIiIiqjw4QoAkIZcBzgwDiIiIiIiIKg2OEKAKxzCAiIiIiIio8mEgQBWKYQAREREREVHlxEcGqMLIZYCzowpO9gwDiIiIiIiIKhuOEKAKwTCAiIiIiIiocmMgQOWOYQAREREREVHlx0CAyhXDACIiIiIioqqBgQCVG7lcxjCAiIiIiIioimAgQOVCqbRDNUc1wwAiIiIiIqIqgoEAPTG5XAYXRxXs1QpLd4WIiIiIiIgeEQMBeiJyGVDNUQ07mcnSXSEiIiIiIqIysLN0B6jqkssAF0cVZIIBRqPR0t0hIiIiIiKiMuAIAXosxWGAI+cMICIiIiIiqpIYCFCZMQwgIiIiIiKq+vjIAJWJXAa4OKnhqFFauitERERERET0BBgI0COTy2RwcVIxDCAiIiIiIrICfGSAHgnDACIiIiIiIuvCQIAeimEAERERERGR9eEjA/RAcpkM1ZxUcGAYQEREREREZFU4QoDui2EAERERERGR9WIgQKViGEBERERERGTdGAhQCQwDiIiIiIiIrB8DATLDMICIiIiIiMg2MBAgEcMAIiIiIiIi28FVBghAURjg6qyCvZphABERERERkS3gCAFiGEBERERERGSDGAjYOIYBREREREREtomBgA1jGEBERERERGS7GAjYKIYBREREREREto2BgA1iGEBEREREREQMBGwMwwAiIiIiIiICGAjYFIYBREREREREVMzO0h0gacjlMrg6MQwgIiIiIiKiIhwhYAMYBhAREREREdG9OELAyinkMlRjGEBERERERET34AgBK8YwgIiIiIiIiO6HIwSsmKuzGhoV/4qJiIiIiIioJH5atFLODioo7TgAhIiIiIiIiErHT4xWimEAERERERERPQg/NRIRERERERHZIAYCRERERERERDaIgQARERERERGRDWIgQERERERERGSDGAgQERERERER2SAGAkREREREREQ2iIEAERERERERkQ1iIEBERERERERkgxgIEBEREREREdkgBgJERERERERENoiBABEREREREZENYiBAREREREREZIMYCBARERERERHZIAYCRERERERERDaIgQARERERERGRDaoUgYDJZMLChQvRoUMHBAUFYdy4cbh69ep999+6dSt8fHxK/ElJSRH32blzJ3r37o2AgABERkbi4MGDZufIyMjAtGnTEBoairCwMMyaNQsFBQUVdo9ERERERERElUmlCATi4uKwdu1azJ49G+vWrYPJZMLYsWOh1+tL3T8hIQFhYWH47bffzP7UqVMHAHDo0CFER0djyJAh2LJlCyIiIjB+/HhcuHBBPMeUKVNw+fJlrFq1CgsWLMD+/fvx/vvvS3G7RERERERERBZn8UBAr9djxYoVmDJlCjp37ozmzZtj/vz5SE1Nxe7du0s95vz58/Dx8YG7u7vZH4VCAQBYunQpunfvjhEjRqBp06Z4/fXX4efnh6+//hoA8Oeff+LIkSP45JNP4Ofnh4iICHzwwQf48ccfkZaWJtm9ExEREREREVmKxQOBc+fOIS8vDxEREWKbi4sLfH19ER8fX+oxCQkJaNq0aanbTCYTjh8/bnY+AAgPDxfPd/ToUbi7u5udIywsDDKZDMeOHXvSWyIiIiIiIiKq9Ows3YHU1FQAEIf7F6tVq5a47W5ZWVlIS0vD0aNHsXbtWmRkZCAgIADR0dFo3LgxsrOzkZ+fDw8Pj/ueLy0trcT1VCoVXF1dcf369ce+F0EQkJ+f/9jHV1XFcy9wDgbpsObSYr2lx5pLi/WWHmsuPdZcWrZeb0EQIJPJLN0NooeyeCBQ/CahUqnM2tVqNbKyskrsn5iYCKDol+zjjz+GVqvFokWLMGzYMGzbtg0Gg+G+59PpdOI1791+7z6Po7CwEGfPnn3s46u65ORkS3fB5rDm0mK9pceaS4v1lh5rLj3WXFq2XO/SPm8QVTYWDwQ0Gg2AorkEiv8dAHQ6Hezt7UvsHxISgoMHD8LNzU1M3WJjY9G5c2ds3rwZgwcPFs93t7vPp9FoSp2wUKfTwcHB4bHvRalUolmzZo99fFVVUFCA5ORkNGrUqNS/Myp/rLm0WG/psebSYr2lx5pLjzWXlq3XOykpydJdIHokFg8Eiofup6eno0GDBmJ7eno6fHx8Sj2mevXqZj/b29vD09MTaWlpcHV1hYODA9LT0832SU9PR+3atQEAHh4e2LNnj9l2vV6PzMxM1KpV67HvRSaTPVGgUNXZ29vb9P1bAmsuLdZbeqy5tFhv6bHm0mPNpWWr9ebjAlRVWHxSwebNm8PJyQmHDx8W27Kzs3HmzBmEhoaW2H/9+vUIDw83e1Y/NzcXycnJaNasGWQyGYKDg3HkyBGz4w4fPoyQkBAAQGhoKFJTU3H58mVxe/H+rVu3Ltf7IyIiIiIiIqqMLB4IqFQqDB8+HJ9//jn27t2Lc+fO4bXXXoOHhwd69OgBo9GIGzduQKvVAgA6duwIk8mEGTNmIDExEadOncLkyZNRvXp1REVFAQBGjx6N7du3Y+XKlbhw4QI+/fRTnD17FiNHjgQABAYGIjg4GK+99hpOnjyJQ4cO4d1330VkZKQ4ioCIiIiIiIjImlk8EACAKVOmYNCgQXj77bcxdOhQKBQKLF++HEqlEtevX0f79u2xY8cOAEWPGKxatQr5+fkYOnQoRo0aBWdnZ6xevRpqtRoA0L59e/znP//Bd999h4EDB+LQoUNYvHixuMygTCZDbGwsPD09MXLkSPzf//0fOnbsiPfff99SJSAiIiIiIiKSlMXnEAAAhUKB6OhoREdHl9jm6emJhIQEszY/Pz+sWLHigeeMjIxEZGTkfbfXqFEDCxcufKz+EhEREREREVV1lWKEABERERERERFJi4EAERERERERkQ1iIEBERERERERkgxgIEBEREREREdkgBgJERERERERENkgmCIJg6U5Yg+PHj0MQBKhUKkt3RXKCIKCwsBBKpRIymczS3bEJrLm0WG/psebSYr2lx5pLjzWXlq3XW6/XQyaTITg42NJdIXqgSrHsoDWwxTe6YjKZzCaDEEtizaXFekuPNZcW6y091lx6rLm0bL3eMpnMpj8fUNXBEQJERERERERENohzCBARERERERHZIAYCRERERERERDaIgQARERERERGRDWIgQERERERERGSDGAgQERERERER2SAGAkREREREREQ2iIEAERERERERkQ1iIEBERERERERkgxgIEBEREREREdkgBgJERERERERENoiBABEREREREZENYiBAREREREREZIMYCBBMJhMWLlyIDh06ICgoCOPGjcPVq1cf6ditW7fCx8cHKSkpZu09evSAj4+P2Z833nhD3J6RkYFp06YhNDQUYWFhmDVrFgoKCsr1viqr8q73vXW++8+1a9cAAMeOHSt1++HDhyvkHiubsta8uM73/rm77jt37kTv3r0REBCAyMhIHDx40OwcfI2XX71NJhOWLVuGnj17IigoCH369MGGDRvMzrFo0aJSz2ErKuI1zvfxByvvmvO9/MHKWu/CwkLMnTtX3H/48OE4e/as2T4HDx5EVFQUAgMD0atXL2zfvt1su06nw6xZsxAREYFWrVph2rRpuH37doXcX2VUETXftGkT+vXrh6CgIPTo0QNLliyB0WgUtz/KexMRlTOBbF5MTIwQHh4u/Pzzz8LZs2eFMWPGCD169BB0Ot0Dj0tJSRFat24teHt7C1evXhXb8/LyhObNmws///yzkJ6eLv7Jzs4W9xk+fLjwzDPPCKdPnxb++OMPoUuXLsKMGTMq7B4rk/Ku9901Tk9PFxITE4Xw8HCzeq5Zs0bo3r17iX0fdk1rUdaaf/rpp8Lw4cNL1MtgMAiCIAgHDx4U/Pz8hK+//lpISkoS5syZI7Rs2VJISkoSz8HXePnVOy4uTggJCRG2b98uXL58WVi3bp3g6+srbNmyRTzHq6++KkRHR5c4h60o75rzffzhyrvmfC9/sLLW+6233hLatm0rHDhwQEhKShImT54stGvXTnwNJyUlCf7+/sK8efOEpKQkYdmyZYKvr6/wxx9/iOd44403hO7duwvx8fHCiRMnhMjISOH555+X5H4rg/Ku+Y8//ij4+fkJ69atEy5fvixs375dCA4OFmJiYsRzPOz3hIjKHwMBG6fT6YRWrVoJa9asEduysrKEgIAAYdu2bfc9zmg0CkOHDhVGjBhR4gPqiRMnBG9vbyEzM7PUY48fPy54e3ubfXj69ddfBR8fHyE1NbUc7qryqoh632vy5MlCr169zP6D/d577wkvv/xy+dxEFfM4NR87dqwwe/bs+55zzJgxwquvvmrW9txzzwnvvPOOIAh8jZd3vTt06CDExcWZtb355pvCsGHDxJ+ffvppYeXKlU/W+SqqImrO9/EHq4ia34vv5XeUtd5XrlwRfHx8hJ9//tls/y5duogf+N955x1h0KBBZsdNnTpVGDNmjCAIgpCamio0b95c+OWXX8TtFy9eFLy9vYXjx4+X5+1VShVR8yFDhggzZ840Oy42Nlbo1KmT+HNZf0+I6MnxkQEbd+7cOeTl5SEiIkJsc3Fxga+vL+Lj4+973OLFi1FYWIiXXnqpxLaEhATUrFkT1apVK/XYo0ePwt3dHU2bNhXbwsLCIJPJcOzYsSe4m8qvIup9t99++w27d+/G7NmzoVKpxPaEhASzetuSx6n5g+plMplw/Phxs/MBQHh4uHg+vsbLt96ffPIJBg4caNYul8uRnZ0NANDr9UhOTkaTJk3K6S6qlvKuefF2vo/fX0XU/G58LzdX1nr//vvvcHZ2RseOHc3237dvn3iOo0ePlngfb9OmDY4dOwZBEMTXcZs2bcTtjRs3Ru3atR/432trURE1nz59Ol588UWz4+RyObKyssSfbfU1TmRJDARsXGpqKgCgTp06Zu21atUSt93r5MmTWLFiBT777DMoFIoS2xMSEuDg4IApU6agffv26NevH1atWgWTyQQASEtLK3E9lUoFV1dXXL9+vTxuq9KqiHrfbd68eejWrRtCQkLM2hMTE3Hx4kVERUWhXbt2GD16NE6ePPkEd1J1lLXmWVlZSEtLw9GjR9GvXz+0b98eEyZMwKVLlwAA2dnZyM/Ph4eHx33Px9d4+dVbLpcjIiLCrN7Xrl3D9u3b0b59ewBAUlISjEYjdu3ahZ49e6Jz586Ijo5Genp6Rd1mpVLeNQf4Pv4wFVHzu/G93FxZ633p0iXUr18fu3fvFms1btw4XLhwweycpb2PFxQUICMjA2lpaXBzc4NarX6ka1qbiqh569at0bhxY/HnnJwcfPfdd+jQoQOAsv+eEFH5YCBg44ongLr7GwgAUKvV0Ol0JfbPz8/H9OnTMX36dDRq1KjUcyYmJiI7Oxs9e/bE8uXLMXToUCxYsAAxMTHiNe+93oOuaU0qot7F4uPj8ffff2PChAlm7devX0dOTg7y8/Px9ttvIy4uDjVr1sTw4cORlJT0ZDdUBZS15omJiQAAQRDw8ccf44svvoBOp8OwYcNw8+ZNaLXah56Pr/Hyq/e9bt68iXHjxqFGjRp45ZVXAADnz58HANjb22PBggX46KOPcPHiRYwYMUL8+7JmFVFzvo8/WEW+zvleXlJZ652bm4vLly8jLi4OU6dOxaJFi2BnZ4dhw4bh1q1bAACtVlvifMU/6/V6vsYroOZ3y8vLw4QJE6DT6TBjxgwAZf/vARGVDztLd4AsS6PRACj6j1/xvwNFM+va29uX2P/DDz9E48aNMWTIkPuec+nSpdDpdHB2dgZQNHNybm4uFi1ahMmTJ0Oj0UCv15c4TqfTwcHB4UlvqVKriHoX27JlCwICAuDn52fWXqdOHcTHx8Pe3h5KpRIA4O/vjzNnzuCbb77BrFmznuSWKr2y1jwkJAQHDx6Em5sbZDIZACA2NhadO3fG5s2bMXjwYPF8d7v7fHyNl1+9x48fL+578eJFjB8/HkajEatXr4aLiwsAIDIyEh07dkT16tXFfb28vNCxY0fs27cPvXv3rpB7rSwqouZ8H3+winyd8728pLLW287ODrm5uZg/f744/Hz+/Pno1KkTtmzZgrFjx0KtVpd4DRf/bG9v/8DXeGnXtDYVUfNiN27cwEsvvYSUlBQsX74cnp6eAMr2e0JE5YcjBGxc8VCwe4fWpqeno3bt2iX237RpE/744w+0atUKrVq1wrhx4wAAffv2xeLFiwEUpcnF/xNZzNvbG/n5+cjKyoKHh0eJ6+n1emRmZqJWrVrldm+VUUXUGyh6znrfvn3o169fqdd1cXER/wcSKBqG3bRpU6SlpT3xPVV2Za05AFSvXl38nxGg6H8OPT09kZaWBldXVzg4ODzwfHyNl1+9ix07dgxDhgyBvb091q1bh/r165c4x91q1aoFV1dXmxjaWxE15/v4g1XU65zv5aUra709PDxgZ2dn9iy6RqNB/fr1xeXr6tSpU+r5HBwc4OzsDA8PD2RmZpYIBR70d2xNKqLmAHDhwgU8++yzuHXrFtasWQN/f3+z8zzK7wkRlS8GAjauefPmcHJyMlvDODs7G2fOnEFoaGiJ/Xfv3o2ffvoJP/zwA3744Qd8+OGHAIAlS5ZgyJAhEAQB3bt3R2xsrNlxp06dgru7O9zc3BAaGorU1FRcvnxZ3H7kyBEARc+XWbPyrnexpKQkZGRkoG3btiXOceDAAbRq1cps7WCDwYBz586hWbNm5Xl7lVJZa75+/XqEh4cjPz9fbMvNzUVycjKaNWsGmUyG4OBg8TVb7PDhw+LzvnyNl1+9gaJ5NMaOHQsvLy+sWbOmxP+Mzp8/Hz179oQgCGJbSkoKMjIy+Bp/jJrzffzhKuJ1DvC9/H7KWu/Q0FAYDAacOnVKbNNqtbh69SoaNmwIoOjb6Hvfxw8dOoTg4GDI5XK0bt0aJpPJbJLMS5cuIS0trdRrWpuKqPnVq1cxcuRIMdj18vIyO8ej/p4QUTmz2PoGVGnMmzdPCAsLE/bs2WO2zqxerxcMBoOQnp4uFBQUlHrsoUOHSiyDN2fOHCEoKMhszfCAgABh/fr1giAIgslkEoYMGSIMHDhQOHHihHDw4EGhS5cuwhtvvCHJ/VpaeddbEARhy5Ytgp+fn2A0Gksck5OTI3Tp0kUYOnSocOrUKeHcuXPC1KlThdDQUOHGjRsVco+VTVlqfu3aNSEkJESYOHGicP78eeHkyZPCqFGjhO7duwtarVYQhKLl1Vq0aCGsWLFCSEpKEj755BMhICBAXIKNr/Hyq3dhYaHw1FNPCd26dROuXLliti71rVu3BEEQhFOnTgl+fn7Cu+++K1y8eFE4cuSIEBkZKQwZMkQwmUyWLIVkyvs1zvfxhyvvmgsC38sfpKz/7Rw1apTw9NNPC/Hx8UJiYqIwefJkISIiQnzfOH/+vODn5yd89tlnQlJSkrB8+XLB19dXXCJPEIqWIezatatw6NAh4cSJE0JkZKQwfPhwye/dUsq75sOHDxdCQ0OFs2fPmr2Xp6enC4Lw6L8nRFS+GAiQYDAYhE8//VRo06aNEBQUJIwbN078wHn16lXB29tb2LRpU6nHlvYBtbCwUIiNjRW6desm+Pn5CT179hT/J7LYzZs3hcmTJwtBQUFCeHi48N5779nMm31511sQBGHJkiVC27Zt73vNy5cvC5MnTxbCwsKEwMBAYcyYMUJCQkL53VQlV9aanz59Whg9erTQunVrITg4WJg8ebJw7do1s3Nu2bJFeOqppwR/f39h4MCBZv8TKQh8jZdXvY8dOyZ4e3uX+qdLly7iOf744w/hueeeE4KCgoSwsDDhzTffFDIzM6W9cQsq79c438cfriLeV/hefn9lrXdOTo7w3nvvCeHh4UJgYKAwevRoITEx0eyc+/fvF/r27Su0bNlS6NWrl7B9+3az7Xl5ecLMmTOFkJAQISQkRJg6dapw+/btir/ZSqI8a56amnrf93Jvb2/xHI/ye0JE5UsmCHeNsSQiIiIiIiIim8A5BIiIiIiIiIhsEAMBIiIiIiIiIhvEQICIiIiIiIjIBjEQICIiIiIiIrJBDASIiIiIiIiIbBADASIiIiIiIiIbxECAiIiIiIiIyAYxECAiIiIiIiKyQQwEiIhIctOmTYOPjw9WrFhh6a5UCZcvX8arr76K9u3bo3Xr1hg6dCgOHjxY5vOkpqbi+eefh7+/PyIiIlBQUPDQY1JSUuDj44PNmzcDAGJiYuDj41PmaxMREVHlw0CAiIgklZOTgz179sDb2xvr16+HIAiW7lKllpGRgeHDh+PixYt46623MH/+fNSsWRNjxozBkSNHynSur7/+Gn/99Rc++eQTxMbGwt7evoJ6TURERFWBnaU7QEREtuWnn34CAMycORMjR47EoUOHEBERYeFeVV4//PADMjIysHHjRtSuXRsA0K5dOwwYMADLly9HWFjYI58rMzMTtWrVQu/evSuqu0RERFSFcIQAERFJatOmTYiIiECbNm3QsGFDrFu3Ttw2ZswYREVFlThmwoQJ6N+/v/jz0aNHMXz4cAQGBiIsLAyvv/46bt++LW7fvHkzfH19sWHDBrRr1w5hYWFISkqC0WjEkiVL0LdvXwQEBCAoKAhDhgzBoUOHzK73yy+/ICoqCgEBAejZsyd++uknPPXUU4iJiRH3yczMxLvvvou2bdvC398fzz77bJmH8S9evBgtW7ZEVlaWWfuqVavg5+eHW7duoXbt2hg1apQYBgCAQqFAw4YNceXKlUe+VteuXbF582Zcu3YNPj4+iImJwebNm+Hj44OUlJQS+77xxhtluhciIiKqehgIEBGRZBITE3Hq1ClERkYCACIjI7F3717cvHkTANC/f3/8/fffuHz5snhMdnY2Dhw4gAEDBgAA4uPjMWrUKGg0GnzxxRd46623cOTIEYwYMQJarVY8zmg0YsWKFfjoo4/w5ptvomnTpvj8888RFxeH5557DsuWLcPs2bORmZmJV199VXye/tChQ5gwYQLq1KmDmJgYPP/883jvvfdw/fp18dw6nQ4jR47E3r178dprryE2NhYeHh4YO3ZsmUKBfv36wWAwYPfu3Wbt27dvR/v27VGjRg307t0b06dPN9uelZWF+Ph4eHl5PfK1YmNj0alTJ7i7u2P9+vUYPHjwIx9LRERE1omBABERSWbTpk1wdXVF165dAQADBw6E0WjExo0bAQA9evSAg4OD+FgBAOzevRtGoxF9+/YFAMydOxeNGzfGV199hS5duiAyMhIrVqzAxYsXsWnTJrPrvfzyy+jcuTMGDhwImUyG9PR0vPbaa3jhhRcQFhaGfv36YerUqcjMzERCQgKAoknzvLy8xA/QI0aMwAcffIDCwkLxvD/++CPOnTuHuLg4DB48GJ06dcKCBQvQqlUrfP75549cj3r16iE0NNTsfq9cuYKTJ0+KAci9TCYT3nnnHeTm5mLs2LGPfC1fX19Ur14dKpUKQUFB8PDweORjiYiIyDoxECAiIkkUFhZi69at6N69O7RaLbKzs+Ho6IjWrVvj+++/h8lkgoODA7p3744dO3aIx23fvh0RERGoXbs2CgoKcOLECXTq1AmCIMBgMMBgMKB+/fpo2rQpfv/9d7NrtmjRwuznuXPnYuTIkbh9+zaOHj2KTZs2YevWrQAAvV4PvV6PP//8Ez169IBMJhOP69WrF+zs7ky7c/DgQbi7u8PPz0/sg9FoRJcuXXD69OkSjwA8SP/+/REfH48bN26I9+vk5CSGJvfWMDo6Grt27cLMmTMREBDwyNchIiIiuhcnFSQiIkn88ssvuHXrFjZu3CiOCLjbr7/+ik6dOmHAgAHYunUrzp07h5o1a+Lw4cP4z3/+A6Do8QGTyYSlS5di6dKlJc6hVqvNfnZwcDD7+dSpU5g1axZOnToFe3t7NGvWDHXr1gUACIKAzMxMGI1G1KhRw+w4hUIBV1dX8efMzEzcuHEDfn5+pd7rjRs3UK1atYcXBUVhw+zZs7Fz506MGDEC27dvR8+ePaHRaMz2y87OxqRJkxAfH4933nkHzz///COdn4iIiOh+GAgQEZEkNm3ahPr16+Ojjz4yaxcEAZMmTcK6devQqVMnREREwN3dHTt37oS7uzvUajV69OgBAHB0dIRMJsOoUaPQp0+fEtd40DJ6xUPsfXx8sH37djRp0gRyuRz79+/Hrl27AAA1atSAUqkU5zQoZjKZkJmZKf7s7OyMRo0a3ffxAE9Pz0eqSfG5unbtip07d6JNmzZITEzEO++8Y7ZPamoqRo8ejZSUFMybNw9PP/30I5//QYpHQZhMJrP2vLy8cjk/ERERVW4MBIiIqMLduHEDv/76K8aOHYvw8PAS23v16oXNmzcjLS0NtWvXRr9+/fDzzz/DxcUF3bt3F7/pd3Jygq+vLy5evAh/f3/xeK1WiylTpqBTp05o1qxZqX24ePEiMjMzMWLECLN9Dhw4AKDoQ7FCoUBwcDD27t2LSZMmifvs27cPBoNB/DksLAy//PILatSoIY4wAIpWDTh79myZ5hEAgAEDBmDChAn47rvvULduXbOlBHNzczFy5EjcvHkTK1euREhISJnO/SBOTk4AigKHBg0aAAAuXLhgFn4QERGR9eIcAkREVOF++OEHGAyGUr/VB4pWGzAajfj+++8BFH1ATkhIwLFjx0pMrjd16lT89ttvmDZtGvbv3499+/aJs/vfbwg/ADRu3BhOTk5YvHgxfvnlF/z222945513sHbtWgAQVxmYMmUKzp07hylTpuDAgQNYt26d+I198TfqUVFRqFu3LkaPHo0tW7bg0KFDmDdvHhYsWIBatWpBqVSWqT4dOnSAq6sr1q9fj379+pnNX7Bw4UIkJydj9OjRsLOzw19//SX+OXPmTJmuc6/w8HBoNBrMmTMH+/fvx44dOzBx4kSzxyOIiIjIejEQICKiCrd582Z4eXnB29u71O2tW7eGp6cnNmzYAKPRiObNm8Pb2xs1atRARESE2b7t27fH8uXLkZqaiilTpmDGjBlQKBRYuXIlgoKC7tsHZ2dnxMXFQRAEvPrqq5gxYwauXbuGb7/9Fo6Ojjh69CgAICQkBDExMbh06RImTJiAlStXioGAo6MjgKK5CdasWYPWrVvjs88+w7hx47B7925MmzYNb775ZpnrY2dnhz59+sBoNKJ///5m24qXJIyJicFzzz1n9ufuUQyPw8XFBTExMTAajZg4cSIWLFiAiRMnomXLlk90XiIiIqoaZIIgCJbuBBERUWWxd+9eeHh4mI02SExMRN++fREXF4du3bpZsHdERERE5YdzCBAREd3lt99+w44dOzB9+nQ0btwYaWlpWLRoEZo0aYL27ds/0jkEQYDRaHzofgqFwuzxgMdhMplKTApYmruXTSQiIiICOEKAiIjIjFarxYIFC7Br1y6kp6fD1dUVHTp0wLRp01CzZs1HOsfmzZsf6dGB1atXlzrJYlnExMQgNjb2ofvt3bu3TKsfEBERkfVjIEBERFTOMjIykJKS8tD9iic6fBJpaWlIT09/6H4+Pj5QqVRPdC0iIiKyLgwEiIiIiIiIiGwQVxkgIiIiIiIiskEMBIiIiIiIiIhsEAMBIiIiIiIiIhvEQICIiIiIiIjIBjEQICIiIiIiIrJBDASIiIiIiIiIbBADASIiIiIiIiIb9P+OnB9OA4lgvAAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAABAkAAANrCAYAAAAknMFeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3QUVf/H8femhyQkNAlVxECAFHroVUpEVLqI0qR3ESSoCIINBUEgEKQIojw+SBGen4CIBVCkSBUkiIBAgkoLgdTdZHd/f8SMLEkggUQQP69zOGTvzL1z5+6Ew/3OLSa73W5HRERERERERP71nO50BURERERERETk7qAggYiIiIiIiIgAChKIiIiIiIiIyJ8UJBARERERERERQEECEREREREREfmTggQiIiIiIiIiAihIICIiIiIiIiJ/UpBARERERERERAAFCURE5B5ht9vvdBVERERE/vEUJBCRf5WePXsSGBjo8Cc4OJjmzZszefJkrly5cqer+K+2Zs0aAgMDiY2NzXWeq1evMm7cOPbs2WOk9ezZk549exZEFXMlPT2dd999l2bNmlG9enV69OjBwYMHb5rvxIkTDB48mJo1axIWFsbIkSM5depUlrIXLFhAmzZtqFGjBo8//jgbNmwooDspOLt27SIwMJBdu3bdsTocOnSInj17UrNmTRo3bsyMGTOwWCw3zffdd9/RuXNnqlevTsuWLVm8eHGWINXp06cZPHgwderUoV69ekyaNInExMSCuhUREZF843KnKyAi8nerVq0akyZNMj6npaXx008/MWPGDKKjo/n4448xmUx3sIaSF9HR0axbt47OnTsbadd+v3fC1KlTWbVqFWPGjKFMmTIsWbKEPn36sHbtWu6///5s88TExPDkk0/i4+PDxIkTKVasGKtWreKJJ55g9erVlC1bFoA5c+awYMEChg0bRu3atdm8eTOjR4/G2dmZtm3b/p23+Y8WExND3759qVGjBu+++y4nTpxg5syZxMfHM2XKlBzzHThwgMGDB/Pwww8zatQo9u7dy7Rp07BarQwcOBDICFz17t2b4sWLM3XqVOLi4pg2bRqxsbEsXrz477pFERGRW6IggYj863h7e1OjRg2HtLp165KUlMTs2bM5ePBgluPyzxIQEHDHrv3777/z8ccf89JLL9GjRw8AGjduTNu2bVm4cCGvvfZatvk++OADUlJSWL16NeXKlTPydevWjZkzZ/LOO+8AsHr1atq3b8/w4cMBaNCgAT/99BMfffSRggR5sHDhQry8vJg3bx5ubm40a9YMDw8PXn31VQYPHkzp0qWzzTdnzhyqVq3KtGnTAGjatCnp6enMnz+fXr164eHhwccff0x8fDxr1qyhaNGiAJQsWZKBAweyd+9eateu/bfdp4iISF5puoGIyJ+Cg4MB+O2334y0L7/8kk6dOhESEkKjRo147bXXSE5Odsj35Zdf0qNHD2rWrElwcDDh4eEsX77cOJ45rPq///0vLVq0oFatWmzfvp24uDjGjBlDo0aNCAkJ4fHHH2ft2rUOZZ86dYqRI0fSqFEjatSoQc+ePdm7d69xPDY2lsDAQDZu3MjIkSONYeoTJkzIUs/rnT9/noiICBo0aEDNmjV5+umn2b9/v3HcbDYzd+5cwsPDCQkJoU2bNixYsACbzWac07NnT8aOHcvIkSOpUaMGffv2Neq0ZMkSwsPDqV69OqtXrwbg2LFjDBo0iFq1alGrVi2GDRtGTEzMDeu5cuVKOnXqRI0aNQgNDeXxxx9n48aNRtv26tULgF69ehlTDK6fbpDbe3nppZdYsGABzZs3JyQkhO7du/Pjjz9mae85c+bkWN8dO3aQnp5O69atjTQ3NzeaN2/O1q1bc8x38uRJAgICjAABgJOTE3Xr1nXIZ7FY8Pb2dsjr5+dHfHx8jmVnZ86cObRs2ZJvvvnG+J66det20+H/8+fPJzg4OMvUnKVLlxIUFMSlS5cA+OGHH+jXrx9169YlODiYli1bMmfOHIc2v9b48eNp2bKlQ1pme69Zs8ZIi4+PZ+LEiTRs2JCQkBC6devGjh07spQVGBh4w/v47rvvaNasGW5ubkZaeHg4NpuN7777Lts8FouFXbt2OXy3AG3btiUpKcn43fzuu++oXbu2ESCAjICPl5cX27Ztu2G9RERE7jQFCURE/vTrr78CGJ20//u//2PYsGFUrFiRuXPnMnz4cP73v/8xdOhQY/7xli1bGDZsGEFBQcybN485c+ZQrlw5pkyZkmUOemRkJBEREUycOJGaNWvy/PPPc+LECSZPnszChQupVq0aERER7Ny5E4Djx4/TqVMnYmNjmTBhAtOnT8dkMtG7d292797tUPakSZMoU6YM8+bNo1+/fqxatYqoqKgc7zUpKYknn3ySXbt28fzzzxMZGYm7uzvPPPMMp06dwm63M3jwYBYtWkTXrl2ZP38+4eHhvPvuu1mG8m/cuBEvLy+ioqLo37+/kT5nzhwGDBjA22+/TaNGjfj111/p3r07ly5d4q233uL11183hthndiyvt3z5ciZOnEirVq147733mD59Om5ubowdO5Y//viDoKAgJk6cCMDEiROznWaQl3vZtGkTX331FRMmTGDGjBlcvHiRESNGYLVaAbjvvvtYsWIFXbt2zbFtT5w4gZeXFyVKlHBIv//++zl//jxJSUnZ5itSpAgXLlwgLS3NIT0mJoaEhAQjCNCrVy/Wrl3Ltm3bSExM5H//+x/ffvstjz/+eI51yklcXBwRERH06NGDWbNm4eHhQb9+/YiOjs4xz6OPPkp6ejpffPGFQ/r69etp3LgxxYoV4+jRo/Tp0wc/Pz9mzpxJVFQUderUITIy0gjw3Aqz2Uzv3r356quvGD16NJGRkfj7+9O/f3+HQMHQoUNZsWJFjuWkpqZy9uxZHnjgAYf0okWL4u3tbfxbcL2YmBjS0tKoUKGCQ3rmFJLMfCdOnMhStrOzM2XLls2xbBERkbuFphuIyL+O3W4nPT3d+HzlyhV2795NVFSUMRrAbrczffp0mjRpwvTp041zK1SoQJ8+fdi6dSvNmzfn+PHjdOzYkZdeesk4p2bNmtSrV49du3ZRvXp1I71Hjx6Eh4cbn3fv3s2wYcNo1aoVAGFhYfj5+RlvNiMjI3Fzc2PZsmXGm+PmzZvTvn173n77bVatWmWU1axZMyIiIoCM4efbt29ny5YtjBkzJts2+PTTTzl79iyffvopVatWBaBWrVp06NCBH374gdOnT/P9998zY8YMHnnkEQAaNWqEh4cHs2bNolevXlSqVAkAV1dXJk+ebNQ7c9HBhx9+2GGdgDFjxuDp6cnSpUuN+2nQoAGtWrVi0aJFRv2vFRMTQ79+/Rg6dKiRVqZMGTp16sTevXt55JFHjKkFAQEB2U4z2LZtW67vJT09ncWLFxv1S0pKIiIigujoaIKDg3Fzc7vpVJSEhIQsb/oBvLy8AEhMTDR+vlanTp347LPPiIiIYPTo0Xh7e7Nu3Tq+/fZbAFJSUvDz86NPnz4cOHCAAQMGGHk7d+7sEKDJrZSUFF555RU6dOgAQP369WnVqhULFixg5syZ2eYpU6YMdevW5bPPPjOCJWfOnOHHH3808hw9epSGDRsybdo0nJwy3kc0atSIr7/+ml27dhnfQ16tW7eOo0eP8sknnxi/W02bNqVnz55Mnz7dGLFSvnx5ypcvn2M5CQkJADl+TzktMJhTvmu/28zzsvuOb1S2iIjI3UJBAhH51/nhhx8ICgpySHNycqJhw4ZMmTIFk8nEiRMn+OOPPxg0aJBDQKFu3bp4e3uzfft2mjdvbnTMkpKS+PXXXzlz5gyHDh0CyLJKemZnPFO9evWYM2cOR44coUmTJg4dfcgIIrRo0cKhQ+Li4sIjjzzC3LlzHd5IX99x9ff35+zZszm2wd69eylbtqxDnTw9Pdm0aRMA06ZNw8XFxSGoAfDYY48xa9Ysdu/ebXSsK1as6DBkO6f73blzJ2FhYXh4eBht6u3tTZ06dfj++++zref48eOBjIXgTp48yenTp43h8LlZhR4y2jG39xIQEODQ3iVLlgQyOtO5dbOtGDM7zddr1KgR06ZN44033mD9+vUANGzYkAEDBjBnzhw8PDywWCw89dRTXLhwgcmTJ1OxYkX2799PVFQUhQoVYsKECbmuJ2Q8T+3btzc+e3h40LRpU2NIvNVqdbgfJycnnJyceOyxx5g0aRIXLlygRIkSrF+/Hm9vb2O6QIcOHejQoQNms5lff/2V06dPEx0djdVqzTJSIi927NhBiRIlCAoKcvi9bNGiBW+//TZXrlzB19f3puXkNOUhU04Ll94sX+Z3e6NnQIuiiojI3U5BAhH51wkKCmLy5MlAxn/Y3d3dKVWqlEPnMHNo9+TJk41zr3X+/HkgY7j2pEmT+PLLLzGZTNx///3UqVMHyNpRKFSokMPnmTNnMn/+fDZu3MimTZscAhVlypThypUrFC9ePMu1ixcvjt1ud3gj6enp6XCOk5PTDTsq8fHxFCtWLMfjV65coUiRIjg7OzukZw6hz3yjCmT7xhSy3m98fDwbNmzIdru+a+duX+vMmTNMnDiRHTt24OrqSsWKFalSpQpw8874rdxLdu0IN+8cXsvb2zvbKQWZ35ePj0+OeR977DEeeeQRYmJi8PT0pGTJksyaNQsnJycKFy7Mhg0bOHr0KEuWLKFhw4ZAxggUb29vpkyZQrdu3ahcuXKu61q8eHFcXBz/K1CsWDHj+W/durVDsKljx45MnTqV8PBwXn31VTZu3EivXr1Yv349bdu2xcPDA8gYzv/qq6+ybt060tPTKVu2LDVr1sTFxSXX31t24uPjuXDhQpYgX6YLFy7kKkhw7UiR6yUmJub4HWWmX58v87vNLPdGz0Bm4ElERORupSCBiPzreHl5ERIScsNzChcuDMC4ceMICwvLcjyzIzJ27FhOnjzJ0qVLqVmzJm5ubqSkpPDJJ5/ctB4+Pj48//zzPP/885w8eZKvvvqKefPmMXnyZBYsWICvry8XL17Mku/ChQtAxhz2zGBFXvn4+BjTAq61b98+fH198fX15fLly1itVofOdeb1ihQpckvXbNiwIX379s1y7PqOKmR0zAcOHIirqyurVq2iatWquLi4cPz4cdatW5fr6xbEvdxIxYoVSUxMJC4uziH4cfr0acqUKWN0pK934sQJDh06RIcOHRzmvB85coTAwECcnZ2NRTVr1arlkLdu3bpAxjoWeQkSZLfY4cWLF40AUlRUlMOIjcy28vHxoWXLlmzcuJH69evzyy+/8PLLLxvnvf7662zatIl3332Xhg0bGgGjBg0a5FgXk8lkrP2Q6frFN318fKhQoYLDFKBrZW4TeTNeXl6ULFmS06dPO6RfunSJpKQkHnzwwWzzlS9fHmdn5yz5zpw5A2Dke+CBB4y0TFarldjYWNq0aZOrOoqIiNwpWrhQRCQbFStWpFixYsTGxhISEmL8KVmyJO+88w5HjhwBMobtt2nThnr16hlD7jOHat/o7fPZs2dp1qwZn3/+uXG9AQMG0LBhQ6MjWLduXb755huHEQNWq5X169cTEhKS7RD/3KpTpw4xMTH88ssvRprZbGbEiBGsWrWKsLAw0tPTjfpl+t///gdwS1u4hYWFcfz4capWrWq0Z3BwMEuXLmXz5s1Zzr98+TK//vorXbp0ISQkxAgkXN++148QyO66+X0vN5L5hv/a61ksFrZs2UKjRo1yzPfLL78QERHByZMnjbTjx4/z3XffGetWVKxYEYA9e/Y45N23bx+Q+05yptTUVGPNg8zP27ZtMzrzgYGBDs//teU//vjjHDhwgI8//pjSpUs7BNP27t1LvXr1aNWqlREgOHz4MHFxcTn+Xnh5eXH58mXMZrNDOdcKCwvj999/p1ixYg712r59O4sWLbrps3CtRo0asWXLFocgyKZNm3B2dqZ+/frZ5nF3d6dOnTps3rzZYUTEpk2b8PHxITQ01Cj7hx9+IC4uzjjnu+++Izk5+YbPgIiIyN1AIwlERLLh7OzM6NGjmThxIs7OzrRo0YKrV68yb948zp07Zwx3Dg0N5f/+7/8ICgrC39+fffv2sWDBAkwm0w3nsZcpUwZ/f39ee+01EhMTKV++PIcPH2br1q0MGjQIgOHDh7Nt2zZ69eplvFH/6KOPiImJYdGiRbd1f506deLDDz9kyJAhjBw5kiJFirBs2TLS0tLo0aMHZcuWpV69ekyYMIFz585RpUoVdu/ezcKFC+nYsWO2CwTezNChQ+nevTuDBg3iySefxN3dnRUrVvDll18ye/bsLOcXK1aMMmXKsHz5cvz9/SlcuDDffvsty5YtA/5aJyBzCPiWLVvw9fU1piNkatq0ab7di8Vi4ciRI/j7++Pv75/tOWXKlKFjx468+eabmM1mKlSowJIlS7h69arD4oJnzpwhLi7OWE+iWbNmlC9fnrFjxzJq1CgSExN5++23KVu2LH369AGgZcuWVK9eneeff54RI0ZQsWJFfvzxR6KiomjZsqXRSY2Li+PMmTNZ1ljIzgsvvMCzzz5LsWLFWLx4McnJyQwZMuSmbdGkSRP8/PxYsWIF/fv3d5hrHxoaysaNG/n444958MEHOXr0KFFRUTf8vWjRogUffvghL730El26dOHYsWMsWbLEoePfqVMnPvroI/r27cvgwYMpVaoU33//PQsXLuTpp5/G1dU127bNTv/+/Vm/fj39+/enb9++nDp1ihkzZtCtWzdKly4NZP99DxkyhL59+zJq1Cg6d+7M/v37Wbx4sbEwJ2QsUppZz+HDhxMfH8+0adNo2rRpllEgIiIidx27iMi/yNNPP21/+umnc33++vXr7R07drQHBwfbw8LC7IMHD7YfPXrUOB4bG2sfNGiQvXbt2vbatWvbO3fubF+3bp29X79+9s6dO9vtdrt9586d9sqVK9t37tzpUPb58+ft48ePtzdu3NgeFBRkb9WqlT0qKsputVqNc44cOWLv37+/vUaNGvaaNWvae/fubf/hhx+M4zExMfbKlSvbV69e7VB2RESEvUWLFje8tz/++MP+3HPP2evUqWOvVauW/ZlnnrFHR0cbx5OTk+1Tp061N2nSxB4UFGRv27atfdGiRQ71y649c6qT3W63Hz582N6vXz97zZo17TVq1LB369bN/uWXXxrHV69eba9cubI9JibGbrfb7dHR0fann37aXqNGDXtYWJi9R48e9m3bttnDw8PtI0eOtNvtdrvVarU/99xz9pCQEPsjjzySbb1u9V6u/+4y72327Nk3bFuz2Wx//fXX7Q0aNLBXr17d3qNHD/uBAwcczomIiLBXrlzZIe3UqVP2gQMH2mvXrm1v2LChffz48fZz5845nJOQkGCfMmWKvVGjRvbg4GD7ww8/bH/vvffsZrM5Szte/8xda/bs2fbKlSvbN2/ebG/RooW9evXq9r59+zo8Azfz6quv2itXrmz/5ZdfHNIvX75sf+655+xhYWH2GjVq2Nu3b2//4IMP7C+//LK9UaNG9vT09Gx/LxYvXmxv3ry5PTg42P7EE0/YDx8+bA8ODnZ4li5evGh/4YUX7A0aNLAHBwfb27Zta1+4cKHDd5ld22bnhx9+sHft2tUeHBxsb9KkiX369Ol2i8ViHM/p+/7iiy/s7du3twcFBdlbtmxpX7x4cZayf/75Z3vv3r3toaGh9gYNGthffvlle0JCws0bVURE5A4z2e23sYKQiIiI3JXGjRtHjx49cnybPmfOHCIjI/n555//3oqJiIjIXU1rEoiIiNxjjh8/zsGDB/O0iKGIiIgIKEggIiJyzylatChLly7Nsg2liIiIyM1ouoGIiIiIiIiIABpJICIiIiIiIiJ/UpBARETueRo0JyIiIpI7ChKIyD3n2LFjjB49mkaNGhEcHEzjxo159tlnOXr06J2umtyGNWvWEBgYSGxsbK7zXL16lXHjxrFnzx4jrWfPnvTs2bMgqpgr6enpvPvuuzRr1ozq1avTo0cPDh48eNN8J06cYPDgwdSsWZOwsDBGjhzJqVOnjOOZ7ZPTn08//bQA7yp/7dq1i8DAQHbt2nXH6nDo0CF69uxJzZo1ady4MTNmzMBisdw033fffUfnzp2pXr06LVu2ZPHixVmCVKdPn2bw4MHUqVOHevXqMWnSJBITEwvqVkRERPLE5U5XQEQkP/3yyy888cQT1KhRgwkTJlCsWDH++OMPPvroI7p168ayZcty3BJO7j3R0dGsW7eOzp07G2mTJk26gzWCqVOnsmrVKsaMGUOZMmVYsmQJffr0Ye3atdx///3Z5omJieHJJ5/Ex8eHiRMnUqxYMVatWsUTTzzB6tWrKVu2LM2bN2fFihVZ8k6YMIHExESaNWtW0Ld2z4iJiaFv377UqFGDd999lxMnTjBz5kzi4+OZMmVKjvkOHDjA4MGDefjhhxk1ahR79+5l2rRpWK1WBg4cCGQErnr37k3x4sWZOnUqcXFxTJs2jdjYWBYvXvx33aKIiEiOFCQQkXvKkiVLKFKkCAsXLsTF5a9/4lq1akV4eDjz5s1jwYIFd7CGcqcFBATcsWv//vvvfPzxx7z00kv06NEDgMaNG9O2bVsWLlzIa6+9lm2+Dz74gJSUFFavXk25cuWMfN26dWPmzJm88847FC1alKJFizrkW7ZsGSdOnOC///1vlmOSs4ULF+Ll5cW8efNwc3OjWbNmeHh48OqrrzJ48GBKly6dbb45c+ZQtWpVpk2bBkDTpk1JT09n/vz59OrVCw8PDz7++GPi4+NZs2aN8Z2ULFmSgQMHsnfvXmrXrv233aeIiEh2NN1ARO4pFy9exG63Y7PZHNILFSrEiy++yMMPP2yk9ezZk/HjxzN//nwaNmxI7dq1GTp0KGfPnnXIe+zYMQYNGkStWrWoVasWw4YNIyYmxuGco0ePMnz4cOrXr09QUBBNmjThtddeIzU11TgnMDCQyMhIOnXqRGhoKJGRkaxZs4aQkBD27NlD586dCQkJoW3btnz99decPHmS3r17U716dVq3bs369esdrvnDDz/Qr18/6tatS3BwMC1btmTOnDnGvcfGxhIYGMjGjRsZOXKkMUx9woQJJCcn37Adz58/T0REBA0aNKBmzZo8/fTT7N+/3zhuNpuZO3cu4eHhhISE0KZNGxYsWODQ7j179mTs2LGMHDmSGjVq0LdvX6NOS5YsITw8nOrVq7N69epct/P1Vq5cSadOnahRowahoaE8/vjjbNy4EcgYst6rVy8AevXqZUwxuH66QW7v5aWXXmLBggU0b96ckJAQunfvzo8//mick3lvc+bMybG+O3bsID09ndatWxtpbm5uNG/enK1bt+aY7+TJkwQEBBgBAgAnJyfq1q2bY76LFy/y7rvv8uSTT1K9evUcy87OnDlzaNmyJd98843xPXXr1u2mw//nz59PcHAwV65ccUhfunQpQUFBXLp0Cbj5s3u98ePH07JlS4e0zPZes2aNkRYfH8/EiRNp2LAhISEhdOvWjR07dmQpKzAw8Ib38d1339GsWTPc3NyMtPDwcGw2G9999122eSwWC7t27XL4bgHatm1LUlISe/fuNcquXbu2Q9CmcePGeHl5sW3bthvWS0RE5O+gIIGI3FOaN2/Ob7/9Rvfu3Vm+fDknTpww5gOHh4fTsWNHh/O/+uor1qxZw4QJE5g8eTLR0dH07NmTlJQUAH799Ve6d+/OpUuXeOutt3j99deNod+ZHZ7z58/z1FNPkZKSwtSpU1m4cCGPPPIIH374IcuWLXO43vz583n00UeZPXs2bdu2BTLmqI8ZM4bu3bsTFRWFp6cnY8eOZfDgwTRv3pz58+dz3333ERERwR9//AFkBCX69OmDn58fM2fOJCoqijp16hAZGWl0kjNNmjSJMmXKMG/ePPr168eqVauIiorKsQ2TkpJ48skn2bVrF88//zyRkZG4u7vzzDPPcOrUKex2O4MHD2bRokV07dqV+fPnEx4ezrvvvptlKP/GjRvx8vIiKiqK/v37G+lz5sxhwIABvP322zRq1ChX7Xy95cuXM3HiRFq1asV7773H9OnTcXNzY+zYsfzxxx8EBQUxceJEACZOnJjtNIO83MumTZv46quvmDBhAjNmzODixYuMGDECq9UKwH333ceKFSvo2rVrjm174sQJvLy8KFGihEP6/fffz/nz50lKSso2X5EiRbhw4QJpaWkO6TExMSQkJBAfH58lz+zZs3FycuLZZ5/NsT43EhcXR0REBD169GDWrFl4eHjQr18/oqOjc8zz6KOPkp6ezhdffOGQvn79eho3bkyxYsXy9Ozmhdlspnfv3nz11VeMHj2ayMhI/P396d+/v0OgYOjQodlOy8iUmprK2bNneeCBBxzSixYtire3N7/++mu2+WJiYkhLS6NChQoO6ZlTSDLznThxIkvZzs7OlC1bNseyRURE/k6abiAi95QePXpw4cIFFi9ebMwdLlKkCI0bN6ZXr16EhoY6nJ+SksKaNWuMN7QVK1akY8eOrF27lieffJLIyEg8PT1ZunQp3t7eADRo0IBWrVqxaNEiIiIiOHbsGFWrVmXWrFnGOQ0bNmT79u3s2rXLmIsMUKdOHfr27Wt8PnToEDabjcGDBxudy6tXrzJ69Gh69+5tnOvj40Pnzp05fPgw/v7+HD16lIYNGzJt2jScnDLivY0aNeLrr79m165dPPLII8Y1mjVrRkREhFH37du3s2XLFsaMGZNtG3766aecPXuWTz/9lKpVqwJQq1YtOnTowA8//MDp06f5/vvvmTFjhnGdRo0a4eHhwaxZs+jVqxeVKlUCwNXVlcmTJxtvZDMXHXz44Ycd1gkYM2bMTdv5ejExMfTr14+hQ4caaWXKlKFTp07s3buXRx55xJhaEBAQkO00g23btuX6XtLT01m8eLFRv6SkJCIiIoiOjiY4OBg3N7ebrneRkJBg5L+Wl5cXAImJicbP1+rUqROfffYZERERjB49Gm9vb9atW8e3334LZDzHfn5+xvmXLl1i7dq19O3bl8KFC9+wTjlJSUnhlVdeoUOHDgDUr1+fVq1asWDBAmbOnJltnjJlylC3bl0+++wz43k+c+YMP/74o5EnL89uXqxbt46jR4/yySefGCMnmjZtSs+ePZk+fboxYqV8+fKUL18+x3ISEhIAcvyeclpgMKd81363medl9x3fqGwREZG/k0YSiMg9Z9SoUXz77be88847dOnSBW9vb/7v//7PWLjwWrVq1XIYwl2tWjXKlSvHDz/8AMDOnTsJCwvDw8OD9PR00tPT8fb2pk6dOnz//fdAxlDhjz76CHd3d44fP85XX31FVFQUcXFxWVZDz+x0X69mzZrGz8WKFQNwGCKe2QG8evUqAB06dGDhwoWkpaVx9OhRNm3axOzZs7FarVneNl/fcfX397/hdIO9e/dStmxZh7p6enqyadMmunbtyu7du3FxcSE8PNwh32OPPQbA7t27jbSKFSs6DNnOdH075Kadrzd+/HjGjh3L1atXOXDgAOvWrWP58uUAuVqFPrOuub2XgIAAhw5gyZIlAYxRJ7lxs60YMzvN12vUqBHTpk3j+++/p1WrVtSvX5+tW7cyYMAAADw8PBzOX7lyJTabjd69e+e6btdzcXGhffv2xmcPDw+aNm1q/G5YrVbju0pPTzemCjz22GP88MMPXLhwAcgYReDt7W1MF8jLs5sXO3bsoESJEgQFBRl1slqttGjRgsOHD2eZApGTnKY8ZDKZTLeUL/O7vdEzkFPZIiIifyeNJBCRe5Kvry/t27c3OjlHjhzh+eefZ9q0aTz66KMUKVIE+Kujd61ixYoZHYr4+Hg2bNjAhg0bspyXOafYZrMxY8YMli9fTnJyMqVKlSI0NBR3d/cseQoVKpRtfbN7a+np6Znj/aWmpvLqq6+ybt060tPTKVu2LDVr1sTFxSVLJ+T6cpycnG7YUYmPjzcCFdm5cuUKRYoUwdnZ2SE9cwh95htVINs3ppC1HXLTztc7c+YMEydOZMeOHbi6ulKxYkWqVKkC3Lwzfiv3kl07ws07h9fy9vbOdkpB5htkHx+fHPM+9thjPPLII8TExODp6UnJkiWZNWsWTk5OWUYLbNq0iUaNGt3WYoXFixd3WPwTMn43Mqc2tG7d2mH9jo4dOzJ16lTCw8N59dVX2bhxI7169WL9+vW0bdvWCGTk5dnNi/j4eC5cuEBQUFC2xy9cuICvr+9Ny7l2pMj1EhMTc/yOMtOvz5f53WaWe6NnILt/j0RERP5uChKIyD3j3LlzdO7cmVGjRmWZF16tWjVGjx5tLIaXGSS4fPlylnIuXrxoDEf28fGhYcOGDlMEMmV2oBYsWMDSpUuZPHkybdq0MToLXbp0ydf7u9brr7/Opk2bePfdd2nYsKHR6W7QoMFtl+3j42NMC7jWvn378PX1xdfXl8uXL2O1Wh061+fPnwcw2jav17xZO1/LZrMxcOBAXF1dWbVqFVWrVsXFxYXjx4+zbt26XF+3IO7lRipWrEhiYiJxcXEOHfjTp09TpkyZLCMCMp04cYJDhw7RoUMHhznvR44cITAw0KHu586d48iRI7c1igDIdp2DixcvGgGkqKgohxEbmW3l4+NDy5Yt2bhxI/Xr1+eXX37h5ZdfNs67lWfXZDIZaz9kun40jI+PDxUqVGD69OnZllG2bNkb3O1fvLy8KFmyJKdPn3ZIv3TpEklJSTz44IPZ5itfvjzOzs5Z8p05cwbAyPfAAw8YaZmsViuxsbG0adMmV3UUEREpSJpuICL3jMw3n//5z38wm81Zjp88eRJ3d3eHvej37t3rECg4fPgwsbGxRoclLCyM48ePU7VqVUJCQggJCSE4OJilS5eyefNmo4yAgAA6d+5sBAjOnTvHsWPH8vSWOS/27t1LvXr1aNWqldHJOnz4MHFxcbd9zTp16hATE8Mvv/xipJnNZkaMGMGqVasICwsjPT2dzz//3CHf//73P4Bb2sItN+18rcuXL/Prr7/SpUsXQkJCjEBC5urwmW1w/QiB7K6b3/dyIw0bNgRwuJ7FYmHLli00atQox3y//PILERERnDx50kg7fvw43333Ha1atXI49+DBg0DGVJrbkZqaaqx5kPl527Ztxu9GYGCg8V2FhIQ4dMIff/xxDhw4wMcff0zp0qUJCwszjt3Ks+vl5cXly5cdfq8zdwvIFBYWxu+//06xYsUc6rV9+3YWLVp002fhWo0aNWLLli0OQZBNmzbh7OxM/fr1s83j7u5OnTp12Lx5s8OIiE2bNuHj42Osh9KoUSN++OEH4uLijHO+++47kpOTb/gMiIiI/F00kkBE7hnOzs688sorDBs2jM6dO/PUU0/x4IMPkpKSwvbt21m+fDmjRo1yGHKckpJC//79GTJkCElJScycOZPKlSsb0xSGDh1K9+7dGTRoEE8++STu7u6sWLGCL7/8ktmzZwMQGhrKvHnzWLBgATVq1OD06dO89957WCyWPM1Xz4vQ0FA2btzIxx9/zIMPPsjRo0eJiorCZDLd9jU7derEhx9+yJAhQxg5ciRFihRh2bJlpKWl0aNHD8qWLUu9evWYMGEC586do0qVKuzevZuFCxfSsWPHbBcIvJnctPO1ihUrRpkyZVi+fDn+/v4ULlyYb7/91lhzIrMNMoM2W7ZswdfX15iOkKlp06b5di8Wi4UjR47g7++Pv79/tueUKVOGjh078uabb2I2m6lQoQJLlizh6tWrDrs/nDlzhri4OGM9iWbNmlG+fHnGjh3LqFGjSExM5O2336Zs2bL06dPH4RrHjh3Dzc0tx8X54uLiOHPmTJY1FrLzwgsv8Oyzz1KsWDEWL15McnIyQ4YMuWlbNGnSBD8/P1asWEH//v0d5trfyrPbokULPvzwQ1566SW6dOnCsWPHWLJkiUPHv1OnTnz00Uf07duXwYMHU6pUKb7//nsWLlzI008/jaurK5C1bbPTv39/1q9fT//+/enbty+nTp1ixowZdOvWjdKlSwPZf99Dhgyhb9++jBo1is6dO7N//34WL15sLMwJGYurZtZz+PDhxMfHM23aNJo2bXrbgR0REZH8oCCBiNxTmjdvzieffMLixYuZP38+cXFxuLm5Ua1aNWbOnJllOG+dOnWoX78+L730EgAtW7Zk3LhxxmJ7VapUYfny5cycOZNx48Zht9upXLkyc+fO5aGHHgJg0KBBXL58mWXLljF37lxKlSrF448/jslk4r333uPq1au3vMJ8TsaPH09aWhrvvvsuFouFsmXLMmTIEI4fP87XX3+dZWh2Xnh7e/PRRx/x9ttv8+qrr2Kz2ahRowbLli0zFnl87733mD17NkuXLiUuLo6yZcvy3HPPZTtdIDdy087XmzdvHq+//jrjx4/Hzc2NgIAAoqKieOONN9izZw89e/akUqVKtG/fnuXLl/Ptt9/y2WefOZSR+R3lx72cP3+eJ554guHDhzNixIgcz5syZQqFCxdm4cKFJCcnExQUxJIlSxxGuMybN49PP/2Un3/+GchYD2HRokW88cYbjBkzBnd3d5o2bWrsdHCtixcv3vB527JlCy+88ALLli2jXr16N7ynV155hTfeeIO4uDhq1arFxx9/7FDPnLi4uBjbgGYuApnpVp7dRo0aERERwYcffsimTZsICgoiMjKS7t27G+cUKlSI5cuX88477zBt2jQSEhIoU6YMY8aM4ZlnnjHOu75ts/Pggw/y/vvv8/bbbxuBsj59+jBy5EjjnOy+7wYNGjBnzhxmz57NsGHDKFmyJOPGjXO4ftGiRVm2bBlvvPEGY8eOxcvLi/DwcMaNG3fTdhUREfk7mOy3s0qQiMg/WM+ePQH48MMP73BNRP5e48aNo0ePHjm+TZ8zZw6RkZE37EiLiIjIvUlrEoiIiPyLHD9+nIMHD1K5cuU7XRURERG5CylIICIi8i9StGhRli5dmuN2nCIiIvLvpukGIiIiIiIiIgJoJIGIiIiIiIiI/ElBAhEREREREREBtAViruzfvx+73W7ssSwiIiIiIv9OaWlpmEwmataseaerIlIgNJIgF+x2O3d66Qa73Y7FYrnj9bhXqX0Lltq3YKl9C47atmCpfQuW2rdgqX0Lzt3etndD30CkIGkkQS5kjiAICQm5Y3VITk4mOjqagIAArUhdANS+BUvtW7DUvgVHbVuw1L4FS+1bsNS+Bedub9tDhw7d6SqIFCiNJBARERERERERQEECEREREREREfmTggQiIiIiIiIiAihIICIiIiIiIiJ/UpBARERERERERAAFCURERERERETkTwoSiIiIiIiIiAigIIGIiIiIiIiI/ElBAhEREREREREBFCQQERERERERkT8pSCAiIiIiIiIigIIEIiIiIiIiIvInBQlEREREREREBFCQQERERERERET+pCCBiIiIiIiIiAAKEoiIiIiIiIjInxQkEBERERERERFAQQIRERERERER+ZOCBCIiIiIiIiICKEggIiIiIiIiIn9SkEBEREREREREAAUJRERERERERORPChKIiIiIiIiICKAggYiIiIiIiIj8SUECEREREREREQEUJBARERERERGRPylIICIiIiIiIiKAggQiIiIiIiIi8icFCUREREREREQEUJBARERERERERP6kIIGIiIiIiIiIAAoSiIiIiIjIPcxut/+rr/93+jfd6+34O9vpVq6lIIGIiIiIiPwjtGzZkvHjx+f6/K+++oqIiAiHtPXr19OiRQuCg4OZOHEiPXv2pGfPnvldVf744w8GDhzI2bNnsxwbMmQIUVFRuS7r9OnTBAYGZvnTvn37/KyyYc2aNQQGBhIbG5ur869evcq4cePYs2dPvlw/MDCQOXPm5Hg8NjY22/bI/PPCCy8AMGfOHAIDA2+pDgXxXNzomSgIe/fuZeDAgXnO51IAdREREREREck3drsdux0iIyPx9vbOdb6lS5dmSZsyZQoVKlRg6tSplCxZkpdffjkfa/qX77//nq1bt2ZJt1gs7Ny5kxEjRuS6rOjoaCDjfjw9PY10Dw+P269oNpo3b86KFSu47777cl2/devW0blz5wKpz/Xuu+8+VqxYkSV9+fLlbNy4MV/qMWnSpNsu43o5PRMFZeXKlZw4cSLP+RQkEBERERGRu5bNZic+wYyPlyvVqlW77fLi4+Np1KgR9erVy4fa5d2ePXvw8vKiatWquc4THR2Nv78/DRo0KMCa/aVo0aIULVr0b7nWrXBzc6NGjRoOaYcPH2bjxo2MHj2aOnXq3PY1AgICbruMfypNNxARERERkbuSzWbncoIZc7oV+Gu6QeZw840bNzJy5Ehq1qxJWFgYEyZMIDk5GcgYLr579252795NYGAgu3btMoaez507N8fh9Gazmblz5xIeHk5ISAht2rRhwYIF2Gw24xyr1cqCBQto3749oaGh1KhRg+7du7Nz504gY7h+5pD3hx56yGGKxNatW2nSpAkmkwnICFpMnDiRhg0bEhISQrdu3dixY4dDnY4ePZqnoEKmNWvWEBISwp49e+jcuTMhISG0bduWr7/+mpMnT9K7d2+qV69O69atWb9+vUO+a9snLi6OMWPG0KhRI0JCQnj88cdZu3YtALt27aJXr14A9OrVyxii37NnT8aOHcvIkSOpUaMGffv2BTKmCowbN47GjRsTFBREgwYNGDduHJcvX87z/WWy2+1MmTKFBx98kD59+mQ5/uWXX9K2bVtCQkLo2rVrlvbNzvXTDQIDA1m+fDkvvfQSYWFh1KxZk1GjRnHx4kXjnDNnzjB48GDq1atH9erVeeKJJ4yRAzk9Ey1btuSNN96gd+/ehIaG8tJLL+U43eP66TYWi4V3332Xhx56iNDQUNq3b8+nn34KwPjx4/n00085e/YsgYGBrFmzJpetqSCBiIiIiIjchex2uJyQSqolPeNDNiZNmkSZMmWYN28e/fr1Y9WqVcZc/0mTJlGtWjWqVavGihUrCAoKMoaod+nSJdvh9Ha7ncGDB7No0SK6du3K/PnzCQ8P591333UYfr58+XLmzZvHE088waJFi3j11VeJj49n1KhRpKSk0Lx5c4YMGQJkTJEYOnSokXfr1q00a9YMyAhI9O7dm6+++orRo0cTGRmJv78//fv3d+jIRkdHk5SURPfu3QkJCaFRo0ZMnz6dtLS0m7Zjeno6Y8aMoXv37kRFReHp6cnYsWMZPHgwzZs3Z/78+dx3331ERETwxx9/ZFvG888/z4kTJ5g8eTILFy6kWrVqREREsHPnToKCgpg4cSIAEydOdGinjRs34uXlRVRUFP379yclJYVevXpx4sQJJk2axOLFi+nVqxfr169n5syZN72XnGzYsIGDBw/y4osv4uzsnOX4Sy+9RK9evZgzZw5eXl4MGDCAQ4cO5fk6M2fOxGazMWPGDMaNG8c333zDG2+8AYDNZmPQoEGkpKTw9ttvM2/ePPz8/BgyZAinT5++4TOxfPlyQkJCmDdvHl26dMl1fcaOHcuSJUvo2rUr7733Ho0bN2b8+PF89tlnDB06lGbNmlGiRAlWrFhB8+bNc12uphuIiIiIiMhdxcXFlSuJZuxOrjc8r1mzZsbChA0aNGD79u1s2bKFMWPGEBAQYKxfkDk0PfNvf3//LMPVAbZt28b333/PjBkzeOSRRwBo1KgRHh4ezJo1y3hjHhcXx+jRox3eNLu7uzNixAh+/vlnatSoQfny5QGoWrUqZcuWBSAmJoaYmBgaNWoEwLp16zh69CiffPIJ1atXB6Bp06b07NmT6dOns3r1auLi4jh37hxWq5Xnn3+e0qVLs2PHDhYuXMjvv//OO++8c8M2stlsDB48mK5duwIZiwyOHj2a3r17G2/3fXx86Ny5M4cPH8bf3z9LGbt372bYsGG0atUKgLCwMPz8/HBzc8Pb29sYmh8QEOAwTN/V1ZXJkyfj5uYG/DVt4q233qJcuXIA1K9fn4MHD7J79+4b3seNLF68mFq1auU4hWTy5MmEh4cDGc/JQw89xMKFC5k9e3aerlO5cmXefPNN4/OPP/7I559/DsClS5c4efKk0TkHCA0NJTIyEovFQtGiRbN9JgBKly7N2LFjjc+//vrrTety7NgxNm3axIsvvkjv3r2Nezt79iy7du2iffv2FC1aNNupGTejIIGIiIiIiNw17HZIMtvwMqfj6emKzWbn7MUkzsWlkJZuc9jS7frOj7+//22tHL97925cXFyMDmWmxx57jFmzZrF7925CQ0N59tlnCQkJIS4ujpMnT3L69Gm++eYbIGMIeE62bdtGzZo18fHxAWDHjh2UKFGCoKAg0tPTjfNatGjB22+/zZUrVyhUqBDvv/8+999/v9GxDAsLw83NjXfffZehQ4fywAMPOEyHAHBx+aurV7NmTePnYsWKARhBCQA/Pz8gI4CQnXr16jFnzhyOHDlCkyZNHIIzN1KxYkUjQAAZneP//Oc/2Gw2Tp06xenTpzl+/DgnT550uP9rWa1Wh+/cyckJJ6e/BsTv27ePn376iblz52ab39XVlTZt2hif3d3dadq0qfF93az8a2X3vKWkpABQvHhxAgICePnll/nuu+9o3LgxTZs2NaYY3MitTCXZu3cvgMO9ATfcFSK3FCQQEREREZG7gtVmJz7RTGKymfuAo6fi+HznKZJS0vBwdeZKopkDxy4QfSoOwGGlf8jo4N3OHvRXrlyhSJEiWYaslyhRAoCEhAQAjh8/zuTJkzl06BCenp4EBARQunRp4Mb70m/dupWmTZsan+Pj47lw4QJBQUHZnn/hwgUCAgKMkQfXat68Oe+++y5Hjx5lw4YNREZGOhz/+eefjZ+z2xHi+ra7kZkzZzJ//nw2btzIpk2bcHJyomHDhkyZMoUyZcrkmM/LyytL2pIlS5g/fz7x8fEUL16c4OBgPD09jba9XuvWrR0CPx07dmTq1KnG502bNuHr62u8vb9ekSJFsnT6ixUrZgREblb+tW70vJlMJt5//32ioqLYvHkza9euxdXVlVatWjF58mR8fX2zLROgUKFCOR7LSXx8vHEv+U1BAhERERERueOsVhuXE8ykmNOx2+38EnOFld+cJNWcTsmihSjs7YbJBIkpaXy4IbpA6uDr68vly5exWq0OgYLz588DGR3O5ORkXn/9dapVq8b69eupWLEiTk5ObN26lU2bNuVYttlsZteuXYwZM8ZI8/HxoUKFCkyfPj3bPGXLluXUqVPs3LmTdu3aUbhwYeNYamoqkLETQZ06dfI05zyvfHx8eP7553n++ec5efIkX331FfPmzWPy5MksWLAg1+X83//9H1OnTuX555+nU6dOxg4Ko0aNynGNgKioKIfRGUWKFHE4vmXLFh566CFcXbOfmpKQkIDdbjcWigS4ePGice2blZ8XJUuW5JVXXmHSpEkcPXqUzz//nIULF1KkSJE8bamYWdfrR4ckJSUZP2c+C3FxcQ5TRE6cOEF8fDy1a9e+5fvQwoUiIiIiInJHZQYIzGkZuxjY7Xa+/CGWVHM6ft5uuLo44WQyYcKEu6tTxmKG3PitPZDjsPGchIWFkZ6ebswzz/S///0PgNq1a3P27FkSEhLo1asXAQEBxjW2bdsG/NWxu/7au3btws/Pz9hhIfN6v//+O8WKFSMkJMT4s337dhYtWoSzszMXLlxg0qRJWeq0YcMGvL29CQoKomTJkg75Q0JC8nTfN3L27FmaNWtmXL9ixYoMGDCAhg0b8ttvvwFku1hgdvbu3UvhwoXp37+/0UlPSkpi7969WTrEmQIDAx3u69q5/PHx8Zw6dYpatWrleM2UlBRj14nM623ZssVYv+BG5efF/v37adiwIT/++CMmk4mqVasyevRoKleubLRTbp/HzJEf1y4kmdn5z5QZBPj6668d8k6fPp3XX389T9e7nkYSiIiIiIjIHWO12oi7mool/a9O4oWr6Zy/nIKXp6vDG+AMJrw8Mt4an7+ccsOyCxcuzP79+9mxYwfVqlW74ZBvyFg0sF69ekyYMIFz585RpUoVdu/ezcKFC+nYsSMBAQFcvHiRQoUKMX/+fFxcXHBxcWHTpk2sWrUKwJijnvmmd/PmzTRt2pRt27Y5TDUA6NSpEx999BF9+/Zl8ODBlCpViu+//56FCxfy9NNP4+rqSu3atWnQoAFTp04lNTWVgIAAtmzZwocffsj48eMdRhcUhDJlyuDv789rr71GYmIi5cuX5/Dhw2zdupVBgwYBGGssbNmyBV9fX6pUqZJtWaGhoXz88cdMnTqVFi1acP78eRYvXszFixdv+t1k59ixYwAOiyVez9XVlRdffJHnnnsOb29vFixYQGpqqsPuAvmhWrVqeHh4MG7cOEaMGEHx4sX5/vvviY6ONha8vP6ZePDBB7Mtq169enh4eDB16lRGjRpFUlISs2fPNtaOAKhSpQrh4eFMmzaN1NRUqlatyrZt2/jmm2+MqSeFCxfm4sWLbN26lapVq2bZzSMnChKIiIiIiMgdkW61cfm6AAFAqtmO1WbHxfn6AEEGFxfTn+dlv9hdpqeeeorDhw8zYMAA3nzzTR599NEbnm8ymXjvvfeYPXs2S5cuJS4ujrJly/Lcc88ZOwF4eXkxbtw4Vq1axahRo/Dy8qJq1ap89NFHDBgwgD179tCyZUvq1atHw4YNeeedd9ixYwenTp1i3LhxDtcrVKgQy5cv55133mHatGkkJCRQpkwZxowZwzPPPANkvA2OjIwkMjKSpUuXcuHCBcqXL8+rr75q7FhQ0CIjI5kxYwazZs3i8uXLlCpViuHDhzNw4EAAKlWqRPv27Vm+fDnffvstn332WbbldOzYkdjYWFavXs1//vMfSpYsSbNmzejRowcvv/wyJ06cyLHjnJ2LFy8C3DBQUrRoUcaMGcOMGTO4cOEC1atX56OPPqJixYp5aIGbc3d35/333+edd97h9ddf5+rVq1SoUIEpU6bQqVMngCzPRE5TNQoXLsycOXN45513GDZsGGXKlGH48OGsXbvW4bxp06YRGRnJBx98wOXLl3nwwQeZPXu2sQtFp06d2Lp1K8OGDWPkyJHG93UzJvvtrOzxL5E5PyY/h+3kVXJyMtHR0VStWvWWFraQG1P7Fiy1b8FS+xYctW3BUvsWLLVvwVL73r6cAgQpKSns2HeUL/Yn4uHugpuLE16ernh7unI5wQyA2WIl1WLlxT5hBJTz+1vrfTf0DUQKktYkEBERERGRv1VaevYBgkwlCrtwXxFPklLTsqw7YLfbSUhJo8x93lQsk/ch6iJyYwoSiIiIiIjI3yYt3UZ8Qs4BAsgY9t+qblk83FyIT7KQlmbDZrdjtli5dNVMIXcXurSshJNT9tMRROTWKUggIiIiIiJ/i7R06w1HEFyrUjlfnmpbhTIlvDGnW7maaCHVYqVCqcIM7VKd6pVK5Eud7uXZ1/fyvUnBUZBAREREREQKnDnNStzVVNKsNw8QZKpSoSgju9WkzyNB9HokiBf7hNH7oZL877/zadu2LdWrV6d27dp0796d//znP6Sn33ghwzVr1hAYGEhsbCwAv/zyC08++aTDOS1btmT8+PE3LCcyMpLAwECaNm2aY0d8+vTpBAYG0rNnzyzHhgwZQlRUlEPa559/zsCBA2nSpAnBwcE0btyYUaNG8eOPP96wLpl69uzpcK2VK1fy1ltvGZ937dpFYGAgu3btumE5gYGBBAYGMmPGjGyP22w2mjRpQmBgIGvWrMmxnNjYWKOs7P688MILwF/fybV/QkNDefjhh5k/fz5WqzVX9y/5R7sbiIiIiIhIgTJb0rmcYMZqy/ubbScnE+VKelPcz5PNX2zihRde4MEHH6Rv37488MADpKamsnXrVt544w2+/fZb5s2bl822iRmaN2/OihUrjK3gPv/8c/bv339L9+Tk5MS5c+fYt2+fsWf9tTZs2JBtPovFws6dOxkxYgQA6enpjBkzhs2bN/PYY4/x8ssvU6RIEX777Tc++eQTunfvzvTp02nXrt0N6zNp0iSHz1FRUYSFhd3yvX3++ec899xzWY798MMPnD9//qZl3HfffaxYsSJL+vLly9m4cSOdO3d2SI+MjKREiRLY7XZSUlLYt28fs2fPJjU1lWefffaW7kNujYIEIiIiIiJSYFLMaVxJtNxSgOBaJ0+e5IUXXqBJkya8++67uLj81ZVp1qwZ9erVY+TIkWzcuDHHDnXRokUpWrTobdUjU6lSpbDb7WzcuDFLkODAgQOcO3eOypUrZ8m3Z88eY9tEgPnz5/P5558ze/Zs2rZt63Duo48+yrBhw5g8eTItW7bEw8Mjx/oEBATkw11lqFWrFnv27OHIkSNUq1bN4dj69eupWrUq0dHRNyzDzc2NGjVqOKQdPnyYjRs3Mnr0aOrUqeNwrGrVqpQtW9b43LBhQ2JiYvjvf/+rIMHfTNMNRERERESkQCSlphGfcPsBAoD331+Mk5MTkydPdggQZGrbti0dOnQwPgcGBhIZGUmnTp0IDQ0lMjLSYbrBnDlziIyMNM6dM2dOnusUHh7OF198kWXKwYYNG2jYsCF+fn5Z8mzdupUmTZpgMplISUlh8eLFhIeHZwkQQMYb/WeffZZ69epx6dIlIGN4frVq1Vi5ciWNGjUiLCyM48ePO0w3aNmyJWfPnuXTTz91mF6RW3Xr1qV48eJ8/vnnDunp6el88cUXPPLII3kqDzLWR5gyZQoPPvggffr0yVWewoUL5zgqRAqOggQiIiIiIpLvEpMtXE00Y8unxfO+/uor6tevT7FixXI856233nIYRTB//nweffTRbN/Sd+3alS5dugCwYsUKunbtmuc6tWvXzphykMlms/H555/n2JHeunUrzZo1A+D7778nOTmZ9u3b53iNwMBAZs+eTZkyZYw0q9XK+++/z+uvv25Mv7hW5tD9Zs2aOUyvyC1nZ2fatm2bJUiwY8cOzGYzLVu2zFN5kBE4OXjwIC+++CLOzs5ZjttsNtLT00lPTycxMZFt27axbt06nnrqqTxfS26PphuIiIiIiEi+SkiykJBiIb8W10+4epWrV69SoUKFLMeuX6zQZDIZndA6derQt29f49ihQ4eMn/39/fH39wfIMiw+t0JCQihXrpzDlIM9e/YQHx9Pq1atWL16tcP5MTExxMTE0KhRI+MzkOW+bDYbNpvjAo9OTk44Of31jnfw4ME0b94823pVq1YNNzc3ihYtesv31q5dO5YvX+4w5WDDhg089NBDuLu757m8xYsXU6tWLerVq5ft8datW2dJCwkJoXfv3nm+ltwejSQQEREREZF8YbfbuZJoztcAQVJKGt8dzH64/OnTpwkKCnL4c21nM3Pef0Fq166dw5SD9evX07x5c7y9vbOcu23bNmrWrImPjw9AlkBAplmzZmW5r7lz5zqcU9D3Vrt2bUqWLGmMJrBYLHz55ZfZjnqwWq3GKID09PQs97Vv3z5++ukn+vXrl+P1oqKiWLVqFatWrWL58uW88sornD9/nu7du5OYmJi/Nyc3pJEEIiIiIiJy2+x2O/GJZlJS08mP+IDdZmfPkT/47+ZjJKak4eTsxqnTMQ7nlCpVilWrVhmf586dy7Fjx4zPhQoVyoea3Fi7du1477332LdvHzVq1OCLL77glVdeyfbcrVu30rRpU+Nz6dKlATh79iyVKlUy0nv06EGrVq2Mz5nTIq5V0PdmMpkIDw83djn49ttvcXJyolGjRpw7d87h3NatW3P27Fnjc8eOHZk6darxedOmTfj6+hrTLLJTuXJlh4UL69SpQ+XKlenRowcrV650GBEiBUtBAhERERERuS02m53LCWbMlvwJEMRetPDprh85eyHJSPMqWY3vv99OYmKi8Zbezc2NkJAQ45zsFgosaFWqVOGBBx7g888/JzU1FbPZnO00ALPZzK5duxgzZoyR1qhRI9zd3fn8888d8pQsWZKSJUv+DbW/sXbt2vHBBx8QHR3Nhg0baNOmDa6urlnOi4qKwmKxGJ+LFCnicHzLli089NBD2ea9kczv9tSpU3mvvNwyTTcQEREREZFbZrXauHw1ldR8ChD8EnOFTfviHQIEAA/WDMdiSWP4s887dEgzpaamGnP8c+vaOf63I3PKwYYNG2jdunW2c/Z37dqFn58fgYGBRpqPjw99+/Zl7dq1bN68Oduyrx0ZkRf5cW81atSgTJkyrFu3jq+//jrHxRgDAwMJCQkx/lw7IiA+Pp5Tp05Rq1atPF//xx9/BLKu2SAFSyMJRERERETklqT/GSCwpGc/tz6vrFYb//3yOCmWv8INJqDMfV74Fy1B4pWn+WHXx3Ts2ImuXbsQGBhIeno6+/fvZ9WqVVy8eJH+/fvn+nqFCxcG4LPPPqN69eqUK1cOgOPHj7N06dIs59eqVSvbLfnatWvH3LlzWbduHfPmzcv2Wtu2bXOYapBp5MiR/PHHH4wYMYLw8HBat27Nfffdx4ULF/jmm2/YuHEjJUuWpEGDBrm+r8x7O3LkCLt37yY0NNRI37RpE9HR0VnO79q1K15eXlnSw8PDWbZsGX5+foSFheWpDvBXkCMgIOCG50VHR3Px4kUgY62GEydOMGfOHEqUKEHHjh3zfF25dQoSiIiIiIhInqWlW7l81UyaNX8CBGnpNuavOUjcVbOR5uRkooSfBz6F3DCZTNxfuS6+xSvwgPsJVq1axdmzZ7Hb7ZQrV4527drRvXv3PL11btOmDevWrWP8+PF06dLFWEvg0KFDDjshZBo1ahRNmjTJkh4QEEDlypW5cOECDRs2zPZa27ZtY9y4cVnSnZ2deeutt2jfvj0rV65k2rRpXLx4ES8vL6pWrcpLL71Ehw4d8PT0zPV9ATzzzDO88cYb9OvXjyVLlhjpy5cvz/b88PDwbIME7dq1Y/HixTz88MO3NDohs+OfGZDJyfDhw42fXVxcKFKkCPXq1WPUqFF3ZBrJv5nJbs+vdUfvXZn/QFw73+nvlpycTHR0NFWrVv1bFmD5t1H7Fiy1b8FS+xYctW3BUvsWLLVvwfq3t685zUp8Qirp1vzpSiSlpDF/zY/8EhNvpLk4m7iviCcuzk54ebri7enKpSupXE4w82z3WtSqcl++XDuv7oa+gUhB0kgCERERERHJtVRLOvEJZqy2/AkQXIhPIfKTA5yLSzbSXJwwAgTXSku34eLsRGEvt3y5tohkpSCBiIiIiIjkSoo5jSuJlnwLEJw8e4Wo1QdJSE4z0goXcsWSlobTdVP/7XY7CSlpVChVmIplfPPl+iKSlYIEIiIiIiJyU0kpFq4mpWHLp9nK+46eZ8lnP5F2zaKHjzZ+gLIlPPnPF0eJT0rD28MVFxcTaWk24tMtFHJ3oUvLSjhdH0EQkXyjIIGIiIiIiNxQQpKFxBQL+TGAwG6389UPMaz++hdjy0RnJxM921WlfnApUlJSaBHiy5GzNi7Ep5JstmHyNlGxtC+t691P9Uolbr8SIpIjBQlERERERCRHVxLNJKWmkR8DCKw2G598+Qtb98UaaYXcXRjUKZTA+4sYaWWLu9G0bkUuXE0nMTkNPx93qlcqjrubui8iBU2/ZSIiIiIikoXdbudKkoXkfAoQpFrSWbzuJw6duGikFfP1YHjXGpQqnnXrPZOTiftLZWyb52RCUwxE/iYKEoiIiIiIiAO73U58opmU1HTyYwWC+AQzc1cdJOZcgpF2f6nCDOsSSmEv93y4gojkFwUJRERERETEYLfbuZxgJtWcPwGCs+cTiVx1gMtXzUZa9Uol6PdYEG6uzvlwBRHJTwoSiIiIiIgIADbbnyMIzOn5Ul70qTje+/RHUs1WI61lnXLaoUDkLqYggYiIiIiIYLP9OYLAkj8Bgu0Hf2P5pqPY/twSwWSCbg9VpkWdcvlSvogUDAUJRERERET+5TICBKmkWqw3P/km7HY7//v2JBu/P2Wkubk60e+xYG1fKPIPoCCBiIiIiMi/WH4GCNLSbSzbcIQfjpwz0gp7uTGsa3Xu9y982+WLSMFTkEBERERE5F/KarMTn08BgsSUNOav/pHjsfFGWuniXgzrWp1ivp63Xb6I/D0UJBARERER+Rey2uzEX00lNe32AwQXLicTufIg5+KSjbQq9xdhYMcQCnm43nb5IvL3UZBARERERORfJj8DBCfPXmHeqoMkpqQZaQ1CSvFUeBVcnJ1uu3wR+XspSCAiIiIi8i9itdm5fDUVcz4ECPYdPc+Sz34iLd1mpD3WpCIPN6yAyaQtDkX+iRQkEBERERH5l7BabVxOMN92gMBut/Pl7jOs+eY49j/TnJ1M9GpXlXrBpW6/oiJyxyhIICIiIiLyL5BfAQKrzcaKzcfYtv+skVbI3YXBnUOpXL7I7VZTRO4wBQlERERERO5x+RUgSLWks3jdYQ6duGSkFff1YFjXGpQq7nW71RSRu4CCBCIiIiIi9zCr1Ubc1VQs16wbcCviE8zMXXWQmHMJRlqFUoUZ2qU6hb3cbreaN+TkZMJJaxyI/C0UJBARERERuUelW21czocAwdnziUSuPMDlBLORVqNyCZ55NAg3V+fbreYNubk44efjjrN2ShD5WyhIICIiIiJyD8qvAMGRXy+x4NNDpFr+mqrQqm55OrUIwMmp4N7um0wmPN1d8PV2x7kAryMijhQkEBERERG5x+RXgGD7wd9Y/vlRbPaMPQxMJujWqjItapfLj2rmyNXVhcJebhQt7FGg1xGRrBQkEBERERG5h6Sl24hPuL0Agc1u53/bTvL5jlNGmpurE/0fCya0Uol8qGXOXFyc8PV2w8tDXRWRO0G/eSIiIiIi94j8CBCkpdtYtuEIPxw5Z6QV9nJjWNfq3O9fOD+qmSM3Fyd8PNy5fO72dmEQkVunIIGIiIiIyD0gPwIEiSlpzF/9I8dj44200sW9GNa1OsV8PfOhltkzAe5uzvh5u2M2p2Kz3d40CRG5dQoSiIiIiIj8w6WlZ6xBkGa99c71hcvJzPnkAOcvpxhpgfcXYVDHEAp5uOZHNbNlMoGXh2uBb6N4K+x2OyZtvSj56O9+pm7letpHRERERETkHywt3ZptgODJLu156/VJuSrj5NkrvDB1KQe/XmKkNQgpRXCx8zzzVAfatqjPjLdfZ/TwgYwePjDf6u7sZMLXy42UxMsMGjSIs2fPZjlnyJAhREVF5ds1sxMbG0tgYCBr1qwB4OrVq4wbN449e/YY5/Ts2ZOePXsCcP78eYfz16xZQ2BgILGxsQVaz9txbf1v18qVKwkMDMzyZ8qUKX9bHQrS9c9Dfvnqq6+IiIjI1zJvZN68eSxevDjP+TSSQERERETkH8qcZuVyQipWqz3LsSlvTKeQl9dNy9h39DxLPvuJ36O/MdIebVKRdg0r0LH9cMqWLc/4lyZTvMR9vPPWa/lWd1fnjAUK3d1c2LTxe7Zu3ZrlHIvFws6dOxkxYkS+XTc3oqOjWbduHZ07dzbSJk3KCLikpKRkOb958+asWLGC++6772+r450UHR3NAw88wNSpUx3SixcvfodqlL/uu+8+VqxYQfny5fO13KVLl+ZreTcza9Yshg8fnud8ChKIiIiIiPwDmS3pXE4wY7VlDRAAVKpc5Yb57XY7X+4+w5pvjpNZgskEfdtXo15wKQCuXrlCnU71qVGrTn5WHQ9XZ3x93HFxvvHA5j179uDl5UXVqlXz9fq3IiAgAIBDhw5lOVa0aFGKFi36d1fpjomOjiYkJIQaNWrc6aoUCDc3t3v23nJD0w1ERERERP5hUsxpNwwQwF/TDf74/TdaNq7Nlq8388qEcTzSugmPP9yCoSPHsnLzEexAzPfzSYk7SfKlk7wwuD0H9u2hZePaACxbspCWjWvzx++/ZbmGxWzmw6UL6d2jE21bNqBn9w58/NFSh4UHrVYrH3+0lGd6diO8ZUPatWrE0EF92PPDbiBjqP4LL7wAwEMPPcTEiRONvFu3bqVJkyaYTCZ27dpFYGAgO3bsoGfPnoSGhtK8eXNWrlzJ+fPnGT58ODVr1qRZs2YOb2xzmgrQsmVLxo8fn+Wedu3aRa9evQDo1auXMTz+RkPlr7/G+PHj6dOnD6tXr6Zt27YEBwfz+OOPs23bNod8+/fv56mnnqJGjRo0b96cDz74gD59+hj1yrznXbt2OeS7vi6pqam88847tGnThuDgYGrVqkXfvn2Jjo7Otr43cuzYMQYNGkStWrWoVasWw4YNIyYmxjhut9v5+eef8yVwExcXx+TJk2nRogXBwcGEhYUxbNgwh++qZ8+ejB07lpEjR1KjRg369u0LZEz5GD16NGFhYdStW5eJEycyc+ZMWrZs6XCNlStX8sgjjxAcHEzz5s2ZM2cOVuuNd8+4frrBmjVrqFatGgcPHuSJJ54gJCSEFi1aZBnK/9lnn/HYY48RGhpK/fr1GTt2LOfOnTPuY/fu3ezevdv4TjO/3//+97+0aNGCWrVqsX379myfteyehZMnTzJ8+HCjDQYNGsSJEycACAwMBCAyMtL4ObcUJBARERER+QdJTk0jPsFywwBBdmZOe4OS/qV4+dW3KR/Sip/3b+XSL18BULnhE1SoWImAyoFEzl9KpcAqRM5fCkC79o8TOX8pRYs5DiW32+28FDGa/y5fRrv2HXj9rZk0a9GKxQvnMXPaG8Z5C+fP4cOlC3ns8c5Ezovitdde48qVK4waNYqUlBSaN2/OkCFDgIwOzYABA4y8W7dupVmzZg7Xfe6552jZsiXvvfceDzzwAJMmTaJXr15UqlSJefPmERoayptvvsmPP/6Yp/bJFBQUZAQqJk6caEwzyKvDhw+zePFiRo4cydy5c3F2dmbEiBFcuXIFgBMnTtCnTx8AZsyYwYgRI1iwYAF79+7N87XGjRvH6tWrGThwIO+//z4vvPACv/zyC2PGjMFuz/1z8uuvv9K9e3cuXbrEW2+9xeuvv05MTAxPPvkkly5dAuDMmTMkJSVx6NAh2rZtS1BQEG3btmXt2rV5qrPdbmfQoEFs376dsWPHsnjxYoYPH86OHTuytPnGjRvx8vIiKiqK/v37Y7FY6N27N/v27ePFF1/kzTff5OjRo7z//vsO+d577z1efvllGjRowPz583nqqadYuHAhL7/8cp7qCmCz2Xj22Wdp164dCxYsoFatWrz99tt8++23AOzdu5dx48bRpk0bFi5cyAsvvMDOnTsZM2YMkDFdpVq1alSrVo0VK1YQFBRklB0ZGUlERAQTJ06kZs2auarPuXPneOKJJzh16hSvvPIK06ZN4+LFi/Tu3Zv4+HhWrFgBQJcuXYyfc0vTDURERERE/iESUywkJFnIY3wAgHoNG/Nk76HMXXWQtKL18Sx2kKTz0dRt+QRDuzRmUsQXAFQLDnH4u3iJksbP19q983v27tnFhFfeoGWrtgDUqVsfd3cPliyKolPXJ3mg4oNcuniBAYNH0L9fb9zdMrof7u7ujBgxgp9//pkaNWoYc7+rVq1K0aJFiY6O5uzZs8TExNCoUSOH63bu3Nl4m1yoUCG6detGaGgoo0aNAqBKlSp88cUX7Nu3j9DQ0Dy3k7e3tzG1ICAgwPg5rxISElizZo1xb4UKFeLpp59m586dtG3blvfeew8fHx8WLVqEp2fG9pIVK1ake/fuebqOxWIhKSmJCRMm0K5dOwDCwsJITExk6tSpXLx4kRIlSuSqrMjISDw9PVm6dCne3t4ANGjQgFatWrFo0SIiIiKM0QmxsbGMHz8eFxcX1q5dS0REBBaLhW7duuXqWufPn8fT05OIiAjq1MmYzlKvXj3OnDmTpVPr6urK5MmTcXPL2AFj1apVnDx5ktWrVxMcHAxA/fr1adWqlZEnISGBefPm8cQTTzBhwgQAGjdujJ+fHxMmTKBv375UqlQpV3WFjKDG0KFD6dq1KwC1a9dm8+bNbNmyhSZNmrB37148PDwYOHCgUU8/Pz8OHTqE3W4nICDAaNPrpzL06NGD8PDwXNcFMtY3sFgsLFmyxPh+q1SpwpNPPsnBgweN4Jq/v3+ep04oSCAiIiIi8g+QkGQhIcXCjV4M22x2Ys4lkJicRrrV5nBumfKVeWvZD1xOMAPg4uGLU9pVnutRCzdX5zzX5+D+vTg7O9OsRSuH9NZt27FkURQ/HtjLAxUf5NXXpuLr487VK/EcOnmS06dP8803GYskWiyWHMv/7rvvqFmzJj4+Pg7p175pLVasGADVq1c30ooUKQJkdBLvpKJFizosfOfv7w/8tfDhzp07adq0qREggIx7K1OmTJ6u4+bmZgx7P3fuHL/++iunTp26YRvbbDaHKSEmkwlnZ2d27txJWFgYHh4epKenAxlBkzp16vD9998DULduXebPn0+9evUoVKgQAE2aNCEuLo7Zs2fTtWtXbDabwwiGzPKvVbJkSZYtW4bdbic2NpbTp09z8uRJ9u3bl6XOFStWNDremW1Xrlw5I0CQWc8WLVoYw/H3799PamoqLVu2NO4FMKYjbN++nYCAgCxTD66v57Wuffbc3NwoWrQoycnJRrvMnDmT9u3b07ZtW5o1a0bjxo2zjITJzq1M3di7dy81atRwCAD5+/sb3/vtUJBAREREROQuZrfbuZpkISk17YYBgqOn4vh85ynOXUrGarORkJxG9KlLnIiNB+Crfb9TqFRp4/zSJbyJt7rcUoAA4GrCFXz9/LJ0qooWzei4JyYm4uXpypmTPzNoyhQOHTqEp6cnAQEBlC5d2ri3nHz33Xc0bdo0S3rm29hrXdvRvltcX6fMveozO+dxcXFGkONat7JDwLfffssbb7zByZMn8fLyokqVKkYHPrs2fvHFF/n000+Nz2XKlOHrr78mPj6eDRs2sGHDhix5MhdmLFasGC1atMhyvFmzZnz//fdcvHiR5557jt27dxvHwsLC+PDDD7Pk+d///seMGTP4/fff8fPzo2rVqnh4eGQ5z+u6XTouX76cbdtdmxYfHw/AwIHZb9l5/vx5du/ebaw/kWnZsmU5Bmqur5uTk5PRvjVr1mTBggUsXbqUJUuWsGDBAooXL87gwYNvuu1j5neVF/Hx8ZQtWzbP+XJDQQIRERERkbuUzWbnSpKZlNR0bjTD4OipOJZvOkqqOR0vT1dcnF0wAcmp6fz3y2MApKVnlGAyQbdWldn9eWEO/HHrdSvs48uV+HisVqtDoODSpYsA+N9XDBfSGDBgAIGBgaxfv56KFSvi5OTE1q1b2bRpU45lWywW9uzZw7hx4269gn+6vnOeKSkp6bbLvh3+/v5cvHgxS/qlS5eoWLEicOO6Z3acz5w5w7Bhw2jVqhXvvfce5cqVw2QysXz5cmO+/PWGDx/OU089ZXzOfEvv4+NDw4YNjekc13Jxyeg67tmzh5iYGDp27Ohw3Gw24+zsjK+vL5MnT3Zo3+s7+ZnlRERE0LNnT/r160fJkiUBePvtt2+6LkPJkiU5depUlvTMdRMAChcuDMD06dOpUKFClnOLFy+Oj48Pq1atckh/4IEHjABDXjVp0oQmTZqQkpLCzp07WbZsGa+99hrVq1fP89SX60c4ZI5YyOTj40NcXFyWfDt27KBs2bKUK1cu7zfwJy1cKCIiIiJyF7La7FxOSCX5JgECm83O5ztPkWpOx8/bDTcXJ5xMJv7sX5Kckmac6+bqxJBOobSonbUD4XST7QivF1qzFlarla3ffOmQ/vXmjQA0bFCPkydPEh8fT69evQgICMDJKeMamav8Z3Z+M9MzRUdH4+vrm+dV2bOTOfLgjz/+ioicOHHihh3BGw05zy9169bl22+/xWw2G2lHjhxxWNk/u7pfuXLFWMEeMhZINJvNDBw4kPLlyxuBhcwAQXYjCcqWLUtISIjxJ7Odw8LCOH78OFWrVjWOBQcHs3TpUjZv3gxkDPUfP348v/76q1GezWZj06ZN1KxZEzc3NypWrOhQfmbQ41r79+/HZrMxYsQII0BgtVqNaQ3XB0auFRYWRmxsrMPuDampqQ5BkerVq+Pq6sq5c+cc6uLi4sKMGTOIjY3F29vb4VhISEi2I1Vy46233qJz587Y7XY8PT1p0aIFERERAPz2W8bOINc/5znx9vZ2+M6BLIGTOnXqcPDgQYdAwaVLl+jfvz9bt27N0/Wup5EEIiIiIiJ3mbR0G/EJqVjSc+4oZYo5l8C5S8l4eboaHUS73Y7VZiftmvzubs6Meao29/sXzrYcb28fjhz+kX17d1OpUhV8Cmd/XqZ69RtRo1Yd3nnrNS5eOE9AQGUO/7ifjz5cQseOHQkICCAhIQFvb2/mz5+Pi4sLLi4ubNq0yXh7mzk/P/Ot7+bNmwkLC+PAgQNZFiy8VfXq1cPDw4OpU6cyatQokpKSmD17Nn5+fjnmyVwHYcuWLfj6+lKlSpV8qcu1Bg8ezIYNG+jfvz/PPPMMV69eZdasWTg5ORnfY2BgIKVKlWLu3Ll4e3tjMpl47733HKYyBAUF4eLiwrRp03jmmWewWCysWbOGLVu2AFnfQN/I0KFD6d69O4MGDeLJJ5/E3d2dFStW8OWXXzJ79mwAunfvzn//+18GDx7MqFGj8PT05D//+Q/Hjh1j+fLlub5W5pv1KVOm0LlzZ65cucLy5cs5evSoUe+cOuzt27dnwYIFDBs2jFGjRlG4cGGWLFnCpUuXjKksRYoUoX///syaNYvExETq1avHuXPnmDVrFiaTKd+/0/r167NkyRLGjx/PY489RlpaGosWLcLPz4/69esDGc/5/v372bFjB9WqVcuxrBYtWvD111/z5ptv0rJlS/bs2ZNl94g+ffqwdu1a+vfvz6BBg3B1dSUqKgp/f38effRR43r79u3jhx9+oE6dOsZzdTMaSSAiIiIichexpFm5fDV3AQKAxOQ0rDYbLs4ZHQCrzc75yylZ1i+oW6VkjgECgA6du+Hs4sILY0eye+f2m17XZDLxxtvv0v7xTqz65D+8MG4UW7d8yXPPPccbb2Rsgejj48O8efOw2+2MGjWKcePG8dtvv/HRRx/h5eXFnj17gIyOfMOGDXnnnXeYMWMGBw4coHHjxrm6/5spXLgwc+bMwWq1MmzYMGbNmsWwYcMcFr27XqVKlWjfvj3Lly9n7Nix+VKP691///0sXrwYs9nMyJEjmTlzJgMGDKBEiRLG8HxnZ2dmz55N8eLFee6553j99dd55JFHaNOmjUM577zzDufOnWPIkCHG9o0ffvghJpPJaOPcqFKlCsuXL8dkMjFu3DhGjhzJhQsXmDt3rnHN4sWLs3z5cgIDA3nttdd49tlnSUlJYenSpQ4LSN5MvXr1mDhxIvv372fAgAFMnTqV0qVLExkZCWR9c34tFxcXFi9eTLVq1XjllVcYN24clSpVonXr1g7z+5999lnGjx/P5s2bGTBgANOmTaN27dp89NFHWRbEvF3NmjVj+vTp/PLLLwwfPpznnnsOT09Pli1bZgSknnrqKVxdXRkwYIAxmiY7nTt3ZsCAAXz22WcMHDiQ/fv3G0GaTKVKleI///kP9913H+PHj+eFF16gVKlSfPDBB/j6+gIZgajDhw8zYMAAfv/991zfi8mel40z/6UOHToEQEhI1q1f/i7JyclER0dTtWrVW1rYQm5M7Vuw1L4FS+1bcNS2BUvtW7DUvgWroNrXbEnncoIZax72ODz9+1Xmr/kRdzdnTMCF+BTSrX/ld3NxopCHC0M6V+f+UjceHXArnJ1M+BRyxcvT7eYn58Ld/uzmV99gx44duLq6Gtv/AVy9epWGDRsybty4LAvqyV9++eUXTp48SZs2bRzejnfp0gV/f38j0CC3RtMNRERERETuAinmNOITLdjyECAAKFfSh5LFCnHmj6ukmq1cm72QhwtOTuBf3ItyJfP3zSmAq7MTvj7uuN/iDgn/Zj/99BOzZ8/mueeeIygoiPj4eJYsWYKPjw/t27e/09W7qyUnJzNq1Ch69OhB69atsVqtbNiwgcOHDxfYyI9/kzseJLDZbERGRrJy5UoSEhKoW7cuEydOzHE1xrS0NGbPns3atWtJSEggODiYl156yWFvyb59+xoLXmTKadsNEREREZE7LTk1jSuJFmy3MMjXycnEg2X8OHrqskO6t6crmOwUcnclvH4FnJxyNx85N0xkrHHg5+2Ocx4XPJQMmesHfPzxx/z+++8UKlSIsLAw3nzzTWO7Qcle9erVeffdd1m8eDFr167FbrdTrVo1Fi1aZMz/l1t3x4ME8+bN4z//+Q9Tp07F39+fadOm0b9/f/7v//7P2IrjWq+88gpbtmwx5qzMmjWLAQMGsHHjRmNeyc8//8wrr7xCq1atjHyurq5/2z2JiIiIiORWUmoaV28xQGC329m8+wwbtv/qkO7h7oyrixMlixUivH4FqlTIv06nyQReHq4U9nLL9UJokpWTkxNDhw5l6NChd7oq/0jh4eGEh4ff6Wrck+5okMBisfD+++8zduxYmjdvDsDMmTNp0qQJX3zxRZZhNjExMaxevZr58+fTpEkTAF577TU6dOjA4cOHadCgAZcuXeLSpUtUr16dEiVK/N23JCIiIiKSa0kpFq4mWcjjDAMArDYbKzYfY9v+s0ZaIQ8XOjZ7kKKFPfEu5Eq5kj75OoLAyWSisFf+rT8gInefOxokOHr0KElJSTRo0MBIK1y4MNWqVeOHH37IEiTYvn07Pj4+NG3a1OH8r7/+2vj8888/YzKZeOCBB/K1rna7PU/bh+S3zO1hMv+W/KX2LVhq34Kl9i04atuCpfYtWGrfgpUf7ZuSauVKsjnPaxAAmC1WPtj4M9Gn4o20ooXdGfh4VUoW/WuxP7M59Zbrdz1XF2d8vd0w2dNJTk7Pt3Kvd7c/u3a7XSMo5J52R4MEf/zxB5CxfcO17rvvPuPYtX799VfKlSvHF198wYIFCzh37hzVqlVj/PjxPPjggwAcO3YMHx8fpkyZwvbt2ylUqBDh4eEMHTo02+kLuZWWlkZ0dPQt588vp06dutNVuKepfQuW2rdgqX0Ljtq2YKl9C5bat2DdSvs6OzuTbnPiapKFtPS8d7YTU62s3x3Phat/5S3p58ojdQuTePl3Ei/fIPMtcHIyUcjDHS8PJy79kc7ftTna3fzs3k6/QuRud0eDBJnRwet/ydzd3bly5UqW8xMTEzl9+jTz5s1j3LhxFC5cmKioKHr06MGGDRsoVqwYx44dw2w2ExoaSt++fYmOjubtt9/mt99+4+23377lurq6uhIQEHDL+W9XSkoKp06dokKFCnh6et6xetyr1L4FS+1bsNS+BUdtW7DUvgVL7Vuwbqd9k1LTSUi24HcLIwh+u5jER1uiiU/8K0AQ8mBRnm5bCbcC2GHAZDLh5eGCTyE3/q6X53f7s3v8+PE7XQWRAnVHgwQeHh5AxtoEmT8DmM3mbP9BcHFxITExkZkzZxojB2bOnEmzZs349NNP6d+/P1OmTCEiIgJfX18AKleujKurK6NHj2bcuHEUL178lupqMpnuin1aPT0974p63KvUvgVL7Vuw1L4FR21bsNS+BUvtW7Dy2r4JSRbSbDbc3T1ufvJ1jvx6iQWfHibVYjXSWtUtT6cWAfm67kAmJxN4e7rh43Vn3prfrc+uphrIve6O7leSOc3g/PnzDunnz5+nZMmSWc739/fHxcXFCBBARqChXLlyxMbGAhmBhMwAQaZKlSoBZDuFQURERETk73A10UJCioVbGa3/3cGzRH5y0AgQmEzQvXVlujxUqUACBM5OJny93e9YgEBE7pw7GiSoUqUK3t7e7Nq1y0i7evUqR44coW7dulnOr1u3Lunp6Rw6dMhIS01NJSYmhvvvvx+Anj178sILLzjkO3ToEK6urlSoUKFgbkRERERE5AauJJpJTM17gMBmt7N263E+2njU2CLRzdWJIZ2r07x2uQKoKbg6O1GksAeFPLSFuMi/0R2dbuDm5sbTTz/N9OnTKVq0KGXKlGHatGn4+/vTpk0brFYrcXFx+Pj44OHhQZ06dWjYsCERERFMmTIFPz8/Zs+ejbOzM48//jgAbdu25Y033iA0NJTGjRtz6NAh3n77bfr164e3t/edvF0RERER+Re6kmgmKTUtzwGCtHQrH6yPZk/0OSOtsJcbw7pW537/wvlcywzurs74+bjj4nxH3yWKyB10R4MEACNHjiQ9PZ0JEyaQmppK3bp1Wbx4Ma6ursTGxvLQQw/x5ptv0qlTJwDmzJnD9OnTGT58OKmpqdSqVYtly5ZRtGhRAJ5++mlMJhMffvghb7zxBiVKlKBPnz4MHDjwTt6miIiIiPzL2O12riRZSL6FAEFiShrzVx/keOxfi3mXLu7F8K41KOqb9/UMbsZkAg83F/y83Qtk+oKI/HPc8SCBs7Mzzz//PM8//3yWY2XLluXnn392SPP29uaVV17hlVdeybHMp556iqeeeiq/qyoiIiIikit2u534RDMpqenkdQmC85eTifzkAOcvpxhpVSoUZVCHEDw98v+/7yYTeHu44ePlqkX5ROTOBwlERERERO4ltxMgOHn2CvNWHSQxJc1IaxhSiqfCq+BcAFMAnEwmCnu74aX1B0TkTwoSiIiIiIjkE5stI0CQas57gGDv0XMs+b8jpFttRtpjTSvycIMKBfKG38XZhJ+3O+5u6hKIyF/0L4KIiIiISD6w2uzEX00lNc2ap3x2u53Nu8+w5pvjRpqLs4le7aoRFuSf39UEMhYo9PV2x9VFCxSKiCMFCUREREREblO61UZ8ghlzHgMEVpuNFZuPsW3/WSOtkIcLQzqFUql8kfyuJiYTeLq74OulBQpFJHsKEoiIiIiI3Ia0dBvxCalY0m03P/kaqeZ0Fv3vMIdPXDLSivt6MLxbDfyLeeV3NXEygbenGz5ebvletojcOxQkEBERERG5BSaTiXSrjYSrqQ7rCOTG5YRU5q08SMz5RCPtgdKFGdK5OoULoBPv7GTC19sNT3ctUCgiN6YggYiIiIjILTA5uRCXYMbV1T1P+WLPJzB35UEuJ5iNtJqBJejbPgg3V+f8riZuLk74+bjj6pL/ZYvIvUdBAhERERGRPDKn2bialIZPug3XPLycP/LrJRZ8eohUy19rF7QOK0/HFgE45fMOBibA3c0FPx93nLX+gIjkkoIEIiIiIiJ5kJRiIT4hlVRLWp7yfXfwLP/5/Gds9ozNEU0meKJ1IM1rlc33OjqZoJCHK4W93Apk+0QRuXcpSCAiIiIikksJSRYSUixYbfZc57HZ7fxv2wk+33HaSHN3dabf48GEBhTP9zo6O5ko7OVGIQ+tPyAieacggYiIiIjITdjtdq4kWUhOTcOe+/gAaelWPlgfzZ7oc0aar7cbw7pUp7x/4Xyvp6uzE74+7rgXwNoGIvLvoCCBiIiIiMgN2Gx24hPNpJrTyUN8gMSUNOavPsjx2CtGWukSXgzvUoOivh75WkcTGaMT/HzccXZ2yteyReTfRUECEREREZEcWK024hPMpKZZb37yNc5fTibykwOcv5xipFWpUJRBHULw9Mjf/4Kb/lx/wFfrD4hIPlCQQEREREQkG2npNuITUrGk2/KU70RsPFGrfyQx5a+FDRuGluKptlXy/S2/k8mEj5cr3p5u+VquiPx7KUggIiIiInIdc5qV+AQz6da8BQj2Hj3Hkv874pDvsaYVebhBhXx/y+/ibMLP2x13N/2XXkTyj/5FERERERG5RqolnfgEc552MLDb7WzefYY13xw30lycTfRqV42wIP98r6O7qzO+3u64umj9ARHJXwoSiIiIiIj8KTk1jSuJFmx52MLAarOxYvMxtu0/a6QV8nBhSKdQKpUvkq/1M5nA090FXy93nJy0/oCI5D8FCUREREREgMRkCwnJFvIwgIBUi5VF//cjP528ZKQV9/NkeNfq+Bfzytf6OZnA29MNHy+tPyAiBUdBAhERERH517uSaCYpNY08DCAgMcVK5KrDnL2QZKQ9ULowQzpXp3A+d+SdnUz4ervh6e6ar+WKiFxPQQIRERER+dey2+3EJ5pJMafnKUDw24UkVm6PIyn1rwUKa1YuQd9Hg3Bzdc7XOrq5OOHr7Z7v5YqIZEdBAhERERH5V7LZ7FxOMGO2pJOH+AA/nbzEgrWHMVv+ChC0CitPpxYBOOXjDgYmwN3NGT9v93zfOlFEJCcKEoiIiIjIv47VaiM+wUxqmjVP+b47eJb/fP6zsbChyQRPtA6kea2y+Vo/kwkKebji6+WW71sniojciIIEIiIiIvKvkpZuIz4hFUu67eYn/8lmt/O/bSf4fMdpI83V2UTvdoHUCSqTr/VzMpnwKeSKdyEtUCgifz8FCURERETkX8OSZiU+wUyaNfcBgrR0Kx+sj2ZP9DkjrbCXK+G1fAiqWDRf66cFCkXkTlOQQERERET+FcyWdC4nmLHmYY/DxJQ05q8+yPHYK0Za6RJe9G9fhbgLsflaPy1QKCJ3AwUJREREROSel2JO40qiJU8BgvOXk4n85ADnL6cYaVUrFGVghxCwpxF3IX/qpgUKReRuoiCBiIiIiNzTElMsJCSlGYsN5sbJs1eYt+ogiSlpRlqj0NL0aBuIs7MTKdek3w6TCbw8XCmsBQpF5C6hIIGIiIiI3LMSkiwkpFjIQ3yAvUfPseT/jpB+zboFjzd9kPAG9+drR97ZKWOBQi9PLVAoIncPBQlERERE5J5jt9u5kmQhOTUt1wECu93OF7vO8OmW40aai7OJXu2qERbkn6/1c3V2wtfbDXc3/XdcRO4u+ldJRERERO4pNpud+EQzqeZ0cjuAwGqzsWLzMbbtP2ukFfJwYUinUCqVL5Kv9fNwdcbXxx0XrT8gInchBQlERERE5J5htdmJT0gl1WLNdZ5UczoL1x3mp5OXjLTifp4M71od/2Je+VY3kwk83V3w9XLHyUnrD4jI3UlBAhERERG5J6RbbVy+mool3Xbzk/90OSGVuSsPEns+0Uh7oHRhhnapjk+h/FsrwMmUsf6Adz6WKSJSEBQkEBEREZF/vLR0K5evmkmz5j5AEHsugchVB4lPMBtpNQNL0Ld9EG6uzvlWN2dnE75ebni6u+ZbmSIiBUVBAhERERH5RzNb0rmcaMZqzf0WBj+dvMTCtYccpiW0DitPxxYBOOXjDgZuLk74+bjj6pJ/QQcRkYKkIIGIiIiI/GOlmNO4kmjBast9gODbA2f5eNPP2P7c9sBkgidaB9K8Vtl8q5cJcHdzwc/HHWetPyAi/yAKEoiIiIjIP1JiioWEpDSjs38zNruddVtPsGnnaSPN3dWZ/o8HExJQPN/q5WQCLw83fLxcMeXjqAQRkb+DggQiIiIi8o9zNdFCYqqFXMYHSEu38sH6I+yJPm+k+Xq7MaxLdcr7F863ejk7mSjs5UYhD60/ICL/TAoSiIiIiMg/ht1uJz7RTIo5PdcBgsRkC1FrfuRE7BUjrXQJL4Z3rUHRwh75VjdXZyd8fdxxz8dFD0VE/m4KEoiIiIjIP4LVZic+IRWzxUpuVyA4fzmZyE8OcP5yipFWtUJRBnYIwdMjf/4rbCJj2oKfjzvOzk75UqaIyJ2iIIGIiIiI3PXSrTbiE8yY06w3P/lPJ2Ljmbf6R5JS0oy0RqGl6dE2MN86804mE4U8XfH1ctP6AyJyT1CQQERERETuapY0K/EJZtKstlzn2RN9jqWfHSH9mjyPN32Q8Ab351tn3s3VFV8vd/y83fOlPBGRu4GCBCIiIiJy18rrFod2u50vdp3h0y3HjTQXZxO92lUjLMg/3+rl6uKMn7crnh5af0BE7i0KEoiIiIjIXSmvWxxabTb++8Uxvj1w1kgr5OHCkM6hVCpXJN/q5ebiRGEPdy79kZ5vZYqI3C0UJBARERGRu4rdbudqkoWk1LRc72CQak5n4brD/HTykpFW3M+T4V2r41/MK1/qZTKBh5sLvt7umFNTsOe2ciIi/yAKEoiIiIjIXcNms3MlyUxKanqudzC4nJDK3JUHiT2faKQ9ULowQ7tUx6eQW77Uy8kE3p5u+HjlT3kiIncrBQlERERE5K5g/XMHg9Q87GAQey6ByFUHiU8wG2k1A0vQt30Qbq75s16As5MJX283PN1d86U8EZG7mYIEIiIiInLHWdKsXEk0Y0nP/Q4GP528xIK1hzBb/goqtK5Xno7NA3DKrx0MXJzw9XbPt4CDiMjdTkECEREREbmj8rqDAcC3B87y8aafjUUNTSbo3jqQZrXK5kudTIC7mzN+3u44OzvlS5kiIv8EChKIiIiIyB2TmGwhITn3OxjY7HbWbT3Bpp2njTR3V2f6Px5MSEDxfKmTyQReHq4U9nLDlE8jEkRE/ikUJBARERGRv53dbudKkoXkPOxgkJZuZelnR9h79LyR5uvtxrAuNSjv75Mv9XIymfDxcsXbUwsUisi/k4IEIiIiIvK3strsxCekYrZYc72DQWKyhag1P3Ii9oqRVqaEN8O6VqdoYY98qZezs4ki3u64u+m/yCLy76V/AUVERETkb5OWbiU+IW8LFJ6LSyZy5QEuXE4x0qpWKMrAjiF4uufPf2fdXJzw8/HA1UXrD4jIv5uCBCIiIiLytzBb0rmcaMZqzf0Chcdj44la/SNJKWlGWqPqpenRJjBfFhQ0AR7uLvh6u+PspPUHREQUJBARERGRAncrOxjsiT7H0s+OkG79a9TB400fJLzB/fmyoKAWKBQRyUpBAhEREREpUBk7GFjIbXzAbrfzxa7TfLrlhJHm4myi9yPVqFvNP1/q5GQyUdjbDS8P13wpT0TkXqEggYiIiIgUCLvdztUkC0l52MHAarPx3y9+5tsDvxlpXh4uDO4cSqVyRfKlXi7OJvy0QKGISLb0L6OIiIiI5DubzU58oplUc3qudzBINaezcN1hfjp5yUgr7ufJ8K7V8S/mlS/10gKFIiI3piCBiIiIiOQrq9VGfIKZ1DRrrvNcvprK3FUHiT2faKQ9ULowQ7tUx6eQ223XyQS4u7ng56MFCkVEbkRBAhERERHJN7eyxWHsuQQiVx0kPsFspNUKvI8+7avh5up823XSAoUiIrmnIIGIiIiI5Itb2eLwp5OXWLD2EGbLX6MOWoeVp2OLAJzyoUPvZDLh4+WKt+ftj0YQEfk3UJBARERERG5bcmoaV5PytsXhtwfO8vGmn7H9uaqhyQTdWwfSrFbZfKmT858LFHpogUIRkVzTv5giIiIiclsSkiwkpuR+i0Ob3c66rSfYtPO0kebu6kz/DsGEPFg8X+qUsUChO64utz9dQUTk30RBAhERERG5JbeyxWFaupWlnx1h79HzRpqvtzvDulSnvL/PbdcpY4FCZ/x8PLRAoYjILVCQQERERETy7Fa2OExMthC15kdOxF4x0sqU8GZY1+oULexx23UymaCQhyu+WqBQROSWKUggIiIiInlitdq4nGDGnIctDs/FJRO58gAXLqcYadUeKMqADiF4ut/+f0mdTCZ8CrninQ/bJYqI/JspSCAiIiIiuXYrWxwej40navWPJKWkGWmNqpemR5tAnJ2dbrtOzk4mfL3d8HR3ve2yRET+7RQkEBEREZFcuZUtDvdEn2PpZ0dIt/4VVOjQ7EHa1r8/X6YEuLk44evtjpurFigUEckPChKIiIiIyE2lmNO4kpj7LQ7tdjtf7DrNp1tOGGkuziZ6P1KNutX886VOHq7O+Pm458toBBERyaAggYiIiIjcUGKKhYSkNGy53MLAarPx3y9+5tsDvxlpXh4uDO4cSqVyRW67PiYTeLq74OftrgUKRUTymYIEIiIiIpKjK4nmPG1xmGJOZ9G6w/x08pKRVsLPk+HdalCyaKHbro+TCbw93fDx0gKFIiIFQUECEREREcnCZrNzJclMijk91wGCy1dTiVx5kLMXEo20imV8GdI5FJ982HVACxSKiBQ8BQlERERExIHVaiM+wUxqHrY4jDmXQOTKg1xJNBtptarcR59HquXLooKuzk74+rjjrgUKRUQKlIIEIiIiImKwpFm5kpi3LQ4Pn7jIwnWHMVv+Ciq0qXc/HZo/iFM+rBng4eqMr487LlqgUESkwClIICIiIiIApFrSiU8w53oHA4BvD5zl400/G4samkzwZJtAmtYse9v1yVyg0NfLHScnLVAoIvJ3UJBARERERPK8g4HNbmfd1hNs2nnaSHN3daZ/h2BCHix+2/XRAoUiIneGggQiIiIi/3B2u/2WtwK02+1cTbLkaQeDtHQrSz87wt6j540y/Hw8GNalOuX9fW6pHtdydjJR2MuNQh63tkDh7bSHiMi/nSZ2iYiIiNxFWrZsyfjx43N9/ldffUVERIRD2vr163n44Yfp1asXr732Gj179qRnz55Z8lptdi4nmElKyX2AIDHZwsyP97P36HnSUuI5u/t9inlaGN+rjkOAYML45/jog8W5vo9MLs5OFPFxz1WAIDY2lsDAQNasWQPA1atXGTduHHv27DHOufberz9/zZo1BAYGEhsbm+d6/l1y+u5uxcqVKwkMDMzyZ8qUKX9bHUTk7qeRBCIiIiJ3kcjISLy9vXN9/tKlS7OkTZkyhXLlytG/f3/q1KnDG2+8keWctHQb8QmpeVqg8FxcMpErD3DhcgoAyRePk3T+KAM7BlOksMdfZaelsW/vbno/MzDXZUPGdAVfb3dcXW7tPVZ0dDTr1q2jc+fORtqkSZNyPL958+asWLGC++6775au908THR3NAw88wNSpUx3Sixe//ekhInLvUJBARERE5C5SrVq12y4jPj6ebt26Ua1aNe6///4sx82WdOITzaRbc79A4fHYeKJW/0hSSpqRVqmcH+cOgoeb438pfzywD69CXgRUCsxV2SbAw90FX293nPN5gcKAgIAcjxUtWpSiRYvm6/XuZtHR0YSEhFCjRo07XRURuYtpuoGIiIjIXSRzukHm0PiNGzcycuRIatasSVhYGBMmTCA5ORnIGAa+e/dudu/eTWBgILt27SIwMKNjvmDBAnr06MFvv/3mUH5yahp/XLzKksUL6N2jE21bNqBn9w58/NFSbLa/RhVYrVY+/mgpz/TsRpsWDRj0VDjRX75L8sXjADxY6DTffbYAgB5dH+Wt1/96Y79r53bq1muIyWTiwL49tGxcm317dvPciIGEt2xI907tWP9/n3Lp4gUmvTiWdq0b0/HRtny47AOjjJymAuQ0HWPXrl306tULgF69ehnD4280VP76a4wfP54+ffqwevVq2rZtS3BwMI8//jjbtm1zyLd//36eeeYZ+vbty8MPP8wHH3xAnz59jHplfg+7du1yyHd9XVJTU3nnnXdo06YNwcHB1KpVi759+xIdHZ1tfW/k2LFjDBo0iFq1alGrVi2GDRtGTEyMcdxut/Pzzz9TtWrVPJd9vbi4OCZPnkyLFi0IDg4mLCyMYcOGOXxXPXv2ZOzYsYwcOZIaNWrQt29fAM6fP8/o0aMJCwujbt26TJw4kZkzZ9KyZUuHa3zzzTd07tyZ4OBgmjdvzpw5c7BarYhIwVOQQEREROQuNmnSJMqUKcO8efPo168fq1atIioqyjhWrVo1qlWrxooVKwgKCmLFihUAdOjQgcmTJzsMJU9IshCfkMoLzz/Lf5cvo137Drz+1kyatWjF4oXzmDntr2kJC+fP4cOlCwmo3hz/On0pGdoFqyWZ3/d+RK/wAAb07MjTvfsBMPn1aTzdp7+Rd9eO76jXoJHDfbz2yos0aNSUN95+l3LlKzBz+ps8N3IwVapUJioqitDQUN58801+/PHHW2qnoKAgJk6cCMDEiRNvOM3gRg4fPszixYsZOXIkc+fOxdnZmREjRnDlyhUATpw4QZ8+fQAYPnw4gwcPZsGCBezduzfP1xo3bhyrV69m4MCBvP/++7zwwgv88ssvjBkzBntuF4kAfv31V7p3786lS5d46623eP3114mJieHJJ5/k0qVLAJw5c4akpCQOHTpE27ZtCQoKom3btqxduzZPdbbb7QwaNIjt27czduxYFi9ezPDhw9mxY0eWNt+4cSNeXl5ERUXRv39/LBYLvXv3Zt++fbz44ou8+eabHD16lPfff98h3+LFi1m0aBH16tVj/vz5PPXUUyxcuJCXX345T3UVkVuj6QYiIiIid7FmzZoZCxM2aNCA7du3s2XLFsaMGUNAQICxfkHmEPLMv0uWLEmlSpVwc3PDbreTbrWTkGJh547v2btnFxNeeYOWrdoCUKdufdzdPViyKIpOXZ/kgYoPcvHCeao36cKZ9EAKFcuoi6eHO7/uWIqfy1X8itxP6TJlAahUuQr+pUoD8PtvZ/ntbCy169ZzuI/wRx6ja/enM8rx9GTYoD5Urx7K2DHPAVClShW++OIL9u3bR2hoaJ7bydvb25haEBAQcMNpBjeSkJDAmjVrKF++PACFChXi6aefZufOnbRt25b33nsPHx8f5s6dy6lTp6hatSpVqlShe/fuebqOxWIhKSmJCRMm0K5dOwDCwsJITExk6tSpXLx4kRIlSuSqrMjISDw9PVm6dKnxPDRo0IBWrVqxaNEiIiIijNEJsbGxjB8/HhcXF9auXUtERAQWi4Vu3brl6lrnz5/H09OTiIgI6tSpA0C9evU4c+aMEaDK5OrqyuTJk3Fzy9jGctWqVZw8eZLVq1cTHBwMQP369WnVqpWRJyEhgUWLFtGyZUvGjRtHoUKF/p+9+w5vqnoDOP7NaJLuslF2AUtLB2WUWUZFQEHZ/BgyZEMZgiwVERSUKQJt2UMQERniAERQGbJBpgzZlD1Kd5u0SX5/1F4ILdBpGe/neXxsT+54c3pLc997znuoU6cObm5ujB49mnfeeYfy5ctnKFYhRNZIkkAIIYQQIo9ZLFbOX40iOs5EUrLF5inyw/PHixYtytWrVzN0XJVKhdliJdlswWIBqxWOHDqIRqOhXoOGNtu+1vgNFi+YzdHDBylarBQunm25ciGCZGMsSXG30VujKWgN5wKQlGR65Dn37t5JRR8/nJxsl0Ks6OOnfF3435vfyv7331u+fPmAlJvEvJQ/f34lQQAp/Q2QkJBSrHHPnj3UrVsXe3t7ZRt/f3+KFSuWqfPodDoWLkxZ/eHmzZtcuHCBixcv8scffwApSYSHWSwWmykhKpUKjUbDnj17CAgIwGAwkJycDKQkTapWrcquXbsAqFatGnPmzKF69eo4ODgAEBgYSEREBDNnzqRt27ZYLLbXXurxH1SkSBGWLl2K1WrlypUrXLp0ifPnz/PXX3+lidnd3V1JEKT2XYkSJZQEQWqcDRo0UKZmHDp0iMTERKpUqUJycrLyflKnI+zcuVOSBELkMkkSCCGEEELkoSNnbrP69zNcvRVLstlCVKyRw//c5uTFCACbm1EAtVqd4aHoKo0d92KMPHBfSXRMFK5ubmlu/vLnTxkucPvuPaZ+fZBzZ05y89g6jFHhaLQ63MuWxf7f0QKPO//ePX9SvUbtNO2ODo6oAL1Oi5uLPt339jR4OCaVKqWQYurNeUREBAUKFEizX1ZWCNixYwefffYZ58+fx9HRkQoVKig38On18QcffMD333+vfF+sWDF+//13IiMj2bBhAxs2bEizT2phxgIFCtCgQYM0r9erV49du3Zx584dhg4dyr59+5TXAgICWLZsWZp9fvzxR7744guuX7+Om5sbnp6eGAyGNNs5OjrafH/v3r10++7BtsjISAAmT57M5MmT02x769atNG1CiJwlSQIhhBBCiDxy5MxtQlcfISExGWdHO+w0dqhUEJuQxLINmS9e96Bks5XoWBPO+WyLvbk4uxIVGYnZbLZJFNy9eweAnX9HoS1whyt7F6J3eYnXu01gYOdXMejt2LP7T7Zv/e2R5zQZjRz+6wC9+g5M85pKBY72drg46kiIefIKBg/fnKeKi4t74r65qWjRoty5cydN+927d3F3dwceH3vqjfPly5cJDg6mYcOGzJ07lxIlSqBSqVi+fDk7duxI99wDBgygU6dOyvepT+mdnZ2pVauWUhzwQVptysf9AwcOEB4eTsuWLW1eNxqNaDQaXF1dGTdunE3/PnyTn3qckSNH0rlzZ3r06EGRIkWAlJv6J9VlKFKkCBcvXkzTnlo3AcDFxQWA4OBgatasmSb5IMs1CpH7pHChEEIIIUQesFisrP79DAmJyRRw1aO306BWq1ChQm+nJtGUMsz6SaMG1Or0P84lmpJJNCWlaff1r4zZbGbbH1ts2pd/uzrlC8fimGJvYUmKp37jlrzXozEGvR0A+/akDF23WC3/ntt2NMLhQwdxcXXDvWza4eAOBjtcnfTKDfSTpM6tv3HjhtJ27tw55Ulzeh4eHZEbqlWrxo4dOzAajUrbiRMnbCr7pxd7VFQU586dU74/fvw4RqOR3r17U7JkSaVfUhME6f3cixcvjo+Pj/Jf6koWAQEBnD17Fk9PT+U1b29vlixZwubNm4GUof6jRo3iwoULyvEsFgubNm3C398fnU6Hu7u7zfFTkx4POnToEBaLhYEDByoJArPZrExreDgx8qCAgACuXLlis3pDYmKiTVLEz88POzs7IiIiqFixohKLVqvliy++SLPahRAi58lIAiGEEEKIPHD+ahRXb8Xi7GiXzo2zCkdDyo35rXsJjz2Oi4sLhw4dYvfu3Xh6emLVpDx5fVRyoXqN2lSqXJVpk8Zz5/YtypZ7hZ9+2cr2X1fjUrwKeuciWJIT0BscOH9oI/t2F0ej1bL9j9/YuP4HABITEgGUugM7tv1O9Zq12bvnTwKq17I5n1qd8t7s9Zn72Fm9enUMBgMTJ05k8ODBxMXFMXPmTNzc3B65j7NzSjxbt27F1dWVChUqZOqcGdG3b182bNjAgAEDqF+/PufPn2f27Nmo1Wrl5+jh4cFLL71EaGgoTk5OqFQq5s6dazOVoWLFimi1WqZMmUL37t0xmUysXbuWrVu3AijLXGZE//79ad++PX369KFDhw7o9XpWrlzJli1bmDlzJgDt27fn22+/pW/fvgwePBh7e3u++eYb/vnnH5YvX57hc6UWlfzkk09o3bo1UVFRLF++nFOnTilxpyZJHtasWTPmzZtHcHAwgwcPxsXFhcWLF3P37l1efjllKku+fPno2rUrX331FY6OjtSpU4ebN28yY8YMVCpVrvxMhRC2ZCSBEEIIIUQeiI4zkWy2YKdJ/+OYVptyw5loTH7scTp16oSdnR29evVi46bfiE98/PYqlYrPJn9Js+atWP3dN4wcNojdf/5BwQqvU8SvLXo7DQM71ODzydPBamXcRyOZ+OkYbt28wZch83FwcOTY0UMA+FeuSpWq1VkwN4TZIdPZt2eXzdKHOq0aFwfdo0J5LBcXF2bNmoXZbCY4OJgZM2YQHBxsU/TuYeXLl6dZs2YsX76cYcOGZem8T1KqVCkWLlyI0WhkxowZhISE0KtXLwoVKqQMz9doNMycOZOCBQsydOhQJkyYQNOmTWnUqJHNcaZNm8bNmzfp16+fsnzjsmXLUKlUHDhwIMMxVahQgeXLl6NSqRgxYgSDBg3i9u3bhIaGKucsWLAgy5cvx8PDg/Hjx/Puu++SkJDAkiVL8PPze8IZ7qtevTpjxozh0KFD9OrVi4kTJ/Lyyy8TEhIC8NgpB1qtloULF+Ll5cXYsWMZMWIE5cuX57XXXlNqMUDKVIO3336b33//nV69ejFlyhSqVKnC119/rSSChBC5R2XNzCKsL6hjx44B4OPjk2cxxMfHc/LkSTw9PW3+ERU5Q/o3d0n/5i7p39wjfZu7XvT+PRseyWdL9mHQa9DbpR0mbzSZSTSZ+aBbAOVKuD32WKYkM1GxRkzJ94d6JyQkcO7cOcqWLZtugUBTkpkl60/w16n7heBcnfQEt/GjZNHs34gZ7DS4OevRPCIJ8qzavXs3dnZ2eHl5KddvcnIytWrVYsSIEXTp0iWvQ3xqnTlzhvPnz9OoUSOb0TNt2rShaNGiSqLhaf+34Wm4NxAiN8l0AyGEEEKIPOBezJVihZ24eD0anYva5qbJarUSk5BE6ZdccC/m+tjjJJqSiYwxYrZk/LlPbLyJsDVHOX81SmkrVsiJAW39yOeStkp9ZqhUKVMLXB31ylSD58nff//NzJkzGThwIPb29ly/fp1vvvkGZ2dnmjVrltfhPdXi4+MZPHgwHTt25LXXXsNsNrNhwwaOHz+eayM/hBCZJ0kCIYQQQog8oFaraBNUntDVR7gbbcTZ3g47rZqkZAsxCUk46LW0CSr/2Bvt2AQTMXFJWDIxMPRmRDwhqw5z+4FaB15l8tOrhU+m6wakeU8qcLLX4eyYtSkGz4LU+gGrVq3i2rVrODo6Ur16dT7//HNluUGRPj8/P7788ksWLlzIunXrsFqteHl5sWDBAmrUqJHX4Qkh/iVJAiGEEEKIPOJXvhDBbfxY/fsZrt6KJTYhCa1GTemXXGgTVB6/8oXS3c9qtRIdZyIuMYnMTBw9Gx7J7DVHiHugbkFtv5fp2Mgj29MCNGoVrk467P9dCeF5pVar6d+/P926dXuqh8Q/rZo0aUKTJk3yOgwhxGNIkkAIIYQQIg/5lS+ET9mCnL8aRXScCRdHHe7FXB85gsBssRIZY8RoSiYzhaX2n7jBV+tPkGy+v1eLemVpXKNUhpclfBQ7jRpXZ326tRWEEEI8WyRJIIQQQgiRx9Rq1ROLEwIkJVuIjEm0KVD4JFarlV92X2TdtnNKm1ajoluzilT1LJKVcG0Y7DS4OuvRPmcFCoUQ4kUlSQIhhBBCiGeA0ZRMZGzKsokZZbZY+e63c+z5+/4KBo4GLf1a+2UoKfE4z3uBQiGEeFFJkkAIIYQQ4imXYEwiKtaUqRUMEo3J/Lw/kvDbJqWtkJs9A9pVokj+7M2hfxEKFAohxItKkgRCCCGEEE+xmDgTsQkmMpEfICI6kVmrjnP97v0EgXsxV/q19sXZIXs39i9KgUIhhHhRSZJACCGEEOIpZLVaiYozEZ/JFQzCb8YQsuoIUbFGpa1yhcJ0a+qFLpuFBaVAoRBCPP8kSSCEEEII8ZTJ6goGx8/dYf4PxzGazEpbUJWXadOwAupsrmCgt9PgJgUKhRDiuSdJAiGEEEKIp0hWVjAA2H7oCt/++g+Wf4cdqFVQ19uZN+uUzlaCQAoUCiHEi0WSBEIIIYQQTwljkpmoGCNJmVjBwGK1sm7rOX7de0lp0+s0dH39FXTmiGzFo1aBo0GHi5MUKBRCiBeFJAmEEEIIIZ4CWVnBwJRkZsn6E/x16v4Sh65Oega09aOgi5Zz57KeJNCoVTg76nA0SIFCIYR4kUiSQAghhBAij8XGm4iJT1KmCmR0n7A1Rzl/NUppK1bIiQFt/cjnYiAhISHL8Wg1Ktyc9Oh18lFRCCFeNPIvvxBCCCFEHrFarUTHmYjL5AoGNyPiCVl1mNv37icCvMrkp1cLH+z12ft4p9OqcXM2YKeVAoVCCPEikiSBEEIIIUQesFisRMYaSTRmbgWDs+GRzF5zhLjEZKWttt/LdGzkgSYbKw+oAL1Oi5uzHo0UKBRCiBeWJAmEEEIIIf5jyWZLyhKHSeYnb/yA/Sdu8NX6EySb76cVWtQrS+MapVBlcwUDR4MdLo66bB1HCCHEs0+SBEIIIYQQ/yFTkpnITK5gYLVa2bTnEuu2nVPatBoV3ZpVpKpnkWzFo1apcHa0w8leVjAQQgghSQIhhBBCiP9MoimZyBhjplYwMJstrPj1NH8euaa0ORq09GvtR7kSbtmKR/NvgUKDFCgUQgjxL/mLIIQQQgjxH4hLMBEdl7kVDBKMycxfd4wTF+4vZVgonz0D2laiSH6HbMWTUqBQj51Wk63jCCGEeL5IkkAIIYQQIpdFx5qITTRlagWDiOhEQlcd4ertWKXNvZgr/Vv74uSQ9akBKQUKNbg56bNV6FAIIcTzSZIEQgghhBC5xGKxEhVnJMGYnKkEQfjNGEJWHSEq1qi0ValQmG7NvLL15F+lAgeDHa5SoFAIIcQjSJJACCGEECIXmM2WlCUOTZlbweDYuTss+OE4xgf2a1yjFM3rlUWdjRt7tQqcHXTZGoUghBDi+SdJAiGEEEKIHJaUbCEyJhFTcsZXMADYfugKK349rYw6UKtUtG/0CnX9i2crHo1ahauTDnu9XbaOI4QQ4vknSQIhhBBCiBxkNCUTGWsk2Zzx+QUWq5Xvt55l897LSptep6FXc2+8yxbMciwqlQq9nYb8LgZ0dlKgUAghxJNJkkAIIYQQIofEJSYRHWfCkoklDk1JZpb8fIK/Tt9S2tyc9Qxo40fxIs5ZjkWlAicHPfmc9ZIgEEIIkWGSJBBCCCGEyAFZWcEgJt7E7DVHOX81SmkrVsiJAW39yOdiyHIsKhU4Guxw1KtQq6VAoRBCiIyTJIEQQgghRDZkdQWDmxHxzPruMHciE5Q2rzL56dXCB3t91j+iqVXgZK9Do1JzNTk5y8cRQgjxYpIkgRBCCCFEFiWbLUTGGDEmZW4FgzPh95iz5ihxifdv4uv4vUyHRh5oNOosx/NggcL4eEkQCCGEyDxJEgghhBBCZIExyUxUjJEkc+ZWMNh/4gZfrT9hU9iwZf2yNKpeClU2lji006hxddajl/oDQgghskGSBEIIIYQQmZRgTCIq1oQ5EwUKrVYrm/ZcYt22c0qbVqOmWzMvqnoWyVY8ejsNbs56tNkYhSCEEEKAJAmEEEIIITIlNt5ETHwSlkwUIDCbLXyz6TQ7j15T2hzt7ejX2pdyxd2yHIsK0Ou0uDnr0UiBQiGEEDlAkgRCCCGEEBlgtVqJjjMRl5iUqQKFCcZk5n1/jJMXI5S2QvnsGdC2EkXyO2Q5HpUKHAx2uDrqsjVNQQghhHiQJAmEEEIIIZ7AbLGmFCg0JZOJ/AAR0YmErjrC1duxSpt7MVf6t/bFyUGX5XhSVzBwdsz6MYQQQoj0SJJACCGEEOIxsrqCweUbMYSuPkJUrFFpq1KhMN2aeWGnzXpxQbVKhYujHY72kiAQQgiR8yRJIIQQQgjxCFldweDYuTssWHfcJrHQuEYpmtcrizobUwM0ahVuznoMOvkIJ4QQInfIXxghhBBCiHRkZQUDgO2HrrDi19NK3QK1SkWHxh4EViqWrXhkiUMhhBD/BUkSCCGEEEI8JCbORGyCiczkByxWK9//cZbN+y4rbXqdht4tfKjoXiBb8ei0avK5GGSJQyGEELlOkgRCCCGEEP+yWKxExRlJMCZnagUDU5KZJT+f4K/Tt5Q2N2c9A9r4UbyIc5bjkSUOhRBC/NckSSCEEEIIAZj/LVCYmMkChTHxJmavOcr5q1FKW/HCTgS38SOfiyHL8cgSh0IIIfKCJAmEEEII8cIzJZmJzEKBwpsR8cz67jB3IhOUtoruBejV3BuDPusfs2SJQyGEEHlFkgRCCCGEeKFltUDhmfB7zFlzlLjEZKUtsFIx2jd6BY0667UD1CoVLk46HA12WT6GEEIIkVWSJBBCCCHECysrBQoB9p+4wVfrT5Bsvr9jy/rlaFS9ZLamBmjUKvI569HLEodCCCHyiPwFEkIIIcQLJ6sFCq1WK5v2XGLdtnNKm1ajplszL6p6FslWTHYaNW7OenSyxKEQQog8JEkCIYQQQrxQks0WorJQoNBstvDNptPsPHpNaXO0t6Nfa1/KFXfLVkw6rRo3ZwN2WlniUAghRN6SJIEQQgghXhjGJDNRWShQmGBMZt73xzh5MUJpK5TPngFtK1Ekv0O2YtLbacjnrEejkQSBEEKIvCdJAiGEEEK8EJ5UoNBqtaZbTyAiOpHQVUe4ejtWaStb3JV+rXxxcsje6gMGnQY3ZwMateqR588rT1s8Qggh/huSshZCCCHEcycoKIhRo0Yp38fEmYiMMT4yQbDzz21MHP+xTdvvWzbRtsUbtG1Wl79++4rwXXMI3zWHqp6Febe9f7YSBCrAwaAln7OB27du0rt3b65evZpmu379+jF79uwsnycjrly5goeHB2vXrgUgOjqaESNGcODAAWWbzp0707lz53S3X7t2LR4eHly5ciVX48yOB+PPrlWrVuHh4YG/vz8dO3bE398fDw8PPvnkk8fuN2rUKIKCgnIkBiGEyE0ykkAIIYQQz52QkBCcnJwyXKBw9bfL07RNnzoRi10+igX0RGtw4ebRNeR3MdD9LW/U2XjCrlKBg8EOV0cdKpWKXbt2sW3btjTbmUwm9uzZw8CBA7N8rqw4efIkP/zwA61bt1baPv7440duX79+fVauXEnhwoX/i/Dy3MmTJylTpgxjx47l4sWLlC5dGoPBQMGCBfM6NCGEyBGSJBBCCCHEc8fLywuz2cK96MRMFygE2H7oCnGx0eQvH4BDwbKoVSqK5HfA1Umf7QSBk0GHi9OTRyEcOHAAR0dHPD09s3y+nFKuXLlHvpY/f37y58//H0aTt06ePImPjw++vr7Y2dnh6emJg0P26lIIIcTTRKYbCCGEEOK506BBA94bNoKLl8MJqlOFrb9vZuzoETR9LZDmrzdg6qRPSUhIAGDIgN4cOXyQI4cPElSnCjMXrmPswOYARJzZwj8/j6B9/SK4OultzmEyGlm2ZD5dO7aicVBNOrdvwYqvl2Cx3C+KaDabWfH1Erp3bkeToFq8/modevfswp49e4CUofrvv/8+AK+++qrNFIlt27YRGBiISqVi7969eHh4sHv3bjp37oyvry/169dn1apV3Lp1iwEDBuDv70+9evVYsmSJzTH8/f3TTAV4eDpGqr1799KlSxcAunTpogzRf9xw/YenG4waNYpu3bqxZs0aGjdujLe3N82bN2f79u02+x06dIhOnTpRqVIl6tevz1dffUW3bt2UuFLf8969e232eziWxMREpk2bRqNGjfD29qZy5cq88847nDx5Mt14H+eff/6hT58+VK5cmcqVKxMcHEx4eLjyutVq5fTp09lK3KxcuZL69evj6+tL165dOXHihM3r+/fvp0ePHlSrVg1vb2+CgoKYNWuWzXX1888/89Zbb+Hr60uNGjUYNmwYN2/etDnOqlWraNq0Kd7e3tSvX59Zs2ZhNmc+YSaEePFIkkAIIYQQz5UEYxIWixXLA/MLpk/5jCJFX+LTz6fxv46d2fjzD3z91QIABr83inKveFC2vAcNO4zm2DU7StQOBqCQew3GfB5KDf/yNuewWq18OHII3y5fyhvNWjBh0nTqNWjIwvlhTJ/ymbLd/DmzWLZkPm81b03o7DmMH/8pkZGRDB48mISEBOrXr0+/fv2AlCkS/fv3V/bdtm0b9erVsznv0KFDCQoKYu7cuZQpU4aPP/6YLl26UL58ecLCwvD19eXzzz/n6NGjWeq7ihUrMmbMGADGjBnz2GkGj3P8+HEWLlzIoEGDCA0NRaPRMHDgQKKiogA4d+4c3bp1A+CLL75g4MCBzJs3j4MHD2b6XCNGjGDNmjX07t2bRYsW8f7773PmzBnee+89rI+bY/KQCxcu0L59e+7evcukSZOYMGEC4eHhdOjQgbt37wJw+fJl4uLiOHbsGC1atKBz5860aNGCdevWZegcN27cICQkhHfffZcvvviCqKgoOnfuzLVrKctqnjp1im7duuHm5sb06dOZPXs2VatWJSQkhI0bNwJw8OBBRowYQaNGjZg/fz7vv/8+e/bs4b333lPOM3fuXD766CNq1qzJnDlz6NSpE/Pnz+ejjz7KcH8IIV5cMt1ACCGEEM+N2AQTMXFJPHxrWL1WHfoNGAJA5aoBHNi/lz27dtCr70BKl3HHYHDg6u1YNDEuaOzAPl8pABrU9KJ+YI0059m3ZxcHD+xl9NjPCGrYGICq1Wqg1xtYvGA2rdp2oIx7We7euU2vPgPo0aMrBl3Kxy69Xs/AgQM5ffo0lSpVomTJkgB4enpSvHhxAMLDwwkPD6d27do2523dujXvvPMOAA4ODrRr1w5fX18GDx4MQIUKFfj111/566+/HjtF4FGcnJyU/cqVK5elYwDExMSwdu1a5b05ODjw9ttvs2fPHho3bszcuXNxdnZmwYIF2NvbA+Du7k779u0zdR6TyURcXByjR4/mjTfeACAgIIDY2FgmTpzInTt3KFSoUIaOFRISgr29PUuWLMHJyQmAmjVr0rBhQxYsWMDIkSOV0QlXrlxh6NChXLt2jaNHjzJy5EhMJhPt2rV77DnMZjOhoaH4+voC4OfnR8OGDVm2bBkjR47k1KlT1KpViylTpqBWpzzLq127Nr///jt79+6ladOmHDx4EIPBQO/evdHpUqatuLm5cezYMaxWK7GxsYSFhfG///2P0aNHA1CnTh3c3NwYPXo077zzDuXLl08/QCGEQJIEQgghhHgOmM0WTl28x82IOAz6tB9vKlb0sfm+UKEi3Lx+HYAbd+O4fCOGpOT7w7kruhfgH1Bu7B925NBBNBoN9Ro0tGl/rfEbLF4wm6OHD1LGvSxjPvmMfE564mKjOX7+PJcuXeKPP/4AUm5wH2X79u34+/vj7Oxs0+7v7698XaBAASDlRjNVvnz5gJSb9LyUP39+JUEAULRoUQBliseePXuoW7eukiCAlPdWrFixTJ1Hp9OxcOFCAG7evMmFCxe4ePHiY/vYYrHYDN1XqVRoNBr27NlDQEAABoOB5ORkICVpUrVqVXbt2gVAtWrVmDNnDtWrVwdS6hO0b9+eqKgoZs6cSdu2bbFYLDYjGFKPD1CiRAklQQBQqFAhKlWqxP79+wFo0aIFLVq0wGg0cuHCBS5dusTJkycxm80kJSUpMUyfPp1mzZrRuHFj6tWrR506dZRRJ4cOHSIxMZGgoCDlfQDKygo7d+6UJIEQ4rEkSSCEEEKIZ9rxc3fYsOsC/1yOJMGYhEatJjrORFSsUdlGbzDY7KNWq7BYLZwJv8fsNUdtEgSBlV6mfSMPvp/56HNGx0Th6uam3Pylyp8/5cY9NjYWO42aq5fPMGD8pxw7dgx7e3vKlSvHyy+/DPDYofDbtm2jbt26adpTn3A/6MEb7afFwzGp/i32mHpzHhERoSQ5HpSVFQJ27NjBZ599xvnz53F0dKRChQpKIcH0+viDDz7g+++/V74vVqwYv//+O5GRkWzYsIENGzak2Se1MGOBAgVo0KABAPHx8crr9erVY9euXdy5c4ehQ4eyb98+5bWAgACWLVv2yPdXoEABrv+bsEpMTOTTTz/lhx9+IDk5meLFi+Pv749Wq1Xei7+/P/PmzWPJkiUsXryYefPmUbBgQfr27Uvnzp2JjIwEoHfv3un2161btx7Rk0IIkUKSBEIIIYR4Zh07d4dlG09y/U4sBp0WF0cdyWYrZrOF81ejOHcl8pH7mpLMzPj2EMnm+zeSLeuXo1H1kspN7aO4OLsSFRmJ2Wy2SRTcvXsHgPz586HTJNO3T288PDxYv3497u7uqNVqtm3bxqZNmx55bKPRyN69e23mmGfVwzfnqeLi4rJ97OwoWrQod+7cSdN+9+5d3N3dgcfH7ujoCKTUCAgODqZhw4bMnTuXEiVKoFKpWL58OTt27Ej33AMGDKBTp07K96lD9p2dnalVq5YyneNBWm3KR+YDBw4QHh5Oy5YtbV43Go1oNBpcXV0ZN26cTf+mxgooNRkedPv2bSUJMWHCBDZt2sSXX35JrVq1lGRHzZo1bfYJDAwkMDCQhIQE9uzZw9KlSxk/fjx+fn64uLgAMHXqVEqXLp3mfLJUoxDiSaRwoRBCCCGeSQnGJH7cfo6rt2JwtrdDp1WjVqlS/q9WYbZY2XboSpr9rFYrV27FEhOfpCQIVGo1LxV0pHGNUk9MEAD4+lfGbDaz7Y8tNu1bfk15Ch1YuwaXL10kMjKSLl26UK5cOWWOeWqV/9Sb39T2VHv37sXNzQ0PD49M9khaqU/0b9y4obSdO3dOedqcnodHR+SGatWqsWPHDozG+6M9Tpw4YbMKQ+qoiQdjj4qK4ty5c8r3x48fx2g00rt3b0qWvJ/cSU0QpDeSoHjx4vj4+Cj/pfZzQEAAZ8+exdPTU3nN29ubJUuWsHnzZiBlmsSoUaO4cOGCcjyLxcKmTZvw9/dHp9Ph7u5uc/zUpAekFEe8fPmy8v3169c5dOiQMn3h4MGDVK9enYYNGyoJguPHjxMREaFcL5MmTaJ169ZYrVbs7e1p0KABI0eOBODatWv4+flhZ2fHzZs3beLQarV88cUXaVa6EEKIh8lIAiGEEEI8c+ISTJy4EMHpS/dwMNile2Ov1ai5cy/Bps1stvDNptOE37w/Z9/R3g4P95e4fO4kfx3cR/nyFXD+92nso1SvUZtKlasybdJ47ty+Rdlyr3D08EG++forWrRogccr5YmJicHJyYk5c+ag1WrRarVs2rSJ1atXA/fn56c++d28eTN169Zl+/bt6U41yAovLy8MBgMTJ05k8ODBxMXFMXPmTNzc3B65T2odhK1bt+Lq6kqFChVyJJYH9e3blw0bNtCzZ0+6d+9OdHQ0M2bMQK1WKz9LDw8PXnrpJUJDQ3FyckKlUjF37lybqQwVK1ZEq9UyZcoUunfvjslkYu3atWzduhWwnRLwJP3796d9+/b06dOHDh06oNfrWblyJVu2bGHmzJS5J+3bt+fbb7+lb9++9O3blzt37hAaGso///zD8uXLn3gOvV5Pv379GDJkCGazmRkzZuDm5kbXrl0B8PX1ZePGjaxYsYKyZcty6tQpZs+ejUqlUq6XGjVqsHjxYkaNGsVbb71FUlISCxYswM3NjRo1auDm5kbPnj2ZMWMGsbGxVK9enZs3bzJjxgxUKlWu/DyFEM8XGUkghBBCiGeG1WolKtZIVJyJe9FGzBYLWk36T/5VKjA/8CQ5ITGZkFVH2Hn0mtJWKJ89IzpX5e23O6HRanl/2CD27dn5xDhUKhWfTf6SZs1bsfq7b/hgxGC2b/2NoUOH8PnnnwMpN9thYWFYrVYGDx7MiBEjuHbtGl9//TWOjo4cOHAAgOrVq1OrVi2mTZvGpEmT2L59e5qlD7PK0dGRqVOnYjabCQ4OZsaMGQQHB+Pt7f3IfcqXL0+zZs1Yvnw5w4YNy5E4HlaqVCkWLlyI0Whk0KBBTJ8+nV69elGoUCFleL5Go2HmzJkULFiQoUOHMmHCBJo2bUqjRo1sjjNt2jRu3rxJv379lOUbly1bhkqlUvo4IypUqMDy5ctRqVSMGDGCQYMGcfv2bUJDQ5VzFixYkOXLl+Ph4cHkyZOZOXMmCQkJLFmyxKaA5KN4eXnRtm1bxo4dy4gRIyhZsiTffPONMt1g1KhRNGzYkC+//JI+ffqwatUq+vXrR7t27Th06BBms5l69eoxdepUzpw5w4ABAxg6dCj29vYsXbpUSf68++67jBo1is2bN9OrVy+mTJlClSpV+Prrr9MUwxRCiIeprJlZQPYFdezYMQB8fHyesGXuiY+P5+TJk3h6eirDz0TOkf7NXdK/uUv6N/dI3+auzPav2ZKSIEg0JmMFLl2PZs7ao+h1GnTatM89TEkWjElm+rbyxdlRR+iqI1y9Hau8Xra4K/1a+eLkoMvW+1CrwMleh7Nj9o6T057m63f37t3Y2dlRtWpVpS06OppatWoxYsQIunTpkofRZczT3L/Puqe9b5+GewMhcpNMNxBCCCHEUy8p2UJkTCKmB1YhKFHEmSIFHLh6OxY7R53NlAOr1UqcMYlihVLmtU9aup+o2PvL4VX1LEzXpl7YabM3/16tUuHipMPRYJet47xo/v77b2bOnMnQoUOpWLEikZGRLF68GGdnZ5o1a5bX4QkhxAstz6cbWCwWZs6cSWBgIJUqVaJXr16Eh4c/cvukpCSmTZumbP/2229z8uRJm212795Nq1at8PPzo0mTJqxfvz6334YQQgghconRlExEdIJNggBSljFsUqM0Bp2WyDgTpiQLFqsVU5KFyDgT9jotHiXz8cU3f9kkCBrXKEX3t7yznSDQqFW4OUuCICu6d+9O3759WbFiBT169OCjjz6iYMGCfPvtt8rQeyGEEHkjz5MEYWFhfPPNN3z66ad8++23WCwWevbsiclkSnf7sWPHsnbtWj777DPWrFlD/vz56dWrFzExKQWIzp07R58+fQgMDGTt2rW0bduWESNGsHv37v/ybQkhhBAiB8QlmIiINtosU/igCqXz06lxBYoVcsKYZCY6zoQxyUyxQk5UdC/AjzvOY0wyAylP/Ts1qUDL+uVQZ2AFg8ex06jJ52LAXi8JgqxQq9X079+fTZs2cfToUfbs2cPMmTMpVapUXocmhBAvvDydbmAymVi0aBHDhg2jfv36AEyfPp3AwEB+/fXXNMPNwsPDWbNmDXPmzCEwMBCA8ePH06JFC44fP07NmjX56quv8PDwYMiQIQCULVuWEydOsGDBgjRrzAohhBDi6WS1WomOMxGXmMSTqidVKJ2fV0rmI/xmDLHxSTjaazlw8hZb9t9fak6v09C7hQ8V3QtkOza9nQY3Zz1aTZ4/axFCCCFyXJ4mCU6dOkVcXJzNzbuLiwteXl7s378/TZJg586dODs72ywL5OLiwu+//658f+DAARo2bGizX40aNZgwYQJWqzVDax+nx2q1ZmoZnZyWuuxN6v9FzpL+zV3Sv7lL+jf3SN/mrkf1r9UKUXEmEoxPThA8qLCbHW5OapZvOsPRsxFKu6uTjl5veVKskEO2fpYqFdjr7bC3U2MyJpL+mMenh1y/uUv6N/c87X2bnXsKIZ4FeZokuHHjBgAvvfSSTXvhwoWV1x504cIFSpQowa+//sq8efO4efMmXl5ejBo1irJlyyrHLFq0aJrjJSQkcO/evSzPc0tKSkpT+yAvXLx4Ma9DeK5J/+Yu6d/cJf2be6Rvc1dq/6pUKtQaO2ITzcQnGMnsAkzxRgvr90dyMzJJaSvooqVZNRcSo29wLjrrMWq1Gpzs9RjsrFxPTs76gfKAXL+5S/o39zzNfavTPV2rmQiRk/I0SZCaHXz4l0yv1xMVFZVm+9jYWC5dukRYWBgjRozAxcWF2bNn07FjRzZs2ECBAgVITExMc7zU7x9V5yAj7OzsKFeuXJb3z66EhAQuXrxI6dKlsbe3z7M4nlfSv7lL+jd3Sf/mHunb3PVw/5qSLETFmXBKNmf6WLfuJbDihxPcjbqfIKhQyo2ub3hg0GW/QKGzgw4Hw7O1KJRcv7lL+jf3PO19e/bs2bwOQYhclad/7QwGA5By8576NYDRaEz3HwStVktsbCzTp09XRg5Mnz6devXq8f3339OzZ0/0en2aZEDq99n5R0alUj0V67Ta29s/FXE8r6R/c5f0b+6S/s090re5y97eHotKS0JSElo7HdpM1gI8c/kes9ceIz7x/hP+wErFaN/oFTTq7NUN0KhVuDrpnukChXL95i7p39zztPatTDUQz7s8TRKkTjO4desWJUuWVNpv3bqFh4dHmu2LFi2KVqtVEgSQkmgoUaIEV65cUY5569Ytm/1u3bqFg4MDzs7OufE2hBBCCJFFWq2WmPgkzFgyVX8g1b6/b7B0wwmb1Q9aNSjHawEls/1BXqtR4+asR2+XvZEIQgghxLMkT8vyVqhQAScnJ/bu3au0RUdHc+LECapVq5Zm+2rVqpGcnMyxY8eUtsTERMLDw5Ulc6pWrcq+ffts9tuzZw+VK1dGnc2nCUIIIYTIOVYrxButxCaYMp0gsFqtbNh1gUU//a0kCLQaNb1aeNOoeqlsJwh0WjX5XQySIBBCCPHCydO7Zp1Ox9tvv83UqVP57bffOHXqFEOGDKFo0aI0atQIs9nM7du3SUxMBFISALVq1WLkyJEcOHCAs2fPMmLECDQaDc2bNwegc+fOHD16lKlTp3Lu3DkWLVrEL7/8Qs+ePfPyrQohhBDiAUnJFu7FGImJN2Y6QWA2W1i28SQ/bj+vtDna2zGkgz9VKhTJdmwGOw35XQzYaeXhghBCiBdPnv/1GzRoEG3atGH06NF06NABjUbDwoULsbOz4/r169SpU4cNGzYo28+aNYuAgAAGDBhAmzZtiI2NZenSpcqqBeXLlycsLIxt27bRokULVq1axZQpU2yWWRRCCCFE3jGakrkbnUCiKTnTKxgkJCYTsuoIu45eV9oK57NnZJeqlC3ulq24VIBBpyWfiwGNJs8/IgkhhBB5Is/L9Go0GoYPH87w4cPTvFa8eHFOnz5t0+bk5MTYsWMZO3bsI49Zt25d6tatm9OhCiGEECKb4hJMRMclYclCAYKIqERCVh/m2u04pa1scVf6tfbDyT57hQVVKnAw2OHqqJOiZEIIIV5oeZ4kEEIIIcTzz2q1Eh1nIi4xKUsFCi/fiCZ09RGiYu+vYFTVswhdm3pip81e3QCVCpwMOlycZN1zIYQQQpIEQgghhMhVZouVqFgjicZkspAf4OjZOyz84TjGJLPS1qRmKd6qWxZ1Np/6q1Xg7KjDyV4SBEIIIQRIkkAIIYQQuSgp2UJUrNHmBj8ztv51hZWbTyujD9QqFR0aexBYqVi2Y1OrVLg66XAwZG+qghBCCPE8kSSBEEIIIXKF0ZRMZKyJZLMl0/tarFbW/nGWLfsuK20GnYZeLXyo6F4g27Fp1CkJAnu9JAiEEEKIB0mSQAghhBA5Lj4xieg4E2ZL5icYmJLMLP75bw6dvq20uTnrGdDGj+JFnLMdm0atIp+zHr1OPgYJIYQQD5O/jkIIIYTIUdGxJuISTWQhP0B0nInZa45w4Vq00laisBP92/qRz9mQ7di0GhVuzgb0dtkrdiiEEEI8ryRJIIQQQogcYbFYiYozkmBMztIKBjfuxhHy3WHuRCUqbRXdC9CruTcGffY/sthp1ORz0Wd7NQQhhBDieSZJAiGEEEJkW7LZQmRM1gsUnrl8j9lrjxKfmKy01fUvxv9eewWNWp3t+HRaNflcDGg12T+WEEII8TyTJIEQQgghssWYZCYqxkhSFgoUAhw8dZtvt5wl2Xx/+EGrBuV4LaAkqmwucQhgsNPg5qxHIwkCIYQQ4okkSSCEEEKILEswJhEVm7UChVarlf1nYtl7+qbSptWoeedNL6pUKJLt2FSAXqcln7MetTr7yQYhhBDiRSBJAiGEEEJkSUycidiErBUoNJstrNxyjr2n45Q2J3s7+rX2pWxxt2zHplKBg8EOV0ddjoxGEEIIIV4UkiQQQgghRKZkt0BhQmIyc9cd49TFCKWtcD57BrarRKF8DtmOT6UCJ4MOFyddto8lhBBCvGgkSSCEEEKIDDP/W6AwMYsFCu9GJRC66gjX7twfQVDmZWeC2/rjZG+X7fjUKhXOjnY42UuCQAghhMgKSRIIIYQQIkNMSWaiYo2YkrNWoPDSjWhCVx0hOs6ktJV/WU+vlhVzJEGgUatwddJhr8/+sYQQQogXlSQJhBBCCPFE2SlQCHD07B0W/HAMU9L9BEPDasXwKJyEnTb7qw5oNCryOenR6+SjjRBCCJEd8pdUCCGEEI8VG28iJj4JS1YKEABb/7rCys2nlfoFapWKjk08qPJKfs6dO5ft+Ow0atyc9ejsNNk+lhBCCPGikySBEEIIIdJltVqJijMRn5iUpQKFFquVtb+fZcv+y0qbQaehd0sfvMoUICEhIdsx6rRq3JwNOTIaQQghhBCSJBBCCCFEOsxmC5GxRowmM1kZP2BKMrP4p7859M9tpS2fs57gtn4UL+ycIzEa7DS4OevRaCRBIIQQQuQUSRIIIYQQwkZSspnImKwXKIyOMzF7zREuXItW2koUdqJ/Wz/yORuyHZ8K0Ou0KQkCtSrbxxNCCCHEfZIkEEIIIYQi0ZRMZKwRszlr9Qdu3I0jZNUR7kTen0rgXbYAPd/yxqDP/scOlQrs9VrcnPSoVJIgEEIIIXKaJAmEEEIIAUBsgomYuKwXKDxz+R6z1x4lPjFZaavrX4z/vfYKGnX2pwSoVOBosMPVSZ/tYwkhhBAifZIkEEIIIV5wVquV6DgTcVksUAiw7+8bLN1wguQHRiC0alCO1wJK5sgTf5UKnO11ODvqsn0sIYQQQjyaJAmEEEKIF5jZYiUyxojRlJylAoVWq5WNuy7y447zSptWo+adN72oUqFIjsSoVoGTJAiEEEKI/4QkCYQQQogXVFKyhciYxCwXKDSbLSzfdIpdR68rbU72dvRr7UvZ4m45EqNaBc6OOpzsJUEghBBC/BckSSCEEEK8gIz/FihMzmKBwoTEZOauO8apixFKW+F89gxsV4lC+RxyJEa1SoWLkw5Hg12OHE8IIYQQTyZJAiGEEOIFE5eYRHSsKcsFCiOiEglZfZhrt+OUtnLFXenb2g8n+5y5oVerVLg66XCQBIEQQgjxn5IkgRBCCPGCyIkChZdvRBOy6gjRcSalrapnEbo29cROq8mRODXqlASBvV4SBEIIIcR/TZIEQgghxAvAYrESGWsk0Zi1AoUAR8/eYeEPxzEmmZW2JjVL8VbdsqhzYAUDSEkQuDnrMejkI4oQQgiRF+QvsBBCCPGcSzZbUlYweODmPrO2/nWFlZtPKyMQ1CoVHZt4UMevWA5FmZIgyOesRy8JAiGEECLPyF9hIYQQ4jmWUqDQRLI5aysYWKxW1v5xli37LittBp2G3i198CpTIKfCRKtR4eZsQG+XM1MWhBBCCJE1kiQQQgghnlPxiUlEx5kwW7I2wcCUZGbxT39z6J/bSls+Zz3Bbf0oXtg5R2JUqUBnpyG/iyHHahoIIYQQIuskSSCEEEI8h2LiTMQmmHg4P2C1WlFloH5AdJyJ2WuOcOFatNJWorATwW0r4easz3JcD5/f3t5APid9niUIMtofQgghxItCndcBCCGEECJrgoKCGDVqlE2bxWLlXkwiMekkCHb+uY2J4z+2aft9yybat25K4wY1+GLyBIYM6E1w3x5MXrrfJkHgXbYA73WqkuUEwe1bN3l/+CBu3riutBl0WpwNat57bwizZ8/O0nEz6sqVK3h4eLB27VoAoqOjGTFiBAcOHFC26dy5M507d053+7Vr1+Lh4cGVK1dyNc7seDD+7Fq1ahUeHh5p/vvkk09y5Pi5Ye/evXh4eLB3714ARo0aRVBQUB5HJYQQzx4ZSSCEEEI8o0JCQnByclK+TzZbiIoxkviIAoWrv12epm3m9EkUL16SUR+Oo2Chwoz/5GOu3o7DXDpR2aaufzH+99oraNRZf7Zw8MBe9u7eqXxvsNNgcFBz9VIC+/bt4913383ysbPi5MmT/PDDD7Ru3Vpp+/jjjx+5ff369Vm5ciWFCxf+L8LLcydPnqRMmTJMnDjRpr1gwYJ5FJEQQoj/iiQJhBBCiGeUl5eX8rUxyUxUjJGkTBYojI6KomqrGlSqXJW9f9/gyq1YZQUDgFYNyvFaQMkcHZJv0GlwczZgTEzg5MmTODo64unpmWPHz6py5co98rX8+fOTP3/+/zCavHXy5El8fHyoVKlSXocihBDiPybTDYQQQohnVOp0g7PnL+Lr7cXmzZsYO3oETV8LpPnrDZg66VMSEhIAGDKgN0cOH+TI4YME1anC4b8OEFSnCgBLF88nqE4V5q3coSQI7LRqerXwpn6lonz91QK6dmxF46CadG7fghVfL8FiuZ+MMJvNrPh6Cd07t6NJUC3eaFibAX3f4dBf+wH4ZcOPTP5sHAAd277J5M/GolGnJB0OHz5M7dq1UalUynDx3bt307lzZ3x9falfvz6rVq3i1q1bDBgwAH9/f+rVq8eSJUuU8z9qKkB60zEgZVh6ly5dAOjSpYsyRP9xw/UfPseoUaPo1q0ba9asoXHjxnh7e9O8eXO2b99us9+hQ4fo1KkTlSpVon79+nz11Vd069ZNievhIfKpHo4lMTGRadOm0ahRI7y9valcuTLvvPMOJ0+eTDfex/nnn3/o06cPlStXpnLlygQHBxMeHq68brVaOX36dJYSNxEREbz33nvUrl0bHx8fmjdvzrp165TX165di4+PDwcOHKB169b4+PjQuHFjfv/9d86fP0/Xrl3x8/PjtddeY/369TbH3r9/Pz169KBatWp4e3sTFBTErFmzbK5FIYQQ2SdJAiGEEOIZlpRsJjrOBMD0KZ9RpOhLfPr5NP7XsTMbf/6Br79aAMDg90ZR7hUPyr3iQcicJZT3qEDInCUAlPerR4nawWj0LgBoNCqGdKhMZY/CfDhyCN8uX8obzVowYdJ06jVoyML5YUyf8pkSw/w5s1i2ZD7Nmrdi0rRZvDdiNDHRUYwbPZLExARq1Aykc9ceAMyaNYvg4GBl39QkwYOGDh1KUFAQc+fOpUyZMnz88cd06dKF8uXLExYWhq+vL59//jlHjx7NUp9VrFiRMWPGADBmzJjHTjN4nOPHj7Nw4UIGDRpEaGgoGo2GgQMHEhUVBcC5c+fo1q0bAF988QUDBw5k3rx5HDx4MNPnGjFiBGvWrKF3794sWrSI999/nzNnzvDee+9htWZ89YoLFy7Qvn177t69y6RJk5gwYQLh4eF06NCBu3fvAnD58mXi4uI4duwYjRs3pmLFijRu3NjmZv9Rhg8fzrlz5xg3bhzz58/Hy8uLkSNHsmfPHmWb5ORk3nvvPdq3b8/s2bOxt7dn2LBh9O3bl/r16zNnzhwKFy7MyJEjuXHjBgCnTp2iW7duuLm5MX36dGbPnk3VqlUJCQlh48aNmetMIYQQjyXTDYQQQohnkNVqxWyxkpRsUZ7+V69Vh34DhgBQuWoAB/bvZc+uHfTqO5DSZdxxdEipX+Dl7QNAmXIpT4rvxttRsEQpAHR2aooVcsK9mCt7d+/k4IG9jB77GUENGwNQtVoN9HoDixfMplXbDpRxL8vdO7fp0TuYVm3aK/HZ6fWM/XA458+epaKPD2VKl045t5cXxYsXB+Dq1avcunWLGjVq2Ly31q1b88477wDg4OBAu3bt8PX1ZfDgwQBUqFCBX3/9lb/++gtfX99M952Tk5MytaBcuXKPnWbwODExMaxdu5aSJUsqsb799tvs2bOHxo0bM3fuXJydnVmwYAH29vYAuLu70759+8cdNg2TyURcXByjR4/mjTfeACAgIIDY2FgmTpzInTt3KFSoUIaOFRISgr29PUuWLFHqWdSsWZOGDRuyYMECRo4cqYxOuHLlCqNGjUKr1bJu3TpGjhyJyWSiXbt2jzz+vn37CA4OpmHDhkqcbm5u6HQ6ZRuLxULfvn1p27YtkFJEcsiQIXTt2lX5uTs7O9O6dWuOHz9O0aJFOXXqFLVq1WLKlCmo/62NUbt2bX7//Xf27t1L06ZNM9OlQgghHkOSBEIIIcQzwmKxcv5qFHGJSWAlzRPkihV9bL4vVKgIN69fJz13oxIIXXXEpq1ccVd0l52VAoVHDh1Eo9FQr0FDm+1ea/wGixfM5ujhg5RxL8uHH08AIPLePcIvX+TKlXB270wZdp+UbMLZXoe9Ie1Hjj///JPy5cvj7Oxs0+7v7698XaBAAQD8/PyUtnz58gEpN+l5KX/+/EqCAKBo0aIAyhSPPXv2ULduXSVBACnvrVixYpk6j06nY+HChQDcvHmTCxcucPHiRf744w8gJYnwMIvFQnJyMmazWfm/RqNhz549BAQEYDAYSE5OBlKSJlWrVmXXrl0AVKtWjTlz5lC9enUcHBwACAwMJCIigpkzZ9K2bVssFovN9adSqdBoNFSvXp1Zs2Zx4sQJAgMDqVevHiNHjkwT35N+xm5ubkBKAgGgRYsWtGjRAqPRyIULF7h06RInT57EbDaTlJSUqf4UQgjxeJIkEEIIIZ4BR87c5oft54iLTyIxyUx0rJHoOBNRsUZlG73BYLOPWq3CYk07X/vSjWhCVx1RpikAVPUsQtemnozYc38mYnRMFK5ubmg0Gpv98+dPuamLjY0F4PSpE3w5bSKnT/6NwWCgVJmyFCmScsPsoNfi7KgjPX/++We6hfEeXLEh1YM32k+Lh2NKLe6YOkc+IiJCuQF+UFZWCNixYwefffYZ58+fx9HRkQoVKig38OlNN/jggw/4/vvvle+LFSvG77//TmRkJBs2bGDDhg1p9kktzFigQAEaNGiQ5vV69eqxa9cu7ty5w9ChQ9m3b5/yWkBAAMuWLWP69OnMmTOHjRs3smnTJtRqNbVq1eKTTz6xSY5k9mecmJjIp59+yg8//EBycjLFixfH398frVabqekWQgghnkySBEIIIcRT7siZ2yz66Th2GjVqtYqkpGS0WjVms4XzV6M4dyUyw8eKTTAxbflBTEn3kwflSrjR/a2KqB9awcDF2ZWoyEjlKXSqu3fvAODq6kZcXCwj3xuAe9nyLFq2ipKlSqNWq9m3eyfbt/6GvT79jxpGo5EDBw7QrFmzTPRE+h6+OU8VFxeX7WNnR9GiRblz506a9rt37+Lu7g48PnZHR0cgpUZA6hD+uXPnUqJECVQqFcuXL2fHjh3pnnvAgAG0bt2aixcvUrp0aVxcUupNODs7U6tWLWVY/4O02pSf1YEDBwgPD6dly5Y2rxuNRjQaDa6urowbN86mf1NjdXZ2Zvjw4QwfPpzz58/z22+/ERYWxrhx45g3b96TO+0RJkyYwKZNm/jyyy+pVauWkiCpWbNmlo8phBAifVK4UAghhHiKWSxWNu66iFajRq2GuIQkVCoVOm1KwsBssbLt0JUnHwiIijNx7U6ckiBITQpUKJU/TYIAwNe/MmazmW1/bLFp3/JrylNob99KXL50keioKFq37UDpMu6o1Wo0ahWH/9rzb/z/nktt+5Fj7969uLq62gzXz6rUp9KpRe4gpWhgZGTkI/d5eHREbqhWrRo7duzAaLw/2uPEiRM2qzCkF3tUVBTnzp1Tvj9+/DhGo5HevXtTsuT95ShTEwTpPUkvXrw4FStWxN3dnYoVK+Lh4QGkPPE/e/Ysnp6e+Pj44OPjg7e3N0uWLGHz5s1AyjSJUaNGceHCBeV4FouFTZs24e/vj06nw93dXdnfx8cHd3d3rl69Sr169fjll1+AlPoLvXr1olatWly7di1bfXnw4EGqV69Ow4YNlQTB8ePHiYiIkNUNhBAih8lIAiGEEOIpdvF6FBHRiSQlW0g0p70Z0mrU3LmX8NhjWKxW1v5+ljsxVkyxt4m/cxaXgiXo3646A3569H7Va9SmUuWqTJs0nju3b1G23CscOXyQFV8vofHrzShdxp3Y2BgcHR35eukiNBoNOjs7dv/5B99/vxa4Pz8/9Un25s2bqVu3Ltu3b0+zqkFWVa9eHYPBwMSJExk8eDBxcXHMnDlTmdeentQ6CFu3bsXV1ZUKFSrkSCwP6tu3Lxs2bKBnz550796d6OhoZsyYgVqtVm70PTw8eOmllwgNDcXJyQmVSsXcuXNtht5XrFgRrVbLlClT6N69OyaTibVr17J161YA4uPjMxxT//79ad++PX369KFDhw7o9XpWrlzJli1bmDlzJgDt27fn22+/pW/fvgwePBh7e3u++eYb/vnnH5YvX/7IYxcrVoyiRYsyfvx4YmNjKVmyJMePH2fbtm306dMnCz14n6+vLxs3bmTFihWULVuWU6dOMXv2bFQqlXKNCSGEyBkykkAIIYR4SpnNFm5GJHA7Mo6kZHO626hUYH7MnGyrFeZ/f4wt+y/jVroWKrWGq/sWUq+cEa8yaefL2x5bxWeTv6RZ81as/u4bPhgxmG1/bKFnnwEMfz9l2UAnJ2c+/fwLsFoZ99EoPvt0DDdv3uDrr7/G0dGRAwcOACk38rVq1WLatGlMmjSJ7du3U6dOnSz2jC0XFxdmzZqF2WwmODiYGTNmEBwcjLe39yP3KV++PM2aNWP58uUMGzYsR+J4WKlSpVi4cCFGo5FBgwYxffp0evXqRaFChZTh+RqNhpkzZ1KwYEGGDh3KhAkTaNq0KY0aNbI5zrRp07h58yb9+vVTlm9ctmwZKpVK6eOMqFChAsuXL0elUjFixAgGDRrE7du3CQ0NVc5ZsGBBli9fjoeHB+PHj+fdd98lISGBJUuW2BQXTE9ISAiBgYHMmDGD7t27s2LFCgYMGGCz7GVWjBo1ioYNG/Lll1/Sp08fVq1aRb9+/WjXrh2HDh3CbE7/90MIIUTmqaxS7eWJjh07BoCPj88Ttsw98fHxnDx5Ek9PT2WYncg50r+5S/o3d0n/5p687NukZDORMUbOXolk9pqj6HUadNq0uX1TkgVjkpm+rXwp9ZKLzWvRcSbCVh/h4vVopa1EYSeC21bCzVmfo/HqtGrcnPXYaTM+jP95v3Z3796NnZ0dVatWVdqio6OpVasWI0aMoEuXLrl6/ue9f/Oa9G/uedr79mm4NxAiN8l0AyGEEOIpk2hKJjLWiNlspXhhZ4oUcODq7VjsHHXKMHVImYseZ0yiWCEnShSxXUbwxt04Qr47zJ2oRKXNu2wBejb3xqDL2T//Oq2afC4GtBoZoPigv//+m5kzZzJ06FAqVqxIZGQkixcvxtnZOUcKNgohhBC5QZIEQgghxFMkLsFEdFwSln8H+qnVKprUKM3yTaeIjDPhqLdDq1WRnJySILDXaWlSozRq9f3kwT+X7zFn7VHiE5OVtrr+xfjfa6+gUefsjbzeTkM+Zz0aSRCkkVo/YMWKFVy/fh0HBwcCAgL4/PPPleUGhRBCiKeNJAmEEEKIp4DVaiU6zkRcYhIPTwSsUDo/nRpX4Jc9F7l5N554owWNWk2xQk40qVGaCqXv33Du/fsGS9efwGy5f5DWDcrRMKCkzSiEnGDQaXBzNqBR5+xxnxdqtZr+/fvTv3//vA5FCCGEyDBJEgghhBB5zGyxEhljxGhK5lGFgiqUzs8rJfMRfjOG2PgknBzsKFHEWRlBYLWmLJX4447zyj52WjXdmnlRpUKRHI1XBRj0Wtyc9DYjGIQQQgjx7JMkgRBCCJGHUgsUmpKfvNa7Wq1KU5wQUlZBWP7LKXYdu660Odnb0b+NH+7FXHM0XpUKHAx2uD5UH0EIIYQQz4csJwnOnTvHzp07uXXrFp07dyY8PJwKFSrg5OSUk/EJIYQQz60HCxRmVUJiMnPXHePUxQilrUh+Bwa09aNQvpytCq5SgZNBh4uTLkePK4QQQoinR6aTBBaLhTFjxrBmzRqsVisqlYrXX3+dsLAwLl++zNdff03RokVzI1YhhBDiuRGbYCLmgQKFWXE3KoHQVUe4didOaStX3I1+rX1xtLfLiTAVahU4O+hwcpAEgRBCCPE8y3Qp4rCwMH766SfGjx/Pzp07sf774Wb48OFYLBamT5+e40EKIYQQzwur1UpUrJHoOFO2EgSXbkQzaekBmwRBNa8iDG7vnwsJAhWuTnpJEAghhBAvgEwnCdasWcOgQYNo3bo1bm5uSrunpyeDBg1i586dORmfEEII8dwwW6xERCcSl5B2BYPMOHrmNtOWHyQ6zqS0vVGrNN3frIidNmeXItSoVeRz0eNgyNnEgxBCCCGeTpmebnDnzh08PT3Tfa1IkSJER0dnOyghhBDieZOZAoWP88fBcL7b8o+SZFCrVHRqUoHafi/nQJS2NBoV+Zz06HVS51gIIYR4UWT6cUOpUqXYtm1buq/t27ePUqVKZTsoIYQQ4nmSaErmblRithIEFouV77b8w8rN9xMEBr2GAe38ciVBYKdRk9/ZIAkCIYQQ4gWT6b/8Xbt2ZcyYMSQlJdGgQQNUKhWXLl1i7969LFq0iFGjRuVGnEIIIcQzKTbeREx89goUmpLMLPzxb46cua205XPRM6BNJYoVzvlVhXRaNW7OhhyfuiCEEEKIp1+mkwRt27YlIiKC2bNns2LFCqxWK0OHDsXOzo6ePXvSoUOH3IhTCCGEeKZYrVai4kzEJ2av/kB0nJGw1Ue5eP3+dL4SRZwJbuOHm7M+ByK1pdOqyediQKuRBIEQQgjxIsrSGMI+ffrQqVMnDh06RGRkJC4uLvj5+dkUMhRCCCFeVGazhchYI0aTmWzkB7h+J46QVYe5G5WotPmULUCP5t4YcmEagMFOg5uzHo0kCIQQQogXVqY/Bbz//vuEh4fj5OREYGAgb775JvXq1cPNzY3z58/Tt2/f3IhTCCGEeCaYksxERCeSmM0EwelL95iy7IBNgqCufzH6tvbNnQSBToObi0ESBEIIIcQLLkOfMq5du6Z8vW7dOho2bIhGo0mz3fbt29m1a1fORSeEEEI8QxKMSUTFmjBbspMegL3Hr7N0w0nlOCqgVYNyNAwoiUqlyoFI71MBBr0WNyc9anXOHlsIIYQQz54MJQnGjRvH9u3ble8HDBiQ7nZWq5XatWvnTGRCCCHEMyQmzkRsgons5AesVisbdl7gpz8vKG12WjXvNKtI5QqFcyBKWyrA3pCSIMjp5IMQQgghnk0ZShJ88skn7Nq1C6vVygcffEC/fv0oWbKkzTZqtRoXFxeqV6+eK4EKIYQQTyOLxUpUnJEEY3K2ChQmmy0s/+UUu49dV9qc7O3o38YP92KuORCpLZUK7PWSIBBCCCGErQwlCYoUKULLli0BUKlU1K9fn3z58uVqYEIIIcTTLtlsISrGSGKSOVvHiU9MYu73xzh96Z7SViS/AwPa+lEon0N2w0xDpQIHgx1uTjm/OoIQQgghnm0ZShLs378fLy8vHB0dKV68OGfPnn3s9tWqVcuR4IQQQoinlTHJTFSMkSSzJVvHuRuVQOiqI1y7E6e0lSvuRr/Wvjja22U3zDRUKnA02OEqCQIhhBBCpCNDSYLOnTvz3Xff4evrS+fOnVGpVFgfGlOZ2qZSqTh58mSuBCuEEEI8DeITk4iOy36BwkvXowldfYToOJPSVs2rCF3e8MJOm/OrDKhU4GTQ4eKky/FjCyGEEOL5kKEkwdKlSylbtqzytRBCCPGiyokChQBHz9xmwY/HMSXdH4nwRq3SvBnonis1AlQqcLbX4ewoCQIhhBBCPFqGkgQBAQHpfi2EEEK8KHKqQCHAHwfD+W7LP8px1CoVnZpUoLbfy9kPNB2SIBBCCCFERmUoSRASEpLhA6pUKoKDg7MckBBCCPE0UalUmC1W7kYlYErOXv0Bi8XKmj/O8Nv+cKXNoNfQu4UPXmUKZDfUdKmw4uKgx8lBEgRC5KSHp96Kp0vqNGghROZlaMJjSEhIpv4TQgghnhcqtZZZoXNYunRJto5jSjIzb90xmwRBPhc9wztVzbUEwYafvmfh3JlPVYLg9OnTtGjRAm9vb1q1apXX4eSaEydO4O/vj8lkevLGT4krV67g4eHB2rVr8zqUp05QUBCjRo1Svg8LC7OZgjtr1iw8PDzS3f5Z6NeH43+W3bhxg969e3P16tXHbrd27Vo8PDy4cuXKfxRZ7vn777/p1asXNWrUoHr16nTv3p2///47r8MSz7AMjSQ4depUlg5+7do1ChcujFabodMIIYQQT5VEo5mouCS+WjSPLu/0zvJxouOMhK0+ysXr0UpbiSLOBLfxw805d1YZUKtULF+2iBrVq+fK8bMqNDSUa9euERoaioNDzi/v+LTYtm0bNWrUQKd7ehI0IufMmDGDPn36KN+3bduWwMDAdLctXLgwK1eupGTJkv9VeC+0Xbt2sW3btrwO4z9z6dIl3n77bby9vZkwYQIqlYpFixbRsWNHvv/+e9zd3fM6RPEMyvnSyf8ym828+uqrnD59OrdOIYQQQuQKq9VKdKyJyDgjRlNSto51/U4ck5YesEkQ+JQtwHudKudqgsDVSYf6KRxqe+/ePV555RXq1atHxYoV8zqcXLN9+3bq1auX12GI/0jRokWpVKlSuq/pdDoqVapE/vz5/9ugxAth2bJl2NvbM3fuXF599VWCgoKYN28e9vb2fP3113kdnnhG5VqSAGSulhBCiNx1/PhxunbtSpUqVfD396dbt24cPnxYef3AgQO8/fbb+Pn5ERAQwMiRI4mIiFBeX7t2LV5eXhw5coT//e9/+Pj4UL9+A0LC5qWsYGCxMqhvVwCWLp5HUJ0qyr4Xzp/lgxGDadaoLs0a1eWj99/j2tX7w1YP/3WAoDpVGDPtG45smsWZDR9ybvMnGCK206tFRQy6lFF2SUlJLJofRqe2b9EkqBbdO7dj08afbN7nzh1b6dvjbRoH1aT1W40I+XIKCQkJ6faJWqXCzVlHszcac/XqVb7//ntlSG3q+121ahW1a9cmICCAs2fPYjabmTdvHs2aNcPX15dKlSrRvn179uzZoxx31qxZvPbaa2zdupU333wTb29vGjduzLp162zO/9VXX9GkSRN8fHwIDAxk7NixxMbGAuDh4cG+ffvYv38/Hh4e/Pjjj0DKk7BBgwZRu3ZtKlWqROfOnTl48KByzNTh2osXL6ZJkyb4+fmxZs0aZs2aRZMmTdi8eTPNmjXDx8eH5s2bc+jQIQ4fPkzbtm3x9fWlWbNm7N692ybOf/75hz59+lC5cmUqV65McHAw4eH3p4Ls3bsXDw8Pvv32Wxo0aEDlypXZuXMnERERvPfee9SuXVs538N9EBUVxZEjR6hbty6QspT0qFGjmDNnDrVq1aJKlSr0798/zXDoLVu20LFjR/z9/fH29qZJkyYsX778iTEBrFq1ilatWlGpUiV8fX1p3rw5mzdvVvZNHVp95MgRWrZsia+vL2+++Sa//PJLmmvo9u3bDBo0CH9/fwICAvjoo4+Ii4tTXg8KCuKzzz6ja9eu+Pr68uGHHwIpI08HDBhAjRo1qFixIoGBgYwfP57ExERl3507d9KuXTv8/f2pVq0a/fr149y5c2n6oVWrVvj4+FC7dm3Gjx9PfHx8mjgf5uHhwYoVKxg1ahRVqlQhICBAOf+kSZOUodgffvghRqMRePRUgFGjRhEUFPTI8wDMnTuXjh07Ao8frv/wOdL7d6dBgwYsXLjQZr9bt24xZMgQAgICqFatGmPGjGH69Ok2cXl4eDBr1iyb/dKLJb3rY+PGjY/tz4elXn+7d++mc+fO+Pr6Ur9+fVatWsWtW7cYMGAA/v7+1KtXjyVLlqTZ788//6RTp074+vrSqFEjvvnmG5vje3h4sHz5cj788EPq1atH9+7dGTFiBHfu3LHZ7nHXx9q1a3n//fcBePXVV22miDzKX3/9pUyBatasGRs2bLB5/cqVK4wYMYI6depQsWJFatasycyZM4mJiVG2edLfInjy36P0fPTRR9SuXRuz2WzTPmHCBKpXr05SUhLu7u50797dZmSWg4MDRYsW5fLly098/0KkJ1eTBEIIIURuiY2NpWfPnuTLl49Zs2Yxffp0EhIS6NGjBzExMezfv59u3bphMBj48ssv+eCDD9i3bx9dunSxuWmxWCy8++67vPHGG8yePRdvHz9CZn7Bvr27ABgy4iMA3mjWnJA5SwAIv3yJgX27c+/ePUZ+OJZhoz7i+rWrDOrfnXv3Uj70nbhwF4CL+77GPn8ZigW8Q5UaDTi662c2rf9ROf+EcR+y6tuveePNFnw2+UuqBdRg0oSx/LY55ebtt1838tH771GyVGk+/WwaXbv3ZvOmDXw0amiaZLxGnZIgsNfbERISQqFChahXrx4rV66kcOHCQMpIv0WLFjFhwgTef/99ypYty9SpUwkLC+N///sfCxYs4NNPPyUyMpLBgwfbJCNu377NJ598QpcuXZg3bx7Fixdn5MiRyk3ezz//zJQpU+jUqRMLFy4kODiYH374gU8//RSAlStX4uXlhZeXFytXrqROnTpcuXKFTp06ceXKFUaPHs3UqVNRqVR07dqVffv22by/WbNm0atXLyZPnkzt2rWBlPnHEydOpG/fvsyYMYPo6GgGDRrE0KFDadu2LaGhoVitVoYMGaL83C9cuED79u25e/cukyZNYsKECYSHh9OhQwfu3r1rc86QkBBGjhzJmDFj8Pf3Z/jw4Zw7d45x48Yxf/58vLy8GDlypE1C5c8//8Td3Z2XX76/WsVvv/3G2rVrGT16NOPGjePkyZN07txZ6d+tW7cSHBxMxYoVCQsLY9asWZQoUYJPPvmEI0eOPDam5cuXM2bMGBo2bMjcuXOZOnUqOp2ODz74IM376dOnD6+++iohISGUKVOGd999N83Q7BkzZvDSSy8RFhZG165d+e6779LUnFq+fDk+Pj6EhYXRpk0bbt26RadOnUhISGDixInMnz+fpk2bsmzZMmXufnh4OP3798fb25vZs2czYcIELly4QO/evbFYUoqC/vTTTwQHB+Pu7k5oaCgDBgzgxx9/pH///hl6+DRlyhR0Oh0hISG0aNGCZcuW0aJFC65fv87UqVPp3Lkzq1evZtmyZU881qOsXLkSgBYtWjBu3LgsHePBf3fmzZtH5cqVmTx5Mjt27ADAZDLRtWtX/vrrLz744AM+//xzTp06xaJFizJ9rkddH8OGDePGjRuZPt7QoUMJCgpi7ty5lClTho8//pguXbpQvnx5wsLC8PX15fPPP+fo0aM2+w0ZMgQvLy9CQ0OpVasW48aNS5MomD59OhaLhYkTJ9KxY0e2b9/OZ599prz+pOujfv369OvXD0j5Penfv/8T38+YMWN4/fXXCQsLo3z58gwZMoQtW7YAkJCQQJcuXTh37hwff/wxCxcupEuXLuzcuVOJ/Ul/i4AM/z16WPPmzblz5w579+5V2iwWCxs3bqRp06bY2dnRsWNHevbsabPfpUuXOHPmDOXLl3/i+xciPVIsQAghxDPp7Nmz3Lt3jy5dulC5cmUA3N3dWblyJXFxcUybNo0yZcowd+5cNBoNAH5+fjRt2pQ1a9bQqVMnIGXUW//+/XmreUsiY428N6oC27b+zu5df+Lt608Z93IAFCxUBC9vHyBlVIHeYGDql2E4OjoBULlqAJ3avcXK5UspUektftl9CQDXktUp6tWId5pVpHKFwnRq+xe7d+3gzRatuXD+LNu3/kbwoPdo3a6jcpwbN66njERo2Jh5c2YRUL0WH4wZr7z34sVLMuzdfuzd/Sc1aqXMg05JEOiVEQpeXl7odDry58+fZhh03759qV+/vvJ96hPLzp07K216vZ6BAwdy+vRpZf+EhAQmTJhAzZo1AShdujQNGjRg27ZtlC1bln379lG8eHE6deqEWq0mICAABwcHoqKiAKhUqRJOTk7K1/Hx8axduxY7OzuWLl2qvFa/fn2aNWvG5MmTWb16tRLT66+/TuvWrW3eS0JCAh9//LHy1P7s2bNMmzaNCRMm0KZNGwDi4+MZNGgQFy5cwNPTk5CQEOzt7VmyZIlyzpo1a9KwYUMWLFjAyJEjleN37NiRJk2aKN/v27eP4OBgGjZsCKQsDe3m5mZTe2D79u1KPA/GuXbtWkqUKAGkXKstW7Zk3bp1dOjQgbNnz9KyZUvlqTyAv78/1atXZ+/evfj5+T0ypvDwcHr06GFzQ1SsWDFatWrF6dOnqVOnjtLeuXNnZRWqwMBAWrZsSWhoqM3UiMaNGytPY2vWrMnOnTttkiAAL7/8MsOGDVO+//PPP/H09GTGjBlKn9aqVYudO3eyd+9eevfuzdGjR0lMTKRPnz4UKVIESBmm/9tvvxEfH4+joyNTp04lMDCQqVOnKscuXbo03bp1Y9u2bTbXbXrKlSvHJ598AqT8bFatWkVSUhJTp05Fq9VSp04dNm3axF9//fXY4zxO6u9DkSJFsnwTlvrvTtu2bQGoUqUKmzdvZuvWrQQGBvLjjz9y/vx51qxZg7e3NwA1atRQrrvMeNz1cfDgQZo2bZqp47Vu3Zp33nkHSHli3a5dO3x9fRk8eDAAFSpU4Ndff+Wvv/7C19dX2e+1115Tru/AwEBu3bpFWFgYHTp0UFYheOWVV/j888+Jj4/Hzc2Nu3fvKiNirFZrhq6P1NoPnp6eFC9e/InvZ+DAgfTo0QOAunXrcvHiRcLCwmjYsCEXL16kaNGiTJo0SfndrVGjBn/++adSGPBJf4ucnZ0z/PfoYVWqVKFYsWL8/PPP1KpVC0gZmXH79m2aN2+e7j6JiYmMHDkSnU7H22+//cT3L0R6JEkghBDimWGxWDl/NYroOBN2DoXJnz8/ffv2pUmTJgQGBlK7dm2GDx9OQkICR44coUePHlitVpKTkwEoUaIEZcuWZefOnTYfyip4ehMRbcRitaLT6XBzcyPxEcP5Af46uJ9K/lUw6A2Y/z22o4Mj3j6V2Pz7NgrF+ijb5i/qzpAOlXEv5gpAwcKFSUxMOfaxo4cBCKxnO6x53IQpAFy+dIHbt27SsfM7ynkA/CpVxtHRkQP796YkCaxmnB0MaNWQnJyMRqN57NJfnp6eNt9PmzYNgIiICM6fP8+lS5f4448/ANJU538w4VC0aFEAZahvjRo1WLlyJa1ataJhw4bUq1ePN99887GxnDhxgvr16ys3lgBarZamTZsSGhpqM8z94bhTpX4wByhYsCCAzU21m5sbANHRKXUh9uzZQ0BAAAaDQbk2nJycqFq1Krt27bI59sPnrF69OrNmzeLEiRMEBgZSr149m6SC1Wplx44dfPHFF2liTL3JgJQkTokSJdi/fz8dOnRQngTGxcVx4cIFLl++zLFjx4C0P4OHY0odUh0dHa38/FKfPCY/cN0AtGzZUvlapVLx2muvMWvWLJunmVWrVrXZp3jx4jbTP9KLoU6dOtSpU4ekpCTOnj3LpUuX+Oeff4iIiFD638/PD71eT5s2bWjSpAl169alevXqyo3kuXPnuHHjBn369LGJu1q1ajg5ObFz507q16+f5j09eL37+/vbtOfLl4+KFSvaFNF2c3OzGSqeVx6MNTWhl/q7tGfPHkqUKKEkCCDlGm3QoIHNU+WMeNz1kd7qG1arNc3w9gf778G4CxRIWZnlwd+3fPnyAaTp4wevPYBGjRrx22+/ceHCBaW43sMJzcKFCyujbc6fP5+h6+NhFotFGamS3vt54403bF5r2LAhs2bNIi4uDk9PT7755hssFgsXL17k0qVLnD17litXrijHLF++/CP/FgEZ/nv08HWtVqtRq9W89dZbfPPNN4wdOxadTsf69espXbq0TZ+nio2NJTg4mGPHjjFjxgyKFSuWZhshMkKSBEIIIZ4JR87cZvXvZ7h6K5ZkswWtRk3AW8OIu7SNjRs3snLlSgwGA82bN6dPnz5YLBbmz5/P/Pnz0xxLr08pGJg6fDnJqsHywFBmlVqN1WpJs1+q6KhI/vjtV/747dc0r2l0jhR64PuOr/soCQIAtUqN1WL99zgpT9jd8qVf0Cz19RnTJjJj2sQ0r9+9c5u7t27QtpXtk8DPP//8scsLPryqwLFjxxg3bhzHjh3D3t6ecuXKKUPlHx7ibW9vf/+9qNU227zxxhtYLBa++eYbZch8sWLFGDZsWJoP4qni4uKUG40HFSxYEKvVqtQzSC/uVA8mGNKL82GRkZFs2LAhzdxjIE1xuYfPOX36dObMmcPGjRvZtGkTarWaWrVq8cknn1CsWDGOHTuG0WikSpUqNvulPjl/UIECBZRRFhEREXz88cds2bIFlUpFqVKllJv1h38GD8d0+fJlxowZw+7du7Gzs8Pd3Z0KFSqku2/qtJMHY7BarUoCBdL2nVqtfmIMFouFL774guXLlxMfH89LL72Er6+v8rsGKcmGr7/+mnnz5rF69WqWLl2Ki4sLHTt25N133yUyMhKAcePGpTuM/9atW1y5coVXX33Vpv3B6z29a+FpXUXDYDDYfP9gP9+7dy/d34v02p4kM9cHwPfff6+MJEn122+/KV9n9vct1cO/A6nvJfV3IL3jPNgnGbk+0hMaGppmusyDhdVTE4sPxpX6b4+joyOLFy9mzpw5REZGUrBgQby9vTEYDEoC09HRkeXLlzN79uw0f4tGjx5NdHR0hv4ePVzIdcCAAQwcOJDmzZsze/ZsduzYQWBgIL/++itdu3ZNc5zr16/Tp08fLly4wPTp07M06kSIVJIkEEII8dQ7cuY2oauPkJCYjLOjHXYaO5LMFiLiHbEv9Rbz3v0AEq7zww8/sGLFCooUKYJKpaJbt27pDqW1t7fHbLESl5jy5CazdXadnJ2pXKU67TqkDOWMjDGy6rd/uBOZAKqUoaTFCztxBXBzevQKBk5OzgBERd6jUOH7H6AvX7pAdFQUjv++3qf/YCpVrppm/wL53CjvXtxmSD6QoSG2qVLn03p4eLB+/Xrc3d1Rq9Vs27aNTZs2Zfg4qZo1a0azZs2IiYnhzz//ZP78+QwfPpwqVaqke6Ps6OiYZt48pNQ/gJSnko/68J9Vzs7O1KpVSxky/aAnLdvs7OzM8OHDGT58OOfPn+e3334jLCyMcePGMW/ePLZv306tWrWws7Oz2e/evXtpjnXnzh1laPSwYcM4f/48S5Yswd/fH51OR0JCAt99991j47FYLPTu3Rs7OztWr16Np6cnWq2Ws2fP8sMPP6TZPvVG58EYNBoNbm5u2ernefPmsWTJEsaNG0ejRo1wdk65dlOnfKTy9fUlJCQEk8nEwYMHWblyJXPmzKFChQqUK5cytWfEiBEEBASkOYerqyuFCxfO1vX+sNQRCA8/Oc9IocTcVKRIES5evJimPb3flcfFntnrA6BBgwZp+rhw4cJpCm1m1r1792yWgUx9LxlNfLi4uACPvz7S065du8dOU4mKikr3d8LV1ZWffvqJiRMnMnz4cFq1aqUkEbt168bZs2eVfdzd3ZkyZQpms5mjR48qf4tKlixJ+/btn/j3CEi3zwHKlCmDr68vGzduRK1WEx0dzVtvvWWz7enTp+nRowdGo5FFixZRrVq1R75fITJCChcKIYR4qlksVlb/foaExGQKuOrR22lQq1VEXD7Moe8/JCoygrXbzuPnV4mxY8fi4uLC3bt38fLy4vz58/j4+Cj/lS9fnlmzZrF79x4iohJISjY/OQDuPzFP5VepCpcunqdcuVcwuBZnzZ5YYqwFuHd+B7E3jlPNqwitGjx5rrKPbyUAdu3cbtM+b/YsQmZMpWSp0uTLl58b16/hUcFL+a9QwUIsnBvCtfBz2NsbbN6jj4+PMtz34bjTc/78eSIjI+nSpQvlypVT9tm+PSWmh4fpPs67776rzHd3dnbm9ddfp3///iQnJz/yBtTT05Pt27fbjBgwm82sX78eHx8fm7n+OSV1VQdPT0+lz7y9vVmyZInNigAPu3r1KvXq1VNWBHB3d6dXr17UqlWLa9euAenXIwA4ePCgTaLg+PHjXLlyRanvcPDgQRo1akT16tWV95yRn8G9e/e4cOECbdq0wcfHR0lypO778JPi1IJsqa/9+uuvVKlSJdv9fPDgQcqVK0fr1q2VBMHNmzf5559/lPiXLFlCgwYNMJlM6HQ6atasqRS1vHbtGu7u7hQoUIArV67YXM9FihRh2rRpnDhxAp1O98jrPStSn4rfvHlTaUtKSkpTdO9hGfndyo6AgACuXLnCyZMnlbbExESlsGEqJycnm9gBm3oLT7o+0ru28uXLl6aPc+L38MFrD+CXX36hWLFiNomDx8nI9QFpfzZFihRJ834etHXrVuVri8XCL7/8gp+fHwaDgYMHD+Li4kLPnj2VBEFcXBynTp1Sfrd++eUXatSowe3bt9FoNPj7+yt/i65du4aTk9Nj/x6lTv14OMYHk6rNmzdnx44drF+/Ps3UpevXr/POO++gUqlYsWKFJAhEjpCRBEIIIZ5q569GcfVWLM6OdjZz2/MVLQdWC+d3LiTxXkO+fymeIwe2ExMTQ6NGjQgKCqJ379689957vPXWW0pV/yNHjvC/t9/BlJzxm19HRyf+PnaEI4f/wtfPn87dejGwbzcGDuhPvKMPZquGqMt7ib3xN806DqX7mxU5cujgE49btvwr1GvQkLlhMzAmJlKuvAd79+xk987tjJswBY1GQ/fe/Zk+5TPUGjU1a9clLjaG5V8t5PbtWzbzldPj4uLCiRMn2Ldvn00BsQeVKVMGJycn5syZg1arRavVsmnTJuWp1qOWWkxPjRo1+Pjjj5k0aRJ169YlOjqakJAQSpcurQxvfljr1q2V6uipTzy//vprwsPDWbBgQYbPnRn9+/enffv29OnThw4dOqDX61m5ciVbtmxh5syZj9yvWLFiFC1alPHjxxMbG0vJkiU5fvw427Zto0+fPkRERHD8+PE0S9JBSj/27NmTfv36ERcXx/Tp03nllVdo1qwZkPKE/aeffqJixYoULVqUv/76i3nz5qFSqR77MyhQoADFihVj+fLlFC1aFBcXF3bs2KGsKJC61F+qyZMnYzQaKVOmDKtWreLcuXN89dVXWelGG76+voSFhTFv3jwqVarEpUuXmDt3LiaTSYm/Ro0aTJ06leDgYN5++200Gg3ffvstOp2OBg0aoNFoGDJkCGPGjEGj0dCgQQOio6MJCwvj5s2baYZj5wRXV1f8/f1ZtmwZpUqVwtXVlaVLl5KYmPjYaQouLi4cPnyYwoULP/Lazo5mzZoxb948goODGTx4MC4uLixevJi7d+/arJpRv3591q9fj5+fH6VKlWLt2rVcunRJef1J10dmfr+za/Hixej1eipVqsSvv/7KH3/8odRDyYiMXh+pIw42b95M3bp1KVu27GOP++WXX2I2m3nppZdYsWIFFy5cYPHixUDKdb1ixQomTpxIgwYNuHXrFgsXLiQyMhJHR0cgpd6IxWIhODiY3r174+joyMaNG5W/RZCyIsSj/h5lZAWGN954g4kTJ7JhwwY+/vhjm9fGjx/P3bt3GTduHLGxsTZLLzo5OSkjdITIjFxLEqjVagYMGJBm7psQQgiRGdFxJpLNFuw0tsO3DY6uVGs6mNP7fuDs7m/4ePdSPF55hVmzZlGjRg0AFi5cSEhICIMGDcLOzo4Knl5M/XI2FTx90jvVI/2vYxe+Xf4V7w8byOKvV1O2XHneDp7AiqXzSLi3AqvVit6lKB37fEjPzi2ffMAHfDBmPF8tmsuaVSuIioykZOnSfDx+MnXqNgCg6ZstcXRw5NtvlvLzj9/j4OBAlcqV+eKLaTZPk9LTvXt3PvvsM3r06KF86H2Ys7MzYWFhTJ48mcGDB+Po6Iinpydff/01vXr14sCBA49cL/5h7du3JykpiW+//ZZvvvkGg8FAzZo1GT58eJrh96mKFy/O4sWLCQsL4/3330elUuHr68vSpUvTFNDLKRUqVGD58uVMnz6dESNGYLVaeeWVVwgNDU0z3/1hISEhfPHFF8yYMYN79+7x0ksvMWDAAHr37s3PP/9M+fLl051WUbVqVWrUqKFUdw8KCmLEiBHKE9qJEyfy6aefKk/WS5cuzbhx4/jxxx85cODAY2MKCwtjwoQJjBo1Cp1OR7ly5ZQlBk+dOmWz7dixY5k7dy7h4eF4eXmxaNGiHOnnPn36cO/ePZYuXUpoaCgvvfQSzZs3R6VSMXfuXKKjo6lQoQJz5swhNDSUoUOHYjab8fb2ZtGiRUrhurZt2+Lo6MiCBQtYuXIlDg4OVK5cmalTpz7xes+q1L4fPXo0Tk5OtGnThipVqrBq1apH7tO3b19CQ0M5fPiwMhokJ2m1WhYuXMiECRMYO3YsWq2Wt956Czc3Ny5cuKBs9/7775OcnMykSZPQarW88cYbvPfee4wePVrZ5lHXx2effcaBAwdsVjXJTR988AHff/89c+fOxd3dnZkzZ9K4ceNMHSMj10f16tWpVasW06ZNY/fu3cybN++xx/z888+ZOHEily5d4pVXXmH+/PnKdIaWLVty5coV1qxZwzfffEORIkWoV68edevWZe7cuZw7d46yZcuyYMECZsyYwYcffkhCQoIySiD1b1GdOnXS/D2qWLEiixcvTlOsMT358+enTp067Ny502ZlE5PJpIyEeDh5ACkjUrKz3Kd4camsGVl0Fjhw4AALFiwgPDyc8uXL07NnzzRPME6ePMmAAQNsips8D1KrCz88POm/FB8fz8mTJ/H09HxqC/A8y6R/c5f0b+563vv3bHgkny3Zh0GvQW+nSfO60WQm0WTmg24BlCvhlu4xrFYr0XEm4hKTMlV/ICEhQfkQmDpvNHX6w+8HwpXtDHoNvVv44FUm80XFMkqlAhcHHU4OOT/8Pq8879duqtSbsP/6w/qD/fvLL7/w/vvv89tvv2VrDr+4Lzev3zNnznD+/HkaNWpkM4KqTZs2FC1aNE0hvqfZ3r176dKlC0uXLqV69eoZ2udp/7fhabg3ECI3ZWgkwe7du+nRowdly5alTJky7N69my1btvDhhx/SoUMHZTuTyaTMyRNCCCFygnsxV4oVduLi9Wh0LmqbD8xWq5WYhCRKv+Ris4LAg8xmC5GxRowmM5msT5iGKcnMop/+5vA/t5W2fC56BrStRLFCaSt+5xSVCpztn68EgRDi0eLj4xk8eDAdO3bktddew2w2s2HDBo4fP86wYcPyOjwhxHMuQ0mCWbNm0bBhQ7788kulqubo0aP55JNPsFqtdOzYMbfjFEII8YJSq1W0CSpP6Ooj3I024mxvh51WTVKyhZiEJBz0WtoElUetVqXZNynZTGSMMVP1Bx4lOs5I2OqjXLx+f6m4kkWc6d/GDzfnR69gkF0qFTgZdDg7SoJAiBeFn58fX375JQsXLmTdunVYrVa8vLxYsGCBMoRdCCFyS4aSBP/88w/9+/dXqoW6uLgwY8YMhg0bxoQJEyhYsKBSmEMIIYTIaX7lCxHcxo/Vv5/h6q1YYhOS0GrUlH7JhTZB5fErXyjNPgnGJKJiTZgt2R0/ADcj4pn/4ynuRiUqbT5lC9CjuTcGXe7VAFapwNFgh4uTJAieZU/DnOBWrVrRqlWrvA5DZEKTJk1s5p8/q6pXr87p06fzOgwhRCZk6JONvb09cXFxNm0qlYpJkyZx+/Zthg8fTsGCBdFo0s4VFUIIIXKCX/lC+JQtyPmrUUTHmXBx1OFezDXdEQSx8SZi4pOwZKYAwSNcuWNi0+ZjJBjvL5dYr3Jx2jUsjyYXl0FTAQ4GO1ydcm+UghBCCCHEwzL06aZy5cqEhYVx+/Ztm3atVktoaCgvv/wyffr04c8//8yVIIUQQghImXpQroQblSsUplwJtzQJAqvVSmSskeh4U44kCPafvMWPe+8pCQIV0CaoPO1feyXXEwT2Bi2uMsVACCGEEP+xDH3Cee+994iIiCAoKIgvvvjC5jVnZ2cWL15M/vz5010XWAghhPgvmM0WIqISiU/I3AoG6bFaraz/8zzf/HqW1NkKdlo1vVv60DCgpE3xxNxg0Gtxc9Ln+nmEEEIIIR6WoekGJUuW5KeffmLdunXprv1btGhR1qxZw8yZM9m8eXOOBymEEEI8jinJTFRszhQoTDZb+HrjKfYcv660OdlrCW5biTIvp7+CQk5RkZIgyOcsCQIhhBBC5I0MV1tyc3OjW7duREREpPu6k5MTH3zwAR988EGOBSeEEEI8SU4WKIxPTGLu98c4feme0ubmqCG4rS8liuZygkD1bw0CR50kCIQQQgiRZzJdkrlu3boEBgbSvHlzgoKC0OlkvqQQQoi8ERNnIjbBRA7kB7gTmUDo6iNcv3O/UG/ZYi40qKinoKsh+yd4DLUKnOxlmUMhhBBC5L1MV10aNmwYd+/e5d1336V27dp89NFHHDhwIDdiE0IIIdJlsVi5F5NITA4lCC5dj2bysgM2CYKAikXp28ILgy73ChQCqFUqXBwlQSCEEEKIp0OmRxJ069aNbt26ER4ezs8//8yGDRtYtWoVL7/8Mm+99RZvvvkmZcuWzY1YhRBCCJLNFqJijCQmmZ+8cQYcOXObhT8ex5R0v57BG7VK82agO4mJiTlyjkfRqFW4Oumw19vl6nmEEEIIITIqy49HSpQoQb9+/fjpp5/46aefqF+/PvPnz6dZs2Y5GZ8QQgihMCaZiYhKzLEEwe8Hwpmz5qiSIFCrVXR5w5O36pbN9boAWo2KfM56SRAIIYQQ4qmS6ZEED7p79y4bN25k48aNHDp0CDc3N954442cik0IIYRQxCcmER2XMwUKLRYrq38/w+8HwpU2g15Dn5a+eJbOn+3jP4lOq8bNWY+dVpPr5xJCCCGEyIxMJwliYmLYtGkT69evZ//+/Wg0GoKCgggLCyMwMBCNRj7wCCGEyFk5WaDQaDKz6KfjHDlzR2nL72IguK0fxQo5Zf8ET6DTqsnnYkCryd1aB0IIIYQQWZHpJEHNmjWxWCxUqVKFsWPH0qRJE5yccv9DlRBCiBePxWIlKs5IgjEZaw4kCKLjjISuPsql69FKW8kizgS39cPVSZ/9EzyB3k5DPmc9GkkQCCGEEOIplekkwcCBA3nzzTd5+eWXn7jt/v37qVixIg4ODlkKTgghxIsr2WwhMsaIMYfqD1y7E0voqiPcjbpfjNCnbEF6NK+IQZet2XcZIgkCIYQQQjwLMv1JpU+fPhlKEJjNZrp06cKFCxeyFJgQQogXl9GUzN2oxBxLEJy+FMGUZQdtEgT1KhenX2vf/yRBYLDTkM/FIAkCIYQQQjz1cvWTkTUnxoYKIYR4ocQlJhGTQwUKAfYcu86yjSeV46mA1kHlebVaiVxfwQDAoNPg5mxAo879cwkhhBBCZFfuPz4RQgghMsBqtRIdZyIuMSlH6g9YrVbW77zAz3/eH9Fmp1XT/c2K+HsUzv4JMsCg05LPWY9aEgRCCCGEeEZIkkAIIUSeM1usKfUHTMnkxPiBZLOFrzeeZM/xG0qbs4Md/dv4UeZl1xw4w5NJgkAIIYQQzyJJEgghhMhTSclmImOMmJItOXK8+MQk5q49xunL95S2IvkdGNCuEoXc7HPkHE9ir09JEPwX0xmEEEIIIXKSJAmEEELkmURTMpGxRszmnKk/cCcygZBVh7lxN15pK1/Cjb6tfHG0t8uRczyOCjBIgkAIIYQQzzBJEgghhMgTsQkmYuKSsORQkdtL16MJXX2E6DiT0hZQsSidX/fETpv7qwpIgkAIIYQQzwNJEgghhPhPWa1WouJMxOdQgUKAI2dus/DH45iS7k9ZeKNWad4MdP9PbtglQSCEEEKI54UkCYQQQvxnzGYLkbFGjCZzjhQoBPjjQDjfbflHOZ5areLtJhWo5ftyDp3h8SRBIIQQQojnSbaSBDExMdy6dYsSJUqg0WjQaDTKaxqNhs8//5zixYtnO0ghhBDPPlOSmajYnCtQaLFYWf37GX4/EK60GfQa+rT0xbN0/hw5x5NIgkAIIYQQz5ssJQn27t3L1KlTOX78OCqVilWrVjF//nyKFi3KqFGjlO1atmyZY4EKIYR4diUYk4iKNWG25Mz4AaPJzKKfjnPkzB2lLb+LgeC2fhQr5JQj53gSSRAIIYQQ4nmU6UpOu3fvpkePHhgMBoYNG4b13wmlFSpUYOnSpSxevDjHgxRCCPHsio03ERmTcwmC6DgjX6z4yyZBULKIMyO7VP1PEwT2hmczQWDNqUIQ/7FnNW7xfJHrUAjxIsh0kuDLL7/k1VdfZdmyZXTt2lX5x7Jv37707NmTVatW5XiQQgghnj1Wq5XIWCPR8aYcW8Hg2p1YJi09wKXr0UqbT9mCDO1UGVcnfY6c40lUKnCwt8PN6dlLEISFhbFw4cK8DiPTVq1axaRJk/I6DBunT5+mRYsWeHt788Ybb+R1OLnmxIkT+Pv7YzKZnrwxMGrUKIKCgpTvPTw8mDVrVm6F95/K6HUYFBRkM7L2WWUymZgzZw5NmjShUqVKNG7cmJCQkAxfC0KIZ1emkwQnT56kdevWAGk+HNWuXZurV6/mTGRCCCGeWWazhYjoROITcm4Fg9OXIpiy7CB3oxKVtnqVi9OvtS8G3X9Th1elAkfDs5kgAJgxYwYJCQl5HUamzZ49m8jIyLwOw0ZoaCjXrl0jNDT0qUtg5KRt27ZRo0YNdDpdXoeS557G6zA3jR8/njlz5tCqVStmz55N69atmT9/PmPHjs3r0IQQuSzTn6qcnZ25fft2uq9dv34dZ2fnbAclhBDi2ZXTBQoB9hy7zrKNJ5UpCyqgdVB5Xq1W4j+7WU9NEPxXIxbE0+3evXu88sor1KtXL69DyVXbt2+nefPmeR2G+I/du3eP7777jmHDhtGzZ08AatasCcC0adMYNmwY+fP/NwVihRD/vUyPJHj11VeZPn06x44dU9pUKhU3btxgzpw51K9fPyfjE0II8QxJMCYREZ2YYwkCq9XKz3+eZ8n6E0qCwE6rpndLHxoGlPzPEgRqlQongy7XEgTHjx+na9euVKlSBX9/f7p168bhw4eV1w8cOMDbb7+Nn58fAQEBjBw5koiICOX1tWvX4uXlxZEjR/jf//6Hj48PDRo0sJla4OHhAUBISIjyNcA///xDnz59qFy5MpUrVyY4OJjw8PsrRuzduxcPDw92795N9+7d8fPzo3bt2kyZMgWz2axsZzKZlCmJvr6+NGvWjO+//97mfW7ZsoVWrVrh4+ND7dq1mTx5MomJiTxOUFAQV69e5fvvv8fDw4MrV64o73fVqlXUrl2bgIAAzp49i9lsZt68eTRr1gxfX18qVapE+/bt2bNnj3K8WbNm8dprr7F161befPNNvL29ady4MevWrbM571dffUWTJk3w8fEhMDCQsWPHEhsbq/Tlvn372L9/Px4eHqxduxaAixcvMmjQIGrXrk2lSpXo3LkzBw8eVI555coVPDw8WLx4MU2aNMHPz481a9Ywa9YsmjRpwubNm2nWrBk+Pj40b96cQ4cOcfjwYdq2bav06e7du23ifNLPL3W6wLfffkuDBg2oXLkyO3fuJCIigvfee4/atWsr53u4D6Kiojhy5Ah169YFyFD/ZkXnzp0ZM2YMYWFhBAYG4ufnR69evbhz5w5r1qzhtddeU34vrly5YrPfqFGjmDNnDrVq1aJKlSr079/fZlRrRn/ekZGRjBkzhlq1auHj40O7du1s+jq96/BxkpKSGD9+PNWqVaNq1appfmchZfpCq1atqFSpEr6+vjRv3pyNGzcqr1ssFqZPn05QUBDe3t4EBQUxbdo0kpKSlG2MRiOTJ0+mXr16eHt78+abb7Jhw4bHxnbjxg08PT35+uuvbdojIiKoWLEiS5YsITY2lvbt29tMHQFwd3cHsLnGhBDPn0wnCd577z0KFChAu3btlITA0KFDadKkCSqViqFDh+Z0jEIIIZ4BMXEmImOMOVagMNls4av1J/j5zwtKm7ODHUM7Vsbfo3COnCMj7LQanB11uDjlznDr2NhYevbsSb58+Zg1axbTp08nISGBHj16EBMTw/79++nWrRsGg4Evv/ySDz74gH379tGlSxebG2yLxcK7777LG2+8wbx586hcuTKTJ09mx44dAKxcuRKANm3aKF9fuHCB9u3bc/fuXSZNmsSECRMIDw+nQ4cO3L171ybOYcOGUaVKFebMmUOzZs1YsGCBTR2iYcOGsXjxYtq2bcvcuXOpU6cOo0aN4ueffwbgp59+Ijg4GHd3d0JDQxkwYADr169n2rRpjy0GFxISQqFChahXrx4rV66kcOGUn73ZbGbRokVMmDCB999/n7JlyzJ16lTCwsL43//+x4IFC/j000+JjIxk8ODBNtMsbt++zSeffEKXLl2YN28exYsXZ+TIkZw7dw6An3/+mSlTptCpUycWLlxIcHAwP/zwA59++qnSl15eXnh5ebFy5Urq16/P2bNnadWqFVeuXGH06NFMnToVlUpF165d2bdvn817mjVrFr169WLy5MnUrl0bSLlxmzhxIn379mXGjBlER0czaNAghg4dStu2bQkNDcVqtTJkyBDl556Zn19ISAgjR45kzJgx+Pv77AurtAAA1CxJREFUM3z4cM6dO8e4ceOYP38+Xl5ejBw50uaG/88//8Td3Z2XX34ZIMP9mxU///wzu3fvZsKECXz44Yfs3r2bt99+m6VLlzJy5Eg++eQTjhw5wieffGKz32+//cbatWsZPXo048aN4+TJk3Tu3DlTP2+j0UjXrl357bffGDJkCCEhIRQtWpSePXsqiYJHXYePsnHjRv7++28mTpzIyJEj2bp1K7169VISa8uXL2fMmDE0bNiQuXPnMnXqVHQ6HcOGDePGjRsAzJ8/nxUrVhAcHMyiRYvo0KEDCxcuZPbs2UBKEjU4OJhvv/2Wd955h9mzZ+Pv78+QIUPSJEEeVLRoUQICAli/fr1N+y+//ILVaqVp06aUKFGCsWPHKkmBB/vbzs6O0qVLP/b9CyGebZmebuDq6sqqVatYt24de/bsITIyEmdnZzp37kyrVq2wt7fPjTiFEEI8pSwWK1FxRhKMyTlWfyAuMYl5a49x+vI9pa1IfgcGtKtEIbf/7u+MWq3CxVGPoyH3ah6cPXuWe/fu0aVLFypXrgykPK1buXIlcXFxTJs2jTJlyjB37lw0Gg0Afn5+NG3alDVr1tCpUycg5Yahf//+tG3bFoAqVaqwefNmtm7dSmBgIJUqVQJSbhBSvw4JCcHe3p4lS5bg5JSyMkTNmjVp2LAhCxYsYOTIkUqcbdu2JTg4WNlmy5YtbN26lfbt2/PPP/+wadMmPvjgA7p27apsc/XqVfbu3UvTpk2ZOnUqgYGBTJ06VTlm0aJF6du3L3/++SeNGzdOt3+8vLzQ6XTkz59fiTtV3759bUYw3rp1iyFDhtC5c2elTa/XM3DgQE6fPq3sn5CQwIQJE5Th06VLl6ZBgwZs27aNsmXLsm/fPooXL06nTp1Qq9UEBATg4OBAVFQUAJUqVVL6K/WYn3zyCTqdjqVLlyqv1a9fn2bNmjF58mRWr16txPT6668r9Z1SJSQk8PHHHytP7c+ePcu0adOYMGECbdq0ASA+Pp5BgwZx4cIFPD09n/jzGzhwoHL8jh070qRJE+X7ffv2ERwcTMOGDQEICAjAzc3NpvbA9u3blXgy079ZkZycTEhICK6urgD8+uuv7Nixgy1btlCiRAkADh8+zA8//JCm39auXats4+7uTsuWLVm3bh0dOnRQtnncz/uHH37g1KlTfPfdd/j5+QFQt25dOnfuzNSpU1mzZk2612FycvIj30++fPlYuHAhDg4OyvfBwcFs376dBg0aEB4eTo8ePejfv7+yT7FixWjVqhUHDx6kadOm7Nu3D29vb+VaCQgIwN7eXpnau2vXLnbs2MH06dOV4pmBgYEkJCQwdepUmjVrhlab/r9dzZs354MPPuDatWtKEmj9+vXUqlWLQoUKpbvP5s2b+f7773n77beVn5MQ4vmUpU89Op2Odu3a0a5du5yORwghxDPEbLYQGWMkMcn85I0z6E5kAiGrDnPjbrzSVr6EG31b+eJob5dj53kStQpcHfREq3OutkIqi8XK+atRRMeZsHMoTP78+enbty9NmjQhMDCQ2rVrM3z4cBISEjhy5Ag9evTAarUqNyUlSpSgbNmy7Ny5U0kSAPj7+ytfp97QxMfHpzl/qj3/Z+/Ow2O63gCOf2eSTCYrsRSNJWKJILHHVluKElsprb1atfwIukXRVkurrb2axb4U1aaWtoilqL1aW0WUKoISS5B9m5nMzO+PNLdGgoRMBO/nefpUzl3OmTM3k7nvPec9v/2Gn58fWq1WObezszMNGzbk119/tdj39nND1g1+9rmzh9S3b9/eYp/srPbnzp3j2rVrDBs2zOLGqkGDBjg4OPDbb7/xwgsv5LjpsrGxueeUEm9vb4ufZ86cCWQNm46OjubixYvs3LkTIEdG9ttvaMuWLQugvJ4mTZoQHh5Ojx49aNu2La1ataJLly73bMvBgwdp06aNcrMOYGtrS6dOnQgNDSU1NfWu7c6WHSQCKFWqFIBy0wpQvHhxAJKSslb3uN/7d3uQ4M46GzduTHBwMCdPnqRFixa0atXKIihkNpvZu3cvs2bNUsry07+5MRqNFqNG1Go1anXWoNYqVapY3HiWKlUKNzc35eY/+/UnJydbnLN+/foW+9SsWZMKFSpw6NAhJUgA936/Dxw4QOnSpalVq5bFNdimTRumTZtGYmJirjfFmZmZGI1GMjMzyczMtHg9rVq1UgIEkDVdwdbWlkOHDtGmTRtl9YOkpCSlL3///Xfgv75s3LgxM2fOpG/fvvj7+9O6dWv69++vnPPAgQOoVCpatWpl0W5/f3/Wr1/PmTNn8PLywmSy/AyztbWlffv2TJo0iU2bNvHGG29w9epVjhw5wvTp03O8TsgK2rzzzjs0aNCAoKCgXPcRQjw58h0kCAkJues2tVqNo6MjlSpVonnz5pIJVwghnmB6g5GEZB0GY8HdRF+4mkTo6mMkp/0359avVlkGdPTGzjbfM+QemFoFrk4aVOZMi3n3BSHyzA3W/HKGmNgUMo0mbG3U+HV9l9SLu9m8eTPh4eFotVq6devGsGHDMJlMLFy4kIULF+Y4l729ZY4ErVZr+TrU6nsO5U9ISGDTpk25zmG+MynZvc6dnfG9ZMmSd60HYNKkSUyaNCnH9hs3bnD58mWef/55i/LPP/+cHj163LX9t9+EAURFRTFp0iSioqJwcHCgatWqylPSO/vh9pGP2Td22fsEBARgMplYtWoVYWFhBAcH4+7uzrvvvnvX5Q4TExOVG/vblSpVCrPZrOQzyK3d2W4PMOTWzjvl5/27s87Zs2czb948Nm/ezNatW1Gr1TRr1ozJkyfj7u5OVFQUOp2OBg0aKMfkp39z065dO4t8Ad27d+eLL76462u/Wz/drkyZMjnKSpYsqYz6yHav9zshIYEbN25Qq1atXOu4ceNGrkGCRo0aWfwcGBioBGbufBqvVqtxc3NTAjz//PMPEydO5MCBA9jZ2eHp6UmNGjUs2vXGG2/g5OTE2rVrmTFjBtOnT6datWp88MEHNGnShISEBMxms0Vw6XaxsbFs3749x3f306dP4+zsTNu2bYmIiOCNN95g06ZNODg4KCNLbrds2TKmTp2Kn58foaGhOT53hBBPnnwHCdavX8+1a9fQ6/XY2tpSvHhxEhISyMzMRKVSKR9sVatWZfny5ZL5VAghnkDpOgOJKfoCyz8AcOzvGyxefwLDbUkPA5p50KWFZ6EuN5gdIHBy0JCWdvfhxA8i8swNQtdEkp6RiYuTHXY2dhiMJuLSnHCo1JUFb06A9Kv89NNPfPvtt5QpUwaVSsWgQYPo1KlTjvM97BQ/FxcXmjVrxmuvvZZj292GKefG1dUVyHrCnP2UFrJGECQkJCjbx44di5+fn7I9IyODCxcu4OvryzPPPGMxJB+gfPnyeW5Ddm4HLy8vIiIi8PT0RK1Ws3v3brZu3Zrn82Tr3LkznTt3Jjk5mX379rFw4UKCgoJo0KBBrjemxYoV4+bNmznKs1eEcnNzIzY2Nt/tuJeHef9cXFwICgoiKCiI6OhoduzYQVhYGJMmTWLBggXs2bOHZs2aYWeXNXqnIPp37ty5FiMO3Nzc8vFqcxcfH5+j7ObNm1SsWDHP53BxccHDw8NiKszt7nYdrly5kgsXLuDh4YFWq7XIU3DnUolGo5H4+HhKliyJyWRi6NCh2NnZsWbNGry9vbG1teXs2bMW0ynUajX9+vWjX79+3Lp1i927dzNv3jxGjRrF/v37cXFxwdHRkeXLl+favkqVKlGjRo27JhXv2rUrQ4cO5eLFi0RERPDCCy9YfKaYzWamTJnCihUr6Ny5M59//rk8ABTiKZHvxzJjxoxBo9Ewa9Ysjh8/zr59+4iKiiIkJAQ3Nze+/PJLNmzYgEqlshiiJoQQ4slQ0AkKAXYc+of5644rAQK1WsXAAG+6tqxSyAECFa7O9jg5FPwXYZPJzJpfzpCekUnJYvbY29mgVquI++cYf/zwPokJcazbHU2dOnX5+OOPcXV15datW9SsWZPo6Gh8fHyU/6pVq0ZwcLAyPDnPr09t+Wc/e1UAb29v5dy1a9dm2bJlbNu2Lc/nzX7a/Msvv1iUz5gxgylTpuDp6UnJkiW5fPmyxet45pln+Pbbb/nrr7/QaDQW23x8fJSbyDvbnZvo6GgSEhIYOHAgVatWVY7Zs2cPQI4h1/fy5ptvKvkXXFxc6NixIyNGjCAzM/OuN/qNGjVi586dFiMGjEYjERER+Pj4WOXm6kHfv5iYGFq1asWWLVuArHn8Q4YMoVmzZly5cgXImY+gIPrXy8vL4v3NTxDobo4cOWIRKDhx4gSXL19W8g/khZ+fH1evXqVkyZIW7du/fz+LFi1ScoHceR3WqlULT09PatWqhY+Pj0XwaP/+/RZTALZu3UpmZiaNGzcmPj6e8+fP07NnT3x8fJSAzp192bt3bz799FMga3REjx496NevH0lJSaSkpODn50daWhpms9mi3X///TehoaFkZmZSpkyZHL9X2Z577jlKlSrF8uXL+fPPP3MsdTlr1ixWrFjBa6+9piRWFEI8HfI9kiA4OFjJnpxNpVLRtm1bbt68yZw5c9i8eTPDhw9XhpAJIYR4/FkjQaHJZGb1jr/ZeeS/5cS09jYM6+6Lt0fhjkRTq1QUc9bgqLVO3oPomERiYlNwcbKzCHy4la0KZhPR+xeTEd+WH8qlEXl4D8nJybRv3x5/f3+GDh3KO++8Q9euXZWs/pGRkRZJz/LC1dWVo0ePcujQIRo2bMiIESPo3bs3w4YNo0+fPtjb2xMeHs727dv56quv8nzeGjVq0KFDB6ZPn05GRgbe3t7s2bOHnTt3EhISgo2NDW+99RYTJ07ExsaGNm3akJSURGhoKFeuXKFmzZr3bffJkyc5ePAgvr6+ue5TuXJlnJ2dmTdvHra2ttja2rJ161ZldEJ+su83adKEjz76iKlTp9KyZUuSkpIICQnBw8NDGRJ+p8DAQPbs2cPAgQOVp8QrV67k0qVLLFq0KM9158eDvn/u7u6ULVuWTz/9lJSUFCpWrMiJEyfYvXs3w4YNIy4ujhMnTig5JaBg+7cgpaen88Ybb/C///2P1NRUZs+eTfXq1encuXOez9GjRw9WrlzJa6+9xvDhwylXrhy//vorCxcupH///spoirxch9lu3LjBqFGjGDBgABcuXGDWrFk0b96cpk2bolKpcHd355tvvqFs2bK4urqyd+9eZURAdl82atSIJUuWUKpUKerVq8f169dZunQpfn5+lChRglatWtGoUSNGjBjBiBEjqFKlCsePH+err76iRYsW9x3Na2NjQ6dOnVi5ciVlypShcePGyrZTp06xcOFCfHx86NChA5GRkRbHVq1aNdcpIkKIJ0O+gwRXr16lUqVKuW5zd3dX5pqVKVMmx3wwIYQQj6fMfxMU6gowQaFOb2TJhhNEnvlviHYJVy0je9XBvXThfvlUq1QUd9HgYG+9xIhJqXoyjSbsbCzr0DoVo1GnMZw++BNnD6ziowPL8apeneDgYJo0aQLA4sWLCQkJYfTo0djZ2VGrVi2WLl2a72zyw4cPJywsjCFDhrBp0yZq1KjBN998w+zZsxk7dixms5nq1asTGhqaIz/A/UyfPp2QkBC+/vpr4uPjqVKlCl999ZUyx7lXr144OTmxaNEiwsPDcXR0pE6dOgwePBh3d/d7nvv111/ns88+Y/DgwSxdujTXfVxcXAgLC2PatGmMGTMGJycnZS34IUOGcPjw4Rxrvt9N7969MRgMfPfdd6xatQqtVkvTpk0JCgpSbhjvVK1aNVatWsWsWbMYP348KpUKX19fli9fTsOGDfNUb37d7/27V9LKkJAQZs2axZw5c4iPj6dcuXIEBgYydOhQNm7cSLVq1SyejBdk/xakhg0b0qRJE95//30gK2nf2LFj8/XU29HRkW+++YaZM2cyffp0kpOTcXd355133uH1119X9rvzOrxXcKtv374kJyczcuRINBoNXbp0ISgoSAkQhoWFMWXKFMaNG4dGo6Fq1arMnTuXzz77jMOHDzNgwABl9O7atWsJDQ3FxcUFf39/3nnnHSBrZMOCBQuYM2cO8+fP59atW5QpU4bXXntNGQlzP926dePrr7+mc+fOFiMlfv75Z8xmM1FRUbzyyis5jlu+fLlFUEEI8WRRmfOSaeY2L730Eu7u7rlGqMeMGcOFCxf46aefWLNmDfPmzWP79u0F1thHJSoqCsBiiFZhS0tL49SpU3h7e+cpkY/IH+lf65L+tS5r96/OYCSxgBMUJqboCFsTycVr/2Uqr1jGhZG96lDMuXCTYt0rQFCQfXv2UgKfLTuI1t4GezubHNt1eiMZeiMTBvlRtULxh6rrcSGfDdb1pPdv9lKMK1aseCT1P+n9+ygV9b4tCvcGQlhTvkcSjBo1ipEjR9K9e3fat29PyZIluXnzJtu3b+f06dN89dVXnDx5kunTp+dYA1gIIcTjJS3DQFJqwSYovHIzhZDvI4lLylDKfKuW4vWutdBqHmhl3gdmo86aYmDNEQTZPN2L4f6MMxeuJqFxVVtMOTCbzSSnG/Ao54qnu6w/LoQQQohHJ9/fxlq3bs3ixYsJDg4mJCQEo9GIra0tDRo04Ouvv6Zhw4b88ssvdOrUiTfffNMKTRZCCFEYklP1pKTrKcD4AKcvxjFvXRTpuv8SerWuX56X21ZHrS68BIVQuAECyErG2NO/GqFrIrmVpMPFwQ47WzWGTBPJ6QYc7W3p6V+t0PtBCCGEEOJ2D/TIpkmTJjRp0gS9Xk9iYiIlS5a0mMfk7+//SOalCSGEeHjWSFAI8FvUVVZsPqWMSlABL/lX4/lGFQp1BQPIChAUd7Ev9JELdaqVZmTPOqz55QwxsSmkpBuwtVHjUc6Vnv7VqFOt9P1PIoQAHt00AyGEeNI90LcjnU7H6dOn0ev1mM1mLly4gMlkIj09ncOHD/Puu+8WdDuFEEIUAmskKDSbzUTsP8/GfeeVMjtbNa93qUU9r2fucaR12KhVuLnYY1/IAYJsdaqVxqdKKaJjEklK1ePqpMHTvZiMIBBCCCFEkZDvb0i///47Y8aMuevKBU5OThIkEEKIx5A1EhRmGk2s3HyK305cU8pcHO0Y0bMOlZ8t/Ln3jzpAkE2tVj01yQmFEEII8XjJ97ek2bNn4+bmxieffML69etRq9X06NGDPXv28O2337Jw4UJrtFMIIYQVWSNBYWqGgQXrojj9T7xSVrakI4G96lKquEOB1ZNXRSVAIIQQQghRlOX7m9Lp06f59NNPadeuHcnJyXz33Xe0atWKVq1aYTAYmDt3LgsWLLBGW4UQQliBNRIU3kxIJ2T1Ma7d+m+d9uoVizOshy9O2sJJFHg7GxsVbi7aXJceFEIIIYQQ/1HffxdLJpOJMmXKAFCpUiXOnDmjbHvhhRc4efJkwbVOCCGE1ZhMZuKTM0gu4ADBhatJTF1+yCJA0LhWWUa9XO+RBAg0tmpKukqAQAghhBAiL/IdJKhYsSKnT58GoHLlyqSnpxMdHQ1AZmYmqampBdtCIYQQBc5oNBGflEFaRsGuYHDs7xvM/OYIyWkGpaxT88oM6lwTO9t8/8l5aFo7G0q4arGzlQCBEEIIIURe5Hu6QZcuXZgxYwZms5n+/ftTu3ZtPvnkEwYMGMC8efOoWrWqNdophBCigOgNRhIKOEEhwI5D/7BmxxmyYw5qtYr+HWrQzPfZAq0nL1SA1t6W4s72smqAEEIIIUQ+5PuxzhtvvEHv3r2JjIwE4KOPPuLUqVOMGDGC6Ohoxo4dW+CNFEIIUTDSdQbikjIKNEBgMpkJ33aa1bcFCBzsbRn9ct1HEyBQgZODHW4uEiAQQgghhMivfI8kOH/+PO+9957ys4+PD9u3byc6OhpPT0+cnZ0LtIFCCCEKRkqanuQ0A6YCnF+g0xtZvP4Ex8/eVMpKuGoJ7FWHZ0sX/t8DtQqcHTS4OGkKvW4hhBBCiCdBvkcS9O3blx9//NGizNnZGV9fXwkQCCFEEWQ2m0lI0ZGUpi/QAEFiio6Zq45YBAgqlnXhvYENH1GAQEUxZ3sJEAghhBBCPIR8jySws7PDzc3NGm0RQghRwIxGEwkpOnR6IwWYn5ArN1IIWR1JXFKGUuZbtRSDu9bGXlP4SQJt1CqKOWtwsC/81ROEEEIIIZ4k+Q4SjBkzhmnTppGcnEyNGjVwdHTMsc+zzxb+HFQhhBCWDJlZCQr1mQWboPCvC3HM/yGKdF2mUtamQXl6PV/9keQAsFGrKO5ij1aT7z9pQgghhBDiDvn+RvXxxx9jNBoJCgq66z6nTp16qEYJIYR4OBn6TBJSdBiNeR8/YDabUanufZN/IOoqKzafwmTKOq8K6Pl8NZ5vVPFhmvvAskcQSIBAiIKXl88EIYQQT5585yT49NNP+fzzz/nss8/u+p8QQohHJyVdT3xS/gIEK5YtIvzbFXfdbjab2bA3mq8jTioBAjtbNUO7+zzSAMEvP29gzuyZj6T+uzl9+jQvvvgitWvXJiAg4FE3x2pOnjxJvXr10Ov1j7opeXb58mW8vLxYt27do25KkePv78+4ceOUn8PCwli8eLHyc3BwMF5eXrnu/zj0653tz0326/Dy8iI8PDzXfZKTk/Hx8cHLy4vDhw8DsH79ery8vLh8+XKe27Nu3TqLY/LSvt9//11p3759+3Ld59y5c8o+92vPne/5/aSkpODv75/r+/znn38yZMgQmjRpQuPGjXn99df5888/83zue7l48SJjxozhueeeo0GDBvTp04cDBw4UyLmFELnL96OX7t27W6MdQgghCkBiio7UDAP5zU+4dNFcBr42NNdtmUYTKzad4vc/ryllLo52jOhZh8rPFnuY5j4wG7UKNxd7Fi1cgJ+f3yNpw92EhoZy5coVQkNDKVGixKNujtXs3r2bJk2aoNFIosgn0Zw5cwgMDFR+7tWrFy1atMh132eeeYbw8HAqVnw0AcOCplar2bJlC6+88kqObdu2bcsRGHvuuecIDw/nmWeeyXMdrVu3zvcxd7bvueeey7Ft06ZN+T5fXiQmJjJixAhiYmJybLt48SL9+/endu3aTJkyBZVKxZIlS+jbty8//PADnp6eD1xvQkIC/fv3p3jx4kyYMAFnZ2dWr17N66+/ztdff13kPv+FeFI80PhMvV7PmjVr+PXXX7lx4wafffYZBw8epFatWvj6+hZ0G4UQQtyHra0dCSl6UNkWaILC1AwD89cd5+9/EpSysiUdCexVl1LFHQqwprzLDhDYF9EpBvHx8VSvXp1WrVo96qZY1Z49e+jWrdujboYoJGXLlqVs2bK5btNoNNStW7dwG2RF9evX5/fffycuLi5HoC8iIgJvb2+LqbUlSpSgfPny+aqjRIkSDxxErF+/Ptu2bePjjz/G1tbyc3DTpk052vewduzYwZQpU0hNTc11+4oVK3BwcGD+/PlKrrImTZrg7+/PypUrmThx4gPXvWHDBuLj41mzZg1lypQBoHnz5nTr1o3FixdLkEAIK8n3dIO4uDheeuklpkyZwsWLFzl+/DgZGRns2rWLAQMG8Mcff1ijnUII8VQ6ceIEr776Kg0aNKBevXoMGjSIY8eOKdsPHz7M4MGD6d+/H10C2vL5pxNJiI9Xtm/ZtJ62rfw49WcUgcMG8YJ/U3q/1InwVcuVffyfawDA8qULlH8DHD32J68OGsymBaM4u+VDYg59jXsxA0EDGlKquAPHjh7G/7kGHD18kKC3RtDx+Wa81LU9C8K+wmg0KucxGAwsWRhGv15d6eDfjNcHvMzWzRssXuf+vbsYPrg/L/g35aWu7Qn5cjrp6ek5+uP2AIG/vz8xMTH88MMPytDadevWUbNmTVavXk3z5s3x8/Pj7NmzGI1GFixYQOfOnfH19aVu3br07t2b3377TTl3cHAw7dq1Y9euXXTp0oXatWvTrVs39u7da9GGr7/+mg4dOuDj40OLFi34+OOPSUlJAcDLy4uDBw9y6NAhi+HXFy5cYPTo0TRv3py6desyYMAAjhw5opwze5jz0qVL6dChA3Xq1GHt2rUEBwfToUMHtm3bRufOnfHx8aFbt2788ccfHDt2jF69euHr60vnzp1zDL/9+++/GTZsGPXr16d+/fqMHDmSS5cuKduzhy5/9913tGnThvr167N//37i4uJ45513aN68uVLfnUsfJyYmEhkZScuWLQEYMGAA48aNY968eTRr1owGDRrk+tRx+/bt9O3bl3r16lG7dm26d+/Ozz//fN82AaxevZoePXpQt25dfH196datG5s3b1aOzR6+HRkZSffu3fH19aVLly5s2bIlx3V048YNRo8eTb169fDz8+PDDz+0uAHy9/fns88+49VXX8XX15f3338fgL/++ovAwECaNGlCrVq1aNGiBZ9++ikZGf+t8rF//35efvll6tWrR6NGjfjf//7HuXPncvRDjx498PHxoXnz5nz66aekpaXlaOedvLy8+Pbbbxk3bhwNGjTAz89PqX/q1KnKUO/3338fnU6nvNZ69erlGCI+btw4/P3971oPQEhIiPLvew2Hv3O6QfbvYWRkJK+88go+Pj60adPGYvoCQGxsLG+99RZ+fn40atSIiRMnMnv2bIt2eXl5ERwcbHFcbm253/WRH+3atUOtVrNt2zaL8vj4eH777Tc6depkUX7ndINx48YxaNAg1q5dywsvvKB8luzZs0c55s7pBvkREBBAQkKCxecXZF2fFy5coGPHjvk+590kJSURGBhIo0aNWLRoUa77eHp68vrrr1skM3d0dKRs2bL8888/dz33vHnzqF27NomJiRbly5Yto1GjRiQmJlKmTBkGDRqkBAgAbGxsqFSp0j3PLYR4OPkOEkybNo3U1FQ2bdrEDz/8gPnfMa1fffUVPj4+fPXVVwXeSCGEeBqlpKTwxhtv4ObmRnBwMLNnzyY9PZ3BgweTnJzMoUOHePXVV1Hbahj0xkiG/m80x/44wtujh6HT/XfTYjaZmDRxHG2eb8/n0+fg41uX+WFzOPT7rwCEzFsGQEDnbsq/Dxw+wdg3h5CanESZOq9QxrcXGlMSkZtnoU9PtmjnlMkf4FunHlOmfcnz7Trw3aqv2bThx/+2T3qf1d+tJKDLi3w27Usa+TVh6pSP2bEt6+Ztx8+b+XD8O1Ss5MEnn83k1deHsm3rJj4c97byNwZyjiAICQmhdOnStGrVymLYrtFoZMmSJUyZMoXx48dTpUoVZsyYQVhYGK+88gqLFi3ik08+ISEhgTFjxlgEI27cuMHkyZMZOHAgCxYswN3dnblz53L+/HkANm7cyPTp0+nXrx+LFy9m5MiR/PTTT3zyyScAhIeHU7NmTWrWrEl4eDitW7fm7Nmz9OjRg8uXL/PBBx8wY8YMVCoVr776KgcPHrToy+DgYIYMGcK0adNo3rw5ANeuXeOLL75g+PDhzJkzh6SkJEaPHs3bb79Nr169CA0NxWw289Zbbyk3q+fPn6d3797cunWLqVOnMmXKFC5dukSfPn24deuWRZ0hISG89957TJw4kXr16hEUFMS5c+eYNGkSCxcupGbNmrz33nsWNyT79u3D09PTYjWjHTt2sG7dOj744AMmTZrEqVOnGDBggNK/u3btYuTIkdSqVYuwsDCCg4Nxd3dn2bJlREVF3bNN33zzDRMnTqRt27bMnz+fGTNmoNFoePfdd7l27ZrFscOGDeP5558nJCSEypUr8+abb7J7926LfebMmUO5cuUICwvj1Vdf5fvvvyckJMRin2+++QYfHx/CwsLo2bMnsbGx9OvXj/T0dL744gsWLlxIp06dWLFiBcuXZwXdLl26xIgRI6hduzZz585lypQpnD9/nqFDh2IyZa0ysmHDBkaOHImnpyehoaEEBgayfv16RowYYXG938306dPRaDSEhITw4osvsmLFCl588UWuXr3KjBkzGDBgAGvWrGHFirvnGLmf7Pn4PXv2vOvc/PsxmUy8+eabBAQEsGDBAurXr8+0adOUoJter+fVV1/l6NGjTJgwgc8//5y//vqLJUuW5Luu/FwfeeHq6krz5s1zBJi2bt3Ks88+m6dRsydOnGDx4sWMHj2a0NBQbGxsGDVqVI4b4gdRtWpVqlWrlqN9ERER+Pn5Ubp06YeuI5tWqyUiIoKpU6fedQn0vn378sYbb1iUXbx4kTNnzlCtWrW7nrtLly5kZmZaBAoh63U0bdqUYsWK0b59e959912L7YmJiRw6dOie5xZCPJx8j9XcuXMnEyZMoFKlShZPiuzt7Xn99dfzlQBFCCHE3Z09e5b4+HgGDhxI/fr1gawnNuHh4aSmpjJ9+gwqVPRg4uSpXLhwgSpVqlCnbgNeG9CLzRvX8+JLLwNZSQcHvjaEgM4vAlDbpy57d+/kwK/7aNS4GTVr+wBQqnQZatb24djfN5g240tUNnaUbzIEGzstnZpXpqXva/R/pRvhq5YzfOSbSjs7dXmRAYOGAFC/gR/79+ziwK976fLiS5yPPsueXTsYOfodXnq5b9Y+Df24du1q1kiEti+wYF4wfo2bMWHip8o5y5evyLtv/o/fD+yjSbMWuU4xqFmzJhqNhhIlSuQY6jx8+HBat26t/Jz9xHLAgAFKmb29PaNGjeL06dPK8enp6UyZMoWmTZsCUKZMGQICAti7dy+1atXi4MGDlC9fnn79+qFWq/Hz88PR0VH54l+3bl2cnZ2VfwNMnjwZjUbD8uXLlW2tW7emc+fOTJs2jTVr1iht6tixIy+99JLFa0lPT+ejjz5SntqfPXuWmTNnMmXKFHr27AlAWloao0eP5vz583h7exMSEoKDgwPLli1T6mzatClt27Zl0aJFvPfee8r5+/btS4cOHZSfDx48yMiRI2nbti0Afn5+FC9e3CL3wJ49e5T23N7OdevWUaFCBSDrWu3evTs//vgjffr04ezZs3Tv3l15Kg9Qo0YNWrduzaFDh2jcuPFd23Tp0iUGDx7MiBEjlDJ3d3d69OjBkSNHLJ7sDhgwgJEjRwLQokULunfvTmhoqMX0jxdeeIHx48cr/bJ///4cT2WfffZZi5uTffv24e3tzZw5c5Q+bdasGfv37+f3339n6NChyujKYcOGKU8+y5Yty44dO0hLS8PJyYkZM2bQokULZsyYoZzbw8ODQYMGsXv3bovrNjdVq1Zl8uTJQNZ7s3r1agwGAzNmzMDW1pbnnnuOrVu3cvToUfr27XvPc91N9rVbtmzZB55GYDabGTFiBL169QKgQYMGbNu2jV27dtGiRQvWr19PdHQ0a9eupXbt2kDWEPXs6y4/8nN95FXHjh2ZMGGCxZSDiIiIPCcjTU5OZt26dUqeBkdHR/r3789vv/3GCy+8kO/25Na+5cuXW0w52LRpE8OHD3/oc99Oo9HkO6dARkYG7733HhqNhv79+991P3d3dxo1asTGjRuV6+Sff/7h+PHjfPHFF7keYzKZ+PDDD5UguhDCOvIdJNDpdBQvXjzXbTY2NhgMhodtkxBCPLVMJjPRMYkkpeqxc3yGEiVKMHz4cDp06ECLFi1o3rw57777LrE3E4mKOs7LfQYAZoxGI0ZjJs8+606lSh4cOfybEiQAqFnrvydfGo2G4sWLk5HLcP4dh/5hzY4zpNw4g2PJKtjaaejXwYumtbPmIvv41uPIod8tjqlZ2/KpWqlnniEjI+vcUcePAdCileWw5klTpgPwz8Xz3Ii9Tt8Br2HMzFS216lbHycnJw4f+p3mz7XE2cEGGzVk/ruPjY3NPZdm8/b2tvh55sysFRDi4uKIjo7m4sWL7Ny5EyBHErLbb4qyb/Syn4Y3adKE8PBwevToQdu2bWnVqhVdunS5Z1sOHjxImzZtlBtLAFtbWzp16kRoaKjFMPc7250tO0gEUKpUKQDq1KmjlGX/XU5KSgLgt99+w8/PD61Wq/SZs7MzDRs25Ndff7U49511Nm7cmODgYE6ePEmLFi1o1aqVRVDBbDazd+9eZs2alaON2QECyAriVKhQgUOHDtGnTx/lC31qairnz5/nn3/+UaYo3vnd4c42ZT+ASEpKUt6/33/Pug7vfP9uT7CsUqlo164dwcHBFlMCGjZsaHFM+fLlLaZ/5NaG5557jueeew6DwcDZs2e5ePEif//9N3FxcUr/16lTB3t7e3r27EmHDh1o2bIljRs3Vp48nzt3jmvXrjFs2DDlfQFo1KgRzs7O7N+/n9atW1tsA8vrvV69ehblbm5u1KpVy2JuevHixUlOthzx8yjc3tbsgF72tIrffvuNChUqKAECyLpG27Rpo7y3eZWf6wOyruHbH3QBOeb2t23blg8//JBt27bxyiuvEBsby+HDh5k4cSJxcXH3bVOJEiUsEjlm53PIbRrVgwgICOCrr77it99+47nnniMyMpLr16/Tvn17duzYYbHvva6ngpaSksLIkSOJiopizpw5uLu7A1kjvG4fKaNWq1Gr1XTt2pWPPvqIGzduULp0aSIiInB2dqZVq1bKCK5sBoOBcePGsXXrViZOnCh50ISwonwHCXx8fFi1alWuCZk2bNhg8WEvhBAi7yLP3GDNL2eIiU0h02jC1kaNX9d3Sb24m82bNxMeHo5Wq6VDQGde6TsIk8nEd998zXfffJ3jXBp7rcXPWq3lzyq1GrPZZFH2Z/RNfrtxBgCjPo3kK5GcvhLJxI2W5y5e3HLIqfaOutQqNeZ/l0lM+vcJe3G33BN0ZW+fM/ML5szM+eQo7tZNUhJv0KqZ5ZO3zz//nB49euR6TsBibixAVFQUkyZNIioqCgcHB6pWraoMlb9ziLeDw38JGdVqtcU+AQEBmEwmVq1aZTFk/t13373rE8bExETlxv52pUqVwmw2K/kMcmt3ttsDDLm1804JCQls2rQp10zndyZLu7PO2bNnM2/ePDZv3szWrVtRq9U0a9aMyZMn4+7uTlRUFDqdjgYNGlgcd/uc4WwlS5ZURlnExcXx0UcfsX37dlQqFZUqVVICMne+B3e26Z9//mHixIkcOHAAOzs7PD09qVGjRq7H3pktvmTJkpjNZiWAAjn7Tq1W37cNJpOJWbNm8c0335CWlka5cuXw9fXF3t5e2ad8+fKsXLmSBQsWsGbNGpYvX46rqyt9+/blzTffJCEhAYBJkyYxadKkHP0VGxvL5cuXef755y3Kb7/ec7sW7nbdPGp3fu7c3s/x8fGULFkyxzG5ld1Pfq4PgB9++EEZSZLtzhtrZ2dnWrZsqaxysGXLFmWYf16CGHdeY9k35dnTTh5W5cqV8fb2VlY52LRpE8899xzFilmuOHO/66kgXb16lWHDhnH+/Hlmz55tMSqkXbt2FjlKunfvzhdffEGHDh345JNP2Lx5MwMHDiQiIoIXXnghx7WTnRvh0KFDfPjhh/Tr16/A2y+E+E++gwRjxoxh0KBBdOvWjVatWqFSqdi4cSPBwcHs27fvrklNhBBC3F3kmRuErokkPSMTFyc77GzsMBhNxKU54VCpKwvemkBmcgxr1v7Aj+tW41aiFCqVip4v96V5izZcjrlMeffy2Guzbljs77hxvxedPuuJ2vkrSZT6NxeYrcaBho0a8+qg13Lsb2Njk+dzOzu7AJCYEE/pZ/67ifzn4nmSEhNx+nf7sBFjqFvf8umuChXu5UpQsfyzFkPygXxlEs8elurl5UVERASenp6o1Wp2797N1q1b83yebJ07d6Zz584kJyezb98+Fi5cSFBQEA0aNMj1RrlYsWLcvHkzR/mNGzcAcHNzIzY2Nt/tuBcXFxeaNWvGa6/lfP/ufGKa27FBQUEEBQURHR3Njh07CAsLY9KkSSxYsIA9e/bQrFkz7OzsLI6Lvy1hZrabN28qT1PfffddoqOjWbZsGfXq1UOj0XDr1q1c11y/nclkYujQodjZ2bFmzRq8vb2xtbXl7Nmz/PTTTzn2T0hIsAjK3Lx5ExsbG4oXL/5Q/bxgwQKWLVvGpEmTaN++PS4uWddu9pSPbL6+voSEhKDX6zly5Ajh4eHMmzePGjVqULVqVQDGjh2ba1b2YsWK8cwzzzzU9X43dz45z0uiRGsqU6YMFy5cyFF+Z84MuHfb83t9ALRp0yZHHz/zzDM5ro+AgACCgoKIi4tj06ZNDzRtwZoCAgJYvHgxH330EVu2bMkxdx+w2vV0p9OnTzN48GB0Oh1LliyhUaNGFtvnzp1rMaojO7+Bi4sL/v7+bN68mSZNmnDmzBk+/PBDi2OvXbvGa6+9xuXLl5k1a1aBJmYUQuQu34kLGzZsyNKlS3FwcGDRokWYzWaWLVvGjRs3mD9/Pk2aNLFGO4UQ4ollMplZ88sZ0jMyKVnMHns7G9RqFXH/HOOPH94nIy2RnUdiqODhxei3x+Hs7EJcXBzVqtfgn38uUM2rBhUrVaaaVw08KnuybPE8Iv84nKe6E1N0zFp1BPhv6Gmlsi40atSI2KuXqFq1Ol41auJVoybVvbxZ/d1K9u3ZmefX5uNbF4Bf9++xKF8wN5iQOTOoWMkDN7cSXLt6RanHq0ZNnin9DEsWhhB99gwajQYfHx+L/7K/YGY/6b+X6OhoEhISGDhwIFWrVlWOyc40np8ne2+++aYy393FxYWOHTsyYsQIMjMz73oD2qhRI3bu3GkxYsBoNBIREYGPj4/FXP+Ckr2qg7e3t9JntWvXZtmyZTkytt8uJiaGVq1aKQnRPD09GTJkCM2aNePKlStA7vkIAI4cOWIRKDhx4gSXL19W8jscOXKE9u3b07hxY+U1Z69ccK/3ID4+nvPnz9OzZ098fHyUIMfd3r/t27cr/zabzfz88880aNDgofv5yJEjVK1alZdeekkJEFy/fp2///5bacOyZcto06YNer0ejUZD06ZNlaSWV65cwdPTk5IlS3L58mWL67lMmTLMnDmTkydP3vN6fxDZT7SvX7+ulBkMBo4fP37P4/Lyu/Uw/Pz8uHz5ssVSfRkZGTlWE3F2drZoO8DRo0eVf+f3+oCsG9Q7+zi366NNmzZoNBpWrlzJsWPHilyQoGPHjiQkJDBv3jwSExNzjBgACvx6ys3Vq1d57bXXUKlUfPvttzkCBJC1SsXtbbg9UNGtWzeOHTvGt99+y7PPPmsRQEtJSeHVV18lNjaWpUuXSoBAiELyQItMN2rUiO+++46MjAwSExNxdnbGycmpoNsmhBBPheiYRGJiU3BxsrOYJ+pWtiqYTZzZswhdfHtczVc5dWw/qakptGztT7PnWjI+aAzTP5+El7cPN2Ov8uO67zl18gQDXh1y33rTMjKZuvwwcUkZqO0cyIi/SDntLd7q04qYS26MGj6ICWPfpGv3nmg0Gjb8tI79e3fx0afT8vzaqlSrTqs2bZkfNgddRgZVq3nx+2/7ObB/D5OmTMfGxobXh45g9vTPUNuoadq8JWkpyXyzfDGxsbHUqlXrnud3dXXl5MmTHDx48K7zUytXroyzszPz5s3D1tYWW1tbtm7dqjxdy88c4SZNmvDRRx8xdepUWrZsSVJSEiEhIXh4eCjDm+8UGBjInj17GDhwoPLEc+XKlVy6dMlqo+9GjBhB7969GTZsGH369MHe3p7w8HC2b99+z1WI3N3dKVu2LJ9++ikpKSlUrFiREydOsHv3boYNG0ZcXBwnTpzIsSQdZPXjG2+8wf/+9z9SU1OZPXs21atXp3PnzkDWE/YNGzZQq1YtypYty9GjR5k/fz4qleqe70HJkiVxd3fnm2++oWzZsri6urJ3715lRYE7j502bRo6nY7KlSuzevVqzp07x9df55ySk1++vr6EhYWxYMEC6taty8WLF5k/fz56vd4iZ8WMGTMYOXIk/fv3x8bGhu+++w6NRkObNm2wsbHhrbfeYuLEidjY2NCmTRuSkpIICwvj+vXr973eH4SzszN16tRhxYoVVKpUiWLFirF8+XIyMjLuOU3B1dWVo0ePcujQoRw5HApC586dWbBgASNHjmTMmDG4urqydOlSbt26ZbFqRuvWrYmIiKBOnTpUqlSJdevWcfHiRWV7fq+P/HB0dKRVq1YsWLAAX19fi5wb1rJs2bIcZa6urrlOD6hQoQI+Pj7Mnz+fdu3aPdC0k7Nnz+ZaZ/369fM85//TTz/l1q1bTJo0iZSUFItlep2dnZURNHfTokULihcvTnh4OG+88YbF38F58+Zx4cIFRo0aha2trcW5NRoNNWvWzFMbhRD5k+8gwYsvvsiLL75I586dKVWqVI45Qw/CZDIREhLC6tWrSU5OVtbKvduH8fr16wkKCspRvmPHDiUy2b59e4s/IvDf/CchhChKklL1ZBpN2NlYDt8uXbo0z7/8Lkd2rSFq93L+3JWJZ5WqfPzpNOrVz3pSM21mCEsXz2P/3hA0Gg3Va3gzY3aYsmLB3RgyTZyIvkVpp6xkbiWr+ZMY/Qu/b5xDYu8WVKlajS9DF7FkQRiffzIRs9lMZc8qTP58Js2fy5mT5l4mTPyUr5fMZ+3qb0lMSKCihwcffTqN51q2AaBTl+44OTrx3arlbFz/A06OjtSvX5+ZM2fe90v566+/zmeffcbgwYNZunRprvu4uLgQFhbGtGnTGDNmDE5OTnh7e7Ny5UqGDBnC4cOH77pe/J169+6NwWDgu+++Y9WqVWi1Wpo2bUpQUFCO4ffZqlWrxqpVq5g1axbjx49HpVLh6+vL8uXLrXLzBVmrBnzzzTfMnj2bsWPHYjabqV69OqGhobk+bbxdSEgIs2bNYs6cOcTHx1OuXDkCAwMZOnQoGzdupFq1arlOq2jYsCFNmjRRVi/w9/dn7NixyhPaL774gk8++UR5su7h4cEHH3zA999/ryQwvJuwsDCmTJnCuHHj0Gg0VK1alblz5/LZZ59x+PBhi1UrPv74Y+bPn8+lS5eoWbMmS5YsKZB+HjZsGPHx8SxfvpzQ0FDKlStHt27dUKlUzJ8/n6SkJGrUqMG8efMIDQ3l7bffxmg0Urt2bZYsWaJkiO/VqxdOTk4sWrSI8PBwHP+93mfMmGG1m9BJkyYxY8YMPvjgA5ydnenZsycNGjRg9erVdz1m+PDhhIWFMWTIkFxzWzwsW1tbFi9ezJQpU5QM/V27dqV48eIWCevGjx9PZmYmU6dOxdbWloCAAN555x0++OADZZ/8XB/5FRAQwJYtW/K8qsHD+vzzz3OUVaxY8a45BAICAoiKinrgUQ5RUVE5liCFrOnFeQkS6PV6du3aBcBHH32UY7ufn999l+PMTuS6YsUKunbtarEtO1dEcHBwjuCku7s7v/zyy33bKITIP5U5L4vy3mbEiBHs3bsXk8lEkyZNePHFF2nXrt1DBQtCQkJYuXIlX3zxBWXLlmX69OlcvnyZDRs25Dr8a/r06Rw/fjxHZuUSJUpgY2NDWloaDRo0YO7cuRZRea1WqwwRzI/sD08fn3t/6bamtLQ0Tp06hbe3d5FNUPQ4k/61Lunfezt7KYHPlh1Ea2+DvZ0NKhW4OmpQq1UkpOhITjWgMxgZ3sOXSuVccxyfnp7OuXPnqFKlyj2T2WU7EHWVFZtPYfo3uaAK6Pl8NZ5vVPHeB1qZWgWuThqcHAp++P2DMJvNpKenP3bXrtlstlrm8rvJvgm7383AnQrys2HdunWMHz/e4oHB064of/aeOXOG6Oho2rdvb3G99uzZk7JlyxISEvIIW5c3Rbl/H3dFvW+Lwr2BENaU7wlnYWFh/Prrr0yaNAmz2cy4ceNo1qwZ7733Hr/++muuWWTvRa/Xs2TJEkaPHk3r1q2pUaMGs2fP5tq1a/z888+5HvP333/j5eVF6dKlLf7LTqZ19uxZTCYT9erVs9j+IAECIYSwNk/3Yrg/40xymgG1Goo724NKxa3EDDJ0maTqDJQp6UiFMg/3GWY2m9mwN5qvI04qAQI7WzXDevhKgOAOYWFhLF68+FE3I99Wr17N1KlTH3UzLJw+fZoXX3yR2rVrF9rT2Efh5MmT1KtXL9cl93Izbtw4ixEsXl5euU7juPMYLy8vWrZsedfvWzNmzMDLy8vi6XlAQICyTGBe+fv7WxyTl/YNGDAALy8vevfufdd93nrrLby8vJg+fTpjxozhk08+4cCBA+zbt48JEyZw4sQJ+vfvz7p16/Dy8uLy5ct5bvOKFStyHRWk1+uZN28eHTp0oG7durzwwgtKcsmHZTKZWL16NV26dKFevXo8//zzfPbZZxY5SIQQ4nHzQDkJXFxc6NmzJz179uTWrVts2bKFLVu2MGTIEEqVKsXu3bvzfK6//vqL1NRUJbERZM29qlmzJocOHVLmMt7u9OnT9xwaevr0aUqVKpVjGZiHYTabH2km4Ow5dQW1vq6wJP1rXdK/99eleUW+3nQasxlS0vWkpBrQZRpJ0xnR2qnxr/8sOl1GrsfqMnQW/89NptFE+PZzHP7rhlLm7GDHG11rUKms8yN9b9RqFcWc7FGZM0lLy7z/AYVgzpw5DBs27LG7dkNDQ2nYsGGh/73Kzj6fW71z5swhJiaGmTNnUqJECYt9CrJ/s2/4MjIyHsnf6+3bt9OoUSMyMzNzrEufm8zMTEwmk0VbDQbDPduemZmJWq3m+vXrHDhwQFlC8nYRERFA1nuS3a+fffYZJUuWzFe/zJgxAycnp3y1z2g0olarOXbsGOfPn88xLSU9PV0ZHu7q6srUqVNZvnw5P/74I2azGS8vL0JDQ/H19VVWPsjr+7llyxa++OILSpcunWP/Tz/9lIiICIYMGUKtWrU4efIkCxYs4J9//uHjjz/OY4/klJ6ezsaNG1m9ejUDBw7Ez8+Pf/75h7CwMP766y/mzp1b6KN6nhRF/bP3UYzYEqIwPVCQ4Ha3bt3i5s2bJCUlYTQa831jfu3aNQDKlStnUf7MM88o226XmJjI9evXOXz4MKtWrSI+Ph5fX1+CgoKoXLkykBUkcHR0ZPTo0Rw9ehQ3NzdeeuklBg4c+MDZeg0Gg0UG3kclt+WCRMGR/rUu6d+709jY8EL9Yhw4lciF62mYjGZUahVuzjY0rOqIWn+Tc+dyLqN3u8sxuT9xy9Cb2HwkgZhbBqXMzdmGLn7FyEyN5dy5gl1+Lz/sbG1xddKQpDblWObsUbtx44ZyzT4u167BYCAxMbHQ/1698847ALnWe/XqVZ599lllWcLc9imI/s3O/ZCYmEhiYuJDny+/tm3bRosWLfLc94mJiTm+W9y4ceOexycmJlKiRAkAvvvuO+zt7S22nzlzhuvXr1OhQgXS0tKUfrW3tyclJSXf18Wdx9yvfWlpaXh4eHD58mVWrlyZY+TIr7/+ikajwdHRkcTERMqXL8+ECRNynOfUqVPKahpnz5695/uZmJjI6tWr+eWXX3B2ds7Rp8nJyaxbt47evXsrD6SaNGlCbGws3333HR07dsTVNec0rrwwmUysX78ef39/2rVrB2QNPx84cCDBwcFs2rRJyUUhHkxR/uy1xso0QhQVDxQkuHTpEhs3bmTTpk2cPXuWUqVK0blzZ6ZOnXrX7M53kx0hvPMXzd7ePtc/CmfOnAGyIniff/45GRkZzJ07l759+7JhwwZKlSrFmTNnSEpK4oUXXmDkyJEcOXKE6dOnk5iYyJgxYx7kJWNnZ3ff7KzWlJ6ezoULF/Dw8MjTnGORP9K/1iX9e3+pGZm4ltLjXdXE5RuppGQYcNbaUb60Eyr1vZ9W6DJ0XI65THn38thrLW8a4pIyWPDTKa7H/RcgqFreldc61cBR+9Bx4oeiVqso5miPg9Ymz8ecPHmSOXPmcPLkSUwmE7Vr12bkyJFKgq2jR48SGhqqLCXXsmVL3nrrLeXGav369UyePJmlS5cyffp0Tp8+TYkSJejTpw8DBw4EoF69ekDWHPd169axatUqPDw8iImJ4auvvlKWX/Pz8+Ptt99W5r8fPnyYIUOGMG/ePJYtW8axY8dwcnKiS5cuBAYGKlPiDAYDCxYsYPPmzdy8eZPy5cszcOBAi4RdO3fuZOHChZw7dw4XFxfat2/PqFGj7vn7ExAQwM2bN9mzZw979uwhIiKCw4cPM3nyZCZMmEBYWBgGg4ElS5bg4eHB8uXLiYiI4PLly6hUKqpXr05gYKCyfNm8efPYtGkTQUFBBAcHc+HCBcqVK8eQIUMsRvmtWrWK1atXc+XKFYoVK0br1q0ZPXo0zs7OSl8C9O3bl0mTJtG1a1cuXrxIcHAwf/zxB6mpqdSqVYtRo0YpT8avXLlCp06dePvtt1m3bh3Xrl1j3LhxXL16la1btzJq1CjCwsK4dOkSHh4eTJgwAZVKxfTp0zlz5gzly5cnKCiIxo0bK/WfPXs2T+/f+++/z+LFi0lOTmbGjBlUr16dGTNmcPDgQZKTk/Hw8KB///506dJFOXdSUhLnzp3jyy+/pFy5chiNxvv2b7FixbCzs8Pb21s5T+nSpS1+vlP2Me3atWPLli18/vnnFk8zIyIiaNKkCRkZWaOOPDw8uHDhAm+//TaNGjVi8uTJSt9OmzaNrVu3cuDAAWxtbXn++ecJCgpSrrGAgAAaNmzI5MmT89w+R0dHHB0d8fT05Pjx40rwKNvChQt54YUX2LdvH8WKFbvnubK/71WtWtVitYM7TZ8+nVOnTjFjxgx2797N4cOHLc4bExNDz549efnll/Hw8FDKr127xnfffYejo2Ou7dDpdLRt25bu3bvz9ttvK+WZmZm0a9eOjh07MmjQIFq0aEH37t0tzqFWqwkODsbW1vaer1HcXVH/3nD27NlH3QQhrCrf3xBfeuklTp48iVarpV27dowbN46mTZsqT+jzO/wmO+GhXq+3SH6o0+ly/VBo2LAhBw4cwM3NTaknJCSE1q1bs27dOoYOHcrChQvR6XRKDgIvLy9SUlKYO3cuo0aNeqDRBCqVqkgkTnFwcCgS7XhSSf9al/RvTmazmYQUHQaTGnt7LdhDdacH6yN7rb3F5+b5K4mErYkiOe2/AEHjWmUZEOCNrY1110C/H7UKXBw1ODvm/UlMSkoKgYGBNGnShODgYPR6PXPnzmXkyJHs2rWLv/76i+HDh9OkSRO+/PJLEhMTmTNnDsOHD2fNmjVotVo0Gg0mk4lx48YxaNAg3nnnHdasWcPs2bOpVasWLVq0IDw8nFdeeYWePXsqN4KxsbEMGjQIT09Ppk6dSmZmJnPnzuX111/np59+omTJkspT3Q8++IC+ffsyfPhwdu3axbJly6hcubIyT3v06NHs3r2b//3vf9SpU4fdu3fz0Ucf4ezsTOfOndmwYQPvvvsuXbp04e233yYmJobZs2dz4cIFli5dete/sWFhYQwdOpSaNWsyYsQIKlasyPHjxzEajXzzzTd89tlnxMfHU7t2baZNm8a3337LO++8g5eXF9evXyc0NJSxY8eya9cuHBwcsLOz4+bNm0ydOpX//e9/uLu7s3jxYj788EMaNGhAlSpV2LhxI19++SXvvfceXl5eREdHM3XqVAwGA1OnTiU8PJxJkyYBWZnPK1asyJUrV+jXrx8eHh6MGzeOa9eusXfvXoYOHcqSJUvw8/NTvg/Mnz+f999/X1nGb/Xq1Vy/fp3Zs2fz1ltv4ejoyCeffMLYsWOxs7Nj+PDhlCtXji+++ILx48eza9cutFot58+fz/P7t2DBAj744AMyMjJo0qQJo0aN4tatW0yePBlnZ2d++uknJk6cSKVKlWjSpAmQFdTx9PSkSpUqAEydOvW+/Wtra4tarbb4TLSzs7vnZ2T2MV27dmX58uX89ddfNGjQAMh6qr19+3befvtt1q5dC6B8HqhUKmxtbXF0dFT6dsqUKbz00ksMGDCA48ePM3v2bEqXLq3c2KvVauWYvLYvOxDWpUsX3nzzTZKSkihbtqzy+7t//36WLl3Kr7/+muPcd8p+eKTVau+5X//+/ZkwYQJ2dnbs378/R59Wq1aNTz/9NMdx+/btw87Ojho1auR6fkdHR1544QW2b9/O+++/r/ze7d69m4SEBF566SWeeeYZXn311RzJ9fbv3w9ArVq15G/eQyqq3xtkqoF40uU7SFC8eHG++OIL2rdvb/FlNDY2lu+//561a9eyc+fOPJ8ve5pBbGwsFSv+lzgrNjYWLy+vXI/JfiKUzcHBgfLly3P9+nUg6w/LnSMTqlevTlpaGomJibi5ueW5fUIIYS1Go4mEZB06g5H8pXy9v2N/32Dx+hMYMk1KWafmlen8XOVH/uXmQQIEkPXkJj4+noEDB1K/fn0APD09CQ8PJzU1lZkzZ1K5cmXmz5+v3KzUqVOHTp06sXbtWvr16wdkBWZGjBhBr169AGjQoAHbtm1j165dtGjRQnmaXbZsWXx9fTl16hTz58/HwcGBZcuW4ezsDEDTpk1p27YtixYt4r333lPa2atXL0aOHKnss337dnbt2kXv3r35+++/2bp1KxMmTODVV19V9omJieH333+nU6dOzJgxgxYtWjBjxgzlnB4eHgwaNIjdu3fTunXrXPunZs2aaDQaSpQokWOu+vDhwy2Oi42N5a233rJIbmdvb8+oUaM4ffq0cnx6ejpTpkxRhml7eHjQpk0bdu/eTZUqVTh48CDly5enX79+qNVq/Pz8lKHkAHXr1lX6K/uckydPRqPRsHz5ctRqNadOnaJPnz68/PLLTJs2jTVr1iht6tixIy+99JLFa0lPT+ejjz6iZcuWynUxc+ZMpkyZQs+ePYGsYe+jR4/m/PnzeHt7ExISkuf3r2/fvnTo0EH5+eDBg4wcOZK2bdsCWSMQihcvbvE9Y8+ePUp78tO/D8rHx4cKFSqwefNmJUhw+PBhEhISaNu2rRIkuJdWrVopr7tp06bs37+fXbt25Xj6/yBat26Ng4MDW7ZsYdCgQUDWdIySJUsq7S0o2YGZ/Ni2bRs//PAD/fv3v+c02W7durF27VqOHDmiLKMZERGBp6cnPj4+ueZKiIyMZMGCBbRp04bq1avnu21CCFEU5PtR0uLFi+nWrZsSINi7dy8jR47E39+fkJAQ5YtZXtWoUQNnZ2d+//13pSwpKYmTJ08qQ/JuFx4eTuPGjS0+mFNSUrhw4QJVq1bFbDbTtm3bHEvnREVFUbp0aQkQCCGKBL3BSFxSBhlWCBDsOPQP89cdVwIEarWKVzvVpEsLz6IRIHDKX4DAZDJz9lICaebiFCvuxvDhw5k4cSLbtm2jVKlSBAUFUaxYMSIjI2nVqhVms1lJHlehQgWqVKmiPNnLdvsw+Owb63slRzt48KDyhDv73M7OzjRs2JBff/31rueGrGBD9rmPHDkCQPv27S32CQ4O5pNPPiE6Oppr167h7++v1JOZmUmjRo1wdnZWXsft2zIzM++7stCdQ55nzpzJq6++SlxcHIcPH2bt2rWsX78eIEfG99tvaLOfCme/niZNmnD+/Hl69OhBSEgIUVFRdOnS5Z7r0h88eJA2bdooN+vw3zrpJ06cIDU19a7tzpYdJAKUXAd16tRRyooXLw5kfZ8A+O233/L8/t1ZZ+PGjQkODmb06NGsXr2amzdv8t577yltMJvN7N271yJIkJ/+zY3RaLR4f00mU459AgIC+Pnnn5X3PiIigtatW1v0673cGai4/Tp9WFqtFn9/f7Zs2aKURURE0LFjxxyfQSaTyeK1Wjs3yc8//8zbb79NgwYNCAoKArD4zLg98aSfnx/PPvuskgxSp9Oxfft2unXrluu5jxw5whtvvEH58uX5/PPPrfo6hBDCmh5oQmpcXBxr1qzh+++/JyYmBmdnZ7p37063bt2USGteaTQa+vfvz4wZMyhRogTu7u5Mnz6dsmXL0r59e4xGI3Fxcbi4uKDVamnZsiUzZsxg7NixjBkzhoyMDGbNmkWJEiXo0aMHKpWKdu3asXjxYjw9PalduzYHDhxg0aJFvP/++w/ycoUQokBl6DNJSNZhNBVseMBkMhO+7TQ7j/yXwNDB3pZh3X2o4VHiHkcWDiVAkI9lDiPP3GDNL2eIiU0h02jCs8VI4s7uYOPGCMLDw9FqtXTr1o1hw4ZhMplYuHAhCxcuzHGeOxO83T69DbKGVt/rRjsxMZFNmzaxadOmHNvuHN12r3MnJCQAULJkyVzryd4+adIkZZj+7WJjY7l8+TLPP/+8Rfnnn39Ojx497tr+O4frRkVFMWnSJKKionBwcLCY931nP9w+avD2qYWQdaNqMplYtWoVYWFhBAcH4+7uzrvvvnvX5Q4TExOVG/vblSpVCrPZbLF03N2GGed2I3yvecsJCQl5fv/urHP27NnMmzePzZs3s3XrVtRqNc2aNWPy5Mm4u7sTFRWFTqezeEKen/7NTbt27YiJiVF+7t69O1988YXFPgEBAcyfP5+jR49St25dfv7553xl6r+zv+73O5BfHTt2JDAwkGvXrmFvb8+BAwd48803c+w3YcIEfvjhB+Vnd3d3ZQWEgrZs2TKmTp2Kn58foaGhyufCDz/8wPjx4y323bFjB+XLl6dLly6sXr2aDz74gJ07d5KWlmaRjyLbpk2bGDduHB4eHixatEgeSgkhHmv5ChL89ttvhIeHs337doxGIw0aNCAmJobQ0FD8/PweuBGjR48mMzNTmQPYqFEjFi9ejJ2dnfJlKPsLULly5Vi2bBkzZ86kT58+mM1mmjdvzvLly5UP+3feeQdnZ2dmzZrFtWvXKF++PO+//z4vv/zyA7dRCCEKQkq6nuRUA6YC/DIOYMg0s2TjX/x5Pl4pK+GqJbBXHZ4tnbcni9akVoGrkwanfAYIQtdEkp6RiYuTHXY2dhgcn8XOuTeVGvWlvY8dfx7dzbfffkuZMmVQqVQMGjSITp065TjXwya+cnZ2pnnz5rz22ms5ttna5v1PaXYW9bi4OOWpPMC5c+dISEhQto8dOzbXv6vFihXjmWeesRiSDyjJ9/IiJSWFN954Ay8vL2XotFqtZvfu3WzdujXP58nWuXNnOnfuTHJyMvv27WPhwoUEBQXRoEGDHEvgZb+GmzdzrtRx40bW8pxubm7ExhbsihsuLi40a9bsgd4/FxcXgoKCCAoKIjo6mh07dhAWFsakSZNYsGABe/bsoVmzZtjZ2QEF079z5861GHGQ2w1njRo1qFy5Mlu2bCEjIwOdTnfXqSiPQsuWLXFycmLLli04OjpSvnx5ateunWO/wMBAZSoQWCdjvNlsZsqUKaxYsYLOnTvz+eefW9TTpk2bHL9TzzzzDJA15WD+/Pn8/vvvbNq0iUaNGuHu7m6x7/Lly/nyyy+V4EN2TiwhhHhc5embzbJlywgPD+f8+fNUqlSJESNG0L17dxwdHfHz83vo4as2NjbKH+A7lS9fntOnT1uU1apViyVLltz1fLa2towcOVKZEyqEEI+a2WwmKVVPaoaBAo4PkJSqZ92BOG4k/rc2e6WyLozoWYdizvb3OLJwPEiAwGQys+aXM6RnZFKymD0qlYqr547w595VPNfrQ1IMWo5ftWfSxI+IiIjg1q1b1KxZk+joaHx8fJTzZGRkMHr0aFq1apWvFWruTHDboEEDzp49i7e3t3JTaTabeffdd6lUqVKeM5hnP23+5Zdf6Nu3r1I+Y8YMrl+/zurVqylZsiSXL19m8ODByvbY2FjGjh1L7969qVixosVrvFe7cxMdHU1CQgIDBw606JM9e/YA5Dq0/W7efPNNDAaDcmPUsWNH7OzsGDlyJLGxsbkGCRo1asTOnTtJSUlR2ms0GomIiMDHx8cqN4l+fn4P9P7FxMTQt29fxo8fT4cOHfD09MTT05Njx45x8eJFIKvfsvNbQMH0791yMt0pICCAtWvXkpaWRrt27XKMmHmUNBoNbdu2ZevWrWi12lyDd5D1PS8/Qa4HMWvWLFasWMFrr73Ge++9l+N7q5ub212f/FepUoVatWoRERHB7t27+eCDDyy279ixg8WLFxMQEMDUqVNlWTwhxBMhT0GCL774Ai8vL5YvX27xZCM5OdlqDRNCiCeF0WTOSlCozyzw/ANXbqQQ/P1x4pP/CxD4Vi3F4K61sdfkL0eMNahV4Opsj5PWLl/HRcckEhObgouTnfKF3q1sVcxmE0e3zqOiTzv+vKHhrXd/Ijk5mfbt2+Pv78/QoUN555136Nq1K0ajkSVLlnDs2DFGjBiRr/pdXV2VYdwODg4MHTqUQYMGMWzYMPr06YO9vb0ysu6rr77K83lr1KhBhw4dmD59OhkZGXh7e7Nnzx527typ5PV56623mDhxImq1Gn9/f5KSkggLC+P69evUqlXrvu0+efIkBw8eVJaFvFPlypVxdnZm3rx52NraYmtry9atW5UnqdlLE+dFkyZN+Oijj5g6dSotW7YkKSmJkJAQPDw87rokcmBgIHv27GHgwIG8+uqrXL9+nTlz5nDp0iUWLVqU57rzY8SIEfTu3Tvf75+7uztly5bl008/JSUlhYoVK3LixAl2797NsGHDiIuL48SJEwQHByvHFGT/3k9AQAChoaH89NNPhIWFFdh57+bYsWMsW7YsR3mLFi1yTSAYEBDAsGHDUKvVOW6u82rt2rU5kguq1WplydL7OXXqFAsXLsTHx4cOHToQGRlpsb1q1ar3zePQrVs3pk6diq2trUVSy5s3b7JixQqeffZZ+vXrx8mTJy2Oq1ixYo7pLEII8TjIU5CgU6dO7Nixg2HDhtG0aVO6d+9OmzZtrN02IYR47BkyjSQk69Bn5v3pbF79dSGO+T9Eka77L0DQpkF5ej1fHbX60S/PpFapcHXW5DtAAFmjIzKNJuxs/jtW61SMRp3G8PfBnzi17xsyM/WkV65CcHCwshTd4sWLCQkJYfTo0djZ2VGsWDF69uyZ72zyw4cPJywsjMDAQKZNm0aLFi345ptvmD17NmPHjsVsNlO9enVCQ0Nz5Ae4n+nTpxMSEsLXX39NfHw8VapU4auvvlKy5/fq1YuoqCg2b97M999/j6OjI/Xr12fGjBlUqFDhnud+/fXX+eyzzxg8eDBLly7NdR8XFxfCwsKYNm0aY8aMwcnJCW9vb1auXMmQIUM4fPgw/v7+OY47ffq0kg3/m2++YdSoUfTu3RuDwcB3333HqlWr0Gq1NG3alKCgIGX4/Z2qVavGqlWrmDVrFh9//DFmsxkfHx+WL1+e77xGeVWjRo08v38XLlxg+PDh/P7772g0GkJCQpg1axZz5swhPj6ecuXKERgYyNChQ9m4cSNOTk706dNHmUffsGFDXnzxRc6ePXvX/v3555/54YcflDwAuY3InDFjBgsXLsTPz48VK1YA5JiuULVqVapXr86NGzdo1qxZrq999OjRFskg83K9DhgwgJiYGHbv3m1Rvm/fPvbt25djfzc3t1yDBM2aNUOr1ZKenn7fUQ5ms5klS5YQHh7O1atXlak3uQU/bGxs8hwkyE7uGBUVxSuvvJJj+/Lly2ncuPE9z9G5c2emTZtGmzZtLKYS7Nu3D71eryzreaf75QoRQoiiSmXOY5aalJQUNmzYwLp164iKisLNzY22bduyZs0aVqxYYbU/7EVBVFQUwF2HdxaGtLQ0Tp06lWMtXlEwpH+t62ntX50+k/gUHUZjQY8fgF+PX2Hllr8w3Zb88MWWHnRolv/lwKxBrVJRzFmD4wMECADOXkrgs2UH0drbYG+Xc0SETm8kQ29kwiA/qlYoftfzeHl5ERgYyKhRox6oHY/q2vX398fPzy9HsrpHafTo0fz2229Mnz6dEiVKFMjfxKL42TB37lyOHz/O3Llz87T/uHHjOHjwoBIkyMs1N27cOH766Scl6WNuywL6+/sTExNjESQ4efIkzs7OFktG30taWhqbN2+mdu3ayhSGvLRvwIABHD58GLPZzK5duyzyZ2Sft1mzZqSnp+eaVPF269atY/z48UoiwLuZOnUqK1asYPTo0fj4+LBnzx6WLFnC5MmTc725zyuz2cyQIUOIiopi9OjReHp6cuDAARYuXEhgYOBDTU0titfvk6Ko921RuDcQwprynG3J2dmZPn360KdPH86cOcPatWvZsGEDZrOZCRMm0KlTJzp16pSvOZ9CCPGkSk3Xk2SFBIVms5mN+84Tsf+8UmZnq6ZdXRda1Xu2QOt6UGq1iuLOGhzsHyxAAODpXgz3Z5y5cDUJjava4kmr2WwmOd2ARzlXPN3vvsa5KFjx8fFUr16dVq1aPeqmWNWePXvuusRdQSpXrhxms5nNmzfnCBIcO3aM69evU716dYvymjVr5rseDw+P+45AyU3NmjU5e/YsW7ZsYdCgQRbbdu7ciYODg/K0/2FdvnyZZcuW8eGHHyq5Opo2bcrVq1fZt2/fQwUJTp48yd69e/nyyy/p2LGjcu7ExEQWLVrEiBEjHvnSsEIIUdTcP8NRLqpVq8a4cePYvXs3wcHBeHp6snDhQrp06ULXrl0Luo1CCPHYMJvNJKboSEzVW2EFAxPLNp60CBC4ONox8qVaeJbV3uPIwmOjVuHmYn/fAMGJEyd49dVXadCgAfXq1WPQoEEcO3ZM2X706BGifp7D4e/f5eelb3N0+xIyUpPQ6Y3cStKR+M8h1nz5BlFRx3nllVfw8fGhTZs2LF68WDlH9pPTkJAQi0Rwf//9N8OGDaN+/frUr1+fkSNHcunSJWX777//jpeXFwcOHOB///sfgwYNom3btkyfPt1iDXe9Xs+XX37J888/j6+vL507d7ZYyg1g+/bt9OjRAx8fH5o3b86nn35637Xos58g//DDD3h5eXH58mXWrVtHzZo1Wb16Nc2bN1cS8RmNRhYsWEDnzp3x9fWlbt269O7dm99++005X3BwMO3atWPXrl106dKF2rVr88ILL/Djjz9a1Pv111/ToUMHfHx8aNGiBR9//LGyHKGXlxcHDx7k0KFDeHl5sW7dOiBraP7o0aNp3rw5devWZcCAARw5ckQ55+XLl/Hy8mLp0qV06NCBOnXqsHbtWoKDg+nQoQO//PILY8eOpXHjxnTr1o0//viDY8eO0atXL6VPDxw4YNHOvL5/3333HW3atKF+/frs37+fuLg43nnnHZo3b46Pjw/dunXL0QeJiYlERkbSsmVLgDz178Po0KGDMhz+dps2baJZs2YUL17cotzf359x48ZZ9O3mzZsZPXo09erVw8/Pjw8++MDiGhs9ejQTJ07Md9scHR1p1aoVW7ZsybFt06ZNvPDCC/la2eNetm/fjr29PT179rQo//LLLy3yPdzp6NGjeHl5sXPnTovyU6dO4eXlxbZt2wB45ZVXaNq0qcU+np6epKWlcevWrQJ5DUII8SR5oCBBNltbW9q1a8e8efPYtWsXb7/9NpmZmfc/UAghnkBGk5n4ZB2p6QW/gkFqhoHg7//g9z+vKWVlSzry3sBGVCpbNJbbyg4QaDX3vnHIXiLOzc2N4OBgZs+eTXp6OoMHDyY5OZlDhw4xaNAgSrm5EPT+pzTw70v81b85sH4mqWnpeJRz5flGFTGbTbz55psEBASwYMEC6tevz7Rp09i7dy8A4eHhAPTs2VP59/nz5+nduze3bt1i6tSpTJkyhUuXLtGnT58cNwvvvvsu9erVIygoiA4dOrBo0SJWr15tsX3p0qX06tWL+fPn89xzzzFu3Dg2btwIwIYNGxg5ciSenp6EhoYSGBjI+vXrGTFixD3Xow8JCaF06dK0atWK8PBwZSm27ESMU6ZMYfz48VSpUoUZM2YQFhbGK6+8wqJFi/jkk09ISEhgzJgxFgnybty4weTJkxk4cCALFiygfPnyvPfee5w7dw6AjRs3Mn36dPr168fixYsZOXIkP/30E5988onSlzVr1qRmzZqEh4fTunVrzp49S48ePbh8+TIffPABM2bMQKVS8eqrr3Lw4EGL1xQcHMyQIUOYNm0azZs3B+DatWvMmjWLF198kWnTppGUlMTo0aN5++236dWrF6GhoZjNZt566y0yMjLy/f6FhITw3nvvMXHiROV9PHfuHJMmTWLhwoXUrFmT9957z+KGf9++fXh6evLss1mjcvLavw8qICCA69evc/ToUaXMZDKxZcuWu64GcKePPvoId3d3wsLCGDx4MGvWrMnzVIm8tO/YsWNcu/bf505KSgp79uyhc+fOBVIHZN3UV6pUiUOHDtG9e3dq1aqFv7+/8nt7N/Xr16dixYpERERYlG/cuJHixYvTqlUratWqxeTJk3MEXLZv306JEiUksaAQQuSiYELAQKlSpRgyZAhDhgwpqFMKIcRjw5BpIjFFh85gvP/O+XQzIZ2Q1ce4duu/p4PVKxZnWA9fnLR2BZot/UHZ2Khwc7bH/j4BAoCzZ88SHx/PwIEDqV+/PpD1VC88PJzU1FRmzpxJ5cqVmT9/PjY2NgzqY2bv7y8w/PXe1Ct9ldFDOvHjj9cwm82MGDFCWX6uQYMGbNu2jV27dtGiRQslWWHZsmWVf4eEhODg4MCyZcuUjOZNmzalbdu2LFq0SEnMB1kJBIcOHcqpU6fo2bMnu3fvZteuXfTu3Zu///6brVu3MmHCBF599VXlPDExMfz+++906tSJGTNm0KJFC2bMmKGc08PDg0GDBrF79+67rmlfs2ZNNBoNJUqUyJFwcfjw4RbHxcbG8tZbbzFgwAClzN7enlGjRnH69Gnl+PT0dKZMmaI8TfXw8KBNmzbs3r2bKlWqcPDgQcqXL0+/fv1Qq9X4+fnh6OhIYmIiAHXr1lX6K/uckydPRqPRsHz5cmVb69atlSRvt68737FjR1566SWL15Kens748eMpUaIE3t7eXLp0iZkzZzJlyhTliXJaWhqjR4/m/PnzeHt75+v969u3r0Um+oMHDzJy5EglQaSfnx/Fixe3WLJuz549yiiC/PTvg/Lx8aFChQoWUw4OHz5MQkICbdu2Ze3atfc9R6tWrZTX3bRpU/bv38+uXbt45513HqptkPV+Ojg4WEw52LZtGyVLlsw1j8KDiouL4/r167z77rsEBgbi6enJpk2blBEQ95pu0LVrV5YsWUJGRgZarRaz2cymTZvo0KHDXZcj/Prrrzl48CDjxo3L07KhQgjxtCmwIIEQQjytdPpMElKysvEXtPNXEglbE0lymkEpa1yrLP07emNnWzS+3NraqHBz0aLJJcFgNpPJTHRMIkmpeuwcn6FEiRIMHz6cDh060KJFC5o3b05QUBDp6elERkYyePBgzGazMjqteaOaVK1ahb9P/mGxckO9evWUf2ffWN9rOP9vv/2Gn58fWq1WObezszMNGzbk119/tdj39nNDVrAh+9zZQ+rbt29vsU/20Ohz585x7do1hg0bZjHCrlGjRjg7O7N//35at26dY/SdjY3NPedHe3t7W/w8c+ZMIOsmKzo6mosXLypDr/V6vcW+t9/QZieiy349TZo0ITw8nB49etC2bVtatWpFly5d7tmWgwcP0qZNG4vl42xtbenUqROhoaEWGfXvbHe2OnXqKFMFSpUqpZRly376m5SUBOTv/buzzsaNGxMcHMzJkydp0aKFxc01ZE0V2rt3L7NmzVLK8tO/uTEajRajRtRqdY6b0oCAAH788Ufef/99VCoVERERtG7d+r7L8mW7M1BRtmxZYmJi8nTs/Wi1Wvz9/S2CBBEREXTs2DHHtWEymTCZ/vsMVKlU2NjkbRlWg8FAfHw8wcHByu9U06ZNuXLlCiEhIbzyyit3PX/Xrl0JCQlh586ddOzYkaNHj3LlypW75pVYuXIln3/+OR07dsyRa0EIIUQWCRIIIcRDSMswkJSqx2gq+BUM/jgdy5INf2K4bfnETs0r0/m5ykUm0ZadjRo3V3vsbO9+MxB55gZrfjlDTGwKmUYTtjZq/Lq+S+rF3WzevJnw8HC0Wi3dunVj2LBhmEwmFi5cyMKFC3Oc685l1LRay1wM2UvK3U1CQgKbNm1i06ZNObbdOez4XudOSEgAoGTJknetB2DSpElMmjQpx/bY2FguX76cYzm6+y2ZdmeW76ioKCZNmkRUVBQODg5UrVpVGSp/Zz84ODhYvJbb9wkICFAy7YeFhREcHIy7uzvvvvsuAQEBubYlMTFRubG/XalSpTCbzUo+g9zanS23G+Hb23mn/Lx/d9Y5e/Zs5s2bx+bNm9m6dStqtZpmzZoxefJk3N3diYqKQqfTWTwhz0//5qZdu3YWN+y5rQQQEBDA/PnzOXr0KHXr1uXnn3/m448/vu+5s93ZX/f7Hcivjh07EhgYyLVr17C3t+fAgQO8+eabOfabMGGCRU4Od3d3ZcWH+3FyckKlUuVIitmiRQv27dvHzZs3mTFjRq7nr1SpEvXq1VOCFxEREVSsWFEZpZTNZDIxbdo0li5dSufOnZk6dWqR+RwVQoiiRoIEQgjxgJJT9aSk6yno+IDZbGbHoUus/eUM2adWq1UM6OhNU59yBVvZQ9DYqinuor3niIbIMzcIXRNJekYmLk522NnYYTCaiEtzwqFSVxa8OQHSr/LTTz/x7bffUqZMGVQqFYMGDcp1Tva9biDzwsXFhWbNmvHaa6/l2JafJGzZWd3j4uIsloc7d+4cCQkJyvaxY8fi5+eX4/hixYrxzDPPWAzJB+65RNydsnM7eHl5ERERgaenJ2q1mt27d7N169Y8nydb586d6dy5M8nJyezbt4+FCxcSFBREgwYNKFOmTK6v4ebNmznKb9y4AYCbmxuxsbH5bse9PMz75+LiQlBQEEFBQURHR7Njxw7CwsKYNGkSCxYsYM+ePTRr1gw7u6ykmwXRv3PnzrUYceDm5pZjnxo1alC5cmW2bNlCRkYGOp3urlNRHoWWLVvi5OTEli1bcHR0pHz58tSuXTvHfoGBgfTr10/5+W5D/XNTqVIlzGYzBoPBIhCYPVpEq9Xe8/xdu3bl888/Jzk5mS1bttCnTx+L8+v1et555x1+/vlnXn/9dcaOHSsBAiGEuAcJEgghRD6ZTGYSU3Wk6zILPEGhyWTm++1/s+voZaXMwd6WYd19qOFRdBJsaWzVuLlqsbW5e4DAZDKz5pczpGdkUrKYvfKlPO78MU7sXUWtDu+xbnc0k4Y0VZ4E3rp1i5o1axIdHW2x/nRGRgajR4+mVatW+Vpq986h3dmrAnh7eys3lWazmXfffZdKlSrddVj8nbKfNv/yyy/Kkm2Qleju+vXrrF69mpIlS3L58mUGDx6sbI+NjWXs2LH07t2bihUr3nWN7bzMk46OjiYhIYGBAwda9MmePXsALIZm38+bb76JwWAgNDQUFxcXOnbsiJ2dHSNHjiQ2NjbXIEGjRo3YuXMnKSkpyogAo9FIREQEPj4++bpJzKsHff9iYmLo27cv48ePp0OHDnh6euLp6cmxY8e4ePEikNVv2fktoGD69/ZVNe4lICCAtWvXkpaWRrt27XKMmHmUNBoNbdu2ZevWrWi12rsmVCxfvny+gly3a9WqFYsXLyYiIsIi/8Avv/yCl5cXzs7OODs73/X8AQEBfPbZZ8yZM4dbt27lWGlr/PjxbNu2jfHjx8sUAyGEyAMJEgghRD5kGk0kJuvIsEKCQp3eyOL1Jzh+9r+nsyVctQS+XIdnS+VtfnJhsLezwc3FHpt7BAgAomMSiYlNwcXJzuKpnVvZqmA2Eb1/MRnxbfmhXBqRh/eQnJxM+/bt8ff3Z+jQobzzzjt07dpVyeofGRnJiBEj8tVWV1dXjh49yqFDh2jYsCEjRoygd+/eDBs2jD59+mBvb094eDjbt2/nq6++yvN5a9SoQYcOHZg+fToZGRl4e3uzZ88edu7cSUhICDY2Nrz11ltMnDgRGxsb2rRpQ1JSEmFhYVy/fp1atWrdt90nT57k4MGD+Pr65rpP5cqVcXZ2Zt68edja2mJra8vWrVuV0Qn5SWjZpEkTPvroI6ZOnUrLli1JSkoiJCQEDw8PatSokesxgYGB7Nmzh4EDBzJ06FDs7OxYuXIlly5dYtGiRXmuOz8e9P1zd3enbNmyfPrpp6SkpFCxYkVOnDjB7t27GTZsGHFxcZw4ccJiub2C7N/7CQgIIDQ0lJ9++omwsLACO+/dHDt2jGXLluUob9GiBVWqVMm1fcOGDUOtVvPBBx88UJ1r166lWLFiFmVqtZqBAwfSuHFj2rRpw+eff056ejrVqlXjxx9/5OjRo3nqj+yVDFatWkW9evWoVKmSsm379u1s3LgRf39/6tata7HUKvyXKFQIIcR/JEgghBB5pDcYSUjWYbBCgsLEFB2hayL551qyUlaprAsjetahmHPReaqotbOhuKsWG/X9h+ompWYlc7SzsbM8h1MxGnUaw+mDP3H2wCo+OrAcr+rVCQ4OpkmTJgAsXryYkJAQRo8ejZ2dHbVq1WLp0qX5ziY/fPhwwsLCGDJkCJs2baJGjRp88803zJ49m7Fjx2I2m6levTqhoaE58gPcz/Tp0wkJCeHrr78mPj6eKlWq8NVXXynZ83v16oWTkxOLFi0iPDwcR0dH6tevz4wZM6hQocI9z/3666/z2WefMXjwYJYuXZrrPi4uLoSFhTFt2jTGjBmDk5MT3t7erFy5kiFDhnD48GH8/f3z9Fp69+6NwWDgu+++Y9WqVWi1Wpo2bUpQUJAy/P5O1apVY9WqVcyaNYvx48ejUqnw9fVl+fLlNGzYME/15tfDvH8hISHMmjWLOXPmEB8fT7ly5QgMDGTo0KFs3LiRatWqWYyYKMj+vZ+qVatSvXp1bty4QbNmzQrknPeyb98+9u3bl6Pczc0t1yBBs2bNcHV1pVy5crluz4vcbvZtbGwYOHAgAHPmzCEkJISlS5cSFxdH1apVCQkJyXMfd+vWje3bt9OlSxeL8p9//hnIGpWQW46EHTt2PPAICCGEeFKpzAWZ3eYJFRUVBXDXYaGFIS0tjVOnTuHt7X3XBFDiwUn/WteT0L/pOgOJKdZJUHjlRgohqyOJS8pQynyrlmJw19rYa+6fHTw9PZ1z585RpUqVh56zfy9ajQ3FXfIWIAA4eymBz5YdRGtvg30uKx/o9EYy9EYmDPKjaoXiBdzagvEkXLtFmfSvdUn/Wpf0r/UU9b4tCvcGQliTjCQQQoj7sFaCQoBTF+KY/8NxMnT/TV9o06A8vZ6vbrHU36Om1dji5mKfrzZ5uhfD/RlnLlxNQuOqtphyYDabSU434FHOFU/3Yvc4ixBCCCGEKExFY5FtIYQogsxmMwkpOpKtFCD49fgVgr8/pgQIVECv56vxSjuvIhMgUJGVOLGEa/4CBJC1IkNP/2o42NtyK0mHTm/EZDKj0xu5laTD0d6Wnv7VisxrFUIIIYQQMpJACCFyZTSaSEjJurEt6PiA2Wxmw95oNv16QSmzs1UzuGtt6lYvXcC1PTgV4KC1pbiz/QMvF1anWmlG9qzDml/OEBObQkq6AVsbNR7lXOnpX4061YrO6xVCCCGEEBIkEEKIHAyZWQkK9ZkFn6DQkGlixeZTHPzzmlLm4mjHyF518SjnWuD1PSiVChy1dhRz0jz0euJ1qpXGp0opomMSSUrV4+qkwdO9mIwgEEIIIYQogiRIIIQQt8nQZ5KQrLNKgsLUdAPz1h3nzKUEpaxsSUcCe9WlVHHrJRzML5UKnLR2BbqqglqtKrLJCYUQQgghxH8kSCCEEP9KSdeTnGrAZIVFX24kpBPy/TGux6UpZV4V3Rjawwcnbe5LzD0KKhU4azW4Osu64UIIIYQQTyMJEgghnnpms5mkVD2pGQassSjs+SuJhK2JJDnNoJQ1qV2W/h29sbUpOvljVSpwcdDg4iQBAiGEEEKIp5UECYQQTzWjyUxCsg6dPrPAExQC/HE6liUb/sRwW36Dzs9VplPzyg89178gqVXg4qjB2VECBEIIIYQQTzMJEgghnlqGTBMJyRlWSVBoNpvZcegSa385owQfbNQqBnT0polPuQKv72GoVeDqpMHJQQIEQgghhBBPOwkSCCGeSjp9JvEpOozGgh8/YDKZ+X773+w6elkpc7C3ZVh3H2p4lCjw+h6GWqXC1VlTpPIiCCGEEEKIR0eCBEKIp05qhoGkFL1VEhRm6DNZsv5Pjp+9qZSVcNUS+HIdni3lXOD1PQy1SkUxZw2OEiAQQgghhBD/kiCBEOKpYTabSU41kJKht0qCwsQUHaGrI/nnerJSVqmsCyN61inQ5QQLglqtorizBgd7CRAIIYQQQoj/SJBACPFUMJnMJKToyNBnWiVAcOVGCiGrI4lLylDK6lQrxetdamOvsSn4Ch+CjTprBIEECIQQQgghxJ0kSCCEeOJlGk1ZKxgYjFY5/6kLccz/4TgZuv/O79+wAj39q6FWF50VDCArQODmYo+9Rj7+hRBCCCFETvItUQjxRNMZjCQm6zAYC34FA4Bfj19h5Za/MJmyhieogJ7PV+P5RhWtUt/DkACBEEIIIYS4H/mmKIR4YqVlGEhK1WM0Ffz8ArPZzIa90Wz69YJSZmerZnDX2tStXrrA63tYNjYq3JwlQCCEEEIIIe5Nvi0KIZ5Iyal6UtL1WCE+gCHTxMrNp/j9z2tKmauThhE96+BRzrXgK3xItjYqirtosbcrWrkRhBBCCCFE0SNBAiHEE8VkMpOYqiNdZ50EhakZBuatPc6ZSwlKWdmSjgT2qkup4g4FX+FDsrVR4+Zij0YCBEIIIYQQIg8kSCCEeGIY/01QmGGlBIU3E9IJWX2Ma7fSlDKvim4M6+GDo7borRRgZ6PGzdUeO1sJEAghhBBCiLyRIIEQ4omgNxhJsGKCwvNXEglbE0lymkEpa1K7LP07emNro7ZKnQ9DY6umuIsWO9ui1zYhhBBCCFF0SZBACPHYS9cZSEyxToJCgD9Ox7Jkw58YMv8LQHR5rjIBzSujUhWtJQ4hK0Dg5qotksELIYQQQghRtEmQQAjxWEtJ05OcZsBkhQQEZrOZHYcusfaXM2Sf3UatYkCAN01qlyvw+gqCBAiEEEIIIcTDkCCBEOKxZDabSUzVk5ZhsEqCQpPJzPfb/2bX0ctKmaO9LcN6+OJVya3gK3xIKpUKrcaWEq5abCRAIIQQQgghHpAECYQQjx2j0URCig6d3og1Jhhk6DNZ/NOfRJ27qZSVLKYlsFddypVyskKND8/JwZ7izhoJEAghhBBCiIciQQIhxGPFkJmVoFCfaZ0EhYkpOkJXR/LP9WSlrFI5V0b29MXVyd4qdT4sB3tbnLVq1Oqilx9BCCGEEEI8XiRIIIR4bGToM0lI0WE0WidBYcyNFEJXRxKXlKGU1alWmsFda6GxK5rLCGo1Nmht1VzLNNx/ZyGEEEIIIe5DggRCiMdCSrqe5FTrJCgEOHUhjvk/HCdDZ1TK/BtWoKd/tSL7hF6rscXNxZ6MjPRH3RQhhBBCCPGEkCCBEKJIM5vNJKXqSbVSgkKAX49fYeWWvzD9u4SiCujVtjr+DStYp8KHpALs/w0QFNUAhhBCCCGEeDxJkEAIUWQZTWYSknXo9JlWSVBoNpvZsDeaTb9eUMrsbNUM7lqbutVLW6HGh6cCtPZZAQKVSgIEQgghhBCiYEmQQAhRJBkyTSQkZ1gtQaEh08SKzac4+Oc1pczVScOInnXwKOdqlToflgpw0NpS3FkCBEIIIYQQwjokSCCEKHJ0/yYozLRSgsLUdAPz1h3nzKUEpaxsSUcCe9WlVHEHq9T5sCRAIIQQQgghCoMECYQQRUpqhoGkVL2SH6Cg3UhIJ+T7Y1yPS1PKvCq6MayHD45aO6vU+bBUKnDU2lHMSSMBAiGEEEIIYVUSJBBCFBlJKXpSM/RYKT7A+SuJhK2JJDntv+UCm9QuS/+O3tjaqK1T6UNSqcBJa0cxZ/tH3RQhhBBCCPEUkCCBEOKRM5nMJKbqSNdlWm0Fgz9Ox7Jkw58Ybstx0OW5ygQ0r1xkn85LgEAIIYQQQhQ2CRIIIR6pTKMpawUDg9Eq5zebzew4dIm1v5xRVkiwUasY0NGbJj7lrFJnQZAAgRBCCCGEeBQkSCCEeGR0BiOJyToMRuusYGA0mfh++xl2H72slDnY2zK8hw9elUpYpc6CoFKBs1aDq7PmUTdFCCGEEEI8ZSRIIIR4JNJ1BhJT9BitlIAgQ5/J4p/+JOrcTaWsZDEtgb3qUq6Uk1XqLAgqFbg4aHBxkgCBEEIIIYQofBIkEEIUuuRUPSnp1ktQmJiiI3R1JP9cT1bKKpVzZWRPX1ydiu7wfbUKXBw1ODtKgEAIIYQQQjwaEiQQQhQasxnikzOsmqAw5kYKoasjiUvKUMrqVCvN4K610NjZWKfSAqBWgYuTBmcHCRAIIYQQQohHR4IEQohCYWNrR0KKDtR2Vqvj1IU45v9wnAzdf0kQ/RtWoKd/NdTqormCAYBapcLVWYOT1np9I4QQQgghRF5IkEAIYXWZRhPJ6UacdZk4OFjnRvjX41dYueUvTP/OYVCp4OXnq9OmYQWr1FdQ1CoVxZw1OEqAQAghhBBCFAESJBBCWFW6zkBcso60dJ1Vzm82m9mwN5pNv15QyjR2agZ3rU2daqWtUmdBUatVFHfW4GAvAQIhhBBCCFE0SJBACGE1Kel6klMNZGZaZ4lDQ6aJFZtPcfDPa0qZq5OGkT3rUKmcq1XqLCg2ahXFXezRauRjWAghhBBCFB3y7VQIUeDMZjNJqXpSMwxWS1CYmm5g3rrjnLmUoJSVK+VEYK86lCzmYJ1KC4iNWoWbiz32EiAQQgghhBBFjHxDFUIUKKPJTEJyBjq9ESvFB7iRkE7I98e4HpemlHlVcmNYd58iP7dfAgRCCCGEEKIok2+pQogCY8g0kpCsQ2+l6QUA0TGJzF0bSXKaQSlrUrsc/TvWwNZGbbV6C4KNjQo3Fy32RXgpRiGEEEII8XSTIIEQokDo9JnEp+gwGq01fgCO/hXL0o1/YrgtCNHlucoENK+MSlV0lzgEsLVR4+Zij0YCBEIIIYQQogiTIIEQ4qGlputJSjVgslICArPZzI5Dl1j7yxllCoONWsWAAG+a1C5nlToLkp2NGjdXe+xsJUAghBBCCCGKNgkSCCEeWGEkKDSaTHy//W92H41RyhzsbRnewwevSiWsU2kB0tiqKe6ixc62aE+FEEIIIYQQAiRIIIR4QEaTmcQUHRm6TKslKMzQZ7L4pxNEnbullJUspiWwV13KlXKyUq0FR2Orxs1VW+RzJQghhBBCCJFNggRCiHwzZJpITNGhMxitVkdCso7QNZFcup6slFUq58rInr64Otlbrd6CIgECIYQQQgjxOJIggRAiX3T6TBJSdGRaMUFhTGwKIWuOEZ+kU8rqVCvN4K61HovEf3Y2WVMMJEAghBBCCCEeNxIkEELkWVqGgaRUPUaT9QIEJ8/fYsGPUWTo/hul4N+wAj39q6FWF+0VDODfVQxc7SUHgRBCCCGEeCxJkEAIkSdJKXpSM/RYMT7A/sgrfLP1L0z/VqJSwcvPV6dNwwrWq7QA2diocHORVQyEEEIIIcTjS4IEQoh7MpnMJKbqSNdlWm0FA7PZzPq90Wz+9YJSprFTM7hrbepUK22dSgtYVoBA+1hMhxBCCCGEEOJuJEgghLgrQ6aRhGQd+kyTFeswsXzTSQ6dvK6UuTppGNmzDpXKuVqt3oJko1bh5myPvQQIhBBCCCHEY06CBEKIXKXrDCSmWDf/QGq6gblrj3P2coJSVq6UE4G96lCymIPV6i1INuqsKQb2Gvk4FUIIIYQQjz/5ViuEsGA2m0lJM5CSbt38AzcS0gn5/hjX49KUMq9Kbgzr7oOj1s56FRcgG7WKYs4aCRAIIYQQQognhnyzFUIojCYzCck6dPpMrBgfIDomkblrI0lOMyhlTX3K0a9Djcdm2UD1vwECB/vHI6AhhBBCCCFEXkiQQAgBgN5gJDHFuvkHAP44HcuSDX9iuK2eLi08CWjmgUpV9Jc4BFCrVBSXAIEQQgghhHgCSZBACEFahoGkVOvmHzCbzew6eoX1ey8ooxRs1CoGBnjTuHY5q9Vb0NQqFcVdJEAghBBCCCGeTBIkEOIpl5yqt3r+AaPJzJ4TyURdjFXKHO1tGdbDF69KbtaruIBJgEAIIYQQQjzpJEggxFPKZDKTmKojXZeJ2YoBggx9Jks2/MXJi+lKWcliWgJ71aVcKSfrVVzAJEAghBBCCCGeBhIkEOIplGk0ZSUoNBitWk9Cso7QNZFcup6slHmUc2VET19cneytWndBkgCBEEIIIYR4WkiQQIinjM5gJDFZh8Fo3QSFMbEphKw5RnySTinzqVKCIS/6orGzsWrdBUmtklUMhBBCCCHE00OCBEI8RQojQSHAyfO3WPBjFBm6/0Yq1KnsyMAAr8crQKCWVQyEEEIIIcTTRYIEQjwlCiNBIcD+yCt8s/UvTP9WpFJB91aVedY5DbX68VjiELJWXijuYo9WIx+TQgghhBDi6SHffoV4whVWgkKz2cz6vdFs/vWCUqaxU/NG19pUK+/MuXPnrFd5AbNRq3BzscdeAgRCCCGEEOIpI9+AhXiCZRpNJCbryLBygkJDponlm05y6OR1pczVScPIXnWoVNaV9PT0exxdtNjYqHBz0WL/GE2LEEIIIYQQoqBIkECIJ1RhJShMSTcwb+1xzl5OUMqeLeXEyF51KFnMwap1FzTbfwMEj1PeBCGEEEIIIQqSBAmEeAKl6wwkplg/QeGN+DRCVkdyPS5NKfOq5Maw7j44ah+vZH92NmrcXO2xs5UAgRBCCCGEeHpJkECIJ0xhJSiMjkkkbE0kKekGpaypTzn6daiBrY3aupUXMI2tmuIuWuxsH692CyGEEEIIUdAkSCDEE6KwEhQCHP0rlqUb/8SQ+d9Uhi4tPAlo5oFK9fisYABZAQI3V+1jF9gQQgghhBDCGiRIIMQTwGg0kVAICQrNZjPbD/7Dup1nyY5D2KhVDAjwpkntclat2xokQCCEEEIIIYQlCRII8ZjTG4wkpujQZ1o3QaHRZCJ829/s+SNGKXO0t2VYD1+8KrlZtW5rsLezwc3FHhsJEAghhBBCCKGQIIEQj7HCSlCYoc9k0U8nOHHullJWqpiWkb3qUq6Uk1XrtgYJEAghhBBCCJE7CRII8ZhKSdOTnGb9BIUJyTpC10Ry6XqyUuZRzpURPevg6qSxbuVWoLWzobirFhv145U7QQghhBBCiMIgQQIhHjNms5nEVD1pGQarJyiMiU0hZPUx4pN1Slnd6qV5vUstNHaP31KBWo0NxV0kQCCEEEIIIcTdSJBAiMeI0WgiIUWHTm/EyvEBTp6/xYIfosjQ/5cMsW2jivRoUxX1Y3iTrdXY4OaifSzbLoQQQgghRGGRIIEQjwlDppGEZOsnKATYH3mFb7b+henfuQwqFbzctjptGlSwet3WIAECIYQQQggh8kaCBEI8BjL0mSQk66yeoNBkNrN+TzRbDlxQyjR2at7o5oNv1VJWrdtaJEAghBBCCCFE3kmQQIgiLiVdT3KqAZOVExAYMk0s33SSQyevK2WuThpG9qpDpbKuVq3bWrR2WTkIJEAghBBCCCFE3kiQQIgiymw2k5SqJ7UQEhSmpBuYt/Y4Zy8nKGXPlnIisFddShTTWrdyK5FVDIQQQgghhMg/CRIIUQQZTWYSkjMKJUHhjfg0gr8/Rmx8ulJWo5Ibw7r74qB9PD8iJEAghBBCCCHEg3k87wCEeIIVZoLC6JhEwtZEkpJuUMqa+ZSjb4ca2NqorV6/NUiAQAghhBBCiAcnQQIhipAMfSYJKTqMRmuPH4Ajf11n6YaTZBr/C0Z0beFJx2YeqFSP5w22vQQIhBBCCCGEeCgSJBCiiCisBIVms5ltB/9h3c6zSpmtjYoBATVpXKusVeu2Jns7G9xc7CVAIIQQQgghxEOQIIEQj1hhJig0mkyEb/ubPX/EKGWOWluG9/ClekU361ZuRRpbdVaA4DGdIiGEEEIIIURRIUECIR6hrASFOnT6TKsnKMzQZ7LopxOcOHdLKStVTEvgy3UpW9LJyrVbj8ZWjZurVgIEQgghhBBCFAAJEgjxiBgyTSQkZxRKgsKEZB2hayK5dD1ZKav8rCv/e6kOrk4aq9dvLdkBgsc1yaIQQgghhBBFjQQJhHgEdPpM4gspQWFMbAohq48Rn6xTyupWL83rXWqhsbOxev3WYmejpriLBAiEEEIIIYQoSBIkEKKQpabrSSqEBIUAJ8/fYsEPUWTojUpZ20YV6dGmKurHOMGfrY0aN1d77GwlQCCEEEIIIURBkiCBEIWkMBMUAuyPvMI3W/5SghEqFbzStjqtG1SwfuVWZGujws3FHjvbx3cUhBBCCCGEEEWVBAmEKARGk5nEFB0ZOusnKDSZzazfE82WAxeUMo2dmje6+eBbtZSVa7eurACB9rGeJiGEEEIIIURRJkECIazMkGkiMUWHzmC8/84FUNfyTSc5dPK6UubqpGFkrzpUKutq9fqtSQIEQgghhBBCWJ8ECYSwIp0+k4QUHZmFkKAwJd3AvLXHOXs5QSl7tpQTgb3qUqKY1ur1W5OtjRo3F3sJEAghhBBCCGFlEiQQwkpSMwwkp+oxmqwfILgRn0bI6kiux6UpZTUquTGsuy8O2sf719zWRk0JV8lBIIQQQgghRGF4vO8ehCiiklL0pGboKYT4ANExiYStiSQl3aCUNfUpR78ONR775QHtlFUMJEAghBBCCCFEYZAggRAFyGQyk5CiI0OfWSgrGBz56zrLNp7EkGlSyrq08CSgmQcq1eO7xCFIgEAIIYQQQohHQYIEQhSQTKOJhOTCSVBoNpvZdvAf1u08q5TZqFUMDPCmce1yVq/f2iRAIIQQQgghxKMhQQIhCoBOn0liih6D0XT/nR+S0WQifNvf7PkjRilz1NoyvIcv1Su6Wb1+a5MAgRBCCCGEEI+OBAmEeEhpGQaSCilBYYY+k0U/neDEuVtKWaliWgJfrkvZkk5Wr9/aJEAghBBCCCHEoyVBAiEeQnKqnpT0wklQmJCsI3RNJJeuJytllZ915X8v1cHVSWP9BliZBAiEEEIIIYR49CRIIMQDMJnMJKbqSNcVToLCy7HJhK6OJD5Zp5TVq16a17rUQmP3+N9US4BACCGEEEKIokGCBELkU2EmKAQ4ef4WC36IIkP/X31tG1Wkh39V1I/5CgYgAQIhhBBCCCGKEgkSCJEPhZmgEGBfZAyrtpzG9O9wBZUKXmlbndYNKhRK/dYmAQIhhBBCCCGKFgkSCJFHhZmg0GQ2s35PNFsOXFDKNHZq3ujmg2/VUlavvzDYSoBACCGEEEKIIkeCBELkQVKKntSMwklQaMg08nXEKQ6fuq6UFXPWMLJnHSqWdbV+AwqBjCAQQgghhBCiaJIggRD3YDKZSUjRkaEvnASFKekG5q2N5OzlRKXs2dJOBPasS4liWus3oBBIgEAIIYQQQoiiS4IEQtyFIdNEYkrhJSiMjU8jdHUk1+PSlLIaHiUY9qIPDton41dVY2cjAQIhhBBCCCGKsCfjzkOIAqbTZ5KQoiezkBIURsckErYmkpR0g1LWzKcc/TrUwMZGXShtsDZHrT1uzhIgEEIIIYQQoiiTIIEQd0jNMJBcSAkKAY78dZ2lG05aBCS6tvSkY1MPVE/AEoeQNYLAxdEWG5sn4/UIIYQQQgjxpJIggRD/srW1JTndgNFsKpT8A2azmW0H/2HdzrP/tcFGxYCAmjSuVdb6DSgkdjZqXLX23LxquP/OQgghhBBCiEdKggRCAGYzpOnNpKTp0WodrF6f0WQifNvf7PkjRilz1NoyvIcv1Su6Wb3+wpKVpFCLQZ+BuTAiL0IIIYQQQoiHIkEC8dQzZJqIT9aRnKqjdCHcx2boMlm0/gQnzt1SykoV0xL4cl3KlnSyfgMKSXaAwM5WjUH/qFsjhBBCCCGEyAsJEoinmk6fSbyyxKH1IwTxyRmErY7kUmyKUlb5WVf+91IdXJ00Vq+/sGhs1RR3yQoQCCGEEEIIIR4fEiQQT63UdD1JqQZMhTQM/nJsMqGrI4lP1ill9aqX5rUutdDYPTkZ/yVAIIQQQgghxONLggTiqWM2m0lK1ZOaYSiUBIUAf0bfYuGPUWTojUpZ20YV6dGmKmr1k5PxXwIEQgghhBBCPN4kSCCeKkaTmYRkHTp9JoWVRm/vsRi+3XpaGbGgUsErbavTukGFQmpB4ZAAgRBCCCGEEI8/CRKIp4Yh00hCsg59pqlQ6jOZzazfc44tBy4qZRo7NW9088G3aqlCaUNh0dhmJSm0tZEAgRBCCCGEEI8zCRKIp0KGPpOEFB1GY+GMHzBkGvk64hSHT11Xyoo5axjZsw4Vy7oWShsKiwQIhBBCCCGEeHJIkEA88VLS9SQXYoLClHQD89ZGcvZyolL2bGknAnvWpUQxbaG0obBIgEAIIYQQQogniwQJxBPrUSQojI1PI+T7Y8TGpytl3h4lGPqiDw7aJ+vXTQIEQgghhBBCPHmerLsWIf5lNJpISNGh0xsLLUHhucsJhK09Tmq6QSlr5luOfi/UwOYJu5HW2Kop4ap94l6XEEIIIYQQTzsJEognTmEnKAQ48td1lm44Sabxvzq7taxCh6aVUKmenCUOQQIEQgghhBBCPMkkSCCeKOk6A4kpeoymwhk/YDab+fn3f/hh11mlzNZGxcCAmvjVKlsobShM9nY2uLnYS4BACCGEEEKIJ5QECcQTIyVNT3Ja4SUoNJpMfPfz3+w9FqOUOWpt+V8PX6pVdCuUNhQmCRAIIYQQQgjx5JMggXjsmc1mElP1pBVigsIMXSYLfzrBn9G3lLJSxR0I7FWHsiWdCqcRhcjezgY3Vy026idr6oQQQgghhBDCkgQJxGPNaDSRkKxDZyi8BIXxyRmEro7kcmyKUlb5WVdG9KyDi6OmkFpReLR2NhSXAIEQQgghhBBPBQkSiMeW3mAkMaVwExRevp5MyJpIEpJ1Slk9r9K81rkWGjubQmtHYdFqbCjuIgECIYQQQgghnhYSJBCPpcJOUAjwZ/QtFvwYhU5vVMra+VWke5uqqJ+wFQwgK0Dg5qJFLQECIYQQQgghnhoSJBCPneRUPSnpegoxPsDeYzF8u/W0khRRpYJX2nnRun75wmtEIVEB9hpb3FzsJUAghBBCCCHEU0aCBOKxYTKZSUzVka7LLLQEhSazmZ92n2PrbxeVMns7G97oVhufqqUKpxGFSAVo7bMCBKoncHSEEEIIIYQQ4t4kSCAeC5lGE4nJOjIMxvvvXEAMmUa+jjjJ4VOxSlkxZw0je9ahYlnXQmtHYVEBDlpbijtLgEAIIYQQQoinlQQJRJGnMxhJTNZhMBZegsKUND1z1x3n3OVEpezZ0k4E9qpLCVdtobWjsEiAQAghhBBCCAGgftQNADCZTHz11Ve0aNGCunXrMmTIEC5dunTX/devX4+Xl1eO/y5fvqzss3nzZgICAvD19eXFF1/kwIEDhfFSRAFLyzAQn5RRqAGC2Pg0pq04bBEgqOFRgqB+DZ/MAIEKHB3sJEAghBBCCCGEKBpBgrCwMFatWsUnn3zCd999h8lk4o033kCv1+e6/+nTp/Hz82Pfvn0W/5UrVw6A3377jaCgIHr37s0PP/xA06ZNGTp0KOfOnSvMlyUeUnKqnsQUXaGuYHD+ShJTlx8mNj5dKWvu+yyjetXBQfvkDbxRqcBJKwECIYQQQgghRJZHHiTQ6/UsWbKE0aNH8//27js+ijr/4/h7s8lmU0no0muCCSWhNymCARWkKAqIFA/Uo3lHsaHyw3ZWEBMB6XonggjY0IOzIBY60qRI6BFJaElIT3bn90dgdQlgAtlsyuv5eOQh+532nQ+zI/POzHe6dOmiRo0aacaMGTp16pTWrl17xWV+/fVXhYaGqlKlSk4/ZnPue+rnzZun7t27a+jQoapfv74ef/xxhYeH69133y3KXcN1stsNnb+QoQtF/AaDgyczNGvlL0pNz3a09elUX0NubySz2e1flUJ3KSAo5+/t7q4AAAAAKCbcfuWzf/9+paamql27do62wMBAhYWFacuWLVdc5sCBA6pfv/4Vp9ntdm3fvt1pfZLUpk2bq64PxUeOza5zyRlKyyi6NxgYhqGvt/6mNduTlGPL3ain2aS/3RWu29vXKZW/YTeZpAAfCwEBAAAAACduv3/61KlTkuR4VOCSypUrO6b9WVJSkuLj47V161YtWbJE58+fV9OmTTV58mTVrVtXycnJSktLU9WqVfO1vvwyDENpaWnXvfyNSk9Pd/pvaZSdY1dSapayivANBja7oRXfHtaGPfGONl+rpx7s1Uj1qweWynp7eJgU4GuR2ZSjtLScItlmWTh+3Yn6ug61dS3q61rU17Wor+sU99oahlEqf4kEXOL2kODSl99isTi1e3t7KykpKc/8Bw8elJT75fzXv/6ljIwMzZ49W4MHD9Znn32mnJycq64vMzPzuvuZnZ2tffv2XffyheXo0aPu7kKhM5vNshkeupCWrcys7L9eoJBk5dj1321JOn76j7EvAn3N6t26nJRxWocOnS6yvhQVL09PBfpZlOxhl81WdGHMJaXx+C1OqK/rUFvXor6uRX1di/q6TnGu7eXXGkBp4vaQwGrNHS0+KyvL8WdJyszMlI+PT575W7ZsqQ0bNig4ONiR4MXExKhLly5auXKlBgwY4Fjfn11tffnl5eWlBg0aXPfyNyo9PV1Hjx5VnTp1bmg/iqOU9BylpGepXBEOQJCYkql5n+zTyTN/HCdVgrw06q6bVSHYv8j6UZQ8PEwq5+stH6u5yLddmo/f4oD6ug61dS3q61rU17Wor+sU99rGxsa6uwuAS7k9JLj0mEFCQoJq1arlaE9ISFBoaOgVlylfvrzTZx8fH9WoUUPx8fEKCgqSr6+vEhISnOZJSEhQlSpVrrufJpNJvr6+1718YfHx8SkW/SgMNruhpJRM5Rge8vYuulcLxsVfUMxHe5R44Y87S5o1qKB2Dc2qEOxfLP9ndKM8TCYF+lvkZ/Vyaz9K0/FbHFFf16G2rkV9XYv6uhb1dZ3iWlseNUBp5/aBCxs1aiR/f39t2rTJ0ZacnKy9e/eqVatWeeZftmyZ2rRp4zQ+QEpKio4ePaoGDRrIZDKpefPm2rx5s9NymzZtUsuWLV23IyiQ7By7zidnKD2z6AYolKRfDp/Va+9vcwoIbmtTS0PvCJGnuXSe8D1MJpUrBgEBAAAAgOLP7SGBxWLRkCFD9Prrr+vrr7/W/v379c9//lNVq1ZVVFSUbDabTp8+rYyMDElSp06dZLfb9dhjj+ngwYPavXu3xo0bp/Lly6t///6SpBEjRmj16tVatGiRDh06pFdffVX79u3TsGHD3LmruCgzK0fnktOVWYQDFErS9zt+09vLdyozK3e7JpM0KCpUd3dtKI9Smgh7mEwKCrDIl4AAAAAAQD64PSSQpPHjx+uee+7R008/rUGDBslsNmvBggXy8vLS77//ro4dO+qLL76QlPt4wuLFi5WWlqZBgwZp+PDhCggI0HvvvSdv79zXuXXs2FEvvfSSPvjgA/Xr108bN27UnDlzrvraRBSd1PQsnUvOdLxqsCjYDUOr1sXq/f/ul/3ibQveXmaNvqeZOjevUWT9KGpmj9yAwMebgAAAAABA/rh9TAIpd3T7yZMna/LkyXmm1ahRQwcOHHBqCw8P18KFC6+5zr59+6pv376F2U3cAMMwlJyapdSM7CJ9vCA7x6bFn+/Vtv1/jFFRzt+iMfdEqFbVgKLrSBHLDQi8ZbUUi684AAAAgBKCKwi4nM1uKPFCpjKzclSE+YBS0rI0e+UuHYr741Wa1Sr5aeyACJUPLLqBEoua2cOk4ABveRMQAAAAACggriLgUtk5diVeyFBWjr1Itxt/Lk0xy3fo9Pl0R9vNdcrrob5N5GMtvYc9AQEAAACAG8GVBFwmMytH51MyZSvC8QckKTYuUbNX7FJqerajrUPTahrcI1Rmc7EYhsMlzB4mBQda5e1ldndXAAAAAJRQhARwiZT0LF1IzXYMFFhUtu6L1+LP9yrH9sedC3061VfPdrVL9TttCQgAAAAAFAZCAhQqdw1QaBiG1m46rlXrYh1tnmaTht0ZplZhVYuuI25AQAAAAACgsBASoNDkDlCYocwsW5EOUGiz27V07a/6fsdvjjZfq6f+fndTNawZXIQ9KXoEBAAAAAAKEyEBCkV2jk2JFzKLfIDCjMwczftkj345fNbRVjHIR2MHNFPVCn5F2peiRkAAAAAAoLAREuCGZWTlKNENAxSeT87Q2x/tVFxCiqOtbrVAjb6nmQJ8LUXal6JGQAAAAADAFQgJcENS0rJ0Ia3oByiMi7+gmI92KvFCpqMtMrSSRvQKl6WUXzgTEAAAAABwFUICXBfDMJSUmqW0Ih6gUJJ+OXxWcz/ercwsm6Pttja11K9LA3mU4jcYSBcDggBvAgIAAAAALkFIgAKz2exKTMks8gEKJen7Hb/pgzUHHHcumEzSwNtC1bl5jSLuSdFzBAQWvrYAAAAAXIOrDRRIVrZNSSlFP0Ch3TD0yXeHtGbjMUebt5dZI/s2VpP6FYu0L+5AQAAAAACgKHDFgXxLz8xWUkqWbPaivX8gO8emxZ/v1bb9CY62cv4WjbknQrWqBhRpX9yBgAAAAABAUeGqA/mSO0Bhloo4H1BKWpZmr9ylQ3FJjrbqlfw1ZkAzlQ+0Fm1n3MDsYVIQAQEAAACAIsKVB67JbjeUlJqp9MycIh+gMP5cmmKW79Dp8+mOtrC65TWqbxP5eJf+Q9fDw6Ry/hZZCQgAAAAAFBGuPnBVOTa7ki5kKiPb9tczF7LYuETNXrFLqenZjrYOzappcFSozGaPIu9PUfMwmRTkb5GPt5e7uwIAAACgDCEkwBVlZuUoMSVLObaiHaBQkrbui9fiz/c6bbtv5/rq0ba2TKX8FYfSxYAggIAAAAAAQNEjJEAeqelZSk7Llr2IByAwDENrNx3XqnWxjjZPs0nD7gxTq7CqRdoXdyEgAAAAAOBOhARwMAxDyalZSs3ILvLxB2x2u5auPaDvd5x0tPlZPfXI3U3VsGZw0XbGTQgIAAAAALgbIQEkSTabXYkpmcrMsqmI8wFlZOZo3id79Mvhs462SkE+GntvhKqU9y3i3rgHAQEAAACA4oCQAMrOsSnxQqaycop+/IHzyRl6+6OdiktIcbTVq15Of7+7qQJ8LUXeH3cgIAAAAABQXBASlHEZWTlKvJApWxGPPyBJJ+IvKGb5TiWlZDramodW1vBeYbJ4mYu8P+5AQAAAAACgOCEkKMNS0rN0ITVb9qIegEDSnkNnNO+TPcrM+uP1ire1qaV+XRrIowy8wUAiIAAAAABQ/BASlEGGYSgpNUtpbhigUJLW/xynpWt/dYQTJpM0KCpUnSJrFH1n3MTsYVI5fwICAAAAAMULIUEZ484BCu2GoY/XHdLaTcccbd5eZo3s21hN6lcs4t64j9nDpKAAb1ktfP0AAAAAFC9cpZQh7hygMDvHpkWf79X2/QmOtnL+Fo25J0K1qgYUeX/cxexhUnCAt7wJCAAAAAAUQ1yplBHuHKAwJS1Ls1bs0uHfkhxt1Sv5a8yAZiofaC3y/rgLAQEAAACA4o6rlTKgMAYoNAxDpusYUDD+XJpilu/Q6fPpjrawuuU1qm8T+Xi7/vC73n4XNgICAAAAACUBVyylWGENUPjvxfPlZbFo4OChBVou9kSiZq/cpdT0bEdbh2bVNDgqVGazx/V3KJ9Wf7ZKx48e0d/HTXD5tq7FbDYpOMAqby+zDhw4oMcff1yxsbGqVauWvvjiC7f2zVX27t2r+++/X5s2bdKzzz6rjRs36o033rihdcbFxalbt25ObZ6enqpQoYK6dOmif/zjHypfvrzT9G+++UbvvvuufvnlF2VmZqpq1arq0qWLHnnkEVWoUMEx3wMPPKDNmzdfddvNmjXThx9+eEP9BwAAAEoCQoJSqjAHKFw0f7aGjnioQMts3RevxZ/vVY7tj/EP+naurx5taxfZb/b/8+4CRUS2KJJtXY3nxYDA4mWWJL399ts6efKk3n777TwXtKXJd999p7Zt28pisRT6uv/+97+rS5cukqTMzEwdOXJE0dHRio2N1ZIlSxzzrVq1Sk8++aQGDhyo4cOHy8fHR7GxsZo7d66+/fZbrVixQuXKlXPMHxYWpqlTp15xm35+foW+HwAAAEBxREhQCrlzgELDMLR20zGtWnfI0eZpNmnYnWFqFVa1yPvjTl5mDwUFeDsCAkk6f/68QkJC1LlzZzf2zPXWr1+vPn36uGTdtWrVUkREhONzmzZt5OXlpaeeekoHDx5Uw4YNJeUGMnfeeaf+7//+zzFv27Zt1bJlS/Xp00fLly/XyJEjHdP8/f2d1gsAAACURa6/5xsus2fPHg0bNkwtWrRQZGSkhg8frs1btulscoaycuzatfNn/WPsKN3erb363N5VL7/wrBLPn3cs/98vPlX3zq2175fdGvvwcPW4tZ0G3n2nli15zzHPrR1zfxP/3qK5jj9L0pHDsXrqsUfVK6qTekV10jNPTlRc3HEtWbNfq9YdUtqZQ/r188dkTzos4/BSPTuuv+6+K0pzZ70lm83mWE92drYWzpul+wfcpZ63tteDD9yrNV9+5rSfP36/To/8bYh63NpOd98VpZg3X1N6erquZdA9vRR/6net+fJz3dqxhU79ftKxv6s/W6W774pSn9u76uiRw7LZbFq+9D/613NPqd+dt+qO7h009pER+nn7Fsf6Fi94R0Pu66ONP32vkcPuU4+ubTV0YD+t/e9qp+2u+HCJhg3ur563ttPdfXropRefV0pKiiQpNDRUmzdv1pYtWxQaGqqVK1dKko4eParx48erQ4cOioiI0AMPPKBt27Y51hkXF6fQ0FAtWrRIPXv2VLNmzbRixQpFR0erZ8+e+t///qdevXqpSZMm6tOnj37++Wft2LFDAwYMUNOmTdWrVy9t2LDBqZ+//vqrHn74YTVv3lzNmzfXmDFjdOLECcf0TZs2KTQ0VEuXLlXXrl3VvHlz/fjjjzp37pwmTpyoDh06OLb38ccfO607KSlJO3fuVKdOna75d3S5r776SoMHD1ZkZKQaN26snj176v3338/XspfuCPjzXSpnzpyRcYXnbBo1aqQnn3xSjRs3LlD/AAAAgLKAkKCESklJ0ciRIxUcHKzo6GjNmDFDKampeuThh5SUlKydO7Zr0qOPyOpt1bPPvawx4ydqx8/bNGH8w8rMzHCsx7DbNe3ZJ9S1W5T+9dpMNWkaoXdmzdSWTT9JkmLmLJYk3dGrj+PPJ44f07hHHtT58+f1+JT/06QnntHJ3+I06sHh+nbTr079PL37Q7Vt00ovvvqmut3WU0uXvKsvPvvYMf3FaVO0fOl/dEfvvnrp1TfVqnVbvfLi/+nr//1XkvT12i/1zJMTVat2HT3/0hsa9uBD+t+aL/TMExOueAF4yXMvva7yFSqoTbsOipmzWOUrVJQk2S8GApOfeEajx09Q7Tp1NW9OtD54f7Ha39JVz730hiY+9rQuJCdp2tOPKyPjjzDi3NkzemvGq+o/YJBeenWmqlarppdfeFbHjx3J7ev//qu5s9/S3ffcp3fmztOYMWP0ySef6Pnnn5ckLVu2TGFhYQoLC9OyZcvUpUsXxcbGqn///oqLi9PTTz+t119/XSaTScOGDcvzjHx0dLRGjRqlV199VR06dJAknTp1Si+//LIeeeQRzZw5U8nJyRo/frwmTJigAQMG6O2335ZhGPrnP/+pjIzcv/cjR45o4MCBOnv2rF555RW9+OKLOnHihAYNGqSzZ886bTMmJkaPP/64nn32WUVGRmry5Mk6dOiQpk2bpnnz5iksLEyPP/64Nm7c6Fjmhx9+UL169VStWrWr/v1cbt26dRozZozCw8M1a9YsRUdHq2bNmnruuee0c+dOp3ntdrtycnKUk5OjjIwM7d+/X7NmzVLbtm3VoEEDx3xdunTR6tWrNWbMGH3++eeKj493TBs+fLjatm3rtF7DMBzrvfznWscaAAAAUJrwuEEJFRsbq/Pnz2vo0KGKjIxUUmqWHp9SRZ9/skrpaWma/06MataqrRdffVNmc+7t7jeHN9GIBwboy88/Vd+775WUe2E0dMQo3dGrrySpcZMIff/dt9rw0w9q1aa9who3kSRVrFTF8ef3Fs2Vt9Wq19+cJT8/f51PztA3+8069uEzOn9onSqF9dJNlfwUJ6l3n356YPgoSVLzFq314/p12vDT9+rd924dORyr9eu+1pjxE3X3vYNz52nZWqdO/a4d27fq1u49NHdOtFq3aa+nnn3Bse81atTSpH/8XZs2/KC27W+5Yn0ahjSSl5dFQUHBjn5fcv/Qvzktd/bMaQ0b8ZAaN2uh+vXry8fHR17e3vq/KZN1ODbWsXxGRoYmPf6MmrdsnduPmrU06J5e2vjTD6pVu6527dium6pV14Mjhsnb4qn27drK19dXSUm5r36MiIiQv7+/48+S9Nxzz8lisei9995zTOvSpYt69eqlV199VR999JGjn7fffrvuvvtup31JT0/X1KlTHb+1j42N1RtvvKEXX3xR99xzjyQpLS1N48eP15EjR3TzzTcrJiZGPj4+Wrx4sWOb7dq1U/fu3TV//nw9/vjjjvUPHjxYPXv2dHzevHmzxowZo+7du0uSWrduraCgIKexB9avX1/guwhiY2PVr18/TZkyxdEWGRmpNm3aaNOmTWrWrJmjfcqUKU7zSVJQUJD+/e9/O7U9//zzstvtWrt2rb766itJuY8qdOvWTSNGjFCVKlWc5t+yZYvCw8Ov2L+ZM2c61QEAAAAorQgJShC7Yejwb8nKsqXIy7eyypcvr4cffkS3drtNzVu2UYvW7fTQ6PHKyEjXvl92695BD0iGIVtOjiSpWrXqql27jrZt3egICSQpLLyp488Wi0VBQUHKuMbt/Nu3bVFEZAtZva06evK8Zn20S0kpOfIpX1dpZw6qeaPKiqweoC2fSWGNmzotW7FyZcdv53fv2iFJuqXzrU7zTHvxNUnS8WNHdDohXoMfGOHYB0lqFtFcfn5+2rplk9q2v8VpmiR5mM3XHByxQcMQp89Tpr6o9PR07djxs37Zs1OnExK04cf1kqTs7Cynef8cOFSqnHuReWl/WrZspc8+WaH77r1H3bt3V+fOndW7d+9r9mXz5s3q2rWr42Jdyh2x/84779Tbb7+t1NRUR/vNN998xXU0b97c8eeKFXPvmPjzRXVQUJAkKTk5WZK0ceNGtW7dWlarVTkXa+fv76+WLVvqp59+clr35dts06aNoqOjtXfvXt1yyy3q3LmzU6hgGIa+//57TZ8+/ar7fCWXxgZITU3VkSNHdPz4ce3evVuSlJXl/HcwduxYx8CFOTk5+v333/Xee+9p4MCB+ve//+240A8ICNBbb72luLg4fffdd9q0aZM2bdqkRYsWadmyZVq4cKEiIyMd6w0PD9e0adOu2L9atWoVaH8AAACAkoqQoITYc/ic3v/2jJLSz8puN+Tj7anuA5/SiT1r9NX/1ujjVR/J29tbUT17adADI2S327X0/Xe19P1386zL4m11+my1On82eXjIMK4+6GFyUqK+/Xqtvv16bZ5pPn6BGtmnsXb9nPtMvfWybXmYPGTYjYvryf0Ne1DwlUf5vzR95hsva+YbL+eZfvbMaZ36/aQGD+jt1P7YU1PV8467rtp/Hx9fp88H9u/VjNde0q8H9snbalWduvVVpUruIIuX32Zutfr8sS8euU/r2O2GLJ4eGnB3H/n7emnJkiWOW+arV6+uSZMm6Y477rhiX5KSkhwX9n9WsWJFGYbhGM9Aknx9ffPMJ8kpYPhjH32uMGeuxMREffHFF1d8/eLlb1y4fJszZszQnDlz9OWXX2rNmjXy8PBQ+/bt9dxzz6l69eravXu3MjMz1aJFwd4qce7cOU2dOlVfffWVTCaTateurZYtW0rK+3dQvXp1NWnyR1gTGRmpzp07q0uXLoqOjtacOXOc5q9Ro4buv/9+3X///bLb7frqq6/0xBNP6Pnnn3eMCyHlvsHgz+sFAAAAyiJCghJg58HTmvfJXl1IzVZwoFWBft7ytph17oKHzHXu0ssjJ8pI+13/++9qffrxR6pYqZJMJpPuuXewbr0t7y3S3pdduBeUf0CAatZvogu+kY4LOJPJpKi2tdQqrJo88vmKQ3//AElSUuJ5x2/lpdw7CJKTkuR3cfrDox9VRPOWV+xHhYqVNHu+823mVW/K/7PwqakpenziWNWpW19PPvuS2nfoKD8/P23c8IPWr/s6X+vwNJtUPtAqs9lDvXr1Uq9evXThwgX98MMPmjdvniZPnqwWLVrkub1dyh1w78yZM3naT58+LUkKDg5WQkJCvvcnPwICAtS+fXuNGDEi7754XvuUEBAQoMmTJ2vy5Mk6fPiwvv76a82aNUvTpk3T3LlztX79erVv315eXl4F6tOkSZN0+PBhLV68WJGRkbJYLEpPT9eHH36Yr+X9/PxUr149HTt2TJK0Zs0aTZ06VR988IHq1q3rmM/Dw0NRUVHasmVLvtcNAAAAlCUMXFjM2e2GPvrmoNIzbQrw9ZC3l1kBfhYd3bdZ6/49WcmJ57R28wndHNZE/5j0pPz9A3Tu3Dk1DGmk48ePKrRRmOOnTt16Wrxgjnb+vLVAfbj0G3Mp95GHitVC9euvsbIE3CRrUE2Vq1xHE/52hw79/D/9sP7bfK+3SdMISdJPF2/tv2Tu7GjFzHxdtWrXUXBweZ36/aTTflSsWEnzZkcr9tcD8vLycpoW2ihM5coF5en31Rw/dlTJSUm6q98A3VStumOZzRt/uri/f/0aSau3p8xmD/3jH//QmDFjJOVeTN9+++0aPXq0cnJyrnqh36pVK3377bdOdwzYbDatXr1aTZo0cXrWv7C0bt1asbGxuvnmm9WkSRM1adJEjRs31uLFi/W///3vqsv99ttv6ty5s/7739xBJevVq6dRo0apffv2OnnypKTrG49AkrZt26aoqCi1adPGsc/r1+ceF3b7X/8dXLhwQUeOHFHt2rUlSQ0bNlRiYqLefTfvnTRS7hslQkJCrjgNAAAAKMu4k6CYO/xbkn5LSFGAr6dstmxHu3+lejLsdh35aZHSz3fTmvLJ2rfjR6WmpqhTl1vVvmMnPTn5Ub04bYq6Rd0uu82uD5f+W/v27tEDw0YVqA/+/gH6ZfdObd+2RVuOeyunYntlH9iu37YsUrXQTurRJUTLF76iH79fp6kvvJrv9dZvGKLOXbvrnVkzlZmRoQYNQ7Vp44/a8ON6TXvxNZnNZj340GjNeO0leZg91K5DJ6VcuKD/vDtfpxMSFBJ65Wf0/9zvg78e0M6ft6lR2JUHpKtZq7b8/Pz04ZL31LV7TyWdP6uNP32vL1d/IknKSM+44nKSZPXKHRDy0p0Tbdu21dSpU/XKK6+oU6dOSk5OVkxMjOrUqaNGjRpdcR1jx47V+vXrNXToUD300EPy8vLSf/7zH504cULz58//yxpej9GjR2vgwIF6+OGHNWjQIHl7e2vZsmX66quv9NZbb111uerVq6tq1ap64YUXlJKSolq1amnPnj367rvv9PDDD+vcuXPas2ePoqOj8yybmpqqL7/8Utu3b3e6y6BatWqKiopS06ZN9dlnnyk8PFxVq1bV9u3bNXfuXJlMpjyvuzx+/Lh27Njh+HzmzBnNnz/f8cYPKTfAeOihh/TOO+/o5MmTuuuuu1S1alWdPXtWn3zyiTZs2KBFixY5rTclJcVpvZdr0qSJYxBQAAAAoLQiJCjmklOzlGOzy8/qJZvtj3arXzlF9hyrQ9s+0+FNH+jNjTmqV7+B/u+FVxXZvJUk6dU3YvTuorma9vRj8vTyUkjozXp9xqw8o/3/lfuHPqj3Fs/X4xPHq1bnifIOvEk12/9dKYe/0m/blujtbVLdevX13L/eUIeOnQu07qeefUHvLnxHK5Z/oKTERNWqU0dTX3hVHTt1lSTd2buf/Hz9tHTJe/r801Xy8fFR4yYReurZF3RTterXXPe9gx7QrLfe0GMTx+r1GbOuOI+/f4Ce/9d0zY6ZoYVz35afn58ahjTSmzHz9MSk8dq962e175j3N+NWi1lBAc6PbQwcOFDZ2dlaunSplixZIqvVqnbt2mny5MlXvf2+YcOGWrJkiaZPn64nn3xSJpNJTZs21Xvvved4Jr+wNWrUSO+//75mzJihxx57TIZhKCQkRG+//ba6det2zWVjYmI0ffp0zZw5U+fPn9dNN92ksWPH6qGHHtLnn3+uhg0bXvGxiuTk5DxvH5By36oQFRWll19+Wc8//7zjdZF16tTRtGnT9Omnn2rrVuc7X2bPnq3Zs2dLyr1bJCAgQOHh4VqwYIFTzSZMmKCbb75Zy5cvdwQbgYGBatmypT766KM8wc3evXt13333XXXft2zZosDAwGvWBwAAACjpTAYvAP9Ll0ZZd8egZrEnEvXS4s3y9jLJZsuWr9WqisG+Opecocwsm7Ky7crMtumR/k1V+ybXXMDEn0tTzPIdOn3+j9/ohtUtr1F9m8jHu3TkTOnp6Tp06JDjFYjXYrV4KjjAWx4e+Rt7AbmvYdy3b59uvvnmqw7AiOtHfV2H2roW9XUt6uta1Nd1intt3XltABSF0nGFV4rVq15O1Sv768jJJPl4Oec5hmEoNTNb1Sv5q2aVAJdsP/ZEomav3KXU9D8edejQrJoGR4XKbC5bQ1qYJHkTEAAAAAAoxcrWVV4J5OFh0j23NpSPxawL6bl3DdjthrKy7UpMzZKPxVM929ZxyUXr1n3xenPpdqeAoG/n+hrSs1GZDAis3p4qH0hAAAAAAKD0KltXeiVUs4aVNKpPmKoEeSkjy64LaVnKyrapeiV/De7RSI3qlP/rlRSAYRj674ajmv/JHuXYcu9e8DSb9Le7wtWzXR2Z8vmKw9LCJMnHmnsHQVnbdwAAAABlC48blBCN65XXkK4V5VOuhrJsJmXl2FW9kn+h/1bbZrPrg7UH9MPOk442P6unHrm7qRrWDC7UbZUElwKCIH8CAgAAAAClHyFBCeJhMqle9UCZzF46fyFThT3kZHpmjuZ9vFt7j5xztFUK8tHYeyNUpXzxGzTG1UwmydfqpXJ+FgICAAAAAGUCIQEkSeeTMxSzfKd+O53iaKtXvZz+fndTBfha3Ngz9zCZJD+rl8r5e7u7KwAAAABQZAgJoBPxFxSzfKeSUjIdbc0bVdbwO8Nk8TK7sWfuQUAAAAAAoKwiJCjj9hw6o3mf7FFmls3RFtWmtvp2qS+PMniLvYfJJH+rRYH+Ze/uCQAAAAAgJCjD1v8cp6Vrf5X94uAGJpM0KCpUnSJruLln7uHpaVaAHwEBAAAAgLKLkKAMshuGPl53SGs3HXO0eXuZNbJvYzWpX9GNPXMfDw+TAn295WflKwEAAACg7OKKqIzJyrZp8eq92r4/wdFWzt9bY+5pplpVA9zYM/fxMEmBvhYlm+3u7goAAAAAuBUhQRlyIS1Ls1fs0uHfkhxt1Sv5a+yAZgoOtLqxZ+7jYZICfC3yUI5sNttfLwAAAAAApRghQRkRfy5NMR/u0OnEdEdbWN3yGtW3iXy8y+Zh4GGSAvws8vexKC0tx93dAQAAAAC3K5tXh2VM7IlEzV6xU6kZf1wId2hWTYOjQmU2e7ixZ+7jYZIC/Szy82GQQgAAAAC4hJCglNuy95TeXb1XOTbD0da3c331aFtbpjL4ikOJgAAAAAAAroaQoJQyDENrNh7Tx98dcrR5mk0a3itcLW+u4saeuZeHyaRAf4v8rF7u7goAAAAAFDuEBKWQzWbXkrUH9OPOk442P6un/n53MzWoGeS+jrmZh8mkcv4W+RIQAAAAAMAVERKUMumZOZr38W7tPXLO0VYpyEdj741QlfK+buyZexEQAAAAAMBfIyQoRc4nZyhm+U79djrF0Vavejn9/e6mCvAtu8/fe5hMCgqwyMebgAAAAAAAroWQoJQ4EX9BMct3Kikl09HWvFFljegVJi9Psxt75l4EBAAAAACQf4QEpcCeQ2c075M9ysyyOdp6tK2tPp3ry6OMvsFAIiAAAAAAgIIiJCjh1v8cp6Vrf5XdyH3FoYfJpIFRIeoUWcPNPXMvDw+TgvwJCAAAAACgIAgJSii7YWjlt7H636bjjjZvi1mj+jRW4/oV3dgz9zN75A5SSEAAAAAAAAVDSFACZWXbNP/jPdq2P8HRFhTgrbH3NFONKgFu7Jn7mT1MCgrwltXCoQ0AAAAABcWVVAmTnJql15bs1K/HEx1t1Sv5a+yAZgoOtLqvY8UAAQEAAAAA3BiupkqQjGy7npm7WafOpTvawuqW16i+TeTjXbb/Ks0eJgUHeMubgAAAAAAArhtXVCXIwd8ynAKCjs2qaVBUqMxmDzf2yv3MZpOC/QkIAAAAAOBGcVVVgtSsZFGQv0WpGTnqfUs93da6lkxl+BWH0sWAIMAqby+zu7sCAAAAACUeIUEJEuTnqVmTblGW3UOZ2XZdfOthmeVp9lBwgLcsBAQAAAAAUCjK9n3qJZDZ7CFfK6/28zJ7qHwgAQEAAAAAFCbuJECJY/H0UFCAVV6eZFwAAAAAUJi4ykKJQkAAAAAAAK7DnQQoMSyeHgoOtMqzjL/NAQAAAABchZAAJYK3l1nBAd5l/nWPAAAAAOBKXHGh2CMgAAAAAICiwZ0EKNasXmYFBVpl9jC5uysAAAAAUOoREqDYslrMCgogIAAAAACAokJIgGLJavFUcIC3PAgIAAAAAKDIEBKgWDFJsnp7KsifgAAAAAAAihohAYoNkyQfa25AYDIREAAAAABAUWO4eBQLBAQAAAAA4H7cSQC3M5kkX6uXyvlZCAgAAAAAwI0ICeBWJpPkZ/VSOX9vd3cFAAAAAMo8HjeA2xAQAAAAAEDxwp0EcAuTSfK3WhTob3F3VwAAAAAAFxESoMiZTFKAj0UBfgQEAAAAAFCc8LgBihQBAQAAAAAUX4QEKDIEBAAAAABQvBESoEgQEAAAAABA8UdIAJcjIAAAAACAkoGQAC5FQAAAAAAAJQchAVyGgAAAAAAAShZCArgEAQEAAAAAlDye7u4ASh+TSQr0tcjfl4AAAAAAAEoSQgIUKg+TFEBAAAAAAAAlEo8boNAQEAAAAABAycadBCgUHiYpwM8ifx8CAgAAAAAoqbiTADeMgAAAAAAASgdCAtwQAgIAAAAAKD143ADXzcMkBfpZ5EdAAAAAAAClAiEBrouHyaRAf4v8rF7u7goAAAAAoJAQEqDAPEwmlfO3yJeAAAAAAABKFUICFIiHyaSgAIt8vAkIAAAAAKC0YeBC5BsBAQAAAACUbtxJgHwxe+Q+YkBAAAAAAAClFyEB/pLZw6SgAG9ZLRwuAAAAAFCa8bgBromAAAAAAADKDq78cFVmD5OCA7zlTUAAAAAAAGUCdxLgiggIAAAAAKDs4QoQeZjNJgUHWOXtZXZ3VwAAAAAARYg7CeCEgAAAAAAAyi7uJICD58WAwEJAAAAAAABlEiEBJEleZg8FB3rLy5OAAAAAAADKKh43AAEBAAAAAEASdxKUeRZPDwUFWOXlSV4EAAAAAGUdV4ZlGAEBAAAAAODPuJOgjLJ4eig40CpPMwEBAAAAACAXV4hlEAEBAAAAAOBKuJOgjPH2Mis4wFtmAgIAAAAAwGW4UixDCAgAAAAAANfCnQRlhNXLrKBAq8weJnd3BQAAAABQTPEr5TKAgAAAAAAAkB/cSVDKWS1mBQUQEAAAAAAA/hohQSlmtZgVHGCVBwEBAAAAACAfCAlKKavFU8EB3gQEAAAAAIB8IyQoZUySrN6eCvInIAAAAAAAFAwhQSlyKSAIDvCWyURAAAAAAAAoGEKCUsIkyceaewcBAQEAAAAA4HrwCsRSgIAAAAAAAFAYCAlKOAICAAAAAEBhISQowQgIAAAAAACFiTEJSiiTTLJazQQEAAAAAIBCQ0hQQvlYPVXOz0JAAAAAAAAoNIQEJZCXp1lWiycBAQAAAACgUBESlECeZoaSAAAAAAAUPq42AQAAAACAJEICAAAAAABwESEBAAAAAACQREgAAAAAAAAuIiQAAAAAAACSCAkAAAAAAMBFhAQAAAAAAEASIQEAAAAAALiIkAAAAAAAAEgiJAAAAAAAABcREgAAAAAAAEmEBAAAAAAA4CJCAgAAAAAAIImQAAAAAAAAXFQsQgK73a633npLt9xyiyIiIjRq1CidOHEiX8t++umnCg0NVVxcnFN7VFSUQkNDnX6eeOIJV3QfAAAAAIBSwdPdHZCkWbNmacmSJXr55ZdVtWpVvfbaaxo5cqQ+++wzWSyWqy7322+/6bnnnsvTnpaWphMnTuidd95ReHi4o91qtbqk/wAAAAAAlAZuv5MgKytLCxcu1Pjx49WlSxc1atRIM2bM0KlTp7R27dqrLme32zV58mSnEOCS2NhY2e12RUZGqlKlSo6fgIAAV+4KAAAAAAAlmttDgv379ys1NVXt2rVztAUGBiosLExbtmy56nJz5sxRdna2Hn744TzTDhw4oIoVK6pcuXIu6TMAAAAAAKWR2x83OHXqlCTppptucmqvXLmyY9rldu3apYULF+qjjz5SfHx8nukHDhyQr6+vxo8fr+3btys4OFh33323hg4dKg+P68tFDMNQWlradS1bGNLT053+i8JFfV2L+roW9XUdauta1Ne1qK9rUV/XKe61NQxDJpPJ3d0AXMbtIcGlL//lYw94e3srKSkpz/xpaWmaNGmSJk2apDp16lwxJDh48KCSk5PVo0cPjRkzRtu2bdNrr72mpKQkPfroo9fVz+zsbO3bt++6li1MR48edXcXSjXq61rU17Wor+tQW9eivq5FfV2L+rpOca7ttcZNA0o6t4cElwYTzMrKchpYMDMzUz4+Pnnmf+GFF1S3bl0NHDjwquucN2+eMjMzHWMQhIaGKiUlRbNnz9a4ceOu624CLy8vNWjQoMDLFZb09HQdPXpUderUuWJdcGOor2tRX9eivq5DbV2L+roW9XUt6us6xb22sbGx7u4C4FJuDwkuPWaQkJCgWrVqOdoTEhIUGhqaZ/4VK1bIYrEoMjJSkmSz2SRJvXr10iOPPKJHHnlEFoslT7oXEhKitLQ0JSUlKTg4uMD9NJlM8vX1LfByhc3Hx6dY9KO0or6uRX1di/q6DrV1LerrWtTXtaiv6xTX2vKoAUo7t4cEjRo1kr+/vzZt2uQICZKTk7V3714NGTIkz/yXv/Fg586dmjx5subOnauQkBAZhqHbbrtNffv21dixYx3z7d69W5UqVbqugAAAAAAAgLLA7SGBxWLRkCFD9Prrr6t8+fKqXr26XnvtNVWtWlVRUVGy2Ww6d+6cAgICZLVaVbt2baflLw1uWK1aNQUFBUmSbrvtNi1YsED16tVT48aNtWHDBs2fP19Tpkwp6t0DAAAAAKDEcHtIIEnjx49XTk6Onn76aWVkZKhVq1ZasGCBvLy8FBcXp27duulf//qX+vfvn6/1TZw4Uf7+/po+fbpOnTqlGjVqaMqUKbr33ntdvCcAAAAAAJRcxSIkMJvNmjx5siZPnpxnWo0aNXTgwIGrLtumTZs80z09PTVmzBiNGTOm0PsKAAAAAEBpVfBh/gEAAAAAQKlESAAAAAAAACQREgAAAAAAgIsICQAAAAAAgCRCAgAAAAAAcBEhAQAAAAAAkCSZDMMw3N2J4m779u0yDEMWi8VtfTAMQ9nZ2fLy8pLJZHJbP0or6uta1Ne1qK/rUFvXor6uRX1di/q6TnGvbVZWlkwmk5o3b+7urgAu4enuDpQExeHkZDKZ3BpSlHbU17Wor2tRX9ehtq5FfV2L+roW9XWd4l5bk8lULK4PAFfhTgIAAAAAACCJMQkAAAAAAMBFhAQAAAAAAEASIQEAAAAAALiIkAAAAAAAAEgiJAAAAAAAABcREgAAAAAAAEmEBAAAAAAA4CJCAgAAAAAAIImQAAAAAAAAXERIAAAAAAAAJBESAAAAAACAiwgJAAAAAACAJEKCImO32/XWW2/plltuUUREhEaNGqUTJ07ka9lPP/1UoaGhiouLc1rf/Pnz1aNHD0VEROjOO+/U8uXLnZabPXu2QkND8/yUNoVdW0mKiorKU7cnnnjCMf38+fOaOHGiWrVqpdatW2vatGlKT08v1P0qLgq7vlc6Ji/9nDx5UpK0bdu2K07ftGmTS/bRnQpa30s1vfznzzX+8ssvdccdd6hp06bq27evNmzY4LQOjt/rry/nXmeuOH45/+Yq7Npy7nVW0PpmZ2frjTfecMw/ZMgQ7du3z2meDRs2qH///mrWrJl69uyp1atXO03PzMzUtGnT1K5dO0VGRmrixIk6d+6cS/bP3VxR3xUrVqh3796KiIhQVFSU5s6dK5vN5pien/MLgHwyUCSio6ONNm3aGN9++62xb98+48EHHzSioqKMzMzMay4XFxdntGjRwggJCTFOnDjhaJ81a5bRsmVLY/Xq1caxY8eMpUuXGmFhYcaqVasc8zz66KPG5MmTjYSEBKef0qawa5uammo0atTI+Pbbb53qlpyc7JhnyJAhxt13323s2bPH+Omnn4yuXbsajz32mMv20Z0Ku76XH48HDx402rRp41S/999/3+jevXueef9qmyVRQev76quvGkOGDMlTm5ycHMMwDGPDhg1GeHi48e677xqxsbHGyy+/bDRu3NiIjY11rIPj9/rry7nXWWHXl/PvHwq7tpx7nRW0vk899ZTRvn17Y/369UZsbKwxbtw4o0OHDo5jMzY21mjSpIkxffp0IzY21pg/f74RFhZm/PTTT451PPHEE0b37t2NLVu2GDt37jT69u1r3H///UWyv0WtsOv7ySefGOHh4cbSpUuNY8eOGatXrzaaN29uREdHO9bxV98BAPlHSFAEMjMzjcjISOP99993tCUlJRlNmzY1Pvvss6suZ7PZjEGDBhlDhw7Nc6F1yy23GLNmzXKa/8knnzQGDx7s+Hz77bcbixYtKrwdKYZcUdudO3caISEhRmJi4hWX3b59uxESEuJ00fX9998boaGhxqlTpwphr4oPV9T3cuPGjTN69uzp9A+HqVOnGo888kjh7EQxdj31HTlypPH8889fdZ0PPvig8eijjzq13XfffcYzzzxjGAbH743Wl3PvH1xRX86/uVxR28tx7s1/fY8fP26EhoYa3377rdP8Xbt2dYQAzzzzjHHPPfc4LTdhwgTjwQcfNAzDME6dOmU0atTIWLdunWP64cOHjZCQEGP79u2FuXtu54r6Dhw40JgyZYrTcjExMUbnzp0dnwv6HQBwdTxuUAT279+v1NRUtWvXztEWGBiosLAwbdmy5arLzZkzR9nZ2Xr44Yed2u12u1555RX169fPqd3Dw0PJycmSpKysLB09elT16tUrxD0pfgq7tpJ04MABVaxYUeXKlbvislu3blWlSpVUv359R1vr1q1lMpm0bdu2G9ib4scV9f2zH374QWvXrtXzzz8vi8XiaD9w4IBTfUur66nvtWpjt9u1fft2p/VJUps2bRzr4/i9sfpy7v1DYdf30nTOv66p7Z9x7i1YfX/88UcFBASoU6dOTvN/8803jnVs3bo1z7m3bdu22rZtmwzDcByfbdu2dUyvW7euqlSpcs3/n5ZErqjvpEmT9Le//c1pOQ8PDyUlJTk+l5XjFygKhARF4NSpU5Kkm266yam9cuXKjmmX27VrlxYuXKjXXntNZrPZaZqHh4fatWunqlWrOtpOnjyp1atXq2PHjpKk2NhY2Ww2rVmzRj169FCXLl00efJkJSQkFOauuV1h11bK/Z+Mr6+vxo8fr44dO6p3795avHix7Ha7JCk+Pj7P9iwWi4KCgvT7778Xxm4VG66o759Nnz5d3bp1U8uWLZ3aDx48qMOHD6t///7q0KGDRowYoV27dt3AnhRPBa1vUlKS4uPjtXXrVvXu3VsdO3bU6NGjdeTIEUlScnKy0tLSnM4Nl6+P4/f668u511lh11fi/HuJK2r7Z5x7C1bfI0eOqGbNmlq7dq2jNqNGjdKhQ4ec1nmlc296errOnz+v+Ph4BQcHy9vbO1/bLMlcUd8WLVqobt26js8XLlzQBx98oFtuuUVSwb8DAK6NkKAIXBpQ6c9pvSR5e3srMzMzz/xpaWmaNGmSJk2apDp16vzl+s+cOaNRo0apQoUK+vvf/y5J+vXXXyVJPj4+mjlzpl588UUdPnxYQ4cOVUZGxg3uUfHhitoePHhQycnJ6tGjhxYsWKBBgwZp5syZio6Odmzz8u1da5slmSuP3S1btuiXX37R6NGjndp///13XbhwQWlpaXr66ac1a9YsVaxYUUOGDFFsbOyN7VAxU9D6Hjx4UJJkGIb+9a9/6c0331RmZqYGDx6sM2fOOL7b11ofx+/11/dyZfncK7mmvpx/c7ny2OXcW/D6pqSk6NixY5o1a5YmTJig2bNny9PTU4MHD9bZs2clSRkZGXnWd+lzVlZWmTl2JdfU989SU1M1evRoZWZm6rHHHpNU8PM3gGvzdHcHygKr1Sop938Sl/4s5Y5y6+Pjk2f+F154QXXr1tXAgQP/ct2HDx/WQw89JJvNpvfee0+BgYGSpL59+6pTp04qX768Y96GDRuqU6dO+uabb3THHXfc6G4VC66o7bx585SZmamAgABJuSNCp6SkaPbs2Ro3bpysVquysrLyLJeZmSlfX98b3aVixZXH7qpVq9S0aVOFh4c7td90003asmWLfHx85OXlJUlq0qSJ9u7dq3//+9+aNm3ajexSsVLQ+rZs2VIbNmxQcHCwTCaTJCkmJkZdunTRypUrNWDAAMf6/uzP6+P4vf76PvTQQ455y/q5V3JNfTn/5nLlscu5t+D19fT0VEpKimbMmOG4nX3GjBnq3LmzVq1apZEjR8rb2zvPsXnps4+PzzWP3SttsyRzRX0vOX36tB5++GHFxcVpwYIFqlGjhqSCfQcA/DXuJCgCl263uvx204SEBFWpUiXP/CtWrNBPP/2kyMhIRUZGatSoUZKkXr16ac6cOY75tm3bpoEDB8rHx0dLly5VzZo1ndbz53+kSrm3eQUFBZWq29pcUVuLxeL4B+olISEhSktLU1JSkqpWrZpne1lZWUpMTFTlypULbd+KA1cdu3a7Xd9884169+59xe0GBgY6/pEq5d7mXb9+fcXHx9/wPhUnBa2vlPu9vvQPICn3H581atRQfHy8goKC5Ovre831cfxef30v4dybyxX15fyby1XHLufeXAWtb9WqVeXp6en0vLvValXNmjUdr9e76aabrrg+X19fBQQEqGrVqkpMTMwTFFzr77SkckV9JenQoUO69957dfbsWb3//vtq0qSJ03ry8x0AkD+EBEWgUaNG8vf3d3rPcHJysvbu3atWrVrlmX/t2rX6/PPP9fHHH+vjjz/WCy+8IEmaO3eu4ze0u3bt0siRI9WwYUO9//77eU66M2bMUI8ePWQYhqMtLi5O58+fV4MGDVyxm25R2LU1DEPdu3dXTEyM03K7d+9WpUqVFBwcrFatWunUqVM6duyYY/rmzZsl5T4zV5q44tiVcp/bPn/+vNq3b59nHevXr1dkZKTT+5RzcnK0f//+UnXsSgWv77Jly9SmTRulpaU52lJSUnT06FE1aNBAJpNJzZs3dxyPl2zatMnx7DHH7/XXV+Lc+2eFXV/Ov39wxbErce69pKD1bdWqlXJycrR7925HW0ZGhk6cOKHatWtLyv1N9uXn3o0bN6p58+by8PBQixYtZLfbnQbYPHLkiOLj46+4zZLMFfU9ceKEhg0b5ghnGzZs6LSO/H4HAOSTu16rUNZMnz7daN26tfHVV185vS82KyvLyMnJMRISEoz09PQrLrtx40an18hlZ2cbt912m9GtWzfj+PHjTu+CPXv2rGEYhrF7924jPDzcePbZZ43Dhw8bmzdvNvr27WsMHDjQsNvtRbbfRaEwa2sYhvHyyy8bERERTu9Bb9q0qbFs2TLDMAzDbrcbAwcONPr162fs3LnT2LBhg9G1a1fjiSeeKJL9LWqFXV/DMIxVq1YZ4eHhhs1my7PMhQsXjK5duxqDBg0ydu/ebezfv9+YMGGC0apVK+P06dMu2Ud3Kkh9T548abRs2dIYM2aM8euvvxq7du0yhg8fbnTv3t3IyMgwDCP3dXA333yzsXDhQiM2NtZ45ZVXjKZNmzpeGcfxe/315dybV2Efv5x//1DYtTUMzr1/VtD/tw0fPty4/fbbjS1bthgHDx40xo0bZ7Rr187x3f/111+N8PBw47XXXjNiY2ONBQsWGGFhYY5X+BlG7isRb731VmPjxo3Gzp07jb59+xpDhgwp8n0vCoVd3yFDhhitWrUy9u3b53TuTUhIMAwj/98BAPlDSFBEcnJyjFdffdVo27atERERYYwaNcpx4XTixAkjJCTEWLFixRWXvfxCa9u2bUZISMgVf7p27epY7qeffjLuu+8+IyIiwmjdurXx5JNPXvXd0yVZYdbWMHJDmJiYGKNbt25GeHi40aNHD8c/UC85c+aMMW7cOCMiIsJo06aNMXXq1FL7P6HCrq9hGMbcuXON9u3bX3Wbx44dM8aNG2e0bt3aaNasmfHggw8aBw4cKLydKkYKWt89e/YYI0aMMFq0aGE0b97cGDdunHHy5Emnda5atcq47bbbjCZNmhj9+vVz+keqYXD8Xm99OffmVdjHL+ffP7ji3MC59w8Fre+FCxeMqVOnGm3atDGaNWtmjBgxwjh48KDTOr/77jujV69eRuPGjY2ePXsaq1evdpqemppqTJkyxWjZsqXRsmVLY8KECca5c+dcv7NuUJj1PXXq1FXPvSEhIY515Oc7ACB/TIbxp3siAQAAAABAmcWYBAAAAAAAQBIhAQAAAAAAuIiQAAAAAAAASCIkAAAAAAAAFxESAAAAAAAASYQEAAAAAADgIkICAAAAAAAgiZAAAAAAAABcREgAACh0EydOVGhoqBYuXOjurpQIx44d06OPPqqOHTuqRYsWGjRokDZs2FDg9Zw6dUr333+/mjRponbt2ik9Pf0vl4mLi1NoaKhWrlwpSYqOjlZoaGiBtw0AAEoHQgIAQKG6cOGCvvrqK4WEhGjZsmUyDMPdXSrWzp8/ryFDhujw4cN66qmnNGPGDFWsWFEPPvigNm/eXKB1vfvuu9qxY4deeeUVxcTEyMfHx0W9BgAApZWnuzsAAChdPv/8c0nSlClTNGzYMG3cuFHt2rVzc6+Kr48//ljnz5/XRx99pCpVqkiSOnTooD59+mjBggVq3bp1vteVmJioypUr64477nBVdwEAQCnHnQQAgEK1YsUKtWvXTm3btlXt2rW1dOlSx7QHH3xQ/fv3z7PM6NGjdddddzk+b926VUOGDFGzZs3UunVrPf744zp37pxj+sqVKxUWFqbly5erQ4cOat26tWJjY2Wz2TR37lz16tVLTZs2VUREhAYOHKiNGzc6bW/dunXq37+/mjZtqh49eujzzz/XbbfdpujoaMc8iYmJevbZZ9W+fXs1adJE9957b4EfAZgzZ44aN26spKQkp/bFixcrPDxcZ8+eVZUqVTR8+HBHQCBJZrNZtWvX1vHjx/O9rVtvvVUrV67UyZMnFRoaqujoaK1cuVKhoaGKi4vLM+8TTzxRoH0BAABlAyEBAKDQHDx4ULt371bfvn0lSX379tXXX3+tM2fOSJLuuusu/fLLLzp27JhjmeTkZK1fv159+vSRJG3ZskXDhw+X1WrVm2++qaeeekqbN2/W0KFDlZGR4VjOZrNp4cKFevHFF/Xkk0+qfv36ev311zVr1izdd999mj9/vp5//nklJibq0UcfdTyfv3HjRo0ePVo33XSToqOjdf/992vq1Kn6/fffHevOzMzUsGHD9PXXX+uf//ynYmJiVLVqVY0cObJAQUHv3r2Vk5OjtWvXOrWvXr1aHTt2VIUKFXTHHXdo0qRJTtOTkpK0ZcsWNWzYMN/biomJUefOnVWpUiUtW7ZMAwYMyPeyAAAAlxASAAAKzYoVKxQUFKRbb71VktSvXz/ZbDZ99NFHkqSoqCj5+vo6HkmQpLVr18pms6lXr16SpDfeeEN169bVO++8o65du6pv375auHChDh8+rBUrVjht75FHHlGXLl3Ur18/mUwmJSQk6J///KceeOABtW7dWr1799aECROUmJioAwcOSModmK9hw4aOi+qhQ4fqueeeU3Z2tmO9n3zyifbv369Zs2ZpwIAB6ty5s2bOnKnIyEi9/vrr+a5H9erV1apVK6f9PX78uHbt2uUIRS5nt9v1zDPPKCUlRSNHjsz3tsLCwlS+fHlZLBZFRESoatWq+V4WAADgEkICAEChyM7O1qeffqru3bsrIyNDycnJ8vPzU4sWLfThhx/KbrfL19dX3bt31xdffOFYbvXq1WrXrp2qVKmi9PR07dy5U507d5ZhGMrJyVFOTo5q1qyp+vXr68cff3Ta5s033+z0+Y033tCwYcN07tw5bd26VStWrNCnn34qScrKylJWVpZ+/vlnRUVFyWQyOZbr2bOnPD3/GKZnw4YNqlSpksLDwx19sNls6tq1q/bs2ZPn8YFrueuuu7RlyxadPn3asb/+/v6OIOXyGk6ePFlr1qzRlClT1LRp03xvBwAAoDAwcCEAoFCsW7dOZ8+e1UcffeS4c+DPvv/+e3Xu3Fl9+vTRp59+qv3796tixYratGmTXnrpJUm5jx7Y7XbNmzdP8+bNy7MOb29vp8++vr5On3fv3q1p06Zp9+7d8vHxUYMGDVStWjVJkmEYSkxMlM1mU4UKFZyWM5vNCgoKcnxOTEzU6dOnFR4efsV9PX36tMqVK/fXRVFuAPH888/ryy+/1NChQ7V69Wr16NFDVqvVab7k5GSNHTtWW7Zs0TPPPKP7778/X+sHAAAoTIQEAIBCsWLFCtWsWVMvvviiU7thGBo7dqyWLl2qzp07q127dqpUqZK+/PJLVapUSd7e3oqKipIk+fn5yWQyafjw4brzzjvzbONar/S7dHt+aGioVq9erXr16snDw0Pfffed1qxZI0mqUKGCvLy8HGMkXGK325WYmOj4HBAQoDp16lz10YIaNWrkqyaX1nXrrbfqyy+/VNu2bXXw4EE988wzTvOcOnVKI0aMUFxcnKZPn67bb7893+u/lkt3S9jtdqf21NTUQlk/AAAofQgJAAA37PTp0/r+++81cuRItWnTJs/0nj17auXKlYqPj1eVKlXUu3dvffvttwoMDFT37t0ddwT4+/srLCxMhw8fVpMmTRzLZ2RkaPz48ercubMaNGhwxT4cPnxYiYmJGjp0qNM869evl5R7oWw2m9W8eXN9/fXXGjt2rGOeb775Rjk5OY7PrVu31rp161ShQgXHnQhS7tsK9u3bV6BxCSSpT58+Gj16tD744ANVq1bN6bWGKSkpGjZsmM6cOaNFixapZcuWBVr3tfj7+0vKDSFq1aolSTp06JBTIAIAAPBnjEkAALhhH3/8sXJycq74238p9y0HNptNH374oaTci+YDBw5o27ZteQbwmzBhgn744QdNnDhR3333nb755hvHWwWudvu/JNWtW1f+/v6aM2eO1q1bpx9++EHPPPOMlixZIkmOtxuMHz9e+/fv1/jx47V+/XotXbrU8Zv9S79579+/v6pVq6YRI0Zo1apV2rhxo6ZPn66ZM2eqcuXK8vLyKlB9brnlFgUFBWnZsmXq3bu303gIb731lo4ePaoRI0bI09NTO3bscPzs3bu3QNu5XJs2bWS1WvXyyy/ru+++0xdffKExY8Y4PVoBAADwZ4QEAIAbtnLlSjVs2FAhISFXnN6iRQvVqFFDy5cvl81mU6NGjRQSEqIKFSqoXbt2TvN27NhRCxYs0KlTpzR+/Hg99thjMpvNWrRokSIiIq7ah4CAAM2aNUuGYejRRx/VY489ppMnT+o///mP/Pz8tHXrVklSy5YtFR0drSNHjmj06NFatGiRIyTw8/OTlDvWwfvvv68WLVrotdde06hRo7R27VpNnDhRTz75ZIHr4+npqTvvvFM2m0133XWX07RLr0eMjo7Wfffd5/Tz57sdrkdgYKCio6Nls9k0ZswYzZw5U2PGjFHjxo1vaL0AAKD0MhmGYbi7EwAAFJWvv/5aVatWdbor4eDBg+rVq5dmzZqlbt26ubF3AAAA7sWYBACAMuWHH37QF198oUmTJqlu3bqKj4/X7NmzVa9ePXXs2DFf6zAMQzab7S/nM5vNTo8WXA+73Z5n4MEr+fMrHAEAAK4XdxIAAMqUjIwMzZw5U2vWrFFCQoKCgoJ0yy23aOLEiapYsWK+1rFy5cp8PXbw3nvvXXEgx4KIjo5WTEzMX8739ddfF+itCwAAAFdCSAAAQAGdP39ecXFxfznfpcEUb0R8fLwSEhL+cr7Q0FBZLJYb2hYAAAAhAQAAAAAAkMTbDQAAAAAAwEWEBAAAAAAAQBIhAQAAAAAAuIiQAAAAAAAASCIkAAAAAAAAFxESAAAAAAAASYQEAAAAAADgov8HIz4WWgfBq7wAAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -1512,7 +1707,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -1548,30 +1743,30 @@ " \n", " \n", " 0\n", - " GritLM/GritLM-7B\n", + " intfloat/e5-mistral-7b-instruct\n", " 0.63\n", " 1.0\n", - " 0.69\n", - " 2.0\n", - " 66.76\n", + " 0.67\n", " 1.0\n", + " 66.63\n", + " 2.0\n", " \n", " \n", " 1\n", - " intfloat/e5-mistral-7b-instruct\n", + " GritLM/GritLM-7B\n", " 0.63\n", " 2.0\n", - " 0.69\n", - " 1.0\n", - " 66.63\n", + " 0.66\n", " 2.0\n", + " 66.76\n", + " 1.0\n", " \n", " \n", " 2\n", " intfloat/multilingual-e5-large-instruct\n", " 0.61\n", " 3.0\n", - " 0.68\n", + " 0.65\n", " 3.0\n", " 64.41\n", " 3.0\n", @@ -1581,7 +1776,7 @@ " intfloat/multilingual-e5-large\n", " 0.57\n", " 4.0\n", - " 0.64\n", + " 0.62\n", " 4.0\n", " 60.89\n", " 4.0\n", @@ -1591,37 +1786,37 @@ " intfloat/multilingual-e5-base\n", " 0.56\n", " 5.0\n", - " 0.63\n", + " 0.60\n", " 5.0\n", " 59.11\n", " 5.0\n", " \n", " \n", " 5\n", - " sentence-transformers/all-mpnet-base-v2\n", + " intfloat/multilingual-e5-small\n", " 0.54\n", " 6.0\n", - " 0.60\n", - " 8.0\n", - " 57.78\n", + " 0.58\n", " 6.0\n", + " 57.04\n", + " 7.0\n", " \n", " \n", " 6\n", - " intfloat/multilingual-e5-small\n", - " 0.54\n", + " sentence-transformers/all-mpnet-base-v2\n", + " 0.53\n", " 7.0\n", - " 0.61\n", + " 0.56\n", + " 8.0\n", + " 57.78\n", " 6.0\n", - " 57.04\n", - " 7.0\n", " \n", " \n", " 7\n", " sentence-transformers/paraphrase-multilingual-...\n", - " 0.53\n", + " 0.52\n", " 8.0\n", - " 0.60\n", + " 0.57\n", " 7.0\n", " 54.64\n", " 10.0\n", @@ -1631,7 +1826,7 @@ " sentence-transformers/all-MiniLM-L12-v2\n", " 0.52\n", " 9.0\n", - " 0.58\n", + " 0.55\n", " 10.0\n", " 56.53\n", " 8.0\n", @@ -1639,9 +1834,9 @@ " \n", " 9\n", " sentence-transformers/all-MiniLM-L6-v2\n", - " 0.52\n", + " 0.51\n", " 10.0\n", - " 0.58\n", + " 0.54\n", " 11.0\n", " 56.10\n", " 9.0\n", @@ -1649,9 +1844,9 @@ " \n", " 10\n", " sentence-transformers/paraphrase-multilingual-...\n", - " 0.51\n", + " 0.50\n", " 11.0\n", - " 0.58\n", + " 0.55\n", " 9.0\n", " 52.45\n", " 11.0\n", @@ -1659,9 +1854,9 @@ " \n", " 11\n", " sentence-transformers/LaBSE\n", - " 0.44\n", + " 0.43\n", " 12.0\n", - " 0.53\n", + " 0.49\n", " 12.0\n", " 45.21\n", " 12.0\n", @@ -1672,35 +1867,35 @@ ], "text/plain": [ " model Average_v2_full \\\n", - "0 GritLM/GritLM-7B 0.63 \n", - "1 intfloat/e5-mistral-7b-instruct 0.63 \n", + "0 intfloat/e5-mistral-7b-instruct 0.63 \n", + "1 GritLM/GritLM-7B 0.63 \n", "2 intfloat/multilingual-e5-large-instruct 0.61 \n", "3 intfloat/multilingual-e5-large 0.57 \n", "4 intfloat/multilingual-e5-base 0.56 \n", - "5 sentence-transformers/all-mpnet-base-v2 0.54 \n", - "6 intfloat/multilingual-e5-small 0.54 \n", - "7 sentence-transformers/paraphrase-multilingual-... 0.53 \n", + "5 intfloat/multilingual-e5-small 0.54 \n", + "6 sentence-transformers/all-mpnet-base-v2 0.53 \n", + "7 sentence-transformers/paraphrase-multilingual-... 0.52 \n", "8 sentence-transformers/all-MiniLM-L12-v2 0.52 \n", - "9 sentence-transformers/all-MiniLM-L6-v2 0.52 \n", - "10 sentence-transformers/paraphrase-multilingual-... 0.51 \n", - "11 sentence-transformers/LaBSE 0.44 \n", + "9 sentence-transformers/all-MiniLM-L6-v2 0.51 \n", + "10 sentence-transformers/paraphrase-multilingual-... 0.50 \n", + "11 sentence-transformers/LaBSE 0.43 \n", "\n", " Rank_v2_full Average_v2_lite Rank_v2_lite Average Rank \n", - "0 1.0 0.69 2.0 66.76 1.0 \n", - "1 2.0 0.69 1.0 66.63 2.0 \n", - "2 3.0 0.68 3.0 64.41 3.0 \n", - "3 4.0 0.64 4.0 60.89 4.0 \n", - "4 5.0 0.63 5.0 59.11 5.0 \n", - "5 6.0 0.60 8.0 57.78 6.0 \n", - "6 7.0 0.61 6.0 57.04 7.0 \n", - "7 8.0 0.60 7.0 54.64 10.0 \n", - "8 9.0 0.58 10.0 56.53 8.0 \n", - "9 10.0 0.58 11.0 56.10 9.0 \n", - "10 11.0 0.58 9.0 52.45 11.0 \n", - "11 12.0 0.53 12.0 45.21 12.0 " + "0 1.0 0.67 1.0 66.63 2.0 \n", + "1 2.0 0.66 2.0 66.76 1.0 \n", + "2 3.0 0.65 3.0 64.41 3.0 \n", + "3 4.0 0.62 4.0 60.89 4.0 \n", + "4 5.0 0.60 5.0 59.11 5.0 \n", + "5 6.0 0.58 6.0 57.04 7.0 \n", + "6 7.0 0.56 8.0 57.78 6.0 \n", + "7 8.0 0.57 7.0 54.64 10.0 \n", + "8 9.0 0.55 10.0 56.53 8.0 \n", + "9 10.0 0.54 11.0 56.10 9.0 \n", + "10 11.0 0.55 9.0 52.45 11.0 \n", + "11 12.0 0.49 12.0 45.21 12.0 " ] }, - "execution_count": 83, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -1721,12 +1916,12 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 24, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAMzCAYAAADnL2V5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3hUZaIG8Pec6ZmZFGooUkNCSUKRKlJEBWkqYEVREUVgva67d6+w666u7l3d9a5rB0QQ61opohRFxS4oKCTU0EInAdKmnynn/jFkkoFAkqlnMu/veXyUb9r3SUheTnk/QZZlGUREREQUV2K8J0BEREREDGVEREREisBQRkRERKQADGVERERECsBQRkRERKQADGVERERECsBQRkRERKQA6nhPINZ+/fVXyLIMjUYT76kQERFRE+d2uyEIAvr27Vvvc5PuSJksy0jEvlxZliFJUkLOPRzJum4gedfOdSfXuoHkXTvXnRzrbkzuSLojZdVHyPLy8uI8k8ax2+3YtWsXsrKykJKSEu/pxEyyrhtI3rVz3cm1biB51851J8e6CwsLG/zcpDtSRkRERKREDGVERERECsBQRkRERKQADGVERERECsBQRkRERKQADGVERERECsBQRkRERKQADGVERERECsBQRkRERKQADGVERERECsBQRkRERKQADGVERERECsBQRkRERKQADGVERERECsBQRkRERKQADGVERERECsBQRkRERKQADGVERERECsBQRkRERKQADGVERERECsBQRkRERKQADGVERERECsBQRkRERKQADGVERESUtFxuL9weX7ynAYChjIiIiJKUU/KgosoJWZbjPRUAgDreEyAiIiKKNbvTjUqrBEAZgQxgKCMiIqIkY3VIsNgk+GRAEOI9mxoMZURERJQ0LDYJFocEhZyxDMJQRkRERE2eLMuoskmwOd2KDGQAQxkRERE1cbIso8LqgsPpUdAVZOdjKCMiIqImy+uTUWFxwil54z2VejGUERERUZPk9fpQbnHB5VZ+IAMYyoiIiKgJcnt8qLA4ISmkGLYhGMqIiIioSZHcXlRYXHB7EyeQAQxlRERE1IS4JA/KrS54vUq+pL9uDGVERETUJDhc/pZ+ry/xAhnAUEZERERNgM3pRpVVgk+pJWQNwFBGRERECc1ql2Cx+7dNSmQMZURERJSwqqwSrE5lbpvUWAxlRERElHASYdukxmIoIyIiooSSKNsmNRZDGRERESUMn88fyJyuphXIAIYyIiIiShCJtI9lKBjKiIiISPESbR/LUDCUERERkaJ5vD6UVyXWPpahYCgjIiIixXJ7vCivSrx9LEPBUEZERESK5HJ7UWFxwnPOPpY6jQopPg/Es5f6+wA4RDWc7sQObgxlREREpDguyYNyiytoH0uTWoBRlCEXHYDvh18hV1kBWYYq1YjUwb2R1qMr7D4BVo+ckHdmMpQRERGRopy7sbggAC00AH74FZ5vNgMuKej5coUF3sMnAY0a+sv6Qj9yIM645YTbdkmM9wSIiIiIqtmdblRYpKAjZC00AuRl6+Fb/8N5gSyI2wPf1z9Dfmc1WmgECOc8rFGLaGn0Qa3yP1J8ogpLP96BTzceCjzH56iCz2mJ5JIajKGMiIiIFMHmkFBpleCrtW9SmhrAlxshb9/b4PeRi4ohr/kaaeqa99GoRaSjApblDyFDZcWm7Sfwr7c2Y+P2k1i0shBHSizwOS2wbfkQjh2fwue0RnJpDcLTl0RERBR3FpsEq0M675SjHj54vv+l0e/n+3k7dFdfBkFQQ63yB7LyN++Bu7IUbz+3BOsrewWea9CpkSK6YNu8CtavXqoZ73UNRL0p5DU1Fo+UERERUVxVWSVY6ghkKRoRvs3bEerFYb4ffoVZqwoEMktFJV6wTg0KZOYUDf48LRfaPcGBrGrtk3DsWBfTI2YMZURERBQXsiyj0uqC1SlBriN3GWUvfN/9GvL7+7YVwajxoPzNe3Cs3IMnLTOw09M18Pgl2jN4elY/dD7zRVAgq1a19km4Dm6Ez3OR69giiKcviYiIKC4qrC44nBfeWFwUAJ/FFvoHlFfC53ajqMt9eH6DE07oAw9dqtmBu1JWAa+/DKvPU+fLNW16QNuhH0S1NvQ5NAJDGREREcWUWqNBpVWCLKij2icmA1jx7TG8uQGQawWy6/QbME7/LQQB/ubZOmja9EDGLc9DZWwWxRkGYygjIiKimPH5ZNicPtidbugNF48hsiAAogj4Gt/U74KAl4VMfPvlgcCYTg3crXsPfbV7LvraeAQygNeUERERUYy4PT5UWCVY7a4GHSFz+QChV1ajP+cM1HgUl+BbpAbGWqTp8fjdl2LYtZMv+lp1y65xCWQAQxkRERHFgOT2orzKCafkgVzXVf11sPoEqK4c3KjP2Qs95qEj9guGwFhOhwzMu3MAOmUaIdvLLvp62e0EfN5GfWakMJQRERFRVLkkD8qqnHB7G3ca0uuT4TUbIbRu3qDnf41UPIpLUCHUnBYd0a89Hri5Dy5JF+ApWA7bNy9f/DMrjuHMmzPhtZxq1FwjgaGMiIiIosbmdKOsKnhj8caokFVQzZgMpOgv+BwvgNfREi8KbeAW/NFGJQqYNrY7bh2dgxYGrz+QfT2/QZ/pLTscl2DGUEZERERRYbFJqLK6grZNaiyP14cKlRbq394OpJvPe9wGEU+iPT4Raq4BSzWo8dBt/TC0dztkaN0XDWSCIa3O8XgEM4YyIiIiiihZllFucdbZ0h8Kl0dGmVoP1W+nQTXtWghtWwEAjkGDP6IjtgnGwHM7tTbhzzMGo2O7dAgCoFELcG5fU+f7atr0QIs5K5E64ZE6H/eWH4W7dC98Hlf4i2gAhjIiIiKKGK9PRlmV018KG8ESMrfHh1IJqOjcEfKMKdh2yw34k74bTgg1xa79u7fC727rj1STDgAgy0CZS4P0qQugat4p6P00bXog9cZnccahgjZ7FNImPBr8gYKIjCn/hLZ9H4hqXeQWchEMZURERBQRHq8PZZUOOCVv1EphXZIH73x/GH9/vxB2V81dkhMv74y7r8uFTqs6Z04yyr3GoGBWHcjK3AZ4fTJkTQr03WsFs+pA1nkwRF1KlFZyPpbHEhERUdjcHi/Kq1yNvsOysZ/x1trd2LTjZGBMp1Hhrgk90Ten1QVf5/HKKIcRGVMXwPblszBe+ftAIBME/3NEvQn67qMC/x3rQAYwlBEREVGYXJIH5VYXvN7obZpUYXFh4fICFJ+oCow1T9NjzpTeaNfKVO/rq4NZ2tVzUeZS13k3aCCYCULMAxnAUEZERERhcLjcqLRKIVdeNMTB45VYuLwAlVYpMNbtknTcNykPppSGbxbu8co441ABFzm5KurrD3jRwlBGREREIbE53aiySmFVXtRn0/YTeHPtbnhqnRYd3rcdbr4qGypV07o0nqGMiIiIGs1ik2CNUOVFXXw+GSu+3of1mw4HxkRRwC1XZ2N43/bR+dA4YygjIiKiRqm0umBzuiNaeVGb3enGklU7sOPAmcCY0aDBfZPykN0hIzofqgAMZURERNQgsiyj0ibBHsVAVlJmx/wPt6GkzB4Ya9fShNlT8tEi3XCRVyY+hjIiIiKql88no8LqgtPliVoH2Y4DZ7D4o+1wuDyBsT7ZLXHXhJ7Qa5t+ZGn6KyQiIqKweH0yKixOOCVv/U8OgSzL+OLnI1i2YW/QEbjxQztj/OWdIVaXiTVxDGVERER0QR6vDxUWF1zu6AQyt8eLt9ftwcbtJwJjWo2Iu8b3Qr/uFy6EbYoYyoiIiKhObo8XFRYXJE90Wvorrf5C2IPHawphm6XqMWdKPtq3NkflM5WMoYyIiIjO43J7UWFxwhOllv7DJy14dXURKq2uwFhW+3TcNzkP5kYUwjYlDGVEREQUxCV5UG5xRa2lf89RBzYUbg8KfMP6tMPNV2dD3cQKYRuDoYyIiIgCorltks8n4+PvivHl1prTlaIo4OarsjGiX9MshG0MhjIiIiICANgcEqps7qhsm+RwerDk4+3Yvj+4EHbm9bnI6dgs4p+XiBjKiIiICBabBItDikopbEmZHQuWbcPJMzWFsG2ap2DOjX3QsokXwjYGQxkREVESk2UZVTYpatsm7Tx4BotXboe9ViFsl9Y63Ds5DxlpDGS1MZQRERElKVn2t/Q7nJFv6b9QIezoge3RraUEvVYV4U9MfAxlREREScjf0u+CS4p8IHN7fPjPp7vxY2FwIeyd43uiZ8dU7N+/P8KfGDol7RXAUEZERJRkvF4fyqPU0l9pdeHlFYU4cKwyMNYsVY/ZU/JxSWszHA5HxD8zVKIgIEWvVkwNB0MZERFREnF7fKiwOKPS0n/oZBUWLCtAhaV2IWwaZk7KR6pROYWwggBo1SqYUzTQKWijc+XMhIiIiKJKcvu3TXJ7Ix/Ift55Em+s2QV3rbB3ee+2uGV0jmKORAGAShRgMmhgNGggKGyjc4YyIiKiJOCUPKiIQku/T5ax6pv9WPfjocCYKAi46apuGNGvvWKCjyAAeq0a5hQtNGrlhMTaFBHKVq5ciUWLFuHIkSPo0KED7r//fowdOxYAsGDBAjz77LPnvWbPnj0xniUREVFisjndqLJKES+Fdbg8eHXVDhTuPx0YM+rVuPf6PHTvpJxCWI1KhNmogUGnifdULiruoeyjjz7Cww8/jD/96U8YNmwYVq9ejd///vfIzMxE3759sWfPHlx33XX4n//5n3hPlYiIKOFYbBKsDgmR3jWptNyO+R8GF8K2bWHE7Cn5aJmREtkPC1H1hfymFC1UojKO2F1MXEOZLMt47rnncMcdd+C2224DAMyePRubN2/GTz/9hL59+6KoqAg33XQTWrZsGc+pEhERJZRolsLuKi7DKysLYXfWFMLmZ7XA3RN7Qa+L+/EeCAA0ahGpRq2iLuSvT1xnevDgQRw7dgwTJ04MGl+yZAkAQJIkFBcXo0uXLvGYHhERUULy+fylsE5XZDvIZFnGhi1H8eEXe4NOhY69rBMmDusCUQHXj4mCAKNeA7NReRfy1yfuoQwA7HY7ZsyYgZ07d6J9+/aYPXs2Ro0ahX379sHr9eLTTz/F3//+d7hcLgwYMAD/8z//g1atWoX8ubIsw2631/9EBanudVFSv0ssJOu6geRdO9edXOsGknft0Vq3zyej0iZFPJB5PD58uOEANu0sDYxp1CJuvToLfbNbwOV0Nuh9XE5X0L8jRRAAnUYNo1EDteiBw+Gp/0UxIMtyg8OhIMvR2OmqYT766CM89NBDaN++Pe6//350794dn376KRYuXIilS5eipKQEc+fOxbRp03DDDTfgzJkz+Pe//w2bzYaVK1dCr9c3+jMLCwshSVIUVkNERBQ/giBAVGlgcXjgcEqI5I93m9OLtVsqcbLcHRgz6UWMG5COVmnxv3heq/FXXGjVMrweZYSx2rRaLfLy8up9XlyPlGk0/t/IGTNmYNKkSQCAHj16YOfOnVi6dCkWLVqE4cOHo1mzmjs4unXrhuHDh+PLL7/EuHHjQv7crKys8BcQQw6HA8XFxejUqRMMhuTZwDVZ1w0k79q57uRaN5C8a4/0ut0eHyqsEkyeyLb0Hymx4q2vdqPCWhPIOrUx4+7xOTCHUAjrcrpw9NhRtG/XHjq9Lqy5CYIAvdZfAqukLrTa9u3b1+DnxjWUtW7dGgCQnZ0dNJ6VlYWvvvoKAIICGQC0atUK6enpOHnyZMifKwgCUlKUcWdIYxkMhoSdeziSdd1A8q6d604+ybr2SKzb4XLD4ZCg1mihjuCBq7oKYYfm+wthw+360ul1YYVRlSjAlKKBUa/sa8caM7e4xspevXrBaDRi27ZtQeNFRUXo0KEDnnnmGYwZMyboEOzRo0dRXl6ecEe6iIiIosHmkFBhkSJaCuuTZaz8ej+WrNoRCGSiIODmq7Jx+9jucS1fFQTAoFOjRboBJoNW0YGsseJ6pEyv1+Oee+7BSy+9hNatWyM/Px+rV6/G999/j9deew1GoxFLlizBX//6V9x11104ffo0nnjiCfTr1w/Dhg2L59SJiIjizmKTYHFIEa28cLg8WPrxDhTsqymETTlbCNsjjoWw1TUXphTll8CGKu7lHXPmzIHBYMAzzzyDkpISdO3aFS+88AIGDRoEAHjllVfw3HPPYfLkydBqtbjyyisxd+7cJpWMiYiIGkOW/XdY2iPcQXaq3I75ywpw4rQtMNbmbCFsqzgWwlbXXJhSNBAToAQ2VHEPZQAwffp0TJ8+vc7HhgwZgiFDhsR4RkRERMoUrQ6y3cVlWHROIWxe1xa4+9peMMSpENZfc6GCOUULrUYVlznEkiJCGREREdXP65NRYXHCKUXuDktZlvHVlqP44JxC2GuGdMS1w7rG7ciUWiXAlKKFUd80T1XWhaGMiIgoAXi9PpRbXHC5IxfIPF4f3vlsD77fdjwwplGLuGNcDwzomdmo98pI8UD2XrgQVlDpUG6vP3aIAmDQa2A2aKBSaM1FtDCUERERKZzb40OFxQmpVjVFuKpsLry8ohD7j1YGxtLNOsyenI+ObVIb/X6y14Uf37lwf+iQW9fgYrFDEACt2t85lkj7VUZScq6aiIgoQUhuLyosLri9kQtkh09asGD5NpRX1RzZ6tw2FbMm5yPNFF6hayhUogCTwd/Kn8w38jGUERERKZRL8qDc6oLXG7lL+jfvKsHrq3cGFcIOyWuDqWNi3z8mCoBep4Y5RavYRv5YYigjIiJSIIfLjUpr5EphfbKMT749gDU/FAfGBAGYckU3XDngkpgeoRJFETqNCulmXZPtHAsFQxkREZHC2J3+QOaLUAmZ0+XB0k92YNveWoWwOjXuuT4XPTs3j8hnNJRKFJFu0qNZqp6B7BwMZURERApidUiw2CREatekUxUOLPhwG47XKoTNbJ6C2VN6o3Wz2BbC6rUqpOr1qDjlRRJfOnZBDGVEREQKEeltk3YXl+GVlYWwBRXCNsfd1+bGvBBWFIHmJgPsdjt8vsjdtNCUMJQREREpQKXVBVuEtk2SZRlf/3IU738eXAg7ZnBHXDc8OoWwgkp3tvai1hj8160JggCVRh/xz2xqGMqIiIjirNzihMPliUgg83h9ePezPfjunELYaWN7YGCvxhXCNoa/GNYfK9g5Fhr+nyIiIooTtVqNSqsEWVBHZB/LKpuERSsKsC9ChbChYOdY6BjKiIiI4kCWAZtLht3pht4Q/o/jIyUWLFhWgLIqZ2AsloWw/s3D1Ug1amPed9ZUMJQRERHFWPU+lla7C60i8H5bdvsLYSV3zQX0g3Pb4LZrcqBRqyLwCRenUglITdEiJYk2D48GhjIiIqIYqt7H0il5IId5EZlPlrH6u4NY/f3BwFgsC2EFATDo1EhN0Sbd5uHRwFBGREQUI26PF+VVkdnH0il58NonO7G16FRgLFaFsAL8Nw+YUjQsgI0ghjIiIqIYcLm9qLA44YnAPpanKxxYsKwAx05ZA2Otm6Vgzg3RL4RViQKMev+F/NGo1khmDGVERERR5pI8KLe4IrKP5Z5D5Vi0shA2hzswltu1OWZMzIVBH70f64IA6LX+zcN5IX90MJQRERFFUaQ2FpdlGV//egzvry8KKoQdPagjrh8RnULYamqViFQjT1VGG0MZERFRlNicblRFYGNxj9eH99YX4dutxwJjapWIaWO7Y1Bum3CneUG8kD+2GMqIiIiiwGqXYLGHv7G4xS7h5eWF2He0IjCWZtJh9pR8dIpiIaxKFGA2amFkzUXMMJQRERFFWKQ2Fj9aYsH8cwphO7XxF8Kmm6NTCOsvgVUh1ajjtWMxxlBGREQUIbIso8omRWRj8V92l+K11TuCCmEH9crE7WO7R60QllskxRdDGRERUQTIsowKqwsOpyesfSwvVAg7eWQWrhrYISphSQCg1aiQatRCq4n+DgBUN4YyIiKiMHl98tmWfm9Y7+OUPHj9k534tVYhrEGnxj3X5aJXl+gUwoqC/+iYKYVHx+KNoYyIiCgM1dsmSZ7wWvovVAg7e0o+Mpsbw51mnbRqEalGLXRaxgEl4O8CERFRiCS3FxWW8LdNKjpcjkUrCmGtVQjbq0tzzLi2V1Q2+RYFIEWvgTlFy1Z+BWEoIyIiCoFT8qAiAi39PxSexLKvDsJX632uHtgBk0ZmRSUw8eiYcvF3hIiIqJEiUQrr9frwVWEVth8qCYypVSJuH9sdg6NQCFt9dMyUooWKR8cUiaGMiIioESw2CVZHeKWwVruEhSt3Yt9RR2AszaTFrMn56Nw2LQKzrCEA0PDoWELg7w4REVEDyLKMSpsEe5gdZEdLLViwrABnKmsKYTu2ScXsKBTCsncssTCUERER1cPn83eQOV3hdZD9uqcUr32yEy53TXXGpd1b4s7xvSLaD+Zv5Vcj1ahlK38CYSgjIiK6CK/XhwqLC0536B1kPlnG2u8P4uPvahXCAhjSw4Qbrs6KaCBTiQJMKRqYDNqIvSfFBkMZERHRBUSig8wlefHa6h34dU9NIaxep8K0MdnQ+coidlqxupU/zaSN2jZMFF0MZURERHWIRAfZmUp/IezR0ppC2FYZBsy5oTfSUkTs318WiakG7qxMNWp57VgCYygjIiI6h0vyoDzMDrK9R/yFsBZ7TSFsz87NMOO6XBj1Gjgcjou8uuE0KhGpJi30vLMy4fF3kIiIqBaHy41KqxRWIPt26zG889meoELYqwZ2wKSRXaESI3PhvSgAhrOt/OwdaxoYyoiIiM4Kt4PM6/Xh/S/24utfjgbG1CoBt13TA0PyIlcIq1GJMBs1MOgivwUTxQ9DGRERJT2fT0alzQWHyxNyB5nVLuGVldux53B5YCzV6C+E7dIuMoWwPDrWtDGUERFRUnN7fKi0uoK6wxrr2CkrFny4DadrF8JmmjFrSj4yzPqw58hW/uTA31kiIkpaLsmDCqsLHm/o149tLTqFpZ/sgEuqCXUDerbGtLE9ItI/phIFGPUamFLYyt/UMZQREVFSCndTcVmWsfaHYqz69kBgTABw/ciuGD2oY9gBqqaVX8PesSTBUEZERElFlmVU2STYwtjD0iV58fqanfhld2lgTK9VYca1ucjLahH2HFWiAHOKBka28icVhjIiIkoaXp+MyjD3sCyrdGLB8gIcKbEExlpmGDBnSm+0aWEMa37VR8fSTFqoVdyzMtkwlBERUVKIxJZJ+45U4OUVBUGFsD06NcM91/sLYcPBo2PEUEZERE1eJC7o/27bMbzz6Z6gUtlR/S/BlFFZYRXC+o+OqZBm0kGj5tGxZMZQRkRETZrd6UaVLfSGfq/Phw+/2IsNW4ILYaeO6Y7L8tuGNTe1WgWjXoNmqXqI7B1LegxlRETUZFVZJdicoTf02xxuvLKyELsPRb4QViUKSDPqkGrUMpARAIYyIiJqgiLR0H/8lBXzlxXgdEXNxuEdMs2YPTkfGanhFcJqVCJMqXpUnAq9sJaaHoYyIiJqUjxeHyos4TX0F+w9hSUfR74QNrBNkkEDl8sJny/0mw6o6WEoIyKiJsMleVBpleD2hhZ2ZFnGuh8PYdU3+wOVGQKA60Z0xZjBoRfCcpskagh+ZRARUZNgc7phCeOCfsntxRtrdmLzruBC2LuvzUV+GIWw3CaJGoqhjIiIElokGvrLqpxYuKwAh2sXwqYbMPuGfLRtYQp5blq1iDSTLiJ7YFLTx1BGREQJy+uT/dePSaE39O8/WoGFy4MLYbt3zMC91+fBaAitEFYQgBS9BqkpvLOSGo6hjIiIEpLb40WFxRVWQ//3247jP5/uDjrlecWl7XHDld1CLoRViQJSjVqkhNnwT8mHoYyIiBKO82xDvzfEhn6vz4dlX+7Dl5uPBMZUor8Qdmjv0AphBfib+VPZzE8hYigjIqKEYrVLsNjd8IV4AZnN4cYrH23H7uKywJg5RYP7Jucjq316SO8pVp+uNGp5MT+FjKGMiIgSgizLqLRJsIdxQf/x01Ys+LAAp2oVwl7S2l8I2ywttEJYnq6kSGEoIyIixYvEBf0Fe0/h1Y93wFmrELZ/j1a4Y1zPkO+O1KpFpJt10Kh5dyWFj6GMiIgULdyGflmW8enGQ/jo6/1Bge664V1xzZDQCmEFATDo1Egz6nh3JUUMQxkRESmW5PbfYRlqQ7/k9uLNtbvw886SwJhOq8LdE3uhd7eWIb2nShRgNmph5OlKijCGMiIiUiSHy41Ka+gN/eVVTixYXoDDJ2sKYVukGzBnSj7atgytEFanUSHNpOXpSooKhjIiIlIUn0+Gxe6/oD/EPIYDxyqxcHkBqmxSYCznbCGsKYRCWEEAjHoNzCyDpShiKCMiIsVwe7yotEohXz8GAD8U+AthPd5zCmFHdYNK1fj+MN5dSbHCUEZERHGnVqvhcHlR5XCGfLrS6/Nh+YZ9+OLn4ELYW8fk4PLe7UJ6T95dSbHEUEZERHEly4BDAiqtLuj0oXWF2ZxuLPloO3YePKcQdlI+si5Jb/T7Vd9dmWrUQcXTlRQjDGVERBQ3bo8X5RYXLHYnWoTYCHvitA0Llm1DaXmtQthWJsye0jukQlhREGA2amAyaEOaD1GoGMqIiCgu7E43qmwSnJIHvhBPWRbuP40lq7bD6aq5Bq1f91a4c1xP6LSNP+WoUYlIM2mh0/LHI8Uev+qIiCimZFlGlU2CLYztkmRZxmebDmHlV8GFsNcO74KxQzo1uhC2ejPxNLMO6hBuBiCKBIYyIiKKmeq7KyW3N+TtkiS3F2+t242fdpwMjOm0Kkyf0At9shtfCCuc3Uw8jZuJU5wxlBERUUzYHBIsdnfId1cCQLnFiYXLCnCodiFsmh6zp/RGu1aNL4QVRQGpKRoYef0YKQBDGRERRZXX60Pl2WvHQj1dCVygELZDBu6dFFohLK8fI6XhVyIREUWNS/Kg0iqFvHdltR8LT+DtdbuCCmFH9GuPm65sfCEsrx8jpWIoIyKiqAh370rAXwi7YsN+fP7z4cCYKAq4dXQOhvVpfCEsrx8jJWMoIyKiiLM5JFTZ3PCFcb6y7kJYNX5/ay+0admi0e8nCgJSjbx+jJSLoYyIiCLKYpNgcUhhXT928owN8z8MLoRNNZzEpZ3fRXaHd2BxXOTFdVCrBKSbdLx+jBSNX51ERBQRsiyj0ibBHkb/GABs338ai88phM1M34neHVdCrXI3+v10GhXSTDpo1Lx+jJSNoYyIiMLm8fpQaXXBJYXeP1ZdCLtiw76g95hyRTs4Kh5DYy8BEwRAr1Uj3aSDyP0rKQEwlBERUVickgeVVlfQnZGN5fHKePuzfdiy+1RgTKcVMXtSV+R1FbB8Xa3Pc5XDbPD/tyjqUWk7/0eZKAAmgxZmI68fo8TBUEZERCGRZRlWuxtWR3gX9FdYXVj+QxlKKz2BMYO2HP27vIu9e0uxd2/w85evuy3w31OvWwMguDRWJQpINWqRom98dxlRPDGUERFRo3m9PlRYJbjc4RXCHjxeiQXLClBlqwlkPTqZ0T71KWjVjbyaH2cLYc066DSN34ycKN4YyoiIqFEcLjeqbG54wiyE3bj9BN5auzvofYb3bYcZE9vi/U8aF8iqC2HTzbpGl8kSKQVDGRERNYjPJ6PKLsHhdCOMPlj4fDJWfL0P6zfVKoQVgMkju+CqQZ2h1XjOnpb0c7rKg05ZTr7mbeh1Gf7XiXoWwlKTwVBGRET1ckkeVNkkSJ7wjo7ZnW4sWbUDOw6cCYwZ9WqM7mvG0PxMADh74X7NdWLVF/VX0+syYHH4H68uhDWxEJaaAIYyIiK6KItNCvtifgAoKbNj/ofbUFJmD4y1a2nC9PHZqDh9rNHvp1aJSOeG4tSE8CuZiIjq5PXJqLS64JTCu5gfAHYcOIPFH22Hw1VzQX/f7Ja4c0JPyF43Kk5f+LWiqA86nSmKeui5oTg1QQxlRER0HsntRaXVFfbpSlmW8cXPR7Bsw96gYDd+aGeMv7wzREGAw3Hxlv7apzOrrx9rlsbrx6jpYSgjIqIgNqcbFpsEbzhX8wNwe7x4e90ebNx+IjCm1Yi4a3wv9OveqtHvJwoCzLx+jJowhjIiIgJQc3dluHtXAkCl1YWFywtw8HhVYKxZqh5zpuSjfWtzo99PJQpIM2lh0LEQlpouhjIiIoLb40WlVYLL7a3/yfUoPlGFBcsKUGl1Bcay2qdj5qQ8pIaw7ZFGJSLdrIOWhbDUxDGUERElOYfLjUpr+KcrAWDTjpN4c82uoELYYX3a4uarc0K6KL+6EJYX9FMyYCgjIkpSsiyjyuY/XRluHvP5ZKz8ej8+23QoMCYKAm66qhtG9Gvf6IvyBQA6rdrf0C/ygn5KDgxlRERJyO3xocrqgsvtRbjHxxxOD5Z8vB3b99cqhDVoMPP6XOR0bNbo9xMEwKjXIJUN/ZRkGMqIiJKMU/KgwuqC1xv+6cqSMjsWLNuGk2dqCmHbtjRizpTeaJFuuMgr61bd0G/kHZaUhBRxkn7lypUYN24c8vLyMH78eKxduzbw2NGjR3HfffehX79+uPzyy/Hss8/C6w3/QlQiomQjyzIsNgnlVc6IBLKdB8/gn6//HBTIendriYdu7x9SINOoVchI1TGQUdKK+5Gyjz76CA8//DD+9Kc/YdiwYVi9ejV+//vfIzMzE7m5uZgxYwY6deqEd999F4cPH8bDDz8MURTxwAMPxHvqREQJw+v1ocIqweUOv53/QoWw4y7rhAnDukAM4ZRjSooezcw66LllEiWxuH71y7KM5557DnfccQduu+02AMDs2bOxefNm/PTTTzh27BiOHz+O999/H2lpacjOzsaZM2fw1FNPYdasWdBq+bcpIqL6uCQPKq0S3N7w2vkB/7Vo//l0N34sDC6EvXN8T1zavXVI76nXqmHWi1CpeP0YJbe4hrKDBw/i2LFjmDhxYtD4kiVLAAB//etf0atXL6SlpQUeGzx4MKxWK3bt2oXevXvHdL5ERIlElmVY7e6IbCYOXLgQdvaUfFwSQiEsAOi1KujVIkqOXXyrJaJkEPdQBgB2ux0zZszAzp070b59e8yePRujRo3CyZMnkZmZGfSaVq38W3OcOHEi5FAmyzLsdnv9T1QQh8MR9O9kkazrBpJ37Vx3ZNbt9fmvH3NEYDNxADhcYsWrn+xGpVUKjHVua8b08Tkwp6gbPe/qOyz1ahFOJ3/Pk0myrVuW5QbfRRzXUGa1WgEAc+fOxf33348//OEP+PTTTzFnzhwsXboUTqcTqampQa/R6XQAAJfLdd77NZTb7cauXbtCn3gcFRcXx3sKcZGs6waSd+1cd2jUajU8sgCrwwuXS4IcgURWdMyBL7ZVofbZz56XGDAiz4DSE0dQ2sj3U6lUMKfoYNXIOObxBMb5e55ckmndDb3cKq6hTKPx72E2Y8YMTJo0CQDQo0cP7Ny5E0uXLoVer4ckSUGvqQ5jKSkpYX1uVlZWyK+PB4fDgeLiYnTq1AkGQ+PvakpUybpuIHnXznWHvm6fT4bF7obD5UFaBMKYzydj7Y+H8fmvNacrRQG4fkRnXJ6fGVKHmCgKMKdoYdTX/Pjh7znX3ZTt27evwc+Nayhr3dp/UWh2dnbQeFZWFr766isMHDgQRUVFQY+VlpYGvTYUgiCEFeriyWAwJOzcw5Gs6waSd+1cd+PYnG5YnRJ8gho6ffjf2h0uD177ZAcK958OjBn1atx7fR66d2p8ISxwtoPMpIVRX/em4vw9Ty7Jsu7G/OUlrj1lvXr1gtFoxLZt24LGi4qK0KFDBwwYMAA7d+4MnOYEgI0bN8JoNKJ79+6xni4RkeJ4vD6UVTlRGaEyWAAoLbfjn2/8HBTI2rQwYt6dA0IOZCqVgGapugsGMiKK85EyvV6Pe+65By+99BJat26N/Px8rF69Gt9//z1ee+019OnTB88++ywefPBB/OEPf8DRo0fx73//G3fffTfrMIgo6dmdblTZpYiFMQDYVVyGV1YWwu6sudYrP6sF7p7YC3pdaD8ytGoR6WYdNGpVpKZJ1CTFvaVvzpw5MBgMeOaZZ1BSUoKuXbvihRdewKBBgwAAixcvxmOPPYabbroJaWlpmDp1KubMmRPnWRMRxY/XJ6PK5oLDFZk7KwH/HWIbthzFh1/sDarPCKcQFgD0GpV/U3GVIjaQIVK0uIcyAJg+fTqmT59e52MdO3bEq6++GuMZEREpk1PyoCpCRbDV3B4f3vlsN34oqCmE1aj9hbD9e4R2/a4AQK9TI92kgyiyFJaoIRQRyoiI6OL8d1ZKsDvd8EXubCUqrS68vKIQB45VBsYyUnWYPTkfHTJTL/LKC6vuIEs1akO6Q5MoWTGUEREpnEvyoMomQfJE7ugYABw+WYUFywpQbqnpfezaPg33TcpDqlEX0nuKAmBO0cKUwut+iRqLoYyISKGidXQMADbvKsHrq3fCXSvoDc1vi1tG50CjDu36L5UoINWoRQrvsCQKCUMZEZECRevomE+WseqbA1j3Y3FgTBQE3HBlN1xxafuQTzeqRAEZZh10Wv5YIQoV//QQESmEIAjw+WSUW5xwujwRPzrmcHmw9OMdKNhX0z+WoldjZhiFsIC/gyzDxEBGFC7+CSIiUgifoMaZKhfUmshfj3W6woGXPtyGE6dtgbE2LYyYPSUfrTLC2LZO5e8g02rYQUYULoYyIqI4k9xelFtcqLC4kN7cC3WEL8nae6QcC5cXwOYILoSdPrEXDCEWwgLVpbD6kK9BI6JgDGVERHHi88mw2t2wOf2biHu93oi+vyzL+PrXo3j/8yL4al2ads2QTrh2eOiFsIA/kGWk6qFmKSxRxDCUERHFQfWF/G6PDxG+dAyAvxD2vc/34Lutx2sGRTdGXyXipss6QHKGFsgEADqt2t/Sz1JYoohiKCMiiiHv2ZoLRxRqLqpV2VxYtKIQ+47WFMJCWwVV7rv40nUC2adl9GsxCpKzcedJWQpLFF0MZUREMRKtmovaDp+0YOHybSirqimEReoRqHq9B0Hrv8h/YeHfMSsPjQpmoiDAbNTAZGApLFG08GIAIqIo8/lkVFpdKKtyRjWQbd5Vgv97a3NQIBMyf4Wq9+uBQFZtYeHfcVo6DL22/rsmVaKAdLOWgYwoynikjIgoilxuL6qsrqhdOwb4C2E//vYA1v5QHBgTBECd9Tl8bb5HXWcax3W6Ba30HeC0X/zmAlZeEMUOQxkRUZRY7RIsdjd8crTiGOB0ebD0kx3YtrdWIaxOjfum9EL7zl3xlx+3QPI6g14zrtMtmNz1XjjtFz91ycoLotjinzQiogjz+mSUVTlRZZeiGshOVTjw1JubgwJZZvMUzL1zAHI6tEC62BF/G7IIWpU+8HhDA5leo0KzVAYyoljinzYioghyub0oq3TA4fIginkMu4vL8I/XfsLxWg39eV2bY+4dA9C6mb+h3+0WgoJZQwKZAMCgUyMjVQ8VO8iIYoqnL4mIIsTqOHu6MlpdFzhbCPvLUbz/+d6go3BjBnfEdcO7QjynO8ztFpCu6YjnRn4IDQwXD2QCkKLXII2VF0RxwVBGRBQmt8eHKpsElzu6R8c8Xh/e/WwPvttWUwirUYuYNrYHBvbKvPD83AI0cjqcF7nzUxAAk16LVBPvsCSKF4YyIqIw2M4eHfNG8egYAFTZJCxaURBUCJtu1mH25Hx0bJNa7+vdFwlk7CAjUgaGMiKiEMTq6BgAHCmxYMGyApRV1dxF2bltKmZNzkeaSRfWe6tEAWkmLQy6CO+CTkSNxlBGRNRIsTo6BgBbdpfg9dU7IblrjnQNyWuDqWO6h31npEolIMOkg07LHwVESsA/iUREDeQ/OuaCy+2N+tExnyxj9XcHsfr7g4ExQQCmXNENVw64JOwL8VkKS6Q8DGVERA0Qy6NjTsmD1z7Zia1FpwJjKTo17rk+Fz07Nw/7/VkKS6RMYYWyAwcO4OjRo7BarcjIyEDbtm3RsWPHSM2NiCjuYnl0DADOVDqxdHUBjp2yBsYym6dg9pTegf6xcOg1KqSbdewgI1KgRoey06dPY+nSpfjkk09QWloKudZ3KUEQ0L59e4wdOxZ33HEHWrRoEdHJEhHFktUhwRqjo2MAcPS0hKWfF8Dm9ATGcrs2x4yJuTDowzuxIQDQadXIMOvO6zIjImVo8J9yr9eLl156CYsXL0bbtm0xadIk5OXloV27dkhJSUFlZSVKSkqwZcsWfPnll3jjjTdw55134v7774dGw7t6iChxxProGAB8X3ASqzaVo3b+Gz2oI64fcX4hbGMJAAx6NdJNOpbCEilYg0PZlClT0L59e/znP/9Bbm5unc/Jy8vDVVddhblz52Lz5s1YvHgxbrzxRqxcuTJS8yUiiqpYHx3zeH14b30Rvt16LDCmVomYNrY7BuW2Cfv9BQEw6jVIZUs/keI1OJTNmzcPgwcPbvAb9+/fH/3798ePP/4Y0sSIiGLJ7fGe7R2L3dExi13Cy8sLse9oRWAszajF7Bt6o1MDCmHrw5Z+osTS4FDWmEBW25AhQ0J6HRFRLMiyDJvDDasjdkfHAOBoiQXzzymEbZ2uxuwp+chsGX4gEwXAZNDCbGQgI0oUIV05erHTkYIgwGg0okOHDsjOzg51XkREUSe5/UfHJLcXsYtjwC+7S/Ha6h1BhbD9e7RE/07+dv1wiQKQatTCyG2TiBJKSKHs4Ycfhs/n/2Zy7t2X1WOCIGDQoEFYsGABDAZDBKZKRBQZPl/N0TFfrM5V4uKFsJfltsCBAwfC/gxR8Ae7FD1vsCJKNCEV1SxevBgGgwG/+93v8OWXX6KgoAAbNmzA3LlzYTAY8MQTT2DBggUoLi7G888/H+k5ExGFzCV5cKbSAYtdimkgc0oevLKiMCiQGXRq3H9jH1w1sENELsJXiQIyUnUMZEQJKqQjZf/85z9x7733YubMmYGxNm3a4K677oLH48Fbb72F5cuX47/+67/w0ksvYe7cuRGbMBFRKLw+GVa7BLvTjRheOgYAOF3hwIJlwYWwrZulYM4NkSmEBQC1SkC6WQ8dt00iSlghHSk7cOAA8vPz63ysR48e2LdvHwCgY8eOOH36dOizIyKKAIfLjTMVjrOnK2P72UWHy/GP138OCmS9ujTH3Dv6RyyQadUimqUykBElupCOlF1yySX49NNPMXTo0PMeW79+Pdq08XfrnDx5Es2aNQtvhkSUcBweCV7ZB5NGH9d5eL0+VNklOFyemNVc1PbNr0fx7voi+GolwasHdcCkEVkRa9XXqkVkpOqh5rZJRAkvpFB2zz334I9//CPOnDmDMWPGoHnz5jh9+jQ+//xzfP7553j88cdx8OBBPPvssxg+fHik50xEClclOeGWvXENZXanG1V2CV5v7NOY1+vDe58X4Ztfgwthbx/bHYMjUAhbjftYEjUtIYWySZMmQRAEPP/88/jiiy8C4x06dMD//d//YcKECVi9ejW6du2K//7v/47YZIlI+SxuJ17Y+SWqJAf+t//1SNXG9u5rj9eHKpsEpxSfo2MWu4RFKwqx90hFYCzNpMWsyfno3DYtYp+j5z6WRE1OyDvcXn/99bj++utx+PBhlJWVITMzE5mZmYHHx48fj/Hjx0dkkkSUOE47rfj06A4AwFFbBXrGKJTJsgyb0x3TLZLOdbTUggXLCnCmsqYQtmObVMyenI90sy4in8F9LImarpBDGQBUVlZCq9WiVatW8Pl8OH78eOCxtm3bhj05IkoclZIdB6pO47FfPg6M/c+mD/BIvwnISctEui4yF7XXxe3xosoqweWJ3RZJ5/p1Tyle+2QnXG5vYGxQr0zcdk13aCN0Ab4gACl6DdK4jyVRkxRSKDt06BDmzp2Lbdu2XfA5u3btCnlSRJQ4HB43KiU7/rLlI2w9cyTosVKnBff/8A56prfF3/tfjwxdCoyayBwxAvxHx6z22JfA1uaTZaz5/iA++a5WISyASVdk4eoI9Y8BNRuLp5ki9/+PiJQlpFD2t7/9DcXFxbj//vuRmZkJUeRFpkTJyqDWwOnV4PasQThhr0CJwxL0eHOdEVOzBsKo0UU0kLncXljisEVS0BwkL15bvQO/7jkVGNPrVJhxbS7yuraI2OcIAmDmPpZETV5Ioeznn3/G3//+d0yYMCHS8yGiBJShS8GwzGzkN2uPl3d9g2XFvwAAxrTvhd/nXY00jQGqCP3lzefzHx2zOeN3dAwAzlT6C2GPltb0j7XKMGDODb2R2dwYsc8RBcBs1MLEfSyJmryQQpnJZEJaWuTuIiKixCcKAjJ0RszsPhzrj+2E5PPiwdwr0UwXuYDikjyosklwe3xxOzoGAHuPlOPl5YWwOtyBsZ6dm2HGdbkwRnCLI1EQkGrSRvQ9iUi5Qgpl1113Hd5++21cfvnlvNiUiIKkavW4v9coVEkOpGoic+el1+uDxeGGIw5bJJ3r263H8M5ne4IKYa8a0AGTrugasaOBgH8fyzSTFgYdAxlRsggplBkMBmzZsgVXX3018vLyoNcHF0QKgoAnnngiIhMkosSiFlUY2SYbMgCtKqwbvAHgbM2FBE8cSmBr83p9eP+Lvfj6l6OBMbVKwG3X9MCQvMgVwgL+lv5Uk47bJhElmZC+Y65YsQJmsxk+n6/OOzB59IwouaVrU+AN43ovURTh8fpwptIBlzt+NRfVrHYJr6zcjj2HywNjqUZ/IWyXdpG7lEOlUsGo16BZmgEqlsISJZ2QQtmXX34Z6XkQURMiCALUYfzlzONT4UylE1pdfPfOBIBjpVYsWLYNp2sVwnbINGP25HxkpEZufipRQLpJhzSTloGMKEmFdW7B5/OhqKgIpaWl6NevHzweD9LT0yM0NSJKNm6PF+UWFyptTmTE++IxAFuLTmHpJzvgkmoKYQf0bI1pY3tErBAW8O+LaUrVo+KUt/4nE1GTFXIo++ijj/D000+jtLQUgiDgww8/xAsvvACNRoOnn34aWi1v3yaihrM6JFjtbjhcHni9vrjORZZlrP2hGKu+PRAYEwBcPzILowdFrhAW8F8/lm7WwS254PPFd91EFF8h3Sq0Zs0azJ07F4MHD8YzzzwD+ewFH1dffTW+/vprzJ8/P6KTJKKmS3J7cabSgSqbFLc9K2tzSV688tH2oECm16ow54beGDO4Y0QDmU6jQkaqHho1L+gnohCPlC1cuBC33HIL/vrXv8LrrTncPmXKFJSVleH999/Hgw8+GKk5ElET5PX6YHW4YXd64loCW1tZpRMLlm3DkVqFsC0zDJgzpTfatIhc3xrgD3rpZj2vHyOigJCOlB08eBBXX311nY/17t0bJSUlYU2KiJouWZZhdUg4VemI656V59p3pAJPvv5TUCDr2bUZ/njngIgGMgGAQadGBgMZEZ0jpFDWvHlz7N+/v87H9u/fj+bNm4c1KSJqmlySB6crHKiySvDGuXestu+2HcMz7/wCi72moT+3d3NkXmZAWmrk9usUBMCgVyPDrIPIQEZE5wgplI0bNw7PP/881q1bB0mSAPhvgd++fTvmz5+Pa665JqKTJKLE5vXJqLC6UFblhBTnLZJq83p9eG/9Hry1dnfgeja1SsD0Sb2wrUUJlhfvhayWEYn4JAiAUa9BhlnPLkciqlNI15Q9+OCDKCoqwoMPPgjx7LYi06ZNg91uR//+/fHb3/42opMkosRld7phsbvhifMdleeyOtx4ZWUh9hwKLoS9/6beOGV04Eypv5fs3f27cWvHHnBLoc9fFACTQQuzkXelE9GFhRTKtFotFi9ejO+//x4bN25ERUUFzGYzBg4ciBEjRvBvgUQEt8eHKpsEl9sT90b+cx0/ZcX8ZQU4XeEIjFUXwrZuk4K/ffVjYPzDA0W4pWt3CBJCOsInCgJSjRoYDQxkRHRxDQ5lbrcbGk3wxrhDhw7F0KFDG/06ImraqjvHlFBxca5te0/h1Y/rLoQ1pWjwY8kJnHHWtPdLPl/IR8tEwb+peIqe3wOJqH4NvqZs4sSJjd5ead26dZgwYUKjJ0VEicntUVbnWG3VhbALlxUEApkA4PoRXXH3xF7+hn4tsHDX1vNe++GBokZfW6YSBWSk6hjIiKjBGnyk7KmnnsK8efPw3HPPYcKECRg9ejQ6dux43vP27t2Lr7/+Gh988AF8Ph+eeuqpiE6YiJRHlmXYHG5YHco8Oia5vXhjzU5s3lUaGNNrVbh7Yi/kd2sJANBqxPOOkgVe38ijZSqVgAyTDjptWDvZEVGSafB3jPz8fKxcuRJvv/02XnvtNfz73/9Gamoq2rVrB4PBgKqqKpSUlMBisaBZs2a45557MHXqVOh0kbudnIiUR3J7YbFJcHm8irt2DADKqpxYuKwAh0ssgbGW6QbMviEfbVuYap54gaNk1Rp6bZlG5d82KZJ7YxJRcmjUX+O0Wi2mT5+O22+/HRs3bsSmTZtw5MgRWK1WZGZm4oorrsDQoUPRv39/qFT8hkTUlMmyDKvdragC2HPtP1qBhcsLgvrHunfMwL3X58FoqDmtqBIFaEQRr424eJ2PWhQhqmVInrqPlvn3sdRDow6pbYiIklxIx9Y1Gg2GDRuGYcOGRXo+RATA5vbBLuvQqksvVHrVqLJ6z3uOTi1ArxJg0MT+bmeX5EGVTYJbQZ1j5/p+23H859PdQadTr7i0PW64shtUYnBo8vpk2Cyeet/TjfN/H6rpNCpkmHVQqRjIiCg0EbngYdeuXSguLkanTp3Qo0ePSLwlUVLTiAJW73dgccH51zdV+/eVZgxsE9uaBZ9PhsUuKWq/ynN5fT58+MVebNhyNDCmEgVMHdMdQ3u3jcpnch9LIoqERoWyH3/8Ee+99x4EQcDUqVMxYMAAPPTQQ/j4448hyzIEQcDll1+O559/HgaDIVpzJmrytCoBE7P0eHunE446DuB0SlMhOyO2F5FXHx270Kk7JbA53Hjlo+3YXVwWGDOnaHDf5HxktU+P+OcJAPQ6NdJN3DaJiMLX4O/q69evxwMPPIC2bdvCbDZj+vTpuOmmm7Bu3Tr89re/RW5uLrZt24YFCxZg/vz5+O///u9ozpuoydMJHtzaQ4dXC13nPfZA/xSk62Nzmszrk2G1S7A73VDgjZUBx09bseDDApyqXQjb2oxZU/LRLFUf8c8TBP/G4ukmHQuziSgiGhzKFi9ejAkTJuD//u//AABvvPEGnnzySfzmN7/BrFmzAACXX345BEHAqlWrGMqIwiTKXozvrME7u1xBR8tieZTM4XLDYnPDrbAtks5VcLYQ1lmrELZ/j1a4Y1zPqNwFWb2PZZqJd5cTUeQ0+K/a+/btCyqCvfbaayHLMgYMGBD0vEGDBuH48eORmyFRErOWleC2nsFHeWJxlMzj9aHc4kS5xaXoQCbLMtb9WIwFywoCgUwAcN3wrphxbW7UApnZoGUgI6KIa/Bft202G9LS0gK/NplMQf8OvKFaDbfbDSIKX2XZaUzs3i5wbVm0j5LJshzYQFyJJbC1+Qthd2HzrpLAmE6rwoxahbCRJgqAOUULUwr3sSSiyGvUd3ex1m3kvIaCKPpkWYZO8OD2Xga8ss0R1aNkLskDq92t2BLY2uoqhG2RbsCcKflo29J0kVeGjhuLE1G0hf1XboYzougSZS8mZunx/VF3VI6Sebw+WOwSnC6Poi/kr7b/aAVeXlGIKpsUGMs5WwhrMkRnn0lREJBu1sKg4z6WRBQ9jfoO/5vf/AZabfDfEmfNmgWNpuYblSRJ576MiEJUfXTapBXw7FWpSIlgUazP59+v0uZU/qnKaj8U+AthPd5zCmFHdYtaaatKFJBmYiAjouhrcCibNGlSNOdBlPR8Phk+H6BW+4OXAB0yW7cH4O8tE4XIBafquyo9XuU28tfm9fmwfMM+fPHzkcCYShRw65gcXN67XdQ+VyUKyDBzY3Eiio0Gf6d58sknozkPoqTnsss4ediLdl3U8HhkHC7ywdii5qYZdQTKSb1eH6oS6FQlANicbiz5aDt2HjynEHZSPrIuSY/a56pVAtLNeui4sTgRxUjE//rncrlw8OBBdO/ePdJvTdS0CcAPayW0bOdB5RkfuvRSQ5dxfnFsqBwuN6psUtCpP6U7cdqG+cu24VR5TSHsJa1MmD2lN5qlRb4QtppGJSIjVQeNmoGMiGKnwRdhXH755di1a1fQ2NKlS1FWVhY0tnv3bp7qJGokp12GpcIflk4d80FyAlVnZDTPaAshzL871e4cS6RAtvNgOf755s9Bgaxf91b4w+39oxrItGoRzdL0DGREFHMN/m5/+vTpoP4xr9eLp556CgMHDkSzZs2iMjmipk4+2z1R8IOEvduCN7k8XuzFmf8Ao6boodXKUKkbf/rS5pBgcbjhTaAwJssytuyzYePukqDr3a4d1gVjL+sU1Tu+dRoVMsy6qN00QER0MWH9FVxWepkRkcJVB4zeQzXolq/GmjedgcfaZ6nQd4QAjc4Dlbpx7fFujxdVNgkut/I7x2qT3F68/elebNljDYzpNCpMn9gLfbKjUwhbTa9VId2sh4obixNRnPCWIiIF0BlE+M5uZ9S2swrlpT4YUwWcKT+O1q1bN/h9ZNlfc2F1JE7NRbVyi78Q9tDJWoWwaXrMntIb7VpFpxAW8G/LpNOqkWHWQWQgI6I4YigjUgoBGDlJh5btVPB6ZBzd74Va1fAjZC7JA4vdDcntTYiai9oOHKvEwuUFQYWwWe1TMWty76huaSQAMOjVSDfpWIRNRHHHUEakEDq9gDYdVRBVAqAT0L6rDxWV9ReWen0yrHYJdqcHvkQ6V3nWj4Un8Pa6XUE3IeR1NODOiT2jG8gEIEWvQZpRy0BGRIrAbZaIFEJUBf9ZkgUJJ04eQVr6hetlqktg3WdPfSYSr8+HFRv24/OfDwfGRFHAlJGd0TrFFtWL7QUBMOo1SDM17lo9IqJo4jZLRAp2oZtpqverdLg8CXUhf7W6CmFNBg3um5yH9i302L9/f9Q+WxAAs0ELs5EbixORsjQ4lF1//fU8KkakADanGxa7lFA1F7WdPGPD/A+3obRW/1j7VibMnpKP5mkGOByOi7w6PKIAmI1amAwMZESkPA0OZf/4xz+iOQ8iqofb40OVzZVwNRe1Fe4/jSWrtsPp8gbG+uW0wp3je0KnjW5ZqygISDVpYdRzY3EiUqYGX7Txxz/+EUeOHKn/iUQUcVa7hDOVDjilxAxksizjs02HMP+DbUGBbOKwLrj3+tyYBLJ0MwMZESlbg0PZihUrUF5eHs25ENE5RLUWZVUuVNmlhOsdqya5vXjtk51YvmFfoKpDp1Hhvkl5GD+0c9Qvi1CJAjJSdTDoGMiISNlYiUGkQD6fDJvTgwqrBHOGBwZVYgaKcosTC5cX4tCJqsBY8zQ95kS5ELaaShSQYdZBp+W3OiJSPn6nIlIYl+RBlU1Clc0FSXLX/wKFOnjcXwhbaa25I7vbJem4b1JeVPvHqqlVAtLNeug03FiciBJDo0LZX//6V5hM9f/tVhAEvP766yFPiigZeb0+VNklOF0e+GQk5LVj1TZtP4E31+6Gp1Z/2vC+7XDzVdkx2exboxKRkaqDRs1ARkSJo9FHyhqyCTk3KidqHJtDgsXhTtiai2o+n4wVX+3D+p+CC2FvuTobw/u2j8kctGoR6WY9NOrohz8iokhq9JGy/Pz8aM2FKOlIbi8sNgkuT2LeVVmb3enGklU7sOPAmcCY0aDBfZPykN0hIyZz0GlUyDDrYnI0jogo0nhNGVEcJPp+lec6ecaGBcsKUFJmD4y1a+kvhG2RbojJHPRaFdLNeqhEllwTUWJiKCOKMZvTDavdHXS9VSLbceAMFn+0HQ6XJzDWN7sl7pzQE/oY3PUoANDr1Eg36SAykBFRAmvwd8wBAwbAaDRGcy5ETVpTOlUJ+K8d/fynw1j+1b6g9Uy4vDPGDe0MMQbbsgkCYDgbyLgNHBElugaHsvbt22Px4sUNeq4gCHjiiSca9NySkhIMHz78vPEnn3wSkydPxp///Gd88MEHQY+1a9cOX375ZYPenyjemtqpSgBwe7x4e91ubNx+MjCm1Yi4a3wv9OveKiZzEATAqNcgzaSLyecREUVbg0PZihUrIAgCWrduDVG8+EW0jfkb6+7du6HT6fD5558Hvc5sNgMA9uzZg1mzZuH2228PPKZS8TZ3SgwOlxsWmxvuJnKqEgAqLC68vKIAB4/XFMI2S9VjzpR8tG9tjskcRAEwGbQwG7mxOBE1HQ0OZWPHjsVXX30FSZJwzTXXYPz48bj00kvDnkBRURE6deqEVq3O/9u1LMvYt28fZs6ciZYtW4b9WUSx4vZ4UWVzw+X2NIlTldWKT1RhwbICVFpdgbFul6Rj5qQ8mGNQCAtwY3EiaroaHMqeeeYZOBwObNiwAWvWrMH06dPRokULjBs3DuPHj0ePHj1CmsCePXvQtWvXOh87fPgw7HY7unTpEtJ7E8WazyfD5nDD5nQn7F6VF3KhQtibrsqGOkYVFCpRQJpJy30siahJatStUQaDAePGjcO4ceNgtVqxfv16rFmzBq+99hrat2+PCRMmYPz48ejcuXOD37OoqAgZGRm47bbbcPDgQXTs2BGzZ8/G8OHDUVRUBAB488038c0330AURQwfPhy/+93vAqc3QyHLMux2e/1PVBCHwxH072SRSOt2uX2w2iVIEbqQ3+V0Bf07Xnw+GZ/8cAgbthwPjImigMkjOmNofibckguR3AzqQuvWqFUwmbSQvW7Y7Ym7/dSFJNLXeqQl69q57uRYtyzLDb6sS5AjUL9fUVGB9evXY+3atfjpp5+QnZ2N5cuX1/s6j8eDPn36ICsrC/PmzYPJZMLq1auxdOlSLF26FL/88gtefPFF/Nd//ReuuuoqHD58GE899RRat26N119/vd5r2+pSWFgISZLqfyJRA4iiCEFUwSH54HC64fZ44z2liHK5ffjs10ocKq35M6PXCLjm0nS0bxGb05WCABj0OpgNavi8bu4YQkQJR6vVIi8vr97nRaREyOVyweFwwOl0wuv14tixYw16nVqtxqZNm6BSqaDX6wEAubm52Lt3L5YsWYJFixZh6tSpyMjwt4FnZ2ejZcuWuOmmm1BYWIjevXuHNF+NRoOsrKyQXhsvDocDxcXF6NSpEwyG2JRxKoHS121zemBzuGGKwoX8LqcLR48dRft27aHTx/4Ow9JyB97/eDdKy2sCWZvmKZgxsTuap+mj9rnnrluvVSPNpG3ypbBK/1qPpmRdO9edHOvet29fg58bcigrKSnBunXrsG7dOmzbtg0pKSm46qqrcN9992Ho0KENfp+6us+6deuG7777DqIoBgJZ7ccA4OTJkyGHMkEQkJKSEtJr481gMCTs3MOhtHW7JA+qbBLcXhEarQ7RvMJJp9fF/BvXzoNnsHjldthrFcL2yW6Ju2JUCAsAeoMO6ammpGvpV9rXeiwl69q57qatMY0UjfruWjuIbd26FQaDAVdccQXuueceDBs2DFpt405n7N27FzfffDMWLFiAQYMGBca3b9+OrKwsPPTQQygtLcVrr70WeKywsBAAEu5IFzUNXq8PFocbDqcbTew6fgD+ax+++PkIlm3YG3Rd3LjLOmHCsC4xKYQFAJUoIkWnQYZZX29Lf6XLjbQ6Lvx3eb3w+GQYNdy4hIgSQ4O/W916663Ytm0bdDodRowYgeeeew4jRoyAThf6aZWuXbuiS5cuePzxx/HYY48hIyMD77//PrZu3Yply5bhyJEjmDNnDl588UVce+21OHjwIB5//HFMmDDhgndsEkWLzemGxS7B622CaQyA2+PDfz7djR8LTwTGtBoRd47viUu7t47ZPERBgNmoR5pJ26Btk0rsTqw9dBKXt22BFLW/w7C4yo6NJ8/g1uxLYOSNmkSUIBocyn799VeoVCpkZWWhrKwMb731Ft566606nysIAl5//fV631MURSxcuBBPP/00HnzwQVRVVaFnz55YunQpsrOzkZ2djWeffRaLFi3CK6+8ArPZjIkTJ+LBBx9s8AKJwuXvHJPgcjeN7ZHqUml1YeHy8wthZ0/JxyUxKoQF/Bf1m41aWDQN/x/dTK/F89v247lt+5Gu1cDi9sAryxjfMRMpGhZNE1HiaNTel9Xqu/upMXdHtWjRAk8++eQFHx87dizGjh3b4PcjipTqzjGrw91ktkeqy6ETVViwvAAVlpoKiqz2/kLY1Bg25ouCgFSjBoLsgcfjqf8FZ6lEATkZZuwut6BCqqnKGNupNXTc/YOIEkiDQ9mbb74ZzXkQKYpT8sBik+D2+NB04xjw046TeHPtLrg9NXePDuvTFjdfnROzQlgguBTWbm94IAOAVI0avZr5Q1ltHcxN/wJiImpaeAUsUS1erw9VdglOl6dJXshfzeeT8dE3+/HpxkOBMVEQcNNV3TCiX/tG3S0ULpVKQIZJB12Id3VWSR5sPV153vj+SitaGrhZORElDoYyorNsDgkWh7vJXshfzeHy4NVV21G4/0xgzKhXY+akPOR0bBbTuahVIjLMOmjDuPbLI8vYX2kDAHQ0p+CMU4LV7cGa4hL0aZEOvZqnMIkoMTCUUdJze3yosrma9IX81UrK7FiwbBtOnqnZZqxtSyNmT+mNlumx7ULTqkWkm/XQqMM7TVolufG3wT3Ru0Ua1IIAGUC5S8JPJ8vg8HgZyogoYTCUUdKS5ZoL+Zva5uF1qasQNj+rBe6e2At6XWy/FWjVIjJS9RG5bi0zRY+uaaagsWZ6LTqZU+CKwk4LRETRwlBGSUlye2GxSXBFaPNwJZNlGV9uPoIPv4xvIWw1vUaF9NTItfRfqBxWJYpICWF/XCKieGEoo6SSLDUX1dweH975dDd+qFUIq1H7C2H794hdISwACAB0WjUyzLoGlcISESUbhjJKGoH9Kpt4zUW1SqsLL68oxIFjNXcmZqTqMHtyb3TIjF0hLOAPZAa9GukmXUzv7CQiSiQMZdTkeX0yLHapye5XWZdDJ6uwcFkBymsVwnZtn4b7JuUh1RjbmghBAFL0GqQZtQxkREQXwVBGTZr97H6VniZec1HbzztP4o01wYWwQ/Pb4pbROWHf6dhYggAY9RqkmdgXRkRUH4YyapL8NRcSXG5Pk7+Qv5pPlrHqm/1Y92NwIeyNV3bDyEtjWwjr/2zAZNDCHMOtmoiIEhlDGTUpyVZzUc3h8mDpxztQsO90YCxFr8bM6/PQvVNsC2EBfyBLNWphNDCQERE1FEMZNRmS24sqmwQpCWouajtVbsf8ZQU4cdoWGGvTwog5U/LRMiP2+z+Kgn8fyxS9JuafTUSUyBjKKOH5fDKsdjdszuSouahtd3EZFq0shN0ZXAg7fWIvGGJcCAsEbyxORESNw1BGCS3Zai6qybKMr7YcxQdf7A0KomMv64SJcSiEBfyBLMMc+sbiRETJjt89KSElY81FNY/Xh3c+24Pvtx0PjGnUIu4Y1wMDembGZU5qlYB0sx66MDYWJyJKdgxllHBsTjesSVZzUa3K5i+E3X+0ViGsWYdZU/LRMTM1LnPSqERkpOqg4cbfRERhYSijhCG5vbDYJbjcyXUhf7XDJ6uwYHkByqtqCmG7tPMXwsarByySG4sTESU7hjJSPLVaA6vDA69dTroL+att3lWC11fvDCqEvSy/DW4d3T3mhbDVdBoVMsw6qBjIiIgigqGMFEuWZThdXlTZvTDZXdDrDfGeUszJsow1PxzG+p+PBsYEAbjxymxcEYdC2Gp6rQrpZj1U3FiciChiGMpIkVySBxa7G5VWF+xOV1KernRKXqzZXImDJTWnK1P0atxzXS56dm4elzkJAPQ6/8biIgMZEVFEMZSRoni9PlTZJThdHvhk/5GiZHSq3I6XPizAyTM1gSyzeQpmT+mN1s1iXwgL+AOZQe8PZNxYnIgo8hjKSDFsZzcP9ybhXZW17S4uwysrC2GrVQib17UF7r42PoWwQM3G4qlGLQMZEVGUMJRR3Lk93rObhyfnXZXVZFnGV78cxQefBxfCXtW/HSaPyonb6UJBAEx6LVJN3MeSiCiaGMoobny+s5uHO93wJVsD7Dk8Xh/e/WwPvqtdCKsScUW+CWOHdoxbIBMFwJyihSmFgYyIKNoYyiguHC43LDY3PN7k2h6pLlU2CYtWFGBfrULYdLMO08fnwG0tidu8REFAqlEDo4GBjIgoFhjKKKaSvQD2XEdKLFiwrABlVc7AWOe2qZg1OR9alQ/74xTKREFAupkbixMRxRJDGcWE7+xelXanJ2kLYM+1Zbe/EFZy1xTCDslrg6lj/IWwDocjLvNSiQLSzTroubE4EVFM8bsuRZ1L8qDKJkGq1UafzHyyjE++PYA1PxQHxgQBuGFUN4zqf0lc725UqQRkmHTQMZAREcUcv/NS1Hh9Mqx2CXanG0l+HX+A0+XB0k92YtveU4GxFJ0a91wfv0LYamqViAyzDloNNxYnIooHhjKKCqfkgYVHx4KcqnBgwbJtOH7KFhiLdyFsNa1aRLpZH7d9NImIiKGMIuzcRn7y23OoDItWbofN4Q6M5XZtjhkTc2HQx/ePoVYtIiNVDzU3FiciiiuGMooY+9lGfk+SN/LXJssyvv71GN5fXxR0g8OYwR1x3fCucd8/Uq9RId2sg4qBjIgo7hjKKGxujw8WuwSn5GHNRS0erw/vrS/Ct1uPBcbUKhHTxvXAoF6ZcZyZn16rQrpZDxU3FiciUgSGMgqZLJ9t5He44eW5yiAWu4SXlxdi39GKwFiaSYfZU/LRqU1q/CYGbixORKRUDGUUEpfbe/ZCfpbAnutoiQXzL1AIm2bSxXFm3FiciEjJGMqoUWpqLlgCW5dfdpfitdU7ggphB+dm4rZrukOjjm/VhCgAJoMWZiO3TSIiUiKGMmowu9MNq90Nt5c1F+fyyTJWf3cQq78/GBgTBGDyyCxcNbBD3I9KiYKAVJMWRj23TSIiUiqGMqoX96u8OKfkweuf7MSvRTWFsAadGvdcl4teXeJbCAv4t01KM3EfSyIipWMoowviqcr6na5wYMGyAhw7ZQ2MtW6Wgjk3xL8QFgDUKgHp3DaJiCgh8Ds11cl29lSlh6cqL6jocDkWrSiEtVYhbK8uzTHj2l5IUcBpQn9Lvy7u17IREVHDMJRREJfkgdXuhot3VV7U178cxXufF8FXqwrk6oEdMGlkVtwLYQFAd7YUli39RESJg6GMAPi3R7I43HBw8/CL8nh9eP/zInzza3Ah7O1ju2Nwbpu4zcskaqCD/4iYIAhQCwLgAGCK25SIiKiRGMqSnCzL/u2RHG54uT3SRVnsEhatKMTeIxWBsTSTFrMm56Nz27S4zUujFqE7o0LVQlfQuL6vGqbJWoiG+B+5IyKi+jGUJTHpbAEsT1XW72ipBfM/DC6E7dTGXwibbo5vIWyaoIPlDRdkS/C441sPUkZpGMqIiBIEQ1kS4l2VjfPrnlK89slOuNzewNigXpm4fWz8C2ENajU8273wltTx+ygDla+5kPFfeogmBjMiIqVjKEsyDpcbFhsLYBvCJ8tY+/1BfPydMgthAcAsaFC+zHnBxz3FPrgPeKHNVUFQwA0IRER0YQxlScLt8cFil+CUPDxV2QAuyYvXV+/EL3tKA2MGnRozru2F3K4t4jizGqLgv6hfUAuQceHfVMEgMJARESUAhrImTpZl2BxuWB1ueHlbZYOcqfQXwh4tDS6EnT0lH5nNjXGcWQ1R8Lf0q7QCzFO1qDznIv9q2h4i1G0ZyIiIEgFDWRPmkjyw2N2Q3N6LHEeh2vYe8RfCWuw1hbA9OzfDjOtyFbNvpEoUkG7WQX+2pV/bTQV1RxGeQ+eckhYB8206iEZ2lRERJQKGsiao5kJ+do41xrdbj+Gdz/YEFcJeNaADJl3RFSpRGcFGpRKQcc62SaJJQNqdOlQuDT5apuurgmjmUTIiokTBUNbE2J1uWLg9UqN4vT68/8VefP3L0cCYWiXgtmt6YEhe/Aphz6VR+bdN0mrOv+NTbCEg/bfB1RyCSoCoZygjIkoUDGVNhNvjQ5VNgsvNC/kbw2qX8MrK7dhzuDwwlmr0F8J2aRe/Qthz+fex1EOjrvuInagTAB0DGBFRImMoS3C8kD90x0qtWLBsG05X1lRKdMw0Y9aUfGSY9XGcWQ1BEKDXqpGRquc+lkRETRxDWQJznW3k54X8jbe16BSWfrwjqBB2QM/WmDa2R52nB+NBAGBK0SGDG4sTESUFhrIE5PPJsLCRPySyLGPtD8VY9e2BwJgA4PqRWRg9SBmFsIB/Tga9Bka9CIVMiYiIooyhLIGoVCpIHh9slQ5IHl7I31guyYvX1+zEL7trCmH1OhVmTMxFXpYyCmGB6kCmhk4l4oTbXe/ziYioaWAoSxCyDLg8IsqqnNDplHG9UyIpq3RiwbJtOFKrELZlhgFzpvRGmxbKKIQF/Ns4peg1SDNq4XA44j0dIiKKIYayBOBwuVFW5USVzYHmvJi/0fYdqcDLKwoUXQgL+AOZSa9Fqkkb76kQEVEcMJQpmOT2wmJ3w+X2wOX2BpWaUsN8t+0Y3vl0T9CdqUorhAX8+1iaDFqYjQxkRETJiqFMgWoa+Xkhf6i8Xh8++GIvvjqnEHbqmO64LL9tHGd2PlEQkGrUwGhgICMiSmYMZQpjc7phsUvwehnGQmV1uPHKykLsOaTsQligZmPxFAWdRiUiovhgKFMI6WznmMvjZSN/GI6fsmL+sgKcrqi5SL5DphmzJ+cjI1VZN0icu7E4ERElN/40iDN2jkXOtr2n8OrHO+CSlFsIW62ujcWJiCi58SdCHDlcblhsbri5eXhYZFnG2h+L8fE3BwI7G/gLYbti9KCOiimEraZWici4wMbiRESUvBjK4sDt8cFil+CUuHl4uCS3F2+s2YnNu2oVwmpVuPvaXOQrqBC2mn9jcR00agYyIiIKxlAWQ9w8PLLKqpxYuKwAh0ssgbGW6QbMviEfbVuY4jizumnVIjcWJyKiC2IoixGX5IHF7ubm4RGy72gFXl4eXAjbvVMz3HtdLowG5d3JqNeokG7WQcVARkREF8BQFmXsHIu877cdx38+3R10tHFU/0swZVSWogphq+m1KqSb9VCJyrq2jYiIlIWhLIrsTjesdl7IHylenw8ffrEXG7bUFMKqRH8h7NDeyiqEBfw3G+h1aqSbdBAZyIiIqB4MZVFSVuXkhfwRZHO68fJHu7G7uCwwZk7R4L7J+chqnx6/iV2AIACGs4FMaXd/EhGRMjGURYnkZglspJyxePDuu4U4XekMjHVobcasKfloprBCWMAfyIx6DdJMunhPhYiIEghDGSna9gNl+PD7Mrg9NQm3f4/WuGOc8gphAX8gM3NjcSIiCgFDGSmSLMv4dOMhfPT1/qBC2OtGdMWYwcorhAUAUQDMRi1M3FiciIhCwFBGiuMvhN2FzbtKAmM6jYgZ1+Yiv1vLOM7swkRBQKpJCyM3FiciohAxlJGi1FUIm5aiwn2T89ClffM4zuzCVKKANJMWBh0DGRERhY6hjBRj/9EKvLyiEFU2KTDW7ZI0jOihRZvmKXGc2YWpRAEZZm4sTkRE4eNPElKEHwr8hbAeb80F/Vdc2h7jL7sExQcPxHFmF6ZWCUg366FT4A0HRESUeBjKKK68Ph+WfbkPX24+EhirXQjrcDjiOLsL06hEZKRyY3EiIoochjKKG5vTjcUrt2NXghTCVtOqRaSb9dColbelExERJS6GMoqLE6dtmL9sG06V1xwJu6S1GbMn56NZmvIKYavpNCpkcGNxIiKKAoYyirnCfaexZNV2OCVvYOzS7q1w5/ieiiyErabXqJCeyo3FiYgoOhjKKGZkWcZnmw5h5Vc1hbAAcO3wLhg7pJMiC2EBf2mtTqtGhpkbixMRUfQwlFFMSG4v3ly7Cz/vrFUIq1Vh+oRe6JOtzEJYwB/IDHpuLE5ERNHHUEZRV17lxMLlBTh0sqYQtkWaHrNv6I12LU1xnNnFVW8snmrUMpAREVHUMZRRVB04VomFywuCCmFzOmTg3kl5MBmU24AvCIBJr0WqiftYEhFRbDCUUdT8WHgCb6/bFVQIO6Jfe9x0ZTdF370oCoA5RQtTCgMZERHFDkMZRZzX58OKDfvx+c+HA2OiKODW0TkY1qddHGdWP1EQkGrUwGhgICMiothiKKOIsjndWPLRduw8GFwIO3NSHrpdkhHHmdVPFAWkc2NxIiKKE4YyipgTp21YsGwbSmsVwrZvZcLsKflonmaI48zqpxIFpJt10HNjcSIiihP+BKKIKNx/thDWVVMI2y/HXwir0yq3EBYAVCoBGSYddAxkREQUR/wpRGHxF8Iexsqv9gUVwk4c1gXjLlNuIWw1jUpEulmn6J0EiIgoOTCUUcgktxdvr9uNTTtOBsZ0GhXumtATfXNaxXFmDcONxYmISEkYyigk5RYnFi4vxKETVYGx5ml6zJnSG+1aKbcQtppWLSIjVQ+1gqs5iIgoucT9J1JJSQlycnLO+2f58uUAgF27duH2229Hnz59MGrUKLzxxhtxnjEdPF6Jf7z+c1Agy+6Qjj/eOSAhApleo0IzBjIiIlKYuB8p2717N3Q6HT7//POg64/MZjPKy8sxffp0jBo1Co899hi2bt2Kxx57DEajEVOmTInjrJPXxu0n8Nba3fB4fYGxEf3a4aYrsxVdCFtNr1Uh3ayHihuLExGRwsQ9lBUVFaFTp05o1er8a5Bef/11aDQaPP7441Cr1ejatSsOHTqERYsWMZTFmM8nY8VX+7D+p+BC2Fuuzsbwvu3jOLOGEQDodf6NxUUGMiIiUqC4H9rYs2cPunbtWudjmzdvxsCBA6FW12THwYMHo7i4GKdPn47VFJOe3enGSx9uCwpkJoMGv7ulb8IEMoNejQwzAxkRESmXIo6UZWRk4LbbbsPBgwfRsWNHzJ49G8OHD8fJkyeRnZ0d9PzqI2onTpxAixYtQvpMWZZht9vDnvvFOF1OeDy++p/YQC6nK+jfsVJa7sDiVbtwqsIZGGvbIgUzJnZHs1Q9HA7HRV4dvnDXLQgCTAYNdCox6nONtOr5Jtq8w8V1J9e6geRdO9edHOuWZbnB9VBxDWUejwcHDhxAVlYW5s2bB5PJhNWrV2PmzJlYunQpnE4ntNrgPQh1Oh0AwOUKPZy43W7s2rUrrLlfjFqtRoXNC6dLivh7Hz12NOLveSGHSl349JdKSJ6aBrKumTpc2ceI8lPHUH4qZlMJad1qtQrmFB2sahlHPZ4ozCo2iouL4z2FuOC6k0+yrp3rbvrOzTIXEtdQplarsWnTJqhUKuj1egBAbm4u9u7diyVLlkCv10OSgoNNdRhLSUkJ+XM1Gg2ysrJCn3gDlFY4In6k7Oixo2jfrj10el3E3rcusizjq1+P45OfSyDXaoQdM+gSjB7UHmIMC2FDXbcoCkhN0SJFH/eDwSFzOBwoLi5Gp06dYDAoe5uqSOK6k2vdQPKunetOjnXv27evwc+N+08so9F43li3bt3w3XffITMzE6WlpUGPVf+6devWIX+mIAhhhbqG0DtkeDVy/U9sJJ1eF9UvYrfHi7fWBhfCajUipk/oFddC2MasWxQEpJm0SNE3jY3FDQZD1L9elYjrTj7Junauu2lrzM42cb3Qf+/evejXrx82bdoUNL59+3ZkZWVhwIAB2LJlC7zemv0UN27ciM6dO6N58+axnm6TV2Fx4em3fwkKZM1S9Xjo9v4J0dAPVG8s3nQCGRERJY+4hrKuXbuiS5cuePzxx7F582bs378fTz75JLZu3YrZs2djypQpsFqtePjhh7Fv3z4sX74cr732Gu677754TrtJOni8Ek++/hOKaxXCdrskHX+8awDatzbHcWYNp1IJyDDrYNAxkBERUeKJ6+lLURSxcOFCPP3003jwwQdRVVWFnj17YunSpYG7LhcvXoy///3vmDRpElq2bImHHnoIkyZNiue0m5xN20/gzXMKYYf3bYebrspOmNZ7tUpEBjcWJyKiBBb3a8patGiBJ5988oKP5+fn47333ovhjJKHzydjxdf7sH5TcCHszVdlY0Q/5fePVePG4kRE1BTEPZRRfNidbixZtQM7DpwJjBkNGsy8Pg85HTPiOLPG0WlUSDfrEuaIHhER0YUwlCWhkjI75n+4DSVlNQW67VqaMHtKPlqkJ87tyXqNCump3MeSiIiaBoayJLPjwBks/mg7HK6aMtU+2S1x14Se0GsT48uB+1gSEVFTlBg/hSlssizji5+PYNmGvUGFsOOHdsb4yzvHtBA2HIIAGM4GssZ0vxARESkdQ1kScHu8eHvdHmzcfiIwptWIuHN8T1zaPfQS3lgTBMCo1yDNFN0dDYiIiOKBoayJq7S6sHB5AQ4er+kfa5aqx+wp+bgkQfrHAP9doWaDFmZjw/YPIyIiSjQMZU3YoRNVmL+sAJXWms3bs9qnY+akPKQmULjRatRIM+oYyIiIqEljKGuiftpxEm+u3QV3rU3Rh/Vpi5uvzkmo+gi1SkSqSQuDjqWwRETUtDGUNTE+n4yPvtmPTzceCoyJgoCbrs7GiL7tEurieJUowGjSobzUW/+TiYiIEhxDWRPicHnw6qrtKNxfqxBWr8a91+ehe6dmcZxZ46lVAtLNenjdLvh8vvpfQERElOAYypqI0nJ/IezJMzWFsG1bGjF7Sm+0TKBCWADQqERkpOqgUatgd8d7NkRERLHBUNYE7Dx4BotXboe9ViFs724tMH1CL+h1ifVbrFWLyEjVJ9R1b0RERJGQWD+xKYgsy/hy8xF8+GVwIey4yzphwrAuCVMIW01/dh9LFQMZERElIYayBOX2+PDOp7vxQ2FNIaxG7S+E7d8jcQphAf+2STqtGhlmbptERETJi6EsAVVaXXh5RSEOHKsMjGWk6jB7cm90yEycQljAH8gMem6bRERExFCWYI6UWLF09R6UW2oKYbu2T8N9k/KQakys7Yeqt01KNWoZyIiIKOkxlCWQomNObCjYDre3piJiaH5b3DomsQphAX8g47ZJRERENRjKEoBPlrH6h0P4/Nea05WiIODGK7th5KXtE+4okygAZqMWJgMDGRERUTWGMoVzuDxY+vEOFOw7HRhL1EJYwB8m00xapOg18Z4KERGRojCUKdipcjvmLyvAidO2wFhmcwN+c0MftMxIiePMQqMSBaSbddBr+WVHRER0Lv50VKjdxWV4ZWUhbM6aQthOrbW4b1IeMtITMJCpBGSYdNAxkBEREdWJPyEVRpZlbNhyFB9+sRe+Wo2wVw9oj+xWUsI19AP+bZPSzTpoNap4T4WIiEixEu8nfBPm9vjw7md78H3B8cBYdSFsr06p2L9/fxxnFxqtWkS6WQ+NOrHuDiUiIoo1hjKFqLK5sHD5OYWwZh1mTclHx8xUOByOOM4uNDqNChncNomIiKhBGMoU4PDJKixYXoDyqppC2C7t/IWwaabEKoStpteqkG7WQ8Vtk4iIiBqEoSzONu8qweurd8LtqSmEvSy/DW4d3T0hT/lx2yQiIqLQMJTFiU+W8fG3B7D2h+LAmCAAN16ZjSsSsBAW8M8/Ra9BGrdNIiIiajSGsjhwujx49ZxC2BS9Gvdcl4uenZvHcWahEwTApNci1cSWfiIiolAwlMXYqXI7FiwrwPGgQtgUzLmhN1olYCEscHbbpBQtTCkMZERERKFiKIuhugph87q2wN3X9oIhAfvHAG6bREREFCmJmQQSjCzL+OqXo/jg8+BC2GuGdMS1w7pCTNA7FFWiP5AZdAxkRERE4WIoizKP118I+9224ELYO8b1wICemXGcWXi4bRIREVFk8SdqFFXZJCxaUYB9R2sKYdPNOsyenI+ObVLjOLPwcNskIiKiyGMoi5LDJRa89ME2lFU5A2Od26Zi1uT8hC2EBbhtEhERUbQwlEXBd9uO4Zn//AKpViHskLw2mDomMQthq3HbJCIiouhhKIuwVd/uxysrtwd+LQjADaO6YVT/SxK6UJXbJhEREUUXQ1mEvb1ud+C/U3Rq3HN94hbCAv5tk/Q6/7ZJiXqXKBERUSLgeagIy+vaAgDQtoURc+8ckNiBTPDvY5lhZiAjIiKKNh4pi7A/3TUQJ87YIACQ6322cgkCYNRrEvqmBCIiokTCI2URJooC2rU0JfSRJVEAzAYtAxkREVEM8UgZBREFAalGDYwG7mNJREQUSwxlFMBtk4iIiOKHoYwA+ANZhpnbJhEREcULfwIT1CoB6WY9dNw2iYiIKG4YypKcf9skHTRqBjIiIqJ4YihLYlq1iIxUPdTcNomIiCjuGMqSlF6jQjr3sSQiIlIMhrIkxH0siYiIlIehLIkI8G+blG7SJfTm6ERERE0RQ1mSEAQgRa9BmlHLQEZERKRADGVJgPtYEhERKR9DWRMnCoDJoIXZyG2TiIiIlIyhrAnjPpZERESJg6GsieI+lkRERImFoawJUokC0s066LmPJRERUcLgT+0mRqUSkMF9LImIiBIOQ1kTolGJyEjlPpZERESJiKGsifBvLK6HRs1tk4iIiBIRQ1kToNOokMF9LImIiBIaQ1mC4z6WRERETQNDWYISAOh1/n0sRQYyIiKihMdQloAEATDouLE4ERFRU8JQlmBEQeA+lkRERE0QrwxPIBq1GmajloGMiIioCWIoSxCiKCDVqIVRz4ObRERETRFDWQIQBQHpJh3Uoi/eUyEiIqIo4WEXhavex9LnkeD1euM9HSIiIooSHilTMP8+ltxYnIiIKBnwp71CaVQi0s06aLmxOBERUVJgKFMg7mNJRESUfBjKFEanUSHdrIOa+1gSERElFYYyBdFrVEhP5T6WREREyYihTAEEADqtGhlm7mNJRESUrBjK4kwAYNBzH0siIqJkx1AWR4IAGPUapBq1DGRERERJjqEsTgQBMBu0MBu18Z4KERERKQBDWRyIAmA2amEyMJARERGRH0NZjImCgFSTFka9Jt5TISIiIgVhKIshlSggzaSFQcdARkRERMEYymJEJfr3sdRxH0siIiKqAxNCDKjP7mOp4z6WREREdAEMZVHm38dSB42agYyIiIgujKEsSgT4A1lGqp77WBIREVG9GMqiRK0WkW7SQcVARkRERA3AUBYlGWY997EkIiKiBuNhnChhICMiIqLGYCgjIiIiUgCGMiIiIiIFYCgjIiIiUgCGsiiQvb54T4GIiIgSDENZhMkOD7w/n4Jsc8d7KkRERJRAGMoiTLZ74H6jCLKFoYyIiIgajqEsTLLbB9kiwVclQa6S4F5+EJAB93v7IVf6x2SLBFnyxnuqREREpGAsjw2X2wvfYSukxbsBZ03w8u2ugHPeJkAnQnN7N6h6ZgBa7n9JREREdVPUkbKDBw+ib9++WL58eWDsz3/+M3JycoL+GTVqVBxnGUxI0UDskgrN5M51Pq4e2wGqnhkQUjQxnhkRERElEsUcKXO73fjDH/4Au90eNL5nzx7MmjULt99+e2BMpVLWESfBoIaqf0t4t56Bb2d5zXgnM9SXZzKQERERUb0UE8peeOEFmEymoDFZlrFv3z7MnDkTLVu2jNPMGsjjg1wl+f9bACADqJIArxzPWREREVGCUEQo+/nnn/Hee+9h5cqVGDlyZGD88OHDsNvt6NKlS/wm11AyALsHmrtyoMprBm9RBTwfHIAMf0YjIiIiupi4h7Kqqio89NBD+POf/4w2bdoEPVZUVAQAePPNN/HNN99AFEUMHz4cv/vd72A2m0P+TFmWzztNGi6DXgfNQ73h0cqQZAlijgmah3pDSFFH5LMcDkfQv5NFsq4bSN61c93JtW4gedfOdSfHumVZhiA07PBM3EPZX//6V/Tt2xcTJ04877GioiKIoohWrVph4cKFOHz4MJ566ins3bsXr7/+OkQxtPsU3G43du3aFe7Ug5hMJlit1vPGzWYzLBZLxD6nuLg4Yu+VSJJ13UDyrp3rTj7Junauu+nTarUNel5cQ9nKlSuxefNmfPzxx3U+Pnv2bEydOhUZGRkAgOzsbLRs2RI33XQTCgsL0bt375A+V6PRICsrK+R5x4PD4UBxcTE6deoEg8EQ7+nETLKuG0jetXPdybVuIHnXznUnx7r37dvX4OfGNZQtW7YMZ86cCbqODAAeffRRrFmzBosXLw4EsmrdunUDAJw8eTLkUCYIAlJSUkJ6bbwZDIaEnXs4knXdQPKunetOPsm6dq67aWvoqUsgzqHsX//6F5xOZ9DY6NGj8cADD+Daa6/FQw89hNLSUrz22muBxwsLCwEg4Y50EREREV1MXENZ69at6xxv3rw5WrdujTFjxmDOnDl48cUXce211+LgwYN4/PHHMWHCBHTt2jXGsyUiIiKKnrhf6H8xV155JZ599lksWrQIr7zyCsxmMyZOnIgHH3ww3lMjIiIiiijFhbI9e/YE/Xrs2LEYO3ZsnGZDREREFBuK2vuSiIiIKFkxlBEREREpAEMZERERkQIwlBEREREpAEMZERERkQIwlBEREREpAEMZERERkQIwlBEREREpAEMZERERkQIwlBEREREpAEMZERERkQIwlBEREREpgCDLshzvScTSL7/8AlmWodVq4z2VRpFlGW63GxqNBoIgxHs6MZOs6waSd+1cd3KtG0jetXPdybFuSZIgCAL69etX73PVMZiPoiTqF4AgCAkXJCMhWdcNJO/aue7kk6xr57qTgyAIDc4eSXekjIiIiEiJeE0ZERERkQIwlBEREREpAEMZERERkQIwlBEREREpAEMZERERkQIwlBEREREpAEMZERERkQIwlBEREREpAEMZERERkQIwlBEREREpAEMZERERkQIwlBEREREpAEOZwpSUlCAnJ+e8f5YvX37ec//85z9j1KhRcZhldNS39tLSUvz+979H//79MWjQIPz3f/83ysrK4jzr8NW37h07dmDatGno27cvRo4ciX/961+QJCnOs46MlStXYty4ccjLy8P48eOxdu3awGNHjx7Ffffdh379+uHyyy/Hs88+C6/XG8fZRtbF1v7LL79g2rRpuPTSSzFs2DA8/PDDqKioiN9kI+hi665twYIFyMnJifHsoudi67ZarXj00UcxePBgXHrppZg1axaOHDkSx9lGzsXWffjwYcyaNQv9+/fH5ZdfjkceeQQWiyWOs1UAmRTlq6++kvPy8uSSkhK5tLQ08I/D4Qh63vr16+Xs7Gz5iiuuiNNMI+9ia3e5XPL48ePlm2++Wd6xY4e8detWedy4cfI999wT72mH7WLrLisrkwcOHCg/8sgjcnFxsfzNN9/IQ4YMkf/5z3/Ge9phW7lypdyzZ0/5rbfekg8dOiTPnz9f7t69u/zLL7/IkiTJo0ePlmfOnCnv2bNHXr9+vTxw4ED5ueeei/e0I+Jiaz9w4IDcp08f+W9/+5u8b98++eeff5YnTJgg33HHHfGedtgutu7atm3bJvfs2VPOzs6O00wjq75133XXXfKYMWPkzZs3y7t27ZKnTp0qjx8/XvZ6vXGeeXga8mf8N7/5TeDrfPTo0fIDDzwQ72nHFUOZwixatEieOHHiRZ9TUlIiDx48WL799tubVCi72NqXLVsm9+nTRz516lRg7JtvvpGvvPJK2WKxxGqKUXGxdVeH79prfOKJJ+QJEybEanpR4fP55CuuuEL+xz/+ETR+9913ywsXLpQ//vhjOTc3V66oqAg89u6778r9+vWTXS5XrKcbUfWt/d///rc8evRo2efzBR77+eef5ezsbPnw4cOxnm7E1LfuajabTR49erR8xx13NIlQVt+6N27cKOfk5Mi7d+8OPLZ371555MiR8v79+2M93Yipb907d+6Us7Ozg9b9+uuvy3379o31VBVFHe8jdRRsz5496Nq16wUfl2UZ8+bNw3XXXQej0YgVK1bEcHbRdbG1f/fddxg8eDBatGgRGBs2bBg+//zzWE0vai627mbNmgEA3nnnHdx99904ceIEvv76a/Tv3z+WU4y4gwcP4tixY5g4cWLQ+JIlSwAAf/3rX9GrVy+kpaUFHhs8eDCsVit27dqF3r17x3S+kVTf2vfv348rrrgCgiAEHqv+78rKSlxyySWxm2wE1bfuan//+9+RnZ2NK664Ahs3bozlFKOivnU//fTTyM7ODjpVm5WVhQ0bNsR0npFW37pPnjwJURTx/vvvY+7cubBarVi3bl1C/9mOBF5TpjBFRUUoKyvDbbfdhssuuwy33norvvnmm8Djr732Gk6dOoXf//73cZxldFxs7QcPHkT79u3x0ksv4eqrr8YVV1yBv/zlL6iqqorzrMN3sXX369cPs2fPxnPPPYe8vDxceeWVaN26NR555JE4zzo8Bw8eBADY7XbMmDEDQ4YMwY033ogvv/wSgP8bdmZmZtBrWrVqBQA4ceJEbCcbYfWtvWvXrujTp0/Qa1555RW0bNkyoa+xqm/dAPDZZ5/h66+/xuOPPx6vaUZcfes+ePAgOnbsiP/85z8YP348hg0bhgcffBAlJSXxnHbY6lt3ZmYm/vznP2P58uXo3bs3hgwZAovFgn//+9/xnHbcMZQpiMfjwYEDB1BZWYn/+q//wqJFi9CnTx/MnDkTP/74I3bv3o0XX3wR//d//wetVhvv6UZUfWu3Wq1YuXIl9uzZg6effhqPP/44tmzZgjlz5kCW5XhPP2QNWfeBAwdw22234YMPPsBzzz2H4uJi/OUvf4n31MNitVoBAHPnzsWECRPw6quvYujQoZgzZw5+/PFHOJ3O877GdTodAMDlcsV8vpFU39rP9c9//hNfffUV/vrXv0Kj0cR6uhFT37pLSkrwyCOP4IknnkBGRkacZxs59a3barVi48aNWLNmDR577DE888wzOHnyJO64446E/lqvb92SJGHPnj0YPXo03nvvPSxatAg+nw8PPvhgk7qhp7F4+lJB1Go1Nm3aBJVKBb1eDwDIzc3F3r17MX/+fJSXl2P27Nno3r17nGcaeRdb+5IlS6BWq5GSkoKnn3468IMpLS0NN954IwoLC5Gfnx/P6YesvnW3a9cOlZWVeP755wEgcErvrrvuwl133YUePXrEc/ohq/49nDFjBiZNmgQA6NGjB3bu3ImlS5dCr9efd4dp9Q+olJSU2E42wupb+5AhQwAAbrcbjzzyCFauXIm//e1vuOqqq+I250iob91utxtjx47F8OHD4znNiKtv3Wq1Gi6XCy+99FLgdP2LL76IYcOG4csvv8TYsWPjNvdw1LfuwsJCbNq0CWvWrIFKpQIAdOrUCaNHj8aGDRsS/us9VDxSpjBGozHww7lat27d8NNPP2Hv3r148cUX0bdvX/Tt2xcvv/wyjh8/jr59+2Lz5s1xmnHkXGjtJSUlyMzMROfOnYOOFHTr1g2AvzohkV1s3Vu2bEFeXl7QY9XXXBQXF8dqihHXunVrAEB2dnbQeFZWFo4ePYrMzEyUlpYGPVb96+rXJqr61g74jzLce++9+Pjjj/Hvf/8bN954Y8znGWkXW/e+ffvwww8/YPny5YHvb48++igAoG/fvli1alXM5xspDflab926ddD1ky1atEB6enpCf2+rb91btmxBz549A4EMADp27IiMjIyE/t4WLoYyBdm7dy/69euHTZs2BY1v374dY8eOxWeffYaPPvoIK1euxMqVK3HLLbegVatWWLlyJXJzc+M068i42NqzsrIwYMAA7N69G06nM/BYUVERAP8f5ERV37pbt26NPXv2BD1W/evOnTvHbJ6R1qtXLxiNRmzbti1ovKioCB06dMCAAQOwc+fOwCkQANi4cSOMRmPCHymub+2SJOG+++5DQUEBlixZkrBHSs51sXV37doVn332GVatWhX4/vbAAw8A8PdcJXIfY0O+1o8fPx70l5DS0lKUl5cn9Pe2+tbdunVr7N27N+jyk5KSElRUVKBTp04xnq2CxPnuT6rF6/XKU6ZMkceNGyf//PPP8r59++QnnnhCzs3Nlffs2XPe859//vkmU4lR39rPnDkjX3bZZfKcOXPkoqIiefPmzfKECRPkadOmxXvqYalv3d98842ck5MjP/PMM/KhQ4fkH374QR4zZow8c+bMeE89bC+99JLct29f+eOPPw7qMNq4caPsdDrlq666Sp4xY4a8a9euQE/ZCy+8EO9pR8TF1v7888/LOTk58ieffBLUW1daWprwdSAXW/e5li1b1iQqMWT54ut2uVzyuHHj5FtuuUUuLCyUd+zYId92223yNddc06R/v3fv3i3n5ubKDz/8sLxv3z75119/lW+66Sb5uuuukyVJivfU44ahTGFOnTolz5s3Tx46dKicl5cn33zzzfLPP/9c53ObUiiT5frXfvDgQXnmzJly79695QEDBsh//OMf5crKyjjOODLqW/dXX30l33jjjXKfPn3kK664Qn7iiSdkm80WxxlHzquvviqPGjVK7tWrl3zttdfK69evDzxWXFwsT58+Xc7Ly5Mvv/xy+dlnn034Ms3aLrT20aNHy9nZ2XX+U1d4STQX+z2vrSmFMlm++LpLS0vl3//+93K/fv3kvn37yvfff7988uTJOM42ci627l9//VWeNm2afOmll8qXX365PG/ePPnMmTNxnG38CbKcwLeuERERETURvKaMiIiISAEYyoiIiIgUgKGMiIiISAEYyoiIiIgUgKGMiIiISAEYyoiIiIgUgKGMiIiISAEYyoiImgjWThIlNoYyIoqIadOmIScnB7fccssFn/O73/0OOTk5mDdvXtBrLvZP9XPnzZt30ecNHTo08Dl1Pbdfv3646aab8Nlnn9U5t4KCAowZMwaSJEXw/0rkbNq0CTk5OYF9Ul944QXk5OQEHt+yZQtmzpzZ6Pe9/fbbsWbNmojNk4hCp473BIio6RBFEVu3bsXJkyeRmZkZ9JjdbseGDRuCxh599NGgTccfe+yxwHi1Zs2aBf67ZcuWePHFF+v8bI1GE/Tr2s/1+XyorKzEJ598ggceeABLliwJCnEulwtz587F//zP/0Cr1TZmyXFz4403YtiwYYFff/DBB9i/f3+j3+dPf/oTZsyYgUGDBqF58+aRnCIRNRJDGRFFTM+ePbFv3z6sW7cOd911V9BjGzZsgMFgQGpqamAsKysr6DkmkwkA0KdPnzrfX6vVXvCxhjx35MiR+PXXX/Hee+8FhbL//Oc/UKvVuOqqqxr03kqQmZl5XvANRc+ePZGfn48FCxbgz3/+cwRmRkSh4ulLIoqYlJQUjBgxAuvWrTvvsTVr1mDMmDFQq+P3d0FBEGA2myEIQmBMkiQsXboUEyZMCHquy+XCU089hREjRiA3NxcTJ0487zTfqFGj8Pzzz+Of//wnLrvsMuTn52PGjBkoLi4Oet6KFSswbtw45OXl4dprr8WPP/6Inj17Yvny5SGvpfbpy3nz5mHFihU4duwYcnJyAu/bkDUAwMSJE/Hhhx+irKws5PkQUfgYyogoosaNGxc4hVnNarXim2++OS/4hMLj8dT5T10XuVc/5na7UV5ejjfeeAN79+7FrbfeGnjOpk2bUFJSgtGjRwfGZFnGb37zG7z77ruYPn06FixYgL59++J3v/sdVq5cGfQZb7zxBg4cOIAnn3wS//u//4vt27dj7ty5gcdXrlyJefPmoV+/fpg/fz7GjBmDOXPmwOv1hv3/otqcOXMwYsQItGzZEu+99x5GjhzZqDWMGjUKXq8X69evj9iciKjxePqSiCJq5MiRMBgMQacw169fj+bNm+PSSy8N672PHTuGXr161fnYQw89hBkzZtT73FtvvRUDBw4M/Hrjxo1ITU1F586dA2M//PADvv32WzzzzDMYN24cAGDYsGFwOBz417/+hQkTJgSO+KWmpmL+/PlQqVQAgMOHD+OFF15AeXk5MjIy8Nxzz+GKK67A//7v/wbeR6PR4Omnnw7r/0VtHTp0QLNmzYJO2X7//fcNXkNKSgq6du2KH3/8ETfffHPE5kVEjcNQRkQRpdfrMWrUqKBQtnr1aowdOzbotGEoWrZsiQULFtT5WJs2bS76XKvVis2bN2PRokWwWq3417/+BQA4cuQI2rVrF/TaH3/8EYIgYMSIEfB4PIHxUaNGYdWqVdi7dy969OgBAMjLywsEMgCB67wcDgeqqqpw/Phx/Pa3vw16//Hjx0c0lNWlMWsAgHbt2uHo0aNRnRMRXRxDGRFF3NixY3H//ffj5MmT0Ol0+PHHH/Hggw+G/b5arRZ5eXkhP3fIkCFQq9V49tlnMX36dPTq1QtWqxUGgyHoeRUVFZBlGf369avzvUtLSwOB5tzXiqL/qhCfzxe4RuvcuxpbtGjRoDWEozFrAPzrsFgsUZ8XEV0YQxkRRdzw4cNhNBqxbt06pKSkoH379sjNzY33tAAgMI9Dhw6hV69eyMjIQGlpadBzzGYzUlJS8MYbb9T5Hh07dmzQZ1UfNTtz5kzQ+Lm/jobGrqGqqgoZGRlRnxcRXRgv9CeiiNNqtbjqqqvw6aefYu3atRg/fny8pxRQUFAAoCaUtG3bFidPngy6UWDgwIGw2+2QZRl5eXmBf4qKivDSSy8FnQ68mMzMTHTo0OG8C+gvVGAbjuojdNUau4aTJ0+edxqXiGKLR8qIKCrGjRuH++67D6IoRqz/SpIkbN269YKP5+TkBE4nnvtcj8eDn376CQsWLMDll18euAlg6NChWLRoEYqKigIVEyNGjMCAAQMwZ84czJkzB127dkVBQQGef/55DBs2LKjQ9mIEQcADDzyAP/zhD3j00Udx9dVXY/fu3XjppZcABAeprVu3olmzZujQoUNj/pcEpKam4vTp0/j666/Ro0ePRq3BYrFg7969uPvuu0P6bCKKDIYyIoqKyy67DKmpqWjTpg26du0akfc8derURe8OXLlyZeA6qXOfq9Fo0K5dO9xxxx34zW9+Exjv378/mjdvjq+//joQykRRxKJFi/Dcc8/h5ZdfxpkzZ9C6dWtMnz496LUNMXHiRNjtdixZsgTLli1Dt27d8PDDD+Phhx9GSkpK4Hk333wzJk2ahH/84x+Nev9qkydPxtdff43f/OY3eOCBBzBz5swGr+Hbb7+FRqPByJEjQ/psIooMQeYOtkSU5F599VW88847+Oyzz8K+Q/Rcn3zyCXr27IkuXboExr766ivcd999+Oijj9C9e3cA/rsl165di8cffzyin98Qd955J7Kzs/Hwww/H/LOJqAavKSOipDd16lT4fL46dyII16pVq3Dvvffi448/xubNm7Fs2TI8+uijGDhwYCCQ+Xw+LF68OGjrp1gpLCzE7t27Q9rMnIgii0fKiIgA/PLLL5g3bx4++eSTiG5KXl5ejqeffhrffPMNysrK0KJFC4wZMwYPPPAAjEZj4Hk7duy4YDFuNE2dOhVTp06NyG4LRBQehjIiIiIiBeDpSyIiIiIFYCgjIiIiUgCGMiIiIiIFYCgjIiIiUgCGMiIiIiIFYCgjIiIiUgCGMiIiIiIFYCgjIiIiUgCGMiIiIiIFYCgjIiIiUgCGMiIiIiIFUMd7AslIlmV4vV54PJ54T4WIiKjJ02g0UKlU8Z5GvRjKYkiWZVRUVODUqVPwer3xng4REVHSSE9PR2ZmJgRBiPdULoihLIZOnjyJiooKpKamIjU1FWq1WtFfHERERIlOlmXY7XaUlpYCANq0aRPnGV0YQ1mMeL1eVFZWomXLlmjRokW8p0NERJQ0DAYDAKC0tBStWrVS7KlMXugfI263G7Isw2g0xnsqRERESSclJQWA/+exUjGUxRhPVxIREcVeIvz8ZSgjIiIiUgCGMiIiIiIFYCijRhs1ahRycnKwdOnSOh9/5JFHkJOTgxdeeCHsz2nMe9T3/JycHNx444111pFMmzYN8+bNC2meREpT5fLhjMOHU3bvef+ccfhQ5fLFdX4vvPACRo0aFfh1Tk4Oli9ffsHnb9q0CTk5OejZsyfKysrOe1ySJPTv3x85OTk4evQogMb/ma79/WP58uXIycm56PNzcnKQk5ODzz77rM7HZ8yY0eB1Vc+5Pi+//DKmTZt23viXX36JKVOmoG/fvhg1ahT++c9/wul0Nug9SVl49yWFRKPR4NNPP8X06dODxj0eDz777DPFnrsvKCjAkiVLMHPmzHhPhShqtCrgPzsdeGvH+T+Yp/XS4848QxxmFT5RFLF+/XrcfPPNQePffPMNrFZr0NgLL7zQqDvsPvzwQ+h0ukbNp/r74OjRo4PGy8vLsWnTpka9V33efvttPPvss+jfv3/Q+ObNm3H//ffjgQcewDXXXINDhw7hkUceQUVFBZ588smIzoGij0fKKCRDhgzB1q1bcfLkyaDxjRs3IiUlRbE9MJdccgleeOEF7Nu3L95TIYoavVrEjd0N0J6TSXQq4IbuBujVifmtf8iQIVi3bt1542vXrj0vrKSnp8NsNjf4vZs1a9bou+OHDBmCDRs2wOVyBY1/9tln6NOnT6Pe60JKSkowa9Ys/Otf/0KnTp3Oe/zdd9/FoEGDMGvWLHTq1AkjRozA7373O3z88ceQJCkic6DYScw/mRR3+fn5aNu27XnfINesWYOxY8eed6Ts119/xR133IFLL70UgwYNwh//+EeUl5cHHrdYLJg7dy769++PwYMH13lq9JdffsFtt92G/Px8jBw5Eo899th5fzuuzz333IMOHTpg7ty5F91VYfPmzbjjjjvQr18/5ObmYuzYsfjoo48Cj8+bNw8PPfQQ/vd//xf9+/fHwIED8fzzz2P//v2YOnUq8vPzMXHiRGzbti1ojX/5y18wePBgXHrppbjjjjtQWFjYqPkTNZRRA9zUXR80dlN3PYya2Hx+UVER7rvvPgwYMAC5ubm48sor8eqrr4b1nmPHjsVPP/0UdArT6XTiyy+/xLhx44KeW/v05fLly3H11VcH/p2bm4vJkydjy5Ytgec39nIJABg5ciR8Ph++/fbboPE1a9acN59Q7dixAxqNBqtWrULv3r3Pe/zuu+/G3Llzg8ZEUYTb7W7090eKP4YyCtnYsWODQpkkSfj8888xfvz4oOcVFBRg2rRp6NatG95//30899xz2LZtG2bMmBEIRg8++CAKCgqwcOFCLF26FF999RWOHTsWeI/du3dj+vTpGDZsGFatWoV//etf2LFjB+6++27IstzgOWu1Wjz55JPYtWsXXnnllTqfU1JSghkzZiAvLw8rVqzAypUrkZ+fj4cffhinT58OPG/NmjVQqVRYvnw57rrrLrz00kuYNWsWZsyYgQ8++AA6nQ6PPfYYAH+j9L333osjR47g5Zdfxv+3d+dxOaX/48dfrSJJFEKUqNCKhsaWYlSWJAzZktDYGaPs1Yw1W8pMRYMGMwZRhIzlw+Q7tuFjTCGlkOxLhPZ+f/TrfLoVikrNXM/H435wrnOd61zndN/nvM91XeecX3/9FXNzc4YNG0ZcXFyp6y8IpfVma1lltpK9fv0aNzc36tatyy+//ML+/fuxs7Nj+fLlXLly5YPLtbS0RENDgyNHjkhpx48fR0dHB319/Xcue/fuXX755Rf8/PzYs2cPNWvWxMvLq0zHjzfVrFkTa2trDh48KKU9evSICxcuYGdn98HlFlUYLOro6JQ4v02bNhgZGUnT2dnZbN68GWNjY+rVq1cudRAqjwjKhA9mb2/Pf//7X+7fvw/AqVOnqFevHm3atJHJ9+OPP2JoaMiCBQvQ19enU6dOrF69mtjYWGJiYrhx4wYxMTEsXLiQDh060Lp1a1atWoWysrJURmhoKJ07d5aa6Dt06MCqVau4dOkSZ8+eLVO9TU1NcXd3JzAwkPj4+GLzMzMzmTJlCrNmzaJ58+a0bNmS8ePHk52dTXJyspSvbt26eHp60qxZM1xdXQFwcHDA1tYWQ0NDBg4cKJV/+vRp/vvf/7J27VrMzMzQ19dn5syZmJubExYWVqb6C0JpFW0tq8xWstevXzNq1CgWLlyIvr4+urq6TJ06FYBr1659cLlycnL07t1b5mLw4MGDxS4ES5KdnY2Pjw/m5ua0atWKMWPGcOvWLR4+fPjB9YGC4+Dx48elrsJDhw7x2WeffZKAKCcnh9mzZ3P9+nUWLVpU6esXPp4Y6C98MGNjY3R0dIiOjmbUqFEcOHCgxINjfHw8nTt3lkkzMjJCTU2Na9eu8fr1awBMTEyk+ZqamjJXhnFxcdy8eRMLC4ti5ScmJtKxY8cy1X3y5MkcO3YMLy8vfv31V5l5zZo1Y+DAgYSFhREfH8+tW7e4evUqgEyXZ9OmTZGXL7iuKXxSdNE6q6ioSE+Ojo2NJT8/nx49esisKysrq9h4FEEoL4WtZZHXMyt1LFm9evVwcXFh//79xMXFyfyG8vLefefnm7/xqKgomWl7e3tGjx7N06dPUVZW5uTJk3zzzTekpqa+t15FW9MKx5t97NPdu3fvTn5+Pr///ju2trYcOHCAQYMGFcv3vu36WOnp6UyfPp2zZ88SGBiIqalpuZYvVA4RlAkfpbAL88svv+To0aPs3LmzWJ63dQ/k5+ejpKQkjT9782CtqPi/r2deXh79+vXDw8OjWDkfckVa2I05dOhQQkJCZOYlJCTg4uJC27Zt+fzzz/niiy/Q0NBg8ODBMvmUlIo3OxQGaW/Ky8ujdu3aJd4eX7RFUBDKm6oSrP+iTqW1kgE8fPiQL7/8knr16mFjY0OXLl0wMTGhe/fu71127969MtMNGjTg9u3b0nT79u2pX78+R44cQUVFBQMDA3R0dEoVlJX0W/uY7ksouPiysbHh0KFDtG3bltjYWIKDg4vle992fYwHDx4wbtw47ty5Q2hoKJaWluVSrlD5RFAmfBR7e3tCQkLYvXv3W8d1GBoaygyohYIxYunp6VLXBhQM5Le2tgbg+fPn3Lp1S8rfqlUrEhISaN68uZSWmJiIn58fM2fOLNNdVoVMTExwd3fn+++/R1NTkyZNmgAFdzPVr19f5maDY8eOAR9+ADcwMCA9PZ3s7Gxatmwppc+fPx8jIyNGjBjxQeUKwvuoKMrTUJVKveNy//79PHv2jOjoaOnipbDb8n2/oaK/8ZIUdmFGR0ejrKxcbgPqP4a9vT2enp4YGBjQuXPnEo9H79uuD5WWlsbo0aNJT09n27Zt732+mlC1iaBM+CitW7emefPmrFq1igkTJpSYZ8yYMbi4uPDtt9/i4uLCo0eP+Pbbb2nTpg1WVlYoKSlhZ2eHr68vysrKaGpqsnr1apnbud3c3Bg+fDg+Pj6MGDGC58+f4+PjQ0ZGRom3iZfWpEmTOHbsmMzYskaNGnHv3j1OnDhBy5YtiY2N5bvvvgP44FvMu3btSuvWrZkxYwbz5s1DW1ub7du3Ex4eTmho6AfXXxBKo5ZS5Q4fbtSoEa9fv+bQoUO0b9+eGzduSM/MKo/HNNjb2zNq1CiUlJQqdOzUyZMni6UZGhrSsGFDmbSuXbuSn59PUFAQvr6+ZV7PuXPnuHHjhkxa8+bNSxXILV26lNu3b7Nx40bq1asnM0auXr16ZXpWm/DpiaBM+Gj29vb88MMPb71iNTMzY+PGjaxdu5YBAwZQu3Ztevbsyddffy1dRS9fvpzly5czY8YM8vLy+PLLL2Vuezc3N2fjxo34+/vj5ORErVq1sLKywtPT86O6/5SVlVm2bBlDhgyR0kaNGsWNGzeYPXs2WVlZ6OrqMnPmTNatW8fly5fp1q1bmdejoKDAjz/+iJ+fH9OnT+f169fo6+sTGBiIlZXVB9dfEKoiOzs7YmNjWbZsGenp6TRp0oTBgwdz9OhRLl++/NHPMbSwsJDGnb4ZIJWncePGFUtbunQpAwcOlEmrUaMGtra2/Pbbb8XGjZZGSW8emDx5MlOmTHnncrm5uRw4cIDs7GxGjx5dbP7Ro0dp2rRpmesjfDpy+R/boS6USkZGBklJSejp6aGiovL+BQRBEARBKDfV4TwsHokhCIIgCIJQBYigTBAEQRAEoQoQQZkgCIIgCEIVIIIyQRAEQRCEKkAEZYIgCIIgCFWACMoEQRAEQRCqABGUCYIgCIIgVAEiKBMEQRAEQagCRFAmCIIgCIJQBYigTPhg+fn5hIeHM3LkSDp16oSxsTG9evVi8eLFMu9fK4mhoSHh4eEAZGdns3nzZmleeHj4e1+qa2hoiKGhIYcPHy5x/tixY2XWURIvLy+pnDc/gYGBJeZp3bo1Xbp0YeHChaSnp7+zjoIglCwgIAAbGxtp+n2/1TNnzmBoaEibNm1kXr9WKCsriw4dOmBoaEhKSgoAI0eOLPH1RW9jY2NDQEAAUHnHoMLtKqzz2+zduxcHBwdMTEzo06cPBw8efM/WCNWVCMqqsfzXGeQ/Tyfv3kPyn6eT/zqj0tadl5fHpEmTWLZsGT169OCnn37i8OHDzJ8/n8uXL+Ps7Mzjx4/funxMTIz0rsz9+/dLLysuCyUlJaKjo4ulP336lDNnzrx3+Xnz5hETEyPz6dOnD1paWgwePFjKZ2FhIc0/evQoq1at4ty5c8ydO7fMdRaEyvTmW/Sq+1v15OXl+e2334qlnzx5sthFUkBAAPPmzSt12bt27cLNza1M9fnYY1BpREREMG/ePIYPH05UVBR9+/Zl5syZXLx4sVzKF6oW8ULyaio//RXZe46Q999rkJ8PcnLImxuh5GSLXO1aFb7+zZs3c+LECX799Vfatm0rpTdu3JiOHTvSp08fQkNDmT17donLa2lp/W9bPvBEYWVlxfHjx8nMzKRGjRpS+uHDhzE3N+fcuXPvXF5NTQ01NTVp+tixYxw4cIAtW7bIvORYSUlJpr6NGzdm0qRJzJo1i/T0dGrXrv1B9ReEipaTDbm5+ajUlCPjdT4KCqCk/Klr9eGsrKw4dOgQX375pUz6wYMH6dChg8xvvm7dumUqu169eh9Un485Br1Pfn4+/v7+jBo1iuHDhwPw1Vdfcf78ec6ePYuFhcVHlS9UPaKlrBrKf51REJBdvFoQkAHk55N38QrZe45WeItZfn4+W7dupX///jIBWSEVFRXCwsKYPn06KSkpGBoaEhwcTOfOnbG1tSU9PV1q1g8PD2fOnDlAQXdAWa4ura2tycvL4/fff5dJP3DggNQKV1qZmZksXrwYZ2dnOnbs+N78KioqyMnJlWkdglDZcrLzOb47g/u3czm+O4Oc7MprKYuPj2fChAlYWlpibGyMra0tP/7440eVaW9vz9mzZ2W6MDMyMjh27Fix33zR7svw8HB69eol/WtsbMzAgQP5888/pfxFuy9LqzyPQSVJSkrizp079OvXTyY9NDSUCRMmfHT5QtUjgrLqKDunoIWsBHn/vQrZORW6+pSUFO7cucPnn3/+1jxNmjRBWfl/l+R79uxhy5YtrF27VqZlycHBQeoGjImJKdOVX82aNbG2tpYZX/Ho0SMuXLiAnZ1dWTaJnTt38ujRI6ZPn/7evPfu3ePHH3/Ezs5OtJIJVVZhAJb2OI8jv2aQ9jhPJr0ivX79Gjc3N+rWrcsvv/zC/v37sbOzY/ny5Vy5cuWDy7W0tERDQ4MjR45IacePH0dHRwd9ff13Lnv37l1++eUX/Pz82LNnDzVr1sTLy+ujunTL8xhUkqSkJABevXrF2LFjsbKyYvDgwRw7duyjyxaqJhGUVUP5r17/r4Ws2Mx88l9VbEvZo0ePgOLN/R4eHlhYWEifPn36SPNcXFxo2bIlJiYmMsuoqKhIXYhaWloygVxp2Nvbc/z4cbKysgA4dOgQn332WZm6IvLy8tiyZQuDBw+W6aYsdP78eWmbTE1N6d69O4mJiUyePLlMdRWEypKTlc/zp3lEbXlN7v+/RsvNgagtr3n+NI+crIoNzF6/fs2oUaNYuHAh+vr66OrqMnXqVACuXSv5grI05OTk6N27N4cOHZLSDh48KHOseZvs7Gx8fHwwNzenVatWjBkzhlu3br33pqT3KY9j0NsUjpPz9PSkb9++/Pjjj3Tu3JmJEyfyxx9/fHT5QtUjxpRVQ3K1aoKcXMmBmZwccrVUKnT9GhoaAKSlpcmk+/j4kJFREBD+9NNPMldzzZs3r5C6dO/enfz8fH7//XdsbW05cOAAgwYNKpbvzRa4qKgoGjduDMCFCxe4desWw4YNK3EdxsbGrFy5EoDc3FweP35MWFgYX375JTt37kRPT6+ct0oQPo6ishx1NOTpM7omERsLAjMFRegzuiZKNeRQVKzYrvd69erh4uLC/v37iYuL49atW1y9ehUouAh6l5J+q0XZ29szevRonj59irKyMidPnuSbb74hNTX1vfUq2ppWeDGYnZ1dqm16m485Br2PkpISUHAnp5OTEwCtW7cmLi6OTZs2YWVl9VF1F6oeEZRVR0qKyJsbkXexeDeAvLkRKFXsn1VHRwctLS3OnDkjM26i6OB4dXV1mWVUVComUFRRUcHGxoZDhw7Rtm1bYmNjCQ4OLpZv7969MtMNGjSQ/v/bb7/Rpk2bt3Z/qKioyASVLVq0wMzMjI4dO/Lrr7/i6elZPhsjCOVIUUmO7Kx81OvL085amQv/KWjJqeiADODhw4d8+eWX1KtXDxsbG7p06YKJiQndu3d/77Il/VZv374tTbdv35769etz5MgRVFRUMDAwQEdHp1RBWUkt8R97R+rHHIOKbldJCo+pBgYGMuktW7bkP//5z0fVW6iaRFBWDcnVVEHJyZZs/v8YsjfvvqxZsS1lCgoKjBo1ivXr1zNs2DCMjIyK5bl7926py/vYAfP29vZ4enpiYGBA586dZe6oLPSulrpz58590BVnXl5etX/EgPDPpqgkRw9nFVRqFvyroFA5692/fz/Pnj0jOjpaau0p7LZ832/mfa3qhV2Y0dHRKCsrl8uA+o/1scegt2nbti2qqqpcunSJDh06SOnx8fE0a9bso+osVE0iKKum5GrXQmlQL3DsQf6rjIIuSyXFCg/ICrm7uxMXF4eLiwvjx4/H2tqa2rVrEx8fz9atWzl16hTOzs6lKqtWrYJHePz999+0bNlSSj958mSxvIaGhjItcgBdu3YlPz+foKAgfH19y7Qdubm5xMfH4+rq+tY82dnZMuNOnj59SkhICFlZWfTt27dM6xOEyqSoBErKBRc9KjXlKu0iolGjRrx+/ZpDhw7Rvn17bty4IT2LsHDs1cewt7dn1KhRKCkpsWjRoo8u720q4xgEBReGN27ckElr3rw5zZs3x93dnfXr19OwYUNMTU2Jiori1KlTMg/cFv45RFBWjcnVVIGaIFen8u8AlJeXZ+3atRw8eJDdu3cTFhbG8+fP0dTUpEOHDmzduhVLS8v3PqkaoFOnTpiZmTF06FD8/Pyk9HHjxhXLu3TpUgYOHCiTVqNGDWxtbfntt9/o0aNHmbbj2bNnZGdnv/OZRhcvXqRLly5AwVW6qqoqRkZGBAUFYWxsXKb1CUJlerMVurIe42JnZ0dsbCzLli0jPT2dJk2aMHjwYI4ePcrly5fR1tb+qPItLCzQ1NRER0enWIBUnirjGASU+OaByZMnM2XKFCZOnEjNmjVZs2YN9+/fR19fn4CAgFI9ukeofuTyRf9LpcjIyCApKQk9Pb0KG18lCIIgCELJqsN5WDwSQxAEQRAEoQoQQZkgCIIgCEIVIIIyQRAEQRCEKkAEZYIgCIIgCFWACMoEQRAEQRCqABGUCYIgCIIgVAEiKBMEQRAEQagCRFAmCIIgCIJQBYigTBAEQRAEoQoQQZkgCIIgCEIVIIIyoULMnz8fQ0NDmY+NjU25ruPMmTMYGhqW6v2aANevX+c///nPR60zPDwcQ0PDd+Z5c7uLflJTU0lJScHQ0JAzZ86Uad0BAQHlug+fPn3Kzp07y628kuTn57Nnzx4eP35coesR/h1sbGwICAiQpo8fP05CQgJQ/HhQNG9pfrefgqGhIeHh4W+dX7hNbdq04cmTJ8XmZ2Vl0aFDB5ntHjlyZInv0nybsu6nwmPZ4cOHS5w/duzYUm9XaY/dwcHBjBw5slj6sWPHcHZ2xsLCAhsbG5YvX05GRkapyqyqxAvJq7G818+Rr1nnrdOf0rVr1/Dw8GDEiBFSmoKCQrmuw8LCgpiYGOrVq1eq/BMmTMDJyQlra+tyrcebYmJiZKbT0tIYMWIE3bt3p3HjxqU+EL3Jzc2N4cOHl0cVAVixYgUpKSkMHjy43Mp807lz5/Dy8uLo0aMVtg7h7fIy8yGvSII8yNeonJeSV7Q7d+7g4eFBWFgYLVu2fOfxwMHBga5du36CWpYPeXl5fvvtN7788kuZ9JMnT5Keni6TFhAQUKZj7a5du6hRo0aZ6qOkpER0dDRffPGFTPrTp0/LfLH5Ptu2bWPt2rV06NBBJv38+fNMnjyZqVOnYmdnx82bN1m4cCHPnj1j6dKl5VqHyiRayqqp3BcPeRa5iNwXD0uc/pTy8/NJSEjA2NgYLS0t6VPa4Km0lJWV0dLSKvdg72MV3WYtLS3WrVuHhoYG33777UeVq6qqWq77MD8/v9zK+pTrEEqWn51P5n9zSduQKX0y/5tLfvY/42/y5nfrXccDFRUVtLS0Kqtq5c7KyopDhw4VSz948GCxYKVu3bqoqamVuux69eqhqqpa5vocP36czMxMmfTDhw9jbm5eprLe5v79+3h4eLBy5Up0dXWLzf/ll1/o2LEjHh4e6Orq0r17d2bMmMG+ffvIysoqlzp8CiIoq4byXj8n7cASMq+f5PFWD7JS43i81YPM6ydJO7CEvNfPK7wOL168YMGCBXTq1In27dszatQoLl++DMCtW7d49eoVLVq0KFOZNjY2hISEMH78eMzMzLCxseHIkSMcOXKE3r17Y25uztixY6WusDebwP/66y9cXFywsLDA0tKSKVOmkJqaKpV9584dAgMDpWZwQ0ND1q1bR48ePejSpQvJycmkpqYyY8YMrKysaNu2Ld26dcPPz4+8vLySK/0eMTExHD58mG+//RZlZWWZeRcvXqRfv34YGxszcOBATp8+/c6yinZfFnaBRkdHM3jwYIyNjbGxsWHHjh1S/sePHzN16lQ6duyIqakpQ4cO5ezZswB4eXmxZ88ezp49K3VXjBw5kgULFjB48GA6dOhAZGQkXl5exboN3kx79OgRs2fPpmPHjrRv354JEyZw8+ZNzpw5w6hRowCwtbV9Z3eGUP7yXubzYlsmWbG50ufFtkzyXlZOUGZoaMiOHTtwcXHBxMQEe3t7Lly4wI4dO7C2tqZdu3ZMnz5d6m4qqevsbd1pKSkp2NraAjBq1CgCAgLe2SX2ZjmGhobs2rULV1dXTE1N6dKlC4GBgTLL7Nu3D3t7e0xMTBg8eDBhYWHFynjzO100LS8vj+DgYHr37o2xsTHt2rXD3d2dW7dulWU3AmBvb8/Zs2dlujAzMjI4duwYDg4OMnmLdl+Gh4fTq1cv6d/CY82ff/4p5X+zS7g0rK2tycvL4/fff5dJP3DgQLH6fKjY2FiUlJSIjIzEzMys2Hw3Nzc8PT1l0uTl5cnOzi7WelidiKCsGpKvWQd1h7ko1Ncl93Eyj38cSe7jZBTq66LuMLfCuzDz8/MZN24ct2/fJjg4mF9//RVzc3OGDRtGXFwc8fHxAPz000/Y2NjQs2dPfH19efHixXvL/v7773FwcGDfvn0YGRkxe/ZsgoKC8PPzIygoiMuXL7Nhw4Ziy+Xm5jJhwgQsLS2JjIxk8+bNpKamMnfuXKCgib5Ro0a4ubnJHIC2b9/OunXrCAwMRFdXl6+++ooXL16wadMmDh06hJubGxs3buTYsWMftK9Wr16Nra1tsatZgNDQUL766isiIiJo06YNEyZM4P79+2Uqf+nSpXh4eHDw4EGsra3x9vbm9u3bAHh7e5OZmcnWrVvZt28fenp6TJw4kVevXjFv3jzs7e2lLp9CO3fuZNSoUWzfvr1U3T05OTm4ubmRkJDA999/z6+//kpeXh7u7u5YWFhI+3rnzp3ldrAW3i/vVT7pEdnkyzZkkJ8J6ZHZ5L2unMBszZo1uLu7ExERgZqaGh4eHkRHRxMSEsLSpUs5cuTIB41r1NbWlpYLCAjAzc2tzGUsX74cJycnoqKiGDFiBAEBAZw7dw4oGKvm6enJoEGDiIyMZODAgaxcubJM5YeFhREaGoqXlxfR0dGsX7+e5ORkli1bVua6WlpaoqGhwZEjR6S048ePo6Ojg76+/juXvXv3Lr/88gt+fn7s2bOHmjVr4uXl9VGt2DVr1sTa2pqDBw9KaY8ePeLChQvY2dl9cLlFFQaLOjo6Jc5v06YNRkZG0nR2djabN2/G2Ni43HtlKpMIyqopBTUt6jrKdofVdfwWBbWKb6I/ffo0//3vf1m7di1mZmbo6+szc+ZMzM3NCQsLIz4+Hnl5eRo0aEBQUBBeXl7ExMQwceLE97Y4WVtbM2DAAJo1a8aQIUN4+fIlM2bMwNTUlE6dOvH5559z/fr1Ysulp6fz9OlTGjRoQJMmTWjbti1r165l+vTpQEETvYKCArVq1aJu3brSco6OjpiYmGBubk5GRgaOjo58++23GBkZoaOjg6urK5qamly7dq3M++ncuXPExsYyceLEEudPmTIFBwcH9PX18fb2pn79+mzfvr1M63B1dcXW1hYdHR1mzJhBXl4ely5dAgpaLOvUqYOOjg7Nmzdn3rx5rFu3DgUFBdTU1FBRUUFJSUmmW6d169b069cPAwMDNDQ03rv+P/74g2vXrrFq1Srat2+Pvr4+3333HT179iQ9PR11dXWgYP+rqKiUaduEj5APGX/klDgr4/9yZMeZVSBnZ2dsbGxo0aIFjo6OpKWlsXDhQgwMDOjduzetW7cu8ff8PgoKCtKJV11dvczdbwADBgzA0dERHR0dPDw8qFOnDhcuXAAKLpjs7OwYO3Ysenp6DBs2jGHDhpWp/GbNmrF8+XJ69OhBkyZNsLKyws7OTrpoLQs5OTl69+4t04V58OBB+vTp895ls7Oz8fHxwdzcnFatWjFmzBhu3brFw4cfN9TF3t6e48ePS12Fhw4d4rPPPvskAVFOTg6zZ8/m+vXrLFq0qNLXX57EQP9qKvfFQ55FLJBJexaxgPojgio8MIuNjSU/P58ePXrIpGdlZZGZmcmSJUtwcXGRTuoGBgZoaWkxZMgQLl++zB9//EFwcLC0XL9+/fD19QWgefPmUnrNmjWBgoNbIRUVlRLv5FNXV8fd3Z1vv/2WdevW0alTJ7p37469vf07t6Xo+lRUVBgxYgSHDh3ir7/+4ubNm1y7do1Hjx6VGExGRkbKHADat2/Pxo0bpek9e/ZgampK27ZtS1x3+/btpf8rKirSpk0brl+/TmpqarGD7cWLF0sso+hVcuE4kuzsbAAmT57MN998Q3R0NO3bt6dLly707dv3nYN6i+6P0oiPj0ddXR09PT0prWHDhsW6FYRKJgcqVooFAdgbVD5XrLTL8dL8nj/V+J83W5jU1NSk305sbGyxQeyWlpZs3ry51OXb2Nhw6dIl/P39SUpKIikpiYSEBBo2bFhifgsLC5npqKgomWl7e3tGjx7N06dPUVZW5uTJk3zzzTfSEI13eddx4kN1796d/Px8fv/9d2xtbTlw4ACDBg0qlu992/Wx0tPTmT59OmfPniUwMBBTU9NyLb+yiaCsGiocU1bYZVnX8VueRSwg93EyaQeWULe/T4V2Yebl5VG7du0SxwgpKysjLy9frJWlVatWANy7d4+hQ4fKBEu1a9eW/q+oWPwrKSdXurvFZs2ahYuLCydOnOCPP/7g22+/ZePGjezdu7fYeK5CRVtvXr16xYgRI8jIyMDOzg4nJydMTU3fesejjY2NzFiHomXl5eVx7Nixt7aSQfG7UXNzc6lRowYNGjRg7969pdnkErersFuiV69e/P777/z+++/83//9H5s2bSIwMJBff/1V+nu8qTStWTk5/zvRl/T3Ej49+Vpy1HZUIvPPHJkuTLkaULu/EvI1K+cOzJK+H/LypY8Ic3Nzy7M6Mt7121FUVCzzONKivwuAkJAQ1q9fj5OTE1ZWVri6unL06NG3BiVv/uYbNGggDUWAgou4+vXrc+TIEVRUVDAwMEBHR6dUQdm7tvVDqaioYGNjw6FDh2jbti2xsbEyF9uF3rddH+PBgweMGzeOO3fuEBoaiqWlZbmU+ymJI2o1VDimLO0ABWPL1LSoPyKItANLKmVMmYGBAenp6WRnZ9OyZUspff78+RgZGfHXX3/x4MEDmavKwpsAWrZsSd26dWW6EMvDjRs32LJlC3PnzpW6Gv78809cXFy4evVqqa6eYmJiiI2N5dSpU2hqagLw7NkzHj9+XOIBrHbt2jIBZVEJCQk8ffqUzz///K3r+/vvv6UxEVlZWfz9998MHToURUXFMrdYvSkrK4tVq1bh6OiIg4MDDg4OZGRk0LlzZ/7zn//QqlWrUgW7SkpKxQbN3rx5UwreWrZsSVpaGjdv3pTq/OTJE+zt7QkODi51QC2UP3lVOdSG1yDjzP+CBZWOisjXrpp/EyUlJaCg5aPwd5WcnPzW/BX53TIyMpKGARR6s7X6zd/GzZs3ZeYHBQUxadIkxo8fL6WFhoa+NRh632++sAszOjoaZWXlKjFG097eHk9PTwwMDOjcuXOJd31+7LHsbdLS0hg9ejTp6els27atSj6H7kOIoKyaUlDTkmkRe3O6InXt2pXWrVszY8YM5s2bh7a2Ntu3byc8PJzQ0FC0tbWZOHEigYGB9O/fn6SkJHx9fenbt+97B6V+KA0NDaKiosjIyGD8+PHIy8uzZ88e1NXVpbtAVVVVSU5O5tGjR1LQVVSjRo2Agm7J3r17c/fuXVavXk12dnaZu1ji4uJQUlJ65x2oq1atom7duujq6vL999+TlZVVbs8hU1ZW5vLly5w/f54FCxagqanJyZMnefXqldSdUKtWLR48eMDt27ffOpjW3NycXbt2ERkZiYWFBZGRkcTHx0tBrpWVFcbGxnh6ejJ37lxq1qzJihUrqFevHm3btpXG4l29ehUNDY0PGvsjfBg5JTlqmCtQw7RIi6w8yClWzaDM3NwcOTk5AgICGDlyJJcvX2bPnj1vzV+rVi2goAu9TZs25VqXcePGMWHCBExNTenRowd//vknW7duLVbfnTt3YmlpSX5+PkuXLpVpkdLW1ubUqVPY2NggLy9PREQEhw8fLvHYU1r29vaMGjUKJSWlCh07dfLkyWJphoaGxbpeu3btSn5+PkFBQdIQlLI4d+4cN27ckElr3rx5qQK5pUuXcvv2bTZu3Ei9evVkxsgVjiGujkRQVo29GYBV1oNjFRQU+PHHH/Hz82P69Om8fv0afX19AgMDsbKyAmDt2rWEhISwYcMG1NTU6NevnzToviJoaGiwYcMGVq1axZAhQ8jNzcXc3JxNmzZJV90jR45k+fLlXL9+ncjIyGJlmJqaMmfOHDZv3szatWtp2LAhDg4OaGtrSy19pfXw4UPU1dXf2VUzZcoUVq5cSUpKCqampmzatKlcWxDXrFnD0qVLpTtKW7RowcqVK6U7QQcMGMBvv/1G37593/p07v79+3PlyhW+++47cnJypHEtha0G8vLyfP/99yxdupQxY8YgJydHp06d2LhxI0pKShgYGNC9e3emT5/OzJkzP+guOeHDVacHxero6ODj40NwcDDbt2+nffv2zJ49+63jEzU0NHB2dmbFihXcvHmTXr16lVtdunXrhq+vL8HBwaxatQpjY2OGDRsmE5h5e3vj7e3NkCFDaNCgAdOmTePevXvS/BUrVuDr64uzszOqqqqYmZnh4+ODt7c3qampNG7cuMz1srCwQFNTEx0dnbeOTSsP48aNK5a2dOlSBg4cKJNWo0YNbG1t+e2334qNMS6Nkt48MHnyZKZMmfLO5XJzczlw4ADZ2dmMHj262PyjR4/StGnTMtenKpDLF093rBQZGRkkJSWhp6cn7kITBEGows6ePYumpqZMS3dQUBC7du2SeSyFUL1Uh/OweCSGIAiCIBQRExPD2LFjOX36NKmpqRw9epQtW7bg6Oj4qasm/MOJ7ktBEARBKGLy5Mm8evWK2bNn8+TJE7S1tXF1dcXd3f1TV034hxPdl5WkOjSbCoIgCMI/VXU4D4vuS0EQBEEQhCpABGWCIAiCIAhVgAjKBEEQBEEQqgARlAmCIAiCIFQBIigTBEEQBEGoAkRQJgiCIAiCUAWIoEwQBEEQBKEKEEGZUCXY2NgQEBAgTR8/fpyEhAQAzpw5g6GhISkpKcXyhoeHY2hoWPkVfg9DQ0PCw8M/qozc3FxMTU0xNDSU+RTdT296c19VNeWxX4oq+j2pKKmpqURFRVXoOipafmZupa/zzd90Wb35vTc1NaVfv34lfn9+//13Ro4cSbt27TAzM6Nfv36EhISQnZ0t5QkICChWZtHPoUOHPriuglBexBP9q6msjDTycjOLpcsr1EBZRf0T1Kj83LlzBw8PD8LCwmjZsiUWFhbExMRQr169YnkdHBzo2rXrJ6hlxUtOTiYzM5OIiAjq168vpdeqVesT1urjxMTEoKamVi5lvfk9qSienp40adKEPn36VNg6KlJ+Rg55ic+R16+DnEr1OuTPnTsXBwcHAF69ekVMTAzz58+nXr16WFtbA3Dq1Cm++uorZsyYgbe3N4qKily4cIGlS5eSlJTE0qVLpfIaNWrErl27SlyXunr1Pm4K/wzV6xcqSPJyM4nZal8svcuIg5+gNuXrzZdMKCsro6WlVWJeFRWVKvtk5o917do1ateujZGR0aeuSrl529/xQ4iXkbxfflYu+S9zyAqKo4Z3B5CXQ05Z4VNXq9TU1NRkvjPNmzfn2LFjhIeHS0HZjh076Nq1K2PHjpXJl5GRga+vL3PmzKFOnToAKCgolOt3UBDKm+i+FMrM0NCQHTt24OLigomJCfb29ly4cIEdO3ZgbW1Nu3btmD59OhkZGUDJXYxv63ZMSUnB1tYWgFGjRhEQEPDOLrk3yzE0NGTXrl24urpiampKly5dCAwMlFlm37592NvbY2JiwuDBgwkLCytWxptdJEXT8vLyCA4Opnfv3hgbG9OuXTvc3d25detWWXYjFy5cYPjw4ZiammJtbY2Pjw/p6enS/GvXrqGvr1+mMt+UlpbG/Pnz6dq1K23btsXKyor58+fz+vVroKC7s02bNoSEhNCxY0cGDhxIXl4et27dYty4cVhYWNC1a1c2bdpEr169ZPbL7t27sbe3x9TUFHt7e7Zs2UJeXt4761N0P3p5eeHl5cXy5cuxsrLCzMyMCRMmcP/+fSn/3r176dOnDyYmJnTt2pXFixeTlZX11u/Jm9ty+/ZtDA0NOXPmjFRmSkpKsbTIyEj69++Pqakptra2bNmyBYCRI0dy9uxZ9uzZg42NzUf9LSpTfkYOeSnpZG2+RqbfJcjJJ9PvElmbr5GX8pL8jJxPWr+srCyWL1+OjY0NxsbGfPbZZ0ybNo0nT568d9k3L8Lk5OS4evWqzPcGYMCAAezfv79atywL/z4iKBM+yJo1a3B3dyciIgI1NTU8PDyIjo4mJCSEpUuXcuTIEXbu3FnmcrW1taXlAgICcHNzK3MZy5cvx8nJiaioKEaMGEFAQADnzp0DCsYgeXp6MmjQICIjIxk4cCArV64sU/lhYWGEhobi5eVFdHQ069evJzk5mWXLlpW6jKtXrzJmzBi6du1KZGQkK1euJDY2Fjc3N6kFKD4+npycHMaOHUvnzp0ZOHAgERERZaqrl5cXcXFxBAYGEh0dzZw5c9i7dy87duyQ8uTm5nLixAl27NjB4sWLyczMxNXVlby8PH7++WfWrFlDeHg4t2/flpbZsWMHK1asYPLkyURFRTF9+nQ2bNhQ5n25f/9+nj17xtatW9mwYQOxsbGsXbtW2kfz589nypQpREdHs2TJEiIiIti4ceNbvydvboucnNx763DgwAE8PT1xdHQkMjKSmTNnsnLlSsLDwwkICMDCwgJ7e/u3dntVRXIqishp1EBOsyakZRUkpmUhp1kTOQ3lT96NuWLFCg4fPsyyZcuIjo5m2bJlnD59mh9++OGty+Tl5RETE8OpU6dwdnaW0kePHs3jx4+xsbFh9OjRBAYGcvbsWZSUlNDX10dRUXQICdWH+LYKH8TZ2VlqOXB0dMTX15eFCxeiq6uLgYEBGzdu5Pr162UuV0FBQRo7pq6ujqqqapnLGDBgAI6OjgB4eHgQGhrKhQsXsLS0JDQ0FDs7O6mrQ09Pj+TkZDZv3lzq8ps1a8by5cvp0aMHAE2aNMHOzq5MA4VDQ0Pp3LkzHh4eAOjq6rJq1Sp69uzJ2bNn6dixI9evXycvL4+pU6fSqFEjTpw4wZw5c8jOzmbQoEGlWk/nzp2xtLSUWgKbNm3K1q1biY+Pl8nn5uaGrq4uUNAC9uTJE8LDw6lbty4Afn5+0j4F+P777/nqq6+kcVY6Ojqkp6fj4+PDtGnTqFGjRqnqp6amhq+vr3QCdXBw4MSJE0BBi5acnBxNmjShcePGNG7cmNDQUGrXrv3O70nRbSnNDQ9btmzBwcFB+k7o6ury8uVLVFRUqFu3LkpKSqioqJQ4prEqk1NVQtGmMbm//W8fKNk0Rk5V6RPWqoCJiQl2dnZ06NABKPgNff7558W+l4sWLeLbb78FIDMzk9zcXHr27ImVlZWUp127doSHh7Np0yZOnDjB6dOnAWjQoAGLFi2iZ8+eUt7U1FQsLCyK1UdDQ4Njx46V+3YKQlmJoEz4IM2bN5f+X7NmTaAgWCmkoqJCVlZWpdcLKNblp6amJt2FFRsbyxdffCEz39LSskxBmY2NDZcuXcLf35+kpCSSkpJISEigYcOGJeZ/8yQQFRVFXFwcN2/eLPEEkZiYSMeOHdm/fz+5ublSwGFkZERqaiqhoaEMGjSIPn36kJqaKi23YcOGYmW5uLhw7Ngx9uzZQ3JyMgkJCaSkpNCiRQuZfIVBDEBcXBx6enpSQFa47sIB+k+ePOHevXusXr0af39/KU9eXh6ZmZmkpKSwZcsW9u3bJ82bMGGCFIAW1axZM5SU/hckFP1bde3aFQsLCwYNGkTTpk3p3Lkztra2GBsbFyvnbdtSGvHx8cUG8Q8ZMqRMZVRV+U8yUbDUQtG2CTlH7pD3NBOFuqULmCuSo6Mj//d//8fKlStJTk7mxo0bJCUlSUFaoalTp0q/16ysLK5fv46fnx+TJk2S+b63bNmSxYsXAwW/n99//52tW7cybdo0mSEODRo04KeffipWH3l50WkkVA0iKKum5BVqlDioX16hcg64JXUJlOXAlptbcbfoKysrF0sr7BJUVFR877inN+XkyI6/CQkJYf369Tg5OWFlZYWrqytHjx5962MT9u7dKzPdoEED8vLy6NevX4mBSmGLTEk3MBgYGBAZGSnVo2jdGjZsyKVLl6TpvLw8JkyYwPXr1+nbty8ODg60bduWBQsWFCu3aMuWgoLCO/dR4bw5c+bw+eefF5uvra3NtGnTZAZev+3OtpL+VkXrFBYWRlxcHDExMcTExODh4cGAAQNk7qh717aU5M3v3j+5e0u+WW3kh7VErqYiSi4tQalqBB8LFy4kOjqaAQMGYGNjw6RJkwgNDS02Lqx+/foyF4CtWrUiJyeHb775huvXr9OkSRNWr16Ns7MzrVu3BgouyvT19enfvz89evQgJiZGCsoUFRVlyhOEquafezT6h6tOj70obAlJT0+ndu3aQMHjHt6mNOOAPpSRkZFM4AJw8eJFmWklJSWZAfc3b96UmR8UFMSkSZMYP368lBYaGvrWuwFLOgm0atWKhIQEmXmJiYn4+fkxc+ZM8vPz6dmzJ15eXgwcOFDKc/nyZVq1agUUdPm8y5UrVzh58iS//vorZmZmAGRnZ3Pr1i10dHTeupyRkRG//vorz549k1rLEhMTefHiBVBwoqxXrx63b9+Wqf+BAwf47bffWL58OfXr15d5jMeHOHHiBJcvX2by5Mm0adOG8ePH88MPPxAUFMTSpUtL9T0p+t0r9OZ3T19fn8uXL8ukLV26lLt377Ju3bqP2oZPTU5RHhQLAjG5mlXjcP/06VN27NjBmjVrpMddANy4caNUg/ILf2d5eXmoqKiwb98+srOz8fHxkcmnqqqKgoLCR38PBaEyVY1fqfCPZm5ujpycHAEBAYwcOZLLly+zZ8+et+YvPDDHx8fTpk2bcq3LuHHjmDBhAqampvTo0YM///yTrVu3Fqvvzp07sbS0JD8/n6VLl8q06Ghra3Pq1ClsbGyQl5cnIiKCw4cPo6mpWep6uLm5MXz4cHx8fBgxYgTPnz/Hx8eHjIwMdHV1UVZWplOnTqxZs0ZqLTh8+DCRkZEEBweXah2ampooKipy8OBB6tWrx7NnzwgKCuLhw4fv7Fru27cvAQEBzJo1i1mzZkmPFoCCgFlOTo5x48axZs0aGjduTLdu3bh27Rre3t7Y2tq+s/WrLJSUlFi/fj21a9fG1taWtLQ0/vOf/0hdvqX5njRo0IAmTZqwZcsWdHV1efbsGf7+/jIB3fjx45kyZQqmpqZ0796dS5cu8fPPP0vbrKqqyp07d7h37x6NGjUql237t7h58yYnT56USVNRUUFNTY2jR4/Stm1bMjIy2Lp1K7GxsdLFQ6EXL17w8OFDoCAIu379Ov7+/rRp0wYDAwPk5OSYNWsW8+fPBwrGutarV49bt27x448/oq2tjZ2dnVRebm6uVN6batasKV00CsKnIoIyocLp6Ojg4+NDcHAw27dvp3379syePRtPT88S82toaODs7MyKFSu4efMmvXr1Kre6dOvWDV9fX4KDg1m1ahXGxsYMGzZMJjDz9vbG29ubIUOG0KBBA6ZNm8a9e/ek+StWrMDX1xdnZ2dUVVUxMzPDx8cHb29vUlNTady48XvrYW5uzsaNG/H398fJyYlatWphZWWFp6enFNQsWbKEgIAAFi1axOPHj9HX12fdunWlflhuw4YNWbZsGQEBAWzbtg0tLS2sra1xdXV956BmZWVlNm7ciK+vL0OGDEFdXR0PDw9iY2Ollic3Nzdq1KjBTz/9xLJly9DU1GTIkCFMnTq1VHUrjc8//5zFixfz448/smbNGlRUVOjevTteXl5A6b4ncnJyrFixgiVLluDo6Ejz5s2ZM2eOTCunjY0Nvr6+bNiwgeXLl9OkSRPmzJnDgAEDABg6dCienp7079+fP/74AwWF6vOcr09t3759MmMLoaCF19/fn2XLltGvXz/U1dXp2LEjM2fOJDg4mNevX0vjVJcsWcKSJUsApFavzz//nJkzZ0qB9eDBg9HS0mLLli2MGzeOly9foqmpia2tLStWrJAZBnDv3j26dOlSYl2HDx/OwoULK2I3CEKpyeWLJzBWioyMDJKSktDT0/vHPuy0Ojh79iyampoyA92DgoLYtWsXR44c+YQ1qzpSUlJITk6WOXndv3+fbt26sW3btmKDsQVBEKqD6nAerhqjPgWhksTExDB27FhOnz5NamoqR48eZcuWLTKPe/i3y8zMZPz48YSGhnL79m3i4uJYsGABurq6xbqXBEEQhPIjui+Ff5XJkyfz6tUrZs+ezZMnT9DW1sbV1RV3d/dPXbUqQ19fn9WrVxMUFMS6detQUVHBysqKTZs2yTy+QhAEQShfovuyklSHZlNBEARB+KeqDudh0X0pCIIgCIJQBYigTBAEQRAEoQoQQZkgCIIgCEIVIIIyQRAEQRCEKkAEZYIgCIIgCFWACMoEQRAEQRCqABGUCVWCjY0NAQEB0vTx48dJSEgA4MyZMxgaGpKSklIsb3h4OIaGhpVf4fcwNDQkPDz8o8rIzc3F1NQUQ0NDmU/R/fSmN/eVIHwqI0eOLPbdLfp58uTJB5VbUd/x0pQ7cuRI6TVflfVbu3HjBjNmzMDKygpjY2NsbGzw8fHh0aNHUp6AgABsbGwqtB5C5RAPj63mMjLTyMnNRFGhBio11D91dcrFnTt38PDwICwsjJYtW2JhYUFMTAz16tUrltfBwaHU74KsbpKTk8nMzCQiIoL69etL6YUv4haEqs7e3p558+aVOE9DQ6OSa1O+3nVcKi+PHj3CxcWFHj16sHHjRtTV1UlKSmLFihWMHDmSiIgIlJWVcXNzY/jw4RVWD6HyiKCsmsvJzWTrHjtGOB361FUpN28+z1hZWRktLa0S86qoqFTZhwB+rGvXrlG7dm2MjIw+dVUE4YOoqKi89bdb3b3ruFReDh06RE5ODkuWLJFewN60aVMaN26Mg4MDv//+O7a2tqiqqqKqqlqhdREqh+i+FMrM0NCQHTt24OLigomJCfb29ly4cIEdO3ZgbW1Nu3btmD59OhkZGUDJXYxv63ZMSUnB1tYWgFGjRhEQEPDOboI3yzE0NGTXrl24urpiampKly5dCAwMlFlm37592NvbY2JiwuDBgwkLCytWxptdj0XT8vLyCA4Opnfv3hgbG9OuXTvc3d25detWWXYjFy5cYPjw4ZiammJtbY2Pjw/p6enS/GvXrqGvr1+mMt+UlpbG/Pnz6dq1K23btsXKyor58+fz+vVroKALpk2bNoSEhNCxY0cGDhxIXl4et27dYty4cVhYWNC1a1c2bdpEr169ZPbL7t27sbe3x9TUFHt7e7Zs2UJeXt5H1Vf497GxsSEkJITx48djZmaGjY0NR44c4ciRI/Tu3Rtzc3PGjh3L48ePZZY7duwYPXv2xMTEhJEjR3L16lVpXn5+Phs2bMDW1hYzMzMcHR2JjIyUWf78+fMMHjwYU1NT+vfvL7M8QFZWFkuWLMHKyor27dvj5+cn8/0uaVhFaGgoU6ZMwcLCgo4dO/Ldd9+Rk5MjLRMTE4OTkxMmJib07duX3bt3v7MLVE5OjpcvX3Lu3DmZdH19faKioujUqRMg233p5eVVYndx0e7N48ePM3DgQExNTenVqxdr164lKyvr3X8ooVKIoKwayshMI/3VA9JfPeDlq/sAvHx1X0rLyEyr8DqsWbMGd3d3IiIiUFNTw8PDg+joaEJCQli6dClHjhxh586dZS5XW1tbWi4gIAA3N7cyl7F8+XKcnJyIiopixIgRBAQESAe148eP4+npyaBBg4iMjGTgwIGsXLmyTOWHhYURGhqKl5cX0dHRrF+/nuTkZJYtW1bqMq5evcqYMWPo2rUrkZGRrFy5ktjYWNzc3KSWwvj4eHJychg7diydO3dm4MCBRERElKmuXl5exMXFERgYSHR0NHPmzGHv3r3s2LFDypObm8uJEyfYsWMHixcvJjMzE1dXV/Ly8vj5559Zs2YN4eHh3L59W1pmx44drFixgsmTJxMVFcX06dPZsGFDmfelIAB8//33ODg4sG/fPoyMjJg9ezZBQUH4+fkRFBTE5cuX2bBhg8wyP/74I4sWLWL37t2oqqri7u4uXWysWbOGn3/+mQULFrBv3z5GjRqFt7c327ZtA+D27du4ubnRunVr9uzZw6RJk/j+++9lyv/uu+84cOAAy5Yt45dffuHevXucP3/+ndvh7++PpaUlkZGRzJ49m61bt7J//34Arly5woQJE7CysiIiIoKvvvqK5cuXv7O8Pn36oK2tzciRIxkwYADLli3jyJEjpKen07JlyxJbx+bNm0dMTIz08ff3R0FBgSlTpgBw8uRJpk+fzpAhQ9i/fz+LFi3i4MGDfPPNN++si1A5RPdlNVTYZVnUnujR0v8royvT2dlZuvJydHTE19eXhQsXoquri4GBARs3buT69etlLldBQUEao6Gurv5BTfIDBgzA0dERAA8PD0JDQ7lw4QKWlpaEhoZiZ2fH2LFjAdDT0yM5OZnNmzeXuvxmzZqxfPlyevToAUCTJk2ws7Pj0KHS7/fQ0FA6d+6Mh4cHALq6uqxatYqePXty9uxZOnbsyPXr18nLy2Pq1Kk0atSIEydOMGfOHLKzsxk0aFCp1tO5c2csLS2llsCmTZuydetW4uPjZfK5ubmhq6sLFLSAPXnyhPDwcOrWrQuAn5+ftE+h4CT61Vdf0adPHwB0dHRIT0/Hx8eHadOmUaNGjVLvC+Gfa9++fURHRxdL79mzJ35+ftK0tbU1AwYMAGDIkCEcPXqUGTNmYGpqCsDnn39e7HiyYMECaTzpihUr6N69O/v376dPnz5s3ryZ1atXY21tDRT8Zu/cuUNoaCjDhw/n119/RVNTk0WLFqGgoIC+vj53795l6dKlAKSnpxMeHs6iRYvo3r07AEuWLOH06dPv3N4uXbowatQooOA38dNPP3HhwgUGDBjA5s2bMTY2Zvbs2QC0aNGCx48fs3jx4reWV7duXcLDw9m0aROHDx9m06ZNbNq0CRUVFcaPH8+kSZOKLaOmpoaamhoAt27dYtGiRbi5ueHk5ARAUFAQQ4YMYejQodK+8fHxYfTo0aSkpNC0adN3bqNQsURQVg0pKtSQAq+Xr+6zJ3o0Tr23oFqroTS/ojVv3lz6f82aNYGCH3chFRWVT9Yc/maXn5qaGtnZ2QDExsbyxRdfyMy3tLQsU1BmY2PDpUuX8Pf3JykpiaSkJBISEmjYsGGJ+S0sLGSmo6KiiIuL4+bNm8XmASQmJtKxY0f2799Pbm6uFJgaGRmRmppKaGgogwYNok+fPqSmpkrLvdmSAODi4sKxY8fYs2cPycnJJCQkkJKSQosWLWTyFQZkAHFxcejp6UkBWeG6Cw/0T5484d69e6xevRp/f38pT15eHpmZmaSkpHx0t6vwz2BjY8OsWbOKpb95s0ppjidvdl+2b99e+n+dOnXQ1dUlPj4eQ0NDMjMz+frrr5GX/19nUE5ODllZWWRkZBAfH0+bNm1QUFCQ5rdr1076f1JSEtnZ2ZiYmEhpNWrUoE2bNu/c3ncde+Li4vj8889l5ltaWr6zPCgIzGbMmMGMGTN48OABf/zxBzt37mTdunVoaGjg4uJS4nJpaWmMHz8eS0tLvv76ayk9Li6Ov/76i127dklpha3ziYmJIij7xERQVg2VdJelaq2G1K7VoNLqoKhY/KtT9AD4Prm5ueVZHRnKysrF0goPOoqKimUe91R0TAhASEgI69evx8nJCSsrK1xdXTl69ChRUVElLr93716Z6QYNGpCXl0e/fv2klrKiClsKS7qBwcDAQBobExISIlO3hg0bcunSJWk6Ly+PCRMmcP36dfr27YuDgwNt27ZlwYIFxcot2rKloKDwzn1UOG/OnDnFTjJQ0AUtCACqqqoyAdfblHQ8KRzY/jZFAyooOKYoKytLv/W1a9cWu/iAguODnJxcse940ToUrvvNm45KquebZb+psIz3/a5KEhISQtOmTXFwcAAKjh2Ojo7069ePL7/8khMnTpQYlGVnZzN58mRq1qzJihUrZPZlXl4e7u7uUstZUf/UmzKqEzGmTKhwSkpKADKD2JOTk9+a/30H449hZGQkE7gAXLx4UWZaSUlJpq43b96UmR8UFMSkSZPw9vbmyy+/xNzcnOTk5GIH8ELNmzeX+SgqKtKqVSsSEhJk0nNycli6dCl3797l+fPnfPbZZ8VuOLh8+TKtWrUCCrpNiy7/ZhB35coVTp48ib+/P7NmzaJ///40a9aMW7duvbWuhfvo5s2bPHv2TEpLTEzkxYsXANSvX5969epx+/ZtmfXHxsaydu3at5YrCOXp77//lv7/5MkTkpOTadWqFS1atEBRUZHU1FSZ7+eJEycIDQ1FXl4eIyMj/v77b5nW/KLl6enpUaNGDS5cuCCl5eTkFLsZoCyMjIz466+/ZNLePPa86a+//uKHH34odmEoLy9P7dq1ZR6VU9TChQtJSkrihx9+KNYq2apVK5KSkmT2zb1791ixYgUvX778gC0TypMIyqq5wq7Myuiy/FDm5ubIyckREBBASkoKBw8eZM+ePW/NX3gQiY+PlwKB8jJu3DgOHTrEpk2bSE5OZvfu3WzdurVYfXfu3MmVK1eIi4vD29tb5gpYW1ubU6dOkZCQwI0bN1izZg2HDx8uU3etm5sbcXFx+Pj4kJiYyMWLF/n6669JTk5GV1eXOnXq0KlTJ9asWcOJEydITk4mJCSEyMhIacDu+2hqaqKoqMjBgwe5ffs2ly9fZvr06Tx8+PCdde3bty8aGhrMmjWLq1ev8t///lcaBCwnJ4ecnBzjxo3jp59+YuvWrdy6dYvffvsNb29vVFRUSmwtEP6dMjIyePjwYYmfjx3esHDhQv744w+uXLnCjBkz0NbWxsHBATU1NYYOHYq/vz8RERHcvn2bXbt24efnR4MGBb0Jw4YN4/Xr18ydO5fExESOHz8u81BmVVVVRowYwbp16zh8+DCJiYksWrSI+/fvf3B93dzcuHz5MitXriQpKYnffvuNdevWAW+/EJ00aRIpKSmMHTuWmJgY7ty5w8WLF1m2bBn//e9/GTNmTLFlgoODOXDgACtXrkRJSUlmn+fm5jJu3Diio6MJDAwkKSmJP/74gzlz5vDixQvRUlYFiO7Laq46PDBWR0cHHx8fgoOD2b59O+3bt2f27Nl4enqWmF9DQwNnZ2dWrFjBzZs36dWrV7nVpVu3bvj6+hIcHMyqVaswNjZm2LBhMoGZt7c33t7eDBkyhAYNGjBt2jTu3bsnzV+xYgW+vr44OzujqqqKmZkZPj4+eHt7k5qaSuPGjd9bD3NzczZu3Ii/vz9OTk7UqlULKysrPD09paBmyZIlBAQEsGjRIh4/foy+vj7r1q0r9cNyGzZsyLJlywgICGDbtm1oaWlhbW2Nq6srx44de+tyysrKbNy4EV9fX4YMGYK6ujoeHh7ExsZKrZ5ubm7UqFGDn376iWXLlqGpqcmQIUOYOnVqqeom/DscPHiQgwcPljjP398fOzu7EueVxsSJE5kzZw5PnjyhY8eObNy4UfrtzJkzBw0NDfz9/Xnw4AHa2tpMnToVd3d3oOC3sWXLFpYsWYKTkxPa2tp89dVX+Pj4SOV//fXX1KhRA19fX16+fIm9vf1HPTXfwMCAwMBAVq9ezebNm9HT05PuDi/8Xb2pdevW7Ny5k++//545c+bw9OlTVFVV+eyzz/jll1+kVvOiduzYQUZGBqNHjy427+jRo9jZ2bFmzRqCg4MJCgqibt26bx37J1Q+ufx39WMI5SYjI4OkpCT09PT+sQ87rQ7Onj2LpqamzFiToKAgdu3axZEjRz5hzaqOlJQUkpOT6dKli5R2//59unXrxrZt2+jQocMnrJ0gVE9//fUXioqKMjcL7Nu3j7lz53Lx4sX3jlcTPl51OA+L7kvhXyUmJoaxY8dy+vRpUlNTOXr0KFu2bJF53MO/XWZmJuPHjyc0NJTbt28TFxfHggUL0NXVxczM7FNXTxCqpStXrjBq1CiOHj1Kamoqf/zxBwEBAfTp00cEZIJEtJRVkuoQof8bZGVlsWLFCg4fPsyTJ0/Q1tZm0KBBuLu7F7ub69/s0KFDBAUFkZSUhIqKClZWVsyePbtUXbOCIBSXn5/P+vXr2bNnD/fv36d+/fr06dOHqVOninNCJakO52ERlFWS6vBlEARBEIR/qupwHhbdl4IgCIIgCFWACMoEQRAEQRCqABGUCYIgCIIgVAEiKBMEQRAEQagCRFAmCIIgCIJQBYigTBAEQRAEoQoQQZkgCIIgCEIVIIIyoUqwsbGReSHw8ePHSUhIAODMmTMYGhqSkpJSLG94eDiGhoaVX+H3MDQ0JDw8/KPKyM3NxdTUFENDQ5lP0f30pjf3lfDvlpuby5MnT6RPbm5upa378ePHfPPNN3Tq1AkLCwvGjx9PYmKiND82NpaRI0diYWGBtbU1K1eufOtLypOSkrCwsPjo35QgVHXi3Q7VXG5+Llm5mSgr1EBB7p/xRPo7d+7g4eFBWFgYLVu2xMLCgpiYGOrVq1csr4ODQ6lf0F3dJCcnk5mZSUREBPXr15fSa9Wq9QlrJVQXT58+JSoqip9//pkHDx7QoEEDhg0bRp8+fdDQ0Kjw9U+aNIm8vDxCQkJQVVXF398fV1dXDh8+TEZGBm5ubtjZ2fHdd99x69YtPD09ycvLY/bs2TLlZGdnM2vWLF69elXhdRaET020lFVjufm53H95h29OjuD+yzvk5lfeVXBFevMlE8rKymhpaZX4GiQVFRW0tLQqq2qV6tq1a9SuXRsjIyO0tLSkj6qq6qeumlDFPX36lClTprB27Vru379Pfn4+9+/fZ+3atUydOpWnT59W6PrT0tJo0qQJ3333Haampujr6zNx4kQePHjA9evX+fPPP3n27BnffPMNzZs3p2vXrvTr14/ff/+9WFkBAQHUrl27QusrCFWFCMqqqcKAzCvGlTvpyXjFuFZaYGZoaMiOHTtwcXHBxMQEe3t7Lly4wI4dO7C2tqZdu3ZMnz6djIwMoOQuxrd1O6akpGBrawvAqFGjCAgIeGeX3JvlGBoasmvXLlxdXTE1NaVLly4EBgbKLLNv3z7s7e0xMTFh8ODBhIWFFSvjzW6Soml5eXkEBwfTu3dvjI2NadeuHe7u7ty6dassu5ELFy4wfPhwTE1Nsba2xsfHh/T0dGn+tWvX0NfXL1OZhY4dO0bPnj0xMTFh5MiRXL16VZqXlpbG/Pnz6dq1K23btsXKyor58+fz+vVrKU9oaCg9e/bE2NgYGxsb1q9fLxMsHz9+nIEDB2JqakqvXr1Yu3btW7uehMqVm5tLVFSUzN+8qCtXrnDgwIEK7cpUV1dn1apVGBgYAPDkyRM2b95Mo0aNaNmypdTq/fPPP5Obm0tKSgonTpwo9sL7c+fOsWPHDpYtW1ZhdRWEqkQEZdVQ0YDsRdYzAF5kPavUwGzNmjW4u7sTERGBmpoaHh4eREdHExISwtKlSzly5Ag7d+4sc7na2trScgEBAbi5uZW5jOXLl+Pk5ERUVBQjRowgICCAc+fOAQXBhKenJ4MGDSIyMpKBAweycuXKMpUfFhZGaGgoXl5eREdHs379epKTk8t04rh69Spjxoyha9euREZGsnLlSmJjY3Fzc5OCn/j4eHJychg7diydO3dm4MCBRERElKr8H3/8kUWLFrF7925UVVVxd3eXgi4vLy/i4uIIDAwkOjqaOXPmsHfvXnbs2AEUBHTBwcH4+Phw+PBhZs2axQ8//EBkZCQAJ0+eZPr06QwZMoT9+/ezaNEiDh48yDfffFOW3ShUkLS0NH7++ed35vn5559JS0urlPosWLAAKysroqKiWLx4MbVq1aJdu3Z89dVX+Pv7Y2Jigq2tLQ0bNmThwoXScs+fP2f27NnMnz8fbW3tSqmrIHxqIiirhrJyM1lydroUkBV6kfWMJWenk5WbWeF1cHZ2xsbGhhYtWuDo6EhaWhoLFy7EwMCA3r1707p1a65fv17mchUUFKSraHV19Q/qqhswYACOjo7o6Ojg4eFBnTp1uHDhAlDQAmRnZ8fYsWPR09Nj2LBhDBs2rEzlN2vWjOXLl9OjRw+aNGmClZUVdnZ2xMfHl7qM0NBQOnfujIeHB7q6unTo0IFVq1Zx6dIlzp49C8D169d59uwZI0eOJDQ0lN69ezNnzhx27dr13vIXLFhA165dMTAwYMWKFbx8+ZL9+/cD0LlzZ5YuXYqZmRlNmzalf//+tGnTRqr/rVu3UFZWpkmTJjRu3BgHBwc2b96MpaUlAEFBQQwZMoShQ4fSrFkzunTpgo+PD4cOHRI3GFQR9+/f/6j55Wn06NHs3r2bvn37MmnSJGJjY0lPT+fGjRsMHz6cnTt34u/vT3JyMgsWLJCW8/b2xsLCgn79+lVaXQXhUxMD/ashZYUazP1srUxLGYCacl3mfrYWZYUaFV6H5s2bS/+vWbMmUBCsFFJRUflk3VlvdvmpqamRnZ0NFNzx9cUXX8jMt7S0ZPPmzaUu38bGhkuXLuHv709SUhJJSUkkJCTQsGHDEvNbWFjITEdFRREXF8fNmzeLzQNITEykY8eO7N+/n9zcXCkwNTIyIjU1ldDQUAYNGkSfPn1ITU2VltuwYYP0//bt20v/r1OnDrq6ulLQ5eLiwrFjx9izZw/JyckkJCSQkpJCixYtAOjfvz+7d++md+/etGzZks8//5zevXvTuHFjAOLi4vjrr79kgsPC1r3ExESaNm1a6n0pVIyGDRu+M/B623e1IrRs2RKAxYsXc+nSJbZu3YqysjJpaWmsW7cOgLZt26Kuro6rqyuurq5cu3aN8+fPs2/fvkqrpyBUBSIoq4YU5BRoqNqEZV02S4GZmnJdlnXZTEPVJpVyF6aiYvGvjrx86RteK3I8i7KycrG0wqBBUVGRvLy8MpWXk5MjMx0SEsL69etxcnLCysoKV1dXjh49SlRUVInL7927V2a6QYMG5OXl0a9fPzw8PIrlL2wpVFFRKTbPwMBA6kYMCQmRqVvDhg25dOkSQLGbInJzc1FWViYvL48JEyZw/fp1+vbti4ODA23btpVpoahXrx4RERFcvHiRU6dOERMTQ1hYGFOmTGHy5Mnk5eXh7u6Ok5NTsfr9U2+6qE7U1dUZNmwYa9eufWueYcOGoa6uXmF1ePLkCX/88Qe9e/eWjhXy8vK0bNmSBw8ecP/+faytrWWWKRxPlpyczO7du3n8+HGxPIsWLeLAgQNs3LixwuouCJ+SCMqqqaKB2ZKz05n72dpKC8jKSklJCYD09HTpLqrk5OS35peTk6uwuhgZGUmBS6GLFy/KTCspKckMuL9586bM/KCgICZNmsT48eOltNDQ0GJ3jRYq2qpYqFWrViQkJMjMS0xMxM/Pj5kzZ5Kfn0/Pnj3x8vJi4MCBUp7Lly/TqlUrAJo0afLW7fz777+xsrICCk6QycnJuLm5ceXKFU6ePMmvv/4qnQSzs7O5desWOjo6AERGRvLixQuGDx9O+/btmTp1KvPnz+fAgQNMnjyZVq1akZSUJFP3M2fOEBYWhre3t3hkxyemoKBAnz59iI6O5sqVK8Xmt27dmj59+pR4N3N5efToETNnzmTjxo3SI2uys7OJi4vDxsYGKLiRpajCaT09PVauXCndKFToiy++YOrUqfTv37/C6i0In5oIyqqxwsDMr9vWKv2cMnNzc+Tk5AgICGDkyJFcvnyZPXv2vDV/4Uk9Pj6eNm3alGtdxo0bx4QJEzA1NaVHjx78+eefbN26tVh9d+7ciaWlJfn5+SxdulSm9U1bW5tTp05hY2ODvLw8ERERHD58GE1NzVLXw83NjeHDh+Pj48OIESN4/vw5Pj4+ZGRkoKuri7KyMp06dWLNmjXUr1+f5s2bc/jwYSIjIwkODn5v+QsXLsTX15e6deuybNkytLW1cXBw4OnTpygqKnLw4EHq1avHs2fPCAoK4uHDh1J3c2ZmJsuXL0dVVZUOHTpw7949zp07R4cOHaR9OH36dAIDA+nTpw/37t1j3rx5NG3aVLSUVREaGhqsW7eOAwcO8PPPP3P//n0aNmwoPaesbt26Fbp+AwMDunXrxnfffcd3332Huro6wcHBPH/+HFdXVxITExk3bhxr165l4MCB3LlzBx8fH6ytrTEyMnprufXr16/UrldBqGwiKKvmFOQUqKlYtVsmdHR08PHxITg4mO3bt9O+fXtmz56Np6dnifk1NDRwdnZmxYoV3Lx5k169epVbXbp164avry/BwcGsWrUKY2Njhg0bJhOYeXt74+3tzZAhQ2jQoAHTpk3j3r170vwVK1bg6+uLs7MzqqqqmJmZ4ePjg7e3N6mpqdLYq3cxNzdn48aN+Pv74+TkRK1atbCyssLT01MKAJcsWUJAQACLFi3i8ePH6Ovrs27dulI9LHfixInMmTOHJ0+e0LFjRzZu3IiysjINGzZk2bJlBAQEsG3bNrS0tLC2tsbV1ZVjx44BMHjwYJ49e8b333/P3bt3UVdXp3fv3syaNQsAOzs71qxZQ3BwMEFBQdStWxcbGxtpvlA1aGhoMHToUOzt7aU0dXX1Cm0hK2r16tWsWrWKGTNm8OLFCzp06MC2bdto3LgxjRs3Jjg4mPXr17NlyxY0NDTo1asX06ZNq5S6CUJVJZf/tj4XoVxlZGSQlJSEnp5eiWOFhMpx9uxZNDU1pUHtUNAduWvXLo4cOfIJayYIgiBUpOpwHhaPxBD+VWJiYhg7diynT58mNTWVo0ePsmXLFhwdHT911QRBEIR/OdF9KfyrTJ48mVevXjF79myePHmCtrY2rq6uuLu7f+qqCYIgCP9yovuyklSHZlNBEARB+KeqDudh0X0pCIIgCIJQBYigTBAEQRAEoQoQQZkgCIIgCEIVIIIyQRAEQRCEKkAEZYIgCIIgCFWACMoEQRAEQRCqABGUCeUuICBAeukwgKGhIeHh4Z+wRh/uzz//5Pz58+/MU523T/jnysjI4PHjx5w4cYJffvmFEydO8Pjx42Iv+q4oZ86cwdDQsMSPra0tAOnp6SxatIhOnTrRvn17PDw8uH37tkw5UVFR9O3bFzMzMxwcHNi7d2+l1F8QPgXx8Nh/gFc5WdRSVH5/RqHMXFxcWLp0qfQybkGoDp4/f87+/fv54YcfeP36tZRes2ZNJk6cSJ8+fahTp06F1sHCwoKYmBiZtP/+979MmTKFiRMnAjBlyhTu3r3L+vXrUVVV5dtvv+Wrr74iMjISeXl5Tp8+zezZs1mwYAGdO3fm5MmTzJkzBw0NDbp3716h9ReET0G0lFVzz7Nec/TOFZ5nVc7VryAIVVtGRgZRUVGsXr1aJiADeP36NatWrSIqKqrCW8yUlZXR0tKSPqqqqixduhQnJyecnZ05c+YMf/zxB/7+/rRv3x4jIyN8fHx4+fIlycnJABw9ehRDQ0OGDh2Kjo4Ow4cPx8jIiN9//71C6y4In4oIyqq5B69f4HtxPw8ynlfqeuPj45kwYQKWlpYYGxtja2vLjz/++EFlhYeH06tXL3755Resra0xMzNj6tSp3L9/n1mzZmFhYUG3bt3YtWuXtMzIkSNZvHgxM2fOxMzMjG7duhESEkLhCyrOnDlDmzZtOHHiBH379sXY2Bg7OzuZl47n5+ezYcMGbG1tMTMzw9HRkcjISGm+oaEhAHPmzMHLy+ud23Djxg2GDh2KsbEx9vb2HDx4UJqXl5dHcHAwvXv3xtjYmHbt2uHu7s6tW7ekPCdOnGDgwIGYmZlhZWWFl5cXaWlp0vzExETGjRuHhYUFXbp04euvv+bhw4cftL+Ff7aXL1/y/fffvzPPDz/8wKtXryqpRgWCgoJ4/fo1np6eQMF7aA0MDKTfGUDLli05fvw4LVq0AKB+/fpcv36d06dPk5+fz5kzZ0hMTMTU1LRS6y4IlUUEZdXU86wMEtIe4HVuNwBeZ3eTkPagUlrMXr9+jZubG3Xr1uWXX35h//792NnZsXz5cq5cufJBZaampnLo0CFCQkJYt24dR48epV+/frRt25bdu3fTrVs3vL29efr0qbTMzz//jJqaGuHh4cyYMYP169ezYcMGaX5ubi5+fn7MmzeP/fv3Y2BggKenJy9fvgRgzZo1/PzzzyxYsIB9+/YxatQovL292bZtG4DU9TJ37lzmzZv3zvpv2bKFAQMGsG/fPnr37s2MGTP4+++/AQgLCyM0NBQvLy+io6NZv349ycnJLFu2DIAnT54wefJknJ2dOXDgAIGBgZw7d44VK1YAcP/+fVxcXGjevDm7du0iKCiI9PR0vvzyy0o/sQpV399//12shexNr1694vLly5VUo4Lv+ObNm/Hw8KBu3boAJCUl0bx5c7Zv306fPn3o2rUr06dP5/79+9JyI0eOpGvXrowePZq2bdsyatQoxowZQ//+/Sut7oJQmcSYsmroVU4W/7l7jW8v7pfSbqY/YdjxDSy06Ittk9YVOsbs9evXjBo1iuHDh6OqqgrA1KlT2bhxI9euXfugMnNycliwYAH6+voYGBhgZGSEkpISY8aMAWDMmDHs3LmT5ORkNDQ0ANDT08Pb2xs5OTn09fVJTEwkLCyMcePGSeVOnz4dKysrACZOnEh0dDTx8fEYGhqyefNmVq9ejbW1NQDNmjXjzp07hIaGMnz4cLS0tABQU1NDTU3tnfV3cXFh6NCh0jpPnz7N5s2bWblyJc2aNWP58uX06NEDgCZNmmBnZ8ehQ4eAgqArKyuLxo0b06RJE5o0aUJQUBC5ublAQfDZqFEj5s+fL61v7dq1dOrUiUOHDjFw4MAP2ufCP9Pdu3dLla9o8FPRtm/fjpqaGl9++aWUlp6eTmxsLE+fPsXHxweAlStXMmrUKCIjI6lRowZ3797l6dOnLFy4kHbt2nH69GnWrFmDjo4OgwYNqrT6C0JlEUFZNVRLURlrbUPaaGjjdXY3N9Of0Lx2PZZ95kwDlToVPui/Xr16uLi4sH//fuLi4rh16xZXr14FCrrq3sXCwkJmOioqSvp/s2bNpP/XqlULbW1tabpGjRoAZGVlSWkdO3ZETk5OpuwNGzbItKYVdoMA1K5dG4Ds7GwSEhLIzMzk66+/Rl7+fw3GOTk5ZGVlkZGRUeyFtQsXLmTfvn3S9IQJE/Dw8ACgffv2MnnNzMw4ffo0ADY2Nly6dAl/f3+SkpJISkoiISGBhg0bAtC6dWv69u2Lh4cHWlpadO7cGWtra3r16gVAXFwc169fL7bvMjMzSUxMRBCKKvq7eZfC719l2Lt3LwMGDJD5TSkqKpKZmcn69etRV1cHIDAwkK5du3Ls2DHs7e2ZMmUKffv2Zfjw4UDBbyUtLQ0/Pz8GDhwo89sVhH8CEZRVU3WUVaijrMIyS2eGHd/Ass+caVmnQaWs++HDh3z55ZfUq1cPGxsbunTpgomJSanuhnrzdvYGDf5XZyUlJZl57zvgKirKfn0LA0IFBQUpTVm5eICan58vjT1bu3atTOD2ruWmTZvG2LFjpenCE0lJdc3NzZXKCAkJYf369Tg5OWFlZYWrqytHjx6VCUhXrVrFpEmTOHnyJP/3f//HN998Q/v27dmyZQt5eXl06tSJRYsWFavT+1rwhH8fY2Njatas+c4uzFq1amFiYlIp9bl69Sq3b9+mX79+MumNGjWiYcOGMr8jTU1N6tatS0pKCk+ePOHGjRvF6mlubs4PP/zAs2fPqFevXqVsgyBUFnGZUc01qFmHhRZ9aaBSsbe3F7V//36ePXvGzz//zMSJE+nVq5c0KL0w2Hmb5s2by3zeDKzK4s0xMRcuXKBp06YyB/m3adGiBYqKiqSmpsrU58SJE4SGhpYYENavX18mb+HYGIDY2NhidWnVqhVQMMB50qRJeHt78+WXX2Jubk5ycrK0ry5dusSSJUto0aIFrq6uhISEsGTJEk6fPs3jx49p1aoViYmJaGtrS+tWV1dnyZIlxMfHl3W3Cf9wqqqq0iMn3uarr76iVq1alVKf8+fPU79+fYyMjGTSLS0tSU1N5cGDB1LagwcPePr0qfQdr1mzZrEhEdeuXaNOnToiIBP+kURQVs3VUVbBtklr6iirvD9zOWnUqBGvX7/m0KFDpKamEhMTw8yZMwHZ7sWKdv78edatW0dycjK7du1i27ZtuLu7l2pZNTU1hg4dir+/PxEREdy+fZtdu3bh5+cn03pXq1YtEhMTZbpES7J582b27NnDjRs3pGCpcGybtrY2p06dIiEhgRs3brBmzRoOHz4s7avatWuzfft2/Pz8uHnzJvHx8Rw4cABdXV00NDRwcXHhxYsXzJo1i6tXr3L16lVmzJjB5cuXMTAw+MC9J/xTqaio0KdPH77++mtq1qwpM69WrVp8/fXX9OnTp1j3fEWJi4uTucOykL29Pbq6ukybNo2///6buLg4Zs6ciZ6eHtbW1igoKDBq1Ch++OEH9u7dy+3bt9m7dy/BwcHSsAFB+KcR3Zf/AJX94Fg7OztiY2NZtmwZ6enpNGnShMGDB3P06FEuX75c6jEtH8vW1pbExET69+9PgwYNmDNnDsOGDSv18oUPofT39+fBgwdoa2szdepUmcDOzc2NjRs3kpiYSFBQ0FvLmjhxIj/99BMLFiygZcuWhISEoKenB8CKFSvw9fXF2dkZVVVVzMzM8PHxwdvbm9TUVPT19QkICCAwMJDt27cjLy9Pp06d2LBhA/Ly8ujo6LB161ZWrVrFsGHDUFBQoF27doSFhYnWAqFEderUwcnJid69e3P58mXu379Pw4YNMTExoVatWpUWkEHBcIeircqFlJWV2bx5M8uWLWP06NHk5+fTuXNnVq1aJXX9T5s2DQ0NDYKDg7l79y5Nmzblm2++kW6qEYR/Grn89/U3CeUiIyODpKQk9PT0KvWA+E81cuRImjRpIj1WQhAEQRDepTqch0X3pSAIgiAIQhUggjJBEARBEIQqQIwpE6qln3766VNXQRAEQRDKlWgpEwRBEARBqAJEUCYIgiAIglAFiKBMEARBEAShChBBmSAIgiAIQhUggjJBEARBEIQqQARlgiAIgiAIVYB4JIYgCMI/2MuXL8nOzkZJSQlVVdVPXR1BEN5BtJQJVYKNjQ0BAQHS9PHjx0lISADgzJkzGBoakpKSUixveHh4iS87/tQMDQ0JDw//qDJyc3MxNTXF0NBQ5lN0P1UFb/4N3vxbCp/G06dPOX/+PAsWLGDatGksWLCA8+fP8/Tp00qvS3BwMCNHjpRJu3LlCiNGjMDc3BwbGxvCwsJk5j979oyFCxfSrVs32rVrx7Bhwzh//nyJ5efn5zN27Nhi6xCE6ka0lP0DvMrOppaS0qeuRrm5c+cOHh4ehIWF0bJlSywsLIiJiSnx5dsODg507dr1E9Sy4iUnJ5OZmUlERAT169eX0mvVqvUJayVUBw8ePGDy5MncuHFDJv3kyZO0aNGC9evXo6WlVSl12bZtG2vXrqVDhw5S2tOnTxkzZgw2Njb4+Pjw3//+Fx8fH1RVVXF2dgZg5syZPHz4kNWrV1O/fn1++uknxo4dy549e2jRooXMOrZs2UJMTAyfffZZpWyTIFQU0VJWzWXk5HDv9UsycnI+dVXKTX5+vsy0srIyWlpaKCgoFMuroqJSaSeXynbt2jVq166NkZERWlpa0kd0QQnv8vTp0xIDskI3btxg0qRJFd5idv/+fTw8PFi5ciW6uroy83799VeUlJTw9fVFX18fZ2dnXF1dCQkJAeDmzZucOnUKb29vOnTogJ6eHgsWLKBBgwbs27dPpqxr166xfv16zM3NK3R7BKEyiKCsmkvPzmbCyd9Iz86utHUaGhqyY8cOXFxcMDExwd7engsXLrBjxw6sra1p164d06dPJyMjAyi5i/Ft3Y4pKSnY2toCMGrUKAICAop1X76rHENDQ3bt2oWrqyumpqZ06dKFwMBAmWX27duHvb09JiYmDB48mLCwsGJlvNn1WDQtLy+P4OBgevfujbGxMe3atcPd3Z1bt26VZTdy4cIFhg8fjqmpKdbW1vj4+JCeni7Nv3btGvr6+mUqEyA0NJSePXtibGyMjY0N69evlwLdgIAAXF1dCQwM5PPPP8fCwoKFCxdy9+5dJkyYgJmZGb169eI///mPVF5qaiozZszAysqKtm3b0q1bN/z8/MjLyytz3YSKl5iY+NaArNCNGzfem+djxcbGoqSkRGRkJGZmZjLzzp8/z2effYai4v86azp16kRycjKPHj1CQ0ODkJAQTExMpPlycnLIycnx/PlzKS0zM5NZs2YxdepU9PT0KnR7BKEyiKCsGsvIyWFH4lWeZ2Xx641rldpatmbNGtzd3YmIiEBNTQ0PDw+io6MJCQlh6dKlHDlyhJ07d5a5XG1tbWm5gIAA3NzcylzG8uXLcXJyIioqihEjRhAQEMC5c+eAgrFqnp6eDBo0iMjISAYOHMjKlSvLVH5YWBihoaF4eXkRHR3N+vXrSU5OZtmyZaUu4+rVq4wZM4auXbsSGRnJypUriY2Nxc3NTQqg4uPjycnJYezYsXTu3JmBAwcSERHxznKPHTtGcHAwPj4+HD58mFmzZvHDDz8QGRkp5Tl//jxJSUls27aN+fPns2PHDgYNGoS9vT3h4eHo6+vj5eUl1eOrr77ixYsXbNq0iUOHDuHm5sbGjRs5duxYmfabUPFevnzJ9u3bS5V3+/btvHz5ssLqUji2UEdHp9i8e/fu0ahRI5m0Bg0aAHD37l3q1KlD9+7dUVZWluZHR0dz8+ZNmeEKfn5+NGjQgBEjRlTQVghC5RJBWTWWnp3NjoRrAPyScLVSW8ucnZ2xsbGhRYsWODo6kpaWxsKFCzEwMKB37960bt2a69evl7lcBQUFaeyYurr6B3XVDRgwAEdHR3R0dPDw8KBOnTpcuHABKGhFsrOzY+zYsejp6TFs2DCGDRtWpvKbNWvG8uXL6dGjB02aNMHKygo7Ozvi4+NLXUZoaCidO3fGw8MDXV1dOnTowKpVq7h06RJnz54F4Pr16zx79oyRI0cSGhpK7969mTNnDrt27Xprubdu3UJZWZkmTZrQuHFjHBwc2Lx5M5aWllKevLw8fHx80NPTw9nZGQ0NDTp16sSAAQPQ19dn2LBhPH36lIcPH5KRkYGjoyPffvstRkZG6Ojo4OrqiqamJteuXSvTfhMqXnZ2No8fPy5V3sePH5NdiceMojIyMmQCLoAaNWoABa1fb7pw4QJz5szhiy++wNraGigYH7dv3z6WLFmCnJxchddZECqDGOhfTRW2kmXm5QKQmZvLrzeu4WZojIpixf9ZmzdvLv2/Zs2aQEGwUkhFRYWsrKwKr0dJ3uzyU1NTk04+sbGxfPHFFzLzLS0t2bx5c6nLt7Gx4dKlS/j7+5OUlERSUhIJCQk0bNiwxPwWFhYy01FRUcTFxXHz5s1i86Cg+6ljx47s37+f3NxcKTA1MjIiNTWV0NBQBg0aRJ8+fUhNTZWW27BhA/3792f37t307t2bli1b8vnnn9O7d28aN24s5atfvz61a9eWpmvVqlXsbweQlZWFiooKI0aM4NChQ/z111/cvHmTa9eu8ejRI9F9WQUpKSnJ3BTyLvXr10fpE90gVNLxoTAYe/NGliNHjjBr1izatWsntWo/efKEuXPn4u3t/dbfnSBURyIoq6aKtpIV+iXhKkNaGFZKUKZYwjrk5Uvf8Jqbm1ue1ZHx5hU4/O/mAUVFxTIHEzlvdAuHhISwfv16nJycsLKywtXVlaNHjxIVFVXi8nv37pWZbtCgAXl5efTr1w8PD49i+QtbCguDo6IMDAykrsiQkBCZujVs2BAVFRUiIiK4ePEip06dIiYmhrCwMKZMmcLkyZMBSjwRv+1v9+rVK0aMGEFGRgZ2dnY4OTlhamrK8OHDS8wvfFqqqqq4uLhw8uTJ9+Z1cXH5ZDeNNGrUiAcPHsikFU4XDbK2bt3K4sWLsbOzY/ny5dJv+8SJEzx8+JC5c+cyd+5coOAiIi8vDwsLC6KiomQuRAShuhBBWTX0ZitZocpuLSutwiAgPT1daqFJTk5+a/6K7IowMjLi0qVLMmkXL16UmVZSUpIZcH/z5k2Z+UFBQUyaNInx48dLaaGhocXuGi1UtFWxUKtWrUhISJCZl5iYiJ+fHzNnziQ/P5+ePXvi5eXFwIEDpTyXL1+mVatWADRp0qRYuZGRkbx48YLhw4fTvn17pk6dyvz58zlw4IAUlJVFTEwMsbGxnDp1Ck1NTaDg+VGPHz9+6/YKn5a+vj4tWrR450D+Fi1aFHusRGWytLTkl19+ITc3V7qr+vTp0+jp6Uktfdu3b+fbb79l5MiRzJs3T+a40KtXL9q1aydT5sqVK7l37x4rV66UxqcJQnVTdc7cQqll5eXypb4Rg/WL370ojxxZebmoVKE/rbm5OXJycgQEBDBy5EguX77Mnj173pq/sPsiPj6eNm3alGtdxo0bx4QJEzA1NaVHjx78+eefbN26tVh9d+7ciaWlJfn5+SxdulSm9U1bW5tTp05hY2ODvLw8ERERHD58WApaSsPNzY3hw4fj4+PDiBEjeP78OT4+PmRkZKCrq4uysjKdOnVizZo11K9fn+bNm3P48GEiIyMJDg5+a7mZmZksX74cVVVVOnTowL179zh37pzMM6LKonAwdmRkJL179+bu3busXr2a7OzsT9Y9LbybhoYG69evZ9KkSSUGZoXPKdPQ0PgEtSvg7OzMxo0bmTdvHu7u7vz1119s3rwZHx8fAJKSkliyZAm9evViwoQJPHr0SFpWRUUFNTU1mS54KGglVFFRKfEiSBCqi6pz5hZKrY5yjU9dhTLR0dHBx8eH4OBgtm/fTvv27Zk9ezaenp4l5tfQ0MDZ2ZkVK1Zw8+ZNevXqVW516datG76+vgQHB7Nq1SqMjY0ZNmyYTGDm7e2Nt7c3Q4YMoUGDBkybNo179+5J81esWIGvry/Ozs6oqqpiZmaGj48P3t7epKamlqrbxNzcnI0bN+Lv74+TkxO1atXCysoKT09PKQBcsmQJAQEBLFq0iMePH6Ovr8+6deve+bDcwYMH8+zZM77//nvu3r2Luro6vXv3ZtasWR+0v0xNTZkzZw6bN29m7dq1NGzYEAcHB7S1tbl8+fIHlSlUPC0tLYKDg7lx4wbbt2/n8ePH1K9fHxcXF1q0aPFJAzIoGM+2ceNGFi9ejJOTE1paWsyePRsnJyeg4E7L7OxsfvvtN3777TeZZZ2cnMp0p7MgVCdy+aIPolJkZGSQlJSEnp5eiWOFhMpx9uxZNDU1ZbpugoKC2LVrF0eOHPmENROEiiHefSkIBarDeVg8EkP4V4mJiWHs2LGcPn2a1NRUjh49ypYtW3B0dPzUVROECqGqqkrdunVFQCYI1YDovhT+VSZPnsyrV6+YPXs2T548QVtbG1dXV9zd3T911QRBEIR/OdF9WUmqQ7OpIAiCIPxTVYfzsOi+FARBEARBqAJEUCYIgiAIglAFiKBMEARBEAShChBBmSAIgiAIQhUg7r4UBEH4h8nJyeH58+ckJSVx9uxZ0tLSUFdX57PPPkNPTw91dXXp9UaCIFQdIigTBEH4B3n69Cn79u1j+/btMq8ngoJ3tGpqauLi4kK/fv0++ZP9BUGQJR6JUUmqw624giBUb48ePWLy5MkkJCS8N2/Lli0JDAws0ztbBaE6qw7nYTGmTKgSbGxsCAgIkKaPHz8unVjOnDmDoaEhKSkpxfKGh4djaFj8xeyfmqGhIeHh4Z+6Gh/kzz//5Pz58+/MU52375/q6dOnpQ7IABISEpg8eTJPnz6t4JoVvGDcwsJC5jtz5coVRowYgbm5OTY2NoSFhcksk56ezqJFi+jSpQufffYZs2bN4vHjx8XKHT9+PBYWFnTu3BlfX19ev35d4dsjCBVFBGVClXPnzh08PDykA7CFhQUxMTFoa2sXy+vg4EBMTExlV/EfzcXFhVu3bn3qaghlkJOTw759+0odkBVKSEhg37595ObmVlDNIDs7m1mzZvHq1Ssp7enTp4wZM4ZmzZqxe/duJk2axMqVK9m9e7eUZ9q0aZw4cYLFixezbds2Xr9+zahRo8jKypLKGDFiBIqKiuzcuRM/Pz9+++03li9fXmHbIggVTQRl1dTzrGzSMrNl0tIys3melf2WJaqPN3vUlZWV0dLSKnFgsoqKClpaWpVVNUGokp4/f8727ds/aNnt27eTlpZWzjX6n4CAAGrXri2T9uuvv6KkpISvry/6+vo4Ozvj6upKSEgIUNCKFhMTg6+vL927d6dVq1asWLGCBw8eEBUVBcDWrVtRVFRkzZo1tGzZks8//5ypU6fy119/FTuGCEJ1IYKyaio/H7Zeu0V4wh2Snr8kPOEO267drpR1GxoasmPHDlxcXDAxMcHe3p4LFy6wY8cOrK2tadeuHdOnTycjIwMouYvxbd2OKSkp2NraAjBq1CgCAgKKdV++qxxDQ0N27dqFq6srpqamdOnShcDAQJll9u3bh729PSYmJgwePJiwsLBiZbzZNVc0LS8vj+DgYHr37o2xsTHt2rXD3d29TK1L4eHh9OrVi19++QVra2vMzMyYOnUq9+/fZ9asWVhYWNCtWzd27dolLTNy5EgWL17MzJkzMTMzo1u3boSEhEgnoDNnztCmTRtOnDhB3759MTY2xs7OjiNHjkhl5Ofns2HDBmxtbTEzM8PR0ZHIyEiZ7QSYM2cOXl5e79yGGzduMHToUIyNjbG3t+fgwYPSvNLsoxMnTjBw4EDMzMywsrLCy8tLJjhITExk3LhxWFhY0KVLF77++msePnxY6n38b5KUlFRsUH9pPXr0iKSkpHKuUYFz586xY8cOli1bJpN+/vx5PvvsMxQV/3evWadOnUhOTubRo0ckJycD0KFDB2m+qqoqzZs35+zZswDExMTQq1cvatSoIeUZPHgw4eHhyMnJVcj2CEJFE0FZNaVeQwntWir4XbyOS/Q5/C5eR1tVhTrKSpWy/jVr1uDu7k5ERARqamp4eHgQHR1NSEgIS5cu5ciRI+zcubPM5Wpra0vLBQQE4ObmVuYyli9fjpOTE1FRUYwYMYKAgADOnTsHFIxV8/T0ZNCgQURGRjJw4EBWrlxZpvLDwsIIDQ3Fy8uL6Oho1q9fT3JycrETz/ukpqZy6NAhQkJCWLduHUePHqVfv360bduW3bt3061bN7y9vWXG/Pz888+oqakRHh7OjBkzWL9+PRs2bJDm5+bm4ufnx7x589i/fz8GBgZ4enry8uVLoODv9vPPP7NgwQL27dvHqFGj8Pb2Ztu2bQBSV/DcuXOZN2/eO+u/ZcsWBgwYwL59++jduzczZszg77//LtU+evLkCZMnT8bZ2ZkDBw4QGBjIuXPnWLFiBQD379/HxcWF5s2bs2vXLoKCgkhPT+fLL7+U6QYTChQGKh+q8PdRnp4/f87s2bOZP39+saEH9+7do1GjRjJpDRo0AODu3bsy/y+Um5vLvXv3ePLkCVAQiDZo0IClS5dibW1Nr169WLFiBZmZmeW+LYJQWURQVo1ZNKgrM22upV5p63Z2dsbGxoYWLVrg6OhIWloaCxcuxMDAgN69e9O6dWuuX79e5nIVFBSoV68eAOrq6qiqqpa5jAEDBuDo6IiOjg4eHh7UqVOHCxcuAAWPBLCzs2Ps2LHo6ekxbNgwhg0bVqbymzVrxvLly+nRowdNmjTBysoKOzs74uPjy1ROTk4OCxYswMDAgO7du2NkZESLFi0YM2aM9G92drbUagCgp6eHt7c3+vr6ODk5MXLkSMLCwmS6a6ZPn46VlRW6urpMnDiR9PR04uPjefXqFZs3b2bu3LlYW1vTrFkzqdsoNDQUQOoKVlNTQ01N7Z31d3FxYejQoejp6TF9+nTMzc3ZvHlzqfbR/fv3ycrKonHjxjRp0oT27dsTFBTEyJEjgYLgs1GjRsyfPx99fX2MjY1Zu3Ytjx8/5tChQ2Xaz/8GH9v9+OzZs/KpSBHe3t5YWFjQr1+/YvMyMjJQVlaWSSts8crMzMTExIQWLVqwaNEi7t+/T0ZGBqtWreLp06dkZxcM0UhPT2fDhg1kZmYSGBjIN998w759+5g/f365b4sgVBbxnLJq7OKDZzLT/32Yhl6dsgcxH6J58+bS/2vWrAkUnIgLqaioSANyK5u+vr7MtJqamnQgj42N5YsvvpCZb2lpKQUTpWFjY8OlS5fw9/cnKSmJpKQkEhISaNiwYYn5LSwsZKYLx8SA7D6rVauWTItC4Umq6H7s2LGjTNeMhYUFGzZskGlNa9GihfT/wrE82dnZJCQkkJmZyddff428/P+ux3JycsjKyiIjI6PY8GQGfwAAB8BJREFUbeILFy5k37590vSECRPw8PAAoH379jJ5zczMOH36dKn2UevWrenbty8eHh5oaWnRuXNnqbUDIC4ujuvXrxfbd5mZmSQmJiLIUlf/uAuyunXrlk9F/r+9e/dy/vx5me9OUSUdHwpbuGrVqoWysjKBgYHMnj2bbt26oaSkRL9+/ejRo4f03VVUVJQuUgCMjY3Jzc1l+vTpeHl5Ub9+/XLdJkGoDCIoq6bSMrO59yqT2e0MMNdS578P07j7MoPnWdmV0oVZdCxIoaIn+vepyLu93rwCh//dPKCoqEheXl6ZysvJyZGZDgkJYf369Tg5OWFlZYWrqytHjx6VCbaK2rt3r8x0YdcMgJKS7N/qffvwzf1euC1Fb4J42/YX7oO1a9fKBG7vWm7atGmMHTtWmi568n+zrrm5uVIZpdlHq1atYtKkSZw8eZL/+7//45tvvqF9+/Zs2bKFvLw8OnXqxKJFi4rV6X0teP9Gn332mdTa+SEsLS3LsTawe/duHj9+jLW1tUz6okWLOHDgAI0aNeLBgwcy8wqnCwN3fX19du/ezbNnz1BUVKR27doMGjSITp06AdCoUSNatWolU0bh9J07d0RQJlRLIiirpuTkYISRjhSA6dVRrbJ3XhYGHunp6VLLTdEuuTdV5CBdIyMjLl26JJN28eJFmWklJSXS09Ol6Zs3b8rMDwoKYtKkSYwfP15KCw0NfesdX0VbFT/W5cuXZaYvXLhA06ZNS9VS0qJFCxQVFUlNTaVHjx5SelhYGAkJCfj6+hZbpn79+m89ucXGxtKzZ0+ZuhgZGQHv30eXLl0iKiqKuXPn0qJFC1xdXYmMjOSbb77h8ePHtGrVigMHDqCtrS0Fes+ePcPT05MxY8ZIJ2ahgJ6eHpqamh802F9TUxM9Pb1yrc/KlSulG30KffHFF0ydOpX+/fsTERHBL7/8Qm5urnRBcfr0afT09Khfvz7p6el4eHgwf/586TuVkpJCXFwcX3/9NVAQSBbeaVl4zIiPj0dBQYGmTZuW6/YIQmURY8qqqTrKSsVaxEpKqwrMzc2Rk5MjICCAlJQUDh48yJ49e96av1atWkDBAfbFixflWpdx48Zx6NAhNm3aRHJyMrt372br1q3F6rtz506uXLlCXFwc3t7eMq1I2tranDp1ioSEBG7cuMGaNWs4fPhwpXTXnj9/nnXr1pGcnMyuXbvYtm0b7u7upVpWTU2NoUOH4u/vT0REBLdv32bXrl34+fnJtN7VqlWLxMTE9z5UdPPmzezZs4cbN26wZMkS4uPjGTduHPD+fVS7dm22b9+On58fN2/eJD4+ngMHDqCrq4uGhgYuLi68ePGCWbNmcfXqVa5evcqMGTO4fPkyBgYGH7j3/rnq1KmDi4vLBy3r4uLy0d2fb2rYsCHNmzeX+UBBkN+wYUOcnZ1JT09n3rx5JCQkEB4ezubNm5kwYQJQ8P3Iz89n8eLFXL9+ncuXL/PVV1/RqVMnrKysABg7diy3b99m0aJFJCUl8fvvv7N8+XIcHR2lcamCUN2IoEyocDo6Ovj4+PDbb79hb2/Pjh07mD179lvza2ho4OzszIoVK/D39y/XunTr1g1fX1+2bdtG37592blzJ8OGDZPpRvT29kZdXZ0hQ4YwZcoUBg8eLHOn2IoVK8jIyMDZ2ZkRI0YQHx+Pj48Pjx8/JjU1tVzr+yZbW1sSExPp378/QUFBzJkzp0w3KsyZM4dRo0bh7++Pvb09wcHBTJ06lUmTJkl53Nzc2Lp1K3PmzHlnWRMnTuSnn36if//+nD17lpCQEKnF5X37SF9fn4CAAE6fPs2AAQMYNmwYCgoKbNiwAXl5eXR0dNi6dSsvX75k2LBhjBgxAiUlJcLCwsQJtwSKior069ePli1blmm5Vq1a0a9fv0p/OXn9+vXZuHEjSUlJODk5SePHnJycpDyrV69GXV2dYcOGMWHCBNq3by/z1o8WLVoQFhbGjRs3cHR0xMvLCwcHB3x8fCp1WwShPIl3X1aS6vDOrX+Ds2fPoqmpKTOmKigoiF27dsk8z6sqGjlyJE2aNCnzozeEf4+yvPuyVatWBAQEiHdfCv8a1eE8LFrKhH+VmJgYxo4dy+nTp0lNTeXo0aNs2bIFR0fHT101Qfhompqa/PDDD0ydOvWtwZampiZTp07l+++/FwGZIFQxYqC/8K8yefJkXr16xezZs3ny5Ana2tq4urqWelyWIFR1GhoaDB8+nL59+5KUlMTZs2dJS0ujbt26WFpaoqenh7q6eqV3WQqC8H6i+7KSVIdmU0EQBEH4p6oO52HRfSkIgiAIglAFiKCskomGSUEQBEGofNXh/CuCskqipKSEnJyc9GJoQRAEQRAqz6tXr4Dib1KpSsRA/0qioKCAuro6Dx8+JDMzkzp16qCoqFihT68XBEEQhH+7/Px8Xr16xYMHD6hbt26VvslFDPSvRPn5+aSlpfHgwYMKffejIAiCIAiy6tatS6NGjap0Y4gIyj6B/Px8cnNzi73oWhAEQRCE8qekpFSlW8gKiaBMEARBEAShChAD/QVBEARBEKoAEZQJgiAIgiBUASIoEwRBEARBqAJEUCYIgiAIglAFiKBMEARBEAShChBBmSAIgiAIQhUggjJBEARBEIQq4P8BHZq07bv3MqsAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmoAAAMzCAYAAAAWJD70AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd5wUVb4+/qeququr48yQc2ZIEwDBLCoqiIJKMuuquCjg7ureu4a7+73rur+N966rqwiLIBjvKkFQMYvoGlBQGXIYGHJmQk/H6nB+f4xTzcAwobtnunvmeb9evpBPd1efPnT3PHOqzjmSEEKAiIiIiNKOnOoGEBEREVHtGNSIiIiI0hSDGhEREVGaYlAjIiIiSlMMakRERERpikGNiIiIKE0xqBERERGlKVOqG9DcfvjhBwghYDabU90UIiIiauFCoRAkScKwYcPienyrG1ETQqC1rvErhICu66329Z+KfRHDvohhX8SwL2LYFzHsi5iG9kWiuaPVjahVj6Tl5+enuCXNz+fzYevWrejXrx9sNluqm5NS7IsY9kUM+yKGfRHDvohhX8Q0tC82btyY0PO0uhE1IiIiokzBoEZERESUphjUiIiIiNIUgxoRERFRmmJQIyIiIkpTDGpEREREaYpBjYiIiChNMagRERERpSkGNSIiIqI0xaBGRERElKYY1IiIiIjSFIMaERERUZpiUCMiIiJKUwxqRERERGmKQY2IiIgoTTGoEREREaUpBjUiIiKiNMWgRkRERJSmGNSIiIiI0hSDGhEREVGaYlAjIiIiSlMMakRERERpikGNiIiIKE0xqBEREVGrFdTDCIWjqW7GWTGoERERUavkD4ZQXhmEECLVTTkrU6obQERERNTcvIEQ3B4dQPqGNIBBjYiIiFoZj09HpU9HVACSlOrW1I1BjYiIiFqNCk8Q3kAIaXy2swYGNSIiImrxhBAo9wThD4YzJqQBDGpERETUwkWjAmWVQQT1cJpfkXYmBjUiIiJqsSKRKMorgwiEIqluSlwY1IiIiKhFCoWjKK8MQE/jddLqw6BGRERELY4eiqC8MohQJHNDGsCgRkRERC1MUA+jzBNEJJJpV6SdiUGNiIiIWgx/MIQKj45IVECSkFEzPGvDoEZEREQZRUQjEKEAAAmyxWbUq3cbiAqBNhYdJlnCyaCKSDRz0xr3+iQiIqKMIaIRRMoP4vhzN0Av+QbRoA9A1W4Dbk/QCGnBNQtR/q8H0MbshyKn+fYDdWBQIyIiooxQHdJOLLwbUW8pypY+DL3kG/g9lXD/uCVUdUjzf/sawke2oeKNX9Qa1mQJsJgVaKoprbeRYlAjIiKitHdqSBP+8uoiypY+DLF/LRymSI2QVu30sKaaZLRRBNpHgnAVbYbz398Aq79FZMsuiEovRCicmhd4FrxGjYiIiNKeCAdRtuyxWEgzboiiYtkjcE14HMHDW+Ff+39nPDZ8ZBt8X72AdhffB7HnOKIffYXw/iM17hMBAJsG5YKhMF0yHHDaIaXBUBtH1IiIiCjtyaoNOVP/BtnV6cwbRRTut/671pAGAOaeI+C8+F5EP1yLyAvLIE4LaQZfAJFP1iD4txchjpdBRFO/BhuDGhEREWUEU1YntP3J/NrD2lmYe45A9g1/QuSTHxD94vuGPajSC/2ZVyEqPHG2NHkY1IiIiCgjBPUwKpCNnDueb1BYM/ccAdcNfwIqQoh+9l3jnszrR3jpRxD+QJytTQ4GNSIiIkp7/mAIZZVBBPQIKpCNNnctrPP+SlYnZE/5X+gRFZEPvzrjdgHgXWTjp+iL/0UXhGo5RnTrbkCv7Zbmw6BGREREac3r11FeqRsL1zqUILxrXq7zMRH3Meh71kKTwxBbimvcFgYwDx2xUOqIcsmEbyQn9sFy5kGEQPibDRCRSLJeSqMxqBEREVHaqvTqcHurdhsAgDaWEIJrXqixBEetfpwNGjrwHeQrRhhlL2T8Ed3wsZRt1LqKILpDr/Uw0Z37gEDttzUHBjUiIiJKO0IIlHuCqPRXLWQLNCKkGQeJovzNRxHpFIB81UgchRm/QQ9slOzGXXKFH7/Dfqg4yzZTgSAgUjf7k0GNiIiI0kokKlDqDsLnDxmbqtcX0sydB0F2djjzBhFF+bv/ja0mP/5L7YMDUuwU50XCjd9iP7JQx6lNVQWk1MUlLnhLREREaSMciaK8MohgqGZ4igpAtmbX+piq2Z1/BsJ+lL38U0TdNddJW6sPwaKPQjj1kJPFCdyIk/WOWMm9uwIWNY5XkhwcUSMiIqK0oIciKK0InBHSAKBcN8NUMAn2S2fWqFcvwXEyaEb5aUt3CAGsDIzC856JRkhThMAD4jBubkBIAwDlomGQTEqiLy1uDGpERESUckE9jNLKAEKRs18PVnZaWIuFNBVRAYTCUSOsRZ1dsMh/A1b4LzMe7xAR/D/sx6VwN6hNcv8ekCzmhF5Xonjqk4iIiFLKFwjB7Y0tv1GXMt2MnIJJcGV3gbnXeUZIqxYKR3EgZMc8PITNwUqj3lkVeCy4F51rXTGtFpoK09SxkGzWxr6cpGJQIyIiopTx+HRU+nQ0IKMZynQzHD0vgTuIMx53tNSH2YvX41iZ36j1756N39xUAPvKCMSWXfU/gdUCdebNkHJcDW9UE2FQIyIiomYnhIDbq8MbiM3sbAxP8Mzazv1lmLt0A7yBsFE7P68Tbrt6EAJmGY6pV0PeshPRVd9CnCw/8wAmBfLwwTCPvQhw2SEpqbs2zWhSqhtARERErUv1Gmn+QPhsq5c12ppNh/Hyu1trnD69blQfjLugFyRJghDAcV1AG5QLx8C+UCoqIYq2A14fJNUMqVsnKEP6AqoZUgpneZ6OQY2IiIiaTTRaFdICweSENCEE3v73brz71R6jZlJk/OTaQRg5+MyN2wOhKAKQoLiyoF5+AWQI2G0qTCmc2VkXBjUiIiJqFtGo+HFj9XD9d26AUDiCl97dirVbjho1p82MGZML0adrVp2PjUQE/JEwJAlI8XyBOjGoERERUZOLRAXKKwMI6MnZ4LzSp2PO0g3YfbDCqHVqa8OsqUPRPjuNk1cjMagRERFRk4pEBcrdAQRqWcg2HodPeDF78XqcqAgYtYE9czB9Yj5sWmrXPUs2BjUiIiJqMmfbEipe2/aU4p9vboQ/GDt9elFhF9w6ZgAUpeWt48+gRkRERE0iFI6gvDIIPXz23QYa48uiQ3j1g22I/jizUwIw8fJ+uOrcHpAkKSnPkW4Y1IiIiCjpgqEIyisDCEcSn9sZFQLLV+/Ch9/sNWpmk4x7JgzBsAEdEj5+OmNQIyIioqQK6GGUVwYbtCVUffRQBAvf2Ywfth83ai67iplTCtGrc+p3DmhqDGpERESUNL5ACBUeHdF4ths4TYUniOeWbsDew7FN1Lu2d2DWlEK0ydISPn4mYFAjIiKipKj06vD4G7dv59kcPObB7CVFKHXHZnYO6dMW916fB6ul9cSX1vNKiYiIqElEowIV3iD8wXBc+3aebvPuk3h++cYaa65dOrwbbryyPxS55c3srAuDGhEREcUt8uPyG8laI2319wfw+kfbjcAnScDUK3IxekT3pBw/0zCoERERUVz0UAQVnuQsvxGNCixZtROr1u03ahazgmnX56GgX7uEj5+pGNSIiIio0ZI5szOgh7FgxWZs3HXCqGU7LZg1pRDdOzoTPn4mY1AjIiKiBlMUBf5gBLovmJSZnWXuAJ5bUoT9xzxGrUdHJ2ZOKUS205Lw8TMdgxoRERE1mB6WUeEJwqIlvjzGviNuzF6yARWeoFEr6NcO067Lg0VVEj5+S8CgRkRERPWKRgUqPDrcvgDaJGEkbf2O43jh7U3QQ7Hr264c2QOTLu8HWW6Z20HFg0GNiIiI6lQ9s9MXCCESSWzigBACn6zdj6WrdqI67smShJvH5GLUsG6JN7aFYVAjIiKiswpHoihzB6CHo0h0HC0SjeL1j3bg8x8OGjXNomD6DfkY3LttgkdvmRjUiIiIqFahcARl7iBCCY6iAYA/GMbzyzdiS0mpUWvj0jBraiG6tnckfPyWikGNiIiIzhAMRVBWGUAkkvj1aCcr/Ji9pAiHjnuNWu8uLsyYXACXnTM768KgRkRERDUkc420kkMVmLN0A9xe3agNH9ABd40fDNXMmZ31SYugtnz5csybNw/79+9Hjx498MADD2DcuHEAgDlz5uCpp5464zHbt29v5lYSERG1fP5gCOUeHdEkhLTvth3Fone2IHTKzgVXX9AT143qC1nizM6GSHlQW7FiBX7961/jv/7rv3DJJZdg5cqV+OUvf4lOnTph2LBh2L59O66//nr86le/SnVTiYiIWjSvX4fbG0p4IVshBD5YsxfLP9tl1GRZwu1XD8SFBV0SbWarktKgJoTA008/jTvvvBO33XYbAGDGjBlYt24dvv32WwwbNgw7duzAjTfeiPbt26eyqURERC1apVeHx68j0YG0cCSK1z7Yhq82HDZqNosJ903Kx4CebRJsZeuT0qBWUlKCgwcPYsKECTXqCxYsAADouo49e/agT58+qWgeERFRq1DhCcIbCCHRdWy9gRDmLduI7fvKjFr7bCtmTS1Ep7b2BFvZNKQ0PwWb8qAGAD6fD9OmTcOWLVvQrVs3zJgxA6NHj0ZxcTEikQg++OAD/OEPf0AwGMTIkSPxq1/9Ch06dIj7eYUQ8Pl8yXoZGcPv99f4szVjX8SwL2LYFzHsi5iW3hdurw5vIAzRgJQWDARr/HmqE+UBPP/WVhwri/VT785O3DNhIBxWOe36TwJgNiuw2VSEQ0GEQ417fEPfF0KIhMKgJBryL9NEVqxYgYcffhjdunXDAw88gIEDB+KDDz7A3LlzsXDhQhw9ehSPPPII7rjjDkyZMgUnT57Ek08+Ca/Xi+XLl0OLY5+xjRs3Qtf1+u9IRETUgplMJviCoup0ZzSxddIOlep4d205AqFYpMjtquGKAhcUJf1GrMxmE+yaCs0MRCINC6mJUFUV+fn5cT02pSNqZrMZADBt2jRMnDgRADBo0CBs2bIFCxcuxLx58zBq1Ci0aRM7p92/f3+MGjUKq1atwjXXXBP38/br1y/xF5Bh/H4/9uzZg169esFqtaa6OSnFvohhX8SwL2LYFzEtsS+EAMo9QQSCYTTmCvBgIIgDBw+gW9dusGhV6599t+04VnxTXGO9tbHndcfY87ql3WlFSQIsZhOcNjPMJjmhYzX0fVFcXJzQ86Q0qHXs2BEAkJubW6Per18/rF69GgBqhDQA6NChA7Kzs3HkyJG4n1eSJNhstrgfn+msVmurfv2nYl/EsC9i2Bcx7IuYltIX1VtCQTZDs5rjOoZFs0DTNKz8sgTvfFFi1E2KhDvGDcJ5eZ2T1dykkSUJDqsZDps5qQGyvvdFos+VWJxM0JAhQ2C321FUVFSjvmPHDvTo0QN///vfMXbs2BpDkgcOHEBZWVmrHBEjIiJKRNWWUFX7diYiHI5i4TtbaoQ0u9WMX9w8LO1CmiQBmllB2ywNTruadqN89UnpiJqmabj33nsxe/ZsdOzYEQUFBVi5ciW+/PJLLFq0CHa7HQsWLMDjjz+Ou+66CydOnMAf//hHDB8+HJdcckkqm05ERJRRgqEIyisDCCe4JZRfj+K5Nzej5FClUevYxoZZUwvRISe9RhwVWYLDZoZdS+4oWnNK+YK3M2fOhNVqxd///nccPXoUffv2xTPPPIPzzjsPAPD888/j6aefxqRJk6CqKq644go88sgjGdvhREREzS2oh1GWhC2hjpX5seSLUlT4IkYtt0c27ptYAHucp1GbgiQBmmqCy67CpKT05GHCUh7UAODuu+/G3XffXettF1xwAS644IJmbhEREVHL4A+GUOHREw5p2/eW4Z/LNsIXjIW0C/M749arB6ZVGDIpMpw2M2xa+gTHRKRFUCMiIqLk8wZCcHv0hLeE+mrDIbz6/rYaYe+GS/ti7Pk90+YMlywBVs0Mp02FIqdHm5KBQY2IiKgF8vh0VPoS2xIqKgTe+nw33v96j1FTZOD2q3NxQUH3xBuZJKpJhsuuwqK2vFjT8l4RERFRK5eMLaH0UAQvrtyC77YdM2oOqxlXD3diaP92yLGFISJn7lBQTVIsKPM1bcyQJQk2zQSnTYXcgkbRTsWgRkRE1EIIIVDh1eFLMKS5vUHMWboBJYfcRq1LOzumTRiAsuMHq54rEsTX/3f2hecvuOVdNFXMkACYW/Ao2qla9qsjIiJqJaJRYew2kMgVaYeOezB7SRFOVgSM2uDebfDT6/MBEULZ8cTbmoimWrg2XTGoERERZbhIVKC8MoCAHqn/znXYUnIS85ZvROCUmZ2jhnXFTVflQpFl+P2N3Lk8ycyKjCxHyx9FO1XreaVEREQtUCQSRVllEMFQYiHt3+sP4v8+2G7MEJUATB7dH1eM7J7ykavqddGy7CqUNFoKpDkwqBEREWWoUDiK8srEtoSKRgWWrS7Gx9/uM2qqWcY9E/IwNLcxW7Y3DVmW4LSZ4bCqqW5KSjCoERERZaCqLaGCCEfiD2lBPYIX3t6Eop0njFqWw4JZUwrQo5MrGc1MiGqS4XJYYDErqW5KyjCoERERZZhkbAlVXhnEc0uKsO9obM/Obh0cmDWlEDkuLRnNjJskAVaLCS67pUUtXhsPBjUiIqIM4guE4PYmtiXU/qOVmL2kCOWVsXXQ8vu2w7Trh0Br4IX6kmL5cQmOs98eD+XHU532Vnqq83QMakRERBmi0qvD409st4GNxScwf8WmGpMPRo/ojimj+zdq0diqxWyTGyNUk4wshwVqKz7VeToGNSIiojQXjQpUeIPwB8MJLWT76br9eOOTHcYxJAm46cpcXHZOareDkiXA9uM+nS11h4F4MagRERGlsUgkivLKIAIJLL8RiUax+OOdWP39AaNmURX89Po85PVtl4xmxs2syHA51Aafcm1t2CtERERpSv9xZmcogZmdgWAY89/ahE27Thq1HJcFs6YUolsHZzKaGRdjwoCt9a2N1hgMakRERGnIHwyhwpPYpIFSdwCzFxfh4HGPUevZ2YWZkwuQ5YjvYv9kUBQJTpsKu2ZOWRsyBYMaERFRmknGpIG9h92YvaQIbq9u1IbltsfdE4ak7GJ9WQI0jqI1CoMaERFRmhCiamP1RCcNrN9xHAve2oTQKTsWjD2/J66/tC/kFGwHJQEwm2Q47bwWrbHYW0RERGmgetJAMBRBvBlNCIGPvt2HNz8tNo4hyxJuHTsAFxd2TVZTG0WWJDisZtitZs7ojAODGhERUYrpoQgqPMGE9uyMRKL410fb8e/1h4ya1WLCfRPzMbBXm2Q0s1EkAKpZgcuucl20BDCoERERpVAyJg34AiHMW74J2/aUGrV2WRpmTR2Kzu3syWhmoyiyBLtmhsNmhpSCU60tCYMaERFRinh8Oip9IUQTuCDtRLkfs5cU4fAJr1Hr0zULMyYXwGlr3m2YJAmwmBS4HCrMJo6iJQODGhERUTMTQqDCq8MXCCU0aWD3wQrMWVqESl/IqI0Y1BE/uXZQswclRZbgsJlh1ziKlkwMakRERM0oEomi3BNEUI9/0gAArNt6FIve2YLwKYvhXnNhL0y4pE+zBiVJAixmBS67BWYTl9xINgY1IiKiZhIKV+00kMikASEE3vt6D976fLdRU2QJd4wbhPPzOyejmQ2myBKcNjPs1uY9xdqaMKgRERE1g4AeRnllMKFJA+FIFK+8tw1rNh02anbNhPsnFaB/j5xkNLNBZEmCppqQ5VBh4sK1TYpBjYiIqIl5/DoqvYlNGvD6Q5i7bAN27i83ah1yrJg1dSg6trEloZUNo1lUZDksaJulNdtztmYMakRERE1ECAG3V4c3wUkDR0t9mL14PY6V+Y1a/+7ZuG9SARzW5tkvU5IAm2ZGtsMEq4UzOpsLgxoREVETiEartoMKBMMJTRrYua8Mc5dtgDcQNmrnDemE28cNaraL96uvRZOEjMOhUP0PoKRhUCMiIkqy6u2gAqFIQsdZs/EwXn5va43r2q67pA/GXdirWWZ2SvhxRqejakanzxeu9zGUXAxqRERESRQKR1FeGUh4Zufb/96Nd7/aY9RMioyfXDsIIwd3SkIr61e9Ryd3F0gtBjUiIqIkCYaqlt84dW2zxgqFI3hx5Vas23rUqDmsZsyYXIC+3bKT0Mr6qSYZLrsKi8qYkGr8FyAiIkqCZCy/UenTMWfpBuw+WGHUOrW1YdbUoWifbU1GM+skS7E9OmWZo2jpgEGNiIgoQb5A1cbqiSy/cfiEF7MXr8eJioBRG9gzB9Mn5sOmNe3Mzuo9Op12FaqZMzrTCYMaERFRAqrWSNORwEAatu0pxT/f3Ah/MHax/kWFXXDrmAFQmnhBWUWR4LRyd4F0xaBGREQUJ7dHhyegJ7RG2pdFh/DqB9sQ/THpSQAmXt4PV53bo0kv4pclQLOY4LKpTR4GKX4MakRERI0khECFV4cvgYVso0JgxWe78MGavUbNbJJxz4QhGDagQ5JaWjvVJMNpV6FxskDa478QERFRIyRjIVs9FMHCdzbjh+3HjZrLrmLWlEL07OxKTkNrwckCmYdBjYiIqIEiUYHyygACevwL2VZ4gnhu6QbsPew2al3bOzBraiHauJpm/0wJgGpW4OJkgYzDoEZERFQPSZIQiQqUuQMIJrDbwMFjHsxeUoRSd2xm55A+bfHT6/OgWZrmR7Iix0bRuHBt5mFQIyIiqoesmFFeGYRsin9m5KZdJzB/xaYao3GXDe+GqVf2hyIn/2L+6iU3XA4VZhNH0TIVgxoREVEdwpEo3L4wHKEIrHH+1Fz93X68/vEOY+KBJAE3XpGLy0d0T15DT6HIVds/2a0cRct0DGpERERnEdDDKK0Mwh8IxvX4aFRgyaqdWLVuv1GzmBVMuz4PBf3aJauZBkn6cRN1e9Um6pT5GNSIiIhqUb3bQDjOzdUDehgvvLUZG4pPGLVspwUPTClEt47OZDXToMgSnDYuXNvSMKgRERGdxuPTUemLf7eBMncAzy0pwv5jHqPWo6MTM6cUIttpSVIrq1SNopngsqscRWuBGNSIiIh+JISA26vDm8BCtvuOuDF7yQZUeGKnSwv6tcO06/JgUZN7Ub+iSHDaVNibeC9QSh0GNSIiIgCRSBTlniCCeiTuhWzX7ziOF97eBD0UO1165cgemHR5v6QuMCtJgKZWjaKZuP1Ti8agRkRErZ4eiqDCE4Qe5/VoQgh8snY/lq7aaYQ8WZJw85hcjBrWLXkNRdUomsumwsZRtFaBQY2IiFo1XyAEt1dHJM4L0iLRKF7/aAc+/+GgUdMsCqbfkI/Bvdsmq5nGKFqWnZuotyYMakRE1CoJIVDpDcEbiH/SgD8QxvMrNmJLSalRa+PSMGtqIbq2dySppZzR2ZoxqBERUatTtWdnEEE9/o3VT5T78dySIhw64TVqvbu4MGNyAVz25MzslPDjumjcXaDVYlAjIqJWJRSOoLwy/uvRAKDkUAWeW1KESl/IqA0f2AF3XTs4aZueyxLgsKrco7OVY1AjIqJWwx8MocKrIxKJdxwN+H7bMSx8ZzNCpwS9qy/oietG9YWcpEClKBKyHRZoKn9Mt3Z8BxARUatQtYhtCNE4F0gTQuCDNXvw5updRk2WJdx+9UBcWNAlWc2Exawgy8EtoKgKgxoREbVoyVjENhIVeP2TXfhm8zGjZrOYcN+kfAzo2SYp7ZQkwGoxIctuSeqaa5TZGNSIiKjFikYFyj1BBILxTxrwBcJ4+5tyHDipG7X22VbMmlqITm3tSWmnLElw2s1wcFYnnYZBjYiIWqRIJIqyyiCCoUjcxzhe5sMzb2zEsbJYSOvbLQszJhXAYUtOqFJkCVkOFVYLF7ClMzGoERFRi5OMmZ3FB8oxd+kGePyxmZ3nDumEO8YNStr1Y4oiIcdhgYWTBugs+M4gIqIWJaiHUeYJJjSz89vNR/DSu1sQPuUYY8/rhhsuy03aUhlmRUa205K05TyoZWJQIyKiFsMXqFp+IxrnVgNCCKz8sgTvfFFi1BRFwuh8J64+v0fSQppqkpHt1Dizk+rFoEZERC1CpVeHxx//dlChcBSvvLcV32w+YtTsVjPuuXYAROB4UtooS4BVM8NlUzmzkxqEQY2IiDKaEAIVXh2+BJbf8Ph0zF22AcUHKoxaxzY2PDC1EA5Nwq5diQc1kyLBZeekAWocBjUiIspYkUgU5Z4ggnok7uU3jpz0YvbiIhwv9xu1AT1yMH1SPuyaGX6/v45H10+SAE01IcuuQlF4qpMah0GNiIgyUlAPo8KjIxSJf2bn9r1l+OebG+ALhI3ahfmdcevVA2FKQqhSZAkOG9dHo/gxqBERUcZJdDsoAPhqwyG8+v42RE65qO2GS/ti7Pk9kzJpQDXJcDkssHBWJyWAQY2IiDJGJCrg9gbhD4bjvh4tKgTe+nw33v96j1Ezm2TcNX4wzhnYMeE2Vm8F5bJboHDCACWIQY2IiDJCKBxFeWUgoUVs9VAEL67cgu+2xfbsdNrMmDmlEL27ZCXcRm4FRcnGoEZERGkvqIdR7tERTuB6NLc3iDlLN6DkkNuodW5nx6wphWiXbU2ofRKqRuV4qpOSjUGNiIjSmj8YQoVHr3EtWWMdOu7B7CVFOFkRMGqDerXB9BvyYdUS+1EoSxLsmhlOuzlpC+ISVWNQIyKitJXoIrYAsKXkJOYt34hAMLY5+yVDu+Lmq3ITXi5DNclw2VXu1UlNhu8sIiJKO9GoQEWCkwYA4N/rD+L/PthuzA6VAEwe3R9XjOye0OiXLEmwaSY4ucMANTEGNSIiSivhSBTllUEEQ5H673wW0ajAstXF+PjbfUZNNcu4Z0Iehua2j/u4xrVoHEWjZsJ3GRERpY1kTBoI6hG88PZmFO2MbfuU5bBg1pQC9Ojkivu4iizBYVXhsJk5ikbNhkGNiIjSgtevw+1NbBHb8sognltahH1HKo1atw4OzJpSiByXFvdx7TYNbVwaXA4uu0HNi0GNiIhSKhoVcPsS21QdAA4crcTsJUUoqwwatfy+7TDt+iHQ4jxNKQGwaWY4NRlmE/fppObHoEZERCkTCkdR4UnsejQA2Fh8AvPf2oSgHjvO5ed0w9QrcuM+TSlJVSFNlWUcDocSah9RvBjUiIgoJfzBECq8OiKRBIbRAHy6bj/e+GSHMRonScBNV+bisnO6x31MWQIcVhVOuwqfz5dQ+4gSwaBGRETNSgiBSm8I3kBi66NFolEs/ngnVn9/wKhZVAU/vT4PeX3bnfVxWfYwotGqhW9lWUOFt+aPQkWWkOVQYbWY428cUZIwqBERUbOJRKIo9+gI6mEkMo4WCIYx/61N2LTrpFHLcVkwa0ohunVw1vnYaDSA11ZcAwC49fp3ATiM28yKjCwnt4Gi9MGgRkREzUIPRVDhCSa0qToAlFYEMHtJEQ4e9xi1Hp2cmDWlEFkOS1zHlFA1GpftsCS8WwFRMjGoERFRk/MFQnB7E9uvEwD2HnZj9pIiuL26URua2x53jx8CixrfKJgkAXbNDJdd5V6dlHYY1IiIqMkIIeD2Vi29kWBGww/bj+GFtzcjdMqI3JjzeuKGy/pCriNgnXpNGgAEgmU1/t9plSBJErx+wGzSYFHjXxSXKNkY1IiIqElUXY8WRFCPJHQ9mhACH327D29+WmwcR5Yl3DJmAC4Z2rXex596Tdrplr1/W42/3z7xfQY1SisMakRElHTJuh4tEoni/z7cji+KDhk1q8WE6RPzMahXm0SbSZT2GNSIiCipknU9mi8Qwrzlm7BtT6lRa5elYdbUoejczt7g48iy9uPsziqBYJkxkjZ53GuwarHAZzbFv80UUVNgUCMioqRI5vVoJ8r9eHbxehw5GVtstk/XLNw/qQAue+P226xaJ61qCQ7VJMOmxa5ns2pt4LB1SKyxRE2IQY2IiBKWrOvRAGD3wQrMWVqESl9s26YRgzriJ9cOgtkU//pmFrOCHKcF/qCn/jsTpQkGNSIiSkhQD6PCoyMUSex6NABYt/UoFr2zBeFTjnXNhb0w/pI+dc7srI9mVpDt0qDEue8nUaowqBERUdwqvTo8/hCiIrFxNCEE3vt6D976fLdRU2QJd4wbhPPzO8d9XAmAZjEh22ExNmc3mzTcPvF94/+J0hmDGhERNVokKlDhCSKgh5FgRkM4EsUr723Dmk2HjZpdM+H+SQXo3yMn7uNKEmDTzMg6bSFbi+riEhyUMRjUiIioUZK19AYAeP0hzF22ATv3lxu1DjlWzJo6FB3b2OI+riQBDk2Fy9G4iQdE6YZBjYiIGixZS28AwNFSH2YvXo9jZX6j1r97Nu6fVAC71Rz3cWUJcNpUOGwMaZT5GNSIiKheyVx6AwB27i/D3KUb4A2Ejdr5eZ1w+7hBMCWwKbosSXDZzbBbGdKoZWBQIyKiOoUjUVQkaekNAFiz6TBefndrjVG560b1wbgLeiW0KbosSch2qrBa4h+NI0o3DGpERFQrk8mEgB6BO+BHJJJ4RBNC4O1/78a7X+2JPYci467xgzFiUMeEjq3IErIcDGnU8jCoERHRGaJRAb8OlFcGYdESX8IiFI7gxZVbsW7rUaPmtJkxY3Ih+nTNSujYiiwhx2mBReWPNGp5+K4mIqIa/MEQSiuDcHsDaJfo2hsAKn065izdgN0HK4xap7Y2zJo6FO2zrQkdWzXJyHJYoJrj37GAKJ3Ff8VmEi1fvhzXXHMN8vPzce211+K9994zbjtw4ADuu+8+DB8+HBdffDGeeuopRCKRFLaWiKhlikYFyj1BlFcGoYciEEkIaYdPePGXF9fWCGkDe+bg4TtGJBTSJACaqqCNS2NIoxYt5SNqK1aswK9//Wv813/9Fy655BKsXLkSv/zlL9GpUyfk5eVh2rRp6NWrF/71r39h3759+PWvfw1ZlvHzn/881U0nImoxgnoYbq+elLXRqm3bU4p/vrkR/mBsZudFhV1w65gBUBKY2Xm2hWyJWqKUBjUhBJ5++mnceeeduO222wAAM2bMwLp16/Dtt9/i4MGDOHToEN544w1kZWUhNzcXJ0+exF//+lfcf//9UFVOvyYiSoQQAh5fKCnbQJ3qi6KDeO2D7Yj+OLNTAjDx8n646tweCc/sdNrNcHD5DWolUhrUSkpKcPDgQUyYMKFGfcGCBQCAxx9/HEOGDEFWVuxC0/PPPx8ejwdbt25FYWFhs7aXiKglCYWjcHuCCIaSs+wGAESFwPLVu/DhN3uNmtkk454JQzBsQIeEjq3IErKdFmicNECtSMqDGgD4fD5MmzYNW7ZsQbdu3TBjxgyMHj0aR44cQadOnWo8pkOHqg/64cOH4w5qQgj4fL7EGp+B/H5/jT9bM/ZFDPsipjX1hR6KosKrIxSu/ZrfYCBY48+GHTOCVz/YiQ27So2a02bGvRMGokcnZ0L9ajErcDpURMM6fGE97uPEozW9L+rDvohpaF8IIRIaRU5pUPN4PACARx55BA888AD+8z//Ex988AFmzpyJhQsXIhAIwOWquXGuxWIBAASDDf/yOF0oFMLWrVvjb3iG27NnT6qbkDbYFzHsi5iW3BcmkwmhiIRKnw49FK73/gcOHmjQcb2BCFauLcexitgx2zpNGH9uFkLeY9i161hc7ZVlCXarBTaLjOOHw0mZ4BCvlvy+aCz2RUxD+iKRS7VSGtTM5qqFCadNm4aJEycCAAYNGoQtW7Zg4cKF0DQNul7zN6fqgGazxb9Zr9lsRr9+/eJ+fKby+/3Ys2cPevXqBas1sSnxmY59EcO+iGnpfSFE1VIZvkAY2fUEnmAgiAMHD6Bb126waJY673vohBevfrYVZZWxkDaoZzbuHJcLzRL/jxlJkmDXTHDaVKRyzkBLf180BvsipqF9UVxcnNDzpDSodexYtRJ1bm5ujXq/fv2wevVqnHvuudixY0eN244dO1bjsfGQJCmhoJfprFZrq379p2JfxLAvYlpiX4TCVdtARWCCRWv4V79Fs9T5Q2jz7pN4fvkmBPTYKdRLh3fDjVf2hyInsmcnYNdUuBzpM2mgJb4v4sW+iKmvLxKdmZzSddSGDBkCu92OoqKiGvUdO3agR48eGDlyJLZs2WKcIgWANWvWwG63Y+DAgc3dXCKijOQLhHCywo9gKLlrUK7+/gCeXbzeCGmSBEy9oj9uGTMgoZBWtR2UJa1CGlGqpHRETdM03HvvvZg9ezY6duyIgoICrFy5El9++SUWLVqEoUOH4qmnnsKDDz6I//zP/8SBAwfw5JNP4p577uHSHERE9YhGBSp9OryBEJJ5aVc0KrBk1U6sWrffqFnMCqZdn4eCfu0SOrZZkZHlULkdFNGPUv5JmDlzJqxWK/7+97/j6NGj6Nu3L5555hmcd955AID58+fjd7/7HW688UZkZWXh1ltvxcyZM1PcaiKi9BYKR1Dh0ZM+ihbQw1iwYjM27jph1LKdFsyaUojuHZ0JHdtiVpDttMCUwGK4RC1NyoMaANx99924++67a72tZ8+eeOGFF5q5RUREmcsXCMHt1RGJJneGZJk7gOeWFGH/sdjlKD06OjFzSiGynXVPOKiLBECzmJDtsECWudMA0anSIqgREVHihBBwe5N/qhMA9h1xY/aSDajwxJZGKujXDtOuy4NFjX+vTW4HRVQ3BjUiohYgHImiojK5uwxUK9p5HAve2gQ9FNsH9MqRPTDp8n4JjYBJEuBIs5mdROmGQY2IKMMF9TDKPUGEI8mNaEIIfPbDIaz4fI8R/mRJws1jcjFqWLeEji1LgNOmwmFjSCOqC4MaEVEGq/TqSd9QHQAiUYHPNlVi097YjgKaRcH0G/IxuHfbhI4tSxKyHCpsmjnRZhK1eAxqREQZKBIVqPAEEdDDSb8ezR8MY/5bW7Ftb2wPwzYuDbOmFqJre0dCxzYpErIdFi6/QdRA/KQQEWWYYCgCtycIPRyt/86NdLLCj9lLinDouNeo9erswswpBXDZE5vZaTEryOLyG0SNwqBGRJRBvH4dlb5Q0pfeAICSQxV4bkkRKn0ho1bYvy2mXZcP1Vw1s1ORJZjNUQSCDZ9EUD2z02VTufwGUSPx1xoiogwQiQqUVQZQ0QTrowHAd9uO4snXvq8R0s7pZ8Od43JrhjSrH1sr10DVQmc7VA2yJCHLrnKNNKI4cUSNiCjNNeWpTiEEPlizF8s/22XUZFnCjaP7oJ3mgfzj2mbVIe2Pa3+O3RXbMD3vMYxofyX0wNknBCiyhBwnr0cjSgQ/PUREaczjqzrVmexZnUDV2muvfbANX204bNRsFhPum5SPHh2s2LWrageC00MaAMzb9CcgD2cNa4oiIcepwWKOfzFcImJQIyJKS5FIFBVevUlmdQKANxDCvDc3YvveMqPWPtuKWVML0amtHX5/1YxPkyKfEdKqnS2smRUZ2U6LccqUiOLHoEZElGZ8gRDcPh2RJC9gW+14mQ+zlxThyEmfUevbLQszJhXUWIBWs6hwZEfxx7W/OCOkVTs9rKkmGdlODWYTL4EmSgYGNSKiNBGOROFuwlE0ACg+UI65SzfA449NBjh3SCfcMW7QGeHKZrOgMlSGg549dR7zy8Mf4cIuYyBFFOQ4LVC4/AZR0vDTRESUBryBEE5U+OEPNl1I+3bzETz1f9/XCGnjL+6Nu8cPrnUErLSsEmqoLX5/4XxYFK3WYw5pcw5+OfzPiOgqclwaQxpRkvETRUSUQqFwFCcrAqjwBJvsVKcQAiu/2I0X3t5s7AdqUiTcM2EIxl/cB5J09mUz/P4osqTutYa1IW3OwS/P+QvkiA05Tg0Kl98gSjoGNSKiFBBCwOPXcbLC36SnOkPhKBa9swVvf1Fi1BxWMx68ZTjOHdKpYccISWeEteqQZorakWW31Bn2iCh+vEaNiKiZhcIRuD06guFIkwU0oGppj7nLNqD4QIVR69jGhgemFqJ9jq1RxwqFJGSZq8Lav7bPxazCx6FGHXDY1fofTERxY1AjImomQgh4/SF4/E2zBdSpjpz0YvbiIhwvj22sPqBHDqZPyoddO/sitXWpDms/H/p7mGCD3R7fcYio4RjUiIiaQSgchdsbRDDUtKNoALB9byn+uWwjfMGwUbuwoDNuHTsw4Q3RoxEZFtkKjbsNEDULftKIiJpYU26kfrqvNhzCK+9vQ/SU57rh0r4Ye37PhK8j40K2RM2PQY2IqIk09e4Cp4oKgbc+34X3v95r1MwmGXeNH4xzBnZM+PiqSUaOS0t4RI6IGodBjYioCeihCCqaaCP12p5r0cot+H7bMaPmtJkxc0ohenfJSujYkgRoqoJsLr9BlBIMakRESeYPhlDh0ZvlVKfbG8ScpRtQcsht1Lq0s2PmlEK0y7YmdGxFlmGzmJHj1CAzpBGlBIMaEVGSCCFQ6Q3BG9DRDBkNh4578OziIpS6A0ZtcO82+On1+bBqiX29S5IEh82CLIfKkEaUQgxqRERJEIkKVHiCzXI9GgBsKTmJecs3IhCMGLVRw7ripqtyociJXUcmSYDDZoaHS6QRpRyDGhFRgoJ6GBUeHaFI01+PBgCf/3AA//pwB6I/JkIJwOTR/XHFyO4Jz+yUJcBhVaFIYRwIh+t/ABE1KQY1IqI4CSHg8VUtYBtthmG0aFRg2afF+HjtPqOmmmXcMyEPQ3PbJ3x8WQKcdhUOqwqfjyGNKB0wqBERxSEciaLCE0RQj6AZznQiqEfwwtubULTzhFHLclgwa0oBenRyJXz8U0MaEaUPBjUiokbyB0Oo8OqIRJojogHllUE8t6QI+45WGrVuHRyYNaUQOS4t4ePLEuCyq7AzpBGlHQY1IqIGikYFKn06fIFQs8zqBID9Rysxe0kRyiuDRi2/bztMu35IUrZxYkgjSm8MakREDRAKR1Dh0aGHmudUJwBsLD6B+Ss2IRiKzewcPaI7pozun5QlM2QJcDkscW/STkRNL6Ggtnv3bhw4cAAejwc5OTno0qULevbsmay2ERGlBW8ghEpv8yxgW23Vuv1Y/MkOY6kPSQJuujIXl53TPSnHlyUJWQ4VNoY0orTW6KB24sQJLFy4EO+88w6OHTsGccpMJ0mS0K1bN4wbNw533nkn2rVrl9TGEhE1p3AkCncz7dVZLRKNYvHHO7H6+wNGzaIq+On1ecjrm5zvVEWuCmlWC0MaUbprcFCLRCKYPXs25s+fjy5dumDixInIz89H165dYbPZUFFRgaNHj+K7777DqlWr8NJLL+EnP/kJHnjgAZjN/DIgoszi8evw+ELNOormD4Yxf8UmbN590qjluCyYNaUQ3To4k/Iciiwh22lJyvVtRNT0GvxJnTx5Mrp164bXXnsNeXl5td4nPz8fV155JR555BGsW7cO8+fPx9SpU7F8+fJktZeIqEnJJhVllUFISrTZRtEAoLQigNlLinDwuMeo9ezkxMwphchyWJLyHIosIcdpgYUhjShjNPjT+uijj+L8889v8IFHjBiBESNG4Ouvv46rYUTUMoUiEZgVJdXNOIMQAt5AGOUeHc5gGFZr850J2HvYjdlLiuD26kZtWG573D1hCFRzcvpKUSTkODVYknQ8ImoeDQ5qjQlpp7rgggviehwRtTyVug4BkXZBrXpGp9sbhK6HmvW5f9h+DC+8vRmhcGz7qTHn9cQNl/WFnOB2UNXMioxspyVpoY+Imk9c4991ncqUJAl2ux09evRAbm5uvO0iohaoTA/g6Q3f47cjLoBLTc7pvEQIIeD1V20BFYmKZj3VKYTAR9/uw5ufFhvLfciyhFvGDMAlQ7sm7XlUk4xspwazKbGN2okoNeIKar/+9a8RjVb99nf6rM/qmiRJOO+88zBnzhxYrdYkNJWIMlmlruPZjT/giyMHURYMpDyohcIRuD06guFIswY0AIhEovi/D7fji6JDRs1qMWH6xHwM6tUmac9jMSvIcVqgKAxpRJkqrk/v/PnzYbVa8dBDD2HVqlXYsGEDPv30UzzyyCOwWq344x//iDlz5mDPnj34xz/+kew2E1EGKtMD+Oxw1ZITT2/4AW49WM8jmkbVRuo6TlYEEAg1f0jzBUJ4ZnFRjZDWLkvDr+44J6khTTMryHFpDGlEGS6uEbW//OUv+OlPf4rp06cbtc6dO+Ouu+5COBzGK6+8gmXLluFnP/sZZs+ejUceeSRpDSaizFM9mlbty6OpGVVL5SgaAJwo92P2kiIcPuE1an26ZuH+SQVw2ZOzhZMEwKKakOO0JGX3AiJKrbh+1dq9ezcKCgpqvW3QoEEoLi4GAPTs2RMnTpyIv3VE1CKcOppWrTlH1VI9igYAuw9W4C8vra0R0kYM6oiHbhmW1JBm1Uxo42JII2op4gpq3bt3xwcffFDrbR999BE6d+4MADhy5AjatEneUD4RZZ7TR9OqVY+qNbVwJIpSdwBuX/NuAXWqdVuP4snXvkelLzaj9JoLe+Ge64bAbErOTExJAuxWM7IdFuN6YSLKfHGd+rz33nvx2GOP4eTJkxg7dizatm2LEydO4OOPP8bHH3+MJ554AiUlJXjqqacwatSoZLeZiDJIbaNp1Z7e8AMeH9l0M0D9wRDcXh3hSGoCmhAC7321B2/9e7dRU2QJd1wzCOfndU7a80gS4NBUuBzJGZkjovQRV1CbOHEiJEnCP/7xD3zyySdGvUePHvif//kfjB8/HitXrkTfvn3xH//xH0lrLBFllrONplVrqmvVhBBwe3X4AiGkaBANoXAUr76/FWs2HTFqds2E+ycVoH+PnKQ9jywBTpsKh40hjaglinsfkRtuuAE33HAD9u3bh9LSUnTq1AmdOnUybr/22mtx7bXXJqWRRJSZ6hpNq5bsUbXqxWv1UAQpymjw+EP457IN2Lm/3Kh1yLFi1tSh6NjGlrTnkSUJLrsZditDGlFLldCGbxUVFVBVFR06dEA0GsWhQ7Hp5l26dEm4cUSUuSLRKFxmC94eN7He+5rl5Fyn5fXrqGzmjdRPd7TUh9mL1+NYmd+o9e+ejfsmFcCRxG2pFFlClkOF1dJ8W10RUfOLK6jt3bsXjzzyCIqKis56n61bt8bdKCLKfIosI9vSPMtvRKICbm8Q/mA4JTM6q+3cV4a5yzbAGwgbtfPzOuG2qwcldWcARZaQ7bRA4+bqRC1eXJ/y3//+99izZw8eeOABdOrUCbLMBRWJKDWCehhurw79lL0yU2HNxsN4+b2tNUbzrhvVB+Mu6JXUWZiKIiHHYYGFIY2oVYjrk7527Vr84Q9/wPjx45PdHiKiBglHovD4Q/AHwoimcBhNCIG3/70b7361x6iZFBl3jR+MEYM6JvW5uLk6UesTV1BzOBzIyspKdluIiOoVjVZtpO4NpPZaNKBq4sKLK7di3dajRs1pM2PG5EL06Zrc70hurk7UOsX1ib/++uvx6quv1tiQnYioqXkDIZwo96MyhYvXVqv06fj7//1QI6R1amvDI3eOTHpIs5gVtHExpBG1RnGNqFmtVnz33Xe46qqrkJ+fD03TatwuSRL++Mc/JqWBREShcBRubxDBFG3/dLrDJ7yYvXg9TlTEdlYY2KsNpt+QB5tW9yxM1SzDYpVR6Q7Xeb9qmllBtkuDwi2hiFqluILam2++CafTiWg0WuvMT25fQkTJkg5Lbpxq255S/PPNjfAHY0Hr4sIuuGXMAChKA0a81Ci+PlaC4a5e0PWzvyZurk5EQJxBbdWqVcluBxFRDek2igYAXxQdxGsfbEf0x9AoAZh4eT9cdW6PBv2CajZL+OTQFszdthpvjL4fki7Vuihv9ebq3LeTiBK64CEajWLbtm34/PPP4fF4UF5enqRmEVFr5g2EcLLCj4CeHiEtKgSWfVqMV97bZoQ0s0nG9In5GHNez4aFNFVC2BzC3G2fwRvWsXDHlzDbqmaInkqSABs3VyeiH8W9EM+KFSvwt7/9DceOHYMkSViyZAmeeeYZmM1m/O1vf4OqcksTImqcdFm49lR6KIKFb2/GDzuOGzWXXcXMKYXo1dlV7+NVswyoUby7vwgLdnwBfyQEAFiy5ztsKjuIxwrHoa3mRFSXIYTg5upEVENcI2rvvvsuHnnkEZx//vn4+9//bsz+vOqqq/DZZ5/hueeeS2ojiajlC+phlFb44QukT0ir8ATxt1e/qxHSurZ34NE7RzYopAGArArs8RzHGyXrjJBWbVvFEbxY/DVUVYFZAVw2hjQiqimuEbW5c+fi5ptvxuOPP45IJGLUJ0+ejNLSUrzxxht48MEHk9VGImrBhBDw+ELw+EMpXbj2dAePefDskvUocweN2pA+bXHv9XmwWhr+1RnwCvRSO2LRqHvwwcFNeHrzJwCA7vYc/HnkZGQpNugeAYeNm6sT0ZniGlErKSnBVVddVetthYWFOHr0aK23ERGdKhSOorQigEqfnlYhbeueMvzPK+tqhLRLh3fDzCkFjQpp1fRQFLoXGNs5Dxd37AcA+P/OuQHZouqUp8uhMqQRUa3iGlFr27Ytdu3ahYsuuuiM23bt2oW2bdsm3DAiatl8gRDcPh2RSPoENADYuMeHzzcfNU6/ShIwZXR/XDGyR8LHDgcl/DLvKkiQ0NbsRFQXyOHm6kRUh7i+Ha655hr84x//QIcOHXDppZcCqFo7bdOmTXjuuee4BygRnVU0KuD26fAFQmlzLRpQ1a43PyvB55sqjZrFrGDa9Xko6Ncuac9hESqeGHE9fJ4Isp0qN1cnojrF9Q3x4IMPYseOHXjwwQchy1VnT++44w74fD6MGDECv/jFL5LaSCJqGfzBEDy+EPRwNNVNqSGgh/HCW5uxofiEUct2WjBrSiG6d3Qm9bnCOhCCQLaDm6sTUf3iCmqqqmL+/Pn48ssvsWbNGpSXl8PpdOLcc8/FpZdeyrV/iFIoGBYIRAS8obMPVykS4FJlWM3N81kNhSNwe0MIhtJnRme1MncAzy0pwv5jHqPWrYMdD0wdhmynJenPZ1ZkOGwq9+0kogZpcFALhUIwm2vuYXfRRRfVep1afY8joqZjMUkoqYjg3vcqznqfWwdruKfAiqo18JtOJCrg9YXgDaTXjM5q+45UYvaSIlR4YpMGene04L7JeU0S0lSTjByXdsYit0REZ9Pgb4sJEyY0euuo999/n9erEaVAZ7uM/Pa1/x6mKsCNA63QmnhExxcI4WS5H5X+9JrRWa1o53H876vraoS0y4Z3wbgRWbA0wSlJzaygTZaVIY2IGqXBI2p//etf8eijj+Lpp5/G+PHjMWbMGPTs2fOM++3cuROfffYZFi9ejGg0ir/+9a9JbTAR1S9Lk/GLEfZaR9WmDNDQlGuqVu3RqaflaU6gat22T9bux9JVO419NmVJws1jcjFyYNWM9mTTuLk6EcWpwUGtoKAAy5cvx6uvvopFixbhySefhMvlQteuXWG1WuF2u3H06FFUVlaiTZs2uPfee3HrrbfCYkn+6QMiql/1qNrG42Gj1pSjadUL13oDIUSiaZjQAESiUbz+0Q58/sNBo6ZZFEy/IR+De7eF3+9P6vNxc3UiSlSjJhOoqoq7774bt99+O9asWYNvvvkG+/fvh8fjQadOnXD55ZfjoosuwogRI6AonM1ElEq1jao11WiaHoqgwhNEKBxFekY0wB8I4/kVG7GlpNSotXFpeGBqIbq0dyT9+SQJsFoY0ogoMXHN+jSbzbjkkktwySWXJLs9RJREp46qNcVomhACXn8Ilb70nCxQ7US5H88tKcKhE16j1ruLCzMmF8BlT/6ovyQBds2MLAfPKBBRYpLyjb1161a899572Lp1azIOR0RJUj2qBiR/NC0ciaLUHYA7zbZ/Ol3JoQr85aW1NULa8IEd8NAtw5sspDk0lSGNiJKiUSNqX3/9NV5//XVIkoRbb70VI0eOxMMPP4y3334bQghIkoSLL74Y//jHP2C1WpuqzUTUCJ3tMs7pZErqaFq6bv90uu+2HcWid7YgdMoCu1df0BPXjeoLuQlOR0oS4LSqcNq5bycRJUeDg9pHH32En//85+jSpQucTifuvvtu3HjjjXj//ffxi1/8Anl5eSgqKsKcOXPw3HPP4T/+4z+ast1E1EBZmow/X+qCJCUeqiJRAbc3CH8wPWd0VhNC4IM1e7H8s9gMTlmWcPvVA3FhQZcmeU5ZApx2FQ5urk5ESdTgoDZ//nyMHz8e//M//wMAeOmll/CnP/0Js2bNwv333w8AuPjiiyFJEt566y0GNaImFIkIhEOARYuNCoV0AbNa+yiRWQFMcmKjaUE9DLdXT7vtn04XjkTx2gfb8NWGw0bNZjHhvkn5GNCzTZM8pyxJcDlU2DUu7k1EydXgb+7i4uIai9ded911EEJg5MiRNe533nnn4dChQ8lrIRGdQZaBk0ciCPiiiIQFKsuj0INnH+IyJbB+lxACbo+OUncw7UOaNxDCM6+vrxHS2mdb8fCdI5oupMkSsp0MaUTUNBo8oub1epGVlWX83eFw1PjTOKDJhFAolKTmEVFtJEmCu1Rg7ccB9BpkwtZ1IYy/K/nXhYbCEVR4dOihSNouu1HteJkPzy4uwtFSn1Hr2y0LMyYVwGFrmtORiiwh22mBpsY1gZ6IqF6N+naRTzl1wnWBiFIn6BcI6QKeCoFNa6p+MQrpgB4UUC3J+Wx6/Do8vvRdvPZUxfvLMWfZBnj9sV8Szx3SCXeMG9Rkm58rsoQcpwUWhjQiakIJf8MwsBE1n3BIIOAX+HxFEGXHap6GfO9lP3oPMuHcq1RIEiAr8X02I5EoKrw6Anp6Txio9u3mI3jp3S0InzID9dqLemP8xb2b7PvJpEjIdmpNsicoEdGpGhXUZs2aBVWteQrh/vvvh9kcuzZD1/XktIyIzmAyS1AjwKXXW/DtJ0Ec2h0La0MvMaPP4KrPYrwhzR8Mwe3Va4SedCWEwMovS/DOFyVGzaRIuOOawThvSKcme16zIiPHZYHZxJBGRE2vwUFt4sSJTdkOImogVZOgahJ69DfhcIkOu0uCp0Kg10ATNHt8Ac1kNlftLoAoMuBMJ0LhKF55byu+2XzEqNmtZsyYVIB+3bOb7HlVk4xsp9Zkp1OJiE7X4KD2pz/9qSnbQUSNFAoKjL1VgyNbxp6tYcQ7hycciaLSH4XDr0PT0n+hao9Px9xlG1B8ILaHacc2NjwwtRDtc2xN9ryqSUaOS4NJYUgjouaT9Ktgg8EgSkpKMHDgwGQfmoh+FAkL9M0zw6RWXSfaN88EpZFn4oQQ8PhCOFkRgNcXyIjr0Y6c9GL24iIcL/cbtdwe2bhvUkGTLo9hMSvIcVqgMKQRUTNr8LfOxRdffMZengsXLkRpaWmN2rZt23ialKiJKSYJZotkXCxvMkuQGrFWWihctU9npV/PiFmdALB9byn++tK6GiHtwvzO+PlNw5o0pGlmBTkujSGNiFKiwSNqJ06cqLE+WiQSwV//+lece+65aNOmaRaSJKLk8wWqJgxkSkADgK82HMIr729D9JQ233BpX4w9v2eTzeyUAFhUE3KcFsgJLBhMRJSIhE59ikw4V0JEADJnn85TRYXAW5/vwvtf7zVqZpOMn1w7GCMGdWyy55UkwKqZkO2wcAkiIkoprtRI1AoE9TAqPDpCkfTeAupUeiiCRSu34Pttx4ya02bGzCmF6N0lq45HJkZRFDisKkMaEaUFBjWiFkwIgUpvCN6AnhHLblRze4OYs3QDSg65jVqXdnbMnFKIdtlNNzNVliS4bBY4bWaGNCJKCwxqRC2UHorA7c2MfTpPdei4B88uLkKpO2DUBvdug59enw+r1nRfWbIEOO0qKs2Z1FtE1NJxCymiFiiT9uk81ZaSk5i3fCMCwYhRGzWsK266KheK3HSzLmVJgstuhiTCCIfDTfY8RESNxS2kiFqQUDgKt1dHMJQ5Ewaqff7DAfzrwx2I/thwCcDk0f1xxcjuTfoLoSJLyHKosFrM8PkY0ogovTQ4qN1www0cPSNKU0IIeP0hePyZN4oWjQos+7QYH6/dZ9RUs4x7JuRhaG77Jn1uRZaQ47TAovIqECJKTw3+dvrzn//clO0gojgZ16KFIxk3ihbUI3jh7U0o2nnCqGU5LJg1pQA9Orma9LlNioRspwaLmZurE1H6avBFH4899hj279/flG0hokaomtGp42RFAMFQ5oW08sog/vbqdzVCWrcODjx654gmD2lmRUYbF0MaEaW/Bge1N998E2VlZU3ZFiJqoFA4itKKACp9unFNVybZf7QSf35pLfYdrTRq+X3b4T9vPwc5Lq1Jn7t6c3WziSGNiNIfL8wgyjDeQAiVGbYF1Kk2Fp/A/BWbEAzFZnZefk43TL0it8m3auLm6kSUaRjUiDJEJm4BdbpP1+3HG5/sMNovScBNV+bisnO6N/lza2YF2S4NCvftJKIM0qig9vjjj8PhcNR7P0mS8OKLL8bdKCKqyRcIweMLZdQWUKeKRKNY8slOfPrdAaOmqQruvT4PeX3bNfnza6qCbCdDGhFlnkaPqDVkI/bGbNZ+9OhRjBo16oz6n/70J0yaNAm/+c1vsHjx4hq3de3aFatWrWrwcxBlKj0UQaVPz8jJAtUCwTDmv7UJm3adNGptXBpmTSlE1w71/+KXCAmAZqnaXL2pT6sSETWFRo+oFRQUJLUB27Ztg8Viwccff1xjnTan0wkA2L59O+6//37cfvvtxm2KwouAqWWLRAU8Ph2+QDgjJwtUK3UHMHtxEQ4e9xi1np2cmDmlEFkOS5M+twTAqpm4uToRZbSUX6O2Y8cO9OrVCx06dDjjNiEEiouLMX36dLRv37QLXxKlAyEEvIEfF66NZG5AA4C9h92YvaQIbm9st5JhA9rj7vFDoDbxshiSBNg0M7LsKkMaEWW0lAe17du3o2/fvrXetm/fPvh8PvTp06eZW0XU/AJ6GJVeHaFINGNPc1Zbv+M4Fry1CaFw7Jq6sef3xPWX9oXcxMFJkgC7Zm7yETsioubQ4KA2cuRI2O32pDdgx44dyMnJwW233YaSkhL07NkTM2bMwKhRo7Bjxw4AwMsvv4zPP/8csixj1KhReOihh4xTo/EQQsDn8yXrJWQMv99f48/WLJ36IhIRqPSHENDDiKZgyY1gIFjjz0QIIbD6+0N4+4u9qH4lsixh6uV9cH5eRwQDgYSfoy6yJMFhU2GWI3F9xtPpfZFq7IsY9kUM+yKmoX0hhEhoZF8SDbzy/7HHHmv4QSUJf/zjH+u9XzgcxtChQ9GvXz88+uijcDgcWLlyJRYuXIiFCxfi+++/x7PPPouf/exnuPLKK7Fv3z789a9/RceOHfHiiy9Clhu/FtLGjRu5cTylnKIogCQjoAv4g2GEwqGMH0WLRAU+21SJLftiX1qqScK4EVno3q7pR7fMJgVOmwWqSSAc5ubqRJQ+VFVFfn5+XI9tcFAbOHAgJElCx44d6w1IkiThk08+aVADvF4vFEWBpsVWI7/33nsBAPPmzUNFRQVycnKM24qKinDjjTfijTfeQGFhYYOe41QbN26EEAL9+vVr9GMznd/vx549e9CrVy9YrdZUNyelUtkXkaiALxCGL5AeG6gHA0EcOHgA3bp2g0WLL1D5g2EsWrkdO/ZXGLU2LgumXz8IHdvYktXUs5JlCVk2C6xaYte+8TMSw76IYV/EsC9iGtoXxcXFkCQp7qDW4FOf48aNw+rVq6HrOq6++mpce+21OOecc+J60lPVdjq1f//++OKLLyDLco2QVn0bABw5ciSuoAZUBUmbrel/eKQrq9Xaql//qZqzLyJRAa8vBF8whIhQoFrSa/ayRbPE9cV7otyPZxdvwpGTsVONfbpm4f5JBXDZ1WQ2sVayJMHlUGHXzEk7Jj8jMeyLGPZFDPsipr6+SHRCU4PPHf7973/HV199hd/85jc4duwY7r77bowePRr/+7//i61bt8b15Dt37sTw4cPxzTff1Khv2rQJ/fr1w8MPP4y77rqrxm0bN24EgFY5IkaZyxcI4WS5H5X+zN36qTa7D1bgLy+trRHSRgzqiIduGdZsIS2rgSHNEwrDG6r9lGhFMAQ9mpmLCRNRy9aoWZ9WqxXXXHMNrrnmGng8Hnz00Ud49913sWjRInTr1g3jx4/Htddei969ezfoeH379kWfPn3wxBNP4He/+x1ycnLwxhtvYP369Vi6dCn279+PmTNn4tlnn8V1112HkpISPPHEExg/fvxZZ4oSpZNQOAK3N7MXrD2bdVuPYtE7WxA+ZbeEay7shfGX9GnymZ1AVUjLdqqwWho2khaORrG0+BD6ZtmR19YFRZLgC0fw1eGT6OKw4sJObcAr24go3cS9PIfD4cDEiRMxceJElJeX46OPPsJ7772HuXPnIjc3F8uWLav3GLIsY+7cufjb3/6GBx98EG63G4MHD8bChQuRm5uL3NxcPPXUU5g3bx6ef/55OJ1OTJgwAQ8++GC8zSZqFkIIeHxV66Fl8oK1tRFC4L2v9uCtf+82aoos4Y5xg3B+fudmaYMiV42kNTSkAUCWaoZmkvHY15sBAC6zCe4fR9gWjzuX660RUVpKyjpqwWAQfr8fgUAAkUgEBw8ebPBj27Vrhz/96U9nvX3cuHEYN25cMppJ1CyCoQjcniBC4ShaVkQDQuEoXn1/K9ZsOmLU7JoJ908qQP8eOXU8MnkUWUK20wJNbdzXlyRJGNWlHZ7dUBUwq0NaW02FzZTyJSWJiGoV97fT0aNH8f777+P9999HUVERbDYbrrzyStx333246KKLktlGoowghEClNwRvoOWNogGAxx/CP5dtwM795UatQ44Vs6YObZaZnUD8Ia2aw2yCDODUq9H6ZzugxrHUDxFRc2jUt92p4Wz9+vWwWq24/PLLce+99+KSSy6Bqjb9xcNE6aglj6IBwNFSH2YvXo9jZbE10vp3z8Z9kwrgsCZvtmVdFFlCjtMCS5whDQDKgjpOnzKw+aSbEwmIKG01+BvvlltuQVFRESwWCy699FI8/fTTuPTSS2GxcJsWar1a8rVo1XbuK8PcZRvgDcQutT8/rxNuu3oQzKbmGYlSFAk5jsRCWiQaxSf7jwMAbCYF3RxW7KrwojIURkUwhDYaf9EkovTT4G+9H374AYqioF+/figtLcUrr7yCV155pdb7SpKEF198MWmNJEpHoXAUbm+wRc7orLZm02G8/O7WGkuKXDeqD8Zd0KvZLr43KRJynFrCG7m79TA62TS8dNUItNVUSBIQFcDuCg9Kgzp6Cq4JRUTpp1F7fVarbzODBm52QJSxfIEQ3D4dkUjLfK8LIfDOFyVY+WWJUTMpMn5y7SCMHNyp2dphVmTkuCwwmxJfHFgzKbimV0cop12P1lZrg0A40qLWtyOilqPBQe3ll19uynYQZYRIVMDtDcIfDLfYUbRQOIKX3t2KtVuOGjWnzYwZkwvRp2tWs7VDNcnIdmpJO71qrSPsaT/eFkrKMxERJQ/npBM1UFAPw+3VoYdb7oXnlT4dc5ZuwO6DsT07O7W1YdbUoWif3Xz7+qkmGTkuDSaFszGJqHVjUCOqR0tfdqPa0VIf5r+1DScqAkZtYM8cTJ+YD1sS99Gsj8WsIMdpgcKQRkTEoEZUl1A4ggqPDj0UaZHLblTbfyKIDz/aCH8wYtQuKuyCW8cMaNbApJkVZLs0KDJ3CSAiAhjUiM7K69dR6Qu1+IvM12w6ire/KUf1y5QATLy8H646t0ezbqukqSbkOC2QGdKIiAwMakSniUSiqPDqCOgtd8IAAESFwIrPduGDNXuNmtkk454JQzBsQIdma4cEQLNUhTTut0lEVBODGtEpAnoYFR4d4UjLnTAAAHoogoXvbMYP248bNafNjFlTh6JXZ1eztUMCYNVMyHYwpBER1YZBjQhANCpQ6dPhC4TQws90osITxHNLN2DvYbdRa+s0YdaUAnTp2IwhTQLsmhkuu8qQRkR0Fgxq1OrpoQgqPMEWvexGtYPHPJi9pAil7tjMzkE9s3HJQDNyXM23HVx1SMtycAs6IqK6MKhRq2UymeALhBH2ixY/YQAANu06gfkrNiGgx2Z2Xjq8GyZc1B17SnY3WzskCXBaVTjt3FuTiKg+DGrUKkWiAr6gQIVXh6ZpqW5Ok1v93X68/vEOY3KEJAFTr8jF6BHd4ff7m60dsgQ47SocVoY0IqKGYFCjVscbCKHUHUSlL4j2LXlaJ6quvVuyaidWrdtv1CxmBdOuz0NBv3bN2hZZkuByqLA34+K5RESZjkGNWo1wJIoKj45gKIxQOALRwkNaQA9jwYrN2LjrhFHLdlowa0ohund0NmtbZElCtlOF1cKQRkTUGAxq1Cq0lsVrq5W5A3huSRH2H/MYtR4dnZg5pRDZzua9gF+RJWQ5GNKIiOLBoEYt2qmjaC18AM2w74gbs5dsQIUnaNQK+rXDtOvyYFGVZm2LIkvIcVpgUflVQ0QUD357UovlDYRQ6dVbzSgaABTtPI4Fb22CHootNXLlyB6YdHm/Zt+aSVEk5Dg1WMzNGw6JiFoSBjVqccKRKNytYAuoUwkhsGrdfiz5ZKexebwsSbh5TC5GDevW7O0xKzKynRaoDGlERAlhUKMWxRcIwe3TEYm0koQGIBKN4vWPduDzHw4aNc2iYPoN+Rjcu22zt0c1VYU0s4khjYgoUQxq1CLooQgqvTqC4UirGUUDAH8wjOeXb8SWklKj1salYdbUQnRt72j29qgmGTkuDSZFbvbnJiJqiRjUKKNFogIenw5fIIxoa0poAE5W+DF7SREOHfcatd5dXJgxuQAue/NvzWQxK8hxWqAwpBERJQ2DGmUkIQR8gRAq/aFWdZqzWsmhCjy3pAiVvpBRGz6wA+66dnBKrgvTzAqyXRqUZp6wQETU0jGoUcYJ6mFU+kLQW9lpzmrfbTuKRe9sQeiUTeSvvqAXrhvVB7LU/EFJUxVkOxnSiIiaAoMaZYxQOAqPX0cgGEYrWnHDIITAB2v2Yvlnu4yaIku47eqBuLCgS7O3RwKgWUzIdliafekPIqLWgkGN0p4QAh5fCN5A69lZ4HThSBSvfbANX204bNRsmgn3TczHgJ5tmr09EgCrVhXSpBSM4hERtRYMapTWgnoYbq+OUDiK1hnRqhbunbdsI7bvKzNq7bOtmDW1EJ3a2pu9PZIE2DQzsuwqQxoRURNjUKO0FJvNGWqVpzmrHS/z4dnFRTha6jNqfbtlYcakAjhsarO3R5IAu2ZGlqP5Z5USEbVGDGqUdvzBECq9IYQi0frv3IIV7y/HnGUb4PXHZnaeO6QT7hg3CGZT8y+BIUmA06rCaW/+gEhE1FoxqFHaiEQFKn06/K18FA0Avt18BC+9uwXhU5Yeufai3hh/ce+UnG6UJcBpU1MyikdE1JoxqFFa4ChaFSEEVn5Zgne+KDFqJkXCHdcMxnlDOqWkTbIEuOwq7FaGNCKi5sagRinFUbSYUDiKl9/bim83HzFqdqsZMyYVoF/37JS0SZYkZDlU2DRzSp6fiKi1Y1CjlOEoWozHp2Pusg0oPlBh1Dq2seGBqYVon2NLSZsUuSqkWS0MaUREqcKgRs2Oo2g1HTnpxezFRThe7jdqA3rkYPqkfNhTNJKlyBKynRZoKr8iiIhSid/C1KwCehhuj85RtB9t31uGf765Ab5A2KhdWNAZt44dCFOKNjdXFAk5DgssDGlERCnHb2JqFhxFO9NXGw7h1fe31dhtYeJlfTHmvJ4pW0jWpEjIdmqwpGBjdyIiOhODGjU5jqLVFBUCb32+G+9/vceomU0y7ho/GOcM7JiydqlmBW1cGswmhjQionTBoEZNhrsLnEkPRbBo5RZ8v+2YUXPZVcyYXIDeXbJS1i6b1YIch4UhjYgozTCoUZOo3qNTD3MUrZrbG8ScpRtQcsht1Lq0s2PmlEK0y7amrF2aaoLTqkBRuG8nEVG6YVCjpOIoWu0OHffg2cVFKHUHjNrg3m3w0+vzYdVS9zG0mBVoVglHD4bqvzMRETU7BjVKGo6i1W5LyUnMW74RgWDEqI0a1hU3XZULRU7NzE4A0FQF2U4NwYAfNqsNJt2M6KnpWgJkO0fZiIhSiUGNEhb9cUYnR9HO9PkPB/CvD3cgKqo6RgIweXR/XDGye8pmdkoALKoJOU4LZLmqDb069IZ3YRgiFFsmRM6W4LzVAsXBsEZElCoMapQQjqLVLhoVWPZpMT5eu8+oqWYZ067LQ2H/9ilrlwTAqpmQ7bAYQVGJKgh+GYW+7cx/Q21kBHKhAklmWCMiSgUGNYoLR9HOLqhH8MLbm1C084RRy3JYMGtKAXp0cqWsXZIEWC01QxoAKEET/B8Gan1M5Ws6zH01KC4GNSKiVGBQo0bjKNrZlVcG8dySIuw7WmnUundwYOaUQuS4tJS1S5IAu2ZGlsNSox71CnjeCAHh2h8XdQv4PwvDNsYM2cKwRkTU3BjUqME4ila3/UcrMXtJEcorg0Ytv287TLt+SEr3zJQkwKGpcDnUM2+MApHyuv8xI6UCYCYnIkqJ1E05o4wS1MM4WeGHx8+QVpuNxSfwv698VyOkXX5ON8yYXJDykOa0niWkAZCdElw/UasuXqvt8RbAcb0ZspWjaUREqcARNaoTR9Hqt2rdfiz+ZAd+nNgJSQJuujIXl53TPaXtkiXAaVfhsNYe0qoJZxSW82UEvz5z2Mx+vcolOoiIUohBjc6K16LVLRKNYvHHO7H6+wNGzaIq+On1ecjr2y6FLQNkSYLLocKumeu9b1gJQbtWgb4pCuinHCNbgnauAsnMoEZElCoManSG2Cha2Fj/i2ryB8OYv2ITNu8+adRyXBbMmlKIbh2cKWxZVUjLdqqwWuoPadUOlR1Ej9/0rLm2mwTIXEONiCilGNSohupRtFA4Cka02pW6A5i9uAgHj3uMWs9OTsycUnjGrMrmpsgSsp2WRl8XV1ZZhk7dOsFmszVRy4iIKB4MagSgahTN4wvBGwhxFK0Oew+7MXtJEdze2DnCYbntcfeEIVDNSgpbVhXScpwWWFI4eYGIiJKL3+jEUbQGWr/jOBa8tQmhU67ZG3NeT9xwWV/IKdoOqpqiSMhxarCkOCwSEVFyMai1YhxFaxghBD76dh/e/LTYCLKyLOGWMQNwydCuKW0bAJgVGdlOS8pH9IiIKPkY1FopjqI1TCQSxb8+2o5/rz9k1KwWE6ZPzMegXm1S2LIqqklGtlOD2cQlEYmIWiIGtVZGMZng8YcREYKjaPXwBUKYt3wTtu0pNWrtsjTMmjoUndvZU9iyKqpJRo5Lg0lhSCMiaqkY1FqRcCQKb0Cg0heEpllT3Zy0dqLcj2cXr8eRkz6j1qdrFmZMLoDTVvcCss3BYlaQ47RAYUgjImrRGNRagepr0U5WBODxBcCBtLrtPliBOUuLUOkLGbURgzriJ9cOgtmU+uvANLOCbJcGReYaZ0RELR2DWgsXDEXg9gQRCkcR4R5Q9Vq39SgWvbMF4UhsZuc1F/bChEv61FwMNkU0VUGOU4PMkEZE1CowqLVQQghUejmjs6GEEHjvqz1469+7jZoiS7jjmkE4P69zCltWRQKgWUzIdlgY0oiIWhEGtRbo1FE0RrT6hcJRvPr+NqzZdNio2TUT7p9UgP49clLYsiqSBNg0M7LsalqM6hERUfNhUGtBhKi6Fs3j5yhaQ3n9Iby4bAt27i83ah1yrJg1dSg6tkn9dkqSBNg1c8q3piIiotRgUGshOIrWeOWeMF5/YyOOlweMWv/u2bhvUgEc1oZvaN5UZAlwWFU47amfZUpERKnBoJbhOIoWn10HK7D4y1IEQ7E+O29IJ9w+blBaLB4rSxJcDhV2LfWBkYiIUodBLYNxFC0+azYexsvvba0xC/a6UX0w7oJeaXENmCJLyHKosFoY0oiIWjsGtQzEUbT4CCHw9r93492v9hg1kyLhJ9cOxsjBnVLXsFMosoQcpwUWlR9NIiJiUMs4wVAElV4deijCUbRGCIUjeHHlVqzbetSoWVUJ02/Iw6A+HVLYshhFkZDj1GDh5upERPQjBrUMwVG0+FX6dMxZugG7D1YYtY5trBhTaEOvzs4UtizGrMjIdlqgMqQREdEpGNQygB6KwM1RtLgcPuHF7MXrcaIiNrNzYM8c3DmuPw4d2JvClsWoJhnZTi0tJjEQEVF6YVBLYxxFS8y2PaWY9+ZG+IJho3ZxYRfcMmYAdD2YwpbFqCYZOS4NJm6uTkREtWBQS1McRUvMl0WH8OoH2xD9cWanBGDi5f1w1bk90mJmJ/Dj5upOCxSGNCIiOgsGtTTDUbTERIXAis924YM1sdOaZpOMeyYMwbAB6TFpAKjaXD3bqUHhvp1ERFQHBrU0wlG0xOihCBa+sxk/bD9u1Fx2FbOmFKJnZ1cKWxbDzdWJiKgxGNTSAEfRElfhCeK5pRuw97DbqHVt78CsKYVok6WlsGUxEgCrVhXS0uX0KxERpTcGtRTjKFriDh7zYPaSIpS6YzM7h/Rpi3uvz4PVkh5v8erN1V12lSGNiIgaLD1+irVCHEVLjs27T+L55RsR0CNG7dLh3XDjlf2hyOlxkb4kAQ5NhcvBzdWJiKhxGNRSgKNoybH6+wN4/aPtqM65kgRMvSIXo0d0T23DTiFJgNOqwmlnSCMiosZjUGtGHEVLjmhUYMmqnVi1br9Rs5gVTLs+DwX92qWwZTXJEuC0q3BYGdKIiCg+DGrNhKNoyRHQw1iwYjM27jph1LKdFsyaUojuHdNjOygAkCUJWQ4VNs2c6qYQEVEGY1BrYhxFS54ydwDPLSnC/mMeo9ajoxMzpxQi22lJYctqUuSqkGa1MKQREVFiGNSaEEfRkmffETdmL9mACk9s66fC/u1wz4Q8WNT02chckSXkOC2wqPxoERFR4vjTpIlUenV4AiFjCyOKX9HO41jw1ibooahRu/LcHph0Wb+0WjRWUSTkOBjSiIgoefgTpYl4GdISJoTAJ2v3Y+mqncaIpCxJuHlMLkYN65bStp3OpMjIcVqgmtNndI+IiDIfgxqlpUg0itc/2oHPfzho1DSLguk35GNw77YpbNmZVJOMbKcFZhNDGhERJReDGqUdfzCM55dvxJaSUqPWxqXhgamF6NLekcKWnUk1ychxaTAp6bG4LhERtSwMapRWTlb4MXtJEQ4d9xq13l1cmDG5AC57+szsBKrWbstxWqAwpBERURNhUKO0UXKoAnOWboDbqxu14QM74K5rB6fdtV+aWUG2S4OSRpMZiIio5WFQo7Tw3bajWPTOFoTCsZmdV1/QC9eN6gM5jTYxlwBYVBNynJa0mnFKREQtE4MapZQQAh+s2Yvln+0yaoos4barB+LCgi4pbNmZJABWzYRshwVSGoVHIiJquRjUKGXCkShe+2Abvtpw2KjZNBPum5iPAT3bpLBlZ5IkwKaZkWVXGdKIiKjZMKhRSngDIcxbthHb95UZtfbZVsyaWohObe0pbNmZJAmwa2ZkOdJrMgMREbV8DGrU7I6X+fDs4iIcLfUZtX7dsnD/pAI4bGoKW3YmSQKcVhVOe3q1i4iIWgcGNWpWxfvLMWfZBnj9IaN23pBOuH3cIJhN6bXMhSwBTpuaduGRiIhaDwY1ajbfbj6Cl97dgnAktrXW+It749qLeqfddV+yJMFlN8NuZUgjIqLUSfkQxtGjRzFgwIAz/lu2bBkAYOvWrbj99tsxdOhQjB49Gi+99FKKW0yNJYTAO1/sxgtvbzZCmkmRcM+EIRh/cZ+0DGlZDpUhjYiIUi7lI2rbtm2DxWLBxx9/XOMHttPpRFlZGe6++26MHj0av/vd77B+/Xr87ne/g91ux+TJk1PYamqoUDiKl9/bim83HzFqdqsZMyYXoF+37NQ17CwUuSqkWS3mVDeFiIgo9UFtx44d6NWrFzp06HDGbS+++CLMZjOeeOIJmEwm9O3bF3v37sW8efMY1DKAx6dj7rINKD5QYdQ6trHhgamFaJ9jS2HLaqfIEnKcFljUlH8siIiIAKRBUNu+fTv69u1b623r1q3DueeeC5Mp1szzzz8f//znP3HixAm0a9curucUQsDn89V/xwQEggGET1llPx0EA8EafzalY2V+PL9iK05UBIxav24u3H3tQNg0CX6/v8nbUJfT+8JsUuB0qoiEdfjCel0PbXGq/y1S/W+SDtgXMeyLGPZFDPsipqF9IYRI6BKflAe1HTt2ICcnB7fddhtKSkrQs2dPzJgxA6NGjcKRI0eQm5tb4/7VI2+HDx+OO6iFQiFs3bo14bafjclkQrk3gkAwPX/gHzh4oGmPf0LHe9+VIxiKTRoY1F3DZfkaDh/c26TP3VgHDh6ATbPAYTPh5JEQhBD1P6iF2rNnT6qbkDbYFzHsixj2RQz7IqYhfaGq8V/znNKgFg6HsXv3bvTr1w+PPvooHA4HVq5cienTp2PhwoUIBAJnvDiLpWrR0WAw/lEhs9mMfv36JdT2+hwr96fliNqBgwfQrWs3WLSmWbz1m81H8da3uxGNxgLPtRf2wBUjuqbVpIFgIIiDhw6gT69e6NjWCUVJn7Y1N7/fjz179qBXr16wWq2pbk5KsS9i2Bcx7IsY9kVMQ/uiuLg4oedJaVAzmUz45ptvoCgKNE0DAOTl5WHnzp1YsGABNE2DrtcclaoOaDZb/Nc4SZKU0OMbQvMLRMzpOTpj0SxJ/4BFhcBbn+/C+1/HRszMJhl3jR+McwZ2TOpzJYvNqqFjOyecjvTaCSFVrFZrk38uMgX7IoZ9EcO+iGFfxNTXF4kOUqT81KfdfuYPyf79++OLL75Ap06dcOzYsRq3Vf+9Y8f0/OHfGumhCBat3ILvt8X+rZw2M2ZOKUTvLlkpbNnZaaoJDk2GIrfekTQiIkp/KV1HbefOnRg+fDi++eabGvVNmzahX79+GDlyJL777jtEIhHjtjVr1qB3795o27ZtczeXauH2BvH3//u+Rkjr0s6OR38yMn1DmllBjtOCSDhU/52JiIhSKKVBrW/fvujTpw+eeOIJrFu3Drt27cKf/vQnrF+/HjNmzMDkyZPh8Xjw61//GsXFxVi2bBkWLVqE++67L5XNph8dOu7Bn19ch5JDbqM2uHcb/Or2EWiblZ7XLmiqgmyXhjS6XI6IiOisUnrqU5ZlzJ07F3/729/w4IMPwu12Y/DgwVi4cKEx23P+/Pn4wx/+gIkTJ6J9+/Z4+OGHMXHixFQ2mwBsKTmJecs3IhCMjXaOGtYVN12VC0VO+YYXtdJUE3KcFsg83UlERBki5deotWvXDn/605/OentBQQFef/31ZmwR1efzHw7gXx/uQPTHpSwkAJNH98cVI7un1czOahIAzWJCtoMhjYiIMkvKgxpljmhUYNmnxfh47T6jpppl3DMhD0Nz26ewZWdXHdJynJa0DJFERER1YVCjBgnqEbzw9iYU7Txh1LIcFsyaUoAenVwpbNnZSQCsWtVIGkMaERFlIgY1qld5ZRDPLSnCvqOVRq1bBwdmTSlEjktLYcvOTpIAm2ZGll1lSCMioozFoEZ12n+0ErOXFKG8MrYTRH7ftph2fR60NN28XJIAu2ZGlqNpdl8gIiJqLun5k5bSwsbiE5i/YhOCodjMztEjumPK6P5pe1E+QxoREbUkDGpUq1Xr9mPxJztQvUe5JAE3XZmLy87pntqG1UGSAIemwuWIf/NbIiKidMKgRjVEolEs/ngnVn9/wKhZVAU/vT4PeX3bpbBldZMkwGlV4bQzpBERUcvBoEYGfzCM+Ss2YfPuk0Ytx2XBrCmF6NbBmcKW1U2WAKdNhcPGkEZERC0LgxoBAEorApi9pAgHj3uMWs9OTsycUpjW13vJEuC0q3BYGdKIiKjlYVAj7D3sxuwlRXB7daM2LLc97p4wBKpZSWHL6iZLElwOFXbNnOqmEBERNQkGtVbuh+3H8MLbmxEKR43amPN64obL+kJO4/XHZElClkOFjSGNiIhaMAa1VkoIgY++3Yc3Py3GjxM7IcsSbhkzAJcM7ZrSttVHliRkO1VYLQxpRETUsjGotUKRSBSvvr8NXxQdMmpWiwn3TczHwF5tUtiy+ily1UgaQxoREbUGDGqtTDAUxbwVW7Fjf4VRa5elYdbUoejczp7CltVPkSVkOy1puyMCERFRsvEnXitysiKApV+WotQT22mgT9cszJhcAGeaL22hyBJynBZYGNKIiKgV4U+9VmL3wQo8t2QDPP5YSBsxqCN+cu0gmE3pO7MTABRFQo6DIY2IiFof/uRrBdZtPYpF72xBOBKb2XnNhb0w/pI+aT2zEwBMiowcpyWtlwkhIiJqKgxqLZgQAu99vQdvfb7bqMkScPNV/TBqeM8UtqxhzIqMHJcl7Uf8iIiImgqDWgsVjkTxynvbsGbTYaNm00wYO8yJkYM6pLBlDaOaZGQ7NZhNcqqbQkRElDIMai2Q1x/C3GUbsHN/uVHrkGPFvRMGwl166OwPTBOqSUaOS4NJYUgjIqLWjUGthTla6sPsxetxrMxv1Pp3z8Z9kwqgIAx3aQob1wAWs4IcpwUKQxoRERGDWkuyc18Z5i7bAG8gbNTOz+uE264eBLNJht8fruPRqaeZFWS7NChyek9wICIiai4Mai3Emk2H8fK7WxGJCqN23ag+GHdBL0hpPrMTADTVhBynBTJDGhERkYFBLcMJIfD2v3fj3a/2GDWTIuMn1w7CyMGdUtewBpIAaBYTsh0MaURERKdjUMtgoXAEL67cinVbjxo1p82MGZML0adrVgpb1jASAKtWFdIyYdSPiIiouTGoZahKn445Szdg98HYnp2d2towa+pQtM+2prBlDSNJgF0zw2VXGdKIiIjOgkEtAx0+4cXsxetxoiJg1Ab2zMH0ifmwaeYUtqxhqkNalsOS6qYQERGlNQa1DLNtTyn++eZG+IOxGZwXF3bBLWMGZMSSFpIEOK0qnPb03gSeiIgoHTCoZZAvig7itQ+2I/rjzE4JwMTL++Gqc3tkxOlDWQKcNhUOG0MaERFRQzCoZYCoEFi+ehc+/GavUTObZNwzYQiGDUj/7aAAQJYkuOxm2K0MaURERA3FoJbm9FAEC9/ejB92HDdqLruKWVMK0bOzK4UtazhZkpDlUDPi+jkiIqJ0wqCWxio8QTy3pAh7j1Qata7tHZg1tRBtXFoKW9ZwilwV0qwWhjQiIqLGYlBLUwePefDskvUocweN2pA+bfHT6/OgWTLjn02RJeQ4LbComdFeIiKidMOfoGlo064TmL9iEwJ6xKhdNrwbpl7ZH4qc/jM7AUBRJOQ4GNKIiIgSwZ+iaWb1d/vx+sc7IH7cslOSgBuvyMXlI7qntmGNYFJk5DgtUM1KqptCRESU0RjU0kQ0KrBk1U6sWrffqFlUBfdel4f8fu1S2LLGMSsyclwWmE0MaURERIliUEsDAT2MBSs2Y+OuE0Ytx2nBrCmF6NbRmcKWNY5qkpHt1GA2ZcbpWSIionTHoJZiZe4AnltShP3HPEatR0cnZk4pRLYzc7ZYUk0y2ri0jNgdgYiIKFMwqKXQviNuzF6yARWe2MzOwv7tcM+EPFjUzDl1qJkVZDstDGlERERJxqCWIkU7j2PBW5ugh6JG7cqRPTDp8n6Q5fTfDqqapirIdmpQMqjNREREmYJBrZkJIfDJ2v1YumonfpzYCVmScPOYXIwa1i2lbWsMCYBFNSHHacmoYElERJRJGNSaUSQaxesf7cDnPxw0appFwfQb8jG4d9sUtqxxJABWzYRshyUjNoMnIiLKVAxqzcQfCOP5FRuxpaTUqLVxaZg1tRBd2ztS2LLGkSTAppmRZVcZ0oiIiJoYg1ozOFnhx+zFRTh0wmvUendxYcbkArjsmTOzU5IAu2ZGliNz2kxERJTJGNSaWMmhCjy3pAiVvpBRGz6wA+66dnBGrdwvSYBDU+FyqKluChERUavBoNaEvtt2FIve2YJQODaz8+oLeuK6UX0hZ9BpQ1kCHFYVTjtDGhERUXNiUGsCQgi899UeLFtdbNRkWcLtVw/EhQVdUtiyxpMlwGlX4bAypBERETU3BrUkC4WjmLO0CB99u8+o2TQT7puYjwE926SwZY0nSxJcDhV2zZzqphAREbVKDGpJ9o83fsDq7w4Yf2+fbcWsqYXo1NaewlY1nixJyHaqsFoY0oiIiFKFQS3J1mw8bPx/v25ZuH9SARy2zDptqMgSsp0WaCrfHkRERKnEzRmT7OarBqCNy4LLz+mGX9w8PCNDWg5DGhERUVrgT+Mkmzy6PyaP7o8jJ72IREX9D0gjiiIhx6nBkkHLhhAREbVkDGoEADArMrKdloxa242IiKilY1AjqCYZ2U4NZhPPhBMREaUTBrVWTjXJyHFpMCkMaUREROmGQa0Vs5gV5DgtUBjSiIiI0hKDWiskAdDMCrJdGhQ5c7ayIiIiam0Y1FoZWZZgtZiQ49IgM6QRERGlNZ7zakUkCXDYLMhyWBjSiIiIMgCDWishAbBrZthUCRIzGhERUUZgUGsFJAmwWc1w2VWEw+FUN4eIiIgaiEGthZOkqpG0bIcl1U0hIiKiRuJkghZMkgCnVYXTnln7jRIREVEVBrUWSpYAp03NuE3hiYiIKIZBrQWSJQkuuxl2K0MaERFRJmNQa2FkSUKWQ4VNM6e6KURERJQgBrUWRJGrQprVwpBGRETUEjCotRCKLCHHaYFF5T8pERFRS8Gf6i2AokjIcTCkERERtTT8yZ7hTIqMHKcFqllJdVOIiIgoyRjUMphZkZHjssBsYkgjIiJqiRjUMpRqkpHt1GA2cXMJIiKilopBLQOpJhltXBoUhSGNiIioJWNQyzCaWUG208KQRkRE1AowqGUQTVWQ7dSgyFKqm0JERETNgEEtA0gALKoJOU4LZIY0IiKiVoNBLc1JAKyaCdkOCySJIY2IiKg1YVBLY5IE2DQzsuwqQxoREVErxKCWpiQJsGtmZDksqW4KERERpQiDWhqSJMChqXA51FQ3hYiIiFKIQS3NyBLgsKpw2hnSiIiIWjsGtTQiS4DTrsJhZUgjIiIiBrW0IUsSXA4Vds2c6qYQERFRmmBQSwOyLCHbocJqYUgjIiKiGAa1FFNkCdlOCzSV/xRERERUE9NBCimyhBynBRaGNCIiIqoFE0KKmBQJ2U4NFrOS6qYQERFRmmJQSwGzIiPHZYHZxJBGREREZ8eg1sxUk4xspwazSU51U4iIiCjNMag1I9UkI8elwaQwpBEREVH9GNSaicWsIMdpgcKQRkRERA3E1NAERCQKRZaMv2tmBTkujSGNiIiIGoXJIcmEP4zI2uNwoCqoaaqpKqSdEtyIiIiIGoJBLcmEL4zQSztgDkRh10zIcVogM6QRERFRHBjUEiRCUYhKHVG3DuHWEVpWAggg/MZuuKIyJE8IolKH0COpbioRERFlGE4mSFQogug+D/T524BALIyJbeUIPvoNYJFhvr0/lME5gMp104iIiKjhOKKWIMlmhtzHBfOk3rXebhrXA8rgHEg2brhOREREjZNWQa2kpATDhg3DsmXLjNpvfvMbDBgwoMZ/o0ePTmErzyRZTVBGtIc8OKdmvZcTpos7MaQRERFRXNLm1GcoFMJ//ud/wufz1ahv374d999/P26//XajpihpeAoxHIVw61X/LwEQANw6EBGpbBURERFlsLQJas888wwcDkeNmhACxcXFmD59Otq3b5+iljWQAOALw3zXACj5bRDZUY7w4t0QADjnk4iIiOKRFkFt7dq1eP3117F8+XJcdtllRn3fvn3w+Xzo06dPUp9PCHHGyF2irJoF5ocLEVYFdKFDHuCA+eFCSDZT0p8rXn6/v8afrRn7IoZ9EcO+iGFfxLAvYtgXMQ3tCyEEJCn+IZuUBzW3242HH34Yv/nNb9C5c+cat+3YsQMA8PLLL+Pzzz+HLMsYNWoUHnroITidzrifMxQKYevWrQm1+3QOhwMej+eMutPpRGVlZVKfK1F79uxJdRPSBvsihn0Rw76IYV/EsC9i2BcxDekLVVXjPn7Kg9rjjz+OYcOGYcKECWfctmPHDsiyjA4dOmDu3LnYt28f/vrXv2Lnzp148cUXIcvxzYUwm83o169fok3POH6/H3v27EGvXr1gtVpT3ZyUYl/EsC9i2Bcx7IsY9kUM+yKmoX1RXFyc0POkNKgtX74c69atw9tvv13r7TNmzMCtt96KnJyq2ZS5ublo3749brzxRmzcuBGFhYVxPa8kSbDZbHG3O9NZrdZW/fpPxb6IYV/EsC9i2Bcx7IsY9kVMfX2RyGlPIMXLcyxduhQnT57EZZddhmHDhmHYsGEAgN/+9re49957IcuyEdKq9e/fHwBw5MiRZm8vERERUXNK6Yja//7v/yIQCNSojRkzBj//+c9x3XXX4eGHH8axY8ewaNEi4/aNGzcCQKs8dUlEREStS0qDWseOHWutt23bFh07dsTYsWMxc+ZMPPvss7juuutQUlKCJ554AuPHj0ffvn2bubVEREREzSvlkwnqcsUVV+Cpp57CvHnz8Pzzz8PpdGLChAl48MEHU900IiIioiaXdkFt+/btNf4+btw4jBs3LkWtISIiIkqdtNrrk4iIiIhiGNSIiIiI0hSDGhEREVGaYlAjIiIiSlMMakRERERpikGNiIiIKE0xqBERERGlKQY1IiIiojTFoEZERESUphjUiIiIiNIUgxoRERFRmpKEECLVjWhO33//PYQQUFU11U1pdkIIhEIhmM1mSJKU6uakFPsihn0Rw76IYV/EsC9i2BcxDe0LXdchSRKGDx8e1/Ok3absTa01v7EkSWqVAbU27IsY9kUM+yKGfRHDvohhX8Q0tC8kSUooe7S6ETUiIiKiTMFr1IiIiIjSFIMaERERUZpiUCMiIiJKUwxqRERERGmKQY2IiIgoTTGoEREREaUpBjUiIiKiNMWgRkRERJSmGNSIiIiI0hSDGhEREVGaYlAjIiIiSlMMakRERERpikEtgx09ehQDBgw4479ly5YBALZu3Yrbb78dQ4cOxejRo/HSSy/Ve8z33nsP11xzDQoKCnDDDTfg66+/buqXkRT19cWqVaswefJkDBs2DKNHj8Zf/vIXBAKBsx4vEomgoKDgjOM988wzzfWS4lZfX/zmN78547bRo0fXecyW+L644447ar1twIABWL58+VmPeffdd59x/zvuuKP5XlQCli9fjmuuuQb5+fm49tpr8d577xm3HThwAPfddx+GDx+Oiy++GE899RQikUidx/v6668xadIkFBYW4uqrr8bKlSub+iUkTV198f333+OOO+7AOeecg0suuQS//vWvUV5eXufxxowZc8b74tFHH23iV5EcdfXFnDlzav2M1KUlvi8effTRs35fPPvss2c9Xjzft2cQlLFWr14t8vPzxdGjR8WxY8eM//x+vygtLRXnnXeeeOyxx0RxcbFYsmSJyM/PF0uWLDnr8b7++msxZMgQ8eKLL4ri4mLx5z//WeTl5Yni4uJmfFXxqasv1q5dKwYNGiTmzJkjSkpKxOrVq8WoUaPEo48+etbjFRcXi9zcXLF169Yax/N4PM34quJTV18IIcSUKVPEk08+WeO2kydPnvV4LfV9UVZWVqN29OhRceutt4prr722zn/nCy64QLz22ms1HltWVtZ8LypOy5cvF4MHDxavvPKK2Lt3r3juuefEwIEDxffffy90XRdjxowR06dPF9u3bxcfffSROPfcc8XTTz991uMVFxeL/Px88eSTT4ri4mIxf/58MXjwYPHVV18146uKT119sXv3bjF06FDx+9//XhQXF4u1a9eK8ePHizvvvPOsx/N6vWLgwIHi008/rfG+cLvdzfiq4lNXXwghxC9+8Qvxq1/9qsbrOnbs2FmP11LfF263+4w+eOihh8RFF10kjhw5ctZjNvb7tjYMahls3rx5YsKECbXeNnfuXHHxxReLUChk1P72t7+JMWPGnPV499xzj/jFL35Ro3bTTTeJ//f//l9S2tuU6uqL//iP/xB33XVXjdqbb74phgwZIoLBYK2PWblypRg+fHjS29kc6uqLaDQqhg4dKj788MMGH6+lvi9O9/LLL4u8vDyxa9eus97nxIkTIjc3V2zevDlZTWwW0WhUXH755eLPf/5zjfo999wj5s6dK95++22Rl5cnysvLjdv+9a9/ieHDh5/1M/L//t//E1OmTKlR++Uvfynuueee5L+AJKqvL5588kkxZswYEY1GjdvWrl0rcnNzxb59+2o9ZlFRkcjNza3Rf5mgvr4QQohx48aJhQsXNviYLfV9cbpPPvlEDBgwQKxZs6bOYzb2+7Y2pkSGCCm1tm/fjr59+9Z627p163DuuefCZIr9E59//vn45z//iRMnTqBdu3Y17h+NRvH999+fMVR/3nnn4cMPP0x+45Osrr645557IMs1z/LLsoxQKASPx4M2bdo06njprq6279u3Dz6fD3369GnQsVry++JUpaWleOqppzBjxow6+2b79u2QJAm9e/dOZjObXElJCQ4ePIgJEybUqC9YsAAA8Pjjj2PIkCHIysoybjv//PPh8XiwdetWFBYWnnHMdevW4corr6xRO//88/GHP/wBQghIktQEryRx9fXFrl27cPnll9dof/X/V1RUoHv37mccc/v27WjXrl2N/ssE9fWFruvYs2dPg78vgJb7vjhVMBjEH/7wB0yePBnnnXfeWY/Z2O/bs+E1ahlsx44dKC0txW233YYLL7wQt9xyCz7//HMAwJEjR9CpU6ca9+/QoQMA4PDhw2ccy+12w+fz1fqYI0eONNErSJ66+mLw4MEYOHCgcd9QKIRFixYhLy+v1pBWfbxwOIxp06bhoosuwqRJk7BixYpmeS2JqqsvduzYAQB4+eWXMXr0aFx55ZV44oknUFlZWeuxWvL74lTPP/88NE3DtGnT6j2e0+nEE088gVGjRuHqq6/GU089BV3Xm+olJEVJSQkAwOfzYdq0abjgggswdepUrFq1CkDjvy/qeozf70dZWVmyX0LS1NcXffv2xdChQ2s85vnnn0f79u3Pem3W9u3bYbPZ8POf/xwXX3wxJkyYgEWLFiEajTbpa0lUfX1RXFyMSCSCDz74AGPHjsVll12GX/3qVzh27NhZj9lS3xenWrx4MU6cOIEHH3ywzmM29vv2bBjUMlQ4HMbu3btRUVGBn/3sZ5g3bx6GDh2K6dOn4+uvv0YgEICqqjUeY7FYAFT9NnC66gvra3tMbfdPJ/X1xen3ffjhh7Fz50789re/Pesxd+7cifLyctxxxx1YsGABxo4di8ceewxLlixp6peTkPr6YseOHZBlGR06dMDcuXPx6KOP4osvvsDMmTNr/aHSGt4XHo8Hb7zxBqZNm2Z8Rs5mx44dCAaDKCgowPz58zFjxgwsXrwYv/nNb5r65STE4/EAAB555BGMHz8eL7zwAi666CLMnDkzru8LALU+pvrv6Rxc6+uL0/3lL3/B6tWr8fjjj8NsNtd6zJ07d8LtdmPs2LFYsGABbrnlFjz99NNpP/movr6oDhpWqxVPP/00/vCHP2D37t248847zzoZq6W/L6LRKF588UVMnToV7du3r/OYjf2+PRue+sxQJpMJ33zzDRRFgaZpAIC8vDzs3LkTCxYsgKZpZ3woqr9wbTbbGcer/lKu7TFWq7UpXkLS1NcXF1xwAYCqD+KDDz6Ib7/9Fs8++ywKCgrOesx33nkHkUgEdrsdADBw4EAcOnQICxYswJQpU5r+RcWpvr6YN28ebr31VuTk5AAAcnNz0b59e9x4443YuHHjGae4WsP74uOPP4au65g8eXK9x3ziiSfwyCOPGKe4cnNzYTab8dBDD+Hhhx8+45KCdFEdMKZNm4aJEycCAAYNGoQtW7Zg4cKFjf6+AKreG6c/pvrv6fzeqK8vqt8XoVAI//3f/43ly5fj97///Rmn8071/PPPIxgMwul0AgAGDBgAj8eDOXPm4Gc/+9kZl16ki/r6Yt68eRg1alSNMw/9+/fHqFGjsGrVKlxzzTVnHLOlvy++//577Nu3D7fccku9x5wxY0ajvm/PJj3fPdQgdrvd+AFUrX///jh69Cg6dep0xvB09d87dux4xrGys7Nhs9lqfUxt9083dfUFUPU6brvtNqxfvx4LFizApZdeWufxNE0zQlq13NzcjDjdV1dfyLJsfGmcehuAWl9bS39fAFVB7dJLL4XL5ar3eCaT6YzrkOrqv3RR/W+Vm5tbo96vXz8cOHCg0d8XANC5c+daH2Oz2YzAko7q6wug6pe6n/70p3j77bfx5JNPYurUqXUeU1XVM15zbm4ufD4fKioqktj65GpIX5x+eUiHDh2QnZ191vd7S35fAMBHH32EwYMHN+ja18Z+3571OA2+J6WVnTt3Yvjw4fjmm29q1Ddt2oR+/fph5MiR+O6772qsg7RmzRr07t0bbdu2PeN4kiRh+PDh+Pbbb2vUv/nmG4wYMaJpXkSS1NcXFRUV+MlPfoLS0lK8+uqrGDlyZJ3Hc7vdOPfcc411x6pt3LjR+JClq/r64uGHH8Zdd91V47aNGzcCqPpCOl1Lfl9UW7dunfHbcn3uuOMOPPbYYzVqGzduhNlsRq9evRJuc1MZMmQI7HY7ioqKatR37NiBHj16YOTIkdiyZYtx+geo+r6w2+01ru881YgRI854X6xZswbDhw9P2xEkoP6+0HUd9913HzZs2IAFCxZg3LhxdR5PCIErr7zyjLW0Nm7ciPbt25/xgzqd1NcXf//73zF27FgIIYzbDhw4gLKyslq/L4CW+76otnbt2gZ/XzT2+/asEpozSikTiUTE5MmTxTXXXCPWrl0riouLxR//+EeRl5cntm/fLk6cOCFGjhwpHnnkEbFz506xdOlSkZ+fL5YtW2Ycw+1211jP5d///rcYNGiQeOGFF0RxcbH4y1/+IgoKCtJ+vaz6+uKRRx4RQ4YMEV9//fUZ6+CEw2EhhBBlZWU11sL62c9+Ji6++GKxevVqUVJSIv75z3+KQYMGic8//zxFr7Jh6uuLjz/+WOTm5opnnnlG7N27V6xevVqMHj1a/PKXvzSO0VreF0IIcejQIZGbmyvWrVtX6zE8Hk+NNaNefvllMWjQIPHaa6+Jffv2iZUrV4rzzjtPPPnkk83ymhIxe/ZsMWzYMPH222/XWCNqzZo1IhAIiCuvvFJMmzZNbN261VhH7ZlnnjEef3pf7NixQwwZMkT8z//8jyguLhYLFizImPWy6uqLf/zjH2LAgAHinXfeOeP7onqpktM/I3/+85/F0KFDxcqVK8XevXvFv/71L1FQUCBef/31VL3EBqurLzZu3CiGDBki/vu//1vs3r1bfPvtt+KGG24QN998s7F8SWt5XwghRDgcFkOGDBErVqyo9fF+v7/Gz5WGfN82BINaBjt+/Lh49NFHxUUXXSTy8/PFTTfdJNauXWvcXlRUJG688UaRl5cnLr/8cvHyyy/XePwjjzwiLr/88hq1N998U1x11VUiPz9fTJw4MSM+XEKcvS/C4bDIz88Xubm5tf63f/9+IYQQt99+u7j99tuN41VWVoo//vGP4tJLLxV5eXni+uuvFx999FGqXl6j1Pe+ePfdd8UNN9wgCgoKxEUXXST+/Oc/i0AgYNzeGt4X1arXvzpb6PzHP/4hcnNza9ReeeUVMW7cOONzNWfOHBGJRJr0dSTLCy+8IEaPHi2GDBkirrvuuhrv6T179oi7775b5Ofni4svvlg89dRTNV5XbX3x2WefifHjx4u8vDxx9dVXi5UrVzbba0nU2fpizJgxZ/2+qP6BffpnJBQKiWeffVZcccUVYsiQIWLs2LEZEdKq1fW++Oqrr8RNN90khg4dKs4991zx2GOP1VgvrrW8L4SIraP42Wef1frYpUuX1vi5IkT937cNIQlxypgmEREREaWN9D1hTERERNTKMagRERERpSkGNSIiIqI0xaBGRERElKYY1IiIiIjSFIMaERERUZpiUCMiIiJKUwxqREQtBJfFJGp5GNSIKCnuuOMODBgwADfffPNZ7/PQQw9hwIABePTRR2s8pq7/qu/76KOP1nm/iy66yHie2u47fPhw3Hjjjfjwww9rbduGDRswduxY6LqexF5Jnm+++QYDBgww9i595plnMGDAAOP27777DtOnT2/0cW+//Xa8++67SWsnESWXKdUNIKKWQ5ZlrF+/HkeOHEGnTp1q3Obz+fDpp5/WqP32t7+tsRH47373O6Ne7f9n787jckr/x4+/WkWSIoSQqNCKkDXFtFiTNSLZGrv5oOzVzMg6JGYqGjQYBiFC1g+Tz9iGwWRJFJJdmNB69/ujX+frrlC0muv5eNwPnetc5zrXOe77nPe5ruuco62tLf2to6OT7+XXuVRUVOSm380rk8l4+fIl+/btY/LkyYSGhsoFdmlpaXh5eTFjxgxUVVWLssllZsCAAXTq1Ema3r59O7du3SpyObNnz2bUqFG0bduWGjVqFGcVBUEoBiJQEwSh2DRv3py4uDgOHjyIu7u73Lzjx49TuXJlqlWrJqU1adJELk/VqlUBsLCwKLB8VVXV984rTF4bGxsuXrzItm3b5AK1LVu2oKysTLdu3QpVdnlQp06dfMHwp2jevDlmZmb89NNPzJ07txhqJghCcRJdn4IgFJsqVarQpUsXDh48mG/e/v37sbe3R1m57K4PFRQU0NDQQEFBQUpLT09n/fr19OzZUy5vWloaS5YsoUuXLpiYmNCrV698XYS2trasWrWKxYsX0759e8zMzBg1ahQJCQly+Xbt2oWTkxOmpqb07t2bP/74g+bNmxMeHv7J2/Ju16e3tze7du3i/v37GBkZSeUWZhsAevXqxY4dO3j+/Pkn10cQhJIhAjVBEIqVk5OT1P2ZKyUlhZMnT+YLhj5FZmZmgZ+CBtLnzsvIyCA5OZmwsDBu3rzJkCFDpDxnzpzh0aNHfPXVV1JadnY2EyZMYOvWrYwcOZKffvoJS0tLpk2bxu7du+XWERYWxu3bt/H39+e7777j77//xsvLS5q/e/duvL29admyJT/++CP29vaMHz+erKysz94XucaPH0+XLl3Q0dFh27Zt2NjYFGkbbG1tycrK4vDhw8VWJ0EQiofo+hQEoVjZ2NhQuXJlue7Pw4cPU6NGDVq1avVZZd+/f58WLVoUOG/mzJmMGjXqo3mHDBlCmzZtpOnTp09TrVo19PX1pbT//e9//P7776xYsQInJycAOnXqxNu3b1m2bBk9e/aUWgarVavGjz/+iJKSEgB3794lMDCQ5ORktLS0CAgIoGvXrnz33XdSOSoqKixfvvyz9sW7GjRogLa2tlx376lTpwq9DVWqVMHAwIA//viDQYMGFVu9BEH4fCJQEwShWKmpqWFraysXqEVGRuLo6CjX5fgpdHR0+Omnnwqcp6ur+8G8KSkpnD9/npCQEFJSUli2bBkA9+7do169enLL/vHHHygoKNClSxcyMzOldFtbWyIiIrh58ybNmjUDwNTUVArSAGnc2Nu3b3n16hVJSUlMmTJFrvwePXoUa6BWkKJsA0C9evVITEws0ToJglB0IlATBKHYOTo6MnHiRB4+fEilSpX4448/mDp16meXq6qqiqmp6Sfntba2RllZmZUrVzJy5EhatGhBSkoKlStXlsv34sULsrOzadmyZYFlP378WApy8i6rqJgzokQmk0ljvvLeTVmzZs1CbcPnKMo2QM52/PPPPyVeL0EQikYEaoIgFLvOnTujrq7OwYMHqVKlCvXr18fExKSsqwUg1ePOnTu0aNECLS0tHj9+LJdHQ0ODKlWqEBYWVmAZDRs2LNS6clvXnj17Jpeed7okFHUbXr16hZaWVonXSxCEohE3EwiCUOxUVVXp1q0bUVFRHDhwgB49epR1lSSXL18G/i9QqVu3Lg8fPpS7GaFNmza8efOG7OxsTE1NpU9sbCxr1qyR60r8kDp16tCgQYN8g/Tf99Ddz5HbkperqNvw8OHDfF3AgiCUPdGiJghCiXBycmLcuHEoKioW2/O50tPT+euvv94738jISOqKzJs3MzOTs2fP8tNPP9GxY0fpRoMOHToQEhJCbGys9LiLLl26YGVlxfjx4xk/fjwGBgZcvnyZVatW0alTJ7mH8H6IgoICkydPZvr06SxYsIDu3btz/fp11qxZA8gHV3/99Rfa2to0aNCgKLtEUq1aNZ4+fcqJEydo1qxZkbbhn3/+4ebNm3h4eHzSugVBKDkiUBMEoUS0b9+eatWqoauri4GBQbGU+eTJkw/elbh7925p3FXevCoqKtSrV4/hw4czYcIEKb1169bUqFGDEydOSIGaoqIiISEhBAQEEBwczLNnz6hduzYjR46UW7YwevXqxZs3bwgNDWXnzp00bdqUOXPmMGfOHKpUqSLlGzRoEM7OzixatKhI5efq168fJ06cYMKECUyePJmxY8cWeht+//13VFRUsLGx+aR1C4JQchSyxVt8BUH4l/v555/59ddfOXTo0GffmZrXvn37aN68OY0bN5bS/vvf/zJu3Dj27NmDsbExkHOX5oEDB/Dz8yvW9RfGiBEjMDQ0ZM6cOaW+bkEQPkyMURME4V/P1dUVmUxW4BsVPldERARjxoxh7969nD9/np07d7JgwQLatGkjBWkymYx169bJvdaqtFy5coXr169/0gvdBUEoeaJFTRAEAbhw4QLe3t7s27evWF/MnpyczPLlyzl58iTPnz+nZs2a2NvbM3nyZNTV1aV8MTEx732Yb0lydXXF1dW1WN4aIQhC8ROBmiAIgiAIQjkluj4FQRAEQRDKKRGoCYIgCIIglFMiUBMEQRAEQSinRKAmCIIgCIJQTolATRAEQRAEoZwSgZogCIIgCEI5JQI1QRAEQRCEckoEaoIgCIIgCOWUCNQEQRAEQRDKKRGoCYIgCIIglFMiUBMEQRAEQSinlMu6Av9G2dnZZGVlkZmZWdZVEQRBEIQvnoqKCkpKSmVdjU8iArVSlJ2dzYsXL3jy5AlZWVllXR1BEARB+NeoXr06derUQUFBoayrUiQiUCtFDx8+5MWLF1SrVo1q1aqhrKxc4b4wgiAIglCRZGdn8+bNGx4/fgyArq5uGdeoaESgVkqysrJ4+fIlOjo61KxZs6yrIwiCIAj/GpUrVwbg8ePH1KpVq0J1g4qbCUpJRkYG2dnZqKurl3VVBEEQBOFfp0qVKkDO+bgiEYFaKRNdnYIgCIJQ+irq+VcEaoIgCIIgCOWUCNQEQRAEQRDKKRGoCUVma2uLkZER69evL3D+/PnzMTIyIjAw8LPXU5QyPpbfyMiIAQMGFPhoFDc3N7y9vT+pnoJQ3rxKk/HsrYwnb7LyfZ69lfEqTVam9QsMDMTW1laaNjIyIjw8/L35z5w5g5GREc2bN+f58+f55qenp9O6dWuMjIxITEwEiv6bfvf4ER4ejpGR0QfzGxkZYWRkxKFDhwqcP2rUqEJvV26dPyY4OBg3N7d86ceOHcPFxQVLS0tsbW1ZvHgxqamphSpTKP/EXZ/CJ1FRUSEqKoqRI0fKpWdmZnLo0KFyOxbg8uXLhIaGMnbs2LKuiiCUGFUl2HL1LZti8p+s3VqoMcK0chnU6vMpKipy+PBhBg0aJJd+8uRJUlJS5NICAwOLdGffjh07qFSpUpHqk3sc/Oqrr+TSk5OTOXPmTJHK+pjNmzezcuVKWrduLZd+/vx5Jk6cyOTJk3FwcODOnTvMnz+fFy9e4O/vX6x1EMqGaFETPom1tTV//fUXDx8+lEs/ffo0VapUKbfPqdHT0yMwMJC4uLiyrooglBg1ZUUGGFdGNU+cUkkJ+htXRk25Yh76ra2tOXjwYL70AwcO5AtgqlevjoaGRqHL1tbWLvJd+dbW1hw/fpy0tDS59EOHDmFhYVGkst7n0aNHeHp6smzZMho1apRv/tatW2nbti2enp40atSILl26MG3aNPbu3Ut6enqx1EEoWxXz1yqUOTMzM+rWrZvvoLl//34cHR3ztahdvHiR4cOH06pVK9q2bcusWbNITk6W5v/zzz94eXnRunVr2rVrV2C36oULFxg6dChmZmbY2Njg6+ub7yr6Y0aPHk2DBg3w8vL64Nshzp8/z/Dhw2nZsiUmJiY4OjqyZ88eab63tzczZ87ku+++o3Xr1rRp04ZVq1Zx69YtXF1dMTMzo1evXly6dEluG+fNm0e7du1o1aoVw4cP58qVK0WqvyAUlroKDDRWk0sbaKyGukrprD82NpZx48ZhZWWFiYkJdnZ2/Pzzz59VpqOjI2fPnpXr/kxNTeXYsWM4OTnJ5X236zM8PJzu3btL/5qYmNCvXz/+/PNPKX9Rh1oA2NjYIJPJ+P333+XS9+/fn68+nyomJgYVFRUiIiIwNzfPN9/DwwMvLy+5NEVFRTIyMop8fBTKJxGoCZ/M0dFRLlBLT0/nyJEj9OjRQy7f5cuXcXNzo2nTpvz2228EBARw6dIlRo0aJQVLU6dO5fLlywQFBbF+/Xr++9//cv/+famM69evM3LkSDp16kRERATLli0jJiYGDw8PsrOzC11nVVVV/P39uXbtGmvXri0wz6NHjxg1ahSmpqbs2rWL3bt3Y2Zmxpw5c3j69KmUb//+/SgpKREeHo67uztr1qzB09OTUaNGsX37dipVqoSvry+Q82TsMWPGcO/ePYKDg/ntt9+wsLBgyJAhXL16tdD1F4TCytuqVpqtaW/fvsXDw4Pq1auzdetW9u3bh4ODA4sXL+batWufXK6VlRVaWlocOXJESjt+/Dh6enoYGBh8cNkHDx6wdetWli5dyq5du6hcuTLe3t5FOn7kVblyZWxsbDhw4ICU9vTpUy5cuICDg8Mnl/uu3ABST0+vwPnNmzfH2NhYms7IyGDDhg2YmJigra1dLHUQypYI1IRP5ujoyF9//cWjR48AOHXqFNra2jRv3lwu388//4yRkRHz5s3DwMCAdu3a8cMPPxATE0N0dDS3b98mOjqa+fPn07p1a5o1a8by5ctRVVWVyggNDaVDhw5S837r1q1Zvnw5ly5d4uzZs0Wqt5mZGaNHj2b16tXExsbmm5+WlsakSZOYPn06DRs2pEmTJowdO5aMjAwSEhKkfNWrV8fLy4sGDRrg7u4OgJOTE3Z2dhgZGdGvXz+p/NOnT/PXX3+xcuVKzM3NMTAw4JtvvsHCwoKwsLAi1V8QCuvdVrXSbE17+/Ytw4cPZ/78+RgYGNCoUSMmT54MwI0bNz65XAUFBezt7eUuEA8cOJDv4rAgGRkZ+Pr6YmFhQdOmTRk5ciR3797lyZMnn1wfyDkOHj9+XOpmPHjwIG3atCmTICkzM5OZM2dy8+ZNFixYUOrrF0qGuJlA+GQmJibo6ekRFRXF8OHD2b9/f4EHzNjYWDp06CCXZmxsjIaGBjdu3ODt27cAmJqaSvNr1qwpdwV59epV7ty5g6WlZb7yb926Rdu2bYtU94kTJ3Ls2DG8vb357bff5OY1aNCAfv36ERYWRmxsLHfv3uX69esAct2l9evXR1Ex51on94nX79ZZTU1NegJ2TEwM2dnZdO3aVW5d6enp+ca3CEJxyW1Vi7iZVqpj07S1tXF1dWXfvn1cvXpV7jckk334jtO8v/HIyEi5aUdHR0aMGEFycjKqqqqcPHmSGTNmkJSU9NF6vdvqljt+7XOfUt+lSxeys7P5/fffsbOzY//+/fTv3z9fvo9t1+dKSUlh6tSpnD17ltWrV2NmZlas5QtlRwRqwmfJ7f4cNGgQR48eZfv27fnyvK9rITs7GxUVFWk8W94DuLLy/309ZTIZvXr1wtPTM185n3LlmtsFOnjwYEJCQuTmxcXF4erqSosWLWjfvj1fffUVWlpaDBgwQC6fikr+5oncwC0vmUxG1apVC7xV/92WQ0EobuoqsOaraqXWmgbw5MkTBg0ahLa2Nra2tnTs2BFTU1O6dOny0WV3794tN12rVi3u3bsnTbdq1YoaNWpw5MgR1NTUMDQ0RE9Pr1CBWkG/tc/p+oScCzJbW1sOHjxIixYtiImJITg4OF++j23X53j8+DFjxozh/v37hIaGYmVlVSzlCuWDCNSEz+Lo6EhISAg7d+587zgRIyMjuUG7kDPmLCUlReoWgZybBWxsbAB49eoVd+/elfI3bdqUuLg4GjZsKKXdunWLpUuX8s033xTp7q5cpqamjB49mh9//JGaNWtSr149IOcuqho1asjd0HDs2DHg0w/qhoaGpKSkkJGRQZMmTaT0uXPnYmxszLBhwz6pXEH4GDVlRWqrU6p3eu7bt48XL14QFRUlXdDkdnl+7Df07m+8ILndn1FRUaiqqhbboP3P4ejoiJeXF4aGhnTo0KHA49HHtutTvXz5khEjRpCSksLmzZs/+vw3oeIRgZrwWZo1a0bDhg1Zvnw548aNKzDPyJEjcXV15dtvv8XV1ZWnT5/y7bff0rx5c6ytrVFRUcHBwQE/Pz9UVVWpWbMmP/zwg9yt5R4eHgwdOhRfX1+GDRvGq1ev8PX1JTU1tcBb1gtrwoQJHDt2TG6sWp06dXj48CEnTpygSZMmxMTE8N133wF88u3unTp1olmzZkybNo05c+agq6vLli1bCA8PJzQ09JPrLwiFUUWldIcj16lTh7dv33Lw4EFatWrF7du3pWd6FccjIxwdHRk+fDgqKiolOhbr5MmT+dKMjIyoXbu2XFqnTp3Izs4mKCgIPz+/Iq/n3Llz3L59Wy6tYcOGhQru/P39uXfvHuvWrUNbW1tuzJ22tnaRniUnlE8iUBM+m6OjIz/99NN7r2zNzc1Zt24dK1eupG/fvlStWpVu3brxn//8R7raXrx4MYsXL2batGnIZDIGDRokdwu+hYUF69atIyAgAGdnZ6pUqYK1tTVeXl6f1XWoqqrKokWLGDhwoJQ2fPhwbt++zcyZM0lPT6dRo0Z88803rFq1iitXrtC5c+cir0dJSYmff/6ZpUuXMnXqVN6+fYuBgQGrV6/G2tr6k+svCOWRg4MDMTExLFq0iJSUFOrVq8eAAQM4evQoV65c+eznLFpaWkrjWPMGTcVpzJgx+dL8/f3p16+fXFqlSpWws7Pj8OHD+cahFkZBb1CYOHEikyZN+uByWVlZ7N+/n4yMDEaMGJFv/tGjR6lfv36R6yOULwrZn9tBLxRKamoq8fHx6Ovro6am9vEFBEEQBEEoNhX1PCwezyEIgiAIglBOiUBNEARBEAShnBKBmiAIgiAIQjklAjVBEARBEIRySgRqgiAIgiAI5ZQI1ARBEARBEMopEagJgiAIgiCUUyJQEwRBEARBKKdEoCYIgiAIglBOiUBNKBFz587FyMhI7mNra1us6zhz5gxGRkYkJiYWKv/Nmzf573//+1nrDA8P/+hLj/Nu97ufpKQkEhMTMTIy4syZM0Vad2BgYLHuw+TkZLZv315s5RUkOzubXbt28ezZsxJdj1Cx5P0uGxkZER4e/t78ub/15s2by71aLld6ejqtW7eWOx64ubkV+Gqm97G1tSUwMBAo2u/80KFDBc4fNWpUobfrY8ew3bt34+TkhKmpKT169ODAgQMf2RrhSyLe9VmBZb9NhYxMst+8RaFKZVBRRqFy+Xgtxo0bN/D09GTYsGFSWnG/HNjS0pLo6Gi0tbULlX/cuHE4OztjY2NTrPXIKzo6Wm765cuXDBs2jC5dulC3bt1CB5Z55b6YvrgsWbKExMREBgwYUGxl5nXu3Dm8vb05evRoia1DeL/s7GwUFBTeO13RKCoqcvjwYQYNGiSXfvLkSVJSUuTSAgMDi3TM2bFjB5UqVSpSfVRUVIiKiuKrr76SS09OTi7yhdj77Nmzhzlz5jB79mw6depEZGQk33zzDXXq1MHS0rJY1iGUbyJQq6CyU96QsesIsr9uQHY2KCigaGGMirMdClWrlG3dsrOJi4tj7Nix6OjolNh6VFVVS7T8T5W3Tt9++y1aWlp8++23n1Wuuro66urqn1XGu0rjNb/iVcJlKzMDsrKyUausQOrbbJSUQEW1rGv16aytrTl48GC+QO3AgQO0bt2ac+fOSWnVq1cvUtmFveDLW5/jx4+TlpYmF+QdOnQICwsLufp8iuzsbAICAhg+fLh0kfb1119z/vx5zp49KwK1fwnR9VkBZb9NzQnSLl7PCdIAsrORXbxGxq6jOS1tJeyff/5h3rx5tGvXjlatWjF8+HCuXLkCwN27d3nz5g2NGzcuUpm2traEhIQwduxYzM3NsbW15ciRIxw5cgR7e3ssLCwYNWqU1I2Wt9vg8uXLuLq6YmlpiZWVFZMmTSIpKUkq+/79+6xevRo3Nzcgp+ti1apVdO3alY4dO5KQkEBSUhLTpk3D2tqaFi1a0LlzZ5YuXYpMJvuk/RQdHc2hQ4f49ttvUVWVP0NevHiRXr16YWJiQr9+/Th9+vQHy3q3uyi3+zQqKooBAwZgYmKCra0t27Ztk/I/e/aMyZMn07ZtW8zMzBg8eDBnz54FwNvbm127dnH27Fmpi8fNzY158+YxYMAAWrduTUREBN7e3tL+ypU37enTp8ycOZO2bdvSqlUrxo0bx507dzhz5gzDhw8HwM7O7oNdQELJyMzI5vjOVB7dy+L4zlQyM0ovcI6NjWXcuHFYWVlhYmKCnZ0dP//882eV6ejoyNmzZ+W6P1NTUzl27BhOTk5yed/t+gwPD6d79+7Sv7m/uT///FPK/27XZ2HZ2Nggk8n4/fff5dL379+frz6fIj4+nvv379OrVy+59NDQUMaNG/fZ5QsVgwjUKqKMzJyWtALI/roOGZkluvrs7GzGjBnDvXv3CA4O5rfffsPCwoIhQ4Zw9epVYmNjAfjll1+wtbWlW7du+Pn58c8//3y07B9//BEnJyf27t2LsbExM2fOJCgoiKVLlxIUFMSVK1dYu3ZtvuWysrKkk0JERAQbNmwgKSmJ2bNnAzndGnXq1MHDw0PuYLxlyxZWrVrF6tWradSoEV9//TX//PMP69ev5+DBg3h4eLBu3TqOHTv2Sfvqhx9+wM7OjtatW+ebFxoaytdff82ePXto3rw548aN49GjR0Uq39/fH09PTw4cOICNjQ0+Pj7cu3cPAB8fH9LS0ti0aRN79+5FX1+f8ePH8+bNG+bMmYOjo6PUfZxr+/btDB8+nC1bttCpU6ePrj8zMxMPDw/i4uL48ccf+e2335DJZIwePRpLS0tpX2/fvr1YTlxC4eUGZS+fyTjyWyovn8nk0kvS27dv8fDwoHr16mzdupV9+/bh4ODA4sWLuXbt2ieXa2VlhZaWFkeOHJHSjh8/jp6eHgYGBh9c9sGDB2zdupWlS5eya9cuKleujLe392e1+lauXBkbGxu5MWNPnz7lwoULODg4fHK5ueLj4wF48+YNo0aNwtramgEDBnzy8UiomESgVgFlv3n7fy1p+WZmk/2mZFvUTp8+zV9//cXKlSsxNzfHwMCAb775BgsLC8LCwoiNjUVRUZFatWoRFBSEt7c30dHRjB8//qMtUzY2NvTt25cGDRowcOBAXr9+zbRp0zAzM6Ndu3a0b9+emzdv5lsuJSWF5ORkatWqRb169WjRogUrV65k6tSpQE63hpKSElWqVJHrEunTpw+mpqZYWFiQmppKnz59+PbbbzE2NkZPTw93d3dq1qzJjRsFB8Yfcu7cOWJiYhg/fnyB8ydNmoSTkxMGBgb4+PhQo0YNtmzZUqR1uLu7Y2dnh56eHtOmTUMmk3Hp0iUgp2WzWrVq6Onp0bBhQ+bMmcOqVatQUlJCQ0MDNTU1VFRU5LpqmzVrRq9evTA0NERLS+uj6//jjz+4ceMGy5cvp1WrVhgYGPDdd9/RrVs3UlJS0NTUBHL2v5pa+Rg/+W+QmZ7Nq2QZkRvfkvX/r9uyMiFy41teJcvITC/ZYO3t27cMHz6c+fPnY2BgQKNGjZg8eTLAJ/2WcikoKGBvb8/BgweltAMHDtCjR4+PLpuRkYGvry8WFhY0bdqUkSNHcvfuXZ48efLJ9YGcVr7jx4+Tnp4OwMGDB2nTps0ndaXmlTvuzsvLi549e/Lzzz/ToUMHxo8fzx9//PHZ5QsVgxijVgEpVKkMCgoFB2sKCihUKdkTYkxMDNnZ2XTt2lUuPT09nbS0NBYuXIirq6t0ojc0NERHR4eBAwdy5coV/vjjD4KDg6XlevXqhZ+fHwANGzaU0itXrgxAgwYNpDQ1NbUC7yDU1NRk9OjRfPvtt6xatYp27drRpUsXHB0dP7gt765PTU2NYcOGcfDgQS5fvsydO3e4ceMGT58+LTDAjIiIYMGCBdJ0q1atWLdunTS9a9cuzMzMaNGiRYHrbtWqlfS3srIyzZs35+bNmyQlJeU78Vy8eLHAMt5tRdDQ0AByTkgAEydOZMaMGURFRdGqVSs6duxIz549Pzhg+t39URixsbFoamqir68vpdWuXRsvL68ilSMUL2VVBappKdJjRGX2rMsJ1pSUoceIyqhUUkBZuWRvKNDW1sbV1ZV9+/Zx9epV7t69y/Xr1wE+erGWd9xVZGSk3LSjoyMjRowgOTkZVVVVTp48yYwZM6RhDh/yod/Lp+rSpQvZ2dn8/vvv2NnZsX//fvr3758v38e2qyAqKipAzh2kzs7OQM7F1NWrV1m/fj3W1tafVXehYhCBWkWkooyihTGyi/m7EBQtjEGlZP9bZTIZVatWLXDMkaqqKoqKivlaY5o2bQrAw4cPGTx4sFwAVbVqVelvZeX8dS/sXWrTp0/H1dWVEydO8Mcff/Dtt9+ybt06du/enW98WK53W3nevHnDsGHDSE1NxcHBAWdnZ8zMzN57p6WtrS3m5uYFliWTyTh27Nh7W9Mg/12wWVlZVKpUiVq1arF79+7CbHKB25XbldO9e3d+//13fv/9d/73v/+xfv16Vq9ezW+//Sb9f+RVmFavzMz/61ov6P9LKB+UVRTISM9Gs4YiLW1UufDfnBafkg7SAJ48ecKgQYPQ1tbG1taWjh07YmpqSpcuXT66bN7vfq1ataTufMi5wKlRowZHjhxBTU0NQ0ND9PT0ChWofej38qnU1NSwtbXl4MGDtGjRgpiYGLkL0Vwf266C1K5dG8i52H1XkyZNPvtRQ0LFIY6yFZBCZTVUnO3I4P+PSct712cJP6LD0NCQlJQUMjIyaNKkiZQ+d+5cjI2NuXz5Mo8fP2bDhg3SvNwbDZo0aUL16tWLfEfWx9y+fZuNGzcye/ZshgwZwpAhQ/jzzz9xdXXl+vXrmJmZfbSM6OhoYmJiOHXqFDVr1gTgxYsXPHv2rMCDedWqVeWCzHfFxcWRnJxM+/bt37u+v//+G2NjYyCnNfLvv/9m8ODBKCsrF7llK6/09HSWL19Onz59cHJywsnJidTUVDp06MB///tfmjZtWqgAWEVFJd9jD+7cuSMFdE2aNOHly5fcuXNHqvPz589xdHQkODi4Qj8K4kugrKJAVxc11Crn/FvMT8h5r3379vHixQuioqKkVqHcLs+PBUYf++7ndn9GRUWhqqpaLsY+Ojo64uXlhaGhIR06dJBa6971Kb/pFi1aoK6uzqVLl+TGucbGxsr1NAhfNhGoVVAKVaug0r879OlK9pvUnO7OUnqOWqdOnWjWrBnTpk1jzpw56OrqsmXLFsLDwwkNDUVXV5fx48ezevVqevfuTXx8PH5+fvTs2fOjA34/lZaWFpGRkaSmpjJ27FgUFRXZtWsXmpqa0t2n6urqJCQk8PTpUykQe1edOnWAnC5Ne3t7Hjx4wA8//EBGRoY0/qSwrl69ioqKygfvfF2+fDnVq1enUaNG/Pjjj6Snpxfbc9JUVVW5cuUK58+fZ968edSsWZOTJ0/y5s0bqQumSpUqPH78mHv37qGnp1dgORYWFuzYsYOIiAgsLS2JiIggNjZWCnytra0xMTHBy8uL2bNnU7lyZZYsWYK2tjYtWrSQTs7Xr19HS0urWB8vInycsgqoqOYEy2qVFUrtcSl16tTh7du3HDx4kFatWnH79m38/f0BivxbKoijoyPDhw9HRUVFbvhBcTt58mS+NCMjI6mlK1enTp3Izs4mKChIGsZRFOfOneP27dtyaQ0bNqRhw4aMHj2aNWvWULt2bczMzIiMjOTUqVNyF8LCl00EahWYQmU1qAwK1Qpu1SkpSkpK/PzzzyxdupSpU6fy9u1bDAwMWL16tTRmYuXKlYSEhLB27Vo0NDTo1auXNLC/JGhpabF27VqWL1/OwIEDycrKwsLCgvXr10utXm5ubixevJibN28SERGRrwwzMzNmzZrFhg0bWLlyJbVr18bJyQldXV2pRbCwnjx5gqamJoqK779fZ9KkSSxbtozExETMzMxYv359sbY0rlixAn9/f+lO1saNG7Ns2TLpyrxv374cPnyYnj17vvfp6r179+batWt89913ZGZmSuODcsfMKSoq8uOPP+Lv78/IkSNRUFCgXbt2rFu3DhUVFQwNDenSpQtTp07lm2++wcPDo9i2T/i4vC2apdXC6eDgQExMDIsWLSIlJYV69eoxYMAAjh49ypUrV9DV1f2s8i0tLalZsyZ6enr5gqbiNGbMmHxp/v7+9OvXTy6tUqVK2NnZcfjw4XxjdwujoDcoTJw4kUmTJjF+/HgqV67MihUrePToEQYGBgQGBtK2bdsir0eomBSyxRMpS0Vqairx8fHo6+uLu98EQRAEoZRV1POweDyHIAiCIAhCOSUCNUEQBEEQhHJKBGqCIAiCIAjllAjUBEEQBEEQyikRqAmCIAiCIJRTIlATBEEQBEEop0SgJgiCIAiCUE6JQE0QBEEQBKGcEoGaIAiCIAhCOSUCNUEQBEEQhHJKBGrCJ8vOziY8PBw3NzfatWuHiYkJ3bt35/vvv+fJkycfXNbIyIjw8HAAMjIy5F4wHB4ejpGR0UeXNzIyeu87KkeNGiW3joJ4e3tL5eT9rF69usA8zZo1o2PHjsyfP5+UlJQP1lEQhOJla2tLYGCgNH38+HHi4uIAOHPmDEZGRiQmJubLW5hjSln42DEqd5uaN2/O8+fP881PT0+ndevWctvt5uZW4LtD36eo+6k4jr15/68+Jjg4GDc3t3zpx44dw8XFBUtLS2xtbVm8eDGpqamFKrMiEYFaBSZ7++qD0yW6bpmMCRMmsGjRIrp27covv/zCoUOHmDt3LleuXMHFxYVnz569d/no6GicnJwA2LdvH/7+/kWug4qKClFRUfnSk5OTOXPmzEeXnzNnDtHR0XKfHj16oKOjw4ABA6R8lpaW0vyjR4+yfPlyzp07x+zZs4tcZ0EoTbK0bGRv3/mkfTmvdr5//z6enp7ScSb3d1rQC9+dnJyIjo4u7SoWG0VFRQ4fPpwv/eTJk/kuGAMDA5kzZ06hy96xYwceHh5Fqs/nHnuLYvPmzaxcuTJf+vnz55k4cSLdu3dn165dLFiwgP379+Pr61us6y8PRKBWQWX984QXEQvI+udJgdMlbcOGDZw4cYL169fj4eFB06ZNqVu3Ll26dGHDhg2oqKgQGhr63uV1dHSkl+JmZ3/aycPa2prjx4+TlpYml37o0CEsLCw+uryGhgY6OjrS58qVK+zfv5/ly5dTu3ZtKZ+KioqUp27durRt25YJEyZw6NAh0aomlFvZGdmk/ZXFy7Vp0iftryyyM76MYC3vcUNVVRUdHR2UlJTy5VVTU0NHR6e0qlbsrK2tOXjwYL70AwcO0Lp1a7m06tWro6GhUeiytbW1UVdXL3J9PufYWxiPHj3C09OTZcuW0ahRo3zzt27dStu2bfH09KRRo0Z06dKFadOmsXfvXtLT04ulDuWFCNQqINnbV7zcv5C0myd5tsmT9KSrPNvkSdrNk7zcv7DEW9ays7PZtGkTvXv3pkWLFvnmq6mpERYWxtSpU0lMTMTIyIjg4GA6dOiAnZ0dKSkpUtN4eHg4s2bNAnKa1ItyNWZjY4NMJuP333+XS9+/f7/UWldYaWlpfP/997i4uNC2bduP5ldTU0NBQaFI6xCE0iR7nc0/m9NIj8mSPv9sTkP2unQCNSMjI7Zt24arqyumpqY4Ojpy4cIFtm3bho2NDS1btmTq1KlSV1VB3W7v64pLTEzEzs4OgOHDhxMYGPjB7rS85RgZGbFjxw7c3d0xMzOjY8eO0nCHXHv37sXR0RFTU1MGDBhAWFhYvjLydu+9myaTyQgODsbe3h4TExNatmzJ6NGjuXv3blF2IwCOjo6cPXtWrvszNTWVY8eO5TvWvdv1GR4eTvfu3aV/TUxM6NevH3/++aeUP293cmEU57H3fWJiYlBRUSEiIgJzc/N88z08PPDy8pJLU1RUJCMj44u7gBaBWgWkWLkamk6zUarRiKxnCTz72Y2sZwko1WiEptNsFCtXK9H1JyYmcv/+fdq3b//ePPXq1UNVVVWa3rVrFxs3bmTlypVUrVpVSndycpK6EKOjo7G0tCx0PSpXroyNjQ0HDhyQ0p4+fcqFCxdwcHAoyiaxfft2nj59ytSpUz+a9+HDh/z88884ODjIbYsglBeyN9mk7MkgW77Bg+w0SInIQPa2dIK1FStWMHr0aPbs2YOGhgaenp5ERUUREhKCv78/R44cYfv27UUuV1dXV1ouMDCwyF13AIsXL8bZ2ZnIyEiGDRtGYGAg586dA3LGvnl5edG/f38iIiLo168fy5YtK1L5YWFhhIaG4u3tTVRUFGvWrCEhIYFFixYVua5WVlZoaWlx5MgRKe348ePo6elhYGDwwWUfPHjA1q1bWbp0Kbt27aJy5cp4e3t/ck8GFO+x931yA0g9Pb0C5zdv3hxjY2NpOness4mJCdra2sVSh/JCBGoVlJKGDtX7fCuXVr3PtyhplHzz/tOnTwHy/Rg8PT2xtLSUPj169JDmubq60qRJE0xNTeWWUVNTk5rpdXR05IK7wnB0dOT48eNSU/fBgwdp06ZNkX6oMpmMjRs3MmDAgAK7R86fPy9tk5mZGV26dOHWrVtMnDixSHUVhFKTDal/ZBY4K/V/mSArnWq4uLhga2tL48aN6dOnDy9fvmT+/PkYGhpib29Ps2bNuHnzZpHLVVJSkn7jmpqaRe66A+jbty99+vRBT08PT09PqlWrxoULFwAIDQ3FwcGBUaNGoa+vz5AhQxgyZEiRym/QoAGLFy+ma9eu1KtXD2traxwcHIiNjS1yXRUUFLC3t5fr/jxw4IDcMfZ9MjIy8PX1xcLCgqZNmzJy5Eju3r370Ru+PqY4jr3FJTMzk5kzZ3Lz5k0WLFhQ6usvacplXQHh02T984QXe+bJpb3YM48aw4JKPFjT0tIC4OXLl3Lpvr6+UjfGL7/8wrFjx6R5DRs2LJG6dOnShezsbH7//Xfs7OzYv38//fv3z5cvb0tdZGQkdevWBeDChQvcvXv3vQdiExMT6Wo6KyuLZ8+eERYWxqBBg9i+fTv6+vrFvFWC8JkUQM1aOScoy0OtvXKpXaK/+7uvXLkykBPASHVRUyuz8UR5W6I0NDTIyMgAcrrdvvrqK7n5VlZWcnenf4ytrS2XLl0iICCA+Ph44uPjiYuLkxv/+q6CjlHvcnR0ZMSIESQnJ6OqqsrJkyeZMWMGSUlJH63Lu9uae2Gcu62f6nOOvcUpJSWFqVOncvbsWVavXo2ZmVmxll8eiECtAsodo5bb3Vm9z7e82DOPrGcJvNy/kOq9fUu0+1NPTw8dHR3OnDkjNx7h3QOQpqam3DK5Nw4UNzU1NWxtbTl48CAtWrQgJiaG4ODgfPl2794tN12rVi3p78OHD9O8efP3diGoqanJnXAaN26Mubk5bdu25bfffss3TkIQyppiFQWq9lEh7c9Mue5PhUpQtbcKipVLZ3ylsnL+U4yiYuGjxKysrOKsjpyCWu9zuwOVlZWRyYrW7JiZKR8Uh4SEsGbNGpydnbG2tsbd3Z2jR4++N1Ap6Bh17949abpVq1bUqFGDI0eOoKamhqGhIXp6eoUK1D60rZ/qc469727X53j8+DFjxozh/v37hIaGYmVlVSzlljciUKuAcseovdxPzlg1DR1qDAvi5f6FpTJGTUlJieHDh7NmzRqGDBkiN04g14MHDwpd3ucOynd0dMTLywtDQ0M6dOhQ4B1PH2rRO3fuHNbW1kVer0wm++yDnSCUFEV1BTSGViL1zP8FEGptlVGsWj5vglFRUQFyWkhyx34mJCS8N39J3sxjbGzMpUuX5NIuXrwoN62ioiI3aP3OnTty84OCgpgwYQJjx46V0kJDQ997zPhYr0Nu92dUVBSqqqrFNmj/c3zusfdzvHz5khEjRpCSksLmzZvL5XPyiosI1CooJQ0duZazvNMlbfTo0Vy9ehVXV1fGjh2LjY0NVatWJTY2lk2bNnHq1ClcXFwKVVaVKlUA+Pvvv2nSpImUfvLkyXx5jYyM8nUddOrUiezsbIKCgvDz8yvSdmRlZREbG4u7u/t782RkZMiN50hOTiYkJIT09HR69uxZpPUJQmlRUFGgkoUSlczeeVyFIigol89AzcLCAgUFBQIDA3Fzc+PKlSvs2rXrvflzjxuxsbE0b968WOsyZswYxo0bh5mZGV27duXPP/9k06ZN+eq7fft2rKysyM7Oxt/fX67lSldXl1OnTmFra4uioiJ79uzh0KFD1KxZ85Pr5ejoyPDhw1FRUSnRsVilceyFnIvk27dvy6U1bNiwUMGdv78/9+7dY926dWhra8sdo7W1tQt8TEtFJQK1CixvUFZaQRrkdF+sXLmSAwcOsHPnTsLCwnj16hU1a9akdevWbNq0CSsrq0I9ebpdu3aYm5szePBgli5dKqWPGTMmX15/f3/69esnl1apUiXs7Ow4fPgwXbt2LdJ2vHjxgoyMDKpXr/7ePBcvXqRjx45AzlWturo6xsbGBAUFYWJiUqT1CUJpUqxUPoOygujp6eHr60twcDBbtmyhVatWzJw5871DC7S0tHBxcWHJkiXcuXOH7t27F1tdOnfujJ+fH8HBwSxfvhwTExOGDBkiF6z5+Pjg4+PDwIEDqVWrFlOmTOHhw4fS/CVLluDn54eLiwvq6uqYm5vj6+uLj48PSUlJ0hjZorC0tKRmzZro6em9d6xbcSiNYy9Q4BsUJk6cyKRJkz64XFZWFvv37ycjI4MRI0bkm3/06FHq169f5PqUVwrZou+mVKSmphIfH4++vn6JjdcSBEEQPt/Zs2epWbMmjRs3ltKCgoLYsWOH3CMyhIqlop6HxeM5BEEQBOEd0dHRjBo1itOnT5OUlMTRo0fZuHEjffr0KeuqCf9CoutTEARBEN4xceJE3rx5w8yZM3n+/Dm6urq4u7szevTosq6a8C8kuj5LSUVtchUEQRCEL0FFPQ+Lrk9BEARBEIRySgRqgiAIgiAI5ZQI1ARBEARBEMopEagJgiAIgiCUUyJQEwRBEARBKKdEoCYIgiAIglBOiUBNEARBEAShnBKBmlAu2NraEhgYKE0fP36cuLg4AM6cOYORkZH03tB384aHh2NkZFT6Ff4IIyMjwsPDP6uMrKwszMzMMDIykvu8u5/yyruvypvi2C/vevd7UlKSkpKIjIws0XWUtOy0rFJfZ97fdFHl/d6bmZnRq1evAr8/v//+O25ubrRs2RJzc3N69epFSEgIGRkZUp7AwMB8Zb77OXjw4CfXVRBKkngzQQWVnvoSWVZavnRFpUqoqmmWQY2Kz/379/H09CQsLIwmTZpgaWlJdHQ02tra+fI6OTnRqVOnMqhlyUtISCAtLY09e/ZQo0YNKb1KlSplWKvPEx0djYaGRrGUlfd7UlK8vLyoV68ePXr0KLF1lKTs1Exkt16haFANBbWKdcifPXs2Tk5OALx584bo6Gjmzp2LtrY2NjY2AJw6dYqvv/6aadOm4ePjg7KyMhcuXMDf35/4+Hj8/f2l8urUqcOOHTsKXJemZsU+bgpfror1qxUksqw0ojc55kvvOOxAGdSmeOV9WYaqqio6OjoF5lVTU6tQT5guihs3blC1alWMjY3LuirF5n3/j59CvFTl47LTs8h+nUl60FUq+bQGRQUUVJXKulqFpqGhIfedadiwIceOHSM8PFwK1LZt20anTp0YNWqUXL7U1FT8/PyYNWsW1apVA0BJSalYv4OCUBpE16dQZEZGRmzbtg1XV1dMTU1xdHTkwoULbNu2DRsbG1q2bMnUqVNJTU0FCu6efF+XZWJiInZ2dgAMHz6cwMDAD3bn5S3HyMiIHTt24O7ujpmZGR07dmT16tVyy+zduxdHR0dMTU0ZMGAAYWFh+crI273ybppMJiM4OBh7e3tMTExo2bIlo0eP5u7du0XZjVy4cIGhQ4diZmaGjY0Nvr6+pKSkSPNv3LiBgYFBkcrM6+XLl8ydO5dOnTrRokULrK2tmTt3Lm/fvgVyukqbN29OSEgIbdu2pV+/fshkMu7evcuYMWOwtLSkU6dOrF+/nu7du8vtl507d+Lo6IiZmRmOjo5s3LgRmUz2wfq8ux+9vb3x9vZm8eLFWFtbY25uzrhx43j06JGUf/fu3fTo0QNTU1M6derE999/T3p6+nu/J3m35d69exgZGXHmzBmpzMTExHxpERER9O7dGzMzM+zs7Ni4cSMAbm5unD17ll27dmFra/tZ/xelKTs1E1liCukbbpC29BJkZpO29BLpG24gS3xNdmpmmdYvPT2dxYsXY2tri4mJCW3atGHKlCk8f/78o8vmvTBTUFDg+vXrct8bgL59+7Jv374K3QItCCACNeETrVixgtGjR7Nnzx40NDTw9PQkKiqKkJAQ/P39OXLkCNu3by9yubq6utJygYGBeHh4FLmMxYsX4+zsTGRkJMOGDSMwMJBz584BOWOavLy86N+/PxEREfTr149ly5YVqfywsDBCQ0Px9vYmKiqKNWvWkJCQwKJFiwpdxvXr1xk5ciSdOnUiIiKCZcuWERMTg4eHh9RSFBsbS2ZmJqNGjaJDhw7069ePPXv2FKmu3t7eXL16ldWrVxMVFcWsWbPYvXs327Ztk/JkZWVx4sQJtm3bxvfff09aWhru7u7IZDJ+/fVXVqxYQXh4OPfu3ZOW2bZtG0uWLGHixIlERkYydepU1q5dW+R9uW/fPl68eMGmTZtYu3YtMTExrFy5UtpHc+fOZdKkSURFRbFw4UL27NnDunXr3vs9ybstCgoKH63D/v378fLyok+fPkRERPDNN9+wbNkywsPDCQwMxNLSEkdHx/d2mZVHCmrKKGhVQqFmZXiZnpP4Mh2FmpVR0FIt8y7QJUuWcOjQIRYtWkRUVBSLFi3i9OnT/PTTT+9dRiaTER0dzalTp3BxcZHSR4wYwbNnz7C1tWXEiBGsXr2as2fPoqKigoGBAcrKouNIqNjEN1j4JC4uLlILQ58+ffDz82P+/Pk0atQIQ0ND1q1bx82bN4tcrpKSkjQWTVNTE3V19SKX0bdvX/r06QOAp6cnoaGhXLhwASsrK0JDQ3FwcJC6SfT19UlISGDDhg2FLr9BgwYsXryYrl27AlCvXj0cHByKNBg5NDSUDh064OnpCUCjRo1Yvnw53bp14+zZs7Rt25abN28ik8mYPHkyderU4cSJE8yaNYuMjAz69+9fqPV06NABKysrqcWwfv36bNq0idjYWLl8Hh4eNGrUCMhpKXv+/Dnh4eFUr14dgKVLl0r7FODHH3/k66+/lsZt6enpkZKSgq+vL1OmTKFSpUqFqp+GhgZ+fn7SSdXJyYkTJ04AOS1fCgoK1KtXj7p161K3bl1CQ0OpWrXqB78n725LYW6q2LhxI05OTtJ3olGjRrx+/Ro1NTWqV6+OiooKampqBY6RLM8U1FVQtq1L1uH/2wcqtnVRUFcpw1rlMDU1xcHBgdatWwM5v6H27dvn+14uWLCAb7/9FoC0tDSysrLo1q0b1tbWUp6WLVsSHh7O+vXrOXHiBKdPnwagVq1aLFiwgG7dukl5k5KSsLS0zFcfLS0tjh07VuzbKQjFQQRqwidp2LCh9HflypWBnAAml5qaGunp6aVeLyBfd6GGhoZ091dMTAxfffWV3HwrK6siBWq2trZcunSJgIAA4uPjiY+PJy4ujtq1axeYP++JITIykqtXr3Lnzp0CTxq3bt2ibdu27Nu3j6ysLCkIMTY2JikpidDQUPr370+PHj1ISkqSllu7dm2+slxdXTl27Bi7du0iISGBuLg4EhMTady4sVy+3MAG4OrVq+jr60tBWu66c28CeP78OQ8fPuSHH34gICBAyiOTyUhLSyMxMZGNGzeyd+9ead64ceOkoPRdDRo0QEXl/wKHd/+vOnXqhKWlJf3796d+/fp06NABOzs7TExM8pXzvm0pjNjY2Hw3CgwcOLBIZZRX2c/TULLSQdmuHplH7iNLTkOpeuGC6JLUp08f/ve//7Fs2TISEhK4ffs28fHxUuCWa/LkydLvNT09nZs3b7J06VImTJgg931v0qQJ33//PZDz+/n999/ZtGkTU6ZMkRseUatWLX755Zd89VFUFJ1LQvklArUKSlGpUoE3Digqlc5BuKDuhKIc7LKySu5xAaqqqvnScrsTlZWVPzqOKq/MTPnxPCEhIaxZswZnZ2esra1xd3fn6NGj732Ew+7du+Wma9WqhUwmo1evXgUGL7ktNwXdJGFoaEhERIRUj3frVrt2bS5duiRNy2Qyxo0bx82bN+nZsydOTk60aNGCefPm5Sv33RYwJSWlD+6j3HmzZs2iffv2+ebr6uoyZcoUucHd77ujrqD/q3frFBYWxtWrV4mOjiY6OhpPT0/69u0rdyffh7alIHm/e19y15hig6ooDmmCQmVlVFybgEr5CEjmz59PVFQUffv2xdbWlgkTJhAaGppvnFmNGjXkLgqbNm1KZmYmM2bM4ObNm9SrV48ffvgBFxcXmjVrBuRcqBkYGNC7d2+6du1KdHS0FKgpKyvLlScIFcGXe4T6wlWkR3DktpikpKRQtWpVIOfRE+9TmHFFn8rY2FgumAG4ePGi3LSKiorcoP47d+7IzQ8KCmLChAmMHTtWSgsNDX3vXYgFnRiaNm1KXFyc3Lxbt26xdOlSvvnmG7Kzs+nWrRve3t7069dPynPlyhWaNm0K5HQXfci1a9c4efIkv/32G+bm5gBkZGRw9+5d9PT03rucsbExv/32Gy9evJBa1W7dusU///wD5Jw8tbW1uXfvnlz99+/fz+HDh1m8eDE1atSQe6TIpzhx4gRXrlxh4sSJNG/enLFjx/LTTz8RFBSEv79/ob4n7373cuX97hkYGHDlyhW5NH9/fx48eMCqVas+axvKmoKyIijnBGcKlcvH4T45OZlt27axYsUK6dEbALdv3y7UwP/c35lMJkNNTY29e/eSkZGBr6+vXD51dXWUlJQ++3soCGWtfPxyhS+ahYUFCgoKBAYG4ubmxpUrV9i1a9d78+cerGNjY2nevHmx1mXMmDGMGzcOMzMzunbtyp9//smmTZvy1Xf79u1YWVmRnZ2Nv7+/XMuPrq4up06dwtbWFkVFRfbs2cOhQ4eoWbNmoevh4eHB0KFD8fX1ZdiwYbx69QpfX19SU1Np1KgRqqqqtGvXjhUrVkitCocOHSIiIoLg4OBCraNmzZooKytz4MABtLW1efHiBUFBQTx58uSD3dI9e/YkMDCQ6dOnM336dOkxB5ATRCsoKDBmzBhWrFhB3bp16dy5Mzdu3MDHxwc7O7sPtpIVhYqKCmvWrKFq1arY2dnx8uVL/vvf/0rdxYX5ntSqVYt69eqxceNGGjVqxIsXLwgICJAL8saOHcukSZMwMzOjS5cuXLp0iV9//VXaZnV1de7fv8/Dhw+pU6dOsWzbv8WdO3c4efKkXJqamhoaGhocPXqUFi1akJqayqZNm4iJiZEuKHL9888/PHnyBMgJzG7evElAQADNmzfH0NAQBQUFpk+fzty5c4GcsbPa2trcvXuXn3/+GV1dXRwcHKTysrKypPLyqly5snQhKQjliQjUhBKnp6eHr68vwcHBbNmyhVatWjFz5ky8vLwKzK+lpYWLiwtLlizhzp07dO/evdjq0rlzZ/z8/AgODmb58uWYmJgwZMgQuWDNx8cHHx8fBg4cSK1atZgyZQoPHz6U5i9ZsgQ/Pz9cXFxQV1fH3NwcX19ffHx8SEpKom7duh+th4WFBevWrSMgIABnZ2eqVKmCtbU1Xl5eUqCzcOFCAgMDWbBgAc+ePcPAwIBVq1YV+gG/tWvXZtGiRQQGBrJ582Z0dHSwsbHB3d39gwOnVVVVWbduHX5+fgwcOBBNTU08PT2JiYmRWqg8PDyoVKkSv/zyC4sWLaJmzZoMHDiQyZMnF6puhdG+fXu+//57fv75Z1asWIGamhpdunTB29sbKNz3REFBgSVLlrBw4UL69OlDw4YNmTVrllxrqK2tLX5+fqxdu5bFixdTr149Zs2aRd++fQEYPHgwXl5e9O7dmz/++AMlpYrzHLKytnfvXrmxipDTEhwQEMCiRYvo1asXmpqatG3blm+++Ybg4GDevn0rjXtduHAhCxcuBJBax9q3b88333wjBdsDBgxAR0eHjRs3MmbMGF6/fk3NmjWxs7NjyZIlckMIHj58SMeOHQus69ChQ5k/f35J7AZB+CwK2eKpkaUiNTWV+Ph49PX1v9gHtFYEZ8+epWbNmnKD6YOCgtixYwdHjhwpw5qVH4mJiSQkJMid0B49ekTnzp3ZvHlzvgHfgiAIFUFFPQ+Xj5GlglBKoqOjGTVqFKdPnyYpKYmjR4+yceNGuUdP/NulpaUxduxYQkNDuXfvHlevXmXevHk0atQoX9eUIAiCULJE16fwrzJx4kTevHnDzJkzef78Obq6uri7uzN69Oiyrlq5YWBgwA8//EBQUBCrVq1CTU0Na2tr1q9fL/coDUEQBKHkia7PUlJRm1wFQRAE4UtQUc/DoutTEARBEAShnBKBmiAIgiAIQjklAjVBEARBEIRySgRqgiAIgiAI5ZQI1ARBEARBEMopEagJgiAIgiCUUyJQE8oFW1tbAgMDpenjx48TFxcHwJkzZzAyMiIxMTFf3vDwcIyMjEq/wh9hZGREeHj4Z5WRlZWFmZkZRkZGcp9391NeefeVIJQVNze3fN/ddz/Pnz//pHJL6jtemHLd3NykV5iV1m/t9u3bTJs2DWtra0xMTLC1tcXX15enT59KeQIDA7G1tS3ReghlRzzwtoJLTXtJZlYaykqVUKukWdbVKRb379/H09OTsLAwmjRpgqWlJdHR0Whra+fL6+TkVOh3X1Y0CQkJpKWlsWfPHmrUqCGl576MXBDKO0dHR+bMmVPgPC0trVKuTfH60HGpuDx9+hRXV1e6du3KunXr0NTUJD4+niVLluDm5saePXtQVVXFw8ODoUOHllg9hLIlArUKLjMrjU27HBjmfLCsq1Js8j6DWVVVFR0dnQLzqqmpVagHFxbFjRs3qFq1KsbGxmVdFUH4JGpqau/97VZ0HzouFZeDBw+SmZnJwoULpZfQ169fn7p16+Lk5MTvv/+OnZ0d6urqqKurl2hdhLIjuj6FIjMyMmLbtm24urpiamqKo6MjFy5cYNu2bdjY2NCyZUumTp1KamoqUHD35Pu6LBMTE7GzswNg+PDhBAYGfrCLIW85RkZG7NixA3d3d8zMzOjYsSOrV6+WW2bv3r04OjpiamrKgAEDCAsLy1dG3m7Ld9NkMhnBwcHY29tjYmJCy5YtGT16NHfv3i3KbuTChQsMHToUMzMzbGxs8PX1JSUlRZp/48YNDAwMilRmXi9fvmTu3Ll06tSJFi1aYG1tzdy5c3n79i2Q033TvHlzQkJCaNu2Lf369UMmk3H37l3GjBmDpaUlnTp1Yv369XTv3l1uv+zcuRNHR0fMzMxwdHRk48aNyGSyz6qv8O9ja2tLSEgIY8eOxdzcHFtbW44cOcKRI0ewt7fHwsKCUaNG8ezZM7nljh07Rrdu3TA1NcXNzY3r169L87Kzs1m7di12dnaYm5vTp08fIiIi5JY/f/48AwYMwMzMjN69e8stD5Cens7ChQuxtramVatWLF26VO77XdCQjNDQUCZNmoSlpSVt27blu+++IzMzU1omOjoaZ2dnTE1N6dmzJzt37vxg96mCggKvX7/m3LlzcukGBgZERkbSrl07QL7r09vbu8Cu5ne7Ro8fP06/fv0wMzOje/furFy5kvT09A//RwllRgRqFVBq2ktS3jwm5c1jXr95BMDrN4+ktNS0lyVehxUrVjB69Gj27NmDhoYGnp6eREVFERISgr+/P0eOHGH79u1FLldXV1daLjAwEA8PjyKXsXjxYpydnYmMjGTYsGEEBgZKB7rjx4/j5eVF//79iYiIoF+/fixbtqxI5YeFhREaGoq3tzdRUVGsWbOGhIQEFi1aVOgyrl+/zsiRI+nUqRMREREsW7aMmJgYPDw8pBbF2NhYMjMzGTVqFB06dKBfv37s2bOnSHX19vbm6tWrrF69mqioKGbNmsXu3bvZtm2blCcrK4sTJ06wbds2vv/+e9LS0nB3d0cmk/Hrr7+yYsUKwsPDuXfvnrTMtm3bWLJkCRMnTiQyMpKpU6eydu3aIu9LQQD48ccfcXJyYu/evRgbGzNz5kyCgoJYunQpQUFBXLlyhbVr18ot8/PPP7NgwQJ27tyJuro6o0ePli5AVqxYwa+//sq8efPYu3cvw4cPx8fHh82bNwNw7949PDw8aNasGbt27WLChAn8+OOPcuV/99137N+/n0WLFrF161YePnzI+fPnP7gdAQEBWFlZERERwcyZM9m0aRP79u0D4Nq1a4wbNw5ra2v27NnD119/zeLFiz9YXo8ePdDV1cXNzY2+ffuyaNEijhw5QkpKCk2aNCmwFW3OnDlER0dLn4CAAJSUlJg0aRIAJ0+eZOrUqQwcOJB9+/axYMECDhw4wIwZMz5YF6HsiK7PCii3u/Ndu6JGSH+XRjeoi4uLdIXWp08f/Pz8mD9/Po0aNcLQ0JB169Zx8+bNIperpKQkjfnQ1NT8pOb8vn370qdPHwA8PT0JDQ3lwoULWFlZERoaioODA6NGjQJAX1+fhIQENmzYUOjyGzRowOLFi+natSsA9erVw8HBgYMHC7/fQ0ND6dChA56engA0atSI5cuX061bN86ePUvbtm25efMmMpmMyZMnU6dOHU6cOMGsWbPIyMigf//+hVpPhw4dsLKykloM69evz6ZNm4iNjZXL5+HhQaNGjYCclrLnz58THh5O9erVAVi6dKm0TyHnxPr111/To0cPAPT09EhJScHX15cpU6ZQqVKlQu8L4cu1d+9eoqKi8qV369aNpUuXStM2Njb07dsXgIEDB3L06FGmTZuGmZkZAO3bt893PJk3b540PnXJkiV06dKFffv20aNHDzZs2MAPP/yAjY0NkPObvX//PqGhoQwdOpTffvuNmjVrsmDBApSUlDAwMODBgwf4+/sDkJKSQnh4OAsWLKBLly4ALFy4kNOnT39wezt27Mjw4cOBnN/EL7/8woULF+jbty8bNmzAxMSEmTNnAtC4cWOePXvG999//97yqlevTnh4OOvXr+fQoUOsX7+e9evXo6amxtixY5kwYUK+ZTQ0NNDQ0ADg7t27LFiwAA8PD5ydnQEICgpi4MCBDB48WNo3vr6+jBgxgsTEROrXr//BbRRKnwjUKiBlpUpSMPb6zSN2RY3A2X4j6lVqS/NLWsOGDaW/K1euDOT84HOpqamVWVN63u5CDQ0NMjIyAIiJieGrr76Sm29lZVWkQM3W1pZLly4REBBAfHw88fHxxMXFUbt27QLzW1payk1HRkZy9epV7ty5k28ewK1bt2jbti379u0jKytLClaNjY1JSkoiNDSU/v3706NHD5KSkqTl8rY4ALi6unLs2DF27dpFQkICcXFxJCYm0rhxY7l8uUEawNWrV9HX15eCtNx15x78nz9/zsOHD/nhhx8ICAiQ8shkMtLS0khMTPzsLlvhy2Bra8v06dPzpee9IaYwx5O8XZ+tWrWS/q5WrRqNGjUiNjYWIyMj0tLS+M9//oOi4v91GmVmZpKenk5qaiqxsbE0b94cJSUlaX7Lli2lv+Pj48nIyMDU1FRKq1SpEs2bN//g9n7o2HP16lXat28vN9/KyuqD5UFOsDZt2jSmTZvG48eP+eOPP9i+fTurVq1CS0sLV1fXApd7+fIlY8eOxcrKiv/85z9S+tWrV7l8+TI7duyQ0nJb8W/duiUCtXJIBGoVUEF3d6pXqU3VKrVKrQ7Kyvm/Ou8eFD8mKyurOKsjR1VVNV9a7oFIWVm5yOOo3h1jAhASEsKaNWtwdnbG2toad3d3jh49SmRkZIHL7969W266Vq1ayGQyevXqJbWovSu3RbGgmyQMDQ2lsTYhISFydatduzaXLl2SpmUyGePGjePmzZv07NkTJycnWrRowbx58/KV+24LmJKS0gf3Ue68WbNm5TvxQE73tSAAqKurywVh71PQ8SR38Pz7vBtkQc4xRVVVVfqtr1y5Mt8FCeQcHxQUFPJ9x9+tQ+66897YVFA985adV24ZH/tdFSQkJIT69evj5OQE5Bw7+vTpQ69evRg0aBAnTpwoMFDLyMhg4sSJVK5cmSVLlsjtS5lMxujRo6UWtnd9qTd+VHRijJpQ4lRUVADkBsonJCS8N//HDtCfw9jYWC6YAbh48aLctIqKilxd79y5Izc/KCiICRMm4OPjw6BBg7CwsCAhISHfQT1Xw4YN5T7Kyso0bdqUuLg4ufTMzEz8/f158OABr169ok2bNvluarhy5QpNmzYFcrpc310+b2B37do1Tp48SUBAANOnT6d37940aNCAu3fvvreuufvozp07vHjxQkq7desW//zzDwA1atRAW1ube/fuya0/JiaGlStXvrdcQShOf//9t/T38+fPSUhIoGnTpjRu3BhlZWWSkpLkvp8nTpwgNDQURUVFjI2N+fvvv+Va/d8tT19fn0qVKnHhwgUpLTMzM98NB0VhbGzM5cuX5dLyHnvyunz5Mj/99FO+i0VFRUWqVq0q99ied82fP5/4+Hh++umnfK2XTZs2JT4+Xm7fPHz4kCVLlvD69etP2DKhpIlArYLL7QYtje7OT2VhYYGCggKBgYEkJiZy4MABdu3a9d78uQeW2NhYKTgoLmPGjOHgwYOsX7+ehIQEdu7cyaZNm/LVd/v27Vy7do2rV6/i4+Mjd6Wsq6vLqVOniIuL4/bt26xYsYJDhw4VqavXw8ODq1ev4uvry61bt7h48SL/+c9/SEhIoFGjRlSrVo127dqxYsUKTpw4QUJCAiEhIUREREiDgj+mZs2aKCsrc+DAAe7du8eVK1eYOnUqT548+WBde/bsiZaWFtOnT+f69ev89ddf0kBjBQUFFBQUGDNmDL/88gubNm3i7t27HD58GB8fH9TU1ApsVRD+nVJTU3ny5EmBn88dGjF//nz++OMPrl27xrRp09DV1cXJyQkNDQ0GDx5MQEAAe/bs4d69e+zYsYOlS5dSq1ZOr8OQIUN4+/Yts2fP5tatWxw/flzuQdLq6uoMGzaMVatWcejQIW7dusWCBQt49OjRJ9fXw8ODK1eusGzZMuLj4zl8+DCrVq0C3n9xOmHCBBITExk1ahTR0dHcv3+fixcvsmjRIv766y9GjhyZb5ng4GD279/PsmXLUFFRkdvnWVlZjBkzhqioKFavXk18fDx//PEHs2bN4p9//hEtauWU6Pqs4CrCQ2719PTw9fUlODiYLVu20KpVK2bOnImXl1eB+bW0tHBxcWHJkiXcuXOH7t27F1tdOnfujJ+fH8HBwSxfvhwTExOGDBkiF6z5+Pjg4+PDwIEDqVWrFlOmTOHhw4fS/CVLluDn54eLiwvq6uqYm5vj6+uLj48PSUlJ1K1b96P1sLCwYN26dQQEBODs7EyVKlWwtrbGy8tLCnQWLlxIYGAgCxYs4NmzZxgYGLBq1apCP+C3du3aLFq0iMDAQDZv3oyOjg42Nja4u7tz7Nix9y6nqqrKunXr8PPzY+DAgWhqauLp6UlMTIzUOurh4UGlSpX45ZdfWLRoETVr1mTgwIFMnjy5UHUT/h0OHDjAgQMHCpwXEBCAg4NDgfMKY/z48cyaNYvnz5/Ttm1b1q1bJ/12Zs2ahZaWFgEBATx+/BhdXV0mT57M6NGjgZzfxsaNG1m4cCHOzs7o6ury9ddf4+vrK5X/n//8h0qVKuHn58fr169xdHT8rKf/Gxoasnr1an744Qc2bNiAvr6+dFd67u8qr2bNmrF9+3Z+/PFHZs2aRXJyMurq6rRp04atW7dKrevv2rZtG6mpqYwYMSLfvKNHj+Lg4MCKFSsIDg4mKCiI6tWrv3csoVA+KGR/qA9EKDapqanEx8ejr6//xT6gtSI4e/YsNWvWlBu7EhQUxI4dOzhy5EgZ1qz8SExMJCEhgY4dO0ppjx49onPnzmzevJnWrVuXYe0EoWK6fPkyysrKcjck7N27l9mzZ3Px4sWPjn8TPl9FPQ+Lrk/hXyU6OppRo0Zx+vRpkpKSOHr0KBs3bpR79MS/XVpaGmPHjiU0NJR79+5x9epV5s2bR6NGjTA3Ny/r6glChXTt2jWGDx/O0aNHSUpK4o8//iAwMJAePXqIIE34INGiVkoqaiT/pUlPT2fJkiUcOnSI58+fo6urS//+/Rk9enS+u8j+zQ4ePEhQUBDx8fGoqalhbW3NzJkzC9WtKwhCftnZ2axZs4Zdu3bx6NEjatSoQY8ePZg8ebI4J5SSinoeFoFaKamoXxBBEARB+BJU1POw6PoUBEEQBEEop0SgJgiCIAiCUE6JQE0QBEEQBKGcEoGaIAiCIAhCOSUCNUEQBEEQhHJKBGqCIAiCIAjllAjUBEEQBEEQyikRqAnlgq2trdxLkY8fP05cXBwAZ86cwcjIiMTExHx5w8PDMTIyKv0Kf4SRkRHh4eGfVUZWVhZmZmYYGRnJfd7dT3nl3VfCv1tWVhbPnz+XPllZWaW27mfPnjFjxgzatWuHpaUlY8eO5datW9L8mJgY3NzcsLS0xMbGhmXLlr33Re3x8fFYWlp+9m9KECoi8d6KCi4rO4v0rDRUlSqhpPBlPFn//v37eHp6EhYWRpMmTbC0tCQ6Ohptbe18eZ2cnAr9kvKKJiEhgbS0NPbs2UONGjWk9CpVqpRhrYSKIjk5mcjISH799VceP35MrVq1GDJkCD169EBLS6vE1z9hwgRkMhkhISGoq6sTEBCAu7s7hw4dIjU1FQ8PDxwcHPjuu++4e/cuXl5eyGQyZs6cKVdORkYG06dP582bNyVeZ0Eoj0SLWgWWlZ3Fo9f3mXFyGI9e3ycru/SulktS3pdlqKqqoqOjU+ArntTU1NDR0SmtqpWqGzduULVqVYyNjdHR0ZE+6urqZV01oZxLTk5m0qRJrFy5kkePHpGdnc2jR49YuXIlkydPJjk5uUTX//LlS+rVq8d3332HmZkZBgYGjB8/nsePH3Pz5k3+/PNPXrx4wYwZM2jYsCGdOnWiV69e/P777/nKCgwMpGrVqiVaX0Eoz0SgVkHlBmne0e7cT0nAO9q91II1IyMjtm3bhqurK6ampjg6OnLhwgW2bduGjY0NLVu2ZOrUqaSmpgIFd0++r8syMTEROzs7AIYPH05gYOAHu/PylmNkZMSOHTtwd3fHzMyMjh07snr1arll9u7di6OjI6ampgwYMICwsLB8ZeTtYnk3TSaTERwcjL29PSYmJrRs2ZLRo0dz9+7douxGLly4wNChQzEzM8PGxgZfX19SUlKk+Tdu3MDAwKBIZeY6duwY3bp1w9TUFDc3N65fvy7Ne/nyJXPnzqVTp060aNECa2tr5s6dy9u3b6U8oaGhdOvWDRMTE2xtbVmzZo1cAH38+HH69euHmZkZ3bt3Z+XKle/tthJKV1ZWFpGRkXL/5++6du0a+/fvL9FuUE1NTZYvX46hoSEAz58/Z8OGDdSpU4cmTZpIreO//vorWVlZJCYmcuLECczNzeXKOXfuHNu2bWPRokUlVldBKO9EoFYBvRuk/ZP+AoB/0l+UarC2YsUKRo8ezZ49e9DQ0MDT05OoqChCQkLw9/fnyJEjbN++vcjl6urqSssFBgbi4eFR5DIWL16Ms7MzkZGRDBs2jMDAQM6dOwfkBBheXl7079+fiIgI+vXrx7Jly4pUflhYGKGhoXh7exMVFcWaNWtISEgo0snk+vXrjBw5kk6dOhEREcGyZcuIiYnBw8NDCohiY2PJzMxk1KhRdOjQgX79+rFnz55Clf/zzz+zYMECdu7cibq6OqNHj5YCMW9vb65evcrq1auJiopi1qxZ7N69m23btgE5QV5wcDC+vr4cOnSI6dOn89NPPxEREQHAyZMnmTp1KgMHDmTfvn0sWLCAAwcOMGPGjKLsRqGEvHz5kl9//fWDeX799VdevnxZKvWZN28e1tbWREZG8v3331OlShVatmzJ119/TUBAAKamptjZ2VG7dm3mz58vLffq1StmzpzJ3Llz0dXVLZW6CkJ5JAK1Cig9K42FZ6dKQVquf9JfsPDsVNKz0kq8Di4uLtja2tK4cWP69OnDy5cvmT9/PoaGhtjb29OsWTNu3rxZ5HKVlJSkq21NTc1P6ubr27cvffr0QU9PD09PT6pVq8aFCxeAnJYiBwcHRo0ahb6+PkOGDGHIkCFFKr9BgwYsXryYrl27Uq9ePaytrXFwcCA2NrbQZYSGhtKhQwc8PT1p1KgRrVu3Zvny5Vy6dImzZ88CcPPmTV68eIGbmxuhoaHY29sza9YsduzY8dHy582bR6dOnTA0NGTJkiW8fv2affv2AdChQwf8/f0xNzenfv369O7dm+bNm0v1v3v3LqqqqtSrV4+6devi5OTEhg0bsLKyAiAoKIiBAwcyePBgGjRoQMeOHfH19eXgwYPiJoZy4tGjR581vziNGDGCnTt30rNnTyZMmEBMTAwpKSncvn2boUOHsn37dgICAkhISGDevHnScj4+PlhaWtKrV69Sq6sglEfiZoIKSFWpErPbrJRrUQPQUK3O7DYrUVWqVOJ1aNiwofR35cqVgZwAJpeamlqZdYXl7S7U0NAgIyMDyLnT7KuvvpKbb2VlxYYNGwpdvq2tLZcuXSIgIID4+Hji4+OJi4ujdu3aBea3tLSUm46MjOTq1avcuXMn3zyAW7du0bZtW/bt20dWVpYUrBobG5OUlERoaCj9+/enR48eJCUlScutXbtW+rtVq1bS39WqVaNRo0ZSIObq6sqxY8fYtWsXCQkJxMXFkZiYSOPGjQHo3bs3O3fuxN7eniZNmtC+fXvs7e2pW7cuAFevXuXy5ctyAWNuK+CtW7eoX79+ofelUDJq1679wWDsfd/VktCkSRMAvv/+ey5dusSmTZtQVVXl5cuXrFq1CoAWLVqgqamJu7s77u7u3Lhxg/Pnz7N3795Sq6cglFciUKuAlBSUqK1ej0UdN0jBmoZqdRZ13EBt9XqlcvensnL+r46iYuEbaEtyfIyqqmq+tNxAQllZGZlMVqTyMjMz5aZDQkJYs2YNzs7OWFtb4+7uztGjR4mMjCxw+d27d8tN16pVC5lMRq9evfD09MyXP7dFUU1NLd88Q0NDqQsyJCRErm61a9fm0qVLAPluvMjKykJVVRWZTMa4ceO4efMmPXv2xMnJiRYtWsi1ZGhra7Nnzx4uXrzIqVOniI6OJiwsjEmTJjFx4kRkMhmjR4/G2dk5X/2+1Bs7KhJNTU2GDBnCypUr35tnyJAhaGpqllgdnj9/zh9//IG9vb10rFBUVKRJkyY8fvyYR48eYWNjI7dM7vi0hIQEdu7cybNnz/LlWbBgAfv372fdunUlVndBKG9EoFZBvRusLTw7ldltVpZakFZUKioqAKSkpEh3byUkJLw3v4KCQonVxdjYWApmcl28eFFuWkVFRW5Q/507d+TmBwUFMWHCBMaOHSulhYaG5rtbNde7rY+5mjZtSlxcnNy8W7dusXTpUr755huys7Pp1q0b3t7e9OvXT8pz5coVmjZtCkC9evXeu51///031tbWQM5JMyEhAQ8PD65du8bJkyf57bffpBNjRkYGd+/eRU9PD4CIiAj++ecfhg4dSqtWrZg8eTJz585l//79TJw4kaZNmxIfHy9X9zNnzhAWFoaPj494fEgZU1JSokePHkRFRXHt2rV885s1a0aPHj0KvIu6uDx9+pRvvvmGdevWSY/PycjI4OrVq9ja2gI5N8u8K3daX1+fZcuWSTcj5frqq6+YPHkyvXv3LrF6C0J5JAK1Ciw3WFvaeVO5fo6ahYUFCgoKBAYG4ubmxpUrV9i1a9d78+ee6GNjY2nevHmx1mXMmDGMGzcOMzMzunbtyp9//smmTZvy1Xf79u1YWVmRnZ2Nv7+/XCudrq4up06dwtbWFkVFRfbs2cOhQ4eoWbNmoevh4eHB0KFD8fX1ZdiwYbx69QpfX19SU1Np1KgRqqqqtGvXjhUrVlCjRg0aNmzIoUOHiIiIIDg4+KPlz58/Hz8/P6pXr86iRYvQ1dXFycmJ5ORklJWVOXDgANra2rx48YKgoCCePHkidVWnpaWxePFi1NXVad26NQ8fPuTcuXO0bt1a2odTp05l9erV9OjRg4cPHzJnzhzq168vWtTKCS0tLVatWsX+/fv59ddfefToEbVr15aeo1a9evUSXb+hoSGdO3fmu+++47vvvkNTU5Pg4GBevXqFu7s7t27dYsyYMaxcuZJ+/fpx//59fH19sbGxwdjY+L3l1qhRo1S7bQWhPBCBWgWnpKBEZeXy3YKhp6eHr68vwcHBbNmyhVatWjFz5ky8vLwKzK+lpYWLiwtLlizhzp07dO/evdjq0rlzZ/z8/AgODmb58uWYmJgwZMgQuWDNx8cHHx8fBg4cSK1atZgyZQoPHz6U5i9ZsgQ/Pz9cXFxQV1fH3NwcX19ffHx8SEpKksZyfYiFhQXr1q0jICAAZ2dnqlSpgrW1NV5eXlJQuHDhQgIDA1mwYAHPnj3DwMCAVatWFeoBv+PHj2fWrFk8f/6ctm3bsm7dOlRVValduzaLFi0iMDCQzZs3o6Ojg42NDe7u7hw7dgyAAQMG8OLFC3788UcePHiApqYm9vb2TJ8+HQAHBwdWrFhBcHAwQUFBVK9eHVtbW2m+UD5oaWkxePBgHB0dpTRNTc0SbUl71w8//MDy5cuZNm0a//zzD61bt2bz5s3UrVuXunXrEhwczJo1a9i4cSNaWlp0796dKVOmlErdBKEiUch+X3+NUKxSU1OJj49HX1+/wLFHQuk4e/YsNWvWlAbOQ05X5o4dOzhy5EgZ1kwQBEEoSRX1PCwezyH8q0RHRzNq1ChOnz5NUlISR48eZePGjfTp06esqyYIgiAI+YiuT+FfZeLEibx584aZM2fy/PlzdHV1cXd3Z/To0WVdNUEQBEHIR3R9lpKK2uQqCIIgCF+CinoeFl2fgiAIgiAI5ZQI1ARBEARBEMopEagJgiAIgiCUUyJQEwRBEARBKKdEoCYIgiAIglBOiUBNEARBEAShnBKBmlAu2NraEhgYKE0fP36cuLg4IOeF30ZGRiQmJubLGx4ejpGRUelX+COMjIwIDw//rDKysrIwMzPDyMhI7vPufioP8v4f5P2/FMpGamoqz54948SJE2zdupUTJ07w7NmzfC87Lym5v9uCPnZ2dgCkpKSwYMEC2rVrR6tWrfD09OTevXty5URGRtKzZ0/Mzc1xcnJi9+7dpVJ/QSgvxANvvwBvMtOpoqz68YwVxP379/H09CQsLIwmTZpgaWlJdHQ02tra+fI6OTkV6t2XFVFCQgJpaWns2bOHGjVqSOm5L60XhPd59eoV+/bt46effuLt27dSeuXKlRk/fjw9evSgWrVqJVqH3N/tu/766y8mTZrE+PHjAZg0aRIPHjxgzZo1qKur8+233/L1118TERGBoqIip0+fZubMmcybN48OHTpw8uRJZs2ahZaWFl26dCnR+gtCeSFa1Cq4V+lvOXr/Gq/SS+cquTTkfQazqqoqOjo6Bb5MWk1NDR0dndKqWqm6ceMGVatWxdjYGB0dHemjrq5e1lUTyrHU1FQiIyP54Ycf5II0gLdv37J8+XIiIyNLvGUt93f77vfW398fZ2dnXFxcOHPmDH/88QcBAQG0atUKY2NjfH19ef36NQkJCQAcPXoUIyMjBg8ejJ6eHkOHDsXY2Jjff/+9ROsuCOWJCNQquMdv/8Hv4j4ep74qtXUaGRmxbds2XF1dMTU1xdHRkQsXLrBt2zZsbGxo2bIlU6dOlU4EBXVPvq/LMjExUeoWGT58OIGBgfm6Pj9UjpGRETt27MDd3R0zMzM6duzI6tWr5ZbZu3cvjo6OmJqaMmDAAMLCwvKVkbfb8t00mUxGcHAw9vb2mJiY0LJlS0aPHs3du3eLshu5cOECQ4cOxczMDBsbG3x9fUlJSZHm37hxAwMDgyKVCRAaGkq3bt0wMTHB1taWNWvWSMFvYGAg7u7urF69mvbt22Npacn8+fN58OAB48aNw9zcnO7du/Pf//5XKi8pKYlp06ZhbW1NixYt6Ny5M0uXLkUmkxW5bkLJe/36NT/++OMH8/z000+8efOmlGqUIygoiLdv3+Ll5QXkvHfX0NBQ7rfXpEkTjh8/TuPGjQGoUaMGN2/e5PTp02RnZ3PmzBlu3bqFmZlZqdZdEMqSCNQqqFfpqcS9fIz3uZ0AeJ/dSdzLx6XWsrZixQpGjx7Nnj170NDQwNPTk6ioKEJCQvD39+fIkSNs3769yOXq6upKywUGBuLh4VHkMhYvXoyzszORkZEMGzaMwMBAzp07B+SMffPy8qJ///5ERETQr18/li1bVqTyw8LCCA0Nxdvbm6ioKNasWUNCQgKLFi0qdBnXr19n5MiRdOrUiYiICJYtW0ZMTAweHh5SUBUbG0tmZiajRo2iQ4cO9OvXjz179nyw3GPHjhEcHIyvry+HDh1i+vTp/PTTT0REREh5zp8/T3x8PJs3b2bu3Lls27aN/v374+joSHh4OAYGBnh7e0v1+Prrr/nnn39Yv349Bw8exMPDg3Xr1nHs2LEi7TehdPz999/5WtLyevPmDVeuXCmlGsHz58/ZsGEDnp6eVK9eHYD4+HgaNmzIli1b6NGjB506dWLq1Kk8evRIWs7NzY1OnToxYsQIWrRowfDhwxk5ciS9e/cutboLQlkTgVoF9CYznf8+uMGQ42u5k/IcgDspzxlyfC0nHtzgTWZ6idfBxcUFW1tbGjduTJ8+fXj58iXz58/H0NAQe3t7mjVrxs2bN4tcrpKSkjQWTVNT85O6+fr27UufPn3Q09PD09OTatWqceHCBSCntcnBwYFRo0ahr6/PkCFDGDJkSJHKb9CgAYsXL6Zr167Uq1cPa2trHBwciI2NLXQZoaGhdOjQAU9PTxo1akTr1q1Zvnw5ly5d4uzZswDcvHmTFy9e4ObmRmhoKPb29syaNYsdO3a8t9y7d++iqqpKvXr1qFu3Lk5OTmzYsAErKyspj0wmw9fXF319fVxcXNDS0qJdu3b07dsXAwMDhgwZQnJyMk+ePCE1NZU+ffrw7bffYmxsjJ6eHu7u7tSsWZMbN24Uab8JpePBgweFyvduQFTStmzZgoaGBoMGDZLSUlJSOH36NPv378fX15cVK1bw8OFDhg8fTlpaGpCzLcnJycyfP5+dO3fi7e3N+vXrP/gbEIQvjbiZoAKqoqyKja4RzbV08T67kzspz2lYVZtFbVyopVatVG4saNiwofR35cqVgZwAJpeamhrp6SUfMBYkb3ehhoYGGRkZAMTExPDVV1/JzbeysmLDhg2FLt/W1pZLly4REBBAfHw88fHxxMXFUbt27QLzW1payk1HRkZy9epV7ty5k28ewK1bt2jbti379u0jKytLClaNjY1JSkoiNDSU/v3706NHD5KSkqTl1q5dS+/evdm5cyf29vY0adKE9u3bY29vT926daV8NWrUoGrVqtJ0lSpV8v3fAaSnp6OmpsawYcM4ePAgly9f5s6dO9y4cYOnT5+Krs9ySldXt1D53vd9LQm7d++mb9++ci/CVlZWJi0tjTVr1qCpqQnA6tWr6dSpE8eOHcPR0ZFJkybRs2dPhg4dCkCzZs14+fIlS5cupV+/figqirYG4csnArUKqpqqGtVU1Vhk5cKQ42tZ1MaFJtVqldr6lZXzf3WKctDMysoqzurIUVXNH6jmduMpKysXOcDIzMyUmw4JCWHNmjU4OztjbW2Nu7s7R48eJTIyssDl8z5OoFatWshkMnr16oWnp2e+/Lktiu+e1HIZGhpK3ZghISFydatduzZqamrs2bOHixcvcurUKaKjowkLC2PSpElMnDgRABUVlXzlvu//7s2bNwwbNozU1FQcHBxwdnbGzMxMOnEK5Y+JiQmVK1f+YPdnlSpVMDU1LZX6XL9+nXv37tGrVy+59Dp16lC7dm0pSAOoWbMm1atXJzExkefPn3P79u189bSwsOCnn37ixYsXBd4JLghfGnE5UsHVqlyN+ZY9qaVWsrfaf47cwODdgfK5d3UVREFBocTqYmxszKVLl+TSLl68KDetoqIiV9c7d+7IzQ8KCmLChAn4+PgwaNAgLCwsSEhIyHe3aq6GDRvKfZSVlWnatClxcXFy6ZmZmfj7+/PgwQNevXpFmzZt8t3UcOXKFZo2bQpAvXr15JZXU1MjIiKCX3/9lVatWjF58mR+++03BgwYwP79+z9pf0VHRxMTE0NYWBiTJ0/GycmJqlWr8uzZs/dur1C21NXVpcdfvM/XX39dao95OX/+PDVq1MDY2Fgu3crKiqSkJB4/fiylPX78mOTkZBo2bIimpiaVK1fO18V+48YNqlWrJoI04V9DtKhVcNVU1bCr16xcP0fNwsICBQUFAgMDcXNz48qVK+zateu9+XNPILGxsTRv3rxY6zJmzBjGjRuHmZkZXbt25c8//2TTpk356rt9+3asrKzIzs7G399frpVOV1eXU6dOYWtri6KiInv27OHQoUPUrFmz0PXw8PBg6NCh+Pr6MmzYMF69eoWvry+pqak0atQIVVVV2rVrx4oVK6hRowYNGzbk0KFDREREEBwc/N5y09LSWLx4Merq6rRu3ZqHDx9y7tw5WrduXfSdRU6rB0BERAT29vY8ePCAH374gYyMjDLr2hY+TE1NjR49egDw448/yrWsValSha+//poePXoU2GJbEq5evVrgHd6Ojo6EhIQwZcoU5syZg6KiIgsXLkRfXx8bGxuUlJQYPnw4P/30Ezo6OrRq1Yo///yT4OBgJkyYUCp1F4TyQARqX4DyHKQB6Onp4evrS3BwMFu2bKFVq1bMnDlTuk0/Ly0tLVxcXFiyZAl37tyhe/fuxVaXzp074+fnR3BwMMuXL8fExIQhQ4bIBWs+Pj74+PgwcOBAatWqxZQpU3j48KE0f8mSJfj5+eHi4oK6ujrm5ub4+vri4+NDUlKS3Hiw97GwsGDdunUEBATg7OxMlSpVsLa2xsvLSwoKFy5cSGBgIAsWLODZs2cYGBiwatWqDz7gd8CAAbx48YIff/yRBw8eoKmpib29PdOnT/+k/WVmZsasWbPYsGEDK1eupHbt2jg5OaGrq1uqdw0KRVOtWjWcnZ2xt7fnypUrPHr0iNq1a2NqakqVKlVKLUgDePLkiXSn57tUVVXZsGEDixYtYsSIEWRnZ9OhQweWL18u/QamTJmClpYWwcHBPHjwgPr16zNjxgwGDx5cavUXhLKmkC36L0pFamoq8fHx6Ovrl+pBUpB39uxZatasKT2nCXK6Mnfs2MGRI0fKsGaCIAhCSaqo52ExRk34V4mOjmbUqFGcPn2apKQkjh49ysaNG+nTp09ZV00QBEEQ8hFdn8K/ysSJE3nz5g0zZ87k+fPn6Orq4u7uzujRo8u6aoIgCIKQj+j6LCUVtclVEARBEL4EFfU8LLo+BUEQBEEQyikRqAmCIAiCIJRTIlATBEEQBEEop0SgJgiCIAiCUE6JQE0QBEEQBKGcEoGaIAiCIAhCOSWeoyYIgvAFe/36NRkZGaioqKCurl7W1REEoYhEi5pQ7AIDA7G1tZWmjYyMCA8PL8Mafbo///yT8+fPfzBPRd4+4cuVnJzM+fPnmTdvHlOmTGHevHmcP3+e5OTkUq9LcHAwbm5ucmnXrl1j2LBhWFhYYGtrS1hYmNz8Fy9eMH/+fDp37kzLli0ZMmTIe3+L2dnZjBo1Kt86BOFLIAK1L8CbjIyyrsIXy9XVlbt375Z1NQShSB4/fsy4cePw9PTk5MmTxMTEcPLkSTw9PRk3bhxPnjwptbps3ryZlStXyqUlJyczcuRIGjRowM6dO5kwYQLLli1j586dUp5vvvmGixcv8sMPP7Bz506aNWvGqFGjuH37dr51bNy4kejo6JLeFEEoEyJQq+BSMzN5+PY1qZmZZV0VQRDKgeTkZCZOnFhgQANw+/ZtJkyYUOIta48ePcLT05Nly5bRqFEjuXm//fYbKioq+Pn5YWBggIuLC+7u7oSEhABw584dTp06hY+PD61bt0ZfX5958+ZRq1Yt9u7dK1fWjRs3WLNmDRYWFiW6PYJQVkSgVsGlZGQw7uRhUkq5VS02NpZx48ZhZWWFiYkJdnZ2/Pzzz59UVnh4ON27d2fr1q3Y2Nhgbm7O5MmTefToEdOnT8fS0pLOnTuzY8cOaRk3Nze+//57vvnmG8zNzencuTMhISHkvhHtzJkzNG/enBMnTtCzZ09MTExwcHDgyJEjUhnZ2dmsXbsWOzs7zM3N6dOnDxEREdJ8IyMjAGbNmoW3t/cHt+H27dsMHjwYExMTHB0dOXDggDRPJpMRHByMvb09JiYmtGzZktGjR8u11J04cYJ+/fphbm6OtbU13t7evHz5Upp/69YtxowZg6WlJR07duQ///lPqbaKCBXHrVu33huk5bp9+/ZH83yumJgYVFRUiIiIwNzcXG7e+fPnadOmDcrK/zdMul27diQkJPD06VO0tLQICQnB1NRUmq+goICCggKvXr2S0tLS0pg+fTqTJ09GX1+/RLdHEMqKCNQqsNTMTLbdus6r9HR+u32j1FrV3r59i4eHB9WrV2fr1q3s27cPBwcHFi9ezLVr1z6pzKSkJA4ePEhISAirVq3i6NGj9OrVixYtWrBz5046d+6Mj4+PXCvAr7/+ioaGBuHh4UybNo01a9awdu1aaX5WVhZLly5lzpw57Nu3D0NDQ7y8vHj9+jUAK1as4Ndff2XevHns3buX4cOH4+Pjw+bNmwGkrpTZs2czZ86cD9Z/48aN9O3bl71792Jvb8+0adP4+++/AQgLCyM0NBRvb2+ioqJYs2YNCQkJLFq0CIDnz58zceJEXFxc2L9/P6tXr+bcuXMsWbIEyGmZcHV1pWHDhuzYsYOgoCBSUlIYNGgQb968+aT9LXyZXr9+zZYtWwqVd8uWLdJvoSTY2toSGBiInp5evnkPHz6kTp06cmm1atUC4MGDB1SrVo0uXbqgqqoqzY+KiuLOnTt06tRJSlu6dCm1atVi2LBhJbQVglD2xF2fFVhKRgbb4m4AsDXuOgMbG6GmXPL/pW/fvmX48OEMHTpUuots8uTJrFu3jhs3bnxSmZmZmcybNw8DAwMMDQ0xNjZGRUWFkSNHAjBy5Ei2b99OQkICWlpaAOjr6+Pj44OCggIGBgbcunWLsLAwxowZI5U7depUrK2tARg/fjxRUVHExsZiZGTEhg0b+OGHH7CxsQGgQYMG3L9/n9DQUIYOHYqOjg4AGhoaaGhofLD+rq6uDB48WFrn6dOn2bBhA8uWLaNBgwYsXryYrl27AlCvXj0cHBw4ePAgkBOIpaenU7duXerVq0e9evUICgoiKysLyAlI69Spw9y5c6X1rVy5knbt2nHw4EH69ev3Sftc+PJkZGTw7NmzQuV99uwZGWU0vjU1NVUuCAOoVKkSkNNKlteFCxeYNWsWX331lfR7PXnyJHv37iUiIgIFBYUSr7MglBURqFVQua1pabKck3laVha/3b6Bh5FJiQdr2trauLq6sm/fPq5evcrdu3e5fv06kNPN9yGWlpZy05GRkdLfDRo0kP6uUqUKurq60nTuQTw9PV1Ka9u2rdwB2tLSkrVr18q1ujVu3Fj6u2rVqkDOySwuLo60tDT+85//oKj4fw3LmZmZpKenk5qaipqamlxd58+fLzc+JnewNkCrVq3k8pqbm3P69Gkgp2Xh0qVLBAQEEB8fT3x8PHFxcdSuXRuAZs2a0bNnTzw9PdHR0aFDhw7Y2NjQvXt3AK5evcrNmzfz7bu0tDRu3bqFIORSUVGhRo0ahcpbo0YNVFRUSrhGBVNTU5P7LcP/BWhVqlSRSz9y5AjTp0+nZcuWLFu2DMhphZ49ezY+Pj7S70gQvlQiUKug3m1Ny1VarWpPnjxh0KBBaGtrY2trS8eOHTE1NaVLly4fXXb37t1y07ndHUC+k8a7AVRBlPNsZ26QqKSkJKXlvWqHnLFpuWPZVq5cKRfMfWi5KVOmMGrUKGlaU1PzvXXNysqSyggJCWHNmjU4OztjbW2Nu7s7R48elQtSly9fzoQJEzh58iT/+9//mDFjBq1atWLjxo3IZDLatWvHggUL8tXpYy19wr+Luro6rq6unDx58qN5XV1dy+y5anXq1OHx48dyabnT7wZemzZt4vvvv5eGVuT+pk6cOMGTJ0+YPXs2s2fPBnIu4mQyGZaWlkRGRlK3bt1S2hpBKFkiUKuA8ram5SqtVrV9+/bx4sULoqKipOAqt8szNwB6n4YNGxZbPa5cuSI3feHCBerXry8XQL1P48aNUVZWJikpSeqShJzxZHFxcfj5+eVbpkaNGu9trYiJiaFbt25ydTE2NgYgKCiICRMmMHbsWGl+aGiotK8uXbpEZGQks2fPpnHjxri7uxMREcGMGTN49uwZTZs2Zf/+/ejq6konqhcvXuDl5cXIkSNp167dR7dX+PcwMDCgcePGH7xZoHHjxgVeoJQWKysrtm7dSlZWlnRhdfr0afT19aXf2JYtW/j2229xc3Njzpw5cq3n3bt3p2XLlnJlLlu2jIcPH7Js2TK5C0BBqOhEoFYBpcuyGGRgzAADo3zzFFEgXZaFWgn+19apU4e3b99y8OBBWrVqxe3bt/H398+pW57ujJJ0/vx5Vq1aRe/evTl//jybN29m1qxZhVpWQ0ODwYMHExAQQNWqVWnZsiVnzpxh6dKljBs3TspXpUoVbt26RXJysjQ2riAbNmygQYMGmJubs3XrVmJjY1m+fDkAurq6nDp1CltbWxQVFdmzZw+HDh2iZs2aQE6X7JYtW1BRUWHgwIGkpaWxf/9+GjVqhJaWFq6urmzbto3p06czfvx4ABYvXsyNGzcwNDT81N0nfKG0tLRYs2YNEyZMKDBYa9y4MWvWrPng97mkubi4sG7dOubMmcPo0aO5fPkyGzZswNfXF4D4+HgWLlxI9+7dGTduHE+fPpWWVVNTQ0NDQxrKkEtdXR01NbVivRgUhPJABGoVUDXVSmW6fgcHB2JiYli0aBEpKSnUq1ePAQMGcPToUa5cuSI3tqwk2dnZcevWLXr37k2tWrWYNWsWQ4YMKfTys2bNQktLi4CAAB4/foyuri6TJ09m9OjRUh4PDw/WrVvHpMhdEgAAEUVJREFUrVu3CAoKem9Z48eP55dffmHevHk0adKEkJAQ6XEBS5Yswc/PDxcXF9TV1TE3N8fX1xcfHx+SkpIwMDAgMDCQ1atXs2XLFhQVFWnXrh1r165FUVERPT09Nm3axPLlyxkyZAhKSkq0bNmSsLAwtLW1P30HCl8sHR0dgoODuX37Nlu2bOHZs2fUqFEDV1dXGjduXKZBGuS0Tq9bt47vv/8eZ2dndHR0mDlzJs7OzkDOHZ4ZGRkcPnyYw4cPyy3r7Ows3TEtCP8GCtkf66sSikVqairx8fHo6+vnG6QuFJ2bmxv16tUTB2xB+Ajxrk9ByFFRz8OiRU0QBOELJoIzQajYxANvBUEQBEEQyinRoiZUSL/88ktZV0EQBEEQSpxoURMEQRAEQSinRKAmCIIgCIJQTolATRAEQRAEoZwSgZogCIIgCEI5JW4mEARB+MJkZmby6tUr4uPjOXv2LC9fvkRTU5M2bdqgr6+Ppqam3DtxBUEov0SgJgiC8AVJTk5m7969bNmyRe7VS5DzjtmaNWvi6upKr169yvwNBYIgfJx4M0EpqahPRBYEoeJ4+vQpEydOJC4u7qN5mzRpwurVq6V3zgrCl66inofFGDWhXLC1tSUwMFCaPn78uHSyOXPmDEZGRiQmJubLGx4ejpFR/pfTlzUjIyPCw8PLuhqf5M8//+T8+fMfzFORt+9LlZycXOggDSAuLo6JEyeSnJxcwjXLecm6paWl3Hfm2rVrDBs2DAsLC2xtbQkLC5NbJiUlhQULFtCxY0fatGnD9OnTefbsWb5yx44di6WlJR06dMDPz4+3b9+W+PYIQmkSgZpQ7ty/fx9PT0/poGxpaUl0dHSBL3t3cnIiOjq6tKv4RXN1deXu3btlXQ2hCDIzM9m7d2+hg7RccXFx7N27l6ysrBKqGWRkZDB9+nTevHkjpSUnJzNy5EgaNGjAzp07mTBhAsuWLWPnzp1SnilTpnDixAm+//57Nm/ezNu3bxk+fDjp6elSGcOGDUNZWZnt27ezdOlSDh8+zOLFi0tsWwShLIhArYJ6lZ7By7QMubSXaRm8Ss94zxIVR97eeFVVVXR0dAoc/KympoaOjk5pVU0QyqVXr16xZcuWT1p2y5YtvHz5sphr9H8CAwOpWrWqXNpvv/2GiooKfn5+GBgY4OLigru7OyEhIUBOa1t0dDR+fn506dKFpk2bsmTJEh4/fkxkZCQAmzZtQllZmRUrVtCkSRPat2/P5MmTuXz5cr5jiCBUZCJQq6Cys2HTjbuEx90n/tVrwuPus/nGvVJZt5GREdu2bcPV1RVTU1McHR25cOEC27Ztw8bGhpYtWzJ16lRSU1OBgrsn39dlmZiYiJ2dHQDDhw8nMDAwX9fnh8oxMjJix44duLu7Y2ZmRseOHVm9erXcMnv37sXR0RFTU1MGDBhAWFhYvjLyduu9myaTyQgODsbe3h4TExNatmzJ6NGji9QKFR4eTvfu3dm6dSs2NjaYm5szefJkHj16xPTp07G0tKRz587s2LFDWsbNzY3vv/+eb775BnNzczp37kxISIh0Ujpz5gzNmzfnxIkT9OzZExMTExwcHDhy5IhURnZ2NmvXrsXOzg5zc3P69OlDRESE3HYCzJo1C29v7w9uw+3btxk8eDAmJiY4Ojpy4MABaV5h9tGJEyfo168f5ubmWFtb4+3tLRcw3Lp1izFjxmBpaUnHjh35z3/+w5MnTwq9j/9N4uPj8904UFhPnz4lPj6+mGuU49y5c2zbto1FixbJpZ8/f542bdqgrPx/97O1a9eOhIQEnj59SkJCAgCtW7eW5qurq9OwYUPOnj0LQHR0NN27d6dSpUpSngEDBhAeHo6CgkKJbI8glAURqFVQmpVU0K2ixtKLN3GNOsfSizfRVVejmqpKqax/xYoVjB49mj179qChoYGnpydRUVGEhITg7+/PkSNH2L59e5HL1dXVlZYLDAzEw8OjyGUsXrwYZ2dnIiMjGTZsGIGBgZw7dw7IGfvm5eVF//79iYiIoF+/fixbtqxI5YeFhREaGoq3tzdRUVGsWbOGhISEfCejj0lKSuLgwYOEhISwatUqjh49Sq9evWjRogU7d+6kc+fO+Pj4yI0h+vXXX9HQ0CA8PJxp06axZs0a1q5dK83Pyspi6dKlzJkzh3379mFoaIiXlxevX78Gcv7ffv31V+bNm8fevXsZPnw4Pj4+bN68GUDqRp49ezZz5sz5YP03btxI37592bt3L/b29kybNo2///67UPvo+fPnTJw4ERcXF/bv38/q1as5d+4cS5YsAeDRo0e4urrSsGFDduzYQVBQECkpKQwaNEiuC03IkRu8fKrc30dxevXqFTNnzmTu3Ln5hi08fPiQOnXqyKXVqlULgAcPHsj9nSsrK4uHDx/y/PlzICc4rVWrFv7+/tjY2NC9e3eWLFlCWlpasW+LIJQlEahVYJa1qstNW+holtq6XVxcsLW1pXHjxvTp04eXL18yf/58DA0Nsbe3p1mzZty8ebPI5SopKaGtrQ2ApqYm6urqRS6jb9++9OnTBz09PTw9PalWrRoXLlwAch5P4ODgwKhRo9DX12fIkCEMGTKkSOU3aNCAxYsX07VrV+rVq4e1tTUODg7ExsYWqZzMzEzmzZuHoaEhXbp0wdjYmMaNGzNy5Ejp34yMDKl1AUBfXx8fHx8MDAxwdnbGzc2NsLAwua6eqVOnYm1tTaNGjRg/fjwpKSnExsby5s0bNmzYwOzZs7GxsaFBgwZSl1NoaCiA1I2soaGBhobGB+vv6urK4MGD0dfXZ+rUqVhYWLBhw4ZC7aNHjx6Rnp5O3bp1qVevHq1atSIoKAg3NzcgJyCtU6cOc+fOxcDAABMTE1auXMmzZ884ePBgkfbzv8Hndl2+ePGieCryDh8fHywtLenVq1e+eampqaiqqsql5baMpaWlYWpqSuPGjVmwYAGPHj0iNTWV5cuXk5ycTEZGzvCOlJQU1q5dS1paGqtXr2bGjBns3buXuXPnFvu2CEJZEs9Rq8AuPn4hN/3Xk5foVyt6YPMpGjZsKP1duXJlIOfknEtNTU0a9FvaDAwM5KY1NDSkg3tMTAxfffWV3HwrKyspwCgMW1tbLl26REBAAPHx8cTHxxMXF0ft2rULzG9paSk3nTvGBuT3WZUqVeRaHnJPXO/ux7Zt28p161haWrJ27Vq5VrfGjRtLf+eODcrIyCAuLo60tDT+85//oKj4f9domZmZpKenk5qamu+W9fnz57N3715pety4cXh6egLQqlUrubzm5uacPn26UPuoWbNm9OzZE09PT3R0dOjQoYPUKgJw9epVbt68mW/fpaWlcevWLQR5mpqfd5FWvXr14qnI/7d7927Onz8v9915V0HHh9yWsCpVqqCqqsrq1auZOXMmnTt3RkVFhV69etG1a1fpu6usrCxduACYmJiQlZXF1KlT8fb2pkaNGsW6TYJQVkSgVkG9TMvg4Zs0ZrY0xEJHk7+evOTB61RepWeUSvfnu2NLcr178v+YkrzLLO+VOvzfDQrKysrIZLIilZeZmSk3HRISwpo1a3B2dsba2hp3d3eOHj0qF4C9a/fu3XLTud06ACoq8v9XH9uHefd77ra8e6PF+7Y/dx+sXLlSLpj70HJTpkxh1KhR0vS7AUHeumZlZUllFGYfLV++nAkTJnDy5En+97//MWPGDFq1asXGjRuRyWS0a9eOBQsW5KvTx1r6/o3atGkjtYp+Cisrq2KsDezcuZNnz55hY2Mjl75gwQL2799PnTp1ePz4sdy83OncYN7AwICdO3fy4sULlJWVqVq1Kv3796ddu3YA1KlTh6ZNm8qVkTt9//59EagJXwwRqFVQCgowzFhPCsr0q6mX2zs+c4ORlJQUqYXn3e68vEpyILCxsTGXLl2SS7t48aLctIqKCikpKdL0nTt35OYHBQUxYcIExo4dK6WFhoa+906zd1sfP9eVK1fkpi9cuED9+vUL1aLSuHFjlJWVSUpKomvXrlJ6WFgYcXFx+Pn55VumRo0a7z3hxcTE0K1bN7m6GBsbAx/fR5cuXSIyMpLZs2fTuHFj3N3diYiIYMaMGTx79oymTZuyf//+/9fe/YU02YZxHP/a3Hsg5TYc2CipVXYcRFAnnQSBoVsyirYsKhP7O4jCGgW5oCCNYg3K/kBmMwpX4klF0VEFY3QmmCzdDMUDo0i0UQey90AarD9v5atry98HdjKeZ9zPw9h97brv67mw2Wzp4O/Dhw8cPXqUnTt3pidrmWS327FarVMqKLBardjt9mkdz7lz59LFRF+sX78er9eLw+Ggq6uLO3fuMDExkf6TEYlEsNvtlJSUMD4+zp49ezhx4kT6OzU0NERPTw+HDx8GJoPLLxWeX34zYrEYBoOBhQsXTuv1iPxJ2qOWp4r/MX6TOfvee7lgxYoVFBQUEAwGGRoa4uHDh3R2dv7w+KKiImDyR3dsbGxax1JXV8ejR4+4ceMGAwMD3Lt3j1Ao9M14Ozo6ePXqFT09PTQ2NmZkm2w2Gy9evKCvr494PM6FCxd4/PhxVpZ6X758ycWLFxkYGCAcDtPe3s7u3bt/6dx58+axZcsWAoEAXV1dDA4OEg6HaW5uzsjyFRUV0d/f/9MHoba2ttLZ2Uk8HufMmTPEYjHq6uqAn9+juXPncvv2bZqbm3nz5g2xWIwHDx6wePFiLBYLHo+HsbExjhw5Qm9vL729vRw6dIju7m6WL18+xbv39youLsbj8UzpXI/H87+XTr9WWlrKokWLMl4wGfiXlpbicrkYHx/n+PHj9PX1cf/+fVpbW6mvrwcmvx+pVIrTp0/z+vVruru72bt3L6tXr2bNmjUA1NbWMjg4yMmTJ0kkEjx79oyzZ8/idDrT+1xF/gYK1GTGlZWV4ff7efLkCRUVFdy9e5eGhoYfHm+xWHC5XDQ1NREIBKZ1LGvXruXUqVO0t7dTWVlJR0cHbrc7YwmysbERk8nE5s2bOXjwIJs2bcqoUGtqauLTp0+4XC5qamqIxWL4/X7evXvH8PDwtI73a+vWraO/vx+Hw0FLSws+n++3iiF8Ph/bt28nEAhQUVHBlStX8Hq97N+/P33Mrl27CIVC+Hy+//ysffv2cevWLRwOB9FolKtXr6YzMz+7R0uXLiUYDBKJRNi4cSNutxuDwcC1a9eYM2cOZWVlhEIhPn78iNvtpqamBqPRSFtbmybh7ygsLKSqqoply5b91nnl5eVUVVVlvUF7SUkJ169fJ5FIUF1dnd6PVl1dnT7m/PnzmEwm3G439fX1rFy5MqN7yZIlS2hrayMej+N0Ojl27BgbNmzA7/dn9VpEZpp6fWZJvvYY+9tEo1GsVmvGHq2WlhbC4XDG88Zy0bZt21iwYMFvPwZEZo/f6fVZXl5OMBhUr0+ZNfJ1HlZGTWaV58+fU1tbSyQSYXh4mKdPn3Lz5k2cTuefHprI/2a1Wrl8+TJer/eHAZjVasXr9XLp0iUFaSJ5QMUEMqscOHCAZDJJQ0MD79+/x2azsWPHjl/e5yWS6ywWC1u3bqWyspJEIkE0GmV0dBSz2cyqVauw2+2YTKasL3eKyNRo6TNL8jXlKiIi8jfI13lYS58iIiIiOUqBWpYpgSkiIpJ9+Tr/KlDLEqPRSEFBQbo5toiIiGRPMpkEvu0Ik+tUTJAlBoMBk8nE27dv+fz5M8XFxRQWFs7oU/hFRERmu1QqRTKZZGRkBLPZnHeFNComyKJUKsXo6CgjIyMz2utSREREMpnNZubPn593CRIFan9AKpViYmLim2bfIiIiMv2MRmPeZdK+UKAmIiIikqNUTCAiIiKSoxSoiYiIiOQoBWoiIiIiOUqBmoiIiEiOUqAmIiIikqMUqImIiIjkKAVqIiIiIjnqX3N8Sy2WZ/AkAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -1818,16 +2013,16 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "PearsonRResult(statistic=0.9563266778922446, pvalue=1.1626644107421448e-06)" + "PearsonRResult(statistic=0.9579163089066131, pvalue=9.685276711455158e-07)" ] }, - "execution_count": 106, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -1838,7 +2033,7 @@ }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -1847,7 +2042,7 @@ "SignificanceResult(statistic=0.9020979020979022, pvalue=5.997857446537695e-05)" ] }, - "execution_count": 107, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -1858,7 +2053,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -1889,204 +2084,204 @@ " Rank_v2_lite\n", " Average\n", " Rank\n", - " Model Name\n", " Embedding Size\n", + " Model Name\n", " \n", " \n", " \n", " \n", " 0\n", - " GritLM-7B\n", - " 0.632563\n", + " intfloat/e5-mistral-7b-instruct\n", + " 0.626230\n", " 1.0\n", - " 68.600860\n", - " 2.0\n", - " 66.76\n", + " 66.963823\n", " 1.0\n", - " GritLM-7B\n", - " NaN\n", + " 66.63\n", + " 2.0\n", + " 4096\n", + " e5-mistral-7b-instruct\n", " \n", " \n", " 1\n", - " e5-mistral-7b-instruct\n", - " 0.628811\n", + " GritLM/GritLM-7B\n", + " 0.625762\n", " 2.0\n", - " 69.136519\n", - " 1.0\n", - " 66.63\n", + " 66.448032\n", " 2.0\n", - " e5-mistral-7b-instruct\n", - " NaN\n", + " 66.76\n", + " 1.0\n", + " 4096\n", + " GritLM-7B\n", " \n", " \n", " 2\n", - " multilingual-e5-large-instruct\n", - " 0.614539\n", + " intfloat/multilingual-e5-large-instruct\n", + " 0.611273\n", " 3.0\n", - " 67.764793\n", + " 65.236396\n", " 3.0\n", " 64.41\n", " 3.0\n", + " 1024\n", " multilingual-e5-large-instruct\n", - " NaN\n", " \n", " \n", " 3\n", - " multilingual-e5-large\n", - " 0.574409\n", + " intfloat/multilingual-e5-large\n", + " 0.569256\n", " 4.0\n", - " 64.288174\n", + " 62.070275\n", " 4.0\n", " 60.89\n", " 4.0\n", + " 1024\n", " multilingual-e5-large\n", - " NaN\n", " \n", " \n", " 4\n", - " multilingual-e5-base\n", - " 0.556909\n", + " intfloat/multilingual-e5-base\n", + " 0.555790\n", " 5.0\n", - " 62.748222\n", + " 60.235039\n", " 5.0\n", " 59.11\n", " 5.0\n", + " 768\n", " multilingual-e5-base\n", - " NaN\n", " \n", " \n", " 5\n", - " all-mpnet-base\n", - " 0.541223\n", + " intfloat/multilingual-e5-small\n", + " 0.535567\n", " 6.0\n", - " 59.517229\n", - " 8.0\n", - " 57.78\n", + " 58.444347\n", " 6.0\n", - " all-mpnet-base\n", - " NaN\n", + " 57.04\n", + " 7.0\n", + " 384\n", + " multilingual-e5-small\n", " \n", " \n", " 6\n", - " multilingual-e5-small\n", - " 0.536997\n", + " sentence-transformers/all-mpnet-base-v2\n", + " 0.531485\n", " 7.0\n", - " 61.091356\n", + " 56.019392\n", + " 8.0\n", + " 57.78\n", " 6.0\n", - " 57.04\n", - " 7.0\n", - " multilingual-e5-small\n", - " NaN\n", + " 768\n", + " all-mpnet-base\n", " \n", " \n", " 7\n", - " multilingual-mpnet-base\n", - " 0.526547\n", + " sentence-transformers/paraphrase-multilingual-...\n", + " 0.517245\n", " 8.0\n", - " 60.471889\n", + " 57.290940\n", " 7.0\n", " 54.64\n", " 10.0\n", + " 768\n", " multilingual-mpnet-base\n", - " NaN\n", " \n", " \n", " 8\n", - " all-MiniLM-L12\n", - " 0.523151\n", + " sentence-transformers/all-MiniLM-L12-v2\n", + " 0.516339\n", " 9.0\n", - " 58.117131\n", + " 54.728697\n", " 10.0\n", " 56.53\n", " 8.0\n", + " 384\n", " all-MiniLM-L12\n", - " NaN\n", " \n", " \n", " 9\n", - " all-MiniLM-L6\n", - " 0.519588\n", + " sentence-transformers/all-MiniLM-L6-v2\n", + " 0.512155\n", " 10.0\n", - " 57.935782\n", + " 54.381772\n", " 11.0\n", " 56.10\n", " 9.0\n", + " 384\n", " all-MiniLM-L6\n", - " NaN\n", " \n", " \n", " 10\n", - " multilingual-MiniLM-L12\n", - " 0.506291\n", + " sentence-transformers/paraphrase-multilingual-...\n", + " 0.496854\n", " 11.0\n", - " 58.430607\n", + " 55.130507\n", " 9.0\n", " 52.45\n", " 11.0\n", + " 384\n", " multilingual-MiniLM-L12\n", - " NaN\n", " \n", " \n", " 11\n", - " LaBSE\n", - " 0.442359\n", + " sentence-transformers/LaBSE\n", + " 0.425548\n", " 12.0\n", - " 53.306358\n", + " 48.570700\n", " 12.0\n", " 45.21\n", " 12.0\n", + " 768\n", " LaBSE\n", - " NaN\n", " \n", " \n", "\n", "" ], "text/plain": [ - " model Average_v2_full Rank_v2_full \\\n", - "0 GritLM-7B 0.632563 1.0 \n", - "1 e5-mistral-7b-instruct 0.628811 2.0 \n", - "2 multilingual-e5-large-instruct 0.614539 3.0 \n", - "3 multilingual-e5-large 0.574409 4.0 \n", - "4 multilingual-e5-base 0.556909 5.0 \n", - "5 all-mpnet-base 0.541223 6.0 \n", - "6 multilingual-e5-small 0.536997 7.0 \n", - "7 multilingual-mpnet-base 0.526547 8.0 \n", - "8 all-MiniLM-L12 0.523151 9.0 \n", - "9 all-MiniLM-L6 0.519588 10.0 \n", - "10 multilingual-MiniLM-L12 0.506291 11.0 \n", - "11 LaBSE 0.442359 12.0 \n", + " model Average_v2_full \\\n", + "0 intfloat/e5-mistral-7b-instruct 0.626230 \n", + "1 GritLM/GritLM-7B 0.625762 \n", + "2 intfloat/multilingual-e5-large-instruct 0.611273 \n", + "3 intfloat/multilingual-e5-large 0.569256 \n", + "4 intfloat/multilingual-e5-base 0.555790 \n", + "5 intfloat/multilingual-e5-small 0.535567 \n", + "6 sentence-transformers/all-mpnet-base-v2 0.531485 \n", + "7 sentence-transformers/paraphrase-multilingual-... 0.517245 \n", + "8 sentence-transformers/all-MiniLM-L12-v2 0.516339 \n", + "9 sentence-transformers/all-MiniLM-L6-v2 0.512155 \n", + "10 sentence-transformers/paraphrase-multilingual-... 0.496854 \n", + "11 sentence-transformers/LaBSE 0.425548 \n", "\n", - " Average_v2_lite Rank_v2_lite Average Rank \\\n", - "0 68.600860 2.0 66.76 1.0 \n", - "1 69.136519 1.0 66.63 2.0 \n", - "2 67.764793 3.0 64.41 3.0 \n", - "3 64.288174 4.0 60.89 4.0 \n", - "4 62.748222 5.0 59.11 5.0 \n", - "5 59.517229 8.0 57.78 6.0 \n", - "6 61.091356 6.0 57.04 7.0 \n", - "7 60.471889 7.0 54.64 10.0 \n", - "8 58.117131 10.0 56.53 8.0 \n", - "9 57.935782 11.0 56.10 9.0 \n", - "10 58.430607 9.0 52.45 11.0 \n", - "11 53.306358 12.0 45.21 12.0 \n", + " Rank_v2_full Average_v2_lite Rank_v2_lite Average Rank \\\n", + "0 1.0 66.963823 1.0 66.63 2.0 \n", + "1 2.0 66.448032 2.0 66.76 1.0 \n", + "2 3.0 65.236396 3.0 64.41 3.0 \n", + "3 4.0 62.070275 4.0 60.89 4.0 \n", + "4 5.0 60.235039 5.0 59.11 5.0 \n", + "5 6.0 58.444347 6.0 57.04 7.0 \n", + "6 7.0 56.019392 8.0 57.78 6.0 \n", + "7 8.0 57.290940 7.0 54.64 10.0 \n", + "8 9.0 54.728697 10.0 56.53 8.0 \n", + "9 10.0 54.381772 11.0 56.10 9.0 \n", + "10 11.0 55.130507 9.0 52.45 11.0 \n", + "11 12.0 48.570700 12.0 45.21 12.0 \n", "\n", - " Model Name Embedding Size \n", - "0 GritLM-7B NaN \n", - "1 e5-mistral-7b-instruct NaN \n", - "2 multilingual-e5-large-instruct NaN \n", - "3 multilingual-e5-large NaN \n", - "4 multilingual-e5-base NaN \n", - "5 all-mpnet-base NaN \n", - "6 multilingual-e5-small NaN \n", - "7 multilingual-mpnet-base NaN \n", - "8 all-MiniLM-L12 NaN \n", - "9 all-MiniLM-L6 NaN \n", - "10 multilingual-MiniLM-L12 NaN \n", - "11 LaBSE NaN " + " Embedding Size Model Name \n", + "0 4096 e5-mistral-7b-instruct \n", + "1 4096 GritLM-7B \n", + "2 1024 multilingual-e5-large-instruct \n", + "3 1024 multilingual-e5-large \n", + "4 768 multilingual-e5-base \n", + "5 384 multilingual-e5-small \n", + "6 768 all-mpnet-base \n", + "7 768 multilingual-mpnet-base \n", + "8 384 all-MiniLM-L12 \n", + "9 384 all-MiniLM-L6 \n", + "10 384 multilingual-MiniLM-L12 \n", + "11 768 LaBSE " ] }, - "execution_count": 82, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -2097,7 +2292,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -2110,12 +2305,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# MTEB Lite Benchmarking\n" + "# MTEB(eng) Benchmarking\n" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -2131,9 +2326,17 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 30, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/au561649/.virtualenvs/mteb/lib/python3.8/site-packages/huggingface_hub/file_download.py:1150: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + }, { "name": "stdout", "output_type": "stream", @@ -2177,7 +2380,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -2187,7 +2390,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -2200,7 +2403,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -2213,6 +2416,8 @@ " res.evaluation_time for res in mteb_results[model_name][rev]\n", " )\n", "\n", + " total_co2 = sum(res.kg_co2_emissions for res in mteb_results[model_name][rev])\n", + "\n", " data.append(\n", " {\n", " \"model\": model_name,\n", @@ -2221,6 +2426,7 @@ " **weighted_mean[model_name][rev],\n", " **avg_score,\n", " \"Total Evaluation time (hours)\": total_eval_time / 3600,\n", + " \"Total CO2-eq emissions (kg)\": total_co2,\n", " }\n", " )\n", "\n", @@ -2234,7 +2440,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 41, "metadata": {}, "outputs": [ { @@ -2246,18 +2452,18 @@ " & Rank (Borda Count) & mean & mean (weighted by task type) & mean (PairClassification) & mean (Classification) & mean (STS) & mean (Retrieval) & mean (Clustering) & mean (Reranking) \\\\\n", "model & & & & & & & & & \\\\\n", "\\midrule\n", - "e5-mistral-7b-instruct & 1 (275) & 69.1 & 66.5 & 88.4 & 67.4 & 84.0 & 57.3 & 51.8 & 49.8 \\\\\n", - "GritLM-7B & 2 (265) & 68.6 & 66.1 & 87.3 & 69.5 & 82.8 & 57.2 & 50.0 & 49.6 \\\\\n", - "multilingual-e5-large-instruct & 3 (256) & 67.8 & 64.7 & 86.2 & 64.2 & 84.6 & 54.7 & 49.9 & 48.7 \\\\\n", - "multilingual-e5-large & 4 (190) & 64.3 & 61.5 & 84.7 & 65.8 & 81.5 & 49.3 & 42.7 & 44.7 \\\\\n", - "multilingual-e5-base & 5 (147) & 62.7 & 60.0 & 83.6 & 63.9 & 79.9 & 45.9 & 42.7 & 44.3 \\\\\n", - "all-mpnet-base-v2 & 6 (143) & 59.5 & 58.0 & 83.0 & 51.3 & 72.4 & 46.9 & 45.8 & 48.4 \\\\\n", - "paraphrase-multilingual-mpnet-base-v2 & 7 (130) & 60.5 & 57.9 & 81.7 & 63.7 & 80.1 & 34.2 & 42.3 & 45.2 \\\\\n", - "all-MiniLM-L12-v2 & 8 (122) & 58.1 & 56.8 & 82.5 & 52.3 & 71.1 & 43.3 & 43.8 & 47.5 \\\\\n", - "all-MiniLM-L6-v2 & 9 (106) & 57.9 & 56.6 & 82.4 & 51.5 & 70.8 & 43.0 & 44.6 & 47.1 \\\\\n", - "multilingual-e5-small & 10 (102) & 61.1 & 58.4 & 82.7 & 62.0 & 78.5 & 43.0 & 41.3 & 43.2 \\\\\n", - "paraphrase-multilingual-MiniLM-L12-v2 & 11 (76) & 58.4 & 56.0 & 80.0 & 59.0 & 77.8 & 32.8 & 41.1 & 45.4 \\\\\n", - "LaBSE & 12 (36) & 53.3 & 51.8 & 78.9 & 63.7 & 71.2 & 18.4 & 37.0 & 41.3 \\\\\n", + "e5-mistral-7b-instruct & 1 (393) & 67.0 & 67.2 & 88.4 & 75.2 & 83.6 & 54.8 & 51.4 & 49.8 \\\\\n", + "GritLM-7B & 2 (384) & 66.4 & 66.7 & 87.3 & 77.0 & 82.5 & 53.2 & 50.8 & 49.6 \\\\\n", + "multilingual-e5-large-instruct & 3 (357) & 65.2 & 65.6 & 86.2 & 73.2 & 84.3 & 51.0 & 49.9 & 48.7 \\\\\n", + "multilingual-e5-large & 4 (270) & 62.1 & 62.4 & 84.7 & 72.8 & 80.6 & 49.0 & 42.8 & 44.7 \\\\\n", + "all-mpnet-base-v2 & 5 (211) & 56.0 & 58.1 & 83.0 & 56.6 & 72.2 & 41.9 & 46.6 & 48.4 \\\\\n", + "multilingual-e5-base & 6 (211) & 60.2 & 60.9 & 83.6 & 70.0 & 79.1 & 46.1 & 42.2 & 44.3 \\\\\n", + "paraphrase-multilingual-mpnet-base-v2 & 7 (188) & 57.3 & 58.8 & 81.7 & 68.6 & 79.8 & 34.1 & 43.5 & 45.2 \\\\\n", + "all-MiniLM-L12-v2 & 8 (172) & 54.7 & 57.0 & 82.5 & 55.8 & 70.7 & 40.7 & 44.6 & 47.5 \\\\\n", + "all-MiniLM-L6-v2 & 9 (149) & 54.4 & 56.7 & 82.4 & 55.4 & 70.4 & 39.8 & 44.9 & 47.1 \\\\\n", + "multilingual-e5-small & 10 (147) & 58.4 & 59.3 & 82.7 & 67.7 & 77.6 & 43.7 & 40.8 & 43.2 \\\\\n", + "paraphrase-multilingual-MiniLM-L12-v2 & 11 (109) & 55.1 & 57.0 & 80.0 & 64.4 & 77.5 & 32.8 & 41.7 & 45.4 \\\\\n", + "LaBSE & 12 (49) & 48.6 & 51.7 & 78.9 & 66.8 & 70.2 & 16.8 & 36.1 & 41.3 \\\\\n", "\\bottomrule\n", "\\end{tabular}\n", "\n" @@ -2316,54 +2522,454 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 35, "metadata": {}, "outputs": [ { - "ename": "TypeError", - "evalue": "object of type 'NoneType' has no len()", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[31], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtasks\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_latex\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Github/mteb/mteb/overview.py:209\u001b[0m, in \u001b[0;36mMTEBTasks.to_latex\u001b[0;34m(self, properties, group_indices, include_citation_in_name, limit_n_entries)\u001b[0m\n\u001b[1;32m 207\u001b[0m _col \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 208\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m val \u001b[38;5;129;01min\u001b[39;00m df[col]:\n\u001b[0;32m--> 209\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mval\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;241m>\u001b[39m limit_n_entries:\n\u001b[1;32m 210\u001b[0m ending \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m]\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(val, \u001b[38;5;28mlist\u001b[39m) \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m}\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 211\u001b[0m str_col \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(val[:limit_n_entries])[:\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, ...\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m ending\n", - "\u001b[0;31mTypeError\u001b[0m: object of type 'NoneType' has no len()" + "name": "stdout", + "output_type": "stream", + "text": [ + "intfloat/multilingual-e5-small: 0.83 hours\n", + "sentence-transformers/LaBSE: 1.02 hours\n", + "GritLM/GritLM-7B: 3.11 hours\n", + "intfloat/multilingual-e5-large: 2.55 hours\n", + "sentence-transformers/paraphrase-multilingual-mpnet-base-v2: 1.02 hours\n", + "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2: 0.88 hours\n", + "sentence-transformers/all-mpnet-base-v2: 1.19 hours\n", + "intfloat/multilingual-e5-large-instruct: 2.03 hours\n", + "sentence-transformers/all-MiniLM-L12-v2: 0.81 hours\n", + "intfloat/multilingual-e5-base: 1.17 hours\n", + "sentence-transformers/all-MiniLM-L6-v2: 0.73 hours\n", + "intfloat/e5-mistral-7b-instruct: 2.50 hours\n" ] } ], - "source": [] + "source": [ + "for model, revision in mteb_results.items():\n", + " for rev, results in revision.items():\n", + " print(\n", + " f\"{model}: {sum(res.evaluation_time for res in results) / 3600 :.2f} hours\"\n", + " )" + ] }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 39, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:mteb.abstasks.TaskMetadata:Citation contains whitespace. Please ensure that the citation is correctly formatted.\n", + "WARNING:mteb.abstasks.TaskMetadata:Citation contains whitespace. Please ensure that the citation is correctly formatted.\n", + "WARNING:mteb.abstasks.TaskMetadata:Citation contains whitespace. Please ensure that the citation is correctly formatted.\n", + "WARNING:mteb.abstasks.TaskMetadata:Citation contains whitespace. Please ensure that the citation is correctly formatted.\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "intfloat/multilingual-e5-small: 1.09 hours\n", - "sentence-transformers/LaBSE: 1.33 hours\n", - "GritLM/GritLM-7B: 3.90 hours\n", - "intfloat/multilingual-e5-large: 3.33 hours\n", - "sentence-transformers/paraphrase-multilingual-mpnet-base-v2: 1.33 hours\n", - "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2: 1.14 hours\n", - "sentence-transformers/all-mpnet-base-v2: 1.55 hours\n", - "intfloat/multilingual-e5-large-instruct: 2.66 hours\n", - "sentence-transformers/all-MiniLM-L12-v2: 1.06 hours\n", - "intfloat/multilingual-e5-base: 1.54 hours\n", - "sentence-transformers/all-MiniLM-L6-v2: 0.95 hours\n", - "intfloat/e5-mistral-7b-instruct: 3.21 hours\n" + "\\begin{tabular}{llll}\n", + "\\toprule\n", + " & & languages & domains \\\\\n", + "type & name & & \\\\\n", + "\\midrule\n", + "Classification & AmazonCounterfactualClassification \\cite{oneill-etal-2021-wish} & ['deu', 'eng', 'jpn'] & ['Reviews', 'Written'] \\\\\n", + "\\cline{1-4}\n", + "Retrieval & ArguAna \\cite{boteva2016} & ['eng'] & ['Medical', 'Written'] \\\\\n", + "\\cline{1-4}\n", + "\\multirow[t]{2}{*}{Clustering} & ArXivHierarchicalClusteringP2P \\cite{arXiv.org e-Print archive} & ['eng'] & ['Academic', 'Written'] \\\\\n", + " & ArXivHierarchicalClusteringS2S \\cite{arXiv.org e-Print archive} & ['eng'] & ['Academic', 'Written'] \\\\\n", + "\\cline{1-4}\n", + "Reranking & AskUbuntuDupQuestions \\cite{wang-2021-TSDAE} & ['eng'] & None \\\\\n", + "\\cline{1-4}\n", + "STS & BIOSSES \\cite{10.1093/bioinformatics/btx238} & ['eng'] & None \\\\\n", + "\\cline{1-4}\n", + "Classification & Banking77Classification \\cite{casanueva-etal-2020-efficient} & ['eng'] & ['Written'] \\\\\n", + "\\cline{1-4}\n", + "Clustering & BiorxivClusteringP2P.v2 & ['eng'] & ['Academic', 'Written'] \\\\\n", + "\\cline{1-4}\n", + "\\multirow[t]{6}{*}{Retrieval} & CQADupstackGamingRetrieval \\cite{hoogeveen2015} & ['eng'] & None \\\\\n", + " & CQADupstackUnixRetrieval \\cite{hoogeveen2015} & ['eng'] & None \\\\\n", + " & ClimateFEVERHardNegatives \\cite{diggelmann2021climatefever} & ['eng'] & None \\\\\n", + " & FEVERHardNegatives \\cite{thorne-etal-2018-fever} & ['eng'] & None \\\\\n", + " & FiQA2018 \\cite{\n", + "thakur2021beir} & ['eng'] & None \\\\\n", + " & HotpotQAHardNegatives \\cite{yang-etal-2018-hotpotqa} & ['eng'] & ['Web', 'Written'] \\\\\n", + "\\cline{1-4}\n", + "\\multirow[t]{4}{*}{Classification} & ImdbClassification \\cite{maas-etal-2011-learning} & ['eng'] & ['Reviews', 'Written'] \\\\\n", + " & MTOPDomainClassification \\cite{li-etal-2021-mtop} & ['deu', 'eng', 'fra', ...] & ['Spoken', 'Spoken'] \\\\\n", + " & MassiveIntentClassification \\cite{fitzgerald2022massive} & ['afr', 'amh', 'ara', ...] & ['Spoken'] \\\\\n", + " & MassiveScenarioClassification \\cite{fitzgerald2022massive} & ['afr', 'amh', 'ara', ...] & ['Spoken'] \\\\\n", + "\\cline{1-4}\n", + "\\multirow[t]{2}{*}{Clustering} & MedrxivClusteringP2P.v2 & ['eng'] & ['Academic', 'Medical', 'Written'] \\\\\n", + " & MedrxivClusteringS2S.v2 & ['eng'] & ['Academic', 'Medical', 'Written'] \\\\\n", + "\\cline{1-4}\n", + "Reranking & MindSmallReranking \\cite{wu-etal-2020-mind} & ['eng'] & ['News', 'Written'] \\\\\n", + "\\cline{1-4}\n", + "Retrieval & SCIDOCS \\cite{specter2020cohan} & ['eng'] & ['Academic', 'Written', 'Non-fiction'] \\\\\n", + "\\cline{1-4}\n", + "\\multirow[t]{8}{*}{STS} & SICK-R \\cite{dadas-etal-2020-evaluation} & ['eng'] & None \\\\\n", + " & STS12 \\cite{10.5555/2387636.2387697} & ['eng'] & ['Encyclopaedic', 'News', 'Written'] \\\\\n", + " & STS13 \\cite{Agirre2013SEM2S} & ['eng'] & ['Web', 'News', 'Non-fiction', ...] \\\\\n", + " & STS14 \\cite{bandhakavi-etal-2014-generating} & ['eng'] & ['Blog', 'Web', 'Spoken'] \\\\\n", + " & STS15 \\cite{bicici-2015-rtm} & ['eng'] & ['Blog', 'News', 'Web', ...] \\\\\n", + " & STS17 \\cite{cer-etal-2017-semeval} & ['ara', 'deu', 'eng', ...] & ['News', 'Web', 'Written'] \\\\\n", + " & STS22.v2 \\cite{chen-etal-2022-semeval} & ['ara', 'cmn', 'deu', ...] & ['News', 'Written'] \\\\\n", + " & STSBenchmark \\cite{huggingface:dataset:stsb_multi_mt} & ['eng'] & None \\\\\n", + "\\cline{1-4}\n", + "PairClassification & SprintDuplicateQuestions \\cite{shah-etal-2018-adversarial} & ['eng'] & ['Programming', 'Written'] \\\\\n", + "\\cline{1-4}\n", + "\\multirow[t]{2}{*}{Clustering} & StackExchangeClustering.v2 \\cite{geigle:2021:arxiv} & ['eng'] & ['Web', 'Written'] \\\\\n", + " & StackExchangeClusteringP2P.v2 \\cite{geigle:2021:arxiv} & ['eng'] & ['Web', 'Written'] \\\\\n", + "\\cline{1-4}\n", + "\\multirow[t]{2}{*}{Retrieval} & TRECCOVID \\cite{roberts2021searching} & ['eng'] & None \\\\\n", + " & Touche2020 \\cite{potthast_2022_6862281} & ['eng'] & None \\\\\n", + "\\cline{1-4}\n", + "\\multirow[t]{2}{*}{Classification} & ToxicConversationsClassification \\cite{jigsaw-unintended-bias-in-toxicity-classification} & ['eng'] & ['Social', 'Written'] \\\\\n", + " & TweetSentimentExtractionClassification \\cite{tweet-sentiment-extraction} & ['eng'] & ['Social', 'Written'] \\\\\n", + "\\cline{1-4}\n", + "Clustering & TwentyNewsgroupsClustering.v2 \\cite{LANG1995331} & ['eng'] & ['News', 'Written'] \\\\\n", + "\\cline{1-4}\n", + "\\multirow[t]{2}{*}{PairClassification} & TwitterSemEval2015 \\cite{xu-etal-2015-semeval} & ['eng'] & None \\\\\n", + " & TwitterURLCorpus \\cite{lan-etal-2017-continuously} & ['eng'] & None \\\\\n", + "\\cline{1-4}\n", + "\\bottomrule\n", + "\\end{tabular}\n", + "\n" ] } ], "source": [ - "for model, revision in mteb_results.items():\n", - " for rev, results in revision.items():\n", - " print(\n", - " f\"{model}: {sum(res.evaluation_time for res in results) / 3600 :.2f} hours\"\n", - " )" + "print(tasks.to_latex(properties=[\"name\", \"type\", \"languages\", \"domains\"]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compare CO2-eq emissions" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
modelrevisionmeanmean (Clustering)mean (STS)mean (Classification)mean (Reranking)mean (Retrieval)mean (PairClassification)mean (weighted by task type)borda_countTotal Evaluation time (hours)Total CO2-eq emissions (kg)
11intfloat/e5-mistral-7b-instruct07163b72af1488142a360786df853f237b1a3ca10.6700.5140.8360.7520.4980.5480.8840.672393.02.5022.971
2GritLM/GritLM-7B13f00a0e36500c80ce12870ea513846a066004af0.6640.5080.8250.7700.4960.5320.8730.667384.03.1113.409
7intfloat/multilingual-e5-large-instructbaa7be480a7de1539afce709c8f13f833a510e0a0.6520.4990.8430.7320.4870.5100.8620.656357.02.0331.418
3intfloat/multilingual-e5-large4dc6d853a804b9c8886ede6dda8a073b7dc08a810.6210.4280.8060.7280.4470.4900.8470.624270.02.5491.563
6sentence-transformers/all-mpnet-base-v284f2bcc00d77236f9e89c8a360a00fb1139bf47d0.5600.4660.7220.5660.4840.4190.8300.581211.01.1900.688
9intfloat/multilingual-e5-based13f1b27baf31030b7fd040960d60d909913633f0.6020.4220.7910.7000.4430.4610.8360.609211.01.1700.691
4sentence-transformers/paraphrase-multilingual-...79f2382ceacceacdf38563d7c5d16b9ff8d725d60.5730.4350.7980.6860.4520.3410.8170.588188.01.0170.563
8sentence-transformers/all-MiniLM-L12-v2a05860a77cef7b37e0048a7864658139bc18a8540.5470.4460.7070.5580.4750.4070.8250.570172.00.8140.442
10sentence-transformers/all-MiniLM-L6-v28b3219a92973c328a8e22fadcfa821b5dc75636a0.5440.4490.7040.5540.4710.3980.8240.567149.00.7330.391
0intfloat/multilingual-e5-smalle4ce9877abf3edfe10b0d82785e83bdcb973e22e0.5840.4080.7760.6770.4320.4370.8270.593147.00.8330.459
5sentence-transformers/paraphrase-multilingual-...bf3bf13ab40c3157080a7ab344c831b9ad18b5eb0.5510.4170.7750.6440.4540.3280.8000.570109.00.8790.469
1sentence-transformers/LaBSEe34fab64a3011d2176c99545a93d5cbddc9a91b70.4860.3610.7020.6680.4130.1680.7890.51749.01.0200.582
\n", + "
" + ], + "text/plain": [ + " model \\\n", + "11 intfloat/e5-mistral-7b-instruct \n", + "2 GritLM/GritLM-7B \n", + "7 intfloat/multilingual-e5-large-instruct \n", + "3 intfloat/multilingual-e5-large \n", + "6 sentence-transformers/all-mpnet-base-v2 \n", + "9 intfloat/multilingual-e5-base \n", + "4 sentence-transformers/paraphrase-multilingual-... \n", + "8 sentence-transformers/all-MiniLM-L12-v2 \n", + "10 sentence-transformers/all-MiniLM-L6-v2 \n", + "0 intfloat/multilingual-e5-small \n", + "5 sentence-transformers/paraphrase-multilingual-... \n", + "1 sentence-transformers/LaBSE \n", + "\n", + " revision mean mean (Clustering) \\\n", + "11 07163b72af1488142a360786df853f237b1a3ca1 0.670 0.514 \n", + "2 13f00a0e36500c80ce12870ea513846a066004af 0.664 0.508 \n", + "7 baa7be480a7de1539afce709c8f13f833a510e0a 0.652 0.499 \n", + "3 4dc6d853a804b9c8886ede6dda8a073b7dc08a81 0.621 0.428 \n", + "6 84f2bcc00d77236f9e89c8a360a00fb1139bf47d 0.560 0.466 \n", + "9 d13f1b27baf31030b7fd040960d60d909913633f 0.602 0.422 \n", + "4 79f2382ceacceacdf38563d7c5d16b9ff8d725d6 0.573 0.435 \n", + "8 a05860a77cef7b37e0048a7864658139bc18a854 0.547 0.446 \n", + "10 8b3219a92973c328a8e22fadcfa821b5dc75636a 0.544 0.449 \n", + "0 e4ce9877abf3edfe10b0d82785e83bdcb973e22e 0.584 0.408 \n", + "5 bf3bf13ab40c3157080a7ab344c831b9ad18b5eb 0.551 0.417 \n", + "1 e34fab64a3011d2176c99545a93d5cbddc9a91b7 0.486 0.361 \n", + "\n", + " mean (STS) mean (Classification) mean (Reranking) mean (Retrieval) \\\n", + "11 0.836 0.752 0.498 0.548 \n", + "2 0.825 0.770 0.496 0.532 \n", + "7 0.843 0.732 0.487 0.510 \n", + "3 0.806 0.728 0.447 0.490 \n", + "6 0.722 0.566 0.484 0.419 \n", + "9 0.791 0.700 0.443 0.461 \n", + "4 0.798 0.686 0.452 0.341 \n", + "8 0.707 0.558 0.475 0.407 \n", + "10 0.704 0.554 0.471 0.398 \n", + "0 0.776 0.677 0.432 0.437 \n", + "5 0.775 0.644 0.454 0.328 \n", + "1 0.702 0.668 0.413 0.168 \n", + "\n", + " mean (PairClassification) mean (weighted by task type) borda_count \\\n", + "11 0.884 0.672 393.0 \n", + "2 0.873 0.667 384.0 \n", + "7 0.862 0.656 357.0 \n", + "3 0.847 0.624 270.0 \n", + "6 0.830 0.581 211.0 \n", + "9 0.836 0.609 211.0 \n", + "4 0.817 0.588 188.0 \n", + "8 0.825 0.570 172.0 \n", + "10 0.824 0.567 149.0 \n", + "0 0.827 0.593 147.0 \n", + "5 0.800 0.570 109.0 \n", + "1 0.789 0.517 49.0 \n", + "\n", + " Total Evaluation time (hours) Total CO2-eq emissions (kg) \n", + "11 2.502 2.971 \n", + "2 3.111 3.409 \n", + "7 2.033 1.418 \n", + "3 2.549 1.563 \n", + "6 1.190 0.688 \n", + "9 1.170 0.691 \n", + "4 1.017 0.563 \n", + "8 0.814 0.442 \n", + "10 0.733 0.391 \n", + "0 0.833 0.459 \n", + "5 0.879 0.469 \n", + "1 1.020 0.582 " + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# plot co2 consumption" ] }, { diff --git a/scripts/task_selection/task_selection_mult.ipynb b/scripts/task_selection/task_selection_mult.ipynb index 2abc981622..eb40e62b2b 100644 --- a/scripts/task_selection/task_selection_mult.ipynb +++ b/scripts/task_selection/task_selection_mult.ipynb @@ -3530,18 +3530,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 29, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/au561649/.virtualenvs/mteb/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "# It is possible to start the notebok from here:\n", "import pandas as pd\n", @@ -3554,9 +3545,17 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 30, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/au561649/.virtualenvs/mteb/lib/python3.8/site-packages/huggingface_hub/file_download.py:1150: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + }, { "name": "stdout", "output_type": "stream", @@ -3582,6 +3581,9 @@ " \"intfloat/multilingual-e5-small\",\n", " \"intfloat/multilingual-e5-base\",\n", " \"intfloat/multilingual-e5-large\",\n", + " # additional models\n", + " # \"Alibaba-NLP/gte-multilingual-base\",\n", + " # \"BAAI/bge-m3\",\n", " ]\n", " models: list[mteb.ModelMeta] = [mteb.get_model_meta(name) for name in model_names]\n", "\n", @@ -3600,17 +3602,32 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ "# load task results for the specified models from mteb/results repository\n", + "task_names += [\"MIRACLRetrievalHardNegatives\"]\n", "mteb_results = mteb.load_results(models=models, tasks=task_names, download_latest=False)" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "# import mteb.task_selection as task_selection\n", + "# test = task_selection.results_to_dataframe(mteb_results, drop_na=False)\n", + "\n", + "\n", + "# # columsn with NA\n", + "# test[test.columns[test.isna().any()]].loc[[\"BAAI/bge-m3\", \"Alibaba-NLP/gte-multilingual-base\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -3627,7 +3644,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -3662,7 +3679,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -3671,21 +3688,21 @@ "text": [ "\\begin{tabular}{llrrrrrrrrrr}\n", "\\toprule\n", - " & Rank (Borda Count) & mean & mean (wieghted by task type) & mean (BitextMining) & mean (PairClassification) & mean (Classification) & mean (STS) & mean (Retrieval) & mean (MultilabelClassification) & mean (Clustering) & mean (Reranking) \\\\\n", + " & Rank (Borda Count) & mean & mean (weighted by task type) & mean (BitextMining) & mean (PairClassification) & mean (Classification) & mean (STS) & mean (Retrieval) & mean (MultilabelClassification) & mean (Clustering) & mean (Reranking) \\\\\n", "model & & & & & & & & & & & \\\\\n", "\\midrule\n", - "multilingual-e5-large-instruct & 1 (1237) & 63.40 & 55.20 & 80.10 & 81.10 & 65.00 & 76.70 & 58.00 & 22.20 & 51.50 & 62.50 \\\\\n", - "GritLM-7B & 2 (1114) & 61.00 & 53.70 & 70.50 & 80.20 & 61.90 & 73.20 & 59.50 & 21.20 & 50.40 & 62.80 \\\\\n", - "e5-mistral-7b-instruct & 3 (1087) & 60.10 & 52.80 & 70.60 & 81.30 & 60.30 & 73.90 & 55.30 & 20.00 & 51.40 & 63.10 \\\\\n", - "multilingual-e5-large & 4 (972) & 58.70 & 51.40 & 71.70 & 79.30 & 59.90 & 73.40 & 54.30 & 21.30 & 43.10 & 62.60 \\\\\n", - "multilingual-e5-base & 5 (802) & 57.10 & 50.00 & 69.40 & 77.60 & 58.20 & 71.20 & 53.00 & 20.20 & 42.80 & 59.90 \\\\\n", - "paraphrase-multilingual-mpnet-base-v2 & 6 (693) & 52.20 & 45.30 & 52.10 & 81.60 & 55.10 & 69.50 & 40.00 & 16.40 & 41.20 & 53.20 \\\\\n", - "multilingual-e5-small & 7 (645) & 55.50 & 48.80 & 67.50 & 76.80 & 56.50 & 69.90 & 49.60 & 19.10 & 41.80 & 60.20 \\\\\n", - "LaBSE & 8 (586) & 52.40 & 45.90 & 76.30 & 76.10 & 54.60 & 65.20 & 33.80 & 20.10 & 39.40 & 50.40 \\\\\n", - "paraphrase-multilingual-MiniLM-L12-v2 & 9 (471) & 49.00 & 42.60 & 44.50 & 79.40 & 51.70 & 66.40 & 37.00 & 14.90 & 39.60 & 51.00 \\\\\n", - "all-mpnet-base-v2 & 10 (398) & 42.70 & 36.30 & 21.20 & 71.00 & 47.00 & 57.10 & 34.20 & 16.30 & 41.10 & 42.10 \\\\\n", - "all-MiniLM-L12-v2 & 11 (353) & 42.30 & 36.30 & 22.90 & 71.90 & 46.80 & 56.60 & 33.60 & 14.60 & 36.80 & 44.30 \\\\\n", - "all-MiniLM-L6-v2 & 12 (288) & 41.70 & 35.40 & 20.10 & 71.30 & 46.30 & 55.60 & 34.50 & 15.10 & 38.30 & 40.00 \\\\\n", + "multilingual-e5-large-instruct & 1 (1244) & 63.4 & 55.3 & 80.1 & 81.2 & 65.0 & 76.7 & 58.0 & 22.9 & 51.5 & 63.0 \\\\\n", + "GritLM-7B & 2 (1119) & 60.9 & 53.6 & 70.5 & 80.2 & 61.9 & 73.2 & 59.1 & 21.2 & 50.4 & 62.8 \\\\\n", + "e5-mistral-7b-instruct & 3 (1100) & 60.2 & 53.1 & 70.6 & 81.4 & 60.3 & 73.9 & 55.4 & 22.2 & 51.4 & 63.4 \\\\\n", + "multilingual-e5-large & 4 (980) & 58.7 & 51.5 & 71.7 & 79.3 & 59.9 & 73.4 & 55.0 & 21.3 & 43.1 & 62.6 \\\\\n", + "multilingual-e5-base & 5 (811) & 57.1 & 50.0 & 69.4 & 77.6 & 58.2 & 71.2 & 53.6 & 20.2 & 42.8 & 59.9 \\\\\n", + "paraphrase-multilingual-mpnet-base-v2 & 6 (698) & 52.0 & 45.2 & 52.1 & 81.6 & 55.1 & 69.5 & 39.3 & 16.4 & 41.2 & 53.2 \\\\\n", + "multilingual-e5-small & 7 (654) & 55.6 & 48.8 & 67.5 & 76.8 & 56.5 & 69.9 & 50.2 & 19.1 & 41.8 & 60.2 \\\\\n", + "LaBSE & 8 (589) & 52.1 & 45.8 & 76.3 & 76.1 & 54.6 & 65.2 & 32.9 & 20.1 & 39.4 & 50.4 \\\\\n", + "paraphrase-multilingual-MiniLM-L12-v2 & 9 (475) & 48.8 & 42.5 & 44.5 & 79.4 & 51.7 & 66.4 & 36.2 & 14.9 & 39.6 & 51.0 \\\\\n", + "all-mpnet-base-v2 & 10 (398) & 42.4 & 36.2 & 21.2 & 71.0 & 47.0 & 57.1 & 32.8 & 16.3 & 41.1 & 42.1 \\\\\n", + "all-MiniLM-L12-v2 & 11 (355) & 42.1 & 36.2 & 22.9 & 71.9 & 46.8 & 56.6 & 32.4 & 14.6 & 36.8 & 44.3 \\\\\n", + "all-MiniLM-L6-v2 & 12 (290) & 41.5 & 35.2 & 20.1 & 71.3 & 46.3 & 55.6 & 33.1 & 15.1 & 38.3 & 40.0 \\\\\n", "\\bottomrule\n", "\\end{tabular}\n", "\n" @@ -3708,7 +3725,7 @@ " \"mean (Clustering)\",\n", " \"mean (Reranking)\",\n", " \"mean (InstructionRetrieval)\",\n", - " \"mean (wieghted by task type)\",\n", + " \"mean (weighted by task type)\",\n", "]\n", "\n", "borda_col_name = \"borda_count\"\n", @@ -3726,7 +3743,7 @@ "cols = [\n", " \"Rank (Borda Count)\",\n", " \"mean\",\n", - " \"mean (wieghted by task type)\",\n", + " \"mean (weighted by task type)\",\n", " \"mean (BitextMining)\",\n", " \"mean (PairClassification)\",\n", " \"mean (Classification)\",\n", @@ -3739,7 +3756,7 @@ "\n", "latex_df = latex_df[cols]\n", "\n", - "table_latex = latex_df.to_latex(index=True, float_format=\"%.2f\")\n", + "table_latex = latex_df.to_latex(index=True, float_format=\"%.1f\")\n", "\n", "\n", "print(table_latex)" @@ -3747,16 +3764,373 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Rank (Borda Count)meanmean (weighted by task type)mean (BitextMining)mean (PairClassification)mean (Classification)mean (STS)mean (Retrieval)mean (MultilabelClassification)mean (Clustering)mean (Reranking)
model
multilingual-e5-large-instruct1 (1244)63.455.380.181.265.076.758.022.951.563.0
GritLM-7B2 (1119)60.953.670.580.261.973.259.121.250.462.8
e5-mistral-7b-instruct3 (1100)60.253.170.681.460.373.955.422.251.463.4
multilingual-e5-large4 (980)58.751.571.779.359.973.455.021.343.162.6
multilingual-e5-base5 (811)57.150.069.477.658.271.253.620.242.859.9
paraphrase-multilingual-mpnet-base-v26 (698)52.045.252.181.655.169.539.316.441.253.2
multilingual-e5-small7 (654)55.648.867.576.856.569.950.219.141.860.2
LaBSE8 (589)52.145.876.376.154.665.232.920.139.450.4
paraphrase-multilingual-MiniLM-L12-v29 (475)48.842.544.579.451.766.436.214.939.651.0
all-mpnet-base-v210 (398)42.436.221.271.047.057.132.816.341.142.1
all-MiniLM-L12-v211 (355)42.136.222.971.946.856.632.414.636.844.3
all-MiniLM-L6-v212 (290)41.535.220.171.346.355.633.115.138.340.0
\n", + "
" + ], + "text/plain": [ + " Rank (Borda Count) mean \\\n", + "model \n", + "multilingual-e5-large-instruct 1 (1244) 63.4 \n", + "GritLM-7B 2 (1119) 60.9 \n", + "e5-mistral-7b-instruct 3 (1100) 60.2 \n", + "multilingual-e5-large 4 (980) 58.7 \n", + "multilingual-e5-base 5 (811) 57.1 \n", + "paraphrase-multilingual-mpnet-base-v2 6 (698) 52.0 \n", + "multilingual-e5-small 7 (654) 55.6 \n", + "LaBSE 8 (589) 52.1 \n", + "paraphrase-multilingual-MiniLM-L12-v2 9 (475) 48.8 \n", + "all-mpnet-base-v2 10 (398) 42.4 \n", + "all-MiniLM-L12-v2 11 (355) 42.1 \n", + "all-MiniLM-L6-v2 12 (290) 41.5 \n", + "\n", + " mean (weighted by task type) \\\n", + "model \n", + "multilingual-e5-large-instruct 55.3 \n", + "GritLM-7B 53.6 \n", + "e5-mistral-7b-instruct 53.1 \n", + "multilingual-e5-large 51.5 \n", + "multilingual-e5-base 50.0 \n", + "paraphrase-multilingual-mpnet-base-v2 45.2 \n", + "multilingual-e5-small 48.8 \n", + "LaBSE 45.8 \n", + "paraphrase-multilingual-MiniLM-L12-v2 42.5 \n", + "all-mpnet-base-v2 36.2 \n", + "all-MiniLM-L12-v2 36.2 \n", + "all-MiniLM-L6-v2 35.2 \n", + "\n", + " mean (BitextMining) \\\n", + "model \n", + "multilingual-e5-large-instruct 80.1 \n", + "GritLM-7B 70.5 \n", + "e5-mistral-7b-instruct 70.6 \n", + "multilingual-e5-large 71.7 \n", + "multilingual-e5-base 69.4 \n", + "paraphrase-multilingual-mpnet-base-v2 52.1 \n", + "multilingual-e5-small 67.5 \n", + "LaBSE 76.3 \n", + "paraphrase-multilingual-MiniLM-L12-v2 44.5 \n", + "all-mpnet-base-v2 21.2 \n", + "all-MiniLM-L12-v2 22.9 \n", + "all-MiniLM-L6-v2 20.1 \n", + "\n", + " mean (PairClassification) \\\n", + "model \n", + "multilingual-e5-large-instruct 81.2 \n", + "GritLM-7B 80.2 \n", + "e5-mistral-7b-instruct 81.4 \n", + "multilingual-e5-large 79.3 \n", + "multilingual-e5-base 77.6 \n", + "paraphrase-multilingual-mpnet-base-v2 81.6 \n", + "multilingual-e5-small 76.8 \n", + "LaBSE 76.1 \n", + "paraphrase-multilingual-MiniLM-L12-v2 79.4 \n", + "all-mpnet-base-v2 71.0 \n", + "all-MiniLM-L12-v2 71.9 \n", + "all-MiniLM-L6-v2 71.3 \n", + "\n", + " mean (Classification) mean (STS) \\\n", + "model \n", + "multilingual-e5-large-instruct 65.0 76.7 \n", + "GritLM-7B 61.9 73.2 \n", + "e5-mistral-7b-instruct 60.3 73.9 \n", + "multilingual-e5-large 59.9 73.4 \n", + "multilingual-e5-base 58.2 71.2 \n", + "paraphrase-multilingual-mpnet-base-v2 55.1 69.5 \n", + "multilingual-e5-small 56.5 69.9 \n", + "LaBSE 54.6 65.2 \n", + "paraphrase-multilingual-MiniLM-L12-v2 51.7 66.4 \n", + "all-mpnet-base-v2 47.0 57.1 \n", + "all-MiniLM-L12-v2 46.8 56.6 \n", + "all-MiniLM-L6-v2 46.3 55.6 \n", + "\n", + " mean (Retrieval) \\\n", + "model \n", + "multilingual-e5-large-instruct 58.0 \n", + "GritLM-7B 59.1 \n", + "e5-mistral-7b-instruct 55.4 \n", + "multilingual-e5-large 55.0 \n", + "multilingual-e5-base 53.6 \n", + "paraphrase-multilingual-mpnet-base-v2 39.3 \n", + "multilingual-e5-small 50.2 \n", + "LaBSE 32.9 \n", + "paraphrase-multilingual-MiniLM-L12-v2 36.2 \n", + "all-mpnet-base-v2 32.8 \n", + "all-MiniLM-L12-v2 32.4 \n", + "all-MiniLM-L6-v2 33.1 \n", + "\n", + " mean (MultilabelClassification) \\\n", + "model \n", + "multilingual-e5-large-instruct 22.9 \n", + "GritLM-7B 21.2 \n", + "e5-mistral-7b-instruct 22.2 \n", + "multilingual-e5-large 21.3 \n", + "multilingual-e5-base 20.2 \n", + "paraphrase-multilingual-mpnet-base-v2 16.4 \n", + "multilingual-e5-small 19.1 \n", + "LaBSE 20.1 \n", + "paraphrase-multilingual-MiniLM-L12-v2 14.9 \n", + "all-mpnet-base-v2 16.3 \n", + "all-MiniLM-L12-v2 14.6 \n", + "all-MiniLM-L6-v2 15.1 \n", + "\n", + " mean (Clustering) mean (Reranking) \n", + "model \n", + "multilingual-e5-large-instruct 51.5 63.0 \n", + "GritLM-7B 50.4 62.8 \n", + "e5-mistral-7b-instruct 51.4 63.4 \n", + "multilingual-e5-large 43.1 62.6 \n", + "multilingual-e5-base 42.8 59.9 \n", + "paraphrase-multilingual-mpnet-base-v2 41.2 53.2 \n", + "multilingual-e5-small 41.8 60.2 \n", + "LaBSE 39.4 50.4 \n", + "paraphrase-multilingual-MiniLM-L12-v2 39.6 51.0 \n", + "all-mpnet-base-v2 41.1 42.1 \n", + "all-MiniLM-L12-v2 36.8 44.3 \n", + "all-MiniLM-L6-v2 38.3 40.0 " + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "latex_df" + ] + }, + { + "cell_type": "code", + "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "131" + "132" ] }, - "execution_count": 30, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -3767,6 +4141,32 @@ "sum(Counter([mteb.get_task(task).metadata.type for task in task_names]).values())" ] }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\\begin{tabular}{lllll}\n", + "\\toprule\n", + " & & languages & domains & license \\\\\n", + "type & name & & & \\\\\n", + "\\midrule\n", + "Retrieval & MIRACLRetrievalHardNegatives \\cite{10.1162/tacl_a_00595} & ['ara', 'ben', 'deu', ...] & ['Encyclopaedic', 'Written'] & cc-by-sa-4.0 \\\\\n", + "\\cline{1-5}\n", + "\\bottomrule\n", + "\\end{tabular}\n", + "\n" + ] + } + ], + "source": [ + "print(mteb.get_tasks(tasks=[\"MIRACLRetrievalHardNegatives\"]).to_latex())" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/tasks_per_language.pdf b/tasks_per_language.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6f6f267870386d17de1ff52c11af2cbbe683ecd4 GIT binary patch literal 18297 zcmb`v2Rzo__W&LuTZj_zAY?v^hbJRDTOP8q_nwatO7`BnP*!D=Rc1C(XjvgEL{@gj z|NWrvr%(C)KfV61|J&=j_kG^y-h0lu=bm%k_rCALazj#@8_L5=$WrhLlwVB<1wlae zMplGELLji(Z3j~jSi%r(Xk%{<0^cw+H+2TV00b0BM1;`P&IE%f@Jj`0dpk4;eux0p zR5r3QHAaK@k8Y*h(brYchGcD_ZVdHKrT}$Ex~IO@RgAicj2&svUr8JZ zLy&rVYVirHqO5sQZb`gheOweAK3uU;Oc}{v^{=s|E_)FxjhUF6NPc0QKt?XrMdFW@ zjKwwR9o+rCu)}a|&wanjJ3W1EEPW(xYL|1rUSRlp@z#$yu9=6+o2{J_RsC$Gm7jx# ze=ww8-nF=P@M9J2ao1xN$A1a5`i@=NSQp7=FL$N4kdJnAx_`K*;p%+2TZISKnwIg) zp(p#U^w$dT%rDoJZ5fG^%V;x9Z11gaH}u%Q`6#L3C!@qTnPlWlpy##0W}O$8Eayb? z7EGTgCy?ZBaBP8cno1CT``4uz689Vf@dAn*)vh=1qjRQ|3MG8IsEuVluuq1q zo?2?=$(pgMb4k-}H0@~p@pjNfhEN0?9OWV5HjOnJq`UGw>$0RWS&C>@AZkvmMig6` zfI*`yw*Cs)vdBUn99eB*Y|f@0PK3anTf>i2Bj!mCi`bC5b8mB;E%|Kr79Az|xiERg z7jGqHlP^P&1HBgdX0?$H)N znyoU)+&?9cc7qeHN4KZwQp~`yr#i<#*gHuo09GaWq ze)lvES0OGOFj_0o)Iu0S7P`-sv*W|> z*w3(h=fYK5tZ}C3s_R=ezW!%BhbdVn#40r{Tfkq5?OT*HSD@GH?(hp)0Ze}k_(yH1~Ovw`$rS> zomfM!Q!DMlU?5HrXk%mqam{`avEgkJMw8XmA`p)6>n4#B<3kxWGi?^7+%d>~eEA{8 zkC(@o`TO3lQeb)-hE@Ss=WHl3Qg{Gc`*?B2IVa@(t(JJ6kCJ!OA*BQ_4By`zy@g`< zoVIl{j)IQvwnx+>8XBIp5QFx5CilLvgbX!5u8!NQ>!YTW@{S>1!pGxsIgk86V~cgY z!{;=E=xBOLj3F?Fj9kS8dJk$d$U1y7TZLtbkA+V>xb@ZNtc2!GLgI|nq$5;AwKDOQ zc-XeTkc>{})DpovDNkA}s{UmCwKHDKf$TKrnPYDnCC3m37+l)+Y?2Pt_0|YP-dBU6 z@QgIEyMrxSelkxHA>*h}!bJ0|Q*dMRQvTOdX?|f1EVk18MjC!0SSs+_KMZ1s#uK}z zz7f#Qx~vOt=Z;i>(pZ!)w4^+OO%V{hdlpKE+IVQ`t#0nle0L&34)2wRAEm0*>vONr z76$zZ{H8N>)zSz_J^E*D%pGe}(*(3zF6)Zhx1X2|ii*(Ny_}k|VPsGqcv+lp{|X%3 ziI8IF6$)R%d{|$LKu2 z8&(p^MvfKMa@>U+lx6T0t`3$eWBSuqR671Xmaa1Hv<*d=?yaDt$t+9sG2PQf0gsbN zNfxSC(-$&tmI+bN8L@S^OX{#JDP5ps1Xzk1f)?5$rVtanJqlOZ(jL~f8&K6(G zhqd}^B^*CG)^GWf>{jnQ-t~g1qrQCQ)*+*`?a--{I%Z0e^7GKvMtFyt7ne?r)PU$x zW*vJ?a5RSo=hQIs-PPwzK3;-`9UtnHp-OIq zn+h=3pGxON=oAOsQmCSlD(YuC;^cgoENLI~c0+vh^1!QNdqLik`Bx8~$+sFT6ZKo= zgrnI8nkux9Md~E2QSX$a-o&d}R@PKsXMDtTZ200U^ujRb>ZN&>Qv0PNK^L=5;@o8! zizMziM@F*x?XZFL-DeT3Gjv-ywuuHhp^P#1Y~dl%z(`%bXfDsy{Mgvu!7!G%CK7zq zLJuy^`Lq|}oSv73c+87>kI9|!D;=aR>S_9BX2g{+Uxt6A^0@YyF0(;fQqT%*(l=L+ zSO|l3d9qK9IEzpTbOPHJQo}sP*Le_AU9vRc^E!S-R9wbw0?yT_dTp z>R6TEDfYH*%hlIynGnX}%0(Qr@_q2!N(|>SIniok7EJ3$a*vK6Vsw;J_qiFI0M{@% zCOWM1WrpQantmuAyT^DaR}{G*l5z;N^H$78koj3Z-zIfC>xz2oSwA4or<0eR0HEEf)H>}dwOKPK!Cc);E26y#F z>~s2j!nc=lYwwgvDEP(mo3f1x>_yOv1Siwke)0I({~H=1;e`3p-ePie&${NZHVIXu zs|O>ta7_d=b5>f4kF^QJD$VW3c9(oksdoYixG5cj?4GwEFItG)plI^Vtxk!*|T^3SvsP_ zjQM=?TLtf;*pS{=_gtTC1vlXH#^g+~W;=PTg@2zhFIK*`@8z+|;6&}VzkOTKerrZw z>A-pa9k`$2z-xc(dylu^-jALq4G-@Vun8G!jp;b8%LmafYP)4482hrU#${{W=yA3j zHhaeO?+n*PCXhA%NF!Qxn5ouMGcBkr>#v%^PO6yQ-D|J07i{07aq$?xQY%5`rtc7h zU@U(+(w?*!SuC?A|7thJdUG%U_xosDnpaCT>rXiiPcGUC3WD$*U}DccwHM z+^eZtVjIcQ;AC7OLbp-l&88y~o>^=cw9CXlyNjo{r^P_^{G!aBrIn&9>uHA>32A~` zTB^`f(G<2;5%1R>bY7bon+QI(5N%5GD+$1I_{tsc$nn(8n7FHPYud{7ntG#Oc*ngb zmobD=7qR9PjzsYkviXzs#~}x`te+J}j?KX;OGK-*_}F=gu~?s8e6@P<)1B&cO%V0C zAIa{=gwh~x2X*UHH{~8^8<1=B;C=qOUhu;zmdApu9qww=SGTB&$_-DXhcb`WZkxVs z$Zz}V3umB{8p!k>xt@)Z8m9T4MW0qti>s%vm@fV##%wi!WVSccTjpNB(3QQlnmw9$sBTA(&6rW-So5zK z7@Xt=nzHi}6r$xUHjzZBZBB_^-#6x}sNHF`uXZfCUGN;hJ{>AuB^FTdjeN5uEUH0i zr@zeHXG@oxKz(9^vv0xN4R_ryd|En zn{*9d&kWD7>BAEBXSa*t+zR4^llS8hcjOA_K2bJ|DF{6w!w-o`c7${eyz&i=KaV2i zwp@!QZF(|OX4AxcMX@7ftKyU~5l*?1eoieSt-7DGwLz~w4gK&erE&GRi)S14$n~ib z4yxWfrE@_9BW3fl;w1Bm3FN@=%a1&WC!8n}QefLpmg=Q9v}mPllOGG`ChpZ< zV-5K7=DF=F73te|1Rd4!DTo8#e4pSoCL)p(Z^=Xr(evTij~zfuPx)9BiL{Jo=S6aJ zJ?(tX8bHmk4iag&{|T?r2<5`q>5#0}PG1SKXswXbvQ6fUN>VRcWL4pE63c!9{Z7FJ zv?r;`D`UB^CU9nb)#_OhwT`rJsHJ~YHps@%he*0XUySUk!_9s-Qs)=_U^cwZ?2*1} z@yh;8Hiq{SsrYj@Ohr4GNu6_Rl;cFHEjeRRx;iXG__s9A_L|4N-N0wqTbrCxAtoPp zC>2@BmGkG`iQ3WA&KVl4*#Xgw1UBgHiIkpNF>g$5BB}}J=IWi$N~pfZS*#nAHM~68 z!V(Zpdm$;R-;L&$=A-FC4j1py*T(8d`*fvR)Rmna&3fYNJvxu14y4dDy8H$6>Fdq7 zMM?c}5U85uON5%#BX2UXp-CrBxzV;Q`UzaB&gl?q<>ude?!Hcc>jt113Bw z8aK&RQVu`LW6ojSi-JY6I~RAGuy_YEspqM064GPSBxr*y>)M=uSQ*{Bt9bKSO41#g zTNggBr(MjBMevEPJs4Eq>siF|f8YQK=m#FHF$~#iIPqv-EtLN-pR#>m7Go8T??yhi zT+h`;y=QHx7x!$le&xyy@4faGIOE3;U@f}QDi@3OdhSL_|Bd?B)zs+N)jT4W0*fZ@tBA*x8p;y8$~+ea^P5&!1gA z&P)2gkg0OdNh%G%)t<~ksSj&K8C>=Ff%b&gki-_nwzqKBgcpt;OdhDo$)8$f7GE2V z0<-R!2Zh6_^oC4rK6vw0`(H?{Z0Gt`Uz2k48mPnN#p+ugsov(c_DN~e+=Kpwp|RM{ zhP?FS-&1pw&O7P7t`(P#PARYS;a8-rECJz~RB{ARYg;%$*y>YG&mfsQKJ*7SSB^z; z@AQmXAoC)I=o3KoDIfhC)8?FZOTbmYNTE5O$gz}fC~ReEymTu|>ZK7g`*7i5Pb`_)iURp{&MOhT$PnK}Z%yIEF;!lj?QjVqq zZUI}j*cLUUt=p@D-r2QUJ;#(MmY2c|9)ndrdvw22XuNMx-|Ui5K=I=$gL7$bzn9>K z0rMbZwxnuRMquT7?b<_0Gu%sRKE~4|$^M0Ta3YwNa3pPEAvvA&OtHQqE7oO!w(FnT zYo%gK0&v+Myvf+0RJf%cjnWvnK?R(YkkW6S>HyKLTmrrHnS0a`?f zi;YUEHFD_U=Ci+s|Oj$`v9a;roiiUI;@R5pfMrl_bIPj-f#7xlK#&=p!qX>KpmpE+uDZrGAq5-mtD?aJDknVmi8~hO$8VnPQI&THo z-RRW5$;(UgjJLSl)xtLxK6LxiMO%2J;{i-P#2s%?E_{2XYID15ck7Gm#@TbH%XeNI z)vAV+NNq|*w*)#JAR4&i=|3W0tLJe}bmooiwo;NC5)tJ|QP2(#!~5ShGlRG6xaPLL zWixy~zxVP6`h#w$-Zg@;0_2kw=l$AXL;J5}7@FeX&9->!Tu{ozPjTYddtgY(V|Gh*6nojmvs6N7>i z&1x6q_8q?r%80~w&r|EGM-Ay%YZ|xKn1>sP$r&d0FJ-kbtlfSTsJ!D^z$G3sphQRb zF)5Q)Y2K*ZBxwBW*xa7!^Q~L;4aU#6{6xt5mu1OFlk9pO9Ykp&71bxFD=&bHwFI$O z?s}82-C=HHSU2qm?D25#=4FwPOw^6np*H?fb)if%#%MO>eiCC1XpG-zO-vsz{My|3 zO6b*ALE*Cs`mI7jdWK;uzSmrf-YaZs2G!fQY-QCG4=!twMWxi&S4@lWyb6PcxvrI# z-w|ur&gb6wItb6uws%>k-zYm+Us-=Qzd2LY({pY9mDPKytHX%%x+8TLbzTS_kZ9{` z77I_$jHS6~&pp!5>2F&mCtY7V7<$pei&@e?UXngu+&^5u#;j@c@goF&uJ|7=d}DyO zfYtGzh?r&m-xj|SP+-CPKNh~>e0&;^8ztqZC>}qSCj26uNXQ9$l9v#CoCAeM ze1U))8@YKRZ}%FwCA}8l_zND-9I_QCH^{ww_EXp13fw5~P@u6GDytv3{K7AZIiBtQ z+kB(-87Z<6hR*a0#3vAS!nObc@!v$j;k;^)8@+tk6lQ^}|B?(YWfxXiw?g+lE@M^I z%ZXUg2pFH=yR$TpupICtz40_IoT33s$^{c9;n>uhsTGqJPiFL%@&L z$TDT;HO~9t{nm3g9QN5tLDvSA@5vyfCwX&r_RRGVCgY^@Gm~Njb8<}<+QA8KQaECZ ztL3je2^NP(tva5a$xVJE$Zl3Lk4Ev>oeyH9?^u1hJe@UQE#9+L!pX8GGY|-OVq3Im z`|`eHB6@!`;Dxmla10+SZpA9U3-~RJKzh>J8a79%M=m|AqkWWFfZ^d%&Q6} zxO{am#X zXOfHPyZA-a_(hvrzUS8E3fR&f@y?RJT6T-K_HvXo-!x*!#}Q zCc{D%*Lq`B!-i;#elu?tOZcVCLn4k|l@uc@=b8bYmlW<7&*$x80Ygmx|Qj zO)e3X~bjw zTyHA;EPKu;X zg*^05^jDh7dGOSiD%_te$*{~%LhR3kq@V3ydK$F5p4Qg|qVO(R&)D~o?UAQh~X@pDR6IIRYkPKpf zbY*iq$XP9Wtuqq#WmX`DEj?^$W0=bod6#sa2ob?ZHM7Ox;fwIgT)4f2ebGtmKzC$| zUaDr61w>0g{ZYkeF?HHszs?m+UvY)PS^T~j4<+O4IgZC?@82jW%X)-TamW>UGF$M_ zds<(I`viGm_N0H87ccU^6-@YsV5B4?@jNmBynK0;*@0dX>USL`O{{hC1e5-89c53*qN!ec}X<<(26@N0(r#*r_jpm?cs z=*PlmSdl-P2>qRDO&8Q<*h2D{BEP~T+0!{ZAG%X>ed|EnVv7}?0*#|kA)Pr7<6m8~ zeHBl5)vWI{7sZRuCX|W-T{W_(aGH47SNg}UOm_-|I-byWh2|$NPrK#-tEb=98)VD0U;pWMzkk5LLPecDGgIsC$m_R$Nak-1X$&C^nzx zcBACZDNKr^v46gpw0S+jHiks1hnq;e-zIh&=5xmDLh^N&Nz0t}kqw*TjonjkY-vL+E3)SrP% zcq@j{nSYP=ZsuT(mPc&o;6d^Xld8%V>M4(rxNb0QPX_t2aE3L7R%zV|k}E64x&918 z!1L4a3@qb2valCVza%hHDiOQTk!oi*J-0S$p3v?Siuh#rL)y%g)t@BFMEui*d%>W2 z>}MCa;t>c+nARJdZzP>0+6Mi(v?^h~obewC(3$EjdEKAOzI=4$m*8Ig{PfPsj)i)X zK|r7Ux~98+fJky~itA#mxW~AnAX7IzVKz*Vo3p~R<*S{~&b2Po@?PiW)}>;y*Sk8Y zCrI#wkP!Jd9xtMi{3Qtg^nscf0U_a#GHV=Qvm?&JybNYY7yWUGDsM|O;3lx@QE8r$ zGyZfVO0;g=iM#U?E|~+H_fW4|YT`RL%NW_4?4PnJZXb}EvqnF?-DjM(No4$kS?B~R zPly@eynkUTrBL#Al2H8N22}?inen9P@8x-znCR96g%7Vk6ByXIcd0%$c~$a)h52%S zyz(6;iGs(wyiS*y9&gDK;JPM6wOh*{hSa0{Ix(r6ONWU&9h>9=~s< zu=fW$LDV`+cQ%dMdG1jc%+xoMo_hL(vK+UAW@r(?ZgJ{nb(m~Z)~bh{Yo<-s&ADy5 zQ*cs<)5~C%TbJtlMo%T)(Vdlw2;JipRnZdN?fBr@I125QaiIQ!7g^^Z)SBWTFVKc9 z8d^7T!DE?a;0suzDER&|flIXsxV9;M=*x_F9KKuODN;R?=ED4}r-|R@gnT#5rJSV0 z--iiYWsKE+i}am%`-on9Qis2E!j`%8g~r$-!+0uKXJh`n{DObW38QmD@C-%#jS@w- zJK_Q%>rx&z3*;W*dO`blvibeuz_oW`9v8%kjg*^8bTk|d(alj++;9@_>okjPJ22zg z51JshSsdb+ko>;v9oJTxytSzhMnzefI^e5J6y33t`J&8TSD&@VRaV0H(0U9Dj7yXU z92_%thFCm#i=x{ohL!hGgDPL|XGC#uTGLw%Y541s%MFY0Jb@{V)Zu84*Cpf&4=r7z zcfGp$C$sPq;&cALh;rbtsvE#N6fYJFybN*6E37i&d*v*t zzQN}Al0mj-TS3&4r70mal9y7kV!A%d;hbU_rtUm}o)cnLDEu#`hGHaO(;10zmTH+8 zM44=4Iy@`2zZw2Gu)c)KOBaF-I9AC9{(JjgVR zh3!W!M0EYUU|eUFs%p=w;P=a`jiMtUZrP|S!X0>MTsDy`^0=?b@^nQ@x^4#DZ%V~p zs7Q(JpV@Xzn#ak%0LDInmlHxr0r+1y_--V{JqSK-Z@TI(vwsKErXyXL`++*7505_I z;}N5PUv*>QxWTJxH06@&}+U>)+Q~e6E@QBL5-e+UuV4X=F8* zXhu#T?Sx>B_b=YcP8{~cv+6wO6}ZlHWA`OG5Hh7VCg7cVVNSw!LboTyJbVKGEfT31 zd8gALe96spU*>zeQ$lfarqCA$2$DA@2-l>id;W6Fk14oj&fSLG>0EtnQaR^_R2QVh zJtuV~x%1|1DR#b~YDD2Rgf_Sa8Z*-BlCXQL*E^00u4hlU9>nbDLu3GVT zu9KP)4iq@k*?fLw+kAmd`a$6$>1GH>nN9V2>o?Hg@)@>>I|=!m4rw+Ueew{}(|h8S z$x~sX^Ta-FuT|HkC>ACsU1xj@>~E9H44&+ zTk=QOc73B*Drc;)PwY=;ta-HiX|9gO!P}=E{|J2Z!XT-}Phn~M+Q*)qeH?mrvz-$} zc~bC$_zUfliWZ~*CW?foQU>!DGmc1)8H^pVX>UAWgL?T+6XaJz>HB`jH)Pf~sFGy_ zcd78Sx$*8C!j6w(>a@!}_3TNDc+{O2t(FPs6$^wjCs85gB{Ncw_$-=7XRlI?e#@M4 zdc62%Tf$*of4#wI1oc$DYo)i~B|3YnqZ@h>&4JGc<4Y;WJ(&JmMIdz!)OaAF%iot;vEz@ zMLpCDqZMghkvJC;@zJ5Y4HJ22JoG$i$Ewip>DGvp?C7+mL z&r5CYdo@*h`?TfE)x@G3AuW#gQIQ!9)($y6;AuA1ffGb@Lg4c^wni$v2D1z$>G{~; z^j+2s5taC+(FT&UEOyV?Muc@P5u~u&!J8%U(uuhObsc@y5+Ou7*Kb6oEo2#fsQ-F8 zV*N4d`x04v`a;1(;5-(#_(ii5NIN0gfx-XcMMv>Qr!6pgKWYg1ATMX>YUs~|2|e5M zmUu-Fn`9wBk+B+oEAf@-CNt8$vwB)0NGZW`)vot-ap`^*F}CZ$4Yjj{ETwYYI&bYt zd1b+a%>ofg))kR6xt{)7S?<=7eF5W@U9qNWSnJ(rBgUZN*>T2_{^|38{|$ zE}la;Pye1JGm3LK8I`8QJ$)hR;jE`uBbY|V8JUg3NpsAGL`gx%=b0RbdyumILc7t<(92Y z4TIVrIwT_;B6dUO@YHsdzv{#l5Tw+p&E%g(fc2i?dxH{Sc3GETPA3>i|j8&XYambftHdAh4+H|Jg+m<&4* zX?5m4&Fv-KdryDrVR7W=?O1Bd&1BnmMII#zf_d2`bT-P)daU}|<}a?&_nhh6C>kI~ z)nK0scmaRoJbvY!x^y|UD?-Lf$HqQ#PgAZpz)nXxlsWogE|bus?9Ogh!ilI{*?&`z{B9iOCk^F0Wwg3FrC*UOZqG~Lz}DBCAiHLJMI*5j1~E zm#Afden459?cdX;q^wl00iS0?QW4}DX=^lEo=Y(z7^cuyRZ+d~L9_p3b#X=T?miZm zvzPM;!*Wt6_7_GMjwrtYxz~J7qKNm@r7t+B6IwnYFoMGW#&<=@lb&V-!hqDw$d~O5 zGmfyi0Ox~m_tNYrjfDf$d|vow2=iFuq){<2(p0WgOfIq4{8~7`pC+p> zb>w`k8O;o<_s+(G5}d%z39%F$(CPo&hu}jY?Ew2WzejinbkA;hX%Hk6v)e$DJ9rMq z5siu$)K+ut3JOrGSU_)=Ck#>WPq~=}v=DUWQ+z`O$~>)AXZ)~~0xPcKGe6yd%4wR) zo^tH;z1E!bgs6SHvb1=*<5Cp8CKYprPE03;6+}XIGrU~l%MknhK9&#h*o~zH03^CG^ z%T|N$n6N7Ylvc@ zycFLtMoN<5txum7o{AX7{*rmkmg)@YL|L=m3m+=E^VYGOSh)(bK63}fD;4$q{S24B z-D^8xa88Uk|KfduTSe9hM-0EFhjY!}?8%sM5@*oVK|lVhGk7FnqG;`jN8t}aTpG30 zG5HqFV{oYN7;eyA&VF0XDN!|$tnm%L4A&KpHTnU^WP^$DuUl zRsE*#&o1Z|3#W*67U;CFP@Y%#V90DvdMyRUQ|>Ix51NtIOfrnNQ@;qOh~y2BVGPK7 zkp1Sg7)5xIbOgfb+>ab^6gu!`#8c>kGqQp?vt|JfId2j*U+ByQ~ zyozPPs;JD%Jv*{>=Vx(a?(#ElsNYR`zV2LYQZs0M&~GP-_%Jb68$9_Vym)42dlyIF z;8ySnGCV2Nv{!}hPiK;F)(-+Qs8M1)iT7soT~-*~kM<2dcH*K8eL|xE0JoV<)hELe$C6_utbRUcrlk+<+i}?L{+b z!pE#%e%W3$AvBwIpwYU|uBmy7hSluL4aVd^Iv!(H=SNtgQf-e7F6Lbe;`HMrIh7Di zv2+4SCq;RG5lQg@QQkcWv81Q+IQc}w;2B23freID2jj7kI6c&a==4?c$j3fxV(Te- zpussTb*Tvz+3o|yx?CyMp2y$v6;HFlBiCK%p5A}w(W`yMHjaAK;m)blOW#f)@Pt@T z;BN?2d_fKnI4>M<{biI)Em^yY|K6RF^56i@BEbQmJg6Sttj!6tc?1y4DK6+56E~UmJ;}BRqk?lY*+hS$Y$WY9wuj z;7crR6CQ|W53px=DL=6aT0AIbmaif5z%~vK=(4jACSJ{AanG|r)$-G0N6MB>bWfFb@l<{lm@MKMbWFb_ld*q zhc}RnjVSgLJr+2B!p<+>3E=|_`WN$uPX&ot$ywN>+84bX4t7`0yz+X#;dOYQI(?%~ zV0k=a$|@@EDdV)Nh(`)fYW1i7ZY$oI#By%ud(SIt!G*CeL(GEx8ZDEbW8D|RlIZVy zsv>cozW05O>XnjDr$s`m?8rupD}0hF8D~O3MptOK#9rr0QBvFXux0Z#0IzDJN#8~~ zRCC%AFZkeLsSD8c&-O1T2rIMpp{Zc+^ZB|i5Ku}n32jF{{y;ge&HENc@xG0`zzU~U za1Ww%>J+787q-bmx%jfF7W>k*GCYl)eXNJq)e`?4Ge8voEcPFrdOSSJ2o^VVHa#vW zs;Ee+b4r?88LGRe8rnHyjE|EuTEfE638)NKFgz~9`1lCHm^zZC&c;rb4rqHP5ESG2 zP$00XixK+pOeLldAjVQMv;|HkivQm9QGM~F(}>(q2n-71<^|Rhp!~r4ITQv&XSzV| zKo^RJXeZ0VBa1u`2xg3dzyFmmCl>+YZU84Kfy0W2=PC)oQg*=EL(DnLf6jOQ1QCKI zEzQhKfm4#0W0AK&m2_yh`BsCBcSj93V1RH_C#vrf>2#mo8fq|2x79g-C2y6ucTZ6zhAh0b6 zYzMRr0y}`9fT+Mgi-361Ag~Jv>*6L=4^~PPl|*ACKb~I=7t-F+47PwbdWK% zG`B#5AcSDepGDxbDsYb0(pb#S+{P56F{)@&TXjr9(a`;nBY+`Z*sn)fe+IL+KnMur z|BpoY|Fs=Bu)@O&Xc#X70uq1=fDmv5Ari23`~naV3~2b&QFNR zF+^aP!XyNsaG1J$0OLn(0JuQEhh^X$83yP+Ch_qj4|60CmmvT+0ty2dJnEMh=$8+e z&u}EB9xsRw$OQmlVe-QS!{8wfg@DO1{T~h*Mt}effItZ`eewZy4u$fQI81)1KbW3) zF*gv5h%jVhSi|H%Ivfv7cSnhk4`A`ICPu#h6aWH`nt%ZuW1e8B0w~8+$Ed;2;rf{a zLk6r1{n7$2KY#>$Jcc13qX+*Y6$3m}CZHuut^c)xAqoN^JbZTC01|-&{Y)4|IMM-( z9{f8U${NrMj86QL4)p<}6F<|Tt{m$GMjw7n7`hJcfi^I@@iSpKM?m<1F$8GE=)egH zDFBQKMlS$Y@@x936F+l|P=IkKJj^lWU((@NA3ew5{4Zhj0yC9{8}2r!?AxJS;3_n0q? zzXCqBI(*^+;yv!q6)6+a0ry2 zj~5E(guvJ!5H|K-0PE|A?Z42nQvH1RhW zKLil-?=Xx${{|BPK27}t2AHtlVMxq8{#RY#2Jw3z5C|MIKDf&KU2VVEHNcNh|g zi++P4;eW^!g2cFz-|_hQ{@@LX{8RQ2OlbV?y1<(O#-d`b|H%&u;RC*p_#GwyVEhJy zLXf}H0)@gbHW73E(;k!$*cAR9#{Xws81zrwg~0^=s0(~V^#^@0B=8yXZ+*aFf9Nw1 z-vHs&?|8u1&ws%9fiUrRJS4`-W3GRA&Dy8!~D?CmiI@6ct)+L_q{G1K7 list[Tensor] | ndarray | Tensor: - return torch.randn(len(sentences), 10) - - -class MockE5Wrapper(E5Wrapper): - def __init__(self, **kwargs): - self.mdl = MockSentenceTransformer() - + return torch.randn(len(sentences), 10).numpy() -class MockBGEWrapper(BGEWrapper): - def __init__(self, **kwargs): - self.mdl = MockSentenceTransformer() +class MockSentenceTransformerWrapper(SentenceTransformerWrapper): + def __init__( + self, + model: str | SentenceTransformer | CrossEncoder, + revision: str | None = None, + model_prompts: dict[str, str] | None = None, + **kwargs, + ) -> None: + """Wrapper for SentenceTransformer models. + + Args: + model: The SentenceTransformer model to use. Can be a string (model name), a SentenceTransformer model, or a CrossEncoder model. + revision: The revision of the model to use. + model_prompts: A dictionary mapping task names to prompt names. + First priority is given to the composed prompt of task name + prompt type (query or passage), then to the specific task prompt, + then to the composed prompt of task type + prompt type, then to the specific task type prompt, + and finally to the specific prompt type. + **kwargs: Additional arguments to pass to the SentenceTransformer model. + """ + if isinstance(model, str): + self.model = SentenceTransformer( + model, revision=revision, trust_remote_code=True, **kwargs + ) + else: + self.model = model + + if ( + model_prompts is None + and hasattr(self.model, "prompts") + and len(self.model.prompts) > 0 + ): + model_prompts = self.model.prompts + elif model_prompts is not None and hasattr(self.model, "prompts"): + self.model.prompts = model_prompts + self.model_prompts = model_prompts -class MockMxbaiWrapper(MxbaiWrapper): - def __init__(self, **kwargs): - self.mdl = MockSentenceTransformer() + def encode( + self, + sentences: Sequence[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + prompt_name = None + if self.model_prompts is not None: + prompt_name = get_mock_prompt_name( + self.model_prompts, task_name, prompt_type + ) + + embeddings = self.model.encode( + sentences, + prompt_name=prompt_name, + **kwargs, # sometimes in kwargs can be return_tensors=True + ) + if isinstance(embeddings, torch.Tensor): + embeddings = embeddings.cpu().detach().float().numpy() + return embeddings + + +def get_mock_prompt_name( + task_to_prompt: dict[str, str], task_name: str, prompt_type: PromptType | None +) -> str | None: + task = [ + mock_task + for mock_task in MOCK_TASK_TEST_GRID + if mock_task.metadata.name == task_name + ][0] + task_type = task.metadata.type + prompt_type_value = prompt_type.value if prompt_type else None + + if ( + task_name + and prompt_type + and f"{task_name}-{prompt_type_value}" in task_to_prompt + ): + return f"{task_name}-{prompt_type_value}" + if task_name and task_name in task_to_prompt: + return task_name + if ( + task_type + and prompt_type + and f"{task_type}-{prompt_type_value}" in task_to_prompt + ): + return f"{task_type}-{prompt_type_value}" + if task_type and task_type in task_to_prompt: + return task_type + if prompt_type and prompt_type in task_to_prompt: + return prompt_type_value + return None diff --git a/tests/test_benchmark/mock_tasks.py b/tests/test_benchmark/mock_tasks.py index eb6d6e0663..60c1a205a0 100644 --- a/tests/test_benchmark/mock_tasks.py +++ b/tests/test_benchmark/mock_tasks.py @@ -34,7 +34,7 @@ "dialect": ["Written"], "domains": [], "task_subtypes": [], - "license": "NA", + "license": "cc-by-4.0", "annotations_creators": "derived", "modalities": ["text"], "sample_creation": "found", @@ -994,8 +994,8 @@ def load_data(self, **kwargs): } self.corpus = { "test": { - "d1": {"text": "This is a positive sentence"}, - "d2": {"text": "This is another positive sentence"}, + "d1": "This is a positive sentence", + "d2": "This is another positive sentence", } } @@ -1052,8 +1052,8 @@ def load_data(self, **kwargs): self.queries = {"eng": queries, "fra": queries} corpus = { "test": { - "d1": {"text": "This is a positive sentence"}, - "d2": {"text": "This is another positive sentence"}, + "d1": "This is a positive sentence", + "d2": "This is another positive sentence", } } self.corpus = {"eng": corpus, "fra": corpus} diff --git a/tests/test_benchmark/task_grid.py b/tests/test_benchmark/task_grid.py index f1422a353c..34af684a70 100644 --- a/tests/test_benchmark/task_grid.py +++ b/tests/test_benchmark/task_grid.py @@ -131,3 +131,5 @@ def dataset_transform(self): MOCK_TASK_TEST_GRID_AS_STRING = [ t.metadata.name if isinstance(t, AbsTask) else t for t in MOCK_TASK_TEST_GRID ] + +MOCK_TASK_REGISTRY = {task.metadata.name: type(task) for task in MOCK_TASK_TEST_GRID} diff --git a/tests/test_benchmark/test_benchmark.py b/tests/test_benchmark/test_benchmark.py index 9b86a2bd94..d898517528 100644 --- a/tests/test_benchmark/test_benchmark.py +++ b/tests/test_benchmark/test_benchmark.py @@ -7,20 +7,29 @@ import numpy as np import pytest +import torch from sentence_transformers import SentenceTransformer import mteb -from mteb.benchmarks import Benchmark +import mteb.overview +from mteb.benchmarks.benchmarks import Benchmark from mteb.create_meta import generate_readme from .mock_models import ( - MockBGEWrapper, - MockE5Wrapper, - MockMxbaiWrapper, MockNumpyEncoder, + MockSentenceTransformer, + MockSentenceTransformerWrapper, MockTorchbf16Encoder, MockTorchEncoder, ) +from .mock_tasks import ( + MockInstructionRetrival, + MockMultilingualInstructionRetrival, + MockMultilingualRerankingTask, + MockMultilingualRetrievalTask, + MockRerankingTask, + MockRetrievalTask, +) from .task_grid import MOCK_TASK_TEST_GRID logging.basicConfig(level=logging.INFO) @@ -29,18 +38,14 @@ @pytest.mark.parametrize("tasks", [MOCK_TASK_TEST_GRID]) @pytest.mark.parametrize("model", [MockNumpyEncoder()]) def test_mulitple_mteb_tasks( - tasks: list[mteb.AbsTask], model: mteb.Encoder, monkeypatch + tasks: list[mteb.AbsTask], model: mteb.Encoder, tmp_path: Path ): """Test that multiple tasks can be run""" eval = mteb.MTEB(tasks=tasks) - output_folder = "tests/results" - eval.run(model, output_folder=output_folder, overwrite_results=True) + eval.run(model, output_folder=str(tmp_path), overwrite_results=True) - tasks_dict = {task.metadata.name: task for task in tasks} - monkeypatch.setattr( - mteb, "get_task", lambda task_name, **kwargs: tasks_dict[task_name] - ) - generate_readme(Path(output_folder)) + # ensure that we can generate a readme from the output folder + generate_readme(tmp_path) @pytest.mark.parametrize("task", MOCK_TASK_TEST_GRID) @@ -50,9 +55,6 @@ def test_mulitple_mteb_tasks( MockNumpyEncoder(), MockTorchEncoder(), MockTorchbf16Encoder(), - MockBGEWrapper(), - MockE5Wrapper(), - MockMxbaiWrapper(), ], ) def test_benchmark_encoders_on_task(task: str | mteb.AbsTask, model: mteb.Encoder): @@ -66,9 +68,31 @@ def test_benchmark_encoders_on_task(task: str | mteb.AbsTask, model: mteb.Encode eval.run(model, output_folder="tests/results", overwrite_results=True) +@pytest.mark.parametrize("task", MOCK_TASK_TEST_GRID[:1]) +@pytest.mark.parametrize("model", [MockNumpyEncoder()]) +def test_reload_results(task: str | mteb.AbsTask, model: mteb.Encoder, tmp_path: Path): + """Test that when rerunning the results are reloaded correctly""" + if isinstance(task, str): + tasks = mteb.get_tasks(tasks=[task]) + else: + tasks = [task] + + eval = mteb.MTEB(tasks=tasks) + results = eval.run(model, output_folder=str(tmp_path), overwrite_results=True) + + assert isinstance(results, list) + assert isinstance(results[0], mteb.TaskResult) + + # reload the results + results = eval.run(model, output_folder=str(tmp_path), overwrite_results=False) + + assert isinstance(results, list) + assert isinstance(results[0], mteb.TaskResult) + + @pytest.mark.parametrize("task_name", MOCK_TASK_TEST_GRID) def test_prompt_name_passed_to_all_encodes(task_name: str | mteb.AbsTask): - """Test that all tasks correctly pass down the task_name to the encoder which supports it, and that the encoder which does not support it does not + """Test that all tasks correctly pass down the prompt_name to the encoder which supports it, and that the encoder which does not support it does not receive it. """ _task_name = ( @@ -82,7 +106,7 @@ def encode(self, sentences, prompt_name: str | None = None, **kwargs): class EncoderWithoutInstructions(SentenceTransformer): def encode(self, sentences, **kwargs): - assert "prompt_name" not in kwargs + assert kwargs["prompt_name"] is None return super().encode(sentences, **kwargs) if isinstance(task_name, mteb.AbsTask): @@ -93,8 +117,16 @@ def encode(self, sentences, **kwargs): eval = mteb.MTEB(tasks=tasks) # Test that the task_name is passed down to the encoder - model = MockEncoderWithInstructions() - eval.run(model, output_folder="tests/results", overwrite_results=True) + model = MockSentenceTransformerWrapper( + MockEncoderWithInstructions(), + model_prompts={tasks[0].metadata.name: tasks[0].metadata.name}, + ) + + eval.run( + model, + output_folder="tests/results", + overwrite_results=True, + ) # Test that the task_name is not passed down to the encoder model = EncoderWithoutInstructions("average_word_embeddings_levy_dependency") assert model.prompts == {}, "The encoder should not have any prompts" @@ -108,7 +140,11 @@ def test_encode_kwargs_passed_to_all_encodes(task_name: str | mteb.AbsTask): class MockEncoderWithKwargs(mteb.Encoder): def encode(self, sentences, prompt_name: str | None = None, **kwargs): - assert kwargs == my_encode_kwargs + assert "no_one_uses_this_args" in kwargs + assert ( + my_encode_kwargs["no_one_uses_this_args"] + == kwargs["no_one_uses_this_args"] + ) return np.zeros((len(sentences), 10)) if isinstance(task_name, mteb.AbsTask): @@ -131,9 +167,163 @@ def encode(self, sentences, prompt_name: str | None = None, **kwargs): @pytest.mark.parametrize("model", [MockNumpyEncoder()]) def test_run_using_benchmark(model: mteb.Encoder): """Test that a benchmark object can be run using the MTEB class.""" - bench = Benchmark(name="test_bench", tasks=["STS12", "SummEval"]) + bench = Benchmark( + name="test_bench", tasks=mteb.get_tasks(tasks=["STS12", "SummEval"]) + ) + + eval = mteb.MTEB(tasks=bench) + eval.run( + model, output_folder="tests/results", overwrite_results=True + ) # we just want to test that it runs + + +@pytest.mark.parametrize("model", [MockNumpyEncoder()]) +def test_run_using_list_of_benchmark(model: mteb.Encoder): + """Test that a list of benchmark objects can be run using the MTEB class.""" + bench = [ + Benchmark(name="test_bench", tasks=mteb.get_tasks(tasks=["STS12", "SummEval"])) + ] eval = mteb.MTEB(tasks=bench) eval.run( model, output_folder="tests/results", overwrite_results=True ) # we just want to test that it runs + + +def test_benchmark_names_must_be_unique(): + import mteb.benchmarks.benchmarks as benchmark_module + + names = [ + inst.name + for nam, inst in benchmark_module.__dict__.items() + if isinstance(inst, Benchmark) + ] + assert len(names) == len(set(names)) + + +@pytest.mark.parametrize("name", ["MTEB(eng)", "MTEB(rus)", "MTEB(Scandinavian)"]) +def test_get_benchmark(name): + benchmark = mteb.get_benchmark(benchmark_name=name) + assert isinstance(benchmark, mteb.Benchmark) + + +@pytest.mark.parametrize("task", MOCK_TASK_TEST_GRID) +@pytest.mark.parametrize("is_task_name", [True, False]) +def test_prompt_name_passed_to_all_encodes_with_prompts( + task: mteb.AbsTask | str, is_task_name: bool +): + """Test that all tasks and task_types correctly pass down the prompt_name to the encoder with prompts.""" + _task_name = task.metadata.name if isinstance(task, mteb.AbsTask) else task + + if isinstance(task, mteb.AbsTask): + tasks = [task] + _task_type = task.metadata.type + else: + tasks = mteb.get_tasks(tasks=[task]) + _task_type = tasks[0].metadata.type + + to_compare = _task_name if is_task_name else _task_type + + class MockEncoderWithPrompts(mteb.Encoder): + prompts = {} + + def encode(self, sentences, prompt_name: str | None = None, **kwargs): + assert prompt_name == to_compare + return np.zeros((len(sentences), 10)) + + eval = mteb.MTEB(tasks=tasks) + + # Test that the task_name is passed down to the encoder + model = MockSentenceTransformerWrapper( + MockEncoderWithPrompts(), model_prompts={to_compare: to_compare} + ) + eval.run( + model, + output_folder="tests/results", + overwrite_results=True, + ) + + class MockEncoderWithExistingPrompts(mteb.Encoder): + prompts = {to_compare: to_compare} + + def encode(self, sentences, prompt_name: str | None = None, **kwargs): + assert prompt_name == to_compare + return np.zeros((len(sentences), 10)) + + eval = mteb.MTEB(tasks=tasks) + + # Test that the task_name is passed down to the encoder + model = MockSentenceTransformerWrapper(MockEncoderWithExistingPrompts()) + eval.run( + model, + output_folder="tests/results", + overwrite_results=True, + ) + + +@pytest.mark.parametrize( + "task", + [ + MockRerankingTask(), + MockMultilingualRerankingTask(), + MockInstructionRetrival(), + MockMultilingualInstructionRetrival(), + MockRetrievalTask(), + MockMultilingualRetrievalTask(), + ], +) +@pytest.mark.parametrize("is_task_name", [True, False]) +def test_model_query_passage_prompts_task_type( + task: mteb.AbsTask | str, is_task_name: bool +): + """Test that the model with prompts is correctly called.""" + tasks = [task] + + task_name = task.metadata.name if is_task_name else task.metadata.type + + def check_prompt(prompt_name, is_query): + prompt_type = "query" if is_query else "passage" + assert prompt_name == f"{task_name}-{prompt_type}" + + prompt_list = { + f"{task_name}-query": "query", + f"{task_name}-passage": "passage", + } + + class MockEncoderWithPrompts(mteb.Encoder): + is_query = True + + def encode(self, sentences, prompt_name: str | None = None, **kwargs): + check_prompt(prompt_name, self.is_query) + self.is_query = not self.is_query + return np.zeros((len(sentences), 10)) + + class MockSentenceEncoderWithPrompts(MockSentenceTransformer): + is_query = True + + def encode(self, sentences, prompt_name: str | None = None, *args, **kwargs): + check_prompt(prompt_name, self.is_query) + self.is_query = not self.is_query + return torch.randn(len(sentences), 10).numpy() + + eval = mteb.MTEB(tasks=tasks) + model = MockSentenceTransformerWrapper( + MockEncoderWithPrompts(), model_prompts=prompt_list + ) + + eval.run( + model, + model_prompts=prompt_list, + output_folder="tests/results", + overwrite_results=True, + ) + model = MockSentenceTransformerWrapper( + MockSentenceEncoderWithPrompts(), model_prompts=prompt_list + ) + + eval.run( + model, + model_prompts=prompt_list, + output_folder="tests/results", + overwrite_results=True, + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 1d4f8c5ded..f79624c61f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,7 @@ from __future__ import annotations import subprocess +import sys from argparse import Namespace from pathlib import Path @@ -13,7 +14,7 @@ def test_available_tasks(): - command = "mteb available_tasks" + command = f"{sys.executable} -m mteb available_tasks" result = subprocess.run(command, shell=True, capture_output=True, text=True) assert result.returncode == 0, "Command failed" assert ( @@ -21,6 +22,15 @@ def test_available_tasks(): ), "Sample task Banking77Classification task not found in available tasks" +def test_available_benchmarks(): + command = f"{sys.executable} -m mteb available_benchmarks" + result = subprocess.run(command, shell=True, capture_output=True, text=True) + assert result.returncode == 0, "Command failed" + assert ( + "MTEB(eng)" in result.stdout + ), "Sample benchmark MTEB(eng) task not found in available benchmarks" + + run_task_fixures = [ ( "average_word_embeddings_komninos", @@ -30,7 +40,7 @@ def test_available_tasks(): ( "intfloat/multilingual-e5-small", "BornholmBitextMining", - "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "fd1525a9fd15316a2d503bf26ab031a61d056e98", ), ] @@ -55,6 +65,7 @@ def test_run_task( co2_tracker=None, overwrite=True, eval_splits=None, + benchmarks=None, ) run(args) @@ -111,7 +122,7 @@ def test_create_meta(): ), f"Value for {key} does not match" # ensure that the command line interface works as well - command = f"mteb create_meta --results_folder {results} --output_path {output_path} --overwrite" + command = f"{sys.executable} -m mteb create_meta --results_folder {results} --output_path {output_path} --overwrite" result = subprocess.run(command, shell=True, capture_output=True, text=True) assert result.returncode == 0, "Command failed" @@ -172,13 +183,13 @@ def test_create_meta_from_existing(existing_readme_name: str, gold_readme_name: ), f"Value for {key} does not match" assert readme_output == gold_readme # ensure that the command line interface works as well - command = f"mteb create_meta --results_folder {results} --output_path {output_path} --from_existing {existing_readme} --overwrite" + command = f"{sys.executable} -m mteb create_meta --results_folder {results} --output_path {output_path} --from_existing {existing_readme} --overwrite" result = subprocess.run(command, shell=True, capture_output=True, text=True) assert result.returncode == 0, "Command failed" def test_save_predictions(): - command = "mteb run -m all-MiniLM-L6-v2 -t NFCorpus --output_folder tests/results --save_predictions" + command = f"{sys.executable} -m mteb run -m all-MiniLM-L6-v2 -t NFCorpus --output_folder tests/results --save_predictions" result = subprocess.run(command, shell=True, capture_output=True, text=True) assert result.returncode == 0, "Command failed" test_folder = Path(__file__).parent diff --git a/tests/test_embedding_caching.py b/tests/test_embedding_caching.py new file mode 100644 index 0000000000..77e0546440 --- /dev/null +++ b/tests/test_embedding_caching.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import shutil + +import numpy as np +import pytest + +from mteb.encoder_interface import Encoder +from mteb.models.cache_wrapper import CachedEmbeddingWrapper + + +class DummyModel(Encoder): + def __init__(self, embedding_dim=768): + self.embedding_dim = embedding_dim + self.call_count = 0 + + def encode(self, texts, **kwargs): + self.call_count += 1 + return np.random.rand(len(texts), self.embedding_dim).astype(np.float32) + + def random_other_function_returns_false(self): + return False + + +class TestCachedEmbeddingWrapper: + @pytest.fixture(scope="function") + def cache_dir(self, tmp_path): + cache_path = tmp_path / "test_cache" + yield cache_path + # Cleanup after test + if cache_path.exists(): + shutil.rmtree(cache_path) + + def test_caching_functionality(self, cache_dir): + # Create a dummy model + dummy_model = DummyModel() + + # Create the wrapper + wrapped_model = CachedEmbeddingWrapper(dummy_model, cache_dir) + + # Simulate data + queries = [ + "What is the effect of vitamin C on common cold?", + "How does exercise affect cardiovascular health?", + ] + corpus = [ + "Vitamin C supplementation does not significantly reduce the incidence of common cold.", + "Regular exercise improves cardiovascular health by strengthening the heart and reducing blood pressure.", + "The impact of vitamin C on common cold duration is minimal according to recent studies.", + ] + + # First call - should use the model to compute embeddings + query_embeddings1 = wrapped_model.encode(queries, task_name="query") + corpus_embeddings1 = wrapped_model.encode(corpus, task_name="corpus") + + assert dummy_model.call_count == 2 # One call for queries, one for corpus + + # Second call - should use cached embeddings + query_embeddings2 = wrapped_model.encode(queries) + corpus_embeddings2 = wrapped_model.encode(corpus) + + assert dummy_model.call_count == 2 # No additional calls to the model + + # Verify that the embeddings are the same + np.testing.assert_allclose(query_embeddings1, query_embeddings2) + np.testing.assert_allclose(corpus_embeddings1, corpus_embeddings2) + + # Verify that cache files were created + assert (cache_dir / "cache" / "vectors.npy").exists() + assert (cache_dir / "cache" / "index.json").exists() + + # Test with a new query - should use cache for existing queries and compute for new one + new_queries = ["What is the role of insulin in diabetes?"] + query_embeddings3 = wrapped_model.encode(new_queries) + + assert dummy_model.call_count == 3 # One additional call for the new query + assert query_embeddings3.shape == (1, dummy_model.embedding_dim) + + # try with a cached query only + _ = wrapped_model.encode(queries) + assert dummy_model.call_count == 3 + + wrapped_model.close() # delete to allow cleanup on Windows + + def test_other_functions_still_work(self, cache_dir): + # Create a dummy model + dummy_model = DummyModel() + + # Create the wrapper + wrapped_model = CachedEmbeddingWrapper(dummy_model, cache_dir) + + # Call a function that is not wrapped + result = wrapped_model.random_other_function_returns_false() + + assert result is False + assert wrapped_model.call_count == 0 + + wrapped_model.close() # delete to allow cleanup on Windows diff --git a/tests/test_encoder_interfaces.py b/tests/test_encoder_interfaces.py index 941b75dca1..546a41152e 100644 --- a/tests/test_encoder_interfaces.py +++ b/tests/test_encoder_interfaces.py @@ -2,7 +2,7 @@ from sentence_transformers import SentenceTransformer -from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from mteb.encoder_interface import Encoder from mteb.evaluation.evaluators.RetrievalEvaluator import DRESModel @@ -16,4 +16,3 @@ def test_wrapped_sentence_is_encoder_with_query_corpus_encode(): model = DRESModel(model) assert isinstance(model, Encoder) - assert isinstance(model, EncoderWithQueryCorpusEncode) diff --git a/tests/test_evaluators/test_ClusteringEvaluator.py b/tests/test_evaluators/test_ClusteringEvaluator.py index a0209857ba..78d72651bb 100644 --- a/tests/test_evaluators/test_ClusteringEvaluator.py +++ b/tests/test_evaluators/test_ClusteringEvaluator.py @@ -11,7 +11,7 @@ class Model: def encode( self, sentences: list[str], - prompt_name: str | None = None, + task_name: str | None = None, batch_size=32, ) -> np.ndarray: return np.eye(len(sentences)) diff --git a/tests/test_evaluators/test_InstructionRetrievalEvaluator.py b/tests/test_evaluators/test_InstructionRetrievalEvaluator.py index 8595378ef6..9fe1cb13c0 100644 --- a/tests/test_evaluators/test_InstructionRetrievalEvaluator.py +++ b/tests/test_evaluators/test_InstructionRetrievalEvaluator.py @@ -1,6 +1,8 @@ from __future__ import annotations +from mteb import SentenceTransformerWrapper from mteb.evaluation.evaluators import InstructionRetrievalEvaluator, utils +from tests.test_benchmark.mock_models import MockNumpyEncoder class TestInstructionRetrievalEvaluator: @@ -11,7 +13,7 @@ def setup_method(self): """ # checks that it loads self.evaluator = InstructionRetrievalEvaluator.InstructionRetrievalEvaluator( - task_name="test" + SentenceTransformerWrapper(MockNumpyEncoder()), task_name="test" ) def test_p_mrr(self): diff --git a/tests/test_evaluators/test_RetrievalEvaluator.py b/tests/test_evaluators/test_RetrievalEvaluator.py index bc5cda2a87..01a4747969 100644 --- a/tests/test_evaluators/test_RetrievalEvaluator.py +++ b/tests/test_evaluators/test_RetrievalEvaluator.py @@ -2,7 +2,9 @@ import pytest +from mteb import SentenceTransformerWrapper from mteb.evaluation.evaluators import RetrievalEvaluator +from tests.test_benchmark.mock_models import MockNumpyEncoder TOL = 0.0001 @@ -13,7 +15,9 @@ def setup_method(self): setup_method is invoked for every test method of a class. """ - self.evaluator = RetrievalEvaluator() + self.evaluator = RetrievalEvaluator( + SentenceTransformerWrapper(MockNumpyEncoder()), + ) @pytest.mark.parametrize( "relevant_docs, results, ignore_identical_ids, expected_metrics", diff --git a/tests/test_load_results/test_mteb_load_results.py b/tests/test_load_results/test_mteb_load_results.py index d5d2ec87ef..57ba1bae54 100644 --- a/tests/test_load_results/test_mteb_load_results.py +++ b/tests/test_load_results/test_mteb_load_results.py @@ -4,6 +4,7 @@ from pathlib import Path import mteb +from mteb.load_results.benchmark_results import BenchmarkResults, ModelResult def test_mteb_load_results(): @@ -13,15 +14,15 @@ def test_mteb_load_results(): results = mteb.load_results(download_latest=False) - assert isinstance(results, dict) - for model in results: - assert isinstance(results[model], dict) - for revision in results[model]: - assert isinstance(results[model][revision], list) - for result in results[model][revision]: - assert isinstance(result, mteb.MTEBResults) + assert isinstance(results, BenchmarkResults) + for model_result in results: + assert isinstance(model_result, ModelResult) + for res in model_result: + assert isinstance(res, mteb.TaskResult) known_model = "sentence-transformers/average_word_embeddings_levy_dependency" known_revision = "6d9c09a789ad5dd126b476323fccfeeafcd90509" - assert known_model in results - assert known_revision in results[known_model] + assert known_model in [res.model_name for res in results] + assert known_revision in [ + res.model_revision for res in results if res.model_name == known_model + ] diff --git a/tests/test_load_results/test_mteb_results.py b/tests/test_load_results/test_mteb_results.py index 4007da270f..6c22b390f3 100644 --- a/tests/test_load_results/test_mteb_results.py +++ b/tests/test_load_results/test_mteb_results.py @@ -7,7 +7,7 @@ import mteb from mteb import AbsTask -from mteb.load_results.mteb_results import MTEBResults +from mteb.load_results.task_results import TaskResult tests_folder = Path(__file__).parent.parent @@ -52,7 +52,7 @@ def _calculate_metrics_from_split( def test_mteb_results(): - """Test MTEBResults class (this is the same as the example in the docstring)""" + """Test TaskResult class (this is the same as the example in the docstring)""" scores = { "train": { "en-de": { @@ -66,7 +66,7 @@ def test_mteb_results(): evaluation_time = 100 - mteb_results = MTEBResults.from_task_results( + mteb_results = TaskResult.from_task_results( task=DummyTask(), scores=scores, evaluation_time=evaluation_time ) @@ -101,5 +101,5 @@ def test_mteb_results(): "path", list((tests_folder / "historic_results").glob("*.json")) ) def test_mteb_results_from_historic(path: Path): - mteb_result = MTEBResults.from_disk(path, load_historic_data=True) - assert isinstance(mteb_result, MTEBResults) + mteb_result = TaskResult.from_disk(path, load_historic_data=True) + assert isinstance(mteb_result, TaskResult) diff --git a/tests/test_overview.py b/tests/test_overview.py index 7103e2dfa2..73df5dc193 100644 --- a/tests/test_overview.py +++ b/tests/test_overview.py @@ -3,7 +3,8 @@ import pytest import mteb -from mteb import get_tasks +from mteb import get_task, get_tasks +from mteb.abstasks.AbsTask import AbsTask from mteb.abstasks.TaskMetadata import TASK_DOMAIN, TASK_TYPE from mteb.overview import MTEBTasks @@ -19,12 +20,25 @@ def test_get_tasks_size_differences(): ) +@pytest.mark.parametrize("task_name", ["BornholmBitextMining"]) +@pytest.mark.parametrize("eval_splits", [["test"], None]) +def test_get_task(task_name: str, eval_splits: list[str] | None): + task = get_task(task_name, eval_splits=eval_splits) + assert isinstance(task, AbsTask) + assert task.metadata.name == task_name + if eval_splits: + for split in task.eval_splits: + assert split in eval_splits + else: + assert task.eval_splits == task.metadata.eval_splits + + @pytest.mark.parametrize("languages", [["eng", "deu"], ["eng"], None]) @pytest.mark.parametrize("script", [["Latn"], ["Cyrl"], None]) @pytest.mark.parametrize("domains", [["Legal"], ["Medical", "Non-fiction"], None]) @pytest.mark.parametrize("task_types", [["Classification"], ["Clustering"], None]) @pytest.mark.parametrize("exclude_superseeded_datasets", [True, False]) -def test_get_task( +def test_get_tasks( languages: list[str], script: list[str], domains: list[TASK_DOMAIN], diff --git a/tests/test_reproducible_workflow.py b/tests/test_reproducible_workflow.py index 7c0cd84ab2..afe9f98c8d 100644 --- a/tests/test_reproducible_workflow.py +++ b/tests/test_reproducible_workflow.py @@ -6,8 +6,10 @@ import mteb from mteb import MTEB -from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode +from mteb.encoder_interface import Encoder from mteb.model_meta import ModelMeta +from mteb.models.sentence_transformer_wrapper import validate_task_to_prompt_name +from tests.test_benchmark.task_grid import TASK_TEST_GRID logging.basicConfig(level=logging.INFO) @@ -24,7 +26,50 @@ def test_reproducibility_workflow(task_name: str, model_name: str, model_revisio assert isinstance(task, mteb.AbsTask) model = mteb.get_model(model_name, revision=model_revision) - assert isinstance(model, (Encoder, EncoderWithQueryCorpusEncode)) + assert isinstance(model, Encoder) eval = MTEB(tasks=[task]) eval.run(model, output_folder="tests/results", overwrite_results=True) + + +@pytest.mark.parametrize( + "task_name", + TASK_TEST_GRID + + [ + "BitextMining", + "Classification", + "MultilabelClassification", + "Clustering", + "PairClassification", + "Reranking", + "Retrieval", + "STS", + "Summarization", + "InstructionRetrieval", + "Speed", + ], +) +def test_validate_task_to_prompt_name(task_name: str | mteb.AbsTask): + if isinstance(task_name, mteb.AbsTask): + task_names = [task_name.metadata.name] + else: + task_names = [task_name] + + model_prompts = {task_name: "prompt_name" for task_name in task_names} + model_prompts |= {task_name + "-query": "prompt_name" for task_name in task_names} + model_prompts |= {task_name + "-passage": "prompt_name" for task_name in task_names} + model_prompts |= { + "query": "prompt_name", + "passage": "prompt_name", + } + validate_task_to_prompt_name(model_prompts) + + +def test_validate_task_to_prompt_name_fail(): + with pytest.raises(KeyError): + validate_task_to_prompt_name( + {"task_name": "prompt_name", "task_name-query": "prompt_name"} + ) + + with pytest.raises(ValueError): + validate_task_to_prompt_name({"task_name-task_name": "prompt_name"}) diff --git a/tests/test_task_aggregation.py b/tests/test_task_aggregation.py index 23228872c6..f0754418c3 100644 --- a/tests/test_task_aggregation.py +++ b/tests/test_task_aggregation.py @@ -2,9 +2,10 @@ import mteb import mteb.task_aggregation as task_aggregation +from mteb.load_results.benchmark_results import BenchmarkResults # define some test data -bitext1_1 = mteb.MTEBResults( +bitext1_1 = mteb.TaskResult( dataset_revision="test_rev", task_name="BornholmBitextMining", mteb_version="test_version", @@ -12,7 +13,7 @@ scores={"test": [{"main_score": 1, "hf_subset": "NaN", "languages": ["eng-Latn"]}]}, ) -bitext1_2 = mteb.MTEBResults( +bitext1_2 = mteb.TaskResult( dataset_revision="test_rev", task_name="BornholmBitextMining", mteb_version="test_version", @@ -20,7 +21,7 @@ scores={"test": [{"main_score": 2, "hf_subset": "NaN", "languages": ["eng-Latn"]}]}, ) -classification1_1 = mteb.MTEBResults( +classification1_1 = mteb.TaskResult( dataset_revision="test_rev", task_name="Banking77Classification", mteb_version="test_version", @@ -28,7 +29,7 @@ scores={"test": [{"main_score": 1, "hf_subset": "NaN", "languages": ["eng-Latn"]}]}, ) -classification1_2 = mteb.MTEBResults( +classification1_2 = mteb.TaskResult( dataset_revision="test_rev", task_name="Banking77Classification", mteb_version="test_version", @@ -36,7 +37,7 @@ scores={"test": [{"main_score": 2, "hf_subset": "NaN", "languages": ["eng-Latn"]}]}, ) -classification2_1 = mteb.MTEBResults( +classification2_1 = mteb.TaskResult( dataset_revision="test_rev", task_name="AfriSentiClassification", mteb_version="test_version", @@ -54,6 +55,7 @@ "rev2": [bitext1_2, classification1_1, classification2_1], }, } +mteb_results = BenchmarkResults.from_legacy_dict(mteb_results) def test_mean(): @@ -103,14 +105,16 @@ def test_task_category_weighted_mean(): def test_borda_count_simple(): - mteb_results_simple = { - "model1": { - "rev1": [bitext1_1], - }, - "model2": { - "rev2": [bitext1_2], - }, - } + mteb_results_simple = BenchmarkResults.from_legacy_dict( + { + "model1": { + "rev1": [bitext1_1], + }, + "model2": { + "rev2": [bitext1_2], + }, + } + ) expected = { "model1": { "rev1": {"borda_count": 0}, @@ -143,6 +147,9 @@ def test_borda_count_simple_with_tie(): "rev2": {"borda_count": 2.5}, }, } + mteb_results_simple_with_tie = BenchmarkResults.from_legacy_dict( + mteb_results_simple_with_tie + ) assert task_aggregation.borda_count(mteb_results_simple_with_tie) == expected diff --git a/tests/test_tasks/test_mteb_rerank.py b/tests/test_tasks/test_mteb_rerank.py index 6920769694..c540bb41ee 100644 --- a/tests/test_tasks/test_mteb_rerank.py +++ b/tests/test_tasks/test_mteb_rerank.py @@ -2,16 +2,17 @@ import json import logging -import os +from pathlib import Path from sentence_transformers import CrossEncoder, SentenceTransformer from mteb import MTEB +from mteb.model_meta import ModelMeta logging.basicConfig(level=logging.INFO) -def test_mteb_rerank(): +def test_mteb_rerank(tmp_path: Path): # Test that reranking works # unfortunately, we need all the query ids to pretend to have this scifact_keys = [ @@ -323,7 +324,8 @@ def test_mteb_rerank(): ] ) # create fake first stage results - with open("tmp.json", "w") as f: + tmp_file = tmp_path / "tmp.json" + with open(tmp_file, "w") as f: f.write( json.dumps( { @@ -344,10 +346,10 @@ def test_mteb_rerank(): overwrite_results=True, eval_splits=["test"], top_k=2, - previous_results="tmp.json", + previous_results=tmp_file, save_predictions=True, ) - os.remove("tmp.json") + tmp_file.unlink() # read in the results with open("tests/results/SciFact_default_predictions.json") as f: @@ -364,6 +366,14 @@ def test_reranker_same_ndcg1(): revision = "21eec43590414cb8e3a6f654857abed0483ae36e" de = SentenceTransformer(de_name, revision=revision) ce = CrossEncoder("cross-encoder/ms-marco-TinyBERT-L-2-v2") + ce_revision = "e9ea2688951463fc2791a2ea2ddfce6762900675" + ce.mteb_model_meta = ModelMeta( + name="cross-encoder/ms-marco-TinyBERT-L-2-v2", + languages=["eng-Latn"], + open_weights=True, + revision=ce_revision, + release_date="2021-04-15", + ) eval = MTEB(tasks=["SciFact"]) eval.run( de, @@ -389,7 +399,7 @@ def test_reranker_same_ndcg1(): stage1 = json.load(f) with open( - "tests/results/stage2/no_model_name_available/no_revision_available/SciFact.json" + f"tests/results/stage2/cross-encoder__ms-marco-TinyBERT-L-2-v2/{ce_revision}/SciFact.json" ) as f: stage2 = json.load(f) From 6979b2ad9931bddc5bf1a735a0b588de3f14fab3 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 28 Oct 2024 14:54:10 +0200 Subject: [PATCH 084/154] [mieb] Add OpenCLIP models (#1335) * add open clip models * Update __init__.py * lint * fix model overview * update jina clip --------- Co-authored-by: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Co-authored-by: gowitheflow-1998 Co-authored-by: gowitheflow-1998 --- mteb/models/datacomp_clip.py | 151 ------------ mteb/models/jina_clip.py | 156 +----------- mteb/models/openclip_models.py | 233 ++++++++++++++++++ mteb/models/overview.py | 27 +- .../MNISTZeroShot.json | 19 ++ .../model_meta.json | 1 + .../MNIST.json | 48 ++++ .../model_meta.json | 1 + .../BLINKIT2IRetrieval.json | 186 ++++++++++++++ .../Caltech101.json | 48 ---- .../MNIST.json | 28 +++ .../MNISTZeroShot.json | 19 ++ .../STS14.json | 26 ++ .../model_meta.json | 2 +- .../MNISTZeroShot.json | 19 ++ .../model_meta.json | 1 + .../MNIST.json | 48 ++++ .../model_meta.json | 1 + 18 files changed, 659 insertions(+), 355 deletions(-) delete mode 100644 mteb/models/datacomp_clip.py create mode 100644 mteb/models/openclip_models.py create mode 100644 results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/MNISTZeroShot.json create mode 100644 results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/model_meta.json create mode 100644 results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/MNIST.json create mode 100644 results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json create mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/BLINKIT2IRetrieval.json delete mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/Caltech101.json create mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNIST.json create mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNISTZeroShot.json create mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/STS14.json create mode 100644 results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/MNISTZeroShot.json create mode 100644 results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/model_meta.json create mode 100644 results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/MNIST.json create mode 100644 results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json diff --git a/mteb/models/datacomp_clip.py b/mteb/models/datacomp_clip.py deleted file mode 100644 index cca9bf5c1a..0000000000 --- a/mteb/models/datacomp_clip.py +++ /dev/null @@ -1,151 +0,0 @@ -from __future__ import annotations - -from functools import partial -from typing import Any - -import torch -from PIL import Image -from torch.utils.data import DataLoader -from tqdm import tqdm -from transformers import AutoModel, AutoProcessor - -from mteb.model_meta import ModelMeta - - -class DataCLIPModelWrapper: - def __init__( - self, - model_name: str = "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", - device: str = "cuda" if torch.cuda.is_available() else "cpu", - **kwargs: Any, - ): - self.model_name = model_name - self.device = device - self.model = AutoModel.from_pretrained(model_name, trust_remote_code=True).to( - self.device - ) - self.processor = AutoProcessor.from_pretrained(model_name) - - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): - all_text_embeddings = [] - - with torch.no_grad(): - for i in tqdm(range(0, len(texts), batch_size)): - batch_texts = texts[i : i + batch_size] - inputs = self.processor( - text=batch_texts, return_tensors="pt", padding=True, truncation=True - ) - inputs = {k: v.to(self.device) for k, v in inputs.items()} - text_outputs = self.model.get_text_features(**inputs) - all_text_embeddings.append(text_outputs.cpu()) - - all_text_embeddings = torch.cat(all_text_embeddings, dim=0) - return all_text_embeddings - - def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 - ): - all_image_embeddings = [] - if isinstance(images, DataLoader): - with torch.no_grad(): - for batch in tqdm(images): - inputs = self.processor( - images=batch, return_tensors="pt", padding=True - ) - inputs = {k: v.to(self.device) for k, v in inputs.items()} - image_outputs = self.model.get_image_features(**inputs) - all_image_embeddings.append(image_outputs.cpu()) - else: - with torch.no_grad(): - for i in tqdm(range(0, len(images), batch_size)): - batch_images = images[i : i + batch_size] - inputs = self.processor( - images=batch_images, return_tensors="pt", padding=True - ) - inputs = {k: v.to(self.device) for k, v in inputs.items()} - image_outputs = self.model.get_image_features(**inputs) - all_image_embeddings.append(image_outputs.cpu()) - - all_image_embeddings = torch.cat(all_image_embeddings, dim=0) - return all_image_embeddings - - def calculate_probs(self, text_embeddings, image_embeddings): - text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) - image_embeddings = image_embeddings / image_embeddings.norm( - dim=-1, keepdim=True - ) - logits = torch.matmul(image_embeddings, text_embeddings.T) - probs = (logits * 100).softmax(dim=-1) - return probs - - def get_fused_embeddings( - self, - texts: list[str] = None, - images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", - batch_size: int = 32, - ): - if texts is None and images is None: - raise ValueError("Either texts or images must be provided") - - text_embeddings = None - image_embeddings = None - - if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) - - if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) - - if text_embeddings is not None and image_embeddings is not None: - if len(text_embeddings) != len(image_embeddings): - raise ValueError( - "The number of texts and images must have the same length" - ) - if fusion_mode == "sum": - fused_embeddings = text_embeddings + image_embeddings - else: - # to do: add other fusion mode - raise ValueError(f"fusion mode {fusion_mode} hasn't been implemented") - return fused_embeddings - elif text_embeddings is not None: - return text_embeddings - elif image_embeddings is not None: - return image_embeddings - - -datacomp_clip_vit_large_patch14 = ModelMeta( - loader=partial( - DataCLIPModelWrapper, - model_name="laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", - ), - name="laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", - languages=["eng_Latn"], - open_source=True, - revision="84c9828e63dc9a9351d1fe637c346d4c1c4db341", - release_date="2023-04-26", -) - -datacomp_clip_vit_base_patch32 = ModelMeta( - loader=partial( - DataCLIPModelWrapper, - model_name="laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", - ), - name="laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", - languages=["eng_Latn"], - open_source=True, - revision="f0e2ffa09cbadab3db6a261ec1ec56407ce42912", - release_date="2023-04-26", -) - -datacomp_clip_vit_base_patch16 = ModelMeta( - loader=partial( - DataCLIPModelWrapper, - model_name="laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", - ), - name="laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", - languages=["eng_Latn"], - open_source=True, - revision="d110532e8d4ff91c574ee60a342323f28468b287", - release_date="2023-04-26", -) diff --git a/mteb/models/jina_clip.py b/mteb/models/jina_clip.py index 969fd7ea9d..eec407dab7 100644 --- a/mteb/models/jina_clip.py +++ b/mteb/models/jina_clip.py @@ -1,164 +1,12 @@ from __future__ import annotations from functools import partial -from typing import Any - -import torch -from PIL import Image -from torch.utils.data import DataLoader -from tqdm import tqdm -from transformers import AutoModel - -from mteb.model_meta import ModelMeta -from mteb.models.text_formatting_utils import corpus_to_texts - - -class JinaCLIPModelWrapper: - def __init__( - self, - model_name: str, - device: str = "cuda" if torch.cuda.is_available() else "cpu", - **kwargs: Any, - ): - self.model_name = model_name - self.device = device - self.model = AutoModel.from_pretrained(model_name, trust_remote_code=True).to( - self.device - ) - - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): - all_text_embeddings = [] - - with torch.no_grad(): - for i in tqdm(range(0, len(texts), batch_size)): - batch_texts = texts[i : i + batch_size] - text_outputs = self.model.encode_text( - batch_texts, convert_to_numpy=False, convert_to_tensor=True - ) - all_text_embeddings.append(text_outputs.cpu()) - - all_text_embeddings = torch.cat(all_text_embeddings, dim=0) - return all_text_embeddings - - def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 - ): - all_image_embeddings = [] - - if isinstance(images, DataLoader): - with torch.no_grad(): - for batch in tqdm(images): - image_outputs = self.model.encode_image( - batch, convert_to_numpy=False, convert_to_tensor=True - ) - all_image_embeddings.append(image_outputs.cpu()) - else: - with torch.no_grad(): - for i in tqdm(range(0, len(images), batch_size)): - batch_images = images[i : i + batch_size] - image_outputs = self.model.encode_image( - batch_images, convert_to_numpy=False, convert_to_tensor=True - ) - all_image_embeddings.append(image_outputs.cpu()) - - all_image_embeddings = torch.cat(all_image_embeddings, dim=0) - return all_image_embeddings - - def calculate_probs(self, text_embeddings, image_embeddings): - text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) - image_embeddings = image_embeddings / image_embeddings.norm( - dim=-1, keepdim=True - ) - logits = torch.matmul(image_embeddings, text_embeddings.T) - probs = (logits * 100).softmax(dim=-1) - return probs - - def get_fused_embeddings( - self, - texts: list[str] = None, - images: list[Image.Image] = None, - fusion_mode="sum", - batch_size: int = 32, - ): - if texts is None and images is None: - raise ValueError("Either texts or images must be provided") - - text_embeddings = None - image_embeddings = None - - if texts is not None: - text_embeddings = self.encode_text( - texts, - batch_size=batch_size, - convert_to_numpy=False, - convert_to_tensor=True, - ) - - if images is not None: - image_embeddings = self.encode_image( - images, - batch_size=batch_size, - convert_to_numpy=False, - convert_to_tensor=True, - ) - - if text_embeddings is not None and image_embeddings is not None: - if len(text_embeddings) != len(image_embeddings): - raise ValueError( - "The number of texts and images must have the same length" - ) - if fusion_mode == "sum": - fused_embeddings = text_embeddings + image_embeddings - else: - # to do: add other fusion mode - raise ValueError(f"fusion mode {fusion_mode} hasn't been implemented") - return fused_embeddings - elif text_embeddings is not None: - return text_embeddings - elif image_embeddings is not None: - return image_embeddings - - def encode( # type: ignore - self, - sentences: list[str], - *, - batch_size: int = 32, - **kwargs: Any, - ): - if "prompt_name" in kwargs: - kwargs.pop("prompt_name") - return self.model.encode_text(sentences, batch_size=batch_size, **kwargs) - - def encode_queries(self, queries: list[str], batch_size: int = 32, **kwargs: Any): - if "prompt_name" in kwargs: - kwargs.pop("prompt_name") - sentences = [ - "Represent this sentence for searching relevant passages: " + sentence - for sentence in queries - ] - emb = self.encode( - sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs - ) - return emb - - def encode_corpus( - self, - corpus: list[dict[str, str]] | dict[str, list[str]], - batch_size: int = 32, - **kwargs: Any, - ): - if "prompt_name" in kwargs: - kwargs.pop("prompt_name") - sentences = corpus_to_texts(corpus) - emb = self.encode( - sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs - ) - return emb +from mteb.model_meta import ModelMeta, sentence_transformers_loader jina_clip_v1 = ModelMeta( loader=partial( - JinaCLIPModelWrapper, + sentence_transformers_loader, model_name="jinaai/jina-clip-v1", ), name="jinaai/jina-clip-v1", diff --git a/mteb/models/openclip_models.py b/mteb/models/openclip_models.py new file mode 100644 index 0000000000..ef7589cf8f --- /dev/null +++ b/mteb/models/openclip_models.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm + +from mteb.model_meta import ModelMeta + + +def openclip_loader(**kwargs): + try: + import open_clip + except ImportError: + raise ImportError("Please run `pip install open_clip_torch`.") + + class OpenCLIPWrapper: + def __init__( + self, + model_name: str = "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + self.model, _, self.img_preprocess = open_clip.create_model_and_transforms( + f"hf-hub:{model_name}", device=device + ) + self.model.eval() + self.tokenizer = open_clip.get_tokenizer(f"hf-hub:{model_name}") + + def encode( # type: ignore + self, + sentences: list[str], + *, + batch_size: int = 32, + **kwargs: Any, + ): + return self.get_text_embeddings(texts=sentences, batch_size=batch_size) + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(), torch.cuda.amp.autocast(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + inputs = self.tokenizer(batch_texts) + text_outputs = self.model.encode_text(inputs.to(self.device)) + all_text_embeddings.append(text_outputs.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + all_image_embeddings = [] + if isinstance(images, DataLoader): + import torchvision.transforms.functional as F + + with torch.no_grad(), torch.cuda.amp.autocast(): + for batch in tqdm(images): + # import pdb; pdb.set_trace() + inputs = torch.vstack( + [ + self.img_preprocess(F.to_pil_image(b)).unsqueeze(0) + for b in batch + ] + ) + image_outputs = self.model.encode_image(inputs.to(self.device)) + all_image_embeddings.append(image_outputs.cpu()) + else: + with torch.no_grad(), torch.cuda.amp.autocast(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + inputs = torch.vstack( + [self.img_preprocess(b) for b in batch_images] + ) + image_outputs = self.model.encode_image(inputs.to(self.device)) + all_image_embeddings.append(image_outputs.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm( + dim=-1, keepdim=True + ) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + else: + # to do: add other fusion mode + raise ValueError( + f"fusion mode {fusion_mode} hasn't been implemented" + ) + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + return OpenCLIPWrapper(**kwargs) + + +CLIP_ViT_L_14_DataComp_XL_s13B_b90K = ModelMeta( + loader=partial( + openclip_loader, + model_name="laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + ), + name="laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + languages=["eng_Latn"], + open_source=True, + revision="84c9828e63dc9a9351d1fe637c346d4c1c4db341", + release_date="2023-04-26", +) + +CLIP_ViT_B_32_DataComp_XL_s13B_b90K = ModelMeta( + loader=partial( + openclip_loader, + model_name="laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", + ), + name="laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", + languages=["eng_Latn"], + open_source=True, + revision="f0e2ffa09cbadab3db6a261ec1ec56407ce42912", + release_date="2023-04-26", +) + +CLIP_ViT_B_16_DataComp_XL_s13B_b90K = ModelMeta( + loader=partial( + openclip_loader, + model_name="laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", + ), + name="laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", + languages=["eng_Latn"], + open_source=True, + revision="d110532e8d4ff91c574ee60a342323f28468b287", + release_date="2023-04-26", +) + +CLIP_ViT_bigG_14_laion2B_39B_b160k = ModelMeta( + loader=partial( + openclip_loader, + model_name="laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", + ), + name="laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", + languages=["eng_Latn"], + open_source=True, + revision="bc7788f151930d91b58474715fdce5524ad9a189", + release_date="2023-01-23", +) + +CLIP_ViT_g_14_laion2B_s34B_b88K = ModelMeta( + loader=partial( + openclip_loader, + model_name="laion/CLIP-ViT-g-14-laion2B-s34B-b88K", + ), + name="laion/CLIP-ViT-g-14-laion2B-s34B-b88K", + languages=["eng_Latn"], + open_source=True, + revision="15efd0f6ac0c40c0f9da7becca03c974d7012604", + release_date="2023-03-06", +) + +CLIP_ViT_H_14_laion2B_s32B_b79K = ModelMeta( + loader=partial( + openclip_loader, + model_name="laion/CLIP-ViT-H-14-laion2B-s32B-b79K", + ), + name="laion/CLIP-ViT-H-14-laion2B-s32B-b79K", + languages=["eng_Latn"], + open_source=True, + revision="de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", + release_date="2022-09-15", +) + +CLIP_ViT_L_14_laion2B_s32B_b82K = ModelMeta( + loader=partial( + openclip_loader, + model_name="laion/CLIP-ViT-L-14-laion2B-s32B-b82K", + ), + name="laion/CLIP-ViT-L-14-laion2B-s32B-b82K", + languages=["eng_Latn"], + open_source=True, + revision="1627032197142fbe2a7cfec626f4ced3ae60d07a", + release_date="2022-09-15", +) + +CLIP_ViT_B_32_laion2B_s34B_b79K = ModelMeta( + loader=partial( + openclip_loader, + model_name="laion/CLIP-ViT-B-32-laion2B-s34B-b79K", + ), + name="laion/CLIP-ViT-B-32-laion2B-s34B-b79K", + languages=["eng_Latn"], + open_source=True, + revision="08f73555f1b2fb7c82058aebbd492887a94968ef", + release_date="2022-09-15", +) diff --git a/mteb/models/overview.py b/mteb/models/overview.py index 0c2175ebdf..70768639bd 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -9,18 +9,28 @@ from mteb.encoder_interface import Encoder from mteb.model_meta import ModelMeta from mteb.models import ( + align_models, bge_models, + blip2_models, + blip_models, bm25, + clip_models, cohere_models, + dino_models, e5_instruct, e5_models, + e5_v, google_models, gritlm_models, gte_models, + jina_clip, llm2vec_models, + moco_models, mxbai_models, nomic_models, + nomic_models_vision, openai_models, + openclip_models, promptriever_models, repllama_models, rerankers_custom, @@ -28,29 +38,44 @@ ru_sentence_models, salesforce_models, sentence_transformers_models, + vista_models, + vlm2vec_models, voyage_models, ) logger = logging.getLogger(__name__) model_modules = [ + align_models, bge_models, + blip_models, + blip2_models, bm25, + clip_models, cohere_models, + dino_models, e5_instruct, e5_models, + e5_v, google_models, gritlm_models, gte_models, + jina_clip, llm2vec_models, + moco_models, mxbai_models, nomic_models, + nomic_models_vision, + cohere_models, + clip_models, openai_models, + openclip_models, ru_sentence_models, salesforce_models, sentence_transformers_models, + vista_models, voyage_models, - google_models, + vlm2vec_models, repllama_models, promptriever_models, rerankers_monot5_based, diff --git a/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/MNISTZeroShot.json b/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/MNISTZeroShot.json new file mode 100644 index 0000000000..a8e2f70693 --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/MNISTZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 24.08267593383789, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.6381, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6381 + } + ] + }, + "task_name": "MNISTZeroShot" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/model_meta.json b/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/model_meta.json new file mode 100644 index 0000000000..4781cc3a98 --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/model_meta.json @@ -0,0 +1 @@ +{"name": "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", "revision": "08f73555f1b2fb7c82058aebbd492887a94968ef", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/MNIST.json b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/MNIST.json new file mode 100644 index 0000000000..e6e6de11f7 --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 184.1278896331787, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.9621999999999999, + "f1": 0.9622432335494757, + "f1_weighted": 0.962254470082415, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9621999999999999, + "scores_per_experiment": [ + { + "accuracy": 0.9623, + "f1": 0.9622679487261954, + "f1_weighted": 0.9624186429667654 + }, + { + "accuracy": 0.956, + "f1": 0.955938319135998, + "f1_weighted": 0.9559668684856993 + }, + { + "accuracy": 0.9645, + "f1": 0.9644946109310863, + "f1_weighted": 0.9645725164278387 + }, + { + "accuracy": 0.9606, + "f1": 0.9608141458212058, + "f1_weighted": 0.9606963363543709 + }, + { + "accuracy": 0.9676, + "f1": 0.9677011431328928, + "f1_weighted": 0.9676179861774005 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json new file mode 100644 index 0000000000..36d3659874 --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json @@ -0,0 +1 @@ +{"name": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "revision": "de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/BLINKIT2IRetrieval.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/BLINKIT2IRetrieval.json new file mode 100644 index 0000000000..dd843816fd --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/BLINKIT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26", + "evaluation_time": 28.74270534515381, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.3607, + "cv_recall_at_10": 0.63184, + "cv_recall_at_100": 0.91045, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.69652, + "cv_recall_at_3": 0.54975, + "cv_recall_at_5": 0.58458, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.50288, + "map_at_1": 0.3607, + "map_at_10": 0.46095, + "map_at_100": 0.47, + "map_at_1000": 0.47054, + "map_at_20": 0.46542, + "map_at_3": 0.44693, + "map_at_5": 0.45464, + "mrr_at_1": 0.36069651741293535, + "mrr_at_10": 0.4614921424622917, + "mrr_at_100": 0.4705570652048439, + "mrr_at_1000": 0.4710736141683453, + "mrr_at_20": 0.465952194964833, + "mrr_at_3": 0.447346600331675, + "mrr_at_5": 0.4551824212271974, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.3251766664892245, + "nauc_cv_recall_at_100_max": 0.4996971324986197, + "nauc_cv_recall_at_100_std": -0.10981886119431823, + "nauc_cv_recall_at_10_diff1": 0.4150802935478081, + "nauc_cv_recall_at_10_max": 0.44521676291214113, + "nauc_cv_recall_at_10_std": -0.4044050580553833, + "nauc_cv_recall_at_1_diff1": 0.5090856549067897, + "nauc_cv_recall_at_1_max": 0.3561884419781478, + "nauc_cv_recall_at_1_std": -0.3108474258223719, + "nauc_cv_recall_at_20_diff1": 0.3928776128813473, + "nauc_cv_recall_at_20_max": 0.40754722638468, + "nauc_cv_recall_at_20_std": -0.4226831259380377, + "nauc_cv_recall_at_3_diff1": 0.41751507599566495, + "nauc_cv_recall_at_3_max": 0.5119599405595536, + "nauc_cv_recall_at_3_std": -0.3880458369320522, + "nauc_cv_recall_at_5_diff1": 0.41110507164831417, + "nauc_cv_recall_at_5_max": 0.4892953426380338, + "nauc_cv_recall_at_5_std": -0.3909994678840402, + "nauc_map_at_1000_diff1": 0.46826505032819066, + "nauc_map_at_1000_max": 0.40797029811237867, + "nauc_map_at_1000_std": -0.3487770618903377, + "nauc_map_at_100_diff1": 0.4680852808240436, + "nauc_map_at_100_max": 0.4079486164295179, + "nauc_map_at_100_std": -0.34864155304932437, + "nauc_map_at_10_diff1": 0.46821531795636995, + "nauc_map_at_10_max": 0.4105873108851904, + "nauc_map_at_10_std": -0.35048690648082625, + "nauc_map_at_1_diff1": 0.5090856549067897, + "nauc_map_at_1_max": 0.3561884419781478, + "nauc_map_at_1_std": -0.3108474258223719, + "nauc_map_at_20_diff1": 0.4672516873903748, + "nauc_map_at_20_max": 0.4078473889033209, + "nauc_map_at_20_std": -0.3510430019216793, + "nauc_map_at_3_diff1": 0.46792266407052935, + "nauc_map_at_3_max": 0.4217139881712332, + "nauc_map_at_3_std": -0.34757820498553116, + "nauc_map_at_5_diff1": 0.46735798854760324, + "nauc_map_at_5_max": 0.4165657756912836, + "nauc_map_at_5_std": -0.3486378662930895, + "nauc_mrr_at_1000_diff1": 0.46804229195204133, + "nauc_mrr_at_1000_max": 0.4071917446826936, + "nauc_mrr_at_1000_std": -0.3476138289564308, + "nauc_mrr_at_100_diff1": 0.46790977187423044, + "nauc_mrr_at_100_max": 0.4072143236882314, + "nauc_mrr_at_100_std": -0.3475446027341017, + "nauc_mrr_at_10_diff1": 0.46800043394283297, + "nauc_mrr_at_10_max": 0.40981114524918816, + "nauc_mrr_at_10_std": -0.34936642009737867, + "nauc_mrr_at_1_diff1": 0.5090856549067897, + "nauc_mrr_at_1_max": 0.3561884419781478, + "nauc_mrr_at_1_std": -0.3108474258223719, + "nauc_mrr_at_20_diff1": 0.4670263761262366, + "nauc_mrr_at_20_max": 0.407078956671809, + "nauc_mrr_at_20_std": -0.34989409756068307, + "nauc_mrr_at_3_diff1": 0.46769515854499266, + "nauc_mrr_at_3_max": 0.4210312389892628, + "nauc_mrr_at_3_std": -0.3470554885966742, + "nauc_mrr_at_5_diff1": 0.4671445165231216, + "nauc_mrr_at_5_max": 0.41579979574754844, + "nauc_mrr_at_5_std": -0.3475262115394731, + "nauc_ndcg_at_1000_diff1": 0.46044333308523944, + "nauc_ndcg_at_1000_max": 0.41294647548488717, + "nauc_ndcg_at_1000_std": -0.35135326672746664, + "nauc_ndcg_at_100_diff1": 0.45473212178018996, + "nauc_ndcg_at_100_max": 0.4151222010165269, + "nauc_ndcg_at_100_std": -0.34213432270326677, + "nauc_ndcg_at_10_diff1": 0.45571130700534823, + "nauc_ndcg_at_10_max": 0.42039077138937964, + "nauc_ndcg_at_10_std": -0.3630517807405077, + "nauc_ndcg_at_1_diff1": 0.5090856549067897, + "nauc_ndcg_at_1_max": 0.3561884419781478, + "nauc_ndcg_at_1_std": -0.3108474258223719, + "nauc_ndcg_at_20_diff1": 0.4520756416944454, + "nauc_ndcg_at_20_max": 0.4113476779046133, + "nauc_ndcg_at_20_std": -0.3655776710972757, + "nauc_ndcg_at_3_diff1": 0.455287916218925, + "nauc_ndcg_at_3_max": 0.4441335745158963, + "nauc_ndcg_at_3_std": -0.3578120161280115, + "nauc_ndcg_at_5_diff1": 0.45370549230640783, + "nauc_ndcg_at_5_max": 0.4346379074583971, + "nauc_ndcg_at_5_std": -0.3591858287964079, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.31176332692683584, + "nauc_precision_at_100_max": 0.4815670235846271, + "nauc_precision_at_100_std": -0.09187319856981026, + "nauc_precision_at_10_diff1": 0.415080293547808, + "nauc_precision_at_10_max": 0.4452167629121412, + "nauc_precision_at_10_std": -0.4044050580553832, + "nauc_precision_at_1_diff1": 0.5090856549067897, + "nauc_precision_at_1_max": 0.3561884419781478, + "nauc_precision_at_1_std": -0.3108474258223719, + "nauc_precision_at_20_diff1": 0.39287761288134654, + "nauc_precision_at_20_max": 0.40754722638467994, + "nauc_precision_at_20_std": -0.4226831259380381, + "nauc_precision_at_3_diff1": 0.41751507599566523, + "nauc_precision_at_3_max": 0.5119599405595534, + "nauc_precision_at_3_std": -0.3880458369320523, + "nauc_precision_at_5_diff1": 0.41110507164831417, + "nauc_precision_at_5_max": 0.4892953426380342, + "nauc_precision_at_5_std": -0.3909994678840406, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.3117633269268345, + "nauc_recall_at_100_max": 0.48156702358462466, + "nauc_recall_at_100_std": -0.0918731985698134, + "nauc_recall_at_10_diff1": 0.4150802935478081, + "nauc_recall_at_10_max": 0.44521676291214113, + "nauc_recall_at_10_std": -0.4044050580553833, + "nauc_recall_at_1_diff1": 0.5090856549067897, + "nauc_recall_at_1_max": 0.3561884419781478, + "nauc_recall_at_1_std": -0.3108474258223719, + "nauc_recall_at_20_diff1": 0.3928776128813473, + "nauc_recall_at_20_max": 0.40754722638468, + "nauc_recall_at_20_std": -0.4226831259380377, + "nauc_recall_at_3_diff1": 0.41751507599566495, + "nauc_recall_at_3_max": 0.5119599405595536, + "nauc_recall_at_3_std": -0.3880458369320522, + "nauc_recall_at_5_diff1": 0.41110507164831417, + "nauc_recall_at_5_max": 0.4892953426380338, + "nauc_recall_at_5_std": -0.3909994678840402, + "ndcg_at_1": 0.3607, + "ndcg_at_10": 0.50288, + "ndcg_at_100": 0.55671, + "ndcg_at_1000": 0.56895, + "ndcg_at_20": 0.51921, + "ndcg_at_3": 0.47346, + "ndcg_at_5": 0.48759, + "precision_at_1": 0.3607, + "precision_at_10": 0.06318, + "precision_at_100": 0.00908, + "precision_at_1000": 0.001, + "precision_at_20": 0.03483, + "precision_at_3": 0.18325, + "precision_at_5": 0.11692, + "recall_at_1": 0.3607, + "recall_at_10": 0.63184, + "recall_at_100": 0.90796, + "recall_at_1000": 1.0, + "recall_at_20": 0.69652, + "recall_at_3": 0.54975, + "recall_at_5": 0.58458 + } + ] + }, + "task_name": "BLINKIT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/Caltech101.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/Caltech101.json deleted file mode 100644 index 84d016d4f2..0000000000 --- a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/Caltech101.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", - "evaluation_time": 336.7735936641693, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.9634451019066403, - "f1": 0.9390393783666946, - "f1_weighted": 0.9629340674668043, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9634451019066403, - "scores_per_experiment": [ - { - "accuracy": 0.9640039447731755, - "f1": 0.9387417673075573, - "f1_weighted": 0.963701162627355 - }, - { - "accuracy": 0.965483234714004, - "f1": 0.9412686220165988, - "f1_weighted": 0.9654077609459234 - }, - { - "accuracy": 0.9598948060486522, - "f1": 0.9360200534285099, - "f1_weighted": 0.9589670335376868 - }, - { - "accuracy": 0.9617028270874425, - "f1": 0.9417554557342505, - "f1_weighted": 0.9606672076394908 - }, - { - "accuracy": 0.9661406969099277, - "f1": 0.9374109933465566, - "f1_weighted": 0.9659271725835652 - } - ] - } - ] - }, - "task_name": "Caltech101" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNIST.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNIST.json new file mode 100644 index 0000000000..0713724334 --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNIST.json @@ -0,0 +1,28 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 84.51716136932373, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.9587, + "f1": 0.9584918381835605, + "f1_weighted": 0.9587174442512109, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9587, + "scores_per_experiment": [ + { + "accuracy": 0.9587, + "f1": 0.9584918381835605, + "f1_weighted": 0.9587174442512109 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNISTZeroShot.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNISTZeroShot.json new file mode 100644 index 0000000000..154f626d2f --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNISTZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 79.63758492469788, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.8144, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8144 + } + ] + }, + "task_name": "MNISTZeroShot" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/STS14.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/STS14.json new file mode 100644 index 0000000000..20751681aa --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/STS14.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", + "evaluation_time": 5.388969898223877, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cosine_pearson": 0.6794914647081003, + "cosine_spearman": 0.6327310416487439, + "euclidean_pearson": 0.6754886733081702, + "euclidean_spearman": 0.6473065461091825, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6327310416487439, + "manhattan_pearson": 0.6833982691624099, + "manhattan_spearman": 0.6526646970883643, + "pearson": 0.6794914647081003, + "spearman": 0.6327310416487439 + } + ] + }, + "task_name": "STS14" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json index 76c7344f9e..1825d8b79c 100644 --- a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json +++ b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json @@ -1 +1 @@ -{"name": "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "revision": "84c9828e63dc9a9351d1fe637c346d4c1c4db341", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DataCLIPModelWrapper"} \ No newline at end of file +{"name": "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "revision": "84c9828e63dc9a9351d1fe637c346d4c1c4db341", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/MNISTZeroShot.json b/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/MNISTZeroShot.json new file mode 100644 index 0000000000..d166f91c44 --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/MNISTZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 73.2168505191803, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.6414, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6414 + } + ] + }, + "task_name": "MNISTZeroShot" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/model_meta.json b/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/model_meta.json new file mode 100644 index 0000000000..ff6252d975 --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/model_meta.json @@ -0,0 +1 @@ +{"name": "laion/CLIP-ViT-L-14-laion2B-s32B-b82K", "revision": "1627032197142fbe2a7cfec626f4ced3ae60d07a", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/MNIST.json b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/MNIST.json new file mode 100644 index 0000000000..c0c5d016a1 --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 259.59732961654663, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.96534, + "f1": 0.9652996312920756, + "f1_weighted": 0.9654146401974364, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.96534, + "scores_per_experiment": [ + { + "accuracy": 0.9656, + "f1": 0.9655583025661676, + "f1_weighted": 0.9656417812003859 + }, + { + "accuracy": 0.963, + "f1": 0.9628025390918398, + "f1_weighted": 0.96301114967701 + }, + { + "accuracy": 0.9676, + "f1": 0.9674695816973904, + "f1_weighted": 0.9676411686849484 + }, + { + "accuracy": 0.9611, + "f1": 0.9613040896869535, + "f1_weighted": 0.9613453097505182 + }, + { + "accuracy": 0.9694, + "f1": 0.9693636434180266, + "f1_weighted": 0.9694337916743194 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json new file mode 100644 index 0000000000..5465d2e2eb --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json @@ -0,0 +1 @@ +{"name": "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", "revision": "15efd0f6ac0c40c0f9da7becca03c974d7012604", "release_date": "2023-03-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file From 45ffa4439711ac5b9c0ea3109948e7b87926b240 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 28 Oct 2024 17:25:55 +0200 Subject: [PATCH 085/154] [mieb] new version with downsampled train split to 32 per class (#1327) * new version with downsampled train split to 32 per class * force load truncated image file * make lint * add open clip models * Update __init__.py * lint * fix model overview * fix ImageCLS undersample; run birdsnap * make lint * make lint --------- Co-authored-by: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Co-authored-by: gowitheflow-1998 Co-authored-by: gowitheflow-1998 --- .../Image/AbsTaskImageClassification.py | 20 +-- .../Image/ClassificationEvaluator.py | 59 +++---- mteb/models/jina_clip.py | 157 +++++++++++++++++- .../eng/BirdsnapClassification.py | 2 +- .../ZeroshotClassification/eng/Birdsnap.py | 2 +- .../Birdsnap.json | 48 ++++++ .../Birdsnap.json | 48 ++++++ 7 files changed, 286 insertions(+), 50 deletions(-) create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Birdsnap.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Birdsnap.json diff --git a/mteb/abstasks/Image/AbsTaskImageClassification.py b/mteb/abstasks/Image/AbsTaskImageClassification.py index 04a6980283..7add58296a 100644 --- a/mteb/abstasks/Image/AbsTaskImageClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageClassification.py @@ -5,6 +5,7 @@ from typing import Any import numpy as np +from PIL import ImageFile from mteb.abstasks.TaskMetadata import HFSubset @@ -16,6 +17,8 @@ ) from ..AbsTask import AbsTask, ScoresDict +ImageFile.LOAD_TRUNCATED_IMAGES = True + logger = logging.getLogger(__name__) @@ -133,7 +136,7 @@ def _evaluate_subset( "=" * 10 + f" Experiment {i+1}/{self.n_experiments} " + "=" * 10 ) # Bootstrap `self.samples_per_label` samples per label for each split - X_sampled, y_sampled, idxs = self._undersample_data( + undersampled_train, idxs = self._undersample_data( train_split, self.label_column_name, self.samples_per_label, @@ -142,8 +145,7 @@ def _evaluate_subset( if self.method == "kNN": evaluator = ImagekNNClassificationEvaluator( - X_sampled, - y_sampled, + undersampled_train, eval_split, self.image_column_name, self.label_column_name, @@ -153,8 +155,7 @@ def _evaluate_subset( ) elif self.method == "kNN-pytorch": evaluator = ImagekNNClassificationEvaluatorPytorch( - X_sampled, - y_sampled, + undersampled_train, eval_split, self.image_column_name, self.label_column_name, @@ -164,8 +165,7 @@ def _evaluate_subset( ) elif self.method == "logReg": evaluator = ImagelogRegClassificationEvaluator( - X_sampled, - y_sampled, + undersampled_train, eval_split, self.image_column_name, self.label_column_name, @@ -199,15 +199,15 @@ def _undersample_data( label_counter = defaultdict(int) selected_indices = [] + labels = dataset_split[label_column_name] for i in idxs: - label = dataset_split[i][label_column_name] + label = labels[i] if label_counter[label] < samples_per_label: selected_indices.append(i) label_counter[label] += 1 undersampled_dataset = dataset_split.select(selected_indices) return ( - undersampled_dataset[self.image_column_name], - undersampled_dataset[self.label_column_name], + undersampled_dataset, idxs, ) diff --git a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py index e129ab0a4c..2f60b5c690 100644 --- a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py @@ -1,11 +1,12 @@ from __future__ import annotations import logging +import math +import os from typing import Any import numpy as np import torch -from datasets import Dataset from sklearn.linear_model import LogisticRegression from sklearn.metrics import ( accuracy_score, @@ -55,8 +56,7 @@ def custom_collate_fn(batch): class ImagekNNClassificationEvaluator(Evaluator): def __init__( self, - images_train, - y_train, + dataset_train, dataset_test, image_column_name, label_column_name, @@ -69,17 +69,13 @@ def __init__( super().__init__(**kwargs) if limit is not None: - images_train = images_train[:limit] - y_train = y_train[:limit] - dataset_test = dataset_test[:limit] + dataset_train = dataset_train.select(list(range(limit))) - self.images_train = images_train - self.y_train = y_train self.dataset_train = ImageDataset( - Dataset.from_dict({"image": images_train, "label": y_train}), - image_column_name=image_column_name, - transform=transform, + dataset_train, image_column_name=image_column_name, transform=transform ) + self.y_train = dataset_train[label_column_name] + self.dataset_test = ImageDataset( dataset_test, image_column_name=image_column_name, transform=transform ) @@ -102,7 +98,7 @@ def __call__(self, model, test_cache=None): batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=16, + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) X_train = model.get_image_embeddings( dataloader_train, batch_size=self.encode_kwargs["batch_size"] @@ -111,7 +107,7 @@ def __call__(self, model, test_cache=None): self.dataset_test, batch_size=self.encode_kwargs["batch_size"], shuffle=False, - num_workers=16, + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) if test_cache is None: X_test = model.get_image_embeddings( @@ -145,8 +141,7 @@ def __call__(self, model, test_cache=None): class ImagekNNClassificationEvaluatorPytorch(Evaluator): def __init__( self, - images_train, - y_train, + dataset_train, dataset_test, image_column_name, label_column_name, @@ -158,17 +153,13 @@ def __init__( ): super().__init__(**kwargs) if limit is not None: - images_train = images_train[:limit] - y_train = y_train[:limit] - dataset_test = dataset_test[:limit] + dataset_train = dataset_train.select(list(range(limit))) - self.images_train = images_train self.dataset_train = ImageDataset( - Dataset.from_dict({"image": images_train, "label": y_train}), - image_column_name=image_column_name, - transform=transform, + dataset_train, image_column_name=image_column_name, transform=transform ) - self.y_train = y_train + self.y_train = dataset_train[label_column_name] + self.dataset_test = ImageDataset( dataset_test, image_column_name=image_column_name, transform=transform ) @@ -192,7 +183,7 @@ def __call__(self, model: Encoder, test_cache=None): batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=16, + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) X_train = model.get_image_embeddings( dataloader_train, batch_size=self.encode_kwargs["batch_size"] @@ -202,7 +193,7 @@ def __call__(self, model: Encoder, test_cache=None): self.dataset_test, batch_size=self.encode_kwargs["batch_size"], shuffle=False, - num_workers=16, + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) if test_cache is None: X_test = model.get_image_embeddings( @@ -311,8 +302,7 @@ def _dot_score(a: Tensor, b: Tensor): class ImagelogRegClassificationEvaluator(Evaluator): def __init__( self, - images_train, - y_train, + dataset_train, dataset_test, image_column_name, label_column_name, @@ -329,17 +319,12 @@ def __init__( self.encode_kwargs["batch_size"] = 32 if limit is not None: - images_train = images_train[:limit] - y_train = y_train[:limit] - dataset_test = dataset_test[:limit] + dataset_train = dataset_train.select(list(range(limit))) - self.images_train = images_train - self.y_train = y_train self.dataset_train = ImageDataset( - Dataset.from_dict({"image": images_train, "label": y_train}), - image_column_name=image_column_name, - transform=transform, + dataset_train, image_column_name=image_column_name, transform=transform ) + self.y_train = dataset_train[label_column_name] self.dataset_test = ImageDataset( dataset_test, image_column_name=image_column_name, transform=transform ) @@ -361,7 +346,7 @@ def __call__(self, model, test_cache=None): batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=16, + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) X_train = model.get_image_embeddings( dataloader_train, batch_size=self.encode_kwargs["batch_size"] @@ -371,7 +356,7 @@ def __call__(self, model, test_cache=None): batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=16, + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) if test_cache is None: X_test = model.get_image_embeddings( diff --git a/mteb/models/jina_clip.py b/mteb/models/jina_clip.py index eec407dab7..8740b9cb21 100644 --- a/mteb/models/jina_clip.py +++ b/mteb/models/jina_clip.py @@ -1,8 +1,163 @@ from __future__ import annotations from functools import partial +from typing import Any -from mteb.model_meta import ModelMeta, sentence_transformers_loader +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import AutoModel + +from mteb.model_meta import ModelMeta + +# from mteb.models.text_formatting_utils import corpus_to_texts + + +class JinaCLIPModelWrapper: + def __init__( + self, + model_name: str, + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + self.model = AutoModel.from_pretrained(model_name, trust_remote_code=True).to( + self.device + ) + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + text_outputs = self.model.encode_text( + batch_texts, convert_to_numpy=False, convert_to_tensor=True + ) + all_text_embeddings.append(text_outputs.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + with torch.no_grad(): + for batch in tqdm(images): + image_outputs = self.model.encode_image( + batch, convert_to_numpy=False, convert_to_tensor=True + ) + all_image_embeddings.append(image_outputs.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + image_outputs = self.model.encode_image( + batch_images, convert_to_numpy=False, convert_to_tensor=True + ) + all_image_embeddings.append(image_outputs.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.encode_text( + texts, + batch_size=batch_size, + convert_to_numpy=False, + convert_to_tensor=True, + ) + + if images is not None: + image_embeddings = self.encode_image( + images, + batch_size=batch_size, + convert_to_numpy=False, + convert_to_tensor=True, + ) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + else: + # to do: add other fusion mode + raise ValueError(f"fusion mode {fusion_mode} hasn't been implemented") + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + def encode( # type: ignore + self, + sentences: list[str], + *, + batch_size: int = 32, + **kwargs: Any, + ): + if "prompt_name" in kwargs: + kwargs.pop("prompt_name") + return self.model.encode_text(sentences, batch_size=batch_size, **kwargs) + + # def encode_queries(self, queries: list[str], batch_size: int = 32, **kwargs: Any): + # if "prompt_name" in kwargs: + # kwargs.pop("prompt_name") + # sentences = [ + # "Represent this sentence for searching relevant passages: " + sentence + # for sentence in queries + # ] + # emb = self.encode( + # sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs + # ) + # return emb + + # def encode_corpus( + # self, + # corpus: list[dict[str, str]] | dict[str, list[str]], + # batch_size: int = 32, + # **kwargs: Any, + # ): + # if "prompt_name" in kwargs: + # kwargs.pop("prompt_name") + # sentences = corpus_to_texts(corpus) + # emb = self.encode( + # sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs + # ) + # return emb + + +from mteb.model_meta import sentence_transformers_loader jina_clip_v1 = ModelMeta( loader=partial( diff --git a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py index 4e90a5e30d..2b08169aae 100644 --- a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py @@ -11,7 +11,7 @@ class BirdsnapClassification(AbsTaskImageClassification): reference="https://openaccess.thecvf.com/content_cvpr_2014/html/Berg_Birdsnap_Large-scale_Fine-grained_2014_CVPR_paper.html", dataset={ "path": "isaacchung/birdsnap", - "revision": "e09b9dea248d579376684268cbedba28cd66b9b4", + "revision": "fd23015508be94f0b5b59d61630e4ea2536509e4", }, type="ImageClassification", category="i2t", diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py b/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py index 2eac4b0214..14609d08a6 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Birdsnap.py @@ -13,7 +13,7 @@ class BirdsnapClassification(AbsTaskZeroshotClassification): reference="https://openaccess.thecvf.com/content_cvpr_2014/html/Berg_Birdsnap_Large-scale_Fine-grained_2014_CVPR_paper.html", dataset={ "path": "isaacchung/birdsnap", - "revision": "e09b9dea248d579376684268cbedba28cd66b9b4", + "revision": "fd23015508be94f0b5b59d61630e4ea2536509e4", }, type="ZeroShotClassification", category="i2t", diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Birdsnap.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Birdsnap.json new file mode 100644 index 0000000000..8fb18fd10f --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Birdsnap.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "fd23015508be94f0b5b59d61630e4ea2536509e4", + "evaluation_time": 1329.5408725738525, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5715829281469476, + "f1": 0.5427579493814982, + "f1_weighted": 0.5675297472237123, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5715829281469476, + "scores_per_experiment": [ + { + "accuracy": 0.5705024311183144, + "f1": 0.5441404980561606, + "f1_weighted": 0.5690425577783762 + }, + { + "accuracy": 0.5764451647757969, + "f1": 0.5486149281330003, + "f1_weighted": 0.573202555293317 + }, + { + "accuracy": 0.5748244192328471, + "f1": 0.5423477063005765, + "f1_weighted": 0.568412911615314 + }, + { + "accuracy": 0.5640194489465153, + "f1": 0.5339722105851138, + "f1_weighted": 0.5562256849939183 + }, + { + "accuracy": 0.5721231766612642, + "f1": 0.5447144038326402, + "f1_weighted": 0.5707650264376358 + } + ] + } + ] + }, + "task_name": "Birdsnap" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Birdsnap.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Birdsnap.json new file mode 100644 index 0000000000..7f46636f26 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Birdsnap.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "fd23015508be94f0b5b59d61630e4ea2536509e4", + "evaluation_time": 1263.4543855190277, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.46904376012965965, + "f1": 0.4359369143749953, + "f1_weighted": 0.46064550492427314, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.46904376012965965, + "scores_per_experiment": [ + { + "accuracy": 0.473257698541329, + "f1": 0.44226358144193817, + "f1_weighted": 0.46374933668288604 + }, + { + "accuracy": 0.4662344678552134, + "f1": 0.43198733439697296, + "f1_weighted": 0.45760840995849095 + }, + { + "accuracy": 0.4737979470556456, + "f1": 0.43945327318821287, + "f1_weighted": 0.4665146936621815 + }, + { + "accuracy": 0.4705564559697461, + "f1": 0.44085583091607183, + "f1_weighted": 0.46342910945828286 + }, + { + "accuracy": 0.4613722312263641, + "f1": 0.42512455193178084, + "f1_weighted": 0.4519259748595243 + } + ] + } + ] + }, + "task_name": "Birdsnap" +} \ No newline at end of file From 805460716a3a527877f31cd8044c0813138f16f0 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 28 Oct 2024 20:13:07 +0200 Subject: [PATCH 086/154] [mieb] Fix Jina CLIP (#1349) fix jina clip v1 --- mteb/models/jina_clip.py | 44 +---- .../MSCOCOI2TRetrieval.json | 186 ++++++++++++++++++ .../STS14.json | 26 +++ .../model_meta.json | 1 + 4 files changed, 222 insertions(+), 35 deletions(-) create mode 100644 results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/MSCOCOI2TRetrieval.json create mode 100644 results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/STS14.json create mode 100644 results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json diff --git a/mteb/models/jina_clip.py b/mteb/models/jina_clip.py index 8740b9cb21..be8a4b0715 100644 --- a/mteb/models/jina_clip.py +++ b/mteb/models/jina_clip.py @@ -11,8 +11,6 @@ from mteb.model_meta import ModelMeta -# from mteb.models.text_formatting_utils import corpus_to_texts - class JinaCLIPModelWrapper: def __init__( @@ -48,9 +46,13 @@ def get_image_embeddings( if isinstance(images, DataLoader): with torch.no_grad(): + import torchvision.transforms.functional as F + for batch in tqdm(images): image_outputs = self.model.encode_image( - batch, convert_to_numpy=False, convert_to_tensor=True + [F.to_pil_image(b.to("cpu")) for b in batch], + convert_to_numpy=False, + convert_to_tensor=True, ) all_image_embeddings.append(image_outputs.cpu()) else: @@ -126,48 +128,20 @@ def encode( # type: ignore batch_size: int = 32, **kwargs: Any, ): - if "prompt_name" in kwargs: - kwargs.pop("prompt_name") + if "task_name" in kwargs: + kwargs.pop("task_name") return self.model.encode_text(sentences, batch_size=batch_size, **kwargs) - # def encode_queries(self, queries: list[str], batch_size: int = 32, **kwargs: Any): - # if "prompt_name" in kwargs: - # kwargs.pop("prompt_name") - # sentences = [ - # "Represent this sentence for searching relevant passages: " + sentence - # for sentence in queries - # ] - # emb = self.encode( - # sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs - # ) - # return emb - - # def encode_corpus( - # self, - # corpus: list[dict[str, str]] | dict[str, list[str]], - # batch_size: int = 32, - # **kwargs: Any, - # ): - # if "prompt_name" in kwargs: - # kwargs.pop("prompt_name") - # sentences = corpus_to_texts(corpus) - # emb = self.encode( - # sentences, batch_size=batch_size, normalize_embeddings=True, **kwargs - # ) - # return emb - - -from mteb.model_meta import sentence_transformers_loader jina_clip_v1 = ModelMeta( loader=partial( - sentence_transformers_loader, + JinaCLIPModelWrapper, model_name="jinaai/jina-clip-v1", ), name="jinaai/jina-clip-v1", languages=["eng_Latn"], open_source=True, - revision="1cbe5e8b11ea3728df0b610d5453dfe739804aa9", + revision="06150c7c382d7a4faedc7d5a0d8cdb59308968f4", release_date="2024-05-30", ) diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/MSCOCOI2TRetrieval.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/MSCOCOI2TRetrieval.json new file mode 100644 index 0000000000..1bc2b143dc --- /dev/null +++ b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/MSCOCOI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", + "evaluation_time": 57.833680629730225, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.544, + "cv_recall_at_10": 0.8592, + "cv_recall_at_100": 0.993, + "cv_recall_at_1000": 0.9998, + "cv_recall_at_20": 0.922, + "cv_recall_at_3": 0.7132, + "cv_recall_at_5": 0.7826, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.48279, + "map_at_1": 0.10886, + "map_at_10": 0.36753, + "map_at_100": 0.41438, + "map_at_1000": 0.41751, + "map_at_20": 0.39363, + "map_at_3": 0.2442, + "map_at_5": 0.31641, + "mrr_at_1": 0.544, + "mrr_at_10": 0.6441334920634935, + "mrr_at_100": 0.6505229831896451, + "mrr_at_1000": 0.6505722508994467, + "mrr_at_20": 0.6485590829105544, + "mrr_at_3": 0.6178333333333357, + "mrr_at_5": 0.6337633333333345, + "nauc_cv_recall_at_1000_diff1": -1.1517273576107072, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": -0.001093770841666523, + "nauc_cv_recall_at_100_max": 0.4157129518474038, + "nauc_cv_recall_at_100_std": -0.3355875683606853, + "nauc_cv_recall_at_10_diff1": 0.3116322316328111, + "nauc_cv_recall_at_10_max": 0.22344380665652067, + "nauc_cv_recall_at_10_std": -0.5484061734930179, + "nauc_cv_recall_at_1_diff1": 0.3620563318972254, + "nauc_cv_recall_at_1_max": 0.22304702323816303, + "nauc_cv_recall_at_1_std": -0.39325847713016515, + "nauc_cv_recall_at_20_diff1": 0.3174411644999864, + "nauc_cv_recall_at_20_max": 0.26129067011419665, + "nauc_cv_recall_at_20_std": -0.5873327108621224, + "nauc_cv_recall_at_3_diff1": 0.3190785659299513, + "nauc_cv_recall_at_3_max": 0.22356865681802765, + "nauc_cv_recall_at_3_std": -0.45729779105093976, + "nauc_cv_recall_at_5_diff1": 0.32854938408837603, + "nauc_cv_recall_at_5_max": 0.21763614808463352, + "nauc_cv_recall_at_5_std": -0.4901185955745346, + "nauc_map_at_1000_diff1": 0.21096215655518416, + "nauc_map_at_1000_max": 0.26437483183618576, + "nauc_map_at_1000_std": -0.3656676562792446, + "nauc_map_at_100_diff1": 0.21094575808032542, + "nauc_map_at_100_max": 0.2647084321742531, + "nauc_map_at_100_std": -0.36659141600302986, + "nauc_map_at_10_diff1": 0.20614205326831164, + "nauc_map_at_10_max": 0.25564678162479904, + "nauc_map_at_10_std": -0.3625738173870285, + "nauc_map_at_1_diff1": 0.3621079914040161, + "nauc_map_at_1_max": 0.22314949459519212, + "nauc_map_at_1_std": -0.39289670930715714, + "nauc_map_at_20_diff1": 0.2088651049333622, + "nauc_map_at_20_max": 0.2606844896581087, + "nauc_map_at_20_std": -0.36749366718366866, + "nauc_map_at_3_diff1": 0.23125083203784524, + "nauc_map_at_3_max": 0.23835700021972306, + "nauc_map_at_3_std": -0.3525765196558998, + "nauc_map_at_5_diff1": 0.20395917414865206, + "nauc_map_at_5_max": 0.24890333320816935, + "nauc_map_at_5_std": -0.3551844385945933, + "nauc_mrr_at_1000_diff1": 0.3462109269235247, + "nauc_mrr_at_1000_max": 0.22341427843542805, + "nauc_mrr_at_1000_std": -0.421124236204143, + "nauc_mrr_at_100_diff1": 0.34615985793627224, + "nauc_mrr_at_100_max": 0.2234417011721278, + "nauc_mrr_at_100_std": -0.42112571190402276, + "nauc_mrr_at_10_diff1": 0.3457260635691663, + "nauc_mrr_at_10_max": 0.22268948650151105, + "nauc_mrr_at_10_std": -0.4235752064832113, + "nauc_mrr_at_1_diff1": 0.3620563318972254, + "nauc_mrr_at_1_max": 0.22304702323816303, + "nauc_mrr_at_1_std": -0.39325847713016515, + "nauc_mrr_at_20_diff1": 0.34612102767284886, + "nauc_mrr_at_20_max": 0.22330420489291178, + "nauc_mrr_at_20_std": -0.42260480382387744, + "nauc_mrr_at_3_diff1": 0.3436568869473764, + "nauc_mrr_at_3_max": 0.22268159334592896, + "nauc_mrr_at_3_std": -0.41793547872411135, + "nauc_mrr_at_5_diff1": 0.3462959273283977, + "nauc_mrr_at_5_max": 0.22190940766444717, + "nauc_mrr_at_5_std": -0.42173732217738524, + "nauc_ndcg_at_1000_diff1": 0.24740773356795673, + "nauc_ndcg_at_1000_max": 0.2633139225090684, + "nauc_ndcg_at_1000_std": -0.3825727328439968, + "nauc_ndcg_at_100_diff1": 0.2439247459069718, + "nauc_ndcg_at_100_max": 0.27221683198323254, + "nauc_ndcg_at_100_std": -0.38503378892448387, + "nauc_ndcg_at_10_diff1": 0.2322351990974726, + "nauc_ndcg_at_10_max": 0.25133559217064416, + "nauc_ndcg_at_10_std": -0.38906206049843284, + "nauc_ndcg_at_1_diff1": 0.3620563318972254, + "nauc_ndcg_at_1_max": 0.22304702323816303, + "nauc_ndcg_at_1_std": -0.39325847713016515, + "nauc_ndcg_at_20_diff1": 0.2368585564555519, + "nauc_ndcg_at_20_max": 0.2586236191288364, + "nauc_ndcg_at_20_std": -0.39571963457704934, + "nauc_ndcg_at_3_diff1": 0.25241282801383136, + "nauc_ndcg_at_3_max": 0.23505398892057644, + "nauc_ndcg_at_3_std": -0.37119292314398367, + "nauc_ndcg_at_5_diff1": 0.2325478110853486, + "nauc_ndcg_at_5_max": 0.24393791588243935, + "nauc_ndcg_at_5_std": -0.37684221871528933, + "nauc_precision_at_1000_diff1": -0.060776645601850005, + "nauc_precision_at_1000_max": 0.45061728395062717, + "nauc_precision_at_1000_std": 0.49837165139806966, + "nauc_precision_at_100_diff1": 0.16399269478318815, + "nauc_precision_at_100_max": 0.3174441756422564, + "nauc_precision_at_100_std": -0.2719045017872643, + "nauc_precision_at_10_diff1": 0.1834674594026287, + "nauc_precision_at_10_max": 0.2475111637901415, + "nauc_precision_at_10_std": -0.36964525963118233, + "nauc_precision_at_1_diff1": 0.3620563318972254, + "nauc_precision_at_1_max": 0.22304702323816303, + "nauc_precision_at_1_std": -0.39325847713016515, + "nauc_precision_at_20_diff1": 0.18190648500233034, + "nauc_precision_at_20_max": 0.2543371579459487, + "nauc_precision_at_20_std": -0.3684848720360581, + "nauc_precision_at_3_diff1": 0.2216308838348382, + "nauc_precision_at_3_max": 0.23889515868420422, + "nauc_precision_at_3_std": -0.3624204875957743, + "nauc_precision_at_5_diff1": 0.19290462095741936, + "nauc_precision_at_5_max": 0.2465663979201694, + "nauc_precision_at_5_std": -0.36446145380745226, + "nauc_recall_at_1000_diff1": -0.04813353912994217, + "nauc_recall_at_1000_max": 0.47849854227402444, + "nauc_recall_at_1000_std": 0.5228805808037352, + "nauc_recall_at_100_diff1": 0.16501981261932044, + "nauc_recall_at_100_max": 0.3183331361282548, + "nauc_recall_at_100_std": -0.2726147056630754, + "nauc_recall_at_10_diff1": 0.1835675928057196, + "nauc_recall_at_10_max": 0.24766746744342197, + "nauc_recall_at_10_std": -0.3696577837871595, + "nauc_recall_at_1_diff1": 0.3621079914040161, + "nauc_recall_at_1_max": 0.22314949459519212, + "nauc_recall_at_1_std": -0.39289670930715714, + "nauc_recall_at_20_diff1": 0.18207077982715833, + "nauc_recall_at_20_max": 0.254625446742885, + "nauc_recall_at_20_std": -0.36840440513586076, + "nauc_recall_at_3_diff1": 0.2218119797611259, + "nauc_recall_at_3_max": 0.23903410739410114, + "nauc_recall_at_3_std": -0.3625477824595922, + "nauc_recall_at_5_diff1": 0.19309140539859648, + "nauc_recall_at_5_max": 0.2467762588423194, + "nauc_recall_at_5_std": -0.36457390258550315, + "ndcg_at_1": 0.544, + "ndcg_at_10": 0.48279, + "ndcg_at_100": 0.6064, + "ndcg_at_1000": 0.63735, + "ndcg_at_20": 0.53283, + "ndcg_at_3": 0.47129, + "ndcg_at_5": 0.41454, + "precision_at_1": 0.544, + "precision_at_10": 0.24992, + "precision_at_100": 0.04259, + "precision_at_1000": 0.00496, + "precision_at_20": 0.15413, + "precision_at_3": 0.45027, + "precision_at_5": 0.37584, + "recall_at_1": 0.10886, + "recall_at_10": 0.50001, + "recall_at_100": 0.85215, + "recall_at_1000": 0.99216, + "recall_at_20": 0.6168, + "recall_at_3": 0.27025, + "recall_at_5": 0.37596 + } + ] + }, + "task_name": "MSCOCOI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/STS14.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/STS14.json new file mode 100644 index 0000000000..55c1a7509b --- /dev/null +++ b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/STS14.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", + "evaluation_time": 4.692688703536987, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cosine_pearson": 0.8302122731320196, + "cosine_spearman": 0.7867521693830515, + "euclidean_pearson": 0.8140308674026765, + "euclidean_spearman": 0.7867522096109674, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7867521693830515, + "manhattan_pearson": 0.813071258051059, + "manhattan_spearman": 0.7856621071659424, + "pearson": 0.8302122731320196, + "spearman": 0.7867521693830515 + } + ] + }, + "task_name": "STS14" +} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json new file mode 100644 index 0000000000..53cf522da6 --- /dev/null +++ b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json @@ -0,0 +1 @@ +{"name": "jinaai/jina-clip-v1", "revision": "06150c7c382d7a4faedc7d5a0d8cdb59308968f4", "release_date": "2024-05-30", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "JinaCLIPModelWrapper"} \ No newline at end of file From 874c1bc484e5e15279ac274e2557cfaefa4a2dd7 Mon Sep 17 00:00:00 2001 From: Kenneth Enevoldsen Date: Tue, 29 Oct 2024 15:47:53 +0100 Subject: [PATCH 087/154] fix: Add clevr license (#1356) --- mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py b/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py index c96c2727c4..9b7397e24b 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py @@ -23,7 +23,7 @@ class CLEVR(AbsTaskZeroshotClassification): date=("2016-01-01", "2016-12-20"), domains=["Constructed"], task_subtypes=["Object recognition"], - license="not specified", + license="cc-by-4.0", annotations_creators="human-annotated", dialect=[], modalities=["text", "image"], From cf8ea1f3a1519e62c3105c78979dc495d88c2d6d Mon Sep 17 00:00:00 2001 From: Jamie-Stirling <36764530+Jamie-Stirling@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:54:10 +0000 Subject: [PATCH 088/154] Add BLINK as multi-choice tasks (#1348) * wip: start adding BLIP models * add other blip variants * wip: add blip2_models.py * make lint * wip: implement blip2 wrapper * feat: add blip2 models, still mismatched names * fix: remove projections from image and text embeddings * make lint * wip: add coco BLIP2 * fix: BLIP2 better zero-shot classification without text_proj and vision_proj * tidy blip2 * add imagenet-dog-15 dataset * tidy and lint * remove unused import * add cluster_accuracy, ari and nmi to Image.ClusteringEvaluator * add imagenet-10 clustering task * add SOPI2IRetrieval * add results forclip on ImageNet10Clustering and ImageNetDog15Clustering * add SOPI2IRetrieval results for clip 32 * add results for clip vit 32/SOPI2IRetrieval * resolve conflict * add RP2kI2IRetrieval dataset * add RP2kI2IRetrieval results with clip-vit-base-patch32 * update image retrieval __init__.py * fix ImageTextPair dataloading for large datasets; more compositionality evaluation datasets * add RP2kI2IRetrieval and METI2IRetrieval * add METI2IRetreival * add SOP results * make lign * new revision for METI2IRetrieval * make lint * reset corpus chunk size * remove wrong classification import * add Flickr30k T2I and I2T * add Flickr30k T2I retriebal * reduced-size MET revision * fix: add Flickr30k T2I * make lint * add two landmark datasets and results * add Sketchy i2i retrieval * add task metadata * add BLINKIT2IRetrieval dataset * add BLINKIT2TRetrieval * add ImageCoDeT2IRetrieval * make lint * add vizwiz retrieval and results * fix vizwiz duplicate texts * add new vizwiz results * add VQA2 results * add GLD v2 I2T retrieval * add gld v2 i2i retrieval * make lint * add AbsTaskAny2AnyMultiChoice * make lint * remove GLDv2I2IRetrieval * exclude AbsTaskAny2AnyMultiChoice from test_load_data * fix e5v&vista * remove duplicate corpus entries from BLINKIT2TRetreival dataset * task type fix for running tasks * update BLINKIT2T metadata * fix wrong meta * run mieb script * split ROxford, RParis into easy, medium and hard * make lint * add BLINK as multi choice tasks * fix: license metadata in wrong format --------- Co-authored-by: gowitheflow-1998 --- .../Image/Any2AnyMultiChoice/__init__.py | 2 + .../eng/BLINKIT2IMultiChoice.py | 49 +++++++++++++++++++ .../eng/BLINKIT2TMultiChoice.py | 48 ++++++++++++++++++ .../eng/ImageCoDeT2IMultiChoice.py | 4 +- .../BLINKIT2IMultiChoice.json | 33 +++++++++++++ .../BLINKIT2TMultiChoice.json | 33 +++++++++++++ 6 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py create mode 100644 mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py b/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py index b317e8cabd..c818af7048 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py @@ -1,3 +1,5 @@ from __future__ import annotations +from .eng.BLINKIT2IMultiChoice import * +from .eng.BLINKIT2TMultiChoice import * from .eng.ImageCoDeT2IMultiChoice import * diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py new file mode 100644 index 0000000000..d600eaa4f2 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class BLINKIT2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="BLINKIT2IMultiChoice", + description="Retrieve images based on images and specific retrieval instructions.", + reference="https://arxiv.org/abs/2404.12390", + dataset={ + "path": "JamieSJS/blink-it2i-multi", + "revision": "780ade70cd769e586502a61dda903e525f945a45", + "trust_remote_code": True, + }, + type="Any2AnyMultiChoice", + category="it2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2018-01-01", "2018-12-31"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{fu2024blink, + title={Blink: Multimodal large language models can see but not perceive}, + author={Fu, Xingyu and Hu, Yushi and Li, Bangzheng and Feng, Yu and Wang, Haoyu and Lin, Xudong and Roth, Dan and Smith, Noah A and Ma, Wei-Chiu and Krishna, Ranjay}, + journal={arXiv preprint arXiv:2404.12390}, + year={2024} +} +""", + descriptive_stats={ + "n_samples": {"test": 402}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 804, + "num_queries": 402, + "average_relevant_docs_per_query": 1, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py new file mode 100644 index 0000000000..fe37216de0 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class BLINKIT2TMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="BLINKIT2TMultiChoice", + description="Retrieve the correct text answer based on images and specific retrieval instructions.", + reference="https://arxiv.org/abs/2404.12390", + dataset={ + "path": "JamieSJS/blink-it2t-multi", + "revision": "b6e18eba186cada040ddb72e8e3cb92edd7ca5e9", + }, + type="Any2AnyMultiChoice", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2018-01-01", "2018-12-31"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@article{fu2024blink, + title={Blink: Multimodal large language models can see but not perceive}, + author={Fu, Xingyu and Hu, Yushi and Li, Bangzheng and Feng, Yu and Wang, Haoyu and Lin, Xudong and Roth, Dan and Smith, Noah A and Ma, Wei-Chiu and Krishna, Ranjay}, + journal={arXiv preprint arXiv:2404.12390}, + year={2024} +} +""", + descriptive_stats={ + "n_samples": {"test": 1073}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 26, + "num_queries": 1073, + "average_relevant_docs_per_query": 1, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py index 7c9ab2e721..3cb875b845 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py @@ -14,10 +14,10 @@ class ImageCoDeT2IMultiChoice(AbsTaskAny2AnyMultiChoice): "revision": "d28adfd8b34fefa546fdf94bdc352622b2575f6c", }, type="Any2AnyMultiChoice", - category="t2i", + category="it2i", eval_splits=["test"], eval_langs=["eng-Latn"], - main_score="ndcg_at_1", + main_score="accuracy", date=("2022-05-22", "2022-05-27"), # conference dates domains=["Web", "Written"], task_subtypes=["Image Text Retrieval"], diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json new file mode 100644 index 0000000000..648d3aa59e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json @@ -0,0 +1,33 @@ +{ + "dataset_revision": "780ade70cd769e586502a61dda903e525f945a45", + "evaluation_time": 56.62301731109619, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.70149, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.70149, + "mrr_at_1": 0.7014925373134329, + "mrr_at_10": 0.8507462686567164, + "mrr_at_100": 0.8507462686567164, + "mrr_at_1000": 0.8507462686567164, + "mrr_at_20": 0.8507462686567164, + "mrr_at_3": 0.8507462686567164, + "mrr_at_5": 0.8507462686567164, + "ndcg_at_1": 0.70149, + "ndcg_at_10": 0.88983, + "ndcg_at_100": 0.88983, + "ndcg_at_1000": 0.88983, + "ndcg_at_20": 0.88983, + "ndcg_at_3": 0.88983, + "ndcg_at_5": 0.88983 + } + ] + }, + "task_name": "BLINKIT2IMultiChoice" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json new file mode 100644 index 0000000000..bda6cd2cb9 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json @@ -0,0 +1,33 @@ +{ + "dataset_revision": "b6e18eba186cada040ddb72e8e3cb92edd7ca5e9", + "evaluation_time": 43.71325731277466, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.38397, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.38397, + "mrr_at_1": 0.38397017707362535, + "mrr_at_10": 0.650512581547066, + "mrr_at_100": 0.650512581547066, + "mrr_at_1000": 0.650512581547066, + "mrr_at_20": 0.650512581547066, + "mrr_at_3": 0.6295433364398889, + "mrr_at_5": 0.650512581547066, + "ndcg_at_1": 0.38397, + "ndcg_at_10": 0.73974, + "ndcg_at_100": 0.73974, + "ndcg_at_1000": 0.73974, + "ndcg_at_20": 0.73974, + "ndcg_at_3": 0.70361, + "ndcg_at_5": 0.73974 + } + ] + }, + "task_name": "BLINKIT2TMultiChoice" +} \ No newline at end of file From 6652e56b4c1b6ebe991615efdbae73d1ba468c29 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Thu, 31 Oct 2024 18:20:47 +0200 Subject: [PATCH 089/154] [mieb] add Eva CLIP models (#1369) * add Eva CLIP models * make lint --- mteb/models/evaclip_models.py | 198 ++++++++++++++++++ mteb/models/overview.py | 2 + .../MNIST.json | 28 +++ .../MSCOCOI2TRetrieval.json | 186 ++++++++++++++++ .../model_meta.json | 1 + .../MSCOCOI2TRetrieval.json | 186 ++++++++++++++++ .../model_meta.json | 1 + .../model_meta.json | 1 + 8 files changed, 603 insertions(+) create mode 100644 mteb/models/evaclip_models.py create mode 100644 results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MNIST.json create mode 100644 results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json create mode 100644 results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json create mode 100644 results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json create mode 100644 results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json create mode 100644 results-mieb/EVA02-CLIP-bigE-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json diff --git a/mteb/models/evaclip_models.py b/mteb/models/evaclip_models.py new file mode 100644 index 0000000000..1e123096ca --- /dev/null +++ b/mteb/models/evaclip_models.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm + +from mteb.model_meta import ModelMeta + + +def evaclip_loader(**kwargs): + try: + import sys + import os + + sys.path.insert(0, os.path.join(os.getcwd(), "EVA/EVA-CLIP/rei")) + + from eva_clip import create_model_and_transforms, get_tokenizer + except ImportError: + # https://github.com/baaivision/EVA/tree/master/EVA-CLIP#setup + raise ImportError( + "Please run `git clone git@github.com:baaivision/EVA.git`," + "`pip install ninja`" + "`pip install -v -U git+https://github.com/facebookresearch/xformers.git@main#egg=xformers`" + "`git clone https://github.com/NVIDIA/apex && cd apex && pip install -v --disable-pip-version-check --no-build-isolation --no-cache-dir ./`" + ) + + class EvaCLIPWrapper: + def __init__( + self, + model_name: str = "EVA02-CLIP-B-16", + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + pretrained = "eva_clip" # or "/path/to/EVA02_CLIP_B_psz16_s8B.pt" + self.model, _, self.img_preprocess = create_model_and_transforms( + model_name, pretrained, force_custom_clip=True, device=device + ) + self.model.eval() + self.tokenizer = get_tokenizer(model_name) + + def encode( # type: ignore + self, + sentences: list[str], + *, + batch_size: int = 32, + **kwargs: Any, + ): + return self.get_text_embeddings(texts=sentences, batch_size=batch_size) + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(), torch.cuda.amp.autocast(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + inputs = self.tokenizer(batch_texts) + text_outputs = self.model.encode_text(inputs.to(self.device)) + all_text_embeddings.append(text_outputs.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + all_image_embeddings = [] + if isinstance(images, DataLoader): + import torchvision.transforms.functional as F + + with torch.no_grad(), torch.cuda.amp.autocast(): + for batch in tqdm(images): + # import pdb; pdb.set_trace() + inputs = torch.vstack( + [ + self.img_preprocess(F.to_pil_image(b)).unsqueeze(0) + for b in batch + ] + ) + image_outputs = self.model.encode_image(inputs.to(self.device)) + all_image_embeddings.append(image_outputs.cpu()) + else: + with torch.no_grad(), torch.cuda.amp.autocast(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + inputs = torch.vstack( + [self.img_preprocess(b) for b in batch_images] + ) + image_outputs = self.model.encode_image(inputs.to(self.device)) + all_image_embeddings.append(image_outputs.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm( + dim=-1, keepdim=True + ) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + else: + # to do: add other fusion mode + raise ValueError( + f"fusion mode {fusion_mode} hasn't been implemented" + ) + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + return EvaCLIPWrapper(**kwargs) + + +EVA02_CLIP_B_16 = ModelMeta( + loader=partial( + evaclip_loader, + model_name="EVA02-CLIP-B-16", + ), + name="EVA02-CLIP-B-16", + languages=["eng_Latn"], + open_source=True, + revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", + release_date="2023-04-26", +) + +EVA02_CLIP_L_14 = ModelMeta( + loader=partial( + evaclip_loader, + model_name="EVA02-CLIP-L-14", + ), + name="EVA02-CLIP-L-14", + languages=["eng_Latn"], + open_source=True, + revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", + release_date="2023-04-26", +) + +EVA02_CLIP_bigE_14 = ModelMeta( + loader=partial( + evaclip_loader, + model_name="EVA02-CLIP-bigE-14", + ), + name="EVA02-CLIP-bigE-14", + languages=["eng_Latn"], + open_source=True, + revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", + release_date="2023-04-26", +) + + +EVA02_CLIP_bigE_14_plus = ModelMeta( + loader=partial( + evaclip_loader, + model_name="EVA02-CLIP-bigE-14-plus", + ), + name="EVA02-CLIP-bigE-14-plus", + languages=["eng_Latn"], + open_source=True, + revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", + release_date="2023-04-26", +) diff --git a/mteb/models/overview.py b/mteb/models/overview.py index 70768639bd..37358ea775 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -20,6 +20,7 @@ e5_instruct, e5_models, e5_v, + evaclip_models, google_models, gritlm_models, gte_models, @@ -57,6 +58,7 @@ e5_instruct, e5_models, e5_v, + evaclip_models, google_models, gritlm_models, gte_models, diff --git a/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MNIST.json b/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MNIST.json new file mode 100644 index 0000000000..3c6bf4be39 --- /dev/null +++ b/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MNIST.json @@ -0,0 +1,28 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 36.933613777160645, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.7848, + "f1": 0.7815922902217035, + "f1_weighted": 0.7830608860261875, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7848, + "scores_per_experiment": [ + { + "accuracy": 0.7848, + "f1": 0.7815922902217035, + "f1_weighted": 0.7830608860261875 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json b/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json new file mode 100644 index 0000000000..243b9cdfb3 --- /dev/null +++ b/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", + "evaluation_time": 123.7929699420929, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.5808, + "cv_recall_at_10": 0.8748, + "cv_recall_at_100": 0.9944, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.9314, + "cv_recall_at_3": 0.7374, + "cv_recall_at_5": 0.7984, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.51767, + "map_at_1": 0.1164, + "map_at_10": 0.4046, + "map_at_100": 0.45014, + "map_at_1000": 0.45303, + "map_at_20": 0.43031, + "map_at_3": 0.26852, + "map_at_5": 0.35112, + "mrr_at_1": 0.5808, + "mrr_at_10": 0.6744126190476204, + "mrr_at_100": 0.6801586440797971, + "mrr_at_1000": 0.6801960917779327, + "mrr_at_20": 0.6784373254841141, + "mrr_at_3": 0.6501000000000023, + "mrr_at_5": 0.6640700000000009, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.18012204881955546, + "nauc_cv_recall_at_100_max": 0.4690209417100159, + "nauc_cv_recall_at_100_std": -0.28182939842600496, + "nauc_cv_recall_at_10_diff1": 0.40434217822752977, + "nauc_cv_recall_at_10_max": 0.4848002316155178, + "nauc_cv_recall_at_10_std": -0.48703213512130683, + "nauc_cv_recall_at_1_diff1": 0.4279641274466745, + "nauc_cv_recall_at_1_max": 0.40462849354795966, + "nauc_cv_recall_at_1_std": -0.3439104150545278, + "nauc_cv_recall_at_20_diff1": 0.40948624347698215, + "nauc_cv_recall_at_20_max": 0.4677421989203846, + "nauc_cv_recall_at_20_std": -0.49209888036847343, + "nauc_cv_recall_at_3_diff1": 0.39274864442318025, + "nauc_cv_recall_at_3_max": 0.43154140979989203, + "nauc_cv_recall_at_3_std": -0.41178966708778997, + "nauc_cv_recall_at_5_diff1": 0.3789987569351505, + "nauc_cv_recall_at_5_max": 0.441581412804759, + "nauc_cv_recall_at_5_std": -0.4429600034948794, + "nauc_map_at_1000_diff1": 0.2551299778398228, + "nauc_map_at_1000_max": 0.45024216293039104, + "nauc_map_at_1000_std": -0.3158410835982976, + "nauc_map_at_100_diff1": 0.2551180137900688, + "nauc_map_at_100_max": 0.4507781940007651, + "nauc_map_at_100_std": -0.3165456782869218, + "nauc_map_at_10_diff1": 0.2470592559213462, + "nauc_map_at_10_max": 0.44129809055151087, + "nauc_map_at_10_std": -0.31273315656889844, + "nauc_map_at_1_diff1": 0.42575307120531464, + "nauc_map_at_1_max": 0.4048588519120006, + "nauc_map_at_1_std": -0.34378381546020276, + "nauc_map_at_20_diff1": 0.2516873789249498, + "nauc_map_at_20_max": 0.4484210309890165, + "nauc_map_at_20_std": -0.31683283226512, + "nauc_map_at_3_diff1": 0.2826357897945762, + "nauc_map_at_3_max": 0.42426410796847175, + "nauc_map_at_3_std": -0.30647778109153573, + "nauc_map_at_5_diff1": 0.24811920497848947, + "nauc_map_at_5_max": 0.43227047102448485, + "nauc_map_at_5_std": -0.30545952750772826, + "nauc_mrr_at_1000_diff1": 0.41413837092161987, + "nauc_mrr_at_1000_max": 0.41676194427293417, + "nauc_mrr_at_1000_std": -0.37106779073618856, + "nauc_mrr_at_100_diff1": 0.4140964307246391, + "nauc_mrr_at_100_max": 0.4167634806381506, + "nauc_mrr_at_100_std": -0.37107069460512243, + "nauc_mrr_at_10_diff1": 0.4139845884692961, + "nauc_mrr_at_10_max": 0.41838787547148604, + "nauc_mrr_at_10_std": -0.373368042710646, + "nauc_mrr_at_1_diff1": 0.4279641274466745, + "nauc_mrr_at_1_max": 0.40462849354795966, + "nauc_mrr_at_1_std": -0.3439104150545278, + "nauc_mrr_at_20_diff1": 0.41422005719482136, + "nauc_mrr_at_20_max": 0.4169421834449322, + "nauc_mrr_at_20_std": -0.37195479331593423, + "nauc_mrr_at_3_diff1": 0.4129850334584388, + "nauc_mrr_at_3_max": 0.4153387916886552, + "nauc_mrr_at_3_std": -0.36938113677751916, + "nauc_mrr_at_5_diff1": 0.41133587690415013, + "nauc_mrr_at_5_max": 0.41658201667599837, + "nauc_mrr_at_5_std": -0.3726272419181216, + "nauc_ndcg_at_1000_diff1": 0.30085799504287825, + "nauc_ndcg_at_1000_max": 0.45175503659717203, + "nauc_ndcg_at_1000_std": -0.33497432412179057, + "nauc_ndcg_at_100_diff1": 0.2971007796129066, + "nauc_ndcg_at_100_max": 0.45770544154865983, + "nauc_ndcg_at_100_std": -0.33709187267575114, + "nauc_ndcg_at_10_diff1": 0.2796457903219738, + "nauc_ndcg_at_10_max": 0.4445160485969821, + "nauc_ndcg_at_10_std": -0.3377133809957326, + "nauc_ndcg_at_1_diff1": 0.4258548467189445, + "nauc_ndcg_at_1_max": 0.40505598965450235, + "nauc_ndcg_at_1_std": -0.3441811229562816, + "nauc_ndcg_at_20_diff1": 0.2865070753592145, + "nauc_ndcg_at_20_max": 0.4537504419231763, + "nauc_ndcg_at_20_std": -0.34287699080243245, + "nauc_ndcg_at_3_diff1": 0.3084620615728682, + "nauc_ndcg_at_3_max": 0.42489968327164407, + "nauc_ndcg_at_3_std": -0.32533589756821285, + "nauc_ndcg_at_5_diff1": 0.2802294933259117, + "nauc_ndcg_at_5_max": 0.43204705198340854, + "nauc_ndcg_at_5_std": -0.3296170450292348, + "nauc_precision_at_1000_diff1": 0.07501926945277763, + "nauc_precision_at_1000_max": 0.34712587495850716, + "nauc_precision_at_1000_std": 0.47869080518110013, + "nauc_precision_at_100_diff1": 0.2148427857809954, + "nauc_precision_at_100_max": 0.43306705779365773, + "nauc_precision_at_100_std": -0.25062075823160657, + "nauc_precision_at_10_diff1": 0.22474307723586737, + "nauc_precision_at_10_max": 0.43423322714260504, + "nauc_precision_at_10_std": -0.3212543682360079, + "nauc_precision_at_1_diff1": 0.4258548467189445, + "nauc_precision_at_1_max": 0.40505598965450235, + "nauc_precision_at_1_std": -0.3441811229562816, + "nauc_precision_at_20_diff1": 0.22416005971431854, + "nauc_precision_at_20_max": 0.4371201909814698, + "nauc_precision_at_20_std": -0.31708914748633776, + "nauc_precision_at_3_diff1": 0.2745706041175571, + "nauc_precision_at_3_max": 0.4278350852940981, + "nauc_precision_at_3_std": -0.31711638180222146, + "nauc_precision_at_5_diff1": 0.2349320970775645, + "nauc_precision_at_5_max": 0.42831671123295334, + "nauc_precision_at_5_std": -0.3190019134502589, + "nauc_recall_at_1000_diff1": 0.10507922975953841, + "nauc_recall_at_1000_max": 0.3930871865654486, + "nauc_recall_at_1000_std": 0.5020094994519381, + "nauc_recall_at_100_diff1": 0.21661240783376975, + "nauc_recall_at_100_max": 0.43497626991291494, + "nauc_recall_at_100_std": -0.25185551819463464, + "nauc_recall_at_10_diff1": 0.22519163965451483, + "nauc_recall_at_10_max": 0.43446322405629173, + "nauc_recall_at_10_std": -0.32141360644158223, + "nauc_recall_at_1_diff1": 0.42575307120531464, + "nauc_recall_at_1_max": 0.4048588519120006, + "nauc_recall_at_1_std": -0.34378381546020276, + "nauc_recall_at_20_diff1": 0.22475741497205412, + "nauc_recall_at_20_max": 0.43757220630004384, + "nauc_recall_at_20_std": -0.31738515586158633, + "nauc_recall_at_3_diff1": 0.2746377320730141, + "nauc_recall_at_3_max": 0.42775643120193435, + "nauc_recall_at_3_std": -0.31711052977780496, + "nauc_recall_at_5_diff1": 0.23517626729390337, + "nauc_recall_at_5_max": 0.4284668289611237, + "nauc_recall_at_5_std": -0.3189996126749059, + "ndcg_at_1": 0.5816, + "ndcg_at_10": 0.51767, + "ndcg_at_100": 0.6347, + "ndcg_at_1000": 0.66318, + "ndcg_at_20": 0.56556, + "ndcg_at_3": 0.50923, + "ndcg_at_5": 0.44845, + "precision_at_1": 0.5816, + "precision_at_10": 0.2667, + "precision_at_100": 0.04328, + "precision_at_1000": 0.00497, + "precision_at_20": 0.16118, + "precision_at_3": 0.4874, + "precision_at_5": 0.40752, + "recall_at_1": 0.1164, + "recall_at_10": 0.53367, + "recall_at_100": 0.86597, + "recall_at_1000": 0.99448, + "recall_at_20": 0.64502, + "recall_at_3": 0.29261, + "recall_at_5": 0.40775 + } + ] + }, + "task_name": "MSCOCOI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json b/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json new file mode 100644 index 0000000000..a639fc4d52 --- /dev/null +++ b/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json @@ -0,0 +1 @@ +{"name": "EVA02-CLIP-B-16", "revision": "11afd202f2ae80869d6cef18b1ec775e79bd8d12", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "evaclip_loader"} \ No newline at end of file diff --git a/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json b/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json new file mode 100644 index 0000000000..fad26b40d9 --- /dev/null +++ b/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", + "evaluation_time": 188.94002294540405, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.6288, + "cv_recall_at_10": 0.8992, + "cv_recall_at_100": 0.9956, + "cv_recall_at_1000": 0.9998, + "cv_recall_at_20": 0.9472, + "cv_recall_at_3": 0.7738, + "cv_recall_at_5": 0.8342, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56069, + "map_at_1": 0.12554, + "map_at_10": 0.44873, + "map_at_100": 0.4927, + "map_at_1000": 0.49528, + "map_at_20": 0.47383, + "map_at_3": 0.29637, + "map_at_5": 0.39313, + "mrr_at_1": 0.6288, + "mrr_at_10": 0.7158076984126989, + "mrr_at_100": 0.7205783122922634, + "mrr_at_1000": 0.7206069527616489, + "mrr_at_20": 0.7192201942055193, + "mrr_at_3": 0.6930666666666684, + "mrr_at_5": 0.7068966666666667, + "nauc_cv_recall_at_1000_diff1": 0.12278244631152727, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.46403106697222163, + "nauc_cv_recall_at_100_max": 0.729055258466997, + "nauc_cv_recall_at_100_std": -0.30457940752058904, + "nauc_cv_recall_at_10_diff1": 0.3904889248253057, + "nauc_cv_recall_at_10_max": 0.5221902215034939, + "nauc_cv_recall_at_10_std": -0.5924430292420227, + "nauc_cv_recall_at_1_diff1": 0.44292967763077234, + "nauc_cv_recall_at_1_max": 0.47715154630664924, + "nauc_cv_recall_at_1_std": -0.40620411297806974, + "nauc_cv_recall_at_20_diff1": 0.43476607814843, + "nauc_cv_recall_at_20_max": 0.5567421413009648, + "nauc_cv_recall_at_20_std": -0.5714798545680903, + "nauc_cv_recall_at_3_diff1": 0.4261256966503181, + "nauc_cv_recall_at_3_max": 0.49484976574587286, + "nauc_cv_recall_at_3_std": -0.4813401812617564, + "nauc_cv_recall_at_5_diff1": 0.39770539047919967, + "nauc_cv_recall_at_5_max": 0.4974364426662427, + "nauc_cv_recall_at_5_std": -0.5283547497680571, + "nauc_map_at_1000_diff1": 0.23277589985950517, + "nauc_map_at_1000_max": 0.4750327210959742, + "nauc_map_at_1000_std": -0.3607727535310882, + "nauc_map_at_100_diff1": 0.23262810652639354, + "nauc_map_at_100_max": 0.475256107901075, + "nauc_map_at_100_std": -0.36143324382132475, + "nauc_map_at_10_diff1": 0.22389555975001613, + "nauc_map_at_10_max": 0.46963591043048414, + "nauc_map_at_10_std": -0.3557278665509837, + "nauc_map_at_1_diff1": 0.44545079244943503, + "nauc_map_at_1_max": 0.47785301021506615, + "nauc_map_at_1_std": -0.4057042536024991, + "nauc_map_at_20_diff1": 0.22998051862002486, + "nauc_map_at_20_max": 0.47305652893594374, + "nauc_map_at_20_std": -0.3603168960770869, + "nauc_map_at_3_diff1": 0.27812725553418577, + "nauc_map_at_3_max": 0.47306973520598594, + "nauc_map_at_3_std": -0.3570706186087874, + "nauc_map_at_5_diff1": 0.2280217674026679, + "nauc_map_at_5_max": 0.4682093275293784, + "nauc_map_at_5_std": -0.3423580014696908, + "nauc_mrr_at_1000_diff1": 0.4353997803612837, + "nauc_mrr_at_1000_max": 0.4829466674716431, + "nauc_mrr_at_1000_std": -0.43724506589884127, + "nauc_mrr_at_100_diff1": 0.4354030270991679, + "nauc_mrr_at_100_max": 0.48297038396857644, + "nauc_mrr_at_100_std": -0.4372610054743306, + "nauc_mrr_at_10_diff1": 0.4339442584396937, + "nauc_mrr_at_10_max": 0.48330696344702784, + "nauc_mrr_at_10_std": -0.4407280111594318, + "nauc_mrr_at_1_diff1": 0.44292967763077234, + "nauc_mrr_at_1_max": 0.47715154630664924, + "nauc_mrr_at_1_std": -0.40620411297806974, + "nauc_mrr_at_20_diff1": 0.435424981370453, + "nauc_mrr_at_20_max": 0.483314664468245, + "nauc_mrr_at_20_std": -0.43817326450406274, + "nauc_mrr_at_3_diff1": 0.4367712306037216, + "nauc_mrr_at_3_max": 0.4825955486267395, + "nauc_mrr_at_3_std": -0.4343397489713263, + "nauc_mrr_at_5_diff1": 0.4330194072228175, + "nauc_mrr_at_5_max": 0.48215440797790354, + "nauc_mrr_at_5_std": -0.4393099993887148, + "nauc_ndcg_at_1000_diff1": 0.286919936116174, + "nauc_ndcg_at_1000_max": 0.48455380332723746, + "nauc_ndcg_at_1000_std": -0.3889019114242068, + "nauc_ndcg_at_100_diff1": 0.281607082470054, + "nauc_ndcg_at_100_max": 0.4862363359092808, + "nauc_ndcg_at_100_std": -0.39104186341881614, + "nauc_ndcg_at_10_diff1": 0.25974640251443, + "nauc_ndcg_at_10_max": 0.4746176277404106, + "nauc_ndcg_at_10_std": -0.3877117875687834, + "nauc_ndcg_at_1_diff1": 0.44645817200139365, + "nauc_ndcg_at_1_max": 0.47805516562130335, + "nauc_ndcg_at_1_std": -0.40610800775079414, + "nauc_ndcg_at_20_diff1": 0.2696500915937151, + "nauc_ndcg_at_20_max": 0.47845385412507335, + "nauc_ndcg_at_20_std": -0.39360516882367463, + "nauc_ndcg_at_3_diff1": 0.30847439164717777, + "nauc_ndcg_at_3_max": 0.4761874477195775, + "nauc_ndcg_at_3_std": -0.3776639638945563, + "nauc_ndcg_at_5_diff1": 0.26569027870667994, + "nauc_ndcg_at_5_max": 0.47347834616528645, + "nauc_ndcg_at_5_std": -0.37106994299369317, + "nauc_precision_at_1000_diff1": 0.1236733225400447, + "nauc_precision_at_1000_max": 0.308697179177513, + "nauc_precision_at_1000_std": 0.40753732685737887, + "nauc_precision_at_100_diff1": 0.19092682607623723, + "nauc_precision_at_100_max": 0.4250180546261426, + "nauc_precision_at_100_std": -0.30733698696941303, + "nauc_precision_at_10_diff1": 0.19311336478366353, + "nauc_precision_at_10_max": 0.44355904903183346, + "nauc_precision_at_10_std": -0.36759162233395865, + "nauc_precision_at_1_diff1": 0.44645817200139365, + "nauc_precision_at_1_max": 0.47805516562130335, + "nauc_precision_at_1_std": -0.40610800775079414, + "nauc_precision_at_20_diff1": 0.19966070829030882, + "nauc_precision_at_20_max": 0.4326136796798698, + "nauc_precision_at_20_std": -0.3669436772279273, + "nauc_precision_at_3_diff1": 0.2708800654517302, + "nauc_precision_at_3_max": 0.4729051262633524, + "nauc_precision_at_3_std": -0.36661790608555717, + "nauc_precision_at_5_diff1": 0.21323688684871933, + "nauc_precision_at_5_max": 0.4617539145838835, + "nauc_precision_at_5_std": -0.3535506470153617, + "nauc_recall_at_1000_diff1": 0.10032584462359877, + "nauc_recall_at_1000_max": 0.35548028735302944, + "nauc_recall_at_1000_std": 0.42345509632425254, + "nauc_recall_at_100_diff1": 0.19003462550964206, + "nauc_recall_at_100_max": 0.42683017998778644, + "nauc_recall_at_100_std": -0.3088713370152116, + "nauc_recall_at_10_diff1": 0.19275938507453244, + "nauc_recall_at_10_max": 0.44376915738937206, + "nauc_recall_at_10_std": -0.3680795761536703, + "nauc_recall_at_1_diff1": 0.44545079244943503, + "nauc_recall_at_1_max": 0.47785301021506615, + "nauc_recall_at_1_std": -0.4057042536024991, + "nauc_recall_at_20_diff1": 0.1990748921216041, + "nauc_recall_at_20_max": 0.4329712829247703, + "nauc_recall_at_20_std": -0.3674964699732755, + "nauc_recall_at_3_diff1": 0.2706407468460583, + "nauc_recall_at_3_max": 0.47295729285142457, + "nauc_recall_at_3_std": -0.3667117594872674, + "nauc_recall_at_5_diff1": 0.2129174058601686, + "nauc_recall_at_5_max": 0.4618647334514804, + "nauc_recall_at_5_std": -0.3537533728865379, + "ndcg_at_1": 0.6274, + "ndcg_at_10": 0.56069, + "ndcg_at_100": 0.67034, + "ndcg_at_1000": 0.695, + "ndcg_at_20": 0.60649, + "ndcg_at_3": 0.55439, + "ndcg_at_5": 0.49188, + "precision_at_1": 0.6274, + "precision_at_10": 0.28744, + "precision_at_100": 0.04423, + "precision_at_1000": 0.00498, + "precision_at_20": 0.17036, + "precision_at_3": 0.53253, + "precision_at_5": 0.44992, + "recall_at_1": 0.12554, + "recall_at_10": 0.57515, + "recall_at_100": 0.88496, + "recall_at_1000": 0.99608, + "recall_at_20": 0.68176, + "recall_at_3": 0.31966, + "recall_at_5": 0.45013 + } + ] + }, + "task_name": "MSCOCOI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json b/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json new file mode 100644 index 0000000000..038364051d --- /dev/null +++ b/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json @@ -0,0 +1 @@ +{"name": "EVA02-CLIP-L-14", "revision": "11afd202f2ae80869d6cef18b1ec775e79bd8d12", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "evaclip_loader"} \ No newline at end of file diff --git a/results-mieb/EVA02-CLIP-bigE-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json b/results-mieb/EVA02-CLIP-bigE-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json new file mode 100644 index 0000000000..5f33638645 --- /dev/null +++ b/results-mieb/EVA02-CLIP-bigE-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json @@ -0,0 +1 @@ +{"name": "EVA02-CLIP-bigE-14", "revision": "11afd202f2ae80869d6cef18b1ec775e79bd8d12", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "evaclip_loader"} \ No newline at end of file From 9b178e60a1cc3ecdf703f3dc5642484121c600c2 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Fri, 1 Nov 2024 00:33:41 +0800 Subject: [PATCH 090/154] [mieb] add siglip, cohere multimodal & some fixes for final run (#1357) * fix dataset type error * fix clustering metrics * add siglip & cohere * update mieb run script * cohere-v import * fix * api key name --- .../evaluators/Image/ClusteringEvaluator.py | 3 - mteb/models/cohere_v.py | 202 ++ mteb/models/overview.py | 4 + mteb/models/siglip_models.py | 243 +++ .../eng/VidoreBenchRetrieval.py | 18 +- mteb/tasks/Image/Clustering/eng/CIFAR.py | 6 +- mteb/tasks/Image/Clustering/eng/ImageNet.py | 6 +- .../Image/Clustering/eng/TinyImageNet.py | 2 +- .../Image/ImageClassification/eng/CIFAR.py | 2 +- .../eng/PatchCamelyon.py | 2 +- .../1/ImageNetDog15Clustering.json | 23 + .../embed-english-v3.0-v/1/model_meta.json | 1 + .../WITT2IRetrieval.json | 1936 +++++++++++++++++ .../model_meta.json | 1 + .../WITT2IRetrieval.json | 1936 +++++++++++++++++ .../model_meta.json | 1 + .../WITT2IRetrieval.json | 1936 +++++++++++++++++ .../model_meta.json | 1 + .../WITT2IRetrieval.json | 1936 +++++++++++++++++ .../model_meta.json | 1 + .../WITT2IRetrieval.json | 1936 +++++++++++++++++ .../model_meta.json | 1 + .../WITT2IRetrieval.json | 1936 +++++++++++++++++ .../model_meta.json | 1 + .../WITT2IRetrieval.json | 1936 +++++++++++++++++ .../model_meta.json | 1 + .../VidoreArxivQARetrieval.json | 186 ++ .../VidoreDocVQARetrieval.json | 186 ++ .../VidoreInfoVQARetrieval.json | 186 ++ .../VidoreShiftProjectRetrieval.json | 186 ++ .../VidoreSyntheticDocQAAIRetrieval.json | 186 ++ .../VidoreSyntheticDocQAEnergyRetrieval.json | 186 ++ ...theticDocQAGovernmentReportsRetrieval.json | 186 ++ ...heticDocQAHealthcareIndustryRetrieval.json | 186 ++ .../VidoreTabfquadRetrieval.json | 186 ++ .../VidoreTatdqaRetrieval.json | 186 ++ .../WITT2IRetrieval.json | 1936 +++++++++++++++++ .../model_meta.json | 1 + scripts/run_mieb.py | 26 +- 39 files changed, 17871 insertions(+), 23 deletions(-) create mode 100644 mteb/models/cohere_v.py create mode 100644 mteb/models/siglip_models.py create mode 100644 results-mieb/embed-english-v3.0-v/1/ImageNetDog15Clustering.json create mode 100644 results-mieb/embed-english-v3.0-v/1/model_meta.json create mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/WITT2IRetrieval.json create mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/model_meta.json create mode 100644 results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/WITT2IRetrieval.json create mode 100644 results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/model_meta.json create mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/WITT2IRetrieval.json create mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/model_meta.json create mode 100644 results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/WITT2IRetrieval.json create mode 100644 results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/model_meta.json create mode 100644 results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/WITT2IRetrieval.json create mode 100644 results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/model_meta.json create mode 100644 results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/WITT2IRetrieval.json create mode 100644 results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/model_meta.json create mode 100644 results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/WITT2IRetrieval.json create mode 100644 results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/model_meta.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/WITT2IRetrieval.json create mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json diff --git a/mteb/evaluation/evaluators/Image/ClusteringEvaluator.py b/mteb/evaluation/evaluators/Image/ClusteringEvaluator.py index f53befe8ef..fbf5e0bb14 100644 --- a/mteb/evaluation/evaluators/Image/ClusteringEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ClusteringEvaluator.py @@ -59,8 +59,6 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): ) ari = metrics.cluster.adjusted_rand_score(self.labels, cluster_assignment) - accuracy = metrics.accuracy_score(self.labels, cluster_assignment) - matrix = metrics.confusion_matrix(self.labels, cluster_assignment) # get linear sum assignment @@ -70,7 +68,6 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): return { "v_measure": v_measure, - "accuracy": accuracy, "nmi": nmi, "ari": ari, "cluster_accuracy": clustering_accuracy, diff --git a/mteb/models/cohere_v.py b/mteb/models/cohere_v.py new file mode 100644 index 0000000000..cf98f7dc3c --- /dev/null +++ b/mteb/models/cohere_v.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from torchvision import transforms +from tqdm import tqdm +import os +import io +import base64 +import mteb +import time +from mteb.model_meta import ModelMeta + +api_key = os.getenv("COHERE_API_KEY") +tensor_to_image = transforms.Compose([transforms.ToPILImage()]) + + +def cohere_v_loader(**kwargs): + try: + import cohere + except ImportError: + raise ImportError("To use cohere models, please run `pip install cohere`.") + + class CohereMultiModalModelWrapper: + def __init__( + self, + model_name: str, + **kwargs: Any, + ): + self.model_name = model_name + self.client = cohere.ClientV2(api_key) + self.image_format = "JPEG" + """ Wrapper for Cohere multimodal embedding model, + + do `export COHERE_API_KEY=` before running eval scripts. + Cohere currently supports 40 images/min, thus time.sleep(1.5) is applied after each image. + Remove or adjust this after Cohere API changes capacity. + """ + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + response = self.client.embed( + texts=batch_texts, + model=self.model_name, + input_type="search_document", + ) + all_text_embeddings.append(torch.tensor(response.embeddings.float)) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + for batch in tqdm(images): + for image in batch: + # cohere only supports 1 image per call + buffered = io.BytesIO() + image = tensor_to_image(image) + image.save(buffered, format=self.image_format) + image_bytes = buffered.getvalue() + stringified_buffer = base64.b64encode(image_bytes).decode( + "utf-8" + ) + content_type = f"image/{self.image_format.lower()}" + image_base64 = ( + f"data:{content_type};base64,{stringified_buffer}" + ) + response = self.client.embed( + model=self.model_name, + input_type="image", + embedding_types=["float"], + images=[image_base64], + ) + all_image_embeddings.append( + torch.tensor(response.embeddings.float) + ) + time.sleep(1.5) + else: + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + for image in batch_images: + # cohere only supports 1 image per call + buffered = io.BytesIO() + image.save(buffered, format=self.image_format) + image_bytes = buffered.getvalue() + stringified_buffer = base64.b64encode(image_bytes).decode( + "utf-8" + ) + content_type = f"image/{self.image_format.lower()}" + image_base64 = ( + f"data:{content_type};base64,{stringified_buffer}" + ) + response = self.client.embed( + model=self.model_name, + input_type="image", + embedding_types=["float"], + images=[image_base64], + ) + all_image_embeddings.append( + torch.tensor(response.embeddings.float) + ) + time.sleep(1.5) + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm( + dim=-1, keepdim=True + ) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + else: + # to do: add other fusion mode + raise ValueError( + f"fusion mode {fusion_mode} hasn't been implemented" + ) + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + return CohereMultiModalModelWrapper(**kwargs) + + +cohere_mult_3 = ModelMeta( + loader=partial(cohere_v_loader, model_name="embed-multilingual-v3.0"), + name="embed-multilingual-v3.0-v", + languages=[], # Unknown, but support >100 languages + open_source=False, + revision="1", + release_date="2024-10-24", + n_parameters=None, + memory_usage=None, + max_tokens=None, + embed_dim=1024, + license=None, + similarity_fn_name="cosine", + framework=[], +) + +cohere_eng_3 = ModelMeta( + loader=partial(cohere_v_loader, model_name="embed-english-v3.0"), + name="embed-english-v3.0-v", + languages=["eng-Latn"], + open_source=False, + revision="1", + release_date="2024-10-24", + n_parameters=None, + memory_usage=None, + max_tokens=None, + embed_dim=1024, + license=None, + similarity_fn_name="cosine", + framework=[], +) + +if __name__ == "__main__": + mdl = mteb.get_model(cohere_mult_3.name, cohere_mult_3.revision) + emb = mdl.encode(["Hello, world!"]) diff --git a/mteb/models/overview.py b/mteb/models/overview.py index 37358ea775..e68a4f65e4 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -16,6 +16,7 @@ bm25, clip_models, cohere_models, + cohere_v, dino_models, e5_instruct, e5_models, @@ -39,6 +40,7 @@ ru_sentence_models, salesforce_models, sentence_transformers_models, + siglip_models, vista_models, vlm2vec_models, voyage_models, @@ -54,6 +56,7 @@ bm25, clip_models, cohere_models, + cohere_v, dino_models, e5_instruct, e5_models, @@ -75,6 +78,7 @@ ru_sentence_models, salesforce_models, sentence_transformers_models, + siglip_models, vista_models, voyage_models, vlm2vec_models, diff --git a/mteb/models/siglip_models.py b/mteb/models/siglip_models.py new file mode 100644 index 0000000000..17e7dd0056 --- /dev/null +++ b/mteb/models/siglip_models.py @@ -0,0 +1,243 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import AutoModel, AutoProcessor + +from mteb.model_meta import ModelMeta + + +class SiglipModelWrapper: + def __init__( + self, + model_name: str, + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + self.model = AutoModel.from_pretrained(model_name).to(self.device) + self.processor = AutoProcessor.from_pretrained(model_name) + + def preprocess( + self, + texts: list[str], + images: list[Image.Image], + ): + return self.processor( + text=texts, images=images, return_tensors="pt", padding=True + ) + + def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + all_text_embeddings = [] + + with torch.no_grad(): + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + inputs = self.processor( + text=batch_texts, return_tensors="pt", padding=True, truncation=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + text_outputs = self.model.get_text_features(**inputs) + text_outputs = text_outputs / text_outputs.norm(dim=-1, keepdim=True) + all_text_embeddings.append(text_outputs.cpu()) + + all_text_embeddings = torch.cat(all_text_embeddings, dim=0) + return all_text_embeddings + + def get_image_embeddings( + self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + with torch.no_grad(): + for batch in tqdm(images): + inputs = self.processor( + images=batch, return_tensors="pt", padding=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model.get_image_features(**inputs) + image_outputs = image_outputs / image_outputs.norm( + dim=-1, keepdim=True + ) + all_image_embeddings.append(image_outputs.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + inputs = self.processor( + images=batch_images, return_tensors="pt", padding=True + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model.get_image_features(**inputs) + image_outputs = image_outputs / image_outputs.norm( + dim=-1, keepdim=True + ) + all_image_embeddings.append(image_outputs.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + # normalized features + image_embeddings = image_embeddings / image_embeddings.norm( + p=2, dim=-1, keepdim=True + ) + text_embeddings = text_embeddings / text_embeddings.norm( + p=2, dim=-1, keepdim=True + ) + + # cosine similarity as logits + logits_per_text = torch.matmul( + text_embeddings, image_embeddings.t().to(text_embeddings.device) + ) * self.model.logit_scale.exp().to( + text_embeddings.device + ) + self.model.logit_bias.to(text_embeddings.device) + logits_per_image = logits_per_text.t() + return logits_per_image + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + if len(text_embeddings) != len(image_embeddings): + raise ValueError( + "The number of texts and images must have the same length" + ) + if fusion_mode == "sum": + fused_embeddings = text_embeddings + image_embeddings + else: + # to do: add other fusion mode + raise ValueError(f"fusion mode {fusion_mode} hasn't been implemented") + return fused_embeddings + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + +siglip_so400m_patch14_384 = ModelMeta( + loader=partial( + SiglipModelWrapper, + model_name="google/siglip-so400m-patch14-384", + ), + name="google/siglip-so400m-patch14-384", + languages=["eng_Latn"], + open_source=True, + revision="9fdffc58afc957d1a03a25b10dba0329ab15c2a3", + release_date="2024-01-08", +) + +siglip_base_patch16_256_multilingual = ModelMeta( + loader=partial( + SiglipModelWrapper, + model_name="google/siglip-base-patch16-256-multilingual", + ), + name="google/siglip-base-patch16-256-multilingual", + languages=["eng_Latn"], + open_source=True, + revision="8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6", + release_date="2024-01-08", +) + +siglip_base_patch16_256 = ModelMeta( + loader=partial( + SiglipModelWrapper, + model_name="google/siglip-base-patch16-256", + ), + name="google/siglip-base-patch16-256", + languages=["eng_Latn"], + open_source=True, + revision="b078df89e446d623010d890864d4207fe6399f61", + release_date="2024-01-08", +) + +siglip_base_patch16_512 = ModelMeta( + loader=partial( + SiglipModelWrapper, + model_name="google/siglip-base-patch16-512", + ), + name="google/siglip-base-patch16-512", + languages=["eng_Latn"], + open_source=True, + revision="753a949581523b60257d93e18391e8c27f72eb22", + release_date="2024-01-08", +) + +siglip_base_patch16_384 = ModelMeta( + loader=partial( + SiglipModelWrapper, + model_name="google/siglip-base-patch16-384", + ), + name="google/siglip-base-patch16-384", + languages=["eng_Latn"], + open_source=True, + revision="41aec1c83b32e0a6fca20ad88ba058aa5b5ea394", + release_date="2024-01-08", +) + +siglip_base_patch16_224 = ModelMeta( + loader=partial( + SiglipModelWrapper, + model_name="google/siglip-base-patch16-224", + ), + name="google/siglip-base-patch16-224", + languages=["eng_Latn"], + open_source=True, + revision="7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed", + release_date="2024-01-08", +) + +siglip_large_patch16_256 = ModelMeta( + loader=partial( + SiglipModelWrapper, + model_name="google/siglip-large-patch16-256", + ), + name="google/siglip-large-patch16-256", + languages=["eng_Latn"], + open_source=True, + revision="d0da9f876e7d66b4e250cd2450c3ba2ce735e447", + release_date="2024-01-08", +) + +siglip_large_patch16_384 = ModelMeta( + loader=partial( + SiglipModelWrapper, + model_name="google/siglip-large-patch16-384", + ), + name="google/siglip-large-patch16-384", + languages=["eng_Latn"], + open_source=True, + revision="ce005573a40965dfd21fd937fbdeeebf2439fc35", + release_date="2024-01-08", +) + +if __name__ == "__main__": + import mteb + + mdl = mteb.get_model( + siglip_so400m_patch14_384.name, siglip_so400m_patch14_384.revision + ) + emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py index 8b8b232e1d..68ae7ed387 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py @@ -153,7 +153,7 @@ class VidoreDocVQARetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/docvqa_test_subsampled", "revision": "b1d89eda849e636676df6ead8002602fb1858600", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -206,7 +206,7 @@ class VidoreInfoVQARetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/infovqa_test_subsampled", "revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -259,7 +259,7 @@ class VidoreTabfquadRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/tabfquad_test_subsampled", "revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -312,7 +312,7 @@ class VidoreTatdqaRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/tatdqa_test", "revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -365,7 +365,7 @@ class VidoreShiftProjectRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/shiftproject_test", "revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -418,7 +418,7 @@ class VidoreSyntheticDocQAAIRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/syntheticDocQA_artificial_intelligence_test", "revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -471,7 +471,7 @@ class VidoreSyntheticDocQAEnergyRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/syntheticDocQA_energy_test", "revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -524,7 +524,7 @@ class VidoreSyntheticDocQAGovernmentReportsRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/syntheticDocQA_government_reports_test", "revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -577,7 +577,7 @@ class VidoreSyntheticDocQAHealthcareIndustryRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/syntheticDocQA_healthcare_industry_test", "revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", }, - type="Retrieval", + type="Any2AnyRetrieval", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Clustering/eng/CIFAR.py b/mteb/tasks/Image/Clustering/eng/CIFAR.py index f20145dbfe..2bde390661 100644 --- a/mteb/tasks/Image/Clustering/eng/CIFAR.py +++ b/mteb/tasks/Image/Clustering/eng/CIFAR.py @@ -17,7 +17,7 @@ class CIFAR10Clustering(AbsTaskImageClustering): category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], - main_score="accuracy", + main_score="nmi", date=( "2008-01-01", "2009-01-01", @@ -53,11 +53,11 @@ class CIFAR100Clustering(AbsTaskImageClustering): "path": "uoft-cs/cifar100", "revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", }, - type="Clustering", + type="ImageClustering", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], - main_score="accuracy", + main_score="nmi", date=( "2008-01-01", "2009-01-01", diff --git a/mteb/tasks/Image/Clustering/eng/ImageNet.py b/mteb/tasks/Image/Clustering/eng/ImageNet.py index 44c9584ebf..a2377e2334 100644 --- a/mteb/tasks/Image/Clustering/eng/ImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/ImageNet.py @@ -17,7 +17,7 @@ class ImageNetDog15Clustering(AbsTaskImageClustering): category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], - main_score="accuracy", + main_score="nmi", date=("2009-06-20", "2009-06-20"), # Conference date domains=["Web"], task_subtypes=["Object recognition"], @@ -53,11 +53,11 @@ class ImageNet10Clustering(AbsTaskImageClustering): "path": "JamieSJS/imagenet-10", "revision": "88f8a6d47c257895094c5ad81e67ba751771fc99", }, - type="Clustering", + type="ImageClustering", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], - main_score="accuracy", + main_score="nmi", date=("2009-06-20", "2009-06-20"), # Conference date domains=["Web"], task_subtypes=["Object recognition"], diff --git a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py index 95d0ee3246..a5760b6b9c 100644 --- a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py @@ -17,7 +17,7 @@ class TinyImageNet(AbsTaskImageClustering): category="s2s", eval_splits=["valid"], eval_langs=["eng-Latn"], - main_score="accuracy", + main_score="nmi", date=( "2012-01-01", "2015-12-31", diff --git a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py index c6e7ef6b16..5b4d096783 100644 --- a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py +++ b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py @@ -53,7 +53,7 @@ class CIFAR100Classification(AbsTaskImageClassification): "path": "uoft-cs/cifar100", "revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", }, - type="Classification", + type="ImageClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py b/mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py index 196026f971..24b3e7a4b1 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/PatchCamelyon.py @@ -17,7 +17,7 @@ class PatchCamelyonClassification(AbsTaskZeroshotClassification): "path": "clip-benchmark/wds_vtab-pcam", "revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", }, - type="Classification", + type="ZeroShotClassification", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/results-mieb/embed-english-v3.0-v/1/ImageNetDog15Clustering.json b/results-mieb/embed-english-v3.0-v/1/ImageNetDog15Clustering.json new file mode 100644 index 0000000000..a791197d49 --- /dev/null +++ b/results-mieb/embed-english-v3.0-v/1/ImageNetDog15Clustering.json @@ -0,0 +1,23 @@ +{ + "dataset_revision": "bfb6ad3b2109d26c9daddf14f98d315daa35ee72", + "evaluation_time": 1831.918389081955, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.11895910780669144, + "ari": 0.4718230736383377, + "cluster_accuracy": 0.578996282527881, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.11895910780669144, + "nmi": 0.6532570693140701, + "v_measure": 0.65325706931407 + } + ] + }, + "task_name": "ImageNetDog15Clustering" +} \ No newline at end of file diff --git a/results-mieb/embed-english-v3.0-v/1/model_meta.json b/results-mieb/embed-english-v3.0-v/1/model_meta.json new file mode 100644 index 0000000000..067881b7e8 --- /dev/null +++ b/results-mieb/embed-english-v3.0-v/1/model_meta.json @@ -0,0 +1 @@ +{"name": "embed-english-v3.0-v", "revision": "1", "release_date": "2024-10-24", "languages": ["eng-Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": 1024, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": "cosine", "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "cohere_v_loader"} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/WITT2IRetrieval.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/WITT2IRetrieval.json new file mode 100644 index 0000000000..5e35a43d4c --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/WITT2IRetrieval.json @@ -0,0 +1,1936 @@ +{ + "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", + "evaluation_time": 66.81307911872864, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.14831, + "cv_recall_at_10": 0.31011, + "cv_recall_at_100": 0.65056, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.39438, + "cv_recall_at_3": 0.20449, + "cv_recall_at_5": 0.25169, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.21879, + "map_at_1": 0.14831, + "map_at_10": 0.19083, + "map_at_100": 0.20226, + "map_at_1000": 0.2037, + "map_at_20": 0.19631, + "map_at_3": 0.17247, + "map_at_5": 0.1832, + "mrr_at_1": 0.14831460674157304, + "mrr_at_10": 0.19082753700731223, + "mrr_at_100": 0.20225932653601883, + "mrr_at_1000": 0.2037004971452857, + "mrr_at_20": 0.19630941951644984, + "mrr_at_3": 0.17247191011235952, + "mrr_at_5": 0.18320224719101122, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.26807037584184995, + "nauc_cv_recall_at_100_max": 0.48453045854865856, + "nauc_cv_recall_at_100_std": 0.5051430893683861, + "nauc_cv_recall_at_10_diff1": 0.3240598073692733, + "nauc_cv_recall_at_10_max": 0.34736522495740113, + "nauc_cv_recall_at_10_std": 0.3320976688540044, + "nauc_cv_recall_at_1_diff1": 0.5290394349724978, + "nauc_cv_recall_at_1_max": 0.48781134075251725, + "nauc_cv_recall_at_1_std": 0.35022750945671843, + "nauc_cv_recall_at_20_diff1": 0.33883138486795833, + "nauc_cv_recall_at_20_max": 0.3700807956874158, + "nauc_cv_recall_at_20_std": 0.35876997956540574, + "nauc_cv_recall_at_3_diff1": 0.4534575089822943, + "nauc_cv_recall_at_3_max": 0.39864663168457115, + "nauc_cv_recall_at_3_std": 0.31624509525368005, + "nauc_cv_recall_at_5_diff1": 0.3859997231322966, + "nauc_cv_recall_at_5_max": 0.39631469310533485, + "nauc_cv_recall_at_5_std": 0.33543675880208573, + "nauc_map_at_1000_diff1": 0.4622335272968312, + "nauc_map_at_1000_max": 0.4430261662650935, + "nauc_map_at_1000_std": 0.34589976349391555, + "nauc_map_at_100_diff1": 0.46143307956438734, + "nauc_map_at_100_max": 0.442923564402142, + "nauc_map_at_100_std": 0.346192058081346, + "nauc_map_at_10_diff1": 0.4618222747153087, + "nauc_map_at_10_max": 0.43767798670996355, + "nauc_map_at_10_std": 0.33999642815590886, + "nauc_map_at_1_diff1": 0.5290394349724978, + "nauc_map_at_1_max": 0.48781134075251725, + "nauc_map_at_1_std": 0.35022750945671843, + "nauc_map_at_20_diff1": 0.4620254574672781, + "nauc_map_at_20_max": 0.4390165307515562, + "nauc_map_at_20_std": 0.3426591907598361, + "nauc_map_at_3_diff1": 0.492887071202886, + "nauc_map_at_3_max": 0.4476533348870664, + "nauc_map_at_3_std": 0.3350468692448414, + "nauc_map_at_5_diff1": 0.4731605872794598, + "nauc_map_at_5_max": 0.44620481412940166, + "nauc_map_at_5_std": 0.3401122978842973, + "nauc_mrr_at_1000_diff1": 0.4622335272968312, + "nauc_mrr_at_1000_max": 0.4430261662650935, + "nauc_mrr_at_1000_std": 0.34589976349391555, + "nauc_mrr_at_100_diff1": 0.46143307956438734, + "nauc_mrr_at_100_max": 0.442923564402142, + "nauc_mrr_at_100_std": 0.346192058081346, + "nauc_mrr_at_10_diff1": 0.4618222747153087, + "nauc_mrr_at_10_max": 0.43767798670996355, + "nauc_mrr_at_10_std": 0.33999642815590886, + "nauc_mrr_at_1_diff1": 0.5290394349724978, + "nauc_mrr_at_1_max": 0.48781134075251725, + "nauc_mrr_at_1_std": 0.35022750945671843, + "nauc_mrr_at_20_diff1": 0.4620254574672781, + "nauc_mrr_at_20_max": 0.4390165307515562, + "nauc_mrr_at_20_std": 0.3426591907598361, + "nauc_mrr_at_3_diff1": 0.492887071202886, + "nauc_mrr_at_3_max": 0.4476533348870664, + "nauc_mrr_at_3_std": 0.3350468692448414, + "nauc_mrr_at_5_diff1": 0.4731605872794598, + "nauc_mrr_at_5_max": 0.44620481412940166, + "nauc_mrr_at_5_std": 0.3401122978842973, + "nauc_ndcg_at_1000_diff1": 0.4387864666079701, + "nauc_ndcg_at_1000_max": 0.43838379944766553, + "nauc_ndcg_at_1000_std": 0.3560604797863434, + "nauc_ndcg_at_100_diff1": 0.41728283501155305, + "nauc_ndcg_at_100_max": 0.4427460292346204, + "nauc_ndcg_at_100_std": 0.372909504646752, + "nauc_ndcg_at_10_diff1": 0.42287678557837377, + "nauc_ndcg_at_10_max": 0.4120351287654675, + "nauc_ndcg_at_10_std": 0.33760808580973, + "nauc_ndcg_at_1_diff1": 0.5290394349724978, + "nauc_ndcg_at_1_max": 0.48781134075251725, + "nauc_ndcg_at_1_std": 0.35022750945671843, + "nauc_ndcg_at_20_diff1": 0.4255068257817746, + "nauc_ndcg_at_20_max": 0.41783754747845137, + "nauc_ndcg_at_20_std": 0.34618769482991557, + "nauc_ndcg_at_3_diff1": 0.48183250013832507, + "nauc_ndcg_at_3_max": 0.4340116328470406, + "nauc_ndcg_at_3_std": 0.32981777269416657, + "nauc_ndcg_at_5_diff1": 0.4486175419457543, + "nauc_ndcg_at_5_max": 0.43177983045623736, + "nauc_ndcg_at_5_std": 0.33853236429784955, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.2680703758418504, + "nauc_precision_at_100_max": 0.48453045854865845, + "nauc_precision_at_100_std": 0.5051430893683858, + "nauc_precision_at_10_diff1": 0.3240598073692732, + "nauc_precision_at_10_max": 0.3473652249574012, + "nauc_precision_at_10_std": 0.3320976688540042, + "nauc_precision_at_1_diff1": 0.5290394349724978, + "nauc_precision_at_1_max": 0.48781134075251725, + "nauc_precision_at_1_std": 0.35022750945671843, + "nauc_precision_at_20_diff1": 0.3388313848679582, + "nauc_precision_at_20_max": 0.3700807956874157, + "nauc_precision_at_20_std": 0.35876997956540574, + "nauc_precision_at_3_diff1": 0.4534575089822941, + "nauc_precision_at_3_max": 0.398646631684571, + "nauc_precision_at_3_std": 0.31624509525368005, + "nauc_precision_at_5_diff1": 0.3859997231322966, + "nauc_precision_at_5_max": 0.3963146931053349, + "nauc_precision_at_5_std": 0.33543675880208584, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.26807037584184995, + "nauc_recall_at_100_max": 0.48453045854865856, + "nauc_recall_at_100_std": 0.5051430893683861, + "nauc_recall_at_10_diff1": 0.3240598073692733, + "nauc_recall_at_10_max": 0.34736522495740113, + "nauc_recall_at_10_std": 0.3320976688540044, + "nauc_recall_at_1_diff1": 0.5290394349724978, + "nauc_recall_at_1_max": 0.48781134075251725, + "nauc_recall_at_1_std": 0.35022750945671843, + "nauc_recall_at_20_diff1": 0.33883138486795833, + "nauc_recall_at_20_max": 0.3700807956874158, + "nauc_recall_at_20_std": 0.35876997956540574, + "nauc_recall_at_3_diff1": 0.4534575089822943, + "nauc_recall_at_3_max": 0.39864663168457115, + "nauc_recall_at_3_std": 0.31624509525368005, + "nauc_recall_at_5_diff1": 0.3859997231322966, + "nauc_recall_at_5_max": 0.39631469310533485, + "nauc_recall_at_5_std": 0.33543675880208573, + "ndcg_at_1": 0.14831, + "ndcg_at_10": 0.21879, + "ndcg_at_100": 0.28592, + "ndcg_at_1000": 0.32918, + "ndcg_at_20": 0.23962, + "ndcg_at_3": 0.18067, + "ndcg_at_5": 0.20006, + "precision_at_1": 0.14831, + "precision_at_10": 0.03101, + "precision_at_100": 0.00651, + "precision_at_1000": 0.001, + "precision_at_20": 0.01972, + "precision_at_3": 0.06816, + "precision_at_5": 0.05034, + "recall_at_1": 0.14831, + "recall_at_10": 0.31011, + "recall_at_100": 0.65056, + "recall_at_1000": 1.0, + "recall_at_20": 0.39438, + "recall_at_3": 0.20449, + "recall_at_5": 0.25169 + }, + { + "cv_recall_at_1": 0.13488, + "cv_recall_at_10": 0.31047, + "cv_recall_at_100": 0.68023, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.40581, + "cv_recall_at_3": 0.20814, + "cv_recall_at_5": 0.25, + "hf_subset": "bg", + "languages": [ + "bul-Cyrl" + ], + "main_score": 0.21438, + "map_at_1": 0.13488, + "map_at_10": 0.18462, + "map_at_100": 0.19748, + "map_at_1000": 0.19905, + "map_at_20": 0.19126, + "map_at_3": 0.16667, + "map_at_5": 0.17643, + "mrr_at_1": 0.13488372093023257, + "mrr_at_10": 0.1846197858988557, + "mrr_at_100": 0.19748470936111007, + "mrr_at_1000": 0.19904904925859657, + "mrr_at_20": 0.1912630101656491, + "mrr_at_3": 0.1666666666666667, + "mrr_at_5": 0.1764341085271318, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2445650317044002, + "nauc_cv_recall_at_100_max": 0.361707515600977, + "nauc_cv_recall_at_100_std": 0.363261081466743, + "nauc_cv_recall_at_10_diff1": 0.3676665275583041, + "nauc_cv_recall_at_10_max": 0.18264771666908935, + "nauc_cv_recall_at_10_std": 0.14698127038758088, + "nauc_cv_recall_at_1_diff1": 0.612561211764825, + "nauc_cv_recall_at_1_max": 0.2835486094917522, + "nauc_cv_recall_at_1_std": 0.18441089696146312, + "nauc_cv_recall_at_20_diff1": 0.27639099909559006, + "nauc_cv_recall_at_20_max": 0.1172433342882384, + "nauc_cv_recall_at_20_std": 0.11048650659259424, + "nauc_cv_recall_at_3_diff1": 0.4615205504007333, + "nauc_cv_recall_at_3_max": 0.19940569139194983, + "nauc_cv_recall_at_3_std": 0.14882662857375184, + "nauc_cv_recall_at_5_diff1": 0.4246482547448188, + "nauc_cv_recall_at_5_max": 0.19074386102961918, + "nauc_cv_recall_at_5_std": 0.16244686401464295, + "nauc_map_at_1000_diff1": 0.5006542328949688, + "nauc_map_at_1000_max": 0.23136238179970012, + "nauc_map_at_1000_std": 0.16686604817646652, + "nauc_map_at_100_diff1": 0.49949888758750094, + "nauc_map_at_100_max": 0.23127076723942885, + "nauc_map_at_100_std": 0.16683027470816547, + "nauc_map_at_10_diff1": 0.5082854436326143, + "nauc_map_at_10_max": 0.23184604408065937, + "nauc_map_at_10_std": 0.1650454145932678, + "nauc_map_at_1_diff1": 0.612561211764825, + "nauc_map_at_1_max": 0.2835486094917522, + "nauc_map_at_1_std": 0.18441089696146312, + "nauc_map_at_20_diff1": 0.49881030031048196, + "nauc_map_at_20_max": 0.22573843880235675, + "nauc_map_at_20_std": 0.16150675906865322, + "nauc_map_at_3_diff1": 0.5328258548948126, + "nauc_map_at_3_max": 0.23753875983065484, + "nauc_map_at_3_std": 0.16377526196925346, + "nauc_map_at_5_diff1": 0.5206969673028455, + "nauc_map_at_5_max": 0.23460276419011938, + "nauc_map_at_5_std": 0.16857161106494534, + "nauc_mrr_at_1000_diff1": 0.5006542740060854, + "nauc_mrr_at_1000_max": 0.23136240008672215, + "nauc_mrr_at_1000_std": 0.16686602588729194, + "nauc_mrr_at_100_diff1": 0.49949888758750094, + "nauc_mrr_at_100_max": 0.23127076723942885, + "nauc_mrr_at_100_std": 0.16683027470816547, + "nauc_mrr_at_10_diff1": 0.5082854436326143, + "nauc_mrr_at_10_max": 0.23184604408065937, + "nauc_mrr_at_10_std": 0.1650454145932678, + "nauc_mrr_at_1_diff1": 0.612561211764825, + "nauc_mrr_at_1_max": 0.2835486094917522, + "nauc_mrr_at_1_std": 0.18441089696146312, + "nauc_mrr_at_20_diff1": 0.49881030031048196, + "nauc_mrr_at_20_max": 0.22573843880235675, + "nauc_mrr_at_20_std": 0.16150675906865322, + "nauc_mrr_at_3_diff1": 0.5328258548948126, + "nauc_mrr_at_3_max": 0.23753875983065484, + "nauc_mrr_at_3_std": 0.16377526196925346, + "nauc_mrr_at_5_diff1": 0.5206969673028455, + "nauc_mrr_at_5_max": 0.23460276419011938, + "nauc_mrr_at_5_std": 0.16857161106494534, + "nauc_ndcg_at_1000_diff1": 0.46721227001489246, + "nauc_ndcg_at_1000_max": 0.2287805352488556, + "nauc_ndcg_at_1000_std": 0.17407191971540306, + "nauc_ndcg_at_100_diff1": 0.439465110267931, + "nauc_ndcg_at_100_max": 0.24227660648463453, + "nauc_ndcg_at_100_std": 0.19339621087015968, + "nauc_ndcg_at_10_diff1": 0.46657913593419037, + "nauc_ndcg_at_10_max": 0.21665052448717026, + "nauc_ndcg_at_10_std": 0.15963159610625768, + "nauc_ndcg_at_1_diff1": 0.612561211764825, + "nauc_ndcg_at_1_max": 0.2835486094917522, + "nauc_ndcg_at_1_std": 0.18441089696146312, + "nauc_ndcg_at_20_diff1": 0.43577425970705724, + "nauc_ndcg_at_20_max": 0.1955854870307893, + "nauc_ndcg_at_20_std": 0.14776732385765973, + "nauc_ndcg_at_3_diff1": 0.5121326130898106, + "nauc_ndcg_at_3_max": 0.2264066500971107, + "nauc_ndcg_at_3_std": 0.1593606756338783, + "nauc_ndcg_at_5_diff1": 0.4924030178160872, + "nauc_ndcg_at_5_max": 0.22153330346602482, + "nauc_ndcg_at_5_std": 0.16658520770421348, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.24456503170440094, + "nauc_precision_at_100_max": 0.3617075156009771, + "nauc_precision_at_100_std": 0.36326108146674346, + "nauc_precision_at_10_diff1": 0.3676665275583044, + "nauc_precision_at_10_max": 0.1826477166690894, + "nauc_precision_at_10_std": 0.14698127038758094, + "nauc_precision_at_1_diff1": 0.612561211764825, + "nauc_precision_at_1_max": 0.2835486094917522, + "nauc_precision_at_1_std": 0.18441089696146312, + "nauc_precision_at_20_diff1": 0.27639099909558995, + "nauc_precision_at_20_max": 0.11724333428823823, + "nauc_precision_at_20_std": 0.11048650659259393, + "nauc_precision_at_3_diff1": 0.46152055040073325, + "nauc_precision_at_3_max": 0.19940569139194977, + "nauc_precision_at_3_std": 0.1488266285737518, + "nauc_precision_at_5_diff1": 0.42464825474481893, + "nauc_precision_at_5_max": 0.19074386102961938, + "nauc_precision_at_5_std": 0.16244686401464292, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2445650317044002, + "nauc_recall_at_100_max": 0.361707515600977, + "nauc_recall_at_100_std": 0.363261081466743, + "nauc_recall_at_10_diff1": 0.3676665275583041, + "nauc_recall_at_10_max": 0.18264771666908935, + "nauc_recall_at_10_std": 0.14698127038758088, + "nauc_recall_at_1_diff1": 0.612561211764825, + "nauc_recall_at_1_max": 0.2835486094917522, + "nauc_recall_at_1_std": 0.18441089696146312, + "nauc_recall_at_20_diff1": 0.27639099909559006, + "nauc_recall_at_20_max": 0.1172433342882384, + "nauc_recall_at_20_std": 0.11048650659259424, + "nauc_recall_at_3_diff1": 0.4615205504007333, + "nauc_recall_at_3_max": 0.19940569139194983, + "nauc_recall_at_3_std": 0.14882662857375184, + "nauc_recall_at_5_diff1": 0.4246482547448188, + "nauc_recall_at_5_max": 0.19074386102961918, + "nauc_recall_at_5_std": 0.16244686401464295, + "ndcg_at_1": 0.13488, + "ndcg_at_10": 0.21438, + "ndcg_at_100": 0.28778, + "ndcg_at_1000": 0.32867, + "ndcg_at_20": 0.23853, + "ndcg_at_3": 0.1773, + "ndcg_at_5": 0.19471, + "precision_at_1": 0.13488, + "precision_at_10": 0.03105, + "precision_at_100": 0.0068, + "precision_at_1000": 0.001, + "precision_at_20": 0.02029, + "precision_at_3": 0.06938, + "precision_at_5": 0.05, + "recall_at_1": 0.13488, + "recall_at_10": 0.31047, + "recall_at_100": 0.68023, + "recall_at_1000": 1.0, + "recall_at_20": 0.40581, + "recall_at_3": 0.20814, + "recall_at_5": 0.25 + }, + { + "cv_recall_at_1": 0.30864, + "cv_recall_at_10": 0.55107, + "cv_recall_at_100": 0.81033, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.63861, + "cv_recall_at_3": 0.41751, + "cv_recall_at_5": 0.46914, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.4196, + "map_at_1": 0.30864, + "map_at_10": 0.37877, + "map_at_100": 0.3888, + "map_at_1000": 0.38968, + "map_at_20": 0.38476, + "map_at_3": 0.35597, + "map_at_5": 0.36758, + "mrr_at_1": 0.30864197530864196, + "mrr_at_10": 0.37877006395524887, + "mrr_at_100": 0.3888033951118189, + "mrr_at_1000": 0.38968141928150263, + "mrr_at_20": 0.38475571429269884, + "mrr_at_3": 0.3559670781893003, + "mrr_at_5": 0.3675832398054617, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.34917861138467304, + "nauc_cv_recall_at_100_max": 0.5809280032918813, + "nauc_cv_recall_at_100_std": 0.5652106785226791, + "nauc_cv_recall_at_10_diff1": 0.5202673828089519, + "nauc_cv_recall_at_10_max": 0.4867137193623612, + "nauc_cv_recall_at_10_std": 0.35862555124365353, + "nauc_cv_recall_at_1_diff1": 0.6978458793185269, + "nauc_cv_recall_at_1_max": 0.4812630250390875, + "nauc_cv_recall_at_1_std": 0.2574662833691913, + "nauc_cv_recall_at_20_diff1": 0.4794186508801584, + "nauc_cv_recall_at_20_max": 0.5114398099151045, + "nauc_cv_recall_at_20_std": 0.4478160589028657, + "nauc_cv_recall_at_3_diff1": 0.5920738440881316, + "nauc_cv_recall_at_3_max": 0.4382773687425336, + "nauc_cv_recall_at_3_std": 0.2790071271396831, + "nauc_cv_recall_at_5_diff1": 0.5511450554138266, + "nauc_cv_recall_at_5_max": 0.453711553179142, + "nauc_cv_recall_at_5_std": 0.3190509058673222, + "nauc_map_at_1000_diff1": 0.636391806736526, + "nauc_map_at_1000_max": 0.4701983153516574, + "nauc_map_at_1000_std": 0.2856963477079405, + "nauc_map_at_100_diff1": 0.6357752143121728, + "nauc_map_at_100_max": 0.47037477407570155, + "nauc_map_at_100_std": 0.28621124775825796, + "nauc_map_at_10_diff1": 0.6380748511735241, + "nauc_map_at_10_max": 0.46869771847749625, + "nauc_map_at_10_std": 0.28119719880382293, + "nauc_map_at_1_diff1": 0.6978458793185269, + "nauc_map_at_1_max": 0.4812630250390875, + "nauc_map_at_1_std": 0.2574662833691913, + "nauc_map_at_20_diff1": 0.6363130940052869, + "nauc_map_at_20_max": 0.4701539151563389, + "nauc_map_at_20_std": 0.2865548455100543, + "nauc_map_at_3_diff1": 0.6502805436566025, + "nauc_map_at_3_max": 0.4604312622649522, + "nauc_map_at_3_std": 0.2670529749642637, + "nauc_map_at_5_diff1": 0.6414768479249554, + "nauc_map_at_5_max": 0.46432565412176646, + "nauc_map_at_5_std": 0.2768796717662677, + "nauc_mrr_at_1000_diff1": 0.636391806736526, + "nauc_mrr_at_1000_max": 0.4701983153516574, + "nauc_mrr_at_1000_std": 0.2856963477079405, + "nauc_mrr_at_100_diff1": 0.6357752143121728, + "nauc_mrr_at_100_max": 0.47037477407570155, + "nauc_mrr_at_100_std": 0.28621124775825796, + "nauc_mrr_at_10_diff1": 0.6380748511735241, + "nauc_mrr_at_10_max": 0.46869771847749625, + "nauc_mrr_at_10_std": 0.28119719880382293, + "nauc_mrr_at_1_diff1": 0.6978458793185269, + "nauc_mrr_at_1_max": 0.4812630250390875, + "nauc_mrr_at_1_std": 0.2574662833691913, + "nauc_mrr_at_20_diff1": 0.6363130940052869, + "nauc_mrr_at_20_max": 0.4701539151563389, + "nauc_mrr_at_20_std": 0.2865548455100543, + "nauc_mrr_at_3_diff1": 0.6502805436566025, + "nauc_mrr_at_3_max": 0.4604312622649522, + "nauc_mrr_at_3_std": 0.2670529749642637, + "nauc_mrr_at_5_diff1": 0.6414768479249554, + "nauc_mrr_at_5_max": 0.46432565412176646, + "nauc_mrr_at_5_std": 0.2768796717662677, + "nauc_ndcg_at_1000_diff1": 0.6168157065098726, + "nauc_ndcg_at_1000_max": 0.4741862963085771, + "nauc_ndcg_at_1000_std": 0.30184717278586776, + "nauc_ndcg_at_100_diff1": 0.5981836215780852, + "nauc_ndcg_at_100_max": 0.48122121391903505, + "nauc_ndcg_at_100_std": 0.3196703358055457, + "nauc_ndcg_at_10_diff1": 0.6103946748770198, + "nauc_ndcg_at_10_max": 0.47184321180216543, + "nauc_ndcg_at_10_std": 0.2985277877145453, + "nauc_ndcg_at_1_diff1": 0.6978458793185269, + "nauc_ndcg_at_1_max": 0.4812630250390875, + "nauc_ndcg_at_1_std": 0.2574662833691913, + "nauc_ndcg_at_20_diff1": 0.6036674298778513, + "nauc_ndcg_at_20_max": 0.4770433048467731, + "nauc_ndcg_at_20_std": 0.3173737824737843, + "nauc_ndcg_at_3_diff1": 0.6351136489526188, + "nauc_ndcg_at_3_max": 0.4545954162776597, + "nauc_ndcg_at_3_std": 0.2701636458988301, + "nauc_ndcg_at_5_diff1": 0.6188271043822667, + "nauc_ndcg_at_5_max": 0.4613224822188785, + "nauc_ndcg_at_5_std": 0.28724444390528403, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3491786113846725, + "nauc_precision_at_100_max": 0.5809280032918817, + "nauc_precision_at_100_std": 0.5652106785226783, + "nauc_precision_at_10_diff1": 0.5202673828089526, + "nauc_precision_at_10_max": 0.486713719362361, + "nauc_precision_at_10_std": 0.3586255512436532, + "nauc_precision_at_1_diff1": 0.6978458793185269, + "nauc_precision_at_1_max": 0.4812630250390875, + "nauc_precision_at_1_std": 0.2574662833691913, + "nauc_precision_at_20_diff1": 0.479418650880159, + "nauc_precision_at_20_max": 0.5114398099151042, + "nauc_precision_at_20_std": 0.44781605890286524, + "nauc_precision_at_3_diff1": 0.5920738440881318, + "nauc_precision_at_3_max": 0.4382773687425334, + "nauc_precision_at_3_std": 0.27900712713968306, + "nauc_precision_at_5_diff1": 0.5511450554138263, + "nauc_precision_at_5_max": 0.45371155317914186, + "nauc_precision_at_5_std": 0.3190509058673224, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.34917861138467304, + "nauc_recall_at_100_max": 0.5809280032918813, + "nauc_recall_at_100_std": 0.5652106785226791, + "nauc_recall_at_10_diff1": 0.5202673828089519, + "nauc_recall_at_10_max": 0.4867137193623612, + "nauc_recall_at_10_std": 0.35862555124365353, + "nauc_recall_at_1_diff1": 0.6978458793185269, + "nauc_recall_at_1_max": 0.4812630250390875, + "nauc_recall_at_1_std": 0.2574662833691913, + "nauc_recall_at_20_diff1": 0.4794186508801584, + "nauc_recall_at_20_max": 0.5114398099151045, + "nauc_recall_at_20_std": 0.4478160589028657, + "nauc_recall_at_3_diff1": 0.5920738440881316, + "nauc_recall_at_3_max": 0.4382773687425336, + "nauc_recall_at_3_std": 0.2790071271396831, + "nauc_recall_at_5_diff1": 0.5511450554138266, + "nauc_recall_at_5_max": 0.453711553179142, + "nauc_recall_at_5_std": 0.3190509058673222, + "ndcg_at_1": 0.30864, + "ndcg_at_10": 0.4196, + "ndcg_at_100": 0.47276, + "ndcg_at_1000": 0.49672, + "ndcg_at_20": 0.44162, + "ndcg_at_3": 0.37175, + "ndcg_at_5": 0.39285, + "precision_at_1": 0.30864, + "precision_at_10": 0.05511, + "precision_at_100": 0.0081, + "precision_at_1000": 0.001, + "precision_at_20": 0.03193, + "precision_at_3": 0.13917, + "precision_at_5": 0.09383, + "recall_at_1": 0.30864, + "recall_at_10": 0.55107, + "recall_at_100": 0.81033, + "recall_at_1000": 1.0, + "recall_at_20": 0.63861, + "recall_at_3": 0.41751, + "recall_at_5": 0.46914 + }, + { + "cv_recall_at_1": 0.10526, + "cv_recall_at_10": 0.25263, + "cv_recall_at_100": 0.57544, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.29825, + "cv_recall_at_3": 0.17018, + "cv_recall_at_5": 0.20175, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.17164, + "map_at_1": 0.10526, + "map_at_10": 0.14659, + "map_at_100": 0.1559, + "map_at_1000": 0.15776, + "map_at_20": 0.14964, + "map_at_3": 0.13275, + "map_at_5": 0.13968, + "mrr_at_1": 0.10526315789473684, + "mrr_at_10": 0.1465866054023949, + "mrr_at_100": 0.15589577807695595, + "mrr_at_1000": 0.15776355208566945, + "mrr_at_20": 0.14964308480582725, + "mrr_at_3": 0.1327485380116959, + "mrr_at_5": 0.13967836257309946, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.18637855746142357, + "nauc_cv_recall_at_100_max": 0.24833841602415352, + "nauc_cv_recall_at_100_std": 0.403742110009017, + "nauc_cv_recall_at_10_diff1": 0.2682111039181232, + "nauc_cv_recall_at_10_max": 0.2003298498720677, + "nauc_cv_recall_at_10_std": 0.35493798616891187, + "nauc_cv_recall_at_1_diff1": 0.5149290393013103, + "nauc_cv_recall_at_1_max": 0.34715247452692855, + "nauc_cv_recall_at_1_std": 0.38980167394468723, + "nauc_cv_recall_at_20_diff1": 0.21777962676809698, + "nauc_cv_recall_at_20_max": 0.19354332580530123, + "nauc_cv_recall_at_20_std": 0.35196481635564003, + "nauc_cv_recall_at_3_diff1": 0.37585939785098305, + "nauc_cv_recall_at_3_max": 0.1919651323127164, + "nauc_cv_recall_at_3_std": 0.34531802943568224, + "nauc_cv_recall_at_5_diff1": 0.3058416721945437, + "nauc_cv_recall_at_5_max": 0.1752755705474715, + "nauc_cv_recall_at_5_std": 0.3182589304616601, + "nauc_map_at_1000_diff1": 0.403902744979074, + "nauc_map_at_1000_max": 0.26715425573574997, + "nauc_map_at_1000_std": 0.36653124352277344, + "nauc_map_at_100_diff1": 0.4030183822085817, + "nauc_map_at_100_max": 0.266996593455788, + "nauc_map_at_100_std": 0.36669005171306873, + "nauc_map_at_10_diff1": 0.41010891346858724, + "nauc_map_at_10_max": 0.26588433202149636, + "nauc_map_at_10_std": 0.3642306694227432, + "nauc_map_at_1_diff1": 0.5149290393013103, + "nauc_map_at_1_max": 0.34715247452692855, + "nauc_map_at_1_std": 0.38980167394468723, + "nauc_map_at_20_diff1": 0.4038257381182442, + "nauc_map_at_20_max": 0.2643456092222034, + "nauc_map_at_20_std": 0.36333876392800185, + "nauc_map_at_3_diff1": 0.44276173515640216, + "nauc_map_at_3_max": 0.27000821438381173, + "nauc_map_at_3_std": 0.3661474539923582, + "nauc_map_at_5_diff1": 0.4193634293904592, + "nauc_map_at_5_max": 0.26298725307683535, + "nauc_map_at_5_std": 0.35770376391128783, + "nauc_mrr_at_1000_diff1": 0.403902744979074, + "nauc_mrr_at_1000_max": 0.26715425573574997, + "nauc_mrr_at_1000_std": 0.36653124352277344, + "nauc_mrr_at_100_diff1": 0.4030183822085817, + "nauc_mrr_at_100_max": 0.266996593455788, + "nauc_mrr_at_100_std": 0.36669005171306873, + "nauc_mrr_at_10_diff1": 0.41010891346858724, + "nauc_mrr_at_10_max": 0.26588433202149636, + "nauc_mrr_at_10_std": 0.3642306694227432, + "nauc_mrr_at_1_diff1": 0.5149290393013103, + "nauc_mrr_at_1_max": 0.34715247452692855, + "nauc_mrr_at_1_std": 0.38980167394468723, + "nauc_mrr_at_20_diff1": 0.4038257381182442, + "nauc_mrr_at_20_max": 0.2643456092222034, + "nauc_mrr_at_20_std": 0.36333876392800185, + "nauc_mrr_at_3_diff1": 0.44276173515640216, + "nauc_mrr_at_3_max": 0.27000821438381173, + "nauc_mrr_at_3_std": 0.3661474539923582, + "nauc_mrr_at_5_diff1": 0.4193634293904592, + "nauc_mrr_at_5_max": 0.26298725307683535, + "nauc_mrr_at_5_std": 0.35770376391128783, + "nauc_ndcg_at_1000_diff1": 0.36876440377313985, + "nauc_ndcg_at_1000_max": 0.25674301571843805, + "nauc_ndcg_at_1000_std": 0.36823676127345956, + "nauc_ndcg_at_100_diff1": 0.3411028514655866, + "nauc_ndcg_at_100_max": 0.2549721813489106, + "nauc_ndcg_at_100_std": 0.373586154530333, + "nauc_ndcg_at_10_diff1": 0.3662572907422116, + "nauc_ndcg_at_10_max": 0.24402177974697598, + "nauc_ndcg_at_10_std": 0.36035887865430855, + "nauc_ndcg_at_1_diff1": 0.5149290393013103, + "nauc_ndcg_at_1_max": 0.34715247452692855, + "nauc_ndcg_at_1_std": 0.38980167394468723, + "nauc_ndcg_at_20_diff1": 0.3465518274198802, + "nauc_ndcg_at_20_max": 0.24022710954348786, + "nauc_ndcg_at_20_std": 0.35861648681751135, + "nauc_ndcg_at_3_diff1": 0.42315037604313344, + "nauc_ndcg_at_3_max": 0.24726612330812991, + "nauc_ndcg_at_3_std": 0.3600189049541695, + "nauc_ndcg_at_5_diff1": 0.38509779167121194, + "nauc_ndcg_at_5_max": 0.23615476637347932, + "nauc_ndcg_at_5_std": 0.34587018634527034, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.18637855746142468, + "nauc_precision_at_100_max": 0.24833841602415418, + "nauc_precision_at_100_std": 0.4037421100090175, + "nauc_precision_at_10_diff1": 0.26821110391812325, + "nauc_precision_at_10_max": 0.20032984987206753, + "nauc_precision_at_10_std": 0.3549379861689121, + "nauc_precision_at_1_diff1": 0.5149290393013103, + "nauc_precision_at_1_max": 0.34715247452692855, + "nauc_precision_at_1_std": 0.38980167394468723, + "nauc_precision_at_20_diff1": 0.2177796267680969, + "nauc_precision_at_20_max": 0.19354332580530145, + "nauc_precision_at_20_std": 0.35196481635564014, + "nauc_precision_at_3_diff1": 0.37585939785098327, + "nauc_precision_at_3_max": 0.19196513231271656, + "nauc_precision_at_3_std": 0.3453180294356824, + "nauc_precision_at_5_diff1": 0.30584167219454383, + "nauc_precision_at_5_max": 0.17527557054747175, + "nauc_precision_at_5_std": 0.31825893046166037, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.18637855746142357, + "nauc_recall_at_100_max": 0.24833841602415352, + "nauc_recall_at_100_std": 0.403742110009017, + "nauc_recall_at_10_diff1": 0.2682111039181232, + "nauc_recall_at_10_max": 0.2003298498720677, + "nauc_recall_at_10_std": 0.35493798616891187, + "nauc_recall_at_1_diff1": 0.5149290393013103, + "nauc_recall_at_1_max": 0.34715247452692855, + "nauc_recall_at_1_std": 0.38980167394468723, + "nauc_recall_at_20_diff1": 0.21777962676809698, + "nauc_recall_at_20_max": 0.19354332580530123, + "nauc_recall_at_20_std": 0.35196481635564003, + "nauc_recall_at_3_diff1": 0.37585939785098305, + "nauc_recall_at_3_max": 0.1919651323127164, + "nauc_recall_at_3_std": 0.34531802943568224, + "nauc_recall_at_5_diff1": 0.3058416721945437, + "nauc_recall_at_5_max": 0.1752755705474715, + "nauc_recall_at_5_std": 0.3182589304616601, + "ndcg_at_1": 0.10526, + "ndcg_at_10": 0.17164, + "ndcg_at_100": 0.23279, + "ndcg_at_1000": 0.28635, + "ndcg_at_20": 0.18302, + "ndcg_at_3": 0.14231, + "ndcg_at_5": 0.15507, + "precision_at_1": 0.10526, + "precision_at_10": 0.02526, + "precision_at_100": 0.00575, + "precision_at_1000": 0.001, + "precision_at_20": 0.01491, + "precision_at_3": 0.05673, + "precision_at_5": 0.04035, + "recall_at_1": 0.10526, + "recall_at_10": 0.25263, + "recall_at_100": 0.57544, + "recall_at_1000": 1.0, + "recall_at_20": 0.29825, + "recall_at_3": 0.17018, + "recall_at_5": 0.20175 + }, + { + "cv_recall_at_1": 0.17277, + "cv_recall_at_10": 0.34325, + "cv_recall_at_100": 0.6762, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.43593, + "cv_recall_at_3": 0.23341, + "cv_recall_at_5": 0.27346, + "hf_subset": "et", + "languages": [ + "est-Latn" + ], + "main_score": 0.24659, + "map_at_1": 0.17277, + "map_at_10": 0.2171, + "map_at_100": 0.22898, + "map_at_1000": 0.23049, + "map_at_20": 0.22329, + "map_at_3": 0.1987, + "map_at_5": 0.2078, + "mrr_at_1": 0.17276887871853547, + "mrr_at_10": 0.2171011768551813, + "mrr_at_100": 0.2289810314309146, + "mrr_at_1000": 0.23048794912664428, + "mrr_at_20": 0.22328900136301424, + "mrr_at_3": 0.1987032799389779, + "mrr_at_5": 0.20779938977879484, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.23707892644673476, + "nauc_cv_recall_at_100_max": 0.5355293934445048, + "nauc_cv_recall_at_100_std": 0.5106409286751152, + "nauc_cv_recall_at_10_diff1": 0.39870287756389644, + "nauc_cv_recall_at_10_max": 0.5015395307021189, + "nauc_cv_recall_at_10_std": 0.39778923076644446, + "nauc_cv_recall_at_1_diff1": 0.6403798630015372, + "nauc_cv_recall_at_1_max": 0.5070352889746105, + "nauc_cv_recall_at_1_std": 0.31104444805117587, + "nauc_cv_recall_at_20_diff1": 0.3353205779178744, + "nauc_cv_recall_at_20_max": 0.48669917318802397, + "nauc_cv_recall_at_20_std": 0.41855181255617585, + "nauc_cv_recall_at_3_diff1": 0.5307956145460796, + "nauc_cv_recall_at_3_max": 0.46477350734939027, + "nauc_cv_recall_at_3_std": 0.32290526314893786, + "nauc_cv_recall_at_5_diff1": 0.47362817758445497, + "nauc_cv_recall_at_5_max": 0.4860244681874373, + "nauc_cv_recall_at_5_std": 0.3426509746871683, + "nauc_map_at_1000_diff1": 0.5518950452222675, + "nauc_map_at_1000_max": 0.49524192662772293, + "nauc_map_at_1000_std": 0.33176304836465365, + "nauc_map_at_100_diff1": 0.5507042720887966, + "nauc_map_at_100_max": 0.4952857109383223, + "nauc_map_at_100_std": 0.3324067505646156, + "nauc_map_at_10_diff1": 0.5583879340502963, + "nauc_map_at_10_max": 0.49526907782206203, + "nauc_map_at_10_std": 0.3300983653566066, + "nauc_map_at_1_diff1": 0.6403798630015372, + "nauc_map_at_1_max": 0.5070352889746105, + "nauc_map_at_1_std": 0.31104444805117587, + "nauc_map_at_20_diff1": 0.5524908076068307, + "nauc_map_at_20_max": 0.49414088668716216, + "nauc_map_at_20_std": 0.3317096629500253, + "nauc_map_at_3_diff1": 0.5866258611416878, + "nauc_map_at_3_max": 0.48824451649660294, + "nauc_map_at_3_std": 0.31506067342848376, + "nauc_map_at_5_diff1": 0.5712264451574552, + "nauc_map_at_5_max": 0.49309396688072177, + "nauc_map_at_5_std": 0.3204667488839716, + "nauc_mrr_at_1000_diff1": 0.5518950452222675, + "nauc_mrr_at_1000_max": 0.49524192662772293, + "nauc_mrr_at_1000_std": 0.33176304836465365, + "nauc_mrr_at_100_diff1": 0.5507042720887966, + "nauc_mrr_at_100_max": 0.4952857109383223, + "nauc_mrr_at_100_std": 0.3324067505646156, + "nauc_mrr_at_10_diff1": 0.5583879340502963, + "nauc_mrr_at_10_max": 0.49526907782206203, + "nauc_mrr_at_10_std": 0.3300983653566066, + "nauc_mrr_at_1_diff1": 0.6403798630015372, + "nauc_mrr_at_1_max": 0.5070352889746105, + "nauc_mrr_at_1_std": 0.31104444805117587, + "nauc_mrr_at_20_diff1": 0.5524908076068307, + "nauc_mrr_at_20_max": 0.49414088668716216, + "nauc_mrr_at_20_std": 0.3317096629500253, + "nauc_mrr_at_3_diff1": 0.5866258611416878, + "nauc_mrr_at_3_max": 0.48824451649660294, + "nauc_mrr_at_3_std": 0.31506067342848376, + "nauc_mrr_at_5_diff1": 0.5712264451574552, + "nauc_mrr_at_5_max": 0.49309396688072177, + "nauc_mrr_at_5_std": 0.3204667488839716, + "nauc_ndcg_at_1000_diff1": 0.5175493259429081, + "nauc_ndcg_at_1000_max": 0.4966318371741495, + "nauc_ndcg_at_1000_std": 0.34574293260303035, + "nauc_ndcg_at_100_diff1": 0.48509068676100753, + "nauc_ndcg_at_100_max": 0.5005582680689583, + "nauc_ndcg_at_100_std": 0.36468181197554345, + "nauc_ndcg_at_10_diff1": 0.5149606519232172, + "nauc_ndcg_at_10_max": 0.49619640915776325, + "nauc_ndcg_at_10_std": 0.34786983476833444, + "nauc_ndcg_at_1_diff1": 0.6403798630015372, + "nauc_ndcg_at_1_max": 0.5070352889746105, + "nauc_ndcg_at_1_std": 0.31104444805117587, + "nauc_ndcg_at_20_diff1": 0.4953903019228144, + "nauc_ndcg_at_20_max": 0.49198524631438506, + "nauc_ndcg_at_20_std": 0.35391185392693036, + "nauc_ndcg_at_3_diff1": 0.5712391453484501, + "nauc_ndcg_at_3_max": 0.4818568955948492, + "nauc_ndcg_at_3_std": 0.3171474374420657, + "nauc_ndcg_at_5_diff1": 0.5443642154064743, + "nauc_ndcg_at_5_max": 0.49077593878515474, + "nauc_ndcg_at_5_std": 0.32642028107092486, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.23707892644673514, + "nauc_precision_at_100_max": 0.5355293934445053, + "nauc_precision_at_100_std": 0.5106409286751148, + "nauc_precision_at_10_diff1": 0.3987028775638965, + "nauc_precision_at_10_max": 0.5015395307021191, + "nauc_precision_at_10_std": 0.39778923076644485, + "nauc_precision_at_1_diff1": 0.6403798630015372, + "nauc_precision_at_1_max": 0.5070352889746105, + "nauc_precision_at_1_std": 0.31104444805117587, + "nauc_precision_at_20_diff1": 0.3353205779178743, + "nauc_precision_at_20_max": 0.48669917318802425, + "nauc_precision_at_20_std": 0.4185518125561754, + "nauc_precision_at_3_diff1": 0.5307956145460793, + "nauc_precision_at_3_max": 0.4647735073493902, + "nauc_precision_at_3_std": 0.3229052631489379, + "nauc_precision_at_5_diff1": 0.4736281775844547, + "nauc_precision_at_5_max": 0.4860244681874368, + "nauc_precision_at_5_std": 0.342650974687168, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.23707892644673476, + "nauc_recall_at_100_max": 0.5355293934445048, + "nauc_recall_at_100_std": 0.5106409286751152, + "nauc_recall_at_10_diff1": 0.39870287756389644, + "nauc_recall_at_10_max": 0.5015395307021189, + "nauc_recall_at_10_std": 0.39778923076644446, + "nauc_recall_at_1_diff1": 0.6403798630015372, + "nauc_recall_at_1_max": 0.5070352889746105, + "nauc_recall_at_1_std": 0.31104444805117587, + "nauc_recall_at_20_diff1": 0.3353205779178744, + "nauc_recall_at_20_max": 0.48669917318802397, + "nauc_recall_at_20_std": 0.41855181255617585, + "nauc_recall_at_3_diff1": 0.5307956145460796, + "nauc_recall_at_3_max": 0.46477350734939027, + "nauc_recall_at_3_std": 0.32290526314893786, + "nauc_recall_at_5_diff1": 0.47362817758445497, + "nauc_recall_at_5_max": 0.4860244681874373, + "nauc_recall_at_5_std": 0.3426509746871683, + "ndcg_at_1": 0.17277, + "ndcg_at_10": 0.24659, + "ndcg_at_100": 0.31337, + "ndcg_at_1000": 0.35447, + "ndcg_at_20": 0.26971, + "ndcg_at_3": 0.20758, + "ndcg_at_5": 0.22403, + "precision_at_1": 0.17277, + "precision_at_10": 0.03432, + "precision_at_100": 0.00676, + "precision_at_1000": 0.001, + "precision_at_20": 0.0218, + "precision_at_3": 0.0778, + "precision_at_5": 0.05469, + "recall_at_1": 0.17277, + "recall_at_10": 0.34325, + "recall_at_100": 0.6762, + "recall_at_1000": 1.0, + "recall_at_20": 0.43593, + "recall_at_3": 0.23341, + "recall_at_5": 0.27346 + }, + { + "cv_recall_at_1": 0.36959, + "cv_recall_at_10": 0.63041, + "cv_recall_at_100": 0.89789, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.72919, + "cv_recall_at_3": 0.50277, + "cv_recall_at_5": 0.56826, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.49355, + "map_at_1": 0.36959, + "map_at_10": 0.45037, + "map_at_100": 0.46169, + "map_at_1000": 0.46219, + "map_at_20": 0.45725, + "map_at_3": 0.42712, + "map_at_5": 0.4421, + "mrr_at_1": 0.36958934517203107, + "mrr_at_10": 0.45036819759350283, + "mrr_at_100": 0.4616881513898778, + "mrr_at_1000": 0.4621883172413923, + "mrr_at_20": 0.45724985417101655, + "mrr_at_3": 0.42711801701812785, + "mrr_at_5": 0.4421013688494263, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4265858736100146, + "nauc_cv_recall_at_100_max": 0.6130105903171712, + "nauc_cv_recall_at_100_std": 0.6479106644247871, + "nauc_cv_recall_at_10_diff1": 0.4763556989410077, + "nauc_cv_recall_at_10_max": 0.39518628189820976, + "nauc_cv_recall_at_10_std": 0.36312964949223575, + "nauc_cv_recall_at_1_diff1": 0.6375084501609795, + "nauc_cv_recall_at_1_max": 0.4004230010736252, + "nauc_cv_recall_at_1_std": 0.2143136715398967, + "nauc_cv_recall_at_20_diff1": 0.4459499331458421, + "nauc_cv_recall_at_20_max": 0.45995435707435284, + "nauc_cv_recall_at_20_std": 0.46913009413801393, + "nauc_cv_recall_at_3_diff1": 0.5344709452451161, + "nauc_cv_recall_at_3_max": 0.3636165152262314, + "nauc_cv_recall_at_3_std": 0.27532160846534987, + "nauc_cv_recall_at_5_diff1": 0.5096097288136904, + "nauc_cv_recall_at_5_max": 0.3825057145648894, + "nauc_cv_recall_at_5_std": 0.3177712488439209, + "nauc_map_at_1000_diff1": 0.5857032193043039, + "nauc_map_at_1000_max": 0.39546456969544974, + "nauc_map_at_1000_std": 0.2547417213718815, + "nauc_map_at_100_diff1": 0.5854429365409518, + "nauc_map_at_100_max": 0.39572854085540565, + "nauc_map_at_100_std": 0.25525466157873605, + "nauc_map_at_10_diff1": 0.5853467092076959, + "nauc_map_at_10_max": 0.3921171573493067, + "nauc_map_at_10_std": 0.25239978453862266, + "nauc_map_at_1_diff1": 0.6375084501609795, + "nauc_map_at_1_max": 0.4004230010736252, + "nauc_map_at_1_std": 0.2143136715398967, + "nauc_map_at_20_diff1": 0.5847518403193711, + "nauc_map_at_20_max": 0.3947421495854351, + "nauc_map_at_20_std": 0.2562116477479193, + "nauc_map_at_3_diff1": 0.5933363215405166, + "nauc_map_at_3_max": 0.3864589857120207, + "nauc_map_at_3_std": 0.24006674573039177, + "nauc_map_at_5_diff1": 0.5883306808360874, + "nauc_map_at_5_max": 0.39028697858525846, + "nauc_map_at_5_std": 0.2484264835473533, + "nauc_mrr_at_1000_diff1": 0.5857036015963674, + "nauc_mrr_at_1000_max": 0.39546283285938855, + "nauc_mrr_at_1000_std": 0.2547396596482983, + "nauc_mrr_at_100_diff1": 0.5854433289477083, + "nauc_mrr_at_100_max": 0.39572683119053226, + "nauc_mrr_at_100_std": 0.2552526336592108, + "nauc_mrr_at_10_diff1": 0.5853467092076959, + "nauc_mrr_at_10_max": 0.3921171573493067, + "nauc_mrr_at_10_std": 0.25239978453862266, + "nauc_mrr_at_1_diff1": 0.6375084501609795, + "nauc_mrr_at_1_max": 0.4004230010736252, + "nauc_mrr_at_1_std": 0.2143136715398967, + "nauc_mrr_at_20_diff1": 0.5847518403193711, + "nauc_mrr_at_20_max": 0.3947421495854351, + "nauc_mrr_at_20_std": 0.2562116477479193, + "nauc_mrr_at_3_diff1": 0.5933363215405166, + "nauc_mrr_at_3_max": 0.3864589857120207, + "nauc_mrr_at_3_std": 0.24006674573039177, + "nauc_mrr_at_5_diff1": 0.5883306808360874, + "nauc_mrr_at_5_max": 0.39028697858525846, + "nauc_mrr_at_5_std": 0.2484264835473533, + "nauc_ndcg_at_1000_diff1": 0.5729010036437998, + "nauc_ndcg_at_1000_max": 0.40063261790125426, + "nauc_ndcg_at_1000_std": 0.2713028785224261, + "nauc_ndcg_at_100_diff1": 0.5663054737618531, + "nauc_ndcg_at_100_max": 0.40972102733006077, + "nauc_ndcg_at_100_std": 0.28757487816844396, + "nauc_ndcg_at_10_diff1": 0.5614605598216259, + "nauc_ndcg_at_10_max": 0.39222010301524096, + "nauc_ndcg_at_10_std": 0.27580314095413405, + "nauc_ndcg_at_1_diff1": 0.6375084501609795, + "nauc_ndcg_at_1_max": 0.4004230010736252, + "nauc_ndcg_at_1_std": 0.2143136715398967, + "nauc_ndcg_at_20_diff1": 0.5591453506371686, + "nauc_ndcg_at_20_max": 0.4036424685141276, + "nauc_ndcg_at_20_std": 0.2916509600022843, + "nauc_ndcg_at_3_diff1": 0.5784145380252298, + "nauc_ndcg_at_3_max": 0.3807448902432253, + "nauc_ndcg_at_3_std": 0.24898665496611744, + "nauc_ndcg_at_5_diff1": 0.5693980880921701, + "nauc_ndcg_at_5_max": 0.388113991989614, + "nauc_ndcg_at_5_std": 0.26469207043279036, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.426585873610016, + "nauc_precision_at_100_max": 0.6130105903171704, + "nauc_precision_at_100_std": 0.6479106644247877, + "nauc_precision_at_10_diff1": 0.4763556989410072, + "nauc_precision_at_10_max": 0.39518628189820937, + "nauc_precision_at_10_std": 0.36312964949223514, + "nauc_precision_at_1_diff1": 0.6375084501609795, + "nauc_precision_at_1_max": 0.4004230010736252, + "nauc_precision_at_1_std": 0.2143136715398967, + "nauc_precision_at_20_diff1": 0.4459499331458413, + "nauc_precision_at_20_max": 0.4599543570743535, + "nauc_precision_at_20_std": 0.4691300941380129, + "nauc_precision_at_3_diff1": 0.5344709452451162, + "nauc_precision_at_3_max": 0.3636165152262313, + "nauc_precision_at_3_std": 0.2753216084653498, + "nauc_precision_at_5_diff1": 0.5096097288136904, + "nauc_precision_at_5_max": 0.3825057145648895, + "nauc_precision_at_5_std": 0.31777124884392066, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4265858736100146, + "nauc_recall_at_100_max": 0.6130105903171712, + "nauc_recall_at_100_std": 0.6479106644247871, + "nauc_recall_at_10_diff1": 0.4763556989410077, + "nauc_recall_at_10_max": 0.39518628189820976, + "nauc_recall_at_10_std": 0.36312964949223575, + "nauc_recall_at_1_diff1": 0.6375084501609795, + "nauc_recall_at_1_max": 0.4004230010736252, + "nauc_recall_at_1_std": 0.2143136715398967, + "nauc_recall_at_20_diff1": 0.4459499331458421, + "nauc_recall_at_20_max": 0.45995435707435284, + "nauc_recall_at_20_std": 0.46913009413801393, + "nauc_recall_at_3_diff1": 0.5344709452451161, + "nauc_recall_at_3_max": 0.3636165152262314, + "nauc_recall_at_3_std": 0.27532160846534987, + "nauc_recall_at_5_diff1": 0.5096097288136904, + "nauc_recall_at_5_max": 0.3825057145648894, + "nauc_recall_at_5_std": 0.3177712488439209, + "ndcg_at_1": 0.36959, + "ndcg_at_10": 0.49355, + "ndcg_at_100": 0.55011, + "ndcg_at_1000": 0.56319, + "ndcg_at_20": 0.51855, + "ndcg_at_3": 0.4465, + "ndcg_at_5": 0.47349, + "precision_at_1": 0.36959, + "precision_at_10": 0.06304, + "precision_at_100": 0.00898, + "precision_at_1000": 0.001, + "precision_at_20": 0.03646, + "precision_at_3": 0.16759, + "precision_at_5": 0.11365, + "recall_at_1": 0.36959, + "recall_at_10": 0.63041, + "recall_at_100": 0.89789, + "recall_at_1000": 1.0, + "recall_at_20": 0.72919, + "recall_at_3": 0.50277, + "recall_at_5": 0.56826 + }, + { + "cv_recall_at_1": 0.087, + "cv_recall_at_10": 0.19656, + "cv_recall_at_100": 0.46079, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.25349, + "cv_recall_at_3": 0.12567, + "cv_recall_at_5": 0.15145, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.13391, + "map_at_1": 0.087, + "map_at_10": 0.1149, + "map_at_100": 0.12368, + "map_at_1000": 0.12578, + "map_at_20": 0.11893, + "map_at_3": 0.10311, + "map_at_5": 0.10924, + "mrr_at_1": 0.08700322234156821, + "mrr_at_10": 0.11489736245375343, + "mrr_at_100": 0.12367750257748027, + "mrr_at_1000": 0.12577974098864964, + "mrr_at_20": 0.11893280521224774, + "mrr_at_3": 0.10311493018259932, + "mrr_at_5": 0.10923737916219116, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.22095843734622075, + "nauc_cv_recall_at_100_max": 0.25029245970753483, + "nauc_cv_recall_at_100_std": 0.3215991312676245, + "nauc_cv_recall_at_10_diff1": 0.3603023327796987, + "nauc_cv_recall_at_10_max": 0.33497378488984614, + "nauc_cv_recall_at_10_std": 0.3351531422726964, + "nauc_cv_recall_at_1_diff1": 0.6847321341274407, + "nauc_cv_recall_at_1_max": 0.47811747616186656, + "nauc_cv_recall_at_1_std": 0.3783668201615523, + "nauc_cv_recall_at_20_diff1": 0.3116302886581901, + "nauc_cv_recall_at_20_max": 0.29917621370786907, + "nauc_cv_recall_at_20_std": 0.31617724986963225, + "nauc_cv_recall_at_3_diff1": 0.5070298216820545, + "nauc_cv_recall_at_3_max": 0.4015417695684175, + "nauc_cv_recall_at_3_std": 0.3687153985936076, + "nauc_cv_recall_at_5_diff1": 0.45106428207293814, + "nauc_cv_recall_at_5_max": 0.3703391700087565, + "nauc_cv_recall_at_5_std": 0.3441693889123974, + "nauc_map_at_1000_diff1": 0.5334132974149114, + "nauc_map_at_1000_max": 0.4064602928580575, + "nauc_map_at_1000_std": 0.35204806392949156, + "nauc_map_at_100_diff1": 0.5322119948770907, + "nauc_map_at_100_max": 0.40531017093092836, + "nauc_map_at_100_std": 0.35167250774892, + "nauc_map_at_10_diff1": 0.5475273770715559, + "nauc_map_at_10_max": 0.41406016843897586, + "nauc_map_at_10_std": 0.35488317945498377, + "nauc_map_at_1_diff1": 0.6847321341274407, + "nauc_map_at_1_max": 0.47811747616186656, + "nauc_map_at_1_std": 0.3783668201615523, + "nauc_map_at_20_diff1": 0.5387576711800679, + "nauc_map_at_20_max": 0.4083036835156604, + "nauc_map_at_20_std": 0.35246396647442946, + "nauc_map_at_3_diff1": 0.5898863721170673, + "nauc_map_at_3_max": 0.43472652790728333, + "nauc_map_at_3_std": 0.36490905144970676, + "nauc_map_at_5_diff1": 0.5694469364779752, + "nauc_map_at_5_max": 0.4230697053023698, + "nauc_map_at_5_std": 0.35765181185393363, + "nauc_mrr_at_1000_diff1": 0.5334127890646034, + "nauc_mrr_at_1000_max": 0.406458644275623, + "nauc_mrr_at_1000_std": 0.3520478053565356, + "nauc_mrr_at_100_diff1": 0.5322114897576498, + "nauc_mrr_at_100_max": 0.4053085319250195, + "nauc_mrr_at_100_std": 0.3516722507420569, + "nauc_mrr_at_10_diff1": 0.5475273770715559, + "nauc_mrr_at_10_max": 0.41406016843897586, + "nauc_mrr_at_10_std": 0.35488317945498377, + "nauc_mrr_at_1_diff1": 0.6847321341274407, + "nauc_mrr_at_1_max": 0.47811747616186656, + "nauc_mrr_at_1_std": 0.3783668201615523, + "nauc_mrr_at_20_diff1": 0.5387576711800679, + "nauc_mrr_at_20_max": 0.4083036835156604, + "nauc_mrr_at_20_std": 0.35246396647442946, + "nauc_mrr_at_3_diff1": 0.5898863721170673, + "nauc_mrr_at_3_max": 0.43472652790728333, + "nauc_mrr_at_3_std": 0.36490905144970676, + "nauc_mrr_at_5_diff1": 0.5694469364779752, + "nauc_mrr_at_5_max": 0.4230697053023698, + "nauc_mrr_at_5_std": 0.35765181185393363, + "nauc_ndcg_at_1000_diff1": 0.476617380656095, + "nauc_ndcg_at_1000_max": 0.3818200888368288, + "nauc_ndcg_at_1000_std": 0.3456048197225887, + "nauc_ndcg_at_100_diff1": 0.43184711588315283, + "nauc_ndcg_at_100_max": 0.3563522077934337, + "nauc_ndcg_at_100_std": 0.34041098252993757, + "nauc_ndcg_at_10_diff1": 0.48809093672959647, + "nauc_ndcg_at_10_max": 0.38870531314884205, + "nauc_ndcg_at_10_std": 0.34846356080211893, + "nauc_ndcg_at_1_diff1": 0.6847321341274407, + "nauc_ndcg_at_1_max": 0.47811747616186656, + "nauc_ndcg_at_1_std": 0.3783668201615523, + "nauc_ndcg_at_20_diff1": 0.4643378062200878, + "nauc_ndcg_at_20_max": 0.37277590099233887, + "nauc_ndcg_at_20_std": 0.34101321085440256, + "nauc_ndcg_at_3_diff1": 0.5658668804015625, + "nauc_ndcg_at_3_max": 0.4249995108514367, + "nauc_ndcg_at_3_std": 0.36565975617467283, + "nauc_ndcg_at_5_diff1": 0.5337700413900782, + "nauc_ndcg_at_5_max": 0.4070769919703173, + "nauc_ndcg_at_5_std": 0.35355366721091624, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.2209584373462211, + "nauc_precision_at_100_max": 0.2502924597075356, + "nauc_precision_at_100_std": 0.3215991312676248, + "nauc_precision_at_10_diff1": 0.3603023327796984, + "nauc_precision_at_10_max": 0.3349737848898459, + "nauc_precision_at_10_std": 0.3351531422726965, + "nauc_precision_at_1_diff1": 0.6847321341274407, + "nauc_precision_at_1_max": 0.47811747616186656, + "nauc_precision_at_1_std": 0.3783668201615523, + "nauc_precision_at_20_diff1": 0.3116302886581901, + "nauc_precision_at_20_max": 0.2991762137078694, + "nauc_precision_at_20_std": 0.31617724986963225, + "nauc_precision_at_3_diff1": 0.5070298216820542, + "nauc_precision_at_3_max": 0.40154176956841753, + "nauc_precision_at_3_std": 0.3687153985936076, + "nauc_precision_at_5_diff1": 0.45106428207293786, + "nauc_precision_at_5_max": 0.3703391700087562, + "nauc_precision_at_5_std": 0.3441693889123974, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.22095843734622075, + "nauc_recall_at_100_max": 0.25029245970753483, + "nauc_recall_at_100_std": 0.3215991312676245, + "nauc_recall_at_10_diff1": 0.3603023327796987, + "nauc_recall_at_10_max": 0.33497378488984614, + "nauc_recall_at_10_std": 0.3351531422726964, + "nauc_recall_at_1_diff1": 0.6847321341274407, + "nauc_recall_at_1_max": 0.47811747616186656, + "nauc_recall_at_1_std": 0.3783668201615523, + "nauc_recall_at_20_diff1": 0.3116302886581901, + "nauc_recall_at_20_max": 0.29917621370786907, + "nauc_recall_at_20_std": 0.31617724986963225, + "nauc_recall_at_3_diff1": 0.5070298216820545, + "nauc_recall_at_3_max": 0.4015417695684175, + "nauc_recall_at_3_std": 0.3687153985936076, + "nauc_recall_at_5_diff1": 0.45106428207293814, + "nauc_recall_at_5_max": 0.3703391700087565, + "nauc_recall_at_5_std": 0.3441693889123974, + "ndcg_at_1": 0.087, + "ndcg_at_10": 0.13391, + "ndcg_at_100": 0.18566, + "ndcg_at_1000": 0.25167, + "ndcg_at_20": 0.14842, + "ndcg_at_3": 0.10887, + "ndcg_at_5": 0.11969, + "precision_at_1": 0.087, + "precision_at_10": 0.01966, + "precision_at_100": 0.00461, + "precision_at_1000": 0.001, + "precision_at_20": 0.01267, + "precision_at_3": 0.04189, + "precision_at_5": 0.03029, + "recall_at_1": 0.087, + "recall_at_10": 0.19656, + "recall_at_100": 0.46079, + "recall_at_1000": 1.0, + "recall_at_20": 0.25349, + "recall_at_3": 0.12567, + "recall_at_5": 0.15145 + }, + { + "cv_recall_at_1": 0.082, + "cv_recall_at_10": 0.26, + "cv_recall_at_100": 0.574, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.32, + "cv_recall_at_3": 0.149, + "cv_recall_at_5": 0.2, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.16092, + "map_at_1": 0.082, + "map_at_10": 0.1303, + "map_at_100": 0.14011, + "map_at_1000": 0.14192, + "map_at_20": 0.13451, + "map_at_3": 0.11017, + "map_at_5": 0.12222, + "mrr_at_1": 0.082, + "mrr_at_10": 0.1302984126984126, + "mrr_at_100": 0.14010643849757057, + "mrr_at_1000": 0.14191966909611997, + "mrr_at_20": 0.13451473022161872, + "mrr_at_3": 0.11016666666666662, + "mrr_at_5": 0.12221666666666663, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.12957698971663834, + "nauc_cv_recall_at_100_max": 0.18513510786430704, + "nauc_cv_recall_at_100_std": 0.2848998619083174, + "nauc_cv_recall_at_10_diff1": 0.27739296348761233, + "nauc_cv_recall_at_10_max": 0.06266225318724539, + "nauc_cv_recall_at_10_std": 0.1555158856349918, + "nauc_cv_recall_at_1_diff1": 0.5317361115655695, + "nauc_cv_recall_at_1_max": 0.12698043937777712, + "nauc_cv_recall_at_1_std": 0.13111310345730237, + "nauc_cv_recall_at_20_diff1": 0.23898239323557738, + "nauc_cv_recall_at_20_max": 0.08588114096114723, + "nauc_cv_recall_at_20_std": 0.1700631488815156, + "nauc_cv_recall_at_3_diff1": 0.35436188037270344, + "nauc_cv_recall_at_3_max": 0.04819250257808713, + "nauc_cv_recall_at_3_std": 0.16807912752262544, + "nauc_cv_recall_at_5_diff1": 0.2936341620923475, + "nauc_cv_recall_at_5_max": 0.0383467872134323, + "nauc_cv_recall_at_5_std": 0.14974652889893447, + "nauc_map_at_1000_diff1": 0.3916961691104259, + "nauc_map_at_1000_max": 0.09453808212623574, + "nauc_map_at_1000_std": 0.15397478560727407, + "nauc_map_at_100_diff1": 0.390296329267604, + "nauc_map_at_100_max": 0.09447881047349718, + "nauc_map_at_100_std": 0.15419789099089493, + "nauc_map_at_10_diff1": 0.3998884462384207, + "nauc_map_at_10_max": 0.08704327678044757, + "nauc_map_at_10_std": 0.14764227122018564, + "nauc_map_at_1_diff1": 0.5317361115655695, + "nauc_map_at_1_max": 0.12698043937777712, + "nauc_map_at_1_std": 0.13111310345730237, + "nauc_map_at_20_diff1": 0.3942263239393362, + "nauc_map_at_20_max": 0.08900280364693693, + "nauc_map_at_20_std": 0.14877466455415778, + "nauc_map_at_3_diff1": 0.4322072311999196, + "nauc_map_at_3_max": 0.09024055839461552, + "nauc_map_at_3_std": 0.15063310629000448, + "nauc_map_at_5_diff1": 0.40613624024338546, + "nauc_map_at_5_max": 0.0835408380843802, + "nauc_map_at_5_std": 0.14696146124460574, + "nauc_mrr_at_1000_diff1": 0.3916961691104259, + "nauc_mrr_at_1000_max": 0.09453808212623574, + "nauc_mrr_at_1000_std": 0.15397478560727407, + "nauc_mrr_at_100_diff1": 0.390296329267604, + "nauc_mrr_at_100_max": 0.09447881047349718, + "nauc_mrr_at_100_std": 0.15419789099089493, + "nauc_mrr_at_10_diff1": 0.3998884462384207, + "nauc_mrr_at_10_max": 0.08704327678044757, + "nauc_mrr_at_10_std": 0.14764227122018564, + "nauc_mrr_at_1_diff1": 0.5317361115655695, + "nauc_mrr_at_1_max": 0.12698043937777712, + "nauc_mrr_at_1_std": 0.13111310345730237, + "nauc_mrr_at_20_diff1": 0.3942263239393362, + "nauc_mrr_at_20_max": 0.08900280364693693, + "nauc_mrr_at_20_std": 0.14877466455415778, + "nauc_mrr_at_3_diff1": 0.4322072311999196, + "nauc_mrr_at_3_max": 0.09024055839461552, + "nauc_mrr_at_3_std": 0.15063310629000448, + "nauc_mrr_at_5_diff1": 0.40613624024338546, + "nauc_mrr_at_5_max": 0.0835408380843802, + "nauc_mrr_at_5_std": 0.14696146124460574, + "nauc_ndcg_at_1000_diff1": 0.3532959414745214, + "nauc_ndcg_at_1000_max": 0.10288254390326823, + "nauc_ndcg_at_1000_std": 0.1700140214238286, + "nauc_ndcg_at_100_diff1": 0.3157125547302426, + "nauc_ndcg_at_100_max": 0.11360475881829207, + "nauc_ndcg_at_100_std": 0.18628635111438582, + "nauc_ndcg_at_10_diff1": 0.35801968786277605, + "nauc_ndcg_at_10_max": 0.07767693928985969, + "nauc_ndcg_at_10_std": 0.1504383247734284, + "nauc_ndcg_at_1_diff1": 0.5317361115655695, + "nauc_ndcg_at_1_max": 0.12698043937777712, + "nauc_ndcg_at_1_std": 0.13111310345730237, + "nauc_ndcg_at_20_diff1": 0.34147465995699866, + "nauc_ndcg_at_20_max": 0.08529823718283426, + "nauc_ndcg_at_20_std": 0.15529041642225935, + "nauc_ndcg_at_3_diff1": 0.4083555318854164, + "nauc_ndcg_at_3_max": 0.07765520581609091, + "nauc_ndcg_at_3_std": 0.1559297532311266, + "nauc_ndcg_at_5_diff1": 0.3691673258075353, + "nauc_ndcg_at_5_max": 0.06867370893589116, + "nauc_ndcg_at_5_std": 0.14830740352720787, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.1295769897166383, + "nauc_precision_at_100_max": 0.1851351078643071, + "nauc_precision_at_100_std": 0.28489986190831773, + "nauc_precision_at_10_diff1": 0.2773929634876122, + "nauc_precision_at_10_max": 0.06266225318724516, + "nauc_precision_at_10_std": 0.15551588563499155, + "nauc_precision_at_1_diff1": 0.5317361115655695, + "nauc_precision_at_1_max": 0.12698043937777712, + "nauc_precision_at_1_std": 0.13111310345730237, + "nauc_precision_at_20_diff1": 0.23898239323557724, + "nauc_precision_at_20_max": 0.08588114096114721, + "nauc_precision_at_20_std": 0.17006314888151536, + "nauc_precision_at_3_diff1": 0.35436188037270333, + "nauc_precision_at_3_max": 0.048192502578087264, + "nauc_precision_at_3_std": 0.16807912752262555, + "nauc_precision_at_5_diff1": 0.29363416209234733, + "nauc_precision_at_5_max": 0.038346787213432404, + "nauc_precision_at_5_std": 0.14974652889893444, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.12957698971663834, + "nauc_recall_at_100_max": 0.18513510786430704, + "nauc_recall_at_100_std": 0.2848998619083174, + "nauc_recall_at_10_diff1": 0.27739296348761233, + "nauc_recall_at_10_max": 0.06266225318724539, + "nauc_recall_at_10_std": 0.1555158856349918, + "nauc_recall_at_1_diff1": 0.5317361115655695, + "nauc_recall_at_1_max": 0.12698043937777712, + "nauc_recall_at_1_std": 0.13111310345730237, + "nauc_recall_at_20_diff1": 0.23898239323557738, + "nauc_recall_at_20_max": 0.08588114096114723, + "nauc_recall_at_20_std": 0.1700631488815156, + "nauc_recall_at_3_diff1": 0.35436188037270344, + "nauc_recall_at_3_max": 0.04819250257808713, + "nauc_recall_at_3_std": 0.16807912752262544, + "nauc_recall_at_5_diff1": 0.2936341620923475, + "nauc_recall_at_5_max": 0.0383467872134323, + "nauc_recall_at_5_std": 0.14974652889893447, + "ndcg_at_1": 0.082, + "ndcg_at_10": 0.16092, + "ndcg_at_100": 0.22146, + "ndcg_at_1000": 0.27457, + "ndcg_at_20": 0.17615, + "ndcg_at_3": 0.12008, + "ndcg_at_5": 0.14143, + "precision_at_1": 0.082, + "precision_at_10": 0.026, + "precision_at_100": 0.00574, + "precision_at_1000": 0.001, + "precision_at_20": 0.016, + "precision_at_3": 0.04967, + "precision_at_5": 0.04, + "recall_at_1": 0.082, + "recall_at_10": 0.26, + "recall_at_100": 0.574, + "recall_at_1000": 1.0, + "recall_at_20": 0.32, + "recall_at_3": 0.149, + "recall_at_5": 0.2 + }, + { + "cv_recall_at_1": 0.30097, + "cv_recall_at_10": 0.56311, + "cv_recall_at_100": 0.83218, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.65049, + "cv_recall_at_3": 0.43689, + "cv_recall_at_5": 0.49376, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.42733, + "map_at_1": 0.30097, + "map_at_10": 0.3845, + "map_at_100": 0.39491, + "map_at_1000": 0.39581, + "map_at_20": 0.39073, + "map_at_3": 0.36223, + "map_at_5": 0.37527, + "mrr_at_1": 0.30097087378640774, + "mrr_at_10": 0.38450399577306643, + "mrr_at_100": 0.39491072235582175, + "mrr_at_1000": 0.3958122134598317, + "mrr_at_20": 0.39072532037400665, + "mrr_at_3": 0.36222838650023137, + "mrr_at_5": 0.3752658344891355, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4625246100334655, + "nauc_cv_recall_at_100_max": 0.4933803291224194, + "nauc_cv_recall_at_100_std": 0.6051973615855294, + "nauc_cv_recall_at_10_diff1": 0.4868176969594762, + "nauc_cv_recall_at_10_max": 0.5056329109842426, + "nauc_cv_recall_at_10_std": 0.4244211679740214, + "nauc_cv_recall_at_1_diff1": 0.6346058198200373, + "nauc_cv_recall_at_1_max": 0.5195356174195791, + "nauc_cv_recall_at_1_std": 0.335413193872272, + "nauc_cv_recall_at_20_diff1": 0.4835757726879039, + "nauc_cv_recall_at_20_max": 0.509186250028205, + "nauc_cv_recall_at_20_std": 0.4963081401316029, + "nauc_cv_recall_at_3_diff1": 0.4778302006907091, + "nauc_cv_recall_at_3_max": 0.4964648160795888, + "nauc_cv_recall_at_3_std": 0.4227748046018749, + "nauc_cv_recall_at_5_diff1": 0.46570883018839004, + "nauc_cv_recall_at_5_max": 0.4985706519151476, + "nauc_cv_recall_at_5_std": 0.40929979622927776, + "nauc_map_at_1000_diff1": 0.5635129914592062, + "nauc_map_at_1000_max": 0.5126271092937562, + "nauc_map_at_1000_std": 0.37999202563630197, + "nauc_map_at_100_diff1": 0.563195913165389, + "nauc_map_at_100_max": 0.5124783062168992, + "nauc_map_at_100_std": 0.38054266238217466, + "nauc_map_at_10_diff1": 0.562849647014662, + "nauc_map_at_10_max": 0.5119277625419975, + "nauc_map_at_10_std": 0.3753089900038274, + "nauc_map_at_1_diff1": 0.6346058198200373, + "nauc_map_at_1_max": 0.5195356174195791, + "nauc_map_at_1_std": 0.335413193872272, + "nauc_map_at_20_diff1": 0.5631407378790679, + "nauc_map_at_20_max": 0.5122987639602273, + "nauc_map_at_20_std": 0.37930908581900735, + "nauc_map_at_3_diff1": 0.5628566451051329, + "nauc_map_at_3_max": 0.5105530097848331, + "nauc_map_at_3_std": 0.3772714487963746, + "nauc_map_at_5_diff1": 0.5603110311518988, + "nauc_map_at_5_max": 0.5112275663372864, + "nauc_map_at_5_std": 0.3735154182994574, + "nauc_mrr_at_1000_diff1": 0.5635112864100491, + "nauc_mrr_at_1000_max": 0.5126242058711608, + "nauc_mrr_at_1000_std": 0.37998936423437424, + "nauc_mrr_at_100_diff1": 0.5631942374373318, + "nauc_mrr_at_100_max": 0.5124753808585685, + "nauc_mrr_at_100_std": 0.38053997223693614, + "nauc_mrr_at_10_diff1": 0.562849647014662, + "nauc_mrr_at_10_max": 0.5119277625419975, + "nauc_mrr_at_10_std": 0.3753089900038274, + "nauc_mrr_at_1_diff1": 0.6346058198200373, + "nauc_mrr_at_1_max": 0.5195356174195791, + "nauc_mrr_at_1_std": 0.335413193872272, + "nauc_mrr_at_20_diff1": 0.5631407378790679, + "nauc_mrr_at_20_max": 0.5122987639602273, + "nauc_mrr_at_20_std": 0.37930908581900735, + "nauc_mrr_at_3_diff1": 0.5628566451051329, + "nauc_mrr_at_3_max": 0.5105530097848331, + "nauc_mrr_at_3_std": 0.3772714487963746, + "nauc_mrr_at_5_diff1": 0.5603110311518988, + "nauc_mrr_at_5_max": 0.5112275663372864, + "nauc_mrr_at_5_std": 0.3735154182994574, + "nauc_ndcg_at_1000_diff1": 0.5521570506994392, + "nauc_ndcg_at_1000_max": 0.5123679273042125, + "nauc_ndcg_at_1000_std": 0.3934861592656324, + "nauc_ndcg_at_100_diff1": 0.5457695675871636, + "nauc_ndcg_at_100_max": 0.5104748090246876, + "nauc_ndcg_at_100_std": 0.40754977276938625, + "nauc_ndcg_at_10_diff1": 0.5431720842235095, + "nauc_ndcg_at_10_max": 0.5100874224849119, + "nauc_ndcg_at_10_std": 0.3872665155700147, + "nauc_ndcg_at_1_diff1": 0.6346058198200373, + "nauc_ndcg_at_1_max": 0.5195356174195791, + "nauc_ndcg_at_1_std": 0.335413193872272, + "nauc_ndcg_at_20_diff1": 0.544019392334048, + "nauc_ndcg_at_20_max": 0.5111400368148176, + "nauc_ndcg_at_20_std": 0.402073504650038, + "nauc_ndcg_at_3_diff1": 0.54071925005883, + "nauc_ndcg_at_3_max": 0.5069523340371278, + "nauc_ndcg_at_3_std": 0.3891988182889605, + "nauc_ndcg_at_5_diff1": 0.5361030708242451, + "nauc_ndcg_at_5_max": 0.5080185032370559, + "nauc_ndcg_at_5_std": 0.3829646873337103, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.46252461003346523, + "nauc_precision_at_100_max": 0.49338032912241975, + "nauc_precision_at_100_std": 0.6051973615855295, + "nauc_precision_at_10_diff1": 0.48681769695947613, + "nauc_precision_at_10_max": 0.5056329109842427, + "nauc_precision_at_10_std": 0.424421167974021, + "nauc_precision_at_1_diff1": 0.6346058198200373, + "nauc_precision_at_1_max": 0.5195356174195791, + "nauc_precision_at_1_std": 0.335413193872272, + "nauc_precision_at_20_diff1": 0.4835757726879036, + "nauc_precision_at_20_max": 0.5091862500282047, + "nauc_precision_at_20_std": 0.4963081401316024, + "nauc_precision_at_3_diff1": 0.47783020069070864, + "nauc_precision_at_3_max": 0.49646481607958853, + "nauc_precision_at_3_std": 0.4227748046018748, + "nauc_precision_at_5_diff1": 0.46570883018839015, + "nauc_precision_at_5_max": 0.4985706519151479, + "nauc_precision_at_5_std": 0.40929979622927737, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4625246100334655, + "nauc_recall_at_100_max": 0.4933803291224194, + "nauc_recall_at_100_std": 0.6051973615855294, + "nauc_recall_at_10_diff1": 0.4868176969594762, + "nauc_recall_at_10_max": 0.5056329109842426, + "nauc_recall_at_10_std": 0.4244211679740214, + "nauc_recall_at_1_diff1": 0.6346058198200373, + "nauc_recall_at_1_max": 0.5195356174195791, + "nauc_recall_at_1_std": 0.335413193872272, + "nauc_recall_at_20_diff1": 0.4835757726879039, + "nauc_recall_at_20_max": 0.509186250028205, + "nauc_recall_at_20_std": 0.4963081401316029, + "nauc_recall_at_3_diff1": 0.4778302006907091, + "nauc_recall_at_3_max": 0.4964648160795888, + "nauc_recall_at_3_std": 0.4227748046018749, + "nauc_recall_at_5_diff1": 0.46570883018839004, + "nauc_recall_at_5_max": 0.4985706519151476, + "nauc_recall_at_5_std": 0.40929979622927776, + "ndcg_at_1": 0.30097, + "ndcg_at_10": 0.42733, + "ndcg_at_100": 0.48236, + "ndcg_at_1000": 0.50435, + "ndcg_at_20": 0.44962, + "ndcg_at_3": 0.38146, + "ndcg_at_5": 0.40492, + "precision_at_1": 0.30097, + "precision_at_10": 0.05631, + "precision_at_100": 0.00832, + "precision_at_1000": 0.001, + "precision_at_20": 0.03252, + "precision_at_3": 0.14563, + "precision_at_5": 0.09875, + "recall_at_1": 0.30097, + "recall_at_10": 0.56311, + "recall_at_100": 0.83218, + "recall_at_1000": 1.0, + "recall_at_20": 0.65049, + "recall_at_3": 0.43689, + "recall_at_5": 0.49376 + }, + { + "cv_recall_at_1": 0.28224, + "cv_recall_at_10": 0.54863, + "cv_recall_at_100": 0.82241, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.62685, + "cv_recall_at_3": 0.40063, + "cv_recall_at_5": 0.45983, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.40522, + "map_at_1": 0.28224, + "map_at_10": 0.36064, + "map_at_100": 0.37109, + "map_at_1000": 0.37193, + "map_at_20": 0.36617, + "map_at_3": 0.3351, + "map_at_5": 0.34863, + "mrr_at_1": 0.2822410147991543, + "mrr_at_10": 0.3606366824390079, + "mrr_at_100": 0.3710940391372773, + "mrr_at_1000": 0.3719294554550222, + "mrr_at_20": 0.3661695515172595, + "mrr_at_3": 0.3350951374207188, + "mrr_at_5": 0.3486257928118391, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.34201335085299767, + "nauc_cv_recall_at_100_max": 0.4623789767253533, + "nauc_cv_recall_at_100_std": 0.6192812082541869, + "nauc_cv_recall_at_10_diff1": 0.41445167336327915, + "nauc_cv_recall_at_10_max": 0.4014545477647777, + "nauc_cv_recall_at_10_std": 0.39772790563947014, + "nauc_cv_recall_at_1_diff1": 0.6337414799404985, + "nauc_cv_recall_at_1_max": 0.5114562833991539, + "nauc_cv_recall_at_1_std": 0.36764760561164883, + "nauc_cv_recall_at_20_diff1": 0.37977308593119635, + "nauc_cv_recall_at_20_max": 0.38489121713355706, + "nauc_cv_recall_at_20_std": 0.40658924970974497, + "nauc_cv_recall_at_3_diff1": 0.4767990083649137, + "nauc_cv_recall_at_3_max": 0.4456114997013859, + "nauc_cv_recall_at_3_std": 0.38757663075347437, + "nauc_cv_recall_at_5_diff1": 0.4368711432697605, + "nauc_cv_recall_at_5_max": 0.41052511580267553, + "nauc_cv_recall_at_5_std": 0.3946283429118848, + "nauc_map_at_1000_diff1": 0.5523103220721189, + "nauc_map_at_1000_max": 0.47666446029624304, + "nauc_map_at_1000_std": 0.38216888015698586, + "nauc_map_at_100_diff1": 0.5518295493827026, + "nauc_map_at_100_max": 0.476549602965261, + "nauc_map_at_100_std": 0.3826181329591628, + "nauc_map_at_10_diff1": 0.5528407331133731, + "nauc_map_at_10_max": 0.4754644451763753, + "nauc_map_at_10_std": 0.3795646303723107, + "nauc_map_at_1_diff1": 0.6337414799404985, + "nauc_map_at_1_max": 0.5114562833991539, + "nauc_map_at_1_std": 0.36764760561164883, + "nauc_map_at_20_diff1": 0.5511520073528766, + "nauc_map_at_20_max": 0.47449026353447393, + "nauc_map_at_20_std": 0.3800723775105995, + "nauc_map_at_3_diff1": 0.5643316970303613, + "nauc_map_at_3_max": 0.4842459956105745, + "nauc_map_at_3_std": 0.3774511286290846, + "nauc_map_at_5_diff1": 0.5553906790668495, + "nauc_map_at_5_max": 0.4764659357568404, + "nauc_map_at_5_std": 0.3794113343603276, + "nauc_mrr_at_1000_diff1": 0.5523103711559296, + "nauc_mrr_at_1000_max": 0.47666445633151716, + "nauc_mrr_at_1000_std": 0.38216891937928554, + "nauc_mrr_at_100_diff1": 0.5518295493827026, + "nauc_mrr_at_100_max": 0.476549602965261, + "nauc_mrr_at_100_std": 0.3826181329591628, + "nauc_mrr_at_10_diff1": 0.5528407331133731, + "nauc_mrr_at_10_max": 0.4754644451763753, + "nauc_mrr_at_10_std": 0.3795646303723107, + "nauc_mrr_at_1_diff1": 0.6337414799404985, + "nauc_mrr_at_1_max": 0.5114562833991539, + "nauc_mrr_at_1_std": 0.36764760561164883, + "nauc_mrr_at_20_diff1": 0.5511520073528766, + "nauc_mrr_at_20_max": 0.47449026353447393, + "nauc_mrr_at_20_std": 0.3800723775105995, + "nauc_mrr_at_3_diff1": 0.5643316970303613, + "nauc_mrr_at_3_max": 0.4842459956105745, + "nauc_mrr_at_3_std": 0.3774511286290846, + "nauc_mrr_at_5_diff1": 0.5553906790668495, + "nauc_mrr_at_5_max": 0.4764659357568404, + "nauc_mrr_at_5_std": 0.3794113343603276, + "nauc_ndcg_at_1000_diff1": 0.5321001402438658, + "nauc_ndcg_at_1000_max": 0.46935887821078137, + "nauc_ndcg_at_1000_std": 0.3906187271528008, + "nauc_ndcg_at_100_diff1": 0.519046454926447, + "nauc_ndcg_at_100_max": 0.4684330980797537, + "nauc_ndcg_at_100_std": 0.40573077981906797, + "nauc_ndcg_at_10_diff1": 0.5192128553827087, + "nauc_ndcg_at_10_max": 0.45768256351389847, + "nauc_ndcg_at_10_std": 0.3839833147055718, + "nauc_ndcg_at_1_diff1": 0.6337414799404985, + "nauc_ndcg_at_1_max": 0.5114562833991539, + "nauc_ndcg_at_1_std": 0.36764760561164883, + "nauc_ndcg_at_20_diff1": 0.5133076590199503, + "nauc_ndcg_at_20_max": 0.4547434106411084, + "nauc_ndcg_at_20_std": 0.38576204358020355, + "nauc_ndcg_at_3_diff1": 0.541276579669424, + "nauc_ndcg_at_3_max": 0.4741475983069668, + "nauc_ndcg_at_3_std": 0.3801582142936978, + "nauc_ndcg_at_5_diff1": 0.5250777835596054, + "nauc_ndcg_at_5_max": 0.459898236719731, + "nauc_ndcg_at_5_std": 0.3833626280258285, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.34201335085299867, + "nauc_precision_at_100_max": 0.4623789767253534, + "nauc_precision_at_100_std": 0.6192812082541859, + "nauc_precision_at_10_diff1": 0.4144516733632791, + "nauc_precision_at_10_max": 0.40145454776477707, + "nauc_precision_at_10_std": 0.3977279056394695, + "nauc_precision_at_1_diff1": 0.6337414799404985, + "nauc_precision_at_1_max": 0.5114562833991539, + "nauc_precision_at_1_std": 0.36764760561164883, + "nauc_precision_at_20_diff1": 0.37977308593119563, + "nauc_precision_at_20_max": 0.3848912171335574, + "nauc_precision_at_20_std": 0.4065892497097443, + "nauc_precision_at_3_diff1": 0.47679900836491307, + "nauc_precision_at_3_max": 0.44561149970138586, + "nauc_precision_at_3_std": 0.38757663075347404, + "nauc_precision_at_5_diff1": 0.43687114326976084, + "nauc_precision_at_5_max": 0.4105251158026755, + "nauc_precision_at_5_std": 0.3946283429118848, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.34201335085299767, + "nauc_recall_at_100_max": 0.4623789767253533, + "nauc_recall_at_100_std": 0.6192812082541869, + "nauc_recall_at_10_diff1": 0.41445167336327915, + "nauc_recall_at_10_max": 0.4014545477647777, + "nauc_recall_at_10_std": 0.39772790563947014, + "nauc_recall_at_1_diff1": 0.6337414799404985, + "nauc_recall_at_1_max": 0.5114562833991539, + "nauc_recall_at_1_std": 0.36764760561164883, + "nauc_recall_at_20_diff1": 0.37977308593119635, + "nauc_recall_at_20_max": 0.38489121713355706, + "nauc_recall_at_20_std": 0.40658924970974497, + "nauc_recall_at_3_diff1": 0.4767990083649137, + "nauc_recall_at_3_max": 0.4456114997013859, + "nauc_recall_at_3_std": 0.38757663075347437, + "nauc_recall_at_5_diff1": 0.4368711432697605, + "nauc_recall_at_5_max": 0.41052511580267553, + "nauc_recall_at_5_std": 0.3946283429118848, + "ndcg_at_1": 0.28224, + "ndcg_at_10": 0.40522, + "ndcg_at_100": 0.46119, + "ndcg_at_1000": 0.48372, + "ndcg_at_20": 0.42513, + "ndcg_at_3": 0.35196, + "ndcg_at_5": 0.37634, + "precision_at_1": 0.28224, + "precision_at_10": 0.05486, + "precision_at_100": 0.00822, + "precision_at_1000": 0.001, + "precision_at_20": 0.03134, + "precision_at_3": 0.13354, + "precision_at_5": 0.09197, + "recall_at_1": 0.28224, + "recall_at_10": 0.54863, + "recall_at_100": 0.82241, + "recall_at_1000": 1.0, + "recall_at_20": 0.62685, + "recall_at_3": 0.40063, + "recall_at_5": 0.45983 + }, + { + "cv_recall_at_1": 0.473, + "cv_recall_at_10": 0.774, + "cv_recall_at_100": 0.971, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.844, + "cv_recall_at_3": 0.617, + "cv_recall_at_5": 0.691, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6154, + "map_at_1": 0.473, + "map_at_10": 0.56563, + "map_at_100": 0.57394, + "map_at_1000": 0.57412, + "map_at_20": 0.57045, + "map_at_3": 0.53783, + "map_at_5": 0.55428, + "mrr_at_1": 0.473, + "mrr_at_10": 0.565631746031746, + "mrr_at_100": 0.5739374783660317, + "mrr_at_1000": 0.5741198530644942, + "mrr_at_20": 0.5704528635923758, + "mrr_at_3": 0.537833333333333, + "mrr_at_5": 0.5542833333333335, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4605267394314036, + "nauc_cv_recall_at_100_max": 0.7279854470523838, + "nauc_cv_recall_at_100_std": 0.7968543739334791, + "nauc_cv_recall_at_10_diff1": 0.6490325040510915, + "nauc_cv_recall_at_10_max": 0.27027474796991563, + "nauc_cv_recall_at_10_std": -0.15112953960537553, + "nauc_cv_recall_at_1_diff1": 0.7308358885481937, + "nauc_cv_recall_at_1_max": 0.35826378265076236, + "nauc_cv_recall_at_1_std": 0.01624337877061942, + "nauc_cv_recall_at_20_diff1": 0.6136489236102389, + "nauc_cv_recall_at_20_max": 0.3500433319098693, + "nauc_cv_recall_at_20_std": -0.07247418925368479, + "nauc_cv_recall_at_3_diff1": 0.6547708840444232, + "nauc_cv_recall_at_3_max": 0.313124884298393, + "nauc_cv_recall_at_3_std": -0.06447773250859423, + "nauc_cv_recall_at_5_diff1": 0.6617254318351731, + "nauc_cv_recall_at_5_max": 0.27884895799272874, + "nauc_cv_recall_at_5_std": -0.11496810614670311, + "nauc_map_at_1000_diff1": 0.6991631273972542, + "nauc_map_at_1000_max": 0.33871405378040526, + "nauc_map_at_1000_std": -0.021408877269458917, + "nauc_map_at_100_diff1": 0.699035903470624, + "nauc_map_at_100_max": 0.33896301834936865, + "nauc_map_at_100_std": -0.020914173868226957, + "nauc_map_at_10_diff1": 0.6991120433673029, + "nauc_map_at_10_max": 0.3347705099742023, + "nauc_map_at_10_std": -0.027856046283506398, + "nauc_map_at_1_diff1": 0.7308358885481937, + "nauc_map_at_1_max": 0.35826378265076236, + "nauc_map_at_1_std": 0.01624337877061942, + "nauc_map_at_20_diff1": 0.698558842795481, + "nauc_map_at_20_max": 0.33807285741066717, + "nauc_map_at_20_std": -0.02376670240251998, + "nauc_map_at_3_diff1": 0.6963433133190773, + "nauc_map_at_3_max": 0.3387796350169068, + "nauc_map_at_3_std": -0.01790309156726825, + "nauc_map_at_5_diff1": 0.6987923943484893, + "nauc_map_at_5_max": 0.3340024651219265, + "nauc_map_at_5_std": -0.02584665881142524, + "nauc_mrr_at_1000_diff1": 0.6991631273972542, + "nauc_mrr_at_1000_max": 0.33871405378040526, + "nauc_mrr_at_1000_std": -0.021408877269458917, + "nauc_mrr_at_100_diff1": 0.699035903470624, + "nauc_mrr_at_100_max": 0.33896301834936865, + "nauc_mrr_at_100_std": -0.020914173868226957, + "nauc_mrr_at_10_diff1": 0.6991120433673029, + "nauc_mrr_at_10_max": 0.3347705099742023, + "nauc_mrr_at_10_std": -0.027856046283506398, + "nauc_mrr_at_1_diff1": 0.7308358885481937, + "nauc_mrr_at_1_max": 0.35826378265076236, + "nauc_mrr_at_1_std": 0.01624337877061942, + "nauc_mrr_at_20_diff1": 0.698558842795481, + "nauc_mrr_at_20_max": 0.33807285741066717, + "nauc_mrr_at_20_std": -0.02376670240251998, + "nauc_mrr_at_3_diff1": 0.6963433133190773, + "nauc_mrr_at_3_max": 0.3387796350169068, + "nauc_mrr_at_3_std": -0.01790309156726825, + "nauc_mrr_at_5_diff1": 0.6987923943484893, + "nauc_mrr_at_5_max": 0.3340024651219265, + "nauc_mrr_at_5_std": -0.02584665881142524, + "nauc_ndcg_at_1000_diff1": 0.6937504114819907, + "nauc_ndcg_at_1000_max": 0.33790132226185615, + "nauc_ndcg_at_1000_std": -0.02382794258355088, + "nauc_ndcg_at_100_diff1": 0.6902411323123153, + "nauc_ndcg_at_100_max": 0.344055739588466, + "nauc_ndcg_at_100_std": -0.011046520811068718, + "nauc_ndcg_at_10_diff1": 0.6898041695743851, + "nauc_ndcg_at_10_max": 0.32335607357229856, + "nauc_ndcg_at_10_std": -0.04982529757937129, + "nauc_ndcg_at_1_diff1": 0.7308358885481937, + "nauc_ndcg_at_1_max": 0.35826378265076236, + "nauc_ndcg_at_1_std": 0.01624337877061942, + "nauc_ndcg_at_20_diff1": 0.6870893166291495, + "nauc_ndcg_at_20_max": 0.33679776689666513, + "nauc_ndcg_at_20_std": -0.03388341318705418, + "nauc_ndcg_at_3_diff1": 0.6861737674452026, + "nauc_ndcg_at_3_max": 0.33255033904787995, + "nauc_ndcg_at_3_std": -0.029180408094922526, + "nauc_ndcg_at_5_diff1": 0.6902989330918204, + "nauc_ndcg_at_5_max": 0.3223485889205435, + "nauc_ndcg_at_5_std": -0.04490685171130491, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.4605267394313974, + "nauc_precision_at_100_max": 0.7279854470523784, + "nauc_precision_at_100_std": 0.7968543739334838, + "nauc_precision_at_10_diff1": 0.6490325040510907, + "nauc_precision_at_10_max": 0.27027474796991413, + "nauc_precision_at_10_std": -0.15112953960537717, + "nauc_precision_at_1_diff1": 0.7308358885481937, + "nauc_precision_at_1_max": 0.35826378265076236, + "nauc_precision_at_1_std": 0.01624337877061942, + "nauc_precision_at_20_diff1": 0.6136489236102396, + "nauc_precision_at_20_max": 0.3500433319098695, + "nauc_precision_at_20_std": -0.07247418925368708, + "nauc_precision_at_3_diff1": 0.6547708840444236, + "nauc_precision_at_3_max": 0.31312488429839336, + "nauc_precision_at_3_std": -0.06447773250859448, + "nauc_precision_at_5_diff1": 0.6617254318351731, + "nauc_precision_at_5_max": 0.2788489579927279, + "nauc_precision_at_5_std": -0.11496810614670348, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4605267394314036, + "nauc_recall_at_100_max": 0.7279854470523838, + "nauc_recall_at_100_std": 0.7968543739334791, + "nauc_recall_at_10_diff1": 0.6490325040510915, + "nauc_recall_at_10_max": 0.27027474796991563, + "nauc_recall_at_10_std": -0.15112953960537553, + "nauc_recall_at_1_diff1": 0.7308358885481937, + "nauc_recall_at_1_max": 0.35826378265076236, + "nauc_recall_at_1_std": 0.01624337877061942, + "nauc_recall_at_20_diff1": 0.6136489236102389, + "nauc_recall_at_20_max": 0.3500433319098693, + "nauc_recall_at_20_std": -0.07247418925368479, + "nauc_recall_at_3_diff1": 0.6547708840444232, + "nauc_recall_at_3_max": 0.313124884298393, + "nauc_recall_at_3_std": -0.06447773250859423, + "nauc_recall_at_5_diff1": 0.6617254318351731, + "nauc_recall_at_5_max": 0.27884895799272874, + "nauc_recall_at_5_std": -0.11496810614670311, + "ndcg_at_1": 0.473, + "ndcg_at_10": 0.6154, + "ndcg_at_100": 0.65714, + "ndcg_at_1000": 0.66107, + "ndcg_at_20": 0.63306, + "ndcg_at_3": 0.55822, + "ndcg_at_5": 0.5883, + "precision_at_1": 0.473, + "precision_at_10": 0.0774, + "precision_at_100": 0.00971, + "precision_at_1000": 0.001, + "precision_at_20": 0.0422, + "precision_at_3": 0.20567, + "precision_at_5": 0.1382, + "recall_at_1": 0.473, + "recall_at_10": 0.774, + "recall_at_100": 0.971, + "recall_at_1000": 1.0, + "recall_at_20": 0.844, + "recall_at_3": 0.617, + "recall_at_5": 0.691 + } + ] + }, + "task_name": "WITT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/model_meta.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/model_meta.json new file mode 100644 index 0000000000..e29c0dd4fe --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/model_meta.json @@ -0,0 +1 @@ +{"name": "google/siglip-base-patch16-224", "revision": "7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/WITT2IRetrieval.json b/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/WITT2IRetrieval.json new file mode 100644 index 0000000000..16b2062a47 --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/WITT2IRetrieval.json @@ -0,0 +1,1936 @@ +{ + "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", + "evaluation_time": 74.72156262397766, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.3236, + "cv_recall_at_10": 0.60337, + "cv_recall_at_100": 0.89438, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.70674, + "cv_recall_at_3": 0.44719, + "cv_recall_at_5": 0.51573, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.45325, + "map_at_1": 0.3236, + "map_at_10": 0.4064, + "map_at_100": 0.41858, + "map_at_1000": 0.41914, + "map_at_20": 0.41382, + "map_at_3": 0.37865, + "map_at_5": 0.39421, + "mrr_at_1": 0.3235955056179775, + "mrr_at_10": 0.4063991439272336, + "mrr_at_100": 0.4185789121078483, + "mrr_at_1000": 0.4191412489262205, + "mrr_at_20": 0.4138152962342439, + "mrr_at_3": 0.37865168539325816, + "mrr_at_5": 0.39421348314606697, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2642337007621988, + "nauc_cv_recall_at_100_max": 0.4364784844059463, + "nauc_cv_recall_at_100_std": 0.6209214395742897, + "nauc_cv_recall_at_10_diff1": 0.39870764492443644, + "nauc_cv_recall_at_10_max": 0.3090094125185994, + "nauc_cv_recall_at_10_std": 0.28345755308066695, + "nauc_cv_recall_at_1_diff1": 0.5662128824302667, + "nauc_cv_recall_at_1_max": 0.3210718551732005, + "nauc_cv_recall_at_1_std": 0.1913141735544631, + "nauc_cv_recall_at_20_diff1": 0.35494897443461426, + "nauc_cv_recall_at_20_max": 0.31353234713474976, + "nauc_cv_recall_at_20_std": 0.3484756279950574, + "nauc_cv_recall_at_3_diff1": 0.47458937595728956, + "nauc_cv_recall_at_3_max": 0.3071588373467233, + "nauc_cv_recall_at_3_std": 0.2534976907450473, + "nauc_cv_recall_at_5_diff1": 0.4608897957568923, + "nauc_cv_recall_at_5_max": 0.31221669884830827, + "nauc_cv_recall_at_5_std": 0.2613788349789335, + "nauc_map_at_1000_diff1": 0.5135593752434391, + "nauc_map_at_1000_max": 0.31686199255548764, + "nauc_map_at_1000_std": 0.22591048552758194, + "nauc_map_at_100_diff1": 0.5130594196105744, + "nauc_map_at_100_max": 0.31696126710125977, + "nauc_map_at_100_std": 0.2264918348015778, + "nauc_map_at_10_diff1": 0.5131969521348017, + "nauc_map_at_10_max": 0.31610024081986743, + "nauc_map_at_10_std": 0.22310702565265525, + "nauc_map_at_1_diff1": 0.5662128824302667, + "nauc_map_at_1_max": 0.3210718551732005, + "nauc_map_at_1_std": 0.1913141735544631, + "nauc_map_at_20_diff1": 0.5124598116914615, + "nauc_map_at_20_max": 0.3160581390682378, + "nauc_map_at_20_std": 0.22532241691526742, + "nauc_map_at_3_diff1": 0.5217299399887494, + "nauc_map_at_3_max": 0.31528970701609593, + "nauc_map_at_3_std": 0.21957741727895302, + "nauc_map_at_5_diff1": 0.5196735760377273, + "nauc_map_at_5_max": 0.31672138456625565, + "nauc_map_at_5_std": 0.22093720684062693, + "nauc_mrr_at_1000_diff1": 0.5135593752434391, + "nauc_mrr_at_1000_max": 0.31686199255548764, + "nauc_mrr_at_1000_std": 0.22591048552758194, + "nauc_mrr_at_100_diff1": 0.5130594196105744, + "nauc_mrr_at_100_max": 0.31696126710125977, + "nauc_mrr_at_100_std": 0.2264918348015778, + "nauc_mrr_at_10_diff1": 0.5131969521348017, + "nauc_mrr_at_10_max": 0.31610024081986743, + "nauc_mrr_at_10_std": 0.22310702565265525, + "nauc_mrr_at_1_diff1": 0.5662128824302667, + "nauc_mrr_at_1_max": 0.3210718551732005, + "nauc_mrr_at_1_std": 0.1913141735544631, + "nauc_mrr_at_20_diff1": 0.5124598116914615, + "nauc_mrr_at_20_max": 0.3160581390682378, + "nauc_mrr_at_20_std": 0.22532241691526742, + "nauc_mrr_at_3_diff1": 0.5217299399887494, + "nauc_mrr_at_3_max": 0.31528970701609593, + "nauc_mrr_at_3_std": 0.21957741727895302, + "nauc_mrr_at_5_diff1": 0.5196735760377273, + "nauc_mrr_at_5_max": 0.31672138456625565, + "nauc_mrr_at_5_std": 0.22093720684062693, + "nauc_ndcg_at_1000_diff1": 0.4999680546074473, + "nauc_ndcg_at_1000_max": 0.3181423090222327, + "nauc_ndcg_at_1000_std": 0.23901249891109946, + "nauc_ndcg_at_100_diff1": 0.48863649358451144, + "nauc_ndcg_at_100_max": 0.32303604460144, + "nauc_ndcg_at_100_std": 0.2561979096000777, + "nauc_ndcg_at_10_diff1": 0.48800886375175123, + "nauc_ndcg_at_10_max": 0.314397962869589, + "nauc_ndcg_at_10_std": 0.23675231404214014, + "nauc_ndcg_at_1_diff1": 0.5662128824302667, + "nauc_ndcg_at_1_max": 0.3210718551732005, + "nauc_ndcg_at_1_std": 0.1913141735544631, + "nauc_ndcg_at_20_diff1": 0.48361593946256537, + "nauc_ndcg_at_20_max": 0.31508639375442066, + "nauc_ndcg_at_20_std": 0.24696843482835842, + "nauc_ndcg_at_3_diff1": 0.5093997655368436, + "nauc_ndcg_at_3_max": 0.3132071543562908, + "nauc_ndcg_at_3_std": 0.2283621617892596, + "nauc_ndcg_at_5_diff1": 0.5049077426042756, + "nauc_ndcg_at_5_max": 0.3155518597169984, + "nauc_ndcg_at_5_std": 0.23107575301440864, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.26423370076219915, + "nauc_precision_at_100_max": 0.4364784844059457, + "nauc_precision_at_100_std": 0.6209214395742907, + "nauc_precision_at_10_diff1": 0.39870764492443644, + "nauc_precision_at_10_max": 0.30900941251859915, + "nauc_precision_at_10_std": 0.28345755308066645, + "nauc_precision_at_1_diff1": 0.5662128824302667, + "nauc_precision_at_1_max": 0.3210718551732005, + "nauc_precision_at_1_std": 0.1913141735544631, + "nauc_precision_at_20_diff1": 0.3549489744346135, + "nauc_precision_at_20_max": 0.31353234713474953, + "nauc_precision_at_20_std": 0.34847562799505694, + "nauc_precision_at_3_diff1": 0.4745893759572897, + "nauc_precision_at_3_max": 0.3071588373467233, + "nauc_precision_at_3_std": 0.2534976907450468, + "nauc_precision_at_5_diff1": 0.4608897957568923, + "nauc_precision_at_5_max": 0.3122166988483081, + "nauc_precision_at_5_std": 0.2613788349789337, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2642337007621988, + "nauc_recall_at_100_max": 0.4364784844059463, + "nauc_recall_at_100_std": 0.6209214395742897, + "nauc_recall_at_10_diff1": 0.39870764492443644, + "nauc_recall_at_10_max": 0.3090094125185994, + "nauc_recall_at_10_std": 0.28345755308066695, + "nauc_recall_at_1_diff1": 0.5662128824302667, + "nauc_recall_at_1_max": 0.3210718551732005, + "nauc_recall_at_1_std": 0.1913141735544631, + "nauc_recall_at_20_diff1": 0.35494897443461426, + "nauc_recall_at_20_max": 0.31353234713474976, + "nauc_recall_at_20_std": 0.3484756279950574, + "nauc_recall_at_3_diff1": 0.47458937595728956, + "nauc_recall_at_3_max": 0.3071588373467233, + "nauc_recall_at_3_std": 0.2534976907450473, + "nauc_recall_at_5_diff1": 0.4608897957568923, + "nauc_recall_at_5_max": 0.31221669884830827, + "nauc_recall_at_5_std": 0.2613788349789335, + "ndcg_at_1": 0.3236, + "ndcg_at_10": 0.45325, + "ndcg_at_100": 0.5144, + "ndcg_at_1000": 0.52816, + "ndcg_at_20": 0.47968, + "ndcg_at_3": 0.39628, + "ndcg_at_5": 0.42442, + "precision_at_1": 0.3236, + "precision_at_10": 0.06034, + "precision_at_100": 0.00894, + "precision_at_1000": 0.001, + "precision_at_20": 0.03534, + "precision_at_3": 0.14906, + "precision_at_5": 0.10315, + "recall_at_1": 0.3236, + "recall_at_10": 0.60337, + "recall_at_100": 0.89438, + "recall_at_1000": 1.0, + "recall_at_20": 0.70674, + "recall_at_3": 0.44719, + "recall_at_5": 0.51573 + }, + { + "cv_recall_at_1": 0.22326, + "cv_recall_at_10": 0.48837, + "cv_recall_at_100": 0.82442, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.57558, + "cv_recall_at_3": 0.34302, + "cv_recall_at_5": 0.40698, + "hf_subset": "bg", + "languages": [ + "bul-Cyrl" + ], + "main_score": 0.34483, + "map_at_1": 0.22326, + "map_at_10": 0.30029, + "map_at_100": 0.31203, + "map_at_1000": 0.31288, + "map_at_20": 0.30642, + "map_at_3": 0.27539, + "map_at_5": 0.28986, + "mrr_at_1": 0.22325581395348837, + "mrr_at_10": 0.3002911590992984, + "mrr_at_100": 0.3120329661071744, + "mrr_at_1000": 0.3128790149867266, + "mrr_at_20": 0.3064209778179915, + "mrr_at_3": 0.27538759689922493, + "mrr_at_5": 0.28986434108527126, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.35012547421390255, + "nauc_cv_recall_at_100_max": 0.45906854233689964, + "nauc_cv_recall_at_100_std": 0.5046356661272047, + "nauc_cv_recall_at_10_diff1": 0.4351618677282965, + "nauc_cv_recall_at_10_max": 0.30553498132888923, + "nauc_cv_recall_at_10_std": 0.15531432274789383, + "nauc_cv_recall_at_1_diff1": 0.58412712095743, + "nauc_cv_recall_at_1_max": 0.33950763616510776, + "nauc_cv_recall_at_1_std": 0.12532505782796716, + "nauc_cv_recall_at_20_diff1": 0.44304747512907705, + "nauc_cv_recall_at_20_max": 0.3206900894093943, + "nauc_cv_recall_at_20_std": 0.17718003609956795, + "nauc_cv_recall_at_3_diff1": 0.5124164506626259, + "nauc_cv_recall_at_3_max": 0.33443732549821936, + "nauc_cv_recall_at_3_std": 0.12647658311091964, + "nauc_cv_recall_at_5_diff1": 0.48301048809398744, + "nauc_cv_recall_at_5_max": 0.30559815145467906, + "nauc_cv_recall_at_5_std": 0.1304319774333429, + "nauc_map_at_1000_diff1": 0.5349942734009903, + "nauc_map_at_1000_max": 0.3276032432755494, + "nauc_map_at_1000_std": 0.13407559730775565, + "nauc_map_at_100_diff1": 0.5346369207616264, + "nauc_map_at_100_max": 0.3276991776209837, + "nauc_map_at_100_std": 0.13456351565102595, + "nauc_map_at_10_diff1": 0.5338501804482444, + "nauc_map_at_10_max": 0.3242258806026739, + "nauc_map_at_10_std": 0.12892421280373964, + "nauc_map_at_1_diff1": 0.58412712095743, + "nauc_map_at_1_max": 0.33950763616510776, + "nauc_map_at_1_std": 0.12532505782796716, + "nauc_map_at_20_diff1": 0.5349091508590145, + "nauc_map_at_20_max": 0.32576659257476576, + "nauc_map_at_20_std": 0.13063204694092, + "nauc_map_at_3_diff1": 0.5481409986549067, + "nauc_map_at_3_max": 0.33003382575724993, + "nauc_map_at_3_std": 0.12367006552300888, + "nauc_map_at_5_diff1": 0.5405251606031292, + "nauc_map_at_5_max": 0.3237417588873691, + "nauc_map_at_5_std": 0.12534857747574496, + "nauc_mrr_at_1000_diff1": 0.5349940938660724, + "nauc_mrr_at_1000_max": 0.3276030551460708, + "nauc_mrr_at_1000_std": 0.13407525677728963, + "nauc_mrr_at_100_diff1": 0.5346369207616264, + "nauc_mrr_at_100_max": 0.3276991776209837, + "nauc_mrr_at_100_std": 0.13456351565102595, + "nauc_mrr_at_10_diff1": 0.5338501804482444, + "nauc_mrr_at_10_max": 0.3242258806026739, + "nauc_mrr_at_10_std": 0.12892421280373964, + "nauc_mrr_at_1_diff1": 0.58412712095743, + "nauc_mrr_at_1_max": 0.33950763616510776, + "nauc_mrr_at_1_std": 0.12532505782796716, + "nauc_mrr_at_20_diff1": 0.5349091508590145, + "nauc_mrr_at_20_max": 0.32576659257476576, + "nauc_mrr_at_20_std": 0.13063204694092, + "nauc_mrr_at_3_diff1": 0.5481409986549067, + "nauc_mrr_at_3_max": 0.33003382575724993, + "nauc_mrr_at_3_std": 0.12367006552300888, + "nauc_mrr_at_5_diff1": 0.5405251606031292, + "nauc_mrr_at_5_max": 0.3237417588873691, + "nauc_mrr_at_5_std": 0.12534857747574496, + "nauc_ndcg_at_1000_diff1": 0.5211836969162896, + "nauc_ndcg_at_1000_max": 0.3309939382643004, + "nauc_ndcg_at_1000_std": 0.14823376349662354, + "nauc_ndcg_at_100_diff1": 0.5095398772996848, + "nauc_ndcg_at_100_max": 0.33878237754346036, + "nauc_ndcg_at_100_std": 0.17106414103216427, + "nauc_ndcg_at_10_diff1": 0.5091209247099993, + "nauc_ndcg_at_10_max": 0.31955058250055207, + "nauc_ndcg_at_10_std": 0.1351478389842136, + "nauc_ndcg_at_1_diff1": 0.58412712095743, + "nauc_ndcg_at_1_max": 0.33950763616510776, + "nauc_ndcg_at_1_std": 0.12532505782796716, + "nauc_ndcg_at_20_diff1": 0.5126350268619967, + "nauc_ndcg_at_20_max": 0.3241335197093869, + "nauc_ndcg_at_20_std": 0.14063500960677952, + "nauc_ndcg_at_3_diff1": 0.5383501967990882, + "nauc_ndcg_at_3_max": 0.330941174321705, + "nauc_ndcg_at_3_std": 0.12434272552034373, + "nauc_ndcg_at_5_diff1": 0.5250679923659893, + "nauc_ndcg_at_5_max": 0.31902551025939385, + "nauc_ndcg_at_5_std": 0.12668388842368603, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3501254742139034, + "nauc_precision_at_100_max": 0.4590685423369001, + "nauc_precision_at_100_std": 0.5046356661272042, + "nauc_precision_at_10_diff1": 0.4351618677282968, + "nauc_precision_at_10_max": 0.3055349813288893, + "nauc_precision_at_10_std": 0.1553143227478939, + "nauc_precision_at_1_diff1": 0.58412712095743, + "nauc_precision_at_1_max": 0.33950763616510776, + "nauc_precision_at_1_std": 0.12532505782796716, + "nauc_precision_at_20_diff1": 0.4430474751290764, + "nauc_precision_at_20_max": 0.32069008940939414, + "nauc_precision_at_20_std": 0.17718003609956776, + "nauc_precision_at_3_diff1": 0.5124164506626258, + "nauc_precision_at_3_max": 0.3344373254982189, + "nauc_precision_at_3_std": 0.12647658311091964, + "nauc_precision_at_5_diff1": 0.48301048809398744, + "nauc_precision_at_5_max": 0.30559815145467917, + "nauc_precision_at_5_std": 0.130431977433343, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.35012547421390255, + "nauc_recall_at_100_max": 0.45906854233689964, + "nauc_recall_at_100_std": 0.5046356661272047, + "nauc_recall_at_10_diff1": 0.4351618677282965, + "nauc_recall_at_10_max": 0.30553498132888923, + "nauc_recall_at_10_std": 0.15531432274789383, + "nauc_recall_at_1_diff1": 0.58412712095743, + "nauc_recall_at_1_max": 0.33950763616510776, + "nauc_recall_at_1_std": 0.12532505782796716, + "nauc_recall_at_20_diff1": 0.44304747512907705, + "nauc_recall_at_20_max": 0.3206900894093943, + "nauc_recall_at_20_std": 0.17718003609956795, + "nauc_recall_at_3_diff1": 0.5124164506626259, + "nauc_recall_at_3_max": 0.33443732549821936, + "nauc_recall_at_3_std": 0.12647658311091964, + "nauc_recall_at_5_diff1": 0.48301048809398744, + "nauc_recall_at_5_max": 0.30559815145467906, + "nauc_recall_at_5_std": 0.1304319774333429, + "ndcg_at_1": 0.22326, + "ndcg_at_10": 0.34483, + "ndcg_at_100": 0.41164, + "ndcg_at_1000": 0.43399, + "ndcg_at_20": 0.36698, + "ndcg_at_3": 0.29273, + "ndcg_at_5": 0.31895, + "precision_at_1": 0.22326, + "precision_at_10": 0.04884, + "precision_at_100": 0.00824, + "precision_at_1000": 0.001, + "precision_at_20": 0.02878, + "precision_at_3": 0.11434, + "precision_at_5": 0.0814, + "recall_at_1": 0.22326, + "recall_at_10": 0.48837, + "recall_at_100": 0.82442, + "recall_at_1000": 1.0, + "recall_at_20": 0.57558, + "recall_at_3": 0.34302, + "recall_at_5": 0.40698 + }, + { + "cv_recall_at_1": 0.3771, + "cv_recall_at_10": 0.67228, + "cv_recall_at_100": 0.90572, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.75421, + "cv_recall_at_3": 0.52974, + "cv_recall_at_5": 0.60831, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.51928, + "map_at_1": 0.3771, + "map_at_10": 0.47085, + "map_at_100": 0.48038, + "map_at_1000": 0.48084, + "map_at_20": 0.47658, + "map_at_3": 0.44463, + "map_at_5": 0.46253, + "mrr_at_1": 0.3771043771043771, + "mrr_at_10": 0.4708504801097391, + "mrr_at_100": 0.48037636290982116, + "mrr_at_1000": 0.4808395265178713, + "mrr_at_20": 0.47657879978154394, + "mrr_at_3": 0.4446315001870555, + "mrr_at_5": 0.4625327347549565, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4906585203122402, + "nauc_cv_recall_at_100_max": 0.5964527979866758, + "nauc_cv_recall_at_100_std": 0.6167906035908517, + "nauc_cv_recall_at_10_diff1": 0.5669748910407173, + "nauc_cv_recall_at_10_max": 0.5612593077746558, + "nauc_cv_recall_at_10_std": 0.42747846311538445, + "nauc_cv_recall_at_1_diff1": 0.7206551036615471, + "nauc_cv_recall_at_1_max": 0.47420094849426403, + "nauc_cv_recall_at_1_std": 0.20649483028278479, + "nauc_cv_recall_at_20_diff1": 0.5160964579986497, + "nauc_cv_recall_at_20_max": 0.5592683856398762, + "nauc_cv_recall_at_20_std": 0.46856803784647566, + "nauc_cv_recall_at_3_diff1": 0.586423414940436, + "nauc_cv_recall_at_3_max": 0.4652708584095005, + "nauc_cv_recall_at_3_std": 0.2671140327676577, + "nauc_cv_recall_at_5_diff1": 0.5843302145213901, + "nauc_cv_recall_at_5_max": 0.5153515422537157, + "nauc_cv_recall_at_5_std": 0.3564964378343796, + "nauc_map_at_1000_diff1": 0.6639911039547133, + "nauc_map_at_1000_max": 0.4848407002702824, + "nauc_map_at_1000_std": 0.2542221685017404, + "nauc_map_at_100_diff1": 0.6637161909591038, + "nauc_map_at_100_max": 0.48488129868471175, + "nauc_map_at_100_std": 0.25458289111532056, + "nauc_map_at_10_diff1": 0.6644504995596737, + "nauc_map_at_10_max": 0.4864333166313017, + "nauc_map_at_10_std": 0.25634573215141104, + "nauc_map_at_1_diff1": 0.7206551036615471, + "nauc_map_at_1_max": 0.47420094849426403, + "nauc_map_at_1_std": 0.20649483028278479, + "nauc_map_at_20_diff1": 0.6629438663191258, + "nauc_map_at_20_max": 0.4855299221132638, + "nauc_map_at_20_std": 0.2560909055516543, + "nauc_map_at_3_diff1": 0.6636902506020562, + "nauc_map_at_3_max": 0.47129485843850816, + "nauc_map_at_3_std": 0.2326786452017618, + "nauc_map_at_5_diff1": 0.6650049442764228, + "nauc_map_at_5_max": 0.4815139197059957, + "nauc_map_at_5_std": 0.2503501372458987, + "nauc_mrr_at_1000_diff1": 0.6639911039547133, + "nauc_mrr_at_1000_max": 0.4848407002702824, + "nauc_mrr_at_1000_std": 0.2542221685017404, + "nauc_mrr_at_100_diff1": 0.6637161909591038, + "nauc_mrr_at_100_max": 0.48488129868471175, + "nauc_mrr_at_100_std": 0.25458289111532056, + "nauc_mrr_at_10_diff1": 0.6644504995596737, + "nauc_mrr_at_10_max": 0.4864333166313017, + "nauc_mrr_at_10_std": 0.25634573215141104, + "nauc_mrr_at_1_diff1": 0.7206551036615471, + "nauc_mrr_at_1_max": 0.47420094849426403, + "nauc_mrr_at_1_std": 0.20649483028278479, + "nauc_mrr_at_20_diff1": 0.6629438663191258, + "nauc_mrr_at_20_max": 0.4855299221132638, + "nauc_mrr_at_20_std": 0.2560909055516543, + "nauc_mrr_at_3_diff1": 0.6636902506020562, + "nauc_mrr_at_3_max": 0.47129485843850816, + "nauc_mrr_at_3_std": 0.2326786452017618, + "nauc_mrr_at_5_diff1": 0.6650049442764228, + "nauc_mrr_at_5_max": 0.4815139197059957, + "nauc_mrr_at_5_std": 0.2503501372458987, + "nauc_ndcg_at_1000_diff1": 0.6516030124664017, + "nauc_ndcg_at_1000_max": 0.490972539609689, + "nauc_ndcg_at_1000_std": 0.2712931231475558, + "nauc_ndcg_at_100_diff1": 0.6447102766805155, + "nauc_ndcg_at_100_max": 0.4946938959773728, + "nauc_ndcg_at_100_std": 0.2849466684001508, + "nauc_ndcg_at_10_diff1": 0.6435389736849311, + "nauc_ndcg_at_10_max": 0.5009031707887753, + "nauc_ndcg_at_10_std": 0.2905583148670199, + "nauc_ndcg_at_1_diff1": 0.7206551036615471, + "nauc_ndcg_at_1_max": 0.47420094849426403, + "nauc_ndcg_at_1_std": 0.20649483028278479, + "nauc_ndcg_at_20_diff1": 0.6374473907758845, + "nauc_ndcg_at_20_max": 0.49791612968842575, + "nauc_ndcg_at_20_std": 0.2915783578120534, + "nauc_ndcg_at_3_diff1": 0.6443682341577142, + "nauc_ndcg_at_3_max": 0.46982593986634175, + "nauc_ndcg_at_3_std": 0.24130930934625286, + "nauc_ndcg_at_5_diff1": 0.6458853061404316, + "nauc_ndcg_at_5_max": 0.4888301578562302, + "nauc_ndcg_at_5_std": 0.2742810964190069, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.49065852031224216, + "nauc_precision_at_100_max": 0.5964527979866789, + "nauc_precision_at_100_std": 0.6167906035908529, + "nauc_precision_at_10_diff1": 0.5669748910407167, + "nauc_precision_at_10_max": 0.5612593077746549, + "nauc_precision_at_10_std": 0.42747846311538346, + "nauc_precision_at_1_diff1": 0.7206551036615471, + "nauc_precision_at_1_max": 0.47420094849426403, + "nauc_precision_at_1_std": 0.20649483028278479, + "nauc_precision_at_20_diff1": 0.516096457998648, + "nauc_precision_at_20_max": 0.5592683856398757, + "nauc_precision_at_20_std": 0.4685680378464741, + "nauc_precision_at_3_diff1": 0.5864234149404364, + "nauc_precision_at_3_max": 0.4652708584095008, + "nauc_precision_at_3_std": 0.2671140327676577, + "nauc_precision_at_5_diff1": 0.5843302145213897, + "nauc_precision_at_5_max": 0.5153515422537154, + "nauc_precision_at_5_std": 0.3564964378343795, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4906585203122402, + "nauc_recall_at_100_max": 0.5964527979866758, + "nauc_recall_at_100_std": 0.6167906035908517, + "nauc_recall_at_10_diff1": 0.5669748910407173, + "nauc_recall_at_10_max": 0.5612593077746558, + "nauc_recall_at_10_std": 0.42747846311538445, + "nauc_recall_at_1_diff1": 0.7206551036615471, + "nauc_recall_at_1_max": 0.47420094849426403, + "nauc_recall_at_1_std": 0.20649483028278479, + "nauc_recall_at_20_diff1": 0.5160964579986497, + "nauc_recall_at_20_max": 0.5592683856398762, + "nauc_recall_at_20_std": 0.46856803784647566, + "nauc_recall_at_3_diff1": 0.586423414940436, + "nauc_recall_at_3_max": 0.4652708584095005, + "nauc_recall_at_3_std": 0.2671140327676577, + "nauc_recall_at_5_diff1": 0.5843302145213901, + "nauc_recall_at_5_max": 0.5153515422537157, + "nauc_recall_at_5_std": 0.3564964378343796, + "ndcg_at_1": 0.3771, + "ndcg_at_10": 0.51928, + "ndcg_at_100": 0.56799, + "ndcg_at_1000": 0.58004, + "ndcg_at_20": 0.54004, + "ndcg_at_3": 0.4665, + "ndcg_at_5": 0.49881, + "precision_at_1": 0.3771, + "precision_at_10": 0.06723, + "precision_at_100": 0.00906, + "precision_at_1000": 0.001, + "precision_at_20": 0.03771, + "precision_at_3": 0.17658, + "precision_at_5": 0.12166, + "recall_at_1": 0.3771, + "recall_at_10": 0.67228, + "recall_at_100": 0.90572, + "recall_at_1000": 1.0, + "recall_at_20": 0.75421, + "recall_at_3": 0.52974, + "recall_at_5": 0.60831 + }, + { + "cv_recall_at_1": 0.25789, + "cv_recall_at_10": 0.51404, + "cv_recall_at_100": 0.85614, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.63333, + "cv_recall_at_3": 0.35088, + "cv_recall_at_5": 0.41228, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.37035, + "map_at_1": 0.25789, + "map_at_10": 0.32633, + "map_at_100": 0.34022, + "map_at_1000": 0.34102, + "map_at_20": 0.33488, + "map_at_3": 0.29854, + "map_at_5": 0.31257, + "mrr_at_1": 0.2578947368421053, + "mrr_at_10": 0.3263297131718183, + "mrr_at_100": 0.3402207310095121, + "mrr_at_1000": 0.34102258220012444, + "mrr_at_20": 0.3348754483315343, + "mrr_at_3": 0.29853801169590644, + "mrr_at_5": 0.3125730994152046, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5166860368587083, + "nauc_cv_recall_at_100_max": 0.43867555323640267, + "nauc_cv_recall_at_100_std": 0.4181767510258299, + "nauc_cv_recall_at_10_diff1": 0.4930973392561108, + "nauc_cv_recall_at_10_max": 0.3048720618334208, + "nauc_cv_recall_at_10_std": 0.25607080770973034, + "nauc_cv_recall_at_1_diff1": 0.6535331862179902, + "nauc_cv_recall_at_1_max": 0.29091931817892586, + "nauc_cv_recall_at_1_std": 0.09693323050530796, + "nauc_cv_recall_at_20_diff1": 0.4097584319236105, + "nauc_cv_recall_at_20_max": 0.27841599057295296, + "nauc_cv_recall_at_20_std": 0.28995084547443667, + "nauc_cv_recall_at_3_diff1": 0.5381549132947977, + "nauc_cv_recall_at_3_max": 0.25885549132947977, + "nauc_cv_recall_at_3_std": 0.14639768786127155, + "nauc_cv_recall_at_5_diff1": 0.49700295961989843, + "nauc_cv_recall_at_5_max": 0.2933130733554832, + "nauc_cv_recall_at_5_std": 0.20155805277421843, + "nauc_map_at_1000_diff1": 0.5832600994027133, + "nauc_map_at_1000_max": 0.28339679288184694, + "nauc_map_at_1000_std": 0.13635175481091114, + "nauc_map_at_100_diff1": 0.5830692435995272, + "nauc_map_at_100_max": 0.283691963519985, + "nauc_map_at_100_std": 0.13685323283451764, + "nauc_map_at_10_diff1": 0.58543822530251, + "nauc_map_at_10_max": 0.28438693411051763, + "nauc_map_at_10_std": 0.1355926756343963, + "nauc_map_at_1_diff1": 0.6535331862179902, + "nauc_map_at_1_max": 0.29091931817892586, + "nauc_map_at_1_std": 0.09693323050530796, + "nauc_map_at_20_diff1": 0.5811818293948084, + "nauc_map_at_20_max": 0.2824903844060562, + "nauc_map_at_20_std": 0.1366295938070716, + "nauc_map_at_3_diff1": 0.5954714526372237, + "nauc_map_at_3_max": 0.27381059912257893, + "nauc_map_at_3_std": 0.1155176122101709, + "nauc_map_at_5_diff1": 0.5847127401758295, + "nauc_map_at_5_max": 0.28204732400406096, + "nauc_map_at_5_std": 0.12870599169498348, + "nauc_mrr_at_1000_diff1": 0.5832600994027133, + "nauc_mrr_at_1000_max": 0.28339679288184694, + "nauc_mrr_at_1000_std": 0.13635175481091114, + "nauc_mrr_at_100_diff1": 0.5830692435995272, + "nauc_mrr_at_100_max": 0.283691963519985, + "nauc_mrr_at_100_std": 0.13685323283451764, + "nauc_mrr_at_10_diff1": 0.58543822530251, + "nauc_mrr_at_10_max": 0.28438693411051763, + "nauc_mrr_at_10_std": 0.1355926756343963, + "nauc_mrr_at_1_diff1": 0.6535331862179902, + "nauc_mrr_at_1_max": 0.29091931817892586, + "nauc_mrr_at_1_std": 0.09693323050530796, + "nauc_mrr_at_20_diff1": 0.5811818293948084, + "nauc_mrr_at_20_max": 0.2824903844060562, + "nauc_mrr_at_20_std": 0.1366295938070716, + "nauc_mrr_at_3_diff1": 0.5954714526372237, + "nauc_mrr_at_3_max": 0.27381059912257893, + "nauc_mrr_at_3_std": 0.1155176122101709, + "nauc_mrr_at_5_diff1": 0.5847127401758295, + "nauc_mrr_at_5_max": 0.28204732400406096, + "nauc_mrr_at_5_std": 0.12870599169498348, + "nauc_ndcg_at_1000_diff1": 0.5682435897735153, + "nauc_ndcg_at_1000_max": 0.28625538891286073, + "nauc_ndcg_at_1000_std": 0.15434381014343437, + "nauc_ndcg_at_100_diff1": 0.5648418395834174, + "nauc_ndcg_at_100_max": 0.29511407669511164, + "nauc_ndcg_at_100_std": 0.16950589407124725, + "nauc_ndcg_at_10_diff1": 0.5618969295076067, + "nauc_ndcg_at_10_max": 0.28892499642810837, + "nauc_ndcg_at_10_std": 0.16425087953198123, + "nauc_ndcg_at_1_diff1": 0.6535331862179902, + "nauc_ndcg_at_1_max": 0.29091931817892586, + "nauc_ndcg_at_1_std": 0.09693323050530796, + "nauc_ndcg_at_20_diff1": 0.5453651886355529, + "nauc_ndcg_at_20_max": 0.2821736243022714, + "nauc_ndcg_at_20_std": 0.16872468062339616, + "nauc_ndcg_at_3_diff1": 0.5800141979555709, + "nauc_ndcg_at_3_max": 0.26973368041527257, + "nauc_ndcg_at_3_std": 0.12356161545209693, + "nauc_ndcg_at_5_diff1": 0.5615004994756322, + "nauc_ndcg_at_5_max": 0.28444838112540044, + "nauc_ndcg_at_5_std": 0.1472280972157901, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5166860368587091, + "nauc_precision_at_100_max": 0.43867555323640356, + "nauc_precision_at_100_std": 0.4181767510258304, + "nauc_precision_at_10_diff1": 0.49309733925611043, + "nauc_precision_at_10_max": 0.3048720618334204, + "nauc_precision_at_10_std": 0.2560708077097302, + "nauc_precision_at_1_diff1": 0.6535331862179902, + "nauc_precision_at_1_max": 0.29091931817892586, + "nauc_precision_at_1_std": 0.09693323050530796, + "nauc_precision_at_20_diff1": 0.40975843192361006, + "nauc_precision_at_20_max": 0.2784159905729532, + "nauc_precision_at_20_std": 0.28995084547443656, + "nauc_precision_at_3_diff1": 0.5381549132947977, + "nauc_precision_at_3_max": 0.2588554913294797, + "nauc_precision_at_3_std": 0.1463976878612719, + "nauc_precision_at_5_diff1": 0.4970029596198983, + "nauc_precision_at_5_max": 0.2933130733554832, + "nauc_precision_at_5_std": 0.20155805277421854, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5166860368587083, + "nauc_recall_at_100_max": 0.43867555323640267, + "nauc_recall_at_100_std": 0.4181767510258299, + "nauc_recall_at_10_diff1": 0.4930973392561108, + "nauc_recall_at_10_max": 0.3048720618334208, + "nauc_recall_at_10_std": 0.25607080770973034, + "nauc_recall_at_1_diff1": 0.6535331862179902, + "nauc_recall_at_1_max": 0.29091931817892586, + "nauc_recall_at_1_std": 0.09693323050530796, + "nauc_recall_at_20_diff1": 0.4097584319236105, + "nauc_recall_at_20_max": 0.27841599057295296, + "nauc_recall_at_20_std": 0.28995084547443667, + "nauc_recall_at_3_diff1": 0.5381549132947977, + "nauc_recall_at_3_max": 0.25885549132947977, + "nauc_recall_at_3_std": 0.14639768786127155, + "nauc_recall_at_5_diff1": 0.49700295961989843, + "nauc_recall_at_5_max": 0.2933130733554832, + "nauc_recall_at_5_std": 0.20155805277421843, + "ndcg_at_1": 0.25789, + "ndcg_at_10": 0.37035, + "ndcg_at_100": 0.44141, + "ndcg_at_1000": 0.46041, + "ndcg_at_20": 0.40083, + "ndcg_at_3": 0.31197, + "ndcg_at_5": 0.33726, + "precision_at_1": 0.25789, + "precision_at_10": 0.0514, + "precision_at_100": 0.00856, + "precision_at_1000": 0.001, + "precision_at_20": 0.03167, + "precision_at_3": 0.11696, + "precision_at_5": 0.08246, + "recall_at_1": 0.25789, + "recall_at_10": 0.51404, + "recall_at_100": 0.85614, + "recall_at_1000": 1.0, + "recall_at_20": 0.63333, + "recall_at_3": 0.35088, + "recall_at_5": 0.41228 + }, + { + "cv_recall_at_1": 0.15103, + "cv_recall_at_10": 0.37414, + "cv_recall_at_100": 0.70709, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.46796, + "cv_recall_at_3": 0.24142, + "cv_recall_at_5": 0.28833, + "hf_subset": "et", + "languages": [ + "est-Latn" + ], + "main_score": 0.25003, + "map_at_1": 0.15103, + "map_at_10": 0.21206, + "map_at_100": 0.22451, + "map_at_1000": 0.22589, + "map_at_20": 0.21867, + "map_at_3": 0.1905, + "map_at_5": 0.20114, + "mrr_at_1": 0.15102974828375287, + "mrr_at_10": 0.21206094947513715, + "mrr_at_100": 0.22450731989004902, + "mrr_at_1000": 0.22588700470049086, + "mrr_at_20": 0.21867350862748539, + "mrr_at_3": 0.19050343249427923, + "mrr_at_5": 0.2011441647597255, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2627987447450525, + "nauc_cv_recall_at_100_max": 0.5780401280458465, + "nauc_cv_recall_at_100_std": 0.5261055025722506, + "nauc_cv_recall_at_10_diff1": 0.37104575403153983, + "nauc_cv_recall_at_10_max": 0.3326454524789175, + "nauc_cv_recall_at_10_std": 0.2408937491984172, + "nauc_cv_recall_at_1_diff1": 0.5818077289177603, + "nauc_cv_recall_at_1_max": 0.30800372272543686, + "nauc_cv_recall_at_1_std": 0.111734743577289, + "nauc_cv_recall_at_20_diff1": 0.32652415973790355, + "nauc_cv_recall_at_20_max": 0.4521984724216397, + "nauc_cv_recall_at_20_std": 0.37226557128815174, + "nauc_cv_recall_at_3_diff1": 0.39849646522091037, + "nauc_cv_recall_at_3_max": 0.2875581846946957, + "nauc_cv_recall_at_3_std": 0.15067000226993205, + "nauc_cv_recall_at_5_diff1": 0.3596038540829789, + "nauc_cv_recall_at_5_max": 0.2881051420511456, + "nauc_cv_recall_at_5_std": 0.167710054668009, + "nauc_map_at_1000_diff1": 0.4607486246089169, + "nauc_map_at_1000_max": 0.31054228952252577, + "nauc_map_at_1000_std": 0.15867686744819928, + "nauc_map_at_100_diff1": 0.45998150020862694, + "nauc_map_at_100_max": 0.3115941847216049, + "nauc_map_at_100_std": 0.16014118304459476, + "nauc_map_at_10_diff1": 0.46615837349470063, + "nauc_map_at_10_max": 0.29888142697456227, + "nauc_map_at_10_std": 0.145690639903157, + "nauc_map_at_1_diff1": 0.5818077289177603, + "nauc_map_at_1_max": 0.30800372272543686, + "nauc_map_at_1_std": 0.111734743577289, + "nauc_map_at_20_diff1": 0.46186180237623736, + "nauc_map_at_20_max": 0.3092479663323469, + "nauc_map_at_20_std": 0.15741007065454662, + "nauc_map_at_3_diff1": 0.4800353699094137, + "nauc_map_at_3_max": 0.2915921003153219, + "nauc_map_at_3_std": 0.12788122656072387, + "nauc_map_at_5_diff1": 0.46572435008499474, + "nauc_map_at_5_max": 0.29108929759167185, + "nauc_map_at_5_std": 0.13290094268462646, + "nauc_mrr_at_1000_diff1": 0.4607486246089169, + "nauc_mrr_at_1000_max": 0.31054228952252577, + "nauc_mrr_at_1000_std": 0.15867686744819928, + "nauc_mrr_at_100_diff1": 0.45998150020862694, + "nauc_mrr_at_100_max": 0.3115941847216049, + "nauc_mrr_at_100_std": 0.16014118304459476, + "nauc_mrr_at_10_diff1": 0.46615837349470063, + "nauc_mrr_at_10_max": 0.29888142697456227, + "nauc_mrr_at_10_std": 0.145690639903157, + "nauc_mrr_at_1_diff1": 0.5818077289177603, + "nauc_mrr_at_1_max": 0.30800372272543686, + "nauc_mrr_at_1_std": 0.111734743577289, + "nauc_mrr_at_20_diff1": 0.46186180237623736, + "nauc_mrr_at_20_max": 0.3092479663323469, + "nauc_mrr_at_20_std": 0.15741007065454662, + "nauc_mrr_at_3_diff1": 0.4800353699094137, + "nauc_mrr_at_3_max": 0.2915921003153219, + "nauc_mrr_at_3_std": 0.12788122656072387, + "nauc_mrr_at_5_diff1": 0.46572435008499474, + "nauc_mrr_at_5_max": 0.29108929759167185, + "nauc_mrr_at_5_std": 0.13290094268462646, + "nauc_ndcg_at_1000_diff1": 0.4353269370557416, + "nauc_ndcg_at_1000_max": 0.33047611867355287, + "nauc_ndcg_at_1000_std": 0.19048707620953562, + "nauc_ndcg_at_100_diff1": 0.41560305720295077, + "nauc_ndcg_at_100_max": 0.35861665089692857, + "nauc_ndcg_at_100_std": 0.22866740291545476, + "nauc_ndcg_at_10_diff1": 0.43670745989749377, + "nauc_ndcg_at_10_max": 0.30736360243562977, + "nauc_ndcg_at_10_std": 0.17170928125657542, + "nauc_ndcg_at_1_diff1": 0.5818077289177603, + "nauc_ndcg_at_1_max": 0.30800372272543686, + "nauc_ndcg_at_1_std": 0.111734743577289, + "nauc_ndcg_at_20_diff1": 0.4229548229039391, + "nauc_ndcg_at_20_max": 0.3431403535337248, + "nauc_ndcg_at_20_std": 0.21132734084355814, + "nauc_ndcg_at_3_diff1": 0.4564002917837622, + "nauc_ndcg_at_3_max": 0.2901989981063535, + "nauc_ndcg_at_3_std": 0.13424490964159908, + "nauc_ndcg_at_5_diff1": 0.4342887056984211, + "nauc_ndcg_at_5_max": 0.28989155178116033, + "nauc_ndcg_at_5_std": 0.14279723581592305, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.26279874474505255, + "nauc_precision_at_100_max": 0.5780401280458474, + "nauc_precision_at_100_std": 0.5261055025722517, + "nauc_precision_at_10_diff1": 0.3710457540315399, + "nauc_precision_at_10_max": 0.3326454524789174, + "nauc_precision_at_10_std": 0.24089374919841736, + "nauc_precision_at_1_diff1": 0.5818077289177603, + "nauc_precision_at_1_max": 0.30800372272543686, + "nauc_precision_at_1_std": 0.111734743577289, + "nauc_precision_at_20_diff1": 0.32652415973790305, + "nauc_precision_at_20_max": 0.4521984724216396, + "nauc_precision_at_20_std": 0.3722655712881512, + "nauc_precision_at_3_diff1": 0.3984964652209105, + "nauc_precision_at_3_max": 0.28755818469469585, + "nauc_precision_at_3_std": 0.150670002269932, + "nauc_precision_at_5_diff1": 0.3596038540829787, + "nauc_precision_at_5_max": 0.28810514205114574, + "nauc_precision_at_5_std": 0.16771005466800906, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2627987447450525, + "nauc_recall_at_100_max": 0.5780401280458465, + "nauc_recall_at_100_std": 0.5261055025722506, + "nauc_recall_at_10_diff1": 0.37104575403153983, + "nauc_recall_at_10_max": 0.3326454524789175, + "nauc_recall_at_10_std": 0.2408937491984172, + "nauc_recall_at_1_diff1": 0.5818077289177603, + "nauc_recall_at_1_max": 0.30800372272543686, + "nauc_recall_at_1_std": 0.111734743577289, + "nauc_recall_at_20_diff1": 0.32652415973790355, + "nauc_recall_at_20_max": 0.4521984724216397, + "nauc_recall_at_20_std": 0.37226557128815174, + "nauc_recall_at_3_diff1": 0.39849646522091037, + "nauc_recall_at_3_max": 0.2875581846946957, + "nauc_recall_at_3_std": 0.15067000226993205, + "nauc_recall_at_5_diff1": 0.3596038540829789, + "nauc_recall_at_5_max": 0.2881051420511456, + "nauc_recall_at_5_std": 0.167710054668009, + "ndcg_at_1": 0.15103, + "ndcg_at_10": 0.25003, + "ndcg_at_100": 0.31768, + "ndcg_at_1000": 0.35499, + "ndcg_at_20": 0.27387, + "ndcg_at_3": 0.20356, + "ndcg_at_5": 0.22282, + "precision_at_1": 0.15103, + "precision_at_10": 0.03741, + "precision_at_100": 0.00707, + "precision_at_1000": 0.001, + "precision_at_20": 0.0234, + "precision_at_3": 0.08047, + "precision_at_5": 0.05767, + "recall_at_1": 0.15103, + "recall_at_10": 0.37414, + "recall_at_100": 0.70709, + "recall_at_1000": 1.0, + "recall_at_20": 0.46796, + "recall_at_3": 0.24142, + "recall_at_5": 0.28833 + }, + { + "cv_recall_at_1": 0.30966, + "cv_recall_at_10": 0.60155, + "cv_recall_at_100": 0.84573, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.66371, + "cv_recall_at_3": 0.45616, + "cv_recall_at_5": 0.52164, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.4485, + "map_at_1": 0.30966, + "map_at_10": 0.4004, + "map_at_100": 0.40957, + "map_at_1000": 0.4103, + "map_at_20": 0.40488, + "map_at_3": 0.37477, + "map_at_5": 0.38981, + "mrr_at_1": 0.30965593784683687, + "mrr_at_10": 0.40039902753554235, + "mrr_at_100": 0.40957344046741884, + "mrr_at_1000": 0.4102965595323894, + "mrr_at_20": 0.4048839867430234, + "mrr_at_3": 0.37476877543470216, + "mrr_at_5": 0.389807621161672, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.413844347837448, + "nauc_cv_recall_at_100_max": 0.6591032685370454, + "nauc_cv_recall_at_100_std": 0.6882492080286065, + "nauc_cv_recall_at_10_diff1": 0.5166753772605301, + "nauc_cv_recall_at_10_max": 0.5890906125901273, + "nauc_cv_recall_at_10_std": 0.5228958235520792, + "nauc_cv_recall_at_1_diff1": 0.7003540691175822, + "nauc_cv_recall_at_1_max": 0.5562337135287849, + "nauc_cv_recall_at_1_std": 0.3222697658148194, + "nauc_cv_recall_at_20_diff1": 0.49453686872493546, + "nauc_cv_recall_at_20_max": 0.570490024802668, + "nauc_cv_recall_at_20_std": 0.5310876413062724, + "nauc_cv_recall_at_3_diff1": 0.5822939098091017, + "nauc_cv_recall_at_3_max": 0.5587282972262209, + "nauc_cv_recall_at_3_std": 0.42902959999002005, + "nauc_cv_recall_at_5_diff1": 0.551476530777103, + "nauc_cv_recall_at_5_max": 0.5698560042206824, + "nauc_cv_recall_at_5_std": 0.489924186816204, + "nauc_map_at_1000_diff1": 0.6385712536554932, + "nauc_map_at_1000_max": 0.5666453665562166, + "nauc_map_at_1000_std": 0.39054233624224277, + "nauc_map_at_100_diff1": 0.6380972125992274, + "nauc_map_at_100_max": 0.5668257371445421, + "nauc_map_at_100_std": 0.39107637572379544, + "nauc_map_at_10_diff1": 0.6382357516010204, + "nauc_map_at_10_max": 0.566426041023328, + "nauc_map_at_10_std": 0.3907557479715132, + "nauc_map_at_1_diff1": 0.7003540691175822, + "nauc_map_at_1_max": 0.5562337135287849, + "nauc_map_at_1_std": 0.3222697658148194, + "nauc_map_at_20_diff1": 0.6380225873024882, + "nauc_map_at_20_max": 0.5653351730614248, + "nauc_map_at_20_std": 0.390364721772839, + "nauc_map_at_3_diff1": 0.6481258672212916, + "nauc_map_at_3_max": 0.5622091310080056, + "nauc_map_at_3_std": 0.3737845333707094, + "nauc_map_at_5_diff1": 0.6421070955397341, + "nauc_map_at_5_max": 0.5638541197609342, + "nauc_map_at_5_std": 0.38713354286032103, + "nauc_mrr_at_1000_diff1": 0.6385712599372024, + "nauc_mrr_at_1000_max": 0.5666453342564297, + "nauc_mrr_at_1000_std": 0.3905423005585837, + "nauc_mrr_at_100_diff1": 0.6380972125992274, + "nauc_mrr_at_100_max": 0.5668257371445421, + "nauc_mrr_at_100_std": 0.39107637572379544, + "nauc_mrr_at_10_diff1": 0.6382357516010204, + "nauc_mrr_at_10_max": 0.566426041023328, + "nauc_mrr_at_10_std": 0.3907557479715132, + "nauc_mrr_at_1_diff1": 0.7003540691175822, + "nauc_mrr_at_1_max": 0.5562337135287849, + "nauc_mrr_at_1_std": 0.3222697658148194, + "nauc_mrr_at_20_diff1": 0.6380225873024882, + "nauc_mrr_at_20_max": 0.5653351730614248, + "nauc_mrr_at_20_std": 0.390364721772839, + "nauc_mrr_at_3_diff1": 0.6481258672212916, + "nauc_mrr_at_3_max": 0.5622091310080056, + "nauc_mrr_at_3_std": 0.3737845333707094, + "nauc_mrr_at_5_diff1": 0.6421070955397341, + "nauc_mrr_at_5_max": 0.5638541197609342, + "nauc_mrr_at_5_std": 0.38713354286032103, + "nauc_ndcg_at_1000_diff1": 0.6221171524663395, + "nauc_ndcg_at_1000_max": 0.5704914017884861, + "nauc_ndcg_at_1000_std": 0.40945199390172643, + "nauc_ndcg_at_100_diff1": 0.6093090345917943, + "nauc_ndcg_at_100_max": 0.5758410316092789, + "nauc_ndcg_at_100_std": 0.4260576549155124, + "nauc_ndcg_at_10_diff1": 0.6104476699376449, + "nauc_ndcg_at_10_max": 0.5712565523303276, + "nauc_ndcg_at_10_std": 0.4209712430612556, + "nauc_ndcg_at_1_diff1": 0.7003540691175822, + "nauc_ndcg_at_1_max": 0.5562337135287849, + "nauc_ndcg_at_1_std": 0.3222697658148194, + "nauc_ndcg_at_20_diff1": 0.608414917728419, + "nauc_ndcg_at_20_max": 0.5669549851368711, + "nauc_ndcg_at_20_std": 0.42011850937905115, + "nauc_ndcg_at_3_diff1": 0.6311285156481615, + "nauc_ndcg_at_3_max": 0.561514062757331, + "nauc_ndcg_at_3_std": 0.3882275785314245, + "nauc_ndcg_at_5_diff1": 0.6196780560521288, + "nauc_ndcg_at_5_max": 0.5652964042526559, + "nauc_ndcg_at_5_std": 0.4124392955924702, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.4138443478374479, + "nauc_precision_at_100_max": 0.6591032685370464, + "nauc_precision_at_100_std": 0.6882492080286061, + "nauc_precision_at_10_diff1": 0.5166753772605301, + "nauc_precision_at_10_max": 0.5890906125901274, + "nauc_precision_at_10_std": 0.5228958235520789, + "nauc_precision_at_1_diff1": 0.7003540691175822, + "nauc_precision_at_1_max": 0.5562337135287849, + "nauc_precision_at_1_std": 0.3222697658148194, + "nauc_precision_at_20_diff1": 0.49453686872493513, + "nauc_precision_at_20_max": 0.5704900248026669, + "nauc_precision_at_20_std": 0.5310876413062724, + "nauc_precision_at_3_diff1": 0.5822939098091017, + "nauc_precision_at_3_max": 0.5587282972262201, + "nauc_precision_at_3_std": 0.42902959999001977, + "nauc_precision_at_5_diff1": 0.5514765307771033, + "nauc_precision_at_5_max": 0.5698560042206825, + "nauc_precision_at_5_std": 0.48992418681620375, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.413844347837448, + "nauc_recall_at_100_max": 0.6591032685370454, + "nauc_recall_at_100_std": 0.6882492080286065, + "nauc_recall_at_10_diff1": 0.5166753772605301, + "nauc_recall_at_10_max": 0.5890906125901273, + "nauc_recall_at_10_std": 0.5228958235520792, + "nauc_recall_at_1_diff1": 0.7003540691175822, + "nauc_recall_at_1_max": 0.5562337135287849, + "nauc_recall_at_1_std": 0.3222697658148194, + "nauc_recall_at_20_diff1": 0.49453686872493546, + "nauc_recall_at_20_max": 0.570490024802668, + "nauc_recall_at_20_std": 0.5310876413062724, + "nauc_recall_at_3_diff1": 0.5822939098091017, + "nauc_recall_at_3_max": 0.5587282972262209, + "nauc_recall_at_3_std": 0.42902959999002005, + "nauc_recall_at_5_diff1": 0.551476530777103, + "nauc_recall_at_5_max": 0.5698560042206824, + "nauc_recall_at_5_std": 0.489924186816204, + "ndcg_at_1": 0.30966, + "ndcg_at_10": 0.4485, + "ndcg_at_100": 0.49826, + "ndcg_at_1000": 0.51785, + "ndcg_at_20": 0.46443, + "ndcg_at_3": 0.3957, + "ndcg_at_5": 0.42273, + "precision_at_1": 0.30966, + "precision_at_10": 0.06016, + "precision_at_100": 0.00846, + "precision_at_1000": 0.001, + "precision_at_20": 0.03319, + "precision_at_3": 0.15205, + "precision_at_5": 0.10433, + "recall_at_1": 0.30966, + "recall_at_10": 0.60155, + "recall_at_100": 0.84573, + "recall_at_1000": 1.0, + "recall_at_20": 0.66371, + "recall_at_3": 0.45616, + "recall_at_5": 0.52164 + }, + { + "cv_recall_at_1": 0.17401, + "cv_recall_at_10": 0.40064, + "cv_recall_at_100": 0.68636, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.47046, + "cv_recall_at_3": 0.2782, + "cv_recall_at_5": 0.32653, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.2794, + "map_at_1": 0.17401, + "map_at_10": 0.24161, + "map_at_100": 0.2515, + "map_at_1000": 0.25285, + "map_at_20": 0.24637, + "map_at_3": 0.22055, + "map_at_5": 0.23135, + "mrr_at_1": 0.17400644468313642, + "mrr_at_10": 0.2416057149676911, + "mrr_at_100": 0.25150072564404563, + "mrr_at_1000": 0.2528496021900727, + "mrr_at_20": 0.24636789319443522, + "mrr_at_3": 0.2205513784461154, + "mrr_at_5": 0.23134622269960614, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.29634773574834217, + "nauc_cv_recall_at_100_max": 0.3944551058132891, + "nauc_cv_recall_at_100_std": 0.5567710352580213, + "nauc_cv_recall_at_10_diff1": 0.36568437737806625, + "nauc_cv_recall_at_10_max": 0.3702981428273432, + "nauc_cv_recall_at_10_std": 0.38843071216997727, + "nauc_cv_recall_at_1_diff1": 0.6028695349379543, + "nauc_cv_recall_at_1_max": 0.42021900033590426, + "nauc_cv_recall_at_1_std": 0.2899746114582274, + "nauc_cv_recall_at_20_diff1": 0.3510794385312679, + "nauc_cv_recall_at_20_max": 0.3635329282691102, + "nauc_cv_recall_at_20_std": 0.42662091019234755, + "nauc_cv_recall_at_3_diff1": 0.4405320603733295, + "nauc_cv_recall_at_3_max": 0.3771266366287866, + "nauc_cv_recall_at_3_std": 0.3338238960381248, + "nauc_cv_recall_at_5_diff1": 0.41682490531795086, + "nauc_cv_recall_at_5_max": 0.3887424281690552, + "nauc_cv_recall_at_5_std": 0.3429433319882689, + "nauc_map_at_1000_diff1": 0.5025158655376364, + "nauc_map_at_1000_max": 0.3992455527715951, + "nauc_map_at_1000_std": 0.32907720916655936, + "nauc_map_at_100_diff1": 0.5017141695851265, + "nauc_map_at_100_max": 0.39906454583873385, + "nauc_map_at_100_std": 0.3295697259307407, + "nauc_map_at_10_diff1": 0.502622132869752, + "nauc_map_at_10_max": 0.39881509550973365, + "nauc_map_at_10_std": 0.3245463759758096, + "nauc_map_at_1_diff1": 0.6028695349379543, + "nauc_map_at_1_max": 0.42021900033590426, + "nauc_map_at_1_std": 0.2899746114582274, + "nauc_map_at_20_diff1": 0.5019572969432622, + "nauc_map_at_20_max": 0.39843955602599274, + "nauc_map_at_20_std": 0.3275556201384039, + "nauc_map_at_3_diff1": 0.5204204057808283, + "nauc_map_at_3_max": 0.40019855440613106, + "nauc_map_at_3_std": 0.31426720952478765, + "nauc_map_at_5_diff1": 0.5118332769762192, + "nauc_map_at_5_max": 0.4019896666115835, + "nauc_map_at_5_std": 0.31667063979916427, + "nauc_mrr_at_1000_diff1": 0.5025158655376364, + "nauc_mrr_at_1000_max": 0.3992455527715951, + "nauc_mrr_at_1000_std": 0.32907720916655936, + "nauc_mrr_at_100_diff1": 0.5017141695851265, + "nauc_mrr_at_100_max": 0.39906454583873385, + "nauc_mrr_at_100_std": 0.3295697259307407, + "nauc_mrr_at_10_diff1": 0.502622132869752, + "nauc_mrr_at_10_max": 0.39881509550973365, + "nauc_mrr_at_10_std": 0.3245463759758096, + "nauc_mrr_at_1_diff1": 0.6028695349379543, + "nauc_mrr_at_1_max": 0.42021900033590426, + "nauc_mrr_at_1_std": 0.2899746114582274, + "nauc_mrr_at_20_diff1": 0.5019572969432622, + "nauc_mrr_at_20_max": 0.39843955602599274, + "nauc_mrr_at_20_std": 0.3275556201384039, + "nauc_mrr_at_3_diff1": 0.5204204057808283, + "nauc_mrr_at_3_max": 0.40019855440613106, + "nauc_mrr_at_3_std": 0.31426720952478765, + "nauc_mrr_at_5_diff1": 0.5118332769762192, + "nauc_mrr_at_5_max": 0.4019896666115835, + "nauc_mrr_at_5_std": 0.31667063979916427, + "nauc_ndcg_at_1000_diff1": 0.47799866792663115, + "nauc_ndcg_at_1000_max": 0.39570991951231005, + "nauc_ndcg_at_1000_std": 0.3479223622205202, + "nauc_ndcg_at_100_diff1": 0.457912000481337, + "nauc_ndcg_at_100_max": 0.39500436807184836, + "nauc_ndcg_at_100_std": 0.3692954991144709, + "nauc_ndcg_at_10_diff1": 0.4643091961538564, + "nauc_ndcg_at_10_max": 0.390803121224411, + "nauc_ndcg_at_10_std": 0.34168265671702486, + "nauc_ndcg_at_1_diff1": 0.6028695349379543, + "nauc_ndcg_at_1_max": 0.42021900033590426, + "nauc_ndcg_at_1_std": 0.2899746114582274, + "nauc_ndcg_at_20_diff1": 0.46111566328912856, + "nauc_ndcg_at_20_max": 0.38916229283952436, + "nauc_ndcg_at_20_std": 0.3522899166330396, + "nauc_ndcg_at_3_diff1": 0.4978605410630954, + "nauc_ndcg_at_3_max": 0.39376004888593885, + "nauc_ndcg_at_3_std": 0.3198731441754594, + "nauc_ndcg_at_5_diff1": 0.48460521512617366, + "nauc_ndcg_at_5_max": 0.39791429841604314, + "nauc_ndcg_at_5_std": 0.3241595021167435, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.2963477357483419, + "nauc_precision_at_100_max": 0.39445510581328863, + "nauc_precision_at_100_std": 0.556771035258021, + "nauc_precision_at_10_diff1": 0.36568437737806625, + "nauc_precision_at_10_max": 0.3702981428273432, + "nauc_precision_at_10_std": 0.3884307121699771, + "nauc_precision_at_1_diff1": 0.6028695349379543, + "nauc_precision_at_1_max": 0.42021900033590426, + "nauc_precision_at_1_std": 0.2899746114582274, + "nauc_precision_at_20_diff1": 0.35107943853126766, + "nauc_precision_at_20_max": 0.3635329282691101, + "nauc_precision_at_20_std": 0.4266209101923477, + "nauc_precision_at_3_diff1": 0.4405320603733291, + "nauc_precision_at_3_max": 0.37712663662878615, + "nauc_precision_at_3_std": 0.3338238960381247, + "nauc_precision_at_5_diff1": 0.4168249053179508, + "nauc_precision_at_5_max": 0.388742428169055, + "nauc_precision_at_5_std": 0.342943331988269, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.29634773574834217, + "nauc_recall_at_100_max": 0.3944551058132891, + "nauc_recall_at_100_std": 0.5567710352580213, + "nauc_recall_at_10_diff1": 0.36568437737806625, + "nauc_recall_at_10_max": 0.3702981428273432, + "nauc_recall_at_10_std": 0.38843071216997727, + "nauc_recall_at_1_diff1": 0.6028695349379543, + "nauc_recall_at_1_max": 0.42021900033590426, + "nauc_recall_at_1_std": 0.2899746114582274, + "nauc_recall_at_20_diff1": 0.3510794385312679, + "nauc_recall_at_20_max": 0.3635329282691102, + "nauc_recall_at_20_std": 0.42662091019234755, + "nauc_recall_at_3_diff1": 0.4405320603733295, + "nauc_recall_at_3_max": 0.3771266366287866, + "nauc_recall_at_3_std": 0.3338238960381248, + "nauc_recall_at_5_diff1": 0.41682490531795086, + "nauc_recall_at_5_max": 0.3887424281690552, + "nauc_recall_at_5_std": 0.3429433319882689, + "ndcg_at_1": 0.17401, + "ndcg_at_10": 0.2794, + "ndcg_at_100": 0.33618, + "ndcg_at_1000": 0.3752, + "ndcg_at_20": 0.29695, + "ndcg_at_3": 0.23538, + "ndcg_at_5": 0.25507, + "precision_at_1": 0.17401, + "precision_at_10": 0.04006, + "precision_at_100": 0.00686, + "precision_at_1000": 0.001, + "precision_at_20": 0.02352, + "precision_at_3": 0.09273, + "precision_at_5": 0.06531, + "recall_at_1": 0.17401, + "recall_at_10": 0.40064, + "recall_at_100": 0.68636, + "recall_at_1000": 1.0, + "recall_at_20": 0.47046, + "recall_at_3": 0.2782, + "recall_at_5": 0.32653 + }, + { + "cv_recall_at_1": 0.189, + "cv_recall_at_10": 0.434, + "cv_recall_at_100": 0.776, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.52, + "cv_recall_at_3": 0.287, + "cv_recall_at_5": 0.351, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.29912, + "map_at_1": 0.189, + "map_at_10": 0.25745, + "map_at_100": 0.27018, + "map_at_1000": 0.27119, + "map_at_20": 0.26366, + "map_at_3": 0.232, + "map_at_5": 0.24625, + "mrr_at_1": 0.189, + "mrr_at_10": 0.257453968253968, + "mrr_at_100": 0.27018074089708155, + "mrr_at_1000": 0.2711940749154071, + "mrr_at_20": 0.26365941537392756, + "mrr_at_3": 0.2320000000000002, + "mrr_at_5": 0.24625000000000002, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.23584547731802377, + "nauc_cv_recall_at_100_max": 0.5048956866760526, + "nauc_cv_recall_at_100_std": 0.5603047119270076, + "nauc_cv_recall_at_10_diff1": 0.38566115088987807, + "nauc_cv_recall_at_10_max": 0.28669233113414205, + "nauc_cv_recall_at_10_std": 0.28326331170607066, + "nauc_cv_recall_at_1_diff1": 0.4977619739879074, + "nauc_cv_recall_at_1_max": 0.3444770313188568, + "nauc_cv_recall_at_1_std": 0.22987253973319294, + "nauc_cv_recall_at_20_diff1": 0.3523029000489689, + "nauc_cv_recall_at_20_max": 0.3360275314217313, + "nauc_cv_recall_at_20_std": 0.31898770335709226, + "nauc_cv_recall_at_3_diff1": 0.42690386232087146, + "nauc_cv_recall_at_3_max": 0.3048319388632666, + "nauc_cv_recall_at_3_std": 0.21794317850827377, + "nauc_cv_recall_at_5_diff1": 0.4093556153855275, + "nauc_cv_recall_at_5_max": 0.3190624353434359, + "nauc_cv_recall_at_5_std": 0.2662802757450084, + "nauc_map_at_1000_diff1": 0.45476595623650545, + "nauc_map_at_1000_max": 0.33102341505163757, + "nauc_map_at_1000_std": 0.24388150035754028, + "nauc_map_at_100_diff1": 0.4541710217692779, + "nauc_map_at_100_max": 0.3314191205195175, + "nauc_map_at_100_std": 0.2445461478717132, + "nauc_map_at_10_diff1": 0.4579805762156703, + "nauc_map_at_10_max": 0.32450039624849997, + "nauc_map_at_10_std": 0.23816668982486297, + "nauc_map_at_1_diff1": 0.4977619739879074, + "nauc_map_at_1_max": 0.3444770313188568, + "nauc_map_at_1_std": 0.22987253973319294, + "nauc_map_at_20_diff1": 0.4556901341073797, + "nauc_map_at_20_max": 0.3284399315434941, + "nauc_map_at_20_std": 0.2410209601625378, + "nauc_map_at_3_diff1": 0.4679324337192478, + "nauc_map_at_3_max": 0.3260264006938065, + "nauc_map_at_3_std": 0.22374444652641026, + "nauc_map_at_5_diff1": 0.46277591167639953, + "nauc_map_at_5_max": 0.3297028251995126, + "nauc_map_at_5_std": 0.23572419801369274, + "nauc_mrr_at_1000_diff1": 0.45476595623650545, + "nauc_mrr_at_1000_max": 0.33102341505163757, + "nauc_mrr_at_1000_std": 0.24388150035754028, + "nauc_mrr_at_100_diff1": 0.4541710217692779, + "nauc_mrr_at_100_max": 0.3314191205195175, + "nauc_mrr_at_100_std": 0.2445461478717132, + "nauc_mrr_at_10_diff1": 0.4579805762156703, + "nauc_mrr_at_10_max": 0.32450039624849997, + "nauc_mrr_at_10_std": 0.23816668982486297, + "nauc_mrr_at_1_diff1": 0.4977619739879074, + "nauc_mrr_at_1_max": 0.3444770313188568, + "nauc_mrr_at_1_std": 0.22987253973319294, + "nauc_mrr_at_20_diff1": 0.4556901341073797, + "nauc_mrr_at_20_max": 0.3284399315434941, + "nauc_mrr_at_20_std": 0.2410209601625378, + "nauc_mrr_at_3_diff1": 0.4679324337192478, + "nauc_mrr_at_3_max": 0.3260264006938065, + "nauc_mrr_at_3_std": 0.22374444652641026, + "nauc_mrr_at_5_diff1": 0.46277591167639953, + "nauc_mrr_at_5_max": 0.3297028251995126, + "nauc_mrr_at_5_std": 0.23572419801369274, + "nauc_ndcg_at_1000_diff1": 0.43772123611759606, + "nauc_ndcg_at_1000_max": 0.33609697923437043, + "nauc_ndcg_at_1000_std": 0.26127496831875935, + "nauc_ndcg_at_100_diff1": 0.4203011357051008, + "nauc_ndcg_at_100_max": 0.3501915540657517, + "nauc_ndcg_at_100_std": 0.2858917313816372, + "nauc_ndcg_at_10_diff1": 0.43839515861227235, + "nauc_ndcg_at_10_max": 0.3146284118343576, + "nauc_ndcg_at_10_std": 0.24952184654188878, + "nauc_ndcg_at_1_diff1": 0.4977619739879074, + "nauc_ndcg_at_1_max": 0.3444770313188568, + "nauc_ndcg_at_1_std": 0.22987253973319294, + "nauc_ndcg_at_20_diff1": 0.42995677255332104, + "nauc_ndcg_at_20_max": 0.3284335639984129, + "nauc_ndcg_at_20_std": 0.2591496857680367, + "nauc_ndcg_at_3_diff1": 0.4567421335185591, + "nauc_ndcg_at_3_max": 0.32018196382995895, + "nauc_ndcg_at_3_std": 0.22212212819728647, + "nauc_ndcg_at_5_diff1": 0.44806804419654234, + "nauc_ndcg_at_5_max": 0.3265209685228764, + "nauc_ndcg_at_5_std": 0.24354943983311997, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.2358454773180231, + "nauc_precision_at_100_max": 0.5048956866760533, + "nauc_precision_at_100_std": 0.5603047119270077, + "nauc_precision_at_10_diff1": 0.3856611508898776, + "nauc_precision_at_10_max": 0.2866923311341418, + "nauc_precision_at_10_std": 0.2832633117060702, + "nauc_precision_at_1_diff1": 0.4977619739879074, + "nauc_precision_at_1_max": 0.3444770313188568, + "nauc_precision_at_1_std": 0.22987253973319294, + "nauc_precision_at_20_diff1": 0.35230290004896864, + "nauc_precision_at_20_max": 0.33602753142173103, + "nauc_precision_at_20_std": 0.31898770335709214, + "nauc_precision_at_3_diff1": 0.4269038623208713, + "nauc_precision_at_3_max": 0.3048319388632666, + "nauc_precision_at_3_std": 0.21794317850827388, + "nauc_precision_at_5_diff1": 0.40935561538552745, + "nauc_precision_at_5_max": 0.3190624353434356, + "nauc_precision_at_5_std": 0.26628027574500823, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.23584547731802377, + "nauc_recall_at_100_max": 0.5048956866760526, + "nauc_recall_at_100_std": 0.5603047119270076, + "nauc_recall_at_10_diff1": 0.38566115088987807, + "nauc_recall_at_10_max": 0.28669233113414205, + "nauc_recall_at_10_std": 0.28326331170607066, + "nauc_recall_at_1_diff1": 0.4977619739879074, + "nauc_recall_at_1_max": 0.3444770313188568, + "nauc_recall_at_1_std": 0.22987253973319294, + "nauc_recall_at_20_diff1": 0.3523029000489689, + "nauc_recall_at_20_max": 0.3360275314217313, + "nauc_recall_at_20_std": 0.31898770335709226, + "nauc_recall_at_3_diff1": 0.42690386232087146, + "nauc_recall_at_3_max": 0.3048319388632666, + "nauc_recall_at_3_std": 0.21794317850827377, + "nauc_recall_at_5_diff1": 0.4093556153855275, + "nauc_recall_at_5_max": 0.3190624353434359, + "nauc_recall_at_5_std": 0.2662802757450084, + "ndcg_at_1": 0.189, + "ndcg_at_10": 0.29912, + "ndcg_at_100": 0.36856, + "ndcg_at_1000": 0.39676, + "ndcg_at_20": 0.32116, + "ndcg_at_3": 0.24612, + "ndcg_at_5": 0.27215, + "precision_at_1": 0.189, + "precision_at_10": 0.0434, + "precision_at_100": 0.00776, + "precision_at_1000": 0.001, + "precision_at_20": 0.026, + "precision_at_3": 0.09567, + "precision_at_5": 0.0702, + "recall_at_1": 0.189, + "recall_at_10": 0.434, + "recall_at_100": 0.776, + "recall_at_1000": 1.0, + "recall_at_20": 0.52, + "recall_at_3": 0.287, + "recall_at_5": 0.351 + }, + { + "cv_recall_at_1": 0.29126, + "cv_recall_at_10": 0.56311, + "cv_recall_at_100": 0.81969, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.64216, + "cv_recall_at_3": 0.43273, + "cv_recall_at_5": 0.49098, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.42183, + "map_at_1": 0.29126, + "map_at_10": 0.37732, + "map_at_100": 0.38717, + "map_at_1000": 0.38804, + "map_at_20": 0.38286, + "map_at_3": 0.3546, + "map_at_5": 0.36771, + "mrr_at_1": 0.2912621359223301, + "mrr_at_10": 0.37731875921889774, + "mrr_at_100": 0.38716967898159166, + "mrr_at_1000": 0.3880350605603339, + "mrr_at_20": 0.38286417328162536, + "mrr_at_3": 0.35460009246417024, + "mrr_at_5": 0.367706888580675, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.43538500388127865, + "nauc_cv_recall_at_100_max": 0.7177252933127443, + "nauc_cv_recall_at_100_std": 0.660412445901951, + "nauc_cv_recall_at_10_diff1": 0.4963201835184886, + "nauc_cv_recall_at_10_max": 0.5597989306303259, + "nauc_cv_recall_at_10_std": 0.48074848708310886, + "nauc_cv_recall_at_1_diff1": 0.6473674367686191, + "nauc_cv_recall_at_1_max": 0.49186622428315996, + "nauc_cv_recall_at_1_std": 0.3119812982341505, + "nauc_cv_recall_at_20_diff1": 0.5001328769811207, + "nauc_cv_recall_at_20_max": 0.6028237648849925, + "nauc_cv_recall_at_20_std": 0.5323859658135447, + "nauc_cv_recall_at_3_diff1": 0.5468222678409858, + "nauc_cv_recall_at_3_max": 0.5128205176152807, + "nauc_cv_recall_at_3_std": 0.40675980904226217, + "nauc_cv_recall_at_5_diff1": 0.5020836460064558, + "nauc_cv_recall_at_5_max": 0.5394861529788975, + "nauc_cv_recall_at_5_std": 0.45600299088374213, + "nauc_map_at_1000_diff1": 0.5870509021904564, + "nauc_map_at_1000_max": 0.5065320181114963, + "nauc_map_at_1000_std": 0.36554314852816566, + "nauc_map_at_100_diff1": 0.5867048660360288, + "nauc_map_at_100_max": 0.5069224205930181, + "nauc_map_at_100_std": 0.3660919004931112, + "nauc_map_at_10_diff1": 0.5864163481883536, + "nauc_map_at_10_max": 0.5047660836484374, + "nauc_map_at_10_std": 0.3641270487884386, + "nauc_map_at_1_diff1": 0.6473674367686191, + "nauc_map_at_1_max": 0.49186622428315996, + "nauc_map_at_1_std": 0.3119812982341505, + "nauc_map_at_20_diff1": 0.5871061465808929, + "nauc_map_at_20_max": 0.5066720694370498, + "nauc_map_at_20_std": 0.36620195781101544, + "nauc_map_at_3_diff1": 0.5973239337484972, + "nauc_map_at_3_max": 0.49778872535897273, + "nauc_map_at_3_std": 0.3513465169058443, + "nauc_map_at_5_diff1": 0.58703502269122, + "nauc_map_at_5_max": 0.5027772921794861, + "nauc_map_at_5_std": 0.36188878315047157, + "nauc_mrr_at_1000_diff1": 0.5870509211623248, + "nauc_mrr_at_1000_max": 0.506532051280673, + "nauc_mrr_at_1000_std": 0.36554319117411654, + "nauc_mrr_at_100_diff1": 0.5867048660360288, + "nauc_mrr_at_100_max": 0.5069224205930181, + "nauc_mrr_at_100_std": 0.3660919004931112, + "nauc_mrr_at_10_diff1": 0.5864163481883536, + "nauc_mrr_at_10_max": 0.5047660836484374, + "nauc_mrr_at_10_std": 0.3641270487884386, + "nauc_mrr_at_1_diff1": 0.6473674367686191, + "nauc_mrr_at_1_max": 0.49186622428315996, + "nauc_mrr_at_1_std": 0.3119812982341505, + "nauc_mrr_at_20_diff1": 0.5871061465808929, + "nauc_mrr_at_20_max": 0.5066720694370498, + "nauc_mrr_at_20_std": 0.36620195781101544, + "nauc_mrr_at_3_diff1": 0.5973239337484972, + "nauc_mrr_at_3_max": 0.49778872535897273, + "nauc_mrr_at_3_std": 0.3513465169058443, + "nauc_mrr_at_5_diff1": 0.58703502269122, + "nauc_mrr_at_5_max": 0.5027772921794861, + "nauc_mrr_at_5_std": 0.36188878315047157, + "nauc_ndcg_at_1000_diff1": 0.5740812385542557, + "nauc_ndcg_at_1000_max": 0.5168841998243363, + "nauc_ndcg_at_1000_std": 0.3843458584172321, + "nauc_ndcg_at_100_diff1": 0.5645003851534279, + "nauc_ndcg_at_100_max": 0.5303028488918393, + "nauc_ndcg_at_100_std": 0.4028550618170588, + "nauc_ndcg_at_10_diff1": 0.5644944296758979, + "nauc_ndcg_at_10_max": 0.5169814341365214, + "nauc_ndcg_at_10_std": 0.39129432844376977, + "nauc_ndcg_at_1_diff1": 0.6473674367686191, + "nauc_ndcg_at_1_max": 0.49186622428315996, + "nauc_ndcg_at_1_std": 0.3119812982341505, + "nauc_ndcg_at_20_diff1": 0.5669757949202917, + "nauc_ndcg_at_20_max": 0.5250403811377805, + "nauc_ndcg_at_20_std": 0.4001112801964311, + "nauc_ndcg_at_3_diff1": 0.5839362186171795, + "nauc_ndcg_at_3_max": 0.5015676540132488, + "nauc_ndcg_at_3_std": 0.3656795310689533, + "nauc_ndcg_at_5_diff1": 0.5655307294741795, + "nauc_ndcg_at_5_max": 0.5115681799159831, + "nauc_ndcg_at_5_std": 0.38525142911874694, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.4353850038812787, + "nauc_precision_at_100_max": 0.7177252933127455, + "nauc_precision_at_100_std": 0.660412445901951, + "nauc_precision_at_10_diff1": 0.496320183518489, + "nauc_precision_at_10_max": 0.559798930630326, + "nauc_precision_at_10_std": 0.4807484870831086, + "nauc_precision_at_1_diff1": 0.6473674367686191, + "nauc_precision_at_1_max": 0.49186622428315996, + "nauc_precision_at_1_std": 0.3119812982341505, + "nauc_precision_at_20_diff1": 0.5001328769811205, + "nauc_precision_at_20_max": 0.602823764884992, + "nauc_precision_at_20_std": 0.5323859658135439, + "nauc_precision_at_3_diff1": 0.5468222678409858, + "nauc_precision_at_3_max": 0.5128205176152807, + "nauc_precision_at_3_std": 0.4067598090422622, + "nauc_precision_at_5_diff1": 0.5020836460064558, + "nauc_precision_at_5_max": 0.5394861529788973, + "nauc_precision_at_5_std": 0.45600299088374197, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.43538500388127865, + "nauc_recall_at_100_max": 0.7177252933127443, + "nauc_recall_at_100_std": 0.660412445901951, + "nauc_recall_at_10_diff1": 0.4963201835184886, + "nauc_recall_at_10_max": 0.5597989306303259, + "nauc_recall_at_10_std": 0.48074848708310886, + "nauc_recall_at_1_diff1": 0.6473674367686191, + "nauc_recall_at_1_max": 0.49186622428315996, + "nauc_recall_at_1_std": 0.3119812982341505, + "nauc_recall_at_20_diff1": 0.5001328769811207, + "nauc_recall_at_20_max": 0.6028237648849925, + "nauc_recall_at_20_std": 0.5323859658135447, + "nauc_recall_at_3_diff1": 0.5468222678409858, + "nauc_recall_at_3_max": 0.5128205176152807, + "nauc_recall_at_3_std": 0.40675980904226217, + "nauc_recall_at_5_diff1": 0.5020836460064558, + "nauc_recall_at_5_max": 0.5394861529788975, + "nauc_recall_at_5_std": 0.45600299088374213, + "ndcg_at_1": 0.29126, + "ndcg_at_10": 0.42183, + "ndcg_at_100": 0.47434, + "ndcg_at_1000": 0.49735, + "ndcg_at_20": 0.44189, + "ndcg_at_3": 0.37471, + "ndcg_at_5": 0.39852, + "precision_at_1": 0.29126, + "precision_at_10": 0.05631, + "precision_at_100": 0.0082, + "precision_at_1000": 0.001, + "precision_at_20": 0.03211, + "precision_at_3": 0.14424, + "precision_at_5": 0.0982, + "recall_at_1": 0.29126, + "recall_at_10": 0.56311, + "recall_at_100": 0.81969, + "recall_at_1000": 1.0, + "recall_at_20": 0.64216, + "recall_at_3": 0.43273, + "recall_at_5": 0.49098 + }, + { + "cv_recall_at_1": 0.45983, + "cv_recall_at_10": 0.75264, + "cv_recall_at_100": 0.94186, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.81395, + "cv_recall_at_3": 0.61416, + "cv_recall_at_5": 0.67759, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.60066, + "map_at_1": 0.45983, + "map_at_10": 0.55269, + "map_at_100": 0.56038, + "map_at_1000": 0.56068, + "map_at_20": 0.55716, + "map_at_3": 0.52819, + "map_at_5": 0.54267, + "mrr_at_1": 0.459830866807611, + "mrr_at_10": 0.552691365482063, + "mrr_at_100": 0.5603828225377104, + "mrr_at_1000": 0.5606781539383635, + "mrr_at_20": 0.5571602161130574, + "mrr_at_3": 0.5281888653981675, + "mrr_at_5": 0.5426708949964759, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4429595399611373, + "nauc_cv_recall_at_100_max": 0.5374944784553677, + "nauc_cv_recall_at_100_std": 0.5020336324941823, + "nauc_cv_recall_at_10_diff1": 0.5042140972034406, + "nauc_cv_recall_at_10_max": 0.4597924352668207, + "nauc_cv_recall_at_10_std": 0.3309821314292971, + "nauc_cv_recall_at_1_diff1": 0.6973331588586901, + "nauc_cv_recall_at_1_max": 0.47206125971867047, + "nauc_cv_recall_at_1_std": 0.16444378947679292, + "nauc_cv_recall_at_20_diff1": 0.5037305064368949, + "nauc_cv_recall_at_20_max": 0.4271691618156776, + "nauc_cv_recall_at_20_std": 0.28189017720507564, + "nauc_cv_recall_at_3_diff1": 0.580064919110858, + "nauc_cv_recall_at_3_max": 0.49645920159223683, + "nauc_cv_recall_at_3_std": 0.2711250691727226, + "nauc_cv_recall_at_5_diff1": 0.5374231247800073, + "nauc_cv_recall_at_5_max": 0.48513326853185296, + "nauc_cv_recall_at_5_std": 0.30095998643913063, + "nauc_map_at_1000_diff1": 0.6441521021101068, + "nauc_map_at_1000_max": 0.48098963639253217, + "nauc_map_at_1000_std": 0.20812381908060446, + "nauc_map_at_100_diff1": 0.643929028420692, + "nauc_map_at_100_max": 0.48105172984074207, + "nauc_map_at_100_std": 0.20837106228113203, + "nauc_map_at_10_diff1": 0.6417540808024643, + "nauc_map_at_10_max": 0.4813076374672515, + "nauc_map_at_10_std": 0.21095403593602866, + "nauc_map_at_1_diff1": 0.6973331588586901, + "nauc_map_at_1_max": 0.47206125971867047, + "nauc_map_at_1_std": 0.16444378947679292, + "nauc_map_at_20_diff1": 0.6432421142666999, + "nauc_map_at_20_max": 0.48038725601830146, + "nauc_map_at_20_std": 0.2078558363364783, + "nauc_map_at_3_diff1": 0.6489600889526264, + "nauc_map_at_3_max": 0.4866454342294112, + "nauc_map_at_3_std": 0.20663618594230385, + "nauc_map_at_5_diff1": 0.6430473848288992, + "nauc_map_at_5_max": 0.4842255766526718, + "nauc_map_at_5_std": 0.21044581143827049, + "nauc_mrr_at_1000_diff1": 0.6441520919455076, + "nauc_mrr_at_1000_max": 0.4809896494420363, + "nauc_mrr_at_1000_std": 0.20812383899078785, + "nauc_mrr_at_100_diff1": 0.643929028420692, + "nauc_mrr_at_100_max": 0.48105172984074207, + "nauc_mrr_at_100_std": 0.20837106228113203, + "nauc_mrr_at_10_diff1": 0.6417540808024643, + "nauc_mrr_at_10_max": 0.4813076374672515, + "nauc_mrr_at_10_std": 0.21095403593602866, + "nauc_mrr_at_1_diff1": 0.6973331588586901, + "nauc_mrr_at_1_max": 0.47206125971867047, + "nauc_mrr_at_1_std": 0.16444378947679292, + "nauc_mrr_at_20_diff1": 0.6432421142666999, + "nauc_mrr_at_20_max": 0.48038725601830146, + "nauc_mrr_at_20_std": 0.2078558363364783, + "nauc_mrr_at_3_diff1": 0.6489600889526264, + "nauc_mrr_at_3_max": 0.4866454342294112, + "nauc_mrr_at_3_std": 0.20663618594230385, + "nauc_mrr_at_5_diff1": 0.6430473848288992, + "nauc_mrr_at_5_max": 0.4842255766526718, + "nauc_mrr_at_5_std": 0.21044581143827049, + "nauc_ndcg_at_1000_diff1": 0.6326592590675244, + "nauc_ndcg_at_1000_max": 0.479746255773527, + "nauc_ndcg_at_1000_std": 0.21821891778897104, + "nauc_ndcg_at_100_diff1": 0.627023184120909, + "nauc_ndcg_at_100_max": 0.4814040178905751, + "nauc_ndcg_at_100_std": 0.22609099066376712, + "nauc_ndcg_at_10_diff1": 0.6160435381553313, + "nauc_ndcg_at_10_max": 0.4780939816399053, + "nauc_ndcg_at_10_std": 0.2332277013810803, + "nauc_ndcg_at_1_diff1": 0.6973331588586901, + "nauc_ndcg_at_1_max": 0.47206125971867047, + "nauc_ndcg_at_1_std": 0.16444378947679292, + "nauc_ndcg_at_20_diff1": 0.6206953233188065, + "nauc_ndcg_at_20_max": 0.4739268715311874, + "nauc_ndcg_at_20_std": 0.22160597418951172, + "nauc_ndcg_at_3_diff1": 0.6323475050574792, + "nauc_ndcg_at_3_max": 0.4892097773834569, + "nauc_ndcg_at_3_std": 0.2221068601383589, + "nauc_ndcg_at_5_diff1": 0.6199812667553398, + "nauc_ndcg_at_5_max": 0.484803228251082, + "nauc_ndcg_at_5_std": 0.23025081492492494, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.44295953996113135, + "nauc_precision_at_100_max": 0.5374944784553698, + "nauc_precision_at_100_std": 0.5020336324941786, + "nauc_precision_at_10_diff1": 0.5042140972034405, + "nauc_precision_at_10_max": 0.4597924352668208, + "nauc_precision_at_10_std": 0.33098213142929594, + "nauc_precision_at_1_diff1": 0.6973331588586901, + "nauc_precision_at_1_max": 0.47206125971867047, + "nauc_precision_at_1_std": 0.16444378947679292, + "nauc_precision_at_20_diff1": 0.5037305064368954, + "nauc_precision_at_20_max": 0.4271691618156773, + "nauc_precision_at_20_std": 0.28189017720507437, + "nauc_precision_at_3_diff1": 0.5800649191108583, + "nauc_precision_at_3_max": 0.49645920159223705, + "nauc_precision_at_3_std": 0.2711250691727229, + "nauc_precision_at_5_diff1": 0.5374231247800063, + "nauc_precision_at_5_max": 0.4851332685318528, + "nauc_precision_at_5_std": 0.3009599864391298, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4429595399611373, + "nauc_recall_at_100_max": 0.5374944784553677, + "nauc_recall_at_100_std": 0.5020336324941823, + "nauc_recall_at_10_diff1": 0.5042140972034406, + "nauc_recall_at_10_max": 0.4597924352668207, + "nauc_recall_at_10_std": 0.3309821314292971, + "nauc_recall_at_1_diff1": 0.6973331588586901, + "nauc_recall_at_1_max": 0.47206125971867047, + "nauc_recall_at_1_std": 0.16444378947679292, + "nauc_recall_at_20_diff1": 0.5037305064368949, + "nauc_recall_at_20_max": 0.4271691618156776, + "nauc_recall_at_20_std": 0.28189017720507564, + "nauc_recall_at_3_diff1": 0.580064919110858, + "nauc_recall_at_3_max": 0.49645920159223683, + "nauc_recall_at_3_std": 0.2711250691727226, + "nauc_recall_at_5_diff1": 0.5374231247800073, + "nauc_recall_at_5_max": 0.48513326853185296, + "nauc_recall_at_5_std": 0.30095998643913063, + "ndcg_at_1": 0.45983, + "ndcg_at_10": 0.60066, + "ndcg_at_100": 0.64004, + "ndcg_at_1000": 0.64756, + "ndcg_at_20": 0.61643, + "ndcg_at_3": 0.55028, + "ndcg_at_5": 0.5764, + "precision_at_1": 0.45983, + "precision_at_10": 0.07526, + "precision_at_100": 0.00942, + "precision_at_1000": 0.001, + "precision_at_20": 0.0407, + "precision_at_3": 0.20472, + "precision_at_5": 0.13552, + "recall_at_1": 0.45983, + "recall_at_10": 0.75264, + "recall_at_100": 0.94186, + "recall_at_1000": 1.0, + "recall_at_20": 0.81395, + "recall_at_3": 0.61416, + "recall_at_5": 0.67759 + }, + { + "cv_recall_at_1": 0.421, + "cv_recall_at_10": 0.697, + "cv_recall_at_100": 0.947, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.769, + "cv_recall_at_3": 0.558, + "cv_recall_at_5": 0.608, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.55025, + "map_at_1": 0.421, + "map_at_10": 0.50454, + "map_at_100": 0.51404, + "map_at_1000": 0.5143, + "map_at_20": 0.5095, + "map_at_3": 0.48167, + "map_at_5": 0.49302, + "mrr_at_1": 0.421, + "mrr_at_10": 0.5045424603174602, + "mrr_at_100": 0.514036278166916, + "mrr_at_1000": 0.5143008176799778, + "mrr_at_20": 0.5094954526424723, + "mrr_at_3": 0.4816666666666664, + "mrr_at_5": 0.49301666666666627, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4855627785705507, + "nauc_cv_recall_at_100_max": 0.731365502175715, + "nauc_cv_recall_at_100_std": 0.7247678945792161, + "nauc_cv_recall_at_10_diff1": 0.5321993122841938, + "nauc_cv_recall_at_10_max": 0.3845373133259707, + "nauc_cv_recall_at_10_std": 0.22025163739727724, + "nauc_cv_recall_at_1_diff1": 0.7210976082522879, + "nauc_cv_recall_at_1_max": 0.4254087701305756, + "nauc_cv_recall_at_1_std": 0.20592471965325038, + "nauc_cv_recall_at_20_diff1": 0.5359568379661741, + "nauc_cv_recall_at_20_max": 0.40377868528873234, + "nauc_cv_recall_at_20_std": 0.2406943573982335, + "nauc_cv_recall_at_3_diff1": 0.621321956788392, + "nauc_cv_recall_at_3_max": 0.38942145623174623, + "nauc_cv_recall_at_3_std": 0.1645052364400703, + "nauc_cv_recall_at_5_diff1": 0.5961893952166478, + "nauc_cv_recall_at_5_max": 0.3920810351887088, + "nauc_cv_recall_at_5_std": 0.18723355572147657, + "nauc_map_at_1000_diff1": 0.6697072108072887, + "nauc_map_at_1000_max": 0.411943976387272, + "nauc_map_at_1000_std": 0.1967222314828611, + "nauc_map_at_100_diff1": 0.6695110001873309, + "nauc_map_at_100_max": 0.4121863201598696, + "nauc_map_at_100_std": 0.19710670362571528, + "nauc_map_at_10_diff1": 0.6672475373507716, + "nauc_map_at_10_max": 0.4098526385755607, + "nauc_map_at_10_std": 0.19445865524250197, + "nauc_map_at_1_diff1": 0.7210976082522879, + "nauc_map_at_1_max": 0.4254087701305756, + "nauc_map_at_1_std": 0.20592471965325038, + "nauc_map_at_20_diff1": 0.668795383818105, + "nauc_map_at_20_max": 0.41071496567002347, + "nauc_map_at_20_std": 0.19505186847070433, + "nauc_map_at_3_diff1": 0.6771890825309842, + "nauc_map_at_3_max": 0.40849482153000877, + "nauc_map_at_3_std": 0.184697047083633, + "nauc_map_at_5_diff1": 0.6726059841502193, + "nauc_map_at_5_max": 0.409503476165295, + "nauc_map_at_5_std": 0.1899677021603648, + "nauc_mrr_at_1000_diff1": 0.6697072108072887, + "nauc_mrr_at_1000_max": 0.411943976387272, + "nauc_mrr_at_1000_std": 0.1967222314828611, + "nauc_mrr_at_100_diff1": 0.6695110001873309, + "nauc_mrr_at_100_max": 0.4121863201598696, + "nauc_mrr_at_100_std": 0.19710670362571528, + "nauc_mrr_at_10_diff1": 0.6672475373507716, + "nauc_mrr_at_10_max": 0.4098526385755607, + "nauc_mrr_at_10_std": 0.19445865524250197, + "nauc_mrr_at_1_diff1": 0.7210976082522879, + "nauc_mrr_at_1_max": 0.4254087701305756, + "nauc_mrr_at_1_std": 0.20592471965325038, + "nauc_mrr_at_20_diff1": 0.668795383818105, + "nauc_mrr_at_20_max": 0.41071496567002347, + "nauc_mrr_at_20_std": 0.19505186847070433, + "nauc_mrr_at_3_diff1": 0.6771890825309842, + "nauc_mrr_at_3_max": 0.40849482153000877, + "nauc_mrr_at_3_std": 0.184697047083633, + "nauc_mrr_at_5_diff1": 0.6726059841502193, + "nauc_mrr_at_5_max": 0.409503476165295, + "nauc_mrr_at_5_std": 0.1899677021603648, + "nauc_ndcg_at_1000_diff1": 0.6581251444086017, + "nauc_ndcg_at_1000_max": 0.4134415641023258, + "nauc_ndcg_at_1000_std": 0.20355955369147752, + "nauc_ndcg_at_100_diff1": 0.6535580361197576, + "nauc_ndcg_at_100_max": 0.4211935378090691, + "nauc_ndcg_at_100_std": 0.21616041137002734, + "nauc_ndcg_at_10_diff1": 0.6404035207915835, + "nauc_ndcg_at_10_max": 0.4047612423982089, + "nauc_ndcg_at_10_std": 0.1987362690556888, + "nauc_ndcg_at_1_diff1": 0.7210976082522879, + "nauc_ndcg_at_1_max": 0.4254087701305756, + "nauc_ndcg_at_1_std": 0.20592471965325038, + "nauc_ndcg_at_20_diff1": 0.6455609884072833, + "nauc_ndcg_at_20_max": 0.40861630089629375, + "nauc_ndcg_at_20_std": 0.20131788577749649, + "nauc_ndcg_at_3_diff1": 0.6633132744102853, + "nauc_ndcg_at_3_max": 0.40370965648596985, + "nauc_ndcg_at_3_std": 0.17954858276017077, + "nauc_ndcg_at_5_diff1": 0.6546348270037611, + "nauc_ndcg_at_5_max": 0.40525686733966526, + "nauc_ndcg_at_5_std": 0.18885692395096065, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.4855627785705482, + "nauc_precision_at_100_max": 0.731365502175715, + "nauc_precision_at_100_std": 0.7247678945792142, + "nauc_precision_at_10_diff1": 0.5321993122841937, + "nauc_precision_at_10_max": 0.3845373133259703, + "nauc_precision_at_10_std": 0.22025163739727607, + "nauc_precision_at_1_diff1": 0.7210976082522879, + "nauc_precision_at_1_max": 0.4254087701305756, + "nauc_precision_at_1_std": 0.20592471965325038, + "nauc_precision_at_20_diff1": 0.535956837966174, + "nauc_precision_at_20_max": 0.40377868528873134, + "nauc_precision_at_20_std": 0.24069435739823442, + "nauc_precision_at_3_diff1": 0.6213219567883927, + "nauc_precision_at_3_max": 0.3894214562317463, + "nauc_precision_at_3_std": 0.16450523644007103, + "nauc_precision_at_5_diff1": 0.5961893952166473, + "nauc_precision_at_5_max": 0.39208103518870874, + "nauc_precision_at_5_std": 0.18723355572147696, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4855627785705507, + "nauc_recall_at_100_max": 0.731365502175715, + "nauc_recall_at_100_std": 0.7247678945792161, + "nauc_recall_at_10_diff1": 0.5321993122841938, + "nauc_recall_at_10_max": 0.3845373133259707, + "nauc_recall_at_10_std": 0.22025163739727724, + "nauc_recall_at_1_diff1": 0.7210976082522879, + "nauc_recall_at_1_max": 0.4254087701305756, + "nauc_recall_at_1_std": 0.20592471965325038, + "nauc_recall_at_20_diff1": 0.5359568379661741, + "nauc_recall_at_20_max": 0.40377868528873234, + "nauc_recall_at_20_std": 0.2406943573982335, + "nauc_recall_at_3_diff1": 0.621321956788392, + "nauc_recall_at_3_max": 0.38942145623174623, + "nauc_recall_at_3_std": 0.1645052364400703, + "nauc_recall_at_5_diff1": 0.5961893952166478, + "nauc_recall_at_5_max": 0.3920810351887088, + "nauc_recall_at_5_std": 0.18723355572147657, + "ndcg_at_1": 0.421, + "ndcg_at_10": 0.55025, + "ndcg_at_100": 0.60141, + "ndcg_at_1000": 0.60826, + "ndcg_at_20": 0.56839, + "ndcg_at_3": 0.50128, + "ndcg_at_5": 0.52181, + "precision_at_1": 0.421, + "precision_at_10": 0.0697, + "precision_at_100": 0.00947, + "precision_at_1000": 0.001, + "precision_at_20": 0.03845, + "precision_at_3": 0.186, + "precision_at_5": 0.1216, + "recall_at_1": 0.421, + "recall_at_10": 0.697, + "recall_at_100": 0.947, + "recall_at_1000": 1.0, + "recall_at_20": 0.769, + "recall_at_3": 0.558, + "recall_at_5": 0.608 + } + ] + }, + "task_name": "WITT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/model_meta.json b/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/model_meta.json new file mode 100644 index 0000000000..d1bb13979d --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/model_meta.json @@ -0,0 +1 @@ +{"name": "google/siglip-base-patch16-256-multilingual", "revision": "8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/WITT2IRetrieval.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/WITT2IRetrieval.json new file mode 100644 index 0000000000..617035f059 --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/WITT2IRetrieval.json @@ -0,0 +1,1936 @@ +{ + "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", + "evaluation_time": 76.18750858306885, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.13483, + "cv_recall_at_10": 0.30337, + "cv_recall_at_100": 0.63933, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.38202, + "cv_recall_at_3": 0.19326, + "cv_recall_at_5": 0.24157, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.20723, + "map_at_1": 0.13483, + "map_at_10": 0.17781, + "map_at_100": 0.18943, + "map_at_1000": 0.19099, + "map_at_20": 0.18309, + "map_at_3": 0.15861, + "map_at_5": 0.16957, + "mrr_at_1": 0.1348314606741573, + "mrr_at_10": 0.1778107722489745, + "mrr_at_100": 0.18943211198548437, + "mrr_at_1000": 0.19098625996171575, + "mrr_at_20": 0.18308938797198454, + "mrr_at_3": 0.15861423220973786, + "mrr_at_5": 0.16956928838951316, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.3002040717601591, + "nauc_cv_recall_at_100_max": 0.4681682095972851, + "nauc_cv_recall_at_100_std": 0.4755266180167234, + "nauc_cv_recall_at_10_diff1": 0.3701965776989649, + "nauc_cv_recall_at_10_max": 0.3761393776807772, + "nauc_cv_recall_at_10_std": 0.3489670955910214, + "nauc_cv_recall_at_1_diff1": 0.6065338821108335, + "nauc_cv_recall_at_1_max": 0.4882184330442451, + "nauc_cv_recall_at_1_std": 0.3549830956930766, + "nauc_cv_recall_at_20_diff1": 0.356259847114431, + "nauc_cv_recall_at_20_max": 0.39515084320476174, + "nauc_cv_recall_at_20_std": 0.4012473011612301, + "nauc_cv_recall_at_3_diff1": 0.4336991857794745, + "nauc_cv_recall_at_3_max": 0.4503825046974262, + "nauc_cv_recall_at_3_std": 0.3552670822273256, + "nauc_cv_recall_at_5_diff1": 0.4053060368334942, + "nauc_cv_recall_at_5_max": 0.40797751394890314, + "nauc_cv_recall_at_5_std": 0.32909342618618104, + "nauc_map_at_1000_diff1": 0.5054655795372572, + "nauc_map_at_1000_max": 0.45675426413623993, + "nauc_map_at_1000_std": 0.3581527976889818, + "nauc_map_at_100_diff1": 0.5043801271441104, + "nauc_map_at_100_max": 0.45653696828813317, + "nauc_map_at_100_std": 0.3583138001490483, + "nauc_map_at_10_diff1": 0.5075131819044263, + "nauc_map_at_10_max": 0.45403109899811495, + "nauc_map_at_10_std": 0.35281856964515634, + "nauc_map_at_1_diff1": 0.6065338821108335, + "nauc_map_at_1_max": 0.4882184330442451, + "nauc_map_at_1_std": 0.3549830956930766, + "nauc_map_at_20_diff1": 0.505531819154646, + "nauc_map_at_20_max": 0.4552086212020412, + "nauc_map_at_20_std": 0.3577827870853602, + "nauc_map_at_3_diff1": 0.5259349787448474, + "nauc_map_at_3_max": 0.47348219240664524, + "nauc_map_at_3_std": 0.3568078210563393, + "nauc_map_at_5_diff1": 0.5155157517292684, + "nauc_map_at_5_max": 0.45978901596979405, + "nauc_map_at_5_std": 0.3485293329187086, + "nauc_mrr_at_1000_diff1": 0.5054655795372572, + "nauc_mrr_at_1000_max": 0.45675426413623993, + "nauc_mrr_at_1000_std": 0.3581527976889818, + "nauc_mrr_at_100_diff1": 0.5043801271441104, + "nauc_mrr_at_100_max": 0.45653696828813317, + "nauc_mrr_at_100_std": 0.3583138001490483, + "nauc_mrr_at_10_diff1": 0.5075131819044263, + "nauc_mrr_at_10_max": 0.45403109899811495, + "nauc_mrr_at_10_std": 0.35281856964515634, + "nauc_mrr_at_1_diff1": 0.6065338821108335, + "nauc_mrr_at_1_max": 0.4882184330442451, + "nauc_mrr_at_1_std": 0.3549830956930766, + "nauc_mrr_at_20_diff1": 0.505531819154646, + "nauc_mrr_at_20_max": 0.4552086212020412, + "nauc_mrr_at_20_std": 0.3577827870853602, + "nauc_mrr_at_3_diff1": 0.5259349787448474, + "nauc_mrr_at_3_max": 0.47348219240664524, + "nauc_mrr_at_3_std": 0.3568078210563393, + "nauc_mrr_at_5_diff1": 0.5155157517292684, + "nauc_mrr_at_5_max": 0.45978901596979405, + "nauc_mrr_at_5_std": 0.3485293329187086, + "nauc_ndcg_at_1000_diff1": 0.47759230790461354, + "nauc_ndcg_at_1000_max": 0.44982400569152814, + "nauc_ndcg_at_1000_std": 0.3667093394535492, + "nauc_ndcg_at_100_diff1": 0.45295974994472493, + "nauc_ndcg_at_100_max": 0.4508470813906076, + "nauc_ndcg_at_100_std": 0.379308242615182, + "nauc_ndcg_at_10_diff1": 0.46603335111945293, + "nauc_ndcg_at_10_max": 0.43139552979443707, + "nauc_ndcg_at_10_std": 0.35151331443779565, + "nauc_ndcg_at_1_diff1": 0.6065338821108335, + "nauc_ndcg_at_1_max": 0.4882184330442451, + "nauc_ndcg_at_1_std": 0.3549830956930766, + "nauc_ndcg_at_20_diff1": 0.4601678130379194, + "nauc_ndcg_at_20_max": 0.4364576544675796, + "nauc_ndcg_at_20_std": 0.3683819085567761, + "nauc_ndcg_at_3_diff1": 0.499746811378489, + "nauc_ndcg_at_3_max": 0.4670335489333946, + "nauc_ndcg_at_3_std": 0.3564343587817275, + "nauc_ndcg_at_5_diff1": 0.48303833414011565, + "nauc_ndcg_at_5_max": 0.4448883386689996, + "nauc_ndcg_at_5_std": 0.3430501197807311, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3002040717601596, + "nauc_precision_at_100_max": 0.46816820959728567, + "nauc_precision_at_100_std": 0.4755266180167234, + "nauc_precision_at_10_diff1": 0.37019657769896475, + "nauc_precision_at_10_max": 0.37613937768077704, + "nauc_precision_at_10_std": 0.34896709559102135, + "nauc_precision_at_1_diff1": 0.6065338821108335, + "nauc_precision_at_1_max": 0.4882184330442451, + "nauc_precision_at_1_std": 0.3549830956930766, + "nauc_precision_at_20_diff1": 0.3562598471144309, + "nauc_precision_at_20_max": 0.3951508432047615, + "nauc_precision_at_20_std": 0.40124730116123025, + "nauc_precision_at_3_diff1": 0.43369918577947425, + "nauc_precision_at_3_max": 0.4503825046974258, + "nauc_precision_at_3_std": 0.35526708222732556, + "nauc_precision_at_5_diff1": 0.40530603683349403, + "nauc_precision_at_5_max": 0.4079775139489031, + "nauc_precision_at_5_std": 0.32909342618618126, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.3002040717601591, + "nauc_recall_at_100_max": 0.4681682095972851, + "nauc_recall_at_100_std": 0.4755266180167234, + "nauc_recall_at_10_diff1": 0.3701965776989649, + "nauc_recall_at_10_max": 0.3761393776807772, + "nauc_recall_at_10_std": 0.3489670955910214, + "nauc_recall_at_1_diff1": 0.6065338821108335, + "nauc_recall_at_1_max": 0.4882184330442451, + "nauc_recall_at_1_std": 0.3549830956930766, + "nauc_recall_at_20_diff1": 0.356259847114431, + "nauc_recall_at_20_max": 0.39515084320476174, + "nauc_recall_at_20_std": 0.4012473011612301, + "nauc_recall_at_3_diff1": 0.4336991857794745, + "nauc_recall_at_3_max": 0.4503825046974262, + "nauc_recall_at_3_std": 0.3552670822273256, + "nauc_recall_at_5_diff1": 0.4053060368334942, + "nauc_recall_at_5_max": 0.40797751394890314, + "nauc_recall_at_5_std": 0.32909342618618104, + "ndcg_at_1": 0.13483, + "ndcg_at_10": 0.20723, + "ndcg_at_100": 0.27408, + "ndcg_at_1000": 0.31905, + "ndcg_at_20": 0.22689, + "ndcg_at_3": 0.16743, + "ndcg_at_5": 0.18725, + "precision_at_1": 0.13483, + "precision_at_10": 0.03034, + "precision_at_100": 0.00639, + "precision_at_1000": 0.001, + "precision_at_20": 0.0191, + "precision_at_3": 0.06442, + "precision_at_5": 0.04831, + "recall_at_1": 0.13483, + "recall_at_10": 0.30337, + "recall_at_100": 0.63933, + "recall_at_1000": 1.0, + "recall_at_20": 0.38202, + "recall_at_3": 0.19326, + "recall_at_5": 0.24157 + }, + { + "cv_recall_at_1": 0.1407, + "cv_recall_at_10": 0.32209, + "cv_recall_at_100": 0.68605, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.40349, + "cv_recall_at_3": 0.20698, + "cv_recall_at_5": 0.25349, + "hf_subset": "bg", + "languages": [ + "bul-Cyrl" + ], + "main_score": 0.21949, + "map_at_1": 0.1407, + "map_at_10": 0.18806, + "map_at_100": 0.19986, + "map_at_1000": 0.2014, + "map_at_20": 0.19363, + "map_at_3": 0.16841, + "map_at_5": 0.17876, + "mrr_at_1": 0.14069767441860465, + "mrr_at_10": 0.18805878552971583, + "mrr_at_100": 0.199857434973446, + "mrr_at_1000": 0.20139668433598573, + "mrr_at_20": 0.1936337157754288, + "mrr_at_3": 0.16841085271317835, + "mrr_at_5": 0.17875968992248067, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2428187864857724, + "nauc_cv_recall_at_100_max": 0.3504993592742509, + "nauc_cv_recall_at_100_std": 0.37675284258487124, + "nauc_cv_recall_at_10_diff1": 0.34495904167942326, + "nauc_cv_recall_at_10_max": 0.1964802730243885, + "nauc_cv_recall_at_10_std": 0.1529343749951434, + "nauc_cv_recall_at_1_diff1": 0.6347504221281157, + "nauc_cv_recall_at_1_max": 0.29627863650775316, + "nauc_cv_recall_at_1_std": 0.17109276495759043, + "nauc_cv_recall_at_20_diff1": 0.3686049944624201, + "nauc_cv_recall_at_20_max": 0.19272426449099975, + "nauc_cv_recall_at_20_std": 0.14347502244255989, + "nauc_cv_recall_at_3_diff1": 0.46476368983897803, + "nauc_cv_recall_at_3_max": 0.23512666550601383, + "nauc_cv_recall_at_3_std": 0.1646336720217216, + "nauc_cv_recall_at_5_diff1": 0.39303577318150434, + "nauc_cv_recall_at_5_max": 0.19109121634257398, + "nauc_cv_recall_at_5_std": 0.14149124626406387, + "nauc_map_at_1000_diff1": 0.5178979975519488, + "nauc_map_at_1000_max": 0.2520281507799221, + "nauc_map_at_1000_std": 0.16253498933831326, + "nauc_map_at_100_diff1": 0.5168410775781308, + "nauc_map_at_100_max": 0.25182300657032847, + "nauc_map_at_100_std": 0.16266888300130025, + "nauc_map_at_10_diff1": 0.5181276295036846, + "nauc_map_at_10_max": 0.25153894042920016, + "nauc_map_at_10_std": 0.16128937040858818, + "nauc_map_at_1_diff1": 0.6347504221281157, + "nauc_map_at_1_max": 0.29627863650775316, + "nauc_map_at_1_std": 0.17109276495759043, + "nauc_map_at_20_diff1": 0.5191927175733649, + "nauc_map_at_20_max": 0.2502218937847514, + "nauc_map_at_20_std": 0.1597487735456185, + "nauc_map_at_3_diff1": 0.5505877763675288, + "nauc_map_at_3_max": 0.2635034544704076, + "nauc_map_at_3_std": 0.1636388695664378, + "nauc_map_at_5_diff1": 0.5287732974391035, + "nauc_map_at_5_max": 0.25086722661365046, + "nauc_map_at_5_std": 0.15793590077616376, + "nauc_mrr_at_1000_diff1": 0.5178980012528572, + "nauc_mrr_at_1000_max": 0.25202814514475286, + "nauc_mrr_at_1000_std": 0.1625349513052187, + "nauc_mrr_at_100_diff1": 0.5168410775781308, + "nauc_mrr_at_100_max": 0.25182300657032847, + "nauc_mrr_at_100_std": 0.16266888300130025, + "nauc_mrr_at_10_diff1": 0.5181276295036846, + "nauc_mrr_at_10_max": 0.25153894042920016, + "nauc_mrr_at_10_std": 0.16128937040858818, + "nauc_mrr_at_1_diff1": 0.6347504221281157, + "nauc_mrr_at_1_max": 0.29627863650775316, + "nauc_mrr_at_1_std": 0.17109276495759043, + "nauc_mrr_at_20_diff1": 0.5191927175733649, + "nauc_mrr_at_20_max": 0.2502218937847514, + "nauc_mrr_at_20_std": 0.1597487735456185, + "nauc_mrr_at_3_diff1": 0.5505877763675288, + "nauc_mrr_at_3_max": 0.2635034544704076, + "nauc_mrr_at_3_std": 0.1636388695664378, + "nauc_mrr_at_5_diff1": 0.5287732974391035, + "nauc_mrr_at_5_max": 0.25086722661365046, + "nauc_mrr_at_5_std": 0.15793590077616376, + "nauc_ndcg_at_1000_diff1": 0.48472063125969944, + "nauc_ndcg_at_1000_max": 0.24838283580088705, + "nauc_ndcg_at_1000_std": 0.17053299626936394, + "nauc_ndcg_at_100_diff1": 0.4554439428367917, + "nauc_ndcg_at_100_max": 0.25784490286848205, + "nauc_ndcg_at_100_std": 0.1919503537996817, + "nauc_ndcg_at_10_diff1": 0.46695823299989325, + "nauc_ndcg_at_10_max": 0.23497369111423905, + "nauc_ndcg_at_10_std": 0.15893086664948175, + "nauc_ndcg_at_1_diff1": 0.6347504221281157, + "nauc_ndcg_at_1_max": 0.29627863650775316, + "nauc_ndcg_at_1_std": 0.17109276495759043, + "nauc_ndcg_at_20_diff1": 0.4724458069597433, + "nauc_ndcg_at_20_max": 0.2324494583873871, + "nauc_ndcg_at_20_std": 0.15515694071655556, + "nauc_ndcg_at_3_diff1": 0.5260471514772517, + "nauc_ndcg_at_3_max": 0.2552904616025047, + "nauc_ndcg_at_3_std": 0.1637548191784826, + "nauc_ndcg_at_5_diff1": 0.4894596334723677, + "nauc_ndcg_at_5_max": 0.23368023426560022, + "nauc_ndcg_at_5_std": 0.1533233974295215, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.24281878648577213, + "nauc_precision_at_100_max": 0.35049935927425113, + "nauc_precision_at_100_std": 0.37675284258487085, + "nauc_precision_at_10_diff1": 0.3449590416794232, + "nauc_precision_at_10_max": 0.19648027302438859, + "nauc_precision_at_10_std": 0.1529343749951435, + "nauc_precision_at_1_diff1": 0.6347504221281157, + "nauc_precision_at_1_max": 0.29627863650775316, + "nauc_precision_at_1_std": 0.17109276495759043, + "nauc_precision_at_20_diff1": 0.3686049944624202, + "nauc_precision_at_20_max": 0.19272426449099955, + "nauc_precision_at_20_std": 0.14347502244256, + "nauc_precision_at_3_diff1": 0.4647636898389777, + "nauc_precision_at_3_max": 0.23512666550601385, + "nauc_precision_at_3_std": 0.16463367202172133, + "nauc_precision_at_5_diff1": 0.39303577318150446, + "nauc_precision_at_5_max": 0.191091216342574, + "nauc_precision_at_5_std": 0.14149124626406415, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2428187864857724, + "nauc_recall_at_100_max": 0.3504993592742509, + "nauc_recall_at_100_std": 0.37675284258487124, + "nauc_recall_at_10_diff1": 0.34495904167942326, + "nauc_recall_at_10_max": 0.1964802730243885, + "nauc_recall_at_10_std": 0.1529343749951434, + "nauc_recall_at_1_diff1": 0.6347504221281157, + "nauc_recall_at_1_max": 0.29627863650775316, + "nauc_recall_at_1_std": 0.17109276495759043, + "nauc_recall_at_20_diff1": 0.3686049944624201, + "nauc_recall_at_20_max": 0.19272426449099975, + "nauc_recall_at_20_std": 0.14347502244255989, + "nauc_recall_at_3_diff1": 0.46476368983897803, + "nauc_recall_at_3_max": 0.23512666550601383, + "nauc_recall_at_3_std": 0.1646336720217216, + "nauc_recall_at_5_diff1": 0.39303577318150434, + "nauc_recall_at_5_max": 0.19109121634257398, + "nauc_recall_at_5_std": 0.14149124626406387, + "ndcg_at_1": 0.1407, + "ndcg_at_10": 0.21949, + "ndcg_at_100": 0.29031, + "ndcg_at_1000": 0.33046, + "ndcg_at_20": 0.23998, + "ndcg_at_3": 0.17825, + "ndcg_at_5": 0.19716, + "precision_at_1": 0.1407, + "precision_at_10": 0.03221, + "precision_at_100": 0.00686, + "precision_at_1000": 0.001, + "precision_at_20": 0.02017, + "precision_at_3": 0.06899, + "precision_at_5": 0.0507, + "recall_at_1": 0.1407, + "recall_at_10": 0.32209, + "recall_at_100": 0.68605, + "recall_at_1000": 1.0, + "recall_at_20": 0.40349, + "recall_at_3": 0.20698, + "recall_at_5": 0.25349 + }, + { + "cv_recall_at_1": 0.31201, + "cv_recall_at_10": 0.55443, + "cv_recall_at_100": 0.81257, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.63412, + "cv_recall_at_3": 0.42088, + "cv_recall_at_5": 0.47026, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.42241, + "map_at_1": 0.31201, + "map_at_10": 0.38151, + "map_at_100": 0.39138, + "map_at_1000": 0.39224, + "map_at_20": 0.38695, + "map_at_3": 0.35896, + "map_at_5": 0.37024, + "mrr_at_1": 0.31200897867564537, + "mrr_at_10": 0.3815139935510306, + "mrr_at_100": 0.3913751785670525, + "mrr_at_1000": 0.39224160925971996, + "mrr_at_20": 0.38694515136032975, + "mrr_at_3": 0.3589599700710812, + "mrr_at_5": 0.37023943135054227, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.42409783935737194, + "nauc_cv_recall_at_100_max": 0.5681665077524001, + "nauc_cv_recall_at_100_std": 0.5351888788683341, + "nauc_cv_recall_at_10_diff1": 0.4908110669756098, + "nauc_cv_recall_at_10_max": 0.44110626843837974, + "nauc_cv_recall_at_10_std": 0.3307889697670433, + "nauc_cv_recall_at_1_diff1": 0.6603157950562596, + "nauc_cv_recall_at_1_max": 0.4919339948870857, + "nauc_cv_recall_at_1_std": 0.19876708948379945, + "nauc_cv_recall_at_20_diff1": 0.43626796524742434, + "nauc_cv_recall_at_20_max": 0.48145219539697703, + "nauc_cv_recall_at_20_std": 0.41036023299874647, + "nauc_cv_recall_at_3_diff1": 0.5627330715157299, + "nauc_cv_recall_at_3_max": 0.49752444289033215, + "nauc_cv_recall_at_3_std": 0.2957201276468252, + "nauc_cv_recall_at_5_diff1": 0.5377169103841068, + "nauc_cv_recall_at_5_max": 0.461998159762653, + "nauc_cv_recall_at_5_std": 0.2905834636220688, + "nauc_map_at_1000_diff1": 0.6048376948334023, + "nauc_map_at_1000_max": 0.4870026289030281, + "nauc_map_at_1000_std": 0.250945646125177, + "nauc_map_at_100_diff1": 0.6044739666710233, + "nauc_map_at_100_max": 0.487066095883574, + "nauc_map_at_100_std": 0.2513750588492708, + "nauc_map_at_10_diff1": 0.6061716820773362, + "nauc_map_at_10_max": 0.4828330168661941, + "nauc_map_at_10_std": 0.24640874505077667, + "nauc_map_at_1_diff1": 0.6603157950562596, + "nauc_map_at_1_max": 0.4919339948870857, + "nauc_map_at_1_std": 0.19876708948379945, + "nauc_map_at_20_diff1": 0.6033902765092166, + "nauc_map_at_20_max": 0.4856995179729451, + "nauc_map_at_20_std": 0.25081409216717315, + "nauc_map_at_3_diff1": 0.6175165157619867, + "nauc_map_at_3_max": 0.49266653072933536, + "nauc_map_at_3_std": 0.2426696726132534, + "nauc_map_at_5_diff1": 0.6114934391371092, + "nauc_map_at_5_max": 0.48455960871116116, + "nauc_map_at_5_std": 0.24137827487506577, + "nauc_mrr_at_1000_diff1": 0.6048376948334023, + "nauc_mrr_at_1000_max": 0.4870026289030281, + "nauc_mrr_at_1000_std": 0.250945646125177, + "nauc_mrr_at_100_diff1": 0.6044739666710233, + "nauc_mrr_at_100_max": 0.487066095883574, + "nauc_mrr_at_100_std": 0.2513750588492708, + "nauc_mrr_at_10_diff1": 0.6061716820773362, + "nauc_mrr_at_10_max": 0.4828330168661941, + "nauc_mrr_at_10_std": 0.24640874505077667, + "nauc_mrr_at_1_diff1": 0.6603157950562596, + "nauc_mrr_at_1_max": 0.4919339948870857, + "nauc_mrr_at_1_std": 0.19876708948379945, + "nauc_mrr_at_20_diff1": 0.6033902765092166, + "nauc_mrr_at_20_max": 0.4856995179729451, + "nauc_mrr_at_20_std": 0.25081409216717315, + "nauc_mrr_at_3_diff1": 0.6175165157619867, + "nauc_mrr_at_3_max": 0.49266653072933536, + "nauc_mrr_at_3_std": 0.2426696726132534, + "nauc_mrr_at_5_diff1": 0.6114934391371092, + "nauc_mrr_at_5_max": 0.48455960871116116, + "nauc_mrr_at_5_std": 0.24137827487506577, + "nauc_ndcg_at_1000_diff1": 0.5880265966614385, + "nauc_ndcg_at_1000_max": 0.4887610935079035, + "nauc_ndcg_at_1000_std": 0.2700524820212304, + "nauc_ndcg_at_100_diff1": 0.5768554508340397, + "nauc_ndcg_at_100_max": 0.4936025136378576, + "nauc_ndcg_at_100_std": 0.287310533772089, + "nauc_ndcg_at_10_diff1": 0.5794624398102854, + "nauc_ndcg_at_10_max": 0.4738347553313095, + "nauc_ndcg_at_10_std": 0.2662128530488792, + "nauc_ndcg_at_1_diff1": 0.6603157950562596, + "nauc_ndcg_at_1_max": 0.4919339948870857, + "nauc_ndcg_at_1_std": 0.19876708948379945, + "nauc_ndcg_at_20_diff1": 0.569235966407218, + "nauc_ndcg_at_20_max": 0.4838935445595701, + "nauc_ndcg_at_20_std": 0.2825462527831911, + "nauc_ndcg_at_3_diff1": 0.6033020386909346, + "nauc_ndcg_at_3_max": 0.4938559154274072, + "nauc_ndcg_at_3_std": 0.2564915255297522, + "nauc_ndcg_at_5_diff1": 0.5928196609086988, + "nauc_ndcg_at_5_max": 0.4792111070485909, + "nauc_ndcg_at_5_std": 0.25417597546292575, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.4240978393573708, + "nauc_precision_at_100_max": 0.5681665077524004, + "nauc_precision_at_100_std": 0.5351888788683329, + "nauc_precision_at_10_diff1": 0.49081106697560933, + "nauc_precision_at_10_max": 0.44110626843837974, + "nauc_precision_at_10_std": 0.3307889697670433, + "nauc_precision_at_1_diff1": 0.6603157950562596, + "nauc_precision_at_1_max": 0.4919339948870857, + "nauc_precision_at_1_std": 0.19876708948379945, + "nauc_precision_at_20_diff1": 0.43626796524742434, + "nauc_precision_at_20_max": 0.48145219539697637, + "nauc_precision_at_20_std": 0.4103602329987468, + "nauc_precision_at_3_diff1": 0.5627330715157296, + "nauc_precision_at_3_max": 0.49752444289033193, + "nauc_precision_at_3_std": 0.29572012764682554, + "nauc_precision_at_5_diff1": 0.5377169103841068, + "nauc_precision_at_5_max": 0.4619981597626524, + "nauc_precision_at_5_std": 0.29058346362206866, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.42409783935737194, + "nauc_recall_at_100_max": 0.5681665077524001, + "nauc_recall_at_100_std": 0.5351888788683341, + "nauc_recall_at_10_diff1": 0.4908110669756098, + "nauc_recall_at_10_max": 0.44110626843837974, + "nauc_recall_at_10_std": 0.3307889697670433, + "nauc_recall_at_1_diff1": 0.6603157950562596, + "nauc_recall_at_1_max": 0.4919339948870857, + "nauc_recall_at_1_std": 0.19876708948379945, + "nauc_recall_at_20_diff1": 0.43626796524742434, + "nauc_recall_at_20_max": 0.48145219539697703, + "nauc_recall_at_20_std": 0.41036023299874647, + "nauc_recall_at_3_diff1": 0.5627330715157299, + "nauc_recall_at_3_max": 0.49752444289033215, + "nauc_recall_at_3_std": 0.2957201276468252, + "nauc_recall_at_5_diff1": 0.5377169103841068, + "nauc_recall_at_5_max": 0.461998159762653, + "nauc_recall_at_5_std": 0.2905834636220688, + "ndcg_at_1": 0.31201, + "ndcg_at_10": 0.42241, + "ndcg_at_100": 0.47523, + "ndcg_at_1000": 0.49886, + "ndcg_at_20": 0.44243, + "ndcg_at_3": 0.37482, + "ndcg_at_5": 0.39515, + "precision_at_1": 0.31201, + "precision_at_10": 0.05544, + "precision_at_100": 0.00813, + "precision_at_1000": 0.001, + "precision_at_20": 0.03171, + "precision_at_3": 0.14029, + "precision_at_5": 0.09405, + "recall_at_1": 0.31201, + "recall_at_10": 0.55443, + "recall_at_100": 0.81257, + "recall_at_1000": 1.0, + "recall_at_20": 0.63412, + "recall_at_3": 0.42088, + "recall_at_5": 0.47026 + }, + { + "cv_recall_at_1": 0.11404, + "cv_recall_at_10": 0.24211, + "cv_recall_at_100": 0.57193, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.30351, + "cv_recall_at_3": 0.15789, + "cv_recall_at_5": 0.18947, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.1699, + "map_at_1": 0.11404, + "map_at_10": 0.14796, + "map_at_100": 0.15834, + "map_at_1000": 0.16026, + "map_at_20": 0.15224, + "map_at_3": 0.13421, + "map_at_5": 0.14114, + "mrr_at_1": 0.11403508771929824, + "mrr_at_10": 0.1479566972988026, + "mrr_at_100": 0.15834412120057514, + "mrr_at_1000": 0.16025762172047456, + "mrr_at_20": 0.1522396953872024, + "mrr_at_3": 0.13421052631578947, + "mrr_at_5": 0.141140350877193, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.14916308807129272, + "nauc_cv_recall_at_100_max": 0.2508666844965659, + "nauc_cv_recall_at_100_std": 0.41207757015476687, + "nauc_cv_recall_at_10_diff1": 0.2920350970876328, + "nauc_cv_recall_at_10_max": 0.23395789815352871, + "nauc_cv_recall_at_10_std": 0.3874919508096308, + "nauc_cv_recall_at_1_diff1": 0.5665683213093571, + "nauc_cv_recall_at_1_max": 0.35968558199633904, + "nauc_cv_recall_at_1_std": 0.4074641972649942, + "nauc_cv_recall_at_20_diff1": 0.2232536897378315, + "nauc_cv_recall_at_20_max": 0.20621593928881596, + "nauc_cv_recall_at_20_std": 0.35304348649166706, + "nauc_cv_recall_at_3_diff1": 0.4337144801769612, + "nauc_cv_recall_at_3_max": 0.2864454653734898, + "nauc_cv_recall_at_3_std": 0.3876535647439169, + "nauc_cv_recall_at_5_diff1": 0.3801795090109267, + "nauc_cv_recall_at_5_max": 0.23712217965091523, + "nauc_cv_recall_at_5_std": 0.3777168771581285, + "nauc_map_at_1000_diff1": 0.4545928364247393, + "nauc_map_at_1000_max": 0.3060927387596978, + "nauc_map_at_1000_std": 0.39824255335755576, + "nauc_map_at_100_diff1": 0.45302741061897966, + "nauc_map_at_100_max": 0.30567588396362266, + "nauc_map_at_100_std": 0.3983424056494272, + "nauc_map_at_10_diff1": 0.4641548511790888, + "nauc_map_at_10_max": 0.3070952684924584, + "nauc_map_at_10_std": 0.40004023276797424, + "nauc_map_at_1_diff1": 0.5665683213093571, + "nauc_map_at_1_max": 0.35968558199633904, + "nauc_map_at_1_std": 0.4074641972649942, + "nauc_map_at_20_diff1": 0.45506833027410765, + "nauc_map_at_20_max": 0.30377682000528866, + "nauc_map_at_20_std": 0.3965571744014492, + "nauc_map_at_3_diff1": 0.49933312991396894, + "nauc_map_at_3_max": 0.32643242734954164, + "nauc_map_at_3_std": 0.4035398807501951, + "nauc_map_at_5_diff1": 0.4821683291085525, + "nauc_map_at_5_max": 0.3116439870265342, + "nauc_map_at_5_std": 0.40056380152957843, + "nauc_mrr_at_1000_diff1": 0.4545928364247393, + "nauc_mrr_at_1000_max": 0.3060927387596978, + "nauc_mrr_at_1000_std": 0.39824255335755576, + "nauc_mrr_at_100_diff1": 0.45302741061897966, + "nauc_mrr_at_100_max": 0.30567588396362266, + "nauc_mrr_at_100_std": 0.3983424056494272, + "nauc_mrr_at_10_diff1": 0.4641548511790888, + "nauc_mrr_at_10_max": 0.3070952684924584, + "nauc_mrr_at_10_std": 0.40004023276797424, + "nauc_mrr_at_1_diff1": 0.5665683213093571, + "nauc_mrr_at_1_max": 0.35968558199633904, + "nauc_mrr_at_1_std": 0.4074641972649942, + "nauc_mrr_at_20_diff1": 0.45506833027410765, + "nauc_mrr_at_20_max": 0.30377682000528866, + "nauc_mrr_at_20_std": 0.3965571744014492, + "nauc_mrr_at_3_diff1": 0.49933312991396894, + "nauc_mrr_at_3_max": 0.32643242734954164, + "nauc_mrr_at_3_std": 0.4035398807501951, + "nauc_mrr_at_5_diff1": 0.4821683291085525, + "nauc_mrr_at_5_max": 0.3116439870265342, + "nauc_mrr_at_5_std": 0.40056380152957843, + "nauc_ndcg_at_1000_diff1": 0.41198031732074153, + "nauc_ndcg_at_1000_max": 0.292407714742196, + "nauc_ndcg_at_1000_std": 0.39510889733579196, + "nauc_ndcg_at_100_diff1": 0.37090134146384546, + "nauc_ndcg_at_100_max": 0.2852556140761665, + "nauc_ndcg_at_100_std": 0.3977790657553699, + "nauc_ndcg_at_10_diff1": 0.4132626869592185, + "nauc_ndcg_at_10_max": 0.2842838711906126, + "nauc_ndcg_at_10_std": 0.39569785122526163, + "nauc_ndcg_at_1_diff1": 0.5665683213093571, + "nauc_ndcg_at_1_max": 0.35968558199633904, + "nauc_ndcg_at_1_std": 0.4074641972649942, + "nauc_ndcg_at_20_diff1": 0.3855757865062933, + "nauc_ndcg_at_20_max": 0.2734964176350558, + "nauc_ndcg_at_20_std": 0.3839196775789696, + "nauc_ndcg_at_3_diff1": 0.4808969265424267, + "nauc_ndcg_at_3_max": 0.3153556314861584, + "nauc_ndcg_at_3_std": 0.39932922170634866, + "nauc_ndcg_at_5_diff1": 0.4523925584034704, + "nauc_ndcg_at_5_max": 0.290241056262286, + "nauc_ndcg_at_5_std": 0.3941396765756552, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.14916308807129303, + "nauc_precision_at_100_max": 0.25086668449656613, + "nauc_precision_at_100_std": 0.41207757015476687, + "nauc_precision_at_10_diff1": 0.29203509708763287, + "nauc_precision_at_10_max": 0.2339578981535289, + "nauc_precision_at_10_std": 0.3874919508096306, + "nauc_precision_at_1_diff1": 0.5665683213093571, + "nauc_precision_at_1_max": 0.35968558199633904, + "nauc_precision_at_1_std": 0.4074641972649942, + "nauc_precision_at_20_diff1": 0.22325368973783147, + "nauc_precision_at_20_max": 0.20621593928881604, + "nauc_precision_at_20_std": 0.353043486491667, + "nauc_precision_at_3_diff1": 0.433714480176961, + "nauc_precision_at_3_max": 0.28644546537349, + "nauc_precision_at_3_std": 0.3876535647439171, + "nauc_precision_at_5_diff1": 0.3801795090109264, + "nauc_precision_at_5_max": 0.2371221796509154, + "nauc_precision_at_5_std": 0.3777168771581287, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.14916308807129272, + "nauc_recall_at_100_max": 0.2508666844965659, + "nauc_recall_at_100_std": 0.41207757015476687, + "nauc_recall_at_10_diff1": 0.2920350970876328, + "nauc_recall_at_10_max": 0.23395789815352871, + "nauc_recall_at_10_std": 0.3874919508096308, + "nauc_recall_at_1_diff1": 0.5665683213093571, + "nauc_recall_at_1_max": 0.35968558199633904, + "nauc_recall_at_1_std": 0.4074641972649942, + "nauc_recall_at_20_diff1": 0.2232536897378315, + "nauc_recall_at_20_max": 0.20621593928881596, + "nauc_recall_at_20_std": 0.35304348649166706, + "nauc_recall_at_3_diff1": 0.4337144801769612, + "nauc_recall_at_3_max": 0.2864454653734898, + "nauc_recall_at_3_std": 0.3876535647439169, + "nauc_recall_at_5_diff1": 0.3801795090109267, + "nauc_recall_at_5_max": 0.23712217965091523, + "nauc_recall_at_5_std": 0.3777168771581285, + "ndcg_at_1": 0.11404, + "ndcg_at_10": 0.1699, + "ndcg_at_100": 0.23369, + "ndcg_at_1000": 0.28782, + "ndcg_at_20": 0.18545, + "ndcg_at_3": 0.14033, + "ndcg_at_5": 0.15308, + "precision_at_1": 0.11404, + "precision_at_10": 0.02421, + "precision_at_100": 0.00572, + "precision_at_1000": 0.001, + "precision_at_20": 0.01518, + "precision_at_3": 0.05263, + "precision_at_5": 0.03789, + "recall_at_1": 0.11404, + "recall_at_10": 0.24211, + "recall_at_100": 0.57193, + "recall_at_1000": 1.0, + "recall_at_20": 0.30351, + "recall_at_3": 0.15789, + "recall_at_5": 0.18947 + }, + { + "cv_recall_at_1": 0.16705, + "cv_recall_at_10": 0.35126, + "cv_recall_at_100": 0.67735, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.43822, + "cv_recall_at_3": 0.24714, + "cv_recall_at_5": 0.29176, + "hf_subset": "et", + "languages": [ + "est-Latn" + ], + "main_score": 0.25082, + "map_at_1": 0.16705, + "map_at_10": 0.2198, + "map_at_100": 0.23141, + "map_at_1000": 0.23296, + "map_at_20": 0.22571, + "map_at_3": 0.20252, + "map_at_5": 0.21219, + "mrr_at_1": 0.16704805491990846, + "mrr_at_10": 0.21979768261232777, + "mrr_at_100": 0.23140874491520524, + "mrr_at_1000": 0.23295989128115277, + "mrr_at_20": 0.22570981921725977, + "mrr_at_3": 0.20251716247139592, + "mrr_at_5": 0.21218535469107544, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2525428564670692, + "nauc_cv_recall_at_100_max": 0.5346628929514341, + "nauc_cv_recall_at_100_std": 0.47650255481546067, + "nauc_cv_recall_at_10_diff1": 0.38458830102538316, + "nauc_cv_recall_at_10_max": 0.5109280947554741, + "nauc_cv_recall_at_10_std": 0.3745191607765455, + "nauc_cv_recall_at_1_diff1": 0.6638275888245915, + "nauc_cv_recall_at_1_max": 0.5504752903293038, + "nauc_cv_recall_at_1_std": 0.31046701629003715, + "nauc_cv_recall_at_20_diff1": 0.3278440033736192, + "nauc_cv_recall_at_20_max": 0.5165547496354531, + "nauc_cv_recall_at_20_std": 0.39395543291064483, + "nauc_cv_recall_at_3_diff1": 0.479147621248729, + "nauc_cv_recall_at_3_max": 0.5183797446519467, + "nauc_cv_recall_at_3_std": 0.3360203436773196, + "nauc_cv_recall_at_5_diff1": 0.4463467292860443, + "nauc_cv_recall_at_5_max": 0.49931333096174124, + "nauc_cv_recall_at_5_std": 0.3332607535748596, + "nauc_map_at_1000_diff1": 0.5448568065314144, + "nauc_map_at_1000_max": 0.5318758450937885, + "nauc_map_at_1000_std": 0.33283444678560264, + "nauc_map_at_100_diff1": 0.5437588464406067, + "nauc_map_at_100_max": 0.5318250750358087, + "nauc_map_at_100_std": 0.33338742536532645, + "nauc_map_at_10_diff1": 0.5492756780946725, + "nauc_map_at_10_max": 0.5311112107540561, + "nauc_map_at_10_std": 0.3299222777160743, + "nauc_map_at_1_diff1": 0.6638275888245915, + "nauc_map_at_1_max": 0.5504752903293038, + "nauc_map_at_1_std": 0.31046701629003715, + "nauc_map_at_20_diff1": 0.544610032644422, + "nauc_map_at_20_max": 0.5311375624808914, + "nauc_map_at_20_std": 0.3313447833782751, + "nauc_map_at_3_diff1": 0.5707234130723992, + "nauc_map_at_3_max": 0.5346313018781644, + "nauc_map_at_3_std": 0.324016696954914, + "nauc_map_at_5_diff1": 0.5594957337059822, + "nauc_map_at_5_max": 0.5284747680149817, + "nauc_map_at_5_std": 0.32302330446027244, + "nauc_mrr_at_1000_diff1": 0.5448568065314144, + "nauc_mrr_at_1000_max": 0.5318758450937885, + "nauc_mrr_at_1000_std": 0.33283444678560264, + "nauc_mrr_at_100_diff1": 0.5437588464406067, + "nauc_mrr_at_100_max": 0.5318250750358087, + "nauc_mrr_at_100_std": 0.33338742536532645, + "nauc_mrr_at_10_diff1": 0.5492756780946725, + "nauc_mrr_at_10_max": 0.5311112107540561, + "nauc_mrr_at_10_std": 0.3299222777160743, + "nauc_mrr_at_1_diff1": 0.6638275888245915, + "nauc_mrr_at_1_max": 0.5504752903293038, + "nauc_mrr_at_1_std": 0.31046701629003715, + "nauc_mrr_at_20_diff1": 0.544610032644422, + "nauc_mrr_at_20_max": 0.5311375624808914, + "nauc_mrr_at_20_std": 0.3313447833782751, + "nauc_mrr_at_3_diff1": 0.5707234130723992, + "nauc_mrr_at_3_max": 0.5346313018781644, + "nauc_mrr_at_3_std": 0.324016696954914, + "nauc_mrr_at_5_diff1": 0.5594957337059822, + "nauc_mrr_at_5_max": 0.5284747680149817, + "nauc_mrr_at_5_std": 0.32302330446027244, + "nauc_ndcg_at_1000_diff1": 0.5106477734657984, + "nauc_ndcg_at_1000_max": 0.5301338839155268, + "nauc_ndcg_at_1000_std": 0.3447217097906694, + "nauc_ndcg_at_100_diff1": 0.48090999547407126, + "nauc_ndcg_at_100_max": 0.530346752198247, + "nauc_ndcg_at_100_std": 0.3600192855287342, + "nauc_ndcg_at_10_diff1": 0.5031977703710548, + "nauc_ndcg_at_10_max": 0.5251715595267603, + "nauc_ndcg_at_10_std": 0.3417303671545655, + "nauc_ndcg_at_1_diff1": 0.6638275888245915, + "nauc_ndcg_at_1_max": 0.5504752903293038, + "nauc_ndcg_at_1_std": 0.31046701629003715, + "nauc_ndcg_at_20_diff1": 0.4864847598490328, + "nauc_ndcg_at_20_max": 0.5262779001040049, + "nauc_ndcg_at_20_std": 0.34723586519080096, + "nauc_ndcg_at_3_diff1": 0.5450553817498409, + "nauc_ndcg_at_3_max": 0.5300900562100276, + "nauc_ndcg_at_3_std": 0.3274088836871772, + "nauc_ndcg_at_5_diff1": 0.5270163829776572, + "nauc_ndcg_at_5_max": 0.5202203374368238, + "nauc_ndcg_at_5_std": 0.32601310394670324, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.2525428564670695, + "nauc_precision_at_100_max": 0.5346628929514347, + "nauc_precision_at_100_std": 0.4765025548154612, + "nauc_precision_at_10_diff1": 0.384588301025383, + "nauc_precision_at_10_max": 0.5109280947554742, + "nauc_precision_at_10_std": 0.37451916077654557, + "nauc_precision_at_1_diff1": 0.6638275888245915, + "nauc_precision_at_1_max": 0.5504752903293038, + "nauc_precision_at_1_std": 0.31046701629003715, + "nauc_precision_at_20_diff1": 0.32784400337361863, + "nauc_precision_at_20_max": 0.5165547496354526, + "nauc_precision_at_20_std": 0.3939554329106448, + "nauc_precision_at_3_diff1": 0.4791476212487288, + "nauc_precision_at_3_max": 0.5183797446519465, + "nauc_precision_at_3_std": 0.33602034367731953, + "nauc_precision_at_5_diff1": 0.4463467292860441, + "nauc_precision_at_5_max": 0.4993133309617412, + "nauc_precision_at_5_std": 0.33326075357485957, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2525428564670692, + "nauc_recall_at_100_max": 0.5346628929514341, + "nauc_recall_at_100_std": 0.47650255481546067, + "nauc_recall_at_10_diff1": 0.38458830102538316, + "nauc_recall_at_10_max": 0.5109280947554741, + "nauc_recall_at_10_std": 0.3745191607765455, + "nauc_recall_at_1_diff1": 0.6638275888245915, + "nauc_recall_at_1_max": 0.5504752903293038, + "nauc_recall_at_1_std": 0.31046701629003715, + "nauc_recall_at_20_diff1": 0.3278440033736192, + "nauc_recall_at_20_max": 0.5165547496354531, + "nauc_recall_at_20_std": 0.39395543291064483, + "nauc_recall_at_3_diff1": 0.479147621248729, + "nauc_recall_at_3_max": 0.5183797446519467, + "nauc_recall_at_3_std": 0.3360203436773196, + "nauc_recall_at_5_diff1": 0.4463467292860443, + "nauc_recall_at_5_max": 0.49931333096174124, + "nauc_recall_at_5_std": 0.3332607535748596, + "ndcg_at_1": 0.16705, + "ndcg_at_10": 0.25082, + "ndcg_at_100": 0.31616, + "ndcg_at_1000": 0.35742, + "ndcg_at_20": 0.27263, + "ndcg_at_3": 0.21398, + "ndcg_at_5": 0.2319, + "precision_at_1": 0.16705, + "precision_at_10": 0.03513, + "precision_at_100": 0.00677, + "precision_at_1000": 0.001, + "precision_at_20": 0.02191, + "precision_at_3": 0.08238, + "precision_at_5": 0.05835, + "recall_at_1": 0.16705, + "recall_at_10": 0.35126, + "recall_at_100": 0.67735, + "recall_at_1000": 1.0, + "recall_at_20": 0.43822, + "recall_at_3": 0.24714, + "recall_at_5": 0.29176 + }, + { + "cv_recall_at_1": 0.36515, + "cv_recall_at_10": 0.65039, + "cv_recall_at_100": 0.89789, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.73585, + "cv_recall_at_3": 0.51165, + "cv_recall_at_5": 0.57381, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.50016, + "map_at_1": 0.36515, + "map_at_10": 0.45294, + "map_at_100": 0.46318, + "map_at_1000": 0.46369, + "map_at_20": 0.45889, + "map_at_3": 0.4286, + "map_at_5": 0.44286, + "mrr_at_1": 0.365149833518313, + "mrr_at_10": 0.4529411764705879, + "mrr_at_100": 0.463181284099606, + "mrr_at_1000": 0.4636910375275689, + "mrr_at_20": 0.4588889551156271, + "mrr_at_3": 0.4285978542360337, + "mrr_at_5": 0.4428597854236029, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.3519632757367839, + "nauc_cv_recall_at_100_max": 0.5896633539169532, + "nauc_cv_recall_at_100_std": 0.6279061759070669, + "nauc_cv_recall_at_10_diff1": 0.49598786888905616, + "nauc_cv_recall_at_10_max": 0.41815541361430786, + "nauc_cv_recall_at_10_std": 0.35841020167020005, + "nauc_cv_recall_at_1_diff1": 0.6832526806736251, + "nauc_cv_recall_at_1_max": 0.38716079008324905, + "nauc_cv_recall_at_1_std": 0.19616342687021568, + "nauc_cv_recall_at_20_diff1": 0.4103941158660433, + "nauc_cv_recall_at_20_max": 0.43701975747528027, + "nauc_cv_recall_at_20_std": 0.4431938868593069, + "nauc_cv_recall_at_3_diff1": 0.5678328869179713, + "nauc_cv_recall_at_3_max": 0.40246002367987804, + "nauc_cv_recall_at_3_std": 0.2518196988828705, + "nauc_cv_recall_at_5_diff1": 0.529681288168825, + "nauc_cv_recall_at_5_max": 0.39142002976373985, + "nauc_cv_recall_at_5_std": 0.2885002963574131, + "nauc_map_at_1000_diff1": 0.6207013621900259, + "nauc_map_at_1000_max": 0.3989618223778303, + "nauc_map_at_1000_std": 0.23971198686244546, + "nauc_map_at_100_diff1": 0.6202909508328959, + "nauc_map_at_100_max": 0.39917405505970993, + "nauc_map_at_100_std": 0.24020801896301075, + "nauc_map_at_10_diff1": 0.6220116794480105, + "nauc_map_at_10_max": 0.3970448836274126, + "nauc_map_at_10_std": 0.2374657253002716, + "nauc_map_at_1_diff1": 0.6832526806736251, + "nauc_map_at_1_max": 0.38716079008324905, + "nauc_map_at_1_std": 0.19616342687021568, + "nauc_map_at_20_diff1": 0.6191197977242113, + "nauc_map_at_20_max": 0.39772268435868907, + "nauc_map_at_20_std": 0.24056952293607264, + "nauc_map_at_3_diff1": 0.6321019056808161, + "nauc_map_at_3_max": 0.3968853497625998, + "nauc_map_at_3_std": 0.22306202122311464, + "nauc_map_at_5_diff1": 0.6249797342785457, + "nauc_map_at_5_max": 0.39455312160663836, + "nauc_map_at_5_std": 0.2307452054244085, + "nauc_mrr_at_1000_diff1": 0.6207013621900259, + "nauc_mrr_at_1000_max": 0.3989618223778303, + "nauc_mrr_at_1000_std": 0.23971198686244546, + "nauc_mrr_at_100_diff1": 0.6202909508328959, + "nauc_mrr_at_100_max": 0.39917405505970993, + "nauc_mrr_at_100_std": 0.24020801896301075, + "nauc_mrr_at_10_diff1": 0.6220116794480105, + "nauc_mrr_at_10_max": 0.3970448836274126, + "nauc_mrr_at_10_std": 0.2374657253002716, + "nauc_mrr_at_1_diff1": 0.6832526806736251, + "nauc_mrr_at_1_max": 0.38716079008324905, + "nauc_mrr_at_1_std": 0.19616342687021568, + "nauc_mrr_at_20_diff1": 0.6191197977242113, + "nauc_mrr_at_20_max": 0.39772268435868907, + "nauc_mrr_at_20_std": 0.24056952293607264, + "nauc_mrr_at_3_diff1": 0.6321019056808161, + "nauc_mrr_at_3_max": 0.3968853497625998, + "nauc_mrr_at_3_std": 0.22306202122311464, + "nauc_mrr_at_5_diff1": 0.6249797342785457, + "nauc_mrr_at_5_max": 0.39455312160663836, + "nauc_mrr_at_5_std": 0.2307452054244085, + "nauc_ndcg_at_1000_diff1": 0.6038061339373723, + "nauc_ndcg_at_1000_max": 0.40478501436504905, + "nauc_ndcg_at_1000_std": 0.25668452905230366, + "nauc_ndcg_at_100_diff1": 0.592446378602503, + "nauc_ndcg_at_100_max": 0.41265629017662886, + "nauc_ndcg_at_100_std": 0.2728434617850929, + "nauc_ndcg_at_10_diff1": 0.5948353501805591, + "nauc_ndcg_at_10_max": 0.40113867845503637, + "nauc_ndcg_at_10_std": 0.26223182571918424, + "nauc_ndcg_at_1_diff1": 0.6832526806736251, + "nauc_ndcg_at_1_max": 0.38716079008324905, + "nauc_ndcg_at_1_std": 0.19616342687021568, + "nauc_ndcg_at_20_diff1": 0.5828639375379747, + "nauc_ndcg_at_20_max": 0.40394839922498804, + "nauc_ndcg_at_20_std": 0.274426191371496, + "nauc_ndcg_at_3_diff1": 0.6157909448194059, + "nauc_ndcg_at_3_max": 0.39842749233436653, + "nauc_ndcg_at_3_std": 0.23045707287814626, + "nauc_ndcg_at_5_diff1": 0.6022492170015042, + "nauc_ndcg_at_5_max": 0.39410237251204805, + "nauc_ndcg_at_5_std": 0.24445390105709308, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3519632757367811, + "nauc_precision_at_100_max": 0.5896633539169541, + "nauc_precision_at_100_std": 0.627906175907067, + "nauc_precision_at_10_diff1": 0.4959878688890556, + "nauc_precision_at_10_max": 0.418155413614308, + "nauc_precision_at_10_std": 0.3584102016701997, + "nauc_precision_at_1_diff1": 0.6832526806736251, + "nauc_precision_at_1_max": 0.38716079008324905, + "nauc_precision_at_1_std": 0.19616342687021568, + "nauc_precision_at_20_diff1": 0.4103941158660434, + "nauc_precision_at_20_max": 0.43701975747528016, + "nauc_precision_at_20_std": 0.4431938868593058, + "nauc_precision_at_3_diff1": 0.5678328869179718, + "nauc_precision_at_3_max": 0.4024600236798784, + "nauc_precision_at_3_std": 0.2518196988828707, + "nauc_precision_at_5_diff1": 0.5296812881688243, + "nauc_precision_at_5_max": 0.3914200297637398, + "nauc_precision_at_5_std": 0.288500296357413, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.3519632757367839, + "nauc_recall_at_100_max": 0.5896633539169532, + "nauc_recall_at_100_std": 0.6279061759070669, + "nauc_recall_at_10_diff1": 0.49598786888905616, + "nauc_recall_at_10_max": 0.41815541361430786, + "nauc_recall_at_10_std": 0.35841020167020005, + "nauc_recall_at_1_diff1": 0.6832526806736251, + "nauc_recall_at_1_max": 0.38716079008324905, + "nauc_recall_at_1_std": 0.19616342687021568, + "nauc_recall_at_20_diff1": 0.4103941158660433, + "nauc_recall_at_20_max": 0.43701975747528027, + "nauc_recall_at_20_std": 0.4431938868593069, + "nauc_recall_at_3_diff1": 0.5678328869179713, + "nauc_recall_at_3_max": 0.40246002367987804, + "nauc_recall_at_3_std": 0.2518196988828705, + "nauc_recall_at_5_diff1": 0.529681288168825, + "nauc_recall_at_5_max": 0.39142002976373985, + "nauc_recall_at_5_std": 0.2885002963574131, + "ndcg_at_1": 0.36515, + "ndcg_at_10": 0.50016, + "ndcg_at_100": 0.55211, + "ndcg_at_1000": 0.56527, + "ndcg_at_20": 0.52179, + "ndcg_at_3": 0.44988, + "ndcg_at_5": 0.47553, + "precision_at_1": 0.36515, + "precision_at_10": 0.06504, + "precision_at_100": 0.00898, + "precision_at_1000": 0.001, + "precision_at_20": 0.03679, + "precision_at_3": 0.17055, + "precision_at_5": 0.11476, + "recall_at_1": 0.36515, + "recall_at_10": 0.65039, + "recall_at_100": 0.89789, + "recall_at_1000": 1.0, + "recall_at_20": 0.73585, + "recall_at_3": 0.51165, + "recall_at_5": 0.57381 + }, + { + "cv_recall_at_1": 0.09023, + "cv_recall_at_10": 0.20086, + "cv_recall_at_100": 0.45542, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.25564, + "cv_recall_at_3": 0.12675, + "cv_recall_at_5": 0.15252, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.13653, + "map_at_1": 0.09023, + "map_at_10": 0.11714, + "map_at_100": 0.12554, + "map_at_1000": 0.12765, + "map_at_20": 0.12095, + "map_at_3": 0.10526, + "map_at_5": 0.11106, + "mrr_at_1": 0.09022556390977443, + "mrr_at_10": 0.11714447683835436, + "mrr_at_100": 0.12554517947850538, + "mrr_at_1000": 0.12765268537614335, + "mrr_at_20": 0.1209542322873805, + "mrr_at_3": 0.10526315789473682, + "mrr_at_5": 0.11106337271750806, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2332453798292854, + "nauc_cv_recall_at_100_max": 0.2604533475265826, + "nauc_cv_recall_at_100_std": 0.34716025012947915, + "nauc_cv_recall_at_10_diff1": 0.35329370842952273, + "nauc_cv_recall_at_10_max": 0.3289408734013955, + "nauc_cv_recall_at_10_std": 0.38988109866697873, + "nauc_cv_recall_at_1_diff1": 0.6484509307804875, + "nauc_cv_recall_at_1_max": 0.4865030100451507, + "nauc_cv_recall_at_1_std": 0.42080120745590044, + "nauc_cv_recall_at_20_diff1": 0.3375684040590422, + "nauc_cv_recall_at_20_max": 0.3364013794750651, + "nauc_cv_recall_at_20_std": 0.3763071751508761, + "nauc_cv_recall_at_3_diff1": 0.5356708174885437, + "nauc_cv_recall_at_3_max": 0.3783391728916529, + "nauc_cv_recall_at_3_std": 0.3777211044732277, + "nauc_cv_recall_at_5_diff1": 0.47021139762415187, + "nauc_cv_recall_at_5_max": 0.3420074129863515, + "nauc_cv_recall_at_5_std": 0.36368502197235486, + "nauc_map_at_1000_diff1": 0.5397229127932685, + "nauc_map_at_1000_max": 0.414317285718928, + "nauc_map_at_1000_std": 0.39325018888031077, + "nauc_map_at_100_diff1": 0.5382353593792347, + "nauc_map_at_100_max": 0.41361154226841035, + "nauc_map_at_100_std": 0.39287405926700186, + "nauc_map_at_10_diff1": 0.5494054617283636, + "nauc_map_at_10_max": 0.4169379207559909, + "nauc_map_at_10_std": 0.3960601530511953, + "nauc_map_at_1_diff1": 0.6484509307804875, + "nauc_map_at_1_max": 0.4865030100451507, + "nauc_map_at_1_std": 0.42080120745590044, + "nauc_map_at_20_diff1": 0.5453892496407874, + "nauc_map_at_20_max": 0.41732748869963604, + "nauc_map_at_20_std": 0.39426197338787017, + "nauc_map_at_3_diff1": 0.5951395368065661, + "nauc_map_at_3_max": 0.4360903241739117, + "nauc_map_at_3_std": 0.3991134950739887, + "nauc_map_at_5_diff1": 0.5750651443076629, + "nauc_map_at_5_max": 0.42288294013194566, + "nauc_map_at_5_std": 0.39387458453693197, + "nauc_mrr_at_1000_diff1": 0.5397252229441754, + "nauc_mrr_at_1000_max": 0.4143152370673888, + "nauc_mrr_at_1000_std": 0.39325041574122255, + "nauc_mrr_at_100_diff1": 0.5382376576035895, + "nauc_mrr_at_100_max": 0.4136095049464029, + "nauc_mrr_at_100_std": 0.39287428504225475, + "nauc_mrr_at_10_diff1": 0.5494054617283636, + "nauc_mrr_at_10_max": 0.4169379207559909, + "nauc_mrr_at_10_std": 0.3960601530511953, + "nauc_mrr_at_1_diff1": 0.6484509307804875, + "nauc_mrr_at_1_max": 0.4865030100451507, + "nauc_mrr_at_1_std": 0.42080120745590044, + "nauc_mrr_at_20_diff1": 0.5453892496407874, + "nauc_mrr_at_20_max": 0.41732748869963604, + "nauc_mrr_at_20_std": 0.39426197338787017, + "nauc_mrr_at_3_diff1": 0.5951395368065661, + "nauc_mrr_at_3_max": 0.4360903241739117, + "nauc_mrr_at_3_std": 0.3991134950739887, + "nauc_mrr_at_5_diff1": 0.5750651443076629, + "nauc_mrr_at_5_max": 0.42288294013194566, + "nauc_mrr_at_5_std": 0.39387458453693197, + "nauc_ndcg_at_1000_diff1": 0.4877079024580914, + "nauc_ndcg_at_1000_max": 0.3894770176532519, + "nauc_ndcg_at_1000_std": 0.3866618739044846, + "nauc_ndcg_at_100_diff1": 0.4421249294826701, + "nauc_ndcg_at_100_max": 0.3665441486565287, + "nauc_ndcg_at_100_std": 0.3789855079726725, + "nauc_ndcg_at_10_diff1": 0.48884465811326916, + "nauc_ndcg_at_10_max": 0.38826077672968173, + "nauc_ndcg_at_10_std": 0.39280940785655283, + "nauc_ndcg_at_1_diff1": 0.6484509307804875, + "nauc_ndcg_at_1_max": 0.4865030100451507, + "nauc_ndcg_at_1_std": 0.42080120745590044, + "nauc_ndcg_at_20_diff1": 0.47829172426006694, + "nauc_ndcg_at_20_max": 0.3894391246306693, + "nauc_ndcg_at_20_std": 0.3875491838687217, + "nauc_ndcg_at_3_diff1": 0.5783208835318133, + "nauc_ndcg_at_3_max": 0.41978489706071076, + "nauc_ndcg_at_3_std": 0.39301265794434825, + "nauc_ndcg_at_5_diff1": 0.5444484907251043, + "nauc_ndcg_at_5_max": 0.3989289367797176, + "nauc_ndcg_at_5_std": 0.38486046761887477, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.23324537982928592, + "nauc_precision_at_100_max": 0.260453347526583, + "nauc_precision_at_100_std": 0.3471602501294796, + "nauc_precision_at_10_diff1": 0.3532937084295228, + "nauc_precision_at_10_max": 0.3289408734013955, + "nauc_precision_at_10_std": 0.3898810986669788, + "nauc_precision_at_1_diff1": 0.6484509307804875, + "nauc_precision_at_1_max": 0.4865030100451507, + "nauc_precision_at_1_std": 0.42080120745590044, + "nauc_precision_at_20_diff1": 0.33756840405904204, + "nauc_precision_at_20_max": 0.33640137947506477, + "nauc_precision_at_20_std": 0.3763071751508759, + "nauc_precision_at_3_diff1": 0.5356708174885437, + "nauc_precision_at_3_max": 0.37833917289165303, + "nauc_precision_at_3_std": 0.37772110447322776, + "nauc_precision_at_5_diff1": 0.4702113976241517, + "nauc_precision_at_5_max": 0.34200741298635134, + "nauc_precision_at_5_std": 0.36368502197235497, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2332453798292854, + "nauc_recall_at_100_max": 0.2604533475265826, + "nauc_recall_at_100_std": 0.34716025012947915, + "nauc_recall_at_10_diff1": 0.35329370842952273, + "nauc_recall_at_10_max": 0.3289408734013955, + "nauc_recall_at_10_std": 0.38988109866697873, + "nauc_recall_at_1_diff1": 0.6484509307804875, + "nauc_recall_at_1_max": 0.4865030100451507, + "nauc_recall_at_1_std": 0.42080120745590044, + "nauc_recall_at_20_diff1": 0.3375684040590422, + "nauc_recall_at_20_max": 0.3364013794750651, + "nauc_recall_at_20_std": 0.3763071751508761, + "nauc_recall_at_3_diff1": 0.5356708174885437, + "nauc_recall_at_3_max": 0.3783391728916529, + "nauc_recall_at_3_std": 0.3777211044732277, + "nauc_recall_at_5_diff1": 0.47021139762415187, + "nauc_recall_at_5_max": 0.3420074129863515, + "nauc_recall_at_5_std": 0.36368502197235486, + "ndcg_at_1": 0.09023, + "ndcg_at_10": 0.13653, + "ndcg_at_100": 0.18638, + "ndcg_at_1000": 0.25295, + "ndcg_at_20": 0.1504, + "ndcg_at_3": 0.11074, + "ndcg_at_5": 0.12127, + "precision_at_1": 0.09023, + "precision_at_10": 0.02009, + "precision_at_100": 0.00455, + "precision_at_1000": 0.001, + "precision_at_20": 0.01278, + "precision_at_3": 0.04225, + "precision_at_5": 0.0305, + "recall_at_1": 0.09023, + "recall_at_10": 0.20086, + "recall_at_100": 0.45542, + "recall_at_1000": 1.0, + "recall_at_20": 0.25564, + "recall_at_3": 0.12675, + "recall_at_5": 0.15252 + }, + { + "cv_recall_at_1": 0.09, + "cv_recall_at_10": 0.241, + "cv_recall_at_100": 0.572, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.298, + "cv_recall_at_3": 0.142, + "cv_recall_at_5": 0.184, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.15616, + "map_at_1": 0.09, + "map_at_10": 0.13016, + "map_at_100": 0.14012, + "map_at_1000": 0.14196, + "map_at_20": 0.1341, + "map_at_3": 0.11267, + "map_at_5": 0.12262, + "mrr_at_1": 0.09, + "mrr_at_10": 0.130156746031746, + "mrr_at_100": 0.14011933620958256, + "mrr_at_1000": 0.14195799808904563, + "mrr_at_20": 0.1341021033447118, + "mrr_at_3": 0.11266666666666662, + "mrr_at_5": 0.12261666666666665, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.09950724604329818, + "nauc_cv_recall_at_100_max": 0.19202302395020154, + "nauc_cv_recall_at_100_std": 0.29821107013617665, + "nauc_cv_recall_at_10_diff1": 0.2609484113161116, + "nauc_cv_recall_at_10_max": 0.04742804090975, + "nauc_cv_recall_at_10_std": 0.16593983737163379, + "nauc_cv_recall_at_1_diff1": 0.519786542646752, + "nauc_cv_recall_at_1_max": 0.06741794115016544, + "nauc_cv_recall_at_1_std": 0.1665653042363534, + "nauc_cv_recall_at_20_diff1": 0.2257939014702732, + "nauc_cv_recall_at_20_max": 0.06763853455264178, + "nauc_cv_recall_at_20_std": 0.18127495980674707, + "nauc_cv_recall_at_3_diff1": 0.43499246295510946, + "nauc_cv_recall_at_3_max": 0.01803852080770247, + "nauc_cv_recall_at_3_std": 0.23359590522218548, + "nauc_cv_recall_at_5_diff1": 0.3353192120401153, + "nauc_cv_recall_at_5_max": 0.021990660909630255, + "nauc_cv_recall_at_5_std": 0.1779186381757737, + "nauc_map_at_1000_diff1": 0.4137939221823387, + "nauc_map_at_1000_max": 0.04585110095108629, + "nauc_map_at_1000_std": 0.1905566893465737, + "nauc_map_at_100_diff1": 0.41218399729561006, + "nauc_map_at_100_max": 0.04606128131195546, + "nauc_map_at_100_std": 0.1902874373233265, + "nauc_map_at_10_diff1": 0.42234901698531985, + "nauc_map_at_10_max": 0.040903992320474654, + "nauc_map_at_10_std": 0.18605702839308597, + "nauc_map_at_1_diff1": 0.519786542646752, + "nauc_map_at_1_max": 0.06741794115016544, + "nauc_map_at_1_std": 0.1665653042363534, + "nauc_map_at_20_diff1": 0.41690209448121784, + "nauc_map_at_20_max": 0.042671874195198585, + "nauc_map_at_20_std": 0.1869251620123988, + "nauc_map_at_3_diff1": 0.47577635534215285, + "nauc_map_at_3_max": 0.03642620873031971, + "nauc_map_at_3_std": 0.20504951607197658, + "nauc_map_at_5_diff1": 0.44141461540766325, + "nauc_map_at_5_max": 0.03624088835029317, + "nauc_map_at_5_std": 0.1888593111268919, + "nauc_mrr_at_1000_diff1": 0.4137939221823387, + "nauc_mrr_at_1000_max": 0.04585110095108629, + "nauc_mrr_at_1000_std": 0.1905566893465737, + "nauc_mrr_at_100_diff1": 0.41218399729561006, + "nauc_mrr_at_100_max": 0.04606128131195546, + "nauc_mrr_at_100_std": 0.1902874373233265, + "nauc_mrr_at_10_diff1": 0.42234901698531985, + "nauc_mrr_at_10_max": 0.040903992320474654, + "nauc_mrr_at_10_std": 0.18605702839308597, + "nauc_mrr_at_1_diff1": 0.519786542646752, + "nauc_mrr_at_1_max": 0.06741794115016544, + "nauc_mrr_at_1_std": 0.1665653042363534, + "nauc_mrr_at_20_diff1": 0.41690209448121784, + "nauc_mrr_at_20_max": 0.042671874195198585, + "nauc_mrr_at_20_std": 0.1869251620123988, + "nauc_mrr_at_3_diff1": 0.47577635534215285, + "nauc_mrr_at_3_max": 0.03642620873031971, + "nauc_mrr_at_3_std": 0.20504951607197658, + "nauc_mrr_at_5_diff1": 0.44141461540766325, + "nauc_mrr_at_5_max": 0.03624088835029317, + "nauc_mrr_at_5_std": 0.1888593111268919, + "nauc_ndcg_at_1000_diff1": 0.36875630031096535, + "nauc_ndcg_at_1000_max": 0.0586483601817311, + "nauc_ndcg_at_1000_std": 0.2007283154206796, + "nauc_ndcg_at_100_diff1": 0.3245616830412682, + "nauc_ndcg_at_100_max": 0.07781147437348998, + "nauc_ndcg_at_100_std": 0.21267434981201533, + "nauc_ndcg_at_10_diff1": 0.37109903302118974, + "nauc_ndcg_at_10_max": 0.04192896271586187, + "nauc_ndcg_at_10_std": 0.18049054255252522, + "nauc_ndcg_at_1_diff1": 0.519786542646752, + "nauc_ndcg_at_1_max": 0.06741794115016544, + "nauc_ndcg_at_1_std": 0.1665653042363534, + "nauc_ndcg_at_20_diff1": 0.3548551379281221, + "nauc_ndcg_at_20_max": 0.04882258202592032, + "nauc_ndcg_at_20_std": 0.18487017063013111, + "nauc_ndcg_at_3_diff1": 0.46386739885940526, + "nauc_ndcg_at_3_max": 0.030844689867467245, + "nauc_ndcg_at_3_std": 0.21353132426323734, + "nauc_ndcg_at_5_diff1": 0.4089763993837865, + "nauc_ndcg_at_5_max": 0.03139894253778136, + "nauc_ndcg_at_5_std": 0.1862933242082388, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.09950724604329804, + "nauc_precision_at_100_max": 0.1920230239502011, + "nauc_precision_at_100_std": 0.29821107013617626, + "nauc_precision_at_10_diff1": 0.26094841131611163, + "nauc_precision_at_10_max": 0.047428040909750106, + "nauc_precision_at_10_std": 0.16593983737163379, + "nauc_precision_at_1_diff1": 0.519786542646752, + "nauc_precision_at_1_max": 0.06741794115016544, + "nauc_precision_at_1_std": 0.1665653042363534, + "nauc_precision_at_20_diff1": 0.22579390147027317, + "nauc_precision_at_20_max": 0.06763853455264163, + "nauc_precision_at_20_std": 0.18127495980674682, + "nauc_precision_at_3_diff1": 0.43499246295510957, + "nauc_precision_at_3_max": 0.01803852080770258, + "nauc_precision_at_3_std": 0.23359590522218557, + "nauc_precision_at_5_diff1": 0.33531921204011533, + "nauc_precision_at_5_max": 0.021990660909630016, + "nauc_precision_at_5_std": 0.17791863817577347, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.09950724604329818, + "nauc_recall_at_100_max": 0.19202302395020154, + "nauc_recall_at_100_std": 0.29821107013617665, + "nauc_recall_at_10_diff1": 0.2609484113161116, + "nauc_recall_at_10_max": 0.04742804090975, + "nauc_recall_at_10_std": 0.16593983737163379, + "nauc_recall_at_1_diff1": 0.519786542646752, + "nauc_recall_at_1_max": 0.06741794115016544, + "nauc_recall_at_1_std": 0.1665653042363534, + "nauc_recall_at_20_diff1": 0.2257939014702732, + "nauc_recall_at_20_max": 0.06763853455264178, + "nauc_recall_at_20_std": 0.18127495980674707, + "nauc_recall_at_3_diff1": 0.43499246295510946, + "nauc_recall_at_3_max": 0.01803852080770247, + "nauc_recall_at_3_std": 0.23359590522218548, + "nauc_recall_at_5_diff1": 0.3353192120401153, + "nauc_recall_at_5_max": 0.021990660909630255, + "nauc_recall_at_5_std": 0.1779186381757737, + "ndcg_at_1": 0.09, + "ndcg_at_10": 0.15616, + "ndcg_at_100": 0.21938, + "ndcg_at_1000": 0.27284, + "ndcg_at_20": 0.17056, + "ndcg_at_3": 0.12019, + "ndcg_at_5": 0.1378, + "precision_at_1": 0.09, + "precision_at_10": 0.0241, + "precision_at_100": 0.00572, + "precision_at_1000": 0.001, + "precision_at_20": 0.0149, + "precision_at_3": 0.04733, + "precision_at_5": 0.0368, + "recall_at_1": 0.09, + "recall_at_10": 0.241, + "recall_at_100": 0.572, + "recall_at_1000": 1.0, + "recall_at_20": 0.298, + "recall_at_3": 0.142, + "recall_at_5": 0.184 + }, + { + "cv_recall_at_1": 0.30513, + "cv_recall_at_10": 0.55895, + "cv_recall_at_100": 0.84605, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.63662, + "cv_recall_at_3": 0.43689, + "cv_recall_at_5": 0.50208, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.42702, + "map_at_1": 0.30513, + "map_at_10": 0.38526, + "map_at_100": 0.3958, + "map_at_1000": 0.39659, + "map_at_20": 0.39083, + "map_at_3": 0.36269, + "map_at_5": 0.37774, + "mrr_at_1": 0.30513176144244103, + "mrr_at_10": 0.38525691830130115, + "mrr_at_100": 0.39579860906812514, + "mrr_at_1000": 0.3965896152678169, + "mrr_at_20": 0.3908313698594654, + "mrr_at_3": 0.3626907073509017, + "mrr_at_5": 0.3777392510402221, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.41165495427230214, + "nauc_cv_recall_at_100_max": 0.48206221054316895, + "nauc_cv_recall_at_100_std": 0.6198649450287124, + "nauc_cv_recall_at_10_diff1": 0.4824079854591921, + "nauc_cv_recall_at_10_max": 0.5027473494385158, + "nauc_cv_recall_at_10_std": 0.48890187766681076, + "nauc_cv_recall_at_1_diff1": 0.644437799394293, + "nauc_cv_recall_at_1_max": 0.48720195113023645, + "nauc_cv_recall_at_1_std": 0.30433696117127595, + "nauc_cv_recall_at_20_diff1": 0.46158293543775514, + "nauc_cv_recall_at_20_max": 0.5333349260221991, + "nauc_cv_recall_at_20_std": 0.5400687456755257, + "nauc_cv_recall_at_3_diff1": 0.5168596861238599, + "nauc_cv_recall_at_3_max": 0.5076731821999175, + "nauc_cv_recall_at_3_std": 0.4135101558667384, + "nauc_cv_recall_at_5_diff1": 0.510299582925276, + "nauc_cv_recall_at_5_max": 0.5222048542887422, + "nauc_cv_recall_at_5_std": 0.4420289365796392, + "nauc_map_at_1000_diff1": 0.5830143383509978, + "nauc_map_at_1000_max": 0.5004906621407894, + "nauc_map_at_1000_std": 0.36274588989880185, + "nauc_map_at_100_diff1": 0.5825830804096754, + "nauc_map_at_100_max": 0.5003766495951647, + "nauc_map_at_100_std": 0.3632585756371978, + "nauc_map_at_10_diff1": 0.5827442300479906, + "nauc_map_at_10_max": 0.4989782918627815, + "nauc_map_at_10_std": 0.36149303848979514, + "nauc_map_at_1_diff1": 0.644437799394293, + "nauc_map_at_1_max": 0.48720195113023645, + "nauc_map_at_1_std": 0.30433696117127595, + "nauc_map_at_20_diff1": 0.582219628948973, + "nauc_map_at_20_max": 0.5010772738483961, + "nauc_map_at_20_std": 0.36377688498932803, + "nauc_map_at_3_diff1": 0.5862003610604175, + "nauc_map_at_3_max": 0.4962801376440491, + "nauc_map_at_3_std": 0.34912297998390074, + "nauc_map_at_5_diff1": 0.5861040029572455, + "nauc_map_at_5_max": 0.50086328296321, + "nauc_map_at_5_std": 0.3560349273683049, + "nauc_mrr_at_1000_diff1": 0.5830133060638379, + "nauc_mrr_at_1000_max": 0.5004875566119603, + "nauc_mrr_at_1000_std": 0.3627424647847617, + "nauc_mrr_at_100_diff1": 0.5825820719456357, + "nauc_mrr_at_100_max": 0.5003735001062519, + "nauc_mrr_at_100_std": 0.3632550948760236, + "nauc_mrr_at_10_diff1": 0.5827442300479906, + "nauc_mrr_at_10_max": 0.4989782918627815, + "nauc_mrr_at_10_std": 0.36149303848979514, + "nauc_mrr_at_1_diff1": 0.644437799394293, + "nauc_mrr_at_1_max": 0.48720195113023645, + "nauc_mrr_at_1_std": 0.30433696117127595, + "nauc_mrr_at_20_diff1": 0.582219628948973, + "nauc_mrr_at_20_max": 0.5010772738483961, + "nauc_mrr_at_20_std": 0.36377688498932803, + "nauc_mrr_at_3_diff1": 0.5862003610604175, + "nauc_mrr_at_3_max": 0.4962801376440491, + "nauc_mrr_at_3_std": 0.34912297998390074, + "nauc_mrr_at_5_diff1": 0.5861040029572455, + "nauc_mrr_at_5_max": 0.50086328296321, + "nauc_mrr_at_5_std": 0.3560349273683049, + "nauc_ndcg_at_1000_diff1": 0.5695323239109163, + "nauc_ndcg_at_1000_max": 0.5033127016866682, + "nauc_ndcg_at_1000_std": 0.3815003423814512, + "nauc_ndcg_at_100_diff1": 0.5594255087859172, + "nauc_ndcg_at_100_max": 0.5014926360153439, + "nauc_ndcg_at_100_std": 0.39584021920450774, + "nauc_ndcg_at_10_diff1": 0.5588656984073319, + "nauc_ndcg_at_10_max": 0.500693692375392, + "nauc_ndcg_at_10_std": 0.3910345980663399, + "nauc_ndcg_at_1_diff1": 0.644437799394293, + "nauc_ndcg_at_1_max": 0.48720195113023645, + "nauc_ndcg_at_1_std": 0.30433696117127595, + "nauc_ndcg_at_20_diff1": 0.5562572880254463, + "nauc_ndcg_at_20_max": 0.507832593764301, + "nauc_ndcg_at_20_std": 0.40000838616396234, + "nauc_ndcg_at_3_diff1": 0.5681528046470169, + "nauc_ndcg_at_3_max": 0.4992340623158375, + "nauc_ndcg_at_3_std": 0.3656656199525087, + "nauc_ndcg_at_5_diff1": 0.5668569119987612, + "nauc_ndcg_at_5_max": 0.5062895140454402, + "nauc_ndcg_at_5_std": 0.3775797241332556, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.4116549542723031, + "nauc_precision_at_100_max": 0.4820622105431704, + "nauc_precision_at_100_std": 0.6198649450287131, + "nauc_precision_at_10_diff1": 0.48240798545919267, + "nauc_precision_at_10_max": 0.5027473494385162, + "nauc_precision_at_10_std": 0.4889018776668112, + "nauc_precision_at_1_diff1": 0.644437799394293, + "nauc_precision_at_1_max": 0.48720195113023645, + "nauc_precision_at_1_std": 0.30433696117127595, + "nauc_precision_at_20_diff1": 0.4615829354377557, + "nauc_precision_at_20_max": 0.5333349260221996, + "nauc_precision_at_20_std": 0.5400687456755257, + "nauc_precision_at_3_diff1": 0.51685968612386, + "nauc_precision_at_3_max": 0.5076731821999171, + "nauc_precision_at_3_std": 0.4135101558667381, + "nauc_precision_at_5_diff1": 0.5102995829252761, + "nauc_precision_at_5_max": 0.5222048542887423, + "nauc_precision_at_5_std": 0.4420289365796393, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.41165495427230214, + "nauc_recall_at_100_max": 0.48206221054316895, + "nauc_recall_at_100_std": 0.6198649450287124, + "nauc_recall_at_10_diff1": 0.4824079854591921, + "nauc_recall_at_10_max": 0.5027473494385158, + "nauc_recall_at_10_std": 0.48890187766681076, + "nauc_recall_at_1_diff1": 0.644437799394293, + "nauc_recall_at_1_max": 0.48720195113023645, + "nauc_recall_at_1_std": 0.30433696117127595, + "nauc_recall_at_20_diff1": 0.46158293543775514, + "nauc_recall_at_20_max": 0.5333349260221991, + "nauc_recall_at_20_std": 0.5400687456755257, + "nauc_recall_at_3_diff1": 0.5168596861238599, + "nauc_recall_at_3_max": 0.5076731821999175, + "nauc_recall_at_3_std": 0.4135101558667384, + "nauc_recall_at_5_diff1": 0.510299582925276, + "nauc_recall_at_5_max": 0.5222048542887422, + "nauc_recall_at_5_std": 0.4420289365796392, + "ndcg_at_1": 0.30513, + "ndcg_at_10": 0.42702, + "ndcg_at_100": 0.48494, + "ndcg_at_1000": 0.50492, + "ndcg_at_20": 0.44689, + "ndcg_at_3": 0.38173, + "ndcg_at_5": 0.40871, + "precision_at_1": 0.30513, + "precision_at_10": 0.05589, + "precision_at_100": 0.00846, + "precision_at_1000": 0.001, + "precision_at_20": 0.03183, + "precision_at_3": 0.14563, + "precision_at_5": 0.10042, + "recall_at_1": 0.30513, + "recall_at_10": 0.55895, + "recall_at_100": 0.84605, + "recall_at_1000": 1.0, + "recall_at_20": 0.63662, + "recall_at_3": 0.43689, + "recall_at_5": 0.50208 + }, + { + "cv_recall_at_1": 0.30127, + "cv_recall_at_10": 0.53277, + "cv_recall_at_100": 0.80761, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.62474, + "cv_recall_at_3": 0.41121, + "cv_recall_at_5": 0.463, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.40991, + "map_at_1": 0.30127, + "map_at_10": 0.37141, + "map_at_100": 0.38227, + "map_at_1000": 0.38319, + "map_at_20": 0.37769, + "map_at_3": 0.34989, + "map_at_5": 0.36173, + "mrr_at_1": 0.3012684989429176, + "mrr_at_10": 0.3714067586160609, + "mrr_at_100": 0.38227085161316743, + "mrr_at_1000": 0.38319274292423033, + "mrr_at_20": 0.37768662546004295, + "mrr_at_3": 0.3498942917547569, + "mrr_at_5": 0.3617336152219871, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2862145489450746, + "nauc_cv_recall_at_100_max": 0.3375859781656063, + "nauc_cv_recall_at_100_std": 0.5712024116215759, + "nauc_cv_recall_at_10_diff1": 0.4501011642607513, + "nauc_cv_recall_at_10_max": 0.38903421251660564, + "nauc_cv_recall_at_10_std": 0.41458618776851075, + "nauc_cv_recall_at_1_diff1": 0.6199769209160442, + "nauc_cv_recall_at_1_max": 0.44486579497059164, + "nauc_cv_recall_at_1_std": 0.34961427339705375, + "nauc_cv_recall_at_20_diff1": 0.3905621048828626, + "nauc_cv_recall_at_20_max": 0.3598445963581698, + "nauc_cv_recall_at_20_std": 0.42593453843243684, + "nauc_cv_recall_at_3_diff1": 0.511281603222867, + "nauc_cv_recall_at_3_max": 0.4027393985883771, + "nauc_cv_recall_at_3_std": 0.3740329209080239, + "nauc_cv_recall_at_5_diff1": 0.4909711786318926, + "nauc_cv_recall_at_5_max": 0.40310060672872056, + "nauc_cv_recall_at_5_std": 0.40779921745492137, + "nauc_map_at_1000_diff1": 0.561306914268901, + "nauc_map_at_1000_max": 0.42384523950028463, + "nauc_map_at_1000_std": 0.37222062517463184, + "nauc_map_at_100_diff1": 0.560649765663532, + "nauc_map_at_100_max": 0.423551125644637, + "nauc_map_at_100_std": 0.3725384962597104, + "nauc_map_at_10_diff1": 0.5629278261027092, + "nauc_map_at_10_max": 0.42477770972747986, + "nauc_map_at_10_std": 0.37068421576959626, + "nauc_map_at_1_diff1": 0.6199769209160442, + "nauc_map_at_1_max": 0.44486579497059164, + "nauc_map_at_1_std": 0.34961427339705375, + "nauc_map_at_20_diff1": 0.5603607763457364, + "nauc_map_at_20_max": 0.42321621824595984, + "nauc_map_at_20_std": 0.3708469512400973, + "nauc_map_at_3_diff1": 0.5721797574981036, + "nauc_map_at_3_max": 0.4263739586019243, + "nauc_map_at_3_std": 0.36200070568332154, + "nauc_map_at_5_diff1": 0.5679047556570016, + "nauc_map_at_5_max": 0.42644946418412055, + "nauc_map_at_5_std": 0.36993011130441666, + "nauc_mrr_at_1000_diff1": 0.561306714388788, + "nauc_mrr_at_1000_max": 0.42384532845744227, + "nauc_mrr_at_1000_std": 0.3722207152690133, + "nauc_mrr_at_100_diff1": 0.5606496443087321, + "nauc_mrr_at_100_max": 0.4235514242888643, + "nauc_mrr_at_100_std": 0.3725388107550537, + "nauc_mrr_at_10_diff1": 0.5629278261027092, + "nauc_mrr_at_10_max": 0.42477770972747986, + "nauc_mrr_at_10_std": 0.37068421576959626, + "nauc_mrr_at_1_diff1": 0.6199769209160442, + "nauc_mrr_at_1_max": 0.44486579497059164, + "nauc_mrr_at_1_std": 0.34961427339705375, + "nauc_mrr_at_20_diff1": 0.5603607763457364, + "nauc_mrr_at_20_max": 0.42321621824595984, + "nauc_mrr_at_20_std": 0.3708469512400973, + "nauc_mrr_at_3_diff1": 0.5721797574981036, + "nauc_mrr_at_3_max": 0.4263739586019243, + "nauc_mrr_at_3_std": 0.36200070568332154, + "nauc_mrr_at_5_diff1": 0.5679047556570016, + "nauc_mrr_at_5_max": 0.42644946418412055, + "nauc_mrr_at_5_std": 0.36993011130441666, + "nauc_ndcg_at_1000_diff1": 0.5428204489189257, + "nauc_ndcg_at_1000_max": 0.4180339024617109, + "nauc_ndcg_at_1000_std": 0.38274231659073416, + "nauc_ndcg_at_100_diff1": 0.5243369526007065, + "nauc_ndcg_at_100_max": 0.411769907569649, + "nauc_ndcg_at_100_std": 0.39542375029587296, + "nauc_ndcg_at_10_diff1": 0.5361932596317941, + "nauc_ndcg_at_10_max": 0.4162362113983208, + "nauc_ndcg_at_10_std": 0.38113049766075247, + "nauc_ndcg_at_1_diff1": 0.6199769209160442, + "nauc_ndcg_at_1_max": 0.44486579497059164, + "nauc_ndcg_at_1_std": 0.34961427339705375, + "nauc_ndcg_at_20_diff1": 0.525438714155031, + "nauc_ndcg_at_20_max": 0.4103350023364189, + "nauc_ndcg_at_20_std": 0.38250427397382974, + "nauc_ndcg_at_3_diff1": 0.556324242979917, + "nauc_ndcg_at_3_max": 0.4202217818769986, + "nauc_ndcg_at_3_std": 0.36520139070196067, + "nauc_ndcg_at_5_diff1": 0.5483766172208181, + "nauc_ndcg_at_5_max": 0.4204092201127261, + "nauc_ndcg_at_5_std": 0.379301524104246, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.2862145489450751, + "nauc_precision_at_100_max": 0.33758597816560554, + "nauc_precision_at_100_std": 0.5712024116215759, + "nauc_precision_at_10_diff1": 0.4501011642607511, + "nauc_precision_at_10_max": 0.3890342125166057, + "nauc_precision_at_10_std": 0.4145861877685106, + "nauc_precision_at_1_diff1": 0.6199769209160442, + "nauc_precision_at_1_max": 0.44486579497059164, + "nauc_precision_at_1_std": 0.34961427339705375, + "nauc_precision_at_20_diff1": 0.39056210488286275, + "nauc_precision_at_20_max": 0.3598445963581694, + "nauc_precision_at_20_std": 0.42593453843243645, + "nauc_precision_at_3_diff1": 0.5112816032228666, + "nauc_precision_at_3_max": 0.40273939858837715, + "nauc_precision_at_3_std": 0.3740329209080242, + "nauc_precision_at_5_diff1": 0.4909711786318925, + "nauc_precision_at_5_max": 0.4031006067287203, + "nauc_precision_at_5_std": 0.4077992174549212, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2862145489450746, + "nauc_recall_at_100_max": 0.3375859781656063, + "nauc_recall_at_100_std": 0.5712024116215759, + "nauc_recall_at_10_diff1": 0.4501011642607513, + "nauc_recall_at_10_max": 0.38903421251660564, + "nauc_recall_at_10_std": 0.41458618776851075, + "nauc_recall_at_1_diff1": 0.6199769209160442, + "nauc_recall_at_1_max": 0.44486579497059164, + "nauc_recall_at_1_std": 0.34961427339705375, + "nauc_recall_at_20_diff1": 0.3905621048828626, + "nauc_recall_at_20_max": 0.3598445963581698, + "nauc_recall_at_20_std": 0.42593453843243684, + "nauc_recall_at_3_diff1": 0.511281603222867, + "nauc_recall_at_3_max": 0.4027393985883771, + "nauc_recall_at_3_std": 0.3740329209080239, + "nauc_recall_at_5_diff1": 0.4909711786318926, + "nauc_recall_at_5_max": 0.40310060672872056, + "nauc_recall_at_5_std": 0.40779921745492137, + "ndcg_at_1": 0.30127, + "ndcg_at_10": 0.40991, + "ndcg_at_100": 0.46679, + "ndcg_at_1000": 0.49132, + "ndcg_at_20": 0.43303, + "ndcg_at_3": 0.36565, + "ndcg_at_5": 0.38698, + "precision_at_1": 0.30127, + "precision_at_10": 0.05328, + "precision_at_100": 0.00808, + "precision_at_1000": 0.001, + "precision_at_20": 0.03124, + "precision_at_3": 0.13707, + "precision_at_5": 0.0926, + "recall_at_1": 0.30127, + "recall_at_10": 0.53277, + "recall_at_100": 0.80761, + "recall_at_1000": 1.0, + "recall_at_20": 0.62474, + "recall_at_3": 0.41121, + "recall_at_5": 0.463 + }, + { + "cv_recall_at_1": 0.474, + "cv_recall_at_10": 0.773, + "cv_recall_at_100": 0.973, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.846, + "cv_recall_at_3": 0.627, + "cv_recall_at_5": 0.697, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.61597, + "map_at_1": 0.474, + "map_at_10": 0.56653, + "map_at_100": 0.57507, + "map_at_1000": 0.57521, + "map_at_20": 0.57157, + "map_at_3": 0.5405, + "map_at_5": 0.55655, + "mrr_at_1": 0.474, + "mrr_at_10": 0.5665349206349206, + "mrr_at_100": 0.5750684463503862, + "mrr_at_1000": 0.5752147141574319, + "mrr_at_20": 0.5715710057443414, + "mrr_at_3": 0.5404999999999998, + "mrr_at_5": 0.55655, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.45353944046754274, + "nauc_cv_recall_at_100_max": 0.7559912854030483, + "nauc_cv_recall_at_100_std": 0.7699623059100126, + "nauc_cv_recall_at_10_diff1": 0.7005081767378533, + "nauc_cv_recall_at_10_max": 0.20818058253744653, + "nauc_cv_recall_at_10_std": -0.28630003460215453, + "nauc_cv_recall_at_1_diff1": 0.7659842765053384, + "nauc_cv_recall_at_1_max": 0.323405137468343, + "nauc_cv_recall_at_1_std": -0.06912237336565674, + "nauc_cv_recall_at_20_diff1": 0.6957153024006514, + "nauc_cv_recall_at_20_max": 0.24541855706787719, + "nauc_cv_recall_at_20_std": -0.21536912069337907, + "nauc_cv_recall_at_3_diff1": 0.6735930290720233, + "nauc_cv_recall_at_3_max": 0.2844436549378181, + "nauc_cv_recall_at_3_std": -0.14239634998620962, + "nauc_cv_recall_at_5_diff1": 0.678067736963919, + "nauc_cv_recall_at_5_max": 0.28032026577835256, + "nauc_cv_recall_at_5_std": -0.18294278641251102, + "nauc_map_at_1000_diff1": 0.7349768037519908, + "nauc_map_at_1000_max": 0.3108191319770147, + "nauc_map_at_1000_std": -0.09577460850617198, + "nauc_map_at_100_diff1": 0.7348596383487171, + "nauc_map_at_100_max": 0.3110408807336249, + "nauc_map_at_100_std": -0.0953690667160771, + "nauc_map_at_10_diff1": 0.7348528563701027, + "nauc_map_at_10_max": 0.3060472302583592, + "nauc_map_at_10_std": -0.10453879471869904, + "nauc_map_at_1_diff1": 0.7659842765053384, + "nauc_map_at_1_max": 0.323405137468343, + "nauc_map_at_1_std": -0.06912237336565674, + "nauc_map_at_20_diff1": 0.7349672844338018, + "nauc_map_at_20_max": 0.30896979717042244, + "nauc_map_at_20_std": -0.0993943729562651, + "nauc_map_at_3_diff1": 0.7293407859255343, + "nauc_map_at_3_max": 0.31154340909320627, + "nauc_map_at_3_std": -0.09141294877318266, + "nauc_map_at_5_diff1": 0.7315184827912649, + "nauc_map_at_5_max": 0.31110890999241614, + "nauc_map_at_5_std": -0.09830711349311756, + "nauc_mrr_at_1000_diff1": 0.7349768037519908, + "nauc_mrr_at_1000_max": 0.3108191319770147, + "nauc_mrr_at_1000_std": -0.09577460850617198, + "nauc_mrr_at_100_diff1": 0.7348596383487171, + "nauc_mrr_at_100_max": 0.3110408807336249, + "nauc_mrr_at_100_std": -0.0953690667160771, + "nauc_mrr_at_10_diff1": 0.7348528563701027, + "nauc_mrr_at_10_max": 0.3060472302583592, + "nauc_mrr_at_10_std": -0.10453879471869904, + "nauc_mrr_at_1_diff1": 0.7659842765053384, + "nauc_mrr_at_1_max": 0.323405137468343, + "nauc_mrr_at_1_std": -0.06912237336565674, + "nauc_mrr_at_20_diff1": 0.7349672844338018, + "nauc_mrr_at_20_max": 0.30896979717042244, + "nauc_mrr_at_20_std": -0.0993943729562651, + "nauc_mrr_at_3_diff1": 0.7293407859255343, + "nauc_mrr_at_3_max": 0.31154340909320627, + "nauc_mrr_at_3_std": -0.09141294877318266, + "nauc_mrr_at_5_diff1": 0.7315184827912649, + "nauc_mrr_at_5_max": 0.31110890999241614, + "nauc_mrr_at_5_std": -0.09830711349311756, + "nauc_ndcg_at_1000_diff1": 0.7300106781628969, + "nauc_ndcg_at_1000_max": 0.30946240469434727, + "nauc_ndcg_at_1000_std": -0.09855705821637864, + "nauc_ndcg_at_100_diff1": 0.726228776286605, + "nauc_ndcg_at_100_max": 0.31581736941064814, + "nauc_ndcg_at_100_std": -0.0863659221103867, + "nauc_ndcg_at_10_diff1": 0.7275424860578261, + "nauc_ndcg_at_10_max": 0.2899275846092146, + "nauc_ndcg_at_10_std": -0.1349627955933614, + "nauc_ndcg_at_1_diff1": 0.7659842765053384, + "nauc_ndcg_at_1_max": 0.323405137468343, + "nauc_ndcg_at_1_std": -0.06912237336565674, + "nauc_ndcg_at_20_diff1": 0.7281192602738402, + "nauc_ndcg_at_20_max": 0.30002130037590796, + "nauc_ndcg_at_20_std": -0.11647828361837598, + "nauc_ndcg_at_3_diff1": 0.7160762699063752, + "nauc_ndcg_at_3_max": 0.30524984633027247, + "nauc_ndcg_at_3_std": -0.10325252208043191, + "nauc_ndcg_at_5_diff1": 0.7196819527437018, + "nauc_ndcg_at_5_max": 0.30450554939861313, + "nauc_ndcg_at_5_std": -0.11613402673386349, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.45353944046753836, + "nauc_precision_at_100_max": 0.7559912854030467, + "nauc_precision_at_100_std": 0.7699623059100184, + "nauc_precision_at_10_diff1": 0.7005081767378538, + "nauc_precision_at_10_max": 0.2081805825374449, + "nauc_precision_at_10_std": -0.2863000346021545, + "nauc_precision_at_1_diff1": 0.7659842765053384, + "nauc_precision_at_1_max": 0.323405137468343, + "nauc_precision_at_1_std": -0.06912237336565674, + "nauc_precision_at_20_diff1": 0.6957153024006515, + "nauc_precision_at_20_max": 0.24541855706787566, + "nauc_precision_at_20_std": -0.2153691206933822, + "nauc_precision_at_3_diff1": 0.6735930290720236, + "nauc_precision_at_3_max": 0.28444365493781915, + "nauc_precision_at_3_std": -0.14239634998620926, + "nauc_precision_at_5_diff1": 0.6780677369639184, + "nauc_precision_at_5_max": 0.28032026577835195, + "nauc_precision_at_5_std": -0.18294278641251163, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.45353944046754274, + "nauc_recall_at_100_max": 0.7559912854030483, + "nauc_recall_at_100_std": 0.7699623059100126, + "nauc_recall_at_10_diff1": 0.7005081767378533, + "nauc_recall_at_10_max": 0.20818058253744653, + "nauc_recall_at_10_std": -0.28630003460215453, + "nauc_recall_at_1_diff1": 0.7659842765053384, + "nauc_recall_at_1_max": 0.323405137468343, + "nauc_recall_at_1_std": -0.06912237336565674, + "nauc_recall_at_20_diff1": 0.6957153024006514, + "nauc_recall_at_20_max": 0.24541855706787719, + "nauc_recall_at_20_std": -0.21536912069337907, + "nauc_recall_at_3_diff1": 0.6735930290720233, + "nauc_recall_at_3_max": 0.2844436549378181, + "nauc_recall_at_3_std": -0.14239634998620962, + "nauc_recall_at_5_diff1": 0.678067736963919, + "nauc_recall_at_5_max": 0.28032026577835256, + "nauc_recall_at_5_std": -0.18294278641251102, + "ndcg_at_1": 0.474, + "ndcg_at_10": 0.61597, + "ndcg_at_100": 0.65853, + "ndcg_at_1000": 0.66207, + "ndcg_at_20": 0.6344, + "ndcg_at_3": 0.56268, + "ndcg_at_5": 0.59155, + "precision_at_1": 0.474, + "precision_at_10": 0.0773, + "precision_at_100": 0.00973, + "precision_at_1000": 0.001, + "precision_at_20": 0.0423, + "precision_at_3": 0.209, + "precision_at_5": 0.1394, + "recall_at_1": 0.474, + "recall_at_10": 0.773, + "recall_at_100": 0.973, + "recall_at_1000": 1.0, + "recall_at_20": 0.846, + "recall_at_3": 0.627, + "recall_at_5": 0.697 + } + ] + }, + "task_name": "WITT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/model_meta.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/model_meta.json new file mode 100644 index 0000000000..e2f5452dbe --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/model_meta.json @@ -0,0 +1 @@ +{"name": "google/siglip-base-patch16-256", "revision": "b078df89e446d623010d890864d4207fe6399f61", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/WITT2IRetrieval.json b/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/WITT2IRetrieval.json new file mode 100644 index 0000000000..112c157a17 --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/WITT2IRetrieval.json @@ -0,0 +1,1936 @@ +{ + "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", + "evaluation_time": 137.17432284355164, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.13933, + "cv_recall_at_10": 0.33596, + "cv_recall_at_100": 0.65056, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.4, + "cv_recall_at_3": 0.20112, + "cv_recall_at_5": 0.25281, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.22251, + "map_at_1": 0.13933, + "map_at_10": 0.18821, + "map_at_100": 0.19874, + "map_at_1000": 0.20028, + "map_at_20": 0.19266, + "map_at_3": 0.16592, + "map_at_5": 0.17749, + "mrr_at_1": 0.1393258426966292, + "mrr_at_10": 0.1882062600321027, + "mrr_at_100": 0.19873749728328727, + "mrr_at_1000": 0.2002817481555001, + "mrr_at_20": 0.19265773240009676, + "mrr_at_3": 0.16591760299625463, + "mrr_at_5": 0.17749063670411983, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2979879802367704, + "nauc_cv_recall_at_100_max": 0.4739733500284659, + "nauc_cv_recall_at_100_std": 0.46487157379083804, + "nauc_cv_recall_at_10_diff1": 0.3857208699452441, + "nauc_cv_recall_at_10_max": 0.41350589842683755, + "nauc_cv_recall_at_10_std": 0.3503705814003786, + "nauc_cv_recall_at_1_diff1": 0.6313007118799755, + "nauc_cv_recall_at_1_max": 0.5155341013472138, + "nauc_cv_recall_at_1_std": 0.35458607624004906, + "nauc_cv_recall_at_20_diff1": 0.382239438086532, + "nauc_cv_recall_at_20_max": 0.39601968317152486, + "nauc_cv_recall_at_20_std": 0.3440345873014037, + "nauc_cv_recall_at_3_diff1": 0.4801863620484621, + "nauc_cv_recall_at_3_max": 0.4433085417748924, + "nauc_cv_recall_at_3_std": 0.3463988285400372, + "nauc_cv_recall_at_5_diff1": 0.39071373942219917, + "nauc_cv_recall_at_5_max": 0.3956512800269739, + "nauc_cv_recall_at_5_std": 0.36737802120083213, + "nauc_map_at_1000_diff1": 0.5211586969479306, + "nauc_map_at_1000_max": 0.4620597681593909, + "nauc_map_at_1000_std": 0.3518606509995756, + "nauc_map_at_100_diff1": 0.5199563889214931, + "nauc_map_at_100_max": 0.4617893581547442, + "nauc_map_at_100_std": 0.35187670213120303, + "nauc_map_at_10_diff1": 0.5228671435456008, + "nauc_map_at_10_max": 0.4627408394245875, + "nauc_map_at_10_std": 0.34995955033850235, + "nauc_map_at_1_diff1": 0.6313007118799755, + "nauc_map_at_1_max": 0.5155341013472138, + "nauc_map_at_1_std": 0.35458607624004906, + "nauc_map_at_20_diff1": 0.5224268312188151, + "nauc_map_at_20_max": 0.461155418295598, + "nauc_map_at_20_std": 0.3494356451137881, + "nauc_map_at_3_diff1": 0.5539130179035476, + "nauc_map_at_3_max": 0.4775300569101307, + "nauc_map_at_3_std": 0.34648304915615186, + "nauc_map_at_5_diff1": 0.527370251204279, + "nauc_map_at_5_max": 0.463098359674595, + "nauc_map_at_5_std": 0.3526771617701944, + "nauc_mrr_at_1000_diff1": 0.5211586936725224, + "nauc_mrr_at_1000_max": 0.46205976295168305, + "nauc_mrr_at_1000_std": 0.3518606434736402, + "nauc_mrr_at_100_diff1": 0.5199563889214931, + "nauc_mrr_at_100_max": 0.4617893581547442, + "nauc_mrr_at_100_std": 0.35187670213120303, + "nauc_mrr_at_10_diff1": 0.5228671435456008, + "nauc_mrr_at_10_max": 0.4627408394245875, + "nauc_mrr_at_10_std": 0.34995955033850235, + "nauc_mrr_at_1_diff1": 0.6313007118799755, + "nauc_mrr_at_1_max": 0.5155341013472138, + "nauc_mrr_at_1_std": 0.35458607624004906, + "nauc_mrr_at_20_diff1": 0.5224268312188151, + "nauc_mrr_at_20_max": 0.461155418295598, + "nauc_mrr_at_20_std": 0.3494356451137881, + "nauc_mrr_at_3_diff1": 0.5539130179035476, + "nauc_mrr_at_3_max": 0.4775300569101307, + "nauc_mrr_at_3_std": 0.34648304915615186, + "nauc_mrr_at_5_diff1": 0.527370251204279, + "nauc_mrr_at_5_max": 0.463098359674595, + "nauc_mrr_at_5_std": 0.3526771617701944, + "nauc_ndcg_at_1000_diff1": 0.4921495850103755, + "nauc_ndcg_at_1000_max": 0.4536351152047796, + "nauc_ndcg_at_1000_std": 0.3590542409723068, + "nauc_ndcg_at_100_diff1": 0.4658043926888635, + "nauc_ndcg_at_100_max": 0.45474316038725227, + "nauc_ndcg_at_100_std": 0.3702553940425927, + "nauc_ndcg_at_10_diff1": 0.48102408933081287, + "nauc_ndcg_at_10_max": 0.4467146470840002, + "nauc_ndcg_at_10_std": 0.3503084336712536, + "nauc_ndcg_at_1_diff1": 0.6313007118799755, + "nauc_ndcg_at_1_max": 0.5155341013472138, + "nauc_ndcg_at_1_std": 0.35458607624004906, + "nauc_ndcg_at_20_diff1": 0.47931001719588917, + "nauc_ndcg_at_20_max": 0.44104335869322414, + "nauc_ndcg_at_20_std": 0.34838234044212724, + "nauc_ndcg_at_3_diff1": 0.5327924389517208, + "nauc_ndcg_at_3_max": 0.4676848915957549, + "nauc_ndcg_at_3_std": 0.3462985691606815, + "nauc_ndcg_at_5_diff1": 0.4879105951367381, + "nauc_ndcg_at_5_max": 0.44358910787060024, + "nauc_ndcg_at_5_std": 0.3566285952669218, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.29798798023677125, + "nauc_precision_at_100_max": 0.47397335002846613, + "nauc_precision_at_100_std": 0.46487157379083816, + "nauc_precision_at_10_diff1": 0.3857208699452443, + "nauc_precision_at_10_max": 0.4135058984268377, + "nauc_precision_at_10_std": 0.3503705814003787, + "nauc_precision_at_1_diff1": 0.6313007118799755, + "nauc_precision_at_1_max": 0.5155341013472138, + "nauc_precision_at_1_std": 0.35458607624004906, + "nauc_precision_at_20_diff1": 0.38223943808653166, + "nauc_precision_at_20_max": 0.39601968317152453, + "nauc_precision_at_20_std": 0.3440345873014034, + "nauc_precision_at_3_diff1": 0.480186362048462, + "nauc_precision_at_3_max": 0.443308541774892, + "nauc_precision_at_3_std": 0.34639882854003723, + "nauc_precision_at_5_diff1": 0.3907137394221993, + "nauc_precision_at_5_max": 0.3956512800269739, + "nauc_precision_at_5_std": 0.3673780212008324, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2979879802367704, + "nauc_recall_at_100_max": 0.4739733500284659, + "nauc_recall_at_100_std": 0.46487157379083804, + "nauc_recall_at_10_diff1": 0.3857208699452441, + "nauc_recall_at_10_max": 0.41350589842683755, + "nauc_recall_at_10_std": 0.3503705814003786, + "nauc_recall_at_1_diff1": 0.6313007118799755, + "nauc_recall_at_1_max": 0.5155341013472138, + "nauc_recall_at_1_std": 0.35458607624004906, + "nauc_recall_at_20_diff1": 0.382239438086532, + "nauc_recall_at_20_max": 0.39601968317152486, + "nauc_recall_at_20_std": 0.3440345873014037, + "nauc_recall_at_3_diff1": 0.4801863620484621, + "nauc_recall_at_3_max": 0.4433085417748924, + "nauc_recall_at_3_std": 0.3463988285400372, + "nauc_recall_at_5_diff1": 0.39071373942219917, + "nauc_recall_at_5_max": 0.3956512800269739, + "nauc_recall_at_5_std": 0.36737802120083213, + "ndcg_at_1": 0.13933, + "ndcg_at_10": 0.22251, + "ndcg_at_100": 0.28451, + "ndcg_at_1000": 0.32833, + "ndcg_at_20": 0.23872, + "ndcg_at_3": 0.17493, + "ndcg_at_5": 0.19601, + "precision_at_1": 0.13933, + "precision_at_10": 0.0336, + "precision_at_100": 0.00651, + "precision_at_1000": 0.001, + "precision_at_20": 0.02, + "precision_at_3": 0.06704, + "precision_at_5": 0.05056, + "recall_at_1": 0.13933, + "recall_at_10": 0.33596, + "recall_at_100": 0.65056, + "recall_at_1000": 1.0, + "recall_at_20": 0.4, + "recall_at_3": 0.20112, + "recall_at_5": 0.25281 + }, + { + "cv_recall_at_1": 0.15233, + "cv_recall_at_10": 0.32558, + "cv_recall_at_100": 0.68488, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.42558, + "cv_recall_at_3": 0.20814, + "cv_recall_at_5": 0.25, + "hf_subset": "bg", + "languages": [ + "bul-Cyrl" + ], + "main_score": 0.22681, + "map_at_1": 0.15233, + "map_at_10": 0.19685, + "map_at_100": 0.20977, + "map_at_1000": 0.21137, + "map_at_20": 0.20388, + "map_at_3": 0.17733, + "map_at_5": 0.18698, + "mrr_at_1": 0.15232558139534882, + "mrr_at_10": 0.19685123661867845, + "mrr_at_100": 0.20976999322473525, + "mrr_at_1000": 0.21136818088041656, + "mrr_at_20": 0.20388234299720012, + "mrr_at_3": 0.17732558139534887, + "mrr_at_5": 0.1869767441860465, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.28928588096595664, + "nauc_cv_recall_at_100_max": 0.3545880113713616, + "nauc_cv_recall_at_100_std": 0.3624408248903762, + "nauc_cv_recall_at_10_diff1": 0.410329527119275, + "nauc_cv_recall_at_10_max": 0.16473563361052504, + "nauc_cv_recall_at_10_std": 0.08078999627652986, + "nauc_cv_recall_at_1_diff1": 0.6586457244685855, + "nauc_cv_recall_at_1_max": 0.2772832920468003, + "nauc_cv_recall_at_1_std": 0.12915926096719418, + "nauc_cv_recall_at_20_diff1": 0.3776209415608929, + "nauc_cv_recall_at_20_max": 0.18776254714901358, + "nauc_cv_recall_at_20_std": 0.14351872823746506, + "nauc_cv_recall_at_3_diff1": 0.5002744242173366, + "nauc_cv_recall_at_3_max": 0.22006604660961354, + "nauc_cv_recall_at_3_std": 0.12469577719340726, + "nauc_cv_recall_at_5_diff1": 0.46741021408349515, + "nauc_cv_recall_at_5_max": 0.19384178519804637, + "nauc_cv_recall_at_5_std": 0.09437635245516056, + "nauc_map_at_1000_diff1": 0.5513265197992123, + "nauc_map_at_1000_max": 0.24016450921419116, + "nauc_map_at_1000_std": 0.1263100552175581, + "nauc_map_at_100_diff1": 0.5503259849465996, + "nauc_map_at_100_max": 0.24015798693197335, + "nauc_map_at_100_std": 0.1266885245132762, + "nauc_map_at_10_diff1": 0.5541550509171621, + "nauc_map_at_10_max": 0.23351168479305368, + "nauc_map_at_10_std": 0.11489690071703798, + "nauc_map_at_1_diff1": 0.6586457244685855, + "nauc_map_at_1_max": 0.2772832920468003, + "nauc_map_at_1_std": 0.12915926096719418, + "nauc_map_at_20_diff1": 0.550752731386968, + "nauc_map_at_20_max": 0.2363155358781987, + "nauc_map_at_20_std": 0.12196053759746485, + "nauc_map_at_3_diff1": 0.5767075860995042, + "nauc_map_at_3_max": 0.24751741022380233, + "nauc_map_at_3_std": 0.12614812315624652, + "nauc_map_at_5_diff1": 0.5661607434146992, + "nauc_map_at_5_max": 0.23941193117024734, + "nauc_map_at_5_std": 0.11767569954794925, + "nauc_mrr_at_1000_diff1": 0.5513265093688596, + "nauc_mrr_at_1000_max": 0.2401645568612914, + "nauc_mrr_at_1000_std": 0.12631007104475458, + "nauc_mrr_at_100_diff1": 0.5503259849465996, + "nauc_mrr_at_100_max": 0.24015798693197335, + "nauc_mrr_at_100_std": 0.1266885245132762, + "nauc_mrr_at_10_diff1": 0.5541550509171621, + "nauc_mrr_at_10_max": 0.23351168479305368, + "nauc_mrr_at_10_std": 0.11489690071703798, + "nauc_mrr_at_1_diff1": 0.6586457244685855, + "nauc_mrr_at_1_max": 0.2772832920468003, + "nauc_mrr_at_1_std": 0.12915926096719418, + "nauc_mrr_at_20_diff1": 0.550752731386968, + "nauc_mrr_at_20_max": 0.2363155358781987, + "nauc_mrr_at_20_std": 0.12196053759746485, + "nauc_mrr_at_3_diff1": 0.5767075860995042, + "nauc_mrr_at_3_max": 0.24751741022380233, + "nauc_mrr_at_3_std": 0.12614812315624652, + "nauc_mrr_at_5_diff1": 0.5661607434146992, + "nauc_mrr_at_5_max": 0.23941193117024734, + "nauc_mrr_at_5_std": 0.11767569954794925, + "nauc_ndcg_at_1000_diff1": 0.5214665780441233, + "nauc_ndcg_at_1000_max": 0.24043521395830333, + "nauc_ndcg_at_1000_std": 0.13920790773883404, + "nauc_ndcg_at_100_diff1": 0.49414511319709525, + "nauc_ndcg_at_100_max": 0.25181424634361693, + "nauc_ndcg_at_100_std": 0.1629480668331953, + "nauc_ndcg_at_10_diff1": 0.5125254924923802, + "nauc_ndcg_at_10_max": 0.21379969221512746, + "nauc_ndcg_at_10_std": 0.10536356101011754, + "nauc_ndcg_at_1_diff1": 0.6586457244685855, + "nauc_ndcg_at_1_max": 0.2772832920468003, + "nauc_ndcg_at_1_std": 0.12915926096719418, + "nauc_ndcg_at_20_diff1": 0.5014840374569179, + "nauc_ndcg_at_20_max": 0.22153633427823136, + "nauc_ndcg_at_20_std": 0.1262308783663143, + "nauc_ndcg_at_3_diff1": 0.5552016979915128, + "nauc_ndcg_at_3_max": 0.23978651146760627, + "nauc_ndcg_at_3_std": 0.12570952954169726, + "nauc_ndcg_at_5_diff1": 0.537869708474716, + "nauc_ndcg_at_5_max": 0.22648568477623546, + "nauc_ndcg_at_5_std": 0.11132774920238546, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.28928588096595753, + "nauc_precision_at_100_max": 0.35458801137136126, + "nauc_precision_at_100_std": 0.36244082489037516, + "nauc_precision_at_10_diff1": 0.41032952711927495, + "nauc_precision_at_10_max": 0.16473563361052518, + "nauc_precision_at_10_std": 0.08078999627652979, + "nauc_precision_at_1_diff1": 0.6586457244685855, + "nauc_precision_at_1_max": 0.2772832920468003, + "nauc_precision_at_1_std": 0.12915926096719418, + "nauc_precision_at_20_diff1": 0.3776209415608924, + "nauc_precision_at_20_max": 0.1877625471490132, + "nauc_precision_at_20_std": 0.14351872823746498, + "nauc_precision_at_3_diff1": 0.5002744242173365, + "nauc_precision_at_3_max": 0.2200660466096136, + "nauc_precision_at_3_std": 0.12469577719340708, + "nauc_precision_at_5_diff1": 0.46741021408349537, + "nauc_precision_at_5_max": 0.19384178519804657, + "nauc_precision_at_5_std": 0.09437635245516063, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.28928588096595664, + "nauc_recall_at_100_max": 0.3545880113713616, + "nauc_recall_at_100_std": 0.3624408248903762, + "nauc_recall_at_10_diff1": 0.410329527119275, + "nauc_recall_at_10_max": 0.16473563361052504, + "nauc_recall_at_10_std": 0.08078999627652986, + "nauc_recall_at_1_diff1": 0.6586457244685855, + "nauc_recall_at_1_max": 0.2772832920468003, + "nauc_recall_at_1_std": 0.12915926096719418, + "nauc_recall_at_20_diff1": 0.3776209415608929, + "nauc_recall_at_20_max": 0.18776254714901358, + "nauc_recall_at_20_std": 0.14351872823746506, + "nauc_recall_at_3_diff1": 0.5002744242173366, + "nauc_recall_at_3_max": 0.22006604660961354, + "nauc_recall_at_3_std": 0.12469577719340726, + "nauc_recall_at_5_diff1": 0.46741021408349515, + "nauc_recall_at_5_max": 0.19384178519804637, + "nauc_recall_at_5_std": 0.09437635245516056, + "ndcg_at_1": 0.15233, + "ndcg_at_10": 0.22681, + "ndcg_at_100": 0.29875, + "ndcg_at_1000": 0.33929, + "ndcg_at_20": 0.2522, + "ndcg_at_3": 0.18526, + "ndcg_at_5": 0.20257, + "precision_at_1": 0.15233, + "precision_at_10": 0.03256, + "precision_at_100": 0.00685, + "precision_at_1000": 0.001, + "precision_at_20": 0.02128, + "precision_at_3": 0.06938, + "precision_at_5": 0.05, + "recall_at_1": 0.15233, + "recall_at_10": 0.32558, + "recall_at_100": 0.68488, + "recall_at_1000": 1.0, + "recall_at_20": 0.42558, + "recall_at_3": 0.20814, + "recall_at_5": 0.25 + }, + { + "cv_recall_at_1": 0.32884, + "cv_recall_at_10": 0.57688, + "cv_recall_at_100": 0.81594, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.65544, + "cv_recall_at_3": 0.43547, + "cv_recall_at_5": 0.49495, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.44096, + "map_at_1": 0.32884, + "map_at_10": 0.39885, + "map_at_100": 0.40826, + "map_at_1000": 0.40911, + "map_at_20": 0.40444, + "map_at_3": 0.37411, + "map_at_5": 0.38803, + "mrr_at_1": 0.3288439955106622, + "mrr_at_10": 0.3988509432953876, + "mrr_at_100": 0.40826340383916915, + "mrr_at_1000": 0.40911460678506656, + "mrr_at_20": 0.40444359872490476, + "mrr_at_3": 0.37411148522259624, + "mrr_at_5": 0.38802843247287666, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.42251653533438616, + "nauc_cv_recall_at_100_max": 0.5974632567614767, + "nauc_cv_recall_at_100_std": 0.5798983552509402, + "nauc_cv_recall_at_10_diff1": 0.4965565401407999, + "nauc_cv_recall_at_10_max": 0.4884093981133426, + "nauc_cv_recall_at_10_std": 0.3476667895125619, + "nauc_cv_recall_at_1_diff1": 0.6881197967738875, + "nauc_cv_recall_at_1_max": 0.5104387821019861, + "nauc_cv_recall_at_1_std": 0.22996512608256198, + "nauc_cv_recall_at_20_diff1": 0.4579569229337732, + "nauc_cv_recall_at_20_max": 0.5357361507438837, + "nauc_cv_recall_at_20_std": 0.41608588175822514, + "nauc_cv_recall_at_3_diff1": 0.5826564933802192, + "nauc_cv_recall_at_3_max": 0.4915891078838359, + "nauc_cv_recall_at_3_std": 0.26513885013921823, + "nauc_cv_recall_at_5_diff1": 0.5361007340951345, + "nauc_cv_recall_at_5_max": 0.46650003271004464, + "nauc_cv_recall_at_5_std": 0.29467856997063335, + "nauc_map_at_1000_diff1": 0.6260425875683999, + "nauc_map_at_1000_max": 0.5031965378036112, + "nauc_map_at_1000_std": 0.2599848007687964, + "nauc_map_at_100_diff1": 0.6257450554561577, + "nauc_map_at_100_max": 0.5033928040860026, + "nauc_map_at_100_std": 0.2606797179062218, + "nauc_map_at_10_diff1": 0.6265320761885688, + "nauc_map_at_10_max": 0.5003685246352216, + "nauc_map_at_10_std": 0.25713351432434206, + "nauc_map_at_1_diff1": 0.6881197967738875, + "nauc_map_at_1_max": 0.5104387821019861, + "nauc_map_at_1_std": 0.22996512608256198, + "nauc_map_at_20_diff1": 0.6251828068786616, + "nauc_map_at_20_max": 0.5032413273960014, + "nauc_map_at_20_std": 0.26008764400434875, + "nauc_map_at_3_diff1": 0.6404464664086379, + "nauc_map_at_3_max": 0.502375813650769, + "nauc_map_at_3_std": 0.24441336490772264, + "nauc_map_at_5_diff1": 0.6304197906675999, + "nauc_map_at_5_max": 0.49722842291628355, + "nauc_map_at_5_std": 0.2509909601892623, + "nauc_mrr_at_1000_diff1": 0.6260425875683999, + "nauc_mrr_at_1000_max": 0.5031965378036112, + "nauc_mrr_at_1000_std": 0.2599848007687964, + "nauc_mrr_at_100_diff1": 0.6257450554561577, + "nauc_mrr_at_100_max": 0.5033928040860026, + "nauc_mrr_at_100_std": 0.2606797179062218, + "nauc_mrr_at_10_diff1": 0.6265320761885688, + "nauc_mrr_at_10_max": 0.5003685246352216, + "nauc_mrr_at_10_std": 0.25713351432434206, + "nauc_mrr_at_1_diff1": 0.6881197967738875, + "nauc_mrr_at_1_max": 0.5104387821019861, + "nauc_mrr_at_1_std": 0.22996512608256198, + "nauc_mrr_at_20_diff1": 0.6251828068786616, + "nauc_mrr_at_20_max": 0.5032413273960014, + "nauc_mrr_at_20_std": 0.26008764400434875, + "nauc_mrr_at_3_diff1": 0.6404464664086379, + "nauc_mrr_at_3_max": 0.502375813650769, + "nauc_mrr_at_3_std": 0.24441336490772264, + "nauc_mrr_at_5_diff1": 0.6304197906675999, + "nauc_mrr_at_5_max": 0.49722842291628355, + "nauc_mrr_at_5_std": 0.2509909601892623, + "nauc_ndcg_at_1000_diff1": 0.6076429265545639, + "nauc_ndcg_at_1000_max": 0.5052332372999279, + "nauc_ndcg_at_1000_std": 0.27652683355425656, + "nauc_ndcg_at_100_diff1": 0.5957468733498283, + "nauc_ndcg_at_100_max": 0.5114361696173704, + "nauc_ndcg_at_100_std": 0.2972555019901694, + "nauc_ndcg_at_10_diff1": 0.5968390935433466, + "nauc_ndcg_at_10_max": 0.4972020381548397, + "nauc_ndcg_at_10_std": 0.276970819778273, + "nauc_ndcg_at_1_diff1": 0.6881197967738875, + "nauc_ndcg_at_1_max": 0.5104387821019861, + "nauc_ndcg_at_1_std": 0.22996512608256198, + "nauc_ndcg_at_20_diff1": 0.5912904398741408, + "nauc_ndcg_at_20_max": 0.5077719844454441, + "nauc_ndcg_at_20_std": 0.2893097949822134, + "nauc_ndcg_at_3_diff1": 0.6254713111075045, + "nauc_ndcg_at_3_max": 0.49960017452239175, + "nauc_ndcg_at_3_std": 0.24972205479769516, + "nauc_ndcg_at_5_diff1": 0.6069956168160815, + "nauc_ndcg_at_5_max": 0.48982469778813303, + "nauc_ndcg_at_5_std": 0.2616678203416795, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.42251653533438577, + "nauc_precision_at_100_max": 0.5974632567614774, + "nauc_precision_at_100_std": 0.5798983552509408, + "nauc_precision_at_10_diff1": 0.49655654014079986, + "nauc_precision_at_10_max": 0.488409398113342, + "nauc_precision_at_10_std": 0.3476667895125617, + "nauc_precision_at_1_diff1": 0.6881197967738875, + "nauc_precision_at_1_max": 0.5104387821019861, + "nauc_precision_at_1_std": 0.22996512608256198, + "nauc_precision_at_20_diff1": 0.45795692293377344, + "nauc_precision_at_20_max": 0.5357361507438838, + "nauc_precision_at_20_std": 0.4160858817582251, + "nauc_precision_at_3_diff1": 0.5826564933802196, + "nauc_precision_at_3_max": 0.49158910788383625, + "nauc_precision_at_3_std": 0.2651388501392188, + "nauc_precision_at_5_diff1": 0.5361007340951345, + "nauc_precision_at_5_max": 0.46650003271004487, + "nauc_precision_at_5_std": 0.29467856997063324, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.42251653533438616, + "nauc_recall_at_100_max": 0.5974632567614767, + "nauc_recall_at_100_std": 0.5798983552509402, + "nauc_recall_at_10_diff1": 0.4965565401407999, + "nauc_recall_at_10_max": 0.4884093981133426, + "nauc_recall_at_10_std": 0.3476667895125619, + "nauc_recall_at_1_diff1": 0.6881197967738875, + "nauc_recall_at_1_max": 0.5104387821019861, + "nauc_recall_at_1_std": 0.22996512608256198, + "nauc_recall_at_20_diff1": 0.4579569229337732, + "nauc_recall_at_20_max": 0.5357361507438837, + "nauc_recall_at_20_std": 0.41608588175822514, + "nauc_recall_at_3_diff1": 0.5826564933802192, + "nauc_recall_at_3_max": 0.4915891078838359, + "nauc_recall_at_3_std": 0.26513885013921823, + "nauc_recall_at_5_diff1": 0.5361007340951345, + "nauc_recall_at_5_max": 0.46650003271004464, + "nauc_recall_at_5_std": 0.29467856997063335, + "ndcg_at_1": 0.32884, + "ndcg_at_10": 0.44096, + "ndcg_at_100": 0.49016, + "ndcg_at_1000": 0.51345, + "ndcg_at_20": 0.461, + "ndcg_at_3": 0.3898, + "ndcg_at_5": 0.41458, + "precision_at_1": 0.32884, + "precision_at_10": 0.05769, + "precision_at_100": 0.00816, + "precision_at_1000": 0.001, + "precision_at_20": 0.03277, + "precision_at_3": 0.14516, + "precision_at_5": 0.09899, + "recall_at_1": 0.32884, + "recall_at_10": 0.57688, + "recall_at_100": 0.81594, + "recall_at_1000": 1.0, + "recall_at_20": 0.65544, + "recall_at_3": 0.43547, + "recall_at_5": 0.49495 + }, + { + "cv_recall_at_1": 0.11754, + "cv_recall_at_10": 0.2614, + "cv_recall_at_100": 0.56667, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.31228, + "cv_recall_at_3": 0.16491, + "cv_recall_at_5": 0.20351, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.17967, + "map_at_1": 0.11754, + "map_at_10": 0.15478, + "map_at_100": 0.16417, + "map_at_1000": 0.16616, + "map_at_20": 0.15821, + "map_at_3": 0.1383, + "map_at_5": 0.14716, + "mrr_at_1": 0.11754385964912281, + "mrr_at_10": 0.1547827903091061, + "mrr_at_100": 0.16417160639151848, + "mrr_at_1000": 0.16615976090692203, + "mrr_at_20": 0.15821471681867644, + "mrr_at_3": 0.13830409356725143, + "mrr_at_5": 0.14716374269005847, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.20686636279942508, + "nauc_cv_recall_at_100_max": 0.17926244045364328, + "nauc_cv_recall_at_100_std": 0.38626426415880044, + "nauc_cv_recall_at_10_diff1": 0.3603919019081563, + "nauc_cv_recall_at_10_max": 0.1680027725567475, + "nauc_cv_recall_at_10_std": 0.33617250564756646, + "nauc_cv_recall_at_1_diff1": 0.6064805637913367, + "nauc_cv_recall_at_1_max": 0.2527065611174607, + "nauc_cv_recall_at_1_std": 0.3772990481171108, + "nauc_cv_recall_at_20_diff1": 0.29609637486864593, + "nauc_cv_recall_at_20_max": 0.1845928383275292, + "nauc_cv_recall_at_20_std": 0.3293436311630296, + "nauc_cv_recall_at_3_diff1": 0.4800736209246735, + "nauc_cv_recall_at_3_max": 0.19174365694492915, + "nauc_cv_recall_at_3_std": 0.3175182241421318, + "nauc_cv_recall_at_5_diff1": 0.40168845928793273, + "nauc_cv_recall_at_5_max": 0.2056786469820408, + "nauc_cv_recall_at_5_std": 0.32768263041814816, + "nauc_map_at_1000_diff1": 0.49721183191633966, + "nauc_map_at_1000_max": 0.21530537794659496, + "nauc_map_at_1000_std": 0.3539650055758818, + "nauc_map_at_100_diff1": 0.49620795978310384, + "nauc_map_at_100_max": 0.21509618634759792, + "nauc_map_at_100_std": 0.3542543673006248, + "nauc_map_at_10_diff1": 0.5090810244617068, + "nauc_map_at_10_max": 0.21482663598645924, + "nauc_map_at_10_std": 0.3526557190520316, + "nauc_map_at_1_diff1": 0.6064805637913367, + "nauc_map_at_1_max": 0.2527065611174607, + "nauc_map_at_1_std": 0.3772990481171108, + "nauc_map_at_20_diff1": 0.5016662903455533, + "nauc_map_at_20_max": 0.21638144190481617, + "nauc_map_at_20_std": 0.352290307891851, + "nauc_map_at_3_diff1": 0.5444623926108907, + "nauc_map_at_3_max": 0.2171326896610425, + "nauc_map_at_3_std": 0.3464784007620425, + "nauc_map_at_5_diff1": 0.5186026114663759, + "nauc_map_at_5_max": 0.21988382288719446, + "nauc_map_at_5_std": 0.3488387290105743, + "nauc_mrr_at_1000_diff1": 0.49721183191633966, + "nauc_mrr_at_1000_max": 0.21530537794659496, + "nauc_mrr_at_1000_std": 0.3539650055758818, + "nauc_mrr_at_100_diff1": 0.49620795978310384, + "nauc_mrr_at_100_max": 0.21509618634759792, + "nauc_mrr_at_100_std": 0.3542543673006248, + "nauc_mrr_at_10_diff1": 0.5090810244617068, + "nauc_mrr_at_10_max": 0.21482663598645924, + "nauc_mrr_at_10_std": 0.3526557190520316, + "nauc_mrr_at_1_diff1": 0.6064805637913367, + "nauc_mrr_at_1_max": 0.2527065611174607, + "nauc_mrr_at_1_std": 0.3772990481171108, + "nauc_mrr_at_20_diff1": 0.5016662903455533, + "nauc_mrr_at_20_max": 0.21638144190481617, + "nauc_mrr_at_20_std": 0.352290307891851, + "nauc_mrr_at_3_diff1": 0.5444623926108907, + "nauc_mrr_at_3_max": 0.2171326896610425, + "nauc_mrr_at_3_std": 0.3464784007620425, + "nauc_mrr_at_5_diff1": 0.5186026114663759, + "nauc_mrr_at_5_max": 0.21988382288719446, + "nauc_mrr_at_5_std": 0.3488387290105743, + "nauc_ndcg_at_1000_diff1": 0.45255995578858577, + "nauc_ndcg_at_1000_max": 0.20829872016114442, + "nauc_ndcg_at_1000_std": 0.35374196329629476, + "nauc_ndcg_at_100_diff1": 0.4167023841930697, + "nauc_ndcg_at_100_max": 0.20346728060732097, + "nauc_ndcg_at_100_std": 0.35912915345884755, + "nauc_ndcg_at_10_diff1": 0.46408001594108844, + "nauc_ndcg_at_10_max": 0.20122977875141929, + "nauc_ndcg_at_10_std": 0.34759731100350105, + "nauc_ndcg_at_1_diff1": 0.6064805637913367, + "nauc_ndcg_at_1_max": 0.2527065611174607, + "nauc_ndcg_at_1_std": 0.3772990481171108, + "nauc_ndcg_at_20_diff1": 0.44021005808238023, + "nauc_ndcg_at_20_max": 0.2062442932637015, + "nauc_ndcg_at_20_std": 0.3455111095383758, + "nauc_ndcg_at_3_diff1": 0.5263189012349082, + "nauc_ndcg_at_3_max": 0.20975189907779873, + "nauc_ndcg_at_3_std": 0.3382583344210606, + "nauc_ndcg_at_5_diff1": 0.4842182837146837, + "nauc_ndcg_at_5_max": 0.21512998355498653, + "nauc_ndcg_at_5_std": 0.34229115985178443, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.20686636279942525, + "nauc_precision_at_100_max": 0.17926244045364373, + "nauc_precision_at_100_std": 0.38626426415880066, + "nauc_precision_at_10_diff1": 0.3603919019081564, + "nauc_precision_at_10_max": 0.16800277255674756, + "nauc_precision_at_10_std": 0.3361725056475665, + "nauc_precision_at_1_diff1": 0.6064805637913367, + "nauc_precision_at_1_max": 0.2527065611174607, + "nauc_precision_at_1_std": 0.3772990481171108, + "nauc_precision_at_20_diff1": 0.29609637486864615, + "nauc_precision_at_20_max": 0.18459283832752935, + "nauc_precision_at_20_std": 0.3293436311630295, + "nauc_precision_at_3_diff1": 0.4800736209246737, + "nauc_precision_at_3_max": 0.19174365694492926, + "nauc_precision_at_3_std": 0.31751822414213193, + "nauc_precision_at_5_diff1": 0.40168845928793273, + "nauc_precision_at_5_max": 0.20567864698204089, + "nauc_precision_at_5_std": 0.32768263041814827, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.20686636279942508, + "nauc_recall_at_100_max": 0.17926244045364328, + "nauc_recall_at_100_std": 0.38626426415880044, + "nauc_recall_at_10_diff1": 0.3603919019081563, + "nauc_recall_at_10_max": 0.1680027725567475, + "nauc_recall_at_10_std": 0.33617250564756646, + "nauc_recall_at_1_diff1": 0.6064805637913367, + "nauc_recall_at_1_max": 0.2527065611174607, + "nauc_recall_at_1_std": 0.3772990481171108, + "nauc_recall_at_20_diff1": 0.29609637486864593, + "nauc_recall_at_20_max": 0.1845928383275292, + "nauc_recall_at_20_std": 0.3293436311630296, + "nauc_recall_at_3_diff1": 0.4800736209246735, + "nauc_recall_at_3_max": 0.19174365694492915, + "nauc_recall_at_3_std": 0.3175182241421318, + "nauc_recall_at_5_diff1": 0.40168845928793273, + "nauc_recall_at_5_max": 0.2056786469820408, + "nauc_recall_at_5_std": 0.32768263041814816, + "ndcg_at_1": 0.11754, + "ndcg_at_10": 0.17967, + "ndcg_at_100": 0.23847, + "ndcg_at_1000": 0.29349, + "ndcg_at_20": 0.19242, + "ndcg_at_3": 0.14513, + "ndcg_at_5": 0.16106, + "precision_at_1": 0.11754, + "precision_at_10": 0.02614, + "precision_at_100": 0.00567, + "precision_at_1000": 0.001, + "precision_at_20": 0.01561, + "precision_at_3": 0.05497, + "precision_at_5": 0.0407, + "recall_at_1": 0.11754, + "recall_at_10": 0.2614, + "recall_at_100": 0.56667, + "recall_at_1000": 1.0, + "recall_at_20": 0.31228, + "recall_at_3": 0.16491, + "recall_at_5": 0.20351 + }, + { + "cv_recall_at_1": 0.17506, + "cv_recall_at_10": 0.35126, + "cv_recall_at_100": 0.67849, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.44394, + "cv_recall_at_3": 0.246, + "cv_recall_at_5": 0.28375, + "hf_subset": "et", + "languages": [ + "est-Latn" + ], + "main_score": 0.2524, + "map_at_1": 0.17506, + "map_at_10": 0.22221, + "map_at_100": 0.2342, + "map_at_1000": 0.23573, + "map_at_20": 0.22847, + "map_at_3": 0.205, + "map_at_5": 0.21381, + "mrr_at_1": 0.17505720823798626, + "mrr_at_10": 0.2222063310450038, + "mrr_at_100": 0.23420477050876534, + "mrr_at_1000": 0.2357283425713766, + "mrr_at_20": 0.22847110561344786, + "mrr_at_3": 0.20499618611746767, + "mrr_at_5": 0.21380625476735324, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.31030606229731295, + "nauc_cv_recall_at_100_max": 0.4920183843935835, + "nauc_cv_recall_at_100_std": 0.4214345474136694, + "nauc_cv_recall_at_10_diff1": 0.4319578595158496, + "nauc_cv_recall_at_10_max": 0.46680225532674047, + "nauc_cv_recall_at_10_std": 0.3069335988465379, + "nauc_cv_recall_at_1_diff1": 0.662791905035046, + "nauc_cv_recall_at_1_max": 0.5136669938374341, + "nauc_cv_recall_at_1_std": 0.22883573101962362, + "nauc_cv_recall_at_20_diff1": 0.38988749437635895, + "nauc_cv_recall_at_20_max": 0.4800254245971985, + "nauc_cv_recall_at_20_std": 0.3312341215165486, + "nauc_cv_recall_at_3_diff1": 0.5405291010324115, + "nauc_cv_recall_at_3_max": 0.4954257412448969, + "nauc_cv_recall_at_3_std": 0.28238790805278113, + "nauc_cv_recall_at_5_diff1": 0.4673198702095926, + "nauc_cv_recall_at_5_max": 0.4819739132028776, + "nauc_cv_recall_at_5_std": 0.30321271891046175, + "nauc_map_at_1000_diff1": 0.5753951970120799, + "nauc_map_at_1000_max": 0.49867755908541817, + "nauc_map_at_1000_std": 0.2629204728875237, + "nauc_map_at_100_diff1": 0.5742791463806061, + "nauc_map_at_100_max": 0.4987120095894289, + "nauc_map_at_100_std": 0.2636667111263593, + "nauc_map_at_10_diff1": 0.5789104428111786, + "nauc_map_at_10_max": 0.4978836819985857, + "nauc_map_at_10_std": 0.2600144113637667, + "nauc_map_at_1_diff1": 0.662791905035046, + "nauc_map_at_1_max": 0.5136669938374341, + "nauc_map_at_1_std": 0.22883573101962362, + "nauc_map_at_20_diff1": 0.5755538396436413, + "nauc_map_at_20_max": 0.49831743106828436, + "nauc_map_at_20_std": 0.26198591051716935, + "nauc_map_at_3_diff1": 0.6059299307856398, + "nauc_map_at_3_max": 0.5053408422172819, + "nauc_map_at_3_std": 0.25371863899923963, + "nauc_map_at_5_diff1": 0.5850701872196608, + "nauc_map_at_5_max": 0.5014935376584141, + "nauc_map_at_5_std": 0.2595483491706154, + "nauc_mrr_at_1000_diff1": 0.5753951970120799, + "nauc_mrr_at_1000_max": 0.49867755908541817, + "nauc_mrr_at_1000_std": 0.2629204728875237, + "nauc_mrr_at_100_diff1": 0.5742791463806061, + "nauc_mrr_at_100_max": 0.4987120095894289, + "nauc_mrr_at_100_std": 0.2636667111263593, + "nauc_mrr_at_10_diff1": 0.5789104428111786, + "nauc_mrr_at_10_max": 0.4978836819985857, + "nauc_mrr_at_10_std": 0.2600144113637667, + "nauc_mrr_at_1_diff1": 0.662791905035046, + "nauc_mrr_at_1_max": 0.5136669938374341, + "nauc_mrr_at_1_std": 0.22883573101962362, + "nauc_mrr_at_20_diff1": 0.5755538396436413, + "nauc_mrr_at_20_max": 0.49831743106828436, + "nauc_mrr_at_20_std": 0.26198591051716935, + "nauc_mrr_at_3_diff1": 0.6059299307856398, + "nauc_mrr_at_3_max": 0.5053408422172819, + "nauc_mrr_at_3_std": 0.25371863899923963, + "nauc_mrr_at_5_diff1": 0.5850701872196608, + "nauc_mrr_at_5_max": 0.5014935376584141, + "nauc_mrr_at_5_std": 0.2595483491706154, + "nauc_ndcg_at_1000_diff1": 0.546210016037982, + "nauc_ndcg_at_1000_max": 0.49504877252113694, + "nauc_ndcg_at_1000_std": 0.27535570292149614, + "nauc_ndcg_at_100_diff1": 0.5185344152265912, + "nauc_ndcg_at_100_max": 0.4948560103106455, + "nauc_ndcg_at_100_std": 0.29277026295247915, + "nauc_ndcg_at_10_diff1": 0.5383424122759396, + "nauc_ndcg_at_10_max": 0.4892576392717773, + "nauc_ndcg_at_10_std": 0.2731410798077506, + "nauc_ndcg_at_1_diff1": 0.662791905035046, + "nauc_ndcg_at_1_max": 0.5136669938374341, + "nauc_ndcg_at_1_std": 0.22883573101962362, + "nauc_ndcg_at_20_diff1": 0.5261286547319675, + "nauc_ndcg_at_20_max": 0.49228912059389557, + "nauc_ndcg_at_20_std": 0.2801804838120073, + "nauc_ndcg_at_3_diff1": 0.5879478465311763, + "nauc_ndcg_at_3_max": 0.5026208826663386, + "nauc_ndcg_at_3_std": 0.2616003966411608, + "nauc_ndcg_at_5_diff1": 0.5526361128325122, + "nauc_ndcg_at_5_max": 0.4961346125428329, + "nauc_ndcg_at_5_std": 0.2716265448866012, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3103060622973135, + "nauc_precision_at_100_max": 0.49201838439358375, + "nauc_precision_at_100_std": 0.42143454741367004, + "nauc_precision_at_10_diff1": 0.43195785951584975, + "nauc_precision_at_10_max": 0.46680225532674036, + "nauc_precision_at_10_std": 0.3069335988465379, + "nauc_precision_at_1_diff1": 0.662791905035046, + "nauc_precision_at_1_max": 0.5136669938374341, + "nauc_precision_at_1_std": 0.22883573101962362, + "nauc_precision_at_20_diff1": 0.3898874943763586, + "nauc_precision_at_20_max": 0.4800254245971982, + "nauc_precision_at_20_std": 0.3312341215165482, + "nauc_precision_at_3_diff1": 0.5405291010324113, + "nauc_precision_at_3_max": 0.49542574124489663, + "nauc_precision_at_3_std": 0.28238790805278136, + "nauc_precision_at_5_diff1": 0.4673198702095924, + "nauc_precision_at_5_max": 0.4819739132028772, + "nauc_precision_at_5_std": 0.30321271891046164, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.31030606229731295, + "nauc_recall_at_100_max": 0.4920183843935835, + "nauc_recall_at_100_std": 0.4214345474136694, + "nauc_recall_at_10_diff1": 0.4319578595158496, + "nauc_recall_at_10_max": 0.46680225532674047, + "nauc_recall_at_10_std": 0.3069335988465379, + "nauc_recall_at_1_diff1": 0.662791905035046, + "nauc_recall_at_1_max": 0.5136669938374341, + "nauc_recall_at_1_std": 0.22883573101962362, + "nauc_recall_at_20_diff1": 0.38988749437635895, + "nauc_recall_at_20_max": 0.4800254245971985, + "nauc_recall_at_20_std": 0.3312341215165486, + "nauc_recall_at_3_diff1": 0.5405291010324115, + "nauc_recall_at_3_max": 0.4954257412448969, + "nauc_recall_at_3_std": 0.28238790805278113, + "nauc_recall_at_5_diff1": 0.4673198702095926, + "nauc_recall_at_5_max": 0.4819739132028776, + "nauc_recall_at_5_std": 0.30321271891046175, + "ndcg_at_1": 0.17506, + "ndcg_at_10": 0.2524, + "ndcg_at_100": 0.31861, + "ndcg_at_1000": 0.35964, + "ndcg_at_20": 0.27562, + "ndcg_at_3": 0.21547, + "ndcg_at_5": 0.23118, + "precision_at_1": 0.17506, + "precision_at_10": 0.03513, + "precision_at_100": 0.00678, + "precision_at_1000": 0.001, + "precision_at_20": 0.0222, + "precision_at_3": 0.082, + "precision_at_5": 0.05675, + "recall_at_1": 0.17506, + "recall_at_10": 0.35126, + "recall_at_100": 0.67849, + "recall_at_1000": 1.0, + "recall_at_20": 0.44394, + "recall_at_3": 0.246, + "recall_at_5": 0.28375 + }, + { + "cv_recall_at_1": 0.3818, + "cv_recall_at_10": 0.65816, + "cv_recall_at_100": 0.90455, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.75472, + "cv_recall_at_3": 0.51831, + "cv_recall_at_5": 0.58713, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.51299, + "map_at_1": 0.3818, + "map_at_10": 0.46725, + "map_at_100": 0.47806, + "map_at_1000": 0.47854, + "map_at_20": 0.47405, + "map_at_3": 0.44192, + "map_at_5": 0.45762, + "mrr_at_1": 0.38179800221975585, + "mrr_at_10": 0.467253756848651, + "mrr_at_100": 0.4780623138117509, + "mrr_at_1000": 0.478538713382236, + "mrr_at_20": 0.4740469779162587, + "mrr_at_3": 0.4419163891971881, + "mrr_at_5": 0.4576211616722157, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5265912201308302, + "nauc_cv_recall_at_100_max": 0.6336587563293079, + "nauc_cv_recall_at_100_std": 0.6311235478687189, + "nauc_cv_recall_at_10_diff1": 0.5851689002249301, + "nauc_cv_recall_at_10_max": 0.4485733805721923, + "nauc_cv_recall_at_10_std": 0.3718632734056777, + "nauc_cv_recall_at_1_diff1": 0.7137156779886052, + "nauc_cv_recall_at_1_max": 0.45559066885439636, + "nauc_cv_recall_at_1_std": 0.21835400435692623, + "nauc_cv_recall_at_20_diff1": 0.533567046952136, + "nauc_cv_recall_at_20_max": 0.484551209017695, + "nauc_cv_recall_at_20_std": 0.47478384921943123, + "nauc_cv_recall_at_3_diff1": 0.6285924276187039, + "nauc_cv_recall_at_3_max": 0.44391634092707977, + "nauc_cv_recall_at_3_std": 0.3112414211969393, + "nauc_cv_recall_at_5_diff1": 0.6021358367466351, + "nauc_cv_recall_at_5_max": 0.4355454957871355, + "nauc_cv_recall_at_5_std": 0.3236237251547708, + "nauc_map_at_1000_diff1": 0.6697722369919447, + "nauc_map_at_1000_max": 0.4508071898827599, + "nauc_map_at_1000_std": 0.2661928201749134, + "nauc_map_at_100_diff1": 0.6695552441716708, + "nauc_map_at_100_max": 0.4510068260181231, + "nauc_map_at_100_std": 0.2666357600178505, + "nauc_map_at_10_diff1": 0.670017679250379, + "nauc_map_at_10_max": 0.44799842203040807, + "nauc_map_at_10_std": 0.2633449828707209, + "nauc_map_at_1_diff1": 0.7137156779886052, + "nauc_map_at_1_max": 0.45559066885439636, + "nauc_map_at_1_std": 0.21835400435692623, + "nauc_map_at_20_diff1": 0.6686318875348207, + "nauc_map_at_20_max": 0.44973314463975794, + "nauc_map_at_20_std": 0.2670816271196027, + "nauc_map_at_3_diff1": 0.6759346113098538, + "nauc_map_at_3_max": 0.4483948069723216, + "nauc_map_at_3_std": 0.25772030300263965, + "nauc_map_at_5_diff1": 0.6710255392841811, + "nauc_map_at_5_max": 0.4466274421733826, + "nauc_map_at_5_std": 0.25962165465555503, + "nauc_mrr_at_1000_diff1": 0.6697719406219865, + "nauc_mrr_at_1000_max": 0.4508059845394589, + "nauc_mrr_at_1000_std": 0.2661912917368203, + "nauc_mrr_at_100_diff1": 0.6695549610314403, + "nauc_mrr_at_100_max": 0.45100559857467465, + "nauc_mrr_at_100_std": 0.26663420897995804, + "nauc_mrr_at_10_diff1": 0.670017679250379, + "nauc_mrr_at_10_max": 0.44799842203040807, + "nauc_mrr_at_10_std": 0.2633449828707209, + "nauc_mrr_at_1_diff1": 0.7137156779886052, + "nauc_mrr_at_1_max": 0.45559066885439636, + "nauc_mrr_at_1_std": 0.21835400435692623, + "nauc_mrr_at_20_diff1": 0.6686318875348207, + "nauc_mrr_at_20_max": 0.44973314463975794, + "nauc_mrr_at_20_std": 0.2670816271196027, + "nauc_mrr_at_3_diff1": 0.6759346113098538, + "nauc_mrr_at_3_max": 0.4483948069723216, + "nauc_mrr_at_3_std": 0.25772030300263965, + "nauc_mrr_at_5_diff1": 0.6710255392841811, + "nauc_mrr_at_5_max": 0.4466274421733826, + "nauc_mrr_at_5_std": 0.25962165465555503, + "nauc_ndcg_at_1000_diff1": 0.65934330876716, + "nauc_ndcg_at_1000_max": 0.45495571849239214, + "nauc_ndcg_at_1000_std": 0.28259581269143613, + "nauc_ndcg_at_100_diff1": 0.6536109125984344, + "nauc_ndcg_at_100_max": 0.46218844729330094, + "nauc_ndcg_at_100_std": 0.29694705857691633, + "nauc_ndcg_at_10_diff1": 0.6517697877827734, + "nauc_ndcg_at_10_max": 0.4476652993910479, + "nauc_ndcg_at_10_std": 0.2858562756359774, + "nauc_ndcg_at_1_diff1": 0.7137156779886052, + "nauc_ndcg_at_1_max": 0.45559066885439636, + "nauc_ndcg_at_1_std": 0.21835400435692623, + "nauc_ndcg_at_20_diff1": 0.6457801636805331, + "nauc_ndcg_at_20_max": 0.4541692965534771, + "nauc_ndcg_at_20_std": 0.3004166175153123, + "nauc_ndcg_at_3_diff1": 0.663972432848074, + "nauc_ndcg_at_3_max": 0.44717519330462746, + "nauc_ndcg_at_3_std": 0.2711624329493742, + "nauc_ndcg_at_5_diff1": 0.6547094368277582, + "nauc_ndcg_at_5_max": 0.44396849750140915, + "nauc_ndcg_at_5_std": 0.27489677323503187, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.5265912201308315, + "nauc_precision_at_100_max": 0.6336587563293098, + "nauc_precision_at_100_std": 0.6311235478687203, + "nauc_precision_at_10_diff1": 0.5851689002249305, + "nauc_precision_at_10_max": 0.4485733805721921, + "nauc_precision_at_10_std": 0.3718632734056779, + "nauc_precision_at_1_diff1": 0.7137156779886052, + "nauc_precision_at_1_max": 0.45559066885439636, + "nauc_precision_at_1_std": 0.21835400435692623, + "nauc_precision_at_20_diff1": 0.5335670469521362, + "nauc_precision_at_20_max": 0.4845512090176942, + "nauc_precision_at_20_std": 0.47478384921943095, + "nauc_precision_at_3_diff1": 0.6285924276187037, + "nauc_precision_at_3_max": 0.4439163409270798, + "nauc_precision_at_3_std": 0.31124142119693926, + "nauc_precision_at_5_diff1": 0.6021358367466348, + "nauc_precision_at_5_max": 0.4355454957871354, + "nauc_precision_at_5_std": 0.3236237251547704, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5265912201308302, + "nauc_recall_at_100_max": 0.6336587563293079, + "nauc_recall_at_100_std": 0.6311235478687189, + "nauc_recall_at_10_diff1": 0.5851689002249301, + "nauc_recall_at_10_max": 0.4485733805721923, + "nauc_recall_at_10_std": 0.3718632734056777, + "nauc_recall_at_1_diff1": 0.7137156779886052, + "nauc_recall_at_1_max": 0.45559066885439636, + "nauc_recall_at_1_std": 0.21835400435692623, + "nauc_recall_at_20_diff1": 0.533567046952136, + "nauc_recall_at_20_max": 0.484551209017695, + "nauc_recall_at_20_std": 0.47478384921943123, + "nauc_recall_at_3_diff1": 0.6285924276187039, + "nauc_recall_at_3_max": 0.44391634092707977, + "nauc_recall_at_3_std": 0.3112414211969393, + "nauc_recall_at_5_diff1": 0.6021358367466351, + "nauc_recall_at_5_max": 0.4355454957871355, + "nauc_recall_at_5_std": 0.3236237251547708, + "ndcg_at_1": 0.3818, + "ndcg_at_10": 0.51299, + "ndcg_at_100": 0.56564, + "ndcg_at_1000": 0.57796, + "ndcg_at_20": 0.53752, + "ndcg_at_3": 0.46154, + "ndcg_at_5": 0.48986, + "precision_at_1": 0.3818, + "precision_at_10": 0.06582, + "precision_at_100": 0.00905, + "precision_at_1000": 0.001, + "precision_at_20": 0.03774, + "precision_at_3": 0.17277, + "precision_at_5": 0.11743, + "recall_at_1": 0.3818, + "recall_at_10": 0.65816, + "recall_at_100": 0.90455, + "recall_at_1000": 1.0, + "recall_at_20": 0.75472, + "recall_at_3": 0.51831, + "recall_at_5": 0.58713 + }, + { + "cv_recall_at_1": 0.07841, + "cv_recall_at_10": 0.21805, + "cv_recall_at_100": 0.47691, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.2739, + "cv_recall_at_3": 0.12137, + "cv_recall_at_5": 0.15897, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.13717, + "map_at_1": 0.07841, + "map_at_10": 0.11268, + "map_at_100": 0.12105, + "map_at_1000": 0.12309, + "map_at_20": 0.11648, + "map_at_3": 0.09649, + "map_at_5": 0.10487, + "mrr_at_1": 0.07841031149301826, + "mrr_at_10": 0.11267837962252575, + "mrr_at_100": 0.12104804694712606, + "mrr_at_1000": 0.12308794285824537, + "mrr_at_20": 0.11648311700466915, + "mrr_at_3": 0.09649122807017542, + "mrr_at_5": 0.10486931614751165, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.18453188206043467, + "nauc_cv_recall_at_100_max": 0.16392815605857672, + "nauc_cv_recall_at_100_std": 0.3509105591030677, + "nauc_cv_recall_at_10_diff1": 0.31782919395535686, + "nauc_cv_recall_at_10_max": 0.26121048660827145, + "nauc_cv_recall_at_10_std": 0.3216704847714983, + "nauc_cv_recall_at_1_diff1": 0.7300323741152004, + "nauc_cv_recall_at_1_max": 0.5875499383325973, + "nauc_cv_recall_at_1_std": 0.4395028543714588, + "nauc_cv_recall_at_20_diff1": 0.2850540894070008, + "nauc_cv_recall_at_20_max": 0.25926111017353165, + "nauc_cv_recall_at_20_std": 0.35138498307149507, + "nauc_cv_recall_at_3_diff1": 0.52950899182385, + "nauc_cv_recall_at_3_max": 0.40695540700911886, + "nauc_cv_recall_at_3_std": 0.37230982704145404, + "nauc_cv_recall_at_5_diff1": 0.44363573917477916, + "nauc_cv_recall_at_5_max": 0.31469487529160706, + "nauc_cv_recall_at_5_std": 0.33104107124420945, + "nauc_map_at_1000_diff1": 0.5460353243518522, + "nauc_map_at_1000_max": 0.42919167158826854, + "nauc_map_at_1000_std": 0.3813551521426623, + "nauc_map_at_100_diff1": 0.544390937711658, + "nauc_map_at_100_max": 0.42768712328820707, + "nauc_map_at_100_std": 0.38100639101593836, + "nauc_map_at_10_diff1": 0.5574473549510108, + "nauc_map_at_10_max": 0.43533414024358197, + "nauc_map_at_10_std": 0.3789419393898596, + "nauc_map_at_1_diff1": 0.7300323741152004, + "nauc_map_at_1_max": 0.5875499383325973, + "nauc_map_at_1_std": 0.4395028543714588, + "nauc_map_at_20_diff1": 0.5508909279002877, + "nauc_map_at_20_max": 0.43386475681313386, + "nauc_map_at_20_std": 0.382263436012735, + "nauc_map_at_3_diff1": 0.6223212772454054, + "nauc_map_at_3_max": 0.48578152509206257, + "nauc_map_at_3_std": 0.39982562617190115, + "nauc_map_at_5_diff1": 0.591480016688883, + "nauc_map_at_5_max": 0.4531842521492073, + "nauc_map_at_5_std": 0.384540400987139, + "nauc_mrr_at_1000_diff1": 0.5460314797682349, + "nauc_mrr_at_1000_max": 0.4291924596284079, + "nauc_mrr_at_1000_std": 0.3813545092878323, + "nauc_mrr_at_100_diff1": 0.5443871148859052, + "nauc_mrr_at_100_max": 0.4276879060108006, + "nauc_mrr_at_100_std": 0.3810057517648782, + "nauc_mrr_at_10_diff1": 0.5574473549510108, + "nauc_mrr_at_10_max": 0.43533414024358197, + "nauc_mrr_at_10_std": 0.3789419393898596, + "nauc_mrr_at_1_diff1": 0.7300323741152004, + "nauc_mrr_at_1_max": 0.5875499383325973, + "nauc_mrr_at_1_std": 0.4395028543714588, + "nauc_mrr_at_20_diff1": 0.5508909279002877, + "nauc_mrr_at_20_max": 0.43386475681313386, + "nauc_mrr_at_20_std": 0.382263436012735, + "nauc_mrr_at_3_diff1": 0.6223212772454054, + "nauc_mrr_at_3_max": 0.48578152509206257, + "nauc_mrr_at_3_std": 0.39982562617190115, + "nauc_mrr_at_5_diff1": 0.591480016688883, + "nauc_mrr_at_5_max": 0.4531842521492073, + "nauc_mrr_at_5_std": 0.384540400987139, + "nauc_ndcg_at_1000_diff1": 0.4788419648481831, + "nauc_ndcg_at_1000_max": 0.38079278911374764, + "nauc_ndcg_at_1000_std": 0.3716402728056327, + "nauc_ndcg_at_100_diff1": 0.42588088426408005, + "nauc_ndcg_at_100_max": 0.3404479375921339, + "nauc_ndcg_at_100_std": 0.36683836687852833, + "nauc_ndcg_at_10_diff1": 0.47794237369134523, + "nauc_ndcg_at_10_max": 0.37645783480827505, + "nauc_ndcg_at_10_std": 0.35898182581341265, + "nauc_ndcg_at_1_diff1": 0.7300323741152004, + "nauc_ndcg_at_1_max": 0.5875499383325973, + "nauc_ndcg_at_1_std": 0.4395028543714588, + "nauc_ndcg_at_20_diff1": 0.4598258570443921, + "nauc_ndcg_at_20_max": 0.37237818193500444, + "nauc_ndcg_at_20_std": 0.369399984258033, + "nauc_ndcg_at_3_diff1": 0.5948678284793605, + "nauc_ndcg_at_3_max": 0.4622721642364097, + "nauc_ndcg_at_3_std": 0.39154133751679404, + "nauc_ndcg_at_5_diff1": 0.5452122159932722, + "nauc_ndcg_at_5_max": 0.40980433669295235, + "nauc_ndcg_at_5_std": 0.36767612935248906, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.18453188206043514, + "nauc_precision_at_100_max": 0.163928156058577, + "nauc_precision_at_100_std": 0.35091055910306784, + "nauc_precision_at_10_diff1": 0.31782919395535664, + "nauc_precision_at_10_max": 0.26121048660827145, + "nauc_precision_at_10_std": 0.3216704847714983, + "nauc_precision_at_1_diff1": 0.7300323741152004, + "nauc_precision_at_1_max": 0.5875499383325973, + "nauc_precision_at_1_std": 0.4395028543714588, + "nauc_precision_at_20_diff1": 0.28505408940700105, + "nauc_precision_at_20_max": 0.2592611101735318, + "nauc_precision_at_20_std": 0.3513849830714949, + "nauc_precision_at_3_diff1": 0.5295089918238496, + "nauc_precision_at_3_max": 0.4069554070091187, + "nauc_precision_at_3_std": 0.37230982704145404, + "nauc_precision_at_5_diff1": 0.4436357391747791, + "nauc_precision_at_5_max": 0.3146948752916072, + "nauc_precision_at_5_std": 0.3310410712442096, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.18453188206043467, + "nauc_recall_at_100_max": 0.16392815605857672, + "nauc_recall_at_100_std": 0.3509105591030677, + "nauc_recall_at_10_diff1": 0.31782919395535686, + "nauc_recall_at_10_max": 0.26121048660827145, + "nauc_recall_at_10_std": 0.3216704847714983, + "nauc_recall_at_1_diff1": 0.7300323741152004, + "nauc_recall_at_1_max": 0.5875499383325973, + "nauc_recall_at_1_std": 0.4395028543714588, + "nauc_recall_at_20_diff1": 0.2850540894070008, + "nauc_recall_at_20_max": 0.25926111017353165, + "nauc_recall_at_20_std": 0.35138498307149507, + "nauc_recall_at_3_diff1": 0.52950899182385, + "nauc_recall_at_3_max": 0.40695540700911886, + "nauc_recall_at_3_std": 0.37230982704145404, + "nauc_recall_at_5_diff1": 0.44363573917477916, + "nauc_recall_at_5_max": 0.31469487529160706, + "nauc_recall_at_5_std": 0.33104107124420945, + "ndcg_at_1": 0.07841, + "ndcg_at_10": 0.13717, + "ndcg_at_100": 0.18755, + "ndcg_at_1000": 0.25155, + "ndcg_at_20": 0.1512, + "ndcg_at_3": 0.10285, + "ndcg_at_5": 0.11814, + "precision_at_1": 0.07841, + "precision_at_10": 0.0218, + "precision_at_100": 0.00477, + "precision_at_1000": 0.001, + "precision_at_20": 0.01369, + "precision_at_3": 0.04046, + "precision_at_5": 0.03179, + "recall_at_1": 0.07841, + "recall_at_10": 0.21805, + "recall_at_100": 0.47691, + "recall_at_1000": 1.0, + "recall_at_20": 0.2739, + "recall_at_3": 0.12137, + "recall_at_5": 0.15897 + }, + { + "cv_recall_at_1": 0.087, + "cv_recall_at_10": 0.251, + "cv_recall_at_100": 0.563, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.313, + "cv_recall_at_3": 0.164, + "cv_recall_at_5": 0.191, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.16084, + "map_at_1": 0.087, + "map_at_10": 0.13302, + "map_at_100": 0.14278, + "map_at_1000": 0.14468, + "map_at_20": 0.13725, + "map_at_3": 0.119, + "map_at_5": 0.12515, + "mrr_at_1": 0.087, + "mrr_at_10": 0.1330154761904761, + "mrr_at_100": 0.14277627814186838, + "mrr_at_1000": 0.14467822399493577, + "mrr_at_20": 0.1372495060022536, + "mrr_at_3": 0.11899999999999991, + "mrr_at_5": 0.12514999999999993, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.1558876013096379, + "nauc_cv_recall_at_100_max": 0.13621597911643635, + "nauc_cv_recall_at_100_std": 0.22131963502183968, + "nauc_cv_recall_at_10_diff1": 0.3552955619811528, + "nauc_cv_recall_at_10_max": 0.08030756714447457, + "nauc_cv_recall_at_10_std": 0.14590495435100892, + "nauc_cv_recall_at_1_diff1": 0.5057347902383663, + "nauc_cv_recall_at_1_max": 0.1060512137277217, + "nauc_cv_recall_at_1_std": 0.03693766210535726, + "nauc_cv_recall_at_20_diff1": 0.3068987784462007, + "nauc_cv_recall_at_20_max": 0.09564743425371655, + "nauc_cv_recall_at_20_std": 0.13610007638906607, + "nauc_cv_recall_at_3_diff1": 0.4561053020422625, + "nauc_cv_recall_at_3_max": 0.055866321891938665, + "nauc_cv_recall_at_3_std": 0.17909752953327257, + "nauc_cv_recall_at_5_diff1": 0.442646228966321, + "nauc_cv_recall_at_5_max": 0.07753053012657463, + "nauc_cv_recall_at_5_std": 0.1616987359280104, + "nauc_map_at_1000_diff1": 0.44417160523549337, + "nauc_map_at_1000_max": 0.0852978835378124, + "nauc_map_at_1000_std": 0.10768953069112178, + "nauc_map_at_100_diff1": 0.4428536887062119, + "nauc_map_at_100_max": 0.08499196853620133, + "nauc_map_at_100_std": 0.10736629195957144, + "nauc_map_at_10_diff1": 0.45481166438543014, + "nauc_map_at_10_max": 0.08291759622326565, + "nauc_map_at_10_std": 0.10690073118185775, + "nauc_map_at_1_diff1": 0.5057347902383663, + "nauc_map_at_1_max": 0.1060512137277217, + "nauc_map_at_1_std": 0.03693766210535726, + "nauc_map_at_20_diff1": 0.44816652230523707, + "nauc_map_at_20_max": 0.08428473121701546, + "nauc_map_at_20_std": 0.10578701542925341, + "nauc_map_at_3_diff1": 0.4784349377587308, + "nauc_map_at_3_max": 0.07575787040300637, + "nauc_map_at_3_std": 0.11229478774442961, + "nauc_map_at_5_diff1": 0.47406143708766296, + "nauc_map_at_5_max": 0.08129161757667618, + "nauc_map_at_5_std": 0.1089115383035945, + "nauc_mrr_at_1000_diff1": 0.44417160523549337, + "nauc_mrr_at_1000_max": 0.0852978835378124, + "nauc_mrr_at_1000_std": 0.10768953069112178, + "nauc_mrr_at_100_diff1": 0.4428536887062119, + "nauc_mrr_at_100_max": 0.08499196853620133, + "nauc_mrr_at_100_std": 0.10736629195957144, + "nauc_mrr_at_10_diff1": 0.45481166438543014, + "nauc_mrr_at_10_max": 0.08291759622326565, + "nauc_mrr_at_10_std": 0.10690073118185775, + "nauc_mrr_at_1_diff1": 0.5057347902383663, + "nauc_mrr_at_1_max": 0.1060512137277217, + "nauc_mrr_at_1_std": 0.03693766210535726, + "nauc_mrr_at_20_diff1": 0.44816652230523707, + "nauc_mrr_at_20_max": 0.08428473121701546, + "nauc_mrr_at_20_std": 0.10578701542925341, + "nauc_mrr_at_3_diff1": 0.4784349377587308, + "nauc_mrr_at_3_max": 0.07575787040300637, + "nauc_mrr_at_3_std": 0.11229478774442961, + "nauc_mrr_at_5_diff1": 0.47406143708766296, + "nauc_mrr_at_5_max": 0.08129161757667618, + "nauc_mrr_at_5_std": 0.1089115383035945, + "nauc_ndcg_at_1000_diff1": 0.4087907427501082, + "nauc_ndcg_at_1000_max": 0.0912358458578291, + "nauc_ndcg_at_1000_std": 0.12493394003702137, + "nauc_ndcg_at_100_diff1": 0.368002561758682, + "nauc_ndcg_at_100_max": 0.09539143507732341, + "nauc_ndcg_at_100_std": 0.13604709329141815, + "nauc_ndcg_at_10_diff1": 0.4239977392449605, + "nauc_ndcg_at_10_max": 0.08170068898856558, + "nauc_ndcg_at_10_std": 0.12101181084019956, + "nauc_ndcg_at_1_diff1": 0.5057347902383663, + "nauc_ndcg_at_1_max": 0.1060512137277217, + "nauc_ndcg_at_1_std": 0.03693766210535726, + "nauc_ndcg_at_20_diff1": 0.4040731057585679, + "nauc_ndcg_at_20_max": 0.08686523266695671, + "nauc_ndcg_at_20_std": 0.11781622112525923, + "nauc_ndcg_at_3_diff1": 0.47157401991603676, + "nauc_ndcg_at_3_max": 0.06954207858405162, + "nauc_ndcg_at_3_std": 0.13271042137472033, + "nauc_ndcg_at_5_diff1": 0.4641953202178358, + "nauc_ndcg_at_5_max": 0.07954983760889357, + "nauc_ndcg_at_5_std": 0.12614454515428244, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.15588760130963777, + "nauc_precision_at_100_max": 0.13621597911643624, + "nauc_precision_at_100_std": 0.2213196350218393, + "nauc_precision_at_10_diff1": 0.3552955619811528, + "nauc_precision_at_10_max": 0.08030756714447448, + "nauc_precision_at_10_std": 0.14590495435100886, + "nauc_precision_at_1_diff1": 0.5057347902383663, + "nauc_precision_at_1_max": 0.1060512137277217, + "nauc_precision_at_1_std": 0.03693766210535726, + "nauc_precision_at_20_diff1": 0.30689877844620056, + "nauc_precision_at_20_max": 0.09564743425371641, + "nauc_precision_at_20_std": 0.13610007638906604, + "nauc_precision_at_3_diff1": 0.45610530204226246, + "nauc_precision_at_3_max": 0.05586632189193886, + "nauc_precision_at_3_std": 0.17909752953327254, + "nauc_precision_at_5_diff1": 0.4426462289663209, + "nauc_precision_at_5_max": 0.07753053012657454, + "nauc_precision_at_5_std": 0.16169873592801015, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.1558876013096379, + "nauc_recall_at_100_max": 0.13621597911643635, + "nauc_recall_at_100_std": 0.22131963502183968, + "nauc_recall_at_10_diff1": 0.3552955619811528, + "nauc_recall_at_10_max": 0.08030756714447457, + "nauc_recall_at_10_std": 0.14590495435100892, + "nauc_recall_at_1_diff1": 0.5057347902383663, + "nauc_recall_at_1_max": 0.1060512137277217, + "nauc_recall_at_1_std": 0.03693766210535726, + "nauc_recall_at_20_diff1": 0.3068987784462007, + "nauc_recall_at_20_max": 0.09564743425371655, + "nauc_recall_at_20_std": 0.13610007638906607, + "nauc_recall_at_3_diff1": 0.4561053020422625, + "nauc_recall_at_3_max": 0.055866321891938665, + "nauc_recall_at_3_std": 0.17909752953327257, + "nauc_recall_at_5_diff1": 0.442646228966321, + "nauc_recall_at_5_max": 0.07753053012657463, + "nauc_recall_at_5_std": 0.1616987359280104, + "ndcg_at_1": 0.087, + "ndcg_at_10": 0.16084, + "ndcg_at_100": 0.22102, + "ndcg_at_1000": 0.27573, + "ndcg_at_20": 0.17643, + "ndcg_at_3": 0.13048, + "ndcg_at_5": 0.14158, + "precision_at_1": 0.087, + "precision_at_10": 0.0251, + "precision_at_100": 0.00563, + "precision_at_1000": 0.001, + "precision_at_20": 0.01565, + "precision_at_3": 0.05467, + "precision_at_5": 0.0382, + "recall_at_1": 0.087, + "recall_at_10": 0.251, + "recall_at_100": 0.563, + "recall_at_1000": 1.0, + "recall_at_20": 0.313, + "recall_at_3": 0.164, + "recall_at_5": 0.191 + }, + { + "cv_recall_at_1": 0.32316, + "cv_recall_at_10": 0.57975, + "cv_recall_at_100": 0.84466, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.6491, + "cv_recall_at_3": 0.44521, + "cv_recall_at_5": 0.50208, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.44335, + "map_at_1": 0.32316, + "map_at_10": 0.40073, + "map_at_100": 0.41023, + "map_at_1000": 0.41109, + "map_at_20": 0.40551, + "map_at_3": 0.37748, + "map_at_5": 0.39025, + "mrr_at_1": 0.3231622746185853, + "mrr_at_10": 0.400730907249631, + "mrr_at_100": 0.41022696716485957, + "mrr_at_1000": 0.41109069877067983, + "mrr_at_20": 0.4055117706081922, + "mrr_at_3": 0.37748497457235325, + "mrr_at_5": 0.39024503005085515, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.48098495877756514, + "nauc_cv_recall_at_100_max": 0.5404607638853961, + "nauc_cv_recall_at_100_std": 0.5785820711736115, + "nauc_cv_recall_at_10_diff1": 0.5077985610322101, + "nauc_cv_recall_at_10_max": 0.52220082238744, + "nauc_cv_recall_at_10_std": 0.4673819128352976, + "nauc_cv_recall_at_1_diff1": 0.6603749690968012, + "nauc_cv_recall_at_1_max": 0.522301229855425, + "nauc_cv_recall_at_1_std": 0.35891873761023935, + "nauc_cv_recall_at_20_diff1": 0.47762772221834277, + "nauc_cv_recall_at_20_max": 0.5148646415990591, + "nauc_cv_recall_at_20_std": 0.5201794202680657, + "nauc_cv_recall_at_3_diff1": 0.5477584110243355, + "nauc_cv_recall_at_3_max": 0.5251152644999395, + "nauc_cv_recall_at_3_std": 0.43168441471824376, + "nauc_cv_recall_at_5_diff1": 0.5197125969701824, + "nauc_cv_recall_at_5_max": 0.5353503012783575, + "nauc_cv_recall_at_5_std": 0.43589790113593907, + "nauc_map_at_1000_diff1": 0.6032375794517476, + "nauc_map_at_1000_max": 0.5234777263347525, + "nauc_map_at_1000_std": 0.3925397811648996, + "nauc_map_at_100_diff1": 0.6029913887345628, + "nauc_map_at_100_max": 0.5234730473183735, + "nauc_map_at_100_std": 0.392993926813872, + "nauc_map_at_10_diff1": 0.6032298753284776, + "nauc_map_at_10_max": 0.523595638946803, + "nauc_map_at_10_std": 0.3908289376631486, + "nauc_map_at_1_diff1": 0.6603749690968012, + "nauc_map_at_1_max": 0.522301229855425, + "nauc_map_at_1_std": 0.35891873761023935, + "nauc_map_at_20_diff1": 0.6019939478294539, + "nauc_map_at_20_max": 0.523271680197658, + "nauc_map_at_20_std": 0.393494827654832, + "nauc_map_at_3_diff1": 0.6097994862720478, + "nauc_map_at_3_max": 0.5226182274555089, + "nauc_map_at_3_std": 0.3865498621153615, + "nauc_map_at_5_diff1": 0.6036483829475393, + "nauc_map_at_5_max": 0.525436647195231, + "nauc_map_at_5_std": 0.387863937482928, + "nauc_mrr_at_1000_diff1": 0.6032366982761342, + "nauc_mrr_at_1000_max": 0.523476478320266, + "nauc_mrr_at_1000_std": 0.3925383463898094, + "nauc_mrr_at_100_diff1": 0.6029905030981351, + "nauc_mrr_at_100_max": 0.523471777559567, + "nauc_mrr_at_100_std": 0.3929924646324689, + "nauc_mrr_at_10_diff1": 0.6032298753284776, + "nauc_mrr_at_10_max": 0.523595638946803, + "nauc_mrr_at_10_std": 0.3908289376631486, + "nauc_mrr_at_1_diff1": 0.6603749690968012, + "nauc_mrr_at_1_max": 0.522301229855425, + "nauc_mrr_at_1_std": 0.35891873761023935, + "nauc_mrr_at_20_diff1": 0.6019939478294539, + "nauc_mrr_at_20_max": 0.523271680197658, + "nauc_mrr_at_20_std": 0.393494827654832, + "nauc_mrr_at_3_diff1": 0.6097994862720478, + "nauc_mrr_at_3_max": 0.5226182274555089, + "nauc_mrr_at_3_std": 0.3865498621153615, + "nauc_mrr_at_5_diff1": 0.6036483829475393, + "nauc_mrr_at_5_max": 0.525436647195231, + "nauc_mrr_at_5_std": 0.387863937482928, + "nauc_ndcg_at_1000_diff1": 0.5905044778644832, + "nauc_ndcg_at_1000_max": 0.5240057340140417, + "nauc_ndcg_at_1000_std": 0.4044954816571764, + "nauc_ndcg_at_100_diff1": 0.5838104029791342, + "nauc_ndcg_at_100_max": 0.5247090283939485, + "nauc_ndcg_at_100_std": 0.41541523853402806, + "nauc_ndcg_at_10_diff1": 0.5807929180968202, + "nauc_ndcg_at_10_max": 0.5235542263682889, + "nauc_ndcg_at_10_std": 0.40815027296979883, + "nauc_ndcg_at_1_diff1": 0.6603749690968012, + "nauc_ndcg_at_1_max": 0.522301229855425, + "nauc_ndcg_at_1_std": 0.35891873761023935, + "nauc_ndcg_at_20_diff1": 0.5759441101599324, + "nauc_ndcg_at_20_max": 0.5221250926205538, + "nauc_ndcg_at_20_std": 0.41829934427865817, + "nauc_ndcg_at_3_diff1": 0.5937567252992616, + "nauc_ndcg_at_3_max": 0.5232237668689338, + "nauc_ndcg_at_3_std": 0.39800682368297036, + "nauc_ndcg_at_5_diff1": 0.5825995713549366, + "nauc_ndcg_at_5_max": 0.527843396831755, + "nauc_ndcg_at_5_std": 0.39994406748733624, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.48098495877756564, + "nauc_precision_at_100_max": 0.540460763885397, + "nauc_precision_at_100_std": 0.5785820711736123, + "nauc_precision_at_10_diff1": 0.5077985610322097, + "nauc_precision_at_10_max": 0.5222008223874395, + "nauc_precision_at_10_std": 0.4673819128352972, + "nauc_precision_at_1_diff1": 0.6603749690968012, + "nauc_precision_at_1_max": 0.522301229855425, + "nauc_precision_at_1_std": 0.35891873761023935, + "nauc_precision_at_20_diff1": 0.4776277222183427, + "nauc_precision_at_20_max": 0.5148646415990584, + "nauc_precision_at_20_std": 0.520179420268066, + "nauc_precision_at_3_diff1": 0.5477584110243353, + "nauc_precision_at_3_max": 0.5251152644999393, + "nauc_precision_at_3_std": 0.431684414718244, + "nauc_precision_at_5_diff1": 0.5197125969701824, + "nauc_precision_at_5_max": 0.5353503012783575, + "nauc_precision_at_5_std": 0.4358979011359394, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.48098495877756514, + "nauc_recall_at_100_max": 0.5404607638853961, + "nauc_recall_at_100_std": 0.5785820711736115, + "nauc_recall_at_10_diff1": 0.5077985610322101, + "nauc_recall_at_10_max": 0.52220082238744, + "nauc_recall_at_10_std": 0.4673819128352976, + "nauc_recall_at_1_diff1": 0.6603749690968012, + "nauc_recall_at_1_max": 0.522301229855425, + "nauc_recall_at_1_std": 0.35891873761023935, + "nauc_recall_at_20_diff1": 0.47762772221834277, + "nauc_recall_at_20_max": 0.5148646415990591, + "nauc_recall_at_20_std": 0.5201794202680657, + "nauc_recall_at_3_diff1": 0.5477584110243355, + "nauc_recall_at_3_max": 0.5251152644999395, + "nauc_recall_at_3_std": 0.43168441471824376, + "nauc_recall_at_5_diff1": 0.5197125969701824, + "nauc_recall_at_5_max": 0.5353503012783575, + "nauc_recall_at_5_std": 0.43589790113593907, + "ndcg_at_1": 0.32316, + "ndcg_at_10": 0.44335, + "ndcg_at_100": 0.49652, + "ndcg_at_1000": 0.51696, + "ndcg_at_20": 0.46084, + "ndcg_at_3": 0.3949, + "ndcg_at_5": 0.41812, + "precision_at_1": 0.32316, + "precision_at_10": 0.05798, + "precision_at_100": 0.00845, + "precision_at_1000": 0.001, + "precision_at_20": 0.03245, + "precision_at_3": 0.1484, + "precision_at_5": 0.10042, + "recall_at_1": 0.32316, + "recall_at_10": 0.57975, + "recall_at_100": 0.84466, + "recall_at_1000": 1.0, + "recall_at_20": 0.6491, + "recall_at_3": 0.44521, + "recall_at_5": 0.50208 + }, + { + "cv_recall_at_1": 0.31712, + "cv_recall_at_10": 0.56237, + "cv_recall_at_100": 0.83827, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.64588, + "cv_recall_at_3": 0.43869, + "cv_recall_at_5": 0.48414, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.43086, + "map_at_1": 0.31712, + "map_at_10": 0.38985, + "map_at_100": 0.40032, + "map_at_1000": 0.40108, + "map_at_20": 0.39546, + "map_at_3": 0.36892, + "map_at_5": 0.37949, + "mrr_at_1": 0.3171247357293869, + "mrr_at_10": 0.389854944796805, + "mrr_at_100": 0.4003152729755614, + "mrr_at_1000": 0.40108303021116537, + "mrr_at_20": 0.39545539710970634, + "mrr_at_3": 0.3689217758985197, + "mrr_at_5": 0.37949260042283245, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.39181767995617045, + "nauc_cv_recall_at_100_max": 0.4050520733318427, + "nauc_cv_recall_at_100_std": 0.5990584601132213, + "nauc_cv_recall_at_10_diff1": 0.46942265389779947, + "nauc_cv_recall_at_10_max": 0.37702464329644436, + "nauc_cv_recall_at_10_std": 0.4060088764375527, + "nauc_cv_recall_at_1_diff1": 0.6636784835703272, + "nauc_cv_recall_at_1_max": 0.47860715314683555, + "nauc_cv_recall_at_1_std": 0.3351963299597727, + "nauc_cv_recall_at_20_diff1": 0.4105037645463755, + "nauc_cv_recall_at_20_max": 0.35846333599786645, + "nauc_cv_recall_at_20_std": 0.42470469111912557, + "nauc_cv_recall_at_3_diff1": 0.5252010506154087, + "nauc_cv_recall_at_3_max": 0.41922611748318617, + "nauc_cv_recall_at_3_std": 0.37931068318115596, + "nauc_cv_recall_at_5_diff1": 0.5043643680663037, + "nauc_cv_recall_at_5_max": 0.3949210102081491, + "nauc_cv_recall_at_5_std": 0.37065381352919147, + "nauc_map_at_1000_diff1": 0.594113403077539, + "nauc_map_at_1000_max": 0.44654400940153666, + "nauc_map_at_1000_std": 0.3592820692140759, + "nauc_map_at_100_diff1": 0.5936780021007206, + "nauc_map_at_100_max": 0.4464329940626086, + "nauc_map_at_100_std": 0.35973466746408383, + "nauc_map_at_10_diff1": 0.5950245929669957, + "nauc_map_at_10_max": 0.4454913141343116, + "nauc_map_at_10_std": 0.35727519098134763, + "nauc_map_at_1_diff1": 0.6636784835703272, + "nauc_map_at_1_max": 0.47860715314683555, + "nauc_map_at_1_std": 0.3351963299597727, + "nauc_map_at_20_diff1": 0.592494503602588, + "nauc_map_at_20_max": 0.44468202478683205, + "nauc_map_at_20_std": 0.35763958254326716, + "nauc_map_at_3_diff1": 0.6028648903906949, + "nauc_map_at_3_max": 0.4523138001749844, + "nauc_map_at_3_std": 0.3546353095693066, + "nauc_map_at_5_diff1": 0.5985843487767859, + "nauc_map_at_5_max": 0.4471703848852369, + "nauc_map_at_5_std": 0.3528030244263197, + "nauc_mrr_at_1000_diff1": 0.5941131184415618, + "nauc_mrr_at_1000_max": 0.4465442760720423, + "nauc_mrr_at_1000_std": 0.3592823694304114, + "nauc_mrr_at_100_diff1": 0.5936777192516401, + "nauc_mrr_at_100_max": 0.4464332457865363, + "nauc_mrr_at_100_std": 0.3597349501870047, + "nauc_mrr_at_10_diff1": 0.5950245929669957, + "nauc_mrr_at_10_max": 0.4454913141343116, + "nauc_mrr_at_10_std": 0.35727519098134763, + "nauc_mrr_at_1_diff1": 0.6636784835703272, + "nauc_mrr_at_1_max": 0.47860715314683555, + "nauc_mrr_at_1_std": 0.3351963299597727, + "nauc_mrr_at_20_diff1": 0.592494503602588, + "nauc_mrr_at_20_max": 0.44468202478683205, + "nauc_mrr_at_20_std": 0.35763958254326716, + "nauc_mrr_at_3_diff1": 0.6028648903906949, + "nauc_mrr_at_3_max": 0.4523138001749844, + "nauc_mrr_at_3_std": 0.3546353095693066, + "nauc_mrr_at_5_diff1": 0.5985843487767859, + "nauc_mrr_at_5_max": 0.4471703848852369, + "nauc_mrr_at_5_std": 0.3528030244263197, + "nauc_ndcg_at_1000_diff1": 0.5762392937292321, + "nauc_ndcg_at_1000_max": 0.43979278940966154, + "nauc_ndcg_at_1000_std": 0.37028794726047176, + "nauc_ndcg_at_100_diff1": 0.5645324093122328, + "nauc_ndcg_at_100_max": 0.43743526699578555, + "nauc_ndcg_at_100_std": 0.3843072006940757, + "nauc_ndcg_at_10_diff1": 0.565573536978437, + "nauc_ndcg_at_10_max": 0.42963997269770055, + "nauc_ndcg_at_10_std": 0.36832076534969604, + "nauc_ndcg_at_1_diff1": 0.6636784835703272, + "nauc_ndcg_at_1_max": 0.47860715314683555, + "nauc_ndcg_at_1_std": 0.3351963299597727, + "nauc_ndcg_at_20_diff1": 0.555405981896656, + "nauc_ndcg_at_20_max": 0.42678212460430726, + "nauc_ndcg_at_20_std": 0.37084255862441984, + "nauc_ndcg_at_3_diff1": 0.5827884096507694, + "nauc_ndcg_at_3_max": 0.4437515122638289, + "nauc_ndcg_at_3_std": 0.3610167467861188, + "nauc_ndcg_at_5_diff1": 0.5747593673859045, + "nauc_ndcg_at_5_max": 0.4341592635536307, + "nauc_ndcg_at_5_std": 0.3575397919597607, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.39181767995617117, + "nauc_precision_at_100_max": 0.405052073331843, + "nauc_precision_at_100_std": 0.5990584601132215, + "nauc_precision_at_10_diff1": 0.4694226538977993, + "nauc_precision_at_10_max": 0.3770246432964444, + "nauc_precision_at_10_std": 0.4060088764375527, + "nauc_precision_at_1_diff1": 0.6636784835703272, + "nauc_precision_at_1_max": 0.47860715314683555, + "nauc_precision_at_1_std": 0.3351963299597727, + "nauc_precision_at_20_diff1": 0.4105037645463753, + "nauc_precision_at_20_max": 0.3584633359978659, + "nauc_precision_at_20_std": 0.4247046911191254, + "nauc_precision_at_3_diff1": 0.5252010506154082, + "nauc_precision_at_3_max": 0.419226117483186, + "nauc_precision_at_3_std": 0.37931068318115574, + "nauc_precision_at_5_diff1": 0.5043643680663035, + "nauc_precision_at_5_max": 0.394921010208149, + "nauc_precision_at_5_std": 0.3706538135291907, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.39181767995617045, + "nauc_recall_at_100_max": 0.4050520733318427, + "nauc_recall_at_100_std": 0.5990584601132213, + "nauc_recall_at_10_diff1": 0.46942265389779947, + "nauc_recall_at_10_max": 0.37702464329644436, + "nauc_recall_at_10_std": 0.4060088764375527, + "nauc_recall_at_1_diff1": 0.6636784835703272, + "nauc_recall_at_1_max": 0.47860715314683555, + "nauc_recall_at_1_std": 0.3351963299597727, + "nauc_recall_at_20_diff1": 0.4105037645463755, + "nauc_recall_at_20_max": 0.35846333599786645, + "nauc_recall_at_20_std": 0.42470469111912557, + "nauc_recall_at_3_diff1": 0.5252010506154087, + "nauc_recall_at_3_max": 0.41922611748318617, + "nauc_recall_at_3_std": 0.37931068318115596, + "nauc_recall_at_5_diff1": 0.5043643680663037, + "nauc_recall_at_5_max": 0.3949210102081491, + "nauc_recall_at_5_std": 0.37065381352919147, + "ndcg_at_1": 0.31712, + "ndcg_at_10": 0.43086, + "ndcg_at_100": 0.4872, + "ndcg_at_1000": 0.50779, + "ndcg_at_20": 0.45174, + "ndcg_at_3": 0.38676, + "ndcg_at_5": 0.40565, + "precision_at_1": 0.31712, + "precision_at_10": 0.05624, + "precision_at_100": 0.00838, + "precision_at_1000": 0.001, + "precision_at_20": 0.03229, + "precision_at_3": 0.14623, + "precision_at_5": 0.09683, + "recall_at_1": 0.31712, + "recall_at_10": 0.56237, + "recall_at_100": 0.83827, + "recall_at_1000": 1.0, + "recall_at_20": 0.64588, + "recall_at_3": 0.43869, + "recall_at_5": 0.48414 + }, + { + "cv_recall_at_1": 0.495, + "cv_recall_at_10": 0.78, + "cv_recall_at_100": 0.979, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.841, + "cv_recall_at_3": 0.655, + "cv_recall_at_5": 0.717, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.63387, + "map_at_1": 0.495, + "map_at_10": 0.58739, + "map_at_100": 0.59567, + "map_at_1000": 0.59578, + "map_at_20": 0.59173, + "map_at_3": 0.56467, + "map_at_5": 0.57897, + "mrr_at_1": 0.495, + "mrr_at_10": 0.5873928571428575, + "mrr_at_100": 0.5956652007256305, + "mrr_at_1000": 0.5957836523327847, + "mrr_at_20": 0.5917261944453897, + "mrr_at_3": 0.5646666666666669, + "mrr_at_5": 0.5789666666666671, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6082655284335997, + "nauc_cv_recall_at_100_max": 0.7598594993552958, + "nauc_cv_recall_at_100_std": 0.8843982037259329, + "nauc_cv_recall_at_10_diff1": 0.6816641164742425, + "nauc_cv_recall_at_10_max": 0.2725900681596879, + "nauc_cv_recall_at_10_std": -0.19088885797746538, + "nauc_cv_recall_at_1_diff1": 0.7404600865904506, + "nauc_cv_recall_at_1_max": 0.4009643329808529, + "nauc_cv_recall_at_1_std": 0.013753383478648351, + "nauc_cv_recall_at_20_diff1": 0.6667880441155599, + "nauc_cv_recall_at_20_max": 0.3064266273321445, + "nauc_cv_recall_at_20_std": -0.1325410883236954, + "nauc_cv_recall_at_3_diff1": 0.6830957330550604, + "nauc_cv_recall_at_3_max": 0.30237216831138086, + "nauc_cv_recall_at_3_std": -0.11962728938481404, + "nauc_cv_recall_at_5_diff1": 0.6895788774983411, + "nauc_cv_recall_at_5_max": 0.29256752540180514, + "nauc_cv_recall_at_5_std": -0.1600261724164667, + "nauc_map_at_1000_diff1": 0.7197162304227843, + "nauc_map_at_1000_max": 0.36638095382714386, + "nauc_map_at_1000_std": -0.03260887184687692, + "nauc_map_at_100_diff1": 0.7196837015487083, + "nauc_map_at_100_max": 0.36653962854973204, + "nauc_map_at_100_std": -0.032241605111186684, + "nauc_map_at_10_diff1": 0.7192984727699232, + "nauc_map_at_10_max": 0.3626920886070452, + "nauc_map_at_10_std": -0.03897821506531741, + "nauc_map_at_1_diff1": 0.7404600865904506, + "nauc_map_at_1_max": 0.4009643329808529, + "nauc_map_at_1_std": 0.013753383478648351, + "nauc_map_at_20_diff1": 0.7191499846305851, + "nauc_map_at_20_max": 0.3649792771195435, + "nauc_map_at_20_std": -0.03538358395201348, + "nauc_map_at_3_diff1": 0.7172464691095869, + "nauc_map_at_3_max": 0.36403675266954516, + "nauc_map_at_3_std": -0.03286408950285727, + "nauc_map_at_5_diff1": 0.7194128350803819, + "nauc_map_at_5_max": 0.3632676565831752, + "nauc_map_at_5_std": -0.03783915587541209, + "nauc_mrr_at_1000_diff1": 0.7197162304227843, + "nauc_mrr_at_1000_max": 0.36638095382714386, + "nauc_mrr_at_1000_std": -0.03260887184687692, + "nauc_mrr_at_100_diff1": 0.7196837015487083, + "nauc_mrr_at_100_max": 0.36653962854973204, + "nauc_mrr_at_100_std": -0.032241605111186684, + "nauc_mrr_at_10_diff1": 0.7192984727699232, + "nauc_mrr_at_10_max": 0.3626920886070452, + "nauc_mrr_at_10_std": -0.03897821506531741, + "nauc_mrr_at_1_diff1": 0.7404600865904506, + "nauc_mrr_at_1_max": 0.4009643329808529, + "nauc_mrr_at_1_std": 0.013753383478648351, + "nauc_mrr_at_20_diff1": 0.7191499846305851, + "nauc_mrr_at_20_max": 0.3649792771195435, + "nauc_mrr_at_20_std": -0.03538358395201348, + "nauc_mrr_at_3_diff1": 0.7172464691095869, + "nauc_mrr_at_3_max": 0.36403675266954516, + "nauc_mrr_at_3_std": -0.03286408950285727, + "nauc_mrr_at_5_diff1": 0.7194128350803819, + "nauc_mrr_at_5_max": 0.3632676565831752, + "nauc_mrr_at_5_std": -0.03783915587541209, + "nauc_ndcg_at_1000_diff1": 0.7161717872946759, + "nauc_ndcg_at_1000_max": 0.36240087391615494, + "nauc_ndcg_at_1000_std": -0.03802915968796838, + "nauc_ndcg_at_100_diff1": 0.7150293653440268, + "nauc_ndcg_at_100_max": 0.3669197218892507, + "nauc_ndcg_at_100_std": -0.02753279118061936, + "nauc_ndcg_at_10_diff1": 0.7122536087594573, + "nauc_ndcg_at_10_max": 0.346227943924156, + "nauc_ndcg_at_10_std": -0.06629529516462444, + "nauc_ndcg_at_1_diff1": 0.7404600865904506, + "nauc_ndcg_at_1_max": 0.4009643329808529, + "nauc_ndcg_at_1_std": 0.013753383478648351, + "nauc_ndcg_at_20_diff1": 0.7114885578859922, + "nauc_ndcg_at_20_max": 0.3545982292278816, + "nauc_ndcg_at_20_std": -0.05241597538665322, + "nauc_ndcg_at_3_diff1": 0.7092500802086553, + "nauc_ndcg_at_3_max": 0.34973063708805896, + "nauc_ndcg_at_3_std": -0.052851060304107446, + "nauc_ndcg_at_5_diff1": 0.7128843624704241, + "nauc_ndcg_at_5_max": 0.34819332706378014, + "nauc_ndcg_at_5_std": -0.06327695695795105, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.6082655284335963, + "nauc_precision_at_100_max": 0.7598594993552936, + "nauc_precision_at_100_std": 0.8843982037259326, + "nauc_precision_at_10_diff1": 0.6816641164742427, + "nauc_precision_at_10_max": 0.27259006815968745, + "nauc_precision_at_10_std": -0.19088885797746732, + "nauc_precision_at_1_diff1": 0.7404600865904506, + "nauc_precision_at_1_max": 0.4009643329808529, + "nauc_precision_at_1_std": 0.013753383478648351, + "nauc_precision_at_20_diff1": 0.6667880441155589, + "nauc_precision_at_20_max": 0.3064266273321421, + "nauc_precision_at_20_std": -0.13254108832369677, + "nauc_precision_at_3_diff1": 0.6830957330550611, + "nauc_precision_at_3_max": 0.3023721683113811, + "nauc_precision_at_3_std": -0.11962728938481379, + "nauc_precision_at_5_diff1": 0.6895788774983402, + "nauc_precision_at_5_max": 0.2925675254018038, + "nauc_precision_at_5_std": -0.16002617241646783, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.6082655284335997, + "nauc_recall_at_100_max": 0.7598594993552958, + "nauc_recall_at_100_std": 0.8843982037259329, + "nauc_recall_at_10_diff1": 0.6816641164742425, + "nauc_recall_at_10_max": 0.2725900681596879, + "nauc_recall_at_10_std": -0.19088885797746538, + "nauc_recall_at_1_diff1": 0.7404600865904506, + "nauc_recall_at_1_max": 0.4009643329808529, + "nauc_recall_at_1_std": 0.013753383478648351, + "nauc_recall_at_20_diff1": 0.6667880441155599, + "nauc_recall_at_20_max": 0.3064266273321445, + "nauc_recall_at_20_std": -0.1325410883236954, + "nauc_recall_at_3_diff1": 0.6830957330550604, + "nauc_recall_at_3_max": 0.30237216831138086, + "nauc_recall_at_3_std": -0.11962728938481404, + "nauc_recall_at_5_diff1": 0.6895788774983411, + "nauc_recall_at_5_max": 0.29256752540180514, + "nauc_recall_at_5_std": -0.1600261724164667, + "ndcg_at_1": 0.495, + "ndcg_at_10": 0.63387, + "ndcg_at_100": 0.67586, + "ndcg_at_1000": 0.67865, + "ndcg_at_20": 0.64941, + "ndcg_at_3": 0.58783, + "ndcg_at_5": 0.61348, + "precision_at_1": 0.495, + "precision_at_10": 0.078, + "precision_at_100": 0.00979, + "precision_at_1000": 0.001, + "precision_at_20": 0.04205, + "precision_at_3": 0.21833, + "precision_at_5": 0.1434, + "recall_at_1": 0.495, + "recall_at_10": 0.78, + "recall_at_100": 0.979, + "recall_at_1000": 1.0, + "recall_at_20": 0.841, + "recall_at_3": 0.655, + "recall_at_5": 0.717 + } + ] + }, + "task_name": "WITT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/model_meta.json b/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/model_meta.json new file mode 100644 index 0000000000..a276a48179 --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/model_meta.json @@ -0,0 +1 @@ +{"name": "google/siglip-base-patch16-384", "revision": "41aec1c83b32e0a6fca20ad88ba058aa5b5ea394", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/WITT2IRetrieval.json b/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/WITT2IRetrieval.json new file mode 100644 index 0000000000..7523e09222 --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/WITT2IRetrieval.json @@ -0,0 +1,1936 @@ +{ + "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", + "evaluation_time": 234.74904417991638, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.14944, + "cv_recall_at_10": 0.33034, + "cv_recall_at_100": 0.66517, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.42247, + "cv_recall_at_3": 0.2, + "cv_recall_at_5": 0.24157, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.22414, + "map_at_1": 0.14944, + "map_at_10": 0.19231, + "map_at_100": 0.20432, + "map_at_1000": 0.2058, + "map_at_20": 0.19873, + "map_at_3": 0.17135, + "map_at_5": 0.18079, + "mrr_at_1": 0.149438202247191, + "mrr_at_10": 0.1923136258248617, + "mrr_at_100": 0.20431598751440663, + "mrr_at_1000": 0.2057957434184432, + "mrr_at_20": 0.19872819244758055, + "mrr_at_3": 0.17134831460674158, + "mrr_at_5": 0.18078651685393257, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2634787391962865, + "nauc_cv_recall_at_100_max": 0.4645672109653449, + "nauc_cv_recall_at_100_std": 0.48648923584685244, + "nauc_cv_recall_at_10_diff1": 0.4120188813008714, + "nauc_cv_recall_at_10_max": 0.39580884178534775, + "nauc_cv_recall_at_10_std": 0.3375481230874158, + "nauc_cv_recall_at_1_diff1": 0.6461541257146491, + "nauc_cv_recall_at_1_max": 0.5172890010471299, + "nauc_cv_recall_at_1_std": 0.3714290257148764, + "nauc_cv_recall_at_20_diff1": 0.3539802242264886, + "nauc_cv_recall_at_20_max": 0.3648519440541502, + "nauc_cv_recall_at_20_std": 0.3389938958623623, + "nauc_cv_recall_at_3_diff1": 0.525914894080172, + "nauc_cv_recall_at_3_max": 0.44017407204507436, + "nauc_cv_recall_at_3_std": 0.340840900762971, + "nauc_cv_recall_at_5_diff1": 0.5011352099676971, + "nauc_cv_recall_at_5_max": 0.4099173553719009, + "nauc_cv_recall_at_5_std": 0.34841297143096883, + "nauc_map_at_1000_diff1": 0.555169070381733, + "nauc_map_at_1000_max": 0.4631488311423136, + "nauc_map_at_1000_std": 0.36157268097851425, + "nauc_map_at_100_diff1": 0.5539155701362769, + "nauc_map_at_100_max": 0.46292444260745313, + "nauc_map_at_100_std": 0.36168212855581966, + "nauc_map_at_10_diff1": 0.5602553663022992, + "nauc_map_at_10_max": 0.46339619753852673, + "nauc_map_at_10_std": 0.3583151485756733, + "nauc_map_at_1_diff1": 0.6461541257146491, + "nauc_map_at_1_max": 0.5172890010471299, + "nauc_map_at_1_std": 0.3714290257148764, + "nauc_map_at_20_diff1": 0.5545223104794634, + "nauc_map_at_20_max": 0.460308754159618, + "nauc_map_at_20_std": 0.3583429353435337, + "nauc_map_at_3_diff1": 0.5873248593499638, + "nauc_map_at_3_max": 0.4774872915845461, + "nauc_map_at_3_std": 0.3582007033781603, + "nauc_map_at_5_diff1": 0.5791085109357881, + "nauc_map_at_5_max": 0.468387528653673, + "nauc_map_at_5_std": 0.35930898584402726, + "nauc_mrr_at_1000_diff1": 0.555169070381733, + "nauc_mrr_at_1000_max": 0.4631488311423136, + "nauc_mrr_at_1000_std": 0.36157268097851425, + "nauc_mrr_at_100_diff1": 0.5539155701362769, + "nauc_mrr_at_100_max": 0.46292444260745313, + "nauc_mrr_at_100_std": 0.36168212855581966, + "nauc_mrr_at_10_diff1": 0.5602553663022992, + "nauc_mrr_at_10_max": 0.46339619753852673, + "nauc_mrr_at_10_std": 0.3583151485756733, + "nauc_mrr_at_1_diff1": 0.6461541257146491, + "nauc_mrr_at_1_max": 0.5172890010471299, + "nauc_mrr_at_1_std": 0.3714290257148764, + "nauc_mrr_at_20_diff1": 0.5545223104794634, + "nauc_mrr_at_20_max": 0.460308754159618, + "nauc_mrr_at_20_std": 0.3583429353435337, + "nauc_mrr_at_3_diff1": 0.5873248593499638, + "nauc_mrr_at_3_max": 0.4774872915845461, + "nauc_mrr_at_3_std": 0.3582007033781603, + "nauc_mrr_at_5_diff1": 0.5791085109357881, + "nauc_mrr_at_5_max": 0.468387528653673, + "nauc_mrr_at_5_std": 0.35930898584402726, + "nauc_ndcg_at_1000_diff1": 0.5222129913095555, + "nauc_ndcg_at_1000_max": 0.4535832261680661, + "nauc_ndcg_at_1000_std": 0.36699755338097423, + "nauc_ndcg_at_100_diff1": 0.4898715477765656, + "nauc_ndcg_at_100_max": 0.45382252122114575, + "nauc_ndcg_at_100_std": 0.37968453864410084, + "nauc_ndcg_at_10_diff1": 0.5175455563050221, + "nauc_ndcg_at_10_max": 0.4430790495975126, + "nauc_ndcg_at_10_std": 0.3524409492818574, + "nauc_ndcg_at_1_diff1": 0.6461541257146491, + "nauc_ndcg_at_1_max": 0.5172890010471299, + "nauc_ndcg_at_1_std": 0.3714290257148764, + "nauc_ndcg_at_20_diff1": 0.49847417961819085, + "nauc_ndcg_at_20_max": 0.432915540811886, + "nauc_ndcg_at_20_std": 0.3527195903365103, + "nauc_ndcg_at_3_diff1": 0.5701266655769082, + "nauc_ndcg_at_3_max": 0.4669535549260126, + "nauc_ndcg_at_3_std": 0.3534094141369618, + "nauc_ndcg_at_5_diff1": 0.5567781257664445, + "nauc_ndcg_at_5_max": 0.4517477421856985, + "nauc_ndcg_at_5_std": 0.3560158696137673, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.26347873919628667, + "nauc_precision_at_100_max": 0.46456721096534553, + "nauc_precision_at_100_std": 0.48648923584685333, + "nauc_precision_at_10_diff1": 0.41201888130087116, + "nauc_precision_at_10_max": 0.39580884178534803, + "nauc_precision_at_10_std": 0.3375481230874158, + "nauc_precision_at_1_diff1": 0.6461541257146491, + "nauc_precision_at_1_max": 0.5172890010471299, + "nauc_precision_at_1_std": 0.3714290257148764, + "nauc_precision_at_20_diff1": 0.35398022422648834, + "nauc_precision_at_20_max": 0.36485194405414995, + "nauc_precision_at_20_std": 0.33899389586236256, + "nauc_precision_at_3_diff1": 0.5259148940801714, + "nauc_precision_at_3_max": 0.44017407204507436, + "nauc_precision_at_3_std": 0.3408409007629708, + "nauc_precision_at_5_diff1": 0.5011352099676972, + "nauc_precision_at_5_max": 0.40991735537190055, + "nauc_precision_at_5_std": 0.3484129714309687, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2634787391962865, + "nauc_recall_at_100_max": 0.4645672109653449, + "nauc_recall_at_100_std": 0.48648923584685244, + "nauc_recall_at_10_diff1": 0.4120188813008714, + "nauc_recall_at_10_max": 0.39580884178534775, + "nauc_recall_at_10_std": 0.3375481230874158, + "nauc_recall_at_1_diff1": 0.6461541257146491, + "nauc_recall_at_1_max": 0.5172890010471299, + "nauc_recall_at_1_std": 0.3714290257148764, + "nauc_recall_at_20_diff1": 0.3539802242264886, + "nauc_recall_at_20_max": 0.3648519440541502, + "nauc_recall_at_20_std": 0.3389938958623623, + "nauc_recall_at_3_diff1": 0.525914894080172, + "nauc_recall_at_3_max": 0.44017407204507436, + "nauc_recall_at_3_std": 0.340840900762971, + "nauc_recall_at_5_diff1": 0.5011352099676971, + "nauc_recall_at_5_max": 0.4099173553719009, + "nauc_recall_at_5_std": 0.34841297143096883, + "ndcg_at_1": 0.14944, + "ndcg_at_10": 0.22414, + "ndcg_at_100": 0.2912, + "ndcg_at_1000": 0.33314, + "ndcg_at_20": 0.24746, + "ndcg_at_3": 0.17869, + "ndcg_at_5": 0.19576, + "precision_at_1": 0.14944, + "precision_at_10": 0.03303, + "precision_at_100": 0.00665, + "precision_at_1000": 0.001, + "precision_at_20": 0.02112, + "precision_at_3": 0.06667, + "precision_at_5": 0.04831, + "recall_at_1": 0.14944, + "recall_at_10": 0.33034, + "recall_at_100": 0.66517, + "recall_at_1000": 1.0, + "recall_at_20": 0.42247, + "recall_at_3": 0.2, + "recall_at_5": 0.24157 + }, + { + "cv_recall_at_1": 0.15698, + "cv_recall_at_10": 0.31977, + "cv_recall_at_100": 0.69535, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.40465, + "cv_recall_at_3": 0.21163, + "cv_recall_at_5": 0.25465, + "hf_subset": "bg", + "languages": [ + "bul-Cyrl" + ], + "main_score": 0.22603, + "map_at_1": 0.15698, + "map_at_10": 0.19746, + "map_at_100": 0.2101, + "map_at_1000": 0.21161, + "map_at_20": 0.20326, + "map_at_3": 0.17888, + "map_at_5": 0.1887, + "mrr_at_1": 0.1569767441860465, + "mrr_at_10": 0.1974617017349575, + "mrr_at_100": 0.21009563393962438, + "mrr_at_1000": 0.21160873316997192, + "mrr_at_20": 0.20325988139847134, + "mrr_at_3": 0.17887596899224803, + "mrr_at_5": 0.1887015503875969, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.3363919574226251, + "nauc_cv_recall_at_100_max": 0.37234669199849013, + "nauc_cv_recall_at_100_std": 0.3986816943007544, + "nauc_cv_recall_at_10_diff1": 0.3841938620716482, + "nauc_cv_recall_at_10_max": 0.1668038430121746, + "nauc_cv_recall_at_10_std": 0.12322377377572048, + "nauc_cv_recall_at_1_diff1": 0.6067466385745514, + "nauc_cv_recall_at_1_max": 0.29588926269525695, + "nauc_cv_recall_at_1_std": 0.15656596549653945, + "nauc_cv_recall_at_20_diff1": 0.3517457376056227, + "nauc_cv_recall_at_20_max": 0.15635326148882445, + "nauc_cv_recall_at_20_std": 0.10895267737737557, + "nauc_cv_recall_at_3_diff1": 0.49341354079088906, + "nauc_cv_recall_at_3_max": 0.1917544432899988, + "nauc_cv_recall_at_3_std": 0.10965683592961223, + "nauc_cv_recall_at_5_diff1": 0.45467439639098584, + "nauc_cv_recall_at_5_max": 0.20165656339442797, + "nauc_cv_recall_at_5_std": 0.13592845763496914, + "nauc_map_at_1000_diff1": 0.5287548795852088, + "nauc_map_at_1000_max": 0.24669405029122785, + "nauc_map_at_1000_std": 0.14398072701997205, + "nauc_map_at_100_diff1": 0.5281431111016485, + "nauc_map_at_100_max": 0.24680733420777262, + "nauc_map_at_100_std": 0.144483705635419, + "nauc_map_at_10_diff1": 0.5305009981241716, + "nauc_map_at_10_max": 0.2434189183238275, + "nauc_map_at_10_std": 0.14042474981401215, + "nauc_map_at_1_diff1": 0.6067466385745514, + "nauc_map_at_1_max": 0.29588926269525695, + "nauc_map_at_1_std": 0.15656596549653945, + "nauc_map_at_20_diff1": 0.5267607229546266, + "nauc_map_at_20_max": 0.24250663560859556, + "nauc_map_at_20_std": 0.13913708344341796, + "nauc_map_at_3_diff1": 0.5560856112730492, + "nauc_map_at_3_max": 0.24808399445761734, + "nauc_map_at_3_std": 0.13552614213434122, + "nauc_map_at_5_diff1": 0.5450619717717616, + "nauc_map_at_5_max": 0.2493579314145244, + "nauc_map_at_5_std": 0.14285677163632696, + "nauc_mrr_at_1000_diff1": 0.5287548970738191, + "nauc_mrr_at_1000_max": 0.2466940493792957, + "nauc_mrr_at_1000_std": 0.14398070943942484, + "nauc_mrr_at_100_diff1": 0.5281431111016485, + "nauc_mrr_at_100_max": 0.24680733420777262, + "nauc_mrr_at_100_std": 0.144483705635419, + "nauc_mrr_at_10_diff1": 0.5305009981241716, + "nauc_mrr_at_10_max": 0.2434189183238275, + "nauc_mrr_at_10_std": 0.14042474981401215, + "nauc_mrr_at_1_diff1": 0.6067466385745514, + "nauc_mrr_at_1_max": 0.29588926269525695, + "nauc_mrr_at_1_std": 0.15656596549653945, + "nauc_mrr_at_20_diff1": 0.5267607229546266, + "nauc_mrr_at_20_max": 0.24250663560859556, + "nauc_mrr_at_20_std": 0.13913708344341796, + "nauc_mrr_at_3_diff1": 0.5560856112730492, + "nauc_mrr_at_3_max": 0.24808399445761734, + "nauc_mrr_at_3_std": 0.13552614213434122, + "nauc_mrr_at_5_diff1": 0.5450619717717616, + "nauc_mrr_at_5_max": 0.2493579314145244, + "nauc_mrr_at_5_std": 0.14285677163632696, + "nauc_ndcg_at_1000_diff1": 0.5022253943978927, + "nauc_ndcg_at_1000_max": 0.24300420710339146, + "nauc_ndcg_at_1000_std": 0.15266396324514417, + "nauc_ndcg_at_100_diff1": 0.4836824082066605, + "nauc_ndcg_at_100_max": 0.2563419064021609, + "nauc_ndcg_at_100_std": 0.1788962463598874, + "nauc_ndcg_at_10_diff1": 0.4890532105795847, + "nauc_ndcg_at_10_max": 0.22147463946489696, + "nauc_ndcg_at_10_std": 0.1353636012280422, + "nauc_ndcg_at_1_diff1": 0.6067466385745514, + "nauc_ndcg_at_1_max": 0.29588926269525695, + "nauc_ndcg_at_1_std": 0.15656596549653945, + "nauc_ndcg_at_20_diff1": 0.4774594920142693, + "nauc_ndcg_at_20_max": 0.2179566948337125, + "nauc_ndcg_at_20_std": 0.13087037961967188, + "nauc_ndcg_at_3_diff1": 0.5387369747763605, + "nauc_ndcg_at_3_max": 0.2324434366269725, + "nauc_ndcg_at_3_std": 0.12836241066655732, + "nauc_ndcg_at_5_diff1": 0.5196578856648923, + "nauc_ndcg_at_5_max": 0.2354217175402108, + "nauc_ndcg_at_5_std": 0.14060419032243368, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3363919574226251, + "nauc_precision_at_100_max": 0.37234669199849024, + "nauc_precision_at_100_std": 0.3986816943007548, + "nauc_precision_at_10_diff1": 0.38419386207164824, + "nauc_precision_at_10_max": 0.16680384301217463, + "nauc_precision_at_10_std": 0.12322377377572062, + "nauc_precision_at_1_diff1": 0.6067466385745514, + "nauc_precision_at_1_max": 0.29588926269525695, + "nauc_precision_at_1_std": 0.15656596549653945, + "nauc_precision_at_20_diff1": 0.35174573760562283, + "nauc_precision_at_20_max": 0.15635326148882472, + "nauc_precision_at_20_std": 0.1089526773773756, + "nauc_precision_at_3_diff1": 0.4934135407908889, + "nauc_precision_at_3_max": 0.19175444328999902, + "nauc_precision_at_3_std": 0.10965683592961223, + "nauc_precision_at_5_diff1": 0.4546743963909857, + "nauc_precision_at_5_max": 0.20165656339442822, + "nauc_precision_at_5_std": 0.1359284576349691, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.3363919574226251, + "nauc_recall_at_100_max": 0.37234669199849013, + "nauc_recall_at_100_std": 0.3986816943007544, + "nauc_recall_at_10_diff1": 0.3841938620716482, + "nauc_recall_at_10_max": 0.1668038430121746, + "nauc_recall_at_10_std": 0.12322377377572048, + "nauc_recall_at_1_diff1": 0.6067466385745514, + "nauc_recall_at_1_max": 0.29588926269525695, + "nauc_recall_at_1_std": 0.15656596549653945, + "nauc_recall_at_20_diff1": 0.3517457376056227, + "nauc_recall_at_20_max": 0.15635326148882445, + "nauc_recall_at_20_std": 0.10895267737737557, + "nauc_recall_at_3_diff1": 0.49341354079088906, + "nauc_recall_at_3_max": 0.1917544432899988, + "nauc_recall_at_3_std": 0.10965683592961223, + "nauc_recall_at_5_diff1": 0.45467439639098584, + "nauc_recall_at_5_max": 0.20165656339442797, + "nauc_recall_at_5_std": 0.13592845763496914, + "ndcg_at_1": 0.15698, + "ndcg_at_10": 0.22603, + "ndcg_at_100": 0.30011, + "ndcg_at_1000": 0.33915, + "ndcg_at_20": 0.24738, + "ndcg_at_3": 0.18719, + "ndcg_at_5": 0.20491, + "precision_at_1": 0.15698, + "precision_at_10": 0.03198, + "precision_at_100": 0.00695, + "precision_at_1000": 0.001, + "precision_at_20": 0.02023, + "precision_at_3": 0.07054, + "precision_at_5": 0.05093, + "recall_at_1": 0.15698, + "recall_at_10": 0.31977, + "recall_at_100": 0.69535, + "recall_at_1000": 1.0, + "recall_at_20": 0.40465, + "recall_at_3": 0.21163, + "recall_at_5": 0.25465 + }, + { + "cv_recall_at_1": 0.3266, + "cv_recall_at_10": 0.58025, + "cv_recall_at_100": 0.80808, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.65208, + "cv_recall_at_3": 0.43098, + "cv_recall_at_5": 0.48822, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.44195, + "map_at_1": 0.3266, + "map_at_10": 0.39925, + "map_at_100": 0.408, + "map_at_1000": 0.4089, + "map_at_20": 0.40416, + "map_at_3": 0.37374, + "map_at_5": 0.38698, + "mrr_at_1": 0.3265993265993266, + "mrr_at_10": 0.39925222240037056, + "mrr_at_100": 0.40800305571393053, + "mrr_at_1000": 0.4088972381164094, + "mrr_at_20": 0.40416376707982127, + "mrr_at_3": 0.3737373737373736, + "mrr_at_5": 0.38698092031425335, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.40351361544537445, + "nauc_cv_recall_at_100_max": 0.5912530427570891, + "nauc_cv_recall_at_100_std": 0.5493433744386865, + "nauc_cv_recall_at_10_diff1": 0.4944214665489543, + "nauc_cv_recall_at_10_max": 0.4785322813825731, + "nauc_cv_recall_at_10_std": 0.3430216902502382, + "nauc_cv_recall_at_1_diff1": 0.6965218420770751, + "nauc_cv_recall_at_1_max": 0.5148871193449024, + "nauc_cv_recall_at_1_std": 0.24287634889441037, + "nauc_cv_recall_at_20_diff1": 0.46800885812695175, + "nauc_cv_recall_at_20_max": 0.5306725749263748, + "nauc_cv_recall_at_20_std": 0.41009891879163524, + "nauc_cv_recall_at_3_diff1": 0.6054092945422833, + "nauc_cv_recall_at_3_max": 0.4670388069523152, + "nauc_cv_recall_at_3_std": 0.2818495744111195, + "nauc_cv_recall_at_5_diff1": 0.568210566338316, + "nauc_cv_recall_at_5_max": 0.4618802871835425, + "nauc_cv_recall_at_5_std": 0.29267636145228926, + "nauc_map_at_1000_diff1": 0.635473936446471, + "nauc_map_at_1000_max": 0.49963553162495394, + "nauc_map_at_1000_std": 0.2754941978206276, + "nauc_map_at_100_diff1": 0.6349601801057578, + "nauc_map_at_100_max": 0.49976685033613943, + "nauc_map_at_100_std": 0.2759674358452211, + "nauc_map_at_10_diff1": 0.6354531528874251, + "nauc_map_at_10_max": 0.4962877765100082, + "nauc_map_at_10_std": 0.271362809907974, + "nauc_map_at_1_diff1": 0.6965218420770751, + "nauc_map_at_1_max": 0.5148871193449024, + "nauc_map_at_1_std": 0.24287634889441037, + "nauc_map_at_20_diff1": 0.6347142005930541, + "nauc_map_at_20_max": 0.4995845447680013, + "nauc_map_at_20_std": 0.27520502762627513, + "nauc_map_at_3_diff1": 0.6522163384845291, + "nauc_map_at_3_max": 0.49628358740147205, + "nauc_map_at_3_std": 0.2643331138403577, + "nauc_map_at_5_diff1": 0.6440086437871753, + "nauc_map_at_5_max": 0.49385387561881533, + "nauc_map_at_5_std": 0.26506225369101805, + "nauc_mrr_at_1000_diff1": 0.635473936446471, + "nauc_mrr_at_1000_max": 0.49963553162495394, + "nauc_mrr_at_1000_std": 0.2754941978206276, + "nauc_mrr_at_100_diff1": 0.6349601801057578, + "nauc_mrr_at_100_max": 0.49976685033613943, + "nauc_mrr_at_100_std": 0.2759674358452211, + "nauc_mrr_at_10_diff1": 0.6354531528874251, + "nauc_mrr_at_10_max": 0.4962877765100082, + "nauc_mrr_at_10_std": 0.271362809907974, + "nauc_mrr_at_1_diff1": 0.6965218420770751, + "nauc_mrr_at_1_max": 0.5148871193449024, + "nauc_mrr_at_1_std": 0.24287634889441037, + "nauc_mrr_at_20_diff1": 0.6347142005930541, + "nauc_mrr_at_20_max": 0.4995845447680013, + "nauc_mrr_at_20_std": 0.27520502762627513, + "nauc_mrr_at_3_diff1": 0.6522163384845291, + "nauc_mrr_at_3_max": 0.49628358740147205, + "nauc_mrr_at_3_std": 0.2643331138403577, + "nauc_mrr_at_5_diff1": 0.6440086437871753, + "nauc_mrr_at_5_max": 0.49385387561881533, + "nauc_mrr_at_5_std": 0.26506225369101805, + "nauc_ndcg_at_1000_diff1": 0.6171301068346469, + "nauc_ndcg_at_1000_max": 0.5013093969812261, + "nauc_ndcg_at_1000_std": 0.2909876037167272, + "nauc_ndcg_at_100_diff1": 0.6019744132439804, + "nauc_ndcg_at_100_max": 0.507234518974889, + "nauc_ndcg_at_100_std": 0.3085171017338535, + "nauc_ndcg_at_10_diff1": 0.6039190661012934, + "nauc_ndcg_at_10_max": 0.49142066303076054, + "nauc_ndcg_at_10_std": 0.28700506210534843, + "nauc_ndcg_at_1_diff1": 0.6965218420770751, + "nauc_ndcg_at_1_max": 0.5148871193449024, + "nauc_ndcg_at_1_std": 0.24287634889441037, + "nauc_ndcg_at_20_diff1": 0.6008855449309235, + "nauc_ndcg_at_20_max": 0.5032878495174833, + "nauc_ndcg_at_20_std": 0.3006504702690925, + "nauc_ndcg_at_3_diff1": 0.639951550242921, + "nauc_ndcg_at_3_max": 0.4888323450700999, + "nauc_ndcg_at_3_std": 0.2690366601897462, + "nauc_ndcg_at_5_diff1": 0.625009496386489, + "nauc_ndcg_at_5_max": 0.4856757482705781, + "nauc_ndcg_at_5_std": 0.27192265649173825, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.4035136154453741, + "nauc_precision_at_100_max": 0.5912530427570886, + "nauc_precision_at_100_std": 0.5493433744386861, + "nauc_precision_at_10_diff1": 0.4944214665489537, + "nauc_precision_at_10_max": 0.4785322813825728, + "nauc_precision_at_10_std": 0.34302169025023754, + "nauc_precision_at_1_diff1": 0.6965218420770751, + "nauc_precision_at_1_max": 0.5148871193449024, + "nauc_precision_at_1_std": 0.24287634889441037, + "nauc_precision_at_20_diff1": 0.4680088581269517, + "nauc_precision_at_20_max": 0.5306725749263749, + "nauc_precision_at_20_std": 0.4100989187916348, + "nauc_precision_at_3_diff1": 0.6054092945422834, + "nauc_precision_at_3_max": 0.4670388069523155, + "nauc_precision_at_3_std": 0.2818495744111192, + "nauc_precision_at_5_diff1": 0.5682105663383159, + "nauc_precision_at_5_max": 0.4618802871835421, + "nauc_precision_at_5_std": 0.292676361452289, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.40351361544537445, + "nauc_recall_at_100_max": 0.5912530427570891, + "nauc_recall_at_100_std": 0.5493433744386865, + "nauc_recall_at_10_diff1": 0.4944214665489543, + "nauc_recall_at_10_max": 0.4785322813825731, + "nauc_recall_at_10_std": 0.3430216902502382, + "nauc_recall_at_1_diff1": 0.6965218420770751, + "nauc_recall_at_1_max": 0.5148871193449024, + "nauc_recall_at_1_std": 0.24287634889441037, + "nauc_recall_at_20_diff1": 0.46800885812695175, + "nauc_recall_at_20_max": 0.5306725749263748, + "nauc_recall_at_20_std": 0.41009891879163524, + "nauc_recall_at_3_diff1": 0.6054092945422833, + "nauc_recall_at_3_max": 0.4670388069523152, + "nauc_recall_at_3_std": 0.2818495744111195, + "nauc_recall_at_5_diff1": 0.568210566338316, + "nauc_recall_at_5_max": 0.4618802871835425, + "nauc_recall_at_5_std": 0.29267636145228926, + "ndcg_at_1": 0.3266, + "ndcg_at_10": 0.44195, + "ndcg_at_100": 0.48864, + "ndcg_at_1000": 0.51296, + "ndcg_at_20": 0.46002, + "ndcg_at_3": 0.38849, + "ndcg_at_5": 0.4122, + "precision_at_1": 0.3266, + "precision_at_10": 0.05802, + "precision_at_100": 0.00808, + "precision_at_1000": 0.001, + "precision_at_20": 0.0326, + "precision_at_3": 0.14366, + "precision_at_5": 0.09764, + "recall_at_1": 0.3266, + "recall_at_10": 0.58025, + "recall_at_100": 0.80808, + "recall_at_1000": 1.0, + "recall_at_20": 0.65208, + "recall_at_3": 0.43098, + "recall_at_5": 0.48822 + }, + { + "cv_recall_at_1": 0.12105, + "cv_recall_at_10": 0.25439, + "cv_recall_at_100": 0.57193, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.31754, + "cv_recall_at_3": 0.16667, + "cv_recall_at_5": 0.19649, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.17855, + "map_at_1": 0.12105, + "map_at_10": 0.15548, + "map_at_100": 0.16549, + "map_at_1000": 0.16744, + "map_at_20": 0.1599, + "map_at_3": 0.14094, + "map_at_5": 0.14778, + "mrr_at_1": 0.12105263157894737, + "mrr_at_10": 0.1554817599554442, + "mrr_at_100": 0.16548549401385365, + "mrr_at_1000": 0.16743933738760186, + "mrr_at_20": 0.15990474455176268, + "mrr_at_3": 0.1409356725146199, + "mrr_at_5": 0.14777777777777779, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.12178494113610103, + "nauc_cv_recall_at_100_max": 0.12027108175345279, + "nauc_cv_recall_at_100_std": 0.36680812841564475, + "nauc_cv_recall_at_10_diff1": 0.30282348421793964, + "nauc_cv_recall_at_10_max": 0.13271784365196235, + "nauc_cv_recall_at_10_std": 0.33726102823484205, + "nauc_cv_recall_at_1_diff1": 0.49365195440479587, + "nauc_cv_recall_at_1_max": 0.2075044516883363, + "nauc_cv_recall_at_1_std": 0.3627661200695744, + "nauc_cv_recall_at_20_diff1": 0.21236255278748464, + "nauc_cv_recall_at_20_max": 0.1251083950019623, + "nauc_cv_recall_at_20_std": 0.3781734179029766, + "nauc_cv_recall_at_3_diff1": 0.3641915835197674, + "nauc_cv_recall_at_3_max": 0.14471642960954986, + "nauc_cv_recall_at_3_std": 0.3275568692130575, + "nauc_cv_recall_at_5_diff1": 0.32097758270041704, + "nauc_cv_recall_at_5_max": 0.14551136951201293, + "nauc_cv_recall_at_5_std": 0.32077075984419345, + "nauc_map_at_1000_diff1": 0.39639028419999567, + "nauc_map_at_1000_max": 0.16661154492698063, + "nauc_map_at_1000_std": 0.34102672479597745, + "nauc_map_at_100_diff1": 0.39497721421640114, + "nauc_map_at_100_max": 0.1661677982631193, + "nauc_map_at_100_std": 0.3410565655998729, + "nauc_map_at_10_diff1": 0.408843256856627, + "nauc_map_at_10_max": 0.16857868932527764, + "nauc_map_at_10_std": 0.3395154299876011, + "nauc_map_at_1_diff1": 0.49365195440479587, + "nauc_map_at_1_max": 0.2075044516883363, + "nauc_map_at_1_std": 0.3627661200695744, + "nauc_map_at_20_diff1": 0.39823470808304207, + "nauc_map_at_20_max": 0.16717533160175635, + "nauc_map_at_20_std": 0.3426694553686378, + "nauc_map_at_3_diff1": 0.42857037037951523, + "nauc_map_at_3_max": 0.17223986953199172, + "nauc_map_at_3_std": 0.3403057010618428, + "nauc_map_at_5_diff1": 0.4139839634441962, + "nauc_map_at_5_max": 0.17065659328509294, + "nauc_map_at_5_std": 0.33635437824819064, + "nauc_mrr_at_1000_diff1": 0.39639028419999567, + "nauc_mrr_at_1000_max": 0.16661154492698063, + "nauc_mrr_at_1000_std": 0.34102672479597745, + "nauc_mrr_at_100_diff1": 0.39497721421640114, + "nauc_mrr_at_100_max": 0.1661677982631193, + "nauc_mrr_at_100_std": 0.3410565655998729, + "nauc_mrr_at_10_diff1": 0.408843256856627, + "nauc_mrr_at_10_max": 0.16857868932527764, + "nauc_mrr_at_10_std": 0.3395154299876011, + "nauc_mrr_at_1_diff1": 0.49365195440479587, + "nauc_mrr_at_1_max": 0.2075044516883363, + "nauc_mrr_at_1_std": 0.3627661200695744, + "nauc_mrr_at_20_diff1": 0.39823470808304207, + "nauc_mrr_at_20_max": 0.16717533160175635, + "nauc_mrr_at_20_std": 0.3426694553686378, + "nauc_mrr_at_3_diff1": 0.42857037037951523, + "nauc_mrr_at_3_max": 0.17223986953199172, + "nauc_mrr_at_3_std": 0.3403057010618428, + "nauc_mrr_at_5_diff1": 0.4139839634441962, + "nauc_mrr_at_5_max": 0.17065659328509294, + "nauc_mrr_at_5_std": 0.33635437824819064, + "nauc_ndcg_at_1000_diff1": 0.3606522195615228, + "nauc_ndcg_at_1000_max": 0.15851435669428843, + "nauc_ndcg_at_1000_std": 0.3422661885031045, + "nauc_ndcg_at_100_diff1": 0.3238716697397056, + "nauc_ndcg_at_100_max": 0.1516195499748372, + "nauc_ndcg_at_100_std": 0.3455179391449604, + "nauc_ndcg_at_10_diff1": 0.376502809924429, + "nauc_ndcg_at_10_max": 0.15758508541947475, + "nauc_ndcg_at_10_std": 0.338090100250942, + "nauc_ndcg_at_1_diff1": 0.49365195440479587, + "nauc_ndcg_at_1_max": 0.2075044516883363, + "nauc_ndcg_at_1_std": 0.3627661200695744, + "nauc_ndcg_at_20_diff1": 0.34349790053029533, + "nauc_ndcg_at_20_max": 0.15398122499247816, + "nauc_ndcg_at_20_std": 0.35070963021538853, + "nauc_ndcg_at_3_diff1": 0.4104244965053797, + "nauc_ndcg_at_3_max": 0.1643291299872034, + "nauc_ndcg_at_3_std": 0.3365155277170668, + "nauc_ndcg_at_5_diff1": 0.38677310970881057, + "nauc_ndcg_at_5_max": 0.16285640227119183, + "nauc_ndcg_at_5_std": 0.33140501975502823, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.12178494113610133, + "nauc_precision_at_100_max": 0.12027108175345302, + "nauc_precision_at_100_std": 0.36680812841564553, + "nauc_precision_at_10_diff1": 0.3028234842179398, + "nauc_precision_at_10_max": 0.13271784365196254, + "nauc_precision_at_10_std": 0.33726102823484233, + "nauc_precision_at_1_diff1": 0.49365195440479587, + "nauc_precision_at_1_max": 0.2075044516883363, + "nauc_precision_at_1_std": 0.3627661200695744, + "nauc_precision_at_20_diff1": 0.21236255278748467, + "nauc_precision_at_20_max": 0.1251083950019622, + "nauc_precision_at_20_std": 0.3781734179029766, + "nauc_precision_at_3_diff1": 0.36419158351976755, + "nauc_precision_at_3_max": 0.14471642960954978, + "nauc_precision_at_3_std": 0.3275568692130576, + "nauc_precision_at_5_diff1": 0.320977582700417, + "nauc_precision_at_5_max": 0.1455113695120129, + "nauc_precision_at_5_std": 0.3207707598441935, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.12178494113610103, + "nauc_recall_at_100_max": 0.12027108175345279, + "nauc_recall_at_100_std": 0.36680812841564475, + "nauc_recall_at_10_diff1": 0.30282348421793964, + "nauc_recall_at_10_max": 0.13271784365196235, + "nauc_recall_at_10_std": 0.33726102823484205, + "nauc_recall_at_1_diff1": 0.49365195440479587, + "nauc_recall_at_1_max": 0.2075044516883363, + "nauc_recall_at_1_std": 0.3627661200695744, + "nauc_recall_at_20_diff1": 0.21236255278748464, + "nauc_recall_at_20_max": 0.1251083950019623, + "nauc_recall_at_20_std": 0.3781734179029766, + "nauc_recall_at_3_diff1": 0.3641915835197674, + "nauc_recall_at_3_max": 0.14471642960954986, + "nauc_recall_at_3_std": 0.3275568692130575, + "nauc_recall_at_5_diff1": 0.32097758270041704, + "nauc_recall_at_5_max": 0.14551136951201293, + "nauc_recall_at_5_std": 0.32077075984419345, + "ndcg_at_1": 0.12105, + "ndcg_at_10": 0.17855, + "ndcg_at_100": 0.23991, + "ndcg_at_1000": 0.29421, + "ndcg_at_20": 0.19457, + "ndcg_at_3": 0.14753, + "ndcg_at_5": 0.15984, + "precision_at_1": 0.12105, + "precision_at_10": 0.02544, + "precision_at_100": 0.00572, + "precision_at_1000": 0.001, + "precision_at_20": 0.01588, + "precision_at_3": 0.05556, + "precision_at_5": 0.0393, + "recall_at_1": 0.12105, + "recall_at_10": 0.25439, + "recall_at_100": 0.57193, + "recall_at_1000": 1.0, + "recall_at_20": 0.31754, + "recall_at_3": 0.16667, + "recall_at_5": 0.19649 + }, + { + "cv_recall_at_1": 0.17506, + "cv_recall_at_10": 0.36041, + "cv_recall_at_100": 0.67506, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.44394, + "cv_recall_at_3": 0.24828, + "cv_recall_at_5": 0.28375, + "hf_subset": "et", + "languages": [ + "est-Latn" + ], + "main_score": 0.25628, + "map_at_1": 0.17506, + "map_at_10": 0.22449, + "map_at_100": 0.23626, + "map_at_1000": 0.23781, + "map_at_20": 0.23033, + "map_at_3": 0.20633, + "map_at_5": 0.21445, + "mrr_at_1": 0.17505720823798626, + "mrr_at_10": 0.22448830409356724, + "mrr_at_100": 0.23626441711645363, + "mrr_at_1000": 0.2378086038142683, + "mrr_at_20": 0.23033407615650203, + "mrr_at_3": 0.2063310450038139, + "mrr_at_5": 0.2144546147978643, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.26730111731361084, + "nauc_cv_recall_at_100_max": 0.5047094672424741, + "nauc_cv_recall_at_100_std": 0.3852903729729708, + "nauc_cv_recall_at_10_diff1": 0.4619307725560599, + "nauc_cv_recall_at_10_max": 0.5193135193608889, + "nauc_cv_recall_at_10_std": 0.34891718392354365, + "nauc_cv_recall_at_1_diff1": 0.65277071094691, + "nauc_cv_recall_at_1_max": 0.5511794943258216, + "nauc_cv_recall_at_1_std": 0.27970900052689085, + "nauc_cv_recall_at_20_diff1": 0.3967413563077368, + "nauc_cv_recall_at_20_max": 0.4866601040699307, + "nauc_cv_recall_at_20_std": 0.3427263005944641, + "nauc_cv_recall_at_3_diff1": 0.5551163506532734, + "nauc_cv_recall_at_3_max": 0.5456634262644148, + "nauc_cv_recall_at_3_std": 0.3272531670651522, + "nauc_cv_recall_at_5_diff1": 0.5006540117355701, + "nauc_cv_recall_at_5_max": 0.5105691605247302, + "nauc_cv_recall_at_5_std": 0.30568708640340475, + "nauc_map_at_1000_diff1": 0.5709223381463038, + "nauc_map_at_1000_max": 0.534545552565501, + "nauc_map_at_1000_std": 0.3044079913907693, + "nauc_map_at_100_diff1": 0.5698204666019357, + "nauc_map_at_100_max": 0.5345082705210916, + "nauc_map_at_100_std": 0.30478680324037544, + "nauc_map_at_10_diff1": 0.5781645404095507, + "nauc_map_at_10_max": 0.5370703656719066, + "nauc_map_at_10_std": 0.30422950515207864, + "nauc_map_at_1_diff1": 0.65277071094691, + "nauc_map_at_1_max": 0.5511794943258216, + "nauc_map_at_1_std": 0.27970900052689085, + "nauc_map_at_20_diff1": 0.5725853252115725, + "nauc_map_at_20_max": 0.5340676903708351, + "nauc_map_at_20_std": 0.3040158147251269, + "nauc_map_at_3_diff1": 0.6017990320281447, + "nauc_map_at_3_max": 0.5447037216519792, + "nauc_map_at_3_std": 0.30153453842423195, + "nauc_map_at_5_diff1": 0.5858313223784389, + "nauc_map_at_5_max": 0.5359906741690218, + "nauc_map_at_5_std": 0.2972934243394176, + "nauc_mrr_at_1000_diff1": 0.5709223381463038, + "nauc_mrr_at_1000_max": 0.534545552565501, + "nauc_mrr_at_1000_std": 0.3044079913907693, + "nauc_mrr_at_100_diff1": 0.5698204666019357, + "nauc_mrr_at_100_max": 0.5345082705210916, + "nauc_mrr_at_100_std": 0.30478680324037544, + "nauc_mrr_at_10_diff1": 0.5781645404095507, + "nauc_mrr_at_10_max": 0.5370703656719066, + "nauc_mrr_at_10_std": 0.30422950515207864, + "nauc_mrr_at_1_diff1": 0.65277071094691, + "nauc_mrr_at_1_max": 0.5511794943258216, + "nauc_mrr_at_1_std": 0.27970900052689085, + "nauc_mrr_at_20_diff1": 0.5725853252115725, + "nauc_mrr_at_20_max": 0.5340676903708351, + "nauc_mrr_at_20_std": 0.3040158147251269, + "nauc_mrr_at_3_diff1": 0.6017990320281447, + "nauc_mrr_at_3_max": 0.5447037216519792, + "nauc_mrr_at_3_std": 0.30153453842423195, + "nauc_mrr_at_5_diff1": 0.5858313223784389, + "nauc_mrr_at_5_max": 0.5359906741690218, + "nauc_mrr_at_5_std": 0.2972934243394176, + "nauc_ndcg_at_1000_diff1": 0.5408301481972125, + "nauc_ndcg_at_1000_max": 0.5292889263873282, + "nauc_ndcg_at_1000_std": 0.3118164266892448, + "nauc_ndcg_at_100_diff1": 0.5095321706069865, + "nauc_ndcg_at_100_max": 0.5266240841593062, + "nauc_ndcg_at_100_std": 0.3206075047355115, + "nauc_ndcg_at_10_diff1": 0.5458492116430604, + "nauc_ndcg_at_10_max": 0.5320211872393085, + "nauc_ndcg_at_10_std": 0.3162506797195962, + "nauc_ndcg_at_1_diff1": 0.65277071094691, + "nauc_ndcg_at_1_max": 0.5511794943258216, + "nauc_ndcg_at_1_std": 0.27970900052689085, + "nauc_ndcg_at_20_diff1": 0.5266549124213301, + "nauc_ndcg_at_20_max": 0.5221614113899488, + "nauc_ndcg_at_20_std": 0.3148100103251451, + "nauc_ndcg_at_3_diff1": 0.5887353752832222, + "nauc_ndcg_at_3_max": 0.544813909186769, + "nauc_ndcg_at_3_std": 0.3085999345388481, + "nauc_ndcg_at_5_diff1": 0.562131568560372, + "nauc_ndcg_at_5_max": 0.5292013580654052, + "nauc_ndcg_at_5_std": 0.30010487958695203, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.2673011173136117, + "nauc_precision_at_100_max": 0.5047094672424749, + "nauc_precision_at_100_std": 0.38529037297297136, + "nauc_precision_at_10_diff1": 0.46193077255606, + "nauc_precision_at_10_max": 0.5193135193608887, + "nauc_precision_at_10_std": 0.3489171839235438, + "nauc_precision_at_1_diff1": 0.65277071094691, + "nauc_precision_at_1_max": 0.5511794943258216, + "nauc_precision_at_1_std": 0.27970900052689085, + "nauc_precision_at_20_diff1": 0.39674135630773644, + "nauc_precision_at_20_max": 0.4866601040699304, + "nauc_precision_at_20_std": 0.3427263005944638, + "nauc_precision_at_3_diff1": 0.5551163506532731, + "nauc_precision_at_3_max": 0.5456634262644146, + "nauc_precision_at_3_std": 0.3272531670651523, + "nauc_precision_at_5_diff1": 0.5006540117355697, + "nauc_precision_at_5_max": 0.51056916052473, + "nauc_precision_at_5_std": 0.3056870864034045, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.26730111731361084, + "nauc_recall_at_100_max": 0.5047094672424741, + "nauc_recall_at_100_std": 0.3852903729729708, + "nauc_recall_at_10_diff1": 0.4619307725560599, + "nauc_recall_at_10_max": 0.5193135193608889, + "nauc_recall_at_10_std": 0.34891718392354365, + "nauc_recall_at_1_diff1": 0.65277071094691, + "nauc_recall_at_1_max": 0.5511794943258216, + "nauc_recall_at_1_std": 0.27970900052689085, + "nauc_recall_at_20_diff1": 0.3967413563077368, + "nauc_recall_at_20_max": 0.4866601040699307, + "nauc_recall_at_20_std": 0.3427263005944641, + "nauc_recall_at_3_diff1": 0.5551163506532734, + "nauc_recall_at_3_max": 0.5456634262644148, + "nauc_recall_at_3_std": 0.3272531670651522, + "nauc_recall_at_5_diff1": 0.5006540117355701, + "nauc_recall_at_5_max": 0.5105691605247302, + "nauc_recall_at_5_std": 0.30568708640340475, + "ndcg_at_1": 0.17506, + "ndcg_at_10": 0.25628, + "ndcg_at_100": 0.32037, + "ndcg_at_1000": 0.36188, + "ndcg_at_20": 0.27746, + "ndcg_at_3": 0.21706, + "ndcg_at_5": 0.23169, + "precision_at_1": 0.17506, + "precision_at_10": 0.03604, + "precision_at_100": 0.00675, + "precision_at_1000": 0.001, + "precision_at_20": 0.0222, + "precision_at_3": 0.08276, + "precision_at_5": 0.05675, + "recall_at_1": 0.17506, + "recall_at_10": 0.36041, + "recall_at_100": 0.67506, + "recall_at_1000": 1.0, + "recall_at_20": 0.44394, + "recall_at_3": 0.24828, + "recall_at_5": 0.28375 + }, + { + "cv_recall_at_1": 0.39401, + "cv_recall_at_10": 0.65927, + "cv_recall_at_100": 0.91343, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.7525, + "cv_recall_at_3": 0.52275, + "cv_recall_at_5": 0.57603, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.51787, + "map_at_1": 0.39401, + "map_at_10": 0.47368, + "map_at_100": 0.48465, + "map_at_1000": 0.48511, + "map_at_20": 0.48026, + "map_at_3": 0.45006, + "map_at_5": 0.46243, + "mrr_at_1": 0.3940066592674806, + "mrr_at_10": 0.4736831210471608, + "mrr_at_100": 0.4846487645407278, + "mrr_at_1000": 0.4851121293358373, + "mrr_at_20": 0.4802649192099936, + "mrr_at_3": 0.4500554938956712, + "mrr_at_5": 0.46243063263041023, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.3606790371733193, + "nauc_cv_recall_at_100_max": 0.6460443767082419, + "nauc_cv_recall_at_100_std": 0.6438264729443607, + "nauc_cv_recall_at_10_diff1": 0.5070064486702217, + "nauc_cv_recall_at_10_max": 0.4369360291530516, + "nauc_cv_recall_at_10_std": 0.4019920484623479, + "nauc_cv_recall_at_1_diff1": 0.6787729267117939, + "nauc_cv_recall_at_1_max": 0.4483448292991838, + "nauc_cv_recall_at_1_std": 0.2086975409018547, + "nauc_cv_recall_at_20_diff1": 0.44455980762664943, + "nauc_cv_recall_at_20_max": 0.43252662858604923, + "nauc_cv_recall_at_20_std": 0.4659697186595289, + "nauc_cv_recall_at_3_diff1": 0.5711627394926658, + "nauc_cv_recall_at_3_max": 0.4173021015328537, + "nauc_cv_recall_at_3_std": 0.303306184425467, + "nauc_cv_recall_at_5_diff1": 0.5555793126760551, + "nauc_cv_recall_at_5_max": 0.42026344838912505, + "nauc_cv_recall_at_5_std": 0.3200825079115572, + "nauc_map_at_1000_diff1": 0.625727698845203, + "nauc_map_at_1000_max": 0.43833007897791126, + "nauc_map_at_1000_std": 0.2609118106038958, + "nauc_map_at_100_diff1": 0.6253584933406903, + "nauc_map_at_100_max": 0.43857408105349865, + "nauc_map_at_100_std": 0.26139464324542994, + "nauc_map_at_10_diff1": 0.6256223969666678, + "nauc_map_at_10_max": 0.43741404152936747, + "nauc_map_at_10_std": 0.2606781274913183, + "nauc_map_at_1_diff1": 0.6787729267117939, + "nauc_map_at_1_max": 0.4483448292991838, + "nauc_map_at_1_std": 0.2086975409018547, + "nauc_map_at_20_diff1": 0.6241272092493547, + "nauc_map_at_20_max": 0.4364857418691703, + "nauc_map_at_20_std": 0.2619722199064384, + "nauc_map_at_3_diff1": 0.6331140514702438, + "nauc_map_at_3_max": 0.43570365229430424, + "nauc_map_at_3_std": 0.2510720421324455, + "nauc_map_at_5_diff1": 0.6300959184867589, + "nauc_map_at_5_max": 0.43632773692723537, + "nauc_map_at_5_std": 0.254354628949991, + "nauc_mrr_at_1000_diff1": 0.6257275882226064, + "nauc_mrr_at_1000_max": 0.4383296579750604, + "nauc_mrr_at_1000_std": 0.26091128658086987, + "nauc_mrr_at_100_diff1": 0.6253583638531727, + "nauc_mrr_at_100_max": 0.4385736325944442, + "nauc_mrr_at_100_std": 0.26139408315403995, + "nauc_mrr_at_10_diff1": 0.6256223969666678, + "nauc_mrr_at_10_max": 0.43741404152936747, + "nauc_mrr_at_10_std": 0.2606781274913183, + "nauc_mrr_at_1_diff1": 0.6787729267117939, + "nauc_mrr_at_1_max": 0.4483448292991838, + "nauc_mrr_at_1_std": 0.2086975409018547, + "nauc_mrr_at_20_diff1": 0.6241272092493547, + "nauc_mrr_at_20_max": 0.4364857418691703, + "nauc_mrr_at_20_std": 0.2619722199064384, + "nauc_mrr_at_3_diff1": 0.6331140514702438, + "nauc_mrr_at_3_max": 0.43570365229430424, + "nauc_mrr_at_3_std": 0.2510720421324455, + "nauc_mrr_at_5_diff1": 0.6300959184867589, + "nauc_mrr_at_5_max": 0.43632773692723537, + "nauc_mrr_at_5_std": 0.254354628949991, + "nauc_ndcg_at_1000_diff1": 0.6119100731636649, + "nauc_ndcg_at_1000_max": 0.44040411165998694, + "nauc_ndcg_at_1000_std": 0.2770059786004134, + "nauc_ndcg_at_100_diff1": 0.6020858600990256, + "nauc_ndcg_at_100_max": 0.4482027804597055, + "nauc_ndcg_at_100_std": 0.29109437970273, + "nauc_ndcg_at_10_diff1": 0.6006859120736538, + "nauc_ndcg_at_10_max": 0.43643008773180164, + "nauc_ndcg_at_10_std": 0.28916183770833015, + "nauc_ndcg_at_1_diff1": 0.6787729267117939, + "nauc_ndcg_at_1_max": 0.4483448292991838, + "nauc_ndcg_at_1_std": 0.2086975409018547, + "nauc_ndcg_at_20_diff1": 0.5939399449347809, + "nauc_ndcg_at_20_max": 0.43464448439569164, + "nauc_ndcg_at_20_std": 0.29524167525362205, + "nauc_ndcg_at_3_diff1": 0.6176093944148927, + "nauc_ndcg_at_3_max": 0.4311207827652941, + "nauc_ndcg_at_3_std": 0.2642395616086192, + "nauc_ndcg_at_5_diff1": 0.6121797185622866, + "nauc_ndcg_at_5_max": 0.4323791466523419, + "nauc_ndcg_at_5_std": 0.27020359365711794, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3606790371733184, + "nauc_precision_at_100_max": 0.6460443767082423, + "nauc_precision_at_100_std": 0.64382647294436, + "nauc_precision_at_10_diff1": 0.5070064486702209, + "nauc_precision_at_10_max": 0.436936029153051, + "nauc_precision_at_10_std": 0.40199204846234754, + "nauc_precision_at_1_diff1": 0.6787729267117939, + "nauc_precision_at_1_max": 0.4483448292991838, + "nauc_precision_at_1_std": 0.2086975409018547, + "nauc_precision_at_20_diff1": 0.4445598076266497, + "nauc_precision_at_20_max": 0.4325266285860495, + "nauc_precision_at_20_std": 0.46596971865952863, + "nauc_precision_at_3_diff1": 0.5711627394926657, + "nauc_precision_at_3_max": 0.4173021015328535, + "nauc_precision_at_3_std": 0.30330618442546653, + "nauc_precision_at_5_diff1": 0.5555793126760548, + "nauc_precision_at_5_max": 0.420263448389125, + "nauc_precision_at_5_std": 0.3200825079115574, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.3606790371733193, + "nauc_recall_at_100_max": 0.6460443767082419, + "nauc_recall_at_100_std": 0.6438264729443607, + "nauc_recall_at_10_diff1": 0.5070064486702217, + "nauc_recall_at_10_max": 0.4369360291530516, + "nauc_recall_at_10_std": 0.4019920484623479, + "nauc_recall_at_1_diff1": 0.6787729267117939, + "nauc_recall_at_1_max": 0.4483448292991838, + "nauc_recall_at_1_std": 0.2086975409018547, + "nauc_recall_at_20_diff1": 0.44455980762664943, + "nauc_recall_at_20_max": 0.43252662858604923, + "nauc_recall_at_20_std": 0.4659697186595289, + "nauc_recall_at_3_diff1": 0.5711627394926658, + "nauc_recall_at_3_max": 0.4173021015328537, + "nauc_recall_at_3_std": 0.303306184425467, + "nauc_recall_at_5_diff1": 0.5555793126760551, + "nauc_recall_at_5_max": 0.42026344838912505, + "nauc_recall_at_5_std": 0.3200825079115572, + "ndcg_at_1": 0.39401, + "ndcg_at_10": 0.51787, + "ndcg_at_100": 0.57193, + "ndcg_at_1000": 0.58323, + "ndcg_at_20": 0.54158, + "ndcg_at_3": 0.4687, + "ndcg_at_5": 0.49081, + "precision_at_1": 0.39401, + "precision_at_10": 0.06593, + "precision_at_100": 0.00913, + "precision_at_1000": 0.001, + "precision_at_20": 0.03762, + "precision_at_3": 0.17425, + "precision_at_5": 0.11521, + "recall_at_1": 0.39401, + "recall_at_10": 0.65927, + "recall_at_100": 0.91343, + "recall_at_1000": 1.0, + "recall_at_20": 0.7525, + "recall_at_3": 0.52275, + "recall_at_5": 0.57603 + }, + { + "cv_recall_at_1": 0.08378, + "cv_recall_at_10": 0.21375, + "cv_recall_at_100": 0.48013, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.28571, + "cv_recall_at_3": 0.12675, + "cv_recall_at_5": 0.15682, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.13899, + "map_at_1": 0.08378, + "map_at_10": 0.11634, + "map_at_100": 0.12578, + "map_at_1000": 0.12783, + "map_at_20": 0.12128, + "map_at_3": 0.10204, + "map_at_5": 0.10881, + "mrr_at_1": 0.08378088077336197, + "mrr_at_10": 0.11634358003853172, + "mrr_at_100": 0.12577563297812794, + "mrr_at_1000": 0.1278294672817507, + "mrr_at_20": 0.12128160331334957, + "mrr_at_3": 0.10204081632653061, + "mrr_at_5": 0.10880773361976372, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.08094760047818926, + "nauc_cv_recall_at_100_max": 0.17066956375947087, + "nauc_cv_recall_at_100_std": 0.38563047630654224, + "nauc_cv_recall_at_10_diff1": 0.2839044210475601, + "nauc_cv_recall_at_10_max": 0.31545350408678147, + "nauc_cv_recall_at_10_std": 0.3441278004612633, + "nauc_cv_recall_at_1_diff1": 0.6291818808071652, + "nauc_cv_recall_at_1_max": 0.5054935879352562, + "nauc_cv_recall_at_1_std": 0.3865691801255298, + "nauc_cv_recall_at_20_diff1": 0.21817134251307765, + "nauc_cv_recall_at_20_max": 0.2592512547042223, + "nauc_cv_recall_at_20_std": 0.34702007706662635, + "nauc_cv_recall_at_3_diff1": 0.4864187243485422, + "nauc_cv_recall_at_3_max": 0.38426154022878617, + "nauc_cv_recall_at_3_std": 0.3491919668762798, + "nauc_cv_recall_at_5_diff1": 0.4063794488396466, + "nauc_cv_recall_at_5_max": 0.33969774274368847, + "nauc_cv_recall_at_5_std": 0.34046314170571884, + "nauc_map_at_1000_diff1": 0.47827949302761363, + "nauc_map_at_1000_max": 0.4064512152065326, + "nauc_map_at_1000_std": 0.3649575210259258, + "nauc_map_at_100_diff1": 0.4759593348882534, + "nauc_map_at_100_max": 0.40529052275374966, + "nauc_map_at_100_std": 0.36497875541695995, + "nauc_map_at_10_diff1": 0.49485792134613193, + "nauc_map_at_10_max": 0.4188337948334773, + "nauc_map_at_10_std": 0.3637559483655701, + "nauc_map_at_1_diff1": 0.6291818808071652, + "nauc_map_at_1_max": 0.5054935879352562, + "nauc_map_at_1_std": 0.3865691801255298, + "nauc_map_at_20_diff1": 0.4839601045926048, + "nauc_map_at_20_max": 0.4102131481142997, + "nauc_map_at_20_std": 0.3642448061238864, + "nauc_map_at_3_diff1": 0.5544338142604952, + "nauc_map_at_3_max": 0.44292832818211825, + "nauc_map_at_3_std": 0.3667412598594845, + "nauc_map_at_5_diff1": 0.526947072293995, + "nauc_map_at_5_max": 0.4278398208874142, + "nauc_map_at_5_std": 0.36383180100575213, + "nauc_mrr_at_1000_diff1": 0.47827803118281714, + "nauc_mrr_at_1000_max": 0.4064523613247372, + "nauc_mrr_at_1000_std": 0.3649567863374473, + "nauc_mrr_at_100_diff1": 0.47595788183689713, + "nauc_mrr_at_100_max": 0.40529166241657266, + "nauc_mrr_at_100_std": 0.3649780249729875, + "nauc_mrr_at_10_diff1": 0.49485792134613193, + "nauc_mrr_at_10_max": 0.4188337948334773, + "nauc_mrr_at_10_std": 0.3637559483655701, + "nauc_mrr_at_1_diff1": 0.6291818808071652, + "nauc_mrr_at_1_max": 0.5054935879352562, + "nauc_mrr_at_1_std": 0.3865691801255298, + "nauc_mrr_at_20_diff1": 0.4839601045926048, + "nauc_mrr_at_20_max": 0.4102131481142997, + "nauc_mrr_at_20_std": 0.3642448061238864, + "nauc_mrr_at_3_diff1": 0.5544338142604952, + "nauc_mrr_at_3_max": 0.44292832818211825, + "nauc_mrr_at_3_std": 0.3667412598594845, + "nauc_mrr_at_5_diff1": 0.526947072293995, + "nauc_mrr_at_5_max": 0.4278398208874142, + "nauc_mrr_at_5_std": 0.36383180100575213, + "nauc_ndcg_at_1000_diff1": 0.4134594567842365, + "nauc_ndcg_at_1000_max": 0.3679622197162452, + "nauc_ndcg_at_1000_std": 0.36383698541450976, + "nauc_ndcg_at_100_diff1": 0.3522286150645984, + "nauc_ndcg_at_100_max": 0.3324224846540905, + "nauc_ndcg_at_100_std": 0.36735232562757786, + "nauc_ndcg_at_10_diff1": 0.42677518649871554, + "nauc_ndcg_at_10_max": 0.3845568065271432, + "nauc_ndcg_at_10_std": 0.3570034792300479, + "nauc_ndcg_at_1_diff1": 0.6291818808071652, + "nauc_ndcg_at_1_max": 0.5054935879352562, + "nauc_ndcg_at_1_std": 0.3865691801255298, + "nauc_ndcg_at_20_diff1": 0.3954488082966329, + "nauc_ndcg_at_20_max": 0.35973541870986864, + "nauc_ndcg_at_20_std": 0.3579997255648102, + "nauc_ndcg_at_3_diff1": 0.534581543463932, + "nauc_ndcg_at_3_max": 0.4258413031304955, + "nauc_ndcg_at_3_std": 0.3616085694555746, + "nauc_ndcg_at_5_diff1": 0.49034708200589217, + "nauc_ndcg_at_5_max": 0.40104217893271416, + "nauc_ndcg_at_5_std": 0.35668991327782323, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.08094760047818939, + "nauc_precision_at_100_max": 0.17066956375947098, + "nauc_precision_at_100_std": 0.3856304763065422, + "nauc_precision_at_10_diff1": 0.2839044210475601, + "nauc_precision_at_10_max": 0.31545350408678124, + "nauc_precision_at_10_std": 0.3441278004612635, + "nauc_precision_at_1_diff1": 0.6291818808071652, + "nauc_precision_at_1_max": 0.5054935879352562, + "nauc_precision_at_1_std": 0.3865691801255298, + "nauc_precision_at_20_diff1": 0.21817134251307746, + "nauc_precision_at_20_max": 0.2592512547042219, + "nauc_precision_at_20_std": 0.34702007706662613, + "nauc_precision_at_3_diff1": 0.48641872434854205, + "nauc_precision_at_3_max": 0.384261540228786, + "nauc_precision_at_3_std": 0.34919196687627996, + "nauc_precision_at_5_diff1": 0.40637944883964683, + "nauc_precision_at_5_max": 0.33969774274368847, + "nauc_precision_at_5_std": 0.3404631417057189, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.08094760047818926, + "nauc_recall_at_100_max": 0.17066956375947087, + "nauc_recall_at_100_std": 0.38563047630654224, + "nauc_recall_at_10_diff1": 0.2839044210475601, + "nauc_recall_at_10_max": 0.31545350408678147, + "nauc_recall_at_10_std": 0.3441278004612633, + "nauc_recall_at_1_diff1": 0.6291818808071652, + "nauc_recall_at_1_max": 0.5054935879352562, + "nauc_recall_at_1_std": 0.3865691801255298, + "nauc_recall_at_20_diff1": 0.21817134251307765, + "nauc_recall_at_20_max": 0.2592512547042223, + "nauc_recall_at_20_std": 0.34702007706662635, + "nauc_recall_at_3_diff1": 0.4864187243485422, + "nauc_recall_at_3_max": 0.38426154022878617, + "nauc_recall_at_3_std": 0.3491919668762798, + "nauc_recall_at_5_diff1": 0.4063794488396466, + "nauc_recall_at_5_max": 0.33969774274368847, + "nauc_recall_at_5_std": 0.34046314170571884, + "ndcg_at_1": 0.08378, + "ndcg_at_10": 0.13899, + "ndcg_at_100": 0.1922, + "ndcg_at_1000": 0.25598, + "ndcg_at_20": 0.15711, + "ndcg_at_3": 0.10836, + "ndcg_at_5": 0.12065, + "precision_at_1": 0.08378, + "precision_at_10": 0.02137, + "precision_at_100": 0.0048, + "precision_at_1000": 0.001, + "precision_at_20": 0.01429, + "precision_at_3": 0.04225, + "precision_at_5": 0.03136, + "recall_at_1": 0.08378, + "recall_at_10": 0.21375, + "recall_at_100": 0.48013, + "recall_at_1000": 1.0, + "recall_at_20": 0.28571, + "recall_at_3": 0.12675, + "recall_at_5": 0.15682 + }, + { + "cv_recall_at_1": 0.096, + "cv_recall_at_10": 0.247, + "cv_recall_at_100": 0.58, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.311, + "cv_recall_at_3": 0.158, + "cv_recall_at_5": 0.198, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.16391, + "map_at_1": 0.096, + "map_at_10": 0.13824, + "map_at_100": 0.14853, + "map_at_1000": 0.15039, + "map_at_20": 0.14257, + "map_at_3": 0.12283, + "map_at_5": 0.13183, + "mrr_at_1": 0.096, + "mrr_at_10": 0.13823849206349206, + "mrr_at_100": 0.14852892899091763, + "mrr_at_1000": 0.1503909681270283, + "mrr_at_20": 0.14257447598411227, + "mrr_at_3": 0.12283333333333327, + "mrr_at_5": 0.1318333333333333, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.03122877789708922, + "nauc_cv_recall_at_100_max": 0.15807673331978997, + "nauc_cv_recall_at_100_std": 0.25700873552543785, + "nauc_cv_recall_at_10_diff1": 0.1979977193701204, + "nauc_cv_recall_at_10_max": 0.08385825781766487, + "nauc_cv_recall_at_10_std": 0.1336889853859249, + "nauc_cv_recall_at_1_diff1": 0.4098442670601914, + "nauc_cv_recall_at_1_max": 0.1960049414184777, + "nauc_cv_recall_at_1_std": 0.11682485913603434, + "nauc_cv_recall_at_20_diff1": 0.147580351467539, + "nauc_cv_recall_at_20_max": 0.10735523163073482, + "nauc_cv_recall_at_20_std": 0.12651030656989257, + "nauc_cv_recall_at_3_diff1": 0.30751900982190294, + "nauc_cv_recall_at_3_max": 0.09457216752684254, + "nauc_cv_recall_at_3_std": 0.1525148026507118, + "nauc_cv_recall_at_5_diff1": 0.2456197739175154, + "nauc_cv_recall_at_5_max": 0.06958399651079145, + "nauc_cv_recall_at_5_std": 0.14042599590538776, + "nauc_map_at_1000_diff1": 0.3108026337423453, + "nauc_map_at_1000_max": 0.13603764722136522, + "nauc_map_at_1000_std": 0.12868613440315627, + "nauc_map_at_100_diff1": 0.3091343337291221, + "nauc_map_at_100_max": 0.13531151633977603, + "nauc_map_at_100_std": 0.12846135145876467, + "nauc_map_at_10_diff1": 0.3199492115034543, + "nauc_map_at_10_max": 0.13191522051200616, + "nauc_map_at_10_std": 0.12544174366583696, + "nauc_map_at_1_diff1": 0.4098442670601914, + "nauc_map_at_1_max": 0.1960049414184777, + "nauc_map_at_1_std": 0.11682485913603434, + "nauc_map_at_20_diff1": 0.31352289693470164, + "nauc_map_at_20_max": 0.13359715775447498, + "nauc_map_at_20_std": 0.12474902480926435, + "nauc_map_at_3_diff1": 0.35261281588271315, + "nauc_map_at_3_max": 0.1419533508215326, + "nauc_map_at_3_std": 0.12829207923693836, + "nauc_map_at_5_diff1": 0.33059012872746757, + "nauc_map_at_5_max": 0.13068614410775703, + "nauc_map_at_5_std": 0.1269370499395121, + "nauc_mrr_at_1000_diff1": 0.3108026337423453, + "nauc_mrr_at_1000_max": 0.13603764722136522, + "nauc_mrr_at_1000_std": 0.12868613440315627, + "nauc_mrr_at_100_diff1": 0.3091343337291221, + "nauc_mrr_at_100_max": 0.13531151633977603, + "nauc_mrr_at_100_std": 0.12846135145876467, + "nauc_mrr_at_10_diff1": 0.3199492115034543, + "nauc_mrr_at_10_max": 0.13191522051200616, + "nauc_mrr_at_10_std": 0.12544174366583696, + "nauc_mrr_at_1_diff1": 0.4098442670601914, + "nauc_mrr_at_1_max": 0.1960049414184777, + "nauc_mrr_at_1_std": 0.11682485913603434, + "nauc_mrr_at_20_diff1": 0.31352289693470164, + "nauc_mrr_at_20_max": 0.13359715775447498, + "nauc_mrr_at_20_std": 0.12474902480926435, + "nauc_mrr_at_3_diff1": 0.35261281588271315, + "nauc_mrr_at_3_max": 0.1419533508215326, + "nauc_mrr_at_3_std": 0.12829207923693836, + "nauc_mrr_at_5_diff1": 0.33059012872746757, + "nauc_mrr_at_5_max": 0.13068614410775703, + "nauc_mrr_at_5_std": 0.1269370499395121, + "nauc_ndcg_at_1000_diff1": 0.27465338314694493, + "nauc_ndcg_at_1000_max": 0.13532791914955453, + "nauc_ndcg_at_1000_std": 0.14176692393314377, + "nauc_ndcg_at_100_diff1": 0.23470449632345772, + "nauc_ndcg_at_100_max": 0.13451761349838698, + "nauc_ndcg_at_100_std": 0.1555639746804492, + "nauc_ndcg_at_10_diff1": 0.2818115912878626, + "nauc_ndcg_at_10_max": 0.11547465009598505, + "nauc_ndcg_at_10_std": 0.12840701176314118, + "nauc_ndcg_at_1_diff1": 0.4098442670601914, + "nauc_ndcg_at_1_max": 0.1960049414184777, + "nauc_ndcg_at_1_std": 0.11682485913603434, + "nauc_ndcg_at_20_diff1": 0.2615957787270437, + "nauc_ndcg_at_20_max": 0.12234254332047498, + "nauc_ndcg_at_20_std": 0.1260256036365097, + "nauc_ndcg_at_3_diff1": 0.33920706600399225, + "nauc_ndcg_at_3_max": 0.12798246596280738, + "nauc_ndcg_at_3_std": 0.13512915509383155, + "nauc_ndcg_at_5_diff1": 0.30427309180580736, + "nauc_ndcg_at_5_max": 0.11143239360648992, + "nauc_ndcg_at_5_std": 0.1313015918668763, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.031228777897089503, + "nauc_precision_at_100_max": 0.1580767333197903, + "nauc_precision_at_100_std": 0.2570087355254377, + "nauc_precision_at_10_diff1": 0.19799771937012006, + "nauc_precision_at_10_max": 0.083858257817665, + "nauc_precision_at_10_std": 0.1336889853859248, + "nauc_precision_at_1_diff1": 0.4098442670601914, + "nauc_precision_at_1_max": 0.1960049414184777, + "nauc_precision_at_1_std": 0.11682485913603434, + "nauc_precision_at_20_diff1": 0.14758035146753876, + "nauc_precision_at_20_max": 0.10735523163073456, + "nauc_precision_at_20_std": 0.1265103065698924, + "nauc_precision_at_3_diff1": 0.30751900982190283, + "nauc_precision_at_3_max": 0.09457216752684251, + "nauc_precision_at_3_std": 0.15251480265071174, + "nauc_precision_at_5_diff1": 0.2456197739175152, + "nauc_precision_at_5_max": 0.06958399651079158, + "nauc_precision_at_5_std": 0.14042599590538776, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.03122877789708922, + "nauc_recall_at_100_max": 0.15807673331978997, + "nauc_recall_at_100_std": 0.25700873552543785, + "nauc_recall_at_10_diff1": 0.1979977193701204, + "nauc_recall_at_10_max": 0.08385825781766487, + "nauc_recall_at_10_std": 0.1336889853859249, + "nauc_recall_at_1_diff1": 0.4098442670601914, + "nauc_recall_at_1_max": 0.1960049414184777, + "nauc_recall_at_1_std": 0.11682485913603434, + "nauc_recall_at_20_diff1": 0.147580351467539, + "nauc_recall_at_20_max": 0.10735523163073482, + "nauc_recall_at_20_std": 0.12651030656989257, + "nauc_recall_at_3_diff1": 0.30751900982190294, + "nauc_recall_at_3_max": 0.09457216752684254, + "nauc_recall_at_3_std": 0.1525148026507118, + "nauc_recall_at_5_diff1": 0.2456197739175154, + "nauc_recall_at_5_max": 0.06958399651079145, + "nauc_recall_at_5_std": 0.14042599590538776, + "ndcg_at_1": 0.096, + "ndcg_at_10": 0.16391, + "ndcg_at_100": 0.22792, + "ndcg_at_1000": 0.28067, + "ndcg_at_20": 0.17996, + "ndcg_at_3": 0.13184, + "ndcg_at_5": 0.14819, + "precision_at_1": 0.096, + "precision_at_10": 0.0247, + "precision_at_100": 0.0058, + "precision_at_1000": 0.001, + "precision_at_20": 0.01555, + "precision_at_3": 0.05267, + "precision_at_5": 0.0396, + "recall_at_1": 0.096, + "recall_at_10": 0.247, + "recall_at_100": 0.58, + "recall_at_1000": 1.0, + "recall_at_20": 0.311, + "recall_at_3": 0.158, + "recall_at_5": 0.198 + }, + { + "cv_recall_at_1": 0.32594, + "cv_recall_at_10": 0.58807, + "cv_recall_at_100": 0.83911, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.66297, + "cv_recall_at_3": 0.45492, + "cv_recall_at_5": 0.52011, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.44964, + "map_at_1": 0.32594, + "map_at_10": 0.40614, + "map_at_100": 0.41528, + "map_at_1000": 0.41617, + "map_at_20": 0.41108, + "map_at_3": 0.38234, + "map_at_5": 0.39704, + "mrr_at_1": 0.3259361997226075, + "mrr_at_10": 0.40613841005657914, + "mrr_at_100": 0.4152786021504557, + "mrr_at_1000": 0.41616716847899377, + "mrr_at_20": 0.41107626755006554, + "mrr_at_3": 0.3823393435043923, + "mrr_at_5": 0.3970411465557097, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4143125486845753, + "nauc_cv_recall_at_100_max": 0.5320816743201151, + "nauc_cv_recall_at_100_std": 0.5665864175181943, + "nauc_cv_recall_at_10_diff1": 0.4736723432838038, + "nauc_cv_recall_at_10_max": 0.5155390517675708, + "nauc_cv_recall_at_10_std": 0.43366104906933245, + "nauc_cv_recall_at_1_diff1": 0.6524414738983796, + "nauc_cv_recall_at_1_max": 0.5512090830443038, + "nauc_cv_recall_at_1_std": 0.3502491395104, + "nauc_cv_recall_at_20_diff1": 0.4507518062183285, + "nauc_cv_recall_at_20_max": 0.5160176805499569, + "nauc_cv_recall_at_20_std": 0.519694782745015, + "nauc_cv_recall_at_3_diff1": 0.5535662951368313, + "nauc_cv_recall_at_3_max": 0.564704741066614, + "nauc_cv_recall_at_3_std": 0.4136036979599246, + "nauc_cv_recall_at_5_diff1": 0.5130205909284755, + "nauc_cv_recall_at_5_max": 0.5281322733359631, + "nauc_cv_recall_at_5_std": 0.4146731951698951, + "nauc_map_at_1000_diff1": 0.5923562901465701, + "nauc_map_at_1000_max": 0.5429739061166239, + "nauc_map_at_1000_std": 0.3848112051754737, + "nauc_map_at_100_diff1": 0.5918655013216925, + "nauc_map_at_100_max": 0.5428860352272687, + "nauc_map_at_100_std": 0.38519824890915205, + "nauc_map_at_10_diff1": 0.5913677852925365, + "nauc_map_at_10_max": 0.5421098179062059, + "nauc_map_at_10_std": 0.3813377163920301, + "nauc_map_at_1_diff1": 0.6524414738983796, + "nauc_map_at_1_max": 0.5512090830443038, + "nauc_map_at_1_std": 0.3502491395104, + "nauc_map_at_20_diff1": 0.5910724634089584, + "nauc_map_at_20_max": 0.5422948552748079, + "nauc_map_at_20_std": 0.3852347149072005, + "nauc_map_at_3_diff1": 0.604405473516488, + "nauc_map_at_3_max": 0.5517276066883616, + "nauc_map_at_3_std": 0.37888818845914163, + "nauc_map_at_5_diff1": 0.596029160112715, + "nauc_map_at_5_max": 0.5437518832787203, + "nauc_map_at_5_std": 0.3790704704001802, + "nauc_mrr_at_1000_diff1": 0.5923560278478026, + "nauc_mrr_at_1000_max": 0.5429733763931445, + "nauc_mrr_at_1000_std": 0.3848105803844461, + "nauc_mrr_at_100_diff1": 0.591865231267037, + "nauc_mrr_at_100_max": 0.5428854783615418, + "nauc_mrr_at_100_std": 0.38519758770448553, + "nauc_mrr_at_10_diff1": 0.5913677852925365, + "nauc_mrr_at_10_max": 0.5421098179062059, + "nauc_mrr_at_10_std": 0.3813377163920301, + "nauc_mrr_at_1_diff1": 0.6524414738983796, + "nauc_mrr_at_1_max": 0.5512090830443038, + "nauc_mrr_at_1_std": 0.3502491395104, + "nauc_mrr_at_20_diff1": 0.5910724634089584, + "nauc_mrr_at_20_max": 0.5422948552748079, + "nauc_mrr_at_20_std": 0.3852347149072005, + "nauc_mrr_at_3_diff1": 0.604405473516488, + "nauc_mrr_at_3_max": 0.5517276066883616, + "nauc_mrr_at_3_std": 0.37888818845914163, + "nauc_mrr_at_5_diff1": 0.596029160112715, + "nauc_mrr_at_5_max": 0.5437518832787203, + "nauc_mrr_at_5_std": 0.3790704704001802, + "nauc_ndcg_at_1000_diff1": 0.5783071578366672, + "nauc_ndcg_at_1000_max": 0.5414636189261661, + "nauc_ndcg_at_1000_std": 0.39675982530546133, + "nauc_ndcg_at_100_diff1": 0.5673150067554389, + "nauc_ndcg_at_100_max": 0.5404582633785823, + "nauc_ndcg_at_100_std": 0.4074294639470118, + "nauc_ndcg_at_10_diff1": 0.5644498003714317, + "nauc_ndcg_at_10_max": 0.5363955483158082, + "nauc_ndcg_at_10_std": 0.393513763480844, + "nauc_ndcg_at_1_diff1": 0.6524414738983796, + "nauc_ndcg_at_1_max": 0.5512090830443038, + "nauc_ndcg_at_1_std": 0.3502491395104, + "nauc_ndcg_at_20_diff1": 0.5623783362516817, + "nauc_ndcg_at_20_max": 0.5370550562641554, + "nauc_ndcg_at_20_std": 0.4099708581203125, + "nauc_ndcg_at_3_diff1": 0.5911279841759199, + "nauc_ndcg_at_3_max": 0.5548417220825114, + "nauc_ndcg_at_3_std": 0.3878572153415391, + "nauc_ndcg_at_5_diff1": 0.5755197804997088, + "nauc_ndcg_at_5_max": 0.5401619268149915, + "nauc_ndcg_at_5_std": 0.3881047375243232, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.4143125486845745, + "nauc_precision_at_100_max": 0.5320816743201164, + "nauc_precision_at_100_std": 0.5665864175181946, + "nauc_precision_at_10_diff1": 0.47367234328380414, + "nauc_precision_at_10_max": 0.5155390517675704, + "nauc_precision_at_10_std": 0.43366104906933317, + "nauc_precision_at_1_diff1": 0.6524414738983796, + "nauc_precision_at_1_max": 0.5512090830443038, + "nauc_precision_at_1_std": 0.3502491395104, + "nauc_precision_at_20_diff1": 0.4507518062183285, + "nauc_precision_at_20_max": 0.5160176805499569, + "nauc_precision_at_20_std": 0.519694782745015, + "nauc_precision_at_3_diff1": 0.553566295136831, + "nauc_precision_at_3_max": 0.5647047410666138, + "nauc_precision_at_3_std": 0.41360369795992435, + "nauc_precision_at_5_diff1": 0.513020590928476, + "nauc_precision_at_5_max": 0.528132273335963, + "nauc_precision_at_5_std": 0.41467319516989587, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4143125486845753, + "nauc_recall_at_100_max": 0.5320816743201151, + "nauc_recall_at_100_std": 0.5665864175181943, + "nauc_recall_at_10_diff1": 0.4736723432838038, + "nauc_recall_at_10_max": 0.5155390517675708, + "nauc_recall_at_10_std": 0.43366104906933245, + "nauc_recall_at_1_diff1": 0.6524414738983796, + "nauc_recall_at_1_max": 0.5512090830443038, + "nauc_recall_at_1_std": 0.3502491395104, + "nauc_recall_at_20_diff1": 0.4507518062183285, + "nauc_recall_at_20_max": 0.5160176805499569, + "nauc_recall_at_20_std": 0.519694782745015, + "nauc_recall_at_3_diff1": 0.5535662951368313, + "nauc_recall_at_3_max": 0.564704741066614, + "nauc_recall_at_3_std": 0.4136036979599246, + "nauc_recall_at_5_diff1": 0.5130205909284755, + "nauc_recall_at_5_max": 0.5281322733359631, + "nauc_recall_at_5_std": 0.4146731951698951, + "ndcg_at_1": 0.32594, + "ndcg_at_10": 0.44964, + "ndcg_at_100": 0.50029, + "ndcg_at_1000": 0.52147, + "ndcg_at_20": 0.46825, + "ndcg_at_3": 0.40096, + "ndcg_at_5": 0.42764, + "precision_at_1": 0.32594, + "precision_at_10": 0.05881, + "precision_at_100": 0.00839, + "precision_at_1000": 0.001, + "precision_at_20": 0.03315, + "precision_at_3": 0.15164, + "precision_at_5": 0.10402, + "recall_at_1": 0.32594, + "recall_at_10": 0.58807, + "recall_at_100": 0.83911, + "recall_at_1000": 1.0, + "recall_at_20": 0.66297, + "recall_at_3": 0.45492, + "recall_at_5": 0.52011 + }, + { + "cv_recall_at_1": 0.32558, + "cv_recall_at_10": 0.57294, + "cv_recall_at_100": 0.83298, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.65539, + "cv_recall_at_3": 0.45243, + "cv_recall_at_5": 0.50317, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.44269, + "map_at_1": 0.32558, + "map_at_10": 0.40177, + "map_at_100": 0.41202, + "map_at_1000": 0.41283, + "map_at_20": 0.4075, + "map_at_3": 0.38073, + "map_at_5": 0.39246, + "mrr_at_1": 0.32558139534883723, + "mrr_at_10": 0.4017743884022952, + "mrr_at_100": 0.41201766810451657, + "mrr_at_1000": 0.41282790686568654, + "mrr_at_20": 0.4074975696352814, + "mrr_at_3": 0.38072586328400265, + "mrr_at_5": 0.3924594785059898, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.33319717829563866, + "nauc_cv_recall_at_100_max": 0.4067449927437285, + "nauc_cv_recall_at_100_std": 0.6174517316377853, + "nauc_cv_recall_at_10_diff1": 0.486821458931059, + "nauc_cv_recall_at_10_max": 0.3902786386096522, + "nauc_cv_recall_at_10_std": 0.37508230270815346, + "nauc_cv_recall_at_1_diff1": 0.6367299436732132, + "nauc_cv_recall_at_1_max": 0.4959073257152778, + "nauc_cv_recall_at_1_std": 0.34968642580726267, + "nauc_cv_recall_at_20_diff1": 0.44159761215278753, + "nauc_cv_recall_at_20_max": 0.3932788718769707, + "nauc_cv_recall_at_20_std": 0.4449484107288282, + "nauc_cv_recall_at_3_diff1": 0.5273113591091085, + "nauc_cv_recall_at_3_max": 0.4477604405417835, + "nauc_cv_recall_at_3_std": 0.3882510977951838, + "nauc_cv_recall_at_5_diff1": 0.5219606524762571, + "nauc_cv_recall_at_5_max": 0.4263623280399427, + "nauc_cv_recall_at_5_std": 0.39232890856205144, + "nauc_map_at_1000_diff1": 0.5805297718176744, + "nauc_map_at_1000_max": 0.4675574167497707, + "nauc_map_at_1000_std": 0.3717808238606819, + "nauc_map_at_100_diff1": 0.5799039262803479, + "nauc_map_at_100_max": 0.4673313547760212, + "nauc_map_at_100_std": 0.37223231199122314, + "nauc_map_at_10_diff1": 0.5818443261985234, + "nauc_map_at_10_max": 0.4655432092875299, + "nauc_map_at_10_std": 0.3661278338064773, + "nauc_map_at_1_diff1": 0.6367299436732132, + "nauc_map_at_1_max": 0.4959073257152778, + "nauc_map_at_1_std": 0.34968642580726267, + "nauc_map_at_20_diff1": 0.5800487033799071, + "nauc_map_at_20_max": 0.46633585873697075, + "nauc_map_at_20_std": 0.3703071346179496, + "nauc_map_at_3_diff1": 0.5858912260990712, + "nauc_map_at_3_max": 0.47405430448817926, + "nauc_map_at_3_std": 0.3664630113569763, + "nauc_map_at_5_diff1": 0.5852808145127003, + "nauc_map_at_5_max": 0.4692733799352851, + "nauc_map_at_5_std": 0.367719016060064, + "nauc_mrr_at_1000_diff1": 0.5805298300860103, + "nauc_mrr_at_1000_max": 0.4675573794546013, + "nauc_mrr_at_1000_std": 0.3717808086064955, + "nauc_mrr_at_100_diff1": 0.5799039262803479, + "nauc_mrr_at_100_max": 0.4673313547760212, + "nauc_mrr_at_100_std": 0.37223231199122314, + "nauc_mrr_at_10_diff1": 0.5818443261985234, + "nauc_mrr_at_10_max": 0.4655432092875299, + "nauc_mrr_at_10_std": 0.3661278338064773, + "nauc_mrr_at_1_diff1": 0.6367299436732132, + "nauc_mrr_at_1_max": 0.4959073257152778, + "nauc_mrr_at_1_std": 0.34968642580726267, + "nauc_mrr_at_20_diff1": 0.5800487033799071, + "nauc_mrr_at_20_max": 0.46633585873697075, + "nauc_mrr_at_20_std": 0.3703071346179496, + "nauc_mrr_at_3_diff1": 0.5858912260990712, + "nauc_mrr_at_3_max": 0.47405430448817926, + "nauc_mrr_at_3_std": 0.3664630113569763, + "nauc_mrr_at_5_diff1": 0.5852808145127003, + "nauc_mrr_at_5_max": 0.4692733799352851, + "nauc_mrr_at_5_std": 0.367719016060064, + "nauc_ndcg_at_1000_diff1": 0.5656743421506273, + "nauc_ndcg_at_1000_max": 0.461169825921946, + "nauc_ndcg_at_1000_std": 0.3826545468837048, + "nauc_ndcg_at_100_diff1": 0.5501611366152832, + "nauc_ndcg_at_100_max": 0.4571657863531696, + "nauc_ndcg_at_100_std": 0.3973814457501108, + "nauc_ndcg_at_10_diff1": 0.5598604439318214, + "nauc_ndcg_at_10_max": 0.44870119146458565, + "nauc_ndcg_at_10_std": 0.3690957138781721, + "nauc_ndcg_at_1_diff1": 0.6367299436732132, + "nauc_ndcg_at_1_max": 0.4959073257152778, + "nauc_ndcg_at_1_std": 0.34968642580726267, + "nauc_ndcg_at_20_diff1": 0.5523867049209488, + "nauc_ndcg_at_20_max": 0.4511549065343426, + "nauc_ndcg_at_20_std": 0.3843038335913939, + "nauc_ndcg_at_3_diff1": 0.5706925500420406, + "nauc_ndcg_at_3_max": 0.4672549158811777, + "nauc_ndcg_at_3_std": 0.3720660751976934, + "nauc_ndcg_at_5_diff1": 0.5691547732715783, + "nauc_ndcg_at_5_max": 0.45861800860996577, + "nauc_ndcg_at_5_std": 0.3739796280197634, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.3331971782956386, + "nauc_precision_at_100_max": 0.4067449927437276, + "nauc_precision_at_100_std": 0.6174517316377839, + "nauc_precision_at_10_diff1": 0.4868214589310593, + "nauc_precision_at_10_max": 0.39027863860965145, + "nauc_precision_at_10_std": 0.375082302708153, + "nauc_precision_at_1_diff1": 0.6367299436732132, + "nauc_precision_at_1_max": 0.4959073257152778, + "nauc_precision_at_1_std": 0.34968642580726267, + "nauc_precision_at_20_diff1": 0.441597612152787, + "nauc_precision_at_20_max": 0.39327887187697014, + "nauc_precision_at_20_std": 0.4449484107288272, + "nauc_precision_at_3_diff1": 0.5273113591091089, + "nauc_precision_at_3_max": 0.4477604405417836, + "nauc_precision_at_3_std": 0.38825109779518374, + "nauc_precision_at_5_diff1": 0.5219606524762569, + "nauc_precision_at_5_max": 0.4263623280399423, + "nauc_precision_at_5_std": 0.39232890856205105, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.33319717829563866, + "nauc_recall_at_100_max": 0.4067449927437285, + "nauc_recall_at_100_std": 0.6174517316377853, + "nauc_recall_at_10_diff1": 0.486821458931059, + "nauc_recall_at_10_max": 0.3902786386096522, + "nauc_recall_at_10_std": 0.37508230270815346, + "nauc_recall_at_1_diff1": 0.6367299436732132, + "nauc_recall_at_1_max": 0.4959073257152778, + "nauc_recall_at_1_std": 0.34968642580726267, + "nauc_recall_at_20_diff1": 0.44159761215278753, + "nauc_recall_at_20_max": 0.3932788718769707, + "nauc_recall_at_20_std": 0.4449484107288282, + "nauc_recall_at_3_diff1": 0.5273113591091085, + "nauc_recall_at_3_max": 0.4477604405417835, + "nauc_recall_at_3_std": 0.3882510977951838, + "nauc_recall_at_5_diff1": 0.5219606524762571, + "nauc_recall_at_5_max": 0.4263623280399427, + "nauc_recall_at_5_std": 0.39232890856205144, + "ndcg_at_1": 0.32558, + "ndcg_at_10": 0.44269, + "ndcg_at_100": 0.49634, + "ndcg_at_1000": 0.51769, + "ndcg_at_20": 0.46354, + "ndcg_at_3": 0.39911, + "ndcg_at_5": 0.42013, + "precision_at_1": 0.32558, + "precision_at_10": 0.05729, + "precision_at_100": 0.00833, + "precision_at_1000": 0.001, + "precision_at_20": 0.03277, + "precision_at_3": 0.15081, + "precision_at_5": 0.10063, + "recall_at_1": 0.32558, + "recall_at_10": 0.57294, + "recall_at_100": 0.83298, + "recall_at_1000": 1.0, + "recall_at_20": 0.65539, + "recall_at_3": 0.45243, + "recall_at_5": 0.50317 + }, + { + "cv_recall_at_1": 0.5, + "cv_recall_at_10": 0.785, + "cv_recall_at_100": 0.98, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.844, + "cv_recall_at_3": 0.653, + "cv_recall_at_5": 0.717, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.63886, + "map_at_1": 0.5, + "map_at_10": 0.59253, + "map_at_100": 0.60054, + "map_at_1000": 0.60066, + "map_at_20": 0.59673, + "map_at_3": 0.569, + "map_at_5": 0.58335, + "mrr_at_1": 0.5, + "mrr_at_10": 0.5925329365079366, + "mrr_at_100": 0.6005437190144755, + "mrr_at_1000": 0.600664144421558, + "mrr_at_20": 0.5967313145746579, + "mrr_at_3": 0.5690000000000002, + "mrr_at_5": 0.5833500000000003, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5858309990662961, + "nauc_cv_recall_at_100_max": 0.7176237161531301, + "nauc_cv_recall_at_100_std": 0.8628618113912271, + "nauc_cv_recall_at_10_diff1": 0.7120100093247249, + "nauc_cv_recall_at_10_max": 0.2462671592639371, + "nauc_cv_recall_at_10_std": -0.23614452142916173, + "nauc_cv_recall_at_1_diff1": 0.7841492338441036, + "nauc_cv_recall_at_1_max": 0.3875429713524314, + "nauc_cv_recall_at_1_std": -0.007577614923384203, + "nauc_cv_recall_at_20_diff1": 0.7082412268582482, + "nauc_cv_recall_at_20_max": 0.32142857142857206, + "nauc_cv_recall_at_20_std": -0.2007146625135008, + "nauc_cv_recall_at_3_diff1": 0.7107919769385271, + "nauc_cv_recall_at_3_max": 0.3104524717256328, + "nauc_cv_recall_at_3_std": -0.13919222641101736, + "nauc_cv_recall_at_5_diff1": 0.6963980779158884, + "nauc_cv_recall_at_5_max": 0.29357546600706164, + "nauc_cv_recall_at_5_std": -0.19533624179984505, + "nauc_map_at_1000_diff1": 0.7545027808605635, + "nauc_map_at_1000_max": 0.3567529979219445, + "nauc_map_at_1000_std": -0.062112641943025536, + "nauc_map_at_100_diff1": 0.7544438124532609, + "nauc_map_at_100_max": 0.3568824116725499, + "nauc_map_at_100_std": -0.06173985435885906, + "nauc_map_at_10_diff1": 0.7539830140069392, + "nauc_map_at_10_max": 0.35104102462166586, + "nauc_map_at_10_std": -0.06966701537309626, + "nauc_map_at_1_diff1": 0.7841492338441036, + "nauc_map_at_1_max": 0.3875429713524314, + "nauc_map_at_1_std": -0.007577614923384203, + "nauc_map_at_20_diff1": 0.7543229223023675, + "nauc_map_at_20_max": 0.35552716581356714, + "nauc_map_at_20_std": -0.06605583359387289, + "nauc_map_at_3_diff1": 0.752913416580022, + "nauc_map_at_3_max": 0.3556594165384521, + "nauc_map_at_3_std": -0.060537047966662746, + "nauc_map_at_5_diff1": 0.7510932930040148, + "nauc_map_at_5_max": 0.3534526852569118, + "nauc_map_at_5_std": -0.06868553575707965, + "nauc_mrr_at_1000_diff1": 0.7545027808605635, + "nauc_mrr_at_1000_max": 0.3567529979219445, + "nauc_mrr_at_1000_std": -0.062112641943025536, + "nauc_mrr_at_100_diff1": 0.7544438124532609, + "nauc_mrr_at_100_max": 0.3568824116725499, + "nauc_mrr_at_100_std": -0.06173985435885906, + "nauc_mrr_at_10_diff1": 0.7539830140069392, + "nauc_mrr_at_10_max": 0.35104102462166586, + "nauc_mrr_at_10_std": -0.06966701537309626, + "nauc_mrr_at_1_diff1": 0.7841492338441036, + "nauc_mrr_at_1_max": 0.3875429713524314, + "nauc_mrr_at_1_std": -0.007577614923384203, + "nauc_mrr_at_20_diff1": 0.7543229223023675, + "nauc_mrr_at_20_max": 0.35552716581356714, + "nauc_mrr_at_20_std": -0.06605583359387289, + "nauc_mrr_at_3_diff1": 0.752913416580022, + "nauc_mrr_at_3_max": 0.3556594165384521, + "nauc_mrr_at_3_std": -0.060537047966662746, + "nauc_mrr_at_5_diff1": 0.7510932930040148, + "nauc_mrr_at_5_max": 0.3534526852569118, + "nauc_mrr_at_5_std": -0.06868553575707965, + "nauc_ndcg_at_1000_diff1": 0.7499698607053241, + "nauc_ndcg_at_1000_max": 0.35418485575392594, + "nauc_ndcg_at_1000_std": -0.06718538120073447, + "nauc_ndcg_at_100_diff1": 0.7482164162964144, + "nauc_ndcg_at_100_max": 0.35807404010503563, + "nauc_ndcg_at_100_std": -0.05697537291884474, + "nauc_ndcg_at_10_diff1": 0.7457296191933457, + "nauc_ndcg_at_10_max": 0.3329444136294682, + "nauc_ndcg_at_10_std": -0.09911995188680284, + "nauc_ndcg_at_1_diff1": 0.7841492338441036, + "nauc_ndcg_at_1_max": 0.3875429713524314, + "nauc_ndcg_at_1_std": -0.007577614923384203, + "nauc_ndcg_at_20_diff1": 0.7467491806865436, + "nauc_ndcg_at_20_max": 0.34852864281248525, + "nauc_ndcg_at_20_std": -0.08706237078385107, + "nauc_ndcg_at_3_diff1": 0.7429288606690733, + "nauc_ndcg_at_3_max": 0.34498708703837594, + "nauc_ndcg_at_3_std": -0.07903887707810912, + "nauc_ndcg_at_5_diff1": 0.739339573470649, + "nauc_ndcg_at_5_max": 0.34065155653743817, + "nauc_ndcg_at_5_std": -0.09527532922715241, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.5858309990662923, + "nauc_precision_at_100_max": 0.717623716153127, + "nauc_precision_at_100_std": 0.862861811391217, + "nauc_precision_at_10_diff1": 0.7120100093247237, + "nauc_precision_at_10_max": 0.24626715926393547, + "nauc_precision_at_10_std": -0.23614452142916353, + "nauc_precision_at_1_diff1": 0.7841492338441036, + "nauc_precision_at_1_max": 0.3875429713524314, + "nauc_precision_at_1_std": -0.007577614923384203, + "nauc_precision_at_20_diff1": 0.7082412268582476, + "nauc_precision_at_20_max": 0.32142857142857073, + "nauc_precision_at_20_std": -0.20071466251350326, + "nauc_precision_at_3_diff1": 0.7107919769385278, + "nauc_precision_at_3_max": 0.3104524717256329, + "nauc_precision_at_3_std": -0.13919222641101667, + "nauc_precision_at_5_diff1": 0.6963980779158885, + "nauc_precision_at_5_max": 0.29357546600705997, + "nauc_precision_at_5_std": -0.1953362417998463, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5858309990662961, + "nauc_recall_at_100_max": 0.7176237161531301, + "nauc_recall_at_100_std": 0.8628618113912271, + "nauc_recall_at_10_diff1": 0.7120100093247249, + "nauc_recall_at_10_max": 0.2462671592639371, + "nauc_recall_at_10_std": -0.23614452142916173, + "nauc_recall_at_1_diff1": 0.7841492338441036, + "nauc_recall_at_1_max": 0.3875429713524314, + "nauc_recall_at_1_std": -0.007577614923384203, + "nauc_recall_at_20_diff1": 0.7082412268582482, + "nauc_recall_at_20_max": 0.32142857142857206, + "nauc_recall_at_20_std": -0.2007146625135008, + "nauc_recall_at_3_diff1": 0.7107919769385271, + "nauc_recall_at_3_max": 0.3104524717256328, + "nauc_recall_at_3_std": -0.13919222641101736, + "nauc_recall_at_5_diff1": 0.6963980779158884, + "nauc_recall_at_5_max": 0.29357546600706164, + "nauc_recall_at_5_std": -0.19533624179984505, + "ndcg_at_1": 0.5, + "ndcg_at_10": 0.63886, + "ndcg_at_100": 0.67982, + "ndcg_at_1000": 0.6825, + "ndcg_at_20": 0.65391, + "ndcg_at_3": 0.59064, + "ndcg_at_5": 0.61676, + "precision_at_1": 0.5, + "precision_at_10": 0.0785, + "precision_at_100": 0.0098, + "precision_at_1000": 0.001, + "precision_at_20": 0.0422, + "precision_at_3": 0.21767, + "precision_at_5": 0.1434, + "recall_at_1": 0.5, + "recall_at_10": 0.785, + "recall_at_100": 0.98, + "recall_at_1000": 1.0, + "recall_at_20": 0.844, + "recall_at_3": 0.653, + "recall_at_5": 0.717 + } + ] + }, + "task_name": "WITT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/model_meta.json b/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/model_meta.json new file mode 100644 index 0000000000..756f9c9c7a --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/model_meta.json @@ -0,0 +1 @@ +{"name": "google/siglip-base-patch16-512", "revision": "753a949581523b60257d93e18391e8c27f72eb22", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/WITT2IRetrieval.json b/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/WITT2IRetrieval.json new file mode 100644 index 0000000000..5f06d4f0d2 --- /dev/null +++ b/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/WITT2IRetrieval.json @@ -0,0 +1,1936 @@ +{ + "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", + "evaluation_time": 153.49457120895386, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.21236, + "cv_recall_at_10": 0.44831, + "cv_recall_at_100": 0.78989, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.54944, + "cv_recall_at_3": 0.30562, + "cv_recall_at_5": 0.36292, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.3176, + "map_at_1": 0.21236, + "map_at_10": 0.27738, + "map_at_100": 0.2907, + "map_at_1000": 0.29165, + "map_at_20": 0.28443, + "map_at_3": 0.253, + "map_at_5": 0.26603, + "mrr_at_1": 0.21235955056179776, + "mrr_at_10": 0.2773800606384876, + "mrr_at_100": 0.29070094283629366, + "mrr_at_1000": 0.2916475905772476, + "mrr_at_20": 0.2844331444819757, + "mrr_at_3": 0.25299625468164805, + "mrr_at_5": 0.2660299625468165, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.3003626994779616, + "nauc_cv_recall_at_100_max": 0.5548275759629901, + "nauc_cv_recall_at_100_std": 0.6386697598297497, + "nauc_cv_recall_at_10_diff1": 0.4149575835645685, + "nauc_cv_recall_at_10_max": 0.3425415745769017, + "nauc_cv_recall_at_10_std": 0.33509351614530253, + "nauc_cv_recall_at_1_diff1": 0.6050929690032468, + "nauc_cv_recall_at_1_max": 0.41251419289584673, + "nauc_cv_recall_at_1_std": 0.25816633985365167, + "nauc_cv_recall_at_20_diff1": 0.40426712835286205, + "nauc_cv_recall_at_20_max": 0.4027644407021793, + "nauc_cv_recall_at_20_std": 0.4096739656605129, + "nauc_cv_recall_at_3_diff1": 0.49874039399022607, + "nauc_cv_recall_at_3_max": 0.3525352598845765, + "nauc_cv_recall_at_3_std": 0.2826150147458686, + "nauc_cv_recall_at_5_diff1": 0.4653915178854044, + "nauc_cv_recall_at_5_max": 0.3569189727946339, + "nauc_cv_recall_at_5_std": 0.3028674653031839, + "nauc_map_at_1000_diff1": 0.5409310976173939, + "nauc_map_at_1000_max": 0.3885752414310074, + "nauc_map_at_1000_std": 0.2865922308596221, + "nauc_map_at_100_diff1": 0.5403803030050892, + "nauc_map_at_100_max": 0.38901485287942417, + "nauc_map_at_100_std": 0.28745240579931414, + "nauc_map_at_10_diff1": 0.5415319832681147, + "nauc_map_at_10_max": 0.38328218679842607, + "nauc_map_at_10_std": 0.28077001695899756, + "nauc_map_at_1_diff1": 0.6050929690032468, + "nauc_map_at_1_max": 0.41251419289584673, + "nauc_map_at_1_std": 0.25816633985365167, + "nauc_map_at_20_diff1": 0.5411089156357143, + "nauc_map_at_20_max": 0.38701139904307347, + "nauc_map_at_20_std": 0.2856492071304046, + "nauc_map_at_3_diff1": 0.5576262979028926, + "nauc_map_at_3_max": 0.38367029894992183, + "nauc_map_at_3_std": 0.2697064390376009, + "nauc_map_at_5_diff1": 0.5489353603103189, + "nauc_map_at_5_max": 0.3852603217798598, + "nauc_map_at_5_std": 0.2755182157848625, + "nauc_mrr_at_1000_diff1": 0.540931085899091, + "nauc_mrr_at_1000_max": 0.3885752266951037, + "nauc_mrr_at_1000_std": 0.2865922209021115, + "nauc_mrr_at_100_diff1": 0.5403803030050892, + "nauc_mrr_at_100_max": 0.38901485287942417, + "nauc_mrr_at_100_std": 0.28745240579931414, + "nauc_mrr_at_10_diff1": 0.5415319832681147, + "nauc_mrr_at_10_max": 0.38328218679842607, + "nauc_mrr_at_10_std": 0.28077001695899756, + "nauc_mrr_at_1_diff1": 0.6050929690032468, + "nauc_mrr_at_1_max": 0.41251419289584673, + "nauc_mrr_at_1_std": 0.25816633985365167, + "nauc_mrr_at_20_diff1": 0.5411089156357143, + "nauc_mrr_at_20_max": 0.38701139904307347, + "nauc_mrr_at_20_std": 0.2856492071304046, + "nauc_mrr_at_3_diff1": 0.5576262979028926, + "nauc_mrr_at_3_max": 0.38367029894992183, + "nauc_mrr_at_3_std": 0.2697064390376009, + "nauc_mrr_at_5_diff1": 0.5489353603103189, + "nauc_mrr_at_5_max": 0.3852603217798598, + "nauc_mrr_at_5_std": 0.2755182157848625, + "nauc_ndcg_at_1000_diff1": 0.5201783586237868, + "nauc_ndcg_at_1000_max": 0.39109832441616876, + "nauc_ndcg_at_1000_std": 0.30461652448903986, + "nauc_ndcg_at_100_diff1": 0.5027918968844239, + "nauc_ndcg_at_100_max": 0.40436472190745265, + "nauc_ndcg_at_100_std": 0.331281945296951, + "nauc_ndcg_at_10_diff1": 0.5089323671477836, + "nauc_ndcg_at_10_max": 0.37255982146503086, + "nauc_ndcg_at_10_std": 0.2945653330281739, + "nauc_ndcg_at_1_diff1": 0.6050929690032468, + "nauc_ndcg_at_1_max": 0.41251419289584673, + "nauc_ndcg_at_1_std": 0.25816633985365167, + "nauc_ndcg_at_20_diff1": 0.5075522728112374, + "nauc_ndcg_at_20_max": 0.38771287487979855, + "nauc_ndcg_at_20_std": 0.3128682588994658, + "nauc_ndcg_at_3_diff1": 0.5416767114387823, + "nauc_ndcg_at_3_max": 0.37515635453117174, + "nauc_ndcg_at_3_std": 0.27322728588827544, + "nauc_ndcg_at_5_diff1": 0.5264649237697293, + "nauc_ndcg_at_5_max": 0.3774223708861285, + "nauc_ndcg_at_5_std": 0.28285178016394, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3003626994779613, + "nauc_precision_at_100_max": 0.5548275759629899, + "nauc_precision_at_100_std": 0.63866975982975, + "nauc_precision_at_10_diff1": 0.4149575835645688, + "nauc_precision_at_10_max": 0.3425415745769019, + "nauc_precision_at_10_std": 0.3350935161453028, + "nauc_precision_at_1_diff1": 0.6050929690032468, + "nauc_precision_at_1_max": 0.41251419289584673, + "nauc_precision_at_1_std": 0.25816633985365167, + "nauc_precision_at_20_diff1": 0.4042671283528619, + "nauc_precision_at_20_max": 0.40276444070217876, + "nauc_precision_at_20_std": 0.40967396566051206, + "nauc_precision_at_3_diff1": 0.49874039399022574, + "nauc_precision_at_3_max": 0.3525352598845762, + "nauc_precision_at_3_std": 0.28261501474586836, + "nauc_precision_at_5_diff1": 0.46539151788540417, + "nauc_precision_at_5_max": 0.356918972794634, + "nauc_precision_at_5_std": 0.302867465303184, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.3003626994779616, + "nauc_recall_at_100_max": 0.5548275759629901, + "nauc_recall_at_100_std": 0.6386697598297497, + "nauc_recall_at_10_diff1": 0.4149575835645685, + "nauc_recall_at_10_max": 0.3425415745769017, + "nauc_recall_at_10_std": 0.33509351614530253, + "nauc_recall_at_1_diff1": 0.6050929690032468, + "nauc_recall_at_1_max": 0.41251419289584673, + "nauc_recall_at_1_std": 0.25816633985365167, + "nauc_recall_at_20_diff1": 0.40426712835286205, + "nauc_recall_at_20_max": 0.4027644407021793, + "nauc_recall_at_20_std": 0.4096739656605129, + "nauc_recall_at_3_diff1": 0.49874039399022607, + "nauc_recall_at_3_max": 0.3525352598845765, + "nauc_recall_at_3_std": 0.2826150147458686, + "nauc_recall_at_5_diff1": 0.4653915178854044, + "nauc_recall_at_5_max": 0.3569189727946339, + "nauc_recall_at_5_std": 0.3028674653031839, + "ndcg_at_1": 0.21236, + "ndcg_at_10": 0.3176, + "ndcg_at_100": 0.38789, + "ndcg_at_1000": 0.41438, + "ndcg_at_20": 0.34321, + "ndcg_at_3": 0.26649, + "ndcg_at_5": 0.29004, + "precision_at_1": 0.21236, + "precision_at_10": 0.04483, + "precision_at_100": 0.0079, + "precision_at_1000": 0.001, + "precision_at_20": 0.02747, + "precision_at_3": 0.10187, + "precision_at_5": 0.07258, + "recall_at_1": 0.21236, + "recall_at_10": 0.44831, + "recall_at_100": 0.78989, + "recall_at_1000": 1.0, + "recall_at_20": 0.54944, + "recall_at_3": 0.30562, + "recall_at_5": 0.36292 + }, + { + "cv_recall_at_1": 0.20698, + "cv_recall_at_10": 0.44419, + "cv_recall_at_100": 0.76977, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.54186, + "cv_recall_at_3": 0.30233, + "cv_recall_at_5": 0.36047, + "hf_subset": "bg", + "languages": [ + "bul-Cyrl" + ], + "main_score": 0.31507, + "map_at_1": 0.20698, + "map_at_10": 0.27516, + "map_at_100": 0.28753, + "map_at_1000": 0.28861, + "map_at_20": 0.28201, + "map_at_3": 0.25058, + "map_at_5": 0.26395, + "mrr_at_1": 0.2069767441860465, + "mrr_at_10": 0.2751591915836101, + "mrr_at_100": 0.2875307284962649, + "mrr_at_1000": 0.2886101346549318, + "mrr_at_20": 0.2820103602457893, + "mrr_at_3": 0.2505813953488373, + "mrr_at_5": 0.26395348837209304, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4093627621786638, + "nauc_cv_recall_at_100_max": 0.47392125148869085, + "nauc_cv_recall_at_100_std": 0.32742884186329557, + "nauc_cv_recall_at_10_diff1": 0.4164293507827622, + "nauc_cv_recall_at_10_max": 0.373676020508955, + "nauc_cv_recall_at_10_std": 0.07981411859816971, + "nauc_cv_recall_at_1_diff1": 0.5769623985509127, + "nauc_cv_recall_at_1_max": 0.3842526568455678, + "nauc_cv_recall_at_1_std": 0.06934283587010087, + "nauc_cv_recall_at_20_diff1": 0.37682780365738766, + "nauc_cv_recall_at_20_max": 0.41811018778993125, + "nauc_cv_recall_at_20_std": 0.129614222958865, + "nauc_cv_recall_at_3_diff1": 0.47505414142682256, + "nauc_cv_recall_at_3_max": 0.3386255924170616, + "nauc_cv_recall_at_3_std": 0.06946109663852355, + "nauc_cv_recall_at_5_diff1": 0.4463062678279969, + "nauc_cv_recall_at_5_max": 0.3507802370456244, + "nauc_cv_recall_at_5_std": 0.07886114373751092, + "nauc_map_at_1000_diff1": 0.5146769065802478, + "nauc_map_at_1000_max": 0.37646808082113914, + "nauc_map_at_1000_std": 0.08646507199309812, + "nauc_map_at_100_diff1": 0.5143120882455383, + "nauc_map_at_100_max": 0.3766277483299629, + "nauc_map_at_100_std": 0.08675906951811373, + "nauc_map_at_10_diff1": 0.5152070626723227, + "nauc_map_at_10_max": 0.3727333152390514, + "nauc_map_at_10_std": 0.08099420288084715, + "nauc_map_at_1_diff1": 0.5769623985509127, + "nauc_map_at_1_max": 0.3842526568455678, + "nauc_map_at_1_std": 0.06934283587010087, + "nauc_map_at_20_diff1": 0.5125542972976519, + "nauc_map_at_20_max": 0.37611677581301717, + "nauc_map_at_20_std": 0.0846629410716143, + "nauc_map_at_3_diff1": 0.5270295580868876, + "nauc_map_at_3_max": 0.3663095452924379, + "nauc_map_at_3_std": 0.07738161926762027, + "nauc_map_at_5_diff1": 0.519948492737527, + "nauc_map_at_5_max": 0.3689441605572904, + "nauc_map_at_5_std": 0.08060487493398132, + "nauc_mrr_at_1000_diff1": 0.514676902292366, + "nauc_mrr_at_1000_max": 0.37646806310992137, + "nauc_mrr_at_1000_std": 0.08646505360870692, + "nauc_mrr_at_100_diff1": 0.5143120882455383, + "nauc_mrr_at_100_max": 0.3766277483299629, + "nauc_mrr_at_100_std": 0.08675906951811373, + "nauc_mrr_at_10_diff1": 0.5152070626723227, + "nauc_mrr_at_10_max": 0.3727333152390514, + "nauc_mrr_at_10_std": 0.08099420288084715, + "nauc_mrr_at_1_diff1": 0.5769623985509127, + "nauc_mrr_at_1_max": 0.3842526568455678, + "nauc_mrr_at_1_std": 0.06934283587010087, + "nauc_mrr_at_20_diff1": 0.5125542972976519, + "nauc_mrr_at_20_max": 0.37611677581301717, + "nauc_mrr_at_20_std": 0.0846629410716143, + "nauc_mrr_at_3_diff1": 0.5270295580868876, + "nauc_mrr_at_3_max": 0.3663095452924379, + "nauc_mrr_at_3_std": 0.07738161926762027, + "nauc_mrr_at_5_diff1": 0.519948492737527, + "nauc_mrr_at_5_max": 0.3689441605572904, + "nauc_mrr_at_5_std": 0.08060487493398132, + "nauc_ndcg_at_1000_diff1": 0.49995543334551823, + "nauc_ndcg_at_1000_max": 0.3811523747562341, + "nauc_ndcg_at_1000_std": 0.09738443817453372, + "nauc_ndcg_at_100_diff1": 0.49170577344667954, + "nauc_ndcg_at_100_max": 0.3884156484654683, + "nauc_ndcg_at_100_std": 0.11507089046060567, + "nauc_ndcg_at_10_diff1": 0.4892258089021453, + "nauc_ndcg_at_10_max": 0.3722492080992435, + "nauc_ndcg_at_10_std": 0.081001233642932, + "nauc_ndcg_at_1_diff1": 0.5769623985509127, + "nauc_ndcg_at_1_max": 0.3842526568455678, + "nauc_ndcg_at_1_std": 0.06934283587010087, + "nauc_ndcg_at_20_diff1": 0.47983152317657074, + "nauc_ndcg_at_20_max": 0.3840360365384102, + "nauc_ndcg_at_20_std": 0.0940749988684443, + "nauc_ndcg_at_3_diff1": 0.5127760637828862, + "nauc_ndcg_at_3_max": 0.3589028686816543, + "nauc_ndcg_at_3_std": 0.07554452472073625, + "nauc_ndcg_at_5_diff1": 0.5000070371620252, + "nauc_ndcg_at_5_max": 0.3638454293511532, + "nauc_ndcg_at_5_std": 0.08041489130513192, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.40936276217866474, + "nauc_precision_at_100_max": 0.4739212514886919, + "nauc_precision_at_100_std": 0.3274288418632965, + "nauc_precision_at_10_diff1": 0.4164293507827624, + "nauc_precision_at_10_max": 0.3736760205089551, + "nauc_precision_at_10_std": 0.07981411859816984, + "nauc_precision_at_1_diff1": 0.5769623985509127, + "nauc_precision_at_1_max": 0.3842526568455678, + "nauc_precision_at_1_std": 0.06934283587010087, + "nauc_precision_at_20_diff1": 0.3768278036573875, + "nauc_precision_at_20_max": 0.4181101877899317, + "nauc_precision_at_20_std": 0.12961422295886474, + "nauc_precision_at_3_diff1": 0.47505414142682284, + "nauc_precision_at_3_max": 0.33862559241706147, + "nauc_precision_at_3_std": 0.06946109663852358, + "nauc_precision_at_5_diff1": 0.4463062678279969, + "nauc_precision_at_5_max": 0.35078023704562433, + "nauc_precision_at_5_std": 0.07886114373751069, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4093627621786638, + "nauc_recall_at_100_max": 0.47392125148869085, + "nauc_recall_at_100_std": 0.32742884186329557, + "nauc_recall_at_10_diff1": 0.4164293507827622, + "nauc_recall_at_10_max": 0.373676020508955, + "nauc_recall_at_10_std": 0.07981411859816971, + "nauc_recall_at_1_diff1": 0.5769623985509127, + "nauc_recall_at_1_max": 0.3842526568455678, + "nauc_recall_at_1_std": 0.06934283587010087, + "nauc_recall_at_20_diff1": 0.37682780365738766, + "nauc_recall_at_20_max": 0.41811018778993125, + "nauc_recall_at_20_std": 0.129614222958865, + "nauc_recall_at_3_diff1": 0.47505414142682256, + "nauc_recall_at_3_max": 0.3386255924170616, + "nauc_recall_at_3_std": 0.06946109663852355, + "nauc_recall_at_5_diff1": 0.4463062678279969, + "nauc_recall_at_5_max": 0.3507802370456244, + "nauc_recall_at_5_std": 0.07886114373751092, + "ndcg_at_1": 0.20698, + "ndcg_at_10": 0.31507, + "ndcg_at_100": 0.38147, + "ndcg_at_1000": 0.4107, + "ndcg_at_20": 0.33986, + "ndcg_at_3": 0.26394, + "ndcg_at_5": 0.28796, + "precision_at_1": 0.20698, + "precision_at_10": 0.04442, + "precision_at_100": 0.0077, + "precision_at_1000": 0.001, + "precision_at_20": 0.02709, + "precision_at_3": 0.10078, + "precision_at_5": 0.07209, + "recall_at_1": 0.20698, + "recall_at_10": 0.44419, + "recall_at_100": 0.76977, + "recall_at_1000": 1.0, + "recall_at_20": 0.54186, + "recall_at_3": 0.30233, + "recall_at_5": 0.36047 + }, + { + "cv_recall_at_1": 0.39731, + "cv_recall_at_10": 0.66442, + "cv_recall_at_100": 0.86083, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.72503, + "cv_recall_at_3": 0.53423, + "cv_recall_at_5": 0.59596, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.52466, + "map_at_1": 0.39731, + "map_at_10": 0.48064, + "map_at_100": 0.48803, + "map_at_1000": 0.48868, + "map_at_20": 0.48489, + "map_at_3": 0.45754, + "map_at_5": 0.47162, + "mrr_at_1": 0.39730639730639733, + "mrr_at_10": 0.48064240286462473, + "mrr_at_100": 0.4880324754856483, + "mrr_at_1000": 0.48868275160675934, + "mrr_at_20": 0.48488937567285484, + "mrr_at_3": 0.457538346427235, + "mrr_at_5": 0.4716236438458655, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5578548171577201, + "nauc_cv_recall_at_100_max": 0.6646804698095687, + "nauc_cv_recall_at_100_std": 0.6195215557966229, + "nauc_cv_recall_at_10_diff1": 0.5838135625092163, + "nauc_cv_recall_at_10_max": 0.5383944730775565, + "nauc_cv_recall_at_10_std": 0.29288807484330076, + "nauc_cv_recall_at_1_diff1": 0.7542846884818821, + "nauc_cv_recall_at_1_max": 0.4420202485145873, + "nauc_cv_recall_at_1_std": 0.0004563032507581658, + "nauc_cv_recall_at_20_diff1": 0.5442476748139404, + "nauc_cv_recall_at_20_max": 0.5818878214385517, + "nauc_cv_recall_at_20_std": 0.367573720549519, + "nauc_cv_recall_at_3_diff1": 0.6588395819935658, + "nauc_cv_recall_at_3_max": 0.5000519416651473, + "nauc_cv_recall_at_3_std": 0.150187884602865, + "nauc_cv_recall_at_5_diff1": 0.63254204802205, + "nauc_cv_recall_at_5_max": 0.48845255697186757, + "nauc_cv_recall_at_5_std": 0.18065335713285896, + "nauc_map_at_1000_diff1": 0.7059026049582101, + "nauc_map_at_1000_max": 0.4738480119813671, + "nauc_map_at_1000_std": 0.0822114309839713, + "nauc_map_at_100_diff1": 0.7056131432990436, + "nauc_map_at_100_max": 0.4740255850949717, + "nauc_map_at_100_std": 0.08300869243502265, + "nauc_map_at_10_diff1": 0.7055998064181506, + "nauc_map_at_10_max": 0.4727748241610827, + "nauc_map_at_10_std": 0.08008450961045248, + "nauc_map_at_1_diff1": 0.7542846884818821, + "nauc_map_at_1_max": 0.4420202485145873, + "nauc_map_at_1_std": 0.0004563032507581658, + "nauc_map_at_20_diff1": 0.7045296733768378, + "nauc_map_at_20_max": 0.47417479130977996, + "nauc_map_at_20_std": 0.08193767726788206, + "nauc_map_at_3_diff1": 0.7140923458572953, + "nauc_map_at_3_max": 0.4703630865323559, + "nauc_map_at_3_std": 0.06460714619505123, + "nauc_map_at_5_diff1": 0.7095881165016424, + "nauc_map_at_5_max": 0.4677367980674292, + "nauc_map_at_5_std": 0.0700858298315289, + "nauc_mrr_at_1000_diff1": 0.7059026049582101, + "nauc_mrr_at_1000_max": 0.4738480119813671, + "nauc_mrr_at_1000_std": 0.0822114309839713, + "nauc_mrr_at_100_diff1": 0.7056131432990436, + "nauc_mrr_at_100_max": 0.4740255850949717, + "nauc_mrr_at_100_std": 0.08300869243502265, + "nauc_mrr_at_10_diff1": 0.7055998064181506, + "nauc_mrr_at_10_max": 0.4727748241610827, + "nauc_mrr_at_10_std": 0.08008450961045248, + "nauc_mrr_at_1_diff1": 0.7542846884818821, + "nauc_mrr_at_1_max": 0.4420202485145873, + "nauc_mrr_at_1_std": 0.0004563032507581658, + "nauc_mrr_at_20_diff1": 0.7045296733768378, + "nauc_mrr_at_20_max": 0.47417479130977996, + "nauc_mrr_at_20_std": 0.08193767726788206, + "nauc_mrr_at_3_diff1": 0.7140923458572953, + "nauc_mrr_at_3_max": 0.4703630865323559, + "nauc_mrr_at_3_std": 0.06460714619505123, + "nauc_mrr_at_5_diff1": 0.7095881165016424, + "nauc_mrr_at_5_max": 0.4677367980674292, + "nauc_mrr_at_5_std": 0.0700858298315289, + "nauc_ndcg_at_1000_diff1": 0.6926431148768741, + "nauc_ndcg_at_1000_max": 0.48413074049723803, + "nauc_ndcg_at_1000_std": 0.1119721669195201, + "nauc_ndcg_at_100_diff1": 0.6848643290944362, + "nauc_ndcg_at_100_max": 0.4932360054078029, + "nauc_ndcg_at_100_std": 0.1394665807430548, + "nauc_ndcg_at_10_diff1": 0.6805052582761167, + "nauc_ndcg_at_10_max": 0.4861708577750309, + "nauc_ndcg_at_10_std": 0.12339785683878872, + "nauc_ndcg_at_1_diff1": 0.7542846884818821, + "nauc_ndcg_at_1_max": 0.4420202485145873, + "nauc_ndcg_at_1_std": 0.0004563032507581658, + "nauc_ndcg_at_20_diff1": 0.6757353483087946, + "nauc_ndcg_at_20_max": 0.4926080218871637, + "nauc_ndcg_at_20_std": 0.13236056140731542, + "nauc_ndcg_at_3_diff1": 0.7003536373293086, + "nauc_ndcg_at_3_max": 0.477911487093341, + "nauc_ndcg_at_3_std": 0.0859337040441357, + "nauc_ndcg_at_5_diff1": 0.6915361391824972, + "nauc_ndcg_at_5_max": 0.47306885865040066, + "nauc_ndcg_at_5_std": 0.09623013241600045, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.5578548171577202, + "nauc_precision_at_100_max": 0.664680469809571, + "nauc_precision_at_100_std": 0.6195215557966243, + "nauc_precision_at_10_diff1": 0.5838135625092153, + "nauc_precision_at_10_max": 0.5383944730775557, + "nauc_precision_at_10_std": 0.2928880748432999, + "nauc_precision_at_1_diff1": 0.7542846884818821, + "nauc_precision_at_1_max": 0.4420202485145873, + "nauc_precision_at_1_std": 0.0004563032507581658, + "nauc_precision_at_20_diff1": 0.54424767481394, + "nauc_precision_at_20_max": 0.5818878214385511, + "nauc_precision_at_20_std": 0.367573720549519, + "nauc_precision_at_3_diff1": 0.658839581993566, + "nauc_precision_at_3_max": 0.500051941665147, + "nauc_precision_at_3_std": 0.1501878846028649, + "nauc_precision_at_5_diff1": 0.6325420480220502, + "nauc_precision_at_5_max": 0.488452556971868, + "nauc_precision_at_5_std": 0.18065335713285838, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5578548171577201, + "nauc_recall_at_100_max": 0.6646804698095687, + "nauc_recall_at_100_std": 0.6195215557966229, + "nauc_recall_at_10_diff1": 0.5838135625092163, + "nauc_recall_at_10_max": 0.5383944730775565, + "nauc_recall_at_10_std": 0.29288807484330076, + "nauc_recall_at_1_diff1": 0.7542846884818821, + "nauc_recall_at_1_max": 0.4420202485145873, + "nauc_recall_at_1_std": 0.0004563032507581658, + "nauc_recall_at_20_diff1": 0.5442476748139404, + "nauc_recall_at_20_max": 0.5818878214385517, + "nauc_recall_at_20_std": 0.367573720549519, + "nauc_recall_at_3_diff1": 0.6588395819935658, + "nauc_recall_at_3_max": 0.5000519416651473, + "nauc_recall_at_3_std": 0.150187884602865, + "nauc_recall_at_5_diff1": 0.63254204802205, + "nauc_recall_at_5_max": 0.48845255697186757, + "nauc_recall_at_5_std": 0.18065335713285896, + "ndcg_at_1": 0.39731, + "ndcg_at_10": 0.52466, + "ndcg_at_100": 0.56456, + "ndcg_at_1000": 0.58213, + "ndcg_at_20": 0.54003, + "ndcg_at_3": 0.47723, + "ndcg_at_5": 0.50264, + "precision_at_1": 0.39731, + "precision_at_10": 0.06644, + "precision_at_100": 0.00861, + "precision_at_1000": 0.001, + "precision_at_20": 0.03625, + "precision_at_3": 0.17808, + "precision_at_5": 0.11919, + "recall_at_1": 0.39731, + "recall_at_10": 0.66442, + "recall_at_100": 0.86083, + "recall_at_1000": 1.0, + "recall_at_20": 0.72503, + "recall_at_3": 0.53423, + "recall_at_5": 0.59596 + }, + { + "cv_recall_at_1": 0.15614, + "cv_recall_at_10": 0.33509, + "cv_recall_at_100": 0.68772, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.42807, + "cv_recall_at_3": 0.22456, + "cv_recall_at_5": 0.26491, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.23459, + "map_at_1": 0.15614, + "map_at_10": 0.20374, + "map_at_100": 0.2165, + "map_at_1000": 0.21805, + "map_at_20": 0.2104, + "map_at_3": 0.18509, + "map_at_5": 0.19404, + "mrr_at_1": 0.156140350877193, + "mrr_at_10": 0.20374199387357286, + "mrr_at_100": 0.21650435818001867, + "mrr_at_1000": 0.21805273596584335, + "mrr_at_20": 0.21039989765242342, + "mrr_at_3": 0.18508771929824558, + "mrr_at_5": 0.19403508771929825, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.18910062947313946, + "nauc_cv_recall_at_100_max": 0.19924487860160875, + "nauc_cv_recall_at_100_std": 0.3410333953362322, + "nauc_cv_recall_at_10_diff1": 0.39068306628424715, + "nauc_cv_recall_at_10_max": 0.22095681004294052, + "nauc_cv_recall_at_10_std": 0.23713586068868153, + "nauc_cv_recall_at_1_diff1": 0.628447781785485, + "nauc_cv_recall_at_1_max": 0.33173520761334724, + "nauc_cv_recall_at_1_std": 0.20001166849129834, + "nauc_cv_recall_at_20_diff1": 0.32431002717936575, + "nauc_cv_recall_at_20_max": 0.21413783039346246, + "nauc_cv_recall_at_20_std": 0.29045666806515763, + "nauc_cv_recall_at_3_diff1": 0.49887013590805396, + "nauc_cv_recall_at_3_max": 0.3020034814950967, + "nauc_cv_recall_at_3_std": 0.24365329931126942, + "nauc_cv_recall_at_5_diff1": 0.443961409746126, + "nauc_cv_recall_at_5_max": 0.25070338959611993, + "nauc_cv_recall_at_5_std": 0.21037823376923773, + "nauc_map_at_1000_diff1": 0.5312774816883872, + "nauc_map_at_1000_max": 0.2889799028008814, + "nauc_map_at_1000_std": 0.2172852085582543, + "nauc_map_at_100_diff1": 0.5296079561010105, + "nauc_map_at_100_max": 0.2884402055859716, + "nauc_map_at_100_std": 0.21772815373049878, + "nauc_map_at_10_diff1": 0.5390101378342672, + "nauc_map_at_10_max": 0.28914187258841156, + "nauc_map_at_10_std": 0.21070249040924247, + "nauc_map_at_1_diff1": 0.628447781785485, + "nauc_map_at_1_max": 0.33173520761334724, + "nauc_map_at_1_std": 0.20001166849129834, + "nauc_map_at_20_diff1": 0.5322467062292103, + "nauc_map_at_20_max": 0.28831451293746013, + "nauc_map_at_20_std": 0.21641674001551306, + "nauc_map_at_3_diff1": 0.5662610467436214, + "nauc_map_at_3_max": 0.30828646913451574, + "nauc_map_at_3_std": 0.21400270072668254, + "nauc_map_at_5_diff1": 0.5508293813433176, + "nauc_map_at_5_max": 0.2952789941475848, + "nauc_map_at_5_std": 0.2056254855366493, + "nauc_mrr_at_1000_diff1": 0.5312774816883872, + "nauc_mrr_at_1000_max": 0.2889799028008814, + "nauc_mrr_at_1000_std": 0.2172852085582543, + "nauc_mrr_at_100_diff1": 0.5296079561010105, + "nauc_mrr_at_100_max": 0.2884402055859716, + "nauc_mrr_at_100_std": 0.21772815373049878, + "nauc_mrr_at_10_diff1": 0.5390101378342672, + "nauc_mrr_at_10_max": 0.28914187258841156, + "nauc_mrr_at_10_std": 0.21070249040924247, + "nauc_mrr_at_1_diff1": 0.628447781785485, + "nauc_mrr_at_1_max": 0.33173520761334724, + "nauc_mrr_at_1_std": 0.20001166849129834, + "nauc_mrr_at_20_diff1": 0.5322467062292103, + "nauc_mrr_at_20_max": 0.28831451293746013, + "nauc_mrr_at_20_std": 0.21641674001551306, + "nauc_mrr_at_3_diff1": 0.5662610467436214, + "nauc_mrr_at_3_max": 0.30828646913451574, + "nauc_mrr_at_3_std": 0.21400270072668254, + "nauc_mrr_at_5_diff1": 0.5508293813433176, + "nauc_mrr_at_5_max": 0.2952789941475848, + "nauc_mrr_at_5_std": 0.2056254855366493, + "nauc_ndcg_at_1000_diff1": 0.49664151547278534, + "nauc_ndcg_at_1000_max": 0.27832871617128796, + "nauc_ndcg_at_1000_std": 0.22965922218655035, + "nauc_ndcg_at_100_diff1": 0.4588080600618917, + "nauc_ndcg_at_100_max": 0.2680381852496268, + "nauc_ndcg_at_100_std": 0.2424971601594663, + "nauc_ndcg_at_10_diff1": 0.4966615039112077, + "nauc_ndcg_at_10_max": 0.270000010997535, + "nauc_ndcg_at_10_std": 0.21810824175170437, + "nauc_ndcg_at_1_diff1": 0.628447781785485, + "nauc_ndcg_at_1_max": 0.33173520761334724, + "nauc_ndcg_at_1_std": 0.20001166849129834, + "nauc_ndcg_at_20_diff1": 0.47507837516767815, + "nauc_ndcg_at_20_max": 0.2675238423292141, + "nauc_ndcg_at_20_std": 0.23558037224046202, + "nauc_ndcg_at_3_diff1": 0.5473931129806788, + "nauc_ndcg_at_3_max": 0.30616586794051853, + "nauc_ndcg_at_3_std": 0.22203037329509664, + "nauc_ndcg_at_5_diff1": 0.5207282709796918, + "nauc_ndcg_at_5_max": 0.2828848040725794, + "nauc_ndcg_at_5_std": 0.20724363575113774, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.1891006294731396, + "nauc_precision_at_100_max": 0.19924487860160886, + "nauc_precision_at_100_std": 0.341033395336232, + "nauc_precision_at_10_diff1": 0.39068306628424715, + "nauc_precision_at_10_max": 0.2209568100429408, + "nauc_precision_at_10_std": 0.2371358606886818, + "nauc_precision_at_1_diff1": 0.628447781785485, + "nauc_precision_at_1_max": 0.33173520761334724, + "nauc_precision_at_1_std": 0.20001166849129834, + "nauc_precision_at_20_diff1": 0.3243100271793654, + "nauc_precision_at_20_max": 0.21413783039346218, + "nauc_precision_at_20_std": 0.2904566680651574, + "nauc_precision_at_3_diff1": 0.49887013590805374, + "nauc_precision_at_3_max": 0.3020034814950966, + "nauc_precision_at_3_std": 0.24365329931126947, + "nauc_precision_at_5_diff1": 0.44396140974612597, + "nauc_precision_at_5_max": 0.2507033895961199, + "nauc_precision_at_5_std": 0.210378233769238, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.18910062947313946, + "nauc_recall_at_100_max": 0.19924487860160875, + "nauc_recall_at_100_std": 0.3410333953362322, + "nauc_recall_at_10_diff1": 0.39068306628424715, + "nauc_recall_at_10_max": 0.22095681004294052, + "nauc_recall_at_10_std": 0.23713586068868153, + "nauc_recall_at_1_diff1": 0.628447781785485, + "nauc_recall_at_1_max": 0.33173520761334724, + "nauc_recall_at_1_std": 0.20001166849129834, + "nauc_recall_at_20_diff1": 0.32431002717936575, + "nauc_recall_at_20_max": 0.21413783039346246, + "nauc_recall_at_20_std": 0.29045666806515763, + "nauc_recall_at_3_diff1": 0.49887013590805396, + "nauc_recall_at_3_max": 0.3020034814950967, + "nauc_recall_at_3_std": 0.24365329931126942, + "nauc_recall_at_5_diff1": 0.443961409746126, + "nauc_recall_at_5_max": 0.25070338959611993, + "nauc_recall_at_5_std": 0.21037823376923773, + "ndcg_at_1": 0.15614, + "ndcg_at_10": 0.23459, + "ndcg_at_100": 0.30538, + "ndcg_at_1000": 0.34567, + "ndcg_at_20": 0.25836, + "ndcg_at_3": 0.19517, + "ndcg_at_5": 0.21155, + "precision_at_1": 0.15614, + "precision_at_10": 0.03351, + "precision_at_100": 0.00688, + "precision_at_1000": 0.001, + "precision_at_20": 0.0214, + "precision_at_3": 0.07485, + "precision_at_5": 0.05298, + "recall_at_1": 0.15614, + "recall_at_10": 0.33509, + "recall_at_100": 0.68772, + "recall_at_1000": 1.0, + "recall_at_20": 0.42807, + "recall_at_3": 0.22456, + "recall_at_5": 0.26491 + }, + { + "cv_recall_at_1": 0.21053, + "cv_recall_at_10": 0.40732, + "cv_recall_at_100": 0.75629, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.50343, + "cv_recall_at_3": 0.29176, + "cv_recall_at_5": 0.33524, + "hf_subset": "et", + "languages": [ + "est-Latn" + ], + "main_score": 0.29932, + "map_at_1": 0.21053, + "map_at_10": 0.266, + "map_at_100": 0.27887, + "map_at_1000": 0.28, + "map_at_20": 0.27243, + "map_at_3": 0.24638, + "map_at_5": 0.25633, + "mrr_at_1": 0.21052631578947367, + "mrr_at_10": 0.2659987831898585, + "mrr_at_100": 0.2788741725576141, + "mrr_at_1000": 0.2799981947148587, + "mrr_at_20": 0.2724331888685576, + "mrr_at_3": 0.24637681159420294, + "mrr_at_5": 0.2563310450038138, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4040309098626024, + "nauc_cv_recall_at_100_max": 0.5072303739534308, + "nauc_cv_recall_at_100_std": 0.5322222637960802, + "nauc_cv_recall_at_10_diff1": 0.41350108355389, + "nauc_cv_recall_at_10_max": 0.29351616464676317, + "nauc_cv_recall_at_10_std": 0.19708885406673227, + "nauc_cv_recall_at_1_diff1": 0.6372005102044834, + "nauc_cv_recall_at_1_max": 0.38613114738319604, + "nauc_cv_recall_at_1_std": 0.10131574096517776, + "nauc_cv_recall_at_20_diff1": 0.37351711120057546, + "nauc_cv_recall_at_20_max": 0.3140724644248817, + "nauc_cv_recall_at_20_std": 0.26700917458594203, + "nauc_cv_recall_at_3_diff1": 0.5132176534203662, + "nauc_cv_recall_at_3_max": 0.3226228179276247, + "nauc_cv_recall_at_3_std": 0.13885815939228147, + "nauc_cv_recall_at_5_diff1": 0.4491732287253029, + "nauc_cv_recall_at_5_max": 0.3105780062973309, + "nauc_cv_recall_at_5_std": 0.1721406687192357, + "nauc_map_at_1000_diff1": 0.5558302859344558, + "nauc_map_at_1000_max": 0.3527916196058264, + "nauc_map_at_1000_std": 0.13338075718467787, + "nauc_map_at_100_diff1": 0.555359372389194, + "nauc_map_at_100_max": 0.35310936558202816, + "nauc_map_at_100_std": 0.13464064729062267, + "nauc_map_at_10_diff1": 0.5556162167007911, + "nauc_map_at_10_max": 0.34786519732513577, + "nauc_map_at_10_std": 0.12641240527243502, + "nauc_map_at_1_diff1": 0.6372005102044834, + "nauc_map_at_1_max": 0.38613114738319604, + "nauc_map_at_1_std": 0.10131574096517776, + "nauc_map_at_20_diff1": 0.5532309928003525, + "nauc_map_at_20_max": 0.34939557135589966, + "nauc_map_at_20_std": 0.1313876717359136, + "nauc_map_at_3_diff1": 0.5782131762035241, + "nauc_map_at_3_max": 0.3538089928851203, + "nauc_map_at_3_std": 0.11391828595128985, + "nauc_map_at_5_diff1": 0.5613343361923882, + "nauc_map_at_5_max": 0.3505715403437261, + "nauc_map_at_5_std": 0.12276900548783347, + "nauc_mrr_at_1000_diff1": 0.5558302859344558, + "nauc_mrr_at_1000_max": 0.3527916196058264, + "nauc_mrr_at_1000_std": 0.13338075718467787, + "nauc_mrr_at_100_diff1": 0.555359372389194, + "nauc_mrr_at_100_max": 0.35310936558202816, + "nauc_mrr_at_100_std": 0.13464064729062267, + "nauc_mrr_at_10_diff1": 0.5556162167007911, + "nauc_mrr_at_10_max": 0.34786519732513577, + "nauc_mrr_at_10_std": 0.12641240527243502, + "nauc_mrr_at_1_diff1": 0.6372005102044834, + "nauc_mrr_at_1_max": 0.38613114738319604, + "nauc_mrr_at_1_std": 0.10131574096517776, + "nauc_mrr_at_20_diff1": 0.5532309928003525, + "nauc_mrr_at_20_max": 0.34939557135589966, + "nauc_mrr_at_20_std": 0.1313876717359136, + "nauc_mrr_at_3_diff1": 0.5782131762035241, + "nauc_mrr_at_3_max": 0.3538089928851203, + "nauc_mrr_at_3_std": 0.11391828595128985, + "nauc_mrr_at_5_diff1": 0.5613343361923882, + "nauc_mrr_at_5_max": 0.3505715403437261, + "nauc_mrr_at_5_std": 0.12276900548783347, + "nauc_ndcg_at_1000_diff1": 0.5349171072515045, + "nauc_ndcg_at_1000_max": 0.35374501452199153, + "nauc_ndcg_at_1000_std": 0.1556987345481063, + "nauc_ndcg_at_100_diff1": 0.5228210882759149, + "nauc_ndcg_at_100_max": 0.3668209735804216, + "nauc_ndcg_at_100_std": 0.1901665181862775, + "nauc_ndcg_at_10_diff1": 0.5179873440321987, + "nauc_ndcg_at_10_max": 0.3333289869905906, + "nauc_ndcg_at_10_std": 0.14459459944320588, + "nauc_ndcg_at_1_diff1": 0.6372005102044834, + "nauc_ndcg_at_1_max": 0.38613114738319604, + "nauc_ndcg_at_1_std": 0.10131574096517776, + "nauc_ndcg_at_20_diff1": 0.508535366784914, + "nauc_ndcg_at_20_max": 0.33907869749189457, + "nauc_ndcg_at_20_std": 0.16281917120234862, + "nauc_ndcg_at_3_diff1": 0.5605384341570335, + "nauc_ndcg_at_3_max": 0.34524089763458526, + "nauc_ndcg_at_3_std": 0.1204822260900556, + "nauc_ndcg_at_5_diff1": 0.5313293612744379, + "nauc_ndcg_at_5_max": 0.339634674431103, + "nauc_ndcg_at_5_std": 0.13570399957682294, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.4040309098626027, + "nauc_precision_at_100_max": 0.5072303739534312, + "nauc_precision_at_100_std": 0.5322222637960811, + "nauc_precision_at_10_diff1": 0.4135010835538899, + "nauc_precision_at_10_max": 0.29351616464676256, + "nauc_precision_at_10_std": 0.19708885406673196, + "nauc_precision_at_1_diff1": 0.6372005102044834, + "nauc_precision_at_1_max": 0.38613114738319604, + "nauc_precision_at_1_std": 0.10131574096517776, + "nauc_precision_at_20_diff1": 0.37351711120057546, + "nauc_precision_at_20_max": 0.3140724644248813, + "nauc_precision_at_20_std": 0.2670091745859422, + "nauc_precision_at_3_diff1": 0.5132176534203662, + "nauc_precision_at_3_max": 0.3226228179276242, + "nauc_precision_at_3_std": 0.1388581593922816, + "nauc_precision_at_5_diff1": 0.4491732287253025, + "nauc_precision_at_5_max": 0.31057800629733096, + "nauc_precision_at_5_std": 0.17214066871923572, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4040309098626024, + "nauc_recall_at_100_max": 0.5072303739534308, + "nauc_recall_at_100_std": 0.5322222637960802, + "nauc_recall_at_10_diff1": 0.41350108355389, + "nauc_recall_at_10_max": 0.29351616464676317, + "nauc_recall_at_10_std": 0.19708885406673227, + "nauc_recall_at_1_diff1": 0.6372005102044834, + "nauc_recall_at_1_max": 0.38613114738319604, + "nauc_recall_at_1_std": 0.10131574096517776, + "nauc_recall_at_20_diff1": 0.37351711120057546, + "nauc_recall_at_20_max": 0.3140724644248817, + "nauc_recall_at_20_std": 0.26700917458594203, + "nauc_recall_at_3_diff1": 0.5132176534203662, + "nauc_recall_at_3_max": 0.3226228179276247, + "nauc_recall_at_3_std": 0.13885815939228147, + "nauc_recall_at_5_diff1": 0.4491732287253029, + "nauc_recall_at_5_max": 0.3105780062973309, + "nauc_recall_at_5_std": 0.1721406687192357, + "ndcg_at_1": 0.21053, + "ndcg_at_10": 0.29932, + "ndcg_at_100": 0.3702, + "ndcg_at_1000": 0.40113, + "ndcg_at_20": 0.32333, + "ndcg_at_3": 0.25804, + "ndcg_at_5": 0.27596, + "precision_at_1": 0.21053, + "precision_at_10": 0.04073, + "precision_at_100": 0.00756, + "precision_at_1000": 0.001, + "precision_at_20": 0.02517, + "precision_at_3": 0.09725, + "precision_at_5": 0.06705, + "recall_at_1": 0.21053, + "recall_at_10": 0.40732, + "recall_at_100": 0.75629, + "recall_at_1000": 1.0, + "recall_at_20": 0.50343, + "recall_at_3": 0.29176, + "recall_at_5": 0.33524 + }, + { + "cv_recall_at_1": 0.48835, + "cv_recall_at_10": 0.74362, + "cv_recall_at_100": 0.94007, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.80577, + "cv_recall_at_3": 0.63263, + "cv_recall_at_5": 0.6859, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.61394, + "map_at_1": 0.48835, + "map_at_10": 0.57268, + "map_at_100": 0.58029, + "map_at_1000": 0.58059, + "map_at_20": 0.57698, + "map_at_3": 0.55272, + "map_at_5": 0.56509, + "mrr_at_1": 0.48834628190899, + "mrr_at_10": 0.5726758627979491, + "mrr_at_100": 0.5802929567946048, + "mrr_at_1000": 0.580593260804546, + "mrr_at_20": 0.5769766502585726, + "mrr_at_3": 0.5527192008879022, + "mrr_at_5": 0.5650943396226412, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6049915033432466, + "nauc_cv_recall_at_100_max": 0.708759882091787, + "nauc_cv_recall_at_100_std": 0.6297270823652008, + "nauc_cv_recall_at_10_diff1": 0.6642776906089205, + "nauc_cv_recall_at_10_max": 0.447059980454553, + "nauc_cv_recall_at_10_std": 0.4052465772306079, + "nauc_cv_recall_at_1_diff1": 0.7754459218216824, + "nauc_cv_recall_at_1_max": 0.44289750168720743, + "nauc_cv_recall_at_1_std": 0.16301588689160468, + "nauc_cv_recall_at_20_diff1": 0.620224538962043, + "nauc_cv_recall_at_20_max": 0.5276603799108296, + "nauc_cv_recall_at_20_std": 0.48463918745627865, + "nauc_cv_recall_at_3_diff1": 0.691775032709587, + "nauc_cv_recall_at_3_max": 0.43592366553042383, + "nauc_cv_recall_at_3_std": 0.26442498020166966, + "nauc_cv_recall_at_5_diff1": 0.6720042848528613, + "nauc_cv_recall_at_5_max": 0.4150629324157825, + "nauc_cv_recall_at_5_std": 0.3241888821441566, + "nauc_map_at_1000_diff1": 0.7371547683757999, + "nauc_map_at_1000_max": 0.4413120271242708, + "nauc_map_at_1000_std": 0.21533414606637508, + "nauc_map_at_100_diff1": 0.7369944021436499, + "nauc_map_at_100_max": 0.4414884543850126, + "nauc_map_at_100_std": 0.21562166277637476, + "nauc_map_at_10_diff1": 0.7368532064652646, + "nauc_map_at_10_max": 0.4381081429166812, + "nauc_map_at_10_std": 0.2155827439033546, + "nauc_map_at_1_diff1": 0.7754459218216824, + "nauc_map_at_1_max": 0.44289750168720743, + "nauc_map_at_1_std": 0.16301588689160468, + "nauc_map_at_20_diff1": 0.7358744047217303, + "nauc_map_at_20_max": 0.44124782849192085, + "nauc_map_at_20_std": 0.2168435407857295, + "nauc_map_at_3_diff1": 0.7397448692197428, + "nauc_map_at_3_max": 0.43880422998312507, + "nauc_map_at_3_std": 0.20032815912248597, + "nauc_map_at_5_diff1": 0.7365303036094254, + "nauc_map_at_5_max": 0.43466506293500423, + "nauc_map_at_5_std": 0.21054343286527433, + "nauc_mrr_at_1000_diff1": 0.7371547497308951, + "nauc_mrr_at_1000_max": 0.4413102095955693, + "nauc_mrr_at_1000_std": 0.2153316915574059, + "nauc_mrr_at_100_diff1": 0.7369943730503353, + "nauc_mrr_at_100_max": 0.44148662838789476, + "nauc_mrr_at_100_std": 0.21561921494775707, + "nauc_mrr_at_10_diff1": 0.7368532064652646, + "nauc_mrr_at_10_max": 0.4381081429166812, + "nauc_mrr_at_10_std": 0.2155827439033546, + "nauc_mrr_at_1_diff1": 0.7754459218216824, + "nauc_mrr_at_1_max": 0.44289750168720743, + "nauc_mrr_at_1_std": 0.16301588689160468, + "nauc_mrr_at_20_diff1": 0.7358744047217303, + "nauc_mrr_at_20_max": 0.44124782849192085, + "nauc_mrr_at_20_std": 0.2168435407857295, + "nauc_mrr_at_3_diff1": 0.7397448692197428, + "nauc_mrr_at_3_max": 0.43880422998312507, + "nauc_mrr_at_3_std": 0.20032815912248597, + "nauc_mrr_at_5_diff1": 0.7365303036094254, + "nauc_mrr_at_5_max": 0.43466506293500423, + "nauc_mrr_at_5_std": 0.21054343286527433, + "nauc_ndcg_at_1000_diff1": 0.7294461405142091, + "nauc_ndcg_at_1000_max": 0.44643934374513766, + "nauc_ndcg_at_1000_std": 0.23427381610688863, + "nauc_ndcg_at_100_diff1": 0.7255991832726855, + "nauc_ndcg_at_100_max": 0.4536474578380291, + "nauc_ndcg_at_100_std": 0.2452493736090266, + "nauc_ndcg_at_10_diff1": 0.7224014716492231, + "nauc_ndcg_at_10_max": 0.4391711764126902, + "nauc_ndcg_at_10_std": 0.2505369918544025, + "nauc_ndcg_at_1_diff1": 0.7754459218216824, + "nauc_ndcg_at_1_max": 0.44289750168720743, + "nauc_ndcg_at_1_std": 0.16301588689160468, + "nauc_ndcg_at_20_diff1": 0.7177104753698911, + "nauc_ndcg_at_20_max": 0.4518920628491788, + "nauc_ndcg_at_20_std": 0.25709137115309527, + "nauc_ndcg_at_3_diff1": 0.7282250550431725, + "nauc_ndcg_at_3_max": 0.43806163687845706, + "nauc_ndcg_at_3_std": 0.21544985963804733, + "nauc_ndcg_at_5_diff1": 0.722169746497287, + "nauc_ndcg_at_5_max": 0.43047402062795503, + "nauc_ndcg_at_5_std": 0.23506811710924785, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.6049915033432456, + "nauc_precision_at_100_max": 0.7087598820917866, + "nauc_precision_at_100_std": 0.6297270823652044, + "nauc_precision_at_10_diff1": 0.6642776906089205, + "nauc_precision_at_10_max": 0.4470599804545534, + "nauc_precision_at_10_std": 0.405246577230608, + "nauc_precision_at_1_diff1": 0.7754459218216824, + "nauc_precision_at_1_max": 0.44289750168720743, + "nauc_precision_at_1_std": 0.16301588689160468, + "nauc_precision_at_20_diff1": 0.6202245389620432, + "nauc_precision_at_20_max": 0.527660379910829, + "nauc_precision_at_20_std": 0.48463918745627876, + "nauc_precision_at_3_diff1": 0.6917750327095873, + "nauc_precision_at_3_max": 0.4359236655304239, + "nauc_precision_at_3_std": 0.26442498020167043, + "nauc_precision_at_5_diff1": 0.6720042848528609, + "nauc_precision_at_5_max": 0.41506293241578196, + "nauc_precision_at_5_std": 0.3241888821441566, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.6049915033432466, + "nauc_recall_at_100_max": 0.708759882091787, + "nauc_recall_at_100_std": 0.6297270823652008, + "nauc_recall_at_10_diff1": 0.6642776906089205, + "nauc_recall_at_10_max": 0.447059980454553, + "nauc_recall_at_10_std": 0.4052465772306079, + "nauc_recall_at_1_diff1": 0.7754459218216824, + "nauc_recall_at_1_max": 0.44289750168720743, + "nauc_recall_at_1_std": 0.16301588689160468, + "nauc_recall_at_20_diff1": 0.620224538962043, + "nauc_recall_at_20_max": 0.5276603799108296, + "nauc_recall_at_20_std": 0.48463918745627865, + "nauc_recall_at_3_diff1": 0.691775032709587, + "nauc_recall_at_3_max": 0.43592366553042383, + "nauc_recall_at_3_std": 0.26442498020166966, + "nauc_recall_at_5_diff1": 0.6720042848528613, + "nauc_recall_at_5_max": 0.4150629324157825, + "nauc_recall_at_5_std": 0.3241888821441566, + "ndcg_at_1": 0.48835, + "ndcg_at_10": 0.61394, + "ndcg_at_100": 0.6543, + "ndcg_at_1000": 0.66198, + "ndcg_at_20": 0.62963, + "ndcg_at_3": 0.57328, + "ndcg_at_5": 0.59539, + "precision_at_1": 0.48835, + "precision_at_10": 0.07436, + "precision_at_100": 0.0094, + "precision_at_1000": 0.001, + "precision_at_20": 0.04029, + "precision_at_3": 0.21088, + "precision_at_5": 0.13718, + "recall_at_1": 0.48835, + "recall_at_10": 0.74362, + "recall_at_100": 0.94007, + "recall_at_1000": 1.0, + "recall_at_20": 0.80577, + "recall_at_3": 0.63263, + "recall_at_5": 0.6859 + }, + { + "cv_recall_at_1": 0.13963, + "cv_recall_at_10": 0.30827, + "cv_recall_at_100": 0.63802, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.39098, + "cv_recall_at_3": 0.19334, + "cv_recall_at_5": 0.23523, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.21135, + "map_at_1": 0.13963, + "map_at_10": 0.18189, + "map_at_100": 0.19343, + "map_at_1000": 0.19487, + "map_at_20": 0.18754, + "map_at_3": 0.16255, + "map_at_5": 0.17195, + "mrr_at_1": 0.13963480128893663, + "mrr_at_10": 0.1818909518694696, + "mrr_at_100": 0.1934289349559609, + "mrr_at_1000": 0.19487283351496834, + "mrr_at_20": 0.18754334327729894, + "mrr_at_3": 0.16254923021840312, + "mrr_at_5": 0.17194772645900466, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2534089559430589, + "nauc_cv_recall_at_100_max": 0.32270816808890607, + "nauc_cv_recall_at_100_std": 0.4730302314141176, + "nauc_cv_recall_at_10_diff1": 0.40325013114727315, + "nauc_cv_recall_at_10_max": 0.2689458126198791, + "nauc_cv_recall_at_10_std": 0.3160539404459094, + "nauc_cv_recall_at_1_diff1": 0.6317084909028625, + "nauc_cv_recall_at_1_max": 0.35727465463798985, + "nauc_cv_recall_at_1_std": 0.3002148677470887, + "nauc_cv_recall_at_20_diff1": 0.36362317744747646, + "nauc_cv_recall_at_20_max": 0.26919980206684513, + "nauc_cv_recall_at_20_std": 0.3217120395483907, + "nauc_cv_recall_at_3_diff1": 0.5051748031641702, + "nauc_cv_recall_at_3_max": 0.3418598559556601, + "nauc_cv_recall_at_3_std": 0.3128990941534339, + "nauc_cv_recall_at_5_diff1": 0.45631176762085995, + "nauc_cv_recall_at_5_max": 0.28861197328428334, + "nauc_cv_recall_at_5_std": 0.29915893751084915, + "nauc_map_at_1000_diff1": 0.5391322227919676, + "nauc_map_at_1000_max": 0.3367418055203415, + "nauc_map_at_1000_std": 0.31472976459112945, + "nauc_map_at_100_diff1": 0.5383827313100662, + "nauc_map_at_100_max": 0.33638933710186986, + "nauc_map_at_100_std": 0.3150002028628421, + "nauc_map_at_10_diff1": 0.5448935246611308, + "nauc_map_at_10_max": 0.33535253238633284, + "nauc_map_at_10_std": 0.31061338437486985, + "nauc_map_at_1_diff1": 0.6317084909028625, + "nauc_map_at_1_max": 0.35727465463798985, + "nauc_map_at_1_std": 0.3002148677470887, + "nauc_map_at_20_diff1": 0.5405184997278059, + "nauc_map_at_20_max": 0.3354746238530943, + "nauc_map_at_20_std": 0.31168100786202535, + "nauc_map_at_3_diff1": 0.5723147050217703, + "nauc_map_at_3_max": 0.35630187388769957, + "nauc_map_at_3_std": 0.31013478532479555, + "nauc_map_at_5_diff1": 0.5564096737028025, + "nauc_map_at_5_max": 0.34086704005792656, + "nauc_map_at_5_std": 0.30680693510326507, + "nauc_mrr_at_1000_diff1": 0.5391323019533246, + "nauc_mrr_at_1000_max": 0.33674149853590685, + "nauc_mrr_at_1000_std": 0.3147295182343123, + "nauc_mrr_at_100_diff1": 0.5383828099646272, + "nauc_mrr_at_100_max": 0.33638903127372455, + "nauc_mrr_at_100_std": 0.3149999575713113, + "nauc_mrr_at_10_diff1": 0.5448935246611308, + "nauc_mrr_at_10_max": 0.33535253238633284, + "nauc_mrr_at_10_std": 0.31061338437486985, + "nauc_mrr_at_1_diff1": 0.6317084909028625, + "nauc_mrr_at_1_max": 0.35727465463798985, + "nauc_mrr_at_1_std": 0.3002148677470887, + "nauc_mrr_at_20_diff1": 0.5405184997278059, + "nauc_mrr_at_20_max": 0.3354746238530943, + "nauc_mrr_at_20_std": 0.31168100786202535, + "nauc_mrr_at_3_diff1": 0.5723147050217703, + "nauc_mrr_at_3_max": 0.35630187388769957, + "nauc_mrr_at_3_std": 0.31013478532479555, + "nauc_mrr_at_5_diff1": 0.5564096737028025, + "nauc_mrr_at_5_max": 0.34086704005792656, + "nauc_mrr_at_5_std": 0.30680693510326507, + "nauc_ndcg_at_1000_diff1": 0.503172222064133, + "nauc_ndcg_at_1000_max": 0.32939523795204545, + "nauc_ndcg_at_1000_std": 0.32587298565830985, + "nauc_ndcg_at_100_diff1": 0.4727440949155755, + "nauc_ndcg_at_100_max": 0.3269834083639128, + "nauc_ndcg_at_100_std": 0.343141914249046, + "nauc_ndcg_at_10_diff1": 0.5031512824454686, + "nauc_ndcg_at_10_max": 0.3160242337211601, + "nauc_ndcg_at_10_std": 0.31234805584479797, + "nauc_ndcg_at_1_diff1": 0.6317084909028625, + "nauc_ndcg_at_1_max": 0.35727465463798985, + "nauc_ndcg_at_1_std": 0.3002148677470887, + "nauc_ndcg_at_20_diff1": 0.48908232206391783, + "nauc_ndcg_at_20_max": 0.31591981917950757, + "nauc_ndcg_at_20_std": 0.31480058661935906, + "nauc_ndcg_at_3_diff1": 0.5534585464528963, + "nauc_ndcg_at_3_max": 0.3524868799024196, + "nauc_ndcg_at_3_std": 0.31106377515333117, + "nauc_ndcg_at_5_diff1": 0.5276047716802393, + "nauc_ndcg_at_5_max": 0.32645814171648835, + "nauc_ndcg_at_5_std": 0.3049512756948914, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.2534089559430588, + "nauc_precision_at_100_max": 0.32270816808890573, + "nauc_precision_at_100_std": 0.473030231414118, + "nauc_precision_at_10_diff1": 0.40325013114727315, + "nauc_precision_at_10_max": 0.2689458126198787, + "nauc_precision_at_10_std": 0.3160539404459094, + "nauc_precision_at_1_diff1": 0.6317084909028625, + "nauc_precision_at_1_max": 0.35727465463798985, + "nauc_precision_at_1_std": 0.3002148677470887, + "nauc_precision_at_20_diff1": 0.36362317744747635, + "nauc_precision_at_20_max": 0.26919980206684496, + "nauc_precision_at_20_std": 0.32171203954839084, + "nauc_precision_at_3_diff1": 0.5051748031641701, + "nauc_precision_at_3_max": 0.34185985595565993, + "nauc_precision_at_3_std": 0.31289909415343403, + "nauc_precision_at_5_diff1": 0.4563117676208598, + "nauc_precision_at_5_max": 0.28861197328428334, + "nauc_precision_at_5_std": 0.29915893751084904, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2534089559430589, + "nauc_recall_at_100_max": 0.32270816808890607, + "nauc_recall_at_100_std": 0.4730302314141176, + "nauc_recall_at_10_diff1": 0.40325013114727315, + "nauc_recall_at_10_max": 0.2689458126198791, + "nauc_recall_at_10_std": 0.3160539404459094, + "nauc_recall_at_1_diff1": 0.6317084909028625, + "nauc_recall_at_1_max": 0.35727465463798985, + "nauc_recall_at_1_std": 0.3002148677470887, + "nauc_recall_at_20_diff1": 0.36362317744747646, + "nauc_recall_at_20_max": 0.26919980206684513, + "nauc_recall_at_20_std": 0.3217120395483907, + "nauc_recall_at_3_diff1": 0.5051748031641702, + "nauc_recall_at_3_max": 0.3418598559556601, + "nauc_recall_at_3_std": 0.3128990941534339, + "nauc_recall_at_5_diff1": 0.45631176762085995, + "nauc_recall_at_5_max": 0.28861197328428334, + "nauc_recall_at_5_std": 0.29915893751084915, + "ndcg_at_1": 0.13963, + "ndcg_at_10": 0.21135, + "ndcg_at_100": 0.27706, + "ndcg_at_1000": 0.32153, + "ndcg_at_20": 0.23215, + "ndcg_at_3": 0.17043, + "ndcg_at_5": 0.18753, + "precision_at_1": 0.13963, + "precision_at_10": 0.03083, + "precision_at_100": 0.00638, + "precision_at_1000": 0.001, + "precision_at_20": 0.01955, + "precision_at_3": 0.06445, + "precision_at_5": 0.04705, + "recall_at_1": 0.13963, + "recall_at_10": 0.30827, + "recall_at_100": 0.63802, + "recall_at_1000": 1.0, + "recall_at_20": 0.39098, + "recall_at_3": 0.19334, + "recall_at_5": 0.23523 + }, + { + "cv_recall_at_1": 0.14, + "cv_recall_at_10": 0.357, + "cv_recall_at_100": 0.658, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.425, + "cv_recall_at_3": 0.219, + "cv_recall_at_5": 0.285, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.23555, + "map_at_1": 0.14, + "map_at_10": 0.19815, + "map_at_100": 0.20838, + "map_at_1000": 0.20992, + "map_at_20": 0.20287, + "map_at_3": 0.17333, + "map_at_5": 0.18873, + "mrr_at_1": 0.14, + "mrr_at_10": 0.19815277777777784, + "mrr_at_100": 0.20837673242317895, + "mrr_at_1000": 0.20991525905046582, + "mrr_at_20": 0.20286907916215202, + "mrr_at_3": 0.17333333333333342, + "mrr_at_5": 0.1887333333333334, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.31186007531151105, + "nauc_cv_recall_at_100_max": 0.4811622040684086, + "nauc_cv_recall_at_100_std": 0.42809040026183837, + "nauc_cv_recall_at_10_diff1": 0.413408863642038, + "nauc_cv_recall_at_10_max": 0.27114828933311785, + "nauc_cv_recall_at_10_std": 0.12190628617682368, + "nauc_cv_recall_at_1_diff1": 0.5133140990283847, + "nauc_cv_recall_at_1_max": 0.14333658619372894, + "nauc_cv_recall_at_1_std": -0.0564724850439138, + "nauc_cv_recall_at_20_diff1": 0.34592748872082824, + "nauc_cv_recall_at_20_max": 0.27579084033886203, + "nauc_cv_recall_at_20_std": 0.16246291668019083, + "nauc_cv_recall_at_3_diff1": 0.4245150101271459, + "nauc_cv_recall_at_3_max": 0.20982885824019645, + "nauc_cv_recall_at_3_std": 0.048116786775706716, + "nauc_cv_recall_at_5_diff1": 0.42818025055575165, + "nauc_cv_recall_at_5_max": 0.22702373285133542, + "nauc_cv_recall_at_5_std": 0.06251428333991231, + "nauc_map_at_1000_diff1": 0.45644452104799366, + "nauc_map_at_1000_max": 0.18966182165595427, + "nauc_map_at_1000_std": 0.015142223758896131, + "nauc_map_at_100_diff1": 0.45603098814474735, + "nauc_map_at_100_max": 0.19037424673457165, + "nauc_map_at_100_std": 0.01627979300863379, + "nauc_map_at_10_diff1": 0.4631346811594673, + "nauc_map_at_10_max": 0.18414084623389668, + "nauc_map_at_10_std": 0.006601657669226161, + "nauc_map_at_1_diff1": 0.5133140990283847, + "nauc_map_at_1_max": 0.14333658619372894, + "nauc_map_at_1_std": -0.0564724850439138, + "nauc_map_at_20_diff1": 0.45687943682993415, + "nauc_map_at_20_max": 0.18467676506949465, + "nauc_map_at_20_std": 0.010065050228116441, + "nauc_map_at_3_diff1": 0.46608467224631916, + "nauc_map_at_3_max": 0.16995784715010084, + "nauc_map_at_3_std": -0.008846294825584775, + "nauc_map_at_5_diff1": 0.4663183772981682, + "nauc_map_at_5_max": 0.1762843693554193, + "nauc_map_at_5_std": -0.0032305300628583337, + "nauc_mrr_at_1000_diff1": 0.45644452104799366, + "nauc_mrr_at_1000_max": 0.18966182165595427, + "nauc_mrr_at_1000_std": 0.015142223758896131, + "nauc_mrr_at_100_diff1": 0.45603098814474735, + "nauc_mrr_at_100_max": 0.19037424673457165, + "nauc_mrr_at_100_std": 0.01627979300863379, + "nauc_mrr_at_10_diff1": 0.4631346811594673, + "nauc_mrr_at_10_max": 0.18414084623389668, + "nauc_mrr_at_10_std": 0.006601657669226161, + "nauc_mrr_at_1_diff1": 0.5133140990283847, + "nauc_mrr_at_1_max": 0.14333658619372894, + "nauc_mrr_at_1_std": -0.0564724850439138, + "nauc_mrr_at_20_diff1": 0.45687943682993415, + "nauc_mrr_at_20_max": 0.18467676506949465, + "nauc_mrr_at_20_std": 0.010065050228116441, + "nauc_mrr_at_3_diff1": 0.46608467224631916, + "nauc_mrr_at_3_max": 0.16995784715010084, + "nauc_mrr_at_3_std": -0.008846294825584775, + "nauc_mrr_at_5_diff1": 0.4663183772981682, + "nauc_mrr_at_5_max": 0.1762843693554193, + "nauc_mrr_at_5_std": -0.0032305300628583337, + "nauc_ndcg_at_1000_diff1": 0.4380427564007995, + "nauc_ndcg_at_1000_max": 0.21748512763611927, + "nauc_ndcg_at_1000_std": 0.053149538455247586, + "nauc_ndcg_at_100_diff1": 0.4231968489839242, + "nauc_ndcg_at_100_max": 0.24795527816693558, + "nauc_ndcg_at_100_std": 0.09710467925441554, + "nauc_ndcg_at_10_diff1": 0.4480123666430948, + "nauc_ndcg_at_10_max": 0.20910479565113282, + "nauc_ndcg_at_10_std": 0.039700166847957956, + "nauc_ndcg_at_1_diff1": 0.5133140990283847, + "nauc_ndcg_at_1_max": 0.14333658619372894, + "nauc_ndcg_at_1_std": -0.0564724850439138, + "nauc_ndcg_at_20_diff1": 0.4268055180867815, + "nauc_ndcg_at_20_max": 0.21078731991897864, + "nauc_ndcg_at_20_std": 0.05221280409745525, + "nauc_ndcg_at_3_diff1": 0.4540115532109377, + "nauc_ndcg_at_3_max": 0.18117151911696, + "nauc_ndcg_at_3_std": 0.007362526259160504, + "nauc_ndcg_at_5_diff1": 0.4546328571737963, + "nauc_ndcg_at_5_max": 0.19120405311322689, + "nauc_ndcg_at_5_std": 0.016307727896751327, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.31186007531151105, + "nauc_precision_at_100_max": 0.4811622040684084, + "nauc_precision_at_100_std": 0.428090400261838, + "nauc_precision_at_10_diff1": 0.41340886364203755, + "nauc_precision_at_10_max": 0.27114828933311763, + "nauc_precision_at_10_std": 0.12190628617682332, + "nauc_precision_at_1_diff1": 0.5133140990283847, + "nauc_precision_at_1_max": 0.14333658619372894, + "nauc_precision_at_1_std": -0.0564724850439138, + "nauc_precision_at_20_diff1": 0.34592748872082796, + "nauc_precision_at_20_max": 0.2757908403388615, + "nauc_precision_at_20_std": 0.16246291668019033, + "nauc_precision_at_3_diff1": 0.4245150101271458, + "nauc_precision_at_3_max": 0.2098288582401962, + "nauc_precision_at_3_std": 0.04811678677570672, + "nauc_precision_at_5_diff1": 0.4281802505557515, + "nauc_precision_at_5_max": 0.22702373285133526, + "nauc_precision_at_5_std": 0.06251428333991228, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.31186007531151105, + "nauc_recall_at_100_max": 0.4811622040684086, + "nauc_recall_at_100_std": 0.42809040026183837, + "nauc_recall_at_10_diff1": 0.413408863642038, + "nauc_recall_at_10_max": 0.27114828933311785, + "nauc_recall_at_10_std": 0.12190628617682368, + "nauc_recall_at_1_diff1": 0.5133140990283847, + "nauc_recall_at_1_max": 0.14333658619372894, + "nauc_recall_at_1_std": -0.0564724850439138, + "nauc_recall_at_20_diff1": 0.34592748872082824, + "nauc_recall_at_20_max": 0.27579084033886203, + "nauc_recall_at_20_std": 0.16246291668019083, + "nauc_recall_at_3_diff1": 0.4245150101271459, + "nauc_recall_at_3_max": 0.20982885824019645, + "nauc_recall_at_3_std": 0.048116786775706716, + "nauc_recall_at_5_diff1": 0.42818025055575165, + "nauc_recall_at_5_max": 0.22702373285133542, + "nauc_recall_at_5_std": 0.06251428333991231, + "ndcg_at_1": 0.14, + "ndcg_at_10": 0.23555, + "ndcg_at_100": 0.29512, + "ndcg_at_1000": 0.33821, + "ndcg_at_20": 0.25274, + "ndcg_at_3": 0.185, + "ndcg_at_5": 0.21246, + "precision_at_1": 0.14, + "precision_at_10": 0.0357, + "precision_at_100": 0.00658, + "precision_at_1000": 0.001, + "precision_at_20": 0.02125, + "precision_at_3": 0.073, + "precision_at_5": 0.057, + "recall_at_1": 0.14, + "recall_at_10": 0.357, + "recall_at_100": 0.658, + "recall_at_1000": 1.0, + "recall_at_20": 0.425, + "recall_at_3": 0.219, + "recall_at_5": 0.285 + }, + { + "cv_recall_at_1": 0.39667, + "cv_recall_at_10": 0.65049, + "cv_recall_at_100": 0.89043, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.73786, + "cv_recall_at_3": 0.54508, + "cv_recall_at_5": 0.59223, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.52106, + "map_at_1": 0.39667, + "map_at_10": 0.47983, + "map_at_100": 0.48961, + "map_at_1000": 0.49022, + "map_at_20": 0.4859, + "map_at_3": 0.46117, + "map_at_5": 0.47178, + "mrr_at_1": 0.39667128987517336, + "mrr_at_10": 0.4798274002157494, + "mrr_at_100": 0.4896124803939883, + "mrr_at_1000": 0.4902196031014366, + "mrr_at_20": 0.4859049701628912, + "mrr_at_3": 0.4611650485436892, + "mrr_at_5": 0.471775312066574, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5529447924526923, + "nauc_cv_recall_at_100_max": 0.48063873317547207, + "nauc_cv_recall_at_100_std": 0.5084000789063866, + "nauc_cv_recall_at_10_diff1": 0.5501105521317363, + "nauc_cv_recall_at_10_max": 0.4901830644185336, + "nauc_cv_recall_at_10_std": 0.3619536097711491, + "nauc_cv_recall_at_1_diff1": 0.7338512053750866, + "nauc_cv_recall_at_1_max": 0.46598193211631234, + "nauc_cv_recall_at_1_std": 0.18958275403666142, + "nauc_cv_recall_at_20_diff1": 0.5235888915069656, + "nauc_cv_recall_at_20_max": 0.4837916615271111, + "nauc_cv_recall_at_20_std": 0.4264989192652688, + "nauc_cv_recall_at_3_diff1": 0.5941351764246412, + "nauc_cv_recall_at_3_max": 0.47104844256320394, + "nauc_cv_recall_at_3_std": 0.30558236581140236, + "nauc_cv_recall_at_5_diff1": 0.58452247493996, + "nauc_cv_recall_at_5_max": 0.48724150043188547, + "nauc_cv_recall_at_5_std": 0.334328948805313, + "nauc_map_at_1000_diff1": 0.6720195306324316, + "nauc_map_at_1000_max": 0.4740969612188918, + "nauc_map_at_1000_std": 0.24973019804344002, + "nauc_map_at_100_diff1": 0.67180082466236, + "nauc_map_at_100_max": 0.47400960560082256, + "nauc_map_at_100_std": 0.25009040568775737, + "nauc_map_at_10_diff1": 0.6701868031843548, + "nauc_map_at_10_max": 0.47473751185879304, + "nauc_map_at_10_std": 0.2489141754170303, + "nauc_map_at_1_diff1": 0.7338512053750866, + "nauc_map_at_1_max": 0.46598193211631234, + "nauc_map_at_1_std": 0.18958275403666142, + "nauc_map_at_20_diff1": 0.66998861295754, + "nauc_map_at_20_max": 0.474387524103794, + "nauc_map_at_20_std": 0.2510785033312345, + "nauc_map_at_3_diff1": 0.6735418032015751, + "nauc_map_at_3_max": 0.470600944971827, + "nauc_map_at_3_std": 0.2418534158719009, + "nauc_map_at_5_diff1": 0.6725778054575782, + "nauc_map_at_5_max": 0.47383144665195165, + "nauc_map_at_5_std": 0.24658927848582451, + "nauc_mrr_at_1000_diff1": 0.672020119769249, + "nauc_mrr_at_1000_max": 0.4740968793306214, + "nauc_mrr_at_1000_std": 0.24973009739879667, + "nauc_mrr_at_100_diff1": 0.6718014053275473, + "nauc_mrr_at_100_max": 0.47400950902542766, + "nauc_mrr_at_100_std": 0.25009027713865783, + "nauc_mrr_at_10_diff1": 0.6701868031843548, + "nauc_mrr_at_10_max": 0.47473751185879304, + "nauc_mrr_at_10_std": 0.2489141754170303, + "nauc_mrr_at_1_diff1": 0.7338512053750866, + "nauc_mrr_at_1_max": 0.46598193211631234, + "nauc_mrr_at_1_std": 0.18958275403666142, + "nauc_mrr_at_20_diff1": 0.66998861295754, + "nauc_mrr_at_20_max": 0.474387524103794, + "nauc_mrr_at_20_std": 0.2510785033312345, + "nauc_mrr_at_3_diff1": 0.6735418032015751, + "nauc_mrr_at_3_max": 0.470600944971827, + "nauc_mrr_at_3_std": 0.2418534158719009, + "nauc_mrr_at_5_diff1": 0.6725778054575782, + "nauc_mrr_at_5_max": 0.47383144665195165, + "nauc_mrr_at_5_std": 0.24658927848582451, + "nauc_ndcg_at_1000_diff1": 0.6601641574024943, + "nauc_ndcg_at_1000_max": 0.47563774314501484, + "nauc_ndcg_at_1000_std": 0.2649262811876548, + "nauc_ndcg_at_100_diff1": 0.6549206364926009, + "nauc_ndcg_at_100_max": 0.4752963483292056, + "nauc_ndcg_at_100_std": 0.27618420424960716, + "nauc_ndcg_at_10_diff1": 0.6442607607604707, + "nauc_ndcg_at_10_max": 0.4782010194867499, + "nauc_ndcg_at_10_std": 0.273393356452818, + "nauc_ndcg_at_1_diff1": 0.7338512053750866, + "nauc_ndcg_at_1_max": 0.46598193211631234, + "nauc_ndcg_at_1_std": 0.18958275403666142, + "nauc_ndcg_at_20_diff1": 0.6430884721245828, + "nauc_ndcg_at_20_max": 0.47663486711469094, + "nauc_ndcg_at_20_std": 0.28195874052103237, + "nauc_ndcg_at_3_diff1": 0.6538264079602413, + "nauc_ndcg_at_3_max": 0.47081803307646053, + "nauc_ndcg_at_3_std": 0.25777210528271977, + "nauc_ndcg_at_5_diff1": 0.6514996139228869, + "nauc_ndcg_at_5_max": 0.4769085934141065, + "nauc_ndcg_at_5_std": 0.2673431188114841, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5529447924526933, + "nauc_precision_at_100_max": 0.48063873317547595, + "nauc_precision_at_100_std": 0.5084000789063885, + "nauc_precision_at_10_diff1": 0.5501105521317365, + "nauc_precision_at_10_max": 0.4901830644185332, + "nauc_precision_at_10_std": 0.36195360977114904, + "nauc_precision_at_1_diff1": 0.7338512053750866, + "nauc_precision_at_1_max": 0.46598193211631234, + "nauc_precision_at_1_std": 0.18958275403666142, + "nauc_precision_at_20_diff1": 0.5235888915069651, + "nauc_precision_at_20_max": 0.48379166152711106, + "nauc_precision_at_20_std": 0.4264989192652693, + "nauc_precision_at_3_diff1": 0.5941351764246411, + "nauc_precision_at_3_max": 0.47104844256320383, + "nauc_precision_at_3_std": 0.30558236581140213, + "nauc_precision_at_5_diff1": 0.5845224749399608, + "nauc_precision_at_5_max": 0.4872415004318854, + "nauc_precision_at_5_std": 0.33432894880531333, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5529447924526923, + "nauc_recall_at_100_max": 0.48063873317547207, + "nauc_recall_at_100_std": 0.5084000789063866, + "nauc_recall_at_10_diff1": 0.5501105521317363, + "nauc_recall_at_10_max": 0.4901830644185336, + "nauc_recall_at_10_std": 0.3619536097711491, + "nauc_recall_at_1_diff1": 0.7338512053750866, + "nauc_recall_at_1_max": 0.46598193211631234, + "nauc_recall_at_1_std": 0.18958275403666142, + "nauc_recall_at_20_diff1": 0.5235888915069656, + "nauc_recall_at_20_max": 0.4837916615271111, + "nauc_recall_at_20_std": 0.4264989192652688, + "nauc_recall_at_3_diff1": 0.5941351764246412, + "nauc_recall_at_3_max": 0.47104844256320394, + "nauc_recall_at_3_std": 0.30558236581140236, + "nauc_recall_at_5_diff1": 0.58452247493996, + "nauc_recall_at_5_max": 0.48724150043188547, + "nauc_recall_at_5_std": 0.334328948805313, + "ndcg_at_1": 0.39667, + "ndcg_at_10": 0.52106, + "ndcg_at_100": 0.57116, + "ndcg_at_1000": 0.58558, + "ndcg_at_20": 0.54317, + "ndcg_at_3": 0.48268, + "ndcg_at_5": 0.50195, + "precision_at_1": 0.39667, + "precision_at_10": 0.06505, + "precision_at_100": 0.0089, + "precision_at_1000": 0.001, + "precision_at_20": 0.03689, + "precision_at_3": 0.18169, + "precision_at_5": 0.11845, + "recall_at_1": 0.39667, + "recall_at_10": 0.65049, + "recall_at_100": 0.89043, + "recall_at_1000": 1.0, + "recall_at_20": 0.73786, + "recall_at_3": 0.54508, + "recall_at_5": 0.59223 + }, + { + "cv_recall_at_1": 0.43552, + "cv_recall_at_10": 0.6797, + "cv_recall_at_100": 0.87844, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.75793, + "cv_recall_at_3": 0.56025, + "cv_recall_at_5": 0.61628, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.55199, + "map_at_1": 0.43552, + "map_at_10": 0.51178, + "map_at_100": 0.52036, + "map_at_1000": 0.52095, + "map_at_20": 0.51714, + "map_at_3": 0.49084, + "map_at_5": 0.50331, + "mrr_at_1": 0.4355179704016913, + "mrr_at_10": 0.5117814356186449, + "mrr_at_100": 0.5203611719791804, + "mrr_at_1000": 0.5209528247341728, + "mrr_at_20": 0.517142191579205, + "mrr_at_3": 0.4908386187455953, + "mrr_at_5": 0.5033121916842842, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5212447375851132, + "nauc_cv_recall_at_100_max": 0.5043702843064178, + "nauc_cv_recall_at_100_std": 0.5877426834505205, + "nauc_cv_recall_at_10_diff1": 0.6033223572296531, + "nauc_cv_recall_at_10_max": 0.5299932139371255, + "nauc_cv_recall_at_10_std": 0.4435903956565738, + "nauc_cv_recall_at_1_diff1": 0.7464799971953152, + "nauc_cv_recall_at_1_max": 0.5381067495626654, + "nauc_cv_recall_at_1_std": 0.30716607647082733, + "nauc_cv_recall_at_20_diff1": 0.5491916947920159, + "nauc_cv_recall_at_20_max": 0.5114356154308455, + "nauc_cv_recall_at_20_std": 0.4812767256961097, + "nauc_cv_recall_at_3_diff1": 0.663126629290186, + "nauc_cv_recall_at_3_max": 0.5283498125854807, + "nauc_cv_recall_at_3_std": 0.346016951683277, + "nauc_cv_recall_at_5_diff1": 0.626626862939999, + "nauc_cv_recall_at_5_max": 0.5288735395629617, + "nauc_cv_recall_at_5_std": 0.40697999107692173, + "nauc_map_at_1000_diff1": 0.7040752895960932, + "nauc_map_at_1000_max": 0.5360887555262941, + "nauc_map_at_1000_std": 0.34052654689342726, + "nauc_map_at_100_diff1": 0.7037493671725752, + "nauc_map_at_100_max": 0.536044976656049, + "nauc_map_at_100_std": 0.34083807138122624, + "nauc_map_at_10_diff1": 0.7040569754687911, + "nauc_map_at_10_max": 0.5360673529713214, + "nauc_map_at_10_std": 0.33960926752548026, + "nauc_map_at_1_diff1": 0.7464799971953152, + "nauc_map_at_1_max": 0.5381067495626654, + "nauc_map_at_1_std": 0.30716607647082733, + "nauc_map_at_20_diff1": 0.7027070230721864, + "nauc_map_at_20_max": 0.5357565736391446, + "nauc_map_at_20_std": 0.34054086247508, + "nauc_map_at_3_diff1": 0.7122095603541071, + "nauc_map_at_3_max": 0.5354345456540576, + "nauc_map_at_3_std": 0.3246828517710239, + "nauc_map_at_5_diff1": 0.7054295586074079, + "nauc_map_at_5_max": 0.5356815100550746, + "nauc_map_at_5_std": 0.3363066932255336, + "nauc_mrr_at_1000_diff1": 0.7040754269691212, + "nauc_mrr_at_1000_max": 0.5360890407260704, + "nauc_mrr_at_1000_std": 0.3405269228763386, + "nauc_mrr_at_100_diff1": 0.7037495044605455, + "nauc_mrr_at_100_max": 0.5360452613859547, + "nauc_mrr_at_100_std": 0.3408384465735878, + "nauc_mrr_at_10_diff1": 0.7040569754687911, + "nauc_mrr_at_10_max": 0.5360673529713214, + "nauc_mrr_at_10_std": 0.33960926752548026, + "nauc_mrr_at_1_diff1": 0.7464799971953152, + "nauc_mrr_at_1_max": 0.5381067495626654, + "nauc_mrr_at_1_std": 0.30716607647082733, + "nauc_mrr_at_20_diff1": 0.7027070230721864, + "nauc_mrr_at_20_max": 0.5357565736391446, + "nauc_mrr_at_20_std": 0.34054086247508, + "nauc_mrr_at_3_diff1": 0.7122095603541071, + "nauc_mrr_at_3_max": 0.5354345456540576, + "nauc_mrr_at_3_std": 0.3246828517710239, + "nauc_mrr_at_5_diff1": 0.7054295586074079, + "nauc_mrr_at_5_max": 0.5356815100550746, + "nauc_mrr_at_5_std": 0.3363066932255336, + "nauc_ndcg_at_1000_diff1": 0.6922671257795675, + "nauc_ndcg_at_1000_max": 0.535122611480308, + "nauc_ndcg_at_1000_std": 0.35408371391461363, + "nauc_ndcg_at_100_diff1": 0.6831632876161986, + "nauc_ndcg_at_100_max": 0.5335032301662331, + "nauc_ndcg_at_100_std": 0.3657719912849508, + "nauc_ndcg_at_10_diff1": 0.6831155704338709, + "nauc_ndcg_at_10_max": 0.534797858763256, + "nauc_ndcg_at_10_std": 0.3605648365763527, + "nauc_ndcg_at_1_diff1": 0.7464799971953152, + "nauc_ndcg_at_1_max": 0.5381067495626654, + "nauc_ndcg_at_1_std": 0.30716607647082733, + "nauc_ndcg_at_20_diff1": 0.6769283008584868, + "nauc_ndcg_at_20_max": 0.5323259384219705, + "nauc_ndcg_at_20_std": 0.3641816897402438, + "nauc_ndcg_at_3_diff1": 0.7001088518732275, + "nauc_ndcg_at_3_max": 0.5337444781772217, + "nauc_ndcg_at_3_std": 0.3300076684439673, + "nauc_ndcg_at_5_diff1": 0.6872925776029278, + "nauc_ndcg_at_5_max": 0.5341253959019795, + "nauc_ndcg_at_5_std": 0.3521586226596281, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5212447375851109, + "nauc_precision_at_100_max": 0.504370284306418, + "nauc_precision_at_100_std": 0.5877426834505183, + "nauc_precision_at_10_diff1": 0.6033223572296522, + "nauc_precision_at_10_max": 0.5299932139371254, + "nauc_precision_at_10_std": 0.4435903956565733, + "nauc_precision_at_1_diff1": 0.7464799971953152, + "nauc_precision_at_1_max": 0.5381067495626654, + "nauc_precision_at_1_std": 0.30716607647082733, + "nauc_precision_at_20_diff1": 0.5491916947920147, + "nauc_precision_at_20_max": 0.5114356154308448, + "nauc_precision_at_20_std": 0.4812767256961104, + "nauc_precision_at_3_diff1": 0.6631266292901861, + "nauc_precision_at_3_max": 0.5283498125854805, + "nauc_precision_at_3_std": 0.34601695168327695, + "nauc_precision_at_5_diff1": 0.6266268629399989, + "nauc_precision_at_5_max": 0.5288735395629613, + "nauc_precision_at_5_std": 0.40697999107692145, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5212447375851132, + "nauc_recall_at_100_max": 0.5043702843064178, + "nauc_recall_at_100_std": 0.5877426834505205, + "nauc_recall_at_10_diff1": 0.6033223572296531, + "nauc_recall_at_10_max": 0.5299932139371255, + "nauc_recall_at_10_std": 0.4435903956565738, + "nauc_recall_at_1_diff1": 0.7464799971953152, + "nauc_recall_at_1_max": 0.5381067495626654, + "nauc_recall_at_1_std": 0.30716607647082733, + "nauc_recall_at_20_diff1": 0.5491916947920159, + "nauc_recall_at_20_max": 0.5114356154308455, + "nauc_recall_at_20_std": 0.4812767256961097, + "nauc_recall_at_3_diff1": 0.663126629290186, + "nauc_recall_at_3_max": 0.5283498125854807, + "nauc_recall_at_3_std": 0.346016951683277, + "nauc_recall_at_5_diff1": 0.626626862939999, + "nauc_recall_at_5_max": 0.5288735395629617, + "nauc_recall_at_5_std": 0.40697999107692173, + "ndcg_at_1": 0.43552, + "ndcg_at_10": 0.55199, + "ndcg_at_100": 0.59427, + "ndcg_at_1000": 0.60987, + "ndcg_at_20": 0.57167, + "ndcg_at_3": 0.50868, + "ndcg_at_5": 0.53147, + "precision_at_1": 0.43552, + "precision_at_10": 0.06797, + "precision_at_100": 0.00878, + "precision_at_1000": 0.001, + "precision_at_20": 0.0379, + "precision_at_3": 0.18675, + "precision_at_5": 0.12326, + "recall_at_1": 0.43552, + "recall_at_10": 0.6797, + "recall_at_100": 0.87844, + "recall_at_1000": 1.0, + "recall_at_20": 0.75793, + "recall_at_3": 0.56025, + "recall_at_5": 0.61628 + }, + { + "cv_recall_at_1": 0.385, + "cv_recall_at_10": 0.547, + "cv_recall_at_100": 0.698, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.593, + "cv_recall_at_3": 0.463, + "cv_recall_at_5": 0.501, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.46078, + "map_at_1": 0.385, + "map_at_10": 0.43387, + "map_at_100": 0.43989, + "map_at_1000": 0.4406, + "map_at_20": 0.43709, + "map_at_3": 0.4195, + "map_at_5": 0.428, + "mrr_at_1": 0.385, + "mrr_at_10": 0.4338726190476187, + "mrr_at_100": 0.43989320041046814, + "mrr_at_1000": 0.44060486564869955, + "mrr_at_20": 0.43708518317337924, + "mrr_at_3": 0.41949999999999976, + "mrr_at_5": 0.42799999999999966, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5809042215421706, + "nauc_cv_recall_at_100_max": 0.8912379770177623, + "nauc_cv_recall_at_100_std": 0.2424633256438869, + "nauc_cv_recall_at_10_diff1": 0.672465991369597, + "nauc_cv_recall_at_10_max": 0.7786860120568095, + "nauc_cv_recall_at_10_std": 0.13047360840701458, + "nauc_cv_recall_at_1_diff1": 0.8258785548979316, + "nauc_cv_recall_at_1_max": 0.7022669665951418, + "nauc_cv_recall_at_1_std": 0.12157864370960537, + "nauc_cv_recall_at_20_diff1": 0.6343060416479551, + "nauc_cv_recall_at_20_max": 0.7904247690928177, + "nauc_cv_recall_at_20_std": 0.16095404825603088, + "nauc_cv_recall_at_3_diff1": 0.7328075282381276, + "nauc_cv_recall_at_3_max": 0.7167558281627555, + "nauc_cv_recall_at_3_std": 0.11546095857706429, + "nauc_cv_recall_at_5_diff1": 0.6998184687081933, + "nauc_cv_recall_at_5_max": 0.7271842109124178, + "nauc_cv_recall_at_5_std": 0.09769344553327645, + "nauc_map_at_1000_diff1": 0.7702690126570004, + "nauc_map_at_1000_max": 0.7186767063695445, + "nauc_map_at_1000_std": 0.12237612475107258, + "nauc_map_at_100_diff1": 0.7700896448953608, + "nauc_map_at_100_max": 0.7190166037158802, + "nauc_map_at_100_std": 0.12303030698837512, + "nauc_map_at_10_diff1": 0.7728156014659673, + "nauc_map_at_10_max": 0.7168863752172929, + "nauc_map_at_10_std": 0.11930561549242265, + "nauc_map_at_1_diff1": 0.8258785548979316, + "nauc_map_at_1_max": 0.7022669665951418, + "nauc_map_at_1_std": 0.12157864370960537, + "nauc_map_at_20_diff1": 0.7705807827864023, + "nauc_map_at_20_max": 0.7172956148978636, + "nauc_map_at_20_std": 0.12130235445505921, + "nauc_map_at_3_diff1": 0.7822816411140364, + "nauc_map_at_3_max": 0.7085502921868301, + "nauc_map_at_3_std": 0.11914325184351209, + "nauc_map_at_5_diff1": 0.7756209852636891, + "nauc_map_at_5_max": 0.710809137344318, + "nauc_map_at_5_std": 0.11531758273692924, + "nauc_mrr_at_1000_diff1": 0.7702690126570004, + "nauc_mrr_at_1000_max": 0.7186767063695445, + "nauc_mrr_at_1000_std": 0.12237612475107258, + "nauc_mrr_at_100_diff1": 0.7700896448953608, + "nauc_mrr_at_100_max": 0.7190166037158802, + "nauc_mrr_at_100_std": 0.12303030698837512, + "nauc_mrr_at_10_diff1": 0.7728156014659673, + "nauc_mrr_at_10_max": 0.7168863752172929, + "nauc_mrr_at_10_std": 0.11930561549242265, + "nauc_mrr_at_1_diff1": 0.8258785548979316, + "nauc_mrr_at_1_max": 0.7022669665951418, + "nauc_mrr_at_1_std": 0.12157864370960537, + "nauc_mrr_at_20_diff1": 0.7705807827864023, + "nauc_mrr_at_20_max": 0.7172956148978636, + "nauc_mrr_at_20_std": 0.12130235445505921, + "nauc_mrr_at_3_diff1": 0.7822816411140364, + "nauc_mrr_at_3_max": 0.7085502921868301, + "nauc_mrr_at_3_std": 0.11914325184351209, + "nauc_mrr_at_5_diff1": 0.7756209852636891, + "nauc_mrr_at_5_max": 0.710809137344318, + "nauc_mrr_at_5_std": 0.11531758273692924, + "nauc_ndcg_at_1000_diff1": 0.7506684503669501, + "nauc_ndcg_at_1000_max": 0.7309290666490663, + "nauc_ndcg_at_1000_std": 0.12703616553444683, + "nauc_ndcg_at_100_diff1": 0.7368679582830968, + "nauc_ndcg_at_100_max": 0.7452584701830449, + "nauc_ndcg_at_100_std": 0.14006746916659532, + "nauc_ndcg_at_10_diff1": 0.7494588028370229, + "nauc_ndcg_at_10_max": 0.7301937966102181, + "nauc_ndcg_at_10_std": 0.12121843676866292, + "nauc_ndcg_at_1_diff1": 0.8258785548979316, + "nauc_ndcg_at_1_max": 0.7022669665951418, + "nauc_ndcg_at_1_std": 0.12157864370960537, + "nauc_ndcg_at_20_diff1": 0.7413130488234162, + "nauc_ndcg_at_20_max": 0.7321767225115111, + "nauc_ndcg_at_20_std": 0.12834744637602127, + "nauc_ndcg_at_3_diff1": 0.7695347817860865, + "nauc_ndcg_at_3_max": 0.7106417741458194, + "nauc_ndcg_at_3_std": 0.11821387146555883, + "nauc_ndcg_at_5_diff1": 0.7568659031020656, + "nauc_ndcg_at_5_max": 0.7147924475388727, + "nauc_ndcg_at_5_std": 0.11114696240542633, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.5809042215421708, + "nauc_precision_at_100_max": 0.891237977017763, + "nauc_precision_at_100_std": 0.24246332564388692, + "nauc_precision_at_10_diff1": 0.672465991369597, + "nauc_precision_at_10_max": 0.7786860120568103, + "nauc_precision_at_10_std": 0.1304736084070146, + "nauc_precision_at_1_diff1": 0.8258785548979316, + "nauc_precision_at_1_max": 0.7022669665951418, + "nauc_precision_at_1_std": 0.12157864370960537, + "nauc_precision_at_20_diff1": 0.6343060416479551, + "nauc_precision_at_20_max": 0.7904247690928177, + "nauc_precision_at_20_std": 0.16095404825603044, + "nauc_precision_at_3_diff1": 0.7328075282381276, + "nauc_precision_at_3_max": 0.716755828162756, + "nauc_precision_at_3_std": 0.11546095857706405, + "nauc_precision_at_5_diff1": 0.6998184687081934, + "nauc_precision_at_5_max": 0.7271842109124178, + "nauc_precision_at_5_std": 0.0976934455332763, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5809042215421706, + "nauc_recall_at_100_max": 0.8912379770177623, + "nauc_recall_at_100_std": 0.2424633256438869, + "nauc_recall_at_10_diff1": 0.672465991369597, + "nauc_recall_at_10_max": 0.7786860120568095, + "nauc_recall_at_10_std": 0.13047360840701458, + "nauc_recall_at_1_diff1": 0.8258785548979316, + "nauc_recall_at_1_max": 0.7022669665951418, + "nauc_recall_at_1_std": 0.12157864370960537, + "nauc_recall_at_20_diff1": 0.6343060416479551, + "nauc_recall_at_20_max": 0.7904247690928177, + "nauc_recall_at_20_std": 0.16095404825603088, + "nauc_recall_at_3_diff1": 0.7328075282381276, + "nauc_recall_at_3_max": 0.7167558281627555, + "nauc_recall_at_3_std": 0.11546095857706429, + "nauc_recall_at_5_diff1": 0.6998184687081933, + "nauc_recall_at_5_max": 0.7271842109124178, + "nauc_recall_at_5_std": 0.09769344553327645, + "ndcg_at_1": 0.385, + "ndcg_at_10": 0.46078, + "ndcg_at_100": 0.49217, + "ndcg_at_1000": 0.52606, + "ndcg_at_20": 0.47243, + "ndcg_at_3": 0.43068, + "ndcg_at_5": 0.44617, + "precision_at_1": 0.385, + "precision_at_10": 0.0547, + "precision_at_100": 0.00698, + "precision_at_1000": 0.001, + "precision_at_20": 0.02965, + "precision_at_3": 0.15433, + "precision_at_5": 0.1002, + "recall_at_1": 0.385, + "recall_at_10": 0.547, + "recall_at_100": 0.698, + "recall_at_1000": 1.0, + "recall_at_20": 0.593, + "recall_at_3": 0.463, + "recall_at_5": 0.501 + } + ] + }, + "task_name": "WITT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/model_meta.json b/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/model_meta.json new file mode 100644 index 0000000000..e19e174af3 --- /dev/null +++ b/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/model_meta.json @@ -0,0 +1 @@ +{"name": "google/siglip-large-patch16-256", "revision": "d0da9f876e7d66b4e250cd2450c3ba2ce735e447", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/WITT2IRetrieval.json b/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/WITT2IRetrieval.json new file mode 100644 index 0000000000..483cb8cbbb --- /dev/null +++ b/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/WITT2IRetrieval.json @@ -0,0 +1,1936 @@ +{ + "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", + "evaluation_time": 305.99254417419434, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.24157, + "cv_recall_at_10": 0.47978, + "cv_recall_at_100": 0.8, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.58427, + "cv_recall_at_3": 0.33708, + "cv_recall_at_5": 0.38315, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.34712, + "map_at_1": 0.24157, + "map_at_10": 0.30661, + "map_at_100": 0.31911, + "map_at_1000": 0.31999, + "map_at_20": 0.31396, + "map_at_3": 0.28371, + "map_at_5": 0.29416, + "mrr_at_1": 0.24157303370786518, + "mrr_at_10": 0.30660558230782947, + "mrr_at_100": 0.3191084750028444, + "mrr_at_1000": 0.3199914067421952, + "mrr_at_20": 0.313955020399258, + "mrr_at_3": 0.28370786516853946, + "mrr_at_5": 0.29415730337078644, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.38537866627754225, + "nauc_cv_recall_at_100_max": 0.5475086711041762, + "nauc_cv_recall_at_100_std": 0.5867727777840137, + "nauc_cv_recall_at_10_diff1": 0.43437211508014945, + "nauc_cv_recall_at_10_max": 0.3640413926999356, + "nauc_cv_recall_at_10_std": 0.3652366136002463, + "nauc_cv_recall_at_1_diff1": 0.5820698913453876, + "nauc_cv_recall_at_1_max": 0.4373704744724589, + "nauc_cv_recall_at_1_std": 0.31242354323111127, + "nauc_cv_recall_at_20_diff1": 0.4519298188132759, + "nauc_cv_recall_at_20_max": 0.3894351430162841, + "nauc_cv_recall_at_20_std": 0.41435732774155504, + "nauc_cv_recall_at_3_diff1": 0.4916636851520573, + "nauc_cv_recall_at_3_max": 0.38074686940966007, + "nauc_cv_recall_at_3_std": 0.33017441860465124, + "nauc_cv_recall_at_5_diff1": 0.4773076210875021, + "nauc_cv_recall_at_5_max": 0.38264219279226985, + "nauc_cv_recall_at_5_std": 0.35768618948540987, + "nauc_map_at_1000_diff1": 0.5318322145757168, + "nauc_map_at_1000_max": 0.4134388754083382, + "nauc_map_at_1000_std": 0.3306243250581623, + "nauc_map_at_100_diff1": 0.5314312528011818, + "nauc_map_at_100_max": 0.41370193480824047, + "nauc_map_at_100_std": 0.3311231298416902, + "nauc_map_at_10_diff1": 0.5298693442691903, + "nauc_map_at_10_max": 0.40899742619453344, + "nauc_map_at_10_std": 0.32611369397428047, + "nauc_map_at_1_diff1": 0.5820698913453876, + "nauc_map_at_1_max": 0.4373704744724589, + "nauc_map_at_1_std": 0.31242354323111127, + "nauc_map_at_20_diff1": 0.5312960056536469, + "nauc_map_at_20_max": 0.41022137345677856, + "nauc_map_at_20_std": 0.3288185491965263, + "nauc_map_at_3_diff1": 0.5385635093747566, + "nauc_map_at_3_max": 0.41066541045322624, + "nauc_map_at_3_std": 0.31799591730631943, + "nauc_map_at_5_diff1": 0.5356614723750011, + "nauc_map_at_5_max": 0.4113715519215499, + "nauc_map_at_5_std": 0.3249822371835517, + "nauc_mrr_at_1000_diff1": 0.5318322145757168, + "nauc_mrr_at_1000_max": 0.4134388754083382, + "nauc_mrr_at_1000_std": 0.3306243250581623, + "nauc_mrr_at_100_diff1": 0.5314312528011818, + "nauc_mrr_at_100_max": 0.41370193480824047, + "nauc_mrr_at_100_std": 0.3311231298416902, + "nauc_mrr_at_10_diff1": 0.5298693442691903, + "nauc_mrr_at_10_max": 0.40899742619453344, + "nauc_mrr_at_10_std": 0.32611369397428047, + "nauc_mrr_at_1_diff1": 0.5820698913453876, + "nauc_mrr_at_1_max": 0.4373704744724589, + "nauc_mrr_at_1_std": 0.31242354323111127, + "nauc_mrr_at_20_diff1": 0.5312960056536469, + "nauc_mrr_at_20_max": 0.41022137345677856, + "nauc_mrr_at_20_std": 0.3288185491965263, + "nauc_mrr_at_3_diff1": 0.5385635093747566, + "nauc_mrr_at_3_max": 0.41066541045322624, + "nauc_mrr_at_3_std": 0.31799591730631943, + "nauc_mrr_at_5_diff1": 0.5356614723750011, + "nauc_mrr_at_5_max": 0.4113715519215499, + "nauc_mrr_at_5_std": 0.3249822371835517, + "nauc_ndcg_at_1000_diff1": 0.520218088630065, + "nauc_ndcg_at_1000_max": 0.41499229550011485, + "nauc_ndcg_at_1000_std": 0.3446399250064482, + "nauc_ndcg_at_100_diff1": 0.5096653930893602, + "nauc_ndcg_at_100_max": 0.42468681407454717, + "nauc_ndcg_at_100_std": 0.3622985410547835, + "nauc_ndcg_at_10_diff1": 0.5060520176550207, + "nauc_ndcg_at_10_max": 0.3976210920323349, + "nauc_ndcg_at_10_std": 0.3358701849148283, + "nauc_ndcg_at_1_diff1": 0.5820698913453876, + "nauc_ndcg_at_1_max": 0.4373704744724589, + "nauc_ndcg_at_1_std": 0.31242354323111127, + "nauc_ndcg_at_20_diff1": 0.5116710283167186, + "nauc_ndcg_at_20_max": 0.4037066777079507, + "nauc_ndcg_at_20_std": 0.3467836131589162, + "nauc_ndcg_at_3_diff1": 0.5259304404932885, + "nauc_ndcg_at_3_max": 0.40262956715553533, + "nauc_ndcg_at_3_std": 0.3211502538045597, + "nauc_ndcg_at_5_diff1": 0.52011269409434, + "nauc_ndcg_at_5_max": 0.4035901466934185, + "nauc_ndcg_at_5_std": 0.33334212092589155, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.38537866627754325, + "nauc_precision_at_100_max": 0.5475086711041771, + "nauc_precision_at_100_std": 0.5867727777840136, + "nauc_precision_at_10_diff1": 0.43437211508014945, + "nauc_precision_at_10_max": 0.36404139269993546, + "nauc_precision_at_10_std": 0.3652366136002458, + "nauc_precision_at_1_diff1": 0.5820698913453876, + "nauc_precision_at_1_max": 0.4373704744724589, + "nauc_precision_at_1_std": 0.31242354323111127, + "nauc_precision_at_20_diff1": 0.4519298188132756, + "nauc_precision_at_20_max": 0.38943514301628335, + "nauc_precision_at_20_std": 0.41435732774155454, + "nauc_precision_at_3_diff1": 0.49166368515205705, + "nauc_precision_at_3_max": 0.3807468694096599, + "nauc_precision_at_3_std": 0.33017441860465113, + "nauc_precision_at_5_diff1": 0.4773076210875018, + "nauc_precision_at_5_max": 0.38264219279226996, + "nauc_precision_at_5_std": 0.35768618948541014, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.38537866627754225, + "nauc_recall_at_100_max": 0.5475086711041762, + "nauc_recall_at_100_std": 0.5867727777840137, + "nauc_recall_at_10_diff1": 0.43437211508014945, + "nauc_recall_at_10_max": 0.3640413926999356, + "nauc_recall_at_10_std": 0.3652366136002463, + "nauc_recall_at_1_diff1": 0.5820698913453876, + "nauc_recall_at_1_max": 0.4373704744724589, + "nauc_recall_at_1_std": 0.31242354323111127, + "nauc_recall_at_20_diff1": 0.4519298188132759, + "nauc_recall_at_20_max": 0.3894351430162841, + "nauc_recall_at_20_std": 0.41435732774155504, + "nauc_recall_at_3_diff1": 0.4916636851520573, + "nauc_recall_at_3_max": 0.38074686940966007, + "nauc_recall_at_3_std": 0.33017441860465124, + "nauc_recall_at_5_diff1": 0.4773076210875021, + "nauc_recall_at_5_max": 0.38264219279226985, + "nauc_recall_at_5_std": 0.35768618948540987, + "ndcg_at_1": 0.24157, + "ndcg_at_10": 0.34712, + "ndcg_at_100": 0.41285, + "ndcg_at_1000": 0.43797, + "ndcg_at_20": 0.37366, + "ndcg_at_3": 0.29742, + "ndcg_at_5": 0.31632, + "precision_at_1": 0.24157, + "precision_at_10": 0.04798, + "precision_at_100": 0.008, + "precision_at_1000": 0.001, + "precision_at_20": 0.02921, + "precision_at_3": 0.11236, + "precision_at_5": 0.07663, + "recall_at_1": 0.24157, + "recall_at_10": 0.47978, + "recall_at_100": 0.8, + "recall_at_1000": 1.0, + "recall_at_20": 0.58427, + "recall_at_3": 0.33708, + "recall_at_5": 0.38315 + }, + { + "cv_recall_at_1": 0.22558, + "cv_recall_at_10": 0.44884, + "cv_recall_at_100": 0.76512, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.52558, + "cv_recall_at_3": 0.32326, + "cv_recall_at_5": 0.3686, + "hf_subset": "bg", + "languages": [ + "bul-Cyrl" + ], + "main_score": 0.32733, + "map_at_1": 0.22558, + "map_at_10": 0.28986, + "map_at_100": 0.30114, + "map_at_1000": 0.30225, + "map_at_20": 0.29522, + "map_at_3": 0.26919, + "map_at_5": 0.27948, + "mrr_at_1": 0.2255813953488372, + "mrr_at_10": 0.2898629568106311, + "mrr_at_100": 0.3011372315333812, + "mrr_at_1000": 0.3022512651766774, + "mrr_at_20": 0.29521953737457013, + "mrr_at_3": 0.269186046511628, + "mrr_at_5": 0.2794767441860466, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4156195458011944, + "nauc_cv_recall_at_100_max": 0.4321414496076812, + "nauc_cv_recall_at_100_std": 0.2821314689037091, + "nauc_cv_recall_at_10_diff1": 0.4393000325721993, + "nauc_cv_recall_at_10_max": 0.3403794813410009, + "nauc_cv_recall_at_10_std": 0.047695440805355475, + "nauc_cv_recall_at_1_diff1": 0.5456746501489526, + "nauc_cv_recall_at_1_max": 0.3886520586941517, + "nauc_cv_recall_at_1_std": 0.06026950958268227, + "nauc_cv_recall_at_20_diff1": 0.4133092357721504, + "nauc_cv_recall_at_20_max": 0.3480612705948237, + "nauc_cv_recall_at_20_std": 0.07327907139538889, + "nauc_cv_recall_at_3_diff1": 0.4697449310838643, + "nauc_cv_recall_at_3_max": 0.36616880958519854, + "nauc_cv_recall_at_3_std": 0.08114614818289403, + "nauc_cv_recall_at_5_diff1": 0.43915646998582375, + "nauc_cv_recall_at_5_max": 0.33399631384859435, + "nauc_cv_recall_at_5_std": 0.0406646620242838, + "nauc_map_at_1000_diff1": 0.49689439348256015, + "nauc_map_at_1000_max": 0.3694777352607035, + "nauc_map_at_1000_std": 0.06365099267013102, + "nauc_map_at_100_diff1": 0.49652671912506346, + "nauc_map_at_100_max": 0.36949625835147754, + "nauc_map_at_100_std": 0.06390111935590399, + "nauc_map_at_10_diff1": 0.498401126203964, + "nauc_map_at_10_max": 0.3675709111670907, + "nauc_map_at_10_std": 0.059359535273785916, + "nauc_map_at_1_diff1": 0.5456746501489526, + "nauc_map_at_1_max": 0.3886520586941517, + "nauc_map_at_1_std": 0.06026950958268227, + "nauc_map_at_20_diff1": 0.4964328959270357, + "nauc_map_at_20_max": 0.36783464945255584, + "nauc_map_at_20_std": 0.06073670621949507, + "nauc_map_at_3_diff1": 0.5074187337184277, + "nauc_map_at_3_max": 0.37474783652153665, + "nauc_map_at_3_std": 0.06987499036428685, + "nauc_map_at_5_diff1": 0.4994576226731685, + "nauc_map_at_5_max": 0.36685059417933674, + "nauc_map_at_5_std": 0.05981883149689524, + "nauc_mrr_at_1000_diff1": 0.4968944008896699, + "nauc_mrr_at_1000_max": 0.36947775312287956, + "nauc_mrr_at_1000_std": 0.0636510175033233, + "nauc_mrr_at_100_diff1": 0.49652671912506346, + "nauc_mrr_at_100_max": 0.36949625835147754, + "nauc_mrr_at_100_std": 0.06390111935590399, + "nauc_mrr_at_10_diff1": 0.498401126203964, + "nauc_mrr_at_10_max": 0.3675709111670907, + "nauc_mrr_at_10_std": 0.059359535273785916, + "nauc_mrr_at_1_diff1": 0.5456746501489526, + "nauc_mrr_at_1_max": 0.3886520586941517, + "nauc_mrr_at_1_std": 0.06026950958268227, + "nauc_mrr_at_20_diff1": 0.4964328959270357, + "nauc_mrr_at_20_max": 0.36783464945255584, + "nauc_mrr_at_20_std": 0.06073670621949507, + "nauc_mrr_at_3_diff1": 0.5074187337184277, + "nauc_mrr_at_3_max": 0.37474783652153665, + "nauc_mrr_at_3_std": 0.06987499036428685, + "nauc_mrr_at_5_diff1": 0.4994576226731685, + "nauc_mrr_at_5_max": 0.36685059417933674, + "nauc_mrr_at_5_std": 0.05981883149689524, + "nauc_ndcg_at_1000_diff1": 0.48604737697435507, + "nauc_ndcg_at_1000_max": 0.36942860131684324, + "nauc_ndcg_at_1000_std": 0.0715185946116579, + "nauc_ndcg_at_100_diff1": 0.4792033641315331, + "nauc_ndcg_at_100_max": 0.3738987011330743, + "nauc_ndcg_at_100_std": 0.0875981776356468, + "nauc_ndcg_at_10_diff1": 0.4823054910823962, + "nauc_ndcg_at_10_max": 0.3602281989472767, + "nauc_ndcg_at_10_std": 0.056135484386297785, + "nauc_ndcg_at_1_diff1": 0.5456746501489526, + "nauc_ndcg_at_1_max": 0.3886520586941517, + "nauc_ndcg_at_1_std": 0.06026950958268227, + "nauc_ndcg_at_20_diff1": 0.47568071371427356, + "nauc_ndcg_at_20_max": 0.3619843976708002, + "nauc_ndcg_at_20_std": 0.06226665792973339, + "nauc_ndcg_at_3_diff1": 0.49712680073778454, + "nauc_ndcg_at_3_max": 0.3722913564738766, + "nauc_ndcg_at_3_std": 0.0729141340064338, + "nauc_ndcg_at_5_diff1": 0.48331434091040426, + "nauc_ndcg_at_5_max": 0.3582440286439964, + "nauc_ndcg_at_5_std": 0.05518668005370692, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.4156195458011953, + "nauc_precision_at_100_max": 0.4321414496076824, + "nauc_precision_at_100_std": 0.2821314689037095, + "nauc_precision_at_10_diff1": 0.4393000325721987, + "nauc_precision_at_10_max": 0.3403794813410003, + "nauc_precision_at_10_std": 0.047695440805355274, + "nauc_precision_at_1_diff1": 0.5456746501489526, + "nauc_precision_at_1_max": 0.3886520586941517, + "nauc_precision_at_1_std": 0.06026950958268227, + "nauc_precision_at_20_diff1": 0.41330923577215073, + "nauc_precision_at_20_max": 0.34806127059482384, + "nauc_precision_at_20_std": 0.07327907139538871, + "nauc_precision_at_3_diff1": 0.46974493108386384, + "nauc_precision_at_3_max": 0.3661688095851986, + "nauc_precision_at_3_std": 0.08114614818289405, + "nauc_precision_at_5_diff1": 0.43915646998582347, + "nauc_precision_at_5_max": 0.33399631384859413, + "nauc_precision_at_5_std": 0.04066466202428358, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4156195458011944, + "nauc_recall_at_100_max": 0.4321414496076812, + "nauc_recall_at_100_std": 0.2821314689037091, + "nauc_recall_at_10_diff1": 0.4393000325721993, + "nauc_recall_at_10_max": 0.3403794813410009, + "nauc_recall_at_10_std": 0.047695440805355475, + "nauc_recall_at_1_diff1": 0.5456746501489526, + "nauc_recall_at_1_max": 0.3886520586941517, + "nauc_recall_at_1_std": 0.06026950958268227, + "nauc_recall_at_20_diff1": 0.4133092357721504, + "nauc_recall_at_20_max": 0.3480612705948237, + "nauc_recall_at_20_std": 0.07327907139538889, + "nauc_recall_at_3_diff1": 0.4697449310838643, + "nauc_recall_at_3_max": 0.36616880958519854, + "nauc_recall_at_3_std": 0.08114614818289403, + "nauc_recall_at_5_diff1": 0.43915646998582375, + "nauc_recall_at_5_max": 0.33399631384859435, + "nauc_recall_at_5_std": 0.0406646620242838, + "ndcg_at_1": 0.22558, + "ndcg_at_10": 0.32733, + "ndcg_at_100": 0.3908, + "ndcg_at_1000": 0.4207, + "ndcg_at_20": 0.34678, + "ndcg_at_3": 0.2831, + "ndcg_at_5": 0.30171, + "precision_at_1": 0.22558, + "precision_at_10": 0.04488, + "precision_at_100": 0.00765, + "precision_at_1000": 0.001, + "precision_at_20": 0.02628, + "precision_at_3": 0.10775, + "precision_at_5": 0.07372, + "recall_at_1": 0.22558, + "recall_at_10": 0.44884, + "recall_at_100": 0.76512, + "recall_at_1000": 1.0, + "recall_at_20": 0.52558, + "recall_at_3": 0.32326, + "recall_at_5": 0.3686 + }, + { + "cv_recall_at_1": 0.40853, + "cv_recall_at_10": 0.65993, + "cv_recall_at_100": 0.87991, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.72727, + "cv_recall_at_3": 0.5376, + "cv_recall_at_5": 0.60494, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.52913, + "map_at_1": 0.40853, + "map_at_10": 0.4877, + "map_at_100": 0.49611, + "map_at_1000": 0.49663, + "map_at_20": 0.49233, + "map_at_3": 0.46483, + "map_at_5": 0.48043, + "mrr_at_1": 0.40852974186307517, + "mrr_at_10": 0.48770242103575406, + "mrr_at_100": 0.49610506582006364, + "mrr_at_1000": 0.4966320096481985, + "mrr_at_20": 0.49233257586044954, + "mrr_at_3": 0.4648335203890757, + "mrr_at_5": 0.48043396932285776, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5909320870844869, + "nauc_cv_recall_at_100_max": 0.6350295788074445, + "nauc_cv_recall_at_100_std": 0.5885417610554063, + "nauc_cv_recall_at_10_diff1": 0.6333473779929506, + "nauc_cv_recall_at_10_max": 0.5446487146800527, + "nauc_cv_recall_at_10_std": 0.2822394925476734, + "nauc_cv_recall_at_1_diff1": 0.765059299328094, + "nauc_cv_recall_at_1_max": 0.5047735351426804, + "nauc_cv_recall_at_1_std": 0.04360795615900494, + "nauc_cv_recall_at_20_diff1": 0.5950393518016469, + "nauc_cv_recall_at_20_max": 0.560818330778241, + "nauc_cv_recall_at_20_std": 0.34469275409722727, + "nauc_cv_recall_at_3_diff1": 0.6761769057730284, + "nauc_cv_recall_at_3_max": 0.507038537611141, + "nauc_cv_recall_at_3_std": 0.15324466916067125, + "nauc_cv_recall_at_5_diff1": 0.6311219292955862, + "nauc_cv_recall_at_5_max": 0.5258098225560203, + "nauc_cv_recall_at_5_std": 0.22820889829638605, + "nauc_map_at_1000_diff1": 0.7177069863322598, + "nauc_map_at_1000_max": 0.512298998322214, + "nauc_map_at_1000_std": 0.11131691750068567, + "nauc_map_at_100_diff1": 0.7174829160021539, + "nauc_map_at_100_max": 0.5123333682356263, + "nauc_map_at_100_std": 0.11191195506396118, + "nauc_map_at_10_diff1": 0.7185132575262295, + "nauc_map_at_10_max": 0.5119807392194838, + "nauc_map_at_10_std": 0.11013406915416628, + "nauc_map_at_1_diff1": 0.765059299328094, + "nauc_map_at_1_max": 0.5047735351426804, + "nauc_map_at_1_std": 0.04360795615900494, + "nauc_map_at_20_diff1": 0.7172107213295099, + "nauc_map_at_20_max": 0.512652076542774, + "nauc_map_at_20_std": 0.11191137664292765, + "nauc_map_at_3_diff1": 0.7256321861141027, + "nauc_map_at_3_max": 0.5066686915715853, + "nauc_map_at_3_std": 0.0917218401207876, + "nauc_map_at_5_diff1": 0.7171261768894966, + "nauc_map_at_5_max": 0.5102234717791918, + "nauc_map_at_5_std": 0.10560034278408584, + "nauc_mrr_at_1000_diff1": 0.7177069863322598, + "nauc_mrr_at_1000_max": 0.512298998322214, + "nauc_mrr_at_1000_std": 0.11131691750068567, + "nauc_mrr_at_100_diff1": 0.7174829160021539, + "nauc_mrr_at_100_max": 0.5123333682356263, + "nauc_mrr_at_100_std": 0.11191195506396118, + "nauc_mrr_at_10_diff1": 0.7185132575262295, + "nauc_mrr_at_10_max": 0.5119807392194838, + "nauc_mrr_at_10_std": 0.11013406915416628, + "nauc_mrr_at_1_diff1": 0.765059299328094, + "nauc_mrr_at_1_max": 0.5047735351426804, + "nauc_mrr_at_1_std": 0.04360795615900494, + "nauc_mrr_at_20_diff1": 0.7172107213295099, + "nauc_mrr_at_20_max": 0.512652076542774, + "nauc_mrr_at_20_std": 0.11191137664292765, + "nauc_mrr_at_3_diff1": 0.7256321861141027, + "nauc_mrr_at_3_max": 0.5066686915715853, + "nauc_mrr_at_3_std": 0.0917218401207876, + "nauc_mrr_at_5_diff1": 0.7171261768894966, + "nauc_mrr_at_5_max": 0.5102234717791918, + "nauc_mrr_at_5_std": 0.10560034278408584, + "nauc_ndcg_at_1000_diff1": 0.7061380148421971, + "nauc_ndcg_at_1000_max": 0.5171327008339424, + "nauc_ndcg_at_1000_std": 0.13519038473094888, + "nauc_ndcg_at_100_diff1": 0.7001742768553503, + "nauc_ndcg_at_100_max": 0.5219790485043022, + "nauc_ndcg_at_100_std": 0.15685332183761455, + "nauc_ndcg_at_10_diff1": 0.6999437000060731, + "nauc_ndcg_at_10_max": 0.5184168746325684, + "nauc_ndcg_at_10_std": 0.14593465923975651, + "nauc_ndcg_at_1_diff1": 0.765059299328094, + "nauc_ndcg_at_1_max": 0.5047735351426804, + "nauc_ndcg_at_1_std": 0.04360795615900494, + "nauc_ndcg_at_20_diff1": 0.6946562518708675, + "nauc_ndcg_at_20_max": 0.520841997954365, + "nauc_ndcg_at_20_std": 0.15355304474124215, + "nauc_ndcg_at_3_diff1": 0.7132599403846717, + "nauc_ndcg_at_3_max": 0.5068006685743853, + "nauc_ndcg_at_3_std": 0.10709000760145516, + "nauc_ndcg_at_5_diff1": 0.6972127766816861, + "nauc_ndcg_at_5_max": 0.5136594718171129, + "nauc_ndcg_at_5_std": 0.1337092541319943, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.5909320870844862, + "nauc_precision_at_100_max": 0.6350295788074443, + "nauc_precision_at_100_std": 0.5885417610554048, + "nauc_precision_at_10_diff1": 0.6333473779929502, + "nauc_precision_at_10_max": 0.5446487146800514, + "nauc_precision_at_10_std": 0.2822394925476726, + "nauc_precision_at_1_diff1": 0.765059299328094, + "nauc_precision_at_1_max": 0.5047735351426804, + "nauc_precision_at_1_std": 0.04360795615900494, + "nauc_precision_at_20_diff1": 0.5950393518016462, + "nauc_precision_at_20_max": 0.5608183307782408, + "nauc_precision_at_20_std": 0.3446927540972269, + "nauc_precision_at_3_diff1": 0.676176905773028, + "nauc_precision_at_3_max": 0.5070385376111409, + "nauc_precision_at_3_std": 0.1532446691606712, + "nauc_precision_at_5_diff1": 0.6311219292955864, + "nauc_precision_at_5_max": 0.5258098225560205, + "nauc_precision_at_5_std": 0.22820889829638602, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5909320870844869, + "nauc_recall_at_100_max": 0.6350295788074445, + "nauc_recall_at_100_std": 0.5885417610554063, + "nauc_recall_at_10_diff1": 0.6333473779929506, + "nauc_recall_at_10_max": 0.5446487146800527, + "nauc_recall_at_10_std": 0.2822394925476734, + "nauc_recall_at_1_diff1": 0.765059299328094, + "nauc_recall_at_1_max": 0.5047735351426804, + "nauc_recall_at_1_std": 0.04360795615900494, + "nauc_recall_at_20_diff1": 0.5950393518016469, + "nauc_recall_at_20_max": 0.560818330778241, + "nauc_recall_at_20_std": 0.34469275409722727, + "nauc_recall_at_3_diff1": 0.6761769057730284, + "nauc_recall_at_3_max": 0.507038537611141, + "nauc_recall_at_3_std": 0.15324466916067125, + "nauc_recall_at_5_diff1": 0.6311219292955862, + "nauc_recall_at_5_max": 0.5258098225560203, + "nauc_recall_at_5_std": 0.22820889829638605, + "ndcg_at_1": 0.40853, + "ndcg_at_10": 0.52913, + "ndcg_at_100": 0.57416, + "ndcg_at_1000": 0.58917, + "ndcg_at_20": 0.54611, + "ndcg_at_3": 0.4835, + "ndcg_at_5": 0.51142, + "precision_at_1": 0.40853, + "precision_at_10": 0.06599, + "precision_at_100": 0.0088, + "precision_at_1000": 0.001, + "precision_at_20": 0.03636, + "precision_at_3": 0.1792, + "precision_at_5": 0.12099, + "recall_at_1": 0.40853, + "recall_at_10": 0.65993, + "recall_at_100": 0.87991, + "recall_at_1000": 1.0, + "recall_at_20": 0.72727, + "recall_at_3": 0.5376, + "recall_at_5": 0.60494 + }, + { + "cv_recall_at_1": 0.16667, + "cv_recall_at_10": 0.35088, + "cv_recall_at_100": 0.69474, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.45439, + "cv_recall_at_3": 0.23509, + "cv_recall_at_5": 0.27719, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.24657, + "map_at_1": 0.16667, + "map_at_10": 0.21466, + "map_at_100": 0.2275, + "map_at_1000": 0.22898, + "map_at_20": 0.22193, + "map_at_3": 0.19503, + "map_at_5": 0.20477, + "mrr_at_1": 0.16666666666666666, + "mrr_at_10": 0.2146602617655249, + "mrr_at_100": 0.227502583825205, + "mrr_at_1000": 0.22898457017327967, + "mrr_at_20": 0.22192984828555953, + "mrr_at_3": 0.19502923976608177, + "mrr_at_5": 0.20476608187134498, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.330851835183518, + "nauc_cv_recall_at_100_max": 0.2257475747574758, + "nauc_cv_recall_at_100_std": 0.3469628212821287, + "nauc_cv_recall_at_10_diff1": 0.4433734104046245, + "nauc_cv_recall_at_10_max": 0.19268670520231207, + "nauc_cv_recall_at_10_std": 0.24033757225433527, + "nauc_cv_recall_at_1_diff1": 0.6046377190260788, + "nauc_cv_recall_at_1_max": 0.2795826052876528, + "nauc_cv_recall_at_1_std": 0.14331204990446947, + "nauc_cv_recall_at_20_diff1": 0.37560598495707764, + "nauc_cv_recall_at_20_max": 0.1927210478966829, + "nauc_cv_recall_at_20_std": 0.2678942439769638, + "nauc_cv_recall_at_3_diff1": 0.4957581959607422, + "nauc_cv_recall_at_3_max": 0.15605331947677356, + "nauc_cv_recall_at_3_std": 0.12871489244375645, + "nauc_cv_recall_at_5_diff1": 0.4619490553123599, + "nauc_cv_recall_at_5_max": 0.21248947474677848, + "nauc_cv_recall_at_5_std": 0.20337989885908708, + "nauc_map_at_1000_diff1": 0.5323722758486542, + "nauc_map_at_1000_max": 0.23646297993116652, + "nauc_map_at_1000_std": 0.16750923161500914, + "nauc_map_at_100_diff1": 0.5316186665095363, + "nauc_map_at_100_max": 0.23635986712686907, + "nauc_map_at_100_std": 0.16820139297706768, + "nauc_map_at_10_diff1": 0.5377626878903065, + "nauc_map_at_10_max": 0.23435667673382332, + "nauc_map_at_10_std": 0.16307234407663262, + "nauc_map_at_1_diff1": 0.6046377190260788, + "nauc_map_at_1_max": 0.2795826052876528, + "nauc_map_at_1_std": 0.14331204990446947, + "nauc_map_at_20_diff1": 0.5318397901748458, + "nauc_map_at_20_max": 0.23438189527334707, + "nauc_map_at_20_std": 0.16600605966152815, + "nauc_map_at_3_diff1": 0.5525148210101732, + "nauc_map_at_3_max": 0.2214669332663381, + "nauc_map_at_3_std": 0.1354554693488529, + "nauc_map_at_5_diff1": 0.5418733724063102, + "nauc_map_at_5_max": 0.23602458830140594, + "nauc_map_at_5_std": 0.15594974954289342, + "nauc_mrr_at_1000_diff1": 0.5323722758486542, + "nauc_mrr_at_1000_max": 0.23646297993116652, + "nauc_mrr_at_1000_std": 0.16750923161500914, + "nauc_mrr_at_100_diff1": 0.5316186665095363, + "nauc_mrr_at_100_max": 0.23635986712686907, + "nauc_mrr_at_100_std": 0.16820139297706768, + "nauc_mrr_at_10_diff1": 0.5377626878903065, + "nauc_mrr_at_10_max": 0.23435667673382332, + "nauc_mrr_at_10_std": 0.16307234407663262, + "nauc_mrr_at_1_diff1": 0.6046377190260788, + "nauc_mrr_at_1_max": 0.2795826052876528, + "nauc_mrr_at_1_std": 0.14331204990446947, + "nauc_mrr_at_20_diff1": 0.5318397901748458, + "nauc_mrr_at_20_max": 0.23438189527334707, + "nauc_mrr_at_20_std": 0.16600605966152815, + "nauc_mrr_at_3_diff1": 0.5525148210101732, + "nauc_mrr_at_3_max": 0.2214669332663381, + "nauc_mrr_at_3_std": 0.1354554693488529, + "nauc_mrr_at_5_diff1": 0.5418733724063102, + "nauc_mrr_at_5_max": 0.23602458830140594, + "nauc_mrr_at_5_std": 0.15594974954289342, + "nauc_ndcg_at_1000_diff1": 0.5093691679149464, + "nauc_ndcg_at_1000_max": 0.23258162232421242, + "nauc_ndcg_at_1000_std": 0.1852794629243669, + "nauc_ndcg_at_100_diff1": 0.4889330716375234, + "nauc_ndcg_at_100_max": 0.2314467376425257, + "nauc_ndcg_at_100_std": 0.20368758726476266, + "nauc_ndcg_at_10_diff1": 0.5107670584417292, + "nauc_ndcg_at_10_max": 0.2224163298308434, + "nauc_ndcg_at_10_std": 0.18382686225774514, + "nauc_ndcg_at_1_diff1": 0.6046377190260788, + "nauc_ndcg_at_1_max": 0.2795826052876528, + "nauc_ndcg_at_1_std": 0.14331204990446947, + "nauc_ndcg_at_20_diff1": 0.4905452978639291, + "nauc_ndcg_at_20_max": 0.2224535017956619, + "nauc_ndcg_at_20_std": 0.19270082403457917, + "nauc_ndcg_at_3_diff1": 0.536769234211766, + "nauc_ndcg_at_3_max": 0.20336056958214557, + "nauc_ndcg_at_3_std": 0.1335506875360385, + "nauc_ndcg_at_5_diff1": 0.5193278051702171, + "nauc_ndcg_at_5_max": 0.22853851210641513, + "nauc_ndcg_at_5_std": 0.16852042136773013, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.33085183518351857, + "nauc_precision_at_100_max": 0.2257475747574758, + "nauc_precision_at_100_std": 0.3469628212821282, + "nauc_precision_at_10_diff1": 0.44337341040462425, + "nauc_precision_at_10_max": 0.19268670520231215, + "nauc_precision_at_10_std": 0.24033757225433522, + "nauc_precision_at_1_diff1": 0.6046377190260788, + "nauc_precision_at_1_max": 0.2795826052876528, + "nauc_precision_at_1_std": 0.14331204990446947, + "nauc_precision_at_20_diff1": 0.3756059849570778, + "nauc_precision_at_20_max": 0.1927210478966828, + "nauc_precision_at_20_std": 0.26789424397696393, + "nauc_precision_at_3_diff1": 0.49575819596074194, + "nauc_precision_at_3_max": 0.15605331947677353, + "nauc_precision_at_3_std": 0.1287148924437563, + "nauc_precision_at_5_diff1": 0.4619490553123598, + "nauc_precision_at_5_max": 0.21248947474677846, + "nauc_precision_at_5_std": 0.20337989885908728, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.330851835183518, + "nauc_recall_at_100_max": 0.2257475747574758, + "nauc_recall_at_100_std": 0.3469628212821287, + "nauc_recall_at_10_diff1": 0.4433734104046245, + "nauc_recall_at_10_max": 0.19268670520231207, + "nauc_recall_at_10_std": 0.24033757225433527, + "nauc_recall_at_1_diff1": 0.6046377190260788, + "nauc_recall_at_1_max": 0.2795826052876528, + "nauc_recall_at_1_std": 0.14331204990446947, + "nauc_recall_at_20_diff1": 0.37560598495707764, + "nauc_recall_at_20_max": 0.1927210478966829, + "nauc_recall_at_20_std": 0.2678942439769638, + "nauc_recall_at_3_diff1": 0.4957581959607422, + "nauc_recall_at_3_max": 0.15605331947677356, + "nauc_recall_at_3_std": 0.12871489244375645, + "nauc_recall_at_5_diff1": 0.4619490553123599, + "nauc_recall_at_5_max": 0.21248947474677848, + "nauc_recall_at_5_std": 0.20337989885908708, + "ndcg_at_1": 0.16667, + "ndcg_at_10": 0.24657, + "ndcg_at_100": 0.31625, + "ndcg_at_1000": 0.35548, + "ndcg_at_20": 0.27285, + "ndcg_at_3": 0.20524, + "ndcg_at_5": 0.22268, + "precision_at_1": 0.16667, + "precision_at_10": 0.03509, + "precision_at_100": 0.00695, + "precision_at_1000": 0.001, + "precision_at_20": 0.02272, + "precision_at_3": 0.07836, + "precision_at_5": 0.05544, + "recall_at_1": 0.16667, + "recall_at_10": 0.35088, + "recall_at_100": 0.69474, + "recall_at_1000": 1.0, + "recall_at_20": 0.45439, + "recall_at_3": 0.23509, + "recall_at_5": 0.27719 + }, + { + "cv_recall_at_1": 0.22426, + "cv_recall_at_10": 0.40732, + "cv_recall_at_100": 0.76087, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.49542, + "cv_recall_at_3": 0.30549, + "cv_recall_at_5": 0.33982, + "hf_subset": "et", + "languages": [ + "est-Latn" + ], + "main_score": 0.30725, + "map_at_1": 0.22426, + "map_at_10": 0.27627, + "map_at_100": 0.28886, + "map_at_1000": 0.29, + "map_at_20": 0.28234, + "map_at_3": 0.25915, + "map_at_5": 0.26693, + "mrr_at_1": 0.2242562929061785, + "mrr_at_10": 0.27627492644655105, + "mrr_at_100": 0.2888584831291441, + "mrr_at_1000": 0.2900005222322179, + "mrr_at_20": 0.2823424222161656, + "mrr_at_3": 0.2591533180778033, + "mrr_at_5": 0.26693363844393597, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4548905095902432, + "nauc_cv_recall_at_100_max": 0.44644713522120655, + "nauc_cv_recall_at_100_std": 0.35804392058592754, + "nauc_cv_recall_at_10_diff1": 0.4959708344082086, + "nauc_cv_recall_at_10_max": 0.32782195690308946, + "nauc_cv_recall_at_10_std": 0.11774784435354384, + "nauc_cv_recall_at_1_diff1": 0.6688306834003572, + "nauc_cv_recall_at_1_max": 0.41410112769511687, + "nauc_cv_recall_at_1_std": 0.06822755485394784, + "nauc_cv_recall_at_20_diff1": 0.4933700813122085, + "nauc_cv_recall_at_20_max": 0.3493270465823052, + "nauc_cv_recall_at_20_std": 0.21358675720665704, + "nauc_cv_recall_at_3_diff1": 0.5676433428285875, + "nauc_cv_recall_at_3_max": 0.33746528106591145, + "nauc_cv_recall_at_3_std": 0.07763926424380885, + "nauc_cv_recall_at_5_diff1": 0.513300302770812, + "nauc_cv_recall_at_5_max": 0.3473703885687822, + "nauc_cv_recall_at_5_std": 0.09544098700198667, + "nauc_map_at_1000_diff1": 0.601309946818558, + "nauc_map_at_1000_max": 0.3811215060299473, + "nauc_map_at_1000_std": 0.08549562744577777, + "nauc_map_at_100_diff1": 0.6010036129821578, + "nauc_map_at_100_max": 0.3813066301818496, + "nauc_map_at_100_std": 0.0862408417273817, + "nauc_map_at_10_diff1": 0.6007541906369676, + "nauc_map_at_10_max": 0.3775895413975735, + "nauc_map_at_10_std": 0.07752197864393401, + "nauc_map_at_1_diff1": 0.6688306834003572, + "nauc_map_at_1_max": 0.41410112769511687, + "nauc_map_at_1_std": 0.06822755485394784, + "nauc_map_at_20_diff1": 0.6005353949460159, + "nauc_map_at_20_max": 0.3794670839752862, + "nauc_map_at_20_std": 0.08483609607709942, + "nauc_map_at_3_diff1": 0.6183803642609942, + "nauc_map_at_3_max": 0.37855822227622066, + "nauc_map_at_3_std": 0.06914276915890476, + "nauc_map_at_5_diff1": 0.6048073533485189, + "nauc_map_at_5_max": 0.38065436367380495, + "nauc_map_at_5_std": 0.07376843881638116, + "nauc_mrr_at_1000_diff1": 0.601309946818558, + "nauc_mrr_at_1000_max": 0.3811215060299473, + "nauc_mrr_at_1000_std": 0.08549562744577777, + "nauc_mrr_at_100_diff1": 0.6010036129821578, + "nauc_mrr_at_100_max": 0.3813066301818496, + "nauc_mrr_at_100_std": 0.0862408417273817, + "nauc_mrr_at_10_diff1": 0.6007541906369676, + "nauc_mrr_at_10_max": 0.3775895413975735, + "nauc_mrr_at_10_std": 0.07752197864393401, + "nauc_mrr_at_1_diff1": 0.6688306834003572, + "nauc_mrr_at_1_max": 0.41410112769511687, + "nauc_mrr_at_1_std": 0.06822755485394784, + "nauc_mrr_at_20_diff1": 0.6005353949460159, + "nauc_mrr_at_20_max": 0.3794670839752862, + "nauc_mrr_at_20_std": 0.08483609607709942, + "nauc_mrr_at_3_diff1": 0.6183803642609942, + "nauc_mrr_at_3_max": 0.37855822227622066, + "nauc_mrr_at_3_std": 0.06914276915890476, + "nauc_mrr_at_5_diff1": 0.6048073533485189, + "nauc_mrr_at_5_max": 0.38065436367380495, + "nauc_mrr_at_5_std": 0.07376843881638116, + "nauc_ndcg_at_1000_diff1": 0.5845812201325902, + "nauc_ndcg_at_1000_max": 0.37872969489351366, + "nauc_ndcg_at_1000_std": 0.1023087725689833, + "nauc_ndcg_at_100_diff1": 0.5738375546183035, + "nauc_ndcg_at_100_max": 0.38463128334228763, + "nauc_ndcg_at_100_std": 0.12459238801374915, + "nauc_ndcg_at_10_diff1": 0.5727643589394087, + "nauc_ndcg_at_10_max": 0.3643059512756334, + "nauc_ndcg_at_10_std": 0.08759165516814928, + "nauc_ndcg_at_1_diff1": 0.6688306834003572, + "nauc_ndcg_at_1_max": 0.41410112769511687, + "nauc_ndcg_at_1_std": 0.06822755485394784, + "nauc_ndcg_at_20_diff1": 0.5723644631319211, + "nauc_ndcg_at_20_max": 0.3705327377456405, + "nauc_ndcg_at_20_std": 0.11326390907853207, + "nauc_ndcg_at_3_diff1": 0.6045190422968498, + "nauc_ndcg_at_3_max": 0.36744809126982, + "nauc_ndcg_at_3_std": 0.07129906457747395, + "nauc_ndcg_at_5_diff1": 0.5805449517766387, + "nauc_ndcg_at_5_max": 0.3714042780033098, + "nauc_ndcg_at_5_std": 0.07929308964235518, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.4548905095902432, + "nauc_precision_at_100_max": 0.44644713522120666, + "nauc_precision_at_100_std": 0.35804392058592793, + "nauc_precision_at_10_diff1": 0.4959708344082088, + "nauc_precision_at_10_max": 0.32782195690308946, + "nauc_precision_at_10_std": 0.11774784435354392, + "nauc_precision_at_1_diff1": 0.6688306834003572, + "nauc_precision_at_1_max": 0.41410112769511687, + "nauc_precision_at_1_std": 0.06822755485394784, + "nauc_precision_at_20_diff1": 0.49337008131220894, + "nauc_precision_at_20_max": 0.34932704658230473, + "nauc_precision_at_20_std": 0.21358675720665715, + "nauc_precision_at_3_diff1": 0.5676433428285876, + "nauc_precision_at_3_max": 0.3374652810659113, + "nauc_precision_at_3_std": 0.07763926424380893, + "nauc_precision_at_5_diff1": 0.5133003027708118, + "nauc_precision_at_5_max": 0.3473703885687821, + "nauc_precision_at_5_std": 0.0954409870019865, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4548905095902432, + "nauc_recall_at_100_max": 0.44644713522120655, + "nauc_recall_at_100_std": 0.35804392058592754, + "nauc_recall_at_10_diff1": 0.4959708344082086, + "nauc_recall_at_10_max": 0.32782195690308946, + "nauc_recall_at_10_std": 0.11774784435354384, + "nauc_recall_at_1_diff1": 0.6688306834003572, + "nauc_recall_at_1_max": 0.41410112769511687, + "nauc_recall_at_1_std": 0.06822755485394784, + "nauc_recall_at_20_diff1": 0.4933700813122085, + "nauc_recall_at_20_max": 0.3493270465823052, + "nauc_recall_at_20_std": 0.21358675720665704, + "nauc_recall_at_3_diff1": 0.5676433428285875, + "nauc_recall_at_3_max": 0.33746528106591145, + "nauc_recall_at_3_std": 0.07763926424380885, + "nauc_recall_at_5_diff1": 0.513300302770812, + "nauc_recall_at_5_max": 0.3473703885687822, + "nauc_recall_at_5_std": 0.09544098700198667, + "ndcg_at_1": 0.22426, + "ndcg_at_10": 0.30725, + "ndcg_at_100": 0.37817, + "ndcg_at_1000": 0.40869, + "ndcg_at_20": 0.32945, + "ndcg_at_3": 0.27102, + "ndcg_at_5": 0.2851, + "precision_at_1": 0.22426, + "precision_at_10": 0.04073, + "precision_at_100": 0.00761, + "precision_at_1000": 0.001, + "precision_at_20": 0.02477, + "precision_at_3": 0.10183, + "precision_at_5": 0.06796, + "recall_at_1": 0.22426, + "recall_at_10": 0.40732, + "recall_at_100": 0.76087, + "recall_at_1000": 1.0, + "recall_at_20": 0.49542, + "recall_at_3": 0.30549, + "recall_at_5": 0.33982 + }, + { + "cv_recall_at_1": 0.52497, + "cv_recall_at_10": 0.74584, + "cv_recall_at_100": 0.93452, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.81354, + "cv_recall_at_3": 0.63596, + "cv_recall_at_5": 0.68923, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.6295, + "map_at_1": 0.52497, + "map_at_10": 0.5929, + "map_at_100": 0.60088, + "map_at_1000": 0.60123, + "map_at_20": 0.59776, + "map_at_3": 0.57307, + "map_at_5": 0.58555, + "mrr_at_1": 0.5249722530521642, + "mrr_at_10": 0.5929033877702022, + "mrr_at_100": 0.6008830842741953, + "mrr_at_1000": 0.6012302314360111, + "mrr_at_20": 0.5977596078367745, + "mrr_at_3": 0.57306696263411, + "mrr_at_5": 0.5855530891601919, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6190269259232122, + "nauc_cv_recall_at_100_max": 0.7729281092845987, + "nauc_cv_recall_at_100_std": 0.6722070920847195, + "nauc_cv_recall_at_10_diff1": 0.6844620596166869, + "nauc_cv_recall_at_10_max": 0.4891266530015138, + "nauc_cv_recall_at_10_std": 0.3756587095100909, + "nauc_cv_recall_at_1_diff1": 0.8005694894014752, + "nauc_cv_recall_at_1_max": 0.4531124922527121, + "nauc_cv_recall_at_1_std": 0.16807685458452326, + "nauc_cv_recall_at_20_diff1": 0.6384051703535973, + "nauc_cv_recall_at_20_max": 0.5364946891885772, + "nauc_cv_recall_at_20_std": 0.45661774023473733, + "nauc_cv_recall_at_3_diff1": 0.7305967415604132, + "nauc_cv_recall_at_3_max": 0.4380960886436962, + "nauc_cv_recall_at_3_std": 0.25395868506657593, + "nauc_cv_recall_at_5_diff1": 0.7066215095307017, + "nauc_cv_recall_at_5_max": 0.4403031130193901, + "nauc_cv_recall_at_5_std": 0.28131628962301375, + "nauc_map_at_1000_diff1": 0.764661351219403, + "nauc_map_at_1000_max": 0.45355809867633967, + "nauc_map_at_1000_std": 0.2138472959642722, + "nauc_map_at_100_diff1": 0.7644678174458901, + "nauc_map_at_100_max": 0.4538588215763582, + "nauc_map_at_100_std": 0.21434823226077818, + "nauc_map_at_10_diff1": 0.7644185915083609, + "nauc_map_at_10_max": 0.45132329370285257, + "nauc_map_at_10_std": 0.2138793153043489, + "nauc_map_at_1_diff1": 0.8005694894014752, + "nauc_map_at_1_max": 0.4531124922527121, + "nauc_map_at_1_std": 0.16807685458452326, + "nauc_map_at_20_diff1": 0.7636421438964389, + "nauc_map_at_20_max": 0.45293445118961567, + "nauc_map_at_20_std": 0.21543100239398125, + "nauc_map_at_3_diff1": 0.7693383442376206, + "nauc_map_at_3_max": 0.4457495815844052, + "nauc_map_at_3_std": 0.20155794353526787, + "nauc_map_at_5_diff1": 0.7656408863186792, + "nauc_map_at_5_max": 0.44678876509561827, + "nauc_map_at_5_std": 0.20691973647903933, + "nauc_mrr_at_1000_diff1": 0.7646627667860468, + "nauc_mrr_at_1000_max": 0.4535564850362927, + "nauc_mrr_at_1000_std": 0.2138451798583905, + "nauc_mrr_at_100_diff1": 0.7644692275214324, + "nauc_mrr_at_100_max": 0.45385724414089046, + "nauc_mrr_at_100_std": 0.21434611237766832, + "nauc_mrr_at_10_diff1": 0.7644185915083609, + "nauc_mrr_at_10_max": 0.45132329370285257, + "nauc_mrr_at_10_std": 0.2138793153043489, + "nauc_mrr_at_1_diff1": 0.8005694894014752, + "nauc_mrr_at_1_max": 0.4531124922527121, + "nauc_mrr_at_1_std": 0.16807685458452326, + "nauc_mrr_at_20_diff1": 0.7636421438964389, + "nauc_mrr_at_20_max": 0.45293445118961567, + "nauc_mrr_at_20_std": 0.21543100239398125, + "nauc_mrr_at_3_diff1": 0.7693383442376206, + "nauc_mrr_at_3_max": 0.4457495815844052, + "nauc_mrr_at_3_std": 0.20155794353526787, + "nauc_mrr_at_5_diff1": 0.7656408863186792, + "nauc_mrr_at_5_max": 0.44678876509561827, + "nauc_mrr_at_5_std": 0.20691973647903933, + "nauc_ndcg_at_1000_diff1": 0.7562876547851568, + "nauc_ndcg_at_1000_max": 0.4600863415818717, + "nauc_ndcg_at_1000_std": 0.23017546787360782, + "nauc_ndcg_at_100_diff1": 0.7515996116803855, + "nauc_ndcg_at_100_max": 0.47008899439581425, + "nauc_ndcg_at_100_std": 0.2446746833885747, + "nauc_ndcg_at_10_diff1": 0.7488619888294695, + "nauc_ndcg_at_10_max": 0.4574503111316088, + "nauc_ndcg_at_10_std": 0.2436844400834987, + "nauc_ndcg_at_1_diff1": 0.8005694894014752, + "nauc_ndcg_at_1_max": 0.4531124922527121, + "nauc_ndcg_at_1_std": 0.16807685458452326, + "nauc_ndcg_at_20_diff1": 0.7446604086844123, + "nauc_ndcg_at_20_max": 0.4637857510751772, + "nauc_ndcg_at_20_std": 0.2507918044976272, + "nauc_ndcg_at_3_diff1": 0.7599025299332235, + "nauc_ndcg_at_3_max": 0.4438548334441876, + "nauc_ndcg_at_3_std": 0.21409525051528827, + "nauc_ndcg_at_5_diff1": 0.7525017128637891, + "nauc_ndcg_at_5_max": 0.44529280527499515, + "nauc_ndcg_at_5_std": 0.223485373476237, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.6190269259232114, + "nauc_precision_at_100_max": 0.7729281092845983, + "nauc_precision_at_100_std": 0.6722070920847233, + "nauc_precision_at_10_diff1": 0.6844620596166876, + "nauc_precision_at_10_max": 0.48912665300151364, + "nauc_precision_at_10_std": 0.37565870951009117, + "nauc_precision_at_1_diff1": 0.8005694894014752, + "nauc_precision_at_1_max": 0.4531124922527121, + "nauc_precision_at_1_std": 0.16807685458452326, + "nauc_precision_at_20_diff1": 0.6384051703535972, + "nauc_precision_at_20_max": 0.536494689188577, + "nauc_precision_at_20_std": 0.4566177402347358, + "nauc_precision_at_3_diff1": 0.7305967415604135, + "nauc_precision_at_3_max": 0.43809608864369626, + "nauc_precision_at_3_std": 0.2539586850665756, + "nauc_precision_at_5_diff1": 0.7066215095307022, + "nauc_precision_at_5_max": 0.44030311301938957, + "nauc_precision_at_5_std": 0.2813162896230139, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.6190269259232122, + "nauc_recall_at_100_max": 0.7729281092845987, + "nauc_recall_at_100_std": 0.6722070920847195, + "nauc_recall_at_10_diff1": 0.6844620596166869, + "nauc_recall_at_10_max": 0.4891266530015138, + "nauc_recall_at_10_std": 0.3756587095100909, + "nauc_recall_at_1_diff1": 0.8005694894014752, + "nauc_recall_at_1_max": 0.4531124922527121, + "nauc_recall_at_1_std": 0.16807685458452326, + "nauc_recall_at_20_diff1": 0.6384051703535973, + "nauc_recall_at_20_max": 0.5364946891885772, + "nauc_recall_at_20_std": 0.45661774023473733, + "nauc_recall_at_3_diff1": 0.7305967415604132, + "nauc_recall_at_3_max": 0.4380960886436962, + "nauc_recall_at_3_std": 0.25395868506657593, + "nauc_recall_at_5_diff1": 0.7066215095307017, + "nauc_recall_at_5_max": 0.4403031130193901, + "nauc_recall_at_5_std": 0.28131628962301375, + "ndcg_at_1": 0.52497, + "ndcg_at_10": 0.6295, + "ndcg_at_100": 0.66932, + "ndcg_at_1000": 0.67783, + "ndcg_at_20": 0.64681, + "ndcg_at_3": 0.58919, + "ndcg_at_5": 0.6114, + "precision_at_1": 0.52497, + "precision_at_10": 0.07458, + "precision_at_100": 0.00935, + "precision_at_1000": 0.001, + "precision_at_20": 0.04068, + "precision_at_3": 0.21199, + "precision_at_5": 0.13785, + "recall_at_1": 0.52497, + "recall_at_10": 0.74584, + "recall_at_100": 0.93452, + "recall_at_1000": 1.0, + "recall_at_20": 0.81354, + "recall_at_3": 0.63596, + "recall_at_5": 0.68923 + }, + { + "cv_recall_at_1": 0.15575, + "cv_recall_at_10": 0.31794, + "cv_recall_at_100": 0.63373, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.40494, + "cv_recall_at_3": 0.19871, + "cv_recall_at_5": 0.25027, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.22424, + "map_at_1": 0.15575, + "map_at_10": 0.19581, + "map_at_100": 0.20715, + "map_at_1000": 0.20863, + "map_at_20": 0.20175, + "map_at_3": 0.1749, + "map_at_5": 0.18666, + "mrr_at_1": 0.15574650912996776, + "mrr_at_10": 0.19580967725436035, + "mrr_at_100": 0.20715421138921317, + "mrr_at_1000": 0.20863345075667283, + "mrr_at_20": 0.20175038883712101, + "mrr_at_3": 0.17490153956319368, + "mrr_at_5": 0.18666308628714642, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.24444918782474567, + "nauc_cv_recall_at_100_max": 0.32551466197697165, + "nauc_cv_recall_at_100_std": 0.45167668398297006, + "nauc_cv_recall_at_10_diff1": 0.39248291964911963, + "nauc_cv_recall_at_10_max": 0.3066812417261699, + "nauc_cv_recall_at_10_std": 0.3424873919857647, + "nauc_cv_recall_at_1_diff1": 0.6532550450478714, + "nauc_cv_recall_at_1_max": 0.44239290311087814, + "nauc_cv_recall_at_1_std": 0.32573808089156275, + "nauc_cv_recall_at_20_diff1": 0.36388396999886186, + "nauc_cv_recall_at_20_max": 0.29934288856164354, + "nauc_cv_recall_at_20_std": 0.3696664843365718, + "nauc_cv_recall_at_3_diff1": 0.5069636472753279, + "nauc_cv_recall_at_3_max": 0.35407303170096777, + "nauc_cv_recall_at_3_std": 0.2986517151004437, + "nauc_cv_recall_at_5_diff1": 0.47172961423842896, + "nauc_cv_recall_at_5_max": 0.33375077016623395, + "nauc_cv_recall_at_5_std": 0.31794011506865144, + "nauc_map_at_1000_diff1": 0.5508520944278331, + "nauc_map_at_1000_max": 0.38565983075291094, + "nauc_map_at_1000_std": 0.3261751242369309, + "nauc_map_at_100_diff1": 0.5496666524189222, + "nauc_map_at_100_max": 0.3853271693919615, + "nauc_map_at_100_std": 0.3265286270672321, + "nauc_map_at_10_diff1": 0.5551643822996104, + "nauc_map_at_10_max": 0.3859397236650562, + "nauc_map_at_10_std": 0.32287469193293833, + "nauc_map_at_1_diff1": 0.6532550450478714, + "nauc_map_at_1_max": 0.44239290311087814, + "nauc_map_at_1_std": 0.32573808089156275, + "nauc_map_at_20_diff1": 0.5516571306533536, + "nauc_map_at_20_max": 0.38486781606814013, + "nauc_map_at_20_std": 0.32480713401957406, + "nauc_map_at_3_diff1": 0.5815687647615574, + "nauc_map_at_3_max": 0.39889133439982516, + "nauc_map_at_3_std": 0.3144923960506113, + "nauc_map_at_5_diff1": 0.5711653159966787, + "nauc_map_at_5_max": 0.3922705457437804, + "nauc_map_at_5_std": 0.3189425954414773, + "nauc_mrr_at_1000_diff1": 0.5508520944278331, + "nauc_mrr_at_1000_max": 0.38565983075291094, + "nauc_mrr_at_1000_std": 0.3261751242369309, + "nauc_mrr_at_100_diff1": 0.5496666524189222, + "nauc_mrr_at_100_max": 0.3853271693919615, + "nauc_mrr_at_100_std": 0.3265286270672321, + "nauc_mrr_at_10_diff1": 0.5551643822996104, + "nauc_mrr_at_10_max": 0.3859397236650562, + "nauc_mrr_at_10_std": 0.32287469193293833, + "nauc_mrr_at_1_diff1": 0.6532550450478714, + "nauc_mrr_at_1_max": 0.44239290311087814, + "nauc_mrr_at_1_std": 0.32573808089156275, + "nauc_mrr_at_20_diff1": 0.5516571306533536, + "nauc_mrr_at_20_max": 0.38486781606814013, + "nauc_mrr_at_20_std": 0.32480713401957406, + "nauc_mrr_at_3_diff1": 0.5815687647615574, + "nauc_mrr_at_3_max": 0.39889133439982516, + "nauc_mrr_at_3_std": 0.3144923960506113, + "nauc_mrr_at_5_diff1": 0.5711653159966787, + "nauc_mrr_at_5_max": 0.3922705457437804, + "nauc_mrr_at_5_std": 0.3189425954414773, + "nauc_ndcg_at_1000_diff1": 0.5157264617450563, + "nauc_ndcg_at_1000_max": 0.37274339990632316, + "nauc_ndcg_at_1000_std": 0.3355623933659139, + "nauc_ndcg_at_100_diff1": 0.48148009490671284, + "nauc_ndcg_at_100_max": 0.3661030696417765, + "nauc_ndcg_at_100_std": 0.3493858547937965, + "nauc_ndcg_at_10_diff1": 0.5088785336848107, + "nauc_ndcg_at_10_max": 0.36287802189034524, + "nauc_ndcg_at_10_std": 0.3276751318762903, + "nauc_ndcg_at_1_diff1": 0.6532550450478714, + "nauc_ndcg_at_1_max": 0.44239290311087814, + "nauc_ndcg_at_1_std": 0.32573808089156275, + "nauc_ndcg_at_20_diff1": 0.4982450502542284, + "nauc_ndcg_at_20_max": 0.35978126250426545, + "nauc_ndcg_at_20_std": 0.33557435140627284, + "nauc_ndcg_at_3_diff1": 0.560933086731898, + "nauc_ndcg_at_3_max": 0.38648542016608384, + "nauc_ndcg_at_3_std": 0.31019490114977205, + "nauc_ndcg_at_5_diff1": 0.5429858311638133, + "nauc_ndcg_at_5_max": 0.3755963165912826, + "nauc_ndcg_at_5_std": 0.3183859719256547, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.244449187824746, + "nauc_precision_at_100_max": 0.3255146619769719, + "nauc_precision_at_100_std": 0.45167668398296984, + "nauc_precision_at_10_diff1": 0.3924829196491193, + "nauc_precision_at_10_max": 0.30668124172616984, + "nauc_precision_at_10_std": 0.3424873919857649, + "nauc_precision_at_1_diff1": 0.6532550450478714, + "nauc_precision_at_1_max": 0.44239290311087814, + "nauc_precision_at_1_std": 0.32573808089156275, + "nauc_precision_at_20_diff1": 0.36388396999886186, + "nauc_precision_at_20_max": 0.29934288856164326, + "nauc_precision_at_20_std": 0.3696664843365716, + "nauc_precision_at_3_diff1": 0.5069636472753278, + "nauc_precision_at_3_max": 0.35407303170096777, + "nauc_precision_at_3_std": 0.29865171510044364, + "nauc_precision_at_5_diff1": 0.47172961423842874, + "nauc_precision_at_5_max": 0.33375077016623395, + "nauc_precision_at_5_std": 0.31794011506865155, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.24444918782474567, + "nauc_recall_at_100_max": 0.32551466197697165, + "nauc_recall_at_100_std": 0.45167668398297006, + "nauc_recall_at_10_diff1": 0.39248291964911963, + "nauc_recall_at_10_max": 0.3066812417261699, + "nauc_recall_at_10_std": 0.3424873919857647, + "nauc_recall_at_1_diff1": 0.6532550450478714, + "nauc_recall_at_1_max": 0.44239290311087814, + "nauc_recall_at_1_std": 0.32573808089156275, + "nauc_recall_at_20_diff1": 0.36388396999886186, + "nauc_recall_at_20_max": 0.29934288856164354, + "nauc_recall_at_20_std": 0.3696664843365718, + "nauc_recall_at_3_diff1": 0.5069636472753279, + "nauc_recall_at_3_max": 0.35407303170096777, + "nauc_recall_at_3_std": 0.2986517151004437, + "nauc_recall_at_5_diff1": 0.47172961423842896, + "nauc_recall_at_5_max": 0.33375077016623395, + "nauc_recall_at_5_std": 0.31794011506865144, + "ndcg_at_1": 0.15575, + "ndcg_at_10": 0.22424, + "ndcg_at_100": 0.28762, + "ndcg_at_1000": 0.33285, + "ndcg_at_20": 0.24613, + "ndcg_at_3": 0.18103, + "ndcg_at_5": 0.20224, + "precision_at_1": 0.15575, + "precision_at_10": 0.03179, + "precision_at_100": 0.00634, + "precision_at_1000": 0.001, + "precision_at_20": 0.02025, + "precision_at_3": 0.06624, + "precision_at_5": 0.05005, + "recall_at_1": 0.15575, + "recall_at_10": 0.31794, + "recall_at_100": 0.63373, + "recall_at_1000": 1.0, + "recall_at_20": 0.40494, + "recall_at_3": 0.19871, + "recall_at_5": 0.25027 + }, + { + "cv_recall_at_1": 0.133, + "cv_recall_at_10": 0.348, + "cv_recall_at_100": 0.681, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.432, + "cv_recall_at_3": 0.219, + "cv_recall_at_5": 0.272, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.2298, + "map_at_1": 0.133, + "map_at_10": 0.19332, + "map_at_100": 0.20463, + "map_at_1000": 0.20609, + "map_at_20": 0.19876, + "map_at_3": 0.171, + "map_at_5": 0.18315, + "mrr_at_1": 0.133, + "mrr_at_10": 0.19332341269841272, + "mrr_at_100": 0.20462787375291316, + "mrr_at_1000": 0.20608662727509244, + "mrr_at_20": 0.1987633009620239, + "mrr_at_3": 0.17100000000000004, + "mrr_at_5": 0.18315, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.23903970030450739, + "nauc_cv_recall_at_100_max": 0.41640569684568446, + "nauc_cv_recall_at_100_std": 0.39736831627291774, + "nauc_cv_recall_at_10_diff1": 0.37136341320494715, + "nauc_cv_recall_at_10_max": 0.18230049253282243, + "nauc_cv_recall_at_10_std": 0.04904476083387433, + "nauc_cv_recall_at_1_diff1": 0.5180319349233996, + "nauc_cv_recall_at_1_max": 0.1270218074435545, + "nauc_cv_recall_at_1_std": -0.07816591097771257, + "nauc_cv_recall_at_20_diff1": 0.36617498542395466, + "nauc_cv_recall_at_20_max": 0.22647790687989328, + "nauc_cv_recall_at_20_std": 0.10219499625187434, + "nauc_cv_recall_at_3_diff1": 0.42927422380603214, + "nauc_cv_recall_at_3_max": 0.1421225439141381, + "nauc_cv_recall_at_3_std": -0.020967317575151773, + "nauc_cv_recall_at_5_diff1": 0.392570337984777, + "nauc_cv_recall_at_5_max": 0.1601208544762595, + "nauc_cv_recall_at_5_std": 0.008884276458861794, + "nauc_map_at_1000_diff1": 0.44984391381046307, + "nauc_map_at_1000_max": 0.15054300211478264, + "nauc_map_at_1000_std": -0.021052809041627263, + "nauc_map_at_100_diff1": 0.44918790083327276, + "nauc_map_at_100_max": 0.1510241417560371, + "nauc_map_at_100_std": -0.019927498393880156, + "nauc_map_at_10_diff1": 0.45343156466916373, + "nauc_map_at_10_max": 0.1428449015931523, + "nauc_map_at_10_std": -0.031102817230918154, + "nauc_map_at_1_diff1": 0.5180319349233996, + "nauc_map_at_1_max": 0.1270218074435545, + "nauc_map_at_1_std": -0.07816591097771257, + "nauc_map_at_20_diff1": 0.4524836567452582, + "nauc_map_at_20_max": 0.1468680942413539, + "nauc_map_at_20_std": -0.026211756253432503, + "nauc_map_at_3_diff1": 0.4698730760278111, + "nauc_map_at_3_max": 0.13492394255234672, + "nauc_map_at_3_std": -0.04706689363194477, + "nauc_map_at_5_diff1": 0.4585585830085684, + "nauc_map_at_5_max": 0.13931591541765556, + "nauc_map_at_5_std": -0.038330787238466967, + "nauc_mrr_at_1000_diff1": 0.44984391381046307, + "nauc_mrr_at_1000_max": 0.15054300211478264, + "nauc_mrr_at_1000_std": -0.021052809041627263, + "nauc_mrr_at_100_diff1": 0.44918790083327276, + "nauc_mrr_at_100_max": 0.1510241417560371, + "nauc_mrr_at_100_std": -0.019927498393880156, + "nauc_mrr_at_10_diff1": 0.45343156466916373, + "nauc_mrr_at_10_max": 0.1428449015931523, + "nauc_mrr_at_10_std": -0.031102817230918154, + "nauc_mrr_at_1_diff1": 0.5180319349233996, + "nauc_mrr_at_1_max": 0.1270218074435545, + "nauc_mrr_at_1_std": -0.07816591097771257, + "nauc_mrr_at_20_diff1": 0.4524836567452582, + "nauc_mrr_at_20_max": 0.1468680942413539, + "nauc_mrr_at_20_std": -0.026211756253432503, + "nauc_mrr_at_3_diff1": 0.4698730760278111, + "nauc_mrr_at_3_max": 0.13492394255234672, + "nauc_mrr_at_3_std": -0.04706689363194477, + "nauc_mrr_at_5_diff1": 0.4585585830085684, + "nauc_mrr_at_5_max": 0.13931591541765556, + "nauc_mrr_at_5_std": -0.038330787238466967, + "nauc_ndcg_at_1000_diff1": 0.42734696398847494, + "nauc_ndcg_at_1000_max": 0.17217833907343263, + "nauc_ndcg_at_1000_std": 0.012881439336521373, + "nauc_ndcg_at_100_diff1": 0.4053394775042063, + "nauc_ndcg_at_100_max": 0.19895583284204096, + "nauc_ndcg_at_100_std": 0.056812856244403374, + "nauc_ndcg_at_10_diff1": 0.4288409161311715, + "nauc_ndcg_at_10_max": 0.15389774771982148, + "nauc_ndcg_at_10_std": -0.00797049243824064, + "nauc_ndcg_at_1_diff1": 0.5180319349233996, + "nauc_ndcg_at_1_max": 0.1270218074435545, + "nauc_ndcg_at_1_std": -0.07816591097771257, + "nauc_ndcg_at_20_diff1": 0.426561870160365, + "nauc_ndcg_at_20_max": 0.16776057042499307, + "nauc_ndcg_at_20_std": 0.00882600452766406, + "nauc_ndcg_at_3_diff1": 0.4579909913635628, + "nauc_ndcg_at_3_max": 0.1370186461871269, + "nauc_ndcg_at_3_std": -0.0394256678182846, + "nauc_ndcg_at_5_diff1": 0.43906112535844644, + "nauc_ndcg_at_5_max": 0.14522893418960348, + "nauc_ndcg_at_5_std": -0.02448000303769126, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.2390397003045082, + "nauc_precision_at_100_max": 0.41640569684568407, + "nauc_precision_at_100_std": 0.39736831627291763, + "nauc_precision_at_10_diff1": 0.37136341320494715, + "nauc_precision_at_10_max": 0.18230049253282227, + "nauc_precision_at_10_std": 0.04904476083387422, + "nauc_precision_at_1_diff1": 0.5180319349233996, + "nauc_precision_at_1_max": 0.1270218074435545, + "nauc_precision_at_1_std": -0.07816591097771257, + "nauc_precision_at_20_diff1": 0.3661749854239545, + "nauc_precision_at_20_max": 0.2264779068798933, + "nauc_precision_at_20_std": 0.10219499625187378, + "nauc_precision_at_3_diff1": 0.42927422380603214, + "nauc_precision_at_3_max": 0.14212254391413814, + "nauc_precision_at_3_std": -0.020967317575151828, + "nauc_precision_at_5_diff1": 0.3925703379847769, + "nauc_precision_at_5_max": 0.16012085447625948, + "nauc_precision_at_5_std": 0.008884276458861773, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.23903970030450739, + "nauc_recall_at_100_max": 0.41640569684568446, + "nauc_recall_at_100_std": 0.39736831627291774, + "nauc_recall_at_10_diff1": 0.37136341320494715, + "nauc_recall_at_10_max": 0.18230049253282243, + "nauc_recall_at_10_std": 0.04904476083387433, + "nauc_recall_at_1_diff1": 0.5180319349233996, + "nauc_recall_at_1_max": 0.1270218074435545, + "nauc_recall_at_1_std": -0.07816591097771257, + "nauc_recall_at_20_diff1": 0.36617498542395466, + "nauc_recall_at_20_max": 0.22647790687989328, + "nauc_recall_at_20_std": 0.10219499625187434, + "nauc_recall_at_3_diff1": 0.42927422380603214, + "nauc_recall_at_3_max": 0.1421225439141381, + "nauc_recall_at_3_std": -0.020967317575151773, + "nauc_recall_at_5_diff1": 0.392570337984777, + "nauc_recall_at_5_max": 0.1601208544762595, + "nauc_recall_at_5_std": 0.008884276458861794, + "ndcg_at_1": 0.133, + "ndcg_at_10": 0.2298, + "ndcg_at_100": 0.29574, + "ndcg_at_1000": 0.33603, + "ndcg_at_20": 0.25055, + "ndcg_at_3": 0.18333, + "ndcg_at_5": 0.20519, + "precision_at_1": 0.133, + "precision_at_10": 0.0348, + "precision_at_100": 0.00681, + "precision_at_1000": 0.001, + "precision_at_20": 0.0216, + "precision_at_3": 0.073, + "precision_at_5": 0.0544, + "recall_at_1": 0.133, + "recall_at_10": 0.348, + "recall_at_100": 0.681, + "recall_at_1000": 1.0, + "recall_at_20": 0.432, + "recall_at_3": 0.219, + "recall_at_5": 0.272 + }, + { + "cv_recall_at_1": 0.4258, + "cv_recall_at_10": 0.68239, + "cv_recall_at_100": 0.89182, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.74064, + "cv_recall_at_3": 0.55062, + "cv_recall_at_5": 0.61026, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.54674, + "map_at_1": 0.4258, + "map_at_10": 0.50431, + "map_at_100": 0.51226, + "map_at_1000": 0.51286, + "map_at_20": 0.50837, + "map_at_3": 0.48151, + "map_at_5": 0.49496, + "mrr_at_1": 0.42579750346740636, + "mrr_at_10": 0.5043056381128503, + "mrr_at_100": 0.5122619082917252, + "mrr_at_1000": 0.5128598359955646, + "mrr_at_20": 0.5083677746206783, + "mrr_at_3": 0.48150716597318527, + "mrr_at_5": 0.49496070272769277, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6123550386274283, + "nauc_cv_recall_at_100_max": 0.5478399377012312, + "nauc_cv_recall_at_100_std": 0.496080100518916, + "nauc_cv_recall_at_10_diff1": 0.6142929793515284, + "nauc_cv_recall_at_10_max": 0.5190468391127735, + "nauc_cv_recall_at_10_std": 0.385308604962248, + "nauc_cv_recall_at_1_diff1": 0.7747521987597996, + "nauc_cv_recall_at_1_max": 0.5173781139473123, + "nauc_cv_recall_at_1_std": 0.13473303740040443, + "nauc_cv_recall_at_20_diff1": 0.6336034724432345, + "nauc_cv_recall_at_20_max": 0.5115030632455113, + "nauc_cv_recall_at_20_std": 0.39928737908127715, + "nauc_cv_recall_at_3_diff1": 0.6797923699658351, + "nauc_cv_recall_at_3_max": 0.5177177240768384, + "nauc_cv_recall_at_3_std": 0.2711052415459016, + "nauc_cv_recall_at_5_diff1": 0.6492496293194232, + "nauc_cv_recall_at_5_max": 0.495768199463953, + "nauc_cv_recall_at_5_std": 0.27675036787034907, + "nauc_map_at_1000_diff1": 0.728333770648744, + "nauc_map_at_1000_max": 0.516118056668068, + "nauc_map_at_1000_std": 0.2016134228119069, + "nauc_map_at_100_diff1": 0.7281529278617421, + "nauc_map_at_100_max": 0.5161391068643536, + "nauc_map_at_100_std": 0.20210605477046603, + "nauc_map_at_10_diff1": 0.7261188851666448, + "nauc_map_at_10_max": 0.5173169342754899, + "nauc_map_at_10_std": 0.20479249871993216, + "nauc_map_at_1_diff1": 0.7747521987597996, + "nauc_map_at_1_max": 0.5173781139473123, + "nauc_map_at_1_std": 0.13473303740040443, + "nauc_map_at_20_diff1": 0.7276632804464694, + "nauc_map_at_20_max": 0.5166384481658993, + "nauc_map_at_20_std": 0.2037465157952008, + "nauc_map_at_3_diff1": 0.7331507665544107, + "nauc_map_at_3_max": 0.5185691526030635, + "nauc_map_at_3_std": 0.19450591863838512, + "nauc_map_at_5_diff1": 0.7282450588763438, + "nauc_map_at_5_max": 0.5139642537867429, + "nauc_map_at_5_std": 0.19442892713879853, + "nauc_mrr_at_1000_diff1": 0.7283336738950129, + "nauc_mrr_at_1000_max": 0.5161182608501237, + "nauc_mrr_at_1000_std": 0.20161372824442178, + "nauc_mrr_at_100_diff1": 0.7281528313373693, + "nauc_mrr_at_100_max": 0.5161393106782692, + "nauc_mrr_at_100_std": 0.20210635950408978, + "nauc_mrr_at_10_diff1": 0.7261188851666448, + "nauc_mrr_at_10_max": 0.5173169342754899, + "nauc_mrr_at_10_std": 0.20479249871993216, + "nauc_mrr_at_1_diff1": 0.7747521987597996, + "nauc_mrr_at_1_max": 0.5173781139473123, + "nauc_mrr_at_1_std": 0.13473303740040443, + "nauc_mrr_at_20_diff1": 0.7276632804464694, + "nauc_mrr_at_20_max": 0.5166384481658993, + "nauc_mrr_at_20_std": 0.2037465157952008, + "nauc_mrr_at_3_diff1": 0.7331507665544107, + "nauc_mrr_at_3_max": 0.5185691526030635, + "nauc_mrr_at_3_std": 0.19450591863838512, + "nauc_mrr_at_5_diff1": 0.7282450588763438, + "nauc_mrr_at_5_max": 0.5139642537867429, + "nauc_mrr_at_5_std": 0.19442892713879853, + "nauc_ndcg_at_1000_diff1": 0.7183362744659123, + "nauc_ndcg_at_1000_max": 0.5151142335186784, + "nauc_ndcg_at_1000_std": 0.2178540584968831, + "nauc_ndcg_at_100_diff1": 0.7132894716769472, + "nauc_ndcg_at_100_max": 0.5164316539030083, + "nauc_ndcg_at_100_std": 0.23117635336299322, + "nauc_ndcg_at_10_diff1": 0.7032515997687422, + "nauc_ndcg_at_10_max": 0.5174827984990962, + "nauc_ndcg_at_10_std": 0.24096931371599054, + "nauc_ndcg_at_1_diff1": 0.7747521987597996, + "nauc_ndcg_at_1_max": 0.5173781139473123, + "nauc_ndcg_at_1_std": 0.13473303740040443, + "nauc_ndcg_at_20_diff1": 0.7092928733514847, + "nauc_ndcg_at_20_max": 0.5156346782452621, + "nauc_ndcg_at_20_std": 0.23866367889009282, + "nauc_ndcg_at_3_diff1": 0.719853330258983, + "nauc_ndcg_at_3_max": 0.5184029128753416, + "nauc_ndcg_at_3_std": 0.2135964955541178, + "nauc_ndcg_at_5_diff1": 0.7098897269058249, + "nauc_ndcg_at_5_max": 0.5099870789610721, + "nauc_ndcg_at_5_std": 0.21404335193633725, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.6123550386274293, + "nauc_precision_at_100_max": 0.5478399377012316, + "nauc_precision_at_100_std": 0.49608010051891843, + "nauc_precision_at_10_diff1": 0.6142929793515284, + "nauc_precision_at_10_max": 0.5190468391127726, + "nauc_precision_at_10_std": 0.38530860496224717, + "nauc_precision_at_1_diff1": 0.7747521987597996, + "nauc_precision_at_1_max": 0.5173781139473123, + "nauc_precision_at_1_std": 0.13473303740040443, + "nauc_precision_at_20_diff1": 0.6336034724432347, + "nauc_precision_at_20_max": 0.5115030632455123, + "nauc_precision_at_20_std": 0.39928737908127737, + "nauc_precision_at_3_diff1": 0.6797923699658351, + "nauc_precision_at_3_max": 0.5177177240768378, + "nauc_precision_at_3_std": 0.2711052415459004, + "nauc_precision_at_5_diff1": 0.6492496293194232, + "nauc_precision_at_5_max": 0.4957681994639534, + "nauc_precision_at_5_std": 0.276750367870349, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.6123550386274283, + "nauc_recall_at_100_max": 0.5478399377012312, + "nauc_recall_at_100_std": 0.496080100518916, + "nauc_recall_at_10_diff1": 0.6142929793515284, + "nauc_recall_at_10_max": 0.5190468391127735, + "nauc_recall_at_10_std": 0.385308604962248, + "nauc_recall_at_1_diff1": 0.7747521987597996, + "nauc_recall_at_1_max": 0.5173781139473123, + "nauc_recall_at_1_std": 0.13473303740040443, + "nauc_recall_at_20_diff1": 0.6336034724432345, + "nauc_recall_at_20_max": 0.5115030632455113, + "nauc_recall_at_20_std": 0.39928737908127715, + "nauc_recall_at_3_diff1": 0.6797923699658351, + "nauc_recall_at_3_max": 0.5177177240768384, + "nauc_recall_at_3_std": 0.2711052415459016, + "nauc_recall_at_5_diff1": 0.6492496293194232, + "nauc_recall_at_5_max": 0.495768199463953, + "nauc_recall_at_5_std": 0.27675036787034907, + "ndcg_at_1": 0.4258, + "ndcg_at_10": 0.54674, + "ndcg_at_100": 0.58958, + "ndcg_at_1000": 0.60381, + "ndcg_at_20": 0.56149, + "ndcg_at_3": 0.49929, + "ndcg_at_5": 0.5237, + "precision_at_1": 0.4258, + "precision_at_10": 0.06824, + "precision_at_100": 0.00892, + "precision_at_1000": 0.001, + "precision_at_20": 0.03703, + "precision_at_3": 0.18354, + "precision_at_5": 0.12205, + "recall_at_1": 0.4258, + "recall_at_10": 0.68239, + "recall_at_100": 0.89182, + "recall_at_1000": 1.0, + "recall_at_20": 0.74064, + "recall_at_3": 0.55062, + "recall_at_5": 0.61026 + }, + { + "cv_recall_at_1": 0.45455, + "cv_recall_at_10": 0.70296, + "cv_recall_at_100": 0.88584, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.7611, + "cv_recall_at_3": 0.57928, + "cv_recall_at_5": 0.63425, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.5731, + "map_at_1": 0.45455, + "map_at_10": 0.53221, + "map_at_100": 0.53943, + "map_at_1000": 0.54, + "map_at_20": 0.53637, + "map_at_3": 0.51039, + "map_at_5": 0.52271, + "mrr_at_1": 0.45454545454545453, + "mrr_at_10": 0.5322116513976977, + "mrr_at_100": 0.5394335226463994, + "mrr_at_1000": 0.540004150062426, + "mrr_at_20": 0.5363658296845691, + "mrr_at_3": 0.5103946441155741, + "mrr_at_5": 0.5227096546863984, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5155748885294896, + "nauc_cv_recall_at_100_max": 0.4354612206487773, + "nauc_cv_recall_at_100_std": 0.5786132641876298, + "nauc_cv_recall_at_10_diff1": 0.6016904587793288, + "nauc_cv_recall_at_10_max": 0.5453527162042908, + "nauc_cv_recall_at_10_std": 0.4474004366146903, + "nauc_cv_recall_at_1_diff1": 0.7397486518013686, + "nauc_cv_recall_at_1_max": 0.5581512468677294, + "nauc_cv_recall_at_1_std": 0.2786415604449159, + "nauc_cv_recall_at_20_diff1": 0.5539394639480063, + "nauc_cv_recall_at_20_max": 0.5329757970859269, + "nauc_cv_recall_at_20_std": 0.5029165107845112, + "nauc_cv_recall_at_3_diff1": 0.6362736404675333, + "nauc_cv_recall_at_3_max": 0.5600825936851035, + "nauc_cv_recall_at_3_std": 0.3562218259043451, + "nauc_cv_recall_at_5_diff1": 0.6134324595869594, + "nauc_cv_recall_at_5_max": 0.5514024998817529, + "nauc_cv_recall_at_5_std": 0.376335323466691, + "nauc_map_at_1000_diff1": 0.6884311413366787, + "nauc_map_at_1000_max": 0.5545252247296255, + "nauc_map_at_1000_std": 0.32258057532335105, + "nauc_map_at_100_diff1": 0.6881264891740738, + "nauc_map_at_100_max": 0.5542651721296694, + "nauc_map_at_100_std": 0.32298399733590283, + "nauc_map_at_10_diff1": 0.6887002858727944, + "nauc_map_at_10_max": 0.555550891059468, + "nauc_map_at_10_std": 0.3216058018523894, + "nauc_map_at_1_diff1": 0.7397486518013686, + "nauc_map_at_1_max": 0.5581512468677294, + "nauc_map_at_1_std": 0.2786415604449159, + "nauc_map_at_20_diff1": 0.6872412205765465, + "nauc_map_at_20_max": 0.5550520779209613, + "nauc_map_at_20_std": 0.323314768524235, + "nauc_map_at_3_diff1": 0.693482051512695, + "nauc_map_at_3_max": 0.5589401535113256, + "nauc_map_at_3_std": 0.31236577566495405, + "nauc_map_at_5_diff1": 0.6895582892296516, + "nauc_map_at_5_max": 0.5565381934854133, + "nauc_map_at_5_std": 0.31533116013401896, + "nauc_mrr_at_1000_diff1": 0.688431220346766, + "nauc_mrr_at_1000_max": 0.5545254930231911, + "nauc_mrr_at_1000_std": 0.3225809480403303, + "nauc_mrr_at_100_diff1": 0.6881267561091581, + "nauc_mrr_at_100_max": 0.554265682415342, + "nauc_mrr_at_100_std": 0.32298470557702164, + "nauc_mrr_at_10_diff1": 0.6887002858727944, + "nauc_mrr_at_10_max": 0.555550891059468, + "nauc_mrr_at_10_std": 0.3216058018523894, + "nauc_mrr_at_1_diff1": 0.7397486518013686, + "nauc_mrr_at_1_max": 0.5581512468677294, + "nauc_mrr_at_1_std": 0.2786415604449159, + "nauc_mrr_at_20_diff1": 0.6872412205765465, + "nauc_mrr_at_20_max": 0.5550520779209613, + "nauc_mrr_at_20_std": 0.323314768524235, + "nauc_mrr_at_3_diff1": 0.693482051512695, + "nauc_mrr_at_3_max": 0.5589401535113256, + "nauc_mrr_at_3_std": 0.31236577566495405, + "nauc_mrr_at_5_diff1": 0.6895582892296516, + "nauc_mrr_at_5_max": 0.5565381934854133, + "nauc_mrr_at_5_std": 0.31533116013401896, + "nauc_ndcg_at_1000_diff1": 0.6771612397254433, + "nauc_ndcg_at_1000_max": 0.5516918879125599, + "nauc_ndcg_at_1000_std": 0.33737823092057195, + "nauc_ndcg_at_100_diff1": 0.66881208513375, + "nauc_ndcg_at_100_max": 0.5453526920357381, + "nauc_ndcg_at_100_std": 0.34944573233573056, + "nauc_ndcg_at_10_diff1": 0.6701885011667525, + "nauc_ndcg_at_10_max": 0.5534554872062589, + "nauc_ndcg_at_10_std": 0.3462041956274489, + "nauc_ndcg_at_1_diff1": 0.7397486518013686, + "nauc_ndcg_at_1_max": 0.5581512468677294, + "nauc_ndcg_at_1_std": 0.2786415604449159, + "nauc_ndcg_at_20_diff1": 0.6639174288133743, + "nauc_ndcg_at_20_max": 0.5514800394725279, + "nauc_ndcg_at_20_std": 0.35304329050488403, + "nauc_ndcg_at_3_diff1": 0.6792793166204122, + "nauc_ndcg_at_3_max": 0.5592203897905106, + "nauc_ndcg_at_3_std": 0.32320983990013075, + "nauc_ndcg_at_5_diff1": 0.671864236588547, + "nauc_ndcg_at_5_max": 0.5553744878364937, + "nauc_ndcg_at_5_std": 0.32935433616936693, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5155748885294904, + "nauc_precision_at_100_max": 0.435461220648779, + "nauc_precision_at_100_std": 0.578613264187631, + "nauc_precision_at_10_diff1": 0.6016904587793293, + "nauc_precision_at_10_max": 0.5453527162042906, + "nauc_precision_at_10_std": 0.44740043661469014, + "nauc_precision_at_1_diff1": 0.7397486518013686, + "nauc_precision_at_1_max": 0.5581512468677294, + "nauc_precision_at_1_std": 0.2786415604449159, + "nauc_precision_at_20_diff1": 0.5539394639480061, + "nauc_precision_at_20_max": 0.5329757970859269, + "nauc_precision_at_20_std": 0.5029165107845114, + "nauc_precision_at_3_diff1": 0.6362736404675334, + "nauc_precision_at_3_max": 0.5600825936851035, + "nauc_precision_at_3_std": 0.35622182590434565, + "nauc_precision_at_5_diff1": 0.6134324595869591, + "nauc_precision_at_5_max": 0.5514024998817525, + "nauc_precision_at_5_std": 0.3763353234666901, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5155748885294896, + "nauc_recall_at_100_max": 0.4354612206487773, + "nauc_recall_at_100_std": 0.5786132641876298, + "nauc_recall_at_10_diff1": 0.6016904587793288, + "nauc_recall_at_10_max": 0.5453527162042908, + "nauc_recall_at_10_std": 0.4474004366146903, + "nauc_recall_at_1_diff1": 0.7397486518013686, + "nauc_recall_at_1_max": 0.5581512468677294, + "nauc_recall_at_1_std": 0.2786415604449159, + "nauc_recall_at_20_diff1": 0.5539394639480063, + "nauc_recall_at_20_max": 0.5329757970859269, + "nauc_recall_at_20_std": 0.5029165107845112, + "nauc_recall_at_3_diff1": 0.6362736404675333, + "nauc_recall_at_3_max": 0.5600825936851035, + "nauc_recall_at_3_std": 0.3562218259043451, + "nauc_recall_at_5_diff1": 0.6134324595869594, + "nauc_recall_at_5_max": 0.5514024998817529, + "nauc_recall_at_5_std": 0.376335323466691, + "ndcg_at_1": 0.45455, + "ndcg_at_10": 0.5731, + "ndcg_at_100": 0.61083, + "ndcg_at_1000": 0.62555, + "ndcg_at_20": 0.58795, + "ndcg_at_3": 0.52812, + "ndcg_at_5": 0.55055, + "precision_at_1": 0.45455, + "precision_at_10": 0.0703, + "precision_at_100": 0.00886, + "precision_at_1000": 0.001, + "precision_at_20": 0.03805, + "precision_at_3": 0.19309, + "precision_at_5": 0.12685, + "recall_at_1": 0.45455, + "recall_at_10": 0.70296, + "recall_at_100": 0.88584, + "recall_at_1000": 1.0, + "recall_at_20": 0.7611, + "recall_at_3": 0.57928, + "recall_at_5": 0.63425 + }, + { + "cv_recall_at_1": 0.399, + "cv_recall_at_10": 0.551, + "cv_recall_at_100": 0.688, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.591, + "cv_recall_at_3": 0.464, + "cv_recall_at_5": 0.501, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.46795, + "map_at_1": 0.399, + "map_at_10": 0.44221, + "map_at_100": 0.44732, + "map_at_1000": 0.44807, + "map_at_20": 0.44478, + "map_at_3": 0.427, + "map_at_5": 0.43555, + "mrr_at_1": 0.399, + "mrr_at_10": 0.4422138888888887, + "mrr_at_100": 0.4473152600871951, + "mrr_at_1000": 0.44807039515532976, + "mrr_at_20": 0.44478111955124294, + "mrr_at_3": 0.4269999999999997, + "mrr_at_5": 0.43554999999999966, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5528063877023381, + "nauc_cv_recall_at_100_max": 0.8748190900404679, + "nauc_cv_recall_at_100_std": 0.2906485358588136, + "nauc_cv_recall_at_10_diff1": 0.7000307851045077, + "nauc_cv_recall_at_10_max": 0.794016725167737, + "nauc_cv_recall_at_10_std": 0.20150889183463758, + "nauc_cv_recall_at_1_diff1": 0.82023819183368, + "nauc_cv_recall_at_1_max": 0.6932781064174243, + "nauc_cv_recall_at_1_std": 0.15115792347826534, + "nauc_cv_recall_at_20_diff1": 0.6427090816405981, + "nauc_cv_recall_at_20_max": 0.7924660240956776, + "nauc_cv_recall_at_20_std": 0.21235840006581444, + "nauc_cv_recall_at_3_diff1": 0.7611491536485745, + "nauc_cv_recall_at_3_max": 0.7138353193818688, + "nauc_cv_recall_at_3_std": 0.1391295422965181, + "nauc_cv_recall_at_5_diff1": 0.7232374725402242, + "nauc_cv_recall_at_5_max": 0.726577328195504, + "nauc_cv_recall_at_5_std": 0.14901571626165597, + "nauc_map_at_1000_diff1": 0.7782405397444, + "nauc_map_at_1000_max": 0.7174181757910132, + "nauc_map_at_1000_std": 0.15736686552194565, + "nauc_map_at_100_diff1": 0.7779408427101666, + "nauc_map_at_100_max": 0.7177708130161948, + "nauc_map_at_100_std": 0.1580175379669668, + "nauc_map_at_10_diff1": 0.7823335231299184, + "nauc_map_at_10_max": 0.7165499158253605, + "nauc_map_at_10_std": 0.15553616586649552, + "nauc_map_at_1_diff1": 0.82023819183368, + "nauc_map_at_1_max": 0.6932781064174243, + "nauc_map_at_1_std": 0.15115792347826534, + "nauc_map_at_20_diff1": 0.7792753419134761, + "nauc_map_at_20_max": 0.716237065369837, + "nauc_map_at_20_std": 0.15606851161248672, + "nauc_map_at_3_diff1": 0.793410702588044, + "nauc_map_at_3_max": 0.705150050776552, + "nauc_map_at_3_std": 0.1471582027526783, + "nauc_map_at_5_diff1": 0.7851116354537095, + "nauc_map_at_5_max": 0.7078973387400431, + "nauc_map_at_5_std": 0.14925366356269948, + "nauc_mrr_at_1000_diff1": 0.7782405397444, + "nauc_mrr_at_1000_max": 0.7174181757910132, + "nauc_mrr_at_1000_std": 0.15736686552194565, + "nauc_mrr_at_100_diff1": 0.7779408427101666, + "nauc_mrr_at_100_max": 0.7177708130161948, + "nauc_mrr_at_100_std": 0.1580175379669668, + "nauc_mrr_at_10_diff1": 0.7823335231299184, + "nauc_mrr_at_10_max": 0.7165499158253605, + "nauc_mrr_at_10_std": 0.15553616586649552, + "nauc_mrr_at_1_diff1": 0.82023819183368, + "nauc_mrr_at_1_max": 0.6932781064174243, + "nauc_mrr_at_1_std": 0.15115792347826534, + "nauc_mrr_at_20_diff1": 0.7792753419134761, + "nauc_mrr_at_20_max": 0.716237065369837, + "nauc_mrr_at_20_std": 0.15606851161248672, + "nauc_mrr_at_3_diff1": 0.793410702588044, + "nauc_mrr_at_3_max": 0.705150050776552, + "nauc_mrr_at_3_std": 0.1471582027526783, + "nauc_mrr_at_5_diff1": 0.7851116354537095, + "nauc_mrr_at_5_max": 0.7078973387400431, + "nauc_mrr_at_5_std": 0.14925366356269948, + "nauc_ndcg_at_1000_diff1": 0.7582505770654221, + "nauc_ndcg_at_1000_max": 0.7298384671482888, + "nauc_ndcg_at_1000_std": 0.1645956327091848, + "nauc_ndcg_at_100_diff1": 0.7407448752009863, + "nauc_ndcg_at_100_max": 0.7432691565803868, + "nauc_ndcg_at_100_std": 0.17866226472522742, + "nauc_ndcg_at_10_diff1": 0.763357005473246, + "nauc_ndcg_at_10_max": 0.7333588449518234, + "nauc_ndcg_at_10_std": 0.1649121151550678, + "nauc_ndcg_at_1_diff1": 0.82023819183368, + "nauc_ndcg_at_1_max": 0.6932781064174243, + "nauc_ndcg_at_1_std": 0.15115792347826534, + "nauc_ndcg_at_20_diff1": 0.7510964074157357, + "nauc_ndcg_at_20_max": 0.7323663849211619, + "nauc_ndcg_at_20_std": 0.1670241666816858, + "nauc_ndcg_at_3_diff1": 0.7851539363695622, + "nauc_ndcg_at_3_max": 0.7074845339044545, + "nauc_ndcg_at_3_std": 0.1451676802667441, + "nauc_ndcg_at_5_diff1": 0.7700013695497865, + "nauc_ndcg_at_5_max": 0.7125439307598648, + "nauc_ndcg_at_5_std": 0.14908218748608065, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.5528063877023378, + "nauc_precision_at_100_max": 0.8748190900404674, + "nauc_precision_at_100_std": 0.29064853585881273, + "nauc_precision_at_10_diff1": 0.7000307851045076, + "nauc_precision_at_10_max": 0.7940167251677365, + "nauc_precision_at_10_std": 0.20150889183463738, + "nauc_precision_at_1_diff1": 0.82023819183368, + "nauc_precision_at_1_max": 0.6932781064174243, + "nauc_precision_at_1_std": 0.15115792347826534, + "nauc_precision_at_20_diff1": 0.6427090816405984, + "nauc_precision_at_20_max": 0.7924660240956771, + "nauc_precision_at_20_std": 0.2123584000658143, + "nauc_precision_at_3_diff1": 0.7611491536485748, + "nauc_precision_at_3_max": 0.7138353193818688, + "nauc_precision_at_3_std": 0.13912954229651817, + "nauc_precision_at_5_diff1": 0.7232374725402244, + "nauc_precision_at_5_max": 0.7265773281955042, + "nauc_precision_at_5_std": 0.1490157162616557, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5528063877023381, + "nauc_recall_at_100_max": 0.8748190900404679, + "nauc_recall_at_100_std": 0.2906485358588136, + "nauc_recall_at_10_diff1": 0.7000307851045077, + "nauc_recall_at_10_max": 0.794016725167737, + "nauc_recall_at_10_std": 0.20150889183463758, + "nauc_recall_at_1_diff1": 0.82023819183368, + "nauc_recall_at_1_max": 0.6932781064174243, + "nauc_recall_at_1_std": 0.15115792347826534, + "nauc_recall_at_20_diff1": 0.6427090816405981, + "nauc_recall_at_20_max": 0.7924660240956776, + "nauc_recall_at_20_std": 0.21235840006581444, + "nauc_recall_at_3_diff1": 0.7611491536485745, + "nauc_recall_at_3_max": 0.7138353193818688, + "nauc_recall_at_3_std": 0.1391295422965181, + "nauc_recall_at_5_diff1": 0.7232374725402242, + "nauc_recall_at_5_max": 0.726577328195504, + "nauc_recall_at_5_std": 0.14901571626165597, + "ndcg_at_1": 0.399, + "ndcg_at_10": 0.46795, + "ndcg_at_100": 0.49594, + "ndcg_at_1000": 0.53109, + "ndcg_at_20": 0.4778, + "ndcg_at_3": 0.43648, + "ndcg_at_5": 0.4518, + "precision_at_1": 0.399, + "precision_at_10": 0.0551, + "precision_at_100": 0.00688, + "precision_at_1000": 0.001, + "precision_at_20": 0.02955, + "precision_at_3": 0.15467, + "precision_at_5": 0.1002, + "recall_at_1": 0.399, + "recall_at_10": 0.551, + "recall_at_100": 0.688, + "recall_at_1000": 1.0, + "recall_at_20": 0.591, + "recall_at_3": 0.464, + "recall_at_5": 0.501 + } + ] + }, + "task_name": "WITT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/model_meta.json b/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/model_meta.json new file mode 100644 index 0000000000..1f85ed05ba --- /dev/null +++ b/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/model_meta.json @@ -0,0 +1 @@ +{"name": "google/siglip-large-patch16-384", "revision": "ce005573a40965dfd21fd937fbdeeebf2439fc35", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..eeaf17e01d --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", + "evaluation_time": 36.80075192451477, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.14, + "cv_recall_at_10": 0.326, + "cv_recall_at_100": 0.7, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.418, + "cv_recall_at_3": 0.216, + "cv_recall_at_5": 0.26, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.20018, + "map_at_1": 0.14, + "map_at_10": 0.18964, + "map_at_100": 0.20214, + "map_at_1000": 0.20383, + "map_at_20": 0.1961, + "map_at_3": 0.17067, + "map_at_5": 0.18057, + "mrr_at_1": 0.14, + "mrr_at_10": 0.18963650793650794, + "mrr_at_100": 0.20214010057097828, + "mrr_at_1000": 0.20383485850905744, + "mrr_at_20": 0.19610020934517847, + "mrr_at_3": 0.17066666666666666, + "mrr_at_5": 0.18056666666666668, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.1943109349666728, + "nauc_cv_recall_at_100_max": 0.30141596108809204, + "nauc_cv_recall_at_100_std": 0.18449648711943784, + "nauc_cv_recall_at_10_diff1": 0.3404131572137101, + "nauc_cv_recall_at_10_max": 0.41392291253691765, + "nauc_cv_recall_at_10_std": 0.14801998820770446, + "nauc_cv_recall_at_1_diff1": 0.6383384954813527, + "nauc_cv_recall_at_1_max": 0.5636386779243923, + "nauc_cv_recall_at_1_std": 0.14467733039161593, + "nauc_cv_recall_at_20_diff1": 0.2755412102074175, + "nauc_cv_recall_at_20_max": 0.3530009092486914, + "nauc_cv_recall_at_20_std": 0.10041369520222374, + "nauc_cv_recall_at_3_diff1": 0.4578307594032993, + "nauc_cv_recall_at_3_max": 0.45122371272185274, + "nauc_cv_recall_at_3_std": 0.14290948032916234, + "nauc_cv_recall_at_5_diff1": 0.38610621259865296, + "nauc_cv_recall_at_5_max": 0.4313116128472723, + "nauc_cv_recall_at_5_std": 0.12227458009308771, + "nauc_map_at_1000_diff1": 0.5108013348607887, + "nauc_map_at_1000_max": 0.49556810553994785, + "nauc_map_at_1000_std": 0.14340316545286083, + "nauc_map_at_100_diff1": 0.5089368235572634, + "nauc_map_at_100_max": 0.49476477633650834, + "nauc_map_at_100_std": 0.14332987871773434, + "nauc_map_at_10_diff1": 0.5152119169569348, + "nauc_map_at_10_max": 0.5016430069177228, + "nauc_map_at_10_std": 0.14462347870066639, + "nauc_map_at_1_diff1": 0.6383384954813527, + "nauc_map_at_1_max": 0.5636386779243923, + "nauc_map_at_1_std": 0.14467733039161593, + "nauc_map_at_20_diff1": 0.5082693046372923, + "nauc_map_at_20_max": 0.494718342086452, + "nauc_map_at_20_std": 0.14051107087925876, + "nauc_map_at_3_diff1": 0.5483175236158682, + "nauc_map_at_3_max": 0.5119270005626352, + "nauc_map_at_3_std": 0.14619814132999046, + "nauc_map_at_5_diff1": 0.5256920920484943, + "nauc_map_at_5_max": 0.5050437535335612, + "nauc_map_at_5_std": 0.14038793888357393, + "nauc_mrr_at_1000_diff1": 0.5108013348607887, + "nauc_mrr_at_1000_max": 0.49556810553994785, + "nauc_mrr_at_1000_std": 0.14340316545286083, + "nauc_mrr_at_100_diff1": 0.5089368235572634, + "nauc_mrr_at_100_max": 0.49476477633650834, + "nauc_mrr_at_100_std": 0.14332987871773434, + "nauc_mrr_at_10_diff1": 0.5152119169569348, + "nauc_mrr_at_10_max": 0.5016430069177228, + "nauc_mrr_at_10_std": 0.14462347870066639, + "nauc_mrr_at_1_diff1": 0.6383384954813527, + "nauc_mrr_at_1_max": 0.5636386779243923, + "nauc_mrr_at_1_std": 0.14467733039161593, + "nauc_mrr_at_20_diff1": 0.5082693046372923, + "nauc_mrr_at_20_max": 0.494718342086452, + "nauc_mrr_at_20_std": 0.14051107087925876, + "nauc_mrr_at_3_diff1": 0.5483175236158682, + "nauc_mrr_at_3_max": 0.5119270005626352, + "nauc_mrr_at_3_std": 0.14619814132999046, + "nauc_mrr_at_5_diff1": 0.5256920920484943, + "nauc_mrr_at_5_max": 0.5050437535335612, + "nauc_mrr_at_5_std": 0.14038793888357393, + "nauc_ndcg_at_1000_diff1": 0.47546503263846057, + "nauc_ndcg_at_1000_max": 0.4730852930430677, + "nauc_ndcg_at_1000_std": 0.14451699521991915, + "nauc_ndcg_at_100_diff1": 0.43918780468181184, + "nauc_ndcg_at_100_max": 0.4525861264216166, + "nauc_ndcg_at_100_std": 0.14813430958483342, + "nauc_ndcg_at_10_diff1": 0.46325211944837963, + "nauc_ndcg_at_10_max": 0.4755238714506355, + "nauc_ndcg_at_10_std": 0.1451640297423185, + "nauc_ndcg_at_1_diff1": 0.6383384954813527, + "nauc_ndcg_at_1_max": 0.5636386779243923, + "nauc_ndcg_at_1_std": 0.14467733039161593, + "nauc_ndcg_at_20_diff1": 0.4411334690790833, + "nauc_ndcg_at_20_max": 0.4545908778015437, + "nauc_ndcg_at_20_std": 0.13045890758542045, + "nauc_ndcg_at_3_diff1": 0.5221939167041364, + "nauc_ndcg_at_3_max": 0.49457354247115864, + "nauc_ndcg_at_3_std": 0.14534347661530161, + "nauc_ndcg_at_5_diff1": 0.48489323636673515, + "nauc_ndcg_at_5_max": 0.4834143750862925, + "nauc_ndcg_at_5_std": 0.1353775207751414, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.19431093496667312, + "nauc_precision_at_100_max": 0.3014159610880925, + "nauc_precision_at_100_std": 0.18449648711943817, + "nauc_precision_at_10_diff1": 0.3404131572137099, + "nauc_precision_at_10_max": 0.4139229125369173, + "nauc_precision_at_10_std": 0.14801998820770462, + "nauc_precision_at_1_diff1": 0.6383384954813527, + "nauc_precision_at_1_max": 0.5636386779243923, + "nauc_precision_at_1_std": 0.14467733039161593, + "nauc_precision_at_20_diff1": 0.2755412102074174, + "nauc_precision_at_20_max": 0.3530009092486912, + "nauc_precision_at_20_std": 0.10041369520222365, + "nauc_precision_at_3_diff1": 0.4578307594032993, + "nauc_precision_at_3_max": 0.45122371272185285, + "nauc_precision_at_3_std": 0.1429094803291628, + "nauc_precision_at_5_diff1": 0.38610621259865285, + "nauc_precision_at_5_max": 0.4313116128472727, + "nauc_precision_at_5_std": 0.12227458009308804, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.1943109349666728, + "nauc_recall_at_100_max": 0.30141596108809204, + "nauc_recall_at_100_std": 0.18449648711943784, + "nauc_recall_at_10_diff1": 0.3404131572137101, + "nauc_recall_at_10_max": 0.41392291253691765, + "nauc_recall_at_10_std": 0.14801998820770446, + "nauc_recall_at_1_diff1": 0.6383384954813527, + "nauc_recall_at_1_max": 0.5636386779243923, + "nauc_recall_at_1_std": 0.14467733039161593, + "nauc_recall_at_20_diff1": 0.2755412102074175, + "nauc_recall_at_20_max": 0.3530009092486914, + "nauc_recall_at_20_std": 0.10041369520222374, + "nauc_recall_at_3_diff1": 0.4578307594032993, + "nauc_recall_at_3_max": 0.45122371272185274, + "nauc_recall_at_3_std": 0.14290948032916234, + "nauc_recall_at_5_diff1": 0.38610621259865296, + "nauc_recall_at_5_max": 0.4313116128472723, + "nauc_recall_at_5_std": 0.12227458009308771, + "ndcg_at_1": 0.14, + "ndcg_at_10": 0.22178, + "ndcg_at_100": 0.29502, + "ndcg_at_1000": 0.33472, + "ndcg_at_20": 0.24514, + "ndcg_at_3": 0.18219, + "ndcg_at_5": 0.20018, + "precision_at_1": 0.14, + "precision_at_10": 0.0326, + "precision_at_100": 0.007, + "precision_at_1000": 0.001, + "precision_at_20": 0.0209, + "precision_at_3": 0.072, + "precision_at_5": 0.052, + "recall_at_1": 0.14, + "recall_at_10": 0.326, + "recall_at_100": 0.7, + "recall_at_1000": 1.0, + "recall_at_20": 0.418, + "recall_at_3": 0.216, + "recall_at_5": 0.26 + } + ] + }, + "task_name": "VidoreArxivQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..2ba30bc4d6 --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b1d89eda849e636676df6ead8002602fb1858600", + "evaluation_time": 40.50687646865845, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.13, + "cv_recall_at_10": 0.29, + "cv_recall_at_100": 0.592, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.372, + "cv_recall_at_3": 0.192, + "cv_recall_at_5": 0.23, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.18085, + "map_at_1": 0.13, + "map_at_10": 0.17226, + "map_at_100": 0.1827, + "map_at_1000": 0.18458, + "map_at_20": 0.17783, + "map_at_3": 0.15633, + "map_at_5": 0.16473, + "mrr_at_1": 0.13, + "mrr_at_10": 0.17225873015873025, + "mrr_at_100": 0.18270338816739434, + "mrr_at_1000": 0.18458288118569882, + "mrr_at_20": 0.1778293426645595, + "mrr_at_3": 0.15633333333333335, + "mrr_at_5": 0.16473333333333337, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.29668674698795183, + "nauc_cv_recall_at_100_max": 0.3333401967243978, + "nauc_cv_recall_at_100_std": 0.4607668957078311, + "nauc_cv_recall_at_10_diff1": 0.3176627336785572, + "nauc_cv_recall_at_10_max": 0.36628985069296605, + "nauc_cv_recall_at_10_std": 0.45197038583574867, + "nauc_cv_recall_at_1_diff1": 0.5701058852499383, + "nauc_cv_recall_at_1_max": 0.4710169908889438, + "nauc_cv_recall_at_1_std": 0.45840030444807583, + "nauc_cv_recall_at_20_diff1": 0.3173390526584947, + "nauc_cv_recall_at_20_max": 0.36508706837620225, + "nauc_cv_recall_at_20_std": 0.4626756631233985, + "nauc_cv_recall_at_3_diff1": 0.4098228341204408, + "nauc_cv_recall_at_3_max": 0.40064769604311734, + "nauc_cv_recall_at_3_std": 0.447420555511086, + "nauc_cv_recall_at_5_diff1": 0.3935662970363478, + "nauc_cv_recall_at_5_max": 0.3866567013925962, + "nauc_cv_recall_at_5_std": 0.43803175668461425, + "nauc_map_at_1000_diff1": 0.47008151424287187, + "nauc_map_at_1000_max": 0.42564732667200894, + "nauc_map_at_1000_std": 0.45354116369652825, + "nauc_map_at_100_diff1": 0.46917354197138605, + "nauc_map_at_100_max": 0.42511751999498737, + "nauc_map_at_100_std": 0.4536651781812026, + "nauc_map_at_10_diff1": 0.4702955145420076, + "nauc_map_at_10_max": 0.4276602888263265, + "nauc_map_at_10_std": 0.45271558223499087, + "nauc_map_at_1_diff1": 0.5701058852499383, + "nauc_map_at_1_max": 0.4710169908889438, + "nauc_map_at_1_std": 0.45840030444807583, + "nauc_map_at_20_diff1": 0.4699693881258117, + "nauc_map_at_20_max": 0.4263741043059254, + "nauc_map_at_20_std": 0.45377335365237415, + "nauc_map_at_3_diff1": 0.49224148432955067, + "nauc_map_at_3_max": 0.43597783225397924, + "nauc_map_at_3_std": 0.453269429839162, + "nauc_map_at_5_diff1": 0.4859281573788269, + "nauc_map_at_5_max": 0.4305498781499855, + "nauc_map_at_5_std": 0.4504661603812436, + "nauc_mrr_at_1000_diff1": 0.47008151424287187, + "nauc_mrr_at_1000_max": 0.42564732667200894, + "nauc_mrr_at_1000_std": 0.45354116369652825, + "nauc_mrr_at_100_diff1": 0.46917354197138605, + "nauc_mrr_at_100_max": 0.42511751999498737, + "nauc_mrr_at_100_std": 0.4536651781812026, + "nauc_mrr_at_10_diff1": 0.4702955145420076, + "nauc_mrr_at_10_max": 0.4276602888263265, + "nauc_mrr_at_10_std": 0.45271558223499087, + "nauc_mrr_at_1_diff1": 0.5701058852499383, + "nauc_mrr_at_1_max": 0.4710169908889438, + "nauc_mrr_at_1_std": 0.45840030444807583, + "nauc_mrr_at_20_diff1": 0.4699693881258117, + "nauc_mrr_at_20_max": 0.4263741043059254, + "nauc_mrr_at_20_std": 0.45377335365237415, + "nauc_mrr_at_3_diff1": 0.49224148432955067, + "nauc_mrr_at_3_max": 0.43597783225397924, + "nauc_mrr_at_3_std": 0.453269429839162, + "nauc_mrr_at_5_diff1": 0.4859281573788269, + "nauc_mrr_at_5_max": 0.4305498781499855, + "nauc_mrr_at_5_std": 0.4504661603812436, + "nauc_ndcg_at_1000_diff1": 0.44217374572654283, + "nauc_ndcg_at_1000_max": 0.41259692326408354, + "nauc_ndcg_at_1000_std": 0.45335502089769386, + "nauc_ndcg_at_100_diff1": 0.42077496326949876, + "nauc_ndcg_at_100_max": 0.40068408483983037, + "nauc_ndcg_at_100_std": 0.4549589546561871, + "nauc_ndcg_at_10_diff1": 0.42501562481003097, + "nauc_ndcg_at_10_max": 0.40952295092418756, + "nauc_ndcg_at_10_std": 0.4521354844274342, + "nauc_ndcg_at_1_diff1": 0.5701058852499383, + "nauc_ndcg_at_1_max": 0.4710169908889438, + "nauc_ndcg_at_1_std": 0.45840030444807583, + "nauc_ndcg_at_20_diff1": 0.42370856334004775, + "nauc_ndcg_at_20_max": 0.40742912764166467, + "nauc_ndcg_at_20_std": 0.455638693912446, + "nauc_ndcg_at_3_diff1": 0.46866327788004875, + "nauc_ndcg_at_3_max": 0.42583768645337167, + "nauc_ndcg_at_3_std": 0.45160402362953556, + "nauc_ndcg_at_5_diff1": 0.4586274201403904, + "nauc_ndcg_at_5_max": 0.4175435252168309, + "nauc_ndcg_at_5_std": 0.44689833044121, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.29668674698795183, + "nauc_precision_at_100_max": 0.3333401967243981, + "nauc_precision_at_100_std": 0.4607668957078318, + "nauc_precision_at_10_diff1": 0.31766273367855735, + "nauc_precision_at_10_max": 0.36628985069296616, + "nauc_precision_at_10_std": 0.45197038583574883, + "nauc_precision_at_1_diff1": 0.5701058852499383, + "nauc_precision_at_1_max": 0.4710169908889438, + "nauc_precision_at_1_std": 0.45840030444807583, + "nauc_precision_at_20_diff1": 0.31733905265849477, + "nauc_precision_at_20_max": 0.3650870683762021, + "nauc_precision_at_20_std": 0.4626756631233986, + "nauc_precision_at_3_diff1": 0.4098228341204409, + "nauc_precision_at_3_max": 0.4006476960431175, + "nauc_precision_at_3_std": 0.44742055551108645, + "nauc_precision_at_5_diff1": 0.39356629703634816, + "nauc_precision_at_5_max": 0.3866567013925965, + "nauc_precision_at_5_std": 0.43803175668461447, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.29668674698795183, + "nauc_recall_at_100_max": 0.3333401967243978, + "nauc_recall_at_100_std": 0.4607668957078311, + "nauc_recall_at_10_diff1": 0.3176627336785572, + "nauc_recall_at_10_max": 0.36628985069296605, + "nauc_recall_at_10_std": 0.45197038583574867, + "nauc_recall_at_1_diff1": 0.5701058852499383, + "nauc_recall_at_1_max": 0.4710169908889438, + "nauc_recall_at_1_std": 0.45840030444807583, + "nauc_recall_at_20_diff1": 0.3173390526584947, + "nauc_recall_at_20_max": 0.36508706837620225, + "nauc_recall_at_20_std": 0.4626756631233985, + "nauc_recall_at_3_diff1": 0.4098228341204408, + "nauc_recall_at_3_max": 0.40064769604311734, + "nauc_recall_at_3_std": 0.447420555511086, + "nauc_recall_at_5_diff1": 0.3935662970363478, + "nauc_recall_at_5_max": 0.3866567013925962, + "nauc_recall_at_5_std": 0.43803175668461425, + "ndcg_at_1": 0.13, + "ndcg_at_10": 0.19978, + "ndcg_at_100": 0.25958, + "ndcg_at_1000": 0.31147, + "ndcg_at_20": 0.22036, + "ndcg_at_3": 0.16545, + "ndcg_at_5": 0.18085, + "precision_at_1": 0.13, + "precision_at_10": 0.029, + "precision_at_100": 0.00592, + "precision_at_1000": 0.001, + "precision_at_20": 0.0186, + "precision_at_3": 0.064, + "precision_at_5": 0.046, + "recall_at_1": 0.13, + "recall_at_10": 0.29, + "recall_at_100": 0.592, + "recall_at_1000": 1.0, + "recall_at_20": 0.372, + "recall_at_3": 0.192, + "recall_at_5": 0.23 + } + ] + }, + "task_name": "VidoreDocVQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..aea7cf6a30 --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", + "evaluation_time": 39.90301728248596, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.258, + "cv_recall_at_10": 0.548, + "cv_recall_at_100": 0.814, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.606, + "cv_recall_at_3": 0.378, + "cv_recall_at_5": 0.456, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.36058, + "map_at_1": 0.258, + "map_at_10": 0.34146, + "map_at_100": 0.35092, + "map_at_1000": 0.35188, + "map_at_20": 0.34558, + "map_at_3": 0.31167, + "map_at_5": 0.32917, + "mrr_at_1": 0.258, + "mrr_at_10": 0.3414619047619047, + "mrr_at_100": 0.35091516817676005, + "mrr_at_1000": 0.351878944711423, + "mrr_at_20": 0.3455762584165062, + "mrr_at_3": 0.3116666666666667, + "mrr_at_5": 0.3291666666666666, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.22488058728615315, + "nauc_cv_recall_at_100_max": 0.34163790432023017, + "nauc_cv_recall_at_100_std": 0.4398715578684168, + "nauc_cv_recall_at_10_diff1": 0.3593018759002586, + "nauc_cv_recall_at_10_max": 0.43681797546252976, + "nauc_cv_recall_at_10_std": 0.3225292986812089, + "nauc_cv_recall_at_1_diff1": 0.5867050765441552, + "nauc_cv_recall_at_1_max": 0.5015622144280729, + "nauc_cv_recall_at_1_std": 0.22811521802827353, + "nauc_cv_recall_at_20_diff1": 0.32595345879120036, + "nauc_cv_recall_at_20_max": 0.41825288970087515, + "nauc_cv_recall_at_20_std": 0.36061786949621855, + "nauc_cv_recall_at_3_diff1": 0.4670428735336128, + "nauc_cv_recall_at_3_max": 0.49687144704354985, + "nauc_cv_recall_at_3_std": 0.2719720680710684, + "nauc_cv_recall_at_5_diff1": 0.38826252637546027, + "nauc_cv_recall_at_5_max": 0.4553113091676377, + "nauc_cv_recall_at_5_std": 0.29645755256721124, + "nauc_map_at_1000_diff1": 0.5094053800617759, + "nauc_map_at_1000_max": 0.489293114100951, + "nauc_map_at_1000_std": 0.2638562098259416, + "nauc_map_at_100_diff1": 0.5085268178285087, + "nauc_map_at_100_max": 0.4889533291842021, + "nauc_map_at_100_std": 0.26436130382480405, + "nauc_map_at_10_diff1": 0.5091516903713251, + "nauc_map_at_10_max": 0.4899516623083387, + "nauc_map_at_10_std": 0.2610633947620562, + "nauc_map_at_1_diff1": 0.5867050765441552, + "nauc_map_at_1_max": 0.5015622144280729, + "nauc_map_at_1_std": 0.22811521802827353, + "nauc_map_at_20_diff1": 0.5081365189697196, + "nauc_map_at_20_max": 0.48887071333338156, + "nauc_map_at_20_std": 0.2634137688547193, + "nauc_map_at_3_diff1": 0.5306475249114199, + "nauc_map_at_3_max": 0.5025297847699206, + "nauc_map_at_3_std": 0.25148826382598116, + "nauc_map_at_5_diff1": 0.5120080392776135, + "nauc_map_at_5_max": 0.49271028166183345, + "nauc_map_at_5_std": 0.2584032846191391, + "nauc_mrr_at_1000_diff1": 0.5094053800617759, + "nauc_mrr_at_1000_max": 0.489293114100951, + "nauc_mrr_at_1000_std": 0.2638562098259416, + "nauc_mrr_at_100_diff1": 0.5085268178285087, + "nauc_mrr_at_100_max": 0.4889533291842021, + "nauc_mrr_at_100_std": 0.26436130382480405, + "nauc_mrr_at_10_diff1": 0.5091516903713251, + "nauc_mrr_at_10_max": 0.4899516623083387, + "nauc_mrr_at_10_std": 0.2610633947620562, + "nauc_mrr_at_1_diff1": 0.5867050765441552, + "nauc_mrr_at_1_max": 0.5015622144280729, + "nauc_mrr_at_1_std": 0.22811521802827353, + "nauc_mrr_at_20_diff1": 0.5081365189697196, + "nauc_mrr_at_20_max": 0.48887071333338156, + "nauc_mrr_at_20_std": 0.2634137688547193, + "nauc_mrr_at_3_diff1": 0.5306475249114199, + "nauc_mrr_at_3_max": 0.5025297847699206, + "nauc_mrr_at_3_std": 0.25148826382598116, + "nauc_mrr_at_5_diff1": 0.5120080392776135, + "nauc_mrr_at_5_max": 0.49271028166183345, + "nauc_mrr_at_5_std": 0.2584032846191391, + "nauc_ndcg_at_1000_diff1": 0.48845200236250697, + "nauc_ndcg_at_1000_max": 0.48102900753840117, + "nauc_ndcg_at_1000_std": 0.2758573747125474, + "nauc_ndcg_at_100_diff1": 0.46824118560402356, + "nauc_ndcg_at_100_max": 0.4709534856151632, + "nauc_ndcg_at_100_std": 0.28812850918887767, + "nauc_ndcg_at_10_diff1": 0.47290459879999414, + "nauc_ndcg_at_10_max": 0.47758748755214875, + "nauc_ndcg_at_10_std": 0.2757346332309286, + "nauc_ndcg_at_1_diff1": 0.5867050765441552, + "nauc_ndcg_at_1_max": 0.5015622144280729, + "nauc_ndcg_at_1_std": 0.22811521802827353, + "nauc_ndcg_at_20_diff1": 0.4674553529067452, + "nauc_ndcg_at_20_max": 0.4736917036654822, + "nauc_ndcg_at_20_std": 0.28420357989357237, + "nauc_ndcg_at_3_diff1": 0.5136193330507195, + "nauc_ndcg_at_3_max": 0.5011469410434867, + "nauc_ndcg_at_3_std": 0.25709010460637316, + "nauc_ndcg_at_5_diff1": 0.4802867776472517, + "nauc_ndcg_at_5_max": 0.483545793523169, + "nauc_ndcg_at_5_std": 0.2684200549296203, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.2248805872861516, + "nauc_precision_at_100_max": 0.3416379043202278, + "nauc_precision_at_100_std": 0.4398715578684146, + "nauc_precision_at_10_diff1": 0.3593018759002583, + "nauc_precision_at_10_max": 0.43681797546252954, + "nauc_precision_at_10_std": 0.32252929868120905, + "nauc_precision_at_1_diff1": 0.5867050765441552, + "nauc_precision_at_1_max": 0.5015622144280729, + "nauc_precision_at_1_std": 0.22811521802827353, + "nauc_precision_at_20_diff1": 0.3259534587912, + "nauc_precision_at_20_max": 0.4182528897008752, + "nauc_precision_at_20_std": 0.3606178694962182, + "nauc_precision_at_3_diff1": 0.46704287353361273, + "nauc_precision_at_3_max": 0.49687144704354985, + "nauc_precision_at_3_std": 0.2719720680710683, + "nauc_precision_at_5_diff1": 0.38826252637546066, + "nauc_precision_at_5_max": 0.4553113091676376, + "nauc_precision_at_5_std": 0.29645755256721196, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.22488058728615315, + "nauc_recall_at_100_max": 0.34163790432023017, + "nauc_recall_at_100_std": 0.4398715578684168, + "nauc_recall_at_10_diff1": 0.3593018759002586, + "nauc_recall_at_10_max": 0.43681797546252976, + "nauc_recall_at_10_std": 0.3225292986812089, + "nauc_recall_at_1_diff1": 0.5867050765441552, + "nauc_recall_at_1_max": 0.5015622144280729, + "nauc_recall_at_1_std": 0.22811521802827353, + "nauc_recall_at_20_diff1": 0.32595345879120036, + "nauc_recall_at_20_max": 0.41825288970087515, + "nauc_recall_at_20_std": 0.36061786949621855, + "nauc_recall_at_3_diff1": 0.4670428735336128, + "nauc_recall_at_3_max": 0.49687144704354985, + "nauc_recall_at_3_std": 0.2719720680710684, + "nauc_recall_at_5_diff1": 0.38826252637546027, + "nauc_recall_at_5_max": 0.4553113091676377, + "nauc_recall_at_5_std": 0.29645755256721124, + "ndcg_at_1": 0.258, + "ndcg_at_10": 0.39035, + "ndcg_at_100": 0.44372, + "ndcg_at_1000": 0.46792, + "ndcg_at_20": 0.40512, + "ndcg_at_3": 0.32874, + "ndcg_at_5": 0.36058, + "precision_at_1": 0.258, + "precision_at_10": 0.0548, + "precision_at_100": 0.00814, + "precision_at_1000": 0.001, + "precision_at_20": 0.0303, + "precision_at_3": 0.126, + "precision_at_5": 0.0912, + "recall_at_1": 0.258, + "recall_at_10": 0.548, + "recall_at_100": 0.814, + "recall_at_1000": 1.0, + "recall_at_20": 0.606, + "recall_at_3": 0.378, + "recall_at_5": 0.456 + } + ] + }, + "task_name": "VidoreInfoVQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..b417221b6f --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", + "evaluation_time": 75.55646324157715, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.14, + "cv_recall_at_10": 0.43, + "cv_recall_at_100": 0.77, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.54, + "cv_recall_at_3": 0.29, + "cv_recall_at_5": 0.35, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.25044, + "map_at_1": 0.14, + "map_at_10": 0.22716, + "map_at_100": 0.24058, + "map_at_1000": 0.24155, + "map_at_20": 0.23522, + "map_at_3": 0.20333, + "map_at_5": 0.21733, + "mrr_at_1": 0.14, + "mrr_at_10": 0.22716269841269843, + "mrr_at_100": 0.24058436473035777, + "mrr_at_1000": 0.2415510139478348, + "mrr_at_20": 0.23522009934509933, + "mrr_at_3": 0.2033333333333333, + "mrr_at_5": 0.21733333333333335, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.07011929007855676, + "nauc_cv_recall_at_100_max": 0.3579597591817545, + "nauc_cv_recall_at_100_std": 0.31774132181464143, + "nauc_cv_recall_at_10_diff1": 0.2517589834961177, + "nauc_cv_recall_at_10_max": 0.3860920003641519, + "nauc_cv_recall_at_10_std": 0.28466270434771296, + "nauc_cv_recall_at_1_diff1": 0.32462132462132465, + "nauc_cv_recall_at_1_max": 0.07117399974542814, + "nauc_cv_recall_at_1_std": 0.05664219949934226, + "nauc_cv_recall_at_20_diff1": 0.23868467116779174, + "nauc_cv_recall_at_20_max": 0.3474618654663666, + "nauc_cv_recall_at_20_std": 0.2975049317885025, + "nauc_cv_recall_at_3_diff1": 0.3201928491957749, + "nauc_cv_recall_at_3_max": 0.3486257434446383, + "nauc_cv_recall_at_3_std": 0.18352265703336387, + "nauc_cv_recall_at_5_diff1": 0.28013712176148714, + "nauc_cv_recall_at_5_max": 0.3316632605972707, + "nauc_cv_recall_at_5_std": 0.14300217548948513, + "nauc_map_at_1000_diff1": 0.31781193581545547, + "nauc_map_at_1000_max": 0.20583970548908834, + "nauc_map_at_1000_std": 0.11837714875609179, + "nauc_map_at_100_diff1": 0.3162574307400726, + "nauc_map_at_100_max": 0.20600852978387757, + "nauc_map_at_100_std": 0.11890326588085454, + "nauc_map_at_10_diff1": 0.318533633385182, + "nauc_map_at_10_max": 0.21124248494920406, + "nauc_map_at_10_std": 0.12349472262933363, + "nauc_map_at_1_diff1": 0.32462132462132465, + "nauc_map_at_1_max": 0.07117399974542814, + "nauc_map_at_1_std": 0.05664219949934226, + "nauc_map_at_20_diff1": 0.32095854907465377, + "nauc_map_at_20_max": 0.20707743553081426, + "nauc_map_at_20_std": 0.12254998915538791, + "nauc_map_at_3_diff1": 0.3361859443214354, + "nauc_map_at_3_max": 0.20457404361165038, + "nauc_map_at_3_std": 0.11004513709039367, + "nauc_map_at_5_diff1": 0.3256838023956137, + "nauc_map_at_5_max": 0.20191202664591607, + "nauc_map_at_5_std": 0.09959327195117011, + "nauc_mrr_at_1000_diff1": 0.31781193581545547, + "nauc_mrr_at_1000_max": 0.20583970548908834, + "nauc_mrr_at_1000_std": 0.11837714875609179, + "nauc_mrr_at_100_diff1": 0.3162574307400726, + "nauc_mrr_at_100_max": 0.20600852978387757, + "nauc_mrr_at_100_std": 0.11890326588085454, + "nauc_mrr_at_10_diff1": 0.318533633385182, + "nauc_mrr_at_10_max": 0.21124248494920406, + "nauc_mrr_at_10_std": 0.12349472262933363, + "nauc_mrr_at_1_diff1": 0.32462132462132465, + "nauc_mrr_at_1_max": 0.07117399974542814, + "nauc_mrr_at_1_std": 0.05664219949934226, + "nauc_mrr_at_20_diff1": 0.32095854907465377, + "nauc_mrr_at_20_max": 0.20707743553081426, + "nauc_mrr_at_20_std": 0.12254998915538791, + "nauc_mrr_at_3_diff1": 0.3361859443214354, + "nauc_mrr_at_3_max": 0.20457404361165038, + "nauc_mrr_at_3_std": 0.11004513709039367, + "nauc_mrr_at_5_diff1": 0.3256838023956137, + "nauc_mrr_at_5_max": 0.20191202664591607, + "nauc_mrr_at_5_std": 0.09959327195117011, + "nauc_ndcg_at_1000_diff1": 0.30331892551505446, + "nauc_ndcg_at_1000_max": 0.23206326096932686, + "nauc_ndcg_at_1000_std": 0.13914943758556678, + "nauc_ndcg_at_100_diff1": 0.277090040797122, + "nauc_ndcg_at_100_max": 0.24263312120508812, + "nauc_ndcg_at_100_std": 0.15521452351822168, + "nauc_ndcg_at_10_diff1": 0.2998416223952832, + "nauc_ndcg_at_10_max": 0.26258288066776686, + "nauc_ndcg_at_10_std": 0.16778069464708717, + "nauc_ndcg_at_1_diff1": 0.32462132462132465, + "nauc_ndcg_at_1_max": 0.07117399974542814, + "nauc_ndcg_at_1_std": 0.05664219949934226, + "nauc_ndcg_at_20_diff1": 0.3006714776078098, + "nauc_ndcg_at_20_max": 0.24906392668652022, + "nauc_ndcg_at_20_std": 0.16791706808815687, + "nauc_ndcg_at_3_diff1": 0.33193765983303775, + "nauc_ndcg_at_3_max": 0.24710745275143542, + "nauc_ndcg_at_3_std": 0.13147738828592404, + "nauc_ndcg_at_5_diff1": 0.31314624561950827, + "nauc_ndcg_at_5_max": 0.24124707243552312, + "nauc_ndcg_at_5_std": 0.11299960549335761, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.07011929007855745, + "nauc_precision_at_100_max": 0.3579597591817549, + "nauc_precision_at_100_std": 0.3177413218146418, + "nauc_precision_at_10_diff1": 0.2517589834961181, + "nauc_precision_at_10_max": 0.3860920003641523, + "nauc_precision_at_10_std": 0.2846627043477133, + "nauc_precision_at_1_diff1": 0.32462132462132465, + "nauc_precision_at_1_max": 0.07117399974542814, + "nauc_precision_at_1_std": 0.05664219949934226, + "nauc_precision_at_20_diff1": 0.2386846711677925, + "nauc_precision_at_20_max": 0.34746186546636687, + "nauc_precision_at_20_std": 0.29750493178850307, + "nauc_precision_at_3_diff1": 0.32019284919577484, + "nauc_precision_at_3_max": 0.3486257434446383, + "nauc_precision_at_3_std": 0.1835226570333639, + "nauc_precision_at_5_diff1": 0.2801371217614872, + "nauc_precision_at_5_max": 0.3316632605972707, + "nauc_precision_at_5_std": 0.1430021754894849, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.07011929007855676, + "nauc_recall_at_100_max": 0.3579597591817545, + "nauc_recall_at_100_std": 0.31774132181464143, + "nauc_recall_at_10_diff1": 0.2517589834961177, + "nauc_recall_at_10_max": 0.3860920003641519, + "nauc_recall_at_10_std": 0.28466270434771296, + "nauc_recall_at_1_diff1": 0.32462132462132465, + "nauc_recall_at_1_max": 0.07117399974542814, + "nauc_recall_at_1_std": 0.05664219949934226, + "nauc_recall_at_20_diff1": 0.23868467116779174, + "nauc_recall_at_20_max": 0.3474618654663666, + "nauc_recall_at_20_std": 0.2975049317885025, + "nauc_recall_at_3_diff1": 0.3201928491957749, + "nauc_recall_at_3_max": 0.3486257434446383, + "nauc_recall_at_3_std": 0.18352265703336387, + "nauc_recall_at_5_diff1": 0.28013712176148714, + "nauc_recall_at_5_max": 0.3316632605972707, + "nauc_recall_at_5_std": 0.14300217548948513, + "ndcg_at_1": 0.14, + "ndcg_at_10": 0.27548, + "ndcg_at_100": 0.34536, + "ndcg_at_1000": 0.37396, + "ndcg_at_20": 0.3038, + "ndcg_at_3": 0.22547, + "ndcg_at_5": 0.25044, + "precision_at_1": 0.14, + "precision_at_10": 0.043, + "precision_at_100": 0.0077, + "precision_at_1000": 0.001, + "precision_at_20": 0.027, + "precision_at_3": 0.09667, + "precision_at_5": 0.07, + "recall_at_1": 0.14, + "recall_at_10": 0.43, + "recall_at_100": 0.77, + "recall_at_1000": 1.0, + "recall_at_20": 0.54, + "recall_at_3": 0.29, + "recall_at_5": 0.35 + } + ] + }, + "task_name": "VidoreShiftProjectRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..057976d0ba --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", + "evaluation_time": 74.40851640701294, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.23, + "cv_recall_at_10": 0.53, + "cv_recall_at_100": 0.76, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.61, + "cv_recall_at_3": 0.38, + "cv_recall_at_5": 0.43, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.327, + "map_at_1": 0.21, + "map_at_10": 0.30566, + "map_at_100": 0.31547, + "map_at_1000": 0.31657, + "map_at_20": 0.31133, + "map_at_3": 0.28167, + "map_at_5": 0.29267, + "mrr_at_1": 0.23, + "mrr_at_10": 0.3156746031746031, + "mrr_at_100": 0.3254601138348874, + "mrr_at_1000": 0.3265610689913019, + "mrr_at_20": 0.3213241758241757, + "mrr_at_3": 0.29166666666666663, + "mrr_at_5": 0.30316666666666664, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.006359781950334248, + "nauc_cv_recall_at_100_max": 0.5430691355888204, + "nauc_cv_recall_at_100_std": 0.4596997490698272, + "nauc_cv_recall_at_10_diff1": 0.08308017979628572, + "nauc_cv_recall_at_10_max": 0.5721453215851762, + "nauc_cv_recall_at_10_std": 0.2545189624599649, + "nauc_cv_recall_at_1_diff1": 0.17789539512820113, + "nauc_cv_recall_at_1_max": 0.43204038986592636, + "nauc_cv_recall_at_1_std": -0.009472094690441003, + "nauc_cv_recall_at_20_diff1": -0.02098409197347156, + "nauc_cv_recall_at_20_max": 0.5650748568579761, + "nauc_cv_recall_at_20_std": 0.3741181091656218, + "nauc_cv_recall_at_3_diff1": 0.10990824734229203, + "nauc_cv_recall_at_3_max": 0.616510279936579, + "nauc_cv_recall_at_3_std": 0.1248148051880541, + "nauc_cv_recall_at_5_diff1": 0.0806986513376077, + "nauc_cv_recall_at_5_max": 0.6248585660220314, + "nauc_cv_recall_at_5_std": 0.22141733102703817, + "nauc_map_at_1000_diff1": 0.1526572027006709, + "nauc_map_at_1000_max": 0.5299962487999986, + "nauc_map_at_1000_std": 0.07656699308819963, + "nauc_map_at_100_diff1": 0.15241863064910316, + "nauc_map_at_100_max": 0.5301663175741202, + "nauc_map_at_100_std": 0.0776350535363974, + "nauc_map_at_10_diff1": 0.15792022337662764, + "nauc_map_at_10_max": 0.5298177356615489, + "nauc_map_at_10_std": 0.06920520130657719, + "nauc_map_at_1_diff1": 0.2525377294249925, + "nauc_map_at_1_max": 0.45356073922751683, + "nauc_map_at_1_std": -0.04944019510032149, + "nauc_map_at_20_diff1": 0.15111374679870085, + "nauc_map_at_20_max": 0.529961239541217, + "nauc_map_at_20_std": 0.07729346018356126, + "nauc_map_at_3_diff1": 0.16157692832993492, + "nauc_map_at_3_max": 0.5360219292045154, + "nauc_map_at_3_std": 0.04172107834562945, + "nauc_map_at_5_diff1": 0.15570547539001464, + "nauc_map_at_5_max": 0.53800624291612, + "nauc_map_at_5_std": 0.06658104533395676, + "nauc_mrr_at_1000_diff1": 0.11823237600605414, + "nauc_mrr_at_1000_max": 0.5212691085515011, + "nauc_mrr_at_1000_std": 0.09520643817366876, + "nauc_mrr_at_100_diff1": 0.11809775190018582, + "nauc_mrr_at_100_max": 0.5214609438591693, + "nauc_mrr_at_100_std": 0.09620581011777922, + "nauc_mrr_at_10_diff1": 0.12388271932408514, + "nauc_mrr_at_10_max": 0.5215436616204162, + "nauc_mrr_at_10_std": 0.08789608700703568, + "nauc_mrr_at_1_diff1": 0.17789539512820113, + "nauc_mrr_at_1_max": 0.43204038986592636, + "nauc_mrr_at_1_std": -0.009472094690441003, + "nauc_mrr_at_20_diff1": 0.11709458086096404, + "nauc_mrr_at_20_max": 0.5212632072880561, + "nauc_mrr_at_20_std": 0.09564320422360603, + "nauc_mrr_at_3_diff1": 0.12855140902292847, + "nauc_mrr_at_3_max": 0.5246464194448699, + "nauc_mrr_at_3_std": 0.05868082951588032, + "nauc_mrr_at_5_diff1": 0.12130166158692024, + "nauc_mrr_at_5_max": 0.5293317976745179, + "nauc_mrr_at_5_std": 0.0848690117724686, + "nauc_ndcg_at_1000_diff1": 0.13372164670347136, + "nauc_ndcg_at_1000_max": 0.538140945351461, + "nauc_ndcg_at_1000_std": 0.11386350497056077, + "nauc_ndcg_at_100_diff1": 0.12258777943022443, + "nauc_ndcg_at_100_max": 0.539138531606273, + "nauc_ndcg_at_100_std": 0.14404291133909553, + "nauc_ndcg_at_10_diff1": 0.1379040925491282, + "nauc_ndcg_at_10_max": 0.5429870060366613, + "nauc_ndcg_at_10_std": 0.11630716757265272, + "nauc_ndcg_at_1_diff1": 0.2525377294249925, + "nauc_ndcg_at_1_max": 0.45356073922751683, + "nauc_ndcg_at_1_std": -0.04944019510032149, + "nauc_ndcg_at_20_diff1": 0.11338858058945903, + "nauc_ndcg_at_20_max": 0.5416827102797607, + "nauc_ndcg_at_20_std": 0.1438549019021181, + "nauc_ndcg_at_3_diff1": 0.14642325849636342, + "nauc_ndcg_at_3_max": 0.5583865766196738, + "nauc_ndcg_at_3_std": 0.06493605013926239, + "nauc_ndcg_at_5_diff1": 0.1347770230227918, + "nauc_ndcg_at_5_max": 0.5619629664272141, + "nauc_ndcg_at_5_std": 0.10811411065414367, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": -0.006359781950332713, + "nauc_precision_at_100_max": 0.5430691355888206, + "nauc_precision_at_100_std": 0.4596997490698282, + "nauc_precision_at_10_diff1": 0.08308017979628633, + "nauc_precision_at_10_max": 0.5721453215851767, + "nauc_precision_at_10_std": 0.2545189624599656, + "nauc_precision_at_1_diff1": 0.2525377294249925, + "nauc_precision_at_1_max": 0.45356073922751683, + "nauc_precision_at_1_std": -0.04944019510032149, + "nauc_precision_at_20_diff1": -0.020984091973470707, + "nauc_precision_at_20_max": 0.565074856857976, + "nauc_precision_at_20_std": 0.37411810916562194, + "nauc_precision_at_3_diff1": 0.10990824734229175, + "nauc_precision_at_3_max": 0.6165102799365789, + "nauc_precision_at_3_std": 0.12481480518805382, + "nauc_precision_at_5_diff1": 0.08069865133760828, + "nauc_precision_at_5_max": 0.6248585660220317, + "nauc_precision_at_5_std": 0.22141733102703876, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.006359781950334248, + "nauc_recall_at_100_max": 0.5430691355888204, + "nauc_recall_at_100_std": 0.4596997490698272, + "nauc_recall_at_10_diff1": 0.08308017979628572, + "nauc_recall_at_10_max": 0.5721453215851762, + "nauc_recall_at_10_std": 0.2545189624599649, + "nauc_recall_at_1_diff1": 0.2525377294249925, + "nauc_recall_at_1_max": 0.45356073922751683, + "nauc_recall_at_1_std": -0.04944019510032149, + "nauc_recall_at_20_diff1": -0.02098409197347156, + "nauc_recall_at_20_max": 0.5650748568579761, + "nauc_recall_at_20_std": 0.3741181091656218, + "nauc_recall_at_3_diff1": 0.10990824734229203, + "nauc_recall_at_3_max": 0.616510279936579, + "nauc_recall_at_3_std": 0.1248148051880541, + "nauc_recall_at_5_diff1": 0.0806986513376077, + "nauc_recall_at_5_max": 0.6248585660220314, + "nauc_recall_at_5_std": 0.22141733102703817, + "ndcg_at_1": 0.21, + "ndcg_at_10": 0.35903, + "ndcg_at_100": 0.40793, + "ndcg_at_1000": 0.43806, + "ndcg_at_20": 0.37941, + "ndcg_at_3": 0.30678, + "ndcg_at_5": 0.327, + "precision_at_1": 0.21, + "precision_at_10": 0.053, + "precision_at_100": 0.0076, + "precision_at_1000": 0.001, + "precision_at_20": 0.0305, + "precision_at_3": 0.12667, + "precision_at_5": 0.086, + "recall_at_1": 0.21, + "recall_at_10": 0.53, + "recall_at_100": 0.76, + "recall_at_1000": 1.0, + "recall_at_20": 0.61, + "recall_at_3": 0.38, + "recall_at_5": 0.43 + } + ] + }, + "task_name": "VidoreSyntheticDocQAAIRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..62f44290d2 --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", + "evaluation_time": 73.33681082725525, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.28, + "cv_recall_at_10": 0.67, + "cv_recall_at_100": 0.92, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.71, + "cv_recall_at_3": 0.5, + "cv_recall_at_5": 0.58, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.44845, + "map_at_1": 0.28, + "map_at_10": 0.41726, + "map_at_100": 0.425, + "map_at_1000": 0.42549, + "map_at_20": 0.42028, + "map_at_3": 0.38667, + "map_at_5": 0.40417, + "mrr_at_1": 0.28, + "mrr_at_10": 0.41671428571428576, + "mrr_at_100": 0.4244418064279869, + "mrr_at_1000": 0.42492739397981305, + "mrr_at_20": 0.4196951284010108, + "mrr_at_3": 0.38500000000000006, + "mrr_at_5": 0.4035, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.05718954248365932, + "nauc_cv_recall_at_100_max": 0.5615662931839402, + "nauc_cv_recall_at_100_std": 0.7516923436041081, + "nauc_cv_recall_at_10_diff1": -0.19090585905097338, + "nauc_cv_recall_at_10_max": 0.6584110646510131, + "nauc_cv_recall_at_10_std": 0.5330026578185566, + "nauc_cv_recall_at_1_diff1": 0.16598692664394651, + "nauc_cv_recall_at_1_max": 0.4452623051567126, + "nauc_cv_recall_at_1_std": 0.3140538577574166, + "nauc_cv_recall_at_20_diff1": -0.16275966866186092, + "nauc_cv_recall_at_20_max": 0.646906213516669, + "nauc_cv_recall_at_20_std": 0.5563256305246183, + "nauc_cv_recall_at_3_diff1": -0.096122584943371, + "nauc_cv_recall_at_3_max": 0.6684610259826782, + "nauc_cv_recall_at_3_std": 0.4076215856095933, + "nauc_cv_recall_at_5_diff1": -0.1365179789302609, + "nauc_cv_recall_at_5_max": 0.5976289288098207, + "nauc_cv_recall_at_5_std": 0.37226107902603245, + "nauc_map_at_1000_diff1": 0.02290130273084715, + "nauc_map_at_1000_max": 0.548071312346396, + "nauc_map_at_1000_std": 0.3431509175694228, + "nauc_map_at_100_diff1": 0.023035522926500292, + "nauc_map_at_100_max": 0.5481243666012099, + "nauc_map_at_100_std": 0.343765707161693, + "nauc_map_at_10_diff1": 0.019009704736545754, + "nauc_map_at_10_max": 0.5515194516807846, + "nauc_map_at_10_std": 0.3449228525930436, + "nauc_map_at_1_diff1": 0.1698977596513771, + "nauc_map_at_1_max": 0.4552349293256607, + "nauc_map_at_1_std": 0.2536314877926139, + "nauc_map_at_20_diff1": 0.021258373975554346, + "nauc_map_at_20_max": 0.5501977396176536, + "nauc_map_at_20_std": 0.3451008448257592, + "nauc_map_at_3_diff1": 0.027947800501289654, + "nauc_map_at_3_max": 0.5615419194304191, + "nauc_map_at_3_std": 0.3374327981401433, + "nauc_map_at_5_diff1": 0.022444182090860222, + "nauc_map_at_5_max": 0.5443295202128533, + "nauc_map_at_5_std": 0.3265094163994626, + "nauc_mrr_at_1000_diff1": 0.037792737057000106, + "nauc_mrr_at_1000_max": 0.5306581923448304, + "nauc_mrr_at_1000_std": 0.36198678701734915, + "nauc_mrr_at_100_diff1": 0.037904244512512376, + "nauc_mrr_at_100_max": 0.5307371796485331, + "nauc_mrr_at_100_std": 0.3625738495305344, + "nauc_mrr_at_10_diff1": 0.0336578592994239, + "nauc_mrr_at_10_max": 0.5344197389534955, + "nauc_mrr_at_10_std": 0.3632912027558574, + "nauc_mrr_at_1_diff1": 0.16598692664394651, + "nauc_mrr_at_1_max": 0.4452623051567126, + "nauc_mrr_at_1_std": 0.3140538577574166, + "nauc_mrr_at_20_diff1": 0.03592134186381886, + "nauc_mrr_at_20_max": 0.5330587540055658, + "nauc_mrr_at_20_std": 0.36365157309531015, + "nauc_mrr_at_3_diff1": 0.04203509201841499, + "nauc_mrr_at_3_max": 0.5450312778658897, + "nauc_mrr_at_3_std": 0.35278982227092975, + "nauc_mrr_at_5_diff1": 0.036600215680544855, + "nauc_mrr_at_5_max": 0.5283891078779852, + "nauc_mrr_at_5_std": 0.3444903245689106, + "nauc_ndcg_at_1000_diff1": 0.0005409786282632852, + "nauc_ndcg_at_1000_max": 0.557765371086238, + "nauc_ndcg_at_1000_std": 0.3620874514374281, + "nauc_ndcg_at_100_diff1": -0.0004227472957878371, + "nauc_ndcg_at_100_max": 0.5581112142072867, + "nauc_ndcg_at_100_std": 0.3770164912688396, + "nauc_ndcg_at_10_diff1": -0.027451154381326896, + "nauc_ndcg_at_10_max": 0.5759121776535603, + "nauc_ndcg_at_10_std": 0.3837759064006119, + "nauc_ndcg_at_1_diff1": 0.1698977596513771, + "nauc_ndcg_at_1_max": 0.4552349293256607, + "nauc_ndcg_at_1_std": 0.2536314877926139, + "nauc_ndcg_at_20_diff1": -0.018923679643943057, + "nauc_ndcg_at_20_max": 0.5717611328008967, + "nauc_ndcg_at_20_std": 0.3855482939373467, + "nauc_ndcg_at_3_diff1": -0.005345292806580259, + "nauc_ndcg_at_3_max": 0.589864220491107, + "nauc_ndcg_at_3_std": 0.35634772055921166, + "nauc_ndcg_at_5_diff1": -0.01736592260642203, + "nauc_ndcg_at_5_max": 0.559198032314247, + "nauc_ndcg_at_5_std": 0.3388356749504251, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": -0.05718954248365978, + "nauc_precision_at_100_max": 0.5615662931839398, + "nauc_precision_at_100_std": 0.751692343604107, + "nauc_precision_at_10_diff1": -0.1909058590509713, + "nauc_precision_at_10_max": 0.6584110646510136, + "nauc_precision_at_10_std": 0.5330026578185574, + "nauc_precision_at_1_diff1": 0.1698977596513771, + "nauc_precision_at_1_max": 0.4552349293256607, + "nauc_precision_at_1_std": 0.2536314877926139, + "nauc_precision_at_20_diff1": -0.16275966866186015, + "nauc_precision_at_20_max": 0.6469062135166688, + "nauc_precision_at_20_std": 0.5563256305246184, + "nauc_precision_at_3_diff1": -0.09612258494337084, + "nauc_precision_at_3_max": 0.6684610259826779, + "nauc_precision_at_3_std": 0.4076215856095937, + "nauc_precision_at_5_diff1": -0.13651797893026002, + "nauc_precision_at_5_max": 0.597628928809821, + "nauc_precision_at_5_std": 0.3722610790260329, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.05718954248365932, + "nauc_recall_at_100_max": 0.5615662931839402, + "nauc_recall_at_100_std": 0.7516923436041081, + "nauc_recall_at_10_diff1": -0.19090585905097338, + "nauc_recall_at_10_max": 0.6584110646510131, + "nauc_recall_at_10_std": 0.5330026578185566, + "nauc_recall_at_1_diff1": 0.1698977596513771, + "nauc_recall_at_1_max": 0.4552349293256607, + "nauc_recall_at_1_std": 0.2536314877926139, + "nauc_recall_at_20_diff1": -0.16275966866186092, + "nauc_recall_at_20_max": 0.646906213516669, + "nauc_recall_at_20_std": 0.5563256305246183, + "nauc_recall_at_3_diff1": -0.096122584943371, + "nauc_recall_at_3_max": 0.6684610259826782, + "nauc_recall_at_3_std": 0.4076215856095933, + "nauc_recall_at_5_diff1": -0.1365179789302609, + "nauc_recall_at_5_max": 0.5976289288098207, + "nauc_recall_at_5_std": 0.37226107902603245, + "ndcg_at_1": 0.28, + "ndcg_at_10": 0.47865, + "ndcg_at_100": 0.52677, + "ndcg_at_1000": 0.53753, + "ndcg_at_20": 0.48907, + "ndcg_at_3": 0.41619, + "ndcg_at_5": 0.44845, + "precision_at_1": 0.28, + "precision_at_10": 0.067, + "precision_at_100": 0.0092, + "precision_at_1000": 0.001, + "precision_at_20": 0.0355, + "precision_at_3": 0.16667, + "precision_at_5": 0.116, + "recall_at_1": 0.28, + "recall_at_10": 0.67, + "recall_at_100": 0.92, + "recall_at_1000": 1.0, + "recall_at_20": 0.71, + "recall_at_3": 0.5, + "recall_at_5": 0.58 + } + ] + }, + "task_name": "VidoreSyntheticDocQAEnergyRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..c0bd444fb8 --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", + "evaluation_time": 73.8860080242157, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.18, + "cv_recall_at_10": 0.53, + "cv_recall_at_100": 0.86, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.65, + "cv_recall_at_3": 0.38, + "cv_recall_at_5": 0.47, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.33103, + "map_at_1": 0.17, + "map_at_10": 0.29279, + "map_at_100": 0.30536, + "map_at_1000": 0.30619, + "map_at_20": 0.30089, + "map_at_3": 0.26667, + "map_at_5": 0.28467, + "mrr_at_1": 0.18, + "mrr_at_10": 0.29542857142857143, + "mrr_at_100": 0.3081013748465196, + "mrr_at_1000": 0.30893080980505994, + "mrr_at_20": 0.3036155927405928, + "mrr_at_3": 0.26666666666666666, + "mrr_at_5": 0.2871666666666666, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.483303311625328, + "nauc_cv_recall_at_100_max": 0.5174241374532345, + "nauc_cv_recall_at_100_std": 0.6429264237217674, + "nauc_cv_recall_at_10_diff1": -0.4115383029319991, + "nauc_cv_recall_at_10_max": 0.709782952342987, + "nauc_cv_recall_at_10_std": 0.48667335633479486, + "nauc_cv_recall_at_1_diff1": -0.09549871943304222, + "nauc_cv_recall_at_1_max": 0.46247763393327, + "nauc_cv_recall_at_1_std": 0.2309055187173279, + "nauc_cv_recall_at_20_diff1": -0.3380421515176176, + "nauc_cv_recall_at_20_max": 0.5907722353324105, + "nauc_cv_recall_at_20_std": 0.5878102367971356, + "nauc_cv_recall_at_3_diff1": -0.3586021365633043, + "nauc_cv_recall_at_3_max": 0.7061315727913083, + "nauc_cv_recall_at_3_std": 0.3685571699633508, + "nauc_cv_recall_at_5_diff1": -0.4414483313334829, + "nauc_cv_recall_at_5_max": 0.6931262279302206, + "nauc_cv_recall_at_5_std": 0.3961286409367214, + "nauc_map_at_1000_diff1": -0.22866744303816647, + "nauc_map_at_1000_max": 0.6739853793849305, + "nauc_map_at_1000_std": 0.3932776656219446, + "nauc_map_at_100_diff1": -0.22930027603884356, + "nauc_map_at_100_max": 0.6735362617611932, + "nauc_map_at_100_std": 0.3939927989346046, + "nauc_map_at_10_diff1": -0.2395548159357987, + "nauc_map_at_10_max": 0.6804946733702065, + "nauc_map_at_10_std": 0.3909649841540425, + "nauc_map_at_1_diff1": -0.06567777186967136, + "nauc_map_at_1_max": 0.6634701773043143, + "nauc_map_at_1_std": 0.37842644712957696, + "nauc_map_at_20_diff1": -0.2309939061113582, + "nauc_map_at_20_max": 0.6719671219524078, + "nauc_map_at_20_std": 0.3961538541197034, + "nauc_map_at_3_diff1": -0.20825225408792225, + "nauc_map_at_3_max": 0.6758876267128524, + "nauc_map_at_3_std": 0.36635678263970267, + "nauc_map_at_5_diff1": -0.24204925924807408, + "nauc_map_at_5_max": 0.6775739229114318, + "nauc_map_at_5_std": 0.37794651270997526, + "nauc_mrr_at_1000_diff1": -0.24080560474639628, + "nauc_mrr_at_1000_max": 0.5801836691701273, + "nauc_mrr_at_1000_std": 0.31892126838980106, + "nauc_mrr_at_100_diff1": -0.24139498766695452, + "nauc_mrr_at_100_max": 0.579984465673827, + "nauc_mrr_at_100_std": 0.3198150057038841, + "nauc_mrr_at_10_diff1": -0.2508554519682481, + "nauc_mrr_at_10_max": 0.5891124235800822, + "nauc_mrr_at_10_std": 0.31837517767996915, + "nauc_mrr_at_1_diff1": -0.09549871943304222, + "nauc_mrr_at_1_max": 0.46247763393327, + "nauc_mrr_at_1_std": 0.2309055187173279, + "nauc_mrr_at_20_diff1": -0.2429160804923638, + "nauc_mrr_at_20_max": 0.5794110337672059, + "nauc_mrr_at_20_std": 0.3227054157690821, + "nauc_mrr_at_3_diff1": -0.23502707309736295, + "nauc_mrr_at_3_max": 0.5891682542865738, + "nauc_mrr_at_3_std": 0.3009575854807983, + "nauc_mrr_at_5_diff1": -0.2539824551386642, + "nauc_mrr_at_5_max": 0.5869243320318603, + "nauc_mrr_at_5_std": 0.3056964583546606, + "nauc_ndcg_at_1000_diff1": -0.24877049649113786, + "nauc_ndcg_at_1000_max": 0.6716904547953125, + "nauc_ndcg_at_1000_std": 0.4072564448685426, + "nauc_ndcg_at_100_diff1": -0.2639017194977542, + "nauc_ndcg_at_100_max": 0.6613111300385416, + "nauc_ndcg_at_100_std": 0.42279146749234503, + "nauc_ndcg_at_10_diff1": -0.2879680148626686, + "nauc_ndcg_at_10_max": 0.6881786699043809, + "nauc_ndcg_at_10_std": 0.41377797946754535, + "nauc_ndcg_at_1_diff1": -0.06567777186967136, + "nauc_ndcg_at_1_max": 0.6634701773043143, + "nauc_ndcg_at_1_std": 0.37842644712957696, + "nauc_ndcg_at_20_diff1": -0.26264273370370655, + "nauc_ndcg_at_20_max": 0.658052249712014, + "nauc_ndcg_at_20_std": 0.4345548933087738, + "nauc_ndcg_at_3_diff1": -0.23793103616980857, + "nauc_ndcg_at_3_max": 0.6792223783252196, + "nauc_ndcg_at_3_std": 0.36358450135691267, + "nauc_ndcg_at_5_diff1": -0.29747560271749857, + "nauc_ndcg_at_5_max": 0.6819612247682432, + "nauc_ndcg_at_5_std": 0.38260912730366553, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": -0.4833033116253274, + "nauc_precision_at_100_max": 0.5174241374532372, + "nauc_precision_at_100_std": 0.6429264237217707, + "nauc_precision_at_10_diff1": -0.411538302931999, + "nauc_precision_at_10_max": 0.7097829523429878, + "nauc_precision_at_10_std": 0.48667335633479564, + "nauc_precision_at_1_diff1": -0.06567777186967136, + "nauc_precision_at_1_max": 0.6634701773043143, + "nauc_precision_at_1_std": 0.37842644712957696, + "nauc_precision_at_20_diff1": -0.3380421515176169, + "nauc_precision_at_20_max": 0.5907722353324113, + "nauc_precision_at_20_std": 0.5878102367971358, + "nauc_precision_at_3_diff1": -0.30869323399687, + "nauc_precision_at_3_max": 0.6874264353989625, + "nauc_precision_at_3_std": 0.35688694010062977, + "nauc_precision_at_5_diff1": -0.44144833133348227, + "nauc_precision_at_5_max": 0.6931262279302212, + "nauc_precision_at_5_std": 0.3961286409367218, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.483303311625328, + "nauc_recall_at_100_max": 0.5174241374532345, + "nauc_recall_at_100_std": 0.6429264237217674, + "nauc_recall_at_10_diff1": -0.4115383029319991, + "nauc_recall_at_10_max": 0.709782952342987, + "nauc_recall_at_10_std": 0.48667335633479486, + "nauc_recall_at_1_diff1": -0.06567777186967136, + "nauc_recall_at_1_max": 0.6634701773043143, + "nauc_recall_at_1_std": 0.37842644712957696, + "nauc_recall_at_20_diff1": -0.3380421515176176, + "nauc_recall_at_20_max": 0.5907722353324105, + "nauc_recall_at_20_std": 0.5878102367971356, + "nauc_recall_at_3_diff1": -0.3086932339968699, + "nauc_recall_at_3_max": 0.6874264353989629, + "nauc_recall_at_3_std": 0.35688694010062993, + "nauc_recall_at_5_diff1": -0.4414483313334829, + "nauc_recall_at_5_max": 0.6931262279302206, + "nauc_recall_at_5_std": 0.3961286409367214, + "ndcg_at_1": 0.17, + "ndcg_at_10": 0.35054, + "ndcg_at_100": 0.41772, + "ndcg_at_1000": 0.43636, + "ndcg_at_20": 0.38063, + "ndcg_at_3": 0.29833, + "ndcg_at_5": 0.33103, + "precision_at_1": 0.17, + "precision_at_10": 0.053, + "precision_at_100": 0.0086, + "precision_at_1000": 0.001, + "precision_at_20": 0.0325, + "precision_at_3": 0.13, + "precision_at_5": 0.094, + "recall_at_1": 0.17, + "recall_at_10": 0.53, + "recall_at_100": 0.86, + "recall_at_1000": 1.0, + "recall_at_20": 0.65, + "recall_at_3": 0.39, + "recall_at_5": 0.47 + } + ] + }, + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..f80c15a971 --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", + "evaluation_time": 72.91890263557434, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.22, + "cv_recall_at_10": 0.6, + "cv_recall_at_100": 0.91, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.7, + "cv_recall_at_3": 0.49, + "cv_recall_at_5": 0.52, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.38779, + "map_at_1": 0.21, + "map_at_10": 0.35287, + "map_at_100": 0.36535, + "map_at_1000": 0.3659, + "map_at_20": 0.36013, + "map_at_3": 0.33667, + "map_at_5": 0.34267, + "mrr_at_1": 0.22, + "mrr_at_10": 0.355218253968254, + "mrr_at_100": 0.36766650196348827, + "mrr_at_1000": 0.3682164935395407, + "mrr_at_20": 0.36243612819928617, + "mrr_at_3": 0.3383333333333334, + "mrr_at_5": 0.34483333333333327, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.003267973856210241, + "nauc_cv_recall_at_100_max": 0.5420686793235807, + "nauc_cv_recall_at_100_std": 0.7533457827575475, + "nauc_cv_recall_at_10_diff1": -0.2672899970317603, + "nauc_cv_recall_at_10_max": 0.6763431285247846, + "nauc_cv_recall_at_10_std": 0.5030424458296231, + "nauc_cv_recall_at_1_diff1": -0.22708767131802224, + "nauc_cv_recall_at_1_max": 0.46022003294278496, + "nauc_cv_recall_at_1_std": 0.11752183236473257, + "nauc_cv_recall_at_20_diff1": -0.27616645649432564, + "nauc_cv_recall_at_20_max": 0.6204287515762922, + "nauc_cv_recall_at_20_std": 0.5124482075301748, + "nauc_cv_recall_at_3_diff1": -0.2874005815729024, + "nauc_cv_recall_at_3_max": 0.6736420007170077, + "nauc_cv_recall_at_3_std": 0.37669459455870835, + "nauc_cv_recall_at_5_diff1": -0.2977311061537627, + "nauc_cv_recall_at_5_max": 0.7365063387561889, + "nauc_cv_recall_at_5_std": 0.4491131182327658, + "nauc_map_at_1000_diff1": -0.24421377564817673, + "nauc_map_at_1000_max": 0.6057575097002678, + "nauc_map_at_1000_std": 0.3563653280500335, + "nauc_map_at_100_diff1": -0.2438208029837478, + "nauc_map_at_100_max": 0.6056381919626572, + "nauc_map_at_100_std": 0.3571056403427818, + "nauc_map_at_10_diff1": -0.2474064341034369, + "nauc_map_at_10_max": 0.6111031262320629, + "nauc_map_at_10_std": 0.3583715496190217, + "nauc_map_at_1_diff1": -0.19750740336041306, + "nauc_map_at_1_max": 0.5460116870160101, + "nauc_map_at_1_std": 0.3261120876684563, + "nauc_map_at_20_diff1": -0.24703318252196604, + "nauc_map_at_20_max": 0.6060923108554881, + "nauc_map_at_20_std": 0.3571216123471924, + "nauc_map_at_3_diff1": -0.24964070479347697, + "nauc_map_at_3_max": 0.6057022544576557, + "nauc_map_at_3_std": 0.33612306100353556, + "nauc_map_at_5_diff1": -0.25163894718289853, + "nauc_map_at_5_max": 0.6187024411572932, + "nauc_map_at_5_std": 0.3512947369738972, + "nauc_mrr_at_1000_diff1": -0.26262047817609174, + "nauc_mrr_at_1000_max": 0.5657657182275748, + "nauc_mrr_at_1000_std": 0.2676332247249374, + "nauc_mrr_at_100_diff1": -0.26219833115320174, + "nauc_mrr_at_100_max": 0.565716644353467, + "nauc_mrr_at_100_std": 0.2685234476168615, + "nauc_mrr_at_10_diff1": -0.26525556917234877, + "nauc_mrr_at_10_max": 0.5720441464757696, + "nauc_mrr_at_10_std": 0.27167563037113773, + "nauc_mrr_at_1_diff1": -0.22708767131802224, + "nauc_mrr_at_1_max": 0.46022003294278496, + "nauc_mrr_at_1_std": 0.11752183236473257, + "nauc_mrr_at_20_diff1": -0.265224379144156, + "nauc_mrr_at_20_max": 0.566706960324191, + "nauc_mrr_at_20_std": 0.2696353634782542, + "nauc_mrr_at_3_diff1": -0.2671988902639183, + "nauc_mrr_at_3_max": 0.5673596974829038, + "nauc_mrr_at_3_std": 0.2520896624948636, + "nauc_mrr_at_5_diff1": -0.2695098953901908, + "nauc_mrr_at_5_max": 0.5799741955943382, + "nauc_mrr_at_5_std": 0.2656432940902274, + "nauc_ndcg_at_1000_diff1": -0.24407529554426993, + "nauc_ndcg_at_1000_max": 0.611964249388691, + "nauc_ndcg_at_1000_std": 0.37296921118890486, + "nauc_ndcg_at_100_diff1": -0.23370771712739602, + "nauc_ndcg_at_100_max": 0.6088627125657947, + "nauc_ndcg_at_100_std": 0.38985170913350337, + "nauc_ndcg_at_10_diff1": -0.2538099257653232, + "nauc_ndcg_at_10_max": 0.6289827458486196, + "nauc_ndcg_at_10_std": 0.39168461686955036, + "nauc_ndcg_at_1_diff1": -0.19750740336041306, + "nauc_ndcg_at_1_max": 0.5460116870160101, + "nauc_ndcg_at_1_std": 0.3261120876684563, + "nauc_ndcg_at_20_diff1": -0.25434363734303483, + "nauc_ndcg_at_20_max": 0.6135309338030498, + "nauc_ndcg_at_20_std": 0.3886075994148953, + "nauc_ndcg_at_3_diff1": -0.26026416495381194, + "nauc_ndcg_at_3_max": 0.6240980080777259, + "nauc_ndcg_at_3_std": 0.34656841722715415, + "nauc_ndcg_at_5_diff1": -0.2641693037611579, + "nauc_ndcg_at_5_max": 0.6490032981064092, + "nauc_ndcg_at_5_std": 0.37552492374435203, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": -0.003267973856207883, + "nauc_precision_at_100_max": 0.542068679323581, + "nauc_precision_at_100_std": 0.75334578275755, + "nauc_precision_at_10_diff1": -0.26728999703175943, + "nauc_precision_at_10_max": 0.6763431285247852, + "nauc_precision_at_10_std": 0.503042445829623, + "nauc_precision_at_1_diff1": -0.19750740336041306, + "nauc_precision_at_1_max": 0.5460116870160101, + "nauc_precision_at_1_std": 0.3261120876684563, + "nauc_precision_at_20_diff1": -0.27616645649432475, + "nauc_precision_at_20_max": 0.6204287515762927, + "nauc_precision_at_20_std": 0.5124482075301745, + "nauc_precision_at_3_diff1": -0.2874005815729024, + "nauc_precision_at_3_max": 0.6736420007170074, + "nauc_precision_at_3_std": 0.37669459455870846, + "nauc_precision_at_5_diff1": -0.29773110615376197, + "nauc_precision_at_5_max": 0.736506338756189, + "nauc_precision_at_5_std": 0.449113118232766, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.003267973856210241, + "nauc_recall_at_100_max": 0.5420686793235807, + "nauc_recall_at_100_std": 0.7533457827575475, + "nauc_recall_at_10_diff1": -0.2672899970317603, + "nauc_recall_at_10_max": 0.6763431285247846, + "nauc_recall_at_10_std": 0.5030424458296231, + "nauc_recall_at_1_diff1": -0.19750740336041306, + "nauc_recall_at_1_max": 0.5460116870160101, + "nauc_recall_at_1_std": 0.3261120876684563, + "nauc_recall_at_20_diff1": -0.27616645649432564, + "nauc_recall_at_20_max": 0.6204287515762922, + "nauc_recall_at_20_std": 0.5124482075301748, + "nauc_recall_at_3_diff1": -0.2874005815729024, + "nauc_recall_at_3_max": 0.6736420007170077, + "nauc_recall_at_3_std": 0.37669459455870835, + "nauc_recall_at_5_diff1": -0.2977311061537627, + "nauc_recall_at_5_max": 0.7365063387561889, + "nauc_recall_at_5_std": 0.4491131182327658, + "ndcg_at_1": 0.21, + "ndcg_at_10": 0.41321, + "ndcg_at_100": 0.4775, + "ndcg_at_1000": 0.48958, + "ndcg_at_20": 0.43887, + "ndcg_at_3": 0.37619, + "ndcg_at_5": 0.38779, + "precision_at_1": 0.21, + "precision_at_10": 0.06, + "precision_at_100": 0.0091, + "precision_at_1000": 0.001, + "precision_at_20": 0.035, + "precision_at_3": 0.16333, + "precision_at_5": 0.104, + "recall_at_1": 0.21, + "recall_at_10": 0.6, + "recall_at_100": 0.91, + "recall_at_1000": 1.0, + "recall_at_20": 0.7, + "recall_at_3": 0.49, + "recall_at_5": 0.52 + } + ] + }, + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..85526aa435 --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", + "evaluation_time": 17.741925716400146, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.13571, + "cv_recall_at_10": 0.58929, + "cv_recall_at_100": 0.86071, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.7, + "cv_recall_at_3": 0.375, + "cv_recall_at_5": 0.48929, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.30532, + "map_at_1": 0.11786, + "map_at_10": 0.26471, + "map_at_100": 0.27589, + "map_at_1000": 0.27681, + "map_at_20": 0.27145, + "map_at_3": 0.21905, + "map_at_5": 0.2478, + "mrr_at_1": 0.1357142857142857, + "mrr_at_10": 0.275654761904762, + "mrr_at_100": 0.2882710170373999, + "mrr_at_1000": 0.28918278167491335, + "mrr_at_20": 0.28371689890224294, + "mrr_at_3": 0.23452380952380958, + "mrr_at_5": 0.26184523809523824, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.0962614034651627, + "nauc_cv_recall_at_100_max": 0.48130701732581055, + "nauc_cv_recall_at_100_std": 0.5359442193442552, + "nauc_cv_recall_at_10_diff1": 0.07595632707564201, + "nauc_cv_recall_at_10_max": 0.5214463383342498, + "nauc_cv_recall_at_10_std": 0.34694315712303286, + "nauc_cv_recall_at_1_diff1": 0.023979663013656137, + "nauc_cv_recall_at_1_max": 0.11585261885201438, + "nauc_cv_recall_at_1_std": 0.039449413296544805, + "nauc_cv_recall_at_20_diff1": -0.013504645237666892, + "nauc_cv_recall_at_20_max": 0.45511104820238296, + "nauc_cv_recall_at_20_std": 0.3636088735620353, + "nauc_cv_recall_at_3_diff1": 0.12455393491055446, + "nauc_cv_recall_at_3_max": 0.34790052806662475, + "nauc_cv_recall_at_3_std": 0.24388768697513188, + "nauc_cv_recall_at_5_diff1": 0.10118096610707866, + "nauc_cv_recall_at_5_max": 0.48999426347254305, + "nauc_cv_recall_at_5_std": 0.338028436108833, + "nauc_map_at_1000_diff1": 0.14177585744325438, + "nauc_map_at_1000_max": 0.25267701349330973, + "nauc_map_at_1000_std": 0.16204571721159644, + "nauc_map_at_100_diff1": 0.14092375993422507, + "nauc_map_at_100_max": 0.2534882998637386, + "nauc_map_at_100_std": 0.16331431723111056, + "nauc_map_at_10_diff1": 0.14448945692889367, + "nauc_map_at_10_max": 0.25899654113563375, + "nauc_map_at_10_std": 0.1597103307680331, + "nauc_map_at_1_diff1": 0.18523279430290246, + "nauc_map_at_1_max": 0.026814407280767123, + "nauc_map_at_1_std": 0.009923644497004887, + "nauc_map_at_20_diff1": 0.1403834299409469, + "nauc_map_at_20_max": 0.25535009456183255, + "nauc_map_at_20_std": 0.16250878288417944, + "nauc_map_at_3_diff1": 0.1481805843215222, + "nauc_map_at_3_max": 0.24167493327909895, + "nauc_map_at_3_std": 0.15671525636183276, + "nauc_map_at_5_diff1": 0.14976730485212275, + "nauc_map_at_5_max": 0.26080880741487417, + "nauc_map_at_5_std": 0.15898524258324853, + "nauc_mrr_at_1000_diff1": 0.053836794776765395, + "nauc_mrr_at_1000_max": 0.2751316825025726, + "nauc_mrr_at_1000_std": 0.1735453731971639, + "nauc_mrr_at_100_diff1": 0.053306077293060236, + "nauc_mrr_at_100_max": 0.2758338657361141, + "nauc_mrr_at_100_std": 0.17472166458524652, + "nauc_mrr_at_10_diff1": 0.059631240474482726, + "nauc_mrr_at_10_max": 0.2861178647684893, + "nauc_mrr_at_10_std": 0.176909107436406, + "nauc_mrr_at_1_diff1": 0.023979663013656137, + "nauc_mrr_at_1_max": 0.11585261885201438, + "nauc_mrr_at_1_std": 0.039449413296544805, + "nauc_mrr_at_20_diff1": 0.05378318892025374, + "nauc_mrr_at_20_max": 0.27730629708974225, + "nauc_mrr_at_20_std": 0.17378722632932975, + "nauc_mrr_at_3_diff1": 0.07204267342282876, + "nauc_mrr_at_3_max": 0.23918521004131926, + "nauc_mrr_at_3_std": 0.1519262805303006, + "nauc_mrr_at_5_diff1": 0.06501788292981979, + "nauc_mrr_at_5_max": 0.2838511219515895, + "nauc_mrr_at_5_std": 0.17950227251107553, + "nauc_ndcg_at_1000_diff1": 0.12681348634395276, + "nauc_ndcg_at_1000_max": 0.284330263153128, + "nauc_ndcg_at_1000_std": 0.19038243430316637, + "nauc_ndcg_at_100_diff1": 0.10951958445625287, + "nauc_ndcg_at_100_max": 0.29955187926203075, + "nauc_ndcg_at_100_std": 0.2167965732577945, + "nauc_ndcg_at_10_diff1": 0.12444522627830684, + "nauc_ndcg_at_10_max": 0.32109399057607235, + "nauc_ndcg_at_10_std": 0.1984173977368013, + "nauc_ndcg_at_1_diff1": 0.18523279430290246, + "nauc_ndcg_at_1_max": 0.026814407280767123, + "nauc_ndcg_at_1_std": 0.009923644497004887, + "nauc_ndcg_at_20_diff1": 0.1070015172410676, + "nauc_ndcg_at_20_max": 0.3099995592251142, + "nauc_ndcg_at_20_std": 0.21140392873639405, + "nauc_ndcg_at_3_diff1": 0.13374110988425314, + "nauc_ndcg_at_3_max": 0.2965332376419493, + "nauc_ndcg_at_3_std": 0.19616272589026296, + "nauc_ndcg_at_5_diff1": 0.13580773833733265, + "nauc_ndcg_at_5_max": 0.32598979522729604, + "nauc_ndcg_at_5_std": 0.198633639239219, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": -0.09626140346516247, + "nauc_precision_at_100_max": 0.4813070173258093, + "nauc_precision_at_100_std": 0.5359442193442547, + "nauc_precision_at_10_diff1": 0.07110205518543622, + "nauc_precision_at_10_max": 0.4711617958469575, + "nauc_precision_at_10_std": 0.29049963462184697, + "nauc_precision_at_1_diff1": 0.18523279430290246, + "nauc_precision_at_1_max": 0.026814407280767123, + "nauc_precision_at_1_std": 0.009923644497004887, + "nauc_precision_at_20_diff1": -0.013504645237666276, + "nauc_precision_at_20_max": 0.4551110482023827, + "nauc_precision_at_20_std": 0.3636088735620348, + "nauc_precision_at_3_diff1": 0.10070409009355073, + "nauc_precision_at_3_max": 0.41801308867412745, + "nauc_precision_at_3_std": 0.28393615875053274, + "nauc_precision_at_5_diff1": 0.1035717322098263, + "nauc_precision_at_5_max": 0.47273732039788774, + "nauc_precision_at_5_std": 0.28682020423393356, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.0962614034651627, + "nauc_recall_at_100_max": 0.48130701732581055, + "nauc_recall_at_100_std": 0.5359442193442552, + "nauc_recall_at_10_diff1": 0.07110205518543654, + "nauc_recall_at_10_max": 0.47116179584695755, + "nauc_recall_at_10_std": 0.29049963462184736, + "nauc_recall_at_1_diff1": 0.18523279430290246, + "nauc_recall_at_1_max": 0.026814407280767123, + "nauc_recall_at_1_std": 0.009923644497004887, + "nauc_recall_at_20_diff1": -0.013504645237666892, + "nauc_recall_at_20_max": 0.45511104820238296, + "nauc_recall_at_20_std": 0.3636088735620353, + "nauc_recall_at_3_diff1": 0.10070409009355087, + "nauc_recall_at_3_max": 0.4180130886741277, + "nauc_recall_at_3_std": 0.283936158750533, + "nauc_recall_at_5_diff1": 0.10357173220982632, + "nauc_recall_at_5_max": 0.4727373203978876, + "nauc_recall_at_5_std": 0.28682020423393373, + "ndcg_at_1": 0.11786, + "ndcg_at_10": 0.34597, + "ndcg_at_100": 0.40068, + "ndcg_at_1000": 0.41976, + "ndcg_at_20": 0.37041, + "ndcg_at_3": 0.25518, + "ndcg_at_5": 0.30532, + "precision_at_1": 0.11786, + "precision_at_10": 0.06036, + "precision_at_100": 0.00861, + "precision_at_1000": 0.001, + "precision_at_20": 0.035, + "precision_at_3": 0.12024, + "precision_at_5": 0.09571, + "recall_at_1": 0.11786, + "recall_at_10": 0.60357, + "recall_at_100": 0.86071, + "recall_at_1000": 1.0, + "recall_at_20": 0.7, + "recall_at_3": 0.36071, + "recall_at_5": 0.47857 + } + ] + }, + "task_name": "VidoreTabfquadRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..29032ce4c6 --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", + "evaluation_time": 126.18415856361389, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.01744, + "cv_recall_at_10": 0.14311, + "cv_recall_at_100": 0.37703, + "cv_recall_at_1000": 0.87072, + "cv_recall_at_20": 0.19964, + "cv_recall_at_3": 0.05532, + "cv_recall_at_5": 0.0926, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.05505, + "map_at_1": 0.01804, + "map_at_10": 0.04953, + "map_at_100": 0.05703, + "map_at_1000": 0.05869, + "map_at_20": 0.05307, + "map_at_3": 0.03397, + "map_at_5": 0.04248, + "mrr_at_1": 0.017438364401683705, + "mrr_at_10": 0.04915671620422074, + "mrr_at_100": 0.0569115235221346, + "mrr_at_1000": 0.05857065878278318, + "mrr_at_20": 0.053008722607474607, + "mrr_at_3": 0.033674082982561644, + "mrr_at_5": 0.04206253758268193, + "nauc_cv_recall_at_1000_diff1": 0.08237134389710216, + "nauc_cv_recall_at_1000_max": 0.20976894796503254, + "nauc_cv_recall_at_1000_std": 0.4140935117522756, + "nauc_cv_recall_at_100_diff1": -0.02981197371782488, + "nauc_cv_recall_at_100_max": 0.20240240199104623, + "nauc_cv_recall_at_100_std": 0.30510424487535914, + "nauc_cv_recall_at_10_diff1": -0.035047295007147705, + "nauc_cv_recall_at_10_max": 0.3038530304180975, + "nauc_cv_recall_at_10_std": 0.3220085766696133, + "nauc_cv_recall_at_1_diff1": -0.032626900215017456, + "nauc_cv_recall_at_1_max": 0.27684455719867385, + "nauc_cv_recall_at_1_std": 0.30979043565912767, + "nauc_cv_recall_at_20_diff1": -0.02501748001370101, + "nauc_cv_recall_at_20_max": 0.2987998775731927, + "nauc_cv_recall_at_20_std": 0.3289343312056541, + "nauc_cv_recall_at_3_diff1": -0.012493786516726318, + "nauc_cv_recall_at_3_max": 0.35062308469531805, + "nauc_cv_recall_at_3_std": 0.328063698293921, + "nauc_cv_recall_at_5_diff1": -0.02014835181508689, + "nauc_cv_recall_at_5_max": 0.30597083566947125, + "nauc_cv_recall_at_5_std": 0.3136339145032427, + "nauc_map_at_1000_diff1": -0.04875525076147121, + "nauc_map_at_1000_max": 0.2815787298378842, + "nauc_map_at_1000_std": 0.3731437421002045, + "nauc_map_at_100_diff1": -0.04942990178875301, + "nauc_map_at_100_max": 0.28124633969736407, + "nauc_map_at_100_std": 0.37218323907682854, + "nauc_map_at_10_diff1": -0.051458156860137806, + "nauc_map_at_10_max": 0.29098313852215413, + "nauc_map_at_10_std": 0.37750246377662106, + "nauc_map_at_1_diff1": -0.10777077067663393, + "nauc_map_at_1_max": 0.25176210516240183, + "nauc_map_at_1_std": 0.4592822617825615, + "nauc_map_at_20_diff1": -0.04802954975755555, + "nauc_map_at_20_max": 0.2913329148831261, + "nauc_map_at_20_std": 0.3747903249592789, + "nauc_map_at_3_diff1": -0.04904890787369823, + "nauc_map_at_3_max": 0.26399306382725835, + "nauc_map_at_3_std": 0.4183070347637387, + "nauc_map_at_5_diff1": -0.05486746702234904, + "nauc_map_at_5_max": 0.28519023906167823, + "nauc_map_at_5_std": 0.3825112354018944, + "nauc_mrr_at_1000_diff1": -0.024517826873781616, + "nauc_mrr_at_1000_max": 0.3058381571189165, + "nauc_mrr_at_1000_std": 0.3247519566076449, + "nauc_mrr_at_100_diff1": -0.025390095142045613, + "nauc_mrr_at_100_max": 0.30521995190372303, + "nauc_mrr_at_100_std": 0.3240834016745154, + "nauc_mrr_at_10_diff1": -0.02579739125808308, + "nauc_mrr_at_10_max": 0.314897749755248, + "nauc_mrr_at_10_std": 0.32324109310365073, + "nauc_mrr_at_1_diff1": -0.032626900215017456, + "nauc_mrr_at_1_max": 0.27684455719867385, + "nauc_mrr_at_1_std": 0.30979043565912767, + "nauc_mrr_at_20_diff1": -0.02365953039025371, + "nauc_mrr_at_20_max": 0.313964555189022, + "nauc_mrr_at_20_std": 0.32474136190872116, + "nauc_mrr_at_3_diff1": -0.01416562201976808, + "nauc_mrr_at_3_max": 0.34082255893899743, + "nauc_mrr_at_3_std": 0.328619069068735, + "nauc_mrr_at_5_diff1": -0.018886104109413417, + "nauc_mrr_at_5_max": 0.32010631024417296, + "nauc_mrr_at_5_std": 0.322477663211533, + "nauc_ndcg_at_1000_diff1": -0.026807561051169607, + "nauc_ndcg_at_1000_max": 0.2649662980138261, + "nauc_ndcg_at_1000_std": 0.3629955058252187, + "nauc_ndcg_at_100_diff1": -0.03931666574317868, + "nauc_ndcg_at_100_max": 0.2543873594260332, + "nauc_ndcg_at_100_std": 0.3413208990403148, + "nauc_ndcg_at_10_diff1": -0.04356951193981768, + "nauc_ndcg_at_10_max": 0.3023339951965928, + "nauc_ndcg_at_10_std": 0.36022667617586135, + "nauc_ndcg_at_1_diff1": -0.10777077067663393, + "nauc_ndcg_at_1_max": 0.25176210516240183, + "nauc_ndcg_at_1_std": 0.4592822617825615, + "nauc_ndcg_at_20_diff1": -0.03658281028277188, + "nauc_ndcg_at_20_max": 0.3020490621335391, + "nauc_ndcg_at_20_std": 0.3559036914665498, + "nauc_ndcg_at_3_diff1": -0.040372611398516646, + "nauc_ndcg_at_3_max": 0.2616384429014375, + "nauc_ndcg_at_3_std": 0.41172672732998233, + "nauc_ndcg_at_5_diff1": -0.049645484907602344, + "nauc_ndcg_at_5_max": 0.29233089541831914, + "nauc_ndcg_at_5_std": 0.3638542090400901, + "nauc_precision_at_1000_diff1": 0.08237134389710085, + "nauc_precision_at_1000_max": 0.20976894796503143, + "nauc_precision_at_1000_std": 0.4140935117522751, + "nauc_precision_at_100_diff1": -0.027951192620319844, + "nauc_precision_at_100_max": 0.2060014267088026, + "nauc_precision_at_100_std": 0.30489286954854505, + "nauc_precision_at_10_diff1": -0.03373855863600513, + "nauc_precision_at_10_max": 0.3175505020337768, + "nauc_precision_at_10_std": 0.3400442978218774, + "nauc_precision_at_1_diff1": -0.10777077067663393, + "nauc_precision_at_1_max": 0.25176210516240183, + "nauc_precision_at_1_std": 0.4592822617825615, + "nauc_precision_at_20_diff1": -0.023503901593891596, + "nauc_precision_at_20_max": 0.3139544627736064, + "nauc_precision_at_20_std": 0.3366365571580702, + "nauc_precision_at_3_diff1": -0.025149804808346558, + "nauc_precision_at_3_max": 0.2566022001981372, + "nauc_precision_at_3_std": 0.40006698593126916, + "nauc_precision_at_5_diff1": -0.04284288904729967, + "nauc_precision_at_5_max": 0.30312075460709115, + "nauc_precision_at_5_std": 0.3366667367707371, + "nauc_recall_at_1000_diff1": 0.08237134389710216, + "nauc_recall_at_1000_max": 0.20976894796503254, + "nauc_recall_at_1000_std": 0.4140935117522756, + "nauc_recall_at_100_diff1": -0.02795119262032021, + "nauc_recall_at_100_max": 0.20600142670880198, + "nauc_recall_at_100_std": 0.3048928695485446, + "nauc_recall_at_10_diff1": -0.03373855863600514, + "nauc_recall_at_10_max": 0.31755050203377655, + "nauc_recall_at_10_std": 0.3400442978218772, + "nauc_recall_at_1_diff1": -0.10777077067663393, + "nauc_recall_at_1_max": 0.25176210516240183, + "nauc_recall_at_1_std": 0.4592822617825615, + "nauc_recall_at_20_diff1": -0.023503901593891666, + "nauc_recall_at_20_max": 0.313954462773606, + "nauc_recall_at_20_std": 0.33663655715806995, + "nauc_recall_at_3_diff1": -0.025149804808346467, + "nauc_recall_at_3_max": 0.25660220019813706, + "nauc_recall_at_3_std": 0.400066985931269, + "nauc_recall_at_5_diff1": -0.04284288904729986, + "nauc_recall_at_5_max": 0.3031207546070908, + "nauc_recall_at_5_std": 0.3366667367707366, + "ndcg_at_1": 0.01804, + "ndcg_at_10": 0.0718, + "ndcg_at_100": 0.11698, + "ndcg_at_1000": 0.17565, + "ndcg_at_20": 0.08502, + "ndcg_at_3": 0.03958, + "ndcg_at_5": 0.05505, + "precision_at_1": 0.01804, + "precision_at_10": 0.01449, + "precision_at_100": 0.00377, + "precision_at_1000": 0.00087, + "precision_at_20": 0.00989, + "precision_at_3": 0.01864, + "precision_at_5": 0.01876, + "recall_at_1": 0.01804, + "recall_at_10": 0.14492, + "recall_at_100": 0.37703, + "recall_at_1000": 0.87072, + "recall_at_20": 0.19784, + "recall_at_3": 0.05592, + "recall_at_5": 0.09381 + } + ] + }, + "task_name": "VidoreTatdqaRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/WITT2IRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/WITT2IRetrieval.json new file mode 100644 index 0000000000..664c54df0a --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/WITT2IRetrieval.json @@ -0,0 +1,1936 @@ +{ + "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", + "evaluation_time": 481.8342795372009, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.14607, + "cv_recall_at_10": 0.34719, + "cv_recall_at_100": 0.68427, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.4427, + "cv_recall_at_3": 0.23146, + "cv_recall_at_5": 0.27303, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.23827, + "map_at_1": 0.14607, + "map_at_10": 0.20461, + "map_at_100": 0.21649, + "map_at_1000": 0.21784, + "map_at_20": 0.21104, + "map_at_3": 0.18521, + "map_at_5": 0.19464, + "mrr_at_1": 0.14606741573033707, + "mrr_at_10": 0.2046107544141251, + "mrr_at_100": 0.2164916033094639, + "mrr_at_1000": 0.21784263443137813, + "mrr_at_20": 0.2110427533722138, + "mrr_at_3": 0.18520599250936332, + "mrr_at_5": 0.19464419475655428, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.26192336539007566, + "nauc_cv_recall_at_100_max": 0.5024390148377479, + "nauc_cv_recall_at_100_std": 0.5638540155133488, + "nauc_cv_recall_at_10_diff1": 0.3454828119738378, + "nauc_cv_recall_at_10_max": 0.3499346539030384, + "nauc_cv_recall_at_10_std": 0.3866041242955454, + "nauc_cv_recall_at_1_diff1": 0.5661591775441839, + "nauc_cv_recall_at_1_max": 0.4545038697008203, + "nauc_cv_recall_at_1_std": 0.4037703592468525, + "nauc_cv_recall_at_20_diff1": 0.3079409062489909, + "nauc_cv_recall_at_20_max": 0.3350925651419665, + "nauc_cv_recall_at_20_std": 0.36270066543579166, + "nauc_cv_recall_at_3_diff1": 0.45254007267605834, + "nauc_cv_recall_at_3_max": 0.3687574992052122, + "nauc_cv_recall_at_3_std": 0.3689113290557241, + "nauc_cv_recall_at_5_diff1": 0.42398776601066207, + "nauc_cv_recall_at_5_max": 0.3608852174216836, + "nauc_cv_recall_at_5_std": 0.3352722872510206, + "nauc_map_at_1000_diff1": 0.47511340162413196, + "nauc_map_at_1000_max": 0.4030154294650597, + "nauc_map_at_1000_std": 0.38565550112838487, + "nauc_map_at_100_diff1": 0.4742826036682546, + "nauc_map_at_100_max": 0.4032604036779924, + "nauc_map_at_100_std": 0.386053147898408, + "nauc_map_at_10_diff1": 0.478007665938371, + "nauc_map_at_10_max": 0.4001804786278926, + "nauc_map_at_10_std": 0.3829642722521861, + "nauc_map_at_1_diff1": 0.5661591775441839, + "nauc_map_at_1_max": 0.4545038697008203, + "nauc_map_at_1_std": 0.4037703592468525, + "nauc_map_at_20_diff1": 0.4741556594644958, + "nauc_map_at_20_max": 0.39843818761588734, + "nauc_map_at_20_std": 0.38073692069410986, + "nauc_map_at_3_diff1": 0.5033510921033137, + "nauc_map_at_3_max": 0.4073517810296481, + "nauc_map_at_3_std": 0.38473170311630295, + "nauc_map_at_5_diff1": 0.4937221583473709, + "nauc_map_at_5_max": 0.403448394535993, + "nauc_map_at_5_std": 0.3745933463443528, + "nauc_mrr_at_1000_diff1": 0.4751133996184114, + "nauc_mrr_at_1000_max": 0.40301542466256235, + "nauc_mrr_at_1000_std": 0.38565549662339377, + "nauc_mrr_at_100_diff1": 0.4742826036682546, + "nauc_mrr_at_100_max": 0.4032604036779924, + "nauc_mrr_at_100_std": 0.386053147898408, + "nauc_mrr_at_10_diff1": 0.478007665938371, + "nauc_mrr_at_10_max": 0.4001804786278926, + "nauc_mrr_at_10_std": 0.3829642722521861, + "nauc_mrr_at_1_diff1": 0.5661591775441839, + "nauc_mrr_at_1_max": 0.4545038697008203, + "nauc_mrr_at_1_std": 0.4037703592468525, + "nauc_mrr_at_20_diff1": 0.4741556594644958, + "nauc_mrr_at_20_max": 0.39843818761588734, + "nauc_mrr_at_20_std": 0.38073692069410986, + "nauc_mrr_at_3_diff1": 0.5033510921033137, + "nauc_mrr_at_3_max": 0.4073517810296481, + "nauc_mrr_at_3_std": 0.38473170311630295, + "nauc_mrr_at_5_diff1": 0.4937221583473709, + "nauc_mrr_at_5_max": 0.403448394535993, + "nauc_mrr_at_5_std": 0.3745933463443528, + "nauc_ndcg_at_1000_diff1": 0.4491703198195501, + "nauc_ndcg_at_1000_max": 0.4015367916914241, + "nauc_ndcg_at_1000_std": 0.3934259930578348, + "nauc_ndcg_at_100_diff1": 0.42706385403119074, + "nauc_ndcg_at_100_max": 0.4125623973619568, + "nauc_ndcg_at_100_std": 0.4119817880505865, + "nauc_ndcg_at_10_diff1": 0.4401322541912091, + "nauc_ndcg_at_10_max": 0.38487416073368114, + "nauc_ndcg_at_10_std": 0.38267665388460514, + "nauc_ndcg_at_1_diff1": 0.5661591775441839, + "nauc_ndcg_at_1_max": 0.4545038697008203, + "nauc_ndcg_at_1_std": 0.4037703592468525, + "nauc_ndcg_at_20_diff1": 0.4279259037817697, + "nauc_ndcg_at_20_max": 0.3797931060000094, + "nauc_ndcg_at_20_std": 0.37526381187135865, + "nauc_ndcg_at_3_diff1": 0.48866305533593757, + "nauc_ndcg_at_3_max": 0.3962060898287809, + "nauc_ndcg_at_3_std": 0.3801685730128606, + "nauc_ndcg_at_5_diff1": 0.47329560550893107, + "nauc_ndcg_at_5_max": 0.39074518984832857, + "nauc_ndcg_at_5_std": 0.3633995164290717, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.261923365390076, + "nauc_precision_at_100_max": 0.5024390148377484, + "nauc_precision_at_100_std": 0.5638540155133486, + "nauc_precision_at_10_diff1": 0.3454828119738378, + "nauc_precision_at_10_max": 0.3499346539030385, + "nauc_precision_at_10_std": 0.38660412429554525, + "nauc_precision_at_1_diff1": 0.5661591775441839, + "nauc_precision_at_1_max": 0.4545038697008203, + "nauc_precision_at_1_std": 0.4037703592468525, + "nauc_precision_at_20_diff1": 0.30794090624899045, + "nauc_precision_at_20_max": 0.33509256514196617, + "nauc_precision_at_20_std": 0.3627006654357914, + "nauc_precision_at_3_diff1": 0.452540072676058, + "nauc_precision_at_3_max": 0.36875749920521234, + "nauc_precision_at_3_std": 0.3689113290557238, + "nauc_precision_at_5_diff1": 0.4239877660106617, + "nauc_precision_at_5_max": 0.3608852174216834, + "nauc_precision_at_5_std": 0.3352722872510205, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.26192336539007566, + "nauc_recall_at_100_max": 0.5024390148377479, + "nauc_recall_at_100_std": 0.5638540155133488, + "nauc_recall_at_10_diff1": 0.3454828119738378, + "nauc_recall_at_10_max": 0.3499346539030384, + "nauc_recall_at_10_std": 0.3866041242955454, + "nauc_recall_at_1_diff1": 0.5661591775441839, + "nauc_recall_at_1_max": 0.4545038697008203, + "nauc_recall_at_1_std": 0.4037703592468525, + "nauc_recall_at_20_diff1": 0.3079409062489909, + "nauc_recall_at_20_max": 0.3350925651419665, + "nauc_recall_at_20_std": 0.36270066543579166, + "nauc_recall_at_3_diff1": 0.45254007267605834, + "nauc_recall_at_3_max": 0.3687574992052122, + "nauc_recall_at_3_std": 0.3689113290557241, + "nauc_recall_at_5_diff1": 0.42398776601066207, + "nauc_recall_at_5_max": 0.3608852174216836, + "nauc_recall_at_5_std": 0.3352722872510206, + "ndcg_at_1": 0.14607, + "ndcg_at_10": 0.23827, + "ndcg_at_100": 0.30546, + "ndcg_at_1000": 0.34486, + "ndcg_at_20": 0.26217, + "ndcg_at_3": 0.19715, + "ndcg_at_5": 0.21422, + "precision_at_1": 0.14607, + "precision_at_10": 0.03472, + "precision_at_100": 0.00684, + "precision_at_1000": 0.001, + "precision_at_20": 0.02213, + "precision_at_3": 0.07715, + "precision_at_5": 0.05461, + "recall_at_1": 0.14607, + "recall_at_10": 0.34719, + "recall_at_100": 0.68427, + "recall_at_1000": 1.0, + "recall_at_20": 0.4427, + "recall_at_3": 0.23146, + "recall_at_5": 0.27303 + }, + { + "cv_recall_at_1": 0.16744, + "cv_recall_at_10": 0.36512, + "cv_recall_at_100": 0.75698, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.47442, + "cv_recall_at_3": 0.25116, + "cv_recall_at_5": 0.28256, + "hf_subset": "bg", + "languages": [ + "bul-Cyrl" + ], + "main_score": 0.25512, + "map_at_1": 0.16744, + "map_at_10": 0.2215, + "map_at_100": 0.23559, + "map_at_1000": 0.23685, + "map_at_20": 0.22923, + "map_at_3": 0.20368, + "map_at_5": 0.21072, + "mrr_at_1": 0.16744186046511628, + "mrr_at_10": 0.22149686231081575, + "mrr_at_100": 0.2355936212080499, + "mrr_at_1000": 0.23685488711895378, + "mrr_at_20": 0.22923039957338684, + "mrr_at_3": 0.20368217054263568, + "mrr_at_5": 0.21071705426356593, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.3155611666828518, + "nauc_cv_recall_at_100_max": 0.45741229085892576, + "nauc_cv_recall_at_100_std": 0.3904921274057909, + "nauc_cv_recall_at_10_diff1": 0.4543116227676328, + "nauc_cv_recall_at_10_max": 0.3076640924878011, + "nauc_cv_recall_at_10_std": 0.09878880673568347, + "nauc_cv_recall_at_1_diff1": 0.6522816689962211, + "nauc_cv_recall_at_1_max": 0.33478067677865797, + "nauc_cv_recall_at_1_std": 0.08616115338820735, + "nauc_cv_recall_at_20_diff1": 0.3758814279747424, + "nauc_cv_recall_at_20_max": 0.297032013955431, + "nauc_cv_recall_at_20_std": 0.11695658583327197, + "nauc_cv_recall_at_3_diff1": 0.5426482437452204, + "nauc_cv_recall_at_3_max": 0.2972351961105648, + "nauc_cv_recall_at_3_std": 0.09761314596307219, + "nauc_cv_recall_at_5_diff1": 0.4755336920118826, + "nauc_cv_recall_at_5_max": 0.27284958625831, + "nauc_cv_recall_at_5_std": 0.08428460770948648, + "nauc_map_at_1000_diff1": 0.5679377206906693, + "nauc_map_at_1000_max": 0.31379566050626934, + "nauc_map_at_1000_std": 0.09487094415683674, + "nauc_map_at_100_diff1": 0.5672953157822261, + "nauc_map_at_100_max": 0.31403082416608186, + "nauc_map_at_100_std": 0.09546910746381114, + "nauc_map_at_10_diff1": 0.5746252363701051, + "nauc_map_at_10_max": 0.31445558548372626, + "nauc_map_at_10_std": 0.09096313225792232, + "nauc_map_at_1_diff1": 0.6522816689962211, + "nauc_map_at_1_max": 0.33478067677865797, + "nauc_map_at_1_std": 0.08616115338820735, + "nauc_map_at_20_diff1": 0.5670389052295799, + "nauc_map_at_20_max": 0.3130057286872424, + "nauc_map_at_20_std": 0.09273511502840416, + "nauc_map_at_3_diff1": 0.598818834786234, + "nauc_map_at_3_max": 0.31446259263387427, + "nauc_map_at_3_std": 0.09122519390838411, + "nauc_map_at_5_diff1": 0.5801340704820455, + "nauc_map_at_5_max": 0.3080495979250753, + "nauc_map_at_5_std": 0.0880409321335613, + "nauc_mrr_at_1000_diff1": 0.5679381592464301, + "nauc_mrr_at_1000_max": 0.31379554761149336, + "nauc_mrr_at_1000_std": 0.09487076381672727, + "nauc_mrr_at_100_diff1": 0.5672953157822261, + "nauc_mrr_at_100_max": 0.31403082416608186, + "nauc_mrr_at_100_std": 0.09546910746381114, + "nauc_mrr_at_10_diff1": 0.5746252363701051, + "nauc_mrr_at_10_max": 0.31445558548372626, + "nauc_mrr_at_10_std": 0.09096313225792232, + "nauc_mrr_at_1_diff1": 0.6522816689962211, + "nauc_mrr_at_1_max": 0.33478067677865797, + "nauc_mrr_at_1_std": 0.08616115338820735, + "nauc_mrr_at_20_diff1": 0.5670389052295799, + "nauc_mrr_at_20_max": 0.3130057286872424, + "nauc_mrr_at_20_std": 0.09273511502840416, + "nauc_mrr_at_3_diff1": 0.598818834786234, + "nauc_mrr_at_3_max": 0.31446259263387427, + "nauc_mrr_at_3_std": 0.09122519390838411, + "nauc_mrr_at_5_diff1": 0.5801340704820455, + "nauc_mrr_at_5_max": 0.3080495979250753, + "nauc_mrr_at_5_std": 0.0880409321335613, + "nauc_ndcg_at_1000_diff1": 0.5396722787543929, + "nauc_ndcg_at_1000_max": 0.3146022960220083, + "nauc_ndcg_at_1000_std": 0.10577477970824188, + "nauc_ndcg_at_100_diff1": 0.5190228546300492, + "nauc_ndcg_at_100_max": 0.3272980019602004, + "nauc_ndcg_at_100_std": 0.13152036231548603, + "nauc_ndcg_at_10_diff1": 0.5404999650445497, + "nauc_ndcg_at_10_max": 0.311839837720339, + "nauc_ndcg_at_10_std": 0.0931436541198744, + "nauc_ndcg_at_1_diff1": 0.6522816689962211, + "nauc_ndcg_at_1_max": 0.33478067677865797, + "nauc_ndcg_at_1_std": 0.08616115338820735, + "nauc_ndcg_at_20_diff1": 0.5165761612528892, + "nauc_ndcg_at_20_max": 0.30809471053636023, + "nauc_ndcg_at_20_std": 0.09875190211945468, + "nauc_ndcg_at_3_diff1": 0.5831062669245733, + "nauc_ndcg_at_3_max": 0.30956315539138013, + "nauc_ndcg_at_3_std": 0.09299125335033034, + "nauc_ndcg_at_5_diff1": 0.5511165710586254, + "nauc_ndcg_at_5_max": 0.29827585834017506, + "nauc_ndcg_at_5_std": 0.0871820714733284, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3155611666828535, + "nauc_precision_at_100_max": 0.4574122908589268, + "nauc_precision_at_100_std": 0.39049212740579214, + "nauc_precision_at_10_diff1": 0.45431162276763287, + "nauc_precision_at_10_max": 0.30766409248780135, + "nauc_precision_at_10_std": 0.09878880673568358, + "nauc_precision_at_1_diff1": 0.6522816689962211, + "nauc_precision_at_1_max": 0.33478067677865797, + "nauc_precision_at_1_std": 0.08616115338820735, + "nauc_precision_at_20_diff1": 0.3758814279747427, + "nauc_precision_at_20_max": 0.29703201395543116, + "nauc_precision_at_20_std": 0.1169565858332718, + "nauc_precision_at_3_diff1": 0.54264824374522, + "nauc_precision_at_3_max": 0.2972351961105648, + "nauc_precision_at_3_std": 0.0976131459630722, + "nauc_precision_at_5_diff1": 0.4755336920118825, + "nauc_precision_at_5_max": 0.27284958625830985, + "nauc_precision_at_5_std": 0.08428460770948652, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.3155611666828518, + "nauc_recall_at_100_max": 0.45741229085892576, + "nauc_recall_at_100_std": 0.3904921274057909, + "nauc_recall_at_10_diff1": 0.4543116227676328, + "nauc_recall_at_10_max": 0.3076640924878011, + "nauc_recall_at_10_std": 0.09878880673568347, + "nauc_recall_at_1_diff1": 0.6522816689962211, + "nauc_recall_at_1_max": 0.33478067677865797, + "nauc_recall_at_1_std": 0.08616115338820735, + "nauc_recall_at_20_diff1": 0.3758814279747424, + "nauc_recall_at_20_max": 0.297032013955431, + "nauc_recall_at_20_std": 0.11695658583327197, + "nauc_recall_at_3_diff1": 0.5426482437452204, + "nauc_recall_at_3_max": 0.2972351961105648, + "nauc_recall_at_3_std": 0.09761314596307219, + "nauc_recall_at_5_diff1": 0.4755336920118826, + "nauc_recall_at_5_max": 0.27284958625831, + "nauc_recall_at_5_std": 0.08428460770948648, + "ndcg_at_1": 0.16744, + "ndcg_at_10": 0.25512, + "ndcg_at_100": 0.33361, + "ndcg_at_1000": 0.36507, + "ndcg_at_20": 0.28294, + "ndcg_at_3": 0.21585, + "ndcg_at_5": 0.22866, + "precision_at_1": 0.16744, + "precision_at_10": 0.03651, + "precision_at_100": 0.00757, + "precision_at_1000": 0.001, + "precision_at_20": 0.02372, + "precision_at_3": 0.08372, + "precision_at_5": 0.05651, + "recall_at_1": 0.16744, + "recall_at_10": 0.36512, + "recall_at_100": 0.75698, + "recall_at_1000": 1.0, + "recall_at_20": 0.47442, + "recall_at_3": 0.25116, + "recall_at_5": 0.28256 + }, + { + "cv_recall_at_1": 0.50842, + "cv_recall_at_10": 0.74186, + "cv_recall_at_100": 0.90685, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.79798, + "cv_recall_at_3": 0.63749, + "cv_recall_at_5": 0.6835, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.62154, + "map_at_1": 0.50842, + "map_at_10": 0.58334, + "map_at_100": 0.58977, + "map_at_1000": 0.59026, + "map_at_20": 0.58734, + "map_at_3": 0.56472, + "map_at_5": 0.57516, + "mrr_at_1": 0.5084175084175084, + "mrr_at_10": 0.5833409046372006, + "mrr_at_100": 0.5897669028137018, + "mrr_at_1000": 0.590260080374905, + "mrr_at_20": 0.587342077177479, + "mrr_at_3": 0.5647212869435088, + "mrr_at_5": 0.5751589973812189, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6977811922414636, + "nauc_cv_recall_at_100_max": 0.6903446325672074, + "nauc_cv_recall_at_100_std": 0.5508802823143485, + "nauc_cv_recall_at_10_diff1": 0.7102384212164595, + "nauc_cv_recall_at_10_max": 0.531658928303646, + "nauc_cv_recall_at_10_std": 0.27018476526472457, + "nauc_cv_recall_at_1_diff1": 0.8146433011443713, + "nauc_cv_recall_at_1_max": 0.49933124251165467, + "nauc_cv_recall_at_1_std": 0.020398181485709738, + "nauc_cv_recall_at_20_diff1": 0.680292572444529, + "nauc_cv_recall_at_20_max": 0.5617293068211144, + "nauc_cv_recall_at_20_std": 0.38890792758037185, + "nauc_cv_recall_at_3_diff1": 0.7357291583736335, + "nauc_cv_recall_at_3_max": 0.5065655310271268, + "nauc_cv_recall_at_3_std": 0.09673314934363086, + "nauc_cv_recall_at_5_diff1": 0.7182209149801598, + "nauc_cv_recall_at_5_max": 0.5124711032629541, + "nauc_cv_recall_at_5_std": 0.1370912516418801, + "nauc_map_at_1000_diff1": 0.7810117882647332, + "nauc_map_at_1000_max": 0.5087532878090155, + "nauc_map_at_1000_std": 0.07156532369375307, + "nauc_map_at_100_diff1": 0.7808851016482755, + "nauc_map_at_100_max": 0.5089571032641548, + "nauc_map_at_100_std": 0.07219028700236066, + "nauc_map_at_10_diff1": 0.7805002652086618, + "nauc_map_at_10_max": 0.5063502982591215, + "nauc_map_at_10_std": 0.06890074443620031, + "nauc_map_at_1_diff1": 0.8146433011443713, + "nauc_map_at_1_max": 0.49933124251165467, + "nauc_map_at_1_std": 0.020398181485709738, + "nauc_map_at_20_diff1": 0.7798997218692829, + "nauc_map_at_20_max": 0.5073921256789337, + "nauc_map_at_20_std": 0.07219065379286495, + "nauc_map_at_3_diff1": 0.782875759866369, + "nauc_map_at_3_max": 0.5034365061077172, + "nauc_map_at_3_std": 0.0503649362897909, + "nauc_map_at_5_diff1": 0.7805620961748264, + "nauc_map_at_5_max": 0.5043784361078906, + "nauc_map_at_5_std": 0.05651909052654164, + "nauc_mrr_at_1000_diff1": 0.7810117882647332, + "nauc_mrr_at_1000_max": 0.5087532878090155, + "nauc_mrr_at_1000_std": 0.07156532369375307, + "nauc_mrr_at_100_diff1": 0.7808851016482755, + "nauc_mrr_at_100_max": 0.5089571032641548, + "nauc_mrr_at_100_std": 0.07219028700236066, + "nauc_mrr_at_10_diff1": 0.7805002652086618, + "nauc_mrr_at_10_max": 0.5063502982591215, + "nauc_mrr_at_10_std": 0.06890074443620031, + "nauc_mrr_at_1_diff1": 0.8146433011443713, + "nauc_mrr_at_1_max": 0.49933124251165467, + "nauc_mrr_at_1_std": 0.020398181485709738, + "nauc_mrr_at_20_diff1": 0.7798997218692829, + "nauc_mrr_at_20_max": 0.5073921256789337, + "nauc_mrr_at_20_std": 0.07219065379286495, + "nauc_mrr_at_3_diff1": 0.782875759866369, + "nauc_mrr_at_3_max": 0.5034365061077172, + "nauc_mrr_at_3_std": 0.0503649362897909, + "nauc_mrr_at_5_diff1": 0.7805620961748264, + "nauc_mrr_at_5_max": 0.5043784361078906, + "nauc_mrr_at_5_std": 0.05651909052654164, + "nauc_ndcg_at_1000_diff1": 0.7738005617196608, + "nauc_ndcg_at_1000_max": 0.5153371875544834, + "nauc_ndcg_at_1000_std": 0.09528500134059216, + "nauc_ndcg_at_100_diff1": 0.7703938756539106, + "nauc_ndcg_at_100_max": 0.5228755704005162, + "nauc_ndcg_at_100_std": 0.11535325549845764, + "nauc_ndcg_at_10_diff1": 0.7664599452451597, + "nauc_ndcg_at_10_max": 0.5109745148698641, + "nauc_ndcg_at_10_std": 0.10569364055562012, + "nauc_ndcg_at_1_diff1": 0.8146433011443713, + "nauc_ndcg_at_1_max": 0.49933124251165467, + "nauc_ndcg_at_1_std": 0.020398181485709738, + "nauc_ndcg_at_20_diff1": 0.7634906866295377, + "nauc_ndcg_at_20_max": 0.5152927534908216, + "nauc_ndcg_at_20_std": 0.11988886514932286, + "nauc_ndcg_at_3_diff1": 0.7716381069994489, + "nauc_ndcg_at_3_max": 0.504234251640214, + "nauc_ndcg_at_3_std": 0.061383868607826936, + "nauc_ndcg_at_5_diff1": 0.7667298291443791, + "nauc_ndcg_at_5_max": 0.5061599949153522, + "nauc_ndcg_at_5_std": 0.07403612092966083, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.6977811922414628, + "nauc_precision_at_100_max": 0.6903446325672082, + "nauc_precision_at_100_std": 0.5508802823143504, + "nauc_precision_at_10_diff1": 0.7102384212164586, + "nauc_precision_at_10_max": 0.5316589283036451, + "nauc_precision_at_10_std": 0.27018476526472307, + "nauc_precision_at_1_diff1": 0.8146433011443713, + "nauc_precision_at_1_max": 0.49933124251165467, + "nauc_precision_at_1_std": 0.020398181485709738, + "nauc_precision_at_20_diff1": 0.6802925724445281, + "nauc_precision_at_20_max": 0.5617293068211142, + "nauc_precision_at_20_std": 0.38890792758037157, + "nauc_precision_at_3_diff1": 0.7357291583736333, + "nauc_precision_at_3_max": 0.5065655310271261, + "nauc_precision_at_3_std": 0.09673314934363113, + "nauc_precision_at_5_diff1": 0.7182209149801593, + "nauc_precision_at_5_max": 0.5124711032629536, + "nauc_precision_at_5_std": 0.1370912516418799, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.6977811922414636, + "nauc_recall_at_100_max": 0.6903446325672074, + "nauc_recall_at_100_std": 0.5508802823143485, + "nauc_recall_at_10_diff1": 0.7102384212164595, + "nauc_recall_at_10_max": 0.531658928303646, + "nauc_recall_at_10_std": 0.27018476526472457, + "nauc_recall_at_1_diff1": 0.8146433011443713, + "nauc_recall_at_1_max": 0.49933124251165467, + "nauc_recall_at_1_std": 0.020398181485709738, + "nauc_recall_at_20_diff1": 0.680292572444529, + "nauc_recall_at_20_max": 0.5617293068211144, + "nauc_recall_at_20_std": 0.38890792758037185, + "nauc_recall_at_3_diff1": 0.7357291583736335, + "nauc_recall_at_3_max": 0.5065655310271268, + "nauc_recall_at_3_std": 0.09673314934363086, + "nauc_recall_at_5_diff1": 0.7182209149801598, + "nauc_recall_at_5_max": 0.5124711032629541, + "nauc_recall_at_5_std": 0.1370912516418801, + "ndcg_at_1": 0.50842, + "ndcg_at_10": 0.62154, + "ndcg_at_100": 0.65532, + "ndcg_at_1000": 0.66746, + "ndcg_at_20": 0.63586, + "ndcg_at_3": 0.58338, + "ndcg_at_5": 0.60227, + "precision_at_1": 0.50842, + "precision_at_10": 0.07419, + "precision_at_100": 0.00907, + "precision_at_1000": 0.001, + "precision_at_20": 0.0399, + "precision_at_3": 0.2125, + "precision_at_5": 0.1367, + "recall_at_1": 0.50842, + "recall_at_10": 0.74186, + "recall_at_100": 0.90685, + "recall_at_1000": 1.0, + "recall_at_20": 0.79798, + "recall_at_3": 0.63749, + "recall_at_5": 0.6835 + }, + { + "cv_recall_at_1": 0.08246, + "cv_recall_at_10": 0.19649, + "cv_recall_at_100": 0.48596, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.23333, + "cv_recall_at_3": 0.1193, + "cv_recall_at_5": 0.15088, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.13156, + "map_at_1": 0.08246, + "map_at_10": 0.1118, + "map_at_100": 0.11956, + "map_at_1000": 0.12204, + "map_at_20": 0.11438, + "map_at_3": 0.09854, + "map_at_5": 0.10591, + "mrr_at_1": 0.0824561403508772, + "mrr_at_10": 0.11179824561403513, + "mrr_at_100": 0.1195605084094835, + "mrr_at_1000": 0.12204457182360474, + "mrr_at_20": 0.11438025632114773, + "mrr_at_3": 0.09853801169590644, + "mrr_at_5": 0.10590643274853803, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.23848481817399358, + "nauc_cv_recall_at_100_max": 0.2780996955027928, + "nauc_cv_recall_at_100_std": 0.4483157280135105, + "nauc_cv_recall_at_10_diff1": 0.41543817720122717, + "nauc_cv_recall_at_10_max": 0.3404936172168539, + "nauc_cv_recall_at_10_std": 0.40725431168204446, + "nauc_cv_recall_at_1_diff1": 0.7881759900439002, + "nauc_cv_recall_at_1_max": 0.5271366508914248, + "nauc_cv_recall_at_1_std": 0.4374689585486017, + "nauc_cv_recall_at_20_diff1": 0.37609410863557907, + "nauc_cv_recall_at_20_max": 0.3282897728935001, + "nauc_cv_recall_at_20_std": 0.4261219018540338, + "nauc_cv_recall_at_3_diff1": 0.5818612882759543, + "nauc_cv_recall_at_3_max": 0.40298276955955536, + "nauc_cv_recall_at_3_std": 0.38599024944067867, + "nauc_cv_recall_at_5_diff1": 0.49318266650703224, + "nauc_cv_recall_at_5_max": 0.40355374636420616, + "nauc_cv_recall_at_5_std": 0.41270925622046467, + "nauc_map_at_1000_diff1": 0.619423325807602, + "nauc_map_at_1000_max": 0.4443675920403044, + "nauc_map_at_1000_std": 0.4262573861773211, + "nauc_map_at_100_diff1": 0.616446963844894, + "nauc_map_at_100_max": 0.4428822327863043, + "nauc_map_at_100_std": 0.4261337543043992, + "nauc_map_at_10_diff1": 0.6301642227971276, + "nauc_map_at_10_max": 0.44727900681909044, + "nauc_map_at_10_std": 0.4228057973690992, + "nauc_map_at_1_diff1": 0.7881759900439002, + "nauc_map_at_1_max": 0.5271366508914248, + "nauc_map_at_1_std": 0.4374689585486017, + "nauc_map_at_20_diff1": 0.6241634212559133, + "nauc_map_at_20_max": 0.4459319580120189, + "nauc_map_at_20_std": 0.42552264032079873, + "nauc_map_at_3_diff1": 0.6860315933291902, + "nauc_map_at_3_max": 0.4627361884329836, + "nauc_map_at_3_std": 0.4158925405410705, + "nauc_map_at_5_diff1": 0.653263228710077, + "nauc_map_at_5_max": 0.4607797632861452, + "nauc_map_at_5_std": 0.42253211669153967, + "nauc_mrr_at_1000_diff1": 0.619423325807602, + "nauc_mrr_at_1000_max": 0.4443675920403044, + "nauc_mrr_at_1000_std": 0.4262573861773211, + "nauc_mrr_at_100_diff1": 0.616446963844894, + "nauc_mrr_at_100_max": 0.4428822327863043, + "nauc_mrr_at_100_std": 0.4261337543043992, + "nauc_mrr_at_10_diff1": 0.6301642227971276, + "nauc_mrr_at_10_max": 0.44727900681909044, + "nauc_mrr_at_10_std": 0.4228057973690992, + "nauc_mrr_at_1_diff1": 0.7881759900439002, + "nauc_mrr_at_1_max": 0.5271366508914248, + "nauc_mrr_at_1_std": 0.4374689585486017, + "nauc_mrr_at_20_diff1": 0.6241634212559133, + "nauc_mrr_at_20_max": 0.4459319580120189, + "nauc_mrr_at_20_std": 0.42552264032079873, + "nauc_mrr_at_3_diff1": 0.6860315933291902, + "nauc_mrr_at_3_max": 0.4627361884329836, + "nauc_mrr_at_3_std": 0.4158925405410705, + "nauc_mrr_at_5_diff1": 0.653263228710077, + "nauc_mrr_at_5_max": 0.4607797632861452, + "nauc_mrr_at_5_std": 0.42253211669153967, + "nauc_ndcg_at_1000_diff1": 0.5601778359428754, + "nauc_ndcg_at_1000_max": 0.4189585137835097, + "nauc_ndcg_at_1000_std": 0.4282972488935674, + "nauc_ndcg_at_100_diff1": 0.49652500435897334, + "nauc_ndcg_at_100_max": 0.3904312925139352, + "nauc_ndcg_at_100_std": 0.43115952849055644, + "nauc_ndcg_at_10_diff1": 0.5601850127764495, + "nauc_ndcg_at_10_max": 0.4132393273356581, + "nauc_ndcg_at_10_std": 0.41786479744859456, + "nauc_ndcg_at_1_diff1": 0.7881759900439002, + "nauc_ndcg_at_1_max": 0.5271366508914248, + "nauc_ndcg_at_1_std": 0.4374689585486017, + "nauc_ndcg_at_20_diff1": 0.5422695603581896, + "nauc_ndcg_at_20_max": 0.4078713890969411, + "nauc_ndcg_at_20_std": 0.4252348047801439, + "nauc_ndcg_at_3_diff1": 0.6561678022728696, + "nauc_ndcg_at_3_max": 0.4454927391585425, + "nauc_ndcg_at_3_std": 0.4074709868295361, + "nauc_ndcg_at_5_diff1": 0.6047492469394583, + "nauc_ndcg_at_5_max": 0.44300596922196894, + "nauc_ndcg_at_5_std": 0.4192443928941932, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.2384848181739936, + "nauc_precision_at_100_max": 0.27809969550279284, + "nauc_precision_at_100_std": 0.44831572801351055, + "nauc_precision_at_10_diff1": 0.41543817720122694, + "nauc_precision_at_10_max": 0.3404936172168537, + "nauc_precision_at_10_std": 0.40725431168204423, + "nauc_precision_at_1_diff1": 0.7881759900439002, + "nauc_precision_at_1_max": 0.5271366508914248, + "nauc_precision_at_1_std": 0.4374689585486017, + "nauc_precision_at_20_diff1": 0.37609410863557896, + "nauc_precision_at_20_max": 0.32828977289350025, + "nauc_precision_at_20_std": 0.42612190185403387, + "nauc_precision_at_3_diff1": 0.5818612882759547, + "nauc_precision_at_3_max": 0.4029827695595554, + "nauc_precision_at_3_std": 0.38599024944067883, + "nauc_precision_at_5_diff1": 0.49318266650703235, + "nauc_precision_at_5_max": 0.40355374636420643, + "nauc_precision_at_5_std": 0.4127092562204649, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.23848481817399358, + "nauc_recall_at_100_max": 0.2780996955027928, + "nauc_recall_at_100_std": 0.4483157280135105, + "nauc_recall_at_10_diff1": 0.41543817720122717, + "nauc_recall_at_10_max": 0.3404936172168539, + "nauc_recall_at_10_std": 0.40725431168204446, + "nauc_recall_at_1_diff1": 0.7881759900439002, + "nauc_recall_at_1_max": 0.5271366508914248, + "nauc_recall_at_1_std": 0.4374689585486017, + "nauc_recall_at_20_diff1": 0.37609410863557907, + "nauc_recall_at_20_max": 0.3282897728935001, + "nauc_recall_at_20_std": 0.4261219018540338, + "nauc_recall_at_3_diff1": 0.5818612882759543, + "nauc_recall_at_3_max": 0.40298276955955536, + "nauc_recall_at_3_std": 0.38599024944067867, + "nauc_recall_at_5_diff1": 0.49318266650703224, + "nauc_recall_at_5_max": 0.40355374636420616, + "nauc_recall_at_5_std": 0.41270925622046467, + "ndcg_at_1": 0.08246, + "ndcg_at_10": 0.13156, + "ndcg_at_100": 0.18517, + "ndcg_at_1000": 0.25104, + "ndcg_at_20": 0.14091, + "ndcg_at_3": 0.10386, + "ndcg_at_5": 0.117, + "precision_at_1": 0.08246, + "precision_at_10": 0.01965, + "precision_at_100": 0.00486, + "precision_at_1000": 0.001, + "precision_at_20": 0.01167, + "precision_at_3": 0.03977, + "precision_at_5": 0.03018, + "recall_at_1": 0.08246, + "recall_at_10": 0.19649, + "recall_at_100": 0.48596, + "recall_at_1000": 1.0, + "recall_at_20": 0.23333, + "recall_at_3": 0.1193, + "recall_at_5": 0.15088 + }, + { + "cv_recall_at_1": 0.28146, + "cv_recall_at_10": 0.48513, + "cv_recall_at_100": 0.77918, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.56407, + "cv_recall_at_3": 0.35812, + "cv_recall_at_5": 0.40046, + "hf_subset": "et", + "languages": [ + "est-Latn" + ], + "main_score": 0.37248, + "map_at_1": 0.28146, + "map_at_10": 0.33788, + "map_at_100": 0.34847, + "map_at_1000": 0.34953, + "map_at_20": 0.34331, + "map_at_3": 0.31636, + "map_at_5": 0.3262, + "mrr_at_1": 0.2814645308924485, + "mrr_at_10": 0.3378822963205114, + "mrr_at_100": 0.34847209578763483, + "mrr_at_1000": 0.3495290815761699, + "mrr_at_20": 0.34331177017418185, + "mrr_at_3": 0.3163615560640733, + "mrr_at_5": 0.3262013729977116, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.34932998468795345, + "nauc_cv_recall_at_100_max": 0.5753714600936997, + "nauc_cv_recall_at_100_std": 0.4360491728305647, + "nauc_cv_recall_at_10_diff1": 0.5166151838980909, + "nauc_cv_recall_at_10_max": 0.46419054224541006, + "nauc_cv_recall_at_10_std": 0.1876105520707209, + "nauc_cv_recall_at_1_diff1": 0.7440892508386078, + "nauc_cv_recall_at_1_max": 0.4548291267176457, + "nauc_cv_recall_at_1_std": 0.006599870432398388, + "nauc_cv_recall_at_20_diff1": 0.47758396896747074, + "nauc_cv_recall_at_20_max": 0.5097760248029565, + "nauc_cv_recall_at_20_std": 0.2613282694411242, + "nauc_cv_recall_at_3_diff1": 0.6143055626966617, + "nauc_cv_recall_at_3_max": 0.42541096805824885, + "nauc_cv_recall_at_3_std": 0.08912464881100893, + "nauc_cv_recall_at_5_diff1": 0.5908371774719564, + "nauc_cv_recall_at_5_max": 0.4486943218864718, + "nauc_cv_recall_at_5_std": 0.09565149785356179, + "nauc_map_at_1000_diff1": 0.6635556751610107, + "nauc_map_at_1000_max": 0.4516694667637394, + "nauc_map_at_1000_std": 0.06241952508490272, + "nauc_map_at_100_diff1": 0.6626921889571289, + "nauc_map_at_100_max": 0.4519587940605559, + "nauc_map_at_100_std": 0.06340615907295671, + "nauc_map_at_10_diff1": 0.6659223965757382, + "nauc_map_at_10_max": 0.4477204335728731, + "nauc_map_at_10_std": 0.05723630290668432, + "nauc_map_at_1_diff1": 0.7440892508386078, + "nauc_map_at_1_max": 0.4548291267176457, + "nauc_map_at_1_std": 0.006599870432398388, + "nauc_map_at_20_diff1": 0.6640183038107316, + "nauc_map_at_20_max": 0.45117770525747186, + "nauc_map_at_20_std": 0.06228760382026663, + "nauc_map_at_3_diff1": 0.6818687659705938, + "nauc_map_at_3_max": 0.4392047808874562, + "nauc_map_at_3_std": 0.04329965866604979, + "nauc_map_at_5_diff1": 0.6757236325559439, + "nauc_map_at_5_max": 0.44526383671545794, + "nauc_map_at_5_std": 0.044908921967325065, + "nauc_mrr_at_1000_diff1": 0.6635556751610107, + "nauc_mrr_at_1000_max": 0.4516694667637394, + "nauc_mrr_at_1000_std": 0.06241952508490272, + "nauc_mrr_at_100_diff1": 0.6626921889571289, + "nauc_mrr_at_100_max": 0.4519587940605559, + "nauc_mrr_at_100_std": 0.06340615907295671, + "nauc_mrr_at_10_diff1": 0.6659223965757382, + "nauc_mrr_at_10_max": 0.4477204335728731, + "nauc_mrr_at_10_std": 0.05723630290668432, + "nauc_mrr_at_1_diff1": 0.7440892508386078, + "nauc_mrr_at_1_max": 0.4548291267176457, + "nauc_mrr_at_1_std": 0.006599870432398388, + "nauc_mrr_at_20_diff1": 0.6640183038107316, + "nauc_mrr_at_20_max": 0.45117770525747186, + "nauc_mrr_at_20_std": 0.06228760382026663, + "nauc_mrr_at_3_diff1": 0.6818687659705938, + "nauc_mrr_at_3_max": 0.4392047808874562, + "nauc_mrr_at_3_std": 0.04329965866604979, + "nauc_mrr_at_5_diff1": 0.6757236325559439, + "nauc_mrr_at_5_max": 0.44526383671545794, + "nauc_mrr_at_5_std": 0.044908921967325065, + "nauc_ndcg_at_1000_diff1": 0.6383749462925029, + "nauc_ndcg_at_1000_max": 0.45848340658237796, + "nauc_ndcg_at_1000_std": 0.08702927051969854, + "nauc_ndcg_at_100_diff1": 0.6151065936759832, + "nauc_ndcg_at_100_max": 0.4675296810665792, + "nauc_ndcg_at_100_std": 0.11488815515095756, + "nauc_ndcg_at_10_diff1": 0.6295784200068598, + "nauc_ndcg_at_10_max": 0.451212145699924, + "nauc_ndcg_at_10_std": 0.08806375022143757, + "nauc_ndcg_at_1_diff1": 0.7440892508386078, + "nauc_ndcg_at_1_max": 0.4548291267176457, + "nauc_ndcg_at_1_std": 0.006599870432398388, + "nauc_ndcg_at_20_diff1": 0.622080375563178, + "nauc_ndcg_at_20_max": 0.4626560051729655, + "nauc_ndcg_at_20_std": 0.10512648478325441, + "nauc_ndcg_at_3_diff1": 0.663987564213784, + "nauc_ndcg_at_3_max": 0.43548729740830583, + "nauc_ndcg_at_3_std": 0.0553017963802168, + "nauc_ndcg_at_5_diff1": 0.6533851232485005, + "nauc_ndcg_at_5_max": 0.44580433876900977, + "nauc_ndcg_at_5_std": 0.05822191398076574, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.349329984687954, + "nauc_precision_at_100_max": 0.5753714600937007, + "nauc_precision_at_100_std": 0.4360491728305648, + "nauc_precision_at_10_diff1": 0.5166151838980908, + "nauc_precision_at_10_max": 0.4641905422454098, + "nauc_precision_at_10_std": 0.18761055207072072, + "nauc_precision_at_1_diff1": 0.7440892508386078, + "nauc_precision_at_1_max": 0.4548291267176457, + "nauc_precision_at_1_std": 0.006599870432398388, + "nauc_precision_at_20_diff1": 0.47758396896747074, + "nauc_precision_at_20_max": 0.509776024802956, + "nauc_precision_at_20_std": 0.26132826944112364, + "nauc_precision_at_3_diff1": 0.6143055626966614, + "nauc_precision_at_3_max": 0.42541096805824835, + "nauc_precision_at_3_std": 0.08912464881100866, + "nauc_precision_at_5_diff1": 0.5908371774719563, + "nauc_precision_at_5_max": 0.44869432188647174, + "nauc_precision_at_5_std": 0.09565149785356195, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.34932998468795345, + "nauc_recall_at_100_max": 0.5753714600936997, + "nauc_recall_at_100_std": 0.4360491728305647, + "nauc_recall_at_10_diff1": 0.5166151838980909, + "nauc_recall_at_10_max": 0.46419054224541006, + "nauc_recall_at_10_std": 0.1876105520707209, + "nauc_recall_at_1_diff1": 0.7440892508386078, + "nauc_recall_at_1_max": 0.4548291267176457, + "nauc_recall_at_1_std": 0.006599870432398388, + "nauc_recall_at_20_diff1": 0.47758396896747074, + "nauc_recall_at_20_max": 0.5097760248029565, + "nauc_recall_at_20_std": 0.2613282694411242, + "nauc_recall_at_3_diff1": 0.6143055626966617, + "nauc_recall_at_3_max": 0.42541096805824885, + "nauc_recall_at_3_std": 0.08912464881100893, + "nauc_recall_at_5_diff1": 0.5908371774719564, + "nauc_recall_at_5_max": 0.4486943218864718, + "nauc_recall_at_5_std": 0.09565149785356179, + "ndcg_at_1": 0.28146, + "ndcg_at_10": 0.37248, + "ndcg_at_100": 0.43161, + "ndcg_at_1000": 0.45984, + "ndcg_at_20": 0.39237, + "ndcg_at_3": 0.32713, + "ndcg_at_5": 0.34471, + "precision_at_1": 0.28146, + "precision_at_10": 0.04851, + "precision_at_100": 0.00779, + "precision_at_1000": 0.001, + "precision_at_20": 0.0282, + "precision_at_3": 0.11937, + "precision_at_5": 0.08009, + "recall_at_1": 0.28146, + "recall_at_10": 0.48513, + "recall_at_100": 0.77918, + "recall_at_1000": 1.0, + "recall_at_20": 0.56407, + "recall_at_3": 0.35812, + "recall_at_5": 0.40046 + }, + { + "cv_recall_at_1": 0.56493, + "cv_recall_at_10": 0.78579, + "cv_recall_at_100": 0.9545, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.84906, + "cv_recall_at_3": 0.67259, + "cv_recall_at_5": 0.72253, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.66955, + "map_at_1": 0.56493, + "map_at_10": 0.63306, + "map_at_100": 0.64021, + "map_at_1000": 0.64046, + "map_at_20": 0.63745, + "map_at_3": 0.61302, + "map_at_5": 0.62445, + "mrr_at_1": 0.564927857935627, + "mrr_at_10": 0.6330602681324101, + "mrr_at_100": 0.6402059570398387, + "mrr_at_1000": 0.6404573779066952, + "mrr_at_20": 0.6374518773738077, + "mrr_at_3": 0.6130225675175731, + "mrr_at_5": 0.6244543100258969, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.7536092845128153, + "nauc_cv_recall_at_100_max": 0.7685517696368389, + "nauc_cv_recall_at_100_std": 0.7416747448364003, + "nauc_cv_recall_at_10_diff1": 0.7446677172022012, + "nauc_cv_recall_at_10_max": 0.4848156559769109, + "nauc_cv_recall_at_10_std": 0.27822291722547365, + "nauc_cv_recall_at_1_diff1": 0.8529129171302879, + "nauc_cv_recall_at_1_max": 0.4854208594375987, + "nauc_cv_recall_at_1_std": 0.11220419342787168, + "nauc_cv_recall_at_20_diff1": 0.7152210152620466, + "nauc_cv_recall_at_20_max": 0.5093419618089969, + "nauc_cv_recall_at_20_std": 0.31952722781397186, + "nauc_cv_recall_at_3_diff1": 0.7987291872149318, + "nauc_cv_recall_at_3_max": 0.47266289861776534, + "nauc_cv_recall_at_3_std": 0.15022238304790544, + "nauc_cv_recall_at_5_diff1": 0.7702540174147103, + "nauc_cv_recall_at_5_max": 0.46083059121534187, + "nauc_cv_recall_at_5_std": 0.18636651790144276, + "nauc_map_at_1000_diff1": 0.8253362540968048, + "nauc_map_at_1000_max": 0.4837965017623685, + "nauc_map_at_1000_std": 0.1420004533208783, + "nauc_map_at_100_diff1": 0.8252282114430481, + "nauc_map_at_100_max": 0.4839987346975108, + "nauc_map_at_100_std": 0.1424789807481376, + "nauc_map_at_10_diff1": 0.82439509533897, + "nauc_map_at_10_max": 0.48154349516858935, + "nauc_map_at_10_std": 0.14141446616923847, + "nauc_map_at_1_diff1": 0.8529129171302879, + "nauc_map_at_1_max": 0.4854208594375987, + "nauc_map_at_1_std": 0.11220419342787168, + "nauc_map_at_20_diff1": 0.824372991789611, + "nauc_map_at_20_max": 0.48264880992645687, + "nauc_map_at_20_std": 0.1415764288605779, + "nauc_map_at_3_diff1": 0.8306021895214727, + "nauc_map_at_3_max": 0.48060459308194475, + "nauc_map_at_3_std": 0.12697305747126705, + "nauc_map_at_5_diff1": 0.8255314697911217, + "nauc_map_at_5_max": 0.47836790397649964, + "nauc_map_at_5_std": 0.13337887538751342, + "nauc_mrr_at_1000_diff1": 0.8253362900449671, + "nauc_mrr_at_1000_max": 0.48379648836283556, + "nauc_mrr_at_1000_std": 0.1420003680546551, + "nauc_mrr_at_100_diff1": 0.8252282114430481, + "nauc_mrr_at_100_max": 0.4839987346975108, + "nauc_mrr_at_100_std": 0.1424789807481376, + "nauc_mrr_at_10_diff1": 0.82439509533897, + "nauc_mrr_at_10_max": 0.48154349516858935, + "nauc_mrr_at_10_std": 0.14141446616923847, + "nauc_mrr_at_1_diff1": 0.8529129171302879, + "nauc_mrr_at_1_max": 0.4854208594375987, + "nauc_mrr_at_1_std": 0.11220419342787168, + "nauc_mrr_at_20_diff1": 0.824372991789611, + "nauc_mrr_at_20_max": 0.48264880992645687, + "nauc_mrr_at_20_std": 0.1415764288605779, + "nauc_mrr_at_3_diff1": 0.8306021895214727, + "nauc_mrr_at_3_max": 0.48060459308194475, + "nauc_mrr_at_3_std": 0.12697305747126705, + "nauc_mrr_at_5_diff1": 0.8255314697911217, + "nauc_mrr_at_5_max": 0.47836790397649964, + "nauc_mrr_at_5_std": 0.13337887538751342, + "nauc_ndcg_at_1000_diff1": 0.8189740955445418, + "nauc_ndcg_at_1000_max": 0.48739519758688205, + "nauc_ndcg_at_1000_std": 0.1560786057897601, + "nauc_ndcg_at_100_diff1": 0.8170601681119815, + "nauc_ndcg_at_100_max": 0.4941817302873829, + "nauc_ndcg_at_100_std": 0.17053277872707645, + "nauc_ndcg_at_10_diff1": 0.8098947819056849, + "nauc_ndcg_at_10_max": 0.4817583414779039, + "nauc_ndcg_at_10_std": 0.16502964660555786, + "nauc_ndcg_at_1_diff1": 0.8529129171302879, + "nauc_ndcg_at_1_max": 0.4854208594375987, + "nauc_ndcg_at_1_std": 0.11220419342787168, + "nauc_ndcg_at_20_diff1": 0.8090926247820623, + "nauc_ndcg_at_20_max": 0.48551778105850196, + "nauc_ndcg_at_20_std": 0.16566285450230253, + "nauc_ndcg_at_3_diff1": 0.8229903288375536, + "nauc_ndcg_at_3_max": 0.47872741897937043, + "nauc_ndcg_at_3_std": 0.13248601038954932, + "nauc_ndcg_at_5_diff1": 0.8135755859828144, + "nauc_ndcg_at_5_max": 0.4746303106044694, + "nauc_ndcg_at_5_std": 0.14462357276570686, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.7536092845128152, + "nauc_precision_at_100_max": 0.7685517696368366, + "nauc_precision_at_100_std": 0.7416747448364028, + "nauc_precision_at_10_diff1": 0.7446677172022022, + "nauc_precision_at_10_max": 0.4848156559769116, + "nauc_precision_at_10_std": 0.27822291722547354, + "nauc_precision_at_1_diff1": 0.8529129171302879, + "nauc_precision_at_1_max": 0.4854208594375987, + "nauc_precision_at_1_std": 0.11220419342787168, + "nauc_precision_at_20_diff1": 0.7152210152620461, + "nauc_precision_at_20_max": 0.5093419618089969, + "nauc_precision_at_20_std": 0.3195272278139724, + "nauc_precision_at_3_diff1": 0.7987291872149324, + "nauc_precision_at_3_max": 0.4726628986177653, + "nauc_precision_at_3_std": 0.15022238304790592, + "nauc_precision_at_5_diff1": 0.770254017414709, + "nauc_precision_at_5_max": 0.4608305912153412, + "nauc_precision_at_5_std": 0.18636651790144232, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.7536092845128153, + "nauc_recall_at_100_max": 0.7685517696368389, + "nauc_recall_at_100_std": 0.7416747448364003, + "nauc_recall_at_10_diff1": 0.7446677172022012, + "nauc_recall_at_10_max": 0.4848156559769109, + "nauc_recall_at_10_std": 0.27822291722547365, + "nauc_recall_at_1_diff1": 0.8529129171302879, + "nauc_recall_at_1_max": 0.4854208594375987, + "nauc_recall_at_1_std": 0.11220419342787168, + "nauc_recall_at_20_diff1": 0.7152210152620466, + "nauc_recall_at_20_max": 0.5093419618089969, + "nauc_recall_at_20_std": 0.31952722781397186, + "nauc_recall_at_3_diff1": 0.7987291872149318, + "nauc_recall_at_3_max": 0.47266289861776534, + "nauc_recall_at_3_std": 0.15022238304790544, + "nauc_recall_at_5_diff1": 0.7702540174147103, + "nauc_recall_at_5_max": 0.46083059121534187, + "nauc_recall_at_5_std": 0.18636651790144276, + "ndcg_at_1": 0.56493, + "ndcg_at_10": 0.66955, + "ndcg_at_100": 0.70522, + "ndcg_at_1000": 0.71118, + "ndcg_at_20": 0.68554, + "ndcg_at_3": 0.62835, + "ndcg_at_5": 0.64893, + "precision_at_1": 0.56493, + "precision_at_10": 0.07858, + "precision_at_100": 0.00954, + "precision_at_1000": 0.001, + "precision_at_20": 0.04245, + "precision_at_3": 0.2242, + "precision_at_5": 0.14451, + "recall_at_1": 0.56493, + "recall_at_10": 0.78579, + "recall_at_100": 0.9545, + "recall_at_1000": 1.0, + "recall_at_20": 0.84906, + "recall_at_3": 0.67259, + "recall_at_5": 0.72253 + }, + { + "cv_recall_at_1": 0.08163, + "cv_recall_at_10": 0.18904, + "cv_recall_at_100": 0.50161, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.26101, + "cv_recall_at_3": 0.11923, + "cv_recall_at_5": 0.14823, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.12847, + "map_at_1": 0.08163, + "map_at_10": 0.10993, + "map_at_100": 0.12006, + "map_at_1000": 0.12203, + "map_at_20": 0.11476, + "map_at_3": 0.09792, + "map_at_5": 0.10448, + "mrr_at_1": 0.08163265306122448, + "mrr_at_10": 0.1099342744616644, + "mrr_at_100": 0.12005746932036375, + "mrr_at_1000": 0.12203113427399533, + "mrr_at_20": 0.11475762824669429, + "mrr_at_3": 0.0979233798782671, + "mrr_at_5": 0.10447547440028646, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.14751155284105305, + "nauc_cv_recall_at_100_max": 0.22564543811830431, + "nauc_cv_recall_at_100_std": 0.36565185549033236, + "nauc_cv_recall_at_10_diff1": 0.2860643473956765, + "nauc_cv_recall_at_10_max": 0.29643996104309345, + "nauc_cv_recall_at_10_std": 0.3487498575679017, + "nauc_cv_recall_at_1_diff1": 0.6745448554552967, + "nauc_cv_recall_at_1_max": 0.5000925535843469, + "nauc_cv_recall_at_1_std": 0.4339496692158664, + "nauc_cv_recall_at_20_diff1": 0.25203106653182167, + "nauc_cv_recall_at_20_max": 0.27861817937663164, + "nauc_cv_recall_at_20_std": 0.35497189059078726, + "nauc_cv_recall_at_3_diff1": 0.47610663695685046, + "nauc_cv_recall_at_3_max": 0.43568329084244445, + "nauc_cv_recall_at_3_std": 0.41368899691317484, + "nauc_cv_recall_at_5_diff1": 0.3733814405327858, + "nauc_cv_recall_at_5_max": 0.37192948006470095, + "nauc_cv_recall_at_5_std": 0.40205902036298935, + "nauc_map_at_1000_diff1": 0.4978739033571974, + "nauc_map_at_1000_max": 0.419520151261849, + "nauc_map_at_1000_std": 0.4068692740308292, + "nauc_map_at_100_diff1": 0.4958356262477288, + "nauc_map_at_100_max": 0.4177789835056861, + "nauc_map_at_100_std": 0.40616264397711266, + "nauc_map_at_10_diff1": 0.5109920139660944, + "nauc_map_at_10_max": 0.42545602974269947, + "nauc_map_at_10_std": 0.4051907995409085, + "nauc_map_at_1_diff1": 0.6745448554552967, + "nauc_map_at_1_max": 0.5000925535843469, + "nauc_map_at_1_std": 0.4339496692158664, + "nauc_map_at_20_diff1": 0.5022386869091225, + "nauc_map_at_20_max": 0.4211733453178635, + "nauc_map_at_20_std": 0.4061450432164285, + "nauc_map_at_3_diff1": 0.5694582452423288, + "nauc_map_at_3_max": 0.4625966131133485, + "nauc_map_at_3_std": 0.4194557757874592, + "nauc_map_at_5_diff1": 0.5346634805130233, + "nauc_map_at_5_max": 0.4433674733314563, + "nauc_map_at_5_std": 0.41674610130363493, + "nauc_mrr_at_1000_diff1": 0.49787361295714627, + "nauc_mrr_at_1000_max": 0.4195213345637115, + "nauc_mrr_at_1000_std": 0.4068699032112551, + "nauc_mrr_at_100_diff1": 0.49583533792631634, + "nauc_mrr_at_100_max": 0.4177801598674725, + "nauc_mrr_at_100_std": 0.4061632694340152, + "nauc_mrr_at_10_diff1": 0.5109920139660944, + "nauc_mrr_at_10_max": 0.42545602974269947, + "nauc_mrr_at_10_std": 0.4051907995409085, + "nauc_mrr_at_1_diff1": 0.6745448554552967, + "nauc_mrr_at_1_max": 0.5000925535843469, + "nauc_mrr_at_1_std": 0.4339496692158664, + "nauc_mrr_at_20_diff1": 0.5022386869091225, + "nauc_mrr_at_20_max": 0.4211733453178635, + "nauc_mrr_at_20_std": 0.4061450432164285, + "nauc_mrr_at_3_diff1": 0.5694582452423288, + "nauc_mrr_at_3_max": 0.4625966131133485, + "nauc_mrr_at_3_std": 0.4194557757874592, + "nauc_mrr_at_5_diff1": 0.5346634805130233, + "nauc_mrr_at_5_max": 0.4433674733314563, + "nauc_mrr_at_5_std": 0.41674610130363493, + "nauc_ndcg_at_1000_diff1": 0.4358424701809257, + "nauc_ndcg_at_1000_max": 0.38888310258472764, + "nauc_ndcg_at_1000_std": 0.40007110481304475, + "nauc_ndcg_at_100_diff1": 0.382231846590429, + "nauc_ndcg_at_100_max": 0.35564165055226754, + "nauc_ndcg_at_100_std": 0.39143473660956984, + "nauc_ndcg_at_10_diff1": 0.43889613013765655, + "nauc_ndcg_at_10_max": 0.3850896642714146, + "nauc_ndcg_at_10_std": 0.3879631630579122, + "nauc_ndcg_at_1_diff1": 0.6745448554552967, + "nauc_ndcg_at_1_max": 0.5000925535843469, + "nauc_ndcg_at_1_std": 0.4339496692158664, + "nauc_ndcg_at_20_diff1": 0.4168988692194725, + "nauc_ndcg_at_20_max": 0.3736432108951681, + "nauc_ndcg_at_20_std": 0.3894409640165071, + "nauc_ndcg_at_3_diff1": 0.542351084582344, + "nauc_ndcg_at_3_max": 0.4546420999628177, + "nauc_ndcg_at_3_std": 0.4176261772057995, + "nauc_ndcg_at_5_diff1": 0.48597152615731637, + "nauc_ndcg_at_5_max": 0.4220938155113547, + "nauc_ndcg_at_5_std": 0.4123319702203166, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.1475115528410533, + "nauc_precision_at_100_max": 0.2256454381183047, + "nauc_precision_at_100_std": 0.3656518554903324, + "nauc_precision_at_10_diff1": 0.2860643473956765, + "nauc_precision_at_10_max": 0.29643996104309334, + "nauc_precision_at_10_std": 0.3487498575679015, + "nauc_precision_at_1_diff1": 0.6745448554552967, + "nauc_precision_at_1_max": 0.5000925535843469, + "nauc_precision_at_1_std": 0.4339496692158664, + "nauc_precision_at_20_diff1": 0.2520310665318214, + "nauc_precision_at_20_max": 0.27861817937663186, + "nauc_precision_at_20_std": 0.3549718905907873, + "nauc_precision_at_3_diff1": 0.47610663695684996, + "nauc_precision_at_3_max": 0.4356832908424444, + "nauc_precision_at_3_std": 0.41368899691317473, + "nauc_precision_at_5_diff1": 0.37338144053278555, + "nauc_precision_at_5_max": 0.37192948006470095, + "nauc_precision_at_5_std": 0.4020590203629894, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.14751155284105305, + "nauc_recall_at_100_max": 0.22564543811830431, + "nauc_recall_at_100_std": 0.36565185549033236, + "nauc_recall_at_10_diff1": 0.2860643473956765, + "nauc_recall_at_10_max": 0.29643996104309345, + "nauc_recall_at_10_std": 0.3487498575679017, + "nauc_recall_at_1_diff1": 0.6745448554552967, + "nauc_recall_at_1_max": 0.5000925535843469, + "nauc_recall_at_1_std": 0.4339496692158664, + "nauc_recall_at_20_diff1": 0.25203106653182167, + "nauc_recall_at_20_max": 0.27861817937663164, + "nauc_recall_at_20_std": 0.35497189059078726, + "nauc_recall_at_3_diff1": 0.47610663695685046, + "nauc_recall_at_3_max": 0.43568329084244445, + "nauc_recall_at_3_std": 0.41368899691317484, + "nauc_recall_at_5_diff1": 0.3733814405327858, + "nauc_recall_at_5_max": 0.37192948006470095, + "nauc_recall_at_5_std": 0.40205902036298935, + "ndcg_at_1": 0.08163, + "ndcg_at_10": 0.12847, + "ndcg_at_100": 0.18934, + "ndcg_at_1000": 0.25043, + "ndcg_at_20": 0.14645, + "ndcg_at_3": 0.10338, + "ndcg_at_5": 0.11526, + "precision_at_1": 0.08163, + "precision_at_10": 0.0189, + "precision_at_100": 0.00502, + "precision_at_1000": 0.001, + "precision_at_20": 0.01305, + "precision_at_3": 0.03974, + "precision_at_5": 0.02965, + "recall_at_1": 0.08163, + "recall_at_10": 0.18904, + "recall_at_100": 0.50161, + "recall_at_1000": 1.0, + "recall_at_20": 0.26101, + "recall_at_3": 0.11923, + "recall_at_5": 0.14823 + }, + { + "cv_recall_at_1": 0.096, + "cv_recall_at_10": 0.218, + "cv_recall_at_100": 0.462, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.271, + "cv_recall_at_3": 0.149, + "cv_recall_at_5": 0.172, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.15148, + "map_at_1": 0.096, + "map_at_10": 0.13099, + "map_at_100": 0.13896, + "map_at_1000": 0.14107, + "map_at_20": 0.13474, + "map_at_3": 0.11983, + "map_at_5": 0.12488, + "mrr_at_1": 0.096, + "mrr_at_10": 0.13098531746031747, + "mrr_at_100": 0.13896170761443877, + "mrr_at_1000": 0.14107497329376809, + "mrr_at_20": 0.13473580277651484, + "mrr_at_3": 0.11983333333333332, + "mrr_at_5": 0.12488333333333335, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.31076379300822904, + "nauc_cv_recall_at_100_max": 0.3998440487367423, + "nauc_cv_recall_at_100_std": 0.324715250876404, + "nauc_cv_recall_at_10_diff1": 0.38215109042528295, + "nauc_cv_recall_at_10_max": 0.3335709361342387, + "nauc_cv_recall_at_10_std": 0.19664777282468648, + "nauc_cv_recall_at_1_diff1": 0.6314143412932651, + "nauc_cv_recall_at_1_max": 0.41841069671764586, + "nauc_cv_recall_at_1_std": 0.20914944995975326, + "nauc_cv_recall_at_20_diff1": 0.355102914918089, + "nauc_cv_recall_at_20_max": 0.3481563529556926, + "nauc_cv_recall_at_20_std": 0.21010726183773293, + "nauc_cv_recall_at_3_diff1": 0.4801968463257448, + "nauc_cv_recall_at_3_max": 0.4484818156216887, + "nauc_cv_recall_at_3_std": 0.2781092541679922, + "nauc_cv_recall_at_5_diff1": 0.439680987927112, + "nauc_cv_recall_at_5_max": 0.40597175482079656, + "nauc_cv_recall_at_5_std": 0.26277429296196414, + "nauc_map_at_1000_diff1": 0.5160698132598479, + "nauc_map_at_1000_max": 0.41257953784470985, + "nauc_map_at_1000_std": 0.2405028977362211, + "nauc_map_at_100_diff1": 0.5149936007224473, + "nauc_map_at_100_max": 0.4123488602331851, + "nauc_map_at_100_std": 0.24045219275874782, + "nauc_map_at_10_diff1": 0.521754177659713, + "nauc_map_at_10_max": 0.4099643031558265, + "nauc_map_at_10_std": 0.23509281560035314, + "nauc_map_at_1_diff1": 0.6314143412932651, + "nauc_map_at_1_max": 0.41841069671764586, + "nauc_map_at_1_std": 0.20914944995975326, + "nauc_map_at_20_diff1": 0.5178858575458826, + "nauc_map_at_20_max": 0.41082413685719776, + "nauc_map_at_20_std": 0.23591664956917177, + "nauc_map_at_3_diff1": 0.5493791787574669, + "nauc_map_at_3_max": 0.4366534865844311, + "nauc_map_at_3_std": 0.2512776116625236, + "nauc_map_at_5_diff1": 0.536436215744084, + "nauc_map_at_5_max": 0.42485582874292555, + "nauc_map_at_5_std": 0.24757945180493524, + "nauc_mrr_at_1000_diff1": 0.5160698132598479, + "nauc_mrr_at_1000_max": 0.41257953784470985, + "nauc_mrr_at_1000_std": 0.2405028977362211, + "nauc_mrr_at_100_diff1": 0.5149936007224473, + "nauc_mrr_at_100_max": 0.4123488602331851, + "nauc_mrr_at_100_std": 0.24045219275874782, + "nauc_mrr_at_10_diff1": 0.521754177659713, + "nauc_mrr_at_10_max": 0.4099643031558265, + "nauc_mrr_at_10_std": 0.23509281560035314, + "nauc_mrr_at_1_diff1": 0.6314143412932651, + "nauc_mrr_at_1_max": 0.41841069671764586, + "nauc_mrr_at_1_std": 0.20914944995975326, + "nauc_mrr_at_20_diff1": 0.5178858575458826, + "nauc_mrr_at_20_max": 0.41082413685719776, + "nauc_mrr_at_20_std": 0.23591664956917177, + "nauc_mrr_at_3_diff1": 0.5493791787574669, + "nauc_mrr_at_3_max": 0.4366534865844311, + "nauc_mrr_at_3_std": 0.2512776116625236, + "nauc_mrr_at_5_diff1": 0.536436215744084, + "nauc_mrr_at_5_max": 0.42485582874292555, + "nauc_mrr_at_5_std": 0.24757945180493524, + "nauc_ndcg_at_1000_diff1": 0.480757159232348, + "nauc_ndcg_at_1000_max": 0.4060259454455605, + "nauc_ndcg_at_1000_std": 0.24825519139717867, + "nauc_ndcg_at_100_diff1": 0.4513665795585693, + "nauc_ndcg_at_100_max": 0.40384720146777503, + "nauc_ndcg_at_100_std": 0.25896097092497994, + "nauc_ndcg_at_10_diff1": 0.47838296717969875, + "nauc_ndcg_at_10_max": 0.3880846270218068, + "nauc_ndcg_at_10_std": 0.22518898143499563, + "nauc_ndcg_at_1_diff1": 0.6314143412932651, + "nauc_ndcg_at_1_max": 0.41841069671764586, + "nauc_ndcg_at_1_std": 0.20914944995975326, + "nauc_ndcg_at_20_diff1": 0.46631502103347294, + "nauc_ndcg_at_20_max": 0.39150783743703094, + "nauc_ndcg_at_20_std": 0.22872029282256662, + "nauc_ndcg_at_3_diff1": 0.5292375963981599, + "nauc_ndcg_at_3_max": 0.44017880892606553, + "nauc_ndcg_at_3_std": 0.25928897132564943, + "nauc_ndcg_at_5_diff1": 0.5075959544238287, + "nauc_ndcg_at_5_max": 0.4199052075354429, + "nauc_ndcg_at_5_std": 0.2526852162522937, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3107637930082286, + "nauc_precision_at_100_max": 0.39984404873674223, + "nauc_precision_at_100_std": 0.32471525087640407, + "nauc_precision_at_10_diff1": 0.3821510904252831, + "nauc_precision_at_10_max": 0.3335709361342387, + "nauc_precision_at_10_std": 0.19664777282468657, + "nauc_precision_at_1_diff1": 0.6314143412932651, + "nauc_precision_at_1_max": 0.41841069671764586, + "nauc_precision_at_1_std": 0.20914944995975326, + "nauc_precision_at_20_diff1": 0.35510291491808915, + "nauc_precision_at_20_max": 0.34815635295569247, + "nauc_precision_at_20_std": 0.21010726183773287, + "nauc_precision_at_3_diff1": 0.48019684632574444, + "nauc_precision_at_3_max": 0.44848181562168865, + "nauc_precision_at_3_std": 0.27810925416799187, + "nauc_precision_at_5_diff1": 0.43968098792711197, + "nauc_precision_at_5_max": 0.4059717548207963, + "nauc_precision_at_5_std": 0.26277429296196425, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.31076379300822904, + "nauc_recall_at_100_max": 0.3998440487367423, + "nauc_recall_at_100_std": 0.324715250876404, + "nauc_recall_at_10_diff1": 0.38215109042528295, + "nauc_recall_at_10_max": 0.3335709361342387, + "nauc_recall_at_10_std": 0.19664777282468648, + "nauc_recall_at_1_diff1": 0.6314143412932651, + "nauc_recall_at_1_max": 0.41841069671764586, + "nauc_recall_at_1_std": 0.20914944995975326, + "nauc_recall_at_20_diff1": 0.355102914918089, + "nauc_recall_at_20_max": 0.3481563529556926, + "nauc_recall_at_20_std": 0.21010726183773293, + "nauc_recall_at_3_diff1": 0.4801968463257448, + "nauc_recall_at_3_max": 0.4484818156216887, + "nauc_recall_at_3_std": 0.2781092541679922, + "nauc_recall_at_5_diff1": 0.439680987927112, + "nauc_recall_at_5_max": 0.40597175482079656, + "nauc_recall_at_5_std": 0.26277429296196414, + "ndcg_at_1": 0.096, + "ndcg_at_10": 0.15148, + "ndcg_at_100": 0.19904, + "ndcg_at_1000": 0.26528, + "ndcg_at_20": 0.16497, + "ndcg_at_3": 0.12734, + "ndcg_at_5": 0.13664, + "precision_at_1": 0.096, + "precision_at_10": 0.0218, + "precision_at_100": 0.00462, + "precision_at_1000": 0.001, + "precision_at_20": 0.01355, + "precision_at_3": 0.04967, + "precision_at_5": 0.0344, + "recall_at_1": 0.096, + "recall_at_10": 0.218, + "recall_at_100": 0.462, + "recall_at_1000": 1.0, + "recall_at_20": 0.271, + "recall_at_3": 0.149, + "recall_at_5": 0.172 + }, + { + "cv_recall_at_1": 0.52843, + "cv_recall_at_10": 0.76283, + "cv_recall_at_100": 0.94313, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.81969, + "cv_recall_at_3": 0.64216, + "cv_recall_at_5": 0.69071, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.63888, + "map_at_1": 0.52843, + "map_at_10": 0.60011, + "map_at_100": 0.60716, + "map_at_1000": 0.60746, + "map_at_20": 0.60404, + "map_at_3": 0.57929, + "map_at_5": 0.59038, + "mrr_at_1": 0.5284327323162274, + "mrr_at_10": 0.600105122955331, + "mrr_at_100": 0.6071648409715369, + "mrr_at_1000": 0.607455781012974, + "mrr_at_20": 0.6040383115486814, + "mrr_at_3": 0.5792880258899676, + "mrr_at_5": 0.5903837263060561, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6892368194018946, + "nauc_cv_recall_at_100_max": 0.6442594748788051, + "nauc_cv_recall_at_100_std": 0.7110681447725741, + "nauc_cv_recall_at_10_diff1": 0.6599255418627427, + "nauc_cv_recall_at_10_max": 0.5807602290666405, + "nauc_cv_recall_at_10_std": 0.4085739259797813, + "nauc_cv_recall_at_1_diff1": 0.8018268384808368, + "nauc_cv_recall_at_1_max": 0.5936789618921932, + "nauc_cv_recall_at_1_std": 0.2180061753760995, + "nauc_cv_recall_at_20_diff1": 0.6525563271016608, + "nauc_cv_recall_at_20_max": 0.5619932280203499, + "nauc_cv_recall_at_20_std": 0.4972492933035224, + "nauc_cv_recall_at_3_diff1": 0.7063676220555728, + "nauc_cv_recall_at_3_max": 0.590015123976248, + "nauc_cv_recall_at_3_std": 0.30431843772328776, + "nauc_cv_recall_at_5_diff1": 0.6734167377323467, + "nauc_cv_recall_at_5_max": 0.5788347633839778, + "nauc_cv_recall_at_5_std": 0.3329561226259785, + "nauc_map_at_1000_diff1": 0.7568665768481858, + "nauc_map_at_1000_max": 0.590748890592957, + "nauc_map_at_1000_std": 0.271276252596812, + "nauc_map_at_100_diff1": 0.7568082284879722, + "nauc_map_at_100_max": 0.5907960412124283, + "nauc_map_at_100_std": 0.27169107729937547, + "nauc_map_at_10_diff1": 0.7551235887991433, + "nauc_map_at_10_max": 0.5909815677942487, + "nauc_map_at_10_std": 0.27051223484286024, + "nauc_map_at_1_diff1": 0.8018268384808368, + "nauc_map_at_1_max": 0.5936789618921932, + "nauc_map_at_1_std": 0.2180061753760995, + "nauc_map_at_20_diff1": 0.7558559584549839, + "nauc_map_at_20_max": 0.5904422446496017, + "nauc_map_at_20_std": 0.2725636557190311, + "nauc_map_at_3_diff1": 0.7603606500012732, + "nauc_map_at_3_max": 0.593144099292417, + "nauc_map_at_3_std": 0.2587097130516631, + "nauc_map_at_5_diff1": 0.7553714760197734, + "nauc_map_at_5_max": 0.5909112343550218, + "nauc_map_at_5_std": 0.26418869764691005, + "nauc_mrr_at_1000_diff1": 0.756866537016681, + "nauc_mrr_at_1000_max": 0.590748966461794, + "nauc_mrr_at_1000_std": 0.2712762912022093, + "nauc_mrr_at_100_diff1": 0.7568082284879722, + "nauc_mrr_at_100_max": 0.5907960412124283, + "nauc_mrr_at_100_std": 0.27169107729937547, + "nauc_mrr_at_10_diff1": 0.7551235887991433, + "nauc_mrr_at_10_max": 0.5909815677942487, + "nauc_mrr_at_10_std": 0.27051223484286024, + "nauc_mrr_at_1_diff1": 0.8018268384808368, + "nauc_mrr_at_1_max": 0.5936789618921932, + "nauc_mrr_at_1_std": 0.2180061753760995, + "nauc_mrr_at_20_diff1": 0.7558559584549839, + "nauc_mrr_at_20_max": 0.5904422446496017, + "nauc_mrr_at_20_std": 0.2725636557190311, + "nauc_mrr_at_3_diff1": 0.7603606500012732, + "nauc_mrr_at_3_max": 0.593144099292417, + "nauc_mrr_at_3_std": 0.2587097130516631, + "nauc_mrr_at_5_diff1": 0.7553714760197734, + "nauc_mrr_at_5_max": 0.5909112343550218, + "nauc_mrr_at_5_std": 0.26418869764691005, + "nauc_ndcg_at_1000_diff1": 0.7486336031563205, + "nauc_ndcg_at_1000_max": 0.5897078600668323, + "nauc_ndcg_at_1000_std": 0.286561756145907, + "nauc_ndcg_at_100_diff1": 0.7469180499307316, + "nauc_ndcg_at_100_max": 0.5912671824348572, + "nauc_ndcg_at_100_std": 0.298923732369518, + "nauc_ndcg_at_10_diff1": 0.7366787103268794, + "nauc_ndcg_at_10_max": 0.5889900838741208, + "nauc_ndcg_at_10_std": 0.2959522421148646, + "nauc_ndcg_at_1_diff1": 0.8018268384808368, + "nauc_ndcg_at_1_max": 0.5936789618921932, + "nauc_ndcg_at_1_std": 0.2180061753760995, + "nauc_ndcg_at_20_diff1": 0.7387233999152774, + "nauc_ndcg_at_20_max": 0.5864892929565019, + "nauc_ndcg_at_20_std": 0.3052934739148191, + "nauc_ndcg_at_3_diff1": 0.7472825895138822, + "nauc_ndcg_at_3_max": 0.5924345319725394, + "nauc_ndcg_at_3_std": 0.2699037851276475, + "nauc_ndcg_at_5_diff1": 0.737149718941511, + "nauc_ndcg_at_5_max": 0.588353926823294, + "nauc_ndcg_at_5_std": 0.2797707300943979, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.6892368194018897, + "nauc_precision_at_100_max": 0.6442594748788032, + "nauc_precision_at_100_std": 0.7110681447725753, + "nauc_precision_at_10_diff1": 0.6599255418627438, + "nauc_precision_at_10_max": 0.5807602290666415, + "nauc_precision_at_10_std": 0.4085739259797818, + "nauc_precision_at_1_diff1": 0.8018268384808368, + "nauc_precision_at_1_max": 0.5936789618921932, + "nauc_precision_at_1_std": 0.2180061753760995, + "nauc_precision_at_20_diff1": 0.6525563271016613, + "nauc_precision_at_20_max": 0.5619932280203506, + "nauc_precision_at_20_std": 0.49724929330352213, + "nauc_precision_at_3_diff1": 0.7063676220555728, + "nauc_precision_at_3_max": 0.5900151239762477, + "nauc_precision_at_3_std": 0.3043184377232873, + "nauc_precision_at_5_diff1": 0.6734167377323469, + "nauc_precision_at_5_max": 0.578834763383978, + "nauc_precision_at_5_std": 0.33295612262597796, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.6892368194018946, + "nauc_recall_at_100_max": 0.6442594748788051, + "nauc_recall_at_100_std": 0.7110681447725741, + "nauc_recall_at_10_diff1": 0.6599255418627427, + "nauc_recall_at_10_max": 0.5807602290666405, + "nauc_recall_at_10_std": 0.4085739259797813, + "nauc_recall_at_1_diff1": 0.8018268384808368, + "nauc_recall_at_1_max": 0.5936789618921932, + "nauc_recall_at_1_std": 0.2180061753760995, + "nauc_recall_at_20_diff1": 0.6525563271016608, + "nauc_recall_at_20_max": 0.5619932280203499, + "nauc_recall_at_20_std": 0.4972492933035224, + "nauc_recall_at_3_diff1": 0.7063676220555728, + "nauc_recall_at_3_max": 0.590015123976248, + "nauc_recall_at_3_std": 0.30431843772328776, + "nauc_recall_at_5_diff1": 0.6734167377323467, + "nauc_recall_at_5_max": 0.5788347633839778, + "nauc_recall_at_5_std": 0.3329561226259785, + "ndcg_at_1": 0.52843, + "ndcg_at_10": 0.63888, + "ndcg_at_100": 0.67605, + "ndcg_at_1000": 0.68346, + "ndcg_at_20": 0.65324, + "ndcg_at_3": 0.59547, + "ndcg_at_5": 0.61546, + "precision_at_1": 0.52843, + "precision_at_10": 0.07628, + "precision_at_100": 0.00943, + "precision_at_1000": 0.001, + "precision_at_20": 0.04098, + "precision_at_3": 0.21405, + "precision_at_5": 0.13814, + "recall_at_1": 0.52843, + "recall_at_10": 0.76283, + "recall_at_100": 0.94313, + "recall_at_1000": 1.0, + "recall_at_20": 0.81969, + "recall_at_3": 0.64216, + "recall_at_5": 0.69071 + }, + { + "cv_recall_at_1": 0.41015, + "cv_recall_at_10": 0.63636, + "cv_recall_at_100": 0.84884, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.6945, + "cv_recall_at_3": 0.51268, + "cv_recall_at_5": 0.56448, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.51471, + "map_at_1": 0.41015, + "map_at_10": 0.47695, + "map_at_100": 0.48473, + "map_at_1000": 0.48547, + "map_at_20": 0.48101, + "map_at_3": 0.45595, + "map_at_5": 0.46769, + "mrr_at_1": 0.41014799154334036, + "mrr_at_10": 0.4769543441055068, + "mrr_at_100": 0.4847251288666404, + "mrr_at_1000": 0.48546655091217855, + "mrr_at_20": 0.48100566949619306, + "mrr_at_3": 0.4559548978153628, + "mrr_at_5": 0.46768851303734993, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5409143387106966, + "nauc_cv_recall_at_100_max": 0.49919692850959463, + "nauc_cv_recall_at_100_std": 0.579791515955671, + "nauc_cv_recall_at_10_diff1": 0.6391618676932485, + "nauc_cv_recall_at_10_max": 0.5607279113763848, + "nauc_cv_recall_at_10_std": 0.5049355725436926, + "nauc_cv_recall_at_1_diff1": 0.78493057893207, + "nauc_cv_recall_at_1_max": 0.5561248078698109, + "nauc_cv_recall_at_1_std": 0.39527525661071433, + "nauc_cv_recall_at_20_diff1": 0.6027928145194872, + "nauc_cv_recall_at_20_max": 0.5624857110624909, + "nauc_cv_recall_at_20_std": 0.548086828891615, + "nauc_cv_recall_at_3_diff1": 0.6958158815323969, + "nauc_cv_recall_at_3_max": 0.5533491395012368, + "nauc_cv_recall_at_3_std": 0.4420005231182388, + "nauc_cv_recall_at_5_diff1": 0.6709801297614552, + "nauc_cv_recall_at_5_max": 0.5435911362820227, + "nauc_cv_recall_at_5_std": 0.450371666636732, + "nauc_map_at_1000_diff1": 0.7376421388088805, + "nauc_map_at_1000_max": 0.5560148083486584, + "nauc_map_at_1000_std": 0.42565795193083183, + "nauc_map_at_100_diff1": 0.7372071244493027, + "nauc_map_at_100_max": 0.5558178928230634, + "nauc_map_at_100_std": 0.42580840091303673, + "nauc_map_at_10_diff1": 0.7382661772755509, + "nauc_map_at_10_max": 0.5564344976313585, + "nauc_map_at_10_std": 0.4244538090998832, + "nauc_map_at_1_diff1": 0.78493057893207, + "nauc_map_at_1_max": 0.5561248078698109, + "nauc_map_at_1_std": 0.39527525661071433, + "nauc_map_at_20_diff1": 0.7369171276008715, + "nauc_map_at_20_max": 0.5567639433197569, + "nauc_map_at_20_std": 0.42660328682417953, + "nauc_map_at_3_diff1": 0.7454743543551721, + "nauc_map_at_3_max": 0.5559979243971782, + "nauc_map_at_3_std": 0.41690566638467735, + "nauc_map_at_5_diff1": 0.740992091976634, + "nauc_map_at_5_max": 0.5545131399375515, + "nauc_map_at_5_std": 0.4189616480090802, + "nauc_mrr_at_1000_diff1": 0.7376421826035284, + "nauc_mrr_at_1000_max": 0.5560147531236701, + "nauc_mrr_at_1000_std": 0.4256578903304658, + "nauc_mrr_at_100_diff1": 0.7372071244493027, + "nauc_mrr_at_100_max": 0.5558178928230634, + "nauc_mrr_at_100_std": 0.42580840091303673, + "nauc_mrr_at_10_diff1": 0.7382661772755509, + "nauc_mrr_at_10_max": 0.5564344976313585, + "nauc_mrr_at_10_std": 0.4244538090998832, + "nauc_mrr_at_1_diff1": 0.78493057893207, + "nauc_mrr_at_1_max": 0.5561248078698109, + "nauc_mrr_at_1_std": 0.39527525661071433, + "nauc_mrr_at_20_diff1": 0.7369171276008715, + "nauc_mrr_at_20_max": 0.5567639433197569, + "nauc_mrr_at_20_std": 0.42660328682417953, + "nauc_mrr_at_3_diff1": 0.7454743543551721, + "nauc_mrr_at_3_max": 0.5559979243971782, + "nauc_mrr_at_3_std": 0.41690566638467735, + "nauc_mrr_at_5_diff1": 0.740992091976634, + "nauc_mrr_at_5_max": 0.5545131399375515, + "nauc_mrr_at_5_std": 0.4189616480090802, + "nauc_ndcg_at_1000_diff1": 0.7245111337667335, + "nauc_ndcg_at_1000_max": 0.5552434039481599, + "nauc_ndcg_at_1000_std": 0.4366429775562634, + "nauc_ndcg_at_100_diff1": 0.7132014389766401, + "nauc_ndcg_at_100_max": 0.5513373018774222, + "nauc_ndcg_at_100_std": 0.44437309757597077, + "nauc_ndcg_at_10_diff1": 0.7168649688744982, + "nauc_ndcg_at_10_max": 0.5570884075954164, + "nauc_ndcg_at_10_std": 0.4411216802971647, + "nauc_ndcg_at_1_diff1": 0.78493057893207, + "nauc_ndcg_at_1_max": 0.5561248078698109, + "nauc_ndcg_at_1_std": 0.39527525661071433, + "nauc_ndcg_at_20_diff1": 0.7113248768688196, + "nauc_ndcg_at_20_max": 0.557708419773681, + "nauc_ndcg_at_20_std": 0.4488177602434449, + "nauc_ndcg_at_3_diff1": 0.7329128659932554, + "nauc_ndcg_at_3_max": 0.5553759897014252, + "nauc_ndcg_at_3_std": 0.42329473035978266, + "nauc_ndcg_at_5_diff1": 0.7242215193891609, + "nauc_ndcg_at_5_max": 0.5521105842445077, + "nauc_ndcg_at_5_std": 0.42666241366994423, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5409143387106957, + "nauc_precision_at_100_max": 0.49919692850959463, + "nauc_precision_at_100_std": 0.5797915159556701, + "nauc_precision_at_10_diff1": 0.639161867693248, + "nauc_precision_at_10_max": 0.5607279113763848, + "nauc_precision_at_10_std": 0.5049355725436926, + "nauc_precision_at_1_diff1": 0.78493057893207, + "nauc_precision_at_1_max": 0.5561248078698109, + "nauc_precision_at_1_std": 0.39527525661071433, + "nauc_precision_at_20_diff1": 0.6027928145194867, + "nauc_precision_at_20_max": 0.5624857110624905, + "nauc_precision_at_20_std": 0.5480868288916155, + "nauc_precision_at_3_diff1": 0.6958158815323969, + "nauc_precision_at_3_max": 0.5533491395012369, + "nauc_precision_at_3_std": 0.4420005231182392, + "nauc_precision_at_5_diff1": 0.670980129761455, + "nauc_precision_at_5_max": 0.5435911362820229, + "nauc_precision_at_5_std": 0.4503716666367316, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5409143387106966, + "nauc_recall_at_100_max": 0.49919692850959463, + "nauc_recall_at_100_std": 0.579791515955671, + "nauc_recall_at_10_diff1": 0.6391618676932485, + "nauc_recall_at_10_max": 0.5607279113763848, + "nauc_recall_at_10_std": 0.5049355725436926, + "nauc_recall_at_1_diff1": 0.78493057893207, + "nauc_recall_at_1_max": 0.5561248078698109, + "nauc_recall_at_1_std": 0.39527525661071433, + "nauc_recall_at_20_diff1": 0.6027928145194872, + "nauc_recall_at_20_max": 0.5624857110624909, + "nauc_recall_at_20_std": 0.548086828891615, + "nauc_recall_at_3_diff1": 0.6958158815323969, + "nauc_recall_at_3_max": 0.5533491395012368, + "nauc_recall_at_3_std": 0.4420005231182388, + "nauc_recall_at_5_diff1": 0.6709801297614552, + "nauc_recall_at_5_max": 0.5435911362820227, + "nauc_recall_at_5_std": 0.450371666636732, + "ndcg_at_1": 0.41015, + "ndcg_at_10": 0.51471, + "ndcg_at_100": 0.55755, + "ndcg_at_1000": 0.57693, + "ndcg_at_20": 0.52943, + "ndcg_at_3": 0.47055, + "ndcg_at_5": 0.49179, + "precision_at_1": 0.41015, + "precision_at_10": 0.06364, + "precision_at_100": 0.00849, + "precision_at_1000": 0.001, + "precision_at_20": 0.03473, + "precision_at_3": 0.17089, + "precision_at_5": 0.1129, + "recall_at_1": 0.41015, + "recall_at_10": 0.63636, + "recall_at_100": 0.84884, + "recall_at_1000": 1.0, + "recall_at_20": 0.6945, + "recall_at_3": 0.51268, + "recall_at_5": 0.56448 + }, + { + "cv_recall_at_1": 0.625, + "cv_recall_at_10": 0.801, + "cv_recall_at_100": 0.94, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.844, + "cv_recall_at_3": 0.719, + "cv_recall_at_5": 0.748, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.71013, + "map_at_1": 0.625, + "map_at_10": 0.68155, + "map_at_100": 0.68709, + "map_at_1000": 0.68745, + "map_at_20": 0.68467, + "map_at_3": 0.66783, + "map_at_5": 0.67448, + "mrr_at_1": 0.625, + "mrr_at_10": 0.6815527777777778, + "mrr_at_100": 0.6870917599034414, + "mrr_at_1000": 0.6874521809565477, + "mrr_at_20": 0.684674403679396, + "mrr_at_3": 0.6678333333333335, + "mrr_at_5": 0.6744833333333335, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.7051976346093998, + "nauc_cv_recall_at_100_max": 0.9334733893557434, + "nauc_cv_recall_at_100_std": 0.9426548397136629, + "nauc_cv_recall_at_10_diff1": 0.8277429003005983, + "nauc_cv_recall_at_10_max": 0.7307057294411077, + "nauc_cv_recall_at_10_std": 0.11950930651531334, + "nauc_cv_recall_at_1_diff1": 0.9062076460314379, + "nauc_cv_recall_at_1_max": 0.673808655152338, + "nauc_cv_recall_at_1_std": 0.034311663108868774, + "nauc_cv_recall_at_20_diff1": 0.8017533723529852, + "nauc_cv_recall_at_20_max": 0.7809038157200636, + "nauc_cv_recall_at_20_std": 0.2732610716169713, + "nauc_cv_recall_at_3_diff1": 0.8567395906845451, + "nauc_cv_recall_at_3_max": 0.6611800870066405, + "nauc_cv_recall_at_3_std": 0.02962867148430545, + "nauc_cv_recall_at_5_diff1": 0.8484581277919656, + "nauc_cv_recall_at_5_max": 0.6963547429103827, + "nauc_cv_recall_at_5_std": 0.05383034830423127, + "nauc_map_at_1000_diff1": 0.8827887804526993, + "nauc_map_at_1000_max": 0.6807906637408886, + "nauc_map_at_1000_std": 0.04991212739584506, + "nauc_map_at_100_diff1": 0.8825373269740879, + "nauc_map_at_100_max": 0.6811539785400426, + "nauc_map_at_100_std": 0.05113954398193766, + "nauc_map_at_10_diff1": 0.8833213241719006, + "nauc_map_at_10_max": 0.6792061204673818, + "nauc_map_at_10_std": 0.042588582733769274, + "nauc_map_at_1_diff1": 0.9062076460314379, + "nauc_map_at_1_max": 0.673808655152338, + "nauc_map_at_1_std": 0.034311663108868774, + "nauc_map_at_20_diff1": 0.882846654608725, + "nauc_map_at_20_max": 0.6806083956333864, + "nauc_map_at_20_std": 0.04739113496391819, + "nauc_map_at_3_diff1": 0.8854941510333725, + "nauc_map_at_3_max": 0.6688552291574957, + "nauc_map_at_3_std": 0.030730587130158093, + "nauc_map_at_5_diff1": 0.8844906719331308, + "nauc_map_at_5_max": 0.6758096501673212, + "nauc_map_at_5_std": 0.03580337858057999, + "nauc_mrr_at_1000_diff1": 0.8827887804526993, + "nauc_mrr_at_1000_max": 0.6807906637408886, + "nauc_mrr_at_1000_std": 0.04991212739584506, + "nauc_mrr_at_100_diff1": 0.8825373269740879, + "nauc_mrr_at_100_max": 0.6811539785400426, + "nauc_mrr_at_100_std": 0.05113954398193766, + "nauc_mrr_at_10_diff1": 0.8833213241719006, + "nauc_mrr_at_10_max": 0.6792061204673818, + "nauc_mrr_at_10_std": 0.042588582733769274, + "nauc_mrr_at_1_diff1": 0.9062076460314379, + "nauc_mrr_at_1_max": 0.673808655152338, + "nauc_mrr_at_1_std": 0.034311663108868774, + "nauc_mrr_at_20_diff1": 0.882846654608725, + "nauc_mrr_at_20_max": 0.6806083956333864, + "nauc_mrr_at_20_std": 0.04739113496391819, + "nauc_mrr_at_3_diff1": 0.8854941510333725, + "nauc_mrr_at_3_max": 0.6688552291574957, + "nauc_mrr_at_3_std": 0.030730587130158093, + "nauc_mrr_at_5_diff1": 0.8844906719331308, + "nauc_mrr_at_5_max": 0.6758096501673212, + "nauc_mrr_at_5_std": 0.03580337858057999, + "nauc_ndcg_at_1000_diff1": 0.8760726250721362, + "nauc_ndcg_at_1000_max": 0.6876964589525962, + "nauc_ndcg_at_1000_std": 0.068740569878718, + "nauc_ndcg_at_100_diff1": 0.8698242603656416, + "nauc_ndcg_at_100_max": 0.6967397827273338, + "nauc_ndcg_at_100_std": 0.10051970473333424, + "nauc_ndcg_at_10_diff1": 0.8729424325302123, + "nauc_ndcg_at_10_max": 0.6881091554863467, + "nauc_ndcg_at_10_std": 0.05583289621595604, + "nauc_ndcg_at_1_diff1": 0.9062076460314379, + "nauc_ndcg_at_1_max": 0.673808655152338, + "nauc_ndcg_at_1_std": 0.034311663108868774, + "nauc_ndcg_at_20_diff1": 0.8706189980509209, + "nauc_ndcg_at_20_max": 0.6942792549563985, + "nauc_ndcg_at_20_std": 0.07699478300441558, + "nauc_ndcg_at_3_diff1": 0.8786736061073106, + "nauc_ndcg_at_3_max": 0.6670500054524965, + "nauc_ndcg_at_3_std": 0.030395050026420765, + "nauc_ndcg_at_5_diff1": 0.8765023815329428, + "nauc_ndcg_at_5_max": 0.6799301445714997, + "nauc_ndcg_at_5_std": 0.03947961532415831, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.7051976346093989, + "nauc_precision_at_100_max": 0.9334733893557411, + "nauc_precision_at_100_std": 0.9426548397136639, + "nauc_precision_at_10_diff1": 0.8277429003005984, + "nauc_precision_at_10_max": 0.7307057294411068, + "nauc_precision_at_10_std": 0.11950930651531294, + "nauc_precision_at_1_diff1": 0.9062076460314379, + "nauc_precision_at_1_max": 0.673808655152338, + "nauc_precision_at_1_std": 0.034311663108868774, + "nauc_precision_at_20_diff1": 0.8017533723529853, + "nauc_precision_at_20_max": 0.7809038157200632, + "nauc_precision_at_20_std": 0.27326107161697166, + "nauc_precision_at_3_diff1": 0.8567395906845451, + "nauc_precision_at_3_max": 0.6611800870066414, + "nauc_precision_at_3_std": 0.029628671484306243, + "nauc_precision_at_5_diff1": 0.8484581277919659, + "nauc_precision_at_5_max": 0.6963547429103824, + "nauc_precision_at_5_std": 0.053830348304230724, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.7051976346093998, + "nauc_recall_at_100_max": 0.9334733893557434, + "nauc_recall_at_100_std": 0.9426548397136629, + "nauc_recall_at_10_diff1": 0.8277429003005983, + "nauc_recall_at_10_max": 0.7307057294411077, + "nauc_recall_at_10_std": 0.11950930651531334, + "nauc_recall_at_1_diff1": 0.9062076460314379, + "nauc_recall_at_1_max": 0.673808655152338, + "nauc_recall_at_1_std": 0.034311663108868774, + "nauc_recall_at_20_diff1": 0.8017533723529852, + "nauc_recall_at_20_max": 0.7809038157200636, + "nauc_recall_at_20_std": 0.2732610716169713, + "nauc_recall_at_3_diff1": 0.8567395906845451, + "nauc_recall_at_3_max": 0.6611800870066405, + "nauc_recall_at_3_std": 0.02962867148430545, + "nauc_recall_at_5_diff1": 0.8484581277919656, + "nauc_recall_at_5_max": 0.6963547429103827, + "nauc_recall_at_5_std": 0.05383034830423127, + "ndcg_at_1": 0.625, + "ndcg_at_10": 0.71013, + "ndcg_at_100": 0.73893, + "ndcg_at_1000": 0.74698, + "ndcg_at_20": 0.72118, + "ndcg_at_3": 0.68103, + "ndcg_at_5": 0.693, + "precision_at_1": 0.625, + "precision_at_10": 0.0801, + "precision_at_100": 0.0094, + "precision_at_1000": 0.001, + "precision_at_20": 0.0422, + "precision_at_3": 0.23967, + "precision_at_5": 0.1496, + "recall_at_1": 0.625, + "recall_at_10": 0.801, + "recall_at_100": 0.94, + "recall_at_1000": 1.0, + "recall_at_20": 0.844, + "recall_at_3": 0.719, + "recall_at_5": 0.748 + } + ] + }, + "task_name": "WITT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json new file mode 100644 index 0000000000..96ba2e7de4 --- /dev/null +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json @@ -0,0 +1 @@ +{"name": "google/siglip-so400m-patch14-384", "revision": "9fdffc58afc957d1a03a25b10dba0329ab15c2a3", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index cf05096996..3fd3e3c81b 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -25,13 +25,31 @@ "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "nyu-visionx/moco-v3-vit-b", "nyu-visionx/moco-v3-vit-l", - # "google/siglip-so400m-patch14-384",# haven't pushed + "google/siglip-so400m-patch14-384", + "google/siglip-base-patch16-256-multilingual", + "google/siglip-base-patch16-256", + "google/siglip-base-patch16-512", + "google/siglip-base-patch16-384", + "google/siglip-base-patch16-224", + "google/siglip-large-patch16-256", + "google/siglip-large-patch16-384", + "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", + "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", + "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", + "laion/CLIP-ViT-L-14-laion2B-s32B-b82K", + "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", + "TIGER-Lab/VLM2Vec-LoRA", + "TIGER-Lab/VLM2Vec-Full", + # "embed-english-v3.0-v", # not feasible to run due to the 40 images/min constraint ]: model = mteb.get_model(model_name) tasks = mteb.get_tasks( task_types=[ "Any2AnyRetrieval", - "AbsTaskAny2AnyMultiChoice", + "Any2AnyMultiChoice", "Any2TextMutipleChoice", "ImageClustering", "ImageClassification", @@ -41,5 +59,9 @@ "ZeroShotClassification", ] ) + # get i-only tasks for i-only models. + if ("moco" in model_name) or ("dinov2" in model_name): + tasks = [task for task in tasks if "t" not in task.metadata.category] + evaluation = mteb.MTEB(tasks=tasks) results = evaluation.run(model, output_folder="results-mieb-final") From 4b0facca0d9be54bf3d70ec3ac7661126436041d Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Sat, 2 Nov 2024 03:51:05 +0800 Subject: [PATCH 091/154] [mieb] fixes for final run (#1374) * e5_v device arg * dataloader num_workers * vista doc * vista doc * run mieb * fix --- docs/mieb-docs/run_vista.md | 26 +++++++++++++++++++ .../Image/Any2AnyMultiChoiceEvaluator.py | 5 ++-- .../Image/Any2AnyRetrievalEvaluator.py | 5 ++-- .../ImageTextPairClassificationEvaluator.py | 4 ++- .../evaluators/Image/VisualSTSEvaluator.py | 4 +-- .../Image/ZeroshotClassificationEvaluator.py | 4 ++- mteb/models/cohere_v.py | 9 ++++--- mteb/models/e5_v.py | 2 ++ mteb/models/evaclip_models.py | 2 +- scripts/run_mieb.py | 8 ++++++ 10 files changed, 56 insertions(+), 13 deletions(-) create mode 100644 docs/mieb-docs/run_vista.md diff --git a/docs/mieb-docs/run_vista.md b/docs/mieb-docs/run_vista.md new file mode 100644 index 0000000000..038b44bb92 --- /dev/null +++ b/docs/mieb-docs/run_vista.md @@ -0,0 +1,26 @@ +## set up VISTA + +the latest FlagEmbedding repo doesn't support VISTA anymore so we use a old version. +``` +git clone --no-checkout https://github.com/FlagOpen/FlagEmbedding.git +cd FlagEmbedding +git checkout 5c9260277977f8f8e256e56a8e12387552693af9 +pip install -e . +pip install torchvision timm einops ftfy +``` +download the vision tower for bge-base +``` +wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_base_en_v1.5.pth?download=true +``` +rename it to `visualized_base_en_V1.5.pth` +``` +mv Visualized_base_en_v1.5.pth?download=true visualized_base_en_V1.5.pth +``` +download the vision tower for bge-m3 +``` +wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_m3.pth?download=true +``` +rename it to `visualized_m3.pth` +``` +mv Visualized_m3.pth?download=true visualized_m3.pth +``` \ No newline at end of file diff --git a/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py index a391af472b..5fdbb112f3 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py @@ -4,6 +4,7 @@ import io import json import logging +import math import os from collections import defaultdict from typing import Any @@ -132,7 +133,7 @@ def search( batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=max(1, os.cpu_count() // 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) if q_modality == "image": query_embeddings = self.model.get_image_embeddings( @@ -182,7 +183,7 @@ def search( batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=max(1, os.cpu_count() // 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) if corpus_modality == "image": sub_corpus_embeddings = self.model.get_image_embeddings( diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index cb0119ea6e..a321979d26 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -4,6 +4,7 @@ import io import json import logging +import math import os from collections import defaultdict from typing import Any @@ -131,7 +132,7 @@ def search( batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=max(1, os.cpu_count() // 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) if q_modality == "image": query_embeddings = self.model.get_image_embeddings( @@ -181,7 +182,7 @@ def search( batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=max(1, os.cpu_count() // 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) if corpus_modality == "image": sub_corpus_embeddings = self.model.get_image_embeddings( diff --git a/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py index b548da365e..7e3d84bb87 100644 --- a/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py @@ -1,6 +1,8 @@ from __future__ import annotations import logging +import math +import os from typing import Any import torch @@ -106,7 +108,7 @@ def __call__( shuffle=False, # collate_fn=lambda x: x, # Identity collate function collate_fn=custom_collate_fn, - num_workers=4, + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) num_images_per_sample = ( diff --git a/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py b/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py index a442eb6a9a..a042d22f5a 100644 --- a/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py +++ b/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py @@ -78,14 +78,14 @@ def __call__( batch_size=encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=math.floor(os.cpu_count() / 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) sentence2_dataloader = DataLoader( self.sentence2_dataset, batch_size=encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=math.floor(os.cpu_count() / 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) embeddings1 = model.get_image_embeddings( diff --git a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py index d082f13517..0b3b7f8f67 100644 --- a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py @@ -1,6 +1,8 @@ from __future__ import annotations import logging +import math +import os from typing import Any import torch @@ -66,7 +68,7 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): batch_size=encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=16, + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) text_embeddings = model.get_text_embeddings( diff --git a/mteb/models/cohere_v.py b/mteb/models/cohere_v.py index cf98f7dc3c..d53fd662e8 100644 --- a/mteb/models/cohere_v.py +++ b/mteb/models/cohere_v.py @@ -1,5 +1,9 @@ from __future__ import annotations +import base64 +import io +import os +import time from functools import partial from typing import Any @@ -8,11 +12,8 @@ from torch.utils.data import DataLoader from torchvision import transforms from tqdm import tqdm -import os -import io -import base64 + import mteb -import time from mteb.model_meta import ModelMeta api_key = os.getenv("COHERE_API_KEY") diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index 70bc20cabf..5647bee380 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -21,6 +21,8 @@ def __init__( ): self.model_name = model_name self.processor = LlavaNextProcessor.from_pretrained(model_name) + if "device" in kwargs: + self.device = kwargs.pop("device") self.model = LlavaNextForConditionalGeneration.from_pretrained( model_name, **kwargs ) diff --git a/mteb/models/evaclip_models.py b/mteb/models/evaclip_models.py index 1e123096ca..015c965c07 100644 --- a/mteb/models/evaclip_models.py +++ b/mteb/models/evaclip_models.py @@ -13,8 +13,8 @@ def evaclip_loader(**kwargs): try: - import sys import os + import sys sys.path.insert(0, os.path.join(os.getcwd(), "EVA/EVA-CLIP/rei")) diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index 3fd3e3c81b..676fd966a7 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -43,6 +43,14 @@ "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", "TIGER-Lab/VLM2Vec-LoRA", "TIGER-Lab/VLM2Vec-Full", + "Salesforce/blip-itm-base-coco", + "Salesforce/blip-itm-large-coco", + "Salesforce/blip-itm-base-flickr", + "Salesforce/blip-itm-large-flickr", + "EVA02-CLIP-B-16", + "EVA02-CLIP-L-14", + "EVA02-CLIP-bigE-14", + "EVA02-CLIP-bigE-14-plus", # "embed-english-v3.0-v", # not feasible to run due to the 40 images/min constraint ]: model = mteb.get_model(model_name) From a449b244ed964ba277ef83047d5f53fa588045c0 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Sat, 2 Nov 2024 03:53:31 +0800 Subject: [PATCH 092/154] Update run_vista.md --- docs/mieb-docs/run_vista.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/mieb-docs/run_vista.md b/docs/mieb-docs/run_vista.md index 038b44bb92..008342c0ce 100644 --- a/docs/mieb-docs/run_vista.md +++ b/docs/mieb-docs/run_vista.md @@ -8,8 +8,9 @@ git checkout 5c9260277977f8f8e256e56a8e12387552693af9 pip install -e . pip install torchvision timm einops ftfy ``` -download the vision tower for bge-base +back to the root folder of mteb; download the vision tower for bge-base ``` +cd .. wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_base_en_v1.5.pth?download=true ``` rename it to `visualized_base_en_V1.5.pth` @@ -23,4 +24,4 @@ wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_m3.pth?d rename it to `visualized_m3.pth` ``` mv Visualized_m3.pth?download=true visualized_m3.pth -``` \ No newline at end of file +``` From 3a18fbdafa25696080fc4fa18c1875f64d6a4010 Mon Sep 17 00:00:00 2001 From: Niklas Muennighoff Date: Sun, 3 Nov 2024 22:34:00 -0800 Subject: [PATCH 093/154] [mieb] Fix torch no grad (#1378) Fix torch no grad --- mteb/models/e5_v.py | 67 +++++++++++++++++------------------ mteb/models/vlm2vec_models.py | 26 ++++++-------- 2 files changed, 44 insertions(+), 49 deletions(-) diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index 5647bee380..4ead48a7cf 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -64,32 +64,33 @@ def get_image_embeddings( ): all_image_embeddings = [] - if isinstance(images, DataLoader): - for batch_images in tqdm(images): - img_inputs = self.processor( - [self.img_prompt] * len(batch_images), - batch_images, - return_tensors="pt", - padding=True, - ).to("cuda") - image_outputs = self.model( - **img_inputs, output_hidden_states=True, return_dict=True - ).hidden_states[-1][:, -1, :] - all_image_embeddings.append(image_outputs.cpu()) with torch.no_grad(): - for i in tqdm(range(0, len(images), batch_size)): - batch_images = images[i : i + batch_size] - img_inputs = self.processor( - [self.img_prompt] * len(batch_images), - batch_images, - return_tensors="pt", - padding=True, - ).to("cuda") - image_outputs = self.model( - **img_inputs, output_hidden_states=True, return_dict=True - ).hidden_states[-1][:, -1, :] - all_image_embeddings.append(image_outputs.cpu()) - return torch.cat(all_image_embeddings, dim=0) + if isinstance(images, DataLoader): + for batch_images in tqdm(images): + img_inputs = self.processor( + [self.img_prompt] * len(batch_images), + batch_images, + return_tensors="pt", + padding=True, + ).to("cuda") + image_outputs = self.model( + **img_inputs, output_hidden_states=True, return_dict=True + ).hidden_states[-1][:, -1, :] + all_image_embeddings.append(image_outputs.cpu()) + else: + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + img_inputs = self.processor( + [self.img_prompt] * len(batch_images), + batch_images, + return_tensors="pt", + padding=True, + ).to("cuda") + image_outputs = self.model( + **img_inputs, output_hidden_states=True, return_dict=True + ).hidden_states[-1][:, -1, :] + all_image_embeddings.append(image_outputs.cpu()) + return torch.cat(all_image_embeddings, dim=0) def calculate_probs(self, text_embeddings, image_embeddings): text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) @@ -112,8 +113,8 @@ def get_fused_embeddings( all_fused_embeddings = [] if texts is not None and images is not None: - if isinstance(images, DataLoader): - with torch.no_grad(): + with torch.no_grad(): + if isinstance(images, DataLoader): for index, batch_images in enumerate(tqdm(images)): batch_texts = texts[ index * batch_size : (index + 1) * batch_size @@ -128,12 +129,11 @@ def get_fused_embeddings( **inputs, output_hidden_states=True, return_dict=True ).hidden_states[-1][:, -1, :] all_fused_embeddings.append(outputs.cpu()) - else: - if len(texts) != len(images): - raise ValueError( - "The number of texts and images must have the same length" - ) - with torch.no_grad(): + else: + if len(texts) != len(images): + raise ValueError( + "The number of texts and images must have the same length" + ) for i in tqdm(range(0, len(images), batch_size)): batch_texts = texts[i : i + batch_size] batch_images = images[i : i + batch_size] @@ -148,7 +148,6 @@ def get_fused_embeddings( ).hidden_states[-1][:, -1, :] all_fused_embeddings.append(outputs.cpu()) return torch.cat(all_fused_embeddings, dim=0) - elif texts is not None: text_embeddings = self.get_text_embeddings(texts, batch_size) return text_embeddings diff --git a/mteb/models/vlm2vec_models.py b/mteb/models/vlm2vec_models.py index 321cba24a4..d75236a93a 100644 --- a/mteb/models/vlm2vec_models.py +++ b/mteb/models/vlm2vec_models.py @@ -254,22 +254,18 @@ def get_fused_embeddings( all_fused_embeddings = [] if isinstance(images, DataLoader): import torchvision.transforms.functional as F - - for batch in images: - for b in batch: - text = next(texts) - inputs = self.processor( - f"<|image_1|> Represent the given image with the following question: {text}", - [F.to_pil_image(b.to("cpu"))], - ) - inputs = { - key: value.to(self.device) for key, value in inputs.items() - } - outputs = self.encode_input(inputs) - all_fused_embeddings.append(outputs.cpu()) - + with torch.no_grad(): + for batch in images: + for b in batch: + text = next(texts) + inputs = self.processor( + f"<|image_1|> Represent the given image with the following question: {text}", + [F.to_pil_image(b.to("cpu"))], + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + outputs = self.encode_input(inputs) + all_fused_embeddings.append(outputs.cpu()) fused_embeddings = torch.cat(all_fused_embeddings, dim=0) - return fused_embeddings elif text_embeddings is not None: return text_embeddings From 1ef93e472c02bdcd740c5d9348843333049d1132 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Tue, 5 Nov 2024 05:57:41 +0200 Subject: [PATCH 094/154] [mieb] Fix vlm2vec (#1380) * fix vlm2vec return dtype * make lint --- mteb/models/vlm2vec_models.py | 57 +++--- .../BLINKIT2IRetrieval.json | 186 ++++++++++++++++++ 2 files changed, 212 insertions(+), 31 deletions(-) create mode 100644 results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json diff --git a/mteb/models/vlm2vec_models.py b/mteb/models/vlm2vec_models.py index d75236a93a..e9c8653f74 100644 --- a/mteb/models/vlm2vec_models.py +++ b/mteb/models/vlm2vec_models.py @@ -149,7 +149,7 @@ def get_image_embeddings( } image_outputs = self.encode_input(inputs) - all_image_embeddings.append(image_outputs.cpu()) + all_image_embeddings.append(image_outputs.cpu().to(torch.float32)) else: with torch.no_grad(): @@ -186,7 +186,7 @@ def get_image_embeddings( } image_outputs = self.encode_input(inputs) - all_image_embeddings.append(image_outputs.cpu()) + all_image_embeddings.append(image_outputs.cpu().to(torch.float32)) all_image_embeddings = torch.cat(all_image_embeddings, dim=0) return all_image_embeddings @@ -221,7 +221,7 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): } text_outputs = self.encode_input(inputs) - all_text_embeddings.append(text_outputs.cpu()) + all_text_embeddings.append(text_outputs.cpu().to(torch.float32)) all_text_embeddings = torch.cat(all_text_embeddings, dim=0) return all_text_embeddings @@ -239,39 +239,34 @@ def get_fused_embeddings( text_embeddings = None image_embeddings = None - if texts is not None: + if texts is not None and images is None: text_embeddings = self.get_text_embeddings(texts, batch_size) + return text_embeddings - if images is not None: + if images is not None and texts is None: image_embeddings = self.get_image_embeddings(images, batch_size) - - if text_embeddings is not None and image_embeddings is not None: - if len(text_embeddings) != len(image_embeddings): - raise ValueError( - "The number of texts and images must have the same length" - ) - texts = iter(texts) - all_fused_embeddings = [] - if isinstance(images, DataLoader): - import torchvision.transforms.functional as F - with torch.no_grad(): - for batch in images: - for b in batch: - text = next(texts) - inputs = self.processor( - f"<|image_1|> Represent the given image with the following question: {text}", - [F.to_pil_image(b.to("cpu"))], - ) - inputs = {k: v.to(self.device) for k, v in inputs.items()} - outputs = self.encode_input(inputs) - all_fused_embeddings.append(outputs.cpu()) - fused_embeddings = torch.cat(all_fused_embeddings, dim=0) - return fused_embeddings - elif text_embeddings is not None: - return text_embeddings - elif image_embeddings is not None: return image_embeddings + # text_embeddings is not None and image_embeddings is not None + texts = iter(texts) + all_fused_embeddings = [] + if isinstance(images, DataLoader): + import torchvision.transforms.functional as F + + with torch.no_grad(): + for batch in images: + for b in batch: + text = next(texts) + inputs = self.processor( + f"<|image_1|> Represent the given image with the following question: {text}", + [F.to_pil_image(b.to("cpu"))], + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + outputs = self.encode_input(inputs) + all_fused_embeddings.append(outputs.cpu().to(torch.float32)) + fused_embeddings = torch.cat(all_fused_embeddings, dim=0) + return fused_embeddings + vlm2vec_lora = ModelMeta( loader=partial( diff --git a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json new file mode 100644 index 0000000000..9077b8c127 --- /dev/null +++ b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26", + "evaluation_time": 224.3045289516449, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.33582, + "cv_recall_at_10": 0.63184, + "cv_recall_at_100": 0.90547, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.72637, + "cv_recall_at_3": 0.52736, + "cv_recall_at_5": 0.57711, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.48677, + "map_at_1": 0.33582, + "map_at_10": 0.43983, + "map_at_100": 0.45072, + "map_at_1000": 0.45136, + "map_at_20": 0.44629, + "map_at_3": 0.4204, + "map_at_5": 0.43209, + "mrr_at_1": 0.3358208955223881, + "mrr_at_10": 0.43982666034904844, + "mrr_at_100": 0.45071603631711843, + "mrr_at_1000": 0.4513620025552411, + "mrr_at_20": 0.44629389891710264, + "mrr_at_3": 0.42039800995024873, + "mrr_at_5": 0.43208955223880596, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2698006811472229, + "nauc_cv_recall_at_100_max": -0.8185369176412147, + "nauc_cv_recall_at_100_std": -1.0075573331300645, + "nauc_cv_recall_at_10_diff1": 0.1788243841076317, + "nauc_cv_recall_at_10_max": -0.612746605676537, + "nauc_cv_recall_at_10_std": -0.9530034638964193, + "nauc_cv_recall_at_1_diff1": 0.2822466928213175, + "nauc_cv_recall_at_1_max": -0.33285460282946566, + "nauc_cv_recall_at_1_std": -0.5640630200327668, + "nauc_cv_recall_at_20_diff1": 0.12960665874064498, + "nauc_cv_recall_at_20_max": -0.8021930752909697, + "nauc_cv_recall_at_20_std": -1.071727841531356, + "nauc_cv_recall_at_3_diff1": 0.2019990705156492, + "nauc_cv_recall_at_3_max": -0.4459723288652778, + "nauc_cv_recall_at_3_std": -0.7870326458461854, + "nauc_cv_recall_at_5_diff1": 0.23846869773197804, + "nauc_cv_recall_at_5_max": -0.47064146280491637, + "nauc_cv_recall_at_5_std": -0.8505490027952863, + "nauc_map_at_1000_diff1": 0.2409455106402186, + "nauc_map_at_1000_max": -0.407384704974283, + "nauc_map_at_1000_std": -0.6813392812409054, + "nauc_map_at_100_diff1": 0.24104039617379056, + "nauc_map_at_100_max": -0.408250752001871, + "nauc_map_at_100_std": -0.682039826842636, + "nauc_map_at_10_diff1": 0.2411578991609476, + "nauc_map_at_10_max": -0.4044975322245294, + "nauc_map_at_10_std": -0.6840284218652672, + "nauc_map_at_1_diff1": 0.2822466928213175, + "nauc_map_at_1_max": -0.33285460282946566, + "nauc_map_at_1_std": -0.5640630200327668, + "nauc_map_at_20_diff1": 0.2388129109858802, + "nauc_map_at_20_max": -0.4111868495020421, + "nauc_map_at_20_std": -0.6861574124125431, + "nauc_map_at_3_diff1": 0.2415896795362189, + "nauc_map_at_3_max": -0.3816620658278678, + "nauc_map_at_3_std": -0.6597288106948174, + "nauc_map_at_5_diff1": 0.2490237529189825, + "nauc_map_at_5_max": -0.3874920448893905, + "nauc_map_at_5_std": -0.6729363831482362, + "nauc_mrr_at_1000_diff1": 0.2409455106402186, + "nauc_mrr_at_1000_max": -0.407384704974283, + "nauc_mrr_at_1000_std": -0.6813392812409054, + "nauc_mrr_at_100_diff1": 0.24104039617379056, + "nauc_mrr_at_100_max": -0.408250752001871, + "nauc_mrr_at_100_std": -0.682039826842636, + "nauc_mrr_at_10_diff1": 0.2411578991609476, + "nauc_mrr_at_10_max": -0.4044975322245294, + "nauc_mrr_at_10_std": -0.6840284218652672, + "nauc_mrr_at_1_diff1": 0.2822466928213175, + "nauc_mrr_at_1_max": -0.33285460282946566, + "nauc_mrr_at_1_std": -0.5640630200327668, + "nauc_mrr_at_20_diff1": 0.2388129109858802, + "nauc_mrr_at_20_max": -0.4111868495020421, + "nauc_mrr_at_20_std": -0.6861574124125431, + "nauc_mrr_at_3_diff1": 0.2415896795362189, + "nauc_mrr_at_3_max": -0.3816620658278678, + "nauc_mrr_at_3_std": -0.6597288106948174, + "nauc_mrr_at_5_diff1": 0.2490237529189825, + "nauc_mrr_at_5_max": -0.3874920448893905, + "nauc_mrr_at_5_std": -0.6729363831482362, + "nauc_ndcg_at_1000_diff1": 0.2344092745371185, + "nauc_ndcg_at_1000_max": -0.4328446999637412, + "nauc_ndcg_at_1000_std": -0.709474885782739, + "nauc_ndcg_at_100_diff1": 0.23614140799135372, + "nauc_ndcg_at_100_max": -0.45059659752764697, + "nauc_ndcg_at_100_std": -0.7232249956899242, + "nauc_ndcg_at_10_diff1": 0.22732780326872332, + "nauc_ndcg_at_10_max": -0.44870094654610376, + "nauc_ndcg_at_10_std": -0.7429856899210265, + "nauc_ndcg_at_1_diff1": 0.2822466928213175, + "nauc_ndcg_at_1_max": -0.33285460282946566, + "nauc_ndcg_at_1_std": -0.5640630200327668, + "nauc_ndcg_at_20_diff1": 0.21923921046650915, + "nauc_ndcg_at_20_max": -0.4772077987509219, + "nauc_ndcg_at_20_std": -0.7553659757754395, + "nauc_ndcg_at_3_diff1": 0.23134535418875965, + "nauc_ndcg_at_3_max": -0.39788809593745006, + "nauc_ndcg_at_3_std": -0.691825805701685, + "nauc_ndcg_at_5_diff1": 0.24552033909645363, + "nauc_ndcg_at_5_max": -0.4076854503791393, + "nauc_ndcg_at_5_std": -0.7156597141210952, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.2698006811472233, + "nauc_precision_at_100_max": -0.818536917641218, + "nauc_precision_at_100_std": -1.007557333130069, + "nauc_precision_at_10_diff1": 0.17882438410763166, + "nauc_precision_at_10_max": -0.6127466056765365, + "nauc_precision_at_10_std": -0.9530034638964187, + "nauc_precision_at_1_diff1": 0.2822466928213175, + "nauc_precision_at_1_max": -0.33285460282946566, + "nauc_precision_at_1_std": -0.5640630200327668, + "nauc_precision_at_20_diff1": 0.12960665874064572, + "nauc_precision_at_20_max": -0.8021930752909691, + "nauc_precision_at_20_std": -1.0717278415313547, + "nauc_precision_at_3_diff1": 0.20199907051564917, + "nauc_precision_at_3_max": -0.44597232886527743, + "nauc_precision_at_3_std": -0.7870326458461852, + "nauc_precision_at_5_diff1": 0.238468697731978, + "nauc_precision_at_5_max": -0.47064146280491626, + "nauc_precision_at_5_std": -0.8505490027952862, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2698006811472229, + "nauc_recall_at_100_max": -0.8185369176412147, + "nauc_recall_at_100_std": -1.0075573331300645, + "nauc_recall_at_10_diff1": 0.1788243841076317, + "nauc_recall_at_10_max": -0.612746605676537, + "nauc_recall_at_10_std": -0.9530034638964193, + "nauc_recall_at_1_diff1": 0.2822466928213175, + "nauc_recall_at_1_max": -0.33285460282946566, + "nauc_recall_at_1_std": -0.5640630200327668, + "nauc_recall_at_20_diff1": 0.12960665874064498, + "nauc_recall_at_20_max": -0.8021930752909697, + "nauc_recall_at_20_std": -1.071727841531356, + "nauc_recall_at_3_diff1": 0.2019990705156492, + "nauc_recall_at_3_max": -0.4459723288652778, + "nauc_recall_at_3_std": -0.7870326458461854, + "nauc_recall_at_5_diff1": 0.23846869773197804, + "nauc_recall_at_5_max": -0.47064146280491637, + "nauc_recall_at_5_std": -0.8505490027952863, + "ndcg_at_1": 0.33582, + "ndcg_at_10": 0.48677, + "ndcg_at_100": 0.54341, + "ndcg_at_1000": 0.55646, + "ndcg_at_20": 0.51055, + "ndcg_at_3": 0.44788, + "ndcg_at_5": 0.46865, + "precision_at_1": 0.33582, + "precision_at_10": 0.06318, + "precision_at_100": 0.00905, + "precision_at_1000": 0.001, + "precision_at_20": 0.03632, + "precision_at_3": 0.17579, + "precision_at_5": 0.11542, + "recall_at_1": 0.33582, + "recall_at_10": 0.63184, + "recall_at_100": 0.90547, + "recall_at_1000": 1.0, + "recall_at_20": 0.72637, + "recall_at_3": 0.52736, + "recall_at_5": 0.57711 + } + ] + }, + "task_name": "BLINKIT2IRetrieval" +} \ No newline at end of file From 34094ea1dbe962cfb7121453b30e184bc692f236 Mon Sep 17 00:00:00 2001 From: Jamie-Stirling <36764530+Jamie-Stirling@users.noreply.github.com> Date: Tue, 5 Nov 2024 06:39:30 +0000 Subject: [PATCH 095/154] [mieb] Remove null entries from corpus of ROxford, RParis (#1371) * wip: start adding BLIP models * add other blip variants * wip: add blip2_models.py * make lint * wip: implement blip2 wrapper * feat: add blip2 models, still mismatched names * fix: remove projections from image and text embeddings * make lint * wip: add coco BLIP2 * fix: BLIP2 better zero-shot classification without text_proj and vision_proj * tidy blip2 * add imagenet-dog-15 dataset * tidy and lint * remove unused import * add cluster_accuracy, ari and nmi to Image.ClusteringEvaluator * add imagenet-10 clustering task * add SOPI2IRetrieval * add results forclip on ImageNet10Clustering and ImageNetDog15Clustering * add SOPI2IRetrieval results for clip 32 * add results for clip vit 32/SOPI2IRetrieval * resolve conflict * add RP2kI2IRetrieval dataset * add RP2kI2IRetrieval results with clip-vit-base-patch32 * update image retrieval __init__.py * fix ImageTextPair dataloading for large datasets; more compositionality evaluation datasets * add RP2kI2IRetrieval and METI2IRetrieval * add METI2IRetreival * add SOP results * make lign * new revision for METI2IRetrieval * make lint * reset corpus chunk size * remove wrong classification import * add Flickr30k T2I and I2T * add Flickr30k T2I retriebal * reduced-size MET revision * fix: add Flickr30k T2I * make lint * add two landmark datasets and results * add Sketchy i2i retrieval * add task metadata * add BLINKIT2IRetrieval dataset * add BLINKIT2TRetrieval * add ImageCoDeT2IRetrieval * make lint * add vizwiz retrieval and results * fix vizwiz duplicate texts * add new vizwiz results * add VQA2 results * add GLD v2 I2T retrieval * add gld v2 i2i retrieval * make lint * add AbsTaskAny2AnyMultiChoice * make lint * remove GLDv2I2IRetrieval * exclude AbsTaskAny2AnyMultiChoice from test_load_data * fix e5v&vista * remove duplicate corpus entries from BLINKIT2TRetreival dataset * task type fix for running tasks * update BLINKIT2T metadata * fix wrong meta * run mieb script * split ROxford, RParis into easy, medium and hard * make lint * add BLINK as multi choice tasks * fix: license metadata in wrong format * remove null examples from corpus of ROxford and RParis --------- Co-authored-by: gowitheflow-1998 --- .../eng/ROxfordI2IRetrieval.py | 12 +- .../eng/RParisI2IRetrieval.py | 10 +- .../ROxfordEasyI2IRetrieval.json | 334 ++++++++--------- .../ROxfordHardI2IRetrieval.json | 342 +++++++++--------- .../ROxfordMediumI2IRetrieval.json | 334 ++++++++--------- .../RParisEasyI2IRetrieval.json | 294 +++++++-------- .../RParisHardI2IRetrieval.json | 334 ++++++++--------- .../RParisMediumI2IRetrieval.json | 294 +++++++-------- .../model_meta.json | 2 +- 9 files changed, 978 insertions(+), 978 deletions(-) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py index e338eec2d1..dbec8e6ae7 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py @@ -11,7 +11,7 @@ class ROxfordEasyI2IRetrieval(AbsTaskAny2AnyRetrieval): reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", dataset={ "path": "JamieSJS/r-oxford-easy", - "revision": "3f018eb7ad32218a5a4ebd704493e0834a265cf5", + "revision": "b71b5f67a93aa63761b79a67bcf28bd2ae590902", }, type="Any2AnyRetrieval", category="i2i", @@ -40,7 +40,7 @@ class ROxfordEasyI2IRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 4993, + "num_documents": 516, "num_queries": 70, "average_relevant_docs_per_query": 43.3, } @@ -57,7 +57,7 @@ class ROxfordMediumI2IRetrieval(AbsTaskAny2AnyRetrieval): reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", dataset={ "path": "JamieSJS/r-oxford-medium", - "revision": "3bd28e9c45e15f299117c634799f7035c4de2d31", + "revision": "1dfb86730ee4b3f49b441f4896d473c83eb5ff0d", }, type="Any2AnyRetrieval", category="i2i", @@ -86,7 +86,7 @@ class ROxfordMediumI2IRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 4993, + "num_documents": 788, "num_queries": 70, "average_relevant_docs_per_query": 78.9, } @@ -103,7 +103,7 @@ class ROxfordHardI2IRetrieval(AbsTaskAny2AnyRetrieval): reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", dataset={ "path": "JamieSJS/r-oxford-hard", - "revision": "f20b30211b7ba3fc64a02bd83998fe75f3023719", + "revision": "f71ab9d4aabcda93d55a7e65edfb3a34767d89e6", }, type="Any2AnyRetrieval", category="i2i", @@ -132,7 +132,7 @@ class ROxfordHardI2IRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 4993, + "num_documents": 685, "num_queries": 70, "average_relevant_docs_per_query": 35.7, } diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py index f29a9849ef..8c2f6344fb 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py @@ -11,7 +11,7 @@ class RParisEasyI2IRetrieval(AbsTaskAny2AnyRetrieval): reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", dataset={ "path": "JamieSJS/r-paris-easy", - "revision": "a7293da8a341de665ee4dcb2f209281df342d80b", + "revision": "7d821ddebcb30ad343133e3a81e23347ac2a08a8", }, type="Any2AnyRetrieval", category="i2i", @@ -40,7 +40,7 @@ class RParisEasyI2IRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 6322, + "num_documents": 1470, "num_queries": 70, "average_relevant_docs_per_query": 98.2, } @@ -57,7 +57,7 @@ class RParisMediumI2IRetrieval(AbsTaskAny2AnyRetrieval): reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", dataset={ "path": "JamieSJS/r-paris-medium", - "revision": "900267b49003a086979e8d52f6942624236bfc34", + "revision": "3d959815e102785efd628170281f1e65561b03d2", }, type="Any2AnyRetrieval", category="i2i", @@ -86,7 +86,7 @@ class RParisMediumI2IRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 6322, + "num_documents": 2651, "num_queries": 70, "average_relevant_docs_per_query": 147.9, } @@ -103,7 +103,7 @@ class RParisHardI2IRetrieval(AbsTaskAny2AnyRetrieval): reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", dataset={ "path": "JamieSJS/r-paris-hard", - "revision": "fd121b6592fe946616fa85116703b94a4c61fd63", + "revision": "d3e0adf4e942446c04427511ccce281c86861248", }, type="Any2AnyRetrieval", category="i2i", diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json index 319e3d389b..738473a5e4 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json @@ -1,184 +1,184 @@ { - "dataset_revision": "3f018eb7ad32218a5a4ebd704493e0834a265cf5", - "evaluation_time": 17.977893829345703, + "dataset_revision": "b71b5f67a93aa63761b79a67bcf28bd2ae590902", + "evaluation_time": 18.838356256484985, "kg_co2_emissions": null, "mteb_version": "1.12.90", "scores": { "test": [ { - "cv_recall_at_1": 0.73529, - "cv_recall_at_10": 0.88235, - "cv_recall_at_100": 0.95588, + "cv_recall_at_1": 0.88235, + "cv_recall_at_10": 0.97059, + "cv_recall_at_100": 1.0, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.89706, - "cv_recall_at_3": 0.79412, - "cv_recall_at_5": 0.82353, + "cv_recall_at_20": 0.98529, + "cv_recall_at_3": 0.91176, + "cv_recall_at_5": 0.94118, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.73529, - "map_at_1": 0.12315, - "map_at_10": 0.25156, - "map_at_100": 0.39829, - "map_at_1000": 0.46268, - "map_at_20": 0.30547, - "map_at_3": 0.16397, - "map_at_5": 0.20244, - "mrr_at_1": 0.7352941176470589, - "mrr_at_10": 0.7796977124183007, - "mrr_at_100": 0.782038983404294, - "mrr_at_1000": 0.7822887502278202, - "mrr_at_20": 0.7806168300653595, - "mrr_at_3": 0.7647058823529411, - "mrr_at_5": 0.7713235294117647, + "main_score": 0.88235, + "map_at_1": 0.18698, + "map_at_10": 0.44691, + "map_at_100": 0.66159, + "map_at_1000": 0.73348, + "map_at_20": 0.53479, + "map_at_3": 0.30295, + "map_at_5": 0.36197, + "mrr_at_1": 0.8823529411764706, + "mrr_at_10": 0.908700980392157, + "mrr_at_100": 0.9098240302312877, + "mrr_at_1000": 0.9098240302312877, + "mrr_at_20": 0.9095660322952712, + "mrr_at_3": 0.8970588235294118, + "mrr_at_5": 0.9044117647058824, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.43205924601836393, - "nauc_cv_recall_at_100_max": 0.46045430774676144, - "nauc_cv_recall_at_100_std": 0.092928936942703, - "nauc_cv_recall_at_10_diff1": 0.5238482274120124, - "nauc_cv_recall_at_10_max": 0.5447795999610322, - "nauc_cv_recall_at_10_std": -0.06773626862669024, - "nauc_cv_recall_at_1_diff1": 0.12227878218725667, - "nauc_cv_recall_at_1_max": 0.6674736854281768, - "nauc_cv_recall_at_1_std": 0.14078948712021733, - "nauc_cv_recall_at_20_diff1": 0.5375955313486179, - "nauc_cv_recall_at_20_max": 0.46954147068219054, - "nauc_cv_recall_at_20_std": -0.09380403989753433, - "nauc_cv_recall_at_3_diff1": 0.032732730804488214, - "nauc_cv_recall_at_3_max": 0.6199038499058639, - "nauc_cv_recall_at_3_std": 0.1529241887407745, - "nauc_cv_recall_at_5_diff1": 0.05658962411663845, - "nauc_cv_recall_at_5_max": 0.5877996333799903, - "nauc_cv_recall_at_5_std": 0.07730142560650717, - "nauc_map_at_1000_diff1": -0.11043090289526158, - "nauc_map_at_1000_max": 0.3448001394797035, - "nauc_map_at_1000_std": 0.27617723792125437, - "nauc_map_at_100_diff1": -0.01003127759400255, - "nauc_map_at_100_max": 0.1251706076455483, - "nauc_map_at_100_std": 0.03701229955981713, - "nauc_map_at_10_diff1": 0.2139884419568207, - "nauc_map_at_10_max": -0.26177042855989124, - "nauc_map_at_10_std": -0.26537536804238954, - "nauc_map_at_1_diff1": 0.21399681007835022, - "nauc_map_at_1_max": -0.3465990811601411, - "nauc_map_at_1_std": -0.29206561782006085, - "nauc_map_at_20_diff1": 0.1442752699133972, - "nauc_map_at_20_max": -0.15930665035054287, - "nauc_map_at_20_std": -0.21656927646466867, - "nauc_map_at_3_diff1": 0.23213875592486613, - "nauc_map_at_3_max": -0.3123546266529488, - "nauc_map_at_3_std": -0.28815041773010014, - "nauc_map_at_5_diff1": 0.2555448109113392, - "nauc_map_at_5_max": -0.29279690087015, - "nauc_map_at_5_std": -0.2774467752271305, - "nauc_mrr_at_1000_diff1": 0.12342303172342535, - "nauc_mrr_at_1000_max": 0.6362159972261828, - "nauc_mrr_at_1000_std": 0.12097023869090642, - "nauc_mrr_at_100_diff1": 0.12414954187540776, - "nauc_mrr_at_100_max": 0.6357802506977609, - "nauc_mrr_at_100_std": 0.12039621076657632, - "nauc_mrr_at_10_diff1": 0.12909230332110294, - "nauc_mrr_at_10_max": 0.6368549836172475, - "nauc_mrr_at_10_std": 0.11895636170173347, - "nauc_mrr_at_1_diff1": 0.12227878218725667, - "nauc_mrr_at_1_max": 0.6674736854281768, - "nauc_mrr_at_1_std": 0.14078948712021733, - "nauc_mrr_at_20_diff1": 0.1279569336937743, - "nauc_mrr_at_20_max": 0.6347997662966113, - "nauc_mrr_at_20_std": 0.11886311296272292, - "nauc_mrr_at_3_diff1": 0.0819493669888214, - "nauc_mrr_at_3_max": 0.6460493610936507, - "nauc_mrr_at_3_std": 0.14625466858168346, - "nauc_mrr_at_5_diff1": 0.08926776001340989, - "nauc_mrr_at_5_max": 0.6409743370271209, - "nauc_mrr_at_5_std": 0.13086158199344464, - "nauc_ndcg_at_1000_diff1": 0.0019895005695910535, - "nauc_ndcg_at_1000_max": 0.45479785106891296, - "nauc_ndcg_at_1000_std": 0.24337684988960884, - "nauc_ndcg_at_100_diff1": -0.09459953169491665, - "nauc_ndcg_at_100_max": 0.3603949417192614, - "nauc_ndcg_at_100_std": 0.25977974826865124, - "nauc_ndcg_at_10_diff1": -0.13743096053058393, - "nauc_ndcg_at_10_max": 0.5086455404107587, - "nauc_ndcg_at_10_std": 0.2942246989065215, - "nauc_ndcg_at_1_diff1": 0.12227878218725667, - "nauc_ndcg_at_1_max": 0.6674736854281768, - "nauc_ndcg_at_1_std": 0.14078948712021733, - "nauc_ndcg_at_20_diff1": -0.12980804316788766, - "nauc_ndcg_at_20_max": 0.4455786946909776, - "nauc_ndcg_at_20_std": 0.31146671038942925, - "nauc_ndcg_at_3_diff1": -0.12622874187697183, - "nauc_ndcg_at_3_max": 0.6321053376737653, - "nauc_ndcg_at_3_std": 0.2970450461706155, - "nauc_ndcg_at_5_diff1": -0.1167444253887786, - "nauc_ndcg_at_5_max": 0.5830509606014207, - "nauc_ndcg_at_5_std": 0.3092017793979728, - "nauc_precision_at_1000_diff1": -0.20870904079910094, - "nauc_precision_at_1000_max": 0.553838513916762, - "nauc_precision_at_1000_std": 0.5530198613029336, - "nauc_precision_at_100_diff1": -0.35505246743099134, - "nauc_precision_at_100_max": 0.7356593773083951, - "nauc_precision_at_100_std": 0.6881226249275548, - "nauc_precision_at_10_diff1": -0.32094129231704527, - "nauc_precision_at_10_max": 0.6957334058235605, - "nauc_precision_at_10_std": 0.4986922302670923, - "nauc_precision_at_1_diff1": 0.12227878218725667, - "nauc_precision_at_1_max": 0.6674736854281768, - "nauc_precision_at_1_std": 0.14078948712021733, - "nauc_precision_at_20_diff1": -0.348557232862726, - "nauc_precision_at_20_max": 0.7349093491157456, - "nauc_precision_at_20_std": 0.5403176104598045, - "nauc_precision_at_3_diff1": -0.19598105078336095, - "nauc_precision_at_3_max": 0.6900728385079788, - "nauc_precision_at_3_std": 0.38284792684568514, - "nauc_precision_at_5_diff1": -0.24325694519449578, - "nauc_precision_at_5_max": 0.6894112316594818, - "nauc_precision_at_5_std": 0.43885055756284386, - "nauc_recall_at_1000_diff1": 0.1247114496786093, - "nauc_recall_at_1000_max": 0.1972100012754324, - "nauc_recall_at_1000_std": 0.45358985850344147, - "nauc_recall_at_100_diff1": -0.007219441799189107, - "nauc_recall_at_100_max": -0.2726450323976995, - "nauc_recall_at_100_std": -0.09774220148121496, - "nauc_recall_at_10_diff1": 0.2493200864593003, - "nauc_recall_at_10_max": -0.35624261159143006, - "nauc_recall_at_10_std": -0.3178591018768707, - "nauc_recall_at_1_diff1": 0.21399681007835022, - "nauc_recall_at_1_max": -0.3465990811601411, - "nauc_recall_at_1_std": -0.29206561782006085, - "nauc_recall_at_20_diff1": 0.17625014384578389, - "nauc_recall_at_20_max": -0.3372766383490987, - "nauc_recall_at_20_std": -0.30264233629183895, - "nauc_recall_at_3_diff1": 0.23996176062420918, - "nauc_recall_at_3_max": -0.3204556669767577, - "nauc_recall_at_3_std": -0.292832765895384, - "nauc_recall_at_5_diff1": 0.28900424501627087, - "nauc_recall_at_5_max": -0.31760291427784704, - "nauc_recall_at_5_std": -0.2855625586155051, - "ndcg_at_1": 0.73529, - "ndcg_at_10": 0.60178, - "ndcg_at_100": 0.61347, - "ndcg_at_1000": 0.69898, - "ndcg_at_20": 0.58929, - "ndcg_at_3": 0.62141, - "ndcg_at_5": 0.61571, - "precision_at_1": 0.73529, - "precision_at_10": 0.45, - "precision_at_100": 0.19691, - "precision_at_1000": 0.04228, - "precision_at_20": 0.37647, - "precision_at_3": 0.54412, - "precision_at_5": 0.51176, - "recall_at_1": 0.12315, - "recall_at_10": 0.32137, - "recall_at_100": 0.68206, - "recall_at_1000": 0.93425, - "recall_at_20": 0.43253, - "recall_at_3": 0.16748, - "recall_at_5": 0.22328 + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": 0.45647666859182273, + "nauc_cv_recall_at_10_max": 0.7947783849423156, + "nauc_cv_recall_at_10_std": 0.07401479495347633, + "nauc_cv_recall_at_1_diff1": 0.15992222589126076, + "nauc_cv_recall_at_1_max": 0.7066438838407989, + "nauc_cv_recall_at_1_std": 0.34516803717873823, + "nauc_cv_recall_at_20_diff1": 0.5540376442015769, + "nauc_cv_recall_at_20_max": 0.8688524590163946, + "nauc_cv_recall_at_20_std": 0.7207043108682435, + "nauc_cv_recall_at_3_diff1": 0.02979739943425187, + "nauc_cv_recall_at_3_max": 0.6677398031346646, + "nauc_cv_recall_at_3_std": 0.19367311898769965, + "nauc_cv_recall_at_5_diff1": 0.33506965405558836, + "nauc_cv_recall_at_5_max": 0.8646023072252566, + "nauc_cv_recall_at_5_std": 0.3439494354763536, + "nauc_map_at_1000_diff1": -0.0748418292398556, + "nauc_map_at_1000_max": 0.1642828370088438, + "nauc_map_at_1000_std": 0.05154074134715094, + "nauc_map_at_100_diff1": 0.18300297522101533, + "nauc_map_at_100_max": -0.09630611530306471, + "nauc_map_at_100_std": -0.27588270039938373, + "nauc_map_at_10_diff1": 0.3759801235280281, + "nauc_map_at_10_max": -0.4332228813048274, + "nauc_map_at_10_std": -0.513562318981569, + "nauc_map_at_1_diff1": 0.3294708101134731, + "nauc_map_at_1_max": -0.42544232201495896, + "nauc_map_at_1_std": -0.5038005227813385, + "nauc_map_at_20_diff1": 0.32605775541629023, + "nauc_map_at_20_max": -0.3650620149830886, + "nauc_map_at_20_std": -0.48968947212775454, + "nauc_map_at_3_diff1": 0.3734075945845939, + "nauc_map_at_3_max": -0.46014380098464513, + "nauc_map_at_3_std": -0.5246559323671462, + "nauc_map_at_5_diff1": 0.3703663478894585, + "nauc_map_at_5_max": -0.45530691941081636, + "nauc_map_at_5_std": -0.5435081911059385, + "nauc_mrr_at_1000_diff1": 0.1563665959264749, + "nauc_mrr_at_1000_max": 0.7110551871683163, + "nauc_mrr_at_1000_std": 0.3006510045832456, + "nauc_mrr_at_100_diff1": 0.1563665959264749, + "nauc_mrr_at_100_max": 0.7110551871683163, + "nauc_mrr_at_100_std": 0.3006510045832456, + "nauc_mrr_at_10_diff1": 0.15944197884390138, + "nauc_mrr_at_10_max": 0.7115982604675308, + "nauc_mrr_at_10_std": 0.2934876531933919, + "nauc_mrr_at_1_diff1": 0.15992222589126076, + "nauc_mrr_at_1_max": 0.7066438838407989, + "nauc_mrr_at_1_std": 0.34516803717873823, + "nauc_mrr_at_20_diff1": 0.15751333465270692, + "nauc_mrr_at_20_max": 0.7115102171316658, + "nauc_mrr_at_20_std": 0.3018622856043013, + "nauc_mrr_at_3_diff1": 0.10362708820418606, + "nauc_mrr_at_3_max": 0.6898130404147407, + "nauc_mrr_at_3_std": 0.2796276825003452, + "nauc_mrr_at_5_diff1": 0.15680775836822267, + "nauc_mrr_at_5_max": 0.7221234773231954, + "nauc_mrr_at_5_std": 0.30966199967981883, + "nauc_ndcg_at_1000_diff1": -0.08504950934995431, + "nauc_ndcg_at_1000_max": 0.3562828960467138, + "nauc_ndcg_at_1000_std": 0.12738028206826132, + "nauc_ndcg_at_100_diff1": 0.042015479832883486, + "nauc_ndcg_at_100_max": 0.267061759431482, + "nauc_ndcg_at_100_std": 0.07290512182104139, + "nauc_ndcg_at_10_diff1": -0.12122120870754685, + "nauc_ndcg_at_10_max": 0.48196302566814775, + "nauc_ndcg_at_10_std": 0.20073536709572107, + "nauc_ndcg_at_1_diff1": 0.15992222589126076, + "nauc_ndcg_at_1_max": 0.7066438838407989, + "nauc_ndcg_at_1_std": 0.34516803717873823, + "nauc_ndcg_at_20_diff1": -0.10661851294694442, + "nauc_ndcg_at_20_max": 0.33981330679892496, + "nauc_ndcg_at_20_std": 0.20675699745130993, + "nauc_ndcg_at_3_diff1": -0.06822895395133349, + "nauc_ndcg_at_3_max": 0.5801948973031311, + "nauc_ndcg_at_3_std": 0.2644877411998497, + "nauc_ndcg_at_5_diff1": -0.08300096492046799, + "nauc_ndcg_at_5_max": 0.5620424257737597, + "nauc_ndcg_at_5_std": 0.25135047052009385, + "nauc_precision_at_1000_diff1": -0.5039877076914758, + "nauc_precision_at_1000_max": 0.5273281460558504, + "nauc_precision_at_1000_std": 0.6496446422377248, + "nauc_precision_at_100_diff1": -0.4796643779471086, + "nauc_precision_at_100_max": 0.6151414562917783, + "nauc_precision_at_100_std": 0.6678831725250701, + "nauc_precision_at_10_diff1": -0.42975726364512384, + "nauc_precision_at_10_max": 0.7033828091076564, + "nauc_precision_at_10_std": 0.6296497019716161, + "nauc_precision_at_1_diff1": 0.15992222589126076, + "nauc_precision_at_1_max": 0.7066438838407989, + "nauc_precision_at_1_std": 0.34516803717873823, + "nauc_precision_at_20_diff1": -0.4946390997396103, + "nauc_precision_at_20_max": 0.6858303123733283, + "nauc_precision_at_20_std": 0.6298812155793545, + "nauc_precision_at_3_diff1": -0.24876059895537342, + "nauc_precision_at_3_max": 0.6456537706466767, + "nauc_precision_at_3_std": 0.5045016679013927, + "nauc_precision_at_5_diff1": -0.32421116538126843, + "nauc_precision_at_5_max": 0.7016255386813988, + "nauc_precision_at_5_std": 0.5649338219757292, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5157172488761371, + "nauc_recall_at_100_max": -0.2912343469081552, + "nauc_recall_at_100_std": -0.45545038271617866, + "nauc_recall_at_10_diff1": 0.430025555269766, + "nauc_recall_at_10_max": -0.49271570163117323, + "nauc_recall_at_10_std": -0.5498160938181346, + "nauc_recall_at_1_diff1": 0.3294708101134731, + "nauc_recall_at_1_max": -0.42544232201495896, + "nauc_recall_at_1_std": -0.5038005227813385, + "nauc_recall_at_20_diff1": 0.4176823227802248, + "nauc_recall_at_20_max": -0.5026073945178894, + "nauc_recall_at_20_std": -0.5407548592660978, + "nauc_recall_at_3_diff1": 0.36561954570571, + "nauc_recall_at_3_max": -0.4642250997167962, + "nauc_recall_at_3_std": -0.5342346748455666, + "nauc_recall_at_5_diff1": 0.3529315499791077, + "nauc_recall_at_5_max": -0.47879447841104084, + "nauc_recall_at_5_std": -0.5554339290951641, + "ndcg_at_1": 0.88235, + "ndcg_at_10": 0.83499, + "ndcg_at_100": 0.82462, + "ndcg_at_1000": 0.87986, + "ndcg_at_20": 0.82108, + "ndcg_at_3": 0.85596, + "ndcg_at_5": 0.84619, + "precision_at_1": 0.88235, + "precision_at_10": 0.61176, + "precision_at_100": 0.24838, + "precision_at_1000": 0.04454, + "precision_at_20": 0.50882, + "precision_at_3": 0.76471, + "precision_at_5": 0.69412, + "recall_at_1": 0.18698, + "recall_at_10": 0.48422, + "recall_at_100": 0.80718, + "recall_at_1000": 1.0, + "recall_at_20": 0.60501, + "recall_at_3": 0.30707, + "recall_at_5": 0.3807 } ] }, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json index f957d7315e..a944a1a301 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json @@ -1,184 +1,184 @@ { - "dataset_revision": "f20b30211b7ba3fc64a02bd83998fe75f3023719", - "evaluation_time": 17.18506622314453, + "dataset_revision": "f71ab9d4aabcda93d55a7e65edfb3a34767d89e6", + "evaluation_time": 24.67104411125183, "kg_co2_emissions": null, "mteb_version": "1.12.90", "scores": { "test": [ { - "cv_recall_at_1": 0.04286, - "cv_recall_at_10": 0.44286, - "cv_recall_at_100": 0.77143, - "cv_recall_at_1000": 0.92857, - "cv_recall_at_20": 0.52857, - "cv_recall_at_3": 0.18571, - "cv_recall_at_5": 0.27143, + "cv_recall_at_1": 0.27143, + "cv_recall_at_10": 0.61429, + "cv_recall_at_100": 0.85714, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.74286, + "cv_recall_at_3": 0.42857, + "cv_recall_at_5": 0.57143, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.04286, - "map_at_1": 0.00178, - "map_at_10": 0.02316, - "map_at_100": 0.03694, - "map_at_1000": 0.05036, - "map_at_20": 0.02814, - "map_at_3": 0.01285, - "map_at_5": 0.01752, - "mrr_at_1": 0.04285714285714286, - "mrr_at_10": 0.13929705215419497, - "mrr_at_100": 0.15254576393638014, - "mrr_at_1000": 0.15348328701607364, - "mrr_at_20": 0.14564255189255187, - "mrr_at_3": 0.1, - "mrr_at_5": 0.11857142857142858, - "nauc_cv_recall_at_1000_diff1": 0.47861811391223064, - "nauc_cv_recall_at_1000_max": 0.5808590102707742, - "nauc_cv_recall_at_1000_std": 0.09140989729224985, - "nauc_cv_recall_at_100_diff1": 0.5586401902191371, - "nauc_cv_recall_at_100_max": -0.12197159565580537, - "nauc_cv_recall_at_100_std": -0.46324143692564745, - "nauc_cv_recall_at_10_diff1": 0.33702203892735166, - "nauc_cv_recall_at_10_max": -0.2566107451434116, - "nauc_cv_recall_at_10_std": -0.2562935040214977, - "nauc_cv_recall_at_1_diff1": 0.5001341561577676, - "nauc_cv_recall_at_1_max": -0.12592791342455964, - "nauc_cv_recall_at_1_std": 0.13701815580001792, - "nauc_cv_recall_at_20_diff1": 0.33340522734848327, - "nauc_cv_recall_at_20_max": -0.27021038803160713, - "nauc_cv_recall_at_20_std": -0.2604850885276762, - "nauc_cv_recall_at_3_diff1": 0.42356508767210377, - "nauc_cv_recall_at_3_max": -0.30471252842923874, - "nauc_cv_recall_at_3_std": -0.2722848548580372, - "nauc_cv_recall_at_5_diff1": 0.37677403680683497, - "nauc_cv_recall_at_5_max": -0.30530641993804797, - "nauc_cv_recall_at_5_std": -0.2763347032980382, - "nauc_map_at_1000_diff1": 0.32492788395298233, - "nauc_map_at_1000_max": -0.264513294885872, - "nauc_map_at_1000_std": -0.06025413474178467, - "nauc_map_at_100_diff1": 0.33932455574204345, - "nauc_map_at_100_max": -0.3183944729391321, - "nauc_map_at_100_std": -0.11067793875024108, - "nauc_map_at_10_diff1": 0.3627322367418886, - "nauc_map_at_10_max": -0.270787009945954, - "nauc_map_at_10_std": -0.12449041958590679, - "nauc_map_at_1_diff1": 0.8349360037495761, - "nauc_map_at_1_max": 0.04030601005296298, - "nauc_map_at_1_std": 0.28342764489523986, - "nauc_map_at_20_diff1": 0.36471735689250456, - "nauc_map_at_20_max": -0.27443485632176895, - "nauc_map_at_20_std": -0.10351956211548237, - "nauc_map_at_3_diff1": 0.33740985231621823, - "nauc_map_at_3_max": -0.26557891525076527, - "nauc_map_at_3_std": -0.13603337759846046, - "nauc_map_at_5_diff1": 0.3822623605794258, - "nauc_map_at_5_max": -0.26535510732907475, - "nauc_map_at_5_std": -0.15188815608303466, - "nauc_mrr_at_1000_diff1": 0.42803033998914947, - "nauc_mrr_at_1000_max": -0.25330248029839103, - "nauc_mrr_at_1000_std": -0.1498777504287341, - "nauc_mrr_at_100_diff1": 0.428756525260202, - "nauc_mrr_at_100_max": -0.255106401697983, - "nauc_mrr_at_100_std": -0.15247667584774333, - "nauc_mrr_at_10_diff1": 0.4269277134986295, - "nauc_mrr_at_10_max": -0.26024547551332144, - "nauc_mrr_at_10_std": -0.15027887241014917, - "nauc_mrr_at_1_diff1": 0.5001341561577676, - "nauc_mrr_at_1_max": -0.12592791342455964, - "nauc_mrr_at_1_std": 0.13701815580001792, - "nauc_mrr_at_20_diff1": 0.4273548864254061, - "nauc_mrr_at_20_max": -0.2649857851418372, - "nauc_mrr_at_20_std": -0.150811901899745, - "nauc_mrr_at_3_diff1": 0.46234460520174797, - "nauc_mrr_at_3_max": -0.26846705418134004, - "nauc_mrr_at_3_std": -0.14951136379707816, - "nauc_mrr_at_5_diff1": 0.4315603993226756, - "nauc_mrr_at_5_max": -0.2751708043421038, - "nauc_mrr_at_5_std": -0.15676414005890574, - "nauc_ndcg_at_1000_diff1": 0.3031539661086623, - "nauc_ndcg_at_1000_max": -0.06324949257415059, - "nauc_ndcg_at_1000_std": 0.03291344536352667, - "nauc_ndcg_at_100_diff1": 0.3537878100611368, - "nauc_ndcg_at_100_max": -0.3440256642252378, - "nauc_ndcg_at_100_std": -0.13928638317122793, - "nauc_ndcg_at_10_diff1": 0.31423572757944224, - "nauc_ndcg_at_10_max": -0.27799943432453955, - "nauc_ndcg_at_10_std": -0.12000726018713354, - "nauc_ndcg_at_1_diff1": 0.5001341561577676, - "nauc_ndcg_at_1_max": -0.12592791342455964, - "nauc_ndcg_at_1_std": 0.13701815580001792, - "nauc_ndcg_at_20_diff1": 0.3527561180413498, - "nauc_ndcg_at_20_max": -0.29516361937414604, - "nauc_ndcg_at_20_std": -0.10951825882002067, - "nauc_ndcg_at_3_diff1": 0.3265974284434805, - "nauc_ndcg_at_3_max": -0.27320998825572146, - "nauc_ndcg_at_3_std": -0.12237689567430368, - "nauc_ndcg_at_5_diff1": 0.32240119227390085, - "nauc_ndcg_at_5_max": -0.30077766985994786, - "nauc_ndcg_at_5_std": -0.15147497522509035, - "nauc_precision_at_1000_diff1": -0.12639282009702774, - "nauc_precision_at_1000_max": 0.22826737103796205, - "nauc_precision_at_1000_std": 0.32628514757600596, - "nauc_precision_at_100_diff1": 0.2772154726137188, - "nauc_precision_at_100_max": -0.23028476826992986, - "nauc_precision_at_100_std": -0.1303180540104937, - "nauc_precision_at_10_diff1": 0.29107138713345343, - "nauc_precision_at_10_max": -0.30600784337992426, - "nauc_precision_at_10_std": -0.12021874830794935, - "nauc_precision_at_1_diff1": 0.5001341561577676, - "nauc_precision_at_1_max": -0.12592791342455964, - "nauc_precision_at_1_std": 0.13701815580001792, - "nauc_precision_at_20_diff1": 0.27727122281419164, - "nauc_precision_at_20_max": -0.3127822679663893, - "nauc_precision_at_20_std": -0.07951548670204237, - "nauc_precision_at_3_diff1": 0.3064711218627153, - "nauc_precision_at_3_max": -0.29553472432214506, - "nauc_precision_at_3_std": -0.17296643483519816, - "nauc_precision_at_5_diff1": 0.27289717372431765, - "nauc_precision_at_5_max": -0.33688627375325547, - "nauc_precision_at_5_std": -0.1943667406192727, - "nauc_recall_at_1000_diff1": 0.2612937021120264, - "nauc_recall_at_1000_max": -0.04177364682142833, - "nauc_recall_at_1000_std": -0.10225779216767208, - "nauc_recall_at_100_diff1": 0.23045925470740378, - "nauc_recall_at_100_max": -0.32266175361589855, - "nauc_recall_at_100_std": -0.10122639227831537, - "nauc_recall_at_10_diff1": 0.3214491777002914, - "nauc_recall_at_10_max": -0.2191819309348708, - "nauc_recall_at_10_std": -0.13294514318629588, - "nauc_recall_at_1_diff1": 0.8349360037495761, - "nauc_recall_at_1_max": 0.04030601005296298, - "nauc_recall_at_1_std": 0.28342764489523986, - "nauc_recall_at_20_diff1": 0.30304432643409973, - "nauc_recall_at_20_max": -0.2369864752715589, - "nauc_recall_at_20_std": -0.11319354561879752, - "nauc_recall_at_3_diff1": 0.2610862099372524, - "nauc_recall_at_3_max": -0.26808957441898906, - "nauc_recall_at_3_std": -0.1754514836876096, - "nauc_recall_at_5_diff1": 0.3659777743786138, - "nauc_recall_at_5_max": -0.23557478979416202, - "nauc_recall_at_5_std": -0.1749293090721626, - "ndcg_at_1": 0.04286, - "ndcg_at_10": 0.08519, - "ndcg_at_100": 0.12707, - "ndcg_at_1000": 0.26609, - "ndcg_at_20": 0.08785, - "ndcg_at_3": 0.07857, - "ndcg_at_5": 0.08201, - "precision_at_1": 0.04286, - "precision_at_10": 0.07571, - "precision_at_100": 0.03571, - "precision_at_1000": 0.01857, - "precision_at_20": 0.05714, - "precision_at_3": 0.08571, - "precision_at_5": 0.08, - "recall_at_1": 0.00178, - "recall_at_10": 0.07379, - "recall_at_100": 0.22471, - "recall_at_1000": 0.60888, - "recall_at_20": 0.09447, - "recall_at_3": 0.02931, - "recall_at_5": 0.04839 + "main_score": 0.27143, + "map_at_1": 0.06428, + "map_at_10": 0.10339, + "map_at_100": 0.1463, + "map_at_1000": 0.18944, + "map_at_20": 0.11828, + "map_at_3": 0.07918, + "map_at_5": 0.09165, + "mrr_at_1": 0.2714285714285714, + "mrr_at_10": 0.3802551020408163, + "mrr_at_100": 0.3931760616047423, + "mrr_at_1000": 0.39374600845359353, + "mrr_at_20": 0.38933244509925174, + "mrr_at_3": 0.34047619047619043, + "mrr_at_5": 0.374047619047619, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.3957240038872689, + "nauc_cv_recall_at_100_max": -0.45918367346938554, + "nauc_cv_recall_at_100_std": -0.7625364431486866, + "nauc_cv_recall_at_10_diff1": 0.3501381065268928, + "nauc_cv_recall_at_10_max": -0.3230170295134737, + "nauc_cv_recall_at_10_std": -0.28669610039365817, + "nauc_cv_recall_at_1_diff1": 0.31133966351507303, + "nauc_cv_recall_at_1_max": -0.36652967019618166, + "nauc_cv_recall_at_1_std": -0.39039944931468024, + "nauc_cv_recall_at_20_diff1": 0.48971241524433046, + "nauc_cv_recall_at_20_max": -0.35202829085807796, + "nauc_cv_recall_at_20_std": -0.4874327799859716, + "nauc_cv_recall_at_3_diff1": 0.20616527390900677, + "nauc_cv_recall_at_3_max": -0.3163788300835654, + "nauc_cv_recall_at_3_std": -0.34913649025069615, + "nauc_cv_recall_at_5_diff1": 0.24162217659137575, + "nauc_cv_recall_at_5_max": -0.40519507186858295, + "nauc_cv_recall_at_5_std": -0.349342915811088, + "nauc_map_at_1000_diff1": 0.33483050942379805, + "nauc_map_at_1000_max": -0.3354071059234048, + "nauc_map_at_1000_std": -0.325502994839205, + "nauc_map_at_100_diff1": 0.36421675112335855, + "nauc_map_at_100_max": -0.37056372968494616, + "nauc_map_at_100_std": -0.39098657201855935, + "nauc_map_at_10_diff1": 0.43724421619965775, + "nauc_map_at_10_max": -0.3475499176287484, + "nauc_map_at_10_std": -0.3087709853113486, + "nauc_map_at_1_diff1": 0.5328003645974971, + "nauc_map_at_1_max": -0.30820864556110844, + "nauc_map_at_1_std": -0.2525466251963858, + "nauc_map_at_20_diff1": 0.40424948920352965, + "nauc_map_at_20_max": -0.36561679338161, + "nauc_map_at_20_std": -0.34598603785503834, + "nauc_map_at_3_diff1": 0.47569665621843843, + "nauc_map_at_3_max": -0.3350766680518147, + "nauc_map_at_3_std": -0.29588688611880093, + "nauc_map_at_5_diff1": 0.44497494504359997, + "nauc_map_at_5_max": -0.35371869827402724, + "nauc_map_at_5_std": -0.2890820614425645, + "nauc_mrr_at_1000_diff1": 0.2682845806139734, + "nauc_mrr_at_1000_max": -0.36456025639429424, + "nauc_mrr_at_1000_std": -0.37575466392348833, + "nauc_mrr_at_100_diff1": 0.26897531827173626, + "nauc_mrr_at_100_max": -0.3659176034789706, + "nauc_mrr_at_100_std": -0.37740763588526954, + "nauc_mrr_at_10_diff1": 0.2686668033979706, + "nauc_mrr_at_10_max": -0.3628633448193816, + "nauc_mrr_at_10_std": -0.36335706237767257, + "nauc_mrr_at_1_diff1": 0.31133966351507303, + "nauc_mrr_at_1_max": -0.36652967019618166, + "nauc_mrr_at_1_std": -0.39039944931468024, + "nauc_mrr_at_20_diff1": 0.2731473760001198, + "nauc_mrr_at_20_max": -0.36635207234744227, + "nauc_mrr_at_20_std": -0.3759359230442535, + "nauc_mrr_at_3_diff1": 0.24452621954568515, + "nauc_mrr_at_3_max": -0.35272889400073, + "nauc_mrr_at_3_std": -0.3683045738415161, + "nauc_mrr_at_5_diff1": 0.25344916447455645, + "nauc_mrr_at_5_max": -0.3740602466954703, + "nauc_mrr_at_5_std": -0.3711781326955965, + "nauc_ndcg_at_1000_diff1": 0.15289382130510762, + "nauc_ndcg_at_1000_max": -0.11235365434908824, + "nauc_ndcg_at_1000_std": -0.16901273056125712, + "nauc_ndcg_at_100_diff1": 0.33546453963254314, + "nauc_ndcg_at_100_max": -0.39506763446494053, + "nauc_ndcg_at_100_std": -0.47504197799159326, + "nauc_ndcg_at_10_diff1": 0.2602337772209344, + "nauc_ndcg_at_10_max": -0.35707200508700326, + "nauc_ndcg_at_10_std": -0.32627013894830836, + "nauc_ndcg_at_1_diff1": 0.31133966351507303, + "nauc_ndcg_at_1_max": -0.36652967019618166, + "nauc_ndcg_at_1_std": -0.39039944931468024, + "nauc_ndcg_at_20_diff1": 0.2951622910865746, + "nauc_ndcg_at_20_max": -0.4075808329991618, + "nauc_ndcg_at_20_std": -0.4033601223158598, + "nauc_ndcg_at_3_diff1": 0.2583369568946569, + "nauc_ndcg_at_3_max": -0.36460761087963334, + "nauc_ndcg_at_3_std": -0.37460677189461866, + "nauc_ndcg_at_5_diff1": 0.24895791797858352, + "nauc_ndcg_at_5_max": -0.38343316697513324, + "nauc_ndcg_at_5_std": -0.35014891765983175, + "nauc_precision_at_1000_diff1": -0.15221736880156744, + "nauc_precision_at_1000_max": 0.24012671254256857, + "nauc_precision_at_1000_std": 0.31990023300446896, + "nauc_precision_at_100_diff1": -0.0025063624250149666, + "nauc_precision_at_100_max": -0.23039752243674347, + "nauc_precision_at_100_std": -0.2687557194001006, + "nauc_precision_at_10_diff1": 0.04266939397452653, + "nauc_precision_at_10_max": -0.2921849379730692, + "nauc_precision_at_10_std": -0.32849429418487175, + "nauc_precision_at_1_diff1": 0.31133966351507303, + "nauc_precision_at_1_max": -0.36652967019618166, + "nauc_precision_at_1_std": -0.39039944931468024, + "nauc_precision_at_20_diff1": -0.012177956618922228, + "nauc_precision_at_20_max": -0.3166595426664091, + "nauc_precision_at_20_std": -0.38852936131749555, + "nauc_precision_at_3_diff1": 0.151720996554046, + "nauc_precision_at_3_max": -0.3436744959797202, + "nauc_precision_at_3_std": -0.37392561492454535, + "nauc_precision_at_5_diff1": 0.09040511727078884, + "nauc_precision_at_5_max": -0.35160602517367096, + "nauc_precision_at_5_std": -0.35003095123461053, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.26559603138781285, + "nauc_recall_at_100_max": -0.3073116185520707, + "nauc_recall_at_100_std": -0.449657488606374, + "nauc_recall_at_10_diff1": 0.38683534388556096, + "nauc_recall_at_10_max": -0.3358496548040101, + "nauc_recall_at_10_std": -0.2154853289653661, + "nauc_recall_at_1_diff1": 0.5328003645974971, + "nauc_recall_at_1_max": -0.30820864556110844, + "nauc_recall_at_1_std": -0.2525466251963858, + "nauc_recall_at_20_diff1": 0.30135439280777626, + "nauc_recall_at_20_max": -0.33860982229647496, + "nauc_recall_at_20_std": -0.2973578871821516, + "nauc_recall_at_3_diff1": 0.4514271986825242, + "nauc_recall_at_3_max": -0.33100045995309413, + "nauc_recall_at_3_std": -0.2898936824323878, + "nauc_recall_at_5_diff1": 0.3748793011771698, + "nauc_recall_at_5_max": -0.3504567142206052, + "nauc_recall_at_5_std": -0.17845054202949487, + "ndcg_at_1": 0.27143, + "ndcg_at_10": 0.22624, + "ndcg_at_100": 0.29318, + "ndcg_at_1000": 0.52204, + "ndcg_at_20": 0.24135, + "ndcg_at_3": 0.24796, + "ndcg_at_5": 0.24666, + "precision_at_1": 0.27143, + "precision_at_10": 0.15714, + "precision_at_100": 0.07357, + "precision_at_1000": 0.03567, + "precision_at_20": 0.13357, + "precision_at_3": 0.20952, + "precision_at_5": 0.2, + "recall_at_1": 0.06428, + "recall_at_10": 0.14214, + "recall_at_100": 0.38739, + "recall_at_1000": 1.0, + "recall_at_20": 0.2058, + "recall_at_3": 0.08351, + "recall_at_5": 0.11676 } ] }, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json index 27241284b1..f1e28194b9 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json @@ -1,184 +1,184 @@ { - "dataset_revision": "3bd28e9c45e15f299117c634799f7035c4de2d31", - "evaluation_time": 17.523592710494995, + "dataset_revision": "1dfb86730ee4b3f49b441f4896d473c83eb5ff0d", + "evaluation_time": 26.38901996612549, "kg_co2_emissions": null, "mteb_version": "1.12.90", "scores": { "test": [ { - "cv_recall_at_1": 0.75714, - "cv_recall_at_10": 0.87143, - "cv_recall_at_100": 0.94286, + "cv_recall_at_1": 0.91429, + "cv_recall_at_10": 0.94286, + "cv_recall_at_100": 1.0, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.88571, - "cv_recall_at_3": 0.82857, - "cv_recall_at_5": 0.85714, + "cv_recall_at_20": 0.97143, + "cv_recall_at_3": 0.91429, + "cv_recall_at_5": 0.94286, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.75714, - "map_at_1": 0.04928, - "map_at_10": 0.14238, - "map_at_100": 0.25975, - "map_at_1000": 0.33, - "map_at_20": 0.1832, - "map_at_3": 0.08106, - "map_at_5": 0.10574, - "mrr_at_1": 0.7571428571428571, - "mrr_at_10": 0.7992857142857144, - "mrr_at_100": 0.8018753503269448, - "mrr_at_1000": 0.8022392070599913, - "mrr_at_20": 0.8001785714285715, - "mrr_at_3": 0.7904761904761906, - "mrr_at_5": 0.796904761904762, + "main_score": 0.91429, + "map_at_1": 0.08083, + "map_at_10": 0.24687, + "map_at_100": 0.42618, + "map_at_1000": 0.5327, + "map_at_20": 0.31214, + "map_at_3": 0.14225, + "map_at_5": 0.18461, + "mrr_at_1": 0.9142857142857143, + "mrr_at_10": 0.92, + "mrr_at_100": 0.9224459377246834, + "mrr_at_1000": 0.9224459377246834, + "mrr_at_20": 0.9215873015873017, + "mrr_at_3": 0.9142857142857143, + "mrr_at_5": 0.92, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.15581232492997169, - "nauc_cv_recall_at_100_max": 0.2811624649859951, - "nauc_cv_recall_at_100_std": -0.12196545284780497, - "nauc_cv_recall_at_10_diff1": 0.394540014958862, - "nauc_cv_recall_at_10_max": 0.4610535313601873, - "nauc_cv_recall_at_10_std": -0.14937493321936154, - "nauc_cv_recall_at_1_diff1": 0.3099268419602683, - "nauc_cv_recall_at_1_max": 0.6277204689460957, - "nauc_cv_recall_at_1_std": 0.11781811503259924, - "nauc_cv_recall_at_20_diff1": 0.39261983863312777, - "nauc_cv_recall_at_20_max": 0.3849074513526345, - "nauc_cv_recall_at_20_std": -0.18147840531561454, - "nauc_cv_recall_at_3_diff1": 0.29657669519420665, - "nauc_cv_recall_at_3_max": 0.5705233706385782, - "nauc_cv_recall_at_3_std": 0.04756418696510771, - "nauc_cv_recall_at_5_diff1": 0.37162293488824066, - "nauc_cv_recall_at_5_max": 0.5234207968901848, - "nauc_cv_recall_at_5_std": -0.06068999028182738, - "nauc_map_at_1000_diff1": -0.14810539444535628, - "nauc_map_at_1000_max": 0.42639970063449095, - "nauc_map_at_1000_std": 0.35965851851847036, - "nauc_map_at_100_diff1": -0.04654719636323819, - "nauc_map_at_100_max": 0.20124568398625026, - "nauc_map_at_100_std": 0.09779920001336984, - "nauc_map_at_10_diff1": 0.19553982602379108, - "nauc_map_at_10_max": -0.16962211647877953, - "nauc_map_at_10_std": -0.2049718676191143, - "nauc_map_at_1_diff1": 0.19831336261662813, - "nauc_map_at_1_max": -0.26048526225990315, - "nauc_map_at_1_std": -0.24848660044643464, - "nauc_map_at_20_diff1": 0.11533311201293248, - "nauc_map_at_20_max": -0.06940468788280867, - "nauc_map_at_20_std": -0.15394999023265438, - "nauc_map_at_3_diff1": 0.22661360439380449, - "nauc_map_at_3_max": -0.24582246289343465, - "nauc_map_at_3_std": -0.24358060859761363, - "nauc_map_at_5_diff1": 0.2391073300885911, - "nauc_map_at_5_max": -0.21313902222536138, - "nauc_map_at_5_std": -0.22555952910665222, - "nauc_mrr_at_1000_diff1": 0.3118072480358942, - "nauc_mrr_at_1000_max": 0.5896434235555521, - "nauc_mrr_at_1000_std": 0.07565750084460038, - "nauc_mrr_at_100_diff1": 0.3114581767470047, - "nauc_mrr_at_100_max": 0.5888175740478946, - "nauc_mrr_at_100_std": 0.07590573329917173, - "nauc_mrr_at_10_diff1": 0.31501956699434985, - "nauc_mrr_at_10_max": 0.5906694492065899, - "nauc_mrr_at_10_std": 0.07282707502971797, - "nauc_mrr_at_1_diff1": 0.3099268419602683, - "nauc_mrr_at_1_max": 0.6277204689460957, - "nauc_mrr_at_1_std": 0.11781811503259924, - "nauc_mrr_at_20_diff1": 0.31460997067981183, - "nauc_mrr_at_20_max": 0.5883036453949262, - "nauc_mrr_at_20_std": 0.07253787758405468, - "nauc_mrr_at_3_diff1": 0.298829153668517, - "nauc_mrr_at_3_max": 0.6036890630335515, - "nauc_mrr_at_3_std": 0.09929911274966623, - "nauc_mrr_at_5_diff1": 0.31309074738187426, - "nauc_mrr_at_5_max": 0.5968819745150035, - "nauc_mrr_at_5_std": 0.08124058142418439, - "nauc_ndcg_at_1000_diff1": 0.023759962305708315, - "nauc_ndcg_at_1000_max": 0.42118350886771483, - "nauc_ndcg_at_1000_std": 0.21503403862781068, - "nauc_ndcg_at_100_diff1": -0.10602196843045252, - "nauc_ndcg_at_100_max": 0.4040222401408388, - "nauc_ndcg_at_100_std": 0.3422251986340049, - "nauc_ndcg_at_10_diff1": -0.14230825595892613, - "nauc_ndcg_at_10_max": 0.5717619317460108, - "nauc_ndcg_at_10_std": 0.35188240284777716, - "nauc_ndcg_at_1_diff1": 0.3099268419602683, - "nauc_ndcg_at_1_max": 0.6277204689460957, - "nauc_ndcg_at_1_std": 0.11781811503259924, - "nauc_ndcg_at_20_diff1": -0.15743789243142528, - "nauc_ndcg_at_20_max": 0.5724054251457065, - "nauc_ndcg_at_20_std": 0.3797961049903619, - "nauc_ndcg_at_3_diff1": 0.02676887075350004, - "nauc_ndcg_at_3_max": 0.5535721465155271, - "nauc_ndcg_at_3_std": 0.22786549855388236, - "nauc_ndcg_at_5_diff1": -0.04449400646171614, - "nauc_ndcg_at_5_max": 0.555861797739005, - "nauc_ndcg_at_5_std": 0.27833851228984907, - "nauc_precision_at_1000_diff1": -0.19203752528557633, - "nauc_precision_at_1000_max": 0.4830680560318716, - "nauc_precision_at_1000_std": 0.5059538376358517, - "nauc_precision_at_100_diff1": -0.29506043721080727, - "nauc_precision_at_100_max": 0.658086670179558, - "nauc_precision_at_100_std": 0.6166722456503947, - "nauc_precision_at_10_diff1": -0.2282600490658613, - "nauc_precision_at_10_max": 0.5965276467257976, - "nauc_precision_at_10_std": 0.42049443291187055, - "nauc_precision_at_1_diff1": 0.3099268419602683, - "nauc_precision_at_1_max": 0.6277204689460957, - "nauc_precision_at_1_std": 0.11781811503259924, - "nauc_precision_at_20_diff1": -0.2777369235938004, - "nauc_precision_at_20_max": 0.6422763272584597, - "nauc_precision_at_20_std": 0.46973331485105685, - "nauc_precision_at_3_diff1": -0.03905783426100687, - "nauc_precision_at_3_max": 0.548195217505554, - "nauc_precision_at_3_std": 0.2602422302381689, - "nauc_precision_at_5_diff1": -0.12812907648800725, - "nauc_precision_at_5_max": 0.5661902630852285, - "nauc_precision_at_5_std": 0.33254144330289526, - "nauc_recall_at_1000_diff1": 0.11552874226597364, - "nauc_recall_at_1000_max": 0.1991754113531379, - "nauc_recall_at_1000_std": 0.05667927260800616, - "nauc_recall_at_100_diff1": -0.01602892704072587, - "nauc_recall_at_100_max": -0.12257633928876718, - "nauc_recall_at_100_std": -0.01682143204953458, - "nauc_recall_at_10_diff1": 0.22298978148074566, - "nauc_recall_at_10_max": -0.22859971193805304, - "nauc_recall_at_10_std": -0.23715467924390157, - "nauc_recall_at_1_diff1": 0.19831336261662813, - "nauc_recall_at_1_max": -0.26048526225990315, - "nauc_recall_at_1_std": -0.24848660044643464, - "nauc_recall_at_20_diff1": 0.16037912600194174, - "nauc_recall_at_20_max": -0.1835918931237162, - "nauc_recall_at_20_std": -0.20833669696201074, - "nauc_recall_at_3_diff1": 0.2179234117693627, - "nauc_recall_at_3_max": -0.2525912771978034, - "nauc_recall_at_3_std": -0.24319572308223014, - "nauc_recall_at_5_diff1": 0.2530123469958082, - "nauc_recall_at_5_max": -0.22138719273185706, - "nauc_recall_at_5_std": -0.2215811361906131, - "ndcg_at_1": 0.75714, - "ndcg_at_10": 0.58385, - "ndcg_at_100": 0.48629, - "ndcg_at_1000": 0.59891, - "ndcg_at_20": 0.52663, - "ndcg_at_3": 0.6553, - "ndcg_at_5": 0.63094, - "precision_at_1": 0.75714, - "precision_at_10": 0.51286, - "precision_at_100": 0.227, - "precision_at_1000": 0.05964, - "precision_at_20": 0.42286, - "precision_at_3": 0.61429, - "precision_at_5": 0.57714, - "recall_at_1": 0.04928, - "recall_at_10": 0.17721, - "recall_at_100": 0.44393, - "recall_at_1000": 0.76857, - "recall_at_20": 0.24576, - "recall_at_3": 0.08507, - "recall_at_5": 0.11871 + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": -0.4013772175536871, + "nauc_cv_recall_at_10_max": 0.42938842203548166, + "nauc_cv_recall_at_10_std": 0.6586134453781514, + "nauc_cv_recall_at_1_diff1": -0.11554621848739352, + "nauc_cv_recall_at_1_max": 0.5269996887643944, + "nauc_cv_recall_at_1_std": 0.21856520385932235, + "nauc_cv_recall_at_20_diff1": 0.3489729225023323, + "nauc_cv_recall_at_20_max": 0.9346405228758171, + "nauc_cv_recall_at_20_std": 0.5401493930905691, + "nauc_cv_recall_at_3_diff1": -0.11554621848739352, + "nauc_cv_recall_at_3_max": 0.5269996887643944, + "nauc_cv_recall_at_3_std": 0.21856520385932235, + "nauc_cv_recall_at_5_diff1": -0.4013772175536871, + "nauc_cv_recall_at_5_max": 0.42938842203548166, + "nauc_cv_recall_at_5_std": 0.6586134453781514, + "nauc_map_at_1000_diff1": -0.23125835300876216, + "nauc_map_at_1000_max": 0.23519514137370254, + "nauc_map_at_1000_std": 0.2182230501859235, + "nauc_map_at_100_diff1": -0.05015810647551921, + "nauc_map_at_100_max": -0.04327639525818628, + "nauc_map_at_100_std": -0.15828452131450388, + "nauc_map_at_10_diff1": 0.1893778857001354, + "nauc_map_at_10_max": -0.34441772024823814, + "nauc_map_at_10_std": -0.3848968165397017, + "nauc_map_at_1_diff1": 0.1529533401602949, + "nauc_map_at_1_max": -0.4190116365256113, + "nauc_map_at_1_std": -0.3330481576960033, + "nauc_map_at_20_diff1": 0.09589668436288233, + "nauc_map_at_20_max": -0.2673998395876724, + "nauc_map_at_20_std": -0.3314998258254241, + "nauc_map_at_3_diff1": 0.22783602569587666, + "nauc_map_at_3_max": -0.375611479696183, + "nauc_map_at_3_std": -0.3593266287200665, + "nauc_map_at_5_diff1": 0.23171354107352043, + "nauc_map_at_5_max": -0.3834575082863232, + "nauc_map_at_5_std": -0.38241308439234356, + "nauc_mrr_at_1000_diff1": -0.14268770122670912, + "nauc_mrr_at_1000_max": 0.5205774182168708, + "nauc_mrr_at_1000_std": 0.2680405904560385, + "nauc_mrr_at_100_diff1": -0.14268770122670912, + "nauc_mrr_at_100_max": 0.5205774182168708, + "nauc_mrr_at_100_std": 0.2680405904560385, + "nauc_mrr_at_10_diff1": -0.15637921835400717, + "nauc_mrr_at_10_max": 0.5130552220888355, + "nauc_mrr_at_10_std": 0.2814292383620098, + "nauc_mrr_at_1_diff1": -0.11554621848739352, + "nauc_mrr_at_1_max": 0.5269996887643944, + "nauc_mrr_at_1_std": 0.21856520385932235, + "nauc_mrr_at_20_diff1": -0.13623047059579352, + "nauc_mrr_at_20_max": 0.524976657329598, + "nauc_mrr_at_20_std": 0.2713958727890603, + "nauc_mrr_at_3_diff1": -0.11554621848739352, + "nauc_mrr_at_3_max": 0.5269996887643944, + "nauc_mrr_at_3_std": 0.21856520385932235, + "nauc_mrr_at_5_diff1": -0.15637921835400717, + "nauc_mrr_at_5_max": 0.5130552220888355, + "nauc_mrr_at_5_std": 0.2814292383620098, + "nauc_ndcg_at_1000_diff1": -0.24370615505516585, + "nauc_ndcg_at_1000_max": 0.3585208209406902, + "nauc_ndcg_at_1000_std": 0.263969492188972, + "nauc_ndcg_at_100_diff1": -0.14186676166246207, + "nauc_ndcg_at_100_max": 0.2275659208451184, + "nauc_ndcg_at_100_std": 0.16048415696096283, + "nauc_ndcg_at_10_diff1": -0.30589363585428303, + "nauc_ndcg_at_10_max": 0.500142114454778, + "nauc_ndcg_at_10_std": 0.4141408373889108, + "nauc_ndcg_at_1_diff1": -0.11554621848739352, + "nauc_ndcg_at_1_max": 0.5269996887643944, + "nauc_ndcg_at_1_std": 0.21856520385932235, + "nauc_ndcg_at_20_diff1": -0.2982156974982523, + "nauc_ndcg_at_20_max": 0.487494996769566, + "nauc_ndcg_at_20_std": 0.4663494007016063, + "nauc_ndcg_at_3_diff1": -0.13605929183456947, + "nauc_ndcg_at_3_max": 0.5394295944368543, + "nauc_ndcg_at_3_std": 0.2841764519265793, + "nauc_ndcg_at_5_diff1": -0.18517240726502945, + "nauc_ndcg_at_5_max": 0.5018241088290002, + "nauc_ndcg_at_5_std": 0.32986920973664985, + "nauc_precision_at_1000_diff1": -0.2610146663608417, + "nauc_precision_at_1000_max": 0.42090687761865053, + "nauc_precision_at_1000_std": 0.5453516915872082, + "nauc_precision_at_100_diff1": -0.31550298388759973, + "nauc_precision_at_100_max": 0.5382316505653624, + "nauc_precision_at_100_std": 0.5864596701910575, + "nauc_precision_at_10_diff1": -0.3396134608542791, + "nauc_precision_at_10_max": 0.5841486511939392, + "nauc_precision_at_10_std": 0.4649483079756401, + "nauc_precision_at_1_diff1": -0.11554621848739352, + "nauc_precision_at_1_max": 0.5269996887643944, + "nauc_precision_at_1_std": 0.21856520385932235, + "nauc_precision_at_20_diff1": -0.3739204789385173, + "nauc_precision_at_20_max": 0.5974041684159593, + "nauc_precision_at_20_std": 0.5452135797286048, + "nauc_precision_at_3_diff1": -0.16477444647626113, + "nauc_precision_at_3_max": 0.5728888479602177, + "nauc_precision_at_3_std": 0.3239051214923765, + "nauc_precision_at_5_diff1": -0.21896064507078017, + "nauc_precision_at_5_max": 0.5663742826794222, + "nauc_precision_at_5_std": 0.378912278123309, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.18668259021675102, + "nauc_recall_at_100_max": -0.13326671980523305, + "nauc_recall_at_100_std": -0.28507500423807525, + "nauc_recall_at_10_diff1": 0.17850980669329247, + "nauc_recall_at_10_max": -0.40096190909138624, + "nauc_recall_at_10_std": -0.3570134850979618, + "nauc_recall_at_1_diff1": 0.1529533401602949, + "nauc_recall_at_1_max": -0.4190116365256113, + "nauc_recall_at_1_std": -0.3330481576960033, + "nauc_recall_at_20_diff1": 0.1509705328425781, + "nauc_recall_at_20_max": -0.3011359312987517, + "nauc_recall_at_20_std": -0.2929059943600196, + "nauc_recall_at_3_diff1": 0.22113145253276656, + "nauc_recall_at_3_max": -0.3819326194584587, + "nauc_recall_at_3_std": -0.3572099966902034, + "nauc_recall_at_5_diff1": 0.1981700707835856, + "nauc_recall_at_5_max": -0.42018635579579555, + "nauc_recall_at_5_std": -0.3263923639383356, + "ndcg_at_1": 0.91429, + "ndcg_at_10": 0.77153, + "ndcg_at_100": 0.65508, + "ndcg_at_1000": 0.81056, + "ndcg_at_20": 0.70656, + "ndcg_at_3": 0.85518, + "ndcg_at_5": 0.83659, + "precision_at_1": 0.91429, + "precision_at_10": 0.66143, + "precision_at_100": 0.29157, + "precision_at_1000": 0.07894, + "precision_at_20": 0.55, + "precision_at_3": 0.81429, + "precision_at_5": 0.76571, + "recall_at_1": 0.08083, + "recall_at_10": 0.26885, + "recall_at_100": 0.57158, + "recall_at_1000": 1.0, + "recall_at_20": 0.35986, + "recall_at_3": 0.14512, + "recall_at_5": 0.20079 } ] }, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json index bd4a6c53b9..1c5827ef01 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json @@ -1,12 +1,12 @@ { - "dataset_revision": "a7293da8a341de665ee4dcb2f209281df342d80b", - "evaluation_time": 22.57354736328125, + "dataset_revision": "7d821ddebcb30ad343133e3a81e23347ac2a08a8", + "evaluation_time": 44.682684659957886, "kg_co2_emissions": null, "mteb_version": "1.12.90", "scores": { "test": [ { - "cv_recall_at_1": 0.94286, + "cv_recall_at_1": 0.97143, "cv_recall_at_10": 1.0, "cv_recall_at_100": 1.0, "cv_recall_at_1000": 1.0, @@ -17,21 +17,21 @@ "languages": [ "eng-Latn" ], - "main_score": 0.94286, - "map_at_1": 0.02765, - "map_at_10": 0.11901, - "map_at_100": 0.46776, - "map_at_1000": 0.58942, - "map_at_20": 0.19679, - "map_at_3": 0.05048, - "map_at_5": 0.07407, - "mrr_at_1": 0.9428571428571428, - "mrr_at_10": 0.9666666666666666, - "mrr_at_100": 0.9666666666666666, - "mrr_at_1000": 0.9666666666666666, - "mrr_at_20": 0.9666666666666666, - "mrr_at_3": 0.9666666666666666, - "mrr_at_5": 0.9666666666666666, + "main_score": 0.97143, + "map_at_1": 0.02925, + "map_at_10": 0.14539, + "map_at_100": 0.63054, + "map_at_1000": 0.75513, + "map_at_20": 0.24788, + "map_at_3": 0.06378, + "map_at_5": 0.08742, + "mrr_at_1": 0.9714285714285714, + "mrr_at_10": 0.9857142857142858, + "mrr_at_100": 0.9857142857142858, + "mrr_at_1000": 0.9857142857142858, + "mrr_at_20": 0.9857142857142858, + "mrr_at_3": 0.9857142857142858, + "mrr_at_5": 0.9857142857142858, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, @@ -41,9 +41,9 @@ "nauc_cv_recall_at_10_diff1": NaN, "nauc_cv_recall_at_10_max": NaN, "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": -0.303338001867413, - "nauc_cv_recall_at_1_max": -0.11951447245564903, - "nauc_cv_recall_at_1_std": 0.38328664799253065, + "nauc_cv_recall_at_1_diff1": -0.6615312791783367, + "nauc_cv_recall_at_1_max": 0.2754435107376266, + "nauc_cv_recall_at_1_std": 0.7117180205415492, "nauc_cv_recall_at_20_diff1": NaN, "nauc_cv_recall_at_20_max": NaN, "nauc_cv_recall_at_20_std": NaN, @@ -53,132 +53,132 @@ "nauc_cv_recall_at_5_diff1": NaN, "nauc_cv_recall_at_5_max": NaN, "nauc_cv_recall_at_5_std": NaN, - "nauc_map_at_1000_diff1": -0.2774518254418231, - "nauc_map_at_1000_max": 0.25032079891265085, - "nauc_map_at_1000_std": 0.45081359384807485, - "nauc_map_at_100_diff1": -0.04907209740285267, - "nauc_map_at_100_max": 0.18686662832088577, - "nauc_map_at_100_std": 0.31839565681325593, - "nauc_map_at_10_diff1": 0.4545859442374117, - "nauc_map_at_10_max": -0.04495045809094941, - "nauc_map_at_10_std": 0.002947268581625076, - "nauc_map_at_1_diff1": 0.7049183369958152, - "nauc_map_at_1_max": -0.2616079702049637, - "nauc_map_at_1_std": -0.3618487407893112, - "nauc_map_at_20_diff1": 0.30890872824690435, - "nauc_map_at_20_max": 0.07581745603833642, - "nauc_map_at_20_std": 0.15307053171724933, - "nauc_map_at_3_diff1": 0.5935344781456238, - "nauc_map_at_3_max": -0.20146382202873225, - "nauc_map_at_3_std": -0.22821424999017603, - "nauc_map_at_5_diff1": 0.5703897004682528, - "nauc_map_at_5_max": -0.16308005302080678, - "nauc_map_at_5_std": -0.1620575175690372, - "nauc_mrr_at_1000_diff1": -0.39652527677737137, - "nauc_mrr_at_1000_max": -0.10590903027877008, - "nauc_mrr_at_1000_std": 0.452714419100977, - "nauc_mrr_at_100_diff1": -0.39652527677737137, - "nauc_mrr_at_100_max": -0.10590903027877008, - "nauc_mrr_at_100_std": 0.452714419100977, - "nauc_mrr_at_10_diff1": -0.39652527677737137, - "nauc_mrr_at_10_max": -0.10590903027877008, - "nauc_mrr_at_10_std": 0.452714419100977, - "nauc_mrr_at_1_diff1": -0.303338001867413, - "nauc_mrr_at_1_max": -0.11951447245564903, - "nauc_mrr_at_1_std": 0.38328664799253065, - "nauc_mrr_at_20_diff1": -0.39652527677737137, - "nauc_mrr_at_20_max": -0.10590903027877008, - "nauc_mrr_at_20_std": 0.452714419100977, - "nauc_mrr_at_3_diff1": -0.39652527677737137, - "nauc_mrr_at_3_max": -0.10590903027877008, - "nauc_mrr_at_3_std": 0.452714419100977, - "nauc_mrr_at_5_diff1": -0.39652527677737137, - "nauc_mrr_at_5_max": -0.10590903027877008, - "nauc_mrr_at_5_std": 0.452714419100977, - "nauc_ndcg_at_1000_diff1": -0.49032575238867226, - "nauc_ndcg_at_1000_max": 0.30827149571860624, - "nauc_ndcg_at_1000_std": 0.5422742306681069, - "nauc_ndcg_at_100_diff1": -0.2712801258824707, - "nauc_ndcg_at_100_max": 0.2469510949945573, - "nauc_ndcg_at_100_std": 0.4367311454283413, - "nauc_ndcg_at_10_diff1": -0.519586478031318, - "nauc_ndcg_at_10_max": 0.35401668386388874, - "nauc_ndcg_at_10_std": 0.5596598049318611, - "nauc_ndcg_at_1_diff1": -0.303338001867413, - "nauc_ndcg_at_1_max": -0.11951447245564903, - "nauc_ndcg_at_1_std": 0.38328664799253065, - "nauc_ndcg_at_20_diff1": -0.4606517361381654, - "nauc_ndcg_at_20_max": 0.38378869676003413, - "nauc_ndcg_at_20_std": 0.474186229263476, - "nauc_ndcg_at_3_diff1": -0.585149663762276, - "nauc_ndcg_at_3_max": 0.217548589813707, - "nauc_ndcg_at_3_std": 0.550167698282911, - "nauc_ndcg_at_5_diff1": -0.5267224367233376, - "nauc_ndcg_at_5_max": 0.2591401820511711, - "nauc_ndcg_at_5_std": 0.5702945720424282, - "nauc_precision_at_1000_diff1": -0.4499561707406712, - "nauc_precision_at_1000_max": 0.15634621233710536, - "nauc_precision_at_1000_std": 0.17441880422894795, - "nauc_precision_at_100_diff1": -0.4968893518170479, - "nauc_precision_at_100_max": 0.19447257428460527, - "nauc_precision_at_100_std": 0.25819757590889797, - "nauc_precision_at_10_diff1": -0.6167580528722199, - "nauc_precision_at_10_max": 0.3956822166454678, - "nauc_precision_at_10_std": 0.587829591941972, - "nauc_precision_at_1_diff1": -0.303338001867413, - "nauc_precision_at_1_max": -0.11951447245564903, - "nauc_precision_at_1_std": 0.38328664799253065, - "nauc_precision_at_20_diff1": -0.6031642916224456, - "nauc_precision_at_20_max": 0.4138166765495704, - "nauc_precision_at_20_std": 0.5161953717797159, - "nauc_precision_at_3_diff1": -0.6298090956856782, - "nauc_precision_at_3_max": 0.24751396267538517, - "nauc_precision_at_3_std": 0.557806449101913, - "nauc_precision_at_5_diff1": -0.5860255772185323, - "nauc_precision_at_5_max": 0.29785990954926145, - "nauc_precision_at_5_std": 0.5944106636660691, - "nauc_recall_at_1000_diff1": -0.12209326635132679, - "nauc_recall_at_1000_max": 0.008543724924431836, - "nauc_recall_at_1000_std": 0.25794589209932395, - "nauc_recall_at_100_diff1": 0.16704714994186, - "nauc_recall_at_100_max": 0.053397233537995845, - "nauc_recall_at_100_std": 0.10488570154536475, - "nauc_recall_at_10_diff1": 0.5240194164262459, - "nauc_recall_at_10_max": -0.03784601294172749, - "nauc_recall_at_10_std": -0.07182820315172336, - "nauc_recall_at_1_diff1": 0.7049183369958152, - "nauc_recall_at_1_max": -0.2616079702049637, - "nauc_recall_at_1_std": -0.3618487407893112, - "nauc_recall_at_20_diff1": 0.40266587722079766, - "nauc_recall_at_20_max": 0.03757413630827814, - "nauc_recall_at_20_std": 0.042791117828726286, - "nauc_recall_at_3_diff1": 0.6179795805128587, - "nauc_recall_at_3_max": -0.1966728040865525, - "nauc_recall_at_3_std": -0.2470220135910208, - "nauc_recall_at_5_diff1": 0.6188095385694616, - "nauc_recall_at_5_max": -0.1447272403059871, - "nauc_recall_at_5_std": -0.20646683990606104, - "ndcg_at_1": 0.94286, - "ndcg_at_10": 0.85179, - "ndcg_at_100": 0.71775, - "ndcg_at_1000": 0.83657, - "ndcg_at_20": 0.82388, - "ndcg_at_3": 0.8936, - "ndcg_at_5": 0.88343, - "precision_at_1": 0.94286, - "precision_at_10": 0.81429, - "precision_at_100": 0.519, - "precision_at_1000": 0.09099, - "precision_at_20": 0.77143, - "precision_at_3": 0.87619, - "precision_at_5": 0.86286, - "recall_at_1": 0.02765, - "recall_at_10": 0.1307, - "recall_at_100": 0.60631, - "recall_at_1000": 0.93409, - "recall_at_20": 0.22272, - "recall_at_3": 0.05209, - "recall_at_5": 0.07985 + "nauc_map_at_1000_diff1": -0.40591219852099064, + "nauc_map_at_1000_max": 0.26094130889238604, + "nauc_map_at_1000_std": 0.09665505089407778, + "nauc_map_at_100_diff1": -0.12855307032951752, + "nauc_map_at_100_max": 0.15818953768862642, + "nauc_map_at_100_std": -0.2617923598733903, + "nauc_map_at_10_diff1": 0.4267948633570509, + "nauc_map_at_10_max": -0.08091999723228532, + "nauc_map_at_10_std": -0.48144180579841844, + "nauc_map_at_1_diff1": 0.7246477183930717, + "nauc_map_at_1_max": -0.2595633654811421, + "nauc_map_at_1_std": -0.473439121428067, + "nauc_map_at_20_diff1": 0.28638200450299545, + "nauc_map_at_20_max": 0.022722341991735208, + "nauc_map_at_20_std": -0.4223664868261228, + "nauc_map_at_3_diff1": 0.599669109747631, + "nauc_map_at_3_max": -0.18294743939161057, + "nauc_map_at_3_std": -0.500291623756148, + "nauc_map_at_5_diff1": 0.5435589386638962, + "nauc_map_at_5_max": -0.1414910554361904, + "nauc_map_at_5_std": -0.5064288331914394, + "nauc_mrr_at_1000_diff1": -0.6615312791783412, + "nauc_mrr_at_1000_max": 0.2754435107376266, + "nauc_mrr_at_1000_std": 0.7117180205415492, + "nauc_mrr_at_100_diff1": -0.6615312791783412, + "nauc_mrr_at_100_max": 0.2754435107376266, + "nauc_mrr_at_100_std": 0.7117180205415492, + "nauc_mrr_at_10_diff1": -0.6615312791783412, + "nauc_mrr_at_10_max": 0.2754435107376266, + "nauc_mrr_at_10_std": 0.7117180205415492, + "nauc_mrr_at_1_diff1": -0.6615312791783367, + "nauc_mrr_at_1_max": 0.2754435107376266, + "nauc_mrr_at_1_std": 0.7117180205415492, + "nauc_mrr_at_20_diff1": -0.6615312791783412, + "nauc_mrr_at_20_max": 0.2754435107376266, + "nauc_mrr_at_20_std": 0.7117180205415492, + "nauc_mrr_at_3_diff1": -0.6615312791783412, + "nauc_mrr_at_3_max": 0.2754435107376266, + "nauc_mrr_at_3_std": 0.7117180205415492, + "nauc_mrr_at_5_diff1": -0.6615312791783412, + "nauc_mrr_at_5_max": 0.2754435107376266, + "nauc_mrr_at_5_std": 0.7117180205415492, + "nauc_ndcg_at_1000_diff1": -0.6596495665503245, + "nauc_ndcg_at_1000_max": 0.35984465648987496, + "nauc_ndcg_at_1000_std": 0.31222271915833927, + "nauc_ndcg_at_100_diff1": -0.42172915492454077, + "nauc_ndcg_at_100_max": 0.1660609412830679, + "nauc_ndcg_at_100_std": 0.03890643898659772, + "nauc_ndcg_at_10_diff1": -0.795470804963755, + "nauc_ndcg_at_10_max": 0.62218300778039, + "nauc_ndcg_at_10_std": 0.41853096184140837, + "nauc_ndcg_at_1_diff1": -0.6615312791783367, + "nauc_ndcg_at_1_max": 0.2754435107376266, + "nauc_ndcg_at_1_std": 0.7117180205415492, + "nauc_ndcg_at_20_diff1": -0.8166236637349379, + "nauc_ndcg_at_20_max": 0.5512278088652715, + "nauc_ndcg_at_20_std": 0.4393916095824728, + "nauc_ndcg_at_3_diff1": -0.9341977827431088, + "nauc_ndcg_at_3_max": 0.6672676444157623, + "nauc_ndcg_at_3_std": 0.39642947946164325, + "nauc_ndcg_at_5_diff1": -0.9441446617385689, + "nauc_ndcg_at_5_max": 0.680574940437725, + "nauc_ndcg_at_5_std": 0.45573053150824994, + "nauc_precision_at_1000_diff1": -0.4449935554869243, + "nauc_precision_at_1000_max": 0.16030281251691458, + "nauc_precision_at_1000_std": 0.5733632485324373, + "nauc_precision_at_100_diff1": -0.5302407180791104, + "nauc_precision_at_100_max": 0.1551769884391311, + "nauc_precision_at_100_std": 0.4184977473765995, + "nauc_precision_at_10_diff1": -0.9456434208407176, + "nauc_precision_at_10_max": 0.5868765770118642, + "nauc_precision_at_10_std": 0.50484511730284, + "nauc_precision_at_1_diff1": -0.6615312791783367, + "nauc_precision_at_1_max": 0.2754435107376266, + "nauc_precision_at_1_std": 0.7117180205415492, + "nauc_precision_at_20_diff1": -0.926778796672683, + "nauc_precision_at_20_max": 0.5280684464415885, + "nauc_precision_at_20_std": 0.5682403269442342, + "nauc_precision_at_3_diff1": -1.0055244319950187, + "nauc_precision_at_3_max": 0.7123404917522567, + "nauc_precision_at_3_std": 0.37741207594148934, + "nauc_precision_at_5_diff1": -1.0297986387840703, + "nauc_precision_at_5_max": 0.6765172620516983, + "nauc_precision_at_5_std": 0.4879394858412396, + "nauc_recall_at_1000_diff1": -0.033581200345988084, + "nauc_recall_at_1000_max": -0.25140608933946496, + "nauc_recall_at_1000_std": 0.10256335172342733, + "nauc_recall_at_100_diff1": 0.18293184373343338, + "nauc_recall_at_100_max": -0.0562892444564702, + "nauc_recall_at_100_std": -0.4925077606843601, + "nauc_recall_at_10_diff1": 0.48352650007752274, + "nauc_recall_at_10_max": -0.13513020550162758, + "nauc_recall_at_10_std": -0.48966303958842, + "nauc_recall_at_1_diff1": 0.7246477183930717, + "nauc_recall_at_1_max": -0.2595633654811421, + "nauc_recall_at_1_std": -0.473439121428067, + "nauc_recall_at_20_diff1": 0.34593183103928304, + "nauc_recall_at_20_max": -0.054714009747642164, + "nauc_recall_at_20_std": -0.45184646419179664, + "nauc_recall_at_3_diff1": 0.6064436631162091, + "nauc_recall_at_3_max": -0.18329351534591007, + "nauc_recall_at_3_std": -0.5063835059924418, + "nauc_recall_at_5_diff1": 0.5499812724878281, + "nauc_recall_at_5_max": -0.14351977259114093, + "nauc_recall_at_5_std": -0.510558046605686, + "ndcg_at_1": 0.97143, + "ndcg_at_10": 0.92622, + "ndcg_at_100": 0.84274, + "ndcg_at_1000": 0.92213, + "ndcg_at_20": 0.91612, + "ndcg_at_3": 0.95074, + "ndcg_at_5": 0.939, + "precision_at_1": 0.97143, + "precision_at_10": 0.89571, + "precision_at_100": 0.62029, + "precision_at_1000": 0.09697, + "precision_at_20": 0.86929, + "precision_at_3": 0.94286, + "precision_at_5": 0.92286, + "recall_at_1": 0.02925, + "recall_at_10": 0.15836, + "recall_at_100": 0.7168, + "recall_at_1000": 0.98961, + "recall_at_20": 0.26733, + "recall_at_3": 0.06435, + "recall_at_5": 0.08842 } ] }, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json index 5b15d0ae51..2fd705aca4 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json @@ -1,184 +1,184 @@ { - "dataset_revision": "fd121b6592fe946616fa85116703b94a4c61fd63", - "evaluation_time": 22.44923186302185, + "dataset_revision": "d3e0adf4e942446c04427511ccce281c86861248", + "evaluation_time": 60.67172384262085, "kg_co2_emissions": null, "mteb_version": "1.12.90", "scores": { "test": [ { - "cv_recall_at_1": 0.04286, - "cv_recall_at_10": 0.41429, - "cv_recall_at_100": 0.97143, + "cv_recall_at_1": 0.41429, + "cv_recall_at_10": 0.68571, + "cv_recall_at_100": 1.0, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.6, - "cv_recall_at_3": 0.17143, - "cv_recall_at_5": 0.27143, + "cv_recall_at_20": 0.77143, + "cv_recall_at_3": 0.51429, + "cv_recall_at_5": 0.6, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.04286, - "map_at_1": 0.00032, - "map_at_10": 0.00319, - "map_at_100": 0.04427, - "map_at_1000": 0.14044, - "map_at_20": 0.00602, - "map_at_3": 0.00096, - "map_at_5": 0.00146, - "mrr_at_1": 0.04285714285714286, - "mrr_at_10": 0.13515873015873012, - "mrr_at_100": 0.15938730320565236, - "mrr_at_1000": 0.15962520658618265, - "mrr_at_20": 0.14815883634769078, - "mrr_at_3": 0.0976190476190476, - "mrr_at_5": 0.11833333333333333, + "main_score": 0.41429, + "map_at_1": 0.00597, + "map_at_10": 0.05664, + "map_at_100": 0.23596, + "map_at_1000": 0.3628, + "map_at_20": 0.10243, + "map_at_3": 0.01731, + "map_at_5": 0.02872, + "mrr_at_1": 0.4142857142857143, + "mrr_at_10": 0.49003968253968255, + "mrr_at_100": 0.502828543277437, + "mrr_at_1000": 0.502828543277437, + "mrr_at_20": 0.49544343814080655, + "mrr_at_3": 0.4619047619047618, + "mrr_at_5": 0.4804761904761904, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.6909430438842217, - "nauc_cv_recall_at_100_max": 0.6790382819794577, - "nauc_cv_recall_at_100_std": 0.4143323996265152, - "nauc_cv_recall_at_10_diff1": 0.2550034187718293, - "nauc_cv_recall_at_10_max": -0.26788387263688956, - "nauc_cv_recall_at_10_std": -0.3843993125496647, - "nauc_cv_recall_at_1_diff1": 0.3989804132009661, - "nauc_cv_recall_at_1_max": -0.12905822377247125, - "nauc_cv_recall_at_1_std": -0.401663536356319, - "nauc_cv_recall_at_20_diff1": 0.12233388457787381, - "nauc_cv_recall_at_20_max": -0.21169910528770744, - "nauc_cv_recall_at_20_std": -0.2534876818046898, - "nauc_cv_recall_at_3_diff1": 0.35100458047053923, - "nauc_cv_recall_at_3_max": -0.1396262752446388, - "nauc_cv_recall_at_3_std": -0.314985425775557, - "nauc_cv_recall_at_5_diff1": 0.35853258558905104, - "nauc_cv_recall_at_5_max": -0.1935699390601907, - "nauc_cv_recall_at_5_std": -0.4389893304719292, - "nauc_map_at_1000_diff1": -0.048721647577313475, - "nauc_map_at_1000_max": 0.02185883585100512, - "nauc_map_at_1000_std": -0.014590760062175558, - "nauc_map_at_100_diff1": 0.20175041133547802, - "nauc_map_at_100_max": -0.2999406736946551, - "nauc_map_at_100_std": -0.21513200915624464, - "nauc_map_at_10_diff1": 0.4340980160657004, - "nauc_map_at_10_max": -0.30137691467001876, - "nauc_map_at_10_std": -0.4166728188377905, - "nauc_map_at_1_diff1": 0.3225400054210336, - "nauc_map_at_1_max": -0.20455804791357643, - "nauc_map_at_1_std": -0.3502703918125217, - "nauc_map_at_20_diff1": 0.4482686549814925, - "nauc_map_at_20_max": -0.33767777626178874, - "nauc_map_at_20_std": -0.39261152175985364, - "nauc_map_at_3_diff1": 0.3906475942798504, - "nauc_map_at_3_max": -0.1421273626640729, - "nauc_map_at_3_std": -0.3164227766935927, - "nauc_map_at_5_diff1": 0.40607500232859556, - "nauc_map_at_5_max": -0.21636348183658996, - "nauc_map_at_5_std": -0.385395422874435, - "nauc_mrr_at_1000_diff1": 0.33380716561285934, - "nauc_mrr_at_1000_max": -0.19720559281432273, - "nauc_mrr_at_1000_std": -0.3739725633585593, - "nauc_mrr_at_100_diff1": 0.3320423802626735, - "nauc_mrr_at_100_max": -0.19584551091041755, - "nauc_mrr_at_100_std": -0.37267220068248286, - "nauc_mrr_at_10_diff1": 0.33810313585026747, - "nauc_mrr_at_10_max": -0.20898927155954197, - "nauc_mrr_at_10_std": -0.39317334419879707, - "nauc_mrr_at_1_diff1": 0.3989804132009661, - "nauc_mrr_at_1_max": -0.12905822377247125, - "nauc_mrr_at_1_std": -0.401663536356319, - "nauc_mrr_at_20_diff1": 0.3270365458084192, - "nauc_mrr_at_20_max": -0.206603016304459, - "nauc_mrr_at_20_std": -0.3731460382346558, - "nauc_mrr_at_3_diff1": 0.3569994314378864, - "nauc_mrr_at_3_max": -0.17174174325462235, - "nauc_mrr_at_3_std": -0.3561501867618589, - "nauc_mrr_at_5_diff1": 0.3571575774665211, - "nauc_mrr_at_5_max": -0.18834228874209263, - "nauc_mrr_at_5_std": -0.40200287994954503, - "nauc_ndcg_at_1000_diff1": -0.0743652757576749, - "nauc_ndcg_at_1000_max": 0.09391843454021216, - "nauc_ndcg_at_1000_std": 0.08425685683454673, - "nauc_ndcg_at_100_diff1": 0.023516077444591114, - "nauc_ndcg_at_100_max": -0.11892313686780007, - "nauc_ndcg_at_100_std": -0.16710001090586352, - "nauc_ndcg_at_10_diff1": 0.3882520721308202, - "nauc_ndcg_at_10_max": -0.2924404653976282, - "nauc_ndcg_at_10_std": -0.4189010025783995, - "nauc_ndcg_at_1_diff1": 0.3989804132009661, - "nauc_ndcg_at_1_max": -0.12905822377247125, - "nauc_ndcg_at_1_std": -0.401663536356319, - "nauc_ndcg_at_20_diff1": 0.3805388505094879, - "nauc_ndcg_at_20_max": -0.30519760291519316, - "nauc_ndcg_at_20_std": -0.3772566200501027, - "nauc_ndcg_at_3_diff1": 0.412201762045363, - "nauc_ndcg_at_3_max": -0.17116948069763432, - "nauc_ndcg_at_3_std": -0.3608861222611007, - "nauc_ndcg_at_5_diff1": 0.35696244084680845, - "nauc_ndcg_at_5_max": -0.25186160853169576, - "nauc_ndcg_at_5_std": -0.400509495428499, - "nauc_precision_at_1000_diff1": -0.22093427992384315, - "nauc_precision_at_1000_max": 0.2196784647696767, - "nauc_precision_at_1000_std": 0.04361074681539203, - "nauc_precision_at_100_diff1": -0.016149306619419775, - "nauc_precision_at_100_max": 0.034799312614065006, - "nauc_precision_at_100_std": -0.13882957890950748, - "nauc_precision_at_10_diff1": 0.38489780804914814, - "nauc_precision_at_10_max": -0.30500607571435273, - "nauc_precision_at_10_std": -0.41532331666205985, - "nauc_precision_at_1_diff1": 0.3989804132009661, - "nauc_precision_at_1_max": -0.12905822377247125, - "nauc_precision_at_1_std": -0.401663536356319, - "nauc_precision_at_20_diff1": 0.3692528574157947, - "nauc_precision_at_20_max": -0.30111599366142605, - "nauc_precision_at_20_std": -0.35015003540161194, - "nauc_precision_at_3_diff1": 0.428985570425912, - "nauc_precision_at_3_max": -0.1563930424168448, - "nauc_precision_at_3_std": -0.3449801647848643, - "nauc_precision_at_5_diff1": 0.34983601813446513, - "nauc_precision_at_5_max": -0.2741390952059421, - "nauc_precision_at_5_std": -0.4086645123951, - "nauc_recall_at_1000_diff1": -0.08562125010420926, - "nauc_recall_at_1000_max": 0.07813883934551541, - "nauc_recall_at_1000_std": 0.23218650354926193, - "nauc_recall_at_100_diff1": -0.01041195221018056, - "nauc_recall_at_100_max": -0.23261397534923414, - "nauc_recall_at_100_std": -0.05979942942574269, - "nauc_recall_at_10_diff1": 0.3361904724346995, - "nauc_recall_at_10_max": -0.28110372866797806, - "nauc_recall_at_10_std": -0.36784022778417585, - "nauc_recall_at_1_diff1": 0.3225400054210336, - "nauc_recall_at_1_max": -0.20455804791357643, - "nauc_recall_at_1_std": -0.3502703918125217, - "nauc_recall_at_20_diff1": 0.31944679632949374, - "nauc_recall_at_20_max": -0.32588194303425905, - "nauc_recall_at_20_std": -0.25546971593163326, - "nauc_recall_at_3_diff1": 0.36197730614062007, - "nauc_recall_at_3_max": -0.07376636861137077, - "nauc_recall_at_3_std": -0.2538555661579059, - "nauc_recall_at_5_diff1": 0.3711994561836934, - "nauc_recall_at_5_max": -0.1684006987187415, - "nauc_recall_at_5_std": -0.373838797717674, - "ndcg_at_1": 0.04286, - "ndcg_at_10": 0.08888, - "ndcg_at_100": 0.20767, - "ndcg_at_1000": 0.46304, - "ndcg_at_20": 0.0984, - "ndcg_at_3": 0.06137, - "ndcg_at_5": 0.07145, - "precision_at_1": 0.04286, - "precision_at_10": 0.10143, - "precision_at_100": 0.202, - "precision_at_1000": 0.09564, - "precision_at_20": 0.10857, - "precision_at_3": 0.06667, - "precision_at_5": 0.08, - "recall_at_1": 0.00032, - "recall_at_10": 0.0087, - "recall_at_100": 0.18965, - "recall_at_1000": 0.67973, - "recall_at_20": 0.01989, - "recall_at_3": 0.00187, - "recall_at_5": 0.00327 + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": 0.13932174607931247, + "nauc_cv_recall_at_10_max": -0.8963390270702233, + "nauc_cv_recall_at_10_std": -0.8396014384177408, + "nauc_cv_recall_at_1_diff1": 0.2060503021455104, + "nauc_cv_recall_at_1_max": -0.5281725278583708, + "nauc_cv_recall_at_1_std": -0.7573041598137233, + "nauc_cv_recall_at_20_diff1": 0.0628172996594046, + "nauc_cv_recall_at_20_max": -0.7762997236681444, + "nauc_cv_recall_at_20_std": -0.7122935543988169, + "nauc_cv_recall_at_3_diff1": 0.35040951939422, + "nauc_cv_recall_at_3_max": -0.6603886570854577, + "nauc_cv_recall_at_3_std": -0.80932236130428, + "nauc_cv_recall_at_5_diff1": 0.345291099520841, + "nauc_cv_recall_at_5_max": -0.6727939617521097, + "nauc_cv_recall_at_5_std": -0.7926260441843702, + "nauc_map_at_1000_diff1": 0.2637485190424332, + "nauc_map_at_1000_max": -0.3214602514655989, + "nauc_map_at_1000_std": -0.3732986822642579, + "nauc_map_at_100_diff1": 0.31321538209822075, + "nauc_map_at_100_max": -0.4434862100043841, + "nauc_map_at_100_std": -0.5807650790699361, + "nauc_map_at_10_diff1": 0.3581935661941152, + "nauc_map_at_10_max": -0.4787594793701487, + "nauc_map_at_10_std": -0.5695584780209748, + "nauc_map_at_1_diff1": 0.35392215263704485, + "nauc_map_at_1_max": -0.47033776664547916, + "nauc_map_at_1_std": -0.581374357285355, + "nauc_map_at_20_diff1": 0.33452504163824237, + "nauc_map_at_20_max": -0.4689043535361316, + "nauc_map_at_20_std": -0.5815981719022251, + "nauc_map_at_3_diff1": 0.3418415128674503, + "nauc_map_at_3_max": -0.48587780235367684, + "nauc_map_at_3_std": -0.5797047003739161, + "nauc_map_at_5_diff1": 0.3476248106096462, + "nauc_map_at_5_max": -0.4810067114931804, + "nauc_map_at_5_std": -0.5755673287021957, + "nauc_mrr_at_1000_diff1": 0.25879474150167453, + "nauc_mrr_at_1000_max": -0.5954028468975692, + "nauc_mrr_at_1000_std": -0.7782793535054984, + "nauc_mrr_at_100_diff1": 0.25879474150167453, + "nauc_mrr_at_100_max": -0.5954028468975692, + "nauc_mrr_at_100_std": -0.7782793535054984, + "nauc_mrr_at_10_diff1": 0.25457966886240146, + "nauc_mrr_at_10_max": -0.6072896469946196, + "nauc_mrr_at_10_std": -0.7834200101118244, + "nauc_mrr_at_1_diff1": 0.2060503021455104, + "nauc_mrr_at_1_max": -0.5281725278583708, + "nauc_mrr_at_1_std": -0.7573041598137233, + "nauc_mrr_at_20_diff1": 0.25380616623495234, + "nauc_mrr_at_20_max": -0.5999473178413075, + "nauc_mrr_at_20_std": -0.778340949953845, + "nauc_mrr_at_3_diff1": 0.276790929933105, + "nauc_mrr_at_3_max": -0.5865299456675351, + "nauc_mrr_at_3_std": -0.7820043340347346, + "nauc_mrr_at_5_diff1": 0.2734971388552989, + "nauc_mrr_at_5_max": -0.5883165176933902, + "nauc_mrr_at_5_std": -0.7787421973872966, + "nauc_ndcg_at_1000_diff1": 0.29912959985776905, + "nauc_ndcg_at_1000_max": -0.28569354071871356, + "nauc_ndcg_at_1000_std": -0.3302978581904054, + "nauc_ndcg_at_100_diff1": 0.33339067606556777, + "nauc_ndcg_at_100_max": -0.3794361926736525, + "nauc_ndcg_at_100_std": -0.4883736139930127, + "nauc_ndcg_at_10_diff1": 0.26397022259588276, + "nauc_ndcg_at_10_max": -0.5817578270842201, + "nauc_ndcg_at_10_std": -0.7307563120549173, + "nauc_ndcg_at_1_diff1": 0.2060503021455104, + "nauc_ndcg_at_1_max": -0.5281725278583708, + "nauc_ndcg_at_1_std": -0.7573041598137233, + "nauc_ndcg_at_20_diff1": 0.2623409031073529, + "nauc_ndcg_at_20_max": -0.5359230413673359, + "nauc_ndcg_at_20_std": -0.6975403451344717, + "nauc_ndcg_at_3_diff1": 0.23259798801223266, + "nauc_ndcg_at_3_max": -0.573906976133018, + "nauc_ndcg_at_3_std": -0.7515043175758601, + "nauc_ndcg_at_5_diff1": 0.2390793878627882, + "nauc_ndcg_at_5_max": -0.5712510242428932, + "nauc_ndcg_at_5_std": -0.7286974701355182, + "nauc_precision_at_1000_diff1": -0.24245498329943332, + "nauc_precision_at_1000_max": 0.3291646825531801, + "nauc_precision_at_1000_std": 0.5255011870667545, + "nauc_precision_at_100_diff1": 0.13623543103107302, + "nauc_precision_at_100_max": -0.07853246653158767, + "nauc_precision_at_100_std": -0.14296193871521895, + "nauc_precision_at_10_diff1": 0.2775400329964219, + "nauc_precision_at_10_max": -0.5859260716532688, + "nauc_precision_at_10_std": -0.7212562014378959, + "nauc_precision_at_1_diff1": 0.2060503021455104, + "nauc_precision_at_1_max": -0.5281725278583708, + "nauc_precision_at_1_std": -0.7573041598137233, + "nauc_precision_at_20_diff1": 0.26665797590030293, + "nauc_precision_at_20_max": -0.5123924265023908, + "nauc_precision_at_20_std": -0.6708786566904752, + "nauc_precision_at_3_diff1": 0.2363702775381167, + "nauc_precision_at_3_max": -0.5803676577092874, + "nauc_precision_at_3_std": -0.7427529614771804, + "nauc_precision_at_5_diff1": 0.2456529314233418, + "nauc_precision_at_5_max": -0.573319088810008, + "nauc_precision_at_5_std": -0.7135127683771583, + "nauc_recall_at_1000_diff1": 0.38214489204648355, + "nauc_recall_at_1000_max": -0.09155063706261089, + "nauc_recall_at_1000_std": -0.022172073157980317, + "nauc_recall_at_100_diff1": 0.3802641372927062, + "nauc_recall_at_100_max": -0.3842566138245613, + "nauc_recall_at_100_std": -0.5385141624854306, + "nauc_recall_at_10_diff1": 0.3783267103081628, + "nauc_recall_at_10_max": -0.488856095520999, + "nauc_recall_at_10_std": -0.58247849954068, + "nauc_recall_at_1_diff1": 0.35392215263704485, + "nauc_recall_at_1_max": -0.47033776664547916, + "nauc_recall_at_1_std": -0.581374357285355, + "nauc_recall_at_20_diff1": 0.3567752100934632, + "nauc_recall_at_20_max": -0.4784863097401327, + "nauc_recall_at_20_std": -0.5895318668296887, + "nauc_recall_at_3_diff1": 0.35482016307799724, + "nauc_recall_at_3_max": -0.492800686077188, + "nauc_recall_at_3_std": -0.5845744958893286, + "nauc_recall_at_5_diff1": 0.36822693817594376, + "nauc_recall_at_5_max": -0.4820944786320667, + "nauc_recall_at_5_std": -0.5821080266388661, + "ndcg_at_1": 0.41429, + "ndcg_at_10": 0.43851, + "ndcg_at_100": 0.44725, + "ndcg_at_1000": 0.67745, + "ndcg_at_20": 0.42956, + "ndcg_at_3": 0.42275, + "ndcg_at_5": 0.43186, + "precision_at_1": 0.41429, + "precision_at_10": 0.44429, + "precision_at_100": 0.36357, + "precision_at_1000": 0.12124, + "precision_at_20": 0.42857, + "precision_at_3": 0.42381, + "precision_at_5": 0.43714, + "recall_at_1": 0.00597, + "recall_at_10": 0.06062, + "recall_at_100": 0.35778, + "recall_at_1000": 0.83615, + "recall_at_20": 0.11218, + "recall_at_3": 0.01773, + "recall_at_5": 0.03016 } ] }, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json index 700a4a2774..3f2c633d86 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json @@ -1,12 +1,12 @@ { - "dataset_revision": "900267b49003a086979e8d52f6942624236bfc34", - "evaluation_time": 23.04814124107361, + "dataset_revision": "3d959815e102785efd628170281f1e65561b03d2", + "evaluation_time": 79.89404535293579, "kg_co2_emissions": null, "mteb_version": "1.12.90", "scores": { "test": [ { - "cv_recall_at_1": 0.98571, + "cv_recall_at_1": 1.0, "cv_recall_at_10": 1.0, "cv_recall_at_100": 1.0, "cv_recall_at_1000": 1.0, @@ -17,21 +17,21 @@ "languages": [ "eng-Latn" ], - "main_score": 0.98571, - "map_at_1": 0.0055, - "map_at_10": 0.0505, - "map_at_100": 0.32128, - "map_at_1000": 0.53905, - "map_at_20": 0.09538, - "map_at_3": 0.01586, - "map_at_5": 0.02626, - "mrr_at_1": 0.9857142857142858, - "mrr_at_10": 0.9928571428571429, - "mrr_at_100": 0.9928571428571429, - "mrr_at_1000": 0.9928571428571429, - "mrr_at_20": 0.9928571428571429, - "mrr_at_3": 0.9928571428571429, - "mrr_at_5": 0.9928571428571429, + "main_score": 1.0, + "map_at_1": 0.00556, + "map_at_10": 0.05371, + "map_at_100": 0.40433, + "map_at_1000": 0.65468, + "map_at_20": 0.10419, + "map_at_3": 0.01659, + "map_at_5": 0.02755, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, @@ -41,9 +41,9 @@ "nauc_cv_recall_at_10_diff1": NaN, "nauc_cv_recall_at_10_max": NaN, "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": 0.8692810457516342, - "nauc_cv_recall_at_1_max": -1.151727357609713, - "nauc_cv_recall_at_1_std": -0.5634920634920669, + "nauc_cv_recall_at_1_diff1": NaN, + "nauc_cv_recall_at_1_max": NaN, + "nauc_cv_recall_at_1_std": NaN, "nauc_cv_recall_at_20_diff1": NaN, "nauc_cv_recall_at_20_max": NaN, "nauc_cv_recall_at_20_std": NaN, @@ -53,132 +53,132 @@ "nauc_cv_recall_at_5_diff1": NaN, "nauc_cv_recall_at_5_max": NaN, "nauc_cv_recall_at_5_std": NaN, - "nauc_map_at_1000_diff1": -0.5202144222378728, - "nauc_map_at_1000_max": 0.11126568280766698, - "nauc_map_at_1000_std": 0.32813155264948524, - "nauc_map_at_100_diff1": -0.17082735069359992, - "nauc_map_at_100_max": -0.07951969633170079, - "nauc_map_at_100_std": 0.13798406208276, - "nauc_map_at_10_diff1": 0.08925396593997569, - "nauc_map_at_10_max": -0.078676032579074, - "nauc_map_at_10_std": 0.13021224342356594, - "nauc_map_at_1_diff1": 0.20034146659034022, - "nauc_map_at_1_max": -0.1488526539306114, - "nauc_map_at_1_std": 0.036837238597286596, - "nauc_map_at_20_diff1": 0.044029095752394776, - "nauc_map_at_20_max": -0.044115438949865464, - "nauc_map_at_20_std": 0.16589359171211718, - "nauc_map_at_3_diff1": 0.12936241106392424, - "nauc_map_at_3_max": -0.11171026414768116, - "nauc_map_at_3_std": 0.09557523635145118, - "nauc_map_at_5_diff1": 0.12090379123610429, - "nauc_map_at_5_max": -0.11656881579212829, - "nauc_map_at_5_std": 0.09921919672363319, - "nauc_mrr_at_1000_diff1": 0.8692810457516355, - "nauc_mrr_at_1000_max": -1.1517273576096931, - "nauc_mrr_at_1000_std": -0.5634920634920526, - "nauc_mrr_at_100_diff1": 0.8692810457516355, - "nauc_mrr_at_100_max": -1.1517273576096931, - "nauc_mrr_at_100_std": -0.5634920634920526, - "nauc_mrr_at_10_diff1": 0.8692810457516355, - "nauc_mrr_at_10_max": -1.1517273576096931, - "nauc_mrr_at_10_std": -0.5634920634920526, - "nauc_mrr_at_1_diff1": 0.8692810457516342, - "nauc_mrr_at_1_max": -1.151727357609713, - "nauc_mrr_at_1_std": -0.5634920634920669, - "nauc_mrr_at_20_diff1": 0.8692810457516355, - "nauc_mrr_at_20_max": -1.1517273576096931, - "nauc_mrr_at_20_std": -0.5634920634920526, - "nauc_mrr_at_3_diff1": 0.8692810457516355, - "nauc_mrr_at_3_max": -1.1517273576096931, - "nauc_mrr_at_3_std": -0.5634920634920526, - "nauc_mrr_at_5_diff1": 0.8692810457516355, - "nauc_mrr_at_5_max": -1.1517273576096931, - "nauc_mrr_at_5_std": -0.5634920634920526, - "nauc_ndcg_at_1000_diff1": -0.44820569134469457, - "nauc_ndcg_at_1000_max": 0.14278328260517373, - "nauc_ndcg_at_1000_std": 0.35909204639356485, - "nauc_ndcg_at_100_diff1": -0.6018888319473394, - "nauc_ndcg_at_100_max": 0.21192336225080224, - "nauc_ndcg_at_100_std": 0.25475321800547673, - "nauc_ndcg_at_10_diff1": -0.5815570895956712, - "nauc_ndcg_at_10_max": 0.2384364105122267, - "nauc_ndcg_at_10_std": 0.47588628797056887, - "nauc_ndcg_at_1_diff1": 0.8692810457516342, - "nauc_ndcg_at_1_max": -1.151727357609713, - "nauc_ndcg_at_1_std": -0.5634920634920669, - "nauc_ndcg_at_20_diff1": -0.5953841029416077, - "nauc_ndcg_at_20_max": 0.32157620990503083, - "nauc_ndcg_at_20_std": 0.47071964995241883, - "nauc_ndcg_at_3_diff1": -0.47333802126140007, - "nauc_ndcg_at_3_max": 0.13407004770022382, - "nauc_ndcg_at_3_std": 0.46679765020950414, - "nauc_ndcg_at_5_diff1": -0.5587389095148461, - "nauc_ndcg_at_5_max": 0.07339891244234566, - "nauc_ndcg_at_5_std": 0.47215237618630984, - "nauc_precision_at_1000_diff1": -0.3950971876695561, - "nauc_precision_at_1000_max": 0.24406161426957987, - "nauc_precision_at_1000_std": 0.12174947607128742, - "nauc_precision_at_100_diff1": -0.5472068296947269, - "nauc_precision_at_100_max": 0.23378022737023205, - "nauc_precision_at_100_std": 0.1829472692928344, - "nauc_precision_at_10_diff1": -0.5950517610712319, - "nauc_precision_at_10_max": 0.28672134365425656, - "nauc_precision_at_10_std": 0.48780862485384807, - "nauc_precision_at_1_diff1": 0.8692810457516342, - "nauc_precision_at_1_max": -1.151727357609713, - "nauc_precision_at_1_std": -0.5634920634920669, - "nauc_precision_at_20_diff1": -0.5947364591012498, - "nauc_precision_at_20_max": 0.35422908868857506, - "nauc_precision_at_20_std": 0.4705389149562903, - "nauc_precision_at_3_diff1": -0.510094916344916, - "nauc_precision_at_3_max": 0.22297297297297156, - "nauc_precision_at_3_std": 0.5162886100386108, - "nauc_precision_at_5_diff1": -0.6028283796740131, - "nauc_precision_at_5_max": 0.11490891658676922, - "nauc_precision_at_5_std": 0.504458293384468, - "nauc_recall_at_1000_diff1": -0.2460203893111549, - "nauc_recall_at_1000_max": 0.07681666757366377, - "nauc_recall_at_1000_std": 0.29402261311504047, - "nauc_recall_at_100_diff1": -0.10390008545301123, - "nauc_recall_at_100_max": -0.09447732649357599, - "nauc_recall_at_100_std": 0.0928830407002215, - "nauc_recall_at_10_diff1": 0.10590480631127752, - "nauc_recall_at_10_max": -0.08162621409653509, - "nauc_recall_at_10_std": 0.11847839880663995, - "nauc_recall_at_1_diff1": 0.20034146659034022, - "nauc_recall_at_1_max": -0.1488526539306114, - "nauc_recall_at_1_std": 0.036837238597286596, - "nauc_recall_at_20_diff1": 0.06586007946887822, - "nauc_recall_at_20_max": -0.05281307817884481, - "nauc_recall_at_20_std": 0.14970188120798883, - "nauc_recall_at_3_diff1": 0.13126830970729877, - "nauc_recall_at_3_max": -0.10596900293507791, - "nauc_recall_at_3_std": 0.09476793236811286, - "nauc_recall_at_5_diff1": 0.12983221018552543, - "nauc_recall_at_5_max": -0.11532911944953175, - "nauc_recall_at_5_std": 0.09212571294283788, - "ndcg_at_1": 0.98571, - "ndcg_at_10": 0.92841, - "ndcg_at_100": 0.77588, - "ndcg_at_1000": 0.7685, - "ndcg_at_20": 0.89894, - "ndcg_at_3": 0.95291, - "ndcg_at_5": 0.94991, - "precision_at_1": 0.98571, - "precision_at_10": 0.91571, - "precision_at_100": 0.721, - "precision_at_1000": 0.18663, - "precision_at_20": 0.88, - "precision_at_3": 0.94286, - "precision_at_5": 0.94286, - "recall_at_1": 0.0055, - "recall_at_10": 0.05165, - "recall_at_100": 0.36172, - "recall_at_1000": 0.79065, - "recall_at_20": 0.0987, - "recall_at_3": 0.01595, - "recall_at_5": 0.0266 + "nauc_map_at_1000_diff1": -0.37033768465220357, + "nauc_map_at_1000_max": 0.2394188102952743, + "nauc_map_at_1000_std": 0.35189302913259624, + "nauc_map_at_100_diff1": -0.0008350440689574251, + "nauc_map_at_100_max": 0.06810001616925912, + "nauc_map_at_100_std": -0.08292235454963914, + "nauc_map_at_10_diff1": 0.137914576743156, + "nauc_map_at_10_max": -0.09262735632646613, + "nauc_map_at_10_std": -0.19844704585800624, + "nauc_map_at_1_diff1": 0.18523999970603441, + "nauc_map_at_1_max": -0.1265328999786591, + "nauc_map_at_1_std": -0.230231416885371, + "nauc_map_at_20_diff1": 0.08800700279626224, + "nauc_map_at_20_max": -0.05754868196581078, + "nauc_map_at_20_std": -0.1590299889904529, + "nauc_map_at_3_diff1": 0.1740453429583345, + "nauc_map_at_3_max": -0.11523868645571757, + "nauc_map_at_3_std": -0.22138850054902404, + "nauc_map_at_5_diff1": 0.1634612763199569, + "nauc_map_at_5_max": -0.11473486161026644, + "nauc_map_at_5_std": -0.21287472489660672, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": -0.22205760084759807, + "nauc_ndcg_at_1000_max": 0.10796195890373474, + "nauc_ndcg_at_1000_std": 0.26231183916047035, + "nauc_ndcg_at_100_diff1": -0.6398984813885491, + "nauc_ndcg_at_100_max": 0.4842652819900039, + "nauc_ndcg_at_100_std": 0.5107149500380104, + "nauc_ndcg_at_10_diff1": -0.9371088266265549, + "nauc_ndcg_at_10_max": 0.6084341828001775, + "nauc_ndcg_at_10_std": 0.5430117586762949, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": -0.9268541652920639, + "nauc_ndcg_at_20_max": 0.597237167140367, + "nauc_ndcg_at_20_std": 0.5649258862662437, + "nauc_ndcg_at_3_diff1": -0.8999624476329307, + "nauc_ndcg_at_3_max": 1.0, + "nauc_ndcg_at_3_std": 0.6408995549373118, + "nauc_ndcg_at_5_diff1": -1.1761404982018615, + "nauc_ndcg_at_5_max": 0.8247853976989885, + "nauc_ndcg_at_5_std": 0.7965821461413944, + "nauc_precision_at_1000_diff1": -0.31783317612679535, + "nauc_precision_at_1000_max": 0.1930712065318718, + "nauc_precision_at_1000_std": 0.4208950298526623, + "nauc_precision_at_100_diff1": -0.539592973425393, + "nauc_precision_at_100_max": 0.4460055144887294, + "nauc_precision_at_100_std": 0.4485043771568464, + "nauc_precision_at_10_diff1": -0.9108685686074025, + "nauc_precision_at_10_max": 0.5802094720880185, + "nauc_precision_at_10_std": 0.5333427372430083, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": -0.9101954663166522, + "nauc_precision_at_20_max": 0.5867536982510865, + "nauc_precision_at_20_std": 0.5645109794600713, + "nauc_precision_at_3_diff1": -0.8266417678182514, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 0.6095549330843374, + "nauc_precision_at_5_diff1": -1.1852629940865032, + "nauc_precision_at_5_max": 0.8047774665421796, + "nauc_precision_at_5_std": 0.8047774665421796, + "nauc_recall_at_1000_diff1": -0.01696359441028296, + "nauc_recall_at_1000_max": -0.06214643338578423, + "nauc_recall_at_1000_std": 0.12422436451417722, + "nauc_recall_at_100_diff1": 0.05923389523513676, + "nauc_recall_at_100_max": 0.023919554920948157, + "nauc_recall_at_100_std": -0.13407391685199146, + "nauc_recall_at_10_diff1": 0.14007744935778899, + "nauc_recall_at_10_max": -0.09598247717834782, + "nauc_recall_at_10_std": -0.19995201976434931, + "nauc_recall_at_1_diff1": 0.18523999970603441, + "nauc_recall_at_1_max": -0.1265328999786591, + "nauc_recall_at_1_std": -0.230231416885371, + "nauc_recall_at_20_diff1": 0.09732767803178609, + "nauc_recall_at_20_max": -0.06812531189724261, + "nauc_recall_at_20_std": -0.16830905580000974, + "nauc_recall_at_3_diff1": 0.1740453429583345, + "nauc_recall_at_3_max": -0.11523868645571757, + "nauc_recall_at_3_std": -0.22138850054902404, + "nauc_recall_at_5_diff1": 0.16407715576816015, + "nauc_recall_at_5_max": -0.11553420065887736, + "nauc_recall_at_5_std": -0.2140734184274355, + "ndcg_at_1": 1.0, + "ndcg_at_10": 0.97012, + "ndcg_at_100": 0.87092, + "ndcg_at_1000": 0.84355, + "ndcg_at_20": 0.95286, + "ndcg_at_3": 0.98907, + "ndcg_at_5": 0.98605, + "precision_at_1": 1.0, + "precision_at_10": 0.96143, + "precision_at_100": 0.821, + "precision_at_1000": 0.20586, + "precision_at_20": 0.94143, + "precision_at_3": 0.98571, + "precision_at_5": 0.98286, + "recall_at_1": 0.00556, + "recall_at_10": 0.05391, + "recall_at_100": 0.42614, + "recall_at_1000": 0.86053, + "recall_at_20": 0.10535, + "recall_at_3": 0.01659, + "recall_at_5": 0.02758 } ] }, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json index cdfeb8c90d..0b09a17fae 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json @@ -1 +1 @@ -{"name": "openai/clip-vit-base-patch32", "revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "CLIPModelWrapper"} \ No newline at end of file +{"name": "openai/clip-vit-base-patch32", "revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "CLIPModelWrapper"} \ No newline at end of file From 2b56317b73047d4a242f9b86f44f08c23b094011 Mon Sep 17 00:00:00 2001 From: Niklas Muennighoff Date: Tue, 5 Nov 2024 07:32:19 -0800 Subject: [PATCH 096/154] [mieb] fixes (#1390) * Fix torch no grad * simplify * make lint --------- Co-authored-by: Isaac Chung --- mteb/models/jina_clip.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/mteb/models/jina_clip.py b/mteb/models/jina_clip.py index be8a4b0715..01563a0649 100644 --- a/mteb/models/jina_clip.py +++ b/mteb/models/jina_clip.py @@ -25,14 +25,22 @@ def __init__( self.device ) - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + batch_size: int = 32, + convert_to_numpy=False, + convert_to_tensor=True, + ): all_text_embeddings = [] with torch.no_grad(): for i in tqdm(range(0, len(texts), batch_size)): batch_texts = texts[i : i + batch_size] text_outputs = self.model.encode_text( - batch_texts, convert_to_numpy=False, convert_to_tensor=True + batch_texts, + convert_to_numpy=convert_to_numpy, + convert_to_tensor=convert_to_tensor, ) all_text_embeddings.append(text_outputs.cpu()) @@ -40,7 +48,11 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): return all_text_embeddings def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + batch_size: int = 32, + convert_to_numpy=False, + convert_to_tensor=True, ): all_image_embeddings = [] @@ -51,8 +63,8 @@ def get_image_embeddings( for batch in tqdm(images): image_outputs = self.model.encode_image( [F.to_pil_image(b.to("cpu")) for b in batch], - convert_to_numpy=False, - convert_to_tensor=True, + convert_to_numpy=convert_to_numpy, + convert_to_tensor=convert_to_tensor, ) all_image_embeddings.append(image_outputs.cpu()) else: @@ -90,7 +102,7 @@ def get_fused_embeddings( image_embeddings = None if texts is not None: - text_embeddings = self.encode_text( + text_embeddings = self.get_text_embeddings( texts, batch_size=batch_size, convert_to_numpy=False, @@ -98,7 +110,7 @@ def get_fused_embeddings( ) if images is not None: - image_embeddings = self.encode_image( + image_embeddings = self.get_image_embeddings( images, batch_size=batch_size, convert_to_numpy=False, From 286232304d90e74eeba78802bc7d8ce7f7ab4d92 Mon Sep 17 00:00:00 2001 From: Imene Kerboua <33312980+imenelydiaker@users.noreply.github.com> Date: Tue, 5 Nov 2024 22:45:09 +0100 Subject: [PATCH 097/154] [MIEB] Remove non-existent method for blip (#1394) remove non-existent method for blip --- mteb/models/blip_models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mteb/models/blip_models.py b/mteb/models/blip_models.py index dff6014246..7dff913fd4 100644 --- a/mteb/models/blip_models.py +++ b/mteb/models/blip_models.py @@ -83,7 +83,6 @@ def get_image_embeddings( images=batch_images, return_tensors="pt", padding=True ) inputs = {k: v.to(self.device) for k, v in inputs.items()} - image_outputs = self.model.get_image_features(**inputs) image_outputs = self.model.vision_model(**inputs) image_outputs = image_outputs[0] image_outputs = normalize( From 8a8b8b7aa696b4414e2cf374cc05da227083d53b Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Wed, 6 Nov 2024 17:23:37 +0800 Subject: [PATCH 098/154] [mieb] fix ALIGN; update Winoground revision id; update run script (#1391) * fix align & winoground * lint * Convert task category to i2i for tasks that only calls image encode * update categories should include img cls, clustering, and multi label clf * no op * no op * make lint --------- Co-authored-by: Isaac Chung --- mteb/models/align_models.py | 6 ++++++ mteb/tasks/Image/Clustering/eng/CIFAR.py | 2 +- mteb/tasks/Image/Clustering/eng/ImageNet.py | 2 +- mteb/tasks/Image/Clustering/eng/TinyImageNet.py | 2 +- .../Image/ImageClassification/eng/BirdsnapClassification.py | 2 +- mteb/tasks/Image/ImageClassification/eng/CIFAR.py | 2 +- .../ImageClassification/eng/Caltech101Classification.py | 2 +- .../ImageClassification/eng/Country211Classification.py | 2 +- .../Image/ImageClassification/eng/DTDClassification.py | 2 +- .../Image/ImageClassification/eng/EuroSATClassification.py | 2 +- .../Image/ImageClassification/eng/FER2013Classification.py | 2 +- .../ImageClassification/eng/FGVCAircraftClassification.py | 2 +- .../Image/ImageClassification/eng/Food101Classification.py | 2 +- .../Image/ImageClassification/eng/GTSRBClassification.py | 2 +- mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py | 2 +- .../Image/ImageClassification/eng/MNISTClassification.py | 2 +- .../ImageClassification/eng/OxfordFlowersClassification.py | 2 +- .../ImageClassification/eng/OxfordPetsClassification.py | 2 +- .../ImageClassification/eng/PatchCamelyonClassification.py | 2 +- .../Image/ImageClassification/eng/RESISC45Classification.py | 2 +- .../Image/ImageClassification/eng/STL10Classification.py | 2 +- .../Image/ImageClassification/eng/SUN397Classification.py | 2 +- .../ImageClassification/eng/StanfordCarsClassification.py | 2 +- .../Image/ImageClassification/eng/UCF101Classification.py | 2 +- .../ImageMultilabelClassification/eng/PascalVOC2007.py | 2 +- mteb/tasks/Image/ImageTextPairClassification/Winoground.py | 3 +-- 26 files changed, 31 insertions(+), 26 deletions(-) diff --git a/mteb/models/align_models.py b/mteb/models/align_models.py index a47e199209..e8f55c64ba 100644 --- a/mteb/models/align_models.py +++ b/mteb/models/align_models.py @@ -57,6 +57,12 @@ def get_image_embeddings( with torch.no_grad(): for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] + batch_images = [ + img.convert("RGB") + if isinstance(img, Image.Image) and img.mode != "RGB" + else img + for img in batch_images + ] inputs = self.processor( images=batch_images, return_tensors="pt", padding=True ) diff --git a/mteb/tasks/Image/Clustering/eng/CIFAR.py b/mteb/tasks/Image/Clustering/eng/CIFAR.py index 2bde390661..f9d08b684a 100644 --- a/mteb/tasks/Image/Clustering/eng/CIFAR.py +++ b/mteb/tasks/Image/Clustering/eng/CIFAR.py @@ -14,7 +14,7 @@ class CIFAR10Clustering(AbsTaskImageClustering): "revision": "0b2714987fa478483af9968de7c934580d0bb9a2", }, type="ImageClustering", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="nmi", diff --git a/mteb/tasks/Image/Clustering/eng/ImageNet.py b/mteb/tasks/Image/Clustering/eng/ImageNet.py index a2377e2334..aa0ab5720b 100644 --- a/mteb/tasks/Image/Clustering/eng/ImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/ImageNet.py @@ -14,7 +14,7 @@ class ImageNetDog15Clustering(AbsTaskImageClustering): "revision": "bfb6ad3b2109d26c9daddf14f98d315daa35ee72", }, type="ImageClustering", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="nmi", diff --git a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py index a5760b6b9c..d49ebbfde6 100644 --- a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py @@ -14,7 +14,7 @@ class TinyImageNet(AbsTaskImageClustering): "revision": "5a77092c28e51558c5586e9c5eb71a7e17a5e43f", }, type="ImageClustering", - category="s2s", + category="i2i", eval_splits=["valid"], eval_langs=["eng-Latn"], main_score="nmi", diff --git a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py index 2b08169aae..2e11094b09 100644 --- a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py @@ -14,7 +14,7 @@ class BirdsnapClassification(AbsTaskImageClassification): "revision": "fd23015508be94f0b5b59d61630e4ea2536509e4", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py index 5b4d096783..abed2ad617 100644 --- a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py +++ b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py @@ -14,7 +14,7 @@ class CIFAR10Classification(AbsTaskImageClassification): "revision": "0b2714987fa478483af9968de7c934580d0bb9a2", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py index 76843f50ba..30112cdf1d 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py @@ -16,7 +16,7 @@ class Caltech101Classification(AbsTaskImageClassification): "trust_remote_code": True, }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py index 14427cd530..b73f895595 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py @@ -14,7 +14,7 @@ class Country211Classification(AbsTaskImageClassification): "revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py index 48a6ca5243..eb7360f088 100644 --- a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py @@ -14,7 +14,7 @@ class DTDClassification(AbsTaskImageClassification): "revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py index 588cce89aa..5cac334c3d 100644 --- a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py @@ -14,7 +14,7 @@ class EuroSATClassification(AbsTaskImageClassification): "revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py index 81c2fc5857..074e92529a 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py @@ -14,7 +14,7 @@ class FER2013Classification(AbsTaskImageClassification): "revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py index 2971faf863..8b2a41bd50 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py @@ -15,7 +15,7 @@ class FGVCAircraftClassification(AbsTaskImageClassification): "trust_remote_code": True, }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py index a0dee80ad0..1bbe8e106b 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py @@ -14,7 +14,7 @@ class Food101Classification(AbsTaskImageClassification): "revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["validation"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py index 4a77c47e98..3244b47dc8 100644 --- a/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py @@ -14,7 +14,7 @@ class GTSRBClassification(AbsTaskImageClassification): "revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py index ff91119015..bed879d282 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py +++ b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py @@ -14,7 +14,7 @@ class Imagenet1kClassification(AbsTaskImageClassification): "revision": "b24c7a5a3ef12df09089055d1795e2ce7c7e7397", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py index 7e4b81f3f6..8230938a14 100644 --- a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py @@ -14,7 +14,7 @@ class MNISTClassification(AbsTaskImageClassification): "revision": "77f3279092a1c1579b2250db8eafed0ad422088c", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py index 316107534f..7f607d6aac 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -14,7 +14,7 @@ class OxfordFlowersClassification(AbsTaskImageClassification): "revision": "a37b1891609c0376fa81eced756e7863e1bd873b", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py index 39620326ab..28a2357d5c 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py @@ -14,7 +14,7 @@ class OxfordPetsClassification(AbsTaskImageClassification): "revision": "557b480fae8d69247be74d9503b378a09425096f", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py index e2518dc221..27508c8c17 100644 --- a/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py @@ -14,7 +14,7 @@ class PatchCamelyonClassification(AbsTaskImageClassification): "revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py index b14d31cdc0..7fa7cd5d3d 100644 --- a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py @@ -14,7 +14,7 @@ class RESISC45Classification(AbsTaskImageClassification): "revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py index 7acd6bb0eb..11ea833477 100644 --- a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py @@ -14,7 +14,7 @@ class STL10Classification(AbsTaskImageClassification): "revision": "49ae7f94508f7feae62baf836db284306eab0b0f", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py index 4d0f987564..b4b5a8b931 100644 --- a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py @@ -14,7 +14,7 @@ class SUN397Classification(AbsTaskImageClassification): "revision": "7e6af6a2499ad708618be868e1471eac0aca1168", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py index f9836e4b05..74fa5e92b8 100644 --- a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py @@ -14,7 +14,7 @@ class StanfordCarsClassification(AbsTaskImageClassification): "revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py index 68d2cb74b7..dc4021b490 100644 --- a/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py @@ -18,7 +18,7 @@ class UCF101Classification(AbsTaskImageClassification): "revision": "1098eed48f2929443f47c39f3b5c814e16369c11", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index bd5d0aad6d..4ceae17ff9 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -18,7 +18,7 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): "trust_remote_code": True, }, type="ImageMultilabelClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py index 0b8a8bedd7..6169182286 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py +++ b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py @@ -16,8 +16,7 @@ class Winoground(AbsTaskImageTextPairClassification): reference="https://openaccess.thecvf.com/content/CVPR2022/html/Thrush_Winoground_Probing_Vision_and_Language_Models_for_Visio-Linguistic_Compositionality_CVPR_2022_paper", dataset={ "path": "facebook/winoground", - "revision": "521ec2ba6f9a5d7380f7cca5a7b44aea5c1d677c", - "trust_remote_code": True, + "revision": "b400e173549071916ad1b3d449293bc8d8b4b763", }, type="ImageTextPairClassification", category="i2t", From 01b7f287c89f61cfe1ca83e4a3533072c6cc9d41 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Thu, 7 Nov 2024 12:57:54 +0200 Subject: [PATCH 099/154] [mieb] Fix open clip for cv bench count (#1397) fix shape mismatch --- mteb/models/openclip_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/models/openclip_models.py b/mteb/models/openclip_models.py index ef7589cf8f..068789267c 100644 --- a/mteb/models/openclip_models.py +++ b/mteb/models/openclip_models.py @@ -77,7 +77,7 @@ def get_image_embeddings( for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] inputs = torch.vstack( - [self.img_preprocess(b) for b in batch_images] + [self.img_preprocess(b).unsqueeze(0) for b in batch_images] ) image_outputs = self.model.encode_image(inputs.to(self.device)) all_image_embeddings.append(image_outputs.cpu()) From cdb92c659bb7d3d0402c88c9e9c69f2d71b221e1 Mon Sep 17 00:00:00 2001 From: Jamie-Stirling <36764530+Jamie-Stirling@users.noreply.github.com> Date: Thu, 7 Nov 2024 17:45:21 +0000 Subject: [PATCH 100/154] [mieb] Update subtasks of BLINKIT2TMultiChoice and BLINKIT2IMultiChoice (#1403) * wip: start adding BLIP models * add other blip variants * wip: add blip2_models.py * make lint * wip: implement blip2 wrapper * feat: add blip2 models, still mismatched names * fix: remove projections from image and text embeddings * make lint * wip: add coco BLIP2 * fix: BLIP2 better zero-shot classification without text_proj and vision_proj * tidy blip2 * add imagenet-dog-15 dataset * tidy and lint * remove unused import * add cluster_accuracy, ari and nmi to Image.ClusteringEvaluator * add imagenet-10 clustering task * add SOPI2IRetrieval * add results forclip on ImageNet10Clustering and ImageNetDog15Clustering * add SOPI2IRetrieval results for clip 32 * add results for clip vit 32/SOPI2IRetrieval * resolve conflict * add RP2kI2IRetrieval dataset * add RP2kI2IRetrieval results with clip-vit-base-patch32 * update image retrieval __init__.py * fix ImageTextPair dataloading for large datasets; more compositionality evaluation datasets * add RP2kI2IRetrieval and METI2IRetrieval * add METI2IRetreival * add SOP results * make lign * new revision for METI2IRetrieval * make lint * reset corpus chunk size * remove wrong classification import * add Flickr30k T2I and I2T * add Flickr30k T2I retriebal * reduced-size MET revision * fix: add Flickr30k T2I * make lint * add two landmark datasets and results * add Sketchy i2i retrieval * add task metadata * add BLINKIT2IRetrieval dataset * add BLINKIT2TRetrieval * add ImageCoDeT2IRetrieval * make lint * add vizwiz retrieval and results * fix vizwiz duplicate texts * add new vizwiz results * add VQA2 results * add GLD v2 I2T retrieval * add gld v2 i2i retrieval * make lint * add AbsTaskAny2AnyMultiChoice * make lint * remove GLDv2I2IRetrieval * exclude AbsTaskAny2AnyMultiChoice from test_load_data * fix e5v&vista * remove duplicate corpus entries from BLINKIT2TRetreival dataset * task type fix for running tasks * update BLINKIT2T metadata * fix wrong meta * run mieb script * split ROxford, RParis into easy, medium and hard * make lint * add BLINK as multi choice tasks * fix: license metadata in wrong format * remove null examples from corpus of ROxford and RParis * fix: add/remove subtasks from BLINKIT2IMultiChoice and BLINKIT2TMultiChoice * update blink metadata * add updated BLINK results --------- Co-authored-by: gowitheflow-1998 --- .../eng/BLINKIT2IMultiChoice.py | 8 ++--- .../eng/BLINKIT2TMultiChoice.py | 8 ++--- .../BLINKIT2IMultiChoice.json | 36 +++++++++---------- .../BLINKIT2TMultiChoice.json | 36 +++++++++---------- 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py index d600eaa4f2..98b0a0120b 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py @@ -11,7 +11,7 @@ class BLINKIT2IMultiChoice(AbsTaskAny2AnyMultiChoice): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2i-multi", - "revision": "780ade70cd769e586502a61dda903e525f945a45", + "revision": "b7b46b72d1ed1fa44d25e2b9c4726afab4a7ce53", "trust_remote_code": True, }, type="Any2AnyMultiChoice", @@ -35,13 +35,13 @@ class BLINKIT2IMultiChoice(AbsTaskAny2AnyMultiChoice): } """, descriptive_stats={ - "n_samples": {"test": 402}, + "n_samples": {"test": 534}, "avg_character_length": { "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 804, - "num_queries": 402, + "num_documents": 1200, + "num_queries": 534, "average_relevant_docs_per_query": 1, } }, diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py index fe37216de0..60f42b8b05 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py @@ -11,7 +11,7 @@ class BLINKIT2TMultiChoice(AbsTaskAny2AnyMultiChoice): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2t-multi", - "revision": "b6e18eba186cada040ddb72e8e3cb92edd7ca5e9", + "revision": "ae713b03ae68e343f16c3bcdbd1b1ee760975d55", }, type="Any2AnyMultiChoice", category="it2t", @@ -34,13 +34,13 @@ class BLINKIT2TMultiChoice(AbsTaskAny2AnyMultiChoice): } """, descriptive_stats={ - "n_samples": {"test": 1073}, + "n_samples": {"test": 923}, "avg_character_length": { "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 26, - "num_queries": 1073, + "num_documents": 24, + "num_queries": 923, "average_relevant_docs_per_query": 1, } }, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json index 648d3aa59e..6b300b8219 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json @@ -1,31 +1,31 @@ { - "dataset_revision": "780ade70cd769e586502a61dda903e525f945a45", - "evaluation_time": 56.62301731109619, + "dataset_revision": "b7b46b72d1ed1fa44d25e2b9c4726afab4a7ce53", + "evaluation_time": 30.517717123031616, "kg_co2_emissions": null, "mteb_version": "1.12.90", "scores": { "test": [ { - "accuracy": 0.70149, + "accuracy": 0.6161, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.70149, - "mrr_at_1": 0.7014925373134329, - "mrr_at_10": 0.8507462686567164, - "mrr_at_100": 0.8507462686567164, - "mrr_at_1000": 0.8507462686567164, - "mrr_at_20": 0.8507462686567164, - "mrr_at_3": 0.8507462686567164, - "mrr_at_5": 0.8507462686567164, - "ndcg_at_1": 0.70149, - "ndcg_at_10": 0.88983, - "ndcg_at_100": 0.88983, - "ndcg_at_1000": 0.88983, - "ndcg_at_20": 0.88983, - "ndcg_at_3": 0.88983, - "ndcg_at_5": 0.88983 + "main_score": 0.6161, + "mrr_at_1": 0.6161048689138576, + "mrr_at_10": 0.7762172284644191, + "mrr_at_100": 0.7762172284644191, + "mrr_at_1000": 0.7762172284644191, + "mrr_at_20": 0.7762172284644191, + "mrr_at_3": 0.7762172284644191, + "mrr_at_5": 0.7762172284644191, + "ndcg_at_1": 0.6161, + "ndcg_at_10": 0.81993, + "ndcg_at_100": 0.81993, + "ndcg_at_1000": 0.81993, + "ndcg_at_20": 0.81993, + "ndcg_at_3": 0.81993, + "ndcg_at_5": 0.81993 } ] }, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json index bda6cd2cb9..55aca4a70e 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json @@ -1,31 +1,31 @@ { - "dataset_revision": "b6e18eba186cada040ddb72e8e3cb92edd7ca5e9", - "evaluation_time": 43.71325731277466, + "dataset_revision": "ae713b03ae68e343f16c3bcdbd1b1ee760975d55", + "evaluation_time": 10.223464965820312, "kg_co2_emissions": null, "mteb_version": "1.12.90", "scores": { "test": [ { - "accuracy": 0.38397, + "accuracy": 0.36945, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.38397, - "mrr_at_1": 0.38397017707362535, - "mrr_at_10": 0.650512581547066, - "mrr_at_100": 0.650512581547066, - "mrr_at_1000": 0.650512581547066, - "mrr_at_20": 0.650512581547066, - "mrr_at_3": 0.6295433364398889, - "mrr_at_5": 0.650512581547066, - "ndcg_at_1": 0.38397, - "ndcg_at_10": 0.73974, - "ndcg_at_100": 0.73974, - "ndcg_at_1000": 0.73974, - "ndcg_at_20": 0.73974, - "ndcg_at_3": 0.70361, - "ndcg_at_5": 0.73974 + "main_score": 0.36945, + "mrr_at_1": 0.3694474539544962, + "mrr_at_10": 0.6365113759479949, + "mrr_at_100": 0.6365113759479949, + "mrr_at_1000": 0.6365113759479949, + "mrr_at_20": 0.6365113759479949, + "mrr_at_3": 0.61213434452871, + "mrr_at_5": 0.6365113759479949, + "ndcg_at_1": 0.36945, + "ndcg_at_10": 0.72903, + "ndcg_at_100": 0.72903, + "ndcg_at_1000": 0.72903, + "ndcg_at_20": 0.72903, + "ndcg_at_3": 0.68704, + "ndcg_at_5": 0.72903 } ] }, From a06227e51f99b20e2ea352e0af994968a1f0980a Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sun, 10 Nov 2024 04:01:29 +0200 Subject: [PATCH 101/154] [mieb] Fix EVA CLIP for CV Bench (#1414) * unsqueeze after preprocess * make lint --- mteb/models/evaclip_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/models/evaclip_models.py b/mteb/models/evaclip_models.py index 015c965c07..2a98277d08 100644 --- a/mteb/models/evaclip_models.py +++ b/mteb/models/evaclip_models.py @@ -89,7 +89,7 @@ def get_image_embeddings( for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] inputs = torch.vstack( - [self.img_preprocess(b) for b in batch_images] + [self.img_preprocess(b).unsqueeze(0) for b in batch_images] ) image_outputs = self.model.encode_image(inputs.to(self.device)) all_image_embeddings.append(image_outputs.cpu()) From f7578921508b7e6f2336fdee476977d6e3054e5a Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sun, 10 Nov 2024 04:01:58 +0200 Subject: [PATCH 102/154] [mieb] Add calculate probs for vlm2vec (#1418) * add method * make lint --- mteb/models/vlm2vec_models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mteb/models/vlm2vec_models.py b/mteb/models/vlm2vec_models.py index e9c8653f74..8a1deab6bd 100644 --- a/mteb/models/vlm2vec_models.py +++ b/mteb/models/vlm2vec_models.py @@ -226,6 +226,15 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): all_text_embeddings = torch.cat(all_text_embeddings, dim=0) return all_text_embeddings + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + def get_fused_embeddings( self, texts: list[str] = None, From f60465a558eabfe83b95baf10533db35eaa62487 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Sun, 10 Nov 2024 20:07:51 +0800 Subject: [PATCH 103/154] [mieb] Fix siglip bug & add retrieval datasets (#1424) * fix siglip * add edis&gld-v2 i2i * results * siglip updated results * fix siglip non-dataloader tasks --- mteb/models/siglip_models.py | 18 +- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 2 + .../Any2AnyRetrieval/eng/EDIST2ITRetrieval.py | 48 +++ .../Any2AnyRetrieval/eng/GLDv2I2IRetrieval.py | 50 +++ .../GLDv2I2IRetrieval.json | 186 ++++++++++ .../CIFAR10.json | 48 +++ .../CIFAR100.json | 48 +++ .../ImageNet10Clustering.json | 22 ++ .../MNIST.json | 48 +++ .../MNISTZeroShot.json | 19 + .../CIFAR10.json | 48 +++ .../CIFAR100.json | 48 +++ .../MNIST.json | 48 +++ .../MNISTZeroShot.json | 19 + .../ImageNet10Clustering.json | 22 ++ .../VidoreArxivQARetrieval.json | 316 ++++++++-------- .../VidoreDocVQARetrieval.json | 316 ++++++++-------- .../VidoreInfoVQARetrieval.json | 316 ++++++++-------- .../VidoreShiftProjectRetrieval.json | 2 +- .../VidoreSyntheticDocQAAIRetrieval.json | 316 ++++++++-------- .../VidoreSyntheticDocQAEnergyRetrieval.json | 316 ++++++++-------- ...theticDocQAGovernmentReportsRetrieval.json | 316 ++++++++-------- ...heticDocQAHealthcareIndustryRetrieval.json | 316 ++++++++-------- .../VidoreTabfquadRetrieval.json | 316 ++++++++-------- .../VidoreTatdqaRetrieval.json | 340 +++++++++--------- .../EDIST2ITRetrieval.json | 186 ++++++++++ .../GLDv2I2IRetrieval.json | 186 ++++++++++ 27 files changed, 2473 insertions(+), 1443 deletions(-) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/EDIST2ITRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2IRetrieval.py create mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/GLDv2I2IRetrieval.json create mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR10.json create mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR100.json create mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/ImageNet10Clustering.json create mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNIST.json create mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNISTZeroShot.json create mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR10.json create mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR100.json create mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNIST.json create mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNISTZeroShot.json create mode 100644 results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/ImageNet10Clustering.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EDIST2ITRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2IRetrieval.json diff --git a/mteb/models/siglip_models.py b/mteb/models/siglip_models.py index 17e7dd0056..3c33139920 100644 --- a/mteb/models/siglip_models.py +++ b/mteb/models/siglip_models.py @@ -40,11 +40,13 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): for i in tqdm(range(0, len(texts), batch_size)): batch_texts = texts[i : i + batch_size] inputs = self.processor( - text=batch_texts, return_tensors="pt", padding=True, truncation=True + text=batch_texts, + return_tensors="pt", + padding="max_length", + truncation=True, ) inputs = {k: v.to(self.device) for k, v in inputs.items()} text_outputs = self.model.get_text_features(**inputs) - text_outputs = text_outputs / text_outputs.norm(dim=-1, keepdim=True) all_text_embeddings.append(text_outputs.cpu()) all_text_embeddings = torch.cat(all_text_embeddings, dim=0) @@ -63,22 +65,22 @@ def get_image_embeddings( ) inputs = {k: v.to(self.device) for k, v in inputs.items()} image_outputs = self.model.get_image_features(**inputs) - image_outputs = image_outputs / image_outputs.norm( - dim=-1, keepdim=True - ) all_image_embeddings.append(image_outputs.cpu()) else: with torch.no_grad(): for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] + batch_images = [ + img.convert("RGB") + if isinstance(img, Image.Image) and img.mode != "RGB" + else img + for img in batch_images + ] inputs = self.processor( images=batch_images, return_tensors="pt", padding=True ) inputs = {k: v.to(self.device) for k, v in inputs.items()} image_outputs = self.model.get_image_features(**inputs) - image_outputs = image_outputs / image_outputs.norm( - dim=-1, keepdim=True - ) all_image_embeddings.append(image_outputs.cpu()) all_image_embeddings = torch.cat(all_image_embeddings, dim=0) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index d4822a3473..59fbb1a11a 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -4,12 +4,14 @@ from .eng.BLINKIT2TRetrieval import * from .eng.CIRRIT2IRetrieval import * from .eng.CUB200I2IRetrieval import * +from .eng.EDIST2ITRetrieval import * from .eng.Fashion200kI2TRetrieval import * from .eng.Fashion200kT2IRetrieval import * from .eng.FashionIQIT2IRetrieval import * from .eng.Flickr30kI2TRetrieval import * from .eng.Flickr30kT2IRetrieval import * from .eng.FORBI2IRetrieval import * +from .eng.GLDv2I2IRetrieval import * from .eng.GLDv2I2TRetrieval import * from .eng.HatefulMemesI2TRetrieval import * from .eng.HatefulMemesT2IRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/EDIST2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/EDIST2ITRetrieval.py new file mode 100644 index 0000000000..e1f8309066 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/EDIST2ITRetrieval.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class EDIST2ITRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="EDIST2ITRetrieval", + description="Retrieve news images and titles based on news content.", + reference="https://aclanthology.org/2023.emnlp-main.297/", + dataset={ + "path": "MRBench/mbeir_edis_task2", + "revision": "68c47ef3e49ef883073b3358bd4243eeca0aee9a", + }, + type="Any2AnyRetrieval", + category="t2it", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2023-01-01", "2023-12-31"), + domains=["News"], + task_subtypes=["Image Text Retrieval"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="created", + bibtex_citation="""@inproceedings{liu2023edis, + title={EDIS: Entity-Driven Image Search over Multimodal Web Content}, + author={Liu, Siqi and Feng, Weixi and Fu, Tsu-Jui and Chen, Wenhu and Wang, William}, + booktitle={Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing}, + pages={4877--4894}, + year={2023} +}""", + descriptive_stats={ + "n_samples": {"test": 3241}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 1047067, + "num_queries": 3241, + "average_relevant_docs_per_query": 2.57, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2IRetrieval.py new file mode 100644 index 0000000000..1d0c2c3bcf --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/GLDv2I2IRetrieval.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class GLDv2I2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="GLDv2I2IRetrieval", + description="Retrieve names of landmarks based on their image.", + reference="https://openaccess.thecvf.com/content_CVPR_2020/html/Weyand_Google_Landmarks_Dataset_v2_-_A_Large-Scale_Benchmark_for_Instance-Level_CVPR_2020_paper.html", + dataset={ + "path": "gowitheflow/gld-v2", + "revision": "c6b162ee349adb293901128a18c0b446f7b43457", + }, + type="Any2AnyRetrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2017-01-01", "2017-12-31"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@InProceedings{Weyand_2020_CVPR, +author = {Weyand, Tobias and Araujo, Andre and Cao, Bingyi and Sim, Jack}, +title = {Google Landmarks Dataset v2 - A Large-Scale Benchmark for Instance-Level Recognition and Retrieval}, +booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, +month = {June}, +year = {2020} +} + +""", + descriptive_stats={ + "n_samples": {"test": 1129}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 847728, + "num_queries": 1129, + "average_relevant_docs_per_query": 13.49, + } + }, + }, + ) diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/GLDv2I2IRetrieval.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/GLDv2I2IRetrieval.json new file mode 100644 index 0000000000..2ce64ec37b --- /dev/null +++ b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/GLDv2I2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c6b162ee349adb293901128a18c0b446f7b43457", + "evaluation_time": 4408.689854860306, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.4535, + "cv_recall_at_10": 0.57219, + "cv_recall_at_100": 0.6519, + "cv_recall_at_1000": 0.73339, + "cv_recall_at_20": 0.59433, + "cv_recall_at_3": 0.50753, + "cv_recall_at_5": 0.5403, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.28941, + "map_at_1": 0.05847, + "map_at_10": 0.15245, + "map_at_100": 0.17257, + "map_at_1000": 0.1746, + "map_at_20": 0.16358, + "map_at_3": 0.10746, + "map_at_5": 0.1285, + "mrr_at_1": 0.4534986713906112, + "mrr_at_10": 0.48867237476626324, + "mrr_at_100": 0.4915899189723126, + "mrr_at_1000": 0.49191288914802445, + "mrr_at_20": 0.4902356580948968, + "mrr_at_3": 0.4769707705934455, + "mrr_at_5": 0.48436669619131967, + "nauc_cv_recall_at_1000_diff1": 0.34148107488406604, + "nauc_cv_recall_at_1000_max": 0.5657029091377153, + "nauc_cv_recall_at_1000_std": 0.3365411347443642, + "nauc_cv_recall_at_100_diff1": 0.3808148655813085, + "nauc_cv_recall_at_100_max": 0.5494694185142655, + "nauc_cv_recall_at_100_std": 0.2927349479831974, + "nauc_cv_recall_at_10_diff1": 0.40118856921921564, + "nauc_cv_recall_at_10_max": 0.5702709786136617, + "nauc_cv_recall_at_10_std": 0.2506097969647151, + "nauc_cv_recall_at_1_diff1": 0.43232239408517925, + "nauc_cv_recall_at_1_max": 0.6111730350701059, + "nauc_cv_recall_at_1_std": 0.2626457116510796, + "nauc_cv_recall_at_20_diff1": 0.3976444877640721, + "nauc_cv_recall_at_20_max": 0.5561717117356078, + "nauc_cv_recall_at_20_std": 0.258312177592681, + "nauc_cv_recall_at_3_diff1": 0.4162409238600696, + "nauc_cv_recall_at_3_max": 0.5731658448651727, + "nauc_cv_recall_at_3_std": 0.24259357095035114, + "nauc_cv_recall_at_5_diff1": 0.40222757559525246, + "nauc_cv_recall_at_5_max": 0.5740443308429867, + "nauc_cv_recall_at_5_std": 0.2638681693812113, + "nauc_map_at_1000_diff1": 0.18861481679640155, + "nauc_map_at_1000_max": 0.49541010487450027, + "nauc_map_at_1000_std": 0.25479149329445355, + "nauc_map_at_100_diff1": 0.19036469102447715, + "nauc_map_at_100_max": 0.49424236992956605, + "nauc_map_at_100_std": 0.2508171110383186, + "nauc_map_at_10_diff1": 0.22996775077271184, + "nauc_map_at_10_max": 0.4659523068740507, + "nauc_map_at_10_std": 0.17873558257018604, + "nauc_map_at_1_diff1": 0.4814554391164708, + "nauc_map_at_1_max": 0.22663592739850807, + "nauc_map_at_1_std": -0.016374913627465144, + "nauc_map_at_20_diff1": 0.20523019715428878, + "nauc_map_at_20_max": 0.48572148693290934, + "nauc_map_at_20_std": 0.21802056174164897, + "nauc_map_at_3_diff1": 0.3578012044738236, + "nauc_map_at_3_max": 0.35368283917955673, + "nauc_map_at_3_std": 0.07546794596340074, + "nauc_map_at_5_diff1": 0.2966436966851955, + "nauc_map_at_5_max": 0.4113893523345609, + "nauc_map_at_5_std": 0.12432827131997834, + "nauc_mrr_at_1000_diff1": 0.4219354542268342, + "nauc_mrr_at_1000_max": 0.5938517870610833, + "nauc_mrr_at_1000_std": 0.25789989416553666, + "nauc_mrr_at_100_diff1": 0.42195820028912456, + "nauc_mrr_at_100_max": 0.5938569414601591, + "nauc_mrr_at_100_std": 0.25780053121453195, + "nauc_mrr_at_10_diff1": 0.4226196920315362, + "nauc_mrr_at_10_max": 0.5949390240575794, + "nauc_mrr_at_10_std": 0.2568484636479838, + "nauc_mrr_at_1_diff1": 0.43232239408517925, + "nauc_mrr_at_1_max": 0.6111730350701059, + "nauc_mrr_at_1_std": 0.2626457116510796, + "nauc_mrr_at_20_diff1": 0.4223978068861933, + "nauc_mrr_at_20_max": 0.59408807498729, + "nauc_mrr_at_20_std": 0.2573461830046759, + "nauc_mrr_at_3_diff1": 0.4262502276324796, + "nauc_mrr_at_3_max": 0.5954118493110601, + "nauc_mrr_at_3_std": 0.2545125910785488, + "nauc_mrr_at_5_diff1": 0.4228118558706074, + "nauc_mrr_at_5_max": 0.5953723133930325, + "nauc_mrr_at_5_std": 0.25854738381668874, + "nauc_ndcg_at_1000_diff1": 0.22343292588387825, + "nauc_ndcg_at_1000_max": 0.5276606021449388, + "nauc_ndcg_at_1000_std": 0.29728763467934904, + "nauc_ndcg_at_100_diff1": 0.23190304701399703, + "nauc_ndcg_at_100_max": 0.5245071773764476, + "nauc_ndcg_at_100_std": 0.2783843293138034, + "nauc_ndcg_at_10_diff1": 0.19316555007961791, + "nauc_ndcg_at_10_max": 0.5825075587136279, + "nauc_ndcg_at_10_std": 0.3092465370973502, + "nauc_ndcg_at_1_diff1": 0.43232239408517925, + "nauc_ndcg_at_1_max": 0.6111730350701059, + "nauc_ndcg_at_1_std": 0.2626457116510796, + "nauc_ndcg_at_20_diff1": 0.21755951662398187, + "nauc_ndcg_at_20_max": 0.5475682374017373, + "nauc_ndcg_at_20_std": 0.2860733738075765, + "nauc_ndcg_at_3_diff1": 0.24971063630292664, + "nauc_ndcg_at_3_max": 0.6086291585528392, + "nauc_ndcg_at_3_std": 0.2992078517726314, + "nauc_ndcg_at_5_diff1": 0.21400973879521373, + "nauc_ndcg_at_5_max": 0.605819692716699, + "nauc_ndcg_at_5_std": 0.31456299770488383, + "nauc_precision_at_1000_diff1": -0.044719214820325454, + "nauc_precision_at_1000_max": 0.469245361680825, + "nauc_precision_at_1000_std": 0.459126550464243, + "nauc_precision_at_100_diff1": -0.0361873993762923, + "nauc_precision_at_100_max": 0.5372410748582264, + "nauc_precision_at_100_std": 0.49655377124356126, + "nauc_precision_at_10_diff1": 0.02048413751839119, + "nauc_precision_at_10_max": 0.601718588539165, + "nauc_precision_at_10_std": 0.38924805371157156, + "nauc_precision_at_1_diff1": 0.43232239408517925, + "nauc_precision_at_1_max": 0.6111730350701059, + "nauc_precision_at_1_std": 0.2626457116510796, + "nauc_precision_at_20_diff1": -0.01598332604841292, + "nauc_precision_at_20_max": 0.5860681313676941, + "nauc_precision_at_20_std": 0.4447422992419558, + "nauc_precision_at_3_diff1": 0.18237525035807778, + "nauc_precision_at_3_max": 0.6073405177582907, + "nauc_precision_at_3_std": 0.3148368780578123, + "nauc_precision_at_5_diff1": 0.10080092161444594, + "nauc_precision_at_5_max": 0.6113824415227623, + "nauc_precision_at_5_std": 0.3542916057954643, + "nauc_recall_at_1000_diff1": 0.11156694655914166, + "nauc_recall_at_1000_max": 0.4029532703125521, + "nauc_recall_at_1000_std": 0.3191218466863474, + "nauc_recall_at_100_diff1": 0.15346884478259795, + "nauc_recall_at_100_max": 0.4263756821168572, + "nauc_recall_at_100_std": 0.2667922963100197, + "nauc_recall_at_10_diff1": 0.20481260563318826, + "nauc_recall_at_10_max": 0.42307599013145597, + "nauc_recall_at_10_std": 0.16281281054955174, + "nauc_recall_at_1_diff1": 0.4814554391164708, + "nauc_recall_at_1_max": 0.22663592739850807, + "nauc_recall_at_1_std": -0.016374913627465144, + "nauc_recall_at_20_diff1": 0.1805511540924871, + "nauc_recall_at_20_max": 0.4341772622005704, + "nauc_recall_at_20_std": 0.20225716907182484, + "nauc_recall_at_3_diff1": 0.343757341612119, + "nauc_recall_at_3_max": 0.33208304495011726, + "nauc_recall_at_3_std": 0.0633600297459121, + "nauc_recall_at_5_diff1": 0.2784926120476109, + "nauc_recall_at_5_max": 0.3814328242345907, + "nauc_recall_at_5_std": 0.11379188945081205, + "ndcg_at_1": 0.4535, + "ndcg_at_10": 0.28941, + "ndcg_at_100": 0.26716, + "ndcg_at_1000": 0.29187, + "ndcg_at_20": 0.26099, + "ndcg_at_3": 0.37311, + "ndcg_at_5": 0.33283, + "precision_at_1": 0.4535, + "precision_at_10": 0.19699, + "precision_at_100": 0.03368, + "precision_at_1000": 0.00467, + "precision_at_20": 0.12161, + "precision_at_3": 0.34485, + "precision_at_5": 0.28131, + "recall_at_1": 0.05847, + "recall_at_10": 0.17682, + "recall_at_100": 0.25581, + "recall_at_1000": 0.34658, + "recall_at_20": 0.20188, + "recall_at_3": 0.11247, + "recall_at_5": 0.13964 + } + ] + }, + "task_name": "GLDv2I2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR10.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR10.json new file mode 100644 index 0000000000..328e3b46b4 --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR10.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + "evaluation_time": 48.409629821777344, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.92594, + "f1": 0.9260547788569216, + "f1_weighted": 0.9260547788569216, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.92594, + "scores_per_experiment": [ + { + "accuracy": 0.9249, + "f1": 0.9250702364213208, + "f1_weighted": 0.9250702364213208 + }, + { + "accuracy": 0.9232, + "f1": 0.9231630424362665, + "f1_weighted": 0.9231630424362666 + }, + { + "accuracy": 0.9217, + "f1": 0.9218520263643336, + "f1_weighted": 0.9218520263643335 + }, + { + "accuracy": 0.9308, + "f1": 0.9308028706566229, + "f1_weighted": 0.930802870656623 + }, + { + "accuracy": 0.9291, + "f1": 0.9293857184060645, + "f1_weighted": 0.9293857184060643 + } + ] + } + ] + }, + "task_name": "CIFAR10" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR100.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR100.json new file mode 100644 index 0000000000..0ef0c4ba83 --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR100.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + "evaluation_time": 82.63413286209106, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.7413199999999999, + "f1": 0.7440710594068098, + "f1_weighted": 0.7440710594068098, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7413199999999999, + "scores_per_experiment": [ + { + "accuracy": 0.7393, + "f1": 0.7415575159167781, + "f1_weighted": 0.7415575159167781 + }, + { + "accuracy": 0.7449, + "f1": 0.7471305285419396, + "f1_weighted": 0.7471305285419396 + }, + { + "accuracy": 0.7428, + "f1": 0.7465020754943685, + "f1_weighted": 0.7465020754943685 + }, + { + "accuracy": 0.7423, + "f1": 0.744902146154522, + "f1_weighted": 0.7449021461545219 + }, + { + "accuracy": 0.7373, + "f1": 0.7402630309264411, + "f1_weighted": 0.7402630309264413 + } + ] + } + ] + }, + "task_name": "CIFAR100" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/ImageNet10Clustering.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/ImageNet10Clustering.json new file mode 100644 index 0000000000..866c25f1fe --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/ImageNet10Clustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "88f8a6d47c257895094c5ad81e67ba751771fc99", + "evaluation_time": 50.5283944606781, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "ari": 0.9791753845317047, + "cluster_accuracy": 0.9905384615384616, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9739308018003532, + "nmi": 0.9739308018003532, + "v_measure": 0.9739308018003532 + } + ] + }, + "task_name": "ImageNet10Clustering" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNIST.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNIST.json new file mode 100644 index 0000000000..264a93931c --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 46.19513821601868, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.95434, + "f1": 0.9542219615719318, + "f1_weighted": 0.9544309958560502, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.95434, + "scores_per_experiment": [ + { + "accuracy": 0.9543, + "f1": 0.9542073639183843, + "f1_weighted": 0.9544394896400503 + }, + { + "accuracy": 0.949, + "f1": 0.9487069493447204, + "f1_weighted": 0.9489175533097969 + }, + { + "accuracy": 0.9586, + "f1": 0.9583645832273339, + "f1_weighted": 0.958733263357083 + }, + { + "accuracy": 0.9522, + "f1": 0.9522190790543403, + "f1_weighted": 0.9523957230838139 + }, + { + "accuracy": 0.9576, + "f1": 0.9576118323148799, + "f1_weighted": 0.957668949889507 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNISTZeroShot.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNISTZeroShot.json new file mode 100644 index 0000000000..06769b48eb --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNISTZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 38.23384070396423, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.8361, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8361 + } + ] + }, + "task_name": "MNISTZeroShot" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR10.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR10.json new file mode 100644 index 0000000000..a94a3ac8a1 --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR10.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + "evaluation_time": 59.87628698348999, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.93338, + "f1": 0.9334631436619831, + "f1_weighted": 0.9334631436619831, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.93338, + "scores_per_experiment": [ + { + "accuracy": 0.9326, + "f1": 0.9325960933629046, + "f1_weighted": 0.9325960933629045 + }, + { + "accuracy": 0.9351, + "f1": 0.935126693421535, + "f1_weighted": 0.935126693421535 + }, + { + "accuracy": 0.9286, + "f1": 0.9287279749504849, + "f1_weighted": 0.9287279749504851 + }, + { + "accuracy": 0.9345, + "f1": 0.9344698910282221, + "f1_weighted": 0.9344698910282219 + }, + { + "accuracy": 0.9361, + "f1": 0.9363950655467697, + "f1_weighted": 0.9363950655467698 + } + ] + } + ] + }, + "task_name": "CIFAR10" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR100.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR100.json new file mode 100644 index 0000000000..f7b2bb3c5c --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR100.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + "evaluation_time": 101.40493559837341, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.75654, + "f1": 0.7586602884829271, + "f1_weighted": 0.7586602884829272, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.75654, + "scores_per_experiment": [ + { + "accuracy": 0.7565, + "f1": 0.757924035988359, + "f1_weighted": 0.7579240359883591 + }, + { + "accuracy": 0.7598, + "f1": 0.7613388743142541, + "f1_weighted": 0.7613388743142538 + }, + { + "accuracy": 0.759, + "f1": 0.7617133389971635, + "f1_weighted": 0.7617133389971638 + }, + { + "accuracy": 0.7498, + "f1": 0.7524641724151415, + "f1_weighted": 0.7524641724151419 + }, + { + "accuracy": 0.7576, + "f1": 0.7598610206997175, + "f1_weighted": 0.7598610206997176 + } + ] + } + ] + }, + "task_name": "CIFAR100" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNIST.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNIST.json new file mode 100644 index 0000000000..5c6ed62858 --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 56.67082715034485, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.9572800000000001, + "f1": 0.9571422576266858, + "f1_weighted": 0.9573362168194761, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9572800000000001, + "scores_per_experiment": [ + { + "accuracy": 0.9592, + "f1": 0.9590539585386963, + "f1_weighted": 0.9592951422536539 + }, + { + "accuracy": 0.9499, + "f1": 0.9494882828072895, + "f1_weighted": 0.9498086431430818 + }, + { + "accuracy": 0.9597, + "f1": 0.9594446389305078, + "f1_weighted": 0.9597836962699936 + }, + { + "accuracy": 0.9578, + "f1": 0.9578834947792313, + "f1_weighted": 0.9580006084749224 + }, + { + "accuracy": 0.9598, + "f1": 0.9598409130777045, + "f1_weighted": 0.9597929939557287 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNISTZeroShot.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNISTZeroShot.json new file mode 100644 index 0000000000..b77069c7c0 --- /dev/null +++ b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNISTZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 50.64418292045593, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.8457, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8457 + } + ] + }, + "task_name": "MNISTZeroShot" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/ImageNet10Clustering.json b/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/ImageNet10Clustering.json new file mode 100644 index 0000000000..185e72dbf7 --- /dev/null +++ b/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/ImageNet10Clustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "88f8a6d47c257895094c5ad81e67ba751771fc99", + "evaluation_time": 408.774094581604, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "ari": 0.9850555604145985, + "cluster_accuracy": 0.9932307692307693, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9806773175345154, + "nmi": 0.9806773175345154, + "v_measure": 0.9806773175345154 + } + ] + }, + "task_name": "ImageNet10Clustering" +} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json index eeaf17e01d..6c5b4b0d05 100644 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json @@ -1,184 +1,184 @@ { "dataset_revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", - "evaluation_time": 36.80075192451477, + "evaluation_time": 37.123581647872925, "kg_co2_emissions": null, "mteb_version": "1.14.15", "scores": { "test": [ { - "cv_recall_at_1": 0.14, - "cv_recall_at_10": 0.326, - "cv_recall_at_100": 0.7, + "cv_recall_at_1": 0.408, + "cv_recall_at_10": 0.658, + "cv_recall_at_100": 0.9, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.418, - "cv_recall_at_3": 0.216, - "cv_recall_at_5": 0.26, + "cv_recall_at_20": 0.73, + "cv_recall_at_3": 0.524, + "cv_recall_at_5": 0.58, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.20018, - "map_at_1": 0.14, - "map_at_10": 0.18964, - "map_at_100": 0.20214, - "map_at_1000": 0.20383, - "map_at_20": 0.1961, - "map_at_3": 0.17067, - "map_at_5": 0.18057, - "mrr_at_1": 0.14, - "mrr_at_10": 0.18963650793650794, - "mrr_at_100": 0.20214010057097828, - "mrr_at_1000": 0.20383485850905744, - "mrr_at_20": 0.19610020934517847, - "mrr_at_3": 0.17066666666666666, - "mrr_at_5": 0.18056666666666668, + "main_score": 0.50041, + "map_at_1": 0.408, + "map_at_10": 0.48463, + "map_at_100": 0.49363, + "map_at_1000": 0.49423, + "map_at_20": 0.48953, + "map_at_3": 0.461, + "map_at_5": 0.4739, + "mrr_at_1": 0.408, + "mrr_at_10": 0.48463492063492053, + "mrr_at_100": 0.4936256394439807, + "mrr_at_1000": 0.49422833528767024, + "mrr_at_20": 0.4895304532842458, + "mrr_at_3": 0.46100000000000013, + "mrr_at_5": 0.4739000000000001, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.1943109349666728, - "nauc_cv_recall_at_100_max": 0.30141596108809204, - "nauc_cv_recall_at_100_std": 0.18449648711943784, - "nauc_cv_recall_at_10_diff1": 0.3404131572137101, - "nauc_cv_recall_at_10_max": 0.41392291253691765, - "nauc_cv_recall_at_10_std": 0.14801998820770446, - "nauc_cv_recall_at_1_diff1": 0.6383384954813527, - "nauc_cv_recall_at_1_max": 0.5636386779243923, - "nauc_cv_recall_at_1_std": 0.14467733039161593, - "nauc_cv_recall_at_20_diff1": 0.2755412102074175, - "nauc_cv_recall_at_20_max": 0.3530009092486914, - "nauc_cv_recall_at_20_std": 0.10041369520222374, - "nauc_cv_recall_at_3_diff1": 0.4578307594032993, - "nauc_cv_recall_at_3_max": 0.45122371272185274, - "nauc_cv_recall_at_3_std": 0.14290948032916234, - "nauc_cv_recall_at_5_diff1": 0.38610621259865296, - "nauc_cv_recall_at_5_max": 0.4313116128472723, - "nauc_cv_recall_at_5_std": 0.12227458009308771, - "nauc_map_at_1000_diff1": 0.5108013348607887, - "nauc_map_at_1000_max": 0.49556810553994785, - "nauc_map_at_1000_std": 0.14340316545286083, - "nauc_map_at_100_diff1": 0.5089368235572634, - "nauc_map_at_100_max": 0.49476477633650834, - "nauc_map_at_100_std": 0.14332987871773434, - "nauc_map_at_10_diff1": 0.5152119169569348, - "nauc_map_at_10_max": 0.5016430069177228, - "nauc_map_at_10_std": 0.14462347870066639, - "nauc_map_at_1_diff1": 0.6383384954813527, - "nauc_map_at_1_max": 0.5636386779243923, - "nauc_map_at_1_std": 0.14467733039161593, - "nauc_map_at_20_diff1": 0.5082693046372923, - "nauc_map_at_20_max": 0.494718342086452, - "nauc_map_at_20_std": 0.14051107087925876, - "nauc_map_at_3_diff1": 0.5483175236158682, - "nauc_map_at_3_max": 0.5119270005626352, - "nauc_map_at_3_std": 0.14619814132999046, - "nauc_map_at_5_diff1": 0.5256920920484943, - "nauc_map_at_5_max": 0.5050437535335612, - "nauc_map_at_5_std": 0.14038793888357393, - "nauc_mrr_at_1000_diff1": 0.5108013348607887, - "nauc_mrr_at_1000_max": 0.49556810553994785, - "nauc_mrr_at_1000_std": 0.14340316545286083, - "nauc_mrr_at_100_diff1": 0.5089368235572634, - "nauc_mrr_at_100_max": 0.49476477633650834, - "nauc_mrr_at_100_std": 0.14332987871773434, - "nauc_mrr_at_10_diff1": 0.5152119169569348, - "nauc_mrr_at_10_max": 0.5016430069177228, - "nauc_mrr_at_10_std": 0.14462347870066639, - "nauc_mrr_at_1_diff1": 0.6383384954813527, - "nauc_mrr_at_1_max": 0.5636386779243923, - "nauc_mrr_at_1_std": 0.14467733039161593, - "nauc_mrr_at_20_diff1": 0.5082693046372923, - "nauc_mrr_at_20_max": 0.494718342086452, - "nauc_mrr_at_20_std": 0.14051107087925876, - "nauc_mrr_at_3_diff1": 0.5483175236158682, - "nauc_mrr_at_3_max": 0.5119270005626352, - "nauc_mrr_at_3_std": 0.14619814132999046, - "nauc_mrr_at_5_diff1": 0.5256920920484943, - "nauc_mrr_at_5_max": 0.5050437535335612, - "nauc_mrr_at_5_std": 0.14038793888357393, - "nauc_ndcg_at_1000_diff1": 0.47546503263846057, - "nauc_ndcg_at_1000_max": 0.4730852930430677, - "nauc_ndcg_at_1000_std": 0.14451699521991915, - "nauc_ndcg_at_100_diff1": 0.43918780468181184, - "nauc_ndcg_at_100_max": 0.4525861264216166, - "nauc_ndcg_at_100_std": 0.14813430958483342, - "nauc_ndcg_at_10_diff1": 0.46325211944837963, - "nauc_ndcg_at_10_max": 0.4755238714506355, - "nauc_ndcg_at_10_std": 0.1451640297423185, - "nauc_ndcg_at_1_diff1": 0.6383384954813527, - "nauc_ndcg_at_1_max": 0.5636386779243923, - "nauc_ndcg_at_1_std": 0.14467733039161593, - "nauc_ndcg_at_20_diff1": 0.4411334690790833, - "nauc_ndcg_at_20_max": 0.4545908778015437, - "nauc_ndcg_at_20_std": 0.13045890758542045, - "nauc_ndcg_at_3_diff1": 0.5221939167041364, - "nauc_ndcg_at_3_max": 0.49457354247115864, - "nauc_ndcg_at_3_std": 0.14534347661530161, - "nauc_ndcg_at_5_diff1": 0.48489323636673515, - "nauc_ndcg_at_5_max": 0.4834143750862925, - "nauc_ndcg_at_5_std": 0.1353775207751414, + "nauc_cv_recall_at_100_diff1": 0.48634920634920553, + "nauc_cv_recall_at_100_max": 0.591447245564892, + "nauc_cv_recall_at_100_std": 0.4420074696545277, + "nauc_cv_recall_at_10_diff1": 0.5607732158147564, + "nauc_cv_recall_at_10_max": 0.525200263160505, + "nauc_cv_recall_at_10_std": 0.2014632517513726, + "nauc_cv_recall_at_1_diff1": 0.6848492828154529, + "nauc_cv_recall_at_1_max": 0.49441071078203574, + "nauc_cv_recall_at_1_std": 0.11512903125871315, + "nauc_cv_recall_at_20_diff1": 0.5547543690656394, + "nauc_cv_recall_at_20_max": 0.532280956967898, + "nauc_cv_recall_at_20_std": 0.2484735300477695, + "nauc_cv_recall_at_3_diff1": 0.610174677499549, + "nauc_cv_recall_at_3_max": 0.47892341399714783, + "nauc_cv_recall_at_3_std": 0.13839790627407458, + "nauc_cv_recall_at_5_diff1": 0.6229851699219318, + "nauc_cv_recall_at_5_max": 0.5293293089938185, + "nauc_cv_recall_at_5_std": 0.160623385669095, + "nauc_map_at_1000_diff1": 0.6491101252075504, + "nauc_map_at_1000_max": 0.5000671746105353, + "nauc_map_at_1000_std": 0.13430585104690468, + "nauc_map_at_100_diff1": 0.6488183522188637, + "nauc_map_at_100_max": 0.5002268240822477, + "nauc_map_at_100_std": 0.13485377824353093, + "nauc_map_at_10_diff1": 0.6479922280771185, + "nauc_map_at_10_max": 0.499988268241859, + "nauc_map_at_10_std": 0.13238514930345616, + "nauc_map_at_1_diff1": 0.6848492828154529, + "nauc_map_at_1_max": 0.49441071078203574, + "nauc_map_at_1_std": 0.11512903125871315, + "nauc_map_at_20_diff1": 0.6485457083737673, + "nauc_map_at_20_max": 0.500196334435762, + "nauc_map_at_20_std": 0.13466389043096438, + "nauc_map_at_3_diff1": 0.6507960840023341, + "nauc_map_at_3_max": 0.489709383637825, + "nauc_map_at_3_std": 0.12304998513330462, + "nauc_map_at_5_diff1": 0.6546180635941539, + "nauc_map_at_5_max": 0.5013720423294509, + "nauc_map_at_5_std": 0.12821821598924624, + "nauc_mrr_at_1000_diff1": 0.6491101252075504, + "nauc_mrr_at_1000_max": 0.5000671746105353, + "nauc_mrr_at_1000_std": 0.13430585104690468, + "nauc_mrr_at_100_diff1": 0.6488183522188637, + "nauc_mrr_at_100_max": 0.5002268240822477, + "nauc_mrr_at_100_std": 0.13485377824353093, + "nauc_mrr_at_10_diff1": 0.6479922280771185, + "nauc_mrr_at_10_max": 0.499988268241859, + "nauc_mrr_at_10_std": 0.13238514930345616, + "nauc_mrr_at_1_diff1": 0.6848492828154529, + "nauc_mrr_at_1_max": 0.49441071078203574, + "nauc_mrr_at_1_std": 0.11512903125871315, + "nauc_mrr_at_20_diff1": 0.6485457083737673, + "nauc_mrr_at_20_max": 0.500196334435762, + "nauc_mrr_at_20_std": 0.13466389043096438, + "nauc_mrr_at_3_diff1": 0.6507960840023341, + "nauc_mrr_at_3_max": 0.489709383637825, + "nauc_mrr_at_3_std": 0.12304998513330462, + "nauc_mrr_at_5_diff1": 0.6546180635941539, + "nauc_mrr_at_5_max": 0.5013720423294509, + "nauc_mrr_at_5_std": 0.12821821598924624, + "nauc_ndcg_at_1000_diff1": 0.6405236118047709, + "nauc_ndcg_at_1000_max": 0.5029849206775402, + "nauc_ndcg_at_1000_std": 0.14436126589668163, + "nauc_ndcg_at_100_diff1": 0.6334691351131405, + "nauc_ndcg_at_100_max": 0.5070066248549946, + "nauc_ndcg_at_100_std": 0.15797425676621923, + "nauc_ndcg_at_10_diff1": 0.6301087837132273, + "nauc_ndcg_at_10_max": 0.5051015889632752, + "nauc_ndcg_at_10_std": 0.14632520256105425, + "nauc_ndcg_at_1_diff1": 0.6848492828154529, + "nauc_ndcg_at_1_max": 0.49441071078203574, + "nauc_ndcg_at_1_std": 0.11512903125871315, + "nauc_ndcg_at_20_diff1": 0.6314997303399957, + "nauc_ndcg_at_20_max": 0.5058510110615413, + "nauc_ndcg_at_20_std": 0.15408489217326618, + "nauc_ndcg_at_3_diff1": 0.6405250822588264, + "nauc_ndcg_at_3_max": 0.48708846743740086, + "nauc_ndcg_at_3_std": 0.12680957085946576, + "nauc_ndcg_at_5_diff1": 0.6467436522794359, + "nauc_ndcg_at_5_max": 0.5076083996525415, + "nauc_ndcg_at_5_std": 0.13575029438286534, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.19431093496667312, - "nauc_precision_at_100_max": 0.3014159610880925, - "nauc_precision_at_100_std": 0.18449648711943817, - "nauc_precision_at_10_diff1": 0.3404131572137099, - "nauc_precision_at_10_max": 0.4139229125369173, - "nauc_precision_at_10_std": 0.14801998820770462, - "nauc_precision_at_1_diff1": 0.6383384954813527, - "nauc_precision_at_1_max": 0.5636386779243923, - "nauc_precision_at_1_std": 0.14467733039161593, - "nauc_precision_at_20_diff1": 0.2755412102074174, - "nauc_precision_at_20_max": 0.3530009092486912, - "nauc_precision_at_20_std": 0.10041369520222365, - "nauc_precision_at_3_diff1": 0.4578307594032993, - "nauc_precision_at_3_max": 0.45122371272185285, - "nauc_precision_at_3_std": 0.1429094803291628, - "nauc_precision_at_5_diff1": 0.38610621259865285, - "nauc_precision_at_5_max": 0.4313116128472727, - "nauc_precision_at_5_std": 0.12227458009308804, + "nauc_precision_at_100_diff1": 0.48634920634920636, + "nauc_precision_at_100_max": 0.5914472455648909, + "nauc_precision_at_100_std": 0.44200746965452886, + "nauc_precision_at_10_diff1": 0.5607732158147563, + "nauc_precision_at_10_max": 0.5252002631605046, + "nauc_precision_at_10_std": 0.2014632517513726, + "nauc_precision_at_1_diff1": 0.6848492828154529, + "nauc_precision_at_1_max": 0.49441071078203574, + "nauc_precision_at_1_std": 0.11512903125871315, + "nauc_precision_at_20_diff1": 0.5547543690656398, + "nauc_precision_at_20_max": 0.5322809569678982, + "nauc_precision_at_20_std": 0.24847353004776943, + "nauc_precision_at_3_diff1": 0.6101746774995492, + "nauc_precision_at_3_max": 0.4789234139971478, + "nauc_precision_at_3_std": 0.1383979062740747, + "nauc_precision_at_5_diff1": 0.6229851699219321, + "nauc_precision_at_5_max": 0.529329308993818, + "nauc_precision_at_5_std": 0.16062338566909454, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.1943109349666728, - "nauc_recall_at_100_max": 0.30141596108809204, - "nauc_recall_at_100_std": 0.18449648711943784, - "nauc_recall_at_10_diff1": 0.3404131572137101, - "nauc_recall_at_10_max": 0.41392291253691765, - "nauc_recall_at_10_std": 0.14801998820770446, - "nauc_recall_at_1_diff1": 0.6383384954813527, - "nauc_recall_at_1_max": 0.5636386779243923, - "nauc_recall_at_1_std": 0.14467733039161593, - "nauc_recall_at_20_diff1": 0.2755412102074175, - "nauc_recall_at_20_max": 0.3530009092486914, - "nauc_recall_at_20_std": 0.10041369520222374, - "nauc_recall_at_3_diff1": 0.4578307594032993, - "nauc_recall_at_3_max": 0.45122371272185274, - "nauc_recall_at_3_std": 0.14290948032916234, - "nauc_recall_at_5_diff1": 0.38610621259865296, - "nauc_recall_at_5_max": 0.4313116128472723, - "nauc_recall_at_5_std": 0.12227458009308771, - "ndcg_at_1": 0.14, - "ndcg_at_10": 0.22178, - "ndcg_at_100": 0.29502, - "ndcg_at_1000": 0.33472, - "ndcg_at_20": 0.24514, - "ndcg_at_3": 0.18219, - "ndcg_at_5": 0.20018, - "precision_at_1": 0.14, - "precision_at_10": 0.0326, - "precision_at_100": 0.007, + "nauc_recall_at_100_diff1": 0.48634920634920553, + "nauc_recall_at_100_max": 0.591447245564892, + "nauc_recall_at_100_std": 0.4420074696545277, + "nauc_recall_at_10_diff1": 0.5607732158147564, + "nauc_recall_at_10_max": 0.525200263160505, + "nauc_recall_at_10_std": 0.2014632517513726, + "nauc_recall_at_1_diff1": 0.6848492828154529, + "nauc_recall_at_1_max": 0.49441071078203574, + "nauc_recall_at_1_std": 0.11512903125871315, + "nauc_recall_at_20_diff1": 0.5547543690656394, + "nauc_recall_at_20_max": 0.532280956967898, + "nauc_recall_at_20_std": 0.2484735300477695, + "nauc_recall_at_3_diff1": 0.610174677499549, + "nauc_recall_at_3_max": 0.47892341399714783, + "nauc_recall_at_3_std": 0.13839790627407458, + "nauc_recall_at_5_diff1": 0.6229851699219318, + "nauc_recall_at_5_max": 0.5293293089938185, + "nauc_recall_at_5_std": 0.160623385669095, + "ndcg_at_1": 0.408, + "ndcg_at_10": 0.52596, + "ndcg_at_100": 0.57503, + "ndcg_at_1000": 0.58846, + "ndcg_at_20": 0.54404, + "ndcg_at_3": 0.47726, + "ndcg_at_5": 0.50041, + "precision_at_1": 0.408, + "precision_at_10": 0.0658, + "precision_at_100": 0.009, "precision_at_1000": 0.001, - "precision_at_20": 0.0209, - "precision_at_3": 0.072, - "precision_at_5": 0.052, - "recall_at_1": 0.14, - "recall_at_10": 0.326, - "recall_at_100": 0.7, + "precision_at_20": 0.0365, + "precision_at_3": 0.17467, + "precision_at_5": 0.116, + "recall_at_1": 0.408, + "recall_at_10": 0.658, + "recall_at_100": 0.9, "recall_at_1000": 1.0, - "recall_at_20": 0.418, - "recall_at_3": 0.216, - "recall_at_5": 0.26 + "recall_at_20": 0.73, + "recall_at_3": 0.524, + "recall_at_5": 0.58 } ] }, diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json index 2ba30bc4d6..53c2fdfc97 100644 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json @@ -1,184 +1,184 @@ { "dataset_revision": "b1d89eda849e636676df6ead8002602fb1858600", - "evaluation_time": 40.50687646865845, + "evaluation_time": 42.08629059791565, "kg_co2_emissions": null, "mteb_version": "1.14.15", "scores": { "test": [ { - "cv_recall_at_1": 0.13, - "cv_recall_at_10": 0.29, - "cv_recall_at_100": 0.592, + "cv_recall_at_1": 0.222, + "cv_recall_at_10": 0.45, + "cv_recall_at_100": 0.726, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.372, - "cv_recall_at_3": 0.192, - "cv_recall_at_5": 0.23, + "cv_recall_at_20": 0.532, + "cv_recall_at_3": 0.348, + "cv_recall_at_5": 0.388, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.18085, - "map_at_1": 0.13, - "map_at_10": 0.17226, - "map_at_100": 0.1827, - "map_at_1000": 0.18458, - "map_at_20": 0.17783, - "map_at_3": 0.15633, - "map_at_5": 0.16473, - "mrr_at_1": 0.13, - "mrr_at_10": 0.17225873015873025, - "mrr_at_100": 0.18270338816739434, - "mrr_at_1000": 0.18458288118569882, - "mrr_at_20": 0.1778293426645595, - "mrr_at_3": 0.15633333333333335, - "mrr_at_5": 0.16473333333333337, + "main_score": 0.31278, + "map_at_1": 0.222, + "map_at_10": 0.29608, + "map_at_100": 0.30637, + "map_at_1000": 0.30768, + "map_at_20": 0.3017, + "map_at_3": 0.27867, + "map_at_5": 0.28757, + "mrr_at_1": 0.222, + "mrr_at_10": 0.2960777777777778, + "mrr_at_100": 0.30637161245152283, + "mrr_at_1000": 0.3076778467647497, + "mrr_at_20": 0.30170205427527425, + "mrr_at_3": 0.2786666666666667, + "mrr_at_5": 0.28756666666666664, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.29668674698795183, - "nauc_cv_recall_at_100_max": 0.3333401967243978, - "nauc_cv_recall_at_100_std": 0.4607668957078311, - "nauc_cv_recall_at_10_diff1": 0.3176627336785572, - "nauc_cv_recall_at_10_max": 0.36628985069296605, - "nauc_cv_recall_at_10_std": 0.45197038583574867, - "nauc_cv_recall_at_1_diff1": 0.5701058852499383, - "nauc_cv_recall_at_1_max": 0.4710169908889438, - "nauc_cv_recall_at_1_std": 0.45840030444807583, - "nauc_cv_recall_at_20_diff1": 0.3173390526584947, - "nauc_cv_recall_at_20_max": 0.36508706837620225, - "nauc_cv_recall_at_20_std": 0.4626756631233985, - "nauc_cv_recall_at_3_diff1": 0.4098228341204408, - "nauc_cv_recall_at_3_max": 0.40064769604311734, - "nauc_cv_recall_at_3_std": 0.447420555511086, - "nauc_cv_recall_at_5_diff1": 0.3935662970363478, - "nauc_cv_recall_at_5_max": 0.3866567013925962, - "nauc_cv_recall_at_5_std": 0.43803175668461425, - "nauc_map_at_1000_diff1": 0.47008151424287187, - "nauc_map_at_1000_max": 0.42564732667200894, - "nauc_map_at_1000_std": 0.45354116369652825, - "nauc_map_at_100_diff1": 0.46917354197138605, - "nauc_map_at_100_max": 0.42511751999498737, - "nauc_map_at_100_std": 0.4536651781812026, - "nauc_map_at_10_diff1": 0.4702955145420076, - "nauc_map_at_10_max": 0.4276602888263265, - "nauc_map_at_10_std": 0.45271558223499087, - "nauc_map_at_1_diff1": 0.5701058852499383, - "nauc_map_at_1_max": 0.4710169908889438, - "nauc_map_at_1_std": 0.45840030444807583, - "nauc_map_at_20_diff1": 0.4699693881258117, - "nauc_map_at_20_max": 0.4263741043059254, - "nauc_map_at_20_std": 0.45377335365237415, - "nauc_map_at_3_diff1": 0.49224148432955067, - "nauc_map_at_3_max": 0.43597783225397924, - "nauc_map_at_3_std": 0.453269429839162, - "nauc_map_at_5_diff1": 0.4859281573788269, - "nauc_map_at_5_max": 0.4305498781499855, - "nauc_map_at_5_std": 0.4504661603812436, - "nauc_mrr_at_1000_diff1": 0.47008151424287187, - "nauc_mrr_at_1000_max": 0.42564732667200894, - "nauc_mrr_at_1000_std": 0.45354116369652825, - "nauc_mrr_at_100_diff1": 0.46917354197138605, - "nauc_mrr_at_100_max": 0.42511751999498737, - "nauc_mrr_at_100_std": 0.4536651781812026, - "nauc_mrr_at_10_diff1": 0.4702955145420076, - "nauc_mrr_at_10_max": 0.4276602888263265, - "nauc_mrr_at_10_std": 0.45271558223499087, - "nauc_mrr_at_1_diff1": 0.5701058852499383, - "nauc_mrr_at_1_max": 0.4710169908889438, - "nauc_mrr_at_1_std": 0.45840030444807583, - "nauc_mrr_at_20_diff1": 0.4699693881258117, - "nauc_mrr_at_20_max": 0.4263741043059254, - "nauc_mrr_at_20_std": 0.45377335365237415, - "nauc_mrr_at_3_diff1": 0.49224148432955067, - "nauc_mrr_at_3_max": 0.43597783225397924, - "nauc_mrr_at_3_std": 0.453269429839162, - "nauc_mrr_at_5_diff1": 0.4859281573788269, - "nauc_mrr_at_5_max": 0.4305498781499855, - "nauc_mrr_at_5_std": 0.4504661603812436, - "nauc_ndcg_at_1000_diff1": 0.44217374572654283, - "nauc_ndcg_at_1000_max": 0.41259692326408354, - "nauc_ndcg_at_1000_std": 0.45335502089769386, - "nauc_ndcg_at_100_diff1": 0.42077496326949876, - "nauc_ndcg_at_100_max": 0.40068408483983037, - "nauc_ndcg_at_100_std": 0.4549589546561871, - "nauc_ndcg_at_10_diff1": 0.42501562481003097, - "nauc_ndcg_at_10_max": 0.40952295092418756, - "nauc_ndcg_at_10_std": 0.4521354844274342, - "nauc_ndcg_at_1_diff1": 0.5701058852499383, - "nauc_ndcg_at_1_max": 0.4710169908889438, - "nauc_ndcg_at_1_std": 0.45840030444807583, - "nauc_ndcg_at_20_diff1": 0.42370856334004775, - "nauc_ndcg_at_20_max": 0.40742912764166467, - "nauc_ndcg_at_20_std": 0.455638693912446, - "nauc_ndcg_at_3_diff1": 0.46866327788004875, - "nauc_ndcg_at_3_max": 0.42583768645337167, - "nauc_ndcg_at_3_std": 0.45160402362953556, - "nauc_ndcg_at_5_diff1": 0.4586274201403904, - "nauc_ndcg_at_5_max": 0.4175435252168309, - "nauc_ndcg_at_5_std": 0.44689833044121, + "nauc_cv_recall_at_100_diff1": 0.45408426850632555, + "nauc_cv_recall_at_100_max": 0.3222744028682529, + "nauc_cv_recall_at_100_std": 0.49769459949735695, + "nauc_cv_recall_at_10_diff1": 0.4583958619786553, + "nauc_cv_recall_at_10_max": 0.4087396058403719, + "nauc_cv_recall_at_10_std": 0.43819027041183795, + "nauc_cv_recall_at_1_diff1": 0.6080247200265034, + "nauc_cv_recall_at_1_max": 0.30684537908272097, + "nauc_cv_recall_at_1_std": 0.3190134282007699, + "nauc_cv_recall_at_20_diff1": 0.4408917344852856, + "nauc_cv_recall_at_20_max": 0.33116607618092503, + "nauc_cv_recall_at_20_std": 0.4635955898153606, + "nauc_cv_recall_at_3_diff1": 0.5135354143241001, + "nauc_cv_recall_at_3_max": 0.3805383547397707, + "nauc_cv_recall_at_3_std": 0.38879738582793566, + "nauc_cv_recall_at_5_diff1": 0.5147748046338926, + "nauc_cv_recall_at_5_max": 0.4052765679780838, + "nauc_cv_recall_at_5_std": 0.41168260840294335, + "nauc_map_at_1000_diff1": 0.5544641523789734, + "nauc_map_at_1000_max": 0.34660178516191575, + "nauc_map_at_1000_std": 0.36292664896805754, + "nauc_map_at_100_diff1": 0.5541714622401884, + "nauc_map_at_100_max": 0.3463162735362515, + "nauc_map_at_100_std": 0.36320862556883937, + "nauc_map_at_10_diff1": 0.5537212140716464, + "nauc_map_at_10_max": 0.3520368843333082, + "nauc_map_at_10_std": 0.36244548454203784, + "nauc_map_at_1_diff1": 0.6080247200265034, + "nauc_map_at_1_max": 0.30684537908272097, + "nauc_map_at_1_std": 0.3190134282007699, + "nauc_map_at_20_diff1": 0.5530202603716357, + "nauc_map_at_20_max": 0.3461352506565878, + "nauc_map_at_20_std": 0.3634854366460339, + "nauc_map_at_3_diff1": 0.56350963435392, + "nauc_map_at_3_max": 0.3450059374839173, + "nauc_map_at_3_std": 0.3531325707685181, + "nauc_map_at_5_diff1": 0.5631217566464541, + "nauc_map_at_5_max": 0.35100571736628244, + "nauc_map_at_5_std": 0.358638105621845, + "nauc_mrr_at_1000_diff1": 0.5544641523789734, + "nauc_mrr_at_1000_max": 0.34660178516191575, + "nauc_mrr_at_1000_std": 0.36292664896805754, + "nauc_mrr_at_100_diff1": 0.5541714622401884, + "nauc_mrr_at_100_max": 0.3463162735362515, + "nauc_mrr_at_100_std": 0.36320862556883937, + "nauc_mrr_at_10_diff1": 0.5537212140716464, + "nauc_mrr_at_10_max": 0.3520368843333082, + "nauc_mrr_at_10_std": 0.36244548454203784, + "nauc_mrr_at_1_diff1": 0.6080247200265034, + "nauc_mrr_at_1_max": 0.30684537908272097, + "nauc_mrr_at_1_std": 0.3190134282007699, + "nauc_mrr_at_20_diff1": 0.5530202603716357, + "nauc_mrr_at_20_max": 0.3461352506565878, + "nauc_mrr_at_20_std": 0.3634854366460339, + "nauc_mrr_at_3_diff1": 0.56350963435392, + "nauc_mrr_at_3_max": 0.3450059374839173, + "nauc_mrr_at_3_std": 0.3531325707685181, + "nauc_mrr_at_5_diff1": 0.5631217566464541, + "nauc_mrr_at_5_max": 0.35100571736628244, + "nauc_mrr_at_5_std": 0.358638105621845, + "nauc_ndcg_at_1000_diff1": 0.5409245673190076, + "nauc_ndcg_at_1000_max": 0.3502701629410077, + "nauc_ndcg_at_1000_std": 0.37632942100540007, + "nauc_ndcg_at_100_diff1": 0.5327132079954523, + "nauc_ndcg_at_100_max": 0.3465358980202094, + "nauc_ndcg_at_100_std": 0.38719600780661023, + "nauc_ndcg_at_10_diff1": 0.5292537182061523, + "nauc_ndcg_at_10_max": 0.3672590009334541, + "nauc_ndcg_at_10_std": 0.38183727402292117, + "nauc_ndcg_at_1_diff1": 0.6080247200265034, + "nauc_ndcg_at_1_max": 0.30684537908272097, + "nauc_ndcg_at_1_std": 0.3190134282007699, + "nauc_ndcg_at_20_diff1": 0.525953721140863, + "nauc_ndcg_at_20_max": 0.346494074796129, + "nauc_ndcg_at_20_std": 0.38703811605768795, + "nauc_ndcg_at_3_diff1": 0.5499370707195624, + "nauc_ndcg_at_3_max": 0.3547957954684597, + "nauc_ndcg_at_3_std": 0.36286921346909046, + "nauc_ndcg_at_5_diff1": 0.5497836024111007, + "nauc_ndcg_at_5_max": 0.3656060271257966, + "nauc_ndcg_at_5_std": 0.37283779755807595, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.29668674698795183, - "nauc_precision_at_100_max": 0.3333401967243981, - "nauc_precision_at_100_std": 0.4607668957078318, - "nauc_precision_at_10_diff1": 0.31766273367855735, - "nauc_precision_at_10_max": 0.36628985069296616, - "nauc_precision_at_10_std": 0.45197038583574883, - "nauc_precision_at_1_diff1": 0.5701058852499383, - "nauc_precision_at_1_max": 0.4710169908889438, - "nauc_precision_at_1_std": 0.45840030444807583, - "nauc_precision_at_20_diff1": 0.31733905265849477, - "nauc_precision_at_20_max": 0.3650870683762021, - "nauc_precision_at_20_std": 0.4626756631233986, - "nauc_precision_at_3_diff1": 0.4098228341204409, - "nauc_precision_at_3_max": 0.4006476960431175, - "nauc_precision_at_3_std": 0.44742055551108645, - "nauc_precision_at_5_diff1": 0.39356629703634816, - "nauc_precision_at_5_max": 0.3866567013925965, - "nauc_precision_at_5_std": 0.43803175668461447, + "nauc_precision_at_100_diff1": 0.4540842685063265, + "nauc_precision_at_100_max": 0.3222744028682524, + "nauc_precision_at_100_std": 0.4976945994973571, + "nauc_precision_at_10_diff1": 0.45839586197865506, + "nauc_precision_at_10_max": 0.4087396058403718, + "nauc_precision_at_10_std": 0.4381902704118379, + "nauc_precision_at_1_diff1": 0.6080247200265034, + "nauc_precision_at_1_max": 0.30684537908272097, + "nauc_precision_at_1_std": 0.3190134282007699, + "nauc_precision_at_20_diff1": 0.4408917344852855, + "nauc_precision_at_20_max": 0.33116607618092536, + "nauc_precision_at_20_std": 0.4635955898153608, + "nauc_precision_at_3_diff1": 0.5135354143241001, + "nauc_precision_at_3_max": 0.38053835473977055, + "nauc_precision_at_3_std": 0.3887973858279356, + "nauc_precision_at_5_diff1": 0.5147748046338925, + "nauc_precision_at_5_max": 0.405276567978084, + "nauc_precision_at_5_std": 0.41168260840294396, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.29668674698795183, - "nauc_recall_at_100_max": 0.3333401967243978, - "nauc_recall_at_100_std": 0.4607668957078311, - "nauc_recall_at_10_diff1": 0.3176627336785572, - "nauc_recall_at_10_max": 0.36628985069296605, - "nauc_recall_at_10_std": 0.45197038583574867, - "nauc_recall_at_1_diff1": 0.5701058852499383, - "nauc_recall_at_1_max": 0.4710169908889438, - "nauc_recall_at_1_std": 0.45840030444807583, - "nauc_recall_at_20_diff1": 0.3173390526584947, - "nauc_recall_at_20_max": 0.36508706837620225, - "nauc_recall_at_20_std": 0.4626756631233985, - "nauc_recall_at_3_diff1": 0.4098228341204408, - "nauc_recall_at_3_max": 0.40064769604311734, - "nauc_recall_at_3_std": 0.447420555511086, - "nauc_recall_at_5_diff1": 0.3935662970363478, - "nauc_recall_at_5_max": 0.3866567013925962, - "nauc_recall_at_5_std": 0.43803175668461425, - "ndcg_at_1": 0.13, - "ndcg_at_10": 0.19978, - "ndcg_at_100": 0.25958, - "ndcg_at_1000": 0.31147, - "ndcg_at_20": 0.22036, - "ndcg_at_3": 0.16545, - "ndcg_at_5": 0.18085, - "precision_at_1": 0.13, - "precision_at_10": 0.029, - "precision_at_100": 0.00592, + "nauc_recall_at_100_diff1": 0.45408426850632555, + "nauc_recall_at_100_max": 0.3222744028682529, + "nauc_recall_at_100_std": 0.49769459949735695, + "nauc_recall_at_10_diff1": 0.4583958619786553, + "nauc_recall_at_10_max": 0.4087396058403719, + "nauc_recall_at_10_std": 0.43819027041183795, + "nauc_recall_at_1_diff1": 0.6080247200265034, + "nauc_recall_at_1_max": 0.30684537908272097, + "nauc_recall_at_1_std": 0.3190134282007699, + "nauc_recall_at_20_diff1": 0.4408917344852856, + "nauc_recall_at_20_max": 0.33116607618092503, + "nauc_recall_at_20_std": 0.4635955898153606, + "nauc_recall_at_3_diff1": 0.5135354143241001, + "nauc_recall_at_3_max": 0.3805383547397707, + "nauc_recall_at_3_std": 0.38879738582793566, + "nauc_recall_at_5_diff1": 0.5147748046338926, + "nauc_recall_at_5_max": 0.4052765679780838, + "nauc_recall_at_5_std": 0.41168260840294335, + "ndcg_at_1": 0.222, + "ndcg_at_10": 0.33307, + "ndcg_at_100": 0.38911, + "ndcg_at_1000": 0.42415, + "ndcg_at_20": 0.35373, + "ndcg_at_3": 0.29652, + "ndcg_at_5": 0.31278, + "precision_at_1": 0.222, + "precision_at_10": 0.045, + "precision_at_100": 0.00726, "precision_at_1000": 0.001, - "precision_at_20": 0.0186, - "precision_at_3": 0.064, - "precision_at_5": 0.046, - "recall_at_1": 0.13, - "recall_at_10": 0.29, - "recall_at_100": 0.592, + "precision_at_20": 0.0266, + "precision_at_3": 0.116, + "precision_at_5": 0.0776, + "recall_at_1": 0.222, + "recall_at_10": 0.45, + "recall_at_100": 0.726, "recall_at_1000": 1.0, - "recall_at_20": 0.372, - "recall_at_3": 0.192, - "recall_at_5": 0.23 + "recall_at_20": 0.532, + "recall_at_3": 0.348, + "recall_at_5": 0.388 } ] }, diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json index aea7cf6a30..031ebfff4e 100644 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json @@ -1,184 +1,184 @@ { "dataset_revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", - "evaluation_time": 39.90301728248596, + "evaluation_time": 41.904603242874146, "kg_co2_emissions": null, "mteb_version": "1.14.15", "scores": { "test": [ { - "cv_recall_at_1": 0.258, - "cv_recall_at_10": 0.548, - "cv_recall_at_100": 0.814, + "cv_recall_at_1": 0.578, + "cv_recall_at_10": 0.842, + "cv_recall_at_100": 0.952, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.606, - "cv_recall_at_3": 0.378, - "cv_recall_at_5": 0.456, + "cv_recall_at_20": 0.886, + "cv_recall_at_3": 0.726, + "cv_recall_at_5": 0.792, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.36058, - "map_at_1": 0.258, - "map_at_10": 0.34146, - "map_at_100": 0.35092, - "map_at_1000": 0.35188, - "map_at_20": 0.34558, - "map_at_3": 0.31167, - "map_at_5": 0.32917, - "mrr_at_1": 0.258, - "mrr_at_10": 0.3414619047619047, - "mrr_at_100": 0.35091516817676005, - "mrr_at_1000": 0.351878944711423, - "mrr_at_20": 0.3455762584165062, - "mrr_at_3": 0.3116666666666667, - "mrr_at_5": 0.3291666666666666, + "main_score": 0.69273, + "map_at_1": 0.578, + "map_at_10": 0.6663, + "map_at_100": 0.67068, + "map_at_1000": 0.67097, + "map_at_20": 0.66919, + "map_at_3": 0.644, + "map_at_5": 0.6596, + "mrr_at_1": 0.578, + "mrr_at_10": 0.6663031746031746, + "mrr_at_100": 0.6706831128053965, + "mrr_at_1000": 0.6709717828298823, + "mrr_at_20": 0.6691926300823358, + "mrr_at_3": 0.644, + "mrr_at_5": 0.6595999999999997, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.22488058728615315, - "nauc_cv_recall_at_100_max": 0.34163790432023017, - "nauc_cv_recall_at_100_std": 0.4398715578684168, - "nauc_cv_recall_at_10_diff1": 0.3593018759002586, - "nauc_cv_recall_at_10_max": 0.43681797546252976, - "nauc_cv_recall_at_10_std": 0.3225292986812089, - "nauc_cv_recall_at_1_diff1": 0.5867050765441552, - "nauc_cv_recall_at_1_max": 0.5015622144280729, - "nauc_cv_recall_at_1_std": 0.22811521802827353, - "nauc_cv_recall_at_20_diff1": 0.32595345879120036, - "nauc_cv_recall_at_20_max": 0.41825288970087515, - "nauc_cv_recall_at_20_std": 0.36061786949621855, - "nauc_cv_recall_at_3_diff1": 0.4670428735336128, - "nauc_cv_recall_at_3_max": 0.49687144704354985, - "nauc_cv_recall_at_3_std": 0.2719720680710684, - "nauc_cv_recall_at_5_diff1": 0.38826252637546027, - "nauc_cv_recall_at_5_max": 0.4553113091676377, - "nauc_cv_recall_at_5_std": 0.29645755256721124, - "nauc_map_at_1000_diff1": 0.5094053800617759, - "nauc_map_at_1000_max": 0.489293114100951, - "nauc_map_at_1000_std": 0.2638562098259416, - "nauc_map_at_100_diff1": 0.5085268178285087, - "nauc_map_at_100_max": 0.4889533291842021, - "nauc_map_at_100_std": 0.26436130382480405, - "nauc_map_at_10_diff1": 0.5091516903713251, - "nauc_map_at_10_max": 0.4899516623083387, - "nauc_map_at_10_std": 0.2610633947620562, - "nauc_map_at_1_diff1": 0.5867050765441552, - "nauc_map_at_1_max": 0.5015622144280729, - "nauc_map_at_1_std": 0.22811521802827353, - "nauc_map_at_20_diff1": 0.5081365189697196, - "nauc_map_at_20_max": 0.48887071333338156, - "nauc_map_at_20_std": 0.2634137688547193, - "nauc_map_at_3_diff1": 0.5306475249114199, - "nauc_map_at_3_max": 0.5025297847699206, - "nauc_map_at_3_std": 0.25148826382598116, - "nauc_map_at_5_diff1": 0.5120080392776135, - "nauc_map_at_5_max": 0.49271028166183345, - "nauc_map_at_5_std": 0.2584032846191391, - "nauc_mrr_at_1000_diff1": 0.5094053800617759, - "nauc_mrr_at_1000_max": 0.489293114100951, - "nauc_mrr_at_1000_std": 0.2638562098259416, - "nauc_mrr_at_100_diff1": 0.5085268178285087, - "nauc_mrr_at_100_max": 0.4889533291842021, - "nauc_mrr_at_100_std": 0.26436130382480405, - "nauc_mrr_at_10_diff1": 0.5091516903713251, - "nauc_mrr_at_10_max": 0.4899516623083387, - "nauc_mrr_at_10_std": 0.2610633947620562, - "nauc_mrr_at_1_diff1": 0.5867050765441552, - "nauc_mrr_at_1_max": 0.5015622144280729, - "nauc_mrr_at_1_std": 0.22811521802827353, - "nauc_mrr_at_20_diff1": 0.5081365189697196, - "nauc_mrr_at_20_max": 0.48887071333338156, - "nauc_mrr_at_20_std": 0.2634137688547193, - "nauc_mrr_at_3_diff1": 0.5306475249114199, - "nauc_mrr_at_3_max": 0.5025297847699206, - "nauc_mrr_at_3_std": 0.25148826382598116, - "nauc_mrr_at_5_diff1": 0.5120080392776135, - "nauc_mrr_at_5_max": 0.49271028166183345, - "nauc_mrr_at_5_std": 0.2584032846191391, - "nauc_ndcg_at_1000_diff1": 0.48845200236250697, - "nauc_ndcg_at_1000_max": 0.48102900753840117, - "nauc_ndcg_at_1000_std": 0.2758573747125474, - "nauc_ndcg_at_100_diff1": 0.46824118560402356, - "nauc_ndcg_at_100_max": 0.4709534856151632, - "nauc_ndcg_at_100_std": 0.28812850918887767, - "nauc_ndcg_at_10_diff1": 0.47290459879999414, - "nauc_ndcg_at_10_max": 0.47758748755214875, - "nauc_ndcg_at_10_std": 0.2757346332309286, - "nauc_ndcg_at_1_diff1": 0.5867050765441552, - "nauc_ndcg_at_1_max": 0.5015622144280729, - "nauc_ndcg_at_1_std": 0.22811521802827353, - "nauc_ndcg_at_20_diff1": 0.4674553529067452, - "nauc_ndcg_at_20_max": 0.4736917036654822, - "nauc_ndcg_at_20_std": 0.28420357989357237, - "nauc_ndcg_at_3_diff1": 0.5136193330507195, - "nauc_ndcg_at_3_max": 0.5011469410434867, - "nauc_ndcg_at_3_std": 0.25709010460637316, - "nauc_ndcg_at_5_diff1": 0.4802867776472517, - "nauc_ndcg_at_5_max": 0.483545793523169, - "nauc_ndcg_at_5_std": 0.2684200549296203, + "nauc_cv_recall_at_100_diff1": 0.7016612200435727, + "nauc_cv_recall_at_100_max": 0.3388966697790251, + "nauc_cv_recall_at_100_std": 0.7045790538437611, + "nauc_cv_recall_at_10_diff1": 0.6544153248333311, + "nauc_cv_recall_at_10_max": 0.5501061465691686, + "nauc_cv_recall_at_10_std": 0.3852996312803394, + "nauc_cv_recall_at_1_diff1": 0.7563944784370051, + "nauc_cv_recall_at_1_max": 0.5450704673985117, + "nauc_cv_recall_at_1_std": 0.09203473708290744, + "nauc_cv_recall_at_20_diff1": 0.6634424892203701, + "nauc_cv_recall_at_20_max": 0.5504020510430014, + "nauc_cv_recall_at_20_std": 0.47475319226863366, + "nauc_cv_recall_at_3_diff1": 0.6899187768752989, + "nauc_cv_recall_at_3_max": 0.5278337780988896, + "nauc_cv_recall_at_3_std": 0.15949409374647983, + "nauc_cv_recall_at_5_diff1": 0.6596313739170878, + "nauc_cv_recall_at_5_max": 0.5408405527453144, + "nauc_cv_recall_at_5_std": 0.27152741438455646, + "nauc_map_at_1000_diff1": 0.728329825559897, + "nauc_map_at_1000_max": 0.5341727992904414, + "nauc_map_at_1000_std": 0.1337047475778167, + "nauc_map_at_100_diff1": 0.7282640631192274, + "nauc_map_at_100_max": 0.5340833390642327, + "nauc_map_at_100_std": 0.13428744788967067, + "nauc_map_at_10_diff1": 0.7269917994209238, + "nauc_map_at_10_max": 0.5352397762689725, + "nauc_map_at_10_std": 0.1339785516273345, + "nauc_map_at_1_diff1": 0.7563944784370051, + "nauc_map_at_1_max": 0.5450704673985117, + "nauc_map_at_1_std": 0.09203473708290744, + "nauc_map_at_20_diff1": 0.7277237287287605, + "nauc_map_at_20_max": 0.5350192237473728, + "nauc_map_at_20_std": 0.13471990708577683, + "nauc_map_at_3_diff1": 0.7292858632329362, + "nauc_map_at_3_max": 0.5331904060611592, + "nauc_map_at_3_std": 0.11234018428495053, + "nauc_map_at_5_diff1": 0.7266994608094846, + "nauc_map_at_5_max": 0.5354412003777747, + "nauc_map_at_5_std": 0.12957191027012394, + "nauc_mrr_at_1000_diff1": 0.728329825559897, + "nauc_mrr_at_1000_max": 0.5341727992904414, + "nauc_mrr_at_1000_std": 0.1337047475778167, + "nauc_mrr_at_100_diff1": 0.7282640631192274, + "nauc_mrr_at_100_max": 0.5340833390642327, + "nauc_mrr_at_100_std": 0.13428744788967067, + "nauc_mrr_at_10_diff1": 0.7269917994209238, + "nauc_mrr_at_10_max": 0.5352397762689725, + "nauc_mrr_at_10_std": 0.1339785516273345, + "nauc_mrr_at_1_diff1": 0.7563944784370051, + "nauc_mrr_at_1_max": 0.5450704673985117, + "nauc_mrr_at_1_std": 0.09203473708290744, + "nauc_mrr_at_20_diff1": 0.7277237287287605, + "nauc_mrr_at_20_max": 0.5350192237473728, + "nauc_mrr_at_20_std": 0.13471990708577683, + "nauc_mrr_at_3_diff1": 0.7292858632329362, + "nauc_mrr_at_3_max": 0.5331904060611592, + "nauc_mrr_at_3_std": 0.11234018428495053, + "nauc_mrr_at_5_diff1": 0.7266994608094846, + "nauc_mrr_at_5_max": 0.5354412003777747, + "nauc_mrr_at_5_std": 0.12957191027012394, + "nauc_ndcg_at_1000_diff1": 0.7236097326031249, + "nauc_ndcg_at_1000_max": 0.5320654575443786, + "nauc_ndcg_at_1000_std": 0.15264521875599046, + "nauc_ndcg_at_100_diff1": 0.7228017117153069, + "nauc_ndcg_at_100_max": 0.5271870696372063, + "nauc_ndcg_at_100_std": 0.16853775142739957, + "nauc_ndcg_at_10_diff1": 0.7151002125670103, + "nauc_ndcg_at_10_max": 0.5367706024907527, + "nauc_ndcg_at_10_std": 0.1710856158422823, + "nauc_ndcg_at_1_diff1": 0.7563944784370051, + "nauc_ndcg_at_1_max": 0.5450704673985117, + "nauc_ndcg_at_1_std": 0.09203473708290744, + "nauc_ndcg_at_20_diff1": 0.7183893546557805, + "nauc_ndcg_at_20_max": 0.5361165591354897, + "nauc_ndcg_at_20_std": 0.1739805415327144, + "nauc_ndcg_at_3_diff1": 0.7203283060605159, + "nauc_ndcg_at_3_max": 0.5317435162233147, + "nauc_ndcg_at_3_std": 0.12273087229898909, + "nauc_ndcg_at_5_diff1": 0.713888248116884, + "nauc_ndcg_at_5_max": 0.536055499438798, + "nauc_ndcg_at_5_std": 0.15592804987391928, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.2248805872861516, - "nauc_precision_at_100_max": 0.3416379043202278, - "nauc_precision_at_100_std": 0.4398715578684146, - "nauc_precision_at_10_diff1": 0.3593018759002583, - "nauc_precision_at_10_max": 0.43681797546252954, - "nauc_precision_at_10_std": 0.32252929868120905, - "nauc_precision_at_1_diff1": 0.5867050765441552, - "nauc_precision_at_1_max": 0.5015622144280729, - "nauc_precision_at_1_std": 0.22811521802827353, - "nauc_precision_at_20_diff1": 0.3259534587912, - "nauc_precision_at_20_max": 0.4182528897008752, - "nauc_precision_at_20_std": 0.3606178694962182, - "nauc_precision_at_3_diff1": 0.46704287353361273, - "nauc_precision_at_3_max": 0.49687144704354985, - "nauc_precision_at_3_std": 0.2719720680710683, - "nauc_precision_at_5_diff1": 0.38826252637546066, - "nauc_precision_at_5_max": 0.4553113091676376, - "nauc_precision_at_5_std": 0.29645755256721196, + "nauc_precision_at_100_diff1": 0.701661220043572, + "nauc_precision_at_100_max": 0.3388966697790239, + "nauc_precision_at_100_std": 0.7045790538437579, + "nauc_precision_at_10_diff1": 0.6544153248333306, + "nauc_precision_at_10_max": 0.5501061465691688, + "nauc_precision_at_10_std": 0.3852996312803368, + "nauc_precision_at_1_diff1": 0.7563944784370051, + "nauc_precision_at_1_max": 0.5450704673985117, + "nauc_precision_at_1_std": 0.09203473708290744, + "nauc_precision_at_20_diff1": 0.6634424892203703, + "nauc_precision_at_20_max": 0.5504020510430019, + "nauc_precision_at_20_std": 0.47475319226863155, + "nauc_precision_at_3_diff1": 0.6899187768752989, + "nauc_precision_at_3_max": 0.5278337780988899, + "nauc_precision_at_3_std": 0.15949409374648, + "nauc_precision_at_5_diff1": 0.6596313739170871, + "nauc_precision_at_5_max": 0.540840552745314, + "nauc_precision_at_5_std": 0.27152741438455674, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.22488058728615315, - "nauc_recall_at_100_max": 0.34163790432023017, - "nauc_recall_at_100_std": 0.4398715578684168, - "nauc_recall_at_10_diff1": 0.3593018759002586, - "nauc_recall_at_10_max": 0.43681797546252976, - "nauc_recall_at_10_std": 0.3225292986812089, - "nauc_recall_at_1_diff1": 0.5867050765441552, - "nauc_recall_at_1_max": 0.5015622144280729, - "nauc_recall_at_1_std": 0.22811521802827353, - "nauc_recall_at_20_diff1": 0.32595345879120036, - "nauc_recall_at_20_max": 0.41825288970087515, - "nauc_recall_at_20_std": 0.36061786949621855, - "nauc_recall_at_3_diff1": 0.4670428735336128, - "nauc_recall_at_3_max": 0.49687144704354985, - "nauc_recall_at_3_std": 0.2719720680710684, - "nauc_recall_at_5_diff1": 0.38826252637546027, - "nauc_recall_at_5_max": 0.4553113091676377, - "nauc_recall_at_5_std": 0.29645755256721124, - "ndcg_at_1": 0.258, - "ndcg_at_10": 0.39035, - "ndcg_at_100": 0.44372, - "ndcg_at_1000": 0.46792, - "ndcg_at_20": 0.40512, - "ndcg_at_3": 0.32874, - "ndcg_at_5": 0.36058, - "precision_at_1": 0.258, - "precision_at_10": 0.0548, - "precision_at_100": 0.00814, + "nauc_recall_at_100_diff1": 0.7016612200435727, + "nauc_recall_at_100_max": 0.3388966697790251, + "nauc_recall_at_100_std": 0.7045790538437611, + "nauc_recall_at_10_diff1": 0.6544153248333311, + "nauc_recall_at_10_max": 0.5501061465691686, + "nauc_recall_at_10_std": 0.3852996312803394, + "nauc_recall_at_1_diff1": 0.7563944784370051, + "nauc_recall_at_1_max": 0.5450704673985117, + "nauc_recall_at_1_std": 0.09203473708290744, + "nauc_recall_at_20_diff1": 0.6634424892203701, + "nauc_recall_at_20_max": 0.5504020510430014, + "nauc_recall_at_20_std": 0.47475319226863366, + "nauc_recall_at_3_diff1": 0.6899187768752989, + "nauc_recall_at_3_max": 0.5278337780988896, + "nauc_recall_at_3_std": 0.15949409374647983, + "nauc_recall_at_5_diff1": 0.6596313739170878, + "nauc_recall_at_5_max": 0.5408405527453144, + "nauc_recall_at_5_std": 0.27152741438455646, + "ndcg_at_1": 0.578, + "ndcg_at_10": 0.70893, + "ndcg_at_100": 0.73168, + "ndcg_at_1000": 0.73811, + "ndcg_at_20": 0.71984, + "ndcg_at_3": 0.66509, + "ndcg_at_5": 0.69273, + "precision_at_1": 0.578, + "precision_at_10": 0.0842, + "precision_at_100": 0.00952, "precision_at_1000": 0.001, - "precision_at_20": 0.0303, - "precision_at_3": 0.126, - "precision_at_5": 0.0912, - "recall_at_1": 0.258, - "recall_at_10": 0.548, - "recall_at_100": 0.814, + "precision_at_20": 0.0443, + "precision_at_3": 0.242, + "precision_at_5": 0.1584, + "recall_at_1": 0.578, + "recall_at_10": 0.842, + "recall_at_100": 0.952, "recall_at_1000": 1.0, - "recall_at_20": 0.606, - "recall_at_3": 0.378, - "recall_at_5": 0.456 + "recall_at_20": 0.886, + "recall_at_3": 0.726, + "recall_at_5": 0.792 } ] }, diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json index b417221b6f..374bde6d4e 100644 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json @@ -1,6 +1,6 @@ { "dataset_revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", - "evaluation_time": 75.55646324157715, + "evaluation_time": 76.06725120544434, "kg_co2_emissions": null, "mteb_version": "1.14.15", "scores": { diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json index 057976d0ba..1836f819bb 100644 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json @@ -1,184 +1,184 @@ { "dataset_revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", - "evaluation_time": 74.40851640701294, + "evaluation_time": 75.31401538848877, "kg_co2_emissions": null, "mteb_version": "1.14.15", "scores": { "test": [ { - "cv_recall_at_1": 0.23, - "cv_recall_at_10": 0.53, - "cv_recall_at_100": 0.76, + "cv_recall_at_1": 0.39, + "cv_recall_at_10": 0.85, + "cv_recall_at_100": 0.95, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.61, - "cv_recall_at_3": 0.38, - "cv_recall_at_5": 0.43, + "cv_recall_at_20": 0.89, + "cv_recall_at_3": 0.67, + "cv_recall_at_5": 0.76, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.327, - "map_at_1": 0.21, - "map_at_10": 0.30566, - "map_at_100": 0.31547, - "map_at_1000": 0.31657, - "map_at_20": 0.31133, - "map_at_3": 0.28167, - "map_at_5": 0.29267, - "mrr_at_1": 0.23, - "mrr_at_10": 0.3156746031746031, - "mrr_at_100": 0.3254601138348874, - "mrr_at_1000": 0.3265610689913019, - "mrr_at_20": 0.3213241758241757, - "mrr_at_3": 0.29166666666666663, - "mrr_at_5": 0.30316666666666664, + "main_score": 0.58256, + "map_at_1": 0.36, + "map_at_10": 0.53621, + "map_at_100": 0.53964, + "map_at_1000": 0.53983, + "map_at_20": 0.53815, + "map_at_3": 0.50167, + "map_at_5": 0.52267, + "mrr_at_1": 0.39, + "mrr_at_10": 0.546825396825397, + "mrr_at_100": 0.5511443104244275, + "mrr_at_1000": 0.5513394682372231, + "mrr_at_20": 0.5496709956709959, + "mrr_at_3": 0.5133333333333333, + "mrr_at_5": 0.5348333333333334, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.006359781950334248, - "nauc_cv_recall_at_100_max": 0.5430691355888204, - "nauc_cv_recall_at_100_std": 0.4596997490698272, - "nauc_cv_recall_at_10_diff1": 0.08308017979628572, - "nauc_cv_recall_at_10_max": 0.5721453215851762, - "nauc_cv_recall_at_10_std": 0.2545189624599649, - "nauc_cv_recall_at_1_diff1": 0.17789539512820113, - "nauc_cv_recall_at_1_max": 0.43204038986592636, - "nauc_cv_recall_at_1_std": -0.009472094690441003, - "nauc_cv_recall_at_20_diff1": -0.02098409197347156, - "nauc_cv_recall_at_20_max": 0.5650748568579761, - "nauc_cv_recall_at_20_std": 0.3741181091656218, - "nauc_cv_recall_at_3_diff1": 0.10990824734229203, - "nauc_cv_recall_at_3_max": 0.616510279936579, - "nauc_cv_recall_at_3_std": 0.1248148051880541, - "nauc_cv_recall_at_5_diff1": 0.0806986513376077, - "nauc_cv_recall_at_5_max": 0.6248585660220314, - "nauc_cv_recall_at_5_std": 0.22141733102703817, - "nauc_map_at_1000_diff1": 0.1526572027006709, - "nauc_map_at_1000_max": 0.5299962487999986, - "nauc_map_at_1000_std": 0.07656699308819963, - "nauc_map_at_100_diff1": 0.15241863064910316, - "nauc_map_at_100_max": 0.5301663175741202, - "nauc_map_at_100_std": 0.0776350535363974, - "nauc_map_at_10_diff1": 0.15792022337662764, - "nauc_map_at_10_max": 0.5298177356615489, - "nauc_map_at_10_std": 0.06920520130657719, - "nauc_map_at_1_diff1": 0.2525377294249925, - "nauc_map_at_1_max": 0.45356073922751683, - "nauc_map_at_1_std": -0.04944019510032149, - "nauc_map_at_20_diff1": 0.15111374679870085, - "nauc_map_at_20_max": 0.529961239541217, - "nauc_map_at_20_std": 0.07729346018356126, - "nauc_map_at_3_diff1": 0.16157692832993492, - "nauc_map_at_3_max": 0.5360219292045154, - "nauc_map_at_3_std": 0.04172107834562945, - "nauc_map_at_5_diff1": 0.15570547539001464, - "nauc_map_at_5_max": 0.53800624291612, - "nauc_map_at_5_std": 0.06658104533395676, - "nauc_mrr_at_1000_diff1": 0.11823237600605414, - "nauc_mrr_at_1000_max": 0.5212691085515011, - "nauc_mrr_at_1000_std": 0.09520643817366876, - "nauc_mrr_at_100_diff1": 0.11809775190018582, - "nauc_mrr_at_100_max": 0.5214609438591693, - "nauc_mrr_at_100_std": 0.09620581011777922, - "nauc_mrr_at_10_diff1": 0.12388271932408514, - "nauc_mrr_at_10_max": 0.5215436616204162, - "nauc_mrr_at_10_std": 0.08789608700703568, - "nauc_mrr_at_1_diff1": 0.17789539512820113, - "nauc_mrr_at_1_max": 0.43204038986592636, - "nauc_mrr_at_1_std": -0.009472094690441003, - "nauc_mrr_at_20_diff1": 0.11709458086096404, - "nauc_mrr_at_20_max": 0.5212632072880561, - "nauc_mrr_at_20_std": 0.09564320422360603, - "nauc_mrr_at_3_diff1": 0.12855140902292847, - "nauc_mrr_at_3_max": 0.5246464194448699, - "nauc_mrr_at_3_std": 0.05868082951588032, - "nauc_mrr_at_5_diff1": 0.12130166158692024, - "nauc_mrr_at_5_max": 0.5293317976745179, - "nauc_mrr_at_5_std": 0.0848690117724686, - "nauc_ndcg_at_1000_diff1": 0.13372164670347136, - "nauc_ndcg_at_1000_max": 0.538140945351461, - "nauc_ndcg_at_1000_std": 0.11386350497056077, - "nauc_ndcg_at_100_diff1": 0.12258777943022443, - "nauc_ndcg_at_100_max": 0.539138531606273, - "nauc_ndcg_at_100_std": 0.14404291133909553, - "nauc_ndcg_at_10_diff1": 0.1379040925491282, - "nauc_ndcg_at_10_max": 0.5429870060366613, - "nauc_ndcg_at_10_std": 0.11630716757265272, - "nauc_ndcg_at_1_diff1": 0.2525377294249925, - "nauc_ndcg_at_1_max": 0.45356073922751683, - "nauc_ndcg_at_1_std": -0.04944019510032149, - "nauc_ndcg_at_20_diff1": 0.11338858058945903, - "nauc_ndcg_at_20_max": 0.5416827102797607, - "nauc_ndcg_at_20_std": 0.1438549019021181, - "nauc_ndcg_at_3_diff1": 0.14642325849636342, - "nauc_ndcg_at_3_max": 0.5583865766196738, - "nauc_ndcg_at_3_std": 0.06493605013926239, - "nauc_ndcg_at_5_diff1": 0.1347770230227918, - "nauc_ndcg_at_5_max": 0.5619629664272141, - "nauc_ndcg_at_5_std": 0.10811411065414367, + "nauc_cv_recall_at_100_diff1": -0.4458450046685341, + "nauc_cv_recall_at_100_max": 0.7343604108309969, + "nauc_cv_recall_at_100_std": 0.16321195144724712, + "nauc_cv_recall_at_10_diff1": -0.5618288317604945, + "nauc_cv_recall_at_10_max": 0.6819720143182559, + "nauc_cv_recall_at_10_std": 0.1256752359258052, + "nauc_cv_recall_at_1_diff1": 0.1994748619249027, + "nauc_cv_recall_at_1_max": 0.14802167811364178, + "nauc_cv_recall_at_1_std": -0.07441180655258509, + "nauc_cv_recall_at_20_diff1": -0.6063482518683952, + "nauc_cv_recall_at_20_max": 0.675972854565759, + "nauc_cv_recall_at_20_std": 0.21076368009621177, + "nauc_cv_recall_at_3_diff1": -0.23034991789541448, + "nauc_cv_recall_at_3_max": 0.37395676389429633, + "nauc_cv_recall_at_3_std": -0.050566267711737146, + "nauc_cv_recall_at_5_diff1": -0.2816258544605003, + "nauc_cv_recall_at_5_max": 0.5499264515012549, + "nauc_cv_recall_at_5_std": -0.1023838366358055, + "nauc_map_at_1000_diff1": 0.06088734314901716, + "nauc_map_at_1000_max": 0.2936805589234933, + "nauc_map_at_1000_std": -0.047660425251865916, + "nauc_map_at_100_diff1": 0.06058775434305587, + "nauc_map_at_100_max": 0.2939707353087287, + "nauc_map_at_100_std": -0.047454525599785205, + "nauc_map_at_10_diff1": 0.05340004843355785, + "nauc_map_at_10_max": 0.2973404838090291, + "nauc_map_at_10_std": -0.04463528240739384, + "nauc_map_at_1_diff1": 0.2620715897758883, + "nauc_map_at_1_max": 0.2259486695008659, + "nauc_map_at_1_std": -0.07092058993334362, + "nauc_map_at_20_diff1": 0.055767839463411815, + "nauc_map_at_20_max": 0.29547949618137764, + "nauc_map_at_20_std": -0.04665077114633524, + "nauc_map_at_3_diff1": 0.05175393143339162, + "nauc_map_at_3_max": 0.26573236127010896, + "nauc_map_at_3_std": -0.06059167319395071, + "nauc_map_at_5_diff1": 0.05975930657081366, + "nauc_map_at_5_max": 0.2964074299438267, + "nauc_map_at_5_std": -0.0660526761176348, + "nauc_mrr_at_1000_diff1": 0.026947462345755967, + "nauc_mrr_at_1000_max": 0.2523382034693512, + "nauc_mrr_at_1000_std": -0.04482570085671642, + "nauc_mrr_at_100_diff1": 0.02667279807328532, + "nauc_mrr_at_100_max": 0.25265291128513884, + "nauc_mrr_at_100_std": -0.04462221468978824, + "nauc_mrr_at_10_diff1": 0.018220797258613936, + "nauc_mrr_at_10_max": 0.2578423858447704, + "nauc_mrr_at_10_std": -0.04528897392769788, + "nauc_mrr_at_1_diff1": 0.1994748619249027, + "nauc_mrr_at_1_max": 0.14802167811364178, + "nauc_mrr_at_1_std": -0.07441180655258509, + "nauc_mrr_at_20_diff1": 0.022114085998288655, + "nauc_mrr_at_20_max": 0.2542766239097509, + "nauc_mrr_at_20_std": -0.04391458858168538, + "nauc_mrr_at_3_diff1": 0.025988536361080748, + "nauc_mrr_at_3_max": 0.23390397415678407, + "nauc_mrr_at_3_std": -0.05373521185814806, + "nauc_mrr_at_5_diff1": 0.027160692501750364, + "nauc_mrr_at_5_max": 0.2576516650528352, + "nauc_mrr_at_5_std": -0.06194421533050859, + "nauc_ndcg_at_1000_diff1": 0.025484026028548535, + "nauc_ndcg_at_1000_max": 0.3155648907709535, + "nauc_ndcg_at_1000_std": -0.03736254089983462, + "nauc_ndcg_at_100_diff1": 0.013805477940072674, + "nauc_ndcg_at_100_max": 0.32603093853073645, + "nauc_ndcg_at_100_std": -0.03205847590051636, + "nauc_ndcg_at_10_diff1": -0.02877794636442936, + "nauc_ndcg_at_10_max": 0.3461160999719953, + "nauc_ndcg_at_10_std": -0.014302025640523356, + "nauc_ndcg_at_1_diff1": 0.2620715897758883, + "nauc_ndcg_at_1_max": 0.2259486695008659, + "nauc_ndcg_at_1_std": -0.07092058993334362, + "nauc_ndcg_at_20_diff1": -0.020531864489671144, + "nauc_ndcg_at_20_max": 0.338473683828385, + "nauc_ndcg_at_20_std": -0.02162764134437804, + "nauc_ndcg_at_3_diff1": -0.017610177016920022, + "nauc_ndcg_at_3_max": 0.2829562771372745, + "nauc_ndcg_at_3_std": -0.059480770234221464, + "nauc_ndcg_at_5_diff1": -0.007820671311857869, + "nauc_ndcg_at_5_max": 0.3432814823079952, + "nauc_ndcg_at_5_std": -0.07202897142385868, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.006359781950332713, - "nauc_precision_at_100_max": 0.5430691355888206, - "nauc_precision_at_100_std": 0.4596997490698282, - "nauc_precision_at_10_diff1": 0.08308017979628633, - "nauc_precision_at_10_max": 0.5721453215851767, - "nauc_precision_at_10_std": 0.2545189624599656, - "nauc_precision_at_1_diff1": 0.2525377294249925, - "nauc_precision_at_1_max": 0.45356073922751683, - "nauc_precision_at_1_std": -0.04944019510032149, - "nauc_precision_at_20_diff1": -0.020984091973470707, - "nauc_precision_at_20_max": 0.565074856857976, - "nauc_precision_at_20_std": 0.37411810916562194, - "nauc_precision_at_3_diff1": 0.10990824734229175, - "nauc_precision_at_3_max": 0.6165102799365789, - "nauc_precision_at_3_std": 0.12481480518805382, - "nauc_precision_at_5_diff1": 0.08069865133760828, - "nauc_precision_at_5_max": 0.6248585660220317, - "nauc_precision_at_5_std": 0.22141733102703876, + "nauc_precision_at_100_diff1": -0.4458450046685348, + "nauc_precision_at_100_max": 0.734360410831003, + "nauc_precision_at_100_std": 0.16321195144724487, + "nauc_precision_at_10_diff1": -0.5562560620756525, + "nauc_precision_at_10_max": 0.6723707911874733, + "nauc_precision_at_10_std": 0.21923929610641618, + "nauc_precision_at_1_diff1": 0.2620715897758883, + "nauc_precision_at_1_max": 0.2259486695008659, + "nauc_precision_at_1_std": -0.07092058993334362, + "nauc_precision_at_20_diff1": -0.6063482518683938, + "nauc_precision_at_20_max": 0.6759728545657586, + "nauc_precision_at_20_std": 0.21076368009621038, + "nauc_precision_at_3_diff1": -0.24954715511841657, + "nauc_precision_at_3_max": 0.342147585109445, + "nauc_precision_at_3_std": -0.05666062873491173, + "nauc_precision_at_5_diff1": -0.2816258544604995, + "nauc_precision_at_5_max": 0.5499264515012546, + "nauc_precision_at_5_std": -0.10238383663580512, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.006359781950334248, - "nauc_recall_at_100_max": 0.5430691355888204, - "nauc_recall_at_100_std": 0.4596997490698272, - "nauc_recall_at_10_diff1": 0.08308017979628572, - "nauc_recall_at_10_max": 0.5721453215851762, - "nauc_recall_at_10_std": 0.2545189624599649, - "nauc_recall_at_1_diff1": 0.2525377294249925, - "nauc_recall_at_1_max": 0.45356073922751683, - "nauc_recall_at_1_std": -0.04944019510032149, - "nauc_recall_at_20_diff1": -0.02098409197347156, - "nauc_recall_at_20_max": 0.5650748568579761, - "nauc_recall_at_20_std": 0.3741181091656218, - "nauc_recall_at_3_diff1": 0.10990824734229203, - "nauc_recall_at_3_max": 0.616510279936579, - "nauc_recall_at_3_std": 0.1248148051880541, - "nauc_recall_at_5_diff1": 0.0806986513376077, - "nauc_recall_at_5_max": 0.6248585660220314, - "nauc_recall_at_5_std": 0.22141733102703817, - "ndcg_at_1": 0.21, - "ndcg_at_10": 0.35903, - "ndcg_at_100": 0.40793, - "ndcg_at_1000": 0.43806, - "ndcg_at_20": 0.37941, - "ndcg_at_3": 0.30678, - "ndcg_at_5": 0.327, - "precision_at_1": 0.21, - "precision_at_10": 0.053, - "precision_at_100": 0.0076, + "nauc_recall_at_100_diff1": -0.4458450046685341, + "nauc_recall_at_100_max": 0.7343604108309969, + "nauc_recall_at_100_std": 0.16321195144724712, + "nauc_recall_at_10_diff1": -0.5562560620756545, + "nauc_recall_at_10_max": 0.6723707911874733, + "nauc_recall_at_10_std": 0.21923929610641552, + "nauc_recall_at_1_diff1": 0.2620715897758883, + "nauc_recall_at_1_max": 0.2259486695008659, + "nauc_recall_at_1_std": -0.07092058993334362, + "nauc_recall_at_20_diff1": -0.6063482518683952, + "nauc_recall_at_20_max": 0.675972854565759, + "nauc_recall_at_20_std": 0.21076368009621177, + "nauc_recall_at_3_diff1": -0.24954715511841757, + "nauc_recall_at_3_max": 0.3421475851094447, + "nauc_recall_at_3_std": -0.05666062873491272, + "nauc_recall_at_5_diff1": -0.2816258544605003, + "nauc_recall_at_5_max": 0.5499264515012549, + "nauc_recall_at_5_std": -0.1023838366358055, + "ndcg_at_1": 0.36, + "ndcg_at_10": 0.61511, + "ndcg_at_100": 0.63354, + "ndcg_at_1000": 0.63964, + "ndcg_at_20": 0.62252, + "ndcg_at_3": 0.54511, + "ndcg_at_5": 0.58256, + "precision_at_1": 0.36, + "precision_at_10": 0.086, + "precision_at_100": 0.0095, "precision_at_1000": 0.001, - "precision_at_20": 0.0305, - "precision_at_3": 0.12667, - "precision_at_5": 0.086, - "recall_at_1": 0.21, - "recall_at_10": 0.53, - "recall_at_100": 0.76, + "precision_at_20": 0.0445, + "precision_at_3": 0.22333, + "precision_at_5": 0.152, + "recall_at_1": 0.36, + "recall_at_10": 0.86, + "recall_at_100": 0.95, "recall_at_1000": 1.0, - "recall_at_20": 0.61, - "recall_at_3": 0.38, - "recall_at_5": 0.43 + "recall_at_20": 0.89, + "recall_at_3": 0.67, + "recall_at_5": 0.76 } ] }, diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json index 62f44290d2..d2071daf29 100644 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json @@ -1,184 +1,184 @@ { "dataset_revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", - "evaluation_time": 73.33681082725525, + "evaluation_time": 74.29262661933899, "kg_co2_emissions": null, "mteb_version": "1.14.15", "scores": { "test": [ { - "cv_recall_at_1": 0.28, - "cv_recall_at_10": 0.67, - "cv_recall_at_100": 0.92, + "cv_recall_at_1": 0.47, + "cv_recall_at_10": 0.87, + "cv_recall_at_100": 0.98, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.71, - "cv_recall_at_3": 0.5, - "cv_recall_at_5": 0.58, + "cv_recall_at_20": 0.92, + "cv_recall_at_3": 0.76, + "cv_recall_at_5": 0.83, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.44845, - "map_at_1": 0.28, - "map_at_10": 0.41726, - "map_at_100": 0.425, - "map_at_1000": 0.42549, - "map_at_20": 0.42028, - "map_at_3": 0.38667, - "map_at_5": 0.40417, - "mrr_at_1": 0.28, - "mrr_at_10": 0.41671428571428576, - "mrr_at_100": 0.4244418064279869, - "mrr_at_1000": 0.42492739397981305, - "mrr_at_20": 0.4196951284010108, - "mrr_at_3": 0.38500000000000006, - "mrr_at_5": 0.4035, + "main_score": 0.65462, + "map_at_1": 0.41, + "map_at_10": 0.59626, + "map_at_100": 0.60145, + "map_at_1000": 0.60156, + "map_at_20": 0.59975, + "map_at_3": 0.575, + "map_at_5": 0.5915, + "mrr_at_1": 0.47, + "mrr_at_10": 0.6217619047619047, + "mrr_at_100": 0.6269392738934335, + "mrr_at_1000": 0.6270515118502697, + "mrr_at_20": 0.6252543290043291, + "mrr_at_3": 0.5983333333333334, + "mrr_at_5": 0.6153333333333333, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.05718954248365932, - "nauc_cv_recall_at_100_max": 0.5615662931839402, - "nauc_cv_recall_at_100_std": 0.7516923436041081, - "nauc_cv_recall_at_10_diff1": -0.19090585905097338, - "nauc_cv_recall_at_10_max": 0.6584110646510131, - "nauc_cv_recall_at_10_std": 0.5330026578185566, - "nauc_cv_recall_at_1_diff1": 0.16598692664394651, - "nauc_cv_recall_at_1_max": 0.4452623051567126, - "nauc_cv_recall_at_1_std": 0.3140538577574166, - "nauc_cv_recall_at_20_diff1": -0.16275966866186092, - "nauc_cv_recall_at_20_max": 0.646906213516669, - "nauc_cv_recall_at_20_std": 0.5563256305246183, - "nauc_cv_recall_at_3_diff1": -0.096122584943371, - "nauc_cv_recall_at_3_max": 0.6684610259826782, - "nauc_cv_recall_at_3_std": 0.4076215856095933, - "nauc_cv_recall_at_5_diff1": -0.1365179789302609, - "nauc_cv_recall_at_5_max": 0.5976289288098207, - "nauc_cv_recall_at_5_std": 0.37226107902603245, - "nauc_map_at_1000_diff1": 0.02290130273084715, - "nauc_map_at_1000_max": 0.548071312346396, - "nauc_map_at_1000_std": 0.3431509175694228, - "nauc_map_at_100_diff1": 0.023035522926500292, - "nauc_map_at_100_max": 0.5481243666012099, - "nauc_map_at_100_std": 0.343765707161693, - "nauc_map_at_10_diff1": 0.019009704736545754, - "nauc_map_at_10_max": 0.5515194516807846, - "nauc_map_at_10_std": 0.3449228525930436, - "nauc_map_at_1_diff1": 0.1698977596513771, - "nauc_map_at_1_max": 0.4552349293256607, - "nauc_map_at_1_std": 0.2536314877926139, - "nauc_map_at_20_diff1": 0.021258373975554346, - "nauc_map_at_20_max": 0.5501977396176536, - "nauc_map_at_20_std": 0.3451008448257592, - "nauc_map_at_3_diff1": 0.027947800501289654, - "nauc_map_at_3_max": 0.5615419194304191, - "nauc_map_at_3_std": 0.3374327981401433, - "nauc_map_at_5_diff1": 0.022444182090860222, - "nauc_map_at_5_max": 0.5443295202128533, - "nauc_map_at_5_std": 0.3265094163994626, - "nauc_mrr_at_1000_diff1": 0.037792737057000106, - "nauc_mrr_at_1000_max": 0.5306581923448304, - "nauc_mrr_at_1000_std": 0.36198678701734915, - "nauc_mrr_at_100_diff1": 0.037904244512512376, - "nauc_mrr_at_100_max": 0.5307371796485331, - "nauc_mrr_at_100_std": 0.3625738495305344, - "nauc_mrr_at_10_diff1": 0.0336578592994239, - "nauc_mrr_at_10_max": 0.5344197389534955, - "nauc_mrr_at_10_std": 0.3632912027558574, - "nauc_mrr_at_1_diff1": 0.16598692664394651, - "nauc_mrr_at_1_max": 0.4452623051567126, - "nauc_mrr_at_1_std": 0.3140538577574166, - "nauc_mrr_at_20_diff1": 0.03592134186381886, - "nauc_mrr_at_20_max": 0.5330587540055658, - "nauc_mrr_at_20_std": 0.36365157309531015, - "nauc_mrr_at_3_diff1": 0.04203509201841499, - "nauc_mrr_at_3_max": 0.5450312778658897, - "nauc_mrr_at_3_std": 0.35278982227092975, - "nauc_mrr_at_5_diff1": 0.036600215680544855, - "nauc_mrr_at_5_max": 0.5283891078779852, - "nauc_mrr_at_5_std": 0.3444903245689106, - "nauc_ndcg_at_1000_diff1": 0.0005409786282632852, - "nauc_ndcg_at_1000_max": 0.557765371086238, - "nauc_ndcg_at_1000_std": 0.3620874514374281, - "nauc_ndcg_at_100_diff1": -0.0004227472957878371, - "nauc_ndcg_at_100_max": 0.5581112142072867, - "nauc_ndcg_at_100_std": 0.3770164912688396, - "nauc_ndcg_at_10_diff1": -0.027451154381326896, - "nauc_ndcg_at_10_max": 0.5759121776535603, - "nauc_ndcg_at_10_std": 0.3837759064006119, - "nauc_ndcg_at_1_diff1": 0.1698977596513771, - "nauc_ndcg_at_1_max": 0.4552349293256607, - "nauc_ndcg_at_1_std": 0.2536314877926139, - "nauc_ndcg_at_20_diff1": -0.018923679643943057, - "nauc_ndcg_at_20_max": 0.5717611328008967, - "nauc_ndcg_at_20_std": 0.3855482939373467, - "nauc_ndcg_at_3_diff1": -0.005345292806580259, - "nauc_ndcg_at_3_max": 0.589864220491107, - "nauc_ndcg_at_3_std": 0.35634772055921166, - "nauc_ndcg_at_5_diff1": -0.01736592260642203, - "nauc_ndcg_at_5_max": 0.559198032314247, - "nauc_ndcg_at_5_std": 0.3388356749504251, + "nauc_cv_recall_at_100_diff1": -0.367413632119516, + "nauc_cv_recall_at_100_max": 0.35807656395891974, + "nauc_cv_recall_at_100_std": -0.0242763772175541, + "nauc_cv_recall_at_10_diff1": 0.01462637932311375, + "nauc_cv_recall_at_10_max": 0.6539657853810261, + "nauc_cv_recall_at_10_std": 0.2829741538917277, + "nauc_cv_recall_at_1_diff1": 0.3496222260314613, + "nauc_cv_recall_at_1_max": 0.1055525521169845, + "nauc_cv_recall_at_1_std": -0.14339587811020701, + "nauc_cv_recall_at_20_diff1": -0.06349206349206456, + "nauc_cv_recall_at_20_max": 0.6151377217553676, + "nauc_cv_recall_at_20_std": 0.36496265172735837, + "nauc_cv_recall_at_3_diff1": 0.08475382884831636, + "nauc_cv_recall_at_3_max": 0.4612572466903178, + "nauc_cv_recall_at_3_std": -0.15386778575754947, + "nauc_cv_recall_at_5_diff1": 0.028588843095141415, + "nauc_cv_recall_at_5_max": 0.44630521855227356, + "nauc_cv_recall_at_5_std": -0.13774888256806125, + "nauc_map_at_1000_diff1": 0.3657009430365933, + "nauc_map_at_1000_max": 0.2996454074298663, + "nauc_map_at_1000_std": -0.14999022266095388, + "nauc_map_at_100_diff1": 0.3654096405697197, + "nauc_map_at_100_max": 0.29966947866950855, + "nauc_map_at_100_std": -0.14994646671270026, + "nauc_map_at_10_diff1": 0.3609021693276199, + "nauc_map_at_10_max": 0.3068046720385847, + "nauc_map_at_10_std": -0.1420792617200844, + "nauc_map_at_1_diff1": 0.499334771432631, + "nauc_map_at_1_max": 0.2027203327434544, + "nauc_map_at_1_std": -0.2031595127685132, + "nauc_map_at_20_diff1": 0.3633362122496361, + "nauc_map_at_20_max": 0.30202217987243185, + "nauc_map_at_20_std": -0.14627098741534908, + "nauc_map_at_3_diff1": 0.34885387763276127, + "nauc_map_at_3_max": 0.309027475843024, + "nauc_map_at_3_std": -0.16955601757062616, + "nauc_map_at_5_diff1": 0.35307672932655393, + "nauc_map_at_5_max": 0.2945922362211287, + "nauc_map_at_5_std": -0.16241885430038053, + "nauc_mrr_at_1000_diff1": 0.26560116161479447, + "nauc_mrr_at_1000_max": 0.2286556037665365, + "nauc_mrr_at_1000_std": -0.12003394418012507, + "nauc_mrr_at_100_diff1": 0.2653509593794469, + "nauc_mrr_at_100_max": 0.22870899026180094, + "nauc_mrr_at_100_std": -0.12000263248162615, + "nauc_mrr_at_10_diff1": 0.2625870606112582, + "nauc_mrr_at_10_max": 0.2370936298550721, + "nauc_mrr_at_10_std": -0.11265461889909462, + "nauc_mrr_at_1_diff1": 0.3496222260314613, + "nauc_mrr_at_1_max": 0.1055525521169845, + "nauc_mrr_at_1_std": -0.14339587811020701, + "nauc_mrr_at_20_diff1": 0.2638864741009738, + "nauc_mrr_at_20_max": 0.23148416292732588, + "nauc_mrr_at_20_std": -0.11651871247529559, + "nauc_mrr_at_3_diff1": 0.2589399601820997, + "nauc_mrr_at_3_max": 0.24019609132343225, + "nauc_mrr_at_3_std": -0.1440308654475832, + "nauc_mrr_at_5_diff1": 0.25871660893688886, + "nauc_mrr_at_5_max": 0.22758080746049747, + "nauc_mrr_at_5_std": -0.1430749326822504, + "nauc_ndcg_at_1000_diff1": 0.34389535687748, + "nauc_ndcg_at_1000_max": 0.314316158167146, + "nauc_ndcg_at_1000_std": -0.13511867578024445, + "nauc_ndcg_at_100_diff1": 0.3352177735053365, + "nauc_ndcg_at_100_max": 0.31485379518438816, + "nauc_ndcg_at_100_std": -0.13380300430512387, + "nauc_ndcg_at_10_diff1": 0.31190857835409885, + "nauc_ndcg_at_10_max": 0.35259278734259186, + "nauc_ndcg_at_10_std": -0.08980541800210258, + "nauc_ndcg_at_1_diff1": 0.499334771432631, + "nauc_ndcg_at_1_max": 0.2027203327434544, + "nauc_ndcg_at_1_std": -0.2031595127685132, + "nauc_ndcg_at_20_diff1": 0.3200803496952673, + "nauc_ndcg_at_20_max": 0.3339057031205548, + "nauc_ndcg_at_20_std": -0.10284104333077189, + "nauc_ndcg_at_3_diff1": 0.29178770526806463, + "nauc_ndcg_at_3_max": 0.3444567331842166, + "nauc_ndcg_at_3_std": -0.16170218021524788, + "nauc_ndcg_at_5_diff1": 0.29483888128565966, + "nauc_ndcg_at_5_max": 0.31916959383433774, + "nauc_ndcg_at_5_std": -0.14266008480433065, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.05718954248365978, - "nauc_precision_at_100_max": 0.5615662931839398, - "nauc_precision_at_100_std": 0.751692343604107, - "nauc_precision_at_10_diff1": -0.1909058590509713, - "nauc_precision_at_10_max": 0.6584110646510136, - "nauc_precision_at_10_std": 0.5330026578185574, - "nauc_precision_at_1_diff1": 0.1698977596513771, - "nauc_precision_at_1_max": 0.4552349293256607, - "nauc_precision_at_1_std": 0.2536314877926139, - "nauc_precision_at_20_diff1": -0.16275966866186015, - "nauc_precision_at_20_max": 0.6469062135166688, - "nauc_precision_at_20_std": 0.5563256305246184, - "nauc_precision_at_3_diff1": -0.09612258494337084, - "nauc_precision_at_3_max": 0.6684610259826779, - "nauc_precision_at_3_std": 0.4076215856095937, - "nauc_precision_at_5_diff1": -0.13651797893026002, - "nauc_precision_at_5_max": 0.597628928809821, - "nauc_precision_at_5_std": 0.3722610790260329, + "nauc_precision_at_100_diff1": -0.3674136321195118, + "nauc_precision_at_100_max": 0.3580765639589172, + "nauc_precision_at_100_std": -0.024276377217548, + "nauc_precision_at_10_diff1": 0.014626379323115002, + "nauc_precision_at_10_max": 0.6539657853810266, + "nauc_precision_at_10_std": 0.282974153891728, + "nauc_precision_at_1_diff1": 0.499334771432631, + "nauc_precision_at_1_max": 0.2027203327434544, + "nauc_precision_at_1_std": -0.2031595127685132, + "nauc_precision_at_20_diff1": -0.06349206349206139, + "nauc_precision_at_20_max": 0.6151377217553686, + "nauc_precision_at_20_std": 0.3649626517273598, + "nauc_precision_at_3_diff1": 0.06112217721179036, + "nauc_precision_at_3_max": 0.4854412390053937, + "nauc_precision_at_3_std": -0.1321814641570239, + "nauc_precision_at_5_diff1": -0.012641138929799524, + "nauc_precision_at_5_max": 0.4398011782032391, + "nauc_precision_at_5_std": -0.03126534118802223, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.05718954248365932, - "nauc_recall_at_100_max": 0.5615662931839402, - "nauc_recall_at_100_std": 0.7516923436041081, - "nauc_recall_at_10_diff1": -0.19090585905097338, - "nauc_recall_at_10_max": 0.6584110646510131, - "nauc_recall_at_10_std": 0.5330026578185566, - "nauc_recall_at_1_diff1": 0.1698977596513771, - "nauc_recall_at_1_max": 0.4552349293256607, - "nauc_recall_at_1_std": 0.2536314877926139, - "nauc_recall_at_20_diff1": -0.16275966866186092, - "nauc_recall_at_20_max": 0.646906213516669, - "nauc_recall_at_20_std": 0.5563256305246183, - "nauc_recall_at_3_diff1": -0.096122584943371, - "nauc_recall_at_3_max": 0.6684610259826782, - "nauc_recall_at_3_std": 0.4076215856095933, - "nauc_recall_at_5_diff1": -0.1365179789302609, - "nauc_recall_at_5_max": 0.5976289288098207, - "nauc_recall_at_5_std": 0.37226107902603245, - "ndcg_at_1": 0.28, - "ndcg_at_10": 0.47865, - "ndcg_at_100": 0.52677, - "ndcg_at_1000": 0.53753, - "ndcg_at_20": 0.48907, - "ndcg_at_3": 0.41619, - "ndcg_at_5": 0.44845, - "precision_at_1": 0.28, - "precision_at_10": 0.067, - "precision_at_100": 0.0092, + "nauc_recall_at_100_diff1": -0.367413632119516, + "nauc_recall_at_100_max": 0.35807656395891974, + "nauc_recall_at_100_std": -0.0242763772175541, + "nauc_recall_at_10_diff1": 0.01462637932311375, + "nauc_recall_at_10_max": 0.6539657853810261, + "nauc_recall_at_10_std": 0.2829741538917277, + "nauc_recall_at_1_diff1": 0.499334771432631, + "nauc_recall_at_1_max": 0.2027203327434544, + "nauc_recall_at_1_std": -0.2031595127685132, + "nauc_recall_at_20_diff1": -0.06349206349206456, + "nauc_recall_at_20_max": 0.6151377217553676, + "nauc_recall_at_20_std": 0.36496265172735837, + "nauc_recall_at_3_diff1": 0.0611221772117895, + "nauc_recall_at_3_max": 0.485441239005394, + "nauc_recall_at_3_std": -0.1321814641570245, + "nauc_recall_at_5_diff1": -0.01264113892979802, + "nauc_recall_at_5_max": 0.43980117820323966, + "nauc_recall_at_5_std": -0.031265341188021364, + "ndcg_at_1": 0.41, + "ndcg_at_10": 0.66508, + "ndcg_at_100": 0.68922, + "ndcg_at_1000": 0.69189, + "ndcg_at_20": 0.67774, + "ndcg_at_3": 0.62535, + "ndcg_at_5": 0.65462, + "precision_at_1": 0.41, + "precision_at_10": 0.087, + "precision_at_100": 0.0098, "precision_at_1000": 0.001, - "precision_at_20": 0.0355, - "precision_at_3": 0.16667, - "precision_at_5": 0.116, - "recall_at_1": 0.28, - "recall_at_10": 0.67, - "recall_at_100": 0.92, + "precision_at_20": 0.046, + "precision_at_3": 0.25667, + "precision_at_5": 0.168, + "recall_at_1": 0.41, + "recall_at_10": 0.87, + "recall_at_100": 0.98, "recall_at_1000": 1.0, - "recall_at_20": 0.71, - "recall_at_3": 0.5, - "recall_at_5": 0.58 + "recall_at_20": 0.92, + "recall_at_3": 0.77, + "recall_at_5": 0.84 } ] }, diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json index c0bd444fb8..1c60fce707 100644 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -1,184 +1,184 @@ { "dataset_revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", - "evaluation_time": 73.8860080242157, + "evaluation_time": 74.7807514667511, "kg_co2_emissions": null, "mteb_version": "1.14.15", "scores": { "test": [ { - "cv_recall_at_1": 0.18, - "cv_recall_at_10": 0.53, - "cv_recall_at_100": 0.86, + "cv_recall_at_1": 0.42, + "cv_recall_at_10": 0.94, + "cv_recall_at_100": 0.98, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.65, - "cv_recall_at_3": 0.38, - "cv_recall_at_5": 0.47, + "cv_recall_at_20": 0.95, + "cv_recall_at_3": 0.78, + "cv_recall_at_5": 0.86, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.33103, - "map_at_1": 0.17, - "map_at_10": 0.29279, - "map_at_100": 0.30536, - "map_at_1000": 0.30619, - "map_at_20": 0.30089, - "map_at_3": 0.26667, - "map_at_5": 0.28467, - "mrr_at_1": 0.18, - "mrr_at_10": 0.29542857142857143, - "mrr_at_100": 0.3081013748465196, - "mrr_at_1000": 0.30893080980505994, - "mrr_at_20": 0.3036155927405928, - "mrr_at_3": 0.26666666666666666, - "mrr_at_5": 0.2871666666666666, + "main_score": 0.66761, + "map_at_1": 0.42, + "map_at_10": 0.61359, + "map_at_100": 0.61548, + "map_at_1000": 0.61559, + "map_at_20": 0.6145, + "map_at_3": 0.585, + "map_at_5": 0.6025, + "mrr_at_1": 0.42, + "mrr_at_10": 0.6167023809523812, + "mrr_at_100": 0.6185586140207295, + "mrr_at_1000": 0.6186628619999807, + "mrr_at_20": 0.617611471861472, + "mrr_at_3": 0.5883333333333334, + "mrr_at_5": 0.6058333333333334, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.483303311625328, - "nauc_cv_recall_at_100_max": 0.5174241374532345, - "nauc_cv_recall_at_100_std": 0.6429264237217674, - "nauc_cv_recall_at_10_diff1": -0.4115383029319991, - "nauc_cv_recall_at_10_max": 0.709782952342987, - "nauc_cv_recall_at_10_std": 0.48667335633479486, - "nauc_cv_recall_at_1_diff1": -0.09549871943304222, - "nauc_cv_recall_at_1_max": 0.46247763393327, - "nauc_cv_recall_at_1_std": 0.2309055187173279, - "nauc_cv_recall_at_20_diff1": -0.3380421515176176, - "nauc_cv_recall_at_20_max": 0.5907722353324105, - "nauc_cv_recall_at_20_std": 0.5878102367971356, - "nauc_cv_recall_at_3_diff1": -0.3586021365633043, - "nauc_cv_recall_at_3_max": 0.7061315727913083, - "nauc_cv_recall_at_3_std": 0.3685571699633508, - "nauc_cv_recall_at_5_diff1": -0.4414483313334829, - "nauc_cv_recall_at_5_max": 0.6931262279302206, - "nauc_cv_recall_at_5_std": 0.3961286409367214, - "nauc_map_at_1000_diff1": -0.22866744303816647, - "nauc_map_at_1000_max": 0.6739853793849305, - "nauc_map_at_1000_std": 0.3932776656219446, - "nauc_map_at_100_diff1": -0.22930027603884356, - "nauc_map_at_100_max": 0.6735362617611932, - "nauc_map_at_100_std": 0.3939927989346046, - "nauc_map_at_10_diff1": -0.2395548159357987, - "nauc_map_at_10_max": 0.6804946733702065, - "nauc_map_at_10_std": 0.3909649841540425, - "nauc_map_at_1_diff1": -0.06567777186967136, - "nauc_map_at_1_max": 0.6634701773043143, - "nauc_map_at_1_std": 0.37842644712957696, - "nauc_map_at_20_diff1": -0.2309939061113582, - "nauc_map_at_20_max": 0.6719671219524078, - "nauc_map_at_20_std": 0.3961538541197034, - "nauc_map_at_3_diff1": -0.20825225408792225, - "nauc_map_at_3_max": 0.6758876267128524, - "nauc_map_at_3_std": 0.36635678263970267, - "nauc_map_at_5_diff1": -0.24204925924807408, - "nauc_map_at_5_max": 0.6775739229114318, - "nauc_map_at_5_std": 0.37794651270997526, - "nauc_mrr_at_1000_diff1": -0.24080560474639628, - "nauc_mrr_at_1000_max": 0.5801836691701273, - "nauc_mrr_at_1000_std": 0.31892126838980106, - "nauc_mrr_at_100_diff1": -0.24139498766695452, - "nauc_mrr_at_100_max": 0.579984465673827, - "nauc_mrr_at_100_std": 0.3198150057038841, - "nauc_mrr_at_10_diff1": -0.2508554519682481, - "nauc_mrr_at_10_max": 0.5891124235800822, - "nauc_mrr_at_10_std": 0.31837517767996915, - "nauc_mrr_at_1_diff1": -0.09549871943304222, - "nauc_mrr_at_1_max": 0.46247763393327, - "nauc_mrr_at_1_std": 0.2309055187173279, - "nauc_mrr_at_20_diff1": -0.2429160804923638, - "nauc_mrr_at_20_max": 0.5794110337672059, - "nauc_mrr_at_20_std": 0.3227054157690821, - "nauc_mrr_at_3_diff1": -0.23502707309736295, - "nauc_mrr_at_3_max": 0.5891682542865738, - "nauc_mrr_at_3_std": 0.3009575854807983, - "nauc_mrr_at_5_diff1": -0.2539824551386642, - "nauc_mrr_at_5_max": 0.5869243320318603, - "nauc_mrr_at_5_std": 0.3056964583546606, - "nauc_ndcg_at_1000_diff1": -0.24877049649113786, - "nauc_ndcg_at_1000_max": 0.6716904547953125, - "nauc_ndcg_at_1000_std": 0.4072564448685426, - "nauc_ndcg_at_100_diff1": -0.2639017194977542, - "nauc_ndcg_at_100_max": 0.6613111300385416, - "nauc_ndcg_at_100_std": 0.42279146749234503, - "nauc_ndcg_at_10_diff1": -0.2879680148626686, - "nauc_ndcg_at_10_max": 0.6881786699043809, - "nauc_ndcg_at_10_std": 0.41377797946754535, - "nauc_ndcg_at_1_diff1": -0.06567777186967136, - "nauc_ndcg_at_1_max": 0.6634701773043143, - "nauc_ndcg_at_1_std": 0.37842644712957696, - "nauc_ndcg_at_20_diff1": -0.26264273370370655, - "nauc_ndcg_at_20_max": 0.658052249712014, - "nauc_ndcg_at_20_std": 0.4345548933087738, - "nauc_ndcg_at_3_diff1": -0.23793103616980857, - "nauc_ndcg_at_3_max": 0.6792223783252196, - "nauc_ndcg_at_3_std": 0.36358450135691267, - "nauc_ndcg_at_5_diff1": -0.29747560271749857, - "nauc_ndcg_at_5_max": 0.6819612247682432, - "nauc_ndcg_at_5_std": 0.38260912730366553, + "nauc_cv_recall_at_100_diff1": -0.0242763772175541, + "nauc_cv_recall_at_100_max": 0.8611111111111092, + "nauc_cv_recall_at_100_std": 0.71171802054155, + "nauc_cv_recall_at_10_diff1": 0.06395891690009405, + "nauc_cv_recall_at_10_max": 0.7930283224400876, + "nauc_cv_recall_at_10_std": 0.3990818549642063, + "nauc_cv_recall_at_1_diff1": 0.3091608989812584, + "nauc_cv_recall_at_1_max": 0.4137569017808538, + "nauc_cv_recall_at_1_std": -0.01810664385514676, + "nauc_cv_recall_at_20_diff1": 0.00513538748832759, + "nauc_cv_recall_at_20_max": 0.8071895424836592, + "nauc_cv_recall_at_20_std": 0.4543417366946761, + "nauc_cv_recall_at_3_diff1": -0.20373719107896363, + "nauc_cv_recall_at_3_max": 0.7115500533222051, + "nauc_cv_recall_at_3_std": 0.06758009922566846, + "nauc_cv_recall_at_5_diff1": -0.19107662463627592, + "nauc_cv_recall_at_5_max": 0.6820701122349996, + "nauc_cv_recall_at_5_std": 0.03606069003741182, + "nauc_map_at_1000_diff1": 0.17693830624484685, + "nauc_map_at_1000_max": 0.40199420044287293, + "nauc_map_at_1000_std": -0.0723223706512287, + "nauc_map_at_100_diff1": 0.17684396080137854, + "nauc_map_at_100_max": 0.40218884057257354, + "nauc_map_at_100_std": -0.07199833339393502, + "nauc_map_at_10_diff1": 0.17680282662193408, + "nauc_map_at_10_max": 0.4046908806180739, + "nauc_map_at_10_std": -0.07074201397090754, + "nauc_map_at_1_diff1": 0.3328796951551442, + "nauc_map_at_1_max": 0.2827980402830702, + "nauc_map_at_1_std": -0.16376338232625642, + "nauc_map_at_20_diff1": 0.1761825278902579, + "nauc_map_at_20_max": 0.4036043233960628, + "nauc_map_at_20_std": -0.0714042333250469, + "nauc_map_at_3_diff1": 0.13335576761291268, + "nauc_map_at_3_max": 0.4300971885040343, + "nauc_map_at_3_std": -0.0699869400443876, + "nauc_map_at_5_diff1": 0.15877224784371946, + "nauc_map_at_5_max": 0.4128887839672196, + "nauc_map_at_5_std": -0.07222002074076582, + "nauc_mrr_at_1000_diff1": 0.16594205369860826, + "nauc_mrr_at_1000_max": 0.4924629063588566, + "nauc_mrr_at_1000_std": 0.03004682497252532, + "nauc_mrr_at_100_diff1": 0.16585082182469363, + "nauc_mrr_at_100_max": 0.49262403678833994, + "nauc_mrr_at_100_std": 0.030334121756698387, + "nauc_mrr_at_10_diff1": 0.16591245155036535, + "nauc_mrr_at_10_max": 0.49445652729464, + "nauc_mrr_at_10_std": 0.030869742208528065, + "nauc_mrr_at_1_diff1": 0.3091608989812584, + "nauc_mrr_at_1_max": 0.4137569017808538, + "nauc_mrr_at_1_std": -0.01810664385514676, + "nauc_mrr_at_20_diff1": 0.16525013780729855, + "nauc_mrr_at_20_max": 0.49367150896807016, + "nauc_mrr_at_20_std": 0.03055295544576017, + "nauc_mrr_at_3_diff1": 0.13051631319395293, + "nauc_mrr_at_3_max": 0.5124847656533753, + "nauc_mrr_at_3_std": 0.023984847710341952, + "nauc_mrr_at_5_diff1": 0.1470586569315778, + "nauc_mrr_at_5_max": 0.4994845463016199, + "nauc_mrr_at_5_std": 0.021489887761373738, + "nauc_ndcg_at_1000_diff1": 0.161837391311996, + "nauc_ndcg_at_1000_max": 0.4176893400412115, + "nauc_ndcg_at_1000_std": -0.05567968591990525, + "nauc_ndcg_at_100_diff1": 0.1594005018759552, + "nauc_ndcg_at_100_max": 0.4233584462924314, + "nauc_ndcg_at_100_std": -0.045922161116991364, + "nauc_ndcg_at_10_diff1": 0.15803940765596416, + "nauc_ndcg_at_10_max": 0.43679112701120815, + "nauc_ndcg_at_10_std": -0.03664740566414989, + "nauc_ndcg_at_1_diff1": 0.3328796951551442, + "nauc_ndcg_at_1_max": 0.2827980402830702, + "nauc_ndcg_at_1_std": -0.16376338232625642, + "nauc_ndcg_at_20_diff1": 0.1554345969800805, + "nauc_ndcg_at_20_max": 0.4330743476999243, + "nauc_ndcg_at_20_std": -0.03872344287722131, + "nauc_ndcg_at_3_diff1": 0.06065482050917286, + "nauc_ndcg_at_3_max": 0.4850976042539053, + "nauc_ndcg_at_3_std": -0.042410523533264216, + "nauc_ndcg_at_5_diff1": 0.10788221180845371, + "nauc_ndcg_at_5_max": 0.4540038684839002, + "nauc_ndcg_at_5_std": -0.04680446883376484, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.4833033116253274, - "nauc_precision_at_100_max": 0.5174241374532372, - "nauc_precision_at_100_std": 0.6429264237217707, - "nauc_precision_at_10_diff1": -0.411538302931999, - "nauc_precision_at_10_max": 0.7097829523429878, - "nauc_precision_at_10_std": 0.48667335633479564, - "nauc_precision_at_1_diff1": -0.06567777186967136, - "nauc_precision_at_1_max": 0.6634701773043143, - "nauc_precision_at_1_std": 0.37842644712957696, - "nauc_precision_at_20_diff1": -0.3380421515176169, - "nauc_precision_at_20_max": 0.5907722353324113, - "nauc_precision_at_20_std": 0.5878102367971358, - "nauc_precision_at_3_diff1": -0.30869323399687, - "nauc_precision_at_3_max": 0.6874264353989625, - "nauc_precision_at_3_std": 0.35688694010062977, - "nauc_precision_at_5_diff1": -0.44144833133348227, - "nauc_precision_at_5_max": 0.6931262279302212, - "nauc_precision_at_5_std": 0.3961286409367218, + "nauc_precision_at_100_diff1": -0.024276377217548, + "nauc_precision_at_100_max": 0.8611111111111196, + "nauc_precision_at_100_std": 0.7117180205415448, + "nauc_precision_at_10_diff1": 0.06395891690009418, + "nauc_precision_at_10_max": 0.7930283224400888, + "nauc_precision_at_10_std": 0.3990818549642083, + "nauc_precision_at_1_diff1": 0.3328796951551442, + "nauc_precision_at_1_max": 0.2827980402830702, + "nauc_precision_at_1_std": -0.16376338232625642, + "nauc_precision_at_20_diff1": 0.005135387488338012, + "nauc_precision_at_20_max": 0.8071895424836621, + "nauc_precision_at_20_std": 0.4543417366946809, + "nauc_precision_at_3_diff1": -0.23781703528538964, + "nauc_precision_at_3_max": 0.7115500533222053, + "nauc_precision_at_3_std": 0.06758009922566896, + "nauc_precision_at_5_diff1": -0.1692531522793392, + "nauc_precision_at_5_max": 0.6820701122350005, + "nauc_precision_at_5_std": 0.09145074130525248, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.483303311625328, - "nauc_recall_at_100_max": 0.5174241374532345, - "nauc_recall_at_100_std": 0.6429264237217674, - "nauc_recall_at_10_diff1": -0.4115383029319991, - "nauc_recall_at_10_max": 0.709782952342987, - "nauc_recall_at_10_std": 0.48667335633479486, - "nauc_recall_at_1_diff1": -0.06567777186967136, - "nauc_recall_at_1_max": 0.6634701773043143, - "nauc_recall_at_1_std": 0.37842644712957696, - "nauc_recall_at_20_diff1": -0.3380421515176176, - "nauc_recall_at_20_max": 0.5907722353324105, - "nauc_recall_at_20_std": 0.5878102367971356, - "nauc_recall_at_3_diff1": -0.3086932339968699, - "nauc_recall_at_3_max": 0.6874264353989629, - "nauc_recall_at_3_std": 0.35688694010062993, - "nauc_recall_at_5_diff1": -0.4414483313334829, - "nauc_recall_at_5_max": 0.6931262279302206, - "nauc_recall_at_5_std": 0.3961286409367214, - "ndcg_at_1": 0.17, - "ndcg_at_10": 0.35054, - "ndcg_at_100": 0.41772, - "ndcg_at_1000": 0.43636, - "ndcg_at_20": 0.38063, - "ndcg_at_3": 0.29833, - "ndcg_at_5": 0.33103, - "precision_at_1": 0.17, - "precision_at_10": 0.053, - "precision_at_100": 0.0086, + "nauc_recall_at_100_diff1": -0.0242763772175541, + "nauc_recall_at_100_max": 0.8611111111111092, + "nauc_recall_at_100_std": 0.71171802054155, + "nauc_recall_at_10_diff1": 0.06395891690009405, + "nauc_recall_at_10_max": 0.7930283224400876, + "nauc_recall_at_10_std": 0.3990818549642063, + "nauc_recall_at_1_diff1": 0.3328796951551442, + "nauc_recall_at_1_max": 0.2827980402830702, + "nauc_recall_at_1_std": -0.16376338232625642, + "nauc_recall_at_20_diff1": 0.00513538748832759, + "nauc_recall_at_20_max": 0.8071895424836592, + "nauc_recall_at_20_std": 0.4543417366946761, + "nauc_recall_at_3_diff1": -0.23781703528539025, + "nauc_recall_at_3_max": 0.7115500533222051, + "nauc_recall_at_3_std": 0.06758009922566846, + "nauc_recall_at_5_diff1": -0.16925315227934043, + "nauc_recall_at_5_max": 0.6820701122349996, + "nauc_recall_at_5_std": 0.09145074130525091, + "ndcg_at_1": 0.42, + "ndcg_at_10": 0.69391, + "ndcg_at_100": 0.70261, + "ndcg_at_1000": 0.70523, + "ndcg_at_20": 0.6967, + "ndcg_at_3": 0.63535, + "ndcg_at_5": 0.66761, + "precision_at_1": 0.42, + "precision_at_10": 0.094, + "precision_at_100": 0.0098, "precision_at_1000": 0.001, - "precision_at_20": 0.0325, - "precision_at_3": 0.13, - "precision_at_5": 0.094, - "recall_at_1": 0.17, - "recall_at_10": 0.53, - "recall_at_100": 0.86, + "precision_at_20": 0.0475, + "precision_at_3": 0.26, + "precision_at_5": 0.172, + "recall_at_1": 0.42, + "recall_at_10": 0.94, + "recall_at_100": 0.98, "recall_at_1000": 1.0, - "recall_at_20": 0.65, - "recall_at_3": 0.39, - "recall_at_5": 0.47 + "recall_at_20": 0.95, + "recall_at_3": 0.78, + "recall_at_5": 0.86 } ] }, diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json index f80c15a971..3828955dfa 100644 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -1,184 +1,184 @@ { "dataset_revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", - "evaluation_time": 72.91890263557434, + "evaluation_time": 73.25277805328369, "kg_co2_emissions": null, "mteb_version": "1.14.15", "scores": { "test": [ { - "cv_recall_at_1": 0.22, - "cv_recall_at_10": 0.6, - "cv_recall_at_100": 0.91, + "cv_recall_at_1": 0.47, + "cv_recall_at_10": 0.94, + "cv_recall_at_100": 0.97, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.7, - "cv_recall_at_3": 0.49, - "cv_recall_at_5": 0.52, + "cv_recall_at_20": 0.94, + "cv_recall_at_3": 0.87, + "cv_recall_at_5": 0.89, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.38779, - "map_at_1": 0.21, - "map_at_10": 0.35287, - "map_at_100": 0.36535, - "map_at_1000": 0.3659, - "map_at_20": 0.36013, - "map_at_3": 0.33667, - "map_at_5": 0.34267, - "mrr_at_1": 0.22, - "mrr_at_10": 0.355218253968254, - "mrr_at_100": 0.36766650196348827, - "mrr_at_1000": 0.3682164935395407, - "mrr_at_20": 0.36243612819928617, - "mrr_at_3": 0.3383333333333334, - "mrr_at_5": 0.34483333333333327, + "main_score": 0.71077, + "map_at_1": 0.45, + "map_at_10": 0.65593, + "map_at_100": 0.6569, + "map_at_1000": 0.65706, + "map_at_20": 0.65593, + "map_at_3": 0.64667, + "map_at_5": 0.64867, + "mrr_at_1": 0.47, + "mrr_at_10": 0.6642976190476192, + "mrr_at_100": 0.6652641390867199, + "mrr_at_1000": 0.6654314232950488, + "mrr_at_20": 0.6642976190476192, + "mrr_at_3": 0.6533333333333333, + "mrr_at_5": 0.6573333333333333, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.003267973856210241, - "nauc_cv_recall_at_100_max": 0.5420686793235807, - "nauc_cv_recall_at_100_std": 0.7533457827575475, - "nauc_cv_recall_at_10_diff1": -0.2672899970317603, - "nauc_cv_recall_at_10_max": 0.6763431285247846, - "nauc_cv_recall_at_10_std": 0.5030424458296231, - "nauc_cv_recall_at_1_diff1": -0.22708767131802224, - "nauc_cv_recall_at_1_max": 0.46022003294278496, - "nauc_cv_recall_at_1_std": 0.11752183236473257, - "nauc_cv_recall_at_20_diff1": -0.27616645649432564, - "nauc_cv_recall_at_20_max": 0.6204287515762922, - "nauc_cv_recall_at_20_std": 0.5124482075301748, - "nauc_cv_recall_at_3_diff1": -0.2874005815729024, - "nauc_cv_recall_at_3_max": 0.6736420007170077, - "nauc_cv_recall_at_3_std": 0.37669459455870835, - "nauc_cv_recall_at_5_diff1": -0.2977311061537627, - "nauc_cv_recall_at_5_max": 0.7365063387561889, - "nauc_cv_recall_at_5_std": 0.4491131182327658, - "nauc_map_at_1000_diff1": -0.24421377564817673, - "nauc_map_at_1000_max": 0.6057575097002678, - "nauc_map_at_1000_std": 0.3563653280500335, - "nauc_map_at_100_diff1": -0.2438208029837478, - "nauc_map_at_100_max": 0.6056381919626572, - "nauc_map_at_100_std": 0.3571056403427818, - "nauc_map_at_10_diff1": -0.2474064341034369, - "nauc_map_at_10_max": 0.6111031262320629, - "nauc_map_at_10_std": 0.3583715496190217, - "nauc_map_at_1_diff1": -0.19750740336041306, - "nauc_map_at_1_max": 0.5460116870160101, - "nauc_map_at_1_std": 0.3261120876684563, - "nauc_map_at_20_diff1": -0.24703318252196604, - "nauc_map_at_20_max": 0.6060923108554881, - "nauc_map_at_20_std": 0.3571216123471924, - "nauc_map_at_3_diff1": -0.24964070479347697, - "nauc_map_at_3_max": 0.6057022544576557, - "nauc_map_at_3_std": 0.33612306100353556, - "nauc_map_at_5_diff1": -0.25163894718289853, - "nauc_map_at_5_max": 0.6187024411572932, - "nauc_map_at_5_std": 0.3512947369738972, - "nauc_mrr_at_1000_diff1": -0.26262047817609174, - "nauc_mrr_at_1000_max": 0.5657657182275748, - "nauc_mrr_at_1000_std": 0.2676332247249374, - "nauc_mrr_at_100_diff1": -0.26219833115320174, - "nauc_mrr_at_100_max": 0.565716644353467, - "nauc_mrr_at_100_std": 0.2685234476168615, - "nauc_mrr_at_10_diff1": -0.26525556917234877, - "nauc_mrr_at_10_max": 0.5720441464757696, - "nauc_mrr_at_10_std": 0.27167563037113773, - "nauc_mrr_at_1_diff1": -0.22708767131802224, - "nauc_mrr_at_1_max": 0.46022003294278496, - "nauc_mrr_at_1_std": 0.11752183236473257, - "nauc_mrr_at_20_diff1": -0.265224379144156, - "nauc_mrr_at_20_max": 0.566706960324191, - "nauc_mrr_at_20_std": 0.2696353634782542, - "nauc_mrr_at_3_diff1": -0.2671988902639183, - "nauc_mrr_at_3_max": 0.5673596974829038, - "nauc_mrr_at_3_std": 0.2520896624948636, - "nauc_mrr_at_5_diff1": -0.2695098953901908, - "nauc_mrr_at_5_max": 0.5799741955943382, - "nauc_mrr_at_5_std": 0.2656432940902274, - "nauc_ndcg_at_1000_diff1": -0.24407529554426993, - "nauc_ndcg_at_1000_max": 0.611964249388691, - "nauc_ndcg_at_1000_std": 0.37296921118890486, - "nauc_ndcg_at_100_diff1": -0.23370771712739602, - "nauc_ndcg_at_100_max": 0.6088627125657947, - "nauc_ndcg_at_100_std": 0.38985170913350337, - "nauc_ndcg_at_10_diff1": -0.2538099257653232, - "nauc_ndcg_at_10_max": 0.6289827458486196, - "nauc_ndcg_at_10_std": 0.39168461686955036, - "nauc_ndcg_at_1_diff1": -0.19750740336041306, - "nauc_ndcg_at_1_max": 0.5460116870160101, - "nauc_ndcg_at_1_std": 0.3261120876684563, - "nauc_ndcg_at_20_diff1": -0.25434363734303483, - "nauc_ndcg_at_20_max": 0.6135309338030498, - "nauc_ndcg_at_20_std": 0.3886075994148953, - "nauc_ndcg_at_3_diff1": -0.26026416495381194, - "nauc_ndcg_at_3_max": 0.6240980080777259, - "nauc_ndcg_at_3_std": 0.34656841722715415, - "nauc_ndcg_at_5_diff1": -0.2641693037611579, - "nauc_ndcg_at_5_max": 0.6490032981064092, - "nauc_ndcg_at_5_std": 0.37552492374435203, + "nauc_cv_recall_at_100_diff1": -0.10597572362278272, + "nauc_cv_recall_at_100_max": 0.4788359788359823, + "nauc_cv_recall_at_100_std": -0.009648303765951848, + "nauc_cv_recall_at_10_diff1": -0.04061624649859804, + "nauc_cv_recall_at_10_max": 0.6651104886398995, + "nauc_cv_recall_at_10_std": 0.016728913787736808, + "nauc_cv_recall_at_1_diff1": 0.2818602565962105, + "nauc_cv_recall_at_1_max": 0.052374108308390115, + "nauc_cv_recall_at_1_std": -0.1893748599005788, + "nauc_cv_recall_at_20_diff1": -0.04061624649859804, + "nauc_cv_recall_at_20_max": 0.6651104886398995, + "nauc_cv_recall_at_20_std": 0.016728913787736808, + "nauc_cv_recall_at_3_diff1": -0.3010812412056582, + "nauc_cv_recall_at_3_max": 0.5909427534621934, + "nauc_cv_recall_at_3_std": -0.1500407316892538, + "nauc_cv_recall_at_5_diff1": -0.2275148183145788, + "nauc_cv_recall_at_5_max": 0.6092260115110386, + "nauc_cv_recall_at_5_std": -0.15514131088394567, + "nauc_map_at_1000_diff1": 0.17737809853138078, + "nauc_map_at_1000_max": 0.3197186933030335, + "nauc_map_at_1000_std": -0.15660439305683332, + "nauc_map_at_100_diff1": 0.17713247274986268, + "nauc_map_at_100_max": 0.31971110754062343, + "nauc_map_at_100_std": -0.15658001231252233, + "nauc_map_at_10_diff1": 0.1766419592404658, + "nauc_map_at_10_max": 0.3218218690272864, + "nauc_map_at_10_std": -0.15584231228112508, + "nauc_map_at_1_diff1": 0.2847508675440319, + "nauc_map_at_1_max": 0.22929352452039553, + "nauc_map_at_1_std": -0.1787599030969685, + "nauc_map_at_20_diff1": 0.1766419592404658, + "nauc_map_at_20_max": 0.3218218690272864, + "nauc_map_at_20_std": -0.15584231228112508, + "nauc_map_at_3_diff1": 0.15674372963365232, + "nauc_map_at_3_max": 0.3338846647575784, + "nauc_map_at_3_std": -0.15814160165893193, + "nauc_map_at_5_diff1": 0.15920039673168618, + "nauc_map_at_5_max": 0.3282159254462605, + "nauc_map_at_5_std": -0.16209944641512158, + "nauc_mrr_at_1000_diff1": 0.15966423921533143, + "nauc_mrr_at_1000_max": 0.17666600015301887, + "nauc_mrr_at_1000_std": -0.17509432480460377, + "nauc_mrr_at_100_diff1": 0.15942769989266647, + "nauc_mrr_at_100_max": 0.17675277810481055, + "nauc_mrr_at_100_std": -0.17506183456061375, + "nauc_mrr_at_10_diff1": 0.15900784081013614, + "nauc_mrr_at_10_max": 0.17942911487138907, + "nauc_mrr_at_10_std": -0.174251393599049, + "nauc_mrr_at_1_diff1": 0.2818602565962105, + "nauc_mrr_at_1_max": 0.052374108308390115, + "nauc_mrr_at_1_std": -0.1893748599005788, + "nauc_mrr_at_20_diff1": 0.15900784081013614, + "nauc_mrr_at_20_max": 0.17942911487138907, + "nauc_mrr_at_20_std": -0.174251393599049, + "nauc_mrr_at_3_diff1": 0.1304221207197815, + "nauc_mrr_at_3_max": 0.19363906685877158, + "nauc_mrr_at_3_std": -0.17807547285490927, + "nauc_mrr_at_5_diff1": 0.14280359493903766, + "nauc_mrr_at_5_max": 0.18970837430829438, + "nauc_mrr_at_5_std": -0.17890776810678843, + "nauc_ndcg_at_1000_diff1": 0.16305570366234656, + "nauc_ndcg_at_1000_max": 0.33387508653430314, + "nauc_ndcg_at_1000_std": -0.1510648449789182, + "nauc_ndcg_at_100_diff1": 0.15718521389079343, + "nauc_ndcg_at_100_max": 0.3361827474796506, + "nauc_ndcg_at_100_std": -0.1485562970765986, + "nauc_ndcg_at_10_diff1": 0.153440821156415, + "nauc_ndcg_at_10_max": 0.35171505689198845, + "nauc_ndcg_at_10_std": -0.14299661493260465, + "nauc_ndcg_at_1_diff1": 0.2847508675440319, + "nauc_ndcg_at_1_max": 0.22929352452039553, + "nauc_ndcg_at_1_std": -0.1787599030969685, + "nauc_ndcg_at_20_diff1": 0.153440821156415, + "nauc_ndcg_at_20_max": 0.35171505689198845, + "nauc_ndcg_at_20_std": -0.14299661493260465, + "nauc_ndcg_at_3_diff1": 0.10268992445948906, + "nauc_ndcg_at_3_max": 0.378473051261699, + "nauc_ndcg_at_3_std": -0.15155969201864058, + "nauc_ndcg_at_5_diff1": 0.1073960551331664, + "nauc_ndcg_at_5_max": 0.3663556421486979, + "nauc_ndcg_at_5_std": -0.16040529466424153, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.003267973856207883, - "nauc_precision_at_100_max": 0.542068679323581, - "nauc_precision_at_100_std": 0.75334578275755, - "nauc_precision_at_10_diff1": -0.26728999703175943, - "nauc_precision_at_10_max": 0.6763431285247852, - "nauc_precision_at_10_std": 0.503042445829623, - "nauc_precision_at_1_diff1": -0.19750740336041306, - "nauc_precision_at_1_max": 0.5460116870160101, - "nauc_precision_at_1_std": 0.3261120876684563, - "nauc_precision_at_20_diff1": -0.27616645649432475, - "nauc_precision_at_20_max": 0.6204287515762927, - "nauc_precision_at_20_std": 0.5124482075301745, - "nauc_precision_at_3_diff1": -0.2874005815729024, - "nauc_precision_at_3_max": 0.6736420007170074, - "nauc_precision_at_3_std": 0.37669459455870846, - "nauc_precision_at_5_diff1": -0.29773110615376197, - "nauc_precision_at_5_max": 0.736506338756189, - "nauc_precision_at_5_std": 0.449113118232766, + "nauc_precision_at_100_diff1": -0.10597572362277796, + "nauc_precision_at_100_max": 0.4788359788359767, + "nauc_precision_at_100_std": -0.009648303765945032, + "nauc_precision_at_10_diff1": -0.04061624649859813, + "nauc_precision_at_10_max": 0.6651104886399004, + "nauc_precision_at_10_std": 0.016728913787735757, + "nauc_precision_at_1_diff1": 0.2847508675440319, + "nauc_precision_at_1_max": 0.22929352452039553, + "nauc_precision_at_1_std": -0.1787599030969685, + "nauc_precision_at_20_diff1": -0.04061624649859813, + "nauc_precision_at_20_max": 0.6651104886399004, + "nauc_precision_at_20_std": 0.016728913787735757, + "nauc_precision_at_3_diff1": -0.2252624880687238, + "nauc_precision_at_3_max": 0.6493000318167349, + "nauc_precision_at_3_std": -0.11314826598790892, + "nauc_precision_at_5_diff1": -0.2275148183145768, + "nauc_precision_at_5_max": 0.6092260115110381, + "nauc_precision_at_5_std": -0.15514131088394403, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.003267973856210241, - "nauc_recall_at_100_max": 0.5420686793235807, - "nauc_recall_at_100_std": 0.7533457827575475, - "nauc_recall_at_10_diff1": -0.2672899970317603, - "nauc_recall_at_10_max": 0.6763431285247846, - "nauc_recall_at_10_std": 0.5030424458296231, - "nauc_recall_at_1_diff1": -0.19750740336041306, - "nauc_recall_at_1_max": 0.5460116870160101, - "nauc_recall_at_1_std": 0.3261120876684563, - "nauc_recall_at_20_diff1": -0.27616645649432564, - "nauc_recall_at_20_max": 0.6204287515762922, - "nauc_recall_at_20_std": 0.5124482075301748, - "nauc_recall_at_3_diff1": -0.2874005815729024, - "nauc_recall_at_3_max": 0.6736420007170077, - "nauc_recall_at_3_std": 0.37669459455870835, - "nauc_recall_at_5_diff1": -0.2977311061537627, - "nauc_recall_at_5_max": 0.7365063387561889, - "nauc_recall_at_5_std": 0.4491131182327658, - "ndcg_at_1": 0.21, - "ndcg_at_10": 0.41321, - "ndcg_at_100": 0.4775, - "ndcg_at_1000": 0.48958, - "ndcg_at_20": 0.43887, - "ndcg_at_3": 0.37619, - "ndcg_at_5": 0.38779, - "precision_at_1": 0.21, - "precision_at_10": 0.06, - "precision_at_100": 0.0091, + "nauc_recall_at_100_diff1": -0.10597572362278272, + "nauc_recall_at_100_max": 0.4788359788359823, + "nauc_recall_at_100_std": -0.009648303765951848, + "nauc_recall_at_10_diff1": -0.04061624649859804, + "nauc_recall_at_10_max": 0.6651104886398995, + "nauc_recall_at_10_std": 0.016728913787736808, + "nauc_recall_at_1_diff1": 0.2847508675440319, + "nauc_recall_at_1_max": 0.22929352452039553, + "nauc_recall_at_1_std": -0.1787599030969685, + "nauc_recall_at_20_diff1": -0.04061624649859804, + "nauc_recall_at_20_max": 0.6651104886398995, + "nauc_recall_at_20_std": 0.016728913787736808, + "nauc_recall_at_3_diff1": -0.22526248806872484, + "nauc_recall_at_3_max": 0.6493000318167352, + "nauc_recall_at_3_std": -0.11314826598791083, + "nauc_recall_at_5_diff1": -0.2275148183145788, + "nauc_recall_at_5_max": 0.6092260115110386, + "nauc_recall_at_5_std": -0.15514131088394567, + "ndcg_at_1": 0.45, + "ndcg_at_10": 0.72753, + "ndcg_at_100": 0.73351, + "ndcg_at_1000": 0.73748, + "ndcg_at_20": 0.72753, + "ndcg_at_3": 0.7069, + "ndcg_at_5": 0.71077, + "precision_at_1": 0.45, + "precision_at_10": 0.094, + "precision_at_100": 0.0097, "precision_at_1000": 0.001, - "precision_at_20": 0.035, - "precision_at_3": 0.16333, - "precision_at_5": 0.104, - "recall_at_1": 0.21, - "recall_at_10": 0.6, - "recall_at_100": 0.91, + "precision_at_20": 0.047, + "precision_at_3": 0.29333, + "precision_at_5": 0.178, + "recall_at_1": 0.45, + "recall_at_10": 0.94, + "recall_at_100": 0.97, "recall_at_1000": 1.0, - "recall_at_20": 0.7, - "recall_at_3": 0.49, - "recall_at_5": 0.52 + "recall_at_20": 0.94, + "recall_at_3": 0.88, + "recall_at_5": 0.89 } ] }, diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json index 85526aa435..817d0b4849 100644 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json @@ -1,184 +1,184 @@ { "dataset_revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", - "evaluation_time": 17.741925716400146, + "evaluation_time": 17.969542264938354, "kg_co2_emissions": null, "mteb_version": "1.14.15", "scores": { "test": [ { - "cv_recall_at_1": 0.13571, - "cv_recall_at_10": 0.58929, - "cv_recall_at_100": 0.86071, + "cv_recall_at_1": 0.14286, + "cv_recall_at_10": 0.59643, + "cv_recall_at_100": 0.87143, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.7, - "cv_recall_at_3": 0.375, - "cv_recall_at_5": 0.48929, + "cv_recall_at_20": 0.71429, + "cv_recall_at_3": 0.38214, + "cv_recall_at_5": 0.5, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.30532, - "map_at_1": 0.11786, - "map_at_10": 0.26471, - "map_at_100": 0.27589, - "map_at_1000": 0.27681, - "map_at_20": 0.27145, - "map_at_3": 0.21905, - "map_at_5": 0.2478, - "mrr_at_1": 0.1357142857142857, - "mrr_at_10": 0.275654761904762, - "mrr_at_100": 0.2882710170373999, - "mrr_at_1000": 0.28918278167491335, - "mrr_at_20": 0.28371689890224294, - "mrr_at_3": 0.23452380952380958, - "mrr_at_5": 0.26184523809523824, + "main_score": 0.31415, + "map_at_1": 0.125, + "map_at_10": 0.27248, + "map_at_100": 0.28394, + "map_at_1000": 0.28477, + "map_at_20": 0.27971, + "map_at_3": 0.22619, + "map_at_5": 0.25601, + "mrr_at_1": 0.14285714285714285, + "mrr_at_10": 0.2834226190476192, + "mrr_at_100": 0.2963289758699489, + "mrr_at_1000": 0.29714896365874977, + "mrr_at_20": 0.2919793288742317, + "mrr_at_3": 0.24166666666666672, + "mrr_at_5": 0.2700595238095239, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.0962614034651627, - "nauc_cv_recall_at_100_max": 0.48130701732581055, - "nauc_cv_recall_at_100_std": 0.5359442193442552, - "nauc_cv_recall_at_10_diff1": 0.07595632707564201, - "nauc_cv_recall_at_10_max": 0.5214463383342498, - "nauc_cv_recall_at_10_std": 0.34694315712303286, - "nauc_cv_recall_at_1_diff1": 0.023979663013656137, - "nauc_cv_recall_at_1_max": 0.11585261885201438, - "nauc_cv_recall_at_1_std": 0.039449413296544805, - "nauc_cv_recall_at_20_diff1": -0.013504645237666892, - "nauc_cv_recall_at_20_max": 0.45511104820238296, - "nauc_cv_recall_at_20_std": 0.3636088735620353, - "nauc_cv_recall_at_3_diff1": 0.12455393491055446, - "nauc_cv_recall_at_3_max": 0.34790052806662475, - "nauc_cv_recall_at_3_std": 0.24388768697513188, - "nauc_cv_recall_at_5_diff1": 0.10118096610707866, - "nauc_cv_recall_at_5_max": 0.48999426347254305, - "nauc_cv_recall_at_5_std": 0.338028436108833, - "nauc_map_at_1000_diff1": 0.14177585744325438, - "nauc_map_at_1000_max": 0.25267701349330973, - "nauc_map_at_1000_std": 0.16204571721159644, - "nauc_map_at_100_diff1": 0.14092375993422507, - "nauc_map_at_100_max": 0.2534882998637386, - "nauc_map_at_100_std": 0.16331431723111056, - "nauc_map_at_10_diff1": 0.14448945692889367, - "nauc_map_at_10_max": 0.25899654113563375, - "nauc_map_at_10_std": 0.1597103307680331, - "nauc_map_at_1_diff1": 0.18523279430290246, - "nauc_map_at_1_max": 0.026814407280767123, - "nauc_map_at_1_std": 0.009923644497004887, - "nauc_map_at_20_diff1": 0.1403834299409469, - "nauc_map_at_20_max": 0.25535009456183255, - "nauc_map_at_20_std": 0.16250878288417944, - "nauc_map_at_3_diff1": 0.1481805843215222, - "nauc_map_at_3_max": 0.24167493327909895, - "nauc_map_at_3_std": 0.15671525636183276, - "nauc_map_at_5_diff1": 0.14976730485212275, - "nauc_map_at_5_max": 0.26080880741487417, - "nauc_map_at_5_std": 0.15898524258324853, - "nauc_mrr_at_1000_diff1": 0.053836794776765395, - "nauc_mrr_at_1000_max": 0.2751316825025726, - "nauc_mrr_at_1000_std": 0.1735453731971639, - "nauc_mrr_at_100_diff1": 0.053306077293060236, - "nauc_mrr_at_100_max": 0.2758338657361141, - "nauc_mrr_at_100_std": 0.17472166458524652, - "nauc_mrr_at_10_diff1": 0.059631240474482726, - "nauc_mrr_at_10_max": 0.2861178647684893, - "nauc_mrr_at_10_std": 0.176909107436406, - "nauc_mrr_at_1_diff1": 0.023979663013656137, - "nauc_mrr_at_1_max": 0.11585261885201438, - "nauc_mrr_at_1_std": 0.039449413296544805, - "nauc_mrr_at_20_diff1": 0.05378318892025374, - "nauc_mrr_at_20_max": 0.27730629708974225, - "nauc_mrr_at_20_std": 0.17378722632932975, - "nauc_mrr_at_3_diff1": 0.07204267342282876, - "nauc_mrr_at_3_max": 0.23918521004131926, - "nauc_mrr_at_3_std": 0.1519262805303006, - "nauc_mrr_at_5_diff1": 0.06501788292981979, - "nauc_mrr_at_5_max": 0.2838511219515895, - "nauc_mrr_at_5_std": 0.17950227251107553, - "nauc_ndcg_at_1000_diff1": 0.12681348634395276, - "nauc_ndcg_at_1000_max": 0.284330263153128, - "nauc_ndcg_at_1000_std": 0.19038243430316637, - "nauc_ndcg_at_100_diff1": 0.10951958445625287, - "nauc_ndcg_at_100_max": 0.29955187926203075, - "nauc_ndcg_at_100_std": 0.2167965732577945, - "nauc_ndcg_at_10_diff1": 0.12444522627830684, - "nauc_ndcg_at_10_max": 0.32109399057607235, - "nauc_ndcg_at_10_std": 0.1984173977368013, - "nauc_ndcg_at_1_diff1": 0.18523279430290246, - "nauc_ndcg_at_1_max": 0.026814407280767123, - "nauc_ndcg_at_1_std": 0.009923644497004887, - "nauc_ndcg_at_20_diff1": 0.1070015172410676, - "nauc_ndcg_at_20_max": 0.3099995592251142, - "nauc_ndcg_at_20_std": 0.21140392873639405, - "nauc_ndcg_at_3_diff1": 0.13374110988425314, - "nauc_ndcg_at_3_max": 0.2965332376419493, - "nauc_ndcg_at_3_std": 0.19616272589026296, - "nauc_ndcg_at_5_diff1": 0.13580773833733265, - "nauc_ndcg_at_5_max": 0.32598979522729604, - "nauc_ndcg_at_5_std": 0.198633639239219, + "nauc_cv_recall_at_100_diff1": -0.09346618228443337, + "nauc_cv_recall_at_100_max": 0.47191206325462104, + "nauc_cv_recall_at_100_std": 0.5897264665028307, + "nauc_cv_recall_at_10_diff1": 0.06153163948878473, + "nauc_cv_recall_at_10_max": 0.527236830297048, + "nauc_cv_recall_at_10_std": 0.34759886185471267, + "nauc_cv_recall_at_1_diff1": 0.04107196177963575, + "nauc_cv_recall_at_1_max": 0.18530904747685875, + "nauc_cv_recall_at_1_std": 0.11839355031352648, + "nauc_cv_recall_at_20_diff1": -0.01519699812382745, + "nauc_cv_recall_at_20_max": 0.44332618600911256, + "nauc_cv_recall_at_20_std": 0.377559635486465, + "nauc_cv_recall_at_3_diff1": 0.12322123253689696, + "nauc_cv_recall_at_3_max": 0.3550718925761234, + "nauc_cv_recall_at_3_std": 0.24882574686208955, + "nauc_cv_recall_at_5_diff1": 0.07709622156657454, + "nauc_cv_recall_at_5_max": 0.5071523746074043, + "nauc_cv_recall_at_5_std": 0.3624536023603309, + "nauc_map_at_1000_diff1": 0.14508719417530674, + "nauc_map_at_1000_max": 0.3028773500015137, + "nauc_map_at_1000_std": 0.22042646887013098, + "nauc_map_at_100_diff1": 0.14428659458613782, + "nauc_map_at_100_max": 0.30343561386061807, + "nauc_map_at_100_std": 0.22154390909649585, + "nauc_map_at_10_diff1": 0.1458055654055668, + "nauc_map_at_10_max": 0.3087857008787967, + "nauc_map_at_10_std": 0.21513465622236783, + "nauc_map_at_1_diff1": 0.21192189105858175, + "nauc_map_at_1_max": 0.1422322713257966, + "nauc_map_at_1_std": 0.12627338129496413, + "nauc_map_at_20_diff1": 0.1434484748653935, + "nauc_map_at_20_max": 0.3048401943472712, + "nauc_map_at_20_std": 0.21989153816245258, + "nauc_map_at_3_diff1": 0.15084588762214982, + "nauc_map_at_3_max": 0.2895541530944628, + "nauc_map_at_3_std": 0.21155741042345294, + "nauc_map_at_5_diff1": 0.14886552885273804, + "nauc_map_at_5_max": 0.3118795633185364, + "nauc_map_at_5_std": 0.21775580485670507, + "nauc_mrr_at_1000_diff1": 0.05695061181399053, + "nauc_mrr_at_1000_max": 0.3111012267827344, + "nauc_mrr_at_1000_std": 0.21393435228582347, + "nauc_mrr_at_100_diff1": 0.056440815201173854, + "nauc_mrr_at_100_max": 0.3116096019154161, + "nauc_mrr_at_100_std": 0.21502116138592015, + "nauc_mrr_at_10_diff1": 0.061021499554143384, + "nauc_mrr_at_10_max": 0.3224409033707004, + "nauc_mrr_at_10_std": 0.21528251710496038, + "nauc_mrr_at_1_diff1": 0.04107196177963575, + "nauc_mrr_at_1_max": 0.18530904747685875, + "nauc_mrr_at_1_std": 0.11839355031352648, + "nauc_mrr_at_20_diff1": 0.0566059945620949, + "nauc_mrr_at_20_max": 0.31286490797934124, + "nauc_mrr_at_20_std": 0.2135380630344629, + "nauc_mrr_at_3_diff1": 0.07833818406529658, + "nauc_mrr_at_3_max": 0.2753016193582217, + "nauc_mrr_at_3_std": 0.18873408566788993, + "nauc_mrr_at_5_diff1": 0.06400650046913868, + "nauc_mrr_at_5_max": 0.32162314506455336, + "nauc_mrr_at_5_std": 0.2214497802320089, + "nauc_ndcg_at_1000_diff1": 0.12888457382784677, + "nauc_ndcg_at_1000_max": 0.325817844139425, + "nauc_ndcg_at_1000_std": 0.24189184208615394, + "nauc_ndcg_at_100_diff1": 0.11282750611807016, + "nauc_ndcg_at_100_max": 0.33632613400391, + "nauc_ndcg_at_100_std": 0.26639957608731973, + "nauc_ndcg_at_10_diff1": 0.12071304892163645, + "nauc_ndcg_at_10_max": 0.3563710929162044, + "nauc_ndcg_at_10_std": 0.2370147085502524, + "nauc_ndcg_at_1_diff1": 0.21192189105858175, + "nauc_ndcg_at_1_max": 0.1422322713257966, + "nauc_ndcg_at_1_std": 0.12627338129496413, + "nauc_ndcg_at_20_diff1": 0.10882639489305446, + "nauc_ndcg_at_20_max": 0.3432251869586007, + "nauc_ndcg_at_20_std": 0.25568218475503307, + "nauc_ndcg_at_3_diff1": 0.13085937145194312, + "nauc_ndcg_at_3_max": 0.32974469478667556, + "nauc_ndcg_at_3_std": 0.23643620953909147, + "nauc_ndcg_at_5_diff1": 0.12727992939268223, + "nauc_ndcg_at_5_max": 0.3649735155042485, + "nauc_ndcg_at_5_std": 0.2454961972349734, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.09626140346516247, - "nauc_precision_at_100_max": 0.4813070173258093, - "nauc_precision_at_100_std": 0.5359442193442547, - "nauc_precision_at_10_diff1": 0.07110205518543622, - "nauc_precision_at_10_max": 0.4711617958469575, - "nauc_precision_at_10_std": 0.29049963462184697, - "nauc_precision_at_1_diff1": 0.18523279430290246, - "nauc_precision_at_1_max": 0.026814407280767123, - "nauc_precision_at_1_std": 0.009923644497004887, - "nauc_precision_at_20_diff1": -0.013504645237666276, - "nauc_precision_at_20_max": 0.4551110482023827, - "nauc_precision_at_20_std": 0.3636088735620348, - "nauc_precision_at_3_diff1": 0.10070409009355073, - "nauc_precision_at_3_max": 0.41801308867412745, - "nauc_precision_at_3_std": 0.28393615875053274, - "nauc_precision_at_5_diff1": 0.1035717322098263, - "nauc_precision_at_5_max": 0.47273732039788774, - "nauc_precision_at_5_std": 0.28682020423393356, + "nauc_precision_at_100_diff1": -0.09346618228443235, + "nauc_precision_at_100_max": 0.47191206325462226, + "nauc_precision_at_100_std": 0.5897264665028321, + "nauc_precision_at_10_diff1": 0.05467766594814778, + "nauc_precision_at_10_max": 0.4735201482417899, + "nauc_precision_at_10_std": 0.2869915668573714, + "nauc_precision_at_1_diff1": 0.21192189105858175, + "nauc_precision_at_1_max": 0.1422322713257966, + "nauc_precision_at_1_std": 0.12627338129496413, + "nauc_precision_at_20_diff1": -0.015196998123827318, + "nauc_precision_at_20_max": 0.4433261860091128, + "nauc_precision_at_20_std": 0.3775596354864649, + "nauc_precision_at_3_diff1": 0.08504628829930763, + "nauc_precision_at_3_max": 0.4204835645025001, + "nauc_precision_at_3_std": 0.292939808871107, + "nauc_precision_at_5_diff1": 0.07691213631127067, + "nauc_precision_at_5_max": 0.48827330523541085, + "nauc_precision_at_5_std": 0.30920357086981864, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.0962614034651627, - "nauc_recall_at_100_max": 0.48130701732581055, - "nauc_recall_at_100_std": 0.5359442193442552, - "nauc_recall_at_10_diff1": 0.07110205518543654, - "nauc_recall_at_10_max": 0.47116179584695755, - "nauc_recall_at_10_std": 0.29049963462184736, - "nauc_recall_at_1_diff1": 0.18523279430290246, - "nauc_recall_at_1_max": 0.026814407280767123, - "nauc_recall_at_1_std": 0.009923644497004887, - "nauc_recall_at_20_diff1": -0.013504645237666892, - "nauc_recall_at_20_max": 0.45511104820238296, - "nauc_recall_at_20_std": 0.3636088735620353, - "nauc_recall_at_3_diff1": 0.10070409009355087, - "nauc_recall_at_3_max": 0.4180130886741277, - "nauc_recall_at_3_std": 0.283936158750533, - "nauc_recall_at_5_diff1": 0.10357173220982632, - "nauc_recall_at_5_max": 0.4727373203978876, - "nauc_recall_at_5_std": 0.28682020423393373, - "ndcg_at_1": 0.11786, - "ndcg_at_10": 0.34597, - "ndcg_at_100": 0.40068, - "ndcg_at_1000": 0.41976, - "ndcg_at_20": 0.37041, - "ndcg_at_3": 0.25518, - "ndcg_at_5": 0.30532, - "precision_at_1": 0.11786, - "precision_at_10": 0.06036, - "precision_at_100": 0.00861, + "nauc_recall_at_100_diff1": -0.09346618228443337, + "nauc_recall_at_100_max": 0.47191206325462104, + "nauc_recall_at_100_std": 0.5897264665028307, + "nauc_recall_at_10_diff1": 0.05467766594814761, + "nauc_recall_at_10_max": 0.4735201482417895, + "nauc_recall_at_10_std": 0.2869915668573714, + "nauc_recall_at_1_diff1": 0.21192189105858175, + "nauc_recall_at_1_max": 0.1422322713257966, + "nauc_recall_at_1_std": 0.12627338129496413, + "nauc_recall_at_20_diff1": -0.01519699812382745, + "nauc_recall_at_20_max": 0.44332618600911256, + "nauc_recall_at_20_std": 0.377559635486465, + "nauc_recall_at_3_diff1": 0.08504628829930751, + "nauc_recall_at_3_max": 0.42048356450249996, + "nauc_recall_at_3_std": 0.29293980887110704, + "nauc_recall_at_5_diff1": 0.07691213631127075, + "nauc_recall_at_5_max": 0.4882733052354108, + "nauc_recall_at_5_std": 0.30920357086981887, + "ndcg_at_1": 0.125, + "ndcg_at_10": 0.35368, + "ndcg_at_100": 0.40933, + "ndcg_at_1000": 0.42685, + "ndcg_at_20": 0.37992, + "ndcg_at_3": 0.26233, + "ndcg_at_5": 0.31415, + "precision_at_1": 0.125, + "precision_at_10": 0.06107, + "precision_at_100": 0.00871, "precision_at_1000": 0.001, - "precision_at_20": 0.035, - "precision_at_3": 0.12024, - "precision_at_5": 0.09571, - "recall_at_1": 0.11786, - "recall_at_10": 0.60357, - "recall_at_100": 0.86071, + "precision_at_20": 0.03571, + "precision_at_3": 0.12262, + "precision_at_5": 0.09786, + "recall_at_1": 0.125, + "recall_at_10": 0.61071, + "recall_at_100": 0.87143, "recall_at_1000": 1.0, - "recall_at_20": 0.7, - "recall_at_3": 0.36071, - "recall_at_5": 0.47857 + "recall_at_20": 0.71429, + "recall_at_3": 0.36786, + "recall_at_5": 0.48929 } ] }, diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json index 29032ce4c6..9aa82d28ab 100644 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json +++ b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json @@ -1,184 +1,184 @@ { "dataset_revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", - "evaluation_time": 126.18415856361389, + "evaluation_time": 130.58521914482117, "kg_co2_emissions": null, "mteb_version": "1.14.15", "scores": { "test": [ { - "cv_recall_at_1": 0.01744, - "cv_recall_at_10": 0.14311, - "cv_recall_at_100": 0.37703, - "cv_recall_at_1000": 0.87072, - "cv_recall_at_20": 0.19964, - "cv_recall_at_3": 0.05532, - "cv_recall_at_5": 0.0926, + "cv_recall_at_1": 0.03969, + "cv_recall_at_10": 0.23211, + "cv_recall_at_100": 0.51473, + "cv_recall_at_1000": 0.92183, + "cv_recall_at_20": 0.30487, + "cv_recall_at_3": 0.10042, + "cv_recall_at_5": 0.16055, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.05505, - "map_at_1": 0.01804, - "map_at_10": 0.04953, - "map_at_100": 0.05703, - "map_at_1000": 0.05869, - "map_at_20": 0.05307, - "map_at_3": 0.03397, - "map_at_5": 0.04248, - "mrr_at_1": 0.017438364401683705, - "mrr_at_10": 0.04915671620422074, - "mrr_at_100": 0.0569115235221346, - "mrr_at_1000": 0.05857065878278318, - "mrr_at_20": 0.053008722607474607, - "mrr_at_3": 0.033674082982561644, - "mrr_at_5": 0.04206253758268193, - "nauc_cv_recall_at_1000_diff1": 0.08237134389710216, - "nauc_cv_recall_at_1000_max": 0.20976894796503254, - "nauc_cv_recall_at_1000_std": 0.4140935117522756, - "nauc_cv_recall_at_100_diff1": -0.02981197371782488, - "nauc_cv_recall_at_100_max": 0.20240240199104623, - "nauc_cv_recall_at_100_std": 0.30510424487535914, - "nauc_cv_recall_at_10_diff1": -0.035047295007147705, - "nauc_cv_recall_at_10_max": 0.3038530304180975, - "nauc_cv_recall_at_10_std": 0.3220085766696133, - "nauc_cv_recall_at_1_diff1": -0.032626900215017456, - "nauc_cv_recall_at_1_max": 0.27684455719867385, - "nauc_cv_recall_at_1_std": 0.30979043565912767, - "nauc_cv_recall_at_20_diff1": -0.02501748001370101, - "nauc_cv_recall_at_20_max": 0.2987998775731927, - "nauc_cv_recall_at_20_std": 0.3289343312056541, - "nauc_cv_recall_at_3_diff1": -0.012493786516726318, - "nauc_cv_recall_at_3_max": 0.35062308469531805, - "nauc_cv_recall_at_3_std": 0.328063698293921, - "nauc_cv_recall_at_5_diff1": -0.02014835181508689, - "nauc_cv_recall_at_5_max": 0.30597083566947125, - "nauc_cv_recall_at_5_std": 0.3136339145032427, - "nauc_map_at_1000_diff1": -0.04875525076147121, - "nauc_map_at_1000_max": 0.2815787298378842, - "nauc_map_at_1000_std": 0.3731437421002045, - "nauc_map_at_100_diff1": -0.04942990178875301, - "nauc_map_at_100_max": 0.28124633969736407, - "nauc_map_at_100_std": 0.37218323907682854, - "nauc_map_at_10_diff1": -0.051458156860137806, - "nauc_map_at_10_max": 0.29098313852215413, - "nauc_map_at_10_std": 0.37750246377662106, - "nauc_map_at_1_diff1": -0.10777077067663393, - "nauc_map_at_1_max": 0.25176210516240183, - "nauc_map_at_1_std": 0.4592822617825615, - "nauc_map_at_20_diff1": -0.04802954975755555, - "nauc_map_at_20_max": 0.2913329148831261, - "nauc_map_at_20_std": 0.3747903249592789, - "nauc_map_at_3_diff1": -0.04904890787369823, - "nauc_map_at_3_max": 0.26399306382725835, - "nauc_map_at_3_std": 0.4183070347637387, - "nauc_map_at_5_diff1": -0.05486746702234904, - "nauc_map_at_5_max": 0.28519023906167823, - "nauc_map_at_5_std": 0.3825112354018944, - "nauc_mrr_at_1000_diff1": -0.024517826873781616, - "nauc_mrr_at_1000_max": 0.3058381571189165, - "nauc_mrr_at_1000_std": 0.3247519566076449, - "nauc_mrr_at_100_diff1": -0.025390095142045613, - "nauc_mrr_at_100_max": 0.30521995190372303, - "nauc_mrr_at_100_std": 0.3240834016745154, - "nauc_mrr_at_10_diff1": -0.02579739125808308, - "nauc_mrr_at_10_max": 0.314897749755248, - "nauc_mrr_at_10_std": 0.32324109310365073, - "nauc_mrr_at_1_diff1": -0.032626900215017456, - "nauc_mrr_at_1_max": 0.27684455719867385, - "nauc_mrr_at_1_std": 0.30979043565912767, - "nauc_mrr_at_20_diff1": -0.02365953039025371, - "nauc_mrr_at_20_max": 0.313964555189022, - "nauc_mrr_at_20_std": 0.32474136190872116, - "nauc_mrr_at_3_diff1": -0.01416562201976808, - "nauc_mrr_at_3_max": 0.34082255893899743, - "nauc_mrr_at_3_std": 0.328619069068735, - "nauc_mrr_at_5_diff1": -0.018886104109413417, - "nauc_mrr_at_5_max": 0.32010631024417296, - "nauc_mrr_at_5_std": 0.322477663211533, - "nauc_ndcg_at_1000_diff1": -0.026807561051169607, - "nauc_ndcg_at_1000_max": 0.2649662980138261, - "nauc_ndcg_at_1000_std": 0.3629955058252187, - "nauc_ndcg_at_100_diff1": -0.03931666574317868, - "nauc_ndcg_at_100_max": 0.2543873594260332, - "nauc_ndcg_at_100_std": 0.3413208990403148, - "nauc_ndcg_at_10_diff1": -0.04356951193981768, - "nauc_ndcg_at_10_max": 0.3023339951965928, - "nauc_ndcg_at_10_std": 0.36022667617586135, - "nauc_ndcg_at_1_diff1": -0.10777077067663393, - "nauc_ndcg_at_1_max": 0.25176210516240183, - "nauc_ndcg_at_1_std": 0.4592822617825615, - "nauc_ndcg_at_20_diff1": -0.03658281028277188, - "nauc_ndcg_at_20_max": 0.3020490621335391, - "nauc_ndcg_at_20_std": 0.3559036914665498, - "nauc_ndcg_at_3_diff1": -0.040372611398516646, - "nauc_ndcg_at_3_max": 0.2616384429014375, - "nauc_ndcg_at_3_std": 0.41172672732998233, - "nauc_ndcg_at_5_diff1": -0.049645484907602344, - "nauc_ndcg_at_5_max": 0.29233089541831914, - "nauc_ndcg_at_5_std": 0.3638542090400901, - "nauc_precision_at_1000_diff1": 0.08237134389710085, - "nauc_precision_at_1000_max": 0.20976894796503143, - "nauc_precision_at_1000_std": 0.4140935117522751, - "nauc_precision_at_100_diff1": -0.027951192620319844, - "nauc_precision_at_100_max": 0.2060014267088026, - "nauc_precision_at_100_std": 0.30489286954854505, - "nauc_precision_at_10_diff1": -0.03373855863600513, - "nauc_precision_at_10_max": 0.3175505020337768, - "nauc_precision_at_10_std": 0.3400442978218774, - "nauc_precision_at_1_diff1": -0.10777077067663393, - "nauc_precision_at_1_max": 0.25176210516240183, - "nauc_precision_at_1_std": 0.4592822617825615, - "nauc_precision_at_20_diff1": -0.023503901593891596, - "nauc_precision_at_20_max": 0.3139544627736064, - "nauc_precision_at_20_std": 0.3366365571580702, - "nauc_precision_at_3_diff1": -0.025149804808346558, - "nauc_precision_at_3_max": 0.2566022001981372, - "nauc_precision_at_3_std": 0.40006698593126916, - "nauc_precision_at_5_diff1": -0.04284288904729967, - "nauc_precision_at_5_max": 0.30312075460709115, - "nauc_precision_at_5_std": 0.3366667367707371, - "nauc_recall_at_1000_diff1": 0.08237134389710216, - "nauc_recall_at_1000_max": 0.20976894796503254, - "nauc_recall_at_1000_std": 0.4140935117522756, - "nauc_recall_at_100_diff1": -0.02795119262032021, - "nauc_recall_at_100_max": 0.20600142670880198, - "nauc_recall_at_100_std": 0.3048928695485446, - "nauc_recall_at_10_diff1": -0.03373855863600514, - "nauc_recall_at_10_max": 0.31755050203377655, - "nauc_recall_at_10_std": 0.3400442978218772, - "nauc_recall_at_1_diff1": -0.10777077067663393, - "nauc_recall_at_1_max": 0.25176210516240183, - "nauc_recall_at_1_std": 0.4592822617825615, - "nauc_recall_at_20_diff1": -0.023503901593891666, - "nauc_recall_at_20_max": 0.313954462773606, - "nauc_recall_at_20_std": 0.33663655715806995, - "nauc_recall_at_3_diff1": -0.025149804808346467, - "nauc_recall_at_3_max": 0.25660220019813706, - "nauc_recall_at_3_std": 0.400066985931269, - "nauc_recall_at_5_diff1": -0.04284288904729986, - "nauc_recall_at_5_max": 0.3031207546070908, - "nauc_recall_at_5_std": 0.3366667367707366, - "ndcg_at_1": 0.01804, - "ndcg_at_10": 0.0718, - "ndcg_at_100": 0.11698, - "ndcg_at_1000": 0.17565, - "ndcg_at_20": 0.08502, - "ndcg_at_3": 0.03958, - "ndcg_at_5": 0.05505, - "precision_at_1": 0.01804, - "precision_at_10": 0.01449, - "precision_at_100": 0.00377, - "precision_at_1000": 0.00087, - "precision_at_20": 0.00989, - "precision_at_3": 0.01864, - "precision_at_5": 0.01876, - "recall_at_1": 0.01804, - "recall_at_10": 0.14492, - "recall_at_100": 0.37703, - "recall_at_1000": 0.87072, - "recall_at_20": 0.19784, - "recall_at_3": 0.05592, - "recall_at_5": 0.09381 + "main_score": 0.08802, + "map_at_1": 0.02886, + "map_at_10": 0.07924, + "map_at_100": 0.08947, + "map_at_1000": 0.09101, + "map_at_20": 0.08455, + "map_at_3": 0.05522, + "map_at_5": 0.06815, + "mrr_at_1": 0.0396873120865905, + "mrr_at_10": 0.08826155637640908, + "mrr_at_100": 0.09808012131592855, + "mrr_at_1000": 0.09962230517717666, + "mrr_at_20": 0.09316889360880878, + "mrr_at_3": 0.06504309480857881, + "mrr_at_5": 0.07857286029264379, + "nauc_cv_recall_at_1000_diff1": 0.0977858066622949, + "nauc_cv_recall_at_1000_max": 0.19289234929852755, + "nauc_cv_recall_at_1000_std": 0.4244384238930124, + "nauc_cv_recall_at_100_diff1": 0.027363809758551075, + "nauc_cv_recall_at_100_max": 0.1472382566698071, + "nauc_cv_recall_at_100_std": 0.3133680991369263, + "nauc_cv_recall_at_10_diff1": -0.015248947065392211, + "nauc_cv_recall_at_10_max": 0.17684865071070638, + "nauc_cv_recall_at_10_std": 0.23399970322736818, + "nauc_cv_recall_at_1_diff1": 0.08496451376348123, + "nauc_cv_recall_at_1_max": 0.2250756456490274, + "nauc_cv_recall_at_1_std": 0.16451661904930748, + "nauc_cv_recall_at_20_diff1": -0.01759476439814012, + "nauc_cv_recall_at_20_max": 0.1384202166086512, + "nauc_cv_recall_at_20_std": 0.24952524851481098, + "nauc_cv_recall_at_3_diff1": 0.0683858086458561, + "nauc_cv_recall_at_3_max": 0.24611576386062065, + "nauc_cv_recall_at_3_std": 0.2046414079176916, + "nauc_cv_recall_at_5_diff1": 0.0389561258425957, + "nauc_cv_recall_at_5_max": 0.21630673696973646, + "nauc_cv_recall_at_5_std": 0.20644440359224767, + "nauc_map_at_1000_diff1": 0.0044848236290639725, + "nauc_map_at_1000_max": 0.1726088992109447, + "nauc_map_at_1000_std": 0.21597266371424678, + "nauc_map_at_100_diff1": 0.004521095517779526, + "nauc_map_at_100_max": 0.17262602441956626, + "nauc_map_at_100_std": 0.21593147204916938, + "nauc_map_at_10_diff1": 0.002769629051444685, + "nauc_map_at_10_max": 0.17949324947822573, + "nauc_map_at_10_std": 0.209214171869861, + "nauc_map_at_1_diff1": -0.02158849186859297, + "nauc_map_at_1_max": 0.13996579430266687, + "nauc_map_at_1_std": 0.17219572255475857, + "nauc_map_at_20_diff1": 0.0017843929653374108, + "nauc_map_at_20_max": 0.1723382974559687, + "nauc_map_at_20_std": 0.21039987703317992, + "nauc_map_at_3_diff1": 0.030158997427946903, + "nauc_map_at_3_max": 0.18440904960369608, + "nauc_map_at_3_std": 0.19515292645565502, + "nauc_map_at_5_diff1": 0.011488449209473625, + "nauc_map_at_5_max": 0.17995271406284225, + "nauc_map_at_5_std": 0.19587606041940772, + "nauc_mrr_at_1000_diff1": 0.045817042591563197, + "nauc_mrr_at_1000_max": 0.20920132305788777, + "nauc_mrr_at_1000_std": 0.2128708534292149, + "nauc_mrr_at_100_diff1": 0.04561537361722436, + "nauc_mrr_at_100_max": 0.20896709990127685, + "nauc_mrr_at_100_std": 0.21279912539406717, + "nauc_mrr_at_10_diff1": 0.04588905114387296, + "nauc_mrr_at_10_max": 0.2146020467703159, + "nauc_mrr_at_10_std": 0.20421262749770086, + "nauc_mrr_at_1_diff1": 0.08496451376348123, + "nauc_mrr_at_1_max": 0.2250756456490274, + "nauc_mrr_at_1_std": 0.16451661904930748, + "nauc_mrr_at_20_diff1": 0.0441332027973836, + "nauc_mrr_at_20_max": 0.20796335061234025, + "nauc_mrr_at_20_std": 0.20809344152536044, + "nauc_mrr_at_3_diff1": 0.07853906035769813, + "nauc_mrr_at_3_max": 0.2391325353905553, + "nauc_mrr_at_3_std": 0.1942148403197897, + "nauc_mrr_at_5_diff1": 0.06523844727902996, + "nauc_mrr_at_5_max": 0.22637957489872684, + "nauc_mrr_at_5_std": 0.19592628829316405, + "nauc_ndcg_at_1000_diff1": 0.009106844527408595, + "nauc_ndcg_at_1000_max": 0.16986530318282603, + "nauc_ndcg_at_1000_std": 0.2477659100109444, + "nauc_ndcg_at_100_diff1": 0.008486238488498946, + "nauc_ndcg_at_100_max": 0.1646870237335375, + "nauc_ndcg_at_100_std": 0.25192946867971616, + "nauc_ndcg_at_10_diff1": -0.006078172191213006, + "nauc_ndcg_at_10_max": 0.18160285165655646, + "nauc_ndcg_at_10_std": 0.22500698386041396, + "nauc_ndcg_at_1_diff1": -0.02158849186859297, + "nauc_ndcg_at_1_max": 0.13996579430266687, + "nauc_ndcg_at_1_std": 0.17219572255475857, + "nauc_ndcg_at_20_diff1": -0.009322631249012595, + "nauc_ndcg_at_20_max": 0.16357791120190782, + "nauc_ndcg_at_20_std": 0.2260169780476658, + "nauc_ndcg_at_3_diff1": 0.03642967796503214, + "nauc_ndcg_at_3_max": 0.192101861179512, + "nauc_ndcg_at_3_std": 0.19946501186770677, + "nauc_ndcg_at_5_diff1": 0.008516418183412523, + "nauc_ndcg_at_5_max": 0.18569505885828344, + "nauc_ndcg_at_5_std": 0.200401149370037, + "nauc_precision_at_1000_diff1": 0.09778580666229689, + "nauc_precision_at_1000_max": 0.19289234929852678, + "nauc_precision_at_1000_std": 0.4244384238930101, + "nauc_precision_at_100_diff1": 0.026851159458335674, + "nauc_precision_at_100_max": 0.1488321813053108, + "nauc_precision_at_100_std": 0.31570240449993014, + "nauc_precision_at_10_diff1": -0.02026084460212222, + "nauc_precision_at_10_max": 0.18235951356296864, + "nauc_precision_at_10_std": 0.24875775274772455, + "nauc_precision_at_1_diff1": -0.02158849186859297, + "nauc_precision_at_1_max": 0.13996579430266687, + "nauc_precision_at_1_std": 0.17219572255475857, + "nauc_precision_at_20_diff1": -0.026345082849132624, + "nauc_precision_at_20_max": 0.14657209751855116, + "nauc_precision_at_20_std": 0.24681362648213306, + "nauc_precision_at_3_diff1": 0.0470327136470777, + "nauc_precision_at_3_max": 0.2057045169689898, + "nauc_precision_at_3_std": 0.20715084952536195, + "nauc_precision_at_5_diff1": 0.0023052267840612817, + "nauc_precision_at_5_max": 0.19417557500790047, + "nauc_precision_at_5_std": 0.20730383037894426, + "nauc_recall_at_1000_diff1": 0.0977858066622949, + "nauc_recall_at_1000_max": 0.19289234929852755, + "nauc_recall_at_1000_std": 0.4244384238930124, + "nauc_recall_at_100_diff1": 0.02685115945833474, + "nauc_recall_at_100_max": 0.14883218130531, + "nauc_recall_at_100_std": 0.3157024044999299, + "nauc_recall_at_10_diff1": -0.020260844602122356, + "nauc_recall_at_10_max": 0.1823595135629684, + "nauc_recall_at_10_std": 0.2487577527477245, + "nauc_recall_at_1_diff1": -0.02158849186859297, + "nauc_recall_at_1_max": 0.13996579430266687, + "nauc_recall_at_1_std": 0.17219572255475857, + "nauc_recall_at_20_diff1": -0.026345082849132815, + "nauc_recall_at_20_max": 0.14657209751855108, + "nauc_recall_at_20_std": 0.24681362648213304, + "nauc_recall_at_3_diff1": 0.04703271364707788, + "nauc_recall_at_3_max": 0.20570451696898984, + "nauc_recall_at_3_std": 0.20715084952536209, + "nauc_recall_at_5_diff1": 0.0023052267840612106, + "nauc_recall_at_5_max": 0.1941755750079004, + "nauc_recall_at_5_std": 0.20730383037894415, + "ndcg_at_1": 0.02886, + "ndcg_at_10": 0.11406, + "ndcg_at_100": 0.17148, + "ndcg_at_1000": 0.22095, + "ndcg_at_20": 0.13338, + "ndcg_at_3": 0.06461, + "ndcg_at_5": 0.08802, + "precision_at_1": 0.02886, + "precision_at_10": 0.02279, + "precision_at_100": 0.00515, + "precision_at_1000": 0.00092, + "precision_at_20": 0.01521, + "precision_at_3": 0.03067, + "precision_at_5": 0.02983, + "recall_at_1": 0.02886, + "recall_at_10": 0.2279, + "recall_at_100": 0.51473, + "recall_at_1000": 0.92183, + "recall_at_20": 0.30427, + "recall_at_3": 0.092, + "recall_at_5": 0.14913 } ] }, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EDIST2ITRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EDIST2ITRetrieval.json new file mode 100644 index 0000000000..b8c457c9ae --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EDIST2ITRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "68c47ef3e49ef883073b3358bd4243eeca0aee9a", + "evaluation_time": 3594.926486968994, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.2058, + "cv_recall_at_10": 0.50231, + "cv_recall_at_100": 0.76797, + "cv_recall_at_1000": 0.92873, + "cv_recall_at_20": 0.5992, + "cv_recall_at_3": 0.34002, + "cv_recall_at_5": 0.415, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.24494, + "map_at_1": 0.07615, + "map_at_10": 0.17644, + "map_at_100": 0.19953, + "map_at_1000": 0.20149, + "map_at_20": 0.18958, + "map_at_3": 0.1292, + "map_at_5": 0.15486, + "mrr_at_1": 0.20580067880283864, + "mrr_at_10": 0.2925150477757707, + "mrr_at_100": 0.3034272424612011, + "mrr_at_1000": 0.30417101259312895, + "mrr_at_20": 0.29926672507651825, + "mrr_at_3": 0.2639103157461694, + "mrr_at_5": 0.28098837807261257, + "nauc_cv_recall_at_1000_diff1": 0.08692340997106167, + "nauc_cv_recall_at_1000_max": -0.08260050261557855, + "nauc_cv_recall_at_1000_std": 0.3207360002955632, + "nauc_cv_recall_at_100_diff1": 0.06500932146729019, + "nauc_cv_recall_at_100_max": 0.03548329590032714, + "nauc_cv_recall_at_100_std": 0.06632350606013385, + "nauc_cv_recall_at_10_diff1": 0.12947130572805773, + "nauc_cv_recall_at_10_max": 0.04226853926632297, + "nauc_cv_recall_at_10_std": -0.017992685315635543, + "nauc_cv_recall_at_1_diff1": 0.2158323552712354, + "nauc_cv_recall_at_1_max": 0.04444742333087822, + "nauc_cv_recall_at_1_std": 0.011407565122854697, + "nauc_cv_recall_at_20_diff1": 0.11043702121642354, + "nauc_cv_recall_at_20_max": 0.07178164802818354, + "nauc_cv_recall_at_20_std": -0.012838329235348646, + "nauc_cv_recall_at_3_diff1": 0.16385487158792883, + "nauc_cv_recall_at_3_max": 0.038761705606404406, + "nauc_cv_recall_at_3_std": -0.015444146140702468, + "nauc_cv_recall_at_5_diff1": 0.1342474969284102, + "nauc_cv_recall_at_5_max": 0.05126831131276464, + "nauc_cv_recall_at_5_std": -0.017244011947458535, + "nauc_map_at_1000_diff1": 0.16203592786234705, + "nauc_map_at_1000_max": 0.030620033890621814, + "nauc_map_at_1000_std": -0.040264811401896076, + "nauc_map_at_100_diff1": 0.16217564027873344, + "nauc_map_at_100_max": 0.030335963317346233, + "nauc_map_at_100_std": -0.04191441091203249, + "nauc_map_at_10_diff1": 0.1668748719138034, + "nauc_map_at_10_max": 0.019665458769663652, + "nauc_map_at_10_std": -0.061030326723274915, + "nauc_map_at_1_diff1": 0.20075357538267108, + "nauc_map_at_1_max": -0.013665387037587046, + "nauc_map_at_1_std": -0.07766759966282027, + "nauc_map_at_20_diff1": 0.16386357584503752, + "nauc_map_at_20_max": 0.027477956932007105, + "nauc_map_at_20_std": -0.05360888917496779, + "nauc_map_at_3_diff1": 0.1828162464833454, + "nauc_map_at_3_max": -0.00042884175525836654, + "nauc_map_at_3_std": -0.07592876118464074, + "nauc_map_at_5_diff1": 0.17337455887715922, + "nauc_map_at_5_max": 0.012236544553715457, + "nauc_map_at_5_std": -0.07362737289120426, + "nauc_mrr_at_1000_diff1": 0.182198101426749, + "nauc_mrr_at_1000_max": 0.0448381948571804, + "nauc_mrr_at_1000_std": -0.0001618378292135322, + "nauc_mrr_at_100_diff1": 0.18191433551703926, + "nauc_mrr_at_100_max": 0.04500572110663768, + "nauc_mrr_at_100_std": -0.0003744867725387041, + "nauc_mrr_at_10_diff1": 0.1826285591048027, + "nauc_mrr_at_10_max": 0.044086235201035766, + "nauc_mrr_at_10_std": -0.00239517797969451, + "nauc_mrr_at_1_diff1": 0.2158323552712354, + "nauc_mrr_at_1_max": 0.04444742333087822, + "nauc_mrr_at_1_std": 0.011407565122854697, + "nauc_mrr_at_20_diff1": 0.18160479504101013, + "nauc_mrr_at_20_max": 0.045950432193568896, + "nauc_mrr_at_20_std": -0.0017360413667605174, + "nauc_mrr_at_3_diff1": 0.19059323121455957, + "nauc_mrr_at_3_max": 0.042015172128788573, + "nauc_mrr_at_3_std": -0.0016791794293602408, + "nauc_mrr_at_5_diff1": 0.18283458974812974, + "nauc_mrr_at_5_max": 0.045060291442748604, + "nauc_mrr_at_5_std": -0.002104815863062162, + "nauc_ndcg_at_1000_diff1": 0.15156096151374038, + "nauc_ndcg_at_1000_max": 0.0406493878022788, + "nauc_ndcg_at_1000_std": 0.016921787353154537, + "nauc_ndcg_at_100_diff1": 0.15054121374308563, + "nauc_ndcg_at_100_max": 0.041154172886530165, + "nauc_ndcg_at_100_std": -0.001532780455810332, + "nauc_ndcg_at_10_diff1": 0.16153464028430178, + "nauc_ndcg_at_10_max": 0.02481835763026216, + "nauc_ndcg_at_10_std": -0.05078346238124407, + "nauc_ndcg_at_1_diff1": 0.21820190922791943, + "nauc_ndcg_at_1_max": 0.04420174217766181, + "nauc_ndcg_at_1_std": 0.009546631680834063, + "nauc_ndcg_at_20_diff1": 0.15686132177509635, + "nauc_ndcg_at_20_max": 0.037578777094029674, + "nauc_ndcg_at_20_std": -0.041502780883555516, + "nauc_ndcg_at_3_diff1": 0.17126079900331548, + "nauc_ndcg_at_3_max": 0.03401910693600456, + "nauc_ndcg_at_3_std": -0.031094315658336043, + "nauc_ndcg_at_5_diff1": 0.16453188548209727, + "nauc_ndcg_at_5_max": 0.029388384612040384, + "nauc_ndcg_at_5_std": -0.04943641033736309, + "nauc_precision_at_1000_diff1": -0.007222168267252968, + "nauc_precision_at_1000_max": 0.1160441913815496, + "nauc_precision_at_1000_std": 0.17313093891904213, + "nauc_precision_at_100_diff1": 0.0325067497088795, + "nauc_precision_at_100_max": 0.11942644169349934, + "nauc_precision_at_100_std": 0.15151622423514086, + "nauc_precision_at_10_diff1": 0.09750520702159386, + "nauc_precision_at_10_max": 0.08979528068901292, + "nauc_precision_at_10_std": 0.04192968105120749, + "nauc_precision_at_1_diff1": 0.21820190922791943, + "nauc_precision_at_1_max": 0.04420174217766181, + "nauc_precision_at_1_std": 0.009546631680834063, + "nauc_precision_at_20_diff1": 0.07494638242908558, + "nauc_precision_at_20_max": 0.11107312948815577, + "nauc_precision_at_20_std": 0.0678622239159027, + "nauc_precision_at_3_diff1": 0.15301239695643, + "nauc_precision_at_3_max": 0.06573995876968261, + "nauc_precision_at_3_std": 0.006095465119184258, + "nauc_precision_at_5_diff1": 0.12418883915813934, + "nauc_precision_at_5_max": 0.08324108054680586, + "nauc_precision_at_5_std": 0.009395259541333896, + "nauc_recall_at_1000_diff1": 0.09463810369253761, + "nauc_recall_at_1000_max": -0.0560747556388996, + "nauc_recall_at_1000_std": 0.3572969720013806, + "nauc_recall_at_100_diff1": 0.08317532906551336, + "nauc_recall_at_100_max": 0.009897416890410404, + "nauc_recall_at_100_std": 0.05893746815666451, + "nauc_recall_at_10_diff1": 0.11874898395838694, + "nauc_recall_at_10_max": -0.004616050285708542, + "nauc_recall_at_10_std": -0.08463423097463776, + "nauc_recall_at_1_diff1": 0.20075357538267108, + "nauc_recall_at_1_max": -0.013665387037587046, + "nauc_recall_at_1_std": -0.07766759966282027, + "nauc_recall_at_20_diff1": 0.10907207863790737, + "nauc_recall_at_20_max": 0.019150773341658055, + "nauc_recall_at_20_std": -0.07091348769187986, + "nauc_recall_at_3_diff1": 0.15218311570784662, + "nauc_recall_at_3_max": -0.01125829888252278, + "nauc_recall_at_3_std": -0.08844619980086296, + "nauc_recall_at_5_diff1": 0.1273697162111514, + "nauc_recall_at_5_max": 0.0008835763863017466, + "nauc_recall_at_5_std": -0.09530587120686718, + "ndcg_at_1": 0.20518, + "ndcg_at_10": 0.24494, + "ndcg_at_100": 0.32936, + "ndcg_at_1000": 0.36533, + "ndcg_at_20": 0.27892, + "ndcg_at_3": 0.19917, + "ndcg_at_5": 0.21463, + "precision_at_1": 0.20518, + "precision_at_10": 0.09327, + "precision_at_100": 0.01747, + "precision_at_1000": 0.00229, + "precision_at_20": 0.06034, + "precision_at_3": 0.16034, + "precision_at_5": 0.13533, + "recall_at_1": 0.07615, + "recall_at_10": 0.31954, + "recall_at_100": 0.64044, + "recall_at_1000": 0.8719, + "recall_at_20": 0.42323, + "recall_at_3": 0.16623, + "recall_at_5": 0.2331 + } + ] + }, + "task_name": "EDIST2ITRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2IRetrieval.json new file mode 100644 index 0000000000..cd8dcc6a44 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c6b162ee349adb293901128a18c0b446f7b43457", + "evaluation_time": 3894.6362721920013, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.23206, + "cv_recall_at_10": 0.39681, + "cv_recall_at_100": 0.54119, + "cv_recall_at_1000": 0.66873, + "cv_recall_at_20": 0.4349, + "cv_recall_at_3": 0.31709, + "cv_recall_at_5": 0.35164, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.15119, + "map_at_1": 0.02397, + "map_at_10": 0.06479, + "map_at_100": 0.08024, + "map_at_1000": 0.08268, + "map_at_20": 0.07251, + "map_at_3": 0.04501, + "map_at_5": 0.0545, + "mrr_at_1": 0.2320637732506643, + "mrr_at_10": 0.2828837853417125, + "mrr_at_100": 0.28837169422300607, + "mrr_at_1000": 0.2888825334937045, + "mrr_at_20": 0.28556164461936007, + "mrr_at_3": 0.2692648361381754, + "mrr_at_5": 0.2770593445527013, + "nauc_cv_recall_at_1000_diff1": 0.12259919578063702, + "nauc_cv_recall_at_1000_max": 0.2470948622454878, + "nauc_cv_recall_at_1000_std": 0.28799480962162954, + "nauc_cv_recall_at_100_diff1": 0.19420908123263017, + "nauc_cv_recall_at_100_max": 0.29687778489700106, + "nauc_cv_recall_at_100_std": 0.3231354986765902, + "nauc_cv_recall_at_10_diff1": 0.15306314553147926, + "nauc_cv_recall_at_10_max": 0.32002948607067444, + "nauc_cv_recall_at_10_std": 0.33790292791220566, + "nauc_cv_recall_at_1_diff1": 0.27344631062903946, + "nauc_cv_recall_at_1_max": 0.3959578109654247, + "nauc_cv_recall_at_1_std": 0.31118842144952946, + "nauc_cv_recall_at_20_diff1": 0.18529666847332407, + "nauc_cv_recall_at_20_max": 0.3300665371997956, + "nauc_cv_recall_at_20_std": 0.34546242556909446, + "nauc_cv_recall_at_3_diff1": 0.2064027555036601, + "nauc_cv_recall_at_3_max": 0.3741850802826935, + "nauc_cv_recall_at_3_std": 0.31988594991760555, + "nauc_cv_recall_at_5_diff1": 0.1704464042473904, + "nauc_cv_recall_at_5_max": 0.36195054598654247, + "nauc_cv_recall_at_5_std": 0.3283984149202229, + "nauc_map_at_1000_diff1": 0.1557976801642832, + "nauc_map_at_1000_max": 0.40047760186355585, + "nauc_map_at_1000_std": 0.3430030328191707, + "nauc_map_at_100_diff1": 0.15789089377828042, + "nauc_map_at_100_max": 0.39699246549039235, + "nauc_map_at_100_std": 0.3351785250251348, + "nauc_map_at_10_diff1": 0.17418470196289046, + "nauc_map_at_10_max": 0.358989373523996, + "nauc_map_at_10_std": 0.2748470090808481, + "nauc_map_at_1_diff1": 0.2670595538891123, + "nauc_map_at_1_max": 0.22266498241056235, + "nauc_map_at_1_std": 0.16770237726987589, + "nauc_map_at_20_diff1": 0.16473144746616353, + "nauc_map_at_20_max": 0.382369999843966, + "nauc_map_at_20_std": 0.30453408973596896, + "nauc_map_at_3_diff1": 0.2129585099540166, + "nauc_map_at_3_max": 0.2993400684456921, + "nauc_map_at_3_std": 0.218588537928361, + "nauc_map_at_5_diff1": 0.19694693530356244, + "nauc_map_at_5_max": 0.3342893645915572, + "nauc_map_at_5_std": 0.24672942576337248, + "nauc_mrr_at_1000_diff1": 0.23236391056686204, + "nauc_mrr_at_1000_max": 0.3756143254195909, + "nauc_mrr_at_1000_std": 0.31925253283001154, + "nauc_mrr_at_100_diff1": 0.23265291802648855, + "nauc_mrr_at_100_max": 0.3757140545121209, + "nauc_mrr_at_100_std": 0.3192347113370852, + "nauc_mrr_at_10_diff1": 0.22970198669237163, + "nauc_mrr_at_10_max": 0.37553145595069753, + "nauc_mrr_at_10_std": 0.31870821442127495, + "nauc_mrr_at_1_diff1": 0.27344631062903946, + "nauc_mrr_at_1_max": 0.3959578109654247, + "nauc_mrr_at_1_std": 0.31118842144952946, + "nauc_mrr_at_20_diff1": 0.23247830829777727, + "nauc_mrr_at_20_max": 0.37634277791741844, + "nauc_mrr_at_20_std": 0.31953209478801003, + "nauc_mrr_at_3_diff1": 0.24061890025051894, + "nauc_mrr_at_3_max": 0.3855162215701596, + "nauc_mrr_at_3_std": 0.31578819749904946, + "nauc_mrr_at_5_diff1": 0.2319139806137186, + "nauc_mrr_at_5_max": 0.3819090578458763, + "nauc_mrr_at_5_std": 0.3175158789349314, + "nauc_ndcg_at_1000_diff1": 0.15244014180873733, + "nauc_ndcg_at_1000_max": 0.37532662485358, + "nauc_ndcg_at_1000_std": 0.3547988725977299, + "nauc_ndcg_at_100_diff1": 0.15885667952123475, + "nauc_ndcg_at_100_max": 0.3729866051910131, + "nauc_ndcg_at_100_std": 0.33820898925869763, + "nauc_ndcg_at_10_diff1": 0.16388891530934463, + "nauc_ndcg_at_10_max": 0.40598231749041, + "nauc_ndcg_at_10_std": 0.3721247127572905, + "nauc_ndcg_at_1_diff1": 0.27344631062903946, + "nauc_ndcg_at_1_max": 0.3959578109654247, + "nauc_ndcg_at_1_std": 0.31118842144952946, + "nauc_ndcg_at_20_diff1": 0.16172703481778403, + "nauc_ndcg_at_20_max": 0.38288168356456037, + "nauc_ndcg_at_20_std": 0.3474159684723018, + "nauc_ndcg_at_3_diff1": 0.19679763706202097, + "nauc_ndcg_at_3_max": 0.4230963505688227, + "nauc_ndcg_at_3_std": 0.347768230242545, + "nauc_ndcg_at_5_diff1": 0.17995417866426847, + "nauc_ndcg_at_5_max": 0.4227315421417604, + "nauc_ndcg_at_5_std": 0.36635238151784116, + "nauc_precision_at_1000_diff1": 0.08337368153007142, + "nauc_precision_at_1000_max": 0.36980358553307513, + "nauc_precision_at_1000_std": 0.4450461238965513, + "nauc_precision_at_100_diff1": 0.08704233522482205, + "nauc_precision_at_100_max": 0.41699203263093915, + "nauc_precision_at_100_std": 0.4858353623514438, + "nauc_precision_at_10_diff1": 0.11122338641251278, + "nauc_precision_at_10_max": 0.43299919237102896, + "nauc_precision_at_10_std": 0.4184487772738942, + "nauc_precision_at_1_diff1": 0.27344631062903946, + "nauc_precision_at_1_max": 0.3959578109654247, + "nauc_precision_at_1_std": 0.31118842144952946, + "nauc_precision_at_20_diff1": 0.10990731374977268, + "nauc_precision_at_20_max": 0.4433545232195496, + "nauc_precision_at_20_std": 0.4496808942169916, + "nauc_precision_at_3_diff1": 0.17030870918458044, + "nauc_precision_at_3_max": 0.42980656332418254, + "nauc_precision_at_3_std": 0.3571964991226058, + "nauc_precision_at_5_diff1": 0.14154739179456016, + "nauc_precision_at_5_max": 0.4355121325859214, + "nauc_precision_at_5_std": 0.38477132537722974, + "nauc_recall_at_1000_diff1": 0.07754568718176012, + "nauc_recall_at_1000_max": 0.2569226744646939, + "nauc_recall_at_1000_std": 0.286809873936029, + "nauc_recall_at_100_diff1": 0.09581782819772992, + "nauc_recall_at_100_max": 0.2854307707331638, + "nauc_recall_at_100_std": 0.2766737038359882, + "nauc_recall_at_10_diff1": 0.114367230618245, + "nauc_recall_at_10_max": 0.2879608088269033, + "nauc_recall_at_10_std": 0.2502157880276106, + "nauc_recall_at_1_diff1": 0.2670595538891123, + "nauc_recall_at_1_max": 0.22266498241056235, + "nauc_recall_at_1_std": 0.16770237726987589, + "nauc_recall_at_20_diff1": 0.10103125164181147, + "nauc_recall_at_20_max": 0.29720895768918904, + "nauc_recall_at_20_std": 0.2590143736205232, + "nauc_recall_at_3_diff1": 0.18358542215665785, + "nauc_recall_at_3_max": 0.2756115962160671, + "nauc_recall_at_3_std": 0.20050115218597597, + "nauc_recall_at_5_diff1": 0.1504626011770959, + "nauc_recall_at_5_max": 0.2922839977079823, + "nauc_recall_at_5_std": 0.22274453322721552, + "ndcg_at_1": 0.23206, + "ndcg_at_10": 0.15119, + "ndcg_at_100": 0.15473, + "ndcg_at_1000": 0.18542, + "ndcg_at_20": 0.14062, + "ndcg_at_3": 0.19526, + "ndcg_at_5": 0.17429, + "precision_at_1": 0.23206, + "precision_at_10": 0.11001, + "precision_at_100": 0.02504, + "precision_at_1000": 0.00415, + "precision_at_20": 0.07516, + "precision_at_3": 0.18364, + "precision_at_5": 0.15182, + "recall_at_1": 0.02397, + "recall_at_10": 0.08901, + "recall_at_100": 0.17505, + "recall_at_1000": 0.28531, + "recall_at_20": 0.11334, + "recall_at_3": 0.05164, + "recall_at_5": 0.06666 + } + ] + }, + "task_name": "GLDv2I2IRetrieval" +} \ No newline at end of file From f0dd6f6aad36dd175956dd2ef06b050b2677376f Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sun, 10 Nov 2024 16:06:21 +0200 Subject: [PATCH 104/154] [mieb] use Logistic Regression classifier for AbsTaskImageMultilabelClassification (#1420) * use moc-lr classifier * set n_experiments=5 * run dinov2 and some laion models * add dinov2-giant results --- .../AbsTaskImageMultilabelClassification.py | 5 +- .../eng/PascalVOC2007.py | 2 +- .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 1 + .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 1 + .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- .../VOC2007.json | 38 +++++++++++---- .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- 21 files changed, 474 insertions(+), 19 deletions(-) create mode 100644 results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json create mode 100644 results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json create mode 100644 results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json create mode 100644 results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json create mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json create mode 100644 results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json create mode 100644 results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json create mode 100644 results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json create mode 100644 results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json diff --git a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py index 3cbd33ab53..d680f5d990 100644 --- a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py @@ -9,7 +9,8 @@ from sklearn.base import ClassifierMixin, clone from sklearn.metrics import f1_score, label_ranking_average_precision_score from sklearn.model_selection import train_test_split -from sklearn.neighbors import KNeighborsClassifier +from sklearn.multioutput import MultiOutputClassifier +from sklearn.linear_model import LogisticRegression from sklearn.preprocessing import MultiLabelBinarizer from mteb.abstasks.TaskMetadata import HFSubset @@ -52,7 +53,7 @@ class AbsTaskImageMultilabelClassification(AbsTask): image_column_name: str = "image" label_column_name: str = "labels" - classifier = KNeighborsClassifier(n_neighbors=5) + classifier = MultiOutputClassifier(estimator=LogisticRegression()) def __init__( self, diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index 4ceae17ff9..0ce54da8eb 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -54,4 +54,4 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): label_column_name: str = "classes" # To be removed when we want full results - n_experiments: int = 1 + n_experiments: int = 5 diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json new file mode 100644 index 0000000000..1240b81aa4 --- /dev/null +++ b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 71.94576096534729, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.3903069466882068, + "f1": 0.6257925811318822, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.5316062533656477, + "main_score": 0.3903069466882068, + "scores_per_experiment": [ + { + "accuracy": 0.3905492730210016, + "f1": 0.6235333142876379, + "lrap": 0.5233687847783186 + }, + { + "accuracy": 0.401453957996769, + "f1": 0.6273491281205146, + "lrap": 0.5416105726081543 + }, + { + "accuracy": 0.37621163166397414, + "f1": 0.6161039070570788, + "lrap": 0.5324994951534777 + }, + { + "accuracy": 0.38126009693053314, + "f1": 0.6219616890190912, + "lrap": 0.5204900376952104 + }, + { + "accuracy": 0.4020597738287561, + "f1": 0.6400148671750886, + "lrap": 0.540062376593077 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json index 6ae5f937c4..4d9fd59747 100644 --- a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json +++ b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json @@ -1 +1 @@ -{"name": "facebook/dinov2-base", "revision": "f9e44c814b77203eaa57a6bdbbd535f21ede1415", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"} \ No newline at end of file +{"name": "facebook/dinov2-base", "revision": "f9e44c814b77203eaa57a6bdbbd535f21ede1415", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json b/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json new file mode 100644 index 0000000000..730e213e2b --- /dev/null +++ b/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 408.0938787460327, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.3851373182552503, + "f1": 0.6315092630757977, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.5298672253634935, + "main_score": 0.3851373182552503, + "scores_per_experiment": [ + { + "accuracy": 0.3939822294022617, + "f1": 0.6271304662163256, + "lrap": 0.53264393735416 + }, + { + "accuracy": 0.3945880452342488, + "f1": 0.6263229032757067, + "lrap": 0.5370607835218142 + }, + { + "accuracy": 0.3784329563812601, + "f1": 0.6339676248065612, + "lrap": 0.5346142411595813 + }, + { + "accuracy": 0.3689418416801292, + "f1": 0.6304914034835388, + "lrap": 0.5144189777418785 + }, + { + "accuracy": 0.3897415185783522, + "f1": 0.6396339175968564, + "lrap": 0.5305981870400338 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json b/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json new file mode 100644 index 0000000000..87a8751f95 --- /dev/null +++ b/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json @@ -0,0 +1 @@ +{"name": "facebook/dinov2-giant", "revision": "611a9d42f2335e0f921f1e313ad3c1b7178d206d", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json new file mode 100644 index 0000000000..a9111cc617 --- /dev/null +++ b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 146.90911436080933, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.35315024232633274, + "f1": 0.5817890400372185, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.49049121567043397, + "main_score": 0.35315024232633274, + "scores_per_experiment": [ + { + "accuracy": 0.3667205169628433, + "f1": 0.591090519122089, + "lrap": 0.5039753859271227 + }, + { + "accuracy": 0.34995961227786754, + "f1": 0.5618180857706963, + "lrap": 0.48571452611739013 + }, + { + "accuracy": 0.34693053311793215, + "f1": 0.5774757520432393, + "lrap": 0.49124652216837067 + }, + { + "accuracy": 0.33582390953150243, + "f1": 0.5718289254309215, + "lrap": 0.4711502647639497 + }, + { + "accuracy": 0.36631663974151857, + "f1": 0.606731917819147, + "lrap": 0.5003693793753363 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json index 1de6c7be30..a84e0519fe 100644 --- a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json +++ b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json @@ -1 +1 @@ -{"name": "facebook/dinov2-large", "revision": "47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"} \ No newline at end of file +{"name": "facebook/dinov2-large", "revision": "47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json new file mode 100644 index 0000000000..e036d66bea --- /dev/null +++ b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 52.176159620285034, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.4340064620355412, + "f1": 0.6759098897017933, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.5801337843295732, + "main_score": 0.4340064620355412, + "scores_per_experiment": [ + { + "accuracy": 0.4184168012924071, + "f1": 0.6581607635477135, + "lrap": 0.563152373900564 + }, + { + "accuracy": 0.44810177705977383, + "f1": 0.6892652132242753, + "lrap": 0.5981864790881464 + }, + { + "accuracy": 0.44426494345718903, + "f1": 0.6797744937645096, + "lrap": 0.5992208535272061 + }, + { + "accuracy": 0.43073505654281097, + "f1": 0.6706176542382969, + "lrap": 0.5713583737210639 + }, + { + "accuracy": 0.42851373182552505, + "f1": 0.6817313237341718, + "lrap": 0.5687508414108857 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json index 2eca9d0047..5789f8b54f 100644 --- a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json +++ b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json @@ -1 +1 @@ -{"name": "facebook/dinov2-small", "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"} \ No newline at end of file +{"name": "facebook/dinov2-small", "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json b/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json new file mode 100644 index 0000000000..4257296d6f --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 39.63397932052612, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.5000807754442649, + "f1": 0.6763180027107614, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.6410416666666808, + "main_score": 0.5000807754442649, + "scores_per_experiment": [ + { + "accuracy": 0.5038368336025848, + "f1": 0.6876380960919419, + "lrap": 0.6470522572249287 + }, + { + "accuracy": 0.5232229402261712, + "f1": 0.6850605274296695, + "lrap": 0.6585363938251828 + }, + { + "accuracy": 0.4888933764135703, + "f1": 0.6592473172415618, + "lrap": 0.6350399950637378 + }, + { + "accuracy": 0.502827140549273, + "f1": 0.6767015784006876, + "lrap": 0.6400778024591767 + }, + { + "accuracy": 0.48162358642972536, + "f1": 0.6729424943899456, + "lrap": 0.6245018847603783 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json b/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json new file mode 100644 index 0000000000..5b161eaaff --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json @@ -0,0 +1 @@ +{"name": "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", "revision": "f0e2ffa09cbadab3db6a261ec1ec56407ce42912", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json new file mode 100644 index 0000000000..b0d4cc3355 --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 106.33361673355103, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.5382875605815831, + "f1": 0.7099573953664177, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.6770555667743827, + "main_score": 0.5382875605815831, + "scores_per_experiment": [ + { + "accuracy": 0.5484652665589661, + "f1": 0.7264512368883388, + "lrap": 0.6877928109854764 + }, + { + "accuracy": 0.5581583198707593, + "f1": 0.7220847968939508, + "lrap": 0.6972356847962836 + }, + { + "accuracy": 0.5282714054927302, + "f1": 0.6913016268300175, + "lrap": 0.6676300821217165 + }, + { + "accuracy": 0.5401857835218093, + "f1": 0.7000594523316099, + "lrap": 0.6753413323460928 + }, + { + "accuracy": 0.5163570274636511, + "f1": 0.7098898638881714, + "lrap": 0.6572779236223446 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json index 36d3659874..4ba9199948 100644 --- a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json +++ b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json @@ -1 +1 @@ -{"name": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "revision": "de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file +{"name": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "revision": "de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json new file mode 100644 index 0000000000..0cc887a6ba --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 139.95830011367798, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.5426898222940226, + "f1": 0.7116762127648723, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.6792292676359876, + "main_score": 0.5426898222940226, + "scores_per_experiment": [ + { + "accuracy": 0.5609854604200323, + "f1": 0.7310080393154441, + "lrap": 0.6944724914737191 + }, + { + "accuracy": 0.559168012924071, + "f1": 0.7125996315013976, + "lrap": 0.6941942649434737 + }, + { + "accuracy": 0.5220113085621971, + "f1": 0.6999211686407317, + "lrap": 0.6659638081134593 + }, + { + "accuracy": 0.5387722132471728, + "f1": 0.702612150503481, + "lrap": 0.6734136600251451 + }, + { + "accuracy": 0.5325121163166397, + "f1": 0.7122400738633065, + "lrap": 0.6681021136241404 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json index 5465d2e2eb..852d6e492a 100644 --- a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json +++ b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json @@ -1 +1 @@ -{"name": "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", "revision": "15efd0f6ac0c40c0f9da7becca03c974d7012604", "release_date": "2023-03-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file +{"name": "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", "revision": "15efd0f6ac0c40c0f9da7becca03c974d7012604", "release_date": "2023-03-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json new file mode 100644 index 0000000000..50150f4e71 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 65.64880728721619, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.523546042003231, + "f1": 0.6886687471443521, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.6682884019924772, + "main_score": 0.523546042003231, + "scores_per_experiment": [ + { + "accuracy": 0.5351373182552503, + "f1": 0.7049899989401454, + "lrap": 0.6795329608688017 + }, + { + "accuracy": 0.5300888529886915, + "f1": 0.6875495899200211, + "lrap": 0.6688218003949185 + }, + { + "accuracy": 0.5119143780290791, + "f1": 0.673686551264662, + "lrap": 0.6658084275713686 + }, + { + "accuracy": 0.5216074313408724, + "f1": 0.6802113731239721, + "lrap": 0.6663693681565408 + }, + { + "accuracy": 0.5189822294022617, + "f1": 0.6969062224729599, + "lrap": 0.6609094529707563 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json index 0b8a2cecb0..2bf553db27 100644 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json @@ -1 +1 @@ -{"name": "openai/clip-vit-base-patch16", "revision": "57c216476eefef5ab752ec549e440a49ae4ae5f3", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "CLIPModelWrapper"} \ No newline at end of file +{"name": "openai/clip-vit-base-patch16", "revision": "57c216476eefef5ab752ec549e440a49ae4ae5f3", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "CLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json index dafbb1f01e..44558b6f3d 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json @@ -1,24 +1,44 @@ { "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 55.247910261154175, + "evaluation_time": 48.62162375450134, "kg_co2_emissions": null, - "mteb_version": "1.12.80", + "mteb_version": "1.16.5", "scores": { "test": [ { - "accuracy": 0.5026252019386107, - "f1": 0.6780762548765893, + "accuracy": 0.5166801292407108, + "f1": 0.6851483326294392, "hf_subset": "default", "languages": [ "eng-Latn" ], - "lrap": 0.6642591096751196, - "main_score": 0.5026252019386107, + "lrap": 0.6602331269072138, + "main_score": 0.5166801292407108, "scores_per_experiment": [ { - "accuracy": 0.5026252019386107, - "f1": 0.6780762548765893, - "lrap": 0.6642591096751196 + "accuracy": 0.52140549273021, + "f1": 0.6981947346229871, + "lrap": 0.6660473882606517 + }, + { + "accuracy": 0.5357431340872375, + "f1": 0.6897556454047876, + "lrap": 0.6734304882427027 + }, + { + "accuracy": 0.5102988691437803, + "f1": 0.6665291163888459, + "lrap": 0.6582500336564507 + }, + { + "accuracy": 0.5149434571890146, + "f1": 0.6906546111504063, + "lrap": 0.6572852158499527 + }, + { + "accuracy": 0.5010096930533118, + "f1": 0.6806075555801697, + "lrap": 0.6461525085263116 } ] } diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json new file mode 100644 index 0000000000..151085b676 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 143.64798212051392, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.5198707592891761, + "f1": 0.6818774389774342, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.6617288749775786, + "main_score": 0.5198707592891761, + "scores_per_experiment": [ + { + "accuracy": 0.5310985460420032, + "f1": 0.696133632301483, + "lrap": 0.6760691527553577 + }, + { + "accuracy": 0.5274636510500808, + "f1": 0.6754522344136807, + "lrap": 0.6636039310716377 + }, + { + "accuracy": 0.507875605815832, + "f1": 0.6621408823160164, + "lrap": 0.6531808135882394 + }, + { + "accuracy": 0.5359450726978998, + "f1": 0.6914673370418792, + "lrap": 0.673578576557188 + }, + { + "accuracy": 0.4969709208400646, + "f1": 0.684193108814112, + "lrap": 0.6422119009154702 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json index 8c47f12e0c..69b52c701c 100644 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json @@ -1 +1 @@ -{"name": "openai/clip-vit-large-patch14", "revision": "32bd64288804d66eefd0ccbe215aa642df71cc41", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "CLIPModelWrapper"} \ No newline at end of file +{"name": "openai/clip-vit-large-patch14", "revision": "32bd64288804d66eefd0ccbe215aa642df71cc41", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "CLIPModelWrapper"} \ No newline at end of file From 66176a0a64651785faecdf07f9f96baf0b0a4d61 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Mon, 11 Nov 2024 03:33:06 +0800 Subject: [PATCH 105/154] [mieb] mieb scripts (siglip rerun & linear probing ablation & params count) (#1429) * mieb scripts * lint --- scripts/run_mieb_get_params.py | 28 ++++++++++++ scripts/run_mieb_kshot_ablation.py | 71 ++++++++++++++++++++++++++++++ scripts/run_mieb_rerun_siglip.py | 30 +++++++++++++ 3 files changed, 129 insertions(+) create mode 100644 scripts/run_mieb_get_params.py create mode 100644 scripts/run_mieb_kshot_ablation.py create mode 100644 scripts/run_mieb_rerun_siglip.py diff --git a/scripts/run_mieb_get_params.py b/scripts/run_mieb_get_params.py new file mode 100644 index 0000000000..a28a35ef65 --- /dev/null +++ b/scripts/run_mieb_get_params.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import pandas as pd +import torch +from tqdm import tqdm + +import mteb + +params = [] + +# add all model names +model_names = [ # "google/siglip-base-patch16-512", + "google/siglip-so400m-patch14-384" + # ... +] + +for model_name in tqdm(model_names): + model = mteb.get_model(model_name) + + total_params = sum(p.numel() for p in model.model.parameters()) + total_params = total_params / 1e6 + params.append([model_name, total_params]) + + del model + torch.cuda.empty_cache() + +param_frame = pd.DataFrame(params, columns=["model name", "# params"]) +param_frame.to_csv("params.csv", index=False) diff --git a/scripts/run_mieb_kshot_ablation.py b/scripts/run_mieb_kshot_ablation.py new file mode 100644 index 0000000000..a277c0d787 --- /dev/null +++ b/scripts/run_mieb_kshot_ablation.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import mteb + +for model_name in [ + # key ones for this ablation (different types of models) + "openai/clip-vit-base-patch32", + "openai/clip-vit-base-patch16", + "openai/clip-vit-large-patch14", + "royokong/e5-v", + "facebook/dinov2-small", + "facebook/dinov2-base", + "facebook/dinov2-large", + "facebook/dinov2-giant", + # more insights + "BAAI/bge-visualized-base", + "BAAI/bge-visualized-m3", + "google/siglip-so400m-patch14-384", + "google/siglip-base-patch16-256-multilingual", + "google/siglip-base-patch16-256", + "google/siglip-base-patch16-512", + "google/siglip-base-patch16-384", + "google/siglip-base-patch16-224", + "google/siglip-large-patch16-256", + "google/siglip-large-patch16-384", + "nyu-visionx/moco-v3-vit-b", + "nyu-visionx/moco-v3-vit-l", + "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", + "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", + "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", + "laion/CLIP-ViT-L-14-laion2B-s32B-b82K", + "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", + "EVA02-CLIP-B-16", + "EVA02-CLIP-L-14", + "EVA02-CLIP-bigE-14", + "EVA02-CLIP-bigE-14-plus", + "TIGER-Lab/VLM2Vec-LoRA", + "TIGER-Lab/VLM2Vec-Full", + # run if enough compute: + # "Salesforce/blip-itm-base-coco", + # "Salesforce/blip-itm-large-coco", + # "Salesforce/blip-itm-base-flickr", + # "Salesforce/blip-itm-large-flickr", + # "kakaobrain/align-base", + # "jinaai/jina-clip-v1", + # "nomic-ai/nomic-embed-vision-v1.5", + # "Salesforce/blip2-opt-2.7b", + # "Salesforce/blip2-opt-6.7b-coco", + # "embed-english-v3.0-v", # not feasible to run due to the 40 images/min constraint +]: + # 16 by default already + + for k_shot in [8, 32, 64, 128, 256]: + model = mteb.get_model(model_name) + tasks = mteb.get_tasks( + task_types=[ + "ImageClassification", + ] + ) + for task in tasks: + task.samples_per_label = k_shot + evaluation = mteb.MTEB(tasks=tasks) + results = evaluation.run( + model, output_folder=f"results-mieb-final/linear_probe_{k_shot}" + ) diff --git a/scripts/run_mieb_rerun_siglip.py b/scripts/run_mieb_rerun_siglip.py new file mode 100644 index 0000000000..539a31e2e7 --- /dev/null +++ b/scripts/run_mieb_rerun_siglip.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import mteb + +for model_name in [ + "google/siglip-so400m-patch14-384", + "google/siglip-base-patch16-256-multilingual", + "google/siglip-base-patch16-256", + "google/siglip-base-patch16-512", + "google/siglip-base-patch16-384", + "google/siglip-base-patch16-224", + "google/siglip-large-patch16-256", + "google/siglip-large-patch16-384", +]: + model = mteb.get_model(model_name) + tasks = mteb.get_tasks( + task_types=[ + "Any2AnyRetrieval", + "Any2AnyMultiChoice", + "Any2TextMutipleChoice", + "ImageClustering", + "ImageClassification", + "ImageMultilabelClassification", + "ImageTextPairClassification", + # "VisualSTS", # visual sts does not need rerun as will be the same after fixed. + "ZeroShotClassification", + ] + ) + evaluation = mteb.MTEB(tasks=tasks) + results = evaluation.run(model, output_folder="results-mieb-final/siglip_rerun") From 7e0779a86a6faf27cd94eb49d42165aafc0699c4 Mon Sep 17 00:00:00 2001 From: Jamie-Stirling <36764530+Jamie-Stirling@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:57:07 +0000 Subject: [PATCH 106/154] [MIEB] Change Flickr30k to test split (#1449) * wip: start adding BLIP models * add other blip variants * wip: add blip2_models.py * make lint * wip: implement blip2 wrapper * feat: add blip2 models, still mismatched names * fix: remove projections from image and text embeddings * make lint * wip: add coco BLIP2 * fix: BLIP2 better zero-shot classification without text_proj and vision_proj * tidy blip2 * add imagenet-dog-15 dataset * tidy and lint * remove unused import * add cluster_accuracy, ari and nmi to Image.ClusteringEvaluator * add imagenet-10 clustering task * add SOPI2IRetrieval * add results forclip on ImageNet10Clustering and ImageNetDog15Clustering * add SOPI2IRetrieval results for clip 32 * add results for clip vit 32/SOPI2IRetrieval * resolve conflict * add RP2kI2IRetrieval dataset * add RP2kI2IRetrieval results with clip-vit-base-patch32 * update image retrieval __init__.py * fix ImageTextPair dataloading for large datasets; more compositionality evaluation datasets * add RP2kI2IRetrieval and METI2IRetrieval * add METI2IRetreival * add SOP results * make lign * new revision for METI2IRetrieval * make lint * reset corpus chunk size * remove wrong classification import * add Flickr30k T2I and I2T * add Flickr30k T2I retriebal * reduced-size MET revision * fix: add Flickr30k T2I * make lint * add two landmark datasets and results * add Sketchy i2i retrieval * add task metadata * add BLINKIT2IRetrieval dataset * add BLINKIT2TRetrieval * add ImageCoDeT2IRetrieval * make lint * add vizwiz retrieval and results * fix vizwiz duplicate texts * add new vizwiz results * add VQA2 results * add GLD v2 I2T retrieval * add gld v2 i2i retrieval * make lint * add AbsTaskAny2AnyMultiChoice * make lint * remove GLDv2I2IRetrieval * exclude AbsTaskAny2AnyMultiChoice from test_load_data * fix e5v&vista * remove duplicate corpus entries from BLINKIT2TRetreival dataset * task type fix for running tasks * update BLINKIT2T metadata * fix wrong meta * run mieb script * split ROxford, RParis into easy, medium and hard * make lint * add BLINK as multi choice tasks * fix: license metadata in wrong format * remove null examples from corpus of ROxford and RParis * fix: add/remove subtasks from BLINKIT2IMultiChoice and BLINKIT2TMultiChoice * update blink metadata * add updated BLINK results * merge upstream mieb * change Flickr30k to test split * change flickr to test split --------- Co-authored-by: gowitheflow-1998 --- .../eng/Flickr30kI2TRetrieval.py | 2 +- .../eng/Flickr30kT2IRetrieval.py | 2 +- .../Flickr30kI2TRetrieval.json | 186 ------------------ .../Flickr30kT2IRetrieval.json | 186 ------------------ 4 files changed, 2 insertions(+), 374 deletions(-) delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py index 3a33733e1f..6ba591cf12 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py @@ -11,7 +11,7 @@ class Flickr30kI2TRetrieval(AbsTaskAny2AnyRetrieval): reference="https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31", dataset={ "path": "JamieSJS/flickr30k", - "revision": "a4cf34ac79215f9e2cd6a10342d84f606fc41cc3", + "revision": "24acb2d0b72e18b03388eb20a6225983c0e3f629", }, type="Any2AnyRetrieval", category="i2t", diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py index 585fcfc255..be56a554e4 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py @@ -11,7 +11,7 @@ class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval): reference="https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31", dataset={ "path": "JamieSJS/flickr30k", - "revision": "a4cf34ac79215f9e2cd6a10342d84f606fc41cc3", + "revision": "24acb2d0b72e18b03388eb20a6225983c0e3f629", }, type="Any2AnyRetrieval", category="t2i", diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json deleted file mode 100644 index 714a658b14..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "a4cf34ac79215f9e2cd6a10342d84f606fc41cc3", - "evaluation_time": 2905.3177580833435, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.4111, - "cv_recall_at_10": 0.74505, - "cv_recall_at_100": 0.94796, - "cv_recall_at_1000": 0.99607, - "cv_recall_at_20": 0.82443, - "cv_recall_at_3": 0.58196, - "cv_recall_at_5": 0.65757, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.33866, - "map_at_1": 0.08222, - "map_at_10": 0.2347, - "map_at_100": 0.26833, - "map_at_1000": 0.27226, - "map_at_20": 0.25195, - "map_at_3": 0.16585, - "map_at_5": 0.20389, - "mrr_at_1": 0.4111046624105243, - "mrr_at_10": 0.5151111351658093, - "mrr_at_100": 0.5239514165075607, - "mrr_at_1000": 0.5242073041502724, - "mrr_at_20": 0.5206470443576335, - "mrr_at_3": 0.4860761806496125, - "mrr_at_5": 0.503347412996305, - "nauc_cv_recall_at_1000_diff1": 0.18030127835739224, - "nauc_cv_recall_at_1000_max": 0.5581466344401298, - "nauc_cv_recall_at_1000_std": 0.8631168401994536, - "nauc_cv_recall_at_100_diff1": 0.272670704205089, - "nauc_cv_recall_at_100_max": 0.4612079444879557, - "nauc_cv_recall_at_100_std": 0.5695575488041834, - "nauc_cv_recall_at_10_diff1": 0.28543669447055997, - "nauc_cv_recall_at_10_max": 0.36347027563947776, - "nauc_cv_recall_at_10_std": 0.2385430096241168, - "nauc_cv_recall_at_1_diff1": 0.3784400483602523, - "nauc_cv_recall_at_1_max": 0.3504797969786945, - "nauc_cv_recall_at_1_std": 0.10634115715764078, - "nauc_cv_recall_at_20_diff1": 0.27508492044537997, - "nauc_cv_recall_at_20_max": 0.3714030729491561, - "nauc_cv_recall_at_20_std": 0.2959015251319611, - "nauc_cv_recall_at_3_diff1": 0.31233109201191445, - "nauc_cv_recall_at_3_max": 0.35059358000714524, - "nauc_cv_recall_at_3_std": 0.15142403433656834, - "nauc_cv_recall_at_5_diff1": 0.2974464195367057, - "nauc_cv_recall_at_5_max": 0.35509762440464737, - "nauc_cv_recall_at_5_std": 0.18640081229405794, - "nauc_map_at_1000_diff1": 0.2662124495945085, - "nauc_map_at_1000_max": 0.40406570060222413, - "nauc_map_at_1000_std": 0.1959460147481183, - "nauc_map_at_100_diff1": 0.2659471804759087, - "nauc_map_at_100_max": 0.40363986652542505, - "nauc_map_at_100_std": 0.1947488241847008, - "nauc_map_at_10_diff1": 0.2666647469717836, - "nauc_map_at_10_max": 0.39968979149613193, - "nauc_map_at_10_std": 0.16723592929025385, - "nauc_map_at_1_diff1": 0.37844004836025164, - "nauc_map_at_1_max": 0.3504797969786941, - "nauc_map_at_1_std": 0.1063411571576403, - "nauc_map_at_20_diff1": 0.2650108128343263, - "nauc_map_at_20_max": 0.4008348432286667, - "nauc_map_at_20_std": 0.17971414027637928, - "nauc_map_at_3_diff1": 0.28590676523718034, - "nauc_map_at_3_max": 0.3781596069511839, - "nauc_map_at_3_std": 0.13040101399163928, - "nauc_map_at_5_diff1": 0.27462944218025, - "nauc_map_at_5_max": 0.39938962890452623, - "nauc_map_at_5_std": 0.14980070399423312, - "nauc_mrr_at_1000_diff1": 0.3494548271148969, - "nauc_mrr_at_1000_max": 0.3521197992435183, - "nauc_mrr_at_1000_std": 0.1344856075273458, - "nauc_mrr_at_100_diff1": 0.349406446391289, - "nauc_mrr_at_100_max": 0.35218688878029347, - "nauc_mrr_at_100_std": 0.1347720116595227, - "nauc_mrr_at_10_diff1": 0.34832734062778187, - "nauc_mrr_at_10_max": 0.35184655254909514, - "nauc_mrr_at_10_std": 0.13412235150155113, - "nauc_mrr_at_1_diff1": 0.3784400483602523, - "nauc_mrr_at_1_max": 0.3504797969786945, - "nauc_mrr_at_1_std": 0.10634115715764078, - "nauc_mrr_at_20_diff1": 0.34874544122097734, - "nauc_mrr_at_20_max": 0.35198594125683075, - "nauc_mrr_at_20_std": 0.1348277834635207, - "nauc_mrr_at_3_diff1": 0.3502458632268556, - "nauc_mrr_at_3_max": 0.3503121014809114, - "nauc_mrr_at_3_std": 0.12461253730902709, - "nauc_mrr_at_5_diff1": 0.3483495916749157, - "nauc_mrr_at_5_max": 0.3512186490276025, - "nauc_mrr_at_5_std": 0.13065045260516017, - "nauc_ndcg_at_1000_diff1": 0.28085910013736953, - "nauc_ndcg_at_1000_max": 0.400173338146715, - "nauc_ndcg_at_1000_std": 0.2454880347293459, - "nauc_ndcg_at_100_diff1": 0.27614261400212725, - "nauc_ndcg_at_100_max": 0.3975804199137475, - "nauc_ndcg_at_100_std": 0.24550705671790238, - "nauc_ndcg_at_10_diff1": 0.2770040634103947, - "nauc_ndcg_at_10_max": 0.386892601870249, - "nauc_ndcg_at_10_std": 0.17593863508925003, - "nauc_ndcg_at_1_diff1": 0.3784400483602523, - "nauc_ndcg_at_1_max": 0.3504797969786945, - "nauc_ndcg_at_1_std": 0.10634115715764078, - "nauc_ndcg_at_20_diff1": 0.27393587281733567, - "nauc_ndcg_at_20_max": 0.3893030504620739, - "nauc_ndcg_at_20_std": 0.1976666061871801, - "nauc_ndcg_at_3_diff1": 0.29659323480247735, - "nauc_ndcg_at_3_max": 0.3710317896505282, - "nauc_ndcg_at_3_std": 0.1331916818585782, - "nauc_ndcg_at_5_diff1": 0.28580433201485833, - "nauc_ndcg_at_5_max": 0.3859484521144905, - "nauc_ndcg_at_5_std": 0.15313433521572764, - "nauc_precision_at_1000_diff1": 0.12752694892887595, - "nauc_precision_at_1000_max": 0.3590535825037547, - "nauc_precision_at_1000_std": 0.6227671512776303, - "nauc_precision_at_100_diff1": 0.18161894144710583, - "nauc_precision_at_100_max": 0.3507884567772516, - "nauc_precision_at_100_std": 0.39162073570098294, - "nauc_precision_at_10_diff1": 0.22500485735582146, - "nauc_precision_at_10_max": 0.37378539642037556, - "nauc_precision_at_10_std": 0.2035430362585049, - "nauc_precision_at_1_diff1": 0.3784400483602523, - "nauc_precision_at_1_max": 0.3504797969786945, - "nauc_precision_at_1_std": 0.10634115715764078, - "nauc_precision_at_20_diff1": 0.20677311842444712, - "nauc_precision_at_20_max": 0.3598552830533254, - "nauc_precision_at_20_std": 0.24219283310766115, - "nauc_precision_at_3_diff1": 0.26529469191714483, - "nauc_precision_at_3_max": 0.37325220096807005, - "nauc_precision_at_3_std": 0.14099019547945105, - "nauc_precision_at_5_diff1": 0.24620492543282482, - "nauc_precision_at_5_max": 0.3867515487955738, - "nauc_precision_at_5_std": 0.16846852569552279, - "nauc_recall_at_1000_diff1": 0.1275269489288761, - "nauc_recall_at_1000_max": 0.35905358250375646, - "nauc_recall_at_1000_std": 0.6227671512776317, - "nauc_recall_at_100_diff1": 0.1816189414471066, - "nauc_recall_at_100_max": 0.35078845677725146, - "nauc_recall_at_100_std": 0.3916207357009826, - "nauc_recall_at_10_diff1": 0.22500485735582146, - "nauc_recall_at_10_max": 0.37378539642037556, - "nauc_recall_at_10_std": 0.2035430362585049, - "nauc_recall_at_1_diff1": 0.37844004836025164, - "nauc_recall_at_1_max": 0.3504797969786941, - "nauc_recall_at_1_std": 0.1063411571576403, - "nauc_recall_at_20_diff1": 0.20677311842444712, - "nauc_recall_at_20_max": 0.3598552830533254, - "nauc_recall_at_20_std": 0.24219283310766115, - "nauc_recall_at_3_diff1": 0.2652946919171448, - "nauc_recall_at_3_max": 0.3732522009680695, - "nauc_recall_at_3_std": 0.1409901954794505, - "nauc_recall_at_5_diff1": 0.24620492543282482, - "nauc_recall_at_5_max": 0.3867515487955738, - "nauc_recall_at_5_std": 0.16846852569552279, - "ndcg_at_1": 0.4111, - "ndcg_at_10": 0.33866, - "ndcg_at_100": 0.44523, - "ndcg_at_1000": 0.49782, - "ndcg_at_20": 0.37695, - "ndcg_at_3": 0.33845, - "ndcg_at_5": 0.29078, - "precision_at_1": 0.4111, - "precision_at_10": 0.17182, - "precision_at_100": 0.03266, - "precision_at_1000": 0.00451, - "precision_at_20": 0.10825, - "precision_at_3": 0.31717, - "precision_at_5": 0.25644, - "recall_at_1": 0.08222, - "recall_at_10": 0.34363, - "recall_at_100": 0.65316, - "recall_at_1000": 0.90261, - "recall_at_20": 0.43299, - "recall_at_3": 0.1903, - "recall_at_5": 0.25644 - } - ] - }, - "task_name": "Flickr30kI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json deleted file mode 100644 index 60797e4e6b..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "a4cf34ac79215f9e2cd6a10342d84f606fc41cc3", - "evaluation_time": 3504.4338762760162, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.2175, - "cv_recall_at_10": 0.50718, - "cv_recall_at_100": 0.80839, - "cv_recall_at_1000": 0.96492, - "cv_recall_at_20": 0.60324, - "cv_recall_at_3": 0.34774, - "cv_recall_at_5": 0.41235, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.3504, - "map_at_1": 0.2175, - "map_at_10": 0.3017, - "map_at_100": 0.31355, - "map_at_1000": 0.31429, - "map_at_20": 0.30838, - "map_at_3": 0.2743, - "map_at_5": 0.28901, - "mrr_at_1": 0.21749532469207455, - "mrr_at_10": 0.3017006349410669, - "mrr_at_100": 0.3135464607899773, - "mrr_at_1000": 0.31428602518127013, - "mrr_at_20": 0.30838107727990743, - "mrr_at_3": 0.2743008533780535, - "mrr_at_5": 0.28900711506631577, - "nauc_cv_recall_at_1000_diff1": 0.2632892740429544, - "nauc_cv_recall_at_1000_max": 0.4683201344263211, - "nauc_cv_recall_at_1000_std": 0.7033795380897453, - "nauc_cv_recall_at_100_diff1": 0.2647647634897656, - "nauc_cv_recall_at_100_max": 0.34490671196475553, - "nauc_cv_recall_at_100_std": 0.3944496412003796, - "nauc_cv_recall_at_10_diff1": 0.2995130617977379, - "nauc_cv_recall_at_10_max": 0.28870905172779393, - "nauc_cv_recall_at_10_std": 0.13919504988489, - "nauc_cv_recall_at_1_diff1": 0.46334150143887837, - "nauc_cv_recall_at_1_max": 0.3188934433784608, - "nauc_cv_recall_at_1_std": 0.03870010544883344, - "nauc_cv_recall_at_20_diff1": 0.2830079535289093, - "nauc_cv_recall_at_20_max": 0.29612199469809664, - "nauc_cv_recall_at_20_std": 0.19159692443206613, - "nauc_cv_recall_at_3_diff1": 0.3535295484138601, - "nauc_cv_recall_at_3_max": 0.29806929455574155, - "nauc_cv_recall_at_3_std": 0.07658894036209522, - "nauc_cv_recall_at_5_diff1": 0.32210572836847934, - "nauc_cv_recall_at_5_max": 0.2888711825797017, - "nauc_cv_recall_at_5_std": 0.09843675944719517, - "nauc_map_at_1000_diff1": 0.39832464563018605, - "nauc_map_at_1000_max": 0.3074377337641309, - "nauc_map_at_1000_std": 0.07177029946282956, - "nauc_map_at_100_diff1": 0.3980484146317034, - "nauc_map_at_100_max": 0.3074089025502927, - "nauc_map_at_100_std": 0.0721090495580087, - "nauc_map_at_10_diff1": 0.39759464570461306, - "nauc_map_at_10_max": 0.3059467184032683, - "nauc_map_at_10_std": 0.06729002523587067, - "nauc_map_at_1_diff1": 0.46334150143887837, - "nauc_map_at_1_max": 0.3188934433784608, - "nauc_map_at_1_std": 0.03870010544883344, - "nauc_map_at_20_diff1": 0.39724585077726443, - "nauc_map_at_20_max": 0.30662830181613326, - "nauc_map_at_20_std": 0.07029104503264444, - "nauc_map_at_3_diff1": 0.4087813508135372, - "nauc_map_at_3_max": 0.3082838424431037, - "nauc_map_at_3_std": 0.05612029637486883, - "nauc_map_at_5_diff1": 0.40044699102255726, - "nauc_map_at_5_max": 0.3058538687154621, - "nauc_map_at_5_std": 0.06168460440199823, - "nauc_mrr_at_1000_diff1": 0.3983251214518209, - "nauc_mrr_at_1000_max": 0.3074398467997274, - "nauc_mrr_at_1000_std": 0.0717737083834719, - "nauc_mrr_at_100_diff1": 0.3980488893862368, - "nauc_mrr_at_100_max": 0.3074110112149868, - "nauc_mrr_at_100_std": 0.07211245146163399, - "nauc_mrr_at_10_diff1": 0.39759509662724174, - "nauc_mrr_at_10_max": 0.3059487942694512, - "nauc_mrr_at_10_std": 0.06729335845497784, - "nauc_mrr_at_1_diff1": 0.46334150143887837, - "nauc_mrr_at_1_max": 0.3188934433784608, - "nauc_mrr_at_1_std": 0.03870010544883344, - "nauc_mrr_at_20_diff1": 0.3972463051807796, - "nauc_mrr_at_20_max": 0.30663039310948675, - "nauc_mrr_at_20_std": 0.07029440252517179, - "nauc_mrr_at_3_diff1": 0.4087846165854502, - "nauc_mrr_at_3_max": 0.3082911152291177, - "nauc_mrr_at_3_std": 0.0561317481536657, - "nauc_mrr_at_5_diff1": 0.40044780533219726, - "nauc_mrr_at_5_max": 0.30585567761349425, - "nauc_mrr_at_5_std": 0.061687450057817825, - "nauc_ndcg_at_1000_diff1": 0.3822651584130574, - "nauc_ndcg_at_1000_max": 0.3096027445146429, - "nauc_ndcg_at_1000_std": 0.09905420670861774, - "nauc_ndcg_at_100_diff1": 0.3751136878938184, - "nauc_ndcg_at_100_max": 0.3096593022400737, - "nauc_ndcg_at_100_std": 0.1114725296151852, - "nauc_ndcg_at_10_diff1": 0.3722441527278821, - "nauc_ndcg_at_10_max": 0.30140653687780977, - "nauc_ndcg_at_10_std": 0.08492137004866748, - "nauc_ndcg_at_1_diff1": 0.46334150143887837, - "nauc_ndcg_at_1_max": 0.3188934433784608, - "nauc_ndcg_at_1_std": 0.03870010544883344, - "nauc_ndcg_at_20_diff1": 0.3702205468461526, - "nauc_ndcg_at_20_max": 0.30363863160972876, - "nauc_ndcg_at_20_std": 0.09630513131602707, - "nauc_ndcg_at_3_diff1": 0.39360911171689245, - "nauc_ndcg_at_3_max": 0.30546309152189277, - "nauc_ndcg_at_3_std": 0.061672579869672305, - "nauc_ndcg_at_5_diff1": 0.37922470007538506, - "nauc_ndcg_at_5_max": 0.3012774286231696, - "nauc_ndcg_at_5_std": 0.07144813701782884, - "nauc_precision_at_1000_diff1": 0.2632892740429434, - "nauc_precision_at_1000_max": 0.46832013442631265, - "nauc_precision_at_1000_std": 0.7033795380897392, - "nauc_precision_at_100_diff1": 0.26476476348976524, - "nauc_precision_at_100_max": 0.3449067119647543, - "nauc_precision_at_100_std": 0.3944496412003801, - "nauc_precision_at_10_diff1": 0.2995130617977381, - "nauc_precision_at_10_max": 0.28870905172779415, - "nauc_precision_at_10_std": 0.13919504988489012, - "nauc_precision_at_1_diff1": 0.46334150143887837, - "nauc_precision_at_1_max": 0.3188934433784608, - "nauc_precision_at_1_std": 0.03870010544883344, - "nauc_precision_at_20_diff1": 0.2830079535289092, - "nauc_precision_at_20_max": 0.29612199469809636, - "nauc_precision_at_20_std": 0.19159692443206594, - "nauc_precision_at_3_diff1": 0.3535203790281254, - "nauc_precision_at_3_max": 0.29804922704221115, - "nauc_precision_at_3_std": 0.07655745610155888, - "nauc_precision_at_5_diff1": 0.32210572836847917, - "nauc_precision_at_5_max": 0.2888711825797015, - "nauc_precision_at_5_std": 0.09843675944719478, - "nauc_recall_at_1000_diff1": 0.2632892740429544, - "nauc_recall_at_1000_max": 0.4683201344263211, - "nauc_recall_at_1000_std": 0.7033795380897453, - "nauc_recall_at_100_diff1": 0.2647647634897656, - "nauc_recall_at_100_max": 0.34490671196475553, - "nauc_recall_at_100_std": 0.3944496412003796, - "nauc_recall_at_10_diff1": 0.2995130617977379, - "nauc_recall_at_10_max": 0.28870905172779393, - "nauc_recall_at_10_std": 0.13919504988489, - "nauc_recall_at_1_diff1": 0.46334150143887837, - "nauc_recall_at_1_max": 0.3188934433784608, - "nauc_recall_at_1_std": 0.03870010544883344, - "nauc_recall_at_20_diff1": 0.2830079535289093, - "nauc_recall_at_20_max": 0.29612199469809664, - "nauc_recall_at_20_std": 0.19159692443206613, - "nauc_recall_at_3_diff1": 0.3535203790281251, - "nauc_recall_at_3_max": 0.29804922704221065, - "nauc_recall_at_3_std": 0.07655745610155873, - "nauc_recall_at_5_diff1": 0.32210572836847934, - "nauc_recall_at_5_max": 0.2888711825797017, - "nauc_recall_at_5_std": 0.09843675944719517, - "ndcg_at_1": 0.2175, - "ndcg_at_10": 0.3504, - "ndcg_at_100": 0.41258, - "ndcg_at_1000": 0.43242, - "ndcg_at_20": 0.3747, - "ndcg_at_3": 0.29313, - "ndcg_at_5": 0.31969, - "precision_at_1": 0.2175, - "precision_at_10": 0.05072, - "precision_at_100": 0.00808, - "precision_at_1000": 0.00096, - "precision_at_20": 0.03016, - "precision_at_3": 0.11591, - "precision_at_5": 0.08247, - "recall_at_1": 0.2175, - "recall_at_10": 0.50718, - "recall_at_100": 0.80839, - "recall_at_1000": 0.96492, - "recall_at_20": 0.60324, - "recall_at_3": 0.34773, - "recall_at_5": 0.41235 - } - ] - }, - "task_name": "Flickr30kT2IRetrieval" -} \ No newline at end of file From 1429ccee9211d59b7ae6c07c844b920a30136477 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 18 Nov 2024 22:47:23 +0200 Subject: [PATCH 107/154] [mieb] Fix VLM2vec dtype (#1462) * propagate dtype * fix fuse embeddings using list of PIL images --- mteb/models/vlm2vec_models.py | 72 ++++- .../CVBenchCount.json | 19 ++ .../model_meta.json | 1 + .../BLINKIT2IRetrieval.json | 302 +++++++++--------- 4 files changed, 238 insertions(+), 156 deletions(-) create mode 100644 results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/CVBenchCount.json create mode 100644 results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json diff --git a/mteb/models/vlm2vec_models.py b/mteb/models/vlm2vec_models.py index 8a1deab6bd..f48502eb0a 100644 --- a/mteb/models/vlm2vec_models.py +++ b/mteb/models/vlm2vec_models.py @@ -63,10 +63,10 @@ def __init__( lora_model = PeftModel.from_pretrained( base_model, checkpoint_path, config=lora_config ) - lora_model = lora_model.merge_and_unload() - model = lora_model + merged_model = lora_model.merge_and_unload() + model = merged_model.to(torch.bfloat16) # propagate dtype. else: - model = base_model + model = base_model.to(torch.bfloat16) model.eval() model.to(device) @@ -264,15 +264,77 @@ def get_fused_embeddings( with torch.no_grad(): for batch in images: + input_ids, pixel_values, image_sizes = [], [], [] for b in batch: text = next(texts) inputs = self.processor( f"<|image_1|> Represent the given image with the following question: {text}", [F.to_pil_image(b.to("cpu"))], + return_tensors="pt", + max_length=256, + truncation=True, + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + input_ids.append(inputs["input_ids"].squeeze(0).unsqueeze(1)) + pixel_values.append(inputs["pixel_values"]) + image_sizes.append(inputs["image_sizes"]) + + input_ids = torch._C._nn.pad_sequence( + input_ids, + batch_first=True, + padding_value=self.processor.tokenizer.pad_token_id, + ).squeeze(2) + attention_mask = input_ids.ne(self.processor.tokenizer.pad_token_id) + + pixel_values = torch.cat(pixel_values, dim=0) + image_sizes = torch.cat(image_sizes, dim=0) + inputs = { + "input_ids": input_ids, + "attention_mask": attention_mask, + "pixel_values": pixel_values, + "image_sizes": image_sizes, + } + + outputs = self.encode_input(inputs) + all_fused_embeddings.append(outputs.cpu().to(torch.float32)) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + input_ids, pixel_values, image_sizes = [], [], [] + for b in batch_images: + text = next(texts) + inputs = self.processor( + f"<|image_1|> Represent the given image with the following question: {text}", + [b], + return_tensors="pt", + max_length=256, + truncation=True, ) inputs = {k: v.to(self.device) for k, v in inputs.items()} - outputs = self.encode_input(inputs) - all_fused_embeddings.append(outputs.cpu().to(torch.float32)) + input_ids.append(inputs["input_ids"].squeeze(0).unsqueeze(1)) + pixel_values.append(inputs["pixel_values"]) + image_sizes.append(inputs["image_sizes"]) + + input_ids = torch._C._nn.pad_sequence( + input_ids, + batch_first=True, + padding_value=self.processor.tokenizer.pad_token_id, + ).squeeze(2) + attention_mask = input_ids.ne(self.processor.tokenizer.pad_token_id) + + pixel_values = torch.cat(pixel_values, dim=0) + image_sizes = torch.cat(image_sizes, dim=0) + inputs = { + "input_ids": input_ids, + "attention_mask": attention_mask, + "pixel_values": pixel_values, + "image_sizes": image_sizes, + } + + outputs = self.encode_input(inputs) + all_fused_embeddings.append(outputs.cpu().to(torch.float32)) + fused_embeddings = torch.cat(all_fused_embeddings, dim=0) return fused_embeddings diff --git a/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/CVBenchCount.json b/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/CVBenchCount.json new file mode 100644 index 0000000000..97fc301f69 --- /dev/null +++ b/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/CVBenchCount.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", + "evaluation_time": 141.80410814285278, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.6205583756345178, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6205583756345178 + } + ] + }, + "task_name": "CVBenchCount" +} \ No newline at end of file diff --git a/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json b/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json new file mode 100644 index 0000000000..c3494de385 --- /dev/null +++ b/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json @@ -0,0 +1 @@ +{"name": "TIGER-Lab/VLM2Vec-Full", "revision": "e9afa98002097ac2471827ba23ea1f2ddd229480", "release_date": "2024-10-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "VLM2VecWrapper"} \ No newline at end of file diff --git a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json index 9077b8c127..6d046886ce 100644 --- a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json +++ b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json @@ -1,184 +1,184 @@ { "dataset_revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26", - "evaluation_time": 224.3045289516449, + "evaluation_time": 213.14108443260193, "kg_co2_emissions": null, - "mteb_version": "1.14.21", + "mteb_version": "1.16.5", "scores": { "test": [ { - "cv_recall_at_1": 0.33582, - "cv_recall_at_10": 0.63184, + "cv_recall_at_1": 0.3209, + "cv_recall_at_10": 0.64179, "cv_recall_at_100": 0.90547, "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.72637, + "cv_recall_at_20": 0.72886, "cv_recall_at_3": 0.52736, - "cv_recall_at_5": 0.57711, + "cv_recall_at_5": 0.5796, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.48677, - "map_at_1": 0.33582, - "map_at_10": 0.43983, - "map_at_100": 0.45072, - "map_at_1000": 0.45136, - "map_at_20": 0.44629, - "map_at_3": 0.4204, - "map_at_5": 0.43209, - "mrr_at_1": 0.3358208955223881, - "mrr_at_10": 0.43982666034904844, - "mrr_at_100": 0.45071603631711843, - "mrr_at_1000": 0.4513620025552411, - "mrr_at_20": 0.44629389891710264, - "mrr_at_3": 0.42039800995024873, - "mrr_at_5": 0.43208955223880596, + "main_score": 0.4847, + "map_at_1": 0.3209, + "map_at_10": 0.434, + "map_at_100": 0.44398, + "map_at_1000": 0.44463, + "map_at_20": 0.43971, + "map_at_3": 0.41335, + "map_at_5": 0.42554, + "mrr_at_1": 0.3208955223880597, + "mrr_at_10": 0.434004580273237, + "mrr_at_100": 0.44398225504608213, + "mrr_at_1000": 0.4446309367234792, + "mrr_at_20": 0.43970614227302923, + "mrr_at_3": 0.4133499170812605, + "mrr_at_5": 0.4255389718076287, "nauc_cv_recall_at_1000_diff1": NaN, "nauc_cv_recall_at_1000_max": NaN, "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2698006811472229, - "nauc_cv_recall_at_100_max": -0.8185369176412147, + "nauc_cv_recall_at_100_diff1": 0.3319455914188296, + "nauc_cv_recall_at_100_max": -0.8496514687248068, "nauc_cv_recall_at_100_std": -1.0075573331300645, - "nauc_cv_recall_at_10_diff1": 0.1788243841076317, - "nauc_cv_recall_at_10_max": -0.612746605676537, - "nauc_cv_recall_at_10_std": -0.9530034638964193, - "nauc_cv_recall_at_1_diff1": 0.2822466928213175, - "nauc_cv_recall_at_1_max": -0.33285460282946566, - "nauc_cv_recall_at_1_std": -0.5640630200327668, - "nauc_cv_recall_at_20_diff1": 0.12960665874064498, - "nauc_cv_recall_at_20_max": -0.8021930752909697, - "nauc_cv_recall_at_20_std": -1.071727841531356, - "nauc_cv_recall_at_3_diff1": 0.2019990705156492, - "nauc_cv_recall_at_3_max": -0.4459723288652778, - "nauc_cv_recall_at_3_std": -0.7870326458461854, - "nauc_cv_recall_at_5_diff1": 0.23846869773197804, - "nauc_cv_recall_at_5_max": -0.47064146280491637, - "nauc_cv_recall_at_5_std": -0.8505490027952863, - "nauc_map_at_1000_diff1": 0.2409455106402186, - "nauc_map_at_1000_max": -0.407384704974283, - "nauc_map_at_1000_std": -0.6813392812409054, - "nauc_map_at_100_diff1": 0.24104039617379056, - "nauc_map_at_100_max": -0.408250752001871, - "nauc_map_at_100_std": -0.682039826842636, - "nauc_map_at_10_diff1": 0.2411578991609476, - "nauc_map_at_10_max": -0.4044975322245294, - "nauc_map_at_10_std": -0.6840284218652672, - "nauc_map_at_1_diff1": 0.2822466928213175, - "nauc_map_at_1_max": -0.33285460282946566, - "nauc_map_at_1_std": -0.5640630200327668, - "nauc_map_at_20_diff1": 0.2388129109858802, - "nauc_map_at_20_max": -0.4111868495020421, - "nauc_map_at_20_std": -0.6861574124125431, - "nauc_map_at_3_diff1": 0.2415896795362189, - "nauc_map_at_3_max": -0.3816620658278678, - "nauc_map_at_3_std": -0.6597288106948174, - "nauc_map_at_5_diff1": 0.2490237529189825, - "nauc_map_at_5_max": -0.3874920448893905, - "nauc_map_at_5_std": -0.6729363831482362, - "nauc_mrr_at_1000_diff1": 0.2409455106402186, - "nauc_mrr_at_1000_max": -0.407384704974283, - "nauc_mrr_at_1000_std": -0.6813392812409054, - "nauc_mrr_at_100_diff1": 0.24104039617379056, - "nauc_mrr_at_100_max": -0.408250752001871, - "nauc_mrr_at_100_std": -0.682039826842636, - "nauc_mrr_at_10_diff1": 0.2411578991609476, - "nauc_mrr_at_10_max": -0.4044975322245294, - "nauc_mrr_at_10_std": -0.6840284218652672, - "nauc_mrr_at_1_diff1": 0.2822466928213175, - "nauc_mrr_at_1_max": -0.33285460282946566, - "nauc_mrr_at_1_std": -0.5640630200327668, - "nauc_mrr_at_20_diff1": 0.2388129109858802, - "nauc_mrr_at_20_max": -0.4111868495020421, - "nauc_mrr_at_20_std": -0.6861574124125431, - "nauc_mrr_at_3_diff1": 0.2415896795362189, - "nauc_mrr_at_3_max": -0.3816620658278678, - "nauc_mrr_at_3_std": -0.6597288106948174, - "nauc_mrr_at_5_diff1": 0.2490237529189825, - "nauc_mrr_at_5_max": -0.3874920448893905, - "nauc_mrr_at_5_std": -0.6729363831482362, - "nauc_ndcg_at_1000_diff1": 0.2344092745371185, - "nauc_ndcg_at_1000_max": -0.4328446999637412, - "nauc_ndcg_at_1000_std": -0.709474885782739, - "nauc_ndcg_at_100_diff1": 0.23614140799135372, - "nauc_ndcg_at_100_max": -0.45059659752764697, - "nauc_ndcg_at_100_std": -0.7232249956899242, - "nauc_ndcg_at_10_diff1": 0.22732780326872332, - "nauc_ndcg_at_10_max": -0.44870094654610376, - "nauc_ndcg_at_10_std": -0.7429856899210265, - "nauc_ndcg_at_1_diff1": 0.2822466928213175, - "nauc_ndcg_at_1_max": -0.33285460282946566, - "nauc_ndcg_at_1_std": -0.5640630200327668, - "nauc_ndcg_at_20_diff1": 0.21923921046650915, - "nauc_ndcg_at_20_max": -0.4772077987509219, - "nauc_ndcg_at_20_std": -0.7553659757754395, - "nauc_ndcg_at_3_diff1": 0.23134535418875965, - "nauc_ndcg_at_3_max": -0.39788809593745006, - "nauc_ndcg_at_3_std": -0.691825805701685, - "nauc_ndcg_at_5_diff1": 0.24552033909645363, - "nauc_ndcg_at_5_max": -0.4076854503791393, - "nauc_ndcg_at_5_std": -0.7156597141210952, + "nauc_cv_recall_at_10_diff1": 0.21632765291249811, + "nauc_cv_recall_at_10_max": -0.616149982302474, + "nauc_cv_recall_at_10_std": -0.9479982037259836, + "nauc_cv_recall_at_1_diff1": 0.3430629866475006, + "nauc_cv_recall_at_1_max": -0.31499658742959696, + "nauc_cv_recall_at_1_std": -0.5455674065129812, + "nauc_cv_recall_at_20_diff1": 0.14497526898295157, + "nauc_cv_recall_at_20_max": -0.8183913780389244, + "nauc_cv_recall_at_20_std": -1.0686032736054183, + "nauc_cv_recall_at_3_diff1": 0.2272558706482997, + "nauc_cv_recall_at_3_max": -0.47485631997681976, + "nauc_cv_recall_at_3_std": -0.7893342152547034, + "nauc_cv_recall_at_5_diff1": 0.23402660393502006, + "nauc_cv_recall_at_5_max": -0.4968352312252504, + "nauc_cv_recall_at_5_std": -0.8652220828722653, + "nauc_map_at_1000_diff1": 0.28084189069845655, + "nauc_map_at_1000_max": -0.4050774810885909, + "nauc_map_at_1000_std": -0.6734464316794351, + "nauc_map_at_100_diff1": 0.2809924262004814, + "nauc_map_at_100_max": -0.4060074400165581, + "nauc_map_at_100_std": -0.6741534540228545, + "nauc_map_at_10_diff1": 0.28170348287831065, + "nauc_map_at_10_max": -0.40186513581417665, + "nauc_map_at_10_std": -0.6758814723680184, + "nauc_map_at_1_diff1": 0.3430629866475006, + "nauc_map_at_1_max": -0.31499658742959696, + "nauc_map_at_1_std": -0.5455674065129812, + "nauc_map_at_20_diff1": 0.278380373705771, + "nauc_map_at_20_max": -0.40905515991168534, + "nauc_map_at_20_std": -0.6780915297694656, + "nauc_map_at_3_diff1": 0.2816431806099485, + "nauc_map_at_3_max": -0.3850630283616802, + "nauc_map_at_3_std": -0.6525319695750113, + "nauc_map_at_5_diff1": 0.2837769102957866, + "nauc_map_at_5_max": -0.3886275351750067, + "nauc_map_at_5_std": -0.66779078741231, + "nauc_mrr_at_1000_diff1": 0.28084189069845655, + "nauc_mrr_at_1000_max": -0.4050774810885909, + "nauc_mrr_at_1000_std": -0.6734464316794351, + "nauc_mrr_at_100_diff1": 0.2809924262004814, + "nauc_mrr_at_100_max": -0.4060074400165581, + "nauc_mrr_at_100_std": -0.6741534540228545, + "nauc_mrr_at_10_diff1": 0.28170348287831065, + "nauc_mrr_at_10_max": -0.40186513581417665, + "nauc_mrr_at_10_std": -0.6758814723680184, + "nauc_mrr_at_1_diff1": 0.3430629866475006, + "nauc_mrr_at_1_max": -0.31499658742959696, + "nauc_mrr_at_1_std": -0.5455674065129812, + "nauc_mrr_at_20_diff1": 0.278380373705771, + "nauc_mrr_at_20_max": -0.40905515991168534, + "nauc_mrr_at_20_std": -0.6780915297694656, + "nauc_mrr_at_3_diff1": 0.2816431806099485, + "nauc_mrr_at_3_max": -0.3850630283616802, + "nauc_mrr_at_3_std": -0.6525319695750113, + "nauc_mrr_at_5_diff1": 0.2837769102957866, + "nauc_mrr_at_5_max": -0.3886275351750067, + "nauc_mrr_at_5_std": -0.66779078741231, + "nauc_ndcg_at_1000_diff1": 0.2723675027709315, + "nauc_ndcg_at_1000_max": -0.4322321611684638, + "nauc_ndcg_at_1000_std": -0.7027813195044652, + "nauc_ndcg_at_100_diff1": 0.27524643357042616, + "nauc_ndcg_at_100_max": -0.4514723022250352, + "nauc_ndcg_at_100_std": -0.7168373347468309, + "nauc_ndcg_at_10_diff1": 0.2663368914050526, + "nauc_ndcg_at_10_max": -0.44757349552373726, + "nauc_ndcg_at_10_std": -0.7355356954078378, + "nauc_ndcg_at_1_diff1": 0.3430629866475006, + "nauc_ndcg_at_1_max": -0.31499658742959696, + "nauc_ndcg_at_1_std": -0.5455674065129812, + "nauc_ndcg_at_20_diff1": 0.2541650696967836, + "nauc_ndcg_at_20_max": -0.47842206029525874, + "nauc_ndcg_at_20_std": -0.7484632948030998, + "nauc_ndcg_at_3_diff1": 0.2674065394667972, + "nauc_ndcg_at_3_max": -0.40781706959442965, + "nauc_ndcg_at_3_std": -0.6872034895658455, + "nauc_ndcg_at_5_diff1": 0.2709785209532911, + "nauc_ndcg_at_5_max": -0.4148914722291144, + "nauc_ndcg_at_5_std": -0.7152726168117532, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.2698006811472233, - "nauc_precision_at_100_max": -0.818536917641218, - "nauc_precision_at_100_std": -1.007557333130069, - "nauc_precision_at_10_diff1": 0.17882438410763166, - "nauc_precision_at_10_max": -0.6127466056765365, - "nauc_precision_at_10_std": -0.9530034638964187, - "nauc_precision_at_1_diff1": 0.2822466928213175, - "nauc_precision_at_1_max": -0.33285460282946566, - "nauc_precision_at_1_std": -0.5640630200327668, - "nauc_precision_at_20_diff1": 0.12960665874064572, - "nauc_precision_at_20_max": -0.8021930752909691, - "nauc_precision_at_20_std": -1.0717278415313547, - "nauc_precision_at_3_diff1": 0.20199907051564917, - "nauc_precision_at_3_max": -0.44597232886527743, - "nauc_precision_at_3_std": -0.7870326458461852, - "nauc_precision_at_5_diff1": 0.238468697731978, - "nauc_precision_at_5_max": -0.47064146280491626, - "nauc_precision_at_5_std": -0.8505490027952862, + "nauc_precision_at_100_diff1": 0.3319455914188293, + "nauc_precision_at_100_max": -0.8496514687248082, + "nauc_precision_at_100_std": -1.007557333130068, + "nauc_precision_at_10_diff1": 0.216327652912498, + "nauc_precision_at_10_max": -0.6161499823024736, + "nauc_precision_at_10_std": -0.9479982037259829, + "nauc_precision_at_1_diff1": 0.3430629866475006, + "nauc_precision_at_1_max": -0.31499658742959696, + "nauc_precision_at_1_std": -0.5455674065129812, + "nauc_precision_at_20_diff1": 0.14497526898295124, + "nauc_precision_at_20_max": -0.8183913780389255, + "nauc_precision_at_20_std": -1.068603273605419, + "nauc_precision_at_3_diff1": 0.22725587064830013, + "nauc_precision_at_3_max": -0.47485631997681943, + "nauc_precision_at_3_std": -0.789334215254703, + "nauc_precision_at_5_diff1": 0.2340266039350195, + "nauc_precision_at_5_max": -0.49683523122525086, + "nauc_precision_at_5_std": -0.8652220828722655, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2698006811472229, - "nauc_recall_at_100_max": -0.8185369176412147, + "nauc_recall_at_100_diff1": 0.3319455914188296, + "nauc_recall_at_100_max": -0.8496514687248068, "nauc_recall_at_100_std": -1.0075573331300645, - "nauc_recall_at_10_diff1": 0.1788243841076317, - "nauc_recall_at_10_max": -0.612746605676537, - "nauc_recall_at_10_std": -0.9530034638964193, - "nauc_recall_at_1_diff1": 0.2822466928213175, - "nauc_recall_at_1_max": -0.33285460282946566, - "nauc_recall_at_1_std": -0.5640630200327668, - "nauc_recall_at_20_diff1": 0.12960665874064498, - "nauc_recall_at_20_max": -0.8021930752909697, - "nauc_recall_at_20_std": -1.071727841531356, - "nauc_recall_at_3_diff1": 0.2019990705156492, - "nauc_recall_at_3_max": -0.4459723288652778, - "nauc_recall_at_3_std": -0.7870326458461854, - "nauc_recall_at_5_diff1": 0.23846869773197804, - "nauc_recall_at_5_max": -0.47064146280491637, - "nauc_recall_at_5_std": -0.8505490027952863, - "ndcg_at_1": 0.33582, - "ndcg_at_10": 0.48677, - "ndcg_at_100": 0.54341, - "ndcg_at_1000": 0.55646, - "ndcg_at_20": 0.51055, - "ndcg_at_3": 0.44788, - "ndcg_at_5": 0.46865, - "precision_at_1": 0.33582, - "precision_at_10": 0.06318, + "nauc_recall_at_10_diff1": 0.21632765291249811, + "nauc_recall_at_10_max": -0.616149982302474, + "nauc_recall_at_10_std": -0.9479982037259836, + "nauc_recall_at_1_diff1": 0.3430629866475006, + "nauc_recall_at_1_max": -0.31499658742959696, + "nauc_recall_at_1_std": -0.5455674065129812, + "nauc_recall_at_20_diff1": 0.14497526898295157, + "nauc_recall_at_20_max": -0.8183913780389244, + "nauc_recall_at_20_std": -1.0686032736054183, + "nauc_recall_at_3_diff1": 0.2272558706482997, + "nauc_recall_at_3_max": -0.47485631997681976, + "nauc_recall_at_3_std": -0.7893342152547034, + "nauc_recall_at_5_diff1": 0.23402660393502006, + "nauc_recall_at_5_max": -0.4968352312252504, + "nauc_recall_at_5_std": -0.8652220828722653, + "ndcg_at_1": 0.3209, + "ndcg_at_10": 0.4847, + "ndcg_at_100": 0.53853, + "ndcg_at_1000": 0.55159, + "ndcg_at_20": 0.50628, + "ndcg_at_3": 0.44269, + "ndcg_at_5": 0.46443, + "precision_at_1": 0.3209, + "precision_at_10": 0.06418, "precision_at_100": 0.00905, "precision_at_1000": 0.001, - "precision_at_20": 0.03632, + "precision_at_20": 0.03644, "precision_at_3": 0.17579, - "precision_at_5": 0.11542, - "recall_at_1": 0.33582, - "recall_at_10": 0.63184, + "precision_at_5": 0.11592, + "recall_at_1": 0.3209, + "recall_at_10": 0.64179, "recall_at_100": 0.90547, "recall_at_1000": 1.0, - "recall_at_20": 0.72637, + "recall_at_20": 0.72886, "recall_at_3": 0.52736, - "recall_at_5": 0.57711 + "recall_at_5": 0.5796 } ] }, From 2fc19e70d651a6d39a8b13ed1a6de475f3df4e6c Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Tue, 19 Nov 2024 05:57:24 +0800 Subject: [PATCH 108/154] [mieb] run script for missing results (#1472) * task type fix * scripts --- mteb/abstasks/TaskMetadata.py | 1 + .../Any2AnyRetrieval/eng/WebQAT2TRetrieval.py | 2 +- scripts/run_mieb_missed_results.py | 141 ++++++++++++++++++ 3 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 scripts/run_mieb_missed_results.py diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 872273e1d2..fa2e8c9e01 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -113,6 +113,7 @@ "s2s", # Sentence-to-sentence "s2p", # Sentence-to-paragraph "p2p", # Paragraph-to-paragraph + "t2t", # specifically for text-only tasks in mieb "i2i", "i2t", "t2i", diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py index c431af4394..eddc6e0fc4 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py @@ -14,7 +14,7 @@ class WebQAT2TRetrieval(AbsTaskAny2AnyRetrieval): "revision": "468b42a2b2e767d80d2d93f5ae5d42f135a10478", }, type="Any2AnyRetrieval", - category="s2p", + category="t2t", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", diff --git a/scripts/run_mieb_missed_results.py b/scripts/run_mieb_missed_results.py new file mode 100644 index 0000000000..f3b5231781 --- /dev/null +++ b/scripts/run_mieb_missed_results.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import mteb + +# missing +model = mteb.get_model("royokong/e5-v") +tasks = mteb.get_tasks(tasks=["EDIST2ITRetrieval", "Winoground"]) +evaluation = mteb.MTEB(tasks=tasks) +results = evaluation.run(model, output_folder="results-mieb-final") + +model = mteb.get_model("nyu-visionx/moco-v3-vit-b") +tasks = mteb.get_tasks( + tasks=[ + "CIFAR10Clustering", + "ImageNetDog15Clustering", + "TinyImageNetClustering", + "UCF101", + "VOC2007", + ] +) +evaluation = mteb.MTEB(tasks=tasks) +results = evaluation.run(model, output_folder="results-mieb-final") + +model = mteb.get_model("nyu-visionx/moco-v3-vit-l") +tasks = mteb.get_tasks( + tasks=[ + "SketchyI2IRetrieval", + "CIFAR10Clustering", + "ImageNetDog15Clustering", + "TinyImageNetClustering", + "UCF101", + "VOC2007", + ] +) +evaluation = mteb.MTEB(tasks=tasks) +results = evaluation.run(model, output_folder="results-mieb-final") + +model = mteb.get_model("BAAI/bge-visualized-base") +tasks = mteb.get_tasks( + tasks=[ + "SciMMIRI2TRetrieval", + "SciMMIRT2IRetrieval", + "VisualNewsI2TRetrieval", + "WITT2IRetrieval", + ] +) +evaluation = mteb.MTEB(tasks=tasks) +results = evaluation.run(model, output_folder="results-mieb-final") + +model = mteb.get_model("TIGER-Lab/VLM2Vec-Full") +tasks = mteb.get_tasks( + tasks=["CVBenchCount", "CVBenchDepth", "CVBenchDistance", "CVBenchRelation"] +) +evaluation = mteb.MTEB(tasks=tasks) +results = evaluation.run(model, output_folder="results-mieb-final") + +model = mteb.get_model("TIGER-Lab/VLM2Vec-LoRA") +tasks = mteb.get_tasks( + task_types=[ + "Any2AnyRetrieval", + "Any2AnyMultiChoice", + "Any2TextMutipleChoice", + "ImageClustering", + "ImageClassification", + "ImageMultilabelClassification", + "ImageTextPairClassification", + "VisualSTS", + "ZeroShotClassification", + ] +) +evaluation = mteb.MTEB(tasks=tasks) +results = evaluation.run(model, output_folder="results-mieb-final") + +# rerun +for model_name in [ + "openai/clip-vit-base-patch32", + "openai/clip-vit-base-patch16", + "openai/clip-vit-large-patch14", + "royokong/e5-v", + "BAAI/bge-visualized-base", + "BAAI/bge-visualized-m3", + "kakaobrain/align-base", + "jinaai/jina-clip-v1", + "nomic-ai/nomic-embed-vision-v1.5", + "Salesforce/blip-image-captioning-large", + "Salesforce/blip-image-captioning-base", + "Salesforce/blip2-opt-2.7b", + "Salesforce/blip2-opt-6.7b-coco", + "facebook/dinov2-small", + "facebook/dinov2-base", + "facebook/dinov2-large", + "facebook/dinov2-giant", + "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + "nyu-visionx/moco-v3-vit-b", + "nyu-visionx/moco-v3-vit-l", + "google/siglip-so400m-patch14-384", + "google/siglip-base-patch16-256-multilingual", + "google/siglip-base-patch16-256", + "google/siglip-base-patch16-512", + "google/siglip-base-patch16-384", + "google/siglip-base-patch16-224", + "google/siglip-large-patch16-256", + "google/siglip-large-patch16-384", + "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", + "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", + "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", + "laion/CLIP-ViT-L-14-laion2B-s32B-b82K", + "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", + "TIGER-Lab/VLM2Vec-LoRA", + "TIGER-Lab/VLM2Vec-Full", + "Salesforce/blip-itm-base-coco", + "Salesforce/blip-itm-large-coco", + "Salesforce/blip-itm-base-flickr", + "Salesforce/blip-itm-large-flickr", + "EVA02-CLIP-B-16", + "EVA02-CLIP-L-14", + "EVA02-CLIP-bigE-14", + "EVA02-CLIP-bigE-14-plus", +]: + model = mteb.get_model(model_name) + tasks = mteb.get_tasks( + tasks=[ + "ROxfordEasyI2IRetrieval", + "ROxfordHardI2IRetrieval", + "ROxfordMediumI2IRetrieval", + "RParisEasyI2IRetrieval", + "RParisHardI2IRetrieval", + "RParisMediumI2IRetrieval", + ] + ) + # get i-only tasks for i-only models. + if ("moco" in model_name) or ("dinov2" in model_name): + tasks = [task for task in tasks if "t" not in task.metadata.category] + + evaluation = mteb.MTEB(tasks=tasks) + results = evaluation.run(model, output_folder="results-mieb-rerun") From fab0b82a5c2680ab4e4823641955cdc7b9bb9891 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Fri, 22 Nov 2024 14:38:10 +0200 Subject: [PATCH 109/154] [mieb] Fix Moco model on CIFAR10Clustering (#1487) Fix Moco model on CIFAR10Clustering --- mteb/models/moco_models.py | 5 ++++- .../CIFAR10Clustering.json | 22 +++++++++++++++++++ .../model_meta.json | 2 +- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/CIFAR10Clustering.json diff --git a/mteb/models/moco_models.py b/mteb/models/moco_models.py index 3fa7dfe203..18e1a63ad9 100644 --- a/mteb/models/moco_models.py +++ b/mteb/models/moco_models.py @@ -78,8 +78,11 @@ def get_image_embeddings( with torch.no_grad(): for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] + inputs = torch.vstack( + [self.processor(b).unsqueeze(0) for b in batch_images] + ) output = self.model( - self.processor(batch_images) + inputs ) # output is (batch_size, num_features) shaped tensor all_image_embeddings.append(output) diff --git a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/CIFAR10Clustering.json b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/CIFAR10Clustering.json new file mode 100644 index 0000000000..981ba63e66 --- /dev/null +++ b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/CIFAR10Clustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + "evaluation_time": 265.58390378952026, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "ari": 0.6686950213456546, + "cluster_accuracy": 0.8305, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7469370718746896, + "nmi": 0.7469370718746896, + "v_measure": 0.7469370718746896 + } + ] + }, + "task_name": "CIFAR10Clustering" +} \ No newline at end of file diff --git a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json index d97b884a3b..6ec67ca445 100644 --- a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json +++ b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json @@ -1 +1 @@ -{"name": "nyu-visionx/moco-v3-vit-b", "revision": "7d091cd70772c5c0ecf7f00b5f12ca609a99d69d", "release_date": "2024-06-03", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "mocov3_loader"} \ No newline at end of file +{"name": "nyu-visionx/moco-v3-vit-b", "revision": "7d091cd70772c5c0ecf7f00b5f12ca609a99d69d", "release_date": "2024-06-03", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "mocov3_loader"} \ No newline at end of file From 67a035d81d483e0df574e1a1c6d223acfd81d6cf Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 27 Nov 2024 08:52:44 +0200 Subject: [PATCH 110/154] [mieb] Fix Flickr30k I2T and T2I (#1505) * remake flickr30k it2 and t2i * add openai clip vit-b32 b16 and jina-clip results * make lint --- .../eng/Flickr30kI2TRetrieval.py | 4 +- .../eng/Flickr30kT2IRetrieval.py | 18 +- .../Flickr30kI2TRetrieval.json | 186 ++++++++++++++++++ .../Flickr30kT2IRetrieval.json | 186 ++++++++++++++++++ .../Flickr30kI2TRetrieval.json | 186 ++++++++++++++++++ .../Flickr30kT2IRetrieval.json | 186 ++++++++++++++++++ .../Flickr30kI2TRetrieval.json | 186 ++++++++++++++++++ .../Flickr30kT2IRetrieval.json | 186 ++++++++++++++++++ .../data/flickr30k/build_flickr_30k_i2t.py | 52 +++++ .../data/flickr30k/build_flickr_30k_t2i.py | 54 +++++ 10 files changed, 1226 insertions(+), 18 deletions(-) create mode 100644 results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json create mode 100644 results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kT2IRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json create mode 100644 scripts/data/flickr30k/build_flickr_30k_i2t.py create mode 100644 scripts/data/flickr30k/build_flickr_30k_t2i.py diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py index 6ba591cf12..eb4a4d692c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py @@ -10,8 +10,8 @@ class Flickr30kI2TRetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve captions based on images.", reference="https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31", dataset={ - "path": "JamieSJS/flickr30k", - "revision": "24acb2d0b72e18b03388eb20a6225983c0e3f629", + "path": "isaacchung/flickr30ki2t", + "revision": "6984df6bd4380034e7766d9a992d8907df363efb", }, type="Any2AnyRetrieval", category="i2t", diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py index be56a554e4..b9775528c8 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py @@ -10,8 +10,8 @@ class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve images based on captions.", reference="https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31", dataset={ - "path": "JamieSJS/flickr30k", - "revision": "24acb2d0b72e18b03388eb20a6225983c0e3f629", + "path": "isaacchung/flickr30kt2i", + "revision": "e819702b287bfbe084e129a61f308a802b7c108e", }, type="Any2AnyRetrieval", category="t2i", @@ -39,17 +39,3 @@ class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval): "n_samples": {"default": 31014}, # qrels }, ) - - def load_data(self, **kwargs): - super().load_data(**kwargs) - # swap corpus and query - for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): - self.queries[split], self.corpus[split] = ( - self.corpus[split], - self.queries[split], - ) - self.relevant_docs[split] = { - cid: {qid: score} - for qid, cid_score in self.relevant_docs[split].items() - for cid, score in cid_score.items() - } diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json new file mode 100644 index 0000000000..b5e7e4bff6 --- /dev/null +++ b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb", + "evaluation_time": 12.655046463012695, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.806, + "cv_recall_at_10": 0.981, + "cv_recall_at_100": 0.999, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.994, + "cv_recall_at_3": 0.922, + "cv_recall_at_5": 0.961, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.75656, + "map_at_1": 0.1612, + "map_at_10": 0.65737, + "map_at_100": 0.6941, + "map_at_1000": 0.69494, + "map_at_20": 0.68213, + "map_at_3": 0.42077, + "map_at_5": 0.5824, + "mrr_at_1": 0.806, + "mrr_at_10": 0.8702051587301588, + "mrr_at_100": 0.8713558453277496, + "mrr_at_1000": 0.8713621346359256, + "mrr_at_20": 0.8711786018505061, + "mrr_at_3": 0.8581666666666669, + "mrr_at_5": 0.8672666666666671, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.3580765639589976, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 1.0, + "nauc_cv_recall_at_10_diff1": 0.4775664651825631, + "nauc_cv_recall_at_10_max": 0.7300113027667245, + "nauc_cv_recall_at_10_std": -0.05830753353973558, + "nauc_cv_recall_at_1_diff1": 0.3443688532171371, + "nauc_cv_recall_at_1_max": 0.4866993124852272, + "nauc_cv_recall_at_1_std": -0.1314705005806373, + "nauc_cv_recall_at_20_diff1": 0.5864456893868585, + "nauc_cv_recall_at_20_max": 0.9782135076252602, + "nauc_cv_recall_at_20_std": 0.5981948334889493, + "nauc_cv_recall_at_3_diff1": 0.3899961694079347, + "nauc_cv_recall_at_3_max": 0.5140295434413065, + "nauc_cv_recall_at_3_std": -0.12323732911968292, + "nauc_cv_recall_at_5_diff1": 0.48242715889774873, + "nauc_cv_recall_at_5_max": 0.5554957025545274, + "nauc_cv_recall_at_5_std": -0.16390145801910372, + "nauc_map_at_1000_diff1": 0.07860412999292432, + "nauc_map_at_1000_max": 0.5193407172877343, + "nauc_map_at_1000_std": -0.03554548015806451, + "nauc_map_at_100_diff1": 0.07860332936377284, + "nauc_map_at_100_max": 0.5202209025325473, + "nauc_map_at_100_std": -0.03402804669053566, + "nauc_map_at_10_diff1": 0.06738608544555293, + "nauc_map_at_10_max": 0.5046712341845588, + "nauc_map_at_10_std": -0.052089661267638164, + "nauc_map_at_1_diff1": 0.344368853217137, + "nauc_map_at_1_max": 0.4866993124852267, + "nauc_map_at_1_std": -0.1314705005806365, + "nauc_map_at_20_diff1": 0.07540132773467392, + "nauc_map_at_20_max": 0.5194163232471378, + "nauc_map_at_20_std": -0.03561110594882118, + "nauc_map_at_3_diff1": 0.13827655728488158, + "nauc_map_at_3_max": 0.48609865047620054, + "nauc_map_at_3_std": -0.09536745328619929, + "nauc_map_at_5_diff1": 0.07059479448464502, + "nauc_map_at_5_max": 0.48086096767511555, + "nauc_map_at_5_std": -0.08160089788651494, + "nauc_mrr_at_1000_diff1": 0.36023929557475076, + "nauc_mrr_at_1000_max": 0.5035860304880058, + "nauc_mrr_at_1000_std": -0.1249299862258316, + "nauc_mrr_at_100_diff1": 0.3602391842803307, + "nauc_mrr_at_100_max": 0.5036115760084787, + "nauc_mrr_at_100_std": -0.12487209719821057, + "nauc_mrr_at_10_diff1": 0.3611514720783506, + "nauc_mrr_at_10_max": 0.5051263876997938, + "nauc_mrr_at_10_std": -0.1257752596646823, + "nauc_mrr_at_1_diff1": 0.3443688532171371, + "nauc_mrr_at_1_max": 0.4866993124852272, + "nauc_mrr_at_1_std": -0.1314705005806373, + "nauc_mrr_at_20_diff1": 0.3606869616253337, + "nauc_mrr_at_20_max": 0.5042795551778488, + "nauc_mrr_at_20_std": -0.1239507002161601, + "nauc_mrr_at_3_diff1": 0.35911327275172067, + "nauc_mrr_at_3_max": 0.49885073455301077, + "nauc_mrr_at_3_std": -0.12597845586993137, + "nauc_mrr_at_5_diff1": 0.36427948002844535, + "nauc_mrr_at_5_max": 0.5025432319868158, + "nauc_mrr_at_5_std": -0.12876002101412964, + "nauc_ndcg_at_1000_diff1": 0.1404445479237733, + "nauc_ndcg_at_1000_max": 0.5347818240932689, + "nauc_ndcg_at_1000_std": -0.03316587882098286, + "nauc_ndcg_at_100_diff1": 0.1358934054564792, + "nauc_ndcg_at_100_max": 0.5460446827666793, + "nauc_ndcg_at_100_std": -0.007831409654410723, + "nauc_ndcg_at_10_diff1": 0.10422124872393643, + "nauc_ndcg_at_10_max": 0.522718634663143, + "nauc_ndcg_at_10_std": -0.042784056691292496, + "nauc_ndcg_at_1_diff1": 0.3443688532171371, + "nauc_ndcg_at_1_max": 0.4866993124852272, + "nauc_ndcg_at_1_std": -0.1314705005806373, + "nauc_ndcg_at_20_diff1": 0.12477162836638274, + "nauc_ndcg_at_20_max": 0.5506038459831472, + "nauc_ndcg_at_20_std": -0.0058328462109728755, + "nauc_ndcg_at_3_diff1": 0.1727965348989873, + "nauc_ndcg_at_3_max": 0.49295039267335533, + "nauc_ndcg_at_3_std": -0.09392106684131309, + "nauc_ndcg_at_5_diff1": 0.1101745223344572, + "nauc_ndcg_at_5_max": 0.4913548608912362, + "nauc_ndcg_at_5_std": -0.08489198959710985, + "nauc_precision_at_1000_diff1": -0.08286647992530825, + "nauc_precision_at_1000_max": 0.854341736694646, + "nauc_precision_at_1000_std": 0.8109243697478463, + "nauc_precision_at_100_diff1": 0.02976321846328062, + "nauc_precision_at_100_max": 0.6817348214942566, + "nauc_precision_at_100_std": 0.4525517335603402, + "nauc_precision_at_10_diff1": 0.03152919182802702, + "nauc_precision_at_10_max": 0.5209015914034817, + "nauc_precision_at_10_std": 0.012785208286159547, + "nauc_precision_at_1_diff1": 0.3443688532171371, + "nauc_precision_at_1_max": 0.4866993124852272, + "nauc_precision_at_1_std": -0.1314705005806373, + "nauc_precision_at_20_diff1": 0.058403294374015056, + "nauc_precision_at_20_max": 0.5976698949057889, + "nauc_precision_at_20_std": 0.15342898310071018, + "nauc_precision_at_3_diff1": 0.13112541027685642, + "nauc_precision_at_3_max": 0.4910052478350366, + "nauc_precision_at_3_std": -0.0846106773972222, + "nauc_precision_at_5_diff1": 0.057600655958168046, + "nauc_precision_at_5_max": 0.4823384872903635, + "nauc_precision_at_5_std": -0.07416100661323119, + "nauc_recall_at_1000_diff1": -0.08286647992533601, + "nauc_recall_at_1000_max": 0.8543417366946594, + "nauc_recall_at_1000_std": 0.8109243697478747, + "nauc_recall_at_100_diff1": 0.029763218463283347, + "nauc_recall_at_100_max": 0.6817348214942569, + "nauc_recall_at_100_std": 0.4525517335603416, + "nauc_recall_at_10_diff1": 0.03152919182802702, + "nauc_recall_at_10_max": 0.5209015914034817, + "nauc_recall_at_10_std": 0.012785208286159547, + "nauc_recall_at_1_diff1": 0.344368853217137, + "nauc_recall_at_1_max": 0.4866993124852267, + "nauc_recall_at_1_std": -0.1314705005806365, + "nauc_recall_at_20_diff1": 0.058403294374015056, + "nauc_recall_at_20_max": 0.5976698949057889, + "nauc_recall_at_20_std": 0.15342898310071018, + "nauc_recall_at_3_diff1": 0.1311254102768568, + "nauc_recall_at_3_max": 0.49100524783503674, + "nauc_recall_at_3_std": -0.0846106773972213, + "nauc_recall_at_5_diff1": 0.057600655958168046, + "nauc_recall_at_5_max": 0.4823384872903635, + "nauc_recall_at_5_std": -0.07416100661323119, + "ndcg_at_1": 0.806, + "ndcg_at_10": 0.75656, + "ndcg_at_100": 0.82771, + "ndcg_at_1000": 0.83522, + "ndcg_at_20": 0.79537, + "ndcg_at_3": 0.7524, + "ndcg_at_5": 0.67901, + "precision_at_1": 0.806, + "precision_at_10": 0.3869, + "precision_at_100": 0.04818, + "precision_at_1000": 0.00499, + "precision_at_20": 0.2159, + "precision_at_3": 0.735, + "precision_at_5": 0.6344, + "recall_at_1": 0.1612, + "recall_at_10": 0.7738, + "recall_at_100": 0.9636, + "recall_at_1000": 0.9976, + "recall_at_20": 0.8636, + "recall_at_3": 0.441, + "recall_at_5": 0.6344 + } + ] + }, + "task_name": "Flickr30kI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kT2IRetrieval.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kT2IRetrieval.json new file mode 100644 index 0000000000..2f5c0c550a --- /dev/null +++ b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "e819702b287bfbe084e129a61f308a802b7c108e", + "evaluation_time": 21.522692918777466, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.678, + "cv_recall_at_10": 0.9334, + "cv_recall_at_100": 0.994, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.9632, + "cv_recall_at_3": 0.8422, + "cv_recall_at_5": 0.89, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8094, + "map_at_1": 0.678, + "map_at_10": 0.76922, + "map_at_100": 0.77224, + "map_at_1000": 0.77227, + "map_at_20": 0.7714, + "map_at_3": 0.7522, + "map_at_5": 0.76322, + "mrr_at_1": 0.678, + "mrr_at_10": 0.7692153968253979, + "mrr_at_100": 0.7722356838203779, + "mrr_at_1000": 0.7722705791355338, + "mrr_at_20": 0.771396252397414, + "mrr_at_3": 0.752200000000002, + "mrr_at_5": 0.7632200000000005, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.7008714596949883, + "nauc_cv_recall_at_100_max": 0.7499999999999728, + "nauc_cv_recall_at_100_std": -0.37489884842825993, + "nauc_cv_recall_at_10_diff1": 0.5953937130407739, + "nauc_cv_recall_at_10_max": 0.5879408820585317, + "nauc_cv_recall_at_10_std": -0.2350459703400894, + "nauc_cv_recall_at_1_diff1": 0.7388944519574545, + "nauc_cv_recall_at_1_max": 0.5035033567670426, + "nauc_cv_recall_at_1_std": -0.10506388352655013, + "nauc_cv_recall_at_20_diff1": 0.6257002801120445, + "nauc_cv_recall_at_20_max": 0.6655198311208591, + "nauc_cv_recall_at_20_std": -0.19918402143466227, + "nauc_cv_recall_at_3_diff1": 0.6472713623239569, + "nauc_cv_recall_at_3_max": 0.5578714550455595, + "nauc_cv_recall_at_3_std": -0.13548415964522142, + "nauc_cv_recall_at_5_diff1": 0.6339326518340327, + "nauc_cv_recall_at_5_max": 0.5687226183317582, + "nauc_cv_recall_at_5_std": -0.15658878103255752, + "nauc_map_at_1000_diff1": 0.7108121576697071, + "nauc_map_at_1000_max": 0.5195954518905794, + "nauc_map_at_1000_std": -0.11880914292849726, + "nauc_map_at_100_diff1": 0.7108064511289186, + "nauc_map_at_100_max": 0.5196339191480183, + "nauc_map_at_100_std": -0.1188526782447686, + "nauc_map_at_10_diff1": 0.7089638573224181, + "nauc_map_at_10_max": 0.5198626919789708, + "nauc_map_at_10_std": -0.12106795154834728, + "nauc_map_at_1_diff1": 0.7388944519574545, + "nauc_map_at_1_max": 0.5035033567670426, + "nauc_map_at_1_std": -0.10506388352655013, + "nauc_map_at_20_diff1": 0.7104379728179321, + "nauc_map_at_20_max": 0.5201383461745716, + "nauc_map_at_20_std": -0.1193396676956454, + "nauc_map_at_3_diff1": 0.7078639395398761, + "nauc_map_at_3_max": 0.5200771632890915, + "nauc_map_at_3_std": -0.11708354472686522, + "nauc_map_at_5_diff1": 0.7087628559704173, + "nauc_map_at_5_max": 0.5201026397276295, + "nauc_map_at_5_std": -0.11894921046259736, + "nauc_mrr_at_1000_diff1": 0.7108121576697071, + "nauc_mrr_at_1000_max": 0.5195954518905794, + "nauc_mrr_at_1000_std": -0.11880914292849726, + "nauc_mrr_at_100_diff1": 0.7108064511289186, + "nauc_mrr_at_100_max": 0.5196339191480183, + "nauc_mrr_at_100_std": -0.1188526782447686, + "nauc_mrr_at_10_diff1": 0.7089638573224181, + "nauc_mrr_at_10_max": 0.5198626919789708, + "nauc_mrr_at_10_std": -0.12106795154834728, + "nauc_mrr_at_1_diff1": 0.7388944519574545, + "nauc_mrr_at_1_max": 0.5035033567670426, + "nauc_mrr_at_1_std": -0.10506388352655013, + "nauc_mrr_at_20_diff1": 0.7104379728179321, + "nauc_mrr_at_20_max": 0.5201383461745716, + "nauc_mrr_at_20_std": -0.1193396676956454, + "nauc_mrr_at_3_diff1": 0.7078639395398761, + "nauc_mrr_at_3_max": 0.5200771632890915, + "nauc_mrr_at_3_std": -0.11708354472686522, + "nauc_mrr_at_5_diff1": 0.7087628559704173, + "nauc_mrr_at_5_max": 0.5201026397276295, + "nauc_mrr_at_5_std": -0.11894921046259736, + "nauc_ndcg_at_1000_diff1": 0.7067180020680146, + "nauc_ndcg_at_1000_max": 0.5231648168494609, + "nauc_ndcg_at_1000_std": -0.12126906171818087, + "nauc_ndcg_at_100_diff1": 0.7066643145995184, + "nauc_ndcg_at_100_max": 0.5243091596475544, + "nauc_ndcg_at_100_std": -0.12259715494304686, + "nauc_ndcg_at_10_diff1": 0.6970018286625054, + "nauc_ndcg_at_10_max": 0.5270891134868764, + "nauc_ndcg_at_10_std": -0.13181013070978706, + "nauc_ndcg_at_1_diff1": 0.7388944519574545, + "nauc_ndcg_at_1_max": 0.5035033567670426, + "nauc_ndcg_at_1_std": -0.10506388352655013, + "nauc_ndcg_at_20_diff1": 0.7033385159678784, + "nauc_ndcg_at_20_max": 0.5287035840738418, + "nauc_ndcg_at_20_std": -0.12480446128344788, + "nauc_ndcg_at_3_diff1": 0.6960393372878148, + "nauc_ndcg_at_3_max": 0.5273632969286177, + "nauc_ndcg_at_3_std": -0.12075786696488118, + "nauc_ndcg_at_5_diff1": 0.6970584144874077, + "nauc_ndcg_at_5_max": 0.5276209122981296, + "nauc_ndcg_at_5_std": -0.1247281409835251, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.7008714596949908, + "nauc_precision_at_100_max": 0.749999999999983, + "nauc_precision_at_100_std": -0.37489884842824256, + "nauc_precision_at_10_diff1": 0.5953937130407729, + "nauc_precision_at_10_max": 0.5879408820585307, + "nauc_precision_at_10_std": -0.23504597034008565, + "nauc_precision_at_1_diff1": 0.7388944519574545, + "nauc_precision_at_1_max": 0.5035033567670426, + "nauc_precision_at_1_std": -0.10506388352655013, + "nauc_precision_at_20_diff1": 0.6257002801120534, + "nauc_precision_at_20_max": 0.6655198311208589, + "nauc_precision_at_20_std": -0.19918402143465108, + "nauc_precision_at_3_diff1": 0.6472713623239564, + "nauc_precision_at_3_max": 0.5578714550455599, + "nauc_precision_at_3_std": -0.13548415964522165, + "nauc_precision_at_5_diff1": 0.6339326518340342, + "nauc_precision_at_5_max": 0.568722618331759, + "nauc_precision_at_5_std": -0.15658878103255713, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.7008714596949883, + "nauc_recall_at_100_max": 0.7499999999999728, + "nauc_recall_at_100_std": -0.37489884842825993, + "nauc_recall_at_10_diff1": 0.5953937130407739, + "nauc_recall_at_10_max": 0.5879408820585317, + "nauc_recall_at_10_std": -0.2350459703400894, + "nauc_recall_at_1_diff1": 0.7388944519574545, + "nauc_recall_at_1_max": 0.5035033567670426, + "nauc_recall_at_1_std": -0.10506388352655013, + "nauc_recall_at_20_diff1": 0.6257002801120445, + "nauc_recall_at_20_max": 0.6655198311208591, + "nauc_recall_at_20_std": -0.19918402143466227, + "nauc_recall_at_3_diff1": 0.6472713623239569, + "nauc_recall_at_3_max": 0.5578714550455595, + "nauc_recall_at_3_std": -0.13548415964522142, + "nauc_recall_at_5_diff1": 0.6339326518340327, + "nauc_recall_at_5_max": 0.5687226183317582, + "nauc_recall_at_5_std": -0.15658878103255752, + "ndcg_at_1": 0.678, + "ndcg_at_10": 0.8094, + "ndcg_at_100": 0.82289, + "ndcg_at_1000": 0.82368, + "ndcg_at_20": 0.81708, + "ndcg_at_3": 0.77539, + "ndcg_at_5": 0.79516, + "precision_at_1": 0.678, + "precision_at_10": 0.09334, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "precision_at_20": 0.04816, + "precision_at_3": 0.28073, + "precision_at_5": 0.178, + "recall_at_1": 0.678, + "recall_at_10": 0.9334, + "recall_at_100": 0.994, + "recall_at_1000": 1.0, + "recall_at_20": 0.9632, + "recall_at_3": 0.8422, + "recall_at_5": 0.89 + } + ] + }, + "task_name": "Flickr30kT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kI2TRetrieval.json new file mode 100644 index 0000000000..a90131d30b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb", + "evaluation_time": 13.612479209899902, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.82, + "cv_recall_at_10": 0.986, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.994, + "cv_recall_at_3": 0.931, + "cv_recall_at_5": 0.97, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.74273, + "map_at_1": 0.164, + "map_at_10": 0.63451, + "map_at_100": 0.67573, + "map_at_1000": 0.67665, + "map_at_20": 0.66234, + "map_at_3": 0.4162, + "map_at_5": 0.56242, + "mrr_at_1": 0.82, + "mrr_at_10": 0.8819170634920639, + "mrr_at_100": 0.8826449941029878, + "mrr_at_1000": 0.8826449941029878, + "mrr_at_20": 0.8824860171691058, + "mrr_at_3": 0.8710000000000003, + "mrr_at_5": 0.8797000000000006, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": 0.19891289849274035, + "nauc_cv_recall_at_10_max": 0.5265772975856937, + "nauc_cv_recall_at_10_std": 0.3455382152861277, + "nauc_cv_recall_at_1_diff1": 0.4480945037999776, + "nauc_cv_recall_at_1_max": 0.3922045379447073, + "nauc_cv_recall_at_1_std": -0.12505231853728344, + "nauc_cv_recall_at_20_diff1": 0.3018985371926489, + "nauc_cv_recall_at_20_max": 0.6433239962651742, + "nauc_cv_recall_at_20_std": 0.9537037037037052, + "nauc_cv_recall_at_3_diff1": 0.2737655448652898, + "nauc_cv_recall_at_3_max": 0.4099108242330735, + "nauc_cv_recall_at_3_std": -0.1392102734813745, + "nauc_cv_recall_at_5_diff1": 0.2479458450046711, + "nauc_cv_recall_at_5_max": 0.45326797385621426, + "nauc_cv_recall_at_5_std": 0.024727668845316842, + "nauc_map_at_1000_diff1": 0.12299692538637216, + "nauc_map_at_1000_max": 0.40383972971788085, + "nauc_map_at_1000_std": -0.03877241232841388, + "nauc_map_at_100_diff1": 0.1224103741610186, + "nauc_map_at_100_max": 0.4045083510460462, + "nauc_map_at_100_std": -0.036813876580229825, + "nauc_map_at_10_diff1": 0.11422185007928484, + "nauc_map_at_10_max": 0.39191511510815347, + "nauc_map_at_10_std": -0.056740898404007624, + "nauc_map_at_1_diff1": 0.4480945037999775, + "nauc_map_at_1_max": 0.39220453794470633, + "nauc_map_at_1_std": -0.12505231853728427, + "nauc_map_at_20_diff1": 0.11978487595245653, + "nauc_map_at_20_max": 0.4010120010081494, + "nauc_map_at_20_std": -0.038459183289332906, + "nauc_map_at_3_diff1": 0.19814408531027977, + "nauc_map_at_3_max": 0.4018668277901667, + "nauc_map_at_3_std": -0.12036750531181802, + "nauc_map_at_5_diff1": 0.11645918188256225, + "nauc_map_at_5_max": 0.38608545461679483, + "nauc_map_at_5_std": -0.08112260515216518, + "nauc_mrr_at_1000_diff1": 0.4117843439551849, + "nauc_mrr_at_1000_max": 0.39603449433424776, + "nauc_mrr_at_1000_std": -0.1224537228873275, + "nauc_mrr_at_100_diff1": 0.4117843439551849, + "nauc_mrr_at_100_max": 0.39603449433424776, + "nauc_mrr_at_100_std": -0.1224537228873275, + "nauc_mrr_at_10_diff1": 0.40988550434587345, + "nauc_mrr_at_10_max": 0.39584154636889374, + "nauc_mrr_at_10_std": -0.12111989031979221, + "nauc_mrr_at_1_diff1": 0.4480945037999776, + "nauc_mrr_at_1_max": 0.3922045379447073, + "nauc_mrr_at_1_std": -0.12505231853728344, + "nauc_mrr_at_20_diff1": 0.41146243588617815, + "nauc_mrr_at_20_max": 0.3962597833666116, + "nauc_mrr_at_20_std": -0.12095286988661123, + "nauc_mrr_at_3_diff1": 0.39902923729559436, + "nauc_mrr_at_3_max": 0.3943872791235097, + "nauc_mrr_at_3_std": -0.13306276199025832, + "nauc_mrr_at_5_diff1": 0.4076485858771895, + "nauc_mrr_at_5_max": 0.39577465779193577, + "nauc_mrr_at_5_std": -0.12265544640203102, + "nauc_ndcg_at_1000_diff1": 0.1844301804081895, + "nauc_ndcg_at_1000_max": 0.42130933536662635, + "nauc_ndcg_at_1000_std": -0.035065914910614955, + "nauc_ndcg_at_100_diff1": 0.17496821020126946, + "nauc_ndcg_at_100_max": 0.43029672300582816, + "nauc_ndcg_at_100_std": -0.002507684096230846, + "nauc_ndcg_at_10_diff1": 0.14481662979912127, + "nauc_ndcg_at_10_max": 0.40520681410698756, + "nauc_ndcg_at_10_std": -0.04220162909542145, + "nauc_ndcg_at_1_diff1": 0.4480945037999776, + "nauc_ndcg_at_1_max": 0.3922045379447073, + "nauc_ndcg_at_1_std": -0.12505231853728344, + "nauc_ndcg_at_20_diff1": 0.15803572920290762, + "nauc_ndcg_at_20_max": 0.42054444917147027, + "nauc_ndcg_at_20_std": -0.0019159299606582648, + "nauc_ndcg_at_3_diff1": 0.21765219679209533, + "nauc_ndcg_at_3_max": 0.40379267362903226, + "nauc_ndcg_at_3_std": -0.12304328773334444, + "nauc_ndcg_at_5_diff1": 0.1462495129764977, + "nauc_ndcg_at_5_max": 0.3934225656825324, + "nauc_ndcg_at_5_std": -0.0786882910212851, + "nauc_precision_at_1000_diff1": 0.62427637721762, + "nauc_precision_at_1000_max": 0.7399626517273514, + "nauc_precision_at_1000_std": 0.7899159663865152, + "nauc_precision_at_100_diff1": 0.028283567165904248, + "nauc_precision_at_100_max": 0.5215659239456432, + "nauc_precision_at_100_std": 0.5279993895443634, + "nauc_precision_at_10_diff1": 0.06152272326044776, + "nauc_precision_at_10_max": 0.3828014005531608, + "nauc_precision_at_10_std": 0.018326040537019386, + "nauc_precision_at_1_diff1": 0.4480945037999776, + "nauc_precision_at_1_max": 0.3922045379447073, + "nauc_precision_at_1_std": -0.12505231853728344, + "nauc_precision_at_20_diff1": 0.06089982228042279, + "nauc_precision_at_20_max": 0.4065587877654105, + "nauc_precision_at_20_std": 0.1612227106912349, + "nauc_precision_at_3_diff1": 0.16178283766113552, + "nauc_precision_at_3_max": 0.3991300609614105, + "nauc_precision_at_3_std": -0.12286918672342817, + "nauc_precision_at_5_diff1": 0.08039511083398675, + "nauc_precision_at_5_max": 0.3789339125780607, + "nauc_precision_at_5_std": -0.05814725911556473, + "nauc_recall_at_1000_diff1": 0.624276377217619, + "nauc_recall_at_1000_max": 0.7399626517273947, + "nauc_recall_at_1000_std": 0.7899159663865074, + "nauc_recall_at_100_diff1": 0.028283567165905115, + "nauc_recall_at_100_max": 0.5215659239456432, + "nauc_recall_at_100_std": 0.5279993895443671, + "nauc_recall_at_10_diff1": 0.06152272326044776, + "nauc_recall_at_10_max": 0.3828014005531608, + "nauc_recall_at_10_std": 0.018326040537019386, + "nauc_recall_at_1_diff1": 0.4480945037999775, + "nauc_recall_at_1_max": 0.39220453794470633, + "nauc_recall_at_1_std": -0.12505231853728427, + "nauc_recall_at_20_diff1": 0.06089982228042279, + "nauc_recall_at_20_max": 0.4065587877654105, + "nauc_recall_at_20_std": 0.1612227106912349, + "nauc_recall_at_3_diff1": 0.161782837661136, + "nauc_recall_at_3_max": 0.3991300609614111, + "nauc_recall_at_3_std": -0.12286918672342777, + "nauc_recall_at_5_diff1": 0.08039511083398675, + "nauc_recall_at_5_max": 0.3789339125780607, + "nauc_recall_at_5_std": -0.05814725911556473, + "ndcg_at_1": 0.82, + "ndcg_at_10": 0.74273, + "ndcg_at_100": 0.82168, + "ndcg_at_1000": 0.82961, + "ndcg_at_20": 0.78527, + "ndcg_at_3": 0.7496, + "ndcg_at_5": 0.6661, + "precision_at_1": 0.82, + "precision_at_10": 0.3759, + "precision_at_100": 0.04818, + "precision_at_1000": 0.00499, + "precision_at_20": 0.2125, + "precision_at_3": 0.72567, + "precision_at_5": 0.6134, + "recall_at_1": 0.164, + "recall_at_10": 0.7518, + "recall_at_100": 0.9636, + "recall_at_1000": 0.999, + "recall_at_20": 0.85, + "recall_at_3": 0.4354, + "recall_at_5": 0.6134 + } + ] + }, + "task_name": "Flickr30kI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kT2IRetrieval.json new file mode 100644 index 0000000000..8b907392b6 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "e819702b287bfbe084e129a61f308a802b7c108e", + "evaluation_time": 22.22239637374878, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.6214, + "cv_recall_at_10": 0.915, + "cv_recall_at_100": 0.9932, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.9532, + "cv_recall_at_3": 0.7944, + "cv_recall_at_5": 0.854, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.76748, + "map_at_1": 0.6214, + "map_at_10": 0.72031, + "map_at_100": 0.72409, + "map_at_1000": 0.72412, + "map_at_20": 0.72299, + "map_at_3": 0.69867, + "map_at_5": 0.71205, + "mrr_at_1": 0.6214, + "mrr_at_10": 0.7203121428571442, + "mrr_at_100": 0.724085885840924, + "mrr_at_1000": 0.7241243831788744, + "mrr_at_20": 0.7229865462203786, + "mrr_at_3": 0.6986666666666691, + "mrr_at_5": 0.7120466666666677, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6346185533036849, + "nauc_cv_recall_at_100_max": 0.6848052946668853, + "nauc_cv_recall_at_100_std": 0.4664002856044359, + "nauc_cv_recall_at_10_diff1": 0.5758345691217669, + "nauc_cv_recall_at_10_max": 0.5616378316032299, + "nauc_cv_recall_at_10_std": 0.15670456417861245, + "nauc_cv_recall_at_1_diff1": 0.6964868193650136, + "nauc_cv_recall_at_1_max": 0.4658840817263219, + "nauc_cv_recall_at_1_std": 0.06009948824657468, + "nauc_cv_recall_at_20_diff1": 0.5942604962212811, + "nauc_cv_recall_at_20_max": 0.5904757914561845, + "nauc_cv_recall_at_20_std": 0.18555826889159976, + "nauc_cv_recall_at_3_diff1": 0.6016933324728311, + "nauc_cv_recall_at_3_max": 0.4996714444125018, + "nauc_cv_recall_at_3_std": 0.11144390402265514, + "nauc_cv_recall_at_5_diff1": 0.596326854581594, + "nauc_cv_recall_at_5_max": 0.5435354775022344, + "nauc_cv_recall_at_5_std": 0.1647132585408802, + "nauc_map_at_1000_diff1": 0.6681451778281712, + "nauc_map_at_1000_max": 0.48153054698905345, + "nauc_map_at_1000_std": 0.08084200415260853, + "nauc_map_at_100_diff1": 0.6681421969134242, + "nauc_map_at_100_max": 0.4815602459122235, + "nauc_map_at_100_std": 0.08089746305416494, + "nauc_map_at_10_diff1": 0.66637927944471, + "nauc_map_at_10_max": 0.4824993978185994, + "nauc_map_at_10_std": 0.08173000407846276, + "nauc_map_at_1_diff1": 0.6964868193650136, + "nauc_map_at_1_max": 0.4658840817263219, + "nauc_map_at_1_std": 0.06009948824657468, + "nauc_map_at_20_diff1": 0.6677072473838044, + "nauc_map_at_20_max": 0.4820334855898056, + "nauc_map_at_20_std": 0.08112000061939652, + "nauc_map_at_3_diff1": 0.6629064040013647, + "nauc_map_at_3_max": 0.47830764546568383, + "nauc_map_at_3_std": 0.07908104385148582, + "nauc_map_at_5_diff1": 0.6651150515375697, + "nauc_map_at_5_max": 0.48320424406293105, + "nauc_map_at_5_std": 0.0841376160310789, + "nauc_mrr_at_1000_diff1": 0.6681451778281712, + "nauc_mrr_at_1000_max": 0.48153054698905345, + "nauc_mrr_at_1000_std": 0.08084200415260853, + "nauc_mrr_at_100_diff1": 0.6681421969134242, + "nauc_mrr_at_100_max": 0.4815602459122235, + "nauc_mrr_at_100_std": 0.08089746305416494, + "nauc_mrr_at_10_diff1": 0.66637927944471, + "nauc_mrr_at_10_max": 0.4824993978185994, + "nauc_mrr_at_10_std": 0.08173000407846276, + "nauc_mrr_at_1_diff1": 0.6964868193650136, + "nauc_mrr_at_1_max": 0.4658840817263219, + "nauc_mrr_at_1_std": 0.06009948824657468, + "nauc_mrr_at_20_diff1": 0.6677072473838044, + "nauc_mrr_at_20_max": 0.4820334855898056, + "nauc_mrr_at_20_std": 0.08112000061939652, + "nauc_mrr_at_3_diff1": 0.6629064040013647, + "nauc_mrr_at_3_max": 0.47830764546568383, + "nauc_mrr_at_3_std": 0.07908104385148582, + "nauc_mrr_at_5_diff1": 0.6651150515375697, + "nauc_mrr_at_5_max": 0.48320424406293105, + "nauc_mrr_at_5_std": 0.0841376160310789, + "nauc_ndcg_at_1000_diff1": 0.6644414321594331, + "nauc_ndcg_at_1000_max": 0.48494330141081976, + "nauc_ndcg_at_1000_std": 0.08499549812045258, + "nauc_ndcg_at_100_diff1": 0.6643084643885205, + "nauc_ndcg_at_100_max": 0.4859086696461096, + "nauc_ndcg_at_100_std": 0.08683324562972554, + "nauc_ndcg_at_10_diff1": 0.6558132357644398, + "nauc_ndcg_at_10_max": 0.4912776808457643, + "nauc_ndcg_at_10_std": 0.09056604431255127, + "nauc_ndcg_at_1_diff1": 0.6964868193650136, + "nauc_ndcg_at_1_max": 0.4658840817263219, + "nauc_ndcg_at_1_std": 0.06009948824657468, + "nauc_ndcg_at_20_diff1": 0.6611087197877977, + "nauc_ndcg_at_20_max": 0.48954890256829625, + "nauc_ndcg_at_20_std": 0.08876502997875993, + "nauc_ndcg_at_3_diff1": 0.6501669512812934, + "nauc_ndcg_at_3_max": 0.4827750854554256, + "nauc_ndcg_at_3_std": 0.08585341799650138, + "nauc_ndcg_at_5_diff1": 0.6534210015801749, + "nauc_ndcg_at_5_max": 0.49293204109100786, + "nauc_ndcg_at_5_std": 0.09712640857173467, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.6346185533036952, + "nauc_precision_at_100_max": 0.6848052946668934, + "nauc_precision_at_100_std": 0.46640028560445, + "nauc_precision_at_10_diff1": 0.5758345691217664, + "nauc_precision_at_10_max": 0.5616378316032302, + "nauc_precision_at_10_std": 0.1567045641786131, + "nauc_precision_at_1_diff1": 0.6964868193650136, + "nauc_precision_at_1_max": 0.4658840817263219, + "nauc_precision_at_1_std": 0.06009948824657468, + "nauc_precision_at_20_diff1": 0.59426049622128, + "nauc_precision_at_20_max": 0.5904757914561855, + "nauc_precision_at_20_std": 0.18555826889160457, + "nauc_precision_at_3_diff1": 0.6016933324728304, + "nauc_precision_at_3_max": 0.49967144441250244, + "nauc_precision_at_3_std": 0.1114439040226548, + "nauc_precision_at_5_diff1": 0.5963268545815941, + "nauc_precision_at_5_max": 0.5435354775022362, + "nauc_precision_at_5_std": 0.16471325854088018, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.6346185533036849, + "nauc_recall_at_100_max": 0.6848052946668853, + "nauc_recall_at_100_std": 0.4664002856044359, + "nauc_recall_at_10_diff1": 0.5758345691217669, + "nauc_recall_at_10_max": 0.5616378316032299, + "nauc_recall_at_10_std": 0.15670456417861245, + "nauc_recall_at_1_diff1": 0.6964868193650136, + "nauc_recall_at_1_max": 0.4658840817263219, + "nauc_recall_at_1_std": 0.06009948824657468, + "nauc_recall_at_20_diff1": 0.5942604962212811, + "nauc_recall_at_20_max": 0.5904757914561845, + "nauc_recall_at_20_std": 0.18555826889159976, + "nauc_recall_at_3_diff1": 0.6016933324728311, + "nauc_recall_at_3_max": 0.4996714444125018, + "nauc_recall_at_3_std": 0.11144390402265514, + "nauc_recall_at_5_diff1": 0.596326854581594, + "nauc_recall_at_5_max": 0.5435354775022344, + "nauc_recall_at_5_std": 0.1647132585408802, + "ndcg_at_1": 0.6214, + "ndcg_at_10": 0.76748, + "ndcg_at_100": 0.78475, + "ndcg_at_1000": 0.78565, + "ndcg_at_20": 0.77717, + "ndcg_at_3": 0.7233, + "ndcg_at_5": 0.74763, + "precision_at_1": 0.6214, + "precision_at_10": 0.0915, + "precision_at_100": 0.00993, + "precision_at_1000": 0.001, + "precision_at_20": 0.04766, + "precision_at_3": 0.2648, + "precision_at_5": 0.1708, + "recall_at_1": 0.6214, + "recall_at_10": 0.915, + "recall_at_100": 0.9932, + "recall_at_1000": 1.0, + "recall_at_20": 0.9532, + "recall_at_3": 0.7944, + "recall_at_5": 0.854 + } + ] + }, + "task_name": "Flickr30kT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json new file mode 100644 index 0000000000..c27e7fd151 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb", + "evaluation_time": 11.527727365493774, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.782, + "cv_recall_at_10": 0.979, + "cv_recall_at_100": 0.998, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.994, + "cv_recall_at_3": 0.919, + "cv_recall_at_5": 0.951, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.71535, + "map_at_1": 0.1564, + "map_at_10": 0.6016, + "map_at_100": 0.64142, + "map_at_1000": 0.64252, + "map_at_20": 0.62777, + "map_at_3": 0.38783, + "map_at_5": 0.52369, + "mrr_at_1": 0.782, + "mrr_at_10": 0.8533876984126986, + "mrr_at_100": 0.8545914967755422, + "mrr_at_1000": 0.8546010863340449, + "mrr_at_20": 0.8544995892995896, + "mrr_at_3": 0.842166666666667, + "mrr_at_5": 0.849416666666667, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6790382819794335, + "nauc_cv_recall_at_100_max": 0.6136788048552808, + "nauc_cv_recall_at_100_std": 0.5401493930905519, + "nauc_cv_recall_at_10_diff1": 0.5072695744964671, + "nauc_cv_recall_at_10_max": 0.6689342403628125, + "nauc_cv_recall_at_10_std": 0.3829754123871726, + "nauc_cv_recall_at_1_diff1": 0.488735446449004, + "nauc_cv_recall_at_1_max": 0.5141133834291504, + "nauc_cv_recall_at_1_std": -0.03618247455901551, + "nauc_cv_recall_at_20_diff1": 0.34002489884843273, + "nauc_cv_recall_at_20_max": 0.6156240273887368, + "nauc_cv_recall_at_20_std": 0.7568471833177612, + "nauc_cv_recall_at_3_diff1": 0.5087895240400684, + "nauc_cv_recall_at_3_max": 0.5634459545134927, + "nauc_cv_recall_at_3_std": 0.10033890099249525, + "nauc_cv_recall_at_5_diff1": 0.5434650050496416, + "nauc_cv_recall_at_5_max": 0.5627870195697343, + "nauc_cv_recall_at_5_std": 0.2778158882600648, + "nauc_map_at_1000_diff1": 0.15465269435050452, + "nauc_map_at_1000_max": 0.46711312406477384, + "nauc_map_at_1000_std": 0.048575938256388396, + "nauc_map_at_100_diff1": 0.15431140329951903, + "nauc_map_at_100_max": 0.46719979617191026, + "nauc_map_at_100_std": 0.05052363203080754, + "nauc_map_at_10_diff1": 0.1529787812866786, + "nauc_map_at_10_max": 0.45711233128671735, + "nauc_map_at_10_std": 0.02449973700621325, + "nauc_map_at_1_diff1": 0.4887354464490032, + "nauc_map_at_1_max": 0.5141133834291495, + "nauc_map_at_1_std": -0.03618247455901751, + "nauc_map_at_20_diff1": 0.15076154146537132, + "nauc_map_at_20_max": 0.46444349461535234, + "nauc_map_at_20_std": 0.04479449662456778, + "nauc_map_at_3_diff1": 0.21351903584890045, + "nauc_map_at_3_max": 0.45698690208877696, + "nauc_map_at_3_std": -0.010887987588539814, + "nauc_map_at_5_diff1": 0.14578035683987364, + "nauc_map_at_5_max": 0.4515880427226945, + "nauc_map_at_5_std": -0.0026874220062081747, + "nauc_mrr_at_1000_diff1": 0.4893310611891473, + "nauc_mrr_at_1000_max": 0.5261133355559909, + "nauc_mrr_at_1000_std": 0.0020315610859939267, + "nauc_mrr_at_100_diff1": 0.4893377672062905, + "nauc_mrr_at_100_max": 0.5261248596328446, + "nauc_mrr_at_100_std": 0.0020657882998518138, + "nauc_mrr_at_10_diff1": 0.48988261824797846, + "nauc_mrr_at_10_max": 0.5277240085344459, + "nauc_mrr_at_10_std": 0.00453451162925253, + "nauc_mrr_at_1_diff1": 0.488735446449004, + "nauc_mrr_at_1_max": 0.5141133834291504, + "nauc_mrr_at_1_std": -0.03618247455901551, + "nauc_mrr_at_20_diff1": 0.48916248000131224, + "nauc_mrr_at_20_max": 0.5262057614175848, + "nauc_mrr_at_20_std": 0.0026394141831601174, + "nauc_mrr_at_3_diff1": 0.48985167550576125, + "nauc_mrr_at_3_max": 0.5300441674581229, + "nauc_mrr_at_3_std": 0.0027730733995076284, + "nauc_mrr_at_5_diff1": 0.4918762189399869, + "nauc_mrr_at_5_max": 0.52702946776199, + "nauc_mrr_at_5_std": 0.011169701853700587, + "nauc_ndcg_at_1000_diff1": 0.22766089836536801, + "nauc_ndcg_at_1000_max": 0.492189505568212, + "nauc_ndcg_at_1000_std": 0.06504915436760614, + "nauc_ndcg_at_100_diff1": 0.2199335774609376, + "nauc_ndcg_at_100_max": 0.4953948669765448, + "nauc_ndcg_at_100_std": 0.0980744945605265, + "nauc_ndcg_at_10_diff1": 0.2044319581978452, + "nauc_ndcg_at_10_max": 0.4763899911130591, + "nauc_ndcg_at_10_std": 0.04503381614955524, + "nauc_ndcg_at_1_diff1": 0.488735446449004, + "nauc_ndcg_at_1_max": 0.5141133834291504, + "nauc_ndcg_at_1_std": -0.03618247455901551, + "nauc_ndcg_at_20_diff1": 0.200495407478228, + "nauc_ndcg_at_20_max": 0.4889831247980535, + "nauc_ndcg_at_20_std": 0.0867421312607528, + "nauc_ndcg_at_3_diff1": 0.25423026936814114, + "nauc_ndcg_at_3_max": 0.4728682169678516, + "nauc_ndcg_at_3_std": 0.001298277045082591, + "nauc_ndcg_at_5_diff1": 0.19418768049527063, + "nauc_ndcg_at_5_max": 0.46699616414823136, + "nauc_ndcg_at_5_std": 0.01093649505873361, + "nauc_precision_at_1000_diff1": 0.12759103641456715, + "nauc_precision_at_1000_max": 0.4844070961717251, + "nauc_precision_at_1000_std": 0.7185340802988052, + "nauc_precision_at_100_diff1": 0.08522333406731375, + "nauc_precision_at_100_max": 0.48179115797715183, + "nauc_precision_at_100_std": 0.5722441057258686, + "nauc_precision_at_10_diff1": 0.13469873984556588, + "nauc_precision_at_10_max": 0.4428990975706562, + "nauc_precision_at_10_std": 0.08769758273258854, + "nauc_precision_at_1_diff1": 0.488735446449004, + "nauc_precision_at_1_max": 0.5141133834291504, + "nauc_precision_at_1_std": -0.03618247455901551, + "nauc_precision_at_20_diff1": 0.09427425323646658, + "nauc_precision_at_20_max": 0.4578914983843951, + "nauc_precision_at_20_std": 0.2235341685316957, + "nauc_precision_at_3_diff1": 0.2010575428533153, + "nauc_precision_at_3_max": 0.46236331299632105, + "nauc_precision_at_3_std": 0.013547375486496331, + "nauc_precision_at_5_diff1": 0.1317349745546741, + "nauc_precision_at_5_max": 0.4500305482173576, + "nauc_precision_at_5_std": 0.022958692069647763, + "nauc_recall_at_1000_diff1": 0.12759103641454075, + "nauc_recall_at_1000_max": 0.48440709617180283, + "nauc_recall_at_1000_std": 0.7185340802988699, + "nauc_recall_at_100_diff1": 0.0852233340673131, + "nauc_recall_at_100_max": 0.48179115797715144, + "nauc_recall_at_100_std": 0.5722441057258676, + "nauc_recall_at_10_diff1": 0.13469873984556588, + "nauc_recall_at_10_max": 0.4428990975706562, + "nauc_recall_at_10_std": 0.08769758273258854, + "nauc_recall_at_1_diff1": 0.4887354464490032, + "nauc_recall_at_1_max": 0.5141133834291495, + "nauc_recall_at_1_std": -0.03618247455901751, + "nauc_recall_at_20_diff1": 0.09427425323646658, + "nauc_recall_at_20_max": 0.4578914983843951, + "nauc_recall_at_20_std": 0.2235341685316957, + "nauc_recall_at_3_diff1": 0.20105754285331554, + "nauc_recall_at_3_max": 0.4623633129963219, + "nauc_recall_at_3_std": 0.013547375486496333, + "nauc_recall_at_5_diff1": 0.1317349745546741, + "nauc_recall_at_5_max": 0.4500305482173576, + "nauc_recall_at_5_std": 0.022958692069647763, + "ndcg_at_1": 0.782, + "ndcg_at_10": 0.71535, + "ndcg_at_100": 0.79531, + "ndcg_at_1000": 0.80538, + "ndcg_at_20": 0.75694, + "ndcg_at_3": 0.70872, + "ndcg_at_5": 0.63009, + "precision_at_1": 0.782, + "precision_at_10": 0.3677, + "precision_at_100": 0.04762, + "precision_at_1000": 0.00499, + "precision_at_20": 0.2081, + "precision_at_3": 0.68767, + "precision_at_5": 0.5814, + "recall_at_1": 0.1564, + "recall_at_10": 0.7354, + "recall_at_100": 0.9524, + "recall_at_1000": 0.998, + "recall_at_20": 0.8324, + "recall_at_3": 0.4126, + "recall_at_5": 0.5814 + } + ] + }, + "task_name": "Flickr30kI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json new file mode 100644 index 0000000000..8ab9b00d39 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "e819702b287bfbe084e129a61f308a802b7c108e", + "evaluation_time": 19.96243405342102, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.582, + "cv_recall_at_10": 0.892, + "cv_recall_at_100": 0.99, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.94, + "cv_recall_at_3": 0.7656, + "cv_recall_at_5": 0.8288, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.73691, + "map_at_1": 0.582, + "map_at_10": 0.6872, + "map_at_100": 0.69194, + "map_at_1000": 0.692, + "map_at_20": 0.69052, + "map_at_3": 0.664, + "map_at_5": 0.6786, + "mrr_at_1": 0.582, + "mrr_at_10": 0.6872000793650814, + "mrr_at_100": 0.6919418796590151, + "mrr_at_1000": 0.6920043726572235, + "mrr_at_20": 0.6905210107419258, + "mrr_at_3": 0.6640000000000026, + "mrr_at_5": 0.6786000000000014, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5202707749766565, + "nauc_cv_recall_at_100_max": 0.5908309990662858, + "nauc_cv_recall_at_100_std": 0.2939682539682632, + "nauc_cv_recall_at_10_diff1": 0.5617933093550299, + "nauc_cv_recall_at_10_max": 0.5589071830149802, + "nauc_cv_recall_at_10_std": 0.2244342982854361, + "nauc_cv_recall_at_1_diff1": 0.696381939657304, + "nauc_cv_recall_at_1_max": 0.47579049630166126, + "nauc_cv_recall_at_1_std": 0.0856787171723342, + "nauc_cv_recall_at_20_diff1": 0.5454154995331454, + "nauc_cv_recall_at_20_max": 0.575666044195457, + "nauc_cv_recall_at_20_std": 0.27880018674136237, + "nauc_cv_recall_at_3_diff1": 0.6071203712031832, + "nauc_cv_recall_at_3_max": 0.5034522530911553, + "nauc_cv_recall_at_3_std": 0.1294339186732062, + "nauc_cv_recall_at_5_diff1": 0.5786903531517902, + "nauc_cv_recall_at_5_max": 0.5299681868096371, + "nauc_cv_recall_at_5_std": 0.18341665359380663, + "nauc_map_at_1000_diff1": 0.6637636015118767, + "nauc_map_at_1000_max": 0.48813186871534664, + "nauc_map_at_1000_std": 0.10620231745962286, + "nauc_map_at_100_diff1": 0.663725501715657, + "nauc_map_at_100_max": 0.48815298386872974, + "nauc_map_at_100_std": 0.10623045894817788, + "nauc_map_at_10_diff1": 0.6621084063953249, + "nauc_map_at_10_max": 0.48930142276760086, + "nauc_map_at_10_std": 0.10756855111381708, + "nauc_map_at_1_diff1": 0.696381939657304, + "nauc_map_at_1_max": 0.47579049630166126, + "nauc_map_at_1_std": 0.0856787171723342, + "nauc_map_at_20_diff1": 0.6630269532116994, + "nauc_map_at_20_max": 0.48862647994909303, + "nauc_map_at_20_std": 0.10717682940880707, + "nauc_map_at_3_diff1": 0.662292666771372, + "nauc_map_at_3_max": 0.48547341513990666, + "nauc_map_at_3_std": 0.10081169237632849, + "nauc_map_at_5_diff1": 0.6602863231828925, + "nauc_map_at_5_max": 0.4883925967971498, + "nauc_map_at_5_std": 0.10706113722330816, + "nauc_mrr_at_1000_diff1": 0.6637636015118767, + "nauc_mrr_at_1000_max": 0.48813186871534664, + "nauc_mrr_at_1000_std": 0.10620231745962286, + "nauc_mrr_at_100_diff1": 0.663725501715657, + "nauc_mrr_at_100_max": 0.48815298386872974, + "nauc_mrr_at_100_std": 0.10623045894817788, + "nauc_mrr_at_10_diff1": 0.6621084063953249, + "nauc_mrr_at_10_max": 0.48930142276760086, + "nauc_mrr_at_10_std": 0.10756855111381708, + "nauc_mrr_at_1_diff1": 0.696381939657304, + "nauc_mrr_at_1_max": 0.47579049630166126, + "nauc_mrr_at_1_std": 0.0856787171723342, + "nauc_mrr_at_20_diff1": 0.6630269532116994, + "nauc_mrr_at_20_max": 0.48862647994909303, + "nauc_mrr_at_20_std": 0.10717682940880707, + "nauc_mrr_at_3_diff1": 0.662292666771372, + "nauc_mrr_at_3_max": 0.48547341513990666, + "nauc_mrr_at_3_std": 0.10081169237632849, + "nauc_mrr_at_5_diff1": 0.6602863231828925, + "nauc_mrr_at_5_max": 0.4883925967971498, + "nauc_mrr_at_5_std": 0.10706113722330816, + "nauc_ndcg_at_1000_diff1": 0.6588020949056204, + "nauc_ndcg_at_1000_max": 0.4911070818710271, + "nauc_ndcg_at_1000_std": 0.11159626078786719, + "nauc_ndcg_at_100_diff1": 0.6578430110166047, + "nauc_ndcg_at_100_max": 0.49175784358270214, + "nauc_ndcg_at_100_std": 0.11271907006804692, + "nauc_ndcg_at_10_diff1": 0.6491199341823203, + "nauc_ndcg_at_10_max": 0.49780129062969325, + "nauc_ndcg_at_10_std": 0.1217823152584022, + "nauc_ndcg_at_1_diff1": 0.696381939657304, + "nauc_ndcg_at_1_max": 0.47579049630166126, + "nauc_ndcg_at_1_std": 0.0856787171723342, + "nauc_ndcg_at_20_diff1": 0.6527562952947522, + "nauc_ndcg_at_20_max": 0.4954501005931264, + "nauc_ndcg_at_20_std": 0.12020249054744403, + "nauc_ndcg_at_3_diff1": 0.6504859796432384, + "nauc_ndcg_at_3_max": 0.4892786954353797, + "nauc_ndcg_at_3_std": 0.10686134169700189, + "nauc_ndcg_at_5_diff1": 0.6458666500847972, + "nauc_ndcg_at_5_max": 0.4954671444369836, + "nauc_ndcg_at_5_std": 0.11996492669439957, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5202707749766482, + "nauc_precision_at_100_max": 0.5908309990662891, + "nauc_precision_at_100_std": 0.29396825396825177, + "nauc_precision_at_10_diff1": 0.5617933093550302, + "nauc_precision_at_10_max": 0.5589071830149824, + "nauc_precision_at_10_std": 0.22443429828543696, + "nauc_precision_at_1_diff1": 0.696381939657304, + "nauc_precision_at_1_max": 0.47579049630166126, + "nauc_precision_at_1_std": 0.0856787171723342, + "nauc_precision_at_20_diff1": 0.545415499533147, + "nauc_precision_at_20_max": 0.5756660441954571, + "nauc_precision_at_20_std": 0.27880018674136353, + "nauc_precision_at_3_diff1": 0.6071203712031833, + "nauc_precision_at_3_max": 0.5034522530911553, + "nauc_precision_at_3_std": 0.1294339186732061, + "nauc_precision_at_5_diff1": 0.5786903531517906, + "nauc_precision_at_5_max": 0.5299681868096384, + "nauc_precision_at_5_std": 0.183416653593807, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5202707749766565, + "nauc_recall_at_100_max": 0.5908309990662858, + "nauc_recall_at_100_std": 0.2939682539682632, + "nauc_recall_at_10_diff1": 0.5617933093550299, + "nauc_recall_at_10_max": 0.5589071830149802, + "nauc_recall_at_10_std": 0.2244342982854361, + "nauc_recall_at_1_diff1": 0.696381939657304, + "nauc_recall_at_1_max": 0.47579049630166126, + "nauc_recall_at_1_std": 0.0856787171723342, + "nauc_recall_at_20_diff1": 0.5454154995331454, + "nauc_recall_at_20_max": 0.575666044195457, + "nauc_recall_at_20_std": 0.27880018674136237, + "nauc_recall_at_3_diff1": 0.6071203712031832, + "nauc_recall_at_3_max": 0.5034522530911553, + "nauc_recall_at_3_std": 0.1294339186732062, + "nauc_recall_at_5_diff1": 0.5786903531517902, + "nauc_recall_at_5_max": 0.5299681868096371, + "nauc_recall_at_5_std": 0.18341665359380663, + "ndcg_at_1": 0.582, + "ndcg_at_10": 0.73691, + "ndcg_at_100": 0.75858, + "ndcg_at_1000": 0.75994, + "ndcg_at_20": 0.74904, + "ndcg_at_3": 0.69014, + "ndcg_at_5": 0.71631, + "precision_at_1": 0.582, + "precision_at_10": 0.0892, + "precision_at_100": 0.0099, + "precision_at_1000": 0.001, + "precision_at_20": 0.047, + "precision_at_3": 0.2552, + "precision_at_5": 0.16576, + "recall_at_1": 0.582, + "recall_at_10": 0.892, + "recall_at_100": 0.99, + "recall_at_1000": 1.0, + "recall_at_20": 0.94, + "recall_at_3": 0.7656, + "recall_at_5": 0.8288 + } + ] + }, + "task_name": "Flickr30kT2IRetrieval" +} \ No newline at end of file diff --git a/scripts/data/flickr30k/build_flickr_30k_i2t.py b/scripts/data/flickr30k/build_flickr_30k_i2t.py new file mode 100644 index 0000000000..2abd1f1682 --- /dev/null +++ b/scripts/data/flickr30k/build_flickr_30k_i2t.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import os + +from datasets import Dataset, DatasetDict, load_dataset +from tqdm import tqdm + +WRITE_TOK = os.environ["HF_TOKEN"] + +eval_split = "test" +data_raw = load_dataset("clip-benchmark/wds_flickr30k")[eval_split] + + +## i2t +queries_ = {"id": [], "modality": [], "image": []} +corpus_ = {"id": [], "modality": [], "text": []} +relevant_docs_ = {"query-id": [], "corpus-id": [], "score": []} + +for row in tqdm(data_raw, total=len(data_raw)): + image = row["jpg"] + texts = row["txt"].split("\n") + key = row["__key__"] + query_id = f"q_{key}" + queries_["id"].append(query_id) + queries_["image"].append(image) + queries_["modality"].append("image") + + for i, text in enumerate(texts): + doc_id = f"d_{key}_{i}" + corpus_["id"].append(doc_id) + corpus_["text"].append(text) + corpus_["modality"].append("text") + + relevant_docs_["query-id"].append(query_id) + relevant_docs_["corpus-id"].append(doc_id) + relevant_docs_["score"].append(1) + +corpus = Dataset.from_dict(corpus_) +queries = Dataset.from_dict(queries_) +relevant_docs = Dataset.from_dict(relevant_docs_) + +corpus = DatasetDict({"corpus": corpus}) +queries = DatasetDict({"test": queries}) +relevant_docs = DatasetDict({"test": relevant_docs}) + + +repo_name = "isaacchung/flickr30ki2t" +# create_repo(repo_name, repo_type="dataset", token=WRITE_TOK) + +corpus.push_to_hub(repo_name, "corpus", token=WRITE_TOK) +queries.push_to_hub(repo_name, "query", token=WRITE_TOK) +relevant_docs.push_to_hub(repo_name, "qrels", token=WRITE_TOK) diff --git a/scripts/data/flickr30k/build_flickr_30k_t2i.py b/scripts/data/flickr30k/build_flickr_30k_t2i.py new file mode 100644 index 0000000000..b000de6e4d --- /dev/null +++ b/scripts/data/flickr30k/build_flickr_30k_t2i.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import os + +from datasets import Dataset, DatasetDict, load_dataset +from huggingface_hub import create_repo +from tqdm import tqdm + +WRITE_TOK = os.environ["HF_TOKEN"] + +eval_split = "test" +data_raw = load_dataset("clip-benchmark/wds_flickr30k")[eval_split] + + +## t2i +queries_ = {"id": [], "modality": [], "text": []} +corpus_ = {"id": [], "modality": [], "image": []} +relevant_docs_ = {"query-id": [], "corpus-id": [], "score": []} + +for row in tqdm(data_raw, total=len(data_raw)): + image = row["jpg"] + texts = row["txt"].split("\n") + key = row["__key__"] + + doc_id = f"d_{key}" + corpus_["id"].append(doc_id) + corpus_["image"].append(image) + corpus_["modality"].append("image") + + for i, text in enumerate(texts): + query_id = f"q_{key}_{i}" + queries_["id"].append(query_id) + queries_["text"].append(text) + queries_["modality"].append("text") + + relevant_docs_["query-id"].append(query_id) + relevant_docs_["corpus-id"].append(doc_id) + relevant_docs_["score"].append(1) + +corpus = Dataset.from_dict(corpus_) +queries = Dataset.from_dict(queries_) +relevant_docs = Dataset.from_dict(relevant_docs_) + +corpus = DatasetDict({"corpus": corpus}) +queries = DatasetDict({"test": queries}) +relevant_docs = DatasetDict({"test": relevant_docs}) + + +repo_name = "isaacchung/flickr30kt2i" +create_repo(repo_name, repo_type="dataset", token=WRITE_TOK) + +corpus.push_to_hub(repo_name, "corpus", token=WRITE_TOK) +queries.push_to_hub(repo_name, "query", token=WRITE_TOK) +relevant_docs.push_to_hub(repo_name, "qrels", token=WRITE_TOK) From ff34ff62965bf929aef1d4ca29d4486ea7afb0be Mon Sep 17 00:00:00 2001 From: Saiteja Utpala <73220310+SaitejaUtpala@users.noreply.github.com> Date: Sat, 30 Nov 2024 22:16:24 +0530 Subject: [PATCH 111/154] [MIEB] add missing siglip models (#1533) * add udpates * lint errors --- .../AbsTaskImageMultilabelClassification.py | 2 +- mteb/models/siglip_models.py | 24 +++++++++++++++++++ scripts/run_mieb.py | 3 ++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py index d680f5d990..6635e89828 100644 --- a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py @@ -7,10 +7,10 @@ import numpy as np from sklearn.base import ClassifierMixin, clone +from sklearn.linear_model import LogisticRegression from sklearn.metrics import f1_score, label_ranking_average_precision_score from sklearn.model_selection import train_test_split from sklearn.multioutput import MultiOutputClassifier -from sklearn.linear_model import LogisticRegression from sklearn.preprocessing import MultiLabelBinarizer from mteb.abstasks.TaskMetadata import HFSubset diff --git a/mteb/models/siglip_models.py b/mteb/models/siglip_models.py index 3c33139920..d60b8bdc30 100644 --- a/mteb/models/siglip_models.py +++ b/mteb/models/siglip_models.py @@ -140,6 +140,18 @@ def get_fused_embeddings( return image_embeddings +siglip_so400m_patch14_224 = ModelMeta( + loader=partial( + SiglipModelWrapper, + model_name="google/siglip-so400m-patch14-224", + ), + name="google/siglip-so400m-patch14-224", + languages=["eng_Latn"], + open_source=True, + revision="d04cf29fca7b6374f74d8bea1969314492266b5e", + release_date="2024-01-08", +) + siglip_so400m_patch14_384 = ModelMeta( loader=partial( SiglipModelWrapper, @@ -152,6 +164,18 @@ def get_fused_embeddings( release_date="2024-01-08", ) +siglip_so400m_patch16_256_i18n = ModelMeta( + loader=partial( + SiglipModelWrapper, + model_name="google/siglip-so400m-patch16-256-i18n", + ), + name="google/siglip-so400m-patch16-256-i18n", + languages=["eng_Latn"], + open_source=True, + revision="365d321c0cfdea96bc28e3a29787a11a062681a1", + release_date="2024-01-08", +) + siglip_base_patch16_256_multilingual = ModelMeta( loader=partial( SiglipModelWrapper, diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index 676fd966a7..b62abf78f7 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -25,7 +25,8 @@ "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "nyu-visionx/moco-v3-vit-b", "nyu-visionx/moco-v3-vit-l", - "google/siglip-so400m-patch14-384", + "google/siglip-so400m-patch14-224" "google/siglip-so400m-patch14-384", + "google/siglip-so400m-patch16-256-i18n" "google/siglip-base-patch16-256-multilingual", "google/siglip-base-patch16-256", "google/siglip-base-patch16-512", From dc35ce3704dd120b4d6a621e66e70f7034d07d3d Mon Sep 17 00:00:00 2001 From: Saiteja Utpala <73220310+SaitejaUtpala@users.noreply.github.com> Date: Sun, 1 Dec 2024 00:58:01 +0530 Subject: [PATCH 112/154] fix typo (#1535) * add udpates * lint errors * fix small typo --- scripts/run_mieb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index b62abf78f7..b3c55b26d5 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -25,8 +25,9 @@ "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "nyu-visionx/moco-v3-vit-b", "nyu-visionx/moco-v3-vit-l", - "google/siglip-so400m-patch14-224" "google/siglip-so400m-patch14-384", - "google/siglip-so400m-patch16-256-i18n" + "google/siglip-so400m-patch14-224", + "google/siglip-so400m-patch14-384", + "google/siglip-so400m-patch16-256-i18n", "google/siglip-base-patch16-256-multilingual", "google/siglip-base-patch16-256", "google/siglip-base-patch16-512", From c77b923cb795b4a5b73fdd5e642309a81e168336 Mon Sep 17 00:00:00 2001 From: Xin Zhang Date: Wed, 4 Dec 2024 16:10:11 +0800 Subject: [PATCH 113/154] [mieb] Fix numbers of CIRR, Fashion200k, FashionIQ, Flickr30k, MSCOCO data statistics (#1544) fix numbers --- .../Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py | 6 +++--- .../Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py | 6 +++--- .../Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py | 6 +++--- .../Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py | 10 +++++++++- .../Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py | 10 +++++++++- .../Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py | 6 +++--- .../Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py | 6 +++--- 7 files changed, 33 insertions(+), 17 deletions(-) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py index 81e9a13328..78291c0f37 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py @@ -35,13 +35,13 @@ class CIRRIT2IRetrieval(AbsTaskAny2AnyRetrieval): year={2021} }""", descriptive_stats={ - "n_samples": {"test": 1172}, + "n_samples": {"test": 4170}, "avg_character_length": { "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 9350, - "num_queries": 1172, + "num_documents": 21551, + "num_queries": 4170, "average_relevant_docs_per_query": 1.0, } }, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py index 0acb328f14..cb67b9ad0b 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py @@ -34,13 +34,13 @@ class Fashion200kI2TRetrieval(AbsTaskAny2AnyRetrieval): year={2017} }""", descriptive_stats={ - "n_samples": {"test": 4890}, + "n_samples": {"test": 4889}, "avg_character_length": { "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 61700, - "num_queries": 4890, + "num_documents": 61707, + "num_queries": 4889, "average_relevant_docs_per_query": 1.0, } }, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py index fd78a663cf..d6099d1e0c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py @@ -35,13 +35,13 @@ class FashionIQIT2IRetrieval(AbsTaskAny2AnyRetrieval): year={2021} }""", descriptive_stats={ - "n_samples": {"test": 6000}, + "n_samples": {"test": 6003}, "avg_character_length": { "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 74400, - "num_queries": 6000, + "num_documents": 74381, + "num_queries": 6003, "average_relevant_docs_per_query": 1.0, } }, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py index eb4a4d692c..9818f4de54 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py @@ -36,6 +36,14 @@ class Flickr30kI2TRetrieval(AbsTaskAny2AnyRetrieval): url={https://api.semanticscholar.org/CorpusID:3104920} }""", descriptive_stats={ - "n_samples": {"default": 155070}, # qrels + "n_samples": {"test": 1000}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 5000, + "num_queries": 1000, + } + }, }, ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py index b9775528c8..6e889f9c54 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py @@ -36,6 +36,14 @@ class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval): url={https://api.semanticscholar.org/CorpusID:3104920} }""", descriptive_stats={ - "n_samples": {"default": 31014}, # qrels + "n_samples": {"test": 5000}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 1000, + "num_queries": 5000, + } + }, }, ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py index 09cfcdec1f..2e84d22ee7 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py @@ -36,13 +36,13 @@ class MSCOCOI2TRetrieval(AbsTaskAny2AnyRetrieval): organization={Springer} }""", descriptive_stats={ - "n_samples": {"test": 1172}, + "n_samples": {"test": 5000}, "avg_character_length": { "test": { "average_document_length": 30.94235294117647, "average_query_length": 131.56569965870307, - "num_documents": 9350, - "num_queries": 1172, + "num_documents": 24809, + "num_queries": 5000, "average_relevant_docs_per_query": 1.0, } }, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py index f47575b9b4..1ad8aa7a04 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py @@ -36,13 +36,13 @@ class MSCOCOT2IRetrieval(AbsTaskAny2AnyRetrieval): organization={Springer} }""", descriptive_stats={ - "n_samples": {"test": 1172}, + "n_samples": {"test": 24809}, "avg_character_length": { "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 9350, - "num_queries": 1172, + "num_documents": 5000, + "num_queries": 24809, "average_relevant_docs_per_query": 1.0, } }, From db5315fae3feb873fbffcb20dbad68acbc05317d Mon Sep 17 00:00:00 2001 From: Kenneth Enevoldsen Date: Wed, 4 Dec 2024 21:12:37 +0100 Subject: [PATCH 114/154] Discussing a standard for ImageEncoders --- mteb/encoder_interface.py | 60 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/mteb/encoder_interface.py b/mteb/encoder_interface.py index c8a45f711d..7c69503ca2 100644 --- a/mteb/encoder_interface.py +++ b/mteb/encoder_interface.py @@ -7,6 +7,9 @@ import numpy as np import torch +from PIL import Image +from torch.utils.data import DataLoader + Corpus = Union[list[dict[str, str]], dict[str, list[str]]] @@ -23,13 +26,15 @@ class Encoder(Protocol): In general the interface is kept aligned with sentence-transformers interface. In cases where exceptions occurs these are handled within MTEB. """ + device: str | None + def __init__(self, device: str | None = None) -> None: """The initialization function for the encoder. Used when calling it from the mteb run CLI. Args: device: The device to use for encoding. Can be ignored if the encoder is not using a device (e.g. for API) """ - self.device = device + pass def encode( self, @@ -151,3 +156,56 @@ def convert_conv_history_to_query(conversations: Sequence[Sequence[str]]) -> str The query. """ ... + + +class ImageEncoder: + """Interface for image encoder designed based on VLM2VecWrapper. + There is not a perfect 1-1 match, e.g. device can be None here. + The intention here is to define the current interface and adapt to as close to MTEB as possible + and align as much as possible with sentencetransformers. + """ + + def __init__( + self, + device: str | None, + **kwargs: Any, + ): + pass + + def encode( # current a 1-1 match with Encoder.encode + self, + sentences: Sequence[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + pass + + def get_image_embeddings( # Seems like sentence transformers use a singular encode for both images and text. Not sure if we want to do the same. + # If not it might be ideal to redefine Encoder.encode + self, + images: list[Image.Image] | DataLoader, + *, + **kwargs, + # removed batch_size, it is not required that it will accept kwargs + ) -> np.ndarray: # added standard output (I believe we actually expect tensors in the code, but would like to be consistent) + pass + + def get_text_embeddings( # any reason for this? + self, + texts: list[str], + *, + **kwargs, + ) -> np.ndarray: + pass + + def get_fused_embeddings( # hmm what if I have a document with images at specific positions? + self, + texts: list[str] | None = None, + images: list[Image.Image] | DataLoader | None = None, # the requirement for these two to be the same seems odd (docs without images, images without associated text, docs with multiple images) + # fusion_mode: str="sum", # will remove this as it should be required in the interface + *, + **kwargs: Any, + ) -> np.ndarray: + pass From d45fbb26ef98f4a31ba98d16cf96fa4f556ac741 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Fri, 6 Dec 2024 02:59:12 +0800 Subject: [PATCH 115/154] Add Voyage's multimodal embedding (#1555) * add voyage multimodal & ran 17 tasks * lint * typo * clean --- mteb/models/overview.py | 2 + mteb/models/voyage_v.py | 179 +++ .../1/CIFAR100Clustering.json | 22 + .../1/CIFAR10Clustering.json | 22 + .../voyage-multimodal-3/1/CVBenchCount.json | 19 + .../voyage-multimodal-3/1/CVBenchDepth.json | 19 + .../1/CVBenchDistance.json | 19 + .../1/CVBenchRelation.json | 19 + .../1/ImageNet10Clustering.json | 22 + .../1/ImageNetDog15Clustering.json | 22 + .../voyage-multimodal-3/1/STS12VisualSTS.json | 26 + .../voyage-multimodal-3/1/STS13VisualSTS.json | 26 + .../voyage-multimodal-3/1/STS14VisualSTS.json | 26 + .../voyage-multimodal-3/1/STS15VisualSTS.json | 26 + .../voyage-multimodal-3/1/STS16VisualSTS.json | 26 + .../1/STS17MultilingualVisualSTS.json | 183 +++ .../1/STSBenchmarkMultilingualVisualSTS.json | 313 ++++ .../1/TinyImageNetClustering.json | 22 + .../1/XFlickr30kCoT2IRetrieval.json | 1411 +++++++++++++++++ .../voyage-multimodal-3/1/model_meta.json | 1 + 20 files changed, 2405 insertions(+) create mode 100644 mteb/models/voyage_v.py create mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR100Clustering.json create mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR10Clustering.json create mode 100644 results-mieb/voyage-multimodal-3/1/CVBenchCount.json create mode 100644 results-mieb/voyage-multimodal-3/1/CVBenchDepth.json create mode 100644 results-mieb/voyage-multimodal-3/1/CVBenchDistance.json create mode 100644 results-mieb/voyage-multimodal-3/1/CVBenchRelation.json create mode 100644 results-mieb/voyage-multimodal-3/1/ImageNet10Clustering.json create mode 100644 results-mieb/voyage-multimodal-3/1/ImageNetDog15Clustering.json create mode 100644 results-mieb/voyage-multimodal-3/1/STS12VisualSTS.json create mode 100644 results-mieb/voyage-multimodal-3/1/STS13VisualSTS.json create mode 100644 results-mieb/voyage-multimodal-3/1/STS14VisualSTS.json create mode 100644 results-mieb/voyage-multimodal-3/1/STS15VisualSTS.json create mode 100644 results-mieb/voyage-multimodal-3/1/STS16VisualSTS.json create mode 100644 results-mieb/voyage-multimodal-3/1/STS17MultilingualVisualSTS.json create mode 100644 results-mieb/voyage-multimodal-3/1/STSBenchmarkMultilingualVisualSTS.json create mode 100644 results-mieb/voyage-multimodal-3/1/TinyImageNetClustering.json create mode 100644 results-mieb/voyage-multimodal-3/1/XFlickr30kCoT2IRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/model_meta.json diff --git a/mteb/models/overview.py b/mteb/models/overview.py index e68a4f65e4..340989baaf 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -44,6 +44,7 @@ vista_models, vlm2vec_models, voyage_models, + voyage_v, ) logger = logging.getLogger(__name__) @@ -81,6 +82,7 @@ siglip_models, vista_models, voyage_models, + voyage_v, vlm2vec_models, repllama_models, promptriever_models, diff --git a/mteb/models/voyage_v.py b/mteb/models/voyage_v.py new file mode 100644 index 0000000000..c1787ea10f --- /dev/null +++ b/mteb/models/voyage_v.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import os +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from torchvision import transforms +from tqdm import tqdm + +import mteb +from mteb.model_meta import ModelMeta + +api_key = os.getenv("VOYAGE_API_KEY") +tensor_to_image = transforms.Compose([transforms.ToPILImage()]) + + +def voyage_v_loader(**kwargs): + try: + import voyageai + except ImportError: + raise ImportError("To use voyage models, please run `pip install -U voyageai`.") + + class VoyageMultiModalModelWrapper: + def __init__( + self, + model_name: str, + **kwargs: Any, + ): + self.model_name = model_name + self.vo = voyageai.Client() + + def get_text_embeddings( + self, texts: list[str], batch_size: int = 32, input_type=None + ): + all_text_embeddings = [] + + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + batch_texts = [[text] for text in batch_texts] + all_text_embeddings += torch.tensor( + self.vo.multimodal_embed( + batch_texts, model=self.model_name, input_type=input_type + ).embeddings + ) + all_text_embeddings = torch.vstack(all_text_embeddings) + return all_text_embeddings + + def get_image_embeddings( + self, + images: list[Image.Image] | DataLoader, + batch_size: int = 32, + input_type=None, + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + for index, batch in enumerate(tqdm(images)): + if index == 0: + assert len(batch) == batch_size + batch_images = [[tensor_to_image(image)] for image in batch] + all_image_embeddings += torch.tensor( + self.vo.multimodal_embed( + batch_images, model=self.model_name, input_type=input_type + ).embeddings + ) + else: + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + batch_images = [[image] for image in batch_images] + all_image_embeddings += torch.tensor( + self.vo.multimodal_embed( + batch_images, model=self.model_name, input_type=input_type + ).embeddings + ) + all_image_embeddings = torch.vstack(all_image_embeddings) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm( + dim=-1, keepdim=True + ) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + batch_size: int = 32, + input_type=None, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + interleaved_embeddings = [] + if texts is not None and images is not None: + # print("encoding interleaved inputs") + if isinstance(images, DataLoader): + for index, batch in tqdm(enumerate(images)): + if index == 0: + assert len(batch) == batch_size + batch_images = [tensor_to_image(image) for image in batch] + batch_texts = texts[ + index * batch_size : (index + 1) * batch_size + ] + interleaved_inputs = [ + [text, image] + for image, text in zip(batch_images, batch_texts) + ] + interleaved_embeddings += torch.tensor( + self.vo.multimodal_embed( + interleaved_inputs, + model=self.model_name, + input_type=input_type, + ).embeddings + ) + else: + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + batch_texts = texts[i : i + batch_size] + interleaved_inputs = [ + [text, image] + for image, text in zip(batch_images, batch_texts) + ] + interleaved_embeddings += torch.tensor( + self.vo.multimodal_embed( + interleaved_inputs, + model=self.model_name, + input_type=input_type, + ).embeddings + ) + interleaved_embeddings = torch.vstack(interleaved_embeddings) + return interleaved_embeddings + + elif texts is not None: + # print("encoding texts only") + text_embeddings = self.get_text_embeddings(texts, batch_size) + + elif images is not None: + # print("encoding images only") + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + return VoyageMultiModalModelWrapper(**kwargs) + + +cohere_mult_3 = ModelMeta( + loader=partial(voyage_v_loader, model_name="voyage-multimodal-3"), + name="voyage-multimodal-3", + languages=[], # Unknown + open_source=False, + revision="1", + release_date="2024-11-10", + n_parameters=None, + memory_usage=None, + max_tokens=None, + embed_dim=1024, + license=None, + similarity_fn_name="cosine", + framework=[], +) + +if __name__ == "__main__": + mdl = mteb.get_model(cohere_mult_3.name, cohere_mult_3.revision) + emb = mdl.encode(["Hello, world!"]) diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR100Clustering.json b/results-mieb/voyage-multimodal-3/1/CIFAR100Clustering.json new file mode 100644 index 0000000000..221465417f --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CIFAR100Clustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + "evaluation_time": 231.98164081573486, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "ari": 0.46531392836213886, + "cluster_accuracy": 0.6042, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7515067668506068, + "nmi": 0.7515067668506068, + "v_measure": 0.7515067668506068 + } + ] + }, + "task_name": "CIFAR100Clustering" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR10Clustering.json b/results-mieb/voyage-multimodal-3/1/CIFAR10Clustering.json new file mode 100644 index 0000000000..5459074f45 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CIFAR10Clustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + "evaluation_time": 236.98327565193176, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "ari": 0.7986159784876974, + "cluster_accuracy": 0.8244, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8622309717386565, + "nmi": 0.8622309717386565, + "v_measure": 0.8622309717386566 + } + ] + }, + "task_name": "CIFAR10Clustering" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchCount.json b/results-mieb/voyage-multimodal-3/1/CVBenchCount.json new file mode 100644 index 0000000000..ff443295ee --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CVBenchCount.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", + "evaluation_time": 444.4619266986847, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.2639593908629442, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.2639593908629442 + } + ] + }, + "task_name": "CVBenchCount" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchDepth.json b/results-mieb/voyage-multimodal-3/1/CVBenchDepth.json new file mode 100644 index 0000000000..185e34679a --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CVBenchDepth.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", + "evaluation_time": 413.3635849952698, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5316666666666666, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5316666666666666 + } + ] + }, + "task_name": "CVBenchDepth" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchDistance.json b/results-mieb/voyage-multimodal-3/1/CVBenchDistance.json new file mode 100644 index 0000000000..a950524030 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CVBenchDistance.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", + "evaluation_time": 402.95056080818176, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.475, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.475 + } + ] + }, + "task_name": "CVBenchDistance" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchRelation.json b/results-mieb/voyage-multimodal-3/1/CVBenchRelation.json new file mode 100644 index 0000000000..9a75e5c540 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CVBenchRelation.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", + "evaluation_time": 201.59757685661316, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5353846153846153, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5353846153846153 + } + ] + }, + "task_name": "CVBenchRelation" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/ImageNet10Clustering.json b/results-mieb/voyage-multimodal-3/1/ImageNet10Clustering.json new file mode 100644 index 0000000000..750953aea8 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/ImageNet10Clustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "88f8a6d47c257895094c5ad81e67ba751771fc99", + "evaluation_time": 914.053968667984, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "ari": 0.9808613040028806, + "cluster_accuracy": 0.9913076923076923, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9758070118084365, + "nmi": 0.9758070118084365, + "v_measure": 0.9758070118084365 + } + ] + }, + "task_name": "ImageNet10Clustering" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/ImageNetDog15Clustering.json b/results-mieb/voyage-multimodal-3/1/ImageNetDog15Clustering.json new file mode 100644 index 0000000000..96f915915f --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/ImageNetDog15Clustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "bfb6ad3b2109d26c9daddf14f98d315daa35ee72", + "evaluation_time": 78.63793706893921, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "ari": 0.8119121646758497, + "cluster_accuracy": 0.820631970260223, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8381839891894807, + "nmi": 0.8381839891894807, + "v_measure": 0.8381839891894807 + } + ] + }, + "task_name": "ImageNetDog15Clustering" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS12VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS12VisualSTS.json new file mode 100644 index 0000000000..0d5231319d --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS12VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "820c25edfba736f3789201b2476208cc62c2ccb9", + "evaluation_time": 381.2039792537689, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.7530408956307458, + "cosine_spearman": 0.7161988759864429, + "euclidean_pearson": 0.7141309522927283, + "euclidean_spearman": 0.7162724788414607, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7161988759864429, + "manhattan_pearson": 0.7140835255725306, + "manhattan_spearman": 0.7154340341352178, + "pearson": 0.7530408956307458, + "spearman": 0.7161988759864429 + } + ] + }, + "task_name": "STS12VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS13VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS13VisualSTS.json new file mode 100644 index 0000000000..f422f81ad5 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS13VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "561ee9ca47ff3e4a657283c59416deca8dc169f2", + "evaluation_time": 178.80085134506226, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.8041898098977802, + "cosine_spearman": 0.8160271504089486, + "euclidean_pearson": 0.8138453622908126, + "euclidean_spearman": 0.8160367132733977, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8160271504089486, + "manhattan_pearson": 0.8141249912528841, + "manhattan_spearman": 0.816315080890506, + "pearson": 0.8041898098977802, + "spearman": 0.8160271504089486 + } + ] + }, + "task_name": "STS13VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS14VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS14VisualSTS.json new file mode 100644 index 0000000000..54abe0a859 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS14VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "824e95e45471024a684b901e0645579ffd9ca288", + "evaluation_time": 477.0949454307556, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.8126428329540769, + "cosine_spearman": 0.7798398073449294, + "euclidean_pearson": 0.8019773377625052, + "euclidean_spearman": 0.779773229647281, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7798398073449294, + "manhattan_pearson": 0.8019854029161337, + "manhattan_spearman": 0.7797635347309039, + "pearson": 0.8126428329540769, + "spearman": 0.7798398073449294 + } + ] + }, + "task_name": "STS14VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS15VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS15VisualSTS.json new file mode 100644 index 0000000000..d2a32dd340 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS15VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "1f8d08d9b9daac7118dfdefeb94b0aac4baf2e5f", + "evaluation_time": 385.04992389678955, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.8596747384607906, + "cosine_spearman": 0.8684963727234637, + "euclidean_pearson": 0.8648294270010821, + "euclidean_spearman": 0.8684429467437428, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8684963727234637, + "manhattan_pearson": 0.8645923546880342, + "manhattan_spearman": 0.8682092035635413, + "pearson": 0.8596747384607906, + "spearman": 0.8684963727234637 + } + ] + }, + "task_name": "STS15VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS16VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS16VisualSTS.json new file mode 100644 index 0000000000..653d939404 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS16VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fc354f19598af93f32c0af1b94046ffdeaacde15", + "evaluation_time": 150.6804370880127, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.8169368709648801, + "cosine_spearman": 0.8261532729608647, + "euclidean_pearson": 0.8216160911211441, + "euclidean_spearman": 0.8260077161971415, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8261532729608647, + "manhattan_pearson": 0.8212353007123708, + "manhattan_spearman": 0.8258613312454957, + "pearson": 0.8169368709648801, + "spearman": 0.8261532729608647 + } + ] + }, + "task_name": "STS16VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS17MultilingualVisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS17MultilingualVisualSTS.json new file mode 100644 index 0000000000..8bde9d8217 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS17MultilingualVisualSTS.json @@ -0,0 +1,183 @@ +{ + "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", + "evaluation_time": 693.5000240802765, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.6295478701037069, + "cosine_spearman": 0.6279910477771019, + "euclidean_pearson": 0.6276223489798609, + "euclidean_spearman": 0.6280662784030709, + "hf_subset": "ko-ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.6279910477771019, + "manhattan_pearson": 0.6272252575072031, + "manhattan_spearman": 0.6275841741159163, + "pearson": 0.6295478701037069, + "spearman": 0.6279910477771019 + }, + { + "cosine_pearson": 0.6640588532698395, + "cosine_spearman": 0.6575414606953666, + "euclidean_pearson": 0.6455970584249845, + "euclidean_spearman": 0.6568954839487309, + "hf_subset": "ar-ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.6575414606953666, + "manhattan_pearson": 0.6456300855443649, + "manhattan_spearman": 0.6581063058387047, + "pearson": 0.6640588532698395, + "spearman": 0.6575414606953666 + }, + { + "cosine_pearson": 0.3785792513239742, + "cosine_spearman": 0.3440485632827937, + "euclidean_pearson": 0.3818330414668889, + "euclidean_spearman": 0.3425467167114543, + "hf_subset": "en-ar", + "languages": [ + "eng-Latn", + "ara-Arab" + ], + "main_score": 0.3440485632827937, + "manhattan_pearson": 0.37280091876138727, + "manhattan_spearman": 0.33497021404416516, + "pearson": 0.3785792513239742, + "spearman": 0.3440485632827937 + }, + { + "cosine_pearson": 0.797388779176213, + "cosine_spearman": 0.8041814174327988, + "euclidean_pearson": 0.8018372869118563, + "euclidean_spearman": 0.8046465372595982, + "hf_subset": "en-de", + "languages": [ + "eng-Latn", + "deu-Latn" + ], + "main_score": 0.8041814174327988, + "manhattan_pearson": 0.8036367691934629, + "manhattan_spearman": 0.806153756136359, + "pearson": 0.797388779176213, + "spearman": 0.8041814174327988 + }, + { + "cosine_pearson": 0.889324389401427, + "cosine_spearman": 0.8968348245174512, + "euclidean_pearson": 0.8988351349236602, + "euclidean_spearman": 0.8966841410694303, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8968348245174512, + "manhattan_pearson": 0.8987261929400799, + "manhattan_spearman": 0.8980541303774577, + "pearson": 0.889324389401427, + "spearman": 0.8968348245174512 + }, + { + "cosine_pearson": 0.46493619455982876, + "cosine_spearman": 0.4498243279387666, + "euclidean_pearson": 0.46435399844913355, + "euclidean_spearman": 0.451091543774284, + "hf_subset": "en-tr", + "languages": [ + "eng-Latn", + "tur-Latn" + ], + "main_score": 0.4498243279387666, + "manhattan_pearson": 0.4687785223213941, + "manhattan_spearman": 0.4568005510381185, + "pearson": 0.46493619455982876, + "spearman": 0.4498243279387666 + }, + { + "cosine_pearson": 0.7413496252477598, + "cosine_spearman": 0.7471866607172193, + "euclidean_pearson": 0.7483417196429633, + "euclidean_spearman": 0.7466169330700719, + "hf_subset": "es-en", + "languages": [ + "spa-Latn", + "eng-Latn" + ], + "main_score": 0.7471866607172193, + "manhattan_pearson": 0.7497169457615618, + "manhattan_spearman": 0.749720065032133, + "pearson": 0.7413496252477598, + "spearman": 0.7471866607172193 + }, + { + "cosine_pearson": 0.8401635646321107, + "cosine_spearman": 0.8370332502012957, + "euclidean_pearson": 0.850259304444977, + "euclidean_spearman": 0.8371101547419473, + "hf_subset": "es-es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.8370332502012957, + "manhattan_pearson": 0.8511002525095128, + "manhattan_spearman": 0.8384332973638572, + "pearson": 0.8401635646321107, + "spearman": 0.8370332502012957 + }, + { + "cosine_pearson": 0.7447131067366383, + "cosine_spearman": 0.750669573557799, + "euclidean_pearson": 0.7452718819569142, + "euclidean_spearman": 0.7508660001953976, + "hf_subset": "fr-en", + "languages": [ + "fra-Latn", + "eng-Latn" + ], + "main_score": 0.750669573557799, + "manhattan_pearson": 0.7480003355590945, + "manhattan_spearman": 0.7524451012272094, + "pearson": 0.7447131067366383, + "spearman": 0.750669573557799 + }, + { + "cosine_pearson": 0.7685526370938169, + "cosine_spearman": 0.7776349916156341, + "euclidean_pearson": 0.7692583708827287, + "euclidean_spearman": 0.7780436051494257, + "hf_subset": "it-en", + "languages": [ + "ita-Latn", + "eng-Latn" + ], + "main_score": 0.7776349916156341, + "manhattan_pearson": 0.7680758399408012, + "manhattan_spearman": 0.776599427307041, + "pearson": 0.7685526370938169, + "spearman": 0.7776349916156341 + }, + { + "cosine_pearson": 0.7848622824203596, + "cosine_spearman": 0.7824649342798778, + "euclidean_pearson": 0.7849080671012388, + "euclidean_spearman": 0.7823646067800476, + "hf_subset": "nl-en", + "languages": [ + "nld-Latn", + "eng-Latn" + ], + "main_score": 0.7824649342798778, + "manhattan_pearson": 0.7863829683396525, + "manhattan_spearman": 0.7856565788434446, + "pearson": 0.7848622824203596, + "spearman": 0.7824649342798778 + } + ] + }, + "task_name": "STS17MultilingualVisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STSBenchmarkMultilingualVisualSTS.json b/results-mieb/voyage-multimodal-3/1/STSBenchmarkMultilingualVisualSTS.json new file mode 100644 index 0000000000..2a7ee75282 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STSBenchmarkMultilingualVisualSTS.json @@ -0,0 +1,313 @@ +{ + "dataset_revision": "9f1ab21f17f497974996ab74b3ff911165a7dbf9", + "evaluation_time": 5075.43874335289, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "dev": [ + { + "cosine_pearson": 0.8563893474191902, + "cosine_spearman": 0.858058860838852, + "euclidean_pearson": 0.8602263464877766, + "euclidean_spearman": 0.8579783803562652, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.858058860838852, + "manhattan_pearson": 0.8598337346826477, + "manhattan_spearman": 0.8575681612119586, + "pearson": 0.8563893474191902, + "spearman": 0.858058860838852 + }, + { + "cosine_pearson": 0.789854359195163, + "cosine_spearman": 0.7899033365671463, + "euclidean_pearson": 0.7916778007557717, + "euclidean_spearman": 0.7898486229980857, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.7899033365671463, + "manhattan_pearson": 0.7910013245869385, + "manhattan_spearman": 0.7894274945306929, + "pearson": 0.789854359195163, + "spearman": 0.7899033365671463 + }, + { + "cosine_pearson": 0.796760464557315, + "cosine_spearman": 0.8013993811687563, + "euclidean_pearson": 0.8032555282023144, + "euclidean_spearman": 0.8015704115360485, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.8013993811687563, + "manhattan_pearson": 0.8020719661864791, + "manhattan_spearman": 0.8003944783391376, + "pearson": 0.796760464557315, + "spearman": 0.8013993811687563 + }, + { + "cosine_pearson": 0.7985932474785588, + "cosine_spearman": 0.8021648661142508, + "euclidean_pearson": 0.8031042634558174, + "euclidean_spearman": 0.8021820102550952, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ], + "main_score": 0.8021648661142508, + "manhattan_pearson": 0.8029724285823261, + "manhattan_spearman": 0.8019900652727078, + "pearson": 0.7985932474785588, + "spearman": 0.8021648661142508 + }, + { + "cosine_pearson": 0.7869557821233382, + "cosine_spearman": 0.7890330736912589, + "euclidean_pearson": 0.7906461233218647, + "euclidean_spearman": 0.7890663388388003, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ], + "main_score": 0.7890330736912589, + "manhattan_pearson": 0.7903840939372646, + "manhattan_spearman": 0.7888588814000735, + "pearson": 0.7869557821233382, + "spearman": 0.7890330736912589 + }, + { + "cosine_pearson": 0.7791772215249965, + "cosine_spearman": 0.781520955767216, + "euclidean_pearson": 0.7801465349415607, + "euclidean_spearman": 0.7815041664138321, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ], + "main_score": 0.781520955767216, + "manhattan_pearson": 0.7789332520700933, + "manhattan_spearman": 0.7801130952911972, + "pearson": 0.7791772215249965, + "spearman": 0.781520955767216 + }, + { + "cosine_pearson": 0.7448849902931678, + "cosine_spearman": 0.7448043158631151, + "euclidean_pearson": 0.7405744069410145, + "euclidean_spearman": 0.7447120862522622, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ], + "main_score": 0.7448043158631151, + "manhattan_pearson": 0.7406202109132378, + "manhattan_spearman": 0.7445824918429544, + "pearson": 0.7448849902931678, + "spearman": 0.7448043158631151 + }, + { + "cosine_pearson": 0.779680602143197, + "cosine_spearman": 0.7826858098266277, + "euclidean_pearson": 0.7852151538285848, + "euclidean_spearman": 0.7826847689006386, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ], + "main_score": 0.7826858098266277, + "manhattan_pearson": 0.7839441476775062, + "manhattan_spearman": 0.7813235864617706, + "pearson": 0.779680602143197, + "spearman": 0.7826858098266277 + }, + { + "cosine_pearson": 0.7884272004913433, + "cosine_spearman": 0.7892230784770122, + "euclidean_pearson": 0.7805909078732953, + "euclidean_spearman": 0.7893512299219531, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7892230784770122, + "manhattan_pearson": 0.7794459614806362, + "manhattan_spearman": 0.7878490798979422, + "pearson": 0.7884272004913433, + "spearman": 0.7892230784770122 + }, + { + "cosine_pearson": 0.7654432441204759, + "cosine_spearman": 0.7647755317149096, + "euclidean_pearson": 0.7593503194362672, + "euclidean_spearman": 0.7647270138720049, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ], + "main_score": 0.7647755317149096, + "manhattan_pearson": 0.7589361597204412, + "manhattan_spearman": 0.764541990552287, + "pearson": 0.7654432441204759, + "spearman": 0.7647755317149096 + } + ], + "test": [ + { + "cosine_pearson": 0.823492051147191, + "cosine_spearman": 0.8255498065512306, + "euclidean_pearson": 0.826698523592022, + "euclidean_spearman": 0.8254643124686717, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8255498065512306, + "manhattan_pearson": 0.8263307385343645, + "manhattan_spearman": 0.8247261005744897, + "pearson": 0.823492051147191, + "spearman": 0.8255498065512306 + }, + { + "cosine_pearson": 0.7492177185806835, + "cosine_spearman": 0.7413351070642012, + "euclidean_pearson": 0.7439940554472478, + "euclidean_spearman": 0.7414010247347829, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.7413351070642012, + "manhattan_pearson": 0.7436240717647824, + "manhattan_spearman": 0.7409998672149616, + "pearson": 0.7492177185806835, + "spearman": 0.7413351070642012 + }, + { + "cosine_pearson": 0.7607906213948373, + "cosine_spearman": 0.7599250153963639, + "euclidean_pearson": 0.7564171074765642, + "euclidean_spearman": 0.7598909226716342, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.7599250153963639, + "manhattan_pearson": 0.7558868227763365, + "manhattan_spearman": 0.7597894599923053, + "pearson": 0.7607906213948373, + "spearman": 0.7599250153963639 + }, + { + "cosine_pearson": 0.7520573823123218, + "cosine_spearman": 0.7442880594186052, + "euclidean_pearson": 0.7420168403959864, + "euclidean_spearman": 0.7442472350562676, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ], + "main_score": 0.7442880594186052, + "manhattan_pearson": 0.7415021156781515, + "manhattan_spearman": 0.7434403372096718, + "pearson": 0.7520573823123218, + "spearman": 0.7442880594186052 + }, + { + "cosine_pearson": 0.7467690218563255, + "cosine_spearman": 0.7395905416647525, + "euclidean_pearson": 0.7357731015973681, + "euclidean_spearman": 0.7394434799461104, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ], + "main_score": 0.7395905416647525, + "manhattan_pearson": 0.7350784821793559, + "manhattan_spearman": 0.7386311414238852, + "pearson": 0.7467690218563255, + "spearman": 0.7395905416647525 + }, + { + "cosine_pearson": 0.7202011364978524, + "cosine_spearman": 0.7134491712781116, + "euclidean_pearson": 0.7118458785635648, + "euclidean_spearman": 0.7134129390083591, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ], + "main_score": 0.7134491712781116, + "manhattan_pearson": 0.7109658343515404, + "manhattan_spearman": 0.7126032622007037, + "pearson": 0.7202011364978524, + "spearman": 0.7134491712781116 + }, + { + "cosine_pearson": 0.6985464798474408, + "cosine_spearman": 0.6883224080503167, + "euclidean_pearson": 0.6874216559416312, + "euclidean_spearman": 0.6881582413464393, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ], + "main_score": 0.6883224080503167, + "manhattan_pearson": 0.6871835930951957, + "manhattan_spearman": 0.6879319205502966, + "pearson": 0.6985464798474408, + "spearman": 0.6883224080503167 + }, + { + "cosine_pearson": 0.7362293011231412, + "cosine_spearman": 0.7348219968439652, + "euclidean_pearson": 0.7298573693202373, + "euclidean_spearman": 0.7346749511887691, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ], + "main_score": 0.7348219968439652, + "manhattan_pearson": 0.7295003107001472, + "manhattan_spearman": 0.73423890877604, + "pearson": 0.7362293011231412, + "spearman": 0.7348219968439652 + }, + { + "cosine_pearson": 0.7368742675667894, + "cosine_spearman": 0.7267810144277265, + "euclidean_pearson": 0.7227644385927088, + "euclidean_spearman": 0.726788301200454, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7267810144277265, + "manhattan_pearson": 0.721898680516144, + "manhattan_spearman": 0.7256356052253394, + "pearson": 0.7368742675667894, + "spearman": 0.7267810144277265 + }, + { + "cosine_pearson": 0.7345816344328195, + "cosine_spearman": 0.7259656655754602, + "euclidean_pearson": 0.7169100386731838, + "euclidean_spearman": 0.7260215086810657, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ], + "main_score": 0.7259656655754602, + "manhattan_pearson": 0.7164561544415515, + "manhattan_spearman": 0.7256700966168421, + "pearson": 0.7345816344328195, + "spearman": 0.7259656655754602 + } + ] + }, + "task_name": "STSBenchmarkMultilingualVisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/TinyImageNetClustering.json b/results-mieb/voyage-multimodal-3/1/TinyImageNetClustering.json new file mode 100644 index 0000000000..0aa996e4ba --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/TinyImageNetClustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "5a77092c28e51558c5586e9c5eb71a7e17a5e43f", + "evaluation_time": 368.8909206390381, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "valid": [ + { + "ari": 0.31776897199787596, + "cluster_accuracy": 0.4675, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6929420627528728, + "nmi": 0.6929420627528728, + "v_measure": 0.6929420627528728 + } + ] + }, + "task_name": "TinyImageNetClustering" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/XFlickr30kCoT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/XFlickr30kCoT2IRetrieval.json new file mode 100644 index 0000000000..8088fb6cce --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/XFlickr30kCoT2IRetrieval.json @@ -0,0 +1,1411 @@ +{ + "dataset_revision": "0af2c2eba58b27a71898787e286be04befdd7a20", + "evaluation_time": 3659.7448897361755, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.6755, + "cv_recall_at_10": 0.929, + "cv_recall_at_100": 0.9865, + "cv_recall_at_1000": 0.9985, + "cv_recall_at_20": 0.9585, + "cv_recall_at_3": 0.8335, + "cv_recall_at_5": 0.887, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.80482, + "map_at_1": 0.6755, + "map_at_10": 0.76463, + "map_at_100": 0.7676, + "map_at_1000": 0.76767, + "map_at_20": 0.76683, + "map_at_3": 0.7465, + "map_at_5": 0.75875, + "mrr_at_1": 0.6755, + "mrr_at_10": 0.764634126984127, + "mrr_at_100": 0.7675982824740856, + "mrr_at_1000": 0.767672981003628, + "mrr_at_20": 0.7668271411380156, + "mrr_at_3": 0.7464999999999994, + "mrr_at_5": 0.7587499999999999, + "nauc_cv_recall_at_1000_diff1": 0.8513849984438709, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.7822561123214679, + "nauc_cv_recall_at_100_max": 0.9683577134557472, + "nauc_cv_recall_at_100_std": 0.8572120206107111, + "nauc_cv_recall_at_10_diff1": 0.6544594363567012, + "nauc_cv_recall_at_10_max": 0.7857274365145103, + "nauc_cv_recall_at_10_std": 0.30422403703265355, + "nauc_cv_recall_at_1_diff1": 0.7463322614632609, + "nauc_cv_recall_at_1_max": 0.5806226318361674, + "nauc_cv_recall_at_1_std": 0.10438451063582097, + "nauc_cv_recall_at_20_diff1": 0.6694002902365792, + "nauc_cv_recall_at_20_max": 0.8863802549132098, + "nauc_cv_recall_at_20_std": 0.5763220951031016, + "nauc_cv_recall_at_3_diff1": 0.6924928185360167, + "nauc_cv_recall_at_3_max": 0.6660863314034848, + "nauc_cv_recall_at_3_std": 0.15626155616629717, + "nauc_cv_recall_at_5_diff1": 0.6826023673900833, + "nauc_cv_recall_at_5_max": 0.7265547008045103, + "nauc_cv_recall_at_5_std": 0.21227234213903992, + "nauc_map_at_1000_diff1": 0.7290871254473019, + "nauc_map_at_1000_max": 0.6120752280056199, + "nauc_map_at_1000_std": 0.1339926740717831, + "nauc_map_at_100_diff1": 0.729116189000848, + "nauc_map_at_100_max": 0.6122070042437942, + "nauc_map_at_100_std": 0.13424625550910035, + "nauc_map_at_10_diff1": 0.7275587072696231, + "nauc_map_at_10_max": 0.6131427813295653, + "nauc_map_at_10_std": 0.13247460462832822, + "nauc_map_at_1_diff1": 0.7463322614632609, + "nauc_map_at_1_max": 0.5806226318361674, + "nauc_map_at_1_std": 0.10438451063582097, + "nauc_map_at_20_diff1": 0.7287129282021074, + "nauc_map_at_20_max": 0.6129948863589249, + "nauc_map_at_20_std": 0.13516965016715282, + "nauc_map_at_3_diff1": 0.7276166067316586, + "nauc_map_at_3_max": 0.6078307595298069, + "nauc_map_at_3_std": 0.12362603641270917, + "nauc_map_at_5_diff1": 0.7281564633744259, + "nauc_map_at_5_max": 0.6130927975568831, + "nauc_map_at_5_std": 0.12986554015100024, + "nauc_mrr_at_1000_diff1": 0.7290871254473019, + "nauc_mrr_at_1000_max": 0.6120752280056199, + "nauc_mrr_at_1000_std": 0.1339926740717831, + "nauc_mrr_at_100_diff1": 0.729116189000848, + "nauc_mrr_at_100_max": 0.6122070042437942, + "nauc_mrr_at_100_std": 0.13424625550910035, + "nauc_mrr_at_10_diff1": 0.7275587072696231, + "nauc_mrr_at_10_max": 0.6131427813295653, + "nauc_mrr_at_10_std": 0.13247460462832822, + "nauc_mrr_at_1_diff1": 0.7463322614632609, + "nauc_mrr_at_1_max": 0.5806226318361674, + "nauc_mrr_at_1_std": 0.10438451063582097, + "nauc_mrr_at_20_diff1": 0.7287129282021074, + "nauc_mrr_at_20_max": 0.6129948863589249, + "nauc_mrr_at_20_std": 0.13516965016715282, + "nauc_mrr_at_3_diff1": 0.7276166067316586, + "nauc_mrr_at_3_max": 0.6078307595298069, + "nauc_mrr_at_3_std": 0.12362603641270917, + "nauc_mrr_at_5_diff1": 0.7281564633744259, + "nauc_mrr_at_5_max": 0.6130927975568831, + "nauc_mrr_at_5_std": 0.12986554015100024, + "nauc_ndcg_at_1000_diff1": 0.7267113760326085, + "nauc_ndcg_at_1000_max": 0.6205977818229345, + "nauc_ndcg_at_1000_std": 0.14558789880807216, + "nauc_ndcg_at_100_diff1": 0.727248678404592, + "nauc_ndcg_at_100_max": 0.6240844070904952, + "nauc_ndcg_at_100_std": 0.15255848317430523, + "nauc_ndcg_at_10_diff1": 0.7195813943287325, + "nauc_ndcg_at_10_max": 0.6312960344952803, + "nauc_ndcg_at_10_std": 0.1500285742121337, + "nauc_ndcg_at_1_diff1": 0.7463322614632609, + "nauc_ndcg_at_1_max": 0.5806226318361674, + "nauc_ndcg_at_1_std": 0.10438451063582097, + "nauc_ndcg_at_20_diff1": 0.7237711267067128, + "nauc_ndcg_at_20_max": 0.6309783007730462, + "nauc_ndcg_at_20_std": 0.16121216549752765, + "nauc_ndcg_at_3_diff1": 0.7206010395502354, + "nauc_ndcg_at_3_max": 0.619345909718247, + "nauc_ndcg_at_3_std": 0.13020291795367483, + "nauc_ndcg_at_5_diff1": 0.7210205657545173, + "nauc_ndcg_at_5_max": 0.6303298538413662, + "nauc_ndcg_at_5_std": 0.14244007171848722, + "nauc_precision_at_1000_diff1": 0.8513849984438441, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.7822561123214696, + "nauc_precision_at_100_max": 0.968357713455748, + "nauc_precision_at_100_std": 0.8572120206107152, + "nauc_precision_at_10_diff1": 0.6544594363567019, + "nauc_precision_at_10_max": 0.7857274365145102, + "nauc_precision_at_10_std": 0.30422403703265183, + "nauc_precision_at_1_diff1": 0.7463322614632609, + "nauc_precision_at_1_max": 0.5806226318361674, + "nauc_precision_at_1_std": 0.10438451063582097, + "nauc_precision_at_20_diff1": 0.6694002902365723, + "nauc_precision_at_20_max": 0.8863802549132097, + "nauc_precision_at_20_std": 0.5763220951030971, + "nauc_precision_at_3_diff1": 0.6924928185360182, + "nauc_precision_at_3_max": 0.6660863314034838, + "nauc_precision_at_3_std": 0.15626155616629744, + "nauc_precision_at_5_diff1": 0.6826023673900811, + "nauc_precision_at_5_max": 0.7265547008045097, + "nauc_precision_at_5_std": 0.21227234213903803, + "nauc_recall_at_1000_diff1": 0.8513849984438709, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.7822561123214679, + "nauc_recall_at_100_max": 0.9683577134557472, + "nauc_recall_at_100_std": 0.8572120206107111, + "nauc_recall_at_10_diff1": 0.6544594363567012, + "nauc_recall_at_10_max": 0.7857274365145103, + "nauc_recall_at_10_std": 0.30422403703265355, + "nauc_recall_at_1_diff1": 0.7463322614632609, + "nauc_recall_at_1_max": 0.5806226318361674, + "nauc_recall_at_1_std": 0.10438451063582097, + "nauc_recall_at_20_diff1": 0.6694002902365792, + "nauc_recall_at_20_max": 0.8863802549132098, + "nauc_recall_at_20_std": 0.5763220951031016, + "nauc_recall_at_3_diff1": 0.6924928185360167, + "nauc_recall_at_3_max": 0.6660863314034848, + "nauc_recall_at_3_std": 0.15626155616629717, + "nauc_recall_at_5_diff1": 0.6826023673900833, + "nauc_recall_at_5_max": 0.7265547008045103, + "nauc_recall_at_5_std": 0.21227234213903992, + "ndcg_at_1": 0.6755, + "ndcg_at_10": 0.80482, + "ndcg_at_100": 0.81775, + "ndcg_at_1000": 0.81937, + "ndcg_at_20": 0.81246, + "ndcg_at_3": 0.7689, + "ndcg_at_5": 0.79096, + "precision_at_1": 0.6755, + "precision_at_10": 0.0929, + "precision_at_100": 0.00987, + "precision_at_1000": 0.001, + "precision_at_20": 0.04792, + "precision_at_3": 0.27783, + "precision_at_5": 0.1774, + "recall_at_1": 0.6755, + "recall_at_10": 0.929, + "recall_at_100": 0.9865, + "recall_at_1000": 0.9985, + "recall_at_20": 0.9585, + "recall_at_3": 0.8335, + "recall_at_5": 0.887 + }, + { + "cv_recall_at_1": 0.8125, + "cv_recall_at_10": 0.979, + "cv_recall_at_100": 0.997, + "cv_recall_at_1000": 0.9995, + "cv_recall_at_20": 0.9905, + "cv_recall_at_3": 0.9245, + "cv_recall_at_5": 0.9505, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89914, + "map_at_1": 0.8125, + "map_at_10": 0.87317, + "map_at_100": 0.87417, + "map_at_1000": 0.87417, + "map_at_20": 0.87394, + "map_at_3": 0.86325, + "map_at_5": 0.86925, + "mrr_at_1": 0.8125, + "mrr_at_10": 0.873170634920634, + "mrr_at_100": 0.874165591919412, + "mrr_at_1000": 0.8741744359007023, + "mrr_at_20": 0.8739376106628999, + "mrr_at_3": 0.8632499999999987, + "mrr_at_5": 0.8692499999999992, + "nauc_cv_recall_at_1000_diff1": 0.869281045751564, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.8175381263616397, + "nauc_cv_recall_at_100_max": 0.9782135076252607, + "nauc_cv_recall_at_100_std": 0.09025832555244799, + "nauc_cv_recall_at_10_diff1": 0.8532746431906091, + "nauc_cv_recall_at_10_max": 0.8753390244986885, + "nauc_cv_recall_at_10_std": -0.21299630963496155, + "nauc_cv_recall_at_1_diff1": 0.8630334769638712, + "nauc_cv_recall_at_1_max": 0.636707988067617, + "nauc_cv_recall_at_1_std": -0.35566721909181354, + "nauc_cv_recall_at_20_diff1": 0.8380755811096317, + "nauc_cv_recall_at_20_max": 0.9094550100742143, + "nauc_cv_recall_at_20_std": -0.16393925991450467, + "nauc_cv_recall_at_3_diff1": 0.7887318282721475, + "nauc_cv_recall_at_3_max": 0.6857210875520191, + "nauc_cv_recall_at_3_std": -0.40794021802981484, + "nauc_cv_recall_at_5_diff1": 0.8004838298955943, + "nauc_cv_recall_at_5_max": 0.7276311197879799, + "nauc_cv_recall_at_5_std": -0.4019230587858072, + "nauc_map_at_1000_diff1": 0.8475728663389887, + "nauc_map_at_1000_max": 0.6502781025937084, + "nauc_map_at_1000_std": -0.36506027547060665, + "nauc_map_at_100_diff1": 0.8475672508185738, + "nauc_map_at_100_max": 0.6503002284229283, + "nauc_map_at_100_std": -0.36505931191358815, + "nauc_map_at_10_diff1": 0.8476975751166981, + "nauc_map_at_10_max": 0.6519234592343327, + "nauc_map_at_10_std": -0.3643551754692845, + "nauc_map_at_1_diff1": 0.8630334769638712, + "nauc_map_at_1_max": 0.636707988067617, + "nauc_map_at_1_std": -0.35566721909181354, + "nauc_map_at_20_diff1": 0.8475754154618236, + "nauc_map_at_20_max": 0.6507046372651616, + "nauc_map_at_20_std": -0.36490811352349883, + "nauc_map_at_3_diff1": 0.8411035099411375, + "nauc_map_at_3_max": 0.6487267627438167, + "nauc_map_at_3_std": -0.3728350091644064, + "nauc_map_at_5_diff1": 0.8448914569338772, + "nauc_map_at_5_max": 0.6511609163961992, + "nauc_map_at_5_std": -0.36969059260196196, + "nauc_mrr_at_1000_diff1": 0.8475728663389887, + "nauc_mrr_at_1000_max": 0.6502781025937084, + "nauc_mrr_at_1000_std": -0.36506027547060665, + "nauc_mrr_at_100_diff1": 0.8475672508185738, + "nauc_mrr_at_100_max": 0.6503002284229283, + "nauc_mrr_at_100_std": -0.36505931191358815, + "nauc_mrr_at_10_diff1": 0.8476975751166981, + "nauc_mrr_at_10_max": 0.6519234592343327, + "nauc_mrr_at_10_std": -0.3643551754692845, + "nauc_mrr_at_1_diff1": 0.8630334769638712, + "nauc_mrr_at_1_max": 0.636707988067617, + "nauc_mrr_at_1_std": -0.35566721909181354, + "nauc_mrr_at_20_diff1": 0.8475754154618236, + "nauc_mrr_at_20_max": 0.6507046372651616, + "nauc_mrr_at_20_std": -0.36490811352349883, + "nauc_mrr_at_3_diff1": 0.8411035099411375, + "nauc_mrr_at_3_max": 0.6487267627438167, + "nauc_mrr_at_3_std": -0.3728350091644064, + "nauc_mrr_at_5_diff1": 0.8448914569338772, + "nauc_mrr_at_5_max": 0.6511609163961992, + "nauc_mrr_at_5_std": -0.36969059260196196, + "nauc_ndcg_at_1000_diff1": 0.8463031464137469, + "nauc_ndcg_at_1000_max": 0.6541835788530691, + "nauc_ndcg_at_1000_std": -0.3627001519386867, + "nauc_ndcg_at_100_diff1": 0.8461539015610023, + "nauc_ndcg_at_100_max": 0.6552171012623584, + "nauc_ndcg_at_100_std": -0.3619261331869923, + "nauc_ndcg_at_10_diff1": 0.8467822882186511, + "nauc_ndcg_at_10_max": 0.663866200724262, + "nauc_ndcg_at_10_std": -0.35788015610205864, + "nauc_ndcg_at_1_diff1": 0.8630334769638712, + "nauc_ndcg_at_1_max": 0.636707988067617, + "nauc_ndcg_at_1_std": -0.35566721909181354, + "nauc_ndcg_at_20_diff1": 0.8461883899116863, + "nauc_ndcg_at_20_max": 0.6582925692614663, + "nauc_ndcg_at_20_std": -0.36084713873893653, + "nauc_ndcg_at_3_diff1": 0.8322438678494426, + "nauc_ndcg_at_3_max": 0.6548607059305832, + "nauc_ndcg_at_3_std": -0.3788620814328986, + "nauc_ndcg_at_5_diff1": 0.8391437301727366, + "nauc_ndcg_at_5_max": 0.6602246180383025, + "nauc_ndcg_at_5_std": -0.3738041857189095, + "nauc_precision_at_1000_diff1": 0.8692810457516023, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.8175381263616008, + "nauc_precision_at_100_max": 0.9782135076252637, + "nauc_precision_at_100_std": 0.09025832555238163, + "nauc_precision_at_10_diff1": 0.8532746431906064, + "nauc_precision_at_10_max": 0.8753390244986764, + "nauc_precision_at_10_std": -0.21299630963496555, + "nauc_precision_at_1_diff1": 0.8630334769638712, + "nauc_precision_at_1_max": 0.636707988067617, + "nauc_precision_at_1_std": -0.35566721909181354, + "nauc_precision_at_20_diff1": 0.8380755811096442, + "nauc_precision_at_20_max": 0.9094550100741999, + "nauc_precision_at_20_std": -0.16393925991451266, + "nauc_precision_at_3_diff1": 0.7887318282721495, + "nauc_precision_at_3_max": 0.6857210875520209, + "nauc_precision_at_3_std": -0.40794021802981506, + "nauc_precision_at_5_diff1": 0.800483829895592, + "nauc_precision_at_5_max": 0.7276311197879809, + "nauc_precision_at_5_std": -0.40192305878580603, + "nauc_recall_at_1000_diff1": 0.869281045751564, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.8175381263616397, + "nauc_recall_at_100_max": 0.9782135076252607, + "nauc_recall_at_100_std": 0.09025832555244799, + "nauc_recall_at_10_diff1": 0.8532746431906091, + "nauc_recall_at_10_max": 0.8753390244986885, + "nauc_recall_at_10_std": -0.21299630963496155, + "nauc_recall_at_1_diff1": 0.8630334769638712, + "nauc_recall_at_1_max": 0.636707988067617, + "nauc_recall_at_1_std": -0.35566721909181354, + "nauc_recall_at_20_diff1": 0.8380755811096317, + "nauc_recall_at_20_max": 0.9094550100742143, + "nauc_recall_at_20_std": -0.16393925991450467, + "nauc_recall_at_3_diff1": 0.7887318282721475, + "nauc_recall_at_3_max": 0.6857210875520191, + "nauc_recall_at_3_std": -0.40794021802981484, + "nauc_recall_at_5_diff1": 0.8004838298955943, + "nauc_recall_at_5_max": 0.7276311197879799, + "nauc_recall_at_5_std": -0.4019230587858072, + "ndcg_at_1": 0.8125, + "ndcg_at_10": 0.89914, + "ndcg_at_100": 0.90333, + "ndcg_at_1000": 0.90363, + "ndcg_at_20": 0.90201, + "ndcg_at_3": 0.87904, + "ndcg_at_5": 0.8898, + "precision_at_1": 0.8125, + "precision_at_10": 0.0979, + "precision_at_100": 0.00997, + "precision_at_1000": 0.001, + "precision_at_20": 0.04952, + "precision_at_3": 0.30817, + "precision_at_5": 0.1901, + "recall_at_1": 0.8125, + "recall_at_10": 0.979, + "recall_at_100": 0.997, + "recall_at_1000": 0.9995, + "recall_at_20": 0.9905, + "recall_at_3": 0.9245, + "recall_at_5": 0.9505 + }, + { + "cv_recall_at_1": 0.738, + "cv_recall_at_10": 0.9655, + "cv_recall_at_100": 0.999, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.9835, + "cv_recall_at_3": 0.902, + "cv_recall_at_5": 0.936, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.85832, + "map_at_1": 0.738, + "map_at_10": 0.82313, + "map_at_100": 0.82485, + "map_at_1000": 0.82485, + "map_at_20": 0.82436, + "map_at_3": 0.81117, + "map_at_5": 0.81909, + "mrr_at_1": 0.738, + "mrr_at_10": 0.8231287698412685, + "mrr_at_100": 0.8248457880053297, + "mrr_at_1000": 0.8248512406319713, + "mrr_at_20": 0.8243573382285052, + "mrr_at_3": 0.8111666666666654, + "mrr_at_5": 0.8190916666666657, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.8611111111111184, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 0.8692810457516946, + "nauc_cv_recall_at_10_diff1": 0.7032774462441969, + "nauc_cv_recall_at_10_max": 0.847054763934563, + "nauc_cv_recall_at_10_std": 0.21517882515325232, + "nauc_cv_recall_at_1_diff1": 0.8228027015155729, + "nauc_cv_recall_at_1_max": 0.657963506973408, + "nauc_cv_recall_at_1_std": 0.07059039237257138, + "nauc_cv_recall_at_20_diff1": 0.6693687576040491, + "nauc_cv_recall_at_20_max": 0.820360467419293, + "nauc_cv_recall_at_20_std": 0.33644568938686364, + "nauc_cv_recall_at_3_diff1": 0.7436331675527346, + "nauc_cv_recall_at_3_max": 0.7652084643381168, + "nauc_cv_recall_at_3_std": 0.13750738390594427, + "nauc_cv_recall_at_5_diff1": 0.7467903828197938, + "nauc_cv_recall_at_5_max": 0.7998694269374397, + "nauc_cv_recall_at_5_std": 0.16819852941176452, + "nauc_map_at_1000_diff1": 0.8055201985370324, + "nauc_map_at_1000_max": 0.6868428387023021, + "nauc_map_at_1000_std": 0.09455811602935846, + "nauc_map_at_100_diff1": 0.8055226699057262, + "nauc_map_at_100_max": 0.686853584905029, + "nauc_map_at_100_std": 0.09458470117793472, + "nauc_map_at_10_diff1": 0.804476460367567, + "nauc_map_at_10_max": 0.6886359840382733, + "nauc_map_at_10_std": 0.09505039704659991, + "nauc_map_at_1_diff1": 0.8228027015155729, + "nauc_map_at_1_max": 0.657963506973408, + "nauc_map_at_1_std": 0.07059039237257138, + "nauc_map_at_20_diff1": 0.8050762144661993, + "nauc_map_at_20_max": 0.687217723852823, + "nauc_map_at_20_std": 0.09495729170023198, + "nauc_map_at_3_diff1": 0.8013479649548041, + "nauc_map_at_3_max": 0.6900708586722066, + "nauc_map_at_3_std": 0.09348015452204224, + "nauc_map_at_5_diff1": 0.8044246497031612, + "nauc_map_at_5_max": 0.689862580590495, + "nauc_map_at_5_std": 0.09487116744284162, + "nauc_mrr_at_1000_diff1": 0.8055201985370324, + "nauc_mrr_at_1000_max": 0.6868428387023021, + "nauc_mrr_at_1000_std": 0.09455811602935846, + "nauc_mrr_at_100_diff1": 0.8055226699057262, + "nauc_mrr_at_100_max": 0.686853584905029, + "nauc_mrr_at_100_std": 0.09458470117793472, + "nauc_mrr_at_10_diff1": 0.804476460367567, + "nauc_mrr_at_10_max": 0.6886359840382733, + "nauc_mrr_at_10_std": 0.09505039704659991, + "nauc_mrr_at_1_diff1": 0.8228027015155729, + "nauc_mrr_at_1_max": 0.657963506973408, + "nauc_mrr_at_1_std": 0.07059039237257138, + "nauc_mrr_at_20_diff1": 0.8050762144661993, + "nauc_mrr_at_20_max": 0.687217723852823, + "nauc_mrr_at_20_std": 0.09495729170023198, + "nauc_mrr_at_3_diff1": 0.8013479649548041, + "nauc_mrr_at_3_max": 0.6900708586722066, + "nauc_mrr_at_3_std": 0.09348015452204224, + "nauc_mrr_at_5_diff1": 0.8044246497031612, + "nauc_mrr_at_5_max": 0.689862580590495, + "nauc_mrr_at_5_std": 0.09487116744284162, + "nauc_ndcg_at_1000_diff1": 0.8029525661469813, + "nauc_ndcg_at_1000_max": 0.6908828889360142, + "nauc_ndcg_at_1000_std": 0.09881132089648269, + "nauc_ndcg_at_100_diff1": 0.8030192391414134, + "nauc_ndcg_at_100_max": 0.6912190957005266, + "nauc_ndcg_at_100_std": 0.09964931117290805, + "nauc_ndcg_at_10_diff1": 0.7971382738540258, + "nauc_ndcg_at_10_max": 0.7003088811485032, + "nauc_ndcg_at_10_std": 0.10401499946098114, + "nauc_ndcg_at_1_diff1": 0.8228027015155729, + "nauc_ndcg_at_1_max": 0.657963506973408, + "nauc_ndcg_at_1_std": 0.07059039237257138, + "nauc_ndcg_at_20_diff1": 0.7995066079500553, + "nauc_ndcg_at_20_max": 0.6940947622326022, + "nauc_ndcg_at_20_std": 0.10398558209170257, + "nauc_ndcg_at_3_diff1": 0.7917995978691086, + "nauc_ndcg_at_3_max": 0.7026512611495505, + "nauc_ndcg_at_3_std": 0.10099772846271363, + "nauc_ndcg_at_5_diff1": 0.7973058952477493, + "nauc_ndcg_at_5_max": 0.7032016467149081, + "nauc_ndcg_at_5_std": 0.10391207396070531, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.8611111111109864, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.8692810457515646, + "nauc_precision_at_10_diff1": 0.703277446244195, + "nauc_precision_at_10_max": 0.8470547639345555, + "nauc_precision_at_10_std": 0.21517882515324474, + "nauc_precision_at_1_diff1": 0.8228027015155729, + "nauc_precision_at_1_max": 0.657963506973408, + "nauc_precision_at_1_std": 0.07059039237257138, + "nauc_precision_at_20_diff1": 0.6693687576040411, + "nauc_precision_at_20_max": 0.82036046741928, + "nauc_precision_at_20_std": 0.33644568938685376, + "nauc_precision_at_3_diff1": 0.7436331675527355, + "nauc_precision_at_3_max": 0.7652084643381156, + "nauc_precision_at_3_std": 0.13750738390594283, + "nauc_precision_at_5_diff1": 0.7467903828197912, + "nauc_precision_at_5_max": 0.7998694269374391, + "nauc_precision_at_5_std": 0.1681985294117573, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.8611111111111184, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.8692810457516946, + "nauc_recall_at_10_diff1": 0.7032774462441969, + "nauc_recall_at_10_max": 0.847054763934563, + "nauc_recall_at_10_std": 0.21517882515325232, + "nauc_recall_at_1_diff1": 0.8228027015155729, + "nauc_recall_at_1_max": 0.657963506973408, + "nauc_recall_at_1_std": 0.07059039237257138, + "nauc_recall_at_20_diff1": 0.6693687576040491, + "nauc_recall_at_20_max": 0.820360467419293, + "nauc_recall_at_20_std": 0.33644568938686364, + "nauc_recall_at_3_diff1": 0.7436331675527346, + "nauc_recall_at_3_max": 0.7652084643381168, + "nauc_recall_at_3_std": 0.13750738390594427, + "nauc_recall_at_5_diff1": 0.7467903828197938, + "nauc_recall_at_5_max": 0.7998694269374397, + "nauc_recall_at_5_std": 0.16819852941176452, + "ndcg_at_1": 0.738, + "ndcg_at_10": 0.85832, + "ndcg_at_100": 0.86589, + "ndcg_at_1000": 0.86603, + "ndcg_at_20": 0.86284, + "ndcg_at_3": 0.83453, + "ndcg_at_5": 0.84867, + "precision_at_1": 0.738, + "precision_at_10": 0.09655, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "precision_at_20": 0.04917, + "precision_at_3": 0.30067, + "precision_at_5": 0.1872, + "recall_at_1": 0.738, + "recall_at_10": 0.9655, + "recall_at_100": 0.999, + "recall_at_1000": 1.0, + "recall_at_20": 0.9835, + "recall_at_3": 0.902, + "recall_at_5": 0.936 + }, + { + "cv_recall_at_1": 0.57, + "cv_recall_at_10": 0.89, + "cv_recall_at_100": 0.9885, + "cv_recall_at_1000": 0.998, + "cv_recall_at_20": 0.936, + "cv_recall_at_3": 0.756, + "cv_recall_at_5": 0.8245, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.72935, + "map_at_1": 0.57, + "map_at_10": 0.67785, + "map_at_100": 0.68246, + "map_at_1000": 0.68253, + "map_at_20": 0.68098, + "map_at_3": 0.65292, + "map_at_5": 0.66864, + "mrr_at_1": 0.57, + "mrr_at_10": 0.6778513888888882, + "mrr_at_100": 0.682463976758287, + "mrr_at_1000": 0.6825288641722557, + "mrr_at_20": 0.6809768672568393, + "mrr_at_3": 0.6529166666666663, + "mrr_at_5": 0.6686416666666668, + "nauc_cv_recall_at_1000_diff1": 0.057773109243720255, + "nauc_cv_recall_at_1000_max": 0.07119514472454563, + "nauc_cv_recall_at_1000_std": 0.3162931839401999, + "nauc_cv_recall_at_100_diff1": 0.5459343157552812, + "nauc_cv_recall_at_100_max": 0.6831283237932791, + "nauc_cv_recall_at_100_std": 0.4349449924897459, + "nauc_cv_recall_at_10_diff1": 0.506069066231423, + "nauc_cv_recall_at_10_max": 0.6634932565930765, + "nauc_cv_recall_at_10_std": 0.18059230306674634, + "nauc_cv_recall_at_1_diff1": 0.7044065096654757, + "nauc_cv_recall_at_1_max": 0.5750721143496796, + "nauc_cv_recall_at_1_std": 0.046992006429299085, + "nauc_cv_recall_at_20_diff1": 0.5504201680672272, + "nauc_cv_recall_at_20_max": 0.7070786647992531, + "nauc_cv_recall_at_20_std": 0.2757972980858999, + "nauc_cv_recall_at_3_diff1": 0.5681348565963951, + "nauc_cv_recall_at_3_max": 0.6170784343861266, + "nauc_cv_recall_at_3_std": 0.09851473313011817, + "nauc_cv_recall_at_5_diff1": 0.5156420439651506, + "nauc_cv_recall_at_5_max": 0.6434716064276265, + "nauc_cv_recall_at_5_std": 0.146063346120565, + "nauc_map_at_1000_diff1": 0.6561740158631391, + "nauc_map_at_1000_max": 0.5903652066345317, + "nauc_map_at_1000_std": 0.06752440959391125, + "nauc_map_at_100_diff1": 0.6561738489049953, + "nauc_map_at_100_max": 0.5904283966716123, + "nauc_map_at_100_std": 0.0676291858616124, + "nauc_map_at_10_diff1": 0.65333134434392, + "nauc_map_at_10_max": 0.5913969782544157, + "nauc_map_at_10_std": 0.06836479826993805, + "nauc_map_at_1_diff1": 0.7044065096654757, + "nauc_map_at_1_max": 0.5750721143496796, + "nauc_map_at_1_std": 0.046992006429299085, + "nauc_map_at_20_diff1": 0.6556880678927408, + "nauc_map_at_20_max": 0.5909962160263641, + "nauc_map_at_20_std": 0.06838331967377713, + "nauc_map_at_3_diff1": 0.653394169001542, + "nauc_map_at_3_max": 0.5888483426159291, + "nauc_map_at_3_std": 0.06312086141745396, + "nauc_map_at_5_diff1": 0.6494305220897459, + "nauc_map_at_5_max": 0.5912390189784108, + "nauc_map_at_5_std": 0.06860456750834616, + "nauc_mrr_at_1000_diff1": 0.6561740158631391, + "nauc_mrr_at_1000_max": 0.5903652066345317, + "nauc_mrr_at_1000_std": 0.06752440959391125, + "nauc_mrr_at_100_diff1": 0.6561738489049953, + "nauc_mrr_at_100_max": 0.5904283966716123, + "nauc_mrr_at_100_std": 0.0676291858616124, + "nauc_mrr_at_10_diff1": 0.65333134434392, + "nauc_mrr_at_10_max": 0.5913969782544157, + "nauc_mrr_at_10_std": 0.06836479826993805, + "nauc_mrr_at_1_diff1": 0.7044065096654757, + "nauc_mrr_at_1_max": 0.5750721143496796, + "nauc_mrr_at_1_std": 0.046992006429299085, + "nauc_mrr_at_20_diff1": 0.6556880678927408, + "nauc_mrr_at_20_max": 0.5909962160263641, + "nauc_mrr_at_20_std": 0.06838331967377713, + "nauc_mrr_at_3_diff1": 0.653394169001542, + "nauc_mrr_at_3_max": 0.5888483426159291, + "nauc_mrr_at_3_std": 0.06312086141745396, + "nauc_mrr_at_5_diff1": 0.6494305220897459, + "nauc_mrr_at_5_max": 0.5912390189784108, + "nauc_mrr_at_5_std": 0.06860456750834616, + "nauc_ndcg_at_1000_diff1": 0.6483006530425621, + "nauc_ndcg_at_1000_max": 0.5933424088703956, + "nauc_ndcg_at_1000_std": 0.07405067389762883, + "nauc_ndcg_at_100_diff1": 0.6483197398489603, + "nauc_ndcg_at_100_max": 0.5947897242846063, + "nauc_ndcg_at_100_std": 0.07655442331368115, + "nauc_ndcg_at_10_diff1": 0.6340715926512107, + "nauc_ndcg_at_10_max": 0.600450440109288, + "nauc_ndcg_at_10_std": 0.08208340679131955, + "nauc_ndcg_at_1_diff1": 0.7044065096654757, + "nauc_ndcg_at_1_max": 0.5750721143496796, + "nauc_ndcg_at_1_std": 0.046992006429299085, + "nauc_ndcg_at_20_diff1": 0.6440067539607353, + "nauc_ndcg_at_20_max": 0.5999888493818042, + "nauc_ndcg_at_20_std": 0.083672066995679, + "nauc_ndcg_at_3_diff1": 0.6350601872400334, + "nauc_ndcg_at_3_max": 0.5948277431038169, + "nauc_ndcg_at_3_std": 0.07058416162338209, + "nauc_ndcg_at_5_diff1": 0.6259232001622383, + "nauc_ndcg_at_5_max": 0.6001942413051717, + "nauc_ndcg_at_5_std": 0.08181067673621384, + "nauc_precision_at_1000_diff1": 0.05777310924364636, + "nauc_precision_at_1000_max": 0.07119514472449866, + "nauc_precision_at_1000_std": 0.31629318394019806, + "nauc_precision_at_100_diff1": 0.5459343157552841, + "nauc_precision_at_100_max": 0.6831283237932635, + "nauc_precision_at_100_std": 0.4349449924897548, + "nauc_precision_at_10_diff1": 0.5060690662314247, + "nauc_precision_at_10_max": 0.6634932565930743, + "nauc_precision_at_10_std": 0.18059230306674617, + "nauc_precision_at_1_diff1": 0.7044065096654757, + "nauc_precision_at_1_max": 0.5750721143496796, + "nauc_precision_at_1_std": 0.046992006429299085, + "nauc_precision_at_20_diff1": 0.5504201680672255, + "nauc_precision_at_20_max": 0.7070786647992503, + "nauc_precision_at_20_std": 0.2757972980858991, + "nauc_precision_at_3_diff1": 0.5681348565963955, + "nauc_precision_at_3_max": 0.617078434386128, + "nauc_precision_at_3_std": 0.09851473313011903, + "nauc_precision_at_5_diff1": 0.5156420439651481, + "nauc_precision_at_5_max": 0.6434716064276249, + "nauc_precision_at_5_std": 0.14606334612056285, + "nauc_recall_at_1000_diff1": 0.057773109243720255, + "nauc_recall_at_1000_max": 0.07119514472454563, + "nauc_recall_at_1000_std": 0.3162931839401999, + "nauc_recall_at_100_diff1": 0.5459343157552812, + "nauc_recall_at_100_max": 0.6831283237932791, + "nauc_recall_at_100_std": 0.4349449924897459, + "nauc_recall_at_10_diff1": 0.506069066231423, + "nauc_recall_at_10_max": 0.6634932565930765, + "nauc_recall_at_10_std": 0.18059230306674634, + "nauc_recall_at_1_diff1": 0.7044065096654757, + "nauc_recall_at_1_max": 0.5750721143496796, + "nauc_recall_at_1_std": 0.046992006429299085, + "nauc_recall_at_20_diff1": 0.5504201680672272, + "nauc_recall_at_20_max": 0.7070786647992531, + "nauc_recall_at_20_std": 0.2757972980858999, + "nauc_recall_at_3_diff1": 0.5681348565963951, + "nauc_recall_at_3_max": 0.6170784343861266, + "nauc_recall_at_3_std": 0.09851473313011817, + "nauc_recall_at_5_diff1": 0.5156420439651506, + "nauc_recall_at_5_max": 0.6434716064276265, + "nauc_recall_at_5_std": 0.146063346120565, + "ndcg_at_1": 0.57, + "ndcg_at_10": 0.72935, + "ndcg_at_100": 0.75091, + "ndcg_at_1000": 0.75221, + "ndcg_at_20": 0.7409, + "ndcg_at_3": 0.67943, + "ndcg_at_5": 0.70771, + "precision_at_1": 0.57, + "precision_at_10": 0.089, + "precision_at_100": 0.00989, + "precision_at_1000": 0.001, + "precision_at_20": 0.0468, + "precision_at_3": 0.252, + "precision_at_5": 0.1649, + "recall_at_1": 0.57, + "recall_at_10": 0.89, + "recall_at_100": 0.9885, + "recall_at_1000": 0.998, + "recall_at_20": 0.936, + "recall_at_3": 0.756, + "recall_at_5": 0.8245 + }, + { + "cv_recall_at_1": 0.6515, + "cv_recall_at_10": 0.9325, + "cv_recall_at_100": 0.9915, + "cv_recall_at_1000": 0.999, + "cv_recall_at_20": 0.962, + "cv_recall_at_3": 0.822, + "cv_recall_at_5": 0.879, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.79245, + "map_at_1": 0.6515, + "map_at_10": 0.74747, + "map_at_100": 0.75038, + "map_at_1000": 0.75042, + "map_at_20": 0.74956, + "map_at_3": 0.72733, + "map_at_5": 0.74018, + "mrr_at_1": 0.6515, + "mrr_at_10": 0.7474712301587296, + "mrr_at_100": 0.7503837624364388, + "mrr_at_1000": 0.7504248142935365, + "mrr_at_20": 0.7495552861139542, + "mrr_at_3": 0.7273333333333322, + "mrr_at_5": 0.740183333333333, + "nauc_cv_recall_at_1000_diff1": 0.7770774976656952, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.8009007524578522, + "nauc_cv_recall_at_100_max": 0.9682814302191535, + "nauc_cv_recall_at_100_std": 0.729856648541773, + "nauc_cv_recall_at_10_diff1": 0.5883217484524672, + "nauc_cv_recall_at_10_max": 0.7557526714389453, + "nauc_cv_recall_at_10_std": 0.3104955562471906, + "nauc_cv_recall_at_1_diff1": 0.7329375789067353, + "nauc_cv_recall_at_1_max": 0.5361300015750745, + "nauc_cv_recall_at_1_std": 0.06148747407860173, + "nauc_cv_recall_at_20_diff1": 0.623163300407883, + "nauc_cv_recall_at_20_max": 0.8448633839500713, + "nauc_cv_recall_at_20_std": 0.42984298982750796, + "nauc_cv_recall_at_3_diff1": 0.6402739291556773, + "nauc_cv_recall_at_3_max": 0.6452442159383034, + "nauc_cv_recall_at_3_std": 0.17984982027398486, + "nauc_cv_recall_at_5_diff1": 0.6157551065526523, + "nauc_cv_recall_at_5_max": 0.6875468807984149, + "nauc_cv_recall_at_5_std": 0.28306329993446605, + "nauc_map_at_1000_diff1": 0.7039322369949266, + "nauc_map_at_1000_max": 0.5719568116443199, + "nauc_map_at_1000_std": 0.10175653818610049, + "nauc_map_at_100_diff1": 0.7039442745246894, + "nauc_map_at_100_max": 0.5720300068067742, + "nauc_map_at_100_std": 0.10188202291296097, + "nauc_map_at_10_diff1": 0.7021694520509648, + "nauc_map_at_10_max": 0.5735604207840197, + "nauc_map_at_10_std": 0.10327802325986768, + "nauc_map_at_1_diff1": 0.7329375789067353, + "nauc_map_at_1_max": 0.5361300015750745, + "nauc_map_at_1_std": 0.06148747407860173, + "nauc_map_at_20_diff1": 0.7035120770713532, + "nauc_map_at_20_max": 0.5729577824792697, + "nauc_map_at_20_std": 0.10260589720880942, + "nauc_map_at_3_diff1": 0.7005784793460021, + "nauc_map_at_3_max": 0.5730788546799835, + "nauc_map_at_3_std": 0.10061225940427539, + "nauc_map_at_5_diff1": 0.7005878046876627, + "nauc_map_at_5_max": 0.5743062534656194, + "nauc_map_at_5_std": 0.10807883793605706, + "nauc_mrr_at_1000_diff1": 0.7039322369949266, + "nauc_mrr_at_1000_max": 0.5719568116443199, + "nauc_mrr_at_1000_std": 0.10175653818610049, + "nauc_mrr_at_100_diff1": 0.7039442745246894, + "nauc_mrr_at_100_max": 0.5720300068067742, + "nauc_mrr_at_100_std": 0.10188202291296097, + "nauc_mrr_at_10_diff1": 0.7021694520509648, + "nauc_mrr_at_10_max": 0.5735604207840197, + "nauc_mrr_at_10_std": 0.10327802325986768, + "nauc_mrr_at_1_diff1": 0.7329375789067353, + "nauc_mrr_at_1_max": 0.5361300015750745, + "nauc_mrr_at_1_std": 0.06148747407860173, + "nauc_mrr_at_20_diff1": 0.7035120770713532, + "nauc_mrr_at_20_max": 0.5729577824792697, + "nauc_mrr_at_20_std": 0.10260589720880942, + "nauc_mrr_at_3_diff1": 0.7005784793460021, + "nauc_mrr_at_3_max": 0.5730788546799835, + "nauc_mrr_at_3_std": 0.10061225940427539, + "nauc_mrr_at_5_diff1": 0.7005878046876627, + "nauc_mrr_at_5_max": 0.5743062534656194, + "nauc_mrr_at_5_std": 0.10807883793605706, + "nauc_ndcg_at_1000_diff1": 0.700087104488326, + "nauc_ndcg_at_1000_max": 0.5795021097985256, + "nauc_ndcg_at_1000_std": 0.11154540403613405, + "nauc_ndcg_at_100_diff1": 0.7006550596100719, + "nauc_ndcg_at_100_max": 0.5817496273763957, + "nauc_ndcg_at_100_std": 0.11502416071470394, + "nauc_ndcg_at_10_diff1": 0.6905913110087234, + "nauc_ndcg_at_10_max": 0.5912371201845855, + "nauc_ndcg_at_10_std": 0.1235839893326974, + "nauc_ndcg_at_1_diff1": 0.7329375789067353, + "nauc_ndcg_at_1_max": 0.5361300015750745, + "nauc_ndcg_at_1_std": 0.06148747407860173, + "nauc_ndcg_at_20_diff1": 0.6966587387995632, + "nauc_ndcg_at_20_max": 0.5891614061644422, + "nauc_ndcg_at_20_std": 0.121914444846387, + "nauc_ndcg_at_3_diff1": 0.6884729340233343, + "nauc_ndcg_at_3_max": 0.587509909021513, + "nauc_ndcg_at_3_std": 0.11641242005242053, + "nauc_ndcg_at_5_diff1": 0.6873311264384971, + "nauc_ndcg_at_5_max": 0.5917469354526042, + "nauc_ndcg_at_5_std": 0.13434677089258235, + "nauc_precision_at_1000_diff1": 0.7770774976657188, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.8009007524578535, + "nauc_precision_at_100_max": 0.9682814302191545, + "nauc_precision_at_100_std": 0.7298566485417957, + "nauc_precision_at_10_diff1": 0.5883217484524637, + "nauc_precision_at_10_max": 0.7557526714389439, + "nauc_precision_at_10_std": 0.31049555624718805, + "nauc_precision_at_1_diff1": 0.7329375789067353, + "nauc_precision_at_1_max": 0.5361300015750745, + "nauc_precision_at_1_std": 0.06148747407860173, + "nauc_precision_at_20_diff1": 0.6231633004078801, + "nauc_precision_at_20_max": 0.8448633839500712, + "nauc_precision_at_20_std": 0.4298429898275066, + "nauc_precision_at_3_diff1": 0.6402739291556786, + "nauc_precision_at_3_max": 0.6452442159383038, + "nauc_precision_at_3_std": 0.17984982027398597, + "nauc_precision_at_5_diff1": 0.6157551065526519, + "nauc_precision_at_5_max": 0.6875468807984131, + "nauc_precision_at_5_std": 0.2830632999344636, + "nauc_recall_at_1000_diff1": 0.7770774976656952, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.8009007524578522, + "nauc_recall_at_100_max": 0.9682814302191535, + "nauc_recall_at_100_std": 0.729856648541773, + "nauc_recall_at_10_diff1": 0.5883217484524672, + "nauc_recall_at_10_max": 0.7557526714389453, + "nauc_recall_at_10_std": 0.3104955562471906, + "nauc_recall_at_1_diff1": 0.7329375789067353, + "nauc_recall_at_1_max": 0.5361300015750745, + "nauc_recall_at_1_std": 0.06148747407860173, + "nauc_recall_at_20_diff1": 0.623163300407883, + "nauc_recall_at_20_max": 0.8448633839500713, + "nauc_recall_at_20_std": 0.42984298982750796, + "nauc_recall_at_3_diff1": 0.6402739291556773, + "nauc_recall_at_3_max": 0.6452442159383034, + "nauc_recall_at_3_std": 0.17984982027398486, + "nauc_recall_at_5_diff1": 0.6157551065526523, + "nauc_recall_at_5_max": 0.6875468807984149, + "nauc_recall_at_5_std": 0.28306329993446605, + "ndcg_at_1": 0.6515, + "ndcg_at_10": 0.79245, + "ndcg_at_100": 0.80558, + "ndcg_at_1000": 0.80656, + "ndcg_at_20": 0.79995, + "ndcg_at_3": 0.75168, + "ndcg_at_5": 0.775, + "precision_at_1": 0.6515, + "precision_at_10": 0.09325, + "precision_at_100": 0.00992, + "precision_at_1000": 0.001, + "precision_at_20": 0.0481, + "precision_at_3": 0.274, + "precision_at_5": 0.1758, + "recall_at_1": 0.6515, + "recall_at_10": 0.9325, + "recall_at_100": 0.9915, + "recall_at_1000": 0.999, + "recall_at_20": 0.962, + "recall_at_3": 0.822, + "recall_at_5": 0.879 + }, + { + "cv_recall_at_1": 0.733, + "cv_recall_at_10": 0.963, + "cv_recall_at_100": 0.9985, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.984, + "cv_recall_at_3": 0.88, + "cv_recall_at_5": 0.9275, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.85232, + "map_at_1": 0.733, + "map_at_10": 0.81632, + "map_at_100": 0.81832, + "map_at_1000": 0.81834, + "map_at_20": 0.81786, + "map_at_3": 0.8005, + "map_at_5": 0.81127, + "mrr_at_1": 0.733, + "mrr_at_10": 0.8163236111111105, + "mrr_at_100": 0.8183248956975758, + "mrr_at_1000": 0.8183350849805338, + "mrr_at_20": 0.8178556814509012, + "mrr_at_3": 0.8004999999999989, + "mrr_at_5": 0.8112749999999994, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.8513849984438709, + "nauc_cv_recall_at_100_max": 0.8638344226580088, + "nauc_cv_recall_at_100_std": -0.10768751945235333, + "nauc_cv_recall_at_10_diff1": 0.7755759961642286, + "nauc_cv_recall_at_10_max": 0.7924899689605557, + "nauc_cv_recall_at_10_std": 0.03805486158427406, + "nauc_cv_recall_at_1_diff1": 0.8133792547946724, + "nauc_cv_recall_at_1_max": 0.6252031052536563, + "nauc_cv_recall_at_1_std": 0.09226384736404669, + "nauc_cv_recall_at_20_diff1": 0.7742471988795507, + "nauc_cv_recall_at_20_max": 0.8285189075630218, + "nauc_cv_recall_at_20_std": 0.1130222922502361, + "nauc_cv_recall_at_3_diff1": 0.7380508272351263, + "nauc_cv_recall_at_3_max": 0.7140669742284433, + "nauc_cv_recall_at_3_std": 0.09508232580337304, + "nauc_cv_recall_at_5_diff1": 0.7234907756205932, + "nauc_cv_recall_at_5_max": 0.753079622653659, + "nauc_cv_recall_at_5_std": 0.051019028300975704, + "nauc_map_at_1000_diff1": 0.7952082628577973, + "nauc_map_at_1000_max": 0.6478391020353417, + "nauc_map_at_1000_std": 0.09081338024626912, + "nauc_map_at_100_diff1": 0.7952086285167334, + "nauc_map_at_100_max": 0.6478494920991844, + "nauc_map_at_100_std": 0.09077674916537626, + "nauc_map_at_10_diff1": 0.7951467350309017, + "nauc_map_at_10_max": 0.6496051997722657, + "nauc_map_at_10_std": 0.09048990915348447, + "nauc_map_at_1_diff1": 0.8133792547946724, + "nauc_map_at_1_max": 0.6252031052536563, + "nauc_map_at_1_std": 0.09226384736404669, + "nauc_map_at_20_diff1": 0.7952202923863493, + "nauc_map_at_20_max": 0.6483931037134688, + "nauc_map_at_20_std": 0.09102570610885713, + "nauc_map_at_3_diff1": 0.790142614160109, + "nauc_map_at_3_max": 0.6517459403239515, + "nauc_map_at_3_std": 0.09487554955026746, + "nauc_map_at_5_diff1": 0.7914810063107847, + "nauc_map_at_5_max": 0.651685378725342, + "nauc_map_at_5_std": 0.09063688167845714, + "nauc_mrr_at_1000_diff1": 0.7952082628577973, + "nauc_mrr_at_1000_max": 0.6478391020353417, + "nauc_mrr_at_1000_std": 0.09081338024626912, + "nauc_mrr_at_100_diff1": 0.7952086285167334, + "nauc_mrr_at_100_max": 0.6478494920991844, + "nauc_mrr_at_100_std": 0.09077674916537626, + "nauc_mrr_at_10_diff1": 0.7951467350309017, + "nauc_mrr_at_10_max": 0.6496051997722657, + "nauc_mrr_at_10_std": 0.09048990915348447, + "nauc_mrr_at_1_diff1": 0.8133792547946724, + "nauc_mrr_at_1_max": 0.6252031052536563, + "nauc_mrr_at_1_std": 0.09226384736404669, + "nauc_mrr_at_20_diff1": 0.7952202923863493, + "nauc_mrr_at_20_max": 0.6483931037134688, + "nauc_mrr_at_20_std": 0.09102570610885713, + "nauc_mrr_at_3_diff1": 0.790142614160109, + "nauc_mrr_at_3_max": 0.6517459403239515, + "nauc_mrr_at_3_std": 0.09487554955026746, + "nauc_mrr_at_5_diff1": 0.7914810063107847, + "nauc_mrr_at_5_max": 0.651685378725342, + "nauc_mrr_at_5_std": 0.09063688167845714, + "nauc_ndcg_at_1000_diff1": 0.793184615751275, + "nauc_ndcg_at_1000_max": 0.6512997647156443, + "nauc_ndcg_at_1000_std": 0.0896376656750505, + "nauc_ndcg_at_100_diff1": 0.7932576505906453, + "nauc_ndcg_at_100_max": 0.6516218237126856, + "nauc_ndcg_at_100_std": 0.08916329598012869, + "nauc_ndcg_at_10_diff1": 0.7923981235421121, + "nauc_ndcg_at_10_max": 0.6601701521909391, + "nauc_ndcg_at_10_std": 0.08679745362754197, + "nauc_ndcg_at_1_diff1": 0.8133792547946724, + "nauc_ndcg_at_1_max": 0.6252031052536563, + "nauc_ndcg_at_1_std": 0.09226384736404669, + "nauc_ndcg_at_20_diff1": 0.7928995512374133, + "nauc_ndcg_at_20_max": 0.6555697297862627, + "nauc_ndcg_at_20_std": 0.09046056892469329, + "nauc_ndcg_at_3_diff1": 0.7805316637468971, + "nauc_ndcg_at_3_max": 0.6632016786911084, + "nauc_ndcg_at_3_std": 0.0949989221869438, + "nauc_ndcg_at_5_diff1": 0.7825497211406456, + "nauc_ndcg_at_5_max": 0.664619339432301, + "nauc_ndcg_at_5_std": 0.08604866742140158, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.851384998443846, + "nauc_precision_at_100_max": 0.8638344226578644, + "nauc_precision_at_100_std": -0.10768751945219786, + "nauc_precision_at_10_diff1": 0.7755759961642249, + "nauc_precision_at_10_max": 0.7924899689605511, + "nauc_precision_at_10_std": 0.0380548615842704, + "nauc_precision_at_1_diff1": 0.8133792547946724, + "nauc_precision_at_1_max": 0.6252031052536563, + "nauc_precision_at_1_std": 0.09226384736404669, + "nauc_precision_at_20_diff1": 0.7742471988795416, + "nauc_precision_at_20_max": 0.8285189075630202, + "nauc_precision_at_20_std": 0.11302229225024674, + "nauc_precision_at_3_diff1": 0.7380508272351264, + "nauc_precision_at_3_max": 0.7140669742284463, + "nauc_precision_at_3_std": 0.0950823258033741, + "nauc_precision_at_5_diff1": 0.7234907756205926, + "nauc_precision_at_5_max": 0.7530796226536576, + "nauc_precision_at_5_std": 0.05101902830097392, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.8513849984438709, + "nauc_recall_at_100_max": 0.8638344226580088, + "nauc_recall_at_100_std": -0.10768751945235333, + "nauc_recall_at_10_diff1": 0.7755759961642286, + "nauc_recall_at_10_max": 0.7924899689605557, + "nauc_recall_at_10_std": 0.03805486158427406, + "nauc_recall_at_1_diff1": 0.8133792547946724, + "nauc_recall_at_1_max": 0.6252031052536563, + "nauc_recall_at_1_std": 0.09226384736404669, + "nauc_recall_at_20_diff1": 0.7742471988795507, + "nauc_recall_at_20_max": 0.8285189075630218, + "nauc_recall_at_20_std": 0.1130222922502361, + "nauc_recall_at_3_diff1": 0.7380508272351263, + "nauc_recall_at_3_max": 0.7140669742284433, + "nauc_recall_at_3_std": 0.09508232580337304, + "nauc_recall_at_5_diff1": 0.7234907756205932, + "nauc_recall_at_5_max": 0.753079622653659, + "nauc_recall_at_5_std": 0.051019028300975704, + "ndcg_at_1": 0.733, + "ndcg_at_10": 0.85232, + "ndcg_at_100": 0.86059, + "ndcg_at_1000": 0.8608, + "ndcg_at_20": 0.85772, + "ndcg_at_3": 0.82103, + "ndcg_at_5": 0.84053, + "precision_at_1": 0.733, + "precision_at_10": 0.0963, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "precision_at_20": 0.0492, + "precision_at_3": 0.29333, + "precision_at_5": 0.1855, + "recall_at_1": 0.733, + "recall_at_10": 0.963, + "recall_at_100": 0.9985, + "recall_at_1000": 1.0, + "recall_at_20": 0.984, + "recall_at_3": 0.88, + "recall_at_5": 0.9275 + }, + { + "cv_recall_at_1": 0.329, + "cv_recall_at_10": 0.6625, + "cv_recall_at_100": 0.898, + "cv_recall_at_1000": 0.9935, + "cv_recall_at_20": 0.753, + "cv_recall_at_3": 0.488, + "cv_recall_at_5": 0.566, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.48558, + "map_at_1": 0.329, + "map_at_10": 0.43015, + "map_at_100": 0.4402, + "map_at_1000": 0.44068, + "map_at_20": 0.43662, + "map_at_3": 0.39908, + "map_at_5": 0.41716, + "mrr_at_1": 0.329, + "mrr_at_10": 0.4301498015873023, + "mrr_at_100": 0.44020367057678783, + "mrr_at_1000": 0.44068107415161467, + "mrr_at_20": 0.43662267512329517, + "mrr_at_3": 0.3990833333333339, + "mrr_at_5": 0.4171583333333347, + "nauc_cv_recall_at_1000_diff1": 0.570746247216831, + "nauc_cv_recall_at_1000_max": 0.9355383178912546, + "nauc_cv_recall_at_1000_std": 0.8143719026071998, + "nauc_cv_recall_at_100_diff1": 0.28561103871074306, + "nauc_cv_recall_at_100_max": 0.6815610029184481, + "nauc_cv_recall_at_100_std": 0.5854242763532237, + "nauc_cv_recall_at_10_diff1": 0.34808730638537383, + "nauc_cv_recall_at_10_max": 0.6230498863849571, + "nauc_cv_recall_at_10_std": 0.38972169526152306, + "nauc_cv_recall_at_1_diff1": 0.4897268439399369, + "nauc_cv_recall_at_1_max": 0.48143439232866647, + "nauc_cv_recall_at_1_std": 0.22002136550057733, + "nauc_cv_recall_at_20_diff1": 0.3732459116186757, + "nauc_cv_recall_at_20_max": 0.6731810217095842, + "nauc_cv_recall_at_20_std": 0.46552168739127814, + "nauc_cv_recall_at_3_diff1": 0.39113799929941523, + "nauc_cv_recall_at_3_max": 0.5270269194433536, + "nauc_cv_recall_at_3_std": 0.28440694428227187, + "nauc_cv_recall_at_5_diff1": 0.3756919063036706, + "nauc_cv_recall_at_5_max": 0.5573107027224745, + "nauc_cv_recall_at_5_std": 0.3082772997579581, + "nauc_map_at_1000_diff1": 0.4414051282408367, + "nauc_map_at_1000_max": 0.514311675803377, + "nauc_map_at_1000_std": 0.2621652878648845, + "nauc_map_at_100_diff1": 0.4411597090934524, + "nauc_map_at_100_max": 0.5145432570532894, + "nauc_map_at_100_std": 0.2625758722812923, + "nauc_map_at_10_diff1": 0.43856234721121634, + "nauc_map_at_10_max": 0.5144546481309277, + "nauc_map_at_10_std": 0.26100856265121153, + "nauc_map_at_1_diff1": 0.4897268439399369, + "nauc_map_at_1_max": 0.48143439232866647, + "nauc_map_at_1_std": 0.22002136550057733, + "nauc_map_at_20_diff1": 0.44113261525244246, + "nauc_map_at_20_max": 0.5157374897438756, + "nauc_map_at_20_std": 0.2631221937442976, + "nauc_map_at_3_diff1": 0.4430321264763373, + "nauc_map_at_3_max": 0.5016682101912822, + "nauc_map_at_3_std": 0.24791585529023333, + "nauc_map_at_5_diff1": 0.44076808631336867, + "nauc_map_at_5_max": 0.5079389409486393, + "nauc_map_at_5_std": 0.25294942024853534, + "nauc_mrr_at_1000_diff1": 0.4414051282408367, + "nauc_mrr_at_1000_max": 0.514311675803377, + "nauc_mrr_at_1000_std": 0.2621652878648845, + "nauc_mrr_at_100_diff1": 0.4411597090934524, + "nauc_mrr_at_100_max": 0.5145432570532894, + "nauc_mrr_at_100_std": 0.2625758722812923, + "nauc_mrr_at_10_diff1": 0.43856234721121634, + "nauc_mrr_at_10_max": 0.5144546481309277, + "nauc_mrr_at_10_std": 0.26100856265121153, + "nauc_mrr_at_1_diff1": 0.4897268439399369, + "nauc_mrr_at_1_max": 0.48143439232866647, + "nauc_mrr_at_1_std": 0.22002136550057733, + "nauc_mrr_at_20_diff1": 0.44113261525244246, + "nauc_mrr_at_20_max": 0.5157374897438756, + "nauc_mrr_at_20_std": 0.2631221937442976, + "nauc_mrr_at_3_diff1": 0.4430321264763373, + "nauc_mrr_at_3_max": 0.5016682101912822, + "nauc_mrr_at_3_std": 0.24791585529023333, + "nauc_mrr_at_5_diff1": 0.44076808631336867, + "nauc_mrr_at_5_max": 0.5079389409486393, + "nauc_mrr_at_5_std": 0.25294942024853534, + "nauc_ndcg_at_1000_diff1": 0.4330018707890607, + "nauc_ndcg_at_1000_max": 0.5266139850395504, + "nauc_ndcg_at_1000_std": 0.2796469830342363, + "nauc_ndcg_at_100_diff1": 0.4260189469019916, + "nauc_ndcg_at_100_max": 0.5325043437016203, + "nauc_ndcg_at_100_std": 0.2916521593222054, + "nauc_ndcg_at_10_diff1": 0.4190897586950866, + "nauc_ndcg_at_10_max": 0.5363395029642329, + "nauc_ndcg_at_10_std": 0.2870156955251191, + "nauc_ndcg_at_1_diff1": 0.4897268439399369, + "nauc_ndcg_at_1_max": 0.48143439232866647, + "nauc_ndcg_at_1_std": 0.22002136550057733, + "nauc_ndcg_at_20_diff1": 0.4274167281601817, + "nauc_ndcg_at_20_max": 0.5416035681782816, + "nauc_ndcg_at_20_std": 0.2959927566759077, + "nauc_ndcg_at_3_diff1": 0.42958629116833824, + "nauc_ndcg_at_3_max": 0.508177698679707, + "nauc_ndcg_at_3_std": 0.25725414826981835, + "nauc_ndcg_at_5_diff1": 0.42485040568184196, + "nauc_ndcg_at_5_max": 0.5196752086294824, + "nauc_ndcg_at_5_std": 0.26626862328931283, + "nauc_precision_at_1000_diff1": 0.5707462472168311, + "nauc_precision_at_1000_max": 0.9355383178912654, + "nauc_precision_at_1000_std": 0.8143719026071755, + "nauc_precision_at_100_diff1": 0.28561103871074167, + "nauc_precision_at_100_max": 0.6815610029184485, + "nauc_precision_at_100_std": 0.5854242763532232, + "nauc_precision_at_10_diff1": 0.3480873063853732, + "nauc_precision_at_10_max": 0.6230498863849561, + "nauc_precision_at_10_std": 0.3897216952615221, + "nauc_precision_at_1_diff1": 0.4897268439399369, + "nauc_precision_at_1_max": 0.48143439232866647, + "nauc_precision_at_1_std": 0.22002136550057733, + "nauc_precision_at_20_diff1": 0.3732459116186754, + "nauc_precision_at_20_max": 0.6731810217095847, + "nauc_precision_at_20_std": 0.4655216873912767, + "nauc_precision_at_3_diff1": 0.3911379992994151, + "nauc_precision_at_3_max": 0.5270269194433539, + "nauc_precision_at_3_std": 0.28440694428227226, + "nauc_precision_at_5_diff1": 0.37569190630366955, + "nauc_precision_at_5_max": 0.557310702722474, + "nauc_precision_at_5_std": 0.3082772997579572, + "nauc_recall_at_1000_diff1": 0.570746247216831, + "nauc_recall_at_1000_max": 0.9355383178912546, + "nauc_recall_at_1000_std": 0.8143719026071998, + "nauc_recall_at_100_diff1": 0.28561103871074306, + "nauc_recall_at_100_max": 0.6815610029184481, + "nauc_recall_at_100_std": 0.5854242763532237, + "nauc_recall_at_10_diff1": 0.34808730638537383, + "nauc_recall_at_10_max": 0.6230498863849571, + "nauc_recall_at_10_std": 0.38972169526152306, + "nauc_recall_at_1_diff1": 0.4897268439399369, + "nauc_recall_at_1_max": 0.48143439232866647, + "nauc_recall_at_1_std": 0.22002136550057733, + "nauc_recall_at_20_diff1": 0.3732459116186757, + "nauc_recall_at_20_max": 0.6731810217095842, + "nauc_recall_at_20_std": 0.46552168739127814, + "nauc_recall_at_3_diff1": 0.39113799929941523, + "nauc_recall_at_3_max": 0.5270269194433536, + "nauc_recall_at_3_std": 0.28440694428227187, + "nauc_recall_at_5_diff1": 0.3756919063036706, + "nauc_recall_at_5_max": 0.5573107027224745, + "nauc_recall_at_5_std": 0.3082772997579581, + "ndcg_at_1": 0.329, + "ndcg_at_10": 0.48558, + "ndcg_at_100": 0.53536, + "ndcg_at_1000": 0.5476, + "ndcg_at_20": 0.5087, + "ndcg_at_3": 0.42192, + "ndcg_at_5": 0.45426, + "precision_at_1": 0.329, + "precision_at_10": 0.06625, + "precision_at_100": 0.00898, + "precision_at_1000": 0.00099, + "precision_at_20": 0.03765, + "precision_at_3": 0.16267, + "precision_at_5": 0.1132, + "recall_at_1": 0.329, + "recall_at_10": 0.6625, + "recall_at_100": 0.898, + "recall_at_1000": 0.9935, + "recall_at_20": 0.753, + "recall_at_3": 0.488, + "recall_at_5": 0.566 + }, + { + "cv_recall_at_1": 0.7535, + "cv_recall_at_10": 0.9685, + "cv_recall_at_100": 0.9985, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.986, + "cv_recall_at_3": 0.8905, + "cv_recall_at_5": 0.939, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ], + "main_score": 0.8647, + "map_at_1": 0.7535, + "map_at_10": 0.83094, + "map_at_100": 0.83264, + "map_at_1000": 0.83265, + "map_at_20": 0.83224, + "map_at_3": 0.81583, + "map_at_5": 0.82686, + "mrr_at_1": 0.7535, + "mrr_at_10": 0.8309436507936507, + "mrr_at_100": 0.8326428461698397, + "mrr_at_1000": 0.8326528711689501, + "mrr_at_20": 0.8322375589172607, + "mrr_at_3": 0.815833333333333, + "mrr_at_5": 0.8268583333333337, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.9074074074074042, + "nauc_cv_recall_at_100_max": 0.9564270152506046, + "nauc_cv_recall_at_100_std": 0.08667911609083245, + "nauc_cv_recall_at_10_diff1": 0.7159604582573755, + "nauc_cv_recall_at_10_max": 0.7671216634802024, + "nauc_cv_recall_at_10_std": -0.3054925673973297, + "nauc_cv_recall_at_1_diff1": 0.810180475530634, + "nauc_cv_recall_at_1_max": 0.5523023359455304, + "nauc_cv_recall_at_1_std": -0.18382166913136913, + "nauc_cv_recall_at_20_diff1": 0.7574196345204801, + "nauc_cv_recall_at_20_max": 0.8418867547018848, + "nauc_cv_recall_at_20_std": -0.17697078831532298, + "nauc_cv_recall_at_3_diff1": 0.7287258127811141, + "nauc_cv_recall_at_3_max": 0.670665706437131, + "nauc_cv_recall_at_3_std": -0.1443513922609512, + "nauc_cv_recall_at_5_diff1": 0.718185088242946, + "nauc_cv_recall_at_5_max": 0.7166850346696072, + "nauc_cv_recall_at_5_std": -0.17488252131453574, + "nauc_map_at_1000_diff1": 0.7907323043547354, + "nauc_map_at_1000_max": 0.5861408086781905, + "nauc_map_at_1000_std": -0.1787536887590416, + "nauc_map_at_100_diff1": 0.7907384415401878, + "nauc_map_at_100_max": 0.5861665619023456, + "nauc_map_at_100_std": -0.17870425477150786, + "nauc_map_at_10_diff1": 0.7898007655152364, + "nauc_map_at_10_max": 0.58760606658666, + "nauc_map_at_10_std": -0.18104252909636068, + "nauc_map_at_1_diff1": 0.810180475530634, + "nauc_map_at_1_max": 0.5523023359455304, + "nauc_map_at_1_std": -0.18382166913136913, + "nauc_map_at_20_diff1": 0.7905669610329621, + "nauc_map_at_20_max": 0.586723738010071, + "nauc_map_at_20_std": -0.17897834402449417, + "nauc_map_at_3_diff1": 0.7848945769498309, + "nauc_map_at_3_max": 0.5899769317245516, + "nauc_map_at_3_std": -0.17212119267453208, + "nauc_map_at_5_diff1": 0.7879028117425809, + "nauc_map_at_5_max": 0.5892120425613406, + "nauc_map_at_5_std": -0.17707379971722223, + "nauc_mrr_at_1000_diff1": 0.7907323043547354, + "nauc_mrr_at_1000_max": 0.5861408086781905, + "nauc_mrr_at_1000_std": -0.1787536887590416, + "nauc_mrr_at_100_diff1": 0.7907384415401878, + "nauc_mrr_at_100_max": 0.5861665619023456, + "nauc_mrr_at_100_std": -0.17870425477150786, + "nauc_mrr_at_10_diff1": 0.7898007655152364, + "nauc_mrr_at_10_max": 0.58760606658666, + "nauc_mrr_at_10_std": -0.18104252909636068, + "nauc_mrr_at_1_diff1": 0.810180475530634, + "nauc_mrr_at_1_max": 0.5523023359455304, + "nauc_mrr_at_1_std": -0.18382166913136913, + "nauc_mrr_at_20_diff1": 0.7905669610329621, + "nauc_mrr_at_20_max": 0.586723738010071, + "nauc_mrr_at_20_std": -0.17897834402449417, + "nauc_mrr_at_3_diff1": 0.7848945769498309, + "nauc_mrr_at_3_max": 0.5899769317245516, + "nauc_mrr_at_3_std": -0.17212119267453208, + "nauc_mrr_at_5_diff1": 0.7879028117425809, + "nauc_mrr_at_5_max": 0.5892120425613406, + "nauc_mrr_at_5_std": -0.17707379971722223, + "nauc_ndcg_at_1000_diff1": 0.7886545545175407, + "nauc_ndcg_at_1000_max": 0.5912553799694619, + "nauc_ndcg_at_1000_std": -0.17878692861317544, + "nauc_ndcg_at_100_diff1": 0.7888518971892833, + "nauc_ndcg_at_100_max": 0.5919000390709643, + "nauc_ndcg_at_100_std": -0.1781301914289305, + "nauc_ndcg_at_10_diff1": 0.7841553474017766, + "nauc_ndcg_at_10_max": 0.6004233460965622, + "nauc_ndcg_at_10_std": -0.18770318177002843, + "nauc_ndcg_at_1_diff1": 0.810180475530634, + "nauc_ndcg_at_1_max": 0.5523023359455304, + "nauc_ndcg_at_1_std": -0.18382166913136913, + "nauc_ndcg_at_20_diff1": 0.7877499835627467, + "nauc_ndcg_at_20_max": 0.5966008749588094, + "nauc_ndcg_at_20_std": -0.17918935976242298, + "nauc_ndcg_at_3_diff1": 0.7746259970314816, + "nauc_ndcg_at_3_max": 0.6047740676129029, + "nauc_ndcg_at_3_std": -0.1670716308750744, + "nauc_ndcg_at_5_diff1": 0.779426755716452, + "nauc_ndcg_at_5_max": 0.6045320925860991, + "nauc_ndcg_at_5_std": -0.17646919507329487, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.9074074074073368, + "nauc_precision_at_100_max": 0.9564270152505274, + "nauc_precision_at_100_std": 0.0866791160907527, + "nauc_precision_at_10_diff1": 0.7159604582573762, + "nauc_precision_at_10_max": 0.7671216634802053, + "nauc_precision_at_10_std": -0.30549256739733754, + "nauc_precision_at_1_diff1": 0.810180475530634, + "nauc_precision_at_1_max": 0.5523023359455304, + "nauc_precision_at_1_std": -0.18382166913136913, + "nauc_precision_at_20_diff1": 0.7574196345204598, + "nauc_precision_at_20_max": 0.8418867547018706, + "nauc_precision_at_20_std": -0.1769707883153585, + "nauc_precision_at_3_diff1": 0.7287258127811156, + "nauc_precision_at_3_max": 0.6706657064371305, + "nauc_precision_at_3_std": -0.14435139226094928, + "nauc_precision_at_5_diff1": 0.7181850882429455, + "nauc_precision_at_5_max": 0.7166850346696044, + "nauc_precision_at_5_std": -0.17488252131453935, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.9074074074074042, + "nauc_recall_at_100_max": 0.9564270152506046, + "nauc_recall_at_100_std": 0.08667911609083245, + "nauc_recall_at_10_diff1": 0.7159604582573755, + "nauc_recall_at_10_max": 0.7671216634802024, + "nauc_recall_at_10_std": -0.3054925673973297, + "nauc_recall_at_1_diff1": 0.810180475530634, + "nauc_recall_at_1_max": 0.5523023359455304, + "nauc_recall_at_1_std": -0.18382166913136913, + "nauc_recall_at_20_diff1": 0.7574196345204801, + "nauc_recall_at_20_max": 0.8418867547018848, + "nauc_recall_at_20_std": -0.17697078831532298, + "nauc_recall_at_3_diff1": 0.7287258127811141, + "nauc_recall_at_3_max": 0.670665706437131, + "nauc_recall_at_3_std": -0.1443513922609512, + "nauc_recall_at_5_diff1": 0.718185088242946, + "nauc_recall_at_5_max": 0.7166850346696072, + "nauc_recall_at_5_std": -0.17488252131453574, + "ndcg_at_1": 0.7535, + "ndcg_at_10": 0.8647, + "ndcg_at_100": 0.87171, + "ndcg_at_1000": 0.87191, + "ndcg_at_20": 0.86922, + "ndcg_at_3": 0.83509, + "ndcg_at_5": 0.85502, + "precision_at_1": 0.7535, + "precision_at_10": 0.09685, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "precision_at_20": 0.0493, + "precision_at_3": 0.29683, + "precision_at_5": 0.1878, + "recall_at_1": 0.7535, + "recall_at_10": 0.9685, + "recall_at_100": 0.9985, + "recall_at_1000": 1.0, + "recall_at_20": 0.986, + "recall_at_3": 0.8905, + "recall_at_5": 0.939 + } + ] + }, + "task_name": "XFlickr30kCoT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/model_meta.json b/results-mieb/voyage-multimodal-3/1/model_meta.json new file mode 100644 index 0000000000..528c812ae0 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/model_meta.json @@ -0,0 +1 @@ +{"name": "voyage-multimodal-3", "revision": "1", "release_date": "2024-11-10", "languages": [], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": 1024, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": "cosine", "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "voyage_v_loader"} \ No newline at end of file From 5f0b9c025e2c3a4388e6691094bf04084d182fcd Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Wed, 11 Dec 2024 02:43:41 +0800 Subject: [PATCH 116/154] [mieb] update script for final re-run (#1576) * mieb final runs * lint --- scripts/run_mieb_missed_results.py | 107 ++++++++--------------------- 1 file changed, 29 insertions(+), 78 deletions(-) diff --git a/scripts/run_mieb_missed_results.py b/scripts/run_mieb_missed_results.py index f3b5231781..5c92289f61 100644 --- a/scripts/run_mieb_missed_results.py +++ b/scripts/run_mieb_missed_results.py @@ -2,75 +2,6 @@ import mteb -# missing -model = mteb.get_model("royokong/e5-v") -tasks = mteb.get_tasks(tasks=["EDIST2ITRetrieval", "Winoground"]) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - -model = mteb.get_model("nyu-visionx/moco-v3-vit-b") -tasks = mteb.get_tasks( - tasks=[ - "CIFAR10Clustering", - "ImageNetDog15Clustering", - "TinyImageNetClustering", - "UCF101", - "VOC2007", - ] -) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - -model = mteb.get_model("nyu-visionx/moco-v3-vit-l") -tasks = mteb.get_tasks( - tasks=[ - "SketchyI2IRetrieval", - "CIFAR10Clustering", - "ImageNetDog15Clustering", - "TinyImageNetClustering", - "UCF101", - "VOC2007", - ] -) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - -model = mteb.get_model("BAAI/bge-visualized-base") -tasks = mteb.get_tasks( - tasks=[ - "SciMMIRI2TRetrieval", - "SciMMIRT2IRetrieval", - "VisualNewsI2TRetrieval", - "WITT2IRetrieval", - ] -) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - -model = mteb.get_model("TIGER-Lab/VLM2Vec-Full") -tasks = mteb.get_tasks( - tasks=["CVBenchCount", "CVBenchDepth", "CVBenchDistance", "CVBenchRelation"] -) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - -model = mteb.get_model("TIGER-Lab/VLM2Vec-LoRA") -tasks = mteb.get_tasks( - task_types=[ - "Any2AnyRetrieval", - "Any2AnyMultiChoice", - "Any2TextMutipleChoice", - "ImageClustering", - "ImageClassification", - "ImageMultilabelClassification", - "ImageTextPairClassification", - "VisualSTS", - "ZeroShotClassification", - ] -) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - # rerun for model_name in [ "openai/clip-vit-base-patch32", @@ -82,8 +13,8 @@ "kakaobrain/align-base", "jinaai/jina-clip-v1", "nomic-ai/nomic-embed-vision-v1.5", - "Salesforce/blip-image-captioning-large", - "Salesforce/blip-image-captioning-base", + # "Salesforce/blip-image-captioning-large", + # "Salesforce/blip-image-captioning-base", "Salesforce/blip2-opt-2.7b", "Salesforce/blip2-opt-6.7b-coco", "facebook/dinov2-small", @@ -125,12 +56,18 @@ model = mteb.get_model(model_name) tasks = mteb.get_tasks( tasks=[ - "ROxfordEasyI2IRetrieval", - "ROxfordHardI2IRetrieval", - "ROxfordMediumI2IRetrieval", - "RParisEasyI2IRetrieval", - "RParisHardI2IRetrieval", - "RParisMediumI2IRetrieval", + "ROxfordEasyI2IMultiChoice", + "ROxfordHardI2IMultiChoice", + "ROxfordMediumI2IMultiChoice", + "RParisEasyI2IMultiChoice", + "RParisHardI2IMultiChoice", + "RParisMediumI2IMultiChoice", + "BLINKIT2IRetrieval", + "BLINKIT2TRetrieval", + "BLINKIT2IMultiChoice", + "BLINKIT2TMultiChoice", + "Flickr30kI2TRetrieval", + "Flickr30kT2IRetrieval", ] ) # get i-only tasks for i-only models. @@ -138,4 +75,18 @@ tasks = [task for task in tasks if "t" not in task.metadata.category] evaluation = mteb.MTEB(tasks=tasks) - results = evaluation.run(model, output_folder="results-mieb-rerun") + results = evaluation.run(model, output_folder="results-mieb-rerun2") + +# # missing task +model_name = "TIGER-Lab/VLM2Vec-Full" +model = mteb.get_model(model_name) +tasks = mteb.get_tasks( + tasks=[ + "CVBenchCount", + "CVBenchDepth", + "CVBenchDistance", + "CVBenchRelation", + ] +) +evaluation = mteb.MTEB(tasks=tasks) +results = evaluation.run(model, output_folder="results-mieb-rerun2") From d2bb0acc92aae9b31428bf3873eee71a4e6e22d8 Mon Sep 17 00:00:00 2001 From: Jamie-Stirling <36764530+Jamie-Stirling@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:47:13 +0000 Subject: [PATCH 117/154] fix: no longer using same query text for all of BLINKIT2TMultiChoice (#1572) * fix: no longer using same query text for all of BLINKIT2TMultiChoice * fix: remove blink subtask * fix: remove subtask from blink it2i * fix: align BLINK retrieval to multi choice * add ROxford and RParis I2I multi choice * add retrieval metrics to multi choice evaluator * fix: remove wrong negatives from revisiting multichoice datasets * fix revisiting datasets * add new results for revisiting multichoice --- .../Image/AbsTaskAny2AnyMultiChoice.py | 12 ++ .../Image/Any2AnyMultiChoice/__init__.py | 2 + .../eng/BLINKIT2IMultiChoice.py | 2 +- .../eng/BLINKIT2TMultiChoice.py | 2 +- .../eng/ROxfordI2IMultiChoice.py | 142 +++++++++++++ .../eng/RParisI2IMultiChoice.py | 142 +++++++++++++ .../eng/BLINKIT2IRetrieval.py | 2 +- .../eng/BLINKIT2TRetrieval.py | 2 +- .../ROxfordEasyI2IMultiChoice.json | 187 ++++++++++++++++++ .../ROxfordHardI2IMultiChoice.json | 187 ++++++++++++++++++ .../ROxfordMediumI2IMultiChoice.json | 187 ++++++++++++++++++ .../RParisEasyI2IMultiChoice.json | 187 ++++++++++++++++++ .../RParisHardI2IMultiChoice.json | 187 ++++++++++++++++++ .../RParisMediumI2IMultiChoice.json | 187 ++++++++++++++++++ 14 files changed, 1424 insertions(+), 4 deletions(-) create mode 100644 mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py create mode 100644 mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IMultiChoice.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IMultiChoice.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IMultiChoice.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IMultiChoice.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IMultiChoice.json create mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IMultiChoice.json diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py b/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py index 4bef3f30b3..66a25c6619 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py @@ -321,7 +321,19 @@ def _evaluate_subset( ) scores = { **{f"ndcg_at_{k.split('@')[1]}": v for (k, v) in ndcg.items()}, + **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()}, + **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()}, + **{f"cv_recall_at_{k.split('@')[1]}": v for (k, v) in cv_recall.items()}, + **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()}, **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, + **{ + k.replace("@", "_at_").replace("_P", "_precision").lower(): v + for k, v in naucs.items() + }, + **{ + k.replace("@", "_at_").replace("_P", "_precision").lower(): v + for k, v in naucs_mrr.items() + }, "accuracy": recall["Recall@1"], } self._add_main_score(scores) diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py b/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py index c818af7048..0e3b6d4505 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py @@ -3,3 +3,5 @@ from .eng.BLINKIT2IMultiChoice import * from .eng.BLINKIT2TMultiChoice import * from .eng.ImageCoDeT2IMultiChoice import * +from .eng.ROxfordI2IMultiChoice import * +from .eng.RParisI2IMultiChoice import * diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py index 98b0a0120b..58db0c8c92 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py @@ -11,7 +11,7 @@ class BLINKIT2IMultiChoice(AbsTaskAny2AnyMultiChoice): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2i-multi", - "revision": "b7b46b72d1ed1fa44d25e2b9c4726afab4a7ce53", + "revision": "a9f994925551c14503d00d86f1307bac6e2ead6a", "trust_remote_code": True, }, type="Any2AnyMultiChoice", diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py index 60f42b8b05..0a1dfcdc42 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py @@ -11,7 +11,7 @@ class BLINKIT2TMultiChoice(AbsTaskAny2AnyMultiChoice): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2t-multi", - "revision": "ae713b03ae68e343f16c3bcdbd1b1ee760975d55", + "revision": "bc8f4c7f62450a4ceb737c8339061cf87aea42d5", }, type="Any2AnyMultiChoice", category="it2t", diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py new file mode 100644 index 0000000000..136848c128 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class ROxfordEasyI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="ROxfordEasyI2IMultiChoice", + description="Retrieve photos of landmarks in Oxford, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-oxford-easy-multi", + "revision": "4c167c3ce529f19457c9b8e694258cc6cf8e7cc7", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 516, + "num_queries": 70, + "average_relevant_docs_per_query": 43.3, + } + }, + }, + ) + skip_first_result = False + + +class ROxfordMediumI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="ROxfordMediumI2IMultiChoice", + description="Retrieve photos of landmarks in Oxford, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-oxford-medium-multi", + "revision": "83bd440268e200a4f60313070618e3f45000fa94", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 788, + "num_queries": 70, + "average_relevant_docs_per_query": 78.9, + } + }, + }, + ) + skip_first_result = False + + +class ROxfordHardI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="ROxfordHardI2IMultiChoice", + description="Retrieve photos of landmarks in Oxford, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-oxford-hard-multi", + "revision": "fc7c4ae6655b1e6b132f3b262a359acef42dfce8", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 685, + "num_queries": 70, + "average_relevant_docs_per_query": 35.7, + } + }, + }, + ) + skip_first_result = False diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py new file mode 100644 index 0000000000..69da75118f --- /dev/null +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class RParisEasyI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="RParisEasyI2IMultiChoice", + description="Retrieve photos of landmarks in Paris, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-paris-easy-multi", + "revision": "db94b5afd0014ab8c978f20a0fbcc52da1612a08", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting paris and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 516, + "num_queries": 70, + "average_relevant_docs_per_query": 43.3, + } + }, + }, + ) + skip_first_result = False + + +class RParisMediumI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="RParisMediumI2IMultiChoice", + description="Retrieve photos of landmarks in Paris, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-paris-medium-multi", + "revision": "372c79fc823e1cebc1d55f8e0039aa239285e177", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting paris and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 788, + "num_queries": 70, + "average_relevant_docs_per_query": 78.9, + } + }, + }, + ) + skip_first_result = False + + +class RParisHardI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="RParisHardI2IMultiChoice", + description="Retrieve photos of landmarks in Paris, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-paris-hard-multi", + "revision": "4e5997e48fb2f2f8bf1c8973851dedeb17e09a83", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting paris and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 685, + "num_queries": 70, + "average_relevant_docs_per_query": 35.7, + } + }, + }, + ) + skip_first_result = False diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py index 6b133f47c3..8202bb133c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py @@ -11,7 +11,7 @@ class BLINKIT2IRetrieval(AbsTaskAny2AnyRetrieval): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2i", - "revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26", + "revision": "7a1a1330565faca9c1aeec6f5acfc64f21296753", "trust_remote_code": True, }, type="Any2AnyRetrieval", diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py index da63f01df4..ff6ec42427 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py @@ -11,7 +11,7 @@ class BLINKIT2TRetrieval(AbsTaskAny2AnyRetrieval): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2t", - "revision": "302cf2008f204285985099dcd46425b00356c610", + "revision": "c6470936de49d6d2ae5fc09612752c75175ce5b6", "trust_remote_code": True, }, type="Any2AnyRetrieval", diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IMultiChoice.json new file mode 100644 index 0000000000..2b9fb1f38e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "4c167c3ce529f19457c9b8e694258cc6cf8e7cc7", + "evaluation_time": 13.940337419509888, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.12011, + "cv_recall_at_1": 0.72857, + "cv_recall_at_10": 0.85714, + "cv_recall_at_100": 0.94286, + "cv_recall_at_1000": 0.97143, + "cv_recall_at_20": 0.87143, + "cv_recall_at_3": 0.78571, + "cv_recall_at_5": 0.84286, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.12011, + "map_at_1": 0.12011, + "map_at_10": 0.27377, + "map_at_100": 0.42059, + "map_at_1000": 0.48363, + "map_at_20": 0.32913, + "map_at_3": 0.1815, + "map_at_5": 0.21622, + "mrr_at_1": 0.7285714285714285, + "mrr_at_10": 0.7696598639455783, + "mrr_at_100": 0.7720894555929709, + "mrr_at_1000": 0.7721998445218422, + "mrr_at_20": 0.7705527210884354, + "mrr_at_3": 0.7547619047619047, + "mrr_at_5": 0.7676190476190478, + "nauc_cv_recall_at_1000_diff1": 0.24042950513538902, + "nauc_cv_recall_at_1000_max": -0.004668534080296818, + "nauc_cv_recall_at_1000_std": -0.5144724556489252, + "nauc_cv_recall_at_100_diff1": 0.22934173669467725, + "nauc_cv_recall_at_100_max": 0.3862044817927167, + "nauc_cv_recall_at_100_std": 0.06185807656395899, + "nauc_cv_recall_at_10_diff1": 0.46904761904761855, + "nauc_cv_recall_at_10_max": 0.45544217687074784, + "nauc_cv_recall_at_10_std": -0.11797862001943518, + "nauc_cv_recall_at_1_diff1": 0.37913486005089103, + "nauc_cv_recall_at_1_max": 0.6452758437491262, + "nauc_cv_recall_at_1_std": 0.13670553365209895, + "nauc_cv_recall_at_20_diff1": 0.4747302062186126, + "nauc_cv_recall_at_20_max": 0.38631264023934025, + "nauc_cv_recall_at_20_std": -0.1437653595469617, + "nauc_cv_recall_at_3_diff1": 0.2941673710904481, + "nauc_cv_recall_at_3_max": 0.5864074387151308, + "nauc_cv_recall_at_3_std": 0.08314454775993227, + "nauc_cv_recall_at_5_diff1": 0.48302290348453764, + "nauc_cv_recall_at_5_max": 0.4865876481597005, + "nauc_cv_recall_at_5_std": -0.012743962213706547, + "nauc_map_at_1000_diff1": 0.039056418726859325, + "nauc_map_at_1000_max": 0.3108646441695373, + "nauc_map_at_1000_std": 0.18387643315553479, + "nauc_map_at_100_diff1": 0.17039421636019683, + "nauc_map_at_100_max": 0.09905795512067436, + "nauc_map_at_100_std": -0.047296000166814706, + "nauc_map_at_10_diff1": 0.39318659870946504, + "nauc_map_at_10_max": -0.2567134684719764, + "nauc_map_at_10_std": -0.3372313247661854, + "nauc_map_at_1_diff1": 0.4052522690135478, + "nauc_map_at_1_max": -0.3133714030581606, + "nauc_map_at_1_std": -0.3279366241134995, + "nauc_map_at_20_diff1": 0.31128533550252463, + "nauc_map_at_20_max": -0.1668988523200872, + "nauc_map_at_20_std": -0.2950570027817496, + "nauc_map_at_3_diff1": 0.47350551731133683, + "nauc_map_at_3_max": -0.3190738829133001, + "nauc_map_at_3_std": -0.34746748342150613, + "nauc_map_at_5_diff1": 0.4297189568601358, + "nauc_map_at_5_max": -0.306734565163091, + "nauc_map_at_5_std": -0.3356128846263281, + "nauc_mrr_at_1000_diff1": 0.3765086703362948, + "nauc_mrr_at_1000_max": 0.6070793223986851, + "nauc_mrr_at_1000_std": 0.09762189861063993, + "nauc_mrr_at_100_diff1": 0.3765699744452178, + "nauc_mrr_at_100_max": 0.6072126617329957, + "nauc_mrr_at_100_std": 0.0978972735979146, + "nauc_mrr_at_10_diff1": 0.3788805046207156, + "nauc_mrr_at_10_max": 0.6082629986581378, + "nauc_mrr_at_10_std": 0.0963034659332654, + "nauc_mrr_at_1_diff1": 0.37913486005089103, + "nauc_mrr_at_1_max": 0.6452758437491262, + "nauc_mrr_at_1_std": 0.13670553365209895, + "nauc_mrr_at_20_diff1": 0.3787562524948725, + "nauc_mrr_at_20_max": 0.6062244307479707, + "nauc_mrr_at_20_std": 0.09613446360209521, + "nauc_mrr_at_3_diff1": 0.34401526617095446, + "nauc_mrr_at_3_max": 0.6246273302929221, + "nauc_mrr_at_3_std": 0.12242294784901903, + "nauc_mrr_at_5_diff1": 0.38107615384083393, + "nauc_mrr_at_5_max": 0.6101526396341715, + "nauc_mrr_at_5_std": 0.10525288237443857, + "nauc_ndcg_at_1000_diff1": 0.12762070579347834, + "nauc_ndcg_at_1000_max": 0.4065392036326618, + "nauc_ndcg_at_1000_std": 0.13600118977631057, + "nauc_ndcg_at_100_diff1": 0.08905540963562808, + "nauc_ndcg_at_100_max": 0.3340876494383974, + "nauc_ndcg_at_100_std": 0.16389116328814335, + "nauc_ndcg_at_10_diff1": 0.031817431716988705, + "nauc_ndcg_at_10_max": 0.48755851455947774, + "nauc_ndcg_at_10_std": 0.1880385361511959, + "nauc_ndcg_at_1_diff1": 0.37913486005089103, + "nauc_ndcg_at_1_max": 0.6452758437491262, + "nauc_ndcg_at_1_std": 0.13670553365209895, + "nauc_ndcg_at_20_diff1": 0.03212844503807327, + "nauc_ndcg_at_20_max": 0.40893860255098513, + "nauc_ndcg_at_20_std": 0.2191836350931408, + "nauc_ndcg_at_3_diff1": 0.10952929105103347, + "nauc_ndcg_at_3_max": 0.5868226539522446, + "nauc_ndcg_at_3_std": 0.20418399857463668, + "nauc_ndcg_at_5_diff1": 0.05132083453784389, + "nauc_ndcg_at_5_max": 0.5305326791254221, + "nauc_ndcg_at_5_std": 0.2280586632195748, + "nauc_precision_at_1000_diff1": -0.3414619911663539, + "nauc_precision_at_1000_max": 0.5560000366090037, + "nauc_precision_at_1000_std": 0.5508816734665237, + "nauc_precision_at_100_diff1": -0.3787961838502515, + "nauc_precision_at_100_max": 0.7160900928470757, + "nauc_precision_at_100_std": 0.6714675209553157, + "nauc_precision_at_10_diff1": -0.3066362781491847, + "nauc_precision_at_10_max": 0.6670893640902829, + "nauc_precision_at_10_std": 0.4078566027551021, + "nauc_precision_at_1_diff1": 0.37913486005089103, + "nauc_precision_at_1_max": 0.6452758437491262, + "nauc_precision_at_1_std": 0.13670553365209895, + "nauc_precision_at_20_diff1": -0.3569525663518488, + "nauc_precision_at_20_max": 0.6799005999378203, + "nauc_precision_at_20_std": 0.47997442491768877, + "nauc_precision_at_3_diff1": -0.022372098222603304, + "nauc_precision_at_3_max": 0.6144576517494506, + "nauc_precision_at_3_std": 0.2747400976264113, + "nauc_precision_at_5_diff1": -0.2176180661349585, + "nauc_precision_at_5_max": 0.6350255399229319, + "nauc_precision_at_5_std": 0.3850972309346714, + "nauc_recall_at_1000_diff1": 0.21587695470985352, + "nauc_recall_at_1000_max": 0.11425941926240299, + "nauc_recall_at_1000_std": 0.05546332909555691, + "nauc_recall_at_100_diff1": 0.21066338253500203, + "nauc_recall_at_100_max": -0.23122605362849036, + "nauc_recall_at_100_std": -0.17781012181323355, + "nauc_recall_at_10_diff1": 0.42259978636740364, + "nauc_recall_at_10_max": -0.34951875178863107, + "nauc_recall_at_10_std": -0.39276608756700837, + "nauc_recall_at_1_diff1": 0.4052522690135478, + "nauc_recall_at_1_max": -0.3133714030581606, + "nauc_recall_at_1_std": -0.3279366241134995, + "nauc_recall_at_20_diff1": 0.3429964665906505, + "nauc_recall_at_20_max": -0.32830343914659427, + "nauc_recall_at_20_std": -0.37518353692962664, + "nauc_recall_at_3_diff1": 0.4793068578557846, + "nauc_recall_at_3_max": -0.3280804979129421, + "nauc_recall_at_3_std": -0.3517807268312443, + "nauc_recall_at_5_diff1": 0.43717782790927456, + "nauc_recall_at_5_max": -0.36275023715800275, + "nauc_recall_at_5_std": -0.34949269884261674, + "ndcg_at_1": 0.72857, + "ndcg_at_10": 0.62501, + "ndcg_at_100": 0.61893, + "ndcg_at_1000": 0.69861, + "ndcg_at_20": 0.60364, + "ndcg_at_3": 0.64316, + "ndcg_at_5": 0.63774, + "precision_at_1": 0.72857, + "precision_at_10": 0.47286, + "precision_at_100": 0.19714, + "precision_at_1000": 0.0413, + "precision_at_20": 0.38857, + "precision_at_3": 0.57619, + "precision_at_5": 0.53429, + "recall_at_1": 0.12011, + "recall_at_10": 0.3365, + "recall_at_100": 0.67186, + "recall_at_1000": 0.915, + "recall_at_20": 0.43313, + "recall_at_3": 0.18464, + "recall_at_5": 0.24446 + } + ] + }, + "task_name": "ROxfordEasyI2IMultiChoice" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IMultiChoice.json new file mode 100644 index 0000000000..0c110c6ead --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "fc7c4ae6655b1e6b132f3b262a359acef42dfce8", + "evaluation_time": 15.003882884979248, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.01148, + "cv_recall_at_1": 0.25714, + "cv_recall_at_10": 0.65714, + "cv_recall_at_100": 0.84286, + "cv_recall_at_1000": 0.92857, + "cv_recall_at_20": 0.68571, + "cv_recall_at_3": 0.45714, + "cv_recall_at_5": 0.57143, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.01148, + "map_at_1": 0.01148, + "map_at_10": 0.05335, + "map_at_100": 0.07243, + "map_at_1000": 0.08784, + "map_at_20": 0.0581, + "map_at_3": 0.03866, + "map_at_5": 0.04476, + "mrr_at_1": 0.2571428571428571, + "mrr_at_10": 0.38481859410430835, + "mrr_at_100": 0.39253963385655055, + "mrr_at_1000": 0.393048468885359, + "mrr_at_20": 0.38730777159348584, + "mrr_at_3": 0.3476190476190476, + "mrr_at_5": 0.3726190476190476, + "nauc_cv_recall_at_1000_diff1": 0.26685340802987884, + "nauc_cv_recall_at_1000_max": 0.07301587301587338, + "nauc_cv_recall_at_1000_std": -0.1830999066293196, + "nauc_cv_recall_at_100_diff1": 0.21040905445147495, + "nauc_cv_recall_at_100_max": 0.11148738971571162, + "nauc_cv_recall_at_100_std": 0.02419570448266679, + "nauc_cv_recall_at_10_diff1": 0.17949926917817913, + "nauc_cv_recall_at_10_max": 0.06129473336791022, + "nauc_cv_recall_at_10_std": 0.20811919468150294, + "nauc_cv_recall_at_1_diff1": 0.4553397252313951, + "nauc_cv_recall_at_1_max": -0.00898601253476105, + "nauc_cv_recall_at_1_std": 0.15209811978583004, + "nauc_cv_recall_at_20_diff1": 0.12448806313055567, + "nauc_cv_recall_at_20_max": 0.16716611727100145, + "nauc_cv_recall_at_20_std": 0.15480471481370445, + "nauc_cv_recall_at_3_diff1": 0.30826644662816005, + "nauc_cv_recall_at_3_max": -0.013483609631685515, + "nauc_cv_recall_at_3_std": 0.12409046583152049, + "nauc_cv_recall_at_5_diff1": 0.1614784394250518, + "nauc_cv_recall_at_5_max": -0.02459958932238193, + "nauc_cv_recall_at_5_std": 0.15406570841889153, + "nauc_map_at_1000_diff1": 0.31728785205951265, + "nauc_map_at_1000_max": -0.21424686509817786, + "nauc_map_at_1000_std": 0.0221906403447802, + "nauc_map_at_100_diff1": 0.31755174177151774, + "nauc_map_at_100_max": -0.19078874190305894, + "nauc_map_at_100_std": -0.023681508955016594, + "nauc_map_at_10_diff1": 0.2799136913282966, + "nauc_map_at_10_max": -0.1520026777231754, + "nauc_map_at_10_std": -0.07629896780635309, + "nauc_map_at_1_diff1": 0.4868843030130222, + "nauc_map_at_1_max": -0.15511847387665895, + "nauc_map_at_1_std": -0.02716611321363879, + "nauc_map_at_20_diff1": 0.2902764596993506, + "nauc_map_at_20_max": -0.12983705173414045, + "nauc_map_at_20_std": -0.04242912291533665, + "nauc_map_at_3_diff1": 0.2199130409925843, + "nauc_map_at_3_max": -0.1300575199918267, + "nauc_map_at_3_std": -0.12177697907382404, + "nauc_map_at_5_diff1": 0.2526425553708014, + "nauc_map_at_5_max": -0.1506194670949868, + "nauc_map_at_5_std": -0.11862027451443419, + "nauc_mrr_at_1000_diff1": 0.3597607815149087, + "nauc_mrr_at_1000_max": 0.008898477350336038, + "nauc_mrr_at_1000_std": 0.14259842374421733, + "nauc_mrr_at_100_diff1": 0.3595466281641209, + "nauc_mrr_at_100_max": 0.008904925581822543, + "nauc_mrr_at_100_std": 0.1429363715769275, + "nauc_mrr_at_10_diff1": 0.3577159847817333, + "nauc_mrr_at_10_max": 0.00585201676533247, + "nauc_mrr_at_10_std": 0.1493154319727965, + "nauc_mrr_at_1_diff1": 0.4553397252313951, + "nauc_mrr_at_1_max": -0.00898601253476105, + "nauc_mrr_at_1_std": 0.15209811978583004, + "nauc_mrr_at_20_diff1": 0.354629000714685, + "nauc_mrr_at_20_max": 0.013433224395754593, + "nauc_mrr_at_20_std": 0.14517705322833865, + "nauc_mrr_at_3_diff1": 0.38374642485486704, + "nauc_mrr_at_3_max": -0.007022664947542781, + "nauc_mrr_at_3_std": 0.13390513839165924, + "nauc_mrr_at_5_diff1": 0.3519534083953614, + "nauc_mrr_at_5_max": -0.004425335185700782, + "nauc_mrr_at_5_std": 0.14223014611578186, + "nauc_ndcg_at_1000_diff1": 0.3270873952168286, + "nauc_ndcg_at_1000_max": -0.23540909631297202, + "nauc_ndcg_at_1000_std": 0.056615205005261696, + "nauc_ndcg_at_100_diff1": 0.3574410299969898, + "nauc_ndcg_at_100_max": -0.2459841410933895, + "nauc_ndcg_at_100_std": -0.004941432018533064, + "nauc_ndcg_at_10_diff1": 0.27284914278868305, + "nauc_ndcg_at_10_max": -0.09860237927268649, + "nauc_ndcg_at_10_std": 0.17143394924414862, + "nauc_ndcg_at_1_diff1": 0.4553397252313951, + "nauc_ndcg_at_1_max": -0.00898601253476105, + "nauc_ndcg_at_1_std": 0.15209811978583004, + "nauc_ndcg_at_20_diff1": 0.24219624937954304, + "nauc_ndcg_at_20_max": -0.09686643301083671, + "nauc_ndcg_at_20_std": 0.1612277453319146, + "nauc_ndcg_at_3_diff1": 0.32690150790533234, + "nauc_ndcg_at_3_max": -0.047784701430556326, + "nauc_ndcg_at_3_std": 0.16012561788746316, + "nauc_ndcg_at_5_diff1": 0.31332748247111847, + "nauc_ndcg_at_5_max": -0.10699011157518731, + "nauc_ndcg_at_5_std": 0.15645669379446075, + "nauc_precision_at_1000_diff1": 0.0819429686636851, + "nauc_precision_at_1000_max": -0.1290271513175996, + "nauc_precision_at_1000_std": 0.2830467579249119, + "nauc_precision_at_100_diff1": 0.1086615495942521, + "nauc_precision_at_100_max": -0.22937147878224026, + "nauc_precision_at_100_std": 0.24534165503695654, + "nauc_precision_at_10_diff1": 0.23400746254303456, + "nauc_precision_at_10_max": -0.12882503413921087, + "nauc_precision_at_10_std": 0.24371550016348392, + "nauc_precision_at_1_diff1": 0.4553397252313951, + "nauc_precision_at_1_max": -0.00898601253476105, + "nauc_precision_at_1_std": 0.15209811978583004, + "nauc_precision_at_20_diff1": 0.12509716785790417, + "nauc_precision_at_20_max": -0.08597893607214858, + "nauc_precision_at_20_std": 0.3218130662392427, + "nauc_precision_at_3_diff1": 0.2960017153008923, + "nauc_precision_at_3_max": -0.05259235057482985, + "nauc_precision_at_3_std": 0.18596720508055797, + "nauc_precision_at_5_diff1": 0.2793466731531475, + "nauc_precision_at_5_max": -0.14856965124532454, + "nauc_precision_at_5_std": 0.18446594198853172, + "nauc_recall_at_1000_diff1": 0.23767465676853808, + "nauc_recall_at_1000_max": -0.21102175240994991, + "nauc_recall_at_1000_std": -0.1566324123319451, + "nauc_recall_at_100_diff1": 0.3128178199504678, + "nauc_recall_at_100_max": -0.2698286972760149, + "nauc_recall_at_100_std": -0.06876008993571361, + "nauc_recall_at_10_diff1": 0.19422816254098801, + "nauc_recall_at_10_max": -0.13751003581798485, + "nauc_recall_at_10_std": -0.07386552580291213, + "nauc_recall_at_1_diff1": 0.4868843030130222, + "nauc_recall_at_1_max": -0.15511847387665895, + "nauc_recall_at_1_std": -0.02716611321363879, + "nauc_recall_at_20_diff1": 0.19903944694380463, + "nauc_recall_at_20_max": -0.07262351965705542, + "nauc_recall_at_20_std": -0.04461273334236608, + "nauc_recall_at_3_diff1": 0.13345389226212073, + "nauc_recall_at_3_max": -0.1026168555200003, + "nauc_recall_at_3_std": -0.13519897599044758, + "nauc_recall_at_5_diff1": 0.16489673880783298, + "nauc_recall_at_5_max": -0.13503944945681817, + "nauc_recall_at_5_std": -0.13677876300364675, + "ndcg_at_1": 0.25714, + "ndcg_at_10": 0.19342, + "ndcg_at_100": 0.19552, + "ndcg_at_1000": 0.32441, + "ndcg_at_20": 0.17295, + "ndcg_at_3": 0.23726, + "ndcg_at_5": 0.21753, + "precision_at_1": 0.25714, + "precision_at_10": 0.15429, + "precision_at_100": 0.05786, + "precision_at_1000": 0.01964, + "precision_at_20": 0.10571, + "precision_at_3": 0.21905, + "precision_at_5": 0.19143, + "recall_at_1": 0.01148, + "recall_at_10": 0.10105, + "recall_at_100": 0.26388, + "recall_at_1000": 0.62162, + "recall_at_20": 0.11767, + "recall_at_3": 0.06125, + "recall_at_5": 0.07694 + } + ] + }, + "task_name": "ROxfordHardI2IMultiChoice" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IMultiChoice.json new file mode 100644 index 0000000000..2579f6ef68 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "83bd440268e200a4f60313070618e3f45000fa94", + "evaluation_time": 14.132107257843018, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.04963, + "cv_recall_at_1": 0.77143, + "cv_recall_at_10": 0.87143, + "cv_recall_at_100": 0.94286, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.88571, + "cv_recall_at_3": 0.82857, + "cv_recall_at_5": 0.87143, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.04963, + "map_at_1": 0.04963, + "map_at_10": 0.15207, + "map_at_100": 0.2724, + "map_at_1000": 0.34306, + "map_at_20": 0.19547, + "map_at_3": 0.087, + "map_at_5": 0.11245, + "mrr_at_1": 0.7714285714285715, + "mrr_at_10": 0.81, + "mrr_at_100": 0.8126116140632085, + "mrr_at_1000": 0.8129819154176776, + "mrr_at_20": 0.8108928571428572, + "mrr_at_3": 0.8, + "mrr_at_5": 0.81, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.22934173669467725, + "nauc_cv_recall_at_100_max": 0.3862044817927167, + "nauc_cv_recall_at_100_std": 0.09862278244631179, + "nauc_cv_recall_at_10_diff1": 0.40127150336574263, + "nauc_cv_recall_at_10_max": 0.5091355914093374, + "nauc_cv_recall_at_10_std": -0.04840260711614463, + "nauc_cv_recall_at_1_diff1": 0.33182314761262127, + "nauc_cv_recall_at_1_max": 0.6395797185270868, + "nauc_cv_recall_at_1_std": 0.17595270226849136, + "nauc_cv_recall_at_20_diff1": 0.40009492168960564, + "nauc_cv_recall_at_20_max": 0.4383009017560512, + "nauc_cv_recall_at_20_std": -0.0693521594684388, + "nauc_cv_recall_at_3_diff1": 0.1894338380513495, + "nauc_cv_recall_at_3_max": 0.6075543120473995, + "nauc_cv_recall_at_3_std": 0.12532916392363366, + "nauc_cv_recall_at_5_diff1": 0.40127150336574263, + "nauc_cv_recall_at_5_max": 0.5091355914093374, + "nauc_cv_recall_at_5_std": -0.04840260711614463, + "nauc_map_at_1000_diff1": -0.13224573449005503, + "nauc_map_at_1000_max": 0.4227560714777927, + "nauc_map_at_1000_std": 0.31351087144984363, + "nauc_map_at_100_diff1": -0.02836497538559071, + "nauc_map_at_100_max": 0.19639808649260623, + "nauc_map_at_100_std": 0.04954005369447823, + "nauc_map_at_10_diff1": 0.25121232316392567, + "nauc_map_at_10_max": -0.15556227534649655, + "nauc_map_at_10_std": -0.2488378494247982, + "nauc_map_at_1_diff1": 0.272345967565727, + "nauc_map_at_1_max": -0.2643088054590121, + "nauc_map_at_1_std": -0.27101749212632714, + "nauc_map_at_20_diff1": 0.13842997076502075, + "nauc_map_at_20_max": -0.06672772445238266, + "nauc_map_at_20_std": -0.19809071582803736, + "nauc_map_at_3_diff1": 0.3224142774768474, + "nauc_map_at_3_max": -0.2326757410552881, + "nauc_map_at_3_std": -0.2675429832114887, + "nauc_map_at_5_diff1": 0.3167741526994269, + "nauc_map_at_5_max": -0.22646890969571978, + "nauc_map_at_5_std": -0.26391954191876693, + "nauc_mrr_at_1000_diff1": 0.3082925432291351, + "nauc_mrr_at_1000_max": 0.6087298059076731, + "nauc_mrr_at_1000_std": 0.128068145081543, + "nauc_mrr_at_100_diff1": 0.30825889635587544, + "nauc_mrr_at_100_max": 0.6080934109778549, + "nauc_mrr_at_100_std": 0.12852823805463925, + "nauc_mrr_at_10_diff1": 0.31165182357930454, + "nauc_mrr_at_10_max": 0.6097725826972008, + "nauc_mrr_at_10_std": 0.12443013146734498, + "nauc_mrr_at_1_diff1": 0.33182314761262127, + "nauc_mrr_at_1_max": 0.6395797185270868, + "nauc_mrr_at_1_std": 0.17595270226849136, + "nauc_mrr_at_20_diff1": 0.31120802031702144, + "nauc_mrr_at_20_max": 0.6073798478505396, + "nauc_mrr_at_20_std": 0.12435570379822256, + "nauc_mrr_at_3_diff1": 0.26938620863854473, + "nauc_mrr_at_3_max": 0.6255367517049751, + "nauc_mrr_at_3_std": 0.1537545556237145, + "nauc_mrr_at_5_diff1": 0.31165182357930454, + "nauc_mrr_at_5_max": 0.6097725826972008, + "nauc_mrr_at_5_std": 0.12443013146734498, + "nauc_ndcg_at_1000_diff1": 0.004789789538060326, + "nauc_ndcg_at_1000_max": 0.4304267036999359, + "nauc_ndcg_at_1000_std": 0.20516350588416282, + "nauc_ndcg_at_100_diff1": -0.0985051399330637, + "nauc_ndcg_at_100_max": 0.4047840878665377, + "nauc_ndcg_at_100_std": 0.3189929050444598, + "nauc_ndcg_at_10_diff1": -0.12943461724680122, + "nauc_ndcg_at_10_max": 0.5959856882638876, + "nauc_ndcg_at_10_std": 0.3322657775788708, + "nauc_ndcg_at_1_diff1": 0.33182314761262127, + "nauc_ndcg_at_1_max": 0.6395797185270868, + "nauc_ndcg_at_1_std": 0.17595270226849136, + "nauc_ndcg_at_20_diff1": -0.15227287405763856, + "nauc_ndcg_at_20_max": 0.577403391414749, + "nauc_ndcg_at_20_std": 0.36970889043336586, + "nauc_ndcg_at_3_diff1": 0.022411758177576204, + "nauc_ndcg_at_3_max": 0.575626107585026, + "nauc_ndcg_at_3_std": 0.24755287374915227, + "nauc_ndcg_at_5_diff1": -0.034454400308166065, + "nauc_ndcg_at_5_max": 0.5655036501426706, + "nauc_ndcg_at_5_std": 0.280337402240637, + "nauc_precision_at_1000_diff1": -0.1979455862208862, + "nauc_precision_at_1000_max": 0.49452629750592836, + "nauc_precision_at_1000_std": 0.5170286173407959, + "nauc_precision_at_100_diff1": -0.31426081091366387, + "nauc_precision_at_100_max": 0.6695443542749885, + "nauc_precision_at_100_std": 0.6261360571210748, + "nauc_precision_at_10_diff1": -0.2527118526563905, + "nauc_precision_at_10_max": 0.6263582372380011, + "nauc_precision_at_10_std": 0.3992628197721442, + "nauc_precision_at_1_diff1": 0.33182314761262127, + "nauc_precision_at_1_max": 0.6395797185270868, + "nauc_precision_at_1_std": 0.17595270226849136, + "nauc_precision_at_20_diff1": -0.31590964166066615, + "nauc_precision_at_20_max": 0.645482829549644, + "nauc_precision_at_20_std": 0.4668678815358941, + "nauc_precision_at_3_diff1": -0.05580043594090598, + "nauc_precision_at_3_max": 0.5759748123032211, + "nauc_precision_at_3_std": 0.2806167756518931, + "nauc_precision_at_5_diff1": -0.14313081953662518, + "nauc_precision_at_5_max": 0.5793499136946008, + "nauc_precision_at_5_std": 0.3373884715435296, + "nauc_recall_at_1000_diff1": 0.07995145169939738, + "nauc_recall_at_1000_max": 0.1842438166265837, + "nauc_recall_at_1000_std": 0.043545956333912506, + "nauc_recall_at_100_diff1": 0.03299050541578269, + "nauc_recall_at_100_max": -0.11358015050633144, + "nauc_recall_at_100_std": -0.03558993365235519, + "nauc_recall_at_10_diff1": 0.31258822262912317, + "nauc_recall_at_10_max": -0.21448543830771524, + "nauc_recall_at_10_std": -0.2702885296246692, + "nauc_recall_at_1_diff1": 0.272345967565727, + "nauc_recall_at_1_max": -0.2643088054590121, + "nauc_recall_at_1_std": -0.27101749212632714, + "nauc_recall_at_20_diff1": 0.20536080239790513, + "nauc_recall_at_20_max": -0.1828385547433498, + "nauc_recall_at_20_std": -0.23751709286812595, + "nauc_recall_at_3_diff1": 0.3339569204499953, + "nauc_recall_at_3_max": -0.23145861780156063, + "nauc_recall_at_3_std": -0.26721917195809763, + "nauc_recall_at_5_diff1": 0.3584675101320892, + "nauc_recall_at_5_max": -0.2639487514014815, + "nauc_recall_at_5_std": -0.2733945840450043, + "ndcg_at_1": 0.77143, + "ndcg_at_10": 0.60443, + "ndcg_at_100": 0.49539, + "ndcg_at_1000": 0.60683, + "ndcg_at_20": 0.54202, + "ndcg_at_3": 0.67149, + "ndcg_at_5": 0.65383, + "precision_at_1": 0.77143, + "precision_at_10": 0.53286, + "precision_at_100": 0.229, + "precision_at_1000": 0.05983, + "precision_at_20": 0.435, + "precision_at_3": 0.62857, + "precision_at_5": 0.6, + "recall_at_1": 0.04963, + "recall_at_10": 0.18302, + "recall_at_100": 0.4477, + "recall_at_1000": 0.76971, + "recall_at_20": 0.25156, + "recall_at_3": 0.09291, + "recall_at_5": 0.12898 + } + ] + }, + "task_name": "ROxfordMediumI2IMultiChoice" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IMultiChoice.json new file mode 100644 index 0000000000..f13cd60fba --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "db94b5afd0014ab8c978f20a0fbcc52da1612a08", + "evaluation_time": 18.75471258163452, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.02814, + "cv_recall_at_1": 0.97143, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.02814, + "map_at_1": 0.02814, + "map_at_10": 0.14604, + "map_at_100": 0.613, + "map_at_1000": 0.71993, + "map_at_20": 0.24425, + "map_at_3": 0.06669, + "map_at_5": 0.09219, + "mrr_at_1": 0.9714285714285714, + "mrr_at_10": 0.9857142857142858, + "mrr_at_100": 0.9857142857142858, + "mrr_at_1000": 0.9857142857142858, + "mrr_at_20": 0.9857142857142858, + "mrr_at_3": 0.9857142857142858, + "mrr_at_5": 0.9857142857142858, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": 0.15289449112978365, + "nauc_cv_recall_at_1_max": 0.4960317460317484, + "nauc_cv_recall_at_1_std": 0.8692810457516342, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_map_at_1000_diff1": -0.5162392334753443, + "nauc_map_at_1000_max": 0.4031740557038615, + "nauc_map_at_1000_std": 0.41563960286590534, + "nauc_map_at_100_diff1": -0.23225006961960362, + "nauc_map_at_100_max": 0.3085695443992811, + "nauc_map_at_100_std": 0.12110491855395035, + "nauc_map_at_10_diff1": 0.5381554521493018, + "nauc_map_at_10_max": -0.10493092334419908, + "nauc_map_at_10_std": -0.2680755265314605, + "nauc_map_at_1_diff1": 0.8594916968198884, + "nauc_map_at_1_max": -0.24256237866427466, + "nauc_map_at_1_std": -0.4444380497243242, + "nauc_map_at_20_diff1": 0.3590188805543108, + "nauc_map_at_20_max": -0.013558846615686978, + "nauc_map_at_20_std": -0.16797583975226438, + "nauc_map_at_3_diff1": 0.7548580604028502, + "nauc_map_at_3_max": -0.2047628239453275, + "nauc_map_at_3_std": -0.4210073714397216, + "nauc_map_at_5_diff1": 0.6593773157320483, + "nauc_map_at_5_max": -0.18318576348363613, + "nauc_map_at_5_std": -0.3667442070438189, + "nauc_mrr_at_1000_diff1": 0.15289449112977452, + "nauc_mrr_at_1000_max": 0.49603174603173467, + "nauc_mrr_at_1000_std": 0.8692810457516342, + "nauc_mrr_at_100_diff1": 0.15289449112977452, + "nauc_mrr_at_100_max": 0.49603174603173467, + "nauc_mrr_at_100_std": 0.8692810457516342, + "nauc_mrr_at_10_diff1": 0.15289449112977452, + "nauc_mrr_at_10_max": 0.49603174603173467, + "nauc_mrr_at_10_std": 0.8692810457516342, + "nauc_mrr_at_1_diff1": 0.15289449112978365, + "nauc_mrr_at_1_max": 0.4960317460317484, + "nauc_mrr_at_1_std": 0.8692810457516342, + "nauc_mrr_at_20_diff1": 0.15289449112977452, + "nauc_mrr_at_20_max": 0.49603174603173467, + "nauc_mrr_at_20_std": 0.8692810457516342, + "nauc_mrr_at_3_diff1": 0.15289449112977452, + "nauc_mrr_at_3_max": 0.49603174603173467, + "nauc_mrr_at_3_std": 0.8692810457516342, + "nauc_mrr_at_5_diff1": 0.15289449112977452, + "nauc_mrr_at_5_max": 0.49603174603173467, + "nauc_mrr_at_5_std": 0.8692810457516342, + "nauc_ndcg_at_1000_diff1": -0.6430087042511464, + "nauc_ndcg_at_1000_max": 0.42655378925643106, + "nauc_ndcg_at_1000_std": 0.5044627676548901, + "nauc_ndcg_at_100_diff1": -0.5428592210544468, + "nauc_ndcg_at_100_max": 0.446739791744679, + "nauc_ndcg_at_100_std": 0.465672473136578, + "nauc_ndcg_at_10_diff1": -0.9208561880491332, + "nauc_ndcg_at_10_max": 0.5967384668371186, + "nauc_ndcg_at_10_std": 0.7022870489893225, + "nauc_ndcg_at_1_diff1": 0.15289449112978365, + "nauc_ndcg_at_1_max": 0.4960317460317484, + "nauc_ndcg_at_1_std": 0.8692810457516342, + "nauc_ndcg_at_20_diff1": -0.8251725302548746, + "nauc_ndcg_at_20_max": 0.5377716300435347, + "nauc_ndcg_at_20_std": 0.5566860440051336, + "nauc_ndcg_at_3_diff1": -0.9569729064943051, + "nauc_ndcg_at_3_max": 0.6190215986665834, + "nauc_ndcg_at_3_std": 0.9222091686879381, + "nauc_ndcg_at_5_diff1": -1.1226587238391097, + "nauc_ndcg_at_5_max": 0.5998868316343212, + "nauc_ndcg_at_5_std": 0.8858441072536013, + "nauc_precision_at_1000_diff1": -0.4777678980183267, + "nauc_precision_at_1000_max": 0.19411501983394858, + "nauc_precision_at_1000_std": 0.4317886018119784, + "nauc_precision_at_100_diff1": -0.6193565917597147, + "nauc_precision_at_100_max": 0.31610227934661556, + "nauc_precision_at_100_std": 0.41899007291320295, + "nauc_precision_at_10_diff1": -1.0942053142464947, + "nauc_precision_at_10_max": 0.57491910971664, + "nauc_precision_at_10_std": 0.7286596725169142, + "nauc_precision_at_1_diff1": 0.15289449112978365, + "nauc_precision_at_1_max": 0.4960317460317484, + "nauc_precision_at_1_std": 0.8692810457516342, + "nauc_precision_at_20_diff1": -1.006987444474548, + "nauc_precision_at_20_max": 0.5295046147956847, + "nauc_precision_at_20_std": 0.6468370068581858, + "nauc_precision_at_3_diff1": -1.188608776844072, + "nauc_precision_at_3_max": 0.6297333748314129, + "nauc_precision_at_3_std": 0.9273783587509071, + "nauc_precision_at_5_diff1": -1.304355295936774, + "nauc_precision_at_5_max": 0.5779142685336303, + "nauc_precision_at_5_std": 0.8799716519199388, + "nauc_recall_at_1000_diff1": -0.1671984561020124, + "nauc_recall_at_1000_max": 0.006203927434206601, + "nauc_recall_at_1000_std": 0.14154184625248917, + "nauc_recall_at_100_diff1": 0.07725532133003771, + "nauc_recall_at_100_max": 0.13911681142723956, + "nauc_recall_at_100_std": -0.13776713676284705, + "nauc_recall_at_10_diff1": 0.5586543206392892, + "nauc_recall_at_10_max": -0.1405893884569057, + "nauc_recall_at_10_std": -0.30419927459760987, + "nauc_recall_at_1_diff1": 0.8594916968198884, + "nauc_recall_at_1_max": -0.24256237866427466, + "nauc_recall_at_1_std": -0.4444380497243242, + "nauc_recall_at_20_diff1": 0.3955687201747704, + "nauc_recall_at_20_max": -0.084623109059799, + "nauc_recall_at_20_std": -0.24589251462208517, + "nauc_recall_at_3_diff1": 0.7237068204895556, + "nauc_recall_at_3_max": -0.21361841875189405, + "nauc_recall_at_3_std": -0.4269612949915014, + "nauc_recall_at_5_diff1": 0.6397655259145639, + "nauc_recall_at_5_max": -0.20024408587215523, + "nauc_recall_at_5_std": -0.38234727563640214, + "ndcg_at_1": 0.97143, + "ndcg_at_10": 0.92763, + "ndcg_at_100": 0.82979, + "ndcg_at_1000": 0.88963, + "ndcg_at_20": 0.90969, + "ndcg_at_3": 0.96255, + "ndcg_at_5": 0.95149, + "precision_at_1": 0.97143, + "precision_at_10": 0.89286, + "precision_at_100": 0.61286, + "precision_at_1000": 0.09256, + "precision_at_20": 0.85429, + "precision_at_3": 0.95714, + "precision_at_5": 0.93714, + "recall_at_1": 0.02814, + "recall_at_10": 0.15323, + "recall_at_100": 0.6921, + "recall_at_1000": 0.94803, + "recall_at_20": 0.26283, + "recall_at_3": 0.06802, + "recall_at_5": 0.09552 + } + ] + }, + "task_name": "RParisEasyI2IMultiChoice" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IMultiChoice.json new file mode 100644 index 0000000000..e6162419e8 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "4e5997e48fb2f2f8bf1c8973851dedeb17e09a83", + "evaluation_time": 18.11166262626648, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.00907, + "cv_recall_at_1": 0.81429, + "cv_recall_at_10": 0.95714, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.97143, + "cv_recall_at_3": 0.9, + "cv_recall_at_5": 0.9, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.00907, + "map_at_1": 0.00907, + "map_at_10": 0.06456, + "map_at_100": 0.23858, + "map_at_1000": 0.33338, + "map_at_20": 0.10802, + "map_at_3": 0.0239, + "map_at_5": 0.03746, + "mrr_at_1": 0.8142857142857143, + "mrr_at_10": 0.8544047619047621, + "mrr_at_100": 0.8560692239858908, + "mrr_at_1000": 0.8560692239858908, + "mrr_at_20": 0.8555952380952383, + "mrr_at_3": 0.8476190476190477, + "mrr_at_5": 0.8476190476190477, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": 0.12651727357610024, + "nauc_cv_recall_at_10_max": 0.742452536570181, + "nauc_cv_recall_at_10_std": 0.5659819483348865, + "nauc_cv_recall_at_1_diff1": 0.07804020484598333, + "nauc_cv_recall_at_1_max": 0.60700909577314, + "nauc_cv_recall_at_1_std": 0.041771764885728854, + "nauc_cv_recall_at_20_diff1": -0.171335200746965, + "nauc_cv_recall_at_20_max": 0.6790382819794577, + "nauc_cv_recall_at_20_std": 0.3489729225023323, + "nauc_cv_recall_at_3_diff1": 0.12058156595971639, + "nauc_cv_recall_at_3_max": 0.6089102307589694, + "nauc_cv_recall_at_3_std": 0.36261171135120646, + "nauc_cv_recall_at_5_diff1": 0.12058156595971639, + "nauc_cv_recall_at_5_max": 0.6089102307589694, + "nauc_cv_recall_at_5_std": 0.36261171135120646, + "nauc_map_at_1000_diff1": -0.14730057025035195, + "nauc_map_at_1000_max": 0.35443918991459294, + "nauc_map_at_1000_std": 0.2592666345486196, + "nauc_map_at_100_diff1": -0.02007929644055961, + "nauc_map_at_100_max": 0.12631888830781082, + "nauc_map_at_100_std": -0.04309297943451873, + "nauc_map_at_10_diff1": 0.316050298469623, + "nauc_map_at_10_max": -0.13310958861916383, + "nauc_map_at_10_std": -0.3020901743993513, + "nauc_map_at_1_diff1": 0.5378248883416507, + "nauc_map_at_1_max": -0.13238109598299855, + "nauc_map_at_1_std": -0.2002514420418722, + "nauc_map_at_20_diff1": 0.21121038948324744, + "nauc_map_at_20_max": -0.06818178937655603, + "nauc_map_at_20_std": -0.2902145910447691, + "nauc_map_at_3_diff1": 0.45725171311121, + "nauc_map_at_3_max": -0.19287806549614692, + "nauc_map_at_3_std": -0.25735181540630836, + "nauc_map_at_5_diff1": 0.42067337504189933, + "nauc_map_at_5_max": -0.18761920119844047, + "nauc_map_at_5_std": -0.2766021627006926, + "nauc_mrr_at_1000_diff1": 0.11587681189811185, + "nauc_mrr_at_1000_max": 0.6097767662452036, + "nauc_mrr_at_1000_std": 0.08966689534414451, + "nauc_mrr_at_100_diff1": 0.11587681189811185, + "nauc_mrr_at_100_max": 0.6097767662452036, + "nauc_mrr_at_100_std": 0.08966689534414451, + "nauc_mrr_at_10_diff1": 0.12009817077420695, + "nauc_mrr_at_10_max": 0.6119594659009971, + "nauc_mrr_at_10_std": 0.09790880388517216, + "nauc_mrr_at_1_diff1": 0.07804020484598333, + "nauc_mrr_at_1_max": 0.60700909577314, + "nauc_mrr_at_1_std": 0.041771764885728854, + "nauc_mrr_at_20_diff1": 0.11488715765282757, + "nauc_mrr_at_20_max": 0.6097325059817074, + "nauc_mrr_at_20_std": 0.09010175972487022, + "nauc_mrr_at_3_diff1": 0.11806489062690863, + "nauc_mrr_at_3_max": 0.6070359281437123, + "nauc_mrr_at_3_std": 0.101521446902114, + "nauc_mrr_at_5_diff1": 0.11806489062690863, + "nauc_mrr_at_5_max": 0.6070359281437123, + "nauc_mrr_at_5_std": 0.101521446902114, + "nauc_ndcg_at_1000_diff1": 0.025969018525589978, + "nauc_ndcg_at_1000_max": 0.36217088530709435, + "nauc_ndcg_at_1000_std": 0.25309337496879, + "nauc_ndcg_at_100_diff1": -0.17968384395623355, + "nauc_ndcg_at_100_max": 0.4239707948301639, + "nauc_ndcg_at_100_std": 0.3667281666093518, + "nauc_ndcg_at_10_diff1": -0.2397422113161351, + "nauc_ndcg_at_10_max": 0.3812994195515857, + "nauc_ndcg_at_10_std": 0.04827273248313771, + "nauc_ndcg_at_1_diff1": 0.07804020484598333, + "nauc_ndcg_at_1_max": 0.60700909577314, + "nauc_ndcg_at_1_std": 0.041771764885728854, + "nauc_ndcg_at_20_diff1": -0.28132858513315473, + "nauc_ndcg_at_20_max": 0.4204685927011596, + "nauc_ndcg_at_20_std": 0.10721091043225721, + "nauc_ndcg_at_3_diff1": -0.09344111682157266, + "nauc_ndcg_at_3_max": 0.379692511012279, + "nauc_ndcg_at_3_std": 0.04006405355649845, + "nauc_ndcg_at_5_diff1": -0.12981305098158163, + "nauc_ndcg_at_5_max": 0.3878036317455262, + "nauc_ndcg_at_5_std": 0.028676296191065903, + "nauc_precision_at_1000_diff1": -0.42027963870829504, + "nauc_precision_at_1000_max": 0.5131562832922288, + "nauc_precision_at_1000_std": 0.6312618988329652, + "nauc_precision_at_100_diff1": -0.3568206317095141, + "nauc_precision_at_100_max": 0.5107076773140476, + "nauc_precision_at_100_std": 0.4725107993998001, + "nauc_precision_at_10_diff1": -0.2953003666954273, + "nauc_precision_at_10_max": 0.361267795513373, + "nauc_precision_at_10_std": 0.055085202761000866, + "nauc_precision_at_1_diff1": 0.07804020484598333, + "nauc_precision_at_1_max": 0.60700909577314, + "nauc_precision_at_1_std": 0.041771764885728854, + "nauc_precision_at_20_diff1": -0.3210375121807198, + "nauc_precision_at_20_max": 0.42343158585680635, + "nauc_precision_at_20_std": 0.13184322004366664, + "nauc_precision_at_3_diff1": -0.15055311012827152, + "nauc_precision_at_3_max": 0.3240710586307241, + "nauc_precision_at_3_std": 0.04530985723140268, + "nauc_precision_at_5_diff1": -0.1735214934469398, + "nauc_precision_at_5_max": 0.3602496967545344, + "nauc_precision_at_5_std": 0.026229993195467694, + "nauc_recall_at_1000_diff1": 0.29022911927159833, + "nauc_recall_at_1000_max": 0.2125654628634255, + "nauc_recall_at_1000_std": 0.28260953057103566, + "nauc_recall_at_100_diff1": 0.19480792831160149, + "nauc_recall_at_100_max": -0.003561859262334835, + "nauc_recall_at_100_std": -0.12640776822732094, + "nauc_recall_at_10_diff1": 0.35131981594744904, + "nauc_recall_at_10_max": -0.17286224306644377, + "nauc_recall_at_10_std": -0.3177220315856104, + "nauc_recall_at_1_diff1": 0.5378248883416507, + "nauc_recall_at_1_max": -0.13238109598299855, + "nauc_recall_at_1_std": -0.2002514420418722, + "nauc_recall_at_20_diff1": 0.276802747494477, + "nauc_recall_at_20_max": -0.13421078554955512, + "nauc_recall_at_20_std": -0.3214777307000227, + "nauc_recall_at_3_diff1": 0.4639750095588438, + "nauc_recall_at_3_max": -0.2163484225516375, + "nauc_recall_at_3_std": -0.2587991550499656, + "nauc_recall_at_5_diff1": 0.45052599994510506, + "nauc_recall_at_5_max": -0.20428456806687892, + "nauc_recall_at_5_std": -0.28400516448621627, + "ndcg_at_1": 0.81429, + "ndcg_at_10": 0.70636, + "ndcg_at_100": 0.50703, + "ndcg_at_1000": 0.62521, + "ndcg_at_20": 0.6597, + "ndcg_at_3": 0.76457, + "ndcg_at_5": 0.74394, + "precision_at_1": 0.81429, + "precision_at_10": 0.68143, + "precision_at_100": 0.39386, + "precision_at_1000": 0.10154, + "precision_at_20": 0.62643, + "precision_at_3": 0.75238, + "precision_at_5": 0.72857, + "recall_at_1": 0.00907, + "recall_at_10": 0.06983, + "recall_at_100": 0.33142, + "recall_at_1000": 0.70794, + "recall_at_20": 0.12346, + "recall_at_3": 0.02464, + "recall_at_5": 0.03943 + } + ] + }, + "task_name": "RParisHardI2IMultiChoice" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IMultiChoice.json new file mode 100644 index 0000000000..229344b315 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "372c79fc823e1cebc1d55f8e0039aa239285e177", + "evaluation_time": 17.42084789276123, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.00556, + "cv_recall_at_1": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.00556, + "map_at_1": 0.00556, + "map_at_10": 0.05258, + "map_at_100": 0.35085, + "map_at_1000": 0.57431, + "map_at_20": 0.1003, + "map_at_3": 0.01627, + "map_at_5": 0.02709, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": NaN, + "nauc_cv_recall_at_1_max": NaN, + "nauc_cv_recall_at_1_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_map_at_1000_diff1": -0.6053563345663608, + "nauc_map_at_1000_max": 0.1905977136179189, + "nauc_map_at_1000_std": 0.32921704776067334, + "nauc_map_at_100_diff1": -0.22879583528362765, + "nauc_map_at_100_max": -0.021084881609830797, + "nauc_map_at_100_std": 0.040852018876228476, + "nauc_map_at_10_diff1": 0.07357334898390998, + "nauc_map_at_10_max": -0.07446530454222411, + "nauc_map_at_10_std": 0.030841288385805625, + "nauc_map_at_1_diff1": 0.17749829551090057, + "nauc_map_at_1_max": -0.1265328999786591, + "nauc_map_at_1_std": -0.04029967299581773, + "nauc_map_at_20_diff1": 0.031681429904680244, + "nauc_map_at_20_max": -0.04579935466000076, + "nauc_map_at_20_std": 0.0600822959918182, + "nauc_map_at_3_diff1": 0.1040211006608559, + "nauc_map_at_3_max": -0.10556625686545555, + "nauc_map_at_3_std": 0.004385201213601456, + "nauc_map_at_5_diff1": 0.10349991026936102, + "nauc_map_at_5_max": -0.10412867049469088, + "nauc_map_at_5_std": 0.0066619256709396935, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": -0.48417134785087246, + "nauc_ndcg_at_1000_max": 0.18405070377983077, + "nauc_ndcg_at_1000_std": 0.31559100185273364, + "nauc_ndcg_at_100_diff1": -0.7053306583324787, + "nauc_ndcg_at_100_max": 0.3110977036980108, + "nauc_ndcg_at_100_std": 0.30984291735802094, + "nauc_ndcg_at_10_diff1": -1.0871954957472245, + "nauc_ndcg_at_10_max": 0.5259679929438033, + "nauc_ndcg_at_10_std": 0.6817427956237951, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": -0.8880065469302778, + "nauc_ndcg_at_20_max": 0.4873208891288291, + "nauc_ndcg_at_20_std": 0.561373434731217, + "nauc_ndcg_at_3_diff1": -1.4590964058433729, + "nauc_ndcg_at_3_max": 0.49010909822533216, + "nauc_ndcg_at_3_std": 0.911235002385481, + "nauc_ndcg_at_5_diff1": -1.4171604522672352, + "nauc_ndcg_at_5_max": 0.5204958630957071, + "nauc_ndcg_at_5_std": 0.9186533100767078, + "nauc_precision_at_1000_diff1": -0.38234481360274825, + "nauc_precision_at_1000_max": 0.24781406894305885, + "nauc_precision_at_1000_std": 0.26403818342266816, + "nauc_precision_at_100_diff1": -0.6128971684961878, + "nauc_precision_at_100_max": 0.31904847150594623, + "nauc_precision_at_100_std": 0.24419530291834549, + "nauc_precision_at_10_diff1": -1.0307736255745676, + "nauc_precision_at_10_max": 0.5403562988547558, + "nauc_precision_at_10_std": 0.650531072272575, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": -0.836691736727432, + "nauc_precision_at_20_max": 0.4856534423121902, + "nauc_precision_at_20_std": 0.5336271166968242, + "nauc_precision_at_3_diff1": -1.4785247432306272, + "nauc_precision_at_3_max": 0.5099595393713005, + "nauc_precision_at_3_std": 0.912854030501091, + "nauc_precision_at_5_diff1": -1.4262371615312837, + "nauc_precision_at_5_max": 0.5442577030812298, + "nauc_precision_at_5_std": 0.9215686274509778, + "nauc_recall_at_1000_diff1": -0.25506002472395295, + "nauc_recall_at_1000_max": 0.10322366296428723, + "nauc_recall_at_1000_std": 0.2469529706691035, + "nauc_recall_at_100_diff1": -0.13201603693344977, + "nauc_recall_at_100_max": -0.0624401543533192, + "nauc_recall_at_100_std": -0.018309376923327982, + "nauc_recall_at_10_diff1": 0.09463286260272151, + "nauc_recall_at_10_max": -0.08130077043564286, + "nauc_recall_at_10_std": 0.017648859246839073, + "nauc_recall_at_1_diff1": 0.17749829551090057, + "nauc_recall_at_1_max": -0.1265328999786591, + "nauc_recall_at_1_std": -0.04029967299581773, + "nauc_recall_at_20_diff1": 0.05416706502986838, + "nauc_recall_at_20_max": -0.05925356554363863, + "nauc_recall_at_20_std": 0.04363713977519275, + "nauc_recall_at_3_diff1": 0.11034320172647584, + "nauc_recall_at_3_max": -0.1059689684621208, + "nauc_recall_at_3_std": -0.0006734851708733446, + "nauc_recall_at_5_diff1": 0.11748996520995737, + "nauc_recall_at_5_max": -0.10649893198419375, + "nauc_recall_at_5_std": -0.0026859671591820122, + "ndcg_at_1": 1.0, + "ndcg_at_10": 0.95692, + "ndcg_at_100": 0.81534, + "ndcg_at_1000": 0.7841, + "ndcg_at_20": 0.93009, + "ndcg_at_3": 0.97638, + "ndcg_at_5": 0.975, + "precision_at_1": 1.0, + "precision_at_10": 0.94714, + "precision_at_100": 0.76243, + "precision_at_1000": 0.18964, + "precision_at_20": 0.91357, + "precision_at_3": 0.97143, + "precision_at_5": 0.97143, + "recall_at_1": 0.00556, + "recall_at_10": 0.05313, + "recall_at_100": 0.38141, + "recall_at_1000": 0.79881, + "recall_at_20": 0.10199, + "recall_at_3": 0.01632, + "recall_at_5": 0.02723 + } + ] + }, + "task_name": "RParisMediumI2IMultiChoice" +} \ No newline at end of file From 0ae63dc9e90193620c28b6d296bcbfcb3600a5d9 Mon Sep 17 00:00:00 2001 From: Xin Zhang Date: Sat, 14 Dec 2024 23:12:42 +0800 Subject: [PATCH 118/154] [MIEB] Make multimodal models compatible to `task_name` and `prompt_type` (#1583) * 1. Make `get_xxx_embeddings` follow `encode`. 2. `ImageDataset.transform` could be `None`. * Apply suggestions from code review Co-authored-by: Kenneth Enevoldsen * Fix arguments * Try to fix tests --------- Co-authored-by: Kenneth Enevoldsen --- .../Image/Any2AnyRetrievalEvaluator.py | 42 +++++++++++++------ .../Image/ZeroshotClassificationEvaluator.py | 2 +- mteb/models/align_models.py | 24 +++++++++-- mteb/models/blip2_models.py | 24 +++++++++-- mteb/models/blip_models.py | 24 +++++++++-- mteb/models/clip_models.py | 24 +++++++++-- mteb/models/cohere_v.py | 24 +++++++++-- mteb/models/dino_models.py | 19 ++++++++- mteb/models/e5_v.py | 23 +++++++++- mteb/models/evaclip_models.py | 27 ++++++++++-- mteb/models/jina_clip.py | 15 ++++++- mteb/models/moco_models.py | 11 ++++- mteb/models/nomic_models_vision.py | 25 +++++++++-- mteb/models/openclip_models.py | 24 +++++++++-- mteb/models/siglip_models.py | 25 +++++++++-- mteb/models/vista_models.py | 33 +++++++++++++-- mteb/models/vlm2vec_models.py | 30 ++++++++++--- mteb/models/voyage_v.py | 36 +++++++++++++++- 18 files changed, 377 insertions(+), 55 deletions(-) diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index a321979d26..777e3b545f 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -17,7 +17,7 @@ from torch.utils.data import DataLoader from torchvision import transforms -from mteb.encoder_interface import Encoder +from mteb.encoder_interface import Encoder, PromptType from ..Evaluator import Evaluator from ..utils import ( @@ -36,7 +36,7 @@ logger = logging.getLogger(__name__) -transform = transforms.Compose([transforms.PILToTensor()]) +DEFAULT_TRANSFORM = transforms.Compose([transforms.PILToTensor()]) class ImageDataset(torch.utils.data.Dataset): @@ -57,7 +57,8 @@ def __getitem__(self, idx): image = image if image.mode != "RGB": image = image.convert("RGB") - image = self.transform(image) + if self.transform is not None: + image = self.transform(image) return image @@ -73,11 +74,13 @@ def __init__( encode_kwargs: dict[str, Any] = {}, corpus_chunk_size: int = 20000, previous_results: str | None = None, + transform=DEFAULT_TRANSFORM, **kwargs: Any, ): # Model is class that provides get_text_embeddings() and get_image_embeddings() self.model = model self.encode_kwargs = encode_kwargs + self.transform = transform if "batch_size" not in encode_kwargs: encode_kwargs["batch_size"] = 128 @@ -104,6 +107,7 @@ def search( queries: Dataset, # solve memoery issues top_k: int, score_function: str, + task_name: str, return_sorted: bool = False, **kwargs, ) -> dict[str, dict[str, float]]: @@ -121,11 +125,14 @@ def search( if q_modality == "text": query_texts = queries["text"] query_embeddings = self.model.get_text_embeddings( - texts=query_texts, batch_size=self.encode_kwargs["batch_size"] + texts=query_texts, + task_name=task_name, + prompt_type=PromptType.query, + **self.encode_kwargs, ) else: queries_dataset = ImageDataset( - queries, image_column_name="image", transform=transform + queries, image_column_name="image", transform=self.transform ) query_image_dataloader = DataLoader( queries_dataset, @@ -137,14 +144,18 @@ def search( if q_modality == "image": query_embeddings = self.model.get_image_embeddings( images=query_image_dataloader, - batch_size=self.encode_kwargs["batch_size"], + task_name=task_name, + prompt_type=PromptType.query, + **self.encode_kwargs, ) elif q_modality == "image,text": query_texts = queries["text"] query_embeddings = self.model.get_fused_embeddings( texts=query_texts, images=query_image_dataloader, - batch_size=self.encode_kwargs["batch_size"], + task_name=task_name, + prompt_type=PromptType.query, + **self.encode_kwargs, ) else: raise ValueError(f"Unsupported modality: {q_modality}") @@ -171,11 +182,14 @@ def search( if corpus_modality == "text": corpus_texts = chunk["text"] sub_corpus_embeddings = self.model.get_text_embeddings( - texts=corpus_texts, batch_size=self.encode_kwargs["batch_size"] + texts=corpus_texts, + task_name=task_name, + prompt_type=PromptType.passage, + **self.encode_kwargs, ) else: corpus_dataset = ImageDataset( - chunk, image_column_name="image", transform=transform + chunk, image_column_name="image", transform=self.transform ) corpus_image_dataloader = DataLoader( corpus_dataset, @@ -187,14 +201,18 @@ def search( if corpus_modality == "image": sub_corpus_embeddings = self.model.get_image_embeddings( images=corpus_image_dataloader, - batch_size=self.encode_kwargs["batch_size"], + task_name=task_name, + prompt_type=PromptType.passage, + **self.encode_kwargs, ) elif corpus_modality == "image,text": corpus_texts = chunk["text"] sub_corpus_embeddings = self.model.get_fused_embeddings( texts=corpus_texts, images=corpus_image_dataloader, - batch_size=self.encode_kwargs["batch_size"], + task_name=task_name, + prompt_type=PromptType.passage, + **self.encode_kwargs, ) else: raise ValueError(f"Unsupported modality: {corpus_modality}") @@ -292,7 +310,7 @@ def __call__( queries, self.top_k, self.score_function, - prompt_name=self.task_name, # type: ignore + task_name=self.task_name, ) @staticmethod diff --git a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py index 0b3b7f8f67..2ef9609ea4 100644 --- a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py @@ -84,6 +84,6 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): logger.info("Evaluating...") - accuracy = metrics.accuracy_score(self.labels, predictions) + accuracy = metrics.accuracy_score(self.labels, predictions.tolist()) return {"accuracy": accuracy} diff --git a/mteb/models/align_models.py b/mteb/models/align_models.py index e8f55c64ba..ca56449303 100644 --- a/mteb/models/align_models.py +++ b/mteb/models/align_models.py @@ -9,6 +9,7 @@ from tqdm import tqdm from transformers import AutoModel, AutoProcessor +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -24,7 +25,15 @@ def __init__( self.model = AutoModel.from_pretrained(model_name).to(self.device) self.processor = AutoProcessor.from_pretrained(model_name) - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): all_text_embeddings = [] with torch.no_grad(): @@ -41,7 +50,13 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): return all_text_embeddings def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, ): all_image_embeddings = [] if isinstance(images, DataLoader): @@ -86,8 +101,11 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") diff --git a/mteb/models/blip2_models.py b/mteb/models/blip2_models.py index 9f950ad525..32f4c1fc9b 100644 --- a/mteb/models/blip2_models.py +++ b/mteb/models/blip2_models.py @@ -9,6 +9,7 @@ from tqdm import tqdm from transformers import Blip2Processor +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -48,7 +49,15 @@ def preprocess( text=texts, images=images, return_tensors="pt", padding=True ) - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): all_text_embeddings = [] with torch.no_grad(): @@ -69,7 +78,13 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): return all_text_embeddings def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, ): all_image_embeddings = [] @@ -159,8 +174,11 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): # TODO: find out if BLIP has a prescribed way of fusing text and image embeddings if texts is None and images is None: diff --git a/mteb/models/blip_models.py b/mteb/models/blip_models.py index 7dff913fd4..0048192fea 100644 --- a/mteb/models/blip_models.py +++ b/mteb/models/blip_models.py @@ -10,6 +10,7 @@ from tqdm import tqdm from transformers import BlipForImageTextRetrieval, BlipProcessor +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -36,7 +37,15 @@ def preprocess( text=texts, images=images, return_tensors="pt", padding=True ) - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): all_text_embeddings = [] with torch.no_grad(): @@ -58,7 +67,13 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): return all_text_embeddings def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, ): all_image_embeddings = [] @@ -106,8 +121,11 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index 0270e5abe3..fc01b62266 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -9,6 +9,7 @@ from tqdm import tqdm from transformers import AutoModel, AutoProcessor +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -33,7 +34,15 @@ def preprocess( text=texts, images=images, return_tensors="pt", padding=True ) - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): all_text_embeddings = [] with torch.no_grad(): @@ -50,7 +59,13 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): return all_text_embeddings def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, ): all_image_embeddings = [] @@ -90,8 +105,11 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") diff --git a/mteb/models/cohere_v.py b/mteb/models/cohere_v.py index d53fd662e8..cc62292c1e 100644 --- a/mteb/models/cohere_v.py +++ b/mteb/models/cohere_v.py @@ -14,6 +14,7 @@ from tqdm import tqdm import mteb +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta api_key = os.getenv("COHERE_API_KEY") @@ -42,7 +43,15 @@ def __init__( Remove or adjust this after Cohere API changes capacity. """ - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): all_text_embeddings = [] for i in tqdm(range(0, len(texts), batch_size)): @@ -58,7 +67,13 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): return all_text_embeddings def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, ): all_image_embeddings = [] @@ -130,8 +145,11 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") diff --git a/mteb/models/dino_models.py b/mteb/models/dino_models.py index 06f9ad8dce..cf37bcdab9 100644 --- a/mteb/models/dino_models.py +++ b/mteb/models/dino_models.py @@ -9,6 +9,7 @@ from tqdm import tqdm from transformers import AutoImageProcessor, AutoModel +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -29,14 +30,25 @@ def __init__( self.processor = AutoImageProcessor.from_pretrained(model_name) @staticmethod - def get_text_embeddings(texts: list[str], batch_size: int = 32): + def get_text_embeddings( + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): raise ValueError("DINO models only support image encoding.") def get_image_embeddings( self, images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, pooling="cls", + **kwargs: Any, ): all_image_embeddings = [] @@ -85,8 +97,11 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("images must be provided for DINO models") diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index 4ead48a7cf..202dd7add6 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -9,6 +9,7 @@ from tqdm import tqdm from transformers import LlavaNextForConditionalGeneration, LlavaNextProcessor +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -42,7 +43,15 @@ def __init__( else: self.composed_prompt = self.template.format(composed_prompt) - def get_text_embeddings(self, texts: list[str], batch_size: int = 8): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 8, + **kwargs: Any, + ): all_text_embeddings = [] with torch.no_grad(): @@ -60,7 +69,13 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 8): return torch.cat(all_text_embeddings, dim=0) def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 8 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 8, + **kwargs: Any, ): all_image_embeddings = [] @@ -105,7 +120,11 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] = None, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 8, + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") diff --git a/mteb/models/evaclip_models.py b/mteb/models/evaclip_models.py index 2a98277d08..3987e755bb 100644 --- a/mteb/models/evaclip_models.py +++ b/mteb/models/evaclip_models.py @@ -8,6 +8,7 @@ from torch.utils.data import DataLoader from tqdm import tqdm +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -48,12 +49,22 @@ def encode( # type: ignore self, sentences: list[str], *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, **kwargs: Any, ): return self.get_text_embeddings(texts=sentences, batch_size=batch_size) - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): all_text_embeddings = [] with torch.no_grad(), torch.cuda.amp.autocast(): @@ -67,7 +78,13 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): return all_text_embeddings def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, ): all_image_embeddings = [] if isinstance(images, DataLoader): @@ -112,8 +129,12 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") diff --git a/mteb/models/jina_clip.py b/mteb/models/jina_clip.py index 01563a0649..78b08c5a57 100644 --- a/mteb/models/jina_clip.py +++ b/mteb/models/jina_clip.py @@ -9,6 +9,7 @@ from tqdm import tqdm from transformers import AutoModel +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -28,9 +29,13 @@ def __init__( def get_text_embeddings( self, texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, convert_to_numpy=False, convert_to_tensor=True, + **kwargs: Any, ): all_text_embeddings = [] @@ -50,9 +55,13 @@ def get_text_embeddings( def get_image_embeddings( self, images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, convert_to_numpy=False, convert_to_tensor=True, + **kwargs: Any, ): all_image_embeddings = [] @@ -92,8 +101,12 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] = None, - fusion_mode="sum", + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") diff --git a/mteb/models/moco_models.py b/mteb/models/moco_models.py index 18e1a63ad9..b8dc8ef0b7 100644 --- a/mteb/models/moco_models.py +++ b/mteb/models/moco_models.py @@ -8,6 +8,7 @@ from torch.utils.data import DataLoader from tqdm import tqdm +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -55,7 +56,11 @@ def get_text_embeddings(texts: list[str], batch_size: int = 32): def get_image_embeddings( self, images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + **kwargs: Any, ): all_image_embeddings = [] @@ -97,8 +102,12 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("images must be provided for MOCO models") diff --git a/mteb/models/nomic_models_vision.py b/mteb/models/nomic_models_vision.py index 6fbf478c51..66a1e4ba8e 100644 --- a/mteb/models/nomic_models_vision.py +++ b/mteb/models/nomic_models_vision.py @@ -10,6 +10,7 @@ from tqdm import tqdm from transformers import AutoImageProcessor, AutoModel, AutoTokenizer +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -45,7 +46,15 @@ def preprocess( text=texts, images=images, return_tensors="pt", padding=True ) - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): all_text_embeddings = [] with torch.no_grad(): @@ -78,7 +87,13 @@ def mean_pooling(self, model_output, attention_mask): ) def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, ): all_image_embeddings = [] @@ -113,8 +128,12 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") diff --git a/mteb/models/openclip_models.py b/mteb/models/openclip_models.py index 068789267c..df18b88561 100644 --- a/mteb/models/openclip_models.py +++ b/mteb/models/openclip_models.py @@ -8,6 +8,7 @@ from torch.utils.data import DataLoader from tqdm import tqdm +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -41,7 +42,14 @@ def encode( # type: ignore ): return self.get_text_embeddings(texts=sentences, batch_size=batch_size) - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + ): all_text_embeddings = [] with torch.no_grad(), torch.cuda.amp.autocast(): @@ -55,7 +63,13 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): return all_text_embeddings def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, ): all_image_embeddings = [] if isinstance(images, DataLoader): @@ -100,8 +114,12 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") diff --git a/mteb/models/siglip_models.py b/mteb/models/siglip_models.py index d60b8bdc30..c3dd3c757e 100644 --- a/mteb/models/siglip_models.py +++ b/mteb/models/siglip_models.py @@ -9,6 +9,7 @@ from tqdm import tqdm from transformers import AutoModel, AutoProcessor +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -33,7 +34,15 @@ def preprocess( text=texts, images=images, return_tensors="pt", padding=True ) - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): all_text_embeddings = [] with torch.no_grad(): @@ -53,7 +62,13 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): return all_text_embeddings def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, ): all_image_embeddings = [] @@ -108,8 +123,12 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py index c86fdcd5b6..32ec76b313 100644 --- a/mteb/models/vista_models.py +++ b/mteb/models/vista_models.py @@ -9,6 +9,7 @@ from torchvision import transforms from tqdm import tqdm +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta tensor_to_image = transforms.Compose([transforms.ToPILImage()]) @@ -94,7 +95,15 @@ def encode_text(self, texts): t_reps = torch.nn.functional.normalize(t_reps, dim=-1) return t_reps.contiguous() - def encode(self, images=None, texts=None, tensors=False): + def encode( + self, + images=None, + texts=None, + tensors=False, + task_name: str | None = None, + prompt_type: PromptType | None = None, + **kwargs: Any, + ): if images is not None: if isinstance(images, list): if not tensors: @@ -122,7 +131,15 @@ def encode(self, images=None, texts=None, tensors=False): else: return None - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): all_text_embeddings = [] for i in tqdm(range(0, len(texts), batch_size)): batch_texts = texts[i : i + batch_size] @@ -132,7 +149,13 @@ def get_text_embeddings(self, texts: list[str], batch_size: int = 32): return torch.cat(all_text_embeddings, dim=0) def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, ): all_image_embeddings = [] @@ -153,7 +176,11 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + **kwargs: Any, ): all_embeddings = [] diff --git a/mteb/models/vlm2vec_models.py b/mteb/models/vlm2vec_models.py index f48502eb0a..28b97c0a6c 100644 --- a/mteb/models/vlm2vec_models.py +++ b/mteb/models/vlm2vec_models.py @@ -10,6 +10,7 @@ from tqdm import tqdm from transformers import AutoConfig, AutoModelForCausalLM, AutoProcessor +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta logging.basicConfig(level=logging.WARNING) @@ -82,8 +83,9 @@ def encode( self, sentences: list[str], *, - prompt_name: str = None, - **kwargs: Any, # noqa + task_name: str | None = None, + prompt_type: PromptType | None = None, + **kwargs: Any, ): return self.get_text_embeddings(texts=sentences) @@ -109,7 +111,13 @@ def _pooling(self, last_hidden_state, attention_mask): # reference: https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/src/collator.py def get_image_embeddings( - self, images: list[Image.Image] | DataLoader, batch_size: int = 32 + self, + images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, ): text = "<|image_1|> Represent the given image." all_image_embeddings = [] @@ -191,7 +199,15 @@ def get_image_embeddings( all_image_embeddings = torch.cat(all_image_embeddings, dim=0) return all_image_embeddings - def get_text_embeddings(self, texts: list[str], batch_size: int = 32): + def get_text_embeddings( + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): all_text_embeddings = [] with torch.no_grad(): @@ -239,8 +255,12 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, + fusion_mode="sum", + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") diff --git a/mteb/models/voyage_v.py b/mteb/models/voyage_v.py index c1787ea10f..0b25cfddef 100644 --- a/mteb/models/voyage_v.py +++ b/mteb/models/voyage_v.py @@ -11,6 +11,7 @@ from tqdm import tqdm import mteb +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta api_key = os.getenv("VOYAGE_API_KEY") @@ -33,8 +34,21 @@ def __init__( self.vo = voyageai.Client() def get_text_embeddings( - self, texts: list[str], batch_size: int = 32, input_type=None + self, + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + input_type=None, + **kwargs: Any, ): + if input_type is None and prompt_type is not None: + if prompt_type == PromptType.passage: + input_type = "document" + elif prompt_type == PromptType.query: + input_type = "query" + all_text_embeddings = [] for i in tqdm(range(0, len(texts), batch_size)): @@ -51,9 +65,19 @@ def get_text_embeddings( def get_image_embeddings( self, images: list[Image.Image] | DataLoader, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, input_type=None, + **kwargs: Any, ): + if input_type is None and prompt_type is not None: + if prompt_type == PromptType.passage: + input_type = "document" + elif prompt_type == PromptType.query: + input_type = "query" + all_image_embeddings = [] if isinstance(images, DataLoader): @@ -93,12 +117,22 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, batch_size: int = 32, input_type=None, + **kwargs: Any, ): if texts is None and images is None: raise ValueError("Either texts or images must be provided") + if input_type is None and prompt_type is not None: + if prompt_type == PromptType.passage: + input_type = "document" + elif prompt_type == PromptType.query: + input_type = "query" + text_embeddings = None image_embeddings = None From 074e5d4320239b3c4168a176ba1ef09fd94494e6 Mon Sep 17 00:00:00 2001 From: Kenneth Enevoldsen Date: Sun, 15 Dec 2024 18:57:19 -0800 Subject: [PATCH 119/154] fix image encoder (#1596) * format * fixed tests * lint --- mteb/encoder_interface.py | 18 ++++++++---------- .../evaluators/RetrievalEvaluator.py | 6 ++++-- mteb/models/sentence_transformer_wrapper.py | 2 ++ mteb/tasks/Image/Clustering/eng/CIFAR.py | 1 + pyproject.toml | 1 + 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/mteb/encoder_interface.py b/mteb/encoder_interface.py index 7c69503ca2..e2725a3ac9 100644 --- a/mteb/encoder_interface.py +++ b/mteb/encoder_interface.py @@ -6,7 +6,6 @@ import numpy as np import torch - from PIL import Image from torch.utils.data import DataLoader @@ -182,30 +181,29 @@ def encode( # current a 1-1 match with Encoder.encode ) -> np.ndarray: pass - def get_image_embeddings( # Seems like sentence transformers use a singular encode for both images and text. Not sure if we want to do the same. + def get_image_embeddings( # Seems like sentence transformers use a singular encode for both images and text. Not sure if we want to do the same. # If not it might be ideal to redefine Encoder.encode self, images: list[Image.Image] | DataLoader, - *, **kwargs, # removed batch_size, it is not required that it will accept kwargs ) -> np.ndarray: # added standard output (I believe we actually expect tensors in the code, but would like to be consistent) pass - def get_text_embeddings( # any reason for this? + def get_text_embeddings( # any reason for this? self, texts: list[str], - *, - **kwargs, + **kwargs, ) -> np.ndarray: pass - def get_fused_embeddings( # hmm what if I have a document with images at specific positions? + def get_fused_embeddings( # hmm what if I have a document with images at specific positions? self, texts: list[str] | None = None, - images: list[Image.Image] | DataLoader | None = None, # the requirement for these two to be the same seems odd (docs without images, images without associated text, docs with multiple images) + images: list[Image.Image] + | DataLoader + | None = None, # the requirement for these two to be the same seems odd (docs without images, images without associated text, docs with multiple images) # fusion_mode: str="sum", # will remove this as it should be required in the interface - *, **kwargs: Any, ) -> np.ndarray: - pass + pass diff --git a/mteb/evaluation/evaluators/RetrievalEvaluator.py b/mteb/evaluation/evaluators/RetrievalEvaluator.py index f81647c3ad..23bbc280e9 100644 --- a/mteb/evaluation/evaluators/RetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/RetrievalEvaluator.py @@ -379,8 +379,9 @@ class DRESModel: mteb_model_meta: ModelMeta | None def __init__(self, model, **kwargs): - self.model = model + self.model: Any = model self.use_sbert_model = isinstance(model, SentenceTransformer) + self.device = model.device if hasattr(model, "device") else None self.save_corpus_embeddings = kwargs.get("save_corpus_embeddings", False) self.corpus_embeddings = {} @@ -418,9 +419,10 @@ def encode_corpus( def encode( self, sentences: list[str], + *, task_name: str, prompt_type: PromptType | None = None, - **kwargs, + **kwargs: Any, ): if prompt_type and prompt_type == PromptType.passage: return self.encode_corpus( diff --git a/mteb/models/sentence_transformer_wrapper.py b/mteb/models/sentence_transformer_wrapper.py index 13f274c186..809b7d50b2 100644 --- a/mteb/models/sentence_transformer_wrapper.py +++ b/mteb/models/sentence_transformer_wrapper.py @@ -39,8 +39,10 @@ def __init__( self.model = SentenceTransformer( model, revision=revision, trust_remote_code=True, **kwargs ) + self.device = self.model.device else: self.model = model + self.device = None if ( model_prompts is None diff --git a/mteb/tasks/Image/Clustering/eng/CIFAR.py b/mteb/tasks/Image/Clustering/eng/CIFAR.py index f9d08b684a..a10906d105 100644 --- a/mteb/tasks/Image/Clustering/eng/CIFAR.py +++ b/mteb/tasks/Image/Clustering/eng/CIFAR.py @@ -41,6 +41,7 @@ class CIFAR10Clustering(AbsTaskImageClustering): "avg_character_length": {"test": 431.4}, }, ) + image_column_name: str = "img" diff --git a/pyproject.toml b/pyproject.toml index 72f47523d7..088613b36a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ speedtask = ["GPUtil>=1.4.0", "psutil>=5.9.8"] peft = ["peft>=0.11.0"] leaderboard = ["gradio>=4.44.0", "gradio_rangeslider>=0.0.6"] flagembedding = ["FlagEmbedding"] +e5v = ["accelerate>=0.26.0"] [tool.coverage.report] From 74cb6e64a161a4c1c06db3a8333579b6fd999414 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Wed, 18 Dec 2024 05:00:56 +0800 Subject: [PATCH 120/154] [mieb] voyage-v: add exponential backoff and other error handling (#1610) * add voyage multimodal & ran 17 tasks * lint * typo * clean * exponential backoff tmp * downsize large images for voyage api call * voyage error handling * lint * add more results * make tenacity optional * lint * log --- mteb/models/voyage_v.py | 126 +- .../voyage-multimodal-3/1/AROCocoOrder.json | 21 + .../voyage-multimodal-3/1/AROFlickrOrder.json | 21 + .../1/AROVisualAttribution.json | 21 + .../1/AROVisualRelation.json | 21 + .../voyage-multimodal-3/1/CIFAR10.json | 48 + .../voyage-multimodal-3/1/CIFAR100.json | 48 + .../1/CIFAR100ZeroShot.json | 19 + .../1/CIFAR10ZeroShot.json | 19 + .../1/CIRRIT2IRetrieval.json | 186 + .../1/CLEVRCountZeroShot.json | 19 + .../voyage-multimodal-3/1/CLEVRZeroShot.json | 19 + .../1/CUB200I2IRetrieval.json | 186 + .../voyage-multimodal-3/1/Caltech101.json | 48 + .../1/Caltech101ZeroShot.json | 19 + .../voyage-multimodal-3/1/Country211.json | 48 + .../1/Country211ZeroShot.json | 19 + results-mieb/voyage-multimodal-3/1/DTD.json | 48 + .../voyage-multimodal-3/1/DTDZeroShot.json | 19 + .../voyage-multimodal-3/1/EuroSAT.json | 48 + .../1/EuroSATZeroShot.json | 19 + .../voyage-multimodal-3/1/FER2013.json | 48 + .../1/FER2013ZeroShot.json | 19 + .../voyage-multimodal-3/1/FGVCAircraft.json | 48 + .../1/FGVCAircraftZeroShot.json | 19 + .../1/Fashion200kI2TRetrieval.json | 186 + .../1/Flickr30kI2TRetrieval.json | 186 + .../1/Flickr30kT2IRetrieval.json | 186 + .../1/Food101Classification.json | 48 + .../1/Food101ZeroShot.json | 19 + results-mieb/voyage-multimodal-3/1/GTSRB.json | 48 + .../voyage-multimodal-3/1/GTSRBZeroShot.json | 19 + .../1/HatefulMemesI2TRetrieval.json | 186 + .../1/InfoSeekIT2ITRetrieval.json | 186 + .../1/InfoSeekIT2TRetrieval.json | 186 + results-mieb/voyage-multimodal-3/1/MNIST.json | 48 + .../voyage-multimodal-3/1/MNISTZeroShot.json | 19 + .../1/MSCOCOI2TRetrieval.json | 186 + .../1/MSCOCOT2IRetrieval.json | 186 + .../1/NIGHTSI2IRetrieval.json | 186 + .../1/OVENIT2TRetrieval.json | 186 + .../1/OxfordFlowersClassification.json | 48 + .../voyage-multimodal-3/1/OxfordPets.json | 48 + .../1/OxfordPetsZeroShot.json | 19 + .../voyage-multimodal-3/1/PatchCamelyon.json | 60 + .../1/PatchCamelyonZeroShot.json | 19 + .../voyage-multimodal-3/1/RESISC45.json | 48 + .../1/RESISC45ZeroShot.json | 19 + .../1/RP2kI2IRetrieval.json | 186 + .../voyage-multimodal-3/1/RenderedSST2.json | 19 + results-mieb/voyage-multimodal-3/1/STL10.json | 48 + .../voyage-multimodal-3/1/STL10ZeroShot.json | 19 + .../voyage-multimodal-3/1/SUN397.json | 48 + .../voyage-multimodal-3/1/SUN397ZeroShot.json | 19 + .../voyage-multimodal-3/1/StanfordCars.json | 48 + .../1/StanfordCarsI2IRetrieval.json | 186 + .../1/StanfordCarsZeroShot.json | 19 + .../voyage-multimodal-3/1/SugarCrepe.json | 21 + .../1/TUBerlinT2IRetrieval.json | 186 + .../1/VidoreArxivQARetrieval.json | 186 + .../1/VidoreDocVQARetrieval.json | 186 + .../1/VidoreInfoVQARetrieval.json | 186 + .../1/VidoreShiftProjectRetrieval.json | 186 + .../1/VidoreSyntheticDocQAAIRetrieval.json | 186 + .../VidoreSyntheticDocQAEnergyRetrieval.json | 186 + ...theticDocQAGovernmentReportsRetrieval.json | 186 + ...heticDocQAHealthcareIndustryRetrieval.json | 186 + .../1/VidoreTabfquadRetrieval.json | 186 + .../1/VidoreTatdqaRetrieval.json | 186 + .../1/VisualNewsI2TRetrieval.json | 186 + .../1/WITT2IRetrieval.json | 1936 +++++ .../1/WebQAT2ITRetrieval.json | 186 + .../1/WebQAT2TRetrieval.json | 186 + .../voyage-multimodal-3/1/Winoground.json | 21 + .../1/XM3600T2IRetrieval.json | 6311 +++++++++++++++++ 75 files changed, 14926 insertions(+), 37 deletions(-) create mode 100644 results-mieb/voyage-multimodal-3/1/AROCocoOrder.json create mode 100644 results-mieb/voyage-multimodal-3/1/AROFlickrOrder.json create mode 100644 results-mieb/voyage-multimodal-3/1/AROVisualAttribution.json create mode 100644 results-mieb/voyage-multimodal-3/1/AROVisualRelation.json create mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR10.json create mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR100.json create mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR100ZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR10ZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/CIRRIT2IRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/CLEVRCountZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/CLEVRZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/CUB200I2IRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/Caltech101.json create mode 100644 results-mieb/voyage-multimodal-3/1/Caltech101ZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/Country211.json create mode 100644 results-mieb/voyage-multimodal-3/1/Country211ZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/DTD.json create mode 100644 results-mieb/voyage-multimodal-3/1/DTDZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/EuroSAT.json create mode 100644 results-mieb/voyage-multimodal-3/1/EuroSATZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/FER2013.json create mode 100644 results-mieb/voyage-multimodal-3/1/FER2013ZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/FGVCAircraft.json create mode 100644 results-mieb/voyage-multimodal-3/1/FGVCAircraftZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/Fashion200kI2TRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/Flickr30kI2TRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/Flickr30kT2IRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/Food101Classification.json create mode 100644 results-mieb/voyage-multimodal-3/1/Food101ZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/GTSRB.json create mode 100644 results-mieb/voyage-multimodal-3/1/GTSRBZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/HatefulMemesI2TRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/InfoSeekIT2ITRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/InfoSeekIT2TRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/MNIST.json create mode 100644 results-mieb/voyage-multimodal-3/1/MNISTZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/MSCOCOI2TRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/MSCOCOT2IRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/NIGHTSI2IRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/OVENIT2TRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/OxfordFlowersClassification.json create mode 100644 results-mieb/voyage-multimodal-3/1/OxfordPets.json create mode 100644 results-mieb/voyage-multimodal-3/1/OxfordPetsZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/PatchCamelyon.json create mode 100644 results-mieb/voyage-multimodal-3/1/PatchCamelyonZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/RESISC45.json create mode 100644 results-mieb/voyage-multimodal-3/1/RESISC45ZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/RP2kI2IRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/RenderedSST2.json create mode 100644 results-mieb/voyage-multimodal-3/1/STL10.json create mode 100644 results-mieb/voyage-multimodal-3/1/STL10ZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/SUN397.json create mode 100644 results-mieb/voyage-multimodal-3/1/SUN397ZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/StanfordCars.json create mode 100644 results-mieb/voyage-multimodal-3/1/StanfordCarsI2IRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/StanfordCarsZeroShot.json create mode 100644 results-mieb/voyage-multimodal-3/1/SugarCrepe.json create mode 100644 results-mieb/voyage-multimodal-3/1/TUBerlinT2IRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/VidoreArxivQARetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/VidoreDocVQARetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/VidoreInfoVQARetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/VidoreShiftProjectRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAAIRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAEnergyRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAGovernmentReportsRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/VidoreTabfquadRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/VidoreTatdqaRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/VisualNewsI2TRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/WITT2IRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/WebQAT2ITRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/WebQAT2TRetrieval.json create mode 100644 results-mieb/voyage-multimodal-3/1/Winoground.json create mode 100644 results-mieb/voyage-multimodal-3/1/XM3600T2IRetrieval.json diff --git a/mteb/models/voyage_v.py b/mteb/models/voyage_v.py index 0b25cfddef..fabff4d6b6 100644 --- a/mteb/models/voyage_v.py +++ b/mteb/models/voyage_v.py @@ -4,6 +4,7 @@ from functools import partial from typing import Any +import logging import torch from PIL import Image from torch.utils.data import DataLoader @@ -11,6 +12,7 @@ from tqdm import tqdm import mteb +from mteb.model_meta import ModelMeta from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -18,11 +20,50 @@ tensor_to_image = transforms.Compose([transforms.ToPILImage()]) +def downsample_image( + image: Image.Image, max_pixels: int = 16000000, target_longest_side: int = 4000 +) -> Image.Image: + """ + if image pixel > max_pixels, downsample it to target_longest_side while keeping the width height ratio. + """ + width, height = image.size + pixels = width * height + + if pixels > max_pixels: + if width > height: + new_width = target_longest_side + new_height = int(height * (target_longest_side / width)) + else: + new_height = target_longest_side + new_width = int(width * (target_longest_side / height)) + + new_size = (new_width, new_height) + logging.info( + f"Downsampling image from {width}x{height} to {new_width}x{new_height}" + ) + return image.resize(new_size, Image.LANCZOS) + if width > height: + if width > 10000: + logging.error("Processing extremely wide images.") + return image.resize((10000, height), Image.LANCZOS) + else: + if height > 10000: + logging.error("Processing extremely high images.") + return image.resize((width, 10000), Image.LANCZOS) + return image + + def voyage_v_loader(**kwargs): try: import voyageai except ImportError: raise ImportError("To use voyage models, please run `pip install -U voyageai`.") + try: + from tenacity import retry, stop_after_attempt, wait_exponential + except ImportError: + raise ImportError( + "please run `pip install tenacity` to use exponential backoff." + ) class VoyageMultiModalModelWrapper: def __init__( @@ -33,6 +74,13 @@ def __init__( self.model_name = model_name self.vo = voyageai.Client() + @retry( + stop=stop_after_attempt(6), # Stop after 6 attempts + wait=wait_exponential(multiplier=1, max=60), # Exponential backoff + ) + def _multimodal_embed(self, inputs, model, input_type): + return self.vo.multimodal_embed(inputs, model=model, input_type=input_type) + def get_text_embeddings( self, texts: list[str], @@ -51,14 +99,17 @@ def get_text_embeddings( all_text_embeddings = [] + batch_size = 128 # for run tasks purpose + for i in tqdm(range(0, len(texts), batch_size)): batch_texts = texts[i : i + batch_size] batch_texts = [[text] for text in batch_texts] - all_text_embeddings += torch.tensor( - self.vo.multimodal_embed( - batch_texts, model=self.model_name, input_type=input_type - ).embeddings - ) + + # with retry mechanism + embeddings = self._multimodal_embed( + batch_texts, model=self.model_name, input_type=input_type + ).embeddings + all_text_embeddings.append(torch.tensor(embeddings)) all_text_embeddings = torch.vstack(all_text_embeddings) return all_text_embeddings @@ -84,21 +135,21 @@ def get_image_embeddings( for index, batch in enumerate(tqdm(images)): if index == 0: assert len(batch) == batch_size - batch_images = [[tensor_to_image(image)] for image in batch] - all_image_embeddings += torch.tensor( - self.vo.multimodal_embed( - batch_images, model=self.model_name, input_type=input_type - ).embeddings - ) + batch_images = [ + [downsample_image(tensor_to_image(image))] for image in batch + ] + embeddings = self._multimodal_embed( + batch_images, model=self.model_name, input_type=input_type + ).embeddings + all_image_embeddings.append(torch.tensor(embeddings)) else: for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] - batch_images = [[image] for image in batch_images] - all_image_embeddings += torch.tensor( - self.vo.multimodal_embed( - batch_images, model=self.model_name, input_type=input_type - ).embeddings - ) + batch_images = [[downsample_image(image)] for image in batch_images] + embeddings = self._multimodal_embed( + batch_images, model=self.model_name, input_type=input_type + ).embeddings + all_image_embeddings.append(torch.tensor(embeddings)) all_image_embeddings = torch.vstack(all_image_embeddings) return all_image_embeddings @@ -138,12 +189,13 @@ def get_fused_embeddings( interleaved_embeddings = [] if texts is not None and images is not None: - # print("encoding interleaved inputs") if isinstance(images, DataLoader): for index, batch in tqdm(enumerate(images)): if index == 0: assert len(batch) == batch_size - batch_images = [tensor_to_image(image) for image in batch] + batch_images = [ + downsample_image(tensor_to_image(image)) for image in batch + ] batch_texts = texts[ index * batch_size : (index + 1) * batch_size ] @@ -151,13 +203,12 @@ def get_fused_embeddings( [text, image] for image, text in zip(batch_images, batch_texts) ] - interleaved_embeddings += torch.tensor( - self.vo.multimodal_embed( - interleaved_inputs, - model=self.model_name, - input_type=input_type, - ).embeddings - ) + embeddings = self._multimodal_embed( + interleaved_inputs, + model=self.model_name, + input_type=input_type, + ).embeddings + interleaved_embeddings.append(torch.tensor(embeddings)) else: for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] @@ -166,23 +217,24 @@ def get_fused_embeddings( [text, image] for image, text in zip(batch_images, batch_texts) ] - interleaved_embeddings += torch.tensor( - self.vo.multimodal_embed( - interleaved_inputs, - model=self.model_name, - input_type=input_type, - ).embeddings - ) + embeddings = self._multimodal_embed( + interleaved_inputs, + model=self.model_name, + input_type=input_type, + ).embeddings + interleaved_embeddings.append(torch.tensor(embeddings)) interleaved_embeddings = torch.vstack(interleaved_embeddings) return interleaved_embeddings elif texts is not None: - # print("encoding texts only") - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings( + texts, batch_size, input_type=input_type + ) elif images is not None: - # print("encoding images only") - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings( + images, batch_size, input_type=input_type + ) if text_embeddings is not None: return text_embeddings diff --git a/results-mieb/voyage-multimodal-3/1/AROCocoOrder.json b/results-mieb/voyage-multimodal-3/1/AROCocoOrder.json new file mode 100644 index 0000000000..b2ccd7997b --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/AROCocoOrder.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "853ec8757226585a38a80886c51fe0f3f268787c", + "evaluation_time": 7601.064672470093, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.4089964032173157, + "text_acc": 0.4089964032173157 + } + ] + }, + "task_name": "AROCocoOrder" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/AROFlickrOrder.json b/results-mieb/voyage-multimodal-3/1/AROFlickrOrder.json new file mode 100644 index 0000000000..75cc95cea6 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/AROFlickrOrder.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "1f9485f69c87947812378a1aedf86410c86a0aa8", + "evaluation_time": 1100.6516120433807, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.3718000054359436, + "text_acc": 0.3718000054359436 + } + ] + }, + "task_name": "AROFlickrOrder" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/AROVisualAttribution.json b/results-mieb/voyage-multimodal-3/1/AROVisualAttribution.json new file mode 100644 index 0000000000..29c10acf9c --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/AROVisualAttribution.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "18f7e01358d91df599d723f00e16a18640e19398", + "evaluation_time": 6468.914260625839, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.6587588787078857, + "text_acc": 0.6587588787078857 + } + ] + }, + "task_name": "AROVisualAttribution" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/AROVisualRelation.json b/results-mieb/voyage-multimodal-3/1/AROVisualRelation.json new file mode 100644 index 0000000000..5d5d540699 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/AROVisualRelation.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "3867ad4f46a1ac2e63be034d1fc77dd8c2ef7209", + "evaluation_time": 5647.357409238815, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.5194051265716553, + "text_acc": 0.5194051265716553 + } + ] + }, + "task_name": "AROVisualRelation" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR10.json b/results-mieb/voyage-multimodal-3/1/CIFAR10.json new file mode 100644 index 0000000000..887d06ad49 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CIFAR10.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + "evaluation_time": 490.16403460502625, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.9553800000000001, + "f1": 0.9555334014223762, + "f1_weighted": 0.955533401422376, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9553800000000001, + "scores_per_experiment": [ + { + "accuracy": 0.9583, + "f1": 0.9585482615112164, + "f1_weighted": 0.9585482615112163 + }, + { + "accuracy": 0.956, + "f1": 0.9561424473749524, + "f1_weighted": 0.9561424473749522 + }, + { + "accuracy": 0.956, + "f1": 0.9561017525171834, + "f1_weighted": 0.9561017525171835 + }, + { + "accuracy": 0.9544, + "f1": 0.9545053818964286, + "f1_weighted": 0.9545053818964285 + }, + { + "accuracy": 0.9522, + "f1": 0.9523691638120997, + "f1_weighted": 0.9523691638120996 + } + ] + } + ] + }, + "task_name": "CIFAR10" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR100.json b/results-mieb/voyage-multimodal-3/1/CIFAR100.json new file mode 100644 index 0000000000..1ed53469ef --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CIFAR100.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + "evaluation_time": 746.6555774211884, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.7813399999999999, + "f1": 0.7742194484494215, + "f1_weighted": 0.7742194484494215, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7813399999999999, + "scores_per_experiment": [ + { + "accuracy": 0.7747, + "f1": 0.7663694868474199, + "f1_weighted": 0.76636948684742 + }, + { + "accuracy": 0.7852, + "f1": 0.7783034822861626, + "f1_weighted": 0.7783034822861625 + }, + { + "accuracy": 0.7808, + "f1": 0.7731842063509314, + "f1_weighted": 0.7731842063509313 + }, + { + "accuracy": 0.7803, + "f1": 0.7736245004512284, + "f1_weighted": 0.7736245004512283 + }, + { + "accuracy": 0.7857, + "f1": 0.7796155663113651, + "f1_weighted": 0.7796155663113651 + } + ] + } + ] + }, + "task_name": "CIFAR100" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR100ZeroShot.json b/results-mieb/voyage-multimodal-3/1/CIFAR100ZeroShot.json new file mode 100644 index 0000000000..9697091e6f --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CIFAR100ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + "evaluation_time": 364.0606734752655, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.7192, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7192 + } + ] + }, + "task_name": "CIFAR100ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR10ZeroShot.json b/results-mieb/voyage-multimodal-3/1/CIFAR10ZeroShot.json new file mode 100644 index 0000000000..8463987f5d --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CIFAR10ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + "evaluation_time": 356.5223488807678, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.9456, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9456 + } + ] + }, + "task_name": "CIFAR10ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIRRIT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/CIRRIT2IRetrieval.json new file mode 100644 index 0000000000..d53cb3d9c3 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CIRRIT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", + "evaluation_time": 4035.3517696857452, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.1729, + "cv_recall_at_10": 0.55516, + "cv_recall_at_100": 0.87146, + "cv_recall_at_1000": 0.98513, + "cv_recall_at_20": 0.65803, + "cv_recall_at_3": 0.35276, + "cv_recall_at_5": 0.43933, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.35005, + "map_at_1": 0.17026, + "map_at_10": 0.28592, + "map_at_100": 0.29878, + "map_at_1000": 0.29934, + "map_at_20": 0.29326, + "map_at_3": 0.25032, + "map_at_5": 0.27043, + "mrr_at_1": 0.17290167865707434, + "mrr_at_10": 0.28662517604963694, + "mrr_at_100": 0.2994069042767556, + "mrr_at_1000": 0.29996016856575075, + "mrr_at_20": 0.29391993573720654, + "mrr_at_3": 0.2514788169464439, + "mrr_at_5": 0.27127498001598865, + "nauc_cv_recall_at_1000_diff1": -0.3170009939459746, + "nauc_cv_recall_at_1000_max": 0.09882985452245488, + "nauc_cv_recall_at_1000_std": 0.7541941507785804, + "nauc_cv_recall_at_100_diff1": -0.12703984241006452, + "nauc_cv_recall_at_100_max": 0.07035034338244234, + "nauc_cv_recall_at_100_std": 0.26590969436450945, + "nauc_cv_recall_at_10_diff1": -0.06282729604578144, + "nauc_cv_recall_at_10_max": 0.05042200343697883, + "nauc_cv_recall_at_10_std": 0.008458354144268477, + "nauc_cv_recall_at_1_diff1": 3.515431224046151e-05, + "nauc_cv_recall_at_1_max": -0.08628430637720942, + "nauc_cv_recall_at_1_std": -0.07433487824238864, + "nauc_cv_recall_at_20_diff1": -0.09258740959337054, + "nauc_cv_recall_at_20_max": 0.06248701680718945, + "nauc_cv_recall_at_20_std": 0.0493702259511631, + "nauc_cv_recall_at_3_diff1": -0.01803245197203495, + "nauc_cv_recall_at_3_max": 0.021054911702306108, + "nauc_cv_recall_at_3_std": -0.03882970308722248, + "nauc_cv_recall_at_5_diff1": -0.03739685944600389, + "nauc_cv_recall_at_5_max": 0.0390507377060466, + "nauc_cv_recall_at_5_std": -0.01965818597776538, + "nauc_map_at_1000_diff1": -0.013888056487772901, + "nauc_map_at_1000_max": -0.025047113581432034, + "nauc_map_at_1000_std": -0.04615687795133258, + "nauc_map_at_100_diff1": -0.013929180383153755, + "nauc_map_at_100_max": -0.024852376888687074, + "nauc_map_at_100_std": -0.045846793015922024, + "nauc_map_at_10_diff1": -0.013282752194913467, + "nauc_map_at_10_max": -0.023507328813512387, + "nauc_map_at_10_std": -0.04880002828070644, + "nauc_map_at_1_diff1": 0.006972875008966575, + "nauc_map_at_1_max": -0.09078674038972498, + "nauc_map_at_1_std": -0.07738315349064624, + "nauc_map_at_20_diff1": -0.014642392810803973, + "nauc_map_at_20_max": -0.023616316784522622, + "nauc_map_at_20_std": -0.046676108387409335, + "nauc_map_at_3_diff1": -0.004152872482186347, + "nauc_map_at_3_max": -0.030055151218383824, + "nauc_map_at_3_std": -0.05893383119480629, + "nauc_map_at_5_diff1": -0.009966154532398824, + "nauc_map_at_5_max": -0.024600214946042556, + "nauc_map_at_5_std": -0.05267076625586234, + "nauc_mrr_at_1000_diff1": -0.015342812781242809, + "nauc_mrr_at_1000_max": -0.02446839675351005, + "nauc_mrr_at_1000_std": -0.04471699907025565, + "nauc_mrr_at_100_diff1": -0.015387382226896854, + "nauc_mrr_at_100_max": -0.024279183900671086, + "nauc_mrr_at_100_std": -0.04441731123645614, + "nauc_mrr_at_10_diff1": -0.01482075150830492, + "nauc_mrr_at_10_max": -0.023072871766092212, + "nauc_mrr_at_10_std": -0.047457658821462065, + "nauc_mrr_at_1_diff1": 3.515431224046151e-05, + "nauc_mrr_at_1_max": -0.08628430637720942, + "nauc_mrr_at_1_std": -0.07433487824238864, + "nauc_mrr_at_20_diff1": -0.01614543662658322, + "nauc_mrr_at_20_max": -0.02310471528284919, + "nauc_mrr_at_20_std": -0.04526342848718061, + "nauc_mrr_at_3_diff1": -0.006574670211446688, + "nauc_mrr_at_3_max": -0.029183469428734166, + "nauc_mrr_at_3_std": -0.0562752472060744, + "nauc_mrr_at_5_diff1": -0.011737536762471967, + "nauc_mrr_at_5_max": -0.024300128014767088, + "nauc_mrr_at_5_std": -0.05139431149525923, + "nauc_ndcg_at_1000_diff1": -0.024295226300813422, + "nauc_ndcg_at_1000_max": -0.014020784847656736, + "nauc_ndcg_at_1000_std": -0.026752128393981025, + "nauc_ndcg_at_100_diff1": -0.027543699842724954, + "nauc_ndcg_at_100_max": -0.009617940963043888, + "nauc_ndcg_at_100_std": -0.01567309309677566, + "nauc_ndcg_at_10_diff1": -0.025739220828095582, + "nauc_ndcg_at_10_max": -0.0033833300342746863, + "nauc_ndcg_at_10_std": -0.03391736967026064, + "nauc_ndcg_at_1_diff1": 3.515431224046151e-05, + "nauc_ndcg_at_1_max": -0.08628430637720942, + "nauc_ndcg_at_1_std": -0.07433487824238864, + "nauc_ndcg_at_20_diff1": -0.03129848142341674, + "nauc_ndcg_at_20_max": -0.0028210068185106894, + "nauc_ndcg_at_20_std": -0.025766809747319912, + "nauc_ndcg_at_3_diff1": -0.007831804331826687, + "nauc_ndcg_at_3_max": -0.015209020075707921, + "nauc_ndcg_at_3_std": -0.05331550077340847, + "nauc_ndcg_at_5_diff1": -0.01753602339841263, + "nauc_ndcg_at_5_max": -0.006182671781771981, + "nauc_ndcg_at_5_std": -0.04310939231430975, + "nauc_precision_at_1000_diff1": -0.3268111988325224, + "nauc_precision_at_1000_max": 0.041709669149306676, + "nauc_precision_at_1000_std": 0.32473064559600107, + "nauc_precision_at_100_diff1": -0.15269962228413458, + "nauc_precision_at_100_max": 0.06541595153406797, + "nauc_precision_at_100_std": 0.23656402842386526, + "nauc_precision_at_10_diff1": -0.07325353438185137, + "nauc_precision_at_10_max": 0.05403993597313785, + "nauc_precision_at_10_std": 0.010807791672858408, + "nauc_precision_at_1_diff1": 3.515431224046151e-05, + "nauc_precision_at_1_max": -0.08628430637720942, + "nauc_precision_at_1_std": -0.07433487824238864, + "nauc_precision_at_20_diff1": -0.10452717120685504, + "nauc_precision_at_20_max": 0.06487413760673558, + "nauc_precision_at_20_std": 0.049045265681017115, + "nauc_precision_at_3_diff1": -0.025008316851500872, + "nauc_precision_at_3_max": 0.026180552745518643, + "nauc_precision_at_3_std": -0.03899051183927358, + "nauc_precision_at_5_diff1": -0.04617245194188806, + "nauc_precision_at_5_max": 0.04457879931255899, + "nauc_precision_at_5_std": -0.01624571071222153, + "nauc_recall_at_1000_diff1": -0.3170009939459746, + "nauc_recall_at_1000_max": 0.09882985452245488, + "nauc_recall_at_1000_std": 0.7541941507785804, + "nauc_recall_at_100_diff1": -0.12609939922364324, + "nauc_recall_at_100_max": 0.07040950128302399, + "nauc_recall_at_100_std": 0.2665222461130575, + "nauc_recall_at_10_diff1": -0.06210442381973989, + "nauc_recall_at_10_max": 0.051164767833312974, + "nauc_recall_at_10_std": 0.008895461448149669, + "nauc_recall_at_1_diff1": 0.006972875008966575, + "nauc_recall_at_1_max": -0.09078674038972498, + "nauc_recall_at_1_std": -0.07738315349064624, + "nauc_recall_at_20_diff1": -0.09152161662419589, + "nauc_recall_at_20_max": 0.06302964517499607, + "nauc_recall_at_20_std": 0.049771567847766096, + "nauc_recall_at_3_diff1": -0.015476344901671572, + "nauc_recall_at_3_max": 0.020647503145370293, + "nauc_recall_at_3_std": -0.04128457833864728, + "nauc_recall_at_5_diff1": -0.036186390273891984, + "nauc_recall_at_5_max": 0.03974749178499069, + "nauc_recall_at_5_std": -0.0193040791268312, + "ndcg_at_1": 0.1729, + "ndcg_at_10": 0.35005, + "ndcg_at_100": 0.41605, + "ndcg_at_1000": 0.43061, + "ndcg_at_20": 0.37634, + "ndcg_at_3": 0.27647, + "ndcg_at_5": 0.31251, + "precision_at_1": 0.1729, + "precision_at_10": 0.05626, + "precision_at_100": 0.00882, + "precision_at_1000": 0.001, + "precision_at_20": 0.03332, + "precision_at_3": 0.11902, + "precision_at_5": 0.08906, + "recall_at_1": 0.17026, + "recall_at_10": 0.55468, + "recall_at_100": 0.87134, + "recall_at_1000": 0.98513, + "recall_at_20": 0.65767, + "recall_at_3": 0.3512, + "recall_at_5": 0.43849 + } + ] + }, + "task_name": "CIRRIT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CLEVRCountZeroShot.json b/results-mieb/voyage-multimodal-3/1/CLEVRCountZeroShot.json new file mode 100644 index 0000000000..7fc596a170 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CLEVRCountZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "8b5dce4d5393a04fb58b9261b22a881b02e379b1", + "evaluation_time": 2673.3230113983154, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.45653333333333335, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.45653333333333335 + } + ] + }, + "task_name": "CLEVRCountZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CLEVRZeroShot.json b/results-mieb/voyage-multimodal-3/1/CLEVRZeroShot.json new file mode 100644 index 0000000000..334c77fbcf --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CLEVRZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "ec9c04224a95836ca0344a6000ec8d8bc8a6d4f2", + "evaluation_time": 2703.7519657611847, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.13906666666666667, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.13906666666666667 + } + ] + }, + "task_name": "CLEVRZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CUB200I2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/CUB200I2IRetrieval.json new file mode 100644 index 0000000000..0fd1e59bcf --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CUB200I2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "ad08c1307b15a226bf1b64e62656a17f1f85f7ec", + "evaluation_time": 2163.535472869873, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.64429, + "cv_recall_at_10": 0.91146, + "cv_recall_at_100": 0.98826, + "cv_recall_at_1000": 0.99983, + "cv_recall_at_20": 0.94546, + "cv_recall_at_3": 0.79651, + "cv_recall_at_5": 0.8533, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.64429, + "map_at_1": 0.0, + "map_at_10": 0.1002, + "map_at_100": 0.24338, + "map_at_1000": 0.26897, + "map_at_20": 0.1634, + "map_at_3": 0.02418, + "map_at_5": 0.04884, + "mrr_at_1": 0.0, + "mrr_at_10": 0.38507918700790505, + "mrr_at_100": 0.3891901905104831, + "mrr_at_1000": 0.38925599889084067, + "mrr_at_20": 0.38798629740580254, + "mrr_at_3": 0.3548786100563762, + "mrr_at_5": 0.3750460246231719, + "nauc_cv_recall_at_1000_diff1": 0.12277185076635803, + "nauc_cv_recall_at_1000_max": 0.8692910721335301, + "nauc_cv_recall_at_1000_std": 0.5541669634720343, + "nauc_cv_recall_at_100_diff1": -0.6321515418902193, + "nauc_cv_recall_at_100_max": 0.07387669514437997, + "nauc_cv_recall_at_100_std": -0.04785958499869987, + "nauc_cv_recall_at_10_diff1": -0.5677044574472964, + "nauc_cv_recall_at_10_max": 0.0009713859993077379, + "nauc_cv_recall_at_10_std": 0.05011589213768149, + "nauc_cv_recall_at_1_diff1": -0.4046106762756965, + "nauc_cv_recall_at_1_max": 0.0004226548426512101, + "nauc_cv_recall_at_1_std": 0.027432944700061697, + "nauc_cv_recall_at_20_diff1": -0.603899401030247, + "nauc_cv_recall_at_20_max": 0.001148593758454481, + "nauc_cv_recall_at_20_std": -0.019548921899248613, + "nauc_cv_recall_at_3_diff1": -0.4455725103414982, + "nauc_cv_recall_at_3_max": 0.008991726540476934, + "nauc_cv_recall_at_3_std": 0.038874900776692395, + "nauc_cv_recall_at_5_diff1": -0.5017947639447603, + "nauc_cv_recall_at_5_max": 0.000468670147831035, + "nauc_cv_recall_at_5_std": 0.03870377084563178, + "nauc_map_at_1000_diff1": -0.2876895130339865, + "nauc_map_at_1000_max": -0.012896517311254253, + "nauc_map_at_1000_std": 0.14461249150690794, + "nauc_map_at_100_diff1": -0.28681032009565455, + "nauc_map_at_100_max": -0.01236690630903875, + "nauc_map_at_100_std": 0.12302542192767915, + "nauc_map_at_10_diff1": -0.3374463371796672, + "nauc_map_at_10_max": -0.00903324753451852, + "nauc_map_at_10_std": 0.047063840155417105, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": -0.3155064667819531, + "nauc_map_at_20_max": -0.01380755932826781, + "nauc_map_at_20_std": 0.07916016466857276, + "nauc_map_at_3_diff1": -0.3565761066694584, + "nauc_map_at_3_max": 0.003993550400848401, + "nauc_map_at_3_std": 0.03848458302964767, + "nauc_map_at_5_diff1": -0.356426360214968, + "nauc_map_at_5_max": -0.0028489029065050923, + "nauc_map_at_5_std": 0.03850371509343761, + "nauc_mrr_at_1000_diff1": -0.43800684377365345, + "nauc_mrr_at_1000_max": -0.0005539340869123282, + "nauc_mrr_at_1000_std": 0.02817514259134202, + "nauc_mrr_at_100_diff1": -0.4381292853682621, + "nauc_mrr_at_100_max": -0.0004748461079494744, + "nauc_mrr_at_100_std": 0.02809781745809359, + "nauc_mrr_at_10_diff1": -0.4391379705707844, + "nauc_mrr_at_10_max": -0.0017781045577203935, + "nauc_mrr_at_10_std": 0.030221395566769144, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": -0.4399000080783371, + "nauc_mrr_at_20_max": -0.0006053657448768487, + "nauc_mrr_at_20_std": 0.028534324700874168, + "nauc_mrr_at_3_diff1": -0.42519681466953113, + "nauc_mrr_at_3_max": -0.002367553551530187, + "nauc_mrr_at_3_std": 0.029485330664110954, + "nauc_mrr_at_5_diff1": -0.43346713040751733, + "nauc_mrr_at_5_max": -0.001378564156485059, + "nauc_mrr_at_5_std": 0.0273542041859884, + "nauc_ndcg_at_1000_diff1": -0.3070099919423358, + "nauc_ndcg_at_1000_max": -0.009340238003473103, + "nauc_ndcg_at_1000_std": 0.23271373655751904, + "nauc_ndcg_at_100_diff1": -0.2955390340514443, + "nauc_ndcg_at_100_max": -0.004923830214676451, + "nauc_ndcg_at_100_std": 0.1223181178316258, + "nauc_ndcg_at_10_diff1": -0.3656129326908537, + "nauc_ndcg_at_10_max": -0.006791901707602765, + "nauc_ndcg_at_10_std": 0.03661090493229275, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": -0.34169185864420143, + "nauc_ndcg_at_20_max": -0.010944008239357693, + "nauc_ndcg_at_20_std": 0.059350958926926344, + "nauc_ndcg_at_3_diff1": -0.3955340807216821, + "nauc_ndcg_at_3_max": 0.0028371458265729526, + "nauc_ndcg_at_3_std": 0.03183977196015828, + "nauc_ndcg_at_5_diff1": -0.3877160890591247, + "nauc_ndcg_at_5_max": -0.0029189946975532806, + "nauc_ndcg_at_5_std": 0.03016904642545801, + "nauc_precision_at_1000_diff1": -0.15317446430623788, + "nauc_precision_at_1000_max": -0.012330932399289705, + "nauc_precision_at_1000_std": 0.46901467409707714, + "nauc_precision_at_100_diff1": -0.2317631455616716, + "nauc_precision_at_100_max": -0.0030020431834504513, + "nauc_precision_at_100_std": 0.15385722543416624, + "nauc_precision_at_10_diff1": -0.35247656600482197, + "nauc_precision_at_10_max": -0.00888510351612272, + "nauc_precision_at_10_std": 0.037710119874103415, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": -0.31990309054950195, + "nauc_precision_at_20_max": -0.013165426936379529, + "nauc_precision_at_20_std": 0.06634509720427917, + "nauc_precision_at_3_diff1": -0.39434779116425517, + "nauc_precision_at_3_max": 0.0031527140838212694, + "nauc_precision_at_3_std": 0.032415733853461234, + "nauc_precision_at_5_diff1": -0.38367802159695974, + "nauc_precision_at_5_max": -0.0043498860313263225, + "nauc_precision_at_5_std": 0.029773528967274605, + "nauc_recall_at_1000_diff1": -0.14659919440214333, + "nauc_recall_at_1000_max": -0.01572890369746866, + "nauc_recall_at_1000_std": 0.5701773317964056, + "nauc_recall_at_100_diff1": -0.22574095098196198, + "nauc_recall_at_100_max": -0.0028660455635216874, + "nauc_recall_at_100_std": 0.16027905776692558, + "nauc_recall_at_10_diff1": -0.3380962218041837, + "nauc_recall_at_10_max": -0.009633461116730245, + "nauc_recall_at_10_std": 0.04188238545019402, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": -0.3097175381529999, + "nauc_recall_at_20_max": -0.013468091696335054, + "nauc_recall_at_20_std": 0.07041109319858846, + "nauc_recall_at_3_diff1": -0.3605831363960561, + "nauc_recall_at_3_max": 0.0027883037507332492, + "nauc_recall_at_3_std": 0.03930001152121736, + "nauc_recall_at_5_diff1": -0.3607461144071461, + "nauc_recall_at_5_max": -0.003960285973988208, + "nauc_recall_at_5_std": 0.03577635038665928, + "ndcg_at_1": 0.0, + "ndcg_at_10": 0.3976, + "ndcg_at_100": 0.44182, + "ndcg_at_1000": 0.58205, + "ndcg_at_20": 0.36201, + "ndcg_at_3": 0.32687, + "ndcg_at_5": 0.38173, + "precision_at_1": 0.0, + "precision_at_10": 0.43961, + "precision_at_100": 0.14667, + "precision_at_1000": 0.02471, + "precision_at_20": 0.36555, + "precision_at_3": 0.40812, + "precision_at_5": 0.45437, + "recall_at_1": 0.0, + "recall_at_10": 0.15711, + "recall_at_100": 0.52043, + "recall_at_1000": 0.87486, + "recall_at_20": 0.26077, + "recall_at_3": 0.04378, + "recall_at_5": 0.08123 + } + ] + }, + "task_name": "CUB200I2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Caltech101.json b/results-mieb/voyage-multimodal-3/1/Caltech101.json new file mode 100644 index 0000000000..a219d45f7e --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/Caltech101.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", + "evaluation_time": 1512.7276003360748, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.913905325443787, + "f1": 0.8977839451811815, + "f1_weighted": 0.9124464844882695, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.913905325443787, + "scores_per_experiment": [ + { + "accuracy": 0.9077909270216963, + "f1": 0.894273201231098, + "f1_weighted": 0.907344141460267 + }, + { + "accuracy": 0.9095989480604865, + "f1": 0.8941508848751586, + "f1_weighted": 0.9101502765347295 + }, + { + "accuracy": 0.9045036160420776, + "f1": 0.8959083825916897, + "f1_weighted": 0.902335674287055 + }, + { + "accuracy": 0.9294871794871795, + "f1": 0.8999318659786849, + "f1_weighted": 0.9275837214048807 + }, + { + "accuracy": 0.9181459566074951, + "f1": 0.9046553912292769, + "f1_weighted": 0.9148186087544156 + } + ] + } + ] + }, + "task_name": "Caltech101" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Caltech101ZeroShot.json b/results-mieb/voyage-multimodal-3/1/Caltech101ZeroShot.json new file mode 100644 index 0000000000..9dbefa140a --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/Caltech101ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", + "evaluation_time": 643.5588819980621, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.7817225509533202, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7817225509533202 + } + ] + }, + "task_name": "Caltech101ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Country211.json b/results-mieb/voyage-multimodal-3/1/Country211.json new file mode 100644 index 0000000000..a6442dcb2d --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/Country211.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", + "evaluation_time": 7564.615749359131, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.08897630331753555, + "f1": 0.06644398101423767, + "f1_weighted": 0.06644398101423768, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.08897630331753555, + "scores_per_experiment": [ + { + "accuracy": 0.08729857819905214, + "f1": 0.06449707199222236, + "f1_weighted": 0.06449707199222238 + }, + { + "accuracy": 0.08725118483412322, + "f1": 0.06513743277476837, + "f1_weighted": 0.06513743277476838 + }, + { + "accuracy": 0.09251184834123223, + "f1": 0.06917324902817186, + "f1_weighted": 0.06917324902817186 + }, + { + "accuracy": 0.09104265402843602, + "f1": 0.06930671211306509, + "f1_weighted": 0.06930671211306509 + }, + { + "accuracy": 0.08677725118483412, + "f1": 0.06410543916296063, + "f1_weighted": 0.06410543916296063 + } + ] + } + ] + }, + "task_name": "Country211" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Country211ZeroShot.json b/results-mieb/voyage-multimodal-3/1/Country211ZeroShot.json new file mode 100644 index 0000000000..4873804f03 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/Country211ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", + "evaluation_time": 4199.378444910049, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.11071090047393364, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.11071090047393364 + } + ] + }, + "task_name": "Country211ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/DTD.json b/results-mieb/voyage-multimodal-3/1/DTD.json new file mode 100644 index 0000000000..653786bec4 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/DTD.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", + "evaluation_time": 1314.5107147693634, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.7256382978723404, + "f1": 0.7125223958188174, + "f1_weighted": 0.7125223958188174, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7256382978723404, + "scores_per_experiment": [ + { + "accuracy": 0.726063829787234, + "f1": 0.7128435451309848, + "f1_weighted": 0.7128435451309848 + }, + { + "accuracy": 0.7303191489361702, + "f1": 0.7175258840982487, + "f1_weighted": 0.7175258840982487 + }, + { + "accuracy": 0.7351063829787234, + "f1": 0.7246822849042761, + "f1_weighted": 0.7246822849042759 + }, + { + "accuracy": 0.7186170212765958, + "f1": 0.7066549466550418, + "f1_weighted": 0.7066549466550417 + }, + { + "accuracy": 0.7180851063829787, + "f1": 0.7009053183055356, + "f1_weighted": 0.7009053183055356 + } + ] + } + ] + }, + "task_name": "DTD" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/DTDZeroShot.json b/results-mieb/voyage-multimodal-3/1/DTDZeroShot.json new file mode 100644 index 0000000000..97b89a1b72 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/DTDZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", + "evaluation_time": 437.40010952949524, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5904255319148937, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5904255319148937 + } + ] + }, + "task_name": "DTDZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/EuroSAT.json b/results-mieb/voyage-multimodal-3/1/EuroSAT.json new file mode 100644 index 0000000000..bb3b8ece72 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/EuroSAT.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", + "evaluation_time": 264.42366337776184, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.7926666666666666, + "f1": 0.7917939849512137, + "f1_weighted": 0.7920934779674137, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7926666666666666, + "scores_per_experiment": [ + { + "accuracy": 0.8044444444444444, + "f1": 0.8037094408204952, + "f1_weighted": 0.8041734004823994 + }, + { + "accuracy": 0.7892592592592592, + "f1": 0.7892676858734221, + "f1_weighted": 0.7890248894183581 + }, + { + "accuracy": 0.7870370370370371, + "f1": 0.7872003711470628, + "f1_weighted": 0.7865408192601453 + }, + { + "accuracy": 0.805, + "f1": 0.8040550271257725, + "f1_weighted": 0.8038929843599751 + }, + { + "accuracy": 0.7775925925925926, + "f1": 0.7747373997893159, + "f1_weighted": 0.7768352963161905 + } + ] + } + ] + }, + "task_name": "EuroSAT" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/EuroSATZeroShot.json b/results-mieb/voyage-multimodal-3/1/EuroSATZeroShot.json new file mode 100644 index 0000000000..02d4005da2 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/EuroSATZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", + "evaluation_time": 233.39685010910034, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5235185185185185, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5235185185185185 + } + ] + }, + "task_name": "EuroSATZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/FER2013.json b/results-mieb/voyage-multimodal-3/1/FER2013.json new file mode 100644 index 0000000000..6426e2e752 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/FER2013.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", + "evaluation_time": 272.1406137943268, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.4668988576205072, + "f1": 0.3944239157839031, + "f1_weighted": 0.46477203838065845, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.4668988576205072, + "scores_per_experiment": [ + { + "accuracy": 0.4821677347450543, + "f1": 0.3961622657921301, + "f1_weighted": 0.46406292230733487 + }, + { + "accuracy": 0.4701866815268877, + "f1": 0.39728952325640365, + "f1_weighted": 0.4745316385258285 + }, + { + "accuracy": 0.4742268041237113, + "f1": 0.3946468406633246, + "f1_weighted": 0.47228603834416566 + }, + { + "accuracy": 0.5050153246029535, + "f1": 0.4180255073056839, + "f1_weighted": 0.48855025062664625 + }, + { + "accuracy": 0.4028977431039287, + "f1": 0.3659954419019734, + "f1_weighted": 0.42442934209931704 + } + ] + } + ] + }, + "task_name": "FER2013" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/FER2013ZeroShot.json b/results-mieb/voyage-multimodal-3/1/FER2013ZeroShot.json new file mode 100644 index 0000000000..26e1c82a1c --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/FER2013ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", + "evaluation_time": 269.1961929798126, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5135135135135135, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5135135135135135 + } + ] + }, + "task_name": "FER2013ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/FGVCAircraft.json b/results-mieb/voyage-multimodal-3/1/FGVCAircraft.json new file mode 100644 index 0000000000..fdd75070bd --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/FGVCAircraft.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "91860adfc9a09aabca5cddb5247442109b38e213", + "evaluation_time": 8436.217485189438, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.2252025202520252, + "f1": 0.20703858415613344, + "f1_weighted": 0.20687263986710783, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.2252025202520252, + "scores_per_experiment": [ + { + "accuracy": 0.23072307230723071, + "f1": 0.2122745870540116, + "f1_weighted": 0.2122443878373498 + }, + { + "accuracy": 0.22652265226522653, + "f1": 0.21175395338696293, + "f1_weighted": 0.21152301004474827 + }, + { + "accuracy": 0.22082208220822083, + "f1": 0.20328120444347306, + "f1_weighted": 0.20296849188917818 + }, + { + "accuracy": 0.22532253225322532, + "f1": 0.20487362177158278, + "f1_weighted": 0.2046900408124526 + }, + { + "accuracy": 0.2226222622262226, + "f1": 0.20300955412463678, + "f1_weighted": 0.20293726875181017 + } + ] + } + ] + }, + "task_name": "FGVCAircraft" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/FGVCAircraftZeroShot.json b/results-mieb/voyage-multimodal-3/1/FGVCAircraftZeroShot.json new file mode 100644 index 0000000000..a5a381b43b --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/FGVCAircraftZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "91860adfc9a09aabca5cddb5247442109b38e213", + "evaluation_time": 2428.0195784568787, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.11671167116711671, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.11671167116711671 + } + ] + }, + "task_name": "FGVCAircraftZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Fashion200kI2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/Fashion200kI2TRetrieval.json new file mode 100644 index 0000000000..9aa4af6e7e --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/Fashion200kI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "96a313715ecf67f5dfe70c4fa52406bc7bdfbeee", + "evaluation_time": 2920.238646507263, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.03191, + "cv_recall_at_10": 0.12722, + "cv_recall_at_100": 0.36265, + "cv_recall_at_1000": 0.71364, + "cv_recall_at_20": 0.17611, + "cv_recall_at_3": 0.06545, + "cv_recall_at_5": 0.08836, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.07318, + "map_at_1": 0.03191, + "map_at_10": 0.05672, + "map_at_100": 0.06447, + "map_at_1000": 0.06583, + "map_at_20": 0.06006, + "map_at_3": 0.04646, + "map_at_5": 0.05162, + "mrr_at_1": 0.031908365718960933, + "mrr_at_10": 0.056717298632823265, + "mrr_at_100": 0.06447384392155588, + "mrr_at_1000": 0.06582834414070698, + "mrr_at_20": 0.06005622816900517, + "mrr_at_3": 0.046464853071521144, + "mrr_at_5": 0.05161928137996866, + "nauc_cv_recall_at_1000_diff1": 0.07064142096733762, + "nauc_cv_recall_at_1000_max": 0.14933892098071952, + "nauc_cv_recall_at_1000_std": 0.1860120425733161, + "nauc_cv_recall_at_100_diff1": 0.06431278693555113, + "nauc_cv_recall_at_100_max": 0.099638842754024, + "nauc_cv_recall_at_100_std": 0.11930789207654446, + "nauc_cv_recall_at_10_diff1": 0.12327773856112313, + "nauc_cv_recall_at_10_max": 0.054975009177794736, + "nauc_cv_recall_at_10_std": 0.066868711233558, + "nauc_cv_recall_at_1_diff1": 0.3139494374020095, + "nauc_cv_recall_at_1_max": 0.13028777354452473, + "nauc_cv_recall_at_1_std": 0.04248164918312027, + "nauc_cv_recall_at_20_diff1": 0.09857331248582667, + "nauc_cv_recall_at_20_max": 0.07617887127857093, + "nauc_cv_recall_at_20_std": 0.07353148089369, + "nauc_cv_recall_at_3_diff1": 0.16925343333548418, + "nauc_cv_recall_at_3_max": 0.06310232394286991, + "nauc_cv_recall_at_3_std": 0.021711082664363137, + "nauc_cv_recall_at_5_diff1": 0.14716519740601317, + "nauc_cv_recall_at_5_max": 0.04865909257707613, + "nauc_cv_recall_at_5_std": 0.04051251752627315, + "nauc_map_at_1000_diff1": 0.18891923417066206, + "nauc_map_at_1000_max": 0.08785537204131776, + "nauc_map_at_1000_std": 0.054198501963607405, + "nauc_map_at_100_diff1": 0.18861241659777042, + "nauc_map_at_100_max": 0.08744669384239624, + "nauc_map_at_100_std": 0.053693641168022804, + "nauc_map_at_10_diff1": 0.2016071032113213, + "nauc_map_at_10_max": 0.0832784376344867, + "nauc_map_at_10_std": 0.047034075026690034, + "nauc_map_at_1_diff1": 0.3139494374020095, + "nauc_map_at_1_max": 0.13028777354452473, + "nauc_map_at_1_std": 0.04248164918312027, + "nauc_map_at_20_diff1": 0.19416427337317085, + "nauc_map_at_20_max": 0.08585800930572945, + "nauc_map_at_20_std": 0.048891821597338306, + "nauc_map_at_3_diff1": 0.22786718131396266, + "nauc_map_at_3_max": 0.09363713840892451, + "nauc_map_at_3_std": 0.03247132719959962, + "nauc_map_at_5_diff1": 0.21265602213173257, + "nauc_map_at_5_max": 0.084718050116957, + "nauc_map_at_5_std": 0.03963097845753907, + "nauc_mrr_at_1000_diff1": 0.18891923417066206, + "nauc_mrr_at_1000_max": 0.08785537204131776, + "nauc_mrr_at_1000_std": 0.054198501963607405, + "nauc_mrr_at_100_diff1": 0.18861241659777042, + "nauc_mrr_at_100_max": 0.08744669384239624, + "nauc_mrr_at_100_std": 0.053693641168022804, + "nauc_mrr_at_10_diff1": 0.2016071032113213, + "nauc_mrr_at_10_max": 0.0832784376344867, + "nauc_mrr_at_10_std": 0.047034075026690034, + "nauc_mrr_at_1_diff1": 0.3139494374020095, + "nauc_mrr_at_1_max": 0.13028777354452473, + "nauc_mrr_at_1_std": 0.04248164918312027, + "nauc_mrr_at_20_diff1": 0.19416427337317085, + "nauc_mrr_at_20_max": 0.08585800930572945, + "nauc_mrr_at_20_std": 0.048891821597338306, + "nauc_mrr_at_3_diff1": 0.22786718131396266, + "nauc_mrr_at_3_max": 0.09363713840892451, + "nauc_mrr_at_3_std": 0.03247132719959962, + "nauc_mrr_at_5_diff1": 0.21265602213173257, + "nauc_mrr_at_5_max": 0.084718050116957, + "nauc_mrr_at_5_std": 0.03963097845753907, + "nauc_ndcg_at_1000_diff1": 0.13976312621484005, + "nauc_ndcg_at_1000_max": 0.0988780200747641, + "nauc_ndcg_at_1000_std": 0.09099886566875824, + "nauc_ndcg_at_100_diff1": 0.13274169983767356, + "nauc_ndcg_at_100_max": 0.08923727522653309, + "nauc_ndcg_at_100_std": 0.07994532547221007, + "nauc_ndcg_at_10_diff1": 0.17057951444650982, + "nauc_ndcg_at_10_max": 0.07155992086707413, + "nauc_ndcg_at_10_std": 0.05413784909723341, + "nauc_ndcg_at_1_diff1": 0.3139494374020095, + "nauc_ndcg_at_1_max": 0.13028777354452473, + "nauc_ndcg_at_1_std": 0.04248164918312027, + "nauc_ndcg_at_20_diff1": 0.15371757602855127, + "nauc_ndcg_at_20_max": 0.07996117756779385, + "nauc_ndcg_at_20_std": 0.05859145892477607, + "nauc_ndcg_at_3_diff1": 0.20879984851659117, + "nauc_ndcg_at_3_max": 0.08382482722194272, + "nauc_ndcg_at_3_std": 0.029056373224856612, + "nauc_ndcg_at_5_diff1": 0.1889200636596775, + "nauc_ndcg_at_5_max": 0.07180390421297643, + "nauc_ndcg_at_5_std": 0.039851440751330514, + "nauc_precision_at_1000_diff1": 0.07064142096733839, + "nauc_precision_at_1000_max": 0.14933892098071938, + "nauc_precision_at_1000_std": 0.18601204257331688, + "nauc_precision_at_100_diff1": 0.06431278693555126, + "nauc_precision_at_100_max": 0.0996388427540241, + "nauc_precision_at_100_std": 0.11930789207654464, + "nauc_precision_at_10_diff1": 0.1232777385611232, + "nauc_precision_at_10_max": 0.054975009177794666, + "nauc_precision_at_10_std": 0.06686871123355793, + "nauc_precision_at_1_diff1": 0.3139494374020095, + "nauc_precision_at_1_max": 0.13028777354452473, + "nauc_precision_at_1_std": 0.04248164918312027, + "nauc_precision_at_20_diff1": 0.09857331248582687, + "nauc_precision_at_20_max": 0.07617887127857105, + "nauc_precision_at_20_std": 0.07353148089369012, + "nauc_precision_at_3_diff1": 0.16925343333548437, + "nauc_precision_at_3_max": 0.06310232394286996, + "nauc_precision_at_3_std": 0.021711082664362967, + "nauc_precision_at_5_diff1": 0.14716519740601297, + "nauc_precision_at_5_max": 0.048659092577076006, + "nauc_precision_at_5_std": 0.04051251752627308, + "nauc_recall_at_1000_diff1": 0.07064142096733762, + "nauc_recall_at_1000_max": 0.14933892098071952, + "nauc_recall_at_1000_std": 0.1860120425733161, + "nauc_recall_at_100_diff1": 0.06431278693555113, + "nauc_recall_at_100_max": 0.099638842754024, + "nauc_recall_at_100_std": 0.11930789207654446, + "nauc_recall_at_10_diff1": 0.12327773856112313, + "nauc_recall_at_10_max": 0.054975009177794736, + "nauc_recall_at_10_std": 0.066868711233558, + "nauc_recall_at_1_diff1": 0.3139494374020095, + "nauc_recall_at_1_max": 0.13028777354452473, + "nauc_recall_at_1_std": 0.04248164918312027, + "nauc_recall_at_20_diff1": 0.09857331248582667, + "nauc_recall_at_20_max": 0.07617887127857093, + "nauc_recall_at_20_std": 0.07353148089369, + "nauc_recall_at_3_diff1": 0.16925343333548418, + "nauc_recall_at_3_max": 0.06310232394286991, + "nauc_recall_at_3_std": 0.021711082664363137, + "nauc_recall_at_5_diff1": 0.14716519740601317, + "nauc_recall_at_5_max": 0.04865909257707613, + "nauc_recall_at_5_std": 0.04051251752627315, + "ndcg_at_1": 0.03191, + "ndcg_at_10": 0.07318, + "ndcg_at_100": 0.11934, + "ndcg_at_1000": 0.16206, + "ndcg_at_20": 0.08547, + "ndcg_at_3": 0.05133, + "ndcg_at_5": 0.0607, + "precision_at_1": 0.03191, + "precision_at_10": 0.01272, + "precision_at_100": 0.00363, + "precision_at_1000": 0.00071, + "precision_at_20": 0.00881, + "precision_at_3": 0.02182, + "precision_at_5": 0.01767, + "recall_at_1": 0.03191, + "recall_at_10": 0.12722, + "recall_at_100": 0.36265, + "recall_at_1000": 0.71364, + "recall_at_20": 0.17611, + "recall_at_3": 0.06545, + "recall_at_5": 0.08836 + } + ] + }, + "task_name": "Fashion200kI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Flickr30kI2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/Flickr30kI2TRetrieval.json new file mode 100644 index 0000000000..9759b31060 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/Flickr30kI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb", + "evaluation_time": 207.93018054962158, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.958, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.92171, + "map_at_1": 0.1916, + "map_at_10": 0.86479, + "map_at_100": 0.88173, + "map_at_1000": 0.88183, + "map_at_20": 0.87868, + "map_at_3": 0.53913, + "map_at_5": 0.79289, + "mrr_at_1": 0.958, + "mrr_at_10": 0.9750166666666669, + "mrr_at_100": 0.9750166666666669, + "mrr_at_1000": 0.9750166666666669, + "mrr_at_20": 0.9750166666666669, + "mrr_at_3": 0.9726666666666668, + "mrr_at_5": 0.9750166666666669, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": 0.4916966786714666, + "nauc_cv_recall_at_1_max": 0.6713240851896326, + "nauc_cv_recall_at_1_std": -0.7526010404161642, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": 0.5285714285714299, + "nauc_cv_recall_at_3_max": 0.6155929038281921, + "nauc_cv_recall_at_3_std": -0.9986461251167119, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_map_at_1000_diff1": -0.02045249189608823, + "nauc_map_at_1000_max": 0.48444705367590757, + "nauc_map_at_1000_std": -0.3509317395111607, + "nauc_map_at_100_diff1": -0.02058308054045463, + "nauc_map_at_100_max": 0.4846637323698576, + "nauc_map_at_100_std": -0.35009368037401234, + "nauc_map_at_10_diff1": -0.029243897097944935, + "nauc_map_at_10_max": 0.48560114666485193, + "nauc_map_at_10_std": -0.3487188193186465, + "nauc_map_at_1_diff1": 0.4916966786714697, + "nauc_map_at_1_max": 0.6713240851896303, + "nauc_map_at_1_std": -0.7526010404161716, + "nauc_map_at_20_diff1": -0.020918191278483735, + "nauc_map_at_20_max": 0.4907827196529326, + "nauc_map_at_20_std": -0.3437478022987934, + "nauc_map_at_3_diff1": 0.06404734735826337, + "nauc_map_at_3_max": 0.49496142191054154, + "nauc_map_at_3_std": -0.5206501004530918, + "nauc_map_at_5_diff1": -0.03213800540833977, + "nauc_map_at_5_max": 0.4343110150737827, + "nauc_map_at_5_std": -0.3532847395080826, + "nauc_mrr_at_1000_diff1": 0.5147667695052229, + "nauc_mrr_at_1000_max": 0.6689794441236546, + "nauc_mrr_at_1000_std": -0.7744101420866352, + "nauc_mrr_at_100_diff1": 0.5147667695052229, + "nauc_mrr_at_100_max": 0.6689794441236546, + "nauc_mrr_at_100_std": -0.7744101420866352, + "nauc_mrr_at_10_diff1": 0.5147667695052229, + "nauc_mrr_at_10_max": 0.6689794441236546, + "nauc_mrr_at_10_std": -0.7744101420866352, + "nauc_mrr_at_1_diff1": 0.4916966786714666, + "nauc_mrr_at_1_max": 0.6713240851896326, + "nauc_mrr_at_1_std": -0.7526010404161642, + "nauc_mrr_at_20_diff1": 0.5147667695052229, + "nauc_mrr_at_20_max": 0.6689794441236546, + "nauc_mrr_at_20_std": -0.7744101420866352, + "nauc_mrr_at_3_diff1": 0.5158644758716511, + "nauc_mrr_at_3_max": 0.6622799526314594, + "nauc_mrr_at_3_std": -0.7943937737696769, + "nauc_mrr_at_5_diff1": 0.5147667695052229, + "nauc_mrr_at_5_max": 0.6689794441236546, + "nauc_mrr_at_5_std": -0.7744101420866352, + "nauc_ndcg_at_1000_diff1": 0.03770546246624843, + "nauc_ndcg_at_1000_max": 0.5150471462106003, + "nauc_ndcg_at_1000_std": -0.40462586903272596, + "nauc_ndcg_at_100_diff1": 0.036148575249940015, + "nauc_ndcg_at_100_max": 0.5184447889732504, + "nauc_ndcg_at_100_std": -0.39038829419383936, + "nauc_ndcg_at_10_diff1": 0.0008624674014364987, + "nauc_ndcg_at_10_max": 0.5191981043207383, + "nauc_ndcg_at_10_std": -0.3598665352164584, + "nauc_ndcg_at_1_diff1": 0.4916966786714666, + "nauc_ndcg_at_1_max": 0.6713240851896326, + "nauc_ndcg_at_1_std": -0.7526010404161642, + "nauc_ndcg_at_20_diff1": 0.026377554690408925, + "nauc_ndcg_at_20_max": 0.5383192158414997, + "nauc_ndcg_at_20_std": -0.35091054444218184, + "nauc_ndcg_at_3_diff1": 0.09388531218398928, + "nauc_ndcg_at_3_max": 0.5082388274289871, + "nauc_ndcg_at_3_std": -0.5390436547567337, + "nauc_ndcg_at_5_diff1": -0.014690636108521834, + "nauc_ndcg_at_5_max": 0.44758188335638405, + "nauc_ndcg_at_5_std": -0.37355202546595306, + "nauc_precision_at_1000_diff1": -0.6909430438838531, + "nauc_precision_at_1000_max": -0.29878618113860494, + "nauc_precision_at_1000_std": -0.00466853408005154, + "nauc_precision_at_100_diff1": -0.11266728913794523, + "nauc_precision_at_100_max": 0.5945637514264249, + "nauc_precision_at_100_std": 0.45455960161837233, + "nauc_precision_at_10_diff1": -0.07461466511359147, + "nauc_precision_at_10_max": 0.5239693994783068, + "nauc_precision_at_10_std": -0.20969420193093674, + "nauc_precision_at_1_diff1": 0.4916966786714666, + "nauc_precision_at_1_max": 0.6713240851896326, + "nauc_precision_at_1_std": -0.7526010404161642, + "nauc_precision_at_20_diff1": -0.06213897988573373, + "nauc_precision_at_20_max": 0.6271463387614928, + "nauc_precision_at_20_std": 0.04860814382249958, + "nauc_precision_at_3_diff1": 0.04620363145654133, + "nauc_precision_at_3_max": 0.48641878979090936, + "nauc_precision_at_3_std": -0.5070883890641662, + "nauc_precision_at_5_diff1": -0.05513998536331827, + "nauc_precision_at_5_max": 0.42244786124143063, + "nauc_precision_at_5_std": -0.3237443035379046, + "nauc_recall_at_1000_diff1": -0.69094304388462, + "nauc_recall_at_1000_max": -0.2987861811394452, + "nauc_recall_at_1000_std": -0.0046685340802375035, + "nauc_recall_at_100_diff1": -0.11266728913786052, + "nauc_recall_at_100_max": 0.5945637514265448, + "nauc_recall_at_100_std": 0.45455960161845066, + "nauc_recall_at_10_diff1": -0.07461466511359147, + "nauc_recall_at_10_max": 0.5239693994783068, + "nauc_recall_at_10_std": -0.20969420193093674, + "nauc_recall_at_1_diff1": 0.4916966786714697, + "nauc_recall_at_1_max": 0.6713240851896303, + "nauc_recall_at_1_std": -0.7526010404161716, + "nauc_recall_at_20_diff1": -0.06213897988573373, + "nauc_recall_at_20_max": 0.6271463387614928, + "nauc_recall_at_20_std": 0.04860814382249958, + "nauc_recall_at_3_diff1": 0.046203631456540835, + "nauc_recall_at_3_max": 0.48641878979091213, + "nauc_recall_at_3_std": -0.5070883890641666, + "nauc_recall_at_5_diff1": -0.05513998536331827, + "nauc_recall_at_5_max": 0.42244786124143063, + "nauc_recall_at_5_std": -0.3237443035379046, + "ndcg_at_1": 0.958, + "ndcg_at_10": 0.92171, + "ndcg_at_100": 0.946, + "ndcg_at_1000": 0.94674, + "ndcg_at_20": 0.9393, + "ndcg_at_3": 0.92272, + "ndcg_at_5": 0.85616, + "precision_at_1": 0.958, + "precision_at_10": 0.4682, + "precision_at_100": 0.04982, + "precision_at_1000": 0.005, + "precision_at_20": 0.2441, + "precision_at_3": 0.91233, + "precision_at_5": 0.8186, + "recall_at_1": 0.1916, + "recall_at_10": 0.9364, + "recall_at_100": 0.9964, + "recall_at_1000": 0.9996, + "recall_at_20": 0.9764, + "recall_at_3": 0.5474, + "recall_at_5": 0.8186 + } + ] + }, + "task_name": "Flickr30kI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Flickr30kT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/Flickr30kT2IRetrieval.json new file mode 100644 index 0000000000..8d5d6c98af --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/Flickr30kT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "e819702b287bfbe084e129a61f308a802b7c108e", + "evaluation_time": 211.6292426586151, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.8684, + "cv_recall_at_10": 0.9932, + "cv_recall_at_100": 0.9994, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.9968, + "cv_recall_at_3": 0.9638, + "cv_recall_at_5": 0.9824, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.93746, + "map_at_1": 0.8684, + "map_at_10": 0.91887, + "map_at_100": 0.91921, + "map_at_1000": 0.91921, + "map_at_20": 0.91913, + "map_at_3": 0.91313, + "map_at_5": 0.91739, + "mrr_at_1": 0.8684, + "mrr_at_10": 0.9188720634920634, + "mrr_at_100": 0.9192070964776662, + "mrr_at_1000": 0.9192105992294939, + "mrr_at_20": 0.9191285436579243, + "mrr_at_3": 0.9131333333333338, + "mrr_at_5": 0.9173933333333328, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.9074074074072582, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 0.043106131341511, + "nauc_cv_recall_at_10_diff1": 0.8142884604822204, + "nauc_cv_recall_at_10_max": 0.8789201955291783, + "nauc_cv_recall_at_10_std": -0.24512550118084836, + "nauc_cv_recall_at_1_diff1": 0.8696890409875206, + "nauc_cv_recall_at_1_max": 0.6075508571662983, + "nauc_cv_recall_at_1_std": -0.2787834127846835, + "nauc_cv_recall_at_20_diff1": 0.832954014939291, + "nauc_cv_recall_at_20_max": 0.9302637721755362, + "nauc_cv_recall_at_20_std": -0.30966970121385284, + "nauc_cv_recall_at_3_diff1": 0.8315897261298597, + "nauc_cv_recall_at_3_max": 0.7461736075645744, + "nauc_cv_recall_at_3_std": -0.42221603190078993, + "nauc_cv_recall_at_5_diff1": 0.8321449792038011, + "nauc_cv_recall_at_5_max": 0.8269140989729185, + "nauc_cv_recall_at_5_std": -0.4130485527544341, + "nauc_map_at_1000_diff1": 0.8642366527631479, + "nauc_map_at_1000_max": 0.6351029904336999, + "nauc_map_at_1000_std": -0.2980863362503044, + "nauc_map_at_100_diff1": 0.8642395287230616, + "nauc_map_at_100_max": 0.635119225458973, + "nauc_map_at_100_std": -0.2980619656916447, + "nauc_map_at_10_diff1": 0.8639983629217577, + "nauc_map_at_10_max": 0.6360145331039044, + "nauc_map_at_10_std": -0.2976928255578576, + "nauc_map_at_1_diff1": 0.8696890409875206, + "nauc_map_at_1_max": 0.6075508571662983, + "nauc_map_at_1_std": -0.2787834127846835, + "nauc_map_at_20_diff1": 0.8642205241547971, + "nauc_map_at_20_max": 0.6354286591080467, + "nauc_map_at_20_std": -0.2979018073376867, + "nauc_map_at_3_diff1": 0.8618419652170195, + "nauc_map_at_3_max": 0.6401380939902368, + "nauc_map_at_3_std": -0.3073303296624219, + "nauc_map_at_5_diff1": 0.8634868750280645, + "nauc_map_at_5_max": 0.6387016700954582, + "nauc_map_at_5_std": -0.3019303288726223, + "nauc_mrr_at_1000_diff1": 0.8642366527631479, + "nauc_mrr_at_1000_max": 0.6351029904336999, + "nauc_mrr_at_1000_std": -0.2980863362503044, + "nauc_mrr_at_100_diff1": 0.8642395287230616, + "nauc_mrr_at_100_max": 0.635119225458973, + "nauc_mrr_at_100_std": -0.2980619656916447, + "nauc_mrr_at_10_diff1": 0.8639983629217577, + "nauc_mrr_at_10_max": 0.6360145331039044, + "nauc_mrr_at_10_std": -0.2976928255578576, + "nauc_mrr_at_1_diff1": 0.8696890409875206, + "nauc_mrr_at_1_max": 0.6075508571662983, + "nauc_mrr_at_1_std": -0.2787834127846835, + "nauc_mrr_at_20_diff1": 0.8642205241547971, + "nauc_mrr_at_20_max": 0.6354286591080467, + "nauc_mrr_at_20_std": -0.2979018073376867, + "nauc_mrr_at_3_diff1": 0.8618419652170195, + "nauc_mrr_at_3_max": 0.6401380939902368, + "nauc_mrr_at_3_std": -0.3073303296624219, + "nauc_mrr_at_5_diff1": 0.8634868750280645, + "nauc_mrr_at_5_max": 0.6387016700954582, + "nauc_mrr_at_5_std": -0.3019303288726223, + "nauc_ndcg_at_1000_diff1": 0.8635726458401701, + "nauc_ndcg_at_1000_max": 0.6388763171254515, + "nauc_ndcg_at_1000_std": -0.30013610087668907, + "nauc_ndcg_at_100_diff1": 0.8636378174190988, + "nauc_ndcg_at_100_max": 0.6393648885077013, + "nauc_ndcg_at_100_std": -0.29961527282803385, + "nauc_ndcg_at_10_diff1": 0.8623106702131149, + "nauc_ndcg_at_10_max": 0.6445380433892065, + "nauc_ndcg_at_10_std": -0.2983077230796854, + "nauc_ndcg_at_1_diff1": 0.8696890409875206, + "nauc_ndcg_at_1_max": 0.6075508571662983, + "nauc_ndcg_at_1_std": -0.2787834127846835, + "nauc_ndcg_at_20_diff1": 0.863308876326781, + "nauc_ndcg_at_20_max": 0.64174443168976, + "nauc_ndcg_at_20_std": -0.29975245292051106, + "nauc_ndcg_at_3_diff1": 0.857902077747856, + "nauc_ndcg_at_3_max": 0.6541603888741332, + "nauc_ndcg_at_3_std": -0.3222399570836121, + "nauc_ndcg_at_5_diff1": 0.8611092724054866, + "nauc_ndcg_at_5_max": 0.6524486999880342, + "nauc_ndcg_at_5_std": -0.3105729128710701, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.9074074074072626, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.04310613134107902, + "nauc_precision_at_10_diff1": 0.814288460482237, + "nauc_precision_at_10_max": 0.8789201955291973, + "nauc_precision_at_10_std": -0.2451255011808053, + "nauc_precision_at_1_diff1": 0.8696890409875206, + "nauc_precision_at_1_max": 0.6075508571662983, + "nauc_precision_at_1_std": -0.2787834127846835, + "nauc_precision_at_20_diff1": 0.8329540149393047, + "nauc_precision_at_20_max": 0.9302637721755742, + "nauc_precision_at_20_std": -0.3096697012137737, + "nauc_precision_at_3_diff1": 0.8315897261298597, + "nauc_precision_at_3_max": 0.7461736075645716, + "nauc_precision_at_3_std": -0.42221603190079987, + "nauc_precision_at_5_diff1": 0.8321449792038005, + "nauc_precision_at_5_max": 0.8269140989729217, + "nauc_precision_at_5_std": -0.4130485527544209, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.9074074074072582, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.043106131341511, + "nauc_recall_at_10_diff1": 0.8142884604822204, + "nauc_recall_at_10_max": 0.8789201955291783, + "nauc_recall_at_10_std": -0.24512550118084836, + "nauc_recall_at_1_diff1": 0.8696890409875206, + "nauc_recall_at_1_max": 0.6075508571662983, + "nauc_recall_at_1_std": -0.2787834127846835, + "nauc_recall_at_20_diff1": 0.832954014939291, + "nauc_recall_at_20_max": 0.9302637721755362, + "nauc_recall_at_20_std": -0.30966970121385284, + "nauc_recall_at_3_diff1": 0.8315897261298597, + "nauc_recall_at_3_max": 0.7461736075645744, + "nauc_recall_at_3_std": -0.42221603190078993, + "nauc_recall_at_5_diff1": 0.8321449792038011, + "nauc_recall_at_5_max": 0.8269140989729185, + "nauc_recall_at_5_std": -0.4130485527544341, + "ndcg_at_1": 0.8684, + "ndcg_at_10": 0.93746, + "ndcg_at_100": 0.93888, + "ndcg_at_1000": 0.93896, + "ndcg_at_20": 0.93838, + "ndcg_at_3": 0.92626, + "ndcg_at_5": 0.93393, + "precision_at_1": 0.8684, + "precision_at_10": 0.09932, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "precision_at_20": 0.04984, + "precision_at_3": 0.32127, + "precision_at_5": 0.19648, + "recall_at_1": 0.8684, + "recall_at_10": 0.9932, + "recall_at_100": 0.9994, + "recall_at_1000": 1.0, + "recall_at_20": 0.9968, + "recall_at_3": 0.9638, + "recall_at_5": 0.9824 + } + ] + }, + "task_name": "Flickr30kT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Food101Classification.json b/results-mieb/voyage-multimodal-3/1/Food101Classification.json new file mode 100644 index 0000000000..3012cfd20e --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/Food101Classification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", + "evaluation_time": 7259.677006721497, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "validation": [ + { + "accuracy": 0.8738376237623763, + "f1": 0.8720096472259975, + "f1_weighted": 0.8720096472259975, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8738376237623763, + "scores_per_experiment": [ + { + "accuracy": 0.8794851485148515, + "f1": 0.8778787574787363, + "f1_weighted": 0.8778787574787366 + }, + { + "accuracy": 0.8722772277227723, + "f1": 0.8703819262029309, + "f1_weighted": 0.8703819262029308 + }, + { + "accuracy": 0.8743762376237624, + "f1": 0.8726621943156567, + "f1_weighted": 0.8726621943156568 + }, + { + "accuracy": 0.8725148514851485, + "f1": 0.87059138861965, + "f1_weighted": 0.8705913886196499 + }, + { + "accuracy": 0.8705346534653465, + "f1": 0.8685339695130132, + "f1_weighted": 0.8685339695130132 + } + ] + } + ] + }, + "task_name": "Food101Classification" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Food101ZeroShot.json b/results-mieb/voyage-multimodal-3/1/Food101ZeroShot.json new file mode 100644 index 0000000000..d17bf8f3d0 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/Food101ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", + "evaluation_time": 5834.1527643203735, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "validation": [ + { + "accuracy": 0.7442376237623762, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7442376237623762 + } + ] + }, + "task_name": "Food101ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/GTSRB.json b/results-mieb/voyage-multimodal-3/1/GTSRB.json new file mode 100644 index 0000000000..acdcbd69f5 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/GTSRB.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", + "evaluation_time": 652.1987547874451, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.755249406175772, + "f1": 0.6720088924349847, + "f1_weighted": 0.7683155357055661, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.755249406175772, + "scores_per_experiment": [ + { + "accuracy": 0.7548693586698337, + "f1": 0.6718940774766187, + "f1_weighted": 0.7714544889259504 + }, + { + "accuracy": 0.7595407759303247, + "f1": 0.6797291490699253, + "f1_weighted": 0.7732718458370778 + }, + { + "accuracy": 0.7379255740300871, + "f1": 0.6516528744630659, + "f1_weighted": 0.7463279498290893 + }, + { + "accuracy": 0.7685669041963579, + "f1": 0.6830864810804418, + "f1_weighted": 0.7791283842787832 + }, + { + "accuracy": 0.7553444180522565, + "f1": 0.6736818800848718, + "f1_weighted": 0.7713950096569298 + } + ] + } + ] + }, + "task_name": "GTSRB" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/GTSRBZeroShot.json b/results-mieb/voyage-multimodal-3/1/GTSRBZeroShot.json new file mode 100644 index 0000000000..6bbaeadc54 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/GTSRBZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", + "evaluation_time": 535.8853387832642, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5511480601741885, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5511480601741885 + } + ] + }, + "task_name": "GTSRBZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/HatefulMemesI2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/HatefulMemesI2TRetrieval.json new file mode 100644 index 0000000000..62c7d29be8 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/HatefulMemesI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", + "evaluation_time": 650.209198474884, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.351, + "cv_recall_at_10": 0.748, + "cv_recall_at_100": 0.928, + "cv_recall_at_1000": 0.992, + "cv_recall_at_20": 0.817, + "cv_recall_at_3": 0.602, + "cv_recall_at_5": 0.674, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.55506, + "map_at_1": 0.355, + "map_at_10": 0.49261, + "map_at_100": 0.50018, + "map_at_1000": 0.5005, + "map_at_20": 0.49749, + "map_at_3": 0.46617, + "map_at_5": 0.48247, + "mrr_at_1": 0.351, + "mrr_at_10": 0.49080277777777725, + "mrr_at_100": 0.49846476493880365, + "mrr_at_1000": 0.49878931223846307, + "mrr_at_20": 0.49577409491952756, + "mrr_at_3": 0.4644999999999996, + "mrr_at_5": 0.4807999999999994, + "nauc_cv_recall_at_1000_diff1": 0.07621381886086587, + "nauc_cv_recall_at_1000_max": 0.8082983193277192, + "nauc_cv_recall_at_1000_std": 0.17851307189543436, + "nauc_cv_recall_at_100_diff1": 0.09988717709305933, + "nauc_cv_recall_at_100_max": 0.6147032887228979, + "nauc_cv_recall_at_100_std": 0.24981196182176554, + "nauc_cv_recall_at_10_diff1": 0.06197436174726118, + "nauc_cv_recall_at_10_max": 0.5141043664889241, + "nauc_cv_recall_at_10_std": 0.041529061400370854, + "nauc_cv_recall_at_1_diff1": 0.3850282742742979, + "nauc_cv_recall_at_1_max": 0.3175284686558156, + "nauc_cv_recall_at_1_std": -0.10299980100603963, + "nauc_cv_recall_at_20_diff1": 0.07301231742174892, + "nauc_cv_recall_at_20_max": 0.4605582162247186, + "nauc_cv_recall_at_20_std": 0.01682458928368046, + "nauc_cv_recall_at_3_diff1": 0.08599160592529945, + "nauc_cv_recall_at_3_max": 0.45713749542019533, + "nauc_cv_recall_at_3_std": -0.007640454793230996, + "nauc_cv_recall_at_5_diff1": 0.07195517798082428, + "nauc_cv_recall_at_5_max": 0.49091067338837635, + "nauc_cv_recall_at_5_std": 0.005269966311808489, + "nauc_map_at_1000_diff1": 0.2533107101417881, + "nauc_map_at_1000_max": 0.38972900961589796, + "nauc_map_at_1000_std": -0.06353846682756097, + "nauc_map_at_100_diff1": 0.2531591611847431, + "nauc_map_at_100_max": 0.38989054771157217, + "nauc_map_at_100_std": -0.06323456068673915, + "nauc_map_at_10_diff1": 0.24912377115374532, + "nauc_map_at_10_max": 0.3926449653117099, + "nauc_map_at_10_std": -0.062216164680505115, + "nauc_map_at_1_diff1": 0.3730153823894987, + "nauc_map_at_1_max": 0.3316677298644587, + "nauc_map_at_1_std": -0.10641009621391966, + "nauc_map_at_20_diff1": 0.25148518715541224, + "nauc_map_at_20_max": 0.38937799165745557, + "nauc_map_at_20_std": -0.06397118323619402, + "nauc_map_at_3_diff1": 0.24482055059226032, + "nauc_map_at_3_max": 0.38720954729341184, + "nauc_map_at_3_std": -0.06613319316079966, + "nauc_map_at_5_diff1": 0.24685220069553152, + "nauc_map_at_5_max": 0.3920841183102921, + "nauc_map_at_5_std": -0.06464680615951957, + "nauc_mrr_at_1000_diff1": 0.2585096359144679, + "nauc_mrr_at_1000_max": 0.38199432135718614, + "nauc_mrr_at_1000_std": -0.05978602963891485, + "nauc_mrr_at_100_diff1": 0.258352622577515, + "nauc_mrr_at_100_max": 0.38216389801366024, + "nauc_mrr_at_100_std": -0.059485839290383584, + "nauc_mrr_at_10_diff1": 0.25416421261734345, + "nauc_mrr_at_10_max": 0.38504436738237263, + "nauc_mrr_at_10_std": -0.05867002311891825, + "nauc_mrr_at_1_diff1": 0.3850282742742979, + "nauc_mrr_at_1_max": 0.3175284686558156, + "nauc_mrr_at_1_std": -0.10299980100603963, + "nauc_mrr_at_20_diff1": 0.25663329766992704, + "nauc_mrr_at_20_max": 0.38169566832125074, + "nauc_mrr_at_20_std": -0.060261518241809524, + "nauc_mrr_at_3_diff1": 0.2500928845703025, + "nauc_mrr_at_3_max": 0.37966197148104364, + "nauc_mrr_at_3_std": -0.06330230125980266, + "nauc_mrr_at_5_diff1": 0.25167385837686956, + "nauc_mrr_at_5_max": 0.384322454672087, + "nauc_mrr_at_5_std": -0.06189888851605895, + "nauc_ndcg_at_1000_diff1": 0.23587589947343313, + "nauc_ndcg_at_1000_max": 0.4022544236196415, + "nauc_ndcg_at_1000_std": -0.05176148282936083, + "nauc_ndcg_at_100_diff1": 0.2318767422114613, + "nauc_ndcg_at_100_max": 0.40775700069666154, + "nauc_ndcg_at_100_std": -0.04240628044464809, + "nauc_ndcg_at_10_diff1": 0.21177187255605082, + "nauc_ndcg_at_10_max": 0.41614606723035347, + "nauc_ndcg_at_10_std": -0.04210221725280141, + "nauc_ndcg_at_1_diff1": 0.3730153823894987, + "nauc_ndcg_at_1_max": 0.3316677298644587, + "nauc_ndcg_at_1_std": -0.10641009621391966, + "nauc_ndcg_at_20_diff1": 0.22009221253620218, + "nauc_ndcg_at_20_max": 0.4035476208118783, + "nauc_ndcg_at_20_std": -0.04956172046512765, + "nauc_ndcg_at_3_diff1": 0.2054629176787454, + "nauc_ndcg_at_3_max": 0.4044812807732281, + "nauc_ndcg_at_3_std": -0.05122687481333138, + "nauc_ndcg_at_5_diff1": 0.20751438759573293, + "nauc_ndcg_at_5_max": 0.41420346702045324, + "nauc_ndcg_at_5_std": -0.04793996715762123, + "nauc_precision_at_1000_diff1": 0.07621381886085157, + "nauc_precision_at_1000_max": 0.8082983193277065, + "nauc_precision_at_1000_std": 0.17851307189538645, + "nauc_precision_at_100_diff1": 0.09988717709305533, + "nauc_precision_at_100_max": 0.6147032887228965, + "nauc_precision_at_100_std": 0.2498119618217622, + "nauc_precision_at_10_diff1": 0.06162567259966079, + "nauc_precision_at_10_max": 0.5152149919569698, + "nauc_precision_at_10_std": 0.04418137397062698, + "nauc_precision_at_1_diff1": 0.3730153823894987, + "nauc_precision_at_1_max": 0.3316677298644587, + "nauc_precision_at_1_std": -0.10641009621391966, + "nauc_precision_at_20_diff1": 0.07301231742174608, + "nauc_precision_at_20_max": 0.46055821622471715, + "nauc_precision_at_20_std": 0.016824589283678037, + "nauc_precision_at_3_diff1": 0.08407925817588253, + "nauc_precision_at_3_max": 0.45783005126402626, + "nauc_precision_at_3_std": -0.0043206548450612, + "nauc_precision_at_5_diff1": 0.07281530201617145, + "nauc_precision_at_5_max": 0.49198582843256056, + "nauc_precision_at_5_std": 0.011004126547455258, + "nauc_recall_at_1000_diff1": 0.07621381886086587, + "nauc_recall_at_1000_max": 0.8082983193277192, + "nauc_recall_at_1000_std": 0.17851307189543436, + "nauc_recall_at_100_diff1": 0.09988717709305933, + "nauc_recall_at_100_max": 0.6147032887228979, + "nauc_recall_at_100_std": 0.24981196182176554, + "nauc_recall_at_10_diff1": 0.061625672599661785, + "nauc_recall_at_10_max": 0.5152149919569704, + "nauc_recall_at_10_std": 0.044181373970627394, + "nauc_recall_at_1_diff1": 0.3730153823894987, + "nauc_recall_at_1_max": 0.3316677298644587, + "nauc_recall_at_1_std": -0.10641009621391966, + "nauc_recall_at_20_diff1": 0.07301231742174892, + "nauc_recall_at_20_max": 0.4605582162247186, + "nauc_recall_at_20_std": 0.01682458928368046, + "nauc_recall_at_3_diff1": 0.08407925817588281, + "nauc_recall_at_3_max": 0.4578300512640267, + "nauc_recall_at_3_std": -0.004320654845060782, + "nauc_recall_at_5_diff1": 0.07281530201617166, + "nauc_recall_at_5_max": 0.49198582843256045, + "nauc_recall_at_5_std": 0.011004126547455015, + "ndcg_at_1": 0.355, + "ndcg_at_10": 0.55506, + "ndcg_at_100": 0.59275, + "ndcg_at_1000": 0.60097, + "ndcg_at_20": 0.57245, + "ndcg_at_3": 0.50115, + "ndcg_at_5": 0.53067, + "precision_at_1": 0.355, + "precision_at_10": 0.0749, + "precision_at_100": 0.00928, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04085, + "precision_at_3": 0.20067, + "precision_at_5": 0.1348, + "recall_at_1": 0.355, + "recall_at_10": 0.749, + "recall_at_100": 0.928, + "recall_at_1000": 0.992, + "recall_at_20": 0.817, + "recall_at_3": 0.602, + "recall_at_5": 0.674 + } + ] + }, + "task_name": "HatefulMemesI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/InfoSeekIT2ITRetrieval.json b/results-mieb/voyage-multimodal-3/1/InfoSeekIT2ITRetrieval.json new file mode 100644 index 0000000000..66f630de1e --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/InfoSeekIT2ITRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "78ee7f7708aac75d3afac5dcab1c9e03cb62664c", + "evaluation_time": 65952.82875466347, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.28403, + "cv_recall_at_10": 0.65719, + "cv_recall_at_100": 0.89257, + "cv_recall_at_1000": 0.97874, + "cv_recall_at_20": 0.75001, + "cv_recall_at_3": 0.47144, + "cv_recall_at_5": 0.55545, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.33953, + "map_at_1": 0.08145, + "map_at_10": 0.21206, + "map_at_100": 0.28257, + "map_at_1000": 0.28611, + "map_at_20": 0.24427, + "map_at_3": 0.14147, + "map_at_5": 0.17264, + "mrr_at_1": 0.28403342238390267, + "mrr_at_10": 0.3996135051910026, + "mrr_at_100": 0.41000322037538744, + "mrr_at_1000": 0.41041189646994997, + "mrr_at_20": 0.4061427559913534, + "mrr_at_3": 0.36668940298224206, + "mrr_at_5": 0.3859271680024156, + "nauc_cv_recall_at_1000_diff1": 0.006255086097224733, + "nauc_cv_recall_at_1000_max": 0.7865445077141644, + "nauc_cv_recall_at_1000_std": 0.8924394061114759, + "nauc_cv_recall_at_100_diff1": -0.013434457267599696, + "nauc_cv_recall_at_100_max": 0.593648393742511, + "nauc_cv_recall_at_100_std": 0.6319873871338207, + "nauc_cv_recall_at_10_diff1": 0.022263238930525824, + "nauc_cv_recall_at_10_max": 0.39839916458848434, + "nauc_cv_recall_at_10_std": 0.2557897885542902, + "nauc_cv_recall_at_1_diff1": 0.01822031583099016, + "nauc_cv_recall_at_1_max": 0.2281715481495325, + "nauc_cv_recall_at_1_std": 0.1716855759958051, + "nauc_cv_recall_at_20_diff1": 0.010652852309121942, + "nauc_cv_recall_at_20_max": 0.4417588169147453, + "nauc_cv_recall_at_20_std": 0.3408777750831783, + "nauc_cv_recall_at_3_diff1": 0.03860765592580623, + "nauc_cv_recall_at_3_max": 0.3102534078441316, + "nauc_cv_recall_at_3_std": 0.1910136003216648, + "nauc_cv_recall_at_5_diff1": 0.025886440425177976, + "nauc_cv_recall_at_5_max": 0.3427393218302827, + "nauc_cv_recall_at_5_std": 0.203679095765542, + "nauc_map_at_1000_diff1": 0.04518541449257291, + "nauc_map_at_1000_max": 0.3083486535985005, + "nauc_map_at_1000_std": 0.19210317667174406, + "nauc_map_at_100_diff1": 0.04724029064314546, + "nauc_map_at_100_max": 0.311711329509263, + "nauc_map_at_100_std": 0.18956658646818914, + "nauc_map_at_10_diff1": 0.08946183195431161, + "nauc_map_at_10_max": 0.2622402614931031, + "nauc_map_at_10_std": 0.024433285370009744, + "nauc_map_at_1_diff1": 0.15993382767564207, + "nauc_map_at_1_max": 0.1073051172298578, + "nauc_map_at_1_std": -0.03657308891148322, + "nauc_map_at_20_diff1": 0.06913273401030226, + "nauc_map_at_20_max": 0.29230128274413936, + "nauc_map_at_20_std": 0.08624682957270653, + "nauc_map_at_3_diff1": 0.13700226942401322, + "nauc_map_at_3_max": 0.18402074614649278, + "nauc_map_at_3_std": -0.017710823650508613, + "nauc_map_at_5_diff1": 0.11491927496912738, + "nauc_map_at_5_max": 0.2200920168842469, + "nauc_map_at_5_std": -0.011576798777768498, + "nauc_mrr_at_1000_diff1": 0.025121567896100157, + "nauc_mrr_at_1000_max": 0.27882633572887006, + "nauc_mrr_at_1000_std": 0.19289373369702298, + "nauc_mrr_at_100_diff1": 0.025086948333758748, + "nauc_mrr_at_100_max": 0.2791026960702822, + "nauc_mrr_at_100_std": 0.19325678818734734, + "nauc_mrr_at_10_diff1": 0.02555860944367029, + "nauc_mrr_at_10_max": 0.2784044740210458, + "nauc_mrr_at_10_std": 0.188423836265687, + "nauc_mrr_at_1_diff1": 0.01822031583099016, + "nauc_mrr_at_1_max": 0.2281715481495325, + "nauc_mrr_at_1_std": 0.1716855759958051, + "nauc_mrr_at_20_diff1": 0.025081365238416566, + "nauc_mrr_at_20_max": 0.278957933480566, + "nauc_mrr_at_20_std": 0.191588318130598, + "nauc_mrr_at_3_diff1": 0.028847425808651184, + "nauc_mrr_at_3_max": 0.2667595353588465, + "nauc_mrr_at_3_std": 0.1806448364151367, + "nauc_mrr_at_5_diff1": 0.02610524326290375, + "nauc_mrr_at_5_max": 0.2732974796911029, + "nauc_mrr_at_5_std": 0.18325469633434754, + "nauc_ndcg_at_1000_diff1": 0.028503470874592905, + "nauc_ndcg_at_1000_max": 0.3216882468647445, + "nauc_ndcg_at_1000_std": 0.2528746110996414, + "nauc_ndcg_at_100_diff1": 0.03207688712074031, + "nauc_ndcg_at_100_max": 0.34667674488041317, + "nauc_ndcg_at_100_std": 0.27229705056106845, + "nauc_ndcg_at_10_diff1": 0.038192209224090994, + "nauc_ndcg_at_10_max": 0.30684379429067615, + "nauc_ndcg_at_10_std": 0.16647699668688587, + "nauc_ndcg_at_1_diff1": 0.01822031583099016, + "nauc_ndcg_at_1_max": 0.2281715481495325, + "nauc_ndcg_at_1_std": 0.1716855759958051, + "nauc_ndcg_at_20_diff1": 0.034938762725658365, + "nauc_ndcg_at_20_max": 0.3264420320738274, + "nauc_ndcg_at_20_std": 0.19022825198564514, + "nauc_ndcg_at_3_diff1": 0.038729342507024075, + "nauc_ndcg_at_3_max": 0.2687825534865985, + "nauc_ndcg_at_3_std": 0.17768045830564053, + "nauc_ndcg_at_5_diff1": 0.03748423929351206, + "nauc_ndcg_at_5_max": 0.28150109846155363, + "nauc_ndcg_at_5_std": 0.16672113376409234, + "nauc_precision_at_1000_diff1": -0.08799871966231278, + "nauc_precision_at_1000_max": 0.12781925314854187, + "nauc_precision_at_1000_std": 0.37277258907442795, + "nauc_precision_at_100_diff1": -0.08320462963055977, + "nauc_precision_at_100_max": 0.2051136071484949, + "nauc_precision_at_100_std": 0.45246451743688454, + "nauc_precision_at_10_diff1": -0.05976693193763614, + "nauc_precision_at_10_max": 0.2968576435115502, + "nauc_precision_at_10_std": 0.2844011395283614, + "nauc_precision_at_1_diff1": 0.01822031583099016, + "nauc_precision_at_1_max": 0.2281715481495325, + "nauc_precision_at_1_std": 0.1716855759958051, + "nauc_precision_at_20_diff1": -0.07361102895671524, + "nauc_precision_at_20_max": 0.2754586684079379, + "nauc_precision_at_20_std": 0.35559497309161087, + "nauc_precision_at_3_diff1": -0.010106575994914213, + "nauc_precision_at_3_max": 0.29340247188852653, + "nauc_precision_at_3_std": 0.22318287453672572, + "nauc_precision_at_5_diff1": -0.03836290413957423, + "nauc_precision_at_5_max": 0.3018368815002715, + "nauc_precision_at_5_std": 0.23624481906440406, + "nauc_recall_at_1000_diff1": -0.008059356781186513, + "nauc_recall_at_1000_max": 0.2304098551929707, + "nauc_recall_at_1000_std": 0.3354339325308451, + "nauc_recall_at_100_diff1": 0.01083728828629382, + "nauc_recall_at_100_max": 0.3836673140413695, + "nauc_recall_at_100_std": 0.42905365137980256, + "nauc_recall_at_10_diff1": 0.06556309837226998, + "nauc_recall_at_10_max": 0.2599183481627226, + "nauc_recall_at_10_std": 0.012531473925899825, + "nauc_recall_at_1_diff1": 0.15993382767564207, + "nauc_recall_at_1_max": 0.1073051172298578, + "nauc_recall_at_1_std": -0.03657308891148322, + "nauc_recall_at_20_diff1": 0.040179941335428404, + "nauc_recall_at_20_max": 0.3090495656432163, + "nauc_recall_at_20_std": 0.11374639642332243, + "nauc_recall_at_3_diff1": 0.1258408660747518, + "nauc_recall_at_3_max": 0.1745820829297796, + "nauc_recall_at_3_std": -0.0251053671266273, + "nauc_recall_at_5_diff1": 0.09408305297650396, + "nauc_recall_at_5_max": 0.20631551260094838, + "nauc_recall_at_5_std": -0.026805180940384123, + "ndcg_at_1": 0.28403, + "ndcg_at_10": 0.33953, + "ndcg_at_100": 0.43251, + "ndcg_at_1000": 0.45665, + "ndcg_at_20": 0.37096, + "ndcg_at_3": 0.29659, + "ndcg_at_5": 0.31142, + "precision_at_1": 0.28403, + "precision_at_10": 0.19173, + "precision_at_100": 0.05487, + "precision_at_1000": 0.00649, + "precision_at_20": 0.14789, + "precision_at_3": 0.25429, + "precision_at_5": 0.23093, + "recall_at_1": 0.08145, + "recall_at_10": 0.36199, + "recall_at_100": 0.68958, + "recall_at_1000": 0.79688, + "recall_at_20": 0.47438, + "recall_at_3": 0.18495, + "recall_at_5": 0.25407 + } + ] + }, + "task_name": "InfoSeekIT2ITRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/InfoSeekIT2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/InfoSeekIT2TRetrieval.json new file mode 100644 index 0000000000..9aac076ec1 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/InfoSeekIT2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "d4f4606f7a42bbf311c2957419ef3734fe81c47f", + "evaluation_time": 6618.2249002456665, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.21046, + "cv_recall_at_10": 0.50799, + "cv_recall_at_100": 0.74565, + "cv_recall_at_1000": 0.9048, + "cv_recall_at_20": 0.58792, + "cv_recall_at_3": 0.35741, + "cv_recall_at_5": 0.4233, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.22647, + "map_at_1": 0.06088, + "map_at_10": 0.14196, + "map_at_100": 0.17037, + "map_at_1000": 0.17435, + "map_at_20": 0.15631, + "map_at_3": 0.10603, + "map_at_5": 0.12422, + "mrr_at_1": 0.21045659277576614, + "mrr_at_10": 0.3008787774287208, + "mrr_at_100": 0.31040670679487925, + "mrr_at_1000": 0.311117781969453, + "mrr_at_20": 0.30655042035402164, + "mrr_at_3": 0.274662191998589, + "mrr_at_5": 0.28961847566899324, + "nauc_cv_recall_at_1000_diff1": 0.17824996811865748, + "nauc_cv_recall_at_1000_max": 0.47987402979389815, + "nauc_cv_recall_at_1000_std": 0.6816462895154791, + "nauc_cv_recall_at_100_diff1": 0.1843143682914582, + "nauc_cv_recall_at_100_max": 0.37905438003510206, + "nauc_cv_recall_at_100_std": 0.5699429810427198, + "nauc_cv_recall_at_10_diff1": 0.15012587648149436, + "nauc_cv_recall_at_10_max": 0.32368974301707476, + "nauc_cv_recall_at_10_std": 0.37409116505337014, + "nauc_cv_recall_at_1_diff1": 0.2097167948031123, + "nauc_cv_recall_at_1_max": 0.2497542239386446, + "nauc_cv_recall_at_1_std": 0.19016398114295133, + "nauc_cv_recall_at_20_diff1": 0.1618159923376429, + "nauc_cv_recall_at_20_max": 0.35277316434660777, + "nauc_cv_recall_at_20_std": 0.4508148652376852, + "nauc_cv_recall_at_3_diff1": 0.17779863645022098, + "nauc_cv_recall_at_3_max": 0.3111112573039595, + "nauc_cv_recall_at_3_std": 0.27563634786514596, + "nauc_cv_recall_at_5_diff1": 0.16305905635229093, + "nauc_cv_recall_at_5_max": 0.3162609882047919, + "nauc_cv_recall_at_5_std": 0.317778809950664, + "nauc_map_at_1000_diff1": 0.18062629663019164, + "nauc_map_at_1000_max": 0.30887386534256817, + "nauc_map_at_1000_std": 0.27501630146891326, + "nauc_map_at_100_diff1": 0.18194687531246523, + "nauc_map_at_100_max": 0.3088618001377212, + "nauc_map_at_100_std": 0.26949909365789393, + "nauc_map_at_10_diff1": 0.19995134632470404, + "nauc_map_at_10_max": 0.2866006053364188, + "nauc_map_at_10_std": 0.19399104469893683, + "nauc_map_at_1_diff1": 0.2387998551076001, + "nauc_map_at_1_max": 0.19279908750721483, + "nauc_map_at_1_std": 0.07437905413912614, + "nauc_map_at_20_diff1": 0.19175528798673863, + "nauc_map_at_20_max": 0.3029648366966638, + "nauc_map_at_20_std": 0.23512831620110114, + "nauc_map_at_3_diff1": 0.21024928339969556, + "nauc_map_at_3_max": 0.2514882852013589, + "nauc_map_at_3_std": 0.12580882754076886, + "nauc_map_at_5_diff1": 0.20305860756214764, + "nauc_map_at_5_max": 0.2736987803243673, + "nauc_map_at_5_std": 0.15805778659737624, + "nauc_mrr_at_1000_diff1": 0.18931473336796248, + "nauc_mrr_at_1000_max": 0.2822758075273321, + "nauc_mrr_at_1000_std": 0.2530585297752362, + "nauc_mrr_at_100_diff1": 0.18933364810876388, + "nauc_mrr_at_100_max": 0.28237055263016825, + "nauc_mrr_at_100_std": 0.2533934194623867, + "nauc_mrr_at_10_diff1": 0.1876819617265076, + "nauc_mrr_at_10_max": 0.28034735163378566, + "nauc_mrr_at_10_std": 0.2477265488055595, + "nauc_mrr_at_1_diff1": 0.2097167948031123, + "nauc_mrr_at_1_max": 0.2497542239386446, + "nauc_mrr_at_1_std": 0.19016398114295133, + "nauc_mrr_at_20_diff1": 0.18886764488043434, + "nauc_mrr_at_20_max": 0.28232620395091035, + "nauc_mrr_at_20_std": 0.2524704902573781, + "nauc_mrr_at_3_diff1": 0.19305391522666082, + "nauc_mrr_at_3_max": 0.2780885313791518, + "nauc_mrr_at_3_std": 0.22948759954151343, + "nauc_mrr_at_5_diff1": 0.18917066668515167, + "nauc_mrr_at_5_max": 0.27927262304030254, + "nauc_mrr_at_5_std": 0.23992936705688953, + "nauc_ndcg_at_1000_diff1": 0.1717625966155328, + "nauc_ndcg_at_1000_max": 0.32055194498333217, + "nauc_ndcg_at_1000_std": 0.3628761510517062, + "nauc_ndcg_at_100_diff1": 0.17792896235260067, + "nauc_ndcg_at_100_max": 0.3242135904641071, + "nauc_ndcg_at_100_std": 0.35207271116750893, + "nauc_ndcg_at_10_diff1": 0.17208049806241, + "nauc_ndcg_at_10_max": 0.3142723749184195, + "nauc_ndcg_at_10_std": 0.28137422414888263, + "nauc_ndcg_at_1_diff1": 0.2097167948031123, + "nauc_ndcg_at_1_max": 0.2497542239386446, + "nauc_ndcg_at_1_std": 0.19016398114295133, + "nauc_ndcg_at_20_diff1": 0.1747475140985028, + "nauc_ndcg_at_20_max": 0.3230887123924791, + "nauc_ndcg_at_20_std": 0.30773262192430567, + "nauc_ndcg_at_3_diff1": 0.1742869872978061, + "nauc_ndcg_at_3_max": 0.29735164173310114, + "nauc_ndcg_at_3_std": 0.23763176754395007, + "nauc_ndcg_at_5_diff1": 0.16989028753503907, + "nauc_ndcg_at_5_max": 0.3073545116664261, + "nauc_ndcg_at_5_std": 0.25833572648743863, + "nauc_precision_at_1000_diff1": -0.021454119112792058, + "nauc_precision_at_1000_max": 0.11979743941369668, + "nauc_precision_at_1000_std": 0.2915837925495167, + "nauc_precision_at_100_diff1": 0.012985694748761932, + "nauc_precision_at_100_max": 0.2099562509077403, + "nauc_precision_at_100_std": 0.407278331288373, + "nauc_precision_at_10_diff1": 0.09491876075712896, + "nauc_precision_at_10_max": 0.3032449465495486, + "nauc_precision_at_10_std": 0.35930676023805985, + "nauc_precision_at_1_diff1": 0.2097167948031123, + "nauc_precision_at_1_max": 0.2497542239386446, + "nauc_precision_at_1_std": 0.19016398114295133, + "nauc_precision_at_20_diff1": 0.0642121578016976, + "nauc_precision_at_20_max": 0.2990014577834674, + "nauc_precision_at_20_std": 0.4111485581458398, + "nauc_precision_at_3_diff1": 0.14574681544852458, + "nauc_precision_at_3_max": 0.31432409778474146, + "nauc_precision_at_3_std": 0.27721588283927534, + "nauc_precision_at_5_diff1": 0.1190395919590016, + "nauc_precision_at_5_max": 0.31560001369555274, + "nauc_precision_at_5_std": 0.3122611999979494, + "nauc_recall_at_1000_diff1": 0.11212176965567214, + "nauc_recall_at_1000_max": 0.2670091990423596, + "nauc_recall_at_1000_std": 0.486655313646483, + "nauc_recall_at_100_diff1": 0.14036796429967138, + "nauc_recall_at_100_max": 0.30035112693144195, + "nauc_recall_at_100_std": 0.4466945003332598, + "nauc_recall_at_10_diff1": 0.14160363251091304, + "nauc_recall_at_10_max": 0.2703810658821937, + "nauc_recall_at_10_std": 0.24857153753984734, + "nauc_recall_at_1_diff1": 0.2387998551076001, + "nauc_recall_at_1_max": 0.19279908750721483, + "nauc_recall_at_1_std": 0.07437905413912614, + "nauc_recall_at_20_diff1": 0.13385820717289768, + "nauc_recall_at_20_max": 0.2951208204679184, + "nauc_recall_at_20_std": 0.3320829918125117, + "nauc_recall_at_3_diff1": 0.17579515035676807, + "nauc_recall_at_3_max": 0.25216176892291103, + "nauc_recall_at_3_std": 0.14486396283814276, + "nauc_recall_at_5_diff1": 0.15470956333707822, + "nauc_recall_at_5_max": 0.26649716278842084, + "nauc_recall_at_5_std": 0.19097762883436376, + "ndcg_at_1": 0.21046, + "ndcg_at_10": 0.22647, + "ndcg_at_100": 0.29013, + "ndcg_at_1000": 0.33478, + "ndcg_at_20": 0.24561, + "ndcg_at_3": 0.2057, + "ndcg_at_5": 0.21249, + "precision_at_1": 0.21046, + "precision_at_10": 0.10974, + "precision_at_100": 0.02777, + "precision_at_1000": 0.00449, + "precision_at_20": 0.0786, + "precision_at_3": 0.1741, + "precision_at_5": 0.14828, + "recall_at_1": 0.06088, + "recall_at_10": 0.24851, + "recall_at_100": 0.48191, + "recall_at_1000": 0.68299, + "recall_at_20": 0.32062, + "recall_at_3": 0.14024, + "recall_at_5": 0.18554 + } + ] + }, + "task_name": "InfoSeekIT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/MNIST.json b/results-mieb/voyage-multimodal-3/1/MNIST.json new file mode 100644 index 0000000000..3c45e466be --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/MNIST.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 355.2941155433655, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.94478, + "f1": 0.9455916630851033, + "f1_weighted": 0.9451557749481427, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94478, + "scores_per_experiment": [ + { + "accuracy": 0.9443, + "f1": 0.945215965609092, + "f1_weighted": 0.9446586990733822 + }, + { + "accuracy": 0.9403, + "f1": 0.9402495134915263, + "f1_weighted": 0.9403240114833532 + }, + { + "accuracy": 0.9468, + "f1": 0.947810464896407, + "f1_weighted": 0.9472892702769644 + }, + { + "accuracy": 0.9484, + "f1": 0.9494772820351903, + "f1_weighted": 0.9489311434850805 + }, + { + "accuracy": 0.9441, + "f1": 0.9452050893933018, + "f1_weighted": 0.9445757504219333 + } + ] + } + ] + }, + "task_name": "MNIST" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/MNISTZeroShot.json b/results-mieb/voyage-multimodal-3/1/MNISTZeroShot.json new file mode 100644 index 0000000000..4f6ea48d74 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/MNISTZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", + "evaluation_time": 305.5367000102997, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.8883, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8883 + } + ] + }, + "task_name": "MNISTZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/MSCOCOI2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/MSCOCOI2TRetrieval.json new file mode 100644 index 0000000000..d42c9cc67d --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/MSCOCOI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", + "evaluation_time": 774.8769903182983, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.6884, + "cv_recall_at_10": 0.9402, + "cv_recall_at_100": 0.9994, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.978, + "cv_recall_at_3": 0.8396, + "cv_recall_at_5": 0.89, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.62155, + "map_at_1": 0.13774, + "map_at_10": 0.50639, + "map_at_100": 0.55195, + "map_at_1000": 0.55375, + "map_at_20": 0.53359, + "map_at_3": 0.33166, + "map_at_5": 0.44173, + "mrr_at_1": 0.6884, + "mrr_at_10": 0.7739456349206367, + "mrr_at_100": 0.777367826450281, + "mrr_at_1000": 0.777372600703197, + "mrr_at_20": 0.7767063895117622, + "mrr_at_3": 0.7555333333333364, + "mrr_at_5": 0.7670633333333351, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.38702147525650704, + "nauc_cv_recall_at_100_max": 0.8513849984437857, + "nauc_cv_recall_at_100_std": 0.6095549330843056, + "nauc_cv_recall_at_10_diff1": 0.32301727826024573, + "nauc_cv_recall_at_10_max": 0.5230288324917483, + "nauc_cv_recall_at_10_std": -0.8189186488419226, + "nauc_cv_recall_at_1_diff1": 0.39010303247213973, + "nauc_cv_recall_at_1_max": 0.40557934934779377, + "nauc_cv_recall_at_1_std": -0.5562807759674807, + "nauc_cv_recall_at_20_diff1": 0.32278669043374947, + "nauc_cv_recall_at_20_max": 0.6116246498599451, + "nauc_cv_recall_at_20_std": -0.8675367116543555, + "nauc_cv_recall_at_3_diff1": 0.3461162635215054, + "nauc_cv_recall_at_3_max": 0.4234316032826311, + "nauc_cv_recall_at_3_std": -0.7107640615089226, + "nauc_cv_recall_at_5_diff1": 0.29777940039515494, + "nauc_cv_recall_at_5_max": 0.4556661798814545, + "nauc_cv_recall_at_5_std": -0.7423692122669878, + "nauc_map_at_1000_diff1": 0.15389759883779974, + "nauc_map_at_1000_max": 0.4039851793741281, + "nauc_map_at_1000_std": -0.5002106432329312, + "nauc_map_at_100_diff1": 0.15400793732497264, + "nauc_map_at_100_max": 0.4046850658511939, + "nauc_map_at_100_std": -0.5006753334767017, + "nauc_map_at_10_diff1": 0.14417566577977628, + "nauc_map_at_10_max": 0.3943356064629755, + "nauc_map_at_10_std": -0.4933375568345805, + "nauc_map_at_1_diff1": 0.39061782604629997, + "nauc_map_at_1_max": 0.40501007654030197, + "nauc_map_at_1_std": -0.5557214169508682, + "nauc_map_at_20_diff1": 0.15204311998140219, + "nauc_map_at_20_max": 0.4010186472342661, + "nauc_map_at_20_std": -0.5030459911389683, + "nauc_map_at_3_diff1": 0.19751394364383174, + "nauc_map_at_3_max": 0.39371898829057755, + "nauc_map_at_3_std": -0.48573229813847163, + "nauc_map_at_5_diff1": 0.1446331422469198, + "nauc_map_at_5_max": 0.38494293310597494, + "nauc_map_at_5_std": -0.46770254575698766, + "nauc_mrr_at_1000_diff1": 0.3763516616711763, + "nauc_mrr_at_1000_max": 0.4150495687962729, + "nauc_mrr_at_1000_std": -0.5983744526758652, + "nauc_mrr_at_100_diff1": 0.37633456843660706, + "nauc_mrr_at_100_max": 0.4150600564683398, + "nauc_mrr_at_100_std": -0.5983453993911649, + "nauc_mrr_at_10_diff1": 0.3754618101299754, + "nauc_mrr_at_10_max": 0.41645403348086485, + "nauc_mrr_at_10_std": -0.6020064260805211, + "nauc_mrr_at_1_diff1": 0.39010303247213973, + "nauc_mrr_at_1_max": 0.40557934934779377, + "nauc_mrr_at_1_std": -0.5562807759674807, + "nauc_mrr_at_20_diff1": 0.3761781415674808, + "nauc_mrr_at_20_max": 0.41572849241686777, + "nauc_mrr_at_20_std": -0.5993859021710495, + "nauc_mrr_at_3_diff1": 0.3763720703975814, + "nauc_mrr_at_3_max": 0.41205472109725566, + "nauc_mrr_at_3_std": -0.6039347601343806, + "nauc_mrr_at_5_diff1": 0.37176138944258397, + "nauc_mrr_at_5_max": 0.4152519913506332, + "nauc_mrr_at_5_std": -0.6027741075483298, + "nauc_ndcg_at_1000_diff1": 0.20766514945642406, + "nauc_ndcg_at_1000_max": 0.41934210758284596, + "nauc_ndcg_at_1000_std": -0.5364815413635738, + "nauc_ndcg_at_100_diff1": 0.20424384491896133, + "nauc_ndcg_at_100_max": 0.4273533185423817, + "nauc_ndcg_at_100_std": -0.535102542515517, + "nauc_ndcg_at_10_diff1": 0.1791208806960087, + "nauc_ndcg_at_10_max": 0.4073289016660168, + "nauc_ndcg_at_10_std": -0.5368173592120352, + "nauc_ndcg_at_1_diff1": 0.39010303247213973, + "nauc_ndcg_at_1_max": 0.40557934934779377, + "nauc_ndcg_at_1_std": -0.5562807759674807, + "nauc_ndcg_at_20_diff1": 0.19275176413625172, + "nauc_ndcg_at_20_max": 0.41722170756908916, + "nauc_ndcg_at_20_std": -0.5492816531157821, + "nauc_ndcg_at_3_diff1": 0.225181694942575, + "nauc_ndcg_at_3_max": 0.399396123599283, + "nauc_ndcg_at_3_std": -0.5164886537494516, + "nauc_ndcg_at_5_diff1": 0.1794399278507418, + "nauc_ndcg_at_5_max": 0.39482220207783547, + "nauc_ndcg_at_5_std": -0.5061581424107108, + "nauc_precision_at_1000_diff1": -0.051082798710932215, + "nauc_precision_at_1000_max": 0.40066564260112725, + "nauc_precision_at_1000_std": 0.28745118118929563, + "nauc_precision_at_100_diff1": 0.112982571814616, + "nauc_precision_at_100_max": 0.44836567702541963, + "nauc_precision_at_100_std": -0.37237152343030927, + "nauc_precision_at_10_diff1": 0.11378576600186772, + "nauc_precision_at_10_max": 0.38720465164652307, + "nauc_precision_at_10_std": -0.5159158443916068, + "nauc_precision_at_1_diff1": 0.39010303247213973, + "nauc_precision_at_1_max": 0.40557934934779377, + "nauc_precision_at_1_std": -0.5562807759674807, + "nauc_precision_at_20_diff1": 0.12756821213828962, + "nauc_precision_at_20_max": 0.39267144771467427, + "nauc_precision_at_20_std": -0.5230524294306809, + "nauc_precision_at_3_diff1": 0.18112368731828446, + "nauc_precision_at_3_max": 0.39285170555391796, + "nauc_precision_at_3_std": -0.5018078606210227, + "nauc_precision_at_5_diff1": 0.12367663772807914, + "nauc_precision_at_5_max": 0.38035879461638944, + "nauc_precision_at_5_std": -0.4811535038733751, + "nauc_recall_at_1000_diff1": 0.01710277606974907, + "nauc_recall_at_1000_max": 0.46253786067273645, + "nauc_recall_at_1000_std": 0.29927580788410824, + "nauc_recall_at_100_diff1": 0.11703851756088672, + "nauc_recall_at_100_max": 0.45101915757917294, + "nauc_recall_at_100_std": -0.37542345400577626, + "nauc_recall_at_10_diff1": 0.11430396402305408, + "nauc_recall_at_10_max": 0.3869821152574072, + "nauc_recall_at_10_std": -0.5162572501669658, + "nauc_recall_at_1_diff1": 0.39061782604629997, + "nauc_recall_at_1_max": 0.40501007654030197, + "nauc_recall_at_1_std": -0.5557214169508682, + "nauc_recall_at_20_diff1": 0.12861124356686862, + "nauc_recall_at_20_max": 0.3930253578243864, + "nauc_recall_at_20_std": -0.5239274880060881, + "nauc_recall_at_3_diff1": 0.18169641263240677, + "nauc_recall_at_3_max": 0.3925592388545264, + "nauc_recall_at_3_std": -0.5019990921945948, + "nauc_recall_at_5_diff1": 0.1240600740311391, + "nauc_recall_at_5_max": 0.38014532371394416, + "nauc_recall_at_5_std": -0.48140440409252105, + "ndcg_at_1": 0.6884, + "ndcg_at_10": 0.62155, + "ndcg_at_100": 0.72631, + "ndcg_at_1000": 0.7415, + "ndcg_at_20": 0.66921, + "ndcg_at_3": 0.61503, + "ndcg_at_5": 0.54496, + "precision_at_1": 0.6884, + "precision_at_10": 0.31916, + "precision_at_100": 0.04645, + "precision_at_1000": 0.00498, + "precision_at_20": 0.18725, + "precision_at_3": 0.593, + "precision_at_5": 0.4996, + "recall_at_1": 0.13774, + "recall_at_10": 0.63858, + "recall_at_100": 0.92951, + "recall_at_1000": 0.99672, + "recall_at_20": 0.74932, + "recall_at_3": 0.35597, + "recall_at_5": 0.49981 + } + ] + }, + "task_name": "MSCOCOI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/MSCOCOT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/MSCOCOT2IRetrieval.json new file mode 100644 index 0000000000..1fda5ba6ae --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/MSCOCOT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", + "evaluation_time": 798.1402442455292, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.54553, + "cv_recall_at_10": 0.8692, + "cv_recall_at_100": 0.99222, + "cv_recall_at_1000": 0.99851, + "cv_recall_at_20": 0.9287, + "cv_recall_at_3": 0.72784, + "cv_recall_at_5": 0.7952, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.70362, + "map_at_1": 0.54513, + "map_at_10": 0.65107, + "map_at_100": 0.65729, + "map_at_1000": 0.65733, + "map_at_20": 0.65538, + "map_at_3": 0.62561, + "map_at_5": 0.64106, + "mrr_at_1": 0.5455278326413802, + "mrr_at_10": 0.6515759449815842, + "mrr_at_100": 0.6576602730040336, + "mrr_at_1000": 0.6576998696355355, + "mrr_at_20": 0.655811964604866, + "mrr_at_3": 0.6261638921359333, + "mrr_at_5": 0.6415957918497558, + "nauc_cv_recall_at_1000_diff1": 0.5913883028915704, + "nauc_cv_recall_at_1000_max": 0.9645439962109348, + "nauc_cv_recall_at_1000_std": 0.9018481347746059, + "nauc_cv_recall_at_100_diff1": 0.5534699359027497, + "nauc_cv_recall_at_100_max": 0.6861774550140799, + "nauc_cv_recall_at_100_std": 0.09057132259190011, + "nauc_cv_recall_at_10_diff1": 0.5839186752981996, + "nauc_cv_recall_at_10_max": 0.3798440454573067, + "nauc_cv_recall_at_10_std": -0.6611604991204363, + "nauc_cv_recall_at_1_diff1": 0.7167795833336132, + "nauc_cv_recall_at_1_max": 0.3455331044071751, + "nauc_cv_recall_at_1_std": -0.47493901255939786, + "nauc_cv_recall_at_20_diff1": 0.5748112965125677, + "nauc_cv_recall_at_20_max": 0.41011759638021583, + "nauc_cv_recall_at_20_std": -0.6670384673426152, + "nauc_cv_recall_at_3_diff1": 0.6251011207741684, + "nauc_cv_recall_at_3_max": 0.35786814470274086, + "nauc_cv_recall_at_3_std": -0.5911605866207055, + "nauc_cv_recall_at_5_diff1": 0.5980744554890782, + "nauc_cv_recall_at_5_max": 0.35849158393841046, + "nauc_cv_recall_at_5_std": -0.627844625782769, + "nauc_map_at_1000_diff1": 0.6828832422396731, + "nauc_map_at_1000_max": 0.35112835281277105, + "nauc_map_at_1000_std": -0.5195228699055376, + "nauc_map_at_100_diff1": 0.6828661396882304, + "nauc_map_at_100_max": 0.351159422258553, + "nauc_map_at_100_std": -0.519495008275733, + "nauc_map_at_10_diff1": 0.6810098053755158, + "nauc_map_at_10_max": 0.35100061273843497, + "nauc_map_at_10_std": -0.5234568934205572, + "nauc_map_at_1_diff1": 0.7171262914350959, + "nauc_map_at_1_max": 0.3453959690825254, + "nauc_map_at_1_std": -0.4759877819944269, + "nauc_map_at_20_diff1": 0.6821066111194438, + "nauc_map_at_20_max": 0.3512342628943979, + "nauc_map_at_20_std": -0.5213520952561683, + "nauc_map_at_3_diff1": 0.6814472223356598, + "nauc_map_at_3_max": 0.35014496482511315, + "nauc_map_at_3_std": -0.5211132052373082, + "nauc_map_at_5_diff1": 0.6794310461672968, + "nauc_map_at_5_max": 0.3500457603735776, + "nauc_map_at_5_std": -0.524235589581454, + "nauc_mrr_at_1000_diff1": 0.6824923157391112, + "nauc_mrr_at_1000_max": 0.35120387598639335, + "nauc_mrr_at_1000_std": -0.5181617693570029, + "nauc_mrr_at_100_diff1": 0.6824742872964704, + "nauc_mrr_at_100_max": 0.3512335430834624, + "nauc_mrr_at_100_std": -0.5181247949793334, + "nauc_mrr_at_10_diff1": 0.6805412633139878, + "nauc_mrr_at_10_max": 0.3511525454141477, + "nauc_mrr_at_10_std": -0.5216875823342743, + "nauc_mrr_at_1_diff1": 0.7167795833336132, + "nauc_mrr_at_1_max": 0.3455331044071751, + "nauc_mrr_at_1_std": -0.47493901255939786, + "nauc_mrr_at_20_diff1": 0.6816926085638637, + "nauc_mrr_at_20_max": 0.3513213303961809, + "nauc_mrr_at_20_std": -0.5197619427467188, + "nauc_mrr_at_3_diff1": 0.6808933173811453, + "nauc_mrr_at_3_max": 0.3502752099018471, + "nauc_mrr_at_3_std": -0.5194553383921099, + "nauc_mrr_at_5_diff1": 0.6789105553784537, + "nauc_mrr_at_5_max": 0.35021487576792626, + "nauc_mrr_at_5_std": -0.5224301407248029, + "nauc_ndcg_at_1000_diff1": 0.6774926006858535, + "nauc_ndcg_at_1000_max": 0.3536992083970472, + "nauc_ndcg_at_1000_std": -0.5230396169325919, + "nauc_ndcg_at_100_diff1": 0.6769870449597041, + "nauc_ndcg_at_100_max": 0.3547231738920983, + "nauc_ndcg_at_100_std": -0.521607000189851, + "nauc_ndcg_at_10_diff1": 0.6673294220619244, + "nauc_ndcg_at_10_max": 0.3546008945512635, + "nauc_ndcg_at_10_std": -0.5429692294158105, + "nauc_ndcg_at_1_diff1": 0.7167795833336132, + "nauc_ndcg_at_1_max": 0.3455331044071751, + "nauc_ndcg_at_1_std": -0.47493901255939786, + "nauc_ndcg_at_20_diff1": 0.6715391142197418, + "nauc_ndcg_at_20_max": 0.3556597729668483, + "nauc_ndcg_at_20_std": -0.5355709965917, + "nauc_ndcg_at_3_diff1": 0.6689984648994431, + "nauc_ndcg_at_3_max": 0.35184351638444844, + "nauc_ndcg_at_3_std": -0.536740668922552, + "nauc_ndcg_at_5_diff1": 0.6641807917915493, + "nauc_ndcg_at_5_max": 0.3516502816268339, + "nauc_ndcg_at_5_std": -0.5439790194614311, + "nauc_precision_at_1000_diff1": -0.29051394786528345, + "nauc_precision_at_1000_max": 0.007168455625121431, + "nauc_precision_at_1000_std": 0.7814796737105516, + "nauc_precision_at_100_diff1": -0.01673301799858297, + "nauc_precision_at_100_max": 0.213340763947866, + "nauc_precision_at_100_std": 0.4911809823345084, + "nauc_precision_at_10_diff1": 0.5604152027291861, + "nauc_precision_at_10_max": 0.36862480817292415, + "nauc_precision_at_10_std": -0.6305299231273225, + "nauc_precision_at_1_diff1": 0.7167795833336132, + "nauc_precision_at_1_max": 0.3455331044071751, + "nauc_precision_at_1_std": -0.47493901255939786, + "nauc_precision_at_20_diff1": 0.5008270292935011, + "nauc_precision_at_20_max": 0.3736600387723433, + "nauc_precision_at_20_std": -0.5638978451960841, + "nauc_precision_at_3_diff1": 0.6231722745154151, + "nauc_precision_at_3_max": 0.3570527652098802, + "nauc_precision_at_3_std": -0.5889077632335837, + "nauc_precision_at_5_diff1": 0.591025634855629, + "nauc_precision_at_5_max": 0.35518613527368537, + "nauc_precision_at_5_std": -0.6201665718909292, + "nauc_recall_at_1000_diff1": 0.5913883028915704, + "nauc_recall_at_1000_max": 0.9645439962109348, + "nauc_recall_at_1000_std": 0.9018481347746059, + "nauc_recall_at_100_diff1": 0.5555338594243105, + "nauc_recall_at_100_max": 0.6846457428695056, + "nauc_recall_at_100_std": 0.07067487943143345, + "nauc_recall_at_10_diff1": 0.585589719395042, + "nauc_recall_at_10_max": 0.37847409422814565, + "nauc_recall_at_10_std": -0.6673654162116813, + "nauc_recall_at_1_diff1": 0.7171262914350959, + "nauc_recall_at_1_max": 0.3453959690825254, + "nauc_recall_at_1_std": -0.4759877819944269, + "nauc_recall_at_20_diff1": 0.5761021751313068, + "nauc_recall_at_20_max": 0.4087068955598216, + "nauc_recall_at_20_std": -0.6760273988990346, + "nauc_recall_at_3_diff1": 0.6259496325614421, + "nauc_recall_at_3_max": 0.3575239405717769, + "nauc_recall_at_3_std": -0.5937210978019355, + "nauc_recall_at_5_diff1": 0.5993470174348666, + "nauc_recall_at_5_max": 0.3578894732514209, + "nauc_recall_at_5_std": -0.6316698946209255, + "ndcg_at_1": 0.54553, + "ndcg_at_10": 0.70362, + "ndcg_at_100": 0.7314, + "ndcg_at_1000": 0.73227, + "ndcg_at_20": 0.71892, + "ndcg_at_3": 0.65181, + "ndcg_at_5": 0.67958, + "precision_at_1": 0.54553, + "precision_at_10": 0.0871, + "precision_at_100": 0.00999, + "precision_at_1000": 0.00101, + "precision_at_20": 0.0466, + "precision_at_3": 0.24271, + "precision_at_5": 0.1592, + "recall_at_1": 0.54513, + "recall_at_10": 0.86799, + "recall_at_100": 0.99213, + "recall_at_1000": 0.99851, + "recall_at_20": 0.9277, + "recall_at_3": 0.72695, + "recall_at_5": 0.79419 + } + ] + }, + "task_name": "MSCOCOT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/NIGHTSI2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/NIGHTSI2IRetrieval.json new file mode 100644 index 0000000000..7dfde434cd --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/NIGHTSI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", + "evaluation_time": 4715.950295448303, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.08255, + "cv_recall_at_10": 0.48113, + "cv_recall_at_100": 0.94387, + "cv_recall_at_1000": 0.99245, + "cv_recall_at_20": 0.70236, + "cv_recall_at_3": 0.20047, + "cv_recall_at_5": 0.29575, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.24863, + "map_at_1": 0.08255, + "map_at_10": 0.17858, + "map_at_100": 0.20121, + "map_at_1000": 0.20146, + "map_at_20": 0.19397, + "map_at_3": 0.13255, + "map_at_5": 0.1542, + "mrr_at_1": 0.08254716981132075, + "mrr_at_10": 0.1785770440251568, + "mrr_at_100": 0.201205823403381, + "mrr_at_1000": 0.20145930114431737, + "mrr_at_20": 0.19396875697904437, + "mrr_at_3": 0.1325471698113209, + "mrr_at_5": 0.15419811320754678, + "nauc_cv_recall_at_1000_diff1": 0.038077731092443015, + "nauc_cv_recall_at_1000_max": 0.8064309056956054, + "nauc_cv_recall_at_1000_std": 0.8878092903828085, + "nauc_cv_recall_at_100_diff1": -0.06317821246145294, + "nauc_cv_recall_at_100_max": 0.39503644595092996, + "nauc_cv_recall_at_100_std": 0.3795557438661738, + "nauc_cv_recall_at_10_diff1": 0.022939846901184988, + "nauc_cv_recall_at_10_max": 0.07836573883290258, + "nauc_cv_recall_at_10_std": -0.022496534672011292, + "nauc_cv_recall_at_1_diff1": 0.056265859174364684, + "nauc_cv_recall_at_1_max": 0.0379362949902258, + "nauc_cv_recall_at_1_std": -0.05939974701981684, + "nauc_cv_recall_at_20_diff1": -0.013337017151137259, + "nauc_cv_recall_at_20_max": 0.1534706439353441, + "nauc_cv_recall_at_20_std": -0.03275147830296917, + "nauc_cv_recall_at_3_diff1": 0.0775182887163471, + "nauc_cv_recall_at_3_max": 0.05816325366059642, + "nauc_cv_recall_at_3_std": -0.03549171253904066, + "nauc_cv_recall_at_5_diff1": 0.07408983351399633, + "nauc_cv_recall_at_5_max": 0.0633687792374156, + "nauc_cv_recall_at_5_std": -0.04857622730560427, + "nauc_map_at_1000_diff1": 0.05657786707298797, + "nauc_map_at_1000_max": 0.06293274475496655, + "nauc_map_at_1000_std": -0.04234689425652198, + "nauc_map_at_100_diff1": 0.056474202269853196, + "nauc_map_at_100_max": 0.06318118028967631, + "nauc_map_at_100_std": -0.042139181358562774, + "nauc_map_at_10_diff1": 0.057573823422659154, + "nauc_map_at_10_max": 0.05735556955687778, + "nauc_map_at_10_std": -0.04449564938916585, + "nauc_map_at_1_diff1": 0.056265859174364684, + "nauc_map_at_1_max": 0.0379362949902258, + "nauc_map_at_1_std": -0.05939974701981684, + "nauc_map_at_20_diff1": 0.055360882192033056, + "nauc_map_at_20_max": 0.06295645186107313, + "nauc_map_at_20_std": -0.046234789714899104, + "nauc_map_at_3_diff1": 0.06704928481972888, + "nauc_map_at_3_max": 0.05150166325545375, + "nauc_map_at_3_std": -0.04372918706365568, + "nauc_map_at_5_diff1": 0.0674125668851068, + "nauc_map_at_5_max": 0.05419732008304022, + "nauc_map_at_5_std": -0.0486370059907162, + "nauc_mrr_at_1000_diff1": 0.05657786707298797, + "nauc_mrr_at_1000_max": 0.06293274475496655, + "nauc_mrr_at_1000_std": -0.04234689425652198, + "nauc_mrr_at_100_diff1": 0.056474202269853196, + "nauc_mrr_at_100_max": 0.06318118028967631, + "nauc_mrr_at_100_std": -0.042139181358562774, + "nauc_mrr_at_10_diff1": 0.057573823422659154, + "nauc_mrr_at_10_max": 0.05735556955687778, + "nauc_mrr_at_10_std": -0.04449564938916585, + "nauc_mrr_at_1_diff1": 0.056265859174364684, + "nauc_mrr_at_1_max": 0.0379362949902258, + "nauc_mrr_at_1_std": -0.05939974701981684, + "nauc_mrr_at_20_diff1": 0.055360882192033056, + "nauc_mrr_at_20_max": 0.06295645186107313, + "nauc_mrr_at_20_std": -0.046234789714899104, + "nauc_mrr_at_3_diff1": 0.06704928481972888, + "nauc_mrr_at_3_max": 0.05150166325545375, + "nauc_mrr_at_3_std": -0.04372918706365568, + "nauc_mrr_at_5_diff1": 0.0674125668851068, + "nauc_mrr_at_5_max": 0.05419732008304022, + "nauc_mrr_at_5_std": -0.0486370059907162, + "nauc_ndcg_at_1000_diff1": 0.05077518917317683, + "nauc_ndcg_at_1000_max": 0.07723869390595439, + "nauc_ndcg_at_1000_std": -0.028531105281549857, + "nauc_ndcg_at_100_diff1": 0.046878435984643675, + "nauc_ndcg_at_100_max": 0.08518688575096077, + "nauc_ndcg_at_100_std": -0.018849997307570036, + "nauc_ndcg_at_10_diff1": 0.04776145512022865, + "nauc_ndcg_at_10_max": 0.06415267011217751, + "nauc_ndcg_at_10_std": -0.03795328556951406, + "nauc_ndcg_at_1_diff1": 0.056265859174364684, + "nauc_ndcg_at_1_max": 0.0379362949902258, + "nauc_ndcg_at_1_std": -0.05939974701981684, + "nauc_ndcg_at_20_diff1": 0.039820501697777864, + "nauc_ndcg_at_20_max": 0.08391303154450644, + "nauc_ndcg_at_20_std": -0.042736635918916216, + "nauc_ndcg_at_3_diff1": 0.07038185031455026, + "nauc_ndcg_at_3_max": 0.05372259050957707, + "nauc_ndcg_at_3_std": -0.040999397625074596, + "nauc_ndcg_at_5_diff1": 0.06984543170279325, + "nauc_ndcg_at_5_max": 0.0574112976802138, + "nauc_ndcg_at_5_std": -0.048435863215603926, + "nauc_precision_at_1000_diff1": 0.03807773109245917, + "nauc_precision_at_1000_max": 0.806430905695622, + "nauc_precision_at_1000_std": 0.8878092903828303, + "nauc_precision_at_100_diff1": -0.06317821246145515, + "nauc_precision_at_100_max": 0.3950364459509304, + "nauc_precision_at_100_std": 0.3795557438661729, + "nauc_precision_at_10_diff1": 0.022939846901185168, + "nauc_precision_at_10_max": 0.07836573883290233, + "nauc_precision_at_10_std": -0.02249653467201138, + "nauc_precision_at_1_diff1": 0.056265859174364684, + "nauc_precision_at_1_max": 0.0379362949902258, + "nauc_precision_at_1_std": -0.05939974701981684, + "nauc_precision_at_20_diff1": -0.013337017151137004, + "nauc_precision_at_20_max": 0.15347064393534382, + "nauc_precision_at_20_std": -0.03275147830296866, + "nauc_precision_at_3_diff1": 0.07751828871634699, + "nauc_precision_at_3_max": 0.05816325366059655, + "nauc_precision_at_3_std": -0.03549171253904067, + "nauc_precision_at_5_diff1": 0.07408983351399642, + "nauc_precision_at_5_max": 0.06336877923741566, + "nauc_precision_at_5_std": -0.048576227305604174, + "nauc_recall_at_1000_diff1": 0.038077731092443015, + "nauc_recall_at_1000_max": 0.8064309056956054, + "nauc_recall_at_1000_std": 0.8878092903828085, + "nauc_recall_at_100_diff1": -0.06317821246145294, + "nauc_recall_at_100_max": 0.39503644595092996, + "nauc_recall_at_100_std": 0.3795557438661738, + "nauc_recall_at_10_diff1": 0.022939846901184988, + "nauc_recall_at_10_max": 0.07836573883290258, + "nauc_recall_at_10_std": -0.022496534672011292, + "nauc_recall_at_1_diff1": 0.056265859174364684, + "nauc_recall_at_1_max": 0.0379362949902258, + "nauc_recall_at_1_std": -0.05939974701981684, + "nauc_recall_at_20_diff1": -0.013337017151137259, + "nauc_recall_at_20_max": 0.1534706439353441, + "nauc_recall_at_20_std": -0.03275147830296917, + "nauc_recall_at_3_diff1": 0.0775182887163471, + "nauc_recall_at_3_max": 0.05816325366059642, + "nauc_recall_at_3_std": -0.03549171253904066, + "nauc_recall_at_5_diff1": 0.07408983351399633, + "nauc_recall_at_5_max": 0.0633687792374156, + "nauc_recall_at_5_std": -0.04857622730560427, + "ndcg_at_1": 0.08255, + "ndcg_at_10": 0.24863, + "ndcg_at_100": 0.35147, + "ndcg_at_1000": 0.35776, + "ndcg_at_20": 0.30462, + "ndcg_at_3": 0.14991, + "ndcg_at_5": 0.18904, + "precision_at_1": 0.08255, + "precision_at_10": 0.04811, + "precision_at_100": 0.00944, + "precision_at_1000": 0.00099, + "precision_at_20": 0.03512, + "precision_at_3": 0.06682, + "precision_at_5": 0.05915, + "recall_at_1": 0.08255, + "recall_at_10": 0.48113, + "recall_at_100": 0.94387, + "recall_at_1000": 0.99245, + "recall_at_20": 0.70236, + "recall_at_3": 0.20047, + "recall_at_5": 0.29575 + } + ] + }, + "task_name": "NIGHTSI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/OVENIT2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/OVENIT2TRetrieval.json new file mode 100644 index 0000000000..cea3ced09d --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/OVENIT2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2192074af29422bc1dc41cf07936f198b8c69bd0", + "evaluation_time": 12739.220294475555, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.21298, + "cv_recall_at_10": 0.44988, + "cv_recall_at_100": 0.67133, + "cv_recall_at_1000": 0.84835, + "cv_recall_at_20": 0.51906, + "cv_recall_at_3": 0.32843, + "cv_recall_at_5": 0.37901, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.16397, + "map_at_1": 0.08389, + "map_at_10": 0.11264, + "map_at_100": 0.11625, + "map_at_1000": 0.11655, + "map_at_20": 0.1147, + "map_at_3": 0.10413, + "map_at_5": 0.10875, + "mrr_at_1": 0.21298296136309094, + "mrr_at_10": 0.28446166370183723, + "mrr_at_100": 0.29304291594034704, + "mrr_at_1000": 0.29376949339180636, + "mrr_at_20": 0.2892705097899374, + "mrr_at_3": 0.2634555902194559, + "mrr_at_5": 0.2750026664533632, + "nauc_cv_recall_at_1000_diff1": 0.23906549629288248, + "nauc_cv_recall_at_1000_max": 0.2807966191767593, + "nauc_cv_recall_at_1000_std": 0.4505710122579727, + "nauc_cv_recall_at_100_diff1": 0.2906996072526638, + "nauc_cv_recall_at_100_max": 0.24695181642790698, + "nauc_cv_recall_at_100_std": 0.3413980973312369, + "nauc_cv_recall_at_10_diff1": 0.3323472647610039, + "nauc_cv_recall_at_10_max": 0.2507787156281116, + "nauc_cv_recall_at_10_std": 0.18034362446270058, + "nauc_cv_recall_at_1_diff1": 0.46901333074242085, + "nauc_cv_recall_at_1_max": 0.26293584739784476, + "nauc_cv_recall_at_1_std": 0.09393200644418588, + "nauc_cv_recall_at_20_diff1": 0.3171006136473147, + "nauc_cv_recall_at_20_max": 0.2492534303712411, + "nauc_cv_recall_at_20_std": 0.22178558628756295, + "nauc_cv_recall_at_3_diff1": 0.3719133094010799, + "nauc_cv_recall_at_3_max": 0.2579938533946551, + "nauc_cv_recall_at_3_std": 0.1326339382457508, + "nauc_cv_recall_at_5_diff1": 0.35066227486275997, + "nauc_cv_recall_at_5_max": 0.2540836536889673, + "nauc_cv_recall_at_5_std": 0.14999446316316967, + "nauc_map_at_1000_diff1": 0.348010615372264, + "nauc_map_at_1000_max": 0.15014507094893012, + "nauc_map_at_1000_std": 0.1717617324425307, + "nauc_map_at_100_diff1": 0.34798249017698873, + "nauc_map_at_100_max": 0.15043936623182197, + "nauc_map_at_100_std": 0.1715964239960731, + "nauc_map_at_10_diff1": 0.34956012147948906, + "nauc_map_at_10_max": 0.15358517354268658, + "nauc_map_at_10_std": 0.166532234878434, + "nauc_map_at_1_diff1": 0.4229161887175077, + "nauc_map_at_1_max": 0.18563915762564656, + "nauc_map_at_1_std": 0.13930048980022755, + "nauc_map_at_20_diff1": 0.34840795188919627, + "nauc_map_at_20_max": 0.15201542141647068, + "nauc_map_at_20_std": 0.16963394984544367, + "nauc_map_at_3_diff1": 0.36108195136505084, + "nauc_map_at_3_max": 0.16270134475236106, + "nauc_map_at_3_std": 0.157948835875016, + "nauc_map_at_5_diff1": 0.3537755602811475, + "nauc_map_at_5_max": 0.15750835781201705, + "nauc_map_at_5_std": 0.162398077711519, + "nauc_mrr_at_1000_diff1": 0.41148171312784426, + "nauc_mrr_at_1000_max": 0.2589681398234147, + "nauc_mrr_at_1000_std": 0.1250147526328914, + "nauc_mrr_at_100_diff1": 0.4114161400120679, + "nauc_mrr_at_100_max": 0.25888484323358824, + "nauc_mrr_at_100_std": 0.12503103420716968, + "nauc_mrr_at_10_diff1": 0.4125753728196354, + "nauc_mrr_at_10_max": 0.2591621686708534, + "nauc_mrr_at_10_std": 0.1200428951411931, + "nauc_mrr_at_1_diff1": 0.46901333074242085, + "nauc_mrr_at_1_max": 0.26293584739784476, + "nauc_mrr_at_1_std": 0.09393200644418588, + "nauc_mrr_at_20_diff1": 0.41153820316953676, + "nauc_mrr_at_20_max": 0.25899728906177827, + "nauc_mrr_at_20_std": 0.12291882408761697, + "nauc_mrr_at_3_diff1": 0.4211323790196329, + "nauc_mrr_at_3_max": 0.26058816436358323, + "nauc_mrr_at_3_std": 0.11128421021474276, + "nauc_mrr_at_5_diff1": 0.4154901347685823, + "nauc_mrr_at_5_max": 0.2596997935959254, + "nauc_mrr_at_5_std": 0.1158003575154667, + "nauc_ndcg_at_1000_diff1": 0.31991443787696583, + "nauc_ndcg_at_1000_max": 0.13581124148871299, + "nauc_ndcg_at_1000_std": 0.19355178755325522, + "nauc_ndcg_at_100_diff1": 0.3224029160648352, + "nauc_ndcg_at_100_max": 0.14466078255754752, + "nauc_ndcg_at_100_std": 0.19269940771241528, + "nauc_ndcg_at_10_diff1": 0.349283174454767, + "nauc_ndcg_at_10_max": 0.17453802215625974, + "nauc_ndcg_at_10_std": 0.16998673115401805, + "nauc_ndcg_at_1_diff1": 0.46901333074242085, + "nauc_ndcg_at_1_max": 0.26293584739784476, + "nauc_ndcg_at_1_std": 0.09393200644418588, + "nauc_ndcg_at_20_diff1": 0.33437010159127156, + "nauc_ndcg_at_20_max": 0.1596751444117642, + "nauc_ndcg_at_20_std": 0.18048101211742304, + "nauc_ndcg_at_3_diff1": 0.3921220985778911, + "nauc_ndcg_at_3_max": 0.2239505034815312, + "nauc_ndcg_at_3_std": 0.14162546946540802, + "nauc_ndcg_at_5_diff1": 0.3715600153430542, + "nauc_ndcg_at_5_max": 0.1999499608847638, + "nauc_ndcg_at_5_std": 0.15656366009111194, + "nauc_precision_at_1000_diff1": 0.239065496292881, + "nauc_precision_at_1000_max": 0.28079661917675847, + "nauc_precision_at_1000_std": 0.45057101225797214, + "nauc_precision_at_100_diff1": 0.29069960725266336, + "nauc_precision_at_100_max": 0.24695181642790687, + "nauc_precision_at_100_std": 0.34139809733123694, + "nauc_precision_at_10_diff1": 0.3323472647610031, + "nauc_precision_at_10_max": 0.25077871562811144, + "nauc_precision_at_10_std": 0.1803436244627001, + "nauc_precision_at_1_diff1": 0.46901333074242085, + "nauc_precision_at_1_max": 0.26293584739784476, + "nauc_precision_at_1_std": 0.09393200644418588, + "nauc_precision_at_20_diff1": 0.31710061364731434, + "nauc_precision_at_20_max": 0.2492534303712408, + "nauc_precision_at_20_std": 0.22178558628756279, + "nauc_precision_at_3_diff1": 0.37191330940108014, + "nauc_precision_at_3_max": 0.257993853394655, + "nauc_precision_at_3_std": 0.13263393824575104, + "nauc_precision_at_5_diff1": 0.3506622748627595, + "nauc_precision_at_5_max": 0.25408365368896674, + "nauc_precision_at_5_std": 0.1499944631631691, + "nauc_recall_at_1000_diff1": 0.10001578043238427, + "nauc_recall_at_1000_max": -0.04662176275850385, + "nauc_recall_at_1000_std": 0.1900591135811226, + "nauc_recall_at_100_diff1": 0.15271776326483577, + "nauc_recall_at_100_max": 0.017407724603021528, + "nauc_recall_at_100_std": 0.21645494602605883, + "nauc_recall_at_10_diff1": 0.22719647788274416, + "nauc_recall_at_10_max": 0.08859833048166742, + "nauc_recall_at_10_std": 0.18795701765526512, + "nauc_recall_at_1_diff1": 0.4229161887175077, + "nauc_recall_at_1_max": 0.18563915762564656, + "nauc_recall_at_1_std": 0.13930048980022755, + "nauc_recall_at_20_diff1": 0.20317559817494252, + "nauc_recall_at_20_max": 0.069132885674504, + "nauc_recall_at_20_std": 0.2030970361116893, + "nauc_recall_at_3_diff1": 0.2881138434807705, + "nauc_recall_at_3_max": 0.1332165142075036, + "nauc_recall_at_3_std": 0.1689131041304317, + "nauc_recall_at_5_diff1": 0.2575729755739856, + "nauc_recall_at_5_max": 0.11253154184070889, + "nauc_recall_at_5_std": 0.1761186732999266, + "ndcg_at_1": 0.21298, + "ndcg_at_10": 0.16397, + "ndcg_at_100": 0.17741, + "ndcg_at_1000": 0.18846, + "ndcg_at_20": 0.16623, + "ndcg_at_3": 0.17523, + "ndcg_at_5": 0.16706, + "precision_at_1": 0.21298, + "precision_at_10": 0.04499, + "precision_at_100": 0.00671, + "precision_at_1000": 0.00085, + "precision_at_20": 0.02595, + "precision_at_3": 0.10948, + "precision_at_5": 0.0758, + "recall_at_1": 0.08389, + "recall_at_10": 0.17962, + "recall_at_100": 0.27225, + "recall_at_1000": 0.34652, + "recall_at_20": 0.20928, + "recall_at_3": 0.13013, + "recall_at_5": 0.15041 + } + ] + }, + "task_name": "OVENIT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/OxfordFlowersClassification.json b/results-mieb/voyage-multimodal-3/1/OxfordFlowersClassification.json new file mode 100644 index 0000000000..1615c93265 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/OxfordFlowersClassification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + "evaluation_time": 2976.4298408031464, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.9264705882352942, + "f1": 0.9260382675718866, + "f1_weighted": 0.9255751571683284, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9264705882352942, + "scores_per_experiment": [ + { + "accuracy": 0.9333333333333333, + "f1": 0.9328345649295149, + "f1_weighted": 0.9322880465495 + }, + { + "accuracy": 0.9245098039215687, + "f1": 0.923887629877471, + "f1_weighted": 0.9234416217263346 + }, + { + "accuracy": 0.9245098039215687, + "f1": 0.9245404681244752, + "f1_weighted": 0.924177246664839 + }, + { + "accuracy": 0.9235294117647059, + "f1": 0.9228630696130874, + "f1_weighted": 0.9222446908845601 + }, + { + "accuracy": 0.9264705882352942, + "f1": 0.9260656053148842, + "f1_weighted": 0.9257241800164078 + } + ] + } + ] + }, + "task_name": "OxfordFlowersClassification" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/OxfordPets.json b/results-mieb/voyage-multimodal-3/1/OxfordPets.json new file mode 100644 index 0000000000..344b6ab2f8 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/OxfordPets.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "557b480fae8d69247be74d9503b378a09425096f", + "evaluation_time": 1211.356342792511, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.8655764513491414, + "f1": 0.8620427475542456, + "f1_weighted": 0.8627648602390383, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8655764513491414, + "scores_per_experiment": [ + { + "accuracy": 0.860452439356773, + "f1": 0.8575450302018354, + "f1_weighted": 0.8581765441792053 + }, + { + "accuracy": 0.8689016080675933, + "f1": 0.8639469162042649, + "f1_weighted": 0.8648596012758134 + }, + { + "accuracy": 0.8639956391387299, + "f1": 0.8603811509947239, + "f1_weighted": 0.8611082432838298 + }, + { + "accuracy": 0.8659035159443991, + "f1": 0.8628250437566548, + "f1_weighted": 0.8635045209980706 + }, + { + "accuracy": 0.868629054238212, + "f1": 0.8655155966137498, + "f1_weighted": 0.866175391458273 + } + ] + } + ] + }, + "task_name": "OxfordPets" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/OxfordPetsZeroShot.json b/results-mieb/voyage-multimodal-3/1/OxfordPetsZeroShot.json new file mode 100644 index 0000000000..d405698708 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/OxfordPetsZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "557b480fae8d69247be74d9503b378a09425096f", + "evaluation_time": 677.260498046875, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.7050967566094304, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7050967566094304 + } + ] + }, + "task_name": "OxfordPetsZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/PatchCamelyon.json b/results-mieb/voyage-multimodal-3/1/PatchCamelyon.json new file mode 100644 index 0000000000..565761adc2 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/PatchCamelyon.json @@ -0,0 +1,60 @@ +{ + "dataset_revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", + "evaluation_time": 1878.3961374759674, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.677899169921875, + "ap": 0.6171958412862929, + "ap_weighted": 0.6171958412862929, + "f1": 0.6716262837899631, + "f1_weighted": 0.6716104944451556, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.677899169921875, + "scores_per_experiment": [ + { + "accuracy": 0.7061767578125, + "ap": 0.6366412143481036, + "ap_weighted": 0.6366412143481036, + "f1": 0.7010964063919244, + "f1_weighted": 0.7010797573001903 + }, + { + "accuracy": 0.635162353515625, + "ap": 0.5809307261507256, + "ap_weighted": 0.5809307261507256, + "f1": 0.6234755309662483, + "f1_weighted": 0.6234471894599793 + }, + { + "accuracy": 0.7178955078125, + "ap": 0.6544682544786682, + "ap_weighted": 0.6544682544786682, + "f1": 0.7177918531207379, + "f1_weighted": 0.7177895423473547 + }, + { + "accuracy": 0.6851806640625, + "ap": 0.6264741993887054, + "ap_weighted": 0.6264741993887054, + "f1": 0.6851780733405203, + "f1_weighted": 0.6851776874883105 + }, + { + "accuracy": 0.64508056640625, + "ap": 0.5874648120652616, + "ap_weighted": 0.5874648120652616, + "f1": 0.6305895551303842, + "f1_weighted": 0.6305582956299431 + } + ] + } + ] + }, + "task_name": "PatchCamelyon" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/PatchCamelyonZeroShot.json b/results-mieb/voyage-multimodal-3/1/PatchCamelyonZeroShot.json new file mode 100644 index 0000000000..faad802aa7 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/PatchCamelyonZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", + "evaluation_time": 1639.0224986076355, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.611114501953125, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.611114501953125 + } + ] + }, + "task_name": "PatchCamelyonZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/RESISC45.json b/results-mieb/voyage-multimodal-3/1/RESISC45.json new file mode 100644 index 0000000000..7b5e2ebe51 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/RESISC45.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", + "evaluation_time": 938.5634450912476, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.8733650793650793, + "f1": 0.8719221769133947, + "f1_weighted": 0.8726249328515951, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8733650793650793, + "scores_per_experiment": [ + { + "accuracy": 0.878095238095238, + "f1": 0.8766732208112427, + "f1_weighted": 0.8773890989875517 + }, + { + "accuracy": 0.8687301587301587, + "f1": 0.8665918615541792, + "f1_weighted": 0.8676431359172864 + }, + { + "accuracy": 0.8744444444444445, + "f1": 0.8729636786793165, + "f1_weighted": 0.8737471540962463 + }, + { + "accuracy": 0.8714285714285714, + "f1": 0.8704211254047678, + "f1_weighted": 0.8707897264966017 + }, + { + "accuracy": 0.8741269841269841, + "f1": 0.8729609981174671, + "f1_weighted": 0.8735555487602894 + } + ] + } + ] + }, + "task_name": "RESISC45" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/RESISC45ZeroShot.json b/results-mieb/voyage-multimodal-3/1/RESISC45ZeroShot.json new file mode 100644 index 0000000000..27bf07b0fe --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/RESISC45ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", + "evaluation_time": 669.4255020618439, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.6173015873015874, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6173015873015874 + } + ] + }, + "task_name": "RESISC45ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/RP2kI2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/RP2kI2IRetrieval.json new file mode 100644 index 0000000000..7ae814b4ed --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/RP2kI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f8f82d4eb1aa4dc4dbf2c768596c8110a3703765", + "evaluation_time": 5771.473861217499, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.5352, + "cv_recall_at_10": 0.76819, + "cv_recall_at_100": 0.90682, + "cv_recall_at_1000": 0.96452, + "cv_recall_at_20": 0.81752, + "cv_recall_at_3": 0.65718, + "cv_recall_at_5": 0.70872, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5352, + "map_at_1": 0.06083, + "map_at_10": 0.13284, + "map_at_100": 0.2051, + "map_at_1000": 0.23144, + "map_at_20": 0.15717, + "map_at_3": 0.09271, + "map_at_5": 0.10891, + "mrr_at_1": 0.9998166867438328, + "mrr_at_10": 0.9998952495679044, + "mrr_at_100": 0.9998952495679044, + "mrr_at_1000": 0.9998952495679044, + "mrr_at_20": 0.9998952495679044, + "mrr_at_3": 0.9998952495679044, + "mrr_at_5": 0.9998952495679044, + "nauc_cv_recall_at_1000_diff1": -0.766019391968589, + "nauc_cv_recall_at_1000_max": -0.06295290555454099, + "nauc_cv_recall_at_1000_std": 0.3959283270329479, + "nauc_cv_recall_at_100_diff1": -0.5678400545768518, + "nauc_cv_recall_at_100_max": -0.02212835197434516, + "nauc_cv_recall_at_100_std": 0.419964891717186, + "nauc_cv_recall_at_10_diff1": -0.4646819473475005, + "nauc_cv_recall_at_10_max": -0.0077422960251187075, + "nauc_cv_recall_at_10_std": 0.3988736082851318, + "nauc_cv_recall_at_1_diff1": -0.3815580121104423, + "nauc_cv_recall_at_1_max": -0.006154143668930801, + "nauc_cv_recall_at_1_std": 0.3183472656015551, + "nauc_cv_recall_at_20_diff1": -0.49646521246826764, + "nauc_cv_recall_at_20_max": -0.013337728131588886, + "nauc_cv_recall_at_20_std": 0.40877102450803326, + "nauc_cv_recall_at_3_diff1": -0.4151287956971939, + "nauc_cv_recall_at_3_max": -0.0016536980365809229, + "nauc_cv_recall_at_3_std": 0.36740802264805184, + "nauc_cv_recall_at_5_diff1": -0.43025372385530897, + "nauc_cv_recall_at_5_max": -0.003428009132052308, + "nauc_cv_recall_at_5_std": 0.381203098652696, + "nauc_map_at_1000_diff1": -0.15120985670983805, + "nauc_map_at_1000_max": 0.009746992690475763, + "nauc_map_at_1000_std": 0.29094231039826757, + "nauc_map_at_100_diff1": -0.10235809229014188, + "nauc_map_at_100_max": 0.01064839570938484, + "nauc_map_at_100_std": 0.20613516282109914, + "nauc_map_at_10_diff1": 0.09725844480380152, + "nauc_map_at_10_max": 0.011271028569452779, + "nauc_map_at_10_std": -0.06270213969677413, + "nauc_map_at_1_diff1": 0.3425746405724122, + "nauc_map_at_1_max": 0.02365490366793686, + "nauc_map_at_1_std": -0.18605605415952398, + "nauc_map_at_20_diff1": 0.02314585270469293, + "nauc_map_at_20_max": 0.009229531572517024, + "nauc_map_at_20_std": 0.01713455805688018, + "nauc_map_at_3_diff1": 0.22558432744395535, + "nauc_map_at_3_max": 0.018412296426926624, + "nauc_map_at_3_std": -0.15025269476059971, + "nauc_map_at_5_diff1": 0.17255685699656403, + "nauc_map_at_5_max": 0.014538356234491148, + "nauc_map_at_5_std": -0.12065309682842128, + "nauc_mrr_at_1000_diff1": 0.8746298450050553, + "nauc_mrr_at_1000_max": -0.08822922720082992, + "nauc_mrr_at_1000_std": -0.26587692280465064, + "nauc_mrr_at_100_diff1": 0.8746298450050553, + "nauc_mrr_at_100_max": -0.08822922720082992, + "nauc_mrr_at_100_std": -0.26587692280465064, + "nauc_mrr_at_10_diff1": 0.8746298450050553, + "nauc_mrr_at_10_max": -0.08822922720082992, + "nauc_mrr_at_10_std": -0.26587692280465064, + "nauc_mrr_at_1_diff1": 0.8779506238559595, + "nauc_mrr_at_1_max": -0.03254161762624195, + "nauc_mrr_at_1_std": -0.26357582606359653, + "nauc_mrr_at_20_diff1": 0.8746298450050553, + "nauc_mrr_at_20_max": -0.08822922720082992, + "nauc_mrr_at_20_std": -0.26587692280465064, + "nauc_mrr_at_3_diff1": 0.8746298450050553, + "nauc_mrr_at_3_max": -0.08822922720082992, + "nauc_mrr_at_3_std": -0.26587692280465064, + "nauc_mrr_at_5_diff1": 0.8746298450050553, + "nauc_mrr_at_5_max": -0.08822922720082992, + "nauc_mrr_at_5_std": -0.26587692280465064, + "nauc_ndcg_at_1000_diff1": -0.21974184350852993, + "nauc_ndcg_at_1000_max": 0.0012162062863128136, + "nauc_ndcg_at_1000_std": 0.34775895764952175, + "nauc_ndcg_at_100_diff1": -0.20146467973477145, + "nauc_ndcg_at_100_max": 0.007725844480498439, + "nauc_ndcg_at_100_std": 0.3486766114714179, + "nauc_ndcg_at_10_diff1": -0.3173040774339957, + "nauc_ndcg_at_10_max": 0.005709132027725912, + "nauc_ndcg_at_10_std": 0.38631272877540723, + "nauc_ndcg_at_1_diff1": 0.8779506238559595, + "nauc_ndcg_at_1_max": -0.03254161762624195, + "nauc_ndcg_at_1_std": -0.26357582606359653, + "nauc_ndcg_at_20_diff1": -0.2909080633079741, + "nauc_ndcg_at_20_max": 0.006541878226556776, + "nauc_ndcg_at_20_std": 0.40254942311031094, + "nauc_ndcg_at_3_diff1": -0.34084820150248, + "nauc_ndcg_at_3_max": 0.001684052550435759, + "nauc_ndcg_at_3_std": 0.33703978667617646, + "nauc_ndcg_at_5_diff1": -0.33192655529088805, + "nauc_ndcg_at_5_max": 0.0035293920900730095, + "nauc_ndcg_at_5_std": 0.36066658957673187, + "nauc_precision_at_1000_diff1": -0.31636500140446766, + "nauc_precision_at_1000_max": -0.002192204079137363, + "nauc_precision_at_1000_std": 0.5138508071474455, + "nauc_precision_at_100_diff1": -0.3834205130450664, + "nauc_precision_at_100_max": 0.002633365597297572, + "nauc_precision_at_100_std": 0.5708202512526964, + "nauc_precision_at_10_diff1": -0.3959065384681834, + "nauc_precision_at_10_max": -0.001494129511746914, + "nauc_precision_at_10_std": 0.43210713213550545, + "nauc_precision_at_1_diff1": 0.8779506238559595, + "nauc_precision_at_1_max": -0.03254161762624195, + "nauc_precision_at_1_std": -0.26357582606359653, + "nauc_precision_at_20_diff1": -0.40001518210288856, + "nauc_precision_at_20_max": -0.0009124698708636982, + "nauc_precision_at_20_std": 0.48403989980888124, + "nauc_precision_at_3_diff1": -0.38229287317154065, + "nauc_precision_at_3_max": -0.0033741789883080554, + "nauc_precision_at_3_std": 0.3523027667471958, + "nauc_precision_at_5_diff1": -0.3863143496226523, + "nauc_precision_at_5_max": -0.00291350156539194, + "nauc_precision_at_5_std": 0.38593312972103216, + "nauc_recall_at_1000_diff1": -0.17969847236289574, + "nauc_recall_at_1000_max": -0.00016524020492468333, + "nauc_recall_at_1000_std": 0.31391704126449405, + "nauc_recall_at_100_diff1": -0.1010639403425878, + "nauc_recall_at_100_max": 0.009107897691813319, + "nauc_recall_at_100_std": 0.20625697881855873, + "nauc_recall_at_10_diff1": 0.10199050893201908, + "nauc_recall_at_10_max": 0.010931865384178865, + "nauc_recall_at_10_std": -0.06471258207066122, + "nauc_recall_at_1_diff1": 0.3425746405724122, + "nauc_recall_at_1_max": 0.02365490366793686, + "nauc_recall_at_1_std": -0.18605605415952398, + "nauc_recall_at_20_diff1": 0.02928013978860769, + "nauc_recall_at_20_max": 0.007750531130897131, + "nauc_recall_at_20_std": 0.01251388074417554, + "nauc_recall_at_3_diff1": 0.2268665131612476, + "nauc_recall_at_3_max": 0.018239264908074454, + "nauc_recall_at_3_std": -0.14999317739606058, + "nauc_recall_at_5_diff1": 0.17448687702754684, + "nauc_recall_at_5_max": 0.014284892663234853, + "nauc_recall_at_5_std": -0.12052944586207395, + "ndcg_at_1": 0.99982, + "ndcg_at_10": 0.56417, + "ndcg_at_100": 0.39403, + "ndcg_at_1000": 0.45364, + "ndcg_at_20": 0.48913, + "ndcg_at_3": 0.74355, + "ndcg_at_5": 0.65817, + "precision_at_1": 0.99982, + "precision_at_10": 0.44004, + "precision_at_100": 0.1663, + "precision_at_1000": 0.03803, + "precision_at_20": 0.34382, + "precision_at_3": 0.66234, + "precision_at_5": 0.55514, + "recall_at_1": 0.06083, + "recall_at_10": 0.14578, + "recall_at_100": 0.29716, + "recall_at_1000": 0.49997, + "recall_at_20": 0.18462, + "recall_at_3": 0.09383, + "recall_at_5": 0.1132 + } + ] + }, + "task_name": "RP2kI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/RenderedSST2.json b/results-mieb/voyage-multimodal-3/1/RenderedSST2.json new file mode 100644 index 0000000000..c7be2dccee --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/RenderedSST2.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", + "evaluation_time": 130.35417199134827, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.8544755628775398, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8544755628775398 + } + ] + }, + "task_name": "RenderedSST2" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STL10.json b/results-mieb/voyage-multimodal-3/1/STL10.json new file mode 100644 index 0000000000..d5a02be6ea --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STL10.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "49ae7f94508f7feae62baf836db284306eab0b0f", + "evaluation_time": 403.07147884368896, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.9878500000000001, + "f1": 0.9878558133843768, + "f1_weighted": 0.9878558133843767, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9878500000000001, + "scores_per_experiment": [ + { + "accuracy": 0.988, + "f1": 0.9880014064377992, + "f1_weighted": 0.9880014064377991 + }, + { + "accuracy": 0.9885, + "f1": 0.9885067563274645, + "f1_weighted": 0.9885067563274644 + }, + { + "accuracy": 0.987375, + "f1": 0.9873823425288343, + "f1_weighted": 0.9873823425288343 + }, + { + "accuracy": 0.98775, + "f1": 0.9877593004896299, + "f1_weighted": 0.9877593004896299 + }, + { + "accuracy": 0.987625, + "f1": 0.9876292611381554, + "f1_weighted": 0.9876292611381554 + } + ] + } + ] + }, + "task_name": "STL10" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STL10ZeroShot.json b/results-mieb/voyage-multimodal-3/1/STL10ZeroShot.json new file mode 100644 index 0000000000..bf6f845917 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STL10ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "49ae7f94508f7feae62baf836db284306eab0b0f", + "evaluation_time": 383.49997425079346, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.987125, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.987125 + } + ] + }, + "task_name": "STL10ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/SUN397.json b/results-mieb/voyage-multimodal-3/1/SUN397.json new file mode 100644 index 0000000000..aa51b639cc --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/SUN397.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "7e6af6a2499ad708618be868e1471eac0aca1168", + "evaluation_time": 9430.040058612823, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.7631080459770115, + "f1": 0.7352522662342722, + "f1_weighted": 0.7637320144217425, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7631080459770115, + "scores_per_experiment": [ + { + "accuracy": 0.7626206896551724, + "f1": 0.7355455453428275, + "f1_weighted": 0.7629772013685338 + }, + { + "accuracy": 0.7569655172413793, + "f1": 0.7319488897671674, + "f1_weighted": 0.7583476931412039 + }, + { + "accuracy": 0.7645057471264368, + "f1": 0.7373019095808092, + "f1_weighted": 0.7650115777182991 + }, + { + "accuracy": 0.7662528735632184, + "f1": 0.7359482181635232, + "f1_weighted": 0.7658961973253269 + }, + { + "accuracy": 0.7651954022988505, + "f1": 0.7355167683170333, + "f1_weighted": 0.7664274025553492 + } + ] + } + ] + }, + "task_name": "SUN397" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/SUN397ZeroShot.json b/results-mieb/voyage-multimodal-3/1/SUN397ZeroShot.json new file mode 100644 index 0000000000..ed71ad10e9 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/SUN397ZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "7e6af6a2499ad708618be868e1471eac0aca1168", + "evaluation_time": 3248.6113753318787, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.6864367816091954, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6864367816091954 + } + ] + }, + "task_name": "SUN397ZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/StanfordCars.json b/results-mieb/voyage-multimodal-3/1/StanfordCars.json new file mode 100644 index 0000000000..5033cbad06 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/StanfordCars.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", + "evaluation_time": 9452.947076797485, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5290635493097874, + "f1": 0.531423599021726, + "f1_weighted": 0.5296608613945841, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5290635493097874, + "scores_per_experiment": [ + { + "accuracy": 0.531525929610745, + "f1": 0.5346764534490921, + "f1_weighted": 0.5336051434306501 + }, + { + "accuracy": 0.53276955602537, + "f1": 0.5323535520016426, + "f1_weighted": 0.5307497038657789 + }, + { + "accuracy": 0.5271732371595572, + "f1": 0.5312445144487528, + "f1_weighted": 0.5289222489776181 + }, + { + "accuracy": 0.5245616216888447, + "f1": 0.5265087065369574, + "f1_weighted": 0.5247431253688526 + }, + { + "accuracy": 0.5292874020644198, + "f1": 0.532334768672185, + "f1_weighted": 0.5302840853300209 + } + ] + } + ] + }, + "task_name": "StanfordCars" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/StanfordCarsI2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/StanfordCarsI2IRetrieval.json new file mode 100644 index 0000000000..af74f4743c --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/StanfordCarsI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b27a0612211af3598bd11fe28af20928f20cce06", + "evaluation_time": 6306.916451454163, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.46785, + "cv_recall_at_10": 0.73672, + "cv_recall_at_100": 0.95075, + "cv_recall_at_1000": 0.9995, + "cv_recall_at_20": 0.81831, + "cv_recall_at_3": 0.60167, + "cv_recall_at_5": 0.65912, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.46785, + "map_at_1": 0.0, + "map_at_10": 0.03844, + "map_at_100": 0.08637, + "map_at_1000": 0.10395, + "map_at_20": 0.05772, + "map_at_3": 0.01142, + "map_at_5": 0.02093, + "mrr_at_1": 0.00012436264146250465, + "mrr_at_10": 0.2937613875712362, + "mrr_at_100": 0.3032439165995846, + "mrr_at_1000": 0.3035436772085266, + "mrr_at_20": 0.29964394708226216, + "mrr_at_3": 0.2634829830452242, + "mrr_at_5": 0.28110516934046254, + "nauc_cv_recall_at_1000_diff1": -1.151825029874656, + "nauc_cv_recall_at_1000_max": -0.2298295685244738, + "nauc_cv_recall_at_1000_std": -0.13833406665530235, + "nauc_cv_recall_at_100_diff1": -0.3065583434061352, + "nauc_cv_recall_at_100_max": 0.08250669884887286, + "nauc_cv_recall_at_100_std": -0.348791493134295, + "nauc_cv_recall_at_10_diff1": -0.395575993710386, + "nauc_cv_recall_at_10_max": -0.006248115883899727, + "nauc_cv_recall_at_10_std": -0.08906794871950323, + "nauc_cv_recall_at_1_diff1": -0.435307778833609, + "nauc_cv_recall_at_1_max": -0.007866420116790255, + "nauc_cv_recall_at_1_std": -0.030415830440148624, + "nauc_cv_recall_at_20_diff1": -0.36737862331436555, + "nauc_cv_recall_at_20_max": -0.006505738205243344, + "nauc_cv_recall_at_20_std": -0.15938166843054247, + "nauc_cv_recall_at_3_diff1": -0.4251008432692228, + "nauc_cv_recall_at_3_max": -0.009223330865472642, + "nauc_cv_recall_at_3_std": -0.051319053449302245, + "nauc_cv_recall_at_5_diff1": -0.4149559790404437, + "nauc_cv_recall_at_5_max": -0.011111361572481566, + "nauc_cv_recall_at_5_std": -0.07638029271951365, + "nauc_map_at_1000_diff1": -0.33739502802420707, + "nauc_map_at_1000_max": 0.0005840703968449165, + "nauc_map_at_1000_std": 0.05069004792828951, + "nauc_map_at_100_diff1": -0.34132369612264735, + "nauc_map_at_100_max": 0.0008966411708180106, + "nauc_map_at_100_std": 0.05068205970232099, + "nauc_map_at_10_diff1": -0.3619601668272867, + "nauc_map_at_10_max": 0.00014890087230641323, + "nauc_map_at_10_std": 0.012024432555931297, + "nauc_map_at_1_diff1": -0.5746685568283685, + "nauc_map_at_1_max": 0.09844700703211207, + "nauc_map_at_1_std": 0.661917627637042, + "nauc_map_at_20_diff1": -0.35182572786027133, + "nauc_map_at_20_max": 0.0013720545342691007, + "nauc_map_at_20_std": 0.02669559294518995, + "nauc_map_at_3_diff1": -0.38399345622920933, + "nauc_map_at_3_max": 0.0022945953431132045, + "nauc_map_at_3_std": -0.023320109114295547, + "nauc_map_at_5_diff1": -0.37108426221595764, + "nauc_map_at_5_max": 0.0038153853032365675, + "nauc_map_at_5_std": -0.009807037280071264, + "nauc_mrr_at_1000_diff1": -0.42454422892630644, + "nauc_mrr_at_1000_max": -0.006784315618668348, + "nauc_mrr_at_1000_std": -0.05235860206269823, + "nauc_mrr_at_100_diff1": -0.42426889507485727, + "nauc_mrr_at_100_max": -0.006583638725419789, + "nauc_mrr_at_100_std": -0.05289654089881731, + "nauc_mrr_at_10_diff1": -0.4252515611778805, + "nauc_mrr_at_10_max": -0.00828493016316975, + "nauc_mrr_at_10_std": -0.049487627586929324, + "nauc_mrr_at_1_diff1": -0.5746685568283684, + "nauc_mrr_at_1_max": 0.32383525527408397, + "nauc_mrr_at_1_std": 0.6619176276370419, + "nauc_mrr_at_20_diff1": -0.42374558442388, + "nauc_mrr_at_20_max": -0.007498321718482753, + "nauc_mrr_at_20_std": -0.05306656104216916, + "nauc_mrr_at_3_diff1": -0.4349144250848074, + "nauc_mrr_at_3_max": -0.005834866917902523, + "nauc_mrr_at_3_std": -0.030070112760466665, + "nauc_mrr_at_5_diff1": -0.4305631366028515, + "nauc_mrr_at_5_max": -0.005579537855220841, + "nauc_mrr_at_5_std": -0.044730291358109804, + "nauc_ndcg_at_1000_diff1": -0.3245598189947247, + "nauc_ndcg_at_1000_max": -0.001956981765244245, + "nauc_ndcg_at_1000_std": -0.005711298478346799, + "nauc_ndcg_at_100_diff1": -0.3384631852576036, + "nauc_ndcg_at_100_max": 0.0013101096062173095, + "nauc_ndcg_at_100_std": 0.010073931380352638, + "nauc_ndcg_at_10_diff1": -0.3680955726064528, + "nauc_ndcg_at_10_max": -0.000348791327047919, + "nauc_ndcg_at_10_std": -0.005616112265719418, + "nauc_ndcg_at_1_diff1": -0.5746685568283684, + "nauc_ndcg_at_1_max": 0.09844700703211205, + "nauc_ndcg_at_1_std": 0.6619176276370419, + "nauc_ndcg_at_20_diff1": -0.3541664733286353, + "nauc_ndcg_at_20_max": 0.0009538962638639399, + "nauc_ndcg_at_20_std": 0.001578883560078981, + "nauc_ndcg_at_3_diff1": -0.4051504575351157, + "nauc_ndcg_at_3_max": 0.0013420721598884624, + "nauc_ndcg_at_3_std": -0.0241111780824001, + "nauc_ndcg_at_5_diff1": -0.3845192804422571, + "nauc_ndcg_at_5_max": 0.0036776018462837904, + "nauc_ndcg_at_5_std": -0.019678161155368832, + "nauc_precision_at_1000_diff1": -0.23443308943299598, + "nauc_precision_at_1000_max": -0.005920538114289565, + "nauc_precision_at_1000_std": -0.00887291260859431, + "nauc_precision_at_100_diff1": -0.3029942937622413, + "nauc_precision_at_100_max": 0.002100537907211134, + "nauc_precision_at_100_std": 0.024000880512538438, + "nauc_precision_at_10_diff1": -0.35274373994854663, + "nauc_precision_at_10_max": -0.0008779745401321433, + "nauc_precision_at_10_std": -0.000516369982727631, + "nauc_precision_at_1_diff1": -0.5746685568283684, + "nauc_precision_at_1_max": 0.09844700703211205, + "nauc_precision_at_1_std": 0.6619176276370419, + "nauc_precision_at_20_diff1": -0.3334397092483834, + "nauc_precision_at_20_max": 0.0009749871462993896, + "nauc_precision_at_20_std": 0.00780023573483833, + "nauc_precision_at_3_diff1": -0.40082900870258426, + "nauc_precision_at_3_max": 0.0025889621419443207, + "nauc_precision_at_3_std": -0.023423965100093654, + "nauc_precision_at_5_diff1": -0.3751583392422432, + "nauc_precision_at_5_max": 0.005117377241896628, + "nauc_precision_at_5_std": -0.017923373435618145, + "nauc_recall_at_1000_diff1": -0.25844392309517555, + "nauc_recall_at_1000_max": -0.004435575371902964, + "nauc_recall_at_1000_std": -0.018739155834388607, + "nauc_recall_at_100_diff1": -0.30620642698455364, + "nauc_recall_at_100_max": 0.0019769607998182674, + "nauc_recall_at_100_std": 0.01672304677306232, + "nauc_recall_at_10_diff1": -0.35096276881889504, + "nauc_recall_at_10_max": -0.002378536023237739, + "nauc_recall_at_10_std": -0.00525163144320875, + "nauc_recall_at_1_diff1": -0.5746685568283685, + "nauc_recall_at_1_max": 0.09844700703211207, + "nauc_recall_at_1_std": 0.661917627637042, + "nauc_recall_at_20_diff1": -0.3327188709289407, + "nauc_recall_at_20_max": -0.00018314878513959012, + "nauc_recall_at_20_std": 0.002013745169352921, + "nauc_recall_at_3_diff1": -0.3866363742979928, + "nauc_recall_at_3_max": 0.0017557406432875135, + "nauc_recall_at_3_std": -0.0241880762963367, + "nauc_recall_at_5_diff1": -0.36821866944017695, + "nauc_recall_at_5_max": 0.003778984799888619, + "nauc_recall_at_5_std": -0.02061686291660384, + "ndcg_at_1": 0.00012, + "ndcg_at_10": 0.24122, + "ndcg_at_100": 0.21367, + "ndcg_at_1000": 0.35718, + "ndcg_at_20": 0.21098, + "ndcg_at_3": 0.22568, + "ndcg_at_5": 0.24666, + "precision_at_1": 0.00012, + "precision_at_10": 0.25882, + "precision_at_100": 0.09024, + "precision_at_1000": 0.02253, + "precision_at_20": 0.2067, + "precision_at_3": 0.27973, + "precision_at_5": 0.28807, + "recall_at_1": 0.0, + "recall_at_10": 0.06538, + "recall_at_100": 0.22602, + "recall_at_1000": 0.5597, + "recall_at_20": 0.1043, + "recall_at_3": 0.02118, + "recall_at_5": 0.03639 + } + ] + }, + "task_name": "StanfordCarsI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/StanfordCarsZeroShot.json b/results-mieb/voyage-multimodal-3/1/StanfordCarsZeroShot.json new file mode 100644 index 0000000000..032e15909d --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/StanfordCarsZeroShot.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", + "evaluation_time": 3249.768146753311, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.49396841188906854, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.49396841188906854 + } + ] + }, + "task_name": "StanfordCarsZeroShot" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/SugarCrepe.json b/results-mieb/voyage-multimodal-3/1/SugarCrepe.json new file mode 100644 index 0000000000..402dbaac7c --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/SugarCrepe.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "134abf9ade6a32f9fdae0e89022ff227a70b87e5", + "evaluation_time": 2256.938088417053, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0, + "hf_subset": "default", + "image_acc": 0.0, + "languages": [ + "eng-Latn" + ], + "main_score": 0.8961523175239563, + "text_acc": 0.8961523175239563 + } + ] + }, + "task_name": "SugarCrepe" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/TUBerlinT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/TUBerlinT2IRetrieval.json new file mode 100644 index 0000000000..f989165017 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/TUBerlinT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "0cd78cd1ddbd3cafa9f319c638ebd77836ec9ff6", + "evaluation_time": 6954.40220117569, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.632, + "cv_recall_at_10": 0.688, + "cv_recall_at_100": 0.768, + "cv_recall_at_1000": 0.88, + "cv_recall_at_20": 0.7, + "cv_recall_at_3": 0.672, + "cv_recall_at_5": 0.676, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.60246, + "map_at_1": 0.0079, + "map_at_10": 0.07075, + "map_at_100": 0.35832, + "map_at_1000": 0.39369, + "map_at_20": 0.13453, + "map_at_3": 0.02261, + "map_at_5": 0.03665, + "mrr_at_1": 0.632, + "mrr_at_10": 0.6523111111111112, + "mrr_at_100": 0.6544858405118993, + "mrr_at_1000": 0.6548076054198555, + "mrr_at_20": 0.6531414141414141, + "mrr_at_3": 0.65, + "mrr_at_5": 0.6507999999999999, + "nauc_cv_recall_at_1000_diff1": 0.43452115812917536, + "nauc_cv_recall_at_1000_max": 0.7407254215717467, + "nauc_cv_recall_at_1000_std": 0.9309576837416487, + "nauc_cv_recall_at_100_diff1": 0.4753903054873923, + "nauc_cv_recall_at_100_max": 0.7488708702300939, + "nauc_cv_recall_at_100_std": 0.9500337849852408, + "nauc_cv_recall_at_10_diff1": 0.4894545301258994, + "nauc_cv_recall_at_10_max": 0.7497962567446042, + "nauc_cv_recall_at_10_std": 0.9646751348920867, + "nauc_cv_recall_at_1_diff1": 0.41400656793789425, + "nauc_cv_recall_at_1_max": 0.6853743850422137, + "nauc_cv_recall_at_1_std": 0.820853831926218, + "nauc_cv_recall_at_20_diff1": 0.5039740587281571, + "nauc_cv_recall_at_20_max": 0.7425761124121779, + "nauc_cv_recall_at_20_std": 0.9761556476310572, + "nauc_cv_recall_at_3_diff1": 0.49802833754861175, + "nauc_cv_recall_at_3_max": 0.7442549835468169, + "nauc_cv_recall_at_3_std": 0.9699967365587014, + "nauc_cv_recall_at_5_diff1": 0.4873986442490248, + "nauc_cv_recall_at_5_max": 0.7356285598744322, + "nauc_cv_recall_at_5_std": 0.9651260820990151, + "nauc_map_at_1000_diff1": 0.18517532377370824, + "nauc_map_at_1000_max": 0.5397099906148431, + "nauc_map_at_1000_std": 0.7018854603300901, + "nauc_map_at_100_diff1": 0.16384249487832003, + "nauc_map_at_100_max": 0.5133118473344895, + "nauc_map_at_100_std": 0.6644666463401745, + "nauc_map_at_10_diff1": 0.33706560106522576, + "nauc_map_at_10_max": 0.644029000779149, + "nauc_map_at_10_std": 0.7258322218404971, + "nauc_map_at_1_diff1": 0.41400656793789353, + "nauc_map_at_1_max": 0.6853743850422134, + "nauc_map_at_1_std": 0.8208538319262177, + "nauc_map_at_20_diff1": 0.2937231437358452, + "nauc_map_at_20_max": 0.6247902961302264, + "nauc_map_at_20_std": 0.71105971054528, + "nauc_map_at_3_diff1": 0.3688228653223755, + "nauc_map_at_3_max": 0.6485263366734154, + "nauc_map_at_3_std": 0.7622131083978515, + "nauc_map_at_5_diff1": 0.35873005431899013, + "nauc_map_at_5_max": 0.6408516884302472, + "nauc_map_at_5_std": 0.7351246510760714, + "nauc_mrr_at_1000_diff1": 0.44699780409031015, + "nauc_mrr_at_1000_max": 0.7147526104841861, + "nauc_mrr_at_1000_std": 0.8853212411559411, + "nauc_mrr_at_100_diff1": 0.4470661493736612, + "nauc_mrr_at_100_max": 0.7148642247167679, + "nauc_mrr_at_100_std": 0.8854262060790729, + "nauc_mrr_at_10_diff1": 0.4473669698354353, + "nauc_mrr_at_10_max": 0.7148720726802916, + "nauc_mrr_at_10_std": 0.8854679886170576, + "nauc_mrr_at_1_diff1": 0.41400656793789425, + "nauc_mrr_at_1_max": 0.6853743850422137, + "nauc_mrr_at_1_std": 0.820853831926218, + "nauc_mrr_at_20_diff1": 0.4477358270590955, + "nauc_mrr_at_20_max": 0.7143799945198087, + "nauc_mrr_at_20_std": 0.8859265713576927, + "nauc_mrr_at_3_diff1": 0.44895760436162346, + "nauc_mrr_at_3_max": 0.7143570130468984, + "nauc_mrr_at_3_std": 0.8866628692326469, + "nauc_mrr_at_5_diff1": 0.4468593663508612, + "nauc_mrr_at_5_max": 0.7126695706582533, + "nauc_mrr_at_5_std": 0.8856072465478423, + "nauc_ndcg_at_1000_diff1": 0.3022533037187929, + "nauc_ndcg_at_1000_max": 0.6408175973611855, + "nauc_ndcg_at_1000_std": 0.8298954905844009, + "nauc_ndcg_at_100_diff1": 0.2540055991335015, + "nauc_ndcg_at_100_max": 0.5881314675701598, + "nauc_ndcg_at_100_std": 0.7623167279503071, + "nauc_ndcg_at_10_diff1": 0.3851681326373794, + "nauc_ndcg_at_10_max": 0.6781755638355185, + "nauc_ndcg_at_10_std": 0.7960845072732069, + "nauc_ndcg_at_1_diff1": 0.41400656793789425, + "nauc_ndcg_at_1_max": 0.6853743850422137, + "nauc_ndcg_at_1_std": 0.820853831926218, + "nauc_ndcg_at_20_diff1": 0.354832824475887, + "nauc_ndcg_at_20_max": 0.6678366209429181, + "nauc_ndcg_at_20_std": 0.7900276393440292, + "nauc_ndcg_at_3_diff1": 0.39433539441866544, + "nauc_ndcg_at_3_max": 0.6710511733456289, + "nauc_ndcg_at_3_std": 0.8067166269026869, + "nauc_ndcg_at_5_diff1": 0.39262145769837264, + "nauc_ndcg_at_5_max": 0.6688008888875651, + "nauc_ndcg_at_5_std": 0.7955553159332548, + "nauc_precision_at_1000_diff1": 0.3013794038214159, + "nauc_precision_at_1000_max": 0.6376415183291911, + "nauc_precision_at_1000_std": 0.84650127290728, + "nauc_precision_at_100_diff1": 0.2207309788254333, + "nauc_precision_at_100_max": 0.5578314150677521, + "nauc_precision_at_100_std": 0.747955127810586, + "nauc_precision_at_10_diff1": 0.37965193897786503, + "nauc_precision_at_10_max": 0.6763534352967545, + "nauc_precision_at_10_std": 0.7888648245686696, + "nauc_precision_at_1_diff1": 0.41400656793789425, + "nauc_precision_at_1_max": 0.6853743850422137, + "nauc_precision_at_1_std": 0.820853831926218, + "nauc_precision_at_20_diff1": 0.3384274323417243, + "nauc_precision_at_20_max": 0.6597557047868738, + "nauc_precision_at_20_std": 0.7803763157657103, + "nauc_precision_at_3_diff1": 0.3895862418304692, + "nauc_precision_at_3_max": 0.6625175539384656, + "nauc_precision_at_3_std": 0.8014294362560389, + "nauc_precision_at_5_diff1": 0.38931076445156515, + "nauc_precision_at_5_max": 0.6617444000758278, + "nauc_precision_at_5_std": 0.7868194632144762, + "nauc_recall_at_1000_diff1": 0.30137940382141565, + "nauc_recall_at_1000_max": 0.6376415183291911, + "nauc_recall_at_1000_std": 0.8465012729072791, + "nauc_recall_at_100_diff1": 0.2207309788254334, + "nauc_recall_at_100_max": 0.5578314150677524, + "nauc_recall_at_100_std": 0.7479551278105863, + "nauc_recall_at_10_diff1": 0.37965193897786503, + "nauc_recall_at_10_max": 0.6763534352967545, + "nauc_recall_at_10_std": 0.7888648245686696, + "nauc_recall_at_1_diff1": 0.41400656793789353, + "nauc_recall_at_1_max": 0.6853743850422134, + "nauc_recall_at_1_std": 0.8208538319262177, + "nauc_recall_at_20_diff1": 0.3384274323417243, + "nauc_recall_at_20_max": 0.6597557047868738, + "nauc_recall_at_20_std": 0.7803763157657103, + "nauc_recall_at_3_diff1": 0.38958624183046886, + "nauc_recall_at_3_max": 0.6625175539384649, + "nauc_recall_at_3_std": 0.8014294362560384, + "nauc_recall_at_5_diff1": 0.38931076445156515, + "nauc_recall_at_5_max": 0.6617444000758278, + "nauc_recall_at_5_std": 0.7868194632144762, + "ndcg_at_1": 0.632, + "ndcg_at_10": 0.60246, + "ndcg_at_100": 0.45777, + "ndcg_at_1000": 0.55125, + "ndcg_at_20": 0.58608, + "ndcg_at_3": 0.61902, + "ndcg_at_5": 0.61088, + "precision_at_1": 0.632, + "precision_at_10": 0.596, + "precision_at_100": 0.33596, + "precision_at_1000": 0.04708, + "precision_at_20": 0.5756, + "precision_at_3": 0.616, + "precision_at_5": 0.6056, + "recall_at_1": 0.0079, + "recall_at_10": 0.0745, + "recall_at_100": 0.41995, + "recall_at_1000": 0.58845, + "recall_at_20": 0.1439, + "recall_at_3": 0.0231, + "recall_at_5": 0.03785 + } + ] + }, + "task_name": "TUBerlinT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreArxivQARetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..4203c356be --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", + "evaluation_time": 503.60697174072266, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.774, + "cv_recall_at_10": 0.93, + "cv_recall_at_100": 0.994, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.958, + "cv_recall_at_3": 0.872, + "cv_recall_at_5": 0.908, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8461, + "map_at_1": 0.774, + "map_at_10": 0.82827, + "map_at_100": 0.83144, + "map_at_1000": 0.83148, + "map_at_20": 0.83034, + "map_at_3": 0.81733, + "map_at_5": 0.82543, + "mrr_at_1": 0.774, + "mrr_at_10": 0.8282658730158727, + "mrr_at_100": 0.8314401062878024, + "mrr_at_1000": 0.8314832333872403, + "mrr_at_20": 0.8303449179475492, + "mrr_at_3": 0.8173333333333332, + "mrr_at_5": 0.8254333333333331, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.8078120136943575, + "nauc_cv_recall_at_100_max": 0.9074074074074102, + "nauc_cv_recall_at_100_std": 0.6934329287270279, + "nauc_cv_recall_at_10_diff1": 0.8166599973322675, + "nauc_cv_recall_at_10_max": 0.7872615712951853, + "nauc_cv_recall_at_10_std": -0.2622248899559833, + "nauc_cv_recall_at_1_diff1": 0.914007925160341, + "nauc_cv_recall_at_1_max": 0.8011901357626264, + "nauc_cv_recall_at_1_std": -0.39925832119140625, + "nauc_cv_recall_at_20_diff1": 0.8250411275621354, + "nauc_cv_recall_at_20_max": 0.8003868213952258, + "nauc_cv_recall_at_20_std": -0.30123160375261115, + "nauc_cv_recall_at_3_diff1": 0.8521416556530105, + "nauc_cv_recall_at_3_max": 0.7492265409107289, + "nauc_cv_recall_at_3_std": -0.3856557130842251, + "nauc_cv_recall_at_5_diff1": 0.8248792270531401, + "nauc_cv_recall_at_5_max": 0.7668980635732549, + "nauc_cv_recall_at_5_std": -0.30081597856534015, + "nauc_map_at_1000_diff1": 0.8962125696737271, + "nauc_map_at_1000_max": 0.7910425116794585, + "nauc_map_at_1000_std": -0.3820849643364578, + "nauc_map_at_100_diff1": 0.8961830489825202, + "nauc_map_at_100_max": 0.7910662571939857, + "nauc_map_at_100_std": -0.38181788077602036, + "nauc_map_at_10_diff1": 0.8944475891541415, + "nauc_map_at_10_max": 0.790785383308978, + "nauc_map_at_10_std": -0.3803408069574615, + "nauc_map_at_1_diff1": 0.914007925160341, + "nauc_map_at_1_max": 0.8011901357626264, + "nauc_map_at_1_std": -0.39925832119140625, + "nauc_map_at_20_diff1": 0.8956877931446563, + "nauc_map_at_20_max": 0.7911494373989493, + "nauc_map_at_20_std": -0.38239572816275114, + "nauc_map_at_3_diff1": 0.8949812790326085, + "nauc_map_at_3_max": 0.7857879975203541, + "nauc_map_at_3_std": -0.38926151758415656, + "nauc_map_at_5_diff1": 0.893513670583005, + "nauc_map_at_5_max": 0.7891425005671933, + "nauc_map_at_5_std": -0.3810974151707644, + "nauc_mrr_at_1000_diff1": 0.8962125696737271, + "nauc_mrr_at_1000_max": 0.7910425116794585, + "nauc_mrr_at_1000_std": -0.3820849643364578, + "nauc_mrr_at_100_diff1": 0.8961830489825202, + "nauc_mrr_at_100_max": 0.7910662571939857, + "nauc_mrr_at_100_std": -0.38181788077602036, + "nauc_mrr_at_10_diff1": 0.8944475891541415, + "nauc_mrr_at_10_max": 0.790785383308978, + "nauc_mrr_at_10_std": -0.3803408069574615, + "nauc_mrr_at_1_diff1": 0.914007925160341, + "nauc_mrr_at_1_max": 0.8011901357626264, + "nauc_mrr_at_1_std": -0.39925832119140625, + "nauc_mrr_at_20_diff1": 0.8956877931446563, + "nauc_mrr_at_20_max": 0.7911494373989493, + "nauc_mrr_at_20_std": -0.38239572816275114, + "nauc_mrr_at_3_diff1": 0.8949812790326085, + "nauc_mrr_at_3_max": 0.7857879975203541, + "nauc_mrr_at_3_std": -0.38926151758415656, + "nauc_mrr_at_5_diff1": 0.893513670583005, + "nauc_mrr_at_5_max": 0.7891425005671933, + "nauc_mrr_at_5_std": -0.3810974151707644, + "nauc_ndcg_at_1000_diff1": 0.8929451112766938, + "nauc_ndcg_at_1000_max": 0.7902064276004215, + "nauc_ndcg_at_1000_std": -0.37646127760601994, + "nauc_ndcg_at_100_diff1": 0.8923458106503991, + "nauc_ndcg_at_100_max": 0.7909366860251333, + "nauc_ndcg_at_100_std": -0.3695077693697132, + "nauc_ndcg_at_10_diff1": 0.8843225087145591, + "nauc_ndcg_at_10_max": 0.7895468920172003, + "nauc_ndcg_at_10_std": -0.3661802754316691, + "nauc_ndcg_at_1_diff1": 0.914007925160341, + "nauc_ndcg_at_1_max": 0.8011901357626264, + "nauc_ndcg_at_1_std": -0.39925832119140625, + "nauc_ndcg_at_20_diff1": 0.8888415250006587, + "nauc_ndcg_at_20_max": 0.7908558372598717, + "nauc_ndcg_at_20_std": -0.37481576312176695, + "nauc_ndcg_at_3_diff1": 0.8863782969130156, + "nauc_ndcg_at_3_max": 0.7784715680451962, + "nauc_ndcg_at_3_std": -0.38827854477910334, + "nauc_ndcg_at_5_diff1": 0.8825150872002235, + "nauc_ndcg_at_5_max": 0.7851998944708328, + "nauc_ndcg_at_5_std": -0.368947880668478, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.8078120136943184, + "nauc_precision_at_100_max": 0.9074074074073771, + "nauc_precision_at_100_std": 0.6934329287270289, + "nauc_precision_at_10_diff1": 0.8166599973322659, + "nauc_precision_at_10_max": 0.7872615712951837, + "nauc_precision_at_10_std": -0.2622248899559834, + "nauc_precision_at_1_diff1": 0.914007925160341, + "nauc_precision_at_1_max": 0.8011901357626264, + "nauc_precision_at_1_std": -0.39925832119140625, + "nauc_precision_at_20_diff1": 0.8250411275621338, + "nauc_precision_at_20_max": 0.8003868213952249, + "nauc_precision_at_20_std": -0.3012316037526172, + "nauc_precision_at_3_diff1": 0.8521416556530094, + "nauc_precision_at_3_max": 0.7492265409107284, + "nauc_precision_at_3_std": -0.3856557130842243, + "nauc_precision_at_5_diff1": 0.8248792270531391, + "nauc_precision_at_5_max": 0.7668980635732537, + "nauc_precision_at_5_std": -0.3008159785653395, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.8078120136943575, + "nauc_recall_at_100_max": 0.9074074074074102, + "nauc_recall_at_100_std": 0.6934329287270279, + "nauc_recall_at_10_diff1": 0.8166599973322675, + "nauc_recall_at_10_max": 0.7872615712951853, + "nauc_recall_at_10_std": -0.2622248899559833, + "nauc_recall_at_1_diff1": 0.914007925160341, + "nauc_recall_at_1_max": 0.8011901357626264, + "nauc_recall_at_1_std": -0.39925832119140625, + "nauc_recall_at_20_diff1": 0.8250411275621354, + "nauc_recall_at_20_max": 0.8003868213952258, + "nauc_recall_at_20_std": -0.30123160375261115, + "nauc_recall_at_3_diff1": 0.8521416556530105, + "nauc_recall_at_3_max": 0.7492265409107289, + "nauc_recall_at_3_std": -0.3856557130842251, + "nauc_recall_at_5_diff1": 0.8248792270531401, + "nauc_recall_at_5_max": 0.7668980635732549, + "nauc_recall_at_5_std": -0.30081597856534015, + "ndcg_at_1": 0.774, + "ndcg_at_10": 0.85311, + "ndcg_at_100": 0.86738, + "ndcg_at_1000": 0.86821, + "ndcg_at_20": 0.86035, + "ndcg_at_3": 0.83138, + "ndcg_at_5": 0.8461, + "precision_at_1": 0.774, + "precision_at_10": 0.093, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "precision_at_20": 0.0479, + "precision_at_3": 0.29067, + "precision_at_5": 0.1816, + "recall_at_1": 0.774, + "recall_at_10": 0.93, + "recall_at_100": 0.994, + "recall_at_1000": 1.0, + "recall_at_20": 0.958, + "recall_at_3": 0.872, + "recall_at_5": 0.908 + } + ] + }, + "task_name": "VidoreArxivQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreDocVQARetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..c376dd98c2 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b1d89eda849e636676df6ead8002602fb1858600", + "evaluation_time": 636.6930768489838, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.4, + "cv_recall_at_10": 0.678, + "cv_recall_at_100": 0.904, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.742, + "cv_recall_at_3": 0.526, + "cv_recall_at_5": 0.586, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.49756, + "map_at_1": 0.4, + "map_at_10": 0.48082, + "map_at_100": 0.48939, + "map_at_1000": 0.48988, + "map_at_20": 0.48533, + "map_at_3": 0.455, + "map_at_5": 0.4683, + "mrr_at_1": 0.4, + "mrr_at_10": 0.4808214285714284, + "mrr_at_100": 0.48938796963272635, + "mrr_at_1000": 0.48987689381415356, + "mrr_at_20": 0.48532962232985455, + "mrr_at_3": 0.45500000000000024, + "mrr_at_5": 0.46830000000000005, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5545440398381588, + "nauc_cv_recall_at_100_max": 0.6224031279178338, + "nauc_cv_recall_at_100_std": 0.7042483660130732, + "nauc_cv_recall_at_10_diff1": 0.5922844253579569, + "nauc_cv_recall_at_10_max": 0.5432800999287697, + "nauc_cv_recall_at_10_std": 0.3076353976332295, + "nauc_cv_recall_at_1_diff1": 0.7808676622039137, + "nauc_cv_recall_at_1_max": 0.42874356333676633, + "nauc_cv_recall_at_1_std": 0.023313594232749747, + "nauc_cv_recall_at_20_diff1": 0.6226725372659271, + "nauc_cv_recall_at_20_max": 0.5880120622044127, + "nauc_cv_recall_at_20_std": 0.38371357569278075, + "nauc_cv_recall_at_3_diff1": 0.6621787729379458, + "nauc_cv_recall_at_3_max": 0.47943079125818877, + "nauc_cv_recall_at_3_std": 0.14791707160827738, + "nauc_cv_recall_at_5_diff1": 0.6327712997671421, + "nauc_cv_recall_at_5_max": 0.5113726338055739, + "nauc_cv_recall_at_5_std": 0.19278695996423892, + "nauc_map_at_1000_diff1": 0.7223029853211699, + "nauc_map_at_1000_max": 0.4573500912745452, + "nauc_map_at_1000_std": 0.0938975472934603, + "nauc_map_at_100_diff1": 0.7220311079046942, + "nauc_map_at_100_max": 0.4575388060411859, + "nauc_map_at_100_std": 0.09469610080340737, + "nauc_map_at_10_diff1": 0.7194427550991311, + "nauc_map_at_10_max": 0.4566455594885925, + "nauc_map_at_10_std": 0.09238228683223873, + "nauc_map_at_1_diff1": 0.7808676622039137, + "nauc_map_at_1_max": 0.42874356333676633, + "nauc_map_at_1_std": 0.023313594232749747, + "nauc_map_at_20_diff1": 0.7219933633150782, + "nauc_map_at_20_max": 0.4582682569401601, + "nauc_map_at_20_std": 0.09425310386243768, + "nauc_map_at_3_diff1": 0.7271767768253766, + "nauc_map_at_3_max": 0.44831239354747643, + "nauc_map_at_3_std": 0.07368394666258558, + "nauc_map_at_5_diff1": 0.7221060762465903, + "nauc_map_at_5_max": 0.45374468132339285, + "nauc_map_at_5_std": 0.08090647086207076, + "nauc_mrr_at_1000_diff1": 0.7223029853211699, + "nauc_mrr_at_1000_max": 0.4573500912745452, + "nauc_mrr_at_1000_std": 0.0938975472934603, + "nauc_mrr_at_100_diff1": 0.7220311079046942, + "nauc_mrr_at_100_max": 0.4575388060411859, + "nauc_mrr_at_100_std": 0.09469610080340737, + "nauc_mrr_at_10_diff1": 0.7194427550991311, + "nauc_mrr_at_10_max": 0.4566455594885925, + "nauc_mrr_at_10_std": 0.09238228683223873, + "nauc_mrr_at_1_diff1": 0.7808676622039137, + "nauc_mrr_at_1_max": 0.42874356333676633, + "nauc_mrr_at_1_std": 0.023313594232749747, + "nauc_mrr_at_20_diff1": 0.7219933633150782, + "nauc_mrr_at_20_max": 0.4582682569401601, + "nauc_mrr_at_20_std": 0.09425310386243768, + "nauc_mrr_at_3_diff1": 0.7271767768253766, + "nauc_mrr_at_3_max": 0.44831239354747643, + "nauc_mrr_at_3_std": 0.07368394666258558, + "nauc_mrr_at_5_diff1": 0.7221060762465903, + "nauc_mrr_at_5_max": 0.45374468132339285, + "nauc_mrr_at_5_std": 0.08090647086207076, + "nauc_ndcg_at_1000_diff1": 0.710686523220205, + "nauc_ndcg_at_1000_max": 0.46774398996740346, + "nauc_ndcg_at_1000_std": 0.12081898918084653, + "nauc_ndcg_at_100_diff1": 0.7038461631864035, + "nauc_ndcg_at_100_max": 0.4740669291705547, + "nauc_ndcg_at_100_std": 0.1453284101083725, + "nauc_ndcg_at_10_diff1": 0.6931946908889765, + "nauc_ndcg_at_10_max": 0.4740638945720107, + "nauc_ndcg_at_10_std": 0.13525061573977884, + "nauc_ndcg_at_1_diff1": 0.7808676622039137, + "nauc_ndcg_at_1_max": 0.42874356333676633, + "nauc_ndcg_at_1_std": 0.023313594232749747, + "nauc_ndcg_at_20_diff1": 0.7022453664871185, + "nauc_ndcg_at_20_max": 0.48027538874049586, + "nauc_ndcg_at_20_std": 0.14356378032475198, + "nauc_ndcg_at_3_diff1": 0.710802049583976, + "nauc_ndcg_at_3_max": 0.4560014006504877, + "nauc_ndcg_at_3_std": 0.09211662046447473, + "nauc_ndcg_at_5_diff1": 0.7008949703273786, + "nauc_ndcg_at_5_max": 0.46696217106895704, + "nauc_ndcg_at_5_std": 0.10693699002475192, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5545440398381573, + "nauc_precision_at_100_max": 0.6224031279178333, + "nauc_precision_at_100_std": 0.7042483660130705, + "nauc_precision_at_10_diff1": 0.5922844253579582, + "nauc_precision_at_10_max": 0.5432800999287695, + "nauc_precision_at_10_std": 0.30763539763322983, + "nauc_precision_at_1_diff1": 0.7808676622039137, + "nauc_precision_at_1_max": 0.42874356333676633, + "nauc_precision_at_1_std": 0.023313594232749747, + "nauc_precision_at_20_diff1": 0.6226725372659271, + "nauc_precision_at_20_max": 0.5880120622044128, + "nauc_precision_at_20_std": 0.3837135756927807, + "nauc_precision_at_3_diff1": 0.6621787729379459, + "nauc_precision_at_3_max": 0.47943079125818866, + "nauc_precision_at_3_std": 0.14791707160827736, + "nauc_precision_at_5_diff1": 0.6327712997671422, + "nauc_precision_at_5_max": 0.5113726338055742, + "nauc_precision_at_5_std": 0.19278695996423872, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5545440398381588, + "nauc_recall_at_100_max": 0.6224031279178338, + "nauc_recall_at_100_std": 0.7042483660130732, + "nauc_recall_at_10_diff1": 0.5922844253579569, + "nauc_recall_at_10_max": 0.5432800999287697, + "nauc_recall_at_10_std": 0.3076353976332295, + "nauc_recall_at_1_diff1": 0.7808676622039137, + "nauc_recall_at_1_max": 0.42874356333676633, + "nauc_recall_at_1_std": 0.023313594232749747, + "nauc_recall_at_20_diff1": 0.6226725372659271, + "nauc_recall_at_20_max": 0.5880120622044127, + "nauc_recall_at_20_std": 0.38371357569278075, + "nauc_recall_at_3_diff1": 0.6621787729379458, + "nauc_recall_at_3_max": 0.47943079125818877, + "nauc_recall_at_3_std": 0.14791707160827738, + "nauc_recall_at_5_diff1": 0.6327712997671421, + "nauc_recall_at_5_max": 0.5113726338055739, + "nauc_recall_at_5_std": 0.19278695996423892, + "ndcg_at_1": 0.4, + "ndcg_at_10": 0.52756, + "ndcg_at_100": 0.57365, + "ndcg_at_1000": 0.58612, + "ndcg_at_20": 0.54381, + "ndcg_at_3": 0.47321, + "ndcg_at_5": 0.49756, + "precision_at_1": 0.4, + "precision_at_10": 0.0678, + "precision_at_100": 0.00904, + "precision_at_1000": 0.001, + "precision_at_20": 0.0371, + "precision_at_3": 0.17533, + "precision_at_5": 0.1172, + "recall_at_1": 0.4, + "recall_at_10": 0.678, + "recall_at_100": 0.904, + "recall_at_1000": 1.0, + "recall_at_20": 0.742, + "recall_at_3": 0.526, + "recall_at_5": 0.586 + } + ] + }, + "task_name": "VidoreDocVQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreInfoVQARetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..ebe098be22 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", + "evaluation_time": 992.0337262153625, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.804, + "cv_recall_at_10": 0.942, + "cv_recall_at_100": 0.98, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.964, + "cv_recall_at_3": 0.878, + "cv_recall_at_5": 0.912, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.86114, + "map_at_1": 0.804, + "map_at_10": 0.8487, + "map_at_100": 0.85081, + "map_at_1000": 0.85089, + "map_at_20": 0.85029, + "map_at_3": 0.83667, + "map_at_5": 0.84427, + "mrr_at_1": 0.804, + "mrr_at_10": 0.8486992063492061, + "mrr_at_100": 0.8508131647390718, + "mrr_at_1000": 0.8508934544613491, + "mrr_at_20": 0.8502916295692607, + "mrr_at_3": 0.8366666666666666, + "mrr_at_5": 0.8442666666666663, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.8513071895424827, + "nauc_cv_recall_at_100_max": 0.8503734827264224, + "nauc_cv_recall_at_100_std": 0.7185340802987861, + "nauc_cv_recall_at_10_diff1": 0.834991467851509, + "nauc_cv_recall_at_10_max": 0.7839112656556875, + "nauc_cv_recall_at_10_std": 0.29012524550049995, + "nauc_cv_recall_at_1_diff1": 0.8625328284370615, + "nauc_cv_recall_at_1_max": 0.4626162991917587, + "nauc_cv_recall_at_1_std": -0.20730267310002226, + "nauc_cv_recall_at_20_diff1": 0.826097105508873, + "nauc_cv_recall_at_20_max": 0.7789189749974079, + "nauc_cv_recall_at_20_std": 0.5447660545699772, + "nauc_cv_recall_at_3_diff1": 0.8494693608816293, + "nauc_cv_recall_at_3_max": 0.5553683121443455, + "nauc_cv_recall_at_3_std": -0.17134862284648175, + "nauc_cv_recall_at_5_diff1": 0.8283783210253801, + "nauc_cv_recall_at_5_max": 0.6534462269756393, + "nauc_cv_recall_at_5_std": 0.020870469399881346, + "nauc_map_at_1000_diff1": 0.8581318188677318, + "nauc_map_at_1000_max": 0.5029151761338965, + "nauc_map_at_1000_std": -0.170525470223046, + "nauc_map_at_100_diff1": 0.8581332599209811, + "nauc_map_at_100_max": 0.503110087695966, + "nauc_map_at_100_std": -0.17004814783619687, + "nauc_map_at_10_diff1": 0.8577013332981014, + "nauc_map_at_10_max": 0.5067936179071768, + "nauc_map_at_10_std": -0.16767133793646685, + "nauc_map_at_1_diff1": 0.8625328284370615, + "nauc_map_at_1_max": 0.4626162991917587, + "nauc_map_at_1_std": -0.20730267310002226, + "nauc_map_at_20_diff1": 0.8579526180444351, + "nauc_map_at_20_max": 0.5035773415391347, + "nauc_map_at_20_std": -0.16871880221874927, + "nauc_map_at_3_diff1": 0.8590778618599331, + "nauc_map_at_3_max": 0.49468576246010537, + "nauc_map_at_3_std": -0.19930147132620046, + "nauc_map_at_5_diff1": 0.856560029179217, + "nauc_map_at_5_max": 0.5033649175384577, + "nauc_map_at_5_std": -0.17793292309864372, + "nauc_mrr_at_1000_diff1": 0.8581318188677318, + "nauc_mrr_at_1000_max": 0.5029151761338965, + "nauc_mrr_at_1000_std": -0.170525470223046, + "nauc_mrr_at_100_diff1": 0.8581332599209811, + "nauc_mrr_at_100_max": 0.503110087695966, + "nauc_mrr_at_100_std": -0.17004814783619687, + "nauc_mrr_at_10_diff1": 0.8577013332981014, + "nauc_mrr_at_10_max": 0.5067936179071768, + "nauc_mrr_at_10_std": -0.16767133793646685, + "nauc_mrr_at_1_diff1": 0.8625328284370615, + "nauc_mrr_at_1_max": 0.4626162991917587, + "nauc_mrr_at_1_std": -0.20730267310002226, + "nauc_mrr_at_20_diff1": 0.8579526180444351, + "nauc_mrr_at_20_max": 0.5035773415391347, + "nauc_mrr_at_20_std": -0.16871880221874927, + "nauc_mrr_at_3_diff1": 0.8590778618599331, + "nauc_mrr_at_3_max": 0.49468576246010537, + "nauc_mrr_at_3_std": -0.19930147132620046, + "nauc_mrr_at_5_diff1": 0.856560029179217, + "nauc_mrr_at_5_max": 0.5033649175384577, + "nauc_mrr_at_5_std": -0.17793292309864372, + "nauc_ndcg_at_1000_diff1": 0.8570009197459093, + "nauc_ndcg_at_1000_max": 0.515089390761954, + "nauc_ndcg_at_1000_std": -0.14762192315949124, + "nauc_ndcg_at_100_diff1": 0.8568954592274121, + "nauc_ndcg_at_100_max": 0.5223246104075928, + "nauc_ndcg_at_100_std": -0.12904095017909736, + "nauc_ndcg_at_10_diff1": 0.8548433259646303, + "nauc_ndcg_at_10_max": 0.5385901444924547, + "nauc_ndcg_at_10_std": -0.11731864860745544, + "nauc_ndcg_at_1_diff1": 0.8625328284370615, + "nauc_ndcg_at_1_max": 0.4626162991917587, + "nauc_ndcg_at_1_std": -0.20730267310002226, + "nauc_ndcg_at_20_diff1": 0.855337897660986, + "nauc_ndcg_at_20_max": 0.5263685689351201, + "nauc_ndcg_at_20_std": -0.11775218712386955, + "nauc_ndcg_at_3_diff1": 0.8571055573097612, + "nauc_ndcg_at_3_max": 0.5074477037297324, + "nauc_ndcg_at_3_std": -0.19362485217798167, + "nauc_ndcg_at_5_diff1": 0.8519901570958626, + "nauc_ndcg_at_5_max": 0.5278319529214093, + "nauc_ndcg_at_5_std": -0.14676364377526235, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.8513071895424774, + "nauc_precision_at_100_max": 0.8503734827264159, + "nauc_precision_at_100_std": 0.7185340802987755, + "nauc_precision_at_10_diff1": 0.8349914678515092, + "nauc_precision_at_10_max": 0.7839112656556898, + "nauc_precision_at_10_std": 0.2901252455005006, + "nauc_precision_at_1_diff1": 0.8625328284370615, + "nauc_precision_at_1_max": 0.4626162991917587, + "nauc_precision_at_1_std": -0.20730267310002226, + "nauc_precision_at_20_diff1": 0.8260971055088702, + "nauc_precision_at_20_max": 0.7789189749974024, + "nauc_precision_at_20_std": 0.5447660545699731, + "nauc_precision_at_3_diff1": 0.8494693608816277, + "nauc_precision_at_3_max": 0.5553683121443457, + "nauc_precision_at_3_std": -0.1713486228464822, + "nauc_precision_at_5_diff1": 0.8283783210253806, + "nauc_precision_at_5_max": 0.6534462269756387, + "nauc_precision_at_5_std": 0.020870469399880635, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.8513071895424827, + "nauc_recall_at_100_max": 0.8503734827264224, + "nauc_recall_at_100_std": 0.7185340802987861, + "nauc_recall_at_10_diff1": 0.834991467851509, + "nauc_recall_at_10_max": 0.7839112656556875, + "nauc_recall_at_10_std": 0.29012524550049995, + "nauc_recall_at_1_diff1": 0.8625328284370615, + "nauc_recall_at_1_max": 0.4626162991917587, + "nauc_recall_at_1_std": -0.20730267310002226, + "nauc_recall_at_20_diff1": 0.826097105508873, + "nauc_recall_at_20_max": 0.7789189749974079, + "nauc_recall_at_20_std": 0.5447660545699772, + "nauc_recall_at_3_diff1": 0.8494693608816293, + "nauc_recall_at_3_max": 0.5553683121443455, + "nauc_recall_at_3_std": -0.17134862284648175, + "nauc_recall_at_5_diff1": 0.8283783210253801, + "nauc_recall_at_5_max": 0.6534462269756393, + "nauc_recall_at_5_std": 0.020870469399881346, + "ndcg_at_1": 0.804, + "ndcg_at_10": 0.87127, + "ndcg_at_100": 0.88009, + "ndcg_at_1000": 0.88257, + "ndcg_at_20": 0.87692, + "ndcg_at_3": 0.84728, + "ndcg_at_5": 0.86114, + "precision_at_1": 0.804, + "precision_at_10": 0.0942, + "precision_at_100": 0.0098, + "precision_at_1000": 0.001, + "precision_at_20": 0.0482, + "precision_at_3": 0.29267, + "precision_at_5": 0.1824, + "recall_at_1": 0.804, + "recall_at_10": 0.942, + "recall_at_100": 0.98, + "recall_at_1000": 1.0, + "recall_at_20": 0.964, + "recall_at_3": 0.878, + "recall_at_5": 0.912 + } + ] + }, + "task_name": "VidoreInfoVQARetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreShiftProjectRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..30cf8399b9 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", + "evaluation_time": 1577.1782314777374, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.56, + "cv_recall_at_10": 0.98, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.99, + "cv_recall_at_3": 0.87, + "cv_recall_at_5": 0.94, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.77482, + "map_at_1": 0.56, + "map_at_10": 0.72388, + "map_at_100": 0.72486, + "map_at_1000": 0.72486, + "map_at_20": 0.7246, + "map_at_3": 0.70167, + "map_at_5": 0.71867, + "mrr_at_1": 0.56, + "mrr_at_10": 0.7238849206349208, + "mrr_at_100": 0.7248623642439433, + "mrr_at_1000": 0.7248623642439433, + "mrr_at_20": 0.7245992063492065, + "mrr_at_3": 0.7016666666666669, + "mrr_at_5": 0.7186666666666668, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": -0.5634920634920563, + "nauc_cv_recall_at_10_max": 0.3384687208216605, + "nauc_cv_recall_at_10_std": -0.5634920634920563, + "nauc_cv_recall_at_1_diff1": 0.6556609130138538, + "nauc_cv_recall_at_1_max": 0.17394390188507777, + "nauc_cv_recall_at_1_std": -0.47217806041335475, + "nauc_cv_recall_at_20_diff1": -0.5634920634920657, + "nauc_cv_recall_at_20_max": 0.12278244631185359, + "nauc_cv_recall_at_20_std": -0.5634920634920657, + "nauc_cv_recall_at_3_diff1": 0.5655409908909134, + "nauc_cv_recall_at_3_max": 0.4225357328001189, + "nauc_cv_recall_at_3_std": -0.6214544915944596, + "nauc_cv_recall_at_5_diff1": 0.30376595082477537, + "nauc_cv_recall_at_5_max": 0.25676937441643427, + "nauc_cv_recall_at_5_std": -0.7432306255835657, + "nauc_map_at_1000_diff1": 0.6230116173866328, + "nauc_map_at_1000_max": 0.20589579326708154, + "nauc_map_at_1000_std": -0.5150012670469309, + "nauc_map_at_100_diff1": 0.6230116173866328, + "nauc_map_at_100_max": 0.20589579326708154, + "nauc_map_at_100_std": -0.5150012670469309, + "nauc_map_at_10_diff1": 0.617681710216042, + "nauc_map_at_10_max": 0.2069385042265956, + "nauc_map_at_10_std": -0.5152190931252199, + "nauc_map_at_1_diff1": 0.6556609130138538, + "nauc_map_at_1_max": 0.17394390188507777, + "nauc_map_at_1_std": -0.47217806041335475, + "nauc_map_at_20_diff1": 0.6215719162765422, + "nauc_map_at_20_max": 0.20579494370394566, + "nauc_map_at_20_std": -0.515060105678952, + "nauc_map_at_3_diff1": 0.6329090417439073, + "nauc_map_at_3_max": 0.23321919704961555, + "nauc_map_at_3_std": -0.5188096812763278, + "nauc_map_at_5_diff1": 0.620811050355049, + "nauc_map_at_5_max": 0.20777525487629664, + "nauc_map_at_5_std": -0.522137936670128, + "nauc_mrr_at_1000_diff1": 0.6230116173866328, + "nauc_mrr_at_1000_max": 0.20589579326708154, + "nauc_mrr_at_1000_std": -0.5150012670469309, + "nauc_mrr_at_100_diff1": 0.6230116173866328, + "nauc_mrr_at_100_max": 0.20589579326708154, + "nauc_mrr_at_100_std": -0.5150012670469309, + "nauc_mrr_at_10_diff1": 0.617681710216042, + "nauc_mrr_at_10_max": 0.2069385042265956, + "nauc_mrr_at_10_std": -0.5152190931252199, + "nauc_mrr_at_1_diff1": 0.6556609130138538, + "nauc_mrr_at_1_max": 0.17394390188507777, + "nauc_mrr_at_1_std": -0.47217806041335475, + "nauc_mrr_at_20_diff1": 0.6215719162765422, + "nauc_mrr_at_20_max": 0.20579494370394566, + "nauc_mrr_at_20_std": -0.515060105678952, + "nauc_mrr_at_3_diff1": 0.6329090417439073, + "nauc_mrr_at_3_max": 0.23321919704961555, + "nauc_mrr_at_3_std": -0.5188096812763278, + "nauc_mrr_at_5_diff1": 0.620811050355049, + "nauc_mrr_at_5_max": 0.20777525487629664, + "nauc_mrr_at_5_std": -0.522137936670128, + "nauc_ndcg_at_1000_diff1": 0.6144858168068215, + "nauc_ndcg_at_1000_max": 0.20826882361687157, + "nauc_ndcg_at_1000_std": -0.519398085858273, + "nauc_ndcg_at_100_diff1": 0.6144858168068215, + "nauc_ndcg_at_100_max": 0.20826882361687157, + "nauc_ndcg_at_100_std": -0.519398085858273, + "nauc_ndcg_at_10_diff1": 0.5834552836723182, + "nauc_ndcg_at_10_max": 0.2125506181876858, + "nauc_ndcg_at_10_std": -0.5205596183937152, + "nauc_ndcg_at_1_diff1": 0.6556609130138538, + "nauc_ndcg_at_1_max": 0.17394390188507777, + "nauc_ndcg_at_1_std": -0.47217806041335475, + "nauc_ndcg_at_20_diff1": 0.6010944498115274, + "nauc_ndcg_at_20_max": 0.20729700617383273, + "nauc_ndcg_at_20_std": -0.5198993504801755, + "nauc_ndcg_at_3_diff1": 0.6224241742598228, + "nauc_ndcg_at_3_max": 0.26251029455353375, + "nauc_ndcg_at_3_std": -0.5352297029710446, + "nauc_ndcg_at_5_diff1": 0.5947188948668818, + "nauc_ndcg_at_5_max": 0.21356792656360682, + "nauc_ndcg_at_5_std": -0.5419445301313862, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": -0.5634920634920675, + "nauc_precision_at_10_max": 0.3384687208216692, + "nauc_precision_at_10_std": -0.5634920634920675, + "nauc_precision_at_1_diff1": 0.6556609130138538, + "nauc_precision_at_1_max": 0.17394390188507777, + "nauc_precision_at_1_std": -0.47217806041335475, + "nauc_precision_at_20_diff1": -0.5634920634920767, + "nauc_precision_at_20_max": 0.12278244631185926, + "nauc_precision_at_20_std": -0.5634920634920767, + "nauc_precision_at_3_diff1": 0.5655409908909123, + "nauc_precision_at_3_max": 0.42253573280011736, + "nauc_precision_at_3_std": -0.6214544915944612, + "nauc_precision_at_5_diff1": 0.3037659508247733, + "nauc_precision_at_5_max": 0.2567693744164348, + "nauc_precision_at_5_std": -0.7432306255835679, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": -0.5634920634920563, + "nauc_recall_at_10_max": 0.3384687208216605, + "nauc_recall_at_10_std": -0.5634920634920563, + "nauc_recall_at_1_diff1": 0.6556609130138538, + "nauc_recall_at_1_max": 0.17394390188507777, + "nauc_recall_at_1_std": -0.47217806041335475, + "nauc_recall_at_20_diff1": -0.5634920634920657, + "nauc_recall_at_20_max": 0.12278244631185359, + "nauc_recall_at_20_std": -0.5634920634920657, + "nauc_recall_at_3_diff1": 0.5655409908909134, + "nauc_recall_at_3_max": 0.4225357328001189, + "nauc_recall_at_3_std": -0.6214544915944596, + "nauc_recall_at_5_diff1": 0.30376595082477537, + "nauc_recall_at_5_max": 0.25676937441643427, + "nauc_recall_at_5_std": -0.7432306255835657, + "ndcg_at_1": 0.56, + "ndcg_at_10": 0.78765, + "ndcg_at_100": 0.79211, + "ndcg_at_1000": 0.79211, + "ndcg_at_20": 0.79021, + "ndcg_at_3": 0.74511, + "ndcg_at_5": 0.77482, + "precision_at_1": 0.56, + "precision_at_10": 0.098, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.0495, + "precision_at_3": 0.29, + "precision_at_5": 0.188, + "recall_at_1": 0.56, + "recall_at_10": 0.98, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.99, + "recall_at_3": 0.87, + "recall_at_5": 0.94 + } + ] + }, + "task_name": "VidoreShiftProjectRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAAIRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..f53016df4c --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", + "evaluation_time": 947.4307334423065, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.59, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 0.99, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.83559, + "map_at_1": 0.6, + "map_at_10": 0.78333, + "map_at_100": 0.78333, + "map_at_1000": 0.78333, + "map_at_20": 0.78333, + "map_at_3": 0.78167, + "map_at_5": 0.78167, + "mrr_at_1": 0.59, + "mrr_at_10": 0.7814285714285714, + "mrr_at_100": 0.7814285714285714, + "mrr_at_1000": 0.7814285714285714, + "mrr_at_20": 0.7814285714285714, + "mrr_at_3": 0.78, + "mrr_at_5": 0.78, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": 0.5448662450291281, + "nauc_cv_recall_at_1_max": 0.12269065402732315, + "nauc_cv_recall_at_1_std": -0.08235138744185386, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": -0.17133520074696149, + "nauc_cv_recall_at_3_max": 1.0, + "nauc_cv_recall_at_3_std": 0.554154995331463, + "nauc_cv_recall_at_5_diff1": -0.17133520074696149, + "nauc_cv_recall_at_5_max": 1.0, + "nauc_cv_recall_at_5_std": 0.554154995331463, + "nauc_map_at_1000_diff1": 0.4738056897477185, + "nauc_map_at_1000_max": 0.15407944176060157, + "nauc_map_at_1000_std": -0.08807478976561002, + "nauc_map_at_100_diff1": 0.4738056897477185, + "nauc_map_at_100_max": 0.15407944176060157, + "nauc_map_at_100_std": -0.08807478976561002, + "nauc_map_at_10_diff1": 0.4738056897477185, + "nauc_map_at_10_max": 0.15407944176060157, + "nauc_map_at_10_std": -0.08807478976561002, + "nauc_map_at_1_diff1": 0.5112644701691897, + "nauc_map_at_1_max": 0.1607153457999401, + "nauc_map_at_1_std": -0.09436034431582087, + "nauc_map_at_20_diff1": 0.4738056897477185, + "nauc_map_at_20_max": 0.15407944176060157, + "nauc_map_at_20_std": -0.08807478976561002, + "nauc_map_at_3_diff1": 0.4676830511027818, + "nauc_map_at_3_max": 0.16210755775314364, + "nauc_map_at_3_std": -0.08197977864617986, + "nauc_map_at_5_diff1": 0.4676830511027818, + "nauc_map_at_5_max": 0.16210755775314364, + "nauc_map_at_5_std": -0.08197977864617986, + "nauc_mrr_at_1000_diff1": 0.5025737502583414, + "nauc_mrr_at_1000_max": 0.15191339579868746, + "nauc_mrr_at_1000_std": -0.04260546581499429, + "nauc_mrr_at_100_diff1": 0.5025737502583414, + "nauc_mrr_at_100_max": 0.15191339579868746, + "nauc_mrr_at_100_std": -0.04260546581499429, + "nauc_mrr_at_10_diff1": 0.5025737502583414, + "nauc_mrr_at_10_max": 0.15191339579868746, + "nauc_mrr_at_10_std": -0.04260546581499429, + "nauc_mrr_at_1_diff1": 0.5448662450291281, + "nauc_mrr_at_1_max": 0.12269065402732315, + "nauc_mrr_at_1_std": -0.08235138744185386, + "nauc_mrr_at_20_diff1": 0.5025737502583414, + "nauc_mrr_at_20_max": 0.15191339579868746, + "nauc_mrr_at_20_std": -0.04260546581499429, + "nauc_mrr_at_3_diff1": 0.4970431657570428, + "nauc_mrr_at_3_max": 0.15887340815934356, + "nauc_mrr_at_3_std": -0.03770801764713768, + "nauc_mrr_at_5_diff1": 0.4970431657570428, + "nauc_mrr_at_5_max": 0.15887340815934356, + "nauc_mrr_at_5_std": -0.03770801764713768, + "nauc_ndcg_at_1000_diff1": 0.4710027496985591, + "nauc_ndcg_at_1000_max": 0.15453799577065144, + "nauc_ndcg_at_1000_std": -0.0870733233833494, + "nauc_ndcg_at_100_diff1": 0.4710027496985591, + "nauc_ndcg_at_100_max": 0.15453799577065144, + "nauc_ndcg_at_100_std": -0.0870733233833494, + "nauc_ndcg_at_10_diff1": 0.4710027496985591, + "nauc_ndcg_at_10_max": 0.15453799577065144, + "nauc_ndcg_at_10_std": -0.0870733233833494, + "nauc_ndcg_at_1_diff1": 0.5112644701691897, + "nauc_ndcg_at_1_max": 0.1607153457999401, + "nauc_ndcg_at_1_std": -0.09436034431582087, + "nauc_ndcg_at_20_diff1": 0.4710027496985591, + "nauc_ndcg_at_20_max": 0.15453799577065144, + "nauc_ndcg_at_20_std": -0.0870733233833494, + "nauc_ndcg_at_3_diff1": 0.45378387388053215, + "nauc_ndcg_at_3_max": 0.17720193030789522, + "nauc_ndcg_at_3_std": -0.06988419298354469, + "nauc_ndcg_at_5_diff1": 0.45378387388053215, + "nauc_ndcg_at_5_max": 0.17720193030789522, + "nauc_ndcg_at_5_std": -0.06988419298354469, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 0.5112644701691897, + "nauc_precision_at_1_max": 0.1607153457999401, + "nauc_precision_at_1_std": -0.09436034431582087, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": -0.1713352007469878, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 0.5541549953314585, + "nauc_precision_at_5_diff1": -0.1713352007469681, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 0.5541549953314738, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": 0.5112644701691897, + "nauc_recall_at_1_max": 0.1607153457999401, + "nauc_recall_at_1_std": -0.09436034431582087, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": -0.17133520074696149, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": 0.554154995331463, + "nauc_recall_at_5_diff1": -0.17133520074696149, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": 0.554154995331463, + "ndcg_at_1": 0.6, + "ndcg_at_10": 0.83915, + "ndcg_at_100": 0.83915, + "ndcg_at_1000": 0.83915, + "ndcg_at_20": 0.83915, + "ndcg_at_3": 0.83559, + "ndcg_at_5": 0.83559, + "precision_at_1": 0.6, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33, + "precision_at_5": 0.198, + "recall_at_1": 0.6, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.99, + "recall_at_5": 0.99 + } + ] + }, + "task_name": "VidoreSyntheticDocQAAIRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAEnergyRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..ca9fa98ef9 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", + "evaluation_time": 974.9262182712555, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.62, + "cv_recall_at_10": 0.97, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.99, + "cv_recall_at_3": 0.91, + "cv_recall_at_5": 0.93, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.79422, + "map_at_1": 0.59, + "map_at_10": 0.75287, + "map_at_100": 0.75479, + "map_at_1000": 0.75479, + "map_at_20": 0.75442, + "map_at_3": 0.745, + "map_at_5": 0.747, + "mrr_at_1": 0.62, + "mrr_at_10": 0.7637063492063493, + "mrr_at_100": 0.7656243386243387, + "mrr_at_1000": 0.7656243386243387, + "mrr_at_20": 0.7652539682539683, + "mrr_at_3": 0.7533333333333333, + "mrr_at_5": 0.7578333333333334, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": 0.27964519140989424, + "nauc_cv_recall_at_10_max": 0.13414254590725253, + "nauc_cv_recall_at_10_std": -0.9229691876750729, + "nauc_cv_recall_at_1_diff1": 0.6289419438205704, + "nauc_cv_recall_at_1_max": -0.04647263329538835, + "nauc_cv_recall_at_1_std": -0.4368212165030922, + "nauc_cv_recall_at_20_diff1": 0.35807656395891085, + "nauc_cv_recall_at_20_max": 1.0, + "nauc_cv_recall_at_20_std": -1.151727357609709, + "nauc_cv_recall_at_3_diff1": 0.2426081543728594, + "nauc_cv_recall_at_3_max": 0.39864093785662147, + "nauc_cv_recall_at_3_std": -0.8358232181761602, + "nauc_cv_recall_at_5_diff1": 0.3188608776844068, + "nauc_cv_recall_at_5_max": 0.4858610110710928, + "nauc_cv_recall_at_5_std": -0.9136321195144755, + "nauc_map_at_1000_diff1": 0.6050487509101241, + "nauc_map_at_1000_max": 0.08450426872364926, + "nauc_map_at_1000_std": -0.529759315257296, + "nauc_map_at_100_diff1": 0.6050487509101241, + "nauc_map_at_100_max": 0.08450426872364926, + "nauc_map_at_100_std": -0.529759315257296, + "nauc_map_at_10_diff1": 0.6018452314830903, + "nauc_map_at_10_max": 0.08272890897551674, + "nauc_map_at_10_std": -0.5336167734838453, + "nauc_map_at_1_diff1": 0.6693031241287217, + "nauc_map_at_1_max": 0.031432051300864135, + "nauc_map_at_1_std": -0.445859685679487, + "nauc_map_at_20_diff1": 0.6045885907957769, + "nauc_map_at_20_max": 0.08621002606252022, + "nauc_map_at_20_std": -0.53091816998107, + "nauc_map_at_3_diff1": 0.5869736640493909, + "nauc_map_at_3_max": 0.10354995309831401, + "nauc_map_at_3_std": -0.5425945788378098, + "nauc_map_at_5_diff1": 0.5943286796830018, + "nauc_map_at_5_max": 0.101081239769278, + "nauc_map_at_5_std": -0.5423918892300909, + "nauc_mrr_at_1000_diff1": 0.5629855995499428, + "nauc_mrr_at_1000_max": 0.025525805506282775, + "nauc_mrr_at_1000_std": -0.519994655773919, + "nauc_mrr_at_100_diff1": 0.5629855995499428, + "nauc_mrr_at_100_max": 0.025525805506282775, + "nauc_mrr_at_100_std": -0.519994655773919, + "nauc_mrr_at_10_diff1": 0.5601335343154699, + "nauc_mrr_at_10_max": 0.024292956739306003, + "nauc_mrr_at_10_std": -0.5240179228031371, + "nauc_mrr_at_1_diff1": 0.6289419438205704, + "nauc_mrr_at_1_max": -0.04647263329538835, + "nauc_mrr_at_1_std": -0.4368212165030922, + "nauc_mrr_at_20_diff1": 0.5625967679302477, + "nauc_mrr_at_20_max": 0.027374949826847985, + "nauc_mrr_at_20_std": -0.5211934201376318, + "nauc_mrr_at_3_diff1": 0.5414891458713736, + "nauc_mrr_at_3_max": 0.04508464642366023, + "nauc_mrr_at_3_std": -0.5338848688412485, + "nauc_mrr_at_5_diff1": 0.5536925278417223, + "nauc_mrr_at_5_max": 0.04466542695249133, + "nauc_mrr_at_5_std": -0.5332272586570691, + "nauc_ndcg_at_1000_diff1": 0.5975908372870623, + "nauc_ndcg_at_1000_max": 0.09253310457244465, + "nauc_ndcg_at_1000_std": -0.53968784190258, + "nauc_ndcg_at_100_diff1": 0.5975908372870623, + "nauc_ndcg_at_100_max": 0.09253310457244465, + "nauc_ndcg_at_100_std": -0.53968784190258, + "nauc_ndcg_at_10_diff1": 0.5826233093198272, + "nauc_ndcg_at_10_max": 0.09067392180823283, + "nauc_ndcg_at_10_std": -0.5575736421728364, + "nauc_ndcg_at_1_diff1": 0.6693031241287217, + "nauc_ndcg_at_1_max": 0.031432051300864135, + "nauc_ndcg_at_1_std": -0.445859685679487, + "nauc_ndcg_at_20_diff1": 0.5943230562558864, + "nauc_ndcg_at_20_max": 0.1049140080822575, + "nauc_ndcg_at_20_std": -0.5480381213743085, + "nauc_ndcg_at_3_diff1": 0.5475794752772646, + "nauc_ndcg_at_3_max": 0.14652511249455774, + "nauc_ndcg_at_3_std": -0.5823793030749803, + "nauc_ndcg_at_5_diff1": 0.5638248903434933, + "nauc_ndcg_at_5_max": 0.14174465488043955, + "nauc_ndcg_at_5_std": -0.582806100658493, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": 0.2796451914099003, + "nauc_precision_at_10_max": 0.13414254590725072, + "nauc_precision_at_10_std": -0.9229691876750711, + "nauc_precision_at_1_diff1": 0.6693031241287217, + "nauc_precision_at_1_max": 0.031432051300864135, + "nauc_precision_at_1_std": -0.445859685679487, + "nauc_precision_at_20_diff1": 0.35807656395891135, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": -1.1517273576097316, + "nauc_precision_at_3_diff1": 0.25758636788048667, + "nauc_precision_at_3_max": 0.469887955182072, + "nauc_precision_at_3_std": -0.86986461251167, + "nauc_precision_at_5_diff1": 0.3188608776844103, + "nauc_precision_at_5_max": 0.48586101107109614, + "nauc_precision_at_5_std": -0.9136321195144689, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.27964519140989424, + "nauc_recall_at_10_max": 0.13414254590725253, + "nauc_recall_at_10_std": -0.9229691876750729, + "nauc_recall_at_1_diff1": 0.6693031241287217, + "nauc_recall_at_1_max": 0.031432051300864135, + "nauc_recall_at_1_std": -0.445859685679487, + "nauc_recall_at_20_diff1": 0.35807656395891085, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": -1.151727357609709, + "nauc_recall_at_3_diff1": 0.2575863678804853, + "nauc_recall_at_3_max": 0.4698879551820732, + "nauc_recall_at_3_std": -0.869864612511674, + "nauc_recall_at_5_diff1": 0.3188608776844068, + "nauc_recall_at_5_max": 0.4858610110710928, + "nauc_recall_at_5_std": -0.9136321195144755, + "ndcg_at_1": 0.59, + "ndcg_at_10": 0.80769, + "ndcg_at_100": 0.81503, + "ndcg_at_1000": 0.81503, + "ndcg_at_20": 0.81295, + "ndcg_at_3": 0.79035, + "ndcg_at_5": 0.79422, + "precision_at_1": 0.59, + "precision_at_10": 0.097, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.0495, + "precision_at_3": 0.30667, + "precision_at_5": 0.186, + "recall_at_1": 0.59, + "recall_at_10": 0.97, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.99, + "recall_at_3": 0.92, + "recall_at_5": 0.93 + } + ] + }, + "task_name": "VidoreSyntheticDocQAEnergyRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..6db6c0dec9 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", + "evaluation_time": 986.1590669155121, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.61, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.83922, + "map_at_1": 0.61, + "map_at_10": 0.78367, + "map_at_100": 0.78367, + "map_at_1000": 0.78367, + "map_at_20": 0.78367, + "map_at_3": 0.78167, + "map_at_5": 0.78367, + "mrr_at_1": 0.61, + "mrr_at_10": 0.7825, + "mrr_at_100": 0.7825, + "mrr_at_1000": 0.7825, + "mrr_at_20": 0.7825, + "mrr_at_3": 0.78, + "mrr_at_5": 0.7825, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": 0.6026921275663585, + "nauc_cv_recall_at_1_max": 0.1182149169851797, + "nauc_cv_recall_at_1_std": -0.09300077047421948, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": 0.12278244631185359, + "nauc_cv_recall_at_3_max": 1.0, + "nauc_cv_recall_at_3_std": 0.35807656395891085, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_map_at_1000_diff1": 0.5422695338733823, + "nauc_map_at_1000_max": 0.26261751641037917, + "nauc_map_at_1000_std": -0.12059887347484721, + "nauc_map_at_100_diff1": 0.5422695338733823, + "nauc_map_at_100_max": 0.26261751641037917, + "nauc_map_at_100_std": -0.12059887347484721, + "nauc_map_at_10_diff1": 0.5422695338733823, + "nauc_map_at_10_max": 0.26261751641037917, + "nauc_map_at_10_std": -0.12059887347484721, + "nauc_map_at_1_diff1": 0.5832036620186425, + "nauc_map_at_1_max": 0.24653664284743093, + "nauc_map_at_1_std": -0.11910624990557943, + "nauc_map_at_20_diff1": 0.5422695338733823, + "nauc_map_at_20_max": 0.26261751641037917, + "nauc_map_at_20_std": -0.12059887347484721, + "nauc_map_at_3_diff1": 0.537540514107023, + "nauc_map_at_3_max": 0.2709302784563632, + "nauc_map_at_3_std": -0.11520260342884224, + "nauc_map_at_5_diff1": 0.5422695338733823, + "nauc_map_at_5_max": 0.26261751641037917, + "nauc_map_at_5_std": -0.12059887347484721, + "nauc_mrr_at_1000_diff1": 0.5704040484258083, + "nauc_mrr_at_1000_max": 0.10875070591557207, + "nauc_mrr_at_1000_std": -0.10805140830156752, + "nauc_mrr_at_100_diff1": 0.5704040484258083, + "nauc_mrr_at_100_max": 0.10875070591557207, + "nauc_mrr_at_100_std": -0.10805140830156752, + "nauc_mrr_at_10_diff1": 0.5704040484258083, + "nauc_mrr_at_10_max": 0.10875070591557207, + "nauc_mrr_at_10_std": -0.10805140830156752, + "nauc_mrr_at_1_diff1": 0.6026921275663585, + "nauc_mrr_at_1_max": 0.1182149169851797, + "nauc_mrr_at_1_std": -0.09300077047421948, + "nauc_mrr_at_20_diff1": 0.5704040484258083, + "nauc_mrr_at_20_max": 0.10875070591557207, + "nauc_mrr_at_20_std": -0.10805140830156752, + "nauc_mrr_at_3_diff1": 0.5641474056962875, + "nauc_mrr_at_3_max": 0.1212081663904222, + "nauc_mrr_at_3_std": -0.10153609229604636, + "nauc_mrr_at_5_diff1": 0.5704040484258083, + "nauc_mrr_at_5_max": 0.10875070591557207, + "nauc_mrr_at_5_std": -0.10805140830156752, + "nauc_ndcg_at_1000_diff1": 0.5397013077011296, + "nauc_ndcg_at_1000_max": 0.26399500098098233, + "nauc_ndcg_at_1000_std": -0.12037456199431834, + "nauc_ndcg_at_100_diff1": 0.5397013077011296, + "nauc_ndcg_at_100_max": 0.26399500098098233, + "nauc_ndcg_at_100_std": -0.12037456199431834, + "nauc_ndcg_at_10_diff1": 0.5397013077011296, + "nauc_ndcg_at_10_max": 0.26399500098098233, + "nauc_ndcg_at_10_std": -0.12037456199431834, + "nauc_ndcg_at_1_diff1": 0.5832036620186425, + "nauc_ndcg_at_1_max": 0.24653664284743093, + "nauc_ndcg_at_1_std": -0.11910624990557943, + "nauc_ndcg_at_20_diff1": 0.5397013077011296, + "nauc_ndcg_at_20_max": 0.26399500098098233, + "nauc_ndcg_at_20_std": -0.12037456199431834, + "nauc_ndcg_at_3_diff1": 0.5277027131062747, + "nauc_ndcg_at_3_max": 0.28517664187290104, + "nauc_ndcg_at_3_std": -0.10660511756420861, + "nauc_ndcg_at_5_diff1": 0.5397013077011296, + "nauc_ndcg_at_5_max": 0.26399500098098233, + "nauc_ndcg_at_5_std": -0.12037456199431834, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 0.5832036620186425, + "nauc_precision_at_1_max": 0.24653664284743093, + "nauc_precision_at_1_std": -0.11910624990557943, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 0.12278244631183229, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 0.35807656395889226, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": 0.5832036620186425, + "nauc_recall_at_1_max": 0.24653664284743093, + "nauc_recall_at_1_std": -0.11910624990557943, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": 0.12278244631185359, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": 0.35807656395891085, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 0.61, + "ndcg_at_10": 0.83922, + "ndcg_at_100": 0.83922, + "ndcg_at_1000": 0.83922, + "ndcg_at_20": 0.83922, + "ndcg_at_3": 0.83535, + "ndcg_at_5": 0.83922, + "precision_at_1": 0.61, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33, + "precision_at_5": 0.2, + "recall_at_1": 0.61, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.99, + "recall_at_5": 1.0 + } + ] + }, + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..59efeb2991 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", + "evaluation_time": 1046.7588300704956, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.56, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.82394, + "map_at_1": 0.57, + "map_at_10": 0.76317, + "map_at_100": 0.76317, + "map_at_1000": 0.76317, + "map_at_20": 0.76317, + "map_at_3": 0.75667, + "map_at_5": 0.76317, + "mrr_at_1": 0.56, + "mrr_at_10": 0.757, + "mrr_at_100": 0.757, + "mrr_at_1000": 0.757, + "mrr_at_20": 0.757, + "mrr_at_3": 0.755, + "mrr_at_5": 0.757, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": 0.6210680218033159, + "nauc_cv_recall_at_1_max": 0.17181467181467122, + "nauc_cv_recall_at_1_std": -0.34974733136497865, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_3_max": 1.0, + "nauc_cv_recall_at_3_std": -1.739962651727339, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_map_at_1000_diff1": 0.5566705457911219, + "nauc_map_at_1000_max": 0.2256523009570009, + "nauc_map_at_1000_std": -0.28558097348570577, + "nauc_map_at_100_diff1": 0.5566705457911219, + "nauc_map_at_100_max": 0.2256523009570009, + "nauc_map_at_100_std": -0.28558097348570577, + "nauc_map_at_10_diff1": 0.5566705457911219, + "nauc_map_at_10_max": 0.2256523009570009, + "nauc_map_at_10_std": -0.28558097348570577, + "nauc_map_at_1_diff1": 0.591997818630617, + "nauc_map_at_1_max": 0.19318035045421283, + "nauc_map_at_1_std": -0.2656247757638381, + "nauc_map_at_20_diff1": 0.5566705457911219, + "nauc_map_at_20_max": 0.2256523009570009, + "nauc_map_at_20_std": -0.28558097348570577, + "nauc_map_at_3_diff1": 0.5472291446587567, + "nauc_map_at_3_max": 0.23894981992601236, + "nauc_map_at_3_std": -0.2930617472275123, + "nauc_map_at_5_diff1": 0.5566705457911219, + "nauc_map_at_5_max": 0.2256523009570009, + "nauc_map_at_5_std": -0.28558097348570577, + "nauc_mrr_at_1000_diff1": 0.6054587201058389, + "nauc_mrr_at_1000_max": 0.18683215614528823, + "nauc_mrr_at_1000_std": -0.3503804825054683, + "nauc_mrr_at_100_diff1": 0.6054587201058389, + "nauc_mrr_at_100_max": 0.18683215614528823, + "nauc_mrr_at_100_std": -0.3503804825054683, + "nauc_mrr_at_10_diff1": 0.6054587201058389, + "nauc_mrr_at_10_max": 0.18683215614528823, + "nauc_mrr_at_10_std": -0.3503804825054683, + "nauc_mrr_at_1_diff1": 0.6210680218033159, + "nauc_mrr_at_1_max": 0.17181467181467122, + "nauc_mrr_at_1_std": -0.34974733136497865, + "nauc_mrr_at_20_diff1": 0.6054587201058389, + "nauc_mrr_at_20_max": 0.18683215614528823, + "nauc_mrr_at_20_std": -0.3503804825054683, + "nauc_mrr_at_3_diff1": 0.609638385387044, + "nauc_mrr_at_3_max": 0.19544663979491714, + "nauc_mrr_at_3_std": -0.3651013460603219, + "nauc_mrr_at_5_diff1": 0.6054587201058389, + "nauc_mrr_at_5_max": 0.18683215614528823, + "nauc_mrr_at_5_std": -0.3503804825054683, + "nauc_ndcg_at_1000_diff1": 0.5544855949125493, + "nauc_ndcg_at_1000_max": 0.22819031621405042, + "nauc_ndcg_at_1000_std": -0.2877622655219696, + "nauc_ndcg_at_100_diff1": 0.5544855949125493, + "nauc_ndcg_at_100_max": 0.22819031621405042, + "nauc_ndcg_at_100_std": -0.2877622655219696, + "nauc_ndcg_at_10_diff1": 0.5544855949125493, + "nauc_ndcg_at_10_max": 0.22819031621405042, + "nauc_ndcg_at_10_std": -0.2877622655219696, + "nauc_ndcg_at_1_diff1": 0.591997818630617, + "nauc_ndcg_at_1_max": 0.19318035045421283, + "nauc_ndcg_at_1_std": -0.2656247757638381, + "nauc_ndcg_at_20_diff1": 0.5544855949125493, + "nauc_ndcg_at_20_max": 0.22819031621405042, + "nauc_ndcg_at_20_std": -0.2877622655219696, + "nauc_ndcg_at_3_diff1": 0.5339051421424691, + "nauc_ndcg_at_3_max": 0.26038319275348876, + "nauc_ndcg_at_3_std": -0.3106305076373389, + "nauc_ndcg_at_5_diff1": 0.5544855949125493, + "nauc_ndcg_at_5_max": 0.22819031621405042, + "nauc_ndcg_at_5_std": -0.2877622655219696, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 0.591997818630617, + "nauc_precision_at_1_max": 0.19318035045421283, + "nauc_precision_at_1_std": -0.2656247757638381, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 0.31714908185496393, + "nauc_precision_at_3_max": 0.6374105197634555, + "nauc_precision_at_3_std": -0.6305633364456867, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": 0.591997818630617, + "nauc_recall_at_1_max": 0.19318035045421283, + "nauc_recall_at_1_std": -0.2656247757638381, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": 0.3171490818549668, + "nauc_recall_at_3_max": 0.6374105197634592, + "nauc_recall_at_3_std": -0.6305633364456882, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 0.57, + "ndcg_at_10": 0.82394, + "ndcg_at_100": 0.82394, + "ndcg_at_1000": 0.82394, + "ndcg_at_20": 0.82394, + "ndcg_at_3": 0.8119, + "ndcg_at_5": 0.82394, + "precision_at_1": 0.57, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.32333, + "precision_at_5": 0.2, + "recall_at_1": 0.57, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.97, + "recall_at_5": 1.0 + } + ] + }, + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreTabfquadRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..fb3a8f4c35 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", + "evaluation_time": 274.94982719421387, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.22857, + "cv_recall_at_10": 0.95714, + "cv_recall_at_100": 0.99286, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.97143, + "cv_recall_at_3": 0.65, + "cv_recall_at_5": 0.89643, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56361, + "map_at_1": 0.21071, + "map_at_10": 0.4602, + "map_at_100": 0.46294, + "map_at_1000": 0.463, + "map_at_20": 0.46228, + "map_at_3": 0.39286, + "map_at_5": 0.45339, + "mrr_at_1": 0.22857142857142856, + "mrr_at_10": 0.4707539682539679, + "mrr_at_100": 0.47251577376236964, + "mrr_at_1000": 0.47257600444588754, + "mrr_at_20": 0.47187420218670195, + "mrr_at_3": 0.4023809523809521, + "mrr_at_5": 0.46291666666666637, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.35807656395892135, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 1.0, + "nauc_cv_recall_at_10_diff1": -0.27170868347338767, + "nauc_cv_recall_at_10_max": 0.9564270152505449, + "nauc_cv_recall_at_10_std": 0.5902583255524414, + "nauc_cv_recall_at_1_diff1": -0.02582364764484841, + "nauc_cv_recall_at_1_max": 0.0113715808791401, + "nauc_cv_recall_at_1_std": -0.027112645285327273, + "nauc_cv_recall_at_20_diff1": -0.2777777777777773, + "nauc_cv_recall_at_20_max": 0.9673202614379085, + "nauc_cv_recall_at_20_std": 0.6473506069094338, + "nauc_cv_recall_at_3_diff1": -0.07400346418980988, + "nauc_cv_recall_at_3_max": 0.29113725399021206, + "nauc_cv_recall_at_3_std": 0.11649210094974545, + "nauc_cv_recall_at_5_diff1": -0.17426178078204416, + "nauc_cv_recall_at_5_max": 0.7885927746692977, + "nauc_cv_recall_at_5_std": 0.24504350076005102, + "nauc_map_at_1000_diff1": 0.13553846809082123, + "nauc_map_at_1000_max": 0.22199520700174338, + "nauc_map_at_1000_std": 0.05778807691192794, + "nauc_map_at_100_diff1": 0.13559236781767062, + "nauc_map_at_100_max": 0.22218364325876425, + "nauc_map_at_100_std": 0.05801628487376212, + "nauc_map_at_10_diff1": 0.1344768398789461, + "nauc_map_at_10_max": 0.22892455842544213, + "nauc_map_at_10_std": 0.060278324955635704, + "nauc_map_at_1_diff1": 0.23368144002078475, + "nauc_map_at_1_max": 0.16144318367946314, + "nauc_map_at_1_std": 0.024416417682840807, + "nauc_map_at_20_diff1": 0.13384108210812262, + "nauc_map_at_20_max": 0.22410132807912986, + "nauc_map_at_20_std": 0.05917900593894214, + "nauc_map_at_3_diff1": 0.07766973381953121, + "nauc_map_at_3_max": 0.21138604876278055, + "nauc_map_at_3_std": 0.06812727676058795, + "nauc_map_at_5_diff1": 0.13235687982693659, + "nauc_map_at_5_max": 0.2386465804104747, + "nauc_map_at_5_std": 0.05423220277094575, + "nauc_mrr_at_1000_diff1": -0.0392865823395486, + "nauc_mrr_at_1000_max": 0.10096326261753061, + "nauc_mrr_at_1000_std": 0.004056299571522236, + "nauc_mrr_at_100_diff1": -0.039192201090724105, + "nauc_mrr_at_100_max": 0.10117680081503029, + "nauc_mrr_at_100_std": 0.004292855002193332, + "nauc_mrr_at_10_diff1": -0.04146431336462882, + "nauc_mrr_at_10_max": 0.10699792280650171, + "nauc_mrr_at_10_std": 0.007644556829231765, + "nauc_mrr_at_1_diff1": -0.02582364764484841, + "nauc_mrr_at_1_max": 0.0113715808791401, + "nauc_mrr_at_1_std": -0.027112645285327273, + "nauc_mrr_at_20_diff1": -0.04028570429277086, + "nauc_mrr_at_20_max": 0.10331896246767898, + "nauc_mrr_at_20_std": 0.005529622938990385, + "nauc_mrr_at_3_diff1": -0.041454567100425964, + "nauc_mrr_at_3_max": 0.10482829405207283, + "nauc_mrr_at_3_std": 0.012387757089927653, + "nauc_mrr_at_5_diff1": -0.045969795381389594, + "nauc_mrr_at_5_max": 0.12346001607950009, + "nauc_mrr_at_5_std": 0.006284038497855093, + "nauc_ndcg_at_1000_diff1": 0.12651591808080723, + "nauc_ndcg_at_1000_max": 0.2367262598374307, + "nauc_ndcg_at_1000_std": 0.06640769582000103, + "nauc_ndcg_at_100_diff1": 0.12774994597272604, + "nauc_ndcg_at_100_max": 0.24079388143590194, + "nauc_ndcg_at_100_std": 0.07138297535711573, + "nauc_ndcg_at_10_diff1": 0.11834485636734318, + "nauc_ndcg_at_10_max": 0.2740394382500109, + "nauc_ndcg_at_10_std": 0.08438885306140487, + "nauc_ndcg_at_1_diff1": 0.23368144002078475, + "nauc_ndcg_at_1_max": 0.16144318367946314, + "nauc_ndcg_at_1_std": 0.024416417682840807, + "nauc_ndcg_at_20_diff1": 0.11455084278957535, + "nauc_ndcg_at_20_max": 0.2558954677030979, + "nauc_ndcg_at_20_std": 0.0809048502571133, + "nauc_ndcg_at_3_diff1": 0.01996065575006399, + "nauc_ndcg_at_3_max": 0.2368828569659345, + "nauc_ndcg_at_3_std": 0.08883647180660824, + "nauc_ndcg_at_5_diff1": 0.11396014515287547, + "nauc_ndcg_at_5_max": 0.29797120024144946, + "nauc_ndcg_at_5_std": 0.06725907831809537, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.35807656395888193, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": -0.03588546529723173, + "nauc_precision_at_10_max": 0.892623716153129, + "nauc_precision_at_10_std": 0.43106131341425424, + "nauc_precision_at_1_diff1": 0.23368144002078475, + "nauc_precision_at_1_max": 0.16144318367946314, + "nauc_precision_at_1_std": 0.024416417682840807, + "nauc_precision_at_20_diff1": -0.277777777777766, + "nauc_precision_at_20_max": 0.9673202614379075, + "nauc_precision_at_20_std": 0.647350606909435, + "nauc_precision_at_3_diff1": -0.15931785685223693, + "nauc_precision_at_3_max": 0.31803703660648913, + "nauc_precision_at_3_std": 0.15442962928519124, + "nauc_precision_at_5_diff1": 0.011255215239820097, + "nauc_precision_at_5_max": 0.7531129726058416, + "nauc_precision_at_5_std": 0.15925482712894967, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.35807656395892135, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 1.0, + "nauc_recall_at_10_diff1": -0.035885465297229795, + "nauc_recall_at_10_max": 0.8926237161531256, + "nauc_recall_at_10_std": 0.43106131341425596, + "nauc_recall_at_1_diff1": 0.23368144002078475, + "nauc_recall_at_1_max": 0.16144318367946314, + "nauc_recall_at_1_std": 0.024416417682840807, + "nauc_recall_at_20_diff1": -0.2777777777777773, + "nauc_recall_at_20_max": 0.9673202614379085, + "nauc_recall_at_20_std": 0.6473506069094338, + "nauc_recall_at_3_diff1": -0.15931785685223723, + "nauc_recall_at_3_max": 0.31803703660648813, + "nauc_recall_at_3_std": 0.15442962928519088, + "nauc_recall_at_5_diff1": 0.011255215239821375, + "nauc_recall_at_5_max": 0.7531129726058414, + "nauc_recall_at_5_std": 0.15925482712894984, + "ndcg_at_1": 0.21071, + "ndcg_at_10": 0.57991, + "ndcg_at_100": 0.59086, + "ndcg_at_1000": 0.59189, + "ndcg_at_20": 0.58666, + "ndcg_at_3": 0.45841, + "ndcg_at_5": 0.56361, + "precision_at_1": 0.21071, + "precision_at_10": 0.09464, + "precision_at_100": 0.00993, + "precision_at_1000": 0.001, + "precision_at_20": 0.04857, + "precision_at_3": 0.21667, + "precision_at_5": 0.17929, + "recall_at_1": 0.21071, + "recall_at_10": 0.94643, + "recall_at_100": 0.99286, + "recall_at_1000": 1.0, + "recall_at_20": 0.97143, + "recall_at_3": 0.65, + "recall_at_5": 0.89643 + } + ] + }, + "task_name": "VidoreTabfquadRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreTatdqaRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..37adcd3e71 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", + "evaluation_time": 1544.8007860183716, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.09321, + "cv_recall_at_10": 0.635, + "cv_recall_at_100": 0.94227, + "cv_recall_at_1000": 0.99579, + "cv_recall_at_20": 0.75646, + "cv_recall_at_3": 0.28443, + "cv_recall_at_5": 0.46603, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.27633, + "map_at_1": 0.09501, + "map_at_10": 0.23984, + "map_at_100": 0.25355, + "map_at_1000": 0.25382, + "map_at_20": 0.24829, + "map_at_3": 0.17619, + "map_at_5": 0.21563, + "mrr_at_1": 0.09320505111244738, + "mrr_at_10": 0.2381229180387323, + "mrr_at_100": 0.25191031872372854, + "mrr_at_1000": 0.252176402965878, + "mrr_at_20": 0.246719851990479, + "mrr_at_3": 0.1733814391661656, + "mrr_at_5": 0.2141511324914806, + "nauc_cv_recall_at_1000_diff1": 0.31050486909740643, + "nauc_cv_recall_at_1000_max": 0.5565879699515275, + "nauc_cv_recall_at_1000_std": 0.7932362042056588, + "nauc_cv_recall_at_100_diff1": 0.017229634643007606, + "nauc_cv_recall_at_100_max": 0.7245328143477694, + "nauc_cv_recall_at_100_std": 0.554035254223751, + "nauc_cv_recall_at_10_diff1": 0.032786105058676726, + "nauc_cv_recall_at_10_max": 0.4898125717664449, + "nauc_cv_recall_at_10_std": -0.04286901530782105, + "nauc_cv_recall_at_1_diff1": -0.030595244757954252, + "nauc_cv_recall_at_1_max": 0.17229432194888586, + "nauc_cv_recall_at_1_std": -0.022750385688563874, + "nauc_cv_recall_at_20_diff1": 0.042499064344320193, + "nauc_cv_recall_at_20_max": 0.5315043626694177, + "nauc_cv_recall_at_20_std": -0.01091688306662299, + "nauc_cv_recall_at_3_diff1": -0.008924384137994422, + "nauc_cv_recall_at_3_max": 0.3078286274347898, + "nauc_cv_recall_at_3_std": 0.017083055748262896, + "nauc_cv_recall_at_5_diff1": 0.024757025565364448, + "nauc_cv_recall_at_5_max": 0.4073620550301354, + "nauc_cv_recall_at_5_std": -0.06215327720340901, + "nauc_map_at_1000_diff1": -0.0003925171649634412, + "nauc_map_at_1000_max": 0.2898040228378256, + "nauc_map_at_1000_std": -0.011252721627844083, + "nauc_map_at_100_diff1": -0.000347784297757475, + "nauc_map_at_100_max": 0.2902793635938703, + "nauc_map_at_100_std": -0.010735880636550706, + "nauc_map_at_10_diff1": -0.0002285351889445233, + "nauc_map_at_10_max": 0.2959599078926981, + "nauc_map_at_10_std": -0.017859607333720634, + "nauc_map_at_1_diff1": -0.027194095074582824, + "nauc_map_at_1_max": 0.1916225301721836, + "nauc_map_at_1_std": -0.001399713359798536, + "nauc_map_at_20_diff1": 0.0017744924766137128, + "nauc_map_at_20_max": 0.29305441547884215, + "nauc_map_at_20_std": -0.015289326952853316, + "nauc_map_at_3_diff1": 0.001522630982425383, + "nauc_map_at_3_max": 0.24227359001382884, + "nauc_map_at_3_std": -0.0027546423271664284, + "nauc_map_at_5_diff1": -0.001963766600180219, + "nauc_map_at_5_max": 0.281071243656603, + "nauc_map_at_5_std": -0.019853852764696617, + "nauc_mrr_at_1000_diff1": -0.008039463989689176, + "nauc_mrr_at_1000_max": 0.28401646367520217, + "nauc_mrr_at_1000_std": -0.013541290133244552, + "nauc_mrr_at_100_diff1": -0.00799442245722196, + "nauc_mrr_at_100_max": 0.2845070670650448, + "nauc_mrr_at_100_std": -0.01303344586909849, + "nauc_mrr_at_10_diff1": -0.00614564411167257, + "nauc_mrr_at_10_max": 0.289135604430979, + "nauc_mrr_at_10_std": -0.019937260545786917, + "nauc_mrr_at_1_diff1": -0.030595244757954252, + "nauc_mrr_at_1_max": 0.17229432194888586, + "nauc_mrr_at_1_std": -0.022750385688563874, + "nauc_mrr_at_20_diff1": -0.006644806046490563, + "nauc_mrr_at_20_max": 0.28733189413446436, + "nauc_mrr_at_20_std": -0.01754671101514299, + "nauc_mrr_at_3_diff1": -0.017339059829432638, + "nauc_mrr_at_3_max": 0.24425253352221082, + "nauc_mrr_at_3_std": 0.0013361903711106577, + "nauc_mrr_at_5_diff1": -0.007130039960877325, + "nauc_mrr_at_5_max": 0.2757222346163175, + "nauc_mrr_at_5_std": -0.023769414978505286, + "nauc_ndcg_at_1000_diff1": 0.003468485965427646, + "nauc_ndcg_at_1000_max": 0.31731576935974565, + "nauc_ndcg_at_1000_std": -0.0024293118052454063, + "nauc_ndcg_at_100_diff1": 0.0035673138731956147, + "nauc_ndcg_at_100_max": 0.3309317704266328, + "nauc_ndcg_at_100_std": 0.014880579527387122, + "nauc_ndcg_at_10_diff1": 0.005795685643303614, + "nauc_ndcg_at_10_max": 0.3528635747427016, + "nauc_ndcg_at_10_std": -0.02502551479265232, + "nauc_ndcg_at_1_diff1": -0.027194095074582824, + "nauc_ndcg_at_1_max": 0.1916225301721836, + "nauc_ndcg_at_1_std": -0.001399713359798536, + "nauc_ndcg_at_20_diff1": 0.014598223077526385, + "nauc_ndcg_at_20_max": 0.34721854443722916, + "nauc_ndcg_at_20_std": -0.016046354479508872, + "nauc_ndcg_at_3_diff1": 0.007765400458555541, + "nauc_ndcg_at_3_max": 0.2575186815309078, + "nauc_ndcg_at_3_std": -0.002035068546740743, + "nauc_ndcg_at_5_diff1": 0.002649179955746284, + "nauc_ndcg_at_5_max": 0.321154170908288, + "nauc_ndcg_at_5_std": -0.030304143245058396, + "nauc_precision_at_1000_diff1": 0.3105048690973943, + "nauc_precision_at_1000_max": 0.556587969951549, + "nauc_precision_at_1000_std": 0.7932362042056358, + "nauc_precision_at_100_diff1": 0.020743590857352306, + "nauc_precision_at_100_max": 0.7187339631723969, + "nauc_precision_at_100_std": 0.5557689545945156, + "nauc_precision_at_10_diff1": 0.02015203520063033, + "nauc_precision_at_10_max": 0.5039818563028681, + "nauc_precision_at_10_std": -0.04236193373091959, + "nauc_precision_at_1_diff1": -0.027194095074582824, + "nauc_precision_at_1_max": 0.1916225301721836, + "nauc_precision_at_1_std": -0.001399713359798536, + "nauc_precision_at_20_diff1": 0.06509303733464025, + "nauc_precision_at_20_max": 0.5326931548142373, + "nauc_precision_at_20_std": -0.009687706719512464, + "nauc_precision_at_3_diff1": 0.020858108362361155, + "nauc_precision_at_3_max": 0.2904392930383802, + "nauc_precision_at_3_std": -0.00029894316449353845, + "nauc_precision_at_5_diff1": 0.011899104360575728, + "nauc_precision_at_5_max": 0.410610008372957, + "nauc_precision_at_5_std": -0.05446107727275674, + "nauc_recall_at_1000_diff1": 0.31050486909740643, + "nauc_recall_at_1000_max": 0.5565879699515275, + "nauc_recall_at_1000_std": 0.7932362042056588, + "nauc_recall_at_100_diff1": 0.020743590857350565, + "nauc_recall_at_100_max": 0.7187339631723954, + "nauc_recall_at_100_std": 0.5557689545945156, + "nauc_recall_at_10_diff1": 0.020152035200631094, + "nauc_recall_at_10_max": 0.5039818563028683, + "nauc_recall_at_10_std": -0.042361933730918684, + "nauc_recall_at_1_diff1": -0.027194095074582824, + "nauc_recall_at_1_max": 0.1916225301721836, + "nauc_recall_at_1_std": -0.001399713359798536, + "nauc_recall_at_20_diff1": 0.06509303733464074, + "nauc_recall_at_20_max": 0.5326931548142366, + "nauc_recall_at_20_std": -0.009687706719513361, + "nauc_recall_at_3_diff1": 0.02085810836236144, + "nauc_recall_at_3_max": 0.2904392930383805, + "nauc_recall_at_3_std": -0.0002989431644934902, + "nauc_recall_at_5_diff1": 0.011899104360575681, + "nauc_recall_at_5_max": 0.41061000837295714, + "nauc_recall_at_5_std": -0.05446107727275678, + "ndcg_at_1": 0.09501, + "ndcg_at_10": 0.33305, + "ndcg_at_100": 0.39919, + "ndcg_at_1000": 0.40611, + "ndcg_at_20": 0.36358, + "ndcg_at_3": 0.2046, + "ndcg_at_5": 0.27633, + "precision_at_1": 0.09501, + "precision_at_10": 0.06344, + "precision_at_100": 0.00942, + "precision_at_1000": 0.001, + "precision_at_20": 0.03773, + "precision_at_3": 0.09581, + "precision_at_5": 0.0926, + "recall_at_1": 0.09501, + "recall_at_10": 0.6344, + "recall_at_100": 0.94167, + "recall_at_1000": 0.99579, + "recall_at_20": 0.75466, + "recall_at_3": 0.28743, + "recall_at_5": 0.46302 + } + ] + }, + "task_name": "VidoreTatdqaRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VisualNewsI2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/VisualNewsI2TRetrieval.json new file mode 100644 index 0000000000..6dd46a1e7a --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/VisualNewsI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "aaee58895a66e4d619168849267ed2bb40d37043", + "evaluation_time": 6107.169147491455, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.06825, + "cv_recall_at_10": 0.19545, + "cv_recall_at_100": 0.39875, + "cv_recall_at_1000": 0.6308, + "cv_recall_at_20": 0.2496, + "cv_recall_at_3": 0.1178, + "cv_recall_at_5": 0.1496, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.12474, + "map_at_1": 0.06825, + "map_at_10": 0.103, + "map_at_100": 0.11027, + "map_at_1000": 0.11118, + "map_at_20": 0.10672, + "map_at_3": 0.0897, + "map_at_5": 0.09691, + "mrr_at_1": 0.06825, + "mrr_at_10": 0.10299886904761851, + "mrr_at_100": 0.11027488430144886, + "mrr_at_1000": 0.11117591475965252, + "mrr_at_20": 0.10671955475724743, + "mrr_at_3": 0.0896999999999995, + "mrr_at_5": 0.09691249999999985, + "nauc_cv_recall_at_1000_diff1": 0.07071177533627818, + "nauc_cv_recall_at_1000_max": 0.26562236167072856, + "nauc_cv_recall_at_1000_std": 0.3641313388165596, + "nauc_cv_recall_at_100_diff1": 0.0879483088139398, + "nauc_cv_recall_at_100_max": 0.19499859114790075, + "nauc_cv_recall_at_100_std": 0.24543736404577246, + "nauc_cv_recall_at_10_diff1": 0.14309800392206096, + "nauc_cv_recall_at_10_max": 0.18275171223289868, + "nauc_cv_recall_at_10_std": 0.14117281838957083, + "nauc_cv_recall_at_1_diff1": 0.3392937980541951, + "nauc_cv_recall_at_1_max": 0.2444093246883695, + "nauc_cv_recall_at_1_std": 0.060403490615457335, + "nauc_cv_recall_at_20_diff1": 0.12719265702804436, + "nauc_cv_recall_at_20_max": 0.18740195316364447, + "nauc_cv_recall_at_20_std": 0.17863831788138756, + "nauc_cv_recall_at_3_diff1": 0.20773411557883753, + "nauc_cv_recall_at_3_max": 0.20086790461930343, + "nauc_cv_recall_at_3_std": 0.08982097516973798, + "nauc_cv_recall_at_5_diff1": 0.17471486468798317, + "nauc_cv_recall_at_5_max": 0.19173329789004723, + "nauc_cv_recall_at_5_std": 0.11246321681756993, + "nauc_map_at_1000_diff1": 0.23557151715228627, + "nauc_map_at_1000_max": 0.21360085336214968, + "nauc_map_at_1000_std": 0.10179157596177511, + "nauc_map_at_100_diff1": 0.23558150842830375, + "nauc_map_at_100_max": 0.2131076975340969, + "nauc_map_at_100_std": 0.10101384389492717, + "nauc_map_at_10_diff1": 0.24205448446889496, + "nauc_map_at_10_max": 0.21288563308201747, + "nauc_map_at_10_std": 0.09145715995583001, + "nauc_map_at_1_diff1": 0.3392937980541951, + "nauc_map_at_1_max": 0.2444093246883695, + "nauc_map_at_1_std": 0.060403490615457335, + "nauc_map_at_20_diff1": 0.23871992868144357, + "nauc_map_at_20_max": 0.21330652626665822, + "nauc_map_at_20_std": 0.09696116661274039, + "nauc_map_at_3_diff1": 0.2657615794348174, + "nauc_map_at_3_max": 0.21969159446682562, + "nauc_map_at_3_std": 0.07592966888480318, + "nauc_map_at_5_diff1": 0.2523244135153835, + "nauc_map_at_5_max": 0.21603781978533362, + "nauc_map_at_5_std": 0.08404111409954902, + "nauc_mrr_at_1000_diff1": 0.2355715148963118, + "nauc_mrr_at_1000_max": 0.21360085206378832, + "nauc_mrr_at_1000_std": 0.10179157561911444, + "nauc_mrr_at_100_diff1": 0.23558150842830375, + "nauc_mrr_at_100_max": 0.2131076975340969, + "nauc_mrr_at_100_std": 0.10101384389492717, + "nauc_mrr_at_10_diff1": 0.24205448446889496, + "nauc_mrr_at_10_max": 0.21288563308201747, + "nauc_mrr_at_10_std": 0.09145715995583001, + "nauc_mrr_at_1_diff1": 0.3392937980541951, + "nauc_mrr_at_1_max": 0.2444093246883695, + "nauc_mrr_at_1_std": 0.060403490615457335, + "nauc_mrr_at_20_diff1": 0.23871992868144357, + "nauc_mrr_at_20_max": 0.21330652626665822, + "nauc_mrr_at_20_std": 0.09696116661274039, + "nauc_mrr_at_3_diff1": 0.2657615794348174, + "nauc_mrr_at_3_max": 0.21969159446682562, + "nauc_mrr_at_3_std": 0.07592966888480318, + "nauc_mrr_at_5_diff1": 0.2523244135153835, + "nauc_mrr_at_5_max": 0.21603781978533362, + "nauc_mrr_at_5_std": 0.08404111409954902, + "nauc_ndcg_at_1000_diff1": 0.18371223467656492, + "nauc_ndcg_at_1000_max": 0.21760781116915476, + "nauc_ndcg_at_1000_std": 0.1651354537417224, + "nauc_ndcg_at_100_diff1": 0.18351509037449174, + "nauc_ndcg_at_100_max": 0.2047563926427989, + "nauc_ndcg_at_100_std": 0.14708224364063993, + "nauc_ndcg_at_10_diff1": 0.20798943664064767, + "nauc_ndcg_at_10_max": 0.2024610973940482, + "nauc_ndcg_at_10_std": 0.10797693655625268, + "nauc_ndcg_at_1_diff1": 0.3392937980541951, + "nauc_ndcg_at_1_max": 0.2444093246883695, + "nauc_ndcg_at_1_std": 0.060403490615457335, + "nauc_ndcg_at_20_diff1": 0.199135794702939, + "nauc_ndcg_at_20_max": 0.2037165138514619, + "nauc_ndcg_at_20_std": 0.12405620210520404, + "nauc_ndcg_at_3_diff1": 0.24804200545089336, + "nauc_ndcg_at_3_max": 0.2139285891161409, + "nauc_ndcg_at_3_std": 0.08013578418891945, + "nauc_ndcg_at_5_diff1": 0.22734280862143041, + "nauc_ndcg_at_5_max": 0.2082046891841067, + "nauc_ndcg_at_5_std": 0.09300248180293258, + "nauc_precision_at_1000_diff1": 0.07071177533627745, + "nauc_precision_at_1000_max": 0.2656223616707277, + "nauc_precision_at_1000_std": 0.364131338816559, + "nauc_precision_at_100_diff1": 0.08794830881393986, + "nauc_precision_at_100_max": 0.19499859114790064, + "nauc_precision_at_100_std": 0.2454373640457725, + "nauc_precision_at_10_diff1": 0.1430980039220609, + "nauc_precision_at_10_max": 0.18275171223289863, + "nauc_precision_at_10_std": 0.1411728183895707, + "nauc_precision_at_1_diff1": 0.3392937980541951, + "nauc_precision_at_1_max": 0.2444093246883695, + "nauc_precision_at_1_std": 0.060403490615457335, + "nauc_precision_at_20_diff1": 0.1271926570280442, + "nauc_precision_at_20_max": 0.18740195316364444, + "nauc_precision_at_20_std": 0.17863831788138737, + "nauc_precision_at_3_diff1": 0.2077341155788376, + "nauc_precision_at_3_max": 0.2008679046193036, + "nauc_precision_at_3_std": 0.08982097516973812, + "nauc_precision_at_5_diff1": 0.1747148646879832, + "nauc_precision_at_5_max": 0.1917332978900472, + "nauc_precision_at_5_std": 0.11246321681756989, + "nauc_recall_at_1000_diff1": 0.07071177533627818, + "nauc_recall_at_1000_max": 0.26562236167072856, + "nauc_recall_at_1000_std": 0.3641313388165596, + "nauc_recall_at_100_diff1": 0.0879483088139398, + "nauc_recall_at_100_max": 0.19499859114790075, + "nauc_recall_at_100_std": 0.24543736404577246, + "nauc_recall_at_10_diff1": 0.14309800392206096, + "nauc_recall_at_10_max": 0.18275171223289868, + "nauc_recall_at_10_std": 0.14117281838957083, + "nauc_recall_at_1_diff1": 0.3392937980541951, + "nauc_recall_at_1_max": 0.2444093246883695, + "nauc_recall_at_1_std": 0.060403490615457335, + "nauc_recall_at_20_diff1": 0.12719265702804436, + "nauc_recall_at_20_max": 0.18740195316364447, + "nauc_recall_at_20_std": 0.17863831788138756, + "nauc_recall_at_3_diff1": 0.20773411557883753, + "nauc_recall_at_3_max": 0.20086790461930343, + "nauc_recall_at_3_std": 0.08982097516973798, + "nauc_recall_at_5_diff1": 0.17471486468798317, + "nauc_recall_at_5_max": 0.19173329789004723, + "nauc_recall_at_5_std": 0.11246321681756993, + "ndcg_at_1": 0.06825, + "ndcg_at_10": 0.12474, + "ndcg_at_100": 0.16551, + "ndcg_at_1000": 0.19379, + "ndcg_at_20": 0.13839, + "ndcg_at_3": 0.0969, + "ndcg_at_5": 0.10995, + "precision_at_1": 0.06825, + "precision_at_10": 0.01955, + "precision_at_100": 0.00399, + "precision_at_1000": 0.00063, + "precision_at_20": 0.01248, + "precision_at_3": 0.03927, + "precision_at_5": 0.02992, + "recall_at_1": 0.06825, + "recall_at_10": 0.19545, + "recall_at_100": 0.39875, + "recall_at_1000": 0.6308, + "recall_at_20": 0.2496, + "recall_at_3": 0.1178, + "recall_at_5": 0.1496 + } + ] + }, + "task_name": "VisualNewsI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/WITT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/WITT2IRetrieval.json new file mode 100644 index 0000000000..c68c3a9c67 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/WITT2IRetrieval.json @@ -0,0 +1,1936 @@ +{ + "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", + "evaluation_time": 1137.1250095367432, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.39775, + "cv_recall_at_10": 0.68764, + "cv_recall_at_100": 0.90337, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.76067, + "cv_recall_at_3": 0.53708, + "cv_recall_at_5": 0.59775, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.53332, + "map_at_1": 0.39775, + "map_at_10": 0.48513, + "map_at_100": 0.49387, + "map_at_1000": 0.49438, + "map_at_20": 0.49008, + "map_at_3": 0.45936, + "map_at_5": 0.47307, + "mrr_at_1": 0.39775280898876403, + "mrr_at_10": 0.4851252898163007, + "mrr_at_100": 0.4938677598278257, + "mrr_at_1000": 0.4943758260759537, + "mrr_at_20": 0.49007998369444744, + "mrr_at_3": 0.4593632958801497, + "mrr_at_5": 0.47307116104868874, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5000271426400019, + "nauc_cv_recall_at_100_max": 0.33645473693353406, + "nauc_cv_recall_at_100_std": 0.2653898768809851, + "nauc_cv_recall_at_10_diff1": 0.5174089324309398, + "nauc_cv_recall_at_10_max": 0.3901568933693027, + "nauc_cv_recall_at_10_std": 0.14601968857967695, + "nauc_cv_recall_at_1_diff1": 0.6433933086018607, + "nauc_cv_recall_at_1_max": 0.4016098624710042, + "nauc_cv_recall_at_1_std": 0.041856713841805254, + "nauc_cv_recall_at_20_diff1": 0.5004080379828553, + "nauc_cv_recall_at_20_max": 0.40080048526069356, + "nauc_cv_recall_at_20_std": 0.21059144801108873, + "nauc_cv_recall_at_3_diff1": 0.5769084735634901, + "nauc_cv_recall_at_3_max": 0.44111111456910174, + "nauc_cv_recall_at_3_std": 0.1138455170439162, + "nauc_cv_recall_at_5_diff1": 0.5616616989567813, + "nauc_cv_recall_at_5_max": 0.4342382770917608, + "nauc_cv_recall_at_5_std": 0.14279892750691942, + "nauc_map_at_1000_diff1": 0.6062730714611361, + "nauc_map_at_1000_max": 0.4110539637034493, + "nauc_map_at_1000_std": 0.0791359300689424, + "nauc_map_at_100_diff1": 0.6061385539345461, + "nauc_map_at_100_max": 0.4110626956856239, + "nauc_map_at_100_std": 0.07934859334144191, + "nauc_map_at_10_diff1": 0.6047402412710131, + "nauc_map_at_10_max": 0.4099565332479481, + "nauc_map_at_10_std": 0.077238302183022, + "nauc_map_at_1_diff1": 0.6433933086018607, + "nauc_map_at_1_max": 0.4016098624710042, + "nauc_map_at_1_std": 0.041856713841805254, + "nauc_map_at_20_diff1": 0.6051415598634673, + "nauc_map_at_20_max": 0.41124616862006846, + "nauc_map_at_20_std": 0.08010265866099041, + "nauc_map_at_3_diff1": 0.6108768857240254, + "nauc_map_at_3_max": 0.41615469088166257, + "nauc_map_at_3_std": 0.07301219530664803, + "nauc_map_at_5_diff1": 0.6082690683739146, + "nauc_map_at_5_max": 0.41430708987714243, + "nauc_map_at_5_std": 0.07783192900428913, + "nauc_mrr_at_1000_diff1": 0.6062730714611361, + "nauc_mrr_at_1000_max": 0.4110539637034493, + "nauc_mrr_at_1000_std": 0.0791359300689424, + "nauc_mrr_at_100_diff1": 0.6061385539345461, + "nauc_mrr_at_100_max": 0.4110626956856239, + "nauc_mrr_at_100_std": 0.07934859334144191, + "nauc_mrr_at_10_diff1": 0.6047402412710131, + "nauc_mrr_at_10_max": 0.4099565332479481, + "nauc_mrr_at_10_std": 0.077238302183022, + "nauc_mrr_at_1_diff1": 0.6433933086018607, + "nauc_mrr_at_1_max": 0.4016098624710042, + "nauc_mrr_at_1_std": 0.041856713841805254, + "nauc_mrr_at_20_diff1": 0.6051415598634673, + "nauc_mrr_at_20_max": 0.41124616862006846, + "nauc_mrr_at_20_std": 0.08010265866099041, + "nauc_mrr_at_3_diff1": 0.6108768857240254, + "nauc_mrr_at_3_max": 0.41615469088166257, + "nauc_mrr_at_3_std": 0.07301219530664803, + "nauc_mrr_at_5_diff1": 0.6082690683739146, + "nauc_mrr_at_5_max": 0.41430708987714243, + "nauc_mrr_at_5_std": 0.07783192900428913, + "nauc_ndcg_at_1000_diff1": 0.5985728701025478, + "nauc_ndcg_at_1000_max": 0.4105306381695453, + "nauc_ndcg_at_1000_std": 0.08997211615828361, + "nauc_ndcg_at_100_diff1": 0.5943985687790737, + "nauc_ndcg_at_100_max": 0.40799386515039965, + "nauc_ndcg_at_100_std": 0.09701143431709665, + "nauc_ndcg_at_10_diff1": 0.5873207409203248, + "nauc_ndcg_at_10_max": 0.4072659270468375, + "nauc_ndcg_at_10_std": 0.0916668746640074, + "nauc_ndcg_at_1_diff1": 0.6433933086018607, + "nauc_ndcg_at_1_max": 0.4016098624710042, + "nauc_ndcg_at_1_std": 0.041856713841805254, + "nauc_ndcg_at_20_diff1": 0.5875622071363173, + "nauc_ndcg_at_20_max": 0.4106690124705196, + "nauc_ndcg_at_20_std": 0.10170412297234428, + "nauc_ndcg_at_3_diff1": 0.6022457729043741, + "nauc_ndcg_at_3_max": 0.42226539179035844, + "nauc_ndcg_at_3_std": 0.08319193818224027, + "nauc_ndcg_at_5_diff1": 0.5971440686463333, + "nauc_ndcg_at_5_max": 0.41907996722877244, + "nauc_ndcg_at_5_std": 0.09291896572761014, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.5000271426400036, + "nauc_precision_at_100_max": 0.33645473693353495, + "nauc_precision_at_100_std": 0.26538987688098664, + "nauc_precision_at_10_diff1": 0.5174089324309394, + "nauc_precision_at_10_max": 0.3901568933693025, + "nauc_precision_at_10_std": 0.14601968857967695, + "nauc_precision_at_1_diff1": 0.6433933086018607, + "nauc_precision_at_1_max": 0.4016098624710042, + "nauc_precision_at_1_std": 0.041856713841805254, + "nauc_precision_at_20_diff1": 0.5004080379828557, + "nauc_precision_at_20_max": 0.4008004852606927, + "nauc_precision_at_20_std": 0.21059144801108778, + "nauc_precision_at_3_diff1": 0.5769084735634901, + "nauc_precision_at_3_max": 0.4411111145691011, + "nauc_precision_at_3_std": 0.11384551704391571, + "nauc_precision_at_5_diff1": 0.5616616989567803, + "nauc_precision_at_5_max": 0.4342382770917606, + "nauc_precision_at_5_std": 0.14279892750691883, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5000271426400019, + "nauc_recall_at_100_max": 0.33645473693353406, + "nauc_recall_at_100_std": 0.2653898768809851, + "nauc_recall_at_10_diff1": 0.5174089324309398, + "nauc_recall_at_10_max": 0.3901568933693027, + "nauc_recall_at_10_std": 0.14601968857967695, + "nauc_recall_at_1_diff1": 0.6433933086018607, + "nauc_recall_at_1_max": 0.4016098624710042, + "nauc_recall_at_1_std": 0.041856713841805254, + "nauc_recall_at_20_diff1": 0.5004080379828553, + "nauc_recall_at_20_max": 0.40080048526069356, + "nauc_recall_at_20_std": 0.21059144801108873, + "nauc_recall_at_3_diff1": 0.5769084735634901, + "nauc_recall_at_3_max": 0.44111111456910174, + "nauc_recall_at_3_std": 0.1138455170439162, + "nauc_recall_at_5_diff1": 0.5616616989567813, + "nauc_recall_at_5_max": 0.4342382770917608, + "nauc_recall_at_5_std": 0.14279892750691942, + "ndcg_at_1": 0.39775, + "ndcg_at_10": 0.53332, + "ndcg_at_100": 0.57838, + "ndcg_at_1000": 0.59093, + "ndcg_at_20": 0.55163, + "ndcg_at_3": 0.47933, + "ndcg_at_5": 0.50418, + "precision_at_1": 0.39775, + "precision_at_10": 0.06876, + "precision_at_100": 0.00903, + "precision_at_1000": 0.001, + "precision_at_20": 0.03803, + "precision_at_3": 0.17903, + "precision_at_5": 0.11955, + "recall_at_1": 0.39775, + "recall_at_10": 0.68764, + "recall_at_100": 0.90337, + "recall_at_1000": 1.0, + "recall_at_20": 0.76067, + "recall_at_3": 0.53708, + "recall_at_5": 0.59775 + }, + { + "cv_recall_at_1": 0.32209, + "cv_recall_at_10": 0.6, + "cv_recall_at_100": 0.87558, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.68953, + "cv_recall_at_3": 0.4407, + "cv_recall_at_5": 0.5093, + "hf_subset": "bg", + "languages": [ + "bul-Cyrl" + ], + "main_score": 0.44843, + "map_at_1": 0.32209, + "map_at_10": 0.4015, + "map_at_100": 0.41219, + "map_at_1000": 0.41284, + "map_at_20": 0.40769, + "map_at_3": 0.37403, + "map_at_5": 0.38961, + "mrr_at_1": 0.32209302325581396, + "mrr_at_10": 0.4015014765596159, + "mrr_at_100": 0.41219135100183496, + "mrr_at_1000": 0.41283779969855594, + "mrr_at_20": 0.4076860035219613, + "mrr_at_3": 0.3740310077519379, + "mrr_at_5": 0.3896124031007749, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.4348271939958975, + "nauc_cv_recall_at_100_max": 0.5359090786963637, + "nauc_cv_recall_at_100_std": 0.24929023706529907, + "nauc_cv_recall_at_10_diff1": 0.4647400719280445, + "nauc_cv_recall_at_10_max": 0.46673155377000985, + "nauc_cv_recall_at_10_std": -0.014356271614653547, + "nauc_cv_recall_at_1_diff1": 0.6469130123286798, + "nauc_cv_recall_at_1_max": 0.4996837405335848, + "nauc_cv_recall_at_1_std": 0.021401518977899162, + "nauc_cv_recall_at_20_diff1": 0.46356765129520633, + "nauc_cv_recall_at_20_max": 0.44654751684700356, + "nauc_cv_recall_at_20_std": -0.04457976228770894, + "nauc_cv_recall_at_3_diff1": 0.5278920774752104, + "nauc_cv_recall_at_3_max": 0.47376992697805126, + "nauc_cv_recall_at_3_std": -0.02027797214207337, + "nauc_cv_recall_at_5_diff1": 0.5037572697381799, + "nauc_cv_recall_at_5_max": 0.4908852578000109, + "nauc_cv_recall_at_5_std": 0.001618973430065162, + "nauc_map_at_1000_diff1": 0.5876636565567583, + "nauc_map_at_1000_max": 0.4868422238450373, + "nauc_map_at_1000_std": 0.006172855129180589, + "nauc_map_at_100_diff1": 0.5872173827802959, + "nauc_map_at_100_max": 0.48681846044840055, + "nauc_map_at_100_std": 0.006174527620908233, + "nauc_map_at_10_diff1": 0.5843873762953701, + "nauc_map_at_10_max": 0.48569620079470033, + "nauc_map_at_10_std": 0.0048121712540381174, + "nauc_map_at_1_diff1": 0.6469130123286798, + "nauc_map_at_1_max": 0.4996837405335848, + "nauc_map_at_1_std": 0.021401518977899162, + "nauc_map_at_20_diff1": 0.5860522812282696, + "nauc_map_at_20_max": 0.4853116062444229, + "nauc_map_at_20_std": 0.003802028482319083, + "nauc_map_at_3_diff1": 0.5930592769877978, + "nauc_map_at_3_max": 0.48448560795502427, + "nauc_map_at_3_std": 0.0019450780457777837, + "nauc_map_at_5_diff1": 0.5876191275019375, + "nauc_map_at_5_max": 0.48860167161632034, + "nauc_map_at_5_std": 0.0071168197778243335, + "nauc_mrr_at_1000_diff1": 0.5876636565567583, + "nauc_mrr_at_1000_max": 0.4868422238450373, + "nauc_mrr_at_1000_std": 0.006172855129180589, + "nauc_mrr_at_100_diff1": 0.5872173827802959, + "nauc_mrr_at_100_max": 0.48681846044840055, + "nauc_mrr_at_100_std": 0.006174527620908233, + "nauc_mrr_at_10_diff1": 0.5843873762953701, + "nauc_mrr_at_10_max": 0.48569620079470033, + "nauc_mrr_at_10_std": 0.0048121712540381174, + "nauc_mrr_at_1_diff1": 0.6469130123286798, + "nauc_mrr_at_1_max": 0.4996837405335848, + "nauc_mrr_at_1_std": 0.021401518977899162, + "nauc_mrr_at_20_diff1": 0.5860522812282696, + "nauc_mrr_at_20_max": 0.4853116062444229, + "nauc_mrr_at_20_std": 0.003802028482319083, + "nauc_mrr_at_3_diff1": 0.5930592769877978, + "nauc_mrr_at_3_max": 0.48448560795502427, + "nauc_mrr_at_3_std": 0.0019450780457777837, + "nauc_mrr_at_5_diff1": 0.5876191275019375, + "nauc_mrr_at_5_max": 0.48860167161632034, + "nauc_mrr_at_5_std": 0.0071168197778243335, + "nauc_ndcg_at_1000_diff1": 0.5764346198771939, + "nauc_ndcg_at_1000_max": 0.48685121856010233, + "nauc_ndcg_at_1000_std": 0.008790975817410052, + "nauc_ndcg_at_100_diff1": 0.5681028972323554, + "nauc_ndcg_at_100_max": 0.48868765751875193, + "nauc_ndcg_at_100_std": 0.018708262636436764, + "nauc_ndcg_at_10_diff1": 0.557269842812324, + "nauc_ndcg_at_10_max": 0.4815575861378846, + "nauc_ndcg_at_10_std": 0.0002687978334403188, + "nauc_ndcg_at_1_diff1": 0.6469130123286798, + "nauc_ndcg_at_1_max": 0.4996837405335848, + "nauc_ndcg_at_1_std": 0.021401518977899162, + "nauc_ndcg_at_20_diff1": 0.5612535119375365, + "nauc_ndcg_at_20_max": 0.4786432157571462, + "nauc_ndcg_at_20_std": -0.004814012882018789, + "nauc_ndcg_at_3_diff1": 0.5761744796564754, + "nauc_ndcg_at_3_max": 0.48156121083060116, + "nauc_ndcg_at_3_std": -0.0038383683766595052, + "nauc_ndcg_at_5_diff1": 0.5665516875226873, + "nauc_ndcg_at_5_max": 0.48879365280033604, + "nauc_ndcg_at_5_std": 0.005361216092415931, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.43482719399589986, + "nauc_precision_at_100_max": 0.5359090786963644, + "nauc_precision_at_100_std": 0.2492902370652986, + "nauc_precision_at_10_diff1": 0.4647400719280439, + "nauc_precision_at_10_max": 0.4667315537700093, + "nauc_precision_at_10_std": -0.01435627161465371, + "nauc_precision_at_1_diff1": 0.6469130123286798, + "nauc_precision_at_1_max": 0.4996837405335848, + "nauc_precision_at_1_std": 0.021401518977899162, + "nauc_precision_at_20_diff1": 0.4635676512952064, + "nauc_precision_at_20_max": 0.4465475168470034, + "nauc_precision_at_20_std": -0.04457976228771008, + "nauc_precision_at_3_diff1": 0.5278920774752098, + "nauc_precision_at_3_max": 0.47376992697805037, + "nauc_precision_at_3_std": -0.020277972142073643, + "nauc_precision_at_5_diff1": 0.5037572697381797, + "nauc_precision_at_5_max": 0.49088525780001036, + "nauc_precision_at_5_std": 0.0016189734300648799, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4348271939958975, + "nauc_recall_at_100_max": 0.5359090786963637, + "nauc_recall_at_100_std": 0.24929023706529907, + "nauc_recall_at_10_diff1": 0.4647400719280445, + "nauc_recall_at_10_max": 0.46673155377000985, + "nauc_recall_at_10_std": -0.014356271614653547, + "nauc_recall_at_1_diff1": 0.6469130123286798, + "nauc_recall_at_1_max": 0.4996837405335848, + "nauc_recall_at_1_std": 0.021401518977899162, + "nauc_recall_at_20_diff1": 0.46356765129520633, + "nauc_recall_at_20_max": 0.44654751684700356, + "nauc_recall_at_20_std": -0.04457976228770894, + "nauc_recall_at_3_diff1": 0.5278920774752104, + "nauc_recall_at_3_max": 0.47376992697805126, + "nauc_recall_at_3_std": -0.02027797214207337, + "nauc_recall_at_5_diff1": 0.5037572697381799, + "nauc_recall_at_5_max": 0.4908852578000109, + "nauc_recall_at_5_std": 0.001618973430065162, + "ndcg_at_1": 0.32209, + "ndcg_at_10": 0.44843, + "ndcg_at_100": 0.50502, + "ndcg_at_1000": 0.52107, + "ndcg_at_20": 0.47104, + "ndcg_at_3": 0.39114, + "ndcg_at_5": 0.41931, + "precision_at_1": 0.32209, + "precision_at_10": 0.06, + "precision_at_100": 0.00876, + "precision_at_1000": 0.001, + "precision_at_20": 0.03448, + "precision_at_3": 0.1469, + "precision_at_5": 0.10186, + "recall_at_1": 0.32209, + "recall_at_10": 0.6, + "recall_at_100": 0.87558, + "recall_at_1000": 1.0, + "recall_at_20": 0.68953, + "recall_at_3": 0.4407, + "recall_at_5": 0.5093 + }, + { + "cv_recall_at_1": 0.39843, + "cv_recall_at_10": 0.66667, + "cv_recall_at_100": 0.8945, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.73625, + "cv_recall_at_3": 0.53872, + "cv_recall_at_5": 0.59484, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.52595, + "map_at_1": 0.39843, + "map_at_10": 0.4817, + "map_at_100": 0.49033, + "map_at_1000": 0.49091, + "map_at_20": 0.48638, + "map_at_3": 0.45978, + "map_at_5": 0.47213, + "mrr_at_1": 0.3984287317620651, + "mrr_at_10": 0.48169704095629967, + "mrr_at_100": 0.49033374059196033, + "mrr_at_1000": 0.4909059829649581, + "mrr_at_20": 0.4863823772433859, + "mrr_at_3": 0.4597830153385704, + "mrr_at_5": 0.47212869435091576, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.35457465941244737, + "nauc_cv_recall_at_100_max": 0.5576807085732769, + "nauc_cv_recall_at_100_std": 0.39068957310710517, + "nauc_cv_recall_at_10_diff1": 0.49672487714012176, + "nauc_cv_recall_at_10_max": 0.6096103856426884, + "nauc_cv_recall_at_10_std": 0.18319856486677483, + "nauc_cv_recall_at_1_diff1": 0.6873254674015392, + "nauc_cv_recall_at_1_max": 0.5296868024822796, + "nauc_cv_recall_at_1_std": -0.0357131227009029, + "nauc_cv_recall_at_20_diff1": 0.45066929394701233, + "nauc_cv_recall_at_20_max": 0.6132154485168014, + "nauc_cv_recall_at_20_std": 0.24783736517811059, + "nauc_cv_recall_at_3_diff1": 0.5782535988484254, + "nauc_cv_recall_at_3_max": 0.5585680345701475, + "nauc_cv_recall_at_3_std": 0.03651369037074517, + "nauc_cv_recall_at_5_diff1": 0.5350363821120877, + "nauc_cv_recall_at_5_max": 0.5901419254871935, + "nauc_cv_recall_at_5_std": 0.1290002567018423, + "nauc_map_at_1000_diff1": 0.6326492352687968, + "nauc_map_at_1000_max": 0.5476258801331528, + "nauc_map_at_1000_std": 0.013729566821103458, + "nauc_map_at_100_diff1": 0.632252036265804, + "nauc_map_at_100_max": 0.5476452208181938, + "nauc_map_at_100_std": 0.014276296948681754, + "nauc_map_at_10_diff1": 0.6324263142199975, + "nauc_map_at_10_max": 0.5489231706174229, + "nauc_map_at_10_std": 0.013307014661679977, + "nauc_map_at_1_diff1": 0.6873254674015392, + "nauc_map_at_1_max": 0.5296868024822796, + "nauc_map_at_1_std": -0.0357131227009029, + "nauc_map_at_20_diff1": 0.631412253292453, + "nauc_map_at_20_max": 0.5484171024649589, + "nauc_map_at_20_std": 0.014782813145197712, + "nauc_map_at_3_diff1": 0.6430234238893437, + "nauc_map_at_3_max": 0.5411032167398862, + "nauc_map_at_3_std": -0.008678979459466659, + "nauc_map_at_5_diff1": 0.63526153323913, + "nauc_map_at_5_max": 0.5477376679115433, + "nauc_map_at_5_std": 0.009739363758753661, + "nauc_mrr_at_1000_diff1": 0.6326492352687968, + "nauc_mrr_at_1000_max": 0.5476258801331528, + "nauc_mrr_at_1000_std": 0.013729566821103458, + "nauc_mrr_at_100_diff1": 0.632252036265804, + "nauc_mrr_at_100_max": 0.5476452208181938, + "nauc_mrr_at_100_std": 0.014276296948681754, + "nauc_mrr_at_10_diff1": 0.6324263142199975, + "nauc_mrr_at_10_max": 0.5489231706174229, + "nauc_mrr_at_10_std": 0.013307014661679977, + "nauc_mrr_at_1_diff1": 0.6873254674015392, + "nauc_mrr_at_1_max": 0.5296868024822796, + "nauc_mrr_at_1_std": -0.0357131227009029, + "nauc_mrr_at_20_diff1": 0.631412253292453, + "nauc_mrr_at_20_max": 0.5484171024649589, + "nauc_mrr_at_20_std": 0.014782813145197712, + "nauc_mrr_at_3_diff1": 0.6430234238893437, + "nauc_mrr_at_3_max": 0.5411032167398862, + "nauc_mrr_at_3_std": -0.008678979459466659, + "nauc_mrr_at_5_diff1": 0.63526153323913, + "nauc_mrr_at_5_max": 0.5477376679115433, + "nauc_mrr_at_5_std": 0.009739363758753661, + "nauc_ndcg_at_1000_diff1": 0.6165259932868026, + "nauc_ndcg_at_1000_max": 0.5525071476684726, + "nauc_ndcg_at_1000_std": 0.03385620702194307, + "nauc_ndcg_at_100_diff1": 0.604737973648432, + "nauc_ndcg_at_100_max": 0.5527301799243698, + "nauc_ndcg_at_100_std": 0.049956953429283336, + "nauc_ndcg_at_10_diff1": 0.6042269549971561, + "nauc_ndcg_at_10_max": 0.5612708687432685, + "nauc_ndcg_at_10_std": 0.04763297007687358, + "nauc_ndcg_at_1_diff1": 0.6873254674015392, + "nauc_ndcg_at_1_max": 0.5296868024822796, + "nauc_ndcg_at_1_std": -0.0357131227009029, + "nauc_ndcg_at_20_diff1": 0.5994153801773628, + "nauc_ndcg_at_20_max": 0.5600367471910729, + "nauc_ndcg_at_20_std": 0.05474081561828843, + "nauc_ndcg_at_3_diff1": 0.6270236848909636, + "nauc_ndcg_at_3_max": 0.5454043787504821, + "nauc_ndcg_at_3_std": 0.0023889403899353366, + "nauc_ndcg_at_5_diff1": 0.6119886766768379, + "nauc_ndcg_at_5_max": 0.5574494515120727, + "nauc_ndcg_at_5_std": 0.03679944430425355, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.35457465941244803, + "nauc_precision_at_100_max": 0.5576807085732789, + "nauc_precision_at_100_std": 0.39068957310710556, + "nauc_precision_at_10_diff1": 0.4967248771401208, + "nauc_precision_at_10_max": 0.6096103856426881, + "nauc_precision_at_10_std": 0.1831985648667747, + "nauc_precision_at_1_diff1": 0.6873254674015392, + "nauc_precision_at_1_max": 0.5296868024822796, + "nauc_precision_at_1_std": -0.0357131227009029, + "nauc_precision_at_20_diff1": 0.4506692939470118, + "nauc_precision_at_20_max": 0.6132154485168013, + "nauc_precision_at_20_std": 0.24783736517811034, + "nauc_precision_at_3_diff1": 0.5782535988484254, + "nauc_precision_at_3_max": 0.5585680345701476, + "nauc_precision_at_3_std": 0.03651369037074524, + "nauc_precision_at_5_diff1": 0.5350363821120875, + "nauc_precision_at_5_max": 0.5901419254871931, + "nauc_precision_at_5_std": 0.12900025670184193, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.35457465941244737, + "nauc_recall_at_100_max": 0.5576807085732769, + "nauc_recall_at_100_std": 0.39068957310710517, + "nauc_recall_at_10_diff1": 0.49672487714012176, + "nauc_recall_at_10_max": 0.6096103856426884, + "nauc_recall_at_10_std": 0.18319856486677483, + "nauc_recall_at_1_diff1": 0.6873254674015392, + "nauc_recall_at_1_max": 0.5296868024822796, + "nauc_recall_at_1_std": -0.0357131227009029, + "nauc_recall_at_20_diff1": 0.45066929394701233, + "nauc_recall_at_20_max": 0.6132154485168014, + "nauc_recall_at_20_std": 0.24783736517811059, + "nauc_recall_at_3_diff1": 0.5782535988484254, + "nauc_recall_at_3_max": 0.5585680345701475, + "nauc_recall_at_3_std": 0.03651369037074517, + "nauc_recall_at_5_diff1": 0.5350363821120877, + "nauc_recall_at_5_max": 0.5901419254871935, + "nauc_recall_at_5_std": 0.1290002567018423, + "ndcg_at_1": 0.39843, + "ndcg_at_10": 0.52595, + "ndcg_at_100": 0.57245, + "ndcg_at_1000": 0.58623, + "ndcg_at_20": 0.54336, + "ndcg_at_3": 0.48004, + "ndcg_at_5": 0.50273, + "precision_at_1": 0.39843, + "precision_at_10": 0.06667, + "precision_at_100": 0.00895, + "precision_at_1000": 0.001, + "precision_at_20": 0.03681, + "precision_at_3": 0.17957, + "precision_at_5": 0.11897, + "recall_at_1": 0.39843, + "recall_at_10": 0.66667, + "recall_at_100": 0.8945, + "recall_at_1000": 1.0, + "recall_at_20": 0.73625, + "recall_at_3": 0.53872, + "recall_at_5": 0.59484 + }, + { + "cv_recall_at_1": 0.27895, + "cv_recall_at_10": 0.54035, + "cv_recall_at_100": 0.85789, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.65965, + "cv_recall_at_3": 0.36667, + "cv_recall_at_5": 0.44912, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.39322, + "map_at_1": 0.27895, + "map_at_10": 0.348, + "map_at_100": 0.36124, + "map_at_1000": 0.36205, + "map_at_20": 0.35658, + "map_at_3": 0.31667, + "map_at_5": 0.33579, + "mrr_at_1": 0.2789473684210526, + "mrr_at_10": 0.34799986076301853, + "mrr_at_100": 0.3612379451067614, + "mrr_at_1000": 0.3620538102805267, + "mrr_at_20": 0.35657692868239554, + "mrr_at_3": 0.31666666666666665, + "mrr_at_5": 0.33578947368421047, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.27402837236632377, + "nauc_cv_recall_at_100_max": 0.5046647719777929, + "nauc_cv_recall_at_100_std": 0.43405763211857484, + "nauc_cv_recall_at_10_diff1": 0.36232640243426695, + "nauc_cv_recall_at_10_max": 0.41406189045798525, + "nauc_cv_recall_at_10_std": 0.18209701958336588, + "nauc_cv_recall_at_1_diff1": 0.578766879493732, + "nauc_cv_recall_at_1_max": 0.4193896011763175, + "nauc_cv_recall_at_1_std": 0.023251418233412462, + "nauc_cv_recall_at_20_diff1": 0.27230672547627616, + "nauc_cv_recall_at_20_max": 0.45335866491324034, + "nauc_cv_recall_at_20_std": 0.3078805264934194, + "nauc_cv_recall_at_3_diff1": 0.46300966262279075, + "nauc_cv_recall_at_3_max": 0.4259355718065274, + "nauc_cv_recall_at_3_std": 0.07531384593497516, + "nauc_cv_recall_at_5_diff1": 0.40499343150854816, + "nauc_cv_recall_at_5_max": 0.4071683830486986, + "nauc_cv_recall_at_5_std": 0.1010146711620259, + "nauc_map_at_1000_diff1": 0.5071629288831934, + "nauc_map_at_1000_max": 0.42123422925176146, + "nauc_map_at_1000_std": 0.06497914953890994, + "nauc_map_at_100_diff1": 0.506574340833974, + "nauc_map_at_100_max": 0.4214497059406688, + "nauc_map_at_100_std": 0.06587010366751947, + "nauc_map_at_10_diff1": 0.5082446345792789, + "nauc_map_at_10_max": 0.4189152236357482, + "nauc_map_at_10_std": 0.06010066860993658, + "nauc_map_at_1_diff1": 0.578766879493732, + "nauc_map_at_1_max": 0.4193896011763175, + "nauc_map_at_1_std": 0.023251418233412462, + "nauc_map_at_20_diff1": 0.5041720899117745, + "nauc_map_at_20_max": 0.42110178848378027, + "nauc_map_at_20_std": 0.06636052994372635, + "nauc_map_at_3_diff1": 0.528685422161208, + "nauc_map_at_3_max": 0.42343211512633144, + "nauc_map_at_3_std": 0.04403801855881142, + "nauc_map_at_5_diff1": 0.513704999469193, + "nauc_map_at_5_max": 0.4189232291850256, + "nauc_map_at_5_std": 0.050220770372886234, + "nauc_mrr_at_1000_diff1": 0.5071629288831934, + "nauc_mrr_at_1000_max": 0.42123422925176146, + "nauc_mrr_at_1000_std": 0.06497914953890994, + "nauc_mrr_at_100_diff1": 0.506574340833974, + "nauc_mrr_at_100_max": 0.4214497059406688, + "nauc_mrr_at_100_std": 0.06587010366751947, + "nauc_mrr_at_10_diff1": 0.5082446345792789, + "nauc_mrr_at_10_max": 0.4189152236357482, + "nauc_mrr_at_10_std": 0.06010066860993658, + "nauc_mrr_at_1_diff1": 0.578766879493732, + "nauc_mrr_at_1_max": 0.4193896011763175, + "nauc_mrr_at_1_std": 0.023251418233412462, + "nauc_mrr_at_20_diff1": 0.5041720899117745, + "nauc_mrr_at_20_max": 0.42110178848378027, + "nauc_mrr_at_20_std": 0.06636052994372635, + "nauc_mrr_at_3_diff1": 0.528685422161208, + "nauc_mrr_at_3_max": 0.42343211512633144, + "nauc_mrr_at_3_std": 0.04403801855881142, + "nauc_mrr_at_5_diff1": 0.513704999469193, + "nauc_mrr_at_5_max": 0.4189232291850256, + "nauc_mrr_at_5_std": 0.050220770372886234, + "nauc_ndcg_at_1000_diff1": 0.48697231270265845, + "nauc_ndcg_at_1000_max": 0.4231368217605475, + "nauc_ndcg_at_1000_std": 0.08523789915262146, + "nauc_ndcg_at_100_diff1": 0.47393152314008596, + "nauc_ndcg_at_100_max": 0.42813566600077263, + "nauc_ndcg_at_100_std": 0.10638039759966585, + "nauc_ndcg_at_10_diff1": 0.47317884996832, + "nauc_ndcg_at_10_max": 0.4175207394457012, + "nauc_ndcg_at_10_std": 0.08784415774604273, + "nauc_ndcg_at_1_diff1": 0.578766879493732, + "nauc_ndcg_at_1_max": 0.4193896011763175, + "nauc_ndcg_at_1_std": 0.023251418233412462, + "nauc_ndcg_at_20_diff1": 0.4575951570176458, + "nauc_ndcg_at_20_max": 0.42601612367918157, + "nauc_ndcg_at_20_std": 0.11166824539675238, + "nauc_ndcg_at_3_diff1": 0.5114809115902081, + "nauc_ndcg_at_3_max": 0.42413650929139324, + "nauc_ndcg_at_3_std": 0.05216132426187068, + "nauc_ndcg_at_5_diff1": 0.4858911567361287, + "nauc_ndcg_at_5_max": 0.41614631893275955, + "nauc_ndcg_at_5_std": 0.06310315441584341, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.2740283723663231, + "nauc_precision_at_100_max": 0.5046647719777917, + "nauc_precision_at_100_std": 0.4340576321185744, + "nauc_precision_at_10_diff1": 0.36232640243426684, + "nauc_precision_at_10_max": 0.414061890457985, + "nauc_precision_at_10_std": 0.18209701958336594, + "nauc_precision_at_1_diff1": 0.578766879493732, + "nauc_precision_at_1_max": 0.4193896011763175, + "nauc_precision_at_1_std": 0.023251418233412462, + "nauc_precision_at_20_diff1": 0.2723067254762764, + "nauc_precision_at_20_max": 0.45335866491324045, + "nauc_precision_at_20_std": 0.3078805264934188, + "nauc_precision_at_3_diff1": 0.4630096626227905, + "nauc_precision_at_3_max": 0.42593557180652697, + "nauc_precision_at_3_std": 0.07531384593497523, + "nauc_precision_at_5_diff1": 0.4049934315085483, + "nauc_precision_at_5_max": 0.40716838304869857, + "nauc_precision_at_5_std": 0.1010146711620258, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.27402837236632377, + "nauc_recall_at_100_max": 0.5046647719777929, + "nauc_recall_at_100_std": 0.43405763211857484, + "nauc_recall_at_10_diff1": 0.36232640243426695, + "nauc_recall_at_10_max": 0.41406189045798525, + "nauc_recall_at_10_std": 0.18209701958336588, + "nauc_recall_at_1_diff1": 0.578766879493732, + "nauc_recall_at_1_max": 0.4193896011763175, + "nauc_recall_at_1_std": 0.023251418233412462, + "nauc_recall_at_20_diff1": 0.27230672547627616, + "nauc_recall_at_20_max": 0.45335866491324034, + "nauc_recall_at_20_std": 0.3078805264934194, + "nauc_recall_at_3_diff1": 0.46300966262279075, + "nauc_recall_at_3_max": 0.4259355718065274, + "nauc_recall_at_3_std": 0.07531384593497516, + "nauc_recall_at_5_diff1": 0.40499343150854816, + "nauc_recall_at_5_max": 0.4071683830486986, + "nauc_recall_at_5_std": 0.1010146711620259, + "ndcg_at_1": 0.27895, + "ndcg_at_10": 0.39322, + "ndcg_at_100": 0.45969, + "ndcg_at_1000": 0.47856, + "ndcg_at_20": 0.42374, + "ndcg_at_3": 0.32947, + "ndcg_at_5": 0.36367, + "precision_at_1": 0.27895, + "precision_at_10": 0.05404, + "precision_at_100": 0.00858, + "precision_at_1000": 0.001, + "precision_at_20": 0.03298, + "precision_at_3": 0.12222, + "precision_at_5": 0.08982, + "recall_at_1": 0.27895, + "recall_at_10": 0.54035, + "recall_at_100": 0.85789, + "recall_at_1000": 1.0, + "recall_at_20": 0.65965, + "recall_at_3": 0.36667, + "recall_at_5": 0.44912 + }, + { + "cv_recall_at_1": 0.19108, + "cv_recall_at_10": 0.42906, + "cv_recall_at_100": 0.76659, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.50572, + "cv_recall_at_3": 0.29519, + "cv_recall_at_5": 0.35469, + "hf_subset": "et", + "languages": [ + "est-Latn" + ], + "main_score": 0.2994, + "map_at_1": 0.19108, + "map_at_10": 0.25916, + "map_at_100": 0.27058, + "map_at_1000": 0.2716, + "map_at_20": 0.26439, + "map_at_3": 0.2357, + "map_at_5": 0.24937, + "mrr_at_1": 0.19107551487414187, + "mrr_at_10": 0.25915967454869054, + "mrr_at_100": 0.27057718507760353, + "mrr_at_1000": 0.271598792636963, + "mrr_at_20": 0.2643949666341024, + "mrr_at_3": 0.2356979405034326, + "mrr_at_5": 0.2493707093821511, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.008516376183806164, + "nauc_cv_recall_at_100_max": 0.5283066591673302, + "nauc_cv_recall_at_100_std": 0.5829740280471483, + "nauc_cv_recall_at_10_diff1": 0.2220847558311369, + "nauc_cv_recall_at_10_max": 0.43731525006271427, + "nauc_cv_recall_at_10_std": 0.20381677609332463, + "nauc_cv_recall_at_1_diff1": 0.5605226558512929, + "nauc_cv_recall_at_1_max": 0.5336666192190369, + "nauc_cv_recall_at_1_std": 0.0588461370327258, + "nauc_cv_recall_at_20_diff1": 0.15852763873374953, + "nauc_cv_recall_at_20_max": 0.4520595706685236, + "nauc_cv_recall_at_20_std": 0.31579778409402076, + "nauc_cv_recall_at_3_diff1": 0.3743596257745175, + "nauc_cv_recall_at_3_max": 0.46859264567363157, + "nauc_cv_recall_at_3_std": 0.06856746069956918, + "nauc_cv_recall_at_5_diff1": 0.2959683249078792, + "nauc_cv_recall_at_5_max": 0.42952579478833075, + "nauc_cv_recall_at_5_std": 0.09896414249666792, + "nauc_map_at_1000_diff1": 0.4317225812730425, + "nauc_map_at_1000_max": 0.4951111047823308, + "nauc_map_at_1000_std": 0.09733934003605733, + "nauc_map_at_100_diff1": 0.4308421866301083, + "nauc_map_at_100_max": 0.4953343634107362, + "nauc_map_at_100_std": 0.09853358301236757, + "nauc_map_at_10_diff1": 0.43592345198798077, + "nauc_map_at_10_max": 0.49290073487004293, + "nauc_map_at_10_std": 0.08683872112304801, + "nauc_map_at_1_diff1": 0.5605226558512929, + "nauc_map_at_1_max": 0.5336666192190369, + "nauc_map_at_1_std": 0.0588461370327258, + "nauc_map_at_20_diff1": 0.4316814136771351, + "nauc_map_at_20_max": 0.49457613365867314, + "nauc_map_at_20_std": 0.09548935501608316, + "nauc_map_at_3_diff1": 0.46875777456087836, + "nauc_map_at_3_max": 0.5034907821086785, + "nauc_map_at_3_std": 0.06354789645788499, + "nauc_map_at_5_diff1": 0.44783010602361834, + "nauc_map_at_5_max": 0.49306058616671694, + "nauc_map_at_5_std": 0.07173277743503308, + "nauc_mrr_at_1000_diff1": 0.4317225812730425, + "nauc_mrr_at_1000_max": 0.4951111047823308, + "nauc_mrr_at_1000_std": 0.09733934003605733, + "nauc_mrr_at_100_diff1": 0.4308421866301083, + "nauc_mrr_at_100_max": 0.4953343634107362, + "nauc_mrr_at_100_std": 0.09853358301236757, + "nauc_mrr_at_10_diff1": 0.43592345198798077, + "nauc_mrr_at_10_max": 0.49290073487004293, + "nauc_mrr_at_10_std": 0.08683872112304801, + "nauc_mrr_at_1_diff1": 0.5605226558512929, + "nauc_mrr_at_1_max": 0.5336666192190369, + "nauc_mrr_at_1_std": 0.0588461370327258, + "nauc_mrr_at_20_diff1": 0.4316814136771351, + "nauc_mrr_at_20_max": 0.49457613365867314, + "nauc_mrr_at_20_std": 0.09548935501608316, + "nauc_mrr_at_3_diff1": 0.46875777456087836, + "nauc_mrr_at_3_max": 0.5034907821086785, + "nauc_mrr_at_3_std": 0.06354789645788499, + "nauc_mrr_at_5_diff1": 0.44783010602361834, + "nauc_mrr_at_5_max": 0.49306058616671694, + "nauc_mrr_at_5_std": 0.07173277743503308, + "nauc_ndcg_at_1000_diff1": 0.3902812726294797, + "nauc_ndcg_at_1000_max": 0.4883252404313833, + "nauc_ndcg_at_1000_std": 0.12918188527168667, + "nauc_ndcg_at_100_diff1": 0.3583473582903687, + "nauc_ndcg_at_100_max": 0.4926795176760152, + "nauc_ndcg_at_100_std": 0.16837332429694266, + "nauc_ndcg_at_10_diff1": 0.37884464737747714, + "nauc_ndcg_at_10_max": 0.4771396846330027, + "nauc_ndcg_at_10_std": 0.11567882387987487, + "nauc_ndcg_at_1_diff1": 0.5605226558512929, + "nauc_ndcg_at_1_max": 0.5336666192190369, + "nauc_ndcg_at_1_std": 0.0588461370327258, + "nauc_ndcg_at_20_diff1": 0.3629320685513456, + "nauc_ndcg_at_20_max": 0.48203568883383097, + "nauc_ndcg_at_20_std": 0.14601766668739585, + "nauc_ndcg_at_3_diff1": 0.44244034795034554, + "nauc_ndcg_at_3_max": 0.4938399382557547, + "nauc_ndcg_at_3_std": 0.06494358833093328, + "nauc_ndcg_at_5_diff1": 0.4060786418896771, + "nauc_ndcg_at_5_max": 0.47576035017023, + "nauc_ndcg_at_5_std": 0.07898953790711814, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.008516376183805786, + "nauc_precision_at_100_max": 0.5283066591673299, + "nauc_precision_at_100_std": 0.5829740280471479, + "nauc_precision_at_10_diff1": 0.2220847558311367, + "nauc_precision_at_10_max": 0.437315250062714, + "nauc_precision_at_10_std": 0.20381677609332435, + "nauc_precision_at_1_diff1": 0.5605226558512929, + "nauc_precision_at_1_max": 0.5336666192190369, + "nauc_precision_at_1_std": 0.0588461370327258, + "nauc_precision_at_20_diff1": 0.15852763873374898, + "nauc_precision_at_20_max": 0.45205957066852337, + "nauc_precision_at_20_std": 0.3157977840940208, + "nauc_precision_at_3_diff1": 0.3743596257745172, + "nauc_precision_at_3_max": 0.4685926456736312, + "nauc_precision_at_3_std": 0.06856746069956916, + "nauc_precision_at_5_diff1": 0.2959683249078796, + "nauc_precision_at_5_max": 0.429525794788331, + "nauc_precision_at_5_std": 0.09896414249666852, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.008516376183806164, + "nauc_recall_at_100_max": 0.5283066591673302, + "nauc_recall_at_100_std": 0.5829740280471483, + "nauc_recall_at_10_diff1": 0.2220847558311369, + "nauc_recall_at_10_max": 0.43731525006271427, + "nauc_recall_at_10_std": 0.20381677609332463, + "nauc_recall_at_1_diff1": 0.5605226558512929, + "nauc_recall_at_1_max": 0.5336666192190369, + "nauc_recall_at_1_std": 0.0588461370327258, + "nauc_recall_at_20_diff1": 0.15852763873374953, + "nauc_recall_at_20_max": 0.4520595706685236, + "nauc_recall_at_20_std": 0.31579778409402076, + "nauc_recall_at_3_diff1": 0.3743596257745175, + "nauc_recall_at_3_max": 0.46859264567363157, + "nauc_recall_at_3_std": 0.06856746069956918, + "nauc_recall_at_5_diff1": 0.2959683249078792, + "nauc_recall_at_5_max": 0.42952579478833075, + "nauc_recall_at_5_std": 0.09896414249666792, + "ndcg_at_1": 0.19108, + "ndcg_at_10": 0.2994, + "ndcg_at_100": 0.36608, + "ndcg_at_1000": 0.3953, + "ndcg_at_20": 0.31868, + "ndcg_at_3": 0.25092, + "ndcg_at_5": 0.2755, + "precision_at_1": 0.19108, + "precision_at_10": 0.04291, + "precision_at_100": 0.00767, + "precision_at_1000": 0.001, + "precision_at_20": 0.02529, + "precision_at_3": 0.0984, + "precision_at_5": 0.07094, + "recall_at_1": 0.19108, + "recall_at_10": 0.42906, + "recall_at_100": 0.76659, + "recall_at_1000": 1.0, + "recall_at_20": 0.50572, + "recall_at_3": 0.29519, + "recall_at_5": 0.35469 + }, + { + "cv_recall_at_1": 0.43507, + "cv_recall_at_10": 0.73141, + "cv_recall_at_100": 0.94007, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.81021, + "cv_recall_at_3": 0.56715, + "cv_recall_at_5": 0.62819, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.57023, + "map_at_1": 0.43507, + "map_at_10": 0.52031, + "map_at_100": 0.52893, + "map_at_1000": 0.52923, + "map_at_20": 0.52575, + "map_at_3": 0.4926, + "map_at_5": 0.50653, + "mrr_at_1": 0.43507214206437295, + "mrr_at_10": 0.5203103253880169, + "mrr_at_100": 0.5289324885399981, + "mrr_at_1000": 0.5292325136237984, + "mrr_at_20": 0.5257454133953585, + "mrr_at_3": 0.4926008139104697, + "mrr_at_5": 0.5065297817240101, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6978733265275376, + "nauc_cv_recall_at_100_max": 0.5949865305257486, + "nauc_cv_recall_at_100_std": 0.5166498497442625, + "nauc_cv_recall_at_10_diff1": 0.5870400374929791, + "nauc_cv_recall_at_10_max": 0.4192351294053403, + "nauc_cv_recall_at_10_std": 0.15748320564522547, + "nauc_cv_recall_at_1_diff1": 0.7306432624255598, + "nauc_cv_recall_at_1_max": 0.514466513106884, + "nauc_cv_recall_at_1_std": 0.067281314126223, + "nauc_cv_recall_at_20_diff1": 0.5648059742557373, + "nauc_cv_recall_at_20_max": 0.4139453010958553, + "nauc_cv_recall_at_20_std": 0.22809417514046965, + "nauc_cv_recall_at_3_diff1": 0.6414846050634647, + "nauc_cv_recall_at_3_max": 0.47781893080666504, + "nauc_cv_recall_at_3_std": 0.09515707826623646, + "nauc_cv_recall_at_5_diff1": 0.6142780982117747, + "nauc_cv_recall_at_5_max": 0.43914771938140457, + "nauc_cv_recall_at_5_std": 0.09945784932203011, + "nauc_map_at_1000_diff1": 0.6895559381936318, + "nauc_map_at_1000_max": 0.49563279664453025, + "nauc_map_at_1000_std": 0.08673011670743808, + "nauc_map_at_100_diff1": 0.689562613982922, + "nauc_map_at_100_max": 0.4957215240433665, + "nauc_map_at_100_std": 0.08710243504689028, + "nauc_map_at_10_diff1": 0.6874136307446468, + "nauc_map_at_10_max": 0.49376751093121124, + "nauc_map_at_10_std": 0.08555924018421786, + "nauc_map_at_1_diff1": 0.7306432624255598, + "nauc_map_at_1_max": 0.514466513106884, + "nauc_map_at_1_std": 0.067281314126223, + "nauc_map_at_20_diff1": 0.6875938630174272, + "nauc_map_at_20_max": 0.49392152305217696, + "nauc_map_at_20_std": 0.08669227777785457, + "nauc_map_at_3_diff1": 0.6929406454047352, + "nauc_map_at_3_max": 0.5020126401504487, + "nauc_map_at_3_std": 0.0800419124746124, + "nauc_map_at_5_diff1": 0.6882651563349085, + "nauc_map_at_5_max": 0.4944791856163838, + "nauc_map_at_5_std": 0.08138282547856554, + "nauc_mrr_at_1000_diff1": 0.6895559381936318, + "nauc_mrr_at_1000_max": 0.49563279664453025, + "nauc_mrr_at_1000_std": 0.08673011670743808, + "nauc_mrr_at_100_diff1": 0.689562613982922, + "nauc_mrr_at_100_max": 0.4957215240433665, + "nauc_mrr_at_100_std": 0.08710243504689028, + "nauc_mrr_at_10_diff1": 0.6874136307446468, + "nauc_mrr_at_10_max": 0.49376751093121124, + "nauc_mrr_at_10_std": 0.08555924018421786, + "nauc_mrr_at_1_diff1": 0.7306432624255598, + "nauc_mrr_at_1_max": 0.514466513106884, + "nauc_mrr_at_1_std": 0.067281314126223, + "nauc_mrr_at_20_diff1": 0.6875938630174272, + "nauc_mrr_at_20_max": 0.49392152305217696, + "nauc_mrr_at_20_std": 0.08669227777785457, + "nauc_mrr_at_3_diff1": 0.6929406454047352, + "nauc_mrr_at_3_max": 0.5020126401504487, + "nauc_mrr_at_3_std": 0.0800419124746124, + "nauc_mrr_at_5_diff1": 0.6882651563349085, + "nauc_mrr_at_5_max": 0.4944791856163838, + "nauc_mrr_at_5_std": 0.08138282547856554, + "nauc_ndcg_at_1000_diff1": 0.6822856007923828, + "nauc_ndcg_at_1000_max": 0.49173586713551026, + "nauc_ndcg_at_1000_std": 0.09620762284631088, + "nauc_ndcg_at_100_diff1": 0.6827195804493991, + "nauc_ndcg_at_100_max": 0.494646191162205, + "nauc_ndcg_at_100_std": 0.10798696761972172, + "nauc_ndcg_at_10_diff1": 0.6681409781657833, + "nauc_ndcg_at_10_max": 0.47984251929370647, + "nauc_ndcg_at_10_std": 0.09819022365147305, + "nauc_ndcg_at_1_diff1": 0.7306432624255598, + "nauc_ndcg_at_1_max": 0.514466513106884, + "nauc_ndcg_at_1_std": 0.067281314126223, + "nauc_ndcg_at_20_diff1": 0.6684980553980436, + "nauc_ndcg_at_20_max": 0.48128478811001296, + "nauc_ndcg_at_20_std": 0.10510187950480612, + "nauc_ndcg_at_3_diff1": 0.6802566039971267, + "nauc_ndcg_at_3_max": 0.49618043172313236, + "nauc_ndcg_at_3_std": 0.08380990206914761, + "nauc_ndcg_at_5_diff1": 0.6712860737099376, + "nauc_ndcg_at_5_max": 0.48218117316614384, + "nauc_ndcg_at_5_std": 0.08570823218004223, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.697873326527537, + "nauc_precision_at_100_max": 0.5949865305257479, + "nauc_precision_at_100_std": 0.5166498497442638, + "nauc_precision_at_10_diff1": 0.5870400374929786, + "nauc_precision_at_10_max": 0.41923512940533914, + "nauc_precision_at_10_std": 0.15748320564522525, + "nauc_precision_at_1_diff1": 0.7306432624255598, + "nauc_precision_at_1_max": 0.514466513106884, + "nauc_precision_at_1_std": 0.067281314126223, + "nauc_precision_at_20_diff1": 0.5648059742557369, + "nauc_precision_at_20_max": 0.4139453010958548, + "nauc_precision_at_20_std": 0.22809417514046984, + "nauc_precision_at_3_diff1": 0.6414846050634644, + "nauc_precision_at_3_max": 0.4778189308066647, + "nauc_precision_at_3_std": 0.09515707826623591, + "nauc_precision_at_5_diff1": 0.614278098211774, + "nauc_precision_at_5_max": 0.4391477193814045, + "nauc_precision_at_5_std": 0.09945784932202985, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.6978733265275376, + "nauc_recall_at_100_max": 0.5949865305257486, + "nauc_recall_at_100_std": 0.5166498497442625, + "nauc_recall_at_10_diff1": 0.5870400374929791, + "nauc_recall_at_10_max": 0.4192351294053403, + "nauc_recall_at_10_std": 0.15748320564522547, + "nauc_recall_at_1_diff1": 0.7306432624255598, + "nauc_recall_at_1_max": 0.514466513106884, + "nauc_recall_at_1_std": 0.067281314126223, + "nauc_recall_at_20_diff1": 0.5648059742557373, + "nauc_recall_at_20_max": 0.4139453010958553, + "nauc_recall_at_20_std": 0.22809417514046965, + "nauc_recall_at_3_diff1": 0.6414846050634647, + "nauc_recall_at_3_max": 0.47781893080666504, + "nauc_recall_at_3_std": 0.09515707826623646, + "nauc_recall_at_5_diff1": 0.6142780982117747, + "nauc_recall_at_5_max": 0.43914771938140457, + "nauc_recall_at_5_std": 0.09945784932203011, + "ndcg_at_1": 0.43507, + "ndcg_at_10": 0.57023, + "ndcg_at_100": 0.61394, + "ndcg_at_1000": 0.62169, + "ndcg_at_20": 0.59012, + "ndcg_at_3": 0.51172, + "ndcg_at_5": 0.53684, + "precision_at_1": 0.43507, + "precision_at_10": 0.07314, + "precision_at_100": 0.0094, + "precision_at_1000": 0.001, + "precision_at_20": 0.04051, + "precision_at_3": 0.18905, + "precision_at_5": 0.12564, + "recall_at_1": 0.43507, + "recall_at_10": 0.73141, + "recall_at_100": 0.94007, + "recall_at_1000": 1.0, + "recall_at_20": 0.81021, + "recall_at_3": 0.56715, + "recall_at_5": 0.62819 + }, + { + "cv_recall_at_1": 0.25242, + "cv_recall_at_10": 0.49087, + "cv_recall_at_100": 0.75295, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.55854, + "cv_recall_at_3": 0.35661, + "cv_recall_at_5": 0.41998, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.36077, + "map_at_1": 0.25242, + "map_at_10": 0.32029, + "map_at_100": 0.32975, + "map_at_1000": 0.33084, + "map_at_20": 0.32497, + "map_at_3": 0.29646, + "map_at_5": 0.31074, + "mrr_at_1": 0.25241675617615467, + "mrr_at_10": 0.3202913746952415, + "mrr_at_100": 0.32974560595937136, + "mrr_at_1000": 0.3308396030425244, + "mrr_at_20": 0.3249678191109887, + "mrr_at_3": 0.29645542427497334, + "mrr_at_5": 0.3107411385606875, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5126757026738264, + "nauc_cv_recall_at_100_max": 0.18510105009062797, + "nauc_cv_recall_at_100_std": 0.5888298680979758, + "nauc_cv_recall_at_10_diff1": 0.5370004026652652, + "nauc_cv_recall_at_10_max": 0.36735113267523206, + "nauc_cv_recall_at_10_std": 0.3517130623928045, + "nauc_cv_recall_at_1_diff1": 0.7038058763022518, + "nauc_cv_recall_at_1_max": 0.4635485682447222, + "nauc_cv_recall_at_1_std": 0.23949641642304445, + "nauc_cv_recall_at_20_diff1": 0.5231357787700573, + "nauc_cv_recall_at_20_max": 0.3204339578087355, + "nauc_cv_recall_at_20_std": 0.3921653472617399, + "nauc_cv_recall_at_3_diff1": 0.5830257867486661, + "nauc_cv_recall_at_3_max": 0.40642348260668365, + "nauc_cv_recall_at_3_std": 0.2693784583886975, + "nauc_cv_recall_at_5_diff1": 0.5673927390985861, + "nauc_cv_recall_at_5_max": 0.4032859392730803, + "nauc_cv_recall_at_5_std": 0.3144014157382476, + "nauc_map_at_1000_diff1": 0.6411599264391882, + "nauc_map_at_1000_max": 0.4289680674918197, + "nauc_map_at_1000_std": 0.2784491957470269, + "nauc_map_at_100_diff1": 0.6407788736651042, + "nauc_map_at_100_max": 0.4285537008083134, + "nauc_map_at_100_std": 0.27905660807460186, + "nauc_map_at_10_diff1": 0.6411016894025999, + "nauc_map_at_10_max": 0.43353902064107813, + "nauc_map_at_10_std": 0.2737477676391273, + "nauc_map_at_1_diff1": 0.7038058763022518, + "nauc_map_at_1_max": 0.4635485682447222, + "nauc_map_at_1_std": 0.23949641642304445, + "nauc_map_at_20_diff1": 0.6406624642001553, + "nauc_map_at_20_max": 0.43069966751305555, + "nauc_map_at_20_std": 0.27629602423470784, + "nauc_map_at_3_diff1": 0.6488533514998196, + "nauc_map_at_3_max": 0.4386996076405115, + "nauc_map_at_3_std": 0.2573084499673646, + "nauc_map_at_5_diff1": 0.6447966801619837, + "nauc_map_at_5_max": 0.4383966725195242, + "nauc_map_at_5_std": 0.26829757773957064, + "nauc_mrr_at_1000_diff1": 0.6411599264391882, + "nauc_mrr_at_1000_max": 0.4289680674918197, + "nauc_mrr_at_1000_std": 0.2784491957470269, + "nauc_mrr_at_100_diff1": 0.6407788736651042, + "nauc_mrr_at_100_max": 0.4285537008083134, + "nauc_mrr_at_100_std": 0.27905660807460186, + "nauc_mrr_at_10_diff1": 0.6411016894025999, + "nauc_mrr_at_10_max": 0.43353902064107813, + "nauc_mrr_at_10_std": 0.2737477676391273, + "nauc_mrr_at_1_diff1": 0.7038058763022518, + "nauc_mrr_at_1_max": 0.4635485682447222, + "nauc_mrr_at_1_std": 0.23949641642304445, + "nauc_mrr_at_20_diff1": 0.6406624642001553, + "nauc_mrr_at_20_max": 0.43069966751305555, + "nauc_mrr_at_20_std": 0.27629602423470784, + "nauc_mrr_at_3_diff1": 0.6488533514998196, + "nauc_mrr_at_3_max": 0.4386996076405115, + "nauc_mrr_at_3_std": 0.2573084499673646, + "nauc_mrr_at_5_diff1": 0.6447966801619837, + "nauc_mrr_at_5_max": 0.4383966725195242, + "nauc_mrr_at_5_std": 0.26829757773957064, + "nauc_ndcg_at_1000_diff1": 0.6256559860728634, + "nauc_ndcg_at_1000_max": 0.4109741897935761, + "nauc_ndcg_at_1000_std": 0.2988253387764826, + "nauc_ndcg_at_100_diff1": 0.6157338383229767, + "nauc_ndcg_at_100_max": 0.3930122773391341, + "nauc_ndcg_at_100_std": 0.3224691211574583, + "nauc_ndcg_at_10_diff1": 0.6148314521713852, + "nauc_ndcg_at_10_max": 0.4173940060757511, + "nauc_ndcg_at_10_std": 0.29279317802765203, + "nauc_ndcg_at_1_diff1": 0.7038058763022518, + "nauc_ndcg_at_1_max": 0.4635485682447222, + "nauc_ndcg_at_1_std": 0.23949641642304445, + "nauc_ndcg_at_20_diff1": 0.6127351090974947, + "nauc_ndcg_at_20_max": 0.4066455428917083, + "nauc_ndcg_at_20_std": 0.3020330912108028, + "nauc_ndcg_at_3_diff1": 0.6312889534711793, + "nauc_ndcg_at_3_max": 0.43013404859610777, + "nauc_ndcg_at_3_std": 0.260700720898194, + "nauc_ndcg_at_5_diff1": 0.6242322616276245, + "nauc_ndcg_at_5_max": 0.4291320833572419, + "nauc_ndcg_at_5_std": 0.28018555183852706, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5126757026738261, + "nauc_precision_at_100_max": 0.18510105009062786, + "nauc_precision_at_100_std": 0.5888298680979758, + "nauc_precision_at_10_diff1": 0.5370004026652653, + "nauc_precision_at_10_max": 0.367351132675232, + "nauc_precision_at_10_std": 0.3517130623928041, + "nauc_precision_at_1_diff1": 0.7038058763022518, + "nauc_precision_at_1_max": 0.4635485682447222, + "nauc_precision_at_1_std": 0.23949641642304445, + "nauc_precision_at_20_diff1": 0.5231357787700573, + "nauc_precision_at_20_max": 0.3204339578087353, + "nauc_precision_at_20_std": 0.3921653472617399, + "nauc_precision_at_3_diff1": 0.5830257867486658, + "nauc_precision_at_3_max": 0.40642348260668354, + "nauc_precision_at_3_std": 0.2693784583886974, + "nauc_precision_at_5_diff1": 0.567392739098586, + "nauc_precision_at_5_max": 0.40328593927308015, + "nauc_precision_at_5_std": 0.31440141573824765, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5126757026738264, + "nauc_recall_at_100_max": 0.18510105009062797, + "nauc_recall_at_100_std": 0.5888298680979758, + "nauc_recall_at_10_diff1": 0.5370004026652652, + "nauc_recall_at_10_max": 0.36735113267523206, + "nauc_recall_at_10_std": 0.3517130623928045, + "nauc_recall_at_1_diff1": 0.7038058763022518, + "nauc_recall_at_1_max": 0.4635485682447222, + "nauc_recall_at_1_std": 0.23949641642304445, + "nauc_recall_at_20_diff1": 0.5231357787700573, + "nauc_recall_at_20_max": 0.3204339578087355, + "nauc_recall_at_20_std": 0.3921653472617399, + "nauc_recall_at_3_diff1": 0.5830257867486661, + "nauc_recall_at_3_max": 0.40642348260668365, + "nauc_recall_at_3_std": 0.2693784583886975, + "nauc_recall_at_5_diff1": 0.5673927390985861, + "nauc_recall_at_5_max": 0.4032859392730803, + "nauc_recall_at_5_std": 0.3144014157382476, + "ndcg_at_1": 0.25242, + "ndcg_at_10": 0.36077, + "ndcg_at_100": 0.41347, + "ndcg_at_1000": 0.4444, + "ndcg_at_20": 0.37786, + "ndcg_at_3": 0.31182, + "ndcg_at_5": 0.33775, + "precision_at_1": 0.25242, + "precision_at_10": 0.04909, + "precision_at_100": 0.00753, + "precision_at_1000": 0.001, + "precision_at_20": 0.02793, + "precision_at_3": 0.11887, + "precision_at_5": 0.084, + "recall_at_1": 0.25242, + "recall_at_10": 0.49087, + "recall_at_100": 0.75295, + "recall_at_1000": 1.0, + "recall_at_20": 0.55854, + "recall_at_3": 0.35661, + "recall_at_5": 0.41998 + }, + { + "cv_recall_at_1": 0.3, + "cv_recall_at_10": 0.604, + "cv_recall_at_100": 0.911, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.694, + "cv_recall_at_3": 0.443, + "cv_recall_at_5": 0.497, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.43895, + "map_at_1": 0.3, + "map_at_10": 0.38779, + "map_at_100": 0.3999, + "map_at_1000": 0.4004, + "map_at_20": 0.39416, + "map_at_3": 0.36117, + "map_at_5": 0.37342, + "mrr_at_1": 0.3, + "mrr_at_10": 0.387792857142857, + "mrr_at_100": 0.39990229179090303, + "mrr_at_1000": 0.40040463142649246, + "mrr_at_20": 0.3941621968459814, + "mrr_at_3": 0.36116666666666647, + "mrr_at_5": 0.3734166666666663, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.36345849201103536, + "nauc_cv_recall_at_100_max": 0.544057323303853, + "nauc_cv_recall_at_100_std": 0.4570022765660564, + "nauc_cv_recall_at_10_diff1": 0.4760467048899043, + "nauc_cv_recall_at_10_max": 0.14560452700238558, + "nauc_cv_recall_at_10_std": -0.18205012465398782, + "nauc_cv_recall_at_1_diff1": 0.618448858262397, + "nauc_cv_recall_at_1_max": 0.2139305499256856, + "nauc_cv_recall_at_1_std": -0.15225645183083375, + "nauc_cv_recall_at_20_diff1": 0.4649302232925785, + "nauc_cv_recall_at_20_max": 0.16381429399774458, + "nauc_cv_recall_at_20_std": -0.13788681339399703, + "nauc_cv_recall_at_3_diff1": 0.5061653239554741, + "nauc_cv_recall_at_3_max": 0.23292893058180955, + "nauc_cv_recall_at_3_std": -0.13830338958916558, + "nauc_cv_recall_at_5_diff1": 0.48439149248824326, + "nauc_cv_recall_at_5_max": 0.2202481335411537, + "nauc_cv_recall_at_5_std": -0.1419893010545874, + "nauc_map_at_1000_diff1": 0.5687810962778976, + "nauc_map_at_1000_max": 0.21659951806752817, + "nauc_map_at_1000_std": -0.1486960241996032, + "nauc_map_at_100_diff1": 0.5685672958859157, + "nauc_map_at_100_max": 0.21699988255049374, + "nauc_map_at_100_std": -0.148001824259214, + "nauc_map_at_10_diff1": 0.5668997355688797, + "nauc_map_at_10_max": 0.2117857851360944, + "nauc_map_at_10_std": -0.15404347629409737, + "nauc_map_at_1_diff1": 0.618448858262397, + "nauc_map_at_1_max": 0.2139305499256856, + "nauc_map_at_1_std": -0.15225645183083375, + "nauc_map_at_20_diff1": 0.5676571870804599, + "nauc_map_at_20_max": 0.21318841167959945, + "nauc_map_at_20_std": -0.15183481227253412, + "nauc_map_at_3_diff1": 0.5712598391235426, + "nauc_map_at_3_max": 0.22358983160989943, + "nauc_map_at_3_std": -0.14827136439311678, + "nauc_map_at_5_diff1": 0.5668118056008307, + "nauc_map_at_5_max": 0.22092235324327933, + "nauc_map_at_5_std": -0.1487400368543234, + "nauc_mrr_at_1000_diff1": 0.5687810962778976, + "nauc_mrr_at_1000_max": 0.21659951806752817, + "nauc_mrr_at_1000_std": -0.1486960241996032, + "nauc_mrr_at_100_diff1": 0.5685672958859157, + "nauc_mrr_at_100_max": 0.21699988255049374, + "nauc_mrr_at_100_std": -0.148001824259214, + "nauc_mrr_at_10_diff1": 0.5668997355688797, + "nauc_mrr_at_10_max": 0.2117857851360944, + "nauc_mrr_at_10_std": -0.15404347629409737, + "nauc_mrr_at_1_diff1": 0.618448858262397, + "nauc_mrr_at_1_max": 0.2139305499256856, + "nauc_mrr_at_1_std": -0.15225645183083375, + "nauc_mrr_at_20_diff1": 0.5676571870804599, + "nauc_mrr_at_20_max": 0.21318841167959945, + "nauc_mrr_at_20_std": -0.15183481227253412, + "nauc_mrr_at_3_diff1": 0.5712598391235426, + "nauc_mrr_at_3_max": 0.22358983160989943, + "nauc_mrr_at_3_std": -0.14827136439311678, + "nauc_mrr_at_5_diff1": 0.5668118056008307, + "nauc_mrr_at_5_max": 0.22092235324327933, + "nauc_mrr_at_5_std": -0.1487400368543234, + "nauc_ndcg_at_1000_diff1": 0.5577030015084231, + "nauc_ndcg_at_1000_max": 0.21916185280007355, + "nauc_ndcg_at_1000_std": -0.14041310480318414, + "nauc_ndcg_at_100_diff1": 0.5505621433052558, + "nauc_ndcg_at_100_max": 0.2316041550327303, + "nauc_ndcg_at_100_std": -0.11784099812524322, + "nauc_ndcg_at_10_diff1": 0.5457623705560521, + "nauc_ndcg_at_10_max": 0.1986485025025215, + "nauc_ndcg_at_10_std": -0.1595026539038139, + "nauc_ndcg_at_1_diff1": 0.618448858262397, + "nauc_ndcg_at_1_max": 0.2139305499256856, + "nauc_ndcg_at_1_std": -0.15225645183083375, + "nauc_ndcg_at_20_diff1": 0.5467444705576753, + "nauc_ndcg_at_20_max": 0.2041154912575809, + "nauc_ndcg_at_20_std": -0.15017440615387506, + "nauc_ndcg_at_3_diff1": 0.5544831112496268, + "nauc_ndcg_at_3_max": 0.226063809860779, + "nauc_ndcg_at_3_std": -0.1457775189902536, + "nauc_ndcg_at_5_diff1": 0.5461086332262867, + "nauc_ndcg_at_5_max": 0.22102516107083806, + "nauc_ndcg_at_5_std": -0.1469887491451484, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.3634584920110345, + "nauc_precision_at_100_max": 0.5440573233038528, + "nauc_precision_at_100_std": 0.45700227656605796, + "nauc_precision_at_10_diff1": 0.4760467048899036, + "nauc_precision_at_10_max": 0.145604527002385, + "nauc_precision_at_10_std": -0.18205012465398845, + "nauc_precision_at_1_diff1": 0.618448858262397, + "nauc_precision_at_1_max": 0.2139305499256856, + "nauc_precision_at_1_std": -0.15225645183083375, + "nauc_precision_at_20_diff1": 0.46493022329257805, + "nauc_precision_at_20_max": 0.16381429399774428, + "nauc_precision_at_20_std": -0.1378868133939973, + "nauc_precision_at_3_diff1": 0.5061653239554742, + "nauc_precision_at_3_max": 0.23292893058180972, + "nauc_precision_at_3_std": -0.13830338958916566, + "nauc_precision_at_5_diff1": 0.4843914924882435, + "nauc_precision_at_5_max": 0.22024813354115405, + "nauc_precision_at_5_std": -0.14198930105458732, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.36345849201103536, + "nauc_recall_at_100_max": 0.544057323303853, + "nauc_recall_at_100_std": 0.4570022765660564, + "nauc_recall_at_10_diff1": 0.4760467048899043, + "nauc_recall_at_10_max": 0.14560452700238558, + "nauc_recall_at_10_std": -0.18205012465398782, + "nauc_recall_at_1_diff1": 0.618448858262397, + "nauc_recall_at_1_max": 0.2139305499256856, + "nauc_recall_at_1_std": -0.15225645183083375, + "nauc_recall_at_20_diff1": 0.4649302232925785, + "nauc_recall_at_20_max": 0.16381429399774458, + "nauc_recall_at_20_std": -0.13788681339399703, + "nauc_recall_at_3_diff1": 0.5061653239554741, + "nauc_recall_at_3_max": 0.23292893058180955, + "nauc_recall_at_3_std": -0.13830338958916558, + "nauc_recall_at_5_diff1": 0.48439149248824326, + "nauc_recall_at_5_max": 0.2202481335411537, + "nauc_recall_at_5_std": -0.1419893010545874, + "ndcg_at_1": 0.3, + "ndcg_at_10": 0.43895, + "ndcg_at_100": 0.50257, + "ndcg_at_1000": 0.51426, + "ndcg_at_20": 0.46186, + "ndcg_at_3": 0.38211, + "ndcg_at_5": 0.40427, + "precision_at_1": 0.3, + "precision_at_10": 0.0604, + "precision_at_100": 0.00911, + "precision_at_1000": 0.001, + "precision_at_20": 0.0347, + "precision_at_3": 0.14767, + "precision_at_5": 0.0994, + "recall_at_1": 0.3, + "recall_at_10": 0.604, + "recall_at_100": 0.911, + "recall_at_1000": 1.0, + "recall_at_20": 0.694, + "recall_at_3": 0.443, + "recall_at_5": 0.497 + }, + { + "cv_recall_at_1": 0.36616, + "cv_recall_at_10": 0.681, + "cv_recall_at_100": 0.90291, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.75312, + "cv_recall_at_3": 0.51456, + "cv_recall_at_5": 0.58114, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.5121, + "map_at_1": 0.36616, + "map_at_10": 0.45946, + "map_at_100": 0.46798, + "map_at_1000": 0.46848, + "map_at_20": 0.46429, + "map_at_3": 0.43088, + "map_at_5": 0.44607, + "mrr_at_1": 0.36615811373092927, + "mrr_at_10": 0.459455562160139, + "mrr_at_100": 0.46798337728849837, + "mrr_at_1000": 0.4684780705303624, + "mrr_at_20": 0.46429064697191796, + "mrr_at_3": 0.4308830328247804, + "mrr_at_5": 0.4460702727693017, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5412854653754147, + "nauc_cv_recall_at_100_max": 0.6184196906683408, + "nauc_cv_recall_at_100_std": 0.5350992879255435, + "nauc_cv_recall_at_10_diff1": 0.5393654107724762, + "nauc_cv_recall_at_10_max": 0.5480476304207523, + "nauc_cv_recall_at_10_std": 0.31205880886496196, + "nauc_cv_recall_at_1_diff1": 0.6942671543678366, + "nauc_cv_recall_at_1_max": 0.5683430205647082, + "nauc_cv_recall_at_1_std": 0.11179006846134261, + "nauc_cv_recall_at_20_diff1": 0.5364223430251741, + "nauc_cv_recall_at_20_max": 0.547757810534148, + "nauc_cv_recall_at_20_std": 0.37293722690682907, + "nauc_cv_recall_at_3_diff1": 0.5898032624856843, + "nauc_cv_recall_at_3_max": 0.5381986930768126, + "nauc_cv_recall_at_3_std": 0.20251897425233206, + "nauc_cv_recall_at_5_diff1": 0.5594173118504862, + "nauc_cv_recall_at_5_max": 0.5450843125766591, + "nauc_cv_recall_at_5_std": 0.26971020091218706, + "nauc_map_at_1000_diff1": 0.6418669842366795, + "nauc_map_at_1000_max": 0.555669417389876, + "nauc_map_at_1000_std": 0.16327461338547425, + "nauc_map_at_100_diff1": 0.641654783339766, + "nauc_map_at_100_max": 0.5557261032650354, + "nauc_map_at_100_std": 0.16371740784880098, + "nauc_map_at_10_diff1": 0.6399662792835568, + "nauc_map_at_10_max": 0.5551146107747434, + "nauc_map_at_10_std": 0.16320993121786914, + "nauc_map_at_1_diff1": 0.6942671543678366, + "nauc_map_at_1_max": 0.5683430205647082, + "nauc_map_at_1_std": 0.11179006846134261, + "nauc_map_at_20_diff1": 0.6407387457589852, + "nauc_map_at_20_max": 0.5550219526965132, + "nauc_map_at_20_std": 0.16405741588426367, + "nauc_map_at_3_diff1": 0.646333023545593, + "nauc_map_at_3_max": 0.5532091489280436, + "nauc_map_at_3_std": 0.1476639087107855, + "nauc_map_at_5_diff1": 0.6412698510592656, + "nauc_map_at_5_max": 0.5553110808740862, + "nauc_map_at_5_std": 0.1611907197031228, + "nauc_mrr_at_1000_diff1": 0.6418669842366795, + "nauc_mrr_at_1000_max": 0.555669417389876, + "nauc_mrr_at_1000_std": 0.16327461338547425, + "nauc_mrr_at_100_diff1": 0.641654783339766, + "nauc_mrr_at_100_max": 0.5557261032650354, + "nauc_mrr_at_100_std": 0.16371740784880098, + "nauc_mrr_at_10_diff1": 0.6399662792835568, + "nauc_mrr_at_10_max": 0.5551146107747434, + "nauc_mrr_at_10_std": 0.16320993121786914, + "nauc_mrr_at_1_diff1": 0.6942671543678366, + "nauc_mrr_at_1_max": 0.5683430205647082, + "nauc_mrr_at_1_std": 0.11179006846134261, + "nauc_mrr_at_20_diff1": 0.6407387457589852, + "nauc_mrr_at_20_max": 0.5550219526965132, + "nauc_mrr_at_20_std": 0.16405741588426367, + "nauc_mrr_at_3_diff1": 0.646333023545593, + "nauc_mrr_at_3_max": 0.5532091489280436, + "nauc_mrr_at_3_std": 0.1476639087107855, + "nauc_mrr_at_5_diff1": 0.6412698510592656, + "nauc_mrr_at_5_max": 0.5553110808740862, + "nauc_mrr_at_5_std": 0.1611907197031228, + "nauc_ndcg_at_1000_diff1": 0.6323074138798798, + "nauc_ndcg_at_1000_max": 0.5556067762911258, + "nauc_ndcg_at_1000_std": 0.1816086842456634, + "nauc_ndcg_at_100_diff1": 0.6279765940181444, + "nauc_ndcg_at_100_max": 0.5580944570852119, + "nauc_ndcg_at_100_std": 0.1962220514043483, + "nauc_ndcg_at_10_diff1": 0.6187016187672618, + "nauc_ndcg_at_10_max": 0.553114482711755, + "nauc_ndcg_at_10_std": 0.1932531729558633, + "nauc_ndcg_at_1_diff1": 0.6942671543678366, + "nauc_ndcg_at_1_max": 0.5683430205647082, + "nauc_ndcg_at_1_std": 0.11179006846134261, + "nauc_ndcg_at_20_diff1": 0.6211872614610904, + "nauc_ndcg_at_20_max": 0.5530597590465678, + "nauc_ndcg_at_20_std": 0.19883341794091794, + "nauc_ndcg_at_3_diff1": 0.6319328518327908, + "nauc_ndcg_at_3_max": 0.549327389660898, + "nauc_ndcg_at_3_std": 0.16138071986818298, + "nauc_ndcg_at_5_diff1": 0.621779492307214, + "nauc_ndcg_at_5_max": 0.5526865020578606, + "nauc_ndcg_at_5_std": 0.18642976010409754, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5412854653754162, + "nauc_precision_at_100_max": 0.6184196906683425, + "nauc_precision_at_100_std": 0.5350992879255447, + "nauc_precision_at_10_diff1": 0.5393654107724762, + "nauc_precision_at_10_max": 0.5480476304207519, + "nauc_precision_at_10_std": 0.31205880886496185, + "nauc_precision_at_1_diff1": 0.6942671543678366, + "nauc_precision_at_1_max": 0.5683430205647082, + "nauc_precision_at_1_std": 0.11179006846134261, + "nauc_precision_at_20_diff1": 0.5364223430251746, + "nauc_precision_at_20_max": 0.5477578105341474, + "nauc_precision_at_20_std": 0.3729372269068284, + "nauc_precision_at_3_diff1": 0.589803262485684, + "nauc_precision_at_3_max": 0.5381986930768128, + "nauc_precision_at_3_std": 0.20251897425233165, + "nauc_precision_at_5_diff1": 0.559417311850486, + "nauc_precision_at_5_max": 0.5450843125766585, + "nauc_precision_at_5_std": 0.26971020091218756, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5412854653754147, + "nauc_recall_at_100_max": 0.6184196906683408, + "nauc_recall_at_100_std": 0.5350992879255435, + "nauc_recall_at_10_diff1": 0.5393654107724762, + "nauc_recall_at_10_max": 0.5480476304207523, + "nauc_recall_at_10_std": 0.31205880886496196, + "nauc_recall_at_1_diff1": 0.6942671543678366, + "nauc_recall_at_1_max": 0.5683430205647082, + "nauc_recall_at_1_std": 0.11179006846134261, + "nauc_recall_at_20_diff1": 0.5364223430251741, + "nauc_recall_at_20_max": 0.547757810534148, + "nauc_recall_at_20_std": 0.37293722690682907, + "nauc_recall_at_3_diff1": 0.5898032624856843, + "nauc_recall_at_3_max": 0.5381986930768126, + "nauc_recall_at_3_std": 0.20251897425233206, + "nauc_recall_at_5_diff1": 0.5594173118504862, + "nauc_recall_at_5_max": 0.5450843125766591, + "nauc_recall_at_5_std": 0.26971020091218706, + "ndcg_at_1": 0.36616, + "ndcg_at_10": 0.5121, + "ndcg_at_100": 0.55758, + "ndcg_at_1000": 0.57013, + "ndcg_at_20": 0.53012, + "ndcg_at_3": 0.45235, + "ndcg_at_5": 0.47974, + "precision_at_1": 0.36616, + "precision_at_10": 0.0681, + "precision_at_100": 0.00903, + "precision_at_1000": 0.001, + "precision_at_20": 0.03766, + "precision_at_3": 0.17152, + "precision_at_5": 0.11623, + "recall_at_1": 0.36616, + "recall_at_10": 0.681, + "recall_at_100": 0.90291, + "recall_at_1000": 1.0, + "recall_at_20": 0.75312, + "recall_at_3": 0.51456, + "recall_at_5": 0.58114 + }, + { + "cv_recall_at_1": 0.44926, + "cv_recall_at_10": 0.72304, + "cv_recall_at_100": 0.92495, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.78541, + "cv_recall_at_3": 0.59408, + "cv_recall_at_5": 0.65328, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.58154, + "map_at_1": 0.44926, + "map_at_10": 0.53678, + "map_at_100": 0.5448, + "map_at_1000": 0.54521, + "map_at_20": 0.5411, + "map_at_3": 0.51374, + "map_at_5": 0.52733, + "mrr_at_1": 0.4492600422832981, + "mrr_at_10": 0.5367776603241715, + "mrr_at_100": 0.5448034191893545, + "mrr_at_1000": 0.5452066948845224, + "mrr_at_20": 0.5411048749935305, + "mrr_at_3": 0.5137420718816064, + "mrr_at_5": 0.5273255813953482, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5604890359223188, + "nauc_cv_recall_at_100_max": 0.600611877989979, + "nauc_cv_recall_at_100_std": 0.5128565005478855, + "nauc_cv_recall_at_10_diff1": 0.5478845958212548, + "nauc_cv_recall_at_10_max": 0.5569566649688461, + "nauc_cv_recall_at_10_std": 0.2380854206882242, + "nauc_cv_recall_at_1_diff1": 0.7076954521901431, + "nauc_cv_recall_at_1_max": 0.6081804572874958, + "nauc_cv_recall_at_1_std": 0.18855092140607752, + "nauc_cv_recall_at_20_diff1": 0.519804538332009, + "nauc_cv_recall_at_20_max": 0.5699259306994402, + "nauc_cv_recall_at_20_std": 0.3267690220002922, + "nauc_cv_recall_at_3_diff1": 0.6196183890984953, + "nauc_cv_recall_at_3_max": 0.5625131608580892, + "nauc_cv_recall_at_3_std": 0.18026450762612506, + "nauc_cv_recall_at_5_diff1": 0.5837624234626474, + "nauc_cv_recall_at_5_max": 0.5955172595825997, + "nauc_cv_recall_at_5_std": 0.2345211583004191, + "nauc_map_at_1000_diff1": 0.662767027607681, + "nauc_map_at_1000_max": 0.5891614953909932, + "nauc_map_at_1000_std": 0.19271138919076922, + "nauc_map_at_100_diff1": 0.6626694252918228, + "nauc_map_at_100_max": 0.5891853733373635, + "nauc_map_at_100_std": 0.1931273640612953, + "nauc_map_at_10_diff1": 0.6611057630068342, + "nauc_map_at_10_max": 0.5885808385362452, + "nauc_map_at_10_std": 0.19038392512862137, + "nauc_map_at_1_diff1": 0.7076954521901431, + "nauc_map_at_1_max": 0.6081804572874958, + "nauc_map_at_1_std": 0.18855092140607752, + "nauc_map_at_20_diff1": 0.6611052423433593, + "nauc_map_at_20_max": 0.589097186728387, + "nauc_map_at_20_std": 0.19301781252750633, + "nauc_map_at_3_diff1": 0.6698031561319879, + "nauc_map_at_3_max": 0.5864773804656612, + "nauc_map_at_3_std": 0.18035627546412655, + "nauc_map_at_5_diff1": 0.6634047508034983, + "nauc_map_at_5_max": 0.5925821858489451, + "nauc_map_at_5_std": 0.19006959491476397, + "nauc_mrr_at_1000_diff1": 0.662767027607681, + "nauc_mrr_at_1000_max": 0.5891614953909932, + "nauc_mrr_at_1000_std": 0.19271138919076922, + "nauc_mrr_at_100_diff1": 0.6626694252918228, + "nauc_mrr_at_100_max": 0.5891853733373635, + "nauc_mrr_at_100_std": 0.1931273640612953, + "nauc_mrr_at_10_diff1": 0.6611057630068342, + "nauc_mrr_at_10_max": 0.5885808385362452, + "nauc_mrr_at_10_std": 0.19038392512862137, + "nauc_mrr_at_1_diff1": 0.7076954521901431, + "nauc_mrr_at_1_max": 0.6081804572874958, + "nauc_mrr_at_1_std": 0.18855092140607752, + "nauc_mrr_at_20_diff1": 0.6611052423433593, + "nauc_mrr_at_20_max": 0.589097186728387, + "nauc_mrr_at_20_std": 0.19301781252750633, + "nauc_mrr_at_3_diff1": 0.6698031561319879, + "nauc_mrr_at_3_max": 0.5864773804656612, + "nauc_mrr_at_3_std": 0.18035627546412655, + "nauc_mrr_at_5_diff1": 0.6634047508034983, + "nauc_mrr_at_5_max": 0.5925821858489451, + "nauc_mrr_at_5_std": 0.19006959491476397, + "nauc_ndcg_at_1000_diff1": 0.6527791339329123, + "nauc_ndcg_at_1000_max": 0.5866662271791159, + "nauc_ndcg_at_1000_std": 0.2011792752392176, + "nauc_ndcg_at_100_diff1": 0.6496398779949115, + "nauc_ndcg_at_100_max": 0.587267185991599, + "nauc_ndcg_at_100_std": 0.2124728126641152, + "nauc_ndcg_at_10_diff1": 0.6389497640400255, + "nauc_ndcg_at_10_max": 0.5823909084092497, + "nauc_ndcg_at_10_std": 0.1992511948037301, + "nauc_ndcg_at_1_diff1": 0.7076954521901431, + "nauc_ndcg_at_1_max": 0.6081804572874958, + "nauc_ndcg_at_1_std": 0.18855092140607752, + "nauc_ndcg_at_20_diff1": 0.6378501985977086, + "nauc_ndcg_at_20_max": 0.5850990776613686, + "nauc_ndcg_at_20_std": 0.2118455632925891, + "nauc_ndcg_at_3_diff1": 0.6574992976045798, + "nauc_ndcg_at_3_max": 0.5805105572521726, + "nauc_ndcg_at_3_std": 0.18012778697797263, + "nauc_ndcg_at_5_diff1": 0.6454889254857838, + "nauc_ndcg_at_5_max": 0.5925209317588707, + "nauc_ndcg_at_5_std": 0.19914914490304186, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5604890359223192, + "nauc_precision_at_100_max": 0.6006118779899806, + "nauc_precision_at_100_std": 0.5128565005478841, + "nauc_precision_at_10_diff1": 0.5478845958212546, + "nauc_precision_at_10_max": 0.5569566649688457, + "nauc_precision_at_10_std": 0.23808542068822366, + "nauc_precision_at_1_diff1": 0.7076954521901431, + "nauc_precision_at_1_max": 0.6081804572874958, + "nauc_precision_at_1_std": 0.18855092140607752, + "nauc_precision_at_20_diff1": 0.5198045383320089, + "nauc_precision_at_20_max": 0.5699259306994408, + "nauc_precision_at_20_std": 0.32676902200029057, + "nauc_precision_at_3_diff1": 0.6196183890984959, + "nauc_precision_at_3_max": 0.5625131608580891, + "nauc_precision_at_3_std": 0.18026450762612492, + "nauc_precision_at_5_diff1": 0.5837624234626474, + "nauc_precision_at_5_max": 0.5955172595825996, + "nauc_precision_at_5_std": 0.23452115830041848, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5604890359223188, + "nauc_recall_at_100_max": 0.600611877989979, + "nauc_recall_at_100_std": 0.5128565005478855, + "nauc_recall_at_10_diff1": 0.5478845958212548, + "nauc_recall_at_10_max": 0.5569566649688461, + "nauc_recall_at_10_std": 0.2380854206882242, + "nauc_recall_at_1_diff1": 0.7076954521901431, + "nauc_recall_at_1_max": 0.6081804572874958, + "nauc_recall_at_1_std": 0.18855092140607752, + "nauc_recall_at_20_diff1": 0.519804538332009, + "nauc_recall_at_20_max": 0.5699259306994402, + "nauc_recall_at_20_std": 0.3267690220002922, + "nauc_recall_at_3_diff1": 0.6196183890984953, + "nauc_recall_at_3_max": 0.5625131608580892, + "nauc_recall_at_3_std": 0.18026450762612506, + "nauc_recall_at_5_diff1": 0.5837624234626474, + "nauc_recall_at_5_max": 0.5955172595825997, + "nauc_recall_at_5_std": 0.2345211583004191, + "ndcg_at_1": 0.44926, + "ndcg_at_10": 0.58154, + "ndcg_at_100": 0.62338, + "ndcg_at_1000": 0.63315, + "ndcg_at_20": 0.59731, + "ndcg_at_3": 0.5344, + "ndcg_at_5": 0.55883, + "precision_at_1": 0.44926, + "precision_at_10": 0.0723, + "precision_at_100": 0.00925, + "precision_at_1000": 0.001, + "precision_at_20": 0.03927, + "precision_at_3": 0.19803, + "precision_at_5": 0.13066, + "recall_at_1": 0.44926, + "recall_at_10": 0.72304, + "recall_at_100": 0.92495, + "recall_at_1000": 1.0, + "recall_at_20": 0.78541, + "recall_at_3": 0.59408, + "recall_at_5": 0.65328 + }, + { + "cv_recall_at_1": 0.517, + "cv_recall_at_10": 0.781, + "cv_recall_at_100": 0.971, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.827, + "cv_recall_at_3": 0.659, + "cv_recall_at_5": 0.722, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.64447, + "map_at_1": 0.517, + "map_at_10": 0.60113, + "map_at_100": 0.608, + "map_at_1000": 0.60816, + "map_at_20": 0.60429, + "map_at_3": 0.57867, + "map_at_5": 0.59307, + "mrr_at_1": 0.517, + "mrr_at_10": 0.6011313492063491, + "mrr_at_100": 0.6079956742603368, + "mrr_at_1000": 0.6081601475225663, + "mrr_at_20": 0.6042893008694089, + "mrr_at_3": 0.5786666666666664, + "mrr_at_5": 0.5930666666666664, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.2455004990501933, + "nauc_cv_recall_at_100_max": 0.777117743649181, + "nauc_cv_recall_at_100_std": 0.580137802247331, + "nauc_cv_recall_at_10_diff1": 0.6447662014039314, + "nauc_cv_recall_at_10_max": 0.09860095441293794, + "nauc_cv_recall_at_10_std": -0.5034772553298917, + "nauc_cv_recall_at_1_diff1": 0.751608559639086, + "nauc_cv_recall_at_1_max": 0.312891122377997, + "nauc_cv_recall_at_1_std": -0.21080843597152954, + "nauc_cv_recall_at_20_diff1": 0.6256890376607283, + "nauc_cv_recall_at_20_max": 0.13114136053877856, + "nauc_cv_recall_at_20_std": -0.4660522212003668, + "nauc_cv_recall_at_3_diff1": 0.6906157507780316, + "nauc_cv_recall_at_3_max": 0.23405883395415814, + "nauc_cv_recall_at_3_std": -0.35375070184359253, + "nauc_cv_recall_at_5_diff1": 0.6616474688427189, + "nauc_cv_recall_at_5_max": 0.15739656780522973, + "nauc_cv_recall_at_5_std": -0.42995129215876704, + "nauc_map_at_1000_diff1": 0.7218586514551526, + "nauc_map_at_1000_max": 0.276378975392984, + "nauc_map_at_1000_std": -0.2741734105052388, + "nauc_map_at_100_diff1": 0.7215779428420441, + "nauc_map_at_100_max": 0.2766641973079267, + "nauc_map_at_100_std": -0.27366533367406815, + "nauc_map_at_10_diff1": 0.7217438447391757, + "nauc_map_at_10_max": 0.2703046067353741, + "nauc_map_at_10_std": -0.2820292800514403, + "nauc_map_at_1_diff1": 0.751608559639086, + "nauc_map_at_1_max": 0.312891122377997, + "nauc_map_at_1_std": -0.21080843597152954, + "nauc_map_at_20_diff1": 0.7217143483142994, + "nauc_map_at_20_max": 0.27281254198769883, + "nauc_map_at_20_std": -0.2790188913967356, + "nauc_map_at_3_diff1": 0.7261837934787818, + "nauc_map_at_3_max": 0.2859687992340636, + "nauc_map_at_3_std": -0.2664707654182384, + "nauc_map_at_5_diff1": 0.7220012985109051, + "nauc_map_at_5_max": 0.2736003653205686, + "nauc_map_at_5_std": -0.2780137338090851, + "nauc_mrr_at_1000_diff1": 0.7218586514551526, + "nauc_mrr_at_1000_max": 0.276378975392984, + "nauc_mrr_at_1000_std": -0.2741734105052388, + "nauc_mrr_at_100_diff1": 0.7215779428420441, + "nauc_mrr_at_100_max": 0.2766641973079267, + "nauc_mrr_at_100_std": -0.27366533367406815, + "nauc_mrr_at_10_diff1": 0.7217438447391757, + "nauc_mrr_at_10_max": 0.2703046067353741, + "nauc_mrr_at_10_std": -0.2820292800514403, + "nauc_mrr_at_1_diff1": 0.751608559639086, + "nauc_mrr_at_1_max": 0.312891122377997, + "nauc_mrr_at_1_std": -0.21080843597152954, + "nauc_mrr_at_20_diff1": 0.7217143483142994, + "nauc_mrr_at_20_max": 0.27281254198769883, + "nauc_mrr_at_20_std": -0.2790188913967356, + "nauc_mrr_at_3_diff1": 0.7261837934787818, + "nauc_mrr_at_3_max": 0.2859687992340636, + "nauc_mrr_at_3_std": -0.2664707654182384, + "nauc_mrr_at_5_diff1": 0.7220012985109051, + "nauc_mrr_at_5_max": 0.2736003653205686, + "nauc_mrr_at_5_std": -0.2780137338090851, + "nauc_ndcg_at_1000_diff1": 0.7137999194741065, + "nauc_ndcg_at_1000_max": 0.2702799148084682, + "nauc_ndcg_at_1000_std": -0.28243336701775806, + "nauc_ndcg_at_100_diff1": 0.7063295481485343, + "nauc_ndcg_at_100_max": 0.27827257821700485, + "nauc_ndcg_at_100_std": -0.26866795728820436, + "nauc_ndcg_at_10_diff1": 0.7078987305730228, + "nauc_ndcg_at_10_max": 0.2404606593292584, + "nauc_ndcg_at_10_std": -0.32119201147324616, + "nauc_ndcg_at_1_diff1": 0.751608559639086, + "nauc_ndcg_at_1_max": 0.312891122377997, + "nauc_ndcg_at_1_std": -0.21080843597152954, + "nauc_ndcg_at_20_diff1": 0.7072336259985014, + "nauc_ndcg_at_20_max": 0.25013100556604273, + "nauc_ndcg_at_20_std": -0.30950587073132424, + "nauc_ndcg_at_3_diff1": 0.7177845158741234, + "nauc_ndcg_at_3_max": 0.2739795946205384, + "nauc_ndcg_at_3_std": -0.28690383831825556, + "nauc_ndcg_at_5_diff1": 0.7093666031080256, + "nauc_ndcg_at_5_max": 0.24987529421961602, + "nauc_ndcg_at_5_std": -0.3096330840540708, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.24550049905018675, + "nauc_precision_at_100_max": 0.7771177436491825, + "nauc_precision_at_100_std": 0.5801378022473334, + "nauc_precision_at_10_diff1": 0.6447662014039321, + "nauc_precision_at_10_max": 0.09860095441293745, + "nauc_precision_at_10_std": -0.5034772553298922, + "nauc_precision_at_1_diff1": 0.751608559639086, + "nauc_precision_at_1_max": 0.312891122377997, + "nauc_precision_at_1_std": -0.21080843597152954, + "nauc_precision_at_20_diff1": 0.6256890376607267, + "nauc_precision_at_20_max": 0.13114136053877531, + "nauc_precision_at_20_std": -0.46605222120036965, + "nauc_precision_at_3_diff1": 0.6906157507780325, + "nauc_precision_at_3_max": 0.23405883395415797, + "nauc_precision_at_3_std": -0.3537507018435914, + "nauc_precision_at_5_diff1": 0.661647468842718, + "nauc_precision_at_5_max": 0.15739656780522981, + "nauc_precision_at_5_std": -0.42995129215876715, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2455004990501933, + "nauc_recall_at_100_max": 0.777117743649181, + "nauc_recall_at_100_std": 0.580137802247331, + "nauc_recall_at_10_diff1": 0.6447662014039314, + "nauc_recall_at_10_max": 0.09860095441293794, + "nauc_recall_at_10_std": -0.5034772553298917, + "nauc_recall_at_1_diff1": 0.751608559639086, + "nauc_recall_at_1_max": 0.312891122377997, + "nauc_recall_at_1_std": -0.21080843597152954, + "nauc_recall_at_20_diff1": 0.6256890376607283, + "nauc_recall_at_20_max": 0.13114136053877856, + "nauc_recall_at_20_std": -0.4660522212003668, + "nauc_recall_at_3_diff1": 0.6906157507780316, + "nauc_recall_at_3_max": 0.23405883395415814, + "nauc_recall_at_3_std": -0.35375070184359253, + "nauc_recall_at_5_diff1": 0.6616474688427189, + "nauc_recall_at_5_max": 0.15739656780522973, + "nauc_recall_at_5_std": -0.42995129215876704, + "ndcg_at_1": 0.517, + "ndcg_at_10": 0.64447, + "ndcg_at_100": 0.68284, + "ndcg_at_1000": 0.68668, + "ndcg_at_20": 0.65606, + "ndcg_at_3": 0.59926, + "ndcg_at_5": 0.62521, + "precision_at_1": 0.517, + "precision_at_10": 0.0781, + "precision_at_100": 0.00971, + "precision_at_1000": 0.001, + "precision_at_20": 0.04135, + "precision_at_3": 0.21967, + "precision_at_5": 0.1444, + "recall_at_1": 0.517, + "recall_at_10": 0.781, + "recall_at_100": 0.971, + "recall_at_1000": 1.0, + "recall_at_20": 0.827, + "recall_at_3": 0.659, + "recall_at_5": 0.722 + } + ] + }, + "task_name": "WITT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/WebQAT2ITRetrieval.json b/results-mieb/voyage-multimodal-3/1/WebQAT2ITRetrieval.json new file mode 100644 index 0000000000..bc7a9a6655 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/WebQAT2ITRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "53db4c9f9c93cb74926a1c9d04dea7d7acac2f21", + "evaluation_time": 53264.06916642189, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.51971, + "cv_recall_at_10": 0.88371, + "cv_recall_at_100": 0.98606, + "cv_recall_at_1000": 0.99801, + "cv_recall_at_20": 0.93986, + "cv_recall_at_3": 0.71446, + "cv_recall_at_5": 0.7965, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.65641, + "map_at_1": 0.41219, + "map_at_10": 0.5806, + "map_at_100": 0.59162, + "map_at_1000": 0.59181, + "map_at_20": 0.58823, + "map_at_3": 0.53322, + "map_at_5": 0.5607, + "mrr_at_1": 0.5197132616487455, + "mrr_at_10": 0.6376906689929384, + "mrr_at_100": 0.6430317621020902, + "mrr_at_1000": 0.6430981364740875, + "mrr_at_20": 0.6417126087487943, + "mrr_at_3": 0.6069958847736621, + "mrr_at_5": 0.6257334395327226, + "nauc_cv_recall_at_1000_diff1": -0.16359871061056505, + "nauc_cv_recall_at_1000_max": 0.9738573659551857, + "nauc_cv_recall_at_1000_std": 0.8716355426212542, + "nauc_cv_recall_at_100_diff1": 0.37040443651434385, + "nauc_cv_recall_at_100_max": 0.5536562443724028, + "nauc_cv_recall_at_100_std": -0.2029133108392488, + "nauc_cv_recall_at_10_diff1": 0.4373814649783142, + "nauc_cv_recall_at_10_max": 0.2623278235230551, + "nauc_cv_recall_at_10_std": -0.4777458069030308, + "nauc_cv_recall_at_1_diff1": 0.6296700135608259, + "nauc_cv_recall_at_1_max": 0.18595989336420365, + "nauc_cv_recall_at_1_std": -0.308000860424921, + "nauc_cv_recall_at_20_diff1": 0.44396751777858534, + "nauc_cv_recall_at_20_max": 0.37139002202609384, + "nauc_cv_recall_at_20_std": -0.46416365148674804, + "nauc_cv_recall_at_3_diff1": 0.5250228583228617, + "nauc_cv_recall_at_3_max": 0.21452132324025366, + "nauc_cv_recall_at_3_std": -0.376613676516204, + "nauc_cv_recall_at_5_diff1": 0.48969921934185057, + "nauc_cv_recall_at_5_max": 0.2201012342193106, + "nauc_cv_recall_at_5_std": -0.3966907506361567, + "nauc_map_at_1000_diff1": 0.5606927411639652, + "nauc_map_at_1000_max": 0.23695502946687824, + "nauc_map_at_1000_std": -0.3219725297843542, + "nauc_map_at_100_diff1": 0.560601290642645, + "nauc_map_at_100_max": 0.2369761010127848, + "nauc_map_at_100_std": -0.322177283851876, + "nauc_map_at_10_diff1": 0.557362556735293, + "nauc_map_at_10_max": 0.23837454920688683, + "nauc_map_at_10_std": -0.32505856270238065, + "nauc_map_at_1_diff1": 0.5956803415962526, + "nauc_map_at_1_max": 0.2511508848721171, + "nauc_map_at_1_std": -0.2533533014832028, + "nauc_map_at_20_diff1": 0.5593378056108329, + "nauc_map_at_20_max": 0.2378486279109965, + "nauc_map_at_20_std": -0.3236293166425821, + "nauc_map_at_3_diff1": 0.5639181501834886, + "nauc_map_at_3_max": 0.24187972587351358, + "nauc_map_at_3_std": -0.31665507581977087, + "nauc_map_at_5_diff1": 0.5580745238384635, + "nauc_map_at_5_max": 0.23962606930376834, + "nauc_map_at_5_std": -0.31997561410739733, + "nauc_mrr_at_1000_diff1": 0.5885577140835274, + "nauc_mrr_at_1000_max": 0.19740465284088993, + "nauc_mrr_at_1000_std": -0.33687489687989824, + "nauc_mrr_at_100_diff1": 0.5885256709976912, + "nauc_mrr_at_100_max": 0.19746031034257694, + "nauc_mrr_at_100_std": -0.3369123512013644, + "nauc_mrr_at_10_diff1": 0.5857089403879366, + "nauc_mrr_at_10_max": 0.1971397736486086, + "nauc_mrr_at_10_std": -0.3400377337570375, + "nauc_mrr_at_1_diff1": 0.6296700135608259, + "nauc_mrr_at_1_max": 0.18595989336420365, + "nauc_mrr_at_1_std": -0.308000860424921, + "nauc_mrr_at_20_diff1": 0.5878546914502075, + "nauc_mrr_at_20_max": 0.1979369859133071, + "nauc_mrr_at_20_std": -0.33782493585182194, + "nauc_mrr_at_3_diff1": 0.5876131256705309, + "nauc_mrr_at_3_max": 0.1952677167723346, + "nauc_mrr_at_3_std": -0.33690000733845876, + "nauc_mrr_at_5_diff1": 0.5848840257463417, + "nauc_mrr_at_5_max": 0.19573848094590507, + "nauc_mrr_at_5_std": -0.33754136097831156, + "nauc_ndcg_at_1000_diff1": 0.5578263978615521, + "nauc_ndcg_at_1000_max": 0.2287106730052526, + "nauc_ndcg_at_1000_std": -0.32790750604660857, + "nauc_ndcg_at_100_diff1": 0.5553967772122925, + "nauc_ndcg_at_100_max": 0.23045027433227894, + "nauc_ndcg_at_100_std": -0.33087731960853317, + "nauc_ndcg_at_10_diff1": 0.5423985101785965, + "nauc_ndcg_at_10_max": 0.23676469280404897, + "nauc_ndcg_at_10_std": -0.34577355251853725, + "nauc_ndcg_at_1_diff1": 0.6206472732725669, + "nauc_ndcg_at_1_max": 0.1884184377108565, + "nauc_ndcg_at_1_std": -0.30647209320694563, + "nauc_ndcg_at_20_diff1": 0.5488316930039923, + "nauc_ndcg_at_20_max": 0.23729811515756516, + "nauc_ndcg_at_20_std": -0.3397743302343221, + "nauc_ndcg_at_3_diff1": 0.5554620170205623, + "nauc_ndcg_at_3_max": 0.23676777748450337, + "nauc_ndcg_at_3_std": -0.32826473567801495, + "nauc_ndcg_at_5_diff1": 0.5453608942107281, + "nauc_ndcg_at_5_max": 0.2362765393024237, + "nauc_ndcg_at_5_std": -0.33441704181011644, + "nauc_precision_at_1000_diff1": -0.2490286905418845, + "nauc_precision_at_1000_max": -0.2823172572380373, + "nauc_precision_at_1000_std": 0.06940935099459479, + "nauc_precision_at_100_diff1": -0.20322438082025343, + "nauc_precision_at_100_max": -0.23944899930888527, + "nauc_precision_at_100_std": 0.014691925313461496, + "nauc_precision_at_10_diff1": 0.04685689049403343, + "nauc_precision_at_10_max": -0.048833759034091796, + "nauc_precision_at_10_std": -0.19287228031286877, + "nauc_precision_at_1_diff1": 0.6206472732725669, + "nauc_precision_at_1_max": 0.1884184377108565, + "nauc_precision_at_1_std": -0.30647209320694563, + "nauc_precision_at_20_diff1": -0.059356183920344925, + "nauc_precision_at_20_max": -0.11647543513941302, + "nauc_precision_at_20_std": -0.11040894004972752, + "nauc_precision_at_3_diff1": 0.2963034493624533, + "nauc_precision_at_3_max": 0.06193015454029306, + "nauc_precision_at_3_std": -0.2847934074807809, + "nauc_precision_at_5_diff1": 0.1818704510358003, + "nauc_precision_at_5_max": 0.012838216995005054, + "nauc_precision_at_5_std": -0.23678360333260248, + "nauc_recall_at_1000_diff1": -0.031658167426407764, + "nauc_recall_at_1000_max": 0.761294904963372, + "nauc_recall_at_1000_std": 0.589836641693236, + "nauc_recall_at_100_diff1": 0.30495269315830603, + "nauc_recall_at_100_max": 0.40319937864466737, + "nauc_recall_at_100_std": -0.3076823635453479, + "nauc_recall_at_10_diff1": 0.40320059333569697, + "nauc_recall_at_10_max": 0.2820181081363864, + "nauc_recall_at_10_std": -0.4051693238202271, + "nauc_recall_at_1_diff1": 0.5956803415962526, + "nauc_recall_at_1_max": 0.2511508848721171, + "nauc_recall_at_1_std": -0.2533533014832028, + "nauc_recall_at_20_diff1": 0.38601321807115213, + "nauc_recall_at_20_max": 0.33457449244668686, + "nauc_recall_at_20_std": -0.4021984802798079, + "nauc_recall_at_3_diff1": 0.49455357752002116, + "nauc_recall_at_3_max": 0.254980097750559, + "nauc_recall_at_3_std": -0.33207921509986293, + "nauc_recall_at_5_diff1": 0.4489604939387345, + "nauc_recall_at_5_max": 0.2583193566941697, + "nauc_recall_at_5_std": -0.34658019377545485, + "ndcg_at_1": 0.5237, + "ndcg_at_10": 0.65641, + "ndcg_at_100": 0.69529, + "ndcg_at_1000": 0.69876, + "ndcg_at_20": 0.67833, + "ndcg_at_3": 0.57495, + "ndcg_at_5": 0.61754, + "precision_at_1": 0.5237, + "precision_at_10": 0.11489, + "precision_at_100": 0.014, + "precision_at_1000": 0.00144, + "precision_at_20": 0.06356, + "precision_at_3": 0.28169, + "precision_at_5": 0.19681, + "recall_at_1": 0.41219, + "recall_at_10": 0.82059, + "recall_at_100": 0.97292, + "recall_at_1000": 0.99642, + "recall_at_20": 0.89626, + "recall_at_3": 0.62186, + "recall_at_5": 0.71505 + } + ] + }, + "task_name": "WebQAT2ITRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/WebQAT2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/WebQAT2TRetrieval.json new file mode 100644 index 0000000000..5a5b13bc1d --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/WebQAT2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "468b42a2b2e767d80d2d93f5ae5d42f135a10478", + "evaluation_time": 2916.9777975082397, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.589, + "cv_recall_at_10": 0.92912, + "cv_recall_at_100": 0.99267, + "cv_recall_at_1000": 0.99959, + "cv_recall_at_20": 0.96375, + "cv_recall_at_3": 0.8167, + "cv_recall_at_5": 0.87413, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6262, + "map_at_1": 0.29219, + "map_at_10": 0.51981, + "map_at_100": 0.53586, + "map_at_1000": 0.53636, + "map_at_20": 0.53029, + "map_at_3": 0.45758, + "map_at_5": 0.49438, + "mrr_at_1": 0.5890020366598778, + "mrr_at_10": 0.7122235153396038, + "mrr_at_100": 0.7155789027275217, + "mrr_at_1000": 0.7156207060411495, + "mrr_at_20": 0.7146892001931302, + "mrr_at_3": 0.6912423625254568, + "mrr_at_5": 0.7045417515274939, + "nauc_cv_recall_at_1000_diff1": -1.741603926146025, + "nauc_cv_recall_at_1000_max": -1.741603926146025, + "nauc_cv_recall_at_1000_std": -0.17150958406100894, + "nauc_cv_recall_at_100_diff1": 0.32561412897936853, + "nauc_cv_recall_at_100_max": 0.46515820318395235, + "nauc_cv_recall_at_100_std": 0.3833298186528627, + "nauc_cv_recall_at_10_diff1": 0.4575429865422298, + "nauc_cv_recall_at_10_max": 0.515849278224097, + "nauc_cv_recall_at_10_std": 0.04406300734960047, + "nauc_cv_recall_at_1_diff1": 0.4966656191251561, + "nauc_cv_recall_at_1_max": 0.27772683782666246, + "nauc_cv_recall_at_1_std": -0.06175533593483985, + "nauc_cv_recall_at_20_diff1": 0.5055617524351645, + "nauc_cv_recall_at_20_max": 0.5636112160370098, + "nauc_cv_recall_at_20_std": 0.14768827018120337, + "nauc_cv_recall_at_3_diff1": 0.4427478637599672, + "nauc_cv_recall_at_3_max": 0.37453726047706076, + "nauc_cv_recall_at_3_std": -0.10573679542784171, + "nauc_cv_recall_at_5_diff1": 0.467397733168413, + "nauc_cv_recall_at_5_max": 0.44894693050871626, + "nauc_cv_recall_at_5_std": -0.07286478058455781, + "nauc_map_at_1000_diff1": 0.29202796941658243, + "nauc_map_at_1000_max": 0.25787082263263295, + "nauc_map_at_1000_std": -0.03977740340077741, + "nauc_map_at_100_diff1": 0.29188967264538324, + "nauc_map_at_100_max": 0.25805786714594836, + "nauc_map_at_100_std": -0.039356656861644475, + "nauc_map_at_10_diff1": 0.28739222005560244, + "nauc_map_at_10_max": 0.2564964868236401, + "nauc_map_at_10_std": -0.046076737174336496, + "nauc_map_at_1_diff1": 0.49075731029230757, + "nauc_map_at_1_max": 0.27779701957024744, + "nauc_map_at_1_std": -0.05748861166572812, + "nauc_map_at_20_diff1": 0.2902754739335047, + "nauc_map_at_20_max": 0.25793116771651126, + "nauc_map_at_20_std": -0.04153755449200511, + "nauc_map_at_3_diff1": 0.27680439152954867, + "nauc_map_at_3_max": 0.22708615796427953, + "nauc_map_at_3_std": -0.06030884464319331, + "nauc_map_at_5_diff1": 0.2819460660292493, + "nauc_map_at_5_max": 0.24510310548741399, + "nauc_map_at_5_std": -0.05476259007222099, + "nauc_mrr_at_1000_diff1": 0.4851985065498464, + "nauc_mrr_at_1000_max": 0.30846768529245516, + "nauc_mrr_at_1000_std": -0.06793412217064555, + "nauc_mrr_at_100_diff1": 0.48518671596935053, + "nauc_mrr_at_100_max": 0.30851404606767807, + "nauc_mrr_at_100_std": -0.06786147285634107, + "nauc_mrr_at_10_diff1": 0.48435566911818745, + "nauc_mrr_at_10_max": 0.31096037838142404, + "nauc_mrr_at_10_std": -0.06727771912167632, + "nauc_mrr_at_1_diff1": 0.4966656191251561, + "nauc_mrr_at_1_max": 0.27772683782666246, + "nauc_mrr_at_1_std": -0.06175533593483985, + "nauc_mrr_at_20_diff1": 0.4853842400252949, + "nauc_mrr_at_20_max": 0.3094884777639313, + "nauc_mrr_at_20_std": -0.06731324685156856, + "nauc_mrr_at_3_diff1": 0.47912398040972315, + "nauc_mrr_at_3_max": 0.3067047656670131, + "nauc_mrr_at_3_std": -0.07596845472211204, + "nauc_mrr_at_5_diff1": 0.48407057547709265, + "nauc_mrr_at_5_max": 0.31223132247632523, + "nauc_mrr_at_5_std": -0.07178600314134528, + "nauc_ndcg_at_1000_diff1": 0.3371883578414304, + "nauc_ndcg_at_1000_max": 0.2850891815328272, + "nauc_ndcg_at_1000_std": -0.0260621294242751, + "nauc_ndcg_at_100_diff1": 0.33162470243382997, + "nauc_ndcg_at_100_max": 0.2896799083593092, + "nauc_ndcg_at_100_std": -0.013478968978344635, + "nauc_ndcg_at_10_diff1": 0.3165447269095804, + "nauc_ndcg_at_10_max": 0.2891288151784892, + "nauc_ndcg_at_10_std": -0.03508962570767255, + "nauc_ndcg_at_1_diff1": 0.4966656191251561, + "nauc_ndcg_at_1_max": 0.27772683782666246, + "nauc_ndcg_at_1_std": -0.06175533593483985, + "nauc_ndcg_at_20_diff1": 0.32393047592260726, + "nauc_ndcg_at_20_max": 0.2915271314676446, + "nauc_ndcg_at_20_std": -0.022416010844775933, + "nauc_ndcg_at_3_diff1": 0.3060273038108367, + "nauc_ndcg_at_3_max": 0.2483764803867023, + "nauc_ndcg_at_3_std": -0.06481599681457599, + "nauc_ndcg_at_5_diff1": 0.31042685093662403, + "nauc_ndcg_at_5_max": 0.2712696876035037, + "nauc_ndcg_at_5_std": -0.054821583327499285, + "nauc_precision_at_1000_diff1": -0.05502022885141667, + "nauc_precision_at_1000_max": 0.018828263504736552, + "nauc_precision_at_1000_std": 0.10707652007203783, + "nauc_precision_at_100_diff1": 0.04617918393088877, + "nauc_precision_at_100_max": 0.21887404884172, + "nauc_precision_at_100_std": 0.24637836543125177, + "nauc_precision_at_10_diff1": 0.16899025415585744, + "nauc_precision_at_10_max": 0.27988248388439646, + "nauc_precision_at_10_std": 0.020501892255375364, + "nauc_precision_at_1_diff1": 0.4966656191251561, + "nauc_precision_at_1_max": 0.27772683782666246, + "nauc_precision_at_1_std": -0.06175533593483985, + "nauc_precision_at_20_diff1": 0.1414296996662419, + "nauc_precision_at_20_max": 0.26899190108516663, + "nauc_precision_at_20_std": 0.08692554067249968, + "nauc_precision_at_3_diff1": 0.20830973485598295, + "nauc_precision_at_3_max": 0.2243587062044391, + "nauc_precision_at_3_std": -0.06454460552554597, + "nauc_precision_at_5_diff1": 0.19572312072504147, + "nauc_precision_at_5_max": 0.25367040781044475, + "nauc_precision_at_5_std": -0.041281502254642145, + "nauc_recall_at_1000_diff1": 0.1623570160659502, + "nauc_recall_at_1000_max": 0.33549516337261165, + "nauc_recall_at_1000_std": 0.43557417309518925, + "nauc_recall_at_100_diff1": 0.12776533061990886, + "nauc_recall_at_100_max": 0.33731010803929673, + "nauc_recall_at_100_std": 0.34635643583394, + "nauc_recall_at_10_diff1": 0.18871296693272535, + "nauc_recall_at_10_max": 0.2989969596246417, + "nauc_recall_at_10_std": 0.02753041423599101, + "nauc_recall_at_1_diff1": 0.49075731029230757, + "nauc_recall_at_1_max": 0.27779701957024744, + "nauc_recall_at_1_std": -0.05748861166572812, + "nauc_recall_at_20_diff1": 0.17416536192966864, + "nauc_recall_at_20_max": 0.3061574059562882, + "nauc_recall_at_20_std": 0.10364372354738594, + "nauc_recall_at_3_diff1": 0.21276002628565463, + "nauc_recall_at_3_max": 0.22813299646040489, + "nauc_recall_at_3_std": -0.06009924853869038, + "nauc_recall_at_5_diff1": 0.2030057553831004, + "nauc_recall_at_5_max": 0.25995381724385386, + "nauc_recall_at_5_std": -0.0356346062273783, + "ndcg_at_1": 0.589, + "ndcg_at_10": 0.6262, + "ndcg_at_100": 0.67704, + "ndcg_at_1000": 0.6853, + "ndcg_at_20": 0.65185, + "ndcg_at_3": 0.53576, + "ndcg_at_5": 0.58335, + "precision_at_1": 0.589, + "precision_at_10": 0.14982, + "precision_at_100": 0.01886, + "precision_at_1000": 0.002, + "precision_at_20": 0.08322, + "precision_at_3": 0.36307, + "precision_at_5": 0.25589, + "recall_at_1": 0.29219, + "recall_at_10": 0.73836, + "recall_at_100": 0.9278, + "recall_at_1000": 0.98001, + "recall_at_20": 0.82006, + "recall_at_3": 0.53856, + "recall_at_5": 0.63208 + } + ] + }, + "task_name": "WebQAT2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Winoground.json b/results-mieb/voyage-multimodal-3/1/Winoground.json new file mode 100644 index 0000000000..b008641bfc --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/Winoground.json @@ -0,0 +1,21 @@ +{ + "dataset_revision": "b400e173549071916ad1b3d449293bc8d8b4b763", + "evaluation_time": 1458.447455406189, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.0625, + "hf_subset": "default", + "image_acc": 0.0925000011920929, + "languages": [ + "eng-Latn" + ], + "main_score": 0.0625, + "text_acc": 0.27000001072883606 + } + ] + }, + "task_name": "Winoground" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/XM3600T2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/XM3600T2IRetrieval.json new file mode 100644 index 0000000000..e68e5e1d02 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/XM3600T2IRetrieval.json @@ -0,0 +1,6311 @@ +{ + "dataset_revision": "8d3e5665526c55a5855cd6ddfbaba2032bc7cee4", + "evaluation_time": 37511.4420838356, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.34125, + "cv_recall_at_10": 0.67287, + "cv_recall_at_100": 0.91421, + "cv_recall_at_1000": 0.99525, + "cv_recall_at_20": 0.76164, + "cv_recall_at_3": 0.50821, + "cv_recall_at_5": 0.58681, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.49827, + "map_at_1": 0.34125, + "map_at_10": 0.44329, + "map_at_100": 0.45355, + "map_at_1000": 0.45395, + "map_at_20": 0.44954, + "map_at_3": 0.41378, + "map_at_5": 0.43164, + "mrr_at_1": 0.3412515270802226, + "mrr_at_10": 0.44329409787534, + "mrr_at_100": 0.4535499457119047, + "mrr_at_1000": 0.45395489734902766, + "mrr_at_20": 0.4495370855277087, + "mrr_at_3": 0.41378218180173004, + "mrr_at_5": 0.43163883987149987, + "nauc_cv_recall_at_1000_diff1": 0.4337496830686869, + "nauc_cv_recall_at_1000_max": 0.8412941148071966, + "nauc_cv_recall_at_1000_std": 0.947111521299572, + "nauc_cv_recall_at_100_diff1": 0.3580251362565319, + "nauc_cv_recall_at_100_max": 0.6640640610501624, + "nauc_cv_recall_at_100_std": 0.5675327555710422, + "nauc_cv_recall_at_10_diff1": 0.4196819928283325, + "nauc_cv_recall_at_10_max": 0.4847935456565573, + "nauc_cv_recall_at_10_std": 0.17822153608684996, + "nauc_cv_recall_at_1_diff1": 0.5748737887556816, + "nauc_cv_recall_at_1_max": 0.4046463836442685, + "nauc_cv_recall_at_1_std": 0.03730800730770686, + "nauc_cv_recall_at_20_diff1": 0.3998000878261237, + "nauc_cv_recall_at_20_max": 0.5260038074914021, + "nauc_cv_recall_at_20_std": 0.2575834089930464, + "nauc_cv_recall_at_3_diff1": 0.4671406343027723, + "nauc_cv_recall_at_3_max": 0.43526122324655075, + "nauc_cv_recall_at_3_std": 0.09393815190552963, + "nauc_cv_recall_at_5_diff1": 0.4436903813254511, + "nauc_cv_recall_at_5_max": 0.44982329761040435, + "nauc_cv_recall_at_5_std": 0.1137201501321779, + "nauc_map_at_1000_diff1": 0.5233639336100439, + "nauc_map_at_1000_max": 0.42625971848572086, + "nauc_map_at_1000_std": 0.0741080415578734, + "nauc_map_at_100_diff1": 0.5231536531540483, + "nauc_map_at_100_max": 0.4264913079765343, + "nauc_map_at_100_std": 0.07459475731986948, + "nauc_map_at_10_diff1": 0.5222653578421395, + "nauc_map_at_10_max": 0.42578248644685257, + "nauc_map_at_10_std": 0.0722300615883244, + "nauc_map_at_1_diff1": 0.5748737887556816, + "nauc_map_at_1_max": 0.4046463836442685, + "nauc_map_at_1_std": 0.03730800730770686, + "nauc_map_at_20_diff1": 0.5224573371553266, + "nauc_map_at_20_max": 0.42670066441524435, + "nauc_map_at_20_std": 0.07428043686934636, + "nauc_map_at_3_diff1": 0.5272012215426869, + "nauc_map_at_3_max": 0.41957225459462183, + "nauc_map_at_3_std": 0.062243752690080166, + "nauc_map_at_5_diff1": 0.523499038800135, + "nauc_map_at_5_max": 0.42241837980697394, + "nauc_map_at_5_std": 0.06596759643358041, + "nauc_mrr_at_1000_diff1": 0.5233639336100439, + "nauc_mrr_at_1000_max": 0.42625971848572086, + "nauc_mrr_at_1000_std": 0.0741080415578734, + "nauc_mrr_at_100_diff1": 0.5231536531540483, + "nauc_mrr_at_100_max": 0.4264913079765343, + "nauc_mrr_at_100_std": 0.07459475731986948, + "nauc_mrr_at_10_diff1": 0.5222653578421395, + "nauc_mrr_at_10_max": 0.42578248644685257, + "nauc_mrr_at_10_std": 0.0722300615883244, + "nauc_mrr_at_1_diff1": 0.5748737887556816, + "nauc_mrr_at_1_max": 0.4046463836442685, + "nauc_mrr_at_1_std": 0.03730800730770686, + "nauc_mrr_at_20_diff1": 0.5224573371553266, + "nauc_mrr_at_20_max": 0.42670066441524435, + "nauc_mrr_at_20_std": 0.07428043686934636, + "nauc_mrr_at_3_diff1": 0.5272012215426869, + "nauc_mrr_at_3_max": 0.41957225459462183, + "nauc_mrr_at_3_std": 0.062243752690080166, + "nauc_mrr_at_5_diff1": 0.523499038800135, + "nauc_mrr_at_5_max": 0.42241837980697394, + "nauc_mrr_at_5_std": 0.06596759643358041, + "nauc_ndcg_at_1000_diff1": 0.5121095899758535, + "nauc_ndcg_at_1000_max": 0.4350773199360631, + "nauc_ndcg_at_1000_std": 0.09165177551394452, + "nauc_ndcg_at_100_diff1": 0.5063075594145833, + "nauc_ndcg_at_100_max": 0.44272557540194885, + "nauc_ndcg_at_100_std": 0.10759226879109529, + "nauc_ndcg_at_10_diff1": 0.5005565980631268, + "nauc_ndcg_at_10_max": 0.4376288011730624, + "nauc_ndcg_at_10_std": 0.09344467085604088, + "nauc_ndcg_at_1_diff1": 0.5748737887556816, + "nauc_ndcg_at_1_max": 0.4046463836442685, + "nauc_ndcg_at_1_std": 0.03730800730770686, + "nauc_ndcg_at_20_diff1": 0.5007512854572422, + "nauc_ndcg_at_20_max": 0.44231034997606367, + "nauc_ndcg_at_20_std": 0.10301397573730728, + "nauc_ndcg_at_3_diff1": 0.5118924631476841, + "nauc_ndcg_at_3_max": 0.42363017314963436, + "nauc_ndcg_at_3_std": 0.07031701275239649, + "nauc_ndcg_at_5_diff1": 0.5044496190290501, + "nauc_ndcg_at_5_max": 0.42893401043585694, + "nauc_ndcg_at_5_std": 0.07729988422114781, + "nauc_precision_at_1000_diff1": 0.43374968306866063, + "nauc_precision_at_1000_max": 0.8412941148072187, + "nauc_precision_at_1000_std": 0.9471115212995683, + "nauc_precision_at_100_diff1": 0.35802513625653104, + "nauc_precision_at_100_max": 0.6640640610501625, + "nauc_precision_at_100_std": 0.5675327555710422, + "nauc_precision_at_10_diff1": 0.41968199282833213, + "nauc_precision_at_10_max": 0.48479354565655736, + "nauc_precision_at_10_std": 0.1782215360868503, + "nauc_precision_at_1_diff1": 0.5748737887556816, + "nauc_precision_at_1_max": 0.4046463836442685, + "nauc_precision_at_1_std": 0.03730800730770686, + "nauc_precision_at_20_diff1": 0.3998000878261242, + "nauc_precision_at_20_max": 0.526003807491402, + "nauc_precision_at_20_std": 0.25758340899304577, + "nauc_precision_at_3_diff1": 0.46714063430277264, + "nauc_precision_at_3_max": 0.4352612232465508, + "nauc_precision_at_3_std": 0.0939381519055294, + "nauc_precision_at_5_diff1": 0.44369038132545086, + "nauc_precision_at_5_max": 0.44982329761040407, + "nauc_precision_at_5_std": 0.11372015013217776, + "nauc_recall_at_1000_diff1": 0.4337496830686869, + "nauc_recall_at_1000_max": 0.8412941148071966, + "nauc_recall_at_1000_std": 0.947111521299572, + "nauc_recall_at_100_diff1": 0.3580251362565319, + "nauc_recall_at_100_max": 0.6640640610501624, + "nauc_recall_at_100_std": 0.5675327555710422, + "nauc_recall_at_10_diff1": 0.4196819928283325, + "nauc_recall_at_10_max": 0.4847935456565573, + "nauc_recall_at_10_std": 0.17822153608684996, + "nauc_recall_at_1_diff1": 0.5748737887556816, + "nauc_recall_at_1_max": 0.4046463836442685, + "nauc_recall_at_1_std": 0.03730800730770686, + "nauc_recall_at_20_diff1": 0.3998000878261237, + "nauc_recall_at_20_max": 0.5260038074914021, + "nauc_recall_at_20_std": 0.2575834089930464, + "nauc_recall_at_3_diff1": 0.4671406343027723, + "nauc_recall_at_3_max": 0.43526122324655075, + "nauc_recall_at_3_std": 0.09393815190552963, + "nauc_recall_at_5_diff1": 0.4436903813254511, + "nauc_recall_at_5_max": 0.44982329761040435, + "nauc_recall_at_5_std": 0.1137201501321779, + "ndcg_at_1": 0.34125, + "ndcg_at_10": 0.49827, + "ndcg_at_100": 0.54933, + "ndcg_at_1000": 0.55972, + "ndcg_at_20": 0.52082, + "ndcg_at_3": 0.43799, + "ndcg_at_5": 0.47027, + "precision_at_1": 0.34125, + "precision_at_10": 0.06729, + "precision_at_100": 0.00914, + "precision_at_1000": 0.001, + "precision_at_20": 0.03808, + "precision_at_3": 0.1694, + "precision_at_5": 0.11736, + "recall_at_1": 0.34125, + "recall_at_10": 0.67287, + "recall_at_100": 0.91421, + "recall_at_1000": 0.99525, + "recall_at_20": 0.76164, + "recall_at_3": 0.50821, + "recall_at_5": 0.58681 + }, + { + "cv_recall_at_1": 0.09139, + "cv_recall_at_10": 0.30556, + "cv_recall_at_100": 0.65111, + "cv_recall_at_1000": 0.94139, + "cv_recall_at_20": 0.40417, + "cv_recall_at_3": 0.175, + "cv_recall_at_5": 0.2225, + "hf_subset": "bn", + "languages": [ + "ben-Beng" + ], + "main_score": 0.18545, + "map_at_1": 0.09139, + "map_at_10": 0.14866, + "map_at_100": 0.16133, + "map_at_1000": 0.16255, + "map_at_20": 0.15547, + "map_at_3": 0.12685, + "map_at_5": 0.13764, + "mrr_at_1": 0.0913888888888889, + "mrr_at_10": 0.14865861992945317, + "mrr_at_100": 0.16133118432056448, + "mrr_at_1000": 0.16254998543628785, + "mrr_at_20": 0.15547410346433166, + "mrr_at_3": 0.12685185185185158, + "mrr_at_5": 0.13764351851851805, + "nauc_cv_recall_at_1000_diff1": 0.1463176107725881, + "nauc_cv_recall_at_1000_max": 0.432051809665412, + "nauc_cv_recall_at_1000_std": 0.6566857390665582, + "nauc_cv_recall_at_100_diff1": 0.09256493988960171, + "nauc_cv_recall_at_100_max": 0.31923931441163583, + "nauc_cv_recall_at_100_std": 0.43131748148846194, + "nauc_cv_recall_at_10_diff1": 0.14427143766141398, + "nauc_cv_recall_at_10_max": 0.23871243028783165, + "nauc_cv_recall_at_10_std": 0.23834337687614615, + "nauc_cv_recall_at_1_diff1": 0.2843768018313742, + "nauc_cv_recall_at_1_max": 0.22642949706528298, + "nauc_cv_recall_at_1_std": 0.10465485168200821, + "nauc_cv_recall_at_20_diff1": 0.12648614968120367, + "nauc_cv_recall_at_20_max": 0.25141099395352257, + "nauc_cv_recall_at_20_std": 0.2806707883697071, + "nauc_cv_recall_at_3_diff1": 0.165651630383353, + "nauc_cv_recall_at_3_max": 0.2255198450052544, + "nauc_cv_recall_at_3_std": 0.16683700150910694, + "nauc_cv_recall_at_5_diff1": 0.14574711280727373, + "nauc_cv_recall_at_5_max": 0.22042896416792016, + "nauc_cv_recall_at_5_std": 0.19751873920296673, + "nauc_map_at_1000_diff1": 0.20534403160355588, + "nauc_map_at_1000_max": 0.2315760856254332, + "nauc_map_at_1000_std": 0.1682451290346834, + "nauc_map_at_100_diff1": 0.2049094254754938, + "nauc_map_at_100_max": 0.2316968894689101, + "nauc_map_at_100_std": 0.16869208240463784, + "nauc_map_at_10_diff1": 0.20802098159538998, + "nauc_map_at_10_max": 0.22787432381186667, + "nauc_map_at_10_std": 0.1587117311493885, + "nauc_map_at_1_diff1": 0.2843768018313742, + "nauc_map_at_1_max": 0.22642949706528298, + "nauc_map_at_1_std": 0.10465485168200821, + "nauc_map_at_20_diff1": 0.20541601683277563, + "nauc_map_at_20_max": 0.22918516814373516, + "nauc_map_at_20_std": 0.16411208132577196, + "nauc_map_at_3_diff1": 0.21836768475010676, + "nauc_map_at_3_max": 0.2269940196497221, + "nauc_map_at_3_std": 0.13857795813754806, + "nauc_map_at_5_diff1": 0.21000082961031727, + "nauc_map_at_5_max": 0.22530968187638267, + "nauc_map_at_5_std": 0.14928697575884342, + "nauc_mrr_at_1000_diff1": 0.20534403160355588, + "nauc_mrr_at_1000_max": 0.2315760856254332, + "nauc_mrr_at_1000_std": 0.1682451290346834, + "nauc_mrr_at_100_diff1": 0.2049094254754938, + "nauc_mrr_at_100_max": 0.2316968894689101, + "nauc_mrr_at_100_std": 0.16869208240463784, + "nauc_mrr_at_10_diff1": 0.20802098159538998, + "nauc_mrr_at_10_max": 0.22787432381186667, + "nauc_mrr_at_10_std": 0.1587117311493885, + "nauc_mrr_at_1_diff1": 0.2843768018313742, + "nauc_mrr_at_1_max": 0.22642949706528298, + "nauc_mrr_at_1_std": 0.10465485168200821, + "nauc_mrr_at_20_diff1": 0.20541601683277563, + "nauc_mrr_at_20_max": 0.22918516814373516, + "nauc_mrr_at_20_std": 0.16411208132577196, + "nauc_mrr_at_3_diff1": 0.21836768475010676, + "nauc_mrr_at_3_max": 0.2269940196497221, + "nauc_mrr_at_3_std": 0.13857795813754806, + "nauc_mrr_at_5_diff1": 0.21000082961031727, + "nauc_mrr_at_5_max": 0.22530968187638267, + "nauc_mrr_at_5_std": 0.14928697575884342, + "nauc_ndcg_at_1000_diff1": 0.18662594605709457, + "nauc_ndcg_at_1000_max": 0.2457099584894618, + "nauc_ndcg_at_1000_std": 0.21375918674731395, + "nauc_ndcg_at_100_diff1": 0.17441894991358925, + "nauc_ndcg_at_100_max": 0.25014125671538306, + "nauc_ndcg_at_100_std": 0.2301861852193149, + "nauc_ndcg_at_10_diff1": 0.1862184690915828, + "nauc_ndcg_at_10_max": 0.23081719080160418, + "nauc_ndcg_at_10_std": 0.1842803191599018, + "nauc_ndcg_at_1_diff1": 0.2843768018313742, + "nauc_ndcg_at_1_max": 0.22642949706528298, + "nauc_ndcg_at_1_std": 0.10465485168200821, + "nauc_ndcg_at_20_diff1": 0.1788396032268059, + "nauc_ndcg_at_20_max": 0.2353158331926509, + "nauc_ndcg_at_20_std": 0.20060602016675325, + "nauc_ndcg_at_3_diff1": 0.2021741614985555, + "nauc_ndcg_at_3_max": 0.22658302834150937, + "nauc_ndcg_at_3_std": 0.14723330002930776, + "nauc_ndcg_at_5_diff1": 0.1893465371489313, + "nauc_ndcg_at_5_max": 0.22383697227093177, + "nauc_ndcg_at_5_std": 0.16449380903854632, + "nauc_precision_at_1000_diff1": 0.14631761077258681, + "nauc_precision_at_1000_max": 0.4320518096654112, + "nauc_precision_at_1000_std": 0.6566857390665592, + "nauc_precision_at_100_diff1": 0.09256493988960134, + "nauc_precision_at_100_max": 0.31923931441163605, + "nauc_precision_at_100_std": 0.431317481488462, + "nauc_precision_at_10_diff1": 0.1442714376614143, + "nauc_precision_at_10_max": 0.23871243028783182, + "nauc_precision_at_10_std": 0.2383433768761465, + "nauc_precision_at_1_diff1": 0.2843768018313742, + "nauc_precision_at_1_max": 0.22642949706528298, + "nauc_precision_at_1_std": 0.10465485168200821, + "nauc_precision_at_20_diff1": 0.1264861496812037, + "nauc_precision_at_20_max": 0.25141099395352234, + "nauc_precision_at_20_std": 0.28067078836970705, + "nauc_precision_at_3_diff1": 0.165651630383353, + "nauc_precision_at_3_max": 0.2255198450052545, + "nauc_precision_at_3_std": 0.16683700150910682, + "nauc_precision_at_5_diff1": 0.1457471128072736, + "nauc_precision_at_5_max": 0.22042896416792016, + "nauc_precision_at_5_std": 0.1975187392029669, + "nauc_recall_at_1000_diff1": 0.1463176107725881, + "nauc_recall_at_1000_max": 0.432051809665412, + "nauc_recall_at_1000_std": 0.6566857390665582, + "nauc_recall_at_100_diff1": 0.09256493988960171, + "nauc_recall_at_100_max": 0.31923931441163583, + "nauc_recall_at_100_std": 0.43131748148846194, + "nauc_recall_at_10_diff1": 0.14427143766141398, + "nauc_recall_at_10_max": 0.23871243028783165, + "nauc_recall_at_10_std": 0.23834337687614615, + "nauc_recall_at_1_diff1": 0.2843768018313742, + "nauc_recall_at_1_max": 0.22642949706528298, + "nauc_recall_at_1_std": 0.10465485168200821, + "nauc_recall_at_20_diff1": 0.12648614968120367, + "nauc_recall_at_20_max": 0.25141099395352257, + "nauc_recall_at_20_std": 0.2806707883697071, + "nauc_recall_at_3_diff1": 0.165651630383353, + "nauc_recall_at_3_max": 0.2255198450052544, + "nauc_recall_at_3_std": 0.16683700150910694, + "nauc_recall_at_5_diff1": 0.14574711280727373, + "nauc_recall_at_5_max": 0.22042896416792016, + "nauc_recall_at_5_std": 0.19751873920296673, + "ndcg_at_1": 0.09139, + "ndcg_at_10": 0.18545, + "ndcg_at_100": 0.25521, + "ndcg_at_1000": 0.29117, + "ndcg_at_20": 0.21035, + "ndcg_at_3": 0.13916, + "ndcg_at_5": 0.15867, + "precision_at_1": 0.09139, + "precision_at_10": 0.03056, + "precision_at_100": 0.00651, + "precision_at_1000": 0.00094, + "precision_at_20": 0.02021, + "precision_at_3": 0.05833, + "precision_at_5": 0.0445, + "recall_at_1": 0.09139, + "recall_at_10": 0.30556, + "recall_at_100": 0.65111, + "recall_at_1000": 0.94139, + "recall_at_20": 0.40417, + "recall_at_3": 0.175, + "recall_at_5": 0.2225 + }, + { + "cv_recall_at_1": 0.35202, + "cv_recall_at_10": 0.69807, + "cv_recall_at_100": 0.91716, + "cv_recall_at_1000": 0.99043, + "cv_recall_at_20": 0.7791, + "cv_recall_at_3": 0.52033, + "cv_recall_at_5": 0.59609, + "hf_subset": "cs", + "languages": [ + "ces-Latn" + ], + "main_score": 0.51423, + "map_at_1": 0.35202, + "map_at_10": 0.45667, + "map_at_100": 0.46613, + "map_at_1000": 0.46649, + "map_at_20": 0.46242, + "map_at_3": 0.4256, + "map_at_5": 0.44292, + "mrr_at_1": 0.3520188705425281, + "mrr_at_10": 0.4566733510850353, + "mrr_at_100": 0.4661325189110714, + "mrr_at_1000": 0.4664901648332512, + "mrr_at_20": 0.46242210594362715, + "mrr_at_3": 0.42560473613616545, + "mrr_at_5": 0.44292123398547745, + "nauc_cv_recall_at_1000_diff1": 0.43984952545943873, + "nauc_cv_recall_at_1000_max": 0.9333946385512961, + "nauc_cv_recall_at_1000_std": 0.9622112512663857, + "nauc_cv_recall_at_100_diff1": 0.4365013750273154, + "nauc_cv_recall_at_100_max": 0.781445962038063, + "nauc_cv_recall_at_100_std": 0.6866576843293807, + "nauc_cv_recall_at_10_diff1": 0.4440013860687523, + "nauc_cv_recall_at_10_max": 0.5457172777528433, + "nauc_cv_recall_at_10_std": 0.24115786357421992, + "nauc_cv_recall_at_1_diff1": 0.5917764418026163, + "nauc_cv_recall_at_1_max": 0.432157993286181, + "nauc_cv_recall_at_1_std": 0.06054202201021012, + "nauc_cv_recall_at_20_diff1": 0.43832825010501836, + "nauc_cv_recall_at_20_max": 0.6175497235851238, + "nauc_cv_recall_at_20_std": 0.3565597779245211, + "nauc_cv_recall_at_3_diff1": 0.4759945496272966, + "nauc_cv_recall_at_3_max": 0.4690612568061717, + "nauc_cv_recall_at_3_std": 0.11757529811594443, + "nauc_cv_recall_at_5_diff1": 0.45499226706325, + "nauc_cv_recall_at_5_max": 0.48788633462122094, + "nauc_cv_recall_at_5_std": 0.14809518063666652, + "nauc_map_at_1000_diff1": 0.5399112943616813, + "nauc_map_at_1000_max": 0.4600690969544165, + "nauc_map_at_1000_std": 0.10168618302358416, + "nauc_map_at_100_diff1": 0.5397977611488858, + "nauc_map_at_100_max": 0.46034560041005235, + "nauc_map_at_100_std": 0.10218056859319692, + "nauc_map_at_10_diff1": 0.5380519968078198, + "nauc_map_at_10_max": 0.45838195646878466, + "nauc_map_at_10_std": 0.09868025608798509, + "nauc_map_at_1_diff1": 0.5917764418026163, + "nauc_map_at_1_max": 0.432157993286181, + "nauc_map_at_1_std": 0.06054202201021012, + "nauc_map_at_20_diff1": 0.5389459887198278, + "nauc_map_at_20_max": 0.4605616437478957, + "nauc_map_at_20_std": 0.10203876555994912, + "nauc_map_at_3_diff1": 0.5406680525792159, + "nauc_map_at_3_max": 0.4494874340157258, + "nauc_map_at_3_std": 0.08437790118672613, + "nauc_map_at_5_diff1": 0.5371655685684823, + "nauc_map_at_5_max": 0.45302351717025974, + "nauc_map_at_5_std": 0.09022498915722132, + "nauc_mrr_at_1000_diff1": 0.5399112943616813, + "nauc_mrr_at_1000_max": 0.4600690969544165, + "nauc_mrr_at_1000_std": 0.10168618302358416, + "nauc_mrr_at_100_diff1": 0.5397977611488858, + "nauc_mrr_at_100_max": 0.46034560041005235, + "nauc_mrr_at_100_std": 0.10218056859319692, + "nauc_mrr_at_10_diff1": 0.5380519968078198, + "nauc_mrr_at_10_max": 0.45838195646878466, + "nauc_mrr_at_10_std": 0.09868025608798509, + "nauc_mrr_at_1_diff1": 0.5917764418026163, + "nauc_mrr_at_1_max": 0.432157993286181, + "nauc_mrr_at_1_std": 0.06054202201021012, + "nauc_mrr_at_20_diff1": 0.5389459887198278, + "nauc_mrr_at_20_max": 0.4605616437478957, + "nauc_mrr_at_20_std": 0.10203876555994912, + "nauc_mrr_at_3_diff1": 0.5406680525792159, + "nauc_mrr_at_3_max": 0.4494874340157258, + "nauc_mrr_at_3_std": 0.08437790118672613, + "nauc_mrr_at_5_diff1": 0.5371655685684823, + "nauc_mrr_at_5_max": 0.45302351717025974, + "nauc_mrr_at_5_std": 0.09022498915722132, + "nauc_ndcg_at_1000_diff1": 0.530352272874443, + "nauc_ndcg_at_1000_max": 0.47375450416916254, + "nauc_ndcg_at_1000_std": 0.12512049368825368, + "nauc_ndcg_at_100_diff1": 0.5272470054380564, + "nauc_ndcg_at_100_max": 0.48297319995546817, + "nauc_ndcg_at_100_std": 0.1418827176716715, + "nauc_ndcg_at_10_diff1": 0.5184077234967509, + "nauc_ndcg_at_10_max": 0.4751084362503045, + "nauc_ndcg_at_10_std": 0.12585944973703592, + "nauc_ndcg_at_1_diff1": 0.5917764418026163, + "nauc_ndcg_at_1_max": 0.432157993286181, + "nauc_ndcg_at_1_std": 0.06054202201021012, + "nauc_ndcg_at_20_diff1": 0.5210283417403792, + "nauc_ndcg_at_20_max": 0.48403106565510695, + "nauc_ndcg_at_20_std": 0.13995236921918247, + "nauc_ndcg_at_3_diff1": 0.5242927568943412, + "nauc_ndcg_at_3_max": 0.4544883574193214, + "nauc_ndcg_at_3_std": 0.09272567738930695, + "nauc_ndcg_at_5_diff1": 0.5175607308176516, + "nauc_ndcg_at_5_max": 0.46120957487887876, + "nauc_ndcg_at_5_std": 0.10370762506625461, + "nauc_precision_at_1000_diff1": 0.43984952545942857, + "nauc_precision_at_1000_max": 0.9333946385512796, + "nauc_precision_at_1000_std": 0.9622112512663694, + "nauc_precision_at_100_diff1": 0.4365013750273164, + "nauc_precision_at_100_max": 0.7814459620380634, + "nauc_precision_at_100_std": 0.6866576843293785, + "nauc_precision_at_10_diff1": 0.4440013860687516, + "nauc_precision_at_10_max": 0.5457172777528427, + "nauc_precision_at_10_std": 0.24115786357421895, + "nauc_precision_at_1_diff1": 0.5917764418026163, + "nauc_precision_at_1_max": 0.432157993286181, + "nauc_precision_at_1_std": 0.06054202201021012, + "nauc_precision_at_20_diff1": 0.4383282501050174, + "nauc_precision_at_20_max": 0.6175497235851231, + "nauc_precision_at_20_std": 0.3565597779245206, + "nauc_precision_at_3_diff1": 0.4759945496272968, + "nauc_precision_at_3_max": 0.4690612568061714, + "nauc_precision_at_3_std": 0.11757529811594437, + "nauc_precision_at_5_diff1": 0.45499226706324974, + "nauc_precision_at_5_max": 0.4878863346212209, + "nauc_precision_at_5_std": 0.14809518063666594, + "nauc_recall_at_1000_diff1": 0.43984952545943873, + "nauc_recall_at_1000_max": 0.9333946385512961, + "nauc_recall_at_1000_std": 0.9622112512663857, + "nauc_recall_at_100_diff1": 0.4365013750273154, + "nauc_recall_at_100_max": 0.781445962038063, + "nauc_recall_at_100_std": 0.6866576843293807, + "nauc_recall_at_10_diff1": 0.4440013860687523, + "nauc_recall_at_10_max": 0.5457172777528433, + "nauc_recall_at_10_std": 0.24115786357421992, + "nauc_recall_at_1_diff1": 0.5917764418026163, + "nauc_recall_at_1_max": 0.432157993286181, + "nauc_recall_at_1_std": 0.06054202201021012, + "nauc_recall_at_20_diff1": 0.43832825010501836, + "nauc_recall_at_20_max": 0.6175497235851238, + "nauc_recall_at_20_std": 0.3565597779245211, + "nauc_recall_at_3_diff1": 0.4759945496272966, + "nauc_recall_at_3_max": 0.4690612568061717, + "nauc_recall_at_3_std": 0.11757529811594443, + "nauc_recall_at_5_diff1": 0.45499226706325, + "nauc_recall_at_5_max": 0.48788633462122094, + "nauc_recall_at_5_std": 0.14809518063666652, + "ndcg_at_1": 0.35202, + "ndcg_at_10": 0.51423, + "ndcg_at_100": 0.56081, + "ndcg_at_1000": 0.57017, + "ndcg_at_20": 0.53488, + "ndcg_at_3": 0.44991, + "ndcg_at_5": 0.48111, + "precision_at_1": 0.35202, + "precision_at_10": 0.06981, + "precision_at_100": 0.00917, + "precision_at_1000": 0.00099, + "precision_at_20": 0.03896, + "precision_at_3": 0.17344, + "precision_at_5": 0.11922, + "recall_at_1": 0.35202, + "recall_at_10": 0.69807, + "recall_at_100": 0.91716, + "recall_at_1000": 0.99043, + "recall_at_20": 0.7791, + "recall_at_3": 0.52033, + "recall_at_5": 0.59609 + }, + { + "cv_recall_at_1": 0.39868, + "cv_recall_at_10": 0.72205, + "cv_recall_at_100": 0.91534, + "cv_recall_at_1000": 0.99023, + "cv_recall_at_20": 0.79447, + "cv_recall_at_3": 0.56911, + "cv_recall_at_5": 0.6378, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ], + "main_score": 0.55393, + "map_at_1": 0.39868, + "map_at_10": 0.50082, + "map_at_100": 0.50903, + "map_at_1000": 0.50939, + "map_at_20": 0.50592, + "map_at_3": 0.47359, + "map_at_5": 0.48935, + "mrr_at_1": 0.3986784140969163, + "mrr_at_10": 0.5008177968323918, + "mrr_at_100": 0.5090339944872555, + "mrr_at_1000": 0.5093904636645038, + "mrr_at_20": 0.505920593279936, + "mrr_at_3": 0.473591226138035, + "mrr_at_5": 0.48934700807635967, + "nauc_cv_recall_at_1000_diff1": 0.5263074522842123, + "nauc_cv_recall_at_1000_max": 0.9194863959419592, + "nauc_cv_recall_at_1000_std": 0.9357992764288658, + "nauc_cv_recall_at_100_diff1": 0.47055254295863874, + "nauc_cv_recall_at_100_max": 0.7334811069618354, + "nauc_cv_recall_at_100_std": 0.6566165938186991, + "nauc_cv_recall_at_10_diff1": 0.47823304517462406, + "nauc_cv_recall_at_10_max": 0.5706203482207864, + "nauc_cv_recall_at_10_std": 0.30137527176231743, + "nauc_cv_recall_at_1_diff1": 0.627152858930617, + "nauc_cv_recall_at_1_max": 0.4658527019227254, + "nauc_cv_recall_at_1_std": 0.10458940196560304, + "nauc_cv_recall_at_20_diff1": 0.48853535388484476, + "nauc_cv_recall_at_20_max": 0.6242157780019971, + "nauc_cv_recall_at_20_std": 0.40844620006462734, + "nauc_cv_recall_at_3_diff1": 0.5282633012642057, + "nauc_cv_recall_at_3_max": 0.5080606004209856, + "nauc_cv_recall_at_3_std": 0.1679506090067389, + "nauc_cv_recall_at_5_diff1": 0.511101454871359, + "nauc_cv_recall_at_5_max": 0.5367492026393453, + "nauc_cv_recall_at_5_std": 0.22728158638352508, + "nauc_map_at_1000_diff1": 0.5818138831820504, + "nauc_map_at_1000_max": 0.4918722831473778, + "nauc_map_at_1000_std": 0.14951051959313508, + "nauc_map_at_100_diff1": 0.5816662994279276, + "nauc_map_at_100_max": 0.49207753096794643, + "nauc_map_at_100_std": 0.14995210436964093, + "nauc_map_at_10_diff1": 0.5796002758151477, + "nauc_map_at_10_max": 0.4910260270825821, + "nauc_map_at_10_std": 0.14736297265232623, + "nauc_map_at_1_diff1": 0.627152858930617, + "nauc_map_at_1_max": 0.4658527019227254, + "nauc_map_at_1_std": 0.10458940196560304, + "nauc_map_at_20_diff1": 0.5811449028241016, + "nauc_map_at_20_max": 0.49239153644280603, + "nauc_map_at_20_std": 0.15016380990114506, + "nauc_map_at_3_diff1": 0.5842145172396421, + "nauc_map_at_3_max": 0.4836508860144376, + "nauc_map_at_3_std": 0.13078463307068863, + "nauc_map_at_5_diff1": 0.5817855309317947, + "nauc_map_at_5_max": 0.4887406792626373, + "nauc_map_at_5_std": 0.1416335599236373, + "nauc_mrr_at_1000_diff1": 0.5818138831820504, + "nauc_mrr_at_1000_max": 0.4918722831473778, + "nauc_mrr_at_1000_std": 0.14951051959313508, + "nauc_mrr_at_100_diff1": 0.5816662994279276, + "nauc_mrr_at_100_max": 0.49207753096794643, + "nauc_mrr_at_100_std": 0.14995210436964093, + "nauc_mrr_at_10_diff1": 0.5796002758151477, + "nauc_mrr_at_10_max": 0.4910260270825821, + "nauc_mrr_at_10_std": 0.14736297265232623, + "nauc_mrr_at_1_diff1": 0.627152858930617, + "nauc_mrr_at_1_max": 0.4658527019227254, + "nauc_mrr_at_1_std": 0.10458940196560304, + "nauc_mrr_at_20_diff1": 0.5811449028241016, + "nauc_mrr_at_20_max": 0.49239153644280603, + "nauc_mrr_at_20_std": 0.15016380990114506, + "nauc_mrr_at_3_diff1": 0.5842145172396421, + "nauc_mrr_at_3_max": 0.4836508860144376, + "nauc_mrr_at_3_std": 0.13078463307068863, + "nauc_mrr_at_5_diff1": 0.5817855309317947, + "nauc_mrr_at_5_max": 0.4887406792626373, + "nauc_mrr_at_5_std": 0.1416335599236373, + "nauc_ndcg_at_1000_diff1": 0.5729456825600561, + "nauc_ndcg_at_1000_max": 0.503419434860686, + "nauc_ndcg_at_1000_std": 0.17228177959800048, + "nauc_ndcg_at_100_diff1": 0.5690964391255721, + "nauc_ndcg_at_100_max": 0.5101963241609828, + "nauc_ndcg_at_100_std": 0.1870203735628991, + "nauc_ndcg_at_10_diff1": 0.5597447480902777, + "nauc_ndcg_at_10_max": 0.5059287479367308, + "nauc_ndcg_at_10_std": 0.1759310934875448, + "nauc_ndcg_at_1_diff1": 0.627152858930617, + "nauc_ndcg_at_1_max": 0.4658527019227254, + "nauc_ndcg_at_1_std": 0.10458940196560304, + "nauc_ndcg_at_20_diff1": 0.5650278051813951, + "nauc_ndcg_at_20_max": 0.5118452737194403, + "nauc_ndcg_at_20_std": 0.18793319055399638, + "nauc_ndcg_at_3_diff1": 0.5704529161273854, + "nauc_ndcg_at_3_max": 0.4896311500930583, + "nauc_ndcg_at_3_std": 0.13986740335109915, + "nauc_ndcg_at_5_diff1": 0.5656134964841482, + "nauc_ndcg_at_5_max": 0.49939996963906663, + "nauc_ndcg_at_5_std": 0.16050641016164127, + "nauc_precision_at_1000_diff1": 0.5263074522841861, + "nauc_precision_at_1000_max": 0.9194863959419753, + "nauc_precision_at_1000_std": 0.9357992764288523, + "nauc_precision_at_100_diff1": 0.47055254295863985, + "nauc_precision_at_100_max": 0.7334811069618357, + "nauc_precision_at_100_std": 0.656616593818699, + "nauc_precision_at_10_diff1": 0.4782330451746229, + "nauc_precision_at_10_max": 0.5706203482207856, + "nauc_precision_at_10_std": 0.30137527176231704, + "nauc_precision_at_1_diff1": 0.627152858930617, + "nauc_precision_at_1_max": 0.4658527019227254, + "nauc_precision_at_1_std": 0.10458940196560304, + "nauc_precision_at_20_diff1": 0.48853535388484326, + "nauc_precision_at_20_max": 0.624215778001996, + "nauc_precision_at_20_std": 0.4084462000646254, + "nauc_precision_at_3_diff1": 0.5282633012642054, + "nauc_precision_at_3_max": 0.5080606004209852, + "nauc_precision_at_3_std": 0.16795060900673922, + "nauc_precision_at_5_diff1": 0.5111014548713589, + "nauc_precision_at_5_max": 0.536749202639345, + "nauc_precision_at_5_std": 0.2272815863835243, + "nauc_recall_at_1000_diff1": 0.5263074522842123, + "nauc_recall_at_1000_max": 0.9194863959419592, + "nauc_recall_at_1000_std": 0.9357992764288658, + "nauc_recall_at_100_diff1": 0.47055254295863874, + "nauc_recall_at_100_max": 0.7334811069618354, + "nauc_recall_at_100_std": 0.6566165938186991, + "nauc_recall_at_10_diff1": 0.47823304517462406, + "nauc_recall_at_10_max": 0.5706203482207864, + "nauc_recall_at_10_std": 0.30137527176231743, + "nauc_recall_at_1_diff1": 0.627152858930617, + "nauc_recall_at_1_max": 0.4658527019227254, + "nauc_recall_at_1_std": 0.10458940196560304, + "nauc_recall_at_20_diff1": 0.48853535388484476, + "nauc_recall_at_20_max": 0.6242157780019971, + "nauc_recall_at_20_std": 0.40844620006462734, + "nauc_recall_at_3_diff1": 0.5282633012642057, + "nauc_recall_at_3_max": 0.5080606004209856, + "nauc_recall_at_3_std": 0.1679506090067389, + "nauc_recall_at_5_diff1": 0.511101454871359, + "nauc_recall_at_5_max": 0.5367492026393453, + "nauc_recall_at_5_std": 0.22728158638352508, + "ndcg_at_1": 0.39868, + "ndcg_at_10": 0.55393, + "ndcg_at_100": 0.5948, + "ndcg_at_1000": 0.6043, + "ndcg_at_20": 0.57234, + "ndcg_at_3": 0.49811, + "ndcg_at_5": 0.52646, + "precision_at_1": 0.39868, + "precision_at_10": 0.07221, + "precision_at_100": 0.00915, + "precision_at_1000": 0.00099, + "precision_at_20": 0.03972, + "precision_at_3": 0.1897, + "precision_at_5": 0.12756, + "recall_at_1": 0.39868, + "recall_at_10": 0.72205, + "recall_at_100": 0.91534, + "recall_at_1000": 0.99023, + "recall_at_20": 0.79447, + "recall_at_3": 0.56911, + "recall_at_5": 0.6378 + }, + { + "cv_recall_at_1": 0.74048, + "cv_recall_at_10": 0.96344, + "cv_recall_at_100": 0.99803, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.98288, + "cv_recall_at_3": 0.88985, + "cv_recall_at_5": 0.92861, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.85632, + "map_at_1": 0.74048, + "map_at_10": 0.82147, + "map_at_100": 0.82329, + "map_at_1000": 0.8233, + "map_at_20": 0.82284, + "map_at_3": 0.80778, + "map_at_5": 0.81671, + "mrr_at_1": 0.740483628369779, + "mrr_at_10": 0.8214715000119333, + "mrr_at_100": 0.8232923058252596, + "mrr_at_1000": 0.8233045665997936, + "mrr_at_20": 0.8228402912251109, + "mrr_at_3": 0.8077827914690076, + "mrr_at_5": 0.8167090902078693, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6080298926938646, + "nauc_cv_recall_at_100_max": 0.8094408328267139, + "nauc_cv_recall_at_100_std": 0.15159729318007661, + "nauc_cv_recall_at_10_diff1": 0.7262385280719563, + "nauc_cv_recall_at_10_max": 0.6053348863156468, + "nauc_cv_recall_at_10_std": -0.3586274961626783, + "nauc_cv_recall_at_1_diff1": 0.813003513000172, + "nauc_cv_recall_at_1_max": 0.45234481733298193, + "nauc_cv_recall_at_1_std": -0.3639220449247165, + "nauc_cv_recall_at_20_diff1": 0.7458693329791404, + "nauc_cv_recall_at_20_max": 0.7318334192674248, + "nauc_cv_recall_at_20_std": -0.14523783296087528, + "nauc_cv_recall_at_3_diff1": 0.7358042843081358, + "nauc_cv_recall_at_3_max": 0.525921917798875, + "nauc_cv_recall_at_3_std": -0.4153844768358637, + "nauc_cv_recall_at_5_diff1": 0.7216698778631316, + "nauc_cv_recall_at_5_max": 0.542507469966757, + "nauc_cv_recall_at_5_std": -0.4223158051137424, + "nauc_map_at_1000_diff1": 0.7934790235857491, + "nauc_map_at_1000_max": 0.47296201032733654, + "nauc_map_at_1000_std": -0.37694405501566375, + "nauc_map_at_100_diff1": 0.7934662202964456, + "nauc_map_at_100_max": 0.472987098543922, + "nauc_map_at_100_std": -0.3769046329700942, + "nauc_map_at_10_diff1": 0.7926637368909875, + "nauc_map_at_10_max": 0.47381616393015213, + "nauc_map_at_10_std": -0.3778287683589719, + "nauc_map_at_1_diff1": 0.813003513000172, + "nauc_map_at_1_max": 0.45234481733298193, + "nauc_map_at_1_std": -0.3639220449247165, + "nauc_map_at_20_diff1": 0.7933995647470824, + "nauc_map_at_20_max": 0.4736339018818771, + "nauc_map_at_20_std": -0.37649670320119927, + "nauc_map_at_3_diff1": 0.789140185230195, + "nauc_map_at_3_max": 0.4745045467764235, + "nauc_map_at_3_std": -0.38230865280163173, + "nauc_map_at_5_diff1": 0.7905349301308817, + "nauc_map_at_5_max": 0.4736720556975195, + "nauc_map_at_5_std": -0.3812223296792622, + "nauc_mrr_at_1000_diff1": 0.7934790235857491, + "nauc_mrr_at_1000_max": 0.47296201032733654, + "nauc_mrr_at_1000_std": -0.37694405501566375, + "nauc_mrr_at_100_diff1": 0.7934662202964456, + "nauc_mrr_at_100_max": 0.472987098543922, + "nauc_mrr_at_100_std": -0.3769046329700942, + "nauc_mrr_at_10_diff1": 0.7926637368909875, + "nauc_mrr_at_10_max": 0.47381616393015213, + "nauc_mrr_at_10_std": -0.3778287683589719, + "nauc_mrr_at_1_diff1": 0.813003513000172, + "nauc_mrr_at_1_max": 0.45234481733298193, + "nauc_mrr_at_1_std": -0.3639220449247165, + "nauc_mrr_at_20_diff1": 0.7933995647470824, + "nauc_mrr_at_20_max": 0.4736339018818771, + "nauc_mrr_at_20_std": -0.37649670320119927, + "nauc_mrr_at_3_diff1": 0.789140185230195, + "nauc_mrr_at_3_max": 0.4745045467764235, + "nauc_mrr_at_3_std": -0.38230865280163173, + "nauc_mrr_at_5_diff1": 0.7905349301308817, + "nauc_mrr_at_5_max": 0.4736720556975195, + "nauc_mrr_at_5_std": -0.3812223296792622, + "nauc_ndcg_at_1000_diff1": 0.7910361385308725, + "nauc_ndcg_at_1000_max": 0.47684137941910887, + "nauc_ndcg_at_1000_std": -0.3765196296277837, + "nauc_ndcg_at_100_diff1": 0.7906493843256589, + "nauc_ndcg_at_100_max": 0.4775492752123655, + "nauc_ndcg_at_100_std": -0.3754033971076211, + "nauc_ndcg_at_10_diff1": 0.7869969737209641, + "nauc_ndcg_at_10_max": 0.48352106502398234, + "nauc_ndcg_at_10_std": -0.37766974970717637, + "nauc_ndcg_at_1_diff1": 0.813003513000172, + "nauc_ndcg_at_1_max": 0.45234481733298193, + "nauc_ndcg_at_1_std": -0.3639220449247165, + "nauc_ndcg_at_20_diff1": 0.790077876058056, + "nauc_ndcg_at_20_max": 0.48309721524984295, + "nauc_ndcg_at_20_std": -0.3712496025867674, + "nauc_ndcg_at_3_diff1": 0.7796023612066416, + "nauc_ndcg_at_3_max": 0.4836702178151078, + "nauc_ndcg_at_3_std": -0.3883468622656064, + "nauc_ndcg_at_5_diff1": 0.7814603879909108, + "nauc_ndcg_at_5_max": 0.4827126092034757, + "nauc_ndcg_at_5_std": -0.386787692131109, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.6080298926938649, + "nauc_precision_at_100_max": 0.8094408328266878, + "nauc_precision_at_100_std": 0.15159729317989815, + "nauc_precision_at_10_diff1": 0.7262385280719623, + "nauc_precision_at_10_max": 0.6053348863156497, + "nauc_precision_at_10_std": -0.3586274961626875, + "nauc_precision_at_1_diff1": 0.813003513000172, + "nauc_precision_at_1_max": 0.45234481733298193, + "nauc_precision_at_1_std": -0.3639220449247165, + "nauc_precision_at_20_diff1": 0.7458693329791394, + "nauc_precision_at_20_max": 0.7318334192674198, + "nauc_precision_at_20_std": -0.14523783296088458, + "nauc_precision_at_3_diff1": 0.7358042843081359, + "nauc_precision_at_3_max": 0.5259219177988752, + "nauc_precision_at_3_std": -0.4153844768358635, + "nauc_precision_at_5_diff1": 0.7216698778631329, + "nauc_precision_at_5_max": 0.5425074699667538, + "nauc_precision_at_5_std": -0.4223158051137469, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.6080298926938646, + "nauc_recall_at_100_max": 0.8094408328267139, + "nauc_recall_at_100_std": 0.15159729318007661, + "nauc_recall_at_10_diff1": 0.7262385280719563, + "nauc_recall_at_10_max": 0.6053348863156468, + "nauc_recall_at_10_std": -0.3586274961626783, + "nauc_recall_at_1_diff1": 0.813003513000172, + "nauc_recall_at_1_max": 0.45234481733298193, + "nauc_recall_at_1_std": -0.3639220449247165, + "nauc_recall_at_20_diff1": 0.7458693329791404, + "nauc_recall_at_20_max": 0.7318334192674248, + "nauc_recall_at_20_std": -0.14523783296087528, + "nauc_recall_at_3_diff1": 0.7358042843081358, + "nauc_recall_at_3_max": 0.525921917798875, + "nauc_recall_at_3_std": -0.4153844768358637, + "nauc_recall_at_5_diff1": 0.7216698778631316, + "nauc_recall_at_5_max": 0.542507469966757, + "nauc_recall_at_5_std": -0.4223158051137424, + "ndcg_at_1": 0.74048, + "ndcg_at_10": 0.85632, + "ndcg_at_100": 0.8642, + "ndcg_at_1000": 0.86447, + "ndcg_at_20": 0.86126, + "ndcg_at_3": 0.82892, + "ndcg_at_5": 0.84495, + "precision_at_1": 0.74048, + "precision_at_10": 0.09634, + "precision_at_100": 0.00998, + "precision_at_1000": 0.001, + "precision_at_20": 0.04914, + "precision_at_3": 0.29662, + "precision_at_5": 0.18572, + "recall_at_1": 0.74048, + "recall_at_10": 0.96344, + "recall_at_100": 0.99803, + "recall_at_1000": 1.0, + "recall_at_20": 0.98288, + "recall_at_3": 0.88985, + "recall_at_5": 0.92861 + }, + { + "cv_recall_at_1": 0.13631, + "cv_recall_at_10": 0.37882, + "cv_recall_at_100": 0.70086, + "cv_recall_at_1000": 0.94142, + "cv_recall_at_20": 0.47043, + "cv_recall_at_3": 0.23778, + "cv_recall_at_5": 0.29511, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ], + "main_score": 0.24596, + "map_at_1": 0.13631, + "map_at_10": 0.2049, + "map_at_100": 0.21693, + "map_at_1000": 0.21796, + "map_at_20": 0.21121, + "map_at_3": 0.18069, + "map_at_5": 0.19379, + "mrr_at_1": 0.13631315935591337, + "mrr_at_10": 0.2049025893903735, + "mrr_at_100": 0.2169267661146188, + "mrr_at_1000": 0.2179567802285265, + "mrr_at_20": 0.21121163783023364, + "mrr_at_3": 0.18068665556172492, + "mrr_at_5": 0.19379048676661156, + "nauc_cv_recall_at_1000_diff1": 0.12315048052987966, + "nauc_cv_recall_at_1000_max": 0.684113262807908, + "nauc_cv_recall_at_1000_std": 0.8107300733807016, + "nauc_cv_recall_at_100_diff1": 0.12615226785532357, + "nauc_cv_recall_at_100_max": 0.49369246006412104, + "nauc_cv_recall_at_100_std": 0.5612294690418357, + "nauc_cv_recall_at_10_diff1": 0.2041646423498855, + "nauc_cv_recall_at_10_max": 0.3876832008855094, + "nauc_cv_recall_at_10_std": 0.3430556378240179, + "nauc_cv_recall_at_1_diff1": 0.38364736725904475, + "nauc_cv_recall_at_1_max": 0.38427880845344464, + "nauc_cv_recall_at_1_std": 0.24976199043925765, + "nauc_cv_recall_at_20_diff1": 0.17985737581721026, + "nauc_cv_recall_at_20_max": 0.41034482476852835, + "nauc_cv_recall_at_20_std": 0.39534016418479334, + "nauc_cv_recall_at_3_diff1": 0.2550695105023992, + "nauc_cv_recall_at_3_max": 0.3720164931669786, + "nauc_cv_recall_at_3_std": 0.2852382529047927, + "nauc_cv_recall_at_5_diff1": 0.23154248441240807, + "nauc_cv_recall_at_5_max": 0.37603218061608035, + "nauc_cv_recall_at_5_std": 0.29973737854224664, + "nauc_map_at_1000_diff1": 0.29537867204369916, + "nauc_map_at_1000_max": 0.3836994455065988, + "nauc_map_at_1000_std": 0.2875121245790976, + "nauc_map_at_100_diff1": 0.2948369157392871, + "nauc_map_at_100_max": 0.38363691195770255, + "nauc_map_at_100_std": 0.28777583177941146, + "nauc_map_at_10_diff1": 0.297555214002202, + "nauc_map_at_10_max": 0.3794912846856662, + "nauc_map_at_10_std": 0.2795066005348579, + "nauc_map_at_1_diff1": 0.38364736725904475, + "nauc_map_at_1_max": 0.38427880845344464, + "nauc_map_at_1_std": 0.24976199043925765, + "nauc_map_at_20_diff1": 0.2953827791586574, + "nauc_map_at_20_max": 0.3817202452937388, + "nauc_map_at_20_std": 0.2843268638555573, + "nauc_map_at_3_diff1": 0.3109444517252625, + "nauc_map_at_3_max": 0.3759194965449022, + "nauc_map_at_3_std": 0.26660162570747864, + "nauc_map_at_5_diff1": 0.3026825934896626, + "nauc_map_at_5_max": 0.3771281027413023, + "nauc_map_at_5_std": 0.27143091554943366, + "nauc_mrr_at_1000_diff1": 0.29537867204369916, + "nauc_mrr_at_1000_max": 0.3836994455065988, + "nauc_mrr_at_1000_std": 0.2875121245790976, + "nauc_mrr_at_100_diff1": 0.2948369157392871, + "nauc_mrr_at_100_max": 0.38363691195770255, + "nauc_mrr_at_100_std": 0.28777583177941146, + "nauc_mrr_at_10_diff1": 0.297555214002202, + "nauc_mrr_at_10_max": 0.3794912846856662, + "nauc_mrr_at_10_std": 0.2795066005348579, + "nauc_mrr_at_1_diff1": 0.38364736725904475, + "nauc_mrr_at_1_max": 0.38427880845344464, + "nauc_mrr_at_1_std": 0.24976199043925765, + "nauc_mrr_at_20_diff1": 0.2953827791586574, + "nauc_mrr_at_20_max": 0.3817202452937388, + "nauc_mrr_at_20_std": 0.2843268638555573, + "nauc_mrr_at_3_diff1": 0.3109444517252625, + "nauc_mrr_at_3_max": 0.3759194965449022, + "nauc_mrr_at_3_std": 0.26660162570747864, + "nauc_mrr_at_5_diff1": 0.3026825934896626, + "nauc_mrr_at_5_max": 0.3771281027413023, + "nauc_mrr_at_5_std": 0.27143091554943366, + "nauc_ndcg_at_1000_diff1": 0.27099643551398866, + "nauc_ndcg_at_1000_max": 0.4000380546347878, + "nauc_ndcg_at_1000_std": 0.3252095778830688, + "nauc_ndcg_at_100_diff1": 0.2578745995168681, + "nauc_ndcg_at_100_max": 0.40221959905473226, + "nauc_ndcg_at_100_std": 0.33799821839115624, + "nauc_ndcg_at_10_diff1": 0.2696423737390834, + "nauc_ndcg_at_10_max": 0.38158703798004134, + "nauc_ndcg_at_10_std": 0.29754569894191735, + "nauc_ndcg_at_1_diff1": 0.38364736725904475, + "nauc_ndcg_at_1_max": 0.38427880845344464, + "nauc_ndcg_at_1_std": 0.24976199043925765, + "nauc_ndcg_at_20_diff1": 0.2623574521934335, + "nauc_ndcg_at_20_max": 0.38873844027747434, + "nauc_ndcg_at_20_std": 0.31360247763855964, + "nauc_ndcg_at_3_diff1": 0.2944299126805184, + "nauc_ndcg_at_3_max": 0.3747043268401905, + "nauc_ndcg_at_3_std": 0.2719708163834449, + "nauc_ndcg_at_5_diff1": 0.2811396024950695, + "nauc_ndcg_at_5_max": 0.37667440905687843, + "nauc_ndcg_at_5_std": 0.2797929568888234, + "nauc_precision_at_1000_diff1": 0.12315048052987512, + "nauc_precision_at_1000_max": 0.684113262807911, + "nauc_precision_at_1000_std": 0.8107300733807009, + "nauc_precision_at_100_diff1": 0.1261522678553237, + "nauc_precision_at_100_max": 0.49369246006412154, + "nauc_precision_at_100_std": 0.5612294690418358, + "nauc_precision_at_10_diff1": 0.20416464234988557, + "nauc_precision_at_10_max": 0.3876832008855091, + "nauc_precision_at_10_std": 0.3430556378240175, + "nauc_precision_at_1_diff1": 0.38364736725904475, + "nauc_precision_at_1_max": 0.38427880845344464, + "nauc_precision_at_1_std": 0.24976199043925765, + "nauc_precision_at_20_diff1": 0.17985737581721012, + "nauc_precision_at_20_max": 0.4103448247685279, + "nauc_precision_at_20_std": 0.3953401641847935, + "nauc_precision_at_3_diff1": 0.255069510502399, + "nauc_precision_at_3_max": 0.37201649316697843, + "nauc_precision_at_3_std": 0.28523825290479277, + "nauc_precision_at_5_diff1": 0.23154248441240788, + "nauc_precision_at_5_max": 0.3760321806160802, + "nauc_precision_at_5_std": 0.29973737854224647, + "nauc_recall_at_1000_diff1": 0.12315048052987966, + "nauc_recall_at_1000_max": 0.684113262807908, + "nauc_recall_at_1000_std": 0.8107300733807016, + "nauc_recall_at_100_diff1": 0.12615226785532357, + "nauc_recall_at_100_max": 0.49369246006412104, + "nauc_recall_at_100_std": 0.5612294690418357, + "nauc_recall_at_10_diff1": 0.2041646423498855, + "nauc_recall_at_10_max": 0.3876832008855094, + "nauc_recall_at_10_std": 0.3430556378240179, + "nauc_recall_at_1_diff1": 0.38364736725904475, + "nauc_recall_at_1_max": 0.38427880845344464, + "nauc_recall_at_1_std": 0.24976199043925765, + "nauc_recall_at_20_diff1": 0.17985737581721026, + "nauc_recall_at_20_max": 0.41034482476852835, + "nauc_recall_at_20_std": 0.39534016418479334, + "nauc_recall_at_3_diff1": 0.2550695105023992, + "nauc_recall_at_3_max": 0.3720164931669786, + "nauc_recall_at_3_std": 0.2852382529047927, + "nauc_recall_at_5_diff1": 0.23154248441240807, + "nauc_recall_at_5_max": 0.37603218061608035, + "nauc_recall_at_5_std": 0.29973737854224664, + "ndcg_at_1": 0.13631, + "ndcg_at_10": 0.24596, + "ndcg_at_100": 0.31142, + "ndcg_at_1000": 0.34132, + "ndcg_at_20": 0.26906, + "ndcg_at_3": 0.19534, + "ndcg_at_5": 0.21895, + "precision_at_1": 0.13631, + "precision_at_10": 0.03788, + "precision_at_100": 0.00701, + "precision_at_1000": 0.00094, + "precision_at_20": 0.02352, + "precision_at_3": 0.07926, + "precision_at_5": 0.05902, + "recall_at_1": 0.13631, + "recall_at_10": 0.37882, + "recall_at_100": 0.70086, + "recall_at_1000": 0.94142, + "recall_at_20": 0.47043, + "recall_at_3": 0.23778, + "recall_at_5": 0.29511 + }, + { + "cv_recall_at_1": 0.59639, + "cv_recall_at_10": 0.90028, + "cv_recall_at_100": 0.9925, + "cv_recall_at_1000": 0.99972, + "cv_recall_at_20": 0.94361, + "cv_recall_at_3": 0.77264, + "cv_recall_at_5": 0.83444, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.74696, + "map_at_1": 0.59639, + "map_at_10": 0.698, + "map_at_100": 0.7025, + "map_at_1000": 0.70255, + "map_at_20": 0.7011, + "map_at_3": 0.67491, + "map_at_5": 0.68905, + "mrr_at_1": 0.5963888888888889, + "mrr_at_10": 0.6979968033509697, + "mrr_at_100": 0.7025036513629882, + "mrr_at_1000": 0.7025546795050475, + "mrr_at_20": 0.7011014931156818, + "mrr_at_3": 0.6749074074074076, + "mrr_at_5": 0.6890462962962937, + "nauc_cv_recall_at_1000_diff1": -0.8576097105504197, + "nauc_cv_recall_at_1000_max": 0.7957516339868551, + "nauc_cv_recall_at_1000_std": 0.638188608776782, + "nauc_cv_recall_at_100_diff1": 0.5582529308019509, + "nauc_cv_recall_at_100_max": 0.5609157243144177, + "nauc_cv_recall_at_100_std": -0.4681156413182839, + "nauc_cv_recall_at_10_diff1": 0.5970066763938622, + "nauc_cv_recall_at_10_max": 0.3365284832595995, + "nauc_cv_recall_at_10_std": -0.6122197254017663, + "nauc_cv_recall_at_1_diff1": 0.7377487579296046, + "nauc_cv_recall_at_1_max": 0.3024684160415429, + "nauc_cv_recall_at_1_std": -0.3981561808321467, + "nauc_cv_recall_at_20_diff1": 0.6089677250210449, + "nauc_cv_recall_at_20_max": 0.3989549842925695, + "nauc_cv_recall_at_20_std": -0.6511236678579452, + "nauc_cv_recall_at_3_diff1": 0.6471234666158875, + "nauc_cv_recall_at_3_max": 0.32663422677411136, + "nauc_cv_recall_at_3_std": -0.53340168067965, + "nauc_cv_recall_at_5_diff1": 0.6179094330520296, + "nauc_cv_recall_at_5_max": 0.3198350834990974, + "nauc_cv_recall_at_5_std": -0.5785430943899972, + "nauc_map_at_1000_diff1": 0.7053641370418658, + "nauc_map_at_1000_max": 0.30906898400039495, + "nauc_map_at_1000_std": -0.44711110357925826, + "nauc_map_at_100_diff1": 0.7053426977769025, + "nauc_map_at_100_max": 0.30912545216268755, + "nauc_map_at_100_std": -0.44712648955253825, + "nauc_map_at_10_diff1": 0.7033110878186942, + "nauc_map_at_10_max": 0.3087413776346512, + "nauc_map_at_10_std": -0.4495718484061258, + "nauc_map_at_1_diff1": 0.7377487579296046, + "nauc_map_at_1_max": 0.3024684160415429, + "nauc_map_at_1_std": -0.3981561808321467, + "nauc_map_at_20_diff1": 0.704843842130606, + "nauc_map_at_20_max": 0.30953620116106784, + "nauc_map_at_20_std": -0.44821432469327055, + "nauc_map_at_3_diff1": 0.703340423649336, + "nauc_map_at_3_max": 0.31031179232153905, + "nauc_map_at_3_std": -0.4480133791974383, + "nauc_map_at_5_diff1": 0.7015623635558061, + "nauc_map_at_5_max": 0.3083947625804306, + "nauc_map_at_5_std": -0.4508028481773592, + "nauc_mrr_at_1000_diff1": 0.7053641370418658, + "nauc_mrr_at_1000_max": 0.30906898400039495, + "nauc_mrr_at_1000_std": -0.44711110357925826, + "nauc_mrr_at_100_diff1": 0.7053426977769025, + "nauc_mrr_at_100_max": 0.30912545216268755, + "nauc_mrr_at_100_std": -0.44712648955253825, + "nauc_mrr_at_10_diff1": 0.7033110878186942, + "nauc_mrr_at_10_max": 0.3087413776346512, + "nauc_mrr_at_10_std": -0.4495718484061258, + "nauc_mrr_at_1_diff1": 0.7377487579296046, + "nauc_mrr_at_1_max": 0.3024684160415429, + "nauc_mrr_at_1_std": -0.3981561808321467, + "nauc_mrr_at_20_diff1": 0.704843842130606, + "nauc_mrr_at_20_max": 0.30953620116106784, + "nauc_mrr_at_20_std": -0.44821432469327055, + "nauc_mrr_at_3_diff1": 0.703340423649336, + "nauc_mrr_at_3_max": 0.31031179232153905, + "nauc_mrr_at_3_std": -0.4480133791974383, + "nauc_mrr_at_5_diff1": 0.7015623635558061, + "nauc_mrr_at_5_max": 0.3083947625804306, + "nauc_mrr_at_5_std": -0.4508028481773592, + "nauc_ndcg_at_1000_diff1": 0.7003430081375407, + "nauc_ndcg_at_1000_max": 0.31097470498087065, + "nauc_ndcg_at_1000_std": -0.45496562187963535, + "nauc_ndcg_at_100_diff1": 0.6998734859936994, + "nauc_ndcg_at_100_max": 0.31226100914742533, + "nauc_ndcg_at_100_std": -0.4552630761942781, + "nauc_ndcg_at_10_diff1": 0.6901158602766978, + "nauc_ndcg_at_10_max": 0.3119355925051725, + "nauc_ndcg_at_10_std": -0.47001381290428546, + "nauc_ndcg_at_1_diff1": 0.7377487579296046, + "nauc_ndcg_at_1_max": 0.3024684160415429, + "nauc_ndcg_at_1_std": -0.3981561808321467, + "nauc_ndcg_at_20_diff1": 0.6960519472233067, + "nauc_ndcg_at_20_max": 0.315223759324874, + "nauc_ndcg_at_20_std": -0.4650794295364529, + "nauc_ndcg_at_3_diff1": 0.6913575792745672, + "nauc_ndcg_at_3_max": 0.31372725712292077, + "nauc_ndcg_at_3_std": -0.4661427966293675, + "nauc_ndcg_at_5_diff1": 0.6869440581528756, + "nauc_ndcg_at_5_max": 0.31045414986716, + "nauc_ndcg_at_5_std": -0.47303632455277805, + "nauc_precision_at_1000_diff1": -0.857609710551402, + "nauc_precision_at_1000_max": 0.7957516339864209, + "nauc_precision_at_1000_std": 0.638188608776549, + "nauc_precision_at_100_diff1": 0.5582529308019456, + "nauc_precision_at_100_max": 0.5609157243144026, + "nauc_precision_at_100_std": -0.468115641318239, + "nauc_precision_at_10_diff1": 0.597006676393863, + "nauc_precision_at_10_max": 0.3365284832595971, + "nauc_precision_at_10_std": -0.6122197254017686, + "nauc_precision_at_1_diff1": 0.7377487579296046, + "nauc_precision_at_1_max": 0.3024684160415429, + "nauc_precision_at_1_std": -0.3981561808321467, + "nauc_precision_at_20_diff1": 0.6089677250210448, + "nauc_precision_at_20_max": 0.3989549842925696, + "nauc_precision_at_20_std": -0.6511236678579463, + "nauc_precision_at_3_diff1": 0.6471234666158877, + "nauc_precision_at_3_max": 0.326634226774111, + "nauc_precision_at_3_std": -0.5334016806796505, + "nauc_precision_at_5_diff1": 0.6179094330520296, + "nauc_precision_at_5_max": 0.3198350834990983, + "nauc_precision_at_5_std": -0.5785430943899962, + "nauc_recall_at_1000_diff1": -0.8576097105504197, + "nauc_recall_at_1000_max": 0.7957516339868551, + "nauc_recall_at_1000_std": 0.638188608776782, + "nauc_recall_at_100_diff1": 0.5582529308019509, + "nauc_recall_at_100_max": 0.5609157243144177, + "nauc_recall_at_100_std": -0.4681156413182839, + "nauc_recall_at_10_diff1": 0.5970066763938622, + "nauc_recall_at_10_max": 0.3365284832595995, + "nauc_recall_at_10_std": -0.6122197254017663, + "nauc_recall_at_1_diff1": 0.7377487579296046, + "nauc_recall_at_1_max": 0.3024684160415429, + "nauc_recall_at_1_std": -0.3981561808321467, + "nauc_recall_at_20_diff1": 0.6089677250210449, + "nauc_recall_at_20_max": 0.3989549842925695, + "nauc_recall_at_20_std": -0.6511236678579452, + "nauc_recall_at_3_diff1": 0.6471234666158875, + "nauc_recall_at_3_max": 0.32663422677411136, + "nauc_recall_at_3_std": -0.53340168067965, + "nauc_recall_at_5_diff1": 0.6179094330520296, + "nauc_recall_at_5_max": 0.3198350834990974, + "nauc_recall_at_5_std": -0.5785430943899972, + "ndcg_at_1": 0.59639, + "ndcg_at_10": 0.74696, + "ndcg_at_100": 0.7674, + "ndcg_at_1000": 0.76841, + "ndcg_at_20": 0.75804, + "ndcg_at_3": 0.70004, + "ndcg_at_5": 0.72551, + "precision_at_1": 0.59639, + "precision_at_10": 0.09003, + "precision_at_100": 0.00992, + "precision_at_1000": 0.001, + "precision_at_20": 0.04718, + "precision_at_3": 0.25755, + "precision_at_5": 0.16689, + "recall_at_1": 0.59639, + "recall_at_10": 0.90028, + "recall_at_100": 0.9925, + "recall_at_1000": 0.99972, + "recall_at_20": 0.94361, + "recall_at_3": 0.77264, + "recall_at_5": 0.83444 + }, + { + "cv_recall_at_1": 0.60169, + "cv_recall_at_10": 0.9127, + "cv_recall_at_100": 0.99512, + "cv_recall_at_1000": 0.99977, + "cv_recall_at_20": 0.9538, + "cv_recall_at_3": 0.79057, + "cv_recall_at_5": 0.84711, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.75655, + "map_at_1": 0.60169, + "map_at_10": 0.70649, + "map_at_100": 0.71061, + "map_at_1000": 0.71064, + "map_at_20": 0.70942, + "map_at_3": 0.68441, + "map_at_5": 0.69742, + "mrr_at_1": 0.6016949152542372, + "mrr_at_10": 0.7064896661396504, + "mrr_at_100": 0.7106148496171836, + "mrr_at_1000": 0.7106424374843844, + "mrr_at_20": 0.709417724462585, + "mrr_at_3": 0.684409101462732, + "mrr_at_5": 0.6974228000928664, + "nauc_cv_recall_at_1000_diff1": 0.6136786139686762, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 0.8611123491271107, + "nauc_cv_recall_at_100_diff1": 0.6200398503013391, + "nauc_cv_recall_at_100_max": 0.6930038548827888, + "nauc_cv_recall_at_100_std": 0.24702230513887494, + "nauc_cv_recall_at_10_diff1": 0.590287713177001, + "nauc_cv_recall_at_10_max": 0.43903135906364865, + "nauc_cv_recall_at_10_std": -0.2547022176587369, + "nauc_cv_recall_at_1_diff1": 0.7212591196929298, + "nauc_cv_recall_at_1_max": 0.3424258071570245, + "nauc_cv_recall_at_1_std": -0.2610397341026975, + "nauc_cv_recall_at_20_diff1": 0.5829385232149739, + "nauc_cv_recall_at_20_max": 0.5086830360137641, + "nauc_cv_recall_at_20_std": -0.14825110719122006, + "nauc_cv_recall_at_3_diff1": 0.6246144891401507, + "nauc_cv_recall_at_3_max": 0.4017382109009011, + "nauc_cv_recall_at_3_std": -0.28904230971819406, + "nauc_cv_recall_at_5_diff1": 0.6093640088946269, + "nauc_cv_recall_at_5_max": 0.42715394711549926, + "nauc_cv_recall_at_5_std": -0.26842751256748754, + "nauc_map_at_1000_diff1": 0.691433722593808, + "nauc_map_at_1000_max": 0.3614960036233303, + "nauc_map_at_1000_std": -0.2678824364782566, + "nauc_map_at_100_diff1": 0.6914272321601181, + "nauc_map_at_100_max": 0.3615230313975254, + "nauc_map_at_100_std": -0.2678474806057638, + "nauc_map_at_10_diff1": 0.6896377884065275, + "nauc_map_at_10_max": 0.36170263160915467, + "nauc_map_at_10_std": -0.2692104736323017, + "nauc_map_at_1_diff1": 0.7212591196929298, + "nauc_map_at_1_max": 0.3424258071570245, + "nauc_map_at_1_std": -0.2610397341026975, + "nauc_map_at_20_diff1": 0.6907849900168619, + "nauc_map_at_20_max": 0.361939109058859, + "nauc_map_at_20_std": -0.2676976848761723, + "nauc_map_at_3_diff1": 0.6875917923426406, + "nauc_map_at_3_max": 0.36170270426098095, + "nauc_map_at_3_std": -0.27333141519414944, + "nauc_map_at_5_diff1": 0.6882704724520071, + "nauc_map_at_5_max": 0.36360481691410973, + "nauc_map_at_5_std": -0.269604778112949, + "nauc_mrr_at_1000_diff1": 0.691433722593808, + "nauc_mrr_at_1000_max": 0.3614960036233303, + "nauc_mrr_at_1000_std": -0.2678824364782566, + "nauc_mrr_at_100_diff1": 0.6914272321601181, + "nauc_mrr_at_100_max": 0.3615230313975254, + "nauc_mrr_at_100_std": -0.2678474806057638, + "nauc_mrr_at_10_diff1": 0.6896377884065275, + "nauc_mrr_at_10_max": 0.36170263160915467, + "nauc_mrr_at_10_std": -0.2692104736323017, + "nauc_mrr_at_1_diff1": 0.7212591196929298, + "nauc_mrr_at_1_max": 0.3424258071570245, + "nauc_mrr_at_1_std": -0.2610397341026975, + "nauc_mrr_at_20_diff1": 0.6907849900168619, + "nauc_mrr_at_20_max": 0.361939109058859, + "nauc_mrr_at_20_std": -0.2676976848761723, + "nauc_mrr_at_3_diff1": 0.6875917923426406, + "nauc_mrr_at_3_max": 0.36170270426098095, + "nauc_mrr_at_3_std": -0.27333141519414944, + "nauc_mrr_at_5_diff1": 0.6882704724520071, + "nauc_mrr_at_5_max": 0.36360481691410973, + "nauc_mrr_at_5_std": -0.269604778112949, + "nauc_ndcg_at_1000_diff1": 0.6872101463698389, + "nauc_ndcg_at_1000_max": 0.3658212859632228, + "nauc_ndcg_at_1000_std": -0.26604171691781336, + "nauc_ndcg_at_100_diff1": 0.6870017228600985, + "nauc_ndcg_at_100_max": 0.36679058513416024, + "nauc_ndcg_at_100_std": -0.26458444374726653, + "nauc_ndcg_at_10_diff1": 0.6780702361160329, + "nauc_ndcg_at_10_max": 0.37034357646106747, + "nauc_ndcg_at_10_std": -0.26829335037077523, + "nauc_ndcg_at_1_diff1": 0.7212591196929298, + "nauc_ndcg_at_1_max": 0.3424258071570245, + "nauc_ndcg_at_1_std": -0.2610397341026975, + "nauc_ndcg_at_20_diff1": 0.6823980035535215, + "nauc_ndcg_at_20_max": 0.3712004190242446, + "nauc_ndcg_at_20_std": -0.26223920912492443, + "nauc_ndcg_at_3_diff1": 0.6746910152153665, + "nauc_ndcg_at_3_max": 0.3698392560465834, + "nauc_ndcg_at_3_std": -0.2766681307398376, + "nauc_ndcg_at_5_diff1": 0.6749598180573865, + "nauc_ndcg_at_5_max": 0.3741053208981384, + "nauc_ndcg_at_5_std": -0.26977667295231167, + "nauc_precision_at_1000_diff1": 0.6136786139684929, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.8611123491272562, + "nauc_precision_at_100_diff1": 0.6200398503013244, + "nauc_precision_at_100_max": 0.6930038548827688, + "nauc_precision_at_100_std": 0.24702230513882242, + "nauc_precision_at_10_diff1": 0.5902877131770019, + "nauc_precision_at_10_max": 0.4390313590636452, + "nauc_precision_at_10_std": -0.25470221765874135, + "nauc_precision_at_1_diff1": 0.7212591196929298, + "nauc_precision_at_1_max": 0.3424258071570245, + "nauc_precision_at_1_std": -0.2610397341026975, + "nauc_precision_at_20_diff1": 0.5829385232149772, + "nauc_precision_at_20_max": 0.5086830360137653, + "nauc_precision_at_20_std": -0.1482511071912294, + "nauc_precision_at_3_diff1": 0.6246144891401509, + "nauc_precision_at_3_max": 0.40173821090090217, + "nauc_precision_at_3_std": -0.28904230971819217, + "nauc_precision_at_5_diff1": 0.6093640088946262, + "nauc_precision_at_5_max": 0.42715394711549876, + "nauc_precision_at_5_std": -0.26842751256748654, + "nauc_recall_at_1000_diff1": 0.6136786139686762, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.8611123491271107, + "nauc_recall_at_100_diff1": 0.6200398503013391, + "nauc_recall_at_100_max": 0.6930038548827888, + "nauc_recall_at_100_std": 0.24702230513887494, + "nauc_recall_at_10_diff1": 0.590287713177001, + "nauc_recall_at_10_max": 0.43903135906364865, + "nauc_recall_at_10_std": -0.2547022176587369, + "nauc_recall_at_1_diff1": 0.7212591196929298, + "nauc_recall_at_1_max": 0.3424258071570245, + "nauc_recall_at_1_std": -0.2610397341026975, + "nauc_recall_at_20_diff1": 0.5829385232149739, + "nauc_recall_at_20_max": 0.5086830360137641, + "nauc_recall_at_20_std": -0.14825110719122006, + "nauc_recall_at_3_diff1": 0.6246144891401507, + "nauc_recall_at_3_max": 0.4017382109009011, + "nauc_recall_at_3_std": -0.28904230971819406, + "nauc_recall_at_5_diff1": 0.6093640088946269, + "nauc_recall_at_5_max": 0.42715394711549926, + "nauc_recall_at_5_std": -0.26842751256748754, + "ndcg_at_1": 0.60169, + "ndcg_at_10": 0.75655, + "ndcg_at_100": 0.77497, + "ndcg_at_1000": 0.77559, + "ndcg_at_20": 0.76703, + "ndcg_at_3": 0.71165, + "ndcg_at_5": 0.73502, + "precision_at_1": 0.60169, + "precision_at_10": 0.09127, + "precision_at_100": 0.00995, + "precision_at_1000": 0.001, + "precision_at_20": 0.04769, + "precision_at_3": 0.26352, + "precision_at_5": 0.16942, + "recall_at_1": 0.60169, + "recall_at_10": 0.9127, + "recall_at_100": 0.99512, + "recall_at_1000": 0.99977, + "recall_at_20": 0.9538, + "recall_at_3": 0.79057, + "recall_at_5": 0.84711 + }, + { + "cv_recall_at_1": 0.23188, + "cv_recall_at_10": 0.53333, + "cv_recall_at_100": 0.82374, + "cv_recall_at_1000": 0.9793, + "cv_recall_at_20": 0.62692, + "cv_recall_at_3": 0.36729, + "cv_recall_at_5": 0.43547, + "hf_subset": "fa", + "languages": [ + "fas-Arab" + ], + "main_score": 0.36969, + "map_at_1": 0.23188, + "map_at_10": 0.31889, + "map_at_100": 0.33036, + "map_at_1000": 0.33108, + "map_at_20": 0.32538, + "map_at_3": 0.29043, + "map_at_5": 0.30588, + "mrr_at_1": 0.2318840579710145, + "mrr_at_10": 0.318891025009036, + "mrr_at_100": 0.3303591182389066, + "mrr_at_1000": 0.33107825961711584, + "mrr_at_20": 0.3253773380134024, + "mrr_at_3": 0.2904301817345277, + "mrr_at_5": 0.3058753163100979, + "nauc_cv_recall_at_1000_diff1": 0.32211872213818943, + "nauc_cv_recall_at_1000_max": 0.7659603007995041, + "nauc_cv_recall_at_1000_std": 0.830699251839353, + "nauc_cv_recall_at_100_diff1": 0.25863051315437807, + "nauc_cv_recall_at_100_max": 0.5732638373083003, + "nauc_cv_recall_at_100_std": 0.5865637398670023, + "nauc_cv_recall_at_10_diff1": 0.2869595891354477, + "nauc_cv_recall_at_10_max": 0.4643134653282664, + "nauc_cv_recall_at_10_std": 0.3382897937481642, + "nauc_cv_recall_at_1_diff1": 0.457735843069892, + "nauc_cv_recall_at_1_max": 0.36856484473201834, + "nauc_cv_recall_at_1_std": 0.14545186619790604, + "nauc_cv_recall_at_20_diff1": 0.2724295353745255, + "nauc_cv_recall_at_20_max": 0.4988732153083199, + "nauc_cv_recall_at_20_std": 0.39954793525470816, + "nauc_cv_recall_at_3_diff1": 0.3506172985202539, + "nauc_cv_recall_at_3_max": 0.414967683433225, + "nauc_cv_recall_at_3_std": 0.23232033769502783, + "nauc_cv_recall_at_5_diff1": 0.31547657274500124, + "nauc_cv_recall_at_5_max": 0.4216833529163241, + "nauc_cv_recall_at_5_std": 0.2677498638456819, + "nauc_map_at_1000_diff1": 0.3947235492658944, + "nauc_map_at_1000_max": 0.3984217284223239, + "nauc_map_at_1000_std": 0.20562945107832167, + "nauc_map_at_100_diff1": 0.39447621630777163, + "nauc_map_at_100_max": 0.3986690453603573, + "nauc_map_at_100_std": 0.20621371414865497, + "nauc_map_at_10_diff1": 0.39359027833782984, + "nauc_map_at_10_max": 0.39713016437567766, + "nauc_map_at_10_std": 0.20280200415718957, + "nauc_map_at_1_diff1": 0.457735843069892, + "nauc_map_at_1_max": 0.36856484473201834, + "nauc_map_at_1_std": 0.14545186619790604, + "nauc_map_at_20_diff1": 0.3935234674955701, + "nauc_map_at_20_max": 0.3985802315211193, + "nauc_map_at_20_std": 0.20542407715586428, + "nauc_map_at_3_diff1": 0.40614112697680893, + "nauc_map_at_3_max": 0.39010031302332215, + "nauc_map_at_3_std": 0.18497814770151996, + "nauc_map_at_5_diff1": 0.3969890334451716, + "nauc_map_at_5_max": 0.3913816377867295, + "nauc_map_at_5_std": 0.19352248177676157, + "nauc_mrr_at_1000_diff1": 0.3947235492658944, + "nauc_mrr_at_1000_max": 0.3984217284223239, + "nauc_mrr_at_1000_std": 0.20562945107832167, + "nauc_mrr_at_100_diff1": 0.39447621630777163, + "nauc_mrr_at_100_max": 0.3986690453603573, + "nauc_mrr_at_100_std": 0.20621371414865497, + "nauc_mrr_at_10_diff1": 0.39359027833782984, + "nauc_mrr_at_10_max": 0.39713016437567766, + "nauc_mrr_at_10_std": 0.20280200415718957, + "nauc_mrr_at_1_diff1": 0.457735843069892, + "nauc_mrr_at_1_max": 0.36856484473201834, + "nauc_mrr_at_1_std": 0.14545186619790604, + "nauc_mrr_at_20_diff1": 0.3935234674955701, + "nauc_mrr_at_20_max": 0.3985802315211193, + "nauc_mrr_at_20_std": 0.20542407715586428, + "nauc_mrr_at_3_diff1": 0.40614112697680893, + "nauc_mrr_at_3_max": 0.39010031302332215, + "nauc_mrr_at_3_std": 0.18497814770151996, + "nauc_mrr_at_5_diff1": 0.3969890334451716, + "nauc_mrr_at_5_max": 0.3913816377867295, + "nauc_mrr_at_5_std": 0.19352248177676157, + "nauc_ndcg_at_1000_diff1": 0.3797599734049316, + "nauc_ndcg_at_1000_max": 0.41316958890282335, + "nauc_ndcg_at_1000_std": 0.23518241317437694, + "nauc_ndcg_at_100_diff1": 0.3720305312998685, + "nauc_ndcg_at_100_max": 0.42098508813898644, + "nauc_ndcg_at_100_std": 0.2538830227370447, + "nauc_ndcg_at_10_diff1": 0.3669227076502561, + "nauc_ndcg_at_10_max": 0.4131996641700761, + "nauc_ndcg_at_10_std": 0.23542173233196248, + "nauc_ndcg_at_1_diff1": 0.457735843069892, + "nauc_ndcg_at_1_max": 0.36856484473201834, + "nauc_ndcg_at_1_std": 0.14545186619790604, + "nauc_ndcg_at_20_diff1": 0.3659279059662814, + "nauc_ndcg_at_20_max": 0.41951719608107246, + "nauc_ndcg_at_20_std": 0.24647748233394168, + "nauc_ndcg_at_3_diff1": 0.39103472570202114, + "nauc_ndcg_at_3_max": 0.39683204553553547, + "nauc_ndcg_at_3_std": 0.19776213687706054, + "nauc_ndcg_at_5_diff1": 0.3752911647015171, + "nauc_ndcg_at_5_max": 0.39944245600088607, + "nauc_ndcg_at_5_std": 0.21304738210858173, + "nauc_precision_at_1000_diff1": 0.3221187221381867, + "nauc_precision_at_1000_max": 0.7659603007994962, + "nauc_precision_at_1000_std": 0.8306992518393461, + "nauc_precision_at_100_diff1": 0.2586305131543791, + "nauc_precision_at_100_max": 0.5732638373083007, + "nauc_precision_at_100_std": 0.5865637398670032, + "nauc_precision_at_10_diff1": 0.2869595891354475, + "nauc_precision_at_10_max": 0.4643134653282665, + "nauc_precision_at_10_std": 0.3382897937481639, + "nauc_precision_at_1_diff1": 0.457735843069892, + "nauc_precision_at_1_max": 0.36856484473201834, + "nauc_precision_at_1_std": 0.14545186619790604, + "nauc_precision_at_20_diff1": 0.2724295353745257, + "nauc_precision_at_20_max": 0.4988732153083194, + "nauc_precision_at_20_std": 0.3995479352547079, + "nauc_precision_at_3_diff1": 0.35061729852025403, + "nauc_precision_at_3_max": 0.41496768343322493, + "nauc_precision_at_3_std": 0.23232033769502788, + "nauc_precision_at_5_diff1": 0.3154765727450011, + "nauc_precision_at_5_max": 0.4216833529163243, + "nauc_precision_at_5_std": 0.26774986384568156, + "nauc_recall_at_1000_diff1": 0.32211872213818943, + "nauc_recall_at_1000_max": 0.7659603007995041, + "nauc_recall_at_1000_std": 0.830699251839353, + "nauc_recall_at_100_diff1": 0.25863051315437807, + "nauc_recall_at_100_max": 0.5732638373083003, + "nauc_recall_at_100_std": 0.5865637398670023, + "nauc_recall_at_10_diff1": 0.2869595891354477, + "nauc_recall_at_10_max": 0.4643134653282664, + "nauc_recall_at_10_std": 0.3382897937481642, + "nauc_recall_at_1_diff1": 0.457735843069892, + "nauc_recall_at_1_max": 0.36856484473201834, + "nauc_recall_at_1_std": 0.14545186619790604, + "nauc_recall_at_20_diff1": 0.2724295353745255, + "nauc_recall_at_20_max": 0.4988732153083199, + "nauc_recall_at_20_std": 0.39954793525470816, + "nauc_recall_at_3_diff1": 0.3506172985202539, + "nauc_recall_at_3_max": 0.414967683433225, + "nauc_recall_at_3_std": 0.23232033769502783, + "nauc_recall_at_5_diff1": 0.31547657274500124, + "nauc_recall_at_5_max": 0.4216833529163241, + "nauc_recall_at_5_std": 0.2677498638456819, + "ndcg_at_1": 0.23188, + "ndcg_at_10": 0.36969, + "ndcg_at_100": 0.42974, + "ndcg_at_1000": 0.44938, + "ndcg_at_20": 0.39334, + "ndcg_at_3": 0.31012, + "ndcg_at_5": 0.33808, + "precision_at_1": 0.23188, + "precision_at_10": 0.05333, + "precision_at_100": 0.00824, + "precision_at_1000": 0.00098, + "precision_at_20": 0.03135, + "precision_at_3": 0.12243, + "precision_at_5": 0.08709, + "recall_at_1": 0.23188, + "recall_at_10": 0.53333, + "recall_at_100": 0.82374, + "recall_at_1000": 0.9793, + "recall_at_20": 0.62692, + "recall_at_3": 0.36729, + "recall_at_5": 0.43547 + }, + { + "cv_recall_at_1": 0.12726, + "cv_recall_at_10": 0.33717, + "cv_recall_at_100": 0.62537, + "cv_recall_at_1000": 0.90908, + "cv_recall_at_20": 0.41336, + "cv_recall_at_3": 0.21454, + "cv_recall_at_5": 0.26365, + "hf_subset": "fi", + "languages": [ + "fin-Latn" + ], + "main_score": 0.22208, + "map_at_1": 0.12726, + "map_at_10": 0.18652, + "map_at_100": 0.19685, + "map_at_1000": 0.19799, + "map_at_20": 0.19179, + "map_at_3": 0.1655, + "map_at_5": 0.17668, + "mrr_at_1": 0.12726252280061737, + "mrr_at_10": 0.18652129950712795, + "mrr_at_100": 0.1968498342359693, + "mrr_at_1000": 0.19799300280423807, + "mrr_at_20": 0.19179013391721447, + "mrr_at_3": 0.1654974042374072, + "mrr_at_5": 0.1766802301108467, + "nauc_cv_recall_at_1000_diff1": 0.1723409475154773, + "nauc_cv_recall_at_1000_max": 0.5806492335928906, + "nauc_cv_recall_at_1000_std": 0.6978656115925028, + "nauc_cv_recall_at_100_diff1": 0.15949208699029174, + "nauc_cv_recall_at_100_max": 0.5081647501763306, + "nauc_cv_recall_at_100_std": 0.5648517323131209, + "nauc_cv_recall_at_10_diff1": 0.20978800071637904, + "nauc_cv_recall_at_10_max": 0.4590187077320782, + "nauc_cv_recall_at_10_std": 0.40395526036896534, + "nauc_cv_recall_at_1_diff1": 0.4194797367320295, + "nauc_cv_recall_at_1_max": 0.48084040784838245, + "nauc_cv_recall_at_1_std": 0.3249433866361136, + "nauc_cv_recall_at_20_diff1": 0.19576055879244425, + "nauc_cv_recall_at_20_max": 0.48531777680211063, + "nauc_cv_recall_at_20_std": 0.4645460165139727, + "nauc_cv_recall_at_3_diff1": 0.2660061870437314, + "nauc_cv_recall_at_3_max": 0.43734745197707886, + "nauc_cv_recall_at_3_std": 0.33147816691693927, + "nauc_cv_recall_at_5_diff1": 0.24040063953680255, + "nauc_cv_recall_at_5_max": 0.43914265734822067, + "nauc_cv_recall_at_5_std": 0.35044859851275406, + "nauc_map_at_1000_diff1": 0.3179538832843002, + "nauc_map_at_1000_max": 0.46396708740385206, + "nauc_map_at_1000_std": 0.35230253784996296, + "nauc_map_at_100_diff1": 0.317458246872419, + "nauc_map_at_100_max": 0.46384790020380545, + "nauc_map_at_100_std": 0.35241899135604876, + "nauc_map_at_10_diff1": 0.3197482521656431, + "nauc_map_at_10_max": 0.4608621677461234, + "nauc_map_at_10_std": 0.34404064722784683, + "nauc_map_at_1_diff1": 0.4194797367320295, + "nauc_map_at_1_max": 0.48084040784838245, + "nauc_map_at_1_std": 0.3249433866361136, + "nauc_map_at_20_diff1": 0.3180910840899835, + "nauc_map_at_20_max": 0.46339777801996374, + "nauc_map_at_20_std": 0.34997322868109837, + "nauc_map_at_3_diff1": 0.33613678125759255, + "nauc_map_at_3_max": 0.45845354566336505, + "nauc_map_at_3_std": 0.329036739176186, + "nauc_map_at_5_diff1": 0.32697875909516155, + "nauc_map_at_5_max": 0.4581622492089542, + "nauc_map_at_5_std": 0.33448880015691235, + "nauc_mrr_at_1000_diff1": 0.3179538832843002, + "nauc_mrr_at_1000_max": 0.46396708740385206, + "nauc_mrr_at_1000_std": 0.35230253784996296, + "nauc_mrr_at_100_diff1": 0.317458246872419, + "nauc_mrr_at_100_max": 0.46384790020380545, + "nauc_mrr_at_100_std": 0.35241899135604876, + "nauc_mrr_at_10_diff1": 0.3197482521656431, + "nauc_mrr_at_10_max": 0.4608621677461234, + "nauc_mrr_at_10_std": 0.34404064722784683, + "nauc_mrr_at_1_diff1": 0.4194797367320295, + "nauc_mrr_at_1_max": 0.48084040784838245, + "nauc_mrr_at_1_std": 0.3249433866361136, + "nauc_mrr_at_20_diff1": 0.3180910840899835, + "nauc_mrr_at_20_max": 0.46339777801996374, + "nauc_mrr_at_20_std": 0.34997322868109837, + "nauc_mrr_at_3_diff1": 0.33613678125759255, + "nauc_mrr_at_3_max": 0.45845354566336505, + "nauc_mrr_at_3_std": 0.329036739176186, + "nauc_mrr_at_5_diff1": 0.32697875909516155, + "nauc_mrr_at_5_max": 0.4581622492089542, + "nauc_mrr_at_5_std": 0.33448880015691235, + "nauc_ndcg_at_1000_diff1": 0.2891655168605115, + "nauc_ndcg_at_1000_max": 0.4719452565725258, + "nauc_ndcg_at_1000_std": 0.38864092349062784, + "nauc_ndcg_at_100_diff1": 0.2766580840757124, + "nauc_ndcg_at_100_max": 0.47154980205352626, + "nauc_ndcg_at_100_std": 0.397782913341144, + "nauc_ndcg_at_10_diff1": 0.2861483545191091, + "nauc_ndcg_at_10_max": 0.45935636504902333, + "nauc_ndcg_at_10_std": 0.3607452554429696, + "nauc_ndcg_at_1_diff1": 0.4194797367320295, + "nauc_ndcg_at_1_max": 0.48084040784838245, + "nauc_ndcg_at_1_std": 0.3249433866361136, + "nauc_ndcg_at_20_diff1": 0.2809672887972353, + "nauc_ndcg_at_20_max": 0.46783014778517384, + "nauc_ndcg_at_20_std": 0.3805501614476263, + "nauc_ndcg_at_3_diff1": 0.3154473467289752, + "nauc_ndcg_at_3_max": 0.4522775531898738, + "nauc_ndcg_at_3_std": 0.32977935791371676, + "nauc_ndcg_at_5_diff1": 0.30081531429951247, + "nauc_ndcg_at_5_max": 0.45229813529152296, + "nauc_ndcg_at_5_std": 0.3390725044860204, + "nauc_precision_at_1000_diff1": 0.17234094751547313, + "nauc_precision_at_1000_max": 0.5806492335928894, + "nauc_precision_at_1000_std": 0.697865611592502, + "nauc_precision_at_100_diff1": 0.15949208699029158, + "nauc_precision_at_100_max": 0.5081647501763312, + "nauc_precision_at_100_std": 0.5648517323131214, + "nauc_precision_at_10_diff1": 0.20978800071637896, + "nauc_precision_at_10_max": 0.45901870773207815, + "nauc_precision_at_10_std": 0.40395526036896495, + "nauc_precision_at_1_diff1": 0.4194797367320295, + "nauc_precision_at_1_max": 0.48084040784838245, + "nauc_precision_at_1_std": 0.3249433866361136, + "nauc_precision_at_20_diff1": 0.19576055879244383, + "nauc_precision_at_20_max": 0.4853177768021108, + "nauc_precision_at_20_std": 0.4645460165139723, + "nauc_precision_at_3_diff1": 0.26600618704373113, + "nauc_precision_at_3_max": 0.4373474519770788, + "nauc_precision_at_3_std": 0.3314781669169393, + "nauc_precision_at_5_diff1": 0.24040063953680246, + "nauc_precision_at_5_max": 0.43914265734822083, + "nauc_precision_at_5_std": 0.350448598512754, + "nauc_recall_at_1000_diff1": 0.1723409475154773, + "nauc_recall_at_1000_max": 0.5806492335928906, + "nauc_recall_at_1000_std": 0.6978656115925028, + "nauc_recall_at_100_diff1": 0.15949208699029174, + "nauc_recall_at_100_max": 0.5081647501763306, + "nauc_recall_at_100_std": 0.5648517323131209, + "nauc_recall_at_10_diff1": 0.20978800071637904, + "nauc_recall_at_10_max": 0.4590187077320782, + "nauc_recall_at_10_std": 0.40395526036896534, + "nauc_recall_at_1_diff1": 0.4194797367320295, + "nauc_recall_at_1_max": 0.48084040784838245, + "nauc_recall_at_1_std": 0.3249433866361136, + "nauc_recall_at_20_diff1": 0.19576055879244425, + "nauc_recall_at_20_max": 0.48531777680211063, + "nauc_recall_at_20_std": 0.4645460165139727, + "nauc_recall_at_3_diff1": 0.2660061870437314, + "nauc_recall_at_3_max": 0.43734745197707886, + "nauc_recall_at_3_std": 0.33147816691693927, + "nauc_recall_at_5_diff1": 0.24040063953680255, + "nauc_recall_at_5_max": 0.43914265734822067, + "nauc_recall_at_5_std": 0.35044859851275406, + "ndcg_at_1": 0.12726, + "ndcg_at_10": 0.22208, + "ndcg_at_100": 0.27988, + "ndcg_at_1000": 0.3147, + "ndcg_at_20": 0.24132, + "ndcg_at_3": 0.17808, + "ndcg_at_5": 0.19827, + "precision_at_1": 0.12726, + "precision_at_10": 0.03372, + "precision_at_100": 0.00625, + "precision_at_1000": 0.00091, + "precision_at_20": 0.02067, + "precision_at_3": 0.07151, + "precision_at_5": 0.05273, + "recall_at_1": 0.12726, + "recall_at_10": 0.33717, + "recall_at_100": 0.62537, + "recall_at_1000": 0.90908, + "recall_at_20": 0.41336, + "recall_at_3": 0.21454, + "recall_at_5": 0.26365 + }, + { + "cv_recall_at_1": 0.10409, + "cv_recall_at_10": 0.26994, + "cv_recall_at_100": 0.48052, + "cv_recall_at_1000": 0.77761, + "cv_recall_at_20": 0.32747, + "cv_recall_at_3": 0.17626, + "cv_recall_at_5": 0.21691, + "hf_subset": "fil", + "languages": [ + "fil-Latn" + ], + "main_score": 0.17979, + "map_at_1": 0.10409, + "map_at_10": 0.15181, + "map_at_100": 0.15935, + "map_at_1000": 0.16042, + "map_at_20": 0.15576, + "map_at_3": 0.13551, + "map_at_5": 0.14481, + "mrr_at_1": 0.10409340272893515, + "mrr_at_10": 0.15181348480687365, + "mrr_at_100": 0.15934884398364807, + "mrr_at_1000": 0.16041844070172975, + "mrr_at_20": 0.15576377193412375, + "mrr_at_3": 0.1355089792282088, + "mrr_at_5": 0.14481408543161325, + "nauc_cv_recall_at_1000_diff1": 0.21424855009198726, + "nauc_cv_recall_at_1000_max": 0.6396145976080426, + "nauc_cv_recall_at_1000_std": 0.6832780714438885, + "nauc_cv_recall_at_100_diff1": 0.2408891924307149, + "nauc_cv_recall_at_100_max": 0.6415385547577763, + "nauc_cv_recall_at_100_std": 0.6241230779955738, + "nauc_cv_recall_at_10_diff1": 0.31301465223645725, + "nauc_cv_recall_at_10_max": 0.6259268235668424, + "nauc_cv_recall_at_10_std": 0.5067559222657122, + "nauc_cv_recall_at_1_diff1": 0.48340507367292174, + "nauc_cv_recall_at_1_max": 0.5918060803445095, + "nauc_cv_recall_at_1_std": 0.3803959830733999, + "nauc_cv_recall_at_20_diff1": 0.2842973728738141, + "nauc_cv_recall_at_20_max": 0.6290968057402018, + "nauc_cv_recall_at_20_std": 0.5382641411507488, + "nauc_cv_recall_at_3_diff1": 0.36960796666684165, + "nauc_cv_recall_at_3_max": 0.6189674816818908, + "nauc_cv_recall_at_3_std": 0.4484934591367724, + "nauc_cv_recall_at_5_diff1": 0.3386517993718759, + "nauc_cv_recall_at_5_max": 0.6155180401590951, + "nauc_cv_recall_at_5_std": 0.46547602875489463, + "nauc_map_at_1000_diff1": 0.39924658085767, + "nauc_map_at_1000_max": 0.6091345749330558, + "nauc_map_at_1000_std": 0.437122064911765, + "nauc_map_at_100_diff1": 0.3990805059906888, + "nauc_map_at_100_max": 0.609073942129352, + "nauc_map_at_100_std": 0.43701505321298145, + "nauc_map_at_10_diff1": 0.4047246467831402, + "nauc_map_at_10_max": 0.6080048359197399, + "nauc_map_at_10_std": 0.4297648942019917, + "nauc_map_at_1_diff1": 0.48340507367292174, + "nauc_map_at_1_max": 0.5918060803445095, + "nauc_map_at_1_std": 0.3803959830733999, + "nauc_map_at_20_diff1": 0.4010306354033571, + "nauc_map_at_20_max": 0.6084782287906838, + "nauc_map_at_20_std": 0.43363590478649666, + "nauc_map_at_3_diff1": 0.4226552701229157, + "nauc_map_at_3_max": 0.6069539099308938, + "nauc_map_at_3_std": 0.4149357350267575, + "nauc_map_at_5_diff1": 0.41071077410712703, + "nauc_map_at_5_max": 0.6059960903085639, + "nauc_map_at_5_std": 0.421177696907373, + "nauc_mrr_at_1000_diff1": 0.39924658085767, + "nauc_mrr_at_1000_max": 0.6091345749330558, + "nauc_mrr_at_1000_std": 0.437122064911765, + "nauc_mrr_at_100_diff1": 0.3990805059906888, + "nauc_mrr_at_100_max": 0.609073942129352, + "nauc_mrr_at_100_std": 0.43701505321298145, + "nauc_mrr_at_10_diff1": 0.4047246467831402, + "nauc_mrr_at_10_max": 0.6080048359197399, + "nauc_mrr_at_10_std": 0.4297648942019917, + "nauc_mrr_at_1_diff1": 0.48340507367292174, + "nauc_mrr_at_1_max": 0.5918060803445095, + "nauc_mrr_at_1_std": 0.3803959830733999, + "nauc_mrr_at_20_diff1": 0.4010306354033571, + "nauc_mrr_at_20_max": 0.6084782287906838, + "nauc_mrr_at_20_std": 0.43363590478649666, + "nauc_mrr_at_3_diff1": 0.4226552701229157, + "nauc_mrr_at_3_max": 0.6069539099308938, + "nauc_mrr_at_3_std": 0.4149357350267575, + "nauc_mrr_at_5_diff1": 0.41071077410712703, + "nauc_mrr_at_5_max": 0.6059960903085639, + "nauc_mrr_at_5_std": 0.421177696907373, + "nauc_ndcg_at_1000_diff1": 0.358454534190773, + "nauc_ndcg_at_1000_max": 0.6170885217468562, + "nauc_ndcg_at_1000_std": 0.48463916900891485, + "nauc_ndcg_at_100_diff1": 0.35325035842887503, + "nauc_ndcg_at_100_max": 0.6182234443263771, + "nauc_ndcg_at_100_std": 0.4869165041620259, + "nauc_ndcg_at_10_diff1": 0.3759730083967871, + "nauc_ndcg_at_10_max": 0.6135123636072259, + "nauc_ndcg_at_10_std": 0.4531881757711014, + "nauc_ndcg_at_1_diff1": 0.48340507367292174, + "nauc_ndcg_at_1_max": 0.5918060803445095, + "nauc_ndcg_at_1_std": 0.3803959830733999, + "nauc_ndcg_at_20_diff1": 0.36471685952311034, + "nauc_ndcg_at_20_max": 0.6148799438274387, + "nauc_ndcg_at_20_std": 0.4651709626901215, + "nauc_ndcg_at_3_diff1": 0.4069453017377879, + "nauc_ndcg_at_3_max": 0.6105376936705731, + "nauc_ndcg_at_3_std": 0.4248011868786331, + "nauc_ndcg_at_5_diff1": 0.38850935119559227, + "nauc_ndcg_at_5_max": 0.6090221015664716, + "nauc_ndcg_at_5_std": 0.43472779438740256, + "nauc_precision_at_1000_diff1": 0.2142485500919868, + "nauc_precision_at_1000_max": 0.6396145976080425, + "nauc_precision_at_1000_std": 0.6832780714438873, + "nauc_precision_at_100_diff1": 0.2408891924307154, + "nauc_precision_at_100_max": 0.641538554757777, + "nauc_precision_at_100_std": 0.6241230779955743, + "nauc_precision_at_10_diff1": 0.313014652236457, + "nauc_precision_at_10_max": 0.6259268235668421, + "nauc_precision_at_10_std": 0.5067559222657121, + "nauc_precision_at_1_diff1": 0.48340507367292174, + "nauc_precision_at_1_max": 0.5918060803445095, + "nauc_precision_at_1_std": 0.3803959830733999, + "nauc_precision_at_20_diff1": 0.2842973728738142, + "nauc_precision_at_20_max": 0.6290968057402014, + "nauc_precision_at_20_std": 0.5382641411507488, + "nauc_precision_at_3_diff1": 0.3696079666668415, + "nauc_precision_at_3_max": 0.6189674816818908, + "nauc_precision_at_3_std": 0.4484934591367722, + "nauc_precision_at_5_diff1": 0.3386517993718761, + "nauc_precision_at_5_max": 0.6155180401590954, + "nauc_precision_at_5_std": 0.465476028754895, + "nauc_recall_at_1000_diff1": 0.21424855009198726, + "nauc_recall_at_1000_max": 0.6396145976080426, + "nauc_recall_at_1000_std": 0.6832780714438885, + "nauc_recall_at_100_diff1": 0.2408891924307149, + "nauc_recall_at_100_max": 0.6415385547577763, + "nauc_recall_at_100_std": 0.6241230779955738, + "nauc_recall_at_10_diff1": 0.31301465223645725, + "nauc_recall_at_10_max": 0.6259268235668424, + "nauc_recall_at_10_std": 0.5067559222657122, + "nauc_recall_at_1_diff1": 0.48340507367292174, + "nauc_recall_at_1_max": 0.5918060803445095, + "nauc_recall_at_1_std": 0.3803959830733999, + "nauc_recall_at_20_diff1": 0.2842973728738141, + "nauc_recall_at_20_max": 0.6290968057402018, + "nauc_recall_at_20_std": 0.5382641411507488, + "nauc_recall_at_3_diff1": 0.36960796666684165, + "nauc_recall_at_3_max": 0.6189674816818908, + "nauc_recall_at_3_std": 0.4484934591367724, + "nauc_recall_at_5_diff1": 0.3386517993718759, + "nauc_recall_at_5_max": 0.6155180401590951, + "nauc_recall_at_5_std": 0.46547602875489463, + "ndcg_at_1": 0.10409, + "ndcg_at_10": 0.17979, + "ndcg_at_100": 0.22198, + "ndcg_at_1000": 0.25766, + "ndcg_at_20": 0.19428, + "ndcg_at_3": 0.14596, + "ndcg_at_5": 0.16271, + "precision_at_1": 0.10409, + "precision_at_10": 0.02699, + "precision_at_100": 0.00481, + "precision_at_1000": 0.00078, + "precision_at_20": 0.01637, + "precision_at_3": 0.05875, + "precision_at_5": 0.04338, + "recall_at_1": 0.10409, + "recall_at_10": 0.26994, + "recall_at_100": 0.48052, + "recall_at_1000": 0.77761, + "recall_at_20": 0.32747, + "recall_at_3": 0.17626, + "recall_at_5": 0.21691 + }, + { + "cv_recall_at_1": 0.68862, + "cv_recall_at_10": 0.94441, + "cv_recall_at_100": 0.99696, + "cv_recall_at_1000": 0.99988, + "cv_recall_at_20": 0.97384, + "cv_recall_at_3": 0.8491, + "cv_recall_at_5": 0.89757, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ], + "main_score": 0.81861, + "map_at_1": 0.68862, + "map_at_10": 0.77802, + "map_at_100": 0.78078, + "map_at_1000": 0.78079, + "map_at_20": 0.7801, + "map_at_3": 0.76039, + "map_at_5": 0.77162, + "mrr_at_1": 0.6886241532352254, + "mrr_at_10": 0.7780242989510651, + "mrr_at_100": 0.780775407295502, + "mrr_at_1000": 0.780794622085721, + "mrr_at_20": 0.7800977472520112, + "mrr_at_3": 0.7603947675776656, + "mrr_at_5": 0.7716187806587189, + "nauc_cv_recall_at_1000_diff1": 0.8692844384055428, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 0.8692844384055428, + "nauc_cv_recall_at_100_diff1": 0.6691478259630901, + "nauc_cv_recall_at_100_max": 0.7475953505943645, + "nauc_cv_recall_at_100_std": 0.27056257097105973, + "nauc_cv_recall_at_10_diff1": 0.6880948148322499, + "nauc_cv_recall_at_10_max": 0.5467060331144916, + "nauc_cv_recall_at_10_std": -0.2425628667761323, + "nauc_cv_recall_at_1_diff1": 0.7838184527245065, + "nauc_cv_recall_at_1_max": 0.4390541428215865, + "nauc_cv_recall_at_1_std": -0.2511815664212096, + "nauc_cv_recall_at_20_diff1": 0.6995952401305364, + "nauc_cv_recall_at_20_max": 0.649355920590757, + "nauc_cv_recall_at_20_std": -0.07975196053310503, + "nauc_cv_recall_at_3_diff1": 0.7060942351703845, + "nauc_cv_recall_at_3_max": 0.5002462807404169, + "nauc_cv_recall_at_3_std": -0.26503056328244606, + "nauc_cv_recall_at_5_diff1": 0.6879778818833843, + "nauc_cv_recall_at_5_max": 0.5076987150481848, + "nauc_cv_recall_at_5_std": -0.27728822139617615, + "nauc_map_at_1000_diff1": 0.7607228052600944, + "nauc_map_at_1000_max": 0.4572493121820054, + "nauc_map_at_1000_std": -0.25597521847200916, + "nauc_map_at_100_diff1": 0.7607113854693041, + "nauc_map_at_100_max": 0.4572754191998793, + "nauc_map_at_100_std": -0.255932878139392, + "nauc_map_at_10_diff1": 0.7596199821431895, + "nauc_map_at_10_max": 0.4577533173217279, + "nauc_map_at_10_std": -0.25703026926053885, + "nauc_map_at_1_diff1": 0.7838184527245065, + "nauc_map_at_1_max": 0.4390541428215865, + "nauc_map_at_1_std": -0.2511815664212096, + "nauc_map_at_20_diff1": 0.7605122214228532, + "nauc_map_at_20_max": 0.4578668309208731, + "nauc_map_at_20_std": -0.2556437602241119, + "nauc_map_at_3_diff1": 0.7571372016948014, + "nauc_map_at_3_max": 0.45891292471001566, + "nauc_map_at_3_std": -0.2575630818052683, + "nauc_map_at_5_diff1": 0.7572888701559919, + "nauc_map_at_5_max": 0.4577447733020841, + "nauc_map_at_5_std": -0.25873803768931697, + "nauc_mrr_at_1000_diff1": 0.7607228052600944, + "nauc_mrr_at_1000_max": 0.4572493121820054, + "nauc_mrr_at_1000_std": -0.25597521847200916, + "nauc_mrr_at_100_diff1": 0.7607113854693041, + "nauc_mrr_at_100_max": 0.4572754191998793, + "nauc_mrr_at_100_std": -0.255932878139392, + "nauc_mrr_at_10_diff1": 0.7596199821431895, + "nauc_mrr_at_10_max": 0.4577533173217279, + "nauc_mrr_at_10_std": -0.25703026926053885, + "nauc_mrr_at_1_diff1": 0.7838184527245065, + "nauc_mrr_at_1_max": 0.4390541428215865, + "nauc_mrr_at_1_std": -0.2511815664212096, + "nauc_mrr_at_20_diff1": 0.7605122214228532, + "nauc_mrr_at_20_max": 0.4578668309208731, + "nauc_mrr_at_20_std": -0.2556437602241119, + "nauc_mrr_at_3_diff1": 0.7571372016948014, + "nauc_mrr_at_3_max": 0.45891292471001566, + "nauc_mrr_at_3_std": -0.2575630818052683, + "nauc_mrr_at_5_diff1": 0.7572888701559919, + "nauc_mrr_at_5_max": 0.4577447733020841, + "nauc_mrr_at_5_std": -0.25873803768931697, + "nauc_ndcg_at_1000_diff1": 0.7578805872601987, + "nauc_ndcg_at_1000_max": 0.46077069695263406, + "nauc_ndcg_at_1000_std": -0.2547039501696542, + "nauc_ndcg_at_100_diff1": 0.7576123162409897, + "nauc_ndcg_at_100_max": 0.4614898486574226, + "nauc_ndcg_at_100_std": -0.25343030205722045, + "nauc_ndcg_at_10_diff1": 0.7524321328014326, + "nauc_ndcg_at_10_max": 0.4658254372938121, + "nauc_ndcg_at_10_std": -0.2564521956378803, + "nauc_ndcg_at_1_diff1": 0.7838184527245065, + "nauc_ndcg_at_1_max": 0.4390541428215865, + "nauc_ndcg_at_1_std": -0.2511815664212096, + "nauc_ndcg_at_20_diff1": 0.7560684300061722, + "nauc_ndcg_at_20_max": 0.46645964477127183, + "nauc_ndcg_at_20_std": -0.25013633353776266, + "nauc_ndcg_at_3_diff1": 0.7472704480142333, + "nauc_ndcg_at_3_max": 0.46684629344125356, + "nauc_ndcg_at_3_std": -0.25908669463469447, + "nauc_ndcg_at_5_diff1": 0.7468090942832256, + "nauc_ndcg_at_5_max": 0.46528854262350067, + "nauc_ndcg_at_5_std": -0.2615309959446548, + "nauc_precision_at_1000_diff1": 0.8692844384061307, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.8692844384061307, + "nauc_precision_at_100_diff1": 0.6691478259631217, + "nauc_precision_at_100_max": 0.7475953505944476, + "nauc_precision_at_100_std": 0.27056257097103237, + "nauc_precision_at_10_diff1": 0.6880948148322517, + "nauc_precision_at_10_max": 0.5467060331144964, + "nauc_precision_at_10_std": -0.24256286677613534, + "nauc_precision_at_1_diff1": 0.7838184527245065, + "nauc_precision_at_1_max": 0.4390541428215865, + "nauc_precision_at_1_std": -0.2511815664212096, + "nauc_precision_at_20_diff1": 0.6995952401305368, + "nauc_precision_at_20_max": 0.6493559205907617, + "nauc_precision_at_20_std": -0.07975196053311341, + "nauc_precision_at_3_diff1": 0.7060942351703826, + "nauc_precision_at_3_max": 0.5002462807404171, + "nauc_precision_at_3_std": -0.26503056328244434, + "nauc_precision_at_5_diff1": 0.687977881883383, + "nauc_precision_at_5_max": 0.5076987150481839, + "nauc_precision_at_5_std": -0.2772882213961815, + "nauc_recall_at_1000_diff1": 0.8692844384055428, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.8692844384055428, + "nauc_recall_at_100_diff1": 0.6691478259630901, + "nauc_recall_at_100_max": 0.7475953505943645, + "nauc_recall_at_100_std": 0.27056257097105973, + "nauc_recall_at_10_diff1": 0.6880948148322499, + "nauc_recall_at_10_max": 0.5467060331144916, + "nauc_recall_at_10_std": -0.2425628667761323, + "nauc_recall_at_1_diff1": 0.7838184527245065, + "nauc_recall_at_1_max": 0.4390541428215865, + "nauc_recall_at_1_std": -0.2511815664212096, + "nauc_recall_at_20_diff1": 0.6995952401305364, + "nauc_recall_at_20_max": 0.649355920590757, + "nauc_recall_at_20_std": -0.07975196053310503, + "nauc_recall_at_3_diff1": 0.7060942351703845, + "nauc_recall_at_3_max": 0.5002462807404169, + "nauc_recall_at_3_std": -0.26503056328244606, + "nauc_recall_at_5_diff1": 0.6879778818833843, + "nauc_recall_at_5_max": 0.5076987150481848, + "nauc_recall_at_5_std": -0.27728822139617615, + "ndcg_at_1": 0.68862, + "ndcg_at_10": 0.81861, + "ndcg_at_100": 0.83055, + "ndcg_at_1000": 0.83095, + "ndcg_at_20": 0.82609, + "ndcg_at_3": 0.78322, + "ndcg_at_5": 0.80331, + "precision_at_1": 0.68862, + "precision_at_10": 0.09444, + "precision_at_100": 0.00997, + "precision_at_1000": 0.001, + "precision_at_20": 0.04869, + "precision_at_3": 0.28303, + "precision_at_5": 0.17951, + "recall_at_1": 0.68862, + "recall_at_10": 0.94441, + "recall_at_100": 0.99696, + "recall_at_1000": 0.99988, + "recall_at_20": 0.97384, + "recall_at_3": 0.8491, + "recall_at_5": 0.89757 + }, + { + "cv_recall_at_1": 0.26514, + "cv_recall_at_10": 0.54569, + "cv_recall_at_100": 0.8125, + "cv_recall_at_1000": 0.97514, + "cv_recall_at_20": 0.62986, + "cv_recall_at_3": 0.39319, + "cv_recall_at_5": 0.45528, + "hf_subset": "he", + "languages": [ + "heb-Hebr" + ], + "main_score": 0.39416, + "map_at_1": 0.26514, + "map_at_10": 0.34707, + "map_at_100": 0.35737, + "map_at_1000": 0.35809, + "map_at_20": 0.35287, + "map_at_3": 0.32095, + "map_at_5": 0.33505, + "mrr_at_1": 0.26513888888888887, + "mrr_at_10": 0.34706889329806007, + "mrr_at_100": 0.3573730842851282, + "mrr_at_1000": 0.35808713808956977, + "mrr_at_20": 0.35286717176979276, + "mrr_at_3": 0.3209490740740735, + "mrr_at_5": 0.3350532407407405, + "nauc_cv_recall_at_1000_diff1": 0.3269851702319663, + "nauc_cv_recall_at_1000_max": 0.8007266221199839, + "nauc_cv_recall_at_1000_std": 0.8604030066402758, + "nauc_cv_recall_at_100_diff1": 0.30351047766360917, + "nauc_cv_recall_at_100_max": 0.6370308989798541, + "nauc_cv_recall_at_100_std": 0.6387209516443855, + "nauc_cv_recall_at_10_diff1": 0.37820827033091164, + "nauc_cv_recall_at_10_max": 0.5316589717810623, + "nauc_cv_recall_at_10_std": 0.4018630792445418, + "nauc_cv_recall_at_1_diff1": 0.5573653194549761, + "nauc_cv_recall_at_1_max": 0.4823866545239, + "nauc_cv_recall_at_1_std": 0.23133760983158186, + "nauc_cv_recall_at_20_diff1": 0.35068725924169186, + "nauc_cv_recall_at_20_max": 0.5735204752668261, + "nauc_cv_recall_at_20_std": 0.4857388404002906, + "nauc_cv_recall_at_3_diff1": 0.44625816393502604, + "nauc_cv_recall_at_3_max": 0.5034472787959484, + "nauc_cv_recall_at_3_std": 0.30912719460546667, + "nauc_cv_recall_at_5_diff1": 0.4136262213740317, + "nauc_cv_recall_at_5_max": 0.5175696018757037, + "nauc_cv_recall_at_5_std": 0.3483951091470087, + "nauc_map_at_1000_diff1": 0.4927579225630324, + "nauc_map_at_1000_max": 0.5000060861294572, + "nauc_map_at_1000_std": 0.2886077873016723, + "nauc_map_at_100_diff1": 0.4923931506441432, + "nauc_map_at_100_max": 0.5001400591264515, + "nauc_map_at_100_std": 0.289072191366851, + "nauc_map_at_10_diff1": 0.4927509866960801, + "nauc_map_at_10_max": 0.49742219248009734, + "nauc_map_at_10_std": 0.2840858256437432, + "nauc_map_at_1_diff1": 0.5573653194549761, + "nauc_map_at_1_max": 0.4823866545239, + "nauc_map_at_1_std": 0.23133760983158186, + "nauc_map_at_20_diff1": 0.49190107685036644, + "nauc_map_at_20_max": 0.49970774159254117, + "nauc_map_at_20_std": 0.2883899724055493, + "nauc_map_at_3_diff1": 0.5049691154013799, + "nauc_map_at_3_max": 0.49270595560391245, + "nauc_map_at_3_std": 0.26857534912218023, + "nauc_map_at_5_diff1": 0.49706923863072766, + "nauc_map_at_5_max": 0.49588840312255467, + "nauc_map_at_5_std": 0.27777587422769684, + "nauc_mrr_at_1000_diff1": 0.4927578897775547, + "nauc_mrr_at_1000_max": 0.5000060857020273, + "nauc_mrr_at_1000_std": 0.2886077752540086, + "nauc_mrr_at_100_diff1": 0.49239311795880597, + "nauc_mrr_at_100_max": 0.5001400586862619, + "nauc_mrr_at_100_std": 0.2890721792953119, + "nauc_mrr_at_10_diff1": 0.4927509866960801, + "nauc_mrr_at_10_max": 0.49742219248009734, + "nauc_mrr_at_10_std": 0.2840858256437432, + "nauc_mrr_at_1_diff1": 0.5573653194549761, + "nauc_mrr_at_1_max": 0.4823866545239, + "nauc_mrr_at_1_std": 0.23133760983158186, + "nauc_mrr_at_20_diff1": 0.49190107685036644, + "nauc_mrr_at_20_max": 0.49970774159254117, + "nauc_mrr_at_20_std": 0.2883899724055493, + "nauc_mrr_at_3_diff1": 0.5049691154013799, + "nauc_mrr_at_3_max": 0.49270595560391245, + "nauc_mrr_at_3_std": 0.26857534912218023, + "nauc_mrr_at_5_diff1": 0.49706923863072766, + "nauc_mrr_at_5_max": 0.49588840312255467, + "nauc_mrr_at_5_std": 0.27777587422769684, + "nauc_ndcg_at_1000_diff1": 0.4744650521139254, + "nauc_ndcg_at_1000_max": 0.5116627297111898, + "nauc_ndcg_at_1000_std": 0.31760456896239264, + "nauc_ndcg_at_100_diff1": 0.4638063088866722, + "nauc_ndcg_at_100_max": 0.5171831990090189, + "nauc_ndcg_at_100_std": 0.3341316651561419, + "nauc_ndcg_at_10_diff1": 0.46499963403197564, + "nauc_ndcg_at_10_max": 0.505441465700467, + "nauc_ndcg_at_10_std": 0.31177303332403883, + "nauc_ndcg_at_1_diff1": 0.5573653194549761, + "nauc_ndcg_at_1_max": 0.4823866545239, + "nauc_ndcg_at_1_std": 0.23133760983158186, + "nauc_ndcg_at_20_diff1": 0.46109580163026737, + "nauc_ndcg_at_20_max": 0.5142500463015515, + "nauc_ndcg_at_20_std": 0.3284962350243677, + "nauc_ndcg_at_3_diff1": 0.48924758859092904, + "nauc_ndcg_at_3_max": 0.49559857295157583, + "nauc_ndcg_at_3_std": 0.2794575762925081, + "nauc_ndcg_at_5_diff1": 0.4753797790260948, + "nauc_ndcg_at_5_max": 0.5014202054503818, + "nauc_ndcg_at_5_std": 0.29593140631395104, + "nauc_precision_at_1000_diff1": 0.3269851702319637, + "nauc_precision_at_1000_max": 0.8007266221199862, + "nauc_precision_at_1000_std": 0.8604030066402799, + "nauc_precision_at_100_diff1": 0.3035104776636093, + "nauc_precision_at_100_max": 0.6370308989798543, + "nauc_precision_at_100_std": 0.6387209516443859, + "nauc_precision_at_10_diff1": 0.3782082703309115, + "nauc_precision_at_10_max": 0.5316589717810623, + "nauc_precision_at_10_std": 0.4018630792445415, + "nauc_precision_at_1_diff1": 0.5573653194549761, + "nauc_precision_at_1_max": 0.4823866545239, + "nauc_precision_at_1_std": 0.23133760983158186, + "nauc_precision_at_20_diff1": 0.3506872592416918, + "nauc_precision_at_20_max": 0.5735204752668259, + "nauc_precision_at_20_std": 0.4857388404002907, + "nauc_precision_at_3_diff1": 0.4462581639350264, + "nauc_precision_at_3_max": 0.5034472787959483, + "nauc_precision_at_3_std": 0.30912719460546667, + "nauc_precision_at_5_diff1": 0.4136262213740317, + "nauc_precision_at_5_max": 0.5175696018757038, + "nauc_precision_at_5_std": 0.34839510914700866, + "nauc_recall_at_1000_diff1": 0.3269851702319663, + "nauc_recall_at_1000_max": 0.8007266221199839, + "nauc_recall_at_1000_std": 0.8604030066402758, + "nauc_recall_at_100_diff1": 0.30351047766360917, + "nauc_recall_at_100_max": 0.6370308989798541, + "nauc_recall_at_100_std": 0.6387209516443855, + "nauc_recall_at_10_diff1": 0.37820827033091164, + "nauc_recall_at_10_max": 0.5316589717810623, + "nauc_recall_at_10_std": 0.4018630792445418, + "nauc_recall_at_1_diff1": 0.5573653194549761, + "nauc_recall_at_1_max": 0.4823866545239, + "nauc_recall_at_1_std": 0.23133760983158186, + "nauc_recall_at_20_diff1": 0.35068725924169186, + "nauc_recall_at_20_max": 0.5735204752668261, + "nauc_recall_at_20_std": 0.4857388404002906, + "nauc_recall_at_3_diff1": 0.44625816393502604, + "nauc_recall_at_3_max": 0.5034472787959484, + "nauc_recall_at_3_std": 0.30912719460546667, + "nauc_recall_at_5_diff1": 0.4136262213740317, + "nauc_recall_at_5_max": 0.5175696018757037, + "nauc_recall_at_5_std": 0.3483951091470087, + "ndcg_at_1": 0.26514, + "ndcg_at_10": 0.39416, + "ndcg_at_100": 0.44892, + "ndcg_at_1000": 0.46923, + "ndcg_at_20": 0.41539, + "ndcg_at_3": 0.33948, + "ndcg_at_5": 0.36497, + "precision_at_1": 0.26514, + "precision_at_10": 0.05457, + "precision_at_100": 0.00812, + "precision_at_1000": 0.00098, + "precision_at_20": 0.03149, + "precision_at_3": 0.13106, + "precision_at_5": 0.09106, + "recall_at_1": 0.26514, + "recall_at_10": 0.54569, + "recall_at_100": 0.8125, + "recall_at_1000": 0.97514, + "recall_at_20": 0.62986, + "recall_at_3": 0.39319, + "recall_at_5": 0.45528 + }, + { + "cv_recall_at_1": 0.05833, + "cv_recall_at_10": 0.21804, + "cv_recall_at_100": 0.54463, + "cv_recall_at_1000": 0.90909, + "cv_recall_at_20": 0.29989, + "cv_recall_at_3": 0.11372, + "cv_recall_at_5": 0.15242, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ], + "main_score": 0.12728, + "map_at_1": 0.05833, + "map_at_10": 0.09962, + "map_at_100": 0.11096, + "map_at_1000": 0.11246, + "map_at_20": 0.10526, + "map_at_3": 0.08211, + "map_at_5": 0.09091, + "mrr_at_1": 0.05833235328707515, + "mrr_at_10": 0.09962170214434136, + "mrr_at_100": 0.11096139209081642, + "mrr_at_1000": 0.11246454103870365, + "mrr_at_20": 0.1052608397121846, + "mrr_at_3": 0.08210827551060423, + "mrr_at_5": 0.09091105100160765, + "nauc_cv_recall_at_1000_diff1": 0.05707411396874622, + "nauc_cv_recall_at_1000_max": 0.4305314431765789, + "nauc_cv_recall_at_1000_std": 0.6174045190761287, + "nauc_cv_recall_at_100_diff1": 0.10092964522892446, + "nauc_cv_recall_at_100_max": 0.29425618904662393, + "nauc_cv_recall_at_100_std": 0.3670656043747271, + "nauc_cv_recall_at_10_diff1": 0.14125562730402333, + "nauc_cv_recall_at_10_max": 0.22871058991291543, + "nauc_cv_recall_at_10_std": 0.19129187889176197, + "nauc_cv_recall_at_1_diff1": 0.3300486422524422, + "nauc_cv_recall_at_1_max": 0.3317286526321265, + "nauc_cv_recall_at_1_std": 0.18794368017024868, + "nauc_cv_recall_at_20_diff1": 0.12612567776569172, + "nauc_cv_recall_at_20_max": 0.2318086938059483, + "nauc_cv_recall_at_20_std": 0.2284786449365189, + "nauc_cv_recall_at_3_diff1": 0.20819459246454827, + "nauc_cv_recall_at_3_max": 0.27691548648139885, + "nauc_cv_recall_at_3_std": 0.18434800757157196, + "nauc_cv_recall_at_5_diff1": 0.17383631683254794, + "nauc_cv_recall_at_5_max": 0.2524853389049951, + "nauc_cv_recall_at_5_std": 0.18646658605239053, + "nauc_map_at_1000_diff1": 0.2234782052076568, + "nauc_map_at_1000_max": 0.2848397578636281, + "nauc_map_at_1000_std": 0.19887907283255718, + "nauc_map_at_100_diff1": 0.22313787879826344, + "nauc_map_at_100_max": 0.2843108110506845, + "nauc_map_at_100_std": 0.1985558390711766, + "nauc_map_at_10_diff1": 0.22840032784693742, + "nauc_map_at_10_max": 0.28147844267923133, + "nauc_map_at_10_std": 0.18601453853576044, + "nauc_map_at_1_diff1": 0.3300486422524422, + "nauc_map_at_1_max": 0.3317286526321265, + "nauc_map_at_1_std": 0.18794368017024868, + "nauc_map_at_20_diff1": 0.22437804015760093, + "nauc_map_at_20_max": 0.2812076570563848, + "nauc_map_at_20_std": 0.19193729091168843, + "nauc_map_at_3_diff1": 0.25646565525415654, + "nauc_map_at_3_max": 0.2999563478016542, + "nauc_map_at_3_std": 0.18478880360825953, + "nauc_map_at_5_diff1": 0.2409823131635885, + "nauc_map_at_5_max": 0.2895968259498882, + "nauc_map_at_5_std": 0.18532356610291725, + "nauc_mrr_at_1000_diff1": 0.2234782052076568, + "nauc_mrr_at_1000_max": 0.2848397578636281, + "nauc_mrr_at_1000_std": 0.19887907283255718, + "nauc_mrr_at_100_diff1": 0.22313787879826344, + "nauc_mrr_at_100_max": 0.2843108110506845, + "nauc_mrr_at_100_std": 0.1985558390711766, + "nauc_mrr_at_10_diff1": 0.22840032784693742, + "nauc_mrr_at_10_max": 0.28147844267923133, + "nauc_mrr_at_10_std": 0.18601453853576044, + "nauc_mrr_at_1_diff1": 0.3300486422524422, + "nauc_mrr_at_1_max": 0.3317286526321265, + "nauc_mrr_at_1_std": 0.18794368017024868, + "nauc_mrr_at_20_diff1": 0.22437804015760093, + "nauc_mrr_at_20_max": 0.2812076570563848, + "nauc_mrr_at_20_std": 0.19193729091168843, + "nauc_mrr_at_3_diff1": 0.25646565525415654, + "nauc_mrr_at_3_max": 0.2999563478016542, + "nauc_mrr_at_3_std": 0.18478880360825953, + "nauc_mrr_at_5_diff1": 0.2409823131635885, + "nauc_mrr_at_5_max": 0.2895968259498882, + "nauc_mrr_at_5_std": 0.18532356610291725, + "nauc_ndcg_at_1000_diff1": 0.1890100975712789, + "nauc_ndcg_at_1000_max": 0.28911610663847015, + "nauc_ndcg_at_1000_std": 0.24423313156730872, + "nauc_ndcg_at_100_diff1": 0.17960027600642617, + "nauc_ndcg_at_100_max": 0.28107378010359907, + "nauc_ndcg_at_100_std": 0.24509423028735777, + "nauc_ndcg_at_10_diff1": 0.19636121499921674, + "nauc_ndcg_at_10_max": 0.2623750702242693, + "nauc_ndcg_at_10_std": 0.18766127725556314, + "nauc_ndcg_at_1_diff1": 0.3300486422524422, + "nauc_ndcg_at_1_max": 0.3317286526321265, + "nauc_ndcg_at_1_std": 0.18794368017024868, + "nauc_ndcg_at_20_diff1": 0.18666766339752958, + "nauc_ndcg_at_20_max": 0.2620930821244513, + "nauc_ndcg_at_20_std": 0.2041134409992863, + "nauc_ndcg_at_3_diff1": 0.24103897094024151, + "nauc_ndcg_at_3_max": 0.2926455989044745, + "nauc_ndcg_at_3_std": 0.18460883817440596, + "nauc_ndcg_at_5_diff1": 0.2181687546993421, + "nauc_ndcg_at_5_max": 0.2771137878870709, + "nauc_ndcg_at_5_std": 0.1856287474451102, + "nauc_precision_at_1000_diff1": 0.05707411396874407, + "nauc_precision_at_1000_max": 0.4305314431765774, + "nauc_precision_at_1000_std": 0.6174045190761295, + "nauc_precision_at_100_diff1": 0.10092964522892454, + "nauc_precision_at_100_max": 0.2942561890466242, + "nauc_precision_at_100_std": 0.36706560437472713, + "nauc_precision_at_10_diff1": 0.14125562730402325, + "nauc_precision_at_10_max": 0.2287105899129151, + "nauc_precision_at_10_std": 0.19129187889176166, + "nauc_precision_at_1_diff1": 0.3300486422524422, + "nauc_precision_at_1_max": 0.3317286526321265, + "nauc_precision_at_1_std": 0.18794368017024868, + "nauc_precision_at_20_diff1": 0.12612567776569164, + "nauc_precision_at_20_max": 0.23180869380594815, + "nauc_precision_at_20_std": 0.22847864493651887, + "nauc_precision_at_3_diff1": 0.20819459246454838, + "nauc_precision_at_3_max": 0.27691548648139863, + "nauc_precision_at_3_std": 0.18434800757157188, + "nauc_precision_at_5_diff1": 0.1738363168325479, + "nauc_precision_at_5_max": 0.2524853389049951, + "nauc_precision_at_5_std": 0.1864665860523905, + "nauc_recall_at_1000_diff1": 0.05707411396874622, + "nauc_recall_at_1000_max": 0.4305314431765789, + "nauc_recall_at_1000_std": 0.6174045190761287, + "nauc_recall_at_100_diff1": 0.10092964522892446, + "nauc_recall_at_100_max": 0.29425618904662393, + "nauc_recall_at_100_std": 0.3670656043747271, + "nauc_recall_at_10_diff1": 0.14125562730402333, + "nauc_recall_at_10_max": 0.22871058991291543, + "nauc_recall_at_10_std": 0.19129187889176197, + "nauc_recall_at_1_diff1": 0.3300486422524422, + "nauc_recall_at_1_max": 0.3317286526321265, + "nauc_recall_at_1_std": 0.18794368017024868, + "nauc_recall_at_20_diff1": 0.12612567776569172, + "nauc_recall_at_20_max": 0.2318086938059483, + "nauc_recall_at_20_std": 0.2284786449365189, + "nauc_recall_at_3_diff1": 0.20819459246454827, + "nauc_recall_at_3_max": 0.27691548648139885, + "nauc_recall_at_3_std": 0.18434800757157196, + "nauc_recall_at_5_diff1": 0.17383631683254794, + "nauc_recall_at_5_max": 0.2524853389049951, + "nauc_recall_at_5_std": 0.18646658605239053, + "ndcg_at_1": 0.05833, + "ndcg_at_10": 0.12728, + "ndcg_at_100": 0.19216, + "ndcg_at_1000": 0.23713, + "ndcg_at_20": 0.14792, + "ndcg_at_3": 0.0902, + "ndcg_at_5": 0.1061, + "precision_at_1": 0.05833, + "precision_at_10": 0.0218, + "precision_at_100": 0.00545, + "precision_at_1000": 0.00091, + "precision_at_20": 0.01499, + "precision_at_3": 0.03791, + "precision_at_5": 0.03048, + "recall_at_1": 0.05833, + "recall_at_10": 0.21804, + "recall_at_100": 0.54463, + "recall_at_1000": 0.90909, + "recall_at_20": 0.29989, + "recall_at_3": 0.11372, + "recall_at_5": 0.15242 + }, + { + "cv_recall_at_1": 0.30014, + "cv_recall_at_10": 0.61264, + "cv_recall_at_100": 0.86044, + "cv_recall_at_1000": 0.97569, + "cv_recall_at_20": 0.69876, + "cv_recall_at_3": 0.45371, + "cv_recall_at_5": 0.51909, + "hf_subset": "hr", + "languages": [ + "hrv-Latn" + ], + "main_score": 0.44666, + "map_at_1": 0.30014, + "map_at_10": 0.39473, + "map_at_100": 0.40487, + "map_at_1000": 0.4054, + "map_at_20": 0.40078, + "map_at_3": 0.36726, + "map_at_5": 0.38221, + "mrr_at_1": 0.3001373626373626, + "mrr_at_10": 0.39472549276120766, + "mrr_at_100": 0.4048696521882351, + "mrr_at_1000": 0.4054039192989718, + "mrr_at_20": 0.4007758807961287, + "mrr_at_3": 0.3672619047619057, + "mrr_at_5": 0.38220695970695995, + "nauc_cv_recall_at_1000_diff1": 0.3586726592708677, + "nauc_cv_recall_at_1000_max": 0.8499844382197388, + "nauc_cv_recall_at_1000_std": 0.9014728301866921, + "nauc_cv_recall_at_100_diff1": 0.3620273025034925, + "nauc_cv_recall_at_100_max": 0.7129467387403892, + "nauc_cv_recall_at_100_std": 0.6609581262359034, + "nauc_cv_recall_at_10_diff1": 0.3842176703377357, + "nauc_cv_recall_at_10_max": 0.5471484927211087, + "nauc_cv_recall_at_10_std": 0.3463565927031771, + "nauc_cv_recall_at_1_diff1": 0.5663570758468127, + "nauc_cv_recall_at_1_max": 0.4682186108963969, + "nauc_cv_recall_at_1_std": 0.17349940936789837, + "nauc_cv_recall_at_20_diff1": 0.37088773390466634, + "nauc_cv_recall_at_20_max": 0.5907304165964303, + "nauc_cv_recall_at_20_std": 0.4283686973553312, + "nauc_cv_recall_at_3_diff1": 0.4475332706202563, + "nauc_cv_recall_at_3_max": 0.49930415860465016, + "nauc_cv_recall_at_3_std": 0.24217892102168642, + "nauc_cv_recall_at_5_diff1": 0.41165724944078125, + "nauc_cv_recall_at_5_max": 0.5111652414669552, + "nauc_cv_recall_at_5_std": 0.2742803051020646, + "nauc_map_at_1000_diff1": 0.5042398721002027, + "nauc_map_at_1000_max": 0.4905559146440463, + "nauc_map_at_1000_std": 0.22119873498521436, + "nauc_map_at_100_diff1": 0.5040238826821266, + "nauc_map_at_100_max": 0.49077281353278235, + "nauc_map_at_100_std": 0.22164398487988576, + "nauc_map_at_10_diff1": 0.5025445743495411, + "nauc_map_at_10_max": 0.4881575618691484, + "nauc_map_at_10_std": 0.2174519067162409, + "nauc_map_at_1_diff1": 0.5663570758468127, + "nauc_map_at_1_max": 0.4682186108963969, + "nauc_map_at_1_std": 0.17349940936789837, + "nauc_map_at_20_diff1": 0.5028888029066095, + "nauc_map_at_20_max": 0.49009005537891887, + "nauc_map_at_20_std": 0.22069609562067313, + "nauc_map_at_3_diff1": 0.5123125544273271, + "nauc_map_at_3_max": 0.4813389503783199, + "nauc_map_at_3_std": 0.20237276561219283, + "nauc_map_at_5_diff1": 0.504540343021302, + "nauc_map_at_5_max": 0.4839795117311297, + "nauc_map_at_5_std": 0.20949681898134187, + "nauc_mrr_at_1000_diff1": 0.5042398721002027, + "nauc_mrr_at_1000_max": 0.4905559146440463, + "nauc_mrr_at_1000_std": 0.22119873498521436, + "nauc_mrr_at_100_diff1": 0.5040238826821266, + "nauc_mrr_at_100_max": 0.49077281353278235, + "nauc_mrr_at_100_std": 0.22164398487988576, + "nauc_mrr_at_10_diff1": 0.5025445743495411, + "nauc_mrr_at_10_max": 0.4881575618691484, + "nauc_mrr_at_10_std": 0.2174519067162409, + "nauc_mrr_at_1_diff1": 0.5663570758468127, + "nauc_mrr_at_1_max": 0.4682186108963969, + "nauc_mrr_at_1_std": 0.17349940936789837, + "nauc_mrr_at_20_diff1": 0.5028888029066095, + "nauc_mrr_at_20_max": 0.49009005537891887, + "nauc_mrr_at_20_std": 0.22069609562067313, + "nauc_mrr_at_3_diff1": 0.5123125544273271, + "nauc_mrr_at_3_max": 0.4813389503783199, + "nauc_mrr_at_3_std": 0.20237276561219283, + "nauc_mrr_at_5_diff1": 0.504540343021302, + "nauc_mrr_at_5_max": 0.4839795117311297, + "nauc_mrr_at_5_std": 0.20949681898134187, + "nauc_ndcg_at_1000_diff1": 0.48946212025389374, + "nauc_ndcg_at_1000_max": 0.5051410930621354, + "nauc_ndcg_at_1000_std": 0.25004833305588986, + "nauc_ndcg_at_100_diff1": 0.48360996856535743, + "nauc_ndcg_at_100_max": 0.5128779666752031, + "nauc_ndcg_at_100_std": 0.2657005413472347, + "nauc_ndcg_at_10_diff1": 0.4756950853648613, + "nauc_ndcg_at_10_max": 0.5008947200817642, + "nauc_ndcg_at_10_std": 0.24531428168736577, + "nauc_ndcg_at_1_diff1": 0.5663570758468127, + "nauc_ndcg_at_1_max": 0.4682186108963969, + "nauc_ndcg_at_1_std": 0.17349940936789837, + "nauc_ndcg_at_20_diff1": 0.47626368707300626, + "nauc_ndcg_at_20_max": 0.5082725864196117, + "nauc_ndcg_at_20_std": 0.25838814733070764, + "nauc_ndcg_at_3_diff1": 0.49549491996306794, + "nauc_ndcg_at_3_max": 0.4859596037574002, + "nauc_ndcg_at_3_std": 0.21260631844181213, + "nauc_ndcg_at_5_diff1": 0.4814150548165864, + "nauc_ndcg_at_5_max": 0.49068965409631315, + "nauc_ndcg_at_5_std": 0.22542235043204617, + "nauc_precision_at_1000_diff1": 0.35867265927085856, + "nauc_precision_at_1000_max": 0.8499844382197332, + "nauc_precision_at_1000_std": 0.9014728301866906, + "nauc_precision_at_100_diff1": 0.3620273025034949, + "nauc_precision_at_100_max": 0.712946738740391, + "nauc_precision_at_100_std": 0.6609581262359061, + "nauc_precision_at_10_diff1": 0.38421767033773546, + "nauc_precision_at_10_max": 0.5471484927211082, + "nauc_precision_at_10_std": 0.3463565927031767, + "nauc_precision_at_1_diff1": 0.5663570758468127, + "nauc_precision_at_1_max": 0.4682186108963969, + "nauc_precision_at_1_std": 0.17349940936789837, + "nauc_precision_at_20_diff1": 0.3708877339046656, + "nauc_precision_at_20_max": 0.5907304165964296, + "nauc_precision_at_20_std": 0.4283686973553297, + "nauc_precision_at_3_diff1": 0.4475332706202558, + "nauc_precision_at_3_max": 0.49930415860465016, + "nauc_precision_at_3_std": 0.24217892102168648, + "nauc_precision_at_5_diff1": 0.4116572494407812, + "nauc_precision_at_5_max": 0.5111652414669557, + "nauc_precision_at_5_std": 0.2742803051020649, + "nauc_recall_at_1000_diff1": 0.3586726592708677, + "nauc_recall_at_1000_max": 0.8499844382197388, + "nauc_recall_at_1000_std": 0.9014728301866921, + "nauc_recall_at_100_diff1": 0.3620273025034925, + "nauc_recall_at_100_max": 0.7129467387403892, + "nauc_recall_at_100_std": 0.6609581262359034, + "nauc_recall_at_10_diff1": 0.3842176703377357, + "nauc_recall_at_10_max": 0.5471484927211087, + "nauc_recall_at_10_std": 0.3463565927031771, + "nauc_recall_at_1_diff1": 0.5663570758468127, + "nauc_recall_at_1_max": 0.4682186108963969, + "nauc_recall_at_1_std": 0.17349940936789837, + "nauc_recall_at_20_diff1": 0.37088773390466634, + "nauc_recall_at_20_max": 0.5907304165964303, + "nauc_recall_at_20_std": 0.4283686973553312, + "nauc_recall_at_3_diff1": 0.4475332706202563, + "nauc_recall_at_3_max": 0.49930415860465016, + "nauc_recall_at_3_std": 0.24217892102168642, + "nauc_recall_at_5_diff1": 0.41165724944078125, + "nauc_recall_at_5_max": 0.5111652414669552, + "nauc_recall_at_5_std": 0.2742803051020646, + "ndcg_at_1": 0.30014, + "ndcg_at_10": 0.44666, + "ndcg_at_100": 0.4984, + "ndcg_at_1000": 0.51294, + "ndcg_at_20": 0.46853, + "ndcg_at_3": 0.38944, + "ndcg_at_5": 0.41637, + "precision_at_1": 0.30014, + "precision_at_10": 0.06126, + "precision_at_100": 0.0086, + "precision_at_1000": 0.00098, + "precision_at_20": 0.03494, + "precision_at_3": 0.15124, + "precision_at_5": 0.10382, + "recall_at_1": 0.30014, + "recall_at_10": 0.61264, + "recall_at_100": 0.86044, + "recall_at_1000": 0.97569, + "recall_at_20": 0.69876, + "recall_at_3": 0.45371, + "recall_at_5": 0.51909 + }, + { + "cv_recall_at_1": 0.12957, + "cv_recall_at_10": 0.34202, + "cv_recall_at_100": 0.61572, + "cv_recall_at_1000": 0.89676, + "cv_recall_at_20": 0.41602, + "cv_recall_at_3": 0.21605, + "cv_recall_at_5": 0.264, + "hf_subset": "hu", + "languages": [ + "hun-Latn" + ], + "main_score": 0.22415, + "map_at_1": 0.12957, + "map_at_10": 0.18788, + "map_at_100": 0.1978, + "map_at_1000": 0.19892, + "map_at_20": 0.19295, + "map_at_3": 0.16655, + "map_at_5": 0.1775, + "mrr_at_1": 0.12957317073170732, + "mrr_at_10": 0.1878756533101041, + "mrr_at_100": 0.1978017747317187, + "mrr_at_1000": 0.19891865771814843, + "mrr_at_20": 0.1929485400897659, + "mrr_at_3": 0.16655118255728013, + "mrr_at_5": 0.17749907612712537, + "nauc_cv_recall_at_1000_diff1": 0.1926685407697348, + "nauc_cv_recall_at_1000_max": 0.6660348086677469, + "nauc_cv_recall_at_1000_std": 0.7471546962869257, + "nauc_cv_recall_at_100_diff1": 0.21347335173344084, + "nauc_cv_recall_at_100_max": 0.5658371871490846, + "nauc_cv_recall_at_100_std": 0.6158093204180968, + "nauc_cv_recall_at_10_diff1": 0.24759927370543935, + "nauc_cv_recall_at_10_max": 0.48185439575113903, + "nauc_cv_recall_at_10_std": 0.4256128948881968, + "nauc_cv_recall_at_1_diff1": 0.46692725912228233, + "nauc_cv_recall_at_1_max": 0.48419673225706406, + "nauc_cv_recall_at_1_std": 0.32274242950108506, + "nauc_cv_recall_at_20_diff1": 0.22756116239806018, + "nauc_cv_recall_at_20_max": 0.5193004941893448, + "nauc_cv_recall_at_20_std": 0.49708893210813393, + "nauc_cv_recall_at_3_diff1": 0.32531876464137166, + "nauc_cv_recall_at_3_max": 0.46145983642570054, + "nauc_cv_recall_at_3_std": 0.35095283380672254, + "nauc_cv_recall_at_5_diff1": 0.28815505370161415, + "nauc_cv_recall_at_5_max": 0.4686756489676717, + "nauc_cv_recall_at_5_std": 0.38257551233467973, + "nauc_map_at_1000_diff1": 0.3702845352984357, + "nauc_map_at_1000_max": 0.4817317513479292, + "nauc_map_at_1000_std": 0.3647306036257812, + "nauc_map_at_100_diff1": 0.3698677927453357, + "nauc_map_at_100_max": 0.4816904952589892, + "nauc_map_at_100_std": 0.36499745995427846, + "nauc_map_at_10_diff1": 0.3718401888534464, + "nauc_map_at_10_max": 0.47669317984440557, + "nauc_map_at_10_std": 0.35466092452935705, + "nauc_map_at_1_diff1": 0.46692725912228233, + "nauc_map_at_1_max": 0.48419673225706406, + "nauc_map_at_1_std": 0.32274242950108506, + "nauc_map_at_20_diff1": 0.3694959663934745, + "nauc_map_at_20_max": 0.4804261018834059, + "nauc_map_at_20_std": 0.3618955962942498, + "nauc_map_at_3_diff1": 0.39376791566150743, + "nauc_map_at_3_max": 0.4729620605043809, + "nauc_map_at_3_std": 0.33694067222910934, + "nauc_map_at_5_diff1": 0.38131559850812324, + "nauc_map_at_5_max": 0.47484262669734634, + "nauc_map_at_5_std": 0.3465880790071037, + "nauc_mrr_at_1000_diff1": 0.3702845352984357, + "nauc_mrr_at_1000_max": 0.4817317513479292, + "nauc_mrr_at_1000_std": 0.3647306036257812, + "nauc_mrr_at_100_diff1": 0.3698677927453357, + "nauc_mrr_at_100_max": 0.4816904952589892, + "nauc_mrr_at_100_std": 0.36499745995427846, + "nauc_mrr_at_10_diff1": 0.3718401888534464, + "nauc_mrr_at_10_max": 0.47669317984440557, + "nauc_mrr_at_10_std": 0.35466092452935705, + "nauc_mrr_at_1_diff1": 0.46692725912228233, + "nauc_mrr_at_1_max": 0.48419673225706406, + "nauc_mrr_at_1_std": 0.32274242950108506, + "nauc_mrr_at_20_diff1": 0.3694959663934745, + "nauc_mrr_at_20_max": 0.4804261018834059, + "nauc_mrr_at_20_std": 0.3618955962942498, + "nauc_mrr_at_3_diff1": 0.39376791566150743, + "nauc_mrr_at_3_max": 0.4729620605043809, + "nauc_mrr_at_3_std": 0.33694067222910934, + "nauc_mrr_at_5_diff1": 0.38131559850812324, + "nauc_mrr_at_5_max": 0.47484262669734634, + "nauc_mrr_at_5_std": 0.3465880790071037, + "nauc_ndcg_at_1000_diff1": 0.33817509632392545, + "nauc_ndcg_at_1000_max": 0.49769845184107936, + "nauc_ndcg_at_1000_std": 0.40885320217687693, + "nauc_ndcg_at_100_diff1": 0.32797720523689905, + "nauc_ndcg_at_100_max": 0.49853128913777867, + "nauc_ndcg_at_100_std": 0.4199722248423589, + "nauc_ndcg_at_10_diff1": 0.3343020690828065, + "nauc_ndcg_at_10_max": 0.47766588097578827, + "nauc_ndcg_at_10_std": 0.37506780176899107, + "nauc_ndcg_at_1_diff1": 0.46692725912228233, + "nauc_ndcg_at_1_max": 0.48419673225706406, + "nauc_ndcg_at_1_std": 0.32274242950108506, + "nauc_ndcg_at_20_diff1": 0.3270780139351324, + "nauc_ndcg_at_20_max": 0.4898620209866191, + "nauc_ndcg_at_20_std": 0.39862570492352634, + "nauc_ndcg_at_3_diff1": 0.37372291907645316, + "nauc_ndcg_at_3_max": 0.4696137532695068, + "nauc_ndcg_at_3_std": 0.34102932389855384, + "nauc_ndcg_at_5_diff1": 0.3534976171106256, + "nauc_ndcg_at_5_max": 0.47287581533045453, + "nauc_ndcg_at_5_std": 0.35706313498705516, + "nauc_precision_at_1000_diff1": 0.19266854076973416, + "nauc_precision_at_1000_max": 0.6660348086677451, + "nauc_precision_at_1000_std": 0.7471546962869254, + "nauc_precision_at_100_diff1": 0.21347335173344126, + "nauc_precision_at_100_max": 0.5658371871490853, + "nauc_precision_at_100_std": 0.6158093204180971, + "nauc_precision_at_10_diff1": 0.24759927370543952, + "nauc_precision_at_10_max": 0.4818543957511388, + "nauc_precision_at_10_std": 0.42561289488819687, + "nauc_precision_at_1_diff1": 0.46692725912228233, + "nauc_precision_at_1_max": 0.48419673225706406, + "nauc_precision_at_1_std": 0.32274242950108506, + "nauc_precision_at_20_diff1": 0.22756116239805987, + "nauc_precision_at_20_max": 0.5193004941893443, + "nauc_precision_at_20_std": 0.49708893210813343, + "nauc_precision_at_3_diff1": 0.32531876464137177, + "nauc_precision_at_3_max": 0.4614598364257008, + "nauc_precision_at_3_std": 0.35095283380672265, + "nauc_precision_at_5_diff1": 0.2881550537016138, + "nauc_precision_at_5_max": 0.4686756489676716, + "nauc_precision_at_5_std": 0.38257551233467957, + "nauc_recall_at_1000_diff1": 0.1926685407697348, + "nauc_recall_at_1000_max": 0.6660348086677469, + "nauc_recall_at_1000_std": 0.7471546962869257, + "nauc_recall_at_100_diff1": 0.21347335173344084, + "nauc_recall_at_100_max": 0.5658371871490846, + "nauc_recall_at_100_std": 0.6158093204180968, + "nauc_recall_at_10_diff1": 0.24759927370543935, + "nauc_recall_at_10_max": 0.48185439575113903, + "nauc_recall_at_10_std": 0.4256128948881968, + "nauc_recall_at_1_diff1": 0.46692725912228233, + "nauc_recall_at_1_max": 0.48419673225706406, + "nauc_recall_at_1_std": 0.32274242950108506, + "nauc_recall_at_20_diff1": 0.22756116239806018, + "nauc_recall_at_20_max": 0.5193004941893448, + "nauc_recall_at_20_std": 0.49708893210813393, + "nauc_recall_at_3_diff1": 0.32531876464137166, + "nauc_recall_at_3_max": 0.46145983642570054, + "nauc_recall_at_3_std": 0.35095283380672254, + "nauc_recall_at_5_diff1": 0.28815505370161415, + "nauc_recall_at_5_max": 0.4686756489676717, + "nauc_recall_at_5_std": 0.38257551233467973, + "ndcg_at_1": 0.12957, + "ndcg_at_10": 0.22415, + "ndcg_at_100": 0.27927, + "ndcg_at_1000": 0.31368, + "ndcg_at_20": 0.24278, + "ndcg_at_3": 0.17922, + "ndcg_at_5": 0.19895, + "precision_at_1": 0.12957, + "precision_at_10": 0.0342, + "precision_at_100": 0.00616, + "precision_at_1000": 0.0009, + "precision_at_20": 0.0208, + "precision_at_3": 0.07202, + "precision_at_5": 0.0528, + "recall_at_1": 0.12957, + "recall_at_10": 0.34202, + "recall_at_100": 0.61572, + "recall_at_1000": 0.89676, + "recall_at_20": 0.41602, + "recall_at_3": 0.21605, + "recall_at_5": 0.264 + }, + { + "cv_recall_at_1": 0.62686, + "cv_recall_at_10": 0.90668, + "cv_recall_at_100": 0.98976, + "cv_recall_at_1000": 0.99958, + "cv_recall_at_20": 0.94583, + "cv_recall_at_3": 0.79006, + "cv_recall_at_5": 0.8462, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.76524, + "map_at_1": 0.62686, + "map_at_10": 0.72007, + "map_at_100": 0.72415, + "map_at_1000": 0.72421, + "map_at_20": 0.72287, + "map_at_3": 0.69894, + "map_at_5": 0.71199, + "mrr_at_1": 0.6268593881560482, + "mrr_at_10": 0.7200742084653106, + "mrr_at_100": 0.7241543067524624, + "mrr_at_1000": 0.7242108276755566, + "mrr_at_20": 0.7228741256671652, + "mrr_at_3": 0.6989428384320333, + "mrr_at_5": 0.7119866217606866, + "nauc_cv_recall_at_1000_diff1": 0.8078060515091644, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 0.9074064093008918, + "nauc_cv_recall_at_100_diff1": 0.6432568018093953, + "nauc_cv_recall_at_100_max": 0.8536611954812929, + "nauc_cv_recall_at_100_std": 0.6027490773953507, + "nauc_cv_recall_at_10_diff1": 0.6657814723494743, + "nauc_cv_recall_at_10_max": 0.6957225433890553, + "nauc_cv_recall_at_10_std": 0.0022072977408623567, + "nauc_cv_recall_at_1_diff1": 0.7792706869820992, + "nauc_cv_recall_at_1_max": 0.5719054031807366, + "nauc_cv_recall_at_1_std": -0.09248351745761402, + "nauc_cv_recall_at_20_diff1": 0.6667407251770318, + "nauc_cv_recall_at_20_max": 0.7637506812219734, + "nauc_cv_recall_at_20_std": 0.11928232129370707, + "nauc_cv_recall_at_3_diff1": 0.6908769200523394, + "nauc_cv_recall_at_3_max": 0.6350382272098436, + "nauc_cv_recall_at_3_std": -0.08527690593618807, + "nauc_cv_recall_at_5_diff1": 0.6860261523249013, + "nauc_cv_recall_at_5_max": 0.6741805654950536, + "nauc_cv_recall_at_5_std": -0.03915985834128877, + "nauc_map_at_1000_diff1": 0.7512516676979538, + "nauc_map_at_1000_max": 0.5969291195432564, + "nauc_map_at_1000_std": -0.08207223388574669, + "nauc_map_at_100_diff1": 0.7512142978388329, + "nauc_map_at_100_max": 0.5969853338639733, + "nauc_map_at_100_std": -0.08191885206401718, + "nauc_map_at_10_diff1": 0.7497104407031654, + "nauc_map_at_10_max": 0.5977098586582251, + "nauc_map_at_10_std": -0.08277371394360927, + "nauc_map_at_1_diff1": 0.7792706869820992, + "nauc_map_at_1_max": 0.5719054031807366, + "nauc_map_at_1_std": -0.09248351745761402, + "nauc_map_at_20_diff1": 0.7507293057842352, + "nauc_map_at_20_max": 0.5976806632240285, + "nauc_map_at_20_std": -0.08176207226315944, + "nauc_map_at_3_diff1": 0.7473253850027027, + "nauc_map_at_3_max": 0.595246565048443, + "nauc_map_at_3_std": -0.08948415715129338, + "nauc_map_at_5_diff1": 0.7490489669338255, + "nauc_map_at_5_max": 0.5987530936259463, + "nauc_map_at_5_std": -0.08367155879418912, + "nauc_mrr_at_1000_diff1": 0.7512516676979538, + "nauc_mrr_at_1000_max": 0.5969291195432564, + "nauc_mrr_at_1000_std": -0.08207223388574669, + "nauc_mrr_at_100_diff1": 0.7512142978388329, + "nauc_mrr_at_100_max": 0.5969853338639733, + "nauc_mrr_at_100_std": -0.08191885206401718, + "nauc_mrr_at_10_diff1": 0.7497104407031654, + "nauc_mrr_at_10_max": 0.5977098586582251, + "nauc_mrr_at_10_std": -0.08277371394360927, + "nauc_mrr_at_1_diff1": 0.7792706869820992, + "nauc_mrr_at_1_max": 0.5719054031807366, + "nauc_mrr_at_1_std": -0.09248351745761402, + "nauc_mrr_at_20_diff1": 0.7507293057842352, + "nauc_mrr_at_20_max": 0.5976806632240285, + "nauc_mrr_at_20_std": -0.08176207226315944, + "nauc_mrr_at_3_diff1": 0.7473253850027027, + "nauc_mrr_at_3_max": 0.595246565048443, + "nauc_mrr_at_3_std": -0.08948415715129338, + "nauc_mrr_at_5_diff1": 0.7490489669338255, + "nauc_mrr_at_5_max": 0.5987530936259463, + "nauc_mrr_at_5_std": -0.08367155879418912, + "nauc_ndcg_at_1000_diff1": 0.7472767756637274, + "nauc_ndcg_at_1000_max": 0.6023894445439884, + "nauc_ndcg_at_1000_std": -0.07606050765907078, + "nauc_ndcg_at_100_diff1": 0.746430499512041, + "nauc_ndcg_at_100_max": 0.6040946792686888, + "nauc_ndcg_at_100_std": -0.07141106721599925, + "nauc_ndcg_at_10_diff1": 0.7391483459861079, + "nauc_ndcg_at_10_max": 0.6095221940049876, + "nauc_ndcg_at_10_std": -0.07343255503981704, + "nauc_ndcg_at_1_diff1": 0.7792706869820992, + "nauc_ndcg_at_1_max": 0.5719054031807366, + "nauc_ndcg_at_1_std": -0.09248351745761402, + "nauc_ndcg_at_20_diff1": 0.7430368326541804, + "nauc_ndcg_at_20_max": 0.6099604556357914, + "nauc_ndcg_at_20_std": -0.0687887602859378, + "nauc_ndcg_at_3_diff1": 0.735431504924604, + "nauc_ndcg_at_3_max": 0.6036553110308045, + "nauc_ndcg_at_3_std": -0.08857924253599371, + "nauc_ndcg_at_5_diff1": 0.7379351281643888, + "nauc_ndcg_at_5_max": 0.6115526644589184, + "nauc_ndcg_at_5_std": -0.07649475310316721, + "nauc_precision_at_1000_diff1": 0.8078060515090997, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.9074064093009346, + "nauc_precision_at_100_diff1": 0.643256801809399, + "nauc_precision_at_100_max": 0.8536611954813077, + "nauc_precision_at_100_std": 0.6027490773953564, + "nauc_precision_at_10_diff1": 0.665781472349474, + "nauc_precision_at_10_max": 0.6957225433890558, + "nauc_precision_at_10_std": 0.002207297740855719, + "nauc_precision_at_1_diff1": 0.7792706869820992, + "nauc_precision_at_1_max": 0.5719054031807366, + "nauc_precision_at_1_std": -0.09248351745761402, + "nauc_precision_at_20_diff1": 0.6667407251770338, + "nauc_precision_at_20_max": 0.7637506812219743, + "nauc_precision_at_20_std": 0.11928232129370485, + "nauc_precision_at_3_diff1": 0.690876920052339, + "nauc_precision_at_3_max": 0.6350382272098447, + "nauc_precision_at_3_std": -0.08527690593618771, + "nauc_precision_at_5_diff1": 0.6860261523249014, + "nauc_precision_at_5_max": 0.6741805654950541, + "nauc_precision_at_5_std": -0.03915985834128767, + "nauc_recall_at_1000_diff1": 0.8078060515091644, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.9074064093008918, + "nauc_recall_at_100_diff1": 0.6432568018093953, + "nauc_recall_at_100_max": 0.8536611954812929, + "nauc_recall_at_100_std": 0.6027490773953507, + "nauc_recall_at_10_diff1": 0.6657814723494743, + "nauc_recall_at_10_max": 0.6957225433890553, + "nauc_recall_at_10_std": 0.0022072977408623567, + "nauc_recall_at_1_diff1": 0.7792706869820992, + "nauc_recall_at_1_max": 0.5719054031807366, + "nauc_recall_at_1_std": -0.09248351745761402, + "nauc_recall_at_20_diff1": 0.6667407251770318, + "nauc_recall_at_20_max": 0.7637506812219734, + "nauc_recall_at_20_std": 0.11928232129370707, + "nauc_recall_at_3_diff1": 0.6908769200523394, + "nauc_recall_at_3_max": 0.6350382272098436, + "nauc_recall_at_3_std": -0.08527690593618807, + "nauc_recall_at_5_diff1": 0.6860261523249013, + "nauc_recall_at_5_max": 0.6741805654950536, + "nauc_recall_at_5_std": -0.03915985834128877, + "ndcg_at_1": 0.62686, + "ndcg_at_10": 0.76524, + "ndcg_at_100": 0.78368, + "ndcg_at_1000": 0.78498, + "ndcg_at_20": 0.77524, + "ndcg_at_3": 0.72235, + "ndcg_at_5": 0.74566, + "precision_at_1": 0.62686, + "precision_at_10": 0.09067, + "precision_at_100": 0.0099, + "precision_at_1000": 0.001, + "precision_at_20": 0.04729, + "precision_at_3": 0.26335, + "precision_at_5": 0.16924, + "recall_at_1": 0.62686, + "recall_at_10": 0.90668, + "recall_at_100": 0.98976, + "recall_at_1000": 0.99958, + "recall_at_20": 0.94583, + "recall_at_3": 0.79006, + "recall_at_5": 0.8462 + }, + { + "cv_recall_at_1": 0.67631, + "cv_recall_at_10": 0.93873, + "cv_recall_at_100": 0.99469, + "cv_recall_at_1000": 0.99988, + "cv_recall_at_20": 0.96777, + "cv_recall_at_3": 0.8391, + "cv_recall_at_5": 0.89045, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ], + "main_score": 0.8093, + "map_at_1": 0.67631, + "map_at_10": 0.76757, + "map_at_100": 0.77043, + "map_at_1000": 0.77046, + "map_at_20": 0.76965, + "map_at_3": 0.7491, + "map_at_5": 0.76093, + "mrr_at_1": 0.6763074017235273, + "mrr_at_10": 0.7675706846701964, + "mrr_at_100": 0.7704339580269138, + "mrr_at_1000": 0.7704636853468644, + "mrr_at_20": 0.7696493226681896, + "mrr_at_3": 0.7491047888875737, + "mrr_at_5": 0.7609274780624032, + "nauc_cv_recall_at_1000_diff1": 0.7222282767327224, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.6081278002284253, + "nauc_cv_recall_at_100_max": 0.8488622992394228, + "nauc_cv_recall_at_100_std": 0.5571362488972622, + "nauc_cv_recall_at_10_diff1": 0.6628892757554061, + "nauc_cv_recall_at_10_max": 0.6565974764765896, + "nauc_cv_recall_at_10_std": -0.08043915108181712, + "nauc_cv_recall_at_1_diff1": 0.7623652046686186, + "nauc_cv_recall_at_1_max": 0.4681979463974185, + "nauc_cv_recall_at_1_std": -0.21274061728203167, + "nauc_cv_recall_at_20_diff1": 0.6662127485138235, + "nauc_cv_recall_at_20_max": 0.7161591219904855, + "nauc_cv_recall_at_20_std": 0.018820578206631276, + "nauc_cv_recall_at_3_diff1": 0.6943259930732723, + "nauc_cv_recall_at_3_max": 0.5333482867271512, + "nauc_cv_recall_at_3_std": -0.22973946875310663, + "nauc_cv_recall_at_5_diff1": 0.6836306948389111, + "nauc_cv_recall_at_5_max": 0.5859049187975784, + "nauc_cv_recall_at_5_std": -0.16199264140836703, + "nauc_map_at_1000_diff1": 0.7411736126116106, + "nauc_map_at_1000_max": 0.4917828756376569, + "nauc_map_at_1000_std": -0.20683083313104692, + "nauc_map_at_100_diff1": 0.7411562255890868, + "nauc_map_at_100_max": 0.4918349144235231, + "nauc_map_at_100_std": -0.20672737625036824, + "nauc_map_at_10_diff1": 0.7400330232348077, + "nauc_map_at_10_max": 0.49357475828973923, + "nauc_map_at_10_std": -0.20606748745004916, + "nauc_map_at_1_diff1": 0.7623652046686186, + "nauc_map_at_1_max": 0.4681979463974185, + "nauc_map_at_1_std": -0.21274061728203167, + "nauc_map_at_20_diff1": 0.7408903405171129, + "nauc_map_at_20_max": 0.49254548529020414, + "nauc_map_at_20_std": -0.20637267290486805, + "nauc_map_at_3_diff1": 0.7382633309068931, + "nauc_map_at_3_max": 0.4892418217193733, + "nauc_map_at_3_std": -0.21665425397028518, + "nauc_map_at_5_diff1": 0.7391457643885568, + "nauc_map_at_5_max": 0.4935053812177111, + "nauc_map_at_5_std": -0.2079626656780284, + "nauc_mrr_at_1000_diff1": 0.7411736126116106, + "nauc_mrr_at_1000_max": 0.4917828756376569, + "nauc_mrr_at_1000_std": -0.20683083313104692, + "nauc_mrr_at_100_diff1": 0.7411562255890868, + "nauc_mrr_at_100_max": 0.4918349144235231, + "nauc_mrr_at_100_std": -0.20672737625036824, + "nauc_mrr_at_10_diff1": 0.7400330232348077, + "nauc_mrr_at_10_max": 0.49357475828973923, + "nauc_mrr_at_10_std": -0.20606748745004916, + "nauc_mrr_at_1_diff1": 0.7623652046686186, + "nauc_mrr_at_1_max": 0.4681979463974185, + "nauc_mrr_at_1_std": -0.21274061728203167, + "nauc_mrr_at_20_diff1": 0.7408903405171129, + "nauc_mrr_at_20_max": 0.49254548529020414, + "nauc_mrr_at_20_std": -0.20637267290486805, + "nauc_mrr_at_3_diff1": 0.7382633309068931, + "nauc_mrr_at_3_max": 0.4892418217193733, + "nauc_mrr_at_3_std": -0.21665425397028518, + "nauc_mrr_at_5_diff1": 0.7391457643885568, + "nauc_mrr_at_5_max": 0.4935053812177111, + "nauc_mrr_at_5_std": -0.2079626656780284, + "nauc_ndcg_at_1000_diff1": 0.7382481720449412, + "nauc_ndcg_at_1000_max": 0.4972038613490746, + "nauc_ndcg_at_1000_std": -0.20261425964460525, + "nauc_ndcg_at_100_diff1": 0.7376880004808867, + "nauc_ndcg_at_100_max": 0.4987282147380828, + "nauc_ndcg_at_100_std": -0.1993796792055653, + "nauc_ndcg_at_10_diff1": 0.7323219789374692, + "nauc_ndcg_at_10_max": 0.508551124757385, + "nauc_ndcg_at_10_std": -0.19555820350978895, + "nauc_ndcg_at_1_diff1": 0.7623652046686186, + "nauc_ndcg_at_1_max": 0.4681979463974185, + "nauc_ndcg_at_1_std": -0.21274061728203167, + "nauc_ndcg_at_20_diff1": 0.7357179194450744, + "nauc_ndcg_at_20_max": 0.5047377774302884, + "nauc_ndcg_at_20_std": -0.19602191052106646, + "nauc_ndcg_at_3_diff1": 0.7295970107460835, + "nauc_ndcg_at_3_max": 0.4978410741544815, + "nauc_ndcg_at_3_std": -0.21913117291610995, + "nauc_ndcg_at_5_diff1": 0.7305310266263622, + "nauc_ndcg_at_5_max": 0.5072305632059156, + "nauc_ndcg_at_5_std": -0.20161380941344245, + "nauc_precision_at_1000_diff1": 0.7222282767317681, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.6081278002284122, + "nauc_precision_at_100_max": 0.8488622992394487, + "nauc_precision_at_100_std": 0.5571362488972942, + "nauc_precision_at_10_diff1": 0.6628892757554079, + "nauc_precision_at_10_max": 0.6565974764765872, + "nauc_precision_at_10_std": -0.08043915108182106, + "nauc_precision_at_1_diff1": 0.7623652046686186, + "nauc_precision_at_1_max": 0.4681979463974185, + "nauc_precision_at_1_std": -0.21274061728203167, + "nauc_precision_at_20_diff1": 0.6662127485138202, + "nauc_precision_at_20_max": 0.7161591219904893, + "nauc_precision_at_20_std": 0.018820578206631296, + "nauc_precision_at_3_diff1": 0.6943259930732729, + "nauc_precision_at_3_max": 0.5333482867271515, + "nauc_precision_at_3_std": -0.22973946875310539, + "nauc_precision_at_5_diff1": 0.6836306948389095, + "nauc_precision_at_5_max": 0.5859049187975767, + "nauc_precision_at_5_std": -0.16199264140836858, + "nauc_recall_at_1000_diff1": 0.7222282767327224, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.6081278002284253, + "nauc_recall_at_100_max": 0.8488622992394228, + "nauc_recall_at_100_std": 0.5571362488972622, + "nauc_recall_at_10_diff1": 0.6628892757554061, + "nauc_recall_at_10_max": 0.6565974764765896, + "nauc_recall_at_10_std": -0.08043915108181712, + "nauc_recall_at_1_diff1": 0.7623652046686186, + "nauc_recall_at_1_max": 0.4681979463974185, + "nauc_recall_at_1_std": -0.21274061728203167, + "nauc_recall_at_20_diff1": 0.6662127485138235, + "nauc_recall_at_20_max": 0.7161591219904855, + "nauc_recall_at_20_std": 0.018820578206631276, + "nauc_recall_at_3_diff1": 0.6943259930732723, + "nauc_recall_at_3_max": 0.5333482867271512, + "nauc_recall_at_3_std": -0.22973946875310663, + "nauc_recall_at_5_diff1": 0.6836306948389111, + "nauc_recall_at_5_max": 0.5859049187975784, + "nauc_recall_at_5_std": -0.16199264140836703, + "ndcg_at_1": 0.67631, + "ndcg_at_10": 0.8093, + "ndcg_at_100": 0.82191, + "ndcg_at_1000": 0.82259, + "ndcg_at_20": 0.81672, + "ndcg_at_3": 0.77226, + "ndcg_at_5": 0.79349, + "precision_at_1": 0.67631, + "precision_at_10": 0.09387, + "precision_at_100": 0.00995, + "precision_at_1000": 0.001, + "precision_at_20": 0.04839, + "precision_at_3": 0.2797, + "precision_at_5": 0.17809, + "recall_at_1": 0.67631, + "recall_at_10": 0.93873, + "recall_at_100": 0.99469, + "recall_at_1000": 0.99988, + "recall_at_20": 0.96777, + "recall_at_3": 0.8391, + "recall_at_5": 0.89045 + }, + { + "cv_recall_at_1": 0.6366, + "cv_recall_at_10": 0.91496, + "cv_recall_at_100": 0.99165, + "cv_recall_at_1000": 0.9993, + "cv_recall_at_20": 0.95254, + "cv_recall_at_3": 0.80264, + "cv_recall_at_5": 0.8611, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.77708, + "map_at_1": 0.6366, + "map_at_10": 0.7328, + "map_at_100": 0.73663, + "map_at_1000": 0.73667, + "map_at_20": 0.7355, + "map_at_3": 0.71211, + "map_at_5": 0.72556, + "mrr_at_1": 0.6366040361864996, + "mrr_at_10": 0.7327972186322893, + "mrr_at_100": 0.736627017488815, + "mrr_at_1000": 0.7366699895317544, + "mrr_at_20": 0.7355008701473986, + "mrr_at_3": 0.7121085594989554, + "mrr_at_5": 0.7255601948503794, + "nauc_cv_recall_at_1000_diff1": 0.4870112277730652, + "nauc_cv_recall_at_1000_max": 0.9738582304731981, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.7360194174964291, + "nauc_cv_recall_at_100_max": 0.8283919638695952, + "nauc_cv_recall_at_100_std": 0.6650081686379116, + "nauc_cv_recall_at_10_diff1": 0.6654660906057306, + "nauc_cv_recall_at_10_max": 0.5986711490902606, + "nauc_cv_recall_at_10_std": -0.044115384650931296, + "nauc_cv_recall_at_1_diff1": 0.7705764177641267, + "nauc_cv_recall_at_1_max": 0.46467207153346846, + "nauc_cv_recall_at_1_std": -0.15797317602015953, + "nauc_cv_recall_at_20_diff1": 0.6725663450558479, + "nauc_cv_recall_at_20_max": 0.671566686556366, + "nauc_cv_recall_at_20_std": 0.1500791693559291, + "nauc_cv_recall_at_3_diff1": 0.682922952048957, + "nauc_cv_recall_at_3_max": 0.5255304200904859, + "nauc_cv_recall_at_3_std": -0.14608349755474395, + "nauc_cv_recall_at_5_diff1": 0.6707935677999222, + "nauc_cv_recall_at_5_max": 0.5559591575751662, + "nauc_cv_recall_at_5_std": -0.10345018603810813, + "nauc_map_at_1000_diff1": 0.7441881259546695, + "nauc_map_at_1000_max": 0.48932443569077916, + "nauc_map_at_1000_std": -0.14707622380005525, + "nauc_map_at_100_diff1": 0.7441879554181916, + "nauc_map_at_100_max": 0.489378471692625, + "nauc_map_at_100_std": -0.14693806269457682, + "nauc_map_at_10_diff1": 0.7427115484863952, + "nauc_map_at_10_max": 0.4901723060092325, + "nauc_map_at_10_std": -0.1479163009747065, + "nauc_map_at_1_diff1": 0.7705764177641267, + "nauc_map_at_1_max": 0.46467207153346846, + "nauc_map_at_1_std": -0.15797317602015953, + "nauc_map_at_20_diff1": 0.7437556217723685, + "nauc_map_at_20_max": 0.49013412945893137, + "nauc_map_at_20_std": -0.14605028470717732, + "nauc_map_at_3_diff1": 0.739929554116238, + "nauc_map_at_3_max": 0.4878653280639865, + "nauc_map_at_3_std": -0.15494221973189937, + "nauc_map_at_5_diff1": 0.7410872828829704, + "nauc_map_at_5_max": 0.48992805135002115, + "nauc_map_at_5_std": -0.1497645067760917, + "nauc_mrr_at_1000_diff1": 0.7441881259546695, + "nauc_mrr_at_1000_max": 0.48932443569077916, + "nauc_mrr_at_1000_std": -0.14707622380005525, + "nauc_mrr_at_100_diff1": 0.7441879554181916, + "nauc_mrr_at_100_max": 0.489378471692625, + "nauc_mrr_at_100_std": -0.14693806269457682, + "nauc_mrr_at_10_diff1": 0.7427115484863952, + "nauc_mrr_at_10_max": 0.4901723060092325, + "nauc_mrr_at_10_std": -0.1479163009747065, + "nauc_mrr_at_1_diff1": 0.7705764177641267, + "nauc_mrr_at_1_max": 0.46467207153346846, + "nauc_mrr_at_1_std": -0.15797317602015953, + "nauc_mrr_at_20_diff1": 0.7437556217723685, + "nauc_mrr_at_20_max": 0.49013412945893137, + "nauc_mrr_at_20_std": -0.14605028470717732, + "nauc_mrr_at_3_diff1": 0.739929554116238, + "nauc_mrr_at_3_max": 0.4878653280639865, + "nauc_mrr_at_3_std": -0.15494221973189937, + "nauc_mrr_at_5_diff1": 0.7410872828829704, + "nauc_mrr_at_5_max": 0.48992805135002115, + "nauc_mrr_at_5_std": -0.1497645067760917, + "nauc_ndcg_at_1000_diff1": 0.740475731829745, + "nauc_ndcg_at_1000_max": 0.4947914358930953, + "nauc_ndcg_at_1000_std": -0.14023034742460014, + "nauc_ndcg_at_100_diff1": 0.7405599849365426, + "nauc_ndcg_at_100_max": 0.49657575192060943, + "nauc_ndcg_at_100_std": -0.13586349043153753, + "nauc_ndcg_at_10_diff1": 0.7331826599258588, + "nauc_ndcg_at_10_max": 0.5023841859205426, + "nauc_ndcg_at_10_std": -0.13705141672240842, + "nauc_ndcg_at_1_diff1": 0.7705764177641267, + "nauc_ndcg_at_1_max": 0.46467207153346846, + "nauc_ndcg_at_1_std": -0.15797317602015953, + "nauc_ndcg_at_20_diff1": 0.7372168346745342, + "nauc_ndcg_at_20_max": 0.5024041914908172, + "nauc_ndcg_at_20_std": -0.12880153641013473, + "nauc_ndcg_at_3_diff1": 0.728071926540345, + "nauc_ndcg_at_3_max": 0.495789424094431, + "nauc_ndcg_at_3_std": -0.15315224992624585, + "nauc_ndcg_at_5_diff1": 0.7293058439381539, + "nauc_ndcg_at_5_max": 0.5008014824465146, + "nauc_ndcg_at_5_std": -0.14255901932569748, + "nauc_precision_at_1000_diff1": 0.4870112277732604, + "nauc_precision_at_1000_max": 0.9738582304732849, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.7360194174964301, + "nauc_precision_at_100_max": 0.8283919638695951, + "nauc_precision_at_100_std": 0.6650081686379157, + "nauc_precision_at_10_diff1": 0.6654660906057299, + "nauc_precision_at_10_max": 0.5986711490902608, + "nauc_precision_at_10_std": -0.044115384650932864, + "nauc_precision_at_1_diff1": 0.7705764177641267, + "nauc_precision_at_1_max": 0.46467207153346846, + "nauc_precision_at_1_std": -0.15797317602015953, + "nauc_precision_at_20_diff1": 0.672566345055847, + "nauc_precision_at_20_max": 0.671566686556363, + "nauc_precision_at_20_std": 0.15007916935592425, + "nauc_precision_at_3_diff1": 0.6829229520489568, + "nauc_precision_at_3_max": 0.5255304200904857, + "nauc_precision_at_3_std": -0.14608349755474373, + "nauc_precision_at_5_diff1": 0.6707935677999219, + "nauc_precision_at_5_max": 0.5559591575751665, + "nauc_precision_at_5_std": -0.10345018603810868, + "nauc_recall_at_1000_diff1": 0.4870112277730652, + "nauc_recall_at_1000_max": 0.9738582304731981, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.7360194174964291, + "nauc_recall_at_100_max": 0.8283919638695952, + "nauc_recall_at_100_std": 0.6650081686379116, + "nauc_recall_at_10_diff1": 0.6654660906057306, + "nauc_recall_at_10_max": 0.5986711490902606, + "nauc_recall_at_10_std": -0.044115384650931296, + "nauc_recall_at_1_diff1": 0.7705764177641267, + "nauc_recall_at_1_max": 0.46467207153346846, + "nauc_recall_at_1_std": -0.15797317602015953, + "nauc_recall_at_20_diff1": 0.6725663450558479, + "nauc_recall_at_20_max": 0.671566686556366, + "nauc_recall_at_20_std": 0.1500791693559291, + "nauc_recall_at_3_diff1": 0.682922952048957, + "nauc_recall_at_3_max": 0.5255304200904859, + "nauc_recall_at_3_std": -0.14608349755474395, + "nauc_recall_at_5_diff1": 0.6707935677999222, + "nauc_recall_at_5_max": 0.5559591575751662, + "nauc_recall_at_5_std": -0.10345018603810813, + "ndcg_at_1": 0.6366, + "ndcg_at_10": 0.77708, + "ndcg_at_100": 0.7942, + "ndcg_at_1000": 0.79521, + "ndcg_at_20": 0.7867, + "ndcg_at_3": 0.73546, + "ndcg_at_5": 0.75962, + "precision_at_1": 0.6366, + "precision_at_10": 0.0915, + "precision_at_100": 0.00992, + "precision_at_1000": 0.001, + "precision_at_20": 0.04763, + "precision_at_3": 0.26755, + "precision_at_5": 0.17222, + "recall_at_1": 0.6366, + "recall_at_10": 0.91496, + "recall_at_100": 0.99165, + "recall_at_1000": 0.9993, + "recall_at_20": 0.95254, + "recall_at_3": 0.80264, + "recall_at_5": 0.8611 + }, + { + "cv_recall_at_1": 0.32824, + "cv_recall_at_10": 0.66092, + "cv_recall_at_100": 0.90157, + "cv_recall_at_1000": 0.98967, + "cv_recall_at_20": 0.74654, + "cv_recall_at_3": 0.49673, + "cv_recall_at_5": 0.5698, + "hf_subset": "ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.48615, + "map_at_1": 0.32824, + "map_at_10": 0.43119, + "map_at_100": 0.44096, + "map_at_1000": 0.44138, + "map_at_20": 0.43704, + "map_at_3": 0.40227, + "map_at_5": 0.41891, + "mrr_at_1": 0.32823529411764707, + "mrr_at_10": 0.4311886087768451, + "mrr_at_100": 0.44095616656130077, + "mrr_at_1000": 0.44137718862191744, + "mrr_at_20": 0.43703863946848, + "mrr_at_3": 0.4022657952069736, + "mrr_at_5": 0.4189128540305019, + "nauc_cv_recall_at_1000_diff1": 0.31606566677303044, + "nauc_cv_recall_at_1000_max": 0.7567280076587494, + "nauc_cv_recall_at_1000_std": 0.8719876136108337, + "nauc_cv_recall_at_100_diff1": 0.36533294149886736, + "nauc_cv_recall_at_100_max": 0.6483074858982988, + "nauc_cv_recall_at_100_std": 0.6143704050898802, + "nauc_cv_recall_at_10_diff1": 0.41043823786876577, + "nauc_cv_recall_at_10_max": 0.501184849572785, + "nauc_cv_recall_at_10_std": 0.24836789796558764, + "nauc_cv_recall_at_1_diff1": 0.5724479741203551, + "nauc_cv_recall_at_1_max": 0.4093246568520635, + "nauc_cv_recall_at_1_std": 0.08187266516484734, + "nauc_cv_recall_at_20_diff1": 0.4008308929393305, + "nauc_cv_recall_at_20_max": 0.5438192664450267, + "nauc_cv_recall_at_20_std": 0.34315894629094834, + "nauc_cv_recall_at_3_diff1": 0.4557066058908472, + "nauc_cv_recall_at_3_max": 0.4461926964630562, + "nauc_cv_recall_at_3_std": 0.15308129925712008, + "nauc_cv_recall_at_5_diff1": 0.4327341488634024, + "nauc_cv_recall_at_5_max": 0.47990161573683054, + "nauc_cv_recall_at_5_std": 0.19948489678898848, + "nauc_map_at_1000_diff1": 0.5148975188038908, + "nauc_map_at_1000_max": 0.4355824398849164, + "nauc_map_at_1000_std": 0.13065522167678348, + "nauc_map_at_100_diff1": 0.514698319934614, + "nauc_map_at_100_max": 0.4357570122065423, + "nauc_map_at_100_std": 0.1311025115276141, + "nauc_map_at_10_diff1": 0.513418485624799, + "nauc_map_at_10_max": 0.43463205566792706, + "nauc_map_at_10_std": 0.12744454519859516, + "nauc_map_at_1_diff1": 0.5724479741203551, + "nauc_map_at_1_max": 0.4093246568520635, + "nauc_map_at_1_std": 0.08187266516484734, + "nauc_map_at_20_diff1": 0.5140149448972925, + "nauc_map_at_20_max": 0.43588645221616207, + "nauc_map_at_20_std": 0.13058737794323919, + "nauc_map_at_3_diff1": 0.5189536861901586, + "nauc_map_at_3_max": 0.4260398579272175, + "nauc_map_at_3_std": 0.11384666123814757, + "nauc_map_at_5_diff1": 0.5147617743556974, + "nauc_map_at_5_max": 0.4329202598408649, + "nauc_map_at_5_std": 0.12324129047164137, + "nauc_mrr_at_1000_diff1": 0.5148975188038908, + "nauc_mrr_at_1000_max": 0.4355824398849164, + "nauc_mrr_at_1000_std": 0.13065522167678348, + "nauc_mrr_at_100_diff1": 0.514698319934614, + "nauc_mrr_at_100_max": 0.4357570122065423, + "nauc_mrr_at_100_std": 0.1311025115276141, + "nauc_mrr_at_10_diff1": 0.513418485624799, + "nauc_mrr_at_10_max": 0.43463205566792706, + "nauc_mrr_at_10_std": 0.12744454519859516, + "nauc_mrr_at_1_diff1": 0.5724479741203551, + "nauc_mrr_at_1_max": 0.4093246568520635, + "nauc_mrr_at_1_std": 0.08187266516484734, + "nauc_mrr_at_20_diff1": 0.5140149448972925, + "nauc_mrr_at_20_max": 0.43588645221616207, + "nauc_mrr_at_20_std": 0.13058737794323919, + "nauc_mrr_at_3_diff1": 0.5189536861901586, + "nauc_mrr_at_3_max": 0.4260398579272175, + "nauc_mrr_at_3_std": 0.11384666123814757, + "nauc_mrr_at_5_diff1": 0.5147617743556974, + "nauc_mrr_at_5_max": 0.4329202598408649, + "nauc_mrr_at_5_std": 0.12324129047164137, + "nauc_ndcg_at_1000_diff1": 0.5028633000738268, + "nauc_ndcg_at_1000_max": 0.4467397582711455, + "nauc_ndcg_at_1000_std": 0.15358645176436325, + "nauc_ndcg_at_100_diff1": 0.49762609351535697, + "nauc_ndcg_at_100_max": 0.4535299823361077, + "nauc_ndcg_at_100_std": 0.16943724112556677, + "nauc_ndcg_at_10_diff1": 0.491020691200883, + "nauc_ndcg_at_10_max": 0.4485124450008812, + "nauc_ndcg_at_10_std": 0.15250408970563734, + "nauc_ndcg_at_1_diff1": 0.5724479741203551, + "nauc_ndcg_at_1_max": 0.4093246568520635, + "nauc_ndcg_at_1_std": 0.08187266516484734, + "nauc_ndcg_at_20_diff1": 0.49270587037633035, + "nauc_ndcg_at_20_max": 0.453954581379322, + "nauc_ndcg_at_20_std": 0.16561882754616208, + "nauc_ndcg_at_3_diff1": 0.5026632707669226, + "nauc_ndcg_at_3_max": 0.431222781113809, + "nauc_ndcg_at_3_std": 0.12392363092639624, + "nauc_ndcg_at_5_diff1": 0.49479976602682435, + "nauc_ndcg_at_5_max": 0.4439816644355392, + "nauc_ndcg_at_5_std": 0.1413280531203506, + "nauc_precision_at_1000_diff1": 0.31606566677303394, + "nauc_precision_at_1000_max": 0.7567280076587406, + "nauc_precision_at_1000_std": 0.8719876136108287, + "nauc_precision_at_100_diff1": 0.3653329414988664, + "nauc_precision_at_100_max": 0.6483074858983007, + "nauc_precision_at_100_std": 0.6143704050898817, + "nauc_precision_at_10_diff1": 0.41043823786876493, + "nauc_precision_at_10_max": 0.5011848495727845, + "nauc_precision_at_10_std": 0.2483678979655874, + "nauc_precision_at_1_diff1": 0.5724479741203551, + "nauc_precision_at_1_max": 0.4093246568520635, + "nauc_precision_at_1_std": 0.08187266516484734, + "nauc_precision_at_20_diff1": 0.4008308929393299, + "nauc_precision_at_20_max": 0.5438192664450257, + "nauc_precision_at_20_std": 0.3431589462909468, + "nauc_precision_at_3_diff1": 0.4557066058908471, + "nauc_precision_at_3_max": 0.44619269646305626, + "nauc_precision_at_3_std": 0.15308129925712013, + "nauc_precision_at_5_diff1": 0.4327341488634025, + "nauc_precision_at_5_max": 0.47990161573683066, + "nauc_precision_at_5_std": 0.1994848967889882, + "nauc_recall_at_1000_diff1": 0.31606566677303044, + "nauc_recall_at_1000_max": 0.7567280076587494, + "nauc_recall_at_1000_std": 0.8719876136108337, + "nauc_recall_at_100_diff1": 0.36533294149886736, + "nauc_recall_at_100_max": 0.6483074858982988, + "nauc_recall_at_100_std": 0.6143704050898802, + "nauc_recall_at_10_diff1": 0.41043823786876577, + "nauc_recall_at_10_max": 0.501184849572785, + "nauc_recall_at_10_std": 0.24836789796558764, + "nauc_recall_at_1_diff1": 0.5724479741203551, + "nauc_recall_at_1_max": 0.4093246568520635, + "nauc_recall_at_1_std": 0.08187266516484734, + "nauc_recall_at_20_diff1": 0.4008308929393305, + "nauc_recall_at_20_max": 0.5438192664450267, + "nauc_recall_at_20_std": 0.34315894629094834, + "nauc_recall_at_3_diff1": 0.4557066058908472, + "nauc_recall_at_3_max": 0.4461926964630562, + "nauc_recall_at_3_std": 0.15308129925712008, + "nauc_recall_at_5_diff1": 0.4327341488634024, + "nauc_recall_at_5_max": 0.47990161573683054, + "nauc_recall_at_5_std": 0.19948489678898848, + "ndcg_at_1": 0.32824, + "ndcg_at_10": 0.48615, + "ndcg_at_100": 0.53635, + "ndcg_at_1000": 0.54755, + "ndcg_at_20": 0.50768, + "ndcg_at_3": 0.42652, + "ndcg_at_5": 0.45657, + "precision_at_1": 0.32824, + "precision_at_10": 0.06609, + "precision_at_100": 0.00902, + "precision_at_1000": 0.00099, + "precision_at_20": 0.03733, + "precision_at_3": 0.16558, + "precision_at_5": 0.11396, + "recall_at_1": 0.32824, + "recall_at_10": 0.66092, + "recall_at_100": 0.90157, + "recall_at_1000": 0.98967, + "recall_at_20": 0.74654, + "recall_at_3": 0.49673, + "recall_at_5": 0.5698 + }, + { + "cv_recall_at_1": 0.00254, + "cv_recall_at_10": 0.01205, + "cv_recall_at_100": 0.07101, + "cv_recall_at_1000": 0.38821, + "cv_recall_at_20": 0.01902, + "cv_recall_at_3": 0.00592, + "cv_recall_at_5": 0.00824, + "hf_subset": "mi", + "languages": [ + "mri-Latn" + ], + "main_score": 0.00661, + "map_at_1": 0.00254, + "map_at_10": 0.00496, + "map_at_100": 0.00656, + "map_at_1000": 0.00746, + "map_at_20": 0.00544, + "map_at_3": 0.00394, + "map_at_5": 0.00446, + "mrr_at_1": 0.00253592561284869, + "mrr_at_10": 0.004960233600343489, + "mrr_at_100": 0.006561841673598242, + "mrr_at_1000": 0.007462404192643936, + "mrr_at_20": 0.005441873817226138, + "mrr_at_3": 0.0039447731755424065, + "mrr_at_5": 0.004462524654832346, + "nauc_cv_recall_at_1000_diff1": -0.001754921851847047, + "nauc_cv_recall_at_1000_max": 0.0813775510944442, + "nauc_cv_recall_at_1000_std": 0.1703090139503763, + "nauc_cv_recall_at_100_diff1": 0.01629770853022363, + "nauc_cv_recall_at_100_max": 0.15059830692876489, + "nauc_cv_recall_at_100_std": 0.2651021745283781, + "nauc_cv_recall_at_10_diff1": 0.16620091601208944, + "nauc_cv_recall_at_10_max": 0.44683741032627683, + "nauc_cv_recall_at_10_std": 0.5502644434148982, + "nauc_cv_recall_at_1_diff1": 0.38191644037549527, + "nauc_cv_recall_at_1_max": 0.8121549617389423, + "nauc_cv_recall_at_1_std": 0.7411893196695267, + "nauc_cv_recall_at_20_diff1": 0.09707278061979692, + "nauc_cv_recall_at_20_max": 0.3273986828212527, + "nauc_cv_recall_at_20_std": 0.47419581902757957, + "nauc_cv_recall_at_3_diff1": 0.24906706569566026, + "nauc_cv_recall_at_3_max": 0.6659004348431463, + "nauc_cv_recall_at_3_std": 0.6936494424405377, + "nauc_cv_recall_at_5_diff1": 0.22487748592748064, + "nauc_cv_recall_at_5_max": 0.5868842812113284, + "nauc_cv_recall_at_5_std": 0.638464710612695, + "nauc_map_at_1000_diff1": 0.1766150827944164, + "nauc_map_at_1000_max": 0.46607818255706707, + "nauc_map_at_1000_std": 0.515329367279344, + "nauc_map_at_100_diff1": 0.1927671971676861, + "nauc_map_at_100_max": 0.5007461423878604, + "nauc_map_at_100_std": 0.5474677908875887, + "nauc_map_at_10_diff1": 0.2611240007883676, + "nauc_map_at_10_max": 0.6300118926644409, + "nauc_map_at_10_std": 0.6460166321923634, + "nauc_map_at_1_diff1": 0.38191644037549527, + "nauc_map_at_1_max": 0.8121549617389423, + "nauc_map_at_1_std": 0.7411893196695267, + "nauc_map_at_20_diff1": 0.2374761310339923, + "nauc_map_at_20_max": 0.5851741900653458, + "nauc_map_at_20_std": 0.619475842174224, + "nauc_map_at_3_diff1": 0.2967462852583165, + "nauc_map_at_3_max": 0.7195420182534291, + "nauc_map_at_3_std": 0.7092403257642388, + "nauc_map_at_5_diff1": 0.28388313400981, + "nauc_map_at_5_max": 0.6826019446607527, + "nauc_map_at_5_std": 0.6809347447189633, + "nauc_mrr_at_1000_diff1": 0.1766150827944164, + "nauc_mrr_at_1000_max": 0.46607818255706707, + "nauc_mrr_at_1000_std": 0.515329367279344, + "nauc_mrr_at_100_diff1": 0.1927671971676861, + "nauc_mrr_at_100_max": 0.5007461423878604, + "nauc_mrr_at_100_std": 0.5474677908875887, + "nauc_mrr_at_10_diff1": 0.2611240007883676, + "nauc_mrr_at_10_max": 0.6300118926644409, + "nauc_mrr_at_10_std": 0.6460166321923634, + "nauc_mrr_at_1_diff1": 0.38191644037549527, + "nauc_mrr_at_1_max": 0.8121549617389423, + "nauc_mrr_at_1_std": 0.7411893196695267, + "nauc_mrr_at_20_diff1": 0.2374761310339923, + "nauc_mrr_at_20_max": 0.5851741900653458, + "nauc_mrr_at_20_std": 0.619475842174224, + "nauc_mrr_at_3_diff1": 0.2967462852583165, + "nauc_mrr_at_3_max": 0.7195420182534291, + "nauc_mrr_at_3_std": 0.7092403257642388, + "nauc_mrr_at_5_diff1": 0.28388313400981, + "nauc_mrr_at_5_max": 0.6826019446607527, + "nauc_mrr_at_5_std": 0.6809347447189633, + "nauc_ndcg_at_1000_diff1": 0.0322965161207195, + "nauc_ndcg_at_1000_max": 0.15979964836745464, + "nauc_ndcg_at_1000_std": 0.2455940665585908, + "nauc_ndcg_at_100_diff1": 0.0732874730991388, + "nauc_ndcg_at_100_max": 0.26639849711214036, + "nauc_ndcg_at_100_std": 0.3626258542492295, + "nauc_ndcg_at_10_diff1": 0.22212352710492203, + "nauc_ndcg_at_10_max": 0.5555024514690303, + "nauc_ndcg_at_10_std": 0.6066786205790419, + "nauc_ndcg_at_1_diff1": 0.38191644037549527, + "nauc_ndcg_at_1_max": 0.8121549617389423, + "nauc_ndcg_at_1_std": 0.7411893196695267, + "nauc_ndcg_at_20_diff1": 0.17186485474703883, + "nauc_ndcg_at_20_max": 0.46433618491655965, + "nauc_ndcg_at_20_std": 0.5514430613917896, + "nauc_ndcg_at_3_diff1": 0.28048949881588553, + "nauc_ndcg_at_3_max": 0.7012956749530924, + "nauc_ndcg_at_3_std": 0.7038690040172996, + "nauc_ndcg_at_5_diff1": 0.2618439126388269, + "nauc_ndcg_at_5_max": 0.647270534174721, + "nauc_ndcg_at_5_std": 0.6650220312022473, + "nauc_precision_at_1000_diff1": -0.0017549218518471566, + "nauc_precision_at_1000_max": 0.08137755109444397, + "nauc_precision_at_1000_std": 0.1703090139503762, + "nauc_precision_at_100_diff1": 0.016297708530223557, + "nauc_precision_at_100_max": 0.15059830692876505, + "nauc_precision_at_100_std": 0.2651021745283781, + "nauc_precision_at_10_diff1": 0.16620091601208986, + "nauc_precision_at_10_max": 0.44683741032627705, + "nauc_precision_at_10_std": 0.5502644434148986, + "nauc_precision_at_1_diff1": 0.38191644037549527, + "nauc_precision_at_1_max": 0.8121549617389423, + "nauc_precision_at_1_std": 0.7411893196695267, + "nauc_precision_at_20_diff1": 0.09707278061979728, + "nauc_precision_at_20_max": 0.32739868282125306, + "nauc_precision_at_20_std": 0.47419581902757973, + "nauc_precision_at_3_diff1": 0.24906706569565998, + "nauc_precision_at_3_max": 0.6659004348431463, + "nauc_precision_at_3_std": 0.6936494424405376, + "nauc_precision_at_5_diff1": 0.2248774859274806, + "nauc_precision_at_5_max": 0.5868842812113281, + "nauc_precision_at_5_std": 0.6384647106126947, + "nauc_recall_at_1000_diff1": -0.001754921851847047, + "nauc_recall_at_1000_max": 0.0813775510944442, + "nauc_recall_at_1000_std": 0.1703090139503763, + "nauc_recall_at_100_diff1": 0.01629770853022363, + "nauc_recall_at_100_max": 0.15059830692876489, + "nauc_recall_at_100_std": 0.2651021745283781, + "nauc_recall_at_10_diff1": 0.16620091601208944, + "nauc_recall_at_10_max": 0.44683741032627683, + "nauc_recall_at_10_std": 0.5502644434148982, + "nauc_recall_at_1_diff1": 0.38191644037549527, + "nauc_recall_at_1_max": 0.8121549617389423, + "nauc_recall_at_1_std": 0.7411893196695267, + "nauc_recall_at_20_diff1": 0.09707278061979692, + "nauc_recall_at_20_max": 0.3273986828212527, + "nauc_recall_at_20_std": 0.47419581902757957, + "nauc_recall_at_3_diff1": 0.24906706569566026, + "nauc_recall_at_3_max": 0.6659004348431463, + "nauc_recall_at_3_std": 0.6936494424405377, + "nauc_recall_at_5_diff1": 0.22487748592748064, + "nauc_recall_at_5_max": 0.5868842812113284, + "nauc_recall_at_5_std": 0.638464710612695, + "ndcg_at_1": 0.00254, + "ndcg_at_10": 0.00661, + "ndcg_at_100": 0.01758, + "ndcg_at_1000": 0.0542, + "ndcg_at_20": 0.00837, + "ndcg_at_3": 0.00445, + "ndcg_at_5": 0.00539, + "precision_at_1": 0.00254, + "precision_at_10": 0.0012, + "precision_at_100": 0.00071, + "precision_at_1000": 0.00039, + "precision_at_20": 0.00095, + "precision_at_3": 0.00197, + "precision_at_5": 0.00165, + "recall_at_1": 0.00254, + "recall_at_10": 0.01205, + "recall_at_100": 0.07101, + "recall_at_1000": 0.38821, + "recall_at_20": 0.01902, + "recall_at_3": 0.00592, + "recall_at_5": 0.00824 + }, + { + "cv_recall_at_1": 0.52488, + "cv_recall_at_10": 0.84874, + "cv_recall_at_100": 0.9732, + "cv_recall_at_1000": 0.99752, + "cv_recall_at_20": 0.90135, + "cv_recall_at_3": 0.70852, + "cv_recall_at_5": 0.77503, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ], + "main_score": 0.68389, + "map_at_1": 0.52488, + "map_at_10": 0.63139, + "map_at_100": 0.63708, + "map_at_1000": 0.63722, + "map_at_20": 0.63508, + "map_at_3": 0.60613, + "map_at_5": 0.6214, + "mrr_at_1": 0.5248790172477975, + "mrr_at_10": 0.6313939655359169, + "mrr_at_100": 0.6370779085742311, + "mrr_at_1000": 0.6372166028814777, + "mrr_at_20": 0.6350809687459603, + "mrr_at_3": 0.6061339289407283, + "mrr_at_5": 0.621396368449349, + "nauc_cv_recall_at_1000_diff1": 0.5744083171531605, + "nauc_cv_recall_at_1000_max": 0.9317200295261857, + "nauc_cv_recall_at_1000_std": 0.8472486786395506, + "nauc_cv_recall_at_100_diff1": 0.6148330048914848, + "nauc_cv_recall_at_100_max": 0.8057075120537615, + "nauc_cv_recall_at_100_std": 0.5279928320652953, + "nauc_cv_recall_at_10_diff1": 0.5678154941414436, + "nauc_cv_recall_at_10_max": 0.5581864497311916, + "nauc_cv_recall_at_10_std": 0.06369656555600225, + "nauc_cv_recall_at_1_diff1": 0.704020495398862, + "nauc_cv_recall_at_1_max": 0.43716341122428626, + "nauc_cv_recall_at_1_std": -0.08216752278968253, + "nauc_cv_recall_at_20_diff1": 0.5613842322724212, + "nauc_cv_recall_at_20_max": 0.6061462124421776, + "nauc_cv_recall_at_20_std": 0.1645691248854774, + "nauc_cv_recall_at_3_diff1": 0.6111622811567877, + "nauc_cv_recall_at_3_max": 0.4913209733378915, + "nauc_cv_recall_at_3_std": -0.05312478344784343, + "nauc_cv_recall_at_5_diff1": 0.5802937631290497, + "nauc_cv_recall_at_5_max": 0.5161516497709878, + "nauc_cv_recall_at_5_std": -0.00560283531961672, + "nauc_map_at_1000_diff1": 0.6686512103954919, + "nauc_map_at_1000_max": 0.46228001523632817, + "nauc_map_at_1000_std": -0.061803739790496234, + "nauc_map_at_100_diff1": 0.6686285521104891, + "nauc_map_at_100_max": 0.4624367851182591, + "nauc_map_at_100_std": -0.06156955965185607, + "nauc_map_at_10_diff1": 0.666741377527361, + "nauc_map_at_10_max": 0.4624129783306096, + "nauc_map_at_10_std": -0.06258848985899317, + "nauc_map_at_1_diff1": 0.704020495398862, + "nauc_map_at_1_max": 0.43716341122428626, + "nauc_map_at_1_std": -0.08216752278968253, + "nauc_map_at_20_diff1": 0.6678504831149329, + "nauc_map_at_20_max": 0.46272393004272905, + "nauc_map_at_20_std": -0.061428251168442145, + "nauc_map_at_3_diff1": 0.6678284118591633, + "nauc_map_at_3_max": 0.4581214019816706, + "nauc_map_at_3_std": -0.07277191030264109, + "nauc_map_at_5_diff1": 0.6651378761592399, + "nauc_map_at_5_max": 0.461044446009575, + "nauc_map_at_5_std": -0.06575410993244067, + "nauc_mrr_at_1000_diff1": 0.6686512103954919, + "nauc_mrr_at_1000_max": 0.46228001523632817, + "nauc_mrr_at_1000_std": -0.061803739790496234, + "nauc_mrr_at_100_diff1": 0.6686285521104891, + "nauc_mrr_at_100_max": 0.4624367851182591, + "nauc_mrr_at_100_std": -0.06156955965185607, + "nauc_mrr_at_10_diff1": 0.666741377527361, + "nauc_mrr_at_10_max": 0.4624129783306096, + "nauc_mrr_at_10_std": -0.06258848985899317, + "nauc_mrr_at_1_diff1": 0.704020495398862, + "nauc_mrr_at_1_max": 0.43716341122428626, + "nauc_mrr_at_1_std": -0.08216752278968253, + "nauc_mrr_at_20_diff1": 0.6678504831149329, + "nauc_mrr_at_20_max": 0.46272393004272905, + "nauc_mrr_at_20_std": -0.061428251168442145, + "nauc_mrr_at_3_diff1": 0.6678284118591633, + "nauc_mrr_at_3_max": 0.4581214019816706, + "nauc_mrr_at_3_std": -0.07277191030264109, + "nauc_mrr_at_5_diff1": 0.6651378761592399, + "nauc_mrr_at_5_max": 0.461044446009575, + "nauc_mrr_at_5_std": -0.06575410993244067, + "nauc_ndcg_at_1000_diff1": 0.6626036419000162, + "nauc_ndcg_at_1000_max": 0.47013464056151005, + "nauc_ndcg_at_1000_std": -0.05030975463253996, + "nauc_ndcg_at_100_diff1": 0.6620050846517636, + "nauc_ndcg_at_100_max": 0.47467890036858557, + "nauc_ndcg_at_100_std": -0.04281287475600159, + "nauc_ndcg_at_10_diff1": 0.651813517049161, + "nauc_ndcg_at_10_max": 0.4760935858527636, + "nauc_ndcg_at_10_std": -0.04531223138002542, + "nauc_ndcg_at_1_diff1": 0.704020495398862, + "nauc_ndcg_at_1_max": 0.43716341122428626, + "nauc_ndcg_at_1_std": -0.08216752278968253, + "nauc_ndcg_at_20_diff1": 0.6554275763205546, + "nauc_ndcg_at_20_max": 0.4774537795303503, + "nauc_ndcg_at_20_std": -0.03987783535410952, + "nauc_ndcg_at_3_diff1": 0.6550643982877136, + "nauc_ndcg_at_3_max": 0.46559271959208537, + "nauc_ndcg_at_3_std": -0.06843432861398872, + "nauc_ndcg_at_5_diff1": 0.6487639755957474, + "nauc_ndcg_at_5_max": 0.471639310185695, + "nauc_ndcg_at_5_std": -0.05459590263752243, + "nauc_precision_at_1000_diff1": 0.5744083171532188, + "nauc_precision_at_1000_max": 0.9317200295261956, + "nauc_precision_at_1000_std": 0.8472486786394815, + "nauc_precision_at_100_diff1": 0.6148330048914886, + "nauc_precision_at_100_max": 0.8057075120537643, + "nauc_precision_at_100_std": 0.5279928320652951, + "nauc_precision_at_10_diff1": 0.567815494141444, + "nauc_precision_at_10_max": 0.5581864497311915, + "nauc_precision_at_10_std": 0.06369656555600109, + "nauc_precision_at_1_diff1": 0.704020495398862, + "nauc_precision_at_1_max": 0.43716341122428626, + "nauc_precision_at_1_std": -0.08216752278968253, + "nauc_precision_at_20_diff1": 0.5613842322724212, + "nauc_precision_at_20_max": 0.6061462124421773, + "nauc_precision_at_20_std": 0.16456912488547504, + "nauc_precision_at_3_diff1": 0.6111622811567875, + "nauc_precision_at_3_max": 0.49132097333789176, + "nauc_precision_at_3_std": -0.05312478344784239, + "nauc_precision_at_5_diff1": 0.5802937631290486, + "nauc_precision_at_5_max": 0.5161516497709874, + "nauc_precision_at_5_std": -0.005602835319617839, + "nauc_recall_at_1000_diff1": 0.5744083171531605, + "nauc_recall_at_1000_max": 0.9317200295261857, + "nauc_recall_at_1000_std": 0.8472486786395506, + "nauc_recall_at_100_diff1": 0.6148330048914848, + "nauc_recall_at_100_max": 0.8057075120537615, + "nauc_recall_at_100_std": 0.5279928320652953, + "nauc_recall_at_10_diff1": 0.5678154941414436, + "nauc_recall_at_10_max": 0.5581864497311916, + "nauc_recall_at_10_std": 0.06369656555600225, + "nauc_recall_at_1_diff1": 0.704020495398862, + "nauc_recall_at_1_max": 0.43716341122428626, + "nauc_recall_at_1_std": -0.08216752278968253, + "nauc_recall_at_20_diff1": 0.5613842322724212, + "nauc_recall_at_20_max": 0.6061462124421776, + "nauc_recall_at_20_std": 0.1645691248854774, + "nauc_recall_at_3_diff1": 0.6111622811567877, + "nauc_recall_at_3_max": 0.4913209733378915, + "nauc_recall_at_3_std": -0.05312478344784343, + "nauc_recall_at_5_diff1": 0.5802937631290497, + "nauc_recall_at_5_max": 0.5161516497709878, + "nauc_recall_at_5_std": -0.00560283531961672, + "ndcg_at_1": 0.52488, + "ndcg_at_10": 0.68389, + "ndcg_at_100": 0.71087, + "ndcg_at_1000": 0.71408, + "ndcg_at_20": 0.69723, + "ndcg_at_3": 0.63244, + "ndcg_at_5": 0.65989, + "precision_at_1": 0.52488, + "precision_at_10": 0.08487, + "precision_at_100": 0.00973, + "precision_at_1000": 0.001, + "precision_at_20": 0.04507, + "precision_at_3": 0.23617, + "precision_at_5": 0.15501, + "recall_at_1": 0.52488, + "recall_at_10": 0.84874, + "recall_at_100": 0.9732, + "recall_at_1000": 0.99752, + "recall_at_20": 0.90135, + "recall_at_3": 0.70852, + "recall_at_5": 0.77503 + }, + { + "cv_recall_at_1": 0.42659, + "cv_recall_at_10": 0.75863, + "cv_recall_at_100": 0.94011, + "cv_recall_at_1000": 0.9939, + "cv_recall_at_20": 0.83169, + "cv_recall_at_3": 0.60003, + "cv_recall_at_5": 0.67184, + "hf_subset": "no", + "languages": [ + "nor-Latn" + ], + "main_score": 0.58599, + "map_at_1": 0.42659, + "map_at_10": 0.53152, + "map_at_100": 0.53948, + "map_at_1000": 0.53977, + "map_at_20": 0.53661, + "map_at_3": 0.50344, + "map_at_5": 0.51986, + "mrr_at_1": 0.42659087758214337, + "mrr_at_10": 0.531517718229213, + "mrr_at_100": 0.5394811572813161, + "mrr_at_1000": 0.5397675971072246, + "mrr_at_20": 0.5366131694602159, + "mrr_at_3": 0.5034428578030439, + "mrr_at_5": 0.5198576644022371, + "nauc_cv_recall_at_1000_diff1": 0.5365388365819015, + "nauc_cv_recall_at_1000_max": 0.8820883601096524, + "nauc_cv_recall_at_1000_std": 0.8560327455019514, + "nauc_cv_recall_at_100_diff1": 0.4291196561694799, + "nauc_cv_recall_at_100_max": 0.7317432023038329, + "nauc_cv_recall_at_100_std": 0.6364378846573594, + "nauc_cv_recall_at_10_diff1": 0.4727513594062488, + "nauc_cv_recall_at_10_max": 0.565976388070829, + "nauc_cv_recall_at_10_std": 0.2629214710555805, + "nauc_cv_recall_at_1_diff1": 0.6612644123585855, + "nauc_cv_recall_at_1_max": 0.4708878245975592, + "nauc_cv_recall_at_1_std": 0.048464967034370754, + "nauc_cv_recall_at_20_diff1": 0.44156738177753563, + "nauc_cv_recall_at_20_max": 0.6029237011588209, + "nauc_cv_recall_at_20_std": 0.3588193861745839, + "nauc_cv_recall_at_3_diff1": 0.5273771194696363, + "nauc_cv_recall_at_3_max": 0.5060320514918131, + "nauc_cv_recall_at_3_std": 0.1341634219173162, + "nauc_cv_recall_at_5_diff1": 0.5064775255421811, + "nauc_cv_recall_at_5_max": 0.5328714909701933, + "nauc_cv_recall_at_5_std": 0.18744484006449222, + "nauc_map_at_1000_diff1": 0.604318482617079, + "nauc_map_at_1000_max": 0.4925757726192787, + "nauc_map_at_1000_std": 0.09783991699344406, + "nauc_map_at_100_diff1": 0.6041514366932171, + "nauc_map_at_100_max": 0.4927559851221211, + "nauc_map_at_100_std": 0.09826761490615679, + "nauc_map_at_10_diff1": 0.6024278938638531, + "nauc_map_at_10_max": 0.4927211325536286, + "nauc_map_at_10_std": 0.09765222129343051, + "nauc_map_at_1_diff1": 0.6612644123585855, + "nauc_map_at_1_max": 0.4708878245975592, + "nauc_map_at_1_std": 0.048464967034370754, + "nauc_map_at_20_diff1": 0.6029216146779116, + "nauc_map_at_20_max": 0.4929855196311677, + "nauc_map_at_20_std": 0.09885380406817676, + "nauc_map_at_3_diff1": 0.6050308340493391, + "nauc_map_at_3_max": 0.48646198146939745, + "nauc_map_at_3_std": 0.08468646272239062, + "nauc_map_at_5_diff1": 0.6033105252155057, + "nauc_map_at_5_max": 0.4909150228364483, + "nauc_map_at_5_std": 0.09326129242686755, + "nauc_mrr_at_1000_diff1": 0.604318482617079, + "nauc_mrr_at_1000_max": 0.4925757726192787, + "nauc_mrr_at_1000_std": 0.09783991699344406, + "nauc_mrr_at_100_diff1": 0.6041514366932171, + "nauc_mrr_at_100_max": 0.4927559851221211, + "nauc_mrr_at_100_std": 0.09826761490615679, + "nauc_mrr_at_10_diff1": 0.6024278938638531, + "nauc_mrr_at_10_max": 0.4927211325536286, + "nauc_mrr_at_10_std": 0.09765222129343051, + "nauc_mrr_at_1_diff1": 0.6612644123585855, + "nauc_mrr_at_1_max": 0.4708878245975592, + "nauc_mrr_at_1_std": 0.048464967034370754, + "nauc_mrr_at_20_diff1": 0.6029216146779116, + "nauc_mrr_at_20_max": 0.4929855196311677, + "nauc_mrr_at_20_std": 0.09885380406817676, + "nauc_mrr_at_3_diff1": 0.6050308340493391, + "nauc_mrr_at_3_max": 0.48646198146939745, + "nauc_mrr_at_3_std": 0.08468646272239062, + "nauc_mrr_at_5_diff1": 0.6033105252155057, + "nauc_mrr_at_5_max": 0.4909150228364483, + "nauc_mrr_at_5_std": 0.09326129242686755, + "nauc_ndcg_at_1000_diff1": 0.5930751206638605, + "nauc_ndcg_at_1000_max": 0.5008525904823516, + "nauc_ndcg_at_1000_std": 0.11665718259501222, + "nauc_ndcg_at_100_diff1": 0.5884476368333909, + "nauc_ndcg_at_100_max": 0.5063649050617395, + "nauc_ndcg_at_100_std": 0.12946071575730214, + "nauc_ndcg_at_10_diff1": 0.5784819770863457, + "nauc_ndcg_at_10_max": 0.5055103201730918, + "nauc_ndcg_at_10_std": 0.12658901227383149, + "nauc_ndcg_at_1_diff1": 0.6612644123585855, + "nauc_ndcg_at_1_max": 0.4708878245975592, + "nauc_ndcg_at_1_std": 0.048464967034370754, + "nauc_ndcg_at_20_diff1": 0.5794062043265107, + "nauc_ndcg_at_20_max": 0.5075170784122032, + "nauc_ndcg_at_20_std": 0.13313497986974157, + "nauc_ndcg_at_3_diff1": 0.5861725439494837, + "nauc_ndcg_at_3_max": 0.4912503969392426, + "nauc_ndcg_at_3_std": 0.09671201364172016, + "nauc_ndcg_at_5_diff1": 0.5818577136220319, + "nauc_ndcg_at_5_max": 0.49996681584677094, + "nauc_ndcg_at_5_std": 0.11362132485163863, + "nauc_precision_at_1000_diff1": 0.5365388365818697, + "nauc_precision_at_1000_max": 0.8820883601096454, + "nauc_precision_at_1000_std": 0.8560327455019321, + "nauc_precision_at_100_diff1": 0.42911965616948045, + "nauc_precision_at_100_max": 0.7317432023038345, + "nauc_precision_at_100_std": 0.6364378846573614, + "nauc_precision_at_10_diff1": 0.4727513594062487, + "nauc_precision_at_10_max": 0.565976388070829, + "nauc_precision_at_10_std": 0.2629214710555803, + "nauc_precision_at_1_diff1": 0.6612644123585855, + "nauc_precision_at_1_max": 0.4708878245975592, + "nauc_precision_at_1_std": 0.048464967034370754, + "nauc_precision_at_20_diff1": 0.4415673817775351, + "nauc_precision_at_20_max": 0.6029237011588202, + "nauc_precision_at_20_std": 0.358819386174583, + "nauc_precision_at_3_diff1": 0.5273771194696367, + "nauc_precision_at_3_max": 0.506032051491813, + "nauc_precision_at_3_std": 0.13416342191731678, + "nauc_precision_at_5_diff1": 0.5064775255421813, + "nauc_precision_at_5_max": 0.5328714909701934, + "nauc_precision_at_5_std": 0.18744484006449183, + "nauc_recall_at_1000_diff1": 0.5365388365819015, + "nauc_recall_at_1000_max": 0.8820883601096524, + "nauc_recall_at_1000_std": 0.8560327455019514, + "nauc_recall_at_100_diff1": 0.4291196561694799, + "nauc_recall_at_100_max": 0.7317432023038329, + "nauc_recall_at_100_std": 0.6364378846573594, + "nauc_recall_at_10_diff1": 0.4727513594062488, + "nauc_recall_at_10_max": 0.565976388070829, + "nauc_recall_at_10_std": 0.2629214710555805, + "nauc_recall_at_1_diff1": 0.6612644123585855, + "nauc_recall_at_1_max": 0.4708878245975592, + "nauc_recall_at_1_std": 0.048464967034370754, + "nauc_recall_at_20_diff1": 0.44156738177753563, + "nauc_recall_at_20_max": 0.6029237011588209, + "nauc_recall_at_20_std": 0.3588193861745839, + "nauc_recall_at_3_diff1": 0.5273771194696363, + "nauc_recall_at_3_max": 0.5060320514918131, + "nauc_recall_at_3_std": 0.1341634219173162, + "nauc_recall_at_5_diff1": 0.5064775255421811, + "nauc_recall_at_5_max": 0.5328714909701933, + "nauc_recall_at_5_std": 0.18744484006449222, + "ndcg_at_1": 0.42659, + "ndcg_at_10": 0.58599, + "ndcg_at_100": 0.62479, + "ndcg_at_1000": 0.63179, + "ndcg_at_20": 0.60449, + "ndcg_at_3": 0.52827, + "ndcg_at_5": 0.55785, + "precision_at_1": 0.42659, + "precision_at_10": 0.07586, + "precision_at_100": 0.0094, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04158, + "precision_at_3": 0.20001, + "precision_at_5": 0.13437, + "recall_at_1": 0.42659, + "recall_at_10": 0.75863, + "recall_at_100": 0.94011, + "recall_at_1000": 0.9939, + "recall_at_20": 0.83169, + "recall_at_3": 0.60003, + "recall_at_5": 0.67184 + }, + { + "cv_recall_at_1": 0.50217, + "cv_recall_at_10": 0.8384, + "cv_recall_at_100": 0.97325, + "cv_recall_at_1000": 0.99762, + "cv_recall_at_20": 0.89693, + "cv_recall_at_3": 0.68758, + "cv_recall_at_5": 0.76012, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ], + "main_score": 0.66649, + "map_at_1": 0.50217, + "map_at_10": 0.61184, + "map_at_100": 0.61806, + "map_at_1000": 0.61819, + "map_at_20": 0.61596, + "map_at_3": 0.58461, + "map_at_5": 0.60119, + "mrr_at_1": 0.5021705643467301, + "mrr_at_10": 0.611844746300706, + "mrr_at_100": 0.6180557676786029, + "mrr_at_1000": 0.6181902933760485, + "mrr_at_20": 0.6159593318440516, + "mrr_at_3": 0.5846053307193233, + "mrr_at_5": 0.6011926434206231, + "nauc_cv_recall_at_1000_diff1": 0.6238997805255307, + "nauc_cv_recall_at_1000_max": 0.9737748390295698, + "nauc_cv_recall_at_1000_std": 0.9221778368717609, + "nauc_cv_recall_at_100_diff1": 0.4493973192301335, + "nauc_cv_recall_at_100_max": 0.7550416988642245, + "nauc_cv_recall_at_100_std": 0.5710271382364848, + "nauc_cv_recall_at_10_diff1": 0.5346408354506577, + "nauc_cv_recall_at_10_max": 0.5740556765410139, + "nauc_cv_recall_at_10_std": 0.11800066290994378, + "nauc_cv_recall_at_1_diff1": 0.6800424383383243, + "nauc_cv_recall_at_1_max": 0.44862302660287423, + "nauc_cv_recall_at_1_std": -0.04130418695693082, + "nauc_cv_recall_at_20_diff1": 0.5260408734169024, + "nauc_cv_recall_at_20_max": 0.6071704973496403, + "nauc_cv_recall_at_20_std": 0.20817459838843785, + "nauc_cv_recall_at_3_diff1": 0.5690289851202682, + "nauc_cv_recall_at_3_max": 0.49770044477603553, + "nauc_cv_recall_at_3_std": -0.005964362544451287, + "nauc_cv_recall_at_5_diff1": 0.5393013571222156, + "nauc_cv_recall_at_5_max": 0.5175617234123552, + "nauc_cv_recall_at_5_std": 0.035400452964310235, + "nauc_map_at_1000_diff1": 0.6376403336847224, + "nauc_map_at_1000_max": 0.47264358864353895, + "nauc_map_at_1000_std": -0.019520330304244982, + "nauc_map_at_100_diff1": 0.6375513504358796, + "nauc_map_at_100_max": 0.4727485339786326, + "nauc_map_at_100_std": -0.019298350612240715, + "nauc_map_at_10_diff1": 0.6358558627801074, + "nauc_map_at_10_max": 0.47356484765601586, + "nauc_map_at_10_std": -0.019397646543862737, + "nauc_map_at_1_diff1": 0.6800424383383243, + "nauc_map_at_1_max": 0.44862302660287423, + "nauc_map_at_1_std": -0.04130418695693082, + "nauc_map_at_20_diff1": 0.6369311067316696, + "nauc_map_at_20_max": 0.47322037710465403, + "nauc_map_at_20_std": -0.018869432047767736, + "nauc_map_at_3_diff1": 0.6357415873057528, + "nauc_map_at_3_max": 0.46874622110718456, + "nauc_map_at_3_std": -0.0291487680219996, + "nauc_map_at_5_diff1": 0.6335952409143635, + "nauc_map_at_5_max": 0.4711353116370656, + "nauc_map_at_5_std": -0.023090682569859827, + "nauc_mrr_at_1000_diff1": 0.6376403336847224, + "nauc_mrr_at_1000_max": 0.47264358864353895, + "nauc_mrr_at_1000_std": -0.019520330304244982, + "nauc_mrr_at_100_diff1": 0.6375513504358796, + "nauc_mrr_at_100_max": 0.4727485339786326, + "nauc_mrr_at_100_std": -0.019298350612240715, + "nauc_mrr_at_10_diff1": 0.6358558627801074, + "nauc_mrr_at_10_max": 0.47356484765601586, + "nauc_mrr_at_10_std": -0.019397646543862737, + "nauc_mrr_at_1_diff1": 0.6800424383383243, + "nauc_mrr_at_1_max": 0.44862302660287423, + "nauc_mrr_at_1_std": -0.04130418695693082, + "nauc_mrr_at_20_diff1": 0.6369311067316696, + "nauc_mrr_at_20_max": 0.47322037710465403, + "nauc_mrr_at_20_std": -0.018869432047767736, + "nauc_mrr_at_3_diff1": 0.6357415873057528, + "nauc_mrr_at_3_max": 0.46874622110718456, + "nauc_mrr_at_3_std": -0.0291487680219996, + "nauc_mrr_at_5_diff1": 0.6335952409143635, + "nauc_mrr_at_5_max": 0.4711353116370656, + "nauc_mrr_at_5_std": -0.023090682569859827, + "nauc_ndcg_at_1000_diff1": 0.6305642442540332, + "nauc_ndcg_at_1000_max": 0.4797823361804251, + "nauc_ndcg_at_1000_std": -0.008377140285703567, + "nauc_ndcg_at_100_diff1": 0.6278877828219978, + "nauc_ndcg_at_100_max": 0.48314104583429296, + "nauc_ndcg_at_100_std": -0.0011767182837512037, + "nauc_ndcg_at_10_diff1": 0.6199082824561567, + "nauc_ndcg_at_10_max": 0.48793329890789217, + "nauc_ndcg_at_10_std": -0.000376817961762273, + "nauc_ndcg_at_1_diff1": 0.6800424383383243, + "nauc_ndcg_at_1_max": 0.44862302660287423, + "nauc_ndcg_at_1_std": -0.04130418695693082, + "nauc_ndcg_at_20_diff1": 0.6235155170852444, + "nauc_ndcg_at_20_max": 0.4870249224862057, + "nauc_ndcg_at_20_std": 0.0028991542134987796, + "nauc_ndcg_at_3_diff1": 0.6204971834552783, + "nauc_ndcg_at_3_max": 0.4753871652584329, + "nauc_ndcg_at_3_std": -0.023940094636684603, + "nauc_ndcg_at_5_diff1": 0.6150442795355727, + "nauc_ndcg_at_5_max": 0.4802621376780359, + "nauc_ndcg_at_5_std": -0.011994226824793612, + "nauc_precision_at_1000_diff1": 0.6238997805255041, + "nauc_precision_at_1000_max": 0.9737748390295885, + "nauc_precision_at_1000_std": 0.922177836871779, + "nauc_precision_at_100_diff1": 0.44939731923013837, + "nauc_precision_at_100_max": 0.755041698864231, + "nauc_precision_at_100_std": 0.5710271382364832, + "nauc_precision_at_10_diff1": 0.534640835450658, + "nauc_precision_at_10_max": 0.5740556765410155, + "nauc_precision_at_10_std": 0.1180006629099428, + "nauc_precision_at_1_diff1": 0.6800424383383243, + "nauc_precision_at_1_max": 0.44862302660287423, + "nauc_precision_at_1_std": -0.04130418695693082, + "nauc_precision_at_20_diff1": 0.5260408734169023, + "nauc_precision_at_20_max": 0.6071704973496397, + "nauc_precision_at_20_std": 0.20817459838843586, + "nauc_precision_at_3_diff1": 0.5690289851202678, + "nauc_precision_at_3_max": 0.49770044477603614, + "nauc_precision_at_3_std": -0.005964362544451902, + "nauc_precision_at_5_diff1": 0.539301357122216, + "nauc_precision_at_5_max": 0.5175617234123555, + "nauc_precision_at_5_std": 0.03540045296431024, + "nauc_recall_at_1000_diff1": 0.6238997805255307, + "nauc_recall_at_1000_max": 0.9737748390295698, + "nauc_recall_at_1000_std": 0.9221778368717609, + "nauc_recall_at_100_diff1": 0.4493973192301335, + "nauc_recall_at_100_max": 0.7550416988642245, + "nauc_recall_at_100_std": 0.5710271382364848, + "nauc_recall_at_10_diff1": 0.5346408354506577, + "nauc_recall_at_10_max": 0.5740556765410139, + "nauc_recall_at_10_std": 0.11800066290994378, + "nauc_recall_at_1_diff1": 0.6800424383383243, + "nauc_recall_at_1_max": 0.44862302660287423, + "nauc_recall_at_1_std": -0.04130418695693082, + "nauc_recall_at_20_diff1": 0.5260408734169024, + "nauc_recall_at_20_max": 0.6071704973496403, + "nauc_recall_at_20_std": 0.20817459838843785, + "nauc_recall_at_3_diff1": 0.5690289851202682, + "nauc_recall_at_3_max": 0.49770044477603553, + "nauc_recall_at_3_std": -0.005964362544451287, + "nauc_recall_at_5_diff1": 0.5393013571222156, + "nauc_recall_at_5_max": 0.5175617234123552, + "nauc_recall_at_5_std": 0.035400452964310235, + "ndcg_at_1": 0.50217, + "ndcg_at_10": 0.66649, + "ndcg_at_100": 0.69579, + "ndcg_at_1000": 0.69899, + "ndcg_at_20": 0.68135, + "ndcg_at_3": 0.61108, + "ndcg_at_5": 0.64097, + "precision_at_1": 0.50217, + "precision_at_10": 0.08384, + "precision_at_100": 0.00973, + "precision_at_1000": 0.001, + "precision_at_20": 0.04485, + "precision_at_3": 0.22919, + "precision_at_5": 0.15202, + "recall_at_1": 0.50217, + "recall_at_10": 0.8384, + "recall_at_100": 0.97325, + "recall_at_1000": 0.99762, + "recall_at_20": 0.89693, + "recall_at_3": 0.68758, + "recall_at_5": 0.76012 + }, + { + "cv_recall_at_1": 0.62502, + "cv_recall_at_10": 0.90156, + "cv_recall_at_100": 0.98826, + "cv_recall_at_1000": 0.99945, + "cv_recall_at_20": 0.93953, + "cv_recall_at_3": 0.78421, + "cv_recall_at_5": 0.84123, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ], + "main_score": 0.76169, + "map_at_1": 0.62502, + "map_at_10": 0.71706, + "map_at_100": 0.72117, + "map_at_1000": 0.72123, + "map_at_20": 0.71975, + "map_at_3": 0.69582, + "map_at_5": 0.70887, + "mrr_at_1": 0.6250172580422477, + "mrr_at_10": 0.7170563696968486, + "mrr_at_100": 0.721167438819844, + "mrr_at_1000": 0.721228572929272, + "mrr_at_20": 0.7197508171057015, + "mrr_at_3": 0.69582125270376, + "mrr_at_5": 0.7088683326430089, + "nauc_cv_recall_at_1000_diff1": 0.6038682984684438, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.6397914759225447, + "nauc_cv_recall_at_100_max": 0.7941256297154522, + "nauc_cv_recall_at_100_std": 0.5105784983752181, + "nauc_cv_recall_at_10_diff1": 0.6213519862367571, + "nauc_cv_recall_at_10_max": 0.5579690033478077, + "nauc_cv_recall_at_10_std": -0.04098643332655127, + "nauc_cv_recall_at_1_diff1": 0.7490694455433466, + "nauc_cv_recall_at_1_max": 0.441556203916839, + "nauc_cv_recall_at_1_std": -0.1558626182312551, + "nauc_cv_recall_at_20_diff1": 0.6381809212766827, + "nauc_cv_recall_at_20_max": 0.601919642260997, + "nauc_cv_recall_at_20_std": 0.06712229461141428, + "nauc_cv_recall_at_3_diff1": 0.6620309591987041, + "nauc_cv_recall_at_3_max": 0.4771822776364749, + "nauc_cv_recall_at_3_std": -0.15353925517434355, + "nauc_cv_recall_at_5_diff1": 0.6476318184903693, + "nauc_cv_recall_at_5_max": 0.5013983365902959, + "nauc_cv_recall_at_5_std": -0.13494507379350235, + "nauc_map_at_1000_diff1": 0.7192701129566936, + "nauc_map_at_1000_max": 0.4583892756334078, + "nauc_map_at_1000_std": -0.14998938985883367, + "nauc_map_at_100_diff1": 0.7192547571330423, + "nauc_map_at_100_max": 0.4584642094024353, + "nauc_map_at_100_std": -0.14984408132811972, + "nauc_map_at_10_diff1": 0.7175606048461799, + "nauc_map_at_10_max": 0.4592566978388754, + "nauc_map_at_10_std": -0.14999184568623453, + "nauc_map_at_1_diff1": 0.7490694455433466, + "nauc_map_at_1_max": 0.441556203916839, + "nauc_map_at_1_std": -0.1558626182312551, + "nauc_map_at_20_diff1": 0.7188548018807867, + "nauc_map_at_20_max": 0.4589323914725497, + "nauc_map_at_20_std": -0.1493844447508069, + "nauc_map_at_3_diff1": 0.7168356111328161, + "nauc_map_at_3_max": 0.45474550225239724, + "nauc_map_at_3_std": -0.15736836775108023, + "nauc_map_at_5_diff1": 0.7170041475021451, + "nauc_map_at_5_max": 0.4571011403468147, + "nauc_map_at_5_std": -0.1550665804562101, + "nauc_mrr_at_1000_diff1": 0.7192701129566936, + "nauc_mrr_at_1000_max": 0.4583892756334078, + "nauc_mrr_at_1000_std": -0.14998938985883367, + "nauc_mrr_at_100_diff1": 0.7192547571330423, + "nauc_mrr_at_100_max": 0.4584642094024353, + "nauc_mrr_at_100_std": -0.14984408132811972, + "nauc_mrr_at_10_diff1": 0.7175606048461799, + "nauc_mrr_at_10_max": 0.4592566978388754, + "nauc_mrr_at_10_std": -0.14999184568623453, + "nauc_mrr_at_1_diff1": 0.7490694455433466, + "nauc_mrr_at_1_max": 0.441556203916839, + "nauc_mrr_at_1_std": -0.1558626182312551, + "nauc_mrr_at_20_diff1": 0.7188548018807867, + "nauc_mrr_at_20_max": 0.4589323914725497, + "nauc_mrr_at_20_std": -0.1493844447508069, + "nauc_mrr_at_3_diff1": 0.7168356111328161, + "nauc_mrr_at_3_max": 0.45474550225239724, + "nauc_mrr_at_3_std": -0.15736836775108023, + "nauc_mrr_at_5_diff1": 0.7170041475021451, + "nauc_mrr_at_5_max": 0.4571011403468147, + "nauc_mrr_at_5_std": -0.1550665804562101, + "nauc_ndcg_at_1000_diff1": 0.7145970154353453, + "nauc_ndcg_at_1000_max": 0.4635310945979437, + "nauc_ndcg_at_1000_std": -0.14364931096142944, + "nauc_ndcg_at_100_diff1": 0.7140530995986156, + "nauc_ndcg_at_100_max": 0.46598217186904245, + "nauc_ndcg_at_100_std": -0.13883751906403827, + "nauc_ndcg_at_10_diff1": 0.705282613937573, + "nauc_ndcg_at_10_max": 0.4708566499849579, + "nauc_ndcg_at_10_std": -0.13809732008953995, + "nauc_ndcg_at_1_diff1": 0.7490694455433466, + "nauc_ndcg_at_1_max": 0.441556203916839, + "nauc_ndcg_at_1_std": -0.1558626182312551, + "nauc_ndcg_at_20_diff1": 0.7106077672375423, + "nauc_ndcg_at_20_max": 0.4700208803914439, + "nauc_ndcg_at_20_std": -0.13454089853888754, + "nauc_ndcg_at_3_diff1": 0.7051459214080644, + "nauc_ndcg_at_3_max": 0.4595309984418196, + "nauc_ndcg_at_3_std": -0.1566627942812548, + "nauc_ndcg_at_5_diff1": 0.7046798864341236, + "nauc_ndcg_at_5_max": 0.46471324477089654, + "nauc_ndcg_at_5_std": -0.15186183615748197, + "nauc_precision_at_1000_diff1": 0.6038682984685291, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.6397914759225701, + "nauc_precision_at_100_max": 0.794125629715474, + "nauc_precision_at_100_std": 0.5105784983752383, + "nauc_precision_at_10_diff1": 0.6213519862367571, + "nauc_precision_at_10_max": 0.5579690033478083, + "nauc_precision_at_10_std": -0.04098643332655233, + "nauc_precision_at_1_diff1": 0.7490694455433466, + "nauc_precision_at_1_max": 0.441556203916839, + "nauc_precision_at_1_std": -0.1558626182312551, + "nauc_precision_at_20_diff1": 0.6381809212766826, + "nauc_precision_at_20_max": 0.6019196422609968, + "nauc_precision_at_20_std": 0.06712229461141557, + "nauc_precision_at_3_diff1": 0.6620309591987041, + "nauc_precision_at_3_max": 0.4771822776364753, + "nauc_precision_at_3_std": -0.15353925517434228, + "nauc_precision_at_5_diff1": 0.6476318184903682, + "nauc_precision_at_5_max": 0.5013983365902952, + "nauc_precision_at_5_std": -0.13494507379350357, + "nauc_recall_at_1000_diff1": 0.6038682984684438, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.6397914759225447, + "nauc_recall_at_100_max": 0.7941256297154522, + "nauc_recall_at_100_std": 0.5105784983752181, + "nauc_recall_at_10_diff1": 0.6213519862367571, + "nauc_recall_at_10_max": 0.5579690033478077, + "nauc_recall_at_10_std": -0.04098643332655127, + "nauc_recall_at_1_diff1": 0.7490694455433466, + "nauc_recall_at_1_max": 0.441556203916839, + "nauc_recall_at_1_std": -0.1558626182312551, + "nauc_recall_at_20_diff1": 0.6381809212766827, + "nauc_recall_at_20_max": 0.601919642260997, + "nauc_recall_at_20_std": 0.06712229461141428, + "nauc_recall_at_3_diff1": 0.6620309591987041, + "nauc_recall_at_3_max": 0.4771822776364749, + "nauc_recall_at_3_std": -0.15353925517434355, + "nauc_recall_at_5_diff1": 0.6476318184903693, + "nauc_recall_at_5_max": 0.5013983365902959, + "nauc_recall_at_5_std": -0.13494507379350235, + "ndcg_at_1": 0.62502, + "ndcg_at_10": 0.76169, + "ndcg_at_100": 0.78073, + "ndcg_at_1000": 0.78219, + "ndcg_at_20": 0.77137, + "ndcg_at_3": 0.71855, + "ndcg_at_5": 0.74205, + "precision_at_1": 0.62502, + "precision_at_10": 0.09016, + "precision_at_100": 0.00988, + "precision_at_1000": 0.001, + "precision_at_20": 0.04698, + "precision_at_3": 0.2614, + "precision_at_5": 0.16825, + "recall_at_1": 0.62502, + "recall_at_10": 0.90156, + "recall_at_100": 0.98826, + "recall_at_1000": 0.99945, + "recall_at_20": 0.93953, + "recall_at_3": 0.78421, + "recall_at_5": 0.84123 + }, + { + "cv_recall_at_1": 0.02292, + "cv_recall_at_10": 0.08417, + "cv_recall_at_100": 0.20569, + "cv_recall_at_1000": 0.52736, + "cv_recall_at_20": 0.11222, + "cv_recall_at_3": 0.04569, + "cv_recall_at_5": 0.05917, + "hf_subset": "quz", + "languages": [ + "quz-Latn" + ], + "main_score": 0.0496, + "map_at_1": 0.02292, + "map_at_10": 0.03905, + "map_at_100": 0.04313, + "map_at_1000": 0.04414, + "map_at_20": 0.04097, + "map_at_3": 0.03266, + "map_at_5": 0.03575, + "mrr_at_1": 0.022916666666666665, + "mrr_at_10": 0.03905042989417988, + "mrr_at_100": 0.04313296486782967, + "mrr_at_1000": 0.044140873349093286, + "mrr_at_20": 0.0409663441169826, + "mrr_at_3": 0.03266203703703707, + "mrr_at_5": 0.03574537037037037, + "nauc_cv_recall_at_1000_diff1": 0.0556925167648709, + "nauc_cv_recall_at_1000_max": 0.41275860775657136, + "nauc_cv_recall_at_1000_std": 0.4770003851498549, + "nauc_cv_recall_at_100_diff1": 0.10493534615151756, + "nauc_cv_recall_at_100_max": 0.5412833456822512, + "nauc_cv_recall_at_100_std": 0.5932599080938897, + "nauc_cv_recall_at_10_diff1": 0.19215412284453554, + "nauc_cv_recall_at_10_max": 0.6037828494413702, + "nauc_cv_recall_at_10_std": 0.6119495502005258, + "nauc_cv_recall_at_1_diff1": 0.43177793497085165, + "nauc_cv_recall_at_1_max": 0.7547918140351733, + "nauc_cv_recall_at_1_std": 0.7091445715540161, + "nauc_cv_recall_at_20_diff1": 0.15027188078341383, + "nauc_cv_recall_at_20_max": 0.5905053410817863, + "nauc_cv_recall_at_20_std": 0.6199595523034361, + "nauc_cv_recall_at_3_diff1": 0.2763078594304439, + "nauc_cv_recall_at_3_max": 0.6678081493545414, + "nauc_cv_recall_at_3_std": 0.642089312932898, + "nauc_cv_recall_at_5_diff1": 0.23357027956127785, + "nauc_cv_recall_at_5_max": 0.644227947058075, + "nauc_cv_recall_at_5_std": 0.6378463968679261, + "nauc_map_at_1000_diff1": 0.2796256774634396, + "nauc_map_at_1000_max": 0.6650125561397447, + "nauc_map_at_1000_std": 0.6516817769669179, + "nauc_map_at_100_diff1": 0.2814989691212026, + "nauc_map_at_100_max": 0.6673002299701124, + "nauc_map_at_100_std": 0.6531316581428467, + "nauc_map_at_10_diff1": 0.3016853183868804, + "nauc_map_at_10_max": 0.6788300049355528, + "nauc_map_at_10_std": 0.6563158970332853, + "nauc_map_at_1_diff1": 0.43177793497085165, + "nauc_map_at_1_max": 0.7547918140351733, + "nauc_map_at_1_std": 0.7091445715540161, + "nauc_map_at_20_diff1": 0.28995599574333136, + "nauc_map_at_20_max": 0.6727288716216021, + "nauc_map_at_20_std": 0.6551859095502353, + "nauc_map_at_3_diff1": 0.3416224280952469, + "nauc_map_at_3_max": 0.7058933820795898, + "nauc_map_at_3_std": 0.6695571249470176, + "nauc_map_at_5_diff1": 0.32038815315465746, + "nauc_map_at_5_max": 0.6941412662777497, + "nauc_map_at_5_std": 0.6659616445790018, + "nauc_mrr_at_1000_diff1": 0.27962567785184494, + "nauc_mrr_at_1000_max": 0.6650125299359211, + "nauc_mrr_at_1000_std": 0.6516817509750019, + "nauc_mrr_at_100_diff1": 0.2814989691212026, + "nauc_mrr_at_100_max": 0.6673002299701124, + "nauc_mrr_at_100_std": 0.6531316581428467, + "nauc_mrr_at_10_diff1": 0.3016853183868804, + "nauc_mrr_at_10_max": 0.6788300049355528, + "nauc_mrr_at_10_std": 0.6563158970332853, + "nauc_mrr_at_1_diff1": 0.43177793497085165, + "nauc_mrr_at_1_max": 0.7547918140351733, + "nauc_mrr_at_1_std": 0.7091445715540161, + "nauc_mrr_at_20_diff1": 0.28995599574333136, + "nauc_mrr_at_20_max": 0.6727288716216021, + "nauc_mrr_at_20_std": 0.6551859095502353, + "nauc_mrr_at_3_diff1": 0.3416224280952469, + "nauc_mrr_at_3_max": 0.7058933820795898, + "nauc_mrr_at_3_std": 0.6695571249470176, + "nauc_mrr_at_5_diff1": 0.32038815315465746, + "nauc_mrr_at_5_max": 0.6941412662777497, + "nauc_mrr_at_5_std": 0.6659616445790018, + "nauc_ndcg_at_1000_diff1": 0.17514521567782645, + "nauc_ndcg_at_1000_max": 0.5643365826227588, + "nauc_ndcg_at_1000_std": 0.5877111863477217, + "nauc_ndcg_at_100_diff1": 0.19774191621612744, + "nauc_ndcg_at_100_max": 0.6098229146580955, + "nauc_ndcg_at_100_std": 0.625863941031239, + "nauc_ndcg_at_10_diff1": 0.2585250624624412, + "nauc_ndcg_at_10_max": 0.6496629866590963, + "nauc_ndcg_at_10_std": 0.6389591527927075, + "nauc_ndcg_at_1_diff1": 0.43177793497085165, + "nauc_ndcg_at_1_max": 0.7547918140351733, + "nauc_ndcg_at_1_std": 0.7091445715540161, + "nauc_ndcg_at_20_diff1": 0.23119336697129492, + "nauc_ndcg_at_20_max": 0.637617837219863, + "nauc_ndcg_at_20_std": 0.6392094102892021, + "nauc_ndcg_at_3_diff1": 0.32051666816022684, + "nauc_ndcg_at_3_max": 0.6936419572495052, + "nauc_ndcg_at_3_std": 0.660655911138741, + "nauc_ndcg_at_5_diff1": 0.2899701948259364, + "nauc_ndcg_at_5_max": 0.6767013412913847, + "nauc_ndcg_at_5_std": 0.6559718938617654, + "nauc_precision_at_1000_diff1": 0.055692516764870165, + "nauc_precision_at_1000_max": 0.4127586077565709, + "nauc_precision_at_1000_std": 0.4770003851498547, + "nauc_precision_at_100_diff1": 0.1049353461515177, + "nauc_precision_at_100_max": 0.5412833456822514, + "nauc_precision_at_100_std": 0.59325990809389, + "nauc_precision_at_10_diff1": 0.19215412284453554, + "nauc_precision_at_10_max": 0.6037828494413703, + "nauc_precision_at_10_std": 0.6119495502005261, + "nauc_precision_at_1_diff1": 0.43177793497085165, + "nauc_precision_at_1_max": 0.7547918140351733, + "nauc_precision_at_1_std": 0.7091445715540161, + "nauc_precision_at_20_diff1": 0.15027188078341375, + "nauc_precision_at_20_max": 0.5905053410817863, + "nauc_precision_at_20_std": 0.619959552303436, + "nauc_precision_at_3_diff1": 0.2763078594304441, + "nauc_precision_at_3_max": 0.6678081493545418, + "nauc_precision_at_3_std": 0.6420893129328985, + "nauc_precision_at_5_diff1": 0.2335702795612778, + "nauc_precision_at_5_max": 0.6442279470580751, + "nauc_precision_at_5_std": 0.6378463968679261, + "nauc_recall_at_1000_diff1": 0.0556925167648709, + "nauc_recall_at_1000_max": 0.41275860775657136, + "nauc_recall_at_1000_std": 0.4770003851498549, + "nauc_recall_at_100_diff1": 0.10493534615151756, + "nauc_recall_at_100_max": 0.5412833456822512, + "nauc_recall_at_100_std": 0.5932599080938897, + "nauc_recall_at_10_diff1": 0.19215412284453554, + "nauc_recall_at_10_max": 0.6037828494413702, + "nauc_recall_at_10_std": 0.6119495502005258, + "nauc_recall_at_1_diff1": 0.43177793497085165, + "nauc_recall_at_1_max": 0.7547918140351733, + "nauc_recall_at_1_std": 0.7091445715540161, + "nauc_recall_at_20_diff1": 0.15027188078341383, + "nauc_recall_at_20_max": 0.5905053410817863, + "nauc_recall_at_20_std": 0.6199595523034361, + "nauc_recall_at_3_diff1": 0.2763078594304439, + "nauc_recall_at_3_max": 0.6678081493545414, + "nauc_recall_at_3_std": 0.642089312932898, + "nauc_recall_at_5_diff1": 0.23357027956127785, + "nauc_recall_at_5_max": 0.644227947058075, + "nauc_recall_at_5_std": 0.6378463968679261, + "ndcg_at_1": 0.02292, + "ndcg_at_10": 0.0496, + "ndcg_at_100": 0.07353, + "ndcg_at_1000": 0.11124, + "ndcg_at_20": 0.05666, + "ndcg_at_3": 0.036, + "ndcg_at_5": 0.04155, + "precision_at_1": 0.02292, + "precision_at_10": 0.00842, + "precision_at_100": 0.00206, + "precision_at_1000": 0.00053, + "precision_at_20": 0.00561, + "precision_at_3": 0.01523, + "precision_at_5": 0.01183, + "recall_at_1": 0.02292, + "recall_at_10": 0.08417, + "recall_at_100": 0.20569, + "recall_at_1000": 0.52736, + "recall_at_20": 0.11222, + "recall_at_3": 0.04569, + "recall_at_5": 0.05917 + }, + { + "cv_recall_at_1": 0.36768, + "cv_recall_at_10": 0.67556, + "cv_recall_at_100": 0.89723, + "cv_recall_at_1000": 0.98708, + "cv_recall_at_20": 0.75516, + "cv_recall_at_3": 0.52352, + "cv_recall_at_5": 0.58964, + "hf_subset": "ro", + "languages": [ + "ron-Latn" + ], + "main_score": 0.51408, + "map_at_1": 0.36768, + "map_at_10": 0.46332, + "map_at_100": 0.47262, + "map_at_1000": 0.47304, + "map_at_20": 0.46897, + "map_at_3": 0.43661, + "map_at_5": 0.45175, + "mrr_at_1": 0.36768215639477747, + "mrr_at_10": 0.4633156842689346, + "mrr_at_100": 0.4726189598420947, + "mrr_at_1000": 0.47304219633255584, + "mrr_at_20": 0.46896763103749506, + "mrr_at_3": 0.4366137863259877, + "mrr_at_5": 0.4517478590481541, + "nauc_cv_recall_at_1000_diff1": 0.4610798974007453, + "nauc_cv_recall_at_1000_max": 0.8955736220454759, + "nauc_cv_recall_at_1000_std": 0.9412424160278959, + "nauc_cv_recall_at_100_diff1": 0.4108601422390299, + "nauc_cv_recall_at_100_max": 0.6956175426234881, + "nauc_cv_recall_at_100_std": 0.6343823723345061, + "nauc_cv_recall_at_10_diff1": 0.4540269841800468, + "nauc_cv_recall_at_10_max": 0.5812748871363784, + "nauc_cv_recall_at_10_std": 0.330293834235662, + "nauc_cv_recall_at_1_diff1": 0.6328181504850688, + "nauc_cv_recall_at_1_max": 0.516415613700369, + "nauc_cv_recall_at_1_std": 0.1613423219822137, + "nauc_cv_recall_at_20_diff1": 0.4215794118456787, + "nauc_cv_recall_at_20_max": 0.6175782422623985, + "nauc_cv_recall_at_20_std": 0.42480252175436806, + "nauc_cv_recall_at_3_diff1": 0.5112768505535457, + "nauc_cv_recall_at_3_max": 0.5433888785986725, + "nauc_cv_recall_at_3_std": 0.21539500930972913, + "nauc_cv_recall_at_5_diff1": 0.4767674258764573, + "nauc_cv_recall_at_5_max": 0.5603999867601904, + "nauc_cv_recall_at_5_std": 0.26000574623796224, + "nauc_map_at_1000_diff1": 0.5725380617205134, + "nauc_map_at_1000_max": 0.5348171726552217, + "nauc_map_at_1000_std": 0.2024265935259748, + "nauc_map_at_100_diff1": 0.5723428685740998, + "nauc_map_at_100_max": 0.5349390652281885, + "nauc_map_at_100_std": 0.20281321546350395, + "nauc_map_at_10_diff1": 0.571185933778101, + "nauc_map_at_10_max": 0.5338641080299099, + "nauc_map_at_10_std": 0.19995969194038202, + "nauc_map_at_1_diff1": 0.6328181504850688, + "nauc_map_at_1_max": 0.516415613700369, + "nauc_map_at_1_std": 0.1613423219822137, + "nauc_map_at_20_diff1": 0.5710745751223294, + "nauc_map_at_20_max": 0.5350518763447834, + "nauc_map_at_20_std": 0.20302780619493885, + "nauc_map_at_3_diff1": 0.5781262316460313, + "nauc_map_at_3_max": 0.5287022746286781, + "nauc_map_at_3_std": 0.18451064634106332, + "nauc_map_at_5_diff1": 0.5720343476770393, + "nauc_map_at_5_max": 0.5322655936266794, + "nauc_map_at_5_std": 0.19351069901578477, + "nauc_mrr_at_1000_diff1": 0.5725380514865761, + "nauc_mrr_at_1000_max": 0.5348171727699603, + "nauc_mrr_at_1000_std": 0.20242659444987554, + "nauc_mrr_at_100_diff1": 0.5723428685740998, + "nauc_mrr_at_100_max": 0.5349390652281885, + "nauc_mrr_at_100_std": 0.20281321546350395, + "nauc_mrr_at_10_diff1": 0.571185933778101, + "nauc_mrr_at_10_max": 0.5338641080299099, + "nauc_mrr_at_10_std": 0.19995969194038202, + "nauc_mrr_at_1_diff1": 0.6328181504850688, + "nauc_mrr_at_1_max": 0.516415613700369, + "nauc_mrr_at_1_std": 0.1613423219822137, + "nauc_mrr_at_20_diff1": 0.5710745751223294, + "nauc_mrr_at_20_max": 0.5350518763447834, + "nauc_mrr_at_20_std": 0.20302780619493885, + "nauc_mrr_at_3_diff1": 0.5781262316460313, + "nauc_mrr_at_3_max": 0.5287022746286781, + "nauc_mrr_at_3_std": 0.18451064634106332, + "nauc_mrr_at_5_diff1": 0.5720343476770393, + "nauc_mrr_at_5_max": 0.5322655936266794, + "nauc_mrr_at_5_std": 0.19351069901578477, + "nauc_ndcg_at_1000_diff1": 0.55932571244852, + "nauc_ndcg_at_1000_max": 0.5438659750081213, + "nauc_ndcg_at_1000_std": 0.22493836703581904, + "nauc_ndcg_at_100_diff1": 0.5533220616630024, + "nauc_ndcg_at_100_max": 0.5483763183365132, + "nauc_ndcg_at_100_std": 0.23857994788319828, + "nauc_ndcg_at_10_diff1": 0.5464225493257165, + "nauc_ndcg_at_10_max": 0.5434379756441989, + "nauc_ndcg_at_10_std": 0.225836309928415, + "nauc_ndcg_at_1_diff1": 0.6328181504850688, + "nauc_ndcg_at_1_max": 0.516415613700369, + "nauc_ndcg_at_1_std": 0.1613423219822137, + "nauc_ndcg_at_20_diff1": 0.5447313710226348, + "nauc_ndcg_at_20_max": 0.5483598729493525, + "nauc_ndcg_at_20_std": 0.23846248552590435, + "nauc_ndcg_at_3_diff1": 0.5612078893755854, + "nauc_ndcg_at_3_max": 0.5324257577340359, + "nauc_ndcg_at_3_std": 0.1922759753998782, + "nauc_ndcg_at_5_diff1": 0.5494157617079797, + "nauc_ndcg_at_5_max": 0.5388763844572947, + "nauc_ndcg_at_5_std": 0.20893951759575607, + "nauc_precision_at_1000_diff1": 0.4610798974007486, + "nauc_precision_at_1000_max": 0.8955736220454706, + "nauc_precision_at_1000_std": 0.9412424160278849, + "nauc_precision_at_100_diff1": 0.4108601422390304, + "nauc_precision_at_100_max": 0.6956175426234907, + "nauc_precision_at_100_std": 0.6343823723345081, + "nauc_precision_at_10_diff1": 0.45402698418004667, + "nauc_precision_at_10_max": 0.5812748871363789, + "nauc_precision_at_10_std": 0.33029383423566167, + "nauc_precision_at_1_diff1": 0.6328181504850688, + "nauc_precision_at_1_max": 0.516415613700369, + "nauc_precision_at_1_std": 0.1613423219822137, + "nauc_precision_at_20_diff1": 0.42157941184567777, + "nauc_precision_at_20_max": 0.6175782422623981, + "nauc_precision_at_20_std": 0.42480252175436756, + "nauc_precision_at_3_diff1": 0.511276850553546, + "nauc_precision_at_3_max": 0.5433888785986725, + "nauc_precision_at_3_std": 0.21539500930972955, + "nauc_precision_at_5_diff1": 0.4767674258764569, + "nauc_precision_at_5_max": 0.5603999867601905, + "nauc_precision_at_5_std": 0.2600057462379625, + "nauc_recall_at_1000_diff1": 0.4610798974007453, + "nauc_recall_at_1000_max": 0.8955736220454759, + "nauc_recall_at_1000_std": 0.9412424160278959, + "nauc_recall_at_100_diff1": 0.4108601422390299, + "nauc_recall_at_100_max": 0.6956175426234881, + "nauc_recall_at_100_std": 0.6343823723345061, + "nauc_recall_at_10_diff1": 0.4540269841800468, + "nauc_recall_at_10_max": 0.5812748871363784, + "nauc_recall_at_10_std": 0.330293834235662, + "nauc_recall_at_1_diff1": 0.6328181504850688, + "nauc_recall_at_1_max": 0.516415613700369, + "nauc_recall_at_1_std": 0.1613423219822137, + "nauc_recall_at_20_diff1": 0.4215794118456787, + "nauc_recall_at_20_max": 0.6175782422623985, + "nauc_recall_at_20_std": 0.42480252175436806, + "nauc_recall_at_3_diff1": 0.5112768505535457, + "nauc_recall_at_3_max": 0.5433888785986725, + "nauc_recall_at_3_std": 0.21539500930972913, + "nauc_recall_at_5_diff1": 0.4767674258764573, + "nauc_recall_at_5_max": 0.5603999867601904, + "nauc_recall_at_5_std": 0.26000574623796224, + "ndcg_at_1": 0.36768, + "ndcg_at_10": 0.51408, + "ndcg_at_100": 0.56075, + "ndcg_at_1000": 0.57214, + "ndcg_at_20": 0.53437, + "ndcg_at_3": 0.45894, + "ndcg_at_5": 0.4862, + "precision_at_1": 0.36768, + "precision_at_10": 0.06756, + "precision_at_100": 0.00897, + "precision_at_1000": 0.00099, + "precision_at_20": 0.03776, + "precision_at_3": 0.17451, + "precision_at_5": 0.11793, + "recall_at_1": 0.36768, + "recall_at_10": 0.67556, + "recall_at_100": 0.89723, + "recall_at_1000": 0.98708, + "recall_at_20": 0.75516, + "recall_at_3": 0.52352, + "recall_at_5": 0.58964 + }, + { + "cv_recall_at_1": 0.6875, + "cv_recall_at_10": 0.94847, + "cv_recall_at_100": 0.99667, + "cv_recall_at_1000": 0.99958, + "cv_recall_at_20": 0.97556, + "cv_recall_at_3": 0.8525, + "cv_recall_at_5": 0.905, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.82082, + "map_at_1": 0.6875, + "map_at_10": 0.77952, + "map_at_100": 0.78212, + "map_at_1000": 0.78214, + "map_at_20": 0.78149, + "map_at_3": 0.76155, + "map_at_5": 0.77351, + "mrr_at_1": 0.6875, + "mrr_at_10": 0.7795203373015837, + "mrr_at_100": 0.7821174627343088, + "mrr_at_1000": 0.7821387642208056, + "mrr_at_20": 0.7814853532093554, + "mrr_at_3": 0.7615509259259242, + "mrr_at_5": 0.7735092592592552, + "nauc_cv_recall_at_1000_diff1": 0.22455648926279748, + "nauc_cv_recall_at_1000_max": 0.7424525365703915, + "nauc_cv_recall_at_1000_std": 0.28275754746343273, + "nauc_cv_recall_at_100_diff1": 0.6293183940242689, + "nauc_cv_recall_at_100_max": 0.6672307812013722, + "nauc_cv_recall_at_100_std": 0.021241830065380926, + "nauc_cv_recall_at_10_diff1": 0.6660374841760586, + "nauc_cv_recall_at_10_max": 0.5524939535562631, + "nauc_cv_recall_at_10_std": -0.2843640600894453, + "nauc_cv_recall_at_1_diff1": 0.7775443213802683, + "nauc_cv_recall_at_1_max": 0.4261422426923034, + "nauc_cv_recall_at_1_std": -0.2797555774877853, + "nauc_cv_recall_at_20_diff1": 0.6828128978864284, + "nauc_cv_recall_at_20_max": 0.6325571895424784, + "nauc_cv_recall_at_20_std": -0.14440094219505392, + "nauc_cv_recall_at_3_diff1": 0.6895588157008296, + "nauc_cv_recall_at_3_max": 0.4706256505146292, + "nauc_cv_recall_at_3_std": -0.350748309809257, + "nauc_cv_recall_at_5_diff1": 0.6537804205502865, + "nauc_cv_recall_at_5_max": 0.48176268558105567, + "nauc_cv_recall_at_5_std": -0.35012299266685093, + "nauc_map_at_1000_diff1": 0.7523940563023707, + "nauc_map_at_1000_max": 0.4397824911340208, + "nauc_map_at_1000_std": -0.2945689733295087, + "nauc_map_at_100_diff1": 0.7523891794383645, + "nauc_map_at_100_max": 0.4398054057499954, + "nauc_map_at_100_std": -0.2945364909899665, + "nauc_map_at_10_diff1": 0.7511612685448151, + "nauc_map_at_10_max": 0.44062613325240885, + "nauc_map_at_10_std": -0.2953424639709468, + "nauc_map_at_1_diff1": 0.7775443213802683, + "nauc_map_at_1_max": 0.4261422426923034, + "nauc_map_at_1_std": -0.2797555774877853, + "nauc_map_at_20_diff1": 0.7521905452599742, + "nauc_map_at_20_max": 0.4403356639427312, + "nauc_map_at_20_std": -0.2942524185462377, + "nauc_map_at_3_diff1": 0.7485436882265807, + "nauc_map_at_3_max": 0.43994611267483863, + "nauc_map_at_3_std": -0.30159040020254646, + "nauc_map_at_5_diff1": 0.747667991627285, + "nauc_map_at_5_max": 0.4393237311004251, + "nauc_map_at_5_std": -0.29926015528317973, + "nauc_mrr_at_1000_diff1": 0.7523940563023707, + "nauc_mrr_at_1000_max": 0.4397824911340208, + "nauc_mrr_at_1000_std": -0.2945689733295087, + "nauc_mrr_at_100_diff1": 0.7523891794383645, + "nauc_mrr_at_100_max": 0.4398054057499954, + "nauc_mrr_at_100_std": -0.2945364909899665, + "nauc_mrr_at_10_diff1": 0.7511612685448151, + "nauc_mrr_at_10_max": 0.44062613325240885, + "nauc_mrr_at_10_std": -0.2953424639709468, + "nauc_mrr_at_1_diff1": 0.7775443213802683, + "nauc_mrr_at_1_max": 0.4261422426923034, + "nauc_mrr_at_1_std": -0.2797555774877853, + "nauc_mrr_at_20_diff1": 0.7521905452599742, + "nauc_mrr_at_20_max": 0.4403356639427312, + "nauc_mrr_at_20_std": -0.2942524185462377, + "nauc_mrr_at_3_diff1": 0.7485436882265807, + "nauc_mrr_at_3_max": 0.43994611267483863, + "nauc_mrr_at_3_std": -0.30159040020254646, + "nauc_mrr_at_5_diff1": 0.747667991627285, + "nauc_mrr_at_5_max": 0.4393237311004251, + "nauc_mrr_at_5_std": -0.29926015528317973, + "nauc_ndcg_at_1000_diff1": 0.7489550956775304, + "nauc_ndcg_at_1000_max": 0.44306305389667855, + "nauc_ndcg_at_1000_std": -0.29462236952032134, + "nauc_ndcg_at_100_diff1": 0.7487981693510771, + "nauc_ndcg_at_100_max": 0.4436446927443267, + "nauc_ndcg_at_100_std": -0.2938447251654681, + "nauc_ndcg_at_10_diff1": 0.7430649301150828, + "nauc_ndcg_at_10_max": 0.44957064968735844, + "nauc_ndcg_at_10_std": -0.295938010067524, + "nauc_ndcg_at_1_diff1": 0.7775443213802683, + "nauc_ndcg_at_1_max": 0.4261422426923034, + "nauc_ndcg_at_1_std": -0.2797555774877853, + "nauc_ndcg_at_20_diff1": 0.747269951253347, + "nauc_ndcg_at_20_max": 0.44835823024848326, + "nauc_ndcg_at_20_std": -0.2908477349842589, + "nauc_ndcg_at_3_diff1": 0.7373188355954172, + "nauc_ndcg_at_3_max": 0.44574261091161155, + "nauc_ndcg_at_3_std": -0.31086889192437706, + "nauc_ndcg_at_5_diff1": 0.7343963293135684, + "nauc_ndcg_at_5_max": 0.4453075054682102, + "nauc_ndcg_at_5_std": -0.30669292975040996, + "nauc_precision_at_1000_diff1": 0.2245564892621984, + "nauc_precision_at_1000_max": 0.7424525365701791, + "nauc_precision_at_1000_std": 0.2827575474633392, + "nauc_precision_at_100_diff1": 0.6293183940242882, + "nauc_precision_at_100_max": 0.6672307812013673, + "nauc_precision_at_100_std": 0.021241830065376183, + "nauc_precision_at_10_diff1": 0.6660374841760579, + "nauc_precision_at_10_max": 0.5524939535562643, + "nauc_precision_at_10_std": -0.2843640600894466, + "nauc_precision_at_1_diff1": 0.7775443213802683, + "nauc_precision_at_1_max": 0.4261422426923034, + "nauc_precision_at_1_std": -0.2797555774877853, + "nauc_precision_at_20_diff1": 0.6828128978864274, + "nauc_precision_at_20_max": 0.6325571895424831, + "nauc_precision_at_20_std": -0.1444009421950585, + "nauc_precision_at_3_diff1": 0.6895588157008311, + "nauc_precision_at_3_max": 0.47062565051463034, + "nauc_precision_at_3_std": -0.3507483098092554, + "nauc_precision_at_5_diff1": 0.6537804205502861, + "nauc_precision_at_5_max": 0.4817626855810538, + "nauc_precision_at_5_std": -0.35012299266685065, + "nauc_recall_at_1000_diff1": 0.22455648926279748, + "nauc_recall_at_1000_max": 0.7424525365703915, + "nauc_recall_at_1000_std": 0.28275754746343273, + "nauc_recall_at_100_diff1": 0.6293183940242689, + "nauc_recall_at_100_max": 0.6672307812013722, + "nauc_recall_at_100_std": 0.021241830065380926, + "nauc_recall_at_10_diff1": 0.6660374841760586, + "nauc_recall_at_10_max": 0.5524939535562631, + "nauc_recall_at_10_std": -0.2843640600894453, + "nauc_recall_at_1_diff1": 0.7775443213802683, + "nauc_recall_at_1_max": 0.4261422426923034, + "nauc_recall_at_1_std": -0.2797555774877853, + "nauc_recall_at_20_diff1": 0.6828128978864284, + "nauc_recall_at_20_max": 0.6325571895424784, + "nauc_recall_at_20_std": -0.14440094219505392, + "nauc_recall_at_3_diff1": 0.6895588157008296, + "nauc_recall_at_3_max": 0.4706256505146292, + "nauc_recall_at_3_std": -0.350748309809257, + "nauc_recall_at_5_diff1": 0.6537804205502865, + "nauc_recall_at_5_max": 0.48176268558105567, + "nauc_recall_at_5_std": -0.35012299266685093, + "ndcg_at_1": 0.6875, + "ndcg_at_10": 0.82082, + "ndcg_at_100": 0.83187, + "ndcg_at_1000": 0.83228, + "ndcg_at_20": 0.82778, + "ndcg_at_3": 0.78497, + "ndcg_at_5": 0.80655, + "precision_at_1": 0.6875, + "precision_at_10": 0.09485, + "precision_at_100": 0.00997, + "precision_at_1000": 0.001, + "precision_at_20": 0.04878, + "precision_at_3": 0.28417, + "precision_at_5": 0.181, + "recall_at_1": 0.6875, + "recall_at_10": 0.94847, + "recall_at_100": 0.99667, + "recall_at_1000": 0.99958, + "recall_at_20": 0.97556, + "recall_at_3": 0.8525, + "recall_at_5": 0.905 + }, + { + "cv_recall_at_1": 0.44191, + "cv_recall_at_10": 0.75375, + "cv_recall_at_100": 0.93249, + "cv_recall_at_1000": 0.99161, + "cv_recall_at_20": 0.82511, + "cv_recall_at_3": 0.60608, + "cv_recall_at_5": 0.67042, + "hf_subset": "sv", + "languages": [ + "swe-Latn" + ], + "main_score": 0.59198, + "map_at_1": 0.44191, + "map_at_10": 0.5409, + "map_at_100": 0.54873, + "map_at_1000": 0.54902, + "map_at_20": 0.54593, + "map_at_3": 0.51476, + "map_at_5": 0.52958, + "mrr_at_1": 0.44190842843393374, + "mrr_at_10": 0.5408954406273279, + "mrr_at_100": 0.5487300488511478, + "mrr_at_1000": 0.5490215246024334, + "mrr_at_20": 0.5459287610986369, + "mrr_at_3": 0.5147577799165894, + "mrr_at_5": 0.5295797240936813, + "nauc_cv_recall_at_1000_diff1": 0.6018887730673989, + "nauc_cv_recall_at_1000_max": 0.9335011023870879, + "nauc_cv_recall_at_1000_std": 0.9682392039474562, + "nauc_cv_recall_at_100_diff1": 0.5028102786422908, + "nauc_cv_recall_at_100_max": 0.7927753764073203, + "nauc_cv_recall_at_100_std": 0.7094087560669563, + "nauc_cv_recall_at_10_diff1": 0.5276450066271778, + "nauc_cv_recall_at_10_max": 0.6308089434631886, + "nauc_cv_recall_at_10_std": 0.30694843082843076, + "nauc_cv_recall_at_1_diff1": 0.6637448075588903, + "nauc_cv_recall_at_1_max": 0.5210610346172718, + "nauc_cv_recall_at_1_std": 0.10086716825348936, + "nauc_cv_recall_at_20_diff1": 0.51394371240035, + "nauc_cv_recall_at_20_max": 0.6816926536113006, + "nauc_cv_recall_at_20_std": 0.4167650335983907, + "nauc_cv_recall_at_3_diff1": 0.5634036392278551, + "nauc_cv_recall_at_3_max": 0.562466498612833, + "nauc_cv_recall_at_3_std": 0.17652617589734698, + "nauc_cv_recall_at_5_diff1": 0.5507036238602347, + "nauc_cv_recall_at_5_max": 0.590022693778581, + "nauc_cv_recall_at_5_std": 0.22280541751586838, + "nauc_map_at_1000_diff1": 0.6205079039809748, + "nauc_map_at_1000_max": 0.5475945312783034, + "nauc_map_at_1000_std": 0.14706532562026975, + "nauc_map_at_100_diff1": 0.6203751026229967, + "nauc_map_at_100_max": 0.5477794610615384, + "nauc_map_at_100_std": 0.1474929591918372, + "nauc_map_at_10_diff1": 0.6191456711326916, + "nauc_map_at_10_max": 0.547395199632512, + "nauc_map_at_10_std": 0.14573244740440383, + "nauc_map_at_1_diff1": 0.6637448075588903, + "nauc_map_at_1_max": 0.5210610346172718, + "nauc_map_at_1_std": 0.10086716825348936, + "nauc_map_at_20_diff1": 0.6196065865981142, + "nauc_map_at_20_max": 0.5481024012855527, + "nauc_map_at_20_std": 0.1477613806133426, + "nauc_map_at_3_diff1": 0.6209787798337106, + "nauc_map_at_3_max": 0.5398502671603651, + "nauc_map_at_3_std": 0.13226616364245533, + "nauc_map_at_5_diff1": 0.6200512056953228, + "nauc_map_at_5_max": 0.5447501900175638, + "nauc_map_at_5_std": 0.1398100581454376, + "nauc_mrr_at_1000_diff1": 0.6205079039809748, + "nauc_mrr_at_1000_max": 0.5475945312783034, + "nauc_mrr_at_1000_std": 0.14706532562026975, + "nauc_mrr_at_100_diff1": 0.6203751026229967, + "nauc_mrr_at_100_max": 0.5477794610615384, + "nauc_mrr_at_100_std": 0.1474929591918372, + "nauc_mrr_at_10_diff1": 0.6191456711326916, + "nauc_mrr_at_10_max": 0.547395199632512, + "nauc_mrr_at_10_std": 0.14573244740440383, + "nauc_mrr_at_1_diff1": 0.6637448075588903, + "nauc_mrr_at_1_max": 0.5210610346172718, + "nauc_mrr_at_1_std": 0.10086716825348936, + "nauc_mrr_at_20_diff1": 0.6196065865981142, + "nauc_mrr_at_20_max": 0.5481024012855527, + "nauc_mrr_at_20_std": 0.1477613806133426, + "nauc_mrr_at_3_diff1": 0.6209787798337106, + "nauc_mrr_at_3_max": 0.5398502671603651, + "nauc_mrr_at_3_std": 0.13226616364245533, + "nauc_mrr_at_5_diff1": 0.6200512056953228, + "nauc_mrr_at_5_max": 0.5447501900175638, + "nauc_mrr_at_5_std": 0.1398100581454376, + "nauc_ndcg_at_1000_diff1": 0.6123639151635346, + "nauc_ndcg_at_1000_max": 0.5579031740640773, + "nauc_ndcg_at_1000_std": 0.16845132021900988, + "nauc_ndcg_at_100_diff1": 0.6087985522272299, + "nauc_ndcg_at_100_max": 0.5638215386490731, + "nauc_ndcg_at_100_std": 0.1823456244021935, + "nauc_ndcg_at_10_diff1": 0.6018289811050774, + "nauc_ndcg_at_10_max": 0.5622477594382406, + "nauc_ndcg_at_10_std": 0.1742020463281658, + "nauc_ndcg_at_1_diff1": 0.6637448075588903, + "nauc_ndcg_at_1_max": 0.5210610346172718, + "nauc_ndcg_at_1_std": 0.10086716825348936, + "nauc_ndcg_at_20_diff1": 0.6032490714691485, + "nauc_ndcg_at_20_max": 0.5661797134702593, + "nauc_ndcg_at_20_std": 0.18382793567563085, + "nauc_ndcg_at_3_diff1": 0.6069939876443473, + "nauc_ndcg_at_3_max": 0.5453946451688797, + "nauc_ndcg_at_3_std": 0.1429789340404351, + "nauc_ndcg_at_5_diff1": 0.6045274257458881, + "nauc_ndcg_at_5_max": 0.5546680908480792, + "nauc_ndcg_at_5_std": 0.15784285604573, + "nauc_precision_at_1000_diff1": 0.6018887730673832, + "nauc_precision_at_1000_max": 0.9335011023870675, + "nauc_precision_at_1000_std": 0.9682392039474488, + "nauc_precision_at_100_diff1": 0.5028102786422958, + "nauc_precision_at_100_max": 0.7927753764073243, + "nauc_precision_at_100_std": 0.7094087560669561, + "nauc_precision_at_10_diff1": 0.5276450066271775, + "nauc_precision_at_10_max": 0.6308089434631875, + "nauc_precision_at_10_std": 0.3069484308284308, + "nauc_precision_at_1_diff1": 0.6637448075588903, + "nauc_precision_at_1_max": 0.5210610346172718, + "nauc_precision_at_1_std": 0.10086716825348936, + "nauc_precision_at_20_diff1": 0.5139437124003483, + "nauc_precision_at_20_max": 0.6816926536113007, + "nauc_precision_at_20_std": 0.4167650335983901, + "nauc_precision_at_3_diff1": 0.563403639227856, + "nauc_precision_at_3_max": 0.5624664986128333, + "nauc_precision_at_3_std": 0.17652617589734707, + "nauc_precision_at_5_diff1": 0.5507036238602353, + "nauc_precision_at_5_max": 0.5900226937785804, + "nauc_precision_at_5_std": 0.22280541751586855, + "nauc_recall_at_1000_diff1": 0.6018887730673989, + "nauc_recall_at_1000_max": 0.9335011023870879, + "nauc_recall_at_1000_std": 0.9682392039474562, + "nauc_recall_at_100_diff1": 0.5028102786422908, + "nauc_recall_at_100_max": 0.7927753764073203, + "nauc_recall_at_100_std": 0.7094087560669563, + "nauc_recall_at_10_diff1": 0.5276450066271778, + "nauc_recall_at_10_max": 0.6308089434631886, + "nauc_recall_at_10_std": 0.30694843082843076, + "nauc_recall_at_1_diff1": 0.6637448075588903, + "nauc_recall_at_1_max": 0.5210610346172718, + "nauc_recall_at_1_std": 0.10086716825348936, + "nauc_recall_at_20_diff1": 0.51394371240035, + "nauc_recall_at_20_max": 0.6816926536113006, + "nauc_recall_at_20_std": 0.4167650335983907, + "nauc_recall_at_3_diff1": 0.5634036392278551, + "nauc_recall_at_3_max": 0.562466498612833, + "nauc_recall_at_3_std": 0.17652617589734698, + "nauc_recall_at_5_diff1": 0.5507036238602347, + "nauc_recall_at_5_max": 0.590022693778581, + "nauc_recall_at_5_std": 0.22280541751586838, + "ndcg_at_1": 0.44191, + "ndcg_at_10": 0.59198, + "ndcg_at_100": 0.63016, + "ndcg_at_1000": 0.63771, + "ndcg_at_20": 0.61012, + "ndcg_at_3": 0.53823, + "ndcg_at_5": 0.56484, + "precision_at_1": 0.44191, + "precision_at_10": 0.07537, + "precision_at_100": 0.00932, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04126, + "precision_at_3": 0.20203, + "precision_at_5": 0.13408, + "recall_at_1": 0.44191, + "recall_at_10": 0.75375, + "recall_at_100": 0.93249, + "recall_at_1000": 0.99161, + "recall_at_20": 0.82511, + "recall_at_3": 0.60608, + "recall_at_5": 0.67042 + }, + { + "cv_recall_at_1": 0.03321, + "cv_recall_at_10": 0.10914, + "cv_recall_at_100": 0.27462, + "cv_recall_at_1000": 0.65995, + "cv_recall_at_20": 0.14647, + "cv_recall_at_3": 0.06117, + "cv_recall_at_5": 0.07934, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ], + "main_score": 0.06635, + "map_at_1": 0.03321, + "map_at_10": 0.05328, + "map_at_100": 0.05867, + "map_at_1000": 0.05996, + "map_at_20": 0.05579, + "map_at_3": 0.0452, + "map_at_5": 0.04938, + "mrr_at_1": 0.033210332103321034, + "mrr_at_10": 0.05327907537317126, + "mrr_at_100": 0.05867479582017056, + "mrr_at_1000": 0.059961352964132415, + "mrr_at_20": 0.05578620053772376, + "mrr_at_3": 0.045202952029520266, + "mrr_at_5": 0.04938262844166893, + "nauc_cv_recall_at_1000_diff1": 0.09433291988275253, + "nauc_cv_recall_at_1000_max": 0.3899071785651053, + "nauc_cv_recall_at_1000_std": 0.5473068856604867, + "nauc_cv_recall_at_100_diff1": 0.13672128250729124, + "nauc_cv_recall_at_100_max": 0.4333965593081663, + "nauc_cv_recall_at_100_std": 0.5290505913112106, + "nauc_cv_recall_at_10_diff1": 0.20374907353063168, + "nauc_cv_recall_at_10_max": 0.528916904832453, + "nauc_cv_recall_at_10_std": 0.5137024217892807, + "nauc_cv_recall_at_1_diff1": 0.4569530640746109, + "nauc_cv_recall_at_1_max": 0.6778468188971734, + "nauc_cv_recall_at_1_std": 0.5496011931528505, + "nauc_cv_recall_at_20_diff1": 0.18314325393687095, + "nauc_cv_recall_at_20_max": 0.5104320443662624, + "nauc_cv_recall_at_20_std": 0.5336469498387514, + "nauc_cv_recall_at_3_diff1": 0.3050165529276982, + "nauc_cv_recall_at_3_max": 0.6152380149146236, + "nauc_cv_recall_at_3_std": 0.5439195330633968, + "nauc_cv_recall_at_5_diff1": 0.26018121898972457, + "nauc_cv_recall_at_5_max": 0.5685319189687142, + "nauc_cv_recall_at_5_std": 0.5204816360035446, + "nauc_map_at_1000_diff1": 0.31171531791929985, + "nauc_map_at_1000_max": 0.5920984209133583, + "nauc_map_at_1000_std": 0.5351020291315202, + "nauc_map_at_100_diff1": 0.31233497958791867, + "nauc_map_at_100_max": 0.5928125136050744, + "nauc_map_at_100_std": 0.5347639542407587, + "nauc_map_at_10_diff1": 0.32661816730495424, + "nauc_map_at_10_max": 0.6063397910374565, + "nauc_map_at_10_std": 0.5326493130326762, + "nauc_map_at_1_diff1": 0.4569530640746109, + "nauc_map_at_1_max": 0.6778468188971734, + "nauc_map_at_1_std": 0.5496011931528505, + "nauc_map_at_20_diff1": 0.31983437901763073, + "nauc_map_at_20_max": 0.6018794532035682, + "nauc_map_at_20_std": 0.5357468910437008, + "nauc_map_at_3_diff1": 0.3703688368182306, + "nauc_map_at_3_max": 0.6405968692379383, + "nauc_map_at_3_std": 0.5451280846967744, + "nauc_map_at_5_diff1": 0.3484623067657344, + "nauc_map_at_5_max": 0.6212741811071596, + "nauc_map_at_5_std": 0.5368538853770519, + "nauc_mrr_at_1000_diff1": 0.31171531791929985, + "nauc_mrr_at_1000_max": 0.5920984209133583, + "nauc_mrr_at_1000_std": 0.5351020291315202, + "nauc_mrr_at_100_diff1": 0.31233497958791867, + "nauc_mrr_at_100_max": 0.5928125136050744, + "nauc_mrr_at_100_std": 0.5347639542407587, + "nauc_mrr_at_10_diff1": 0.32661816730495424, + "nauc_mrr_at_10_max": 0.6063397910374565, + "nauc_mrr_at_10_std": 0.5326493130326762, + "nauc_mrr_at_1_diff1": 0.4569530640746109, + "nauc_mrr_at_1_max": 0.6778468188971734, + "nauc_mrr_at_1_std": 0.5496011931528505, + "nauc_mrr_at_20_diff1": 0.31983437901763073, + "nauc_mrr_at_20_max": 0.6018794532035682, + "nauc_mrr_at_20_std": 0.5357468910437008, + "nauc_mrr_at_3_diff1": 0.3703688368182306, + "nauc_mrr_at_3_max": 0.6405968692379383, + "nauc_mrr_at_3_std": 0.5451280846967744, + "nauc_mrr_at_5_diff1": 0.3484623067657344, + "nauc_mrr_at_5_max": 0.6212741811071596, + "nauc_mrr_at_5_std": 0.5368538853770519, + "nauc_ndcg_at_1000_diff1": 0.22668553918732787, + "nauc_ndcg_at_1000_max": 0.5173234138399592, + "nauc_ndcg_at_1000_std": 0.5363578196999412, + "nauc_ndcg_at_100_diff1": 0.23407015585595642, + "nauc_ndcg_at_100_max": 0.5254226946529976, + "nauc_ndcg_at_100_std": 0.5311337513021962, + "nauc_ndcg_at_10_diff1": 0.28018176043152276, + "nauc_ndcg_at_10_max": 0.5773268253715648, + "nauc_ndcg_at_10_std": 0.5253766421907764, + "nauc_ndcg_at_1_diff1": 0.4569530640746109, + "nauc_ndcg_at_1_max": 0.6778468188971734, + "nauc_ndcg_at_1_std": 0.5496011931528505, + "nauc_ndcg_at_20_diff1": 0.26444563897191153, + "nauc_ndcg_at_20_max": 0.5655565090014336, + "nauc_ndcg_at_20_std": 0.5339288064473993, + "nauc_ndcg_at_3_diff1": 0.34967108251727236, + "nauc_ndcg_at_3_max": 0.6325070835957037, + "nauc_ndcg_at_3_std": 0.5446994265508098, + "nauc_ndcg_at_5_diff1": 0.3180407180415853, + "nauc_ndcg_at_5_max": 0.6033328171533652, + "nauc_ndcg_at_5_std": 0.531442568868304, + "nauc_precision_at_1000_diff1": 0.09433291988275161, + "nauc_precision_at_1000_max": 0.3899071785651051, + "nauc_precision_at_1000_std": 0.5473068856604861, + "nauc_precision_at_100_diff1": 0.13672128250729124, + "nauc_precision_at_100_max": 0.4333965593081666, + "nauc_precision_at_100_std": 0.5290505913112108, + "nauc_precision_at_10_diff1": 0.20374907353063154, + "nauc_precision_at_10_max": 0.5289169048324531, + "nauc_precision_at_10_std": 0.513702421789281, + "nauc_precision_at_1_diff1": 0.4569530640746109, + "nauc_precision_at_1_max": 0.6778468188971734, + "nauc_precision_at_1_std": 0.5496011931528505, + "nauc_precision_at_20_diff1": 0.18314325393687095, + "nauc_precision_at_20_max": 0.5104320443662624, + "nauc_precision_at_20_std": 0.5336469498387516, + "nauc_precision_at_3_diff1": 0.3050165529276982, + "nauc_precision_at_3_max": 0.6152380149146237, + "nauc_precision_at_3_std": 0.5439195330633968, + "nauc_precision_at_5_diff1": 0.2601812189897246, + "nauc_precision_at_5_max": 0.5685319189687144, + "nauc_precision_at_5_std": 0.5204816360035447, + "nauc_recall_at_1000_diff1": 0.09433291988275253, + "nauc_recall_at_1000_max": 0.3899071785651053, + "nauc_recall_at_1000_std": 0.5473068856604867, + "nauc_recall_at_100_diff1": 0.13672128250729124, + "nauc_recall_at_100_max": 0.4333965593081663, + "nauc_recall_at_100_std": 0.5290505913112106, + "nauc_recall_at_10_diff1": 0.20374907353063168, + "nauc_recall_at_10_max": 0.528916904832453, + "nauc_recall_at_10_std": 0.5137024217892807, + "nauc_recall_at_1_diff1": 0.4569530640746109, + "nauc_recall_at_1_max": 0.6778468188971734, + "nauc_recall_at_1_std": 0.5496011931528505, + "nauc_recall_at_20_diff1": 0.18314325393687095, + "nauc_recall_at_20_max": 0.5104320443662624, + "nauc_recall_at_20_std": 0.5336469498387514, + "nauc_recall_at_3_diff1": 0.3050165529276982, + "nauc_recall_at_3_max": 0.6152380149146236, + "nauc_recall_at_3_std": 0.5439195330633968, + "nauc_recall_at_5_diff1": 0.26018121898972457, + "nauc_recall_at_5_max": 0.5685319189687142, + "nauc_recall_at_5_std": 0.5204816360035446, + "ndcg_at_1": 0.03321, + "ndcg_at_10": 0.06635, + "ndcg_at_100": 0.09866, + "ndcg_at_1000": 0.14433, + "ndcg_at_20": 0.07569, + "ndcg_at_3": 0.04929, + "ndcg_at_5": 0.0568, + "precision_at_1": 0.03321, + "precision_at_10": 0.01091, + "precision_at_100": 0.00275, + "precision_at_1000": 0.00066, + "precision_at_20": 0.00732, + "precision_at_3": 0.02039, + "precision_at_5": 0.01587, + "recall_at_1": 0.03321, + "recall_at_10": 0.10914, + "recall_at_100": 0.27462, + "recall_at_1000": 0.65995, + "recall_at_20": 0.14647, + "recall_at_3": 0.06117, + "recall_at_5": 0.07934 + }, + { + "cv_recall_at_1": 0.0175, + "cv_recall_at_10": 0.07792, + "cv_recall_at_100": 0.27306, + "cv_recall_at_1000": 0.69653, + "cv_recall_at_20": 0.11569, + "cv_recall_at_3": 0.03625, + "cv_recall_at_5": 0.05181, + "hf_subset": "te", + "languages": [ + "tel-Telu" + ], + "main_score": 0.04301, + "map_at_1": 0.0175, + "map_at_10": 0.03245, + "map_at_100": 0.03862, + "map_at_1000": 0.0401, + "map_at_20": 0.03506, + "map_at_3": 0.02551, + "map_at_5": 0.02903, + "mrr_at_1": 0.0175, + "mrr_at_10": 0.032452270723104, + "mrr_at_100": 0.03861571397773774, + "mrr_at_1000": 0.040102337223021915, + "mrr_at_20": 0.035063106461487484, + "mrr_at_3": 0.02550925925925927, + "mrr_at_5": 0.02903009259259258, + "nauc_cv_recall_at_1000_diff1": 0.05792005006960345, + "nauc_cv_recall_at_1000_max": 0.3388544839738773, + "nauc_cv_recall_at_1000_std": 0.4881706837009283, + "nauc_cv_recall_at_100_diff1": 0.04207486335429808, + "nauc_cv_recall_at_100_max": 0.26806131714561543, + "nauc_cv_recall_at_100_std": 0.40190030075205696, + "nauc_cv_recall_at_10_diff1": 0.04582783914844086, + "nauc_cv_recall_at_10_max": 0.2544246422622029, + "nauc_cv_recall_at_10_std": 0.3615142571407665, + "nauc_cv_recall_at_1_diff1": 0.2181847607122626, + "nauc_cv_recall_at_1_max": 0.36452996367136414, + "nauc_cv_recall_at_1_std": 0.3431075676849756, + "nauc_cv_recall_at_20_diff1": 0.056574970520330975, + "nauc_cv_recall_at_20_max": 0.2636871580046122, + "nauc_cv_recall_at_20_std": 0.3689271290937749, + "nauc_cv_recall_at_3_diff1": 0.06931709889621863, + "nauc_cv_recall_at_3_max": 0.26279597881052325, + "nauc_cv_recall_at_3_std": 0.3422328724735209, + "nauc_cv_recall_at_5_diff1": 0.06066304073383788, + "nauc_cv_recall_at_5_max": 0.27006102127004533, + "nauc_cv_recall_at_5_std": 0.36088581908268846, + "nauc_map_at_1000_diff1": 0.10326350163816736, + "nauc_map_at_1000_max": 0.2928593760781644, + "nauc_map_at_1000_std": 0.36085285884092044, + "nauc_map_at_100_diff1": 0.10282362577640569, + "nauc_map_at_100_max": 0.29160155095442103, + "nauc_map_at_100_std": 0.3589087984267018, + "nauc_map_at_10_diff1": 0.10783350494599624, + "nauc_map_at_10_max": 0.29312380079082967, + "nauc_map_at_10_std": 0.35285556380874783, + "nauc_map_at_1_diff1": 0.2181847607122626, + "nauc_map_at_1_max": 0.36452996367136414, + "nauc_map_at_1_std": 0.3431075676849756, + "nauc_map_at_20_diff1": 0.10668443333534292, + "nauc_map_at_20_max": 0.29228742620962267, + "nauc_map_at_20_std": 0.3541220639866919, + "nauc_map_at_3_diff1": 0.12890605027228108, + "nauc_map_at_3_max": 0.3036582022490514, + "nauc_map_at_3_std": 0.3447869254476457, + "nauc_map_at_5_diff1": 0.1187492842098791, + "nauc_map_at_5_max": 0.3028427904848807, + "nauc_map_at_5_std": 0.35328503511968795, + "nauc_mrr_at_1000_diff1": 0.10326350163816736, + "nauc_mrr_at_1000_max": 0.2928593760781644, + "nauc_mrr_at_1000_std": 0.36085285884092044, + "nauc_mrr_at_100_diff1": 0.10282362577640569, + "nauc_mrr_at_100_max": 0.29160155095442103, + "nauc_mrr_at_100_std": 0.3589087984267018, + "nauc_mrr_at_10_diff1": 0.10783350494599624, + "nauc_mrr_at_10_max": 0.29312380079082967, + "nauc_mrr_at_10_std": 0.35285556380874783, + "nauc_mrr_at_1_diff1": 0.2181847607122626, + "nauc_mrr_at_1_max": 0.36452996367136414, + "nauc_mrr_at_1_std": 0.3431075676849756, + "nauc_mrr_at_20_diff1": 0.10668443333534292, + "nauc_mrr_at_20_max": 0.29228742620962267, + "nauc_mrr_at_20_std": 0.3541220639866919, + "nauc_mrr_at_3_diff1": 0.12890605027228108, + "nauc_mrr_at_3_max": 0.3036582022490514, + "nauc_mrr_at_3_std": 0.3447869254476457, + "nauc_mrr_at_5_diff1": 0.1187492842098791, + "nauc_mrr_at_5_max": 0.3028427904848807, + "nauc_mrr_at_5_std": 0.35328503511968795, + "nauc_ndcg_at_1000_diff1": 0.07655630184733145, + "nauc_ndcg_at_1000_max": 0.2993228122542535, + "nauc_ndcg_at_1000_std": 0.40555622921563655, + "nauc_ndcg_at_100_diff1": 0.06938793828736407, + "nauc_ndcg_at_100_max": 0.2772667362111841, + "nauc_ndcg_at_100_std": 0.37949298578727964, + "nauc_ndcg_at_10_diff1": 0.0813050607259139, + "nauc_ndcg_at_10_max": 0.2764629145165488, + "nauc_ndcg_at_10_std": 0.35630363144556443, + "nauc_ndcg_at_1_diff1": 0.2181847607122626, + "nauc_ndcg_at_1_max": 0.36452996367136414, + "nauc_ndcg_at_1_std": 0.3431075676849756, + "nauc_ndcg_at_20_diff1": 0.08195382927419637, + "nauc_ndcg_at_20_max": 0.2776948814786933, + "nauc_ndcg_at_20_std": 0.36038885937927667, + "nauc_ndcg_at_3_diff1": 0.10939001881105939, + "nauc_ndcg_at_3_max": 0.2902805163223652, + "nauc_ndcg_at_3_std": 0.344031325078653, + "nauc_ndcg_at_5_diff1": 0.09712085095822343, + "nauc_ndcg_at_5_max": 0.2905608812025033, + "nauc_ndcg_at_5_std": 0.35607370989095327, + "nauc_precision_at_1000_diff1": 0.057920050069601936, + "nauc_precision_at_1000_max": 0.33885448397387585, + "nauc_precision_at_1000_std": 0.4881706837009277, + "nauc_precision_at_100_diff1": 0.04207486335429823, + "nauc_precision_at_100_max": 0.26806131714561515, + "nauc_precision_at_100_std": 0.4019003007520569, + "nauc_precision_at_10_diff1": 0.04582783914844074, + "nauc_precision_at_10_max": 0.25442464226220274, + "nauc_precision_at_10_std": 0.36151425714076646, + "nauc_precision_at_1_diff1": 0.2181847607122626, + "nauc_precision_at_1_max": 0.36452996367136414, + "nauc_precision_at_1_std": 0.3431075676849756, + "nauc_precision_at_20_diff1": 0.05657497052033086, + "nauc_precision_at_20_max": 0.26368715800461207, + "nauc_precision_at_20_std": 0.368927129093775, + "nauc_precision_at_3_diff1": 0.06931709889621865, + "nauc_precision_at_3_max": 0.2627959788105232, + "nauc_precision_at_3_std": 0.3422328724735208, + "nauc_precision_at_5_diff1": 0.060663040733837666, + "nauc_precision_at_5_max": 0.2700610212700452, + "nauc_precision_at_5_std": 0.36088581908268846, + "nauc_recall_at_1000_diff1": 0.05792005006960345, + "nauc_recall_at_1000_max": 0.3388544839738773, + "nauc_recall_at_1000_std": 0.4881706837009283, + "nauc_recall_at_100_diff1": 0.04207486335429808, + "nauc_recall_at_100_max": 0.26806131714561543, + "nauc_recall_at_100_std": 0.40190030075205696, + "nauc_recall_at_10_diff1": 0.04582783914844086, + "nauc_recall_at_10_max": 0.2544246422622029, + "nauc_recall_at_10_std": 0.3615142571407665, + "nauc_recall_at_1_diff1": 0.2181847607122626, + "nauc_recall_at_1_max": 0.36452996367136414, + "nauc_recall_at_1_std": 0.3431075676849756, + "nauc_recall_at_20_diff1": 0.056574970520330975, + "nauc_recall_at_20_max": 0.2636871580046122, + "nauc_recall_at_20_std": 0.3689271290937749, + "nauc_recall_at_3_diff1": 0.06931709889621863, + "nauc_recall_at_3_max": 0.26279597881052325, + "nauc_recall_at_3_std": 0.3422328724735209, + "nauc_recall_at_5_diff1": 0.06066304073383788, + "nauc_recall_at_5_max": 0.27006102127004533, + "nauc_recall_at_5_std": 0.36088581908268846, + "ndcg_at_1": 0.0175, + "ndcg_at_10": 0.04301, + "ndcg_at_100": 0.08077, + "ndcg_at_1000": 0.13143, + "ndcg_at_20": 0.05255, + "ndcg_at_3": 0.02826, + "ndcg_at_5": 0.03463, + "precision_at_1": 0.0175, + "precision_at_10": 0.00779, + "precision_at_100": 0.00273, + "precision_at_1000": 0.0007, + "precision_at_20": 0.00578, + "precision_at_3": 0.01208, + "precision_at_5": 0.01036, + "recall_at_1": 0.0175, + "recall_at_10": 0.07792, + "recall_at_100": 0.27306, + "recall_at_1000": 0.69653, + "recall_at_20": 0.11569, + "recall_at_3": 0.03625, + "recall_at_5": 0.05181 + }, + { + "cv_recall_at_1": 0.40139, + "cv_recall_at_10": 0.74375, + "cv_recall_at_100": 0.94611, + "cv_recall_at_1000": 0.99708, + "cv_recall_at_20": 0.82389, + "cv_recall_at_3": 0.57778, + "cv_recall_at_5": 0.65111, + "hf_subset": "th", + "languages": [ + "tha-Thai" + ], + "main_score": 0.56517, + "map_at_1": 0.40139, + "map_at_10": 0.50891, + "map_at_100": 0.51779, + "map_at_1000": 0.51806, + "map_at_20": 0.51453, + "map_at_3": 0.4797, + "map_at_5": 0.4964, + "mrr_at_1": 0.4013888888888889, + "mrr_at_10": 0.5089053681657874, + "mrr_at_100": 0.5177867851592552, + "mrr_at_1000": 0.5180590992722832, + "mrr_at_20": 0.514527151608116, + "mrr_at_3": 0.4796990740740768, + "mrr_at_5": 0.4964004629629642, + "nauc_cv_recall_at_1000_diff1": 0.5401271619758887, + "nauc_cv_recall_at_1000_max": 0.961095549330813, + "nauc_cv_recall_at_1000_std": 0.9379974211906545, + "nauc_cv_recall_at_100_diff1": 0.448168442634786, + "nauc_cv_recall_at_100_max": 0.7295438794074325, + "nauc_cv_recall_at_100_std": 0.6025585491928738, + "nauc_cv_recall_at_10_diff1": 0.47379036107157435, + "nauc_cv_recall_at_10_max": 0.5529269123201439, + "nauc_cv_recall_at_10_std": 0.24143568087558764, + "nauc_cv_recall_at_1_diff1": 0.6164133423250504, + "nauc_cv_recall_at_1_max": 0.43162943869145476, + "nauc_cv_recall_at_1_std": 0.05381972083937815, + "nauc_cv_recall_at_20_diff1": 0.4589103478632773, + "nauc_cv_recall_at_20_max": 0.5918795480867991, + "nauc_cv_recall_at_20_std": 0.3251142582423592, + "nauc_cv_recall_at_3_diff1": 0.5199180551981475, + "nauc_cv_recall_at_3_max": 0.48830316360010284, + "nauc_cv_recall_at_3_std": 0.12179900604734932, + "nauc_cv_recall_at_5_diff1": 0.4933589969881374, + "nauc_cv_recall_at_5_max": 0.5115624626235463, + "nauc_cv_recall_at_5_std": 0.17317824898789116, + "nauc_map_at_1000_diff1": 0.5728540706664647, + "nauc_map_at_1000_max": 0.4592079702642643, + "nauc_map_at_1000_std": 0.09332696144631226, + "nauc_map_at_100_diff1": 0.5727418480818869, + "nauc_map_at_100_max": 0.45939586065791144, + "nauc_map_at_100_std": 0.09367322367533219, + "nauc_map_at_10_diff1": 0.571122576444961, + "nauc_map_at_10_max": 0.4597304926570427, + "nauc_map_at_10_std": 0.09303375964132092, + "nauc_map_at_1_diff1": 0.6164133423250504, + "nauc_map_at_1_max": 0.43162943869145476, + "nauc_map_at_1_std": 0.05381972083937815, + "nauc_map_at_20_diff1": 0.5718838115024688, + "nauc_map_at_20_max": 0.4598623581030198, + "nauc_map_at_20_std": 0.09418505747711978, + "nauc_map_at_3_diff1": 0.5753151106225249, + "nauc_map_at_3_max": 0.45362185426138985, + "nauc_map_at_3_std": 0.07996167733667833, + "nauc_map_at_5_diff1": 0.5714216051987776, + "nauc_map_at_5_max": 0.4574917315602048, + "nauc_map_at_5_std": 0.08896721117497133, + "nauc_mrr_at_1000_diff1": 0.5728540706664647, + "nauc_mrr_at_1000_max": 0.4592079702642643, + "nauc_mrr_at_1000_std": 0.09332696144631226, + "nauc_mrr_at_100_diff1": 0.5727418480818869, + "nauc_mrr_at_100_max": 0.45939586065791144, + "nauc_mrr_at_100_std": 0.09367322367533219, + "nauc_mrr_at_10_diff1": 0.571122576444961, + "nauc_mrr_at_10_max": 0.4597304926570427, + "nauc_mrr_at_10_std": 0.09303375964132092, + "nauc_mrr_at_1_diff1": 0.6164133423250504, + "nauc_mrr_at_1_max": 0.43162943869145476, + "nauc_mrr_at_1_std": 0.05381972083937815, + "nauc_mrr_at_20_diff1": 0.5718838115024688, + "nauc_mrr_at_20_max": 0.4598623581030198, + "nauc_mrr_at_20_std": 0.09418505747711978, + "nauc_mrr_at_3_diff1": 0.5753151106225249, + "nauc_mrr_at_3_max": 0.45362185426138985, + "nauc_mrr_at_3_std": 0.07996167733667833, + "nauc_mrr_at_5_diff1": 0.5714216051987776, + "nauc_mrr_at_5_max": 0.4574917315602048, + "nauc_mrr_at_5_std": 0.08896721117497133, + "nauc_ndcg_at_1000_diff1": 0.5644879483816287, + "nauc_ndcg_at_1000_max": 0.46864246103483886, + "nauc_ndcg_at_1000_std": 0.10961745828253958, + "nauc_ndcg_at_100_diff1": 0.5614769460541409, + "nauc_ndcg_at_100_max": 0.47454605148821766, + "nauc_ndcg_at_100_std": 0.12083764507191204, + "nauc_ndcg_at_10_diff1": 0.552642154001218, + "nauc_ndcg_at_10_max": 0.4764490613074466, + "nauc_ndcg_at_10_std": 0.11939195000827524, + "nauc_ndcg_at_1_diff1": 0.6164133423250504, + "nauc_ndcg_at_1_max": 0.43162943869145476, + "nauc_ndcg_at_1_std": 0.05381972083937815, + "nauc_ndcg_at_20_diff1": 0.5547211132622595, + "nauc_ndcg_at_20_max": 0.47794734067506456, + "nauc_ndcg_at_20_std": 0.1251463269383499, + "nauc_ndcg_at_3_diff1": 0.5617637090081964, + "nauc_ndcg_at_3_max": 0.46201121733709233, + "nauc_ndcg_at_3_std": 0.09007225455387947, + "nauc_ndcg_at_5_diff1": 0.5539623110541856, + "nauc_ndcg_at_5_max": 0.46943015360238455, + "nauc_ndcg_at_5_std": 0.10731330075560759, + "nauc_precision_at_1000_diff1": 0.5401271619758738, + "nauc_precision_at_1000_max": 0.961095549330848, + "nauc_precision_at_1000_std": 0.9379974211907273, + "nauc_precision_at_100_diff1": 0.4481684426347872, + "nauc_precision_at_100_max": 0.7295438794074368, + "nauc_precision_at_100_std": 0.6025585491928742, + "nauc_precision_at_10_diff1": 0.4737903610715738, + "nauc_precision_at_10_max": 0.5529269123201438, + "nauc_precision_at_10_std": 0.24143568087558764, + "nauc_precision_at_1_diff1": 0.6164133423250504, + "nauc_precision_at_1_max": 0.43162943869145476, + "nauc_precision_at_1_std": 0.05381972083937815, + "nauc_precision_at_20_diff1": 0.4589103478632771, + "nauc_precision_at_20_max": 0.5918795480867984, + "nauc_precision_at_20_std": 0.3251142582423566, + "nauc_precision_at_3_diff1": 0.5199180551981472, + "nauc_precision_at_3_max": 0.4883031636001032, + "nauc_precision_at_3_std": 0.12179900604734974, + "nauc_precision_at_5_diff1": 0.49335899698813696, + "nauc_precision_at_5_max": 0.5115624626235458, + "nauc_precision_at_5_std": 0.17317824898789044, + "nauc_recall_at_1000_diff1": 0.5401271619758887, + "nauc_recall_at_1000_max": 0.961095549330813, + "nauc_recall_at_1000_std": 0.9379974211906545, + "nauc_recall_at_100_diff1": 0.448168442634786, + "nauc_recall_at_100_max": 0.7295438794074325, + "nauc_recall_at_100_std": 0.6025585491928738, + "nauc_recall_at_10_diff1": 0.47379036107157435, + "nauc_recall_at_10_max": 0.5529269123201439, + "nauc_recall_at_10_std": 0.24143568087558764, + "nauc_recall_at_1_diff1": 0.6164133423250504, + "nauc_recall_at_1_max": 0.43162943869145476, + "nauc_recall_at_1_std": 0.05381972083937815, + "nauc_recall_at_20_diff1": 0.4589103478632773, + "nauc_recall_at_20_max": 0.5918795480867991, + "nauc_recall_at_20_std": 0.3251142582423592, + "nauc_recall_at_3_diff1": 0.5199180551981475, + "nauc_recall_at_3_max": 0.48830316360010284, + "nauc_recall_at_3_std": 0.12179900604734932, + "nauc_recall_at_5_diff1": 0.4933589969881374, + "nauc_recall_at_5_max": 0.5115624626235463, + "nauc_recall_at_5_std": 0.17317824898789116, + "ndcg_at_1": 0.40139, + "ndcg_at_10": 0.56517, + "ndcg_at_100": 0.60842, + "ndcg_at_1000": 0.61505, + "ndcg_at_20": 0.5855, + "ndcg_at_3": 0.50491, + "ndcg_at_5": 0.53507, + "precision_at_1": 0.40139, + "precision_at_10": 0.07438, + "precision_at_100": 0.00946, + "precision_at_1000": 0.001, + "precision_at_20": 0.04119, + "precision_at_3": 0.19259, + "precision_at_5": 0.13022, + "recall_at_1": 0.40139, + "recall_at_10": 0.74375, + "recall_at_100": 0.94611, + "recall_at_1000": 0.99708, + "recall_at_20": 0.82389, + "recall_at_3": 0.57778, + "recall_at_5": 0.65111 + }, + { + "cv_recall_at_1": 0.24679, + "cv_recall_at_10": 0.55095, + "cv_recall_at_100": 0.8399, + "cv_recall_at_1000": 0.98092, + "cv_recall_at_20": 0.65464, + "cv_recall_at_3": 0.38366, + "cv_recall_at_5": 0.45279, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.38614, + "map_at_1": 0.24679, + "map_at_10": 0.33501, + "map_at_100": 0.34688, + "map_at_1000": 0.34754, + "map_at_20": 0.3422, + "map_at_3": 0.3064, + "map_at_5": 0.3221, + "mrr_at_1": 0.24678556615512234, + "mrr_at_10": 0.335005617989856, + "mrr_at_100": 0.3468761477353053, + "mrr_at_1000": 0.34754482431159706, + "mrr_at_20": 0.3422046159949083, + "mrr_at_3": 0.30639660813862274, + "mrr_at_5": 0.32210240103230525, + "nauc_cv_recall_at_1000_diff1": 0.28818656533654663, + "nauc_cv_recall_at_1000_max": 0.7881388928706085, + "nauc_cv_recall_at_1000_std": 0.8586308516367452, + "nauc_cv_recall_at_100_diff1": 0.3077133849819943, + "nauc_cv_recall_at_100_max": 0.6228904388563352, + "nauc_cv_recall_at_100_std": 0.5832133669363783, + "nauc_cv_recall_at_10_diff1": 0.3333065129762372, + "nauc_cv_recall_at_10_max": 0.4456554564461219, + "nauc_cv_recall_at_10_std": 0.2561096026594872, + "nauc_cv_recall_at_1_diff1": 0.519216885422269, + "nauc_cv_recall_at_1_max": 0.41701938956923496, + "nauc_cv_recall_at_1_std": 0.12515186524794, + "nauc_cv_recall_at_20_diff1": 0.33771403630592267, + "nauc_cv_recall_at_20_max": 0.4986739517512237, + "nauc_cv_recall_at_20_std": 0.34355706815645887, + "nauc_cv_recall_at_3_diff1": 0.3983883376536369, + "nauc_cv_recall_at_3_max": 0.42284870112868844, + "nauc_cv_recall_at_3_std": 0.16867702576877472, + "nauc_cv_recall_at_5_diff1": 0.37441532248421466, + "nauc_cv_recall_at_5_max": 0.43958306723412155, + "nauc_cv_recall_at_5_std": 0.20772423785543728, + "nauc_map_at_1000_diff1": 0.4525338584708267, + "nauc_map_at_1000_max": 0.4293850432847313, + "nauc_map_at_1000_std": 0.166782289898531, + "nauc_map_at_100_diff1": 0.45224768945865956, + "nauc_map_at_100_max": 0.42965326556217975, + "nauc_map_at_100_std": 0.16737932916906623, + "nauc_map_at_10_diff1": 0.4504471657225726, + "nauc_map_at_10_max": 0.4255150942788997, + "nauc_map_at_10_std": 0.16125390697135564, + "nauc_map_at_1_diff1": 0.519216885422269, + "nauc_map_at_1_max": 0.41701938956923496, + "nauc_map_at_1_std": 0.12515186524794, + "nauc_map_at_20_diff1": 0.45163270802450295, + "nauc_map_at_20_max": 0.4285784046915898, + "nauc_map_at_20_std": 0.165738020819407, + "nauc_map_at_3_diff1": 0.4608529827096026, + "nauc_map_at_3_max": 0.4206307446552375, + "nauc_map_at_3_std": 0.14550345666637035, + "nauc_map_at_5_diff1": 0.45511022081691893, + "nauc_map_at_5_max": 0.42481744042818403, + "nauc_map_at_5_std": 0.15509451723166393, + "nauc_mrr_at_1000_diff1": 0.4525338584708267, + "nauc_mrr_at_1000_max": 0.4293850432847313, + "nauc_mrr_at_1000_std": 0.166782289898531, + "nauc_mrr_at_100_diff1": 0.45224768945865956, + "nauc_mrr_at_100_max": 0.42965326556217975, + "nauc_mrr_at_100_std": 0.16737932916906623, + "nauc_mrr_at_10_diff1": 0.4504471657225726, + "nauc_mrr_at_10_max": 0.4255150942788997, + "nauc_mrr_at_10_std": 0.16125390697135564, + "nauc_mrr_at_1_diff1": 0.519216885422269, + "nauc_mrr_at_1_max": 0.41701938956923496, + "nauc_mrr_at_1_std": 0.12515186524794, + "nauc_mrr_at_20_diff1": 0.45163270802450295, + "nauc_mrr_at_20_max": 0.4285784046915898, + "nauc_mrr_at_20_std": 0.165738020819407, + "nauc_mrr_at_3_diff1": 0.4608529827096026, + "nauc_mrr_at_3_max": 0.4206307446552375, + "nauc_mrr_at_3_std": 0.14550345666637035, + "nauc_mrr_at_5_diff1": 0.45511022081691893, + "nauc_mrr_at_5_max": 0.42481744042818403, + "nauc_mrr_at_5_std": 0.15509451723166393, + "nauc_ndcg_at_1000_diff1": 0.4373081282043892, + "nauc_ndcg_at_1000_max": 0.4409003063361399, + "nauc_ndcg_at_1000_std": 0.19381797221676839, + "nauc_ndcg_at_100_diff1": 0.429992205334297, + "nauc_ndcg_at_100_max": 0.44949420342200636, + "nauc_ndcg_at_100_std": 0.21286903391313913, + "nauc_ndcg_at_10_diff1": 0.42196328661774485, + "nauc_ndcg_at_10_max": 0.4303021596534981, + "nauc_ndcg_at_10_std": 0.18350106704037714, + "nauc_ndcg_at_1_diff1": 0.519216885422269, + "nauc_ndcg_at_1_max": 0.41701938956923496, + "nauc_ndcg_at_1_std": 0.12515186524794, + "nauc_ndcg_at_20_diff1": 0.4259642730438616, + "nauc_ndcg_at_20_max": 0.44168870941829563, + "nauc_ndcg_at_20_std": 0.2005691635345835, + "nauc_ndcg_at_3_diff1": 0.4439390916850469, + "nauc_ndcg_at_3_max": 0.4212649609693536, + "nauc_ndcg_at_3_std": 0.1517498527473349, + "nauc_ndcg_at_5_diff1": 0.43375968280368277, + "nauc_ndcg_at_5_max": 0.428572455649159, + "nauc_ndcg_at_5_std": 0.16862594213907914, + "nauc_precision_at_1000_diff1": 0.28818656533653536, + "nauc_precision_at_1000_max": 0.788138892870599, + "nauc_precision_at_1000_std": 0.8586308516367483, + "nauc_precision_at_100_diff1": 0.30771338498199347, + "nauc_precision_at_100_max": 0.6228904388563357, + "nauc_precision_at_100_std": 0.5832133669363774, + "nauc_precision_at_10_diff1": 0.33330651297623737, + "nauc_precision_at_10_max": 0.4456554564461219, + "nauc_precision_at_10_std": 0.2561096026594872, + "nauc_precision_at_1_diff1": 0.519216885422269, + "nauc_precision_at_1_max": 0.41701938956923496, + "nauc_precision_at_1_std": 0.12515186524794, + "nauc_precision_at_20_diff1": 0.3377140363059223, + "nauc_precision_at_20_max": 0.49867395175122375, + "nauc_precision_at_20_std": 0.3435570681564585, + "nauc_precision_at_3_diff1": 0.3983883376536368, + "nauc_precision_at_3_max": 0.4228487011286886, + "nauc_precision_at_3_std": 0.16867702576877458, + "nauc_precision_at_5_diff1": 0.37441532248421455, + "nauc_precision_at_5_max": 0.43958306723412105, + "nauc_precision_at_5_std": 0.20772423785543706, + "nauc_recall_at_1000_diff1": 0.28818656533654663, + "nauc_recall_at_1000_max": 0.7881388928706085, + "nauc_recall_at_1000_std": 0.8586308516367452, + "nauc_recall_at_100_diff1": 0.3077133849819943, + "nauc_recall_at_100_max": 0.6228904388563352, + "nauc_recall_at_100_std": 0.5832133669363783, + "nauc_recall_at_10_diff1": 0.3333065129762372, + "nauc_recall_at_10_max": 0.4456554564461219, + "nauc_recall_at_10_std": 0.2561096026594872, + "nauc_recall_at_1_diff1": 0.519216885422269, + "nauc_recall_at_1_max": 0.41701938956923496, + "nauc_recall_at_1_std": 0.12515186524794, + "nauc_recall_at_20_diff1": 0.33771403630592267, + "nauc_recall_at_20_max": 0.4986739517512237, + "nauc_recall_at_20_std": 0.34355706815645887, + "nauc_recall_at_3_diff1": 0.3983883376536369, + "nauc_recall_at_3_max": 0.42284870112868844, + "nauc_recall_at_3_std": 0.16867702576877472, + "nauc_recall_at_5_diff1": 0.37441532248421466, + "nauc_recall_at_5_max": 0.43958306723412155, + "nauc_recall_at_5_std": 0.20772423785543728, + "ndcg_at_1": 0.24679, + "ndcg_at_10": 0.38614, + "ndcg_at_100": 0.44657, + "ndcg_at_1000": 0.46446, + "ndcg_at_20": 0.41236, + "ndcg_at_3": 0.32621, + "ndcg_at_5": 0.3546, + "precision_at_1": 0.24679, + "precision_at_10": 0.05509, + "precision_at_100": 0.0084, + "precision_at_1000": 0.00098, + "precision_at_20": 0.03273, + "precision_at_3": 0.12789, + "precision_at_5": 0.09056, + "recall_at_1": 0.24679, + "recall_at_10": 0.55095, + "recall_at_100": 0.8399, + "recall_at_1000": 0.98092, + "recall_at_20": 0.65464, + "recall_at_3": 0.38366, + "recall_at_5": 0.45279 + }, + { + "cv_recall_at_1": 0.47997, + "cv_recall_at_10": 0.81816, + "cv_recall_at_100": 0.96798, + "cv_recall_at_1000": 0.99861, + "cv_recall_at_20": 0.87845, + "cv_recall_at_3": 0.66778, + "cv_recall_at_5": 0.73818, + "hf_subset": "uk", + "languages": [ + "ukr-Cyrl" + ], + "main_score": 0.64496, + "map_at_1": 0.47997, + "map_at_10": 0.58998, + "map_at_100": 0.59669, + "map_at_1000": 0.59685, + "map_at_20": 0.5942, + "map_at_3": 0.56322, + "map_at_5": 0.57925, + "mrr_at_1": 0.47997227997228, + "mrr_at_10": 0.5899797599797617, + "mrr_at_100": 0.5966925968730221, + "mrr_at_1000": 0.5968492732003331, + "mrr_at_20": 0.5942022639319625, + "mrr_at_3": 0.563224763224767, + "mrr_at_5": 0.5792469392469406, + "nauc_cv_recall_at_1000_diff1": 0.359619386307682, + "nauc_cv_recall_at_1000_max": 0.8289359591762638, + "nauc_cv_recall_at_1000_std": 0.8567147435685029, + "nauc_cv_recall_at_100_diff1": 0.46590847395195073, + "nauc_cv_recall_at_100_max": 0.6662723523534553, + "nauc_cv_recall_at_100_std": 0.5424559953823549, + "nauc_cv_recall_at_10_diff1": 0.48640162038929124, + "nauc_cv_recall_at_10_max": 0.5388902567020947, + "nauc_cv_recall_at_10_std": 0.14897840060568757, + "nauc_cv_recall_at_1_diff1": 0.6461339577495517, + "nauc_cv_recall_at_1_max": 0.4259055740159931, + "nauc_cv_recall_at_1_std": -0.033787027088644075, + "nauc_cv_recall_at_20_diff1": 0.47921561047789607, + "nauc_cv_recall_at_20_max": 0.5846110809233728, + "nauc_cv_recall_at_20_std": 0.24702918047480954, + "nauc_cv_recall_at_3_diff1": 0.533961427938337, + "nauc_cv_recall_at_3_max": 0.4791315812220909, + "nauc_cv_recall_at_3_std": 0.01770929535518152, + "nauc_cv_recall_at_5_diff1": 0.5062471209011927, + "nauc_cv_recall_at_5_max": 0.5035739906001356, + "nauc_cv_recall_at_5_std": 0.062482575188251704, + "nauc_map_at_1000_diff1": 0.5998859220426689, + "nauc_map_at_1000_max": 0.44981533649256034, + "nauc_map_at_1000_std": -0.0034501414561704364, + "nauc_map_at_100_diff1": 0.5998140517633854, + "nauc_map_at_100_max": 0.4498957671854623, + "nauc_map_at_100_std": -0.0032166715611345533, + "nauc_map_at_10_diff1": 0.5977066123970523, + "nauc_map_at_10_max": 0.45028403262092415, + "nauc_map_at_10_std": -0.0035025904796239958, + "nauc_map_at_1_diff1": 0.6461339577495517, + "nauc_map_at_1_max": 0.4259055740159931, + "nauc_map_at_1_std": -0.033787027088644075, + "nauc_map_at_20_diff1": 0.5989064282726737, + "nauc_map_at_20_max": 0.45060898042681835, + "nauc_map_at_20_std": -0.0024880705803015054, + "nauc_map_at_3_diff1": 0.5997550415090089, + "nauc_map_at_3_max": 0.44650429192658436, + "nauc_map_at_3_std": -0.01452171041883708, + "nauc_map_at_5_diff1": 0.5970255575961725, + "nauc_map_at_5_max": 0.4491223043095994, + "nauc_map_at_5_std": -0.008338990928095574, + "nauc_mrr_at_1000_diff1": 0.5998859220426689, + "nauc_mrr_at_1000_max": 0.44981533649256034, + "nauc_mrr_at_1000_std": -0.0034501414561704364, + "nauc_mrr_at_100_diff1": 0.5998140517633854, + "nauc_mrr_at_100_max": 0.4498957671854623, + "nauc_mrr_at_100_std": -0.0032166715611345533, + "nauc_mrr_at_10_diff1": 0.5977066123970523, + "nauc_mrr_at_10_max": 0.45028403262092415, + "nauc_mrr_at_10_std": -0.0035025904796239958, + "nauc_mrr_at_1_diff1": 0.6461339577495517, + "nauc_mrr_at_1_max": 0.4259055740159931, + "nauc_mrr_at_1_std": -0.033787027088644075, + "nauc_mrr_at_20_diff1": 0.5989064282726737, + "nauc_mrr_at_20_max": 0.45060898042681835, + "nauc_mrr_at_20_std": -0.0024880705803015054, + "nauc_mrr_at_3_diff1": 0.5997550415090089, + "nauc_mrr_at_3_max": 0.44650429192658436, + "nauc_mrr_at_3_std": -0.01452171041883708, + "nauc_mrr_at_5_diff1": 0.5970255575961725, + "nauc_mrr_at_5_max": 0.4491223043095994, + "nauc_mrr_at_5_std": -0.008338990928095574, + "nauc_ndcg_at_1000_diff1": 0.5920290475390783, + "nauc_ndcg_at_1000_max": 0.4568058683689263, + "nauc_ndcg_at_1000_std": 0.009398550737737928, + "nauc_ndcg_at_100_diff1": 0.5900339801932035, + "nauc_ndcg_at_100_max": 0.45991286069602905, + "nauc_ndcg_at_100_std": 0.01755545805425444, + "nauc_ndcg_at_10_diff1": 0.5793414880989026, + "nauc_ndcg_at_10_max": 0.46396882476782636, + "nauc_ndcg_at_10_std": 0.019337122970605566, + "nauc_ndcg_at_1_diff1": 0.6461339577495517, + "nauc_ndcg_at_1_max": 0.4259055740159931, + "nauc_ndcg_at_1_std": -0.033787027088644075, + "nauc_ndcg_at_20_diff1": 0.5833380066387772, + "nauc_ndcg_at_20_max": 0.4656530069396211, + "nauc_ndcg_at_20_std": 0.02432957786471851, + "nauc_ndcg_at_3_diff1": 0.5844841161839118, + "nauc_ndcg_at_3_max": 0.45401230876657694, + "nauc_ndcg_at_3_std": -0.007134162372538832, + "nauc_ndcg_at_5_diff1": 0.5784652389857332, + "nauc_ndcg_at_5_max": 0.4599763929238898, + "nauc_ndcg_at_5_std": 0.005591487123884887, + "nauc_precision_at_1000_diff1": 0.3596193863074833, + "nauc_precision_at_1000_max": 0.8289359591760609, + "nauc_precision_at_1000_std": 0.8567147435683159, + "nauc_precision_at_100_diff1": 0.465908473951957, + "nauc_precision_at_100_max": 0.6662723523534598, + "nauc_precision_at_100_std": 0.5424559953823621, + "nauc_precision_at_10_diff1": 0.4864016203892922, + "nauc_precision_at_10_max": 0.5388902567020952, + "nauc_precision_at_10_std": 0.14897840060568837, + "nauc_precision_at_1_diff1": 0.6461339577495517, + "nauc_precision_at_1_max": 0.4259055740159931, + "nauc_precision_at_1_std": -0.033787027088644075, + "nauc_precision_at_20_diff1": 0.47921561047789474, + "nauc_precision_at_20_max": 0.584611080923373, + "nauc_precision_at_20_std": 0.24702918047481107, + "nauc_precision_at_3_diff1": 0.5339614279383376, + "nauc_precision_at_3_max": 0.479131581222091, + "nauc_precision_at_3_std": 0.017709295355182442, + "nauc_precision_at_5_diff1": 0.506247120901192, + "nauc_precision_at_5_max": 0.5035739906001366, + "nauc_precision_at_5_std": 0.06248257518825092, + "nauc_recall_at_1000_diff1": 0.359619386307682, + "nauc_recall_at_1000_max": 0.8289359591762638, + "nauc_recall_at_1000_std": 0.8567147435685029, + "nauc_recall_at_100_diff1": 0.46590847395195073, + "nauc_recall_at_100_max": 0.6662723523534553, + "nauc_recall_at_100_std": 0.5424559953823549, + "nauc_recall_at_10_diff1": 0.48640162038929124, + "nauc_recall_at_10_max": 0.5388902567020947, + "nauc_recall_at_10_std": 0.14897840060568757, + "nauc_recall_at_1_diff1": 0.6461339577495517, + "nauc_recall_at_1_max": 0.4259055740159931, + "nauc_recall_at_1_std": -0.033787027088644075, + "nauc_recall_at_20_diff1": 0.47921561047789607, + "nauc_recall_at_20_max": 0.5846110809233728, + "nauc_recall_at_20_std": 0.24702918047480954, + "nauc_recall_at_3_diff1": 0.533961427938337, + "nauc_recall_at_3_max": 0.4791315812220909, + "nauc_recall_at_3_std": 0.01770929535518152, + "nauc_recall_at_5_diff1": 0.5062471209011927, + "nauc_recall_at_5_max": 0.5035739906001356, + "nauc_recall_at_5_std": 0.062482575188251704, + "ndcg_at_1": 0.47997, + "ndcg_at_10": 0.64496, + "ndcg_at_100": 0.67724, + "ndcg_at_1000": 0.68119, + "ndcg_at_20": 0.66025, + "ndcg_at_3": 0.5901, + "ndcg_at_5": 0.61904, + "precision_at_1": 0.47997, + "precision_at_10": 0.08182, + "precision_at_100": 0.00968, + "precision_at_1000": 0.001, + "precision_at_20": 0.04392, + "precision_at_3": 0.22259, + "precision_at_5": 0.14764, + "recall_at_1": 0.47997, + "recall_at_10": 0.81816, + "recall_at_100": 0.96798, + "recall_at_1000": 0.99861, + "recall_at_20": 0.87845, + "recall_at_3": 0.66778, + "recall_at_5": 0.73818 + }, + { + "cv_recall_at_1": 0.56735, + "cv_recall_at_10": 0.88136, + "cv_recall_at_100": 0.98449, + "cv_recall_at_1000": 0.99973, + "cv_recall_at_20": 0.93048, + "cv_recall_at_3": 0.75116, + "cv_recall_at_5": 0.81429, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ], + "main_score": 0.7231, + "map_at_1": 0.56735, + "map_at_10": 0.67252, + "map_at_100": 0.67759, + "map_at_1000": 0.67768, + "map_at_20": 0.67607, + "map_at_3": 0.64896, + "map_at_5": 0.66347, + "mrr_at_1": 0.5673469387755102, + "mrr_at_10": 0.6725237555339588, + "mrr_at_100": 0.6775858165821529, + "mrr_at_1000": 0.67767650718611, + "mrr_at_20": 0.676070937221595, + "mrr_at_3": 0.6489569160997744, + "mrr_at_5": 0.663467120181404, + "nauc_cv_recall_at_1000_diff1": 0.9346405228757856, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.6620923223090417, + "nauc_cv_recall_at_100_max": 0.8255770144315029, + "nauc_cv_recall_at_100_std": 0.4641382868937102, + "nauc_cv_recall_at_10_diff1": 0.6058370101844033, + "nauc_cv_recall_at_10_max": 0.6163717897701185, + "nauc_cv_recall_at_10_std": 0.041965843547617905, + "nauc_cv_recall_at_1_diff1": 0.7259662689257877, + "nauc_cv_recall_at_1_max": 0.43676348337684495, + "nauc_cv_recall_at_1_std": -0.1449503651311885, + "nauc_cv_recall_at_20_diff1": 0.5987335573498791, + "nauc_cv_recall_at_20_max": 0.6816936820390254, + "nauc_cv_recall_at_20_std": 0.1540250803517749, + "nauc_cv_recall_at_3_diff1": 0.6377585424746157, + "nauc_cv_recall_at_3_max": 0.5040415194409423, + "nauc_cv_recall_at_3_std": -0.11346754850123454, + "nauc_cv_recall_at_5_diff1": 0.6239492907917592, + "nauc_cv_recall_at_5_max": 0.5487095212616887, + "nauc_cv_recall_at_5_std": -0.052386412028695956, + "nauc_map_at_1000_diff1": 0.6928319955996259, + "nauc_map_at_1000_max": 0.4662204138391279, + "nauc_map_at_1000_std": -0.12257467776910304, + "nauc_map_at_100_diff1": 0.6928192827730435, + "nauc_map_at_100_max": 0.4663371019670773, + "nauc_map_at_100_std": -0.12240526964653446, + "nauc_map_at_10_diff1": 0.6911981783560863, + "nauc_map_at_10_max": 0.46793659396050186, + "nauc_map_at_10_std": -0.12145300386378471, + "nauc_map_at_1_diff1": 0.7259662689257877, + "nauc_map_at_1_max": 0.43676348337684495, + "nauc_map_at_1_std": -0.1449503651311885, + "nauc_map_at_20_diff1": 0.6921305592473023, + "nauc_map_at_20_max": 0.46723871180782983, + "nauc_map_at_20_std": -0.12153973319496549, + "nauc_map_at_3_diff1": 0.6905332223021816, + "nauc_map_at_3_max": 0.46149727185955913, + "nauc_map_at_3_std": -0.13361290779652502, + "nauc_map_at_5_diff1": 0.6904150526689351, + "nauc_map_at_5_max": 0.46635245248076457, + "nauc_map_at_5_std": -0.12489992753838132, + "nauc_mrr_at_1000_diff1": 0.6928319955996259, + "nauc_mrr_at_1000_max": 0.4662204138391279, + "nauc_mrr_at_1000_std": -0.12257467776910304, + "nauc_mrr_at_100_diff1": 0.6928192827730435, + "nauc_mrr_at_100_max": 0.4663371019670773, + "nauc_mrr_at_100_std": -0.12240526964653446, + "nauc_mrr_at_10_diff1": 0.6911981783560863, + "nauc_mrr_at_10_max": 0.46793659396050186, + "nauc_mrr_at_10_std": -0.12145300386378471, + "nauc_mrr_at_1_diff1": 0.7259662689257877, + "nauc_mrr_at_1_max": 0.43676348337684495, + "nauc_mrr_at_1_std": -0.1449503651311885, + "nauc_mrr_at_20_diff1": 0.6921305592473023, + "nauc_mrr_at_20_max": 0.46723871180782983, + "nauc_mrr_at_20_std": -0.12153973319496549, + "nauc_mrr_at_3_diff1": 0.6905332223021816, + "nauc_mrr_at_3_max": 0.46149727185955913, + "nauc_mrr_at_3_std": -0.13361290779652502, + "nauc_mrr_at_5_diff1": 0.6904150526689351, + "nauc_mrr_at_5_max": 0.46635245248076457, + "nauc_mrr_at_5_std": -0.12489992753838132, + "nauc_ndcg_at_1000_diff1": 0.6883667434948514, + "nauc_ndcg_at_1000_max": 0.4738703537940675, + "nauc_ndcg_at_1000_std": -0.1136077195297271, + "nauc_ndcg_at_100_diff1": 0.6880647036947598, + "nauc_ndcg_at_100_max": 0.4772551525584301, + "nauc_ndcg_at_100_std": -0.10821757655795085, + "nauc_ndcg_at_10_diff1": 0.6794314751585946, + "nauc_ndcg_at_10_max": 0.4866239032204006, + "nauc_ndcg_at_10_std": -0.10157943275930026, + "nauc_ndcg_at_1_diff1": 0.7259662689257877, + "nauc_ndcg_at_1_max": 0.43676348337684495, + "nauc_ndcg_at_1_std": -0.1449503651311885, + "nauc_ndcg_at_20_diff1": 0.6828716896544922, + "nauc_ndcg_at_20_max": 0.48476789923178737, + "nauc_ndcg_at_20_std": -0.10049463526827918, + "nauc_ndcg_at_3_diff1": 0.6790131486012451, + "nauc_ndcg_at_3_max": 0.4706728176618378, + "nauc_ndcg_at_3_std": -0.12927902751301362, + "nauc_ndcg_at_5_diff1": 0.6780737481870208, + "nauc_ndcg_at_5_max": 0.48101881603227314, + "nauc_ndcg_at_5_std": -0.11223564267542065, + "nauc_precision_at_1000_diff1": 0.9346405228756913, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.6620923223090409, + "nauc_precision_at_100_max": 0.8255770144315049, + "nauc_precision_at_100_std": 0.4641382868936997, + "nauc_precision_at_10_diff1": 0.6058370101844027, + "nauc_precision_at_10_max": 0.6163717897701152, + "nauc_precision_at_10_std": 0.04196584354761412, + "nauc_precision_at_1_diff1": 0.7259662689257877, + "nauc_precision_at_1_max": 0.43676348337684495, + "nauc_precision_at_1_std": -0.1449503651311885, + "nauc_precision_at_20_diff1": 0.5987335573498817, + "nauc_precision_at_20_max": 0.6816936820390233, + "nauc_precision_at_20_std": 0.15402508035177356, + "nauc_precision_at_3_diff1": 0.6377585424746148, + "nauc_precision_at_3_max": 0.504041519440942, + "nauc_precision_at_3_std": -0.11346754850123358, + "nauc_precision_at_5_diff1": 0.6239492907917575, + "nauc_precision_at_5_max": 0.5487095212616888, + "nauc_precision_at_5_std": -0.0523864120286966, + "nauc_recall_at_1000_diff1": 0.9346405228757856, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.6620923223090417, + "nauc_recall_at_100_max": 0.8255770144315029, + "nauc_recall_at_100_std": 0.4641382868937102, + "nauc_recall_at_10_diff1": 0.6058370101844033, + "nauc_recall_at_10_max": 0.6163717897701185, + "nauc_recall_at_10_std": 0.041965843547617905, + "nauc_recall_at_1_diff1": 0.7259662689257877, + "nauc_recall_at_1_max": 0.43676348337684495, + "nauc_recall_at_1_std": -0.1449503651311885, + "nauc_recall_at_20_diff1": 0.5987335573498791, + "nauc_recall_at_20_max": 0.6816936820390254, + "nauc_recall_at_20_std": 0.1540250803517749, + "nauc_recall_at_3_diff1": 0.6377585424746157, + "nauc_recall_at_3_max": 0.5040415194409423, + "nauc_recall_at_3_std": -0.11346754850123454, + "nauc_recall_at_5_diff1": 0.6239492907917592, + "nauc_recall_at_5_max": 0.5487095212616887, + "nauc_recall_at_5_std": -0.052386412028695956, + "ndcg_at_1": 0.56735, + "ndcg_at_10": 0.7231, + "ndcg_at_100": 0.74596, + "ndcg_at_1000": 0.74799, + "ndcg_at_20": 0.73569, + "ndcg_at_3": 0.67523, + "ndcg_at_5": 0.7013, + "precision_at_1": 0.56735, + "precision_at_10": 0.08814, + "precision_at_100": 0.00984, + "precision_at_1000": 0.001, + "precision_at_20": 0.04652, + "precision_at_3": 0.25039, + "precision_at_5": 0.16286, + "recall_at_1": 0.56735, + "recall_at_10": 0.88136, + "recall_at_100": 0.98449, + "recall_at_1000": 0.99973, + "recall_at_20": 0.93048, + "recall_at_3": 0.75116, + "recall_at_5": 0.81429 + }, + { + "cv_recall_at_1": 0.71536, + "cv_recall_at_10": 0.94271, + "cv_recall_at_100": 0.99345, + "cv_recall_at_1000": 0.99902, + "cv_recall_at_20": 0.97031, + "cv_recall_at_3": 0.86117, + "cv_recall_at_5": 0.90131, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ], + "main_score": 0.83161, + "map_at_1": 0.71536, + "map_at_10": 0.79571, + "map_at_100": 0.79833, + "map_at_1000": 0.79836, + "map_at_20": 0.79769, + "map_at_3": 0.78076, + "map_at_5": 0.79005, + "mrr_at_1": 0.7153610259269585, + "mrr_at_10": 0.7957073824790565, + "mrr_at_100": 0.7983271878223961, + "mrr_at_1000": 0.7983598682574501, + "mrr_at_20": 0.7976864767200028, + "mrr_at_3": 0.7807592231205266, + "mrr_at_5": 0.7900497165690891, + "nauc_cv_recall_at_1000_diff1": 0.8312673556411718, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.7479815757407692, + "nauc_cv_recall_at_100_max": 0.9122042907617619, + "nauc_cv_recall_at_100_std": 0.5432500070608071, + "nauc_cv_recall_at_10_diff1": 0.7198905770773609, + "nauc_cv_recall_at_10_max": 0.645919725401898, + "nauc_cv_recall_at_10_std": -0.11963773343079089, + "nauc_cv_recall_at_1_diff1": 0.8099670140993427, + "nauc_cv_recall_at_1_max": 0.5193603728956219, + "nauc_cv_recall_at_1_std": -0.1858820928055574, + "nauc_cv_recall_at_20_diff1": 0.7224495327813171, + "nauc_cv_recall_at_20_max": 0.7747247259994867, + "nauc_cv_recall_at_20_std": 0.1537137334369623, + "nauc_cv_recall_at_3_diff1": 0.7284515534007231, + "nauc_cv_recall_at_3_max": 0.5667370329867204, + "nauc_cv_recall_at_3_std": -0.2152454936985054, + "nauc_cv_recall_at_5_diff1": 0.7158676556641865, + "nauc_cv_recall_at_5_max": 0.5848997160398609, + "nauc_cv_recall_at_5_std": -0.21527788827804065, + "nauc_map_at_1000_diff1": 0.7868766395441894, + "nauc_map_at_1000_max": 0.5354946244045167, + "nauc_map_at_1000_std": -0.19143339710059779, + "nauc_map_at_100_diff1": 0.7868643565417248, + "nauc_map_at_100_max": 0.5355539423405813, + "nauc_map_at_100_std": -0.19134157963282333, + "nauc_map_at_10_diff1": 0.785880091957294, + "nauc_map_at_10_max": 0.5359410279488298, + "nauc_map_at_10_std": -0.19304472971847336, + "nauc_map_at_1_diff1": 0.8099670140993427, + "nauc_map_at_1_max": 0.5193603728956219, + "nauc_map_at_1_std": -0.1858820928055574, + "nauc_map_at_20_diff1": 0.7866387909363147, + "nauc_map_at_20_max": 0.536205919161459, + "nauc_map_at_20_std": -0.19059204221385653, + "nauc_map_at_3_diff1": 0.7829163122175201, + "nauc_map_at_3_max": 0.5346738369730953, + "nauc_map_at_3_std": -0.19793512002963232, + "nauc_map_at_5_diff1": 0.7836488568017643, + "nauc_map_at_5_max": 0.5351251307032907, + "nauc_map_at_5_std": -0.19758825403953903, + "nauc_mrr_at_1000_diff1": 0.7868766395441894, + "nauc_mrr_at_1000_max": 0.5354946244045167, + "nauc_mrr_at_1000_std": -0.19143339710059779, + "nauc_mrr_at_100_diff1": 0.7868643565417248, + "nauc_mrr_at_100_max": 0.5355539423405813, + "nauc_mrr_at_100_std": -0.19134157963282333, + "nauc_mrr_at_10_diff1": 0.785880091957294, + "nauc_mrr_at_10_max": 0.5359410279488298, + "nauc_mrr_at_10_std": -0.19304472971847336, + "nauc_mrr_at_1_diff1": 0.8099670140993427, + "nauc_mrr_at_1_max": 0.5193603728956219, + "nauc_mrr_at_1_std": -0.1858820928055574, + "nauc_mrr_at_20_diff1": 0.7866387909363147, + "nauc_mrr_at_20_max": 0.536205919161459, + "nauc_mrr_at_20_std": -0.19059204221385653, + "nauc_mrr_at_3_diff1": 0.7829163122175201, + "nauc_mrr_at_3_max": 0.5346738369730953, + "nauc_mrr_at_3_std": -0.19793512002963232, + "nauc_mrr_at_5_diff1": 0.7836488568017643, + "nauc_mrr_at_5_max": 0.5351251307032907, + "nauc_mrr_at_5_std": -0.19758825403953903, + "nauc_ndcg_at_1000_diff1": 0.7839030308467452, + "nauc_ndcg_at_1000_max": 0.5403658515167975, + "nauc_ndcg_at_1000_std": -0.18649224306281725, + "nauc_ndcg_at_100_diff1": 0.783623949402799, + "nauc_ndcg_at_100_max": 0.5422041371843155, + "nauc_ndcg_at_100_std": -0.18323455028455227, + "nauc_ndcg_at_10_diff1": 0.7786406797093983, + "nauc_ndcg_at_10_max": 0.5460790489887855, + "nauc_ndcg_at_10_std": -0.18765922119721604, + "nauc_ndcg_at_1_diff1": 0.8099670140993427, + "nauc_ndcg_at_1_max": 0.5193603728956219, + "nauc_ndcg_at_1_std": -0.1858820928055574, + "nauc_ndcg_at_20_diff1": 0.7816313221107242, + "nauc_ndcg_at_20_max": 0.5478768608659305, + "nauc_ndcg_at_20_std": -0.1766526940959122, + "nauc_ndcg_at_3_diff1": 0.7724560802243667, + "nauc_ndcg_at_3_max": 0.5408118753435333, + "nauc_ndcg_at_3_std": -0.20137195399903116, + "nauc_ndcg_at_5_diff1": 0.7731218817284778, + "nauc_ndcg_at_5_max": 0.5426458166494701, + "nauc_ndcg_at_5_std": -0.20055915517570486, + "nauc_precision_at_1000_diff1": 0.8312673556410209, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.7479815757407771, + "nauc_precision_at_100_max": 0.9122042907617612, + "nauc_precision_at_100_std": 0.5432500070608429, + "nauc_precision_at_10_diff1": 0.7198905770773566, + "nauc_precision_at_10_max": 0.6459197254018942, + "nauc_precision_at_10_std": -0.11963773343079387, + "nauc_precision_at_1_diff1": 0.8099670140993427, + "nauc_precision_at_1_max": 0.5193603728956219, + "nauc_precision_at_1_std": -0.1858820928055574, + "nauc_precision_at_20_diff1": 0.7224495327813097, + "nauc_precision_at_20_max": 0.774724725999475, + "nauc_precision_at_20_std": 0.15371373343695663, + "nauc_precision_at_3_diff1": 0.7284515534007242, + "nauc_precision_at_3_max": 0.5667370329867211, + "nauc_precision_at_3_std": -0.21524549369850432, + "nauc_precision_at_5_diff1": 0.7158676556641869, + "nauc_precision_at_5_max": 0.5848997160398612, + "nauc_precision_at_5_std": -0.2152778882780402, + "nauc_recall_at_1000_diff1": 0.8312673556411718, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.7479815757407692, + "nauc_recall_at_100_max": 0.9122042907617619, + "nauc_recall_at_100_std": 0.5432500070608071, + "nauc_recall_at_10_diff1": 0.7198905770773609, + "nauc_recall_at_10_max": 0.645919725401898, + "nauc_recall_at_10_std": -0.11963773343079089, + "nauc_recall_at_1_diff1": 0.8099670140993427, + "nauc_recall_at_1_max": 0.5193603728956219, + "nauc_recall_at_1_std": -0.1858820928055574, + "nauc_recall_at_20_diff1": 0.7224495327813171, + "nauc_recall_at_20_max": 0.7747247259994867, + "nauc_recall_at_20_std": 0.1537137334369623, + "nauc_recall_at_3_diff1": 0.7284515534007231, + "nauc_recall_at_3_max": 0.5667370329867204, + "nauc_recall_at_3_std": -0.2152454936985054, + "nauc_recall_at_5_diff1": 0.7158676556641865, + "nauc_recall_at_5_max": 0.5848997160398609, + "nauc_recall_at_5_std": -0.21527788827804065, + "ndcg_at_1": 0.71536, + "ndcg_at_10": 0.83161, + "ndcg_at_100": 0.84306, + "ndcg_at_1000": 0.8438, + "ndcg_at_20": 0.83867, + "ndcg_at_3": 0.80146, + "ndcg_at_5": 0.81809, + "precision_at_1": 0.71536, + "precision_at_10": 0.09427, + "precision_at_100": 0.00993, + "precision_at_1000": 0.001, + "precision_at_20": 0.04852, + "precision_at_3": 0.28706, + "precision_at_5": 0.18026, + "recall_at_1": 0.71536, + "recall_at_10": 0.94271, + "recall_at_100": 0.99345, + "recall_at_1000": 0.99902, + "recall_at_20": 0.97031, + "recall_at_3": 0.86117, + "recall_at_5": 0.90131 + } + ] + }, + "task_name": "XM3600T2IRetrieval" +} \ No newline at end of file From 674020755bcdd7a7d2bb45f1ae198ab4c9d774d9 Mon Sep 17 00:00:00 2001 From: Xin Zhang Date: Mon, 23 Dec 2024 01:50:02 +0800 Subject: [PATCH 121/154] [MIEB] Fix `get_fused_emebddings` (#1612) * Fix fused * fix vlm2vec * Fix lint --- mteb/models/align_models.py | 7 ++----- mteb/models/blip2_models.py | 9 +++------ mteb/models/blip_models.py | 7 ++----- mteb/models/clip_models.py | 7 ++----- mteb/models/cohere_v.py | 7 ++----- mteb/models/dino_models.py | 7 ++----- mteb/models/e5_v.py | 8 +++----- mteb/models/evaclip_models.py | 8 ++------ mteb/models/jina_clip.py | 14 ++------------ mteb/models/nomic_models_vision.py | 8 ++------ mteb/models/openclip_models.py | 8 ++------ mteb/models/siglip_models.py | 8 ++------ mteb/models/vista_models.py | 1 - mteb/models/vlm2vec_models.py | 7 +++++-- mteb/models/voyage_v.py | 7 ++----- 15 files changed, 33 insertions(+), 80 deletions(-) diff --git a/mteb/models/align_models.py b/mteb/models/align_models.py index ca56449303..201541999a 100644 --- a/mteb/models/align_models.py +++ b/mteb/models/align_models.py @@ -101,9 +101,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - task_name: str | None = None, - prompt_type: PromptType | None = None, - batch_size: int = 32, fusion_mode="sum", **kwargs: Any, ): @@ -114,10 +111,10 @@ def get_fused_embeddings( image_embeddings = None if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) if text_embeddings is not None and image_embeddings is not None: if len(text_embeddings) != len(image_embeddings): diff --git a/mteb/models/blip2_models.py b/mteb/models/blip2_models.py index 32f4c1fc9b..59443e46e3 100644 --- a/mteb/models/blip2_models.py +++ b/mteb/models/blip2_models.py @@ -174,9 +174,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - task_name: str | None = None, - prompt_type: PromptType | None = None, - batch_size: int = 32, fusion_mode="sum", **kwargs: Any, ): @@ -188,10 +185,10 @@ def get_fused_embeddings( image_embeddings = None if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) if text_embeddings is not None and image_embeddings is not None: if len(text_embeddings) != len(image_embeddings): @@ -202,7 +199,7 @@ def get_fused_embeddings( fused_embeddings = text_embeddings + image_embeddings elif fusion_mode == "multimodal": fused_embeddings = self.get_multimodal_embeddings( - texts, images, batch_size + texts, images, kwargs.get("batch_size", 32) ) else: # to do: add other fusion mode diff --git a/mteb/models/blip_models.py b/mteb/models/blip_models.py index 0048192fea..dc1b29cd97 100644 --- a/mteb/models/blip_models.py +++ b/mteb/models/blip_models.py @@ -121,9 +121,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - task_name: str | None = None, - prompt_type: PromptType | None = None, - batch_size: int = 32, fusion_mode="sum", **kwargs: Any, ): @@ -134,10 +131,10 @@ def get_fused_embeddings( image_embeddings = None if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) if text_embeddings is not None and image_embeddings is not None: if len(text_embeddings) != len(image_embeddings): diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index fc01b62266..0e53accf1a 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -105,9 +105,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - task_name: str | None = None, - prompt_type: PromptType | None = None, - batch_size: int = 32, fusion_mode="sum", **kwargs: Any, ): @@ -118,10 +115,10 @@ def get_fused_embeddings( image_embeddings = None if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) if text_embeddings is not None and image_embeddings is not None: if len(text_embeddings) != len(image_embeddings): diff --git a/mteb/models/cohere_v.py b/mteb/models/cohere_v.py index cc62292c1e..f4c655fbf6 100644 --- a/mteb/models/cohere_v.py +++ b/mteb/models/cohere_v.py @@ -145,9 +145,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - task_name: str | None = None, - prompt_type: PromptType | None = None, - batch_size: int = 32, fusion_mode="sum", **kwargs: Any, ): @@ -158,10 +155,10 @@ def get_fused_embeddings( image_embeddings = None if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) if text_embeddings is not None and image_embeddings is not None: if len(text_embeddings) != len(image_embeddings): diff --git a/mteb/models/dino_models.py b/mteb/models/dino_models.py index cf37bcdab9..bbac832bfb 100644 --- a/mteb/models/dino_models.py +++ b/mteb/models/dino_models.py @@ -97,9 +97,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - task_name: str | None = None, - prompt_type: PromptType | None = None, - batch_size: int = 32, fusion_mode="sum", **kwargs: Any, ): @@ -110,10 +107,10 @@ def get_fused_embeddings( image_embeddings = None if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) if text_embeddings is not None and image_embeddings is not None: raise ValueError("DINO models only support image encoding.") diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index 202dd7add6..c918086a1a 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -120,9 +120,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] = None, - *, - task_name: str | None = None, - prompt_type: PromptType | None = None, batch_size: int = 8, **kwargs: Any, ): @@ -130,6 +127,7 @@ def get_fused_embeddings( raise ValueError("Either texts or images must be provided") all_fused_embeddings = [] + kwargs.update(batch_size=batch_size) if texts is not None and images is not None: with torch.no_grad(): @@ -168,10 +166,10 @@ def get_fused_embeddings( all_fused_embeddings.append(outputs.cpu()) return torch.cat(all_fused_embeddings, dim=0) elif texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) return text_embeddings elif images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) return image_embeddings diff --git a/mteb/models/evaclip_models.py b/mteb/models/evaclip_models.py index 3987e755bb..017309debb 100644 --- a/mteb/models/evaclip_models.py +++ b/mteb/models/evaclip_models.py @@ -129,10 +129,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - *, - task_name: str | None = None, - prompt_type: PromptType | None = None, - batch_size: int = 32, fusion_mode="sum", **kwargs: Any, ): @@ -143,10 +139,10 @@ def get_fused_embeddings( image_embeddings = None if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) if text_embeddings is not None and image_embeddings is not None: if len(text_embeddings) != len(image_embeddings): diff --git a/mteb/models/jina_clip.py b/mteb/models/jina_clip.py index 78b08c5a57..e463f3e826 100644 --- a/mteb/models/jina_clip.py +++ b/mteb/models/jina_clip.py @@ -101,10 +101,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] = None, - *, - task_name: str | None = None, - prompt_type: PromptType | None = None, - batch_size: int = 32, fusion_mode="sum", **kwargs: Any, ): @@ -116,18 +112,12 @@ def get_fused_embeddings( if texts is not None: text_embeddings = self.get_text_embeddings( - texts, - batch_size=batch_size, - convert_to_numpy=False, - convert_to_tensor=True, + texts, convert_to_numpy=False, convert_to_tensor=True, **kwargs ) if images is not None: image_embeddings = self.get_image_embeddings( - images, - batch_size=batch_size, - convert_to_numpy=False, - convert_to_tensor=True, + images, convert_to_numpy=False, convert_to_tensor=True, **kwargs ) if text_embeddings is not None and image_embeddings is not None: diff --git a/mteb/models/nomic_models_vision.py b/mteb/models/nomic_models_vision.py index 66a1e4ba8e..564b18729f 100644 --- a/mteb/models/nomic_models_vision.py +++ b/mteb/models/nomic_models_vision.py @@ -128,10 +128,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - *, - task_name: str | None = None, - prompt_type: PromptType | None = None, - batch_size: int = 32, fusion_mode="sum", **kwargs: Any, ): @@ -142,10 +138,10 @@ def get_fused_embeddings( image_embeddings = None if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) if text_embeddings is not None and image_embeddings is not None: if len(text_embeddings) != len(image_embeddings): diff --git a/mteb/models/openclip_models.py b/mteb/models/openclip_models.py index df18b88561..3c12641ee6 100644 --- a/mteb/models/openclip_models.py +++ b/mteb/models/openclip_models.py @@ -114,10 +114,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - *, - task_name: str | None = None, - prompt_type: PromptType | None = None, - batch_size: int = 32, fusion_mode="sum", **kwargs: Any, ): @@ -128,10 +124,10 @@ def get_fused_embeddings( image_embeddings = None if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) if text_embeddings is not None and image_embeddings is not None: if len(text_embeddings) != len(image_embeddings): diff --git a/mteb/models/siglip_models.py b/mteb/models/siglip_models.py index c3dd3c757e..48575b642a 100644 --- a/mteb/models/siglip_models.py +++ b/mteb/models/siglip_models.py @@ -123,10 +123,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - *, - task_name: str | None = None, - prompt_type: PromptType | None = None, - batch_size: int = 32, fusion_mode="sum", **kwargs: Any, ): @@ -137,10 +133,10 @@ def get_fused_embeddings( image_embeddings = None if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) if text_embeddings is not None and image_embeddings is not None: if len(text_embeddings) != len(image_embeddings): diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py index 32ec76b313..16188f9191 100644 --- a/mteb/models/vista_models.py +++ b/mteb/models/vista_models.py @@ -176,7 +176,6 @@ def get_fused_embeddings( self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, - *, task_name: str | None = None, prompt_type: PromptType | None = None, batch_size: int = 32, diff --git a/mteb/models/vlm2vec_models.py b/mteb/models/vlm2vec_models.py index 28b97c0a6c..aa2d779f11 100644 --- a/mteb/models/vlm2vec_models.py +++ b/mteb/models/vlm2vec_models.py @@ -267,13 +267,16 @@ def get_fused_embeddings( text_embeddings = None image_embeddings = None + kwargs.update( + task_name=task_name, prompt_type=prompt_type, batch_size=batch_size + ) if texts is not None and images is None: - text_embeddings = self.get_text_embeddings(texts, batch_size) + text_embeddings = self.get_text_embeddings(texts, **kwargs) return text_embeddings if images is not None and texts is None: - image_embeddings = self.get_image_embeddings(images, batch_size) + image_embeddings = self.get_image_embeddings(images, **kwargs) return image_embeddings # text_embeddings is not None and image_embeddings is not None diff --git a/mteb/models/voyage_v.py b/mteb/models/voyage_v.py index fabff4d6b6..ceaf9bc6ae 100644 --- a/mteb/models/voyage_v.py +++ b/mteb/models/voyage_v.py @@ -1,10 +1,10 @@ from __future__ import annotations +import logging import os from functools import partial from typing import Any -import logging import torch from PIL import Image from torch.utils.data import DataLoader @@ -12,7 +12,6 @@ from tqdm import tqdm import mteb -from mteb.model_meta import ModelMeta from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -23,9 +22,7 @@ def downsample_image( image: Image.Image, max_pixels: int = 16000000, target_longest_side: int = 4000 ) -> Image.Image: - """ - if image pixel > max_pixels, downsample it to target_longest_side while keeping the width height ratio. - """ + """If image pixel > max_pixels, downsample it to target_longest_side while keeping the width height ratio.""" width, height = image.size pixels = width * height From 24c37098b71b8cf81922098d3dd11fed51205889 Mon Sep 17 00:00:00 2001 From: Xin Zhang Date: Tue, 24 Dec 2024 18:55:13 +0800 Subject: [PATCH 122/154] [MIEB] Add new multimodal retrieval tasks (#1611) * Add new tasks * Fix score type --- mteb/tasks/Image/Any2AnyRetrieval/__init__.py | 4 ++ .../eng/EncyclopediaVQAIT2ITRetrieval.py | 48 +++++++++++++++ .../eng/LLaVAIT2TRetrieval.py | 59 ++++++++++++++++++ .../eng/OKVQAIT2TRetrieval.py | 48 +++++++++++++++ .../eng/ReMuQIT2TRetrieval.py | 60 +++++++++++++++++++ 5 files changed, 219 insertions(+) create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/EncyclopediaVQAIT2ITRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/LLaVAIT2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/OKVQAIT2TRetrieval.py create mode 100644 mteb/tasks/Image/Any2AnyRetrieval/eng/ReMuQIT2TRetrieval.py diff --git a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py index 59fbb1a11a..9628a41d84 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/__init__.py @@ -5,6 +5,7 @@ from .eng.CIRRIT2IRetrieval import * from .eng.CUB200I2IRetrieval import * from .eng.EDIST2ITRetrieval import * +from .eng.EncyclopediaVQAIT2ITRetrieval import * from .eng.Fashion200kI2TRetrieval import * from .eng.Fashion200kT2IRetrieval import * from .eng.FashionIQIT2IRetrieval import * @@ -18,14 +19,17 @@ from .eng.ImageCoDeT2IRetrieval import * from .eng.InfoSeekIT2ITRetrieval import * from .eng.InfoSeekIT2TRetrieval import * +from .eng.LLaVAIT2TRetrieval import * from .eng.MemotionI2TRetrieval import * from .eng.MemotionT2IRetrieval import * from .eng.METI2IRetrieval import * from .eng.MSCOCOI2TRetrieval import * from .eng.MSCOCOT2IRetrieval import * from .eng.NIGHTSI2IRetrieval import * +from .eng.OKVQAIT2TRetrieval import * from .eng.OVENIT2ITRetrieval import * from .eng.OVENIT2TRetrieval import * +from .eng.ReMuQIT2TRetrieval import * from .eng.ROxfordI2IRetrieval import * from .eng.RP2kI2IRetrieval import * from .eng.RParisI2IRetrieval import * diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/EncyclopediaVQAIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/EncyclopediaVQAIT2ITRetrieval.py new file mode 100644 index 0000000000..4e24d13f7d --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/EncyclopediaVQAIT2ITRetrieval.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class EncyclopediaVQAIT2ITRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="EncyclopediaVQAIT2ITRetrieval", + description="Retrieval Wiki passage and image and passage to answer query about an image.", + reference="https://github.com/google-research/google-research/tree/master/encyclopedic_vqa", + dataset={ + "path": "izhx/UMRB-EncyclopediaVQA", + "revision": "d6eae4f06e260664eb3f276fd1bdb5d4d4c9f32b", + }, + type="Any2AnyRetrieval", + category="it2it", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_5", + date=("2023-01-01", "2023-07-20"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="cc-by-4.0", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@inproceedings{mensink2023encyclopedic, + title={Encyclopedic VQA: Visual questions about detailed properties of fine-grained categories}, + author={Mensink, Thomas and Uijlings, Jasper and Castrejon, Lluis and Goel, Arushi and Cadar, Felipe and Zhou, Howard and Sha, Fei and Araujo, Andr{\'e} and Ferrari, Vittorio}, + booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision}, + pages={3113--3124}, + year={2023} +}""", + descriptive_stats={ + "n_samples": {"test": 3743}, + "avg_character_length": { + "test": { + "average_document_length": 1294.368802424136, + "average_query_length": 51.703713598717606, + "num_documents": 68313, + "num_queries": 3743, + "average_relevant_docs_per_query": 1.3056371894202512, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/LLaVAIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/LLaVAIT2TRetrieval.py new file mode 100644 index 0000000000..e1c4d9ba2a --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/LLaVAIT2TRetrieval.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class LLaVAIT2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="LLaVAIT2TRetrieval", + description="Retrieve responses to answer questions about images.", + reference="https://github.com/LinWeizheDragon/FLMR/blob/main/docs/Datasets.md", + dataset={ + "path": "izhx/UMRB-LLaVA", + "revision": "2a5ed414aab388d8cdd244ba2cf8c8960df4d44d", + }, + type="Any2AnyRetrieval", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_5", + date=("2024-07-06", "2024-02-26"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="cc-by-4.0", + annotations_creators="derived", + dialect=[], + modalities=["text", "image"], + sample_creation="found", + bibtex_citation="""@inproceedings{lin-etal-2024-preflmr, + title = "{P}re{FLMR}: Scaling Up Fine-Grained Late-Interaction Multi-modal Retrievers", + author = "Lin, Weizhe and + Mei, Jingbiao and + Chen, Jinghong and + Byrne, Bill", + editor = "Ku, Lun-Wei and + Martins, Andre and + Srikumar, Vivek", + booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = aug, + year = "2024", + address = "Bangkok, Thailand", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2024.acl-long.289", + doi = "10.18653/v1/2024.acl-long.289", + pages = "5294--5316", +}""", + descriptive_stats={ + "n_samples": {"test": 5120}, + "avg_character_length": { + "test": { + "average_document_length": 546.1925258591925, + "average_query_length": 59.580859375, + "num_documents": 5994, + "num_queries": 5120, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OKVQAIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OKVQAIT2TRetrieval.py new file mode 100644 index 0000000000..a072f896e2 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OKVQAIT2TRetrieval.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class OKVQAIT2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="OKVQAIT2TRetrieval", + description="Retrieval a Wiki passage to answer query about an image.", + reference="https://okvqa.allenai.org", + dataset={ + "path": "izhx/UMRB-OKVQA", + "revision": "96a84a043f5465893670cf616f90e64086c0417a", + }, + type="Any2AnyRetrieval", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_10", + date=("2019-01-01", "2020-07-29"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="cc-by-4.0", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@inproceedings{marino2019ok, + title={Ok-vqa: A visual question answering benchmark requiring external knowledge}, + author={Marino, Kenneth and Rastegari, Mohammad and Farhadi, Ali and Mottaghi, Roozbeh}, + booktitle={Proceedings of the IEEE/cvf conference on computer vision and pattern recognition}, + pages={3195--3204}, + year={2019} +}""", + descriptive_stats={ + "n_samples": {"test": 5046}, + "avg_character_length": { + "test": { + "average_document_length": 41.7072929052715, + "average_query_length": 631.7119703796849, + "num_documents": 114516, + "num_queries": 5046, + "average_relevant_docs_per_query": 7.426674593737614, + } + }, + }, + ) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ReMuQIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ReMuQIT2TRetrieval.py new file mode 100644 index 0000000000..00577d4ce8 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ReMuQIT2TRetrieval.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class ReMuQIT2TRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="ReMuQIT2TRetrieval", + description="Retrieval a Wiki passage to answer query about an image.", + reference="https://github.com/luomancs/ReMuQ", + dataset={ + "path": "izhx/UMRB-ReMuQ", + "revision": "f0bd5955d2897bd1bed56546e88082d966c90a80", + }, + type="Any2AnyRetrieval", + category="it2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_5", + date=("2023-05-15", "2023-07-09"), + domains=["Encyclopaedic"], + task_subtypes=["Image Text Retrieval"], + license="cc0-1.0", + annotations_creators="derived", + dialect=[], + modalities=["image", "text"], + sample_creation="created", + bibtex_citation="""@inproceedings{luo-etal-2023-end, + title = "End-to-end Knowledge Retrieval with Multi-modal Queries", + author = "Luo, Man and + Fang, Zhiyuan and + Gokhale, Tejas and + Yang, Yezhou and + Baral, Chitta", + editor = "Rogers, Anna and + Boyd-Graber, Jordan and + Okazaki, Naoaki", + booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = jul, + year = "2023", + address = "Toronto, Canada", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.acl-long.478", + doi = "10.18653/v1/2023.acl-long.478", + pages = "8573--8589", +}""", + descriptive_stats={ + "n_samples": {"test": 3609}, + "avg_character_length": { + "test": { + "average_document_length": 208.18675158868538, + "average_query_length": 73.85508451094486, + "num_documents": 138794, + "num_queries": 3609, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) From ff74380542e3942b7e60e782636dc2aa7a4c0d3e Mon Sep 17 00:00:00 2001 From: Xin Zhang Date: Wed, 25 Dec 2024 23:25:40 +0800 Subject: [PATCH 123/154] [MIEB] Switch to ViDoRe BEIR version (#1607) * Fix ViDoRe corpus * fix lint * ViDoRe beir version --- .../eng/VidoreBenchRetrieval.py | 175 ++++++++---------- 1 file changed, 77 insertions(+), 98 deletions(-) diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py index 68ae7ed387..f85e1dadd7 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py @@ -6,88 +6,67 @@ from mteb.abstasks.TaskMetadata import TaskMetadata -def _load_data(path: str, splits: str, cache_dir: str = None, revision: str = None): +def _load_data( + path: str, + splits: str, + cache_dir: str | None = None, + revision: str | None = None, +): corpus = {} queries = {} relevant_docs = {} - dataset = load_dataset( - path, - cache_dir=cache_dir, - revision=revision, - ) - for split in splits: - split_dataset = dataset[split] - split_dataset = split_dataset.rename_column("query", "text") - corpus[split] = split_dataset.map( - lambda x, idx: { - "id": f"corpus-{split}-{idx}", - "modality": "image", - "text": None, - }, - with_indices=True, + query_ds = load_dataset( + path, + "queries", + split=split, + cache_dir=cache_dir, + revision=revision, ) - - queries[split] = split_dataset.map( - lambda x, idx: { - "id": f"query-{split}-{idx}", + query_ds = query_ds.map( + lambda x: { + "id": f"query-{split}-{x['query-id']}", + "text": x["query"], "image": None, "modality": "text", }, - with_indices=True, + remove_columns=["query-id", "query"], ) - relevant_docs[split] = {} - for index in range(len(split_dataset)): - query_id = f"query-{split}-{index}" - doc_id = f"corpus-{split}-{index}" - if query_id not in relevant_docs[split]: - relevant_docs[split][query_id] = {} - relevant_docs[split][query_id][doc_id] = 1 - return corpus, queries, relevant_docs - - -def _load_data_qc_unmatched( - path: str, splits: str, cache_dir: str = None, revision: str = None, num_queries=100 -): - corpus = {} - queries = {} - relevant_docs = {} - - dataset = load_dataset( - path, - cache_dir=cache_dir, - revision=revision, - ) - - for split in splits: - split_dataset = dataset[split] - split_dataset = split_dataset.rename_column("query", "text") - corpus[split] = split_dataset.map( - lambda x, idx: { - "id": f"corpus-{split}-{idx}", - "modality": "image", + queries[split] = query_ds + + corpus_ds = load_dataset( + path, + "corpus", + split=split, + cache_dir=cache_dir, + revision=revision, + ) + corpus_ds = corpus_ds.map( + lambda x: { + "id": f"corpus-{split}-{x['corpus-id']}", "text": None, + "modality": "image", }, - with_indices=True, + remove_columns=["corpus-id"], ) - - split_dataset = split_dataset.select(range(num_queries)) - queries[split] = split_dataset.map( - lambda x, idx: { - "id": f"query-{split}-{idx}", - "image": None, - "modality": "text", - }, - with_indices=True, + corpus[split] = corpus_ds + + qrels_ds = load_dataset( + path, + "qrels", + split=split, + cache_dir=cache_dir, + revision=revision, ) relevant_docs[split] = {} - for index in range(len(queries[split])): - query_id = f"query-{split}-{index}" - doc_id = f"corpus-{split}-{index}" - if query_id not in relevant_docs[split]: - relevant_docs[split][query_id] = {} - relevant_docs[split][query_id][doc_id] = 1 + for row in qrels_ds: + qid = f"query-{split}-{row['query-id']}" + did = f"corpus-{split}-{row['corpus-id']}" + if qid not in relevant_docs[split]: + relevant_docs[split][qid] = {} + relevant_docs[split][qid][did] = int(row["score"]) + return corpus, queries, relevant_docs @@ -97,8 +76,8 @@ class VidoreArxivQARetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve associated pages according to questions.", reference="https://arxiv.org/pdf/2407.01449", dataset={ - "path": "vidore/arxivqa_test_subsampled", - "revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", + "path": "vidore/arxivqa_test_subsampled_beir", + "revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", }, type="Any2AnyRetrieval", category="t2i", @@ -150,8 +129,8 @@ class VidoreDocVQARetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve associated pages according to questions.", reference="https://arxiv.org/pdf/2407.01449", dataset={ - "path": "vidore/docvqa_test_subsampled", - "revision": "b1d89eda849e636676df6ead8002602fb1858600", + "path": "vidore/docvqa_test_subsampled_beir", + "revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", }, type="Any2AnyRetrieval", category="t2i", @@ -203,8 +182,8 @@ class VidoreInfoVQARetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve associated pages according to questions.", reference="https://arxiv.org/pdf/2407.01449", dataset={ - "path": "vidore/infovqa_test_subsampled", - "revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", + "path": "vidore/infovqa_test_subsampled_beir", + "revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", }, type="Any2AnyRetrieval", category="t2i", @@ -256,8 +235,8 @@ class VidoreTabfquadRetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve associated pages according to questions.", reference="https://arxiv.org/pdf/2407.01449", dataset={ - "path": "vidore/tabfquad_test_subsampled", - "revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", + "path": "vidore/tabfquad_test_subsampled_beir", + "revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", }, type="Any2AnyRetrieval", category="t2i", @@ -284,7 +263,7 @@ class VidoreTabfquadRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 1.0, "average_query_length": 100.63214285714285, - "num_documents": 280, + "num_documents": 70, "num_queries": 280, "average_relevant_docs_per_query": 1.0, } @@ -309,8 +288,8 @@ class VidoreTatdqaRetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve associated pages according to questions.", reference="https://arxiv.org/pdf/2407.01449", dataset={ - "path": "vidore/tatdqa_test", - "revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", + "path": "vidore/tatdqa_test_beir", + "revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", }, type="Any2AnyRetrieval", category="t2i", @@ -337,7 +316,7 @@ class VidoreTatdqaRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 1.0, "average_query_length": 72.76368009621167, - "num_documents": 1663, + "num_documents": 277, "num_queries": 1663, "average_relevant_docs_per_query": 1.0, } @@ -362,8 +341,8 @@ class VidoreShiftProjectRetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve associated pages according to questions.", reference="https://arxiv.org/pdf/2407.01449", dataset={ - "path": "vidore/shiftproject_test", - "revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", + "path": "vidore/shiftproject_test_beir", + "revision": "84a382e05c4473fed9cff2bbae95fe2379416117", }, type="Any2AnyRetrieval", category="t2i", @@ -399,7 +378,7 @@ class VidoreShiftProjectRetrieval(AbsTaskAny2AnyRetrieval): ) def load_data(self, **kwargs): - self.corpus, self.queries, self.relevant_docs = _load_data_qc_unmatched( + self.corpus, self.queries, self.relevant_docs = _load_data( path=self.metadata_dict["dataset"]["path"], splits=self.metadata_dict["eval_splits"], cache_dir=kwargs.get("cache_dir", None), @@ -415,8 +394,8 @@ class VidoreSyntheticDocQAAIRetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve associated pages according to questions.", reference="https://arxiv.org/pdf/2407.01449", dataset={ - "path": "vidore/syntheticDocQA_artificial_intelligence_test", - "revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", + "path": "vidore/syntheticDocQA_artificial_intelligence_test_beir", + "revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", }, type="Any2AnyRetrieval", category="t2i", @@ -443,7 +422,7 @@ class VidoreSyntheticDocQAAIRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 1.0, "average_query_length": 77.71, - "num_documents": 1000, + "num_documents": 968, "num_queries": 100, "average_relevant_docs_per_query": 1.0, } @@ -452,7 +431,7 @@ class VidoreSyntheticDocQAAIRetrieval(AbsTaskAny2AnyRetrieval): ) def load_data(self, **kwargs): - self.corpus, self.queries, self.relevant_docs = _load_data_qc_unmatched( + self.corpus, self.queries, self.relevant_docs = _load_data( path=self.metadata_dict["dataset"]["path"], splits=self.metadata_dict["eval_splits"], cache_dir=kwargs.get("cache_dir", None), @@ -468,8 +447,8 @@ class VidoreSyntheticDocQAEnergyRetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve associated pages according to questions.", reference="https://arxiv.org/pdf/2407.01449", dataset={ - "path": "vidore/syntheticDocQA_energy_test", - "revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", + "path": "vidore/syntheticDocQA_energy_test_beir", + "revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", }, type="Any2AnyRetrieval", category="t2i", @@ -496,7 +475,7 @@ class VidoreSyntheticDocQAEnergyRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 1.0, "average_query_length": 83.69, - "num_documents": 1000, + "num_documents": 977, "num_queries": 100, "average_relevant_docs_per_query": 1.0, } @@ -505,7 +484,7 @@ class VidoreSyntheticDocQAEnergyRetrieval(AbsTaskAny2AnyRetrieval): ) def load_data(self, **kwargs): - self.corpus, self.queries, self.relevant_docs = _load_data_qc_unmatched( + self.corpus, self.queries, self.relevant_docs = _load_data( path=self.metadata_dict["dataset"]["path"], splits=self.metadata_dict["eval_splits"], cache_dir=kwargs.get("cache_dir", None), @@ -521,8 +500,8 @@ class VidoreSyntheticDocQAGovernmentReportsRetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve associated pages according to questions.", reference="https://arxiv.org/pdf/2407.01449", dataset={ - "path": "vidore/syntheticDocQA_government_reports_test", - "revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", + "path": "vidore/syntheticDocQA_government_reports_test_beir", + "revision": "b4909afa930f81282fd20601e860668073ad02aa", }, type="Any2AnyRetrieval", category="t2i", @@ -549,7 +528,7 @@ class VidoreSyntheticDocQAGovernmentReportsRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 1.0, "average_query_length": 82.53, - "num_documents": 1000, + "num_documents": 972, "num_queries": 100, "average_relevant_docs_per_query": 1.0, } @@ -558,7 +537,7 @@ class VidoreSyntheticDocQAGovernmentReportsRetrieval(AbsTaskAny2AnyRetrieval): ) def load_data(self, **kwargs): - self.corpus, self.queries, self.relevant_docs = _load_data_qc_unmatched( + self.corpus, self.queries, self.relevant_docs = _load_data( path=self.metadata_dict["dataset"]["path"], splits=self.metadata_dict["eval_splits"], cache_dir=kwargs.get("cache_dir", None), @@ -574,8 +553,8 @@ class VidoreSyntheticDocQAHealthcareIndustryRetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve associated pages according to questions.", reference="https://arxiv.org/pdf/2407.01449", dataset={ - "path": "vidore/syntheticDocQA_healthcare_industry_test", - "revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", + "path": "vidore/syntheticDocQA_healthcare_industry_test_beir", + "revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", }, type="Any2AnyRetrieval", category="t2i", @@ -602,7 +581,7 @@ class VidoreSyntheticDocQAHealthcareIndustryRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 1.0, "average_query_length": 80.43, - "num_documents": 1000, + "num_documents": 965, "num_queries": 100, "average_relevant_docs_per_query": 1.0, } @@ -611,7 +590,7 @@ class VidoreSyntheticDocQAHealthcareIndustryRetrieval(AbsTaskAny2AnyRetrieval): ) def load_data(self, **kwargs): - self.corpus, self.queries, self.relevant_docs = _load_data_qc_unmatched( + self.corpus, self.queries, self.relevant_docs = _load_data( path=self.metadata_dict["dataset"]["path"], splits=self.metadata_dict["eval_splits"], cache_dir=kwargs.get("cache_dir", None), From c14c0066ae45581a6bc926bafa8ad4ac5ce5b27a Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 25 Dec 2024 17:44:06 +0200 Subject: [PATCH 124/154] Extend MIEB test coverage (#1629) * add one task from each image AbsTask to test grid * add visual sts to test grid --- tests/test_benchmark/task_grid.py | 37 +++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/tests/test_benchmark/task_grid.py b/tests/test_benchmark/task_grid.py index 34af684a70..afe9ff2218 100644 --- a/tests/test_benchmark/task_grid.py +++ b/tests/test_benchmark/task_grid.py @@ -10,8 +10,14 @@ from mteb.tasks.Clustering.eng.TwentyNewsgroupsClustering import ( TwentyNewsgroupsClusteringFast, ) +from mteb.tasks.Image.Any2AnyMultiChoice import ROxfordEasyI2IMultiChoice +from mteb.tasks.Image.Any2AnyRetrieval import Flickr30kI2TRetrieval +from mteb.tasks.Image.Any2TextMultipleChoice import CVBenchCount from mteb.tasks.Image.Clustering import TinyImageNet -from mteb.tasks.Image.ImageTextPairClassification import Winoground +from mteb.tasks.Image.ImageClassification import OxfordPetsClassification +from mteb.tasks.Image.ImageMultilabelClassification import VOC2007Classification +from mteb.tasks.Image.ImageTextPairClassification import AROFlickrOrder +from mteb.tasks.Image.VisualSTS import STS16VisualSTS from mteb.tasks.Image.ZeroshotClassification import RenderedSST2 from .mock_tasks import ( @@ -76,23 +82,36 @@ def dataset_transform(self): tiny_imagenet = TinyImageNet() renderedSST2 = RenderedSST2() -winoground = Winoground() +aro = AROFlickrOrder() +oxford_pets = OxfordPetsClassification() +voc2007 = VOC2007Classification() +flickr = Flickr30kI2TRetrieval() +roxford_mc = ROxfordEasyI2IMultiChoice() +cvbench_count = CVBenchCount() +sts16 = STS16VisualSTS() ## method override to speed up tests tiny_imagenet.dataset_transform = dataset_transform.__get__(tiny_imagenet) renderedSST2.dataset_transform = dataset_transform.__get__(renderedSST2) -winoground.dataset_transform = dataset_transform.__get__(winoground) +aro.dataset_transform = dataset_transform.__get__(aro) +oxford_pets.dataset_transform = dataset_transform.__get__(oxford_pets) +voc2007.dataset_transform = dataset_transform.__get__(voc2007) +flickr.dataset_transform = dataset_transform.__get__(flickr) +roxford_mc.dataset_transform = dataset_transform.__get__(roxford_mc) +cvbench_count.dataset_transform = dataset_transform.__get__(cvbench_count) +sts16.dataset_transform = dataset_transform.__get__(sts16) MIEB_TASK_TEST_GRID = [ tiny_imagenet, # image clustering - # winoground, # pair classification. Gated + aro, # pair classification renderedSST2, # zero shot classification - # The following takes a long time. Consider creating a mock class. - # "CIRRIT2TRetrieval", # it2i retrieval - # "MSCOCOI2TRetrieval", # i2t retrieval - # "MSCOCOT2IRetrieval", # t2i retrieval - # oxford_flowers, # image classification + oxford_pets, # image classification + voc2007, # multilabel classification + flickr, # I2T retrieval + roxford_mc, # Any2Any MultiChoice + cvbench_count, # Any2Any Text MultiChoice + sts16, # visual sts ] MIEB_TASK_TEST_GRID_AS_STRING = [ From 10ba68d9683b4412a7bce116f9f9d9c6fb7e6e24 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Thu, 26 Dec 2024 14:20:33 +0200 Subject: [PATCH 125/154] [mieb] Task filtering by modality supported by models (#1633) * fix function signature for moco loader * filter out tasks by model modalities * correct conditions * add model meta to relevant models * use modalities instead and separate out constants --- mteb/abstasks/TaskMetadata.py | 6 +----- mteb/evaluation/MTEB.py | 9 +++++++++ mteb/modalities.py | 8 ++++++++ mteb/model_meta.py | 3 +++ mteb/models/align_models.py | 1 + mteb/models/blip2_models.py | 2 ++ mteb/models/blip_models.py | 8 ++++++++ mteb/models/clip_models.py | 3 +++ mteb/models/cohere_v.py | 2 ++ mteb/models/dino_models.py | 4 ++++ mteb/models/e5_v.py | 1 + mteb/models/evaclip_models.py | 4 ++++ mteb/models/jina_clip.py | 1 + mteb/models/moco_models.py | 11 ++++++++++- mteb/models/nomic_models_vision.py | 1 + mteb/models/openclip_models.py | 8 ++++++++ mteb/models/siglip_models.py | 10 ++++++++++ mteb/models/vista_models.py | 2 ++ mteb/models/vlm2vec_models.py | 2 ++ mteb/models/voyage_v.py | 5 +++-- 20 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 mteb/modalities.py diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index fa2e8c9e01..2bd7f18bdd 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -15,6 +15,7 @@ path_to_lang_codes, path_to_lang_scripts, ) +from ..modalities import MODALITIES TASK_SUBTYPE = Literal[ "Article retrieval", @@ -124,11 +125,6 @@ "it2it", ] -MODALITIES = Literal[ - "text", - "image", -] - ANNOTATOR_TYPE = Literal[ "expert-annotated", "human-annotated", diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py index d6c1b43ab4..8c64202d1e 100644 --- a/mteb/evaluation/MTEB.py +++ b/mteb/evaluation/MTEB.py @@ -372,6 +372,15 @@ def run( f"\n\n********************** Evaluating {task.metadata.name} **********************" ) + # skip evaluation if the model does not support the task modalities. + task_modalities = "".join(sorted(task.metadata.modalities)) + if "".join(sorted(meta.modalities)) != task_modalities: + logger.info( + f"{meta.name} only supports {meta.modalities}, but the task modalities are {task.metadata.modalities}." + ) + del self.tasks[0] # empty memory + continue + # skip evaluation if results folder exists and overwrite_results is False if output_path: save_path = output_path / f"{task.metadata.name}{task.save_suffix}.json" diff --git a/mteb/modalities.py b/mteb/modalities.py new file mode 100644 index 0000000000..ff83f963af --- /dev/null +++ b/mteb/modalities.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +from typing_extensions import Literal + +MODALITIES = Literal[ + "text", + "image", +] diff --git a/mteb/model_meta.py b/mteb/model_meta.py index 7acb806b81..b81dd1e1c5 100644 --- a/mteb/model_meta.py +++ b/mteb/model_meta.py @@ -10,6 +10,7 @@ from mteb.encoder_interface import Encoder from .languages import ISO_LANGUAGE_SCRIPT +from .modalities import MODALITIES if TYPE_CHECKING: from .models.sentence_transformer_wrapper import SentenceTransformerWrapper @@ -74,6 +75,7 @@ class ModelMeta(BaseModel): input such as "query: {document}" or "passage: {document}". zero_shot_benchmarks: A list of benchmarks on which the model has been evaluated in a zero-shot setting. By default we assume that all models are evaluated non-zero-shot unless specified otherwise. + modalities: A list of strings representing the modalities the model supports. Default is ["text]. """ name: str | None @@ -94,6 +96,7 @@ class ModelMeta(BaseModel): similarity_fn_name: DISTANCE_METRICS | None = None use_instuctions: bool | None = None zero_shot_benchmarks: list[str] | None = None + modalities: list[MODALITIES] = ["text"] def to_dict(self): dict_repr = self.model_dump() diff --git a/mteb/models/align_models.py b/mteb/models/align_models.py index 201541999a..911e601750 100644 --- a/mteb/models/align_models.py +++ b/mteb/models/align_models.py @@ -143,6 +143,7 @@ def get_fused_embeddings( open_source=True, revision="e96a37facc7b1f59090ece82293226b817afd6ba", release_date="2023-02-24", + modalities=["image", "text"], ) if __name__ == "__main__": diff --git a/mteb/models/blip2_models.py b/mteb/models/blip2_models.py index 59443e46e3..d013e49c28 100644 --- a/mteb/models/blip2_models.py +++ b/mteb/models/blip2_models.py @@ -225,6 +225,7 @@ def get_fused_embeddings( open_source=True, revision="51572668da0eb669e01a189dc22abe6088589a24", release_date="2024-03-22", + modalities=["image", "text"], ) blip2_opt_6_7b_coco = ModelMeta( @@ -237,6 +238,7 @@ def get_fused_embeddings( open_source=True, revision="0d580de59320a25a4d2c386387bcef310d5f286e", release_date="2024-03-31", + modalities=["image", "text"], ) diff --git a/mteb/models/blip_models.py b/mteb/models/blip_models.py index dc1b29cd97..1be907e9b8 100644 --- a/mteb/models/blip_models.py +++ b/mteb/models/blip_models.py @@ -164,6 +164,7 @@ def get_fused_embeddings( open_source=True, revision="2227ac38c9f16105cb0412e7cab4759978a8fd90", release_date="2023-12-07", + modalities=["image", "text"], ) blip_image_captioning_base = ModelMeta( @@ -176,6 +177,7 @@ def get_fused_embeddings( open_source=True, revision="89b09ea1789f7addf2f6d6f0dfc4ce10ab58ef84", release_date="2023-08-01", + modalities=["image", "text"], ) @@ -189,6 +191,7 @@ def get_fused_embeddings( open_source=True, revision="c7df8e7cd7aa2ee9af18f56e2b29e59a92651b64", release_date="2023-12-07", + modalities=["image", "text"], ) blip_vqa_capfilt_large = ModelMeta( @@ -201,6 +204,7 @@ def get_fused_embeddings( open_source=True, revision="e53f95265aeab69013fabb5380500ab984adbbb4", release_date="2023-01-22", + modalities=["image", "text"], ) blip_itm_base_coco = ModelMeta( @@ -213,6 +217,7 @@ def get_fused_embeddings( open_source=True, revision="7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f", release_date="2023-08-01", + modalities=["image", "text"], ) blip_itm_large_coco = ModelMeta( @@ -225,6 +230,7 @@ def get_fused_embeddings( open_source=True, revision="fef05cafc05298067cbbca00b125749394a77a6f", release_date="2023-08-01", + modalities=["image", "text"], ) blip_itm_base_flickr = ModelMeta( @@ -237,6 +243,7 @@ def get_fused_embeddings( open_source=True, revision="1de29e660d91ae1786c1876212ea805a22eab251", release_date="2023-08-01", + modalities=["image", "text"], ) blip_itm_large_flickr = ModelMeta( @@ -249,6 +256,7 @@ def get_fused_embeddings( open_source=True, revision="bda12e6506758f54261b5ab174b2c55a3ba143fb", release_date="2023-08-01", + modalities=["image", "text"], ) diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index 0e53accf1a..141a684da8 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -147,6 +147,7 @@ def get_fused_embeddings( open_source=True, revision="32bd64288804d66eefd0ccbe215aa642df71cc41", release_date="2021-02-26", + modalities=["image", "text"], ) clip_vit_base_patch32 = ModelMeta( @@ -159,6 +160,7 @@ def get_fused_embeddings( open_source=True, revision="3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", release_date="2021-02-26", + modalities=["image", "text"], ) clip_vit_base_patch16 = ModelMeta( @@ -171,6 +173,7 @@ def get_fused_embeddings( open_source=True, revision="57c216476eefef5ab752ec549e440a49ae4ae5f3", release_date="2021-02-26", + modalities=["image", "text"], ) if __name__ == "__main__": diff --git a/mteb/models/cohere_v.py b/mteb/models/cohere_v.py index f4c655fbf6..749fc42016 100644 --- a/mteb/models/cohere_v.py +++ b/mteb/models/cohere_v.py @@ -195,6 +195,7 @@ def get_fused_embeddings( license=None, similarity_fn_name="cosine", framework=[], + modalities=["image", "text"], ) cohere_eng_3 = ModelMeta( @@ -211,6 +212,7 @@ def get_fused_embeddings( license=None, similarity_fn_name="cosine", framework=[], + modalities=["image", "text"], ) if __name__ == "__main__": diff --git a/mteb/models/dino_models.py b/mteb/models/dino_models.py index bbac832bfb..76512b30f9 100644 --- a/mteb/models/dino_models.py +++ b/mteb/models/dino_models.py @@ -130,6 +130,7 @@ def get_fused_embeddings( open_source=True, revision="ed25f3a31f01632728cabb09d1542f84ab7b0056", release_date="2023-07-18", + modalities=["image"], ) dinov2_base = ModelMeta( @@ -142,6 +143,7 @@ def get_fused_embeddings( open_source=True, revision="f9e44c814b77203eaa57a6bdbbd535f21ede1415", release_date="2023-07-18", + modalities=["image"], ) dinov2_large = ModelMeta( @@ -154,6 +156,7 @@ def get_fused_embeddings( open_source=True, revision="47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", release_date="2023-07-18", + modalities=["image"], ) dinov2_giant = ModelMeta( @@ -166,6 +169,7 @@ def get_fused_embeddings( open_source=True, revision="611a9d42f2335e0f921f1e313ad3c1b7178d206d", release_date="2023-07-18", + modalities=["image"], ) if __name__ == "__main__": diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index c918086a1a..ddb1f12551 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -185,6 +185,7 @@ def get_fused_embeddings( open_source=True, revision="0c1f22679417b3ae925d779442221c40cd1861ab", release_date="2024-07-17", + modalities=["image", "text"], ) if __name__ == "__main__": diff --git a/mteb/models/evaclip_models.py b/mteb/models/evaclip_models.py index 017309debb..9be55cfc85 100644 --- a/mteb/models/evaclip_models.py +++ b/mteb/models/evaclip_models.py @@ -175,6 +175,7 @@ def get_fused_embeddings( open_source=True, revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", + modalities=["image", "text"], ) EVA02_CLIP_L_14 = ModelMeta( @@ -187,6 +188,7 @@ def get_fused_embeddings( open_source=True, revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", + modalities=["image", "text"], ) EVA02_CLIP_bigE_14 = ModelMeta( @@ -199,6 +201,7 @@ def get_fused_embeddings( open_source=True, revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", + modalities=["image", "text"], ) @@ -212,4 +215,5 @@ def get_fused_embeddings( open_source=True, revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", + modalities=["image", "text"], ) diff --git a/mteb/models/jina_clip.py b/mteb/models/jina_clip.py index e463f3e826..d24dcf1827 100644 --- a/mteb/models/jina_clip.py +++ b/mteb/models/jina_clip.py @@ -158,6 +158,7 @@ def encode( # type: ignore open_source=True, revision="06150c7c382d7a4faedc7d5a0d8cdb59308968f4", release_date="2024-05-30", + modalities=["image", "text"], ) diff --git a/mteb/models/moco_models.py b/mteb/models/moco_models.py index b8dc8ef0b7..a64cef499e 100644 --- a/mteb/models/moco_models.py +++ b/mteb/models/moco_models.py @@ -50,7 +50,14 @@ def __init__( ) @staticmethod - def get_text_embeddings(texts: list[str], batch_size: int = 32): + def get_text_embeddings( + texts: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): raise ValueError("MOCO models only support image encoding.") def get_image_embeddings( @@ -141,6 +148,7 @@ def get_fused_embeddings( open_source=True, revision="7d091cd70772c5c0ecf7f00b5f12ca609a99d69d", release_date="2024-06-03", + modalities=["image"], ) mocov3_vit_large = ModelMeta( @@ -153,4 +161,5 @@ def get_fused_embeddings( open_source=True, revision="7bf75358d616f39b9716148bf4e3425f3bd35b47", release_date="2024-06-03", + modalities=["image"], ) diff --git a/mteb/models/nomic_models_vision.py b/mteb/models/nomic_models_vision.py index 564b18729f..098e0d8502 100644 --- a/mteb/models/nomic_models_vision.py +++ b/mteb/models/nomic_models_vision.py @@ -171,6 +171,7 @@ def get_fused_embeddings( open_source=True, revision="af2246fffdab78d8458418480e4886a8e48b70a7", release_date="2024-06-08", + modalities=["image", "text"], ) if __name__ == "__main__": diff --git a/mteb/models/openclip_models.py b/mteb/models/openclip_models.py index 3c12641ee6..39469a7385 100644 --- a/mteb/models/openclip_models.py +++ b/mteb/models/openclip_models.py @@ -160,6 +160,7 @@ def get_fused_embeddings( open_source=True, revision="84c9828e63dc9a9351d1fe637c346d4c1c4db341", release_date="2023-04-26", + modalities=["image", "text"], ) CLIP_ViT_B_32_DataComp_XL_s13B_b90K = ModelMeta( @@ -172,6 +173,7 @@ def get_fused_embeddings( open_source=True, revision="f0e2ffa09cbadab3db6a261ec1ec56407ce42912", release_date="2023-04-26", + modalities=["image", "text"], ) CLIP_ViT_B_16_DataComp_XL_s13B_b90K = ModelMeta( @@ -184,6 +186,7 @@ def get_fused_embeddings( open_source=True, revision="d110532e8d4ff91c574ee60a342323f28468b287", release_date="2023-04-26", + modalities=["image", "text"], ) CLIP_ViT_bigG_14_laion2B_39B_b160k = ModelMeta( @@ -196,6 +199,7 @@ def get_fused_embeddings( open_source=True, revision="bc7788f151930d91b58474715fdce5524ad9a189", release_date="2023-01-23", + modalities=["image", "text"], ) CLIP_ViT_g_14_laion2B_s34B_b88K = ModelMeta( @@ -208,6 +212,7 @@ def get_fused_embeddings( open_source=True, revision="15efd0f6ac0c40c0f9da7becca03c974d7012604", release_date="2023-03-06", + modalities=["image", "text"], ) CLIP_ViT_H_14_laion2B_s32B_b79K = ModelMeta( @@ -220,6 +225,7 @@ def get_fused_embeddings( open_source=True, revision="de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", release_date="2022-09-15", + modalities=["image", "text"], ) CLIP_ViT_L_14_laion2B_s32B_b82K = ModelMeta( @@ -232,6 +238,7 @@ def get_fused_embeddings( open_source=True, revision="1627032197142fbe2a7cfec626f4ced3ae60d07a", release_date="2022-09-15", + modalities=["image", "text"], ) CLIP_ViT_B_32_laion2B_s34B_b79K = ModelMeta( @@ -244,4 +251,5 @@ def get_fused_embeddings( open_source=True, revision="08f73555f1b2fb7c82058aebbd492887a94968ef", release_date="2022-09-15", + modalities=["image", "text"], ) diff --git a/mteb/models/siglip_models.py b/mteb/models/siglip_models.py index 48575b642a..14ddb81002 100644 --- a/mteb/models/siglip_models.py +++ b/mteb/models/siglip_models.py @@ -165,6 +165,7 @@ def get_fused_embeddings( open_source=True, revision="d04cf29fca7b6374f74d8bea1969314492266b5e", release_date="2024-01-08", + modalities=["image", "text"], ) siglip_so400m_patch14_384 = ModelMeta( @@ -177,6 +178,7 @@ def get_fused_embeddings( open_source=True, revision="9fdffc58afc957d1a03a25b10dba0329ab15c2a3", release_date="2024-01-08", + modalities=["image", "text"], ) siglip_so400m_patch16_256_i18n = ModelMeta( @@ -189,6 +191,7 @@ def get_fused_embeddings( open_source=True, revision="365d321c0cfdea96bc28e3a29787a11a062681a1", release_date="2024-01-08", + modalities=["image", "text"], ) siglip_base_patch16_256_multilingual = ModelMeta( @@ -201,6 +204,7 @@ def get_fused_embeddings( open_source=True, revision="8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6", release_date="2024-01-08", + modalities=["image", "text"], ) siglip_base_patch16_256 = ModelMeta( @@ -213,6 +217,7 @@ def get_fused_embeddings( open_source=True, revision="b078df89e446d623010d890864d4207fe6399f61", release_date="2024-01-08", + modalities=["image", "text"], ) siglip_base_patch16_512 = ModelMeta( @@ -225,6 +230,7 @@ def get_fused_embeddings( open_source=True, revision="753a949581523b60257d93e18391e8c27f72eb22", release_date="2024-01-08", + modalities=["image", "text"], ) siglip_base_patch16_384 = ModelMeta( @@ -237,6 +243,7 @@ def get_fused_embeddings( open_source=True, revision="41aec1c83b32e0a6fca20ad88ba058aa5b5ea394", release_date="2024-01-08", + modalities=["image", "text"], ) siglip_base_patch16_224 = ModelMeta( @@ -249,6 +256,7 @@ def get_fused_embeddings( open_source=True, revision="7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed", release_date="2024-01-08", + modalities=["image", "text"], ) siglip_large_patch16_256 = ModelMeta( @@ -261,6 +269,7 @@ def get_fused_embeddings( open_source=True, revision="d0da9f876e7d66b4e250cd2450c3ba2ce735e447", release_date="2024-01-08", + modalities=["image", "text"], ) siglip_large_patch16_384 = ModelMeta( @@ -273,6 +282,7 @@ def get_fused_embeddings( open_source=True, revision="ce005573a40965dfd21fd937fbdeeebf2439fc35", release_date="2024-01-08", + modalities=["image", "text"], ) if __name__ == "__main__": diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py index 16188f9191..3adb7607b1 100644 --- a/mteb/models/vista_models.py +++ b/mteb/models/vista_models.py @@ -230,6 +230,7 @@ def calculate_probs(self, text_embeddings, image_embeddings): open_source=True, revision="98db10b10d22620010d06f11733346e1c98c34aa", release_date="2024-06-06", + modalities=["image", "text"], ) visualized_bge_m3 = ModelMeta( @@ -243,6 +244,7 @@ def calculate_probs(self, text_embeddings, image_embeddings): open_source=True, revision="98db10b10d22620010d06f11733346e1c98c34aa", release_date="2024-06-06", + modalities=["image", "text"], ) if __name__ == "__main__": diff --git a/mteb/models/vlm2vec_models.py b/mteb/models/vlm2vec_models.py index aa2d779f11..e5a160753c 100644 --- a/mteb/models/vlm2vec_models.py +++ b/mteb/models/vlm2vec_models.py @@ -372,6 +372,7 @@ def get_fused_embeddings( open_source=True, revision="7403b6327958071c1e33c822c7453adadccc7298", release_date="2024-10-08", + modalities=["image", "text"], ) vlm2vec_full = ModelMeta( @@ -384,4 +385,5 @@ def get_fused_embeddings( open_source=True, revision="e9afa98002097ac2471827ba23ea1f2ddd229480", release_date="2024-10-08", + modalities=["image", "text"], ) diff --git a/mteb/models/voyage_v.py b/mteb/models/voyage_v.py index ceaf9bc6ae..7fbf20e350 100644 --- a/mteb/models/voyage_v.py +++ b/mteb/models/voyage_v.py @@ -241,7 +241,7 @@ def get_fused_embeddings( return VoyageMultiModalModelWrapper(**kwargs) -cohere_mult_3 = ModelMeta( +voyage_v = ModelMeta( loader=partial(voyage_v_loader, model_name="voyage-multimodal-3"), name="voyage-multimodal-3", languages=[], # Unknown @@ -255,8 +255,9 @@ def get_fused_embeddings( license=None, similarity_fn_name="cosine", framework=[], + modalities=["image", "text"], ) if __name__ == "__main__": - mdl = mteb.get_model(cohere_mult_3.name, cohere_mult_3.revision) + mdl = mteb.get_model(voyage_v.name, voyage_v.revision) emb = mdl.encode(["Hello, world!"]) From f994dc6e2a9faedd022bf619b33ac32583d819f0 Mon Sep 17 00:00:00 2001 From: Xin Zhang Date: Mon, 30 Dec 2024 01:54:33 +0800 Subject: [PATCH 126/154] [MIEB] Fix VISTA model (#1638) Fix vista --- mteb/models/vista_models.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py index 3adb7607b1..ee4a5ca0ad 100644 --- a/mteb/models/vista_models.py +++ b/mteb/models/vista_models.py @@ -17,10 +17,10 @@ def vista_loader(**kwargs): try: # a temporal fix for the dependency issues of vista models. - from FlagEmbedding.visual.modeling import Visualized_BGE + from visual_bge.modeling import Visualized_BGE except ImportError: raise ImportError( - "Please install `pip install FlagEmbedding` to use VisualizedBGE models." + "Please install `visual_bge`, refer to https://github.com/FlagOpen/FlagEmbedding/tree/master/research/visual_bge#install-flagembedding." ) class VisualizedBGEWrapper(Visualized_BGE): @@ -33,6 +33,7 @@ def __init__( negatives_cross_device: bool = False, temperature: float = 0.02, from_pretrained=None, + image_tokens_num: int = None, **kwargs: Any, ): super().__init__( @@ -44,6 +45,10 @@ def __init__( temperature=temperature, from_pretrained=from_pretrained, ) + self.image_tokens_num = image_tokens_num + self.max_text_len_with_image = ( + self.tokenizer.model_max_length - image_tokens_num + ) self.eval() def encode_text(self, texts): @@ -120,13 +125,21 @@ def encode( ] images = torch.stack(images) if texts is not None: - texts = self.tokenizer(texts, return_tensors="pt", padding=True) + texts = self.tokenizer( + texts, + return_tensors="pt", + padding=True, + truncation=True, + max_length=self.max_text_len_with_image, + ) return self.encode_mm(images.to(self.device), texts.to(self.device)) else: return self.encode_image(images.to(self.device)) else: if texts is not None: - texts = self.tokenizer(texts, return_tensors="pt", padding=True) + texts = self.tokenizer( + texts, return_tensors="pt", padding=True, truncation=True + ) return self.encode_text(texts.to(self.device)) else: return None @@ -224,6 +237,7 @@ def calculate_probs(self, text_embeddings, image_embeddings): vista_loader, model_name_bge="BAAI/bge-base-en-v1.5", model_weight="visualized_base_en_V1.5.pth", + image_tokens_num=196, ), name="BAAI/bge-visualized-base", languages=["eng_Latn"], @@ -238,6 +252,7 @@ def calculate_probs(self, text_embeddings, image_embeddings): vista_loader, model_name_bge="BAAI/bge-m3", model_weight="visualized_m3.pth", + image_tokens_num=256, ), name="BAAI/bge-visualized-m3", languages=["eng_Latn"], From e83acbb29dd8035b54c30d6113ba8260f133427a Mon Sep 17 00:00:00 2001 From: Niklas Muennighoff Date: Sun, 29 Dec 2024 11:09:24 -0800 Subject: [PATCH 127/154] Warn (#1639) --- mteb/evaluation/MTEB.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py index 8c64202d1e..09cead8e66 100644 --- a/mteb/evaluation/MTEB.py +++ b/mteb/evaluation/MTEB.py @@ -375,8 +375,8 @@ def run( # skip evaluation if the model does not support the task modalities. task_modalities = "".join(sorted(task.metadata.modalities)) if "".join(sorted(meta.modalities)) != task_modalities: - logger.info( - f"{meta.name} only supports {meta.modalities}, but the task modalities are {task.metadata.modalities}." + logger.warning( + f"{meta.name} only supports {meta.modalities}, but the task modalities are {task.metadata.modalities}. Skipping task." ) del self.tasks[0] # empty memory continue From ee26ebe5dd0677fcd6f0f018aa94542de8f211b9 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Mon, 30 Dec 2024 04:31:09 +0800 Subject: [PATCH 128/154] [mieb] model task modalities matching logic (#1640) fixing task & model modalities matching logic --- mteb/evaluation/MTEB.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py index 09cead8e66..c58d3d38de 100644 --- a/mteb/evaluation/MTEB.py +++ b/mteb/evaluation/MTEB.py @@ -374,7 +374,9 @@ def run( # skip evaluation if the model does not support the task modalities. task_modalities = "".join(sorted(task.metadata.modalities)) - if "".join(sorted(meta.modalities)) != task_modalities: + if ("".join(sorted(meta.modalities)) != task_modalities) and ( + not set(task.metadata.modalities).issubset(set(meta.modalities)) + ): logger.warning( f"{meta.name} only supports {meta.modalities}, but the task modalities are {task.metadata.modalities}. Skipping task." ) From 96e6106e895ba1fa260ca4746421626ba9896ca4 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 1 Jan 2025 15:25:17 +0200 Subject: [PATCH 129/154] [mieb] Use mock abstask classes (#1648) * rename to downsampled_dataset_transform * add mock tasks for mieb * wip getting to 57% * make lint * update mock classes to improve coverage * omit mock tasks from some tests --- .../Image/ClassificationEvaluator.py | 1 + tests/test_benchmark/mock_models.py | 2 +- tests/test_benchmark/mock_tasks.py | 905 ++++++++++++++++++ tests/test_benchmark/task_grid.py | 96 +- tests/test_tasks/test_all_abstasks.py | 13 +- tests/test_tasks/test_mieb_datasets.py | 4 +- 6 files changed, 960 insertions(+), 61 deletions(-) diff --git a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py index 2f60b5c690..a0d84d5714 100644 --- a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py @@ -216,6 +216,7 @@ def __call__(self, model: Encoder, test_cache=None): y_pred = torch.mode( y_train[neigh_indices], dim=1 ).values # TODO: case where there is no majority + y_pred = y_pred.tolist() accuracy = accuracy_score(self.y_test, y_pred) f1 = f1_score(self.y_test, y_pred, average="macro") scores["accuracy_" + metric] = accuracy diff --git a/tests/test_benchmark/mock_models.py b/tests/test_benchmark/mock_models.py index 1cd7c050a5..ad401cf3dc 100644 --- a/tests/test_benchmark/mock_models.py +++ b/tests/test_benchmark/mock_models.py @@ -60,7 +60,7 @@ def get_image_embeddings(self, images, **kwargs): return torch.randn(len(images), 10) def get_fused_embeddings(self, texts, images, **kwargs): - return torch.randn(len(images), 10) + return torch.randn(len(texts), 10) def calculate_probs(self, text_embeddings, image_embeddings): return torch.randn(image_embeddings.shape[0], text_embeddings.shape[0]) diff --git a/tests/test_benchmark/mock_tasks.py b/tests/test_benchmark/mock_tasks.py index 60c1a205a0..bbd07b4019 100644 --- a/tests/test_benchmark/mock_tasks.py +++ b/tests/test_benchmark/mock_tasks.py @@ -2,7 +2,9 @@ from __future__ import annotations +import numpy as np from datasets import Dataset, DatasetDict +from PIL import Image from mteb.abstasks import MultilingualTask from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining @@ -18,6 +20,23 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.AbsTaskSTS import AbsTaskSTS from mteb.abstasks.AbsTaskSummarization import AbsTaskSummarization +from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.Image.AbsTaskAny2TextMultipleChoice import ( + AbsTaskAny2TextMultipleChoice, +) +from mteb.abstasks.Image.AbsTaskImageClassification import AbsTaskImageClassification +from mteb.abstasks.Image.AbsTaskImageClustering import AbsTaskImageClustering +from mteb.abstasks.Image.AbsTaskImageMultilabelClassification import ( # noqa + AbsTaskImageMultilabelClassification, +) +from mteb.abstasks.Image.AbsTaskImageTextPairClassification import ( + AbsTaskImageTextPairClassification, +) +from mteb.abstasks.Image.AbsTaskVisualSTS import AbsTaskVisualSTS +from mteb.abstasks.Image.AbsTaskZeroshotClassification import ( + AbsTaskZeroshotClassification, +) from mteb.abstasks.TaskMetadata import TaskMetadata general_args = { @@ -1399,3 +1418,889 @@ def load_data(self, **kwargs): "fra": short_instructions, } self.data_loaded = True + + +class MockMultiChoiceTask(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + type="Any2AnyMultiChoice", + name="MockMultiChoice", + main_score="accuracy", + descriptive_stats={ + "test": { + "num_samples": 2, + "average_question_length": 26.0, + "average_choice_length": 30.5, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + } + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image", "text"] + metadata.category = "it2i" + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + + self.corpus = { + "test": Dataset.from_dict( + { + "id": ["d1", "d2"], + "image": [images[i] for i in range(2)], + "modality": ["image" for _ in range(2)], + } + ) + } + + self.queries = { + "test": Dataset.from_dict( + { + "id": [f"q{i}" for i in range(2)], + "image": [images[i] for i in range(2)], + "text": [ + "This is a positive sentence", + "This is another positive sentence", + ], + "modality": ["image,text" for _ in range(2)], + } + ) + } + + self.relevant_docs = { + "test": { + "q0": {"d1": 1, "d2": 0}, + "q1": {"d1": 0, "d2": 1}, + }, + } + self.data_loaded = True + + +class MockMultilingualMultiChoiceTask(AbsTaskAny2AnyMultiChoice, MultilingualTask): + metadata = TaskMetadata( + type="Any2AnyMultiChoice", + name="MockMultilingualMultiChoice", + main_score="accuracy", + descriptive_stats={ + "test": { + "num_samples": 4, + "average_question_length": 26.0, + "average_choice_length": 30.5, + "unique_labels": 2, + "labels": {"1": {"count": 2}, "0": {"count": 2}}, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 2, + "average_question_length": 26.0, + "average_choice_length": 30.5, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, + "fra": { + "num_samples": 2, + "average_question_length": 26.0, + "average_choice_length": 30.5, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, + }, + } + }, + **general_args, # type: ignore + ) + metadata.eval_langs = multilingual_eval_langs + metadata.modalities = ["image", "text"] + metadata.category = "it2i" + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + + corpus = { + "test": Dataset.from_dict( + { + "id": ["d1", "d2"], + "image": [images[i] for i in range(2)], + "modality": ["image" for _ in range(2)], + } + ) + } + self.corpus = { + "eng": corpus, + "fra": corpus, + } + + queries = { + "test": Dataset.from_dict( + { + "id": [f"q{i}" for i in range(2)], + "image": [images[i] for i in range(2)], + "text": [ + "This is a positive sentence", + "This is another positive sentence", + ], + "modality": ["image,text" for _ in range(2)], + } + ) + } + self.queries = { + "eng": queries, + "fra": queries, + } + + relevant_docs = { + "test": { + "q0": {"d1": 1, "d2": 0}, + "q1": {"d1": 0, "d2": 1}, + }, + } + self.relevant_docs = { + "eng": relevant_docs, + "fra": relevant_docs, + } + + self.data_loaded = True + + +class MockAny2AnyRetrievalI2TTask(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + type="Any2AnyRetrieval", + name="MockAny2AnyRetrievalI2T", + main_score="ndcg_at_10", + descriptive_stats={ + "test": { + "average_document_length": 30.0, + "average_query_length": 26.0, + "num_documents": 2, + "num_queries": 2, + "average_relevant_docs_per_query": 1.0, + } + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image", "text"] + metadata.category = "i2t" + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + + self.queries = { + "test": Dataset.from_dict( + { + "id": [f"q{i}" for i in range(2)], + "image": [images[i] for i in range(2)], + "modality": ["image" for _ in range(2)], + } + ) + } + self.corpus = { + "test": Dataset.from_dict( + { + "id": ["d1", "d2"], + "text": [ + "This is a positive sentence", + "This is another positive sentence", + ], + "modality": ["text" for _ in range(2)], + } + ) + } + + self.relevant_docs = { + "test": { + "q0": {"d1": 1, "d2": 0}, + "q1": {"d1": 0, "d2": 1}, + }, + } + self.data_loaded = True + + +class MockAny2AnyRetrievalT2ITask(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + type="Any2AnyRetrieval", + name="MockAny2AnyRetrievalT2I", + main_score="ndcg_at_10", + descriptive_stats={ + "test": { + "average_document_length": 30.0, + "average_query_length": 26.0, + "num_documents": 2, + "num_queries": 2, + "average_relevant_docs_per_query": 1.0, + } + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image", "text"] + metadata.category = "t2i" + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + + self.queries = { + "test": Dataset.from_dict( + { + "id": [f"q{i}" for i in range(2)], + "text": [ + "This is a positive sentence", + "This is another positive sentence", + ], + "modality": ["text" for _ in range(2)], + } + ) + } + self.corpus = { + "test": Dataset.from_dict( + { + "id": ["d1", "d2"], + "image": [images[i] for i in range(2)], + "modality": ["image" for _ in range(2)], + } + ) + } + + self.relevant_docs = { + "test": { + "q0": {"d1": 1, "d2": 0}, + "q1": {"d1": 0, "d2": 1}, + }, + } + self.data_loaded = True + + +class MockTextMultipleChoiceTask(AbsTaskAny2TextMultipleChoice): + metadata = TaskMetadata( + type="Any2TextMutipleChoice", + name="MockTextMultipleChoice", + main_score="accuracy", + descriptive_stats={ + "test": { + # TODO: Add descriptive stats + } + }, + **general_args, # type: ignore + ) + metadata.modalities = ["text", "image"] + metadata.category = "it2i" + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + + self.dataset = DatasetDict( + { + "test": Dataset.from_dict( + { + "id": [f"q{i}" for i in range(2)], + "image": [images[i] for i in range(2)], + "question": [ + "This is a positive sentence", + "This is another positive sentence", + ], + "choices": [["3", "2", "1", "0"], ["3", "2", "1", "0"]], + "answer": ["1", "0"], + } + ) + } + ) + + +class MockImageClassificationTask(AbsTaskImageClassification): + metadata = TaskMetadata( + type="ImageClassification", + name="MockImageClassification", + main_score="accuracy", + descriptive_stats={ + "test": { + "num_samples": 2, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, + "train": { + "num_samples": 10, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 5}, "0": {"count": 5}}, + }, + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image"] + metadata.category = "i2i" + + def __init__(self, **kwargs): + super().__init__(n_experiments=1, samples_per_label=5, **kwargs) + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + labels = [1, 0] + + self.dataset = DatasetDict( + { + "test": Dataset.from_dict( + { + "image": images, + "label": labels, + } + ), + "train": Dataset.from_dict( + { + "image": images * 5, + "label": labels * 5, + } + ), + } + ) + self.data_loaded = True + + +class MockImageClassificationKNNPTTask(AbsTaskImageClassification): + metadata = TaskMetadata( + type="ImageClassification", + name="MockImageClassificationKNNPT", + main_score="accuracy", + descriptive_stats={ + "test": { + "num_samples": 2, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, + "train": { + "num_samples": 10, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 5}, "0": {"count": 5}}, + }, + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image"] + metadata.category = "i2i" + + def __init__(self, **kwargs): + super().__init__( + method="kNN-pytorch", n_experiments=1, samples_per_label=5, **kwargs + ) + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + labels = [1, 0] + + self.dataset = DatasetDict( + { + "test": Dataset.from_dict( + { + "image": images, + "label": labels, + } + ), + "train": Dataset.from_dict( + { + "image": images * 5, + "label": labels * 5, + } + ), + } + ) + self.data_loaded = True + + +class MockImageClassificationKNNTask(AbsTaskImageClassification): + metadata = TaskMetadata( + type="ImageClassification", + name="MockImageClassificationKNN", + main_score="accuracy", + descriptive_stats={ + "test": { + "num_samples": 2, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, + "train": { + "num_samples": 10, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 5}, "0": {"count": 5}}, + }, + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image"] + metadata.category = "i2i" + + def __init__(self, **kwargs): + super().__init__(method="kNN", n_experiments=1, samples_per_label=5, **kwargs) + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + labels = [1, 0] + + self.dataset = DatasetDict( + { + "test": Dataset.from_dict( + { + "image": images, + "label": labels, + } + ), + "train": Dataset.from_dict( + { + "image": images * 5, + "label": labels * 5, + } + ), + } + ) + self.data_loaded = True + + +class MockMultilingualImageClassificationTask( + AbsTaskImageClassification, MultilingualTask +): + n_experiments = 1 + samples_per_label = 5 + metadata = TaskMetadata( + type="ImageClassification", + name="MockMultilingualImageClassification", + main_score="accuracy", + descriptive_stats={ + "test": { + "num_samples": 4, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 2}, "0": {"count": 2}}, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 2, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, + "fra": { + "num_samples": 2, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, + }, + }, + "train": { + "num_samples": 20, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 10}, "0": {"count": 10}}, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 10, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 5}, "0": {"count": 5}}, + }, + "fra": { + "num_samples": 10, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 5}, "0": {"count": 5}}, + }, + }, + }, + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image"] + metadata.category = "i2i" + metadata.eval_langs = multilingual_eval_langs + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + labels = [1, 0] + data = { + "test": Dataset.from_dict( + { + "image": images, + "label": labels, + } + ), + "train": Dataset.from_dict( + { + "image": images * 5, + "label": labels * 5, + } + ), + } + + self.dataset = DatasetDict( + { + "eng": data, + "fra": data, + } + ) + self.data_loaded = True + + +class MockImageClusteringTask(AbsTaskImageClustering): + metadata = TaskMetadata( + type="ImageClustering", + name="MockImageClustering", + main_score="nmi", + descriptive_stats={ + "test": { + "num_samples": 2, + "average_image_size": 26.0, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + } + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image"] + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + labels = [1, 0] + + self.dataset = DatasetDict( + { + "test": Dataset.from_dict( + { + "image": images, + "label": labels, + } + ), + } + ) + self.data_loaded = True + + +class MockImageMultilabelClassificationTask(AbsTaskImageMultilabelClassification): + metadata = TaskMetadata( + type="ImageMultilabelClassification", + name="MockImageMultilabelClassification", + main_score="accuracy", + descriptive_stats={ + "test": { + "average_image_size": 26.0, + "average_label_per_image": 2.0, + "num_samples": 6, + "unique_labels": 2, + "labels": {"0": {"count": 6}, "1": {"count": 6}}, + } + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image"] + metadata.category = "i2i" + n_experiments = 1 + samples_per_label = 3 + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + labels = [["0", "3"], ["1", "2"]] + + self.dataset = DatasetDict( + { + "test": Dataset.from_dict( + { + "image": images * 2, + "labels": labels * 2, + } + ), + "train": Dataset.from_dict( + { + "image": images * 5, + "labels": labels * 5, + } + ), + } + ) + self.data_loaded = True + + +class MockMultilingualImageMultilabelClassificationTask( + AbsTaskImageMultilabelClassification, MultilingualTask +): + metadata = TaskMetadata( + type="ImageMultilabelClassification", + name="MockMultilingualImageMultilabelClassification", + main_score="accuracy", + descriptive_stats={ + "test": { + "average_image_size": 26.0, + "average_label_per_image": 2.0, + "num_samples": 12, + "unique_labels": 2, + "labels": {"0": {"count": 12}, "1": {"count": 12}}, + "hf_subset_descriptive_stats": { + "eng": { + "average_image_size": 26.0, + "average_label_per_image": 2.0, + "num_samples": 6, + "unique_labels": 2, + "labels": {"0": {"count": 6}, "1": {"count": 6}}, + }, + "fra": { + "average_image_size": 26.0, + "average_label_per_image": 2.0, + "num_samples": 6, + "unique_labels": 2, + "labels": {"0": {"count": 6}, "1": {"count": 6}}, + }, + }, + } + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image"] + metadata.eval_langs = multilingual_eval_langs + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + labels = [["0", "3"], ["1", "2"]] + + data = { + "test": Dataset.from_dict( + { + "image": images * 2, + "labels": labels * 2, + } + ), + "train": Dataset.from_dict( + { + "image": images * 5, + "labels": labels * 5, + } + ), + } + + self.dataset = DatasetDict( + { + "eng": data, + "fra": data, + } + ) + self.data_loaded = True + + +class MockImageTextPairClassificationTask(AbsTaskImageTextPairClassification): + metadata = TaskMetadata( + type="ImageTextPairClassification", + name="MockImageTextPairClassification", + main_score="text_acc", + descriptive_stats={ + "test": { + "average_image_size": 26.0, + "average_text_length": 30.0, + "num_samples": 2, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + } + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image", "text"] + metadata.category = "i2t" + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + [Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images] + ] + texts = [["This is a test sentence", "This is another test sentence"]] + + self.dataset = DatasetDict( + { + "test": Dataset.from_dict( + { + "image": images, + "caption": texts, + } + ), + } + ) + self.data_loaded = True + + +class MockMultilingualImageTextPairClassificationTask( + AbsTaskImageTextPairClassification, MultilingualTask +): + metadata = TaskMetadata( + type="ImageTextPairClassification", + name="MockMultilingualImageTextPairClassification", + main_score="accuracy", + descriptive_stats={ + "test": { + "average_image_size": 26.0, + "average_text_length": 30.0, + "num_samples": 4, + "unique_labels": 2, + "labels": {"1": {"count": 2}, "0": {"count": 2}}, + "hf_subset_descriptive_stats": { + "eng": { + "average_image_size": 26.0, + "average_text_length": 30.0, + "num_samples": 2, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, + "fra": { + "average_image_size": 26.0, + "average_text_length": 30.0, + "num_samples": 2, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, + }, + } + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image", "text"] + metadata.category = "i2t" + + metadata.eval_langs = multilingual_eval_langs + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + [Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images] + ] + texts = [["This is a test sentence", "This is another test sentence"]] + data = { + "test": Dataset.from_dict( + { + "image": images, + "caption": texts, + } + ), + } + + self.dataset = DatasetDict( + { + "eng": data, + "fra": data, + } + ) + self.data_loaded = True + + +class MockVisualSTSTask(AbsTaskVisualSTS): + metadata = TaskMetadata( + type="VisualSTS", + name="MockVisualSTS", + main_score="cosine_spearman", + descriptive_stats={ + "test": { + "average_image_size": 26.0, + "average_text_length": 30.0, + "num_samples": 2, + "average_score": 0.5, + } + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image", "text"] + metadata.category = "i2i" + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + scores = [0.5, 0.5] + + self.dataset = DatasetDict( + { + "test": Dataset.from_dict( + { + "sentence1": images, + "sentence2": images, + "score": scores, + } + ), + } + ) + self.data_loaded = True + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 0 + metadata_dict["max_score"] = 5 + return metadata_dict + + +class MockZeroshotClassificationTask(AbsTaskZeroshotClassification): + metadata = TaskMetadata( + type="ZeroShotClassification", + name="MockZeroshotClassification", + main_score="accuracy", + descriptive_stats={ + "test": { + "average_text_length": 26.0, + "num_samples": 2, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + } + }, + **general_args, # type: ignore + ) + metadata.modalities = ["image", "text"] + metadata.category = "i2t" + + def load_data(self, **kwargs): + images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] + images = [ + Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images + ] + labels = ["label1", "label2"] + + self.dataset = DatasetDict( + { + "test": Dataset.from_dict( + { + "image": images, + "label": labels, + } + ), + } + ) + self.data_loaded = True + + def get_candidate_labels(self) -> list[str]: + return ["This is a test sentence", "This is another test sentence"] diff --git a/tests/test_benchmark/task_grid.py b/tests/test_benchmark/task_grid.py index afe9ff2218..77d8f4c8e7 100644 --- a/tests/test_benchmark/task_grid.py +++ b/tests/test_benchmark/task_grid.py @@ -10,28 +10,32 @@ from mteb.tasks.Clustering.eng.TwentyNewsgroupsClustering import ( TwentyNewsgroupsClusteringFast, ) -from mteb.tasks.Image.Any2AnyMultiChoice import ROxfordEasyI2IMultiChoice -from mteb.tasks.Image.Any2AnyRetrieval import Flickr30kI2TRetrieval -from mteb.tasks.Image.Any2TextMultipleChoice import CVBenchCount -from mteb.tasks.Image.Clustering import TinyImageNet -from mteb.tasks.Image.ImageClassification import OxfordPetsClassification -from mteb.tasks.Image.ImageMultilabelClassification import VOC2007Classification -from mteb.tasks.Image.ImageTextPairClassification import AROFlickrOrder -from mteb.tasks.Image.VisualSTS import STS16VisualSTS -from mteb.tasks.Image.ZeroshotClassification import RenderedSST2 from .mock_tasks import ( + MockAny2AnyRetrievalI2TTask, + MockAny2AnyRetrievalT2ITask, MockBitextMiningTask, MockClassificationTask, MockClusteringFastTask, MockClusteringTask, + MockImageClassificationKNNPTTask, + MockImageClassificationKNNTask, + MockImageClassificationTask, + MockImageClusteringTask, + MockImageMultilabelClassificationTask, + MockImageTextPairClassificationTask, MockInstructionRetrival, + MockMultiChoiceTask, MockMultilabelClassification, MockMultilingualBitextMiningTask, MockMultilingualClassificationTask, MockMultilingualClusteringFastTask, MockMultilingualClusteringTask, + MockMultilingualImageClassificationTask, + MockMultilingualImageMultilabelClassificationTask, + MockMultilingualImageTextPairClassificationTask, MockMultilingualInstructionRetrival, + MockMultilingualMultiChoiceTask, MockMultilingualMultilabelClassification, MockMultilingualPairClassificationTask, MockMultilingualParallelBitextMiningTask, @@ -44,6 +48,9 @@ MockRetrievalTask, MockSTSTask, MockSummarizationTask, + MockTextMultipleChoiceTask, + MockVisualSTSTask, + MockZeroshotClassificationTask, ) twenty_news = TwentyNewsgroupsClusteringFast() @@ -75,50 +82,6 @@ ] -def dataset_transform(self): - for split in self.metadata.eval_splits: - self.dataset[split] = self.dataset[split].select([0, 1]) - - -tiny_imagenet = TinyImageNet() -renderedSST2 = RenderedSST2() -aro = AROFlickrOrder() -oxford_pets = OxfordPetsClassification() -voc2007 = VOC2007Classification() -flickr = Flickr30kI2TRetrieval() -roxford_mc = ROxfordEasyI2IMultiChoice() -cvbench_count = CVBenchCount() -sts16 = STS16VisualSTS() - -## method override to speed up tests -tiny_imagenet.dataset_transform = dataset_transform.__get__(tiny_imagenet) -renderedSST2.dataset_transform = dataset_transform.__get__(renderedSST2) -aro.dataset_transform = dataset_transform.__get__(aro) -oxford_pets.dataset_transform = dataset_transform.__get__(oxford_pets) -voc2007.dataset_transform = dataset_transform.__get__(voc2007) -flickr.dataset_transform = dataset_transform.__get__(flickr) -roxford_mc.dataset_transform = dataset_transform.__get__(roxford_mc) -cvbench_count.dataset_transform = dataset_transform.__get__(cvbench_count) -sts16.dataset_transform = dataset_transform.__get__(sts16) - - -MIEB_TASK_TEST_GRID = [ - tiny_imagenet, # image clustering - aro, # pair classification - renderedSST2, # zero shot classification - oxford_pets, # image classification - voc2007, # multilabel classification - flickr, # I2T retrieval - roxford_mc, # Any2Any MultiChoice - cvbench_count, # Any2Any Text MultiChoice - sts16, # visual sts -] - -MIEB_TASK_TEST_GRID_AS_STRING = [ - t.metadata.name if isinstance(t, AbsTask) else t for t in MIEB_TASK_TEST_GRID -] - - # Mock tasks for testing - intended to be faster and avoid downloading data leading to false positive potential failures in CI # Not all tasks are implemented as Mock tasks yet MOCK_TASK_TEST_GRID = [ @@ -152,3 +115,30 @@ def dataset_transform(self): ] MOCK_TASK_REGISTRY = {task.metadata.name: type(task) for task in MOCK_TASK_TEST_GRID} + +MOCK_MIEB_TASK_GRID = [ + MockAny2AnyRetrievalI2TTask(), + MockAny2AnyRetrievalT2ITask(), + MockTextMultipleChoiceTask(), + MockMultiChoiceTask(), + MockImageClassificationTask(), + MockImageClassificationKNNPTTask(), + MockImageClassificationKNNTask(), + MockImageClusteringTask(), + MockImageTextPairClassificationTask(), + MockVisualSTSTask(), + MockZeroshotClassificationTask(), + MockImageMultilabelClassificationTask(), + MockMultilingualImageClassificationTask(), + MockMultilingualImageTextPairClassificationTask(), + MockMultilingualMultiChoiceTask(), + MockMultilingualImageMultilabelClassificationTask(), +] + +MOCK_MIEB_TASK_GRID_AS_STRING = [ + t.metadata.name if isinstance(t, AbsTask) else t for t in MOCK_MIEB_TASK_GRID +] + +MOCK_MIEB_TASK_REGISTRY = { + task.metadata.name: type(task) for task in MOCK_MIEB_TASK_GRID +} diff --git a/tests/test_tasks/test_all_abstasks.py b/tests/test_tasks/test_all_abstasks.py index d4c8e44a88..dda3634231 100644 --- a/tests/test_tasks/test_all_abstasks.py +++ b/tests/test_tasks/test_all_abstasks.py @@ -18,13 +18,16 @@ from mteb.abstasks.MultiSubsetLoader import MultiSubsetLoader from mteb.overview import TASKS_REGISTRY -from ..test_benchmark.task_grid import MOCK_TASK_TEST_GRID_AS_STRING +from ..test_benchmark.task_grid import ( + MOCK_MIEB_TASK_GRID_AS_STRING, + MOCK_TASK_TEST_GRID_AS_STRING, +) logging.basicConfig(level=logging.INFO) -tasks = [ - t for t in MTEB().tasks_cls if t.metadata.name not in MOCK_TASK_TEST_GRID_AS_STRING -] +ALL_MOCK_TASKS = MOCK_TASK_TEST_GRID_AS_STRING + MOCK_MIEB_TASK_GRID_AS_STRING + +tasks = [t for t in MTEB().tasks_cls if t.metadata.name not in ALL_MOCK_TASKS] @pytest.mark.parametrize("task", tasks) @@ -90,7 +93,7 @@ async def check_datasets_are_available_on_hf(tasks): def test_dataset_availability(): """Checks if the datasets are available on Hugging Face using both their name and revision.""" tasks = MTEB().tasks_cls - tasks = [t for t in tasks if t.metadata.name not in MOCK_TASK_TEST_GRID_AS_STRING] + tasks = [t for t in tasks if t.metadata.name not in ALL_MOCK_TASKS] asyncio.run(check_datasets_are_available_on_hf(tasks)) diff --git a/tests/test_tasks/test_mieb_datasets.py b/tests/test_tasks/test_mieb_datasets.py index c2122945d6..26e60931ec 100644 --- a/tests/test_tasks/test_mieb_datasets.py +++ b/tests/test_tasks/test_mieb_datasets.py @@ -11,12 +11,12 @@ from mteb.abstasks import AbsTask from ..test_benchmark.mock_models import MockCLIPEncoder -from ..test_benchmark.task_grid import MIEB_TASK_TEST_GRID +from ..test_benchmark.task_grid import MOCK_MIEB_TASK_GRID logging.basicConfig(level=logging.INFO) -@pytest.mark.parametrize("task", MIEB_TASK_TEST_GRID) +@pytest.mark.parametrize("task", MOCK_MIEB_TASK_GRID) @pytest.mark.parametrize("model", [MockCLIPEncoder()]) def test_benchmark_sentence_transformer(task: str | AbsTask, model: mteb.Encoder): """Test that a task can be fetched and run""" From 075873f9c954d29aac927782bd30219c5dbd84ce Mon Sep 17 00:00:00 2001 From: Xin Zhang Date: Thu, 2 Jan 2025 00:34:03 +0800 Subject: [PATCH 130/154] [MIEB] Add code for GME models (#1635) * Add GME * Fix infoseek prompts * Merge instructions --- mteb/models/gme_models.py | 449 ++++++++++++++++++++++++++++++++++++ mteb/models/instructions.py | 117 ++++++++++ mteb/models/overview.py | 2 + 3 files changed, 568 insertions(+) create mode 100644 mteb/models/gme_models.py diff --git a/mteb/models/gme_models.py b/mteb/models/gme_models.py new file mode 100644 index 0000000000..b04dbf68d0 --- /dev/null +++ b/mteb/models/gme_models.py @@ -0,0 +1,449 @@ +from __future__ import annotations + +import logging +import math +import os +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm.autonotebook import tqdm +from transformers import AutoModelForVision2Seq, AutoProcessor + +import mteb +from mteb.encoder_interface import PromptType +from mteb.model_meta import ModelMeta + +from .instructions import DEFAULT_PROMPTS, TASKNAME2INSTRUCTIONS + +logging.basicConfig(level=logging.WARNING) +logger = logging.getLogger(__name__) + +HF_GME_QWEN2VL_2B = "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct" +HF_GME_QWEN2VL_7B = "Alibaba-NLP/gme-Qwen2-VL-7B-Instruct" + + +def get_gme_instruction(task_name: str, is_query: bool = True) -> str: + # TODO Prompts for other multimodal tasks. + if task_name in TASKNAME2INSTRUCTIONS: + prompt = TASKNAME2INSTRUCTIONS[task_name] + if isinstance(prompt, tuple): + prompt = prompt[0] if is_query else prompt[1] + else: + meta = mteb.get_task(task_name).metadata + prompt = DEFAULT_PROMPTS.get(meta.type, None) + + if isinstance(prompt, str) and prompt[-1] != ".": + prompt += "." + return prompt + + +class Encoder(torch.nn.Module): + def __init__( + self, + base, + processor, + max_length=1800, + normalize=True, + ) -> None: + super().__init__() + self.base = base + self.processor = processor + self.max_length = max_length + self.normalize = normalize + self.processor.tokenizer.padding_side = "right" + self.defualt_instruction = "You are a helpful assistant." + + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: list[torch.FloatTensor] | None = None, + inputs_embeds: torch.FloatTensor | None = None, + pixel_values: torch.Tensor | None = None, + # pixel_values_videos: torch.FloatTensor | None = None, + image_grid_thw: torch.LongTensor | None = None, + # video_grid_thw: torch.LongTensor | None = None, + pooling_mask: torch.LongTensor | None = None, + **kwargs, + ) -> torch.Tensor: + if inputs_embeds is None: + inputs_embeds = self.base.model.embed_tokens(input_ids) + if pixel_values is not None: + pixel_values = pixel_values.type(self.base.visual.get_dtype()) + image_embeds = self.base.visual( + pixel_values, grid_thw=image_grid_thw + ).to(inputs_embeds.device) + image_mask = input_ids == self.base.config.image_token_id + inputs_embeds[image_mask] = image_embeds + # if pixel_values_videos is not None: + # pixel_values_videos = pixel_values_videos.type(self.base.visual.get_dtype()) + # video_embeds = self.base.visual(pixel_values_videos, grid_thw=video_grid_thw).to(inputs_embeds.device) + # video_mask = input_ids == self.base.config.video_token_id + # inputs_embeds[video_mask] = video_embeds + if attention_mask is not None: + attention_mask = attention_mask.to(inputs_embeds.device) + + outputs = self.base.model( + input_ids=None, + position_ids=position_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + ) + + pooling_mask = attention_mask if pooling_mask is None else pooling_mask + left_padding = pooling_mask[:, -1].sum() == pooling_mask.shape[0] # TODO + if left_padding: + embeddings = outputs.last_hidden_state[:, -1] + else: + sequence_lengths = pooling_mask.sum(dim=1) - 1 + batch_size = outputs.last_hidden_state.shape[0] + embeddings = outputs.last_hidden_state[ + torch.arange(batch_size, device=outputs.last_hidden_state.device), + sequence_lengths, + ] + if self.normalize: + embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1) + return embeddings.contiguous() + + def embed( + self, + texts: list[str], + images: list[Image.Image], + device, + instruction=None, + **kwargs, + ): + instruction = instruction or self.defualt_instruction + # Inputs must be batched + input_texts, input_images = [], [] + for t, i in zip(texts, images): + input_str = "" + if i is None: + input_images = None # All examples in the same batch are consistent + else: + input_str += "<|vision_start|><|image_pad|><|vision_end|>" + i = fetch_image(i) + input_images.append(i) + if t is not None: + input_str += t + msg = f"<|im_start|>system\n{instruction}<|im_end|>\n<|im_start|>user\n{input_str}<|im_end|>\n<|im_start|>assistant\n<|endoftext|>" + input_texts.append(msg) + + inputs = self.processor( + text=input_texts, + images=input_images, + padding=True, + truncation=True, + max_length=self.max_length, + return_tensors="pt", + ) + inputs = {k: v.to(device) for k, v in inputs.items()} # TODO + embeddings = self.forward(**inputs) + return embeddings + + +class GmeQwen2VL: + def __init__( + self, + model_name: str = HF_GME_QWEN2VL_2B, + model_path: str | None = None, + device: str = "cuda" if torch.cuda.is_available() else "cpu", + min_image_tokens=4, + max_image_tokens=1280, + max_length=1800, + **kwargs, + ) -> None: + model_name = model_path or model_name + base = AutoModelForVision2Seq.from_pretrained( + model_name, torch_dtype=torch.float16, **kwargs + ) + min_pixels = min_image_tokens * 28 * 28 + max_pixels = max_image_tokens * 28 * 28 + processor = AutoProcessor.from_pretrained( + model_name, min_pixels=min_pixels, max_pixels=max_pixels, **kwargs + ) + self.model = Encoder(base, processor, max_length=max_length) + self.model.eval() + self.device = device + self.sep = " " + + def encode( + self, + sentences: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + **kwargs: Any, + ): + return self.get_fused_embeddings( + texts=sentences, task_name=task_name, prompt_type=prompt_type, **kwargs + ) + + def encode_queries(self, queries: list[str], **kwargs): + kwargs.update(prompt_type=PromptType.query) + embeddings = self.encode(queries, **kwargs) + return embeddings + + def encode_corpus(self, corpus: list[dict[str, str]], **kwargs): + if type(corpus) is dict: + sentences = [ + (corpus["title"][i] + self.sep + corpus["text"][i]).strip() + if "title" in corpus + else corpus["text"][i].strip() + for i in range(len(corpus["text"])) + ] + else: + sentences = [ + (doc["title"] + self.sep + doc["text"]).strip() + if "title" in doc + else doc["text"].strip() + for doc in corpus + ] + kwargs.update(prompt_type=PromptType.passage) + embeddings = self.encode(sentences, is_query=False, **kwargs) + return embeddings + + def get_image_embeddings(self, images: list[Image.Image] | DataLoader, **kwargs): + return self.get_fused_embeddings(images=images, **kwargs) + + def get_text_embeddings(self, texts: list[str], **kwargs): + return self.get_fused_embeddings(texts=texts, **kwargs) + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] | None = None, + images: list[Image.Image] | DataLoader | None = None, + task_name: str | None = None, + prompt_type: PromptType | None = None, + tqdm_mininterval: int = 15, + instruction=None, + **kwargs: Any, + ): + if prompt_type == PromptType.passage: + instruction = None + elif instruction is None: + instruction = get_gme_instruction(task_name) + self.model = self.model.to(self.device) + + if isinstance(images, DataLoader): + image_loader = images + batch_size = image_loader.batch_size + image_loader.dataset.transform = None + else: + batch_size = kwargs.pop("batch_size", 32) + if images is None: + image_loader = None + else: + image_loader = DataLoader( + images, + batch_size=batch_size, + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=min(math.floor(os.cpu_count() / 2), 8), + ) + + if texts is None: + assert image_loader is not None + n_batch = len(image_loader) + else: + n_batch = len(texts) // batch_size + int(len(texts) % batch_size > 0) + image_loader = image_loader or [None] * n_batch + + all_embeddings = [] + none_batch = [None] * batch_size + show_progress_bar = kwargs.pop("show_progress_bar", True) + pbar = tqdm( + total=n_batch, + disable=not show_progress_bar, + mininterval=tqdm_mininterval, + miniters=n_batch // 10, + desc="encode", + ) + for n, (i, img_batch) in enumerate( + zip(range(0, n_batch * batch_size, batch_size), image_loader) + ): + text_batch = none_batch if texts is None else texts[i : i + batch_size] + img_batch = none_batch if img_batch is None else img_batch + inputs = dict( + texts=text_batch, images=img_batch, instruction=instruction, **kwargs + ) + with torch.inference_mode(): + embeddings = self.model.embed(**inputs, device=self.device) + all_embeddings.append(embeddings.cpu()) + pbar.update(1) + pbar.close() + all_embeddings = torch.cat(all_embeddings, dim=0) + return all_embeddings + + +def custom_collate_fn(batch): + return batch + + +### Copied from qwen_vl_utils.vision_process.py +IMAGE_FACTOR = 28 +MIN_PIXELS = 4 * 28 * 28 +MAX_PIXELS = 16384 * 28 * 28 +MAX_RATIO = 200 + + +def round_by_factor(number: int, factor: int) -> int: + """Returns the closest integer to 'number' that is divisible by 'factor'.""" + return round(number / factor) * factor + + +def ceil_by_factor(number: int, factor: int) -> int: + """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'.""" + return math.ceil(number / factor) * factor + + +def floor_by_factor(number: int, factor: int) -> int: + """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'.""" + return math.floor(number / factor) * factor + + +def smart_resize( + height: int, + width: int, + factor: int = IMAGE_FACTOR, + min_pixels: int = MIN_PIXELS, + max_pixels: int = MAX_PIXELS, +) -> tuple[int, int]: + """Rescales the image so that the following conditions are met: + + 1. Both dimensions (height and width) are divisible by 'factor'. + + 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. + + 3. The aspect ratio of the image is maintained as closely as possible. + """ + h_bar = max(factor, round_by_factor(height, factor)) + w_bar = max(factor, round_by_factor(width, factor)) + if h_bar * w_bar > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + h_bar = floor_by_factor(height / beta, factor) + w_bar = floor_by_factor(width / beta, factor) + elif h_bar * w_bar < min_pixels: + beta = math.sqrt(min_pixels / (height * width)) + h_bar = ceil_by_factor(height * beta, factor) + w_bar = ceil_by_factor(width * beta, factor) + + if max(h_bar, w_bar) / min(h_bar, w_bar) > MAX_RATIO: + logger.warning( + f"Absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(h_bar, w_bar) / min(h_bar, w_bar)}" + ) + if h_bar > w_bar: + h_bar = w_bar * MAX_RATIO + else: + w_bar = h_bar * MAX_RATIO + return h_bar, w_bar + + +def fetch_image( + image: str | Image.Image, size_factor: int = IMAGE_FACTOR +) -> Image.Image: + image_obj = None + if isinstance(image, Image.Image): + image_obj = image + elif image.startswith("http://") or image.startswith("https://"): + import requests + + image_obj = Image.open(requests.get(image, stream=True).raw) + elif image.startswith("file://"): + image_obj = Image.open(image[7:]) + elif image.startswith("data:image"): + import base64 + from io import BytesIO + + if "base64," in image: + _, base64_data = image.split("base64,", 1) + data = base64.b64decode(base64_data) + image_obj = Image.open(BytesIO(data)) + else: + image_obj = Image.open(image) + if image_obj is None: + raise ValueError( + f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}" + ) + image = image_obj.convert("RGB") + ## resize + # if "resized_height" in ele and "resized_width" in ele: + # resized_height, resized_width = smart_resize( + # ele["resized_height"], + # ele["resized_width"], + # factor=size_factor, + # ) + # else: + width, height = image.size + # min_pixels = ele.get("min_pixels", MIN_PIXELS) + # max_pixels = ele.get("max_pixels", MAX_PIXELS) + resized_height, resized_width = smart_resize( + height, + width, + factor=size_factor, + min_pixels=MIN_PIXELS, + max_pixels=MAX_PIXELS, + ) + image = image.resize((resized_width, resized_height)) + + return image + + +### + + +gme_qwen2vl_2b = ModelMeta( + loader=partial( + GmeQwen2VL, + model_name=HF_GME_QWEN2VL_2B, + ), + name=HF_GME_QWEN2VL_2B, + languages=["eng_Latn", "cmn-Hans"], + open_weights=True, + revision="ce765ae71b8cdb208203cd8fb64a170b1b84293a", + release_date="2024-12-24", + n_parameters=2_210_000_000, + memory_usage=None, + embed_dim=1536, + license="apache-2.0", + max_tokens=32768, + reference="https://huggingface.co/" + HF_GME_QWEN2VL_2B, + similarity_fn_name="cosine", + framework=["PyTorch"], + use_instuctions=True, +) + +gme_qwen2vl_7b = ModelMeta( + loader=partial( + GmeQwen2VL, + model_name=HF_GME_QWEN2VL_7B, + ), + name=HF_GME_QWEN2VL_7B, + languages=["eng_Latn", "cmn-Hans"], + open_weights=True, + revision="477027a6480f8630363be77751f169cc3434b673", + release_date="2024-12-24", + n_parameters=8_290_000_000, + memory_usage=None, + embed_dim=3584, + license="apache-2.0", + max_tokens=32768, + reference="https://huggingface.co/" + HF_GME_QWEN2VL_2B, + similarity_fn_name="cosine", + framework=["PyTorch"], + use_instuctions=True, +) diff --git a/mteb/models/instructions.py b/mteb/models/instructions.py index 4a31f8da02..ef439e42bb 100644 --- a/mteb/models/instructions.py +++ b/mteb/models/instructions.py @@ -296,6 +296,123 @@ "", ), "RiaNewsRetrieval": ("Given a news title, retrieve relevant news article", ""), + # Any2Any Retrieval + "WebQAT2TRetrieval": ( + "Retrieve passages from Wikipedia that provide answers to the following question.", + "", + ), + "NIGHTSI2IRetrieval": ( + "Find a day-to-day image that looks similar to the provided image.", + "", + ), + "VisualNewsT2IRetrieval": ( + "Identify the news-related image in line with the described event.", + "", + ), + "Fashion200kT2IRetrieval": ( + "Based on the following fashion description, retrieve the best matching image.", + "", + ), + "MSCOCOT2IRetrieval": ( + "Identify the image showcasing the described everyday scene.", + "", + ), + "Flickr30kT2IRetrieval": ("Find an image that matches the given caption.", ""), + "VidoreTatdqaRetrieval": ( + "Find a screenshot that relevant to the user's question.", + "", + ), + "VidoreArxivQARetrieval": ( + "Find a screenshot that relevant to the user's question.", + "", + ), + "VidoreDocVQARetrieval": ( + "Find a screenshot that relevant to the user's question.", + "", + ), + "VidoreInfoVQARetrieval": ( + "Find a screenshot that relevant to the user's question.", + "", + ), + "VidoreShiftProjectRetrieval": ( + "Find a screenshot that relevant to the user's question.", + "", + ), + "VidoreSyntheticDocQAAIRetrieval": ( + "Find a screenshot that relevant to the user's question.", + "", + ), + "VidoreSyntheticDocQAGovernmentReportsRetrieval": ( + "Find a screenshot that relevant to the user's question.", + "", + ), + "VidoreSyntheticDocQAHealthcareIndustryRetrieval": ( + "Find a screenshot that relevant to the user's question.", + "", + ), + "VidoreSyntheticDocQAEnergyRetrieval": ( + "Find a screenshot that relevant to the user's question.", + "", + ), + "VidoreTabfquadRetrieval": ( + "Find a screenshot that relevant to the user's question.", + "", + ), + "VisualNewsI2TRetrieval": ("Find a caption for the news in the given photo.", ""), + "Fashion200kI2TRetrieval": ( + "Based on the following fashion description, retrieve the best matching image.", + "", + ), + "MSCOCOI2TRetrieval": ( + "Find an image caption describing the following everyday image.", + "", + ), + "Flickr30kI2TRetrieval": ( + "Find an image caption describing the following image.", + "", + ), + "WebQAT2ITRetrieval": ("Find a Wikipedia image that answers this question.", ""), + "EDIST2ITRetrieval": ("Identify the news photo for the given caption.", ""), + "OVENIT2TRetrieval": ( + "Retrieve a Wikipedia paragraph that provides an answer to the given query about the image.", + "", + ), + "InfoSeekIT2TRetrieval": ( + "Find a paragraph from Wikipedia that answers my question about this image.", + "", + ), + "ReMuQIT2TRetrieval": ( + "Retrieve a fact-based paragraph that provides an answer to the given query about the image.", + "", + ), + "OKVQAIT2TRetrieval": ( + "Retrieve documents that provide an answer to the question alongside the image.", + "", + ), + "LLaVAIT2TRetrieval": ( + "Provide a specific decription of the image along with the following question.", + "", + ), + "FashionIQIT2IRetrieval": ( + "Find a fashion image that aligns with the reference image and style note.", + "", + ), + "CIRRIT2IRetrieval": ( + "Retrieve a day-to-day image that aligns with the modification instructions of the provided image.", + "", + ), + "OVENIT2ITRetrieval": ( + "Retrieve a Wikipedia image-description pair that provides evidence for the question of this image.", + "", + ), + "InfoSeekIT2ITRetrieval": ( + "Find an image and subject description from Wikipedia that answers my question about this image.", + "", + ), + "EncyclopediaVQAIT2ITRetrieval": ( + "Obtain illustrated documents that correspond to the inquiry alongside the provided image.", + "", + ), } diff --git a/mteb/models/overview.py b/mteb/models/overview.py index 340989baaf..8b94aafcd0 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -22,6 +22,7 @@ e5_models, e5_v, evaclip_models, + gme_models, google_models, gritlm_models, gte_models, @@ -63,6 +64,7 @@ e5_models, e5_v, evaclip_models, + gme_models, google_models, gritlm_models, gte_models, From 07befcbf34b9ea8e0a502d15d30f2f2f9edfc8ee Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 8 Jan 2025 23:36:19 +0200 Subject: [PATCH 131/154] fix: add version check e5-v in mieb (#1723) * add version check for e5v model * Update e5_v.py * make lint --- mteb/models/e5_v.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index ddb1f12551..fe8099a432 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -4,6 +4,8 @@ from typing import Any import torch +import transformers +from packaging import version from PIL import Image from torch.utils.data import DataLoader from tqdm import tqdm @@ -12,6 +14,10 @@ from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta +E5_V_TRANSFORMERS_VERSION = ( + "4.44.2" # Issue 1647: Only works with transformers==4.44.2. +) + class E5VWrapper: def __init__( @@ -20,6 +26,13 @@ def __init__( composed_prompt=None, **kwargs: Any, ): + if version.parse(transformers.__version__) != version.parse( + E5_V_TRANSFORMERS_VERSION + ): + raise ImportError( + f"This wrapper only works with transformers=={E5_V_TRANSFORMERS_VERSION}" + ) + self.model_name = model_name self.processor = LlavaNextProcessor.from_pretrained(model_name) if "device" in kwargs: From 38fe14e12765e96d13255c63c9e7295b9649797b Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Thu, 9 Jan 2025 20:40:18 +0200 Subject: [PATCH 132/154] fix: change comparison to bigger than (#1743) change comparison to bigger than --- mteb/models/e5_v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index fe8099a432..8b96427f61 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -26,7 +26,7 @@ def __init__( composed_prompt=None, **kwargs: Any, ): - if version.parse(transformers.__version__) != version.parse( + if version.parse(transformers.__version__) > version.parse( E5_V_TRANSFORMERS_VERSION ): raise ImportError( From b021b9bf21c1c7e28cb77876f0e4f08b961da313 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 15 Jan 2025 21:17:28 +0900 Subject: [PATCH 133/154] docs: Rework MIEB docs (#1802) * combine mieb docs and move to main docs folder * make flow more coherent * tidy up * skip AfriSentiLID for now #1785 * fix typo: exclude MIEB mock tests * update vista doc * Apply suggestions from code review --------- Co-authored-by: Isaac Chung --- docs/mieb-docs/README.md | 34 -------- docs/mieb-docs/run_vista.md | 27 ------ docs/mieb.md | 116 ++++++++++++++++++++++++++ tests/test_tasks/test_all_abstasks.py | 9 +- 4 files changed, 124 insertions(+), 62 deletions(-) delete mode 100644 docs/mieb-docs/README.md delete mode 100644 docs/mieb-docs/run_vista.md create mode 100644 docs/mieb.md diff --git a/docs/mieb-docs/README.md b/docs/mieb-docs/README.md deleted file mode 100644 index 4c7f388702..0000000000 --- a/docs/mieb-docs/README.md +++ /dev/null @@ -1,34 +0,0 @@ -To make a MIEB task `X` fully runable from scratch for dataset `Y` with model `Z`, we need to implemenet an `AbsTaskX` class for it, subclassing `AbsTask`; a task-specific `XEvaluator`, which will be called in `AbsTaskX`; a dataset-specific (e.g., Dataset `Y`) class `class Y(AbsTaskX)` subclassing the corresponding `AbsTaskX`, which is itself the subclass of `AbsTask`; and some model class `ZModelWrapper` that has needed functions. - -## Example - -Here is an example implementing zero-shot image classification from scratch. - -To solve this task, we basically need to encode the `images`, encode the `class label candidates with prompts` (things like "this is a dog pic", "this is a cat pic"), and similarity-compare them, to argmax out the class prediction for each image. - -#### ModelWrapper -Since we don't have an established class like `SentenceTransformer` or `DRES` anymore now, we first decide for this task so far, we need the model class to have `get_text_embeddings`, `get_image_embeddings`, and `calculate_probs`. As an example, [CLIPModelWrapper](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/models/clip_models.py) is first implemented, with MetaData defined. - -#### X Evaluator -With the model, [ZeroshotClassificationEvaluator](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py) is implemented here, basically the pipeline of using the defined models to do zero-shot classification. - -#### AbsTask X -With the evaluator, [AbsTaskZeroshotClassification](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/abstasks/Image/AbsTaskZeroshotClassification.py) is defined, operating on the dataset, calling the defined Evaluator, and gives out results. - -#### Dataset class -With all these, we can then define the dataset. Here I choose Rendered SST2 as an example, which is to classify SST2 movie reviews, with reviews rendered into images. [RenderedSST2](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/tasks/Image/ZeroshotClassification/eng/RenderedSST2.py) is implemented like this, subclassing `AbsTaskZeroshotClassification`, and overwrite the `get_candidate_labels` function, which gives `["a negative review of a movie", "a positive review of a movie"]` to be used in the evaluator. - -With all these, we can then -```python - -import mteb - -model_name = "openai/clip-vit-large-patch14" -model = mteb.get_model(model_name = model_name) - -tasks = mteb.get_tasks(tasks=["RenderedSST2"]) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder=f"results-mieb/{model_name}") -``` -And yeah, the results will be under [`results-mieb/openai/clip-vit-large-patch14`](https://github.com/embeddings-benchmark/mteb/blob/mieb/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/RenderedSST2.json) and look legit with an `"accuracy": 0.6979681493684788,`, a bit higher than the original CLIP paper but might be resolution/layout difference of images in the remake of the dataset by the CLIP benchmark team. - diff --git a/docs/mieb-docs/run_vista.md b/docs/mieb-docs/run_vista.md deleted file mode 100644 index 008342c0ce..0000000000 --- a/docs/mieb-docs/run_vista.md +++ /dev/null @@ -1,27 +0,0 @@ -## set up VISTA - -the latest FlagEmbedding repo doesn't support VISTA anymore so we use a old version. -``` -git clone --no-checkout https://github.com/FlagOpen/FlagEmbedding.git -cd FlagEmbedding -git checkout 5c9260277977f8f8e256e56a8e12387552693af9 -pip install -e . -pip install torchvision timm einops ftfy -``` -back to the root folder of mteb; download the vision tower for bge-base -``` -cd .. -wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_base_en_v1.5.pth?download=true -``` -rename it to `visualized_base_en_V1.5.pth` -``` -mv Visualized_base_en_v1.5.pth?download=true visualized_base_en_V1.5.pth -``` -download the vision tower for bge-m3 -``` -wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_m3.pth?download=true -``` -rename it to `visualized_m3.pth` -``` -mv Visualized_m3.pth?download=true visualized_m3.pth -``` diff --git a/docs/mieb.md b/docs/mieb.md new file mode 100644 index 0000000000..e2135730d5 --- /dev/null +++ b/docs/mieb.md @@ -0,0 +1,116 @@ +# Welcome to MIEB! 👋 + +The Massive Image Embedding Benchmark (MIEB) is an image extension of [MTEB](https://arxiv.org/abs/2210.07316) to cover embedding tasks for image-text tasks. + +## 🌱 Background + +MIEB intends to extend MTEB and MMTEB to cover image representation learning and image-text alignment tasks. + +## 🪴 Contributing to MIEB + +The FIRST step is to _always_ create an issue in the MTEB repo (this one), and add the `mieb` label. PRs without issues will not be accepted. + +There are a few ways for anyone to contribute to MIEB: + + 1. Add a dataset as an existing task type. This means that the `AbsTask` already exists, e.g. `AbsTaskImageClassification`, and the effort is solely in adding an instance of it. + 2. Add a model. This could mean either: a) The model wrapper, e.g. `OpenCLIPWrapper`, already exists, and the effort is solely in adding a filled out `ModelMeta` object, and/or b) Add a new model wrapper. + 3. Add a new task type. This means that the existing task types do not cover this new task. An accompanying evaluator should also be implemented. + +Let's go through an example. + +## Example + +Here is an example implementing a zero-shot image classification from scratch. Let's say we wish to implement CIFAR10 as a task and evaluate an OpenCLIP model on it. + +To solve this task, we need to encode the `images`, encode the `class label candidates with prompts` (e.g. "this is a dog pic", "this is a cat pic"), and compare them by calculating similarity, and then argmax out the class prediction for each image. We begin by implementing a model wrapper. + +### Model Wrapper +See the [`ImageEncoder` class](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/encoder_interface.py) for more details. The model class implements `get_text_embeddings`, `get_image_embeddings`, and `calculate_probs` methods. +As an example, [`OpenCLIPWrapper`](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/models/openclip_models.py) is first implemented, with metadata defined below. +```python +class OpenCLIPWrapper: + ... +``` +See also [adding a model](adding_a_model.md) for reference. + +### X Evaluator +With the model, [ZeroshotClassificationEvaluator](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py) is implemented here. This defines how the model are used to do zero-shot classification and get back results on desired metrics. +```python +class ZeroshotClassificationEvaluator(Evaluator): + def __init__(self, ...): + ... + def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): + """Get embeddings and calculate scores.""" + ... +``` + +### AbsTask X +With the evaluator, [AbsTaskZeroshotClassification](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/abstasks/Image/AbsTaskZeroshotClassification.py) is defined, operating on the dataset, calling the defined Evaluator, and gives out results. +```python +class AbsTaskZeroshotClassification(AbsTask): + ... +``` + + +### Dataset class +With all these, we can then define the dataset. [CIFAR10](https://github.com/embeddings-benchmark/mteb/blob/mieb/mteb/tasks/Image/ZeroshotClassification/eng/CIFAR.py) is implemented like this, subclassing `AbsTaskZeroshotClassification`, and overwrite the `get_candidate_labels` function, which gives `["a photo of {label_name}"]` to be used in the evaluator. +```python +class CIFAR10ZeroShotClassification(AbsTaskZeroshotClassification): + metadata = TaskMetadata(...) + + def get_candidate_labels(self) -> list[str]: + ... +``` +See also [adding a dataset](adding_a_dataset.md) for reference. + +### Putting them all together +With all these, we can then +```python +import mteb + +model_name = "laion/CLIP-ViT-L-14-laion2B-s32B-b82K" +model = mteb.get_model(model_name=model_name) + +tasks = mteb.get_tasks(tasks=["CIFAR10ZeroShot"]) +evaluation = mteb.MTEB(tasks=tasks) +results = evaluation.run(model) +``` + +By default, results will be under `results/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/REVISION/CIFAR10ZeroShot.json`. Sometimes metrics can be a bit different than what the original paper claimed. This might be due to the resolution/layout difference of images in the remake of the dataset. + + +## Specific Model running Instructions + +Some models require some specific steps before running. Those are collected here. + +
+ Vista + + ## set up VISTA + + ``` + git clone https://github.com/FlagOpen/FlagEmbedding.git + cd FlagEmbedding/research/visual_bge + pip install -e . + pip install torchvision timm einops ftfy + ``` + back to the root folder of mteb; download the vision tower for bge-base + ``` + cd .. + wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_base_en_v1.5.pth?download=true + ``` + rename it to `visualized_base_en_V1.5.pth` + ``` + mv Visualized_base_en_v1.5.pth?download=true visualized_base_en_V1.5.pth + ``` + download the vision tower for bge-m3 + ``` + wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_m3.pth?download=true + ``` + rename it to `visualized_m3.pth` + ``` + mv Visualized_m3.pth?download=true visualized_m3.pth + ``` + + +
\ No newline at end of file diff --git a/tests/test_tasks/test_all_abstasks.py b/tests/test_tasks/test_all_abstasks.py index dda3634231..e74b43e60b 100644 --- a/tests/test_tasks/test_all_abstasks.py +++ b/tests/test_tasks/test_all_abstasks.py @@ -93,7 +93,14 @@ async def check_datasets_are_available_on_hf(tasks): def test_dataset_availability(): """Checks if the datasets are available on Hugging Face using both their name and revision.""" tasks = MTEB().tasks_cls - tasks = [t for t in tasks if t.metadata.name not in ALL_MOCK_TASKS] + tasks = [ + t + for t in tasks + if t.metadata.name not in MOCK_TASK_TEST_GRID_AS_STRING + if t.metadata.name not in MOCK_MIEB_TASK_GRID_AS_STRING + and t.metadata.name + != "AfriSentiLangClassification" # HOTFIX: Issue#1777. Remove this line when issue is resolved. + ] asyncio.run(check_datasets_are_available_on_hf(tasks)) From 5194033f7769e5c98bda2e199babc3376220f417 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Thu, 16 Jan 2025 18:38:07 +0900 Subject: [PATCH 134/154] [mieb] Remove results-mieb folder (#1815) remove results-mieb folder --- .../CIRRIT2TRetrieval.json | 158 - .../model_meta.json | 1 - .../CIRRIT2TRetrieval.json | 158 - .../model_meta.json | 1 - .../MNIST.json | 28 - .../MSCOCOI2TRetrieval.json | 186 - .../model_meta.json | 1 - .../MSCOCOI2TRetrieval.json | 186 - .../model_meta.json | 1 - .../model_meta.json | 1 - .../Fashion200kI2TRetrieval.json | 186 - .../MNIST.json | 48 - .../MNISTZeroShot.json | 19 - .../NIGHTSI2IRetrieval.json | 186 - .../OxfordFlowersClassification.json | 48 - .../RenderedSST2.json | 19 - .../model_meta.json | 1 - .../RenderedSST2.json | 19 - .../model_meta.json | 1 - .../MNIST.json | 48 - .../MNISTZeroShot.json | 19 - .../NIGHTSI2IRetrieval.json | 186 - .../OxfordFlowersClassification.json | 48 - .../RenderedSST2.json | 19 - .../model_meta.json | 1 - .../CVBenchCount.json | 19 - .../model_meta.json | 1 - .../BLINKIT2IRetrieval.json | 186 - .../Caltech101.json | 28 - .../STS12.json | 26 - .../model_meta.json | 1 - .../1/ImageNetDog15Clustering.json | 23 - .../embed-english-v3.0-v/1/model_meta.json | 1 - .../MNIST.json | 48 - .../NIGHTSI2IRetrieval.json | 186 - .../OxfordFlowersClassification.json | 48 - .../VOC2007.json | 48 - .../model_meta.json | 1 - .../VOC2007.json | 48 - .../model_meta.json | 1 - .../MNIST.json | 48 - .../NIGHTSI2IRetrieval.json | 186 - .../OxfordFlowersClassification.json | 48 - .../VOC2007.json | 48 - .../model_meta.json | 1 - .../GLDv2I2IRetrieval.json | 186 - .../MNIST.json | 48 - .../NIGHTSI2IRetrieval.json | 186 - .../OxfordFlowersClassification.json | 48 - .../STS17MultilingualVisualSTS.json | 183 - .../VOC2007.json | 48 - .../model_meta.json | 1 - .../CIFAR10.json | 48 - .../CIFAR100.json | 48 - .../ImageNet10Clustering.json | 22 - .../MNIST.json | 48 - .../MNISTZeroShot.json | 19 - .../WITT2IRetrieval.json | 1936 ----- .../model_meta.json | 1 - .../WITT2IRetrieval.json | 1936 ----- .../model_meta.json | 1 - .../CIFAR10.json | 48 - .../CIFAR100.json | 48 - .../MNIST.json | 48 - .../MNISTZeroShot.json | 19 - .../WITT2IRetrieval.json | 1936 ----- .../model_meta.json | 1 - .../WITT2IRetrieval.json | 1936 ----- .../model_meta.json | 1 - .../WITT2IRetrieval.json | 1936 ----- .../model_meta.json | 1 - .../WITT2IRetrieval.json | 1936 ----- .../model_meta.json | 1 - .../ImageNet10Clustering.json | 22 - .../WITT2IRetrieval.json | 1936 ----- .../model_meta.json | 1 - .../VidoreArxivQARetrieval.json | 186 - .../VidoreDocVQARetrieval.json | 186 - .../VidoreInfoVQARetrieval.json | 186 - .../VidoreShiftProjectRetrieval.json | 186 - .../VidoreSyntheticDocQAAIRetrieval.json | 186 - .../VidoreSyntheticDocQAEnergyRetrieval.json | 186 - ...theticDocQAGovernmentReportsRetrieval.json | 186 - ...heticDocQAHealthcareIndustryRetrieval.json | 186 - .../VidoreTabfquadRetrieval.json | 186 - .../VidoreTatdqaRetrieval.json | 186 - .../WITT2IRetrieval.json | 1936 ----- .../model_meta.json | 1 - .../Flickr30kI2TRetrieval.json | 186 - .../Flickr30kT2IRetrieval.json | 186 - .../MSCOCOI2TRetrieval.json | 186 - .../STS14.json | 26 - .../model_meta.json | 1 - .../MSCOCOI2TRetrieval.json | 158 - .../MSCOCOT2IRetrieval.json | 158 - .../STS12.json | 26 - .../model_meta.json | 1 - .../MNIST.json | 48 - .../MNISTZeroShot.json | 19 - .../NIGHTSI2IRetrieval.json | 186 - .../OxfordFlowersClassification.json | 48 - .../RenderedSST2.json | 19 - .../STS17MultilingualVisualSTS.json | 183 - .../model_meta.json | 1 - .../VOC2007.json | 48 - .../model_meta.json | 1 - .../MNISTZeroShot.json | 19 - .../model_meta.json | 1 - .../MNIST.json | 48 - .../VOC2007.json | 48 - .../model_meta.json | 1 - .../BLINKIT2IRetrieval.json | 186 - .../MNIST.json | 28 - .../MNISTZeroShot.json | 19 - .../STS14.json | 26 - .../model_meta.json | 1 - .../MNISTZeroShot.json | 19 - .../model_meta.json | 1 - .../MNIST.json | 48 - .../VOC2007.json | 48 - .../model_meta.json | 1 - .../CIRRIT2IRetrieval.json | 158 - .../MSCOCOI2TRetrieval.json | 158 - .../MSCOCOT2IRetrieval.json | 158 - .../NIGHTSI2IRetrieval.json | 158 - .../WebQAT2ITRetrieval.json | 158 - .../WebQAT2TRetrieval.json | 158 - .../model_meta.json | 1 - .../CIFAR10Clustering.json | 22 - .../OxfordFlowersClassification.json | 48 - .../model_meta.json | 1 - .../OxfordFlowersClassification.json | 48 - .../model_meta.json | 1 - .../AROCocoOrder.json | 21 - .../AROFlickrOrder.json | 21 - .../AROVisualAttribution.json | 21 - .../AROVisualRelation.json | 21 - .../Birdsnap.json | 48 - .../CIRRIT2TRetrieval.json | 158 - .../CVBenchCount.json | 19 - .../CVBenchDepth.json | 19 - .../CVBenchDistance.json | 19 - .../CVBenchRelation.json | 19 - .../Caltech101.json | 48 - .../FORBI2IRetrieval.json | 186 - .../Flickr30kI2TRetrieval.json | 186 - .../Flickr30kT2IRetrieval.json | 186 - .../HatefulMemesI2TRetrieval.json | 186 - .../HatefulMemesT2IRetrieval.json | 186 - .../MSCOCOI2TRetrieval.json | 158 - .../MSCOCOT2IRetrieval.json | 158 - .../MemotionI2TRetrieval.json | 186 - .../MemotionT2IRetrieval.json | 186 - .../RenderedSST2.json | 19 - .../STS12VisualSTS.json | 26 - .../STS13VisualSTS.json | 26 - .../STS14VisualSTS.json | 26 - .../STS15VisualSTS.json | 26 - .../STS16VisualSTS.json | 26 - .../STS17MultilingualVisualSTS.json | 183 - .../SciMMIRI2TRetrieval.json | 186 - .../SciMMIRT2IRetrieval.json | 186 - .../SugarCrepe.json | 21 - .../TUBerlinT2IRetrieval.json | 186 - .../TinyImageNetClustering.json | 19 - .../VOC2007.json | 48 - .../VidoreArxivQARetrieval.json | 186 - .../VidoreDocVQARetrieval.json | 186 - .../VidoreInfoVQARetrieval.json | 186 - .../VidoreShiftProjectRetrieval.json | 186 - .../VidoreSyntheticDocQAAIRetrieval.json | 186 - .../VidoreSyntheticDocQAEnergyRetrieval.json | 186 - ...theticDocQAGovernmentReportsRetrieval.json | 186 - ...heticDocQAHealthcareIndustryRetrieval.json | 186 - .../VidoreTabfquadRetrieval.json | 186 - .../VidoreTatdqaRetrieval.json | 186 - .../Winoground.json | 21 - .../model_meta.json | 1 - .../AROCocoOrder.json | 21 - .../AROFlickrOrder.json | 21 - .../AROVisualAttribution.json | 21 - .../AROVisualRelation.json | 21 - .../BLINKIT2IMultiChoice.json | 33 - .../BLINKIT2IRetrieval.json | 186 - .../BLINKIT2TMultiChoice.json | 33 - .../BLINKIT2TRetrieval.json | 186 - .../Birdsnap.json | 48 - .../BirdsnapZeroShot.json | 19 - .../CIFAR10.json | 48 - .../CIFAR100.json | 48 - .../CIFAR100Clustering.json | 20 - .../CIFAR100ZeroShot.json | 19 - .../CIFAR10Clustering.json | 20 - .../CIFAR10ZeroShot.json | 19 - .../CIRRIT2IRetrieval.json | 186 - .../CLEVR.json | 19 - .../CLEVRCount.json | 19 - .../CUB200I2IRetrieval.json | 186 - .../CVBenchCount.json | 19 - .../CVBenchDepth.json | 19 - .../CVBenchDistance.json | 19 - .../CVBenchRelation.json | 19 - .../Caltech101.json | 48 - .../Caltech101ZeroShot.json | 19 - .../Country211.json | 28 - .../Country211ZeroShot.json | 19 - .../DTD.json | 48 - .../DTDZeroShot.json | 19 - .../EDIST2ITRetrieval.json | 186 - .../EuroSAT.json | 48 - .../EuroSATZeroShot.json | 19 - .../FER2013.json | 48 - .../FER2013ZeroShot.json | 19 - .../FGVCAircraft.json | 73 - .../FGVCAircraftZeroShot.json | 19 - .../FORBI2IRetrieval.json | 186 - .../Fashion200kI2TRetrieval.json | 158 - .../Fashion200kT2IRetrieval.json | 158 - .../FashionIQIT2IRetrieval.json | 158 - .../Flickr30kI2TRetrieval.json | 186 - .../Flickr30kT2IRetrieval.json | 186 - .../Food101Classification.json | 48 - .../Food101ZeroShot.json | 19 - .../GLDv2I2IRetrieval.json | 186 - .../GLDv2I2TRetrieval.json | 186 - .../GTSRB.json | 28 - .../GTSRBZeroShot.json | 19 - .../HatefulMemesI2TRetrieval.json | 186 - .../HatefulMemesT2IRetrieval.json | 186 - .../ImageCoDeT2IMultiChoice.json | 33 - .../ImageCoDeT2IRetrieval.json | 186 - .../ImageNet10Clustering.json | 23 - .../ImageNetDog15Clustering.json | 23 - .../Imagenet1k.json | 28 - .../Imagenet1kZeroShot.json | 19 - .../InfoSeekIT2ITRetrieval.json | 186 - .../InfoSeekIT2TRetrieval.json | 186 - .../MNIST.json | 48 - .../MNISTZeroShot.json | 19 - .../MSCOCOI2TRetrieval.json | 186 - .../MSCOCOT2IRetrieval.json | 186 - .../MemotionI2TRetrieval.json | 186 - .../MemotionT2IRetrieval.json | 186 - .../NIGHTSI2IRetrieval.json | 186 - .../OVENIT2ITRetrieval.json | 158 - .../OVENIT2TRetrieval.json | 158 - .../OxfordPets.json | 73 - .../OxfordPetsZeroShot.json | 19 - .../PatchCamelyon.json | 32 - .../PatchCamelyonZeroShot.json | 19 - .../RESISC45.json | 48 - .../RESISC45ZeroShot.json | 19 - .../ROxfordEasyI2IMultiChoice.json | 187 - .../ROxfordEasyI2IRetrieval.json | 186 - .../ROxfordHardI2IMultiChoice.json | 187 - .../ROxfordHardI2IRetrieval.json | 186 - .../ROxfordMediumI2IMultiChoice.json | 187 - .../ROxfordMediumI2IRetrieval.json | 186 - .../RP2kI2IRetrieval.json | 186 - .../RParisEasyI2IMultiChoice.json | 187 - .../RParisEasyI2IRetrieval.json | 186 - .../RParisHardI2IMultiChoice.json | 187 - .../RParisHardI2IRetrieval.json | 186 - .../RParisMediumI2IMultiChoice.json | 187 - .../RParisMediumI2IRetrieval.json | 186 - .../RenderedSST2.json | 19 - .../SOPI2IRetrieval.json | 186 - .../STL10.json | 48 - .../STL10ZeroShot.json | 19 - .../STS12VisualSTS.json | 26 - .../STS13VisualSTS.json | 26 - .../STS14VisualSTS.json | 26 - .../STS15VisualSTS.json | 26 - .../STS16VisualSTS.json | 26 - .../STS17MultilingualVisualSTS.json | 183 - .../STSBenchmarkMultilingualVisualSTS.json | 313 - .../SUN397.json | 48 - .../SUN397ZeroShot.json | 19 - .../SciMMIR.json | 19 - .../SciMMIRI2TRetrieval.json | 186 - .../SciMMIRT2IRetrieval.json | 186 - .../StanfordCars.json | 48 - .../StanfordCarsI2IRetrieval.json | 186 - .../StanfordCarsZeroShot.json | 19 - .../SugarCrepe.json | 21 - .../TUBerlinT2IRetrieval.json | 186 - .../UCF101.json | 28 - .../UCF101ZeroShot.json | 19 - .../VOC2007.json | 48 - .../VQA2IT2TRetrieval.json | 186 - .../VidoreArxivQARetrieval.json | 186 - .../VidoreDocVQARetrieval.json | 186 - .../VidoreInfoVQARetrieval.json | 186 - .../VidoreShiftProjectRetrieval.json | 186 - .../VidoreSyntheticDocQAAIRetrieval.json | 186 - .../VidoreSyntheticDocQAEnergyRetrieval.json | 186 - ...theticDocQAGovernmentReportsRetrieval.json | 186 - ...heticDocQAHealthcareIndustryRetrieval.json | 186 - .../VidoreTabfquadRetrieval.json | 186 - .../VidoreTatdqaRetrieval.json | 186 - .../VisualNewsI2TRetrieval.json | 186 - .../VisualNewsT2IRetrieval.json | 186 - .../VizWizIT2TRetrieval.json | 186 - .../WITT2IRetrieval.json | 1936 ----- .../WebQAT2ITRetrieval.json | 158 - .../WebQAT2TRetrieval.json | 158 - .../Winoground.json | 21 - .../XFlickr30kCoT2IRetrieval.json | 1411 ---- .../XM3600T2IRetrieval.json | 6311 ----------------- .../model_meta.json | 1 - .../AROCocoOrder.json | 21 - .../AROFlickrOrder.json | 21 - .../AROVisualAttribution.json | 21 - .../AROVisualRelation.json | 21 - .../CIRRIT2TRetrieval.json | 158 - .../CVBenchCount.json | 19 - .../CVBenchDepth.json | 19 - .../CVBenchDistance.json | 19 - .../CVBenchRelation.json | 19 - .../FORBI2IRetrieval.json | 186 - .../Fashion200kI2TRetrieval.json | 186 - .../HatefulMemesI2TRetrieval.json | 186 - .../HatefulMemesT2IRetrieval.json | 186 - .../MNIST.json | 48 - .../MSCOCOI2TRetrieval.json | 158 - .../MSCOCOT2IRetrieval.json | 158 - .../MemotionI2TRetrieval.json | 186 - .../MemotionT2IRetrieval.json | 186 - .../RenderedSST2.json | 19 - .../STS12VisualSTS.json | 26 - .../STS13VisualSTS.json | 26 - .../STS14VisualSTS.json | 26 - .../STS15VisualSTS.json | 26 - .../STS16VisualSTS.json | 26 - .../STS17MultilingualVisualSTS.json | 183 - .../STSBenchmarkMultilingualVisualSTS.json | 313 - .../SciMMIRI2TRetrieval.json | 186 - .../SciMMIRT2IRetrieval.json | 186 - .../SugarCrepe.json | 21 - .../TUBerlinT2IRetrieval.json | 186 - .../VOC2007.json | 48 - .../VidoreArxivQARetrieval.json | 186 - .../VidoreDocVQARetrieval.json | 186 - .../VidoreInfoVQARetrieval.json | 186 - .../VidoreShiftProjectRetrieval.json | 186 - .../VidoreSyntheticDocQAAIRetrieval.json | 186 - .../VidoreSyntheticDocQAEnergyRetrieval.json | 186 - ...theticDocQAGovernmentReportsRetrieval.json | 186 - ...heticDocQAHealthcareIndustryRetrieval.json | 186 - .../VidoreTabfquadRetrieval.json | 186 - .../VidoreTatdqaRetrieval.json | 186 - .../Winoground.json | 21 - .../model_meta.json | 1 - .../CIRRIT2TRetrieval.json | 158 - .../model_meta.json | 1 - .../voyage-multimodal-3/1/AROCocoOrder.json | 21 - .../voyage-multimodal-3/1/AROFlickrOrder.json | 21 - .../1/AROVisualAttribution.json | 21 - .../1/AROVisualRelation.json | 21 - .../voyage-multimodal-3/1/CIFAR10.json | 48 - .../voyage-multimodal-3/1/CIFAR100.json | 48 - .../1/CIFAR100Clustering.json | 22 - .../1/CIFAR100ZeroShot.json | 19 - .../1/CIFAR10Clustering.json | 22 - .../1/CIFAR10ZeroShot.json | 19 - .../1/CIRRIT2IRetrieval.json | 186 - .../1/CLEVRCountZeroShot.json | 19 - .../voyage-multimodal-3/1/CLEVRZeroShot.json | 19 - .../1/CUB200I2IRetrieval.json | 186 - .../voyage-multimodal-3/1/CVBenchCount.json | 19 - .../voyage-multimodal-3/1/CVBenchDepth.json | 19 - .../1/CVBenchDistance.json | 19 - .../1/CVBenchRelation.json | 19 - .../voyage-multimodal-3/1/Caltech101.json | 48 - .../1/Caltech101ZeroShot.json | 19 - .../voyage-multimodal-3/1/Country211.json | 48 - .../1/Country211ZeroShot.json | 19 - results-mieb/voyage-multimodal-3/1/DTD.json | 48 - .../voyage-multimodal-3/1/DTDZeroShot.json | 19 - .../voyage-multimodal-3/1/EuroSAT.json | 48 - .../1/EuroSATZeroShot.json | 19 - .../voyage-multimodal-3/1/FER2013.json | 48 - .../1/FER2013ZeroShot.json | 19 - .../voyage-multimodal-3/1/FGVCAircraft.json | 48 - .../1/FGVCAircraftZeroShot.json | 19 - .../1/Fashion200kI2TRetrieval.json | 186 - .../1/Flickr30kI2TRetrieval.json | 186 - .../1/Flickr30kT2IRetrieval.json | 186 - .../1/Food101Classification.json | 48 - .../1/Food101ZeroShot.json | 19 - results-mieb/voyage-multimodal-3/1/GTSRB.json | 48 - .../voyage-multimodal-3/1/GTSRBZeroShot.json | 19 - .../1/HatefulMemesI2TRetrieval.json | 186 - .../1/ImageNet10Clustering.json | 22 - .../1/ImageNetDog15Clustering.json | 22 - .../1/InfoSeekIT2ITRetrieval.json | 186 - .../1/InfoSeekIT2TRetrieval.json | 186 - results-mieb/voyage-multimodal-3/1/MNIST.json | 48 - .../voyage-multimodal-3/1/MNISTZeroShot.json | 19 - .../1/MSCOCOI2TRetrieval.json | 186 - .../1/MSCOCOT2IRetrieval.json | 186 - .../1/NIGHTSI2IRetrieval.json | 186 - .../1/OVENIT2TRetrieval.json | 186 - .../1/OxfordFlowersClassification.json | 48 - .../voyage-multimodal-3/1/OxfordPets.json | 48 - .../1/OxfordPetsZeroShot.json | 19 - .../voyage-multimodal-3/1/PatchCamelyon.json | 60 - .../1/PatchCamelyonZeroShot.json | 19 - .../voyage-multimodal-3/1/RESISC45.json | 48 - .../1/RESISC45ZeroShot.json | 19 - .../1/RP2kI2IRetrieval.json | 186 - .../voyage-multimodal-3/1/RenderedSST2.json | 19 - results-mieb/voyage-multimodal-3/1/STL10.json | 48 - .../voyage-multimodal-3/1/STL10ZeroShot.json | 19 - .../voyage-multimodal-3/1/STS12VisualSTS.json | 26 - .../voyage-multimodal-3/1/STS13VisualSTS.json | 26 - .../voyage-multimodal-3/1/STS14VisualSTS.json | 26 - .../voyage-multimodal-3/1/STS15VisualSTS.json | 26 - .../voyage-multimodal-3/1/STS16VisualSTS.json | 26 - .../1/STS17MultilingualVisualSTS.json | 183 - .../1/STSBenchmarkMultilingualVisualSTS.json | 313 - .../voyage-multimodal-3/1/SUN397.json | 48 - .../voyage-multimodal-3/1/SUN397ZeroShot.json | 19 - .../voyage-multimodal-3/1/StanfordCars.json | 48 - .../1/StanfordCarsI2IRetrieval.json | 186 - .../1/StanfordCarsZeroShot.json | 19 - .../voyage-multimodal-3/1/SugarCrepe.json | 21 - .../1/TUBerlinT2IRetrieval.json | 186 - .../1/TinyImageNetClustering.json | 22 - .../1/VidoreArxivQARetrieval.json | 186 - .../1/VidoreDocVQARetrieval.json | 186 - .../1/VidoreInfoVQARetrieval.json | 186 - .../1/VidoreShiftProjectRetrieval.json | 186 - .../1/VidoreSyntheticDocQAAIRetrieval.json | 186 - .../VidoreSyntheticDocQAEnergyRetrieval.json | 186 - ...theticDocQAGovernmentReportsRetrieval.json | 186 - ...heticDocQAHealthcareIndustryRetrieval.json | 186 - .../1/VidoreTabfquadRetrieval.json | 186 - .../1/VidoreTatdqaRetrieval.json | 186 - .../1/VisualNewsI2TRetrieval.json | 186 - .../1/WITT2IRetrieval.json | 1936 ----- .../1/WebQAT2ITRetrieval.json | 186 - .../1/WebQAT2TRetrieval.json | 186 - .../voyage-multimodal-3/1/Winoground.json | 21 - .../1/XFlickr30kCoT2IRetrieval.json | 1411 ---- .../1/XM3600T2IRetrieval.json | 6311 ----------------- .../voyage-multimodal-3/1/model_meta.json | 1 - 447 files changed, 73973 deletions(-) delete mode 100644 results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json delete mode 100644 results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json delete mode 100644 results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json delete mode 100644 results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json delete mode 100644 results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MNIST.json delete mode 100644 results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json delete mode 100644 results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json delete mode 100644 results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json delete mode 100644 results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json delete mode 100644 results-mieb/EVA02-CLIP-bigE-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json delete mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/Fashion200kI2TRetrieval.json delete mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNIST.json delete mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNISTZeroShot.json delete mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/NIGHTSI2IRetrieval.json delete mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/OxfordFlowersClassification.json delete mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/RenderedSST2.json delete mode 100644 results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/model_meta.json delete mode 100644 results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/RenderedSST2.json delete mode 100644 results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/model_meta.json delete mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNIST.json delete mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNISTZeroShot.json delete mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/NIGHTSI2IRetrieval.json delete mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/OxfordFlowersClassification.json delete mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/RenderedSST2.json delete mode 100644 results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/model_meta.json delete mode 100644 results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/CVBenchCount.json delete mode 100644 results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json delete mode 100644 results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json delete mode 100644 results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/Caltech101.json delete mode 100644 results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/STS12.json delete mode 100644 results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/model_meta.json delete mode 100644 results-mieb/embed-english-v3.0-v/1/ImageNetDog15Clustering.json delete mode 100644 results-mieb/embed-english-v3.0-v/1/model_meta.json delete mode 100644 results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/MNIST.json delete mode 100644 results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/NIGHTSI2IRetrieval.json delete mode 100644 results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/OxfordFlowersClassification.json delete mode 100644 results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json delete mode 100644 results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json delete mode 100644 results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json delete mode 100644 results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json delete mode 100644 results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/MNIST.json delete mode 100644 results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/NIGHTSI2IRetrieval.json delete mode 100644 results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/OxfordFlowersClassification.json delete mode 100644 results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json delete mode 100644 results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json delete mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/GLDv2I2IRetrieval.json delete mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/MNIST.json delete mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/NIGHTSI2IRetrieval.json delete mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/OxfordFlowersClassification.json delete mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/STS17MultilingualVisualSTS.json delete mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json delete mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json delete mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR10.json delete mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR100.json delete mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/ImageNet10Clustering.json delete mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNIST.json delete mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNISTZeroShot.json delete mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/WITT2IRetrieval.json delete mode 100644 results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/model_meta.json delete mode 100644 results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/WITT2IRetrieval.json delete mode 100644 results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/model_meta.json delete mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR10.json delete mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR100.json delete mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNIST.json delete mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNISTZeroShot.json delete mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/WITT2IRetrieval.json delete mode 100644 results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/model_meta.json delete mode 100644 results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/WITT2IRetrieval.json delete mode 100644 results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/model_meta.json delete mode 100644 results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/WITT2IRetrieval.json delete mode 100644 results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/model_meta.json delete mode 100644 results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/WITT2IRetrieval.json delete mode 100644 results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/model_meta.json delete mode 100644 results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/ImageNet10Clustering.json delete mode 100644 results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/WITT2IRetrieval.json delete mode 100644 results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/model_meta.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/WITT2IRetrieval.json delete mode 100644 results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json delete mode 100644 results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json delete mode 100644 results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kT2IRetrieval.json delete mode 100644 results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/MSCOCOI2TRetrieval.json delete mode 100644 results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/STS14.json delete mode 100644 results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json delete mode 100644 results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOI2TRetrieval.json delete mode 100644 results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOT2IRetrieval.json delete mode 100644 results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/STS12.json delete mode 100644 results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/model_meta.json delete mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNIST.json delete mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNISTZeroShot.json delete mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/NIGHTSI2IRetrieval.json delete mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/OxfordFlowersClassification.json delete mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/RenderedSST2.json delete mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/STS17MultilingualVisualSTS.json delete mode 100644 results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/model_meta.json delete mode 100644 results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json delete mode 100644 results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json delete mode 100644 results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/MNISTZeroShot.json delete mode 100644 results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/model_meta.json delete mode 100644 results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/MNIST.json delete mode 100644 results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json delete mode 100644 results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json delete mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/BLINKIT2IRetrieval.json delete mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNIST.json delete mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNISTZeroShot.json delete mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/STS14.json delete mode 100644 results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json delete mode 100644 results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/MNISTZeroShot.json delete mode 100644 results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/model_meta.json delete mode 100644 results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/MNIST.json delete mode 100644 results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json delete mode 100644 results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json delete mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/CIRRIT2IRetrieval.json delete mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOI2TRetrieval.json delete mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOT2IRetrieval.json delete mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/NIGHTSI2IRetrieval.json delete mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2ITRetrieval.json delete mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2TRetrieval.json delete mode 100644 results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/model_meta.json delete mode 100644 results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/CIFAR10Clustering.json delete mode 100644 results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json delete mode 100644 results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json delete mode 100644 results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json delete mode 100644 results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROCocoOrder.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROFlickrOrder.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualAttribution.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualRelation.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Birdsnap.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CIRRIT2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchCount.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDepth.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDistance.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchRelation.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Caltech101.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/FORBI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/RenderedSST2.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS12VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS13VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS14VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS15VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS16VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS17MultilingualVisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SugarCrepe.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TUBerlinT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TinyImageNetClustering.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreArxivQARetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreDocVQARetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreInfoVQARetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreShiftProjectRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAAIRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAEnergyRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTabfquadRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTatdqaRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Winoground.json delete mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROCocoOrder.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROFlickrOrder.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualAttribution.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualRelation.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Birdsnap.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BirdsnapZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100Clustering.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100ZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10Clustering.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10ZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVR.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVRCount.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CUB200I2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchCount.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDepth.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDistance.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchRelation.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101ZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211ZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTD.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTDZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EDIST2ITRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSAT.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013ZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraft.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraftZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FashionIQIT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101Classification.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101ZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRB.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRBZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IMultiChoice.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNet10Clustering.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNetDog15Clustering.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1k.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1kZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2ITRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNIST.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNISTZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2ITRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPets.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPetsZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyon.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyonZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45ZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IMultiChoice.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IMultiChoice.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IMultiChoice.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RP2kI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IMultiChoice.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IMultiChoice.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IMultiChoice.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RenderedSST2.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10ZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS12VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS13VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS14VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS15VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS16VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS17MultilingualVisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STSBenchmarkMultilingualVisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIR.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCars.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SugarCrepe.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/TUBerlinT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101ZeroShot.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VQA2IT2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreArxivQARetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreDocVQARetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreInfoVQARetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreShiftProjectRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAAIRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAEnergyRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAGovernmentReportsRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTabfquadRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTatdqaRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VizWizIT2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WITT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2ITRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Winoground.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XFlickr30kCoT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XM3600T2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROCocoOrder.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROFlickrOrder.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualAttribution.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualRelation.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CIRRIT2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchCount.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDepth.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDistance.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchRelation.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/FORBI2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Fashion200kI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MNIST.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/RenderedSST2.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS12VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS13VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS14VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS15VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS16VisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS17MultilingualVisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STSBenchmarkMultilingualVisualSTS.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRI2TRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SugarCrepe.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/TUBerlinT2IRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreArxivQARetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreDocVQARetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreInfoVQARetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreShiftProjectRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAAIRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAEnergyRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAGovernmentReportsRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTabfquadRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTatdqaRetrieval.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Winoground.json delete mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json delete mode 100644 results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/CIRRIT2TRetrieval.json delete mode 100644 results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json delete mode 100644 results-mieb/voyage-multimodal-3/1/AROCocoOrder.json delete mode 100644 results-mieb/voyage-multimodal-3/1/AROFlickrOrder.json delete mode 100644 results-mieb/voyage-multimodal-3/1/AROVisualAttribution.json delete mode 100644 results-mieb/voyage-multimodal-3/1/AROVisualRelation.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR10.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR100.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR100Clustering.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR100ZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR10Clustering.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CIFAR10ZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CIRRIT2IRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CLEVRCountZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CLEVRZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CUB200I2IRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CVBenchCount.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CVBenchDepth.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CVBenchDistance.json delete mode 100644 results-mieb/voyage-multimodal-3/1/CVBenchRelation.json delete mode 100644 results-mieb/voyage-multimodal-3/1/Caltech101.json delete mode 100644 results-mieb/voyage-multimodal-3/1/Caltech101ZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/Country211.json delete mode 100644 results-mieb/voyage-multimodal-3/1/Country211ZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/DTD.json delete mode 100644 results-mieb/voyage-multimodal-3/1/DTDZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/EuroSAT.json delete mode 100644 results-mieb/voyage-multimodal-3/1/EuroSATZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/FER2013.json delete mode 100644 results-mieb/voyage-multimodal-3/1/FER2013ZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/FGVCAircraft.json delete mode 100644 results-mieb/voyage-multimodal-3/1/FGVCAircraftZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/Fashion200kI2TRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/Flickr30kI2TRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/Flickr30kT2IRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/Food101Classification.json delete mode 100644 results-mieb/voyage-multimodal-3/1/Food101ZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/GTSRB.json delete mode 100644 results-mieb/voyage-multimodal-3/1/GTSRBZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/HatefulMemesI2TRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/ImageNet10Clustering.json delete mode 100644 results-mieb/voyage-multimodal-3/1/ImageNetDog15Clustering.json delete mode 100644 results-mieb/voyage-multimodal-3/1/InfoSeekIT2ITRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/InfoSeekIT2TRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/MNIST.json delete mode 100644 results-mieb/voyage-multimodal-3/1/MNISTZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/MSCOCOI2TRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/MSCOCOT2IRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/NIGHTSI2IRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/OVENIT2TRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/OxfordFlowersClassification.json delete mode 100644 results-mieb/voyage-multimodal-3/1/OxfordPets.json delete mode 100644 results-mieb/voyage-multimodal-3/1/OxfordPetsZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/PatchCamelyon.json delete mode 100644 results-mieb/voyage-multimodal-3/1/PatchCamelyonZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/RESISC45.json delete mode 100644 results-mieb/voyage-multimodal-3/1/RESISC45ZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/RP2kI2IRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/RenderedSST2.json delete mode 100644 results-mieb/voyage-multimodal-3/1/STL10.json delete mode 100644 results-mieb/voyage-multimodal-3/1/STL10ZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/STS12VisualSTS.json delete mode 100644 results-mieb/voyage-multimodal-3/1/STS13VisualSTS.json delete mode 100644 results-mieb/voyage-multimodal-3/1/STS14VisualSTS.json delete mode 100644 results-mieb/voyage-multimodal-3/1/STS15VisualSTS.json delete mode 100644 results-mieb/voyage-multimodal-3/1/STS16VisualSTS.json delete mode 100644 results-mieb/voyage-multimodal-3/1/STS17MultilingualVisualSTS.json delete mode 100644 results-mieb/voyage-multimodal-3/1/STSBenchmarkMultilingualVisualSTS.json delete mode 100644 results-mieb/voyage-multimodal-3/1/SUN397.json delete mode 100644 results-mieb/voyage-multimodal-3/1/SUN397ZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/StanfordCars.json delete mode 100644 results-mieb/voyage-multimodal-3/1/StanfordCarsI2IRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/StanfordCarsZeroShot.json delete mode 100644 results-mieb/voyage-multimodal-3/1/SugarCrepe.json delete mode 100644 results-mieb/voyage-multimodal-3/1/TUBerlinT2IRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/TinyImageNetClustering.json delete mode 100644 results-mieb/voyage-multimodal-3/1/VidoreArxivQARetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/VidoreDocVQARetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/VidoreInfoVQARetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/VidoreShiftProjectRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAAIRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAEnergyRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAGovernmentReportsRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/VidoreTabfquadRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/VidoreTatdqaRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/VisualNewsI2TRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/WITT2IRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/WebQAT2ITRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/WebQAT2TRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/Winoground.json delete mode 100644 results-mieb/voyage-multimodal-3/1/XFlickr30kCoT2IRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/XM3600T2IRetrieval.json delete mode 100644 results-mieb/voyage-multimodal-3/1/model_meta.json diff --git a/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json b/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json deleted file mode 100644 index d64eff78ed..0000000000 --- a/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", - "evaluation_time": 170.71532011032104, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.15458, - "map_at_1": 0.01763, - "map_at_10": 0.10611, - "map_at_100": 0.11949, - "map_at_1000": 0.12077, - "map_at_20": 0.11329, - "map_at_3": 0.07918, - "map_at_5": 0.09356, - "mrr_at_1": 0.02182254196642686, - "mrr_at_10": 0.10637946785428773, - "mrr_at_100": 0.11975036168307936, - "mrr_at_1000": 0.12103153782524949, - "mrr_at_20": 0.11357094520966761, - "mrr_at_3": 0.07957633892885677, - "mrr_at_5": 0.09383293365307707, - "nauc_map_at_1000_diff1": -0.1398093587192006, - "nauc_map_at_1000_max": -0.011571981373852963, - "nauc_map_at_1000_std": 0.016242835917060484, - "nauc_map_at_100_diff1": -0.13932006959252094, - "nauc_map_at_100_max": -0.011471402747623032, - "nauc_map_at_100_std": 0.01571359503578373, - "nauc_map_at_10_diff1": -0.13667051086803197, - "nauc_map_at_10_max": -0.011921920148726927, - "nauc_map_at_10_std": 0.007179174808104729, - "nauc_map_at_1_diff1": -0.4350127493981227, - "nauc_map_at_1_max": -0.15259852263953608, - "nauc_map_at_1_std": 0.0970061577980624, - "nauc_map_at_20_diff1": -0.13618307626254747, - "nauc_map_at_20_max": -0.01109762645661432, - "nauc_map_at_20_std": 0.010507207905837899, - "nauc_map_at_3_diff1": -0.15118399083124537, - "nauc_map_at_3_max": -0.011017137393102167, - "nauc_map_at_3_std": -0.012575390001070445, - "nauc_map_at_5_diff1": -0.1447814690330949, - "nauc_map_at_5_max": -0.013474206558154681, - "nauc_map_at_5_std": 0.00019795325134277809, - "nauc_mrr_at_1000_diff1": -0.14068353124277078, - "nauc_mrr_at_1000_max": -0.01256002651149935, - "nauc_mrr_at_1000_std": 0.017068554858189054, - "nauc_mrr_at_100_diff1": -0.14019671870704672, - "nauc_mrr_at_100_max": -0.012457880868046342, - "nauc_mrr_at_100_std": 0.016528429573257345, - "nauc_mrr_at_10_diff1": -0.13737917913365197, - "nauc_mrr_at_10_max": -0.01310536806893329, - "nauc_mrr_at_10_std": 0.007847056495460521, - "nauc_mrr_at_1_diff1": -0.43519962731124523, - "nauc_mrr_at_1_max": -0.12632792482537586, - "nauc_mrr_at_1_std": 0.07526013026415497, - "nauc_mrr_at_20_diff1": -0.13713739546158307, - "nauc_mrr_at_20_max": -0.012046383412244238, - "nauc_mrr_at_20_std": 0.011333750815465542, - "nauc_mrr_at_3_diff1": -0.15256650395775093, - "nauc_mrr_at_3_max": -0.01285034024259544, - "nauc_mrr_at_3_std": -0.011306437596869121, - "nauc_mrr_at_5_diff1": -0.14586054456834394, - "nauc_mrr_at_5_max": -0.014513358256981268, - "nauc_mrr_at_5_std": 0.0018048979112297826, - "nauc_ndcg_at_1000_diff1": -0.14124685268905293, - "nauc_ndcg_at_1000_max": -0.009951401237880533, - "nauc_ndcg_at_1000_std": 0.057750817408098905, - "nauc_ndcg_at_100_diff1": -0.13265632530925167, - "nauc_ndcg_at_100_max": -0.0040308311439358245, - "nauc_ndcg_at_100_std": 0.05781381999577725, - "nauc_ndcg_at_10_diff1": -0.11414370292413088, - "nauc_ndcg_at_10_max": -0.004236214064962545, - "nauc_ndcg_at_10_std": 0.01241974149052717, - "nauc_ndcg_at_1_diff1": -0.433649345616075, - "nauc_ndcg_at_1_max": -0.12218048475091685, - "nauc_ndcg_at_1_std": 0.07249798765763336, - "nauc_ndcg_at_20_diff1": -0.11345335703573077, - "nauc_ndcg_at_20_max": -0.002148129039034098, - "nauc_ndcg_at_20_std": 0.021976127484270407, - "nauc_ndcg_at_3_diff1": -0.13328966349326943, - "nauc_ndcg_at_3_max": -0.0031192254944289468, - "nauc_ndcg_at_3_std": -0.01881648682054409, - "nauc_ndcg_at_5_diff1": -0.1266611748844269, - "nauc_ndcg_at_5_max": -0.006687530952537075, - "nauc_ndcg_at_5_std": -0.0005726739692759715, - "nauc_precision_at_1000_diff1": -0.3860299158889171, - "nauc_precision_at_1000_max": -0.08377411665854645, - "nauc_precision_at_1000_std": 0.4127832513033843, - "nauc_precision_at_100_diff1": -0.16470827851656045, - "nauc_precision_at_100_max": 0.005707104600219941, - "nauc_precision_at_100_std": 0.17540404629596543, - "nauc_precision_at_10_diff1": -0.09948700092707702, - "nauc_precision_at_10_max": 0.004571406776459961, - "nauc_precision_at_10_std": 0.01970423348809215, - "nauc_precision_at_1_diff1": -0.433649345616075, - "nauc_precision_at_1_max": -0.12218048475091685, - "nauc_precision_at_1_std": 0.07249798765763336, - "nauc_precision_at_20_diff1": -0.09926523747780691, - "nauc_precision_at_20_max": 0.008832676339450061, - "nauc_precision_at_20_std": 0.04036571606012274, - "nauc_precision_at_3_diff1": -0.12470591591535352, - "nauc_precision_at_3_max": 0.009684827445411333, - "nauc_precision_at_3_std": -0.029676750441070248, - "nauc_precision_at_5_diff1": -0.11820147403739824, - "nauc_precision_at_5_max": 0.0012091241273901022, - "nauc_precision_at_5_std": -0.0027024137371278, - "nauc_recall_at_1000_diff1": -0.37734982882041684, - "nauc_recall_at_1000_max": -0.09944561157796639, - "nauc_recall_at_1000_std": 0.5693141145347029, - "nauc_recall_at_100_diff1": -0.14823260134885624, - "nauc_recall_at_100_max": 0.0070610596185191644, - "nauc_recall_at_100_std": 0.18890427192465584, - "nauc_recall_at_10_diff1": -0.0839031142002386, - "nauc_recall_at_10_max": 0.005932685093745821, - "nauc_recall_at_10_std": 0.022366941779985956, - "nauc_recall_at_1_diff1": -0.4350127493981227, - "nauc_recall_at_1_max": -0.15259852263953608, - "nauc_recall_at_1_std": 0.0970061577980624, - "nauc_recall_at_20_diff1": -0.08368591470272047, - "nauc_recall_at_20_max": 0.01010481550534646, - "nauc_recall_at_20_std": 0.043779155070841055, - "nauc_recall_at_3_diff1": -0.10537518414674711, - "nauc_recall_at_3_max": 0.00995675521551123, - "nauc_recall_at_3_std": -0.028879437342584083, - "nauc_recall_at_5_diff1": -0.10152978106995561, - "nauc_recall_at_5_max": 0.002831093824884129, - "nauc_recall_at_5_std": -0.0006341965296450934, - "ndcg_at_1": 0.02158, - "ndcg_at_10": 0.15458, - "ndcg_at_100": 0.22831, - "ndcg_at_1000": 0.26227, - "ndcg_at_20": 0.18104, - "ndcg_at_3": 0.09835, - "ndcg_at_5": 0.12423, - "precision_at_1": 0.02158, - "precision_at_10": 0.03199, - "precision_at_100": 0.00686, - "precision_at_1000": 0.00095, - "precision_at_20": 0.02127, - "precision_at_3": 0.0542, - "precision_at_5": 0.04523, - "recall_at_1": 0.01763, - "recall_at_10": 0.31007, - "recall_at_100": 0.6753, - "recall_at_1000": 0.94245, - "recall_at_20": 0.41535, - "recall_at_3": 0.15348, - "recall_at_5": 0.21643 - } - ] - }, - "task_name": "CIRRIT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json b/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json deleted file mode 100644 index a73369f513..0000000000 --- a/results-mieb/BAAI__bge-visualized-base/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "BAAI/bge-visualized-base", "revision": "98db10b10d22620010d06f11733346e1c98c34aa", "release_date": "2024-06-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "vista_loader"} \ No newline at end of file diff --git a/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json b/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json deleted file mode 100644 index f7bfc2691e..0000000000 --- a/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/CIRRIT2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", - "evaluation_time": 626.7223904132843, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.15486, - "map_at_1": 0.04269, - "map_at_10": 0.11306, - "map_at_100": 0.1261, - "map_at_1000": 0.12744, - "map_at_20": 0.11979, - "map_at_3": 0.08855, - "map_at_5": 0.10125, - "mrr_at_1": 0.045563549160671464, - "mrr_at_10": 0.11386700163678554, - "mrr_at_100": 0.126761432509432, - "mrr_at_1000": 0.1280940096115024, - "mrr_at_20": 0.12048092549117054, - "mrr_at_3": 0.08960831334932043, - "mrr_at_5": 0.1021622701838525, - "nauc_map_at_1000_diff1": -0.07791148946970866, - "nauc_map_at_1000_max": 5.143872615612968e-06, - "nauc_map_at_1000_std": 0.04083408199460632, - "nauc_map_at_100_diff1": -0.07731489078138438, - "nauc_map_at_100_max": 3.918221027999972e-05, - "nauc_map_at_100_std": 0.0404276988719768, - "nauc_map_at_10_diff1": -0.07729492524522034, - "nauc_map_at_10_max": 0.0012864822248743548, - "nauc_map_at_10_std": 0.029465548613110498, - "nauc_map_at_1_diff1": -0.1736241959423541, - "nauc_map_at_1_max": -0.07374443402682516, - "nauc_map_at_1_std": -0.04852583988286722, - "nauc_map_at_20_diff1": -0.07680229687469181, - "nauc_map_at_20_max": 0.0010002432624082133, - "nauc_map_at_20_std": 0.03666477523677474, - "nauc_map_at_3_diff1": -0.08922170584380983, - "nauc_map_at_3_max": -0.008861115632143349, - "nauc_map_at_3_std": 0.01094507983539612, - "nauc_map_at_5_diff1": -0.08011065395630808, - "nauc_map_at_5_max": 0.0016127729720874162, - "nauc_map_at_5_std": 0.020427568697340345, - "nauc_mrr_at_1000_diff1": -0.07770330269715102, - "nauc_mrr_at_1000_max": 0.0009131779135089956, - "nauc_mrr_at_1000_std": 0.038979658723853165, - "nauc_mrr_at_100_diff1": -0.0771211519155922, - "nauc_mrr_at_100_max": 0.0009403242167022057, - "nauc_mrr_at_100_std": 0.03858151021299973, - "nauc_mrr_at_10_diff1": -0.07755851577891724, - "nauc_mrr_at_10_max": 0.0023019907938825845, - "nauc_mrr_at_10_std": 0.02763783882670773, - "nauc_mrr_at_1_diff1": -0.1785604338186492, - "nauc_mrr_at_1_max": -0.05415389829551078, - "nauc_mrr_at_1_std": -0.055220086707242946, - "nauc_mrr_at_20_diff1": -0.07661887536133825, - "nauc_mrr_at_20_max": 0.0019978449944686866, - "nauc_mrr_at_20_std": 0.03474258189581495, - "nauc_mrr_at_3_diff1": -0.08926232455695343, - "nauc_mrr_at_3_max": -0.007285948429449863, - "nauc_mrr_at_3_std": 0.008561270537733506, - "nauc_mrr_at_5_diff1": -0.08043180840287993, - "nauc_mrr_at_5_max": 0.0024895489049257397, - "nauc_mrr_at_5_std": 0.018656997997976786, - "nauc_ndcg_at_1000_diff1": -0.07338267741428946, - "nauc_ndcg_at_1000_max": 0.0030417252830778666, - "nauc_ndcg_at_1000_std": 0.09821559898845032, - "nauc_ndcg_at_100_diff1": -0.062286101465836415, - "nauc_ndcg_at_100_max": 0.004482736065877528, - "nauc_ndcg_at_100_std": 0.10277505818112917, - "nauc_ndcg_at_10_diff1": -0.061254511757178484, - "nauc_ndcg_at_10_max": 0.01189902750651278, - "nauc_ndcg_at_10_std": 0.04977362354287121, - "nauc_ndcg_at_1_diff1": -0.18063459374565402, - "nauc_ndcg_at_1_max": -0.05604355931036713, - "nauc_ndcg_at_1_std": -0.054415423322380765, - "nauc_ndcg_at_20_diff1": -0.060054754551758655, - "nauc_ndcg_at_20_max": 0.01086549847664498, - "nauc_ndcg_at_20_std": 0.07039195680705373, - "nauc_ndcg_at_3_diff1": -0.07753134923526465, - "nauc_ndcg_at_3_max": -0.0002772674733751852, - "nauc_ndcg_at_3_std": 0.02141475124849136, - "nauc_ndcg_at_5_diff1": -0.06571664406877105, - "nauc_ndcg_at_5_max": 0.013233449066821962, - "nauc_ndcg_at_5_std": 0.0342274046868592, - "nauc_precision_at_1000_diff1": -0.19656576649998095, - "nauc_precision_at_1000_max": 0.0008923436399845466, - "nauc_precision_at_1000_std": 0.48358706669108, - "nauc_precision_at_100_diff1": -0.052150565375847446, - "nauc_precision_at_100_max": 0.009548025017867385, - "nauc_precision_at_100_std": 0.26024269132004735, - "nauc_precision_at_10_diff1": -0.046839952542220155, - "nauc_precision_at_10_max": 0.0331080559803692, - "nauc_precision_at_10_std": 0.07899523358542766, - "nauc_precision_at_1_diff1": -0.18063459374565402, - "nauc_precision_at_1_max": -0.05604355931036713, - "nauc_precision_at_1_std": -0.054415423322380765, - "nauc_precision_at_20_diff1": -0.0462560523472062, - "nauc_precision_at_20_max": 0.029442616882690585, - "nauc_precision_at_20_std": 0.12415894016012202, - "nauc_precision_at_3_diff1": -0.06709941561395527, - "nauc_precision_at_3_max": 0.024028017239341025, - "nauc_precision_at_3_std": 0.03720152948553412, - "nauc_precision_at_5_diff1": -0.05180083321883926, - "nauc_precision_at_5_max": 0.03899872250699904, - "nauc_precision_at_5_std": 0.05454906284290042, - "nauc_recall_at_1000_diff1": -0.17163650330108518, - "nauc_recall_at_1000_max": -0.043117719901198975, - "nauc_recall_at_1000_std": 0.6414764487355223, - "nauc_recall_at_100_diff1": -0.036997744471389225, - "nauc_recall_at_100_max": 0.0010422392633025453, - "nauc_recall_at_100_std": 0.27951527516127295, - "nauc_recall_at_10_diff1": -0.03608087156730888, - "nauc_recall_at_10_max": 0.026974486642378233, - "nauc_recall_at_10_std": 0.08294573285460217, - "nauc_recall_at_1_diff1": -0.1736241959423541, - "nauc_recall_at_1_max": -0.07374443402682516, - "nauc_recall_at_1_std": -0.04852583988286722, - "nauc_recall_at_20_diff1": -0.03472320683684399, - "nauc_recall_at_20_max": 0.023755047800559374, - "nauc_recall_at_20_std": 0.13014236460370307, - "nauc_recall_at_3_diff1": -0.05544788258407132, - "nauc_recall_at_3_max": 0.015039569782739954, - "nauc_recall_at_3_std": 0.04220096699168211, - "nauc_recall_at_5_diff1": -0.04120717983373631, - "nauc_recall_at_5_max": 0.03222632622898795, - "nauc_recall_at_5_std": 0.05790997866890883, - "ndcg_at_1": 0.0458, - "ndcg_at_10": 0.15486, - "ndcg_at_100": 0.22705, - "ndcg_at_1000": 0.26306, - "ndcg_at_20": 0.17901, - "ndcg_at_3": 0.10353, - "ndcg_at_5": 0.12623, - "precision_at_1": 0.0458, - "precision_at_10": 0.02976, - "precision_at_100": 0.00658, - "precision_at_1000": 0.00094, - "precision_at_20": 0.01966, - "precision_at_3": 0.05068, - "precision_at_5": 0.04158, - "recall_at_1": 0.04269, - "recall_at_10": 0.28921, - "recall_at_100": 0.64796, - "recall_at_1000": 0.93237, - "recall_at_20": 0.38413, - "recall_at_3": 0.14584, - "recall_at_5": 0.20064 - } - ] - }, - "task_name": "CIRRIT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json b/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json deleted file mode 100644 index a2b05c9208..0000000000 --- a/results-mieb/BAAI__bge-visualized-m3/98db10b10d22620010d06f11733346e1c98c34aa/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "BAAI/bge-visualized-m3", "revision": "98db10b10d22620010d06f11733346e1c98c34aa", "release_date": "2024-06-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "vista_loader"} \ No newline at end of file diff --git a/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MNIST.json b/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MNIST.json deleted file mode 100644 index 3c6bf4be39..0000000000 --- a/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MNIST.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 36.933613777160645, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.7848, - "f1": 0.7815922902217035, - "f1_weighted": 0.7830608860261875, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7848, - "scores_per_experiment": [ - { - "accuracy": 0.7848, - "f1": 0.7815922902217035, - "f1_weighted": 0.7830608860261875 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json b/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json deleted file mode 100644 index 243b9cdfb3..0000000000 --- a/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", - "evaluation_time": 123.7929699420929, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.5808, - "cv_recall_at_10": 0.8748, - "cv_recall_at_100": 0.9944, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.9314, - "cv_recall_at_3": 0.7374, - "cv_recall_at_5": 0.7984, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.51767, - "map_at_1": 0.1164, - "map_at_10": 0.4046, - "map_at_100": 0.45014, - "map_at_1000": 0.45303, - "map_at_20": 0.43031, - "map_at_3": 0.26852, - "map_at_5": 0.35112, - "mrr_at_1": 0.5808, - "mrr_at_10": 0.6744126190476204, - "mrr_at_100": 0.6801586440797971, - "mrr_at_1000": 0.6801960917779327, - "mrr_at_20": 0.6784373254841141, - "mrr_at_3": 0.6501000000000023, - "mrr_at_5": 0.6640700000000009, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.18012204881955546, - "nauc_cv_recall_at_100_max": 0.4690209417100159, - "nauc_cv_recall_at_100_std": -0.28182939842600496, - "nauc_cv_recall_at_10_diff1": 0.40434217822752977, - "nauc_cv_recall_at_10_max": 0.4848002316155178, - "nauc_cv_recall_at_10_std": -0.48703213512130683, - "nauc_cv_recall_at_1_diff1": 0.4279641274466745, - "nauc_cv_recall_at_1_max": 0.40462849354795966, - "nauc_cv_recall_at_1_std": -0.3439104150545278, - "nauc_cv_recall_at_20_diff1": 0.40948624347698215, - "nauc_cv_recall_at_20_max": 0.4677421989203846, - "nauc_cv_recall_at_20_std": -0.49209888036847343, - "nauc_cv_recall_at_3_diff1": 0.39274864442318025, - "nauc_cv_recall_at_3_max": 0.43154140979989203, - "nauc_cv_recall_at_3_std": -0.41178966708778997, - "nauc_cv_recall_at_5_diff1": 0.3789987569351505, - "nauc_cv_recall_at_5_max": 0.441581412804759, - "nauc_cv_recall_at_5_std": -0.4429600034948794, - "nauc_map_at_1000_diff1": 0.2551299778398228, - "nauc_map_at_1000_max": 0.45024216293039104, - "nauc_map_at_1000_std": -0.3158410835982976, - "nauc_map_at_100_diff1": 0.2551180137900688, - "nauc_map_at_100_max": 0.4507781940007651, - "nauc_map_at_100_std": -0.3165456782869218, - "nauc_map_at_10_diff1": 0.2470592559213462, - "nauc_map_at_10_max": 0.44129809055151087, - "nauc_map_at_10_std": -0.31273315656889844, - "nauc_map_at_1_diff1": 0.42575307120531464, - "nauc_map_at_1_max": 0.4048588519120006, - "nauc_map_at_1_std": -0.34378381546020276, - "nauc_map_at_20_diff1": 0.2516873789249498, - "nauc_map_at_20_max": 0.4484210309890165, - "nauc_map_at_20_std": -0.31683283226512, - "nauc_map_at_3_diff1": 0.2826357897945762, - "nauc_map_at_3_max": 0.42426410796847175, - "nauc_map_at_3_std": -0.30647778109153573, - "nauc_map_at_5_diff1": 0.24811920497848947, - "nauc_map_at_5_max": 0.43227047102448485, - "nauc_map_at_5_std": -0.30545952750772826, - "nauc_mrr_at_1000_diff1": 0.41413837092161987, - "nauc_mrr_at_1000_max": 0.41676194427293417, - "nauc_mrr_at_1000_std": -0.37106779073618856, - "nauc_mrr_at_100_diff1": 0.4140964307246391, - "nauc_mrr_at_100_max": 0.4167634806381506, - "nauc_mrr_at_100_std": -0.37107069460512243, - "nauc_mrr_at_10_diff1": 0.4139845884692961, - "nauc_mrr_at_10_max": 0.41838787547148604, - "nauc_mrr_at_10_std": -0.373368042710646, - "nauc_mrr_at_1_diff1": 0.4279641274466745, - "nauc_mrr_at_1_max": 0.40462849354795966, - "nauc_mrr_at_1_std": -0.3439104150545278, - "nauc_mrr_at_20_diff1": 0.41422005719482136, - "nauc_mrr_at_20_max": 0.4169421834449322, - "nauc_mrr_at_20_std": -0.37195479331593423, - "nauc_mrr_at_3_diff1": 0.4129850334584388, - "nauc_mrr_at_3_max": 0.4153387916886552, - "nauc_mrr_at_3_std": -0.36938113677751916, - "nauc_mrr_at_5_diff1": 0.41133587690415013, - "nauc_mrr_at_5_max": 0.41658201667599837, - "nauc_mrr_at_5_std": -0.3726272419181216, - "nauc_ndcg_at_1000_diff1": 0.30085799504287825, - "nauc_ndcg_at_1000_max": 0.45175503659717203, - "nauc_ndcg_at_1000_std": -0.33497432412179057, - "nauc_ndcg_at_100_diff1": 0.2971007796129066, - "nauc_ndcg_at_100_max": 0.45770544154865983, - "nauc_ndcg_at_100_std": -0.33709187267575114, - "nauc_ndcg_at_10_diff1": 0.2796457903219738, - "nauc_ndcg_at_10_max": 0.4445160485969821, - "nauc_ndcg_at_10_std": -0.3377133809957326, - "nauc_ndcg_at_1_diff1": 0.4258548467189445, - "nauc_ndcg_at_1_max": 0.40505598965450235, - "nauc_ndcg_at_1_std": -0.3441811229562816, - "nauc_ndcg_at_20_diff1": 0.2865070753592145, - "nauc_ndcg_at_20_max": 0.4537504419231763, - "nauc_ndcg_at_20_std": -0.34287699080243245, - "nauc_ndcg_at_3_diff1": 0.3084620615728682, - "nauc_ndcg_at_3_max": 0.42489968327164407, - "nauc_ndcg_at_3_std": -0.32533589756821285, - "nauc_ndcg_at_5_diff1": 0.2802294933259117, - "nauc_ndcg_at_5_max": 0.43204705198340854, - "nauc_ndcg_at_5_std": -0.3296170450292348, - "nauc_precision_at_1000_diff1": 0.07501926945277763, - "nauc_precision_at_1000_max": 0.34712587495850716, - "nauc_precision_at_1000_std": 0.47869080518110013, - "nauc_precision_at_100_diff1": 0.2148427857809954, - "nauc_precision_at_100_max": 0.43306705779365773, - "nauc_precision_at_100_std": -0.25062075823160657, - "nauc_precision_at_10_diff1": 0.22474307723586737, - "nauc_precision_at_10_max": 0.43423322714260504, - "nauc_precision_at_10_std": -0.3212543682360079, - "nauc_precision_at_1_diff1": 0.4258548467189445, - "nauc_precision_at_1_max": 0.40505598965450235, - "nauc_precision_at_1_std": -0.3441811229562816, - "nauc_precision_at_20_diff1": 0.22416005971431854, - "nauc_precision_at_20_max": 0.4371201909814698, - "nauc_precision_at_20_std": -0.31708914748633776, - "nauc_precision_at_3_diff1": 0.2745706041175571, - "nauc_precision_at_3_max": 0.4278350852940981, - "nauc_precision_at_3_std": -0.31711638180222146, - "nauc_precision_at_5_diff1": 0.2349320970775645, - "nauc_precision_at_5_max": 0.42831671123295334, - "nauc_precision_at_5_std": -0.3190019134502589, - "nauc_recall_at_1000_diff1": 0.10507922975953841, - "nauc_recall_at_1000_max": 0.3930871865654486, - "nauc_recall_at_1000_std": 0.5020094994519381, - "nauc_recall_at_100_diff1": 0.21661240783376975, - "nauc_recall_at_100_max": 0.43497626991291494, - "nauc_recall_at_100_std": -0.25185551819463464, - "nauc_recall_at_10_diff1": 0.22519163965451483, - "nauc_recall_at_10_max": 0.43446322405629173, - "nauc_recall_at_10_std": -0.32141360644158223, - "nauc_recall_at_1_diff1": 0.42575307120531464, - "nauc_recall_at_1_max": 0.4048588519120006, - "nauc_recall_at_1_std": -0.34378381546020276, - "nauc_recall_at_20_diff1": 0.22475741497205412, - "nauc_recall_at_20_max": 0.43757220630004384, - "nauc_recall_at_20_std": -0.31738515586158633, - "nauc_recall_at_3_diff1": 0.2746377320730141, - "nauc_recall_at_3_max": 0.42775643120193435, - "nauc_recall_at_3_std": -0.31711052977780496, - "nauc_recall_at_5_diff1": 0.23517626729390337, - "nauc_recall_at_5_max": 0.4284668289611237, - "nauc_recall_at_5_std": -0.3189996126749059, - "ndcg_at_1": 0.5816, - "ndcg_at_10": 0.51767, - "ndcg_at_100": 0.6347, - "ndcg_at_1000": 0.66318, - "ndcg_at_20": 0.56556, - "ndcg_at_3": 0.50923, - "ndcg_at_5": 0.44845, - "precision_at_1": 0.5816, - "precision_at_10": 0.2667, - "precision_at_100": 0.04328, - "precision_at_1000": 0.00497, - "precision_at_20": 0.16118, - "precision_at_3": 0.4874, - "precision_at_5": 0.40752, - "recall_at_1": 0.1164, - "recall_at_10": 0.53367, - "recall_at_100": 0.86597, - "recall_at_1000": 0.99448, - "recall_at_20": 0.64502, - "recall_at_3": 0.29261, - "recall_at_5": 0.40775 - } - ] - }, - "task_name": "MSCOCOI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json b/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json deleted file mode 100644 index a639fc4d52..0000000000 --- a/results-mieb/EVA02-CLIP-B-16/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "EVA02-CLIP-B-16", "revision": "11afd202f2ae80869d6cef18b1ec775e79bd8d12", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "evaclip_loader"} \ No newline at end of file diff --git a/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json b/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json deleted file mode 100644 index fad26b40d9..0000000000 --- a/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/MSCOCOI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", - "evaluation_time": 188.94002294540405, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.6288, - "cv_recall_at_10": 0.8992, - "cv_recall_at_100": 0.9956, - "cv_recall_at_1000": 0.9998, - "cv_recall_at_20": 0.9472, - "cv_recall_at_3": 0.7738, - "cv_recall_at_5": 0.8342, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56069, - "map_at_1": 0.12554, - "map_at_10": 0.44873, - "map_at_100": 0.4927, - "map_at_1000": 0.49528, - "map_at_20": 0.47383, - "map_at_3": 0.29637, - "map_at_5": 0.39313, - "mrr_at_1": 0.6288, - "mrr_at_10": 0.7158076984126989, - "mrr_at_100": 0.7205783122922634, - "mrr_at_1000": 0.7206069527616489, - "mrr_at_20": 0.7192201942055193, - "mrr_at_3": 0.6930666666666684, - "mrr_at_5": 0.7068966666666667, - "nauc_cv_recall_at_1000_diff1": 0.12278244631152727, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": 0.46403106697222163, - "nauc_cv_recall_at_100_max": 0.729055258466997, - "nauc_cv_recall_at_100_std": -0.30457940752058904, - "nauc_cv_recall_at_10_diff1": 0.3904889248253057, - "nauc_cv_recall_at_10_max": 0.5221902215034939, - "nauc_cv_recall_at_10_std": -0.5924430292420227, - "nauc_cv_recall_at_1_diff1": 0.44292967763077234, - "nauc_cv_recall_at_1_max": 0.47715154630664924, - "nauc_cv_recall_at_1_std": -0.40620411297806974, - "nauc_cv_recall_at_20_diff1": 0.43476607814843, - "nauc_cv_recall_at_20_max": 0.5567421413009648, - "nauc_cv_recall_at_20_std": -0.5714798545680903, - "nauc_cv_recall_at_3_diff1": 0.4261256966503181, - "nauc_cv_recall_at_3_max": 0.49484976574587286, - "nauc_cv_recall_at_3_std": -0.4813401812617564, - "nauc_cv_recall_at_5_diff1": 0.39770539047919967, - "nauc_cv_recall_at_5_max": 0.4974364426662427, - "nauc_cv_recall_at_5_std": -0.5283547497680571, - "nauc_map_at_1000_diff1": 0.23277589985950517, - "nauc_map_at_1000_max": 0.4750327210959742, - "nauc_map_at_1000_std": -0.3607727535310882, - "nauc_map_at_100_diff1": 0.23262810652639354, - "nauc_map_at_100_max": 0.475256107901075, - "nauc_map_at_100_std": -0.36143324382132475, - "nauc_map_at_10_diff1": 0.22389555975001613, - "nauc_map_at_10_max": 0.46963591043048414, - "nauc_map_at_10_std": -0.3557278665509837, - "nauc_map_at_1_diff1": 0.44545079244943503, - "nauc_map_at_1_max": 0.47785301021506615, - "nauc_map_at_1_std": -0.4057042536024991, - "nauc_map_at_20_diff1": 0.22998051862002486, - "nauc_map_at_20_max": 0.47305652893594374, - "nauc_map_at_20_std": -0.3603168960770869, - "nauc_map_at_3_diff1": 0.27812725553418577, - "nauc_map_at_3_max": 0.47306973520598594, - "nauc_map_at_3_std": -0.3570706186087874, - "nauc_map_at_5_diff1": 0.2280217674026679, - "nauc_map_at_5_max": 0.4682093275293784, - "nauc_map_at_5_std": -0.3423580014696908, - "nauc_mrr_at_1000_diff1": 0.4353997803612837, - "nauc_mrr_at_1000_max": 0.4829466674716431, - "nauc_mrr_at_1000_std": -0.43724506589884127, - "nauc_mrr_at_100_diff1": 0.4354030270991679, - "nauc_mrr_at_100_max": 0.48297038396857644, - "nauc_mrr_at_100_std": -0.4372610054743306, - "nauc_mrr_at_10_diff1": 0.4339442584396937, - "nauc_mrr_at_10_max": 0.48330696344702784, - "nauc_mrr_at_10_std": -0.4407280111594318, - "nauc_mrr_at_1_diff1": 0.44292967763077234, - "nauc_mrr_at_1_max": 0.47715154630664924, - "nauc_mrr_at_1_std": -0.40620411297806974, - "nauc_mrr_at_20_diff1": 0.435424981370453, - "nauc_mrr_at_20_max": 0.483314664468245, - "nauc_mrr_at_20_std": -0.43817326450406274, - "nauc_mrr_at_3_diff1": 0.4367712306037216, - "nauc_mrr_at_3_max": 0.4825955486267395, - "nauc_mrr_at_3_std": -0.4343397489713263, - "nauc_mrr_at_5_diff1": 0.4330194072228175, - "nauc_mrr_at_5_max": 0.48215440797790354, - "nauc_mrr_at_5_std": -0.4393099993887148, - "nauc_ndcg_at_1000_diff1": 0.286919936116174, - "nauc_ndcg_at_1000_max": 0.48455380332723746, - "nauc_ndcg_at_1000_std": -0.3889019114242068, - "nauc_ndcg_at_100_diff1": 0.281607082470054, - "nauc_ndcg_at_100_max": 0.4862363359092808, - "nauc_ndcg_at_100_std": -0.39104186341881614, - "nauc_ndcg_at_10_diff1": 0.25974640251443, - "nauc_ndcg_at_10_max": 0.4746176277404106, - "nauc_ndcg_at_10_std": -0.3877117875687834, - "nauc_ndcg_at_1_diff1": 0.44645817200139365, - "nauc_ndcg_at_1_max": 0.47805516562130335, - "nauc_ndcg_at_1_std": -0.40610800775079414, - "nauc_ndcg_at_20_diff1": 0.2696500915937151, - "nauc_ndcg_at_20_max": 0.47845385412507335, - "nauc_ndcg_at_20_std": -0.39360516882367463, - "nauc_ndcg_at_3_diff1": 0.30847439164717777, - "nauc_ndcg_at_3_max": 0.4761874477195775, - "nauc_ndcg_at_3_std": -0.3776639638945563, - "nauc_ndcg_at_5_diff1": 0.26569027870667994, - "nauc_ndcg_at_5_max": 0.47347834616528645, - "nauc_ndcg_at_5_std": -0.37106994299369317, - "nauc_precision_at_1000_diff1": 0.1236733225400447, - "nauc_precision_at_1000_max": 0.308697179177513, - "nauc_precision_at_1000_std": 0.40753732685737887, - "nauc_precision_at_100_diff1": 0.19092682607623723, - "nauc_precision_at_100_max": 0.4250180546261426, - "nauc_precision_at_100_std": -0.30733698696941303, - "nauc_precision_at_10_diff1": 0.19311336478366353, - "nauc_precision_at_10_max": 0.44355904903183346, - "nauc_precision_at_10_std": -0.36759162233395865, - "nauc_precision_at_1_diff1": 0.44645817200139365, - "nauc_precision_at_1_max": 0.47805516562130335, - "nauc_precision_at_1_std": -0.40610800775079414, - "nauc_precision_at_20_diff1": 0.19966070829030882, - "nauc_precision_at_20_max": 0.4326136796798698, - "nauc_precision_at_20_std": -0.3669436772279273, - "nauc_precision_at_3_diff1": 0.2708800654517302, - "nauc_precision_at_3_max": 0.4729051262633524, - "nauc_precision_at_3_std": -0.36661790608555717, - "nauc_precision_at_5_diff1": 0.21323688684871933, - "nauc_precision_at_5_max": 0.4617539145838835, - "nauc_precision_at_5_std": -0.3535506470153617, - "nauc_recall_at_1000_diff1": 0.10032584462359877, - "nauc_recall_at_1000_max": 0.35548028735302944, - "nauc_recall_at_1000_std": 0.42345509632425254, - "nauc_recall_at_100_diff1": 0.19003462550964206, - "nauc_recall_at_100_max": 0.42683017998778644, - "nauc_recall_at_100_std": -0.3088713370152116, - "nauc_recall_at_10_diff1": 0.19275938507453244, - "nauc_recall_at_10_max": 0.44376915738937206, - "nauc_recall_at_10_std": -0.3680795761536703, - "nauc_recall_at_1_diff1": 0.44545079244943503, - "nauc_recall_at_1_max": 0.47785301021506615, - "nauc_recall_at_1_std": -0.4057042536024991, - "nauc_recall_at_20_diff1": 0.1990748921216041, - "nauc_recall_at_20_max": 0.4329712829247703, - "nauc_recall_at_20_std": -0.3674964699732755, - "nauc_recall_at_3_diff1": 0.2706407468460583, - "nauc_recall_at_3_max": 0.47295729285142457, - "nauc_recall_at_3_std": -0.3667117594872674, - "nauc_recall_at_5_diff1": 0.2129174058601686, - "nauc_recall_at_5_max": 0.4618647334514804, - "nauc_recall_at_5_std": -0.3537533728865379, - "ndcg_at_1": 0.6274, - "ndcg_at_10": 0.56069, - "ndcg_at_100": 0.67034, - "ndcg_at_1000": 0.695, - "ndcg_at_20": 0.60649, - "ndcg_at_3": 0.55439, - "ndcg_at_5": 0.49188, - "precision_at_1": 0.6274, - "precision_at_10": 0.28744, - "precision_at_100": 0.04423, - "precision_at_1000": 0.00498, - "precision_at_20": 0.17036, - "precision_at_3": 0.53253, - "precision_at_5": 0.44992, - "recall_at_1": 0.12554, - "recall_at_10": 0.57515, - "recall_at_100": 0.88496, - "recall_at_1000": 0.99608, - "recall_at_20": 0.68176, - "recall_at_3": 0.31966, - "recall_at_5": 0.45013 - } - ] - }, - "task_name": "MSCOCOI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json b/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json deleted file mode 100644 index 038364051d..0000000000 --- a/results-mieb/EVA02-CLIP-L-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "EVA02-CLIP-L-14", "revision": "11afd202f2ae80869d6cef18b1ec775e79bd8d12", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "evaclip_loader"} \ No newline at end of file diff --git a/results-mieb/EVA02-CLIP-bigE-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json b/results-mieb/EVA02-CLIP-bigE-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json deleted file mode 100644 index 5f33638645..0000000000 --- a/results-mieb/EVA02-CLIP-bigE-14/11afd202f2ae80869d6cef18b1ec775e79bd8d12/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "EVA02-CLIP-bigE-14", "revision": "11afd202f2ae80869d6cef18b1ec775e79bd8d12", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "evaclip_loader"} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/Fashion200kI2TRetrieval.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/Fashion200kI2TRetrieval.json deleted file mode 100644 index 9225a6062a..0000000000 --- a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/Fashion200kI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "96a313715ecf67f5dfe70c4fa52406bc7bdfbeee", - "evaluation_time": 181.5803382396698, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.03886, - "cv_recall_at_10": 0.15484, - "cv_recall_at_100": 0.39701, - "cv_recall_at_1000": 0.7658, - "cv_recall_at_20": 0.21334, - "cv_recall_at_3": 0.07568, - "cv_recall_at_5": 0.1035, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.08809, - "map_at_1": 0.03886, - "map_at_10": 0.06791, - "map_at_100": 0.07632, - "map_at_1000": 0.07774, - "map_at_20": 0.07197, - "map_at_3": 0.05489, - "map_at_5": 0.06124, - "mrr_at_1": 0.03886275311924729, - "mrr_at_10": 0.06791249874191167, - "mrr_at_100": 0.07631679834894635, - "mrr_at_1000": 0.07773539145986881, - "mrr_at_20": 0.0719660846478045, - "mrr_at_3": 0.05488511624735805, - "mrr_at_5": 0.06123610827026652, - "nauc_cv_recall_at_1000_diff1": 0.0218733952063869, - "nauc_cv_recall_at_1000_max": 0.16706592598167, - "nauc_cv_recall_at_1000_std": 0.33597938688032986, - "nauc_cv_recall_at_100_diff1": 0.07193545061359337, - "nauc_cv_recall_at_100_max": 0.03727221469537029, - "nauc_cv_recall_at_100_std": 0.1561269787000631, - "nauc_cv_recall_at_10_diff1": 0.0797701915257812, - "nauc_cv_recall_at_10_max": 0.020430887616611652, - "nauc_cv_recall_at_10_std": 0.06318639559887353, - "nauc_cv_recall_at_1_diff1": 0.18888757745716797, - "nauc_cv_recall_at_1_max": 0.03510598124425557, - "nauc_cv_recall_at_1_std": -0.0011818918393113542, - "nauc_cv_recall_at_20_diff1": 0.07098442271206883, - "nauc_cv_recall_at_20_max": 0.017202068195442434, - "nauc_cv_recall_at_20_std": 0.09427426491566825, - "nauc_cv_recall_at_3_diff1": 0.14514396355155457, - "nauc_cv_recall_at_3_max": 0.022088685493555183, - "nauc_cv_recall_at_3_std": 0.02219092083634193, - "nauc_cv_recall_at_5_diff1": 0.11843084765533575, - "nauc_cv_recall_at_5_max": 0.028181652993821218, - "nauc_cv_recall_at_5_std": 0.03378133752500267, - "nauc_map_at_1000_diff1": 0.1318292645003046, - "nauc_map_at_1000_max": 0.025968459599028312, - "nauc_map_at_1000_std": 0.03596161753365205, - "nauc_map_at_100_diff1": 0.13207865359812196, - "nauc_map_at_100_max": 0.02503274769433611, - "nauc_map_at_100_std": 0.034811909972038466, - "nauc_map_at_10_diff1": 0.13675377201318661, - "nauc_map_at_10_max": 0.023988302044714947, - "nauc_map_at_10_std": 0.024038623147938202, - "nauc_map_at_1_diff1": 0.18888757745716797, - "nauc_map_at_1_max": 0.03510598124425557, - "nauc_map_at_1_std": -0.0011818918393113542, - "nauc_map_at_20_diff1": 0.13291861173066144, - "nauc_map_at_20_max": 0.02328223207465244, - "nauc_map_at_20_std": 0.030123558300912807, - "nauc_map_at_3_diff1": 0.161277155146534, - "nauc_map_at_3_max": 0.024402585884692875, - "nauc_map_at_3_std": 0.010083727677318252, - "nauc_map_at_5_diff1": 0.1491311529395408, - "nauc_map_at_5_max": 0.026749124262949692, - "nauc_map_at_5_std": 0.015564562618014838, - "nauc_mrr_at_1000_diff1": 0.13183050469281174, - "nauc_mrr_at_1000_max": 0.025968889811347606, - "nauc_mrr_at_1000_std": 0.035962054245207556, - "nauc_mrr_at_100_diff1": 0.13207988989141017, - "nauc_mrr_at_100_max": 0.02503317589140467, - "nauc_mrr_at_100_std": 0.03481234450834155, - "nauc_mrr_at_10_diff1": 0.13675377201318661, - "nauc_mrr_at_10_max": 0.023988302044714947, - "nauc_mrr_at_10_std": 0.024038623147938202, - "nauc_mrr_at_1_diff1": 0.18888757745716797, - "nauc_mrr_at_1_max": 0.03510598124425557, - "nauc_mrr_at_1_std": -0.0011818918393113542, - "nauc_mrr_at_20_diff1": 0.13291861173066144, - "nauc_mrr_at_20_max": 0.02328223207465244, - "nauc_mrr_at_20_std": 0.030123558300912807, - "nauc_mrr_at_3_diff1": 0.161277155146534, - "nauc_mrr_at_3_max": 0.024402585884692875, - "nauc_mrr_at_3_std": 0.010083727677318252, - "nauc_mrr_at_5_diff1": 0.1491311529395408, - "nauc_mrr_at_5_max": 0.026749124262949692, - "nauc_mrr_at_5_std": 0.015564562618014838, - "nauc_ndcg_at_1000_diff1": 0.10002387998068846, - "nauc_ndcg_at_1000_max": 0.04997596715187386, - "nauc_ndcg_at_1000_std": 0.10395875200499329, - "nauc_ndcg_at_100_diff1": 0.10528225366714257, - "nauc_ndcg_at_100_max": 0.028766853087252058, - "nauc_ndcg_at_100_std": 0.08065845933284391, - "nauc_ndcg_at_10_diff1": 0.11527617893060818, - "nauc_ndcg_at_10_max": 0.022532453546031717, - "nauc_ndcg_at_10_std": 0.038512971156843226, - "nauc_ndcg_at_1_diff1": 0.18888757745716797, - "nauc_ndcg_at_1_max": 0.03510598124425557, - "nauc_ndcg_at_1_std": -0.0011818918393113542, - "nauc_ndcg_at_20_diff1": 0.10745173167011737, - "nauc_ndcg_at_20_max": 0.02078812380101046, - "nauc_ndcg_at_20_std": 0.05495270418039658, - "nauc_ndcg_at_3_diff1": 0.15602598314224742, - "nauc_ndcg_at_3_max": 0.023546212034123336, - "nauc_ndcg_at_3_std": 0.013879753309827844, - "nauc_ndcg_at_5_diff1": 0.13832101043102465, - "nauc_ndcg_at_5_max": 0.027109131323799444, - "nauc_ndcg_at_5_std": 0.021889859067966443, - "nauc_precision_at_1000_diff1": 0.02187339520638577, - "nauc_precision_at_1000_max": 0.1670659259816692, - "nauc_precision_at_1000_std": 0.3359793868803299, - "nauc_precision_at_100_diff1": 0.0719354506135936, - "nauc_precision_at_100_max": 0.03727221469537065, - "nauc_precision_at_100_std": 0.15612697870006317, - "nauc_precision_at_10_diff1": 0.07977019152578149, - "nauc_precision_at_10_max": 0.020430887616611822, - "nauc_precision_at_10_std": 0.06318639559887362, - "nauc_precision_at_1_diff1": 0.18888757745716797, - "nauc_precision_at_1_max": 0.03510598124425557, - "nauc_precision_at_1_std": -0.0011818918393113542, - "nauc_precision_at_20_diff1": 0.0709844227120689, - "nauc_precision_at_20_max": 0.017202068195442757, - "nauc_precision_at_20_std": 0.09427426491566834, - "nauc_precision_at_3_diff1": 0.14514396355155454, - "nauc_precision_at_3_max": 0.022088685493555214, - "nauc_precision_at_3_std": 0.022190920836341964, - "nauc_precision_at_5_diff1": 0.11843084765533585, - "nauc_precision_at_5_max": 0.028181652993821266, - "nauc_precision_at_5_std": 0.03378133752500258, - "nauc_recall_at_1000_diff1": 0.0218733952063869, - "nauc_recall_at_1000_max": 0.16706592598167, - "nauc_recall_at_1000_std": 0.33597938688032986, - "nauc_recall_at_100_diff1": 0.07193545061359337, - "nauc_recall_at_100_max": 0.03727221469537029, - "nauc_recall_at_100_std": 0.1561269787000631, - "nauc_recall_at_10_diff1": 0.0797701915257812, - "nauc_recall_at_10_max": 0.020430887616611652, - "nauc_recall_at_10_std": 0.06318639559887353, - "nauc_recall_at_1_diff1": 0.18888757745716797, - "nauc_recall_at_1_max": 0.03510598124425557, - "nauc_recall_at_1_std": -0.0011818918393113542, - "nauc_recall_at_20_diff1": 0.07098442271206883, - "nauc_recall_at_20_max": 0.017202068195442434, - "nauc_recall_at_20_std": 0.09427426491566825, - "nauc_recall_at_3_diff1": 0.14514396355155457, - "nauc_recall_at_3_max": 0.022088685493555183, - "nauc_recall_at_3_std": 0.02219092083634193, - "nauc_recall_at_5_diff1": 0.11843084765533575, - "nauc_recall_at_5_max": 0.028181652993821218, - "nauc_recall_at_5_std": 0.03378133752500267, - "ndcg_at_1": 0.03886, - "ndcg_at_10": 0.08809, - "ndcg_at_100": 0.13619, - "ndcg_at_1000": 0.18105, - "ndcg_at_20": 0.10288, - "ndcg_at_3": 0.06022, - "ndcg_at_5": 0.07167, - "precision_at_1": 0.03886, - "precision_at_10": 0.01548, - "precision_at_100": 0.00397, - "precision_at_1000": 0.00077, - "precision_at_20": 0.01067, - "precision_at_3": 0.02523, - "precision_at_5": 0.0207, - "recall_at_1": 0.03886, - "recall_at_10": 0.15484, - "recall_at_100": 0.39701, - "recall_at_1000": 0.7658, - "recall_at_20": 0.21334, - "recall_at_3": 0.07568, - "recall_at_5": 0.1035 - } - ] - }, - "task_name": "Fashion200kI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNIST.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNIST.json deleted file mode 100644 index e0ac824a95..0000000000 --- a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 297.4648160934448, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.85734, - "f1": 0.8534562890978588, - "f1_weighted": 0.8556807192174982, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.85734, - "scores_per_experiment": [ - { - "accuracy": 0.8733, - "f1": 0.8705401169868281, - "f1_weighted": 0.8720762577071195 - }, - { - "accuracy": 0.8022, - "f1": 0.7946331419529729, - "f1_weighted": 0.7995418875379899 - }, - { - "accuracy": 0.8581, - "f1": 0.8542121810766361, - "f1_weighted": 0.8563303310872925 - }, - { - "accuracy": 0.878, - "f1": 0.8754710153202587, - "f1_weighted": 0.8767339121591414 - }, - { - "accuracy": 0.8751, - "f1": 0.8724249901525976, - "f1_weighted": 0.8737212075959484 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNISTZeroShot.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNISTZeroShot.json deleted file mode 100644 index b058c0b7a3..0000000000 --- a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/MNISTZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 231.2214720249176, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.5762, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5762 - } - ] - }, - "task_name": "MNISTZeroShot" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/NIGHTSI2IRetrieval.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/NIGHTSI2IRetrieval.json deleted file mode 100644 index 238904b1c3..0000000000 --- a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/NIGHTSI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", - "evaluation_time": 1032.8710505962372, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.07925, - "cv_recall_at_10": 0.48821, - "cv_recall_at_100": 0.9316, - "cv_recall_at_1000": 0.99481, - "cv_recall_at_20": 0.69764, - "cv_recall_at_3": 0.20142, - "cv_recall_at_5": 0.30708, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.24968, - "map_at_1": 0.07925, - "map_at_10": 0.17764, - "map_at_100": 0.19909, - "map_at_1000": 0.19941, - "map_at_20": 0.1922, - "map_at_3": 0.12995, - "map_at_5": 0.15373, - "mrr_at_1": 0.07924528301886792, - "mrr_at_10": 0.1776362683438151, - "mrr_at_100": 0.19909399968942385, - "mrr_at_1000": 0.19940749680294112, - "mrr_at_20": 0.1921998785828074, - "mrr_at_3": 0.12995283018867934, - "mrr_at_5": 0.15372641509433924, - "nauc_cv_recall_at_1000_diff1": -0.6306765130294512, - "nauc_cv_recall_at_1000_max": 0.4255156608097682, - "nauc_cv_recall_at_1000_std": 0.8273066802478575, - "nauc_cv_recall_at_100_diff1": -0.20477156379793215, - "nauc_cv_recall_at_100_max": 0.3289094948324168, - "nauc_cv_recall_at_100_std": 0.3540262081844245, - "nauc_cv_recall_at_10_diff1": -0.0002766589364904998, - "nauc_cv_recall_at_10_max": 0.05573488312411784, - "nauc_cv_recall_at_10_std": -0.08384768736288326, - "nauc_cv_recall_at_1_diff1": 0.0778329308648601, - "nauc_cv_recall_at_1_max": 0.017467451160770153, - "nauc_cv_recall_at_1_std": -0.09230422783548616, - "nauc_cv_recall_at_20_diff1": -0.03663843577705176, - "nauc_cv_recall_at_20_max": 0.09299368334901667, - "nauc_cv_recall_at_20_std": -0.06832906783953004, - "nauc_cv_recall_at_3_diff1": 0.031859526758706465, - "nauc_cv_recall_at_3_max": 0.0014239615902498597, - "nauc_cv_recall_at_3_std": -0.11472573740325614, - "nauc_cv_recall_at_5_diff1": 0.008440048599030739, - "nauc_cv_recall_at_5_max": 0.007294935526259778, - "nauc_cv_recall_at_5_std": -0.09334608645063842, - "nauc_map_at_1000_diff1": 0.03606146943658301, - "nauc_map_at_1000_max": 0.023462582774634224, - "nauc_map_at_1000_std": -0.09234216583449868, - "nauc_map_at_100_diff1": 0.035779413386981485, - "nauc_map_at_100_max": 0.0238874941734106, - "nauc_map_at_100_std": -0.09185556818208644, - "nauc_map_at_10_diff1": 0.03806883475990293, - "nauc_map_at_10_max": 0.020307608194289003, - "nauc_map_at_10_std": -0.09595197128913463, - "nauc_map_at_1_diff1": 0.0778329308648601, - "nauc_map_at_1_max": 0.017467451160770153, - "nauc_map_at_1_std": -0.09230422783548616, - "nauc_map_at_20_diff1": 0.03544017675471238, - "nauc_map_at_20_max": 0.02225673468383308, - "nauc_map_at_20_std": -0.09543789353271984, - "nauc_map_at_3_diff1": 0.05000879822749758, - "nauc_map_at_3_max": 0.00721164104466641, - "nauc_map_at_3_std": -0.10583005601628986, - "nauc_map_at_5_diff1": 0.04007888239793958, - "nauc_map_at_5_max": 0.009848450000869181, - "nauc_map_at_5_std": -0.09855947438234418, - "nauc_mrr_at_1000_diff1": 0.03606146943658301, - "nauc_mrr_at_1000_max": 0.023462582774634224, - "nauc_mrr_at_1000_std": -0.09234216583449868, - "nauc_mrr_at_100_diff1": 0.035779413386981485, - "nauc_mrr_at_100_max": 0.0238874941734106, - "nauc_mrr_at_100_std": -0.09185556818208644, - "nauc_mrr_at_10_diff1": 0.03806883475990293, - "nauc_mrr_at_10_max": 0.020307608194289003, - "nauc_mrr_at_10_std": -0.09595197128913463, - "nauc_mrr_at_1_diff1": 0.0778329308648601, - "nauc_mrr_at_1_max": 0.017467451160770153, - "nauc_mrr_at_1_std": -0.09230422783548616, - "nauc_mrr_at_20_diff1": 0.03544017675471238, - "nauc_mrr_at_20_max": 0.02225673468383308, - "nauc_mrr_at_20_std": -0.09543789353271984, - "nauc_mrr_at_3_diff1": 0.05000879822749758, - "nauc_mrr_at_3_max": 0.00721164104466641, - "nauc_mrr_at_3_std": -0.10583005601628986, - "nauc_mrr_at_5_diff1": 0.04007888239793958, - "nauc_mrr_at_5_max": 0.009848450000869181, - "nauc_mrr_at_5_std": -0.09855947438234418, - "nauc_ndcg_at_1000_diff1": 0.023142396023156372, - "nauc_ndcg_at_1000_max": 0.035375916635258325, - "nauc_ndcg_at_1000_std": -0.07958245047320736, - "nauc_ndcg_at_100_diff1": 0.015119475959438647, - "nauc_ndcg_at_100_max": 0.04731669040790856, - "nauc_ndcg_at_100_std": -0.0635412170264141, - "nauc_ndcg_at_10_diff1": 0.025144530735346976, - "nauc_ndcg_at_10_max": 0.03090893341127276, - "nauc_ndcg_at_10_std": -0.09226635650097872, - "nauc_ndcg_at_1_diff1": 0.0778329308648601, - "nauc_ndcg_at_1_max": 0.017467451160770153, - "nauc_ndcg_at_1_std": -0.09230422783548616, - "nauc_ndcg_at_20_diff1": 0.016460308621990286, - "nauc_ndcg_at_20_max": 0.038935887157304706, - "nauc_ndcg_at_20_std": -0.08916836990042935, - "nauc_ndcg_at_3_diff1": 0.04402254814866491, - "nauc_ndcg_at_3_max": 0.005282526107844626, - "nauc_ndcg_at_3_std": -0.10876252624429479, - "nauc_ndcg_at_5_diff1": 0.029223974622285064, - "nauc_ndcg_at_5_max": 0.0089255171139576, - "nauc_ndcg_at_5_std": -0.09706274219479245, - "nauc_precision_at_1000_diff1": -0.6306765130294122, - "nauc_precision_at_1000_max": 0.42551566080976494, - "nauc_precision_at_1000_std": 0.8273066802478539, - "nauc_precision_at_100_diff1": -0.20477156379793393, - "nauc_precision_at_100_max": 0.3289094948324158, - "nauc_precision_at_100_std": 0.3540262081844231, - "nauc_precision_at_10_diff1": -0.00027665893649036046, - "nauc_precision_at_10_max": 0.055734883124117916, - "nauc_precision_at_10_std": -0.08384768736288313, - "nauc_precision_at_1_diff1": 0.0778329308648601, - "nauc_precision_at_1_max": 0.017467451160770153, - "nauc_precision_at_1_std": -0.09230422783548616, - "nauc_precision_at_20_diff1": -0.036638435777051476, - "nauc_precision_at_20_max": 0.09299368334901673, - "nauc_precision_at_20_std": -0.06832906783952929, - "nauc_precision_at_3_diff1": 0.03185952675870671, - "nauc_precision_at_3_max": 0.0014239615902501407, - "nauc_precision_at_3_std": -0.11472573740325583, - "nauc_precision_at_5_diff1": 0.00844004859903081, - "nauc_precision_at_5_max": 0.0072949355262599425, - "nauc_precision_at_5_std": -0.09334608645063842, - "nauc_recall_at_1000_diff1": -0.6306765130294512, - "nauc_recall_at_1000_max": 0.4255156608097682, - "nauc_recall_at_1000_std": 0.8273066802478575, - "nauc_recall_at_100_diff1": -0.20477156379793215, - "nauc_recall_at_100_max": 0.3289094948324168, - "nauc_recall_at_100_std": 0.3540262081844245, - "nauc_recall_at_10_diff1": -0.0002766589364904998, - "nauc_recall_at_10_max": 0.05573488312411784, - "nauc_recall_at_10_std": -0.08384768736288326, - "nauc_recall_at_1_diff1": 0.0778329308648601, - "nauc_recall_at_1_max": 0.017467451160770153, - "nauc_recall_at_1_std": -0.09230422783548616, - "nauc_recall_at_20_diff1": -0.03663843577705176, - "nauc_recall_at_20_max": 0.09299368334901667, - "nauc_recall_at_20_std": -0.06832906783953004, - "nauc_recall_at_3_diff1": 0.031859526758706465, - "nauc_recall_at_3_max": 0.0014239615902498597, - "nauc_recall_at_3_std": -0.11472573740325614, - "nauc_recall_at_5_diff1": 0.008440048599030739, - "nauc_recall_at_5_max": 0.007294935526259778, - "nauc_recall_at_5_std": -0.09334608645063842, - "ndcg_at_1": 0.07925, - "ndcg_at_10": 0.24968, - "ndcg_at_100": 0.34784, - "ndcg_at_1000": 0.35593, - "ndcg_at_20": 0.30266, - "ndcg_at_3": 0.14817, - "ndcg_at_5": 0.19136, - "precision_at_1": 0.07925, - "precision_at_10": 0.04882, - "precision_at_100": 0.00932, - "precision_at_1000": 0.00099, - "precision_at_20": 0.03488, - "precision_at_3": 0.06714, - "precision_at_5": 0.06142, - "recall_at_1": 0.07925, - "recall_at_10": 0.48821, - "recall_at_100": 0.9316, - "recall_at_1000": 0.99481, - "recall_at_20": 0.69764, - "recall_at_3": 0.20142, - "recall_at_5": 0.30708 - } - ] - }, - "task_name": "NIGHTSI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/OxfordFlowersClassification.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/OxfordFlowersClassification.json deleted file mode 100644 index 6b9742156a..0000000000 --- a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/OxfordFlowersClassification.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", - "evaluation_time": 351.2027907371521, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.8762745098039215, - "f1": 0.8707019249469784, - "f1_weighted": 0.8703980266170556, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8762745098039215, - "scores_per_experiment": [ - { - "accuracy": 0.8784313725490196, - "f1": 0.8709034893034996, - "f1_weighted": 0.8708997095974342 - }, - { - "accuracy": 0.8823529411764706, - "f1": 0.879309449928005, - "f1_weighted": 0.8791989291482171 - }, - { - "accuracy": 0.865686274509804, - "f1": 0.8593490468264394, - "f1_weighted": 0.858799777548312 - }, - { - "accuracy": 0.8715686274509804, - "f1": 0.866195738562844, - "f1_weighted": 0.8655759996721402 - }, - { - "accuracy": 0.8833333333333333, - "f1": 0.8777519001141039, - "f1_weighted": 0.877515717119174 - } - ] - } - ] - }, - "task_name": "OxfordFlowersClassification" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/RenderedSST2.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/RenderedSST2.json deleted file mode 100644 index bce1d97cbb..0000000000 --- a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/RenderedSST2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", - "evaluation_time": 34.560497760772705, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.49203734211971445, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.49203734211971445 - } - ] - }, - "task_name": "RenderedSST2" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/model_meta.json b/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/model_meta.json deleted file mode 100644 index 2445c6c1a5..0000000000 --- a/results-mieb/Salesforce__blip-itm-base-coco/7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "Salesforce/blip-itm-base-coco", "revision": "7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f", "release_date": "2023-08-01", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "BLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/RenderedSST2.json b/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/RenderedSST2.json deleted file mode 100644 index 79964f28b8..0000000000 --- a/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/RenderedSST2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", - "evaluation_time": 76.84005379676819, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.500823723228995, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.500823723228995 - } - ] - }, - "task_name": "RenderedSST2" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/model_meta.json b/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/model_meta.json deleted file mode 100644 index 69acd258db..0000000000 --- a/results-mieb/Salesforce__blip-itm-large-coco/fef05cafc05298067cbbca00b125749394a77a6f/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "Salesforce/blip-itm-large-coco", "revision": "fef05cafc05298067cbbca00b125749394a77a6f", "release_date": "2023-08-01", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "BLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNIST.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNIST.json deleted file mode 100644 index ac83e833d4..0000000000 --- a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 716.1278786659241, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.9285, - "f1": 0.9276672727830793, - "f1_weighted": 0.9282674252781511, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9285, - "scores_per_experiment": [ - { - "accuracy": 0.9284, - "f1": 0.9278192557850607, - "f1_weighted": 0.9285648611883027 - }, - { - "accuracy": 0.9127, - "f1": 0.9108312967490848, - "f1_weighted": 0.9117442017991534 - }, - { - "accuracy": 0.9321, - "f1": 0.931401132729107, - "f1_weighted": 0.9320989666694753 - }, - { - "accuracy": 0.9354, - "f1": 0.9350616561525831, - "f1_weighted": 0.9354103679494147 - }, - { - "accuracy": 0.9339, - "f1": 0.9332230224995612, - "f1_weighted": 0.9335187287844099 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNISTZeroShot.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNISTZeroShot.json deleted file mode 100644 index b0fd292049..0000000000 --- a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/MNISTZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 636.4780406951904, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.5049, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5049 - } - ] - }, - "task_name": "MNISTZeroShot" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/NIGHTSI2IRetrieval.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/NIGHTSI2IRetrieval.json deleted file mode 100644 index d452d34222..0000000000 --- a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/NIGHTSI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", - "evaluation_time": 2631.542476415634, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.05943, - "cv_recall_at_10": 0.40142, - "cv_recall_at_100": 0.88679, - "cv_recall_at_1000": 0.99104, - "cv_recall_at_20": 0.59764, - "cv_recall_at_3": 0.16698, - "cv_recall_at_5": 0.24717, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.20235, - "map_at_1": 0.05943, - "map_at_10": 0.14229, - "map_at_100": 0.16407, - "map_at_1000": 0.16457, - "map_at_20": 0.1558, - "map_at_3": 0.10409, - "map_at_5": 0.12215, - "mrr_at_1": 0.059433962264150944, - "mrr_at_10": 0.14228998203054774, - "mrr_at_100": 0.16406642527076057, - "mrr_at_1000": 0.1645732756182432, - "mrr_at_20": 0.15579824218803512, - "mrr_at_3": 0.10408805031446555, - "mrr_at_5": 0.12215408805031443, - "nauc_cv_recall_at_1000_diff1": -0.26463216865694905, - "nauc_cv_recall_at_1000_max": 0.5278883483217763, - "nauc_cv_recall_at_1000_std": 0.7581453634085121, - "nauc_cv_recall_at_100_diff1": 0.014236330594184277, - "nauc_cv_recall_at_100_max": 0.3171598451327442, - "nauc_cv_recall_at_100_std": 0.37266908975979723, - "nauc_cv_recall_at_10_diff1": -0.002210170307048272, - "nauc_cv_recall_at_10_max": 0.10455296298298879, - "nauc_cv_recall_at_10_std": 0.02712166932863189, - "nauc_cv_recall_at_1_diff1": 0.06336429572276098, - "nauc_cv_recall_at_1_max": 0.07874966460960554, - "nauc_cv_recall_at_1_std": -0.029650640329470464, - "nauc_cv_recall_at_20_diff1": -0.015820807363866755, - "nauc_cv_recall_at_20_max": 0.12973690402299562, - "nauc_cv_recall_at_20_std": 0.04547749025861378, - "nauc_cv_recall_at_3_diff1": -0.0053137478758161424, - "nauc_cv_recall_at_3_max": 0.05351014259433647, - "nauc_cv_recall_at_3_std": -0.003411847027194816, - "nauc_cv_recall_at_5_diff1": 0.0004749523652767695, - "nauc_cv_recall_at_5_max": 0.0679335317764752, - "nauc_cv_recall_at_5_std": -0.0006472184948264357, - "nauc_map_at_1000_diff1": 0.016633966898488534, - "nauc_map_at_1000_max": 0.08068623366557101, - "nauc_map_at_1000_std": -0.0017960556231574202, - "nauc_map_at_100_diff1": 0.01661909546774381, - "nauc_map_at_100_max": 0.0811695484351021, - "nauc_map_at_100_std": -0.0010905676199076005, - "nauc_map_at_10_diff1": 0.017095173553333582, - "nauc_map_at_10_max": 0.07644073407248878, - "nauc_map_at_10_std": -0.007324964157984001, - "nauc_map_at_1_diff1": 0.06336429572276098, - "nauc_map_at_1_max": 0.07874966460960554, - "nauc_map_at_1_std": -0.029650640329470464, - "nauc_map_at_20_diff1": 0.016082386042899254, - "nauc_map_at_20_max": 0.07877478749954128, - "nauc_map_at_20_std": -0.0062248086434086415, - "nauc_map_at_3_diff1": 0.017880266235441065, - "nauc_map_at_3_max": 0.06255089186760056, - "nauc_map_at_3_std": -0.015893007578431876, - "nauc_map_at_5_diff1": 0.018500079315787177, - "nauc_map_at_5_max": 0.06573480071106033, - "nauc_map_at_5_std": -0.01479218234208926, - "nauc_mrr_at_1000_diff1": 0.016633966898488534, - "nauc_mrr_at_1000_max": 0.08068623366557101, - "nauc_mrr_at_1000_std": -0.0017960556231574202, - "nauc_mrr_at_100_diff1": 0.01661909546774381, - "nauc_mrr_at_100_max": 0.0811695484351021, - "nauc_mrr_at_100_std": -0.0010905676199076005, - "nauc_mrr_at_10_diff1": 0.017095173553333582, - "nauc_mrr_at_10_max": 0.07644073407248878, - "nauc_mrr_at_10_std": -0.007324964157984001, - "nauc_mrr_at_1_diff1": 0.06336429572276098, - "nauc_mrr_at_1_max": 0.07874966460960554, - "nauc_mrr_at_1_std": -0.029650640329470464, - "nauc_mrr_at_20_diff1": 0.016082386042899254, - "nauc_mrr_at_20_max": 0.07877478749954128, - "nauc_mrr_at_20_std": -0.0062248086434086415, - "nauc_mrr_at_3_diff1": 0.017880266235441065, - "nauc_mrr_at_3_max": 0.06255089186760056, - "nauc_mrr_at_3_std": -0.015893007578431876, - "nauc_mrr_at_5_diff1": 0.018500079315787177, - "nauc_mrr_at_5_max": 0.06573480071106033, - "nauc_mrr_at_5_std": -0.01479218234208926, - "nauc_ndcg_at_1000_diff1": 0.009960736287420825, - "nauc_ndcg_at_1000_max": 0.09531647626699342, - "nauc_ndcg_at_1000_std": 0.020883086622059264, - "nauc_ndcg_at_100_diff1": 0.011572508647446316, - "nauc_ndcg_at_100_max": 0.10937633012181702, - "nauc_ndcg_at_100_std": 0.042541337399181214, - "nauc_ndcg_at_10_diff1": 0.00952815561968612, - "nauc_ndcg_at_10_max": 0.08583341700567972, - "nauc_ndcg_at_10_std": 0.0047805496391580115, - "nauc_ndcg_at_1_diff1": 0.06336429572276098, - "nauc_ndcg_at_1_max": 0.07874966460960554, - "nauc_ndcg_at_1_std": -0.029650640329470464, - "nauc_ndcg_at_20_diff1": 0.005486565789239601, - "nauc_ndcg_at_20_max": 0.09377542572693116, - "nauc_ndcg_at_20_std": 0.009702452362775931, - "nauc_ndcg_at_3_diff1": 0.009923888782701359, - "nauc_ndcg_at_3_max": 0.0594695188640034, - "nauc_ndcg_at_3_std": -0.011751091386382948, - "nauc_ndcg_at_5_diff1": 0.01169898585355613, - "nauc_ndcg_at_5_max": 0.06611397346239468, - "nauc_ndcg_at_5_std": -0.009801904972316896, - "nauc_precision_at_1000_diff1": -0.26463216865691536, - "nauc_precision_at_1000_max": 0.5278883483217904, - "nauc_precision_at_1000_std": 0.7581453634085238, - "nauc_precision_at_100_diff1": 0.01423633059418428, - "nauc_precision_at_100_max": 0.31715984513274537, - "nauc_precision_at_100_std": 0.37266908975979857, - "nauc_precision_at_10_diff1": -0.002210170307048316, - "nauc_precision_at_10_max": 0.10455296298298879, - "nauc_precision_at_10_std": 0.027121669328632015, - "nauc_precision_at_1_diff1": 0.06336429572276098, - "nauc_precision_at_1_max": 0.07874966460960554, - "nauc_precision_at_1_std": -0.029650640329470464, - "nauc_precision_at_20_diff1": -0.015820807363866707, - "nauc_precision_at_20_max": 0.12973690402299493, - "nauc_precision_at_20_std": 0.04547749025861332, - "nauc_precision_at_3_diff1": -0.0053137478758161095, - "nauc_precision_at_3_max": 0.05351014259433666, - "nauc_precision_at_3_std": -0.0034118470271947503, - "nauc_precision_at_5_diff1": 0.00047495236527695035, - "nauc_precision_at_5_max": 0.06793353177647533, - "nauc_precision_at_5_std": -0.000647218494826177, - "nauc_recall_at_1000_diff1": -0.26463216865694905, - "nauc_recall_at_1000_max": 0.5278883483217763, - "nauc_recall_at_1000_std": 0.7581453634085121, - "nauc_recall_at_100_diff1": 0.014236330594184277, - "nauc_recall_at_100_max": 0.3171598451327442, - "nauc_recall_at_100_std": 0.37266908975979723, - "nauc_recall_at_10_diff1": -0.002210170307048272, - "nauc_recall_at_10_max": 0.10455296298298879, - "nauc_recall_at_10_std": 0.02712166932863189, - "nauc_recall_at_1_diff1": 0.06336429572276098, - "nauc_recall_at_1_max": 0.07874966460960554, - "nauc_recall_at_1_std": -0.029650640329470464, - "nauc_recall_at_20_diff1": -0.015820807363866755, - "nauc_recall_at_20_max": 0.12973690402299562, - "nauc_recall_at_20_std": 0.04547749025861378, - "nauc_recall_at_3_diff1": -0.0053137478758161424, - "nauc_recall_at_3_max": 0.05351014259433647, - "nauc_recall_at_3_std": -0.003411847027194816, - "nauc_recall_at_5_diff1": 0.0004749523652767695, - "nauc_recall_at_5_max": 0.0679335317764752, - "nauc_recall_at_5_std": -0.0006472184948264357, - "ndcg_at_1": 0.05943, - "ndcg_at_10": 0.20235, - "ndcg_at_100": 0.30717, - "ndcg_at_1000": 0.32046, - "ndcg_at_20": 0.25183, - "ndcg_at_3": 0.12012, - "ndcg_at_5": 0.15292, - "precision_at_1": 0.05943, - "precision_at_10": 0.04014, - "precision_at_100": 0.00887, - "precision_at_1000": 0.00099, - "precision_at_20": 0.02988, - "precision_at_3": 0.05566, - "precision_at_5": 0.04943, - "recall_at_1": 0.05943, - "recall_at_10": 0.40142, - "recall_at_100": 0.88679, - "recall_at_1000": 0.99104, - "recall_at_20": 0.59764, - "recall_at_3": 0.16698, - "recall_at_5": 0.24717 - } - ] - }, - "task_name": "NIGHTSI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/OxfordFlowersClassification.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/OxfordFlowersClassification.json deleted file mode 100644 index f49aa0a7d7..0000000000 --- a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/OxfordFlowersClassification.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", - "evaluation_time": 684.7742538452148, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.966078431372549, - "f1": 0.9665318717720475, - "f1_weighted": 0.96610958525568, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.966078431372549, - "scores_per_experiment": [ - { - "accuracy": 0.9686274509803922, - "f1": 0.969270715608082, - "f1_weighted": 0.9689539772636104 - }, - { - "accuracy": 0.9696078431372549, - "f1": 0.9698978889543093, - "f1_weighted": 0.9692619935891078 - }, - { - "accuracy": 0.9647058823529412, - "f1": 0.9649920270608272, - "f1_weighted": 0.9646380547095591 - }, - { - "accuracy": 0.9598039215686275, - "f1": 0.9602800604286633, - "f1_weighted": 0.9600010155147554 - }, - { - "accuracy": 0.9676470588235294, - "f1": 0.9682186668083551, - "f1_weighted": 0.9676928852013674 - } - ] - } - ] - }, - "task_name": "OxfordFlowersClassification" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/RenderedSST2.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/RenderedSST2.json deleted file mode 100644 index c593e878e3..0000000000 --- a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/RenderedSST2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", - "evaluation_time": 117.32345771789551, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.5178473366282262, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5178473366282262 - } - ] - }, - "task_name": "RenderedSST2" -} \ No newline at end of file diff --git a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/model_meta.json b/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/model_meta.json deleted file mode 100644 index 92e563becc..0000000000 --- a/results-mieb/Salesforce__blip2-opt-2.7b/51572668da0eb669e01a189dc22abe6088589a24/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "Salesforce/blip2-opt-2.7b", "revision": "51572668da0eb669e01a189dc22abe6088589a24", "release_date": "2024-03-22", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "blip2_loader"} \ No newline at end of file diff --git a/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/CVBenchCount.json b/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/CVBenchCount.json deleted file mode 100644 index 97fc301f69..0000000000 --- a/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/CVBenchCount.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", - "evaluation_time": 141.80410814285278, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "accuracy": 0.6205583756345178, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6205583756345178 - } - ] - }, - "task_name": "CVBenchCount" -} \ No newline at end of file diff --git a/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json b/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json deleted file mode 100644 index c3494de385..0000000000 --- a/results-mieb/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "TIGER-Lab/VLM2Vec-Full", "revision": "e9afa98002097ac2471827ba23ea1f2ddd229480", "release_date": "2024-10-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "VLM2VecWrapper"} \ No newline at end of file diff --git a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json deleted file mode 100644 index 6d046886ce..0000000000 --- a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/BLINKIT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26", - "evaluation_time": 213.14108443260193, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.3209, - "cv_recall_at_10": 0.64179, - "cv_recall_at_100": 0.90547, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.72886, - "cv_recall_at_3": 0.52736, - "cv_recall_at_5": 0.5796, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4847, - "map_at_1": 0.3209, - "map_at_10": 0.434, - "map_at_100": 0.44398, - "map_at_1000": 0.44463, - "map_at_20": 0.43971, - "map_at_3": 0.41335, - "map_at_5": 0.42554, - "mrr_at_1": 0.3208955223880597, - "mrr_at_10": 0.434004580273237, - "mrr_at_100": 0.44398225504608213, - "mrr_at_1000": 0.4446309367234792, - "mrr_at_20": 0.43970614227302923, - "mrr_at_3": 0.4133499170812605, - "mrr_at_5": 0.4255389718076287, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.3319455914188296, - "nauc_cv_recall_at_100_max": -0.8496514687248068, - "nauc_cv_recall_at_100_std": -1.0075573331300645, - "nauc_cv_recall_at_10_diff1": 0.21632765291249811, - "nauc_cv_recall_at_10_max": -0.616149982302474, - "nauc_cv_recall_at_10_std": -0.9479982037259836, - "nauc_cv_recall_at_1_diff1": 0.3430629866475006, - "nauc_cv_recall_at_1_max": -0.31499658742959696, - "nauc_cv_recall_at_1_std": -0.5455674065129812, - "nauc_cv_recall_at_20_diff1": 0.14497526898295157, - "nauc_cv_recall_at_20_max": -0.8183913780389244, - "nauc_cv_recall_at_20_std": -1.0686032736054183, - "nauc_cv_recall_at_3_diff1": 0.2272558706482997, - "nauc_cv_recall_at_3_max": -0.47485631997681976, - "nauc_cv_recall_at_3_std": -0.7893342152547034, - "nauc_cv_recall_at_5_diff1": 0.23402660393502006, - "nauc_cv_recall_at_5_max": -0.4968352312252504, - "nauc_cv_recall_at_5_std": -0.8652220828722653, - "nauc_map_at_1000_diff1": 0.28084189069845655, - "nauc_map_at_1000_max": -0.4050774810885909, - "nauc_map_at_1000_std": -0.6734464316794351, - "nauc_map_at_100_diff1": 0.2809924262004814, - "nauc_map_at_100_max": -0.4060074400165581, - "nauc_map_at_100_std": -0.6741534540228545, - "nauc_map_at_10_diff1": 0.28170348287831065, - "nauc_map_at_10_max": -0.40186513581417665, - "nauc_map_at_10_std": -0.6758814723680184, - "nauc_map_at_1_diff1": 0.3430629866475006, - "nauc_map_at_1_max": -0.31499658742959696, - "nauc_map_at_1_std": -0.5455674065129812, - "nauc_map_at_20_diff1": 0.278380373705771, - "nauc_map_at_20_max": -0.40905515991168534, - "nauc_map_at_20_std": -0.6780915297694656, - "nauc_map_at_3_diff1": 0.2816431806099485, - "nauc_map_at_3_max": -0.3850630283616802, - "nauc_map_at_3_std": -0.6525319695750113, - "nauc_map_at_5_diff1": 0.2837769102957866, - "nauc_map_at_5_max": -0.3886275351750067, - "nauc_map_at_5_std": -0.66779078741231, - "nauc_mrr_at_1000_diff1": 0.28084189069845655, - "nauc_mrr_at_1000_max": -0.4050774810885909, - "nauc_mrr_at_1000_std": -0.6734464316794351, - "nauc_mrr_at_100_diff1": 0.2809924262004814, - "nauc_mrr_at_100_max": -0.4060074400165581, - "nauc_mrr_at_100_std": -0.6741534540228545, - "nauc_mrr_at_10_diff1": 0.28170348287831065, - "nauc_mrr_at_10_max": -0.40186513581417665, - "nauc_mrr_at_10_std": -0.6758814723680184, - "nauc_mrr_at_1_diff1": 0.3430629866475006, - "nauc_mrr_at_1_max": -0.31499658742959696, - "nauc_mrr_at_1_std": -0.5455674065129812, - "nauc_mrr_at_20_diff1": 0.278380373705771, - "nauc_mrr_at_20_max": -0.40905515991168534, - "nauc_mrr_at_20_std": -0.6780915297694656, - "nauc_mrr_at_3_diff1": 0.2816431806099485, - "nauc_mrr_at_3_max": -0.3850630283616802, - "nauc_mrr_at_3_std": -0.6525319695750113, - "nauc_mrr_at_5_diff1": 0.2837769102957866, - "nauc_mrr_at_5_max": -0.3886275351750067, - "nauc_mrr_at_5_std": -0.66779078741231, - "nauc_ndcg_at_1000_diff1": 0.2723675027709315, - "nauc_ndcg_at_1000_max": -0.4322321611684638, - "nauc_ndcg_at_1000_std": -0.7027813195044652, - "nauc_ndcg_at_100_diff1": 0.27524643357042616, - "nauc_ndcg_at_100_max": -0.4514723022250352, - "nauc_ndcg_at_100_std": -0.7168373347468309, - "nauc_ndcg_at_10_diff1": 0.2663368914050526, - "nauc_ndcg_at_10_max": -0.44757349552373726, - "nauc_ndcg_at_10_std": -0.7355356954078378, - "nauc_ndcg_at_1_diff1": 0.3430629866475006, - "nauc_ndcg_at_1_max": -0.31499658742959696, - "nauc_ndcg_at_1_std": -0.5455674065129812, - "nauc_ndcg_at_20_diff1": 0.2541650696967836, - "nauc_ndcg_at_20_max": -0.47842206029525874, - "nauc_ndcg_at_20_std": -0.7484632948030998, - "nauc_ndcg_at_3_diff1": 0.2674065394667972, - "nauc_ndcg_at_3_max": -0.40781706959442965, - "nauc_ndcg_at_3_std": -0.6872034895658455, - "nauc_ndcg_at_5_diff1": 0.2709785209532911, - "nauc_ndcg_at_5_max": -0.4148914722291144, - "nauc_ndcg_at_5_std": -0.7152726168117532, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3319455914188293, - "nauc_precision_at_100_max": -0.8496514687248082, - "nauc_precision_at_100_std": -1.007557333130068, - "nauc_precision_at_10_diff1": 0.216327652912498, - "nauc_precision_at_10_max": -0.6161499823024736, - "nauc_precision_at_10_std": -0.9479982037259829, - "nauc_precision_at_1_diff1": 0.3430629866475006, - "nauc_precision_at_1_max": -0.31499658742959696, - "nauc_precision_at_1_std": -0.5455674065129812, - "nauc_precision_at_20_diff1": 0.14497526898295124, - "nauc_precision_at_20_max": -0.8183913780389255, - "nauc_precision_at_20_std": -1.068603273605419, - "nauc_precision_at_3_diff1": 0.22725587064830013, - "nauc_precision_at_3_max": -0.47485631997681943, - "nauc_precision_at_3_std": -0.789334215254703, - "nauc_precision_at_5_diff1": 0.2340266039350195, - "nauc_precision_at_5_max": -0.49683523122525086, - "nauc_precision_at_5_std": -0.8652220828722655, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.3319455914188296, - "nauc_recall_at_100_max": -0.8496514687248068, - "nauc_recall_at_100_std": -1.0075573331300645, - "nauc_recall_at_10_diff1": 0.21632765291249811, - "nauc_recall_at_10_max": -0.616149982302474, - "nauc_recall_at_10_std": -0.9479982037259836, - "nauc_recall_at_1_diff1": 0.3430629866475006, - "nauc_recall_at_1_max": -0.31499658742959696, - "nauc_recall_at_1_std": -0.5455674065129812, - "nauc_recall_at_20_diff1": 0.14497526898295157, - "nauc_recall_at_20_max": -0.8183913780389244, - "nauc_recall_at_20_std": -1.0686032736054183, - "nauc_recall_at_3_diff1": 0.2272558706482997, - "nauc_recall_at_3_max": -0.47485631997681976, - "nauc_recall_at_3_std": -0.7893342152547034, - "nauc_recall_at_5_diff1": 0.23402660393502006, - "nauc_recall_at_5_max": -0.4968352312252504, - "nauc_recall_at_5_std": -0.8652220828722653, - "ndcg_at_1": 0.3209, - "ndcg_at_10": 0.4847, - "ndcg_at_100": 0.53853, - "ndcg_at_1000": 0.55159, - "ndcg_at_20": 0.50628, - "ndcg_at_3": 0.44269, - "ndcg_at_5": 0.46443, - "precision_at_1": 0.3209, - "precision_at_10": 0.06418, - "precision_at_100": 0.00905, - "precision_at_1000": 0.001, - "precision_at_20": 0.03644, - "precision_at_3": 0.17579, - "precision_at_5": 0.11592, - "recall_at_1": 0.3209, - "recall_at_10": 0.64179, - "recall_at_100": 0.90547, - "recall_at_1000": 1.0, - "recall_at_20": 0.72886, - "recall_at_3": 0.52736, - "recall_at_5": 0.5796 - } - ] - }, - "task_name": "BLINKIT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/Caltech101.json b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/Caltech101.json deleted file mode 100644 index cac75612d1..0000000000 --- a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/Caltech101.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", - "evaluation_time": 1317.9743084907532, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.9301446416831032, - "f1": 0.8863632422649081, - "f1_weighted": 0.9270094006117223, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9301446416831032, - "scores_per_experiment": [ - { - "accuracy": 0.9301446416831032, - "f1": 0.8863632422649081, - "f1_weighted": 0.9270094006117223 - } - ] - } - ] - }, - "task_name": "Caltech101" -} \ No newline at end of file diff --git a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/STS12.json b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/STS12.json deleted file mode 100644 index 2f7a702114..0000000000 --- a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/STS12.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", - "evaluation_time": 33.679136514663696, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "cosine_pearson": 0.6128856131150828, - "cosine_spearman": 0.5375376750091784, - "euclidean_pearson": 0.5866571133163221, - "euclidean_spearman": 0.5376001641683719, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5375376750091784, - "manhattan_pearson": 0.5912422177023093, - "manhattan_spearman": 0.5413588869937086, - "pearson": 0.6128856131150828, - "spearman": 0.5375376750091784 - } - ] - }, - "task_name": "STS12" -} \ No newline at end of file diff --git a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/model_meta.json b/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/model_meta.json deleted file mode 100644 index 07f5788002..0000000000 --- a/results-mieb/TIGER-Lab__VLM2Vec-LoRA/7403b6327958071c1e33c822c7453adadccc7298/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "TIGER-Lab/VLM2Vec-LoRA", "revision": "7403b6327958071c1e33c822c7453adadccc7298", "release_date": "2024-10-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "VLM2VecWrapper"} \ No newline at end of file diff --git a/results-mieb/embed-english-v3.0-v/1/ImageNetDog15Clustering.json b/results-mieb/embed-english-v3.0-v/1/ImageNetDog15Clustering.json deleted file mode 100644 index a791197d49..0000000000 --- a/results-mieb/embed-english-v3.0-v/1/ImageNetDog15Clustering.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "dataset_revision": "bfb6ad3b2109d26c9daddf14f98d315daa35ee72", - "evaluation_time": 1831.918389081955, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.11895910780669144, - "ari": 0.4718230736383377, - "cluster_accuracy": 0.578996282527881, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.11895910780669144, - "nmi": 0.6532570693140701, - "v_measure": 0.65325706931407 - } - ] - }, - "task_name": "ImageNetDog15Clustering" -} \ No newline at end of file diff --git a/results-mieb/embed-english-v3.0-v/1/model_meta.json b/results-mieb/embed-english-v3.0-v/1/model_meta.json deleted file mode 100644 index 067881b7e8..0000000000 --- a/results-mieb/embed-english-v3.0-v/1/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "embed-english-v3.0-v", "revision": "1", "release_date": "2024-10-24", "languages": ["eng-Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": 1024, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": "cosine", "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "cohere_v_loader"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/MNIST.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/MNIST.json deleted file mode 100644 index 1c4bf7a7c8..0000000000 --- a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 77.37463116645813, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "accuracy": 0.86678, - "f1": 0.8645767524624768, - "f1_weighted": 0.8666851685943376, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.86678, - "scores_per_experiment": [ - { - "accuracy": 0.8689, - "f1": 0.8670967814543469, - "f1_weighted": 0.8697569866148404 - }, - { - "accuracy": 0.8349, - "f1": 0.8309818535418068, - "f1_weighted": 0.83417961551265 - }, - { - "accuracy": 0.8675, - "f1": 0.8665245386493894, - "f1_weighted": 0.8684789478397935 - }, - { - "accuracy": 0.8789, - "f1": 0.8764053509532486, - "f1_weighted": 0.877569373830515 - }, - { - "accuracy": 0.8837, - "f1": 0.8818752377135922, - "f1_weighted": 0.8834409191738892 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/NIGHTSI2IRetrieval.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/NIGHTSI2IRetrieval.json deleted file mode 100644 index ed5b188152..0000000000 --- a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/NIGHTSI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", - "evaluation_time": 199.69204115867615, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.08491, - "cv_recall_at_10": 0.5033, - "cv_recall_at_100": 0.94811, - "cv_recall_at_1000": 0.99528, - "cv_recall_at_20": 0.73915, - "cv_recall_at_3": 0.21368, - "cv_recall_at_5": 0.32075, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.25976, - "map_at_1": 0.08491, - "map_at_10": 0.18614, - "map_at_100": 0.20832, - "map_at_1000": 0.20858, - "map_at_20": 0.20229, - "map_at_3": 0.13797, - "map_at_5": 0.16217, - "mrr_at_1": 0.08490566037735849, - "mrr_at_10": 0.18614180892482757, - "mrr_at_100": 0.20831945787163383, - "mrr_at_1000": 0.20858401988895778, - "mrr_at_20": 0.20229258313366166, - "mrr_at_3": 0.1379716981132077, - "mrr_at_5": 0.16216981132075434, - "nauc_cv_recall_at_1000_diff1": -0.9262371615312536, - "nauc_cv_recall_at_1000_max": 0.8307656395891835, - "nauc_cv_recall_at_1000_std": 0.9330065359477094, - "nauc_cv_recall_at_100_diff1": -0.47821492233256935, - "nauc_cv_recall_at_100_max": 0.653713606654785, - "nauc_cv_recall_at_100_std": 0.43830744418979667, - "nauc_cv_recall_at_10_diff1": -0.03606188090870156, - "nauc_cv_recall_at_10_max": 0.1143223995091099, - "nauc_cv_recall_at_10_std": -0.035964131725537925, - "nauc_cv_recall_at_1_diff1": 0.020150254896699814, - "nauc_cv_recall_at_1_max": 0.04705154577706238, - "nauc_cv_recall_at_1_std": -0.01978058015085107, - "nauc_cv_recall_at_20_diff1": -0.08100163008591106, - "nauc_cv_recall_at_20_max": 0.25923993548331703, - "nauc_cv_recall_at_20_std": -0.06323951198528052, - "nauc_cv_recall_at_3_diff1": 0.010570189395540861, - "nauc_cv_recall_at_3_max": 0.05478547707900785, - "nauc_cv_recall_at_3_std": -0.03527327833867772, - "nauc_cv_recall_at_5_diff1": -0.009090564990284886, - "nauc_cv_recall_at_5_max": 0.09606475056145755, - "nauc_cv_recall_at_5_std": -0.00945835120744882, - "nauc_map_at_1000_diff1": -0.006652494750020363, - "nauc_map_at_1000_max": 0.07794470071908056, - "nauc_map_at_1000_std": -0.021676746990813362, - "nauc_map_at_100_diff1": -0.007161878633089917, - "nauc_map_at_100_max": 0.07855793447874261, - "nauc_map_at_100_std": -0.021314555769920304, - "nauc_map_at_10_diff1": -0.0011759754726835372, - "nauc_map_at_10_max": 0.0688532273816496, - "nauc_map_at_10_std": -0.024844900739844897, - "nauc_map_at_1_diff1": 0.020150254896699814, - "nauc_map_at_1_max": 0.04705154577706238, - "nauc_map_at_1_std": -0.01978058015085107, - "nauc_map_at_20_diff1": -0.004673155845604268, - "nauc_map_at_20_max": 0.07787233394995861, - "nauc_map_at_20_std": -0.026666636294813202, - "nauc_map_at_3_diff1": 0.012874623252847255, - "nauc_map_at_3_max": 0.05289142275583296, - "nauc_map_at_3_std": -0.028444137731526885, - "nauc_map_at_5_diff1": 0.004486822990023081, - "nauc_map_at_5_max": 0.06691112758480544, - "nauc_map_at_5_std": -0.018957410356361237, - "nauc_mrr_at_1000_diff1": -0.006652494750020363, - "nauc_mrr_at_1000_max": 0.07794470071908056, - "nauc_mrr_at_1000_std": -0.021676746990813362, - "nauc_mrr_at_100_diff1": -0.007161878633089917, - "nauc_mrr_at_100_max": 0.07855793447874261, - "nauc_mrr_at_100_std": -0.021314555769920304, - "nauc_mrr_at_10_diff1": -0.0011759754726835372, - "nauc_mrr_at_10_max": 0.0688532273816496, - "nauc_mrr_at_10_std": -0.024844900739844897, - "nauc_mrr_at_1_diff1": 0.020150254896699814, - "nauc_mrr_at_1_max": 0.04705154577706238, - "nauc_mrr_at_1_std": -0.01978058015085107, - "nauc_mrr_at_20_diff1": -0.004673155845604268, - "nauc_mrr_at_20_max": 0.07787233394995861, - "nauc_mrr_at_20_std": -0.026666636294813202, - "nauc_mrr_at_3_diff1": 0.012874623252847255, - "nauc_mrr_at_3_max": 0.05289142275583296, - "nauc_mrr_at_3_std": -0.028444137731526885, - "nauc_mrr_at_5_diff1": 0.004486822990023081, - "nauc_mrr_at_5_max": 0.06691112758480544, - "nauc_mrr_at_5_std": -0.018957410356361237, - "nauc_ndcg_at_1000_diff1": -0.022411213599227923, - "nauc_ndcg_at_1000_max": 0.0986689095954933, - "nauc_ndcg_at_1000_std": -0.013359705039231978, - "nauc_ndcg_at_100_diff1": -0.0355840262798586, - "nauc_ndcg_at_100_max": 0.11558385385862016, - "nauc_ndcg_at_100_std": -0.0011908869944293231, - "nauc_ndcg_at_10_diff1": -0.012042479286208913, - "nauc_ndcg_at_10_max": 0.08275755290912527, - "nauc_ndcg_at_10_std": -0.0280393669746386, - "nauc_ndcg_at_1_diff1": 0.020150254896699814, - "nauc_ndcg_at_1_max": 0.04705154577706238, - "nauc_ndcg_at_1_std": -0.01978058015085107, - "nauc_ndcg_at_20_diff1": -0.022260903773348162, - "nauc_ndcg_at_20_max": 0.11601171113415618, - "nauc_ndcg_at_20_std": -0.03428564651751476, - "nauc_ndcg_at_3_diff1": 0.012064737993632191, - "nauc_ndcg_at_3_max": 0.05355497149851039, - "nauc_ndcg_at_3_std": -0.030649951361174387, - "nauc_ndcg_at_5_diff1": -0.00011617957032628835, - "nauc_ndcg_at_5_max": 0.07633348175009125, - "nauc_ndcg_at_5_std": -0.015978649311778212, - "nauc_precision_at_1000_diff1": -0.9262371615311449, - "nauc_precision_at_1000_max": 0.830765639589156, - "nauc_precision_at_1000_std": 0.933006535947729, - "nauc_precision_at_100_diff1": -0.47821492233257124, - "nauc_precision_at_100_max": 0.6537136066547907, - "nauc_precision_at_100_std": 0.4383074441897986, - "nauc_precision_at_10_diff1": -0.03606188090870162, - "nauc_precision_at_10_max": 0.11432239950910993, - "nauc_precision_at_10_std": -0.03596413172553808, - "nauc_precision_at_1_diff1": 0.020150254896699814, - "nauc_precision_at_1_max": 0.04705154577706238, - "nauc_precision_at_1_std": -0.01978058015085107, - "nauc_precision_at_20_diff1": -0.08100163008591092, - "nauc_precision_at_20_max": 0.2592399354833167, - "nauc_precision_at_20_std": -0.06323951198528094, - "nauc_precision_at_3_diff1": 0.010570189395540917, - "nauc_precision_at_3_max": 0.05478547707900796, - "nauc_precision_at_3_std": -0.03527327833867778, - "nauc_precision_at_5_diff1": -0.009090564990284862, - "nauc_precision_at_5_max": 0.0960647505614577, - "nauc_precision_at_5_std": -0.009458351207449148, - "nauc_recall_at_1000_diff1": -0.9262371615312536, - "nauc_recall_at_1000_max": 0.8307656395891835, - "nauc_recall_at_1000_std": 0.9330065359477094, - "nauc_recall_at_100_diff1": -0.47821492233256935, - "nauc_recall_at_100_max": 0.653713606654785, - "nauc_recall_at_100_std": 0.43830744418979667, - "nauc_recall_at_10_diff1": -0.03606188090870156, - "nauc_recall_at_10_max": 0.1143223995091099, - "nauc_recall_at_10_std": -0.035964131725537925, - "nauc_recall_at_1_diff1": 0.020150254896699814, - "nauc_recall_at_1_max": 0.04705154577706238, - "nauc_recall_at_1_std": -0.01978058015085107, - "nauc_recall_at_20_diff1": -0.08100163008591106, - "nauc_recall_at_20_max": 0.25923993548331703, - "nauc_recall_at_20_std": -0.06323951198528052, - "nauc_recall_at_3_diff1": 0.010570189395540861, - "nauc_recall_at_3_max": 0.05478547707900785, - "nauc_recall_at_3_std": -0.03527327833867772, - "nauc_recall_at_5_diff1": -0.009090564990284886, - "nauc_recall_at_5_max": 0.09606475056145755, - "nauc_recall_at_5_std": -0.00945835120744882, - "ndcg_at_1": 0.08491, - "ndcg_at_10": 0.25976, - "ndcg_at_100": 0.35919, - "ndcg_at_1000": 0.3654, - "ndcg_at_20": 0.31912, - "ndcg_at_3": 0.15726, - "ndcg_at_5": 0.20112, - "precision_at_1": 0.08491, - "precision_at_10": 0.05033, - "precision_at_100": 0.00948, - "precision_at_1000": 0.001, - "precision_at_20": 0.03696, - "precision_at_3": 0.07123, - "precision_at_5": 0.06415, - "recall_at_1": 0.08491, - "recall_at_10": 0.5033, - "recall_at_100": 0.94811, - "recall_at_1000": 0.99528, - "recall_at_20": 0.73915, - "recall_at_3": 0.21368, - "recall_at_5": 0.32075 - } - ] - }, - "task_name": "NIGHTSI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/OxfordFlowersClassification.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/OxfordFlowersClassification.json deleted file mode 100644 index d02203db75..0000000000 --- a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/OxfordFlowersClassification.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", - "evaluation_time": 176.6004662513733, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "accuracy": 0.9950980392156863, - "f1": 0.9951496593312846, - "f1_weighted": 0.9950563841695577, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9950980392156863, - "scores_per_experiment": [ - { - "accuracy": 0.9950980392156863, - "f1": 0.9951492931717489, - "f1_weighted": 0.9950790860607716 - }, - { - "accuracy": 0.996078431372549, - "f1": 0.9961903907253213, - "f1_weighted": 0.9960625134874698 - }, - { - "accuracy": 0.9950980392156863, - "f1": 0.9951492931717489, - "f1_weighted": 0.9950790860607716 - }, - { - "accuracy": 0.9941176470588236, - "f1": 0.9941091110170156, - "f1_weighted": 0.9940389039060383 - }, - { - "accuracy": 0.9950980392156863, - "f1": 0.9951502085705881, - "f1_weighted": 0.9950223313327365 - } - ] - } - ] - }, - "task_name": "OxfordFlowersClassification" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json deleted file mode 100644 index 1240b81aa4..0000000000 --- a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 71.94576096534729, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "accuracy": 0.3903069466882068, - "f1": 0.6257925811318822, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "lrap": 0.5316062533656477, - "main_score": 0.3903069466882068, - "scores_per_experiment": [ - { - "accuracy": 0.3905492730210016, - "f1": 0.6235333142876379, - "lrap": 0.5233687847783186 - }, - { - "accuracy": 0.401453957996769, - "f1": 0.6273491281205146, - "lrap": 0.5416105726081543 - }, - { - "accuracy": 0.37621163166397414, - "f1": 0.6161039070570788, - "lrap": 0.5324994951534777 - }, - { - "accuracy": 0.38126009693053314, - "f1": 0.6219616890190912, - "lrap": 0.5204900376952104 - }, - { - "accuracy": 0.4020597738287561, - "f1": 0.6400148671750886, - "lrap": 0.540062376593077 - } - ] - } - ] - }, - "task_name": "VOC2007" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json deleted file mode 100644 index 4d9fd59747..0000000000 --- a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "facebook/dinov2-base", "revision": "f9e44c814b77203eaa57a6bdbbd535f21ede1415", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json b/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json deleted file mode 100644 index 730e213e2b..0000000000 --- a/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 408.0938787460327, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "accuracy": 0.3851373182552503, - "f1": 0.6315092630757977, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "lrap": 0.5298672253634935, - "main_score": 0.3851373182552503, - "scores_per_experiment": [ - { - "accuracy": 0.3939822294022617, - "f1": 0.6271304662163256, - "lrap": 0.53264393735416 - }, - { - "accuracy": 0.3945880452342488, - "f1": 0.6263229032757067, - "lrap": 0.5370607835218142 - }, - { - "accuracy": 0.3784329563812601, - "f1": 0.6339676248065612, - "lrap": 0.5346142411595813 - }, - { - "accuracy": 0.3689418416801292, - "f1": 0.6304914034835388, - "lrap": 0.5144189777418785 - }, - { - "accuracy": 0.3897415185783522, - "f1": 0.6396339175968564, - "lrap": 0.5305981870400338 - } - ] - } - ] - }, - "task_name": "VOC2007" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json b/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json deleted file mode 100644 index 87a8751f95..0000000000 --- a/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "facebook/dinov2-giant", "revision": "611a9d42f2335e0f921f1e313ad3c1b7178d206d", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/MNIST.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/MNIST.json deleted file mode 100644 index 8327e4aa22..0000000000 --- a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 158.008056640625, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "accuracy": 0.8246600000000001, - "f1": 0.8221838432987795, - "f1_weighted": 0.8249356935234966, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8246600000000001, - "scores_per_experiment": [ - { - "accuracy": 0.8232, - "f1": 0.8218126192019535, - "f1_weighted": 0.8247912885329193 - }, - { - "accuracy": 0.8027, - "f1": 0.7986794416362917, - "f1_weighted": 0.8019845617911158 - }, - { - "accuracy": 0.8306, - "f1": 0.8297001183400369, - "f1_weighted": 0.8325294903919632 - }, - { - "accuracy": 0.839, - "f1": 0.8351043070584623, - "f1_weighted": 0.8372439745097275 - }, - { - "accuracy": 0.8278, - "f1": 0.8256227302571535, - "f1_weighted": 0.8281291523917571 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/NIGHTSI2IRetrieval.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/NIGHTSI2IRetrieval.json deleted file mode 100644 index e389cfeb4a..0000000000 --- a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/NIGHTSI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", - "evaluation_time": 508.99595761299133, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.07689, - "cv_recall_at_10": 0.48491, - "cv_recall_at_100": 0.94387, - "cv_recall_at_1000": 0.99575, - "cv_recall_at_20": 0.71698, - "cv_recall_at_3": 0.20142, - "cv_recall_at_5": 0.29528, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.24754, - "map_at_1": 0.07689, - "map_at_10": 0.17599, - "map_at_100": 0.19935, - "map_at_1000": 0.19966, - "map_at_20": 0.19239, - "map_at_3": 0.1298, - "map_at_5": 0.15107, - "mrr_at_1": 0.07688679245283019, - "mrr_at_10": 0.17599225067385418, - "mrr_at_100": 0.19935156300390885, - "mrr_at_1000": 0.19965908590632203, - "mrr_at_20": 0.19239222583377735, - "mrr_at_3": 0.12979559748427677, - "mrr_at_5": 0.15106918238993675, - "nauc_cv_recall_at_1000_diff1": 0.3851540616246801, - "nauc_cv_recall_at_1000_max": 0.8244631185807924, - "nauc_cv_recall_at_1000_std": 0.925562817719677, - "nauc_cv_recall_at_100_diff1": -0.4385715070341862, - "nauc_cv_recall_at_100_max": 0.6730613814153109, - "nauc_cv_recall_at_100_std": 0.45218479548682355, - "nauc_cv_recall_at_10_diff1": 0.012871715486821715, - "nauc_cv_recall_at_10_max": 0.08577496630146382, - "nauc_cv_recall_at_10_std": -0.08241386554096461, - "nauc_cv_recall_at_1_diff1": 0.08460726813618427, - "nauc_cv_recall_at_1_max": -0.0065678904232255786, - "nauc_cv_recall_at_1_std": -0.08707639987423882, - "nauc_cv_recall_at_20_diff1": -0.054615919794490805, - "nauc_cv_recall_at_20_max": 0.2859854431283007, - "nauc_cv_recall_at_20_std": -0.059189025260453935, - "nauc_cv_recall_at_3_diff1": 0.059515518902995126, - "nauc_cv_recall_at_3_max": 0.05847203984965998, - "nauc_cv_recall_at_3_std": -0.03672985512045184, - "nauc_cv_recall_at_5_diff1": 0.026571959649107007, - "nauc_cv_recall_at_5_max": 0.06516748857696698, - "nauc_cv_recall_at_5_std": -0.045787832815866014, - "nauc_map_at_1000_diff1": 0.047321574512003584, - "nauc_map_at_1000_max": 0.05438206812877677, - "nauc_map_at_1000_std": -0.05744428851185208, - "nauc_map_at_100_diff1": 0.04650275080611146, - "nauc_map_at_100_max": 0.05516022097644162, - "nauc_map_at_100_std": -0.0569522165815035, - "nauc_map_at_10_diff1": 0.05173169037370162, - "nauc_map_at_10_max": 0.043084012251647076, - "nauc_map_at_10_std": -0.06272816529553181, - "nauc_map_at_1_diff1": 0.08460726813618427, - "nauc_map_at_1_max": -0.0065678904232255786, - "nauc_map_at_1_std": -0.08707639987423882, - "nauc_map_at_20_diff1": 0.04752612557302544, - "nauc_map_at_20_max": 0.057067886386831156, - "nauc_map_at_20_std": -0.06083000950759207, - "nauc_map_at_3_diff1": 0.06836493519828814, - "nauc_map_at_3_max": 0.033865120590951175, - "nauc_map_at_3_std": -0.05438069275392832, - "nauc_map_at_5_diff1": 0.05660743085479349, - "nauc_map_at_5_max": 0.03746650010933686, - "nauc_map_at_5_std": -0.05718514991261317, - "nauc_mrr_at_1000_diff1": 0.047321574512003584, - "nauc_mrr_at_1000_max": 0.05438206812877677, - "nauc_mrr_at_1000_std": -0.05744428851185208, - "nauc_mrr_at_100_diff1": 0.04650275080611146, - "nauc_mrr_at_100_max": 0.05516022097644162, - "nauc_mrr_at_100_std": -0.0569522165815035, - "nauc_mrr_at_10_diff1": 0.05173169037370162, - "nauc_mrr_at_10_max": 0.043084012251647076, - "nauc_mrr_at_10_std": -0.06272816529553181, - "nauc_mrr_at_1_diff1": 0.08460726813618427, - "nauc_mrr_at_1_max": -0.0065678904232255786, - "nauc_mrr_at_1_std": -0.08707639987423882, - "nauc_mrr_at_20_diff1": 0.04752612557302544, - "nauc_mrr_at_20_max": 0.057067886386831156, - "nauc_mrr_at_20_std": -0.06083000950759207, - "nauc_mrr_at_3_diff1": 0.06836493519828814, - "nauc_mrr_at_3_max": 0.033865120590951175, - "nauc_mrr_at_3_std": -0.05438069275392832, - "nauc_mrr_at_5_diff1": 0.05660743085479349, - "nauc_mrr_at_5_max": 0.03746650010933686, - "nauc_mrr_at_5_std": -0.05718514991261317, - "nauc_ndcg_at_1000_diff1": 0.03410252328119208, - "nauc_ndcg_at_1000_max": 0.07841621563109125, - "nauc_ndcg_at_1000_std": -0.046895103992041366, - "nauc_ndcg_at_100_diff1": 0.014228107355737629, - "nauc_ndcg_at_100_max": 0.09921968541645529, - "nauc_ndcg_at_100_std": -0.031053659815862337, - "nauc_ndcg_at_10_diff1": 0.03861706367387873, - "nauc_ndcg_at_10_max": 0.057559536819589835, - "nauc_ndcg_at_10_std": -0.06759475458350936, - "nauc_ndcg_at_1_diff1": 0.08460726813618427, - "nauc_ndcg_at_1_max": -0.0065678904232255786, - "nauc_ndcg_at_1_std": -0.08707639987423882, - "nauc_ndcg_at_20_diff1": 0.02278919178762813, - "nauc_ndcg_at_20_max": 0.10903022698508856, - "nauc_ndcg_at_20_std": -0.06014408602190423, - "nauc_ndcg_at_3_diff1": 0.06540639729834469, - "nauc_ndcg_at_3_max": 0.04202468283776928, - "nauc_ndcg_at_3_std": -0.048475424580323556, - "nauc_ndcg_at_5_diff1": 0.04659329897322925, - "nauc_ndcg_at_5_max": 0.04714149856887462, - "nauc_ndcg_at_5_std": -0.0530008861423805, - "nauc_precision_at_1000_diff1": 0.3851540616246652, - "nauc_precision_at_1000_max": 0.8244631185807481, - "nauc_precision_at_1000_std": 0.925562817719698, - "nauc_precision_at_100_diff1": -0.43857150703418407, - "nauc_precision_at_100_max": 0.6730613814153111, - "nauc_precision_at_100_std": 0.4521847954868242, - "nauc_precision_at_10_diff1": 0.0128717154868219, - "nauc_precision_at_10_max": 0.08577496630146376, - "nauc_precision_at_10_std": -0.08241386554096451, - "nauc_precision_at_1_diff1": 0.08460726813618427, - "nauc_precision_at_1_max": -0.0065678904232255786, - "nauc_precision_at_1_std": -0.08707639987423882, - "nauc_precision_at_20_diff1": -0.05461591979449159, - "nauc_precision_at_20_max": 0.2859854431282994, - "nauc_precision_at_20_std": -0.0591890252604538, - "nauc_precision_at_3_diff1": 0.05951551890299547, - "nauc_precision_at_3_max": 0.05847203984966021, - "nauc_precision_at_3_std": -0.036729855120451856, - "nauc_precision_at_5_diff1": 0.026571959649107142, - "nauc_precision_at_5_max": 0.06516748857696697, - "nauc_precision_at_5_std": -0.04578783281586591, - "nauc_recall_at_1000_diff1": 0.3851540616246801, - "nauc_recall_at_1000_max": 0.8244631185807924, - "nauc_recall_at_1000_std": 0.925562817719677, - "nauc_recall_at_100_diff1": -0.4385715070341862, - "nauc_recall_at_100_max": 0.6730613814153109, - "nauc_recall_at_100_std": 0.45218479548682355, - "nauc_recall_at_10_diff1": 0.012871715486821715, - "nauc_recall_at_10_max": 0.08577496630146382, - "nauc_recall_at_10_std": -0.08241386554096461, - "nauc_recall_at_1_diff1": 0.08460726813618427, - "nauc_recall_at_1_max": -0.0065678904232255786, - "nauc_recall_at_1_std": -0.08707639987423882, - "nauc_recall_at_20_diff1": -0.054615919794490805, - "nauc_recall_at_20_max": 0.2859854431283007, - "nauc_recall_at_20_std": -0.059189025260453935, - "nauc_recall_at_3_diff1": 0.059515518902995126, - "nauc_recall_at_3_max": 0.05847203984965998, - "nauc_recall_at_3_std": -0.03672985512045184, - "nauc_recall_at_5_diff1": 0.026571959649107007, - "nauc_recall_at_5_max": 0.06516748857696698, - "nauc_recall_at_5_std": -0.045787832815866014, - "ndcg_at_1": 0.07689, - "ndcg_at_10": 0.24754, - "ndcg_at_100": 0.35089, - "ndcg_at_1000": 0.35778, - "ndcg_at_20": 0.30659, - "ndcg_at_3": 0.14811, - "ndcg_at_5": 0.18661, - "precision_at_1": 0.07689, - "precision_at_10": 0.04849, - "precision_at_100": 0.00944, - "precision_at_1000": 0.001, - "precision_at_20": 0.03585, - "precision_at_3": 0.06714, - "precision_at_5": 0.05906, - "recall_at_1": 0.07689, - "recall_at_10": 0.48491, - "recall_at_100": 0.94387, - "recall_at_1000": 0.99575, - "recall_at_20": 0.71698, - "recall_at_3": 0.20142, - "recall_at_5": 0.29528 - } - ] - }, - "task_name": "NIGHTSI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/OxfordFlowersClassification.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/OxfordFlowersClassification.json deleted file mode 100644 index 918613b671..0000000000 --- a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/OxfordFlowersClassification.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", - "evaluation_time": 242.11567449569702, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "accuracy": 0.996078431372549, - "f1": 0.9960915465032875, - "f1_weighted": 0.9960639651382607, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.996078431372549, - "scores_per_experiment": [ - { - "accuracy": 0.996078431372549, - "f1": 0.9960915465032875, - "f1_weighted": 0.9960639651382607 - }, - { - "accuracy": 0.996078431372549, - "f1": 0.9960915465032875, - "f1_weighted": 0.9960639651382607 - }, - { - "accuracy": 0.996078431372549, - "f1": 0.9960915465032875, - "f1_weighted": 0.9960639651382607 - }, - { - "accuracy": 0.996078431372549, - "f1": 0.9960915465032875, - "f1_weighted": 0.9960639651382607 - }, - { - "accuracy": 0.996078431372549, - "f1": 0.9960915465032875, - "f1_weighted": 0.9960639651382607 - } - ] - } - ] - }, - "task_name": "OxfordFlowersClassification" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json deleted file mode 100644 index a9111cc617..0000000000 --- a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 146.90911436080933, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "accuracy": 0.35315024232633274, - "f1": 0.5817890400372185, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "lrap": 0.49049121567043397, - "main_score": 0.35315024232633274, - "scores_per_experiment": [ - { - "accuracy": 0.3667205169628433, - "f1": 0.591090519122089, - "lrap": 0.5039753859271227 - }, - { - "accuracy": 0.34995961227786754, - "f1": 0.5618180857706963, - "lrap": 0.48571452611739013 - }, - { - "accuracy": 0.34693053311793215, - "f1": 0.5774757520432393, - "lrap": 0.49124652216837067 - }, - { - "accuracy": 0.33582390953150243, - "f1": 0.5718289254309215, - "lrap": 0.4711502647639497 - }, - { - "accuracy": 0.36631663974151857, - "f1": 0.606731917819147, - "lrap": 0.5003693793753363 - } - ] - } - ] - }, - "task_name": "VOC2007" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json deleted file mode 100644 index a84e0519fe..0000000000 --- a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "facebook/dinov2-large", "revision": "47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/GLDv2I2IRetrieval.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/GLDv2I2IRetrieval.json deleted file mode 100644 index 2ce64ec37b..0000000000 --- a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/GLDv2I2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c6b162ee349adb293901128a18c0b446f7b43457", - "evaluation_time": 4408.689854860306, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.4535, - "cv_recall_at_10": 0.57219, - "cv_recall_at_100": 0.6519, - "cv_recall_at_1000": 0.73339, - "cv_recall_at_20": 0.59433, - "cv_recall_at_3": 0.50753, - "cv_recall_at_5": 0.5403, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.28941, - "map_at_1": 0.05847, - "map_at_10": 0.15245, - "map_at_100": 0.17257, - "map_at_1000": 0.1746, - "map_at_20": 0.16358, - "map_at_3": 0.10746, - "map_at_5": 0.1285, - "mrr_at_1": 0.4534986713906112, - "mrr_at_10": 0.48867237476626324, - "mrr_at_100": 0.4915899189723126, - "mrr_at_1000": 0.49191288914802445, - "mrr_at_20": 0.4902356580948968, - "mrr_at_3": 0.4769707705934455, - "mrr_at_5": 0.48436669619131967, - "nauc_cv_recall_at_1000_diff1": 0.34148107488406604, - "nauc_cv_recall_at_1000_max": 0.5657029091377153, - "nauc_cv_recall_at_1000_std": 0.3365411347443642, - "nauc_cv_recall_at_100_diff1": 0.3808148655813085, - "nauc_cv_recall_at_100_max": 0.5494694185142655, - "nauc_cv_recall_at_100_std": 0.2927349479831974, - "nauc_cv_recall_at_10_diff1": 0.40118856921921564, - "nauc_cv_recall_at_10_max": 0.5702709786136617, - "nauc_cv_recall_at_10_std": 0.2506097969647151, - "nauc_cv_recall_at_1_diff1": 0.43232239408517925, - "nauc_cv_recall_at_1_max": 0.6111730350701059, - "nauc_cv_recall_at_1_std": 0.2626457116510796, - "nauc_cv_recall_at_20_diff1": 0.3976444877640721, - "nauc_cv_recall_at_20_max": 0.5561717117356078, - "nauc_cv_recall_at_20_std": 0.258312177592681, - "nauc_cv_recall_at_3_diff1": 0.4162409238600696, - "nauc_cv_recall_at_3_max": 0.5731658448651727, - "nauc_cv_recall_at_3_std": 0.24259357095035114, - "nauc_cv_recall_at_5_diff1": 0.40222757559525246, - "nauc_cv_recall_at_5_max": 0.5740443308429867, - "nauc_cv_recall_at_5_std": 0.2638681693812113, - "nauc_map_at_1000_diff1": 0.18861481679640155, - "nauc_map_at_1000_max": 0.49541010487450027, - "nauc_map_at_1000_std": 0.25479149329445355, - "nauc_map_at_100_diff1": 0.19036469102447715, - "nauc_map_at_100_max": 0.49424236992956605, - "nauc_map_at_100_std": 0.2508171110383186, - "nauc_map_at_10_diff1": 0.22996775077271184, - "nauc_map_at_10_max": 0.4659523068740507, - "nauc_map_at_10_std": 0.17873558257018604, - "nauc_map_at_1_diff1": 0.4814554391164708, - "nauc_map_at_1_max": 0.22663592739850807, - "nauc_map_at_1_std": -0.016374913627465144, - "nauc_map_at_20_diff1": 0.20523019715428878, - "nauc_map_at_20_max": 0.48572148693290934, - "nauc_map_at_20_std": 0.21802056174164897, - "nauc_map_at_3_diff1": 0.3578012044738236, - "nauc_map_at_3_max": 0.35368283917955673, - "nauc_map_at_3_std": 0.07546794596340074, - "nauc_map_at_5_diff1": 0.2966436966851955, - "nauc_map_at_5_max": 0.4113893523345609, - "nauc_map_at_5_std": 0.12432827131997834, - "nauc_mrr_at_1000_diff1": 0.4219354542268342, - "nauc_mrr_at_1000_max": 0.5938517870610833, - "nauc_mrr_at_1000_std": 0.25789989416553666, - "nauc_mrr_at_100_diff1": 0.42195820028912456, - "nauc_mrr_at_100_max": 0.5938569414601591, - "nauc_mrr_at_100_std": 0.25780053121453195, - "nauc_mrr_at_10_diff1": 0.4226196920315362, - "nauc_mrr_at_10_max": 0.5949390240575794, - "nauc_mrr_at_10_std": 0.2568484636479838, - "nauc_mrr_at_1_diff1": 0.43232239408517925, - "nauc_mrr_at_1_max": 0.6111730350701059, - "nauc_mrr_at_1_std": 0.2626457116510796, - "nauc_mrr_at_20_diff1": 0.4223978068861933, - "nauc_mrr_at_20_max": 0.59408807498729, - "nauc_mrr_at_20_std": 0.2573461830046759, - "nauc_mrr_at_3_diff1": 0.4262502276324796, - "nauc_mrr_at_3_max": 0.5954118493110601, - "nauc_mrr_at_3_std": 0.2545125910785488, - "nauc_mrr_at_5_diff1": 0.4228118558706074, - "nauc_mrr_at_5_max": 0.5953723133930325, - "nauc_mrr_at_5_std": 0.25854738381668874, - "nauc_ndcg_at_1000_diff1": 0.22343292588387825, - "nauc_ndcg_at_1000_max": 0.5276606021449388, - "nauc_ndcg_at_1000_std": 0.29728763467934904, - "nauc_ndcg_at_100_diff1": 0.23190304701399703, - "nauc_ndcg_at_100_max": 0.5245071773764476, - "nauc_ndcg_at_100_std": 0.2783843293138034, - "nauc_ndcg_at_10_diff1": 0.19316555007961791, - "nauc_ndcg_at_10_max": 0.5825075587136279, - "nauc_ndcg_at_10_std": 0.3092465370973502, - "nauc_ndcg_at_1_diff1": 0.43232239408517925, - "nauc_ndcg_at_1_max": 0.6111730350701059, - "nauc_ndcg_at_1_std": 0.2626457116510796, - "nauc_ndcg_at_20_diff1": 0.21755951662398187, - "nauc_ndcg_at_20_max": 0.5475682374017373, - "nauc_ndcg_at_20_std": 0.2860733738075765, - "nauc_ndcg_at_3_diff1": 0.24971063630292664, - "nauc_ndcg_at_3_max": 0.6086291585528392, - "nauc_ndcg_at_3_std": 0.2992078517726314, - "nauc_ndcg_at_5_diff1": 0.21400973879521373, - "nauc_ndcg_at_5_max": 0.605819692716699, - "nauc_ndcg_at_5_std": 0.31456299770488383, - "nauc_precision_at_1000_diff1": -0.044719214820325454, - "nauc_precision_at_1000_max": 0.469245361680825, - "nauc_precision_at_1000_std": 0.459126550464243, - "nauc_precision_at_100_diff1": -0.0361873993762923, - "nauc_precision_at_100_max": 0.5372410748582264, - "nauc_precision_at_100_std": 0.49655377124356126, - "nauc_precision_at_10_diff1": 0.02048413751839119, - "nauc_precision_at_10_max": 0.601718588539165, - "nauc_precision_at_10_std": 0.38924805371157156, - "nauc_precision_at_1_diff1": 0.43232239408517925, - "nauc_precision_at_1_max": 0.6111730350701059, - "nauc_precision_at_1_std": 0.2626457116510796, - "nauc_precision_at_20_diff1": -0.01598332604841292, - "nauc_precision_at_20_max": 0.5860681313676941, - "nauc_precision_at_20_std": 0.4447422992419558, - "nauc_precision_at_3_diff1": 0.18237525035807778, - "nauc_precision_at_3_max": 0.6073405177582907, - "nauc_precision_at_3_std": 0.3148368780578123, - "nauc_precision_at_5_diff1": 0.10080092161444594, - "nauc_precision_at_5_max": 0.6113824415227623, - "nauc_precision_at_5_std": 0.3542916057954643, - "nauc_recall_at_1000_diff1": 0.11156694655914166, - "nauc_recall_at_1000_max": 0.4029532703125521, - "nauc_recall_at_1000_std": 0.3191218466863474, - "nauc_recall_at_100_diff1": 0.15346884478259795, - "nauc_recall_at_100_max": 0.4263756821168572, - "nauc_recall_at_100_std": 0.2667922963100197, - "nauc_recall_at_10_diff1": 0.20481260563318826, - "nauc_recall_at_10_max": 0.42307599013145597, - "nauc_recall_at_10_std": 0.16281281054955174, - "nauc_recall_at_1_diff1": 0.4814554391164708, - "nauc_recall_at_1_max": 0.22663592739850807, - "nauc_recall_at_1_std": -0.016374913627465144, - "nauc_recall_at_20_diff1": 0.1805511540924871, - "nauc_recall_at_20_max": 0.4341772622005704, - "nauc_recall_at_20_std": 0.20225716907182484, - "nauc_recall_at_3_diff1": 0.343757341612119, - "nauc_recall_at_3_max": 0.33208304495011726, - "nauc_recall_at_3_std": 0.0633600297459121, - "nauc_recall_at_5_diff1": 0.2784926120476109, - "nauc_recall_at_5_max": 0.3814328242345907, - "nauc_recall_at_5_std": 0.11379188945081205, - "ndcg_at_1": 0.4535, - "ndcg_at_10": 0.28941, - "ndcg_at_100": 0.26716, - "ndcg_at_1000": 0.29187, - "ndcg_at_20": 0.26099, - "ndcg_at_3": 0.37311, - "ndcg_at_5": 0.33283, - "precision_at_1": 0.4535, - "precision_at_10": 0.19699, - "precision_at_100": 0.03368, - "precision_at_1000": 0.00467, - "precision_at_20": 0.12161, - "precision_at_3": 0.34485, - "precision_at_5": 0.28131, - "recall_at_1": 0.05847, - "recall_at_10": 0.17682, - "recall_at_100": 0.25581, - "recall_at_1000": 0.34658, - "recall_at_20": 0.20188, - "recall_at_3": 0.11247, - "recall_at_5": 0.13964 - } - ] - }, - "task_name": "GLDv2I2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/MNIST.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/MNIST.json deleted file mode 100644 index 2a1a442fe1..0000000000 --- a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 53.76097369194031, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "accuracy": 0.8656600000000001, - "f1": 0.8637301373219758, - "f1_weighted": 0.8657989045644241, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8656600000000001, - "scores_per_experiment": [ - { - "accuracy": 0.8746, - "f1": 0.873539621523118, - "f1_weighted": 0.8756435923724779 - }, - { - "accuracy": 0.8274, - "f1": 0.8235176307759297, - "f1_weighted": 0.8265648513841878 - }, - { - "accuracy": 0.8719, - "f1": 0.8699976650278503, - "f1_weighted": 0.8715310177539896 - }, - { - "accuracy": 0.8787, - "f1": 0.8776085155251211, - "f1_weighted": 0.8795306016429042 - }, - { - "accuracy": 0.8757, - "f1": 0.8739872537578602, - "f1_weighted": 0.8757244596685608 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/NIGHTSI2IRetrieval.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/NIGHTSI2IRetrieval.json deleted file mode 100644 index d589588bc8..0000000000 --- a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/NIGHTSI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", - "evaluation_time": 108.04806113243103, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.08208, - "cv_recall_at_10": 0.50566, - "cv_recall_at_100": 0.94104, - "cv_recall_at_1000": 0.99623, - "cv_recall_at_20": 0.71557, - "cv_recall_at_3": 0.21085, - "cv_recall_at_5": 0.31321, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.25876, - "map_at_1": 0.08208, - "map_at_10": 0.18424, - "map_at_100": 0.20549, - "map_at_1000": 0.20578, - "map_at_20": 0.19902, - "map_at_3": 0.13569, - "map_at_5": 0.15878, - "mrr_at_1": 0.0820754716981132, - "mrr_at_10": 0.1842383572926024, - "mrr_at_100": 0.20549248220385244, - "mrr_at_1000": 0.2057831548741384, - "mrr_at_20": 0.19901605762390864, - "mrr_at_3": 0.13569182389937115, - "mrr_at_5": 0.15878144654088, - "nauc_cv_recall_at_1000_diff1": -0.16304855275446334, - "nauc_cv_recall_at_1000_max": 0.7718837535013962, - "nauc_cv_recall_at_1000_std": 0.9489379084967337, - "nauc_cv_recall_at_100_diff1": -0.32002987861811505, - "nauc_cv_recall_at_100_max": 0.6210308123249305, - "nauc_cv_recall_at_100_std": 0.412720821662, - "nauc_cv_recall_at_10_diff1": -0.013857139967228161, - "nauc_cv_recall_at_10_max": 0.10358654643659096, - "nauc_cv_recall_at_10_std": -0.051303658487245135, - "nauc_cv_recall_at_1_diff1": 0.02898698222662205, - "nauc_cv_recall_at_1_max": 0.042667826269317664, - "nauc_cv_recall_at_1_std": -0.03785054078809807, - "nauc_cv_recall_at_20_diff1": -0.08321837405985853, - "nauc_cv_recall_at_20_max": 0.190030565990472, - "nauc_cv_recall_at_20_std": -0.03864319335986944, - "nauc_cv_recall_at_3_diff1": -0.02082871885741471, - "nauc_cv_recall_at_3_max": 0.05479275316599656, - "nauc_cv_recall_at_3_std": -0.04492840197809061, - "nauc_cv_recall_at_5_diff1": -0.017987852753899084, - "nauc_cv_recall_at_5_max": 0.058657445633728725, - "nauc_cv_recall_at_5_std": -0.05043285328578423, - "nauc_map_at_1000_diff1": -0.006510507398643199, - "nauc_map_at_1000_max": 0.0721993746295309, - "nauc_map_at_1000_std": -0.036777170133303096, - "nauc_map_at_100_diff1": -0.007006312689406556, - "nauc_map_at_100_max": 0.07287810601748373, - "nauc_map_at_100_std": -0.03630568232615193, - "nauc_map_at_10_diff1": -0.001302596447271446, - "nauc_map_at_10_max": 0.06448303401621405, - "nauc_map_at_10_std": -0.04200841918198155, - "nauc_map_at_1_diff1": 0.02898698222662205, - "nauc_map_at_1_max": 0.042667826269317664, - "nauc_map_at_1_std": -0.03785054078809807, - "nauc_map_at_20_diff1": -0.006772172318400011, - "nauc_map_at_20_max": 0.070340640864153, - "nauc_map_at_20_std": -0.04049217683866346, - "nauc_map_at_3_diff1": -0.0007283763921229228, - "nauc_map_at_3_max": 0.05298823394700488, - "nauc_map_at_3_std": -0.04169663282302213, - "nauc_map_at_5_diff1": -0.001801122555208517, - "nauc_map_at_5_max": 0.05479514191945225, - "nauc_map_at_5_std": -0.04264537947699984, - "nauc_mrr_at_1000_diff1": -0.006510507398643199, - "nauc_mrr_at_1000_max": 0.0721993746295309, - "nauc_mrr_at_1000_std": -0.036777170133303096, - "nauc_mrr_at_100_diff1": -0.007006312689406556, - "nauc_mrr_at_100_max": 0.07287810601748373, - "nauc_mrr_at_100_std": -0.03630568232615193, - "nauc_mrr_at_10_diff1": -0.001302596447271446, - "nauc_mrr_at_10_max": 0.06448303401621405, - "nauc_mrr_at_10_std": -0.04200841918198155, - "nauc_mrr_at_1_diff1": 0.02898698222662205, - "nauc_mrr_at_1_max": 0.042667826269317664, - "nauc_mrr_at_1_std": -0.03785054078809807, - "nauc_mrr_at_20_diff1": -0.006772172318400011, - "nauc_mrr_at_20_max": 0.070340640864153, - "nauc_mrr_at_20_std": -0.04049217683866346, - "nauc_mrr_at_3_diff1": -0.0007283763921229228, - "nauc_mrr_at_3_max": 0.05298823394700488, - "nauc_mrr_at_3_std": -0.04169663282302213, - "nauc_mrr_at_5_diff1": -0.001801122555208517, - "nauc_mrr_at_5_max": 0.05479514191945225, - "nauc_mrr_at_5_std": -0.04264537947699984, - "nauc_ndcg_at_1000_diff1": -0.016298224329016942, - "nauc_ndcg_at_1000_max": 0.09062695649942037, - "nauc_ndcg_at_1000_std": -0.02731715426532132, - "nauc_ndcg_at_100_diff1": -0.028264767453912674, - "nauc_ndcg_at_100_max": 0.10964458205541326, - "nauc_ndcg_at_100_std": -0.012838091552479945, - "nauc_ndcg_at_10_diff1": -0.006124175666309968, - "nauc_ndcg_at_10_max": 0.07645936667113105, - "nauc_ndcg_at_10_std": -0.04477882894615783, - "nauc_ndcg_at_1_diff1": 0.02898698222662205, - "nauc_ndcg_at_1_max": 0.042667826269317664, - "nauc_ndcg_at_1_std": -0.03785054078809807, - "nauc_ndcg_at_20_diff1": -0.02473085630921952, - "nauc_ndcg_at_20_max": 0.09743286781620096, - "nauc_ndcg_at_20_std": -0.040343009950076295, - "nauc_ndcg_at_3_diff1": -0.007308502881026242, - "nauc_ndcg_at_3_max": 0.05369162294449823, - "nauc_ndcg_at_3_std": -0.04274088343527608, - "nauc_ndcg_at_5_diff1": -0.007620898518130469, - "nauc_ndcg_at_5_max": 0.056222806231225546, - "nauc_ndcg_at_5_std": -0.04512864978013532, - "nauc_precision_at_1000_diff1": -0.16304855275437521, - "nauc_precision_at_1000_max": 0.771883753501379, - "nauc_precision_at_1000_std": 0.9489379084967005, - "nauc_precision_at_100_diff1": -0.3200298786181148, - "nauc_precision_at_100_max": 0.6210308123249351, - "nauc_precision_at_100_std": 0.4127208216619996, - "nauc_precision_at_10_diff1": -0.013857139967228201, - "nauc_precision_at_10_max": 0.10358654643659095, - "nauc_precision_at_10_std": -0.05130365848724515, - "nauc_precision_at_1_diff1": 0.02898698222662205, - "nauc_precision_at_1_max": 0.042667826269317664, - "nauc_precision_at_1_std": -0.03785054078809807, - "nauc_precision_at_20_diff1": -0.08321837405985895, - "nauc_precision_at_20_max": 0.19003056599047163, - "nauc_precision_at_20_std": -0.03864319335986984, - "nauc_precision_at_3_diff1": -0.020828718857414826, - "nauc_precision_at_3_max": 0.05479275316599652, - "nauc_precision_at_3_std": -0.04492840197809076, - "nauc_precision_at_5_diff1": -0.01798785275389913, - "nauc_precision_at_5_max": 0.05865744563372872, - "nauc_precision_at_5_std": -0.05043285328578428, - "nauc_recall_at_1000_diff1": -0.16304855275446334, - "nauc_recall_at_1000_max": 0.7718837535013962, - "nauc_recall_at_1000_std": 0.9489379084967337, - "nauc_recall_at_100_diff1": -0.32002987861811505, - "nauc_recall_at_100_max": 0.6210308123249305, - "nauc_recall_at_100_std": 0.412720821662, - "nauc_recall_at_10_diff1": -0.013857139967228161, - "nauc_recall_at_10_max": 0.10358654643659096, - "nauc_recall_at_10_std": -0.051303658487245135, - "nauc_recall_at_1_diff1": 0.02898698222662205, - "nauc_recall_at_1_max": 0.042667826269317664, - "nauc_recall_at_1_std": -0.03785054078809807, - "nauc_recall_at_20_diff1": -0.08321837405985853, - "nauc_recall_at_20_max": 0.190030565990472, - "nauc_recall_at_20_std": -0.03864319335986944, - "nauc_recall_at_3_diff1": -0.02082871885741471, - "nauc_recall_at_3_max": 0.05479275316599656, - "nauc_recall_at_3_std": -0.04492840197809061, - "nauc_recall_at_5_diff1": -0.017987852753899084, - "nauc_recall_at_5_max": 0.058657445633728725, - "nauc_recall_at_5_std": -0.05043285328578423, - "ndcg_at_1": 0.08208, - "ndcg_at_10": 0.25876, - "ndcg_at_100": 0.3553, - "ndcg_at_1000": 0.36245, - "ndcg_at_20": 0.3121, - "ndcg_at_3": 0.15486, - "ndcg_at_5": 0.19675, - "precision_at_1": 0.08208, - "precision_at_10": 0.05057, - "precision_at_100": 0.00941, - "precision_at_1000": 0.001, - "precision_at_20": 0.03578, - "precision_at_3": 0.07028, - "precision_at_5": 0.06264, - "recall_at_1": 0.08208, - "recall_at_10": 0.50566, - "recall_at_100": 0.94104, - "recall_at_1000": 0.99623, - "recall_at_20": 0.71557, - "recall_at_3": 0.21085, - "recall_at_5": 0.31321 - } - ] - }, - "task_name": "NIGHTSI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/OxfordFlowersClassification.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/OxfordFlowersClassification.json deleted file mode 100644 index 546e17103a..0000000000 --- a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/OxfordFlowersClassification.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", - "evaluation_time": 154.27621984481812, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "accuracy": 0.9950980392156863, - "f1": 0.9951626579948023, - "f1_weighted": 0.9950347807569507, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9950980392156863, - "scores_per_experiment": [ - { - "accuracy": 0.9950980392156863, - "f1": 0.9951502085705881, - "f1_weighted": 0.9950223313327365 - }, - { - "accuracy": 0.996078431372549, - "f1": 0.9961903907253213, - "f1_weighted": 0.9960625134874698 - }, - { - "accuracy": 0.9941176470588236, - "f1": 0.9941722735369257, - "f1_weighted": 0.9940443962990739 - }, - { - "accuracy": 0.9950980392156863, - "f1": 0.9951502085705881, - "f1_weighted": 0.9950223313327365 - }, - { - "accuracy": 0.9950980392156863, - "f1": 0.9951502085705881, - "f1_weighted": 0.9950223313327365 - } - ] - } - ] - }, - "task_name": "OxfordFlowersClassification" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/STS17MultilingualVisualSTS.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/STS17MultilingualVisualSTS.json deleted file mode 100644 index 7d7cd4eef5..0000000000 --- a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/STS17MultilingualVisualSTS.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", - "evaluation_time": 59.09429907798767, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.09188810319729813, - "cosine_spearman": 0.14346112848523435, - "euclidean_pearson": 0.13082342242931153, - "euclidean_spearman": 0.1468297994946006, - "hf_subset": "ko-ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.14346112848523435, - "manhattan_pearson": 0.13001422733840193, - "manhattan_spearman": 0.14598848320610913, - "pearson": 0.09188810319729813, - "spearman": 0.14346112848523435 - }, - { - "cosine_pearson": 0.27592688750203054, - "cosine_spearman": 0.32201421731334023, - "euclidean_pearson": 0.3443186066520749, - "euclidean_spearman": 0.31841020538106835, - "hf_subset": "ar-ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.32201421731334023, - "manhattan_pearson": 0.3436882295376996, - "manhattan_spearman": 0.31904272427881586, - "pearson": 0.27592688750203054, - "spearman": 0.32201421731334023 - }, - { - "cosine_pearson": 0.11597578416806917, - "cosine_spearman": 0.12294269023753306, - "euclidean_pearson": 0.09300559683007897, - "euclidean_spearman": 0.1211560040681997, - "hf_subset": "en-ar", - "languages": [ - "eng-Latn", - "ara-Arab" - ], - "main_score": 0.12294269023753306, - "manhattan_pearson": 0.09641990638923058, - "manhattan_spearman": 0.12381604457848083, - "pearson": 0.11597578416806917, - "spearman": 0.12294269023753306 - }, - { - "cosine_pearson": 0.2786109978789415, - "cosine_spearman": 0.3115303408523445, - "euclidean_pearson": 0.3040768135184991, - "euclidean_spearman": 0.31091107800856443, - "hf_subset": "en-de", - "languages": [ - "eng-Latn", - "deu-Latn" - ], - "main_score": 0.3115303408523445, - "manhattan_pearson": 0.3032684317040234, - "manhattan_spearman": 0.3090748156840843, - "pearson": 0.2786109978789415, - "spearman": 0.3115303408523445 - }, - { - "cosine_pearson": 0.3800008798157441, - "cosine_spearman": 0.42635650979604245, - "euclidean_pearson": 0.41789590060032134, - "euclidean_spearman": 0.4265325634164343, - "hf_subset": "en-en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.42635650979604245, - "manhattan_pearson": 0.41787926409683357, - "manhattan_spearman": 0.4250380296258589, - "pearson": 0.3800008798157441, - "spearman": 0.42635650979604245 - }, - { - "cosine_pearson": 0.27677049410622256, - "cosine_spearman": 0.252972959285875, - "euclidean_pearson": 0.27226752994699055, - "euclidean_spearman": 0.2526223218944939, - "hf_subset": "en-tr", - "languages": [ - "eng-Latn", - "tur-Latn" - ], - "main_score": 0.252972959285875, - "manhattan_pearson": 0.2685258863887162, - "manhattan_spearman": 0.2487845341304967, - "pearson": 0.27677049410622256, - "spearman": 0.252972959285875 - }, - { - "cosine_pearson": 0.152944923551701, - "cosine_spearman": 0.11263692422646802, - "euclidean_pearson": 0.14461819162711873, - "euclidean_spearman": 0.11089890891421848, - "hf_subset": "es-en", - "languages": [ - "spa-Latn", - "eng-Latn" - ], - "main_score": 0.11263692422646802, - "manhattan_pearson": 0.14243235664712942, - "manhattan_spearman": 0.10971716614273982, - "pearson": 0.152944923551701, - "spearman": 0.11263692422646802 - }, - { - "cosine_pearson": 0.3281526313284088, - "cosine_spearman": 0.35267614845127765, - "euclidean_pearson": 0.3866300928881876, - "euclidean_spearman": 0.35291147634567144, - "hf_subset": "es-es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.35267614845127765, - "manhattan_pearson": 0.38706458323264775, - "manhattan_spearman": 0.35592459624839945, - "pearson": 0.3281526313284088, - "spearman": 0.35267614845127765 - }, - { - "cosine_pearson": 0.18223800347528196, - "cosine_spearman": 0.20383511232761864, - "euclidean_pearson": 0.19417671758750618, - "euclidean_spearman": 0.20500559982563873, - "hf_subset": "fr-en", - "languages": [ - "fra-Latn", - "eng-Latn" - ], - "main_score": 0.20383511232761864, - "manhattan_pearson": 0.1925857152284029, - "manhattan_spearman": 0.20148375862469953, - "pearson": 0.18223800347528196, - "spearman": 0.20383511232761864 - }, - { - "cosine_pearson": 0.29206885904401914, - "cosine_spearman": 0.2793232914240653, - "euclidean_pearson": 0.29837255050901446, - "euclidean_spearman": 0.27901538978665513, - "hf_subset": "it-en", - "languages": [ - "ita-Latn", - "eng-Latn" - ], - "main_score": 0.2793232914240653, - "manhattan_pearson": 0.29586809569328343, - "manhattan_spearman": 0.2752967375515661, - "pearson": 0.29206885904401914, - "spearman": 0.2793232914240653 - }, - { - "cosine_pearson": 0.2843218379631862, - "cosine_spearman": 0.2952572972591813, - "euclidean_pearson": 0.29344751852344736, - "euclidean_spearman": 0.2932607415729033, - "hf_subset": "nl-en", - "languages": [ - "nld-Latn", - "eng-Latn" - ], - "main_score": 0.2952572972591813, - "manhattan_pearson": 0.2935374530503633, - "manhattan_spearman": 0.2921060143334773, - "pearson": 0.2843218379631862, - "spearman": 0.2952572972591813 - } - ] - }, - "task_name": "STS17MultilingualVisualSTS" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json deleted file mode 100644 index e036d66bea..0000000000 --- a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 52.176159620285034, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "accuracy": 0.4340064620355412, - "f1": 0.6759098897017933, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "lrap": 0.5801337843295732, - "main_score": 0.4340064620355412, - "scores_per_experiment": [ - { - "accuracy": 0.4184168012924071, - "f1": 0.6581607635477135, - "lrap": 0.563152373900564 - }, - { - "accuracy": 0.44810177705977383, - "f1": 0.6892652132242753, - "lrap": 0.5981864790881464 - }, - { - "accuracy": 0.44426494345718903, - "f1": 0.6797744937645096, - "lrap": 0.5992208535272061 - }, - { - "accuracy": 0.43073505654281097, - "f1": 0.6706176542382969, - "lrap": 0.5713583737210639 - }, - { - "accuracy": 0.42851373182552505, - "f1": 0.6817313237341718, - "lrap": 0.5687508414108857 - } - ] - } - ] - }, - "task_name": "VOC2007" -} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json deleted file mode 100644 index 5789f8b54f..0000000000 --- a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "facebook/dinov2-small", "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR10.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR10.json deleted file mode 100644 index 328e3b46b4..0000000000 --- a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR10.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", - "evaluation_time": 48.409629821777344, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.92594, - "f1": 0.9260547788569216, - "f1_weighted": 0.9260547788569216, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.92594, - "scores_per_experiment": [ - { - "accuracy": 0.9249, - "f1": 0.9250702364213208, - "f1_weighted": 0.9250702364213208 - }, - { - "accuracy": 0.9232, - "f1": 0.9231630424362665, - "f1_weighted": 0.9231630424362666 - }, - { - "accuracy": 0.9217, - "f1": 0.9218520263643336, - "f1_weighted": 0.9218520263643335 - }, - { - "accuracy": 0.9308, - "f1": 0.9308028706566229, - "f1_weighted": 0.930802870656623 - }, - { - "accuracy": 0.9291, - "f1": 0.9293857184060645, - "f1_weighted": 0.9293857184060643 - } - ] - } - ] - }, - "task_name": "CIFAR10" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR100.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR100.json deleted file mode 100644 index 0ef0c4ba83..0000000000 --- a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/CIFAR100.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", - "evaluation_time": 82.63413286209106, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.7413199999999999, - "f1": 0.7440710594068098, - "f1_weighted": 0.7440710594068098, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7413199999999999, - "scores_per_experiment": [ - { - "accuracy": 0.7393, - "f1": 0.7415575159167781, - "f1_weighted": 0.7415575159167781 - }, - { - "accuracy": 0.7449, - "f1": 0.7471305285419396, - "f1_weighted": 0.7471305285419396 - }, - { - "accuracy": 0.7428, - "f1": 0.7465020754943685, - "f1_weighted": 0.7465020754943685 - }, - { - "accuracy": 0.7423, - "f1": 0.744902146154522, - "f1_weighted": 0.7449021461545219 - }, - { - "accuracy": 0.7373, - "f1": 0.7402630309264411, - "f1_weighted": 0.7402630309264413 - } - ] - } - ] - }, - "task_name": "CIFAR100" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/ImageNet10Clustering.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/ImageNet10Clustering.json deleted file mode 100644 index 866c25f1fe..0000000000 --- a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/ImageNet10Clustering.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "dataset_revision": "88f8a6d47c257895094c5ad81e67ba751771fc99", - "evaluation_time": 50.5283944606781, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "ari": 0.9791753845317047, - "cluster_accuracy": 0.9905384615384616, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9739308018003532, - "nmi": 0.9739308018003532, - "v_measure": 0.9739308018003532 - } - ] - }, - "task_name": "ImageNet10Clustering" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNIST.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNIST.json deleted file mode 100644 index 264a93931c..0000000000 --- a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 46.19513821601868, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.95434, - "f1": 0.9542219615719318, - "f1_weighted": 0.9544309958560502, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.95434, - "scores_per_experiment": [ - { - "accuracy": 0.9543, - "f1": 0.9542073639183843, - "f1_weighted": 0.9544394896400503 - }, - { - "accuracy": 0.949, - "f1": 0.9487069493447204, - "f1_weighted": 0.9489175533097969 - }, - { - "accuracy": 0.9586, - "f1": 0.9583645832273339, - "f1_weighted": 0.958733263357083 - }, - { - "accuracy": 0.9522, - "f1": 0.9522190790543403, - "f1_weighted": 0.9523957230838139 - }, - { - "accuracy": 0.9576, - "f1": 0.9576118323148799, - "f1_weighted": 0.957668949889507 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNISTZeroShot.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNISTZeroShot.json deleted file mode 100644 index 06769b48eb..0000000000 --- a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/MNISTZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 38.23384070396423, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.8361, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8361 - } - ] - }, - "task_name": "MNISTZeroShot" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/WITT2IRetrieval.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/WITT2IRetrieval.json deleted file mode 100644 index 5e35a43d4c..0000000000 --- a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/WITT2IRetrieval.json +++ /dev/null @@ -1,1936 +0,0 @@ -{ - "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", - "evaluation_time": 66.81307911872864, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.14831, - "cv_recall_at_10": 0.31011, - "cv_recall_at_100": 0.65056, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.39438, - "cv_recall_at_3": 0.20449, - "cv_recall_at_5": 0.25169, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.21879, - "map_at_1": 0.14831, - "map_at_10": 0.19083, - "map_at_100": 0.20226, - "map_at_1000": 0.2037, - "map_at_20": 0.19631, - "map_at_3": 0.17247, - "map_at_5": 0.1832, - "mrr_at_1": 0.14831460674157304, - "mrr_at_10": 0.19082753700731223, - "mrr_at_100": 0.20225932653601883, - "mrr_at_1000": 0.2037004971452857, - "mrr_at_20": 0.19630941951644984, - "mrr_at_3": 0.17247191011235952, - "mrr_at_5": 0.18320224719101122, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.26807037584184995, - "nauc_cv_recall_at_100_max": 0.48453045854865856, - "nauc_cv_recall_at_100_std": 0.5051430893683861, - "nauc_cv_recall_at_10_diff1": 0.3240598073692733, - "nauc_cv_recall_at_10_max": 0.34736522495740113, - "nauc_cv_recall_at_10_std": 0.3320976688540044, - "nauc_cv_recall_at_1_diff1": 0.5290394349724978, - "nauc_cv_recall_at_1_max": 0.48781134075251725, - "nauc_cv_recall_at_1_std": 0.35022750945671843, - "nauc_cv_recall_at_20_diff1": 0.33883138486795833, - "nauc_cv_recall_at_20_max": 0.3700807956874158, - "nauc_cv_recall_at_20_std": 0.35876997956540574, - "nauc_cv_recall_at_3_diff1": 0.4534575089822943, - "nauc_cv_recall_at_3_max": 0.39864663168457115, - "nauc_cv_recall_at_3_std": 0.31624509525368005, - "nauc_cv_recall_at_5_diff1": 0.3859997231322966, - "nauc_cv_recall_at_5_max": 0.39631469310533485, - "nauc_cv_recall_at_5_std": 0.33543675880208573, - "nauc_map_at_1000_diff1": 0.4622335272968312, - "nauc_map_at_1000_max": 0.4430261662650935, - "nauc_map_at_1000_std": 0.34589976349391555, - "nauc_map_at_100_diff1": 0.46143307956438734, - "nauc_map_at_100_max": 0.442923564402142, - "nauc_map_at_100_std": 0.346192058081346, - "nauc_map_at_10_diff1": 0.4618222747153087, - "nauc_map_at_10_max": 0.43767798670996355, - "nauc_map_at_10_std": 0.33999642815590886, - "nauc_map_at_1_diff1": 0.5290394349724978, - "nauc_map_at_1_max": 0.48781134075251725, - "nauc_map_at_1_std": 0.35022750945671843, - "nauc_map_at_20_diff1": 0.4620254574672781, - "nauc_map_at_20_max": 0.4390165307515562, - "nauc_map_at_20_std": 0.3426591907598361, - "nauc_map_at_3_diff1": 0.492887071202886, - "nauc_map_at_3_max": 0.4476533348870664, - "nauc_map_at_3_std": 0.3350468692448414, - "nauc_map_at_5_diff1": 0.4731605872794598, - "nauc_map_at_5_max": 0.44620481412940166, - "nauc_map_at_5_std": 0.3401122978842973, - "nauc_mrr_at_1000_diff1": 0.4622335272968312, - "nauc_mrr_at_1000_max": 0.4430261662650935, - "nauc_mrr_at_1000_std": 0.34589976349391555, - "nauc_mrr_at_100_diff1": 0.46143307956438734, - "nauc_mrr_at_100_max": 0.442923564402142, - "nauc_mrr_at_100_std": 0.346192058081346, - "nauc_mrr_at_10_diff1": 0.4618222747153087, - "nauc_mrr_at_10_max": 0.43767798670996355, - "nauc_mrr_at_10_std": 0.33999642815590886, - "nauc_mrr_at_1_diff1": 0.5290394349724978, - "nauc_mrr_at_1_max": 0.48781134075251725, - "nauc_mrr_at_1_std": 0.35022750945671843, - "nauc_mrr_at_20_diff1": 0.4620254574672781, - "nauc_mrr_at_20_max": 0.4390165307515562, - "nauc_mrr_at_20_std": 0.3426591907598361, - "nauc_mrr_at_3_diff1": 0.492887071202886, - "nauc_mrr_at_3_max": 0.4476533348870664, - "nauc_mrr_at_3_std": 0.3350468692448414, - "nauc_mrr_at_5_diff1": 0.4731605872794598, - "nauc_mrr_at_5_max": 0.44620481412940166, - "nauc_mrr_at_5_std": 0.3401122978842973, - "nauc_ndcg_at_1000_diff1": 0.4387864666079701, - "nauc_ndcg_at_1000_max": 0.43838379944766553, - "nauc_ndcg_at_1000_std": 0.3560604797863434, - "nauc_ndcg_at_100_diff1": 0.41728283501155305, - "nauc_ndcg_at_100_max": 0.4427460292346204, - "nauc_ndcg_at_100_std": 0.372909504646752, - "nauc_ndcg_at_10_diff1": 0.42287678557837377, - "nauc_ndcg_at_10_max": 0.4120351287654675, - "nauc_ndcg_at_10_std": 0.33760808580973, - "nauc_ndcg_at_1_diff1": 0.5290394349724978, - "nauc_ndcg_at_1_max": 0.48781134075251725, - "nauc_ndcg_at_1_std": 0.35022750945671843, - "nauc_ndcg_at_20_diff1": 0.4255068257817746, - "nauc_ndcg_at_20_max": 0.41783754747845137, - "nauc_ndcg_at_20_std": 0.34618769482991557, - "nauc_ndcg_at_3_diff1": 0.48183250013832507, - "nauc_ndcg_at_3_max": 0.4340116328470406, - "nauc_ndcg_at_3_std": 0.32981777269416657, - "nauc_ndcg_at_5_diff1": 0.4486175419457543, - "nauc_ndcg_at_5_max": 0.43177983045623736, - "nauc_ndcg_at_5_std": 0.33853236429784955, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.2680703758418504, - "nauc_precision_at_100_max": 0.48453045854865845, - "nauc_precision_at_100_std": 0.5051430893683858, - "nauc_precision_at_10_diff1": 0.3240598073692732, - "nauc_precision_at_10_max": 0.3473652249574012, - "nauc_precision_at_10_std": 0.3320976688540042, - "nauc_precision_at_1_diff1": 0.5290394349724978, - "nauc_precision_at_1_max": 0.48781134075251725, - "nauc_precision_at_1_std": 0.35022750945671843, - "nauc_precision_at_20_diff1": 0.3388313848679582, - "nauc_precision_at_20_max": 0.3700807956874157, - "nauc_precision_at_20_std": 0.35876997956540574, - "nauc_precision_at_3_diff1": 0.4534575089822941, - "nauc_precision_at_3_max": 0.398646631684571, - "nauc_precision_at_3_std": 0.31624509525368005, - "nauc_precision_at_5_diff1": 0.3859997231322966, - "nauc_precision_at_5_max": 0.3963146931053349, - "nauc_precision_at_5_std": 0.33543675880208584, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.26807037584184995, - "nauc_recall_at_100_max": 0.48453045854865856, - "nauc_recall_at_100_std": 0.5051430893683861, - "nauc_recall_at_10_diff1": 0.3240598073692733, - "nauc_recall_at_10_max": 0.34736522495740113, - "nauc_recall_at_10_std": 0.3320976688540044, - "nauc_recall_at_1_diff1": 0.5290394349724978, - "nauc_recall_at_1_max": 0.48781134075251725, - "nauc_recall_at_1_std": 0.35022750945671843, - "nauc_recall_at_20_diff1": 0.33883138486795833, - "nauc_recall_at_20_max": 0.3700807956874158, - "nauc_recall_at_20_std": 0.35876997956540574, - "nauc_recall_at_3_diff1": 0.4534575089822943, - "nauc_recall_at_3_max": 0.39864663168457115, - "nauc_recall_at_3_std": 0.31624509525368005, - "nauc_recall_at_5_diff1": 0.3859997231322966, - "nauc_recall_at_5_max": 0.39631469310533485, - "nauc_recall_at_5_std": 0.33543675880208573, - "ndcg_at_1": 0.14831, - "ndcg_at_10": 0.21879, - "ndcg_at_100": 0.28592, - "ndcg_at_1000": 0.32918, - "ndcg_at_20": 0.23962, - "ndcg_at_3": 0.18067, - "ndcg_at_5": 0.20006, - "precision_at_1": 0.14831, - "precision_at_10": 0.03101, - "precision_at_100": 0.00651, - "precision_at_1000": 0.001, - "precision_at_20": 0.01972, - "precision_at_3": 0.06816, - "precision_at_5": 0.05034, - "recall_at_1": 0.14831, - "recall_at_10": 0.31011, - "recall_at_100": 0.65056, - "recall_at_1000": 1.0, - "recall_at_20": 0.39438, - "recall_at_3": 0.20449, - "recall_at_5": 0.25169 - }, - { - "cv_recall_at_1": 0.13488, - "cv_recall_at_10": 0.31047, - "cv_recall_at_100": 0.68023, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.40581, - "cv_recall_at_3": 0.20814, - "cv_recall_at_5": 0.25, - "hf_subset": "bg", - "languages": [ - "bul-Cyrl" - ], - "main_score": 0.21438, - "map_at_1": 0.13488, - "map_at_10": 0.18462, - "map_at_100": 0.19748, - "map_at_1000": 0.19905, - "map_at_20": 0.19126, - "map_at_3": 0.16667, - "map_at_5": 0.17643, - "mrr_at_1": 0.13488372093023257, - "mrr_at_10": 0.1846197858988557, - "mrr_at_100": 0.19748470936111007, - "mrr_at_1000": 0.19904904925859657, - "mrr_at_20": 0.1912630101656491, - "mrr_at_3": 0.1666666666666667, - "mrr_at_5": 0.1764341085271318, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2445650317044002, - "nauc_cv_recall_at_100_max": 0.361707515600977, - "nauc_cv_recall_at_100_std": 0.363261081466743, - "nauc_cv_recall_at_10_diff1": 0.3676665275583041, - "nauc_cv_recall_at_10_max": 0.18264771666908935, - "nauc_cv_recall_at_10_std": 0.14698127038758088, - "nauc_cv_recall_at_1_diff1": 0.612561211764825, - "nauc_cv_recall_at_1_max": 0.2835486094917522, - "nauc_cv_recall_at_1_std": 0.18441089696146312, - "nauc_cv_recall_at_20_diff1": 0.27639099909559006, - "nauc_cv_recall_at_20_max": 0.1172433342882384, - "nauc_cv_recall_at_20_std": 0.11048650659259424, - "nauc_cv_recall_at_3_diff1": 0.4615205504007333, - "nauc_cv_recall_at_3_max": 0.19940569139194983, - "nauc_cv_recall_at_3_std": 0.14882662857375184, - "nauc_cv_recall_at_5_diff1": 0.4246482547448188, - "nauc_cv_recall_at_5_max": 0.19074386102961918, - "nauc_cv_recall_at_5_std": 0.16244686401464295, - "nauc_map_at_1000_diff1": 0.5006542328949688, - "nauc_map_at_1000_max": 0.23136238179970012, - "nauc_map_at_1000_std": 0.16686604817646652, - "nauc_map_at_100_diff1": 0.49949888758750094, - "nauc_map_at_100_max": 0.23127076723942885, - "nauc_map_at_100_std": 0.16683027470816547, - "nauc_map_at_10_diff1": 0.5082854436326143, - "nauc_map_at_10_max": 0.23184604408065937, - "nauc_map_at_10_std": 0.1650454145932678, - "nauc_map_at_1_diff1": 0.612561211764825, - "nauc_map_at_1_max": 0.2835486094917522, - "nauc_map_at_1_std": 0.18441089696146312, - "nauc_map_at_20_diff1": 0.49881030031048196, - "nauc_map_at_20_max": 0.22573843880235675, - "nauc_map_at_20_std": 0.16150675906865322, - "nauc_map_at_3_diff1": 0.5328258548948126, - "nauc_map_at_3_max": 0.23753875983065484, - "nauc_map_at_3_std": 0.16377526196925346, - "nauc_map_at_5_diff1": 0.5206969673028455, - "nauc_map_at_5_max": 0.23460276419011938, - "nauc_map_at_5_std": 0.16857161106494534, - "nauc_mrr_at_1000_diff1": 0.5006542740060854, - "nauc_mrr_at_1000_max": 0.23136240008672215, - "nauc_mrr_at_1000_std": 0.16686602588729194, - "nauc_mrr_at_100_diff1": 0.49949888758750094, - "nauc_mrr_at_100_max": 0.23127076723942885, - "nauc_mrr_at_100_std": 0.16683027470816547, - "nauc_mrr_at_10_diff1": 0.5082854436326143, - "nauc_mrr_at_10_max": 0.23184604408065937, - "nauc_mrr_at_10_std": 0.1650454145932678, - "nauc_mrr_at_1_diff1": 0.612561211764825, - "nauc_mrr_at_1_max": 0.2835486094917522, - "nauc_mrr_at_1_std": 0.18441089696146312, - "nauc_mrr_at_20_diff1": 0.49881030031048196, - "nauc_mrr_at_20_max": 0.22573843880235675, - "nauc_mrr_at_20_std": 0.16150675906865322, - "nauc_mrr_at_3_diff1": 0.5328258548948126, - "nauc_mrr_at_3_max": 0.23753875983065484, - "nauc_mrr_at_3_std": 0.16377526196925346, - "nauc_mrr_at_5_diff1": 0.5206969673028455, - "nauc_mrr_at_5_max": 0.23460276419011938, - "nauc_mrr_at_5_std": 0.16857161106494534, - "nauc_ndcg_at_1000_diff1": 0.46721227001489246, - "nauc_ndcg_at_1000_max": 0.2287805352488556, - "nauc_ndcg_at_1000_std": 0.17407191971540306, - "nauc_ndcg_at_100_diff1": 0.439465110267931, - "nauc_ndcg_at_100_max": 0.24227660648463453, - "nauc_ndcg_at_100_std": 0.19339621087015968, - "nauc_ndcg_at_10_diff1": 0.46657913593419037, - "nauc_ndcg_at_10_max": 0.21665052448717026, - "nauc_ndcg_at_10_std": 0.15963159610625768, - "nauc_ndcg_at_1_diff1": 0.612561211764825, - "nauc_ndcg_at_1_max": 0.2835486094917522, - "nauc_ndcg_at_1_std": 0.18441089696146312, - "nauc_ndcg_at_20_diff1": 0.43577425970705724, - "nauc_ndcg_at_20_max": 0.1955854870307893, - "nauc_ndcg_at_20_std": 0.14776732385765973, - "nauc_ndcg_at_3_diff1": 0.5121326130898106, - "nauc_ndcg_at_3_max": 0.2264066500971107, - "nauc_ndcg_at_3_std": 0.1593606756338783, - "nauc_ndcg_at_5_diff1": 0.4924030178160872, - "nauc_ndcg_at_5_max": 0.22153330346602482, - "nauc_ndcg_at_5_std": 0.16658520770421348, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.24456503170440094, - "nauc_precision_at_100_max": 0.3617075156009771, - "nauc_precision_at_100_std": 0.36326108146674346, - "nauc_precision_at_10_diff1": 0.3676665275583044, - "nauc_precision_at_10_max": 0.1826477166690894, - "nauc_precision_at_10_std": 0.14698127038758094, - "nauc_precision_at_1_diff1": 0.612561211764825, - "nauc_precision_at_1_max": 0.2835486094917522, - "nauc_precision_at_1_std": 0.18441089696146312, - "nauc_precision_at_20_diff1": 0.27639099909558995, - "nauc_precision_at_20_max": 0.11724333428823823, - "nauc_precision_at_20_std": 0.11048650659259393, - "nauc_precision_at_3_diff1": 0.46152055040073325, - "nauc_precision_at_3_max": 0.19940569139194977, - "nauc_precision_at_3_std": 0.1488266285737518, - "nauc_precision_at_5_diff1": 0.42464825474481893, - "nauc_precision_at_5_max": 0.19074386102961938, - "nauc_precision_at_5_std": 0.16244686401464292, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2445650317044002, - "nauc_recall_at_100_max": 0.361707515600977, - "nauc_recall_at_100_std": 0.363261081466743, - "nauc_recall_at_10_diff1": 0.3676665275583041, - "nauc_recall_at_10_max": 0.18264771666908935, - "nauc_recall_at_10_std": 0.14698127038758088, - "nauc_recall_at_1_diff1": 0.612561211764825, - "nauc_recall_at_1_max": 0.2835486094917522, - "nauc_recall_at_1_std": 0.18441089696146312, - "nauc_recall_at_20_diff1": 0.27639099909559006, - "nauc_recall_at_20_max": 0.1172433342882384, - "nauc_recall_at_20_std": 0.11048650659259424, - "nauc_recall_at_3_diff1": 0.4615205504007333, - "nauc_recall_at_3_max": 0.19940569139194983, - "nauc_recall_at_3_std": 0.14882662857375184, - "nauc_recall_at_5_diff1": 0.4246482547448188, - "nauc_recall_at_5_max": 0.19074386102961918, - "nauc_recall_at_5_std": 0.16244686401464295, - "ndcg_at_1": 0.13488, - "ndcg_at_10": 0.21438, - "ndcg_at_100": 0.28778, - "ndcg_at_1000": 0.32867, - "ndcg_at_20": 0.23853, - "ndcg_at_3": 0.1773, - "ndcg_at_5": 0.19471, - "precision_at_1": 0.13488, - "precision_at_10": 0.03105, - "precision_at_100": 0.0068, - "precision_at_1000": 0.001, - "precision_at_20": 0.02029, - "precision_at_3": 0.06938, - "precision_at_5": 0.05, - "recall_at_1": 0.13488, - "recall_at_10": 0.31047, - "recall_at_100": 0.68023, - "recall_at_1000": 1.0, - "recall_at_20": 0.40581, - "recall_at_3": 0.20814, - "recall_at_5": 0.25 - }, - { - "cv_recall_at_1": 0.30864, - "cv_recall_at_10": 0.55107, - "cv_recall_at_100": 0.81033, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.63861, - "cv_recall_at_3": 0.41751, - "cv_recall_at_5": 0.46914, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.4196, - "map_at_1": 0.30864, - "map_at_10": 0.37877, - "map_at_100": 0.3888, - "map_at_1000": 0.38968, - "map_at_20": 0.38476, - "map_at_3": 0.35597, - "map_at_5": 0.36758, - "mrr_at_1": 0.30864197530864196, - "mrr_at_10": 0.37877006395524887, - "mrr_at_100": 0.3888033951118189, - "mrr_at_1000": 0.38968141928150263, - "mrr_at_20": 0.38475571429269884, - "mrr_at_3": 0.3559670781893003, - "mrr_at_5": 0.3675832398054617, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.34917861138467304, - "nauc_cv_recall_at_100_max": 0.5809280032918813, - "nauc_cv_recall_at_100_std": 0.5652106785226791, - "nauc_cv_recall_at_10_diff1": 0.5202673828089519, - "nauc_cv_recall_at_10_max": 0.4867137193623612, - "nauc_cv_recall_at_10_std": 0.35862555124365353, - "nauc_cv_recall_at_1_diff1": 0.6978458793185269, - "nauc_cv_recall_at_1_max": 0.4812630250390875, - "nauc_cv_recall_at_1_std": 0.2574662833691913, - "nauc_cv_recall_at_20_diff1": 0.4794186508801584, - "nauc_cv_recall_at_20_max": 0.5114398099151045, - "nauc_cv_recall_at_20_std": 0.4478160589028657, - "nauc_cv_recall_at_3_diff1": 0.5920738440881316, - "nauc_cv_recall_at_3_max": 0.4382773687425336, - "nauc_cv_recall_at_3_std": 0.2790071271396831, - "nauc_cv_recall_at_5_diff1": 0.5511450554138266, - "nauc_cv_recall_at_5_max": 0.453711553179142, - "nauc_cv_recall_at_5_std": 0.3190509058673222, - "nauc_map_at_1000_diff1": 0.636391806736526, - "nauc_map_at_1000_max": 0.4701983153516574, - "nauc_map_at_1000_std": 0.2856963477079405, - "nauc_map_at_100_diff1": 0.6357752143121728, - "nauc_map_at_100_max": 0.47037477407570155, - "nauc_map_at_100_std": 0.28621124775825796, - "nauc_map_at_10_diff1": 0.6380748511735241, - "nauc_map_at_10_max": 0.46869771847749625, - "nauc_map_at_10_std": 0.28119719880382293, - "nauc_map_at_1_diff1": 0.6978458793185269, - "nauc_map_at_1_max": 0.4812630250390875, - "nauc_map_at_1_std": 0.2574662833691913, - "nauc_map_at_20_diff1": 0.6363130940052869, - "nauc_map_at_20_max": 0.4701539151563389, - "nauc_map_at_20_std": 0.2865548455100543, - "nauc_map_at_3_diff1": 0.6502805436566025, - "nauc_map_at_3_max": 0.4604312622649522, - "nauc_map_at_3_std": 0.2670529749642637, - "nauc_map_at_5_diff1": 0.6414768479249554, - "nauc_map_at_5_max": 0.46432565412176646, - "nauc_map_at_5_std": 0.2768796717662677, - "nauc_mrr_at_1000_diff1": 0.636391806736526, - "nauc_mrr_at_1000_max": 0.4701983153516574, - "nauc_mrr_at_1000_std": 0.2856963477079405, - "nauc_mrr_at_100_diff1": 0.6357752143121728, - "nauc_mrr_at_100_max": 0.47037477407570155, - "nauc_mrr_at_100_std": 0.28621124775825796, - "nauc_mrr_at_10_diff1": 0.6380748511735241, - "nauc_mrr_at_10_max": 0.46869771847749625, - "nauc_mrr_at_10_std": 0.28119719880382293, - "nauc_mrr_at_1_diff1": 0.6978458793185269, - "nauc_mrr_at_1_max": 0.4812630250390875, - "nauc_mrr_at_1_std": 0.2574662833691913, - "nauc_mrr_at_20_diff1": 0.6363130940052869, - "nauc_mrr_at_20_max": 0.4701539151563389, - "nauc_mrr_at_20_std": 0.2865548455100543, - "nauc_mrr_at_3_diff1": 0.6502805436566025, - "nauc_mrr_at_3_max": 0.4604312622649522, - "nauc_mrr_at_3_std": 0.2670529749642637, - "nauc_mrr_at_5_diff1": 0.6414768479249554, - "nauc_mrr_at_5_max": 0.46432565412176646, - "nauc_mrr_at_5_std": 0.2768796717662677, - "nauc_ndcg_at_1000_diff1": 0.6168157065098726, - "nauc_ndcg_at_1000_max": 0.4741862963085771, - "nauc_ndcg_at_1000_std": 0.30184717278586776, - "nauc_ndcg_at_100_diff1": 0.5981836215780852, - "nauc_ndcg_at_100_max": 0.48122121391903505, - "nauc_ndcg_at_100_std": 0.3196703358055457, - "nauc_ndcg_at_10_diff1": 0.6103946748770198, - "nauc_ndcg_at_10_max": 0.47184321180216543, - "nauc_ndcg_at_10_std": 0.2985277877145453, - "nauc_ndcg_at_1_diff1": 0.6978458793185269, - "nauc_ndcg_at_1_max": 0.4812630250390875, - "nauc_ndcg_at_1_std": 0.2574662833691913, - "nauc_ndcg_at_20_diff1": 0.6036674298778513, - "nauc_ndcg_at_20_max": 0.4770433048467731, - "nauc_ndcg_at_20_std": 0.3173737824737843, - "nauc_ndcg_at_3_diff1": 0.6351136489526188, - "nauc_ndcg_at_3_max": 0.4545954162776597, - "nauc_ndcg_at_3_std": 0.2701636458988301, - "nauc_ndcg_at_5_diff1": 0.6188271043822667, - "nauc_ndcg_at_5_max": 0.4613224822188785, - "nauc_ndcg_at_5_std": 0.28724444390528403, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3491786113846725, - "nauc_precision_at_100_max": 0.5809280032918817, - "nauc_precision_at_100_std": 0.5652106785226783, - "nauc_precision_at_10_diff1": 0.5202673828089526, - "nauc_precision_at_10_max": 0.486713719362361, - "nauc_precision_at_10_std": 0.3586255512436532, - "nauc_precision_at_1_diff1": 0.6978458793185269, - "nauc_precision_at_1_max": 0.4812630250390875, - "nauc_precision_at_1_std": 0.2574662833691913, - "nauc_precision_at_20_diff1": 0.479418650880159, - "nauc_precision_at_20_max": 0.5114398099151042, - "nauc_precision_at_20_std": 0.44781605890286524, - "nauc_precision_at_3_diff1": 0.5920738440881318, - "nauc_precision_at_3_max": 0.4382773687425334, - "nauc_precision_at_3_std": 0.27900712713968306, - "nauc_precision_at_5_diff1": 0.5511450554138263, - "nauc_precision_at_5_max": 0.45371155317914186, - "nauc_precision_at_5_std": 0.3190509058673224, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.34917861138467304, - "nauc_recall_at_100_max": 0.5809280032918813, - "nauc_recall_at_100_std": 0.5652106785226791, - "nauc_recall_at_10_diff1": 0.5202673828089519, - "nauc_recall_at_10_max": 0.4867137193623612, - "nauc_recall_at_10_std": 0.35862555124365353, - "nauc_recall_at_1_diff1": 0.6978458793185269, - "nauc_recall_at_1_max": 0.4812630250390875, - "nauc_recall_at_1_std": 0.2574662833691913, - "nauc_recall_at_20_diff1": 0.4794186508801584, - "nauc_recall_at_20_max": 0.5114398099151045, - "nauc_recall_at_20_std": 0.4478160589028657, - "nauc_recall_at_3_diff1": 0.5920738440881316, - "nauc_recall_at_3_max": 0.4382773687425336, - "nauc_recall_at_3_std": 0.2790071271396831, - "nauc_recall_at_5_diff1": 0.5511450554138266, - "nauc_recall_at_5_max": 0.453711553179142, - "nauc_recall_at_5_std": 0.3190509058673222, - "ndcg_at_1": 0.30864, - "ndcg_at_10": 0.4196, - "ndcg_at_100": 0.47276, - "ndcg_at_1000": 0.49672, - "ndcg_at_20": 0.44162, - "ndcg_at_3": 0.37175, - "ndcg_at_5": 0.39285, - "precision_at_1": 0.30864, - "precision_at_10": 0.05511, - "precision_at_100": 0.0081, - "precision_at_1000": 0.001, - "precision_at_20": 0.03193, - "precision_at_3": 0.13917, - "precision_at_5": 0.09383, - "recall_at_1": 0.30864, - "recall_at_10": 0.55107, - "recall_at_100": 0.81033, - "recall_at_1000": 1.0, - "recall_at_20": 0.63861, - "recall_at_3": 0.41751, - "recall_at_5": 0.46914 - }, - { - "cv_recall_at_1": 0.10526, - "cv_recall_at_10": 0.25263, - "cv_recall_at_100": 0.57544, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.29825, - "cv_recall_at_3": 0.17018, - "cv_recall_at_5": 0.20175, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.17164, - "map_at_1": 0.10526, - "map_at_10": 0.14659, - "map_at_100": 0.1559, - "map_at_1000": 0.15776, - "map_at_20": 0.14964, - "map_at_3": 0.13275, - "map_at_5": 0.13968, - "mrr_at_1": 0.10526315789473684, - "mrr_at_10": 0.1465866054023949, - "mrr_at_100": 0.15589577807695595, - "mrr_at_1000": 0.15776355208566945, - "mrr_at_20": 0.14964308480582725, - "mrr_at_3": 0.1327485380116959, - "mrr_at_5": 0.13967836257309946, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.18637855746142357, - "nauc_cv_recall_at_100_max": 0.24833841602415352, - "nauc_cv_recall_at_100_std": 0.403742110009017, - "nauc_cv_recall_at_10_diff1": 0.2682111039181232, - "nauc_cv_recall_at_10_max": 0.2003298498720677, - "nauc_cv_recall_at_10_std": 0.35493798616891187, - "nauc_cv_recall_at_1_diff1": 0.5149290393013103, - "nauc_cv_recall_at_1_max": 0.34715247452692855, - "nauc_cv_recall_at_1_std": 0.38980167394468723, - "nauc_cv_recall_at_20_diff1": 0.21777962676809698, - "nauc_cv_recall_at_20_max": 0.19354332580530123, - "nauc_cv_recall_at_20_std": 0.35196481635564003, - "nauc_cv_recall_at_3_diff1": 0.37585939785098305, - "nauc_cv_recall_at_3_max": 0.1919651323127164, - "nauc_cv_recall_at_3_std": 0.34531802943568224, - "nauc_cv_recall_at_5_diff1": 0.3058416721945437, - "nauc_cv_recall_at_5_max": 0.1752755705474715, - "nauc_cv_recall_at_5_std": 0.3182589304616601, - "nauc_map_at_1000_diff1": 0.403902744979074, - "nauc_map_at_1000_max": 0.26715425573574997, - "nauc_map_at_1000_std": 0.36653124352277344, - "nauc_map_at_100_diff1": 0.4030183822085817, - "nauc_map_at_100_max": 0.266996593455788, - "nauc_map_at_100_std": 0.36669005171306873, - "nauc_map_at_10_diff1": 0.41010891346858724, - "nauc_map_at_10_max": 0.26588433202149636, - "nauc_map_at_10_std": 0.3642306694227432, - "nauc_map_at_1_diff1": 0.5149290393013103, - "nauc_map_at_1_max": 0.34715247452692855, - "nauc_map_at_1_std": 0.38980167394468723, - "nauc_map_at_20_diff1": 0.4038257381182442, - "nauc_map_at_20_max": 0.2643456092222034, - "nauc_map_at_20_std": 0.36333876392800185, - "nauc_map_at_3_diff1": 0.44276173515640216, - "nauc_map_at_3_max": 0.27000821438381173, - "nauc_map_at_3_std": 0.3661474539923582, - "nauc_map_at_5_diff1": 0.4193634293904592, - "nauc_map_at_5_max": 0.26298725307683535, - "nauc_map_at_5_std": 0.35770376391128783, - "nauc_mrr_at_1000_diff1": 0.403902744979074, - "nauc_mrr_at_1000_max": 0.26715425573574997, - "nauc_mrr_at_1000_std": 0.36653124352277344, - "nauc_mrr_at_100_diff1": 0.4030183822085817, - "nauc_mrr_at_100_max": 0.266996593455788, - "nauc_mrr_at_100_std": 0.36669005171306873, - "nauc_mrr_at_10_diff1": 0.41010891346858724, - "nauc_mrr_at_10_max": 0.26588433202149636, - "nauc_mrr_at_10_std": 0.3642306694227432, - "nauc_mrr_at_1_diff1": 0.5149290393013103, - "nauc_mrr_at_1_max": 0.34715247452692855, - "nauc_mrr_at_1_std": 0.38980167394468723, - "nauc_mrr_at_20_diff1": 0.4038257381182442, - "nauc_mrr_at_20_max": 0.2643456092222034, - "nauc_mrr_at_20_std": 0.36333876392800185, - "nauc_mrr_at_3_diff1": 0.44276173515640216, - "nauc_mrr_at_3_max": 0.27000821438381173, - "nauc_mrr_at_3_std": 0.3661474539923582, - "nauc_mrr_at_5_diff1": 0.4193634293904592, - "nauc_mrr_at_5_max": 0.26298725307683535, - "nauc_mrr_at_5_std": 0.35770376391128783, - "nauc_ndcg_at_1000_diff1": 0.36876440377313985, - "nauc_ndcg_at_1000_max": 0.25674301571843805, - "nauc_ndcg_at_1000_std": 0.36823676127345956, - "nauc_ndcg_at_100_diff1": 0.3411028514655866, - "nauc_ndcg_at_100_max": 0.2549721813489106, - "nauc_ndcg_at_100_std": 0.373586154530333, - "nauc_ndcg_at_10_diff1": 0.3662572907422116, - "nauc_ndcg_at_10_max": 0.24402177974697598, - "nauc_ndcg_at_10_std": 0.36035887865430855, - "nauc_ndcg_at_1_diff1": 0.5149290393013103, - "nauc_ndcg_at_1_max": 0.34715247452692855, - "nauc_ndcg_at_1_std": 0.38980167394468723, - "nauc_ndcg_at_20_diff1": 0.3465518274198802, - "nauc_ndcg_at_20_max": 0.24022710954348786, - "nauc_ndcg_at_20_std": 0.35861648681751135, - "nauc_ndcg_at_3_diff1": 0.42315037604313344, - "nauc_ndcg_at_3_max": 0.24726612330812991, - "nauc_ndcg_at_3_std": 0.3600189049541695, - "nauc_ndcg_at_5_diff1": 0.38509779167121194, - "nauc_ndcg_at_5_max": 0.23615476637347932, - "nauc_ndcg_at_5_std": 0.34587018634527034, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.18637855746142468, - "nauc_precision_at_100_max": 0.24833841602415418, - "nauc_precision_at_100_std": 0.4037421100090175, - "nauc_precision_at_10_diff1": 0.26821110391812325, - "nauc_precision_at_10_max": 0.20032984987206753, - "nauc_precision_at_10_std": 0.3549379861689121, - "nauc_precision_at_1_diff1": 0.5149290393013103, - "nauc_precision_at_1_max": 0.34715247452692855, - "nauc_precision_at_1_std": 0.38980167394468723, - "nauc_precision_at_20_diff1": 0.2177796267680969, - "nauc_precision_at_20_max": 0.19354332580530145, - "nauc_precision_at_20_std": 0.35196481635564014, - "nauc_precision_at_3_diff1": 0.37585939785098327, - "nauc_precision_at_3_max": 0.19196513231271656, - "nauc_precision_at_3_std": 0.3453180294356824, - "nauc_precision_at_5_diff1": 0.30584167219454383, - "nauc_precision_at_5_max": 0.17527557054747175, - "nauc_precision_at_5_std": 0.31825893046166037, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.18637855746142357, - "nauc_recall_at_100_max": 0.24833841602415352, - "nauc_recall_at_100_std": 0.403742110009017, - "nauc_recall_at_10_diff1": 0.2682111039181232, - "nauc_recall_at_10_max": 0.2003298498720677, - "nauc_recall_at_10_std": 0.35493798616891187, - "nauc_recall_at_1_diff1": 0.5149290393013103, - "nauc_recall_at_1_max": 0.34715247452692855, - "nauc_recall_at_1_std": 0.38980167394468723, - "nauc_recall_at_20_diff1": 0.21777962676809698, - "nauc_recall_at_20_max": 0.19354332580530123, - "nauc_recall_at_20_std": 0.35196481635564003, - "nauc_recall_at_3_diff1": 0.37585939785098305, - "nauc_recall_at_3_max": 0.1919651323127164, - "nauc_recall_at_3_std": 0.34531802943568224, - "nauc_recall_at_5_diff1": 0.3058416721945437, - "nauc_recall_at_5_max": 0.1752755705474715, - "nauc_recall_at_5_std": 0.3182589304616601, - "ndcg_at_1": 0.10526, - "ndcg_at_10": 0.17164, - "ndcg_at_100": 0.23279, - "ndcg_at_1000": 0.28635, - "ndcg_at_20": 0.18302, - "ndcg_at_3": 0.14231, - "ndcg_at_5": 0.15507, - "precision_at_1": 0.10526, - "precision_at_10": 0.02526, - "precision_at_100": 0.00575, - "precision_at_1000": 0.001, - "precision_at_20": 0.01491, - "precision_at_3": 0.05673, - "precision_at_5": 0.04035, - "recall_at_1": 0.10526, - "recall_at_10": 0.25263, - "recall_at_100": 0.57544, - "recall_at_1000": 1.0, - "recall_at_20": 0.29825, - "recall_at_3": 0.17018, - "recall_at_5": 0.20175 - }, - { - "cv_recall_at_1": 0.17277, - "cv_recall_at_10": 0.34325, - "cv_recall_at_100": 0.6762, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.43593, - "cv_recall_at_3": 0.23341, - "cv_recall_at_5": 0.27346, - "hf_subset": "et", - "languages": [ - "est-Latn" - ], - "main_score": 0.24659, - "map_at_1": 0.17277, - "map_at_10": 0.2171, - "map_at_100": 0.22898, - "map_at_1000": 0.23049, - "map_at_20": 0.22329, - "map_at_3": 0.1987, - "map_at_5": 0.2078, - "mrr_at_1": 0.17276887871853547, - "mrr_at_10": 0.2171011768551813, - "mrr_at_100": 0.2289810314309146, - "mrr_at_1000": 0.23048794912664428, - "mrr_at_20": 0.22328900136301424, - "mrr_at_3": 0.1987032799389779, - "mrr_at_5": 0.20779938977879484, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.23707892644673476, - "nauc_cv_recall_at_100_max": 0.5355293934445048, - "nauc_cv_recall_at_100_std": 0.5106409286751152, - "nauc_cv_recall_at_10_diff1": 0.39870287756389644, - "nauc_cv_recall_at_10_max": 0.5015395307021189, - "nauc_cv_recall_at_10_std": 0.39778923076644446, - "nauc_cv_recall_at_1_diff1": 0.6403798630015372, - "nauc_cv_recall_at_1_max": 0.5070352889746105, - "nauc_cv_recall_at_1_std": 0.31104444805117587, - "nauc_cv_recall_at_20_diff1": 0.3353205779178744, - "nauc_cv_recall_at_20_max": 0.48669917318802397, - "nauc_cv_recall_at_20_std": 0.41855181255617585, - "nauc_cv_recall_at_3_diff1": 0.5307956145460796, - "nauc_cv_recall_at_3_max": 0.46477350734939027, - "nauc_cv_recall_at_3_std": 0.32290526314893786, - "nauc_cv_recall_at_5_diff1": 0.47362817758445497, - "nauc_cv_recall_at_5_max": 0.4860244681874373, - "nauc_cv_recall_at_5_std": 0.3426509746871683, - "nauc_map_at_1000_diff1": 0.5518950452222675, - "nauc_map_at_1000_max": 0.49524192662772293, - "nauc_map_at_1000_std": 0.33176304836465365, - "nauc_map_at_100_diff1": 0.5507042720887966, - "nauc_map_at_100_max": 0.4952857109383223, - "nauc_map_at_100_std": 0.3324067505646156, - "nauc_map_at_10_diff1": 0.5583879340502963, - "nauc_map_at_10_max": 0.49526907782206203, - "nauc_map_at_10_std": 0.3300983653566066, - "nauc_map_at_1_diff1": 0.6403798630015372, - "nauc_map_at_1_max": 0.5070352889746105, - "nauc_map_at_1_std": 0.31104444805117587, - "nauc_map_at_20_diff1": 0.5524908076068307, - "nauc_map_at_20_max": 0.49414088668716216, - "nauc_map_at_20_std": 0.3317096629500253, - "nauc_map_at_3_diff1": 0.5866258611416878, - "nauc_map_at_3_max": 0.48824451649660294, - "nauc_map_at_3_std": 0.31506067342848376, - "nauc_map_at_5_diff1": 0.5712264451574552, - "nauc_map_at_5_max": 0.49309396688072177, - "nauc_map_at_5_std": 0.3204667488839716, - "nauc_mrr_at_1000_diff1": 0.5518950452222675, - "nauc_mrr_at_1000_max": 0.49524192662772293, - "nauc_mrr_at_1000_std": 0.33176304836465365, - "nauc_mrr_at_100_diff1": 0.5507042720887966, - "nauc_mrr_at_100_max": 0.4952857109383223, - "nauc_mrr_at_100_std": 0.3324067505646156, - "nauc_mrr_at_10_diff1": 0.5583879340502963, - "nauc_mrr_at_10_max": 0.49526907782206203, - "nauc_mrr_at_10_std": 0.3300983653566066, - "nauc_mrr_at_1_diff1": 0.6403798630015372, - "nauc_mrr_at_1_max": 0.5070352889746105, - "nauc_mrr_at_1_std": 0.31104444805117587, - "nauc_mrr_at_20_diff1": 0.5524908076068307, - "nauc_mrr_at_20_max": 0.49414088668716216, - "nauc_mrr_at_20_std": 0.3317096629500253, - "nauc_mrr_at_3_diff1": 0.5866258611416878, - "nauc_mrr_at_3_max": 0.48824451649660294, - "nauc_mrr_at_3_std": 0.31506067342848376, - "nauc_mrr_at_5_diff1": 0.5712264451574552, - "nauc_mrr_at_5_max": 0.49309396688072177, - "nauc_mrr_at_5_std": 0.3204667488839716, - "nauc_ndcg_at_1000_diff1": 0.5175493259429081, - "nauc_ndcg_at_1000_max": 0.4966318371741495, - "nauc_ndcg_at_1000_std": 0.34574293260303035, - "nauc_ndcg_at_100_diff1": 0.48509068676100753, - "nauc_ndcg_at_100_max": 0.5005582680689583, - "nauc_ndcg_at_100_std": 0.36468181197554345, - "nauc_ndcg_at_10_diff1": 0.5149606519232172, - "nauc_ndcg_at_10_max": 0.49619640915776325, - "nauc_ndcg_at_10_std": 0.34786983476833444, - "nauc_ndcg_at_1_diff1": 0.6403798630015372, - "nauc_ndcg_at_1_max": 0.5070352889746105, - "nauc_ndcg_at_1_std": 0.31104444805117587, - "nauc_ndcg_at_20_diff1": 0.4953903019228144, - "nauc_ndcg_at_20_max": 0.49198524631438506, - "nauc_ndcg_at_20_std": 0.35391185392693036, - "nauc_ndcg_at_3_diff1": 0.5712391453484501, - "nauc_ndcg_at_3_max": 0.4818568955948492, - "nauc_ndcg_at_3_std": 0.3171474374420657, - "nauc_ndcg_at_5_diff1": 0.5443642154064743, - "nauc_ndcg_at_5_max": 0.49077593878515474, - "nauc_ndcg_at_5_std": 0.32642028107092486, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.23707892644673514, - "nauc_precision_at_100_max": 0.5355293934445053, - "nauc_precision_at_100_std": 0.5106409286751148, - "nauc_precision_at_10_diff1": 0.3987028775638965, - "nauc_precision_at_10_max": 0.5015395307021191, - "nauc_precision_at_10_std": 0.39778923076644485, - "nauc_precision_at_1_diff1": 0.6403798630015372, - "nauc_precision_at_1_max": 0.5070352889746105, - "nauc_precision_at_1_std": 0.31104444805117587, - "nauc_precision_at_20_diff1": 0.3353205779178743, - "nauc_precision_at_20_max": 0.48669917318802425, - "nauc_precision_at_20_std": 0.4185518125561754, - "nauc_precision_at_3_diff1": 0.5307956145460793, - "nauc_precision_at_3_max": 0.4647735073493902, - "nauc_precision_at_3_std": 0.3229052631489379, - "nauc_precision_at_5_diff1": 0.4736281775844547, - "nauc_precision_at_5_max": 0.4860244681874368, - "nauc_precision_at_5_std": 0.342650974687168, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.23707892644673476, - "nauc_recall_at_100_max": 0.5355293934445048, - "nauc_recall_at_100_std": 0.5106409286751152, - "nauc_recall_at_10_diff1": 0.39870287756389644, - "nauc_recall_at_10_max": 0.5015395307021189, - "nauc_recall_at_10_std": 0.39778923076644446, - "nauc_recall_at_1_diff1": 0.6403798630015372, - "nauc_recall_at_1_max": 0.5070352889746105, - "nauc_recall_at_1_std": 0.31104444805117587, - "nauc_recall_at_20_diff1": 0.3353205779178744, - "nauc_recall_at_20_max": 0.48669917318802397, - "nauc_recall_at_20_std": 0.41855181255617585, - "nauc_recall_at_3_diff1": 0.5307956145460796, - "nauc_recall_at_3_max": 0.46477350734939027, - "nauc_recall_at_3_std": 0.32290526314893786, - "nauc_recall_at_5_diff1": 0.47362817758445497, - "nauc_recall_at_5_max": 0.4860244681874373, - "nauc_recall_at_5_std": 0.3426509746871683, - "ndcg_at_1": 0.17277, - "ndcg_at_10": 0.24659, - "ndcg_at_100": 0.31337, - "ndcg_at_1000": 0.35447, - "ndcg_at_20": 0.26971, - "ndcg_at_3": 0.20758, - "ndcg_at_5": 0.22403, - "precision_at_1": 0.17277, - "precision_at_10": 0.03432, - "precision_at_100": 0.00676, - "precision_at_1000": 0.001, - "precision_at_20": 0.0218, - "precision_at_3": 0.0778, - "precision_at_5": 0.05469, - "recall_at_1": 0.17277, - "recall_at_10": 0.34325, - "recall_at_100": 0.6762, - "recall_at_1000": 1.0, - "recall_at_20": 0.43593, - "recall_at_3": 0.23341, - "recall_at_5": 0.27346 - }, - { - "cv_recall_at_1": 0.36959, - "cv_recall_at_10": 0.63041, - "cv_recall_at_100": 0.89789, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.72919, - "cv_recall_at_3": 0.50277, - "cv_recall_at_5": 0.56826, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.49355, - "map_at_1": 0.36959, - "map_at_10": 0.45037, - "map_at_100": 0.46169, - "map_at_1000": 0.46219, - "map_at_20": 0.45725, - "map_at_3": 0.42712, - "map_at_5": 0.4421, - "mrr_at_1": 0.36958934517203107, - "mrr_at_10": 0.45036819759350283, - "mrr_at_100": 0.4616881513898778, - "mrr_at_1000": 0.4621883172413923, - "mrr_at_20": 0.45724985417101655, - "mrr_at_3": 0.42711801701812785, - "mrr_at_5": 0.4421013688494263, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4265858736100146, - "nauc_cv_recall_at_100_max": 0.6130105903171712, - "nauc_cv_recall_at_100_std": 0.6479106644247871, - "nauc_cv_recall_at_10_diff1": 0.4763556989410077, - "nauc_cv_recall_at_10_max": 0.39518628189820976, - "nauc_cv_recall_at_10_std": 0.36312964949223575, - "nauc_cv_recall_at_1_diff1": 0.6375084501609795, - "nauc_cv_recall_at_1_max": 0.4004230010736252, - "nauc_cv_recall_at_1_std": 0.2143136715398967, - "nauc_cv_recall_at_20_diff1": 0.4459499331458421, - "nauc_cv_recall_at_20_max": 0.45995435707435284, - "nauc_cv_recall_at_20_std": 0.46913009413801393, - "nauc_cv_recall_at_3_diff1": 0.5344709452451161, - "nauc_cv_recall_at_3_max": 0.3636165152262314, - "nauc_cv_recall_at_3_std": 0.27532160846534987, - "nauc_cv_recall_at_5_diff1": 0.5096097288136904, - "nauc_cv_recall_at_5_max": 0.3825057145648894, - "nauc_cv_recall_at_5_std": 0.3177712488439209, - "nauc_map_at_1000_diff1": 0.5857032193043039, - "nauc_map_at_1000_max": 0.39546456969544974, - "nauc_map_at_1000_std": 0.2547417213718815, - "nauc_map_at_100_diff1": 0.5854429365409518, - "nauc_map_at_100_max": 0.39572854085540565, - "nauc_map_at_100_std": 0.25525466157873605, - "nauc_map_at_10_diff1": 0.5853467092076959, - "nauc_map_at_10_max": 0.3921171573493067, - "nauc_map_at_10_std": 0.25239978453862266, - "nauc_map_at_1_diff1": 0.6375084501609795, - "nauc_map_at_1_max": 0.4004230010736252, - "nauc_map_at_1_std": 0.2143136715398967, - "nauc_map_at_20_diff1": 0.5847518403193711, - "nauc_map_at_20_max": 0.3947421495854351, - "nauc_map_at_20_std": 0.2562116477479193, - "nauc_map_at_3_diff1": 0.5933363215405166, - "nauc_map_at_3_max": 0.3864589857120207, - "nauc_map_at_3_std": 0.24006674573039177, - "nauc_map_at_5_diff1": 0.5883306808360874, - "nauc_map_at_5_max": 0.39028697858525846, - "nauc_map_at_5_std": 0.2484264835473533, - "nauc_mrr_at_1000_diff1": 0.5857036015963674, - "nauc_mrr_at_1000_max": 0.39546283285938855, - "nauc_mrr_at_1000_std": 0.2547396596482983, - "nauc_mrr_at_100_diff1": 0.5854433289477083, - "nauc_mrr_at_100_max": 0.39572683119053226, - "nauc_mrr_at_100_std": 0.2552526336592108, - "nauc_mrr_at_10_diff1": 0.5853467092076959, - "nauc_mrr_at_10_max": 0.3921171573493067, - "nauc_mrr_at_10_std": 0.25239978453862266, - "nauc_mrr_at_1_diff1": 0.6375084501609795, - "nauc_mrr_at_1_max": 0.4004230010736252, - "nauc_mrr_at_1_std": 0.2143136715398967, - "nauc_mrr_at_20_diff1": 0.5847518403193711, - "nauc_mrr_at_20_max": 0.3947421495854351, - "nauc_mrr_at_20_std": 0.2562116477479193, - "nauc_mrr_at_3_diff1": 0.5933363215405166, - "nauc_mrr_at_3_max": 0.3864589857120207, - "nauc_mrr_at_3_std": 0.24006674573039177, - "nauc_mrr_at_5_diff1": 0.5883306808360874, - "nauc_mrr_at_5_max": 0.39028697858525846, - "nauc_mrr_at_5_std": 0.2484264835473533, - "nauc_ndcg_at_1000_diff1": 0.5729010036437998, - "nauc_ndcg_at_1000_max": 0.40063261790125426, - "nauc_ndcg_at_1000_std": 0.2713028785224261, - "nauc_ndcg_at_100_diff1": 0.5663054737618531, - "nauc_ndcg_at_100_max": 0.40972102733006077, - "nauc_ndcg_at_100_std": 0.28757487816844396, - "nauc_ndcg_at_10_diff1": 0.5614605598216259, - "nauc_ndcg_at_10_max": 0.39222010301524096, - "nauc_ndcg_at_10_std": 0.27580314095413405, - "nauc_ndcg_at_1_diff1": 0.6375084501609795, - "nauc_ndcg_at_1_max": 0.4004230010736252, - "nauc_ndcg_at_1_std": 0.2143136715398967, - "nauc_ndcg_at_20_diff1": 0.5591453506371686, - "nauc_ndcg_at_20_max": 0.4036424685141276, - "nauc_ndcg_at_20_std": 0.2916509600022843, - "nauc_ndcg_at_3_diff1": 0.5784145380252298, - "nauc_ndcg_at_3_max": 0.3807448902432253, - "nauc_ndcg_at_3_std": 0.24898665496611744, - "nauc_ndcg_at_5_diff1": 0.5693980880921701, - "nauc_ndcg_at_5_max": 0.388113991989614, - "nauc_ndcg_at_5_std": 0.26469207043279036, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.426585873610016, - "nauc_precision_at_100_max": 0.6130105903171704, - "nauc_precision_at_100_std": 0.6479106644247877, - "nauc_precision_at_10_diff1": 0.4763556989410072, - "nauc_precision_at_10_max": 0.39518628189820937, - "nauc_precision_at_10_std": 0.36312964949223514, - "nauc_precision_at_1_diff1": 0.6375084501609795, - "nauc_precision_at_1_max": 0.4004230010736252, - "nauc_precision_at_1_std": 0.2143136715398967, - "nauc_precision_at_20_diff1": 0.4459499331458413, - "nauc_precision_at_20_max": 0.4599543570743535, - "nauc_precision_at_20_std": 0.4691300941380129, - "nauc_precision_at_3_diff1": 0.5344709452451162, - "nauc_precision_at_3_max": 0.3636165152262313, - "nauc_precision_at_3_std": 0.2753216084653498, - "nauc_precision_at_5_diff1": 0.5096097288136904, - "nauc_precision_at_5_max": 0.3825057145648895, - "nauc_precision_at_5_std": 0.31777124884392066, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4265858736100146, - "nauc_recall_at_100_max": 0.6130105903171712, - "nauc_recall_at_100_std": 0.6479106644247871, - "nauc_recall_at_10_diff1": 0.4763556989410077, - "nauc_recall_at_10_max": 0.39518628189820976, - "nauc_recall_at_10_std": 0.36312964949223575, - "nauc_recall_at_1_diff1": 0.6375084501609795, - "nauc_recall_at_1_max": 0.4004230010736252, - "nauc_recall_at_1_std": 0.2143136715398967, - "nauc_recall_at_20_diff1": 0.4459499331458421, - "nauc_recall_at_20_max": 0.45995435707435284, - "nauc_recall_at_20_std": 0.46913009413801393, - "nauc_recall_at_3_diff1": 0.5344709452451161, - "nauc_recall_at_3_max": 0.3636165152262314, - "nauc_recall_at_3_std": 0.27532160846534987, - "nauc_recall_at_5_diff1": 0.5096097288136904, - "nauc_recall_at_5_max": 0.3825057145648894, - "nauc_recall_at_5_std": 0.3177712488439209, - "ndcg_at_1": 0.36959, - "ndcg_at_10": 0.49355, - "ndcg_at_100": 0.55011, - "ndcg_at_1000": 0.56319, - "ndcg_at_20": 0.51855, - "ndcg_at_3": 0.4465, - "ndcg_at_5": 0.47349, - "precision_at_1": 0.36959, - "precision_at_10": 0.06304, - "precision_at_100": 0.00898, - "precision_at_1000": 0.001, - "precision_at_20": 0.03646, - "precision_at_3": 0.16759, - "precision_at_5": 0.11365, - "recall_at_1": 0.36959, - "recall_at_10": 0.63041, - "recall_at_100": 0.89789, - "recall_at_1000": 1.0, - "recall_at_20": 0.72919, - "recall_at_3": 0.50277, - "recall_at_5": 0.56826 - }, - { - "cv_recall_at_1": 0.087, - "cv_recall_at_10": 0.19656, - "cv_recall_at_100": 0.46079, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.25349, - "cv_recall_at_3": 0.12567, - "cv_recall_at_5": 0.15145, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.13391, - "map_at_1": 0.087, - "map_at_10": 0.1149, - "map_at_100": 0.12368, - "map_at_1000": 0.12578, - "map_at_20": 0.11893, - "map_at_3": 0.10311, - "map_at_5": 0.10924, - "mrr_at_1": 0.08700322234156821, - "mrr_at_10": 0.11489736245375343, - "mrr_at_100": 0.12367750257748027, - "mrr_at_1000": 0.12577974098864964, - "mrr_at_20": 0.11893280521224774, - "mrr_at_3": 0.10311493018259932, - "mrr_at_5": 0.10923737916219116, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.22095843734622075, - "nauc_cv_recall_at_100_max": 0.25029245970753483, - "nauc_cv_recall_at_100_std": 0.3215991312676245, - "nauc_cv_recall_at_10_diff1": 0.3603023327796987, - "nauc_cv_recall_at_10_max": 0.33497378488984614, - "nauc_cv_recall_at_10_std": 0.3351531422726964, - "nauc_cv_recall_at_1_diff1": 0.6847321341274407, - "nauc_cv_recall_at_1_max": 0.47811747616186656, - "nauc_cv_recall_at_1_std": 0.3783668201615523, - "nauc_cv_recall_at_20_diff1": 0.3116302886581901, - "nauc_cv_recall_at_20_max": 0.29917621370786907, - "nauc_cv_recall_at_20_std": 0.31617724986963225, - "nauc_cv_recall_at_3_diff1": 0.5070298216820545, - "nauc_cv_recall_at_3_max": 0.4015417695684175, - "nauc_cv_recall_at_3_std": 0.3687153985936076, - "nauc_cv_recall_at_5_diff1": 0.45106428207293814, - "nauc_cv_recall_at_5_max": 0.3703391700087565, - "nauc_cv_recall_at_5_std": 0.3441693889123974, - "nauc_map_at_1000_diff1": 0.5334132974149114, - "nauc_map_at_1000_max": 0.4064602928580575, - "nauc_map_at_1000_std": 0.35204806392949156, - "nauc_map_at_100_diff1": 0.5322119948770907, - "nauc_map_at_100_max": 0.40531017093092836, - "nauc_map_at_100_std": 0.35167250774892, - "nauc_map_at_10_diff1": 0.5475273770715559, - "nauc_map_at_10_max": 0.41406016843897586, - "nauc_map_at_10_std": 0.35488317945498377, - "nauc_map_at_1_diff1": 0.6847321341274407, - "nauc_map_at_1_max": 0.47811747616186656, - "nauc_map_at_1_std": 0.3783668201615523, - "nauc_map_at_20_diff1": 0.5387576711800679, - "nauc_map_at_20_max": 0.4083036835156604, - "nauc_map_at_20_std": 0.35246396647442946, - "nauc_map_at_3_diff1": 0.5898863721170673, - "nauc_map_at_3_max": 0.43472652790728333, - "nauc_map_at_3_std": 0.36490905144970676, - "nauc_map_at_5_diff1": 0.5694469364779752, - "nauc_map_at_5_max": 0.4230697053023698, - "nauc_map_at_5_std": 0.35765181185393363, - "nauc_mrr_at_1000_diff1": 0.5334127890646034, - "nauc_mrr_at_1000_max": 0.406458644275623, - "nauc_mrr_at_1000_std": 0.3520478053565356, - "nauc_mrr_at_100_diff1": 0.5322114897576498, - "nauc_mrr_at_100_max": 0.4053085319250195, - "nauc_mrr_at_100_std": 0.3516722507420569, - "nauc_mrr_at_10_diff1": 0.5475273770715559, - "nauc_mrr_at_10_max": 0.41406016843897586, - "nauc_mrr_at_10_std": 0.35488317945498377, - "nauc_mrr_at_1_diff1": 0.6847321341274407, - "nauc_mrr_at_1_max": 0.47811747616186656, - "nauc_mrr_at_1_std": 0.3783668201615523, - "nauc_mrr_at_20_diff1": 0.5387576711800679, - "nauc_mrr_at_20_max": 0.4083036835156604, - "nauc_mrr_at_20_std": 0.35246396647442946, - "nauc_mrr_at_3_diff1": 0.5898863721170673, - "nauc_mrr_at_3_max": 0.43472652790728333, - "nauc_mrr_at_3_std": 0.36490905144970676, - "nauc_mrr_at_5_diff1": 0.5694469364779752, - "nauc_mrr_at_5_max": 0.4230697053023698, - "nauc_mrr_at_5_std": 0.35765181185393363, - "nauc_ndcg_at_1000_diff1": 0.476617380656095, - "nauc_ndcg_at_1000_max": 0.3818200888368288, - "nauc_ndcg_at_1000_std": 0.3456048197225887, - "nauc_ndcg_at_100_diff1": 0.43184711588315283, - "nauc_ndcg_at_100_max": 0.3563522077934337, - "nauc_ndcg_at_100_std": 0.34041098252993757, - "nauc_ndcg_at_10_diff1": 0.48809093672959647, - "nauc_ndcg_at_10_max": 0.38870531314884205, - "nauc_ndcg_at_10_std": 0.34846356080211893, - "nauc_ndcg_at_1_diff1": 0.6847321341274407, - "nauc_ndcg_at_1_max": 0.47811747616186656, - "nauc_ndcg_at_1_std": 0.3783668201615523, - "nauc_ndcg_at_20_diff1": 0.4643378062200878, - "nauc_ndcg_at_20_max": 0.37277590099233887, - "nauc_ndcg_at_20_std": 0.34101321085440256, - "nauc_ndcg_at_3_diff1": 0.5658668804015625, - "nauc_ndcg_at_3_max": 0.4249995108514367, - "nauc_ndcg_at_3_std": 0.36565975617467283, - "nauc_ndcg_at_5_diff1": 0.5337700413900782, - "nauc_ndcg_at_5_max": 0.4070769919703173, - "nauc_ndcg_at_5_std": 0.35355366721091624, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.2209584373462211, - "nauc_precision_at_100_max": 0.2502924597075356, - "nauc_precision_at_100_std": 0.3215991312676248, - "nauc_precision_at_10_diff1": 0.3603023327796984, - "nauc_precision_at_10_max": 0.3349737848898459, - "nauc_precision_at_10_std": 0.3351531422726965, - "nauc_precision_at_1_diff1": 0.6847321341274407, - "nauc_precision_at_1_max": 0.47811747616186656, - "nauc_precision_at_1_std": 0.3783668201615523, - "nauc_precision_at_20_diff1": 0.3116302886581901, - "nauc_precision_at_20_max": 0.2991762137078694, - "nauc_precision_at_20_std": 0.31617724986963225, - "nauc_precision_at_3_diff1": 0.5070298216820542, - "nauc_precision_at_3_max": 0.40154176956841753, - "nauc_precision_at_3_std": 0.3687153985936076, - "nauc_precision_at_5_diff1": 0.45106428207293786, - "nauc_precision_at_5_max": 0.3703391700087562, - "nauc_precision_at_5_std": 0.3441693889123974, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.22095843734622075, - "nauc_recall_at_100_max": 0.25029245970753483, - "nauc_recall_at_100_std": 0.3215991312676245, - "nauc_recall_at_10_diff1": 0.3603023327796987, - "nauc_recall_at_10_max": 0.33497378488984614, - "nauc_recall_at_10_std": 0.3351531422726964, - "nauc_recall_at_1_diff1": 0.6847321341274407, - "nauc_recall_at_1_max": 0.47811747616186656, - "nauc_recall_at_1_std": 0.3783668201615523, - "nauc_recall_at_20_diff1": 0.3116302886581901, - "nauc_recall_at_20_max": 0.29917621370786907, - "nauc_recall_at_20_std": 0.31617724986963225, - "nauc_recall_at_3_diff1": 0.5070298216820545, - "nauc_recall_at_3_max": 0.4015417695684175, - "nauc_recall_at_3_std": 0.3687153985936076, - "nauc_recall_at_5_diff1": 0.45106428207293814, - "nauc_recall_at_5_max": 0.3703391700087565, - "nauc_recall_at_5_std": 0.3441693889123974, - "ndcg_at_1": 0.087, - "ndcg_at_10": 0.13391, - "ndcg_at_100": 0.18566, - "ndcg_at_1000": 0.25167, - "ndcg_at_20": 0.14842, - "ndcg_at_3": 0.10887, - "ndcg_at_5": 0.11969, - "precision_at_1": 0.087, - "precision_at_10": 0.01966, - "precision_at_100": 0.00461, - "precision_at_1000": 0.001, - "precision_at_20": 0.01267, - "precision_at_3": 0.04189, - "precision_at_5": 0.03029, - "recall_at_1": 0.087, - "recall_at_10": 0.19656, - "recall_at_100": 0.46079, - "recall_at_1000": 1.0, - "recall_at_20": 0.25349, - "recall_at_3": 0.12567, - "recall_at_5": 0.15145 - }, - { - "cv_recall_at_1": 0.082, - "cv_recall_at_10": 0.26, - "cv_recall_at_100": 0.574, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.32, - "cv_recall_at_3": 0.149, - "cv_recall_at_5": 0.2, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.16092, - "map_at_1": 0.082, - "map_at_10": 0.1303, - "map_at_100": 0.14011, - "map_at_1000": 0.14192, - "map_at_20": 0.13451, - "map_at_3": 0.11017, - "map_at_5": 0.12222, - "mrr_at_1": 0.082, - "mrr_at_10": 0.1302984126984126, - "mrr_at_100": 0.14010643849757057, - "mrr_at_1000": 0.14191966909611997, - "mrr_at_20": 0.13451473022161872, - "mrr_at_3": 0.11016666666666662, - "mrr_at_5": 0.12221666666666663, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.12957698971663834, - "nauc_cv_recall_at_100_max": 0.18513510786430704, - "nauc_cv_recall_at_100_std": 0.2848998619083174, - "nauc_cv_recall_at_10_diff1": 0.27739296348761233, - "nauc_cv_recall_at_10_max": 0.06266225318724539, - "nauc_cv_recall_at_10_std": 0.1555158856349918, - "nauc_cv_recall_at_1_diff1": 0.5317361115655695, - "nauc_cv_recall_at_1_max": 0.12698043937777712, - "nauc_cv_recall_at_1_std": 0.13111310345730237, - "nauc_cv_recall_at_20_diff1": 0.23898239323557738, - "nauc_cv_recall_at_20_max": 0.08588114096114723, - "nauc_cv_recall_at_20_std": 0.1700631488815156, - "nauc_cv_recall_at_3_diff1": 0.35436188037270344, - "nauc_cv_recall_at_3_max": 0.04819250257808713, - "nauc_cv_recall_at_3_std": 0.16807912752262544, - "nauc_cv_recall_at_5_diff1": 0.2936341620923475, - "nauc_cv_recall_at_5_max": 0.0383467872134323, - "nauc_cv_recall_at_5_std": 0.14974652889893447, - "nauc_map_at_1000_diff1": 0.3916961691104259, - "nauc_map_at_1000_max": 0.09453808212623574, - "nauc_map_at_1000_std": 0.15397478560727407, - "nauc_map_at_100_diff1": 0.390296329267604, - "nauc_map_at_100_max": 0.09447881047349718, - "nauc_map_at_100_std": 0.15419789099089493, - "nauc_map_at_10_diff1": 0.3998884462384207, - "nauc_map_at_10_max": 0.08704327678044757, - "nauc_map_at_10_std": 0.14764227122018564, - "nauc_map_at_1_diff1": 0.5317361115655695, - "nauc_map_at_1_max": 0.12698043937777712, - "nauc_map_at_1_std": 0.13111310345730237, - "nauc_map_at_20_diff1": 0.3942263239393362, - "nauc_map_at_20_max": 0.08900280364693693, - "nauc_map_at_20_std": 0.14877466455415778, - "nauc_map_at_3_diff1": 0.4322072311999196, - "nauc_map_at_3_max": 0.09024055839461552, - "nauc_map_at_3_std": 0.15063310629000448, - "nauc_map_at_5_diff1": 0.40613624024338546, - "nauc_map_at_5_max": 0.0835408380843802, - "nauc_map_at_5_std": 0.14696146124460574, - "nauc_mrr_at_1000_diff1": 0.3916961691104259, - "nauc_mrr_at_1000_max": 0.09453808212623574, - "nauc_mrr_at_1000_std": 0.15397478560727407, - "nauc_mrr_at_100_diff1": 0.390296329267604, - "nauc_mrr_at_100_max": 0.09447881047349718, - "nauc_mrr_at_100_std": 0.15419789099089493, - "nauc_mrr_at_10_diff1": 0.3998884462384207, - "nauc_mrr_at_10_max": 0.08704327678044757, - "nauc_mrr_at_10_std": 0.14764227122018564, - "nauc_mrr_at_1_diff1": 0.5317361115655695, - "nauc_mrr_at_1_max": 0.12698043937777712, - "nauc_mrr_at_1_std": 0.13111310345730237, - "nauc_mrr_at_20_diff1": 0.3942263239393362, - "nauc_mrr_at_20_max": 0.08900280364693693, - "nauc_mrr_at_20_std": 0.14877466455415778, - "nauc_mrr_at_3_diff1": 0.4322072311999196, - "nauc_mrr_at_3_max": 0.09024055839461552, - "nauc_mrr_at_3_std": 0.15063310629000448, - "nauc_mrr_at_5_diff1": 0.40613624024338546, - "nauc_mrr_at_5_max": 0.0835408380843802, - "nauc_mrr_at_5_std": 0.14696146124460574, - "nauc_ndcg_at_1000_diff1": 0.3532959414745214, - "nauc_ndcg_at_1000_max": 0.10288254390326823, - "nauc_ndcg_at_1000_std": 0.1700140214238286, - "nauc_ndcg_at_100_diff1": 0.3157125547302426, - "nauc_ndcg_at_100_max": 0.11360475881829207, - "nauc_ndcg_at_100_std": 0.18628635111438582, - "nauc_ndcg_at_10_diff1": 0.35801968786277605, - "nauc_ndcg_at_10_max": 0.07767693928985969, - "nauc_ndcg_at_10_std": 0.1504383247734284, - "nauc_ndcg_at_1_diff1": 0.5317361115655695, - "nauc_ndcg_at_1_max": 0.12698043937777712, - "nauc_ndcg_at_1_std": 0.13111310345730237, - "nauc_ndcg_at_20_diff1": 0.34147465995699866, - "nauc_ndcg_at_20_max": 0.08529823718283426, - "nauc_ndcg_at_20_std": 0.15529041642225935, - "nauc_ndcg_at_3_diff1": 0.4083555318854164, - "nauc_ndcg_at_3_max": 0.07765520581609091, - "nauc_ndcg_at_3_std": 0.1559297532311266, - "nauc_ndcg_at_5_diff1": 0.3691673258075353, - "nauc_ndcg_at_5_max": 0.06867370893589116, - "nauc_ndcg_at_5_std": 0.14830740352720787, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.1295769897166383, - "nauc_precision_at_100_max": 0.1851351078643071, - "nauc_precision_at_100_std": 0.28489986190831773, - "nauc_precision_at_10_diff1": 0.2773929634876122, - "nauc_precision_at_10_max": 0.06266225318724516, - "nauc_precision_at_10_std": 0.15551588563499155, - "nauc_precision_at_1_diff1": 0.5317361115655695, - "nauc_precision_at_1_max": 0.12698043937777712, - "nauc_precision_at_1_std": 0.13111310345730237, - "nauc_precision_at_20_diff1": 0.23898239323557724, - "nauc_precision_at_20_max": 0.08588114096114721, - "nauc_precision_at_20_std": 0.17006314888151536, - "nauc_precision_at_3_diff1": 0.35436188037270333, - "nauc_precision_at_3_max": 0.048192502578087264, - "nauc_precision_at_3_std": 0.16807912752262555, - "nauc_precision_at_5_diff1": 0.29363416209234733, - "nauc_precision_at_5_max": 0.038346787213432404, - "nauc_precision_at_5_std": 0.14974652889893444, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.12957698971663834, - "nauc_recall_at_100_max": 0.18513510786430704, - "nauc_recall_at_100_std": 0.2848998619083174, - "nauc_recall_at_10_diff1": 0.27739296348761233, - "nauc_recall_at_10_max": 0.06266225318724539, - "nauc_recall_at_10_std": 0.1555158856349918, - "nauc_recall_at_1_diff1": 0.5317361115655695, - "nauc_recall_at_1_max": 0.12698043937777712, - "nauc_recall_at_1_std": 0.13111310345730237, - "nauc_recall_at_20_diff1": 0.23898239323557738, - "nauc_recall_at_20_max": 0.08588114096114723, - "nauc_recall_at_20_std": 0.1700631488815156, - "nauc_recall_at_3_diff1": 0.35436188037270344, - "nauc_recall_at_3_max": 0.04819250257808713, - "nauc_recall_at_3_std": 0.16807912752262544, - "nauc_recall_at_5_diff1": 0.2936341620923475, - "nauc_recall_at_5_max": 0.0383467872134323, - "nauc_recall_at_5_std": 0.14974652889893447, - "ndcg_at_1": 0.082, - "ndcg_at_10": 0.16092, - "ndcg_at_100": 0.22146, - "ndcg_at_1000": 0.27457, - "ndcg_at_20": 0.17615, - "ndcg_at_3": 0.12008, - "ndcg_at_5": 0.14143, - "precision_at_1": 0.082, - "precision_at_10": 0.026, - "precision_at_100": 0.00574, - "precision_at_1000": 0.001, - "precision_at_20": 0.016, - "precision_at_3": 0.04967, - "precision_at_5": 0.04, - "recall_at_1": 0.082, - "recall_at_10": 0.26, - "recall_at_100": 0.574, - "recall_at_1000": 1.0, - "recall_at_20": 0.32, - "recall_at_3": 0.149, - "recall_at_5": 0.2 - }, - { - "cv_recall_at_1": 0.30097, - "cv_recall_at_10": 0.56311, - "cv_recall_at_100": 0.83218, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.65049, - "cv_recall_at_3": 0.43689, - "cv_recall_at_5": 0.49376, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.42733, - "map_at_1": 0.30097, - "map_at_10": 0.3845, - "map_at_100": 0.39491, - "map_at_1000": 0.39581, - "map_at_20": 0.39073, - "map_at_3": 0.36223, - "map_at_5": 0.37527, - "mrr_at_1": 0.30097087378640774, - "mrr_at_10": 0.38450399577306643, - "mrr_at_100": 0.39491072235582175, - "mrr_at_1000": 0.3958122134598317, - "mrr_at_20": 0.39072532037400665, - "mrr_at_3": 0.36222838650023137, - "mrr_at_5": 0.3752658344891355, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4625246100334655, - "nauc_cv_recall_at_100_max": 0.4933803291224194, - "nauc_cv_recall_at_100_std": 0.6051973615855294, - "nauc_cv_recall_at_10_diff1": 0.4868176969594762, - "nauc_cv_recall_at_10_max": 0.5056329109842426, - "nauc_cv_recall_at_10_std": 0.4244211679740214, - "nauc_cv_recall_at_1_diff1": 0.6346058198200373, - "nauc_cv_recall_at_1_max": 0.5195356174195791, - "nauc_cv_recall_at_1_std": 0.335413193872272, - "nauc_cv_recall_at_20_diff1": 0.4835757726879039, - "nauc_cv_recall_at_20_max": 0.509186250028205, - "nauc_cv_recall_at_20_std": 0.4963081401316029, - "nauc_cv_recall_at_3_diff1": 0.4778302006907091, - "nauc_cv_recall_at_3_max": 0.4964648160795888, - "nauc_cv_recall_at_3_std": 0.4227748046018749, - "nauc_cv_recall_at_5_diff1": 0.46570883018839004, - "nauc_cv_recall_at_5_max": 0.4985706519151476, - "nauc_cv_recall_at_5_std": 0.40929979622927776, - "nauc_map_at_1000_diff1": 0.5635129914592062, - "nauc_map_at_1000_max": 0.5126271092937562, - "nauc_map_at_1000_std": 0.37999202563630197, - "nauc_map_at_100_diff1": 0.563195913165389, - "nauc_map_at_100_max": 0.5124783062168992, - "nauc_map_at_100_std": 0.38054266238217466, - "nauc_map_at_10_diff1": 0.562849647014662, - "nauc_map_at_10_max": 0.5119277625419975, - "nauc_map_at_10_std": 0.3753089900038274, - "nauc_map_at_1_diff1": 0.6346058198200373, - "nauc_map_at_1_max": 0.5195356174195791, - "nauc_map_at_1_std": 0.335413193872272, - "nauc_map_at_20_diff1": 0.5631407378790679, - "nauc_map_at_20_max": 0.5122987639602273, - "nauc_map_at_20_std": 0.37930908581900735, - "nauc_map_at_3_diff1": 0.5628566451051329, - "nauc_map_at_3_max": 0.5105530097848331, - "nauc_map_at_3_std": 0.3772714487963746, - "nauc_map_at_5_diff1": 0.5603110311518988, - "nauc_map_at_5_max": 0.5112275663372864, - "nauc_map_at_5_std": 0.3735154182994574, - "nauc_mrr_at_1000_diff1": 0.5635112864100491, - "nauc_mrr_at_1000_max": 0.5126242058711608, - "nauc_mrr_at_1000_std": 0.37998936423437424, - "nauc_mrr_at_100_diff1": 0.5631942374373318, - "nauc_mrr_at_100_max": 0.5124753808585685, - "nauc_mrr_at_100_std": 0.38053997223693614, - "nauc_mrr_at_10_diff1": 0.562849647014662, - "nauc_mrr_at_10_max": 0.5119277625419975, - "nauc_mrr_at_10_std": 0.3753089900038274, - "nauc_mrr_at_1_diff1": 0.6346058198200373, - "nauc_mrr_at_1_max": 0.5195356174195791, - "nauc_mrr_at_1_std": 0.335413193872272, - "nauc_mrr_at_20_diff1": 0.5631407378790679, - "nauc_mrr_at_20_max": 0.5122987639602273, - "nauc_mrr_at_20_std": 0.37930908581900735, - "nauc_mrr_at_3_diff1": 0.5628566451051329, - "nauc_mrr_at_3_max": 0.5105530097848331, - "nauc_mrr_at_3_std": 0.3772714487963746, - "nauc_mrr_at_5_diff1": 0.5603110311518988, - "nauc_mrr_at_5_max": 0.5112275663372864, - "nauc_mrr_at_5_std": 0.3735154182994574, - "nauc_ndcg_at_1000_diff1": 0.5521570506994392, - "nauc_ndcg_at_1000_max": 0.5123679273042125, - "nauc_ndcg_at_1000_std": 0.3934861592656324, - "nauc_ndcg_at_100_diff1": 0.5457695675871636, - "nauc_ndcg_at_100_max": 0.5104748090246876, - "nauc_ndcg_at_100_std": 0.40754977276938625, - "nauc_ndcg_at_10_diff1": 0.5431720842235095, - "nauc_ndcg_at_10_max": 0.5100874224849119, - "nauc_ndcg_at_10_std": 0.3872665155700147, - "nauc_ndcg_at_1_diff1": 0.6346058198200373, - "nauc_ndcg_at_1_max": 0.5195356174195791, - "nauc_ndcg_at_1_std": 0.335413193872272, - "nauc_ndcg_at_20_diff1": 0.544019392334048, - "nauc_ndcg_at_20_max": 0.5111400368148176, - "nauc_ndcg_at_20_std": 0.402073504650038, - "nauc_ndcg_at_3_diff1": 0.54071925005883, - "nauc_ndcg_at_3_max": 0.5069523340371278, - "nauc_ndcg_at_3_std": 0.3891988182889605, - "nauc_ndcg_at_5_diff1": 0.5361030708242451, - "nauc_ndcg_at_5_max": 0.5080185032370559, - "nauc_ndcg_at_5_std": 0.3829646873337103, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.46252461003346523, - "nauc_precision_at_100_max": 0.49338032912241975, - "nauc_precision_at_100_std": 0.6051973615855295, - "nauc_precision_at_10_diff1": 0.48681769695947613, - "nauc_precision_at_10_max": 0.5056329109842427, - "nauc_precision_at_10_std": 0.424421167974021, - "nauc_precision_at_1_diff1": 0.6346058198200373, - "nauc_precision_at_1_max": 0.5195356174195791, - "nauc_precision_at_1_std": 0.335413193872272, - "nauc_precision_at_20_diff1": 0.4835757726879036, - "nauc_precision_at_20_max": 0.5091862500282047, - "nauc_precision_at_20_std": 0.4963081401316024, - "nauc_precision_at_3_diff1": 0.47783020069070864, - "nauc_precision_at_3_max": 0.49646481607958853, - "nauc_precision_at_3_std": 0.4227748046018748, - "nauc_precision_at_5_diff1": 0.46570883018839015, - "nauc_precision_at_5_max": 0.4985706519151479, - "nauc_precision_at_5_std": 0.40929979622927737, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4625246100334655, - "nauc_recall_at_100_max": 0.4933803291224194, - "nauc_recall_at_100_std": 0.6051973615855294, - "nauc_recall_at_10_diff1": 0.4868176969594762, - "nauc_recall_at_10_max": 0.5056329109842426, - "nauc_recall_at_10_std": 0.4244211679740214, - "nauc_recall_at_1_diff1": 0.6346058198200373, - "nauc_recall_at_1_max": 0.5195356174195791, - "nauc_recall_at_1_std": 0.335413193872272, - "nauc_recall_at_20_diff1": 0.4835757726879039, - "nauc_recall_at_20_max": 0.509186250028205, - "nauc_recall_at_20_std": 0.4963081401316029, - "nauc_recall_at_3_diff1": 0.4778302006907091, - "nauc_recall_at_3_max": 0.4964648160795888, - "nauc_recall_at_3_std": 0.4227748046018749, - "nauc_recall_at_5_diff1": 0.46570883018839004, - "nauc_recall_at_5_max": 0.4985706519151476, - "nauc_recall_at_5_std": 0.40929979622927776, - "ndcg_at_1": 0.30097, - "ndcg_at_10": 0.42733, - "ndcg_at_100": 0.48236, - "ndcg_at_1000": 0.50435, - "ndcg_at_20": 0.44962, - "ndcg_at_3": 0.38146, - "ndcg_at_5": 0.40492, - "precision_at_1": 0.30097, - "precision_at_10": 0.05631, - "precision_at_100": 0.00832, - "precision_at_1000": 0.001, - "precision_at_20": 0.03252, - "precision_at_3": 0.14563, - "precision_at_5": 0.09875, - "recall_at_1": 0.30097, - "recall_at_10": 0.56311, - "recall_at_100": 0.83218, - "recall_at_1000": 1.0, - "recall_at_20": 0.65049, - "recall_at_3": 0.43689, - "recall_at_5": 0.49376 - }, - { - "cv_recall_at_1": 0.28224, - "cv_recall_at_10": 0.54863, - "cv_recall_at_100": 0.82241, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.62685, - "cv_recall_at_3": 0.40063, - "cv_recall_at_5": 0.45983, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.40522, - "map_at_1": 0.28224, - "map_at_10": 0.36064, - "map_at_100": 0.37109, - "map_at_1000": 0.37193, - "map_at_20": 0.36617, - "map_at_3": 0.3351, - "map_at_5": 0.34863, - "mrr_at_1": 0.2822410147991543, - "mrr_at_10": 0.3606366824390079, - "mrr_at_100": 0.3710940391372773, - "mrr_at_1000": 0.3719294554550222, - "mrr_at_20": 0.3661695515172595, - "mrr_at_3": 0.3350951374207188, - "mrr_at_5": 0.3486257928118391, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.34201335085299767, - "nauc_cv_recall_at_100_max": 0.4623789767253533, - "nauc_cv_recall_at_100_std": 0.6192812082541869, - "nauc_cv_recall_at_10_diff1": 0.41445167336327915, - "nauc_cv_recall_at_10_max": 0.4014545477647777, - "nauc_cv_recall_at_10_std": 0.39772790563947014, - "nauc_cv_recall_at_1_diff1": 0.6337414799404985, - "nauc_cv_recall_at_1_max": 0.5114562833991539, - "nauc_cv_recall_at_1_std": 0.36764760561164883, - "nauc_cv_recall_at_20_diff1": 0.37977308593119635, - "nauc_cv_recall_at_20_max": 0.38489121713355706, - "nauc_cv_recall_at_20_std": 0.40658924970974497, - "nauc_cv_recall_at_3_diff1": 0.4767990083649137, - "nauc_cv_recall_at_3_max": 0.4456114997013859, - "nauc_cv_recall_at_3_std": 0.38757663075347437, - "nauc_cv_recall_at_5_diff1": 0.4368711432697605, - "nauc_cv_recall_at_5_max": 0.41052511580267553, - "nauc_cv_recall_at_5_std": 0.3946283429118848, - "nauc_map_at_1000_diff1": 0.5523103220721189, - "nauc_map_at_1000_max": 0.47666446029624304, - "nauc_map_at_1000_std": 0.38216888015698586, - "nauc_map_at_100_diff1": 0.5518295493827026, - "nauc_map_at_100_max": 0.476549602965261, - "nauc_map_at_100_std": 0.3826181329591628, - "nauc_map_at_10_diff1": 0.5528407331133731, - "nauc_map_at_10_max": 0.4754644451763753, - "nauc_map_at_10_std": 0.3795646303723107, - "nauc_map_at_1_diff1": 0.6337414799404985, - "nauc_map_at_1_max": 0.5114562833991539, - "nauc_map_at_1_std": 0.36764760561164883, - "nauc_map_at_20_diff1": 0.5511520073528766, - "nauc_map_at_20_max": 0.47449026353447393, - "nauc_map_at_20_std": 0.3800723775105995, - "nauc_map_at_3_diff1": 0.5643316970303613, - "nauc_map_at_3_max": 0.4842459956105745, - "nauc_map_at_3_std": 0.3774511286290846, - "nauc_map_at_5_diff1": 0.5553906790668495, - "nauc_map_at_5_max": 0.4764659357568404, - "nauc_map_at_5_std": 0.3794113343603276, - "nauc_mrr_at_1000_diff1": 0.5523103711559296, - "nauc_mrr_at_1000_max": 0.47666445633151716, - "nauc_mrr_at_1000_std": 0.38216891937928554, - "nauc_mrr_at_100_diff1": 0.5518295493827026, - "nauc_mrr_at_100_max": 0.476549602965261, - "nauc_mrr_at_100_std": 0.3826181329591628, - "nauc_mrr_at_10_diff1": 0.5528407331133731, - "nauc_mrr_at_10_max": 0.4754644451763753, - "nauc_mrr_at_10_std": 0.3795646303723107, - "nauc_mrr_at_1_diff1": 0.6337414799404985, - "nauc_mrr_at_1_max": 0.5114562833991539, - "nauc_mrr_at_1_std": 0.36764760561164883, - "nauc_mrr_at_20_diff1": 0.5511520073528766, - "nauc_mrr_at_20_max": 0.47449026353447393, - "nauc_mrr_at_20_std": 0.3800723775105995, - "nauc_mrr_at_3_diff1": 0.5643316970303613, - "nauc_mrr_at_3_max": 0.4842459956105745, - "nauc_mrr_at_3_std": 0.3774511286290846, - "nauc_mrr_at_5_diff1": 0.5553906790668495, - "nauc_mrr_at_5_max": 0.4764659357568404, - "nauc_mrr_at_5_std": 0.3794113343603276, - "nauc_ndcg_at_1000_diff1": 0.5321001402438658, - "nauc_ndcg_at_1000_max": 0.46935887821078137, - "nauc_ndcg_at_1000_std": 0.3906187271528008, - "nauc_ndcg_at_100_diff1": 0.519046454926447, - "nauc_ndcg_at_100_max": 0.4684330980797537, - "nauc_ndcg_at_100_std": 0.40573077981906797, - "nauc_ndcg_at_10_diff1": 0.5192128553827087, - "nauc_ndcg_at_10_max": 0.45768256351389847, - "nauc_ndcg_at_10_std": 0.3839833147055718, - "nauc_ndcg_at_1_diff1": 0.6337414799404985, - "nauc_ndcg_at_1_max": 0.5114562833991539, - "nauc_ndcg_at_1_std": 0.36764760561164883, - "nauc_ndcg_at_20_diff1": 0.5133076590199503, - "nauc_ndcg_at_20_max": 0.4547434106411084, - "nauc_ndcg_at_20_std": 0.38576204358020355, - "nauc_ndcg_at_3_diff1": 0.541276579669424, - "nauc_ndcg_at_3_max": 0.4741475983069668, - "nauc_ndcg_at_3_std": 0.3801582142936978, - "nauc_ndcg_at_5_diff1": 0.5250777835596054, - "nauc_ndcg_at_5_max": 0.459898236719731, - "nauc_ndcg_at_5_std": 0.3833626280258285, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.34201335085299867, - "nauc_precision_at_100_max": 0.4623789767253534, - "nauc_precision_at_100_std": 0.6192812082541859, - "nauc_precision_at_10_diff1": 0.4144516733632791, - "nauc_precision_at_10_max": 0.40145454776477707, - "nauc_precision_at_10_std": 0.3977279056394695, - "nauc_precision_at_1_diff1": 0.6337414799404985, - "nauc_precision_at_1_max": 0.5114562833991539, - "nauc_precision_at_1_std": 0.36764760561164883, - "nauc_precision_at_20_diff1": 0.37977308593119563, - "nauc_precision_at_20_max": 0.3848912171335574, - "nauc_precision_at_20_std": 0.4065892497097443, - "nauc_precision_at_3_diff1": 0.47679900836491307, - "nauc_precision_at_3_max": 0.44561149970138586, - "nauc_precision_at_3_std": 0.38757663075347404, - "nauc_precision_at_5_diff1": 0.43687114326976084, - "nauc_precision_at_5_max": 0.4105251158026755, - "nauc_precision_at_5_std": 0.3946283429118848, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.34201335085299767, - "nauc_recall_at_100_max": 0.4623789767253533, - "nauc_recall_at_100_std": 0.6192812082541869, - "nauc_recall_at_10_diff1": 0.41445167336327915, - "nauc_recall_at_10_max": 0.4014545477647777, - "nauc_recall_at_10_std": 0.39772790563947014, - "nauc_recall_at_1_diff1": 0.6337414799404985, - "nauc_recall_at_1_max": 0.5114562833991539, - "nauc_recall_at_1_std": 0.36764760561164883, - "nauc_recall_at_20_diff1": 0.37977308593119635, - "nauc_recall_at_20_max": 0.38489121713355706, - "nauc_recall_at_20_std": 0.40658924970974497, - "nauc_recall_at_3_diff1": 0.4767990083649137, - "nauc_recall_at_3_max": 0.4456114997013859, - "nauc_recall_at_3_std": 0.38757663075347437, - "nauc_recall_at_5_diff1": 0.4368711432697605, - "nauc_recall_at_5_max": 0.41052511580267553, - "nauc_recall_at_5_std": 0.3946283429118848, - "ndcg_at_1": 0.28224, - "ndcg_at_10": 0.40522, - "ndcg_at_100": 0.46119, - "ndcg_at_1000": 0.48372, - "ndcg_at_20": 0.42513, - "ndcg_at_3": 0.35196, - "ndcg_at_5": 0.37634, - "precision_at_1": 0.28224, - "precision_at_10": 0.05486, - "precision_at_100": 0.00822, - "precision_at_1000": 0.001, - "precision_at_20": 0.03134, - "precision_at_3": 0.13354, - "precision_at_5": 0.09197, - "recall_at_1": 0.28224, - "recall_at_10": 0.54863, - "recall_at_100": 0.82241, - "recall_at_1000": 1.0, - "recall_at_20": 0.62685, - "recall_at_3": 0.40063, - "recall_at_5": 0.45983 - }, - { - "cv_recall_at_1": 0.473, - "cv_recall_at_10": 0.774, - "cv_recall_at_100": 0.971, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.844, - "cv_recall_at_3": 0.617, - "cv_recall_at_5": 0.691, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6154, - "map_at_1": 0.473, - "map_at_10": 0.56563, - "map_at_100": 0.57394, - "map_at_1000": 0.57412, - "map_at_20": 0.57045, - "map_at_3": 0.53783, - "map_at_5": 0.55428, - "mrr_at_1": 0.473, - "mrr_at_10": 0.565631746031746, - "mrr_at_100": 0.5739374783660317, - "mrr_at_1000": 0.5741198530644942, - "mrr_at_20": 0.5704528635923758, - "mrr_at_3": 0.537833333333333, - "mrr_at_5": 0.5542833333333335, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4605267394314036, - "nauc_cv_recall_at_100_max": 0.7279854470523838, - "nauc_cv_recall_at_100_std": 0.7968543739334791, - "nauc_cv_recall_at_10_diff1": 0.6490325040510915, - "nauc_cv_recall_at_10_max": 0.27027474796991563, - "nauc_cv_recall_at_10_std": -0.15112953960537553, - "nauc_cv_recall_at_1_diff1": 0.7308358885481937, - "nauc_cv_recall_at_1_max": 0.35826378265076236, - "nauc_cv_recall_at_1_std": 0.01624337877061942, - "nauc_cv_recall_at_20_diff1": 0.6136489236102389, - "nauc_cv_recall_at_20_max": 0.3500433319098693, - "nauc_cv_recall_at_20_std": -0.07247418925368479, - "nauc_cv_recall_at_3_diff1": 0.6547708840444232, - "nauc_cv_recall_at_3_max": 0.313124884298393, - "nauc_cv_recall_at_3_std": -0.06447773250859423, - "nauc_cv_recall_at_5_diff1": 0.6617254318351731, - "nauc_cv_recall_at_5_max": 0.27884895799272874, - "nauc_cv_recall_at_5_std": -0.11496810614670311, - "nauc_map_at_1000_diff1": 0.6991631273972542, - "nauc_map_at_1000_max": 0.33871405378040526, - "nauc_map_at_1000_std": -0.021408877269458917, - "nauc_map_at_100_diff1": 0.699035903470624, - "nauc_map_at_100_max": 0.33896301834936865, - "nauc_map_at_100_std": -0.020914173868226957, - "nauc_map_at_10_diff1": 0.6991120433673029, - "nauc_map_at_10_max": 0.3347705099742023, - "nauc_map_at_10_std": -0.027856046283506398, - "nauc_map_at_1_diff1": 0.7308358885481937, - "nauc_map_at_1_max": 0.35826378265076236, - "nauc_map_at_1_std": 0.01624337877061942, - "nauc_map_at_20_diff1": 0.698558842795481, - "nauc_map_at_20_max": 0.33807285741066717, - "nauc_map_at_20_std": -0.02376670240251998, - "nauc_map_at_3_diff1": 0.6963433133190773, - "nauc_map_at_3_max": 0.3387796350169068, - "nauc_map_at_3_std": -0.01790309156726825, - "nauc_map_at_5_diff1": 0.6987923943484893, - "nauc_map_at_5_max": 0.3340024651219265, - "nauc_map_at_5_std": -0.02584665881142524, - "nauc_mrr_at_1000_diff1": 0.6991631273972542, - "nauc_mrr_at_1000_max": 0.33871405378040526, - "nauc_mrr_at_1000_std": -0.021408877269458917, - "nauc_mrr_at_100_diff1": 0.699035903470624, - "nauc_mrr_at_100_max": 0.33896301834936865, - "nauc_mrr_at_100_std": -0.020914173868226957, - "nauc_mrr_at_10_diff1": 0.6991120433673029, - "nauc_mrr_at_10_max": 0.3347705099742023, - "nauc_mrr_at_10_std": -0.027856046283506398, - "nauc_mrr_at_1_diff1": 0.7308358885481937, - "nauc_mrr_at_1_max": 0.35826378265076236, - "nauc_mrr_at_1_std": 0.01624337877061942, - "nauc_mrr_at_20_diff1": 0.698558842795481, - "nauc_mrr_at_20_max": 0.33807285741066717, - "nauc_mrr_at_20_std": -0.02376670240251998, - "nauc_mrr_at_3_diff1": 0.6963433133190773, - "nauc_mrr_at_3_max": 0.3387796350169068, - "nauc_mrr_at_3_std": -0.01790309156726825, - "nauc_mrr_at_5_diff1": 0.6987923943484893, - "nauc_mrr_at_5_max": 0.3340024651219265, - "nauc_mrr_at_5_std": -0.02584665881142524, - "nauc_ndcg_at_1000_diff1": 0.6937504114819907, - "nauc_ndcg_at_1000_max": 0.33790132226185615, - "nauc_ndcg_at_1000_std": -0.02382794258355088, - "nauc_ndcg_at_100_diff1": 0.6902411323123153, - "nauc_ndcg_at_100_max": 0.344055739588466, - "nauc_ndcg_at_100_std": -0.011046520811068718, - "nauc_ndcg_at_10_diff1": 0.6898041695743851, - "nauc_ndcg_at_10_max": 0.32335607357229856, - "nauc_ndcg_at_10_std": -0.04982529757937129, - "nauc_ndcg_at_1_diff1": 0.7308358885481937, - "nauc_ndcg_at_1_max": 0.35826378265076236, - "nauc_ndcg_at_1_std": 0.01624337877061942, - "nauc_ndcg_at_20_diff1": 0.6870893166291495, - "nauc_ndcg_at_20_max": 0.33679776689666513, - "nauc_ndcg_at_20_std": -0.03388341318705418, - "nauc_ndcg_at_3_diff1": 0.6861737674452026, - "nauc_ndcg_at_3_max": 0.33255033904787995, - "nauc_ndcg_at_3_std": -0.029180408094922526, - "nauc_ndcg_at_5_diff1": 0.6902989330918204, - "nauc_ndcg_at_5_max": 0.3223485889205435, - "nauc_ndcg_at_5_std": -0.04490685171130491, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.4605267394313974, - "nauc_precision_at_100_max": 0.7279854470523784, - "nauc_precision_at_100_std": 0.7968543739334838, - "nauc_precision_at_10_diff1": 0.6490325040510907, - "nauc_precision_at_10_max": 0.27027474796991413, - "nauc_precision_at_10_std": -0.15112953960537717, - "nauc_precision_at_1_diff1": 0.7308358885481937, - "nauc_precision_at_1_max": 0.35826378265076236, - "nauc_precision_at_1_std": 0.01624337877061942, - "nauc_precision_at_20_diff1": 0.6136489236102396, - "nauc_precision_at_20_max": 0.3500433319098695, - "nauc_precision_at_20_std": -0.07247418925368708, - "nauc_precision_at_3_diff1": 0.6547708840444236, - "nauc_precision_at_3_max": 0.31312488429839336, - "nauc_precision_at_3_std": -0.06447773250859448, - "nauc_precision_at_5_diff1": 0.6617254318351731, - "nauc_precision_at_5_max": 0.2788489579927279, - "nauc_precision_at_5_std": -0.11496810614670348, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4605267394314036, - "nauc_recall_at_100_max": 0.7279854470523838, - "nauc_recall_at_100_std": 0.7968543739334791, - "nauc_recall_at_10_diff1": 0.6490325040510915, - "nauc_recall_at_10_max": 0.27027474796991563, - "nauc_recall_at_10_std": -0.15112953960537553, - "nauc_recall_at_1_diff1": 0.7308358885481937, - "nauc_recall_at_1_max": 0.35826378265076236, - "nauc_recall_at_1_std": 0.01624337877061942, - "nauc_recall_at_20_diff1": 0.6136489236102389, - "nauc_recall_at_20_max": 0.3500433319098693, - "nauc_recall_at_20_std": -0.07247418925368479, - "nauc_recall_at_3_diff1": 0.6547708840444232, - "nauc_recall_at_3_max": 0.313124884298393, - "nauc_recall_at_3_std": -0.06447773250859423, - "nauc_recall_at_5_diff1": 0.6617254318351731, - "nauc_recall_at_5_max": 0.27884895799272874, - "nauc_recall_at_5_std": -0.11496810614670311, - "ndcg_at_1": 0.473, - "ndcg_at_10": 0.6154, - "ndcg_at_100": 0.65714, - "ndcg_at_1000": 0.66107, - "ndcg_at_20": 0.63306, - "ndcg_at_3": 0.55822, - "ndcg_at_5": 0.5883, - "precision_at_1": 0.473, - "precision_at_10": 0.0774, - "precision_at_100": 0.00971, - "precision_at_1000": 0.001, - "precision_at_20": 0.0422, - "precision_at_3": 0.20567, - "precision_at_5": 0.1382, - "recall_at_1": 0.473, - "recall_at_10": 0.774, - "recall_at_100": 0.971, - "recall_at_1000": 1.0, - "recall_at_20": 0.844, - "recall_at_3": 0.617, - "recall_at_5": 0.691 - } - ] - }, - "task_name": "WITT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/model_meta.json b/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/model_meta.json deleted file mode 100644 index e29c0dd4fe..0000000000 --- a/results-mieb/google__siglip-base-patch16-224/7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "google/siglip-base-patch16-224", "revision": "7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/WITT2IRetrieval.json b/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/WITT2IRetrieval.json deleted file mode 100644 index 16b2062a47..0000000000 --- a/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/WITT2IRetrieval.json +++ /dev/null @@ -1,1936 +0,0 @@ -{ - "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", - "evaluation_time": 74.72156262397766, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.3236, - "cv_recall_at_10": 0.60337, - "cv_recall_at_100": 0.89438, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.70674, - "cv_recall_at_3": 0.44719, - "cv_recall_at_5": 0.51573, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.45325, - "map_at_1": 0.3236, - "map_at_10": 0.4064, - "map_at_100": 0.41858, - "map_at_1000": 0.41914, - "map_at_20": 0.41382, - "map_at_3": 0.37865, - "map_at_5": 0.39421, - "mrr_at_1": 0.3235955056179775, - "mrr_at_10": 0.4063991439272336, - "mrr_at_100": 0.4185789121078483, - "mrr_at_1000": 0.4191412489262205, - "mrr_at_20": 0.4138152962342439, - "mrr_at_3": 0.37865168539325816, - "mrr_at_5": 0.39421348314606697, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2642337007621988, - "nauc_cv_recall_at_100_max": 0.4364784844059463, - "nauc_cv_recall_at_100_std": 0.6209214395742897, - "nauc_cv_recall_at_10_diff1": 0.39870764492443644, - "nauc_cv_recall_at_10_max": 0.3090094125185994, - "nauc_cv_recall_at_10_std": 0.28345755308066695, - "nauc_cv_recall_at_1_diff1": 0.5662128824302667, - "nauc_cv_recall_at_1_max": 0.3210718551732005, - "nauc_cv_recall_at_1_std": 0.1913141735544631, - "nauc_cv_recall_at_20_diff1": 0.35494897443461426, - "nauc_cv_recall_at_20_max": 0.31353234713474976, - "nauc_cv_recall_at_20_std": 0.3484756279950574, - "nauc_cv_recall_at_3_diff1": 0.47458937595728956, - "nauc_cv_recall_at_3_max": 0.3071588373467233, - "nauc_cv_recall_at_3_std": 0.2534976907450473, - "nauc_cv_recall_at_5_diff1": 0.4608897957568923, - "nauc_cv_recall_at_5_max": 0.31221669884830827, - "nauc_cv_recall_at_5_std": 0.2613788349789335, - "nauc_map_at_1000_diff1": 0.5135593752434391, - "nauc_map_at_1000_max": 0.31686199255548764, - "nauc_map_at_1000_std": 0.22591048552758194, - "nauc_map_at_100_diff1": 0.5130594196105744, - "nauc_map_at_100_max": 0.31696126710125977, - "nauc_map_at_100_std": 0.2264918348015778, - "nauc_map_at_10_diff1": 0.5131969521348017, - "nauc_map_at_10_max": 0.31610024081986743, - "nauc_map_at_10_std": 0.22310702565265525, - "nauc_map_at_1_diff1": 0.5662128824302667, - "nauc_map_at_1_max": 0.3210718551732005, - "nauc_map_at_1_std": 0.1913141735544631, - "nauc_map_at_20_diff1": 0.5124598116914615, - "nauc_map_at_20_max": 0.3160581390682378, - "nauc_map_at_20_std": 0.22532241691526742, - "nauc_map_at_3_diff1": 0.5217299399887494, - "nauc_map_at_3_max": 0.31528970701609593, - "nauc_map_at_3_std": 0.21957741727895302, - "nauc_map_at_5_diff1": 0.5196735760377273, - "nauc_map_at_5_max": 0.31672138456625565, - "nauc_map_at_5_std": 0.22093720684062693, - "nauc_mrr_at_1000_diff1": 0.5135593752434391, - "nauc_mrr_at_1000_max": 0.31686199255548764, - "nauc_mrr_at_1000_std": 0.22591048552758194, - "nauc_mrr_at_100_diff1": 0.5130594196105744, - "nauc_mrr_at_100_max": 0.31696126710125977, - "nauc_mrr_at_100_std": 0.2264918348015778, - "nauc_mrr_at_10_diff1": 0.5131969521348017, - "nauc_mrr_at_10_max": 0.31610024081986743, - "nauc_mrr_at_10_std": 0.22310702565265525, - "nauc_mrr_at_1_diff1": 0.5662128824302667, - "nauc_mrr_at_1_max": 0.3210718551732005, - "nauc_mrr_at_1_std": 0.1913141735544631, - "nauc_mrr_at_20_diff1": 0.5124598116914615, - "nauc_mrr_at_20_max": 0.3160581390682378, - "nauc_mrr_at_20_std": 0.22532241691526742, - "nauc_mrr_at_3_diff1": 0.5217299399887494, - "nauc_mrr_at_3_max": 0.31528970701609593, - "nauc_mrr_at_3_std": 0.21957741727895302, - "nauc_mrr_at_5_diff1": 0.5196735760377273, - "nauc_mrr_at_5_max": 0.31672138456625565, - "nauc_mrr_at_5_std": 0.22093720684062693, - "nauc_ndcg_at_1000_diff1": 0.4999680546074473, - "nauc_ndcg_at_1000_max": 0.3181423090222327, - "nauc_ndcg_at_1000_std": 0.23901249891109946, - "nauc_ndcg_at_100_diff1": 0.48863649358451144, - "nauc_ndcg_at_100_max": 0.32303604460144, - "nauc_ndcg_at_100_std": 0.2561979096000777, - "nauc_ndcg_at_10_diff1": 0.48800886375175123, - "nauc_ndcg_at_10_max": 0.314397962869589, - "nauc_ndcg_at_10_std": 0.23675231404214014, - "nauc_ndcg_at_1_diff1": 0.5662128824302667, - "nauc_ndcg_at_1_max": 0.3210718551732005, - "nauc_ndcg_at_1_std": 0.1913141735544631, - "nauc_ndcg_at_20_diff1": 0.48361593946256537, - "nauc_ndcg_at_20_max": 0.31508639375442066, - "nauc_ndcg_at_20_std": 0.24696843482835842, - "nauc_ndcg_at_3_diff1": 0.5093997655368436, - "nauc_ndcg_at_3_max": 0.3132071543562908, - "nauc_ndcg_at_3_std": 0.2283621617892596, - "nauc_ndcg_at_5_diff1": 0.5049077426042756, - "nauc_ndcg_at_5_max": 0.3155518597169984, - "nauc_ndcg_at_5_std": 0.23107575301440864, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.26423370076219915, - "nauc_precision_at_100_max": 0.4364784844059457, - "nauc_precision_at_100_std": 0.6209214395742907, - "nauc_precision_at_10_diff1": 0.39870764492443644, - "nauc_precision_at_10_max": 0.30900941251859915, - "nauc_precision_at_10_std": 0.28345755308066645, - "nauc_precision_at_1_diff1": 0.5662128824302667, - "nauc_precision_at_1_max": 0.3210718551732005, - "nauc_precision_at_1_std": 0.1913141735544631, - "nauc_precision_at_20_diff1": 0.3549489744346135, - "nauc_precision_at_20_max": 0.31353234713474953, - "nauc_precision_at_20_std": 0.34847562799505694, - "nauc_precision_at_3_diff1": 0.4745893759572897, - "nauc_precision_at_3_max": 0.3071588373467233, - "nauc_precision_at_3_std": 0.2534976907450468, - "nauc_precision_at_5_diff1": 0.4608897957568923, - "nauc_precision_at_5_max": 0.3122166988483081, - "nauc_precision_at_5_std": 0.2613788349789337, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2642337007621988, - "nauc_recall_at_100_max": 0.4364784844059463, - "nauc_recall_at_100_std": 0.6209214395742897, - "nauc_recall_at_10_diff1": 0.39870764492443644, - "nauc_recall_at_10_max": 0.3090094125185994, - "nauc_recall_at_10_std": 0.28345755308066695, - "nauc_recall_at_1_diff1": 0.5662128824302667, - "nauc_recall_at_1_max": 0.3210718551732005, - "nauc_recall_at_1_std": 0.1913141735544631, - "nauc_recall_at_20_diff1": 0.35494897443461426, - "nauc_recall_at_20_max": 0.31353234713474976, - "nauc_recall_at_20_std": 0.3484756279950574, - "nauc_recall_at_3_diff1": 0.47458937595728956, - "nauc_recall_at_3_max": 0.3071588373467233, - "nauc_recall_at_3_std": 0.2534976907450473, - "nauc_recall_at_5_diff1": 0.4608897957568923, - "nauc_recall_at_5_max": 0.31221669884830827, - "nauc_recall_at_5_std": 0.2613788349789335, - "ndcg_at_1": 0.3236, - "ndcg_at_10": 0.45325, - "ndcg_at_100": 0.5144, - "ndcg_at_1000": 0.52816, - "ndcg_at_20": 0.47968, - "ndcg_at_3": 0.39628, - "ndcg_at_5": 0.42442, - "precision_at_1": 0.3236, - "precision_at_10": 0.06034, - "precision_at_100": 0.00894, - "precision_at_1000": 0.001, - "precision_at_20": 0.03534, - "precision_at_3": 0.14906, - "precision_at_5": 0.10315, - "recall_at_1": 0.3236, - "recall_at_10": 0.60337, - "recall_at_100": 0.89438, - "recall_at_1000": 1.0, - "recall_at_20": 0.70674, - "recall_at_3": 0.44719, - "recall_at_5": 0.51573 - }, - { - "cv_recall_at_1": 0.22326, - "cv_recall_at_10": 0.48837, - "cv_recall_at_100": 0.82442, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.57558, - "cv_recall_at_3": 0.34302, - "cv_recall_at_5": 0.40698, - "hf_subset": "bg", - "languages": [ - "bul-Cyrl" - ], - "main_score": 0.34483, - "map_at_1": 0.22326, - "map_at_10": 0.30029, - "map_at_100": 0.31203, - "map_at_1000": 0.31288, - "map_at_20": 0.30642, - "map_at_3": 0.27539, - "map_at_5": 0.28986, - "mrr_at_1": 0.22325581395348837, - "mrr_at_10": 0.3002911590992984, - "mrr_at_100": 0.3120329661071744, - "mrr_at_1000": 0.3128790149867266, - "mrr_at_20": 0.3064209778179915, - "mrr_at_3": 0.27538759689922493, - "mrr_at_5": 0.28986434108527126, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.35012547421390255, - "nauc_cv_recall_at_100_max": 0.45906854233689964, - "nauc_cv_recall_at_100_std": 0.5046356661272047, - "nauc_cv_recall_at_10_diff1": 0.4351618677282965, - "nauc_cv_recall_at_10_max": 0.30553498132888923, - "nauc_cv_recall_at_10_std": 0.15531432274789383, - "nauc_cv_recall_at_1_diff1": 0.58412712095743, - "nauc_cv_recall_at_1_max": 0.33950763616510776, - "nauc_cv_recall_at_1_std": 0.12532505782796716, - "nauc_cv_recall_at_20_diff1": 0.44304747512907705, - "nauc_cv_recall_at_20_max": 0.3206900894093943, - "nauc_cv_recall_at_20_std": 0.17718003609956795, - "nauc_cv_recall_at_3_diff1": 0.5124164506626259, - "nauc_cv_recall_at_3_max": 0.33443732549821936, - "nauc_cv_recall_at_3_std": 0.12647658311091964, - "nauc_cv_recall_at_5_diff1": 0.48301048809398744, - "nauc_cv_recall_at_5_max": 0.30559815145467906, - "nauc_cv_recall_at_5_std": 0.1304319774333429, - "nauc_map_at_1000_diff1": 0.5349942734009903, - "nauc_map_at_1000_max": 0.3276032432755494, - "nauc_map_at_1000_std": 0.13407559730775565, - "nauc_map_at_100_diff1": 0.5346369207616264, - "nauc_map_at_100_max": 0.3276991776209837, - "nauc_map_at_100_std": 0.13456351565102595, - "nauc_map_at_10_diff1": 0.5338501804482444, - "nauc_map_at_10_max": 0.3242258806026739, - "nauc_map_at_10_std": 0.12892421280373964, - "nauc_map_at_1_diff1": 0.58412712095743, - "nauc_map_at_1_max": 0.33950763616510776, - "nauc_map_at_1_std": 0.12532505782796716, - "nauc_map_at_20_diff1": 0.5349091508590145, - "nauc_map_at_20_max": 0.32576659257476576, - "nauc_map_at_20_std": 0.13063204694092, - "nauc_map_at_3_diff1": 0.5481409986549067, - "nauc_map_at_3_max": 0.33003382575724993, - "nauc_map_at_3_std": 0.12367006552300888, - "nauc_map_at_5_diff1": 0.5405251606031292, - "nauc_map_at_5_max": 0.3237417588873691, - "nauc_map_at_5_std": 0.12534857747574496, - "nauc_mrr_at_1000_diff1": 0.5349940938660724, - "nauc_mrr_at_1000_max": 0.3276030551460708, - "nauc_mrr_at_1000_std": 0.13407525677728963, - "nauc_mrr_at_100_diff1": 0.5346369207616264, - "nauc_mrr_at_100_max": 0.3276991776209837, - "nauc_mrr_at_100_std": 0.13456351565102595, - "nauc_mrr_at_10_diff1": 0.5338501804482444, - "nauc_mrr_at_10_max": 0.3242258806026739, - "nauc_mrr_at_10_std": 0.12892421280373964, - "nauc_mrr_at_1_diff1": 0.58412712095743, - "nauc_mrr_at_1_max": 0.33950763616510776, - "nauc_mrr_at_1_std": 0.12532505782796716, - "nauc_mrr_at_20_diff1": 0.5349091508590145, - "nauc_mrr_at_20_max": 0.32576659257476576, - "nauc_mrr_at_20_std": 0.13063204694092, - "nauc_mrr_at_3_diff1": 0.5481409986549067, - "nauc_mrr_at_3_max": 0.33003382575724993, - "nauc_mrr_at_3_std": 0.12367006552300888, - "nauc_mrr_at_5_diff1": 0.5405251606031292, - "nauc_mrr_at_5_max": 0.3237417588873691, - "nauc_mrr_at_5_std": 0.12534857747574496, - "nauc_ndcg_at_1000_diff1": 0.5211836969162896, - "nauc_ndcg_at_1000_max": 0.3309939382643004, - "nauc_ndcg_at_1000_std": 0.14823376349662354, - "nauc_ndcg_at_100_diff1": 0.5095398772996848, - "nauc_ndcg_at_100_max": 0.33878237754346036, - "nauc_ndcg_at_100_std": 0.17106414103216427, - "nauc_ndcg_at_10_diff1": 0.5091209247099993, - "nauc_ndcg_at_10_max": 0.31955058250055207, - "nauc_ndcg_at_10_std": 0.1351478389842136, - "nauc_ndcg_at_1_diff1": 0.58412712095743, - "nauc_ndcg_at_1_max": 0.33950763616510776, - "nauc_ndcg_at_1_std": 0.12532505782796716, - "nauc_ndcg_at_20_diff1": 0.5126350268619967, - "nauc_ndcg_at_20_max": 0.3241335197093869, - "nauc_ndcg_at_20_std": 0.14063500960677952, - "nauc_ndcg_at_3_diff1": 0.5383501967990882, - "nauc_ndcg_at_3_max": 0.330941174321705, - "nauc_ndcg_at_3_std": 0.12434272552034373, - "nauc_ndcg_at_5_diff1": 0.5250679923659893, - "nauc_ndcg_at_5_max": 0.31902551025939385, - "nauc_ndcg_at_5_std": 0.12668388842368603, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3501254742139034, - "nauc_precision_at_100_max": 0.4590685423369001, - "nauc_precision_at_100_std": 0.5046356661272042, - "nauc_precision_at_10_diff1": 0.4351618677282968, - "nauc_precision_at_10_max": 0.3055349813288893, - "nauc_precision_at_10_std": 0.1553143227478939, - "nauc_precision_at_1_diff1": 0.58412712095743, - "nauc_precision_at_1_max": 0.33950763616510776, - "nauc_precision_at_1_std": 0.12532505782796716, - "nauc_precision_at_20_diff1": 0.4430474751290764, - "nauc_precision_at_20_max": 0.32069008940939414, - "nauc_precision_at_20_std": 0.17718003609956776, - "nauc_precision_at_3_diff1": 0.5124164506626258, - "nauc_precision_at_3_max": 0.3344373254982189, - "nauc_precision_at_3_std": 0.12647658311091964, - "nauc_precision_at_5_diff1": 0.48301048809398744, - "nauc_precision_at_5_max": 0.30559815145467917, - "nauc_precision_at_5_std": 0.130431977433343, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.35012547421390255, - "nauc_recall_at_100_max": 0.45906854233689964, - "nauc_recall_at_100_std": 0.5046356661272047, - "nauc_recall_at_10_diff1": 0.4351618677282965, - "nauc_recall_at_10_max": 0.30553498132888923, - "nauc_recall_at_10_std": 0.15531432274789383, - "nauc_recall_at_1_diff1": 0.58412712095743, - "nauc_recall_at_1_max": 0.33950763616510776, - "nauc_recall_at_1_std": 0.12532505782796716, - "nauc_recall_at_20_diff1": 0.44304747512907705, - "nauc_recall_at_20_max": 0.3206900894093943, - "nauc_recall_at_20_std": 0.17718003609956795, - "nauc_recall_at_3_diff1": 0.5124164506626259, - "nauc_recall_at_3_max": 0.33443732549821936, - "nauc_recall_at_3_std": 0.12647658311091964, - "nauc_recall_at_5_diff1": 0.48301048809398744, - "nauc_recall_at_5_max": 0.30559815145467906, - "nauc_recall_at_5_std": 0.1304319774333429, - "ndcg_at_1": 0.22326, - "ndcg_at_10": 0.34483, - "ndcg_at_100": 0.41164, - "ndcg_at_1000": 0.43399, - "ndcg_at_20": 0.36698, - "ndcg_at_3": 0.29273, - "ndcg_at_5": 0.31895, - "precision_at_1": 0.22326, - "precision_at_10": 0.04884, - "precision_at_100": 0.00824, - "precision_at_1000": 0.001, - "precision_at_20": 0.02878, - "precision_at_3": 0.11434, - "precision_at_5": 0.0814, - "recall_at_1": 0.22326, - "recall_at_10": 0.48837, - "recall_at_100": 0.82442, - "recall_at_1000": 1.0, - "recall_at_20": 0.57558, - "recall_at_3": 0.34302, - "recall_at_5": 0.40698 - }, - { - "cv_recall_at_1": 0.3771, - "cv_recall_at_10": 0.67228, - "cv_recall_at_100": 0.90572, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.75421, - "cv_recall_at_3": 0.52974, - "cv_recall_at_5": 0.60831, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.51928, - "map_at_1": 0.3771, - "map_at_10": 0.47085, - "map_at_100": 0.48038, - "map_at_1000": 0.48084, - "map_at_20": 0.47658, - "map_at_3": 0.44463, - "map_at_5": 0.46253, - "mrr_at_1": 0.3771043771043771, - "mrr_at_10": 0.4708504801097391, - "mrr_at_100": 0.48037636290982116, - "mrr_at_1000": 0.4808395265178713, - "mrr_at_20": 0.47657879978154394, - "mrr_at_3": 0.4446315001870555, - "mrr_at_5": 0.4625327347549565, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4906585203122402, - "nauc_cv_recall_at_100_max": 0.5964527979866758, - "nauc_cv_recall_at_100_std": 0.6167906035908517, - "nauc_cv_recall_at_10_diff1": 0.5669748910407173, - "nauc_cv_recall_at_10_max": 0.5612593077746558, - "nauc_cv_recall_at_10_std": 0.42747846311538445, - "nauc_cv_recall_at_1_diff1": 0.7206551036615471, - "nauc_cv_recall_at_1_max": 0.47420094849426403, - "nauc_cv_recall_at_1_std": 0.20649483028278479, - "nauc_cv_recall_at_20_diff1": 0.5160964579986497, - "nauc_cv_recall_at_20_max": 0.5592683856398762, - "nauc_cv_recall_at_20_std": 0.46856803784647566, - "nauc_cv_recall_at_3_diff1": 0.586423414940436, - "nauc_cv_recall_at_3_max": 0.4652708584095005, - "nauc_cv_recall_at_3_std": 0.2671140327676577, - "nauc_cv_recall_at_5_diff1": 0.5843302145213901, - "nauc_cv_recall_at_5_max": 0.5153515422537157, - "nauc_cv_recall_at_5_std": 0.3564964378343796, - "nauc_map_at_1000_diff1": 0.6639911039547133, - "nauc_map_at_1000_max": 0.4848407002702824, - "nauc_map_at_1000_std": 0.2542221685017404, - "nauc_map_at_100_diff1": 0.6637161909591038, - "nauc_map_at_100_max": 0.48488129868471175, - "nauc_map_at_100_std": 0.25458289111532056, - "nauc_map_at_10_diff1": 0.6644504995596737, - "nauc_map_at_10_max": 0.4864333166313017, - "nauc_map_at_10_std": 0.25634573215141104, - "nauc_map_at_1_diff1": 0.7206551036615471, - "nauc_map_at_1_max": 0.47420094849426403, - "nauc_map_at_1_std": 0.20649483028278479, - "nauc_map_at_20_diff1": 0.6629438663191258, - "nauc_map_at_20_max": 0.4855299221132638, - "nauc_map_at_20_std": 0.2560909055516543, - "nauc_map_at_3_diff1": 0.6636902506020562, - "nauc_map_at_3_max": 0.47129485843850816, - "nauc_map_at_3_std": 0.2326786452017618, - "nauc_map_at_5_diff1": 0.6650049442764228, - "nauc_map_at_5_max": 0.4815139197059957, - "nauc_map_at_5_std": 0.2503501372458987, - "nauc_mrr_at_1000_diff1": 0.6639911039547133, - "nauc_mrr_at_1000_max": 0.4848407002702824, - "nauc_mrr_at_1000_std": 0.2542221685017404, - "nauc_mrr_at_100_diff1": 0.6637161909591038, - "nauc_mrr_at_100_max": 0.48488129868471175, - "nauc_mrr_at_100_std": 0.25458289111532056, - "nauc_mrr_at_10_diff1": 0.6644504995596737, - "nauc_mrr_at_10_max": 0.4864333166313017, - "nauc_mrr_at_10_std": 0.25634573215141104, - "nauc_mrr_at_1_diff1": 0.7206551036615471, - "nauc_mrr_at_1_max": 0.47420094849426403, - "nauc_mrr_at_1_std": 0.20649483028278479, - "nauc_mrr_at_20_diff1": 0.6629438663191258, - "nauc_mrr_at_20_max": 0.4855299221132638, - "nauc_mrr_at_20_std": 0.2560909055516543, - "nauc_mrr_at_3_diff1": 0.6636902506020562, - "nauc_mrr_at_3_max": 0.47129485843850816, - "nauc_mrr_at_3_std": 0.2326786452017618, - "nauc_mrr_at_5_diff1": 0.6650049442764228, - "nauc_mrr_at_5_max": 0.4815139197059957, - "nauc_mrr_at_5_std": 0.2503501372458987, - "nauc_ndcg_at_1000_diff1": 0.6516030124664017, - "nauc_ndcg_at_1000_max": 0.490972539609689, - "nauc_ndcg_at_1000_std": 0.2712931231475558, - "nauc_ndcg_at_100_diff1": 0.6447102766805155, - "nauc_ndcg_at_100_max": 0.4946938959773728, - "nauc_ndcg_at_100_std": 0.2849466684001508, - "nauc_ndcg_at_10_diff1": 0.6435389736849311, - "nauc_ndcg_at_10_max": 0.5009031707887753, - "nauc_ndcg_at_10_std": 0.2905583148670199, - "nauc_ndcg_at_1_diff1": 0.7206551036615471, - "nauc_ndcg_at_1_max": 0.47420094849426403, - "nauc_ndcg_at_1_std": 0.20649483028278479, - "nauc_ndcg_at_20_diff1": 0.6374473907758845, - "nauc_ndcg_at_20_max": 0.49791612968842575, - "nauc_ndcg_at_20_std": 0.2915783578120534, - "nauc_ndcg_at_3_diff1": 0.6443682341577142, - "nauc_ndcg_at_3_max": 0.46982593986634175, - "nauc_ndcg_at_3_std": 0.24130930934625286, - "nauc_ndcg_at_5_diff1": 0.6458853061404316, - "nauc_ndcg_at_5_max": 0.4888301578562302, - "nauc_ndcg_at_5_std": 0.2742810964190069, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.49065852031224216, - "nauc_precision_at_100_max": 0.5964527979866789, - "nauc_precision_at_100_std": 0.6167906035908529, - "nauc_precision_at_10_diff1": 0.5669748910407167, - "nauc_precision_at_10_max": 0.5612593077746549, - "nauc_precision_at_10_std": 0.42747846311538346, - "nauc_precision_at_1_diff1": 0.7206551036615471, - "nauc_precision_at_1_max": 0.47420094849426403, - "nauc_precision_at_1_std": 0.20649483028278479, - "nauc_precision_at_20_diff1": 0.516096457998648, - "nauc_precision_at_20_max": 0.5592683856398757, - "nauc_precision_at_20_std": 0.4685680378464741, - "nauc_precision_at_3_diff1": 0.5864234149404364, - "nauc_precision_at_3_max": 0.4652708584095008, - "nauc_precision_at_3_std": 0.2671140327676577, - "nauc_precision_at_5_diff1": 0.5843302145213897, - "nauc_precision_at_5_max": 0.5153515422537154, - "nauc_precision_at_5_std": 0.3564964378343795, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4906585203122402, - "nauc_recall_at_100_max": 0.5964527979866758, - "nauc_recall_at_100_std": 0.6167906035908517, - "nauc_recall_at_10_diff1": 0.5669748910407173, - "nauc_recall_at_10_max": 0.5612593077746558, - "nauc_recall_at_10_std": 0.42747846311538445, - "nauc_recall_at_1_diff1": 0.7206551036615471, - "nauc_recall_at_1_max": 0.47420094849426403, - "nauc_recall_at_1_std": 0.20649483028278479, - "nauc_recall_at_20_diff1": 0.5160964579986497, - "nauc_recall_at_20_max": 0.5592683856398762, - "nauc_recall_at_20_std": 0.46856803784647566, - "nauc_recall_at_3_diff1": 0.586423414940436, - "nauc_recall_at_3_max": 0.4652708584095005, - "nauc_recall_at_3_std": 0.2671140327676577, - "nauc_recall_at_5_diff1": 0.5843302145213901, - "nauc_recall_at_5_max": 0.5153515422537157, - "nauc_recall_at_5_std": 0.3564964378343796, - "ndcg_at_1": 0.3771, - "ndcg_at_10": 0.51928, - "ndcg_at_100": 0.56799, - "ndcg_at_1000": 0.58004, - "ndcg_at_20": 0.54004, - "ndcg_at_3": 0.4665, - "ndcg_at_5": 0.49881, - "precision_at_1": 0.3771, - "precision_at_10": 0.06723, - "precision_at_100": 0.00906, - "precision_at_1000": 0.001, - "precision_at_20": 0.03771, - "precision_at_3": 0.17658, - "precision_at_5": 0.12166, - "recall_at_1": 0.3771, - "recall_at_10": 0.67228, - "recall_at_100": 0.90572, - "recall_at_1000": 1.0, - "recall_at_20": 0.75421, - "recall_at_3": 0.52974, - "recall_at_5": 0.60831 - }, - { - "cv_recall_at_1": 0.25789, - "cv_recall_at_10": 0.51404, - "cv_recall_at_100": 0.85614, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.63333, - "cv_recall_at_3": 0.35088, - "cv_recall_at_5": 0.41228, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.37035, - "map_at_1": 0.25789, - "map_at_10": 0.32633, - "map_at_100": 0.34022, - "map_at_1000": 0.34102, - "map_at_20": 0.33488, - "map_at_3": 0.29854, - "map_at_5": 0.31257, - "mrr_at_1": 0.2578947368421053, - "mrr_at_10": 0.3263297131718183, - "mrr_at_100": 0.3402207310095121, - "mrr_at_1000": 0.34102258220012444, - "mrr_at_20": 0.3348754483315343, - "mrr_at_3": 0.29853801169590644, - "mrr_at_5": 0.3125730994152046, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5166860368587083, - "nauc_cv_recall_at_100_max": 0.43867555323640267, - "nauc_cv_recall_at_100_std": 0.4181767510258299, - "nauc_cv_recall_at_10_diff1": 0.4930973392561108, - "nauc_cv_recall_at_10_max": 0.3048720618334208, - "nauc_cv_recall_at_10_std": 0.25607080770973034, - "nauc_cv_recall_at_1_diff1": 0.6535331862179902, - "nauc_cv_recall_at_1_max": 0.29091931817892586, - "nauc_cv_recall_at_1_std": 0.09693323050530796, - "nauc_cv_recall_at_20_diff1": 0.4097584319236105, - "nauc_cv_recall_at_20_max": 0.27841599057295296, - "nauc_cv_recall_at_20_std": 0.28995084547443667, - "nauc_cv_recall_at_3_diff1": 0.5381549132947977, - "nauc_cv_recall_at_3_max": 0.25885549132947977, - "nauc_cv_recall_at_3_std": 0.14639768786127155, - "nauc_cv_recall_at_5_diff1": 0.49700295961989843, - "nauc_cv_recall_at_5_max": 0.2933130733554832, - "nauc_cv_recall_at_5_std": 0.20155805277421843, - "nauc_map_at_1000_diff1": 0.5832600994027133, - "nauc_map_at_1000_max": 0.28339679288184694, - "nauc_map_at_1000_std": 0.13635175481091114, - "nauc_map_at_100_diff1": 0.5830692435995272, - "nauc_map_at_100_max": 0.283691963519985, - "nauc_map_at_100_std": 0.13685323283451764, - "nauc_map_at_10_diff1": 0.58543822530251, - "nauc_map_at_10_max": 0.28438693411051763, - "nauc_map_at_10_std": 0.1355926756343963, - "nauc_map_at_1_diff1": 0.6535331862179902, - "nauc_map_at_1_max": 0.29091931817892586, - "nauc_map_at_1_std": 0.09693323050530796, - "nauc_map_at_20_diff1": 0.5811818293948084, - "nauc_map_at_20_max": 0.2824903844060562, - "nauc_map_at_20_std": 0.1366295938070716, - "nauc_map_at_3_diff1": 0.5954714526372237, - "nauc_map_at_3_max": 0.27381059912257893, - "nauc_map_at_3_std": 0.1155176122101709, - "nauc_map_at_5_diff1": 0.5847127401758295, - "nauc_map_at_5_max": 0.28204732400406096, - "nauc_map_at_5_std": 0.12870599169498348, - "nauc_mrr_at_1000_diff1": 0.5832600994027133, - "nauc_mrr_at_1000_max": 0.28339679288184694, - "nauc_mrr_at_1000_std": 0.13635175481091114, - "nauc_mrr_at_100_diff1": 0.5830692435995272, - "nauc_mrr_at_100_max": 0.283691963519985, - "nauc_mrr_at_100_std": 0.13685323283451764, - "nauc_mrr_at_10_diff1": 0.58543822530251, - "nauc_mrr_at_10_max": 0.28438693411051763, - "nauc_mrr_at_10_std": 0.1355926756343963, - "nauc_mrr_at_1_diff1": 0.6535331862179902, - "nauc_mrr_at_1_max": 0.29091931817892586, - "nauc_mrr_at_1_std": 0.09693323050530796, - "nauc_mrr_at_20_diff1": 0.5811818293948084, - "nauc_mrr_at_20_max": 0.2824903844060562, - "nauc_mrr_at_20_std": 0.1366295938070716, - "nauc_mrr_at_3_diff1": 0.5954714526372237, - "nauc_mrr_at_3_max": 0.27381059912257893, - "nauc_mrr_at_3_std": 0.1155176122101709, - "nauc_mrr_at_5_diff1": 0.5847127401758295, - "nauc_mrr_at_5_max": 0.28204732400406096, - "nauc_mrr_at_5_std": 0.12870599169498348, - "nauc_ndcg_at_1000_diff1": 0.5682435897735153, - "nauc_ndcg_at_1000_max": 0.28625538891286073, - "nauc_ndcg_at_1000_std": 0.15434381014343437, - "nauc_ndcg_at_100_diff1": 0.5648418395834174, - "nauc_ndcg_at_100_max": 0.29511407669511164, - "nauc_ndcg_at_100_std": 0.16950589407124725, - "nauc_ndcg_at_10_diff1": 0.5618969295076067, - "nauc_ndcg_at_10_max": 0.28892499642810837, - "nauc_ndcg_at_10_std": 0.16425087953198123, - "nauc_ndcg_at_1_diff1": 0.6535331862179902, - "nauc_ndcg_at_1_max": 0.29091931817892586, - "nauc_ndcg_at_1_std": 0.09693323050530796, - "nauc_ndcg_at_20_diff1": 0.5453651886355529, - "nauc_ndcg_at_20_max": 0.2821736243022714, - "nauc_ndcg_at_20_std": 0.16872468062339616, - "nauc_ndcg_at_3_diff1": 0.5800141979555709, - "nauc_ndcg_at_3_max": 0.26973368041527257, - "nauc_ndcg_at_3_std": 0.12356161545209693, - "nauc_ndcg_at_5_diff1": 0.5615004994756322, - "nauc_ndcg_at_5_max": 0.28444838112540044, - "nauc_ndcg_at_5_std": 0.1472280972157901, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.5166860368587091, - "nauc_precision_at_100_max": 0.43867555323640356, - "nauc_precision_at_100_std": 0.4181767510258304, - "nauc_precision_at_10_diff1": 0.49309733925611043, - "nauc_precision_at_10_max": 0.3048720618334204, - "nauc_precision_at_10_std": 0.2560708077097302, - "nauc_precision_at_1_diff1": 0.6535331862179902, - "nauc_precision_at_1_max": 0.29091931817892586, - "nauc_precision_at_1_std": 0.09693323050530796, - "nauc_precision_at_20_diff1": 0.40975843192361006, - "nauc_precision_at_20_max": 0.2784159905729532, - "nauc_precision_at_20_std": 0.28995084547443656, - "nauc_precision_at_3_diff1": 0.5381549132947977, - "nauc_precision_at_3_max": 0.2588554913294797, - "nauc_precision_at_3_std": 0.1463976878612719, - "nauc_precision_at_5_diff1": 0.4970029596198983, - "nauc_precision_at_5_max": 0.2933130733554832, - "nauc_precision_at_5_std": 0.20155805277421854, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5166860368587083, - "nauc_recall_at_100_max": 0.43867555323640267, - "nauc_recall_at_100_std": 0.4181767510258299, - "nauc_recall_at_10_diff1": 0.4930973392561108, - "nauc_recall_at_10_max": 0.3048720618334208, - "nauc_recall_at_10_std": 0.25607080770973034, - "nauc_recall_at_1_diff1": 0.6535331862179902, - "nauc_recall_at_1_max": 0.29091931817892586, - "nauc_recall_at_1_std": 0.09693323050530796, - "nauc_recall_at_20_diff1": 0.4097584319236105, - "nauc_recall_at_20_max": 0.27841599057295296, - "nauc_recall_at_20_std": 0.28995084547443667, - "nauc_recall_at_3_diff1": 0.5381549132947977, - "nauc_recall_at_3_max": 0.25885549132947977, - "nauc_recall_at_3_std": 0.14639768786127155, - "nauc_recall_at_5_diff1": 0.49700295961989843, - "nauc_recall_at_5_max": 0.2933130733554832, - "nauc_recall_at_5_std": 0.20155805277421843, - "ndcg_at_1": 0.25789, - "ndcg_at_10": 0.37035, - "ndcg_at_100": 0.44141, - "ndcg_at_1000": 0.46041, - "ndcg_at_20": 0.40083, - "ndcg_at_3": 0.31197, - "ndcg_at_5": 0.33726, - "precision_at_1": 0.25789, - "precision_at_10": 0.0514, - "precision_at_100": 0.00856, - "precision_at_1000": 0.001, - "precision_at_20": 0.03167, - "precision_at_3": 0.11696, - "precision_at_5": 0.08246, - "recall_at_1": 0.25789, - "recall_at_10": 0.51404, - "recall_at_100": 0.85614, - "recall_at_1000": 1.0, - "recall_at_20": 0.63333, - "recall_at_3": 0.35088, - "recall_at_5": 0.41228 - }, - { - "cv_recall_at_1": 0.15103, - "cv_recall_at_10": 0.37414, - "cv_recall_at_100": 0.70709, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.46796, - "cv_recall_at_3": 0.24142, - "cv_recall_at_5": 0.28833, - "hf_subset": "et", - "languages": [ - "est-Latn" - ], - "main_score": 0.25003, - "map_at_1": 0.15103, - "map_at_10": 0.21206, - "map_at_100": 0.22451, - "map_at_1000": 0.22589, - "map_at_20": 0.21867, - "map_at_3": 0.1905, - "map_at_5": 0.20114, - "mrr_at_1": 0.15102974828375287, - "mrr_at_10": 0.21206094947513715, - "mrr_at_100": 0.22450731989004902, - "mrr_at_1000": 0.22588700470049086, - "mrr_at_20": 0.21867350862748539, - "mrr_at_3": 0.19050343249427923, - "mrr_at_5": 0.2011441647597255, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2627987447450525, - "nauc_cv_recall_at_100_max": 0.5780401280458465, - "nauc_cv_recall_at_100_std": 0.5261055025722506, - "nauc_cv_recall_at_10_diff1": 0.37104575403153983, - "nauc_cv_recall_at_10_max": 0.3326454524789175, - "nauc_cv_recall_at_10_std": 0.2408937491984172, - "nauc_cv_recall_at_1_diff1": 0.5818077289177603, - "nauc_cv_recall_at_1_max": 0.30800372272543686, - "nauc_cv_recall_at_1_std": 0.111734743577289, - "nauc_cv_recall_at_20_diff1": 0.32652415973790355, - "nauc_cv_recall_at_20_max": 0.4521984724216397, - "nauc_cv_recall_at_20_std": 0.37226557128815174, - "nauc_cv_recall_at_3_diff1": 0.39849646522091037, - "nauc_cv_recall_at_3_max": 0.2875581846946957, - "nauc_cv_recall_at_3_std": 0.15067000226993205, - "nauc_cv_recall_at_5_diff1": 0.3596038540829789, - "nauc_cv_recall_at_5_max": 0.2881051420511456, - "nauc_cv_recall_at_5_std": 0.167710054668009, - "nauc_map_at_1000_diff1": 0.4607486246089169, - "nauc_map_at_1000_max": 0.31054228952252577, - "nauc_map_at_1000_std": 0.15867686744819928, - "nauc_map_at_100_diff1": 0.45998150020862694, - "nauc_map_at_100_max": 0.3115941847216049, - "nauc_map_at_100_std": 0.16014118304459476, - "nauc_map_at_10_diff1": 0.46615837349470063, - "nauc_map_at_10_max": 0.29888142697456227, - "nauc_map_at_10_std": 0.145690639903157, - "nauc_map_at_1_diff1": 0.5818077289177603, - "nauc_map_at_1_max": 0.30800372272543686, - "nauc_map_at_1_std": 0.111734743577289, - "nauc_map_at_20_diff1": 0.46186180237623736, - "nauc_map_at_20_max": 0.3092479663323469, - "nauc_map_at_20_std": 0.15741007065454662, - "nauc_map_at_3_diff1": 0.4800353699094137, - "nauc_map_at_3_max": 0.2915921003153219, - "nauc_map_at_3_std": 0.12788122656072387, - "nauc_map_at_5_diff1": 0.46572435008499474, - "nauc_map_at_5_max": 0.29108929759167185, - "nauc_map_at_5_std": 0.13290094268462646, - "nauc_mrr_at_1000_diff1": 0.4607486246089169, - "nauc_mrr_at_1000_max": 0.31054228952252577, - "nauc_mrr_at_1000_std": 0.15867686744819928, - "nauc_mrr_at_100_diff1": 0.45998150020862694, - "nauc_mrr_at_100_max": 0.3115941847216049, - "nauc_mrr_at_100_std": 0.16014118304459476, - "nauc_mrr_at_10_diff1": 0.46615837349470063, - "nauc_mrr_at_10_max": 0.29888142697456227, - "nauc_mrr_at_10_std": 0.145690639903157, - "nauc_mrr_at_1_diff1": 0.5818077289177603, - "nauc_mrr_at_1_max": 0.30800372272543686, - "nauc_mrr_at_1_std": 0.111734743577289, - "nauc_mrr_at_20_diff1": 0.46186180237623736, - "nauc_mrr_at_20_max": 0.3092479663323469, - "nauc_mrr_at_20_std": 0.15741007065454662, - "nauc_mrr_at_3_diff1": 0.4800353699094137, - "nauc_mrr_at_3_max": 0.2915921003153219, - "nauc_mrr_at_3_std": 0.12788122656072387, - "nauc_mrr_at_5_diff1": 0.46572435008499474, - "nauc_mrr_at_5_max": 0.29108929759167185, - "nauc_mrr_at_5_std": 0.13290094268462646, - "nauc_ndcg_at_1000_diff1": 0.4353269370557416, - "nauc_ndcg_at_1000_max": 0.33047611867355287, - "nauc_ndcg_at_1000_std": 0.19048707620953562, - "nauc_ndcg_at_100_diff1": 0.41560305720295077, - "nauc_ndcg_at_100_max": 0.35861665089692857, - "nauc_ndcg_at_100_std": 0.22866740291545476, - "nauc_ndcg_at_10_diff1": 0.43670745989749377, - "nauc_ndcg_at_10_max": 0.30736360243562977, - "nauc_ndcg_at_10_std": 0.17170928125657542, - "nauc_ndcg_at_1_diff1": 0.5818077289177603, - "nauc_ndcg_at_1_max": 0.30800372272543686, - "nauc_ndcg_at_1_std": 0.111734743577289, - "nauc_ndcg_at_20_diff1": 0.4229548229039391, - "nauc_ndcg_at_20_max": 0.3431403535337248, - "nauc_ndcg_at_20_std": 0.21132734084355814, - "nauc_ndcg_at_3_diff1": 0.4564002917837622, - "nauc_ndcg_at_3_max": 0.2901989981063535, - "nauc_ndcg_at_3_std": 0.13424490964159908, - "nauc_ndcg_at_5_diff1": 0.4342887056984211, - "nauc_ndcg_at_5_max": 0.28989155178116033, - "nauc_ndcg_at_5_std": 0.14279723581592305, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.26279874474505255, - "nauc_precision_at_100_max": 0.5780401280458474, - "nauc_precision_at_100_std": 0.5261055025722517, - "nauc_precision_at_10_diff1": 0.3710457540315399, - "nauc_precision_at_10_max": 0.3326454524789174, - "nauc_precision_at_10_std": 0.24089374919841736, - "nauc_precision_at_1_diff1": 0.5818077289177603, - "nauc_precision_at_1_max": 0.30800372272543686, - "nauc_precision_at_1_std": 0.111734743577289, - "nauc_precision_at_20_diff1": 0.32652415973790305, - "nauc_precision_at_20_max": 0.4521984724216396, - "nauc_precision_at_20_std": 0.3722655712881512, - "nauc_precision_at_3_diff1": 0.3984964652209105, - "nauc_precision_at_3_max": 0.28755818469469585, - "nauc_precision_at_3_std": 0.150670002269932, - "nauc_precision_at_5_diff1": 0.3596038540829787, - "nauc_precision_at_5_max": 0.28810514205114574, - "nauc_precision_at_5_std": 0.16771005466800906, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2627987447450525, - "nauc_recall_at_100_max": 0.5780401280458465, - "nauc_recall_at_100_std": 0.5261055025722506, - "nauc_recall_at_10_diff1": 0.37104575403153983, - "nauc_recall_at_10_max": 0.3326454524789175, - "nauc_recall_at_10_std": 0.2408937491984172, - "nauc_recall_at_1_diff1": 0.5818077289177603, - "nauc_recall_at_1_max": 0.30800372272543686, - "nauc_recall_at_1_std": 0.111734743577289, - "nauc_recall_at_20_diff1": 0.32652415973790355, - "nauc_recall_at_20_max": 0.4521984724216397, - "nauc_recall_at_20_std": 0.37226557128815174, - "nauc_recall_at_3_diff1": 0.39849646522091037, - "nauc_recall_at_3_max": 0.2875581846946957, - "nauc_recall_at_3_std": 0.15067000226993205, - "nauc_recall_at_5_diff1": 0.3596038540829789, - "nauc_recall_at_5_max": 0.2881051420511456, - "nauc_recall_at_5_std": 0.167710054668009, - "ndcg_at_1": 0.15103, - "ndcg_at_10": 0.25003, - "ndcg_at_100": 0.31768, - "ndcg_at_1000": 0.35499, - "ndcg_at_20": 0.27387, - "ndcg_at_3": 0.20356, - "ndcg_at_5": 0.22282, - "precision_at_1": 0.15103, - "precision_at_10": 0.03741, - "precision_at_100": 0.00707, - "precision_at_1000": 0.001, - "precision_at_20": 0.0234, - "precision_at_3": 0.08047, - "precision_at_5": 0.05767, - "recall_at_1": 0.15103, - "recall_at_10": 0.37414, - "recall_at_100": 0.70709, - "recall_at_1000": 1.0, - "recall_at_20": 0.46796, - "recall_at_3": 0.24142, - "recall_at_5": 0.28833 - }, - { - "cv_recall_at_1": 0.30966, - "cv_recall_at_10": 0.60155, - "cv_recall_at_100": 0.84573, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.66371, - "cv_recall_at_3": 0.45616, - "cv_recall_at_5": 0.52164, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.4485, - "map_at_1": 0.30966, - "map_at_10": 0.4004, - "map_at_100": 0.40957, - "map_at_1000": 0.4103, - "map_at_20": 0.40488, - "map_at_3": 0.37477, - "map_at_5": 0.38981, - "mrr_at_1": 0.30965593784683687, - "mrr_at_10": 0.40039902753554235, - "mrr_at_100": 0.40957344046741884, - "mrr_at_1000": 0.4102965595323894, - "mrr_at_20": 0.4048839867430234, - "mrr_at_3": 0.37476877543470216, - "mrr_at_5": 0.389807621161672, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.413844347837448, - "nauc_cv_recall_at_100_max": 0.6591032685370454, - "nauc_cv_recall_at_100_std": 0.6882492080286065, - "nauc_cv_recall_at_10_diff1": 0.5166753772605301, - "nauc_cv_recall_at_10_max": 0.5890906125901273, - "nauc_cv_recall_at_10_std": 0.5228958235520792, - "nauc_cv_recall_at_1_diff1": 0.7003540691175822, - "nauc_cv_recall_at_1_max": 0.5562337135287849, - "nauc_cv_recall_at_1_std": 0.3222697658148194, - "nauc_cv_recall_at_20_diff1": 0.49453686872493546, - "nauc_cv_recall_at_20_max": 0.570490024802668, - "nauc_cv_recall_at_20_std": 0.5310876413062724, - "nauc_cv_recall_at_3_diff1": 0.5822939098091017, - "nauc_cv_recall_at_3_max": 0.5587282972262209, - "nauc_cv_recall_at_3_std": 0.42902959999002005, - "nauc_cv_recall_at_5_diff1": 0.551476530777103, - "nauc_cv_recall_at_5_max": 0.5698560042206824, - "nauc_cv_recall_at_5_std": 0.489924186816204, - "nauc_map_at_1000_diff1": 0.6385712536554932, - "nauc_map_at_1000_max": 0.5666453665562166, - "nauc_map_at_1000_std": 0.39054233624224277, - "nauc_map_at_100_diff1": 0.6380972125992274, - "nauc_map_at_100_max": 0.5668257371445421, - "nauc_map_at_100_std": 0.39107637572379544, - "nauc_map_at_10_diff1": 0.6382357516010204, - "nauc_map_at_10_max": 0.566426041023328, - "nauc_map_at_10_std": 0.3907557479715132, - "nauc_map_at_1_diff1": 0.7003540691175822, - "nauc_map_at_1_max": 0.5562337135287849, - "nauc_map_at_1_std": 0.3222697658148194, - "nauc_map_at_20_diff1": 0.6380225873024882, - "nauc_map_at_20_max": 0.5653351730614248, - "nauc_map_at_20_std": 0.390364721772839, - "nauc_map_at_3_diff1": 0.6481258672212916, - "nauc_map_at_3_max": 0.5622091310080056, - "nauc_map_at_3_std": 0.3737845333707094, - "nauc_map_at_5_diff1": 0.6421070955397341, - "nauc_map_at_5_max": 0.5638541197609342, - "nauc_map_at_5_std": 0.38713354286032103, - "nauc_mrr_at_1000_diff1": 0.6385712599372024, - "nauc_mrr_at_1000_max": 0.5666453342564297, - "nauc_mrr_at_1000_std": 0.3905423005585837, - "nauc_mrr_at_100_diff1": 0.6380972125992274, - "nauc_mrr_at_100_max": 0.5668257371445421, - "nauc_mrr_at_100_std": 0.39107637572379544, - "nauc_mrr_at_10_diff1": 0.6382357516010204, - "nauc_mrr_at_10_max": 0.566426041023328, - "nauc_mrr_at_10_std": 0.3907557479715132, - "nauc_mrr_at_1_diff1": 0.7003540691175822, - "nauc_mrr_at_1_max": 0.5562337135287849, - "nauc_mrr_at_1_std": 0.3222697658148194, - "nauc_mrr_at_20_diff1": 0.6380225873024882, - "nauc_mrr_at_20_max": 0.5653351730614248, - "nauc_mrr_at_20_std": 0.390364721772839, - "nauc_mrr_at_3_diff1": 0.6481258672212916, - "nauc_mrr_at_3_max": 0.5622091310080056, - "nauc_mrr_at_3_std": 0.3737845333707094, - "nauc_mrr_at_5_diff1": 0.6421070955397341, - "nauc_mrr_at_5_max": 0.5638541197609342, - "nauc_mrr_at_5_std": 0.38713354286032103, - "nauc_ndcg_at_1000_diff1": 0.6221171524663395, - "nauc_ndcg_at_1000_max": 0.5704914017884861, - "nauc_ndcg_at_1000_std": 0.40945199390172643, - "nauc_ndcg_at_100_diff1": 0.6093090345917943, - "nauc_ndcg_at_100_max": 0.5758410316092789, - "nauc_ndcg_at_100_std": 0.4260576549155124, - "nauc_ndcg_at_10_diff1": 0.6104476699376449, - "nauc_ndcg_at_10_max": 0.5712565523303276, - "nauc_ndcg_at_10_std": 0.4209712430612556, - "nauc_ndcg_at_1_diff1": 0.7003540691175822, - "nauc_ndcg_at_1_max": 0.5562337135287849, - "nauc_ndcg_at_1_std": 0.3222697658148194, - "nauc_ndcg_at_20_diff1": 0.608414917728419, - "nauc_ndcg_at_20_max": 0.5669549851368711, - "nauc_ndcg_at_20_std": 0.42011850937905115, - "nauc_ndcg_at_3_diff1": 0.6311285156481615, - "nauc_ndcg_at_3_max": 0.561514062757331, - "nauc_ndcg_at_3_std": 0.3882275785314245, - "nauc_ndcg_at_5_diff1": 0.6196780560521288, - "nauc_ndcg_at_5_max": 0.5652964042526559, - "nauc_ndcg_at_5_std": 0.4124392955924702, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.4138443478374479, - "nauc_precision_at_100_max": 0.6591032685370464, - "nauc_precision_at_100_std": 0.6882492080286061, - "nauc_precision_at_10_diff1": 0.5166753772605301, - "nauc_precision_at_10_max": 0.5890906125901274, - "nauc_precision_at_10_std": 0.5228958235520789, - "nauc_precision_at_1_diff1": 0.7003540691175822, - "nauc_precision_at_1_max": 0.5562337135287849, - "nauc_precision_at_1_std": 0.3222697658148194, - "nauc_precision_at_20_diff1": 0.49453686872493513, - "nauc_precision_at_20_max": 0.5704900248026669, - "nauc_precision_at_20_std": 0.5310876413062724, - "nauc_precision_at_3_diff1": 0.5822939098091017, - "nauc_precision_at_3_max": 0.5587282972262201, - "nauc_precision_at_3_std": 0.42902959999001977, - "nauc_precision_at_5_diff1": 0.5514765307771033, - "nauc_precision_at_5_max": 0.5698560042206825, - "nauc_precision_at_5_std": 0.48992418681620375, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.413844347837448, - "nauc_recall_at_100_max": 0.6591032685370454, - "nauc_recall_at_100_std": 0.6882492080286065, - "nauc_recall_at_10_diff1": 0.5166753772605301, - "nauc_recall_at_10_max": 0.5890906125901273, - "nauc_recall_at_10_std": 0.5228958235520792, - "nauc_recall_at_1_diff1": 0.7003540691175822, - "nauc_recall_at_1_max": 0.5562337135287849, - "nauc_recall_at_1_std": 0.3222697658148194, - "nauc_recall_at_20_diff1": 0.49453686872493546, - "nauc_recall_at_20_max": 0.570490024802668, - "nauc_recall_at_20_std": 0.5310876413062724, - "nauc_recall_at_3_diff1": 0.5822939098091017, - "nauc_recall_at_3_max": 0.5587282972262209, - "nauc_recall_at_3_std": 0.42902959999002005, - "nauc_recall_at_5_diff1": 0.551476530777103, - "nauc_recall_at_5_max": 0.5698560042206824, - "nauc_recall_at_5_std": 0.489924186816204, - "ndcg_at_1": 0.30966, - "ndcg_at_10": 0.4485, - "ndcg_at_100": 0.49826, - "ndcg_at_1000": 0.51785, - "ndcg_at_20": 0.46443, - "ndcg_at_3": 0.3957, - "ndcg_at_5": 0.42273, - "precision_at_1": 0.30966, - "precision_at_10": 0.06016, - "precision_at_100": 0.00846, - "precision_at_1000": 0.001, - "precision_at_20": 0.03319, - "precision_at_3": 0.15205, - "precision_at_5": 0.10433, - "recall_at_1": 0.30966, - "recall_at_10": 0.60155, - "recall_at_100": 0.84573, - "recall_at_1000": 1.0, - "recall_at_20": 0.66371, - "recall_at_3": 0.45616, - "recall_at_5": 0.52164 - }, - { - "cv_recall_at_1": 0.17401, - "cv_recall_at_10": 0.40064, - "cv_recall_at_100": 0.68636, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.47046, - "cv_recall_at_3": 0.2782, - "cv_recall_at_5": 0.32653, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.2794, - "map_at_1": 0.17401, - "map_at_10": 0.24161, - "map_at_100": 0.2515, - "map_at_1000": 0.25285, - "map_at_20": 0.24637, - "map_at_3": 0.22055, - "map_at_5": 0.23135, - "mrr_at_1": 0.17400644468313642, - "mrr_at_10": 0.2416057149676911, - "mrr_at_100": 0.25150072564404563, - "mrr_at_1000": 0.2528496021900727, - "mrr_at_20": 0.24636789319443522, - "mrr_at_3": 0.2205513784461154, - "mrr_at_5": 0.23134622269960614, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.29634773574834217, - "nauc_cv_recall_at_100_max": 0.3944551058132891, - "nauc_cv_recall_at_100_std": 0.5567710352580213, - "nauc_cv_recall_at_10_diff1": 0.36568437737806625, - "nauc_cv_recall_at_10_max": 0.3702981428273432, - "nauc_cv_recall_at_10_std": 0.38843071216997727, - "nauc_cv_recall_at_1_diff1": 0.6028695349379543, - "nauc_cv_recall_at_1_max": 0.42021900033590426, - "nauc_cv_recall_at_1_std": 0.2899746114582274, - "nauc_cv_recall_at_20_diff1": 0.3510794385312679, - "nauc_cv_recall_at_20_max": 0.3635329282691102, - "nauc_cv_recall_at_20_std": 0.42662091019234755, - "nauc_cv_recall_at_3_diff1": 0.4405320603733295, - "nauc_cv_recall_at_3_max": 0.3771266366287866, - "nauc_cv_recall_at_3_std": 0.3338238960381248, - "nauc_cv_recall_at_5_diff1": 0.41682490531795086, - "nauc_cv_recall_at_5_max": 0.3887424281690552, - "nauc_cv_recall_at_5_std": 0.3429433319882689, - "nauc_map_at_1000_diff1": 0.5025158655376364, - "nauc_map_at_1000_max": 0.3992455527715951, - "nauc_map_at_1000_std": 0.32907720916655936, - "nauc_map_at_100_diff1": 0.5017141695851265, - "nauc_map_at_100_max": 0.39906454583873385, - "nauc_map_at_100_std": 0.3295697259307407, - "nauc_map_at_10_diff1": 0.502622132869752, - "nauc_map_at_10_max": 0.39881509550973365, - "nauc_map_at_10_std": 0.3245463759758096, - "nauc_map_at_1_diff1": 0.6028695349379543, - "nauc_map_at_1_max": 0.42021900033590426, - "nauc_map_at_1_std": 0.2899746114582274, - "nauc_map_at_20_diff1": 0.5019572969432622, - "nauc_map_at_20_max": 0.39843955602599274, - "nauc_map_at_20_std": 0.3275556201384039, - "nauc_map_at_3_diff1": 0.5204204057808283, - "nauc_map_at_3_max": 0.40019855440613106, - "nauc_map_at_3_std": 0.31426720952478765, - "nauc_map_at_5_diff1": 0.5118332769762192, - "nauc_map_at_5_max": 0.4019896666115835, - "nauc_map_at_5_std": 0.31667063979916427, - "nauc_mrr_at_1000_diff1": 0.5025158655376364, - "nauc_mrr_at_1000_max": 0.3992455527715951, - "nauc_mrr_at_1000_std": 0.32907720916655936, - "nauc_mrr_at_100_diff1": 0.5017141695851265, - "nauc_mrr_at_100_max": 0.39906454583873385, - "nauc_mrr_at_100_std": 0.3295697259307407, - "nauc_mrr_at_10_diff1": 0.502622132869752, - "nauc_mrr_at_10_max": 0.39881509550973365, - "nauc_mrr_at_10_std": 0.3245463759758096, - "nauc_mrr_at_1_diff1": 0.6028695349379543, - "nauc_mrr_at_1_max": 0.42021900033590426, - "nauc_mrr_at_1_std": 0.2899746114582274, - "nauc_mrr_at_20_diff1": 0.5019572969432622, - "nauc_mrr_at_20_max": 0.39843955602599274, - "nauc_mrr_at_20_std": 0.3275556201384039, - "nauc_mrr_at_3_diff1": 0.5204204057808283, - "nauc_mrr_at_3_max": 0.40019855440613106, - "nauc_mrr_at_3_std": 0.31426720952478765, - "nauc_mrr_at_5_diff1": 0.5118332769762192, - "nauc_mrr_at_5_max": 0.4019896666115835, - "nauc_mrr_at_5_std": 0.31667063979916427, - "nauc_ndcg_at_1000_diff1": 0.47799866792663115, - "nauc_ndcg_at_1000_max": 0.39570991951231005, - "nauc_ndcg_at_1000_std": 0.3479223622205202, - "nauc_ndcg_at_100_diff1": 0.457912000481337, - "nauc_ndcg_at_100_max": 0.39500436807184836, - "nauc_ndcg_at_100_std": 0.3692954991144709, - "nauc_ndcg_at_10_diff1": 0.4643091961538564, - "nauc_ndcg_at_10_max": 0.390803121224411, - "nauc_ndcg_at_10_std": 0.34168265671702486, - "nauc_ndcg_at_1_diff1": 0.6028695349379543, - "nauc_ndcg_at_1_max": 0.42021900033590426, - "nauc_ndcg_at_1_std": 0.2899746114582274, - "nauc_ndcg_at_20_diff1": 0.46111566328912856, - "nauc_ndcg_at_20_max": 0.38916229283952436, - "nauc_ndcg_at_20_std": 0.3522899166330396, - "nauc_ndcg_at_3_diff1": 0.4978605410630954, - "nauc_ndcg_at_3_max": 0.39376004888593885, - "nauc_ndcg_at_3_std": 0.3198731441754594, - "nauc_ndcg_at_5_diff1": 0.48460521512617366, - "nauc_ndcg_at_5_max": 0.39791429841604314, - "nauc_ndcg_at_5_std": 0.3241595021167435, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.2963477357483419, - "nauc_precision_at_100_max": 0.39445510581328863, - "nauc_precision_at_100_std": 0.556771035258021, - "nauc_precision_at_10_diff1": 0.36568437737806625, - "nauc_precision_at_10_max": 0.3702981428273432, - "nauc_precision_at_10_std": 0.3884307121699771, - "nauc_precision_at_1_diff1": 0.6028695349379543, - "nauc_precision_at_1_max": 0.42021900033590426, - "nauc_precision_at_1_std": 0.2899746114582274, - "nauc_precision_at_20_diff1": 0.35107943853126766, - "nauc_precision_at_20_max": 0.3635329282691101, - "nauc_precision_at_20_std": 0.4266209101923477, - "nauc_precision_at_3_diff1": 0.4405320603733291, - "nauc_precision_at_3_max": 0.37712663662878615, - "nauc_precision_at_3_std": 0.3338238960381247, - "nauc_precision_at_5_diff1": 0.4168249053179508, - "nauc_precision_at_5_max": 0.388742428169055, - "nauc_precision_at_5_std": 0.342943331988269, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.29634773574834217, - "nauc_recall_at_100_max": 0.3944551058132891, - "nauc_recall_at_100_std": 0.5567710352580213, - "nauc_recall_at_10_diff1": 0.36568437737806625, - "nauc_recall_at_10_max": 0.3702981428273432, - "nauc_recall_at_10_std": 0.38843071216997727, - "nauc_recall_at_1_diff1": 0.6028695349379543, - "nauc_recall_at_1_max": 0.42021900033590426, - "nauc_recall_at_1_std": 0.2899746114582274, - "nauc_recall_at_20_diff1": 0.3510794385312679, - "nauc_recall_at_20_max": 0.3635329282691102, - "nauc_recall_at_20_std": 0.42662091019234755, - "nauc_recall_at_3_diff1": 0.4405320603733295, - "nauc_recall_at_3_max": 0.3771266366287866, - "nauc_recall_at_3_std": 0.3338238960381248, - "nauc_recall_at_5_diff1": 0.41682490531795086, - "nauc_recall_at_5_max": 0.3887424281690552, - "nauc_recall_at_5_std": 0.3429433319882689, - "ndcg_at_1": 0.17401, - "ndcg_at_10": 0.2794, - "ndcg_at_100": 0.33618, - "ndcg_at_1000": 0.3752, - "ndcg_at_20": 0.29695, - "ndcg_at_3": 0.23538, - "ndcg_at_5": 0.25507, - "precision_at_1": 0.17401, - "precision_at_10": 0.04006, - "precision_at_100": 0.00686, - "precision_at_1000": 0.001, - "precision_at_20": 0.02352, - "precision_at_3": 0.09273, - "precision_at_5": 0.06531, - "recall_at_1": 0.17401, - "recall_at_10": 0.40064, - "recall_at_100": 0.68636, - "recall_at_1000": 1.0, - "recall_at_20": 0.47046, - "recall_at_3": 0.2782, - "recall_at_5": 0.32653 - }, - { - "cv_recall_at_1": 0.189, - "cv_recall_at_10": 0.434, - "cv_recall_at_100": 0.776, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.52, - "cv_recall_at_3": 0.287, - "cv_recall_at_5": 0.351, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.29912, - "map_at_1": 0.189, - "map_at_10": 0.25745, - "map_at_100": 0.27018, - "map_at_1000": 0.27119, - "map_at_20": 0.26366, - "map_at_3": 0.232, - "map_at_5": 0.24625, - "mrr_at_1": 0.189, - "mrr_at_10": 0.257453968253968, - "mrr_at_100": 0.27018074089708155, - "mrr_at_1000": 0.2711940749154071, - "mrr_at_20": 0.26365941537392756, - "mrr_at_3": 0.2320000000000002, - "mrr_at_5": 0.24625000000000002, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.23584547731802377, - "nauc_cv_recall_at_100_max": 0.5048956866760526, - "nauc_cv_recall_at_100_std": 0.5603047119270076, - "nauc_cv_recall_at_10_diff1": 0.38566115088987807, - "nauc_cv_recall_at_10_max": 0.28669233113414205, - "nauc_cv_recall_at_10_std": 0.28326331170607066, - "nauc_cv_recall_at_1_diff1": 0.4977619739879074, - "nauc_cv_recall_at_1_max": 0.3444770313188568, - "nauc_cv_recall_at_1_std": 0.22987253973319294, - "nauc_cv_recall_at_20_diff1": 0.3523029000489689, - "nauc_cv_recall_at_20_max": 0.3360275314217313, - "nauc_cv_recall_at_20_std": 0.31898770335709226, - "nauc_cv_recall_at_3_diff1": 0.42690386232087146, - "nauc_cv_recall_at_3_max": 0.3048319388632666, - "nauc_cv_recall_at_3_std": 0.21794317850827377, - "nauc_cv_recall_at_5_diff1": 0.4093556153855275, - "nauc_cv_recall_at_5_max": 0.3190624353434359, - "nauc_cv_recall_at_5_std": 0.2662802757450084, - "nauc_map_at_1000_diff1": 0.45476595623650545, - "nauc_map_at_1000_max": 0.33102341505163757, - "nauc_map_at_1000_std": 0.24388150035754028, - "nauc_map_at_100_diff1": 0.4541710217692779, - "nauc_map_at_100_max": 0.3314191205195175, - "nauc_map_at_100_std": 0.2445461478717132, - "nauc_map_at_10_diff1": 0.4579805762156703, - "nauc_map_at_10_max": 0.32450039624849997, - "nauc_map_at_10_std": 0.23816668982486297, - "nauc_map_at_1_diff1": 0.4977619739879074, - "nauc_map_at_1_max": 0.3444770313188568, - "nauc_map_at_1_std": 0.22987253973319294, - "nauc_map_at_20_diff1": 0.4556901341073797, - "nauc_map_at_20_max": 0.3284399315434941, - "nauc_map_at_20_std": 0.2410209601625378, - "nauc_map_at_3_diff1": 0.4679324337192478, - "nauc_map_at_3_max": 0.3260264006938065, - "nauc_map_at_3_std": 0.22374444652641026, - "nauc_map_at_5_diff1": 0.46277591167639953, - "nauc_map_at_5_max": 0.3297028251995126, - "nauc_map_at_5_std": 0.23572419801369274, - "nauc_mrr_at_1000_diff1": 0.45476595623650545, - "nauc_mrr_at_1000_max": 0.33102341505163757, - "nauc_mrr_at_1000_std": 0.24388150035754028, - "nauc_mrr_at_100_diff1": 0.4541710217692779, - "nauc_mrr_at_100_max": 0.3314191205195175, - "nauc_mrr_at_100_std": 0.2445461478717132, - "nauc_mrr_at_10_diff1": 0.4579805762156703, - "nauc_mrr_at_10_max": 0.32450039624849997, - "nauc_mrr_at_10_std": 0.23816668982486297, - "nauc_mrr_at_1_diff1": 0.4977619739879074, - "nauc_mrr_at_1_max": 0.3444770313188568, - "nauc_mrr_at_1_std": 0.22987253973319294, - "nauc_mrr_at_20_diff1": 0.4556901341073797, - "nauc_mrr_at_20_max": 0.3284399315434941, - "nauc_mrr_at_20_std": 0.2410209601625378, - "nauc_mrr_at_3_diff1": 0.4679324337192478, - "nauc_mrr_at_3_max": 0.3260264006938065, - "nauc_mrr_at_3_std": 0.22374444652641026, - "nauc_mrr_at_5_diff1": 0.46277591167639953, - "nauc_mrr_at_5_max": 0.3297028251995126, - "nauc_mrr_at_5_std": 0.23572419801369274, - "nauc_ndcg_at_1000_diff1": 0.43772123611759606, - "nauc_ndcg_at_1000_max": 0.33609697923437043, - "nauc_ndcg_at_1000_std": 0.26127496831875935, - "nauc_ndcg_at_100_diff1": 0.4203011357051008, - "nauc_ndcg_at_100_max": 0.3501915540657517, - "nauc_ndcg_at_100_std": 0.2858917313816372, - "nauc_ndcg_at_10_diff1": 0.43839515861227235, - "nauc_ndcg_at_10_max": 0.3146284118343576, - "nauc_ndcg_at_10_std": 0.24952184654188878, - "nauc_ndcg_at_1_diff1": 0.4977619739879074, - "nauc_ndcg_at_1_max": 0.3444770313188568, - "nauc_ndcg_at_1_std": 0.22987253973319294, - "nauc_ndcg_at_20_diff1": 0.42995677255332104, - "nauc_ndcg_at_20_max": 0.3284335639984129, - "nauc_ndcg_at_20_std": 0.2591496857680367, - "nauc_ndcg_at_3_diff1": 0.4567421335185591, - "nauc_ndcg_at_3_max": 0.32018196382995895, - "nauc_ndcg_at_3_std": 0.22212212819728647, - "nauc_ndcg_at_5_diff1": 0.44806804419654234, - "nauc_ndcg_at_5_max": 0.3265209685228764, - "nauc_ndcg_at_5_std": 0.24354943983311997, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.2358454773180231, - "nauc_precision_at_100_max": 0.5048956866760533, - "nauc_precision_at_100_std": 0.5603047119270077, - "nauc_precision_at_10_diff1": 0.3856611508898776, - "nauc_precision_at_10_max": 0.2866923311341418, - "nauc_precision_at_10_std": 0.2832633117060702, - "nauc_precision_at_1_diff1": 0.4977619739879074, - "nauc_precision_at_1_max": 0.3444770313188568, - "nauc_precision_at_1_std": 0.22987253973319294, - "nauc_precision_at_20_diff1": 0.35230290004896864, - "nauc_precision_at_20_max": 0.33602753142173103, - "nauc_precision_at_20_std": 0.31898770335709214, - "nauc_precision_at_3_diff1": 0.4269038623208713, - "nauc_precision_at_3_max": 0.3048319388632666, - "nauc_precision_at_3_std": 0.21794317850827388, - "nauc_precision_at_5_diff1": 0.40935561538552745, - "nauc_precision_at_5_max": 0.3190624353434356, - "nauc_precision_at_5_std": 0.26628027574500823, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.23584547731802377, - "nauc_recall_at_100_max": 0.5048956866760526, - "nauc_recall_at_100_std": 0.5603047119270076, - "nauc_recall_at_10_diff1": 0.38566115088987807, - "nauc_recall_at_10_max": 0.28669233113414205, - "nauc_recall_at_10_std": 0.28326331170607066, - "nauc_recall_at_1_diff1": 0.4977619739879074, - "nauc_recall_at_1_max": 0.3444770313188568, - "nauc_recall_at_1_std": 0.22987253973319294, - "nauc_recall_at_20_diff1": 0.3523029000489689, - "nauc_recall_at_20_max": 0.3360275314217313, - "nauc_recall_at_20_std": 0.31898770335709226, - "nauc_recall_at_3_diff1": 0.42690386232087146, - "nauc_recall_at_3_max": 0.3048319388632666, - "nauc_recall_at_3_std": 0.21794317850827377, - "nauc_recall_at_5_diff1": 0.4093556153855275, - "nauc_recall_at_5_max": 0.3190624353434359, - "nauc_recall_at_5_std": 0.2662802757450084, - "ndcg_at_1": 0.189, - "ndcg_at_10": 0.29912, - "ndcg_at_100": 0.36856, - "ndcg_at_1000": 0.39676, - "ndcg_at_20": 0.32116, - "ndcg_at_3": 0.24612, - "ndcg_at_5": 0.27215, - "precision_at_1": 0.189, - "precision_at_10": 0.0434, - "precision_at_100": 0.00776, - "precision_at_1000": 0.001, - "precision_at_20": 0.026, - "precision_at_3": 0.09567, - "precision_at_5": 0.0702, - "recall_at_1": 0.189, - "recall_at_10": 0.434, - "recall_at_100": 0.776, - "recall_at_1000": 1.0, - "recall_at_20": 0.52, - "recall_at_3": 0.287, - "recall_at_5": 0.351 - }, - { - "cv_recall_at_1": 0.29126, - "cv_recall_at_10": 0.56311, - "cv_recall_at_100": 0.81969, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.64216, - "cv_recall_at_3": 0.43273, - "cv_recall_at_5": 0.49098, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.42183, - "map_at_1": 0.29126, - "map_at_10": 0.37732, - "map_at_100": 0.38717, - "map_at_1000": 0.38804, - "map_at_20": 0.38286, - "map_at_3": 0.3546, - "map_at_5": 0.36771, - "mrr_at_1": 0.2912621359223301, - "mrr_at_10": 0.37731875921889774, - "mrr_at_100": 0.38716967898159166, - "mrr_at_1000": 0.3880350605603339, - "mrr_at_20": 0.38286417328162536, - "mrr_at_3": 0.35460009246417024, - "mrr_at_5": 0.367706888580675, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.43538500388127865, - "nauc_cv_recall_at_100_max": 0.7177252933127443, - "nauc_cv_recall_at_100_std": 0.660412445901951, - "nauc_cv_recall_at_10_diff1": 0.4963201835184886, - "nauc_cv_recall_at_10_max": 0.5597989306303259, - "nauc_cv_recall_at_10_std": 0.48074848708310886, - "nauc_cv_recall_at_1_diff1": 0.6473674367686191, - "nauc_cv_recall_at_1_max": 0.49186622428315996, - "nauc_cv_recall_at_1_std": 0.3119812982341505, - "nauc_cv_recall_at_20_diff1": 0.5001328769811207, - "nauc_cv_recall_at_20_max": 0.6028237648849925, - "nauc_cv_recall_at_20_std": 0.5323859658135447, - "nauc_cv_recall_at_3_diff1": 0.5468222678409858, - "nauc_cv_recall_at_3_max": 0.5128205176152807, - "nauc_cv_recall_at_3_std": 0.40675980904226217, - "nauc_cv_recall_at_5_diff1": 0.5020836460064558, - "nauc_cv_recall_at_5_max": 0.5394861529788975, - "nauc_cv_recall_at_5_std": 0.45600299088374213, - "nauc_map_at_1000_diff1": 0.5870509021904564, - "nauc_map_at_1000_max": 0.5065320181114963, - "nauc_map_at_1000_std": 0.36554314852816566, - "nauc_map_at_100_diff1": 0.5867048660360288, - "nauc_map_at_100_max": 0.5069224205930181, - "nauc_map_at_100_std": 0.3660919004931112, - "nauc_map_at_10_diff1": 0.5864163481883536, - "nauc_map_at_10_max": 0.5047660836484374, - "nauc_map_at_10_std": 0.3641270487884386, - "nauc_map_at_1_diff1": 0.6473674367686191, - "nauc_map_at_1_max": 0.49186622428315996, - "nauc_map_at_1_std": 0.3119812982341505, - "nauc_map_at_20_diff1": 0.5871061465808929, - "nauc_map_at_20_max": 0.5066720694370498, - "nauc_map_at_20_std": 0.36620195781101544, - "nauc_map_at_3_diff1": 0.5973239337484972, - "nauc_map_at_3_max": 0.49778872535897273, - "nauc_map_at_3_std": 0.3513465169058443, - "nauc_map_at_5_diff1": 0.58703502269122, - "nauc_map_at_5_max": 0.5027772921794861, - "nauc_map_at_5_std": 0.36188878315047157, - "nauc_mrr_at_1000_diff1": 0.5870509211623248, - "nauc_mrr_at_1000_max": 0.506532051280673, - "nauc_mrr_at_1000_std": 0.36554319117411654, - "nauc_mrr_at_100_diff1": 0.5867048660360288, - "nauc_mrr_at_100_max": 0.5069224205930181, - "nauc_mrr_at_100_std": 0.3660919004931112, - "nauc_mrr_at_10_diff1": 0.5864163481883536, - "nauc_mrr_at_10_max": 0.5047660836484374, - "nauc_mrr_at_10_std": 0.3641270487884386, - "nauc_mrr_at_1_diff1": 0.6473674367686191, - "nauc_mrr_at_1_max": 0.49186622428315996, - "nauc_mrr_at_1_std": 0.3119812982341505, - "nauc_mrr_at_20_diff1": 0.5871061465808929, - "nauc_mrr_at_20_max": 0.5066720694370498, - "nauc_mrr_at_20_std": 0.36620195781101544, - "nauc_mrr_at_3_diff1": 0.5973239337484972, - "nauc_mrr_at_3_max": 0.49778872535897273, - "nauc_mrr_at_3_std": 0.3513465169058443, - "nauc_mrr_at_5_diff1": 0.58703502269122, - "nauc_mrr_at_5_max": 0.5027772921794861, - "nauc_mrr_at_5_std": 0.36188878315047157, - "nauc_ndcg_at_1000_diff1": 0.5740812385542557, - "nauc_ndcg_at_1000_max": 0.5168841998243363, - "nauc_ndcg_at_1000_std": 0.3843458584172321, - "nauc_ndcg_at_100_diff1": 0.5645003851534279, - "nauc_ndcg_at_100_max": 0.5303028488918393, - "nauc_ndcg_at_100_std": 0.4028550618170588, - "nauc_ndcg_at_10_diff1": 0.5644944296758979, - "nauc_ndcg_at_10_max": 0.5169814341365214, - "nauc_ndcg_at_10_std": 0.39129432844376977, - "nauc_ndcg_at_1_diff1": 0.6473674367686191, - "nauc_ndcg_at_1_max": 0.49186622428315996, - "nauc_ndcg_at_1_std": 0.3119812982341505, - "nauc_ndcg_at_20_diff1": 0.5669757949202917, - "nauc_ndcg_at_20_max": 0.5250403811377805, - "nauc_ndcg_at_20_std": 0.4001112801964311, - "nauc_ndcg_at_3_diff1": 0.5839362186171795, - "nauc_ndcg_at_3_max": 0.5015676540132488, - "nauc_ndcg_at_3_std": 0.3656795310689533, - "nauc_ndcg_at_5_diff1": 0.5655307294741795, - "nauc_ndcg_at_5_max": 0.5115681799159831, - "nauc_ndcg_at_5_std": 0.38525142911874694, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.4353850038812787, - "nauc_precision_at_100_max": 0.7177252933127455, - "nauc_precision_at_100_std": 0.660412445901951, - "nauc_precision_at_10_diff1": 0.496320183518489, - "nauc_precision_at_10_max": 0.559798930630326, - "nauc_precision_at_10_std": 0.4807484870831086, - "nauc_precision_at_1_diff1": 0.6473674367686191, - "nauc_precision_at_1_max": 0.49186622428315996, - "nauc_precision_at_1_std": 0.3119812982341505, - "nauc_precision_at_20_diff1": 0.5001328769811205, - "nauc_precision_at_20_max": 0.602823764884992, - "nauc_precision_at_20_std": 0.5323859658135439, - "nauc_precision_at_3_diff1": 0.5468222678409858, - "nauc_precision_at_3_max": 0.5128205176152807, - "nauc_precision_at_3_std": 0.4067598090422622, - "nauc_precision_at_5_diff1": 0.5020836460064558, - "nauc_precision_at_5_max": 0.5394861529788973, - "nauc_precision_at_5_std": 0.45600299088374197, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.43538500388127865, - "nauc_recall_at_100_max": 0.7177252933127443, - "nauc_recall_at_100_std": 0.660412445901951, - "nauc_recall_at_10_diff1": 0.4963201835184886, - "nauc_recall_at_10_max": 0.5597989306303259, - "nauc_recall_at_10_std": 0.48074848708310886, - "nauc_recall_at_1_diff1": 0.6473674367686191, - "nauc_recall_at_1_max": 0.49186622428315996, - "nauc_recall_at_1_std": 0.3119812982341505, - "nauc_recall_at_20_diff1": 0.5001328769811207, - "nauc_recall_at_20_max": 0.6028237648849925, - "nauc_recall_at_20_std": 0.5323859658135447, - "nauc_recall_at_3_diff1": 0.5468222678409858, - "nauc_recall_at_3_max": 0.5128205176152807, - "nauc_recall_at_3_std": 0.40675980904226217, - "nauc_recall_at_5_diff1": 0.5020836460064558, - "nauc_recall_at_5_max": 0.5394861529788975, - "nauc_recall_at_5_std": 0.45600299088374213, - "ndcg_at_1": 0.29126, - "ndcg_at_10": 0.42183, - "ndcg_at_100": 0.47434, - "ndcg_at_1000": 0.49735, - "ndcg_at_20": 0.44189, - "ndcg_at_3": 0.37471, - "ndcg_at_5": 0.39852, - "precision_at_1": 0.29126, - "precision_at_10": 0.05631, - "precision_at_100": 0.0082, - "precision_at_1000": 0.001, - "precision_at_20": 0.03211, - "precision_at_3": 0.14424, - "precision_at_5": 0.0982, - "recall_at_1": 0.29126, - "recall_at_10": 0.56311, - "recall_at_100": 0.81969, - "recall_at_1000": 1.0, - "recall_at_20": 0.64216, - "recall_at_3": 0.43273, - "recall_at_5": 0.49098 - }, - { - "cv_recall_at_1": 0.45983, - "cv_recall_at_10": 0.75264, - "cv_recall_at_100": 0.94186, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.81395, - "cv_recall_at_3": 0.61416, - "cv_recall_at_5": 0.67759, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.60066, - "map_at_1": 0.45983, - "map_at_10": 0.55269, - "map_at_100": 0.56038, - "map_at_1000": 0.56068, - "map_at_20": 0.55716, - "map_at_3": 0.52819, - "map_at_5": 0.54267, - "mrr_at_1": 0.459830866807611, - "mrr_at_10": 0.552691365482063, - "mrr_at_100": 0.5603828225377104, - "mrr_at_1000": 0.5606781539383635, - "mrr_at_20": 0.5571602161130574, - "mrr_at_3": 0.5281888653981675, - "mrr_at_5": 0.5426708949964759, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4429595399611373, - "nauc_cv_recall_at_100_max": 0.5374944784553677, - "nauc_cv_recall_at_100_std": 0.5020336324941823, - "nauc_cv_recall_at_10_diff1": 0.5042140972034406, - "nauc_cv_recall_at_10_max": 0.4597924352668207, - "nauc_cv_recall_at_10_std": 0.3309821314292971, - "nauc_cv_recall_at_1_diff1": 0.6973331588586901, - "nauc_cv_recall_at_1_max": 0.47206125971867047, - "nauc_cv_recall_at_1_std": 0.16444378947679292, - "nauc_cv_recall_at_20_diff1": 0.5037305064368949, - "nauc_cv_recall_at_20_max": 0.4271691618156776, - "nauc_cv_recall_at_20_std": 0.28189017720507564, - "nauc_cv_recall_at_3_diff1": 0.580064919110858, - "nauc_cv_recall_at_3_max": 0.49645920159223683, - "nauc_cv_recall_at_3_std": 0.2711250691727226, - "nauc_cv_recall_at_5_diff1": 0.5374231247800073, - "nauc_cv_recall_at_5_max": 0.48513326853185296, - "nauc_cv_recall_at_5_std": 0.30095998643913063, - "nauc_map_at_1000_diff1": 0.6441521021101068, - "nauc_map_at_1000_max": 0.48098963639253217, - "nauc_map_at_1000_std": 0.20812381908060446, - "nauc_map_at_100_diff1": 0.643929028420692, - "nauc_map_at_100_max": 0.48105172984074207, - "nauc_map_at_100_std": 0.20837106228113203, - "nauc_map_at_10_diff1": 0.6417540808024643, - "nauc_map_at_10_max": 0.4813076374672515, - "nauc_map_at_10_std": 0.21095403593602866, - "nauc_map_at_1_diff1": 0.6973331588586901, - "nauc_map_at_1_max": 0.47206125971867047, - "nauc_map_at_1_std": 0.16444378947679292, - "nauc_map_at_20_diff1": 0.6432421142666999, - "nauc_map_at_20_max": 0.48038725601830146, - "nauc_map_at_20_std": 0.2078558363364783, - "nauc_map_at_3_diff1": 0.6489600889526264, - "nauc_map_at_3_max": 0.4866454342294112, - "nauc_map_at_3_std": 0.20663618594230385, - "nauc_map_at_5_diff1": 0.6430473848288992, - "nauc_map_at_5_max": 0.4842255766526718, - "nauc_map_at_5_std": 0.21044581143827049, - "nauc_mrr_at_1000_diff1": 0.6441520919455076, - "nauc_mrr_at_1000_max": 0.4809896494420363, - "nauc_mrr_at_1000_std": 0.20812383899078785, - "nauc_mrr_at_100_diff1": 0.643929028420692, - "nauc_mrr_at_100_max": 0.48105172984074207, - "nauc_mrr_at_100_std": 0.20837106228113203, - "nauc_mrr_at_10_diff1": 0.6417540808024643, - "nauc_mrr_at_10_max": 0.4813076374672515, - "nauc_mrr_at_10_std": 0.21095403593602866, - "nauc_mrr_at_1_diff1": 0.6973331588586901, - "nauc_mrr_at_1_max": 0.47206125971867047, - "nauc_mrr_at_1_std": 0.16444378947679292, - "nauc_mrr_at_20_diff1": 0.6432421142666999, - "nauc_mrr_at_20_max": 0.48038725601830146, - "nauc_mrr_at_20_std": 0.2078558363364783, - "nauc_mrr_at_3_diff1": 0.6489600889526264, - "nauc_mrr_at_3_max": 0.4866454342294112, - "nauc_mrr_at_3_std": 0.20663618594230385, - "nauc_mrr_at_5_diff1": 0.6430473848288992, - "nauc_mrr_at_5_max": 0.4842255766526718, - "nauc_mrr_at_5_std": 0.21044581143827049, - "nauc_ndcg_at_1000_diff1": 0.6326592590675244, - "nauc_ndcg_at_1000_max": 0.479746255773527, - "nauc_ndcg_at_1000_std": 0.21821891778897104, - "nauc_ndcg_at_100_diff1": 0.627023184120909, - "nauc_ndcg_at_100_max": 0.4814040178905751, - "nauc_ndcg_at_100_std": 0.22609099066376712, - "nauc_ndcg_at_10_diff1": 0.6160435381553313, - "nauc_ndcg_at_10_max": 0.4780939816399053, - "nauc_ndcg_at_10_std": 0.2332277013810803, - "nauc_ndcg_at_1_diff1": 0.6973331588586901, - "nauc_ndcg_at_1_max": 0.47206125971867047, - "nauc_ndcg_at_1_std": 0.16444378947679292, - "nauc_ndcg_at_20_diff1": 0.6206953233188065, - "nauc_ndcg_at_20_max": 0.4739268715311874, - "nauc_ndcg_at_20_std": 0.22160597418951172, - "nauc_ndcg_at_3_diff1": 0.6323475050574792, - "nauc_ndcg_at_3_max": 0.4892097773834569, - "nauc_ndcg_at_3_std": 0.2221068601383589, - "nauc_ndcg_at_5_diff1": 0.6199812667553398, - "nauc_ndcg_at_5_max": 0.484803228251082, - "nauc_ndcg_at_5_std": 0.23025081492492494, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.44295953996113135, - "nauc_precision_at_100_max": 0.5374944784553698, - "nauc_precision_at_100_std": 0.5020336324941786, - "nauc_precision_at_10_diff1": 0.5042140972034405, - "nauc_precision_at_10_max": 0.4597924352668208, - "nauc_precision_at_10_std": 0.33098213142929594, - "nauc_precision_at_1_diff1": 0.6973331588586901, - "nauc_precision_at_1_max": 0.47206125971867047, - "nauc_precision_at_1_std": 0.16444378947679292, - "nauc_precision_at_20_diff1": 0.5037305064368954, - "nauc_precision_at_20_max": 0.4271691618156773, - "nauc_precision_at_20_std": 0.28189017720507437, - "nauc_precision_at_3_diff1": 0.5800649191108583, - "nauc_precision_at_3_max": 0.49645920159223705, - "nauc_precision_at_3_std": 0.2711250691727229, - "nauc_precision_at_5_diff1": 0.5374231247800063, - "nauc_precision_at_5_max": 0.4851332685318528, - "nauc_precision_at_5_std": 0.3009599864391298, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4429595399611373, - "nauc_recall_at_100_max": 0.5374944784553677, - "nauc_recall_at_100_std": 0.5020336324941823, - "nauc_recall_at_10_diff1": 0.5042140972034406, - "nauc_recall_at_10_max": 0.4597924352668207, - "nauc_recall_at_10_std": 0.3309821314292971, - "nauc_recall_at_1_diff1": 0.6973331588586901, - "nauc_recall_at_1_max": 0.47206125971867047, - "nauc_recall_at_1_std": 0.16444378947679292, - "nauc_recall_at_20_diff1": 0.5037305064368949, - "nauc_recall_at_20_max": 0.4271691618156776, - "nauc_recall_at_20_std": 0.28189017720507564, - "nauc_recall_at_3_diff1": 0.580064919110858, - "nauc_recall_at_3_max": 0.49645920159223683, - "nauc_recall_at_3_std": 0.2711250691727226, - "nauc_recall_at_5_diff1": 0.5374231247800073, - "nauc_recall_at_5_max": 0.48513326853185296, - "nauc_recall_at_5_std": 0.30095998643913063, - "ndcg_at_1": 0.45983, - "ndcg_at_10": 0.60066, - "ndcg_at_100": 0.64004, - "ndcg_at_1000": 0.64756, - "ndcg_at_20": 0.61643, - "ndcg_at_3": 0.55028, - "ndcg_at_5": 0.5764, - "precision_at_1": 0.45983, - "precision_at_10": 0.07526, - "precision_at_100": 0.00942, - "precision_at_1000": 0.001, - "precision_at_20": 0.0407, - "precision_at_3": 0.20472, - "precision_at_5": 0.13552, - "recall_at_1": 0.45983, - "recall_at_10": 0.75264, - "recall_at_100": 0.94186, - "recall_at_1000": 1.0, - "recall_at_20": 0.81395, - "recall_at_3": 0.61416, - "recall_at_5": 0.67759 - }, - { - "cv_recall_at_1": 0.421, - "cv_recall_at_10": 0.697, - "cv_recall_at_100": 0.947, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.769, - "cv_recall_at_3": 0.558, - "cv_recall_at_5": 0.608, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.55025, - "map_at_1": 0.421, - "map_at_10": 0.50454, - "map_at_100": 0.51404, - "map_at_1000": 0.5143, - "map_at_20": 0.5095, - "map_at_3": 0.48167, - "map_at_5": 0.49302, - "mrr_at_1": 0.421, - "mrr_at_10": 0.5045424603174602, - "mrr_at_100": 0.514036278166916, - "mrr_at_1000": 0.5143008176799778, - "mrr_at_20": 0.5094954526424723, - "mrr_at_3": 0.4816666666666664, - "mrr_at_5": 0.49301666666666627, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4855627785705507, - "nauc_cv_recall_at_100_max": 0.731365502175715, - "nauc_cv_recall_at_100_std": 0.7247678945792161, - "nauc_cv_recall_at_10_diff1": 0.5321993122841938, - "nauc_cv_recall_at_10_max": 0.3845373133259707, - "nauc_cv_recall_at_10_std": 0.22025163739727724, - "nauc_cv_recall_at_1_diff1": 0.7210976082522879, - "nauc_cv_recall_at_1_max": 0.4254087701305756, - "nauc_cv_recall_at_1_std": 0.20592471965325038, - "nauc_cv_recall_at_20_diff1": 0.5359568379661741, - "nauc_cv_recall_at_20_max": 0.40377868528873234, - "nauc_cv_recall_at_20_std": 0.2406943573982335, - "nauc_cv_recall_at_3_diff1": 0.621321956788392, - "nauc_cv_recall_at_3_max": 0.38942145623174623, - "nauc_cv_recall_at_3_std": 0.1645052364400703, - "nauc_cv_recall_at_5_diff1": 0.5961893952166478, - "nauc_cv_recall_at_5_max": 0.3920810351887088, - "nauc_cv_recall_at_5_std": 0.18723355572147657, - "nauc_map_at_1000_diff1": 0.6697072108072887, - "nauc_map_at_1000_max": 0.411943976387272, - "nauc_map_at_1000_std": 0.1967222314828611, - "nauc_map_at_100_diff1": 0.6695110001873309, - "nauc_map_at_100_max": 0.4121863201598696, - "nauc_map_at_100_std": 0.19710670362571528, - "nauc_map_at_10_diff1": 0.6672475373507716, - "nauc_map_at_10_max": 0.4098526385755607, - "nauc_map_at_10_std": 0.19445865524250197, - "nauc_map_at_1_diff1": 0.7210976082522879, - "nauc_map_at_1_max": 0.4254087701305756, - "nauc_map_at_1_std": 0.20592471965325038, - "nauc_map_at_20_diff1": 0.668795383818105, - "nauc_map_at_20_max": 0.41071496567002347, - "nauc_map_at_20_std": 0.19505186847070433, - "nauc_map_at_3_diff1": 0.6771890825309842, - "nauc_map_at_3_max": 0.40849482153000877, - "nauc_map_at_3_std": 0.184697047083633, - "nauc_map_at_5_diff1": 0.6726059841502193, - "nauc_map_at_5_max": 0.409503476165295, - "nauc_map_at_5_std": 0.1899677021603648, - "nauc_mrr_at_1000_diff1": 0.6697072108072887, - "nauc_mrr_at_1000_max": 0.411943976387272, - "nauc_mrr_at_1000_std": 0.1967222314828611, - "nauc_mrr_at_100_diff1": 0.6695110001873309, - "nauc_mrr_at_100_max": 0.4121863201598696, - "nauc_mrr_at_100_std": 0.19710670362571528, - "nauc_mrr_at_10_diff1": 0.6672475373507716, - "nauc_mrr_at_10_max": 0.4098526385755607, - "nauc_mrr_at_10_std": 0.19445865524250197, - "nauc_mrr_at_1_diff1": 0.7210976082522879, - "nauc_mrr_at_1_max": 0.4254087701305756, - "nauc_mrr_at_1_std": 0.20592471965325038, - "nauc_mrr_at_20_diff1": 0.668795383818105, - "nauc_mrr_at_20_max": 0.41071496567002347, - "nauc_mrr_at_20_std": 0.19505186847070433, - "nauc_mrr_at_3_diff1": 0.6771890825309842, - "nauc_mrr_at_3_max": 0.40849482153000877, - "nauc_mrr_at_3_std": 0.184697047083633, - "nauc_mrr_at_5_diff1": 0.6726059841502193, - "nauc_mrr_at_5_max": 0.409503476165295, - "nauc_mrr_at_5_std": 0.1899677021603648, - "nauc_ndcg_at_1000_diff1": 0.6581251444086017, - "nauc_ndcg_at_1000_max": 0.4134415641023258, - "nauc_ndcg_at_1000_std": 0.20355955369147752, - "nauc_ndcg_at_100_diff1": 0.6535580361197576, - "nauc_ndcg_at_100_max": 0.4211935378090691, - "nauc_ndcg_at_100_std": 0.21616041137002734, - "nauc_ndcg_at_10_diff1": 0.6404035207915835, - "nauc_ndcg_at_10_max": 0.4047612423982089, - "nauc_ndcg_at_10_std": 0.1987362690556888, - "nauc_ndcg_at_1_diff1": 0.7210976082522879, - "nauc_ndcg_at_1_max": 0.4254087701305756, - "nauc_ndcg_at_1_std": 0.20592471965325038, - "nauc_ndcg_at_20_diff1": 0.6455609884072833, - "nauc_ndcg_at_20_max": 0.40861630089629375, - "nauc_ndcg_at_20_std": 0.20131788577749649, - "nauc_ndcg_at_3_diff1": 0.6633132744102853, - "nauc_ndcg_at_3_max": 0.40370965648596985, - "nauc_ndcg_at_3_std": 0.17954858276017077, - "nauc_ndcg_at_5_diff1": 0.6546348270037611, - "nauc_ndcg_at_5_max": 0.40525686733966526, - "nauc_ndcg_at_5_std": 0.18885692395096065, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.4855627785705482, - "nauc_precision_at_100_max": 0.731365502175715, - "nauc_precision_at_100_std": 0.7247678945792142, - "nauc_precision_at_10_diff1": 0.5321993122841937, - "nauc_precision_at_10_max": 0.3845373133259703, - "nauc_precision_at_10_std": 0.22025163739727607, - "nauc_precision_at_1_diff1": 0.7210976082522879, - "nauc_precision_at_1_max": 0.4254087701305756, - "nauc_precision_at_1_std": 0.20592471965325038, - "nauc_precision_at_20_diff1": 0.535956837966174, - "nauc_precision_at_20_max": 0.40377868528873134, - "nauc_precision_at_20_std": 0.24069435739823442, - "nauc_precision_at_3_diff1": 0.6213219567883927, - "nauc_precision_at_3_max": 0.3894214562317463, - "nauc_precision_at_3_std": 0.16450523644007103, - "nauc_precision_at_5_diff1": 0.5961893952166473, - "nauc_precision_at_5_max": 0.39208103518870874, - "nauc_precision_at_5_std": 0.18723355572147696, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4855627785705507, - "nauc_recall_at_100_max": 0.731365502175715, - "nauc_recall_at_100_std": 0.7247678945792161, - "nauc_recall_at_10_diff1": 0.5321993122841938, - "nauc_recall_at_10_max": 0.3845373133259707, - "nauc_recall_at_10_std": 0.22025163739727724, - "nauc_recall_at_1_diff1": 0.7210976082522879, - "nauc_recall_at_1_max": 0.4254087701305756, - "nauc_recall_at_1_std": 0.20592471965325038, - "nauc_recall_at_20_diff1": 0.5359568379661741, - "nauc_recall_at_20_max": 0.40377868528873234, - "nauc_recall_at_20_std": 0.2406943573982335, - "nauc_recall_at_3_diff1": 0.621321956788392, - "nauc_recall_at_3_max": 0.38942145623174623, - "nauc_recall_at_3_std": 0.1645052364400703, - "nauc_recall_at_5_diff1": 0.5961893952166478, - "nauc_recall_at_5_max": 0.3920810351887088, - "nauc_recall_at_5_std": 0.18723355572147657, - "ndcg_at_1": 0.421, - "ndcg_at_10": 0.55025, - "ndcg_at_100": 0.60141, - "ndcg_at_1000": 0.60826, - "ndcg_at_20": 0.56839, - "ndcg_at_3": 0.50128, - "ndcg_at_5": 0.52181, - "precision_at_1": 0.421, - "precision_at_10": 0.0697, - "precision_at_100": 0.00947, - "precision_at_1000": 0.001, - "precision_at_20": 0.03845, - "precision_at_3": 0.186, - "precision_at_5": 0.1216, - "recall_at_1": 0.421, - "recall_at_10": 0.697, - "recall_at_100": 0.947, - "recall_at_1000": 1.0, - "recall_at_20": 0.769, - "recall_at_3": 0.558, - "recall_at_5": 0.608 - } - ] - }, - "task_name": "WITT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/model_meta.json b/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/model_meta.json deleted file mode 100644 index d1bb13979d..0000000000 --- a/results-mieb/google__siglip-base-patch16-256-multilingual/8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "google/siglip-base-patch16-256-multilingual", "revision": "8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR10.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR10.json deleted file mode 100644 index a94a3ac8a1..0000000000 --- a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR10.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", - "evaluation_time": 59.87628698348999, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.93338, - "f1": 0.9334631436619831, - "f1_weighted": 0.9334631436619831, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.93338, - "scores_per_experiment": [ - { - "accuracy": 0.9326, - "f1": 0.9325960933629046, - "f1_weighted": 0.9325960933629045 - }, - { - "accuracy": 0.9351, - "f1": 0.935126693421535, - "f1_weighted": 0.935126693421535 - }, - { - "accuracy": 0.9286, - "f1": 0.9287279749504849, - "f1_weighted": 0.9287279749504851 - }, - { - "accuracy": 0.9345, - "f1": 0.9344698910282221, - "f1_weighted": 0.9344698910282219 - }, - { - "accuracy": 0.9361, - "f1": 0.9363950655467697, - "f1_weighted": 0.9363950655467698 - } - ] - } - ] - }, - "task_name": "CIFAR10" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR100.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR100.json deleted file mode 100644 index f7b2bb3c5c..0000000000 --- a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/CIFAR100.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", - "evaluation_time": 101.40493559837341, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.75654, - "f1": 0.7586602884829271, - "f1_weighted": 0.7586602884829272, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.75654, - "scores_per_experiment": [ - { - "accuracy": 0.7565, - "f1": 0.757924035988359, - "f1_weighted": 0.7579240359883591 - }, - { - "accuracy": 0.7598, - "f1": 0.7613388743142541, - "f1_weighted": 0.7613388743142538 - }, - { - "accuracy": 0.759, - "f1": 0.7617133389971635, - "f1_weighted": 0.7617133389971638 - }, - { - "accuracy": 0.7498, - "f1": 0.7524641724151415, - "f1_weighted": 0.7524641724151419 - }, - { - "accuracy": 0.7576, - "f1": 0.7598610206997175, - "f1_weighted": 0.7598610206997176 - } - ] - } - ] - }, - "task_name": "CIFAR100" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNIST.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNIST.json deleted file mode 100644 index 5c6ed62858..0000000000 --- a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 56.67082715034485, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.9572800000000001, - "f1": 0.9571422576266858, - "f1_weighted": 0.9573362168194761, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9572800000000001, - "scores_per_experiment": [ - { - "accuracy": 0.9592, - "f1": 0.9590539585386963, - "f1_weighted": 0.9592951422536539 - }, - { - "accuracy": 0.9499, - "f1": 0.9494882828072895, - "f1_weighted": 0.9498086431430818 - }, - { - "accuracy": 0.9597, - "f1": 0.9594446389305078, - "f1_weighted": 0.9597836962699936 - }, - { - "accuracy": 0.9578, - "f1": 0.9578834947792313, - "f1_weighted": 0.9580006084749224 - }, - { - "accuracy": 0.9598, - "f1": 0.9598409130777045, - "f1_weighted": 0.9597929939557287 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNISTZeroShot.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNISTZeroShot.json deleted file mode 100644 index b77069c7c0..0000000000 --- a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/MNISTZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 50.64418292045593, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.8457, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8457 - } - ] - }, - "task_name": "MNISTZeroShot" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/WITT2IRetrieval.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/WITT2IRetrieval.json deleted file mode 100644 index 617035f059..0000000000 --- a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/WITT2IRetrieval.json +++ /dev/null @@ -1,1936 +0,0 @@ -{ - "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", - "evaluation_time": 76.18750858306885, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.13483, - "cv_recall_at_10": 0.30337, - "cv_recall_at_100": 0.63933, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.38202, - "cv_recall_at_3": 0.19326, - "cv_recall_at_5": 0.24157, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.20723, - "map_at_1": 0.13483, - "map_at_10": 0.17781, - "map_at_100": 0.18943, - "map_at_1000": 0.19099, - "map_at_20": 0.18309, - "map_at_3": 0.15861, - "map_at_5": 0.16957, - "mrr_at_1": 0.1348314606741573, - "mrr_at_10": 0.1778107722489745, - "mrr_at_100": 0.18943211198548437, - "mrr_at_1000": 0.19098625996171575, - "mrr_at_20": 0.18308938797198454, - "mrr_at_3": 0.15861423220973786, - "mrr_at_5": 0.16956928838951316, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.3002040717601591, - "nauc_cv_recall_at_100_max": 0.4681682095972851, - "nauc_cv_recall_at_100_std": 0.4755266180167234, - "nauc_cv_recall_at_10_diff1": 0.3701965776989649, - "nauc_cv_recall_at_10_max": 0.3761393776807772, - "nauc_cv_recall_at_10_std": 0.3489670955910214, - "nauc_cv_recall_at_1_diff1": 0.6065338821108335, - "nauc_cv_recall_at_1_max": 0.4882184330442451, - "nauc_cv_recall_at_1_std": 0.3549830956930766, - "nauc_cv_recall_at_20_diff1": 0.356259847114431, - "nauc_cv_recall_at_20_max": 0.39515084320476174, - "nauc_cv_recall_at_20_std": 0.4012473011612301, - "nauc_cv_recall_at_3_diff1": 0.4336991857794745, - "nauc_cv_recall_at_3_max": 0.4503825046974262, - "nauc_cv_recall_at_3_std": 0.3552670822273256, - "nauc_cv_recall_at_5_diff1": 0.4053060368334942, - "nauc_cv_recall_at_5_max": 0.40797751394890314, - "nauc_cv_recall_at_5_std": 0.32909342618618104, - "nauc_map_at_1000_diff1": 0.5054655795372572, - "nauc_map_at_1000_max": 0.45675426413623993, - "nauc_map_at_1000_std": 0.3581527976889818, - "nauc_map_at_100_diff1": 0.5043801271441104, - "nauc_map_at_100_max": 0.45653696828813317, - "nauc_map_at_100_std": 0.3583138001490483, - "nauc_map_at_10_diff1": 0.5075131819044263, - "nauc_map_at_10_max": 0.45403109899811495, - "nauc_map_at_10_std": 0.35281856964515634, - "nauc_map_at_1_diff1": 0.6065338821108335, - "nauc_map_at_1_max": 0.4882184330442451, - "nauc_map_at_1_std": 0.3549830956930766, - "nauc_map_at_20_diff1": 0.505531819154646, - "nauc_map_at_20_max": 0.4552086212020412, - "nauc_map_at_20_std": 0.3577827870853602, - "nauc_map_at_3_diff1": 0.5259349787448474, - "nauc_map_at_3_max": 0.47348219240664524, - "nauc_map_at_3_std": 0.3568078210563393, - "nauc_map_at_5_diff1": 0.5155157517292684, - "nauc_map_at_5_max": 0.45978901596979405, - "nauc_map_at_5_std": 0.3485293329187086, - "nauc_mrr_at_1000_diff1": 0.5054655795372572, - "nauc_mrr_at_1000_max": 0.45675426413623993, - "nauc_mrr_at_1000_std": 0.3581527976889818, - "nauc_mrr_at_100_diff1": 0.5043801271441104, - "nauc_mrr_at_100_max": 0.45653696828813317, - "nauc_mrr_at_100_std": 0.3583138001490483, - "nauc_mrr_at_10_diff1": 0.5075131819044263, - "nauc_mrr_at_10_max": 0.45403109899811495, - "nauc_mrr_at_10_std": 0.35281856964515634, - "nauc_mrr_at_1_diff1": 0.6065338821108335, - "nauc_mrr_at_1_max": 0.4882184330442451, - "nauc_mrr_at_1_std": 0.3549830956930766, - "nauc_mrr_at_20_diff1": 0.505531819154646, - "nauc_mrr_at_20_max": 0.4552086212020412, - "nauc_mrr_at_20_std": 0.3577827870853602, - "nauc_mrr_at_3_diff1": 0.5259349787448474, - "nauc_mrr_at_3_max": 0.47348219240664524, - "nauc_mrr_at_3_std": 0.3568078210563393, - "nauc_mrr_at_5_diff1": 0.5155157517292684, - "nauc_mrr_at_5_max": 0.45978901596979405, - "nauc_mrr_at_5_std": 0.3485293329187086, - "nauc_ndcg_at_1000_diff1": 0.47759230790461354, - "nauc_ndcg_at_1000_max": 0.44982400569152814, - "nauc_ndcg_at_1000_std": 0.3667093394535492, - "nauc_ndcg_at_100_diff1": 0.45295974994472493, - "nauc_ndcg_at_100_max": 0.4508470813906076, - "nauc_ndcg_at_100_std": 0.379308242615182, - "nauc_ndcg_at_10_diff1": 0.46603335111945293, - "nauc_ndcg_at_10_max": 0.43139552979443707, - "nauc_ndcg_at_10_std": 0.35151331443779565, - "nauc_ndcg_at_1_diff1": 0.6065338821108335, - "nauc_ndcg_at_1_max": 0.4882184330442451, - "nauc_ndcg_at_1_std": 0.3549830956930766, - "nauc_ndcg_at_20_diff1": 0.4601678130379194, - "nauc_ndcg_at_20_max": 0.4364576544675796, - "nauc_ndcg_at_20_std": 0.3683819085567761, - "nauc_ndcg_at_3_diff1": 0.499746811378489, - "nauc_ndcg_at_3_max": 0.4670335489333946, - "nauc_ndcg_at_3_std": 0.3564343587817275, - "nauc_ndcg_at_5_diff1": 0.48303833414011565, - "nauc_ndcg_at_5_max": 0.4448883386689996, - "nauc_ndcg_at_5_std": 0.3430501197807311, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3002040717601596, - "nauc_precision_at_100_max": 0.46816820959728567, - "nauc_precision_at_100_std": 0.4755266180167234, - "nauc_precision_at_10_diff1": 0.37019657769896475, - "nauc_precision_at_10_max": 0.37613937768077704, - "nauc_precision_at_10_std": 0.34896709559102135, - "nauc_precision_at_1_diff1": 0.6065338821108335, - "nauc_precision_at_1_max": 0.4882184330442451, - "nauc_precision_at_1_std": 0.3549830956930766, - "nauc_precision_at_20_diff1": 0.3562598471144309, - "nauc_precision_at_20_max": 0.3951508432047615, - "nauc_precision_at_20_std": 0.40124730116123025, - "nauc_precision_at_3_diff1": 0.43369918577947425, - "nauc_precision_at_3_max": 0.4503825046974258, - "nauc_precision_at_3_std": 0.35526708222732556, - "nauc_precision_at_5_diff1": 0.40530603683349403, - "nauc_precision_at_5_max": 0.4079775139489031, - "nauc_precision_at_5_std": 0.32909342618618126, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.3002040717601591, - "nauc_recall_at_100_max": 0.4681682095972851, - "nauc_recall_at_100_std": 0.4755266180167234, - "nauc_recall_at_10_diff1": 0.3701965776989649, - "nauc_recall_at_10_max": 0.3761393776807772, - "nauc_recall_at_10_std": 0.3489670955910214, - "nauc_recall_at_1_diff1": 0.6065338821108335, - "nauc_recall_at_1_max": 0.4882184330442451, - "nauc_recall_at_1_std": 0.3549830956930766, - "nauc_recall_at_20_diff1": 0.356259847114431, - "nauc_recall_at_20_max": 0.39515084320476174, - "nauc_recall_at_20_std": 0.4012473011612301, - "nauc_recall_at_3_diff1": 0.4336991857794745, - "nauc_recall_at_3_max": 0.4503825046974262, - "nauc_recall_at_3_std": 0.3552670822273256, - "nauc_recall_at_5_diff1": 0.4053060368334942, - "nauc_recall_at_5_max": 0.40797751394890314, - "nauc_recall_at_5_std": 0.32909342618618104, - "ndcg_at_1": 0.13483, - "ndcg_at_10": 0.20723, - "ndcg_at_100": 0.27408, - "ndcg_at_1000": 0.31905, - "ndcg_at_20": 0.22689, - "ndcg_at_3": 0.16743, - "ndcg_at_5": 0.18725, - "precision_at_1": 0.13483, - "precision_at_10": 0.03034, - "precision_at_100": 0.00639, - "precision_at_1000": 0.001, - "precision_at_20": 0.0191, - "precision_at_3": 0.06442, - "precision_at_5": 0.04831, - "recall_at_1": 0.13483, - "recall_at_10": 0.30337, - "recall_at_100": 0.63933, - "recall_at_1000": 1.0, - "recall_at_20": 0.38202, - "recall_at_3": 0.19326, - "recall_at_5": 0.24157 - }, - { - "cv_recall_at_1": 0.1407, - "cv_recall_at_10": 0.32209, - "cv_recall_at_100": 0.68605, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.40349, - "cv_recall_at_3": 0.20698, - "cv_recall_at_5": 0.25349, - "hf_subset": "bg", - "languages": [ - "bul-Cyrl" - ], - "main_score": 0.21949, - "map_at_1": 0.1407, - "map_at_10": 0.18806, - "map_at_100": 0.19986, - "map_at_1000": 0.2014, - "map_at_20": 0.19363, - "map_at_3": 0.16841, - "map_at_5": 0.17876, - "mrr_at_1": 0.14069767441860465, - "mrr_at_10": 0.18805878552971583, - "mrr_at_100": 0.199857434973446, - "mrr_at_1000": 0.20139668433598573, - "mrr_at_20": 0.1936337157754288, - "mrr_at_3": 0.16841085271317835, - "mrr_at_5": 0.17875968992248067, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2428187864857724, - "nauc_cv_recall_at_100_max": 0.3504993592742509, - "nauc_cv_recall_at_100_std": 0.37675284258487124, - "nauc_cv_recall_at_10_diff1": 0.34495904167942326, - "nauc_cv_recall_at_10_max": 0.1964802730243885, - "nauc_cv_recall_at_10_std": 0.1529343749951434, - "nauc_cv_recall_at_1_diff1": 0.6347504221281157, - "nauc_cv_recall_at_1_max": 0.29627863650775316, - "nauc_cv_recall_at_1_std": 0.17109276495759043, - "nauc_cv_recall_at_20_diff1": 0.3686049944624201, - "nauc_cv_recall_at_20_max": 0.19272426449099975, - "nauc_cv_recall_at_20_std": 0.14347502244255989, - "nauc_cv_recall_at_3_diff1": 0.46476368983897803, - "nauc_cv_recall_at_3_max": 0.23512666550601383, - "nauc_cv_recall_at_3_std": 0.1646336720217216, - "nauc_cv_recall_at_5_diff1": 0.39303577318150434, - "nauc_cv_recall_at_5_max": 0.19109121634257398, - "nauc_cv_recall_at_5_std": 0.14149124626406387, - "nauc_map_at_1000_diff1": 0.5178979975519488, - "nauc_map_at_1000_max": 0.2520281507799221, - "nauc_map_at_1000_std": 0.16253498933831326, - "nauc_map_at_100_diff1": 0.5168410775781308, - "nauc_map_at_100_max": 0.25182300657032847, - "nauc_map_at_100_std": 0.16266888300130025, - "nauc_map_at_10_diff1": 0.5181276295036846, - "nauc_map_at_10_max": 0.25153894042920016, - "nauc_map_at_10_std": 0.16128937040858818, - "nauc_map_at_1_diff1": 0.6347504221281157, - "nauc_map_at_1_max": 0.29627863650775316, - "nauc_map_at_1_std": 0.17109276495759043, - "nauc_map_at_20_diff1": 0.5191927175733649, - "nauc_map_at_20_max": 0.2502218937847514, - "nauc_map_at_20_std": 0.1597487735456185, - "nauc_map_at_3_diff1": 0.5505877763675288, - "nauc_map_at_3_max": 0.2635034544704076, - "nauc_map_at_3_std": 0.1636388695664378, - "nauc_map_at_5_diff1": 0.5287732974391035, - "nauc_map_at_5_max": 0.25086722661365046, - "nauc_map_at_5_std": 0.15793590077616376, - "nauc_mrr_at_1000_diff1": 0.5178980012528572, - "nauc_mrr_at_1000_max": 0.25202814514475286, - "nauc_mrr_at_1000_std": 0.1625349513052187, - "nauc_mrr_at_100_diff1": 0.5168410775781308, - "nauc_mrr_at_100_max": 0.25182300657032847, - "nauc_mrr_at_100_std": 0.16266888300130025, - "nauc_mrr_at_10_diff1": 0.5181276295036846, - "nauc_mrr_at_10_max": 0.25153894042920016, - "nauc_mrr_at_10_std": 0.16128937040858818, - "nauc_mrr_at_1_diff1": 0.6347504221281157, - "nauc_mrr_at_1_max": 0.29627863650775316, - "nauc_mrr_at_1_std": 0.17109276495759043, - "nauc_mrr_at_20_diff1": 0.5191927175733649, - "nauc_mrr_at_20_max": 0.2502218937847514, - "nauc_mrr_at_20_std": 0.1597487735456185, - "nauc_mrr_at_3_diff1": 0.5505877763675288, - "nauc_mrr_at_3_max": 0.2635034544704076, - "nauc_mrr_at_3_std": 0.1636388695664378, - "nauc_mrr_at_5_diff1": 0.5287732974391035, - "nauc_mrr_at_5_max": 0.25086722661365046, - "nauc_mrr_at_5_std": 0.15793590077616376, - "nauc_ndcg_at_1000_diff1": 0.48472063125969944, - "nauc_ndcg_at_1000_max": 0.24838283580088705, - "nauc_ndcg_at_1000_std": 0.17053299626936394, - "nauc_ndcg_at_100_diff1": 0.4554439428367917, - "nauc_ndcg_at_100_max": 0.25784490286848205, - "nauc_ndcg_at_100_std": 0.1919503537996817, - "nauc_ndcg_at_10_diff1": 0.46695823299989325, - "nauc_ndcg_at_10_max": 0.23497369111423905, - "nauc_ndcg_at_10_std": 0.15893086664948175, - "nauc_ndcg_at_1_diff1": 0.6347504221281157, - "nauc_ndcg_at_1_max": 0.29627863650775316, - "nauc_ndcg_at_1_std": 0.17109276495759043, - "nauc_ndcg_at_20_diff1": 0.4724458069597433, - "nauc_ndcg_at_20_max": 0.2324494583873871, - "nauc_ndcg_at_20_std": 0.15515694071655556, - "nauc_ndcg_at_3_diff1": 0.5260471514772517, - "nauc_ndcg_at_3_max": 0.2552904616025047, - "nauc_ndcg_at_3_std": 0.1637548191784826, - "nauc_ndcg_at_5_diff1": 0.4894596334723677, - "nauc_ndcg_at_5_max": 0.23368023426560022, - "nauc_ndcg_at_5_std": 0.1533233974295215, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.24281878648577213, - "nauc_precision_at_100_max": 0.35049935927425113, - "nauc_precision_at_100_std": 0.37675284258487085, - "nauc_precision_at_10_diff1": 0.3449590416794232, - "nauc_precision_at_10_max": 0.19648027302438859, - "nauc_precision_at_10_std": 0.1529343749951435, - "nauc_precision_at_1_diff1": 0.6347504221281157, - "nauc_precision_at_1_max": 0.29627863650775316, - "nauc_precision_at_1_std": 0.17109276495759043, - "nauc_precision_at_20_diff1": 0.3686049944624202, - "nauc_precision_at_20_max": 0.19272426449099955, - "nauc_precision_at_20_std": 0.14347502244256, - "nauc_precision_at_3_diff1": 0.4647636898389777, - "nauc_precision_at_3_max": 0.23512666550601385, - "nauc_precision_at_3_std": 0.16463367202172133, - "nauc_precision_at_5_diff1": 0.39303577318150446, - "nauc_precision_at_5_max": 0.191091216342574, - "nauc_precision_at_5_std": 0.14149124626406415, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2428187864857724, - "nauc_recall_at_100_max": 0.3504993592742509, - "nauc_recall_at_100_std": 0.37675284258487124, - "nauc_recall_at_10_diff1": 0.34495904167942326, - "nauc_recall_at_10_max": 0.1964802730243885, - "nauc_recall_at_10_std": 0.1529343749951434, - "nauc_recall_at_1_diff1": 0.6347504221281157, - "nauc_recall_at_1_max": 0.29627863650775316, - "nauc_recall_at_1_std": 0.17109276495759043, - "nauc_recall_at_20_diff1": 0.3686049944624201, - "nauc_recall_at_20_max": 0.19272426449099975, - "nauc_recall_at_20_std": 0.14347502244255989, - "nauc_recall_at_3_diff1": 0.46476368983897803, - "nauc_recall_at_3_max": 0.23512666550601383, - "nauc_recall_at_3_std": 0.1646336720217216, - "nauc_recall_at_5_diff1": 0.39303577318150434, - "nauc_recall_at_5_max": 0.19109121634257398, - "nauc_recall_at_5_std": 0.14149124626406387, - "ndcg_at_1": 0.1407, - "ndcg_at_10": 0.21949, - "ndcg_at_100": 0.29031, - "ndcg_at_1000": 0.33046, - "ndcg_at_20": 0.23998, - "ndcg_at_3": 0.17825, - "ndcg_at_5": 0.19716, - "precision_at_1": 0.1407, - "precision_at_10": 0.03221, - "precision_at_100": 0.00686, - "precision_at_1000": 0.001, - "precision_at_20": 0.02017, - "precision_at_3": 0.06899, - "precision_at_5": 0.0507, - "recall_at_1": 0.1407, - "recall_at_10": 0.32209, - "recall_at_100": 0.68605, - "recall_at_1000": 1.0, - "recall_at_20": 0.40349, - "recall_at_3": 0.20698, - "recall_at_5": 0.25349 - }, - { - "cv_recall_at_1": 0.31201, - "cv_recall_at_10": 0.55443, - "cv_recall_at_100": 0.81257, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.63412, - "cv_recall_at_3": 0.42088, - "cv_recall_at_5": 0.47026, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.42241, - "map_at_1": 0.31201, - "map_at_10": 0.38151, - "map_at_100": 0.39138, - "map_at_1000": 0.39224, - "map_at_20": 0.38695, - "map_at_3": 0.35896, - "map_at_5": 0.37024, - "mrr_at_1": 0.31200897867564537, - "mrr_at_10": 0.3815139935510306, - "mrr_at_100": 0.3913751785670525, - "mrr_at_1000": 0.39224160925971996, - "mrr_at_20": 0.38694515136032975, - "mrr_at_3": 0.3589599700710812, - "mrr_at_5": 0.37023943135054227, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.42409783935737194, - "nauc_cv_recall_at_100_max": 0.5681665077524001, - "nauc_cv_recall_at_100_std": 0.5351888788683341, - "nauc_cv_recall_at_10_diff1": 0.4908110669756098, - "nauc_cv_recall_at_10_max": 0.44110626843837974, - "nauc_cv_recall_at_10_std": 0.3307889697670433, - "nauc_cv_recall_at_1_diff1": 0.6603157950562596, - "nauc_cv_recall_at_1_max": 0.4919339948870857, - "nauc_cv_recall_at_1_std": 0.19876708948379945, - "nauc_cv_recall_at_20_diff1": 0.43626796524742434, - "nauc_cv_recall_at_20_max": 0.48145219539697703, - "nauc_cv_recall_at_20_std": 0.41036023299874647, - "nauc_cv_recall_at_3_diff1": 0.5627330715157299, - "nauc_cv_recall_at_3_max": 0.49752444289033215, - "nauc_cv_recall_at_3_std": 0.2957201276468252, - "nauc_cv_recall_at_5_diff1": 0.5377169103841068, - "nauc_cv_recall_at_5_max": 0.461998159762653, - "nauc_cv_recall_at_5_std": 0.2905834636220688, - "nauc_map_at_1000_diff1": 0.6048376948334023, - "nauc_map_at_1000_max": 0.4870026289030281, - "nauc_map_at_1000_std": 0.250945646125177, - "nauc_map_at_100_diff1": 0.6044739666710233, - "nauc_map_at_100_max": 0.487066095883574, - "nauc_map_at_100_std": 0.2513750588492708, - "nauc_map_at_10_diff1": 0.6061716820773362, - "nauc_map_at_10_max": 0.4828330168661941, - "nauc_map_at_10_std": 0.24640874505077667, - "nauc_map_at_1_diff1": 0.6603157950562596, - "nauc_map_at_1_max": 0.4919339948870857, - "nauc_map_at_1_std": 0.19876708948379945, - "nauc_map_at_20_diff1": 0.6033902765092166, - "nauc_map_at_20_max": 0.4856995179729451, - "nauc_map_at_20_std": 0.25081409216717315, - "nauc_map_at_3_diff1": 0.6175165157619867, - "nauc_map_at_3_max": 0.49266653072933536, - "nauc_map_at_3_std": 0.2426696726132534, - "nauc_map_at_5_diff1": 0.6114934391371092, - "nauc_map_at_5_max": 0.48455960871116116, - "nauc_map_at_5_std": 0.24137827487506577, - "nauc_mrr_at_1000_diff1": 0.6048376948334023, - "nauc_mrr_at_1000_max": 0.4870026289030281, - "nauc_mrr_at_1000_std": 0.250945646125177, - "nauc_mrr_at_100_diff1": 0.6044739666710233, - "nauc_mrr_at_100_max": 0.487066095883574, - "nauc_mrr_at_100_std": 0.2513750588492708, - "nauc_mrr_at_10_diff1": 0.6061716820773362, - "nauc_mrr_at_10_max": 0.4828330168661941, - "nauc_mrr_at_10_std": 0.24640874505077667, - "nauc_mrr_at_1_diff1": 0.6603157950562596, - "nauc_mrr_at_1_max": 0.4919339948870857, - "nauc_mrr_at_1_std": 0.19876708948379945, - "nauc_mrr_at_20_diff1": 0.6033902765092166, - "nauc_mrr_at_20_max": 0.4856995179729451, - "nauc_mrr_at_20_std": 0.25081409216717315, - "nauc_mrr_at_3_diff1": 0.6175165157619867, - "nauc_mrr_at_3_max": 0.49266653072933536, - "nauc_mrr_at_3_std": 0.2426696726132534, - "nauc_mrr_at_5_diff1": 0.6114934391371092, - "nauc_mrr_at_5_max": 0.48455960871116116, - "nauc_mrr_at_5_std": 0.24137827487506577, - "nauc_ndcg_at_1000_diff1": 0.5880265966614385, - "nauc_ndcg_at_1000_max": 0.4887610935079035, - "nauc_ndcg_at_1000_std": 0.2700524820212304, - "nauc_ndcg_at_100_diff1": 0.5768554508340397, - "nauc_ndcg_at_100_max": 0.4936025136378576, - "nauc_ndcg_at_100_std": 0.287310533772089, - "nauc_ndcg_at_10_diff1": 0.5794624398102854, - "nauc_ndcg_at_10_max": 0.4738347553313095, - "nauc_ndcg_at_10_std": 0.2662128530488792, - "nauc_ndcg_at_1_diff1": 0.6603157950562596, - "nauc_ndcg_at_1_max": 0.4919339948870857, - "nauc_ndcg_at_1_std": 0.19876708948379945, - "nauc_ndcg_at_20_diff1": 0.569235966407218, - "nauc_ndcg_at_20_max": 0.4838935445595701, - "nauc_ndcg_at_20_std": 0.2825462527831911, - "nauc_ndcg_at_3_diff1": 0.6033020386909346, - "nauc_ndcg_at_3_max": 0.4938559154274072, - "nauc_ndcg_at_3_std": 0.2564915255297522, - "nauc_ndcg_at_5_diff1": 0.5928196609086988, - "nauc_ndcg_at_5_max": 0.4792111070485909, - "nauc_ndcg_at_5_std": 0.25417597546292575, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.4240978393573708, - "nauc_precision_at_100_max": 0.5681665077524004, - "nauc_precision_at_100_std": 0.5351888788683329, - "nauc_precision_at_10_diff1": 0.49081106697560933, - "nauc_precision_at_10_max": 0.44110626843837974, - "nauc_precision_at_10_std": 0.3307889697670433, - "nauc_precision_at_1_diff1": 0.6603157950562596, - "nauc_precision_at_1_max": 0.4919339948870857, - "nauc_precision_at_1_std": 0.19876708948379945, - "nauc_precision_at_20_diff1": 0.43626796524742434, - "nauc_precision_at_20_max": 0.48145219539697637, - "nauc_precision_at_20_std": 0.4103602329987468, - "nauc_precision_at_3_diff1": 0.5627330715157296, - "nauc_precision_at_3_max": 0.49752444289033193, - "nauc_precision_at_3_std": 0.29572012764682554, - "nauc_precision_at_5_diff1": 0.5377169103841068, - "nauc_precision_at_5_max": 0.4619981597626524, - "nauc_precision_at_5_std": 0.29058346362206866, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.42409783935737194, - "nauc_recall_at_100_max": 0.5681665077524001, - "nauc_recall_at_100_std": 0.5351888788683341, - "nauc_recall_at_10_diff1": 0.4908110669756098, - "nauc_recall_at_10_max": 0.44110626843837974, - "nauc_recall_at_10_std": 0.3307889697670433, - "nauc_recall_at_1_diff1": 0.6603157950562596, - "nauc_recall_at_1_max": 0.4919339948870857, - "nauc_recall_at_1_std": 0.19876708948379945, - "nauc_recall_at_20_diff1": 0.43626796524742434, - "nauc_recall_at_20_max": 0.48145219539697703, - "nauc_recall_at_20_std": 0.41036023299874647, - "nauc_recall_at_3_diff1": 0.5627330715157299, - "nauc_recall_at_3_max": 0.49752444289033215, - "nauc_recall_at_3_std": 0.2957201276468252, - "nauc_recall_at_5_diff1": 0.5377169103841068, - "nauc_recall_at_5_max": 0.461998159762653, - "nauc_recall_at_5_std": 0.2905834636220688, - "ndcg_at_1": 0.31201, - "ndcg_at_10": 0.42241, - "ndcg_at_100": 0.47523, - "ndcg_at_1000": 0.49886, - "ndcg_at_20": 0.44243, - "ndcg_at_3": 0.37482, - "ndcg_at_5": 0.39515, - "precision_at_1": 0.31201, - "precision_at_10": 0.05544, - "precision_at_100": 0.00813, - "precision_at_1000": 0.001, - "precision_at_20": 0.03171, - "precision_at_3": 0.14029, - "precision_at_5": 0.09405, - "recall_at_1": 0.31201, - "recall_at_10": 0.55443, - "recall_at_100": 0.81257, - "recall_at_1000": 1.0, - "recall_at_20": 0.63412, - "recall_at_3": 0.42088, - "recall_at_5": 0.47026 - }, - { - "cv_recall_at_1": 0.11404, - "cv_recall_at_10": 0.24211, - "cv_recall_at_100": 0.57193, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.30351, - "cv_recall_at_3": 0.15789, - "cv_recall_at_5": 0.18947, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.1699, - "map_at_1": 0.11404, - "map_at_10": 0.14796, - "map_at_100": 0.15834, - "map_at_1000": 0.16026, - "map_at_20": 0.15224, - "map_at_3": 0.13421, - "map_at_5": 0.14114, - "mrr_at_1": 0.11403508771929824, - "mrr_at_10": 0.1479566972988026, - "mrr_at_100": 0.15834412120057514, - "mrr_at_1000": 0.16025762172047456, - "mrr_at_20": 0.1522396953872024, - "mrr_at_3": 0.13421052631578947, - "mrr_at_5": 0.141140350877193, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.14916308807129272, - "nauc_cv_recall_at_100_max": 0.2508666844965659, - "nauc_cv_recall_at_100_std": 0.41207757015476687, - "nauc_cv_recall_at_10_diff1": 0.2920350970876328, - "nauc_cv_recall_at_10_max": 0.23395789815352871, - "nauc_cv_recall_at_10_std": 0.3874919508096308, - "nauc_cv_recall_at_1_diff1": 0.5665683213093571, - "nauc_cv_recall_at_1_max": 0.35968558199633904, - "nauc_cv_recall_at_1_std": 0.4074641972649942, - "nauc_cv_recall_at_20_diff1": 0.2232536897378315, - "nauc_cv_recall_at_20_max": 0.20621593928881596, - "nauc_cv_recall_at_20_std": 0.35304348649166706, - "nauc_cv_recall_at_3_diff1": 0.4337144801769612, - "nauc_cv_recall_at_3_max": 0.2864454653734898, - "nauc_cv_recall_at_3_std": 0.3876535647439169, - "nauc_cv_recall_at_5_diff1": 0.3801795090109267, - "nauc_cv_recall_at_5_max": 0.23712217965091523, - "nauc_cv_recall_at_5_std": 0.3777168771581285, - "nauc_map_at_1000_diff1": 0.4545928364247393, - "nauc_map_at_1000_max": 0.3060927387596978, - "nauc_map_at_1000_std": 0.39824255335755576, - "nauc_map_at_100_diff1": 0.45302741061897966, - "nauc_map_at_100_max": 0.30567588396362266, - "nauc_map_at_100_std": 0.3983424056494272, - "nauc_map_at_10_diff1": 0.4641548511790888, - "nauc_map_at_10_max": 0.3070952684924584, - "nauc_map_at_10_std": 0.40004023276797424, - "nauc_map_at_1_diff1": 0.5665683213093571, - "nauc_map_at_1_max": 0.35968558199633904, - "nauc_map_at_1_std": 0.4074641972649942, - "nauc_map_at_20_diff1": 0.45506833027410765, - "nauc_map_at_20_max": 0.30377682000528866, - "nauc_map_at_20_std": 0.3965571744014492, - "nauc_map_at_3_diff1": 0.49933312991396894, - "nauc_map_at_3_max": 0.32643242734954164, - "nauc_map_at_3_std": 0.4035398807501951, - "nauc_map_at_5_diff1": 0.4821683291085525, - "nauc_map_at_5_max": 0.3116439870265342, - "nauc_map_at_5_std": 0.40056380152957843, - "nauc_mrr_at_1000_diff1": 0.4545928364247393, - "nauc_mrr_at_1000_max": 0.3060927387596978, - "nauc_mrr_at_1000_std": 0.39824255335755576, - "nauc_mrr_at_100_diff1": 0.45302741061897966, - "nauc_mrr_at_100_max": 0.30567588396362266, - "nauc_mrr_at_100_std": 0.3983424056494272, - "nauc_mrr_at_10_diff1": 0.4641548511790888, - "nauc_mrr_at_10_max": 0.3070952684924584, - "nauc_mrr_at_10_std": 0.40004023276797424, - "nauc_mrr_at_1_diff1": 0.5665683213093571, - "nauc_mrr_at_1_max": 0.35968558199633904, - "nauc_mrr_at_1_std": 0.4074641972649942, - "nauc_mrr_at_20_diff1": 0.45506833027410765, - "nauc_mrr_at_20_max": 0.30377682000528866, - "nauc_mrr_at_20_std": 0.3965571744014492, - "nauc_mrr_at_3_diff1": 0.49933312991396894, - "nauc_mrr_at_3_max": 0.32643242734954164, - "nauc_mrr_at_3_std": 0.4035398807501951, - "nauc_mrr_at_5_diff1": 0.4821683291085525, - "nauc_mrr_at_5_max": 0.3116439870265342, - "nauc_mrr_at_5_std": 0.40056380152957843, - "nauc_ndcg_at_1000_diff1": 0.41198031732074153, - "nauc_ndcg_at_1000_max": 0.292407714742196, - "nauc_ndcg_at_1000_std": 0.39510889733579196, - "nauc_ndcg_at_100_diff1": 0.37090134146384546, - "nauc_ndcg_at_100_max": 0.2852556140761665, - "nauc_ndcg_at_100_std": 0.3977790657553699, - "nauc_ndcg_at_10_diff1": 0.4132626869592185, - "nauc_ndcg_at_10_max": 0.2842838711906126, - "nauc_ndcg_at_10_std": 0.39569785122526163, - "nauc_ndcg_at_1_diff1": 0.5665683213093571, - "nauc_ndcg_at_1_max": 0.35968558199633904, - "nauc_ndcg_at_1_std": 0.4074641972649942, - "nauc_ndcg_at_20_diff1": 0.3855757865062933, - "nauc_ndcg_at_20_max": 0.2734964176350558, - "nauc_ndcg_at_20_std": 0.3839196775789696, - "nauc_ndcg_at_3_diff1": 0.4808969265424267, - "nauc_ndcg_at_3_max": 0.3153556314861584, - "nauc_ndcg_at_3_std": 0.39932922170634866, - "nauc_ndcg_at_5_diff1": 0.4523925584034704, - "nauc_ndcg_at_5_max": 0.290241056262286, - "nauc_ndcg_at_5_std": 0.3941396765756552, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.14916308807129303, - "nauc_precision_at_100_max": 0.25086668449656613, - "nauc_precision_at_100_std": 0.41207757015476687, - "nauc_precision_at_10_diff1": 0.29203509708763287, - "nauc_precision_at_10_max": 0.2339578981535289, - "nauc_precision_at_10_std": 0.3874919508096306, - "nauc_precision_at_1_diff1": 0.5665683213093571, - "nauc_precision_at_1_max": 0.35968558199633904, - "nauc_precision_at_1_std": 0.4074641972649942, - "nauc_precision_at_20_diff1": 0.22325368973783147, - "nauc_precision_at_20_max": 0.20621593928881604, - "nauc_precision_at_20_std": 0.353043486491667, - "nauc_precision_at_3_diff1": 0.433714480176961, - "nauc_precision_at_3_max": 0.28644546537349, - "nauc_precision_at_3_std": 0.3876535647439171, - "nauc_precision_at_5_diff1": 0.3801795090109264, - "nauc_precision_at_5_max": 0.2371221796509154, - "nauc_precision_at_5_std": 0.3777168771581287, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.14916308807129272, - "nauc_recall_at_100_max": 0.2508666844965659, - "nauc_recall_at_100_std": 0.41207757015476687, - "nauc_recall_at_10_diff1": 0.2920350970876328, - "nauc_recall_at_10_max": 0.23395789815352871, - "nauc_recall_at_10_std": 0.3874919508096308, - "nauc_recall_at_1_diff1": 0.5665683213093571, - "nauc_recall_at_1_max": 0.35968558199633904, - "nauc_recall_at_1_std": 0.4074641972649942, - "nauc_recall_at_20_diff1": 0.2232536897378315, - "nauc_recall_at_20_max": 0.20621593928881596, - "nauc_recall_at_20_std": 0.35304348649166706, - "nauc_recall_at_3_diff1": 0.4337144801769612, - "nauc_recall_at_3_max": 0.2864454653734898, - "nauc_recall_at_3_std": 0.3876535647439169, - "nauc_recall_at_5_diff1": 0.3801795090109267, - "nauc_recall_at_5_max": 0.23712217965091523, - "nauc_recall_at_5_std": 0.3777168771581285, - "ndcg_at_1": 0.11404, - "ndcg_at_10": 0.1699, - "ndcg_at_100": 0.23369, - "ndcg_at_1000": 0.28782, - "ndcg_at_20": 0.18545, - "ndcg_at_3": 0.14033, - "ndcg_at_5": 0.15308, - "precision_at_1": 0.11404, - "precision_at_10": 0.02421, - "precision_at_100": 0.00572, - "precision_at_1000": 0.001, - "precision_at_20": 0.01518, - "precision_at_3": 0.05263, - "precision_at_5": 0.03789, - "recall_at_1": 0.11404, - "recall_at_10": 0.24211, - "recall_at_100": 0.57193, - "recall_at_1000": 1.0, - "recall_at_20": 0.30351, - "recall_at_3": 0.15789, - "recall_at_5": 0.18947 - }, - { - "cv_recall_at_1": 0.16705, - "cv_recall_at_10": 0.35126, - "cv_recall_at_100": 0.67735, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.43822, - "cv_recall_at_3": 0.24714, - "cv_recall_at_5": 0.29176, - "hf_subset": "et", - "languages": [ - "est-Latn" - ], - "main_score": 0.25082, - "map_at_1": 0.16705, - "map_at_10": 0.2198, - "map_at_100": 0.23141, - "map_at_1000": 0.23296, - "map_at_20": 0.22571, - "map_at_3": 0.20252, - "map_at_5": 0.21219, - "mrr_at_1": 0.16704805491990846, - "mrr_at_10": 0.21979768261232777, - "mrr_at_100": 0.23140874491520524, - "mrr_at_1000": 0.23295989128115277, - "mrr_at_20": 0.22570981921725977, - "mrr_at_3": 0.20251716247139592, - "mrr_at_5": 0.21218535469107544, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2525428564670692, - "nauc_cv_recall_at_100_max": 0.5346628929514341, - "nauc_cv_recall_at_100_std": 0.47650255481546067, - "nauc_cv_recall_at_10_diff1": 0.38458830102538316, - "nauc_cv_recall_at_10_max": 0.5109280947554741, - "nauc_cv_recall_at_10_std": 0.3745191607765455, - "nauc_cv_recall_at_1_diff1": 0.6638275888245915, - "nauc_cv_recall_at_1_max": 0.5504752903293038, - "nauc_cv_recall_at_1_std": 0.31046701629003715, - "nauc_cv_recall_at_20_diff1": 0.3278440033736192, - "nauc_cv_recall_at_20_max": 0.5165547496354531, - "nauc_cv_recall_at_20_std": 0.39395543291064483, - "nauc_cv_recall_at_3_diff1": 0.479147621248729, - "nauc_cv_recall_at_3_max": 0.5183797446519467, - "nauc_cv_recall_at_3_std": 0.3360203436773196, - "nauc_cv_recall_at_5_diff1": 0.4463467292860443, - "nauc_cv_recall_at_5_max": 0.49931333096174124, - "nauc_cv_recall_at_5_std": 0.3332607535748596, - "nauc_map_at_1000_diff1": 0.5448568065314144, - "nauc_map_at_1000_max": 0.5318758450937885, - "nauc_map_at_1000_std": 0.33283444678560264, - "nauc_map_at_100_diff1": 0.5437588464406067, - "nauc_map_at_100_max": 0.5318250750358087, - "nauc_map_at_100_std": 0.33338742536532645, - "nauc_map_at_10_diff1": 0.5492756780946725, - "nauc_map_at_10_max": 0.5311112107540561, - "nauc_map_at_10_std": 0.3299222777160743, - "nauc_map_at_1_diff1": 0.6638275888245915, - "nauc_map_at_1_max": 0.5504752903293038, - "nauc_map_at_1_std": 0.31046701629003715, - "nauc_map_at_20_diff1": 0.544610032644422, - "nauc_map_at_20_max": 0.5311375624808914, - "nauc_map_at_20_std": 0.3313447833782751, - "nauc_map_at_3_diff1": 0.5707234130723992, - "nauc_map_at_3_max": 0.5346313018781644, - "nauc_map_at_3_std": 0.324016696954914, - "nauc_map_at_5_diff1": 0.5594957337059822, - "nauc_map_at_5_max": 0.5284747680149817, - "nauc_map_at_5_std": 0.32302330446027244, - "nauc_mrr_at_1000_diff1": 0.5448568065314144, - "nauc_mrr_at_1000_max": 0.5318758450937885, - "nauc_mrr_at_1000_std": 0.33283444678560264, - "nauc_mrr_at_100_diff1": 0.5437588464406067, - "nauc_mrr_at_100_max": 0.5318250750358087, - "nauc_mrr_at_100_std": 0.33338742536532645, - "nauc_mrr_at_10_diff1": 0.5492756780946725, - "nauc_mrr_at_10_max": 0.5311112107540561, - "nauc_mrr_at_10_std": 0.3299222777160743, - "nauc_mrr_at_1_diff1": 0.6638275888245915, - "nauc_mrr_at_1_max": 0.5504752903293038, - "nauc_mrr_at_1_std": 0.31046701629003715, - "nauc_mrr_at_20_diff1": 0.544610032644422, - "nauc_mrr_at_20_max": 0.5311375624808914, - "nauc_mrr_at_20_std": 0.3313447833782751, - "nauc_mrr_at_3_diff1": 0.5707234130723992, - "nauc_mrr_at_3_max": 0.5346313018781644, - "nauc_mrr_at_3_std": 0.324016696954914, - "nauc_mrr_at_5_diff1": 0.5594957337059822, - "nauc_mrr_at_5_max": 0.5284747680149817, - "nauc_mrr_at_5_std": 0.32302330446027244, - "nauc_ndcg_at_1000_diff1": 0.5106477734657984, - "nauc_ndcg_at_1000_max": 0.5301338839155268, - "nauc_ndcg_at_1000_std": 0.3447217097906694, - "nauc_ndcg_at_100_diff1": 0.48090999547407126, - "nauc_ndcg_at_100_max": 0.530346752198247, - "nauc_ndcg_at_100_std": 0.3600192855287342, - "nauc_ndcg_at_10_diff1": 0.5031977703710548, - "nauc_ndcg_at_10_max": 0.5251715595267603, - "nauc_ndcg_at_10_std": 0.3417303671545655, - "nauc_ndcg_at_1_diff1": 0.6638275888245915, - "nauc_ndcg_at_1_max": 0.5504752903293038, - "nauc_ndcg_at_1_std": 0.31046701629003715, - "nauc_ndcg_at_20_diff1": 0.4864847598490328, - "nauc_ndcg_at_20_max": 0.5262779001040049, - "nauc_ndcg_at_20_std": 0.34723586519080096, - "nauc_ndcg_at_3_diff1": 0.5450553817498409, - "nauc_ndcg_at_3_max": 0.5300900562100276, - "nauc_ndcg_at_3_std": 0.3274088836871772, - "nauc_ndcg_at_5_diff1": 0.5270163829776572, - "nauc_ndcg_at_5_max": 0.5202203374368238, - "nauc_ndcg_at_5_std": 0.32601310394670324, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.2525428564670695, - "nauc_precision_at_100_max": 0.5346628929514347, - "nauc_precision_at_100_std": 0.4765025548154612, - "nauc_precision_at_10_diff1": 0.384588301025383, - "nauc_precision_at_10_max": 0.5109280947554742, - "nauc_precision_at_10_std": 0.37451916077654557, - "nauc_precision_at_1_diff1": 0.6638275888245915, - "nauc_precision_at_1_max": 0.5504752903293038, - "nauc_precision_at_1_std": 0.31046701629003715, - "nauc_precision_at_20_diff1": 0.32784400337361863, - "nauc_precision_at_20_max": 0.5165547496354526, - "nauc_precision_at_20_std": 0.3939554329106448, - "nauc_precision_at_3_diff1": 0.4791476212487288, - "nauc_precision_at_3_max": 0.5183797446519465, - "nauc_precision_at_3_std": 0.33602034367731953, - "nauc_precision_at_5_diff1": 0.4463467292860441, - "nauc_precision_at_5_max": 0.4993133309617412, - "nauc_precision_at_5_std": 0.33326075357485957, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2525428564670692, - "nauc_recall_at_100_max": 0.5346628929514341, - "nauc_recall_at_100_std": 0.47650255481546067, - "nauc_recall_at_10_diff1": 0.38458830102538316, - "nauc_recall_at_10_max": 0.5109280947554741, - "nauc_recall_at_10_std": 0.3745191607765455, - "nauc_recall_at_1_diff1": 0.6638275888245915, - "nauc_recall_at_1_max": 0.5504752903293038, - "nauc_recall_at_1_std": 0.31046701629003715, - "nauc_recall_at_20_diff1": 0.3278440033736192, - "nauc_recall_at_20_max": 0.5165547496354531, - "nauc_recall_at_20_std": 0.39395543291064483, - "nauc_recall_at_3_diff1": 0.479147621248729, - "nauc_recall_at_3_max": 0.5183797446519467, - "nauc_recall_at_3_std": 0.3360203436773196, - "nauc_recall_at_5_diff1": 0.4463467292860443, - "nauc_recall_at_5_max": 0.49931333096174124, - "nauc_recall_at_5_std": 0.3332607535748596, - "ndcg_at_1": 0.16705, - "ndcg_at_10": 0.25082, - "ndcg_at_100": 0.31616, - "ndcg_at_1000": 0.35742, - "ndcg_at_20": 0.27263, - "ndcg_at_3": 0.21398, - "ndcg_at_5": 0.2319, - "precision_at_1": 0.16705, - "precision_at_10": 0.03513, - "precision_at_100": 0.00677, - "precision_at_1000": 0.001, - "precision_at_20": 0.02191, - "precision_at_3": 0.08238, - "precision_at_5": 0.05835, - "recall_at_1": 0.16705, - "recall_at_10": 0.35126, - "recall_at_100": 0.67735, - "recall_at_1000": 1.0, - "recall_at_20": 0.43822, - "recall_at_3": 0.24714, - "recall_at_5": 0.29176 - }, - { - "cv_recall_at_1": 0.36515, - "cv_recall_at_10": 0.65039, - "cv_recall_at_100": 0.89789, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.73585, - "cv_recall_at_3": 0.51165, - "cv_recall_at_5": 0.57381, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.50016, - "map_at_1": 0.36515, - "map_at_10": 0.45294, - "map_at_100": 0.46318, - "map_at_1000": 0.46369, - "map_at_20": 0.45889, - "map_at_3": 0.4286, - "map_at_5": 0.44286, - "mrr_at_1": 0.365149833518313, - "mrr_at_10": 0.4529411764705879, - "mrr_at_100": 0.463181284099606, - "mrr_at_1000": 0.4636910375275689, - "mrr_at_20": 0.4588889551156271, - "mrr_at_3": 0.4285978542360337, - "mrr_at_5": 0.4428597854236029, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.3519632757367839, - "nauc_cv_recall_at_100_max": 0.5896633539169532, - "nauc_cv_recall_at_100_std": 0.6279061759070669, - "nauc_cv_recall_at_10_diff1": 0.49598786888905616, - "nauc_cv_recall_at_10_max": 0.41815541361430786, - "nauc_cv_recall_at_10_std": 0.35841020167020005, - "nauc_cv_recall_at_1_diff1": 0.6832526806736251, - "nauc_cv_recall_at_1_max": 0.38716079008324905, - "nauc_cv_recall_at_1_std": 0.19616342687021568, - "nauc_cv_recall_at_20_diff1": 0.4103941158660433, - "nauc_cv_recall_at_20_max": 0.43701975747528027, - "nauc_cv_recall_at_20_std": 0.4431938868593069, - "nauc_cv_recall_at_3_diff1": 0.5678328869179713, - "nauc_cv_recall_at_3_max": 0.40246002367987804, - "nauc_cv_recall_at_3_std": 0.2518196988828705, - "nauc_cv_recall_at_5_diff1": 0.529681288168825, - "nauc_cv_recall_at_5_max": 0.39142002976373985, - "nauc_cv_recall_at_5_std": 0.2885002963574131, - "nauc_map_at_1000_diff1": 0.6207013621900259, - "nauc_map_at_1000_max": 0.3989618223778303, - "nauc_map_at_1000_std": 0.23971198686244546, - "nauc_map_at_100_diff1": 0.6202909508328959, - "nauc_map_at_100_max": 0.39917405505970993, - "nauc_map_at_100_std": 0.24020801896301075, - "nauc_map_at_10_diff1": 0.6220116794480105, - "nauc_map_at_10_max": 0.3970448836274126, - "nauc_map_at_10_std": 0.2374657253002716, - "nauc_map_at_1_diff1": 0.6832526806736251, - "nauc_map_at_1_max": 0.38716079008324905, - "nauc_map_at_1_std": 0.19616342687021568, - "nauc_map_at_20_diff1": 0.6191197977242113, - "nauc_map_at_20_max": 0.39772268435868907, - "nauc_map_at_20_std": 0.24056952293607264, - "nauc_map_at_3_diff1": 0.6321019056808161, - "nauc_map_at_3_max": 0.3968853497625998, - "nauc_map_at_3_std": 0.22306202122311464, - "nauc_map_at_5_diff1": 0.6249797342785457, - "nauc_map_at_5_max": 0.39455312160663836, - "nauc_map_at_5_std": 0.2307452054244085, - "nauc_mrr_at_1000_diff1": 0.6207013621900259, - "nauc_mrr_at_1000_max": 0.3989618223778303, - "nauc_mrr_at_1000_std": 0.23971198686244546, - "nauc_mrr_at_100_diff1": 0.6202909508328959, - "nauc_mrr_at_100_max": 0.39917405505970993, - "nauc_mrr_at_100_std": 0.24020801896301075, - "nauc_mrr_at_10_diff1": 0.6220116794480105, - "nauc_mrr_at_10_max": 0.3970448836274126, - "nauc_mrr_at_10_std": 0.2374657253002716, - "nauc_mrr_at_1_diff1": 0.6832526806736251, - "nauc_mrr_at_1_max": 0.38716079008324905, - "nauc_mrr_at_1_std": 0.19616342687021568, - "nauc_mrr_at_20_diff1": 0.6191197977242113, - "nauc_mrr_at_20_max": 0.39772268435868907, - "nauc_mrr_at_20_std": 0.24056952293607264, - "nauc_mrr_at_3_diff1": 0.6321019056808161, - "nauc_mrr_at_3_max": 0.3968853497625998, - "nauc_mrr_at_3_std": 0.22306202122311464, - "nauc_mrr_at_5_diff1": 0.6249797342785457, - "nauc_mrr_at_5_max": 0.39455312160663836, - "nauc_mrr_at_5_std": 0.2307452054244085, - "nauc_ndcg_at_1000_diff1": 0.6038061339373723, - "nauc_ndcg_at_1000_max": 0.40478501436504905, - "nauc_ndcg_at_1000_std": 0.25668452905230366, - "nauc_ndcg_at_100_diff1": 0.592446378602503, - "nauc_ndcg_at_100_max": 0.41265629017662886, - "nauc_ndcg_at_100_std": 0.2728434617850929, - "nauc_ndcg_at_10_diff1": 0.5948353501805591, - "nauc_ndcg_at_10_max": 0.40113867845503637, - "nauc_ndcg_at_10_std": 0.26223182571918424, - "nauc_ndcg_at_1_diff1": 0.6832526806736251, - "nauc_ndcg_at_1_max": 0.38716079008324905, - "nauc_ndcg_at_1_std": 0.19616342687021568, - "nauc_ndcg_at_20_diff1": 0.5828639375379747, - "nauc_ndcg_at_20_max": 0.40394839922498804, - "nauc_ndcg_at_20_std": 0.274426191371496, - "nauc_ndcg_at_3_diff1": 0.6157909448194059, - "nauc_ndcg_at_3_max": 0.39842749233436653, - "nauc_ndcg_at_3_std": 0.23045707287814626, - "nauc_ndcg_at_5_diff1": 0.6022492170015042, - "nauc_ndcg_at_5_max": 0.39410237251204805, - "nauc_ndcg_at_5_std": 0.24445390105709308, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3519632757367811, - "nauc_precision_at_100_max": 0.5896633539169541, - "nauc_precision_at_100_std": 0.627906175907067, - "nauc_precision_at_10_diff1": 0.4959878688890556, - "nauc_precision_at_10_max": 0.418155413614308, - "nauc_precision_at_10_std": 0.3584102016701997, - "nauc_precision_at_1_diff1": 0.6832526806736251, - "nauc_precision_at_1_max": 0.38716079008324905, - "nauc_precision_at_1_std": 0.19616342687021568, - "nauc_precision_at_20_diff1": 0.4103941158660434, - "nauc_precision_at_20_max": 0.43701975747528016, - "nauc_precision_at_20_std": 0.4431938868593058, - "nauc_precision_at_3_diff1": 0.5678328869179718, - "nauc_precision_at_3_max": 0.4024600236798784, - "nauc_precision_at_3_std": 0.2518196988828707, - "nauc_precision_at_5_diff1": 0.5296812881688243, - "nauc_precision_at_5_max": 0.3914200297637398, - "nauc_precision_at_5_std": 0.288500296357413, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.3519632757367839, - "nauc_recall_at_100_max": 0.5896633539169532, - "nauc_recall_at_100_std": 0.6279061759070669, - "nauc_recall_at_10_diff1": 0.49598786888905616, - "nauc_recall_at_10_max": 0.41815541361430786, - "nauc_recall_at_10_std": 0.35841020167020005, - "nauc_recall_at_1_diff1": 0.6832526806736251, - "nauc_recall_at_1_max": 0.38716079008324905, - "nauc_recall_at_1_std": 0.19616342687021568, - "nauc_recall_at_20_diff1": 0.4103941158660433, - "nauc_recall_at_20_max": 0.43701975747528027, - "nauc_recall_at_20_std": 0.4431938868593069, - "nauc_recall_at_3_diff1": 0.5678328869179713, - "nauc_recall_at_3_max": 0.40246002367987804, - "nauc_recall_at_3_std": 0.2518196988828705, - "nauc_recall_at_5_diff1": 0.529681288168825, - "nauc_recall_at_5_max": 0.39142002976373985, - "nauc_recall_at_5_std": 0.2885002963574131, - "ndcg_at_1": 0.36515, - "ndcg_at_10": 0.50016, - "ndcg_at_100": 0.55211, - "ndcg_at_1000": 0.56527, - "ndcg_at_20": 0.52179, - "ndcg_at_3": 0.44988, - "ndcg_at_5": 0.47553, - "precision_at_1": 0.36515, - "precision_at_10": 0.06504, - "precision_at_100": 0.00898, - "precision_at_1000": 0.001, - "precision_at_20": 0.03679, - "precision_at_3": 0.17055, - "precision_at_5": 0.11476, - "recall_at_1": 0.36515, - "recall_at_10": 0.65039, - "recall_at_100": 0.89789, - "recall_at_1000": 1.0, - "recall_at_20": 0.73585, - "recall_at_3": 0.51165, - "recall_at_5": 0.57381 - }, - { - "cv_recall_at_1": 0.09023, - "cv_recall_at_10": 0.20086, - "cv_recall_at_100": 0.45542, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.25564, - "cv_recall_at_3": 0.12675, - "cv_recall_at_5": 0.15252, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.13653, - "map_at_1": 0.09023, - "map_at_10": 0.11714, - "map_at_100": 0.12554, - "map_at_1000": 0.12765, - "map_at_20": 0.12095, - "map_at_3": 0.10526, - "map_at_5": 0.11106, - "mrr_at_1": 0.09022556390977443, - "mrr_at_10": 0.11714447683835436, - "mrr_at_100": 0.12554517947850538, - "mrr_at_1000": 0.12765268537614335, - "mrr_at_20": 0.1209542322873805, - "mrr_at_3": 0.10526315789473682, - "mrr_at_5": 0.11106337271750806, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2332453798292854, - "nauc_cv_recall_at_100_max": 0.2604533475265826, - "nauc_cv_recall_at_100_std": 0.34716025012947915, - "nauc_cv_recall_at_10_diff1": 0.35329370842952273, - "nauc_cv_recall_at_10_max": 0.3289408734013955, - "nauc_cv_recall_at_10_std": 0.38988109866697873, - "nauc_cv_recall_at_1_diff1": 0.6484509307804875, - "nauc_cv_recall_at_1_max": 0.4865030100451507, - "nauc_cv_recall_at_1_std": 0.42080120745590044, - "nauc_cv_recall_at_20_diff1": 0.3375684040590422, - "nauc_cv_recall_at_20_max": 0.3364013794750651, - "nauc_cv_recall_at_20_std": 0.3763071751508761, - "nauc_cv_recall_at_3_diff1": 0.5356708174885437, - "nauc_cv_recall_at_3_max": 0.3783391728916529, - "nauc_cv_recall_at_3_std": 0.3777211044732277, - "nauc_cv_recall_at_5_diff1": 0.47021139762415187, - "nauc_cv_recall_at_5_max": 0.3420074129863515, - "nauc_cv_recall_at_5_std": 0.36368502197235486, - "nauc_map_at_1000_diff1": 0.5397229127932685, - "nauc_map_at_1000_max": 0.414317285718928, - "nauc_map_at_1000_std": 0.39325018888031077, - "nauc_map_at_100_diff1": 0.5382353593792347, - "nauc_map_at_100_max": 0.41361154226841035, - "nauc_map_at_100_std": 0.39287405926700186, - "nauc_map_at_10_diff1": 0.5494054617283636, - "nauc_map_at_10_max": 0.4169379207559909, - "nauc_map_at_10_std": 0.3960601530511953, - "nauc_map_at_1_diff1": 0.6484509307804875, - "nauc_map_at_1_max": 0.4865030100451507, - "nauc_map_at_1_std": 0.42080120745590044, - "nauc_map_at_20_diff1": 0.5453892496407874, - "nauc_map_at_20_max": 0.41732748869963604, - "nauc_map_at_20_std": 0.39426197338787017, - "nauc_map_at_3_diff1": 0.5951395368065661, - "nauc_map_at_3_max": 0.4360903241739117, - "nauc_map_at_3_std": 0.3991134950739887, - "nauc_map_at_5_diff1": 0.5750651443076629, - "nauc_map_at_5_max": 0.42288294013194566, - "nauc_map_at_5_std": 0.39387458453693197, - "nauc_mrr_at_1000_diff1": 0.5397252229441754, - "nauc_mrr_at_1000_max": 0.4143152370673888, - "nauc_mrr_at_1000_std": 0.39325041574122255, - "nauc_mrr_at_100_diff1": 0.5382376576035895, - "nauc_mrr_at_100_max": 0.4136095049464029, - "nauc_mrr_at_100_std": 0.39287428504225475, - "nauc_mrr_at_10_diff1": 0.5494054617283636, - "nauc_mrr_at_10_max": 0.4169379207559909, - "nauc_mrr_at_10_std": 0.3960601530511953, - "nauc_mrr_at_1_diff1": 0.6484509307804875, - "nauc_mrr_at_1_max": 0.4865030100451507, - "nauc_mrr_at_1_std": 0.42080120745590044, - "nauc_mrr_at_20_diff1": 0.5453892496407874, - "nauc_mrr_at_20_max": 0.41732748869963604, - "nauc_mrr_at_20_std": 0.39426197338787017, - "nauc_mrr_at_3_diff1": 0.5951395368065661, - "nauc_mrr_at_3_max": 0.4360903241739117, - "nauc_mrr_at_3_std": 0.3991134950739887, - "nauc_mrr_at_5_diff1": 0.5750651443076629, - "nauc_mrr_at_5_max": 0.42288294013194566, - "nauc_mrr_at_5_std": 0.39387458453693197, - "nauc_ndcg_at_1000_diff1": 0.4877079024580914, - "nauc_ndcg_at_1000_max": 0.3894770176532519, - "nauc_ndcg_at_1000_std": 0.3866618739044846, - "nauc_ndcg_at_100_diff1": 0.4421249294826701, - "nauc_ndcg_at_100_max": 0.3665441486565287, - "nauc_ndcg_at_100_std": 0.3789855079726725, - "nauc_ndcg_at_10_diff1": 0.48884465811326916, - "nauc_ndcg_at_10_max": 0.38826077672968173, - "nauc_ndcg_at_10_std": 0.39280940785655283, - "nauc_ndcg_at_1_diff1": 0.6484509307804875, - "nauc_ndcg_at_1_max": 0.4865030100451507, - "nauc_ndcg_at_1_std": 0.42080120745590044, - "nauc_ndcg_at_20_diff1": 0.47829172426006694, - "nauc_ndcg_at_20_max": 0.3894391246306693, - "nauc_ndcg_at_20_std": 0.3875491838687217, - "nauc_ndcg_at_3_diff1": 0.5783208835318133, - "nauc_ndcg_at_3_max": 0.41978489706071076, - "nauc_ndcg_at_3_std": 0.39301265794434825, - "nauc_ndcg_at_5_diff1": 0.5444484907251043, - "nauc_ndcg_at_5_max": 0.3989289367797176, - "nauc_ndcg_at_5_std": 0.38486046761887477, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.23324537982928592, - "nauc_precision_at_100_max": 0.260453347526583, - "nauc_precision_at_100_std": 0.3471602501294796, - "nauc_precision_at_10_diff1": 0.3532937084295228, - "nauc_precision_at_10_max": 0.3289408734013955, - "nauc_precision_at_10_std": 0.3898810986669788, - "nauc_precision_at_1_diff1": 0.6484509307804875, - "nauc_precision_at_1_max": 0.4865030100451507, - "nauc_precision_at_1_std": 0.42080120745590044, - "nauc_precision_at_20_diff1": 0.33756840405904204, - "nauc_precision_at_20_max": 0.33640137947506477, - "nauc_precision_at_20_std": 0.3763071751508759, - "nauc_precision_at_3_diff1": 0.5356708174885437, - "nauc_precision_at_3_max": 0.37833917289165303, - "nauc_precision_at_3_std": 0.37772110447322776, - "nauc_precision_at_5_diff1": 0.4702113976241517, - "nauc_precision_at_5_max": 0.34200741298635134, - "nauc_precision_at_5_std": 0.36368502197235497, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2332453798292854, - "nauc_recall_at_100_max": 0.2604533475265826, - "nauc_recall_at_100_std": 0.34716025012947915, - "nauc_recall_at_10_diff1": 0.35329370842952273, - "nauc_recall_at_10_max": 0.3289408734013955, - "nauc_recall_at_10_std": 0.38988109866697873, - "nauc_recall_at_1_diff1": 0.6484509307804875, - "nauc_recall_at_1_max": 0.4865030100451507, - "nauc_recall_at_1_std": 0.42080120745590044, - "nauc_recall_at_20_diff1": 0.3375684040590422, - "nauc_recall_at_20_max": 0.3364013794750651, - "nauc_recall_at_20_std": 0.3763071751508761, - "nauc_recall_at_3_diff1": 0.5356708174885437, - "nauc_recall_at_3_max": 0.3783391728916529, - "nauc_recall_at_3_std": 0.3777211044732277, - "nauc_recall_at_5_diff1": 0.47021139762415187, - "nauc_recall_at_5_max": 0.3420074129863515, - "nauc_recall_at_5_std": 0.36368502197235486, - "ndcg_at_1": 0.09023, - "ndcg_at_10": 0.13653, - "ndcg_at_100": 0.18638, - "ndcg_at_1000": 0.25295, - "ndcg_at_20": 0.1504, - "ndcg_at_3": 0.11074, - "ndcg_at_5": 0.12127, - "precision_at_1": 0.09023, - "precision_at_10": 0.02009, - "precision_at_100": 0.00455, - "precision_at_1000": 0.001, - "precision_at_20": 0.01278, - "precision_at_3": 0.04225, - "precision_at_5": 0.0305, - "recall_at_1": 0.09023, - "recall_at_10": 0.20086, - "recall_at_100": 0.45542, - "recall_at_1000": 1.0, - "recall_at_20": 0.25564, - "recall_at_3": 0.12675, - "recall_at_5": 0.15252 - }, - { - "cv_recall_at_1": 0.09, - "cv_recall_at_10": 0.241, - "cv_recall_at_100": 0.572, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.298, - "cv_recall_at_3": 0.142, - "cv_recall_at_5": 0.184, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.15616, - "map_at_1": 0.09, - "map_at_10": 0.13016, - "map_at_100": 0.14012, - "map_at_1000": 0.14196, - "map_at_20": 0.1341, - "map_at_3": 0.11267, - "map_at_5": 0.12262, - "mrr_at_1": 0.09, - "mrr_at_10": 0.130156746031746, - "mrr_at_100": 0.14011933620958256, - "mrr_at_1000": 0.14195799808904563, - "mrr_at_20": 0.1341021033447118, - "mrr_at_3": 0.11266666666666662, - "mrr_at_5": 0.12261666666666665, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.09950724604329818, - "nauc_cv_recall_at_100_max": 0.19202302395020154, - "nauc_cv_recall_at_100_std": 0.29821107013617665, - "nauc_cv_recall_at_10_diff1": 0.2609484113161116, - "nauc_cv_recall_at_10_max": 0.04742804090975, - "nauc_cv_recall_at_10_std": 0.16593983737163379, - "nauc_cv_recall_at_1_diff1": 0.519786542646752, - "nauc_cv_recall_at_1_max": 0.06741794115016544, - "nauc_cv_recall_at_1_std": 0.1665653042363534, - "nauc_cv_recall_at_20_diff1": 0.2257939014702732, - "nauc_cv_recall_at_20_max": 0.06763853455264178, - "nauc_cv_recall_at_20_std": 0.18127495980674707, - "nauc_cv_recall_at_3_diff1": 0.43499246295510946, - "nauc_cv_recall_at_3_max": 0.01803852080770247, - "nauc_cv_recall_at_3_std": 0.23359590522218548, - "nauc_cv_recall_at_5_diff1": 0.3353192120401153, - "nauc_cv_recall_at_5_max": 0.021990660909630255, - "nauc_cv_recall_at_5_std": 0.1779186381757737, - "nauc_map_at_1000_diff1": 0.4137939221823387, - "nauc_map_at_1000_max": 0.04585110095108629, - "nauc_map_at_1000_std": 0.1905566893465737, - "nauc_map_at_100_diff1": 0.41218399729561006, - "nauc_map_at_100_max": 0.04606128131195546, - "nauc_map_at_100_std": 0.1902874373233265, - "nauc_map_at_10_diff1": 0.42234901698531985, - "nauc_map_at_10_max": 0.040903992320474654, - "nauc_map_at_10_std": 0.18605702839308597, - "nauc_map_at_1_diff1": 0.519786542646752, - "nauc_map_at_1_max": 0.06741794115016544, - "nauc_map_at_1_std": 0.1665653042363534, - "nauc_map_at_20_diff1": 0.41690209448121784, - "nauc_map_at_20_max": 0.042671874195198585, - "nauc_map_at_20_std": 0.1869251620123988, - "nauc_map_at_3_diff1": 0.47577635534215285, - "nauc_map_at_3_max": 0.03642620873031971, - "nauc_map_at_3_std": 0.20504951607197658, - "nauc_map_at_5_diff1": 0.44141461540766325, - "nauc_map_at_5_max": 0.03624088835029317, - "nauc_map_at_5_std": 0.1888593111268919, - "nauc_mrr_at_1000_diff1": 0.4137939221823387, - "nauc_mrr_at_1000_max": 0.04585110095108629, - "nauc_mrr_at_1000_std": 0.1905566893465737, - "nauc_mrr_at_100_diff1": 0.41218399729561006, - "nauc_mrr_at_100_max": 0.04606128131195546, - "nauc_mrr_at_100_std": 0.1902874373233265, - "nauc_mrr_at_10_diff1": 0.42234901698531985, - "nauc_mrr_at_10_max": 0.040903992320474654, - "nauc_mrr_at_10_std": 0.18605702839308597, - "nauc_mrr_at_1_diff1": 0.519786542646752, - "nauc_mrr_at_1_max": 0.06741794115016544, - "nauc_mrr_at_1_std": 0.1665653042363534, - "nauc_mrr_at_20_diff1": 0.41690209448121784, - "nauc_mrr_at_20_max": 0.042671874195198585, - "nauc_mrr_at_20_std": 0.1869251620123988, - "nauc_mrr_at_3_diff1": 0.47577635534215285, - "nauc_mrr_at_3_max": 0.03642620873031971, - "nauc_mrr_at_3_std": 0.20504951607197658, - "nauc_mrr_at_5_diff1": 0.44141461540766325, - "nauc_mrr_at_5_max": 0.03624088835029317, - "nauc_mrr_at_5_std": 0.1888593111268919, - "nauc_ndcg_at_1000_diff1": 0.36875630031096535, - "nauc_ndcg_at_1000_max": 0.0586483601817311, - "nauc_ndcg_at_1000_std": 0.2007283154206796, - "nauc_ndcg_at_100_diff1": 0.3245616830412682, - "nauc_ndcg_at_100_max": 0.07781147437348998, - "nauc_ndcg_at_100_std": 0.21267434981201533, - "nauc_ndcg_at_10_diff1": 0.37109903302118974, - "nauc_ndcg_at_10_max": 0.04192896271586187, - "nauc_ndcg_at_10_std": 0.18049054255252522, - "nauc_ndcg_at_1_diff1": 0.519786542646752, - "nauc_ndcg_at_1_max": 0.06741794115016544, - "nauc_ndcg_at_1_std": 0.1665653042363534, - "nauc_ndcg_at_20_diff1": 0.3548551379281221, - "nauc_ndcg_at_20_max": 0.04882258202592032, - "nauc_ndcg_at_20_std": 0.18487017063013111, - "nauc_ndcg_at_3_diff1": 0.46386739885940526, - "nauc_ndcg_at_3_max": 0.030844689867467245, - "nauc_ndcg_at_3_std": 0.21353132426323734, - "nauc_ndcg_at_5_diff1": 0.4089763993837865, - "nauc_ndcg_at_5_max": 0.03139894253778136, - "nauc_ndcg_at_5_std": 0.1862933242082388, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.09950724604329804, - "nauc_precision_at_100_max": 0.1920230239502011, - "nauc_precision_at_100_std": 0.29821107013617626, - "nauc_precision_at_10_diff1": 0.26094841131611163, - "nauc_precision_at_10_max": 0.047428040909750106, - "nauc_precision_at_10_std": 0.16593983737163379, - "nauc_precision_at_1_diff1": 0.519786542646752, - "nauc_precision_at_1_max": 0.06741794115016544, - "nauc_precision_at_1_std": 0.1665653042363534, - "nauc_precision_at_20_diff1": 0.22579390147027317, - "nauc_precision_at_20_max": 0.06763853455264163, - "nauc_precision_at_20_std": 0.18127495980674682, - "nauc_precision_at_3_diff1": 0.43499246295510957, - "nauc_precision_at_3_max": 0.01803852080770258, - "nauc_precision_at_3_std": 0.23359590522218557, - "nauc_precision_at_5_diff1": 0.33531921204011533, - "nauc_precision_at_5_max": 0.021990660909630016, - "nauc_precision_at_5_std": 0.17791863817577347, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.09950724604329818, - "nauc_recall_at_100_max": 0.19202302395020154, - "nauc_recall_at_100_std": 0.29821107013617665, - "nauc_recall_at_10_diff1": 0.2609484113161116, - "nauc_recall_at_10_max": 0.04742804090975, - "nauc_recall_at_10_std": 0.16593983737163379, - "nauc_recall_at_1_diff1": 0.519786542646752, - "nauc_recall_at_1_max": 0.06741794115016544, - "nauc_recall_at_1_std": 0.1665653042363534, - "nauc_recall_at_20_diff1": 0.2257939014702732, - "nauc_recall_at_20_max": 0.06763853455264178, - "nauc_recall_at_20_std": 0.18127495980674707, - "nauc_recall_at_3_diff1": 0.43499246295510946, - "nauc_recall_at_3_max": 0.01803852080770247, - "nauc_recall_at_3_std": 0.23359590522218548, - "nauc_recall_at_5_diff1": 0.3353192120401153, - "nauc_recall_at_5_max": 0.021990660909630255, - "nauc_recall_at_5_std": 0.1779186381757737, - "ndcg_at_1": 0.09, - "ndcg_at_10": 0.15616, - "ndcg_at_100": 0.21938, - "ndcg_at_1000": 0.27284, - "ndcg_at_20": 0.17056, - "ndcg_at_3": 0.12019, - "ndcg_at_5": 0.1378, - "precision_at_1": 0.09, - "precision_at_10": 0.0241, - "precision_at_100": 0.00572, - "precision_at_1000": 0.001, - "precision_at_20": 0.0149, - "precision_at_3": 0.04733, - "precision_at_5": 0.0368, - "recall_at_1": 0.09, - "recall_at_10": 0.241, - "recall_at_100": 0.572, - "recall_at_1000": 1.0, - "recall_at_20": 0.298, - "recall_at_3": 0.142, - "recall_at_5": 0.184 - }, - { - "cv_recall_at_1": 0.30513, - "cv_recall_at_10": 0.55895, - "cv_recall_at_100": 0.84605, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.63662, - "cv_recall_at_3": 0.43689, - "cv_recall_at_5": 0.50208, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.42702, - "map_at_1": 0.30513, - "map_at_10": 0.38526, - "map_at_100": 0.3958, - "map_at_1000": 0.39659, - "map_at_20": 0.39083, - "map_at_3": 0.36269, - "map_at_5": 0.37774, - "mrr_at_1": 0.30513176144244103, - "mrr_at_10": 0.38525691830130115, - "mrr_at_100": 0.39579860906812514, - "mrr_at_1000": 0.3965896152678169, - "mrr_at_20": 0.3908313698594654, - "mrr_at_3": 0.3626907073509017, - "mrr_at_5": 0.3777392510402221, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.41165495427230214, - "nauc_cv_recall_at_100_max": 0.48206221054316895, - "nauc_cv_recall_at_100_std": 0.6198649450287124, - "nauc_cv_recall_at_10_diff1": 0.4824079854591921, - "nauc_cv_recall_at_10_max": 0.5027473494385158, - "nauc_cv_recall_at_10_std": 0.48890187766681076, - "nauc_cv_recall_at_1_diff1": 0.644437799394293, - "nauc_cv_recall_at_1_max": 0.48720195113023645, - "nauc_cv_recall_at_1_std": 0.30433696117127595, - "nauc_cv_recall_at_20_diff1": 0.46158293543775514, - "nauc_cv_recall_at_20_max": 0.5333349260221991, - "nauc_cv_recall_at_20_std": 0.5400687456755257, - "nauc_cv_recall_at_3_diff1": 0.5168596861238599, - "nauc_cv_recall_at_3_max": 0.5076731821999175, - "nauc_cv_recall_at_3_std": 0.4135101558667384, - "nauc_cv_recall_at_5_diff1": 0.510299582925276, - "nauc_cv_recall_at_5_max": 0.5222048542887422, - "nauc_cv_recall_at_5_std": 0.4420289365796392, - "nauc_map_at_1000_diff1": 0.5830143383509978, - "nauc_map_at_1000_max": 0.5004906621407894, - "nauc_map_at_1000_std": 0.36274588989880185, - "nauc_map_at_100_diff1": 0.5825830804096754, - "nauc_map_at_100_max": 0.5003766495951647, - "nauc_map_at_100_std": 0.3632585756371978, - "nauc_map_at_10_diff1": 0.5827442300479906, - "nauc_map_at_10_max": 0.4989782918627815, - "nauc_map_at_10_std": 0.36149303848979514, - "nauc_map_at_1_diff1": 0.644437799394293, - "nauc_map_at_1_max": 0.48720195113023645, - "nauc_map_at_1_std": 0.30433696117127595, - "nauc_map_at_20_diff1": 0.582219628948973, - "nauc_map_at_20_max": 0.5010772738483961, - "nauc_map_at_20_std": 0.36377688498932803, - "nauc_map_at_3_diff1": 0.5862003610604175, - "nauc_map_at_3_max": 0.4962801376440491, - "nauc_map_at_3_std": 0.34912297998390074, - "nauc_map_at_5_diff1": 0.5861040029572455, - "nauc_map_at_5_max": 0.50086328296321, - "nauc_map_at_5_std": 0.3560349273683049, - "nauc_mrr_at_1000_diff1": 0.5830133060638379, - "nauc_mrr_at_1000_max": 0.5004875566119603, - "nauc_mrr_at_1000_std": 0.3627424647847617, - "nauc_mrr_at_100_diff1": 0.5825820719456357, - "nauc_mrr_at_100_max": 0.5003735001062519, - "nauc_mrr_at_100_std": 0.3632550948760236, - "nauc_mrr_at_10_diff1": 0.5827442300479906, - "nauc_mrr_at_10_max": 0.4989782918627815, - "nauc_mrr_at_10_std": 0.36149303848979514, - "nauc_mrr_at_1_diff1": 0.644437799394293, - "nauc_mrr_at_1_max": 0.48720195113023645, - "nauc_mrr_at_1_std": 0.30433696117127595, - "nauc_mrr_at_20_diff1": 0.582219628948973, - "nauc_mrr_at_20_max": 0.5010772738483961, - "nauc_mrr_at_20_std": 0.36377688498932803, - "nauc_mrr_at_3_diff1": 0.5862003610604175, - "nauc_mrr_at_3_max": 0.4962801376440491, - "nauc_mrr_at_3_std": 0.34912297998390074, - "nauc_mrr_at_5_diff1": 0.5861040029572455, - "nauc_mrr_at_5_max": 0.50086328296321, - "nauc_mrr_at_5_std": 0.3560349273683049, - "nauc_ndcg_at_1000_diff1": 0.5695323239109163, - "nauc_ndcg_at_1000_max": 0.5033127016866682, - "nauc_ndcg_at_1000_std": 0.3815003423814512, - "nauc_ndcg_at_100_diff1": 0.5594255087859172, - "nauc_ndcg_at_100_max": 0.5014926360153439, - "nauc_ndcg_at_100_std": 0.39584021920450774, - "nauc_ndcg_at_10_diff1": 0.5588656984073319, - "nauc_ndcg_at_10_max": 0.500693692375392, - "nauc_ndcg_at_10_std": 0.3910345980663399, - "nauc_ndcg_at_1_diff1": 0.644437799394293, - "nauc_ndcg_at_1_max": 0.48720195113023645, - "nauc_ndcg_at_1_std": 0.30433696117127595, - "nauc_ndcg_at_20_diff1": 0.5562572880254463, - "nauc_ndcg_at_20_max": 0.507832593764301, - "nauc_ndcg_at_20_std": 0.40000838616396234, - "nauc_ndcg_at_3_diff1": 0.5681528046470169, - "nauc_ndcg_at_3_max": 0.4992340623158375, - "nauc_ndcg_at_3_std": 0.3656656199525087, - "nauc_ndcg_at_5_diff1": 0.5668569119987612, - "nauc_ndcg_at_5_max": 0.5062895140454402, - "nauc_ndcg_at_5_std": 0.3775797241332556, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.4116549542723031, - "nauc_precision_at_100_max": 0.4820622105431704, - "nauc_precision_at_100_std": 0.6198649450287131, - "nauc_precision_at_10_diff1": 0.48240798545919267, - "nauc_precision_at_10_max": 0.5027473494385162, - "nauc_precision_at_10_std": 0.4889018776668112, - "nauc_precision_at_1_diff1": 0.644437799394293, - "nauc_precision_at_1_max": 0.48720195113023645, - "nauc_precision_at_1_std": 0.30433696117127595, - "nauc_precision_at_20_diff1": 0.4615829354377557, - "nauc_precision_at_20_max": 0.5333349260221996, - "nauc_precision_at_20_std": 0.5400687456755257, - "nauc_precision_at_3_diff1": 0.51685968612386, - "nauc_precision_at_3_max": 0.5076731821999171, - "nauc_precision_at_3_std": 0.4135101558667381, - "nauc_precision_at_5_diff1": 0.5102995829252761, - "nauc_precision_at_5_max": 0.5222048542887423, - "nauc_precision_at_5_std": 0.4420289365796393, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.41165495427230214, - "nauc_recall_at_100_max": 0.48206221054316895, - "nauc_recall_at_100_std": 0.6198649450287124, - "nauc_recall_at_10_diff1": 0.4824079854591921, - "nauc_recall_at_10_max": 0.5027473494385158, - "nauc_recall_at_10_std": 0.48890187766681076, - "nauc_recall_at_1_diff1": 0.644437799394293, - "nauc_recall_at_1_max": 0.48720195113023645, - "nauc_recall_at_1_std": 0.30433696117127595, - "nauc_recall_at_20_diff1": 0.46158293543775514, - "nauc_recall_at_20_max": 0.5333349260221991, - "nauc_recall_at_20_std": 0.5400687456755257, - "nauc_recall_at_3_diff1": 0.5168596861238599, - "nauc_recall_at_3_max": 0.5076731821999175, - "nauc_recall_at_3_std": 0.4135101558667384, - "nauc_recall_at_5_diff1": 0.510299582925276, - "nauc_recall_at_5_max": 0.5222048542887422, - "nauc_recall_at_5_std": 0.4420289365796392, - "ndcg_at_1": 0.30513, - "ndcg_at_10": 0.42702, - "ndcg_at_100": 0.48494, - "ndcg_at_1000": 0.50492, - "ndcg_at_20": 0.44689, - "ndcg_at_3": 0.38173, - "ndcg_at_5": 0.40871, - "precision_at_1": 0.30513, - "precision_at_10": 0.05589, - "precision_at_100": 0.00846, - "precision_at_1000": 0.001, - "precision_at_20": 0.03183, - "precision_at_3": 0.14563, - "precision_at_5": 0.10042, - "recall_at_1": 0.30513, - "recall_at_10": 0.55895, - "recall_at_100": 0.84605, - "recall_at_1000": 1.0, - "recall_at_20": 0.63662, - "recall_at_3": 0.43689, - "recall_at_5": 0.50208 - }, - { - "cv_recall_at_1": 0.30127, - "cv_recall_at_10": 0.53277, - "cv_recall_at_100": 0.80761, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.62474, - "cv_recall_at_3": 0.41121, - "cv_recall_at_5": 0.463, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.40991, - "map_at_1": 0.30127, - "map_at_10": 0.37141, - "map_at_100": 0.38227, - "map_at_1000": 0.38319, - "map_at_20": 0.37769, - "map_at_3": 0.34989, - "map_at_5": 0.36173, - "mrr_at_1": 0.3012684989429176, - "mrr_at_10": 0.3714067586160609, - "mrr_at_100": 0.38227085161316743, - "mrr_at_1000": 0.38319274292423033, - "mrr_at_20": 0.37768662546004295, - "mrr_at_3": 0.3498942917547569, - "mrr_at_5": 0.3617336152219871, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2862145489450746, - "nauc_cv_recall_at_100_max": 0.3375859781656063, - "nauc_cv_recall_at_100_std": 0.5712024116215759, - "nauc_cv_recall_at_10_diff1": 0.4501011642607513, - "nauc_cv_recall_at_10_max": 0.38903421251660564, - "nauc_cv_recall_at_10_std": 0.41458618776851075, - "nauc_cv_recall_at_1_diff1": 0.6199769209160442, - "nauc_cv_recall_at_1_max": 0.44486579497059164, - "nauc_cv_recall_at_1_std": 0.34961427339705375, - "nauc_cv_recall_at_20_diff1": 0.3905621048828626, - "nauc_cv_recall_at_20_max": 0.3598445963581698, - "nauc_cv_recall_at_20_std": 0.42593453843243684, - "nauc_cv_recall_at_3_diff1": 0.511281603222867, - "nauc_cv_recall_at_3_max": 0.4027393985883771, - "nauc_cv_recall_at_3_std": 0.3740329209080239, - "nauc_cv_recall_at_5_diff1": 0.4909711786318926, - "nauc_cv_recall_at_5_max": 0.40310060672872056, - "nauc_cv_recall_at_5_std": 0.40779921745492137, - "nauc_map_at_1000_diff1": 0.561306914268901, - "nauc_map_at_1000_max": 0.42384523950028463, - "nauc_map_at_1000_std": 0.37222062517463184, - "nauc_map_at_100_diff1": 0.560649765663532, - "nauc_map_at_100_max": 0.423551125644637, - "nauc_map_at_100_std": 0.3725384962597104, - "nauc_map_at_10_diff1": 0.5629278261027092, - "nauc_map_at_10_max": 0.42477770972747986, - "nauc_map_at_10_std": 0.37068421576959626, - "nauc_map_at_1_diff1": 0.6199769209160442, - "nauc_map_at_1_max": 0.44486579497059164, - "nauc_map_at_1_std": 0.34961427339705375, - "nauc_map_at_20_diff1": 0.5603607763457364, - "nauc_map_at_20_max": 0.42321621824595984, - "nauc_map_at_20_std": 0.3708469512400973, - "nauc_map_at_3_diff1": 0.5721797574981036, - "nauc_map_at_3_max": 0.4263739586019243, - "nauc_map_at_3_std": 0.36200070568332154, - "nauc_map_at_5_diff1": 0.5679047556570016, - "nauc_map_at_5_max": 0.42644946418412055, - "nauc_map_at_5_std": 0.36993011130441666, - "nauc_mrr_at_1000_diff1": 0.561306714388788, - "nauc_mrr_at_1000_max": 0.42384532845744227, - "nauc_mrr_at_1000_std": 0.3722207152690133, - "nauc_mrr_at_100_diff1": 0.5606496443087321, - "nauc_mrr_at_100_max": 0.4235514242888643, - "nauc_mrr_at_100_std": 0.3725388107550537, - "nauc_mrr_at_10_diff1": 0.5629278261027092, - "nauc_mrr_at_10_max": 0.42477770972747986, - "nauc_mrr_at_10_std": 0.37068421576959626, - "nauc_mrr_at_1_diff1": 0.6199769209160442, - "nauc_mrr_at_1_max": 0.44486579497059164, - "nauc_mrr_at_1_std": 0.34961427339705375, - "nauc_mrr_at_20_diff1": 0.5603607763457364, - "nauc_mrr_at_20_max": 0.42321621824595984, - "nauc_mrr_at_20_std": 0.3708469512400973, - "nauc_mrr_at_3_diff1": 0.5721797574981036, - "nauc_mrr_at_3_max": 0.4263739586019243, - "nauc_mrr_at_3_std": 0.36200070568332154, - "nauc_mrr_at_5_diff1": 0.5679047556570016, - "nauc_mrr_at_5_max": 0.42644946418412055, - "nauc_mrr_at_5_std": 0.36993011130441666, - "nauc_ndcg_at_1000_diff1": 0.5428204489189257, - "nauc_ndcg_at_1000_max": 0.4180339024617109, - "nauc_ndcg_at_1000_std": 0.38274231659073416, - "nauc_ndcg_at_100_diff1": 0.5243369526007065, - "nauc_ndcg_at_100_max": 0.411769907569649, - "nauc_ndcg_at_100_std": 0.39542375029587296, - "nauc_ndcg_at_10_diff1": 0.5361932596317941, - "nauc_ndcg_at_10_max": 0.4162362113983208, - "nauc_ndcg_at_10_std": 0.38113049766075247, - "nauc_ndcg_at_1_diff1": 0.6199769209160442, - "nauc_ndcg_at_1_max": 0.44486579497059164, - "nauc_ndcg_at_1_std": 0.34961427339705375, - "nauc_ndcg_at_20_diff1": 0.525438714155031, - "nauc_ndcg_at_20_max": 0.4103350023364189, - "nauc_ndcg_at_20_std": 0.38250427397382974, - "nauc_ndcg_at_3_diff1": 0.556324242979917, - "nauc_ndcg_at_3_max": 0.4202217818769986, - "nauc_ndcg_at_3_std": 0.36520139070196067, - "nauc_ndcg_at_5_diff1": 0.5483766172208181, - "nauc_ndcg_at_5_max": 0.4204092201127261, - "nauc_ndcg_at_5_std": 0.379301524104246, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.2862145489450751, - "nauc_precision_at_100_max": 0.33758597816560554, - "nauc_precision_at_100_std": 0.5712024116215759, - "nauc_precision_at_10_diff1": 0.4501011642607511, - "nauc_precision_at_10_max": 0.3890342125166057, - "nauc_precision_at_10_std": 0.4145861877685106, - "nauc_precision_at_1_diff1": 0.6199769209160442, - "nauc_precision_at_1_max": 0.44486579497059164, - "nauc_precision_at_1_std": 0.34961427339705375, - "nauc_precision_at_20_diff1": 0.39056210488286275, - "nauc_precision_at_20_max": 0.3598445963581694, - "nauc_precision_at_20_std": 0.42593453843243645, - "nauc_precision_at_3_diff1": 0.5112816032228666, - "nauc_precision_at_3_max": 0.40273939858837715, - "nauc_precision_at_3_std": 0.3740329209080242, - "nauc_precision_at_5_diff1": 0.4909711786318925, - "nauc_precision_at_5_max": 0.4031006067287203, - "nauc_precision_at_5_std": 0.4077992174549212, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2862145489450746, - "nauc_recall_at_100_max": 0.3375859781656063, - "nauc_recall_at_100_std": 0.5712024116215759, - "nauc_recall_at_10_diff1": 0.4501011642607513, - "nauc_recall_at_10_max": 0.38903421251660564, - "nauc_recall_at_10_std": 0.41458618776851075, - "nauc_recall_at_1_diff1": 0.6199769209160442, - "nauc_recall_at_1_max": 0.44486579497059164, - "nauc_recall_at_1_std": 0.34961427339705375, - "nauc_recall_at_20_diff1": 0.3905621048828626, - "nauc_recall_at_20_max": 0.3598445963581698, - "nauc_recall_at_20_std": 0.42593453843243684, - "nauc_recall_at_3_diff1": 0.511281603222867, - "nauc_recall_at_3_max": 0.4027393985883771, - "nauc_recall_at_3_std": 0.3740329209080239, - "nauc_recall_at_5_diff1": 0.4909711786318926, - "nauc_recall_at_5_max": 0.40310060672872056, - "nauc_recall_at_5_std": 0.40779921745492137, - "ndcg_at_1": 0.30127, - "ndcg_at_10": 0.40991, - "ndcg_at_100": 0.46679, - "ndcg_at_1000": 0.49132, - "ndcg_at_20": 0.43303, - "ndcg_at_3": 0.36565, - "ndcg_at_5": 0.38698, - "precision_at_1": 0.30127, - "precision_at_10": 0.05328, - "precision_at_100": 0.00808, - "precision_at_1000": 0.001, - "precision_at_20": 0.03124, - "precision_at_3": 0.13707, - "precision_at_5": 0.0926, - "recall_at_1": 0.30127, - "recall_at_10": 0.53277, - "recall_at_100": 0.80761, - "recall_at_1000": 1.0, - "recall_at_20": 0.62474, - "recall_at_3": 0.41121, - "recall_at_5": 0.463 - }, - { - "cv_recall_at_1": 0.474, - "cv_recall_at_10": 0.773, - "cv_recall_at_100": 0.973, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.846, - "cv_recall_at_3": 0.627, - "cv_recall_at_5": 0.697, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.61597, - "map_at_1": 0.474, - "map_at_10": 0.56653, - "map_at_100": 0.57507, - "map_at_1000": 0.57521, - "map_at_20": 0.57157, - "map_at_3": 0.5405, - "map_at_5": 0.55655, - "mrr_at_1": 0.474, - "mrr_at_10": 0.5665349206349206, - "mrr_at_100": 0.5750684463503862, - "mrr_at_1000": 0.5752147141574319, - "mrr_at_20": 0.5715710057443414, - "mrr_at_3": 0.5404999999999998, - "mrr_at_5": 0.55655, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.45353944046754274, - "nauc_cv_recall_at_100_max": 0.7559912854030483, - "nauc_cv_recall_at_100_std": 0.7699623059100126, - "nauc_cv_recall_at_10_diff1": 0.7005081767378533, - "nauc_cv_recall_at_10_max": 0.20818058253744653, - "nauc_cv_recall_at_10_std": -0.28630003460215453, - "nauc_cv_recall_at_1_diff1": 0.7659842765053384, - "nauc_cv_recall_at_1_max": 0.323405137468343, - "nauc_cv_recall_at_1_std": -0.06912237336565674, - "nauc_cv_recall_at_20_diff1": 0.6957153024006514, - "nauc_cv_recall_at_20_max": 0.24541855706787719, - "nauc_cv_recall_at_20_std": -0.21536912069337907, - "nauc_cv_recall_at_3_diff1": 0.6735930290720233, - "nauc_cv_recall_at_3_max": 0.2844436549378181, - "nauc_cv_recall_at_3_std": -0.14239634998620962, - "nauc_cv_recall_at_5_diff1": 0.678067736963919, - "nauc_cv_recall_at_5_max": 0.28032026577835256, - "nauc_cv_recall_at_5_std": -0.18294278641251102, - "nauc_map_at_1000_diff1": 0.7349768037519908, - "nauc_map_at_1000_max": 0.3108191319770147, - "nauc_map_at_1000_std": -0.09577460850617198, - "nauc_map_at_100_diff1": 0.7348596383487171, - "nauc_map_at_100_max": 0.3110408807336249, - "nauc_map_at_100_std": -0.0953690667160771, - "nauc_map_at_10_diff1": 0.7348528563701027, - "nauc_map_at_10_max": 0.3060472302583592, - "nauc_map_at_10_std": -0.10453879471869904, - "nauc_map_at_1_diff1": 0.7659842765053384, - "nauc_map_at_1_max": 0.323405137468343, - "nauc_map_at_1_std": -0.06912237336565674, - "nauc_map_at_20_diff1": 0.7349672844338018, - "nauc_map_at_20_max": 0.30896979717042244, - "nauc_map_at_20_std": -0.0993943729562651, - "nauc_map_at_3_diff1": 0.7293407859255343, - "nauc_map_at_3_max": 0.31154340909320627, - "nauc_map_at_3_std": -0.09141294877318266, - "nauc_map_at_5_diff1": 0.7315184827912649, - "nauc_map_at_5_max": 0.31110890999241614, - "nauc_map_at_5_std": -0.09830711349311756, - "nauc_mrr_at_1000_diff1": 0.7349768037519908, - "nauc_mrr_at_1000_max": 0.3108191319770147, - "nauc_mrr_at_1000_std": -0.09577460850617198, - "nauc_mrr_at_100_diff1": 0.7348596383487171, - "nauc_mrr_at_100_max": 0.3110408807336249, - "nauc_mrr_at_100_std": -0.0953690667160771, - "nauc_mrr_at_10_diff1": 0.7348528563701027, - "nauc_mrr_at_10_max": 0.3060472302583592, - "nauc_mrr_at_10_std": -0.10453879471869904, - "nauc_mrr_at_1_diff1": 0.7659842765053384, - "nauc_mrr_at_1_max": 0.323405137468343, - "nauc_mrr_at_1_std": -0.06912237336565674, - "nauc_mrr_at_20_diff1": 0.7349672844338018, - "nauc_mrr_at_20_max": 0.30896979717042244, - "nauc_mrr_at_20_std": -0.0993943729562651, - "nauc_mrr_at_3_diff1": 0.7293407859255343, - "nauc_mrr_at_3_max": 0.31154340909320627, - "nauc_mrr_at_3_std": -0.09141294877318266, - "nauc_mrr_at_5_diff1": 0.7315184827912649, - "nauc_mrr_at_5_max": 0.31110890999241614, - "nauc_mrr_at_5_std": -0.09830711349311756, - "nauc_ndcg_at_1000_diff1": 0.7300106781628969, - "nauc_ndcg_at_1000_max": 0.30946240469434727, - "nauc_ndcg_at_1000_std": -0.09855705821637864, - "nauc_ndcg_at_100_diff1": 0.726228776286605, - "nauc_ndcg_at_100_max": 0.31581736941064814, - "nauc_ndcg_at_100_std": -0.0863659221103867, - "nauc_ndcg_at_10_diff1": 0.7275424860578261, - "nauc_ndcg_at_10_max": 0.2899275846092146, - "nauc_ndcg_at_10_std": -0.1349627955933614, - "nauc_ndcg_at_1_diff1": 0.7659842765053384, - "nauc_ndcg_at_1_max": 0.323405137468343, - "nauc_ndcg_at_1_std": -0.06912237336565674, - "nauc_ndcg_at_20_diff1": 0.7281192602738402, - "nauc_ndcg_at_20_max": 0.30002130037590796, - "nauc_ndcg_at_20_std": -0.11647828361837598, - "nauc_ndcg_at_3_diff1": 0.7160762699063752, - "nauc_ndcg_at_3_max": 0.30524984633027247, - "nauc_ndcg_at_3_std": -0.10325252208043191, - "nauc_ndcg_at_5_diff1": 0.7196819527437018, - "nauc_ndcg_at_5_max": 0.30450554939861313, - "nauc_ndcg_at_5_std": -0.11613402673386349, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.45353944046753836, - "nauc_precision_at_100_max": 0.7559912854030467, - "nauc_precision_at_100_std": 0.7699623059100184, - "nauc_precision_at_10_diff1": 0.7005081767378538, - "nauc_precision_at_10_max": 0.2081805825374449, - "nauc_precision_at_10_std": -0.2863000346021545, - "nauc_precision_at_1_diff1": 0.7659842765053384, - "nauc_precision_at_1_max": 0.323405137468343, - "nauc_precision_at_1_std": -0.06912237336565674, - "nauc_precision_at_20_diff1": 0.6957153024006515, - "nauc_precision_at_20_max": 0.24541855706787566, - "nauc_precision_at_20_std": -0.2153691206933822, - "nauc_precision_at_3_diff1": 0.6735930290720236, - "nauc_precision_at_3_max": 0.28444365493781915, - "nauc_precision_at_3_std": -0.14239634998620926, - "nauc_precision_at_5_diff1": 0.6780677369639184, - "nauc_precision_at_5_max": 0.28032026577835195, - "nauc_precision_at_5_std": -0.18294278641251163, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.45353944046754274, - "nauc_recall_at_100_max": 0.7559912854030483, - "nauc_recall_at_100_std": 0.7699623059100126, - "nauc_recall_at_10_diff1": 0.7005081767378533, - "nauc_recall_at_10_max": 0.20818058253744653, - "nauc_recall_at_10_std": -0.28630003460215453, - "nauc_recall_at_1_diff1": 0.7659842765053384, - "nauc_recall_at_1_max": 0.323405137468343, - "nauc_recall_at_1_std": -0.06912237336565674, - "nauc_recall_at_20_diff1": 0.6957153024006514, - "nauc_recall_at_20_max": 0.24541855706787719, - "nauc_recall_at_20_std": -0.21536912069337907, - "nauc_recall_at_3_diff1": 0.6735930290720233, - "nauc_recall_at_3_max": 0.2844436549378181, - "nauc_recall_at_3_std": -0.14239634998620962, - "nauc_recall_at_5_diff1": 0.678067736963919, - "nauc_recall_at_5_max": 0.28032026577835256, - "nauc_recall_at_5_std": -0.18294278641251102, - "ndcg_at_1": 0.474, - "ndcg_at_10": 0.61597, - "ndcg_at_100": 0.65853, - "ndcg_at_1000": 0.66207, - "ndcg_at_20": 0.6344, - "ndcg_at_3": 0.56268, - "ndcg_at_5": 0.59155, - "precision_at_1": 0.474, - "precision_at_10": 0.0773, - "precision_at_100": 0.00973, - "precision_at_1000": 0.001, - "precision_at_20": 0.0423, - "precision_at_3": 0.209, - "precision_at_5": 0.1394, - "recall_at_1": 0.474, - "recall_at_10": 0.773, - "recall_at_100": 0.973, - "recall_at_1000": 1.0, - "recall_at_20": 0.846, - "recall_at_3": 0.627, - "recall_at_5": 0.697 - } - ] - }, - "task_name": "WITT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/model_meta.json b/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/model_meta.json deleted file mode 100644 index e2f5452dbe..0000000000 --- a/results-mieb/google__siglip-base-patch16-256/b078df89e446d623010d890864d4207fe6399f61/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "google/siglip-base-patch16-256", "revision": "b078df89e446d623010d890864d4207fe6399f61", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/WITT2IRetrieval.json b/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/WITT2IRetrieval.json deleted file mode 100644 index 112c157a17..0000000000 --- a/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/WITT2IRetrieval.json +++ /dev/null @@ -1,1936 +0,0 @@ -{ - "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", - "evaluation_time": 137.17432284355164, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.13933, - "cv_recall_at_10": 0.33596, - "cv_recall_at_100": 0.65056, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.4, - "cv_recall_at_3": 0.20112, - "cv_recall_at_5": 0.25281, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.22251, - "map_at_1": 0.13933, - "map_at_10": 0.18821, - "map_at_100": 0.19874, - "map_at_1000": 0.20028, - "map_at_20": 0.19266, - "map_at_3": 0.16592, - "map_at_5": 0.17749, - "mrr_at_1": 0.1393258426966292, - "mrr_at_10": 0.1882062600321027, - "mrr_at_100": 0.19873749728328727, - "mrr_at_1000": 0.2002817481555001, - "mrr_at_20": 0.19265773240009676, - "mrr_at_3": 0.16591760299625463, - "mrr_at_5": 0.17749063670411983, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2979879802367704, - "nauc_cv_recall_at_100_max": 0.4739733500284659, - "nauc_cv_recall_at_100_std": 0.46487157379083804, - "nauc_cv_recall_at_10_diff1": 0.3857208699452441, - "nauc_cv_recall_at_10_max": 0.41350589842683755, - "nauc_cv_recall_at_10_std": 0.3503705814003786, - "nauc_cv_recall_at_1_diff1": 0.6313007118799755, - "nauc_cv_recall_at_1_max": 0.5155341013472138, - "nauc_cv_recall_at_1_std": 0.35458607624004906, - "nauc_cv_recall_at_20_diff1": 0.382239438086532, - "nauc_cv_recall_at_20_max": 0.39601968317152486, - "nauc_cv_recall_at_20_std": 0.3440345873014037, - "nauc_cv_recall_at_3_diff1": 0.4801863620484621, - "nauc_cv_recall_at_3_max": 0.4433085417748924, - "nauc_cv_recall_at_3_std": 0.3463988285400372, - "nauc_cv_recall_at_5_diff1": 0.39071373942219917, - "nauc_cv_recall_at_5_max": 0.3956512800269739, - "nauc_cv_recall_at_5_std": 0.36737802120083213, - "nauc_map_at_1000_diff1": 0.5211586969479306, - "nauc_map_at_1000_max": 0.4620597681593909, - "nauc_map_at_1000_std": 0.3518606509995756, - "nauc_map_at_100_diff1": 0.5199563889214931, - "nauc_map_at_100_max": 0.4617893581547442, - "nauc_map_at_100_std": 0.35187670213120303, - "nauc_map_at_10_diff1": 0.5228671435456008, - "nauc_map_at_10_max": 0.4627408394245875, - "nauc_map_at_10_std": 0.34995955033850235, - "nauc_map_at_1_diff1": 0.6313007118799755, - "nauc_map_at_1_max": 0.5155341013472138, - "nauc_map_at_1_std": 0.35458607624004906, - "nauc_map_at_20_diff1": 0.5224268312188151, - "nauc_map_at_20_max": 0.461155418295598, - "nauc_map_at_20_std": 0.3494356451137881, - "nauc_map_at_3_diff1": 0.5539130179035476, - "nauc_map_at_3_max": 0.4775300569101307, - "nauc_map_at_3_std": 0.34648304915615186, - "nauc_map_at_5_diff1": 0.527370251204279, - "nauc_map_at_5_max": 0.463098359674595, - "nauc_map_at_5_std": 0.3526771617701944, - "nauc_mrr_at_1000_diff1": 0.5211586936725224, - "nauc_mrr_at_1000_max": 0.46205976295168305, - "nauc_mrr_at_1000_std": 0.3518606434736402, - "nauc_mrr_at_100_diff1": 0.5199563889214931, - "nauc_mrr_at_100_max": 0.4617893581547442, - "nauc_mrr_at_100_std": 0.35187670213120303, - "nauc_mrr_at_10_diff1": 0.5228671435456008, - "nauc_mrr_at_10_max": 0.4627408394245875, - "nauc_mrr_at_10_std": 0.34995955033850235, - "nauc_mrr_at_1_diff1": 0.6313007118799755, - "nauc_mrr_at_1_max": 0.5155341013472138, - "nauc_mrr_at_1_std": 0.35458607624004906, - "nauc_mrr_at_20_diff1": 0.5224268312188151, - "nauc_mrr_at_20_max": 0.461155418295598, - "nauc_mrr_at_20_std": 0.3494356451137881, - "nauc_mrr_at_3_diff1": 0.5539130179035476, - "nauc_mrr_at_3_max": 0.4775300569101307, - "nauc_mrr_at_3_std": 0.34648304915615186, - "nauc_mrr_at_5_diff1": 0.527370251204279, - "nauc_mrr_at_5_max": 0.463098359674595, - "nauc_mrr_at_5_std": 0.3526771617701944, - "nauc_ndcg_at_1000_diff1": 0.4921495850103755, - "nauc_ndcg_at_1000_max": 0.4536351152047796, - "nauc_ndcg_at_1000_std": 0.3590542409723068, - "nauc_ndcg_at_100_diff1": 0.4658043926888635, - "nauc_ndcg_at_100_max": 0.45474316038725227, - "nauc_ndcg_at_100_std": 0.3702553940425927, - "nauc_ndcg_at_10_diff1": 0.48102408933081287, - "nauc_ndcg_at_10_max": 0.4467146470840002, - "nauc_ndcg_at_10_std": 0.3503084336712536, - "nauc_ndcg_at_1_diff1": 0.6313007118799755, - "nauc_ndcg_at_1_max": 0.5155341013472138, - "nauc_ndcg_at_1_std": 0.35458607624004906, - "nauc_ndcg_at_20_diff1": 0.47931001719588917, - "nauc_ndcg_at_20_max": 0.44104335869322414, - "nauc_ndcg_at_20_std": 0.34838234044212724, - "nauc_ndcg_at_3_diff1": 0.5327924389517208, - "nauc_ndcg_at_3_max": 0.4676848915957549, - "nauc_ndcg_at_3_std": 0.3462985691606815, - "nauc_ndcg_at_5_diff1": 0.4879105951367381, - "nauc_ndcg_at_5_max": 0.44358910787060024, - "nauc_ndcg_at_5_std": 0.3566285952669218, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.29798798023677125, - "nauc_precision_at_100_max": 0.47397335002846613, - "nauc_precision_at_100_std": 0.46487157379083816, - "nauc_precision_at_10_diff1": 0.3857208699452443, - "nauc_precision_at_10_max": 0.4135058984268377, - "nauc_precision_at_10_std": 0.3503705814003787, - "nauc_precision_at_1_diff1": 0.6313007118799755, - "nauc_precision_at_1_max": 0.5155341013472138, - "nauc_precision_at_1_std": 0.35458607624004906, - "nauc_precision_at_20_diff1": 0.38223943808653166, - "nauc_precision_at_20_max": 0.39601968317152453, - "nauc_precision_at_20_std": 0.3440345873014034, - "nauc_precision_at_3_diff1": 0.480186362048462, - "nauc_precision_at_3_max": 0.443308541774892, - "nauc_precision_at_3_std": 0.34639882854003723, - "nauc_precision_at_5_diff1": 0.3907137394221993, - "nauc_precision_at_5_max": 0.3956512800269739, - "nauc_precision_at_5_std": 0.3673780212008324, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2979879802367704, - "nauc_recall_at_100_max": 0.4739733500284659, - "nauc_recall_at_100_std": 0.46487157379083804, - "nauc_recall_at_10_diff1": 0.3857208699452441, - "nauc_recall_at_10_max": 0.41350589842683755, - "nauc_recall_at_10_std": 0.3503705814003786, - "nauc_recall_at_1_diff1": 0.6313007118799755, - "nauc_recall_at_1_max": 0.5155341013472138, - "nauc_recall_at_1_std": 0.35458607624004906, - "nauc_recall_at_20_diff1": 0.382239438086532, - "nauc_recall_at_20_max": 0.39601968317152486, - "nauc_recall_at_20_std": 0.3440345873014037, - "nauc_recall_at_3_diff1": 0.4801863620484621, - "nauc_recall_at_3_max": 0.4433085417748924, - "nauc_recall_at_3_std": 0.3463988285400372, - "nauc_recall_at_5_diff1": 0.39071373942219917, - "nauc_recall_at_5_max": 0.3956512800269739, - "nauc_recall_at_5_std": 0.36737802120083213, - "ndcg_at_1": 0.13933, - "ndcg_at_10": 0.22251, - "ndcg_at_100": 0.28451, - "ndcg_at_1000": 0.32833, - "ndcg_at_20": 0.23872, - "ndcg_at_3": 0.17493, - "ndcg_at_5": 0.19601, - "precision_at_1": 0.13933, - "precision_at_10": 0.0336, - "precision_at_100": 0.00651, - "precision_at_1000": 0.001, - "precision_at_20": 0.02, - "precision_at_3": 0.06704, - "precision_at_5": 0.05056, - "recall_at_1": 0.13933, - "recall_at_10": 0.33596, - "recall_at_100": 0.65056, - "recall_at_1000": 1.0, - "recall_at_20": 0.4, - "recall_at_3": 0.20112, - "recall_at_5": 0.25281 - }, - { - "cv_recall_at_1": 0.15233, - "cv_recall_at_10": 0.32558, - "cv_recall_at_100": 0.68488, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.42558, - "cv_recall_at_3": 0.20814, - "cv_recall_at_5": 0.25, - "hf_subset": "bg", - "languages": [ - "bul-Cyrl" - ], - "main_score": 0.22681, - "map_at_1": 0.15233, - "map_at_10": 0.19685, - "map_at_100": 0.20977, - "map_at_1000": 0.21137, - "map_at_20": 0.20388, - "map_at_3": 0.17733, - "map_at_5": 0.18698, - "mrr_at_1": 0.15232558139534882, - "mrr_at_10": 0.19685123661867845, - "mrr_at_100": 0.20976999322473525, - "mrr_at_1000": 0.21136818088041656, - "mrr_at_20": 0.20388234299720012, - "mrr_at_3": 0.17732558139534887, - "mrr_at_5": 0.1869767441860465, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.28928588096595664, - "nauc_cv_recall_at_100_max": 0.3545880113713616, - "nauc_cv_recall_at_100_std": 0.3624408248903762, - "nauc_cv_recall_at_10_diff1": 0.410329527119275, - "nauc_cv_recall_at_10_max": 0.16473563361052504, - "nauc_cv_recall_at_10_std": 0.08078999627652986, - "nauc_cv_recall_at_1_diff1": 0.6586457244685855, - "nauc_cv_recall_at_1_max": 0.2772832920468003, - "nauc_cv_recall_at_1_std": 0.12915926096719418, - "nauc_cv_recall_at_20_diff1": 0.3776209415608929, - "nauc_cv_recall_at_20_max": 0.18776254714901358, - "nauc_cv_recall_at_20_std": 0.14351872823746506, - "nauc_cv_recall_at_3_diff1": 0.5002744242173366, - "nauc_cv_recall_at_3_max": 0.22006604660961354, - "nauc_cv_recall_at_3_std": 0.12469577719340726, - "nauc_cv_recall_at_5_diff1": 0.46741021408349515, - "nauc_cv_recall_at_5_max": 0.19384178519804637, - "nauc_cv_recall_at_5_std": 0.09437635245516056, - "nauc_map_at_1000_diff1": 0.5513265197992123, - "nauc_map_at_1000_max": 0.24016450921419116, - "nauc_map_at_1000_std": 0.1263100552175581, - "nauc_map_at_100_diff1": 0.5503259849465996, - "nauc_map_at_100_max": 0.24015798693197335, - "nauc_map_at_100_std": 0.1266885245132762, - "nauc_map_at_10_diff1": 0.5541550509171621, - "nauc_map_at_10_max": 0.23351168479305368, - "nauc_map_at_10_std": 0.11489690071703798, - "nauc_map_at_1_diff1": 0.6586457244685855, - "nauc_map_at_1_max": 0.2772832920468003, - "nauc_map_at_1_std": 0.12915926096719418, - "nauc_map_at_20_diff1": 0.550752731386968, - "nauc_map_at_20_max": 0.2363155358781987, - "nauc_map_at_20_std": 0.12196053759746485, - "nauc_map_at_3_diff1": 0.5767075860995042, - "nauc_map_at_3_max": 0.24751741022380233, - "nauc_map_at_3_std": 0.12614812315624652, - "nauc_map_at_5_diff1": 0.5661607434146992, - "nauc_map_at_5_max": 0.23941193117024734, - "nauc_map_at_5_std": 0.11767569954794925, - "nauc_mrr_at_1000_diff1": 0.5513265093688596, - "nauc_mrr_at_1000_max": 0.2401645568612914, - "nauc_mrr_at_1000_std": 0.12631007104475458, - "nauc_mrr_at_100_diff1": 0.5503259849465996, - "nauc_mrr_at_100_max": 0.24015798693197335, - "nauc_mrr_at_100_std": 0.1266885245132762, - "nauc_mrr_at_10_diff1": 0.5541550509171621, - "nauc_mrr_at_10_max": 0.23351168479305368, - "nauc_mrr_at_10_std": 0.11489690071703798, - "nauc_mrr_at_1_diff1": 0.6586457244685855, - "nauc_mrr_at_1_max": 0.2772832920468003, - "nauc_mrr_at_1_std": 0.12915926096719418, - "nauc_mrr_at_20_diff1": 0.550752731386968, - "nauc_mrr_at_20_max": 0.2363155358781987, - "nauc_mrr_at_20_std": 0.12196053759746485, - "nauc_mrr_at_3_diff1": 0.5767075860995042, - "nauc_mrr_at_3_max": 0.24751741022380233, - "nauc_mrr_at_3_std": 0.12614812315624652, - "nauc_mrr_at_5_diff1": 0.5661607434146992, - "nauc_mrr_at_5_max": 0.23941193117024734, - "nauc_mrr_at_5_std": 0.11767569954794925, - "nauc_ndcg_at_1000_diff1": 0.5214665780441233, - "nauc_ndcg_at_1000_max": 0.24043521395830333, - "nauc_ndcg_at_1000_std": 0.13920790773883404, - "nauc_ndcg_at_100_diff1": 0.49414511319709525, - "nauc_ndcg_at_100_max": 0.25181424634361693, - "nauc_ndcg_at_100_std": 0.1629480668331953, - "nauc_ndcg_at_10_diff1": 0.5125254924923802, - "nauc_ndcg_at_10_max": 0.21379969221512746, - "nauc_ndcg_at_10_std": 0.10536356101011754, - "nauc_ndcg_at_1_diff1": 0.6586457244685855, - "nauc_ndcg_at_1_max": 0.2772832920468003, - "nauc_ndcg_at_1_std": 0.12915926096719418, - "nauc_ndcg_at_20_diff1": 0.5014840374569179, - "nauc_ndcg_at_20_max": 0.22153633427823136, - "nauc_ndcg_at_20_std": 0.1262308783663143, - "nauc_ndcg_at_3_diff1": 0.5552016979915128, - "nauc_ndcg_at_3_max": 0.23978651146760627, - "nauc_ndcg_at_3_std": 0.12570952954169726, - "nauc_ndcg_at_5_diff1": 0.537869708474716, - "nauc_ndcg_at_5_max": 0.22648568477623546, - "nauc_ndcg_at_5_std": 0.11132774920238546, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.28928588096595753, - "nauc_precision_at_100_max": 0.35458801137136126, - "nauc_precision_at_100_std": 0.36244082489037516, - "nauc_precision_at_10_diff1": 0.41032952711927495, - "nauc_precision_at_10_max": 0.16473563361052518, - "nauc_precision_at_10_std": 0.08078999627652979, - "nauc_precision_at_1_diff1": 0.6586457244685855, - "nauc_precision_at_1_max": 0.2772832920468003, - "nauc_precision_at_1_std": 0.12915926096719418, - "nauc_precision_at_20_diff1": 0.3776209415608924, - "nauc_precision_at_20_max": 0.1877625471490132, - "nauc_precision_at_20_std": 0.14351872823746498, - "nauc_precision_at_3_diff1": 0.5002744242173365, - "nauc_precision_at_3_max": 0.2200660466096136, - "nauc_precision_at_3_std": 0.12469577719340708, - "nauc_precision_at_5_diff1": 0.46741021408349537, - "nauc_precision_at_5_max": 0.19384178519804657, - "nauc_precision_at_5_std": 0.09437635245516063, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.28928588096595664, - "nauc_recall_at_100_max": 0.3545880113713616, - "nauc_recall_at_100_std": 0.3624408248903762, - "nauc_recall_at_10_diff1": 0.410329527119275, - "nauc_recall_at_10_max": 0.16473563361052504, - "nauc_recall_at_10_std": 0.08078999627652986, - "nauc_recall_at_1_diff1": 0.6586457244685855, - "nauc_recall_at_1_max": 0.2772832920468003, - "nauc_recall_at_1_std": 0.12915926096719418, - "nauc_recall_at_20_diff1": 0.3776209415608929, - "nauc_recall_at_20_max": 0.18776254714901358, - "nauc_recall_at_20_std": 0.14351872823746506, - "nauc_recall_at_3_diff1": 0.5002744242173366, - "nauc_recall_at_3_max": 0.22006604660961354, - "nauc_recall_at_3_std": 0.12469577719340726, - "nauc_recall_at_5_diff1": 0.46741021408349515, - "nauc_recall_at_5_max": 0.19384178519804637, - "nauc_recall_at_5_std": 0.09437635245516056, - "ndcg_at_1": 0.15233, - "ndcg_at_10": 0.22681, - "ndcg_at_100": 0.29875, - "ndcg_at_1000": 0.33929, - "ndcg_at_20": 0.2522, - "ndcg_at_3": 0.18526, - "ndcg_at_5": 0.20257, - "precision_at_1": 0.15233, - "precision_at_10": 0.03256, - "precision_at_100": 0.00685, - "precision_at_1000": 0.001, - "precision_at_20": 0.02128, - "precision_at_3": 0.06938, - "precision_at_5": 0.05, - "recall_at_1": 0.15233, - "recall_at_10": 0.32558, - "recall_at_100": 0.68488, - "recall_at_1000": 1.0, - "recall_at_20": 0.42558, - "recall_at_3": 0.20814, - "recall_at_5": 0.25 - }, - { - "cv_recall_at_1": 0.32884, - "cv_recall_at_10": 0.57688, - "cv_recall_at_100": 0.81594, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.65544, - "cv_recall_at_3": 0.43547, - "cv_recall_at_5": 0.49495, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.44096, - "map_at_1": 0.32884, - "map_at_10": 0.39885, - "map_at_100": 0.40826, - "map_at_1000": 0.40911, - "map_at_20": 0.40444, - "map_at_3": 0.37411, - "map_at_5": 0.38803, - "mrr_at_1": 0.3288439955106622, - "mrr_at_10": 0.3988509432953876, - "mrr_at_100": 0.40826340383916915, - "mrr_at_1000": 0.40911460678506656, - "mrr_at_20": 0.40444359872490476, - "mrr_at_3": 0.37411148522259624, - "mrr_at_5": 0.38802843247287666, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.42251653533438616, - "nauc_cv_recall_at_100_max": 0.5974632567614767, - "nauc_cv_recall_at_100_std": 0.5798983552509402, - "nauc_cv_recall_at_10_diff1": 0.4965565401407999, - "nauc_cv_recall_at_10_max": 0.4884093981133426, - "nauc_cv_recall_at_10_std": 0.3476667895125619, - "nauc_cv_recall_at_1_diff1": 0.6881197967738875, - "nauc_cv_recall_at_1_max": 0.5104387821019861, - "nauc_cv_recall_at_1_std": 0.22996512608256198, - "nauc_cv_recall_at_20_diff1": 0.4579569229337732, - "nauc_cv_recall_at_20_max": 0.5357361507438837, - "nauc_cv_recall_at_20_std": 0.41608588175822514, - "nauc_cv_recall_at_3_diff1": 0.5826564933802192, - "nauc_cv_recall_at_3_max": 0.4915891078838359, - "nauc_cv_recall_at_3_std": 0.26513885013921823, - "nauc_cv_recall_at_5_diff1": 0.5361007340951345, - "nauc_cv_recall_at_5_max": 0.46650003271004464, - "nauc_cv_recall_at_5_std": 0.29467856997063335, - "nauc_map_at_1000_diff1": 0.6260425875683999, - "nauc_map_at_1000_max": 0.5031965378036112, - "nauc_map_at_1000_std": 0.2599848007687964, - "nauc_map_at_100_diff1": 0.6257450554561577, - "nauc_map_at_100_max": 0.5033928040860026, - "nauc_map_at_100_std": 0.2606797179062218, - "nauc_map_at_10_diff1": 0.6265320761885688, - "nauc_map_at_10_max": 0.5003685246352216, - "nauc_map_at_10_std": 0.25713351432434206, - "nauc_map_at_1_diff1": 0.6881197967738875, - "nauc_map_at_1_max": 0.5104387821019861, - "nauc_map_at_1_std": 0.22996512608256198, - "nauc_map_at_20_diff1": 0.6251828068786616, - "nauc_map_at_20_max": 0.5032413273960014, - "nauc_map_at_20_std": 0.26008764400434875, - "nauc_map_at_3_diff1": 0.6404464664086379, - "nauc_map_at_3_max": 0.502375813650769, - "nauc_map_at_3_std": 0.24441336490772264, - "nauc_map_at_5_diff1": 0.6304197906675999, - "nauc_map_at_5_max": 0.49722842291628355, - "nauc_map_at_5_std": 0.2509909601892623, - "nauc_mrr_at_1000_diff1": 0.6260425875683999, - "nauc_mrr_at_1000_max": 0.5031965378036112, - "nauc_mrr_at_1000_std": 0.2599848007687964, - "nauc_mrr_at_100_diff1": 0.6257450554561577, - "nauc_mrr_at_100_max": 0.5033928040860026, - "nauc_mrr_at_100_std": 0.2606797179062218, - "nauc_mrr_at_10_diff1": 0.6265320761885688, - "nauc_mrr_at_10_max": 0.5003685246352216, - "nauc_mrr_at_10_std": 0.25713351432434206, - "nauc_mrr_at_1_diff1": 0.6881197967738875, - "nauc_mrr_at_1_max": 0.5104387821019861, - "nauc_mrr_at_1_std": 0.22996512608256198, - "nauc_mrr_at_20_diff1": 0.6251828068786616, - "nauc_mrr_at_20_max": 0.5032413273960014, - "nauc_mrr_at_20_std": 0.26008764400434875, - "nauc_mrr_at_3_diff1": 0.6404464664086379, - "nauc_mrr_at_3_max": 0.502375813650769, - "nauc_mrr_at_3_std": 0.24441336490772264, - "nauc_mrr_at_5_diff1": 0.6304197906675999, - "nauc_mrr_at_5_max": 0.49722842291628355, - "nauc_mrr_at_5_std": 0.2509909601892623, - "nauc_ndcg_at_1000_diff1": 0.6076429265545639, - "nauc_ndcg_at_1000_max": 0.5052332372999279, - "nauc_ndcg_at_1000_std": 0.27652683355425656, - "nauc_ndcg_at_100_diff1": 0.5957468733498283, - "nauc_ndcg_at_100_max": 0.5114361696173704, - "nauc_ndcg_at_100_std": 0.2972555019901694, - "nauc_ndcg_at_10_diff1": 0.5968390935433466, - "nauc_ndcg_at_10_max": 0.4972020381548397, - "nauc_ndcg_at_10_std": 0.276970819778273, - "nauc_ndcg_at_1_diff1": 0.6881197967738875, - "nauc_ndcg_at_1_max": 0.5104387821019861, - "nauc_ndcg_at_1_std": 0.22996512608256198, - "nauc_ndcg_at_20_diff1": 0.5912904398741408, - "nauc_ndcg_at_20_max": 0.5077719844454441, - "nauc_ndcg_at_20_std": 0.2893097949822134, - "nauc_ndcg_at_3_diff1": 0.6254713111075045, - "nauc_ndcg_at_3_max": 0.49960017452239175, - "nauc_ndcg_at_3_std": 0.24972205479769516, - "nauc_ndcg_at_5_diff1": 0.6069956168160815, - "nauc_ndcg_at_5_max": 0.48982469778813303, - "nauc_ndcg_at_5_std": 0.2616678203416795, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.42251653533438577, - "nauc_precision_at_100_max": 0.5974632567614774, - "nauc_precision_at_100_std": 0.5798983552509408, - "nauc_precision_at_10_diff1": 0.49655654014079986, - "nauc_precision_at_10_max": 0.488409398113342, - "nauc_precision_at_10_std": 0.3476667895125617, - "nauc_precision_at_1_diff1": 0.6881197967738875, - "nauc_precision_at_1_max": 0.5104387821019861, - "nauc_precision_at_1_std": 0.22996512608256198, - "nauc_precision_at_20_diff1": 0.45795692293377344, - "nauc_precision_at_20_max": 0.5357361507438838, - "nauc_precision_at_20_std": 0.4160858817582251, - "nauc_precision_at_3_diff1": 0.5826564933802196, - "nauc_precision_at_3_max": 0.49158910788383625, - "nauc_precision_at_3_std": 0.2651388501392188, - "nauc_precision_at_5_diff1": 0.5361007340951345, - "nauc_precision_at_5_max": 0.46650003271004487, - "nauc_precision_at_5_std": 0.29467856997063324, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.42251653533438616, - "nauc_recall_at_100_max": 0.5974632567614767, - "nauc_recall_at_100_std": 0.5798983552509402, - "nauc_recall_at_10_diff1": 0.4965565401407999, - "nauc_recall_at_10_max": 0.4884093981133426, - "nauc_recall_at_10_std": 0.3476667895125619, - "nauc_recall_at_1_diff1": 0.6881197967738875, - "nauc_recall_at_1_max": 0.5104387821019861, - "nauc_recall_at_1_std": 0.22996512608256198, - "nauc_recall_at_20_diff1": 0.4579569229337732, - "nauc_recall_at_20_max": 0.5357361507438837, - "nauc_recall_at_20_std": 0.41608588175822514, - "nauc_recall_at_3_diff1": 0.5826564933802192, - "nauc_recall_at_3_max": 0.4915891078838359, - "nauc_recall_at_3_std": 0.26513885013921823, - "nauc_recall_at_5_diff1": 0.5361007340951345, - "nauc_recall_at_5_max": 0.46650003271004464, - "nauc_recall_at_5_std": 0.29467856997063335, - "ndcg_at_1": 0.32884, - "ndcg_at_10": 0.44096, - "ndcg_at_100": 0.49016, - "ndcg_at_1000": 0.51345, - "ndcg_at_20": 0.461, - "ndcg_at_3": 0.3898, - "ndcg_at_5": 0.41458, - "precision_at_1": 0.32884, - "precision_at_10": 0.05769, - "precision_at_100": 0.00816, - "precision_at_1000": 0.001, - "precision_at_20": 0.03277, - "precision_at_3": 0.14516, - "precision_at_5": 0.09899, - "recall_at_1": 0.32884, - "recall_at_10": 0.57688, - "recall_at_100": 0.81594, - "recall_at_1000": 1.0, - "recall_at_20": 0.65544, - "recall_at_3": 0.43547, - "recall_at_5": 0.49495 - }, - { - "cv_recall_at_1": 0.11754, - "cv_recall_at_10": 0.2614, - "cv_recall_at_100": 0.56667, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.31228, - "cv_recall_at_3": 0.16491, - "cv_recall_at_5": 0.20351, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.17967, - "map_at_1": 0.11754, - "map_at_10": 0.15478, - "map_at_100": 0.16417, - "map_at_1000": 0.16616, - "map_at_20": 0.15821, - "map_at_3": 0.1383, - "map_at_5": 0.14716, - "mrr_at_1": 0.11754385964912281, - "mrr_at_10": 0.1547827903091061, - "mrr_at_100": 0.16417160639151848, - "mrr_at_1000": 0.16615976090692203, - "mrr_at_20": 0.15821471681867644, - "mrr_at_3": 0.13830409356725143, - "mrr_at_5": 0.14716374269005847, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.20686636279942508, - "nauc_cv_recall_at_100_max": 0.17926244045364328, - "nauc_cv_recall_at_100_std": 0.38626426415880044, - "nauc_cv_recall_at_10_diff1": 0.3603919019081563, - "nauc_cv_recall_at_10_max": 0.1680027725567475, - "nauc_cv_recall_at_10_std": 0.33617250564756646, - "nauc_cv_recall_at_1_diff1": 0.6064805637913367, - "nauc_cv_recall_at_1_max": 0.2527065611174607, - "nauc_cv_recall_at_1_std": 0.3772990481171108, - "nauc_cv_recall_at_20_diff1": 0.29609637486864593, - "nauc_cv_recall_at_20_max": 0.1845928383275292, - "nauc_cv_recall_at_20_std": 0.3293436311630296, - "nauc_cv_recall_at_3_diff1": 0.4800736209246735, - "nauc_cv_recall_at_3_max": 0.19174365694492915, - "nauc_cv_recall_at_3_std": 0.3175182241421318, - "nauc_cv_recall_at_5_diff1": 0.40168845928793273, - "nauc_cv_recall_at_5_max": 0.2056786469820408, - "nauc_cv_recall_at_5_std": 0.32768263041814816, - "nauc_map_at_1000_diff1": 0.49721183191633966, - "nauc_map_at_1000_max": 0.21530537794659496, - "nauc_map_at_1000_std": 0.3539650055758818, - "nauc_map_at_100_diff1": 0.49620795978310384, - "nauc_map_at_100_max": 0.21509618634759792, - "nauc_map_at_100_std": 0.3542543673006248, - "nauc_map_at_10_diff1": 0.5090810244617068, - "nauc_map_at_10_max": 0.21482663598645924, - "nauc_map_at_10_std": 0.3526557190520316, - "nauc_map_at_1_diff1": 0.6064805637913367, - "nauc_map_at_1_max": 0.2527065611174607, - "nauc_map_at_1_std": 0.3772990481171108, - "nauc_map_at_20_diff1": 0.5016662903455533, - "nauc_map_at_20_max": 0.21638144190481617, - "nauc_map_at_20_std": 0.352290307891851, - "nauc_map_at_3_diff1": 0.5444623926108907, - "nauc_map_at_3_max": 0.2171326896610425, - "nauc_map_at_3_std": 0.3464784007620425, - "nauc_map_at_5_diff1": 0.5186026114663759, - "nauc_map_at_5_max": 0.21988382288719446, - "nauc_map_at_5_std": 0.3488387290105743, - "nauc_mrr_at_1000_diff1": 0.49721183191633966, - "nauc_mrr_at_1000_max": 0.21530537794659496, - "nauc_mrr_at_1000_std": 0.3539650055758818, - "nauc_mrr_at_100_diff1": 0.49620795978310384, - "nauc_mrr_at_100_max": 0.21509618634759792, - "nauc_mrr_at_100_std": 0.3542543673006248, - "nauc_mrr_at_10_diff1": 0.5090810244617068, - "nauc_mrr_at_10_max": 0.21482663598645924, - "nauc_mrr_at_10_std": 0.3526557190520316, - "nauc_mrr_at_1_diff1": 0.6064805637913367, - "nauc_mrr_at_1_max": 0.2527065611174607, - "nauc_mrr_at_1_std": 0.3772990481171108, - "nauc_mrr_at_20_diff1": 0.5016662903455533, - "nauc_mrr_at_20_max": 0.21638144190481617, - "nauc_mrr_at_20_std": 0.352290307891851, - "nauc_mrr_at_3_diff1": 0.5444623926108907, - "nauc_mrr_at_3_max": 0.2171326896610425, - "nauc_mrr_at_3_std": 0.3464784007620425, - "nauc_mrr_at_5_diff1": 0.5186026114663759, - "nauc_mrr_at_5_max": 0.21988382288719446, - "nauc_mrr_at_5_std": 0.3488387290105743, - "nauc_ndcg_at_1000_diff1": 0.45255995578858577, - "nauc_ndcg_at_1000_max": 0.20829872016114442, - "nauc_ndcg_at_1000_std": 0.35374196329629476, - "nauc_ndcg_at_100_diff1": 0.4167023841930697, - "nauc_ndcg_at_100_max": 0.20346728060732097, - "nauc_ndcg_at_100_std": 0.35912915345884755, - "nauc_ndcg_at_10_diff1": 0.46408001594108844, - "nauc_ndcg_at_10_max": 0.20122977875141929, - "nauc_ndcg_at_10_std": 0.34759731100350105, - "nauc_ndcg_at_1_diff1": 0.6064805637913367, - "nauc_ndcg_at_1_max": 0.2527065611174607, - "nauc_ndcg_at_1_std": 0.3772990481171108, - "nauc_ndcg_at_20_diff1": 0.44021005808238023, - "nauc_ndcg_at_20_max": 0.2062442932637015, - "nauc_ndcg_at_20_std": 0.3455111095383758, - "nauc_ndcg_at_3_diff1": 0.5263189012349082, - "nauc_ndcg_at_3_max": 0.20975189907779873, - "nauc_ndcg_at_3_std": 0.3382583344210606, - "nauc_ndcg_at_5_diff1": 0.4842182837146837, - "nauc_ndcg_at_5_max": 0.21512998355498653, - "nauc_ndcg_at_5_std": 0.34229115985178443, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.20686636279942525, - "nauc_precision_at_100_max": 0.17926244045364373, - "nauc_precision_at_100_std": 0.38626426415880066, - "nauc_precision_at_10_diff1": 0.3603919019081564, - "nauc_precision_at_10_max": 0.16800277255674756, - "nauc_precision_at_10_std": 0.3361725056475665, - "nauc_precision_at_1_diff1": 0.6064805637913367, - "nauc_precision_at_1_max": 0.2527065611174607, - "nauc_precision_at_1_std": 0.3772990481171108, - "nauc_precision_at_20_diff1": 0.29609637486864615, - "nauc_precision_at_20_max": 0.18459283832752935, - "nauc_precision_at_20_std": 0.3293436311630295, - "nauc_precision_at_3_diff1": 0.4800736209246737, - "nauc_precision_at_3_max": 0.19174365694492926, - "nauc_precision_at_3_std": 0.31751822414213193, - "nauc_precision_at_5_diff1": 0.40168845928793273, - "nauc_precision_at_5_max": 0.20567864698204089, - "nauc_precision_at_5_std": 0.32768263041814827, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.20686636279942508, - "nauc_recall_at_100_max": 0.17926244045364328, - "nauc_recall_at_100_std": 0.38626426415880044, - "nauc_recall_at_10_diff1": 0.3603919019081563, - "nauc_recall_at_10_max": 0.1680027725567475, - "nauc_recall_at_10_std": 0.33617250564756646, - "nauc_recall_at_1_diff1": 0.6064805637913367, - "nauc_recall_at_1_max": 0.2527065611174607, - "nauc_recall_at_1_std": 0.3772990481171108, - "nauc_recall_at_20_diff1": 0.29609637486864593, - "nauc_recall_at_20_max": 0.1845928383275292, - "nauc_recall_at_20_std": 0.3293436311630296, - "nauc_recall_at_3_diff1": 0.4800736209246735, - "nauc_recall_at_3_max": 0.19174365694492915, - "nauc_recall_at_3_std": 0.3175182241421318, - "nauc_recall_at_5_diff1": 0.40168845928793273, - "nauc_recall_at_5_max": 0.2056786469820408, - "nauc_recall_at_5_std": 0.32768263041814816, - "ndcg_at_1": 0.11754, - "ndcg_at_10": 0.17967, - "ndcg_at_100": 0.23847, - "ndcg_at_1000": 0.29349, - "ndcg_at_20": 0.19242, - "ndcg_at_3": 0.14513, - "ndcg_at_5": 0.16106, - "precision_at_1": 0.11754, - "precision_at_10": 0.02614, - "precision_at_100": 0.00567, - "precision_at_1000": 0.001, - "precision_at_20": 0.01561, - "precision_at_3": 0.05497, - "precision_at_5": 0.0407, - "recall_at_1": 0.11754, - "recall_at_10": 0.2614, - "recall_at_100": 0.56667, - "recall_at_1000": 1.0, - "recall_at_20": 0.31228, - "recall_at_3": 0.16491, - "recall_at_5": 0.20351 - }, - { - "cv_recall_at_1": 0.17506, - "cv_recall_at_10": 0.35126, - "cv_recall_at_100": 0.67849, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.44394, - "cv_recall_at_3": 0.246, - "cv_recall_at_5": 0.28375, - "hf_subset": "et", - "languages": [ - "est-Latn" - ], - "main_score": 0.2524, - "map_at_1": 0.17506, - "map_at_10": 0.22221, - "map_at_100": 0.2342, - "map_at_1000": 0.23573, - "map_at_20": 0.22847, - "map_at_3": 0.205, - "map_at_5": 0.21381, - "mrr_at_1": 0.17505720823798626, - "mrr_at_10": 0.2222063310450038, - "mrr_at_100": 0.23420477050876534, - "mrr_at_1000": 0.2357283425713766, - "mrr_at_20": 0.22847110561344786, - "mrr_at_3": 0.20499618611746767, - "mrr_at_5": 0.21380625476735324, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.31030606229731295, - "nauc_cv_recall_at_100_max": 0.4920183843935835, - "nauc_cv_recall_at_100_std": 0.4214345474136694, - "nauc_cv_recall_at_10_diff1": 0.4319578595158496, - "nauc_cv_recall_at_10_max": 0.46680225532674047, - "nauc_cv_recall_at_10_std": 0.3069335988465379, - "nauc_cv_recall_at_1_diff1": 0.662791905035046, - "nauc_cv_recall_at_1_max": 0.5136669938374341, - "nauc_cv_recall_at_1_std": 0.22883573101962362, - "nauc_cv_recall_at_20_diff1": 0.38988749437635895, - "nauc_cv_recall_at_20_max": 0.4800254245971985, - "nauc_cv_recall_at_20_std": 0.3312341215165486, - "nauc_cv_recall_at_3_diff1": 0.5405291010324115, - "nauc_cv_recall_at_3_max": 0.4954257412448969, - "nauc_cv_recall_at_3_std": 0.28238790805278113, - "nauc_cv_recall_at_5_diff1": 0.4673198702095926, - "nauc_cv_recall_at_5_max": 0.4819739132028776, - "nauc_cv_recall_at_5_std": 0.30321271891046175, - "nauc_map_at_1000_diff1": 0.5753951970120799, - "nauc_map_at_1000_max": 0.49867755908541817, - "nauc_map_at_1000_std": 0.2629204728875237, - "nauc_map_at_100_diff1": 0.5742791463806061, - "nauc_map_at_100_max": 0.4987120095894289, - "nauc_map_at_100_std": 0.2636667111263593, - "nauc_map_at_10_diff1": 0.5789104428111786, - "nauc_map_at_10_max": 0.4978836819985857, - "nauc_map_at_10_std": 0.2600144113637667, - "nauc_map_at_1_diff1": 0.662791905035046, - "nauc_map_at_1_max": 0.5136669938374341, - "nauc_map_at_1_std": 0.22883573101962362, - "nauc_map_at_20_diff1": 0.5755538396436413, - "nauc_map_at_20_max": 0.49831743106828436, - "nauc_map_at_20_std": 0.26198591051716935, - "nauc_map_at_3_diff1": 0.6059299307856398, - "nauc_map_at_3_max": 0.5053408422172819, - "nauc_map_at_3_std": 0.25371863899923963, - "nauc_map_at_5_diff1": 0.5850701872196608, - "nauc_map_at_5_max": 0.5014935376584141, - "nauc_map_at_5_std": 0.2595483491706154, - "nauc_mrr_at_1000_diff1": 0.5753951970120799, - "nauc_mrr_at_1000_max": 0.49867755908541817, - "nauc_mrr_at_1000_std": 0.2629204728875237, - "nauc_mrr_at_100_diff1": 0.5742791463806061, - "nauc_mrr_at_100_max": 0.4987120095894289, - "nauc_mrr_at_100_std": 0.2636667111263593, - "nauc_mrr_at_10_diff1": 0.5789104428111786, - "nauc_mrr_at_10_max": 0.4978836819985857, - "nauc_mrr_at_10_std": 0.2600144113637667, - "nauc_mrr_at_1_diff1": 0.662791905035046, - "nauc_mrr_at_1_max": 0.5136669938374341, - "nauc_mrr_at_1_std": 0.22883573101962362, - "nauc_mrr_at_20_diff1": 0.5755538396436413, - "nauc_mrr_at_20_max": 0.49831743106828436, - "nauc_mrr_at_20_std": 0.26198591051716935, - "nauc_mrr_at_3_diff1": 0.6059299307856398, - "nauc_mrr_at_3_max": 0.5053408422172819, - "nauc_mrr_at_3_std": 0.25371863899923963, - "nauc_mrr_at_5_diff1": 0.5850701872196608, - "nauc_mrr_at_5_max": 0.5014935376584141, - "nauc_mrr_at_5_std": 0.2595483491706154, - "nauc_ndcg_at_1000_diff1": 0.546210016037982, - "nauc_ndcg_at_1000_max": 0.49504877252113694, - "nauc_ndcg_at_1000_std": 0.27535570292149614, - "nauc_ndcg_at_100_diff1": 0.5185344152265912, - "nauc_ndcg_at_100_max": 0.4948560103106455, - "nauc_ndcg_at_100_std": 0.29277026295247915, - "nauc_ndcg_at_10_diff1": 0.5383424122759396, - "nauc_ndcg_at_10_max": 0.4892576392717773, - "nauc_ndcg_at_10_std": 0.2731410798077506, - "nauc_ndcg_at_1_diff1": 0.662791905035046, - "nauc_ndcg_at_1_max": 0.5136669938374341, - "nauc_ndcg_at_1_std": 0.22883573101962362, - "nauc_ndcg_at_20_diff1": 0.5261286547319675, - "nauc_ndcg_at_20_max": 0.49228912059389557, - "nauc_ndcg_at_20_std": 0.2801804838120073, - "nauc_ndcg_at_3_diff1": 0.5879478465311763, - "nauc_ndcg_at_3_max": 0.5026208826663386, - "nauc_ndcg_at_3_std": 0.2616003966411608, - "nauc_ndcg_at_5_diff1": 0.5526361128325122, - "nauc_ndcg_at_5_max": 0.4961346125428329, - "nauc_ndcg_at_5_std": 0.2716265448866012, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3103060622973135, - "nauc_precision_at_100_max": 0.49201838439358375, - "nauc_precision_at_100_std": 0.42143454741367004, - "nauc_precision_at_10_diff1": 0.43195785951584975, - "nauc_precision_at_10_max": 0.46680225532674036, - "nauc_precision_at_10_std": 0.3069335988465379, - "nauc_precision_at_1_diff1": 0.662791905035046, - "nauc_precision_at_1_max": 0.5136669938374341, - "nauc_precision_at_1_std": 0.22883573101962362, - "nauc_precision_at_20_diff1": 0.3898874943763586, - "nauc_precision_at_20_max": 0.4800254245971982, - "nauc_precision_at_20_std": 0.3312341215165482, - "nauc_precision_at_3_diff1": 0.5405291010324113, - "nauc_precision_at_3_max": 0.49542574124489663, - "nauc_precision_at_3_std": 0.28238790805278136, - "nauc_precision_at_5_diff1": 0.4673198702095924, - "nauc_precision_at_5_max": 0.4819739132028772, - "nauc_precision_at_5_std": 0.30321271891046164, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.31030606229731295, - "nauc_recall_at_100_max": 0.4920183843935835, - "nauc_recall_at_100_std": 0.4214345474136694, - "nauc_recall_at_10_diff1": 0.4319578595158496, - "nauc_recall_at_10_max": 0.46680225532674047, - "nauc_recall_at_10_std": 0.3069335988465379, - "nauc_recall_at_1_diff1": 0.662791905035046, - "nauc_recall_at_1_max": 0.5136669938374341, - "nauc_recall_at_1_std": 0.22883573101962362, - "nauc_recall_at_20_diff1": 0.38988749437635895, - "nauc_recall_at_20_max": 0.4800254245971985, - "nauc_recall_at_20_std": 0.3312341215165486, - "nauc_recall_at_3_diff1": 0.5405291010324115, - "nauc_recall_at_3_max": 0.4954257412448969, - "nauc_recall_at_3_std": 0.28238790805278113, - "nauc_recall_at_5_diff1": 0.4673198702095926, - "nauc_recall_at_5_max": 0.4819739132028776, - "nauc_recall_at_5_std": 0.30321271891046175, - "ndcg_at_1": 0.17506, - "ndcg_at_10": 0.2524, - "ndcg_at_100": 0.31861, - "ndcg_at_1000": 0.35964, - "ndcg_at_20": 0.27562, - "ndcg_at_3": 0.21547, - "ndcg_at_5": 0.23118, - "precision_at_1": 0.17506, - "precision_at_10": 0.03513, - "precision_at_100": 0.00678, - "precision_at_1000": 0.001, - "precision_at_20": 0.0222, - "precision_at_3": 0.082, - "precision_at_5": 0.05675, - "recall_at_1": 0.17506, - "recall_at_10": 0.35126, - "recall_at_100": 0.67849, - "recall_at_1000": 1.0, - "recall_at_20": 0.44394, - "recall_at_3": 0.246, - "recall_at_5": 0.28375 - }, - { - "cv_recall_at_1": 0.3818, - "cv_recall_at_10": 0.65816, - "cv_recall_at_100": 0.90455, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.75472, - "cv_recall_at_3": 0.51831, - "cv_recall_at_5": 0.58713, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.51299, - "map_at_1": 0.3818, - "map_at_10": 0.46725, - "map_at_100": 0.47806, - "map_at_1000": 0.47854, - "map_at_20": 0.47405, - "map_at_3": 0.44192, - "map_at_5": 0.45762, - "mrr_at_1": 0.38179800221975585, - "mrr_at_10": 0.467253756848651, - "mrr_at_100": 0.4780623138117509, - "mrr_at_1000": 0.478538713382236, - "mrr_at_20": 0.4740469779162587, - "mrr_at_3": 0.4419163891971881, - "mrr_at_5": 0.4576211616722157, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5265912201308302, - "nauc_cv_recall_at_100_max": 0.6336587563293079, - "nauc_cv_recall_at_100_std": 0.6311235478687189, - "nauc_cv_recall_at_10_diff1": 0.5851689002249301, - "nauc_cv_recall_at_10_max": 0.4485733805721923, - "nauc_cv_recall_at_10_std": 0.3718632734056777, - "nauc_cv_recall_at_1_diff1": 0.7137156779886052, - "nauc_cv_recall_at_1_max": 0.45559066885439636, - "nauc_cv_recall_at_1_std": 0.21835400435692623, - "nauc_cv_recall_at_20_diff1": 0.533567046952136, - "nauc_cv_recall_at_20_max": 0.484551209017695, - "nauc_cv_recall_at_20_std": 0.47478384921943123, - "nauc_cv_recall_at_3_diff1": 0.6285924276187039, - "nauc_cv_recall_at_3_max": 0.44391634092707977, - "nauc_cv_recall_at_3_std": 0.3112414211969393, - "nauc_cv_recall_at_5_diff1": 0.6021358367466351, - "nauc_cv_recall_at_5_max": 0.4355454957871355, - "nauc_cv_recall_at_5_std": 0.3236237251547708, - "nauc_map_at_1000_diff1": 0.6697722369919447, - "nauc_map_at_1000_max": 0.4508071898827599, - "nauc_map_at_1000_std": 0.2661928201749134, - "nauc_map_at_100_diff1": 0.6695552441716708, - "nauc_map_at_100_max": 0.4510068260181231, - "nauc_map_at_100_std": 0.2666357600178505, - "nauc_map_at_10_diff1": 0.670017679250379, - "nauc_map_at_10_max": 0.44799842203040807, - "nauc_map_at_10_std": 0.2633449828707209, - "nauc_map_at_1_diff1": 0.7137156779886052, - "nauc_map_at_1_max": 0.45559066885439636, - "nauc_map_at_1_std": 0.21835400435692623, - "nauc_map_at_20_diff1": 0.6686318875348207, - "nauc_map_at_20_max": 0.44973314463975794, - "nauc_map_at_20_std": 0.2670816271196027, - "nauc_map_at_3_diff1": 0.6759346113098538, - "nauc_map_at_3_max": 0.4483948069723216, - "nauc_map_at_3_std": 0.25772030300263965, - "nauc_map_at_5_diff1": 0.6710255392841811, - "nauc_map_at_5_max": 0.4466274421733826, - "nauc_map_at_5_std": 0.25962165465555503, - "nauc_mrr_at_1000_diff1": 0.6697719406219865, - "nauc_mrr_at_1000_max": 0.4508059845394589, - "nauc_mrr_at_1000_std": 0.2661912917368203, - "nauc_mrr_at_100_diff1": 0.6695549610314403, - "nauc_mrr_at_100_max": 0.45100559857467465, - "nauc_mrr_at_100_std": 0.26663420897995804, - "nauc_mrr_at_10_diff1": 0.670017679250379, - "nauc_mrr_at_10_max": 0.44799842203040807, - "nauc_mrr_at_10_std": 0.2633449828707209, - "nauc_mrr_at_1_diff1": 0.7137156779886052, - "nauc_mrr_at_1_max": 0.45559066885439636, - "nauc_mrr_at_1_std": 0.21835400435692623, - "nauc_mrr_at_20_diff1": 0.6686318875348207, - "nauc_mrr_at_20_max": 0.44973314463975794, - "nauc_mrr_at_20_std": 0.2670816271196027, - "nauc_mrr_at_3_diff1": 0.6759346113098538, - "nauc_mrr_at_3_max": 0.4483948069723216, - "nauc_mrr_at_3_std": 0.25772030300263965, - "nauc_mrr_at_5_diff1": 0.6710255392841811, - "nauc_mrr_at_5_max": 0.4466274421733826, - "nauc_mrr_at_5_std": 0.25962165465555503, - "nauc_ndcg_at_1000_diff1": 0.65934330876716, - "nauc_ndcg_at_1000_max": 0.45495571849239214, - "nauc_ndcg_at_1000_std": 0.28259581269143613, - "nauc_ndcg_at_100_diff1": 0.6536109125984344, - "nauc_ndcg_at_100_max": 0.46218844729330094, - "nauc_ndcg_at_100_std": 0.29694705857691633, - "nauc_ndcg_at_10_diff1": 0.6517697877827734, - "nauc_ndcg_at_10_max": 0.4476652993910479, - "nauc_ndcg_at_10_std": 0.2858562756359774, - "nauc_ndcg_at_1_diff1": 0.7137156779886052, - "nauc_ndcg_at_1_max": 0.45559066885439636, - "nauc_ndcg_at_1_std": 0.21835400435692623, - "nauc_ndcg_at_20_diff1": 0.6457801636805331, - "nauc_ndcg_at_20_max": 0.4541692965534771, - "nauc_ndcg_at_20_std": 0.3004166175153123, - "nauc_ndcg_at_3_diff1": 0.663972432848074, - "nauc_ndcg_at_3_max": 0.44717519330462746, - "nauc_ndcg_at_3_std": 0.2711624329493742, - "nauc_ndcg_at_5_diff1": 0.6547094368277582, - "nauc_ndcg_at_5_max": 0.44396849750140915, - "nauc_ndcg_at_5_std": 0.27489677323503187, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.5265912201308315, - "nauc_precision_at_100_max": 0.6336587563293098, - "nauc_precision_at_100_std": 0.6311235478687203, - "nauc_precision_at_10_diff1": 0.5851689002249305, - "nauc_precision_at_10_max": 0.4485733805721921, - "nauc_precision_at_10_std": 0.3718632734056779, - "nauc_precision_at_1_diff1": 0.7137156779886052, - "nauc_precision_at_1_max": 0.45559066885439636, - "nauc_precision_at_1_std": 0.21835400435692623, - "nauc_precision_at_20_diff1": 0.5335670469521362, - "nauc_precision_at_20_max": 0.4845512090176942, - "nauc_precision_at_20_std": 0.47478384921943095, - "nauc_precision_at_3_diff1": 0.6285924276187037, - "nauc_precision_at_3_max": 0.4439163409270798, - "nauc_precision_at_3_std": 0.31124142119693926, - "nauc_precision_at_5_diff1": 0.6021358367466348, - "nauc_precision_at_5_max": 0.4355454957871354, - "nauc_precision_at_5_std": 0.3236237251547704, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5265912201308302, - "nauc_recall_at_100_max": 0.6336587563293079, - "nauc_recall_at_100_std": 0.6311235478687189, - "nauc_recall_at_10_diff1": 0.5851689002249301, - "nauc_recall_at_10_max": 0.4485733805721923, - "nauc_recall_at_10_std": 0.3718632734056777, - "nauc_recall_at_1_diff1": 0.7137156779886052, - "nauc_recall_at_1_max": 0.45559066885439636, - "nauc_recall_at_1_std": 0.21835400435692623, - "nauc_recall_at_20_diff1": 0.533567046952136, - "nauc_recall_at_20_max": 0.484551209017695, - "nauc_recall_at_20_std": 0.47478384921943123, - "nauc_recall_at_3_diff1": 0.6285924276187039, - "nauc_recall_at_3_max": 0.44391634092707977, - "nauc_recall_at_3_std": 0.3112414211969393, - "nauc_recall_at_5_diff1": 0.6021358367466351, - "nauc_recall_at_5_max": 0.4355454957871355, - "nauc_recall_at_5_std": 0.3236237251547708, - "ndcg_at_1": 0.3818, - "ndcg_at_10": 0.51299, - "ndcg_at_100": 0.56564, - "ndcg_at_1000": 0.57796, - "ndcg_at_20": 0.53752, - "ndcg_at_3": 0.46154, - "ndcg_at_5": 0.48986, - "precision_at_1": 0.3818, - "precision_at_10": 0.06582, - "precision_at_100": 0.00905, - "precision_at_1000": 0.001, - "precision_at_20": 0.03774, - "precision_at_3": 0.17277, - "precision_at_5": 0.11743, - "recall_at_1": 0.3818, - "recall_at_10": 0.65816, - "recall_at_100": 0.90455, - "recall_at_1000": 1.0, - "recall_at_20": 0.75472, - "recall_at_3": 0.51831, - "recall_at_5": 0.58713 - }, - { - "cv_recall_at_1": 0.07841, - "cv_recall_at_10": 0.21805, - "cv_recall_at_100": 0.47691, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.2739, - "cv_recall_at_3": 0.12137, - "cv_recall_at_5": 0.15897, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.13717, - "map_at_1": 0.07841, - "map_at_10": 0.11268, - "map_at_100": 0.12105, - "map_at_1000": 0.12309, - "map_at_20": 0.11648, - "map_at_3": 0.09649, - "map_at_5": 0.10487, - "mrr_at_1": 0.07841031149301826, - "mrr_at_10": 0.11267837962252575, - "mrr_at_100": 0.12104804694712606, - "mrr_at_1000": 0.12308794285824537, - "mrr_at_20": 0.11648311700466915, - "mrr_at_3": 0.09649122807017542, - "mrr_at_5": 0.10486931614751165, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.18453188206043467, - "nauc_cv_recall_at_100_max": 0.16392815605857672, - "nauc_cv_recall_at_100_std": 0.3509105591030677, - "nauc_cv_recall_at_10_diff1": 0.31782919395535686, - "nauc_cv_recall_at_10_max": 0.26121048660827145, - "nauc_cv_recall_at_10_std": 0.3216704847714983, - "nauc_cv_recall_at_1_diff1": 0.7300323741152004, - "nauc_cv_recall_at_1_max": 0.5875499383325973, - "nauc_cv_recall_at_1_std": 0.4395028543714588, - "nauc_cv_recall_at_20_diff1": 0.2850540894070008, - "nauc_cv_recall_at_20_max": 0.25926111017353165, - "nauc_cv_recall_at_20_std": 0.35138498307149507, - "nauc_cv_recall_at_3_diff1": 0.52950899182385, - "nauc_cv_recall_at_3_max": 0.40695540700911886, - "nauc_cv_recall_at_3_std": 0.37230982704145404, - "nauc_cv_recall_at_5_diff1": 0.44363573917477916, - "nauc_cv_recall_at_5_max": 0.31469487529160706, - "nauc_cv_recall_at_5_std": 0.33104107124420945, - "nauc_map_at_1000_diff1": 0.5460353243518522, - "nauc_map_at_1000_max": 0.42919167158826854, - "nauc_map_at_1000_std": 0.3813551521426623, - "nauc_map_at_100_diff1": 0.544390937711658, - "nauc_map_at_100_max": 0.42768712328820707, - "nauc_map_at_100_std": 0.38100639101593836, - "nauc_map_at_10_diff1": 0.5574473549510108, - "nauc_map_at_10_max": 0.43533414024358197, - "nauc_map_at_10_std": 0.3789419393898596, - "nauc_map_at_1_diff1": 0.7300323741152004, - "nauc_map_at_1_max": 0.5875499383325973, - "nauc_map_at_1_std": 0.4395028543714588, - "nauc_map_at_20_diff1": 0.5508909279002877, - "nauc_map_at_20_max": 0.43386475681313386, - "nauc_map_at_20_std": 0.382263436012735, - "nauc_map_at_3_diff1": 0.6223212772454054, - "nauc_map_at_3_max": 0.48578152509206257, - "nauc_map_at_3_std": 0.39982562617190115, - "nauc_map_at_5_diff1": 0.591480016688883, - "nauc_map_at_5_max": 0.4531842521492073, - "nauc_map_at_5_std": 0.384540400987139, - "nauc_mrr_at_1000_diff1": 0.5460314797682349, - "nauc_mrr_at_1000_max": 0.4291924596284079, - "nauc_mrr_at_1000_std": 0.3813545092878323, - "nauc_mrr_at_100_diff1": 0.5443871148859052, - "nauc_mrr_at_100_max": 0.4276879060108006, - "nauc_mrr_at_100_std": 0.3810057517648782, - "nauc_mrr_at_10_diff1": 0.5574473549510108, - "nauc_mrr_at_10_max": 0.43533414024358197, - "nauc_mrr_at_10_std": 0.3789419393898596, - "nauc_mrr_at_1_diff1": 0.7300323741152004, - "nauc_mrr_at_1_max": 0.5875499383325973, - "nauc_mrr_at_1_std": 0.4395028543714588, - "nauc_mrr_at_20_diff1": 0.5508909279002877, - "nauc_mrr_at_20_max": 0.43386475681313386, - "nauc_mrr_at_20_std": 0.382263436012735, - "nauc_mrr_at_3_diff1": 0.6223212772454054, - "nauc_mrr_at_3_max": 0.48578152509206257, - "nauc_mrr_at_3_std": 0.39982562617190115, - "nauc_mrr_at_5_diff1": 0.591480016688883, - "nauc_mrr_at_5_max": 0.4531842521492073, - "nauc_mrr_at_5_std": 0.384540400987139, - "nauc_ndcg_at_1000_diff1": 0.4788419648481831, - "nauc_ndcg_at_1000_max": 0.38079278911374764, - "nauc_ndcg_at_1000_std": 0.3716402728056327, - "nauc_ndcg_at_100_diff1": 0.42588088426408005, - "nauc_ndcg_at_100_max": 0.3404479375921339, - "nauc_ndcg_at_100_std": 0.36683836687852833, - "nauc_ndcg_at_10_diff1": 0.47794237369134523, - "nauc_ndcg_at_10_max": 0.37645783480827505, - "nauc_ndcg_at_10_std": 0.35898182581341265, - "nauc_ndcg_at_1_diff1": 0.7300323741152004, - "nauc_ndcg_at_1_max": 0.5875499383325973, - "nauc_ndcg_at_1_std": 0.4395028543714588, - "nauc_ndcg_at_20_diff1": 0.4598258570443921, - "nauc_ndcg_at_20_max": 0.37237818193500444, - "nauc_ndcg_at_20_std": 0.369399984258033, - "nauc_ndcg_at_3_diff1": 0.5948678284793605, - "nauc_ndcg_at_3_max": 0.4622721642364097, - "nauc_ndcg_at_3_std": 0.39154133751679404, - "nauc_ndcg_at_5_diff1": 0.5452122159932722, - "nauc_ndcg_at_5_max": 0.40980433669295235, - "nauc_ndcg_at_5_std": 0.36767612935248906, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.18453188206043514, - "nauc_precision_at_100_max": 0.163928156058577, - "nauc_precision_at_100_std": 0.35091055910306784, - "nauc_precision_at_10_diff1": 0.31782919395535664, - "nauc_precision_at_10_max": 0.26121048660827145, - "nauc_precision_at_10_std": 0.3216704847714983, - "nauc_precision_at_1_diff1": 0.7300323741152004, - "nauc_precision_at_1_max": 0.5875499383325973, - "nauc_precision_at_1_std": 0.4395028543714588, - "nauc_precision_at_20_diff1": 0.28505408940700105, - "nauc_precision_at_20_max": 0.2592611101735318, - "nauc_precision_at_20_std": 0.3513849830714949, - "nauc_precision_at_3_diff1": 0.5295089918238496, - "nauc_precision_at_3_max": 0.4069554070091187, - "nauc_precision_at_3_std": 0.37230982704145404, - "nauc_precision_at_5_diff1": 0.4436357391747791, - "nauc_precision_at_5_max": 0.3146948752916072, - "nauc_precision_at_5_std": 0.3310410712442096, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.18453188206043467, - "nauc_recall_at_100_max": 0.16392815605857672, - "nauc_recall_at_100_std": 0.3509105591030677, - "nauc_recall_at_10_diff1": 0.31782919395535686, - "nauc_recall_at_10_max": 0.26121048660827145, - "nauc_recall_at_10_std": 0.3216704847714983, - "nauc_recall_at_1_diff1": 0.7300323741152004, - "nauc_recall_at_1_max": 0.5875499383325973, - "nauc_recall_at_1_std": 0.4395028543714588, - "nauc_recall_at_20_diff1": 0.2850540894070008, - "nauc_recall_at_20_max": 0.25926111017353165, - "nauc_recall_at_20_std": 0.35138498307149507, - "nauc_recall_at_3_diff1": 0.52950899182385, - "nauc_recall_at_3_max": 0.40695540700911886, - "nauc_recall_at_3_std": 0.37230982704145404, - "nauc_recall_at_5_diff1": 0.44363573917477916, - "nauc_recall_at_5_max": 0.31469487529160706, - "nauc_recall_at_5_std": 0.33104107124420945, - "ndcg_at_1": 0.07841, - "ndcg_at_10": 0.13717, - "ndcg_at_100": 0.18755, - "ndcg_at_1000": 0.25155, - "ndcg_at_20": 0.1512, - "ndcg_at_3": 0.10285, - "ndcg_at_5": 0.11814, - "precision_at_1": 0.07841, - "precision_at_10": 0.0218, - "precision_at_100": 0.00477, - "precision_at_1000": 0.001, - "precision_at_20": 0.01369, - "precision_at_3": 0.04046, - "precision_at_5": 0.03179, - "recall_at_1": 0.07841, - "recall_at_10": 0.21805, - "recall_at_100": 0.47691, - "recall_at_1000": 1.0, - "recall_at_20": 0.2739, - "recall_at_3": 0.12137, - "recall_at_5": 0.15897 - }, - { - "cv_recall_at_1": 0.087, - "cv_recall_at_10": 0.251, - "cv_recall_at_100": 0.563, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.313, - "cv_recall_at_3": 0.164, - "cv_recall_at_5": 0.191, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.16084, - "map_at_1": 0.087, - "map_at_10": 0.13302, - "map_at_100": 0.14278, - "map_at_1000": 0.14468, - "map_at_20": 0.13725, - "map_at_3": 0.119, - "map_at_5": 0.12515, - "mrr_at_1": 0.087, - "mrr_at_10": 0.1330154761904761, - "mrr_at_100": 0.14277627814186838, - "mrr_at_1000": 0.14467822399493577, - "mrr_at_20": 0.1372495060022536, - "mrr_at_3": 0.11899999999999991, - "mrr_at_5": 0.12514999999999993, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.1558876013096379, - "nauc_cv_recall_at_100_max": 0.13621597911643635, - "nauc_cv_recall_at_100_std": 0.22131963502183968, - "nauc_cv_recall_at_10_diff1": 0.3552955619811528, - "nauc_cv_recall_at_10_max": 0.08030756714447457, - "nauc_cv_recall_at_10_std": 0.14590495435100892, - "nauc_cv_recall_at_1_diff1": 0.5057347902383663, - "nauc_cv_recall_at_1_max": 0.1060512137277217, - "nauc_cv_recall_at_1_std": 0.03693766210535726, - "nauc_cv_recall_at_20_diff1": 0.3068987784462007, - "nauc_cv_recall_at_20_max": 0.09564743425371655, - "nauc_cv_recall_at_20_std": 0.13610007638906607, - "nauc_cv_recall_at_3_diff1": 0.4561053020422625, - "nauc_cv_recall_at_3_max": 0.055866321891938665, - "nauc_cv_recall_at_3_std": 0.17909752953327257, - "nauc_cv_recall_at_5_diff1": 0.442646228966321, - "nauc_cv_recall_at_5_max": 0.07753053012657463, - "nauc_cv_recall_at_5_std": 0.1616987359280104, - "nauc_map_at_1000_diff1": 0.44417160523549337, - "nauc_map_at_1000_max": 0.0852978835378124, - "nauc_map_at_1000_std": 0.10768953069112178, - "nauc_map_at_100_diff1": 0.4428536887062119, - "nauc_map_at_100_max": 0.08499196853620133, - "nauc_map_at_100_std": 0.10736629195957144, - "nauc_map_at_10_diff1": 0.45481166438543014, - "nauc_map_at_10_max": 0.08291759622326565, - "nauc_map_at_10_std": 0.10690073118185775, - "nauc_map_at_1_diff1": 0.5057347902383663, - "nauc_map_at_1_max": 0.1060512137277217, - "nauc_map_at_1_std": 0.03693766210535726, - "nauc_map_at_20_diff1": 0.44816652230523707, - "nauc_map_at_20_max": 0.08428473121701546, - "nauc_map_at_20_std": 0.10578701542925341, - "nauc_map_at_3_diff1": 0.4784349377587308, - "nauc_map_at_3_max": 0.07575787040300637, - "nauc_map_at_3_std": 0.11229478774442961, - "nauc_map_at_5_diff1": 0.47406143708766296, - "nauc_map_at_5_max": 0.08129161757667618, - "nauc_map_at_5_std": 0.1089115383035945, - "nauc_mrr_at_1000_diff1": 0.44417160523549337, - "nauc_mrr_at_1000_max": 0.0852978835378124, - "nauc_mrr_at_1000_std": 0.10768953069112178, - "nauc_mrr_at_100_diff1": 0.4428536887062119, - "nauc_mrr_at_100_max": 0.08499196853620133, - "nauc_mrr_at_100_std": 0.10736629195957144, - "nauc_mrr_at_10_diff1": 0.45481166438543014, - "nauc_mrr_at_10_max": 0.08291759622326565, - "nauc_mrr_at_10_std": 0.10690073118185775, - "nauc_mrr_at_1_diff1": 0.5057347902383663, - "nauc_mrr_at_1_max": 0.1060512137277217, - "nauc_mrr_at_1_std": 0.03693766210535726, - "nauc_mrr_at_20_diff1": 0.44816652230523707, - "nauc_mrr_at_20_max": 0.08428473121701546, - "nauc_mrr_at_20_std": 0.10578701542925341, - "nauc_mrr_at_3_diff1": 0.4784349377587308, - "nauc_mrr_at_3_max": 0.07575787040300637, - "nauc_mrr_at_3_std": 0.11229478774442961, - "nauc_mrr_at_5_diff1": 0.47406143708766296, - "nauc_mrr_at_5_max": 0.08129161757667618, - "nauc_mrr_at_5_std": 0.1089115383035945, - "nauc_ndcg_at_1000_diff1": 0.4087907427501082, - "nauc_ndcg_at_1000_max": 0.0912358458578291, - "nauc_ndcg_at_1000_std": 0.12493394003702137, - "nauc_ndcg_at_100_diff1": 0.368002561758682, - "nauc_ndcg_at_100_max": 0.09539143507732341, - "nauc_ndcg_at_100_std": 0.13604709329141815, - "nauc_ndcg_at_10_diff1": 0.4239977392449605, - "nauc_ndcg_at_10_max": 0.08170068898856558, - "nauc_ndcg_at_10_std": 0.12101181084019956, - "nauc_ndcg_at_1_diff1": 0.5057347902383663, - "nauc_ndcg_at_1_max": 0.1060512137277217, - "nauc_ndcg_at_1_std": 0.03693766210535726, - "nauc_ndcg_at_20_diff1": 0.4040731057585679, - "nauc_ndcg_at_20_max": 0.08686523266695671, - "nauc_ndcg_at_20_std": 0.11781622112525923, - "nauc_ndcg_at_3_diff1": 0.47157401991603676, - "nauc_ndcg_at_3_max": 0.06954207858405162, - "nauc_ndcg_at_3_std": 0.13271042137472033, - "nauc_ndcg_at_5_diff1": 0.4641953202178358, - "nauc_ndcg_at_5_max": 0.07954983760889357, - "nauc_ndcg_at_5_std": 0.12614454515428244, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.15588760130963777, - "nauc_precision_at_100_max": 0.13621597911643624, - "nauc_precision_at_100_std": 0.2213196350218393, - "nauc_precision_at_10_diff1": 0.3552955619811528, - "nauc_precision_at_10_max": 0.08030756714447448, - "nauc_precision_at_10_std": 0.14590495435100886, - "nauc_precision_at_1_diff1": 0.5057347902383663, - "nauc_precision_at_1_max": 0.1060512137277217, - "nauc_precision_at_1_std": 0.03693766210535726, - "nauc_precision_at_20_diff1": 0.30689877844620056, - "nauc_precision_at_20_max": 0.09564743425371641, - "nauc_precision_at_20_std": 0.13610007638906604, - "nauc_precision_at_3_diff1": 0.45610530204226246, - "nauc_precision_at_3_max": 0.05586632189193886, - "nauc_precision_at_3_std": 0.17909752953327254, - "nauc_precision_at_5_diff1": 0.4426462289663209, - "nauc_precision_at_5_max": 0.07753053012657454, - "nauc_precision_at_5_std": 0.16169873592801015, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.1558876013096379, - "nauc_recall_at_100_max": 0.13621597911643635, - "nauc_recall_at_100_std": 0.22131963502183968, - "nauc_recall_at_10_diff1": 0.3552955619811528, - "nauc_recall_at_10_max": 0.08030756714447457, - "nauc_recall_at_10_std": 0.14590495435100892, - "nauc_recall_at_1_diff1": 0.5057347902383663, - "nauc_recall_at_1_max": 0.1060512137277217, - "nauc_recall_at_1_std": 0.03693766210535726, - "nauc_recall_at_20_diff1": 0.3068987784462007, - "nauc_recall_at_20_max": 0.09564743425371655, - "nauc_recall_at_20_std": 0.13610007638906607, - "nauc_recall_at_3_diff1": 0.4561053020422625, - "nauc_recall_at_3_max": 0.055866321891938665, - "nauc_recall_at_3_std": 0.17909752953327257, - "nauc_recall_at_5_diff1": 0.442646228966321, - "nauc_recall_at_5_max": 0.07753053012657463, - "nauc_recall_at_5_std": 0.1616987359280104, - "ndcg_at_1": 0.087, - "ndcg_at_10": 0.16084, - "ndcg_at_100": 0.22102, - "ndcg_at_1000": 0.27573, - "ndcg_at_20": 0.17643, - "ndcg_at_3": 0.13048, - "ndcg_at_5": 0.14158, - "precision_at_1": 0.087, - "precision_at_10": 0.0251, - "precision_at_100": 0.00563, - "precision_at_1000": 0.001, - "precision_at_20": 0.01565, - "precision_at_3": 0.05467, - "precision_at_5": 0.0382, - "recall_at_1": 0.087, - "recall_at_10": 0.251, - "recall_at_100": 0.563, - "recall_at_1000": 1.0, - "recall_at_20": 0.313, - "recall_at_3": 0.164, - "recall_at_5": 0.191 - }, - { - "cv_recall_at_1": 0.32316, - "cv_recall_at_10": 0.57975, - "cv_recall_at_100": 0.84466, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.6491, - "cv_recall_at_3": 0.44521, - "cv_recall_at_5": 0.50208, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.44335, - "map_at_1": 0.32316, - "map_at_10": 0.40073, - "map_at_100": 0.41023, - "map_at_1000": 0.41109, - "map_at_20": 0.40551, - "map_at_3": 0.37748, - "map_at_5": 0.39025, - "mrr_at_1": 0.3231622746185853, - "mrr_at_10": 0.400730907249631, - "mrr_at_100": 0.41022696716485957, - "mrr_at_1000": 0.41109069877067983, - "mrr_at_20": 0.4055117706081922, - "mrr_at_3": 0.37748497457235325, - "mrr_at_5": 0.39024503005085515, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.48098495877756514, - "nauc_cv_recall_at_100_max": 0.5404607638853961, - "nauc_cv_recall_at_100_std": 0.5785820711736115, - "nauc_cv_recall_at_10_diff1": 0.5077985610322101, - "nauc_cv_recall_at_10_max": 0.52220082238744, - "nauc_cv_recall_at_10_std": 0.4673819128352976, - "nauc_cv_recall_at_1_diff1": 0.6603749690968012, - "nauc_cv_recall_at_1_max": 0.522301229855425, - "nauc_cv_recall_at_1_std": 0.35891873761023935, - "nauc_cv_recall_at_20_diff1": 0.47762772221834277, - "nauc_cv_recall_at_20_max": 0.5148646415990591, - "nauc_cv_recall_at_20_std": 0.5201794202680657, - "nauc_cv_recall_at_3_diff1": 0.5477584110243355, - "nauc_cv_recall_at_3_max": 0.5251152644999395, - "nauc_cv_recall_at_3_std": 0.43168441471824376, - "nauc_cv_recall_at_5_diff1": 0.5197125969701824, - "nauc_cv_recall_at_5_max": 0.5353503012783575, - "nauc_cv_recall_at_5_std": 0.43589790113593907, - "nauc_map_at_1000_diff1": 0.6032375794517476, - "nauc_map_at_1000_max": 0.5234777263347525, - "nauc_map_at_1000_std": 0.3925397811648996, - "nauc_map_at_100_diff1": 0.6029913887345628, - "nauc_map_at_100_max": 0.5234730473183735, - "nauc_map_at_100_std": 0.392993926813872, - "nauc_map_at_10_diff1": 0.6032298753284776, - "nauc_map_at_10_max": 0.523595638946803, - "nauc_map_at_10_std": 0.3908289376631486, - "nauc_map_at_1_diff1": 0.6603749690968012, - "nauc_map_at_1_max": 0.522301229855425, - "nauc_map_at_1_std": 0.35891873761023935, - "nauc_map_at_20_diff1": 0.6019939478294539, - "nauc_map_at_20_max": 0.523271680197658, - "nauc_map_at_20_std": 0.393494827654832, - "nauc_map_at_3_diff1": 0.6097994862720478, - "nauc_map_at_3_max": 0.5226182274555089, - "nauc_map_at_3_std": 0.3865498621153615, - "nauc_map_at_5_diff1": 0.6036483829475393, - "nauc_map_at_5_max": 0.525436647195231, - "nauc_map_at_5_std": 0.387863937482928, - "nauc_mrr_at_1000_diff1": 0.6032366982761342, - "nauc_mrr_at_1000_max": 0.523476478320266, - "nauc_mrr_at_1000_std": 0.3925383463898094, - "nauc_mrr_at_100_diff1": 0.6029905030981351, - "nauc_mrr_at_100_max": 0.523471777559567, - "nauc_mrr_at_100_std": 0.3929924646324689, - "nauc_mrr_at_10_diff1": 0.6032298753284776, - "nauc_mrr_at_10_max": 0.523595638946803, - "nauc_mrr_at_10_std": 0.3908289376631486, - "nauc_mrr_at_1_diff1": 0.6603749690968012, - "nauc_mrr_at_1_max": 0.522301229855425, - "nauc_mrr_at_1_std": 0.35891873761023935, - "nauc_mrr_at_20_diff1": 0.6019939478294539, - "nauc_mrr_at_20_max": 0.523271680197658, - "nauc_mrr_at_20_std": 0.393494827654832, - "nauc_mrr_at_3_diff1": 0.6097994862720478, - "nauc_mrr_at_3_max": 0.5226182274555089, - "nauc_mrr_at_3_std": 0.3865498621153615, - "nauc_mrr_at_5_diff1": 0.6036483829475393, - "nauc_mrr_at_5_max": 0.525436647195231, - "nauc_mrr_at_5_std": 0.387863937482928, - "nauc_ndcg_at_1000_diff1": 0.5905044778644832, - "nauc_ndcg_at_1000_max": 0.5240057340140417, - "nauc_ndcg_at_1000_std": 0.4044954816571764, - "nauc_ndcg_at_100_diff1": 0.5838104029791342, - "nauc_ndcg_at_100_max": 0.5247090283939485, - "nauc_ndcg_at_100_std": 0.41541523853402806, - "nauc_ndcg_at_10_diff1": 0.5807929180968202, - "nauc_ndcg_at_10_max": 0.5235542263682889, - "nauc_ndcg_at_10_std": 0.40815027296979883, - "nauc_ndcg_at_1_diff1": 0.6603749690968012, - "nauc_ndcg_at_1_max": 0.522301229855425, - "nauc_ndcg_at_1_std": 0.35891873761023935, - "nauc_ndcg_at_20_diff1": 0.5759441101599324, - "nauc_ndcg_at_20_max": 0.5221250926205538, - "nauc_ndcg_at_20_std": 0.41829934427865817, - "nauc_ndcg_at_3_diff1": 0.5937567252992616, - "nauc_ndcg_at_3_max": 0.5232237668689338, - "nauc_ndcg_at_3_std": 0.39800682368297036, - "nauc_ndcg_at_5_diff1": 0.5825995713549366, - "nauc_ndcg_at_5_max": 0.527843396831755, - "nauc_ndcg_at_5_std": 0.39994406748733624, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.48098495877756564, - "nauc_precision_at_100_max": 0.540460763885397, - "nauc_precision_at_100_std": 0.5785820711736123, - "nauc_precision_at_10_diff1": 0.5077985610322097, - "nauc_precision_at_10_max": 0.5222008223874395, - "nauc_precision_at_10_std": 0.4673819128352972, - "nauc_precision_at_1_diff1": 0.6603749690968012, - "nauc_precision_at_1_max": 0.522301229855425, - "nauc_precision_at_1_std": 0.35891873761023935, - "nauc_precision_at_20_diff1": 0.4776277222183427, - "nauc_precision_at_20_max": 0.5148646415990584, - "nauc_precision_at_20_std": 0.520179420268066, - "nauc_precision_at_3_diff1": 0.5477584110243353, - "nauc_precision_at_3_max": 0.5251152644999393, - "nauc_precision_at_3_std": 0.431684414718244, - "nauc_precision_at_5_diff1": 0.5197125969701824, - "nauc_precision_at_5_max": 0.5353503012783575, - "nauc_precision_at_5_std": 0.4358979011359394, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.48098495877756514, - "nauc_recall_at_100_max": 0.5404607638853961, - "nauc_recall_at_100_std": 0.5785820711736115, - "nauc_recall_at_10_diff1": 0.5077985610322101, - "nauc_recall_at_10_max": 0.52220082238744, - "nauc_recall_at_10_std": 0.4673819128352976, - "nauc_recall_at_1_diff1": 0.6603749690968012, - "nauc_recall_at_1_max": 0.522301229855425, - "nauc_recall_at_1_std": 0.35891873761023935, - "nauc_recall_at_20_diff1": 0.47762772221834277, - "nauc_recall_at_20_max": 0.5148646415990591, - "nauc_recall_at_20_std": 0.5201794202680657, - "nauc_recall_at_3_diff1": 0.5477584110243355, - "nauc_recall_at_3_max": 0.5251152644999395, - "nauc_recall_at_3_std": 0.43168441471824376, - "nauc_recall_at_5_diff1": 0.5197125969701824, - "nauc_recall_at_5_max": 0.5353503012783575, - "nauc_recall_at_5_std": 0.43589790113593907, - "ndcg_at_1": 0.32316, - "ndcg_at_10": 0.44335, - "ndcg_at_100": 0.49652, - "ndcg_at_1000": 0.51696, - "ndcg_at_20": 0.46084, - "ndcg_at_3": 0.3949, - "ndcg_at_5": 0.41812, - "precision_at_1": 0.32316, - "precision_at_10": 0.05798, - "precision_at_100": 0.00845, - "precision_at_1000": 0.001, - "precision_at_20": 0.03245, - "precision_at_3": 0.1484, - "precision_at_5": 0.10042, - "recall_at_1": 0.32316, - "recall_at_10": 0.57975, - "recall_at_100": 0.84466, - "recall_at_1000": 1.0, - "recall_at_20": 0.6491, - "recall_at_3": 0.44521, - "recall_at_5": 0.50208 - }, - { - "cv_recall_at_1": 0.31712, - "cv_recall_at_10": 0.56237, - "cv_recall_at_100": 0.83827, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.64588, - "cv_recall_at_3": 0.43869, - "cv_recall_at_5": 0.48414, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.43086, - "map_at_1": 0.31712, - "map_at_10": 0.38985, - "map_at_100": 0.40032, - "map_at_1000": 0.40108, - "map_at_20": 0.39546, - "map_at_3": 0.36892, - "map_at_5": 0.37949, - "mrr_at_1": 0.3171247357293869, - "mrr_at_10": 0.389854944796805, - "mrr_at_100": 0.4003152729755614, - "mrr_at_1000": 0.40108303021116537, - "mrr_at_20": 0.39545539710970634, - "mrr_at_3": 0.3689217758985197, - "mrr_at_5": 0.37949260042283245, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.39181767995617045, - "nauc_cv_recall_at_100_max": 0.4050520733318427, - "nauc_cv_recall_at_100_std": 0.5990584601132213, - "nauc_cv_recall_at_10_diff1": 0.46942265389779947, - "nauc_cv_recall_at_10_max": 0.37702464329644436, - "nauc_cv_recall_at_10_std": 0.4060088764375527, - "nauc_cv_recall_at_1_diff1": 0.6636784835703272, - "nauc_cv_recall_at_1_max": 0.47860715314683555, - "nauc_cv_recall_at_1_std": 0.3351963299597727, - "nauc_cv_recall_at_20_diff1": 0.4105037645463755, - "nauc_cv_recall_at_20_max": 0.35846333599786645, - "nauc_cv_recall_at_20_std": 0.42470469111912557, - "nauc_cv_recall_at_3_diff1": 0.5252010506154087, - "nauc_cv_recall_at_3_max": 0.41922611748318617, - "nauc_cv_recall_at_3_std": 0.37931068318115596, - "nauc_cv_recall_at_5_diff1": 0.5043643680663037, - "nauc_cv_recall_at_5_max": 0.3949210102081491, - "nauc_cv_recall_at_5_std": 0.37065381352919147, - "nauc_map_at_1000_diff1": 0.594113403077539, - "nauc_map_at_1000_max": 0.44654400940153666, - "nauc_map_at_1000_std": 0.3592820692140759, - "nauc_map_at_100_diff1": 0.5936780021007206, - "nauc_map_at_100_max": 0.4464329940626086, - "nauc_map_at_100_std": 0.35973466746408383, - "nauc_map_at_10_diff1": 0.5950245929669957, - "nauc_map_at_10_max": 0.4454913141343116, - "nauc_map_at_10_std": 0.35727519098134763, - "nauc_map_at_1_diff1": 0.6636784835703272, - "nauc_map_at_1_max": 0.47860715314683555, - "nauc_map_at_1_std": 0.3351963299597727, - "nauc_map_at_20_diff1": 0.592494503602588, - "nauc_map_at_20_max": 0.44468202478683205, - "nauc_map_at_20_std": 0.35763958254326716, - "nauc_map_at_3_diff1": 0.6028648903906949, - "nauc_map_at_3_max": 0.4523138001749844, - "nauc_map_at_3_std": 0.3546353095693066, - "nauc_map_at_5_diff1": 0.5985843487767859, - "nauc_map_at_5_max": 0.4471703848852369, - "nauc_map_at_5_std": 0.3528030244263197, - "nauc_mrr_at_1000_diff1": 0.5941131184415618, - "nauc_mrr_at_1000_max": 0.4465442760720423, - "nauc_mrr_at_1000_std": 0.3592823694304114, - "nauc_mrr_at_100_diff1": 0.5936777192516401, - "nauc_mrr_at_100_max": 0.4464332457865363, - "nauc_mrr_at_100_std": 0.3597349501870047, - "nauc_mrr_at_10_diff1": 0.5950245929669957, - "nauc_mrr_at_10_max": 0.4454913141343116, - "nauc_mrr_at_10_std": 0.35727519098134763, - "nauc_mrr_at_1_diff1": 0.6636784835703272, - "nauc_mrr_at_1_max": 0.47860715314683555, - "nauc_mrr_at_1_std": 0.3351963299597727, - "nauc_mrr_at_20_diff1": 0.592494503602588, - "nauc_mrr_at_20_max": 0.44468202478683205, - "nauc_mrr_at_20_std": 0.35763958254326716, - "nauc_mrr_at_3_diff1": 0.6028648903906949, - "nauc_mrr_at_3_max": 0.4523138001749844, - "nauc_mrr_at_3_std": 0.3546353095693066, - "nauc_mrr_at_5_diff1": 0.5985843487767859, - "nauc_mrr_at_5_max": 0.4471703848852369, - "nauc_mrr_at_5_std": 0.3528030244263197, - "nauc_ndcg_at_1000_diff1": 0.5762392937292321, - "nauc_ndcg_at_1000_max": 0.43979278940966154, - "nauc_ndcg_at_1000_std": 0.37028794726047176, - "nauc_ndcg_at_100_diff1": 0.5645324093122328, - "nauc_ndcg_at_100_max": 0.43743526699578555, - "nauc_ndcg_at_100_std": 0.3843072006940757, - "nauc_ndcg_at_10_diff1": 0.565573536978437, - "nauc_ndcg_at_10_max": 0.42963997269770055, - "nauc_ndcg_at_10_std": 0.36832076534969604, - "nauc_ndcg_at_1_diff1": 0.6636784835703272, - "nauc_ndcg_at_1_max": 0.47860715314683555, - "nauc_ndcg_at_1_std": 0.3351963299597727, - "nauc_ndcg_at_20_diff1": 0.555405981896656, - "nauc_ndcg_at_20_max": 0.42678212460430726, - "nauc_ndcg_at_20_std": 0.37084255862441984, - "nauc_ndcg_at_3_diff1": 0.5827884096507694, - "nauc_ndcg_at_3_max": 0.4437515122638289, - "nauc_ndcg_at_3_std": 0.3610167467861188, - "nauc_ndcg_at_5_diff1": 0.5747593673859045, - "nauc_ndcg_at_5_max": 0.4341592635536307, - "nauc_ndcg_at_5_std": 0.3575397919597607, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.39181767995617117, - "nauc_precision_at_100_max": 0.405052073331843, - "nauc_precision_at_100_std": 0.5990584601132215, - "nauc_precision_at_10_diff1": 0.4694226538977993, - "nauc_precision_at_10_max": 0.3770246432964444, - "nauc_precision_at_10_std": 0.4060088764375527, - "nauc_precision_at_1_diff1": 0.6636784835703272, - "nauc_precision_at_1_max": 0.47860715314683555, - "nauc_precision_at_1_std": 0.3351963299597727, - "nauc_precision_at_20_diff1": 0.4105037645463753, - "nauc_precision_at_20_max": 0.3584633359978659, - "nauc_precision_at_20_std": 0.4247046911191254, - "nauc_precision_at_3_diff1": 0.5252010506154082, - "nauc_precision_at_3_max": 0.419226117483186, - "nauc_precision_at_3_std": 0.37931068318115574, - "nauc_precision_at_5_diff1": 0.5043643680663035, - "nauc_precision_at_5_max": 0.394921010208149, - "nauc_precision_at_5_std": 0.3706538135291907, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.39181767995617045, - "nauc_recall_at_100_max": 0.4050520733318427, - "nauc_recall_at_100_std": 0.5990584601132213, - "nauc_recall_at_10_diff1": 0.46942265389779947, - "nauc_recall_at_10_max": 0.37702464329644436, - "nauc_recall_at_10_std": 0.4060088764375527, - "nauc_recall_at_1_diff1": 0.6636784835703272, - "nauc_recall_at_1_max": 0.47860715314683555, - "nauc_recall_at_1_std": 0.3351963299597727, - "nauc_recall_at_20_diff1": 0.4105037645463755, - "nauc_recall_at_20_max": 0.35846333599786645, - "nauc_recall_at_20_std": 0.42470469111912557, - "nauc_recall_at_3_diff1": 0.5252010506154087, - "nauc_recall_at_3_max": 0.41922611748318617, - "nauc_recall_at_3_std": 0.37931068318115596, - "nauc_recall_at_5_diff1": 0.5043643680663037, - "nauc_recall_at_5_max": 0.3949210102081491, - "nauc_recall_at_5_std": 0.37065381352919147, - "ndcg_at_1": 0.31712, - "ndcg_at_10": 0.43086, - "ndcg_at_100": 0.4872, - "ndcg_at_1000": 0.50779, - "ndcg_at_20": 0.45174, - "ndcg_at_3": 0.38676, - "ndcg_at_5": 0.40565, - "precision_at_1": 0.31712, - "precision_at_10": 0.05624, - "precision_at_100": 0.00838, - "precision_at_1000": 0.001, - "precision_at_20": 0.03229, - "precision_at_3": 0.14623, - "precision_at_5": 0.09683, - "recall_at_1": 0.31712, - "recall_at_10": 0.56237, - "recall_at_100": 0.83827, - "recall_at_1000": 1.0, - "recall_at_20": 0.64588, - "recall_at_3": 0.43869, - "recall_at_5": 0.48414 - }, - { - "cv_recall_at_1": 0.495, - "cv_recall_at_10": 0.78, - "cv_recall_at_100": 0.979, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.841, - "cv_recall_at_3": 0.655, - "cv_recall_at_5": 0.717, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.63387, - "map_at_1": 0.495, - "map_at_10": 0.58739, - "map_at_100": 0.59567, - "map_at_1000": 0.59578, - "map_at_20": 0.59173, - "map_at_3": 0.56467, - "map_at_5": 0.57897, - "mrr_at_1": 0.495, - "mrr_at_10": 0.5873928571428575, - "mrr_at_100": 0.5956652007256305, - "mrr_at_1000": 0.5957836523327847, - "mrr_at_20": 0.5917261944453897, - "mrr_at_3": 0.5646666666666669, - "mrr_at_5": 0.5789666666666671, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.6082655284335997, - "nauc_cv_recall_at_100_max": 0.7598594993552958, - "nauc_cv_recall_at_100_std": 0.8843982037259329, - "nauc_cv_recall_at_10_diff1": 0.6816641164742425, - "nauc_cv_recall_at_10_max": 0.2725900681596879, - "nauc_cv_recall_at_10_std": -0.19088885797746538, - "nauc_cv_recall_at_1_diff1": 0.7404600865904506, - "nauc_cv_recall_at_1_max": 0.4009643329808529, - "nauc_cv_recall_at_1_std": 0.013753383478648351, - "nauc_cv_recall_at_20_diff1": 0.6667880441155599, - "nauc_cv_recall_at_20_max": 0.3064266273321445, - "nauc_cv_recall_at_20_std": -0.1325410883236954, - "nauc_cv_recall_at_3_diff1": 0.6830957330550604, - "nauc_cv_recall_at_3_max": 0.30237216831138086, - "nauc_cv_recall_at_3_std": -0.11962728938481404, - "nauc_cv_recall_at_5_diff1": 0.6895788774983411, - "nauc_cv_recall_at_5_max": 0.29256752540180514, - "nauc_cv_recall_at_5_std": -0.1600261724164667, - "nauc_map_at_1000_diff1": 0.7197162304227843, - "nauc_map_at_1000_max": 0.36638095382714386, - "nauc_map_at_1000_std": -0.03260887184687692, - "nauc_map_at_100_diff1": 0.7196837015487083, - "nauc_map_at_100_max": 0.36653962854973204, - "nauc_map_at_100_std": -0.032241605111186684, - "nauc_map_at_10_diff1": 0.7192984727699232, - "nauc_map_at_10_max": 0.3626920886070452, - "nauc_map_at_10_std": -0.03897821506531741, - "nauc_map_at_1_diff1": 0.7404600865904506, - "nauc_map_at_1_max": 0.4009643329808529, - "nauc_map_at_1_std": 0.013753383478648351, - "nauc_map_at_20_diff1": 0.7191499846305851, - "nauc_map_at_20_max": 0.3649792771195435, - "nauc_map_at_20_std": -0.03538358395201348, - "nauc_map_at_3_diff1": 0.7172464691095869, - "nauc_map_at_3_max": 0.36403675266954516, - "nauc_map_at_3_std": -0.03286408950285727, - "nauc_map_at_5_diff1": 0.7194128350803819, - "nauc_map_at_5_max": 0.3632676565831752, - "nauc_map_at_5_std": -0.03783915587541209, - "nauc_mrr_at_1000_diff1": 0.7197162304227843, - "nauc_mrr_at_1000_max": 0.36638095382714386, - "nauc_mrr_at_1000_std": -0.03260887184687692, - "nauc_mrr_at_100_diff1": 0.7196837015487083, - "nauc_mrr_at_100_max": 0.36653962854973204, - "nauc_mrr_at_100_std": -0.032241605111186684, - "nauc_mrr_at_10_diff1": 0.7192984727699232, - "nauc_mrr_at_10_max": 0.3626920886070452, - "nauc_mrr_at_10_std": -0.03897821506531741, - "nauc_mrr_at_1_diff1": 0.7404600865904506, - "nauc_mrr_at_1_max": 0.4009643329808529, - "nauc_mrr_at_1_std": 0.013753383478648351, - "nauc_mrr_at_20_diff1": 0.7191499846305851, - "nauc_mrr_at_20_max": 0.3649792771195435, - "nauc_mrr_at_20_std": -0.03538358395201348, - "nauc_mrr_at_3_diff1": 0.7172464691095869, - "nauc_mrr_at_3_max": 0.36403675266954516, - "nauc_mrr_at_3_std": -0.03286408950285727, - "nauc_mrr_at_5_diff1": 0.7194128350803819, - "nauc_mrr_at_5_max": 0.3632676565831752, - "nauc_mrr_at_5_std": -0.03783915587541209, - "nauc_ndcg_at_1000_diff1": 0.7161717872946759, - "nauc_ndcg_at_1000_max": 0.36240087391615494, - "nauc_ndcg_at_1000_std": -0.03802915968796838, - "nauc_ndcg_at_100_diff1": 0.7150293653440268, - "nauc_ndcg_at_100_max": 0.3669197218892507, - "nauc_ndcg_at_100_std": -0.02753279118061936, - "nauc_ndcg_at_10_diff1": 0.7122536087594573, - "nauc_ndcg_at_10_max": 0.346227943924156, - "nauc_ndcg_at_10_std": -0.06629529516462444, - "nauc_ndcg_at_1_diff1": 0.7404600865904506, - "nauc_ndcg_at_1_max": 0.4009643329808529, - "nauc_ndcg_at_1_std": 0.013753383478648351, - "nauc_ndcg_at_20_diff1": 0.7114885578859922, - "nauc_ndcg_at_20_max": 0.3545982292278816, - "nauc_ndcg_at_20_std": -0.05241597538665322, - "nauc_ndcg_at_3_diff1": 0.7092500802086553, - "nauc_ndcg_at_3_max": 0.34973063708805896, - "nauc_ndcg_at_3_std": -0.052851060304107446, - "nauc_ndcg_at_5_diff1": 0.7128843624704241, - "nauc_ndcg_at_5_max": 0.34819332706378014, - "nauc_ndcg_at_5_std": -0.06327695695795105, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.6082655284335963, - "nauc_precision_at_100_max": 0.7598594993552936, - "nauc_precision_at_100_std": 0.8843982037259326, - "nauc_precision_at_10_diff1": 0.6816641164742427, - "nauc_precision_at_10_max": 0.27259006815968745, - "nauc_precision_at_10_std": -0.19088885797746732, - "nauc_precision_at_1_diff1": 0.7404600865904506, - "nauc_precision_at_1_max": 0.4009643329808529, - "nauc_precision_at_1_std": 0.013753383478648351, - "nauc_precision_at_20_diff1": 0.6667880441155589, - "nauc_precision_at_20_max": 0.3064266273321421, - "nauc_precision_at_20_std": -0.13254108832369677, - "nauc_precision_at_3_diff1": 0.6830957330550611, - "nauc_precision_at_3_max": 0.3023721683113811, - "nauc_precision_at_3_std": -0.11962728938481379, - "nauc_precision_at_5_diff1": 0.6895788774983402, - "nauc_precision_at_5_max": 0.2925675254018038, - "nauc_precision_at_5_std": -0.16002617241646783, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.6082655284335997, - "nauc_recall_at_100_max": 0.7598594993552958, - "nauc_recall_at_100_std": 0.8843982037259329, - "nauc_recall_at_10_diff1": 0.6816641164742425, - "nauc_recall_at_10_max": 0.2725900681596879, - "nauc_recall_at_10_std": -0.19088885797746538, - "nauc_recall_at_1_diff1": 0.7404600865904506, - "nauc_recall_at_1_max": 0.4009643329808529, - "nauc_recall_at_1_std": 0.013753383478648351, - "nauc_recall_at_20_diff1": 0.6667880441155599, - "nauc_recall_at_20_max": 0.3064266273321445, - "nauc_recall_at_20_std": -0.1325410883236954, - "nauc_recall_at_3_diff1": 0.6830957330550604, - "nauc_recall_at_3_max": 0.30237216831138086, - "nauc_recall_at_3_std": -0.11962728938481404, - "nauc_recall_at_5_diff1": 0.6895788774983411, - "nauc_recall_at_5_max": 0.29256752540180514, - "nauc_recall_at_5_std": -0.1600261724164667, - "ndcg_at_1": 0.495, - "ndcg_at_10": 0.63387, - "ndcg_at_100": 0.67586, - "ndcg_at_1000": 0.67865, - "ndcg_at_20": 0.64941, - "ndcg_at_3": 0.58783, - "ndcg_at_5": 0.61348, - "precision_at_1": 0.495, - "precision_at_10": 0.078, - "precision_at_100": 0.00979, - "precision_at_1000": 0.001, - "precision_at_20": 0.04205, - "precision_at_3": 0.21833, - "precision_at_5": 0.1434, - "recall_at_1": 0.495, - "recall_at_10": 0.78, - "recall_at_100": 0.979, - "recall_at_1000": 1.0, - "recall_at_20": 0.841, - "recall_at_3": 0.655, - "recall_at_5": 0.717 - } - ] - }, - "task_name": "WITT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/model_meta.json b/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/model_meta.json deleted file mode 100644 index a276a48179..0000000000 --- a/results-mieb/google__siglip-base-patch16-384/41aec1c83b32e0a6fca20ad88ba058aa5b5ea394/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "google/siglip-base-patch16-384", "revision": "41aec1c83b32e0a6fca20ad88ba058aa5b5ea394", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/WITT2IRetrieval.json b/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/WITT2IRetrieval.json deleted file mode 100644 index 7523e09222..0000000000 --- a/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/WITT2IRetrieval.json +++ /dev/null @@ -1,1936 +0,0 @@ -{ - "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", - "evaluation_time": 234.74904417991638, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.14944, - "cv_recall_at_10": 0.33034, - "cv_recall_at_100": 0.66517, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.42247, - "cv_recall_at_3": 0.2, - "cv_recall_at_5": 0.24157, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.22414, - "map_at_1": 0.14944, - "map_at_10": 0.19231, - "map_at_100": 0.20432, - "map_at_1000": 0.2058, - "map_at_20": 0.19873, - "map_at_3": 0.17135, - "map_at_5": 0.18079, - "mrr_at_1": 0.149438202247191, - "mrr_at_10": 0.1923136258248617, - "mrr_at_100": 0.20431598751440663, - "mrr_at_1000": 0.2057957434184432, - "mrr_at_20": 0.19872819244758055, - "mrr_at_3": 0.17134831460674158, - "mrr_at_5": 0.18078651685393257, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2634787391962865, - "nauc_cv_recall_at_100_max": 0.4645672109653449, - "nauc_cv_recall_at_100_std": 0.48648923584685244, - "nauc_cv_recall_at_10_diff1": 0.4120188813008714, - "nauc_cv_recall_at_10_max": 0.39580884178534775, - "nauc_cv_recall_at_10_std": 0.3375481230874158, - "nauc_cv_recall_at_1_diff1": 0.6461541257146491, - "nauc_cv_recall_at_1_max": 0.5172890010471299, - "nauc_cv_recall_at_1_std": 0.3714290257148764, - "nauc_cv_recall_at_20_diff1": 0.3539802242264886, - "nauc_cv_recall_at_20_max": 0.3648519440541502, - "nauc_cv_recall_at_20_std": 0.3389938958623623, - "nauc_cv_recall_at_3_diff1": 0.525914894080172, - "nauc_cv_recall_at_3_max": 0.44017407204507436, - "nauc_cv_recall_at_3_std": 0.340840900762971, - "nauc_cv_recall_at_5_diff1": 0.5011352099676971, - "nauc_cv_recall_at_5_max": 0.4099173553719009, - "nauc_cv_recall_at_5_std": 0.34841297143096883, - "nauc_map_at_1000_diff1": 0.555169070381733, - "nauc_map_at_1000_max": 0.4631488311423136, - "nauc_map_at_1000_std": 0.36157268097851425, - "nauc_map_at_100_diff1": 0.5539155701362769, - "nauc_map_at_100_max": 0.46292444260745313, - "nauc_map_at_100_std": 0.36168212855581966, - "nauc_map_at_10_diff1": 0.5602553663022992, - "nauc_map_at_10_max": 0.46339619753852673, - "nauc_map_at_10_std": 0.3583151485756733, - "nauc_map_at_1_diff1": 0.6461541257146491, - "nauc_map_at_1_max": 0.5172890010471299, - "nauc_map_at_1_std": 0.3714290257148764, - "nauc_map_at_20_diff1": 0.5545223104794634, - "nauc_map_at_20_max": 0.460308754159618, - "nauc_map_at_20_std": 0.3583429353435337, - "nauc_map_at_3_diff1": 0.5873248593499638, - "nauc_map_at_3_max": 0.4774872915845461, - "nauc_map_at_3_std": 0.3582007033781603, - "nauc_map_at_5_diff1": 0.5791085109357881, - "nauc_map_at_5_max": 0.468387528653673, - "nauc_map_at_5_std": 0.35930898584402726, - "nauc_mrr_at_1000_diff1": 0.555169070381733, - "nauc_mrr_at_1000_max": 0.4631488311423136, - "nauc_mrr_at_1000_std": 0.36157268097851425, - "nauc_mrr_at_100_diff1": 0.5539155701362769, - "nauc_mrr_at_100_max": 0.46292444260745313, - "nauc_mrr_at_100_std": 0.36168212855581966, - "nauc_mrr_at_10_diff1": 0.5602553663022992, - "nauc_mrr_at_10_max": 0.46339619753852673, - "nauc_mrr_at_10_std": 0.3583151485756733, - "nauc_mrr_at_1_diff1": 0.6461541257146491, - "nauc_mrr_at_1_max": 0.5172890010471299, - "nauc_mrr_at_1_std": 0.3714290257148764, - "nauc_mrr_at_20_diff1": 0.5545223104794634, - "nauc_mrr_at_20_max": 0.460308754159618, - "nauc_mrr_at_20_std": 0.3583429353435337, - "nauc_mrr_at_3_diff1": 0.5873248593499638, - "nauc_mrr_at_3_max": 0.4774872915845461, - "nauc_mrr_at_3_std": 0.3582007033781603, - "nauc_mrr_at_5_diff1": 0.5791085109357881, - "nauc_mrr_at_5_max": 0.468387528653673, - "nauc_mrr_at_5_std": 0.35930898584402726, - "nauc_ndcg_at_1000_diff1": 0.5222129913095555, - "nauc_ndcg_at_1000_max": 0.4535832261680661, - "nauc_ndcg_at_1000_std": 0.36699755338097423, - "nauc_ndcg_at_100_diff1": 0.4898715477765656, - "nauc_ndcg_at_100_max": 0.45382252122114575, - "nauc_ndcg_at_100_std": 0.37968453864410084, - "nauc_ndcg_at_10_diff1": 0.5175455563050221, - "nauc_ndcg_at_10_max": 0.4430790495975126, - "nauc_ndcg_at_10_std": 0.3524409492818574, - "nauc_ndcg_at_1_diff1": 0.6461541257146491, - "nauc_ndcg_at_1_max": 0.5172890010471299, - "nauc_ndcg_at_1_std": 0.3714290257148764, - "nauc_ndcg_at_20_diff1": 0.49847417961819085, - "nauc_ndcg_at_20_max": 0.432915540811886, - "nauc_ndcg_at_20_std": 0.3527195903365103, - "nauc_ndcg_at_3_diff1": 0.5701266655769082, - "nauc_ndcg_at_3_max": 0.4669535549260126, - "nauc_ndcg_at_3_std": 0.3534094141369618, - "nauc_ndcg_at_5_diff1": 0.5567781257664445, - "nauc_ndcg_at_5_max": 0.4517477421856985, - "nauc_ndcg_at_5_std": 0.3560158696137673, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.26347873919628667, - "nauc_precision_at_100_max": 0.46456721096534553, - "nauc_precision_at_100_std": 0.48648923584685333, - "nauc_precision_at_10_diff1": 0.41201888130087116, - "nauc_precision_at_10_max": 0.39580884178534803, - "nauc_precision_at_10_std": 0.3375481230874158, - "nauc_precision_at_1_diff1": 0.6461541257146491, - "nauc_precision_at_1_max": 0.5172890010471299, - "nauc_precision_at_1_std": 0.3714290257148764, - "nauc_precision_at_20_diff1": 0.35398022422648834, - "nauc_precision_at_20_max": 0.36485194405414995, - "nauc_precision_at_20_std": 0.33899389586236256, - "nauc_precision_at_3_diff1": 0.5259148940801714, - "nauc_precision_at_3_max": 0.44017407204507436, - "nauc_precision_at_3_std": 0.3408409007629708, - "nauc_precision_at_5_diff1": 0.5011352099676972, - "nauc_precision_at_5_max": 0.40991735537190055, - "nauc_precision_at_5_std": 0.3484129714309687, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2634787391962865, - "nauc_recall_at_100_max": 0.4645672109653449, - "nauc_recall_at_100_std": 0.48648923584685244, - "nauc_recall_at_10_diff1": 0.4120188813008714, - "nauc_recall_at_10_max": 0.39580884178534775, - "nauc_recall_at_10_std": 0.3375481230874158, - "nauc_recall_at_1_diff1": 0.6461541257146491, - "nauc_recall_at_1_max": 0.5172890010471299, - "nauc_recall_at_1_std": 0.3714290257148764, - "nauc_recall_at_20_diff1": 0.3539802242264886, - "nauc_recall_at_20_max": 0.3648519440541502, - "nauc_recall_at_20_std": 0.3389938958623623, - "nauc_recall_at_3_diff1": 0.525914894080172, - "nauc_recall_at_3_max": 0.44017407204507436, - "nauc_recall_at_3_std": 0.340840900762971, - "nauc_recall_at_5_diff1": 0.5011352099676971, - "nauc_recall_at_5_max": 0.4099173553719009, - "nauc_recall_at_5_std": 0.34841297143096883, - "ndcg_at_1": 0.14944, - "ndcg_at_10": 0.22414, - "ndcg_at_100": 0.2912, - "ndcg_at_1000": 0.33314, - "ndcg_at_20": 0.24746, - "ndcg_at_3": 0.17869, - "ndcg_at_5": 0.19576, - "precision_at_1": 0.14944, - "precision_at_10": 0.03303, - "precision_at_100": 0.00665, - "precision_at_1000": 0.001, - "precision_at_20": 0.02112, - "precision_at_3": 0.06667, - "precision_at_5": 0.04831, - "recall_at_1": 0.14944, - "recall_at_10": 0.33034, - "recall_at_100": 0.66517, - "recall_at_1000": 1.0, - "recall_at_20": 0.42247, - "recall_at_3": 0.2, - "recall_at_5": 0.24157 - }, - { - "cv_recall_at_1": 0.15698, - "cv_recall_at_10": 0.31977, - "cv_recall_at_100": 0.69535, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.40465, - "cv_recall_at_3": 0.21163, - "cv_recall_at_5": 0.25465, - "hf_subset": "bg", - "languages": [ - "bul-Cyrl" - ], - "main_score": 0.22603, - "map_at_1": 0.15698, - "map_at_10": 0.19746, - "map_at_100": 0.2101, - "map_at_1000": 0.21161, - "map_at_20": 0.20326, - "map_at_3": 0.17888, - "map_at_5": 0.1887, - "mrr_at_1": 0.1569767441860465, - "mrr_at_10": 0.1974617017349575, - "mrr_at_100": 0.21009563393962438, - "mrr_at_1000": 0.21160873316997192, - "mrr_at_20": 0.20325988139847134, - "mrr_at_3": 0.17887596899224803, - "mrr_at_5": 0.1887015503875969, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.3363919574226251, - "nauc_cv_recall_at_100_max": 0.37234669199849013, - "nauc_cv_recall_at_100_std": 0.3986816943007544, - "nauc_cv_recall_at_10_diff1": 0.3841938620716482, - "nauc_cv_recall_at_10_max": 0.1668038430121746, - "nauc_cv_recall_at_10_std": 0.12322377377572048, - "nauc_cv_recall_at_1_diff1": 0.6067466385745514, - "nauc_cv_recall_at_1_max": 0.29588926269525695, - "nauc_cv_recall_at_1_std": 0.15656596549653945, - "nauc_cv_recall_at_20_diff1": 0.3517457376056227, - "nauc_cv_recall_at_20_max": 0.15635326148882445, - "nauc_cv_recall_at_20_std": 0.10895267737737557, - "nauc_cv_recall_at_3_diff1": 0.49341354079088906, - "nauc_cv_recall_at_3_max": 0.1917544432899988, - "nauc_cv_recall_at_3_std": 0.10965683592961223, - "nauc_cv_recall_at_5_diff1": 0.45467439639098584, - "nauc_cv_recall_at_5_max": 0.20165656339442797, - "nauc_cv_recall_at_5_std": 0.13592845763496914, - "nauc_map_at_1000_diff1": 0.5287548795852088, - "nauc_map_at_1000_max": 0.24669405029122785, - "nauc_map_at_1000_std": 0.14398072701997205, - "nauc_map_at_100_diff1": 0.5281431111016485, - "nauc_map_at_100_max": 0.24680733420777262, - "nauc_map_at_100_std": 0.144483705635419, - "nauc_map_at_10_diff1": 0.5305009981241716, - "nauc_map_at_10_max": 0.2434189183238275, - "nauc_map_at_10_std": 0.14042474981401215, - "nauc_map_at_1_diff1": 0.6067466385745514, - "nauc_map_at_1_max": 0.29588926269525695, - "nauc_map_at_1_std": 0.15656596549653945, - "nauc_map_at_20_diff1": 0.5267607229546266, - "nauc_map_at_20_max": 0.24250663560859556, - "nauc_map_at_20_std": 0.13913708344341796, - "nauc_map_at_3_diff1": 0.5560856112730492, - "nauc_map_at_3_max": 0.24808399445761734, - "nauc_map_at_3_std": 0.13552614213434122, - "nauc_map_at_5_diff1": 0.5450619717717616, - "nauc_map_at_5_max": 0.2493579314145244, - "nauc_map_at_5_std": 0.14285677163632696, - "nauc_mrr_at_1000_diff1": 0.5287548970738191, - "nauc_mrr_at_1000_max": 0.2466940493792957, - "nauc_mrr_at_1000_std": 0.14398070943942484, - "nauc_mrr_at_100_diff1": 0.5281431111016485, - "nauc_mrr_at_100_max": 0.24680733420777262, - "nauc_mrr_at_100_std": 0.144483705635419, - "nauc_mrr_at_10_diff1": 0.5305009981241716, - "nauc_mrr_at_10_max": 0.2434189183238275, - "nauc_mrr_at_10_std": 0.14042474981401215, - "nauc_mrr_at_1_diff1": 0.6067466385745514, - "nauc_mrr_at_1_max": 0.29588926269525695, - "nauc_mrr_at_1_std": 0.15656596549653945, - "nauc_mrr_at_20_diff1": 0.5267607229546266, - "nauc_mrr_at_20_max": 0.24250663560859556, - "nauc_mrr_at_20_std": 0.13913708344341796, - "nauc_mrr_at_3_diff1": 0.5560856112730492, - "nauc_mrr_at_3_max": 0.24808399445761734, - "nauc_mrr_at_3_std": 0.13552614213434122, - "nauc_mrr_at_5_diff1": 0.5450619717717616, - "nauc_mrr_at_5_max": 0.2493579314145244, - "nauc_mrr_at_5_std": 0.14285677163632696, - "nauc_ndcg_at_1000_diff1": 0.5022253943978927, - "nauc_ndcg_at_1000_max": 0.24300420710339146, - "nauc_ndcg_at_1000_std": 0.15266396324514417, - "nauc_ndcg_at_100_diff1": 0.4836824082066605, - "nauc_ndcg_at_100_max": 0.2563419064021609, - "nauc_ndcg_at_100_std": 0.1788962463598874, - "nauc_ndcg_at_10_diff1": 0.4890532105795847, - "nauc_ndcg_at_10_max": 0.22147463946489696, - "nauc_ndcg_at_10_std": 0.1353636012280422, - "nauc_ndcg_at_1_diff1": 0.6067466385745514, - "nauc_ndcg_at_1_max": 0.29588926269525695, - "nauc_ndcg_at_1_std": 0.15656596549653945, - "nauc_ndcg_at_20_diff1": 0.4774594920142693, - "nauc_ndcg_at_20_max": 0.2179566948337125, - "nauc_ndcg_at_20_std": 0.13087037961967188, - "nauc_ndcg_at_3_diff1": 0.5387369747763605, - "nauc_ndcg_at_3_max": 0.2324434366269725, - "nauc_ndcg_at_3_std": 0.12836241066655732, - "nauc_ndcg_at_5_diff1": 0.5196578856648923, - "nauc_ndcg_at_5_max": 0.2354217175402108, - "nauc_ndcg_at_5_std": 0.14060419032243368, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3363919574226251, - "nauc_precision_at_100_max": 0.37234669199849024, - "nauc_precision_at_100_std": 0.3986816943007548, - "nauc_precision_at_10_diff1": 0.38419386207164824, - "nauc_precision_at_10_max": 0.16680384301217463, - "nauc_precision_at_10_std": 0.12322377377572062, - "nauc_precision_at_1_diff1": 0.6067466385745514, - "nauc_precision_at_1_max": 0.29588926269525695, - "nauc_precision_at_1_std": 0.15656596549653945, - "nauc_precision_at_20_diff1": 0.35174573760562283, - "nauc_precision_at_20_max": 0.15635326148882472, - "nauc_precision_at_20_std": 0.1089526773773756, - "nauc_precision_at_3_diff1": 0.4934135407908889, - "nauc_precision_at_3_max": 0.19175444328999902, - "nauc_precision_at_3_std": 0.10965683592961223, - "nauc_precision_at_5_diff1": 0.4546743963909857, - "nauc_precision_at_5_max": 0.20165656339442822, - "nauc_precision_at_5_std": 0.1359284576349691, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.3363919574226251, - "nauc_recall_at_100_max": 0.37234669199849013, - "nauc_recall_at_100_std": 0.3986816943007544, - "nauc_recall_at_10_diff1": 0.3841938620716482, - "nauc_recall_at_10_max": 0.1668038430121746, - "nauc_recall_at_10_std": 0.12322377377572048, - "nauc_recall_at_1_diff1": 0.6067466385745514, - "nauc_recall_at_1_max": 0.29588926269525695, - "nauc_recall_at_1_std": 0.15656596549653945, - "nauc_recall_at_20_diff1": 0.3517457376056227, - "nauc_recall_at_20_max": 0.15635326148882445, - "nauc_recall_at_20_std": 0.10895267737737557, - "nauc_recall_at_3_diff1": 0.49341354079088906, - "nauc_recall_at_3_max": 0.1917544432899988, - "nauc_recall_at_3_std": 0.10965683592961223, - "nauc_recall_at_5_diff1": 0.45467439639098584, - "nauc_recall_at_5_max": 0.20165656339442797, - "nauc_recall_at_5_std": 0.13592845763496914, - "ndcg_at_1": 0.15698, - "ndcg_at_10": 0.22603, - "ndcg_at_100": 0.30011, - "ndcg_at_1000": 0.33915, - "ndcg_at_20": 0.24738, - "ndcg_at_3": 0.18719, - "ndcg_at_5": 0.20491, - "precision_at_1": 0.15698, - "precision_at_10": 0.03198, - "precision_at_100": 0.00695, - "precision_at_1000": 0.001, - "precision_at_20": 0.02023, - "precision_at_3": 0.07054, - "precision_at_5": 0.05093, - "recall_at_1": 0.15698, - "recall_at_10": 0.31977, - "recall_at_100": 0.69535, - "recall_at_1000": 1.0, - "recall_at_20": 0.40465, - "recall_at_3": 0.21163, - "recall_at_5": 0.25465 - }, - { - "cv_recall_at_1": 0.3266, - "cv_recall_at_10": 0.58025, - "cv_recall_at_100": 0.80808, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.65208, - "cv_recall_at_3": 0.43098, - "cv_recall_at_5": 0.48822, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.44195, - "map_at_1": 0.3266, - "map_at_10": 0.39925, - "map_at_100": 0.408, - "map_at_1000": 0.4089, - "map_at_20": 0.40416, - "map_at_3": 0.37374, - "map_at_5": 0.38698, - "mrr_at_1": 0.3265993265993266, - "mrr_at_10": 0.39925222240037056, - "mrr_at_100": 0.40800305571393053, - "mrr_at_1000": 0.4088972381164094, - "mrr_at_20": 0.40416376707982127, - "mrr_at_3": 0.3737373737373736, - "mrr_at_5": 0.38698092031425335, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.40351361544537445, - "nauc_cv_recall_at_100_max": 0.5912530427570891, - "nauc_cv_recall_at_100_std": 0.5493433744386865, - "nauc_cv_recall_at_10_diff1": 0.4944214665489543, - "nauc_cv_recall_at_10_max": 0.4785322813825731, - "nauc_cv_recall_at_10_std": 0.3430216902502382, - "nauc_cv_recall_at_1_diff1": 0.6965218420770751, - "nauc_cv_recall_at_1_max": 0.5148871193449024, - "nauc_cv_recall_at_1_std": 0.24287634889441037, - "nauc_cv_recall_at_20_diff1": 0.46800885812695175, - "nauc_cv_recall_at_20_max": 0.5306725749263748, - "nauc_cv_recall_at_20_std": 0.41009891879163524, - "nauc_cv_recall_at_3_diff1": 0.6054092945422833, - "nauc_cv_recall_at_3_max": 0.4670388069523152, - "nauc_cv_recall_at_3_std": 0.2818495744111195, - "nauc_cv_recall_at_5_diff1": 0.568210566338316, - "nauc_cv_recall_at_5_max": 0.4618802871835425, - "nauc_cv_recall_at_5_std": 0.29267636145228926, - "nauc_map_at_1000_diff1": 0.635473936446471, - "nauc_map_at_1000_max": 0.49963553162495394, - "nauc_map_at_1000_std": 0.2754941978206276, - "nauc_map_at_100_diff1": 0.6349601801057578, - "nauc_map_at_100_max": 0.49976685033613943, - "nauc_map_at_100_std": 0.2759674358452211, - "nauc_map_at_10_diff1": 0.6354531528874251, - "nauc_map_at_10_max": 0.4962877765100082, - "nauc_map_at_10_std": 0.271362809907974, - "nauc_map_at_1_diff1": 0.6965218420770751, - "nauc_map_at_1_max": 0.5148871193449024, - "nauc_map_at_1_std": 0.24287634889441037, - "nauc_map_at_20_diff1": 0.6347142005930541, - "nauc_map_at_20_max": 0.4995845447680013, - "nauc_map_at_20_std": 0.27520502762627513, - "nauc_map_at_3_diff1": 0.6522163384845291, - "nauc_map_at_3_max": 0.49628358740147205, - "nauc_map_at_3_std": 0.2643331138403577, - "nauc_map_at_5_diff1": 0.6440086437871753, - "nauc_map_at_5_max": 0.49385387561881533, - "nauc_map_at_5_std": 0.26506225369101805, - "nauc_mrr_at_1000_diff1": 0.635473936446471, - "nauc_mrr_at_1000_max": 0.49963553162495394, - "nauc_mrr_at_1000_std": 0.2754941978206276, - "nauc_mrr_at_100_diff1": 0.6349601801057578, - "nauc_mrr_at_100_max": 0.49976685033613943, - "nauc_mrr_at_100_std": 0.2759674358452211, - "nauc_mrr_at_10_diff1": 0.6354531528874251, - "nauc_mrr_at_10_max": 0.4962877765100082, - "nauc_mrr_at_10_std": 0.271362809907974, - "nauc_mrr_at_1_diff1": 0.6965218420770751, - "nauc_mrr_at_1_max": 0.5148871193449024, - "nauc_mrr_at_1_std": 0.24287634889441037, - "nauc_mrr_at_20_diff1": 0.6347142005930541, - "nauc_mrr_at_20_max": 0.4995845447680013, - "nauc_mrr_at_20_std": 0.27520502762627513, - "nauc_mrr_at_3_diff1": 0.6522163384845291, - "nauc_mrr_at_3_max": 0.49628358740147205, - "nauc_mrr_at_3_std": 0.2643331138403577, - "nauc_mrr_at_5_diff1": 0.6440086437871753, - "nauc_mrr_at_5_max": 0.49385387561881533, - "nauc_mrr_at_5_std": 0.26506225369101805, - "nauc_ndcg_at_1000_diff1": 0.6171301068346469, - "nauc_ndcg_at_1000_max": 0.5013093969812261, - "nauc_ndcg_at_1000_std": 0.2909876037167272, - "nauc_ndcg_at_100_diff1": 0.6019744132439804, - "nauc_ndcg_at_100_max": 0.507234518974889, - "nauc_ndcg_at_100_std": 0.3085171017338535, - "nauc_ndcg_at_10_diff1": 0.6039190661012934, - "nauc_ndcg_at_10_max": 0.49142066303076054, - "nauc_ndcg_at_10_std": 0.28700506210534843, - "nauc_ndcg_at_1_diff1": 0.6965218420770751, - "nauc_ndcg_at_1_max": 0.5148871193449024, - "nauc_ndcg_at_1_std": 0.24287634889441037, - "nauc_ndcg_at_20_diff1": 0.6008855449309235, - "nauc_ndcg_at_20_max": 0.5032878495174833, - "nauc_ndcg_at_20_std": 0.3006504702690925, - "nauc_ndcg_at_3_diff1": 0.639951550242921, - "nauc_ndcg_at_3_max": 0.4888323450700999, - "nauc_ndcg_at_3_std": 0.2690366601897462, - "nauc_ndcg_at_5_diff1": 0.625009496386489, - "nauc_ndcg_at_5_max": 0.4856757482705781, - "nauc_ndcg_at_5_std": 0.27192265649173825, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.4035136154453741, - "nauc_precision_at_100_max": 0.5912530427570886, - "nauc_precision_at_100_std": 0.5493433744386861, - "nauc_precision_at_10_diff1": 0.4944214665489537, - "nauc_precision_at_10_max": 0.4785322813825728, - "nauc_precision_at_10_std": 0.34302169025023754, - "nauc_precision_at_1_diff1": 0.6965218420770751, - "nauc_precision_at_1_max": 0.5148871193449024, - "nauc_precision_at_1_std": 0.24287634889441037, - "nauc_precision_at_20_diff1": 0.4680088581269517, - "nauc_precision_at_20_max": 0.5306725749263749, - "nauc_precision_at_20_std": 0.4100989187916348, - "nauc_precision_at_3_diff1": 0.6054092945422834, - "nauc_precision_at_3_max": 0.4670388069523155, - "nauc_precision_at_3_std": 0.2818495744111192, - "nauc_precision_at_5_diff1": 0.5682105663383159, - "nauc_precision_at_5_max": 0.4618802871835421, - "nauc_precision_at_5_std": 0.292676361452289, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.40351361544537445, - "nauc_recall_at_100_max": 0.5912530427570891, - "nauc_recall_at_100_std": 0.5493433744386865, - "nauc_recall_at_10_diff1": 0.4944214665489543, - "nauc_recall_at_10_max": 0.4785322813825731, - "nauc_recall_at_10_std": 0.3430216902502382, - "nauc_recall_at_1_diff1": 0.6965218420770751, - "nauc_recall_at_1_max": 0.5148871193449024, - "nauc_recall_at_1_std": 0.24287634889441037, - "nauc_recall_at_20_diff1": 0.46800885812695175, - "nauc_recall_at_20_max": 0.5306725749263748, - "nauc_recall_at_20_std": 0.41009891879163524, - "nauc_recall_at_3_diff1": 0.6054092945422833, - "nauc_recall_at_3_max": 0.4670388069523152, - "nauc_recall_at_3_std": 0.2818495744111195, - "nauc_recall_at_5_diff1": 0.568210566338316, - "nauc_recall_at_5_max": 0.4618802871835425, - "nauc_recall_at_5_std": 0.29267636145228926, - "ndcg_at_1": 0.3266, - "ndcg_at_10": 0.44195, - "ndcg_at_100": 0.48864, - "ndcg_at_1000": 0.51296, - "ndcg_at_20": 0.46002, - "ndcg_at_3": 0.38849, - "ndcg_at_5": 0.4122, - "precision_at_1": 0.3266, - "precision_at_10": 0.05802, - "precision_at_100": 0.00808, - "precision_at_1000": 0.001, - "precision_at_20": 0.0326, - "precision_at_3": 0.14366, - "precision_at_5": 0.09764, - "recall_at_1": 0.3266, - "recall_at_10": 0.58025, - "recall_at_100": 0.80808, - "recall_at_1000": 1.0, - "recall_at_20": 0.65208, - "recall_at_3": 0.43098, - "recall_at_5": 0.48822 - }, - { - "cv_recall_at_1": 0.12105, - "cv_recall_at_10": 0.25439, - "cv_recall_at_100": 0.57193, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.31754, - "cv_recall_at_3": 0.16667, - "cv_recall_at_5": 0.19649, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.17855, - "map_at_1": 0.12105, - "map_at_10": 0.15548, - "map_at_100": 0.16549, - "map_at_1000": 0.16744, - "map_at_20": 0.1599, - "map_at_3": 0.14094, - "map_at_5": 0.14778, - "mrr_at_1": 0.12105263157894737, - "mrr_at_10": 0.1554817599554442, - "mrr_at_100": 0.16548549401385365, - "mrr_at_1000": 0.16743933738760186, - "mrr_at_20": 0.15990474455176268, - "mrr_at_3": 0.1409356725146199, - "mrr_at_5": 0.14777777777777779, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.12178494113610103, - "nauc_cv_recall_at_100_max": 0.12027108175345279, - "nauc_cv_recall_at_100_std": 0.36680812841564475, - "nauc_cv_recall_at_10_diff1": 0.30282348421793964, - "nauc_cv_recall_at_10_max": 0.13271784365196235, - "nauc_cv_recall_at_10_std": 0.33726102823484205, - "nauc_cv_recall_at_1_diff1": 0.49365195440479587, - "nauc_cv_recall_at_1_max": 0.2075044516883363, - "nauc_cv_recall_at_1_std": 0.3627661200695744, - "nauc_cv_recall_at_20_diff1": 0.21236255278748464, - "nauc_cv_recall_at_20_max": 0.1251083950019623, - "nauc_cv_recall_at_20_std": 0.3781734179029766, - "nauc_cv_recall_at_3_diff1": 0.3641915835197674, - "nauc_cv_recall_at_3_max": 0.14471642960954986, - "nauc_cv_recall_at_3_std": 0.3275568692130575, - "nauc_cv_recall_at_5_diff1": 0.32097758270041704, - "nauc_cv_recall_at_5_max": 0.14551136951201293, - "nauc_cv_recall_at_5_std": 0.32077075984419345, - "nauc_map_at_1000_diff1": 0.39639028419999567, - "nauc_map_at_1000_max": 0.16661154492698063, - "nauc_map_at_1000_std": 0.34102672479597745, - "nauc_map_at_100_diff1": 0.39497721421640114, - "nauc_map_at_100_max": 0.1661677982631193, - "nauc_map_at_100_std": 0.3410565655998729, - "nauc_map_at_10_diff1": 0.408843256856627, - "nauc_map_at_10_max": 0.16857868932527764, - "nauc_map_at_10_std": 0.3395154299876011, - "nauc_map_at_1_diff1": 0.49365195440479587, - "nauc_map_at_1_max": 0.2075044516883363, - "nauc_map_at_1_std": 0.3627661200695744, - "nauc_map_at_20_diff1": 0.39823470808304207, - "nauc_map_at_20_max": 0.16717533160175635, - "nauc_map_at_20_std": 0.3426694553686378, - "nauc_map_at_3_diff1": 0.42857037037951523, - "nauc_map_at_3_max": 0.17223986953199172, - "nauc_map_at_3_std": 0.3403057010618428, - "nauc_map_at_5_diff1": 0.4139839634441962, - "nauc_map_at_5_max": 0.17065659328509294, - "nauc_map_at_5_std": 0.33635437824819064, - "nauc_mrr_at_1000_diff1": 0.39639028419999567, - "nauc_mrr_at_1000_max": 0.16661154492698063, - "nauc_mrr_at_1000_std": 0.34102672479597745, - "nauc_mrr_at_100_diff1": 0.39497721421640114, - "nauc_mrr_at_100_max": 0.1661677982631193, - "nauc_mrr_at_100_std": 0.3410565655998729, - "nauc_mrr_at_10_diff1": 0.408843256856627, - "nauc_mrr_at_10_max": 0.16857868932527764, - "nauc_mrr_at_10_std": 0.3395154299876011, - "nauc_mrr_at_1_diff1": 0.49365195440479587, - "nauc_mrr_at_1_max": 0.2075044516883363, - "nauc_mrr_at_1_std": 0.3627661200695744, - "nauc_mrr_at_20_diff1": 0.39823470808304207, - "nauc_mrr_at_20_max": 0.16717533160175635, - "nauc_mrr_at_20_std": 0.3426694553686378, - "nauc_mrr_at_3_diff1": 0.42857037037951523, - "nauc_mrr_at_3_max": 0.17223986953199172, - "nauc_mrr_at_3_std": 0.3403057010618428, - "nauc_mrr_at_5_diff1": 0.4139839634441962, - "nauc_mrr_at_5_max": 0.17065659328509294, - "nauc_mrr_at_5_std": 0.33635437824819064, - "nauc_ndcg_at_1000_diff1": 0.3606522195615228, - "nauc_ndcg_at_1000_max": 0.15851435669428843, - "nauc_ndcg_at_1000_std": 0.3422661885031045, - "nauc_ndcg_at_100_diff1": 0.3238716697397056, - "nauc_ndcg_at_100_max": 0.1516195499748372, - "nauc_ndcg_at_100_std": 0.3455179391449604, - "nauc_ndcg_at_10_diff1": 0.376502809924429, - "nauc_ndcg_at_10_max": 0.15758508541947475, - "nauc_ndcg_at_10_std": 0.338090100250942, - "nauc_ndcg_at_1_diff1": 0.49365195440479587, - "nauc_ndcg_at_1_max": 0.2075044516883363, - "nauc_ndcg_at_1_std": 0.3627661200695744, - "nauc_ndcg_at_20_diff1": 0.34349790053029533, - "nauc_ndcg_at_20_max": 0.15398122499247816, - "nauc_ndcg_at_20_std": 0.35070963021538853, - "nauc_ndcg_at_3_diff1": 0.4104244965053797, - "nauc_ndcg_at_3_max": 0.1643291299872034, - "nauc_ndcg_at_3_std": 0.3365155277170668, - "nauc_ndcg_at_5_diff1": 0.38677310970881057, - "nauc_ndcg_at_5_max": 0.16285640227119183, - "nauc_ndcg_at_5_std": 0.33140501975502823, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.12178494113610133, - "nauc_precision_at_100_max": 0.12027108175345302, - "nauc_precision_at_100_std": 0.36680812841564553, - "nauc_precision_at_10_diff1": 0.3028234842179398, - "nauc_precision_at_10_max": 0.13271784365196254, - "nauc_precision_at_10_std": 0.33726102823484233, - "nauc_precision_at_1_diff1": 0.49365195440479587, - "nauc_precision_at_1_max": 0.2075044516883363, - "nauc_precision_at_1_std": 0.3627661200695744, - "nauc_precision_at_20_diff1": 0.21236255278748467, - "nauc_precision_at_20_max": 0.1251083950019622, - "nauc_precision_at_20_std": 0.3781734179029766, - "nauc_precision_at_3_diff1": 0.36419158351976755, - "nauc_precision_at_3_max": 0.14471642960954978, - "nauc_precision_at_3_std": 0.3275568692130576, - "nauc_precision_at_5_diff1": 0.320977582700417, - "nauc_precision_at_5_max": 0.1455113695120129, - "nauc_precision_at_5_std": 0.3207707598441935, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.12178494113610103, - "nauc_recall_at_100_max": 0.12027108175345279, - "nauc_recall_at_100_std": 0.36680812841564475, - "nauc_recall_at_10_diff1": 0.30282348421793964, - "nauc_recall_at_10_max": 0.13271784365196235, - "nauc_recall_at_10_std": 0.33726102823484205, - "nauc_recall_at_1_diff1": 0.49365195440479587, - "nauc_recall_at_1_max": 0.2075044516883363, - "nauc_recall_at_1_std": 0.3627661200695744, - "nauc_recall_at_20_diff1": 0.21236255278748464, - "nauc_recall_at_20_max": 0.1251083950019623, - "nauc_recall_at_20_std": 0.3781734179029766, - "nauc_recall_at_3_diff1": 0.3641915835197674, - "nauc_recall_at_3_max": 0.14471642960954986, - "nauc_recall_at_3_std": 0.3275568692130575, - "nauc_recall_at_5_diff1": 0.32097758270041704, - "nauc_recall_at_5_max": 0.14551136951201293, - "nauc_recall_at_5_std": 0.32077075984419345, - "ndcg_at_1": 0.12105, - "ndcg_at_10": 0.17855, - "ndcg_at_100": 0.23991, - "ndcg_at_1000": 0.29421, - "ndcg_at_20": 0.19457, - "ndcg_at_3": 0.14753, - "ndcg_at_5": 0.15984, - "precision_at_1": 0.12105, - "precision_at_10": 0.02544, - "precision_at_100": 0.00572, - "precision_at_1000": 0.001, - "precision_at_20": 0.01588, - "precision_at_3": 0.05556, - "precision_at_5": 0.0393, - "recall_at_1": 0.12105, - "recall_at_10": 0.25439, - "recall_at_100": 0.57193, - "recall_at_1000": 1.0, - "recall_at_20": 0.31754, - "recall_at_3": 0.16667, - "recall_at_5": 0.19649 - }, - { - "cv_recall_at_1": 0.17506, - "cv_recall_at_10": 0.36041, - "cv_recall_at_100": 0.67506, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.44394, - "cv_recall_at_3": 0.24828, - "cv_recall_at_5": 0.28375, - "hf_subset": "et", - "languages": [ - "est-Latn" - ], - "main_score": 0.25628, - "map_at_1": 0.17506, - "map_at_10": 0.22449, - "map_at_100": 0.23626, - "map_at_1000": 0.23781, - "map_at_20": 0.23033, - "map_at_3": 0.20633, - "map_at_5": 0.21445, - "mrr_at_1": 0.17505720823798626, - "mrr_at_10": 0.22448830409356724, - "mrr_at_100": 0.23626441711645363, - "mrr_at_1000": 0.2378086038142683, - "mrr_at_20": 0.23033407615650203, - "mrr_at_3": 0.2063310450038139, - "mrr_at_5": 0.2144546147978643, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.26730111731361084, - "nauc_cv_recall_at_100_max": 0.5047094672424741, - "nauc_cv_recall_at_100_std": 0.3852903729729708, - "nauc_cv_recall_at_10_diff1": 0.4619307725560599, - "nauc_cv_recall_at_10_max": 0.5193135193608889, - "nauc_cv_recall_at_10_std": 0.34891718392354365, - "nauc_cv_recall_at_1_diff1": 0.65277071094691, - "nauc_cv_recall_at_1_max": 0.5511794943258216, - "nauc_cv_recall_at_1_std": 0.27970900052689085, - "nauc_cv_recall_at_20_diff1": 0.3967413563077368, - "nauc_cv_recall_at_20_max": 0.4866601040699307, - "nauc_cv_recall_at_20_std": 0.3427263005944641, - "nauc_cv_recall_at_3_diff1": 0.5551163506532734, - "nauc_cv_recall_at_3_max": 0.5456634262644148, - "nauc_cv_recall_at_3_std": 0.3272531670651522, - "nauc_cv_recall_at_5_diff1": 0.5006540117355701, - "nauc_cv_recall_at_5_max": 0.5105691605247302, - "nauc_cv_recall_at_5_std": 0.30568708640340475, - "nauc_map_at_1000_diff1": 0.5709223381463038, - "nauc_map_at_1000_max": 0.534545552565501, - "nauc_map_at_1000_std": 0.3044079913907693, - "nauc_map_at_100_diff1": 0.5698204666019357, - "nauc_map_at_100_max": 0.5345082705210916, - "nauc_map_at_100_std": 0.30478680324037544, - "nauc_map_at_10_diff1": 0.5781645404095507, - "nauc_map_at_10_max": 0.5370703656719066, - "nauc_map_at_10_std": 0.30422950515207864, - "nauc_map_at_1_diff1": 0.65277071094691, - "nauc_map_at_1_max": 0.5511794943258216, - "nauc_map_at_1_std": 0.27970900052689085, - "nauc_map_at_20_diff1": 0.5725853252115725, - "nauc_map_at_20_max": 0.5340676903708351, - "nauc_map_at_20_std": 0.3040158147251269, - "nauc_map_at_3_diff1": 0.6017990320281447, - "nauc_map_at_3_max": 0.5447037216519792, - "nauc_map_at_3_std": 0.30153453842423195, - "nauc_map_at_5_diff1": 0.5858313223784389, - "nauc_map_at_5_max": 0.5359906741690218, - "nauc_map_at_5_std": 0.2972934243394176, - "nauc_mrr_at_1000_diff1": 0.5709223381463038, - "nauc_mrr_at_1000_max": 0.534545552565501, - "nauc_mrr_at_1000_std": 0.3044079913907693, - "nauc_mrr_at_100_diff1": 0.5698204666019357, - "nauc_mrr_at_100_max": 0.5345082705210916, - "nauc_mrr_at_100_std": 0.30478680324037544, - "nauc_mrr_at_10_diff1": 0.5781645404095507, - "nauc_mrr_at_10_max": 0.5370703656719066, - "nauc_mrr_at_10_std": 0.30422950515207864, - "nauc_mrr_at_1_diff1": 0.65277071094691, - "nauc_mrr_at_1_max": 0.5511794943258216, - "nauc_mrr_at_1_std": 0.27970900052689085, - "nauc_mrr_at_20_diff1": 0.5725853252115725, - "nauc_mrr_at_20_max": 0.5340676903708351, - "nauc_mrr_at_20_std": 0.3040158147251269, - "nauc_mrr_at_3_diff1": 0.6017990320281447, - "nauc_mrr_at_3_max": 0.5447037216519792, - "nauc_mrr_at_3_std": 0.30153453842423195, - "nauc_mrr_at_5_diff1": 0.5858313223784389, - "nauc_mrr_at_5_max": 0.5359906741690218, - "nauc_mrr_at_5_std": 0.2972934243394176, - "nauc_ndcg_at_1000_diff1": 0.5408301481972125, - "nauc_ndcg_at_1000_max": 0.5292889263873282, - "nauc_ndcg_at_1000_std": 0.3118164266892448, - "nauc_ndcg_at_100_diff1": 0.5095321706069865, - "nauc_ndcg_at_100_max": 0.5266240841593062, - "nauc_ndcg_at_100_std": 0.3206075047355115, - "nauc_ndcg_at_10_diff1": 0.5458492116430604, - "nauc_ndcg_at_10_max": 0.5320211872393085, - "nauc_ndcg_at_10_std": 0.3162506797195962, - "nauc_ndcg_at_1_diff1": 0.65277071094691, - "nauc_ndcg_at_1_max": 0.5511794943258216, - "nauc_ndcg_at_1_std": 0.27970900052689085, - "nauc_ndcg_at_20_diff1": 0.5266549124213301, - "nauc_ndcg_at_20_max": 0.5221614113899488, - "nauc_ndcg_at_20_std": 0.3148100103251451, - "nauc_ndcg_at_3_diff1": 0.5887353752832222, - "nauc_ndcg_at_3_max": 0.544813909186769, - "nauc_ndcg_at_3_std": 0.3085999345388481, - "nauc_ndcg_at_5_diff1": 0.562131568560372, - "nauc_ndcg_at_5_max": 0.5292013580654052, - "nauc_ndcg_at_5_std": 0.30010487958695203, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.2673011173136117, - "nauc_precision_at_100_max": 0.5047094672424749, - "nauc_precision_at_100_std": 0.38529037297297136, - "nauc_precision_at_10_diff1": 0.46193077255606, - "nauc_precision_at_10_max": 0.5193135193608887, - "nauc_precision_at_10_std": 0.3489171839235438, - "nauc_precision_at_1_diff1": 0.65277071094691, - "nauc_precision_at_1_max": 0.5511794943258216, - "nauc_precision_at_1_std": 0.27970900052689085, - "nauc_precision_at_20_diff1": 0.39674135630773644, - "nauc_precision_at_20_max": 0.4866601040699304, - "nauc_precision_at_20_std": 0.3427263005944638, - "nauc_precision_at_3_diff1": 0.5551163506532731, - "nauc_precision_at_3_max": 0.5456634262644146, - "nauc_precision_at_3_std": 0.3272531670651523, - "nauc_precision_at_5_diff1": 0.5006540117355697, - "nauc_precision_at_5_max": 0.51056916052473, - "nauc_precision_at_5_std": 0.3056870864034045, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.26730111731361084, - "nauc_recall_at_100_max": 0.5047094672424741, - "nauc_recall_at_100_std": 0.3852903729729708, - "nauc_recall_at_10_diff1": 0.4619307725560599, - "nauc_recall_at_10_max": 0.5193135193608889, - "nauc_recall_at_10_std": 0.34891718392354365, - "nauc_recall_at_1_diff1": 0.65277071094691, - "nauc_recall_at_1_max": 0.5511794943258216, - "nauc_recall_at_1_std": 0.27970900052689085, - "nauc_recall_at_20_diff1": 0.3967413563077368, - "nauc_recall_at_20_max": 0.4866601040699307, - "nauc_recall_at_20_std": 0.3427263005944641, - "nauc_recall_at_3_diff1": 0.5551163506532734, - "nauc_recall_at_3_max": 0.5456634262644148, - "nauc_recall_at_3_std": 0.3272531670651522, - "nauc_recall_at_5_diff1": 0.5006540117355701, - "nauc_recall_at_5_max": 0.5105691605247302, - "nauc_recall_at_5_std": 0.30568708640340475, - "ndcg_at_1": 0.17506, - "ndcg_at_10": 0.25628, - "ndcg_at_100": 0.32037, - "ndcg_at_1000": 0.36188, - "ndcg_at_20": 0.27746, - "ndcg_at_3": 0.21706, - "ndcg_at_5": 0.23169, - "precision_at_1": 0.17506, - "precision_at_10": 0.03604, - "precision_at_100": 0.00675, - "precision_at_1000": 0.001, - "precision_at_20": 0.0222, - "precision_at_3": 0.08276, - "precision_at_5": 0.05675, - "recall_at_1": 0.17506, - "recall_at_10": 0.36041, - "recall_at_100": 0.67506, - "recall_at_1000": 1.0, - "recall_at_20": 0.44394, - "recall_at_3": 0.24828, - "recall_at_5": 0.28375 - }, - { - "cv_recall_at_1": 0.39401, - "cv_recall_at_10": 0.65927, - "cv_recall_at_100": 0.91343, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.7525, - "cv_recall_at_3": 0.52275, - "cv_recall_at_5": 0.57603, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.51787, - "map_at_1": 0.39401, - "map_at_10": 0.47368, - "map_at_100": 0.48465, - "map_at_1000": 0.48511, - "map_at_20": 0.48026, - "map_at_3": 0.45006, - "map_at_5": 0.46243, - "mrr_at_1": 0.3940066592674806, - "mrr_at_10": 0.4736831210471608, - "mrr_at_100": 0.4846487645407278, - "mrr_at_1000": 0.4851121293358373, - "mrr_at_20": 0.4802649192099936, - "mrr_at_3": 0.4500554938956712, - "mrr_at_5": 0.46243063263041023, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.3606790371733193, - "nauc_cv_recall_at_100_max": 0.6460443767082419, - "nauc_cv_recall_at_100_std": 0.6438264729443607, - "nauc_cv_recall_at_10_diff1": 0.5070064486702217, - "nauc_cv_recall_at_10_max": 0.4369360291530516, - "nauc_cv_recall_at_10_std": 0.4019920484623479, - "nauc_cv_recall_at_1_diff1": 0.6787729267117939, - "nauc_cv_recall_at_1_max": 0.4483448292991838, - "nauc_cv_recall_at_1_std": 0.2086975409018547, - "nauc_cv_recall_at_20_diff1": 0.44455980762664943, - "nauc_cv_recall_at_20_max": 0.43252662858604923, - "nauc_cv_recall_at_20_std": 0.4659697186595289, - "nauc_cv_recall_at_3_diff1": 0.5711627394926658, - "nauc_cv_recall_at_3_max": 0.4173021015328537, - "nauc_cv_recall_at_3_std": 0.303306184425467, - "nauc_cv_recall_at_5_diff1": 0.5555793126760551, - "nauc_cv_recall_at_5_max": 0.42026344838912505, - "nauc_cv_recall_at_5_std": 0.3200825079115572, - "nauc_map_at_1000_diff1": 0.625727698845203, - "nauc_map_at_1000_max": 0.43833007897791126, - "nauc_map_at_1000_std": 0.2609118106038958, - "nauc_map_at_100_diff1": 0.6253584933406903, - "nauc_map_at_100_max": 0.43857408105349865, - "nauc_map_at_100_std": 0.26139464324542994, - "nauc_map_at_10_diff1": 0.6256223969666678, - "nauc_map_at_10_max": 0.43741404152936747, - "nauc_map_at_10_std": 0.2606781274913183, - "nauc_map_at_1_diff1": 0.6787729267117939, - "nauc_map_at_1_max": 0.4483448292991838, - "nauc_map_at_1_std": 0.2086975409018547, - "nauc_map_at_20_diff1": 0.6241272092493547, - "nauc_map_at_20_max": 0.4364857418691703, - "nauc_map_at_20_std": 0.2619722199064384, - "nauc_map_at_3_diff1": 0.6331140514702438, - "nauc_map_at_3_max": 0.43570365229430424, - "nauc_map_at_3_std": 0.2510720421324455, - "nauc_map_at_5_diff1": 0.6300959184867589, - "nauc_map_at_5_max": 0.43632773692723537, - "nauc_map_at_5_std": 0.254354628949991, - "nauc_mrr_at_1000_diff1": 0.6257275882226064, - "nauc_mrr_at_1000_max": 0.4383296579750604, - "nauc_mrr_at_1000_std": 0.26091128658086987, - "nauc_mrr_at_100_diff1": 0.6253583638531727, - "nauc_mrr_at_100_max": 0.4385736325944442, - "nauc_mrr_at_100_std": 0.26139408315403995, - "nauc_mrr_at_10_diff1": 0.6256223969666678, - "nauc_mrr_at_10_max": 0.43741404152936747, - "nauc_mrr_at_10_std": 0.2606781274913183, - "nauc_mrr_at_1_diff1": 0.6787729267117939, - "nauc_mrr_at_1_max": 0.4483448292991838, - "nauc_mrr_at_1_std": 0.2086975409018547, - "nauc_mrr_at_20_diff1": 0.6241272092493547, - "nauc_mrr_at_20_max": 0.4364857418691703, - "nauc_mrr_at_20_std": 0.2619722199064384, - "nauc_mrr_at_3_diff1": 0.6331140514702438, - "nauc_mrr_at_3_max": 0.43570365229430424, - "nauc_mrr_at_3_std": 0.2510720421324455, - "nauc_mrr_at_5_diff1": 0.6300959184867589, - "nauc_mrr_at_5_max": 0.43632773692723537, - "nauc_mrr_at_5_std": 0.254354628949991, - "nauc_ndcg_at_1000_diff1": 0.6119100731636649, - "nauc_ndcg_at_1000_max": 0.44040411165998694, - "nauc_ndcg_at_1000_std": 0.2770059786004134, - "nauc_ndcg_at_100_diff1": 0.6020858600990256, - "nauc_ndcg_at_100_max": 0.4482027804597055, - "nauc_ndcg_at_100_std": 0.29109437970273, - "nauc_ndcg_at_10_diff1": 0.6006859120736538, - "nauc_ndcg_at_10_max": 0.43643008773180164, - "nauc_ndcg_at_10_std": 0.28916183770833015, - "nauc_ndcg_at_1_diff1": 0.6787729267117939, - "nauc_ndcg_at_1_max": 0.4483448292991838, - "nauc_ndcg_at_1_std": 0.2086975409018547, - "nauc_ndcg_at_20_diff1": 0.5939399449347809, - "nauc_ndcg_at_20_max": 0.43464448439569164, - "nauc_ndcg_at_20_std": 0.29524167525362205, - "nauc_ndcg_at_3_diff1": 0.6176093944148927, - "nauc_ndcg_at_3_max": 0.4311207827652941, - "nauc_ndcg_at_3_std": 0.2642395616086192, - "nauc_ndcg_at_5_diff1": 0.6121797185622866, - "nauc_ndcg_at_5_max": 0.4323791466523419, - "nauc_ndcg_at_5_std": 0.27020359365711794, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3606790371733184, - "nauc_precision_at_100_max": 0.6460443767082423, - "nauc_precision_at_100_std": 0.64382647294436, - "nauc_precision_at_10_diff1": 0.5070064486702209, - "nauc_precision_at_10_max": 0.436936029153051, - "nauc_precision_at_10_std": 0.40199204846234754, - "nauc_precision_at_1_diff1": 0.6787729267117939, - "nauc_precision_at_1_max": 0.4483448292991838, - "nauc_precision_at_1_std": 0.2086975409018547, - "nauc_precision_at_20_diff1": 0.4445598076266497, - "nauc_precision_at_20_max": 0.4325266285860495, - "nauc_precision_at_20_std": 0.46596971865952863, - "nauc_precision_at_3_diff1": 0.5711627394926657, - "nauc_precision_at_3_max": 0.4173021015328535, - "nauc_precision_at_3_std": 0.30330618442546653, - "nauc_precision_at_5_diff1": 0.5555793126760548, - "nauc_precision_at_5_max": 0.420263448389125, - "nauc_precision_at_5_std": 0.3200825079115574, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.3606790371733193, - "nauc_recall_at_100_max": 0.6460443767082419, - "nauc_recall_at_100_std": 0.6438264729443607, - "nauc_recall_at_10_diff1": 0.5070064486702217, - "nauc_recall_at_10_max": 0.4369360291530516, - "nauc_recall_at_10_std": 0.4019920484623479, - "nauc_recall_at_1_diff1": 0.6787729267117939, - "nauc_recall_at_1_max": 0.4483448292991838, - "nauc_recall_at_1_std": 0.2086975409018547, - "nauc_recall_at_20_diff1": 0.44455980762664943, - "nauc_recall_at_20_max": 0.43252662858604923, - "nauc_recall_at_20_std": 0.4659697186595289, - "nauc_recall_at_3_diff1": 0.5711627394926658, - "nauc_recall_at_3_max": 0.4173021015328537, - "nauc_recall_at_3_std": 0.303306184425467, - "nauc_recall_at_5_diff1": 0.5555793126760551, - "nauc_recall_at_5_max": 0.42026344838912505, - "nauc_recall_at_5_std": 0.3200825079115572, - "ndcg_at_1": 0.39401, - "ndcg_at_10": 0.51787, - "ndcg_at_100": 0.57193, - "ndcg_at_1000": 0.58323, - "ndcg_at_20": 0.54158, - "ndcg_at_3": 0.4687, - "ndcg_at_5": 0.49081, - "precision_at_1": 0.39401, - "precision_at_10": 0.06593, - "precision_at_100": 0.00913, - "precision_at_1000": 0.001, - "precision_at_20": 0.03762, - "precision_at_3": 0.17425, - "precision_at_5": 0.11521, - "recall_at_1": 0.39401, - "recall_at_10": 0.65927, - "recall_at_100": 0.91343, - "recall_at_1000": 1.0, - "recall_at_20": 0.7525, - "recall_at_3": 0.52275, - "recall_at_5": 0.57603 - }, - { - "cv_recall_at_1": 0.08378, - "cv_recall_at_10": 0.21375, - "cv_recall_at_100": 0.48013, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.28571, - "cv_recall_at_3": 0.12675, - "cv_recall_at_5": 0.15682, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.13899, - "map_at_1": 0.08378, - "map_at_10": 0.11634, - "map_at_100": 0.12578, - "map_at_1000": 0.12783, - "map_at_20": 0.12128, - "map_at_3": 0.10204, - "map_at_5": 0.10881, - "mrr_at_1": 0.08378088077336197, - "mrr_at_10": 0.11634358003853172, - "mrr_at_100": 0.12577563297812794, - "mrr_at_1000": 0.1278294672817507, - "mrr_at_20": 0.12128160331334957, - "mrr_at_3": 0.10204081632653061, - "mrr_at_5": 0.10880773361976372, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.08094760047818926, - "nauc_cv_recall_at_100_max": 0.17066956375947087, - "nauc_cv_recall_at_100_std": 0.38563047630654224, - "nauc_cv_recall_at_10_diff1": 0.2839044210475601, - "nauc_cv_recall_at_10_max": 0.31545350408678147, - "nauc_cv_recall_at_10_std": 0.3441278004612633, - "nauc_cv_recall_at_1_diff1": 0.6291818808071652, - "nauc_cv_recall_at_1_max": 0.5054935879352562, - "nauc_cv_recall_at_1_std": 0.3865691801255298, - "nauc_cv_recall_at_20_diff1": 0.21817134251307765, - "nauc_cv_recall_at_20_max": 0.2592512547042223, - "nauc_cv_recall_at_20_std": 0.34702007706662635, - "nauc_cv_recall_at_3_diff1": 0.4864187243485422, - "nauc_cv_recall_at_3_max": 0.38426154022878617, - "nauc_cv_recall_at_3_std": 0.3491919668762798, - "nauc_cv_recall_at_5_diff1": 0.4063794488396466, - "nauc_cv_recall_at_5_max": 0.33969774274368847, - "nauc_cv_recall_at_5_std": 0.34046314170571884, - "nauc_map_at_1000_diff1": 0.47827949302761363, - "nauc_map_at_1000_max": 0.4064512152065326, - "nauc_map_at_1000_std": 0.3649575210259258, - "nauc_map_at_100_diff1": 0.4759593348882534, - "nauc_map_at_100_max": 0.40529052275374966, - "nauc_map_at_100_std": 0.36497875541695995, - "nauc_map_at_10_diff1": 0.49485792134613193, - "nauc_map_at_10_max": 0.4188337948334773, - "nauc_map_at_10_std": 0.3637559483655701, - "nauc_map_at_1_diff1": 0.6291818808071652, - "nauc_map_at_1_max": 0.5054935879352562, - "nauc_map_at_1_std": 0.3865691801255298, - "nauc_map_at_20_diff1": 0.4839601045926048, - "nauc_map_at_20_max": 0.4102131481142997, - "nauc_map_at_20_std": 0.3642448061238864, - "nauc_map_at_3_diff1": 0.5544338142604952, - "nauc_map_at_3_max": 0.44292832818211825, - "nauc_map_at_3_std": 0.3667412598594845, - "nauc_map_at_5_diff1": 0.526947072293995, - "nauc_map_at_5_max": 0.4278398208874142, - "nauc_map_at_5_std": 0.36383180100575213, - "nauc_mrr_at_1000_diff1": 0.47827803118281714, - "nauc_mrr_at_1000_max": 0.4064523613247372, - "nauc_mrr_at_1000_std": 0.3649567863374473, - "nauc_mrr_at_100_diff1": 0.47595788183689713, - "nauc_mrr_at_100_max": 0.40529166241657266, - "nauc_mrr_at_100_std": 0.3649780249729875, - "nauc_mrr_at_10_diff1": 0.49485792134613193, - "nauc_mrr_at_10_max": 0.4188337948334773, - "nauc_mrr_at_10_std": 0.3637559483655701, - "nauc_mrr_at_1_diff1": 0.6291818808071652, - "nauc_mrr_at_1_max": 0.5054935879352562, - "nauc_mrr_at_1_std": 0.3865691801255298, - "nauc_mrr_at_20_diff1": 0.4839601045926048, - "nauc_mrr_at_20_max": 0.4102131481142997, - "nauc_mrr_at_20_std": 0.3642448061238864, - "nauc_mrr_at_3_diff1": 0.5544338142604952, - "nauc_mrr_at_3_max": 0.44292832818211825, - "nauc_mrr_at_3_std": 0.3667412598594845, - "nauc_mrr_at_5_diff1": 0.526947072293995, - "nauc_mrr_at_5_max": 0.4278398208874142, - "nauc_mrr_at_5_std": 0.36383180100575213, - "nauc_ndcg_at_1000_diff1": 0.4134594567842365, - "nauc_ndcg_at_1000_max": 0.3679622197162452, - "nauc_ndcg_at_1000_std": 0.36383698541450976, - "nauc_ndcg_at_100_diff1": 0.3522286150645984, - "nauc_ndcg_at_100_max": 0.3324224846540905, - "nauc_ndcg_at_100_std": 0.36735232562757786, - "nauc_ndcg_at_10_diff1": 0.42677518649871554, - "nauc_ndcg_at_10_max": 0.3845568065271432, - "nauc_ndcg_at_10_std": 0.3570034792300479, - "nauc_ndcg_at_1_diff1": 0.6291818808071652, - "nauc_ndcg_at_1_max": 0.5054935879352562, - "nauc_ndcg_at_1_std": 0.3865691801255298, - "nauc_ndcg_at_20_diff1": 0.3954488082966329, - "nauc_ndcg_at_20_max": 0.35973541870986864, - "nauc_ndcg_at_20_std": 0.3579997255648102, - "nauc_ndcg_at_3_diff1": 0.534581543463932, - "nauc_ndcg_at_3_max": 0.4258413031304955, - "nauc_ndcg_at_3_std": 0.3616085694555746, - "nauc_ndcg_at_5_diff1": 0.49034708200589217, - "nauc_ndcg_at_5_max": 0.40104217893271416, - "nauc_ndcg_at_5_std": 0.35668991327782323, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.08094760047818939, - "nauc_precision_at_100_max": 0.17066956375947098, - "nauc_precision_at_100_std": 0.3856304763065422, - "nauc_precision_at_10_diff1": 0.2839044210475601, - "nauc_precision_at_10_max": 0.31545350408678124, - "nauc_precision_at_10_std": 0.3441278004612635, - "nauc_precision_at_1_diff1": 0.6291818808071652, - "nauc_precision_at_1_max": 0.5054935879352562, - "nauc_precision_at_1_std": 0.3865691801255298, - "nauc_precision_at_20_diff1": 0.21817134251307746, - "nauc_precision_at_20_max": 0.2592512547042219, - "nauc_precision_at_20_std": 0.34702007706662613, - "nauc_precision_at_3_diff1": 0.48641872434854205, - "nauc_precision_at_3_max": 0.384261540228786, - "nauc_precision_at_3_std": 0.34919196687627996, - "nauc_precision_at_5_diff1": 0.40637944883964683, - "nauc_precision_at_5_max": 0.33969774274368847, - "nauc_precision_at_5_std": 0.3404631417057189, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.08094760047818926, - "nauc_recall_at_100_max": 0.17066956375947087, - "nauc_recall_at_100_std": 0.38563047630654224, - "nauc_recall_at_10_diff1": 0.2839044210475601, - "nauc_recall_at_10_max": 0.31545350408678147, - "nauc_recall_at_10_std": 0.3441278004612633, - "nauc_recall_at_1_diff1": 0.6291818808071652, - "nauc_recall_at_1_max": 0.5054935879352562, - "nauc_recall_at_1_std": 0.3865691801255298, - "nauc_recall_at_20_diff1": 0.21817134251307765, - "nauc_recall_at_20_max": 0.2592512547042223, - "nauc_recall_at_20_std": 0.34702007706662635, - "nauc_recall_at_3_diff1": 0.4864187243485422, - "nauc_recall_at_3_max": 0.38426154022878617, - "nauc_recall_at_3_std": 0.3491919668762798, - "nauc_recall_at_5_diff1": 0.4063794488396466, - "nauc_recall_at_5_max": 0.33969774274368847, - "nauc_recall_at_5_std": 0.34046314170571884, - "ndcg_at_1": 0.08378, - "ndcg_at_10": 0.13899, - "ndcg_at_100": 0.1922, - "ndcg_at_1000": 0.25598, - "ndcg_at_20": 0.15711, - "ndcg_at_3": 0.10836, - "ndcg_at_5": 0.12065, - "precision_at_1": 0.08378, - "precision_at_10": 0.02137, - "precision_at_100": 0.0048, - "precision_at_1000": 0.001, - "precision_at_20": 0.01429, - "precision_at_3": 0.04225, - "precision_at_5": 0.03136, - "recall_at_1": 0.08378, - "recall_at_10": 0.21375, - "recall_at_100": 0.48013, - "recall_at_1000": 1.0, - "recall_at_20": 0.28571, - "recall_at_3": 0.12675, - "recall_at_5": 0.15682 - }, - { - "cv_recall_at_1": 0.096, - "cv_recall_at_10": 0.247, - "cv_recall_at_100": 0.58, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.311, - "cv_recall_at_3": 0.158, - "cv_recall_at_5": 0.198, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.16391, - "map_at_1": 0.096, - "map_at_10": 0.13824, - "map_at_100": 0.14853, - "map_at_1000": 0.15039, - "map_at_20": 0.14257, - "map_at_3": 0.12283, - "map_at_5": 0.13183, - "mrr_at_1": 0.096, - "mrr_at_10": 0.13823849206349206, - "mrr_at_100": 0.14852892899091763, - "mrr_at_1000": 0.1503909681270283, - "mrr_at_20": 0.14257447598411227, - "mrr_at_3": 0.12283333333333327, - "mrr_at_5": 0.1318333333333333, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.03122877789708922, - "nauc_cv_recall_at_100_max": 0.15807673331978997, - "nauc_cv_recall_at_100_std": 0.25700873552543785, - "nauc_cv_recall_at_10_diff1": 0.1979977193701204, - "nauc_cv_recall_at_10_max": 0.08385825781766487, - "nauc_cv_recall_at_10_std": 0.1336889853859249, - "nauc_cv_recall_at_1_diff1": 0.4098442670601914, - "nauc_cv_recall_at_1_max": 0.1960049414184777, - "nauc_cv_recall_at_1_std": 0.11682485913603434, - "nauc_cv_recall_at_20_diff1": 0.147580351467539, - "nauc_cv_recall_at_20_max": 0.10735523163073482, - "nauc_cv_recall_at_20_std": 0.12651030656989257, - "nauc_cv_recall_at_3_diff1": 0.30751900982190294, - "nauc_cv_recall_at_3_max": 0.09457216752684254, - "nauc_cv_recall_at_3_std": 0.1525148026507118, - "nauc_cv_recall_at_5_diff1": 0.2456197739175154, - "nauc_cv_recall_at_5_max": 0.06958399651079145, - "nauc_cv_recall_at_5_std": 0.14042599590538776, - "nauc_map_at_1000_diff1": 0.3108026337423453, - "nauc_map_at_1000_max": 0.13603764722136522, - "nauc_map_at_1000_std": 0.12868613440315627, - "nauc_map_at_100_diff1": 0.3091343337291221, - "nauc_map_at_100_max": 0.13531151633977603, - "nauc_map_at_100_std": 0.12846135145876467, - "nauc_map_at_10_diff1": 0.3199492115034543, - "nauc_map_at_10_max": 0.13191522051200616, - "nauc_map_at_10_std": 0.12544174366583696, - "nauc_map_at_1_diff1": 0.4098442670601914, - "nauc_map_at_1_max": 0.1960049414184777, - "nauc_map_at_1_std": 0.11682485913603434, - "nauc_map_at_20_diff1": 0.31352289693470164, - "nauc_map_at_20_max": 0.13359715775447498, - "nauc_map_at_20_std": 0.12474902480926435, - "nauc_map_at_3_diff1": 0.35261281588271315, - "nauc_map_at_3_max": 0.1419533508215326, - "nauc_map_at_3_std": 0.12829207923693836, - "nauc_map_at_5_diff1": 0.33059012872746757, - "nauc_map_at_5_max": 0.13068614410775703, - "nauc_map_at_5_std": 0.1269370499395121, - "nauc_mrr_at_1000_diff1": 0.3108026337423453, - "nauc_mrr_at_1000_max": 0.13603764722136522, - "nauc_mrr_at_1000_std": 0.12868613440315627, - "nauc_mrr_at_100_diff1": 0.3091343337291221, - "nauc_mrr_at_100_max": 0.13531151633977603, - "nauc_mrr_at_100_std": 0.12846135145876467, - "nauc_mrr_at_10_diff1": 0.3199492115034543, - "nauc_mrr_at_10_max": 0.13191522051200616, - "nauc_mrr_at_10_std": 0.12544174366583696, - "nauc_mrr_at_1_diff1": 0.4098442670601914, - "nauc_mrr_at_1_max": 0.1960049414184777, - "nauc_mrr_at_1_std": 0.11682485913603434, - "nauc_mrr_at_20_diff1": 0.31352289693470164, - "nauc_mrr_at_20_max": 0.13359715775447498, - "nauc_mrr_at_20_std": 0.12474902480926435, - "nauc_mrr_at_3_diff1": 0.35261281588271315, - "nauc_mrr_at_3_max": 0.1419533508215326, - "nauc_mrr_at_3_std": 0.12829207923693836, - "nauc_mrr_at_5_diff1": 0.33059012872746757, - "nauc_mrr_at_5_max": 0.13068614410775703, - "nauc_mrr_at_5_std": 0.1269370499395121, - "nauc_ndcg_at_1000_diff1": 0.27465338314694493, - "nauc_ndcg_at_1000_max": 0.13532791914955453, - "nauc_ndcg_at_1000_std": 0.14176692393314377, - "nauc_ndcg_at_100_diff1": 0.23470449632345772, - "nauc_ndcg_at_100_max": 0.13451761349838698, - "nauc_ndcg_at_100_std": 0.1555639746804492, - "nauc_ndcg_at_10_diff1": 0.2818115912878626, - "nauc_ndcg_at_10_max": 0.11547465009598505, - "nauc_ndcg_at_10_std": 0.12840701176314118, - "nauc_ndcg_at_1_diff1": 0.4098442670601914, - "nauc_ndcg_at_1_max": 0.1960049414184777, - "nauc_ndcg_at_1_std": 0.11682485913603434, - "nauc_ndcg_at_20_diff1": 0.2615957787270437, - "nauc_ndcg_at_20_max": 0.12234254332047498, - "nauc_ndcg_at_20_std": 0.1260256036365097, - "nauc_ndcg_at_3_diff1": 0.33920706600399225, - "nauc_ndcg_at_3_max": 0.12798246596280738, - "nauc_ndcg_at_3_std": 0.13512915509383155, - "nauc_ndcg_at_5_diff1": 0.30427309180580736, - "nauc_ndcg_at_5_max": 0.11143239360648992, - "nauc_ndcg_at_5_std": 0.1313015918668763, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.031228777897089503, - "nauc_precision_at_100_max": 0.1580767333197903, - "nauc_precision_at_100_std": 0.2570087355254377, - "nauc_precision_at_10_diff1": 0.19799771937012006, - "nauc_precision_at_10_max": 0.083858257817665, - "nauc_precision_at_10_std": 0.1336889853859248, - "nauc_precision_at_1_diff1": 0.4098442670601914, - "nauc_precision_at_1_max": 0.1960049414184777, - "nauc_precision_at_1_std": 0.11682485913603434, - "nauc_precision_at_20_diff1": 0.14758035146753876, - "nauc_precision_at_20_max": 0.10735523163073456, - "nauc_precision_at_20_std": 0.1265103065698924, - "nauc_precision_at_3_diff1": 0.30751900982190283, - "nauc_precision_at_3_max": 0.09457216752684251, - "nauc_precision_at_3_std": 0.15251480265071174, - "nauc_precision_at_5_diff1": 0.2456197739175152, - "nauc_precision_at_5_max": 0.06958399651079158, - "nauc_precision_at_5_std": 0.14042599590538776, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.03122877789708922, - "nauc_recall_at_100_max": 0.15807673331978997, - "nauc_recall_at_100_std": 0.25700873552543785, - "nauc_recall_at_10_diff1": 0.1979977193701204, - "nauc_recall_at_10_max": 0.08385825781766487, - "nauc_recall_at_10_std": 0.1336889853859249, - "nauc_recall_at_1_diff1": 0.4098442670601914, - "nauc_recall_at_1_max": 0.1960049414184777, - "nauc_recall_at_1_std": 0.11682485913603434, - "nauc_recall_at_20_diff1": 0.147580351467539, - "nauc_recall_at_20_max": 0.10735523163073482, - "nauc_recall_at_20_std": 0.12651030656989257, - "nauc_recall_at_3_diff1": 0.30751900982190294, - "nauc_recall_at_3_max": 0.09457216752684254, - "nauc_recall_at_3_std": 0.1525148026507118, - "nauc_recall_at_5_diff1": 0.2456197739175154, - "nauc_recall_at_5_max": 0.06958399651079145, - "nauc_recall_at_5_std": 0.14042599590538776, - "ndcg_at_1": 0.096, - "ndcg_at_10": 0.16391, - "ndcg_at_100": 0.22792, - "ndcg_at_1000": 0.28067, - "ndcg_at_20": 0.17996, - "ndcg_at_3": 0.13184, - "ndcg_at_5": 0.14819, - "precision_at_1": 0.096, - "precision_at_10": 0.0247, - "precision_at_100": 0.0058, - "precision_at_1000": 0.001, - "precision_at_20": 0.01555, - "precision_at_3": 0.05267, - "precision_at_5": 0.0396, - "recall_at_1": 0.096, - "recall_at_10": 0.247, - "recall_at_100": 0.58, - "recall_at_1000": 1.0, - "recall_at_20": 0.311, - "recall_at_3": 0.158, - "recall_at_5": 0.198 - }, - { - "cv_recall_at_1": 0.32594, - "cv_recall_at_10": 0.58807, - "cv_recall_at_100": 0.83911, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.66297, - "cv_recall_at_3": 0.45492, - "cv_recall_at_5": 0.52011, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.44964, - "map_at_1": 0.32594, - "map_at_10": 0.40614, - "map_at_100": 0.41528, - "map_at_1000": 0.41617, - "map_at_20": 0.41108, - "map_at_3": 0.38234, - "map_at_5": 0.39704, - "mrr_at_1": 0.3259361997226075, - "mrr_at_10": 0.40613841005657914, - "mrr_at_100": 0.4152786021504557, - "mrr_at_1000": 0.41616716847899377, - "mrr_at_20": 0.41107626755006554, - "mrr_at_3": 0.3823393435043923, - "mrr_at_5": 0.3970411465557097, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4143125486845753, - "nauc_cv_recall_at_100_max": 0.5320816743201151, - "nauc_cv_recall_at_100_std": 0.5665864175181943, - "nauc_cv_recall_at_10_diff1": 0.4736723432838038, - "nauc_cv_recall_at_10_max": 0.5155390517675708, - "nauc_cv_recall_at_10_std": 0.43366104906933245, - "nauc_cv_recall_at_1_diff1": 0.6524414738983796, - "nauc_cv_recall_at_1_max": 0.5512090830443038, - "nauc_cv_recall_at_1_std": 0.3502491395104, - "nauc_cv_recall_at_20_diff1": 0.4507518062183285, - "nauc_cv_recall_at_20_max": 0.5160176805499569, - "nauc_cv_recall_at_20_std": 0.519694782745015, - "nauc_cv_recall_at_3_diff1": 0.5535662951368313, - "nauc_cv_recall_at_3_max": 0.564704741066614, - "nauc_cv_recall_at_3_std": 0.4136036979599246, - "nauc_cv_recall_at_5_diff1": 0.5130205909284755, - "nauc_cv_recall_at_5_max": 0.5281322733359631, - "nauc_cv_recall_at_5_std": 0.4146731951698951, - "nauc_map_at_1000_diff1": 0.5923562901465701, - "nauc_map_at_1000_max": 0.5429739061166239, - "nauc_map_at_1000_std": 0.3848112051754737, - "nauc_map_at_100_diff1": 0.5918655013216925, - "nauc_map_at_100_max": 0.5428860352272687, - "nauc_map_at_100_std": 0.38519824890915205, - "nauc_map_at_10_diff1": 0.5913677852925365, - "nauc_map_at_10_max": 0.5421098179062059, - "nauc_map_at_10_std": 0.3813377163920301, - "nauc_map_at_1_diff1": 0.6524414738983796, - "nauc_map_at_1_max": 0.5512090830443038, - "nauc_map_at_1_std": 0.3502491395104, - "nauc_map_at_20_diff1": 0.5910724634089584, - "nauc_map_at_20_max": 0.5422948552748079, - "nauc_map_at_20_std": 0.3852347149072005, - "nauc_map_at_3_diff1": 0.604405473516488, - "nauc_map_at_3_max": 0.5517276066883616, - "nauc_map_at_3_std": 0.37888818845914163, - "nauc_map_at_5_diff1": 0.596029160112715, - "nauc_map_at_5_max": 0.5437518832787203, - "nauc_map_at_5_std": 0.3790704704001802, - "nauc_mrr_at_1000_diff1": 0.5923560278478026, - "nauc_mrr_at_1000_max": 0.5429733763931445, - "nauc_mrr_at_1000_std": 0.3848105803844461, - "nauc_mrr_at_100_diff1": 0.591865231267037, - "nauc_mrr_at_100_max": 0.5428854783615418, - "nauc_mrr_at_100_std": 0.38519758770448553, - "nauc_mrr_at_10_diff1": 0.5913677852925365, - "nauc_mrr_at_10_max": 0.5421098179062059, - "nauc_mrr_at_10_std": 0.3813377163920301, - "nauc_mrr_at_1_diff1": 0.6524414738983796, - "nauc_mrr_at_1_max": 0.5512090830443038, - "nauc_mrr_at_1_std": 0.3502491395104, - "nauc_mrr_at_20_diff1": 0.5910724634089584, - "nauc_mrr_at_20_max": 0.5422948552748079, - "nauc_mrr_at_20_std": 0.3852347149072005, - "nauc_mrr_at_3_diff1": 0.604405473516488, - "nauc_mrr_at_3_max": 0.5517276066883616, - "nauc_mrr_at_3_std": 0.37888818845914163, - "nauc_mrr_at_5_diff1": 0.596029160112715, - "nauc_mrr_at_5_max": 0.5437518832787203, - "nauc_mrr_at_5_std": 0.3790704704001802, - "nauc_ndcg_at_1000_diff1": 0.5783071578366672, - "nauc_ndcg_at_1000_max": 0.5414636189261661, - "nauc_ndcg_at_1000_std": 0.39675982530546133, - "nauc_ndcg_at_100_diff1": 0.5673150067554389, - "nauc_ndcg_at_100_max": 0.5404582633785823, - "nauc_ndcg_at_100_std": 0.4074294639470118, - "nauc_ndcg_at_10_diff1": 0.5644498003714317, - "nauc_ndcg_at_10_max": 0.5363955483158082, - "nauc_ndcg_at_10_std": 0.393513763480844, - "nauc_ndcg_at_1_diff1": 0.6524414738983796, - "nauc_ndcg_at_1_max": 0.5512090830443038, - "nauc_ndcg_at_1_std": 0.3502491395104, - "nauc_ndcg_at_20_diff1": 0.5623783362516817, - "nauc_ndcg_at_20_max": 0.5370550562641554, - "nauc_ndcg_at_20_std": 0.4099708581203125, - "nauc_ndcg_at_3_diff1": 0.5911279841759199, - "nauc_ndcg_at_3_max": 0.5548417220825114, - "nauc_ndcg_at_3_std": 0.3878572153415391, - "nauc_ndcg_at_5_diff1": 0.5755197804997088, - "nauc_ndcg_at_5_max": 0.5401619268149915, - "nauc_ndcg_at_5_std": 0.3881047375243232, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.4143125486845745, - "nauc_precision_at_100_max": 0.5320816743201164, - "nauc_precision_at_100_std": 0.5665864175181946, - "nauc_precision_at_10_diff1": 0.47367234328380414, - "nauc_precision_at_10_max": 0.5155390517675704, - "nauc_precision_at_10_std": 0.43366104906933317, - "nauc_precision_at_1_diff1": 0.6524414738983796, - "nauc_precision_at_1_max": 0.5512090830443038, - "nauc_precision_at_1_std": 0.3502491395104, - "nauc_precision_at_20_diff1": 0.4507518062183285, - "nauc_precision_at_20_max": 0.5160176805499569, - "nauc_precision_at_20_std": 0.519694782745015, - "nauc_precision_at_3_diff1": 0.553566295136831, - "nauc_precision_at_3_max": 0.5647047410666138, - "nauc_precision_at_3_std": 0.41360369795992435, - "nauc_precision_at_5_diff1": 0.513020590928476, - "nauc_precision_at_5_max": 0.528132273335963, - "nauc_precision_at_5_std": 0.41467319516989587, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4143125486845753, - "nauc_recall_at_100_max": 0.5320816743201151, - "nauc_recall_at_100_std": 0.5665864175181943, - "nauc_recall_at_10_diff1": 0.4736723432838038, - "nauc_recall_at_10_max": 0.5155390517675708, - "nauc_recall_at_10_std": 0.43366104906933245, - "nauc_recall_at_1_diff1": 0.6524414738983796, - "nauc_recall_at_1_max": 0.5512090830443038, - "nauc_recall_at_1_std": 0.3502491395104, - "nauc_recall_at_20_diff1": 0.4507518062183285, - "nauc_recall_at_20_max": 0.5160176805499569, - "nauc_recall_at_20_std": 0.519694782745015, - "nauc_recall_at_3_diff1": 0.5535662951368313, - "nauc_recall_at_3_max": 0.564704741066614, - "nauc_recall_at_3_std": 0.4136036979599246, - "nauc_recall_at_5_diff1": 0.5130205909284755, - "nauc_recall_at_5_max": 0.5281322733359631, - "nauc_recall_at_5_std": 0.4146731951698951, - "ndcg_at_1": 0.32594, - "ndcg_at_10": 0.44964, - "ndcg_at_100": 0.50029, - "ndcg_at_1000": 0.52147, - "ndcg_at_20": 0.46825, - "ndcg_at_3": 0.40096, - "ndcg_at_5": 0.42764, - "precision_at_1": 0.32594, - "precision_at_10": 0.05881, - "precision_at_100": 0.00839, - "precision_at_1000": 0.001, - "precision_at_20": 0.03315, - "precision_at_3": 0.15164, - "precision_at_5": 0.10402, - "recall_at_1": 0.32594, - "recall_at_10": 0.58807, - "recall_at_100": 0.83911, - "recall_at_1000": 1.0, - "recall_at_20": 0.66297, - "recall_at_3": 0.45492, - "recall_at_5": 0.52011 - }, - { - "cv_recall_at_1": 0.32558, - "cv_recall_at_10": 0.57294, - "cv_recall_at_100": 0.83298, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.65539, - "cv_recall_at_3": 0.45243, - "cv_recall_at_5": 0.50317, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.44269, - "map_at_1": 0.32558, - "map_at_10": 0.40177, - "map_at_100": 0.41202, - "map_at_1000": 0.41283, - "map_at_20": 0.4075, - "map_at_3": 0.38073, - "map_at_5": 0.39246, - "mrr_at_1": 0.32558139534883723, - "mrr_at_10": 0.4017743884022952, - "mrr_at_100": 0.41201766810451657, - "mrr_at_1000": 0.41282790686568654, - "mrr_at_20": 0.4074975696352814, - "mrr_at_3": 0.38072586328400265, - "mrr_at_5": 0.3924594785059898, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.33319717829563866, - "nauc_cv_recall_at_100_max": 0.4067449927437285, - "nauc_cv_recall_at_100_std": 0.6174517316377853, - "nauc_cv_recall_at_10_diff1": 0.486821458931059, - "nauc_cv_recall_at_10_max": 0.3902786386096522, - "nauc_cv_recall_at_10_std": 0.37508230270815346, - "nauc_cv_recall_at_1_diff1": 0.6367299436732132, - "nauc_cv_recall_at_1_max": 0.4959073257152778, - "nauc_cv_recall_at_1_std": 0.34968642580726267, - "nauc_cv_recall_at_20_diff1": 0.44159761215278753, - "nauc_cv_recall_at_20_max": 0.3932788718769707, - "nauc_cv_recall_at_20_std": 0.4449484107288282, - "nauc_cv_recall_at_3_diff1": 0.5273113591091085, - "nauc_cv_recall_at_3_max": 0.4477604405417835, - "nauc_cv_recall_at_3_std": 0.3882510977951838, - "nauc_cv_recall_at_5_diff1": 0.5219606524762571, - "nauc_cv_recall_at_5_max": 0.4263623280399427, - "nauc_cv_recall_at_5_std": 0.39232890856205144, - "nauc_map_at_1000_diff1": 0.5805297718176744, - "nauc_map_at_1000_max": 0.4675574167497707, - "nauc_map_at_1000_std": 0.3717808238606819, - "nauc_map_at_100_diff1": 0.5799039262803479, - "nauc_map_at_100_max": 0.4673313547760212, - "nauc_map_at_100_std": 0.37223231199122314, - "nauc_map_at_10_diff1": 0.5818443261985234, - "nauc_map_at_10_max": 0.4655432092875299, - "nauc_map_at_10_std": 0.3661278338064773, - "nauc_map_at_1_diff1": 0.6367299436732132, - "nauc_map_at_1_max": 0.4959073257152778, - "nauc_map_at_1_std": 0.34968642580726267, - "nauc_map_at_20_diff1": 0.5800487033799071, - "nauc_map_at_20_max": 0.46633585873697075, - "nauc_map_at_20_std": 0.3703071346179496, - "nauc_map_at_3_diff1": 0.5858912260990712, - "nauc_map_at_3_max": 0.47405430448817926, - "nauc_map_at_3_std": 0.3664630113569763, - "nauc_map_at_5_diff1": 0.5852808145127003, - "nauc_map_at_5_max": 0.4692733799352851, - "nauc_map_at_5_std": 0.367719016060064, - "nauc_mrr_at_1000_diff1": 0.5805298300860103, - "nauc_mrr_at_1000_max": 0.4675573794546013, - "nauc_mrr_at_1000_std": 0.3717808086064955, - "nauc_mrr_at_100_diff1": 0.5799039262803479, - "nauc_mrr_at_100_max": 0.4673313547760212, - "nauc_mrr_at_100_std": 0.37223231199122314, - "nauc_mrr_at_10_diff1": 0.5818443261985234, - "nauc_mrr_at_10_max": 0.4655432092875299, - "nauc_mrr_at_10_std": 0.3661278338064773, - "nauc_mrr_at_1_diff1": 0.6367299436732132, - "nauc_mrr_at_1_max": 0.4959073257152778, - "nauc_mrr_at_1_std": 0.34968642580726267, - "nauc_mrr_at_20_diff1": 0.5800487033799071, - "nauc_mrr_at_20_max": 0.46633585873697075, - "nauc_mrr_at_20_std": 0.3703071346179496, - "nauc_mrr_at_3_diff1": 0.5858912260990712, - "nauc_mrr_at_3_max": 0.47405430448817926, - "nauc_mrr_at_3_std": 0.3664630113569763, - "nauc_mrr_at_5_diff1": 0.5852808145127003, - "nauc_mrr_at_5_max": 0.4692733799352851, - "nauc_mrr_at_5_std": 0.367719016060064, - "nauc_ndcg_at_1000_diff1": 0.5656743421506273, - "nauc_ndcg_at_1000_max": 0.461169825921946, - "nauc_ndcg_at_1000_std": 0.3826545468837048, - "nauc_ndcg_at_100_diff1": 0.5501611366152832, - "nauc_ndcg_at_100_max": 0.4571657863531696, - "nauc_ndcg_at_100_std": 0.3973814457501108, - "nauc_ndcg_at_10_diff1": 0.5598604439318214, - "nauc_ndcg_at_10_max": 0.44870119146458565, - "nauc_ndcg_at_10_std": 0.3690957138781721, - "nauc_ndcg_at_1_diff1": 0.6367299436732132, - "nauc_ndcg_at_1_max": 0.4959073257152778, - "nauc_ndcg_at_1_std": 0.34968642580726267, - "nauc_ndcg_at_20_diff1": 0.5523867049209488, - "nauc_ndcg_at_20_max": 0.4511549065343426, - "nauc_ndcg_at_20_std": 0.3843038335913939, - "nauc_ndcg_at_3_diff1": 0.5706925500420406, - "nauc_ndcg_at_3_max": 0.4672549158811777, - "nauc_ndcg_at_3_std": 0.3720660751976934, - "nauc_ndcg_at_5_diff1": 0.5691547732715783, - "nauc_ndcg_at_5_max": 0.45861800860996577, - "nauc_ndcg_at_5_std": 0.3739796280197634, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.3331971782956386, - "nauc_precision_at_100_max": 0.4067449927437276, - "nauc_precision_at_100_std": 0.6174517316377839, - "nauc_precision_at_10_diff1": 0.4868214589310593, - "nauc_precision_at_10_max": 0.39027863860965145, - "nauc_precision_at_10_std": 0.375082302708153, - "nauc_precision_at_1_diff1": 0.6367299436732132, - "nauc_precision_at_1_max": 0.4959073257152778, - "nauc_precision_at_1_std": 0.34968642580726267, - "nauc_precision_at_20_diff1": 0.441597612152787, - "nauc_precision_at_20_max": 0.39327887187697014, - "nauc_precision_at_20_std": 0.4449484107288272, - "nauc_precision_at_3_diff1": 0.5273113591091089, - "nauc_precision_at_3_max": 0.4477604405417836, - "nauc_precision_at_3_std": 0.38825109779518374, - "nauc_precision_at_5_diff1": 0.5219606524762569, - "nauc_precision_at_5_max": 0.4263623280399423, - "nauc_precision_at_5_std": 0.39232890856205105, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.33319717829563866, - "nauc_recall_at_100_max": 0.4067449927437285, - "nauc_recall_at_100_std": 0.6174517316377853, - "nauc_recall_at_10_diff1": 0.486821458931059, - "nauc_recall_at_10_max": 0.3902786386096522, - "nauc_recall_at_10_std": 0.37508230270815346, - "nauc_recall_at_1_diff1": 0.6367299436732132, - "nauc_recall_at_1_max": 0.4959073257152778, - "nauc_recall_at_1_std": 0.34968642580726267, - "nauc_recall_at_20_diff1": 0.44159761215278753, - "nauc_recall_at_20_max": 0.3932788718769707, - "nauc_recall_at_20_std": 0.4449484107288282, - "nauc_recall_at_3_diff1": 0.5273113591091085, - "nauc_recall_at_3_max": 0.4477604405417835, - "nauc_recall_at_3_std": 0.3882510977951838, - "nauc_recall_at_5_diff1": 0.5219606524762571, - "nauc_recall_at_5_max": 0.4263623280399427, - "nauc_recall_at_5_std": 0.39232890856205144, - "ndcg_at_1": 0.32558, - "ndcg_at_10": 0.44269, - "ndcg_at_100": 0.49634, - "ndcg_at_1000": 0.51769, - "ndcg_at_20": 0.46354, - "ndcg_at_3": 0.39911, - "ndcg_at_5": 0.42013, - "precision_at_1": 0.32558, - "precision_at_10": 0.05729, - "precision_at_100": 0.00833, - "precision_at_1000": 0.001, - "precision_at_20": 0.03277, - "precision_at_3": 0.15081, - "precision_at_5": 0.10063, - "recall_at_1": 0.32558, - "recall_at_10": 0.57294, - "recall_at_100": 0.83298, - "recall_at_1000": 1.0, - "recall_at_20": 0.65539, - "recall_at_3": 0.45243, - "recall_at_5": 0.50317 - }, - { - "cv_recall_at_1": 0.5, - "cv_recall_at_10": 0.785, - "cv_recall_at_100": 0.98, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.844, - "cv_recall_at_3": 0.653, - "cv_recall_at_5": 0.717, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.63886, - "map_at_1": 0.5, - "map_at_10": 0.59253, - "map_at_100": 0.60054, - "map_at_1000": 0.60066, - "map_at_20": 0.59673, - "map_at_3": 0.569, - "map_at_5": 0.58335, - "mrr_at_1": 0.5, - "mrr_at_10": 0.5925329365079366, - "mrr_at_100": 0.6005437190144755, - "mrr_at_1000": 0.600664144421558, - "mrr_at_20": 0.5967313145746579, - "mrr_at_3": 0.5690000000000002, - "mrr_at_5": 0.5833500000000003, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5858309990662961, - "nauc_cv_recall_at_100_max": 0.7176237161531301, - "nauc_cv_recall_at_100_std": 0.8628618113912271, - "nauc_cv_recall_at_10_diff1": 0.7120100093247249, - "nauc_cv_recall_at_10_max": 0.2462671592639371, - "nauc_cv_recall_at_10_std": -0.23614452142916173, - "nauc_cv_recall_at_1_diff1": 0.7841492338441036, - "nauc_cv_recall_at_1_max": 0.3875429713524314, - "nauc_cv_recall_at_1_std": -0.007577614923384203, - "nauc_cv_recall_at_20_diff1": 0.7082412268582482, - "nauc_cv_recall_at_20_max": 0.32142857142857206, - "nauc_cv_recall_at_20_std": -0.2007146625135008, - "nauc_cv_recall_at_3_diff1": 0.7107919769385271, - "nauc_cv_recall_at_3_max": 0.3104524717256328, - "nauc_cv_recall_at_3_std": -0.13919222641101736, - "nauc_cv_recall_at_5_diff1": 0.6963980779158884, - "nauc_cv_recall_at_5_max": 0.29357546600706164, - "nauc_cv_recall_at_5_std": -0.19533624179984505, - "nauc_map_at_1000_diff1": 0.7545027808605635, - "nauc_map_at_1000_max": 0.3567529979219445, - "nauc_map_at_1000_std": -0.062112641943025536, - "nauc_map_at_100_diff1": 0.7544438124532609, - "nauc_map_at_100_max": 0.3568824116725499, - "nauc_map_at_100_std": -0.06173985435885906, - "nauc_map_at_10_diff1": 0.7539830140069392, - "nauc_map_at_10_max": 0.35104102462166586, - "nauc_map_at_10_std": -0.06966701537309626, - "nauc_map_at_1_diff1": 0.7841492338441036, - "nauc_map_at_1_max": 0.3875429713524314, - "nauc_map_at_1_std": -0.007577614923384203, - "nauc_map_at_20_diff1": 0.7543229223023675, - "nauc_map_at_20_max": 0.35552716581356714, - "nauc_map_at_20_std": -0.06605583359387289, - "nauc_map_at_3_diff1": 0.752913416580022, - "nauc_map_at_3_max": 0.3556594165384521, - "nauc_map_at_3_std": -0.060537047966662746, - "nauc_map_at_5_diff1": 0.7510932930040148, - "nauc_map_at_5_max": 0.3534526852569118, - "nauc_map_at_5_std": -0.06868553575707965, - "nauc_mrr_at_1000_diff1": 0.7545027808605635, - "nauc_mrr_at_1000_max": 0.3567529979219445, - "nauc_mrr_at_1000_std": -0.062112641943025536, - "nauc_mrr_at_100_diff1": 0.7544438124532609, - "nauc_mrr_at_100_max": 0.3568824116725499, - "nauc_mrr_at_100_std": -0.06173985435885906, - "nauc_mrr_at_10_diff1": 0.7539830140069392, - "nauc_mrr_at_10_max": 0.35104102462166586, - "nauc_mrr_at_10_std": -0.06966701537309626, - "nauc_mrr_at_1_diff1": 0.7841492338441036, - "nauc_mrr_at_1_max": 0.3875429713524314, - "nauc_mrr_at_1_std": -0.007577614923384203, - "nauc_mrr_at_20_diff1": 0.7543229223023675, - "nauc_mrr_at_20_max": 0.35552716581356714, - "nauc_mrr_at_20_std": -0.06605583359387289, - "nauc_mrr_at_3_diff1": 0.752913416580022, - "nauc_mrr_at_3_max": 0.3556594165384521, - "nauc_mrr_at_3_std": -0.060537047966662746, - "nauc_mrr_at_5_diff1": 0.7510932930040148, - "nauc_mrr_at_5_max": 0.3534526852569118, - "nauc_mrr_at_5_std": -0.06868553575707965, - "nauc_ndcg_at_1000_diff1": 0.7499698607053241, - "nauc_ndcg_at_1000_max": 0.35418485575392594, - "nauc_ndcg_at_1000_std": -0.06718538120073447, - "nauc_ndcg_at_100_diff1": 0.7482164162964144, - "nauc_ndcg_at_100_max": 0.35807404010503563, - "nauc_ndcg_at_100_std": -0.05697537291884474, - "nauc_ndcg_at_10_diff1": 0.7457296191933457, - "nauc_ndcg_at_10_max": 0.3329444136294682, - "nauc_ndcg_at_10_std": -0.09911995188680284, - "nauc_ndcg_at_1_diff1": 0.7841492338441036, - "nauc_ndcg_at_1_max": 0.3875429713524314, - "nauc_ndcg_at_1_std": -0.007577614923384203, - "nauc_ndcg_at_20_diff1": 0.7467491806865436, - "nauc_ndcg_at_20_max": 0.34852864281248525, - "nauc_ndcg_at_20_std": -0.08706237078385107, - "nauc_ndcg_at_3_diff1": 0.7429288606690733, - "nauc_ndcg_at_3_max": 0.34498708703837594, - "nauc_ndcg_at_3_std": -0.07903887707810912, - "nauc_ndcg_at_5_diff1": 0.739339573470649, - "nauc_ndcg_at_5_max": 0.34065155653743817, - "nauc_ndcg_at_5_std": -0.09527532922715241, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.5858309990662923, - "nauc_precision_at_100_max": 0.717623716153127, - "nauc_precision_at_100_std": 0.862861811391217, - "nauc_precision_at_10_diff1": 0.7120100093247237, - "nauc_precision_at_10_max": 0.24626715926393547, - "nauc_precision_at_10_std": -0.23614452142916353, - "nauc_precision_at_1_diff1": 0.7841492338441036, - "nauc_precision_at_1_max": 0.3875429713524314, - "nauc_precision_at_1_std": -0.007577614923384203, - "nauc_precision_at_20_diff1": 0.7082412268582476, - "nauc_precision_at_20_max": 0.32142857142857073, - "nauc_precision_at_20_std": -0.20071466251350326, - "nauc_precision_at_3_diff1": 0.7107919769385278, - "nauc_precision_at_3_max": 0.3104524717256329, - "nauc_precision_at_3_std": -0.13919222641101667, - "nauc_precision_at_5_diff1": 0.6963980779158885, - "nauc_precision_at_5_max": 0.29357546600705997, - "nauc_precision_at_5_std": -0.1953362417998463, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5858309990662961, - "nauc_recall_at_100_max": 0.7176237161531301, - "nauc_recall_at_100_std": 0.8628618113912271, - "nauc_recall_at_10_diff1": 0.7120100093247249, - "nauc_recall_at_10_max": 0.2462671592639371, - "nauc_recall_at_10_std": -0.23614452142916173, - "nauc_recall_at_1_diff1": 0.7841492338441036, - "nauc_recall_at_1_max": 0.3875429713524314, - "nauc_recall_at_1_std": -0.007577614923384203, - "nauc_recall_at_20_diff1": 0.7082412268582482, - "nauc_recall_at_20_max": 0.32142857142857206, - "nauc_recall_at_20_std": -0.2007146625135008, - "nauc_recall_at_3_diff1": 0.7107919769385271, - "nauc_recall_at_3_max": 0.3104524717256328, - "nauc_recall_at_3_std": -0.13919222641101736, - "nauc_recall_at_5_diff1": 0.6963980779158884, - "nauc_recall_at_5_max": 0.29357546600706164, - "nauc_recall_at_5_std": -0.19533624179984505, - "ndcg_at_1": 0.5, - "ndcg_at_10": 0.63886, - "ndcg_at_100": 0.67982, - "ndcg_at_1000": 0.6825, - "ndcg_at_20": 0.65391, - "ndcg_at_3": 0.59064, - "ndcg_at_5": 0.61676, - "precision_at_1": 0.5, - "precision_at_10": 0.0785, - "precision_at_100": 0.0098, - "precision_at_1000": 0.001, - "precision_at_20": 0.0422, - "precision_at_3": 0.21767, - "precision_at_5": 0.1434, - "recall_at_1": 0.5, - "recall_at_10": 0.785, - "recall_at_100": 0.98, - "recall_at_1000": 1.0, - "recall_at_20": 0.844, - "recall_at_3": 0.653, - "recall_at_5": 0.717 - } - ] - }, - "task_name": "WITT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/model_meta.json b/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/model_meta.json deleted file mode 100644 index 756f9c9c7a..0000000000 --- a/results-mieb/google__siglip-base-patch16-512/753a949581523b60257d93e18391e8c27f72eb22/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "google/siglip-base-patch16-512", "revision": "753a949581523b60257d93e18391e8c27f72eb22", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/WITT2IRetrieval.json b/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/WITT2IRetrieval.json deleted file mode 100644 index 5f06d4f0d2..0000000000 --- a/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/WITT2IRetrieval.json +++ /dev/null @@ -1,1936 +0,0 @@ -{ - "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", - "evaluation_time": 153.49457120895386, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.21236, - "cv_recall_at_10": 0.44831, - "cv_recall_at_100": 0.78989, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.54944, - "cv_recall_at_3": 0.30562, - "cv_recall_at_5": 0.36292, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.3176, - "map_at_1": 0.21236, - "map_at_10": 0.27738, - "map_at_100": 0.2907, - "map_at_1000": 0.29165, - "map_at_20": 0.28443, - "map_at_3": 0.253, - "map_at_5": 0.26603, - "mrr_at_1": 0.21235955056179776, - "mrr_at_10": 0.2773800606384876, - "mrr_at_100": 0.29070094283629366, - "mrr_at_1000": 0.2916475905772476, - "mrr_at_20": 0.2844331444819757, - "mrr_at_3": 0.25299625468164805, - "mrr_at_5": 0.2660299625468165, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.3003626994779616, - "nauc_cv_recall_at_100_max": 0.5548275759629901, - "nauc_cv_recall_at_100_std": 0.6386697598297497, - "nauc_cv_recall_at_10_diff1": 0.4149575835645685, - "nauc_cv_recall_at_10_max": 0.3425415745769017, - "nauc_cv_recall_at_10_std": 0.33509351614530253, - "nauc_cv_recall_at_1_diff1": 0.6050929690032468, - "nauc_cv_recall_at_1_max": 0.41251419289584673, - "nauc_cv_recall_at_1_std": 0.25816633985365167, - "nauc_cv_recall_at_20_diff1": 0.40426712835286205, - "nauc_cv_recall_at_20_max": 0.4027644407021793, - "nauc_cv_recall_at_20_std": 0.4096739656605129, - "nauc_cv_recall_at_3_diff1": 0.49874039399022607, - "nauc_cv_recall_at_3_max": 0.3525352598845765, - "nauc_cv_recall_at_3_std": 0.2826150147458686, - "nauc_cv_recall_at_5_diff1": 0.4653915178854044, - "nauc_cv_recall_at_5_max": 0.3569189727946339, - "nauc_cv_recall_at_5_std": 0.3028674653031839, - "nauc_map_at_1000_diff1": 0.5409310976173939, - "nauc_map_at_1000_max": 0.3885752414310074, - "nauc_map_at_1000_std": 0.2865922308596221, - "nauc_map_at_100_diff1": 0.5403803030050892, - "nauc_map_at_100_max": 0.38901485287942417, - "nauc_map_at_100_std": 0.28745240579931414, - "nauc_map_at_10_diff1": 0.5415319832681147, - "nauc_map_at_10_max": 0.38328218679842607, - "nauc_map_at_10_std": 0.28077001695899756, - "nauc_map_at_1_diff1": 0.6050929690032468, - "nauc_map_at_1_max": 0.41251419289584673, - "nauc_map_at_1_std": 0.25816633985365167, - "nauc_map_at_20_diff1": 0.5411089156357143, - "nauc_map_at_20_max": 0.38701139904307347, - "nauc_map_at_20_std": 0.2856492071304046, - "nauc_map_at_3_diff1": 0.5576262979028926, - "nauc_map_at_3_max": 0.38367029894992183, - "nauc_map_at_3_std": 0.2697064390376009, - "nauc_map_at_5_diff1": 0.5489353603103189, - "nauc_map_at_5_max": 0.3852603217798598, - "nauc_map_at_5_std": 0.2755182157848625, - "nauc_mrr_at_1000_diff1": 0.540931085899091, - "nauc_mrr_at_1000_max": 0.3885752266951037, - "nauc_mrr_at_1000_std": 0.2865922209021115, - "nauc_mrr_at_100_diff1": 0.5403803030050892, - "nauc_mrr_at_100_max": 0.38901485287942417, - "nauc_mrr_at_100_std": 0.28745240579931414, - "nauc_mrr_at_10_diff1": 0.5415319832681147, - "nauc_mrr_at_10_max": 0.38328218679842607, - "nauc_mrr_at_10_std": 0.28077001695899756, - "nauc_mrr_at_1_diff1": 0.6050929690032468, - "nauc_mrr_at_1_max": 0.41251419289584673, - "nauc_mrr_at_1_std": 0.25816633985365167, - "nauc_mrr_at_20_diff1": 0.5411089156357143, - "nauc_mrr_at_20_max": 0.38701139904307347, - "nauc_mrr_at_20_std": 0.2856492071304046, - "nauc_mrr_at_3_diff1": 0.5576262979028926, - "nauc_mrr_at_3_max": 0.38367029894992183, - "nauc_mrr_at_3_std": 0.2697064390376009, - "nauc_mrr_at_5_diff1": 0.5489353603103189, - "nauc_mrr_at_5_max": 0.3852603217798598, - "nauc_mrr_at_5_std": 0.2755182157848625, - "nauc_ndcg_at_1000_diff1": 0.5201783586237868, - "nauc_ndcg_at_1000_max": 0.39109832441616876, - "nauc_ndcg_at_1000_std": 0.30461652448903986, - "nauc_ndcg_at_100_diff1": 0.5027918968844239, - "nauc_ndcg_at_100_max": 0.40436472190745265, - "nauc_ndcg_at_100_std": 0.331281945296951, - "nauc_ndcg_at_10_diff1": 0.5089323671477836, - "nauc_ndcg_at_10_max": 0.37255982146503086, - "nauc_ndcg_at_10_std": 0.2945653330281739, - "nauc_ndcg_at_1_diff1": 0.6050929690032468, - "nauc_ndcg_at_1_max": 0.41251419289584673, - "nauc_ndcg_at_1_std": 0.25816633985365167, - "nauc_ndcg_at_20_diff1": 0.5075522728112374, - "nauc_ndcg_at_20_max": 0.38771287487979855, - "nauc_ndcg_at_20_std": 0.3128682588994658, - "nauc_ndcg_at_3_diff1": 0.5416767114387823, - "nauc_ndcg_at_3_max": 0.37515635453117174, - "nauc_ndcg_at_3_std": 0.27322728588827544, - "nauc_ndcg_at_5_diff1": 0.5264649237697293, - "nauc_ndcg_at_5_max": 0.3774223708861285, - "nauc_ndcg_at_5_std": 0.28285178016394, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3003626994779613, - "nauc_precision_at_100_max": 0.5548275759629899, - "nauc_precision_at_100_std": 0.63866975982975, - "nauc_precision_at_10_diff1": 0.4149575835645688, - "nauc_precision_at_10_max": 0.3425415745769019, - "nauc_precision_at_10_std": 0.3350935161453028, - "nauc_precision_at_1_diff1": 0.6050929690032468, - "nauc_precision_at_1_max": 0.41251419289584673, - "nauc_precision_at_1_std": 0.25816633985365167, - "nauc_precision_at_20_diff1": 0.4042671283528619, - "nauc_precision_at_20_max": 0.40276444070217876, - "nauc_precision_at_20_std": 0.40967396566051206, - "nauc_precision_at_3_diff1": 0.49874039399022574, - "nauc_precision_at_3_max": 0.3525352598845762, - "nauc_precision_at_3_std": 0.28261501474586836, - "nauc_precision_at_5_diff1": 0.46539151788540417, - "nauc_precision_at_5_max": 0.356918972794634, - "nauc_precision_at_5_std": 0.302867465303184, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.3003626994779616, - "nauc_recall_at_100_max": 0.5548275759629901, - "nauc_recall_at_100_std": 0.6386697598297497, - "nauc_recall_at_10_diff1": 0.4149575835645685, - "nauc_recall_at_10_max": 0.3425415745769017, - "nauc_recall_at_10_std": 0.33509351614530253, - "nauc_recall_at_1_diff1": 0.6050929690032468, - "nauc_recall_at_1_max": 0.41251419289584673, - "nauc_recall_at_1_std": 0.25816633985365167, - "nauc_recall_at_20_diff1": 0.40426712835286205, - "nauc_recall_at_20_max": 0.4027644407021793, - "nauc_recall_at_20_std": 0.4096739656605129, - "nauc_recall_at_3_diff1": 0.49874039399022607, - "nauc_recall_at_3_max": 0.3525352598845765, - "nauc_recall_at_3_std": 0.2826150147458686, - "nauc_recall_at_5_diff1": 0.4653915178854044, - "nauc_recall_at_5_max": 0.3569189727946339, - "nauc_recall_at_5_std": 0.3028674653031839, - "ndcg_at_1": 0.21236, - "ndcg_at_10": 0.3176, - "ndcg_at_100": 0.38789, - "ndcg_at_1000": 0.41438, - "ndcg_at_20": 0.34321, - "ndcg_at_3": 0.26649, - "ndcg_at_5": 0.29004, - "precision_at_1": 0.21236, - "precision_at_10": 0.04483, - "precision_at_100": 0.0079, - "precision_at_1000": 0.001, - "precision_at_20": 0.02747, - "precision_at_3": 0.10187, - "precision_at_5": 0.07258, - "recall_at_1": 0.21236, - "recall_at_10": 0.44831, - "recall_at_100": 0.78989, - "recall_at_1000": 1.0, - "recall_at_20": 0.54944, - "recall_at_3": 0.30562, - "recall_at_5": 0.36292 - }, - { - "cv_recall_at_1": 0.20698, - "cv_recall_at_10": 0.44419, - "cv_recall_at_100": 0.76977, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.54186, - "cv_recall_at_3": 0.30233, - "cv_recall_at_5": 0.36047, - "hf_subset": "bg", - "languages": [ - "bul-Cyrl" - ], - "main_score": 0.31507, - "map_at_1": 0.20698, - "map_at_10": 0.27516, - "map_at_100": 0.28753, - "map_at_1000": 0.28861, - "map_at_20": 0.28201, - "map_at_3": 0.25058, - "map_at_5": 0.26395, - "mrr_at_1": 0.2069767441860465, - "mrr_at_10": 0.2751591915836101, - "mrr_at_100": 0.2875307284962649, - "mrr_at_1000": 0.2886101346549318, - "mrr_at_20": 0.2820103602457893, - "mrr_at_3": 0.2505813953488373, - "mrr_at_5": 0.26395348837209304, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4093627621786638, - "nauc_cv_recall_at_100_max": 0.47392125148869085, - "nauc_cv_recall_at_100_std": 0.32742884186329557, - "nauc_cv_recall_at_10_diff1": 0.4164293507827622, - "nauc_cv_recall_at_10_max": 0.373676020508955, - "nauc_cv_recall_at_10_std": 0.07981411859816971, - "nauc_cv_recall_at_1_diff1": 0.5769623985509127, - "nauc_cv_recall_at_1_max": 0.3842526568455678, - "nauc_cv_recall_at_1_std": 0.06934283587010087, - "nauc_cv_recall_at_20_diff1": 0.37682780365738766, - "nauc_cv_recall_at_20_max": 0.41811018778993125, - "nauc_cv_recall_at_20_std": 0.129614222958865, - "nauc_cv_recall_at_3_diff1": 0.47505414142682256, - "nauc_cv_recall_at_3_max": 0.3386255924170616, - "nauc_cv_recall_at_3_std": 0.06946109663852355, - "nauc_cv_recall_at_5_diff1": 0.4463062678279969, - "nauc_cv_recall_at_5_max": 0.3507802370456244, - "nauc_cv_recall_at_5_std": 0.07886114373751092, - "nauc_map_at_1000_diff1": 0.5146769065802478, - "nauc_map_at_1000_max": 0.37646808082113914, - "nauc_map_at_1000_std": 0.08646507199309812, - "nauc_map_at_100_diff1": 0.5143120882455383, - "nauc_map_at_100_max": 0.3766277483299629, - "nauc_map_at_100_std": 0.08675906951811373, - "nauc_map_at_10_diff1": 0.5152070626723227, - "nauc_map_at_10_max": 0.3727333152390514, - "nauc_map_at_10_std": 0.08099420288084715, - "nauc_map_at_1_diff1": 0.5769623985509127, - "nauc_map_at_1_max": 0.3842526568455678, - "nauc_map_at_1_std": 0.06934283587010087, - "nauc_map_at_20_diff1": 0.5125542972976519, - "nauc_map_at_20_max": 0.37611677581301717, - "nauc_map_at_20_std": 0.0846629410716143, - "nauc_map_at_3_diff1": 0.5270295580868876, - "nauc_map_at_3_max": 0.3663095452924379, - "nauc_map_at_3_std": 0.07738161926762027, - "nauc_map_at_5_diff1": 0.519948492737527, - "nauc_map_at_5_max": 0.3689441605572904, - "nauc_map_at_5_std": 0.08060487493398132, - "nauc_mrr_at_1000_diff1": 0.514676902292366, - "nauc_mrr_at_1000_max": 0.37646806310992137, - "nauc_mrr_at_1000_std": 0.08646505360870692, - "nauc_mrr_at_100_diff1": 0.5143120882455383, - "nauc_mrr_at_100_max": 0.3766277483299629, - "nauc_mrr_at_100_std": 0.08675906951811373, - "nauc_mrr_at_10_diff1": 0.5152070626723227, - "nauc_mrr_at_10_max": 0.3727333152390514, - "nauc_mrr_at_10_std": 0.08099420288084715, - "nauc_mrr_at_1_diff1": 0.5769623985509127, - "nauc_mrr_at_1_max": 0.3842526568455678, - "nauc_mrr_at_1_std": 0.06934283587010087, - "nauc_mrr_at_20_diff1": 0.5125542972976519, - "nauc_mrr_at_20_max": 0.37611677581301717, - "nauc_mrr_at_20_std": 0.0846629410716143, - "nauc_mrr_at_3_diff1": 0.5270295580868876, - "nauc_mrr_at_3_max": 0.3663095452924379, - "nauc_mrr_at_3_std": 0.07738161926762027, - "nauc_mrr_at_5_diff1": 0.519948492737527, - "nauc_mrr_at_5_max": 0.3689441605572904, - "nauc_mrr_at_5_std": 0.08060487493398132, - "nauc_ndcg_at_1000_diff1": 0.49995543334551823, - "nauc_ndcg_at_1000_max": 0.3811523747562341, - "nauc_ndcg_at_1000_std": 0.09738443817453372, - "nauc_ndcg_at_100_diff1": 0.49170577344667954, - "nauc_ndcg_at_100_max": 0.3884156484654683, - "nauc_ndcg_at_100_std": 0.11507089046060567, - "nauc_ndcg_at_10_diff1": 0.4892258089021453, - "nauc_ndcg_at_10_max": 0.3722492080992435, - "nauc_ndcg_at_10_std": 0.081001233642932, - "nauc_ndcg_at_1_diff1": 0.5769623985509127, - "nauc_ndcg_at_1_max": 0.3842526568455678, - "nauc_ndcg_at_1_std": 0.06934283587010087, - "nauc_ndcg_at_20_diff1": 0.47983152317657074, - "nauc_ndcg_at_20_max": 0.3840360365384102, - "nauc_ndcg_at_20_std": 0.0940749988684443, - "nauc_ndcg_at_3_diff1": 0.5127760637828862, - "nauc_ndcg_at_3_max": 0.3589028686816543, - "nauc_ndcg_at_3_std": 0.07554452472073625, - "nauc_ndcg_at_5_diff1": 0.5000070371620252, - "nauc_ndcg_at_5_max": 0.3638454293511532, - "nauc_ndcg_at_5_std": 0.08041489130513192, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.40936276217866474, - "nauc_precision_at_100_max": 0.4739212514886919, - "nauc_precision_at_100_std": 0.3274288418632965, - "nauc_precision_at_10_diff1": 0.4164293507827624, - "nauc_precision_at_10_max": 0.3736760205089551, - "nauc_precision_at_10_std": 0.07981411859816984, - "nauc_precision_at_1_diff1": 0.5769623985509127, - "nauc_precision_at_1_max": 0.3842526568455678, - "nauc_precision_at_1_std": 0.06934283587010087, - "nauc_precision_at_20_diff1": 0.3768278036573875, - "nauc_precision_at_20_max": 0.4181101877899317, - "nauc_precision_at_20_std": 0.12961422295886474, - "nauc_precision_at_3_diff1": 0.47505414142682284, - "nauc_precision_at_3_max": 0.33862559241706147, - "nauc_precision_at_3_std": 0.06946109663852358, - "nauc_precision_at_5_diff1": 0.4463062678279969, - "nauc_precision_at_5_max": 0.35078023704562433, - "nauc_precision_at_5_std": 0.07886114373751069, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4093627621786638, - "nauc_recall_at_100_max": 0.47392125148869085, - "nauc_recall_at_100_std": 0.32742884186329557, - "nauc_recall_at_10_diff1": 0.4164293507827622, - "nauc_recall_at_10_max": 0.373676020508955, - "nauc_recall_at_10_std": 0.07981411859816971, - "nauc_recall_at_1_diff1": 0.5769623985509127, - "nauc_recall_at_1_max": 0.3842526568455678, - "nauc_recall_at_1_std": 0.06934283587010087, - "nauc_recall_at_20_diff1": 0.37682780365738766, - "nauc_recall_at_20_max": 0.41811018778993125, - "nauc_recall_at_20_std": 0.129614222958865, - "nauc_recall_at_3_diff1": 0.47505414142682256, - "nauc_recall_at_3_max": 0.3386255924170616, - "nauc_recall_at_3_std": 0.06946109663852355, - "nauc_recall_at_5_diff1": 0.4463062678279969, - "nauc_recall_at_5_max": 0.3507802370456244, - "nauc_recall_at_5_std": 0.07886114373751092, - "ndcg_at_1": 0.20698, - "ndcg_at_10": 0.31507, - "ndcg_at_100": 0.38147, - "ndcg_at_1000": 0.4107, - "ndcg_at_20": 0.33986, - "ndcg_at_3": 0.26394, - "ndcg_at_5": 0.28796, - "precision_at_1": 0.20698, - "precision_at_10": 0.04442, - "precision_at_100": 0.0077, - "precision_at_1000": 0.001, - "precision_at_20": 0.02709, - "precision_at_3": 0.10078, - "precision_at_5": 0.07209, - "recall_at_1": 0.20698, - "recall_at_10": 0.44419, - "recall_at_100": 0.76977, - "recall_at_1000": 1.0, - "recall_at_20": 0.54186, - "recall_at_3": 0.30233, - "recall_at_5": 0.36047 - }, - { - "cv_recall_at_1": 0.39731, - "cv_recall_at_10": 0.66442, - "cv_recall_at_100": 0.86083, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.72503, - "cv_recall_at_3": 0.53423, - "cv_recall_at_5": 0.59596, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.52466, - "map_at_1": 0.39731, - "map_at_10": 0.48064, - "map_at_100": 0.48803, - "map_at_1000": 0.48868, - "map_at_20": 0.48489, - "map_at_3": 0.45754, - "map_at_5": 0.47162, - "mrr_at_1": 0.39730639730639733, - "mrr_at_10": 0.48064240286462473, - "mrr_at_100": 0.4880324754856483, - "mrr_at_1000": 0.48868275160675934, - "mrr_at_20": 0.48488937567285484, - "mrr_at_3": 0.457538346427235, - "mrr_at_5": 0.4716236438458655, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5578548171577201, - "nauc_cv_recall_at_100_max": 0.6646804698095687, - "nauc_cv_recall_at_100_std": 0.6195215557966229, - "nauc_cv_recall_at_10_diff1": 0.5838135625092163, - "nauc_cv_recall_at_10_max": 0.5383944730775565, - "nauc_cv_recall_at_10_std": 0.29288807484330076, - "nauc_cv_recall_at_1_diff1": 0.7542846884818821, - "nauc_cv_recall_at_1_max": 0.4420202485145873, - "nauc_cv_recall_at_1_std": 0.0004563032507581658, - "nauc_cv_recall_at_20_diff1": 0.5442476748139404, - "nauc_cv_recall_at_20_max": 0.5818878214385517, - "nauc_cv_recall_at_20_std": 0.367573720549519, - "nauc_cv_recall_at_3_diff1": 0.6588395819935658, - "nauc_cv_recall_at_3_max": 0.5000519416651473, - "nauc_cv_recall_at_3_std": 0.150187884602865, - "nauc_cv_recall_at_5_diff1": 0.63254204802205, - "nauc_cv_recall_at_5_max": 0.48845255697186757, - "nauc_cv_recall_at_5_std": 0.18065335713285896, - "nauc_map_at_1000_diff1": 0.7059026049582101, - "nauc_map_at_1000_max": 0.4738480119813671, - "nauc_map_at_1000_std": 0.0822114309839713, - "nauc_map_at_100_diff1": 0.7056131432990436, - "nauc_map_at_100_max": 0.4740255850949717, - "nauc_map_at_100_std": 0.08300869243502265, - "nauc_map_at_10_diff1": 0.7055998064181506, - "nauc_map_at_10_max": 0.4727748241610827, - "nauc_map_at_10_std": 0.08008450961045248, - "nauc_map_at_1_diff1": 0.7542846884818821, - "nauc_map_at_1_max": 0.4420202485145873, - "nauc_map_at_1_std": 0.0004563032507581658, - "nauc_map_at_20_diff1": 0.7045296733768378, - "nauc_map_at_20_max": 0.47417479130977996, - "nauc_map_at_20_std": 0.08193767726788206, - "nauc_map_at_3_diff1": 0.7140923458572953, - "nauc_map_at_3_max": 0.4703630865323559, - "nauc_map_at_3_std": 0.06460714619505123, - "nauc_map_at_5_diff1": 0.7095881165016424, - "nauc_map_at_5_max": 0.4677367980674292, - "nauc_map_at_5_std": 0.0700858298315289, - "nauc_mrr_at_1000_diff1": 0.7059026049582101, - "nauc_mrr_at_1000_max": 0.4738480119813671, - "nauc_mrr_at_1000_std": 0.0822114309839713, - "nauc_mrr_at_100_diff1": 0.7056131432990436, - "nauc_mrr_at_100_max": 0.4740255850949717, - "nauc_mrr_at_100_std": 0.08300869243502265, - "nauc_mrr_at_10_diff1": 0.7055998064181506, - "nauc_mrr_at_10_max": 0.4727748241610827, - "nauc_mrr_at_10_std": 0.08008450961045248, - "nauc_mrr_at_1_diff1": 0.7542846884818821, - "nauc_mrr_at_1_max": 0.4420202485145873, - "nauc_mrr_at_1_std": 0.0004563032507581658, - "nauc_mrr_at_20_diff1": 0.7045296733768378, - "nauc_mrr_at_20_max": 0.47417479130977996, - "nauc_mrr_at_20_std": 0.08193767726788206, - "nauc_mrr_at_3_diff1": 0.7140923458572953, - "nauc_mrr_at_3_max": 0.4703630865323559, - "nauc_mrr_at_3_std": 0.06460714619505123, - "nauc_mrr_at_5_diff1": 0.7095881165016424, - "nauc_mrr_at_5_max": 0.4677367980674292, - "nauc_mrr_at_5_std": 0.0700858298315289, - "nauc_ndcg_at_1000_diff1": 0.6926431148768741, - "nauc_ndcg_at_1000_max": 0.48413074049723803, - "nauc_ndcg_at_1000_std": 0.1119721669195201, - "nauc_ndcg_at_100_diff1": 0.6848643290944362, - "nauc_ndcg_at_100_max": 0.4932360054078029, - "nauc_ndcg_at_100_std": 0.1394665807430548, - "nauc_ndcg_at_10_diff1": 0.6805052582761167, - "nauc_ndcg_at_10_max": 0.4861708577750309, - "nauc_ndcg_at_10_std": 0.12339785683878872, - "nauc_ndcg_at_1_diff1": 0.7542846884818821, - "nauc_ndcg_at_1_max": 0.4420202485145873, - "nauc_ndcg_at_1_std": 0.0004563032507581658, - "nauc_ndcg_at_20_diff1": 0.6757353483087946, - "nauc_ndcg_at_20_max": 0.4926080218871637, - "nauc_ndcg_at_20_std": 0.13236056140731542, - "nauc_ndcg_at_3_diff1": 0.7003536373293086, - "nauc_ndcg_at_3_max": 0.477911487093341, - "nauc_ndcg_at_3_std": 0.0859337040441357, - "nauc_ndcg_at_5_diff1": 0.6915361391824972, - "nauc_ndcg_at_5_max": 0.47306885865040066, - "nauc_ndcg_at_5_std": 0.09623013241600045, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.5578548171577202, - "nauc_precision_at_100_max": 0.664680469809571, - "nauc_precision_at_100_std": 0.6195215557966243, - "nauc_precision_at_10_diff1": 0.5838135625092153, - "nauc_precision_at_10_max": 0.5383944730775557, - "nauc_precision_at_10_std": 0.2928880748432999, - "nauc_precision_at_1_diff1": 0.7542846884818821, - "nauc_precision_at_1_max": 0.4420202485145873, - "nauc_precision_at_1_std": 0.0004563032507581658, - "nauc_precision_at_20_diff1": 0.54424767481394, - "nauc_precision_at_20_max": 0.5818878214385511, - "nauc_precision_at_20_std": 0.367573720549519, - "nauc_precision_at_3_diff1": 0.658839581993566, - "nauc_precision_at_3_max": 0.500051941665147, - "nauc_precision_at_3_std": 0.1501878846028649, - "nauc_precision_at_5_diff1": 0.6325420480220502, - "nauc_precision_at_5_max": 0.488452556971868, - "nauc_precision_at_5_std": 0.18065335713285838, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5578548171577201, - "nauc_recall_at_100_max": 0.6646804698095687, - "nauc_recall_at_100_std": 0.6195215557966229, - "nauc_recall_at_10_diff1": 0.5838135625092163, - "nauc_recall_at_10_max": 0.5383944730775565, - "nauc_recall_at_10_std": 0.29288807484330076, - "nauc_recall_at_1_diff1": 0.7542846884818821, - "nauc_recall_at_1_max": 0.4420202485145873, - "nauc_recall_at_1_std": 0.0004563032507581658, - "nauc_recall_at_20_diff1": 0.5442476748139404, - "nauc_recall_at_20_max": 0.5818878214385517, - "nauc_recall_at_20_std": 0.367573720549519, - "nauc_recall_at_3_diff1": 0.6588395819935658, - "nauc_recall_at_3_max": 0.5000519416651473, - "nauc_recall_at_3_std": 0.150187884602865, - "nauc_recall_at_5_diff1": 0.63254204802205, - "nauc_recall_at_5_max": 0.48845255697186757, - "nauc_recall_at_5_std": 0.18065335713285896, - "ndcg_at_1": 0.39731, - "ndcg_at_10": 0.52466, - "ndcg_at_100": 0.56456, - "ndcg_at_1000": 0.58213, - "ndcg_at_20": 0.54003, - "ndcg_at_3": 0.47723, - "ndcg_at_5": 0.50264, - "precision_at_1": 0.39731, - "precision_at_10": 0.06644, - "precision_at_100": 0.00861, - "precision_at_1000": 0.001, - "precision_at_20": 0.03625, - "precision_at_3": 0.17808, - "precision_at_5": 0.11919, - "recall_at_1": 0.39731, - "recall_at_10": 0.66442, - "recall_at_100": 0.86083, - "recall_at_1000": 1.0, - "recall_at_20": 0.72503, - "recall_at_3": 0.53423, - "recall_at_5": 0.59596 - }, - { - "cv_recall_at_1": 0.15614, - "cv_recall_at_10": 0.33509, - "cv_recall_at_100": 0.68772, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.42807, - "cv_recall_at_3": 0.22456, - "cv_recall_at_5": 0.26491, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.23459, - "map_at_1": 0.15614, - "map_at_10": 0.20374, - "map_at_100": 0.2165, - "map_at_1000": 0.21805, - "map_at_20": 0.2104, - "map_at_3": 0.18509, - "map_at_5": 0.19404, - "mrr_at_1": 0.156140350877193, - "mrr_at_10": 0.20374199387357286, - "mrr_at_100": 0.21650435818001867, - "mrr_at_1000": 0.21805273596584335, - "mrr_at_20": 0.21039989765242342, - "mrr_at_3": 0.18508771929824558, - "mrr_at_5": 0.19403508771929825, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.18910062947313946, - "nauc_cv_recall_at_100_max": 0.19924487860160875, - "nauc_cv_recall_at_100_std": 0.3410333953362322, - "nauc_cv_recall_at_10_diff1": 0.39068306628424715, - "nauc_cv_recall_at_10_max": 0.22095681004294052, - "nauc_cv_recall_at_10_std": 0.23713586068868153, - "nauc_cv_recall_at_1_diff1": 0.628447781785485, - "nauc_cv_recall_at_1_max": 0.33173520761334724, - "nauc_cv_recall_at_1_std": 0.20001166849129834, - "nauc_cv_recall_at_20_diff1": 0.32431002717936575, - "nauc_cv_recall_at_20_max": 0.21413783039346246, - "nauc_cv_recall_at_20_std": 0.29045666806515763, - "nauc_cv_recall_at_3_diff1": 0.49887013590805396, - "nauc_cv_recall_at_3_max": 0.3020034814950967, - "nauc_cv_recall_at_3_std": 0.24365329931126942, - "nauc_cv_recall_at_5_diff1": 0.443961409746126, - "nauc_cv_recall_at_5_max": 0.25070338959611993, - "nauc_cv_recall_at_5_std": 0.21037823376923773, - "nauc_map_at_1000_diff1": 0.5312774816883872, - "nauc_map_at_1000_max": 0.2889799028008814, - "nauc_map_at_1000_std": 0.2172852085582543, - "nauc_map_at_100_diff1": 0.5296079561010105, - "nauc_map_at_100_max": 0.2884402055859716, - "nauc_map_at_100_std": 0.21772815373049878, - "nauc_map_at_10_diff1": 0.5390101378342672, - "nauc_map_at_10_max": 0.28914187258841156, - "nauc_map_at_10_std": 0.21070249040924247, - "nauc_map_at_1_diff1": 0.628447781785485, - "nauc_map_at_1_max": 0.33173520761334724, - "nauc_map_at_1_std": 0.20001166849129834, - "nauc_map_at_20_diff1": 0.5322467062292103, - "nauc_map_at_20_max": 0.28831451293746013, - "nauc_map_at_20_std": 0.21641674001551306, - "nauc_map_at_3_diff1": 0.5662610467436214, - "nauc_map_at_3_max": 0.30828646913451574, - "nauc_map_at_3_std": 0.21400270072668254, - "nauc_map_at_5_diff1": 0.5508293813433176, - "nauc_map_at_5_max": 0.2952789941475848, - "nauc_map_at_5_std": 0.2056254855366493, - "nauc_mrr_at_1000_diff1": 0.5312774816883872, - "nauc_mrr_at_1000_max": 0.2889799028008814, - "nauc_mrr_at_1000_std": 0.2172852085582543, - "nauc_mrr_at_100_diff1": 0.5296079561010105, - "nauc_mrr_at_100_max": 0.2884402055859716, - "nauc_mrr_at_100_std": 0.21772815373049878, - "nauc_mrr_at_10_diff1": 0.5390101378342672, - "nauc_mrr_at_10_max": 0.28914187258841156, - "nauc_mrr_at_10_std": 0.21070249040924247, - "nauc_mrr_at_1_diff1": 0.628447781785485, - "nauc_mrr_at_1_max": 0.33173520761334724, - "nauc_mrr_at_1_std": 0.20001166849129834, - "nauc_mrr_at_20_diff1": 0.5322467062292103, - "nauc_mrr_at_20_max": 0.28831451293746013, - "nauc_mrr_at_20_std": 0.21641674001551306, - "nauc_mrr_at_3_diff1": 0.5662610467436214, - "nauc_mrr_at_3_max": 0.30828646913451574, - "nauc_mrr_at_3_std": 0.21400270072668254, - "nauc_mrr_at_5_diff1": 0.5508293813433176, - "nauc_mrr_at_5_max": 0.2952789941475848, - "nauc_mrr_at_5_std": 0.2056254855366493, - "nauc_ndcg_at_1000_diff1": 0.49664151547278534, - "nauc_ndcg_at_1000_max": 0.27832871617128796, - "nauc_ndcg_at_1000_std": 0.22965922218655035, - "nauc_ndcg_at_100_diff1": 0.4588080600618917, - "nauc_ndcg_at_100_max": 0.2680381852496268, - "nauc_ndcg_at_100_std": 0.2424971601594663, - "nauc_ndcg_at_10_diff1": 0.4966615039112077, - "nauc_ndcg_at_10_max": 0.270000010997535, - "nauc_ndcg_at_10_std": 0.21810824175170437, - "nauc_ndcg_at_1_diff1": 0.628447781785485, - "nauc_ndcg_at_1_max": 0.33173520761334724, - "nauc_ndcg_at_1_std": 0.20001166849129834, - "nauc_ndcg_at_20_diff1": 0.47507837516767815, - "nauc_ndcg_at_20_max": 0.2675238423292141, - "nauc_ndcg_at_20_std": 0.23558037224046202, - "nauc_ndcg_at_3_diff1": 0.5473931129806788, - "nauc_ndcg_at_3_max": 0.30616586794051853, - "nauc_ndcg_at_3_std": 0.22203037329509664, - "nauc_ndcg_at_5_diff1": 0.5207282709796918, - "nauc_ndcg_at_5_max": 0.2828848040725794, - "nauc_ndcg_at_5_std": 0.20724363575113774, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.1891006294731396, - "nauc_precision_at_100_max": 0.19924487860160886, - "nauc_precision_at_100_std": 0.341033395336232, - "nauc_precision_at_10_diff1": 0.39068306628424715, - "nauc_precision_at_10_max": 0.2209568100429408, - "nauc_precision_at_10_std": 0.2371358606886818, - "nauc_precision_at_1_diff1": 0.628447781785485, - "nauc_precision_at_1_max": 0.33173520761334724, - "nauc_precision_at_1_std": 0.20001166849129834, - "nauc_precision_at_20_diff1": 0.3243100271793654, - "nauc_precision_at_20_max": 0.21413783039346218, - "nauc_precision_at_20_std": 0.2904566680651574, - "nauc_precision_at_3_diff1": 0.49887013590805374, - "nauc_precision_at_3_max": 0.3020034814950966, - "nauc_precision_at_3_std": 0.24365329931126947, - "nauc_precision_at_5_diff1": 0.44396140974612597, - "nauc_precision_at_5_max": 0.2507033895961199, - "nauc_precision_at_5_std": 0.210378233769238, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.18910062947313946, - "nauc_recall_at_100_max": 0.19924487860160875, - "nauc_recall_at_100_std": 0.3410333953362322, - "nauc_recall_at_10_diff1": 0.39068306628424715, - "nauc_recall_at_10_max": 0.22095681004294052, - "nauc_recall_at_10_std": 0.23713586068868153, - "nauc_recall_at_1_diff1": 0.628447781785485, - "nauc_recall_at_1_max": 0.33173520761334724, - "nauc_recall_at_1_std": 0.20001166849129834, - "nauc_recall_at_20_diff1": 0.32431002717936575, - "nauc_recall_at_20_max": 0.21413783039346246, - "nauc_recall_at_20_std": 0.29045666806515763, - "nauc_recall_at_3_diff1": 0.49887013590805396, - "nauc_recall_at_3_max": 0.3020034814950967, - "nauc_recall_at_3_std": 0.24365329931126942, - "nauc_recall_at_5_diff1": 0.443961409746126, - "nauc_recall_at_5_max": 0.25070338959611993, - "nauc_recall_at_5_std": 0.21037823376923773, - "ndcg_at_1": 0.15614, - "ndcg_at_10": 0.23459, - "ndcg_at_100": 0.30538, - "ndcg_at_1000": 0.34567, - "ndcg_at_20": 0.25836, - "ndcg_at_3": 0.19517, - "ndcg_at_5": 0.21155, - "precision_at_1": 0.15614, - "precision_at_10": 0.03351, - "precision_at_100": 0.00688, - "precision_at_1000": 0.001, - "precision_at_20": 0.0214, - "precision_at_3": 0.07485, - "precision_at_5": 0.05298, - "recall_at_1": 0.15614, - "recall_at_10": 0.33509, - "recall_at_100": 0.68772, - "recall_at_1000": 1.0, - "recall_at_20": 0.42807, - "recall_at_3": 0.22456, - "recall_at_5": 0.26491 - }, - { - "cv_recall_at_1": 0.21053, - "cv_recall_at_10": 0.40732, - "cv_recall_at_100": 0.75629, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.50343, - "cv_recall_at_3": 0.29176, - "cv_recall_at_5": 0.33524, - "hf_subset": "et", - "languages": [ - "est-Latn" - ], - "main_score": 0.29932, - "map_at_1": 0.21053, - "map_at_10": 0.266, - "map_at_100": 0.27887, - "map_at_1000": 0.28, - "map_at_20": 0.27243, - "map_at_3": 0.24638, - "map_at_5": 0.25633, - "mrr_at_1": 0.21052631578947367, - "mrr_at_10": 0.2659987831898585, - "mrr_at_100": 0.2788741725576141, - "mrr_at_1000": 0.2799981947148587, - "mrr_at_20": 0.2724331888685576, - "mrr_at_3": 0.24637681159420294, - "mrr_at_5": 0.2563310450038138, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4040309098626024, - "nauc_cv_recall_at_100_max": 0.5072303739534308, - "nauc_cv_recall_at_100_std": 0.5322222637960802, - "nauc_cv_recall_at_10_diff1": 0.41350108355389, - "nauc_cv_recall_at_10_max": 0.29351616464676317, - "nauc_cv_recall_at_10_std": 0.19708885406673227, - "nauc_cv_recall_at_1_diff1": 0.6372005102044834, - "nauc_cv_recall_at_1_max": 0.38613114738319604, - "nauc_cv_recall_at_1_std": 0.10131574096517776, - "nauc_cv_recall_at_20_diff1": 0.37351711120057546, - "nauc_cv_recall_at_20_max": 0.3140724644248817, - "nauc_cv_recall_at_20_std": 0.26700917458594203, - "nauc_cv_recall_at_3_diff1": 0.5132176534203662, - "nauc_cv_recall_at_3_max": 0.3226228179276247, - "nauc_cv_recall_at_3_std": 0.13885815939228147, - "nauc_cv_recall_at_5_diff1": 0.4491732287253029, - "nauc_cv_recall_at_5_max": 0.3105780062973309, - "nauc_cv_recall_at_5_std": 0.1721406687192357, - "nauc_map_at_1000_diff1": 0.5558302859344558, - "nauc_map_at_1000_max": 0.3527916196058264, - "nauc_map_at_1000_std": 0.13338075718467787, - "nauc_map_at_100_diff1": 0.555359372389194, - "nauc_map_at_100_max": 0.35310936558202816, - "nauc_map_at_100_std": 0.13464064729062267, - "nauc_map_at_10_diff1": 0.5556162167007911, - "nauc_map_at_10_max": 0.34786519732513577, - "nauc_map_at_10_std": 0.12641240527243502, - "nauc_map_at_1_diff1": 0.6372005102044834, - "nauc_map_at_1_max": 0.38613114738319604, - "nauc_map_at_1_std": 0.10131574096517776, - "nauc_map_at_20_diff1": 0.5532309928003525, - "nauc_map_at_20_max": 0.34939557135589966, - "nauc_map_at_20_std": 0.1313876717359136, - "nauc_map_at_3_diff1": 0.5782131762035241, - "nauc_map_at_3_max": 0.3538089928851203, - "nauc_map_at_3_std": 0.11391828595128985, - "nauc_map_at_5_diff1": 0.5613343361923882, - "nauc_map_at_5_max": 0.3505715403437261, - "nauc_map_at_5_std": 0.12276900548783347, - "nauc_mrr_at_1000_diff1": 0.5558302859344558, - "nauc_mrr_at_1000_max": 0.3527916196058264, - "nauc_mrr_at_1000_std": 0.13338075718467787, - "nauc_mrr_at_100_diff1": 0.555359372389194, - "nauc_mrr_at_100_max": 0.35310936558202816, - "nauc_mrr_at_100_std": 0.13464064729062267, - "nauc_mrr_at_10_diff1": 0.5556162167007911, - "nauc_mrr_at_10_max": 0.34786519732513577, - "nauc_mrr_at_10_std": 0.12641240527243502, - "nauc_mrr_at_1_diff1": 0.6372005102044834, - "nauc_mrr_at_1_max": 0.38613114738319604, - "nauc_mrr_at_1_std": 0.10131574096517776, - "nauc_mrr_at_20_diff1": 0.5532309928003525, - "nauc_mrr_at_20_max": 0.34939557135589966, - "nauc_mrr_at_20_std": 0.1313876717359136, - "nauc_mrr_at_3_diff1": 0.5782131762035241, - "nauc_mrr_at_3_max": 0.3538089928851203, - "nauc_mrr_at_3_std": 0.11391828595128985, - "nauc_mrr_at_5_diff1": 0.5613343361923882, - "nauc_mrr_at_5_max": 0.3505715403437261, - "nauc_mrr_at_5_std": 0.12276900548783347, - "nauc_ndcg_at_1000_diff1": 0.5349171072515045, - "nauc_ndcg_at_1000_max": 0.35374501452199153, - "nauc_ndcg_at_1000_std": 0.1556987345481063, - "nauc_ndcg_at_100_diff1": 0.5228210882759149, - "nauc_ndcg_at_100_max": 0.3668209735804216, - "nauc_ndcg_at_100_std": 0.1901665181862775, - "nauc_ndcg_at_10_diff1": 0.5179873440321987, - "nauc_ndcg_at_10_max": 0.3333289869905906, - "nauc_ndcg_at_10_std": 0.14459459944320588, - "nauc_ndcg_at_1_diff1": 0.6372005102044834, - "nauc_ndcg_at_1_max": 0.38613114738319604, - "nauc_ndcg_at_1_std": 0.10131574096517776, - "nauc_ndcg_at_20_diff1": 0.508535366784914, - "nauc_ndcg_at_20_max": 0.33907869749189457, - "nauc_ndcg_at_20_std": 0.16281917120234862, - "nauc_ndcg_at_3_diff1": 0.5605384341570335, - "nauc_ndcg_at_3_max": 0.34524089763458526, - "nauc_ndcg_at_3_std": 0.1204822260900556, - "nauc_ndcg_at_5_diff1": 0.5313293612744379, - "nauc_ndcg_at_5_max": 0.339634674431103, - "nauc_ndcg_at_5_std": 0.13570399957682294, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.4040309098626027, - "nauc_precision_at_100_max": 0.5072303739534312, - "nauc_precision_at_100_std": 0.5322222637960811, - "nauc_precision_at_10_diff1": 0.4135010835538899, - "nauc_precision_at_10_max": 0.29351616464676256, - "nauc_precision_at_10_std": 0.19708885406673196, - "nauc_precision_at_1_diff1": 0.6372005102044834, - "nauc_precision_at_1_max": 0.38613114738319604, - "nauc_precision_at_1_std": 0.10131574096517776, - "nauc_precision_at_20_diff1": 0.37351711120057546, - "nauc_precision_at_20_max": 0.3140724644248813, - "nauc_precision_at_20_std": 0.2670091745859422, - "nauc_precision_at_3_diff1": 0.5132176534203662, - "nauc_precision_at_3_max": 0.3226228179276242, - "nauc_precision_at_3_std": 0.1388581593922816, - "nauc_precision_at_5_diff1": 0.4491732287253025, - "nauc_precision_at_5_max": 0.31057800629733096, - "nauc_precision_at_5_std": 0.17214066871923572, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4040309098626024, - "nauc_recall_at_100_max": 0.5072303739534308, - "nauc_recall_at_100_std": 0.5322222637960802, - "nauc_recall_at_10_diff1": 0.41350108355389, - "nauc_recall_at_10_max": 0.29351616464676317, - "nauc_recall_at_10_std": 0.19708885406673227, - "nauc_recall_at_1_diff1": 0.6372005102044834, - "nauc_recall_at_1_max": 0.38613114738319604, - "nauc_recall_at_1_std": 0.10131574096517776, - "nauc_recall_at_20_diff1": 0.37351711120057546, - "nauc_recall_at_20_max": 0.3140724644248817, - "nauc_recall_at_20_std": 0.26700917458594203, - "nauc_recall_at_3_diff1": 0.5132176534203662, - "nauc_recall_at_3_max": 0.3226228179276247, - "nauc_recall_at_3_std": 0.13885815939228147, - "nauc_recall_at_5_diff1": 0.4491732287253029, - "nauc_recall_at_5_max": 0.3105780062973309, - "nauc_recall_at_5_std": 0.1721406687192357, - "ndcg_at_1": 0.21053, - "ndcg_at_10": 0.29932, - "ndcg_at_100": 0.3702, - "ndcg_at_1000": 0.40113, - "ndcg_at_20": 0.32333, - "ndcg_at_3": 0.25804, - "ndcg_at_5": 0.27596, - "precision_at_1": 0.21053, - "precision_at_10": 0.04073, - "precision_at_100": 0.00756, - "precision_at_1000": 0.001, - "precision_at_20": 0.02517, - "precision_at_3": 0.09725, - "precision_at_5": 0.06705, - "recall_at_1": 0.21053, - "recall_at_10": 0.40732, - "recall_at_100": 0.75629, - "recall_at_1000": 1.0, - "recall_at_20": 0.50343, - "recall_at_3": 0.29176, - "recall_at_5": 0.33524 - }, - { - "cv_recall_at_1": 0.48835, - "cv_recall_at_10": 0.74362, - "cv_recall_at_100": 0.94007, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.80577, - "cv_recall_at_3": 0.63263, - "cv_recall_at_5": 0.6859, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.61394, - "map_at_1": 0.48835, - "map_at_10": 0.57268, - "map_at_100": 0.58029, - "map_at_1000": 0.58059, - "map_at_20": 0.57698, - "map_at_3": 0.55272, - "map_at_5": 0.56509, - "mrr_at_1": 0.48834628190899, - "mrr_at_10": 0.5726758627979491, - "mrr_at_100": 0.5802929567946048, - "mrr_at_1000": 0.580593260804546, - "mrr_at_20": 0.5769766502585726, - "mrr_at_3": 0.5527192008879022, - "mrr_at_5": 0.5650943396226412, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.6049915033432466, - "nauc_cv_recall_at_100_max": 0.708759882091787, - "nauc_cv_recall_at_100_std": 0.6297270823652008, - "nauc_cv_recall_at_10_diff1": 0.6642776906089205, - "nauc_cv_recall_at_10_max": 0.447059980454553, - "nauc_cv_recall_at_10_std": 0.4052465772306079, - "nauc_cv_recall_at_1_diff1": 0.7754459218216824, - "nauc_cv_recall_at_1_max": 0.44289750168720743, - "nauc_cv_recall_at_1_std": 0.16301588689160468, - "nauc_cv_recall_at_20_diff1": 0.620224538962043, - "nauc_cv_recall_at_20_max": 0.5276603799108296, - "nauc_cv_recall_at_20_std": 0.48463918745627865, - "nauc_cv_recall_at_3_diff1": 0.691775032709587, - "nauc_cv_recall_at_3_max": 0.43592366553042383, - "nauc_cv_recall_at_3_std": 0.26442498020166966, - "nauc_cv_recall_at_5_diff1": 0.6720042848528613, - "nauc_cv_recall_at_5_max": 0.4150629324157825, - "nauc_cv_recall_at_5_std": 0.3241888821441566, - "nauc_map_at_1000_diff1": 0.7371547683757999, - "nauc_map_at_1000_max": 0.4413120271242708, - "nauc_map_at_1000_std": 0.21533414606637508, - "nauc_map_at_100_diff1": 0.7369944021436499, - "nauc_map_at_100_max": 0.4414884543850126, - "nauc_map_at_100_std": 0.21562166277637476, - "nauc_map_at_10_diff1": 0.7368532064652646, - "nauc_map_at_10_max": 0.4381081429166812, - "nauc_map_at_10_std": 0.2155827439033546, - "nauc_map_at_1_diff1": 0.7754459218216824, - "nauc_map_at_1_max": 0.44289750168720743, - "nauc_map_at_1_std": 0.16301588689160468, - "nauc_map_at_20_diff1": 0.7358744047217303, - "nauc_map_at_20_max": 0.44124782849192085, - "nauc_map_at_20_std": 0.2168435407857295, - "nauc_map_at_3_diff1": 0.7397448692197428, - "nauc_map_at_3_max": 0.43880422998312507, - "nauc_map_at_3_std": 0.20032815912248597, - "nauc_map_at_5_diff1": 0.7365303036094254, - "nauc_map_at_5_max": 0.43466506293500423, - "nauc_map_at_5_std": 0.21054343286527433, - "nauc_mrr_at_1000_diff1": 0.7371547497308951, - "nauc_mrr_at_1000_max": 0.4413102095955693, - "nauc_mrr_at_1000_std": 0.2153316915574059, - "nauc_mrr_at_100_diff1": 0.7369943730503353, - "nauc_mrr_at_100_max": 0.44148662838789476, - "nauc_mrr_at_100_std": 0.21561921494775707, - "nauc_mrr_at_10_diff1": 0.7368532064652646, - "nauc_mrr_at_10_max": 0.4381081429166812, - "nauc_mrr_at_10_std": 0.2155827439033546, - "nauc_mrr_at_1_diff1": 0.7754459218216824, - "nauc_mrr_at_1_max": 0.44289750168720743, - "nauc_mrr_at_1_std": 0.16301588689160468, - "nauc_mrr_at_20_diff1": 0.7358744047217303, - "nauc_mrr_at_20_max": 0.44124782849192085, - "nauc_mrr_at_20_std": 0.2168435407857295, - "nauc_mrr_at_3_diff1": 0.7397448692197428, - "nauc_mrr_at_3_max": 0.43880422998312507, - "nauc_mrr_at_3_std": 0.20032815912248597, - "nauc_mrr_at_5_diff1": 0.7365303036094254, - "nauc_mrr_at_5_max": 0.43466506293500423, - "nauc_mrr_at_5_std": 0.21054343286527433, - "nauc_ndcg_at_1000_diff1": 0.7294461405142091, - "nauc_ndcg_at_1000_max": 0.44643934374513766, - "nauc_ndcg_at_1000_std": 0.23427381610688863, - "nauc_ndcg_at_100_diff1": 0.7255991832726855, - "nauc_ndcg_at_100_max": 0.4536474578380291, - "nauc_ndcg_at_100_std": 0.2452493736090266, - "nauc_ndcg_at_10_diff1": 0.7224014716492231, - "nauc_ndcg_at_10_max": 0.4391711764126902, - "nauc_ndcg_at_10_std": 0.2505369918544025, - "nauc_ndcg_at_1_diff1": 0.7754459218216824, - "nauc_ndcg_at_1_max": 0.44289750168720743, - "nauc_ndcg_at_1_std": 0.16301588689160468, - "nauc_ndcg_at_20_diff1": 0.7177104753698911, - "nauc_ndcg_at_20_max": 0.4518920628491788, - "nauc_ndcg_at_20_std": 0.25709137115309527, - "nauc_ndcg_at_3_diff1": 0.7282250550431725, - "nauc_ndcg_at_3_max": 0.43806163687845706, - "nauc_ndcg_at_3_std": 0.21544985963804733, - "nauc_ndcg_at_5_diff1": 0.722169746497287, - "nauc_ndcg_at_5_max": 0.43047402062795503, - "nauc_ndcg_at_5_std": 0.23506811710924785, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.6049915033432456, - "nauc_precision_at_100_max": 0.7087598820917866, - "nauc_precision_at_100_std": 0.6297270823652044, - "nauc_precision_at_10_diff1": 0.6642776906089205, - "nauc_precision_at_10_max": 0.4470599804545534, - "nauc_precision_at_10_std": 0.405246577230608, - "nauc_precision_at_1_diff1": 0.7754459218216824, - "nauc_precision_at_1_max": 0.44289750168720743, - "nauc_precision_at_1_std": 0.16301588689160468, - "nauc_precision_at_20_diff1": 0.6202245389620432, - "nauc_precision_at_20_max": 0.527660379910829, - "nauc_precision_at_20_std": 0.48463918745627876, - "nauc_precision_at_3_diff1": 0.6917750327095873, - "nauc_precision_at_3_max": 0.4359236655304239, - "nauc_precision_at_3_std": 0.26442498020167043, - "nauc_precision_at_5_diff1": 0.6720042848528609, - "nauc_precision_at_5_max": 0.41506293241578196, - "nauc_precision_at_5_std": 0.3241888821441566, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.6049915033432466, - "nauc_recall_at_100_max": 0.708759882091787, - "nauc_recall_at_100_std": 0.6297270823652008, - "nauc_recall_at_10_diff1": 0.6642776906089205, - "nauc_recall_at_10_max": 0.447059980454553, - "nauc_recall_at_10_std": 0.4052465772306079, - "nauc_recall_at_1_diff1": 0.7754459218216824, - "nauc_recall_at_1_max": 0.44289750168720743, - "nauc_recall_at_1_std": 0.16301588689160468, - "nauc_recall_at_20_diff1": 0.620224538962043, - "nauc_recall_at_20_max": 0.5276603799108296, - "nauc_recall_at_20_std": 0.48463918745627865, - "nauc_recall_at_3_diff1": 0.691775032709587, - "nauc_recall_at_3_max": 0.43592366553042383, - "nauc_recall_at_3_std": 0.26442498020166966, - "nauc_recall_at_5_diff1": 0.6720042848528613, - "nauc_recall_at_5_max": 0.4150629324157825, - "nauc_recall_at_5_std": 0.3241888821441566, - "ndcg_at_1": 0.48835, - "ndcg_at_10": 0.61394, - "ndcg_at_100": 0.6543, - "ndcg_at_1000": 0.66198, - "ndcg_at_20": 0.62963, - "ndcg_at_3": 0.57328, - "ndcg_at_5": 0.59539, - "precision_at_1": 0.48835, - "precision_at_10": 0.07436, - "precision_at_100": 0.0094, - "precision_at_1000": 0.001, - "precision_at_20": 0.04029, - "precision_at_3": 0.21088, - "precision_at_5": 0.13718, - "recall_at_1": 0.48835, - "recall_at_10": 0.74362, - "recall_at_100": 0.94007, - "recall_at_1000": 1.0, - "recall_at_20": 0.80577, - "recall_at_3": 0.63263, - "recall_at_5": 0.6859 - }, - { - "cv_recall_at_1": 0.13963, - "cv_recall_at_10": 0.30827, - "cv_recall_at_100": 0.63802, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.39098, - "cv_recall_at_3": 0.19334, - "cv_recall_at_5": 0.23523, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.21135, - "map_at_1": 0.13963, - "map_at_10": 0.18189, - "map_at_100": 0.19343, - "map_at_1000": 0.19487, - "map_at_20": 0.18754, - "map_at_3": 0.16255, - "map_at_5": 0.17195, - "mrr_at_1": 0.13963480128893663, - "mrr_at_10": 0.1818909518694696, - "mrr_at_100": 0.1934289349559609, - "mrr_at_1000": 0.19487283351496834, - "mrr_at_20": 0.18754334327729894, - "mrr_at_3": 0.16254923021840312, - "mrr_at_5": 0.17194772645900466, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2534089559430589, - "nauc_cv_recall_at_100_max": 0.32270816808890607, - "nauc_cv_recall_at_100_std": 0.4730302314141176, - "nauc_cv_recall_at_10_diff1": 0.40325013114727315, - "nauc_cv_recall_at_10_max": 0.2689458126198791, - "nauc_cv_recall_at_10_std": 0.3160539404459094, - "nauc_cv_recall_at_1_diff1": 0.6317084909028625, - "nauc_cv_recall_at_1_max": 0.35727465463798985, - "nauc_cv_recall_at_1_std": 0.3002148677470887, - "nauc_cv_recall_at_20_diff1": 0.36362317744747646, - "nauc_cv_recall_at_20_max": 0.26919980206684513, - "nauc_cv_recall_at_20_std": 0.3217120395483907, - "nauc_cv_recall_at_3_diff1": 0.5051748031641702, - "nauc_cv_recall_at_3_max": 0.3418598559556601, - "nauc_cv_recall_at_3_std": 0.3128990941534339, - "nauc_cv_recall_at_5_diff1": 0.45631176762085995, - "nauc_cv_recall_at_5_max": 0.28861197328428334, - "nauc_cv_recall_at_5_std": 0.29915893751084915, - "nauc_map_at_1000_diff1": 0.5391322227919676, - "nauc_map_at_1000_max": 0.3367418055203415, - "nauc_map_at_1000_std": 0.31472976459112945, - "nauc_map_at_100_diff1": 0.5383827313100662, - "nauc_map_at_100_max": 0.33638933710186986, - "nauc_map_at_100_std": 0.3150002028628421, - "nauc_map_at_10_diff1": 0.5448935246611308, - "nauc_map_at_10_max": 0.33535253238633284, - "nauc_map_at_10_std": 0.31061338437486985, - "nauc_map_at_1_diff1": 0.6317084909028625, - "nauc_map_at_1_max": 0.35727465463798985, - "nauc_map_at_1_std": 0.3002148677470887, - "nauc_map_at_20_diff1": 0.5405184997278059, - "nauc_map_at_20_max": 0.3354746238530943, - "nauc_map_at_20_std": 0.31168100786202535, - "nauc_map_at_3_diff1": 0.5723147050217703, - "nauc_map_at_3_max": 0.35630187388769957, - "nauc_map_at_3_std": 0.31013478532479555, - "nauc_map_at_5_diff1": 0.5564096737028025, - "nauc_map_at_5_max": 0.34086704005792656, - "nauc_map_at_5_std": 0.30680693510326507, - "nauc_mrr_at_1000_diff1": 0.5391323019533246, - "nauc_mrr_at_1000_max": 0.33674149853590685, - "nauc_mrr_at_1000_std": 0.3147295182343123, - "nauc_mrr_at_100_diff1": 0.5383828099646272, - "nauc_mrr_at_100_max": 0.33638903127372455, - "nauc_mrr_at_100_std": 0.3149999575713113, - "nauc_mrr_at_10_diff1": 0.5448935246611308, - "nauc_mrr_at_10_max": 0.33535253238633284, - "nauc_mrr_at_10_std": 0.31061338437486985, - "nauc_mrr_at_1_diff1": 0.6317084909028625, - "nauc_mrr_at_1_max": 0.35727465463798985, - "nauc_mrr_at_1_std": 0.3002148677470887, - "nauc_mrr_at_20_diff1": 0.5405184997278059, - "nauc_mrr_at_20_max": 0.3354746238530943, - "nauc_mrr_at_20_std": 0.31168100786202535, - "nauc_mrr_at_3_diff1": 0.5723147050217703, - "nauc_mrr_at_3_max": 0.35630187388769957, - "nauc_mrr_at_3_std": 0.31013478532479555, - "nauc_mrr_at_5_diff1": 0.5564096737028025, - "nauc_mrr_at_5_max": 0.34086704005792656, - "nauc_mrr_at_5_std": 0.30680693510326507, - "nauc_ndcg_at_1000_diff1": 0.503172222064133, - "nauc_ndcg_at_1000_max": 0.32939523795204545, - "nauc_ndcg_at_1000_std": 0.32587298565830985, - "nauc_ndcg_at_100_diff1": 0.4727440949155755, - "nauc_ndcg_at_100_max": 0.3269834083639128, - "nauc_ndcg_at_100_std": 0.343141914249046, - "nauc_ndcg_at_10_diff1": 0.5031512824454686, - "nauc_ndcg_at_10_max": 0.3160242337211601, - "nauc_ndcg_at_10_std": 0.31234805584479797, - "nauc_ndcg_at_1_diff1": 0.6317084909028625, - "nauc_ndcg_at_1_max": 0.35727465463798985, - "nauc_ndcg_at_1_std": 0.3002148677470887, - "nauc_ndcg_at_20_diff1": 0.48908232206391783, - "nauc_ndcg_at_20_max": 0.31591981917950757, - "nauc_ndcg_at_20_std": 0.31480058661935906, - "nauc_ndcg_at_3_diff1": 0.5534585464528963, - "nauc_ndcg_at_3_max": 0.3524868799024196, - "nauc_ndcg_at_3_std": 0.31106377515333117, - "nauc_ndcg_at_5_diff1": 0.5276047716802393, - "nauc_ndcg_at_5_max": 0.32645814171648835, - "nauc_ndcg_at_5_std": 0.3049512756948914, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.2534089559430588, - "nauc_precision_at_100_max": 0.32270816808890573, - "nauc_precision_at_100_std": 0.473030231414118, - "nauc_precision_at_10_diff1": 0.40325013114727315, - "nauc_precision_at_10_max": 0.2689458126198787, - "nauc_precision_at_10_std": 0.3160539404459094, - "nauc_precision_at_1_diff1": 0.6317084909028625, - "nauc_precision_at_1_max": 0.35727465463798985, - "nauc_precision_at_1_std": 0.3002148677470887, - "nauc_precision_at_20_diff1": 0.36362317744747635, - "nauc_precision_at_20_max": 0.26919980206684496, - "nauc_precision_at_20_std": 0.32171203954839084, - "nauc_precision_at_3_diff1": 0.5051748031641701, - "nauc_precision_at_3_max": 0.34185985595565993, - "nauc_precision_at_3_std": 0.31289909415343403, - "nauc_precision_at_5_diff1": 0.4563117676208598, - "nauc_precision_at_5_max": 0.28861197328428334, - "nauc_precision_at_5_std": 0.29915893751084904, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2534089559430589, - "nauc_recall_at_100_max": 0.32270816808890607, - "nauc_recall_at_100_std": 0.4730302314141176, - "nauc_recall_at_10_diff1": 0.40325013114727315, - "nauc_recall_at_10_max": 0.2689458126198791, - "nauc_recall_at_10_std": 0.3160539404459094, - "nauc_recall_at_1_diff1": 0.6317084909028625, - "nauc_recall_at_1_max": 0.35727465463798985, - "nauc_recall_at_1_std": 0.3002148677470887, - "nauc_recall_at_20_diff1": 0.36362317744747646, - "nauc_recall_at_20_max": 0.26919980206684513, - "nauc_recall_at_20_std": 0.3217120395483907, - "nauc_recall_at_3_diff1": 0.5051748031641702, - "nauc_recall_at_3_max": 0.3418598559556601, - "nauc_recall_at_3_std": 0.3128990941534339, - "nauc_recall_at_5_diff1": 0.45631176762085995, - "nauc_recall_at_5_max": 0.28861197328428334, - "nauc_recall_at_5_std": 0.29915893751084915, - "ndcg_at_1": 0.13963, - "ndcg_at_10": 0.21135, - "ndcg_at_100": 0.27706, - "ndcg_at_1000": 0.32153, - "ndcg_at_20": 0.23215, - "ndcg_at_3": 0.17043, - "ndcg_at_5": 0.18753, - "precision_at_1": 0.13963, - "precision_at_10": 0.03083, - "precision_at_100": 0.00638, - "precision_at_1000": 0.001, - "precision_at_20": 0.01955, - "precision_at_3": 0.06445, - "precision_at_5": 0.04705, - "recall_at_1": 0.13963, - "recall_at_10": 0.30827, - "recall_at_100": 0.63802, - "recall_at_1000": 1.0, - "recall_at_20": 0.39098, - "recall_at_3": 0.19334, - "recall_at_5": 0.23523 - }, - { - "cv_recall_at_1": 0.14, - "cv_recall_at_10": 0.357, - "cv_recall_at_100": 0.658, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.425, - "cv_recall_at_3": 0.219, - "cv_recall_at_5": 0.285, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.23555, - "map_at_1": 0.14, - "map_at_10": 0.19815, - "map_at_100": 0.20838, - "map_at_1000": 0.20992, - "map_at_20": 0.20287, - "map_at_3": 0.17333, - "map_at_5": 0.18873, - "mrr_at_1": 0.14, - "mrr_at_10": 0.19815277777777784, - "mrr_at_100": 0.20837673242317895, - "mrr_at_1000": 0.20991525905046582, - "mrr_at_20": 0.20286907916215202, - "mrr_at_3": 0.17333333333333342, - "mrr_at_5": 0.1887333333333334, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.31186007531151105, - "nauc_cv_recall_at_100_max": 0.4811622040684086, - "nauc_cv_recall_at_100_std": 0.42809040026183837, - "nauc_cv_recall_at_10_diff1": 0.413408863642038, - "nauc_cv_recall_at_10_max": 0.27114828933311785, - "nauc_cv_recall_at_10_std": 0.12190628617682368, - "nauc_cv_recall_at_1_diff1": 0.5133140990283847, - "nauc_cv_recall_at_1_max": 0.14333658619372894, - "nauc_cv_recall_at_1_std": -0.0564724850439138, - "nauc_cv_recall_at_20_diff1": 0.34592748872082824, - "nauc_cv_recall_at_20_max": 0.27579084033886203, - "nauc_cv_recall_at_20_std": 0.16246291668019083, - "nauc_cv_recall_at_3_diff1": 0.4245150101271459, - "nauc_cv_recall_at_3_max": 0.20982885824019645, - "nauc_cv_recall_at_3_std": 0.048116786775706716, - "nauc_cv_recall_at_5_diff1": 0.42818025055575165, - "nauc_cv_recall_at_5_max": 0.22702373285133542, - "nauc_cv_recall_at_5_std": 0.06251428333991231, - "nauc_map_at_1000_diff1": 0.45644452104799366, - "nauc_map_at_1000_max": 0.18966182165595427, - "nauc_map_at_1000_std": 0.015142223758896131, - "nauc_map_at_100_diff1": 0.45603098814474735, - "nauc_map_at_100_max": 0.19037424673457165, - "nauc_map_at_100_std": 0.01627979300863379, - "nauc_map_at_10_diff1": 0.4631346811594673, - "nauc_map_at_10_max": 0.18414084623389668, - "nauc_map_at_10_std": 0.006601657669226161, - "nauc_map_at_1_diff1": 0.5133140990283847, - "nauc_map_at_1_max": 0.14333658619372894, - "nauc_map_at_1_std": -0.0564724850439138, - "nauc_map_at_20_diff1": 0.45687943682993415, - "nauc_map_at_20_max": 0.18467676506949465, - "nauc_map_at_20_std": 0.010065050228116441, - "nauc_map_at_3_diff1": 0.46608467224631916, - "nauc_map_at_3_max": 0.16995784715010084, - "nauc_map_at_3_std": -0.008846294825584775, - "nauc_map_at_5_diff1": 0.4663183772981682, - "nauc_map_at_5_max": 0.1762843693554193, - "nauc_map_at_5_std": -0.0032305300628583337, - "nauc_mrr_at_1000_diff1": 0.45644452104799366, - "nauc_mrr_at_1000_max": 0.18966182165595427, - "nauc_mrr_at_1000_std": 0.015142223758896131, - "nauc_mrr_at_100_diff1": 0.45603098814474735, - "nauc_mrr_at_100_max": 0.19037424673457165, - "nauc_mrr_at_100_std": 0.01627979300863379, - "nauc_mrr_at_10_diff1": 0.4631346811594673, - "nauc_mrr_at_10_max": 0.18414084623389668, - "nauc_mrr_at_10_std": 0.006601657669226161, - "nauc_mrr_at_1_diff1": 0.5133140990283847, - "nauc_mrr_at_1_max": 0.14333658619372894, - "nauc_mrr_at_1_std": -0.0564724850439138, - "nauc_mrr_at_20_diff1": 0.45687943682993415, - "nauc_mrr_at_20_max": 0.18467676506949465, - "nauc_mrr_at_20_std": 0.010065050228116441, - "nauc_mrr_at_3_diff1": 0.46608467224631916, - "nauc_mrr_at_3_max": 0.16995784715010084, - "nauc_mrr_at_3_std": -0.008846294825584775, - "nauc_mrr_at_5_diff1": 0.4663183772981682, - "nauc_mrr_at_5_max": 0.1762843693554193, - "nauc_mrr_at_5_std": -0.0032305300628583337, - "nauc_ndcg_at_1000_diff1": 0.4380427564007995, - "nauc_ndcg_at_1000_max": 0.21748512763611927, - "nauc_ndcg_at_1000_std": 0.053149538455247586, - "nauc_ndcg_at_100_diff1": 0.4231968489839242, - "nauc_ndcg_at_100_max": 0.24795527816693558, - "nauc_ndcg_at_100_std": 0.09710467925441554, - "nauc_ndcg_at_10_diff1": 0.4480123666430948, - "nauc_ndcg_at_10_max": 0.20910479565113282, - "nauc_ndcg_at_10_std": 0.039700166847957956, - "nauc_ndcg_at_1_diff1": 0.5133140990283847, - "nauc_ndcg_at_1_max": 0.14333658619372894, - "nauc_ndcg_at_1_std": -0.0564724850439138, - "nauc_ndcg_at_20_diff1": 0.4268055180867815, - "nauc_ndcg_at_20_max": 0.21078731991897864, - "nauc_ndcg_at_20_std": 0.05221280409745525, - "nauc_ndcg_at_3_diff1": 0.4540115532109377, - "nauc_ndcg_at_3_max": 0.18117151911696, - "nauc_ndcg_at_3_std": 0.007362526259160504, - "nauc_ndcg_at_5_diff1": 0.4546328571737963, - "nauc_ndcg_at_5_max": 0.19120405311322689, - "nauc_ndcg_at_5_std": 0.016307727896751327, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.31186007531151105, - "nauc_precision_at_100_max": 0.4811622040684084, - "nauc_precision_at_100_std": 0.428090400261838, - "nauc_precision_at_10_diff1": 0.41340886364203755, - "nauc_precision_at_10_max": 0.27114828933311763, - "nauc_precision_at_10_std": 0.12190628617682332, - "nauc_precision_at_1_diff1": 0.5133140990283847, - "nauc_precision_at_1_max": 0.14333658619372894, - "nauc_precision_at_1_std": -0.0564724850439138, - "nauc_precision_at_20_diff1": 0.34592748872082796, - "nauc_precision_at_20_max": 0.2757908403388615, - "nauc_precision_at_20_std": 0.16246291668019033, - "nauc_precision_at_3_diff1": 0.4245150101271458, - "nauc_precision_at_3_max": 0.2098288582401962, - "nauc_precision_at_3_std": 0.04811678677570672, - "nauc_precision_at_5_diff1": 0.4281802505557515, - "nauc_precision_at_5_max": 0.22702373285133526, - "nauc_precision_at_5_std": 0.06251428333991228, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.31186007531151105, - "nauc_recall_at_100_max": 0.4811622040684086, - "nauc_recall_at_100_std": 0.42809040026183837, - "nauc_recall_at_10_diff1": 0.413408863642038, - "nauc_recall_at_10_max": 0.27114828933311785, - "nauc_recall_at_10_std": 0.12190628617682368, - "nauc_recall_at_1_diff1": 0.5133140990283847, - "nauc_recall_at_1_max": 0.14333658619372894, - "nauc_recall_at_1_std": -0.0564724850439138, - "nauc_recall_at_20_diff1": 0.34592748872082824, - "nauc_recall_at_20_max": 0.27579084033886203, - "nauc_recall_at_20_std": 0.16246291668019083, - "nauc_recall_at_3_diff1": 0.4245150101271459, - "nauc_recall_at_3_max": 0.20982885824019645, - "nauc_recall_at_3_std": 0.048116786775706716, - "nauc_recall_at_5_diff1": 0.42818025055575165, - "nauc_recall_at_5_max": 0.22702373285133542, - "nauc_recall_at_5_std": 0.06251428333991231, - "ndcg_at_1": 0.14, - "ndcg_at_10": 0.23555, - "ndcg_at_100": 0.29512, - "ndcg_at_1000": 0.33821, - "ndcg_at_20": 0.25274, - "ndcg_at_3": 0.185, - "ndcg_at_5": 0.21246, - "precision_at_1": 0.14, - "precision_at_10": 0.0357, - "precision_at_100": 0.00658, - "precision_at_1000": 0.001, - "precision_at_20": 0.02125, - "precision_at_3": 0.073, - "precision_at_5": 0.057, - "recall_at_1": 0.14, - "recall_at_10": 0.357, - "recall_at_100": 0.658, - "recall_at_1000": 1.0, - "recall_at_20": 0.425, - "recall_at_3": 0.219, - "recall_at_5": 0.285 - }, - { - "cv_recall_at_1": 0.39667, - "cv_recall_at_10": 0.65049, - "cv_recall_at_100": 0.89043, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.73786, - "cv_recall_at_3": 0.54508, - "cv_recall_at_5": 0.59223, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.52106, - "map_at_1": 0.39667, - "map_at_10": 0.47983, - "map_at_100": 0.48961, - "map_at_1000": 0.49022, - "map_at_20": 0.4859, - "map_at_3": 0.46117, - "map_at_5": 0.47178, - "mrr_at_1": 0.39667128987517336, - "mrr_at_10": 0.4798274002157494, - "mrr_at_100": 0.4896124803939883, - "mrr_at_1000": 0.4902196031014366, - "mrr_at_20": 0.4859049701628912, - "mrr_at_3": 0.4611650485436892, - "mrr_at_5": 0.471775312066574, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5529447924526923, - "nauc_cv_recall_at_100_max": 0.48063873317547207, - "nauc_cv_recall_at_100_std": 0.5084000789063866, - "nauc_cv_recall_at_10_diff1": 0.5501105521317363, - "nauc_cv_recall_at_10_max": 0.4901830644185336, - "nauc_cv_recall_at_10_std": 0.3619536097711491, - "nauc_cv_recall_at_1_diff1": 0.7338512053750866, - "nauc_cv_recall_at_1_max": 0.46598193211631234, - "nauc_cv_recall_at_1_std": 0.18958275403666142, - "nauc_cv_recall_at_20_diff1": 0.5235888915069656, - "nauc_cv_recall_at_20_max": 0.4837916615271111, - "nauc_cv_recall_at_20_std": 0.4264989192652688, - "nauc_cv_recall_at_3_diff1": 0.5941351764246412, - "nauc_cv_recall_at_3_max": 0.47104844256320394, - "nauc_cv_recall_at_3_std": 0.30558236581140236, - "nauc_cv_recall_at_5_diff1": 0.58452247493996, - "nauc_cv_recall_at_5_max": 0.48724150043188547, - "nauc_cv_recall_at_5_std": 0.334328948805313, - "nauc_map_at_1000_diff1": 0.6720195306324316, - "nauc_map_at_1000_max": 0.4740969612188918, - "nauc_map_at_1000_std": 0.24973019804344002, - "nauc_map_at_100_diff1": 0.67180082466236, - "nauc_map_at_100_max": 0.47400960560082256, - "nauc_map_at_100_std": 0.25009040568775737, - "nauc_map_at_10_diff1": 0.6701868031843548, - "nauc_map_at_10_max": 0.47473751185879304, - "nauc_map_at_10_std": 0.2489141754170303, - "nauc_map_at_1_diff1": 0.7338512053750866, - "nauc_map_at_1_max": 0.46598193211631234, - "nauc_map_at_1_std": 0.18958275403666142, - "nauc_map_at_20_diff1": 0.66998861295754, - "nauc_map_at_20_max": 0.474387524103794, - "nauc_map_at_20_std": 0.2510785033312345, - "nauc_map_at_3_diff1": 0.6735418032015751, - "nauc_map_at_3_max": 0.470600944971827, - "nauc_map_at_3_std": 0.2418534158719009, - "nauc_map_at_5_diff1": 0.6725778054575782, - "nauc_map_at_5_max": 0.47383144665195165, - "nauc_map_at_5_std": 0.24658927848582451, - "nauc_mrr_at_1000_diff1": 0.672020119769249, - "nauc_mrr_at_1000_max": 0.4740968793306214, - "nauc_mrr_at_1000_std": 0.24973009739879667, - "nauc_mrr_at_100_diff1": 0.6718014053275473, - "nauc_mrr_at_100_max": 0.47400950902542766, - "nauc_mrr_at_100_std": 0.25009027713865783, - "nauc_mrr_at_10_diff1": 0.6701868031843548, - "nauc_mrr_at_10_max": 0.47473751185879304, - "nauc_mrr_at_10_std": 0.2489141754170303, - "nauc_mrr_at_1_diff1": 0.7338512053750866, - "nauc_mrr_at_1_max": 0.46598193211631234, - "nauc_mrr_at_1_std": 0.18958275403666142, - "nauc_mrr_at_20_diff1": 0.66998861295754, - "nauc_mrr_at_20_max": 0.474387524103794, - "nauc_mrr_at_20_std": 0.2510785033312345, - "nauc_mrr_at_3_diff1": 0.6735418032015751, - "nauc_mrr_at_3_max": 0.470600944971827, - "nauc_mrr_at_3_std": 0.2418534158719009, - "nauc_mrr_at_5_diff1": 0.6725778054575782, - "nauc_mrr_at_5_max": 0.47383144665195165, - "nauc_mrr_at_5_std": 0.24658927848582451, - "nauc_ndcg_at_1000_diff1": 0.6601641574024943, - "nauc_ndcg_at_1000_max": 0.47563774314501484, - "nauc_ndcg_at_1000_std": 0.2649262811876548, - "nauc_ndcg_at_100_diff1": 0.6549206364926009, - "nauc_ndcg_at_100_max": 0.4752963483292056, - "nauc_ndcg_at_100_std": 0.27618420424960716, - "nauc_ndcg_at_10_diff1": 0.6442607607604707, - "nauc_ndcg_at_10_max": 0.4782010194867499, - "nauc_ndcg_at_10_std": 0.273393356452818, - "nauc_ndcg_at_1_diff1": 0.7338512053750866, - "nauc_ndcg_at_1_max": 0.46598193211631234, - "nauc_ndcg_at_1_std": 0.18958275403666142, - "nauc_ndcg_at_20_diff1": 0.6430884721245828, - "nauc_ndcg_at_20_max": 0.47663486711469094, - "nauc_ndcg_at_20_std": 0.28195874052103237, - "nauc_ndcg_at_3_diff1": 0.6538264079602413, - "nauc_ndcg_at_3_max": 0.47081803307646053, - "nauc_ndcg_at_3_std": 0.25777210528271977, - "nauc_ndcg_at_5_diff1": 0.6514996139228869, - "nauc_ndcg_at_5_max": 0.4769085934141065, - "nauc_ndcg_at_5_std": 0.2673431188114841, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.5529447924526933, - "nauc_precision_at_100_max": 0.48063873317547595, - "nauc_precision_at_100_std": 0.5084000789063885, - "nauc_precision_at_10_diff1": 0.5501105521317365, - "nauc_precision_at_10_max": 0.4901830644185332, - "nauc_precision_at_10_std": 0.36195360977114904, - "nauc_precision_at_1_diff1": 0.7338512053750866, - "nauc_precision_at_1_max": 0.46598193211631234, - "nauc_precision_at_1_std": 0.18958275403666142, - "nauc_precision_at_20_diff1": 0.5235888915069651, - "nauc_precision_at_20_max": 0.48379166152711106, - "nauc_precision_at_20_std": 0.4264989192652693, - "nauc_precision_at_3_diff1": 0.5941351764246411, - "nauc_precision_at_3_max": 0.47104844256320383, - "nauc_precision_at_3_std": 0.30558236581140213, - "nauc_precision_at_5_diff1": 0.5845224749399608, - "nauc_precision_at_5_max": 0.4872415004318854, - "nauc_precision_at_5_std": 0.33432894880531333, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5529447924526923, - "nauc_recall_at_100_max": 0.48063873317547207, - "nauc_recall_at_100_std": 0.5084000789063866, - "nauc_recall_at_10_diff1": 0.5501105521317363, - "nauc_recall_at_10_max": 0.4901830644185336, - "nauc_recall_at_10_std": 0.3619536097711491, - "nauc_recall_at_1_diff1": 0.7338512053750866, - "nauc_recall_at_1_max": 0.46598193211631234, - "nauc_recall_at_1_std": 0.18958275403666142, - "nauc_recall_at_20_diff1": 0.5235888915069656, - "nauc_recall_at_20_max": 0.4837916615271111, - "nauc_recall_at_20_std": 0.4264989192652688, - "nauc_recall_at_3_diff1": 0.5941351764246412, - "nauc_recall_at_3_max": 0.47104844256320394, - "nauc_recall_at_3_std": 0.30558236581140236, - "nauc_recall_at_5_diff1": 0.58452247493996, - "nauc_recall_at_5_max": 0.48724150043188547, - "nauc_recall_at_5_std": 0.334328948805313, - "ndcg_at_1": 0.39667, - "ndcg_at_10": 0.52106, - "ndcg_at_100": 0.57116, - "ndcg_at_1000": 0.58558, - "ndcg_at_20": 0.54317, - "ndcg_at_3": 0.48268, - "ndcg_at_5": 0.50195, - "precision_at_1": 0.39667, - "precision_at_10": 0.06505, - "precision_at_100": 0.0089, - "precision_at_1000": 0.001, - "precision_at_20": 0.03689, - "precision_at_3": 0.18169, - "precision_at_5": 0.11845, - "recall_at_1": 0.39667, - "recall_at_10": 0.65049, - "recall_at_100": 0.89043, - "recall_at_1000": 1.0, - "recall_at_20": 0.73786, - "recall_at_3": 0.54508, - "recall_at_5": 0.59223 - }, - { - "cv_recall_at_1": 0.43552, - "cv_recall_at_10": 0.6797, - "cv_recall_at_100": 0.87844, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.75793, - "cv_recall_at_3": 0.56025, - "cv_recall_at_5": 0.61628, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.55199, - "map_at_1": 0.43552, - "map_at_10": 0.51178, - "map_at_100": 0.52036, - "map_at_1000": 0.52095, - "map_at_20": 0.51714, - "map_at_3": 0.49084, - "map_at_5": 0.50331, - "mrr_at_1": 0.4355179704016913, - "mrr_at_10": 0.5117814356186449, - "mrr_at_100": 0.5203611719791804, - "mrr_at_1000": 0.5209528247341728, - "mrr_at_20": 0.517142191579205, - "mrr_at_3": 0.4908386187455953, - "mrr_at_5": 0.5033121916842842, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5212447375851132, - "nauc_cv_recall_at_100_max": 0.5043702843064178, - "nauc_cv_recall_at_100_std": 0.5877426834505205, - "nauc_cv_recall_at_10_diff1": 0.6033223572296531, - "nauc_cv_recall_at_10_max": 0.5299932139371255, - "nauc_cv_recall_at_10_std": 0.4435903956565738, - "nauc_cv_recall_at_1_diff1": 0.7464799971953152, - "nauc_cv_recall_at_1_max": 0.5381067495626654, - "nauc_cv_recall_at_1_std": 0.30716607647082733, - "nauc_cv_recall_at_20_diff1": 0.5491916947920159, - "nauc_cv_recall_at_20_max": 0.5114356154308455, - "nauc_cv_recall_at_20_std": 0.4812767256961097, - "nauc_cv_recall_at_3_diff1": 0.663126629290186, - "nauc_cv_recall_at_3_max": 0.5283498125854807, - "nauc_cv_recall_at_3_std": 0.346016951683277, - "nauc_cv_recall_at_5_diff1": 0.626626862939999, - "nauc_cv_recall_at_5_max": 0.5288735395629617, - "nauc_cv_recall_at_5_std": 0.40697999107692173, - "nauc_map_at_1000_diff1": 0.7040752895960932, - "nauc_map_at_1000_max": 0.5360887555262941, - "nauc_map_at_1000_std": 0.34052654689342726, - "nauc_map_at_100_diff1": 0.7037493671725752, - "nauc_map_at_100_max": 0.536044976656049, - "nauc_map_at_100_std": 0.34083807138122624, - "nauc_map_at_10_diff1": 0.7040569754687911, - "nauc_map_at_10_max": 0.5360673529713214, - "nauc_map_at_10_std": 0.33960926752548026, - "nauc_map_at_1_diff1": 0.7464799971953152, - "nauc_map_at_1_max": 0.5381067495626654, - "nauc_map_at_1_std": 0.30716607647082733, - "nauc_map_at_20_diff1": 0.7027070230721864, - "nauc_map_at_20_max": 0.5357565736391446, - "nauc_map_at_20_std": 0.34054086247508, - "nauc_map_at_3_diff1": 0.7122095603541071, - "nauc_map_at_3_max": 0.5354345456540576, - "nauc_map_at_3_std": 0.3246828517710239, - "nauc_map_at_5_diff1": 0.7054295586074079, - "nauc_map_at_5_max": 0.5356815100550746, - "nauc_map_at_5_std": 0.3363066932255336, - "nauc_mrr_at_1000_diff1": 0.7040754269691212, - "nauc_mrr_at_1000_max": 0.5360890407260704, - "nauc_mrr_at_1000_std": 0.3405269228763386, - "nauc_mrr_at_100_diff1": 0.7037495044605455, - "nauc_mrr_at_100_max": 0.5360452613859547, - "nauc_mrr_at_100_std": 0.3408384465735878, - "nauc_mrr_at_10_diff1": 0.7040569754687911, - "nauc_mrr_at_10_max": 0.5360673529713214, - "nauc_mrr_at_10_std": 0.33960926752548026, - "nauc_mrr_at_1_diff1": 0.7464799971953152, - "nauc_mrr_at_1_max": 0.5381067495626654, - "nauc_mrr_at_1_std": 0.30716607647082733, - "nauc_mrr_at_20_diff1": 0.7027070230721864, - "nauc_mrr_at_20_max": 0.5357565736391446, - "nauc_mrr_at_20_std": 0.34054086247508, - "nauc_mrr_at_3_diff1": 0.7122095603541071, - "nauc_mrr_at_3_max": 0.5354345456540576, - "nauc_mrr_at_3_std": 0.3246828517710239, - "nauc_mrr_at_5_diff1": 0.7054295586074079, - "nauc_mrr_at_5_max": 0.5356815100550746, - "nauc_mrr_at_5_std": 0.3363066932255336, - "nauc_ndcg_at_1000_diff1": 0.6922671257795675, - "nauc_ndcg_at_1000_max": 0.535122611480308, - "nauc_ndcg_at_1000_std": 0.35408371391461363, - "nauc_ndcg_at_100_diff1": 0.6831632876161986, - "nauc_ndcg_at_100_max": 0.5335032301662331, - "nauc_ndcg_at_100_std": 0.3657719912849508, - "nauc_ndcg_at_10_diff1": 0.6831155704338709, - "nauc_ndcg_at_10_max": 0.534797858763256, - "nauc_ndcg_at_10_std": 0.3605648365763527, - "nauc_ndcg_at_1_diff1": 0.7464799971953152, - "nauc_ndcg_at_1_max": 0.5381067495626654, - "nauc_ndcg_at_1_std": 0.30716607647082733, - "nauc_ndcg_at_20_diff1": 0.6769283008584868, - "nauc_ndcg_at_20_max": 0.5323259384219705, - "nauc_ndcg_at_20_std": 0.3641816897402438, - "nauc_ndcg_at_3_diff1": 0.7001088518732275, - "nauc_ndcg_at_3_max": 0.5337444781772217, - "nauc_ndcg_at_3_std": 0.3300076684439673, - "nauc_ndcg_at_5_diff1": 0.6872925776029278, - "nauc_ndcg_at_5_max": 0.5341253959019795, - "nauc_ndcg_at_5_std": 0.3521586226596281, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.5212447375851109, - "nauc_precision_at_100_max": 0.504370284306418, - "nauc_precision_at_100_std": 0.5877426834505183, - "nauc_precision_at_10_diff1": 0.6033223572296522, - "nauc_precision_at_10_max": 0.5299932139371254, - "nauc_precision_at_10_std": 0.4435903956565733, - "nauc_precision_at_1_diff1": 0.7464799971953152, - "nauc_precision_at_1_max": 0.5381067495626654, - "nauc_precision_at_1_std": 0.30716607647082733, - "nauc_precision_at_20_diff1": 0.5491916947920147, - "nauc_precision_at_20_max": 0.5114356154308448, - "nauc_precision_at_20_std": 0.4812767256961104, - "nauc_precision_at_3_diff1": 0.6631266292901861, - "nauc_precision_at_3_max": 0.5283498125854805, - "nauc_precision_at_3_std": 0.34601695168327695, - "nauc_precision_at_5_diff1": 0.6266268629399989, - "nauc_precision_at_5_max": 0.5288735395629613, - "nauc_precision_at_5_std": 0.40697999107692145, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5212447375851132, - "nauc_recall_at_100_max": 0.5043702843064178, - "nauc_recall_at_100_std": 0.5877426834505205, - "nauc_recall_at_10_diff1": 0.6033223572296531, - "nauc_recall_at_10_max": 0.5299932139371255, - "nauc_recall_at_10_std": 0.4435903956565738, - "nauc_recall_at_1_diff1": 0.7464799971953152, - "nauc_recall_at_1_max": 0.5381067495626654, - "nauc_recall_at_1_std": 0.30716607647082733, - "nauc_recall_at_20_diff1": 0.5491916947920159, - "nauc_recall_at_20_max": 0.5114356154308455, - "nauc_recall_at_20_std": 0.4812767256961097, - "nauc_recall_at_3_diff1": 0.663126629290186, - "nauc_recall_at_3_max": 0.5283498125854807, - "nauc_recall_at_3_std": 0.346016951683277, - "nauc_recall_at_5_diff1": 0.626626862939999, - "nauc_recall_at_5_max": 0.5288735395629617, - "nauc_recall_at_5_std": 0.40697999107692173, - "ndcg_at_1": 0.43552, - "ndcg_at_10": 0.55199, - "ndcg_at_100": 0.59427, - "ndcg_at_1000": 0.60987, - "ndcg_at_20": 0.57167, - "ndcg_at_3": 0.50868, - "ndcg_at_5": 0.53147, - "precision_at_1": 0.43552, - "precision_at_10": 0.06797, - "precision_at_100": 0.00878, - "precision_at_1000": 0.001, - "precision_at_20": 0.0379, - "precision_at_3": 0.18675, - "precision_at_5": 0.12326, - "recall_at_1": 0.43552, - "recall_at_10": 0.6797, - "recall_at_100": 0.87844, - "recall_at_1000": 1.0, - "recall_at_20": 0.75793, - "recall_at_3": 0.56025, - "recall_at_5": 0.61628 - }, - { - "cv_recall_at_1": 0.385, - "cv_recall_at_10": 0.547, - "cv_recall_at_100": 0.698, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.593, - "cv_recall_at_3": 0.463, - "cv_recall_at_5": 0.501, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.46078, - "map_at_1": 0.385, - "map_at_10": 0.43387, - "map_at_100": 0.43989, - "map_at_1000": 0.4406, - "map_at_20": 0.43709, - "map_at_3": 0.4195, - "map_at_5": 0.428, - "mrr_at_1": 0.385, - "mrr_at_10": 0.4338726190476187, - "mrr_at_100": 0.43989320041046814, - "mrr_at_1000": 0.44060486564869955, - "mrr_at_20": 0.43708518317337924, - "mrr_at_3": 0.41949999999999976, - "mrr_at_5": 0.42799999999999966, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5809042215421706, - "nauc_cv_recall_at_100_max": 0.8912379770177623, - "nauc_cv_recall_at_100_std": 0.2424633256438869, - "nauc_cv_recall_at_10_diff1": 0.672465991369597, - "nauc_cv_recall_at_10_max": 0.7786860120568095, - "nauc_cv_recall_at_10_std": 0.13047360840701458, - "nauc_cv_recall_at_1_diff1": 0.8258785548979316, - "nauc_cv_recall_at_1_max": 0.7022669665951418, - "nauc_cv_recall_at_1_std": 0.12157864370960537, - "nauc_cv_recall_at_20_diff1": 0.6343060416479551, - "nauc_cv_recall_at_20_max": 0.7904247690928177, - "nauc_cv_recall_at_20_std": 0.16095404825603088, - "nauc_cv_recall_at_3_diff1": 0.7328075282381276, - "nauc_cv_recall_at_3_max": 0.7167558281627555, - "nauc_cv_recall_at_3_std": 0.11546095857706429, - "nauc_cv_recall_at_5_diff1": 0.6998184687081933, - "nauc_cv_recall_at_5_max": 0.7271842109124178, - "nauc_cv_recall_at_5_std": 0.09769344553327645, - "nauc_map_at_1000_diff1": 0.7702690126570004, - "nauc_map_at_1000_max": 0.7186767063695445, - "nauc_map_at_1000_std": 0.12237612475107258, - "nauc_map_at_100_diff1": 0.7700896448953608, - "nauc_map_at_100_max": 0.7190166037158802, - "nauc_map_at_100_std": 0.12303030698837512, - "nauc_map_at_10_diff1": 0.7728156014659673, - "nauc_map_at_10_max": 0.7168863752172929, - "nauc_map_at_10_std": 0.11930561549242265, - "nauc_map_at_1_diff1": 0.8258785548979316, - "nauc_map_at_1_max": 0.7022669665951418, - "nauc_map_at_1_std": 0.12157864370960537, - "nauc_map_at_20_diff1": 0.7705807827864023, - "nauc_map_at_20_max": 0.7172956148978636, - "nauc_map_at_20_std": 0.12130235445505921, - "nauc_map_at_3_diff1": 0.7822816411140364, - "nauc_map_at_3_max": 0.7085502921868301, - "nauc_map_at_3_std": 0.11914325184351209, - "nauc_map_at_5_diff1": 0.7756209852636891, - "nauc_map_at_5_max": 0.710809137344318, - "nauc_map_at_5_std": 0.11531758273692924, - "nauc_mrr_at_1000_diff1": 0.7702690126570004, - "nauc_mrr_at_1000_max": 0.7186767063695445, - "nauc_mrr_at_1000_std": 0.12237612475107258, - "nauc_mrr_at_100_diff1": 0.7700896448953608, - "nauc_mrr_at_100_max": 0.7190166037158802, - "nauc_mrr_at_100_std": 0.12303030698837512, - "nauc_mrr_at_10_diff1": 0.7728156014659673, - "nauc_mrr_at_10_max": 0.7168863752172929, - "nauc_mrr_at_10_std": 0.11930561549242265, - "nauc_mrr_at_1_diff1": 0.8258785548979316, - "nauc_mrr_at_1_max": 0.7022669665951418, - "nauc_mrr_at_1_std": 0.12157864370960537, - "nauc_mrr_at_20_diff1": 0.7705807827864023, - "nauc_mrr_at_20_max": 0.7172956148978636, - "nauc_mrr_at_20_std": 0.12130235445505921, - "nauc_mrr_at_3_diff1": 0.7822816411140364, - "nauc_mrr_at_3_max": 0.7085502921868301, - "nauc_mrr_at_3_std": 0.11914325184351209, - "nauc_mrr_at_5_diff1": 0.7756209852636891, - "nauc_mrr_at_5_max": 0.710809137344318, - "nauc_mrr_at_5_std": 0.11531758273692924, - "nauc_ndcg_at_1000_diff1": 0.7506684503669501, - "nauc_ndcg_at_1000_max": 0.7309290666490663, - "nauc_ndcg_at_1000_std": 0.12703616553444683, - "nauc_ndcg_at_100_diff1": 0.7368679582830968, - "nauc_ndcg_at_100_max": 0.7452584701830449, - "nauc_ndcg_at_100_std": 0.14006746916659532, - "nauc_ndcg_at_10_diff1": 0.7494588028370229, - "nauc_ndcg_at_10_max": 0.7301937966102181, - "nauc_ndcg_at_10_std": 0.12121843676866292, - "nauc_ndcg_at_1_diff1": 0.8258785548979316, - "nauc_ndcg_at_1_max": 0.7022669665951418, - "nauc_ndcg_at_1_std": 0.12157864370960537, - "nauc_ndcg_at_20_diff1": 0.7413130488234162, - "nauc_ndcg_at_20_max": 0.7321767225115111, - "nauc_ndcg_at_20_std": 0.12834744637602127, - "nauc_ndcg_at_3_diff1": 0.7695347817860865, - "nauc_ndcg_at_3_max": 0.7106417741458194, - "nauc_ndcg_at_3_std": 0.11821387146555883, - "nauc_ndcg_at_5_diff1": 0.7568659031020656, - "nauc_ndcg_at_5_max": 0.7147924475388727, - "nauc_ndcg_at_5_std": 0.11114696240542633, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.5809042215421708, - "nauc_precision_at_100_max": 0.891237977017763, - "nauc_precision_at_100_std": 0.24246332564388692, - "nauc_precision_at_10_diff1": 0.672465991369597, - "nauc_precision_at_10_max": 0.7786860120568103, - "nauc_precision_at_10_std": 0.1304736084070146, - "nauc_precision_at_1_diff1": 0.8258785548979316, - "nauc_precision_at_1_max": 0.7022669665951418, - "nauc_precision_at_1_std": 0.12157864370960537, - "nauc_precision_at_20_diff1": 0.6343060416479551, - "nauc_precision_at_20_max": 0.7904247690928177, - "nauc_precision_at_20_std": 0.16095404825603044, - "nauc_precision_at_3_diff1": 0.7328075282381276, - "nauc_precision_at_3_max": 0.716755828162756, - "nauc_precision_at_3_std": 0.11546095857706405, - "nauc_precision_at_5_diff1": 0.6998184687081934, - "nauc_precision_at_5_max": 0.7271842109124178, - "nauc_precision_at_5_std": 0.0976934455332763, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5809042215421706, - "nauc_recall_at_100_max": 0.8912379770177623, - "nauc_recall_at_100_std": 0.2424633256438869, - "nauc_recall_at_10_diff1": 0.672465991369597, - "nauc_recall_at_10_max": 0.7786860120568095, - "nauc_recall_at_10_std": 0.13047360840701458, - "nauc_recall_at_1_diff1": 0.8258785548979316, - "nauc_recall_at_1_max": 0.7022669665951418, - "nauc_recall_at_1_std": 0.12157864370960537, - "nauc_recall_at_20_diff1": 0.6343060416479551, - "nauc_recall_at_20_max": 0.7904247690928177, - "nauc_recall_at_20_std": 0.16095404825603088, - "nauc_recall_at_3_diff1": 0.7328075282381276, - "nauc_recall_at_3_max": 0.7167558281627555, - "nauc_recall_at_3_std": 0.11546095857706429, - "nauc_recall_at_5_diff1": 0.6998184687081933, - "nauc_recall_at_5_max": 0.7271842109124178, - "nauc_recall_at_5_std": 0.09769344553327645, - "ndcg_at_1": 0.385, - "ndcg_at_10": 0.46078, - "ndcg_at_100": 0.49217, - "ndcg_at_1000": 0.52606, - "ndcg_at_20": 0.47243, - "ndcg_at_3": 0.43068, - "ndcg_at_5": 0.44617, - "precision_at_1": 0.385, - "precision_at_10": 0.0547, - "precision_at_100": 0.00698, - "precision_at_1000": 0.001, - "precision_at_20": 0.02965, - "precision_at_3": 0.15433, - "precision_at_5": 0.1002, - "recall_at_1": 0.385, - "recall_at_10": 0.547, - "recall_at_100": 0.698, - "recall_at_1000": 1.0, - "recall_at_20": 0.593, - "recall_at_3": 0.463, - "recall_at_5": 0.501 - } - ] - }, - "task_name": "WITT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/model_meta.json b/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/model_meta.json deleted file mode 100644 index e19e174af3..0000000000 --- a/results-mieb/google__siglip-large-patch16-256/d0da9f876e7d66b4e250cd2450c3ba2ce735e447/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "google/siglip-large-patch16-256", "revision": "d0da9f876e7d66b4e250cd2450c3ba2ce735e447", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/ImageNet10Clustering.json b/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/ImageNet10Clustering.json deleted file mode 100644 index 185e72dbf7..0000000000 --- a/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/ImageNet10Clustering.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "dataset_revision": "88f8a6d47c257895094c5ad81e67ba751771fc99", - "evaluation_time": 408.774094581604, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "ari": 0.9850555604145985, - "cluster_accuracy": 0.9932307692307693, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9806773175345154, - "nmi": 0.9806773175345154, - "v_measure": 0.9806773175345154 - } - ] - }, - "task_name": "ImageNet10Clustering" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/WITT2IRetrieval.json b/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/WITT2IRetrieval.json deleted file mode 100644 index 483cb8cbbb..0000000000 --- a/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/WITT2IRetrieval.json +++ /dev/null @@ -1,1936 +0,0 @@ -{ - "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", - "evaluation_time": 305.99254417419434, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.24157, - "cv_recall_at_10": 0.47978, - "cv_recall_at_100": 0.8, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.58427, - "cv_recall_at_3": 0.33708, - "cv_recall_at_5": 0.38315, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.34712, - "map_at_1": 0.24157, - "map_at_10": 0.30661, - "map_at_100": 0.31911, - "map_at_1000": 0.31999, - "map_at_20": 0.31396, - "map_at_3": 0.28371, - "map_at_5": 0.29416, - "mrr_at_1": 0.24157303370786518, - "mrr_at_10": 0.30660558230782947, - "mrr_at_100": 0.3191084750028444, - "mrr_at_1000": 0.3199914067421952, - "mrr_at_20": 0.313955020399258, - "mrr_at_3": 0.28370786516853946, - "mrr_at_5": 0.29415730337078644, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.38537866627754225, - "nauc_cv_recall_at_100_max": 0.5475086711041762, - "nauc_cv_recall_at_100_std": 0.5867727777840137, - "nauc_cv_recall_at_10_diff1": 0.43437211508014945, - "nauc_cv_recall_at_10_max": 0.3640413926999356, - "nauc_cv_recall_at_10_std": 0.3652366136002463, - "nauc_cv_recall_at_1_diff1": 0.5820698913453876, - "nauc_cv_recall_at_1_max": 0.4373704744724589, - "nauc_cv_recall_at_1_std": 0.31242354323111127, - "nauc_cv_recall_at_20_diff1": 0.4519298188132759, - "nauc_cv_recall_at_20_max": 0.3894351430162841, - "nauc_cv_recall_at_20_std": 0.41435732774155504, - "nauc_cv_recall_at_3_diff1": 0.4916636851520573, - "nauc_cv_recall_at_3_max": 0.38074686940966007, - "nauc_cv_recall_at_3_std": 0.33017441860465124, - "nauc_cv_recall_at_5_diff1": 0.4773076210875021, - "nauc_cv_recall_at_5_max": 0.38264219279226985, - "nauc_cv_recall_at_5_std": 0.35768618948540987, - "nauc_map_at_1000_diff1": 0.5318322145757168, - "nauc_map_at_1000_max": 0.4134388754083382, - "nauc_map_at_1000_std": 0.3306243250581623, - "nauc_map_at_100_diff1": 0.5314312528011818, - "nauc_map_at_100_max": 0.41370193480824047, - "nauc_map_at_100_std": 0.3311231298416902, - "nauc_map_at_10_diff1": 0.5298693442691903, - "nauc_map_at_10_max": 0.40899742619453344, - "nauc_map_at_10_std": 0.32611369397428047, - "nauc_map_at_1_diff1": 0.5820698913453876, - "nauc_map_at_1_max": 0.4373704744724589, - "nauc_map_at_1_std": 0.31242354323111127, - "nauc_map_at_20_diff1": 0.5312960056536469, - "nauc_map_at_20_max": 0.41022137345677856, - "nauc_map_at_20_std": 0.3288185491965263, - "nauc_map_at_3_diff1": 0.5385635093747566, - "nauc_map_at_3_max": 0.41066541045322624, - "nauc_map_at_3_std": 0.31799591730631943, - "nauc_map_at_5_diff1": 0.5356614723750011, - "nauc_map_at_5_max": 0.4113715519215499, - "nauc_map_at_5_std": 0.3249822371835517, - "nauc_mrr_at_1000_diff1": 0.5318322145757168, - "nauc_mrr_at_1000_max": 0.4134388754083382, - "nauc_mrr_at_1000_std": 0.3306243250581623, - "nauc_mrr_at_100_diff1": 0.5314312528011818, - "nauc_mrr_at_100_max": 0.41370193480824047, - "nauc_mrr_at_100_std": 0.3311231298416902, - "nauc_mrr_at_10_diff1": 0.5298693442691903, - "nauc_mrr_at_10_max": 0.40899742619453344, - "nauc_mrr_at_10_std": 0.32611369397428047, - "nauc_mrr_at_1_diff1": 0.5820698913453876, - "nauc_mrr_at_1_max": 0.4373704744724589, - "nauc_mrr_at_1_std": 0.31242354323111127, - "nauc_mrr_at_20_diff1": 0.5312960056536469, - "nauc_mrr_at_20_max": 0.41022137345677856, - "nauc_mrr_at_20_std": 0.3288185491965263, - "nauc_mrr_at_3_diff1": 0.5385635093747566, - "nauc_mrr_at_3_max": 0.41066541045322624, - "nauc_mrr_at_3_std": 0.31799591730631943, - "nauc_mrr_at_5_diff1": 0.5356614723750011, - "nauc_mrr_at_5_max": 0.4113715519215499, - "nauc_mrr_at_5_std": 0.3249822371835517, - "nauc_ndcg_at_1000_diff1": 0.520218088630065, - "nauc_ndcg_at_1000_max": 0.41499229550011485, - "nauc_ndcg_at_1000_std": 0.3446399250064482, - "nauc_ndcg_at_100_diff1": 0.5096653930893602, - "nauc_ndcg_at_100_max": 0.42468681407454717, - "nauc_ndcg_at_100_std": 0.3622985410547835, - "nauc_ndcg_at_10_diff1": 0.5060520176550207, - "nauc_ndcg_at_10_max": 0.3976210920323349, - "nauc_ndcg_at_10_std": 0.3358701849148283, - "nauc_ndcg_at_1_diff1": 0.5820698913453876, - "nauc_ndcg_at_1_max": 0.4373704744724589, - "nauc_ndcg_at_1_std": 0.31242354323111127, - "nauc_ndcg_at_20_diff1": 0.5116710283167186, - "nauc_ndcg_at_20_max": 0.4037066777079507, - "nauc_ndcg_at_20_std": 0.3467836131589162, - "nauc_ndcg_at_3_diff1": 0.5259304404932885, - "nauc_ndcg_at_3_max": 0.40262956715553533, - "nauc_ndcg_at_3_std": 0.3211502538045597, - "nauc_ndcg_at_5_diff1": 0.52011269409434, - "nauc_ndcg_at_5_max": 0.4035901466934185, - "nauc_ndcg_at_5_std": 0.33334212092589155, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.38537866627754325, - "nauc_precision_at_100_max": 0.5475086711041771, - "nauc_precision_at_100_std": 0.5867727777840136, - "nauc_precision_at_10_diff1": 0.43437211508014945, - "nauc_precision_at_10_max": 0.36404139269993546, - "nauc_precision_at_10_std": 0.3652366136002458, - "nauc_precision_at_1_diff1": 0.5820698913453876, - "nauc_precision_at_1_max": 0.4373704744724589, - "nauc_precision_at_1_std": 0.31242354323111127, - "nauc_precision_at_20_diff1": 0.4519298188132756, - "nauc_precision_at_20_max": 0.38943514301628335, - "nauc_precision_at_20_std": 0.41435732774155454, - "nauc_precision_at_3_diff1": 0.49166368515205705, - "nauc_precision_at_3_max": 0.3807468694096599, - "nauc_precision_at_3_std": 0.33017441860465113, - "nauc_precision_at_5_diff1": 0.4773076210875018, - "nauc_precision_at_5_max": 0.38264219279226996, - "nauc_precision_at_5_std": 0.35768618948541014, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.38537866627754225, - "nauc_recall_at_100_max": 0.5475086711041762, - "nauc_recall_at_100_std": 0.5867727777840137, - "nauc_recall_at_10_diff1": 0.43437211508014945, - "nauc_recall_at_10_max": 0.3640413926999356, - "nauc_recall_at_10_std": 0.3652366136002463, - "nauc_recall_at_1_diff1": 0.5820698913453876, - "nauc_recall_at_1_max": 0.4373704744724589, - "nauc_recall_at_1_std": 0.31242354323111127, - "nauc_recall_at_20_diff1": 0.4519298188132759, - "nauc_recall_at_20_max": 0.3894351430162841, - "nauc_recall_at_20_std": 0.41435732774155504, - "nauc_recall_at_3_diff1": 0.4916636851520573, - "nauc_recall_at_3_max": 0.38074686940966007, - "nauc_recall_at_3_std": 0.33017441860465124, - "nauc_recall_at_5_diff1": 0.4773076210875021, - "nauc_recall_at_5_max": 0.38264219279226985, - "nauc_recall_at_5_std": 0.35768618948540987, - "ndcg_at_1": 0.24157, - "ndcg_at_10": 0.34712, - "ndcg_at_100": 0.41285, - "ndcg_at_1000": 0.43797, - "ndcg_at_20": 0.37366, - "ndcg_at_3": 0.29742, - "ndcg_at_5": 0.31632, - "precision_at_1": 0.24157, - "precision_at_10": 0.04798, - "precision_at_100": 0.008, - "precision_at_1000": 0.001, - "precision_at_20": 0.02921, - "precision_at_3": 0.11236, - "precision_at_5": 0.07663, - "recall_at_1": 0.24157, - "recall_at_10": 0.47978, - "recall_at_100": 0.8, - "recall_at_1000": 1.0, - "recall_at_20": 0.58427, - "recall_at_3": 0.33708, - "recall_at_5": 0.38315 - }, - { - "cv_recall_at_1": 0.22558, - "cv_recall_at_10": 0.44884, - "cv_recall_at_100": 0.76512, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.52558, - "cv_recall_at_3": 0.32326, - "cv_recall_at_5": 0.3686, - "hf_subset": "bg", - "languages": [ - "bul-Cyrl" - ], - "main_score": 0.32733, - "map_at_1": 0.22558, - "map_at_10": 0.28986, - "map_at_100": 0.30114, - "map_at_1000": 0.30225, - "map_at_20": 0.29522, - "map_at_3": 0.26919, - "map_at_5": 0.27948, - "mrr_at_1": 0.2255813953488372, - "mrr_at_10": 0.2898629568106311, - "mrr_at_100": 0.3011372315333812, - "mrr_at_1000": 0.3022512651766774, - "mrr_at_20": 0.29521953737457013, - "mrr_at_3": 0.269186046511628, - "mrr_at_5": 0.2794767441860466, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4156195458011944, - "nauc_cv_recall_at_100_max": 0.4321414496076812, - "nauc_cv_recall_at_100_std": 0.2821314689037091, - "nauc_cv_recall_at_10_diff1": 0.4393000325721993, - "nauc_cv_recall_at_10_max": 0.3403794813410009, - "nauc_cv_recall_at_10_std": 0.047695440805355475, - "nauc_cv_recall_at_1_diff1": 0.5456746501489526, - "nauc_cv_recall_at_1_max": 0.3886520586941517, - "nauc_cv_recall_at_1_std": 0.06026950958268227, - "nauc_cv_recall_at_20_diff1": 0.4133092357721504, - "nauc_cv_recall_at_20_max": 0.3480612705948237, - "nauc_cv_recall_at_20_std": 0.07327907139538889, - "nauc_cv_recall_at_3_diff1": 0.4697449310838643, - "nauc_cv_recall_at_3_max": 0.36616880958519854, - "nauc_cv_recall_at_3_std": 0.08114614818289403, - "nauc_cv_recall_at_5_diff1": 0.43915646998582375, - "nauc_cv_recall_at_5_max": 0.33399631384859435, - "nauc_cv_recall_at_5_std": 0.0406646620242838, - "nauc_map_at_1000_diff1": 0.49689439348256015, - "nauc_map_at_1000_max": 0.3694777352607035, - "nauc_map_at_1000_std": 0.06365099267013102, - "nauc_map_at_100_diff1": 0.49652671912506346, - "nauc_map_at_100_max": 0.36949625835147754, - "nauc_map_at_100_std": 0.06390111935590399, - "nauc_map_at_10_diff1": 0.498401126203964, - "nauc_map_at_10_max": 0.3675709111670907, - "nauc_map_at_10_std": 0.059359535273785916, - "nauc_map_at_1_diff1": 0.5456746501489526, - "nauc_map_at_1_max": 0.3886520586941517, - "nauc_map_at_1_std": 0.06026950958268227, - "nauc_map_at_20_diff1": 0.4964328959270357, - "nauc_map_at_20_max": 0.36783464945255584, - "nauc_map_at_20_std": 0.06073670621949507, - "nauc_map_at_3_diff1": 0.5074187337184277, - "nauc_map_at_3_max": 0.37474783652153665, - "nauc_map_at_3_std": 0.06987499036428685, - "nauc_map_at_5_diff1": 0.4994576226731685, - "nauc_map_at_5_max": 0.36685059417933674, - "nauc_map_at_5_std": 0.05981883149689524, - "nauc_mrr_at_1000_diff1": 0.4968944008896699, - "nauc_mrr_at_1000_max": 0.36947775312287956, - "nauc_mrr_at_1000_std": 0.0636510175033233, - "nauc_mrr_at_100_diff1": 0.49652671912506346, - "nauc_mrr_at_100_max": 0.36949625835147754, - "nauc_mrr_at_100_std": 0.06390111935590399, - "nauc_mrr_at_10_diff1": 0.498401126203964, - "nauc_mrr_at_10_max": 0.3675709111670907, - "nauc_mrr_at_10_std": 0.059359535273785916, - "nauc_mrr_at_1_diff1": 0.5456746501489526, - "nauc_mrr_at_1_max": 0.3886520586941517, - "nauc_mrr_at_1_std": 0.06026950958268227, - "nauc_mrr_at_20_diff1": 0.4964328959270357, - "nauc_mrr_at_20_max": 0.36783464945255584, - "nauc_mrr_at_20_std": 0.06073670621949507, - "nauc_mrr_at_3_diff1": 0.5074187337184277, - "nauc_mrr_at_3_max": 0.37474783652153665, - "nauc_mrr_at_3_std": 0.06987499036428685, - "nauc_mrr_at_5_diff1": 0.4994576226731685, - "nauc_mrr_at_5_max": 0.36685059417933674, - "nauc_mrr_at_5_std": 0.05981883149689524, - "nauc_ndcg_at_1000_diff1": 0.48604737697435507, - "nauc_ndcg_at_1000_max": 0.36942860131684324, - "nauc_ndcg_at_1000_std": 0.0715185946116579, - "nauc_ndcg_at_100_diff1": 0.4792033641315331, - "nauc_ndcg_at_100_max": 0.3738987011330743, - "nauc_ndcg_at_100_std": 0.0875981776356468, - "nauc_ndcg_at_10_diff1": 0.4823054910823962, - "nauc_ndcg_at_10_max": 0.3602281989472767, - "nauc_ndcg_at_10_std": 0.056135484386297785, - "nauc_ndcg_at_1_diff1": 0.5456746501489526, - "nauc_ndcg_at_1_max": 0.3886520586941517, - "nauc_ndcg_at_1_std": 0.06026950958268227, - "nauc_ndcg_at_20_diff1": 0.47568071371427356, - "nauc_ndcg_at_20_max": 0.3619843976708002, - "nauc_ndcg_at_20_std": 0.06226665792973339, - "nauc_ndcg_at_3_diff1": 0.49712680073778454, - "nauc_ndcg_at_3_max": 0.3722913564738766, - "nauc_ndcg_at_3_std": 0.0729141340064338, - "nauc_ndcg_at_5_diff1": 0.48331434091040426, - "nauc_ndcg_at_5_max": 0.3582440286439964, - "nauc_ndcg_at_5_std": 0.05518668005370692, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.4156195458011953, - "nauc_precision_at_100_max": 0.4321414496076824, - "nauc_precision_at_100_std": 0.2821314689037095, - "nauc_precision_at_10_diff1": 0.4393000325721987, - "nauc_precision_at_10_max": 0.3403794813410003, - "nauc_precision_at_10_std": 0.047695440805355274, - "nauc_precision_at_1_diff1": 0.5456746501489526, - "nauc_precision_at_1_max": 0.3886520586941517, - "nauc_precision_at_1_std": 0.06026950958268227, - "nauc_precision_at_20_diff1": 0.41330923577215073, - "nauc_precision_at_20_max": 0.34806127059482384, - "nauc_precision_at_20_std": 0.07327907139538871, - "nauc_precision_at_3_diff1": 0.46974493108386384, - "nauc_precision_at_3_max": 0.3661688095851986, - "nauc_precision_at_3_std": 0.08114614818289405, - "nauc_precision_at_5_diff1": 0.43915646998582347, - "nauc_precision_at_5_max": 0.33399631384859413, - "nauc_precision_at_5_std": 0.04066466202428358, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4156195458011944, - "nauc_recall_at_100_max": 0.4321414496076812, - "nauc_recall_at_100_std": 0.2821314689037091, - "nauc_recall_at_10_diff1": 0.4393000325721993, - "nauc_recall_at_10_max": 0.3403794813410009, - "nauc_recall_at_10_std": 0.047695440805355475, - "nauc_recall_at_1_diff1": 0.5456746501489526, - "nauc_recall_at_1_max": 0.3886520586941517, - "nauc_recall_at_1_std": 0.06026950958268227, - "nauc_recall_at_20_diff1": 0.4133092357721504, - "nauc_recall_at_20_max": 0.3480612705948237, - "nauc_recall_at_20_std": 0.07327907139538889, - "nauc_recall_at_3_diff1": 0.4697449310838643, - "nauc_recall_at_3_max": 0.36616880958519854, - "nauc_recall_at_3_std": 0.08114614818289403, - "nauc_recall_at_5_diff1": 0.43915646998582375, - "nauc_recall_at_5_max": 0.33399631384859435, - "nauc_recall_at_5_std": 0.0406646620242838, - "ndcg_at_1": 0.22558, - "ndcg_at_10": 0.32733, - "ndcg_at_100": 0.3908, - "ndcg_at_1000": 0.4207, - "ndcg_at_20": 0.34678, - "ndcg_at_3": 0.2831, - "ndcg_at_5": 0.30171, - "precision_at_1": 0.22558, - "precision_at_10": 0.04488, - "precision_at_100": 0.00765, - "precision_at_1000": 0.001, - "precision_at_20": 0.02628, - "precision_at_3": 0.10775, - "precision_at_5": 0.07372, - "recall_at_1": 0.22558, - "recall_at_10": 0.44884, - "recall_at_100": 0.76512, - "recall_at_1000": 1.0, - "recall_at_20": 0.52558, - "recall_at_3": 0.32326, - "recall_at_5": 0.3686 - }, - { - "cv_recall_at_1": 0.40853, - "cv_recall_at_10": 0.65993, - "cv_recall_at_100": 0.87991, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.72727, - "cv_recall_at_3": 0.5376, - "cv_recall_at_5": 0.60494, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.52913, - "map_at_1": 0.40853, - "map_at_10": 0.4877, - "map_at_100": 0.49611, - "map_at_1000": 0.49663, - "map_at_20": 0.49233, - "map_at_3": 0.46483, - "map_at_5": 0.48043, - "mrr_at_1": 0.40852974186307517, - "mrr_at_10": 0.48770242103575406, - "mrr_at_100": 0.49610506582006364, - "mrr_at_1000": 0.4966320096481985, - "mrr_at_20": 0.49233257586044954, - "mrr_at_3": 0.4648335203890757, - "mrr_at_5": 0.48043396932285776, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5909320870844869, - "nauc_cv_recall_at_100_max": 0.6350295788074445, - "nauc_cv_recall_at_100_std": 0.5885417610554063, - "nauc_cv_recall_at_10_diff1": 0.6333473779929506, - "nauc_cv_recall_at_10_max": 0.5446487146800527, - "nauc_cv_recall_at_10_std": 0.2822394925476734, - "nauc_cv_recall_at_1_diff1": 0.765059299328094, - "nauc_cv_recall_at_1_max": 0.5047735351426804, - "nauc_cv_recall_at_1_std": 0.04360795615900494, - "nauc_cv_recall_at_20_diff1": 0.5950393518016469, - "nauc_cv_recall_at_20_max": 0.560818330778241, - "nauc_cv_recall_at_20_std": 0.34469275409722727, - "nauc_cv_recall_at_3_diff1": 0.6761769057730284, - "nauc_cv_recall_at_3_max": 0.507038537611141, - "nauc_cv_recall_at_3_std": 0.15324466916067125, - "nauc_cv_recall_at_5_diff1": 0.6311219292955862, - "nauc_cv_recall_at_5_max": 0.5258098225560203, - "nauc_cv_recall_at_5_std": 0.22820889829638605, - "nauc_map_at_1000_diff1": 0.7177069863322598, - "nauc_map_at_1000_max": 0.512298998322214, - "nauc_map_at_1000_std": 0.11131691750068567, - "nauc_map_at_100_diff1": 0.7174829160021539, - "nauc_map_at_100_max": 0.5123333682356263, - "nauc_map_at_100_std": 0.11191195506396118, - "nauc_map_at_10_diff1": 0.7185132575262295, - "nauc_map_at_10_max": 0.5119807392194838, - "nauc_map_at_10_std": 0.11013406915416628, - "nauc_map_at_1_diff1": 0.765059299328094, - "nauc_map_at_1_max": 0.5047735351426804, - "nauc_map_at_1_std": 0.04360795615900494, - "nauc_map_at_20_diff1": 0.7172107213295099, - "nauc_map_at_20_max": 0.512652076542774, - "nauc_map_at_20_std": 0.11191137664292765, - "nauc_map_at_3_diff1": 0.7256321861141027, - "nauc_map_at_3_max": 0.5066686915715853, - "nauc_map_at_3_std": 0.0917218401207876, - "nauc_map_at_5_diff1": 0.7171261768894966, - "nauc_map_at_5_max": 0.5102234717791918, - "nauc_map_at_5_std": 0.10560034278408584, - "nauc_mrr_at_1000_diff1": 0.7177069863322598, - "nauc_mrr_at_1000_max": 0.512298998322214, - "nauc_mrr_at_1000_std": 0.11131691750068567, - "nauc_mrr_at_100_diff1": 0.7174829160021539, - "nauc_mrr_at_100_max": 0.5123333682356263, - "nauc_mrr_at_100_std": 0.11191195506396118, - "nauc_mrr_at_10_diff1": 0.7185132575262295, - "nauc_mrr_at_10_max": 0.5119807392194838, - "nauc_mrr_at_10_std": 0.11013406915416628, - "nauc_mrr_at_1_diff1": 0.765059299328094, - "nauc_mrr_at_1_max": 0.5047735351426804, - "nauc_mrr_at_1_std": 0.04360795615900494, - "nauc_mrr_at_20_diff1": 0.7172107213295099, - "nauc_mrr_at_20_max": 0.512652076542774, - "nauc_mrr_at_20_std": 0.11191137664292765, - "nauc_mrr_at_3_diff1": 0.7256321861141027, - "nauc_mrr_at_3_max": 0.5066686915715853, - "nauc_mrr_at_3_std": 0.0917218401207876, - "nauc_mrr_at_5_diff1": 0.7171261768894966, - "nauc_mrr_at_5_max": 0.5102234717791918, - "nauc_mrr_at_5_std": 0.10560034278408584, - "nauc_ndcg_at_1000_diff1": 0.7061380148421971, - "nauc_ndcg_at_1000_max": 0.5171327008339424, - "nauc_ndcg_at_1000_std": 0.13519038473094888, - "nauc_ndcg_at_100_diff1": 0.7001742768553503, - "nauc_ndcg_at_100_max": 0.5219790485043022, - "nauc_ndcg_at_100_std": 0.15685332183761455, - "nauc_ndcg_at_10_diff1": 0.6999437000060731, - "nauc_ndcg_at_10_max": 0.5184168746325684, - "nauc_ndcg_at_10_std": 0.14593465923975651, - "nauc_ndcg_at_1_diff1": 0.765059299328094, - "nauc_ndcg_at_1_max": 0.5047735351426804, - "nauc_ndcg_at_1_std": 0.04360795615900494, - "nauc_ndcg_at_20_diff1": 0.6946562518708675, - "nauc_ndcg_at_20_max": 0.520841997954365, - "nauc_ndcg_at_20_std": 0.15355304474124215, - "nauc_ndcg_at_3_diff1": 0.7132599403846717, - "nauc_ndcg_at_3_max": 0.5068006685743853, - "nauc_ndcg_at_3_std": 0.10709000760145516, - "nauc_ndcg_at_5_diff1": 0.6972127766816861, - "nauc_ndcg_at_5_max": 0.5136594718171129, - "nauc_ndcg_at_5_std": 0.1337092541319943, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.5909320870844862, - "nauc_precision_at_100_max": 0.6350295788074443, - "nauc_precision_at_100_std": 0.5885417610554048, - "nauc_precision_at_10_diff1": 0.6333473779929502, - "nauc_precision_at_10_max": 0.5446487146800514, - "nauc_precision_at_10_std": 0.2822394925476726, - "nauc_precision_at_1_diff1": 0.765059299328094, - "nauc_precision_at_1_max": 0.5047735351426804, - "nauc_precision_at_1_std": 0.04360795615900494, - "nauc_precision_at_20_diff1": 0.5950393518016462, - "nauc_precision_at_20_max": 0.5608183307782408, - "nauc_precision_at_20_std": 0.3446927540972269, - "nauc_precision_at_3_diff1": 0.676176905773028, - "nauc_precision_at_3_max": 0.5070385376111409, - "nauc_precision_at_3_std": 0.1532446691606712, - "nauc_precision_at_5_diff1": 0.6311219292955864, - "nauc_precision_at_5_max": 0.5258098225560205, - "nauc_precision_at_5_std": 0.22820889829638602, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5909320870844869, - "nauc_recall_at_100_max": 0.6350295788074445, - "nauc_recall_at_100_std": 0.5885417610554063, - "nauc_recall_at_10_diff1": 0.6333473779929506, - "nauc_recall_at_10_max": 0.5446487146800527, - "nauc_recall_at_10_std": 0.2822394925476734, - "nauc_recall_at_1_diff1": 0.765059299328094, - "nauc_recall_at_1_max": 0.5047735351426804, - "nauc_recall_at_1_std": 0.04360795615900494, - "nauc_recall_at_20_diff1": 0.5950393518016469, - "nauc_recall_at_20_max": 0.560818330778241, - "nauc_recall_at_20_std": 0.34469275409722727, - "nauc_recall_at_3_diff1": 0.6761769057730284, - "nauc_recall_at_3_max": 0.507038537611141, - "nauc_recall_at_3_std": 0.15324466916067125, - "nauc_recall_at_5_diff1": 0.6311219292955862, - "nauc_recall_at_5_max": 0.5258098225560203, - "nauc_recall_at_5_std": 0.22820889829638605, - "ndcg_at_1": 0.40853, - "ndcg_at_10": 0.52913, - "ndcg_at_100": 0.57416, - "ndcg_at_1000": 0.58917, - "ndcg_at_20": 0.54611, - "ndcg_at_3": 0.4835, - "ndcg_at_5": 0.51142, - "precision_at_1": 0.40853, - "precision_at_10": 0.06599, - "precision_at_100": 0.0088, - "precision_at_1000": 0.001, - "precision_at_20": 0.03636, - "precision_at_3": 0.1792, - "precision_at_5": 0.12099, - "recall_at_1": 0.40853, - "recall_at_10": 0.65993, - "recall_at_100": 0.87991, - "recall_at_1000": 1.0, - "recall_at_20": 0.72727, - "recall_at_3": 0.5376, - "recall_at_5": 0.60494 - }, - { - "cv_recall_at_1": 0.16667, - "cv_recall_at_10": 0.35088, - "cv_recall_at_100": 0.69474, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.45439, - "cv_recall_at_3": 0.23509, - "cv_recall_at_5": 0.27719, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.24657, - "map_at_1": 0.16667, - "map_at_10": 0.21466, - "map_at_100": 0.2275, - "map_at_1000": 0.22898, - "map_at_20": 0.22193, - "map_at_3": 0.19503, - "map_at_5": 0.20477, - "mrr_at_1": 0.16666666666666666, - "mrr_at_10": 0.2146602617655249, - "mrr_at_100": 0.227502583825205, - "mrr_at_1000": 0.22898457017327967, - "mrr_at_20": 0.22192984828555953, - "mrr_at_3": 0.19502923976608177, - "mrr_at_5": 0.20476608187134498, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.330851835183518, - "nauc_cv_recall_at_100_max": 0.2257475747574758, - "nauc_cv_recall_at_100_std": 0.3469628212821287, - "nauc_cv_recall_at_10_diff1": 0.4433734104046245, - "nauc_cv_recall_at_10_max": 0.19268670520231207, - "nauc_cv_recall_at_10_std": 0.24033757225433527, - "nauc_cv_recall_at_1_diff1": 0.6046377190260788, - "nauc_cv_recall_at_1_max": 0.2795826052876528, - "nauc_cv_recall_at_1_std": 0.14331204990446947, - "nauc_cv_recall_at_20_diff1": 0.37560598495707764, - "nauc_cv_recall_at_20_max": 0.1927210478966829, - "nauc_cv_recall_at_20_std": 0.2678942439769638, - "nauc_cv_recall_at_3_diff1": 0.4957581959607422, - "nauc_cv_recall_at_3_max": 0.15605331947677356, - "nauc_cv_recall_at_3_std": 0.12871489244375645, - "nauc_cv_recall_at_5_diff1": 0.4619490553123599, - "nauc_cv_recall_at_5_max": 0.21248947474677848, - "nauc_cv_recall_at_5_std": 0.20337989885908708, - "nauc_map_at_1000_diff1": 0.5323722758486542, - "nauc_map_at_1000_max": 0.23646297993116652, - "nauc_map_at_1000_std": 0.16750923161500914, - "nauc_map_at_100_diff1": 0.5316186665095363, - "nauc_map_at_100_max": 0.23635986712686907, - "nauc_map_at_100_std": 0.16820139297706768, - "nauc_map_at_10_diff1": 0.5377626878903065, - "nauc_map_at_10_max": 0.23435667673382332, - "nauc_map_at_10_std": 0.16307234407663262, - "nauc_map_at_1_diff1": 0.6046377190260788, - "nauc_map_at_1_max": 0.2795826052876528, - "nauc_map_at_1_std": 0.14331204990446947, - "nauc_map_at_20_diff1": 0.5318397901748458, - "nauc_map_at_20_max": 0.23438189527334707, - "nauc_map_at_20_std": 0.16600605966152815, - "nauc_map_at_3_diff1": 0.5525148210101732, - "nauc_map_at_3_max": 0.2214669332663381, - "nauc_map_at_3_std": 0.1354554693488529, - "nauc_map_at_5_diff1": 0.5418733724063102, - "nauc_map_at_5_max": 0.23602458830140594, - "nauc_map_at_5_std": 0.15594974954289342, - "nauc_mrr_at_1000_diff1": 0.5323722758486542, - "nauc_mrr_at_1000_max": 0.23646297993116652, - "nauc_mrr_at_1000_std": 0.16750923161500914, - "nauc_mrr_at_100_diff1": 0.5316186665095363, - "nauc_mrr_at_100_max": 0.23635986712686907, - "nauc_mrr_at_100_std": 0.16820139297706768, - "nauc_mrr_at_10_diff1": 0.5377626878903065, - "nauc_mrr_at_10_max": 0.23435667673382332, - "nauc_mrr_at_10_std": 0.16307234407663262, - "nauc_mrr_at_1_diff1": 0.6046377190260788, - "nauc_mrr_at_1_max": 0.2795826052876528, - "nauc_mrr_at_1_std": 0.14331204990446947, - "nauc_mrr_at_20_diff1": 0.5318397901748458, - "nauc_mrr_at_20_max": 0.23438189527334707, - "nauc_mrr_at_20_std": 0.16600605966152815, - "nauc_mrr_at_3_diff1": 0.5525148210101732, - "nauc_mrr_at_3_max": 0.2214669332663381, - "nauc_mrr_at_3_std": 0.1354554693488529, - "nauc_mrr_at_5_diff1": 0.5418733724063102, - "nauc_mrr_at_5_max": 0.23602458830140594, - "nauc_mrr_at_5_std": 0.15594974954289342, - "nauc_ndcg_at_1000_diff1": 0.5093691679149464, - "nauc_ndcg_at_1000_max": 0.23258162232421242, - "nauc_ndcg_at_1000_std": 0.1852794629243669, - "nauc_ndcg_at_100_diff1": 0.4889330716375234, - "nauc_ndcg_at_100_max": 0.2314467376425257, - "nauc_ndcg_at_100_std": 0.20368758726476266, - "nauc_ndcg_at_10_diff1": 0.5107670584417292, - "nauc_ndcg_at_10_max": 0.2224163298308434, - "nauc_ndcg_at_10_std": 0.18382686225774514, - "nauc_ndcg_at_1_diff1": 0.6046377190260788, - "nauc_ndcg_at_1_max": 0.2795826052876528, - "nauc_ndcg_at_1_std": 0.14331204990446947, - "nauc_ndcg_at_20_diff1": 0.4905452978639291, - "nauc_ndcg_at_20_max": 0.2224535017956619, - "nauc_ndcg_at_20_std": 0.19270082403457917, - "nauc_ndcg_at_3_diff1": 0.536769234211766, - "nauc_ndcg_at_3_max": 0.20336056958214557, - "nauc_ndcg_at_3_std": 0.1335506875360385, - "nauc_ndcg_at_5_diff1": 0.5193278051702171, - "nauc_ndcg_at_5_max": 0.22853851210641513, - "nauc_ndcg_at_5_std": 0.16852042136773013, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.33085183518351857, - "nauc_precision_at_100_max": 0.2257475747574758, - "nauc_precision_at_100_std": 0.3469628212821282, - "nauc_precision_at_10_diff1": 0.44337341040462425, - "nauc_precision_at_10_max": 0.19268670520231215, - "nauc_precision_at_10_std": 0.24033757225433522, - "nauc_precision_at_1_diff1": 0.6046377190260788, - "nauc_precision_at_1_max": 0.2795826052876528, - "nauc_precision_at_1_std": 0.14331204990446947, - "nauc_precision_at_20_diff1": 0.3756059849570778, - "nauc_precision_at_20_max": 0.1927210478966828, - "nauc_precision_at_20_std": 0.26789424397696393, - "nauc_precision_at_3_diff1": 0.49575819596074194, - "nauc_precision_at_3_max": 0.15605331947677353, - "nauc_precision_at_3_std": 0.1287148924437563, - "nauc_precision_at_5_diff1": 0.4619490553123598, - "nauc_precision_at_5_max": 0.21248947474677846, - "nauc_precision_at_5_std": 0.20337989885908728, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.330851835183518, - "nauc_recall_at_100_max": 0.2257475747574758, - "nauc_recall_at_100_std": 0.3469628212821287, - "nauc_recall_at_10_diff1": 0.4433734104046245, - "nauc_recall_at_10_max": 0.19268670520231207, - "nauc_recall_at_10_std": 0.24033757225433527, - "nauc_recall_at_1_diff1": 0.6046377190260788, - "nauc_recall_at_1_max": 0.2795826052876528, - "nauc_recall_at_1_std": 0.14331204990446947, - "nauc_recall_at_20_diff1": 0.37560598495707764, - "nauc_recall_at_20_max": 0.1927210478966829, - "nauc_recall_at_20_std": 0.2678942439769638, - "nauc_recall_at_3_diff1": 0.4957581959607422, - "nauc_recall_at_3_max": 0.15605331947677356, - "nauc_recall_at_3_std": 0.12871489244375645, - "nauc_recall_at_5_diff1": 0.4619490553123599, - "nauc_recall_at_5_max": 0.21248947474677848, - "nauc_recall_at_5_std": 0.20337989885908708, - "ndcg_at_1": 0.16667, - "ndcg_at_10": 0.24657, - "ndcg_at_100": 0.31625, - "ndcg_at_1000": 0.35548, - "ndcg_at_20": 0.27285, - "ndcg_at_3": 0.20524, - "ndcg_at_5": 0.22268, - "precision_at_1": 0.16667, - "precision_at_10": 0.03509, - "precision_at_100": 0.00695, - "precision_at_1000": 0.001, - "precision_at_20": 0.02272, - "precision_at_3": 0.07836, - "precision_at_5": 0.05544, - "recall_at_1": 0.16667, - "recall_at_10": 0.35088, - "recall_at_100": 0.69474, - "recall_at_1000": 1.0, - "recall_at_20": 0.45439, - "recall_at_3": 0.23509, - "recall_at_5": 0.27719 - }, - { - "cv_recall_at_1": 0.22426, - "cv_recall_at_10": 0.40732, - "cv_recall_at_100": 0.76087, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.49542, - "cv_recall_at_3": 0.30549, - "cv_recall_at_5": 0.33982, - "hf_subset": "et", - "languages": [ - "est-Latn" - ], - "main_score": 0.30725, - "map_at_1": 0.22426, - "map_at_10": 0.27627, - "map_at_100": 0.28886, - "map_at_1000": 0.29, - "map_at_20": 0.28234, - "map_at_3": 0.25915, - "map_at_5": 0.26693, - "mrr_at_1": 0.2242562929061785, - "mrr_at_10": 0.27627492644655105, - "mrr_at_100": 0.2888584831291441, - "mrr_at_1000": 0.2900005222322179, - "mrr_at_20": 0.2823424222161656, - "mrr_at_3": 0.2591533180778033, - "mrr_at_5": 0.26693363844393597, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4548905095902432, - "nauc_cv_recall_at_100_max": 0.44644713522120655, - "nauc_cv_recall_at_100_std": 0.35804392058592754, - "nauc_cv_recall_at_10_diff1": 0.4959708344082086, - "nauc_cv_recall_at_10_max": 0.32782195690308946, - "nauc_cv_recall_at_10_std": 0.11774784435354384, - "nauc_cv_recall_at_1_diff1": 0.6688306834003572, - "nauc_cv_recall_at_1_max": 0.41410112769511687, - "nauc_cv_recall_at_1_std": 0.06822755485394784, - "nauc_cv_recall_at_20_diff1": 0.4933700813122085, - "nauc_cv_recall_at_20_max": 0.3493270465823052, - "nauc_cv_recall_at_20_std": 0.21358675720665704, - "nauc_cv_recall_at_3_diff1": 0.5676433428285875, - "nauc_cv_recall_at_3_max": 0.33746528106591145, - "nauc_cv_recall_at_3_std": 0.07763926424380885, - "nauc_cv_recall_at_5_diff1": 0.513300302770812, - "nauc_cv_recall_at_5_max": 0.3473703885687822, - "nauc_cv_recall_at_5_std": 0.09544098700198667, - "nauc_map_at_1000_diff1": 0.601309946818558, - "nauc_map_at_1000_max": 0.3811215060299473, - "nauc_map_at_1000_std": 0.08549562744577777, - "nauc_map_at_100_diff1": 0.6010036129821578, - "nauc_map_at_100_max": 0.3813066301818496, - "nauc_map_at_100_std": 0.0862408417273817, - "nauc_map_at_10_diff1": 0.6007541906369676, - "nauc_map_at_10_max": 0.3775895413975735, - "nauc_map_at_10_std": 0.07752197864393401, - "nauc_map_at_1_diff1": 0.6688306834003572, - "nauc_map_at_1_max": 0.41410112769511687, - "nauc_map_at_1_std": 0.06822755485394784, - "nauc_map_at_20_diff1": 0.6005353949460159, - "nauc_map_at_20_max": 0.3794670839752862, - "nauc_map_at_20_std": 0.08483609607709942, - "nauc_map_at_3_diff1": 0.6183803642609942, - "nauc_map_at_3_max": 0.37855822227622066, - "nauc_map_at_3_std": 0.06914276915890476, - "nauc_map_at_5_diff1": 0.6048073533485189, - "nauc_map_at_5_max": 0.38065436367380495, - "nauc_map_at_5_std": 0.07376843881638116, - "nauc_mrr_at_1000_diff1": 0.601309946818558, - "nauc_mrr_at_1000_max": 0.3811215060299473, - "nauc_mrr_at_1000_std": 0.08549562744577777, - "nauc_mrr_at_100_diff1": 0.6010036129821578, - "nauc_mrr_at_100_max": 0.3813066301818496, - "nauc_mrr_at_100_std": 0.0862408417273817, - "nauc_mrr_at_10_diff1": 0.6007541906369676, - "nauc_mrr_at_10_max": 0.3775895413975735, - "nauc_mrr_at_10_std": 0.07752197864393401, - "nauc_mrr_at_1_diff1": 0.6688306834003572, - "nauc_mrr_at_1_max": 0.41410112769511687, - "nauc_mrr_at_1_std": 0.06822755485394784, - "nauc_mrr_at_20_diff1": 0.6005353949460159, - "nauc_mrr_at_20_max": 0.3794670839752862, - "nauc_mrr_at_20_std": 0.08483609607709942, - "nauc_mrr_at_3_diff1": 0.6183803642609942, - "nauc_mrr_at_3_max": 0.37855822227622066, - "nauc_mrr_at_3_std": 0.06914276915890476, - "nauc_mrr_at_5_diff1": 0.6048073533485189, - "nauc_mrr_at_5_max": 0.38065436367380495, - "nauc_mrr_at_5_std": 0.07376843881638116, - "nauc_ndcg_at_1000_diff1": 0.5845812201325902, - "nauc_ndcg_at_1000_max": 0.37872969489351366, - "nauc_ndcg_at_1000_std": 0.1023087725689833, - "nauc_ndcg_at_100_diff1": 0.5738375546183035, - "nauc_ndcg_at_100_max": 0.38463128334228763, - "nauc_ndcg_at_100_std": 0.12459238801374915, - "nauc_ndcg_at_10_diff1": 0.5727643589394087, - "nauc_ndcg_at_10_max": 0.3643059512756334, - "nauc_ndcg_at_10_std": 0.08759165516814928, - "nauc_ndcg_at_1_diff1": 0.6688306834003572, - "nauc_ndcg_at_1_max": 0.41410112769511687, - "nauc_ndcg_at_1_std": 0.06822755485394784, - "nauc_ndcg_at_20_diff1": 0.5723644631319211, - "nauc_ndcg_at_20_max": 0.3705327377456405, - "nauc_ndcg_at_20_std": 0.11326390907853207, - "nauc_ndcg_at_3_diff1": 0.6045190422968498, - "nauc_ndcg_at_3_max": 0.36744809126982, - "nauc_ndcg_at_3_std": 0.07129906457747395, - "nauc_ndcg_at_5_diff1": 0.5805449517766387, - "nauc_ndcg_at_5_max": 0.3714042780033098, - "nauc_ndcg_at_5_std": 0.07929308964235518, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.4548905095902432, - "nauc_precision_at_100_max": 0.44644713522120666, - "nauc_precision_at_100_std": 0.35804392058592793, - "nauc_precision_at_10_diff1": 0.4959708344082088, - "nauc_precision_at_10_max": 0.32782195690308946, - "nauc_precision_at_10_std": 0.11774784435354392, - "nauc_precision_at_1_diff1": 0.6688306834003572, - "nauc_precision_at_1_max": 0.41410112769511687, - "nauc_precision_at_1_std": 0.06822755485394784, - "nauc_precision_at_20_diff1": 0.49337008131220894, - "nauc_precision_at_20_max": 0.34932704658230473, - "nauc_precision_at_20_std": 0.21358675720665715, - "nauc_precision_at_3_diff1": 0.5676433428285876, - "nauc_precision_at_3_max": 0.3374652810659113, - "nauc_precision_at_3_std": 0.07763926424380893, - "nauc_precision_at_5_diff1": 0.5133003027708118, - "nauc_precision_at_5_max": 0.3473703885687821, - "nauc_precision_at_5_std": 0.0954409870019865, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4548905095902432, - "nauc_recall_at_100_max": 0.44644713522120655, - "nauc_recall_at_100_std": 0.35804392058592754, - "nauc_recall_at_10_diff1": 0.4959708344082086, - "nauc_recall_at_10_max": 0.32782195690308946, - "nauc_recall_at_10_std": 0.11774784435354384, - "nauc_recall_at_1_diff1": 0.6688306834003572, - "nauc_recall_at_1_max": 0.41410112769511687, - "nauc_recall_at_1_std": 0.06822755485394784, - "nauc_recall_at_20_diff1": 0.4933700813122085, - "nauc_recall_at_20_max": 0.3493270465823052, - "nauc_recall_at_20_std": 0.21358675720665704, - "nauc_recall_at_3_diff1": 0.5676433428285875, - "nauc_recall_at_3_max": 0.33746528106591145, - "nauc_recall_at_3_std": 0.07763926424380885, - "nauc_recall_at_5_diff1": 0.513300302770812, - "nauc_recall_at_5_max": 0.3473703885687822, - "nauc_recall_at_5_std": 0.09544098700198667, - "ndcg_at_1": 0.22426, - "ndcg_at_10": 0.30725, - "ndcg_at_100": 0.37817, - "ndcg_at_1000": 0.40869, - "ndcg_at_20": 0.32945, - "ndcg_at_3": 0.27102, - "ndcg_at_5": 0.2851, - "precision_at_1": 0.22426, - "precision_at_10": 0.04073, - "precision_at_100": 0.00761, - "precision_at_1000": 0.001, - "precision_at_20": 0.02477, - "precision_at_3": 0.10183, - "precision_at_5": 0.06796, - "recall_at_1": 0.22426, - "recall_at_10": 0.40732, - "recall_at_100": 0.76087, - "recall_at_1000": 1.0, - "recall_at_20": 0.49542, - "recall_at_3": 0.30549, - "recall_at_5": 0.33982 - }, - { - "cv_recall_at_1": 0.52497, - "cv_recall_at_10": 0.74584, - "cv_recall_at_100": 0.93452, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.81354, - "cv_recall_at_3": 0.63596, - "cv_recall_at_5": 0.68923, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.6295, - "map_at_1": 0.52497, - "map_at_10": 0.5929, - "map_at_100": 0.60088, - "map_at_1000": 0.60123, - "map_at_20": 0.59776, - "map_at_3": 0.57307, - "map_at_5": 0.58555, - "mrr_at_1": 0.5249722530521642, - "mrr_at_10": 0.5929033877702022, - "mrr_at_100": 0.6008830842741953, - "mrr_at_1000": 0.6012302314360111, - "mrr_at_20": 0.5977596078367745, - "mrr_at_3": 0.57306696263411, - "mrr_at_5": 0.5855530891601919, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.6190269259232122, - "nauc_cv_recall_at_100_max": 0.7729281092845987, - "nauc_cv_recall_at_100_std": 0.6722070920847195, - "nauc_cv_recall_at_10_diff1": 0.6844620596166869, - "nauc_cv_recall_at_10_max": 0.4891266530015138, - "nauc_cv_recall_at_10_std": 0.3756587095100909, - "nauc_cv_recall_at_1_diff1": 0.8005694894014752, - "nauc_cv_recall_at_1_max": 0.4531124922527121, - "nauc_cv_recall_at_1_std": 0.16807685458452326, - "nauc_cv_recall_at_20_diff1": 0.6384051703535973, - "nauc_cv_recall_at_20_max": 0.5364946891885772, - "nauc_cv_recall_at_20_std": 0.45661774023473733, - "nauc_cv_recall_at_3_diff1": 0.7305967415604132, - "nauc_cv_recall_at_3_max": 0.4380960886436962, - "nauc_cv_recall_at_3_std": 0.25395868506657593, - "nauc_cv_recall_at_5_diff1": 0.7066215095307017, - "nauc_cv_recall_at_5_max": 0.4403031130193901, - "nauc_cv_recall_at_5_std": 0.28131628962301375, - "nauc_map_at_1000_diff1": 0.764661351219403, - "nauc_map_at_1000_max": 0.45355809867633967, - "nauc_map_at_1000_std": 0.2138472959642722, - "nauc_map_at_100_diff1": 0.7644678174458901, - "nauc_map_at_100_max": 0.4538588215763582, - "nauc_map_at_100_std": 0.21434823226077818, - "nauc_map_at_10_diff1": 0.7644185915083609, - "nauc_map_at_10_max": 0.45132329370285257, - "nauc_map_at_10_std": 0.2138793153043489, - "nauc_map_at_1_diff1": 0.8005694894014752, - "nauc_map_at_1_max": 0.4531124922527121, - "nauc_map_at_1_std": 0.16807685458452326, - "nauc_map_at_20_diff1": 0.7636421438964389, - "nauc_map_at_20_max": 0.45293445118961567, - "nauc_map_at_20_std": 0.21543100239398125, - "nauc_map_at_3_diff1": 0.7693383442376206, - "nauc_map_at_3_max": 0.4457495815844052, - "nauc_map_at_3_std": 0.20155794353526787, - "nauc_map_at_5_diff1": 0.7656408863186792, - "nauc_map_at_5_max": 0.44678876509561827, - "nauc_map_at_5_std": 0.20691973647903933, - "nauc_mrr_at_1000_diff1": 0.7646627667860468, - "nauc_mrr_at_1000_max": 0.4535564850362927, - "nauc_mrr_at_1000_std": 0.2138451798583905, - "nauc_mrr_at_100_diff1": 0.7644692275214324, - "nauc_mrr_at_100_max": 0.45385724414089046, - "nauc_mrr_at_100_std": 0.21434611237766832, - "nauc_mrr_at_10_diff1": 0.7644185915083609, - "nauc_mrr_at_10_max": 0.45132329370285257, - "nauc_mrr_at_10_std": 0.2138793153043489, - "nauc_mrr_at_1_diff1": 0.8005694894014752, - "nauc_mrr_at_1_max": 0.4531124922527121, - "nauc_mrr_at_1_std": 0.16807685458452326, - "nauc_mrr_at_20_diff1": 0.7636421438964389, - "nauc_mrr_at_20_max": 0.45293445118961567, - "nauc_mrr_at_20_std": 0.21543100239398125, - "nauc_mrr_at_3_diff1": 0.7693383442376206, - "nauc_mrr_at_3_max": 0.4457495815844052, - "nauc_mrr_at_3_std": 0.20155794353526787, - "nauc_mrr_at_5_diff1": 0.7656408863186792, - "nauc_mrr_at_5_max": 0.44678876509561827, - "nauc_mrr_at_5_std": 0.20691973647903933, - "nauc_ndcg_at_1000_diff1": 0.7562876547851568, - "nauc_ndcg_at_1000_max": 0.4600863415818717, - "nauc_ndcg_at_1000_std": 0.23017546787360782, - "nauc_ndcg_at_100_diff1": 0.7515996116803855, - "nauc_ndcg_at_100_max": 0.47008899439581425, - "nauc_ndcg_at_100_std": 0.2446746833885747, - "nauc_ndcg_at_10_diff1": 0.7488619888294695, - "nauc_ndcg_at_10_max": 0.4574503111316088, - "nauc_ndcg_at_10_std": 0.2436844400834987, - "nauc_ndcg_at_1_diff1": 0.8005694894014752, - "nauc_ndcg_at_1_max": 0.4531124922527121, - "nauc_ndcg_at_1_std": 0.16807685458452326, - "nauc_ndcg_at_20_diff1": 0.7446604086844123, - "nauc_ndcg_at_20_max": 0.4637857510751772, - "nauc_ndcg_at_20_std": 0.2507918044976272, - "nauc_ndcg_at_3_diff1": 0.7599025299332235, - "nauc_ndcg_at_3_max": 0.4438548334441876, - "nauc_ndcg_at_3_std": 0.21409525051528827, - "nauc_ndcg_at_5_diff1": 0.7525017128637891, - "nauc_ndcg_at_5_max": 0.44529280527499515, - "nauc_ndcg_at_5_std": 0.223485373476237, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.6190269259232114, - "nauc_precision_at_100_max": 0.7729281092845983, - "nauc_precision_at_100_std": 0.6722070920847233, - "nauc_precision_at_10_diff1": 0.6844620596166876, - "nauc_precision_at_10_max": 0.48912665300151364, - "nauc_precision_at_10_std": 0.37565870951009117, - "nauc_precision_at_1_diff1": 0.8005694894014752, - "nauc_precision_at_1_max": 0.4531124922527121, - "nauc_precision_at_1_std": 0.16807685458452326, - "nauc_precision_at_20_diff1": 0.6384051703535972, - "nauc_precision_at_20_max": 0.536494689188577, - "nauc_precision_at_20_std": 0.4566177402347358, - "nauc_precision_at_3_diff1": 0.7305967415604135, - "nauc_precision_at_3_max": 0.43809608864369626, - "nauc_precision_at_3_std": 0.2539586850665756, - "nauc_precision_at_5_diff1": 0.7066215095307022, - "nauc_precision_at_5_max": 0.44030311301938957, - "nauc_precision_at_5_std": 0.2813162896230139, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.6190269259232122, - "nauc_recall_at_100_max": 0.7729281092845987, - "nauc_recall_at_100_std": 0.6722070920847195, - "nauc_recall_at_10_diff1": 0.6844620596166869, - "nauc_recall_at_10_max": 0.4891266530015138, - "nauc_recall_at_10_std": 0.3756587095100909, - "nauc_recall_at_1_diff1": 0.8005694894014752, - "nauc_recall_at_1_max": 0.4531124922527121, - "nauc_recall_at_1_std": 0.16807685458452326, - "nauc_recall_at_20_diff1": 0.6384051703535973, - "nauc_recall_at_20_max": 0.5364946891885772, - "nauc_recall_at_20_std": 0.45661774023473733, - "nauc_recall_at_3_diff1": 0.7305967415604132, - "nauc_recall_at_3_max": 0.4380960886436962, - "nauc_recall_at_3_std": 0.25395868506657593, - "nauc_recall_at_5_diff1": 0.7066215095307017, - "nauc_recall_at_5_max": 0.4403031130193901, - "nauc_recall_at_5_std": 0.28131628962301375, - "ndcg_at_1": 0.52497, - "ndcg_at_10": 0.6295, - "ndcg_at_100": 0.66932, - "ndcg_at_1000": 0.67783, - "ndcg_at_20": 0.64681, - "ndcg_at_3": 0.58919, - "ndcg_at_5": 0.6114, - "precision_at_1": 0.52497, - "precision_at_10": 0.07458, - "precision_at_100": 0.00935, - "precision_at_1000": 0.001, - "precision_at_20": 0.04068, - "precision_at_3": 0.21199, - "precision_at_5": 0.13785, - "recall_at_1": 0.52497, - "recall_at_10": 0.74584, - "recall_at_100": 0.93452, - "recall_at_1000": 1.0, - "recall_at_20": 0.81354, - "recall_at_3": 0.63596, - "recall_at_5": 0.68923 - }, - { - "cv_recall_at_1": 0.15575, - "cv_recall_at_10": 0.31794, - "cv_recall_at_100": 0.63373, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.40494, - "cv_recall_at_3": 0.19871, - "cv_recall_at_5": 0.25027, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.22424, - "map_at_1": 0.15575, - "map_at_10": 0.19581, - "map_at_100": 0.20715, - "map_at_1000": 0.20863, - "map_at_20": 0.20175, - "map_at_3": 0.1749, - "map_at_5": 0.18666, - "mrr_at_1": 0.15574650912996776, - "mrr_at_10": 0.19580967725436035, - "mrr_at_100": 0.20715421138921317, - "mrr_at_1000": 0.20863345075667283, - "mrr_at_20": 0.20175038883712101, - "mrr_at_3": 0.17490153956319368, - "mrr_at_5": 0.18666308628714642, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.24444918782474567, - "nauc_cv_recall_at_100_max": 0.32551466197697165, - "nauc_cv_recall_at_100_std": 0.45167668398297006, - "nauc_cv_recall_at_10_diff1": 0.39248291964911963, - "nauc_cv_recall_at_10_max": 0.3066812417261699, - "nauc_cv_recall_at_10_std": 0.3424873919857647, - "nauc_cv_recall_at_1_diff1": 0.6532550450478714, - "nauc_cv_recall_at_1_max": 0.44239290311087814, - "nauc_cv_recall_at_1_std": 0.32573808089156275, - "nauc_cv_recall_at_20_diff1": 0.36388396999886186, - "nauc_cv_recall_at_20_max": 0.29934288856164354, - "nauc_cv_recall_at_20_std": 0.3696664843365718, - "nauc_cv_recall_at_3_diff1": 0.5069636472753279, - "nauc_cv_recall_at_3_max": 0.35407303170096777, - "nauc_cv_recall_at_3_std": 0.2986517151004437, - "nauc_cv_recall_at_5_diff1": 0.47172961423842896, - "nauc_cv_recall_at_5_max": 0.33375077016623395, - "nauc_cv_recall_at_5_std": 0.31794011506865144, - "nauc_map_at_1000_diff1": 0.5508520944278331, - "nauc_map_at_1000_max": 0.38565983075291094, - "nauc_map_at_1000_std": 0.3261751242369309, - "nauc_map_at_100_diff1": 0.5496666524189222, - "nauc_map_at_100_max": 0.3853271693919615, - "nauc_map_at_100_std": 0.3265286270672321, - "nauc_map_at_10_diff1": 0.5551643822996104, - "nauc_map_at_10_max": 0.3859397236650562, - "nauc_map_at_10_std": 0.32287469193293833, - "nauc_map_at_1_diff1": 0.6532550450478714, - "nauc_map_at_1_max": 0.44239290311087814, - "nauc_map_at_1_std": 0.32573808089156275, - "nauc_map_at_20_diff1": 0.5516571306533536, - "nauc_map_at_20_max": 0.38486781606814013, - "nauc_map_at_20_std": 0.32480713401957406, - "nauc_map_at_3_diff1": 0.5815687647615574, - "nauc_map_at_3_max": 0.39889133439982516, - "nauc_map_at_3_std": 0.3144923960506113, - "nauc_map_at_5_diff1": 0.5711653159966787, - "nauc_map_at_5_max": 0.3922705457437804, - "nauc_map_at_5_std": 0.3189425954414773, - "nauc_mrr_at_1000_diff1": 0.5508520944278331, - "nauc_mrr_at_1000_max": 0.38565983075291094, - "nauc_mrr_at_1000_std": 0.3261751242369309, - "nauc_mrr_at_100_diff1": 0.5496666524189222, - "nauc_mrr_at_100_max": 0.3853271693919615, - "nauc_mrr_at_100_std": 0.3265286270672321, - "nauc_mrr_at_10_diff1": 0.5551643822996104, - "nauc_mrr_at_10_max": 0.3859397236650562, - "nauc_mrr_at_10_std": 0.32287469193293833, - "nauc_mrr_at_1_diff1": 0.6532550450478714, - "nauc_mrr_at_1_max": 0.44239290311087814, - "nauc_mrr_at_1_std": 0.32573808089156275, - "nauc_mrr_at_20_diff1": 0.5516571306533536, - "nauc_mrr_at_20_max": 0.38486781606814013, - "nauc_mrr_at_20_std": 0.32480713401957406, - "nauc_mrr_at_3_diff1": 0.5815687647615574, - "nauc_mrr_at_3_max": 0.39889133439982516, - "nauc_mrr_at_3_std": 0.3144923960506113, - "nauc_mrr_at_5_diff1": 0.5711653159966787, - "nauc_mrr_at_5_max": 0.3922705457437804, - "nauc_mrr_at_5_std": 0.3189425954414773, - "nauc_ndcg_at_1000_diff1": 0.5157264617450563, - "nauc_ndcg_at_1000_max": 0.37274339990632316, - "nauc_ndcg_at_1000_std": 0.3355623933659139, - "nauc_ndcg_at_100_diff1": 0.48148009490671284, - "nauc_ndcg_at_100_max": 0.3661030696417765, - "nauc_ndcg_at_100_std": 0.3493858547937965, - "nauc_ndcg_at_10_diff1": 0.5088785336848107, - "nauc_ndcg_at_10_max": 0.36287802189034524, - "nauc_ndcg_at_10_std": 0.3276751318762903, - "nauc_ndcg_at_1_diff1": 0.6532550450478714, - "nauc_ndcg_at_1_max": 0.44239290311087814, - "nauc_ndcg_at_1_std": 0.32573808089156275, - "nauc_ndcg_at_20_diff1": 0.4982450502542284, - "nauc_ndcg_at_20_max": 0.35978126250426545, - "nauc_ndcg_at_20_std": 0.33557435140627284, - "nauc_ndcg_at_3_diff1": 0.560933086731898, - "nauc_ndcg_at_3_max": 0.38648542016608384, - "nauc_ndcg_at_3_std": 0.31019490114977205, - "nauc_ndcg_at_5_diff1": 0.5429858311638133, - "nauc_ndcg_at_5_max": 0.3755963165912826, - "nauc_ndcg_at_5_std": 0.3183859719256547, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.244449187824746, - "nauc_precision_at_100_max": 0.3255146619769719, - "nauc_precision_at_100_std": 0.45167668398296984, - "nauc_precision_at_10_diff1": 0.3924829196491193, - "nauc_precision_at_10_max": 0.30668124172616984, - "nauc_precision_at_10_std": 0.3424873919857649, - "nauc_precision_at_1_diff1": 0.6532550450478714, - "nauc_precision_at_1_max": 0.44239290311087814, - "nauc_precision_at_1_std": 0.32573808089156275, - "nauc_precision_at_20_diff1": 0.36388396999886186, - "nauc_precision_at_20_max": 0.29934288856164326, - "nauc_precision_at_20_std": 0.3696664843365716, - "nauc_precision_at_3_diff1": 0.5069636472753278, - "nauc_precision_at_3_max": 0.35407303170096777, - "nauc_precision_at_3_std": 0.29865171510044364, - "nauc_precision_at_5_diff1": 0.47172961423842874, - "nauc_precision_at_5_max": 0.33375077016623395, - "nauc_precision_at_5_std": 0.31794011506865155, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.24444918782474567, - "nauc_recall_at_100_max": 0.32551466197697165, - "nauc_recall_at_100_std": 0.45167668398297006, - "nauc_recall_at_10_diff1": 0.39248291964911963, - "nauc_recall_at_10_max": 0.3066812417261699, - "nauc_recall_at_10_std": 0.3424873919857647, - "nauc_recall_at_1_diff1": 0.6532550450478714, - "nauc_recall_at_1_max": 0.44239290311087814, - "nauc_recall_at_1_std": 0.32573808089156275, - "nauc_recall_at_20_diff1": 0.36388396999886186, - "nauc_recall_at_20_max": 0.29934288856164354, - "nauc_recall_at_20_std": 0.3696664843365718, - "nauc_recall_at_3_diff1": 0.5069636472753279, - "nauc_recall_at_3_max": 0.35407303170096777, - "nauc_recall_at_3_std": 0.2986517151004437, - "nauc_recall_at_5_diff1": 0.47172961423842896, - "nauc_recall_at_5_max": 0.33375077016623395, - "nauc_recall_at_5_std": 0.31794011506865144, - "ndcg_at_1": 0.15575, - "ndcg_at_10": 0.22424, - "ndcg_at_100": 0.28762, - "ndcg_at_1000": 0.33285, - "ndcg_at_20": 0.24613, - "ndcg_at_3": 0.18103, - "ndcg_at_5": 0.20224, - "precision_at_1": 0.15575, - "precision_at_10": 0.03179, - "precision_at_100": 0.00634, - "precision_at_1000": 0.001, - "precision_at_20": 0.02025, - "precision_at_3": 0.06624, - "precision_at_5": 0.05005, - "recall_at_1": 0.15575, - "recall_at_10": 0.31794, - "recall_at_100": 0.63373, - "recall_at_1000": 1.0, - "recall_at_20": 0.40494, - "recall_at_3": 0.19871, - "recall_at_5": 0.25027 - }, - { - "cv_recall_at_1": 0.133, - "cv_recall_at_10": 0.348, - "cv_recall_at_100": 0.681, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.432, - "cv_recall_at_3": 0.219, - "cv_recall_at_5": 0.272, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.2298, - "map_at_1": 0.133, - "map_at_10": 0.19332, - "map_at_100": 0.20463, - "map_at_1000": 0.20609, - "map_at_20": 0.19876, - "map_at_3": 0.171, - "map_at_5": 0.18315, - "mrr_at_1": 0.133, - "mrr_at_10": 0.19332341269841272, - "mrr_at_100": 0.20462787375291316, - "mrr_at_1000": 0.20608662727509244, - "mrr_at_20": 0.1987633009620239, - "mrr_at_3": 0.17100000000000004, - "mrr_at_5": 0.18315, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.23903970030450739, - "nauc_cv_recall_at_100_max": 0.41640569684568446, - "nauc_cv_recall_at_100_std": 0.39736831627291774, - "nauc_cv_recall_at_10_diff1": 0.37136341320494715, - "nauc_cv_recall_at_10_max": 0.18230049253282243, - "nauc_cv_recall_at_10_std": 0.04904476083387433, - "nauc_cv_recall_at_1_diff1": 0.5180319349233996, - "nauc_cv_recall_at_1_max": 0.1270218074435545, - "nauc_cv_recall_at_1_std": -0.07816591097771257, - "nauc_cv_recall_at_20_diff1": 0.36617498542395466, - "nauc_cv_recall_at_20_max": 0.22647790687989328, - "nauc_cv_recall_at_20_std": 0.10219499625187434, - "nauc_cv_recall_at_3_diff1": 0.42927422380603214, - "nauc_cv_recall_at_3_max": 0.1421225439141381, - "nauc_cv_recall_at_3_std": -0.020967317575151773, - "nauc_cv_recall_at_5_diff1": 0.392570337984777, - "nauc_cv_recall_at_5_max": 0.1601208544762595, - "nauc_cv_recall_at_5_std": 0.008884276458861794, - "nauc_map_at_1000_diff1": 0.44984391381046307, - "nauc_map_at_1000_max": 0.15054300211478264, - "nauc_map_at_1000_std": -0.021052809041627263, - "nauc_map_at_100_diff1": 0.44918790083327276, - "nauc_map_at_100_max": 0.1510241417560371, - "nauc_map_at_100_std": -0.019927498393880156, - "nauc_map_at_10_diff1": 0.45343156466916373, - "nauc_map_at_10_max": 0.1428449015931523, - "nauc_map_at_10_std": -0.031102817230918154, - "nauc_map_at_1_diff1": 0.5180319349233996, - "nauc_map_at_1_max": 0.1270218074435545, - "nauc_map_at_1_std": -0.07816591097771257, - "nauc_map_at_20_diff1": 0.4524836567452582, - "nauc_map_at_20_max": 0.1468680942413539, - "nauc_map_at_20_std": -0.026211756253432503, - "nauc_map_at_3_diff1": 0.4698730760278111, - "nauc_map_at_3_max": 0.13492394255234672, - "nauc_map_at_3_std": -0.04706689363194477, - "nauc_map_at_5_diff1": 0.4585585830085684, - "nauc_map_at_5_max": 0.13931591541765556, - "nauc_map_at_5_std": -0.038330787238466967, - "nauc_mrr_at_1000_diff1": 0.44984391381046307, - "nauc_mrr_at_1000_max": 0.15054300211478264, - "nauc_mrr_at_1000_std": -0.021052809041627263, - "nauc_mrr_at_100_diff1": 0.44918790083327276, - "nauc_mrr_at_100_max": 0.1510241417560371, - "nauc_mrr_at_100_std": -0.019927498393880156, - "nauc_mrr_at_10_diff1": 0.45343156466916373, - "nauc_mrr_at_10_max": 0.1428449015931523, - "nauc_mrr_at_10_std": -0.031102817230918154, - "nauc_mrr_at_1_diff1": 0.5180319349233996, - "nauc_mrr_at_1_max": 0.1270218074435545, - "nauc_mrr_at_1_std": -0.07816591097771257, - "nauc_mrr_at_20_diff1": 0.4524836567452582, - "nauc_mrr_at_20_max": 0.1468680942413539, - "nauc_mrr_at_20_std": -0.026211756253432503, - "nauc_mrr_at_3_diff1": 0.4698730760278111, - "nauc_mrr_at_3_max": 0.13492394255234672, - "nauc_mrr_at_3_std": -0.04706689363194477, - "nauc_mrr_at_5_diff1": 0.4585585830085684, - "nauc_mrr_at_5_max": 0.13931591541765556, - "nauc_mrr_at_5_std": -0.038330787238466967, - "nauc_ndcg_at_1000_diff1": 0.42734696398847494, - "nauc_ndcg_at_1000_max": 0.17217833907343263, - "nauc_ndcg_at_1000_std": 0.012881439336521373, - "nauc_ndcg_at_100_diff1": 0.4053394775042063, - "nauc_ndcg_at_100_max": 0.19895583284204096, - "nauc_ndcg_at_100_std": 0.056812856244403374, - "nauc_ndcg_at_10_diff1": 0.4288409161311715, - "nauc_ndcg_at_10_max": 0.15389774771982148, - "nauc_ndcg_at_10_std": -0.00797049243824064, - "nauc_ndcg_at_1_diff1": 0.5180319349233996, - "nauc_ndcg_at_1_max": 0.1270218074435545, - "nauc_ndcg_at_1_std": -0.07816591097771257, - "nauc_ndcg_at_20_diff1": 0.426561870160365, - "nauc_ndcg_at_20_max": 0.16776057042499307, - "nauc_ndcg_at_20_std": 0.00882600452766406, - "nauc_ndcg_at_3_diff1": 0.4579909913635628, - "nauc_ndcg_at_3_max": 0.1370186461871269, - "nauc_ndcg_at_3_std": -0.0394256678182846, - "nauc_ndcg_at_5_diff1": 0.43906112535844644, - "nauc_ndcg_at_5_max": 0.14522893418960348, - "nauc_ndcg_at_5_std": -0.02448000303769126, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.2390397003045082, - "nauc_precision_at_100_max": 0.41640569684568407, - "nauc_precision_at_100_std": 0.39736831627291763, - "nauc_precision_at_10_diff1": 0.37136341320494715, - "nauc_precision_at_10_max": 0.18230049253282227, - "nauc_precision_at_10_std": 0.04904476083387422, - "nauc_precision_at_1_diff1": 0.5180319349233996, - "nauc_precision_at_1_max": 0.1270218074435545, - "nauc_precision_at_1_std": -0.07816591097771257, - "nauc_precision_at_20_diff1": 0.3661749854239545, - "nauc_precision_at_20_max": 0.2264779068798933, - "nauc_precision_at_20_std": 0.10219499625187378, - "nauc_precision_at_3_diff1": 0.42927422380603214, - "nauc_precision_at_3_max": 0.14212254391413814, - "nauc_precision_at_3_std": -0.020967317575151828, - "nauc_precision_at_5_diff1": 0.3925703379847769, - "nauc_precision_at_5_max": 0.16012085447625948, - "nauc_precision_at_5_std": 0.008884276458861773, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.23903970030450739, - "nauc_recall_at_100_max": 0.41640569684568446, - "nauc_recall_at_100_std": 0.39736831627291774, - "nauc_recall_at_10_diff1": 0.37136341320494715, - "nauc_recall_at_10_max": 0.18230049253282243, - "nauc_recall_at_10_std": 0.04904476083387433, - "nauc_recall_at_1_diff1": 0.5180319349233996, - "nauc_recall_at_1_max": 0.1270218074435545, - "nauc_recall_at_1_std": -0.07816591097771257, - "nauc_recall_at_20_diff1": 0.36617498542395466, - "nauc_recall_at_20_max": 0.22647790687989328, - "nauc_recall_at_20_std": 0.10219499625187434, - "nauc_recall_at_3_diff1": 0.42927422380603214, - "nauc_recall_at_3_max": 0.1421225439141381, - "nauc_recall_at_3_std": -0.020967317575151773, - "nauc_recall_at_5_diff1": 0.392570337984777, - "nauc_recall_at_5_max": 0.1601208544762595, - "nauc_recall_at_5_std": 0.008884276458861794, - "ndcg_at_1": 0.133, - "ndcg_at_10": 0.2298, - "ndcg_at_100": 0.29574, - "ndcg_at_1000": 0.33603, - "ndcg_at_20": 0.25055, - "ndcg_at_3": 0.18333, - "ndcg_at_5": 0.20519, - "precision_at_1": 0.133, - "precision_at_10": 0.0348, - "precision_at_100": 0.00681, - "precision_at_1000": 0.001, - "precision_at_20": 0.0216, - "precision_at_3": 0.073, - "precision_at_5": 0.0544, - "recall_at_1": 0.133, - "recall_at_10": 0.348, - "recall_at_100": 0.681, - "recall_at_1000": 1.0, - "recall_at_20": 0.432, - "recall_at_3": 0.219, - "recall_at_5": 0.272 - }, - { - "cv_recall_at_1": 0.4258, - "cv_recall_at_10": 0.68239, - "cv_recall_at_100": 0.89182, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.74064, - "cv_recall_at_3": 0.55062, - "cv_recall_at_5": 0.61026, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.54674, - "map_at_1": 0.4258, - "map_at_10": 0.50431, - "map_at_100": 0.51226, - "map_at_1000": 0.51286, - "map_at_20": 0.50837, - "map_at_3": 0.48151, - "map_at_5": 0.49496, - "mrr_at_1": 0.42579750346740636, - "mrr_at_10": 0.5043056381128503, - "mrr_at_100": 0.5122619082917252, - "mrr_at_1000": 0.5128598359955646, - "mrr_at_20": 0.5083677746206783, - "mrr_at_3": 0.48150716597318527, - "mrr_at_5": 0.49496070272769277, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.6123550386274283, - "nauc_cv_recall_at_100_max": 0.5478399377012312, - "nauc_cv_recall_at_100_std": 0.496080100518916, - "nauc_cv_recall_at_10_diff1": 0.6142929793515284, - "nauc_cv_recall_at_10_max": 0.5190468391127735, - "nauc_cv_recall_at_10_std": 0.385308604962248, - "nauc_cv_recall_at_1_diff1": 0.7747521987597996, - "nauc_cv_recall_at_1_max": 0.5173781139473123, - "nauc_cv_recall_at_1_std": 0.13473303740040443, - "nauc_cv_recall_at_20_diff1": 0.6336034724432345, - "nauc_cv_recall_at_20_max": 0.5115030632455113, - "nauc_cv_recall_at_20_std": 0.39928737908127715, - "nauc_cv_recall_at_3_diff1": 0.6797923699658351, - "nauc_cv_recall_at_3_max": 0.5177177240768384, - "nauc_cv_recall_at_3_std": 0.2711052415459016, - "nauc_cv_recall_at_5_diff1": 0.6492496293194232, - "nauc_cv_recall_at_5_max": 0.495768199463953, - "nauc_cv_recall_at_5_std": 0.27675036787034907, - "nauc_map_at_1000_diff1": 0.728333770648744, - "nauc_map_at_1000_max": 0.516118056668068, - "nauc_map_at_1000_std": 0.2016134228119069, - "nauc_map_at_100_diff1": 0.7281529278617421, - "nauc_map_at_100_max": 0.5161391068643536, - "nauc_map_at_100_std": 0.20210605477046603, - "nauc_map_at_10_diff1": 0.7261188851666448, - "nauc_map_at_10_max": 0.5173169342754899, - "nauc_map_at_10_std": 0.20479249871993216, - "nauc_map_at_1_diff1": 0.7747521987597996, - "nauc_map_at_1_max": 0.5173781139473123, - "nauc_map_at_1_std": 0.13473303740040443, - "nauc_map_at_20_diff1": 0.7276632804464694, - "nauc_map_at_20_max": 0.5166384481658993, - "nauc_map_at_20_std": 0.2037465157952008, - "nauc_map_at_3_diff1": 0.7331507665544107, - "nauc_map_at_3_max": 0.5185691526030635, - "nauc_map_at_3_std": 0.19450591863838512, - "nauc_map_at_5_diff1": 0.7282450588763438, - "nauc_map_at_5_max": 0.5139642537867429, - "nauc_map_at_5_std": 0.19442892713879853, - "nauc_mrr_at_1000_diff1": 0.7283336738950129, - "nauc_mrr_at_1000_max": 0.5161182608501237, - "nauc_mrr_at_1000_std": 0.20161372824442178, - "nauc_mrr_at_100_diff1": 0.7281528313373693, - "nauc_mrr_at_100_max": 0.5161393106782692, - "nauc_mrr_at_100_std": 0.20210635950408978, - "nauc_mrr_at_10_diff1": 0.7261188851666448, - "nauc_mrr_at_10_max": 0.5173169342754899, - "nauc_mrr_at_10_std": 0.20479249871993216, - "nauc_mrr_at_1_diff1": 0.7747521987597996, - "nauc_mrr_at_1_max": 0.5173781139473123, - "nauc_mrr_at_1_std": 0.13473303740040443, - "nauc_mrr_at_20_diff1": 0.7276632804464694, - "nauc_mrr_at_20_max": 0.5166384481658993, - "nauc_mrr_at_20_std": 0.2037465157952008, - "nauc_mrr_at_3_diff1": 0.7331507665544107, - "nauc_mrr_at_3_max": 0.5185691526030635, - "nauc_mrr_at_3_std": 0.19450591863838512, - "nauc_mrr_at_5_diff1": 0.7282450588763438, - "nauc_mrr_at_5_max": 0.5139642537867429, - "nauc_mrr_at_5_std": 0.19442892713879853, - "nauc_ndcg_at_1000_diff1": 0.7183362744659123, - "nauc_ndcg_at_1000_max": 0.5151142335186784, - "nauc_ndcg_at_1000_std": 0.2178540584968831, - "nauc_ndcg_at_100_diff1": 0.7132894716769472, - "nauc_ndcg_at_100_max": 0.5164316539030083, - "nauc_ndcg_at_100_std": 0.23117635336299322, - "nauc_ndcg_at_10_diff1": 0.7032515997687422, - "nauc_ndcg_at_10_max": 0.5174827984990962, - "nauc_ndcg_at_10_std": 0.24096931371599054, - "nauc_ndcg_at_1_diff1": 0.7747521987597996, - "nauc_ndcg_at_1_max": 0.5173781139473123, - "nauc_ndcg_at_1_std": 0.13473303740040443, - "nauc_ndcg_at_20_diff1": 0.7092928733514847, - "nauc_ndcg_at_20_max": 0.5156346782452621, - "nauc_ndcg_at_20_std": 0.23866367889009282, - "nauc_ndcg_at_3_diff1": 0.719853330258983, - "nauc_ndcg_at_3_max": 0.5184029128753416, - "nauc_ndcg_at_3_std": 0.2135964955541178, - "nauc_ndcg_at_5_diff1": 0.7098897269058249, - "nauc_ndcg_at_5_max": 0.5099870789610721, - "nauc_ndcg_at_5_std": 0.21404335193633725, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.6123550386274293, - "nauc_precision_at_100_max": 0.5478399377012316, - "nauc_precision_at_100_std": 0.49608010051891843, - "nauc_precision_at_10_diff1": 0.6142929793515284, - "nauc_precision_at_10_max": 0.5190468391127726, - "nauc_precision_at_10_std": 0.38530860496224717, - "nauc_precision_at_1_diff1": 0.7747521987597996, - "nauc_precision_at_1_max": 0.5173781139473123, - "nauc_precision_at_1_std": 0.13473303740040443, - "nauc_precision_at_20_diff1": 0.6336034724432347, - "nauc_precision_at_20_max": 0.5115030632455123, - "nauc_precision_at_20_std": 0.39928737908127737, - "nauc_precision_at_3_diff1": 0.6797923699658351, - "nauc_precision_at_3_max": 0.5177177240768378, - "nauc_precision_at_3_std": 0.2711052415459004, - "nauc_precision_at_5_diff1": 0.6492496293194232, - "nauc_precision_at_5_max": 0.4957681994639534, - "nauc_precision_at_5_std": 0.276750367870349, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.6123550386274283, - "nauc_recall_at_100_max": 0.5478399377012312, - "nauc_recall_at_100_std": 0.496080100518916, - "nauc_recall_at_10_diff1": 0.6142929793515284, - "nauc_recall_at_10_max": 0.5190468391127735, - "nauc_recall_at_10_std": 0.385308604962248, - "nauc_recall_at_1_diff1": 0.7747521987597996, - "nauc_recall_at_1_max": 0.5173781139473123, - "nauc_recall_at_1_std": 0.13473303740040443, - "nauc_recall_at_20_diff1": 0.6336034724432345, - "nauc_recall_at_20_max": 0.5115030632455113, - "nauc_recall_at_20_std": 0.39928737908127715, - "nauc_recall_at_3_diff1": 0.6797923699658351, - "nauc_recall_at_3_max": 0.5177177240768384, - "nauc_recall_at_3_std": 0.2711052415459016, - "nauc_recall_at_5_diff1": 0.6492496293194232, - "nauc_recall_at_5_max": 0.495768199463953, - "nauc_recall_at_5_std": 0.27675036787034907, - "ndcg_at_1": 0.4258, - "ndcg_at_10": 0.54674, - "ndcg_at_100": 0.58958, - "ndcg_at_1000": 0.60381, - "ndcg_at_20": 0.56149, - "ndcg_at_3": 0.49929, - "ndcg_at_5": 0.5237, - "precision_at_1": 0.4258, - "precision_at_10": 0.06824, - "precision_at_100": 0.00892, - "precision_at_1000": 0.001, - "precision_at_20": 0.03703, - "precision_at_3": 0.18354, - "precision_at_5": 0.12205, - "recall_at_1": 0.4258, - "recall_at_10": 0.68239, - "recall_at_100": 0.89182, - "recall_at_1000": 1.0, - "recall_at_20": 0.74064, - "recall_at_3": 0.55062, - "recall_at_5": 0.61026 - }, - { - "cv_recall_at_1": 0.45455, - "cv_recall_at_10": 0.70296, - "cv_recall_at_100": 0.88584, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.7611, - "cv_recall_at_3": 0.57928, - "cv_recall_at_5": 0.63425, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.5731, - "map_at_1": 0.45455, - "map_at_10": 0.53221, - "map_at_100": 0.53943, - "map_at_1000": 0.54, - "map_at_20": 0.53637, - "map_at_3": 0.51039, - "map_at_5": 0.52271, - "mrr_at_1": 0.45454545454545453, - "mrr_at_10": 0.5322116513976977, - "mrr_at_100": 0.5394335226463994, - "mrr_at_1000": 0.540004150062426, - "mrr_at_20": 0.5363658296845691, - "mrr_at_3": 0.5103946441155741, - "mrr_at_5": 0.5227096546863984, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5155748885294896, - "nauc_cv_recall_at_100_max": 0.4354612206487773, - "nauc_cv_recall_at_100_std": 0.5786132641876298, - "nauc_cv_recall_at_10_diff1": 0.6016904587793288, - "nauc_cv_recall_at_10_max": 0.5453527162042908, - "nauc_cv_recall_at_10_std": 0.4474004366146903, - "nauc_cv_recall_at_1_diff1": 0.7397486518013686, - "nauc_cv_recall_at_1_max": 0.5581512468677294, - "nauc_cv_recall_at_1_std": 0.2786415604449159, - "nauc_cv_recall_at_20_diff1": 0.5539394639480063, - "nauc_cv_recall_at_20_max": 0.5329757970859269, - "nauc_cv_recall_at_20_std": 0.5029165107845112, - "nauc_cv_recall_at_3_diff1": 0.6362736404675333, - "nauc_cv_recall_at_3_max": 0.5600825936851035, - "nauc_cv_recall_at_3_std": 0.3562218259043451, - "nauc_cv_recall_at_5_diff1": 0.6134324595869594, - "nauc_cv_recall_at_5_max": 0.5514024998817529, - "nauc_cv_recall_at_5_std": 0.376335323466691, - "nauc_map_at_1000_diff1": 0.6884311413366787, - "nauc_map_at_1000_max": 0.5545252247296255, - "nauc_map_at_1000_std": 0.32258057532335105, - "nauc_map_at_100_diff1": 0.6881264891740738, - "nauc_map_at_100_max": 0.5542651721296694, - "nauc_map_at_100_std": 0.32298399733590283, - "nauc_map_at_10_diff1": 0.6887002858727944, - "nauc_map_at_10_max": 0.555550891059468, - "nauc_map_at_10_std": 0.3216058018523894, - "nauc_map_at_1_diff1": 0.7397486518013686, - "nauc_map_at_1_max": 0.5581512468677294, - "nauc_map_at_1_std": 0.2786415604449159, - "nauc_map_at_20_diff1": 0.6872412205765465, - "nauc_map_at_20_max": 0.5550520779209613, - "nauc_map_at_20_std": 0.323314768524235, - "nauc_map_at_3_diff1": 0.693482051512695, - "nauc_map_at_3_max": 0.5589401535113256, - "nauc_map_at_3_std": 0.31236577566495405, - "nauc_map_at_5_diff1": 0.6895582892296516, - "nauc_map_at_5_max": 0.5565381934854133, - "nauc_map_at_5_std": 0.31533116013401896, - "nauc_mrr_at_1000_diff1": 0.688431220346766, - "nauc_mrr_at_1000_max": 0.5545254930231911, - "nauc_mrr_at_1000_std": 0.3225809480403303, - "nauc_mrr_at_100_diff1": 0.6881267561091581, - "nauc_mrr_at_100_max": 0.554265682415342, - "nauc_mrr_at_100_std": 0.32298470557702164, - "nauc_mrr_at_10_diff1": 0.6887002858727944, - "nauc_mrr_at_10_max": 0.555550891059468, - "nauc_mrr_at_10_std": 0.3216058018523894, - "nauc_mrr_at_1_diff1": 0.7397486518013686, - "nauc_mrr_at_1_max": 0.5581512468677294, - "nauc_mrr_at_1_std": 0.2786415604449159, - "nauc_mrr_at_20_diff1": 0.6872412205765465, - "nauc_mrr_at_20_max": 0.5550520779209613, - "nauc_mrr_at_20_std": 0.323314768524235, - "nauc_mrr_at_3_diff1": 0.693482051512695, - "nauc_mrr_at_3_max": 0.5589401535113256, - "nauc_mrr_at_3_std": 0.31236577566495405, - "nauc_mrr_at_5_diff1": 0.6895582892296516, - "nauc_mrr_at_5_max": 0.5565381934854133, - "nauc_mrr_at_5_std": 0.31533116013401896, - "nauc_ndcg_at_1000_diff1": 0.6771612397254433, - "nauc_ndcg_at_1000_max": 0.5516918879125599, - "nauc_ndcg_at_1000_std": 0.33737823092057195, - "nauc_ndcg_at_100_diff1": 0.66881208513375, - "nauc_ndcg_at_100_max": 0.5453526920357381, - "nauc_ndcg_at_100_std": 0.34944573233573056, - "nauc_ndcg_at_10_diff1": 0.6701885011667525, - "nauc_ndcg_at_10_max": 0.5534554872062589, - "nauc_ndcg_at_10_std": 0.3462041956274489, - "nauc_ndcg_at_1_diff1": 0.7397486518013686, - "nauc_ndcg_at_1_max": 0.5581512468677294, - "nauc_ndcg_at_1_std": 0.2786415604449159, - "nauc_ndcg_at_20_diff1": 0.6639174288133743, - "nauc_ndcg_at_20_max": 0.5514800394725279, - "nauc_ndcg_at_20_std": 0.35304329050488403, - "nauc_ndcg_at_3_diff1": 0.6792793166204122, - "nauc_ndcg_at_3_max": 0.5592203897905106, - "nauc_ndcg_at_3_std": 0.32320983990013075, - "nauc_ndcg_at_5_diff1": 0.671864236588547, - "nauc_ndcg_at_5_max": 0.5553744878364937, - "nauc_ndcg_at_5_std": 0.32935433616936693, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.5155748885294904, - "nauc_precision_at_100_max": 0.435461220648779, - "nauc_precision_at_100_std": 0.578613264187631, - "nauc_precision_at_10_diff1": 0.6016904587793293, - "nauc_precision_at_10_max": 0.5453527162042906, - "nauc_precision_at_10_std": 0.44740043661469014, - "nauc_precision_at_1_diff1": 0.7397486518013686, - "nauc_precision_at_1_max": 0.5581512468677294, - "nauc_precision_at_1_std": 0.2786415604449159, - "nauc_precision_at_20_diff1": 0.5539394639480061, - "nauc_precision_at_20_max": 0.5329757970859269, - "nauc_precision_at_20_std": 0.5029165107845114, - "nauc_precision_at_3_diff1": 0.6362736404675334, - "nauc_precision_at_3_max": 0.5600825936851035, - "nauc_precision_at_3_std": 0.35622182590434565, - "nauc_precision_at_5_diff1": 0.6134324595869591, - "nauc_precision_at_5_max": 0.5514024998817525, - "nauc_precision_at_5_std": 0.3763353234666901, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5155748885294896, - "nauc_recall_at_100_max": 0.4354612206487773, - "nauc_recall_at_100_std": 0.5786132641876298, - "nauc_recall_at_10_diff1": 0.6016904587793288, - "nauc_recall_at_10_max": 0.5453527162042908, - "nauc_recall_at_10_std": 0.4474004366146903, - "nauc_recall_at_1_diff1": 0.7397486518013686, - "nauc_recall_at_1_max": 0.5581512468677294, - "nauc_recall_at_1_std": 0.2786415604449159, - "nauc_recall_at_20_diff1": 0.5539394639480063, - "nauc_recall_at_20_max": 0.5329757970859269, - "nauc_recall_at_20_std": 0.5029165107845112, - "nauc_recall_at_3_diff1": 0.6362736404675333, - "nauc_recall_at_3_max": 0.5600825936851035, - "nauc_recall_at_3_std": 0.3562218259043451, - "nauc_recall_at_5_diff1": 0.6134324595869594, - "nauc_recall_at_5_max": 0.5514024998817529, - "nauc_recall_at_5_std": 0.376335323466691, - "ndcg_at_1": 0.45455, - "ndcg_at_10": 0.5731, - "ndcg_at_100": 0.61083, - "ndcg_at_1000": 0.62555, - "ndcg_at_20": 0.58795, - "ndcg_at_3": 0.52812, - "ndcg_at_5": 0.55055, - "precision_at_1": 0.45455, - "precision_at_10": 0.0703, - "precision_at_100": 0.00886, - "precision_at_1000": 0.001, - "precision_at_20": 0.03805, - "precision_at_3": 0.19309, - "precision_at_5": 0.12685, - "recall_at_1": 0.45455, - "recall_at_10": 0.70296, - "recall_at_100": 0.88584, - "recall_at_1000": 1.0, - "recall_at_20": 0.7611, - "recall_at_3": 0.57928, - "recall_at_5": 0.63425 - }, - { - "cv_recall_at_1": 0.399, - "cv_recall_at_10": 0.551, - "cv_recall_at_100": 0.688, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.591, - "cv_recall_at_3": 0.464, - "cv_recall_at_5": 0.501, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.46795, - "map_at_1": 0.399, - "map_at_10": 0.44221, - "map_at_100": 0.44732, - "map_at_1000": 0.44807, - "map_at_20": 0.44478, - "map_at_3": 0.427, - "map_at_5": 0.43555, - "mrr_at_1": 0.399, - "mrr_at_10": 0.4422138888888887, - "mrr_at_100": 0.4473152600871951, - "mrr_at_1000": 0.44807039515532976, - "mrr_at_20": 0.44478111955124294, - "mrr_at_3": 0.4269999999999997, - "mrr_at_5": 0.43554999999999966, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5528063877023381, - "nauc_cv_recall_at_100_max": 0.8748190900404679, - "nauc_cv_recall_at_100_std": 0.2906485358588136, - "nauc_cv_recall_at_10_diff1": 0.7000307851045077, - "nauc_cv_recall_at_10_max": 0.794016725167737, - "nauc_cv_recall_at_10_std": 0.20150889183463758, - "nauc_cv_recall_at_1_diff1": 0.82023819183368, - "nauc_cv_recall_at_1_max": 0.6932781064174243, - "nauc_cv_recall_at_1_std": 0.15115792347826534, - "nauc_cv_recall_at_20_diff1": 0.6427090816405981, - "nauc_cv_recall_at_20_max": 0.7924660240956776, - "nauc_cv_recall_at_20_std": 0.21235840006581444, - "nauc_cv_recall_at_3_diff1": 0.7611491536485745, - "nauc_cv_recall_at_3_max": 0.7138353193818688, - "nauc_cv_recall_at_3_std": 0.1391295422965181, - "nauc_cv_recall_at_5_diff1": 0.7232374725402242, - "nauc_cv_recall_at_5_max": 0.726577328195504, - "nauc_cv_recall_at_5_std": 0.14901571626165597, - "nauc_map_at_1000_diff1": 0.7782405397444, - "nauc_map_at_1000_max": 0.7174181757910132, - "nauc_map_at_1000_std": 0.15736686552194565, - "nauc_map_at_100_diff1": 0.7779408427101666, - "nauc_map_at_100_max": 0.7177708130161948, - "nauc_map_at_100_std": 0.1580175379669668, - "nauc_map_at_10_diff1": 0.7823335231299184, - "nauc_map_at_10_max": 0.7165499158253605, - "nauc_map_at_10_std": 0.15553616586649552, - "nauc_map_at_1_diff1": 0.82023819183368, - "nauc_map_at_1_max": 0.6932781064174243, - "nauc_map_at_1_std": 0.15115792347826534, - "nauc_map_at_20_diff1": 0.7792753419134761, - "nauc_map_at_20_max": 0.716237065369837, - "nauc_map_at_20_std": 0.15606851161248672, - "nauc_map_at_3_diff1": 0.793410702588044, - "nauc_map_at_3_max": 0.705150050776552, - "nauc_map_at_3_std": 0.1471582027526783, - "nauc_map_at_5_diff1": 0.7851116354537095, - "nauc_map_at_5_max": 0.7078973387400431, - "nauc_map_at_5_std": 0.14925366356269948, - "nauc_mrr_at_1000_diff1": 0.7782405397444, - "nauc_mrr_at_1000_max": 0.7174181757910132, - "nauc_mrr_at_1000_std": 0.15736686552194565, - "nauc_mrr_at_100_diff1": 0.7779408427101666, - "nauc_mrr_at_100_max": 0.7177708130161948, - "nauc_mrr_at_100_std": 0.1580175379669668, - "nauc_mrr_at_10_diff1": 0.7823335231299184, - "nauc_mrr_at_10_max": 0.7165499158253605, - "nauc_mrr_at_10_std": 0.15553616586649552, - "nauc_mrr_at_1_diff1": 0.82023819183368, - "nauc_mrr_at_1_max": 0.6932781064174243, - "nauc_mrr_at_1_std": 0.15115792347826534, - "nauc_mrr_at_20_diff1": 0.7792753419134761, - "nauc_mrr_at_20_max": 0.716237065369837, - "nauc_mrr_at_20_std": 0.15606851161248672, - "nauc_mrr_at_3_diff1": 0.793410702588044, - "nauc_mrr_at_3_max": 0.705150050776552, - "nauc_mrr_at_3_std": 0.1471582027526783, - "nauc_mrr_at_5_diff1": 0.7851116354537095, - "nauc_mrr_at_5_max": 0.7078973387400431, - "nauc_mrr_at_5_std": 0.14925366356269948, - "nauc_ndcg_at_1000_diff1": 0.7582505770654221, - "nauc_ndcg_at_1000_max": 0.7298384671482888, - "nauc_ndcg_at_1000_std": 0.1645956327091848, - "nauc_ndcg_at_100_diff1": 0.7407448752009863, - "nauc_ndcg_at_100_max": 0.7432691565803868, - "nauc_ndcg_at_100_std": 0.17866226472522742, - "nauc_ndcg_at_10_diff1": 0.763357005473246, - "nauc_ndcg_at_10_max": 0.7333588449518234, - "nauc_ndcg_at_10_std": 0.1649121151550678, - "nauc_ndcg_at_1_diff1": 0.82023819183368, - "nauc_ndcg_at_1_max": 0.6932781064174243, - "nauc_ndcg_at_1_std": 0.15115792347826534, - "nauc_ndcg_at_20_diff1": 0.7510964074157357, - "nauc_ndcg_at_20_max": 0.7323663849211619, - "nauc_ndcg_at_20_std": 0.1670241666816858, - "nauc_ndcg_at_3_diff1": 0.7851539363695622, - "nauc_ndcg_at_3_max": 0.7074845339044545, - "nauc_ndcg_at_3_std": 0.1451676802667441, - "nauc_ndcg_at_5_diff1": 0.7700013695497865, - "nauc_ndcg_at_5_max": 0.7125439307598648, - "nauc_ndcg_at_5_std": 0.14908218748608065, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.5528063877023378, - "nauc_precision_at_100_max": 0.8748190900404674, - "nauc_precision_at_100_std": 0.29064853585881273, - "nauc_precision_at_10_diff1": 0.7000307851045076, - "nauc_precision_at_10_max": 0.7940167251677365, - "nauc_precision_at_10_std": 0.20150889183463738, - "nauc_precision_at_1_diff1": 0.82023819183368, - "nauc_precision_at_1_max": 0.6932781064174243, - "nauc_precision_at_1_std": 0.15115792347826534, - "nauc_precision_at_20_diff1": 0.6427090816405984, - "nauc_precision_at_20_max": 0.7924660240956771, - "nauc_precision_at_20_std": 0.2123584000658143, - "nauc_precision_at_3_diff1": 0.7611491536485748, - "nauc_precision_at_3_max": 0.7138353193818688, - "nauc_precision_at_3_std": 0.13912954229651817, - "nauc_precision_at_5_diff1": 0.7232374725402244, - "nauc_precision_at_5_max": 0.7265773281955042, - "nauc_precision_at_5_std": 0.1490157162616557, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5528063877023381, - "nauc_recall_at_100_max": 0.8748190900404679, - "nauc_recall_at_100_std": 0.2906485358588136, - "nauc_recall_at_10_diff1": 0.7000307851045077, - "nauc_recall_at_10_max": 0.794016725167737, - "nauc_recall_at_10_std": 0.20150889183463758, - "nauc_recall_at_1_diff1": 0.82023819183368, - "nauc_recall_at_1_max": 0.6932781064174243, - "nauc_recall_at_1_std": 0.15115792347826534, - "nauc_recall_at_20_diff1": 0.6427090816405981, - "nauc_recall_at_20_max": 0.7924660240956776, - "nauc_recall_at_20_std": 0.21235840006581444, - "nauc_recall_at_3_diff1": 0.7611491536485745, - "nauc_recall_at_3_max": 0.7138353193818688, - "nauc_recall_at_3_std": 0.1391295422965181, - "nauc_recall_at_5_diff1": 0.7232374725402242, - "nauc_recall_at_5_max": 0.726577328195504, - "nauc_recall_at_5_std": 0.14901571626165597, - "ndcg_at_1": 0.399, - "ndcg_at_10": 0.46795, - "ndcg_at_100": 0.49594, - "ndcg_at_1000": 0.53109, - "ndcg_at_20": 0.4778, - "ndcg_at_3": 0.43648, - "ndcg_at_5": 0.4518, - "precision_at_1": 0.399, - "precision_at_10": 0.0551, - "precision_at_100": 0.00688, - "precision_at_1000": 0.001, - "precision_at_20": 0.02955, - "precision_at_3": 0.15467, - "precision_at_5": 0.1002, - "recall_at_1": 0.399, - "recall_at_10": 0.551, - "recall_at_100": 0.688, - "recall_at_1000": 1.0, - "recall_at_20": 0.591, - "recall_at_3": 0.464, - "recall_at_5": 0.501 - } - ] - }, - "task_name": "WITT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/model_meta.json b/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/model_meta.json deleted file mode 100644 index 1f85ed05ba..0000000000 --- a/results-mieb/google__siglip-large-patch16-384/ce005573a40965dfd21fd937fbdeeebf2439fc35/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "google/siglip-large-patch16-384", "revision": "ce005573a40965dfd21fd937fbdeeebf2439fc35", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json deleted file mode 100644 index 6c5b4b0d05..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreArxivQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", - "evaluation_time": 37.123581647872925, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.408, - "cv_recall_at_10": 0.658, - "cv_recall_at_100": 0.9, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.73, - "cv_recall_at_3": 0.524, - "cv_recall_at_5": 0.58, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.50041, - "map_at_1": 0.408, - "map_at_10": 0.48463, - "map_at_100": 0.49363, - "map_at_1000": 0.49423, - "map_at_20": 0.48953, - "map_at_3": 0.461, - "map_at_5": 0.4739, - "mrr_at_1": 0.408, - "mrr_at_10": 0.48463492063492053, - "mrr_at_100": 0.4936256394439807, - "mrr_at_1000": 0.49422833528767024, - "mrr_at_20": 0.4895304532842458, - "mrr_at_3": 0.46100000000000013, - "mrr_at_5": 0.4739000000000001, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.48634920634920553, - "nauc_cv_recall_at_100_max": 0.591447245564892, - "nauc_cv_recall_at_100_std": 0.4420074696545277, - "nauc_cv_recall_at_10_diff1": 0.5607732158147564, - "nauc_cv_recall_at_10_max": 0.525200263160505, - "nauc_cv_recall_at_10_std": 0.2014632517513726, - "nauc_cv_recall_at_1_diff1": 0.6848492828154529, - "nauc_cv_recall_at_1_max": 0.49441071078203574, - "nauc_cv_recall_at_1_std": 0.11512903125871315, - "nauc_cv_recall_at_20_diff1": 0.5547543690656394, - "nauc_cv_recall_at_20_max": 0.532280956967898, - "nauc_cv_recall_at_20_std": 0.2484735300477695, - "nauc_cv_recall_at_3_diff1": 0.610174677499549, - "nauc_cv_recall_at_3_max": 0.47892341399714783, - "nauc_cv_recall_at_3_std": 0.13839790627407458, - "nauc_cv_recall_at_5_diff1": 0.6229851699219318, - "nauc_cv_recall_at_5_max": 0.5293293089938185, - "nauc_cv_recall_at_5_std": 0.160623385669095, - "nauc_map_at_1000_diff1": 0.6491101252075504, - "nauc_map_at_1000_max": 0.5000671746105353, - "nauc_map_at_1000_std": 0.13430585104690468, - "nauc_map_at_100_diff1": 0.6488183522188637, - "nauc_map_at_100_max": 0.5002268240822477, - "nauc_map_at_100_std": 0.13485377824353093, - "nauc_map_at_10_diff1": 0.6479922280771185, - "nauc_map_at_10_max": 0.499988268241859, - "nauc_map_at_10_std": 0.13238514930345616, - "nauc_map_at_1_diff1": 0.6848492828154529, - "nauc_map_at_1_max": 0.49441071078203574, - "nauc_map_at_1_std": 0.11512903125871315, - "nauc_map_at_20_diff1": 0.6485457083737673, - "nauc_map_at_20_max": 0.500196334435762, - "nauc_map_at_20_std": 0.13466389043096438, - "nauc_map_at_3_diff1": 0.6507960840023341, - "nauc_map_at_3_max": 0.489709383637825, - "nauc_map_at_3_std": 0.12304998513330462, - "nauc_map_at_5_diff1": 0.6546180635941539, - "nauc_map_at_5_max": 0.5013720423294509, - "nauc_map_at_5_std": 0.12821821598924624, - "nauc_mrr_at_1000_diff1": 0.6491101252075504, - "nauc_mrr_at_1000_max": 0.5000671746105353, - "nauc_mrr_at_1000_std": 0.13430585104690468, - "nauc_mrr_at_100_diff1": 0.6488183522188637, - "nauc_mrr_at_100_max": 0.5002268240822477, - "nauc_mrr_at_100_std": 0.13485377824353093, - "nauc_mrr_at_10_diff1": 0.6479922280771185, - "nauc_mrr_at_10_max": 0.499988268241859, - "nauc_mrr_at_10_std": 0.13238514930345616, - "nauc_mrr_at_1_diff1": 0.6848492828154529, - "nauc_mrr_at_1_max": 0.49441071078203574, - "nauc_mrr_at_1_std": 0.11512903125871315, - "nauc_mrr_at_20_diff1": 0.6485457083737673, - "nauc_mrr_at_20_max": 0.500196334435762, - "nauc_mrr_at_20_std": 0.13466389043096438, - "nauc_mrr_at_3_diff1": 0.6507960840023341, - "nauc_mrr_at_3_max": 0.489709383637825, - "nauc_mrr_at_3_std": 0.12304998513330462, - "nauc_mrr_at_5_diff1": 0.6546180635941539, - "nauc_mrr_at_5_max": 0.5013720423294509, - "nauc_mrr_at_5_std": 0.12821821598924624, - "nauc_ndcg_at_1000_diff1": 0.6405236118047709, - "nauc_ndcg_at_1000_max": 0.5029849206775402, - "nauc_ndcg_at_1000_std": 0.14436126589668163, - "nauc_ndcg_at_100_diff1": 0.6334691351131405, - "nauc_ndcg_at_100_max": 0.5070066248549946, - "nauc_ndcg_at_100_std": 0.15797425676621923, - "nauc_ndcg_at_10_diff1": 0.6301087837132273, - "nauc_ndcg_at_10_max": 0.5051015889632752, - "nauc_ndcg_at_10_std": 0.14632520256105425, - "nauc_ndcg_at_1_diff1": 0.6848492828154529, - "nauc_ndcg_at_1_max": 0.49441071078203574, - "nauc_ndcg_at_1_std": 0.11512903125871315, - "nauc_ndcg_at_20_diff1": 0.6314997303399957, - "nauc_ndcg_at_20_max": 0.5058510110615413, - "nauc_ndcg_at_20_std": 0.15408489217326618, - "nauc_ndcg_at_3_diff1": 0.6405250822588264, - "nauc_ndcg_at_3_max": 0.48708846743740086, - "nauc_ndcg_at_3_std": 0.12680957085946576, - "nauc_ndcg_at_5_diff1": 0.6467436522794359, - "nauc_ndcg_at_5_max": 0.5076083996525415, - "nauc_ndcg_at_5_std": 0.13575029438286534, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.48634920634920636, - "nauc_precision_at_100_max": 0.5914472455648909, - "nauc_precision_at_100_std": 0.44200746965452886, - "nauc_precision_at_10_diff1": 0.5607732158147563, - "nauc_precision_at_10_max": 0.5252002631605046, - "nauc_precision_at_10_std": 0.2014632517513726, - "nauc_precision_at_1_diff1": 0.6848492828154529, - "nauc_precision_at_1_max": 0.49441071078203574, - "nauc_precision_at_1_std": 0.11512903125871315, - "nauc_precision_at_20_diff1": 0.5547543690656398, - "nauc_precision_at_20_max": 0.5322809569678982, - "nauc_precision_at_20_std": 0.24847353004776943, - "nauc_precision_at_3_diff1": 0.6101746774995492, - "nauc_precision_at_3_max": 0.4789234139971478, - "nauc_precision_at_3_std": 0.1383979062740747, - "nauc_precision_at_5_diff1": 0.6229851699219321, - "nauc_precision_at_5_max": 0.529329308993818, - "nauc_precision_at_5_std": 0.16062338566909454, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.48634920634920553, - "nauc_recall_at_100_max": 0.591447245564892, - "nauc_recall_at_100_std": 0.4420074696545277, - "nauc_recall_at_10_diff1": 0.5607732158147564, - "nauc_recall_at_10_max": 0.525200263160505, - "nauc_recall_at_10_std": 0.2014632517513726, - "nauc_recall_at_1_diff1": 0.6848492828154529, - "nauc_recall_at_1_max": 0.49441071078203574, - "nauc_recall_at_1_std": 0.11512903125871315, - "nauc_recall_at_20_diff1": 0.5547543690656394, - "nauc_recall_at_20_max": 0.532280956967898, - "nauc_recall_at_20_std": 0.2484735300477695, - "nauc_recall_at_3_diff1": 0.610174677499549, - "nauc_recall_at_3_max": 0.47892341399714783, - "nauc_recall_at_3_std": 0.13839790627407458, - "nauc_recall_at_5_diff1": 0.6229851699219318, - "nauc_recall_at_5_max": 0.5293293089938185, - "nauc_recall_at_5_std": 0.160623385669095, - "ndcg_at_1": 0.408, - "ndcg_at_10": 0.52596, - "ndcg_at_100": 0.57503, - "ndcg_at_1000": 0.58846, - "ndcg_at_20": 0.54404, - "ndcg_at_3": 0.47726, - "ndcg_at_5": 0.50041, - "precision_at_1": 0.408, - "precision_at_10": 0.0658, - "precision_at_100": 0.009, - "precision_at_1000": 0.001, - "precision_at_20": 0.0365, - "precision_at_3": 0.17467, - "precision_at_5": 0.116, - "recall_at_1": 0.408, - "recall_at_10": 0.658, - "recall_at_100": 0.9, - "recall_at_1000": 1.0, - "recall_at_20": 0.73, - "recall_at_3": 0.524, - "recall_at_5": 0.58 - } - ] - }, - "task_name": "VidoreArxivQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json deleted file mode 100644 index 53c2fdfc97..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreDocVQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "b1d89eda849e636676df6ead8002602fb1858600", - "evaluation_time": 42.08629059791565, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.222, - "cv_recall_at_10": 0.45, - "cv_recall_at_100": 0.726, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.532, - "cv_recall_at_3": 0.348, - "cv_recall_at_5": 0.388, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.31278, - "map_at_1": 0.222, - "map_at_10": 0.29608, - "map_at_100": 0.30637, - "map_at_1000": 0.30768, - "map_at_20": 0.3017, - "map_at_3": 0.27867, - "map_at_5": 0.28757, - "mrr_at_1": 0.222, - "mrr_at_10": 0.2960777777777778, - "mrr_at_100": 0.30637161245152283, - "mrr_at_1000": 0.3076778467647497, - "mrr_at_20": 0.30170205427527425, - "mrr_at_3": 0.2786666666666667, - "mrr_at_5": 0.28756666666666664, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.45408426850632555, - "nauc_cv_recall_at_100_max": 0.3222744028682529, - "nauc_cv_recall_at_100_std": 0.49769459949735695, - "nauc_cv_recall_at_10_diff1": 0.4583958619786553, - "nauc_cv_recall_at_10_max": 0.4087396058403719, - "nauc_cv_recall_at_10_std": 0.43819027041183795, - "nauc_cv_recall_at_1_diff1": 0.6080247200265034, - "nauc_cv_recall_at_1_max": 0.30684537908272097, - "nauc_cv_recall_at_1_std": 0.3190134282007699, - "nauc_cv_recall_at_20_diff1": 0.4408917344852856, - "nauc_cv_recall_at_20_max": 0.33116607618092503, - "nauc_cv_recall_at_20_std": 0.4635955898153606, - "nauc_cv_recall_at_3_diff1": 0.5135354143241001, - "nauc_cv_recall_at_3_max": 0.3805383547397707, - "nauc_cv_recall_at_3_std": 0.38879738582793566, - "nauc_cv_recall_at_5_diff1": 0.5147748046338926, - "nauc_cv_recall_at_5_max": 0.4052765679780838, - "nauc_cv_recall_at_5_std": 0.41168260840294335, - "nauc_map_at_1000_diff1": 0.5544641523789734, - "nauc_map_at_1000_max": 0.34660178516191575, - "nauc_map_at_1000_std": 0.36292664896805754, - "nauc_map_at_100_diff1": 0.5541714622401884, - "nauc_map_at_100_max": 0.3463162735362515, - "nauc_map_at_100_std": 0.36320862556883937, - "nauc_map_at_10_diff1": 0.5537212140716464, - "nauc_map_at_10_max": 0.3520368843333082, - "nauc_map_at_10_std": 0.36244548454203784, - "nauc_map_at_1_diff1": 0.6080247200265034, - "nauc_map_at_1_max": 0.30684537908272097, - "nauc_map_at_1_std": 0.3190134282007699, - "nauc_map_at_20_diff1": 0.5530202603716357, - "nauc_map_at_20_max": 0.3461352506565878, - "nauc_map_at_20_std": 0.3634854366460339, - "nauc_map_at_3_diff1": 0.56350963435392, - "nauc_map_at_3_max": 0.3450059374839173, - "nauc_map_at_3_std": 0.3531325707685181, - "nauc_map_at_5_diff1": 0.5631217566464541, - "nauc_map_at_5_max": 0.35100571736628244, - "nauc_map_at_5_std": 0.358638105621845, - "nauc_mrr_at_1000_diff1": 0.5544641523789734, - "nauc_mrr_at_1000_max": 0.34660178516191575, - "nauc_mrr_at_1000_std": 0.36292664896805754, - "nauc_mrr_at_100_diff1": 0.5541714622401884, - "nauc_mrr_at_100_max": 0.3463162735362515, - "nauc_mrr_at_100_std": 0.36320862556883937, - "nauc_mrr_at_10_diff1": 0.5537212140716464, - "nauc_mrr_at_10_max": 0.3520368843333082, - "nauc_mrr_at_10_std": 0.36244548454203784, - "nauc_mrr_at_1_diff1": 0.6080247200265034, - "nauc_mrr_at_1_max": 0.30684537908272097, - "nauc_mrr_at_1_std": 0.3190134282007699, - "nauc_mrr_at_20_diff1": 0.5530202603716357, - "nauc_mrr_at_20_max": 0.3461352506565878, - "nauc_mrr_at_20_std": 0.3634854366460339, - "nauc_mrr_at_3_diff1": 0.56350963435392, - "nauc_mrr_at_3_max": 0.3450059374839173, - "nauc_mrr_at_3_std": 0.3531325707685181, - "nauc_mrr_at_5_diff1": 0.5631217566464541, - "nauc_mrr_at_5_max": 0.35100571736628244, - "nauc_mrr_at_5_std": 0.358638105621845, - "nauc_ndcg_at_1000_diff1": 0.5409245673190076, - "nauc_ndcg_at_1000_max": 0.3502701629410077, - "nauc_ndcg_at_1000_std": 0.37632942100540007, - "nauc_ndcg_at_100_diff1": 0.5327132079954523, - "nauc_ndcg_at_100_max": 0.3465358980202094, - "nauc_ndcg_at_100_std": 0.38719600780661023, - "nauc_ndcg_at_10_diff1": 0.5292537182061523, - "nauc_ndcg_at_10_max": 0.3672590009334541, - "nauc_ndcg_at_10_std": 0.38183727402292117, - "nauc_ndcg_at_1_diff1": 0.6080247200265034, - "nauc_ndcg_at_1_max": 0.30684537908272097, - "nauc_ndcg_at_1_std": 0.3190134282007699, - "nauc_ndcg_at_20_diff1": 0.525953721140863, - "nauc_ndcg_at_20_max": 0.346494074796129, - "nauc_ndcg_at_20_std": 0.38703811605768795, - "nauc_ndcg_at_3_diff1": 0.5499370707195624, - "nauc_ndcg_at_3_max": 0.3547957954684597, - "nauc_ndcg_at_3_std": 0.36286921346909046, - "nauc_ndcg_at_5_diff1": 0.5497836024111007, - "nauc_ndcg_at_5_max": 0.3656060271257966, - "nauc_ndcg_at_5_std": 0.37283779755807595, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.4540842685063265, - "nauc_precision_at_100_max": 0.3222744028682524, - "nauc_precision_at_100_std": 0.4976945994973571, - "nauc_precision_at_10_diff1": 0.45839586197865506, - "nauc_precision_at_10_max": 0.4087396058403718, - "nauc_precision_at_10_std": 0.4381902704118379, - "nauc_precision_at_1_diff1": 0.6080247200265034, - "nauc_precision_at_1_max": 0.30684537908272097, - "nauc_precision_at_1_std": 0.3190134282007699, - "nauc_precision_at_20_diff1": 0.4408917344852855, - "nauc_precision_at_20_max": 0.33116607618092536, - "nauc_precision_at_20_std": 0.4635955898153608, - "nauc_precision_at_3_diff1": 0.5135354143241001, - "nauc_precision_at_3_max": 0.38053835473977055, - "nauc_precision_at_3_std": 0.3887973858279356, - "nauc_precision_at_5_diff1": 0.5147748046338925, - "nauc_precision_at_5_max": 0.405276567978084, - "nauc_precision_at_5_std": 0.41168260840294396, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.45408426850632555, - "nauc_recall_at_100_max": 0.3222744028682529, - "nauc_recall_at_100_std": 0.49769459949735695, - "nauc_recall_at_10_diff1": 0.4583958619786553, - "nauc_recall_at_10_max": 0.4087396058403719, - "nauc_recall_at_10_std": 0.43819027041183795, - "nauc_recall_at_1_diff1": 0.6080247200265034, - "nauc_recall_at_1_max": 0.30684537908272097, - "nauc_recall_at_1_std": 0.3190134282007699, - "nauc_recall_at_20_diff1": 0.4408917344852856, - "nauc_recall_at_20_max": 0.33116607618092503, - "nauc_recall_at_20_std": 0.4635955898153606, - "nauc_recall_at_3_diff1": 0.5135354143241001, - "nauc_recall_at_3_max": 0.3805383547397707, - "nauc_recall_at_3_std": 0.38879738582793566, - "nauc_recall_at_5_diff1": 0.5147748046338926, - "nauc_recall_at_5_max": 0.4052765679780838, - "nauc_recall_at_5_std": 0.41168260840294335, - "ndcg_at_1": 0.222, - "ndcg_at_10": 0.33307, - "ndcg_at_100": 0.38911, - "ndcg_at_1000": 0.42415, - "ndcg_at_20": 0.35373, - "ndcg_at_3": 0.29652, - "ndcg_at_5": 0.31278, - "precision_at_1": 0.222, - "precision_at_10": 0.045, - "precision_at_100": 0.00726, - "precision_at_1000": 0.001, - "precision_at_20": 0.0266, - "precision_at_3": 0.116, - "precision_at_5": 0.0776, - "recall_at_1": 0.222, - "recall_at_10": 0.45, - "recall_at_100": 0.726, - "recall_at_1000": 1.0, - "recall_at_20": 0.532, - "recall_at_3": 0.348, - "recall_at_5": 0.388 - } - ] - }, - "task_name": "VidoreDocVQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json deleted file mode 100644 index 031ebfff4e..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreInfoVQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", - "evaluation_time": 41.904603242874146, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.578, - "cv_recall_at_10": 0.842, - "cv_recall_at_100": 0.952, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.886, - "cv_recall_at_3": 0.726, - "cv_recall_at_5": 0.792, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.69273, - "map_at_1": 0.578, - "map_at_10": 0.6663, - "map_at_100": 0.67068, - "map_at_1000": 0.67097, - "map_at_20": 0.66919, - "map_at_3": 0.644, - "map_at_5": 0.6596, - "mrr_at_1": 0.578, - "mrr_at_10": 0.6663031746031746, - "mrr_at_100": 0.6706831128053965, - "mrr_at_1000": 0.6709717828298823, - "mrr_at_20": 0.6691926300823358, - "mrr_at_3": 0.644, - "mrr_at_5": 0.6595999999999997, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.7016612200435727, - "nauc_cv_recall_at_100_max": 0.3388966697790251, - "nauc_cv_recall_at_100_std": 0.7045790538437611, - "nauc_cv_recall_at_10_diff1": 0.6544153248333311, - "nauc_cv_recall_at_10_max": 0.5501061465691686, - "nauc_cv_recall_at_10_std": 0.3852996312803394, - "nauc_cv_recall_at_1_diff1": 0.7563944784370051, - "nauc_cv_recall_at_1_max": 0.5450704673985117, - "nauc_cv_recall_at_1_std": 0.09203473708290744, - "nauc_cv_recall_at_20_diff1": 0.6634424892203701, - "nauc_cv_recall_at_20_max": 0.5504020510430014, - "nauc_cv_recall_at_20_std": 0.47475319226863366, - "nauc_cv_recall_at_3_diff1": 0.6899187768752989, - "nauc_cv_recall_at_3_max": 0.5278337780988896, - "nauc_cv_recall_at_3_std": 0.15949409374647983, - "nauc_cv_recall_at_5_diff1": 0.6596313739170878, - "nauc_cv_recall_at_5_max": 0.5408405527453144, - "nauc_cv_recall_at_5_std": 0.27152741438455646, - "nauc_map_at_1000_diff1": 0.728329825559897, - "nauc_map_at_1000_max": 0.5341727992904414, - "nauc_map_at_1000_std": 0.1337047475778167, - "nauc_map_at_100_diff1": 0.7282640631192274, - "nauc_map_at_100_max": 0.5340833390642327, - "nauc_map_at_100_std": 0.13428744788967067, - "nauc_map_at_10_diff1": 0.7269917994209238, - "nauc_map_at_10_max": 0.5352397762689725, - "nauc_map_at_10_std": 0.1339785516273345, - "nauc_map_at_1_diff1": 0.7563944784370051, - "nauc_map_at_1_max": 0.5450704673985117, - "nauc_map_at_1_std": 0.09203473708290744, - "nauc_map_at_20_diff1": 0.7277237287287605, - "nauc_map_at_20_max": 0.5350192237473728, - "nauc_map_at_20_std": 0.13471990708577683, - "nauc_map_at_3_diff1": 0.7292858632329362, - "nauc_map_at_3_max": 0.5331904060611592, - "nauc_map_at_3_std": 0.11234018428495053, - "nauc_map_at_5_diff1": 0.7266994608094846, - "nauc_map_at_5_max": 0.5354412003777747, - "nauc_map_at_5_std": 0.12957191027012394, - "nauc_mrr_at_1000_diff1": 0.728329825559897, - "nauc_mrr_at_1000_max": 0.5341727992904414, - "nauc_mrr_at_1000_std": 0.1337047475778167, - "nauc_mrr_at_100_diff1": 0.7282640631192274, - "nauc_mrr_at_100_max": 0.5340833390642327, - "nauc_mrr_at_100_std": 0.13428744788967067, - "nauc_mrr_at_10_diff1": 0.7269917994209238, - "nauc_mrr_at_10_max": 0.5352397762689725, - "nauc_mrr_at_10_std": 0.1339785516273345, - "nauc_mrr_at_1_diff1": 0.7563944784370051, - "nauc_mrr_at_1_max": 0.5450704673985117, - "nauc_mrr_at_1_std": 0.09203473708290744, - "nauc_mrr_at_20_diff1": 0.7277237287287605, - "nauc_mrr_at_20_max": 0.5350192237473728, - "nauc_mrr_at_20_std": 0.13471990708577683, - "nauc_mrr_at_3_diff1": 0.7292858632329362, - "nauc_mrr_at_3_max": 0.5331904060611592, - "nauc_mrr_at_3_std": 0.11234018428495053, - "nauc_mrr_at_5_diff1": 0.7266994608094846, - "nauc_mrr_at_5_max": 0.5354412003777747, - "nauc_mrr_at_5_std": 0.12957191027012394, - "nauc_ndcg_at_1000_diff1": 0.7236097326031249, - "nauc_ndcg_at_1000_max": 0.5320654575443786, - "nauc_ndcg_at_1000_std": 0.15264521875599046, - "nauc_ndcg_at_100_diff1": 0.7228017117153069, - "nauc_ndcg_at_100_max": 0.5271870696372063, - "nauc_ndcg_at_100_std": 0.16853775142739957, - "nauc_ndcg_at_10_diff1": 0.7151002125670103, - "nauc_ndcg_at_10_max": 0.5367706024907527, - "nauc_ndcg_at_10_std": 0.1710856158422823, - "nauc_ndcg_at_1_diff1": 0.7563944784370051, - "nauc_ndcg_at_1_max": 0.5450704673985117, - "nauc_ndcg_at_1_std": 0.09203473708290744, - "nauc_ndcg_at_20_diff1": 0.7183893546557805, - "nauc_ndcg_at_20_max": 0.5361165591354897, - "nauc_ndcg_at_20_std": 0.1739805415327144, - "nauc_ndcg_at_3_diff1": 0.7203283060605159, - "nauc_ndcg_at_3_max": 0.5317435162233147, - "nauc_ndcg_at_3_std": 0.12273087229898909, - "nauc_ndcg_at_5_diff1": 0.713888248116884, - "nauc_ndcg_at_5_max": 0.536055499438798, - "nauc_ndcg_at_5_std": 0.15592804987391928, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.701661220043572, - "nauc_precision_at_100_max": 0.3388966697790239, - "nauc_precision_at_100_std": 0.7045790538437579, - "nauc_precision_at_10_diff1": 0.6544153248333306, - "nauc_precision_at_10_max": 0.5501061465691688, - "nauc_precision_at_10_std": 0.3852996312803368, - "nauc_precision_at_1_diff1": 0.7563944784370051, - "nauc_precision_at_1_max": 0.5450704673985117, - "nauc_precision_at_1_std": 0.09203473708290744, - "nauc_precision_at_20_diff1": 0.6634424892203703, - "nauc_precision_at_20_max": 0.5504020510430019, - "nauc_precision_at_20_std": 0.47475319226863155, - "nauc_precision_at_3_diff1": 0.6899187768752989, - "nauc_precision_at_3_max": 0.5278337780988899, - "nauc_precision_at_3_std": 0.15949409374648, - "nauc_precision_at_5_diff1": 0.6596313739170871, - "nauc_precision_at_5_max": 0.540840552745314, - "nauc_precision_at_5_std": 0.27152741438455674, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.7016612200435727, - "nauc_recall_at_100_max": 0.3388966697790251, - "nauc_recall_at_100_std": 0.7045790538437611, - "nauc_recall_at_10_diff1": 0.6544153248333311, - "nauc_recall_at_10_max": 0.5501061465691686, - "nauc_recall_at_10_std": 0.3852996312803394, - "nauc_recall_at_1_diff1": 0.7563944784370051, - "nauc_recall_at_1_max": 0.5450704673985117, - "nauc_recall_at_1_std": 0.09203473708290744, - "nauc_recall_at_20_diff1": 0.6634424892203701, - "nauc_recall_at_20_max": 0.5504020510430014, - "nauc_recall_at_20_std": 0.47475319226863366, - "nauc_recall_at_3_diff1": 0.6899187768752989, - "nauc_recall_at_3_max": 0.5278337780988896, - "nauc_recall_at_3_std": 0.15949409374647983, - "nauc_recall_at_5_diff1": 0.6596313739170878, - "nauc_recall_at_5_max": 0.5408405527453144, - "nauc_recall_at_5_std": 0.27152741438455646, - "ndcg_at_1": 0.578, - "ndcg_at_10": 0.70893, - "ndcg_at_100": 0.73168, - "ndcg_at_1000": 0.73811, - "ndcg_at_20": 0.71984, - "ndcg_at_3": 0.66509, - "ndcg_at_5": 0.69273, - "precision_at_1": 0.578, - "precision_at_10": 0.0842, - "precision_at_100": 0.00952, - "precision_at_1000": 0.001, - "precision_at_20": 0.0443, - "precision_at_3": 0.242, - "precision_at_5": 0.1584, - "recall_at_1": 0.578, - "recall_at_10": 0.842, - "recall_at_100": 0.952, - "recall_at_1000": 1.0, - "recall_at_20": 0.886, - "recall_at_3": 0.726, - "recall_at_5": 0.792 - } - ] - }, - "task_name": "VidoreInfoVQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json deleted file mode 100644 index 374bde6d4e..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreShiftProjectRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", - "evaluation_time": 76.06725120544434, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.14, - "cv_recall_at_10": 0.43, - "cv_recall_at_100": 0.77, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.54, - "cv_recall_at_3": 0.29, - "cv_recall_at_5": 0.35, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.25044, - "map_at_1": 0.14, - "map_at_10": 0.22716, - "map_at_100": 0.24058, - "map_at_1000": 0.24155, - "map_at_20": 0.23522, - "map_at_3": 0.20333, - "map_at_5": 0.21733, - "mrr_at_1": 0.14, - "mrr_at_10": 0.22716269841269843, - "mrr_at_100": 0.24058436473035777, - "mrr_at_1000": 0.2415510139478348, - "mrr_at_20": 0.23522009934509933, - "mrr_at_3": 0.2033333333333333, - "mrr_at_5": 0.21733333333333335, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.07011929007855676, - "nauc_cv_recall_at_100_max": 0.3579597591817545, - "nauc_cv_recall_at_100_std": 0.31774132181464143, - "nauc_cv_recall_at_10_diff1": 0.2517589834961177, - "nauc_cv_recall_at_10_max": 0.3860920003641519, - "nauc_cv_recall_at_10_std": 0.28466270434771296, - "nauc_cv_recall_at_1_diff1": 0.32462132462132465, - "nauc_cv_recall_at_1_max": 0.07117399974542814, - "nauc_cv_recall_at_1_std": 0.05664219949934226, - "nauc_cv_recall_at_20_diff1": 0.23868467116779174, - "nauc_cv_recall_at_20_max": 0.3474618654663666, - "nauc_cv_recall_at_20_std": 0.2975049317885025, - "nauc_cv_recall_at_3_diff1": 0.3201928491957749, - "nauc_cv_recall_at_3_max": 0.3486257434446383, - "nauc_cv_recall_at_3_std": 0.18352265703336387, - "nauc_cv_recall_at_5_diff1": 0.28013712176148714, - "nauc_cv_recall_at_5_max": 0.3316632605972707, - "nauc_cv_recall_at_5_std": 0.14300217548948513, - "nauc_map_at_1000_diff1": 0.31781193581545547, - "nauc_map_at_1000_max": 0.20583970548908834, - "nauc_map_at_1000_std": 0.11837714875609179, - "nauc_map_at_100_diff1": 0.3162574307400726, - "nauc_map_at_100_max": 0.20600852978387757, - "nauc_map_at_100_std": 0.11890326588085454, - "nauc_map_at_10_diff1": 0.318533633385182, - "nauc_map_at_10_max": 0.21124248494920406, - "nauc_map_at_10_std": 0.12349472262933363, - "nauc_map_at_1_diff1": 0.32462132462132465, - "nauc_map_at_1_max": 0.07117399974542814, - "nauc_map_at_1_std": 0.05664219949934226, - "nauc_map_at_20_diff1": 0.32095854907465377, - "nauc_map_at_20_max": 0.20707743553081426, - "nauc_map_at_20_std": 0.12254998915538791, - "nauc_map_at_3_diff1": 0.3361859443214354, - "nauc_map_at_3_max": 0.20457404361165038, - "nauc_map_at_3_std": 0.11004513709039367, - "nauc_map_at_5_diff1": 0.3256838023956137, - "nauc_map_at_5_max": 0.20191202664591607, - "nauc_map_at_5_std": 0.09959327195117011, - "nauc_mrr_at_1000_diff1": 0.31781193581545547, - "nauc_mrr_at_1000_max": 0.20583970548908834, - "nauc_mrr_at_1000_std": 0.11837714875609179, - "nauc_mrr_at_100_diff1": 0.3162574307400726, - "nauc_mrr_at_100_max": 0.20600852978387757, - "nauc_mrr_at_100_std": 0.11890326588085454, - "nauc_mrr_at_10_diff1": 0.318533633385182, - "nauc_mrr_at_10_max": 0.21124248494920406, - "nauc_mrr_at_10_std": 0.12349472262933363, - "nauc_mrr_at_1_diff1": 0.32462132462132465, - "nauc_mrr_at_1_max": 0.07117399974542814, - "nauc_mrr_at_1_std": 0.05664219949934226, - "nauc_mrr_at_20_diff1": 0.32095854907465377, - "nauc_mrr_at_20_max": 0.20707743553081426, - "nauc_mrr_at_20_std": 0.12254998915538791, - "nauc_mrr_at_3_diff1": 0.3361859443214354, - "nauc_mrr_at_3_max": 0.20457404361165038, - "nauc_mrr_at_3_std": 0.11004513709039367, - "nauc_mrr_at_5_diff1": 0.3256838023956137, - "nauc_mrr_at_5_max": 0.20191202664591607, - "nauc_mrr_at_5_std": 0.09959327195117011, - "nauc_ndcg_at_1000_diff1": 0.30331892551505446, - "nauc_ndcg_at_1000_max": 0.23206326096932686, - "nauc_ndcg_at_1000_std": 0.13914943758556678, - "nauc_ndcg_at_100_diff1": 0.277090040797122, - "nauc_ndcg_at_100_max": 0.24263312120508812, - "nauc_ndcg_at_100_std": 0.15521452351822168, - "nauc_ndcg_at_10_diff1": 0.2998416223952832, - "nauc_ndcg_at_10_max": 0.26258288066776686, - "nauc_ndcg_at_10_std": 0.16778069464708717, - "nauc_ndcg_at_1_diff1": 0.32462132462132465, - "nauc_ndcg_at_1_max": 0.07117399974542814, - "nauc_ndcg_at_1_std": 0.05664219949934226, - "nauc_ndcg_at_20_diff1": 0.3006714776078098, - "nauc_ndcg_at_20_max": 0.24906392668652022, - "nauc_ndcg_at_20_std": 0.16791706808815687, - "nauc_ndcg_at_3_diff1": 0.33193765983303775, - "nauc_ndcg_at_3_max": 0.24710745275143542, - "nauc_ndcg_at_3_std": 0.13147738828592404, - "nauc_ndcg_at_5_diff1": 0.31314624561950827, - "nauc_ndcg_at_5_max": 0.24124707243552312, - "nauc_ndcg_at_5_std": 0.11299960549335761, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.07011929007855745, - "nauc_precision_at_100_max": 0.3579597591817549, - "nauc_precision_at_100_std": 0.3177413218146418, - "nauc_precision_at_10_diff1": 0.2517589834961181, - "nauc_precision_at_10_max": 0.3860920003641523, - "nauc_precision_at_10_std": 0.2846627043477133, - "nauc_precision_at_1_diff1": 0.32462132462132465, - "nauc_precision_at_1_max": 0.07117399974542814, - "nauc_precision_at_1_std": 0.05664219949934226, - "nauc_precision_at_20_diff1": 0.2386846711677925, - "nauc_precision_at_20_max": 0.34746186546636687, - "nauc_precision_at_20_std": 0.29750493178850307, - "nauc_precision_at_3_diff1": 0.32019284919577484, - "nauc_precision_at_3_max": 0.3486257434446383, - "nauc_precision_at_3_std": 0.1835226570333639, - "nauc_precision_at_5_diff1": 0.2801371217614872, - "nauc_precision_at_5_max": 0.3316632605972707, - "nauc_precision_at_5_std": 0.1430021754894849, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.07011929007855676, - "nauc_recall_at_100_max": 0.3579597591817545, - "nauc_recall_at_100_std": 0.31774132181464143, - "nauc_recall_at_10_diff1": 0.2517589834961177, - "nauc_recall_at_10_max": 0.3860920003641519, - "nauc_recall_at_10_std": 0.28466270434771296, - "nauc_recall_at_1_diff1": 0.32462132462132465, - "nauc_recall_at_1_max": 0.07117399974542814, - "nauc_recall_at_1_std": 0.05664219949934226, - "nauc_recall_at_20_diff1": 0.23868467116779174, - "nauc_recall_at_20_max": 0.3474618654663666, - "nauc_recall_at_20_std": 0.2975049317885025, - "nauc_recall_at_3_diff1": 0.3201928491957749, - "nauc_recall_at_3_max": 0.3486257434446383, - "nauc_recall_at_3_std": 0.18352265703336387, - "nauc_recall_at_5_diff1": 0.28013712176148714, - "nauc_recall_at_5_max": 0.3316632605972707, - "nauc_recall_at_5_std": 0.14300217548948513, - "ndcg_at_1": 0.14, - "ndcg_at_10": 0.27548, - "ndcg_at_100": 0.34536, - "ndcg_at_1000": 0.37396, - "ndcg_at_20": 0.3038, - "ndcg_at_3": 0.22547, - "ndcg_at_5": 0.25044, - "precision_at_1": 0.14, - "precision_at_10": 0.043, - "precision_at_100": 0.0077, - "precision_at_1000": 0.001, - "precision_at_20": 0.027, - "precision_at_3": 0.09667, - "precision_at_5": 0.07, - "recall_at_1": 0.14, - "recall_at_10": 0.43, - "recall_at_100": 0.77, - "recall_at_1000": 1.0, - "recall_at_20": 0.54, - "recall_at_3": 0.29, - "recall_at_5": 0.35 - } - ] - }, - "task_name": "VidoreShiftProjectRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json deleted file mode 100644 index 1836f819bb..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAAIRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", - "evaluation_time": 75.31401538848877, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.39, - "cv_recall_at_10": 0.85, - "cv_recall_at_100": 0.95, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.89, - "cv_recall_at_3": 0.67, - "cv_recall_at_5": 0.76, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.58256, - "map_at_1": 0.36, - "map_at_10": 0.53621, - "map_at_100": 0.53964, - "map_at_1000": 0.53983, - "map_at_20": 0.53815, - "map_at_3": 0.50167, - "map_at_5": 0.52267, - "mrr_at_1": 0.39, - "mrr_at_10": 0.546825396825397, - "mrr_at_100": 0.5511443104244275, - "mrr_at_1000": 0.5513394682372231, - "mrr_at_20": 0.5496709956709959, - "mrr_at_3": 0.5133333333333333, - "mrr_at_5": 0.5348333333333334, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.4458450046685341, - "nauc_cv_recall_at_100_max": 0.7343604108309969, - "nauc_cv_recall_at_100_std": 0.16321195144724712, - "nauc_cv_recall_at_10_diff1": -0.5618288317604945, - "nauc_cv_recall_at_10_max": 0.6819720143182559, - "nauc_cv_recall_at_10_std": 0.1256752359258052, - "nauc_cv_recall_at_1_diff1": 0.1994748619249027, - "nauc_cv_recall_at_1_max": 0.14802167811364178, - "nauc_cv_recall_at_1_std": -0.07441180655258509, - "nauc_cv_recall_at_20_diff1": -0.6063482518683952, - "nauc_cv_recall_at_20_max": 0.675972854565759, - "nauc_cv_recall_at_20_std": 0.21076368009621177, - "nauc_cv_recall_at_3_diff1": -0.23034991789541448, - "nauc_cv_recall_at_3_max": 0.37395676389429633, - "nauc_cv_recall_at_3_std": -0.050566267711737146, - "nauc_cv_recall_at_5_diff1": -0.2816258544605003, - "nauc_cv_recall_at_5_max": 0.5499264515012549, - "nauc_cv_recall_at_5_std": -0.1023838366358055, - "nauc_map_at_1000_diff1": 0.06088734314901716, - "nauc_map_at_1000_max": 0.2936805589234933, - "nauc_map_at_1000_std": -0.047660425251865916, - "nauc_map_at_100_diff1": 0.06058775434305587, - "nauc_map_at_100_max": 0.2939707353087287, - "nauc_map_at_100_std": -0.047454525599785205, - "nauc_map_at_10_diff1": 0.05340004843355785, - "nauc_map_at_10_max": 0.2973404838090291, - "nauc_map_at_10_std": -0.04463528240739384, - "nauc_map_at_1_diff1": 0.2620715897758883, - "nauc_map_at_1_max": 0.2259486695008659, - "nauc_map_at_1_std": -0.07092058993334362, - "nauc_map_at_20_diff1": 0.055767839463411815, - "nauc_map_at_20_max": 0.29547949618137764, - "nauc_map_at_20_std": -0.04665077114633524, - "nauc_map_at_3_diff1": 0.05175393143339162, - "nauc_map_at_3_max": 0.26573236127010896, - "nauc_map_at_3_std": -0.06059167319395071, - "nauc_map_at_5_diff1": 0.05975930657081366, - "nauc_map_at_5_max": 0.2964074299438267, - "nauc_map_at_5_std": -0.0660526761176348, - "nauc_mrr_at_1000_diff1": 0.026947462345755967, - "nauc_mrr_at_1000_max": 0.2523382034693512, - "nauc_mrr_at_1000_std": -0.04482570085671642, - "nauc_mrr_at_100_diff1": 0.02667279807328532, - "nauc_mrr_at_100_max": 0.25265291128513884, - "nauc_mrr_at_100_std": -0.04462221468978824, - "nauc_mrr_at_10_diff1": 0.018220797258613936, - "nauc_mrr_at_10_max": 0.2578423858447704, - "nauc_mrr_at_10_std": -0.04528897392769788, - "nauc_mrr_at_1_diff1": 0.1994748619249027, - "nauc_mrr_at_1_max": 0.14802167811364178, - "nauc_mrr_at_1_std": -0.07441180655258509, - "nauc_mrr_at_20_diff1": 0.022114085998288655, - "nauc_mrr_at_20_max": 0.2542766239097509, - "nauc_mrr_at_20_std": -0.04391458858168538, - "nauc_mrr_at_3_diff1": 0.025988536361080748, - "nauc_mrr_at_3_max": 0.23390397415678407, - "nauc_mrr_at_3_std": -0.05373521185814806, - "nauc_mrr_at_5_diff1": 0.027160692501750364, - "nauc_mrr_at_5_max": 0.2576516650528352, - "nauc_mrr_at_5_std": -0.06194421533050859, - "nauc_ndcg_at_1000_diff1": 0.025484026028548535, - "nauc_ndcg_at_1000_max": 0.3155648907709535, - "nauc_ndcg_at_1000_std": -0.03736254089983462, - "nauc_ndcg_at_100_diff1": 0.013805477940072674, - "nauc_ndcg_at_100_max": 0.32603093853073645, - "nauc_ndcg_at_100_std": -0.03205847590051636, - "nauc_ndcg_at_10_diff1": -0.02877794636442936, - "nauc_ndcg_at_10_max": 0.3461160999719953, - "nauc_ndcg_at_10_std": -0.014302025640523356, - "nauc_ndcg_at_1_diff1": 0.2620715897758883, - "nauc_ndcg_at_1_max": 0.2259486695008659, - "nauc_ndcg_at_1_std": -0.07092058993334362, - "nauc_ndcg_at_20_diff1": -0.020531864489671144, - "nauc_ndcg_at_20_max": 0.338473683828385, - "nauc_ndcg_at_20_std": -0.02162764134437804, - "nauc_ndcg_at_3_diff1": -0.017610177016920022, - "nauc_ndcg_at_3_max": 0.2829562771372745, - "nauc_ndcg_at_3_std": -0.059480770234221464, - "nauc_ndcg_at_5_diff1": -0.007820671311857869, - "nauc_ndcg_at_5_max": 0.3432814823079952, - "nauc_ndcg_at_5_std": -0.07202897142385868, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.4458450046685348, - "nauc_precision_at_100_max": 0.734360410831003, - "nauc_precision_at_100_std": 0.16321195144724487, - "nauc_precision_at_10_diff1": -0.5562560620756525, - "nauc_precision_at_10_max": 0.6723707911874733, - "nauc_precision_at_10_std": 0.21923929610641618, - "nauc_precision_at_1_diff1": 0.2620715897758883, - "nauc_precision_at_1_max": 0.2259486695008659, - "nauc_precision_at_1_std": -0.07092058993334362, - "nauc_precision_at_20_diff1": -0.6063482518683938, - "nauc_precision_at_20_max": 0.6759728545657586, - "nauc_precision_at_20_std": 0.21076368009621038, - "nauc_precision_at_3_diff1": -0.24954715511841657, - "nauc_precision_at_3_max": 0.342147585109445, - "nauc_precision_at_3_std": -0.05666062873491173, - "nauc_precision_at_5_diff1": -0.2816258544604995, - "nauc_precision_at_5_max": 0.5499264515012546, - "nauc_precision_at_5_std": -0.10238383663580512, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.4458450046685341, - "nauc_recall_at_100_max": 0.7343604108309969, - "nauc_recall_at_100_std": 0.16321195144724712, - "nauc_recall_at_10_diff1": -0.5562560620756545, - "nauc_recall_at_10_max": 0.6723707911874733, - "nauc_recall_at_10_std": 0.21923929610641552, - "nauc_recall_at_1_diff1": 0.2620715897758883, - "nauc_recall_at_1_max": 0.2259486695008659, - "nauc_recall_at_1_std": -0.07092058993334362, - "nauc_recall_at_20_diff1": -0.6063482518683952, - "nauc_recall_at_20_max": 0.675972854565759, - "nauc_recall_at_20_std": 0.21076368009621177, - "nauc_recall_at_3_diff1": -0.24954715511841757, - "nauc_recall_at_3_max": 0.3421475851094447, - "nauc_recall_at_3_std": -0.05666062873491272, - "nauc_recall_at_5_diff1": -0.2816258544605003, - "nauc_recall_at_5_max": 0.5499264515012549, - "nauc_recall_at_5_std": -0.1023838366358055, - "ndcg_at_1": 0.36, - "ndcg_at_10": 0.61511, - "ndcg_at_100": 0.63354, - "ndcg_at_1000": 0.63964, - "ndcg_at_20": 0.62252, - "ndcg_at_3": 0.54511, - "ndcg_at_5": 0.58256, - "precision_at_1": 0.36, - "precision_at_10": 0.086, - "precision_at_100": 0.0095, - "precision_at_1000": 0.001, - "precision_at_20": 0.0445, - "precision_at_3": 0.22333, - "precision_at_5": 0.152, - "recall_at_1": 0.36, - "recall_at_10": 0.86, - "recall_at_100": 0.95, - "recall_at_1000": 1.0, - "recall_at_20": 0.89, - "recall_at_3": 0.67, - "recall_at_5": 0.76 - } - ] - }, - "task_name": "VidoreSyntheticDocQAAIRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json deleted file mode 100644 index d2071daf29..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAEnergyRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", - "evaluation_time": 74.29262661933899, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.47, - "cv_recall_at_10": 0.87, - "cv_recall_at_100": 0.98, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.92, - "cv_recall_at_3": 0.76, - "cv_recall_at_5": 0.83, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.65462, - "map_at_1": 0.41, - "map_at_10": 0.59626, - "map_at_100": 0.60145, - "map_at_1000": 0.60156, - "map_at_20": 0.59975, - "map_at_3": 0.575, - "map_at_5": 0.5915, - "mrr_at_1": 0.47, - "mrr_at_10": 0.6217619047619047, - "mrr_at_100": 0.6269392738934335, - "mrr_at_1000": 0.6270515118502697, - "mrr_at_20": 0.6252543290043291, - "mrr_at_3": 0.5983333333333334, - "mrr_at_5": 0.6153333333333333, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.367413632119516, - "nauc_cv_recall_at_100_max": 0.35807656395891974, - "nauc_cv_recall_at_100_std": -0.0242763772175541, - "nauc_cv_recall_at_10_diff1": 0.01462637932311375, - "nauc_cv_recall_at_10_max": 0.6539657853810261, - "nauc_cv_recall_at_10_std": 0.2829741538917277, - "nauc_cv_recall_at_1_diff1": 0.3496222260314613, - "nauc_cv_recall_at_1_max": 0.1055525521169845, - "nauc_cv_recall_at_1_std": -0.14339587811020701, - "nauc_cv_recall_at_20_diff1": -0.06349206349206456, - "nauc_cv_recall_at_20_max": 0.6151377217553676, - "nauc_cv_recall_at_20_std": 0.36496265172735837, - "nauc_cv_recall_at_3_diff1": 0.08475382884831636, - "nauc_cv_recall_at_3_max": 0.4612572466903178, - "nauc_cv_recall_at_3_std": -0.15386778575754947, - "nauc_cv_recall_at_5_diff1": 0.028588843095141415, - "nauc_cv_recall_at_5_max": 0.44630521855227356, - "nauc_cv_recall_at_5_std": -0.13774888256806125, - "nauc_map_at_1000_diff1": 0.3657009430365933, - "nauc_map_at_1000_max": 0.2996454074298663, - "nauc_map_at_1000_std": -0.14999022266095388, - "nauc_map_at_100_diff1": 0.3654096405697197, - "nauc_map_at_100_max": 0.29966947866950855, - "nauc_map_at_100_std": -0.14994646671270026, - "nauc_map_at_10_diff1": 0.3609021693276199, - "nauc_map_at_10_max": 0.3068046720385847, - "nauc_map_at_10_std": -0.1420792617200844, - "nauc_map_at_1_diff1": 0.499334771432631, - "nauc_map_at_1_max": 0.2027203327434544, - "nauc_map_at_1_std": -0.2031595127685132, - "nauc_map_at_20_diff1": 0.3633362122496361, - "nauc_map_at_20_max": 0.30202217987243185, - "nauc_map_at_20_std": -0.14627098741534908, - "nauc_map_at_3_diff1": 0.34885387763276127, - "nauc_map_at_3_max": 0.309027475843024, - "nauc_map_at_3_std": -0.16955601757062616, - "nauc_map_at_5_diff1": 0.35307672932655393, - "nauc_map_at_5_max": 0.2945922362211287, - "nauc_map_at_5_std": -0.16241885430038053, - "nauc_mrr_at_1000_diff1": 0.26560116161479447, - "nauc_mrr_at_1000_max": 0.2286556037665365, - "nauc_mrr_at_1000_std": -0.12003394418012507, - "nauc_mrr_at_100_diff1": 0.2653509593794469, - "nauc_mrr_at_100_max": 0.22870899026180094, - "nauc_mrr_at_100_std": -0.12000263248162615, - "nauc_mrr_at_10_diff1": 0.2625870606112582, - "nauc_mrr_at_10_max": 0.2370936298550721, - "nauc_mrr_at_10_std": -0.11265461889909462, - "nauc_mrr_at_1_diff1": 0.3496222260314613, - "nauc_mrr_at_1_max": 0.1055525521169845, - "nauc_mrr_at_1_std": -0.14339587811020701, - "nauc_mrr_at_20_diff1": 0.2638864741009738, - "nauc_mrr_at_20_max": 0.23148416292732588, - "nauc_mrr_at_20_std": -0.11651871247529559, - "nauc_mrr_at_3_diff1": 0.2589399601820997, - "nauc_mrr_at_3_max": 0.24019609132343225, - "nauc_mrr_at_3_std": -0.1440308654475832, - "nauc_mrr_at_5_diff1": 0.25871660893688886, - "nauc_mrr_at_5_max": 0.22758080746049747, - "nauc_mrr_at_5_std": -0.1430749326822504, - "nauc_ndcg_at_1000_diff1": 0.34389535687748, - "nauc_ndcg_at_1000_max": 0.314316158167146, - "nauc_ndcg_at_1000_std": -0.13511867578024445, - "nauc_ndcg_at_100_diff1": 0.3352177735053365, - "nauc_ndcg_at_100_max": 0.31485379518438816, - "nauc_ndcg_at_100_std": -0.13380300430512387, - "nauc_ndcg_at_10_diff1": 0.31190857835409885, - "nauc_ndcg_at_10_max": 0.35259278734259186, - "nauc_ndcg_at_10_std": -0.08980541800210258, - "nauc_ndcg_at_1_diff1": 0.499334771432631, - "nauc_ndcg_at_1_max": 0.2027203327434544, - "nauc_ndcg_at_1_std": -0.2031595127685132, - "nauc_ndcg_at_20_diff1": 0.3200803496952673, - "nauc_ndcg_at_20_max": 0.3339057031205548, - "nauc_ndcg_at_20_std": -0.10284104333077189, - "nauc_ndcg_at_3_diff1": 0.29178770526806463, - "nauc_ndcg_at_3_max": 0.3444567331842166, - "nauc_ndcg_at_3_std": -0.16170218021524788, - "nauc_ndcg_at_5_diff1": 0.29483888128565966, - "nauc_ndcg_at_5_max": 0.31916959383433774, - "nauc_ndcg_at_5_std": -0.14266008480433065, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.3674136321195118, - "nauc_precision_at_100_max": 0.3580765639589172, - "nauc_precision_at_100_std": -0.024276377217548, - "nauc_precision_at_10_diff1": 0.014626379323115002, - "nauc_precision_at_10_max": 0.6539657853810266, - "nauc_precision_at_10_std": 0.282974153891728, - "nauc_precision_at_1_diff1": 0.499334771432631, - "nauc_precision_at_1_max": 0.2027203327434544, - "nauc_precision_at_1_std": -0.2031595127685132, - "nauc_precision_at_20_diff1": -0.06349206349206139, - "nauc_precision_at_20_max": 0.6151377217553686, - "nauc_precision_at_20_std": 0.3649626517273598, - "nauc_precision_at_3_diff1": 0.06112217721179036, - "nauc_precision_at_3_max": 0.4854412390053937, - "nauc_precision_at_3_std": -0.1321814641570239, - "nauc_precision_at_5_diff1": -0.012641138929799524, - "nauc_precision_at_5_max": 0.4398011782032391, - "nauc_precision_at_5_std": -0.03126534118802223, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.367413632119516, - "nauc_recall_at_100_max": 0.35807656395891974, - "nauc_recall_at_100_std": -0.0242763772175541, - "nauc_recall_at_10_diff1": 0.01462637932311375, - "nauc_recall_at_10_max": 0.6539657853810261, - "nauc_recall_at_10_std": 0.2829741538917277, - "nauc_recall_at_1_diff1": 0.499334771432631, - "nauc_recall_at_1_max": 0.2027203327434544, - "nauc_recall_at_1_std": -0.2031595127685132, - "nauc_recall_at_20_diff1": -0.06349206349206456, - "nauc_recall_at_20_max": 0.6151377217553676, - "nauc_recall_at_20_std": 0.36496265172735837, - "nauc_recall_at_3_diff1": 0.0611221772117895, - "nauc_recall_at_3_max": 0.485441239005394, - "nauc_recall_at_3_std": -0.1321814641570245, - "nauc_recall_at_5_diff1": -0.01264113892979802, - "nauc_recall_at_5_max": 0.43980117820323966, - "nauc_recall_at_5_std": -0.031265341188021364, - "ndcg_at_1": 0.41, - "ndcg_at_10": 0.66508, - "ndcg_at_100": 0.68922, - "ndcg_at_1000": 0.69189, - "ndcg_at_20": 0.67774, - "ndcg_at_3": 0.62535, - "ndcg_at_5": 0.65462, - "precision_at_1": 0.41, - "precision_at_10": 0.087, - "precision_at_100": 0.0098, - "precision_at_1000": 0.001, - "precision_at_20": 0.046, - "precision_at_3": 0.25667, - "precision_at_5": 0.168, - "recall_at_1": 0.41, - "recall_at_10": 0.87, - "recall_at_100": 0.98, - "recall_at_1000": 1.0, - "recall_at_20": 0.92, - "recall_at_3": 0.77, - "recall_at_5": 0.84 - } - ] - }, - "task_name": "VidoreSyntheticDocQAEnergyRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json deleted file mode 100644 index 1c60fce707..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", - "evaluation_time": 74.7807514667511, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.42, - "cv_recall_at_10": 0.94, - "cv_recall_at_100": 0.98, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.95, - "cv_recall_at_3": 0.78, - "cv_recall_at_5": 0.86, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.66761, - "map_at_1": 0.42, - "map_at_10": 0.61359, - "map_at_100": 0.61548, - "map_at_1000": 0.61559, - "map_at_20": 0.6145, - "map_at_3": 0.585, - "map_at_5": 0.6025, - "mrr_at_1": 0.42, - "mrr_at_10": 0.6167023809523812, - "mrr_at_100": 0.6185586140207295, - "mrr_at_1000": 0.6186628619999807, - "mrr_at_20": 0.617611471861472, - "mrr_at_3": 0.5883333333333334, - "mrr_at_5": 0.6058333333333334, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.0242763772175541, - "nauc_cv_recall_at_100_max": 0.8611111111111092, - "nauc_cv_recall_at_100_std": 0.71171802054155, - "nauc_cv_recall_at_10_diff1": 0.06395891690009405, - "nauc_cv_recall_at_10_max": 0.7930283224400876, - "nauc_cv_recall_at_10_std": 0.3990818549642063, - "nauc_cv_recall_at_1_diff1": 0.3091608989812584, - "nauc_cv_recall_at_1_max": 0.4137569017808538, - "nauc_cv_recall_at_1_std": -0.01810664385514676, - "nauc_cv_recall_at_20_diff1": 0.00513538748832759, - "nauc_cv_recall_at_20_max": 0.8071895424836592, - "nauc_cv_recall_at_20_std": 0.4543417366946761, - "nauc_cv_recall_at_3_diff1": -0.20373719107896363, - "nauc_cv_recall_at_3_max": 0.7115500533222051, - "nauc_cv_recall_at_3_std": 0.06758009922566846, - "nauc_cv_recall_at_5_diff1": -0.19107662463627592, - "nauc_cv_recall_at_5_max": 0.6820701122349996, - "nauc_cv_recall_at_5_std": 0.03606069003741182, - "nauc_map_at_1000_diff1": 0.17693830624484685, - "nauc_map_at_1000_max": 0.40199420044287293, - "nauc_map_at_1000_std": -0.0723223706512287, - "nauc_map_at_100_diff1": 0.17684396080137854, - "nauc_map_at_100_max": 0.40218884057257354, - "nauc_map_at_100_std": -0.07199833339393502, - "nauc_map_at_10_diff1": 0.17680282662193408, - "nauc_map_at_10_max": 0.4046908806180739, - "nauc_map_at_10_std": -0.07074201397090754, - "nauc_map_at_1_diff1": 0.3328796951551442, - "nauc_map_at_1_max": 0.2827980402830702, - "nauc_map_at_1_std": -0.16376338232625642, - "nauc_map_at_20_diff1": 0.1761825278902579, - "nauc_map_at_20_max": 0.4036043233960628, - "nauc_map_at_20_std": -0.0714042333250469, - "nauc_map_at_3_diff1": 0.13335576761291268, - "nauc_map_at_3_max": 0.4300971885040343, - "nauc_map_at_3_std": -0.0699869400443876, - "nauc_map_at_5_diff1": 0.15877224784371946, - "nauc_map_at_5_max": 0.4128887839672196, - "nauc_map_at_5_std": -0.07222002074076582, - "nauc_mrr_at_1000_diff1": 0.16594205369860826, - "nauc_mrr_at_1000_max": 0.4924629063588566, - "nauc_mrr_at_1000_std": 0.03004682497252532, - "nauc_mrr_at_100_diff1": 0.16585082182469363, - "nauc_mrr_at_100_max": 0.49262403678833994, - "nauc_mrr_at_100_std": 0.030334121756698387, - "nauc_mrr_at_10_diff1": 0.16591245155036535, - "nauc_mrr_at_10_max": 0.49445652729464, - "nauc_mrr_at_10_std": 0.030869742208528065, - "nauc_mrr_at_1_diff1": 0.3091608989812584, - "nauc_mrr_at_1_max": 0.4137569017808538, - "nauc_mrr_at_1_std": -0.01810664385514676, - "nauc_mrr_at_20_diff1": 0.16525013780729855, - "nauc_mrr_at_20_max": 0.49367150896807016, - "nauc_mrr_at_20_std": 0.03055295544576017, - "nauc_mrr_at_3_diff1": 0.13051631319395293, - "nauc_mrr_at_3_max": 0.5124847656533753, - "nauc_mrr_at_3_std": 0.023984847710341952, - "nauc_mrr_at_5_diff1": 0.1470586569315778, - "nauc_mrr_at_5_max": 0.4994845463016199, - "nauc_mrr_at_5_std": 0.021489887761373738, - "nauc_ndcg_at_1000_diff1": 0.161837391311996, - "nauc_ndcg_at_1000_max": 0.4176893400412115, - "nauc_ndcg_at_1000_std": -0.05567968591990525, - "nauc_ndcg_at_100_diff1": 0.1594005018759552, - "nauc_ndcg_at_100_max": 0.4233584462924314, - "nauc_ndcg_at_100_std": -0.045922161116991364, - "nauc_ndcg_at_10_diff1": 0.15803940765596416, - "nauc_ndcg_at_10_max": 0.43679112701120815, - "nauc_ndcg_at_10_std": -0.03664740566414989, - "nauc_ndcg_at_1_diff1": 0.3328796951551442, - "nauc_ndcg_at_1_max": 0.2827980402830702, - "nauc_ndcg_at_1_std": -0.16376338232625642, - "nauc_ndcg_at_20_diff1": 0.1554345969800805, - "nauc_ndcg_at_20_max": 0.4330743476999243, - "nauc_ndcg_at_20_std": -0.03872344287722131, - "nauc_ndcg_at_3_diff1": 0.06065482050917286, - "nauc_ndcg_at_3_max": 0.4850976042539053, - "nauc_ndcg_at_3_std": -0.042410523533264216, - "nauc_ndcg_at_5_diff1": 0.10788221180845371, - "nauc_ndcg_at_5_max": 0.4540038684839002, - "nauc_ndcg_at_5_std": -0.04680446883376484, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.024276377217548, - "nauc_precision_at_100_max": 0.8611111111111196, - "nauc_precision_at_100_std": 0.7117180205415448, - "nauc_precision_at_10_diff1": 0.06395891690009418, - "nauc_precision_at_10_max": 0.7930283224400888, - "nauc_precision_at_10_std": 0.3990818549642083, - "nauc_precision_at_1_diff1": 0.3328796951551442, - "nauc_precision_at_1_max": 0.2827980402830702, - "nauc_precision_at_1_std": -0.16376338232625642, - "nauc_precision_at_20_diff1": 0.005135387488338012, - "nauc_precision_at_20_max": 0.8071895424836621, - "nauc_precision_at_20_std": 0.4543417366946809, - "nauc_precision_at_3_diff1": -0.23781703528538964, - "nauc_precision_at_3_max": 0.7115500533222053, - "nauc_precision_at_3_std": 0.06758009922566896, - "nauc_precision_at_5_diff1": -0.1692531522793392, - "nauc_precision_at_5_max": 0.6820701122350005, - "nauc_precision_at_5_std": 0.09145074130525248, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.0242763772175541, - "nauc_recall_at_100_max": 0.8611111111111092, - "nauc_recall_at_100_std": 0.71171802054155, - "nauc_recall_at_10_diff1": 0.06395891690009405, - "nauc_recall_at_10_max": 0.7930283224400876, - "nauc_recall_at_10_std": 0.3990818549642063, - "nauc_recall_at_1_diff1": 0.3328796951551442, - "nauc_recall_at_1_max": 0.2827980402830702, - "nauc_recall_at_1_std": -0.16376338232625642, - "nauc_recall_at_20_diff1": 0.00513538748832759, - "nauc_recall_at_20_max": 0.8071895424836592, - "nauc_recall_at_20_std": 0.4543417366946761, - "nauc_recall_at_3_diff1": -0.23781703528539025, - "nauc_recall_at_3_max": 0.7115500533222051, - "nauc_recall_at_3_std": 0.06758009922566846, - "nauc_recall_at_5_diff1": -0.16925315227934043, - "nauc_recall_at_5_max": 0.6820701122349996, - "nauc_recall_at_5_std": 0.09145074130525091, - "ndcg_at_1": 0.42, - "ndcg_at_10": 0.69391, - "ndcg_at_100": 0.70261, - "ndcg_at_1000": 0.70523, - "ndcg_at_20": 0.6967, - "ndcg_at_3": 0.63535, - "ndcg_at_5": 0.66761, - "precision_at_1": 0.42, - "precision_at_10": 0.094, - "precision_at_100": 0.0098, - "precision_at_1000": 0.001, - "precision_at_20": 0.0475, - "precision_at_3": 0.26, - "precision_at_5": 0.172, - "recall_at_1": 0.42, - "recall_at_10": 0.94, - "recall_at_100": 0.98, - "recall_at_1000": 1.0, - "recall_at_20": 0.95, - "recall_at_3": 0.78, - "recall_at_5": 0.86 - } - ] - }, - "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json deleted file mode 100644 index 3828955dfa..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", - "evaluation_time": 73.25277805328369, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.47, - "cv_recall_at_10": 0.94, - "cv_recall_at_100": 0.97, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.94, - "cv_recall_at_3": 0.87, - "cv_recall_at_5": 0.89, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.71077, - "map_at_1": 0.45, - "map_at_10": 0.65593, - "map_at_100": 0.6569, - "map_at_1000": 0.65706, - "map_at_20": 0.65593, - "map_at_3": 0.64667, - "map_at_5": 0.64867, - "mrr_at_1": 0.47, - "mrr_at_10": 0.6642976190476192, - "mrr_at_100": 0.6652641390867199, - "mrr_at_1000": 0.6654314232950488, - "mrr_at_20": 0.6642976190476192, - "mrr_at_3": 0.6533333333333333, - "mrr_at_5": 0.6573333333333333, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.10597572362278272, - "nauc_cv_recall_at_100_max": 0.4788359788359823, - "nauc_cv_recall_at_100_std": -0.009648303765951848, - "nauc_cv_recall_at_10_diff1": -0.04061624649859804, - "nauc_cv_recall_at_10_max": 0.6651104886398995, - "nauc_cv_recall_at_10_std": 0.016728913787736808, - "nauc_cv_recall_at_1_diff1": 0.2818602565962105, - "nauc_cv_recall_at_1_max": 0.052374108308390115, - "nauc_cv_recall_at_1_std": -0.1893748599005788, - "nauc_cv_recall_at_20_diff1": -0.04061624649859804, - "nauc_cv_recall_at_20_max": 0.6651104886398995, - "nauc_cv_recall_at_20_std": 0.016728913787736808, - "nauc_cv_recall_at_3_diff1": -0.3010812412056582, - "nauc_cv_recall_at_3_max": 0.5909427534621934, - "nauc_cv_recall_at_3_std": -0.1500407316892538, - "nauc_cv_recall_at_5_diff1": -0.2275148183145788, - "nauc_cv_recall_at_5_max": 0.6092260115110386, - "nauc_cv_recall_at_5_std": -0.15514131088394567, - "nauc_map_at_1000_diff1": 0.17737809853138078, - "nauc_map_at_1000_max": 0.3197186933030335, - "nauc_map_at_1000_std": -0.15660439305683332, - "nauc_map_at_100_diff1": 0.17713247274986268, - "nauc_map_at_100_max": 0.31971110754062343, - "nauc_map_at_100_std": -0.15658001231252233, - "nauc_map_at_10_diff1": 0.1766419592404658, - "nauc_map_at_10_max": 0.3218218690272864, - "nauc_map_at_10_std": -0.15584231228112508, - "nauc_map_at_1_diff1": 0.2847508675440319, - "nauc_map_at_1_max": 0.22929352452039553, - "nauc_map_at_1_std": -0.1787599030969685, - "nauc_map_at_20_diff1": 0.1766419592404658, - "nauc_map_at_20_max": 0.3218218690272864, - "nauc_map_at_20_std": -0.15584231228112508, - "nauc_map_at_3_diff1": 0.15674372963365232, - "nauc_map_at_3_max": 0.3338846647575784, - "nauc_map_at_3_std": -0.15814160165893193, - "nauc_map_at_5_diff1": 0.15920039673168618, - "nauc_map_at_5_max": 0.3282159254462605, - "nauc_map_at_5_std": -0.16209944641512158, - "nauc_mrr_at_1000_diff1": 0.15966423921533143, - "nauc_mrr_at_1000_max": 0.17666600015301887, - "nauc_mrr_at_1000_std": -0.17509432480460377, - "nauc_mrr_at_100_diff1": 0.15942769989266647, - "nauc_mrr_at_100_max": 0.17675277810481055, - "nauc_mrr_at_100_std": -0.17506183456061375, - "nauc_mrr_at_10_diff1": 0.15900784081013614, - "nauc_mrr_at_10_max": 0.17942911487138907, - "nauc_mrr_at_10_std": -0.174251393599049, - "nauc_mrr_at_1_diff1": 0.2818602565962105, - "nauc_mrr_at_1_max": 0.052374108308390115, - "nauc_mrr_at_1_std": -0.1893748599005788, - "nauc_mrr_at_20_diff1": 0.15900784081013614, - "nauc_mrr_at_20_max": 0.17942911487138907, - "nauc_mrr_at_20_std": -0.174251393599049, - "nauc_mrr_at_3_diff1": 0.1304221207197815, - "nauc_mrr_at_3_max": 0.19363906685877158, - "nauc_mrr_at_3_std": -0.17807547285490927, - "nauc_mrr_at_5_diff1": 0.14280359493903766, - "nauc_mrr_at_5_max": 0.18970837430829438, - "nauc_mrr_at_5_std": -0.17890776810678843, - "nauc_ndcg_at_1000_diff1": 0.16305570366234656, - "nauc_ndcg_at_1000_max": 0.33387508653430314, - "nauc_ndcg_at_1000_std": -0.1510648449789182, - "nauc_ndcg_at_100_diff1": 0.15718521389079343, - "nauc_ndcg_at_100_max": 0.3361827474796506, - "nauc_ndcg_at_100_std": -0.1485562970765986, - "nauc_ndcg_at_10_diff1": 0.153440821156415, - "nauc_ndcg_at_10_max": 0.35171505689198845, - "nauc_ndcg_at_10_std": -0.14299661493260465, - "nauc_ndcg_at_1_diff1": 0.2847508675440319, - "nauc_ndcg_at_1_max": 0.22929352452039553, - "nauc_ndcg_at_1_std": -0.1787599030969685, - "nauc_ndcg_at_20_diff1": 0.153440821156415, - "nauc_ndcg_at_20_max": 0.35171505689198845, - "nauc_ndcg_at_20_std": -0.14299661493260465, - "nauc_ndcg_at_3_diff1": 0.10268992445948906, - "nauc_ndcg_at_3_max": 0.378473051261699, - "nauc_ndcg_at_3_std": -0.15155969201864058, - "nauc_ndcg_at_5_diff1": 0.1073960551331664, - "nauc_ndcg_at_5_max": 0.3663556421486979, - "nauc_ndcg_at_5_std": -0.16040529466424153, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.10597572362277796, - "nauc_precision_at_100_max": 0.4788359788359767, - "nauc_precision_at_100_std": -0.009648303765945032, - "nauc_precision_at_10_diff1": -0.04061624649859813, - "nauc_precision_at_10_max": 0.6651104886399004, - "nauc_precision_at_10_std": 0.016728913787735757, - "nauc_precision_at_1_diff1": 0.2847508675440319, - "nauc_precision_at_1_max": 0.22929352452039553, - "nauc_precision_at_1_std": -0.1787599030969685, - "nauc_precision_at_20_diff1": -0.04061624649859813, - "nauc_precision_at_20_max": 0.6651104886399004, - "nauc_precision_at_20_std": 0.016728913787735757, - "nauc_precision_at_3_diff1": -0.2252624880687238, - "nauc_precision_at_3_max": 0.6493000318167349, - "nauc_precision_at_3_std": -0.11314826598790892, - "nauc_precision_at_5_diff1": -0.2275148183145768, - "nauc_precision_at_5_max": 0.6092260115110381, - "nauc_precision_at_5_std": -0.15514131088394403, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.10597572362278272, - "nauc_recall_at_100_max": 0.4788359788359823, - "nauc_recall_at_100_std": -0.009648303765951848, - "nauc_recall_at_10_diff1": -0.04061624649859804, - "nauc_recall_at_10_max": 0.6651104886398995, - "nauc_recall_at_10_std": 0.016728913787736808, - "nauc_recall_at_1_diff1": 0.2847508675440319, - "nauc_recall_at_1_max": 0.22929352452039553, - "nauc_recall_at_1_std": -0.1787599030969685, - "nauc_recall_at_20_diff1": -0.04061624649859804, - "nauc_recall_at_20_max": 0.6651104886398995, - "nauc_recall_at_20_std": 0.016728913787736808, - "nauc_recall_at_3_diff1": -0.22526248806872484, - "nauc_recall_at_3_max": 0.6493000318167352, - "nauc_recall_at_3_std": -0.11314826598791083, - "nauc_recall_at_5_diff1": -0.2275148183145788, - "nauc_recall_at_5_max": 0.6092260115110386, - "nauc_recall_at_5_std": -0.15514131088394567, - "ndcg_at_1": 0.45, - "ndcg_at_10": 0.72753, - "ndcg_at_100": 0.73351, - "ndcg_at_1000": 0.73748, - "ndcg_at_20": 0.72753, - "ndcg_at_3": 0.7069, - "ndcg_at_5": 0.71077, - "precision_at_1": 0.45, - "precision_at_10": 0.094, - "precision_at_100": 0.0097, - "precision_at_1000": 0.001, - "precision_at_20": 0.047, - "precision_at_3": 0.29333, - "precision_at_5": 0.178, - "recall_at_1": 0.45, - "recall_at_10": 0.94, - "recall_at_100": 0.97, - "recall_at_1000": 1.0, - "recall_at_20": 0.94, - "recall_at_3": 0.88, - "recall_at_5": 0.89 - } - ] - }, - "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json deleted file mode 100644 index 817d0b4849..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTabfquadRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", - "evaluation_time": 17.969542264938354, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.14286, - "cv_recall_at_10": 0.59643, - "cv_recall_at_100": 0.87143, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.71429, - "cv_recall_at_3": 0.38214, - "cv_recall_at_5": 0.5, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.31415, - "map_at_1": 0.125, - "map_at_10": 0.27248, - "map_at_100": 0.28394, - "map_at_1000": 0.28477, - "map_at_20": 0.27971, - "map_at_3": 0.22619, - "map_at_5": 0.25601, - "mrr_at_1": 0.14285714285714285, - "mrr_at_10": 0.2834226190476192, - "mrr_at_100": 0.2963289758699489, - "mrr_at_1000": 0.29714896365874977, - "mrr_at_20": 0.2919793288742317, - "mrr_at_3": 0.24166666666666672, - "mrr_at_5": 0.2700595238095239, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.09346618228443337, - "nauc_cv_recall_at_100_max": 0.47191206325462104, - "nauc_cv_recall_at_100_std": 0.5897264665028307, - "nauc_cv_recall_at_10_diff1": 0.06153163948878473, - "nauc_cv_recall_at_10_max": 0.527236830297048, - "nauc_cv_recall_at_10_std": 0.34759886185471267, - "nauc_cv_recall_at_1_diff1": 0.04107196177963575, - "nauc_cv_recall_at_1_max": 0.18530904747685875, - "nauc_cv_recall_at_1_std": 0.11839355031352648, - "nauc_cv_recall_at_20_diff1": -0.01519699812382745, - "nauc_cv_recall_at_20_max": 0.44332618600911256, - "nauc_cv_recall_at_20_std": 0.377559635486465, - "nauc_cv_recall_at_3_diff1": 0.12322123253689696, - "nauc_cv_recall_at_3_max": 0.3550718925761234, - "nauc_cv_recall_at_3_std": 0.24882574686208955, - "nauc_cv_recall_at_5_diff1": 0.07709622156657454, - "nauc_cv_recall_at_5_max": 0.5071523746074043, - "nauc_cv_recall_at_5_std": 0.3624536023603309, - "nauc_map_at_1000_diff1": 0.14508719417530674, - "nauc_map_at_1000_max": 0.3028773500015137, - "nauc_map_at_1000_std": 0.22042646887013098, - "nauc_map_at_100_diff1": 0.14428659458613782, - "nauc_map_at_100_max": 0.30343561386061807, - "nauc_map_at_100_std": 0.22154390909649585, - "nauc_map_at_10_diff1": 0.1458055654055668, - "nauc_map_at_10_max": 0.3087857008787967, - "nauc_map_at_10_std": 0.21513465622236783, - "nauc_map_at_1_diff1": 0.21192189105858175, - "nauc_map_at_1_max": 0.1422322713257966, - "nauc_map_at_1_std": 0.12627338129496413, - "nauc_map_at_20_diff1": 0.1434484748653935, - "nauc_map_at_20_max": 0.3048401943472712, - "nauc_map_at_20_std": 0.21989153816245258, - "nauc_map_at_3_diff1": 0.15084588762214982, - "nauc_map_at_3_max": 0.2895541530944628, - "nauc_map_at_3_std": 0.21155741042345294, - "nauc_map_at_5_diff1": 0.14886552885273804, - "nauc_map_at_5_max": 0.3118795633185364, - "nauc_map_at_5_std": 0.21775580485670507, - "nauc_mrr_at_1000_diff1": 0.05695061181399053, - "nauc_mrr_at_1000_max": 0.3111012267827344, - "nauc_mrr_at_1000_std": 0.21393435228582347, - "nauc_mrr_at_100_diff1": 0.056440815201173854, - "nauc_mrr_at_100_max": 0.3116096019154161, - "nauc_mrr_at_100_std": 0.21502116138592015, - "nauc_mrr_at_10_diff1": 0.061021499554143384, - "nauc_mrr_at_10_max": 0.3224409033707004, - "nauc_mrr_at_10_std": 0.21528251710496038, - "nauc_mrr_at_1_diff1": 0.04107196177963575, - "nauc_mrr_at_1_max": 0.18530904747685875, - "nauc_mrr_at_1_std": 0.11839355031352648, - "nauc_mrr_at_20_diff1": 0.0566059945620949, - "nauc_mrr_at_20_max": 0.31286490797934124, - "nauc_mrr_at_20_std": 0.2135380630344629, - "nauc_mrr_at_3_diff1": 0.07833818406529658, - "nauc_mrr_at_3_max": 0.2753016193582217, - "nauc_mrr_at_3_std": 0.18873408566788993, - "nauc_mrr_at_5_diff1": 0.06400650046913868, - "nauc_mrr_at_5_max": 0.32162314506455336, - "nauc_mrr_at_5_std": 0.2214497802320089, - "nauc_ndcg_at_1000_diff1": 0.12888457382784677, - "nauc_ndcg_at_1000_max": 0.325817844139425, - "nauc_ndcg_at_1000_std": 0.24189184208615394, - "nauc_ndcg_at_100_diff1": 0.11282750611807016, - "nauc_ndcg_at_100_max": 0.33632613400391, - "nauc_ndcg_at_100_std": 0.26639957608731973, - "nauc_ndcg_at_10_diff1": 0.12071304892163645, - "nauc_ndcg_at_10_max": 0.3563710929162044, - "nauc_ndcg_at_10_std": 0.2370147085502524, - "nauc_ndcg_at_1_diff1": 0.21192189105858175, - "nauc_ndcg_at_1_max": 0.1422322713257966, - "nauc_ndcg_at_1_std": 0.12627338129496413, - "nauc_ndcg_at_20_diff1": 0.10882639489305446, - "nauc_ndcg_at_20_max": 0.3432251869586007, - "nauc_ndcg_at_20_std": 0.25568218475503307, - "nauc_ndcg_at_3_diff1": 0.13085937145194312, - "nauc_ndcg_at_3_max": 0.32974469478667556, - "nauc_ndcg_at_3_std": 0.23643620953909147, - "nauc_ndcg_at_5_diff1": 0.12727992939268223, - "nauc_ndcg_at_5_max": 0.3649735155042485, - "nauc_ndcg_at_5_std": 0.2454961972349734, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.09346618228443235, - "nauc_precision_at_100_max": 0.47191206325462226, - "nauc_precision_at_100_std": 0.5897264665028321, - "nauc_precision_at_10_diff1": 0.05467766594814778, - "nauc_precision_at_10_max": 0.4735201482417899, - "nauc_precision_at_10_std": 0.2869915668573714, - "nauc_precision_at_1_diff1": 0.21192189105858175, - "nauc_precision_at_1_max": 0.1422322713257966, - "nauc_precision_at_1_std": 0.12627338129496413, - "nauc_precision_at_20_diff1": -0.015196998123827318, - "nauc_precision_at_20_max": 0.4433261860091128, - "nauc_precision_at_20_std": 0.3775596354864649, - "nauc_precision_at_3_diff1": 0.08504628829930763, - "nauc_precision_at_3_max": 0.4204835645025001, - "nauc_precision_at_3_std": 0.292939808871107, - "nauc_precision_at_5_diff1": 0.07691213631127067, - "nauc_precision_at_5_max": 0.48827330523541085, - "nauc_precision_at_5_std": 0.30920357086981864, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.09346618228443337, - "nauc_recall_at_100_max": 0.47191206325462104, - "nauc_recall_at_100_std": 0.5897264665028307, - "nauc_recall_at_10_diff1": 0.05467766594814761, - "nauc_recall_at_10_max": 0.4735201482417895, - "nauc_recall_at_10_std": 0.2869915668573714, - "nauc_recall_at_1_diff1": 0.21192189105858175, - "nauc_recall_at_1_max": 0.1422322713257966, - "nauc_recall_at_1_std": 0.12627338129496413, - "nauc_recall_at_20_diff1": -0.01519699812382745, - "nauc_recall_at_20_max": 0.44332618600911256, - "nauc_recall_at_20_std": 0.377559635486465, - "nauc_recall_at_3_diff1": 0.08504628829930751, - "nauc_recall_at_3_max": 0.42048356450249996, - "nauc_recall_at_3_std": 0.29293980887110704, - "nauc_recall_at_5_diff1": 0.07691213631127075, - "nauc_recall_at_5_max": 0.4882733052354108, - "nauc_recall_at_5_std": 0.30920357086981887, - "ndcg_at_1": 0.125, - "ndcg_at_10": 0.35368, - "ndcg_at_100": 0.40933, - "ndcg_at_1000": 0.42685, - "ndcg_at_20": 0.37992, - "ndcg_at_3": 0.26233, - "ndcg_at_5": 0.31415, - "precision_at_1": 0.125, - "precision_at_10": 0.06107, - "precision_at_100": 0.00871, - "precision_at_1000": 0.001, - "precision_at_20": 0.03571, - "precision_at_3": 0.12262, - "precision_at_5": 0.09786, - "recall_at_1": 0.125, - "recall_at_10": 0.61071, - "recall_at_100": 0.87143, - "recall_at_1000": 1.0, - "recall_at_20": 0.71429, - "recall_at_3": 0.36786, - "recall_at_5": 0.48929 - } - ] - }, - "task_name": "VidoreTabfquadRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json deleted file mode 100644 index 9aa82d28ab..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/VidoreTatdqaRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", - "evaluation_time": 130.58521914482117, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.03969, - "cv_recall_at_10": 0.23211, - "cv_recall_at_100": 0.51473, - "cv_recall_at_1000": 0.92183, - "cv_recall_at_20": 0.30487, - "cv_recall_at_3": 0.10042, - "cv_recall_at_5": 0.16055, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.08802, - "map_at_1": 0.02886, - "map_at_10": 0.07924, - "map_at_100": 0.08947, - "map_at_1000": 0.09101, - "map_at_20": 0.08455, - "map_at_3": 0.05522, - "map_at_5": 0.06815, - "mrr_at_1": 0.0396873120865905, - "mrr_at_10": 0.08826155637640908, - "mrr_at_100": 0.09808012131592855, - "mrr_at_1000": 0.09962230517717666, - "mrr_at_20": 0.09316889360880878, - "mrr_at_3": 0.06504309480857881, - "mrr_at_5": 0.07857286029264379, - "nauc_cv_recall_at_1000_diff1": 0.0977858066622949, - "nauc_cv_recall_at_1000_max": 0.19289234929852755, - "nauc_cv_recall_at_1000_std": 0.4244384238930124, - "nauc_cv_recall_at_100_diff1": 0.027363809758551075, - "nauc_cv_recall_at_100_max": 0.1472382566698071, - "nauc_cv_recall_at_100_std": 0.3133680991369263, - "nauc_cv_recall_at_10_diff1": -0.015248947065392211, - "nauc_cv_recall_at_10_max": 0.17684865071070638, - "nauc_cv_recall_at_10_std": 0.23399970322736818, - "nauc_cv_recall_at_1_diff1": 0.08496451376348123, - "nauc_cv_recall_at_1_max": 0.2250756456490274, - "nauc_cv_recall_at_1_std": 0.16451661904930748, - "nauc_cv_recall_at_20_diff1": -0.01759476439814012, - "nauc_cv_recall_at_20_max": 0.1384202166086512, - "nauc_cv_recall_at_20_std": 0.24952524851481098, - "nauc_cv_recall_at_3_diff1": 0.0683858086458561, - "nauc_cv_recall_at_3_max": 0.24611576386062065, - "nauc_cv_recall_at_3_std": 0.2046414079176916, - "nauc_cv_recall_at_5_diff1": 0.0389561258425957, - "nauc_cv_recall_at_5_max": 0.21630673696973646, - "nauc_cv_recall_at_5_std": 0.20644440359224767, - "nauc_map_at_1000_diff1": 0.0044848236290639725, - "nauc_map_at_1000_max": 0.1726088992109447, - "nauc_map_at_1000_std": 0.21597266371424678, - "nauc_map_at_100_diff1": 0.004521095517779526, - "nauc_map_at_100_max": 0.17262602441956626, - "nauc_map_at_100_std": 0.21593147204916938, - "nauc_map_at_10_diff1": 0.002769629051444685, - "nauc_map_at_10_max": 0.17949324947822573, - "nauc_map_at_10_std": 0.209214171869861, - "nauc_map_at_1_diff1": -0.02158849186859297, - "nauc_map_at_1_max": 0.13996579430266687, - "nauc_map_at_1_std": 0.17219572255475857, - "nauc_map_at_20_diff1": 0.0017843929653374108, - "nauc_map_at_20_max": 0.1723382974559687, - "nauc_map_at_20_std": 0.21039987703317992, - "nauc_map_at_3_diff1": 0.030158997427946903, - "nauc_map_at_3_max": 0.18440904960369608, - "nauc_map_at_3_std": 0.19515292645565502, - "nauc_map_at_5_diff1": 0.011488449209473625, - "nauc_map_at_5_max": 0.17995271406284225, - "nauc_map_at_5_std": 0.19587606041940772, - "nauc_mrr_at_1000_diff1": 0.045817042591563197, - "nauc_mrr_at_1000_max": 0.20920132305788777, - "nauc_mrr_at_1000_std": 0.2128708534292149, - "nauc_mrr_at_100_diff1": 0.04561537361722436, - "nauc_mrr_at_100_max": 0.20896709990127685, - "nauc_mrr_at_100_std": 0.21279912539406717, - "nauc_mrr_at_10_diff1": 0.04588905114387296, - "nauc_mrr_at_10_max": 0.2146020467703159, - "nauc_mrr_at_10_std": 0.20421262749770086, - "nauc_mrr_at_1_diff1": 0.08496451376348123, - "nauc_mrr_at_1_max": 0.2250756456490274, - "nauc_mrr_at_1_std": 0.16451661904930748, - "nauc_mrr_at_20_diff1": 0.0441332027973836, - "nauc_mrr_at_20_max": 0.20796335061234025, - "nauc_mrr_at_20_std": 0.20809344152536044, - "nauc_mrr_at_3_diff1": 0.07853906035769813, - "nauc_mrr_at_3_max": 0.2391325353905553, - "nauc_mrr_at_3_std": 0.1942148403197897, - "nauc_mrr_at_5_diff1": 0.06523844727902996, - "nauc_mrr_at_5_max": 0.22637957489872684, - "nauc_mrr_at_5_std": 0.19592628829316405, - "nauc_ndcg_at_1000_diff1": 0.009106844527408595, - "nauc_ndcg_at_1000_max": 0.16986530318282603, - "nauc_ndcg_at_1000_std": 0.2477659100109444, - "nauc_ndcg_at_100_diff1": 0.008486238488498946, - "nauc_ndcg_at_100_max": 0.1646870237335375, - "nauc_ndcg_at_100_std": 0.25192946867971616, - "nauc_ndcg_at_10_diff1": -0.006078172191213006, - "nauc_ndcg_at_10_max": 0.18160285165655646, - "nauc_ndcg_at_10_std": 0.22500698386041396, - "nauc_ndcg_at_1_diff1": -0.02158849186859297, - "nauc_ndcg_at_1_max": 0.13996579430266687, - "nauc_ndcg_at_1_std": 0.17219572255475857, - "nauc_ndcg_at_20_diff1": -0.009322631249012595, - "nauc_ndcg_at_20_max": 0.16357791120190782, - "nauc_ndcg_at_20_std": 0.2260169780476658, - "nauc_ndcg_at_3_diff1": 0.03642967796503214, - "nauc_ndcg_at_3_max": 0.192101861179512, - "nauc_ndcg_at_3_std": 0.19946501186770677, - "nauc_ndcg_at_5_diff1": 0.008516418183412523, - "nauc_ndcg_at_5_max": 0.18569505885828344, - "nauc_ndcg_at_5_std": 0.200401149370037, - "nauc_precision_at_1000_diff1": 0.09778580666229689, - "nauc_precision_at_1000_max": 0.19289234929852678, - "nauc_precision_at_1000_std": 0.4244384238930101, - "nauc_precision_at_100_diff1": 0.026851159458335674, - "nauc_precision_at_100_max": 0.1488321813053108, - "nauc_precision_at_100_std": 0.31570240449993014, - "nauc_precision_at_10_diff1": -0.02026084460212222, - "nauc_precision_at_10_max": 0.18235951356296864, - "nauc_precision_at_10_std": 0.24875775274772455, - "nauc_precision_at_1_diff1": -0.02158849186859297, - "nauc_precision_at_1_max": 0.13996579430266687, - "nauc_precision_at_1_std": 0.17219572255475857, - "nauc_precision_at_20_diff1": -0.026345082849132624, - "nauc_precision_at_20_max": 0.14657209751855116, - "nauc_precision_at_20_std": 0.24681362648213306, - "nauc_precision_at_3_diff1": 0.0470327136470777, - "nauc_precision_at_3_max": 0.2057045169689898, - "nauc_precision_at_3_std": 0.20715084952536195, - "nauc_precision_at_5_diff1": 0.0023052267840612817, - "nauc_precision_at_5_max": 0.19417557500790047, - "nauc_precision_at_5_std": 0.20730383037894426, - "nauc_recall_at_1000_diff1": 0.0977858066622949, - "nauc_recall_at_1000_max": 0.19289234929852755, - "nauc_recall_at_1000_std": 0.4244384238930124, - "nauc_recall_at_100_diff1": 0.02685115945833474, - "nauc_recall_at_100_max": 0.14883218130531, - "nauc_recall_at_100_std": 0.3157024044999299, - "nauc_recall_at_10_diff1": -0.020260844602122356, - "nauc_recall_at_10_max": 0.1823595135629684, - "nauc_recall_at_10_std": 0.2487577527477245, - "nauc_recall_at_1_diff1": -0.02158849186859297, - "nauc_recall_at_1_max": 0.13996579430266687, - "nauc_recall_at_1_std": 0.17219572255475857, - "nauc_recall_at_20_diff1": -0.026345082849132815, - "nauc_recall_at_20_max": 0.14657209751855108, - "nauc_recall_at_20_std": 0.24681362648213304, - "nauc_recall_at_3_diff1": 0.04703271364707788, - "nauc_recall_at_3_max": 0.20570451696898984, - "nauc_recall_at_3_std": 0.20715084952536209, - "nauc_recall_at_5_diff1": 0.0023052267840612106, - "nauc_recall_at_5_max": 0.1941755750079004, - "nauc_recall_at_5_std": 0.20730383037894415, - "ndcg_at_1": 0.02886, - "ndcg_at_10": 0.11406, - "ndcg_at_100": 0.17148, - "ndcg_at_1000": 0.22095, - "ndcg_at_20": 0.13338, - "ndcg_at_3": 0.06461, - "ndcg_at_5": 0.08802, - "precision_at_1": 0.02886, - "precision_at_10": 0.02279, - "precision_at_100": 0.00515, - "precision_at_1000": 0.00092, - "precision_at_20": 0.01521, - "precision_at_3": 0.03067, - "precision_at_5": 0.02983, - "recall_at_1": 0.02886, - "recall_at_10": 0.2279, - "recall_at_100": 0.51473, - "recall_at_1000": 0.92183, - "recall_at_20": 0.30427, - "recall_at_3": 0.092, - "recall_at_5": 0.14913 - } - ] - }, - "task_name": "VidoreTatdqaRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/WITT2IRetrieval.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/WITT2IRetrieval.json deleted file mode 100644 index 664c54df0a..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/WITT2IRetrieval.json +++ /dev/null @@ -1,1936 +0,0 @@ -{ - "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", - "evaluation_time": 481.8342795372009, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.14607, - "cv_recall_at_10": 0.34719, - "cv_recall_at_100": 0.68427, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.4427, - "cv_recall_at_3": 0.23146, - "cv_recall_at_5": 0.27303, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.23827, - "map_at_1": 0.14607, - "map_at_10": 0.20461, - "map_at_100": 0.21649, - "map_at_1000": 0.21784, - "map_at_20": 0.21104, - "map_at_3": 0.18521, - "map_at_5": 0.19464, - "mrr_at_1": 0.14606741573033707, - "mrr_at_10": 0.2046107544141251, - "mrr_at_100": 0.2164916033094639, - "mrr_at_1000": 0.21784263443137813, - "mrr_at_20": 0.2110427533722138, - "mrr_at_3": 0.18520599250936332, - "mrr_at_5": 0.19464419475655428, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.26192336539007566, - "nauc_cv_recall_at_100_max": 0.5024390148377479, - "nauc_cv_recall_at_100_std": 0.5638540155133488, - "nauc_cv_recall_at_10_diff1": 0.3454828119738378, - "nauc_cv_recall_at_10_max": 0.3499346539030384, - "nauc_cv_recall_at_10_std": 0.3866041242955454, - "nauc_cv_recall_at_1_diff1": 0.5661591775441839, - "nauc_cv_recall_at_1_max": 0.4545038697008203, - "nauc_cv_recall_at_1_std": 0.4037703592468525, - "nauc_cv_recall_at_20_diff1": 0.3079409062489909, - "nauc_cv_recall_at_20_max": 0.3350925651419665, - "nauc_cv_recall_at_20_std": 0.36270066543579166, - "nauc_cv_recall_at_3_diff1": 0.45254007267605834, - "nauc_cv_recall_at_3_max": 0.3687574992052122, - "nauc_cv_recall_at_3_std": 0.3689113290557241, - "nauc_cv_recall_at_5_diff1": 0.42398776601066207, - "nauc_cv_recall_at_5_max": 0.3608852174216836, - "nauc_cv_recall_at_5_std": 0.3352722872510206, - "nauc_map_at_1000_diff1": 0.47511340162413196, - "nauc_map_at_1000_max": 0.4030154294650597, - "nauc_map_at_1000_std": 0.38565550112838487, - "nauc_map_at_100_diff1": 0.4742826036682546, - "nauc_map_at_100_max": 0.4032604036779924, - "nauc_map_at_100_std": 0.386053147898408, - "nauc_map_at_10_diff1": 0.478007665938371, - "nauc_map_at_10_max": 0.4001804786278926, - "nauc_map_at_10_std": 0.3829642722521861, - "nauc_map_at_1_diff1": 0.5661591775441839, - "nauc_map_at_1_max": 0.4545038697008203, - "nauc_map_at_1_std": 0.4037703592468525, - "nauc_map_at_20_diff1": 0.4741556594644958, - "nauc_map_at_20_max": 0.39843818761588734, - "nauc_map_at_20_std": 0.38073692069410986, - "nauc_map_at_3_diff1": 0.5033510921033137, - "nauc_map_at_3_max": 0.4073517810296481, - "nauc_map_at_3_std": 0.38473170311630295, - "nauc_map_at_5_diff1": 0.4937221583473709, - "nauc_map_at_5_max": 0.403448394535993, - "nauc_map_at_5_std": 0.3745933463443528, - "nauc_mrr_at_1000_diff1": 0.4751133996184114, - "nauc_mrr_at_1000_max": 0.40301542466256235, - "nauc_mrr_at_1000_std": 0.38565549662339377, - "nauc_mrr_at_100_diff1": 0.4742826036682546, - "nauc_mrr_at_100_max": 0.4032604036779924, - "nauc_mrr_at_100_std": 0.386053147898408, - "nauc_mrr_at_10_diff1": 0.478007665938371, - "nauc_mrr_at_10_max": 0.4001804786278926, - "nauc_mrr_at_10_std": 0.3829642722521861, - "nauc_mrr_at_1_diff1": 0.5661591775441839, - "nauc_mrr_at_1_max": 0.4545038697008203, - "nauc_mrr_at_1_std": 0.4037703592468525, - "nauc_mrr_at_20_diff1": 0.4741556594644958, - "nauc_mrr_at_20_max": 0.39843818761588734, - "nauc_mrr_at_20_std": 0.38073692069410986, - "nauc_mrr_at_3_diff1": 0.5033510921033137, - "nauc_mrr_at_3_max": 0.4073517810296481, - "nauc_mrr_at_3_std": 0.38473170311630295, - "nauc_mrr_at_5_diff1": 0.4937221583473709, - "nauc_mrr_at_5_max": 0.403448394535993, - "nauc_mrr_at_5_std": 0.3745933463443528, - "nauc_ndcg_at_1000_diff1": 0.4491703198195501, - "nauc_ndcg_at_1000_max": 0.4015367916914241, - "nauc_ndcg_at_1000_std": 0.3934259930578348, - "nauc_ndcg_at_100_diff1": 0.42706385403119074, - "nauc_ndcg_at_100_max": 0.4125623973619568, - "nauc_ndcg_at_100_std": 0.4119817880505865, - "nauc_ndcg_at_10_diff1": 0.4401322541912091, - "nauc_ndcg_at_10_max": 0.38487416073368114, - "nauc_ndcg_at_10_std": 0.38267665388460514, - "nauc_ndcg_at_1_diff1": 0.5661591775441839, - "nauc_ndcg_at_1_max": 0.4545038697008203, - "nauc_ndcg_at_1_std": 0.4037703592468525, - "nauc_ndcg_at_20_diff1": 0.4279259037817697, - "nauc_ndcg_at_20_max": 0.3797931060000094, - "nauc_ndcg_at_20_std": 0.37526381187135865, - "nauc_ndcg_at_3_diff1": 0.48866305533593757, - "nauc_ndcg_at_3_max": 0.3962060898287809, - "nauc_ndcg_at_3_std": 0.3801685730128606, - "nauc_ndcg_at_5_diff1": 0.47329560550893107, - "nauc_ndcg_at_5_max": 0.39074518984832857, - "nauc_ndcg_at_5_std": 0.3633995164290717, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.261923365390076, - "nauc_precision_at_100_max": 0.5024390148377484, - "nauc_precision_at_100_std": 0.5638540155133486, - "nauc_precision_at_10_diff1": 0.3454828119738378, - "nauc_precision_at_10_max": 0.3499346539030385, - "nauc_precision_at_10_std": 0.38660412429554525, - "nauc_precision_at_1_diff1": 0.5661591775441839, - "nauc_precision_at_1_max": 0.4545038697008203, - "nauc_precision_at_1_std": 0.4037703592468525, - "nauc_precision_at_20_diff1": 0.30794090624899045, - "nauc_precision_at_20_max": 0.33509256514196617, - "nauc_precision_at_20_std": 0.3627006654357914, - "nauc_precision_at_3_diff1": 0.452540072676058, - "nauc_precision_at_3_max": 0.36875749920521234, - "nauc_precision_at_3_std": 0.3689113290557238, - "nauc_precision_at_5_diff1": 0.4239877660106617, - "nauc_precision_at_5_max": 0.3608852174216834, - "nauc_precision_at_5_std": 0.3352722872510205, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.26192336539007566, - "nauc_recall_at_100_max": 0.5024390148377479, - "nauc_recall_at_100_std": 0.5638540155133488, - "nauc_recall_at_10_diff1": 0.3454828119738378, - "nauc_recall_at_10_max": 0.3499346539030384, - "nauc_recall_at_10_std": 0.3866041242955454, - "nauc_recall_at_1_diff1": 0.5661591775441839, - "nauc_recall_at_1_max": 0.4545038697008203, - "nauc_recall_at_1_std": 0.4037703592468525, - "nauc_recall_at_20_diff1": 0.3079409062489909, - "nauc_recall_at_20_max": 0.3350925651419665, - "nauc_recall_at_20_std": 0.36270066543579166, - "nauc_recall_at_3_diff1": 0.45254007267605834, - "nauc_recall_at_3_max": 0.3687574992052122, - "nauc_recall_at_3_std": 0.3689113290557241, - "nauc_recall_at_5_diff1": 0.42398776601066207, - "nauc_recall_at_5_max": 0.3608852174216836, - "nauc_recall_at_5_std": 0.3352722872510206, - "ndcg_at_1": 0.14607, - "ndcg_at_10": 0.23827, - "ndcg_at_100": 0.30546, - "ndcg_at_1000": 0.34486, - "ndcg_at_20": 0.26217, - "ndcg_at_3": 0.19715, - "ndcg_at_5": 0.21422, - "precision_at_1": 0.14607, - "precision_at_10": 0.03472, - "precision_at_100": 0.00684, - "precision_at_1000": 0.001, - "precision_at_20": 0.02213, - "precision_at_3": 0.07715, - "precision_at_5": 0.05461, - "recall_at_1": 0.14607, - "recall_at_10": 0.34719, - "recall_at_100": 0.68427, - "recall_at_1000": 1.0, - "recall_at_20": 0.4427, - "recall_at_3": 0.23146, - "recall_at_5": 0.27303 - }, - { - "cv_recall_at_1": 0.16744, - "cv_recall_at_10": 0.36512, - "cv_recall_at_100": 0.75698, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.47442, - "cv_recall_at_3": 0.25116, - "cv_recall_at_5": 0.28256, - "hf_subset": "bg", - "languages": [ - "bul-Cyrl" - ], - "main_score": 0.25512, - "map_at_1": 0.16744, - "map_at_10": 0.2215, - "map_at_100": 0.23559, - "map_at_1000": 0.23685, - "map_at_20": 0.22923, - "map_at_3": 0.20368, - "map_at_5": 0.21072, - "mrr_at_1": 0.16744186046511628, - "mrr_at_10": 0.22149686231081575, - "mrr_at_100": 0.2355936212080499, - "mrr_at_1000": 0.23685488711895378, - "mrr_at_20": 0.22923039957338684, - "mrr_at_3": 0.20368217054263568, - "mrr_at_5": 0.21071705426356593, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.3155611666828518, - "nauc_cv_recall_at_100_max": 0.45741229085892576, - "nauc_cv_recall_at_100_std": 0.3904921274057909, - "nauc_cv_recall_at_10_diff1": 0.4543116227676328, - "nauc_cv_recall_at_10_max": 0.3076640924878011, - "nauc_cv_recall_at_10_std": 0.09878880673568347, - "nauc_cv_recall_at_1_diff1": 0.6522816689962211, - "nauc_cv_recall_at_1_max": 0.33478067677865797, - "nauc_cv_recall_at_1_std": 0.08616115338820735, - "nauc_cv_recall_at_20_diff1": 0.3758814279747424, - "nauc_cv_recall_at_20_max": 0.297032013955431, - "nauc_cv_recall_at_20_std": 0.11695658583327197, - "nauc_cv_recall_at_3_diff1": 0.5426482437452204, - "nauc_cv_recall_at_3_max": 0.2972351961105648, - "nauc_cv_recall_at_3_std": 0.09761314596307219, - "nauc_cv_recall_at_5_diff1": 0.4755336920118826, - "nauc_cv_recall_at_5_max": 0.27284958625831, - "nauc_cv_recall_at_5_std": 0.08428460770948648, - "nauc_map_at_1000_diff1": 0.5679377206906693, - "nauc_map_at_1000_max": 0.31379566050626934, - "nauc_map_at_1000_std": 0.09487094415683674, - "nauc_map_at_100_diff1": 0.5672953157822261, - "nauc_map_at_100_max": 0.31403082416608186, - "nauc_map_at_100_std": 0.09546910746381114, - "nauc_map_at_10_diff1": 0.5746252363701051, - "nauc_map_at_10_max": 0.31445558548372626, - "nauc_map_at_10_std": 0.09096313225792232, - "nauc_map_at_1_diff1": 0.6522816689962211, - "nauc_map_at_1_max": 0.33478067677865797, - "nauc_map_at_1_std": 0.08616115338820735, - "nauc_map_at_20_diff1": 0.5670389052295799, - "nauc_map_at_20_max": 0.3130057286872424, - "nauc_map_at_20_std": 0.09273511502840416, - "nauc_map_at_3_diff1": 0.598818834786234, - "nauc_map_at_3_max": 0.31446259263387427, - "nauc_map_at_3_std": 0.09122519390838411, - "nauc_map_at_5_diff1": 0.5801340704820455, - "nauc_map_at_5_max": 0.3080495979250753, - "nauc_map_at_5_std": 0.0880409321335613, - "nauc_mrr_at_1000_diff1": 0.5679381592464301, - "nauc_mrr_at_1000_max": 0.31379554761149336, - "nauc_mrr_at_1000_std": 0.09487076381672727, - "nauc_mrr_at_100_diff1": 0.5672953157822261, - "nauc_mrr_at_100_max": 0.31403082416608186, - "nauc_mrr_at_100_std": 0.09546910746381114, - "nauc_mrr_at_10_diff1": 0.5746252363701051, - "nauc_mrr_at_10_max": 0.31445558548372626, - "nauc_mrr_at_10_std": 0.09096313225792232, - "nauc_mrr_at_1_diff1": 0.6522816689962211, - "nauc_mrr_at_1_max": 0.33478067677865797, - "nauc_mrr_at_1_std": 0.08616115338820735, - "nauc_mrr_at_20_diff1": 0.5670389052295799, - "nauc_mrr_at_20_max": 0.3130057286872424, - "nauc_mrr_at_20_std": 0.09273511502840416, - "nauc_mrr_at_3_diff1": 0.598818834786234, - "nauc_mrr_at_3_max": 0.31446259263387427, - "nauc_mrr_at_3_std": 0.09122519390838411, - "nauc_mrr_at_5_diff1": 0.5801340704820455, - "nauc_mrr_at_5_max": 0.3080495979250753, - "nauc_mrr_at_5_std": 0.0880409321335613, - "nauc_ndcg_at_1000_diff1": 0.5396722787543929, - "nauc_ndcg_at_1000_max": 0.3146022960220083, - "nauc_ndcg_at_1000_std": 0.10577477970824188, - "nauc_ndcg_at_100_diff1": 0.5190228546300492, - "nauc_ndcg_at_100_max": 0.3272980019602004, - "nauc_ndcg_at_100_std": 0.13152036231548603, - "nauc_ndcg_at_10_diff1": 0.5404999650445497, - "nauc_ndcg_at_10_max": 0.311839837720339, - "nauc_ndcg_at_10_std": 0.0931436541198744, - "nauc_ndcg_at_1_diff1": 0.6522816689962211, - "nauc_ndcg_at_1_max": 0.33478067677865797, - "nauc_ndcg_at_1_std": 0.08616115338820735, - "nauc_ndcg_at_20_diff1": 0.5165761612528892, - "nauc_ndcg_at_20_max": 0.30809471053636023, - "nauc_ndcg_at_20_std": 0.09875190211945468, - "nauc_ndcg_at_3_diff1": 0.5831062669245733, - "nauc_ndcg_at_3_max": 0.30956315539138013, - "nauc_ndcg_at_3_std": 0.09299125335033034, - "nauc_ndcg_at_5_diff1": 0.5511165710586254, - "nauc_ndcg_at_5_max": 0.29827585834017506, - "nauc_ndcg_at_5_std": 0.0871820714733284, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3155611666828535, - "nauc_precision_at_100_max": 0.4574122908589268, - "nauc_precision_at_100_std": 0.39049212740579214, - "nauc_precision_at_10_diff1": 0.45431162276763287, - "nauc_precision_at_10_max": 0.30766409248780135, - "nauc_precision_at_10_std": 0.09878880673568358, - "nauc_precision_at_1_diff1": 0.6522816689962211, - "nauc_precision_at_1_max": 0.33478067677865797, - "nauc_precision_at_1_std": 0.08616115338820735, - "nauc_precision_at_20_diff1": 0.3758814279747427, - "nauc_precision_at_20_max": 0.29703201395543116, - "nauc_precision_at_20_std": 0.1169565858332718, - "nauc_precision_at_3_diff1": 0.54264824374522, - "nauc_precision_at_3_max": 0.2972351961105648, - "nauc_precision_at_3_std": 0.0976131459630722, - "nauc_precision_at_5_diff1": 0.4755336920118825, - "nauc_precision_at_5_max": 0.27284958625830985, - "nauc_precision_at_5_std": 0.08428460770948652, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.3155611666828518, - "nauc_recall_at_100_max": 0.45741229085892576, - "nauc_recall_at_100_std": 0.3904921274057909, - "nauc_recall_at_10_diff1": 0.4543116227676328, - "nauc_recall_at_10_max": 0.3076640924878011, - "nauc_recall_at_10_std": 0.09878880673568347, - "nauc_recall_at_1_diff1": 0.6522816689962211, - "nauc_recall_at_1_max": 0.33478067677865797, - "nauc_recall_at_1_std": 0.08616115338820735, - "nauc_recall_at_20_diff1": 0.3758814279747424, - "nauc_recall_at_20_max": 0.297032013955431, - "nauc_recall_at_20_std": 0.11695658583327197, - "nauc_recall_at_3_diff1": 0.5426482437452204, - "nauc_recall_at_3_max": 0.2972351961105648, - "nauc_recall_at_3_std": 0.09761314596307219, - "nauc_recall_at_5_diff1": 0.4755336920118826, - "nauc_recall_at_5_max": 0.27284958625831, - "nauc_recall_at_5_std": 0.08428460770948648, - "ndcg_at_1": 0.16744, - "ndcg_at_10": 0.25512, - "ndcg_at_100": 0.33361, - "ndcg_at_1000": 0.36507, - "ndcg_at_20": 0.28294, - "ndcg_at_3": 0.21585, - "ndcg_at_5": 0.22866, - "precision_at_1": 0.16744, - "precision_at_10": 0.03651, - "precision_at_100": 0.00757, - "precision_at_1000": 0.001, - "precision_at_20": 0.02372, - "precision_at_3": 0.08372, - "precision_at_5": 0.05651, - "recall_at_1": 0.16744, - "recall_at_10": 0.36512, - "recall_at_100": 0.75698, - "recall_at_1000": 1.0, - "recall_at_20": 0.47442, - "recall_at_3": 0.25116, - "recall_at_5": 0.28256 - }, - { - "cv_recall_at_1": 0.50842, - "cv_recall_at_10": 0.74186, - "cv_recall_at_100": 0.90685, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.79798, - "cv_recall_at_3": 0.63749, - "cv_recall_at_5": 0.6835, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.62154, - "map_at_1": 0.50842, - "map_at_10": 0.58334, - "map_at_100": 0.58977, - "map_at_1000": 0.59026, - "map_at_20": 0.58734, - "map_at_3": 0.56472, - "map_at_5": 0.57516, - "mrr_at_1": 0.5084175084175084, - "mrr_at_10": 0.5833409046372006, - "mrr_at_100": 0.5897669028137018, - "mrr_at_1000": 0.590260080374905, - "mrr_at_20": 0.587342077177479, - "mrr_at_3": 0.5647212869435088, - "mrr_at_5": 0.5751589973812189, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.6977811922414636, - "nauc_cv_recall_at_100_max": 0.6903446325672074, - "nauc_cv_recall_at_100_std": 0.5508802823143485, - "nauc_cv_recall_at_10_diff1": 0.7102384212164595, - "nauc_cv_recall_at_10_max": 0.531658928303646, - "nauc_cv_recall_at_10_std": 0.27018476526472457, - "nauc_cv_recall_at_1_diff1": 0.8146433011443713, - "nauc_cv_recall_at_1_max": 0.49933124251165467, - "nauc_cv_recall_at_1_std": 0.020398181485709738, - "nauc_cv_recall_at_20_diff1": 0.680292572444529, - "nauc_cv_recall_at_20_max": 0.5617293068211144, - "nauc_cv_recall_at_20_std": 0.38890792758037185, - "nauc_cv_recall_at_3_diff1": 0.7357291583736335, - "nauc_cv_recall_at_3_max": 0.5065655310271268, - "nauc_cv_recall_at_3_std": 0.09673314934363086, - "nauc_cv_recall_at_5_diff1": 0.7182209149801598, - "nauc_cv_recall_at_5_max": 0.5124711032629541, - "nauc_cv_recall_at_5_std": 0.1370912516418801, - "nauc_map_at_1000_diff1": 0.7810117882647332, - "nauc_map_at_1000_max": 0.5087532878090155, - "nauc_map_at_1000_std": 0.07156532369375307, - "nauc_map_at_100_diff1": 0.7808851016482755, - "nauc_map_at_100_max": 0.5089571032641548, - "nauc_map_at_100_std": 0.07219028700236066, - "nauc_map_at_10_diff1": 0.7805002652086618, - "nauc_map_at_10_max": 0.5063502982591215, - "nauc_map_at_10_std": 0.06890074443620031, - "nauc_map_at_1_diff1": 0.8146433011443713, - "nauc_map_at_1_max": 0.49933124251165467, - "nauc_map_at_1_std": 0.020398181485709738, - "nauc_map_at_20_diff1": 0.7798997218692829, - "nauc_map_at_20_max": 0.5073921256789337, - "nauc_map_at_20_std": 0.07219065379286495, - "nauc_map_at_3_diff1": 0.782875759866369, - "nauc_map_at_3_max": 0.5034365061077172, - "nauc_map_at_3_std": 0.0503649362897909, - "nauc_map_at_5_diff1": 0.7805620961748264, - "nauc_map_at_5_max": 0.5043784361078906, - "nauc_map_at_5_std": 0.05651909052654164, - "nauc_mrr_at_1000_diff1": 0.7810117882647332, - "nauc_mrr_at_1000_max": 0.5087532878090155, - "nauc_mrr_at_1000_std": 0.07156532369375307, - "nauc_mrr_at_100_diff1": 0.7808851016482755, - "nauc_mrr_at_100_max": 0.5089571032641548, - "nauc_mrr_at_100_std": 0.07219028700236066, - "nauc_mrr_at_10_diff1": 0.7805002652086618, - "nauc_mrr_at_10_max": 0.5063502982591215, - "nauc_mrr_at_10_std": 0.06890074443620031, - "nauc_mrr_at_1_diff1": 0.8146433011443713, - "nauc_mrr_at_1_max": 0.49933124251165467, - "nauc_mrr_at_1_std": 0.020398181485709738, - "nauc_mrr_at_20_diff1": 0.7798997218692829, - "nauc_mrr_at_20_max": 0.5073921256789337, - "nauc_mrr_at_20_std": 0.07219065379286495, - "nauc_mrr_at_3_diff1": 0.782875759866369, - "nauc_mrr_at_3_max": 0.5034365061077172, - "nauc_mrr_at_3_std": 0.0503649362897909, - "nauc_mrr_at_5_diff1": 0.7805620961748264, - "nauc_mrr_at_5_max": 0.5043784361078906, - "nauc_mrr_at_5_std": 0.05651909052654164, - "nauc_ndcg_at_1000_diff1": 0.7738005617196608, - "nauc_ndcg_at_1000_max": 0.5153371875544834, - "nauc_ndcg_at_1000_std": 0.09528500134059216, - "nauc_ndcg_at_100_diff1": 0.7703938756539106, - "nauc_ndcg_at_100_max": 0.5228755704005162, - "nauc_ndcg_at_100_std": 0.11535325549845764, - "nauc_ndcg_at_10_diff1": 0.7664599452451597, - "nauc_ndcg_at_10_max": 0.5109745148698641, - "nauc_ndcg_at_10_std": 0.10569364055562012, - "nauc_ndcg_at_1_diff1": 0.8146433011443713, - "nauc_ndcg_at_1_max": 0.49933124251165467, - "nauc_ndcg_at_1_std": 0.020398181485709738, - "nauc_ndcg_at_20_diff1": 0.7634906866295377, - "nauc_ndcg_at_20_max": 0.5152927534908216, - "nauc_ndcg_at_20_std": 0.11988886514932286, - "nauc_ndcg_at_3_diff1": 0.7716381069994489, - "nauc_ndcg_at_3_max": 0.504234251640214, - "nauc_ndcg_at_3_std": 0.061383868607826936, - "nauc_ndcg_at_5_diff1": 0.7667298291443791, - "nauc_ndcg_at_5_max": 0.5061599949153522, - "nauc_ndcg_at_5_std": 0.07403612092966083, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.6977811922414628, - "nauc_precision_at_100_max": 0.6903446325672082, - "nauc_precision_at_100_std": 0.5508802823143504, - "nauc_precision_at_10_diff1": 0.7102384212164586, - "nauc_precision_at_10_max": 0.5316589283036451, - "nauc_precision_at_10_std": 0.27018476526472307, - "nauc_precision_at_1_diff1": 0.8146433011443713, - "nauc_precision_at_1_max": 0.49933124251165467, - "nauc_precision_at_1_std": 0.020398181485709738, - "nauc_precision_at_20_diff1": 0.6802925724445281, - "nauc_precision_at_20_max": 0.5617293068211142, - "nauc_precision_at_20_std": 0.38890792758037157, - "nauc_precision_at_3_diff1": 0.7357291583736333, - "nauc_precision_at_3_max": 0.5065655310271261, - "nauc_precision_at_3_std": 0.09673314934363113, - "nauc_precision_at_5_diff1": 0.7182209149801593, - "nauc_precision_at_5_max": 0.5124711032629536, - "nauc_precision_at_5_std": 0.1370912516418799, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.6977811922414636, - "nauc_recall_at_100_max": 0.6903446325672074, - "nauc_recall_at_100_std": 0.5508802823143485, - "nauc_recall_at_10_diff1": 0.7102384212164595, - "nauc_recall_at_10_max": 0.531658928303646, - "nauc_recall_at_10_std": 0.27018476526472457, - "nauc_recall_at_1_diff1": 0.8146433011443713, - "nauc_recall_at_1_max": 0.49933124251165467, - "nauc_recall_at_1_std": 0.020398181485709738, - "nauc_recall_at_20_diff1": 0.680292572444529, - "nauc_recall_at_20_max": 0.5617293068211144, - "nauc_recall_at_20_std": 0.38890792758037185, - "nauc_recall_at_3_diff1": 0.7357291583736335, - "nauc_recall_at_3_max": 0.5065655310271268, - "nauc_recall_at_3_std": 0.09673314934363086, - "nauc_recall_at_5_diff1": 0.7182209149801598, - "nauc_recall_at_5_max": 0.5124711032629541, - "nauc_recall_at_5_std": 0.1370912516418801, - "ndcg_at_1": 0.50842, - "ndcg_at_10": 0.62154, - "ndcg_at_100": 0.65532, - "ndcg_at_1000": 0.66746, - "ndcg_at_20": 0.63586, - "ndcg_at_3": 0.58338, - "ndcg_at_5": 0.60227, - "precision_at_1": 0.50842, - "precision_at_10": 0.07419, - "precision_at_100": 0.00907, - "precision_at_1000": 0.001, - "precision_at_20": 0.0399, - "precision_at_3": 0.2125, - "precision_at_5": 0.1367, - "recall_at_1": 0.50842, - "recall_at_10": 0.74186, - "recall_at_100": 0.90685, - "recall_at_1000": 1.0, - "recall_at_20": 0.79798, - "recall_at_3": 0.63749, - "recall_at_5": 0.6835 - }, - { - "cv_recall_at_1": 0.08246, - "cv_recall_at_10": 0.19649, - "cv_recall_at_100": 0.48596, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.23333, - "cv_recall_at_3": 0.1193, - "cv_recall_at_5": 0.15088, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.13156, - "map_at_1": 0.08246, - "map_at_10": 0.1118, - "map_at_100": 0.11956, - "map_at_1000": 0.12204, - "map_at_20": 0.11438, - "map_at_3": 0.09854, - "map_at_5": 0.10591, - "mrr_at_1": 0.0824561403508772, - "mrr_at_10": 0.11179824561403513, - "mrr_at_100": 0.1195605084094835, - "mrr_at_1000": 0.12204457182360474, - "mrr_at_20": 0.11438025632114773, - "mrr_at_3": 0.09853801169590644, - "mrr_at_5": 0.10590643274853803, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.23848481817399358, - "nauc_cv_recall_at_100_max": 0.2780996955027928, - "nauc_cv_recall_at_100_std": 0.4483157280135105, - "nauc_cv_recall_at_10_diff1": 0.41543817720122717, - "nauc_cv_recall_at_10_max": 0.3404936172168539, - "nauc_cv_recall_at_10_std": 0.40725431168204446, - "nauc_cv_recall_at_1_diff1": 0.7881759900439002, - "nauc_cv_recall_at_1_max": 0.5271366508914248, - "nauc_cv_recall_at_1_std": 0.4374689585486017, - "nauc_cv_recall_at_20_diff1": 0.37609410863557907, - "nauc_cv_recall_at_20_max": 0.3282897728935001, - "nauc_cv_recall_at_20_std": 0.4261219018540338, - "nauc_cv_recall_at_3_diff1": 0.5818612882759543, - "nauc_cv_recall_at_3_max": 0.40298276955955536, - "nauc_cv_recall_at_3_std": 0.38599024944067867, - "nauc_cv_recall_at_5_diff1": 0.49318266650703224, - "nauc_cv_recall_at_5_max": 0.40355374636420616, - "nauc_cv_recall_at_5_std": 0.41270925622046467, - "nauc_map_at_1000_diff1": 0.619423325807602, - "nauc_map_at_1000_max": 0.4443675920403044, - "nauc_map_at_1000_std": 0.4262573861773211, - "nauc_map_at_100_diff1": 0.616446963844894, - "nauc_map_at_100_max": 0.4428822327863043, - "nauc_map_at_100_std": 0.4261337543043992, - "nauc_map_at_10_diff1": 0.6301642227971276, - "nauc_map_at_10_max": 0.44727900681909044, - "nauc_map_at_10_std": 0.4228057973690992, - "nauc_map_at_1_diff1": 0.7881759900439002, - "nauc_map_at_1_max": 0.5271366508914248, - "nauc_map_at_1_std": 0.4374689585486017, - "nauc_map_at_20_diff1": 0.6241634212559133, - "nauc_map_at_20_max": 0.4459319580120189, - "nauc_map_at_20_std": 0.42552264032079873, - "nauc_map_at_3_diff1": 0.6860315933291902, - "nauc_map_at_3_max": 0.4627361884329836, - "nauc_map_at_3_std": 0.4158925405410705, - "nauc_map_at_5_diff1": 0.653263228710077, - "nauc_map_at_5_max": 0.4607797632861452, - "nauc_map_at_5_std": 0.42253211669153967, - "nauc_mrr_at_1000_diff1": 0.619423325807602, - "nauc_mrr_at_1000_max": 0.4443675920403044, - "nauc_mrr_at_1000_std": 0.4262573861773211, - "nauc_mrr_at_100_diff1": 0.616446963844894, - "nauc_mrr_at_100_max": 0.4428822327863043, - "nauc_mrr_at_100_std": 0.4261337543043992, - "nauc_mrr_at_10_diff1": 0.6301642227971276, - "nauc_mrr_at_10_max": 0.44727900681909044, - "nauc_mrr_at_10_std": 0.4228057973690992, - "nauc_mrr_at_1_diff1": 0.7881759900439002, - "nauc_mrr_at_1_max": 0.5271366508914248, - "nauc_mrr_at_1_std": 0.4374689585486017, - "nauc_mrr_at_20_diff1": 0.6241634212559133, - "nauc_mrr_at_20_max": 0.4459319580120189, - "nauc_mrr_at_20_std": 0.42552264032079873, - "nauc_mrr_at_3_diff1": 0.6860315933291902, - "nauc_mrr_at_3_max": 0.4627361884329836, - "nauc_mrr_at_3_std": 0.4158925405410705, - "nauc_mrr_at_5_diff1": 0.653263228710077, - "nauc_mrr_at_5_max": 0.4607797632861452, - "nauc_mrr_at_5_std": 0.42253211669153967, - "nauc_ndcg_at_1000_diff1": 0.5601778359428754, - "nauc_ndcg_at_1000_max": 0.4189585137835097, - "nauc_ndcg_at_1000_std": 0.4282972488935674, - "nauc_ndcg_at_100_diff1": 0.49652500435897334, - "nauc_ndcg_at_100_max": 0.3904312925139352, - "nauc_ndcg_at_100_std": 0.43115952849055644, - "nauc_ndcg_at_10_diff1": 0.5601850127764495, - "nauc_ndcg_at_10_max": 0.4132393273356581, - "nauc_ndcg_at_10_std": 0.41786479744859456, - "nauc_ndcg_at_1_diff1": 0.7881759900439002, - "nauc_ndcg_at_1_max": 0.5271366508914248, - "nauc_ndcg_at_1_std": 0.4374689585486017, - "nauc_ndcg_at_20_diff1": 0.5422695603581896, - "nauc_ndcg_at_20_max": 0.4078713890969411, - "nauc_ndcg_at_20_std": 0.4252348047801439, - "nauc_ndcg_at_3_diff1": 0.6561678022728696, - "nauc_ndcg_at_3_max": 0.4454927391585425, - "nauc_ndcg_at_3_std": 0.4074709868295361, - "nauc_ndcg_at_5_diff1": 0.6047492469394583, - "nauc_ndcg_at_5_max": 0.44300596922196894, - "nauc_ndcg_at_5_std": 0.4192443928941932, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.2384848181739936, - "nauc_precision_at_100_max": 0.27809969550279284, - "nauc_precision_at_100_std": 0.44831572801351055, - "nauc_precision_at_10_diff1": 0.41543817720122694, - "nauc_precision_at_10_max": 0.3404936172168537, - "nauc_precision_at_10_std": 0.40725431168204423, - "nauc_precision_at_1_diff1": 0.7881759900439002, - "nauc_precision_at_1_max": 0.5271366508914248, - "nauc_precision_at_1_std": 0.4374689585486017, - "nauc_precision_at_20_diff1": 0.37609410863557896, - "nauc_precision_at_20_max": 0.32828977289350025, - "nauc_precision_at_20_std": 0.42612190185403387, - "nauc_precision_at_3_diff1": 0.5818612882759547, - "nauc_precision_at_3_max": 0.4029827695595554, - "nauc_precision_at_3_std": 0.38599024944067883, - "nauc_precision_at_5_diff1": 0.49318266650703235, - "nauc_precision_at_5_max": 0.40355374636420643, - "nauc_precision_at_5_std": 0.4127092562204649, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.23848481817399358, - "nauc_recall_at_100_max": 0.2780996955027928, - "nauc_recall_at_100_std": 0.4483157280135105, - "nauc_recall_at_10_diff1": 0.41543817720122717, - "nauc_recall_at_10_max": 0.3404936172168539, - "nauc_recall_at_10_std": 0.40725431168204446, - "nauc_recall_at_1_diff1": 0.7881759900439002, - "nauc_recall_at_1_max": 0.5271366508914248, - "nauc_recall_at_1_std": 0.4374689585486017, - "nauc_recall_at_20_diff1": 0.37609410863557907, - "nauc_recall_at_20_max": 0.3282897728935001, - "nauc_recall_at_20_std": 0.4261219018540338, - "nauc_recall_at_3_diff1": 0.5818612882759543, - "nauc_recall_at_3_max": 0.40298276955955536, - "nauc_recall_at_3_std": 0.38599024944067867, - "nauc_recall_at_5_diff1": 0.49318266650703224, - "nauc_recall_at_5_max": 0.40355374636420616, - "nauc_recall_at_5_std": 0.41270925622046467, - "ndcg_at_1": 0.08246, - "ndcg_at_10": 0.13156, - "ndcg_at_100": 0.18517, - "ndcg_at_1000": 0.25104, - "ndcg_at_20": 0.14091, - "ndcg_at_3": 0.10386, - "ndcg_at_5": 0.117, - "precision_at_1": 0.08246, - "precision_at_10": 0.01965, - "precision_at_100": 0.00486, - "precision_at_1000": 0.001, - "precision_at_20": 0.01167, - "precision_at_3": 0.03977, - "precision_at_5": 0.03018, - "recall_at_1": 0.08246, - "recall_at_10": 0.19649, - "recall_at_100": 0.48596, - "recall_at_1000": 1.0, - "recall_at_20": 0.23333, - "recall_at_3": 0.1193, - "recall_at_5": 0.15088 - }, - { - "cv_recall_at_1": 0.28146, - "cv_recall_at_10": 0.48513, - "cv_recall_at_100": 0.77918, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.56407, - "cv_recall_at_3": 0.35812, - "cv_recall_at_5": 0.40046, - "hf_subset": "et", - "languages": [ - "est-Latn" - ], - "main_score": 0.37248, - "map_at_1": 0.28146, - "map_at_10": 0.33788, - "map_at_100": 0.34847, - "map_at_1000": 0.34953, - "map_at_20": 0.34331, - "map_at_3": 0.31636, - "map_at_5": 0.3262, - "mrr_at_1": 0.2814645308924485, - "mrr_at_10": 0.3378822963205114, - "mrr_at_100": 0.34847209578763483, - "mrr_at_1000": 0.3495290815761699, - "mrr_at_20": 0.34331177017418185, - "mrr_at_3": 0.3163615560640733, - "mrr_at_5": 0.3262013729977116, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.34932998468795345, - "nauc_cv_recall_at_100_max": 0.5753714600936997, - "nauc_cv_recall_at_100_std": 0.4360491728305647, - "nauc_cv_recall_at_10_diff1": 0.5166151838980909, - "nauc_cv_recall_at_10_max": 0.46419054224541006, - "nauc_cv_recall_at_10_std": 0.1876105520707209, - "nauc_cv_recall_at_1_diff1": 0.7440892508386078, - "nauc_cv_recall_at_1_max": 0.4548291267176457, - "nauc_cv_recall_at_1_std": 0.006599870432398388, - "nauc_cv_recall_at_20_diff1": 0.47758396896747074, - "nauc_cv_recall_at_20_max": 0.5097760248029565, - "nauc_cv_recall_at_20_std": 0.2613282694411242, - "nauc_cv_recall_at_3_diff1": 0.6143055626966617, - "nauc_cv_recall_at_3_max": 0.42541096805824885, - "nauc_cv_recall_at_3_std": 0.08912464881100893, - "nauc_cv_recall_at_5_diff1": 0.5908371774719564, - "nauc_cv_recall_at_5_max": 0.4486943218864718, - "nauc_cv_recall_at_5_std": 0.09565149785356179, - "nauc_map_at_1000_diff1": 0.6635556751610107, - "nauc_map_at_1000_max": 0.4516694667637394, - "nauc_map_at_1000_std": 0.06241952508490272, - "nauc_map_at_100_diff1": 0.6626921889571289, - "nauc_map_at_100_max": 0.4519587940605559, - "nauc_map_at_100_std": 0.06340615907295671, - "nauc_map_at_10_diff1": 0.6659223965757382, - "nauc_map_at_10_max": 0.4477204335728731, - "nauc_map_at_10_std": 0.05723630290668432, - "nauc_map_at_1_diff1": 0.7440892508386078, - "nauc_map_at_1_max": 0.4548291267176457, - "nauc_map_at_1_std": 0.006599870432398388, - "nauc_map_at_20_diff1": 0.6640183038107316, - "nauc_map_at_20_max": 0.45117770525747186, - "nauc_map_at_20_std": 0.06228760382026663, - "nauc_map_at_3_diff1": 0.6818687659705938, - "nauc_map_at_3_max": 0.4392047808874562, - "nauc_map_at_3_std": 0.04329965866604979, - "nauc_map_at_5_diff1": 0.6757236325559439, - "nauc_map_at_5_max": 0.44526383671545794, - "nauc_map_at_5_std": 0.044908921967325065, - "nauc_mrr_at_1000_diff1": 0.6635556751610107, - "nauc_mrr_at_1000_max": 0.4516694667637394, - "nauc_mrr_at_1000_std": 0.06241952508490272, - "nauc_mrr_at_100_diff1": 0.6626921889571289, - "nauc_mrr_at_100_max": 0.4519587940605559, - "nauc_mrr_at_100_std": 0.06340615907295671, - "nauc_mrr_at_10_diff1": 0.6659223965757382, - "nauc_mrr_at_10_max": 0.4477204335728731, - "nauc_mrr_at_10_std": 0.05723630290668432, - "nauc_mrr_at_1_diff1": 0.7440892508386078, - "nauc_mrr_at_1_max": 0.4548291267176457, - "nauc_mrr_at_1_std": 0.006599870432398388, - "nauc_mrr_at_20_diff1": 0.6640183038107316, - "nauc_mrr_at_20_max": 0.45117770525747186, - "nauc_mrr_at_20_std": 0.06228760382026663, - "nauc_mrr_at_3_diff1": 0.6818687659705938, - "nauc_mrr_at_3_max": 0.4392047808874562, - "nauc_mrr_at_3_std": 0.04329965866604979, - "nauc_mrr_at_5_diff1": 0.6757236325559439, - "nauc_mrr_at_5_max": 0.44526383671545794, - "nauc_mrr_at_5_std": 0.044908921967325065, - "nauc_ndcg_at_1000_diff1": 0.6383749462925029, - "nauc_ndcg_at_1000_max": 0.45848340658237796, - "nauc_ndcg_at_1000_std": 0.08702927051969854, - "nauc_ndcg_at_100_diff1": 0.6151065936759832, - "nauc_ndcg_at_100_max": 0.4675296810665792, - "nauc_ndcg_at_100_std": 0.11488815515095756, - "nauc_ndcg_at_10_diff1": 0.6295784200068598, - "nauc_ndcg_at_10_max": 0.451212145699924, - "nauc_ndcg_at_10_std": 0.08806375022143757, - "nauc_ndcg_at_1_diff1": 0.7440892508386078, - "nauc_ndcg_at_1_max": 0.4548291267176457, - "nauc_ndcg_at_1_std": 0.006599870432398388, - "nauc_ndcg_at_20_diff1": 0.622080375563178, - "nauc_ndcg_at_20_max": 0.4626560051729655, - "nauc_ndcg_at_20_std": 0.10512648478325441, - "nauc_ndcg_at_3_diff1": 0.663987564213784, - "nauc_ndcg_at_3_max": 0.43548729740830583, - "nauc_ndcg_at_3_std": 0.0553017963802168, - "nauc_ndcg_at_5_diff1": 0.6533851232485005, - "nauc_ndcg_at_5_max": 0.44580433876900977, - "nauc_ndcg_at_5_std": 0.05822191398076574, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.349329984687954, - "nauc_precision_at_100_max": 0.5753714600937007, - "nauc_precision_at_100_std": 0.4360491728305648, - "nauc_precision_at_10_diff1": 0.5166151838980908, - "nauc_precision_at_10_max": 0.4641905422454098, - "nauc_precision_at_10_std": 0.18761055207072072, - "nauc_precision_at_1_diff1": 0.7440892508386078, - "nauc_precision_at_1_max": 0.4548291267176457, - "nauc_precision_at_1_std": 0.006599870432398388, - "nauc_precision_at_20_diff1": 0.47758396896747074, - "nauc_precision_at_20_max": 0.509776024802956, - "nauc_precision_at_20_std": 0.26132826944112364, - "nauc_precision_at_3_diff1": 0.6143055626966614, - "nauc_precision_at_3_max": 0.42541096805824835, - "nauc_precision_at_3_std": 0.08912464881100866, - "nauc_precision_at_5_diff1": 0.5908371774719563, - "nauc_precision_at_5_max": 0.44869432188647174, - "nauc_precision_at_5_std": 0.09565149785356195, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.34932998468795345, - "nauc_recall_at_100_max": 0.5753714600936997, - "nauc_recall_at_100_std": 0.4360491728305647, - "nauc_recall_at_10_diff1": 0.5166151838980909, - "nauc_recall_at_10_max": 0.46419054224541006, - "nauc_recall_at_10_std": 0.1876105520707209, - "nauc_recall_at_1_diff1": 0.7440892508386078, - "nauc_recall_at_1_max": 0.4548291267176457, - "nauc_recall_at_1_std": 0.006599870432398388, - "nauc_recall_at_20_diff1": 0.47758396896747074, - "nauc_recall_at_20_max": 0.5097760248029565, - "nauc_recall_at_20_std": 0.2613282694411242, - "nauc_recall_at_3_diff1": 0.6143055626966617, - "nauc_recall_at_3_max": 0.42541096805824885, - "nauc_recall_at_3_std": 0.08912464881100893, - "nauc_recall_at_5_diff1": 0.5908371774719564, - "nauc_recall_at_5_max": 0.4486943218864718, - "nauc_recall_at_5_std": 0.09565149785356179, - "ndcg_at_1": 0.28146, - "ndcg_at_10": 0.37248, - "ndcg_at_100": 0.43161, - "ndcg_at_1000": 0.45984, - "ndcg_at_20": 0.39237, - "ndcg_at_3": 0.32713, - "ndcg_at_5": 0.34471, - "precision_at_1": 0.28146, - "precision_at_10": 0.04851, - "precision_at_100": 0.00779, - "precision_at_1000": 0.001, - "precision_at_20": 0.0282, - "precision_at_3": 0.11937, - "precision_at_5": 0.08009, - "recall_at_1": 0.28146, - "recall_at_10": 0.48513, - "recall_at_100": 0.77918, - "recall_at_1000": 1.0, - "recall_at_20": 0.56407, - "recall_at_3": 0.35812, - "recall_at_5": 0.40046 - }, - { - "cv_recall_at_1": 0.56493, - "cv_recall_at_10": 0.78579, - "cv_recall_at_100": 0.9545, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.84906, - "cv_recall_at_3": 0.67259, - "cv_recall_at_5": 0.72253, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.66955, - "map_at_1": 0.56493, - "map_at_10": 0.63306, - "map_at_100": 0.64021, - "map_at_1000": 0.64046, - "map_at_20": 0.63745, - "map_at_3": 0.61302, - "map_at_5": 0.62445, - "mrr_at_1": 0.564927857935627, - "mrr_at_10": 0.6330602681324101, - "mrr_at_100": 0.6402059570398387, - "mrr_at_1000": 0.6404573779066952, - "mrr_at_20": 0.6374518773738077, - "mrr_at_3": 0.6130225675175731, - "mrr_at_5": 0.6244543100258969, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.7536092845128153, - "nauc_cv_recall_at_100_max": 0.7685517696368389, - "nauc_cv_recall_at_100_std": 0.7416747448364003, - "nauc_cv_recall_at_10_diff1": 0.7446677172022012, - "nauc_cv_recall_at_10_max": 0.4848156559769109, - "nauc_cv_recall_at_10_std": 0.27822291722547365, - "nauc_cv_recall_at_1_diff1": 0.8529129171302879, - "nauc_cv_recall_at_1_max": 0.4854208594375987, - "nauc_cv_recall_at_1_std": 0.11220419342787168, - "nauc_cv_recall_at_20_diff1": 0.7152210152620466, - "nauc_cv_recall_at_20_max": 0.5093419618089969, - "nauc_cv_recall_at_20_std": 0.31952722781397186, - "nauc_cv_recall_at_3_diff1": 0.7987291872149318, - "nauc_cv_recall_at_3_max": 0.47266289861776534, - "nauc_cv_recall_at_3_std": 0.15022238304790544, - "nauc_cv_recall_at_5_diff1": 0.7702540174147103, - "nauc_cv_recall_at_5_max": 0.46083059121534187, - "nauc_cv_recall_at_5_std": 0.18636651790144276, - "nauc_map_at_1000_diff1": 0.8253362540968048, - "nauc_map_at_1000_max": 0.4837965017623685, - "nauc_map_at_1000_std": 0.1420004533208783, - "nauc_map_at_100_diff1": 0.8252282114430481, - "nauc_map_at_100_max": 0.4839987346975108, - "nauc_map_at_100_std": 0.1424789807481376, - "nauc_map_at_10_diff1": 0.82439509533897, - "nauc_map_at_10_max": 0.48154349516858935, - "nauc_map_at_10_std": 0.14141446616923847, - "nauc_map_at_1_diff1": 0.8529129171302879, - "nauc_map_at_1_max": 0.4854208594375987, - "nauc_map_at_1_std": 0.11220419342787168, - "nauc_map_at_20_diff1": 0.824372991789611, - "nauc_map_at_20_max": 0.48264880992645687, - "nauc_map_at_20_std": 0.1415764288605779, - "nauc_map_at_3_diff1": 0.8306021895214727, - "nauc_map_at_3_max": 0.48060459308194475, - "nauc_map_at_3_std": 0.12697305747126705, - "nauc_map_at_5_diff1": 0.8255314697911217, - "nauc_map_at_5_max": 0.47836790397649964, - "nauc_map_at_5_std": 0.13337887538751342, - "nauc_mrr_at_1000_diff1": 0.8253362900449671, - "nauc_mrr_at_1000_max": 0.48379648836283556, - "nauc_mrr_at_1000_std": 0.1420003680546551, - "nauc_mrr_at_100_diff1": 0.8252282114430481, - "nauc_mrr_at_100_max": 0.4839987346975108, - "nauc_mrr_at_100_std": 0.1424789807481376, - "nauc_mrr_at_10_diff1": 0.82439509533897, - "nauc_mrr_at_10_max": 0.48154349516858935, - "nauc_mrr_at_10_std": 0.14141446616923847, - "nauc_mrr_at_1_diff1": 0.8529129171302879, - "nauc_mrr_at_1_max": 0.4854208594375987, - "nauc_mrr_at_1_std": 0.11220419342787168, - "nauc_mrr_at_20_diff1": 0.824372991789611, - "nauc_mrr_at_20_max": 0.48264880992645687, - "nauc_mrr_at_20_std": 0.1415764288605779, - "nauc_mrr_at_3_diff1": 0.8306021895214727, - "nauc_mrr_at_3_max": 0.48060459308194475, - "nauc_mrr_at_3_std": 0.12697305747126705, - "nauc_mrr_at_5_diff1": 0.8255314697911217, - "nauc_mrr_at_5_max": 0.47836790397649964, - "nauc_mrr_at_5_std": 0.13337887538751342, - "nauc_ndcg_at_1000_diff1": 0.8189740955445418, - "nauc_ndcg_at_1000_max": 0.48739519758688205, - "nauc_ndcg_at_1000_std": 0.1560786057897601, - "nauc_ndcg_at_100_diff1": 0.8170601681119815, - "nauc_ndcg_at_100_max": 0.4941817302873829, - "nauc_ndcg_at_100_std": 0.17053277872707645, - "nauc_ndcg_at_10_diff1": 0.8098947819056849, - "nauc_ndcg_at_10_max": 0.4817583414779039, - "nauc_ndcg_at_10_std": 0.16502964660555786, - "nauc_ndcg_at_1_diff1": 0.8529129171302879, - "nauc_ndcg_at_1_max": 0.4854208594375987, - "nauc_ndcg_at_1_std": 0.11220419342787168, - "nauc_ndcg_at_20_diff1": 0.8090926247820623, - "nauc_ndcg_at_20_max": 0.48551778105850196, - "nauc_ndcg_at_20_std": 0.16566285450230253, - "nauc_ndcg_at_3_diff1": 0.8229903288375536, - "nauc_ndcg_at_3_max": 0.47872741897937043, - "nauc_ndcg_at_3_std": 0.13248601038954932, - "nauc_ndcg_at_5_diff1": 0.8135755859828144, - "nauc_ndcg_at_5_max": 0.4746303106044694, - "nauc_ndcg_at_5_std": 0.14462357276570686, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.7536092845128152, - "nauc_precision_at_100_max": 0.7685517696368366, - "nauc_precision_at_100_std": 0.7416747448364028, - "nauc_precision_at_10_diff1": 0.7446677172022022, - "nauc_precision_at_10_max": 0.4848156559769116, - "nauc_precision_at_10_std": 0.27822291722547354, - "nauc_precision_at_1_diff1": 0.8529129171302879, - "nauc_precision_at_1_max": 0.4854208594375987, - "nauc_precision_at_1_std": 0.11220419342787168, - "nauc_precision_at_20_diff1": 0.7152210152620461, - "nauc_precision_at_20_max": 0.5093419618089969, - "nauc_precision_at_20_std": 0.3195272278139724, - "nauc_precision_at_3_diff1": 0.7987291872149324, - "nauc_precision_at_3_max": 0.4726628986177653, - "nauc_precision_at_3_std": 0.15022238304790592, - "nauc_precision_at_5_diff1": 0.770254017414709, - "nauc_precision_at_5_max": 0.4608305912153412, - "nauc_precision_at_5_std": 0.18636651790144232, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.7536092845128153, - "nauc_recall_at_100_max": 0.7685517696368389, - "nauc_recall_at_100_std": 0.7416747448364003, - "nauc_recall_at_10_diff1": 0.7446677172022012, - "nauc_recall_at_10_max": 0.4848156559769109, - "nauc_recall_at_10_std": 0.27822291722547365, - "nauc_recall_at_1_diff1": 0.8529129171302879, - "nauc_recall_at_1_max": 0.4854208594375987, - "nauc_recall_at_1_std": 0.11220419342787168, - "nauc_recall_at_20_diff1": 0.7152210152620466, - "nauc_recall_at_20_max": 0.5093419618089969, - "nauc_recall_at_20_std": 0.31952722781397186, - "nauc_recall_at_3_diff1": 0.7987291872149318, - "nauc_recall_at_3_max": 0.47266289861776534, - "nauc_recall_at_3_std": 0.15022238304790544, - "nauc_recall_at_5_diff1": 0.7702540174147103, - "nauc_recall_at_5_max": 0.46083059121534187, - "nauc_recall_at_5_std": 0.18636651790144276, - "ndcg_at_1": 0.56493, - "ndcg_at_10": 0.66955, - "ndcg_at_100": 0.70522, - "ndcg_at_1000": 0.71118, - "ndcg_at_20": 0.68554, - "ndcg_at_3": 0.62835, - "ndcg_at_5": 0.64893, - "precision_at_1": 0.56493, - "precision_at_10": 0.07858, - "precision_at_100": 0.00954, - "precision_at_1000": 0.001, - "precision_at_20": 0.04245, - "precision_at_3": 0.2242, - "precision_at_5": 0.14451, - "recall_at_1": 0.56493, - "recall_at_10": 0.78579, - "recall_at_100": 0.9545, - "recall_at_1000": 1.0, - "recall_at_20": 0.84906, - "recall_at_3": 0.67259, - "recall_at_5": 0.72253 - }, - { - "cv_recall_at_1": 0.08163, - "cv_recall_at_10": 0.18904, - "cv_recall_at_100": 0.50161, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.26101, - "cv_recall_at_3": 0.11923, - "cv_recall_at_5": 0.14823, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.12847, - "map_at_1": 0.08163, - "map_at_10": 0.10993, - "map_at_100": 0.12006, - "map_at_1000": 0.12203, - "map_at_20": 0.11476, - "map_at_3": 0.09792, - "map_at_5": 0.10448, - "mrr_at_1": 0.08163265306122448, - "mrr_at_10": 0.1099342744616644, - "mrr_at_100": 0.12005746932036375, - "mrr_at_1000": 0.12203113427399533, - "mrr_at_20": 0.11475762824669429, - "mrr_at_3": 0.0979233798782671, - "mrr_at_5": 0.10447547440028646, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.14751155284105305, - "nauc_cv_recall_at_100_max": 0.22564543811830431, - "nauc_cv_recall_at_100_std": 0.36565185549033236, - "nauc_cv_recall_at_10_diff1": 0.2860643473956765, - "nauc_cv_recall_at_10_max": 0.29643996104309345, - "nauc_cv_recall_at_10_std": 0.3487498575679017, - "nauc_cv_recall_at_1_diff1": 0.6745448554552967, - "nauc_cv_recall_at_1_max": 0.5000925535843469, - "nauc_cv_recall_at_1_std": 0.4339496692158664, - "nauc_cv_recall_at_20_diff1": 0.25203106653182167, - "nauc_cv_recall_at_20_max": 0.27861817937663164, - "nauc_cv_recall_at_20_std": 0.35497189059078726, - "nauc_cv_recall_at_3_diff1": 0.47610663695685046, - "nauc_cv_recall_at_3_max": 0.43568329084244445, - "nauc_cv_recall_at_3_std": 0.41368899691317484, - "nauc_cv_recall_at_5_diff1": 0.3733814405327858, - "nauc_cv_recall_at_5_max": 0.37192948006470095, - "nauc_cv_recall_at_5_std": 0.40205902036298935, - "nauc_map_at_1000_diff1": 0.4978739033571974, - "nauc_map_at_1000_max": 0.419520151261849, - "nauc_map_at_1000_std": 0.4068692740308292, - "nauc_map_at_100_diff1": 0.4958356262477288, - "nauc_map_at_100_max": 0.4177789835056861, - "nauc_map_at_100_std": 0.40616264397711266, - "nauc_map_at_10_diff1": 0.5109920139660944, - "nauc_map_at_10_max": 0.42545602974269947, - "nauc_map_at_10_std": 0.4051907995409085, - "nauc_map_at_1_diff1": 0.6745448554552967, - "nauc_map_at_1_max": 0.5000925535843469, - "nauc_map_at_1_std": 0.4339496692158664, - "nauc_map_at_20_diff1": 0.5022386869091225, - "nauc_map_at_20_max": 0.4211733453178635, - "nauc_map_at_20_std": 0.4061450432164285, - "nauc_map_at_3_diff1": 0.5694582452423288, - "nauc_map_at_3_max": 0.4625966131133485, - "nauc_map_at_3_std": 0.4194557757874592, - "nauc_map_at_5_diff1": 0.5346634805130233, - "nauc_map_at_5_max": 0.4433674733314563, - "nauc_map_at_5_std": 0.41674610130363493, - "nauc_mrr_at_1000_diff1": 0.49787361295714627, - "nauc_mrr_at_1000_max": 0.4195213345637115, - "nauc_mrr_at_1000_std": 0.4068699032112551, - "nauc_mrr_at_100_diff1": 0.49583533792631634, - "nauc_mrr_at_100_max": 0.4177801598674725, - "nauc_mrr_at_100_std": 0.4061632694340152, - "nauc_mrr_at_10_diff1": 0.5109920139660944, - "nauc_mrr_at_10_max": 0.42545602974269947, - "nauc_mrr_at_10_std": 0.4051907995409085, - "nauc_mrr_at_1_diff1": 0.6745448554552967, - "nauc_mrr_at_1_max": 0.5000925535843469, - "nauc_mrr_at_1_std": 0.4339496692158664, - "nauc_mrr_at_20_diff1": 0.5022386869091225, - "nauc_mrr_at_20_max": 0.4211733453178635, - "nauc_mrr_at_20_std": 0.4061450432164285, - "nauc_mrr_at_3_diff1": 0.5694582452423288, - "nauc_mrr_at_3_max": 0.4625966131133485, - "nauc_mrr_at_3_std": 0.4194557757874592, - "nauc_mrr_at_5_diff1": 0.5346634805130233, - "nauc_mrr_at_5_max": 0.4433674733314563, - "nauc_mrr_at_5_std": 0.41674610130363493, - "nauc_ndcg_at_1000_diff1": 0.4358424701809257, - "nauc_ndcg_at_1000_max": 0.38888310258472764, - "nauc_ndcg_at_1000_std": 0.40007110481304475, - "nauc_ndcg_at_100_diff1": 0.382231846590429, - "nauc_ndcg_at_100_max": 0.35564165055226754, - "nauc_ndcg_at_100_std": 0.39143473660956984, - "nauc_ndcg_at_10_diff1": 0.43889613013765655, - "nauc_ndcg_at_10_max": 0.3850896642714146, - "nauc_ndcg_at_10_std": 0.3879631630579122, - "nauc_ndcg_at_1_diff1": 0.6745448554552967, - "nauc_ndcg_at_1_max": 0.5000925535843469, - "nauc_ndcg_at_1_std": 0.4339496692158664, - "nauc_ndcg_at_20_diff1": 0.4168988692194725, - "nauc_ndcg_at_20_max": 0.3736432108951681, - "nauc_ndcg_at_20_std": 0.3894409640165071, - "nauc_ndcg_at_3_diff1": 0.542351084582344, - "nauc_ndcg_at_3_max": 0.4546420999628177, - "nauc_ndcg_at_3_std": 0.4176261772057995, - "nauc_ndcg_at_5_diff1": 0.48597152615731637, - "nauc_ndcg_at_5_max": 0.4220938155113547, - "nauc_ndcg_at_5_std": 0.4123319702203166, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.1475115528410533, - "nauc_precision_at_100_max": 0.2256454381183047, - "nauc_precision_at_100_std": 0.3656518554903324, - "nauc_precision_at_10_diff1": 0.2860643473956765, - "nauc_precision_at_10_max": 0.29643996104309334, - "nauc_precision_at_10_std": 0.3487498575679015, - "nauc_precision_at_1_diff1": 0.6745448554552967, - "nauc_precision_at_1_max": 0.5000925535843469, - "nauc_precision_at_1_std": 0.4339496692158664, - "nauc_precision_at_20_diff1": 0.2520310665318214, - "nauc_precision_at_20_max": 0.27861817937663186, - "nauc_precision_at_20_std": 0.3549718905907873, - "nauc_precision_at_3_diff1": 0.47610663695684996, - "nauc_precision_at_3_max": 0.4356832908424444, - "nauc_precision_at_3_std": 0.41368899691317473, - "nauc_precision_at_5_diff1": 0.37338144053278555, - "nauc_precision_at_5_max": 0.37192948006470095, - "nauc_precision_at_5_std": 0.4020590203629894, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.14751155284105305, - "nauc_recall_at_100_max": 0.22564543811830431, - "nauc_recall_at_100_std": 0.36565185549033236, - "nauc_recall_at_10_diff1": 0.2860643473956765, - "nauc_recall_at_10_max": 0.29643996104309345, - "nauc_recall_at_10_std": 0.3487498575679017, - "nauc_recall_at_1_diff1": 0.6745448554552967, - "nauc_recall_at_1_max": 0.5000925535843469, - "nauc_recall_at_1_std": 0.4339496692158664, - "nauc_recall_at_20_diff1": 0.25203106653182167, - "nauc_recall_at_20_max": 0.27861817937663164, - "nauc_recall_at_20_std": 0.35497189059078726, - "nauc_recall_at_3_diff1": 0.47610663695685046, - "nauc_recall_at_3_max": 0.43568329084244445, - "nauc_recall_at_3_std": 0.41368899691317484, - "nauc_recall_at_5_diff1": 0.3733814405327858, - "nauc_recall_at_5_max": 0.37192948006470095, - "nauc_recall_at_5_std": 0.40205902036298935, - "ndcg_at_1": 0.08163, - "ndcg_at_10": 0.12847, - "ndcg_at_100": 0.18934, - "ndcg_at_1000": 0.25043, - "ndcg_at_20": 0.14645, - "ndcg_at_3": 0.10338, - "ndcg_at_5": 0.11526, - "precision_at_1": 0.08163, - "precision_at_10": 0.0189, - "precision_at_100": 0.00502, - "precision_at_1000": 0.001, - "precision_at_20": 0.01305, - "precision_at_3": 0.03974, - "precision_at_5": 0.02965, - "recall_at_1": 0.08163, - "recall_at_10": 0.18904, - "recall_at_100": 0.50161, - "recall_at_1000": 1.0, - "recall_at_20": 0.26101, - "recall_at_3": 0.11923, - "recall_at_5": 0.14823 - }, - { - "cv_recall_at_1": 0.096, - "cv_recall_at_10": 0.218, - "cv_recall_at_100": 0.462, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.271, - "cv_recall_at_3": 0.149, - "cv_recall_at_5": 0.172, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.15148, - "map_at_1": 0.096, - "map_at_10": 0.13099, - "map_at_100": 0.13896, - "map_at_1000": 0.14107, - "map_at_20": 0.13474, - "map_at_3": 0.11983, - "map_at_5": 0.12488, - "mrr_at_1": 0.096, - "mrr_at_10": 0.13098531746031747, - "mrr_at_100": 0.13896170761443877, - "mrr_at_1000": 0.14107497329376809, - "mrr_at_20": 0.13473580277651484, - "mrr_at_3": 0.11983333333333332, - "mrr_at_5": 0.12488333333333335, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.31076379300822904, - "nauc_cv_recall_at_100_max": 0.3998440487367423, - "nauc_cv_recall_at_100_std": 0.324715250876404, - "nauc_cv_recall_at_10_diff1": 0.38215109042528295, - "nauc_cv_recall_at_10_max": 0.3335709361342387, - "nauc_cv_recall_at_10_std": 0.19664777282468648, - "nauc_cv_recall_at_1_diff1": 0.6314143412932651, - "nauc_cv_recall_at_1_max": 0.41841069671764586, - "nauc_cv_recall_at_1_std": 0.20914944995975326, - "nauc_cv_recall_at_20_diff1": 0.355102914918089, - "nauc_cv_recall_at_20_max": 0.3481563529556926, - "nauc_cv_recall_at_20_std": 0.21010726183773293, - "nauc_cv_recall_at_3_diff1": 0.4801968463257448, - "nauc_cv_recall_at_3_max": 0.4484818156216887, - "nauc_cv_recall_at_3_std": 0.2781092541679922, - "nauc_cv_recall_at_5_diff1": 0.439680987927112, - "nauc_cv_recall_at_5_max": 0.40597175482079656, - "nauc_cv_recall_at_5_std": 0.26277429296196414, - "nauc_map_at_1000_diff1": 0.5160698132598479, - "nauc_map_at_1000_max": 0.41257953784470985, - "nauc_map_at_1000_std": 0.2405028977362211, - "nauc_map_at_100_diff1": 0.5149936007224473, - "nauc_map_at_100_max": 0.4123488602331851, - "nauc_map_at_100_std": 0.24045219275874782, - "nauc_map_at_10_diff1": 0.521754177659713, - "nauc_map_at_10_max": 0.4099643031558265, - "nauc_map_at_10_std": 0.23509281560035314, - "nauc_map_at_1_diff1": 0.6314143412932651, - "nauc_map_at_1_max": 0.41841069671764586, - "nauc_map_at_1_std": 0.20914944995975326, - "nauc_map_at_20_diff1": 0.5178858575458826, - "nauc_map_at_20_max": 0.41082413685719776, - "nauc_map_at_20_std": 0.23591664956917177, - "nauc_map_at_3_diff1": 0.5493791787574669, - "nauc_map_at_3_max": 0.4366534865844311, - "nauc_map_at_3_std": 0.2512776116625236, - "nauc_map_at_5_diff1": 0.536436215744084, - "nauc_map_at_5_max": 0.42485582874292555, - "nauc_map_at_5_std": 0.24757945180493524, - "nauc_mrr_at_1000_diff1": 0.5160698132598479, - "nauc_mrr_at_1000_max": 0.41257953784470985, - "nauc_mrr_at_1000_std": 0.2405028977362211, - "nauc_mrr_at_100_diff1": 0.5149936007224473, - "nauc_mrr_at_100_max": 0.4123488602331851, - "nauc_mrr_at_100_std": 0.24045219275874782, - "nauc_mrr_at_10_diff1": 0.521754177659713, - "nauc_mrr_at_10_max": 0.4099643031558265, - "nauc_mrr_at_10_std": 0.23509281560035314, - "nauc_mrr_at_1_diff1": 0.6314143412932651, - "nauc_mrr_at_1_max": 0.41841069671764586, - "nauc_mrr_at_1_std": 0.20914944995975326, - "nauc_mrr_at_20_diff1": 0.5178858575458826, - "nauc_mrr_at_20_max": 0.41082413685719776, - "nauc_mrr_at_20_std": 0.23591664956917177, - "nauc_mrr_at_3_diff1": 0.5493791787574669, - "nauc_mrr_at_3_max": 0.4366534865844311, - "nauc_mrr_at_3_std": 0.2512776116625236, - "nauc_mrr_at_5_diff1": 0.536436215744084, - "nauc_mrr_at_5_max": 0.42485582874292555, - "nauc_mrr_at_5_std": 0.24757945180493524, - "nauc_ndcg_at_1000_diff1": 0.480757159232348, - "nauc_ndcg_at_1000_max": 0.4060259454455605, - "nauc_ndcg_at_1000_std": 0.24825519139717867, - "nauc_ndcg_at_100_diff1": 0.4513665795585693, - "nauc_ndcg_at_100_max": 0.40384720146777503, - "nauc_ndcg_at_100_std": 0.25896097092497994, - "nauc_ndcg_at_10_diff1": 0.47838296717969875, - "nauc_ndcg_at_10_max": 0.3880846270218068, - "nauc_ndcg_at_10_std": 0.22518898143499563, - "nauc_ndcg_at_1_diff1": 0.6314143412932651, - "nauc_ndcg_at_1_max": 0.41841069671764586, - "nauc_ndcg_at_1_std": 0.20914944995975326, - "nauc_ndcg_at_20_diff1": 0.46631502103347294, - "nauc_ndcg_at_20_max": 0.39150783743703094, - "nauc_ndcg_at_20_std": 0.22872029282256662, - "nauc_ndcg_at_3_diff1": 0.5292375963981599, - "nauc_ndcg_at_3_max": 0.44017880892606553, - "nauc_ndcg_at_3_std": 0.25928897132564943, - "nauc_ndcg_at_5_diff1": 0.5075959544238287, - "nauc_ndcg_at_5_max": 0.4199052075354429, - "nauc_ndcg_at_5_std": 0.2526852162522937, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3107637930082286, - "nauc_precision_at_100_max": 0.39984404873674223, - "nauc_precision_at_100_std": 0.32471525087640407, - "nauc_precision_at_10_diff1": 0.3821510904252831, - "nauc_precision_at_10_max": 0.3335709361342387, - "nauc_precision_at_10_std": 0.19664777282468657, - "nauc_precision_at_1_diff1": 0.6314143412932651, - "nauc_precision_at_1_max": 0.41841069671764586, - "nauc_precision_at_1_std": 0.20914944995975326, - "nauc_precision_at_20_diff1": 0.35510291491808915, - "nauc_precision_at_20_max": 0.34815635295569247, - "nauc_precision_at_20_std": 0.21010726183773287, - "nauc_precision_at_3_diff1": 0.48019684632574444, - "nauc_precision_at_3_max": 0.44848181562168865, - "nauc_precision_at_3_std": 0.27810925416799187, - "nauc_precision_at_5_diff1": 0.43968098792711197, - "nauc_precision_at_5_max": 0.4059717548207963, - "nauc_precision_at_5_std": 0.26277429296196425, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.31076379300822904, - "nauc_recall_at_100_max": 0.3998440487367423, - "nauc_recall_at_100_std": 0.324715250876404, - "nauc_recall_at_10_diff1": 0.38215109042528295, - "nauc_recall_at_10_max": 0.3335709361342387, - "nauc_recall_at_10_std": 0.19664777282468648, - "nauc_recall_at_1_diff1": 0.6314143412932651, - "nauc_recall_at_1_max": 0.41841069671764586, - "nauc_recall_at_1_std": 0.20914944995975326, - "nauc_recall_at_20_diff1": 0.355102914918089, - "nauc_recall_at_20_max": 0.3481563529556926, - "nauc_recall_at_20_std": 0.21010726183773293, - "nauc_recall_at_3_diff1": 0.4801968463257448, - "nauc_recall_at_3_max": 0.4484818156216887, - "nauc_recall_at_3_std": 0.2781092541679922, - "nauc_recall_at_5_diff1": 0.439680987927112, - "nauc_recall_at_5_max": 0.40597175482079656, - "nauc_recall_at_5_std": 0.26277429296196414, - "ndcg_at_1": 0.096, - "ndcg_at_10": 0.15148, - "ndcg_at_100": 0.19904, - "ndcg_at_1000": 0.26528, - "ndcg_at_20": 0.16497, - "ndcg_at_3": 0.12734, - "ndcg_at_5": 0.13664, - "precision_at_1": 0.096, - "precision_at_10": 0.0218, - "precision_at_100": 0.00462, - "precision_at_1000": 0.001, - "precision_at_20": 0.01355, - "precision_at_3": 0.04967, - "precision_at_5": 0.0344, - "recall_at_1": 0.096, - "recall_at_10": 0.218, - "recall_at_100": 0.462, - "recall_at_1000": 1.0, - "recall_at_20": 0.271, - "recall_at_3": 0.149, - "recall_at_5": 0.172 - }, - { - "cv_recall_at_1": 0.52843, - "cv_recall_at_10": 0.76283, - "cv_recall_at_100": 0.94313, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.81969, - "cv_recall_at_3": 0.64216, - "cv_recall_at_5": 0.69071, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.63888, - "map_at_1": 0.52843, - "map_at_10": 0.60011, - "map_at_100": 0.60716, - "map_at_1000": 0.60746, - "map_at_20": 0.60404, - "map_at_3": 0.57929, - "map_at_5": 0.59038, - "mrr_at_1": 0.5284327323162274, - "mrr_at_10": 0.600105122955331, - "mrr_at_100": 0.6071648409715369, - "mrr_at_1000": 0.607455781012974, - "mrr_at_20": 0.6040383115486814, - "mrr_at_3": 0.5792880258899676, - "mrr_at_5": 0.5903837263060561, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.6892368194018946, - "nauc_cv_recall_at_100_max": 0.6442594748788051, - "nauc_cv_recall_at_100_std": 0.7110681447725741, - "nauc_cv_recall_at_10_diff1": 0.6599255418627427, - "nauc_cv_recall_at_10_max": 0.5807602290666405, - "nauc_cv_recall_at_10_std": 0.4085739259797813, - "nauc_cv_recall_at_1_diff1": 0.8018268384808368, - "nauc_cv_recall_at_1_max": 0.5936789618921932, - "nauc_cv_recall_at_1_std": 0.2180061753760995, - "nauc_cv_recall_at_20_diff1": 0.6525563271016608, - "nauc_cv_recall_at_20_max": 0.5619932280203499, - "nauc_cv_recall_at_20_std": 0.4972492933035224, - "nauc_cv_recall_at_3_diff1": 0.7063676220555728, - "nauc_cv_recall_at_3_max": 0.590015123976248, - "nauc_cv_recall_at_3_std": 0.30431843772328776, - "nauc_cv_recall_at_5_diff1": 0.6734167377323467, - "nauc_cv_recall_at_5_max": 0.5788347633839778, - "nauc_cv_recall_at_5_std": 0.3329561226259785, - "nauc_map_at_1000_diff1": 0.7568665768481858, - "nauc_map_at_1000_max": 0.590748890592957, - "nauc_map_at_1000_std": 0.271276252596812, - "nauc_map_at_100_diff1": 0.7568082284879722, - "nauc_map_at_100_max": 0.5907960412124283, - "nauc_map_at_100_std": 0.27169107729937547, - "nauc_map_at_10_diff1": 0.7551235887991433, - "nauc_map_at_10_max": 0.5909815677942487, - "nauc_map_at_10_std": 0.27051223484286024, - "nauc_map_at_1_diff1": 0.8018268384808368, - "nauc_map_at_1_max": 0.5936789618921932, - "nauc_map_at_1_std": 0.2180061753760995, - "nauc_map_at_20_diff1": 0.7558559584549839, - "nauc_map_at_20_max": 0.5904422446496017, - "nauc_map_at_20_std": 0.2725636557190311, - "nauc_map_at_3_diff1": 0.7603606500012732, - "nauc_map_at_3_max": 0.593144099292417, - "nauc_map_at_3_std": 0.2587097130516631, - "nauc_map_at_5_diff1": 0.7553714760197734, - "nauc_map_at_5_max": 0.5909112343550218, - "nauc_map_at_5_std": 0.26418869764691005, - "nauc_mrr_at_1000_diff1": 0.756866537016681, - "nauc_mrr_at_1000_max": 0.590748966461794, - "nauc_mrr_at_1000_std": 0.2712762912022093, - "nauc_mrr_at_100_diff1": 0.7568082284879722, - "nauc_mrr_at_100_max": 0.5907960412124283, - "nauc_mrr_at_100_std": 0.27169107729937547, - "nauc_mrr_at_10_diff1": 0.7551235887991433, - "nauc_mrr_at_10_max": 0.5909815677942487, - "nauc_mrr_at_10_std": 0.27051223484286024, - "nauc_mrr_at_1_diff1": 0.8018268384808368, - "nauc_mrr_at_1_max": 0.5936789618921932, - "nauc_mrr_at_1_std": 0.2180061753760995, - "nauc_mrr_at_20_diff1": 0.7558559584549839, - "nauc_mrr_at_20_max": 0.5904422446496017, - "nauc_mrr_at_20_std": 0.2725636557190311, - "nauc_mrr_at_3_diff1": 0.7603606500012732, - "nauc_mrr_at_3_max": 0.593144099292417, - "nauc_mrr_at_3_std": 0.2587097130516631, - "nauc_mrr_at_5_diff1": 0.7553714760197734, - "nauc_mrr_at_5_max": 0.5909112343550218, - "nauc_mrr_at_5_std": 0.26418869764691005, - "nauc_ndcg_at_1000_diff1": 0.7486336031563205, - "nauc_ndcg_at_1000_max": 0.5897078600668323, - "nauc_ndcg_at_1000_std": 0.286561756145907, - "nauc_ndcg_at_100_diff1": 0.7469180499307316, - "nauc_ndcg_at_100_max": 0.5912671824348572, - "nauc_ndcg_at_100_std": 0.298923732369518, - "nauc_ndcg_at_10_diff1": 0.7366787103268794, - "nauc_ndcg_at_10_max": 0.5889900838741208, - "nauc_ndcg_at_10_std": 0.2959522421148646, - "nauc_ndcg_at_1_diff1": 0.8018268384808368, - "nauc_ndcg_at_1_max": 0.5936789618921932, - "nauc_ndcg_at_1_std": 0.2180061753760995, - "nauc_ndcg_at_20_diff1": 0.7387233999152774, - "nauc_ndcg_at_20_max": 0.5864892929565019, - "nauc_ndcg_at_20_std": 0.3052934739148191, - "nauc_ndcg_at_3_diff1": 0.7472825895138822, - "nauc_ndcg_at_3_max": 0.5924345319725394, - "nauc_ndcg_at_3_std": 0.2699037851276475, - "nauc_ndcg_at_5_diff1": 0.737149718941511, - "nauc_ndcg_at_5_max": 0.588353926823294, - "nauc_ndcg_at_5_std": 0.2797707300943979, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.6892368194018897, - "nauc_precision_at_100_max": 0.6442594748788032, - "nauc_precision_at_100_std": 0.7110681447725753, - "nauc_precision_at_10_diff1": 0.6599255418627438, - "nauc_precision_at_10_max": 0.5807602290666415, - "nauc_precision_at_10_std": 0.4085739259797818, - "nauc_precision_at_1_diff1": 0.8018268384808368, - "nauc_precision_at_1_max": 0.5936789618921932, - "nauc_precision_at_1_std": 0.2180061753760995, - "nauc_precision_at_20_diff1": 0.6525563271016613, - "nauc_precision_at_20_max": 0.5619932280203506, - "nauc_precision_at_20_std": 0.49724929330352213, - "nauc_precision_at_3_diff1": 0.7063676220555728, - "nauc_precision_at_3_max": 0.5900151239762477, - "nauc_precision_at_3_std": 0.3043184377232873, - "nauc_precision_at_5_diff1": 0.6734167377323469, - "nauc_precision_at_5_max": 0.578834763383978, - "nauc_precision_at_5_std": 0.33295612262597796, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.6892368194018946, - "nauc_recall_at_100_max": 0.6442594748788051, - "nauc_recall_at_100_std": 0.7110681447725741, - "nauc_recall_at_10_diff1": 0.6599255418627427, - "nauc_recall_at_10_max": 0.5807602290666405, - "nauc_recall_at_10_std": 0.4085739259797813, - "nauc_recall_at_1_diff1": 0.8018268384808368, - "nauc_recall_at_1_max": 0.5936789618921932, - "nauc_recall_at_1_std": 0.2180061753760995, - "nauc_recall_at_20_diff1": 0.6525563271016608, - "nauc_recall_at_20_max": 0.5619932280203499, - "nauc_recall_at_20_std": 0.4972492933035224, - "nauc_recall_at_3_diff1": 0.7063676220555728, - "nauc_recall_at_3_max": 0.590015123976248, - "nauc_recall_at_3_std": 0.30431843772328776, - "nauc_recall_at_5_diff1": 0.6734167377323467, - "nauc_recall_at_5_max": 0.5788347633839778, - "nauc_recall_at_5_std": 0.3329561226259785, - "ndcg_at_1": 0.52843, - "ndcg_at_10": 0.63888, - "ndcg_at_100": 0.67605, - "ndcg_at_1000": 0.68346, - "ndcg_at_20": 0.65324, - "ndcg_at_3": 0.59547, - "ndcg_at_5": 0.61546, - "precision_at_1": 0.52843, - "precision_at_10": 0.07628, - "precision_at_100": 0.00943, - "precision_at_1000": 0.001, - "precision_at_20": 0.04098, - "precision_at_3": 0.21405, - "precision_at_5": 0.13814, - "recall_at_1": 0.52843, - "recall_at_10": 0.76283, - "recall_at_100": 0.94313, - "recall_at_1000": 1.0, - "recall_at_20": 0.81969, - "recall_at_3": 0.64216, - "recall_at_5": 0.69071 - }, - { - "cv_recall_at_1": 0.41015, - "cv_recall_at_10": 0.63636, - "cv_recall_at_100": 0.84884, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.6945, - "cv_recall_at_3": 0.51268, - "cv_recall_at_5": 0.56448, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.51471, - "map_at_1": 0.41015, - "map_at_10": 0.47695, - "map_at_100": 0.48473, - "map_at_1000": 0.48547, - "map_at_20": 0.48101, - "map_at_3": 0.45595, - "map_at_5": 0.46769, - "mrr_at_1": 0.41014799154334036, - "mrr_at_10": 0.4769543441055068, - "mrr_at_100": 0.4847251288666404, - "mrr_at_1000": 0.48546655091217855, - "mrr_at_20": 0.48100566949619306, - "mrr_at_3": 0.4559548978153628, - "mrr_at_5": 0.46768851303734993, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5409143387106966, - "nauc_cv_recall_at_100_max": 0.49919692850959463, - "nauc_cv_recall_at_100_std": 0.579791515955671, - "nauc_cv_recall_at_10_diff1": 0.6391618676932485, - "nauc_cv_recall_at_10_max": 0.5607279113763848, - "nauc_cv_recall_at_10_std": 0.5049355725436926, - "nauc_cv_recall_at_1_diff1": 0.78493057893207, - "nauc_cv_recall_at_1_max": 0.5561248078698109, - "nauc_cv_recall_at_1_std": 0.39527525661071433, - "nauc_cv_recall_at_20_diff1": 0.6027928145194872, - "nauc_cv_recall_at_20_max": 0.5624857110624909, - "nauc_cv_recall_at_20_std": 0.548086828891615, - "nauc_cv_recall_at_3_diff1": 0.6958158815323969, - "nauc_cv_recall_at_3_max": 0.5533491395012368, - "nauc_cv_recall_at_3_std": 0.4420005231182388, - "nauc_cv_recall_at_5_diff1": 0.6709801297614552, - "nauc_cv_recall_at_5_max": 0.5435911362820227, - "nauc_cv_recall_at_5_std": 0.450371666636732, - "nauc_map_at_1000_diff1": 0.7376421388088805, - "nauc_map_at_1000_max": 0.5560148083486584, - "nauc_map_at_1000_std": 0.42565795193083183, - "nauc_map_at_100_diff1": 0.7372071244493027, - "nauc_map_at_100_max": 0.5558178928230634, - "nauc_map_at_100_std": 0.42580840091303673, - "nauc_map_at_10_diff1": 0.7382661772755509, - "nauc_map_at_10_max": 0.5564344976313585, - "nauc_map_at_10_std": 0.4244538090998832, - "nauc_map_at_1_diff1": 0.78493057893207, - "nauc_map_at_1_max": 0.5561248078698109, - "nauc_map_at_1_std": 0.39527525661071433, - "nauc_map_at_20_diff1": 0.7369171276008715, - "nauc_map_at_20_max": 0.5567639433197569, - "nauc_map_at_20_std": 0.42660328682417953, - "nauc_map_at_3_diff1": 0.7454743543551721, - "nauc_map_at_3_max": 0.5559979243971782, - "nauc_map_at_3_std": 0.41690566638467735, - "nauc_map_at_5_diff1": 0.740992091976634, - "nauc_map_at_5_max": 0.5545131399375515, - "nauc_map_at_5_std": 0.4189616480090802, - "nauc_mrr_at_1000_diff1": 0.7376421826035284, - "nauc_mrr_at_1000_max": 0.5560147531236701, - "nauc_mrr_at_1000_std": 0.4256578903304658, - "nauc_mrr_at_100_diff1": 0.7372071244493027, - "nauc_mrr_at_100_max": 0.5558178928230634, - "nauc_mrr_at_100_std": 0.42580840091303673, - "nauc_mrr_at_10_diff1": 0.7382661772755509, - "nauc_mrr_at_10_max": 0.5564344976313585, - "nauc_mrr_at_10_std": 0.4244538090998832, - "nauc_mrr_at_1_diff1": 0.78493057893207, - "nauc_mrr_at_1_max": 0.5561248078698109, - "nauc_mrr_at_1_std": 0.39527525661071433, - "nauc_mrr_at_20_diff1": 0.7369171276008715, - "nauc_mrr_at_20_max": 0.5567639433197569, - "nauc_mrr_at_20_std": 0.42660328682417953, - "nauc_mrr_at_3_diff1": 0.7454743543551721, - "nauc_mrr_at_3_max": 0.5559979243971782, - "nauc_mrr_at_3_std": 0.41690566638467735, - "nauc_mrr_at_5_diff1": 0.740992091976634, - "nauc_mrr_at_5_max": 0.5545131399375515, - "nauc_mrr_at_5_std": 0.4189616480090802, - "nauc_ndcg_at_1000_diff1": 0.7245111337667335, - "nauc_ndcg_at_1000_max": 0.5552434039481599, - "nauc_ndcg_at_1000_std": 0.4366429775562634, - "nauc_ndcg_at_100_diff1": 0.7132014389766401, - "nauc_ndcg_at_100_max": 0.5513373018774222, - "nauc_ndcg_at_100_std": 0.44437309757597077, - "nauc_ndcg_at_10_diff1": 0.7168649688744982, - "nauc_ndcg_at_10_max": 0.5570884075954164, - "nauc_ndcg_at_10_std": 0.4411216802971647, - "nauc_ndcg_at_1_diff1": 0.78493057893207, - "nauc_ndcg_at_1_max": 0.5561248078698109, - "nauc_ndcg_at_1_std": 0.39527525661071433, - "nauc_ndcg_at_20_diff1": 0.7113248768688196, - "nauc_ndcg_at_20_max": 0.557708419773681, - "nauc_ndcg_at_20_std": 0.4488177602434449, - "nauc_ndcg_at_3_diff1": 0.7329128659932554, - "nauc_ndcg_at_3_max": 0.5553759897014252, - "nauc_ndcg_at_3_std": 0.42329473035978266, - "nauc_ndcg_at_5_diff1": 0.7242215193891609, - "nauc_ndcg_at_5_max": 0.5521105842445077, - "nauc_ndcg_at_5_std": 0.42666241366994423, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.5409143387106957, - "nauc_precision_at_100_max": 0.49919692850959463, - "nauc_precision_at_100_std": 0.5797915159556701, - "nauc_precision_at_10_diff1": 0.639161867693248, - "nauc_precision_at_10_max": 0.5607279113763848, - "nauc_precision_at_10_std": 0.5049355725436926, - "nauc_precision_at_1_diff1": 0.78493057893207, - "nauc_precision_at_1_max": 0.5561248078698109, - "nauc_precision_at_1_std": 0.39527525661071433, - "nauc_precision_at_20_diff1": 0.6027928145194867, - "nauc_precision_at_20_max": 0.5624857110624905, - "nauc_precision_at_20_std": 0.5480868288916155, - "nauc_precision_at_3_diff1": 0.6958158815323969, - "nauc_precision_at_3_max": 0.5533491395012369, - "nauc_precision_at_3_std": 0.4420005231182392, - "nauc_precision_at_5_diff1": 0.670980129761455, - "nauc_precision_at_5_max": 0.5435911362820229, - "nauc_precision_at_5_std": 0.4503716666367316, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5409143387106966, - "nauc_recall_at_100_max": 0.49919692850959463, - "nauc_recall_at_100_std": 0.579791515955671, - "nauc_recall_at_10_diff1": 0.6391618676932485, - "nauc_recall_at_10_max": 0.5607279113763848, - "nauc_recall_at_10_std": 0.5049355725436926, - "nauc_recall_at_1_diff1": 0.78493057893207, - "nauc_recall_at_1_max": 0.5561248078698109, - "nauc_recall_at_1_std": 0.39527525661071433, - "nauc_recall_at_20_diff1": 0.6027928145194872, - "nauc_recall_at_20_max": 0.5624857110624909, - "nauc_recall_at_20_std": 0.548086828891615, - "nauc_recall_at_3_diff1": 0.6958158815323969, - "nauc_recall_at_3_max": 0.5533491395012368, - "nauc_recall_at_3_std": 0.4420005231182388, - "nauc_recall_at_5_diff1": 0.6709801297614552, - "nauc_recall_at_5_max": 0.5435911362820227, - "nauc_recall_at_5_std": 0.450371666636732, - "ndcg_at_1": 0.41015, - "ndcg_at_10": 0.51471, - "ndcg_at_100": 0.55755, - "ndcg_at_1000": 0.57693, - "ndcg_at_20": 0.52943, - "ndcg_at_3": 0.47055, - "ndcg_at_5": 0.49179, - "precision_at_1": 0.41015, - "precision_at_10": 0.06364, - "precision_at_100": 0.00849, - "precision_at_1000": 0.001, - "precision_at_20": 0.03473, - "precision_at_3": 0.17089, - "precision_at_5": 0.1129, - "recall_at_1": 0.41015, - "recall_at_10": 0.63636, - "recall_at_100": 0.84884, - "recall_at_1000": 1.0, - "recall_at_20": 0.6945, - "recall_at_3": 0.51268, - "recall_at_5": 0.56448 - }, - { - "cv_recall_at_1": 0.625, - "cv_recall_at_10": 0.801, - "cv_recall_at_100": 0.94, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.844, - "cv_recall_at_3": 0.719, - "cv_recall_at_5": 0.748, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.71013, - "map_at_1": 0.625, - "map_at_10": 0.68155, - "map_at_100": 0.68709, - "map_at_1000": 0.68745, - "map_at_20": 0.68467, - "map_at_3": 0.66783, - "map_at_5": 0.67448, - "mrr_at_1": 0.625, - "mrr_at_10": 0.6815527777777778, - "mrr_at_100": 0.6870917599034414, - "mrr_at_1000": 0.6874521809565477, - "mrr_at_20": 0.684674403679396, - "mrr_at_3": 0.6678333333333335, - "mrr_at_5": 0.6744833333333335, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.7051976346093998, - "nauc_cv_recall_at_100_max": 0.9334733893557434, - "nauc_cv_recall_at_100_std": 0.9426548397136629, - "nauc_cv_recall_at_10_diff1": 0.8277429003005983, - "nauc_cv_recall_at_10_max": 0.7307057294411077, - "nauc_cv_recall_at_10_std": 0.11950930651531334, - "nauc_cv_recall_at_1_diff1": 0.9062076460314379, - "nauc_cv_recall_at_1_max": 0.673808655152338, - "nauc_cv_recall_at_1_std": 0.034311663108868774, - "nauc_cv_recall_at_20_diff1": 0.8017533723529852, - "nauc_cv_recall_at_20_max": 0.7809038157200636, - "nauc_cv_recall_at_20_std": 0.2732610716169713, - "nauc_cv_recall_at_3_diff1": 0.8567395906845451, - "nauc_cv_recall_at_3_max": 0.6611800870066405, - "nauc_cv_recall_at_3_std": 0.02962867148430545, - "nauc_cv_recall_at_5_diff1": 0.8484581277919656, - "nauc_cv_recall_at_5_max": 0.6963547429103827, - "nauc_cv_recall_at_5_std": 0.05383034830423127, - "nauc_map_at_1000_diff1": 0.8827887804526993, - "nauc_map_at_1000_max": 0.6807906637408886, - "nauc_map_at_1000_std": 0.04991212739584506, - "nauc_map_at_100_diff1": 0.8825373269740879, - "nauc_map_at_100_max": 0.6811539785400426, - "nauc_map_at_100_std": 0.05113954398193766, - "nauc_map_at_10_diff1": 0.8833213241719006, - "nauc_map_at_10_max": 0.6792061204673818, - "nauc_map_at_10_std": 0.042588582733769274, - "nauc_map_at_1_diff1": 0.9062076460314379, - "nauc_map_at_1_max": 0.673808655152338, - "nauc_map_at_1_std": 0.034311663108868774, - "nauc_map_at_20_diff1": 0.882846654608725, - "nauc_map_at_20_max": 0.6806083956333864, - "nauc_map_at_20_std": 0.04739113496391819, - "nauc_map_at_3_diff1": 0.8854941510333725, - "nauc_map_at_3_max": 0.6688552291574957, - "nauc_map_at_3_std": 0.030730587130158093, - "nauc_map_at_5_diff1": 0.8844906719331308, - "nauc_map_at_5_max": 0.6758096501673212, - "nauc_map_at_5_std": 0.03580337858057999, - "nauc_mrr_at_1000_diff1": 0.8827887804526993, - "nauc_mrr_at_1000_max": 0.6807906637408886, - "nauc_mrr_at_1000_std": 0.04991212739584506, - "nauc_mrr_at_100_diff1": 0.8825373269740879, - "nauc_mrr_at_100_max": 0.6811539785400426, - "nauc_mrr_at_100_std": 0.05113954398193766, - "nauc_mrr_at_10_diff1": 0.8833213241719006, - "nauc_mrr_at_10_max": 0.6792061204673818, - "nauc_mrr_at_10_std": 0.042588582733769274, - "nauc_mrr_at_1_diff1": 0.9062076460314379, - "nauc_mrr_at_1_max": 0.673808655152338, - "nauc_mrr_at_1_std": 0.034311663108868774, - "nauc_mrr_at_20_diff1": 0.882846654608725, - "nauc_mrr_at_20_max": 0.6806083956333864, - "nauc_mrr_at_20_std": 0.04739113496391819, - "nauc_mrr_at_3_diff1": 0.8854941510333725, - "nauc_mrr_at_3_max": 0.6688552291574957, - "nauc_mrr_at_3_std": 0.030730587130158093, - "nauc_mrr_at_5_diff1": 0.8844906719331308, - "nauc_mrr_at_5_max": 0.6758096501673212, - "nauc_mrr_at_5_std": 0.03580337858057999, - "nauc_ndcg_at_1000_diff1": 0.8760726250721362, - "nauc_ndcg_at_1000_max": 0.6876964589525962, - "nauc_ndcg_at_1000_std": 0.068740569878718, - "nauc_ndcg_at_100_diff1": 0.8698242603656416, - "nauc_ndcg_at_100_max": 0.6967397827273338, - "nauc_ndcg_at_100_std": 0.10051970473333424, - "nauc_ndcg_at_10_diff1": 0.8729424325302123, - "nauc_ndcg_at_10_max": 0.6881091554863467, - "nauc_ndcg_at_10_std": 0.05583289621595604, - "nauc_ndcg_at_1_diff1": 0.9062076460314379, - "nauc_ndcg_at_1_max": 0.673808655152338, - "nauc_ndcg_at_1_std": 0.034311663108868774, - "nauc_ndcg_at_20_diff1": 0.8706189980509209, - "nauc_ndcg_at_20_max": 0.6942792549563985, - "nauc_ndcg_at_20_std": 0.07699478300441558, - "nauc_ndcg_at_3_diff1": 0.8786736061073106, - "nauc_ndcg_at_3_max": 0.6670500054524965, - "nauc_ndcg_at_3_std": 0.030395050026420765, - "nauc_ndcg_at_5_diff1": 0.8765023815329428, - "nauc_ndcg_at_5_max": 0.6799301445714997, - "nauc_ndcg_at_5_std": 0.03947961532415831, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.7051976346093989, - "nauc_precision_at_100_max": 0.9334733893557411, - "nauc_precision_at_100_std": 0.9426548397136639, - "nauc_precision_at_10_diff1": 0.8277429003005984, - "nauc_precision_at_10_max": 0.7307057294411068, - "nauc_precision_at_10_std": 0.11950930651531294, - "nauc_precision_at_1_diff1": 0.9062076460314379, - "nauc_precision_at_1_max": 0.673808655152338, - "nauc_precision_at_1_std": 0.034311663108868774, - "nauc_precision_at_20_diff1": 0.8017533723529853, - "nauc_precision_at_20_max": 0.7809038157200632, - "nauc_precision_at_20_std": 0.27326107161697166, - "nauc_precision_at_3_diff1": 0.8567395906845451, - "nauc_precision_at_3_max": 0.6611800870066414, - "nauc_precision_at_3_std": 0.029628671484306243, - "nauc_precision_at_5_diff1": 0.8484581277919659, - "nauc_precision_at_5_max": 0.6963547429103824, - "nauc_precision_at_5_std": 0.053830348304230724, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.7051976346093998, - "nauc_recall_at_100_max": 0.9334733893557434, - "nauc_recall_at_100_std": 0.9426548397136629, - "nauc_recall_at_10_diff1": 0.8277429003005983, - "nauc_recall_at_10_max": 0.7307057294411077, - "nauc_recall_at_10_std": 0.11950930651531334, - "nauc_recall_at_1_diff1": 0.9062076460314379, - "nauc_recall_at_1_max": 0.673808655152338, - "nauc_recall_at_1_std": 0.034311663108868774, - "nauc_recall_at_20_diff1": 0.8017533723529852, - "nauc_recall_at_20_max": 0.7809038157200636, - "nauc_recall_at_20_std": 0.2732610716169713, - "nauc_recall_at_3_diff1": 0.8567395906845451, - "nauc_recall_at_3_max": 0.6611800870066405, - "nauc_recall_at_3_std": 0.02962867148430545, - "nauc_recall_at_5_diff1": 0.8484581277919656, - "nauc_recall_at_5_max": 0.6963547429103827, - "nauc_recall_at_5_std": 0.05383034830423127, - "ndcg_at_1": 0.625, - "ndcg_at_10": 0.71013, - "ndcg_at_100": 0.73893, - "ndcg_at_1000": 0.74698, - "ndcg_at_20": 0.72118, - "ndcg_at_3": 0.68103, - "ndcg_at_5": 0.693, - "precision_at_1": 0.625, - "precision_at_10": 0.0801, - "precision_at_100": 0.0094, - "precision_at_1000": 0.001, - "precision_at_20": 0.0422, - "precision_at_3": 0.23967, - "precision_at_5": 0.1496, - "recall_at_1": 0.625, - "recall_at_10": 0.801, - "recall_at_100": 0.94, - "recall_at_1000": 1.0, - "recall_at_20": 0.844, - "recall_at_3": 0.719, - "recall_at_5": 0.748 - } - ] - }, - "task_name": "WITT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json b/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json deleted file mode 100644 index 96ba2e7de4..0000000000 --- a/results-mieb/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "google/siglip-so400m-patch14-384", "revision": "9fdffc58afc957d1a03a25b10dba0329ab15c2a3", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json deleted file mode 100644 index b5e7e4bff6..0000000000 --- a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb", - "evaluation_time": 12.655046463012695, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.806, - "cv_recall_at_10": 0.981, - "cv_recall_at_100": 0.999, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.994, - "cv_recall_at_3": 0.922, - "cv_recall_at_5": 0.961, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.75656, - "map_at_1": 0.1612, - "map_at_10": 0.65737, - "map_at_100": 0.6941, - "map_at_1000": 0.69494, - "map_at_20": 0.68213, - "map_at_3": 0.42077, - "map_at_5": 0.5824, - "mrr_at_1": 0.806, - "mrr_at_10": 0.8702051587301588, - "mrr_at_100": 0.8713558453277496, - "mrr_at_1000": 0.8713621346359256, - "mrr_at_20": 0.8711786018505061, - "mrr_at_3": 0.8581666666666669, - "mrr_at_5": 0.8672666666666671, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.3580765639589976, - "nauc_cv_recall_at_100_max": 1.0, - "nauc_cv_recall_at_100_std": 1.0, - "nauc_cv_recall_at_10_diff1": 0.4775664651825631, - "nauc_cv_recall_at_10_max": 0.7300113027667245, - "nauc_cv_recall_at_10_std": -0.05830753353973558, - "nauc_cv_recall_at_1_diff1": 0.3443688532171371, - "nauc_cv_recall_at_1_max": 0.4866993124852272, - "nauc_cv_recall_at_1_std": -0.1314705005806373, - "nauc_cv_recall_at_20_diff1": 0.5864456893868585, - "nauc_cv_recall_at_20_max": 0.9782135076252602, - "nauc_cv_recall_at_20_std": 0.5981948334889493, - "nauc_cv_recall_at_3_diff1": 0.3899961694079347, - "nauc_cv_recall_at_3_max": 0.5140295434413065, - "nauc_cv_recall_at_3_std": -0.12323732911968292, - "nauc_cv_recall_at_5_diff1": 0.48242715889774873, - "nauc_cv_recall_at_5_max": 0.5554957025545274, - "nauc_cv_recall_at_5_std": -0.16390145801910372, - "nauc_map_at_1000_diff1": 0.07860412999292432, - "nauc_map_at_1000_max": 0.5193407172877343, - "nauc_map_at_1000_std": -0.03554548015806451, - "nauc_map_at_100_diff1": 0.07860332936377284, - "nauc_map_at_100_max": 0.5202209025325473, - "nauc_map_at_100_std": -0.03402804669053566, - "nauc_map_at_10_diff1": 0.06738608544555293, - "nauc_map_at_10_max": 0.5046712341845588, - "nauc_map_at_10_std": -0.052089661267638164, - "nauc_map_at_1_diff1": 0.344368853217137, - "nauc_map_at_1_max": 0.4866993124852267, - "nauc_map_at_1_std": -0.1314705005806365, - "nauc_map_at_20_diff1": 0.07540132773467392, - "nauc_map_at_20_max": 0.5194163232471378, - "nauc_map_at_20_std": -0.03561110594882118, - "nauc_map_at_3_diff1": 0.13827655728488158, - "nauc_map_at_3_max": 0.48609865047620054, - "nauc_map_at_3_std": -0.09536745328619929, - "nauc_map_at_5_diff1": 0.07059479448464502, - "nauc_map_at_5_max": 0.48086096767511555, - "nauc_map_at_5_std": -0.08160089788651494, - "nauc_mrr_at_1000_diff1": 0.36023929557475076, - "nauc_mrr_at_1000_max": 0.5035860304880058, - "nauc_mrr_at_1000_std": -0.1249299862258316, - "nauc_mrr_at_100_diff1": 0.3602391842803307, - "nauc_mrr_at_100_max": 0.5036115760084787, - "nauc_mrr_at_100_std": -0.12487209719821057, - "nauc_mrr_at_10_diff1": 0.3611514720783506, - "nauc_mrr_at_10_max": 0.5051263876997938, - "nauc_mrr_at_10_std": -0.1257752596646823, - "nauc_mrr_at_1_diff1": 0.3443688532171371, - "nauc_mrr_at_1_max": 0.4866993124852272, - "nauc_mrr_at_1_std": -0.1314705005806373, - "nauc_mrr_at_20_diff1": 0.3606869616253337, - "nauc_mrr_at_20_max": 0.5042795551778488, - "nauc_mrr_at_20_std": -0.1239507002161601, - "nauc_mrr_at_3_diff1": 0.35911327275172067, - "nauc_mrr_at_3_max": 0.49885073455301077, - "nauc_mrr_at_3_std": -0.12597845586993137, - "nauc_mrr_at_5_diff1": 0.36427948002844535, - "nauc_mrr_at_5_max": 0.5025432319868158, - "nauc_mrr_at_5_std": -0.12876002101412964, - "nauc_ndcg_at_1000_diff1": 0.1404445479237733, - "nauc_ndcg_at_1000_max": 0.5347818240932689, - "nauc_ndcg_at_1000_std": -0.03316587882098286, - "nauc_ndcg_at_100_diff1": 0.1358934054564792, - "nauc_ndcg_at_100_max": 0.5460446827666793, - "nauc_ndcg_at_100_std": -0.007831409654410723, - "nauc_ndcg_at_10_diff1": 0.10422124872393643, - "nauc_ndcg_at_10_max": 0.522718634663143, - "nauc_ndcg_at_10_std": -0.042784056691292496, - "nauc_ndcg_at_1_diff1": 0.3443688532171371, - "nauc_ndcg_at_1_max": 0.4866993124852272, - "nauc_ndcg_at_1_std": -0.1314705005806373, - "nauc_ndcg_at_20_diff1": 0.12477162836638274, - "nauc_ndcg_at_20_max": 0.5506038459831472, - "nauc_ndcg_at_20_std": -0.0058328462109728755, - "nauc_ndcg_at_3_diff1": 0.1727965348989873, - "nauc_ndcg_at_3_max": 0.49295039267335533, - "nauc_ndcg_at_3_std": -0.09392106684131309, - "nauc_ndcg_at_5_diff1": 0.1101745223344572, - "nauc_ndcg_at_5_max": 0.4913548608912362, - "nauc_ndcg_at_5_std": -0.08489198959710985, - "nauc_precision_at_1000_diff1": -0.08286647992530825, - "nauc_precision_at_1000_max": 0.854341736694646, - "nauc_precision_at_1000_std": 0.8109243697478463, - "nauc_precision_at_100_diff1": 0.02976321846328062, - "nauc_precision_at_100_max": 0.6817348214942566, - "nauc_precision_at_100_std": 0.4525517335603402, - "nauc_precision_at_10_diff1": 0.03152919182802702, - "nauc_precision_at_10_max": 0.5209015914034817, - "nauc_precision_at_10_std": 0.012785208286159547, - "nauc_precision_at_1_diff1": 0.3443688532171371, - "nauc_precision_at_1_max": 0.4866993124852272, - "nauc_precision_at_1_std": -0.1314705005806373, - "nauc_precision_at_20_diff1": 0.058403294374015056, - "nauc_precision_at_20_max": 0.5976698949057889, - "nauc_precision_at_20_std": 0.15342898310071018, - "nauc_precision_at_3_diff1": 0.13112541027685642, - "nauc_precision_at_3_max": 0.4910052478350366, - "nauc_precision_at_3_std": -0.0846106773972222, - "nauc_precision_at_5_diff1": 0.057600655958168046, - "nauc_precision_at_5_max": 0.4823384872903635, - "nauc_precision_at_5_std": -0.07416100661323119, - "nauc_recall_at_1000_diff1": -0.08286647992533601, - "nauc_recall_at_1000_max": 0.8543417366946594, - "nauc_recall_at_1000_std": 0.8109243697478747, - "nauc_recall_at_100_diff1": 0.029763218463283347, - "nauc_recall_at_100_max": 0.6817348214942569, - "nauc_recall_at_100_std": 0.4525517335603416, - "nauc_recall_at_10_diff1": 0.03152919182802702, - "nauc_recall_at_10_max": 0.5209015914034817, - "nauc_recall_at_10_std": 0.012785208286159547, - "nauc_recall_at_1_diff1": 0.344368853217137, - "nauc_recall_at_1_max": 0.4866993124852267, - "nauc_recall_at_1_std": -0.1314705005806365, - "nauc_recall_at_20_diff1": 0.058403294374015056, - "nauc_recall_at_20_max": 0.5976698949057889, - "nauc_recall_at_20_std": 0.15342898310071018, - "nauc_recall_at_3_diff1": 0.1311254102768568, - "nauc_recall_at_3_max": 0.49100524783503674, - "nauc_recall_at_3_std": -0.0846106773972213, - "nauc_recall_at_5_diff1": 0.057600655958168046, - "nauc_recall_at_5_max": 0.4823384872903635, - "nauc_recall_at_5_std": -0.07416100661323119, - "ndcg_at_1": 0.806, - "ndcg_at_10": 0.75656, - "ndcg_at_100": 0.82771, - "ndcg_at_1000": 0.83522, - "ndcg_at_20": 0.79537, - "ndcg_at_3": 0.7524, - "ndcg_at_5": 0.67901, - "precision_at_1": 0.806, - "precision_at_10": 0.3869, - "precision_at_100": 0.04818, - "precision_at_1000": 0.00499, - "precision_at_20": 0.2159, - "precision_at_3": 0.735, - "precision_at_5": 0.6344, - "recall_at_1": 0.1612, - "recall_at_10": 0.7738, - "recall_at_100": 0.9636, - "recall_at_1000": 0.9976, - "recall_at_20": 0.8636, - "recall_at_3": 0.441, - "recall_at_5": 0.6344 - } - ] - }, - "task_name": "Flickr30kI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kT2IRetrieval.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kT2IRetrieval.json deleted file mode 100644 index 2f5c0c550a..0000000000 --- a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "e819702b287bfbe084e129a61f308a802b7c108e", - "evaluation_time": 21.522692918777466, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.678, - "cv_recall_at_10": 0.9334, - "cv_recall_at_100": 0.994, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.9632, - "cv_recall_at_3": 0.8422, - "cv_recall_at_5": 0.89, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8094, - "map_at_1": 0.678, - "map_at_10": 0.76922, - "map_at_100": 0.77224, - "map_at_1000": 0.77227, - "map_at_20": 0.7714, - "map_at_3": 0.7522, - "map_at_5": 0.76322, - "mrr_at_1": 0.678, - "mrr_at_10": 0.7692153968253979, - "mrr_at_100": 0.7722356838203779, - "mrr_at_1000": 0.7722705791355338, - "mrr_at_20": 0.771396252397414, - "mrr_at_3": 0.752200000000002, - "mrr_at_5": 0.7632200000000005, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.7008714596949883, - "nauc_cv_recall_at_100_max": 0.7499999999999728, - "nauc_cv_recall_at_100_std": -0.37489884842825993, - "nauc_cv_recall_at_10_diff1": 0.5953937130407739, - "nauc_cv_recall_at_10_max": 0.5879408820585317, - "nauc_cv_recall_at_10_std": -0.2350459703400894, - "nauc_cv_recall_at_1_diff1": 0.7388944519574545, - "nauc_cv_recall_at_1_max": 0.5035033567670426, - "nauc_cv_recall_at_1_std": -0.10506388352655013, - "nauc_cv_recall_at_20_diff1": 0.6257002801120445, - "nauc_cv_recall_at_20_max": 0.6655198311208591, - "nauc_cv_recall_at_20_std": -0.19918402143466227, - "nauc_cv_recall_at_3_diff1": 0.6472713623239569, - "nauc_cv_recall_at_3_max": 0.5578714550455595, - "nauc_cv_recall_at_3_std": -0.13548415964522142, - "nauc_cv_recall_at_5_diff1": 0.6339326518340327, - "nauc_cv_recall_at_5_max": 0.5687226183317582, - "nauc_cv_recall_at_5_std": -0.15658878103255752, - "nauc_map_at_1000_diff1": 0.7108121576697071, - "nauc_map_at_1000_max": 0.5195954518905794, - "nauc_map_at_1000_std": -0.11880914292849726, - "nauc_map_at_100_diff1": 0.7108064511289186, - "nauc_map_at_100_max": 0.5196339191480183, - "nauc_map_at_100_std": -0.1188526782447686, - "nauc_map_at_10_diff1": 0.7089638573224181, - "nauc_map_at_10_max": 0.5198626919789708, - "nauc_map_at_10_std": -0.12106795154834728, - "nauc_map_at_1_diff1": 0.7388944519574545, - "nauc_map_at_1_max": 0.5035033567670426, - "nauc_map_at_1_std": -0.10506388352655013, - "nauc_map_at_20_diff1": 0.7104379728179321, - "nauc_map_at_20_max": 0.5201383461745716, - "nauc_map_at_20_std": -0.1193396676956454, - "nauc_map_at_3_diff1": 0.7078639395398761, - "nauc_map_at_3_max": 0.5200771632890915, - "nauc_map_at_3_std": -0.11708354472686522, - "nauc_map_at_5_diff1": 0.7087628559704173, - "nauc_map_at_5_max": 0.5201026397276295, - "nauc_map_at_5_std": -0.11894921046259736, - "nauc_mrr_at_1000_diff1": 0.7108121576697071, - "nauc_mrr_at_1000_max": 0.5195954518905794, - "nauc_mrr_at_1000_std": -0.11880914292849726, - "nauc_mrr_at_100_diff1": 0.7108064511289186, - "nauc_mrr_at_100_max": 0.5196339191480183, - "nauc_mrr_at_100_std": -0.1188526782447686, - "nauc_mrr_at_10_diff1": 0.7089638573224181, - "nauc_mrr_at_10_max": 0.5198626919789708, - "nauc_mrr_at_10_std": -0.12106795154834728, - "nauc_mrr_at_1_diff1": 0.7388944519574545, - "nauc_mrr_at_1_max": 0.5035033567670426, - "nauc_mrr_at_1_std": -0.10506388352655013, - "nauc_mrr_at_20_diff1": 0.7104379728179321, - "nauc_mrr_at_20_max": 0.5201383461745716, - "nauc_mrr_at_20_std": -0.1193396676956454, - "nauc_mrr_at_3_diff1": 0.7078639395398761, - "nauc_mrr_at_3_max": 0.5200771632890915, - "nauc_mrr_at_3_std": -0.11708354472686522, - "nauc_mrr_at_5_diff1": 0.7087628559704173, - "nauc_mrr_at_5_max": 0.5201026397276295, - "nauc_mrr_at_5_std": -0.11894921046259736, - "nauc_ndcg_at_1000_diff1": 0.7067180020680146, - "nauc_ndcg_at_1000_max": 0.5231648168494609, - "nauc_ndcg_at_1000_std": -0.12126906171818087, - "nauc_ndcg_at_100_diff1": 0.7066643145995184, - "nauc_ndcg_at_100_max": 0.5243091596475544, - "nauc_ndcg_at_100_std": -0.12259715494304686, - "nauc_ndcg_at_10_diff1": 0.6970018286625054, - "nauc_ndcg_at_10_max": 0.5270891134868764, - "nauc_ndcg_at_10_std": -0.13181013070978706, - "nauc_ndcg_at_1_diff1": 0.7388944519574545, - "nauc_ndcg_at_1_max": 0.5035033567670426, - "nauc_ndcg_at_1_std": -0.10506388352655013, - "nauc_ndcg_at_20_diff1": 0.7033385159678784, - "nauc_ndcg_at_20_max": 0.5287035840738418, - "nauc_ndcg_at_20_std": -0.12480446128344788, - "nauc_ndcg_at_3_diff1": 0.6960393372878148, - "nauc_ndcg_at_3_max": 0.5273632969286177, - "nauc_ndcg_at_3_std": -0.12075786696488118, - "nauc_ndcg_at_5_diff1": 0.6970584144874077, - "nauc_ndcg_at_5_max": 0.5276209122981296, - "nauc_ndcg_at_5_std": -0.1247281409835251, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.7008714596949908, - "nauc_precision_at_100_max": 0.749999999999983, - "nauc_precision_at_100_std": -0.37489884842824256, - "nauc_precision_at_10_diff1": 0.5953937130407729, - "nauc_precision_at_10_max": 0.5879408820585307, - "nauc_precision_at_10_std": -0.23504597034008565, - "nauc_precision_at_1_diff1": 0.7388944519574545, - "nauc_precision_at_1_max": 0.5035033567670426, - "nauc_precision_at_1_std": -0.10506388352655013, - "nauc_precision_at_20_diff1": 0.6257002801120534, - "nauc_precision_at_20_max": 0.6655198311208589, - "nauc_precision_at_20_std": -0.19918402143465108, - "nauc_precision_at_3_diff1": 0.6472713623239564, - "nauc_precision_at_3_max": 0.5578714550455599, - "nauc_precision_at_3_std": -0.13548415964522165, - "nauc_precision_at_5_diff1": 0.6339326518340342, - "nauc_precision_at_5_max": 0.568722618331759, - "nauc_precision_at_5_std": -0.15658878103255713, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.7008714596949883, - "nauc_recall_at_100_max": 0.7499999999999728, - "nauc_recall_at_100_std": -0.37489884842825993, - "nauc_recall_at_10_diff1": 0.5953937130407739, - "nauc_recall_at_10_max": 0.5879408820585317, - "nauc_recall_at_10_std": -0.2350459703400894, - "nauc_recall_at_1_diff1": 0.7388944519574545, - "nauc_recall_at_1_max": 0.5035033567670426, - "nauc_recall_at_1_std": -0.10506388352655013, - "nauc_recall_at_20_diff1": 0.6257002801120445, - "nauc_recall_at_20_max": 0.6655198311208591, - "nauc_recall_at_20_std": -0.19918402143466227, - "nauc_recall_at_3_diff1": 0.6472713623239569, - "nauc_recall_at_3_max": 0.5578714550455595, - "nauc_recall_at_3_std": -0.13548415964522142, - "nauc_recall_at_5_diff1": 0.6339326518340327, - "nauc_recall_at_5_max": 0.5687226183317582, - "nauc_recall_at_5_std": -0.15658878103255752, - "ndcg_at_1": 0.678, - "ndcg_at_10": 0.8094, - "ndcg_at_100": 0.82289, - "ndcg_at_1000": 0.82368, - "ndcg_at_20": 0.81708, - "ndcg_at_3": 0.77539, - "ndcg_at_5": 0.79516, - "precision_at_1": 0.678, - "precision_at_10": 0.09334, - "precision_at_100": 0.00994, - "precision_at_1000": 0.001, - "precision_at_20": 0.04816, - "precision_at_3": 0.28073, - "precision_at_5": 0.178, - "recall_at_1": 0.678, - "recall_at_10": 0.9334, - "recall_at_100": 0.994, - "recall_at_1000": 1.0, - "recall_at_20": 0.9632, - "recall_at_3": 0.8422, - "recall_at_5": 0.89 - } - ] - }, - "task_name": "Flickr30kT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/MSCOCOI2TRetrieval.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/MSCOCOI2TRetrieval.json deleted file mode 100644 index 1bc2b143dc..0000000000 --- a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/MSCOCOI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", - "evaluation_time": 57.833680629730225, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.544, - "cv_recall_at_10": 0.8592, - "cv_recall_at_100": 0.993, - "cv_recall_at_1000": 0.9998, - "cv_recall_at_20": 0.922, - "cv_recall_at_3": 0.7132, - "cv_recall_at_5": 0.7826, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.48279, - "map_at_1": 0.10886, - "map_at_10": 0.36753, - "map_at_100": 0.41438, - "map_at_1000": 0.41751, - "map_at_20": 0.39363, - "map_at_3": 0.2442, - "map_at_5": 0.31641, - "mrr_at_1": 0.544, - "mrr_at_10": 0.6441334920634935, - "mrr_at_100": 0.6505229831896451, - "mrr_at_1000": 0.6505722508994467, - "mrr_at_20": 0.6485590829105544, - "mrr_at_3": 0.6178333333333357, - "mrr_at_5": 0.6337633333333345, - "nauc_cv_recall_at_1000_diff1": -1.1517273576107072, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": -0.001093770841666523, - "nauc_cv_recall_at_100_max": 0.4157129518474038, - "nauc_cv_recall_at_100_std": -0.3355875683606853, - "nauc_cv_recall_at_10_diff1": 0.3116322316328111, - "nauc_cv_recall_at_10_max": 0.22344380665652067, - "nauc_cv_recall_at_10_std": -0.5484061734930179, - "nauc_cv_recall_at_1_diff1": 0.3620563318972254, - "nauc_cv_recall_at_1_max": 0.22304702323816303, - "nauc_cv_recall_at_1_std": -0.39325847713016515, - "nauc_cv_recall_at_20_diff1": 0.3174411644999864, - "nauc_cv_recall_at_20_max": 0.26129067011419665, - "nauc_cv_recall_at_20_std": -0.5873327108621224, - "nauc_cv_recall_at_3_diff1": 0.3190785659299513, - "nauc_cv_recall_at_3_max": 0.22356865681802765, - "nauc_cv_recall_at_3_std": -0.45729779105093976, - "nauc_cv_recall_at_5_diff1": 0.32854938408837603, - "nauc_cv_recall_at_5_max": 0.21763614808463352, - "nauc_cv_recall_at_5_std": -0.4901185955745346, - "nauc_map_at_1000_diff1": 0.21096215655518416, - "nauc_map_at_1000_max": 0.26437483183618576, - "nauc_map_at_1000_std": -0.3656676562792446, - "nauc_map_at_100_diff1": 0.21094575808032542, - "nauc_map_at_100_max": 0.2647084321742531, - "nauc_map_at_100_std": -0.36659141600302986, - "nauc_map_at_10_diff1": 0.20614205326831164, - "nauc_map_at_10_max": 0.25564678162479904, - "nauc_map_at_10_std": -0.3625738173870285, - "nauc_map_at_1_diff1": 0.3621079914040161, - "nauc_map_at_1_max": 0.22314949459519212, - "nauc_map_at_1_std": -0.39289670930715714, - "nauc_map_at_20_diff1": 0.2088651049333622, - "nauc_map_at_20_max": 0.2606844896581087, - "nauc_map_at_20_std": -0.36749366718366866, - "nauc_map_at_3_diff1": 0.23125083203784524, - "nauc_map_at_3_max": 0.23835700021972306, - "nauc_map_at_3_std": -0.3525765196558998, - "nauc_map_at_5_diff1": 0.20395917414865206, - "nauc_map_at_5_max": 0.24890333320816935, - "nauc_map_at_5_std": -0.3551844385945933, - "nauc_mrr_at_1000_diff1": 0.3462109269235247, - "nauc_mrr_at_1000_max": 0.22341427843542805, - "nauc_mrr_at_1000_std": -0.421124236204143, - "nauc_mrr_at_100_diff1": 0.34615985793627224, - "nauc_mrr_at_100_max": 0.2234417011721278, - "nauc_mrr_at_100_std": -0.42112571190402276, - "nauc_mrr_at_10_diff1": 0.3457260635691663, - "nauc_mrr_at_10_max": 0.22268948650151105, - "nauc_mrr_at_10_std": -0.4235752064832113, - "nauc_mrr_at_1_diff1": 0.3620563318972254, - "nauc_mrr_at_1_max": 0.22304702323816303, - "nauc_mrr_at_1_std": -0.39325847713016515, - "nauc_mrr_at_20_diff1": 0.34612102767284886, - "nauc_mrr_at_20_max": 0.22330420489291178, - "nauc_mrr_at_20_std": -0.42260480382387744, - "nauc_mrr_at_3_diff1": 0.3436568869473764, - "nauc_mrr_at_3_max": 0.22268159334592896, - "nauc_mrr_at_3_std": -0.41793547872411135, - "nauc_mrr_at_5_diff1": 0.3462959273283977, - "nauc_mrr_at_5_max": 0.22190940766444717, - "nauc_mrr_at_5_std": -0.42173732217738524, - "nauc_ndcg_at_1000_diff1": 0.24740773356795673, - "nauc_ndcg_at_1000_max": 0.2633139225090684, - "nauc_ndcg_at_1000_std": -0.3825727328439968, - "nauc_ndcg_at_100_diff1": 0.2439247459069718, - "nauc_ndcg_at_100_max": 0.27221683198323254, - "nauc_ndcg_at_100_std": -0.38503378892448387, - "nauc_ndcg_at_10_diff1": 0.2322351990974726, - "nauc_ndcg_at_10_max": 0.25133559217064416, - "nauc_ndcg_at_10_std": -0.38906206049843284, - "nauc_ndcg_at_1_diff1": 0.3620563318972254, - "nauc_ndcg_at_1_max": 0.22304702323816303, - "nauc_ndcg_at_1_std": -0.39325847713016515, - "nauc_ndcg_at_20_diff1": 0.2368585564555519, - "nauc_ndcg_at_20_max": 0.2586236191288364, - "nauc_ndcg_at_20_std": -0.39571963457704934, - "nauc_ndcg_at_3_diff1": 0.25241282801383136, - "nauc_ndcg_at_3_max": 0.23505398892057644, - "nauc_ndcg_at_3_std": -0.37119292314398367, - "nauc_ndcg_at_5_diff1": 0.2325478110853486, - "nauc_ndcg_at_5_max": 0.24393791588243935, - "nauc_ndcg_at_5_std": -0.37684221871528933, - "nauc_precision_at_1000_diff1": -0.060776645601850005, - "nauc_precision_at_1000_max": 0.45061728395062717, - "nauc_precision_at_1000_std": 0.49837165139806966, - "nauc_precision_at_100_diff1": 0.16399269478318815, - "nauc_precision_at_100_max": 0.3174441756422564, - "nauc_precision_at_100_std": -0.2719045017872643, - "nauc_precision_at_10_diff1": 0.1834674594026287, - "nauc_precision_at_10_max": 0.2475111637901415, - "nauc_precision_at_10_std": -0.36964525963118233, - "nauc_precision_at_1_diff1": 0.3620563318972254, - "nauc_precision_at_1_max": 0.22304702323816303, - "nauc_precision_at_1_std": -0.39325847713016515, - "nauc_precision_at_20_diff1": 0.18190648500233034, - "nauc_precision_at_20_max": 0.2543371579459487, - "nauc_precision_at_20_std": -0.3684848720360581, - "nauc_precision_at_3_diff1": 0.2216308838348382, - "nauc_precision_at_3_max": 0.23889515868420422, - "nauc_precision_at_3_std": -0.3624204875957743, - "nauc_precision_at_5_diff1": 0.19290462095741936, - "nauc_precision_at_5_max": 0.2465663979201694, - "nauc_precision_at_5_std": -0.36446145380745226, - "nauc_recall_at_1000_diff1": -0.04813353912994217, - "nauc_recall_at_1000_max": 0.47849854227402444, - "nauc_recall_at_1000_std": 0.5228805808037352, - "nauc_recall_at_100_diff1": 0.16501981261932044, - "nauc_recall_at_100_max": 0.3183331361282548, - "nauc_recall_at_100_std": -0.2726147056630754, - "nauc_recall_at_10_diff1": 0.1835675928057196, - "nauc_recall_at_10_max": 0.24766746744342197, - "nauc_recall_at_10_std": -0.3696577837871595, - "nauc_recall_at_1_diff1": 0.3621079914040161, - "nauc_recall_at_1_max": 0.22314949459519212, - "nauc_recall_at_1_std": -0.39289670930715714, - "nauc_recall_at_20_diff1": 0.18207077982715833, - "nauc_recall_at_20_max": 0.254625446742885, - "nauc_recall_at_20_std": -0.36840440513586076, - "nauc_recall_at_3_diff1": 0.2218119797611259, - "nauc_recall_at_3_max": 0.23903410739410114, - "nauc_recall_at_3_std": -0.3625477824595922, - "nauc_recall_at_5_diff1": 0.19309140539859648, - "nauc_recall_at_5_max": 0.2467762588423194, - "nauc_recall_at_5_std": -0.36457390258550315, - "ndcg_at_1": 0.544, - "ndcg_at_10": 0.48279, - "ndcg_at_100": 0.6064, - "ndcg_at_1000": 0.63735, - "ndcg_at_20": 0.53283, - "ndcg_at_3": 0.47129, - "ndcg_at_5": 0.41454, - "precision_at_1": 0.544, - "precision_at_10": 0.24992, - "precision_at_100": 0.04259, - "precision_at_1000": 0.00496, - "precision_at_20": 0.15413, - "precision_at_3": 0.45027, - "precision_at_5": 0.37584, - "recall_at_1": 0.10886, - "recall_at_10": 0.50001, - "recall_at_100": 0.85215, - "recall_at_1000": 0.99216, - "recall_at_20": 0.6168, - "recall_at_3": 0.27025, - "recall_at_5": 0.37596 - } - ] - }, - "task_name": "MSCOCOI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/STS14.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/STS14.json deleted file mode 100644 index 55c1a7509b..0000000000 --- a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/STS14.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", - "evaluation_time": 4.692688703536987, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "cosine_pearson": 0.8302122731320196, - "cosine_spearman": 0.7867521693830515, - "euclidean_pearson": 0.8140308674026765, - "euclidean_spearman": 0.7867522096109674, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7867521693830515, - "manhattan_pearson": 0.813071258051059, - "manhattan_spearman": 0.7856621071659424, - "pearson": 0.8302122731320196, - "spearman": 0.7867521693830515 - } - ] - }, - "task_name": "STS14" -} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json deleted file mode 100644 index 53cf522da6..0000000000 --- a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "jinaai/jina-clip-v1", "revision": "06150c7c382d7a4faedc7d5a0d8cdb59308968f4", "release_date": "2024-05-30", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "JinaCLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOI2TRetrieval.json b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOI2TRetrieval.json deleted file mode 100644 index 2be160c172..0000000000 --- a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOI2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", - "evaluation_time": 53.00429034233093, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.48279, - "map_at_1": 0.10886, - "map_at_10": 0.36754, - "map_at_100": 0.41438, - "map_at_1000": 0.41752, - "map_at_20": 0.39363, - "map_at_3": 0.2442, - "map_at_5": 0.31642, - "mrr_at_1": 0.544, - "mrr_at_10": 0.6441401587301602, - "mrr_at_100": 0.650529731505628, - "mrr_at_1000": 0.6505789992154296, - "mrr_at_20": 0.648565749577221, - "mrr_at_3": 0.6178333333333357, - "mrr_at_5": 0.6338033333333345, - "nauc_map_at_1000_diff1": 0.21096399203394334, - "nauc_map_at_1000_max": 0.2643789873573568, - "nauc_map_at_1000_std": -0.3656602505905553, - "nauc_map_at_100_diff1": 0.21094758478668932, - "nauc_map_at_100_max": 0.2647125569770621, - "nauc_map_at_100_std": -0.36658408307145496, - "nauc_map_at_10_diff1": 0.20614378960832005, - "nauc_map_at_10_max": 0.2556506736639637, - "nauc_map_at_10_std": -0.3625667597727067, - "nauc_map_at_1_diff1": 0.3621079914040161, - "nauc_map_at_1_max": 0.22314949459519212, - "nauc_map_at_1_std": -0.39289670930715714, - "nauc_map_at_20_diff1": 0.2088668661351603, - "nauc_map_at_20_max": 0.26068843280053283, - "nauc_map_at_20_std": -0.3674865778762073, - "nauc_map_at_3_diff1": 0.23125083203784524, - "nauc_map_at_3_max": 0.23835700021972306, - "nauc_map_at_3_std": -0.3525765196558998, - "nauc_map_at_5_diff1": 0.20396930020356405, - "nauc_map_at_5_max": 0.24892695318464714, - "nauc_map_at_5_std": -0.3551372962226835, - "nauc_mrr_at_1000_diff1": 0.34622127332034, - "nauc_mrr_at_1000_max": 0.2234325242238912, - "nauc_mrr_at_1000_std": -0.4210887792813959, - "nauc_mrr_at_100_diff1": 0.34617020154363415, - "nauc_mrr_at_100_max": 0.2234599441533202, - "nauc_mrr_at_100_std": -0.4210902614211229, - "nauc_mrr_at_10_diff1": 0.3457361472170574, - "nauc_mrr_at_10_max": 0.22270683416516163, - "nauc_mrr_at_10_std": -0.42354086081618936, - "nauc_mrr_at_1_diff1": 0.3620563318972254, - "nauc_mrr_at_1_max": 0.22304702323816303, - "nauc_mrr_at_1_std": -0.39325847713016515, - "nauc_mrr_at_20_diff1": 0.3461312655849626, - "nauc_mrr_at_20_max": 0.22332181681230495, - "nauc_mrr_at_20_std": -0.42256993930033726, - "nauc_mrr_at_3_diff1": 0.3436568869473764, - "nauc_mrr_at_3_max": 0.22268159334592896, - "nauc_mrr_at_3_std": -0.41793547872411135, - "nauc_mrr_at_5_diff1": 0.3463546807212511, - "nauc_mrr_at_5_max": 0.22201030832472302, - "nauc_mrr_at_5_std": -0.4215372140619334, - "nauc_ndcg_at_1000_diff1": 0.2474117966134631, - "nauc_ndcg_at_1000_max": 0.2633225510736001, - "nauc_ndcg_at_1000_std": -0.38255756076688163, - "nauc_ndcg_at_100_diff1": 0.24392830523701964, - "nauc_ndcg_at_100_max": 0.2722245687662711, - "nauc_ndcg_at_100_std": -0.38502007102726465, - "nauc_ndcg_at_10_diff1": 0.23223810393318697, - "nauc_ndcg_at_10_max": 0.25134167852059397, - "nauc_ndcg_at_10_std": -0.3890511501621063, - "nauc_ndcg_at_1_diff1": 0.3620563318972254, - "nauc_ndcg_at_1_max": 0.22304702323816303, - "nauc_ndcg_at_1_std": -0.39325847713016515, - "nauc_ndcg_at_20_diff1": 0.2368616298300584, - "nauc_ndcg_at_20_max": 0.2586300416187524, - "nauc_ndcg_at_20_std": -0.39570826135429343, - "nauc_ndcg_at_3_diff1": 0.25241282801383136, - "nauc_ndcg_at_3_max": 0.23505398892057644, - "nauc_ndcg_at_3_std": -0.37119292314398367, - "nauc_ndcg_at_5_diff1": 0.23258181324662106, - "nauc_ndcg_at_5_max": 0.24401204310948493, - "nauc_ndcg_at_5_std": -0.3766951207685948, - "nauc_precision_at_1000_diff1": -0.060776645601850005, - "nauc_precision_at_1000_max": 0.45061728395062717, - "nauc_precision_at_1000_std": 0.49837165139806966, - "nauc_precision_at_100_diff1": 0.16399269478318815, - "nauc_precision_at_100_max": 0.3174441756422564, - "nauc_precision_at_100_std": -0.2719045017872643, - "nauc_precision_at_10_diff1": 0.1834674594026287, - "nauc_precision_at_10_max": 0.2475111637901415, - "nauc_precision_at_10_std": -0.36964525963118233, - "nauc_precision_at_1_diff1": 0.3620563318972254, - "nauc_precision_at_1_max": 0.22304702323816303, - "nauc_precision_at_1_std": -0.39325847713016515, - "nauc_precision_at_20_diff1": 0.18190648500233034, - "nauc_precision_at_20_max": 0.2543371579459487, - "nauc_precision_at_20_std": -0.3684848720360581, - "nauc_precision_at_3_diff1": 0.2216308838348382, - "nauc_precision_at_3_max": 0.23889515868420422, - "nauc_precision_at_3_std": -0.3624204875957743, - "nauc_precision_at_5_diff1": 0.19295589426112914, - "nauc_precision_at_5_max": 0.24668879287833217, - "nauc_precision_at_5_std": -0.36421771939542763, - "nauc_recall_at_1000_diff1": -0.04813353912994217, - "nauc_recall_at_1000_max": 0.47849854227402444, - "nauc_recall_at_1000_std": 0.5228805808037352, - "nauc_recall_at_100_diff1": 0.16501981261932044, - "nauc_recall_at_100_max": 0.3183331361282548, - "nauc_recall_at_100_std": -0.2726147056630754, - "nauc_recall_at_10_diff1": 0.1835675928057196, - "nauc_recall_at_10_max": 0.24766746744342197, - "nauc_recall_at_10_std": -0.3696577837871595, - "nauc_recall_at_1_diff1": 0.3621079914040161, - "nauc_recall_at_1_max": 0.22314949459519212, - "nauc_recall_at_1_std": -0.39289670930715714, - "nauc_recall_at_20_diff1": 0.18207077982715833, - "nauc_recall_at_20_max": 0.254625446742885, - "nauc_recall_at_20_std": -0.36840440513586076, - "nauc_recall_at_3_diff1": 0.2218119797611259, - "nauc_recall_at_3_max": 0.23903410739410114, - "nauc_recall_at_3_std": -0.3625477824595922, - "nauc_recall_at_5_diff1": 0.1931427010978174, - "nauc_recall_at_5_max": 0.24689867912760904, - "nauc_recall_at_5_std": -0.364330180957868, - "ndcg_at_1": 0.544, - "ndcg_at_10": 0.48279, - "ndcg_at_100": 0.6064, - "ndcg_at_1000": 0.63736, - "ndcg_at_20": 0.53283, - "ndcg_at_3": 0.47129, - "ndcg_at_5": 0.41457, - "precision_at_1": 0.544, - "precision_at_10": 0.24992, - "precision_at_100": 0.04259, - "precision_at_1000": 0.00496, - "precision_at_20": 0.15413, - "precision_at_3": 0.45027, - "precision_at_5": 0.37588, - "recall_at_1": 0.10886, - "recall_at_10": 0.50001, - "recall_at_100": 0.85215, - "recall_at_1000": 0.99216, - "recall_at_20": 0.6168, - "recall_at_3": 0.27025, - "recall_at_5": 0.376 - } - ] - }, - "task_name": "MSCOCOI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOT2IRetrieval.json b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOT2IRetrieval.json deleted file mode 100644 index 101f427b01..0000000000 --- a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/MSCOCOT2IRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", - "evaluation_time": 93.58067321777344, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.58366, - "map_at_1": 0.41549, - "map_at_10": 0.52488, - "map_at_100": 0.53435, - "map_at_1000": 0.53448, - "map_at_20": 0.5309, - "map_at_3": 0.49362, - "map_at_5": 0.51155, - "mrr_at_1": 0.41601838042645817, - "mrr_at_10": 0.5255243392854887, - "mrr_at_100": 0.5348733342315144, - "mrr_at_1000": 0.5349968576566357, - "mrr_at_20": 0.5314956818071761, - "mrr_at_3": 0.49423596275545995, - "mrr_at_5": 0.5122092788907409, - "nauc_map_at_1000_diff1": 0.5624393151830621, - "nauc_map_at_1000_max": 0.1883984047435345, - "nauc_map_at_1000_std": -0.300281213090816, - "nauc_map_at_100_diff1": 0.5623891134436314, - "nauc_map_at_100_max": 0.18846608709434864, - "nauc_map_at_100_std": -0.3001965535748168, - "nauc_map_at_10_diff1": 0.5603327032580525, - "nauc_map_at_10_max": 0.18715837124430523, - "nauc_map_at_10_std": -0.302544478256696, - "nauc_map_at_1_diff1": 0.6091183832361869, - "nauc_map_at_1_max": 0.19445927993458345, - "nauc_map_at_1_std": -0.2806441521864675, - "nauc_map_at_20_diff1": 0.5614027805327534, - "nauc_map_at_20_max": 0.18786747432769882, - "nauc_map_at_20_std": -0.3015101845331886, - "nauc_map_at_3_diff1": 0.5621560332044606, - "nauc_map_at_3_max": 0.18742943309539936, - "nauc_map_at_3_std": -0.2996055329867434, - "nauc_map_at_5_diff1": 0.5598162631668463, - "nauc_map_at_5_max": 0.18579277507099531, - "nauc_map_at_5_std": -0.30325696648318895, - "nauc_mrr_at_1000_diff1": 0.5618384854801253, - "nauc_mrr_at_1000_max": 0.1893320924160437, - "nauc_mrr_at_1000_std": -0.29919748725758244, - "nauc_mrr_at_100_diff1": 0.5617865037742961, - "nauc_mrr_at_100_max": 0.18940792340998744, - "nauc_mrr_at_100_std": -0.29910507969649847, - "nauc_mrr_at_10_diff1": 0.5595973365694361, - "nauc_mrr_at_10_max": 0.18831353355793895, - "nauc_mrr_at_10_std": -0.3012225279840895, - "nauc_mrr_at_1_diff1": 0.6086677033863201, - "nauc_mrr_at_1_max": 0.19525350096143335, - "nauc_mrr_at_1_std": -0.2797335075064764, - "nauc_mrr_at_20_diff1": 0.5607498445253036, - "nauc_mrr_at_20_max": 0.18897520934910883, - "nauc_mrr_at_20_std": -0.30027624919809554, - "nauc_mrr_at_3_diff1": 0.5614777749783645, - "nauc_mrr_at_3_max": 0.18847496584680384, - "nauc_mrr_at_3_std": -0.29848619493001144, - "nauc_mrr_at_5_diff1": 0.5590866529097342, - "nauc_mrr_at_5_max": 0.18696239905084666, - "nauc_mrr_at_5_std": -0.3019481978381332, - "nauc_ndcg_at_1000_diff1": 0.5544996950498666, - "nauc_ndcg_at_1000_max": 0.1899712267352402, - "nauc_ndcg_at_1000_std": -0.3002746853968903, - "nauc_ndcg_at_100_diff1": 0.5531116875181972, - "nauc_ndcg_at_100_max": 0.19235157602831457, - "nauc_ndcg_at_100_std": -0.2968862811392692, - "nauc_ndcg_at_10_diff1": 0.5418234351636028, - "nauc_ndcg_at_10_max": 0.18525740550465958, - "nauc_ndcg_at_10_std": -0.3108723228378656, - "nauc_ndcg_at_1_diff1": 0.6086677033863201, - "nauc_ndcg_at_1_max": 0.19525350096143335, - "nauc_ndcg_at_1_std": -0.2797335075064764, - "nauc_ndcg_at_20_diff1": 0.5457713006659793, - "nauc_ndcg_at_20_max": 0.18823783560110394, - "nauc_ndcg_at_20_std": -0.30723165105906064, - "nauc_ndcg_at_3_diff1": 0.5465925294717622, - "nauc_ndcg_at_3_max": 0.18498021161717793, - "nauc_ndcg_at_3_std": -0.3052417229375767, - "nauc_ndcg_at_5_diff1": 0.5414141206084832, - "nauc_ndcg_at_5_max": 0.18187192357717108, - "nauc_ndcg_at_5_std": -0.3122724524680724, - "nauc_precision_at_1000_diff1": -0.2717903666306267, - "nauc_precision_at_1000_max": 0.3971860188038004, - "nauc_precision_at_1000_std": 0.552726119141797, - "nauc_precision_at_100_diff1": 0.15897061095455042, - "nauc_precision_at_100_max": 0.42848198127641873, - "nauc_precision_at_100_std": 0.24775900337098317, - "nauc_precision_at_10_diff1": 0.45241283409781885, - "nauc_precision_at_10_max": 0.18182339360260374, - "nauc_precision_at_10_std": -0.34048980216449526, - "nauc_precision_at_1_diff1": 0.6086677033863201, - "nauc_precision_at_1_max": 0.19525350096143335, - "nauc_precision_at_1_std": -0.2797335075064764, - "nauc_precision_at_20_diff1": 0.42977222993638897, - "nauc_precision_at_20_max": 0.20622305981256997, - "nauc_precision_at_20_std": -0.31448081933782945, - "nauc_precision_at_3_diff1": 0.4972012863500112, - "nauc_precision_at_3_max": 0.17750313343860488, - "nauc_precision_at_3_std": -0.3218876469389674, - "nauc_precision_at_5_diff1": 0.47444239516674885, - "nauc_precision_at_5_max": 0.16915063391060323, - "nauc_precision_at_5_std": -0.3422618390030155, - "nauc_recall_at_1000_diff1": 0.3832069088634286, - "nauc_recall_at_1000_max": 0.8901286684038919, - "nauc_recall_at_1000_std": 0.8681810401874042, - "nauc_recall_at_100_diff1": 0.4185975008775339, - "nauc_recall_at_100_max": 0.45763959222653156, - "nauc_recall_at_100_std": 0.1007829133427318, - "nauc_recall_at_10_diff1": 0.4592974110602662, - "nauc_recall_at_10_max": 0.1757829270013686, - "nauc_recall_at_10_std": -0.3502055124676758, - "nauc_recall_at_1_diff1": 0.6091183832361869, - "nauc_recall_at_1_max": 0.19445927993458345, - "nauc_recall_at_1_std": -0.2806441521864675, - "nauc_recall_at_20_diff1": 0.4522393445441416, - "nauc_recall_at_20_max": 0.19461709532525517, - "nauc_recall_at_20_std": -0.3418318635602553, - "nauc_recall_at_3_diff1": 0.49905878884342847, - "nauc_recall_at_3_max": 0.17580402067909676, - "nauc_recall_at_3_std": -0.32419684954654177, - "nauc_recall_at_5_diff1": 0.4770455272089403, - "nauc_recall_at_5_max": 0.1660363394348069, - "nauc_recall_at_5_std": -0.3461249105285916, - "ndcg_at_1": 0.41602, - "ndcg_at_10": 0.58366, - "ndcg_at_100": 0.62879, - "ndcg_at_1000": 0.63154, - "ndcg_at_20": 0.6053, - "ndcg_at_3": 0.51938, - "ndcg_at_5": 0.55166, - "precision_at_1": 0.41602, - "precision_at_10": 0.0772, - "precision_at_100": 0.00985, - "precision_at_1000": 0.00101, - "precision_at_20": 0.04291, - "precision_at_3": 0.1981, - "precision_at_5": 0.1346, - "recall_at_1": 0.41549, - "recall_at_10": 0.76983, - "recall_at_100": 0.9782, - "recall_at_1000": 0.99865, - "recall_at_20": 0.85476, - "recall_at_3": 0.59323, - "recall_at_5": 0.67157 - } - ] - }, - "task_name": "MSCOCOT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/STS12.json b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/STS12.json deleted file mode 100644 index a5b65a252d..0000000000 --- a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/STS12.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", - "evaluation_time": 5.472690582275391, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "cosine_pearson": 0.8338917619581533, - "cosine_spearman": 0.7351972128901206, - "euclidean_pearson": 0.7935770054482947, - "euclidean_spearman": 0.7351757102447369, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7351972128901206, - "manhattan_pearson": 0.7933747110044767, - "manhattan_spearman": 0.7344695134800555, - "pearson": 0.8338917619581533, - "spearman": 0.7351972128901206 - } - ] - }, - "task_name": "STS12" -} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/model_meta.json b/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/model_meta.json deleted file mode 100644 index 1cef0d24b8..0000000000 --- a/results-mieb/jinaai__jina-clip-v1/1cbe5e8b11ea3728df0b610d5453dfe739804aa9/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "jinaai/jina-clip-v1", "revision": "1cbe5e8b11ea3728df0b610d5453dfe739804aa9", "release_date": "2024-05-30", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "JinaCLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNIST.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNIST.json deleted file mode 100644 index fdb32990a8..0000000000 --- a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 88.12602519989014, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "accuracy": 0.8676, - "f1": 0.8647519484598748, - "f1_weighted": 0.8674255847302279, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8676, - "scores_per_experiment": [ - { - "accuracy": 0.8841, - "f1": 0.8822100989512647, - "f1_weighted": 0.8842956096110661 - }, - { - "accuracy": 0.8418, - "f1": 0.8348510573099901, - "f1_weighted": 0.8395228217754684 - }, - { - "accuracy": 0.8695, - "f1": 0.8673706569160476, - "f1_weighted": 0.8697790017131949 - }, - { - "accuracy": 0.8837, - "f1": 0.881759695651958, - "f1_weighted": 0.8836890214883635 - }, - { - "accuracy": 0.8589, - "f1": 0.8575682334701138, - "f1_weighted": 0.8598414690630468 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNISTZeroShot.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNISTZeroShot.json deleted file mode 100644 index d1cc3bbdbe..0000000000 --- a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/MNISTZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 52.55812096595764, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "accuracy": 0.3773, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.3773 - } - ] - }, - "task_name": "MNISTZeroShot" -} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/NIGHTSI2IRetrieval.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/NIGHTSI2IRetrieval.json deleted file mode 100644 index 2a5b209388..0000000000 --- a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/NIGHTSI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", - "evaluation_time": 239.57693338394165, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.07406, - "cv_recall_at_10": 0.45, - "cv_recall_at_100": 0.91651, - "cv_recall_at_1000": 0.98396, - "cv_recall_at_20": 0.66462, - "cv_recall_at_3": 0.18632, - "cv_recall_at_5": 0.27877, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.23151, - "map_at_1": 0.07406, - "map_at_10": 0.16568, - "map_at_100": 0.18777, - "map_at_1000": 0.1881, - "map_at_20": 0.18052, - "map_at_3": 0.12241, - "map_at_5": 0.14358, - "mrr_at_1": 0.0740566037735849, - "mrr_at_10": 0.16568134171907734, - "mrr_at_100": 0.18776707411027074, - "mrr_at_1000": 0.18810404013437873, - "mrr_at_20": 0.1805234731481328, - "mrr_at_3": 0.12240566037735866, - "mrr_at_5": 0.14358490566037718, - "nauc_cv_recall_at_1000_diff1": -0.41488163893008945, - "nauc_cv_recall_at_1000_max": 0.8024386225078316, - "nauc_cv_recall_at_1000_std": 0.6085296863843472, - "nauc_cv_recall_at_100_diff1": -0.1561532334214285, - "nauc_cv_recall_at_100_max": 0.3230256320984146, - "nauc_cv_recall_at_100_std": 0.42613429552612103, - "nauc_cv_recall_at_10_diff1": 0.035585056376463996, - "nauc_cv_recall_at_10_max": 0.041574167756887395, - "nauc_cv_recall_at_10_std": -0.031567607900982, - "nauc_cv_recall_at_1_diff1": 0.07258958982395641, - "nauc_cv_recall_at_1_max": -0.02721575557260758, - "nauc_cv_recall_at_1_std": -0.047981761598526164, - "nauc_cv_recall_at_20_diff1": -0.034360664129741786, - "nauc_cv_recall_at_20_max": 0.13687544288877665, - "nauc_cv_recall_at_20_std": 0.005421709114206555, - "nauc_cv_recall_at_3_diff1": 0.07796405401160618, - "nauc_cv_recall_at_3_max": 0.0022447113971254527, - "nauc_cv_recall_at_3_std": -0.02193083840413663, - "nauc_cv_recall_at_5_diff1": 0.058721315220139715, - "nauc_cv_recall_at_5_max": -0.01734274247588793, - "nauc_cv_recall_at_5_std": -0.06046753905236499, - "nauc_map_at_1000_diff1": 0.05722146065713886, - "nauc_map_at_1000_max": 0.004143611940867708, - "nauc_map_at_1000_std": -0.031865337917475284, - "nauc_map_at_100_diff1": 0.05693723750527088, - "nauc_map_at_100_max": 0.004399669635771658, - "nauc_map_at_100_std": -0.031274561293767475, - "nauc_map_at_10_diff1": 0.0624486940125799, - "nauc_map_at_10_max": -0.0022541784601356617, - "nauc_map_at_10_std": -0.03855323923750413, - "nauc_map_at_1_diff1": 0.07258958982395641, - "nauc_map_at_1_max": -0.02721575557260758, - "nauc_map_at_1_std": -0.047981761598526164, - "nauc_map_at_20_diff1": 0.05655287179993111, - "nauc_map_at_20_max": 0.005117092389360872, - "nauc_map_at_20_std": -0.03549998596719167, - "nauc_map_at_3_diff1": 0.07351556234707848, - "nauc_map_at_3_max": -0.007695467804342238, - "nauc_map_at_3_std": -0.031239859746265138, - "nauc_map_at_5_diff1": 0.06630205643433205, - "nauc_map_at_5_max": -0.01497880630240342, - "nauc_map_at_5_std": -0.045952638027913155, - "nauc_mrr_at_1000_diff1": 0.05722146065713886, - "nauc_mrr_at_1000_max": 0.004143611940867708, - "nauc_mrr_at_1000_std": -0.031865337917475284, - "nauc_mrr_at_100_diff1": 0.05693723750527088, - "nauc_mrr_at_100_max": 0.004399669635771658, - "nauc_mrr_at_100_std": -0.031274561293767475, - "nauc_mrr_at_10_diff1": 0.0624486940125799, - "nauc_mrr_at_10_max": -0.0022541784601356617, - "nauc_mrr_at_10_std": -0.03855323923750413, - "nauc_mrr_at_1_diff1": 0.07258958982395641, - "nauc_mrr_at_1_max": -0.02721575557260758, - "nauc_mrr_at_1_std": -0.047981761598526164, - "nauc_mrr_at_20_diff1": 0.05655287179993111, - "nauc_mrr_at_20_max": 0.005117092389360872, - "nauc_mrr_at_20_std": -0.03549998596719167, - "nauc_mrr_at_3_diff1": 0.07351556234707848, - "nauc_mrr_at_3_max": -0.007695467804342238, - "nauc_mrr_at_3_std": -0.031239859746265138, - "nauc_mrr_at_5_diff1": 0.06630205643433205, - "nauc_mrr_at_5_max": -0.01497880630240342, - "nauc_mrr_at_5_std": -0.045952638027913155, - "nauc_ndcg_at_1000_diff1": 0.042063964651544127, - "nauc_ndcg_at_1000_max": 0.02784904871625325, - "nauc_ndcg_at_1000_std": -0.012916227002940112, - "nauc_ndcg_at_100_diff1": 0.03566953457249385, - "nauc_ndcg_at_100_max": 0.03549475391663095, - "nauc_ndcg_at_100_std": 0.0044041422671843595, - "nauc_ndcg_at_10_diff1": 0.054097030876250334, - "nauc_ndcg_at_10_max": 0.011522468333852415, - "nauc_ndcg_at_10_std": -0.036407332290186015, - "nauc_ndcg_at_1_diff1": 0.07258958982395641, - "nauc_ndcg_at_1_max": -0.02721575557260758, - "nauc_ndcg_at_1_std": -0.047981761598526164, - "nauc_ndcg_at_20_diff1": 0.03398315242507301, - "nauc_ndcg_at_20_max": 0.03796440931298207, - "nauc_ndcg_at_20_std": -0.025618940226155185, - "nauc_ndcg_at_3_diff1": 0.07488855851721819, - "nauc_ndcg_at_3_max": -0.004377117227156189, - "nauc_ndcg_at_3_std": -0.028154289921466857, - "nauc_ndcg_at_5_diff1": 0.06380000100648368, - "nauc_ndcg_at_5_max": -0.01553121813581108, - "nauc_ndcg_at_5_std": -0.05057710684051813, - "nauc_precision_at_1000_diff1": -0.41488163893007834, - "nauc_precision_at_1000_max": 0.8024386225078226, - "nauc_precision_at_1000_std": 0.6085296863843523, - "nauc_precision_at_100_diff1": -0.1561532334214323, - "nauc_precision_at_100_max": 0.3230256320984115, - "nauc_precision_at_100_std": 0.42613429552612053, - "nauc_precision_at_10_diff1": 0.035585056376464114, - "nauc_precision_at_10_max": 0.041574167756887555, - "nauc_precision_at_10_std": -0.03156760790098189, - "nauc_precision_at_1_diff1": 0.07258958982395641, - "nauc_precision_at_1_max": -0.02721575557260758, - "nauc_precision_at_1_std": -0.047981761598526164, - "nauc_precision_at_20_diff1": -0.034360664129741654, - "nauc_precision_at_20_max": 0.13687544288877623, - "nauc_precision_at_20_std": 0.0054217091142065525, - "nauc_precision_at_3_diff1": 0.0779640540116062, - "nauc_precision_at_3_max": 0.0022447113971254535, - "nauc_precision_at_3_std": -0.021930838404136517, - "nauc_precision_at_5_diff1": 0.0587213152201399, - "nauc_precision_at_5_max": -0.017342742475887782, - "nauc_precision_at_5_std": -0.060467539052364704, - "nauc_recall_at_1000_diff1": -0.41488163893008945, - "nauc_recall_at_1000_max": 0.8024386225078316, - "nauc_recall_at_1000_std": 0.6085296863843472, - "nauc_recall_at_100_diff1": -0.1561532334214285, - "nauc_recall_at_100_max": 0.3230256320984146, - "nauc_recall_at_100_std": 0.42613429552612103, - "nauc_recall_at_10_diff1": 0.035585056376463996, - "nauc_recall_at_10_max": 0.041574167756887395, - "nauc_recall_at_10_std": -0.031567607900982, - "nauc_recall_at_1_diff1": 0.07258958982395641, - "nauc_recall_at_1_max": -0.02721575557260758, - "nauc_recall_at_1_std": -0.047981761598526164, - "nauc_recall_at_20_diff1": -0.034360664129741786, - "nauc_recall_at_20_max": 0.13687544288877665, - "nauc_recall_at_20_std": 0.005421709114206555, - "nauc_recall_at_3_diff1": 0.07796405401160618, - "nauc_recall_at_3_max": 0.0022447113971254527, - "nauc_recall_at_3_std": -0.02193083840413663, - "nauc_recall_at_5_diff1": 0.058721315220139715, - "nauc_recall_at_5_max": -0.01734274247588793, - "nauc_recall_at_5_std": -0.06046753905236499, - "ndcg_at_1": 0.07406, - "ndcg_at_10": 0.23151, - "ndcg_at_100": 0.33399, - "ndcg_at_1000": 0.34262, - "ndcg_at_20": 0.2857, - "ndcg_at_3": 0.13877, - "ndcg_at_5": 0.1769, - "precision_at_1": 0.07406, - "precision_at_10": 0.045, - "precision_at_100": 0.00917, - "precision_at_1000": 0.00098, - "precision_at_20": 0.03323, - "precision_at_3": 0.06211, - "precision_at_5": 0.05575, - "recall_at_1": 0.07406, - "recall_at_10": 0.45, - "recall_at_100": 0.91651, - "recall_at_1000": 0.98396, - "recall_at_20": 0.66462, - "recall_at_3": 0.18632, - "recall_at_5": 0.27877 - } - ] - }, - "task_name": "NIGHTSI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/OxfordFlowersClassification.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/OxfordFlowersClassification.json deleted file mode 100644 index 46cd0c0c23..0000000000 --- a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/OxfordFlowersClassification.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", - "evaluation_time": 173.65987730026245, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "accuracy": 0.9609803921568627, - "f1": 0.9607429107615776, - "f1_weighted": 0.9605803695939095, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9609803921568627, - "scores_per_experiment": [ - { - "accuracy": 0.9637254901960784, - "f1": 0.9638390179334648, - "f1_weighted": 0.9636168436654212 - }, - { - "accuracy": 0.9588235294117647, - "f1": 0.9585142741395463, - "f1_weighted": 0.9583485618271098 - }, - { - "accuracy": 0.953921568627451, - "f1": 0.9538204962747494, - "f1_weighted": 0.9536083701514289 - }, - { - "accuracy": 0.9637254901960784, - "f1": 0.962793648645596, - "f1_weighted": 0.9627911409488475 - }, - { - "accuracy": 0.9647058823529412, - "f1": 0.9647471168145315, - "f1_weighted": 0.9645369313767403 - } - ] - } - ] - }, - "task_name": "OxfordFlowersClassification" -} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/RenderedSST2.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/RenderedSST2.json deleted file mode 100644 index 4987452960..0000000000 --- a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/RenderedSST2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", - "evaluation_time": 13.0860013961792, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "accuracy": 0.5744096650192202, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5744096650192202 - } - ] - }, - "task_name": "RenderedSST2" -} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/STS17MultilingualVisualSTS.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/STS17MultilingualVisualSTS.json deleted file mode 100644 index c6710d98f7..0000000000 --- a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/STS17MultilingualVisualSTS.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", - "evaluation_time": 78.60265827178955, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.09597727663103678, - "cosine_spearman": 0.1770016412213255, - "euclidean_pearson": 0.15275214888889865, - "euclidean_spearman": 0.17834721249044283, - "hf_subset": "ko-ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.1770016412213255, - "manhattan_pearson": 0.15188457950850526, - "manhattan_spearman": 0.17694304714523107, - "pearson": 0.09597727663103678, - "spearman": 0.1770016412213255 - }, - { - "cosine_pearson": 0.0922905025352107, - "cosine_spearman": 0.17678403329083517, - "euclidean_pearson": 0.19665930504248638, - "euclidean_spearman": 0.20254389410280788, - "hf_subset": "ar-ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.17678403329083517, - "manhattan_pearson": 0.20372253572199442, - "manhattan_spearman": 0.2072891316208028, - "pearson": 0.0922905025352107, - "spearman": 0.17678403329083517 - }, - { - "cosine_pearson": 0.21248211257701216, - "cosine_spearman": 0.21536141561958996, - "euclidean_pearson": 0.21166158764988635, - "euclidean_spearman": 0.2300915821447306, - "hf_subset": "en-ar", - "languages": [ - "eng-Latn", - "ara-Arab" - ], - "main_score": 0.21536141561958996, - "manhattan_pearson": 0.19881626986599565, - "manhattan_spearman": 0.21889019768380685, - "pearson": 0.21248211257701216, - "spearman": 0.21536141561958996 - }, - { - "cosine_pearson": 0.1999261135570124, - "cosine_spearman": 0.21224263369270732, - "euclidean_pearson": 0.17992560914365363, - "euclidean_spearman": 0.17445760591179463, - "hf_subset": "en-de", - "languages": [ - "eng-Latn", - "deu-Latn" - ], - "main_score": 0.21224263369270732, - "manhattan_pearson": 0.19511854135088197, - "manhattan_spearman": 0.19271029174298845, - "pearson": 0.1999261135570124, - "spearman": 0.21224263369270732 - }, - { - "cosine_pearson": 0.760796960762554, - "cosine_spearman": 0.7757114712740603, - "euclidean_pearson": 0.7805797751436612, - "euclidean_spearman": 0.7855424130677757, - "hf_subset": "en-en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7757114712740603, - "manhattan_pearson": 0.7798776498566931, - "manhattan_spearman": 0.785226823499344, - "pearson": 0.760796960762554, - "spearman": 0.7757114712740603 - }, - { - "cosine_pearson": 0.1983656963882651, - "cosine_spearman": 0.19407395142116296, - "euclidean_pearson": 0.23667518822462527, - "euclidean_spearman": 0.2165070550530714, - "hf_subset": "en-tr", - "languages": [ - "eng-Latn", - "tur-Latn" - ], - "main_score": 0.19407395142116296, - "manhattan_pearson": 0.23941732565618717, - "manhattan_spearman": 0.22726074763183676, - "pearson": 0.1983656963882651, - "spearman": 0.19407395142116296 - }, - { - "cosine_pearson": 0.26668276308314165, - "cosine_spearman": 0.28381663186603195, - "euclidean_pearson": 0.259253083456217, - "euclidean_spearman": 0.263978807455943, - "hf_subset": "es-en", - "languages": [ - "spa-Latn", - "eng-Latn" - ], - "main_score": 0.28381663186603195, - "manhattan_pearson": 0.23501156058662798, - "manhattan_spearman": 0.24264285305871922, - "pearson": 0.26668276308314165, - "spearman": 0.28381663186603195 - }, - { - "cosine_pearson": 0.4476771003066796, - "cosine_spearman": 0.5430314210401244, - "euclidean_pearson": 0.5767962443120037, - "euclidean_spearman": 0.5685268143569289, - "hf_subset": "es-es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.5430314210401244, - "manhattan_pearson": 0.5881074786408972, - "manhattan_spearman": 0.5770066935318731, - "pearson": 0.4476771003066796, - "spearman": 0.5430314210401244 - }, - { - "cosine_pearson": 0.3357957852619192, - "cosine_spearman": 0.3419618626112132, - "euclidean_pearson": 0.33039599951923004, - "euclidean_spearman": 0.3368547700336451, - "hf_subset": "fr-en", - "languages": [ - "fra-Latn", - "eng-Latn" - ], - "main_score": 0.3419618626112132, - "manhattan_pearson": 0.34797372878650235, - "manhattan_spearman": 0.35782629267058474, - "pearson": 0.3357957852619192, - "spearman": 0.3419618626112132 - }, - { - "cosine_pearson": 0.2919124060662022, - "cosine_spearman": 0.30848592016783905, - "euclidean_pearson": 0.27358870246148104, - "euclidean_spearman": 0.28561893813755396, - "hf_subset": "it-en", - "languages": [ - "ita-Latn", - "eng-Latn" - ], - "main_score": 0.30848592016783905, - "manhattan_pearson": 0.28058310574571027, - "manhattan_spearman": 0.29789118242713974, - "pearson": 0.2919124060662022, - "spearman": 0.30848592016783905 - }, - { - "cosine_pearson": 0.2095390948752094, - "cosine_spearman": 0.1954844816520889, - "euclidean_pearson": 0.18902206597452498, - "euclidean_spearman": 0.17608168134008192, - "hf_subset": "nl-en", - "languages": [ - "nld-Latn", - "eng-Latn" - ], - "main_score": 0.1954844816520889, - "manhattan_pearson": 0.2084658256986222, - "manhattan_spearman": 0.20003458362715085, - "pearson": 0.2095390948752094, - "spearman": 0.1954844816520889 - } - ] - }, - "task_name": "STS17MultilingualVisualSTS" -} \ No newline at end of file diff --git a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/model_meta.json b/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/model_meta.json deleted file mode 100644 index 564d2686c3..0000000000 --- a/results-mieb/kakaobrain__align-base/e96a37facc7b1f59090ece82293226b817afd6ba/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "kakaobrain/align-base", "revision": "e96a37facc7b1f59090ece82293226b817afd6ba", "release_date": "2023-02-24", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "ALIGNModelWrapper"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json b/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json deleted file mode 100644 index 4257296d6f..0000000000 --- a/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 39.63397932052612, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "accuracy": 0.5000807754442649, - "f1": 0.6763180027107614, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "lrap": 0.6410416666666808, - "main_score": 0.5000807754442649, - "scores_per_experiment": [ - { - "accuracy": 0.5038368336025848, - "f1": 0.6876380960919419, - "lrap": 0.6470522572249287 - }, - { - "accuracy": 0.5232229402261712, - "f1": 0.6850605274296695, - "lrap": 0.6585363938251828 - }, - { - "accuracy": 0.4888933764135703, - "f1": 0.6592473172415618, - "lrap": 0.6350399950637378 - }, - { - "accuracy": 0.502827140549273, - "f1": 0.6767015784006876, - "lrap": 0.6400778024591767 - }, - { - "accuracy": 0.48162358642972536, - "f1": 0.6729424943899456, - "lrap": 0.6245018847603783 - } - ] - } - ] - }, - "task_name": "VOC2007" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json b/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json deleted file mode 100644 index 5b161eaaff..0000000000 --- a/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", "revision": "f0e2ffa09cbadab3db6a261ec1ec56407ce42912", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/MNISTZeroShot.json b/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/MNISTZeroShot.json deleted file mode 100644 index a8e2f70693..0000000000 --- a/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/MNISTZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 24.08267593383789, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.6381, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6381 - } - ] - }, - "task_name": "MNISTZeroShot" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/model_meta.json b/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/model_meta.json deleted file mode 100644 index 4781cc3a98..0000000000 --- a/results-mieb/laion__CLIP-ViT-B-32-laion2B-s34B-b79K/08f73555f1b2fb7c82058aebbd492887a94968ef/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", "revision": "08f73555f1b2fb7c82058aebbd492887a94968ef", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/MNIST.json b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/MNIST.json deleted file mode 100644 index e6e6de11f7..0000000000 --- a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 184.1278896331787, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.9621999999999999, - "f1": 0.9622432335494757, - "f1_weighted": 0.962254470082415, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9621999999999999, - "scores_per_experiment": [ - { - "accuracy": 0.9623, - "f1": 0.9622679487261954, - "f1_weighted": 0.9624186429667654 - }, - { - "accuracy": 0.956, - "f1": 0.955938319135998, - "f1_weighted": 0.9559668684856993 - }, - { - "accuracy": 0.9645, - "f1": 0.9644946109310863, - "f1_weighted": 0.9645725164278387 - }, - { - "accuracy": 0.9606, - "f1": 0.9608141458212058, - "f1_weighted": 0.9606963363543709 - }, - { - "accuracy": 0.9676, - "f1": 0.9677011431328928, - "f1_weighted": 0.9676179861774005 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json deleted file mode 100644 index b0d4cc3355..0000000000 --- a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 106.33361673355103, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "accuracy": 0.5382875605815831, - "f1": 0.7099573953664177, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "lrap": 0.6770555667743827, - "main_score": 0.5382875605815831, - "scores_per_experiment": [ - { - "accuracy": 0.5484652665589661, - "f1": 0.7264512368883388, - "lrap": 0.6877928109854764 - }, - { - "accuracy": 0.5581583198707593, - "f1": 0.7220847968939508, - "lrap": 0.6972356847962836 - }, - { - "accuracy": 0.5282714054927302, - "f1": 0.6913016268300175, - "lrap": 0.6676300821217165 - }, - { - "accuracy": 0.5401857835218093, - "f1": 0.7000594523316099, - "lrap": 0.6753413323460928 - }, - { - "accuracy": 0.5163570274636511, - "f1": 0.7098898638881714, - "lrap": 0.6572779236223446 - } - ] - } - ] - }, - "task_name": "VOC2007" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json deleted file mode 100644 index 4ba9199948..0000000000 --- a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "revision": "de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/BLINKIT2IRetrieval.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/BLINKIT2IRetrieval.json deleted file mode 100644 index dd843816fd..0000000000 --- a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/BLINKIT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26", - "evaluation_time": 28.74270534515381, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.3607, - "cv_recall_at_10": 0.63184, - "cv_recall_at_100": 0.91045, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.69652, - "cv_recall_at_3": 0.54975, - "cv_recall_at_5": 0.58458, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.50288, - "map_at_1": 0.3607, - "map_at_10": 0.46095, - "map_at_100": 0.47, - "map_at_1000": 0.47054, - "map_at_20": 0.46542, - "map_at_3": 0.44693, - "map_at_5": 0.45464, - "mrr_at_1": 0.36069651741293535, - "mrr_at_10": 0.4614921424622917, - "mrr_at_100": 0.4705570652048439, - "mrr_at_1000": 0.4710736141683453, - "mrr_at_20": 0.465952194964833, - "mrr_at_3": 0.447346600331675, - "mrr_at_5": 0.4551824212271974, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.3251766664892245, - "nauc_cv_recall_at_100_max": 0.4996971324986197, - "nauc_cv_recall_at_100_std": -0.10981886119431823, - "nauc_cv_recall_at_10_diff1": 0.4150802935478081, - "nauc_cv_recall_at_10_max": 0.44521676291214113, - "nauc_cv_recall_at_10_std": -0.4044050580553833, - "nauc_cv_recall_at_1_diff1": 0.5090856549067897, - "nauc_cv_recall_at_1_max": 0.3561884419781478, - "nauc_cv_recall_at_1_std": -0.3108474258223719, - "nauc_cv_recall_at_20_diff1": 0.3928776128813473, - "nauc_cv_recall_at_20_max": 0.40754722638468, - "nauc_cv_recall_at_20_std": -0.4226831259380377, - "nauc_cv_recall_at_3_diff1": 0.41751507599566495, - "nauc_cv_recall_at_3_max": 0.5119599405595536, - "nauc_cv_recall_at_3_std": -0.3880458369320522, - "nauc_cv_recall_at_5_diff1": 0.41110507164831417, - "nauc_cv_recall_at_5_max": 0.4892953426380338, - "nauc_cv_recall_at_5_std": -0.3909994678840402, - "nauc_map_at_1000_diff1": 0.46826505032819066, - "nauc_map_at_1000_max": 0.40797029811237867, - "nauc_map_at_1000_std": -0.3487770618903377, - "nauc_map_at_100_diff1": 0.4680852808240436, - "nauc_map_at_100_max": 0.4079486164295179, - "nauc_map_at_100_std": -0.34864155304932437, - "nauc_map_at_10_diff1": 0.46821531795636995, - "nauc_map_at_10_max": 0.4105873108851904, - "nauc_map_at_10_std": -0.35048690648082625, - "nauc_map_at_1_diff1": 0.5090856549067897, - "nauc_map_at_1_max": 0.3561884419781478, - "nauc_map_at_1_std": -0.3108474258223719, - "nauc_map_at_20_diff1": 0.4672516873903748, - "nauc_map_at_20_max": 0.4078473889033209, - "nauc_map_at_20_std": -0.3510430019216793, - "nauc_map_at_3_diff1": 0.46792266407052935, - "nauc_map_at_3_max": 0.4217139881712332, - "nauc_map_at_3_std": -0.34757820498553116, - "nauc_map_at_5_diff1": 0.46735798854760324, - "nauc_map_at_5_max": 0.4165657756912836, - "nauc_map_at_5_std": -0.3486378662930895, - "nauc_mrr_at_1000_diff1": 0.46804229195204133, - "nauc_mrr_at_1000_max": 0.4071917446826936, - "nauc_mrr_at_1000_std": -0.3476138289564308, - "nauc_mrr_at_100_diff1": 0.46790977187423044, - "nauc_mrr_at_100_max": 0.4072143236882314, - "nauc_mrr_at_100_std": -0.3475446027341017, - "nauc_mrr_at_10_diff1": 0.46800043394283297, - "nauc_mrr_at_10_max": 0.40981114524918816, - "nauc_mrr_at_10_std": -0.34936642009737867, - "nauc_mrr_at_1_diff1": 0.5090856549067897, - "nauc_mrr_at_1_max": 0.3561884419781478, - "nauc_mrr_at_1_std": -0.3108474258223719, - "nauc_mrr_at_20_diff1": 0.4670263761262366, - "nauc_mrr_at_20_max": 0.407078956671809, - "nauc_mrr_at_20_std": -0.34989409756068307, - "nauc_mrr_at_3_diff1": 0.46769515854499266, - "nauc_mrr_at_3_max": 0.4210312389892628, - "nauc_mrr_at_3_std": -0.3470554885966742, - "nauc_mrr_at_5_diff1": 0.4671445165231216, - "nauc_mrr_at_5_max": 0.41579979574754844, - "nauc_mrr_at_5_std": -0.3475262115394731, - "nauc_ndcg_at_1000_diff1": 0.46044333308523944, - "nauc_ndcg_at_1000_max": 0.41294647548488717, - "nauc_ndcg_at_1000_std": -0.35135326672746664, - "nauc_ndcg_at_100_diff1": 0.45473212178018996, - "nauc_ndcg_at_100_max": 0.4151222010165269, - "nauc_ndcg_at_100_std": -0.34213432270326677, - "nauc_ndcg_at_10_diff1": 0.45571130700534823, - "nauc_ndcg_at_10_max": 0.42039077138937964, - "nauc_ndcg_at_10_std": -0.3630517807405077, - "nauc_ndcg_at_1_diff1": 0.5090856549067897, - "nauc_ndcg_at_1_max": 0.3561884419781478, - "nauc_ndcg_at_1_std": -0.3108474258223719, - "nauc_ndcg_at_20_diff1": 0.4520756416944454, - "nauc_ndcg_at_20_max": 0.4113476779046133, - "nauc_ndcg_at_20_std": -0.3655776710972757, - "nauc_ndcg_at_3_diff1": 0.455287916218925, - "nauc_ndcg_at_3_max": 0.4441335745158963, - "nauc_ndcg_at_3_std": -0.3578120161280115, - "nauc_ndcg_at_5_diff1": 0.45370549230640783, - "nauc_ndcg_at_5_max": 0.4346379074583971, - "nauc_ndcg_at_5_std": -0.3591858287964079, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.31176332692683584, - "nauc_precision_at_100_max": 0.4815670235846271, - "nauc_precision_at_100_std": -0.09187319856981026, - "nauc_precision_at_10_diff1": 0.415080293547808, - "nauc_precision_at_10_max": 0.4452167629121412, - "nauc_precision_at_10_std": -0.4044050580553832, - "nauc_precision_at_1_diff1": 0.5090856549067897, - "nauc_precision_at_1_max": 0.3561884419781478, - "nauc_precision_at_1_std": -0.3108474258223719, - "nauc_precision_at_20_diff1": 0.39287761288134654, - "nauc_precision_at_20_max": 0.40754722638467994, - "nauc_precision_at_20_std": -0.4226831259380381, - "nauc_precision_at_3_diff1": 0.41751507599566523, - "nauc_precision_at_3_max": 0.5119599405595534, - "nauc_precision_at_3_std": -0.3880458369320523, - "nauc_precision_at_5_diff1": 0.41110507164831417, - "nauc_precision_at_5_max": 0.4892953426380342, - "nauc_precision_at_5_std": -0.3909994678840406, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.3117633269268345, - "nauc_recall_at_100_max": 0.48156702358462466, - "nauc_recall_at_100_std": -0.0918731985698134, - "nauc_recall_at_10_diff1": 0.4150802935478081, - "nauc_recall_at_10_max": 0.44521676291214113, - "nauc_recall_at_10_std": -0.4044050580553833, - "nauc_recall_at_1_diff1": 0.5090856549067897, - "nauc_recall_at_1_max": 0.3561884419781478, - "nauc_recall_at_1_std": -0.3108474258223719, - "nauc_recall_at_20_diff1": 0.3928776128813473, - "nauc_recall_at_20_max": 0.40754722638468, - "nauc_recall_at_20_std": -0.4226831259380377, - "nauc_recall_at_3_diff1": 0.41751507599566495, - "nauc_recall_at_3_max": 0.5119599405595536, - "nauc_recall_at_3_std": -0.3880458369320522, - "nauc_recall_at_5_diff1": 0.41110507164831417, - "nauc_recall_at_5_max": 0.4892953426380338, - "nauc_recall_at_5_std": -0.3909994678840402, - "ndcg_at_1": 0.3607, - "ndcg_at_10": 0.50288, - "ndcg_at_100": 0.55671, - "ndcg_at_1000": 0.56895, - "ndcg_at_20": 0.51921, - "ndcg_at_3": 0.47346, - "ndcg_at_5": 0.48759, - "precision_at_1": 0.3607, - "precision_at_10": 0.06318, - "precision_at_100": 0.00908, - "precision_at_1000": 0.001, - "precision_at_20": 0.03483, - "precision_at_3": 0.18325, - "precision_at_5": 0.11692, - "recall_at_1": 0.3607, - "recall_at_10": 0.63184, - "recall_at_100": 0.90796, - "recall_at_1000": 1.0, - "recall_at_20": 0.69652, - "recall_at_3": 0.54975, - "recall_at_5": 0.58458 - } - ] - }, - "task_name": "BLINKIT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNIST.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNIST.json deleted file mode 100644 index 0713724334..0000000000 --- a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNIST.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 84.51716136932373, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.9587, - "f1": 0.9584918381835605, - "f1_weighted": 0.9587174442512109, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9587, - "scores_per_experiment": [ - { - "accuracy": 0.9587, - "f1": 0.9584918381835605, - "f1_weighted": 0.9587174442512109 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNISTZeroShot.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNISTZeroShot.json deleted file mode 100644 index 154f626d2f..0000000000 --- a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/MNISTZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 79.63758492469788, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.8144, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8144 - } - ] - }, - "task_name": "MNISTZeroShot" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/STS14.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/STS14.json deleted file mode 100644 index 20751681aa..0000000000 --- a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/STS14.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", - "evaluation_time": 5.388969898223877, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "cosine_pearson": 0.6794914647081003, - "cosine_spearman": 0.6327310416487439, - "euclidean_pearson": 0.6754886733081702, - "euclidean_spearman": 0.6473065461091825, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6327310416487439, - "manhattan_pearson": 0.6833982691624099, - "manhattan_spearman": 0.6526646970883643, - "pearson": 0.6794914647081003, - "spearman": 0.6327310416487439 - } - ] - }, - "task_name": "STS14" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json b/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json deleted file mode 100644 index 1825d8b79c..0000000000 --- a/results-mieb/laion__CLIP-ViT-L-14-DataComp.XL-s13B-b90K/84c9828e63dc9a9351d1fe637c346d4c1c4db341/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "revision": "84c9828e63dc9a9351d1fe637c346d4c1c4db341", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/MNISTZeroShot.json b/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/MNISTZeroShot.json deleted file mode 100644 index d166f91c44..0000000000 --- a/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/MNISTZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 73.2168505191803, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.6414, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6414 - } - ] - }, - "task_name": "MNISTZeroShot" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/model_meta.json b/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/model_meta.json deleted file mode 100644 index ff6252d975..0000000000 --- a/results-mieb/laion__CLIP-ViT-L-14-laion2B-s32B-b82K/1627032197142fbe2a7cfec626f4ced3ae60d07a/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "laion/CLIP-ViT-L-14-laion2B-s32B-b82K", "revision": "1627032197142fbe2a7cfec626f4ced3ae60d07a", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/MNIST.json b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/MNIST.json deleted file mode 100644 index c0c5d016a1..0000000000 --- a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 259.59732961654663, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.96534, - "f1": 0.9652996312920756, - "f1_weighted": 0.9654146401974364, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.96534, - "scores_per_experiment": [ - { - "accuracy": 0.9656, - "f1": 0.9655583025661676, - "f1_weighted": 0.9656417812003859 - }, - { - "accuracy": 0.963, - "f1": 0.9628025390918398, - "f1_weighted": 0.96301114967701 - }, - { - "accuracy": 0.9676, - "f1": 0.9674695816973904, - "f1_weighted": 0.9676411686849484 - }, - { - "accuracy": 0.9611, - "f1": 0.9613040896869535, - "f1_weighted": 0.9613453097505182 - }, - { - "accuracy": 0.9694, - "f1": 0.9693636434180266, - "f1_weighted": 0.9694337916743194 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json deleted file mode 100644 index 0cc887a6ba..0000000000 --- a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 139.95830011367798, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "accuracy": 0.5426898222940226, - "f1": 0.7116762127648723, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "lrap": 0.6792292676359876, - "main_score": 0.5426898222940226, - "scores_per_experiment": [ - { - "accuracy": 0.5609854604200323, - "f1": 0.7310080393154441, - "lrap": 0.6944724914737191 - }, - { - "accuracy": 0.559168012924071, - "f1": 0.7125996315013976, - "lrap": 0.6941942649434737 - }, - { - "accuracy": 0.5220113085621971, - "f1": 0.6999211686407317, - "lrap": 0.6659638081134593 - }, - { - "accuracy": 0.5387722132471728, - "f1": 0.702612150503481, - "lrap": 0.6734136600251451 - }, - { - "accuracy": 0.5325121163166397, - "f1": 0.7122400738633065, - "lrap": 0.6681021136241404 - } - ] - } - ] - }, - "task_name": "VOC2007" -} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json deleted file mode 100644 index 852d6e492a..0000000000 --- a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", "revision": "15efd0f6ac0c40c0f9da7becca03c974d7012604", "release_date": "2023-03-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/CIRRIT2IRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/CIRRIT2IRetrieval.json deleted file mode 100644 index 8bab9a555f..0000000000 --- a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/CIRRIT2IRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", - "evaluation_time": 133.3731541633606, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.09422, - "map_at_1": 0.00408, - "map_at_10": 0.06275, - "map_at_100": 0.07352, - "map_at_1000": 0.07488, - "map_at_20": 0.06782, - "map_at_3": 0.04482, - "map_at_5": 0.05503, - "mrr_at_1": 0.008393285371702638, - "mrr_at_10": 0.06311579307982176, - "mrr_at_100": 0.0738817410625923, - "mrr_at_1000": 0.0752359451782398, - "mrr_at_20": 0.06819202642832276, - "mrr_at_3": 0.04532374100719432, - "mrr_at_5": 0.05547961630695447, - "nauc_map_at_1000_diff1": -0.10338892563096655, - "nauc_map_at_1000_max": 0.177335800834316, - "nauc_map_at_1000_std": 0.04510016072563672, - "nauc_map_at_100_diff1": -0.10237436191846207, - "nauc_map_at_100_max": 0.1764756777393816, - "nauc_map_at_100_std": 0.04276612836249515, - "nauc_map_at_10_diff1": -0.0966658193961709, - "nauc_map_at_10_max": 0.16285956921247727, - "nauc_map_at_10_std": 0.026961252449232854, - "nauc_map_at_1_diff1": -0.5280465285121293, - "nauc_map_at_1_max": 0.08621506021244016, - "nauc_map_at_1_std": 0.18339146766836595, - "nauc_map_at_20_diff1": -0.09819954169398601, - "nauc_map_at_20_max": 0.17274426550167696, - "nauc_map_at_20_std": 0.03509542919549203, - "nauc_map_at_3_diff1": -0.1096647340125396, - "nauc_map_at_3_max": 0.14731937096492292, - "nauc_map_at_3_std": 0.02285630960394209, - "nauc_map_at_5_diff1": -0.10387404872895724, - "nauc_map_at_5_max": 0.15482607000107218, - "nauc_map_at_5_std": 0.024648861558939302, - "nauc_mrr_at_1000_diff1": -0.10575125211055578, - "nauc_mrr_at_1000_max": 0.1758407777736024, - "nauc_mrr_at_1000_std": 0.04794265702233136, - "nauc_mrr_at_100_diff1": -0.10473322024101767, - "nauc_mrr_at_100_max": 0.17499228892267552, - "nauc_mrr_at_100_std": 0.045592396596819566, - "nauc_mrr_at_10_diff1": -0.09922185003516211, - "nauc_mrr_at_10_max": 0.16145148359497702, - "nauc_mrr_at_10_std": 0.02999573674643215, - "nauc_mrr_at_1_diff1": -0.5227107209935222, - "nauc_mrr_at_1_max": 0.06995285369312734, - "nauc_mrr_at_1_std": 0.17595155046188055, - "nauc_mrr_at_20_diff1": -0.10068826746108109, - "nauc_mrr_at_20_max": 0.171302451214174, - "nauc_mrr_at_20_std": 0.03802637704632661, - "nauc_mrr_at_3_diff1": -0.11370621646983334, - "nauc_mrr_at_3_max": 0.1478294385458323, - "nauc_mrr_at_3_std": 0.028261047534815342, - "nauc_mrr_at_5_diff1": -0.1069655081332803, - "nauc_mrr_at_5_max": 0.15385012052759187, - "nauc_mrr_at_5_std": 0.028281108584287444, - "nauc_ndcg_at_1000_diff1": -0.12001070980393722, - "nauc_ndcg_at_1000_max": 0.22010392536361548, - "nauc_ndcg_at_1000_std": 0.11525705750127722, - "nauc_ndcg_at_100_diff1": -0.10703657138256953, - "nauc_ndcg_at_100_max": 0.21257636133707714, - "nauc_ndcg_at_100_std": 0.08674070424465001, - "nauc_ndcg_at_10_diff1": -0.08128772448530021, - "nauc_ndcg_at_10_max": 0.17244036804629503, - "nauc_ndcg_at_10_std": 0.02819345677203796, - "nauc_ndcg_at_1_diff1": -0.5227107209935222, - "nauc_ndcg_at_1_max": 0.09562650925677493, - "nauc_ndcg_at_1_std": 0.16468243320939854, - "nauc_ndcg_at_20_diff1": -0.086056370146497, - "nauc_ndcg_at_20_max": 0.195869806760938, - "nauc_ndcg_at_20_std": 0.04757991969301295, - "nauc_ndcg_at_3_diff1": -0.09491160269887662, - "nauc_ndcg_at_3_max": 0.15135662699498084, - "nauc_ndcg_at_3_std": 0.019722501663945312, - "nauc_ndcg_at_5_diff1": -0.09008866995016263, - "nauc_ndcg_at_5_max": 0.16143679483180548, - "nauc_ndcg_at_5_std": 0.022764770078524912, - "nauc_precision_at_1000_diff1": -0.24045166016587438, - "nauc_precision_at_1000_max": 0.3195034508543935, - "nauc_precision_at_1000_std": 0.34953942622964457, - "nauc_precision_at_100_diff1": -0.14676866267060482, - "nauc_precision_at_100_max": 0.2595133723112115, - "nauc_precision_at_100_std": 0.16414102793250826, - "nauc_precision_at_10_diff1": -0.08958089915422018, - "nauc_precision_at_10_max": 0.18075236692646582, - "nauc_precision_at_10_std": 0.036349247845646616, - "nauc_precision_at_1_diff1": -0.5227107209935222, - "nauc_precision_at_1_max": 0.09562650925677493, - "nauc_precision_at_1_std": 0.16468243320939854, - "nauc_precision_at_20_diff1": -0.09690232093945902, - "nauc_precision_at_20_max": 0.2216329441415861, - "nauc_precision_at_20_std": 0.06967652721476236, - "nauc_precision_at_3_diff1": -0.11403461479629584, - "nauc_precision_at_3_max": 0.1515256188109501, - "nauc_precision_at_3_std": 0.02583697143100736, - "nauc_precision_at_5_diff1": -0.10407436535190769, - "nauc_precision_at_5_max": 0.16618197540243515, - "nauc_precision_at_5_std": 0.027731434908842483, - "nauc_recall_at_1000_diff1": -0.21373976975815448, - "nauc_recall_at_1000_max": 0.340364079088584, - "nauc_recall_at_1000_std": 0.3758106778790492, - "nauc_recall_at_100_diff1": -0.12841173327380748, - "nauc_recall_at_100_max": 0.26630249039089776, - "nauc_recall_at_100_std": 0.16610202623791748, - "nauc_recall_at_10_diff1": -0.06387695134566657, - "nauc_recall_at_10_max": 0.1841498177665857, - "nauc_recall_at_10_std": 0.03059539077954125, - "nauc_recall_at_1_diff1": -0.5280465285121293, - "nauc_recall_at_1_max": 0.08621506021244016, - "nauc_recall_at_1_std": 0.18339146766836595, - "nauc_recall_at_20_diff1": -0.07508476020191192, - "nauc_recall_at_20_max": 0.2266673632206196, - "nauc_recall_at_20_std": 0.06644902770116523, - "nauc_recall_at_3_diff1": -0.07292873956832151, - "nauc_recall_at_3_max": 0.15682509079744078, - "nauc_recall_at_3_std": 0.014168688229387593, - "nauc_recall_at_5_diff1": -0.07206865420575172, - "nauc_recall_at_5_max": 0.1701900002314879, - "nauc_recall_at_5_std": 0.019842639580852355, - "ndcg_at_1": 0.00839, - "ndcg_at_10": 0.09422, - "ndcg_at_100": 0.15655, - "ndcg_at_1000": 0.19407, - "ndcg_at_20": 0.11286, - "ndcg_at_3": 0.0573, - "ndcg_at_5": 0.07574, - "precision_at_1": 0.00839, - "precision_at_10": 0.02046, - "precision_at_100": 0.00519, - "precision_at_1000": 0.00082, - "precision_at_20": 0.01393, - "precision_at_3": 0.03421, - "precision_at_5": 0.02954, - "recall_at_1": 0.00408, - "recall_at_10": 0.1946, - "recall_at_100": 0.50923, - "recall_at_1000": 0.80791, - "recall_at_20": 0.26871, - "recall_at_3": 0.09305, - "recall_at_5": 0.13789 - } - ] - }, - "task_name": "CIRRIT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOI2TRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOI2TRetrieval.json deleted file mode 100644 index c4333ca68f..0000000000 --- a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOI2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", - "evaluation_time": 42.80281734466553, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.37288, - "map_at_1": 0.08071, - "map_at_10": 0.26786, - "map_at_100": 0.31168, - "map_at_1000": 0.31587, - "map_at_20": 0.29106, - "map_at_3": 0.17731, - "map_at_5": 0.2267, - "mrr_at_1": 0.4034, - "mrr_at_10": 0.5154798412698407, - "mrr_at_100": 0.5245965200676513, - "mrr_at_1000": 0.5247295619699504, - "mrr_at_20": 0.521172379941179, - "mrr_at_3": 0.482566666666666, - "mrr_at_5": 0.5011866666666658, - "nauc_map_at_1000_diff1": 0.18316596672602442, - "nauc_map_at_1000_max": 0.34801279014511405, - "nauc_map_at_1000_std": -0.18750172942216528, - "nauc_map_at_100_diff1": 0.18245509021092263, - "nauc_map_at_100_max": 0.3484150820371727, - "nauc_map_at_100_std": -0.18865972347643556, - "nauc_map_at_10_diff1": 0.1851275487650171, - "nauc_map_at_10_max": 0.3370878939903754, - "nauc_map_at_10_std": -0.19389877734594482, - "nauc_map_at_1_diff1": 0.2724994569940933, - "nauc_map_at_1_max": 0.2759939487280074, - "nauc_map_at_1_std": -0.18769761438653643, - "nauc_map_at_20_diff1": 0.18551346146324993, - "nauc_map_at_20_max": 0.34459316188937006, - "nauc_map_at_20_std": -0.19182380187451328, - "nauc_map_at_3_diff1": 0.21373477811733987, - "nauc_map_at_3_max": 0.3060468883646992, - "nauc_map_at_3_std": -0.19267528106941764, - "nauc_map_at_5_diff1": 0.1895485112324686, - "nauc_map_at_5_max": 0.32650067482878825, - "nauc_map_at_5_std": -0.19076772161486724, - "nauc_mrr_at_1000_diff1": 0.25557692572293067, - "nauc_mrr_at_1000_max": 0.2864330733081474, - "nauc_mrr_at_1000_std": -0.20284682486076736, - "nauc_mrr_at_100_diff1": 0.2555410395024608, - "nauc_mrr_at_100_max": 0.2864902886387769, - "nauc_mrr_at_100_std": -0.20281383228526584, - "nauc_mrr_at_10_diff1": 0.2552916397209132, - "nauc_mrr_at_10_max": 0.2866177928313412, - "nauc_mrr_at_10_std": -0.20398130147274418, - "nauc_mrr_at_1_diff1": 0.27278878441171606, - "nauc_mrr_at_1_max": 0.2760128871314018, - "nauc_mrr_at_1_std": -0.1875598816248055, - "nauc_mrr_at_20_diff1": 0.2552527730209484, - "nauc_mrr_at_20_max": 0.2869938940286829, - "nauc_mrr_at_20_std": -0.20347090553389235, - "nauc_mrr_at_3_diff1": 0.2551361915671908, - "nauc_mrr_at_3_max": 0.2853826392458793, - "nauc_mrr_at_3_std": -0.2014489869554216, - "nauc_mrr_at_5_diff1": 0.25424238709065144, - "nauc_mrr_at_5_max": 0.2834915887900367, - "nauc_mrr_at_5_std": -0.2041266754700247, - "nauc_ndcg_at_1000_diff1": 0.19931449927197542, - "nauc_ndcg_at_1000_max": 0.34824914996907946, - "nauc_ndcg_at_1000_std": -0.17612769107106158, - "nauc_ndcg_at_100_diff1": 0.18782042717250694, - "nauc_ndcg_at_100_max": 0.3567017765386365, - "nauc_ndcg_at_100_std": -0.18124611786715342, - "nauc_ndcg_at_10_diff1": 0.19614278113213443, - "nauc_ndcg_at_10_max": 0.3345885498609445, - "nauc_ndcg_at_10_std": -0.1995834302663533, - "nauc_ndcg_at_1_diff1": 0.27278878441171606, - "nauc_ndcg_at_1_max": 0.2760128871314018, - "nauc_ndcg_at_1_std": -0.1875598816248055, - "nauc_ndcg_at_20_diff1": 0.19575779535055157, - "nauc_ndcg_at_20_max": 0.3461848546945705, - "nauc_ndcg_at_20_std": -0.19512601196073395, - "nauc_ndcg_at_3_diff1": 0.21982547126571084, - "nauc_ndcg_at_3_max": 0.3043154912793067, - "nauc_ndcg_at_3_std": -0.19749127050913148, - "nauc_ndcg_at_5_diff1": 0.19883328350299154, - "nauc_ndcg_at_5_max": 0.32007392291215236, - "nauc_ndcg_at_5_std": -0.19678681748663923, - "nauc_precision_at_1000_diff1": 0.022351052380250144, - "nauc_precision_at_1000_max": 0.4725759260447294, - "nauc_precision_at_1000_std": 0.5797080918881626, - "nauc_precision_at_100_diff1": 0.08393111116724335, - "nauc_precision_at_100_max": 0.3771922887973945, - "nauc_precision_at_100_std": -0.08376254395248357, - "nauc_precision_at_10_diff1": 0.15560876491276518, - "nauc_precision_at_10_max": 0.33714785955779036, - "nauc_precision_at_10_std": -0.1875518639116136, - "nauc_precision_at_1_diff1": 0.27278878441171606, - "nauc_precision_at_1_max": 0.2760128871314018, - "nauc_precision_at_1_std": -0.1875598816248055, - "nauc_precision_at_20_diff1": 0.1477270394650292, - "nauc_precision_at_20_max": 0.35052230390958183, - "nauc_precision_at_20_std": -0.16921558322042235, - "nauc_precision_at_3_diff1": 0.19844496546548113, - "nauc_precision_at_3_max": 0.3091703704739135, - "nauc_precision_at_3_std": -0.19757336334013054, - "nauc_precision_at_5_diff1": 0.1653617822470633, - "nauc_precision_at_5_max": 0.3264198806421896, - "nauc_precision_at_5_std": -0.19150011703731976, - "nauc_recall_at_1000_diff1": 0.01459616156901875, - "nauc_recall_at_1000_max": 0.48367786796485557, - "nauc_recall_at_1000_std": 0.5837504997127877, - "nauc_recall_at_100_diff1": 0.08310515225683762, - "nauc_recall_at_100_max": 0.37760531087114085, - "nauc_recall_at_100_std": -0.08463401361245898, - "nauc_recall_at_10_diff1": 0.15530962605623758, - "nauc_recall_at_10_max": 0.33716708957281405, - "nauc_recall_at_10_std": -0.18785611276459221, - "nauc_recall_at_1_diff1": 0.2724994569940933, - "nauc_recall_at_1_max": 0.2759939487280074, - "nauc_recall_at_1_std": -0.18769761438653643, - "nauc_recall_at_20_diff1": 0.1472978746648726, - "nauc_recall_at_20_max": 0.35044375641725917, - "nauc_recall_at_20_std": -0.16960809708140606, - "nauc_recall_at_3_diff1": 0.1981017909103437, - "nauc_recall_at_3_max": 0.3091501498176776, - "nauc_recall_at_3_std": -0.19776288065176492, - "nauc_recall_at_5_diff1": 0.16510836445941884, - "nauc_recall_at_5_max": 0.32640908545278824, - "nauc_recall_at_5_std": -0.19170241637346824, - "ndcg_at_1": 0.4034, - "ndcg_at_10": 0.37288, - "ndcg_at_100": 0.50144, - "ndcg_at_1000": 0.54804, - "ndcg_at_20": 0.42119, - "ndcg_at_3": 0.35371, - "ndcg_at_5": 0.31191, - "precision_at_1": 0.4034, - "precision_at_10": 0.19746, - "precision_at_100": 0.03826, - "precision_at_1000": 0.0049, - "precision_at_20": 0.12689, - "precision_at_3": 0.33847, - "precision_at_5": 0.28372, - "recall_at_1": 0.08071, - "recall_at_10": 0.39509, - "recall_at_100": 0.76558, - "recall_at_1000": 0.97947, - "recall_at_20": 0.50782, - "recall_at_3": 0.20317, - "recall_at_5": 0.28382 - } - ] - }, - "task_name": "MSCOCOI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOT2IRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOT2IRetrieval.json deleted file mode 100644 index 7e6d34afff..0000000000 --- a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/MSCOCOT2IRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", - "evaluation_time": 66.51581335067749, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.46897, - "map_at_1": 0.305, - "map_at_10": 0.40963, - "map_at_100": 0.42183, - "map_at_1000": 0.42212, - "map_at_20": 0.41684, - "map_at_3": 0.37632, - "map_at_5": 0.39495, - "mrr_at_1": 0.30525212624450804, - "mrr_at_10": 0.4099942929057418, - "mrr_at_100": 0.42214037857811404, - "mrr_at_1000": 0.4224150820003196, - "mrr_at_20": 0.4171927718574053, - "mrr_at_3": 0.3766643825493417, - "mrr_at_5": 0.39532696467679795, - "nauc_map_at_1000_diff1": 0.4478711106054124, - "nauc_map_at_1000_max": 0.28050562897695164, - "nauc_map_at_1000_std": -0.21959015997209919, - "nauc_map_at_100_diff1": 0.4477231775321896, - "nauc_map_at_100_max": 0.2806372368499711, - "nauc_map_at_100_std": -0.21940131954025208, - "nauc_map_at_10_diff1": 0.44612476561916625, - "nauc_map_at_10_max": 0.2799994443519687, - "nauc_map_at_10_std": -0.22187850998432965, - "nauc_map_at_1_diff1": 0.5029747421425568, - "nauc_map_at_1_max": 0.272549283076283, - "nauc_map_at_1_std": -0.2124194075262369, - "nauc_map_at_20_diff1": 0.4465097088138646, - "nauc_map_at_20_max": 0.28045224479942427, - "nauc_map_at_20_std": -0.22094628341471756, - "nauc_map_at_3_diff1": 0.4543183470256243, - "nauc_map_at_3_max": 0.27789569085174415, - "nauc_map_at_3_std": -0.22162450074705334, - "nauc_map_at_5_diff1": 0.44768741119006605, - "nauc_map_at_5_max": 0.27857801963069206, - "nauc_map_at_5_std": -0.22171216858692744, - "nauc_mrr_at_1000_diff1": 0.44762123092250417, - "nauc_mrr_at_1000_max": 0.2804655602576719, - "nauc_mrr_at_1000_std": -0.21917729645099004, - "nauc_mrr_at_100_diff1": 0.4474764210942955, - "nauc_mrr_at_100_max": 0.280601026351656, - "nauc_mrr_at_100_std": -0.2189669410434172, - "nauc_mrr_at_10_diff1": 0.4457989624406091, - "nauc_mrr_at_10_max": 0.2799902368295713, - "nauc_mrr_at_10_std": -0.22134188164578128, - "nauc_mrr_at_1_diff1": 0.5027796660701911, - "nauc_mrr_at_1_max": 0.27275111707746236, - "nauc_mrr_at_1_std": -0.21214467131036538, - "nauc_mrr_at_20_diff1": 0.44622688768256885, - "nauc_mrr_at_20_max": 0.28042386811339354, - "nauc_mrr_at_20_std": -0.22041389186782168, - "nauc_mrr_at_3_diff1": 0.4540565519372315, - "nauc_mrr_at_3_max": 0.27789412741657327, - "nauc_mrr_at_3_std": -0.22124654547810124, - "nauc_mrr_at_5_diff1": 0.4473821512144231, - "nauc_mrr_at_5_max": 0.2785729569663956, - "nauc_mrr_at_5_std": -0.22131319479803052, - "nauc_ndcg_at_1000_diff1": 0.43654475958434186, - "nauc_ndcg_at_1000_max": 0.2848558104719407, - "nauc_ndcg_at_1000_std": -0.21518136929725057, - "nauc_ndcg_at_100_diff1": 0.4327107582163226, - "nauc_ndcg_at_100_max": 0.28859948535481533, - "nauc_ndcg_at_100_std": -0.2081893113971199, - "nauc_ndcg_at_10_diff1": 0.42285553095601514, - "nauc_ndcg_at_10_max": 0.2852677139001271, - "nauc_ndcg_at_10_std": -0.2245997440948349, - "nauc_ndcg_at_1_diff1": 0.5027796660701911, - "nauc_ndcg_at_1_max": 0.27275111707746236, - "nauc_ndcg_at_1_std": -0.21214467131036538, - "nauc_ndcg_at_20_diff1": 0.4235238359251292, - "nauc_ndcg_at_20_max": 0.2872035072298614, - "nauc_ndcg_at_20_std": -0.22100876714130083, - "nauc_ndcg_at_3_diff1": 0.43942781361598426, - "nauc_ndcg_at_3_max": 0.27973046846426364, - "nauc_ndcg_at_3_std": -0.22444184638120696, - "nauc_ndcg_at_5_diff1": 0.427247141083044, - "nauc_ndcg_at_5_max": 0.2811124586942207, - "nauc_ndcg_at_5_std": -0.22465662590672983, - "nauc_precision_at_1000_diff1": -0.12179681349517751, - "nauc_precision_at_1000_max": 0.13023305110548689, - "nauc_precision_at_1000_std": 0.5260350786100252, - "nauc_precision_at_100_diff1": 0.18340202723844473, - "nauc_precision_at_100_max": 0.39983881724392506, - "nauc_precision_at_100_std": 0.221650593088434, - "nauc_precision_at_10_diff1": 0.3346141522540821, - "nauc_precision_at_10_max": 0.3061257203288402, - "nauc_precision_at_10_std": -0.22911572795263271, - "nauc_precision_at_1_diff1": 0.5027796660701911, - "nauc_precision_at_1_max": 0.27275111707746236, - "nauc_precision_at_1_std": -0.21214467131036538, - "nauc_precision_at_20_diff1": 0.3090330640648721, - "nauc_precision_at_20_max": 0.32131718134148296, - "nauc_precision_at_20_std": -0.20716374732844292, - "nauc_precision_at_3_diff1": 0.3960170209609372, - "nauc_precision_at_3_max": 0.285285655711078, - "nauc_precision_at_3_std": -0.23167624055155045, - "nauc_precision_at_5_diff1": 0.3625854653247323, - "nauc_precision_at_5_max": 0.28929006766488813, - "nauc_precision_at_5_std": -0.23215194678446605, - "nauc_recall_at_1000_diff1": 0.22969264464007677, - "nauc_recall_at_1000_max": 0.7235125847169891, - "nauc_recall_at_1000_std": 0.8279246959358108, - "nauc_recall_at_100_diff1": 0.2606980752081389, - "nauc_recall_at_100_max": 0.47215991575399363, - "nauc_recall_at_100_std": 0.16730099691638203, - "nauc_recall_at_10_diff1": 0.33881214716475055, - "nauc_recall_at_10_max": 0.30639423349407746, - "nauc_recall_at_10_std": -0.23428259082232605, - "nauc_recall_at_1_diff1": 0.5029747421425568, - "nauc_recall_at_1_max": 0.272549283076283, - "nauc_recall_at_1_std": -0.2124194075262369, - "nauc_recall_at_20_diff1": 0.31710792090285655, - "nauc_recall_at_20_max": 0.32370157795364835, - "nauc_recall_at_20_std": -0.21662838240185825, - "nauc_recall_at_3_diff1": 0.3971631468785135, - "nauc_recall_at_3_max": 0.28502696676818606, - "nauc_recall_at_3_std": -0.23298949690090923, - "nauc_recall_at_5_diff1": 0.3644953058714283, - "nauc_recall_at_5_max": 0.2889906359951118, - "nauc_recall_at_5_std": -0.23398910701923947, - "ndcg_at_1": 0.30525, - "ndcg_at_10": 0.46897, - "ndcg_at_100": 0.53034, - "ndcg_at_1000": 0.53666, - "ndcg_at_20": 0.49515, - "ndcg_at_3": 0.39993, - "ndcg_at_5": 0.4335, - "precision_at_1": 0.30525, - "precision_at_10": 0.06607, - "precision_at_100": 0.00955, - "precision_at_1000": 0.001, - "precision_at_20": 0.03824, - "precision_at_3": 0.15626, - "precision_at_5": 0.11012, - "recall_at_1": 0.305, - "recall_at_10": 0.65879, - "recall_at_100": 0.94938, - "recall_at_1000": 0.99673, - "recall_at_20": 0.76204, - "recall_at_3": 0.46797, - "recall_at_5": 0.54937 - } - ] - }, - "task_name": "MSCOCOT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/NIGHTSI2IRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/NIGHTSI2IRetrieval.json deleted file mode 100644 index 5ba1e219c8..0000000000 --- a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/NIGHTSI2IRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", - "evaluation_time": 199.96433472633362, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.22818, - "map_at_1": 0.06934, - "map_at_10": 0.16097, - "map_at_100": 0.18386, - "map_at_1000": 0.18414, - "map_at_20": 0.17653, - "map_at_3": 0.11635, - "map_at_5": 0.13781, - "mrr_at_1": 0.06933962264150943, - "mrr_at_10": 0.16097334531296778, - "mrr_at_100": 0.1838599085374125, - "mrr_at_1000": 0.18413700903303717, - "mrr_at_20": 0.17653316610403727, - "mrr_at_3": 0.1163522012578618, - "mrr_at_5": 0.13781446540880488, - "nauc_map_at_1000_diff1": 0.057301690218310136, - "nauc_map_at_1000_max": 0.07188842602959197, - "nauc_map_at_1000_std": -0.08154303633505418, - "nauc_map_at_100_diff1": 0.05698354071014854, - "nauc_map_at_100_max": 0.07244030136556417, - "nauc_map_at_100_std": -0.08128920818096987, - "nauc_map_at_10_diff1": 0.05865679320595122, - "nauc_map_at_10_max": 0.06134631667106162, - "nauc_map_at_10_std": -0.08503834658994171, - "nauc_map_at_1_diff1": 0.10488273656658804, - "nauc_map_at_1_max": 0.0639349917589788, - "nauc_map_at_1_std": -0.07575716092715594, - "nauc_map_at_20_diff1": 0.0574817213936499, - "nauc_map_at_20_max": 0.0694885498529206, - "nauc_map_at_20_std": -0.08453835749586952, - "nauc_map_at_3_diff1": 0.07568477970041008, - "nauc_map_at_3_max": 0.06567675283236628, - "nauc_map_at_3_std": -0.09191303028245797, - "nauc_map_at_5_diff1": 0.06830671022162287, - "nauc_map_at_5_max": 0.05388386482539364, - "nauc_map_at_5_std": -0.0869392787560929, - "nauc_mrr_at_1000_diff1": 0.057301690218310136, - "nauc_mrr_at_1000_max": 0.07188842602959197, - "nauc_mrr_at_1000_std": -0.08154303633505418, - "nauc_mrr_at_100_diff1": 0.05698354071014854, - "nauc_mrr_at_100_max": 0.07244030136556417, - "nauc_mrr_at_100_std": -0.08128920818096987, - "nauc_mrr_at_10_diff1": 0.05865679320595122, - "nauc_mrr_at_10_max": 0.06134631667106162, - "nauc_mrr_at_10_std": -0.08503834658994171, - "nauc_mrr_at_1_diff1": 0.10488273656658804, - "nauc_mrr_at_1_max": 0.0639349917589788, - "nauc_mrr_at_1_std": -0.07575716092715594, - "nauc_mrr_at_20_diff1": 0.0574817213936499, - "nauc_mrr_at_20_max": 0.0694885498529206, - "nauc_mrr_at_20_std": -0.08453835749586952, - "nauc_mrr_at_3_diff1": 0.07568477970041008, - "nauc_mrr_at_3_max": 0.06567675283236628, - "nauc_mrr_at_3_std": -0.09191303028245797, - "nauc_mrr_at_5_diff1": 0.06830671022162287, - "nauc_mrr_at_5_max": 0.05388386482539364, - "nauc_mrr_at_5_std": -0.0869392787560929, - "nauc_ndcg_at_1000_diff1": 0.03275032921060949, - "nauc_ndcg_at_1000_max": 0.09370875865082869, - "nauc_ndcg_at_1000_std": -0.06520945518508174, - "nauc_ndcg_at_100_diff1": 0.022434604110621165, - "nauc_ndcg_at_100_max": 0.1088264312707009, - "nauc_ndcg_at_100_std": -0.05625420269516149, - "nauc_ndcg_at_10_diff1": 0.036984429499972896, - "nauc_ndcg_at_10_max": 0.06396551203788203, - "nauc_ndcg_at_10_std": -0.08202279893586943, - "nauc_ndcg_at_1_diff1": 0.10488273656658804, - "nauc_ndcg_at_1_max": 0.0639349917589788, - "nauc_ndcg_at_1_std": -0.07575716092715594, - "nauc_ndcg_at_20_diff1": 0.029682814087520562, - "nauc_ndcg_at_20_max": 0.09156539071118802, - "nauc_ndcg_at_20_std": -0.07992527984856755, - "nauc_ndcg_at_3_diff1": 0.06755595595602068, - "nauc_ndcg_at_3_max": 0.06624062050199891, - "nauc_ndcg_at_3_std": -0.09524111086264671, - "nauc_ndcg_at_5_diff1": 0.05733520128202785, - "nauc_ndcg_at_5_max": 0.047767515545288526, - "nauc_ndcg_at_5_std": -0.08669006844506776, - "nauc_precision_at_1000_diff1": -0.5127678978568057, - "nauc_precision_at_1000_max": 0.6076477102468908, - "nauc_precision_at_1000_std": 0.49423490326363295, - "nauc_precision_at_100_diff1": -0.32390524343909355, - "nauc_precision_at_100_max": 0.524590549007864, - "nauc_precision_at_100_std": 0.27115080833171834, - "nauc_precision_at_10_diff1": -0.005594372755515605, - "nauc_precision_at_10_max": 0.07134091937034583, - "nauc_precision_at_10_std": -0.07449401017787786, - "nauc_precision_at_1_diff1": 0.10488273656658804, - "nauc_precision_at_1_max": 0.0639349917589788, - "nauc_precision_at_1_std": -0.07575716092715594, - "nauc_precision_at_20_diff1": -0.044341437676347024, - "nauc_precision_at_20_max": 0.16246650001126064, - "nauc_precision_at_20_std": -0.06513958194587852, - "nauc_precision_at_3_diff1": 0.05083674702012877, - "nauc_precision_at_3_max": 0.06741308965704264, - "nauc_precision_at_3_std": -0.10189783181241091, - "nauc_precision_at_5_diff1": 0.03711832073718323, - "nauc_precision_at_5_max": 0.03554341529116533, - "nauc_precision_at_5_std": -0.08532792633472566, - "nauc_recall_at_1000_diff1": -0.5127678978568214, - "nauc_recall_at_1000_max": 0.6076477102468842, - "nauc_recall_at_1000_std": 0.49423490326363095, - "nauc_recall_at_100_diff1": -0.32390524343909616, - "nauc_recall_at_100_max": 0.5245905490078637, - "nauc_recall_at_100_std": 0.27115080833171884, - "nauc_recall_at_10_diff1": -0.005594372755515973, - "nauc_recall_at_10_max": 0.0713409193703457, - "nauc_recall_at_10_std": -0.07449401017787816, - "nauc_recall_at_1_diff1": 0.10488273656658804, - "nauc_recall_at_1_max": 0.0639349917589788, - "nauc_recall_at_1_std": -0.07575716092715594, - "nauc_recall_at_20_diff1": -0.04434143767634782, - "nauc_recall_at_20_max": 0.1624665000112601, - "nauc_recall_at_20_std": -0.06513958194587831, - "nauc_recall_at_3_diff1": 0.05083674702012879, - "nauc_recall_at_3_max": 0.06741308965704265, - "nauc_recall_at_3_std": -0.10189783181241095, - "nauc_recall_at_5_diff1": 0.037118320737183075, - "nauc_recall_at_5_max": 0.03554341529116537, - "nauc_recall_at_5_std": -0.0853279263347257, - "ndcg_at_1": 0.06934, - "ndcg_at_10": 0.22818, - "ndcg_at_100": 0.33297, - "ndcg_at_1000": 0.34, - "ndcg_at_20": 0.28462, - "ndcg_at_3": 0.13301, - "ndcg_at_5": 0.17188, - "precision_at_1": 0.06934, - "precision_at_10": 0.04509, - "precision_at_100": 0.00925, - "precision_at_1000": 0.00098, - "precision_at_20": 0.03368, - "precision_at_3": 0.06053, - "precision_at_5": 0.05528, - "recall_at_1": 0.06934, - "recall_at_10": 0.45094, - "recall_at_100": 0.925, - "recall_at_1000": 0.97972, - "recall_at_20": 0.67358, - "recall_at_3": 0.1816, - "recall_at_5": 0.27642 - } - ] - }, - "task_name": "NIGHTSI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2ITRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2ITRetrieval.json deleted file mode 100644 index 60695744d8..0000000000 --- a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2ITRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "53db4c9f9c93cb74926a1c9d04dea7d7acac2f21", - "evaluation_time": 2691.2529063224792, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.52663, - "map_at_1": 0.32198, - "map_at_10": 0.45953, - "map_at_100": 0.47181, - "map_at_1000": 0.4723, - "map_at_20": 0.46711, - "map_at_3": 0.42417, - "map_at_5": 0.44499, - "mrr_at_1": 0.41178813221823973, - "mrr_at_10": 0.5210127818550758, - "mrr_at_100": 0.5288710554330768, - "mrr_at_1000": 0.5290959300255417, - "mrr_at_20": 0.526159218535241, - "mrr_at_3": 0.4948891543873623, - "mrr_at_5": 0.5105402893933366, - "nauc_map_at_1000_diff1": 0.43876271441341697, - "nauc_map_at_1000_max": 0.2713223520198593, - "nauc_map_at_1000_std": -0.16052486820215323, - "nauc_map_at_100_diff1": 0.43857076615081975, - "nauc_map_at_100_max": 0.27148142408822945, - "nauc_map_at_100_std": -0.1604368100365843, - "nauc_map_at_10_diff1": 0.43871121919352996, - "nauc_map_at_10_max": 0.269439210185352, - "nauc_map_at_10_std": -0.16631952646794543, - "nauc_map_at_1_diff1": 0.5024807336255694, - "nauc_map_at_1_max": 0.2285751039496304, - "nauc_map_at_1_std": -0.15553705788344513, - "nauc_map_at_20_diff1": 0.4381920616977959, - "nauc_map_at_20_max": 0.27000963658536997, - "nauc_map_at_20_std": -0.16422634459492544, - "nauc_map_at_3_diff1": 0.44957721321495686, - "nauc_map_at_3_max": 0.26041374421396873, - "nauc_map_at_3_std": -0.17098574514931533, - "nauc_map_at_5_diff1": 0.4379760197271896, - "nauc_map_at_5_max": 0.26650735933197567, - "nauc_map_at_5_std": -0.1678803021826141, - "nauc_mrr_at_1000_diff1": 0.4498242067689463, - "nauc_mrr_at_1000_max": 0.2763503353709589, - "nauc_mrr_at_1000_std": -0.1515498089863098, - "nauc_mrr_at_100_diff1": 0.4497295532312827, - "nauc_mrr_at_100_max": 0.27653912064968234, - "nauc_mrr_at_100_std": -0.15134474459586014, - "nauc_mrr_at_10_diff1": 0.44835612729964286, - "nauc_mrr_at_10_max": 0.2760509111175981, - "nauc_mrr_at_10_std": -0.15401033906315414, - "nauc_mrr_at_1_diff1": 0.5064389691968733, - "nauc_mrr_at_1_max": 0.25538371022385664, - "nauc_mrr_at_1_std": -0.15120047500046432, - "nauc_mrr_at_20_diff1": 0.4491194452623415, - "nauc_mrr_at_20_max": 0.2759581261363807, - "nauc_mrr_at_20_std": -0.15329529609279208, - "nauc_mrr_at_3_diff1": 0.4510015598661547, - "nauc_mrr_at_3_max": 0.26930524592262844, - "nauc_mrr_at_3_std": -0.16055986565559113, - "nauc_mrr_at_5_diff1": 0.4467399021960818, - "nauc_mrr_at_5_max": 0.27441954793660395, - "nauc_mrr_at_5_std": -0.156411952108089, - "nauc_ndcg_at_1000_diff1": 0.42307561167146884, - "nauc_ndcg_at_1000_max": 0.28854455791397793, - "nauc_ndcg_at_1000_std": -0.13415719921037747, - "nauc_ndcg_at_100_diff1": 0.41863495760271513, - "nauc_ndcg_at_100_max": 0.2934231665534968, - "nauc_ndcg_at_100_std": -0.12953624384754348, - "nauc_ndcg_at_10_diff1": 0.4165024926207999, - "nauc_ndcg_at_10_max": 0.2815688455123661, - "nauc_ndcg_at_10_std": -0.16033266463844695, - "nauc_ndcg_at_1_diff1": 0.4991940974293242, - "nauc_ndcg_at_1_max": 0.25257839379065944, - "nauc_ndcg_at_1_std": -0.1476551024812966, - "nauc_ndcg_at_20_diff1": 0.4142260718683874, - "nauc_ndcg_at_20_max": 0.2832424372665771, - "nauc_ndcg_at_20_std": -0.15369250623843952, - "nauc_ndcg_at_3_diff1": 0.43173077808948285, - "nauc_ndcg_at_3_max": 0.26535751742144664, - "nauc_ndcg_at_3_std": -0.17037703654971278, - "nauc_ndcg_at_5_diff1": 0.41505821729386394, - "nauc_ndcg_at_5_max": 0.2753571205036466, - "nauc_ndcg_at_5_std": -0.16436190189593947, - "nauc_precision_at_1000_diff1": -0.09195784735246411, - "nauc_precision_at_1000_max": 0.11346836723919133, - "nauc_precision_at_1000_std": 0.19491442787034385, - "nauc_precision_at_100_diff1": -0.032103070968056875, - "nauc_precision_at_100_max": 0.18552459477832725, - "nauc_precision_at_100_std": 0.17369141585226536, - "nauc_precision_at_10_diff1": 0.14619869365447397, - "nauc_precision_at_10_max": 0.23031678733137417, - "nauc_precision_at_10_std": -0.05167802110000886, - "nauc_precision_at_1_diff1": 0.4991940974293242, - "nauc_precision_at_1_max": 0.25257839379065944, - "nauc_precision_at_1_std": -0.1476551024812966, - "nauc_precision_at_20_diff1": 0.08041747973726056, - "nauc_precision_at_20_max": 0.20668859387848693, - "nauc_precision_at_20_std": 0.0015670064844935908, - "nauc_precision_at_3_diff1": 0.2766345000123392, - "nauc_precision_at_3_max": 0.25180850003621597, - "nauc_precision_at_3_std": -0.1259167659370286, - "nauc_precision_at_5_diff1": 0.19761775316424005, - "nauc_precision_at_5_max": 0.24663783968040612, - "nauc_precision_at_5_std": -0.09202309206478451, - "nauc_recall_at_1000_diff1": 0.21907469666289658, - "nauc_recall_at_1000_max": 0.45355511952440225, - "nauc_recall_at_1000_std": 0.2803857322711674, - "nauc_recall_at_100_diff1": 0.23460647409258342, - "nauc_recall_at_100_max": 0.4368628650169039, - "nauc_recall_at_100_std": 0.15794853700940753, - "nauc_recall_at_10_diff1": 0.309978003824074, - "nauc_recall_at_10_max": 0.29394118609593484, - "nauc_recall_at_10_std": -0.1475438686983404, - "nauc_recall_at_1_diff1": 0.5024807336255694, - "nauc_recall_at_1_max": 0.2285751039496304, - "nauc_recall_at_1_std": -0.15553705788344513, - "nauc_recall_at_20_diff1": 0.27182787851817175, - "nauc_recall_at_20_max": 0.3004972888603483, - "nauc_recall_at_20_std": -0.11618638451955582, - "nauc_recall_at_3_diff1": 0.3666928890381534, - "nauc_recall_at_3_max": 0.25984514795329117, - "nauc_recall_at_3_std": -0.1721856188134978, - "nauc_recall_at_5_diff1": 0.3190261983587683, - "nauc_recall_at_5_max": 0.27727115206859354, - "nauc_recall_at_5_std": -0.1578256131156148, - "ndcg_at_1": 0.41458, - "ndcg_at_10": 0.52663, - "ndcg_at_100": 0.57734, - "ndcg_at_1000": 0.58728, - "ndcg_at_20": 0.54976, - "ndcg_at_3": 0.46369, - "ndcg_at_5": 0.49654, - "precision_at_1": 0.41458, - "precision_at_10": 0.09271, - "precision_at_100": 0.01253, - "precision_at_1000": 0.00136, - "precision_at_20": 0.05247, - "precision_at_3": 0.23032, - "precision_at_5": 0.15978, - "recall_at_1": 0.32198, - "recall_at_10": 0.66169, - "recall_at_100": 0.87535, - "recall_at_1000": 0.94464, - "recall_at_20": 0.74313, - "recall_at_3": 0.50816, - "recall_at_5": 0.57965 - } - ] - }, - "task_name": "WebQAT2ITRetrieval" -} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2TRetrieval.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2TRetrieval.json deleted file mode 100644 index febd96847e..0000000000 --- a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/WebQAT2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "468b42a2b2e767d80d2d93f5ae5d42f135a10478", - "evaluation_time": 988.9164459705353, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.64907, - "map_at_1": 0.31073, - "map_at_10": 0.55199, - "map_at_100": 0.56518, - "map_at_1000": 0.56572, - "map_at_20": 0.56038, - "map_at_3": 0.49469, - "map_at_5": 0.52906, - "mrr_at_1": 0.6276985743380855, - "mrr_at_10": 0.7316605243590977, - "mrr_at_100": 0.7344313618617866, - "mrr_at_1000": 0.7344774340987533, - "mrr_at_20": 0.7334756132529352, - "mrr_at_3": 0.7120162932790214, - "mrr_at_5": 0.7248065173116084, - "nauc_map_at_1000_diff1": 0.29580437839479934, - "nauc_map_at_1000_max": 0.2927721108826568, - "nauc_map_at_1000_std": 0.012872496436827125, - "nauc_map_at_100_diff1": 0.2957568562473137, - "nauc_map_at_100_max": 0.29338281472542477, - "nauc_map_at_100_std": 0.013473543179429661, - "nauc_map_at_10_diff1": 0.28970252769434723, - "nauc_map_at_10_max": 0.2864523752255613, - "nauc_map_at_10_std": 0.006314905902818341, - "nauc_map_at_1_diff1": 0.5159028191255995, - "nauc_map_at_1_max": 0.3174029367347432, - "nauc_map_at_1_std": -0.020320539407606746, - "nauc_map_at_20_diff1": 0.2947811042949462, - "nauc_map_at_20_max": 0.2929089074820099, - "nauc_map_at_20_std": 0.012461044754162466, - "nauc_map_at_3_diff1": 0.2905401755376613, - "nauc_map_at_3_max": 0.2661749034174475, - "nauc_map_at_3_std": -0.012626999843920064, - "nauc_map_at_5_diff1": 0.2861299068739539, - "nauc_map_at_5_max": 0.27446625170508415, - "nauc_map_at_5_std": -0.0058971920193630345, - "nauc_mrr_at_1000_diff1": 0.5086922608252236, - "nauc_mrr_at_1000_max": 0.3691168740765083, - "nauc_mrr_at_1000_std": 0.0006666279755451657, - "nauc_mrr_at_100_diff1": 0.5086748494269067, - "nauc_mrr_at_100_max": 0.36919780738198626, - "nauc_mrr_at_100_std": 0.0007592322141292633, - "nauc_mrr_at_10_diff1": 0.5085215685619419, - "nauc_mrr_at_10_max": 0.3718177164350851, - "nauc_mrr_at_10_std": 0.0027147806825914855, - "nauc_mrr_at_1_diff1": 0.5216513467070573, - "nauc_mrr_at_1_max": 0.320769879618688, - "nauc_mrr_at_1_std": -0.019386687001355655, - "nauc_mrr_at_20_diff1": 0.5083421770335178, - "nauc_mrr_at_20_max": 0.37019594976409614, - "nauc_mrr_at_20_std": 0.0017330204692732748, - "nauc_mrr_at_3_diff1": 0.5078153733991984, - "nauc_mrr_at_3_max": 0.3668773645861323, - "nauc_mrr_at_3_std": -0.008641807033782771, - "nauc_mrr_at_5_diff1": 0.5088459578154817, - "nauc_mrr_at_5_max": 0.3736712576479569, - "nauc_mrr_at_5_std": -2.2624894928488478e-05, - "nauc_ndcg_at_1000_diff1": 0.3374277453741463, - "nauc_ndcg_at_1000_max": 0.3321734054378378, - "nauc_ndcg_at_1000_std": 0.03673244516019259, - "nauc_ndcg_at_100_diff1": 0.3337261963315334, - "nauc_ndcg_at_100_max": 0.3432535360812789, - "nauc_ndcg_at_100_std": 0.049838336252328606, - "nauc_ndcg_at_10_diff1": 0.31654227932875967, - "nauc_ndcg_at_10_max": 0.32618988165002455, - "nauc_ndcg_at_10_std": 0.029231011592918396, - "nauc_ndcg_at_1_diff1": 0.5235838276940898, - "nauc_ndcg_at_1_max": 0.31696736904058387, - "nauc_ndcg_at_1_std": -0.02458631928633079, - "nauc_ndcg_at_20_diff1": 0.3275249333002614, - "nauc_ndcg_at_20_max": 0.34158128877849375, - "nauc_ndcg_at_20_std": 0.04494651591965775, - "nauc_ndcg_at_3_diff1": 0.32117487638130354, - "nauc_ndcg_at_3_max": 0.2941028830915832, - "nauc_ndcg_at_3_std": -0.007316961948953446, - "nauc_ndcg_at_5_diff1": 0.31284200905978193, - "nauc_ndcg_at_5_max": 0.3063806097099824, - "nauc_ndcg_at_5_std": 0.005868083161393718, - "nauc_precision_at_1000_diff1": -0.03635225074805508, - "nauc_precision_at_1000_max": 0.16765441647254156, - "nauc_precision_at_1000_std": 0.20388495902082046, - "nauc_precision_at_100_diff1": 0.09225951347257724, - "nauc_precision_at_100_max": 0.3981187839667785, - "nauc_precision_at_100_std": 0.30543394344414393, - "nauc_precision_at_10_diff1": 0.16625472536477842, - "nauc_precision_at_10_max": 0.3229284843568491, - "nauc_precision_at_10_std": 0.0995415625260059, - "nauc_precision_at_1_diff1": 0.5235838276940898, - "nauc_precision_at_1_max": 0.31696736904058387, - "nauc_precision_at_1_std": -0.02458631928633079, - "nauc_precision_at_20_diff1": 0.17259384065055555, - "nauc_precision_at_20_max": 0.38127315704848685, - "nauc_precision_at_20_std": 0.1789475498025293, - "nauc_precision_at_3_diff1": 0.2321551182804366, - "nauc_precision_at_3_max": 0.2757072760399245, - "nauc_precision_at_3_std": -3.369463890472521e-05, - "nauc_precision_at_5_diff1": 0.19327155462501952, - "nauc_precision_at_5_max": 0.2859361592299849, - "nauc_precision_at_5_std": 0.029196569686768183, - "nauc_recall_at_1000_diff1": 0.05823231057513774, - "nauc_recall_at_1000_max": 0.46812391396558123, - "nauc_recall_at_1000_std": 0.46536941182767955, - "nauc_recall_at_100_diff1": 0.14677934784136976, - "nauc_recall_at_100_max": 0.5113587224498323, - "nauc_recall_at_100_std": 0.38163612647621464, - "nauc_recall_at_10_diff1": 0.17727845662053254, - "nauc_recall_at_10_max": 0.34133843118145557, - "nauc_recall_at_10_std": 0.10554021248418982, - "nauc_recall_at_1_diff1": 0.5159028191255995, - "nauc_recall_at_1_max": 0.3174029367347432, - "nauc_recall_at_1_std": -0.020320539407606746, - "nauc_recall_at_20_diff1": 0.1891750473785528, - "nauc_recall_at_20_max": 0.411016904961582, - "nauc_recall_at_20_std": 0.1909091714508319, - "nauc_recall_at_3_diff1": 0.23406001763218115, - "nauc_recall_at_3_max": 0.28163137574773756, - "nauc_recall_at_3_std": 0.00345900087694381, - "nauc_recall_at_5_diff1": 0.1994571814043668, - "nauc_recall_at_5_max": 0.29485386492986626, - "nauc_recall_at_5_std": 0.032509102635484136, - "ndcg_at_1": 0.62688, - "ndcg_at_10": 0.64907, - "ndcg_at_100": 0.69217, - "ndcg_at_1000": 0.70137, - "ndcg_at_20": 0.6693, - "ndcg_at_3": 0.5656, - "ndcg_at_5": 0.61022, - "precision_at_1": 0.62688, - "precision_at_10": 0.15181, - "precision_at_100": 0.01853, - "precision_at_1000": 0.00197, - "precision_at_20": 0.08242, - "precision_at_3": 0.38031, - "precision_at_5": 0.26403, - "recall_at_1": 0.31073, - "recall_at_10": 0.74932, - "recall_at_100": 0.91236, - "recall_at_1000": 0.97101, - "recall_at_20": 0.81344, - "recall_at_3": 0.56456, - "recall_at_5": 0.65238 - } - ] - }, - "task_name": "WebQAT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/model_meta.json b/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/model_meta.json deleted file mode 100644 index cdc58dc7d4..0000000000 --- a/results-mieb/nomic-ai__nomic-embed-vision-v1.5/af2246fffdab78d8458418480e4886a8e48b70a7/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "nomic-ai/nomic-embed-vision-v1.5", "revision": "af2246fffdab78d8458418480e4886a8e48b70a7", "release_date": "2024-06-08", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "NomicVisionModelWrapper"} \ No newline at end of file diff --git a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/CIFAR10Clustering.json b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/CIFAR10Clustering.json deleted file mode 100644 index 981ba63e66..0000000000 --- a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/CIFAR10Clustering.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", - "evaluation_time": 265.58390378952026, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "ari": 0.6686950213456546, - "cluster_accuracy": 0.8305, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7469370718746896, - "nmi": 0.7469370718746896, - "v_measure": 0.7469370718746896 - } - ] - }, - "task_name": "CIFAR10Clustering" -} \ No newline at end of file diff --git a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json deleted file mode 100644 index f4ea1b602c..0000000000 --- a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", - "evaluation_time": 347.9395191669464, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.8880392156862745, - "f1": 0.8858313070145011, - "f1_weighted": 0.8855993382917493, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8880392156862745, - "scores_per_experiment": [ - { - "accuracy": 0.8921568627450981, - "f1": 0.8908725994288734, - "f1_weighted": 0.890707976140322 - }, - { - "accuracy": 0.888235294117647, - "f1": 0.8862588054389066, - "f1_weighted": 0.8857789823937094 - }, - { - "accuracy": 0.8833333333333333, - "f1": 0.8807271651302992, - "f1_weighted": 0.8807519823317017 - }, - { - "accuracy": 0.884313725490196, - "f1": 0.8818842242107174, - "f1_weighted": 0.8815393805409137 - }, - { - "accuracy": 0.8921568627450981, - "f1": 0.8894137408637092, - "f1_weighted": 0.8892183700520992 - } - ] - } - ] - }, - "task_name": "OxfordFlowersClassification" -} \ No newline at end of file diff --git a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json deleted file mode 100644 index 6ec67ca445..0000000000 --- a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "nyu-visionx/moco-v3-vit-b", "revision": "7d091cd70772c5c0ecf7f00b5f12ca609a99d69d", "release_date": "2024-06-03", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "mocov3_loader"} \ No newline at end of file diff --git a/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json b/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json deleted file mode 100644 index a1fb3e1e31..0000000000 --- a/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", - "evaluation_time": 780.7572722434998, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.8990196078431373, - "f1": 0.8976575858316652, - "f1_weighted": 0.8973426503552833, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8990196078431373, - "scores_per_experiment": [ - { - "accuracy": 0.8960784313725491, - "f1": 0.8954744443526484, - "f1_weighted": 0.8949013011291541 - }, - { - "accuracy": 0.9009803921568628, - "f1": 0.89924527322374, - "f1_weighted": 0.89919388345273 - }, - { - "accuracy": 0.8960784313725491, - "f1": 0.8937528098526334, - "f1_weighted": 0.8936883759623736 - }, - { - "accuracy": 0.8990196078431373, - "f1": 0.8985132758082764, - "f1_weighted": 0.8977617544247366 - }, - { - "accuracy": 0.9029411764705882, - "f1": 0.901302125921028, - "f1_weighted": 0.901167936807422 - } - ] - } - ] - }, - "task_name": "OxfordFlowersClassification" -} \ No newline at end of file diff --git a/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json b/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json deleted file mode 100644 index b9e1a399c2..0000000000 --- a/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "nyu-visionx/moco-v3-vit-l", "revision": "7bf75358d616f39b9716148bf4e3425f3bd35b47", "release_date": "2024-06-03", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "mocov3_loader"} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROCocoOrder.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROCocoOrder.json deleted file mode 100644 index 8924145dd2..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROCocoOrder.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "853ec8757226585a38a80886c51fe0f3f268787c", - "evaluation_time": 207.2516052722931, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.4812075197696686, - "text_acc": 0.4812075197696686 - } - ] - }, - "task_name": "AROCocoOrder" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROFlickrOrder.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROFlickrOrder.json deleted file mode 100644 index b69a24d891..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROFlickrOrder.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "1f9485f69c87947812378a1aedf86410c86a0aa8", - "evaluation_time": 37.06100630760193, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.5583999752998352, - "text_acc": 0.5583999752998352 - } - ] - }, - "task_name": "AROFlickrOrder" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualAttribution.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualAttribution.json deleted file mode 100644 index 98049aafaa..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualAttribution.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "18f7e01358d91df599d723f00e16a18640e19398", - "evaluation_time": 339.30732822418213, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.6186517477035522, - "text_acc": 0.6186517477035522 - } - ] - }, - "task_name": "AROVisualAttribution" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualRelation.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualRelation.json deleted file mode 100644 index 3bc832d5d9..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/AROVisualRelation.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "3867ad4f46a1ac2e63be034d1fc77dd8c2ef7209", - "evaluation_time": 278.83220171928406, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.5365334153175354, - "text_acc": 0.5365334153175354 - } - ] - }, - "task_name": "AROVisualRelation" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Birdsnap.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Birdsnap.json deleted file mode 100644 index 8fb18fd10f..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Birdsnap.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "fd23015508be94f0b5b59d61630e4ea2536509e4", - "evaluation_time": 1329.5408725738525, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5715829281469476, - "f1": 0.5427579493814982, - "f1_weighted": 0.5675297472237123, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5715829281469476, - "scores_per_experiment": [ - { - "accuracy": 0.5705024311183144, - "f1": 0.5441404980561606, - "f1_weighted": 0.5690425577783762 - }, - { - "accuracy": 0.5764451647757969, - "f1": 0.5486149281330003, - "f1_weighted": 0.573202555293317 - }, - { - "accuracy": 0.5748244192328471, - "f1": 0.5423477063005765, - "f1_weighted": 0.568412911615314 - }, - { - "accuracy": 0.5640194489465153, - "f1": 0.5339722105851138, - "f1_weighted": 0.5562256849939183 - }, - { - "accuracy": 0.5721231766612642, - "f1": 0.5447144038326402, - "f1_weighted": 0.5707650264376358 - } - ] - } - ] - }, - "task_name": "Birdsnap" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CIRRIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CIRRIT2TRetrieval.json deleted file mode 100644 index f63b804ff7..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CIRRIT2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", - "evaluation_time": 123.18759036064148, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.09754, - "map_at_1": 0.00408, - "map_at_10": 0.06314, - "map_at_100": 0.07411, - "map_at_1000": 0.07553, - "map_at_20": 0.06877, - "map_at_3": 0.04189, - "map_at_5": 0.05327, - "mrr_at_1": 0.008633093525179856, - "mrr_at_10": 0.06345980358570272, - "mrr_at_100": 0.07443694089212283, - "mrr_at_1000": 0.07586721932715101, - "mrr_at_20": 0.06910360751975989, - "mrr_at_3": 0.04240607513988813, - "mrr_at_5": 0.053569144684252626, - "nauc_map_at_1000_diff1": -0.09430006303115672, - "nauc_map_at_1000_max": 0.07971589724669469, - "nauc_map_at_1000_std": 0.1097849476586519, - "nauc_map_at_100_diff1": -0.09304519843836945, - "nauc_map_at_100_max": 0.07978371174425837, - "nauc_map_at_100_std": 0.10767454860254984, - "nauc_map_at_10_diff1": -0.09265414471119784, - "nauc_map_at_10_max": 0.07156210824751408, - "nauc_map_at_10_std": 0.09100330279710303, - "nauc_map_at_1_diff1": -0.5319922978582364, - "nauc_map_at_1_max": 0.22971479979166354, - "nauc_map_at_1_std": 0.01727457819725688, - "nauc_map_at_20_diff1": -0.08883107818505538, - "nauc_map_at_20_max": 0.07564770187280079, - "nauc_map_at_20_std": 0.09984178318888319, - "nauc_map_at_3_diff1": -0.11451405772196703, - "nauc_map_at_3_max": 0.06998317415517456, - "nauc_map_at_3_std": 0.07144993107855412, - "nauc_map_at_5_diff1": -0.09303955529622057, - "nauc_map_at_5_max": 0.06894560968071584, - "nauc_map_at_5_std": 0.0765640043400923, - "nauc_mrr_at_1000_diff1": -0.09682010815892876, - "nauc_mrr_at_1000_max": 0.08150275059724531, - "nauc_mrr_at_1000_std": 0.10918420856774745, - "nauc_mrr_at_100_diff1": -0.09553424341368219, - "nauc_mrr_at_100_max": 0.08155065521338761, - "nauc_mrr_at_100_std": 0.10708594341679054, - "nauc_mrr_at_10_diff1": -0.09506902538489626, - "nauc_mrr_at_10_max": 0.07339968417405465, - "nauc_mrr_at_10_std": 0.0906630104753227, - "nauc_mrr_at_1_diff1": -0.5290373550368184, - "nauc_mrr_at_1_max": 0.2344229794592015, - "nauc_mrr_at_1_std": 0.005955042780908148, - "nauc_mrr_at_20_diff1": -0.09140047438145477, - "nauc_mrr_at_20_max": 0.07741254897582059, - "nauc_mrr_at_20_std": 0.0993326318120689, - "nauc_mrr_at_3_diff1": -0.11939291114310412, - "nauc_mrr_at_3_max": 0.07384253329310762, - "nauc_mrr_at_3_std": 0.07139864053407562, - "nauc_mrr_at_5_diff1": -0.09696157114944583, - "nauc_mrr_at_5_max": 0.07122590150036714, - "nauc_mrr_at_5_std": 0.07643234788433957, - "nauc_ndcg_at_1000_diff1": -0.10106932279250458, - "nauc_ndcg_at_1000_max": 0.10096178940020763, - "nauc_ndcg_at_1000_std": 0.19577796302889916, - "nauc_ndcg_at_100_diff1": -0.08696795064375475, - "nauc_ndcg_at_100_max": 0.09814750175500678, - "nauc_ndcg_at_100_std": 0.16801654804595473, - "nauc_ndcg_at_10_diff1": -0.07446639262352736, - "nauc_ndcg_at_10_max": 0.070454822695297, - "nauc_ndcg_at_10_std": 0.10445010583766168, - "nauc_ndcg_at_1_diff1": -0.5277319943271112, - "nauc_ndcg_at_1_max": 0.23186783701943348, - "nauc_ndcg_at_1_std": 0.003311740580321219, - "nauc_ndcg_at_20_diff1": -0.06726707326404968, - "nauc_ndcg_at_20_max": 0.0801126382265569, - "nauc_ndcg_at_20_std": 0.12521764835456067, - "nauc_ndcg_at_3_diff1": -0.09563193072427134, - "nauc_ndcg_at_3_max": 0.06551385540947947, - "nauc_ndcg_at_3_std": 0.0750635864151865, - "nauc_ndcg_at_5_diff1": -0.07109394463486146, - "nauc_ndcg_at_5_max": 0.06440364564768383, - "nauc_ndcg_at_5_std": 0.08123581037260018, - "nauc_precision_at_1000_diff1": -0.21894379652044482, - "nauc_precision_at_1000_max": 0.20110532137532738, - "nauc_precision_at_1000_std": 0.4985263956386437, - "nauc_precision_at_100_diff1": -0.11448917033346492, - "nauc_precision_at_100_max": 0.13704560799012916, - "nauc_precision_at_100_std": 0.25670911616573194, - "nauc_precision_at_10_diff1": -0.08078240677660208, - "nauc_precision_at_10_max": 0.07928001837810364, - "nauc_precision_at_10_std": 0.11439470716368563, - "nauc_precision_at_1_diff1": -0.5277319943271112, - "nauc_precision_at_1_max": 0.23186783701943348, - "nauc_precision_at_1_std": 0.003311740580321219, - "nauc_precision_at_20_diff1": -0.06791517816537197, - "nauc_precision_at_20_max": 0.0949095608045591, - "nauc_precision_at_20_std": 0.15010397292626543, - "nauc_precision_at_3_diff1": -0.11153702507708928, - "nauc_precision_at_3_max": 0.07582115029130086, - "nauc_precision_at_3_std": 0.06977064016082565, - "nauc_precision_at_5_diff1": -0.07729490099402637, - "nauc_precision_at_5_max": 0.07163492889720516, - "nauc_precision_at_5_std": 0.08001410639199522, - "nauc_recall_at_1000_diff1": -0.1855830858750129, - "nauc_recall_at_1000_max": 0.1980425044862128, - "nauc_recall_at_1000_std": 0.564878472724284, - "nauc_recall_at_100_diff1": -0.09436834890876052, - "nauc_recall_at_100_max": 0.1315423417990006, - "nauc_recall_at_100_std": 0.2717679584573868, - "nauc_recall_at_10_diff1": -0.05552800142913649, - "nauc_recall_at_10_max": 0.07015781570417769, - "nauc_recall_at_10_std": 0.12162556282236173, - "nauc_recall_at_1_diff1": -0.5319922978582364, - "nauc_recall_at_1_max": 0.22971479979166354, - "nauc_recall_at_1_std": 0.01727457819725688, - "nauc_recall_at_20_diff1": -0.045395108502877965, - "nauc_recall_at_20_max": 0.08780002237291626, - "nauc_recall_at_20_std": 0.1581707521402131, - "nauc_recall_at_3_diff1": -0.06721017685785699, - "nauc_recall_at_3_max": 0.05770885232307196, - "nauc_recall_at_3_std": 0.08003902316354816, - "nauc_recall_at_5_diff1": -0.04262981360165199, - "nauc_recall_at_5_max": 0.05798966850733496, - "nauc_recall_at_5_std": 0.08747605572883835, - "ndcg_at_1": 0.00839, - "ndcg_at_10": 0.09754, - "ndcg_at_100": 0.1599, - "ndcg_at_1000": 0.20034, - "ndcg_at_20": 0.11799, - "ndcg_at_3": 0.0533, - "ndcg_at_5": 0.0736, - "precision_at_1": 0.00839, - "precision_at_10": 0.02189, - "precision_at_100": 0.00533, - "precision_at_1000": 0.00086, - "precision_at_20": 0.01499, - "precision_at_3": 0.03181, - "precision_at_5": 0.02897, - "recall_at_1": 0.00408, - "recall_at_10": 0.20899, - "recall_at_100": 0.52254, - "recall_at_1000": 0.8458, - "recall_at_20": 0.28981, - "recall_at_3": 0.08597, - "recall_at_5": 0.13501 - } - ] - }, - "task_name": "CIRRIT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchCount.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchCount.json deleted file mode 100644 index 2a11efd947..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchCount.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 12.502729654312134, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.2017766497461929, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.2017766497461929 - } - ] - }, - "task_name": "CVBenchCount" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDepth.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDepth.json deleted file mode 100644 index b420a0688b..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDepth.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 9.667015790939331, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5116666666666667, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5116666666666667 - } - ] - }, - "task_name": "CVBenchDepth" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDistance.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDistance.json deleted file mode 100644 index 742d2cc2d1..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchDistance.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 9.924455404281616, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.4633333333333333, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4633333333333333 - } - ] - }, - "task_name": "CVBenchDistance" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchRelation.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchRelation.json deleted file mode 100644 index 7c9c4d1238..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/CVBenchRelation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 6.079412221908569, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5030769230769231, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5030769230769231 - } - ] - }, - "task_name": "CVBenchRelation" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Caltech101.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Caltech101.json deleted file mode 100644 index 9cf71ae0fc..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Caltech101.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", - "evaluation_time": 140.4232726097107, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "accuracy": 0.9374424720578567, - "f1": 0.8896507530328425, - "f1_weighted": 0.9381916904354753, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9374424720578567, - "scores_per_experiment": [ - { - "accuracy": 0.9316239316239316, - "f1": 0.8916896755795425, - "f1_weighted": 0.9320804382676573 - }, - { - "accuracy": 0.9421433267587114, - "f1": 0.8911440787847722, - "f1_weighted": 0.9432545002991353 - }, - { - "accuracy": 0.9349112426035503, - "f1": 0.8901121996677432, - "f1_weighted": 0.9358989633433914 - }, - { - "accuracy": 0.9373767258382643, - "f1": 0.8843712891021908, - "f1_weighted": 0.9381450174798306 - }, - { - "accuracy": 0.9411571334648258, - "f1": 0.8909365220299629, - "f1_weighted": 0.9415795327873617 - } - ] - } - ] - }, - "task_name": "Caltech101" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/FORBI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/FORBI2IRetrieval.json deleted file mode 100644 index 4d3811487d..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/FORBI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", - "evaluation_time": 1242.6785640716553, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.40136, - "cv_recall_at_10": 0.55162, - "cv_recall_at_100": 0.66423, - "cv_recall_at_1000": 0.7397, - "cv_recall_at_20": 0.58755, - "cv_recall_at_3": 0.48317, - "cv_recall_at_5": 0.51185, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.40136, - "map_at_1": 0.40023, - "map_at_10": 0.44883, - "map_at_100": 0.45322, - "map_at_1000": 0.45356, - "map_at_20": 0.45134, - "map_at_3": 0.43694, - "map_at_5": 0.4435, - "mrr_at_1": 0.40135849056603773, - "mrr_at_10": 0.44943953279424914, - "mrr_at_100": 0.4538207184180781, - "mrr_at_1000": 0.4541651229469619, - "mrr_at_20": 0.4519378833805158, - "mrr_at_3": 0.43757232704402443, - "mrr_at_5": 0.44411194968553264, - "nauc_cv_recall_at_1000_diff1": 0.9500334258065578, - "nauc_cv_recall_at_1000_max": -1.2528093194175578, - "nauc_cv_recall_at_1000_std": -1.1856652591596875, - "nauc_cv_recall_at_100_diff1": 0.877926997353908, - "nauc_cv_recall_at_100_max": -0.8870905020174381, - "nauc_cv_recall_at_100_std": -0.8732695918632056, - "nauc_cv_recall_at_10_diff1": 0.8177473807784005, - "nauc_cv_recall_at_10_max": -0.5466647529055957, - "nauc_cv_recall_at_10_std": -0.6554690765334921, - "nauc_cv_recall_at_1_diff1": 0.8246695670414056, - "nauc_cv_recall_at_1_max": -0.3034632255124627, - "nauc_cv_recall_at_1_std": -0.4858988248874936, - "nauc_cv_recall_at_20_diff1": 0.82479953127787, - "nauc_cv_recall_at_20_max": -0.6318860267460972, - "nauc_cv_recall_at_20_std": -0.7024579748752124, - "nauc_cv_recall_at_3_diff1": 0.8072308741555094, - "nauc_cv_recall_at_3_max": -0.4123387172398377, - "nauc_cv_recall_at_3_std": -0.5690279516462133, - "nauc_cv_recall_at_5_diff1": 0.8047845690821774, - "nauc_cv_recall_at_5_max": -0.46313010588875514, - "nauc_cv_recall_at_5_std": -0.5999518918827577, - "nauc_map_at_1000_diff1": 0.821123720578351, - "nauc_map_at_1000_max": -0.380874046553554, - "nauc_map_at_1000_std": -0.5404325379702113, - "nauc_map_at_100_diff1": 0.8209485062871567, - "nauc_map_at_100_max": -0.3800979301805527, - "nauc_map_at_100_std": -0.5397866691260815, - "nauc_map_at_10_diff1": 0.8195428750181966, - "nauc_map_at_10_max": -0.37103423612650216, - "nauc_map_at_10_std": -0.5343816263585759, - "nauc_map_at_1_diff1": 0.8270196566959886, - "nauc_map_at_1_max": -0.30320990154210953, - "nauc_map_at_1_std": -0.4833533184781571, - "nauc_map_at_20_diff1": 0.820009618850825, - "nauc_map_at_20_max": -0.3759310074719576, - "nauc_map_at_20_std": -0.5369871242494627, - "nauc_map_at_3_diff1": 0.8186425667711692, - "nauc_map_at_3_max": -0.3502082517227007, - "nauc_map_at_3_std": -0.5210663781673923, - "nauc_map_at_5_diff1": 0.8178707187282951, - "nauc_map_at_5_max": -0.36111543033237625, - "nauc_map_at_5_std": -0.5277312572627604, - "nauc_mrr_at_1000_diff1": 0.8198732381938787, - "nauc_mrr_at_1000_max": -0.3810706994001967, - "nauc_mrr_at_1000_std": -0.5418434797910091, - "nauc_mrr_at_100_diff1": 0.8196987253945341, - "nauc_mrr_at_100_max": -0.38029428979297347, - "nauc_mrr_at_100_std": -0.5411966208887677, - "nauc_mrr_at_10_diff1": 0.8182978327883684, - "nauc_mrr_at_10_max": -0.3712361113968832, - "nauc_mrr_at_10_std": -0.5358019360627169, - "nauc_mrr_at_1_diff1": 0.8246695670414056, - "nauc_mrr_at_1_max": -0.3034632255124627, - "nauc_mrr_at_1_std": -0.4858988248874936, - "nauc_mrr_at_20_diff1": 0.8187629322864929, - "nauc_mrr_at_20_max": -0.3761251367912585, - "nauc_mrr_at_20_std": -0.538391953833271, - "nauc_mrr_at_3_diff1": 0.8173376655828167, - "nauc_mrr_at_3_max": -0.35042983995133714, - "nauc_mrr_at_3_std": -0.5225353792328652, - "nauc_mrr_at_5_diff1": 0.8166387092255286, - "nauc_mrr_at_5_max": -0.3613393270069024, - "nauc_mrr_at_5_std": -0.5291442800691877, - "nauc_ndcg_at_1000_diff1": 0.8334868024040154, - "nauc_ndcg_at_1000_max": -0.48731436283957213, - "nauc_ndcg_at_1000_std": -0.6163695321444129, - "nauc_ndcg_at_100_diff1": 0.8281459653974711, - "nauc_ndcg_at_100_max": -0.4631929003672613, - "nauc_ndcg_at_100_std": -0.594729965146729, - "nauc_ndcg_at_10_diff1": 0.8186924353738807, - "nauc_ndcg_at_10_max": -0.4106178964102518, - "nauc_ndcg_at_10_std": -0.5618271856100127, - "nauc_ndcg_at_1_diff1": 0.8270196566959886, - "nauc_ndcg_at_1_max": -0.30320990154210953, - "nauc_ndcg_at_1_std": -0.4833533184781571, - "nauc_ndcg_at_20_diff1": 0.8203675518215456, - "nauc_ndcg_at_20_max": -0.42922604110746493, - "nauc_ndcg_at_20_std": -0.5718977180047882, - "nauc_ndcg_at_3_diff1": 0.8158251981916631, - "nauc_ndcg_at_3_max": -0.3659119877900009, - "nauc_ndcg_at_3_std": -0.5331616044670827, - "nauc_ndcg_at_5_diff1": 0.8145904125253537, - "nauc_ndcg_at_5_max": -0.38591697079578874, - "nauc_ndcg_at_5_std": -0.5454027961145099, - "nauc_precision_at_1000_diff1": 0.9500334258065563, - "nauc_precision_at_1000_max": -1.2528093194175558, - "nauc_precision_at_1000_std": -1.1856652591596857, - "nauc_precision_at_100_diff1": 0.877926997353908, - "nauc_precision_at_100_max": -0.8870905020174389, - "nauc_precision_at_100_std": -0.8732695918632062, - "nauc_precision_at_10_diff1": 0.817762219827385, - "nauc_precision_at_10_max": -0.5465388231157847, - "nauc_precision_at_10_std": -0.6552180490907682, - "nauc_precision_at_1_diff1": 0.8270196566959886, - "nauc_precision_at_1_max": -0.30320990154210953, - "nauc_precision_at_1_std": -0.4833533184781571, - "nauc_precision_at_20_diff1": 0.824799531277869, - "nauc_precision_at_20_max": -0.6318860267460973, - "nauc_precision_at_20_std": -0.7024579748752124, - "nauc_precision_at_3_diff1": 0.8075292153589316, - "nauc_precision_at_3_max": -0.4121216998390528, - "nauc_precision_at_3_std": -0.568597762738296, - "nauc_precision_at_5_diff1": 0.8047485091566745, - "nauc_precision_at_5_max": -0.46287592219959217, - "nauc_precision_at_5_std": -0.5997842327013363, - "nauc_recall_at_1000_diff1": 0.9500334258065578, - "nauc_recall_at_1000_max": -1.2528093194175578, - "nauc_recall_at_1000_std": -1.1856652591596875, - "nauc_recall_at_100_diff1": 0.877926997353908, - "nauc_recall_at_100_max": -0.8870905020174381, - "nauc_recall_at_100_std": -0.8732695918632056, - "nauc_recall_at_10_diff1": 0.8177622198273848, - "nauc_recall_at_10_max": -0.5465388231157855, - "nauc_recall_at_10_std": -0.6552180490907689, - "nauc_recall_at_1_diff1": 0.8270196566959886, - "nauc_recall_at_1_max": -0.30320990154210953, - "nauc_recall_at_1_std": -0.4833533184781571, - "nauc_recall_at_20_diff1": 0.82479953127787, - "nauc_recall_at_20_max": -0.6318860267460972, - "nauc_recall_at_20_std": -0.7024579748752124, - "nauc_recall_at_3_diff1": 0.8075292153589314, - "nauc_recall_at_3_max": -0.41212169983905295, - "nauc_recall_at_3_std": -0.5685977627382963, - "nauc_recall_at_5_diff1": 0.8047485091566744, - "nauc_recall_at_5_max": -0.4628759221995928, - "nauc_recall_at_5_std": -0.5997842327013364, - "ndcg_at_1": 0.40023, - "ndcg_at_10": 0.4735, - "ndcg_at_100": 0.49668, - "ndcg_at_1000": 0.50618, - "ndcg_at_20": 0.48261, - "ndcg_at_3": 0.44879, - "ndcg_at_5": 0.46062, - "precision_at_1": 0.40023, - "precision_at_10": 0.05515, - "precision_at_100": 0.00664, - "precision_at_1000": 0.00074, - "precision_at_20": 0.02938, - "precision_at_3": 0.16101, - "precision_at_5": 0.10235, - "recall_at_1": 0.40023, - "recall_at_10": 0.55155, - "recall_at_100": 0.66423, - "recall_at_1000": 0.7397, - "recall_at_20": 0.58755, - "recall_at_3": 0.48302, - "recall_at_5": 0.51177 - } - ] - }, - "task_name": "FORBI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kI2TRetrieval.json deleted file mode 100644 index a90131d30b..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb", - "evaluation_time": 13.612479209899902, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.82, - "cv_recall_at_10": 0.986, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.994, - "cv_recall_at_3": 0.931, - "cv_recall_at_5": 0.97, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.74273, - "map_at_1": 0.164, - "map_at_10": 0.63451, - "map_at_100": 0.67573, - "map_at_1000": 0.67665, - "map_at_20": 0.66234, - "map_at_3": 0.4162, - "map_at_5": 0.56242, - "mrr_at_1": 0.82, - "mrr_at_10": 0.8819170634920639, - "mrr_at_100": 0.8826449941029878, - "mrr_at_1000": 0.8826449941029878, - "mrr_at_20": 0.8824860171691058, - "mrr_at_3": 0.8710000000000003, - "mrr_at_5": 0.8797000000000006, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": 0.19891289849274035, - "nauc_cv_recall_at_10_max": 0.5265772975856937, - "nauc_cv_recall_at_10_std": 0.3455382152861277, - "nauc_cv_recall_at_1_diff1": 0.4480945037999776, - "nauc_cv_recall_at_1_max": 0.3922045379447073, - "nauc_cv_recall_at_1_std": -0.12505231853728344, - "nauc_cv_recall_at_20_diff1": 0.3018985371926489, - "nauc_cv_recall_at_20_max": 0.6433239962651742, - "nauc_cv_recall_at_20_std": 0.9537037037037052, - "nauc_cv_recall_at_3_diff1": 0.2737655448652898, - "nauc_cv_recall_at_3_max": 0.4099108242330735, - "nauc_cv_recall_at_3_std": -0.1392102734813745, - "nauc_cv_recall_at_5_diff1": 0.2479458450046711, - "nauc_cv_recall_at_5_max": 0.45326797385621426, - "nauc_cv_recall_at_5_std": 0.024727668845316842, - "nauc_map_at_1000_diff1": 0.12299692538637216, - "nauc_map_at_1000_max": 0.40383972971788085, - "nauc_map_at_1000_std": -0.03877241232841388, - "nauc_map_at_100_diff1": 0.1224103741610186, - "nauc_map_at_100_max": 0.4045083510460462, - "nauc_map_at_100_std": -0.036813876580229825, - "nauc_map_at_10_diff1": 0.11422185007928484, - "nauc_map_at_10_max": 0.39191511510815347, - "nauc_map_at_10_std": -0.056740898404007624, - "nauc_map_at_1_diff1": 0.4480945037999775, - "nauc_map_at_1_max": 0.39220453794470633, - "nauc_map_at_1_std": -0.12505231853728427, - "nauc_map_at_20_diff1": 0.11978487595245653, - "nauc_map_at_20_max": 0.4010120010081494, - "nauc_map_at_20_std": -0.038459183289332906, - "nauc_map_at_3_diff1": 0.19814408531027977, - "nauc_map_at_3_max": 0.4018668277901667, - "nauc_map_at_3_std": -0.12036750531181802, - "nauc_map_at_5_diff1": 0.11645918188256225, - "nauc_map_at_5_max": 0.38608545461679483, - "nauc_map_at_5_std": -0.08112260515216518, - "nauc_mrr_at_1000_diff1": 0.4117843439551849, - "nauc_mrr_at_1000_max": 0.39603449433424776, - "nauc_mrr_at_1000_std": -0.1224537228873275, - "nauc_mrr_at_100_diff1": 0.4117843439551849, - "nauc_mrr_at_100_max": 0.39603449433424776, - "nauc_mrr_at_100_std": -0.1224537228873275, - "nauc_mrr_at_10_diff1": 0.40988550434587345, - "nauc_mrr_at_10_max": 0.39584154636889374, - "nauc_mrr_at_10_std": -0.12111989031979221, - "nauc_mrr_at_1_diff1": 0.4480945037999776, - "nauc_mrr_at_1_max": 0.3922045379447073, - "nauc_mrr_at_1_std": -0.12505231853728344, - "nauc_mrr_at_20_diff1": 0.41146243588617815, - "nauc_mrr_at_20_max": 0.3962597833666116, - "nauc_mrr_at_20_std": -0.12095286988661123, - "nauc_mrr_at_3_diff1": 0.39902923729559436, - "nauc_mrr_at_3_max": 0.3943872791235097, - "nauc_mrr_at_3_std": -0.13306276199025832, - "nauc_mrr_at_5_diff1": 0.4076485858771895, - "nauc_mrr_at_5_max": 0.39577465779193577, - "nauc_mrr_at_5_std": -0.12265544640203102, - "nauc_ndcg_at_1000_diff1": 0.1844301804081895, - "nauc_ndcg_at_1000_max": 0.42130933536662635, - "nauc_ndcg_at_1000_std": -0.035065914910614955, - "nauc_ndcg_at_100_diff1": 0.17496821020126946, - "nauc_ndcg_at_100_max": 0.43029672300582816, - "nauc_ndcg_at_100_std": -0.002507684096230846, - "nauc_ndcg_at_10_diff1": 0.14481662979912127, - "nauc_ndcg_at_10_max": 0.40520681410698756, - "nauc_ndcg_at_10_std": -0.04220162909542145, - "nauc_ndcg_at_1_diff1": 0.4480945037999776, - "nauc_ndcg_at_1_max": 0.3922045379447073, - "nauc_ndcg_at_1_std": -0.12505231853728344, - "nauc_ndcg_at_20_diff1": 0.15803572920290762, - "nauc_ndcg_at_20_max": 0.42054444917147027, - "nauc_ndcg_at_20_std": -0.0019159299606582648, - "nauc_ndcg_at_3_diff1": 0.21765219679209533, - "nauc_ndcg_at_3_max": 0.40379267362903226, - "nauc_ndcg_at_3_std": -0.12304328773334444, - "nauc_ndcg_at_5_diff1": 0.1462495129764977, - "nauc_ndcg_at_5_max": 0.3934225656825324, - "nauc_ndcg_at_5_std": -0.0786882910212851, - "nauc_precision_at_1000_diff1": 0.62427637721762, - "nauc_precision_at_1000_max": 0.7399626517273514, - "nauc_precision_at_1000_std": 0.7899159663865152, - "nauc_precision_at_100_diff1": 0.028283567165904248, - "nauc_precision_at_100_max": 0.5215659239456432, - "nauc_precision_at_100_std": 0.5279993895443634, - "nauc_precision_at_10_diff1": 0.06152272326044776, - "nauc_precision_at_10_max": 0.3828014005531608, - "nauc_precision_at_10_std": 0.018326040537019386, - "nauc_precision_at_1_diff1": 0.4480945037999776, - "nauc_precision_at_1_max": 0.3922045379447073, - "nauc_precision_at_1_std": -0.12505231853728344, - "nauc_precision_at_20_diff1": 0.06089982228042279, - "nauc_precision_at_20_max": 0.4065587877654105, - "nauc_precision_at_20_std": 0.1612227106912349, - "nauc_precision_at_3_diff1": 0.16178283766113552, - "nauc_precision_at_3_max": 0.3991300609614105, - "nauc_precision_at_3_std": -0.12286918672342817, - "nauc_precision_at_5_diff1": 0.08039511083398675, - "nauc_precision_at_5_max": 0.3789339125780607, - "nauc_precision_at_5_std": -0.05814725911556473, - "nauc_recall_at_1000_diff1": 0.624276377217619, - "nauc_recall_at_1000_max": 0.7399626517273947, - "nauc_recall_at_1000_std": 0.7899159663865074, - "nauc_recall_at_100_diff1": 0.028283567165905115, - "nauc_recall_at_100_max": 0.5215659239456432, - "nauc_recall_at_100_std": 0.5279993895443671, - "nauc_recall_at_10_diff1": 0.06152272326044776, - "nauc_recall_at_10_max": 0.3828014005531608, - "nauc_recall_at_10_std": 0.018326040537019386, - "nauc_recall_at_1_diff1": 0.4480945037999775, - "nauc_recall_at_1_max": 0.39220453794470633, - "nauc_recall_at_1_std": -0.12505231853728427, - "nauc_recall_at_20_diff1": 0.06089982228042279, - "nauc_recall_at_20_max": 0.4065587877654105, - "nauc_recall_at_20_std": 0.1612227106912349, - "nauc_recall_at_3_diff1": 0.161782837661136, - "nauc_recall_at_3_max": 0.3991300609614111, - "nauc_recall_at_3_std": -0.12286918672342777, - "nauc_recall_at_5_diff1": 0.08039511083398675, - "nauc_recall_at_5_max": 0.3789339125780607, - "nauc_recall_at_5_std": -0.05814725911556473, - "ndcg_at_1": 0.82, - "ndcg_at_10": 0.74273, - "ndcg_at_100": 0.82168, - "ndcg_at_1000": 0.82961, - "ndcg_at_20": 0.78527, - "ndcg_at_3": 0.7496, - "ndcg_at_5": 0.6661, - "precision_at_1": 0.82, - "precision_at_10": 0.3759, - "precision_at_100": 0.04818, - "precision_at_1000": 0.00499, - "precision_at_20": 0.2125, - "precision_at_3": 0.72567, - "precision_at_5": 0.6134, - "recall_at_1": 0.164, - "recall_at_10": 0.7518, - "recall_at_100": 0.9636, - "recall_at_1000": 0.999, - "recall_at_20": 0.85, - "recall_at_3": 0.4354, - "recall_at_5": 0.6134 - } - ] - }, - "task_name": "Flickr30kI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kT2IRetrieval.json deleted file mode 100644 index 8b907392b6..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "e819702b287bfbe084e129a61f308a802b7c108e", - "evaluation_time": 22.22239637374878, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.6214, - "cv_recall_at_10": 0.915, - "cv_recall_at_100": 0.9932, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.9532, - "cv_recall_at_3": 0.7944, - "cv_recall_at_5": 0.854, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.76748, - "map_at_1": 0.6214, - "map_at_10": 0.72031, - "map_at_100": 0.72409, - "map_at_1000": 0.72412, - "map_at_20": 0.72299, - "map_at_3": 0.69867, - "map_at_5": 0.71205, - "mrr_at_1": 0.6214, - "mrr_at_10": 0.7203121428571442, - "mrr_at_100": 0.724085885840924, - "mrr_at_1000": 0.7241243831788744, - "mrr_at_20": 0.7229865462203786, - "mrr_at_3": 0.6986666666666691, - "mrr_at_5": 0.7120466666666677, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.6346185533036849, - "nauc_cv_recall_at_100_max": 0.6848052946668853, - "nauc_cv_recall_at_100_std": 0.4664002856044359, - "nauc_cv_recall_at_10_diff1": 0.5758345691217669, - "nauc_cv_recall_at_10_max": 0.5616378316032299, - "nauc_cv_recall_at_10_std": 0.15670456417861245, - "nauc_cv_recall_at_1_diff1": 0.6964868193650136, - "nauc_cv_recall_at_1_max": 0.4658840817263219, - "nauc_cv_recall_at_1_std": 0.06009948824657468, - "nauc_cv_recall_at_20_diff1": 0.5942604962212811, - "nauc_cv_recall_at_20_max": 0.5904757914561845, - "nauc_cv_recall_at_20_std": 0.18555826889159976, - "nauc_cv_recall_at_3_diff1": 0.6016933324728311, - "nauc_cv_recall_at_3_max": 0.4996714444125018, - "nauc_cv_recall_at_3_std": 0.11144390402265514, - "nauc_cv_recall_at_5_diff1": 0.596326854581594, - "nauc_cv_recall_at_5_max": 0.5435354775022344, - "nauc_cv_recall_at_5_std": 0.1647132585408802, - "nauc_map_at_1000_diff1": 0.6681451778281712, - "nauc_map_at_1000_max": 0.48153054698905345, - "nauc_map_at_1000_std": 0.08084200415260853, - "nauc_map_at_100_diff1": 0.6681421969134242, - "nauc_map_at_100_max": 0.4815602459122235, - "nauc_map_at_100_std": 0.08089746305416494, - "nauc_map_at_10_diff1": 0.66637927944471, - "nauc_map_at_10_max": 0.4824993978185994, - "nauc_map_at_10_std": 0.08173000407846276, - "nauc_map_at_1_diff1": 0.6964868193650136, - "nauc_map_at_1_max": 0.4658840817263219, - "nauc_map_at_1_std": 0.06009948824657468, - "nauc_map_at_20_diff1": 0.6677072473838044, - "nauc_map_at_20_max": 0.4820334855898056, - "nauc_map_at_20_std": 0.08112000061939652, - "nauc_map_at_3_diff1": 0.6629064040013647, - "nauc_map_at_3_max": 0.47830764546568383, - "nauc_map_at_3_std": 0.07908104385148582, - "nauc_map_at_5_diff1": 0.6651150515375697, - "nauc_map_at_5_max": 0.48320424406293105, - "nauc_map_at_5_std": 0.0841376160310789, - "nauc_mrr_at_1000_diff1": 0.6681451778281712, - "nauc_mrr_at_1000_max": 0.48153054698905345, - "nauc_mrr_at_1000_std": 0.08084200415260853, - "nauc_mrr_at_100_diff1": 0.6681421969134242, - "nauc_mrr_at_100_max": 0.4815602459122235, - "nauc_mrr_at_100_std": 0.08089746305416494, - "nauc_mrr_at_10_diff1": 0.66637927944471, - "nauc_mrr_at_10_max": 0.4824993978185994, - "nauc_mrr_at_10_std": 0.08173000407846276, - "nauc_mrr_at_1_diff1": 0.6964868193650136, - "nauc_mrr_at_1_max": 0.4658840817263219, - "nauc_mrr_at_1_std": 0.06009948824657468, - "nauc_mrr_at_20_diff1": 0.6677072473838044, - "nauc_mrr_at_20_max": 0.4820334855898056, - "nauc_mrr_at_20_std": 0.08112000061939652, - "nauc_mrr_at_3_diff1": 0.6629064040013647, - "nauc_mrr_at_3_max": 0.47830764546568383, - "nauc_mrr_at_3_std": 0.07908104385148582, - "nauc_mrr_at_5_diff1": 0.6651150515375697, - "nauc_mrr_at_5_max": 0.48320424406293105, - "nauc_mrr_at_5_std": 0.0841376160310789, - "nauc_ndcg_at_1000_diff1": 0.6644414321594331, - "nauc_ndcg_at_1000_max": 0.48494330141081976, - "nauc_ndcg_at_1000_std": 0.08499549812045258, - "nauc_ndcg_at_100_diff1": 0.6643084643885205, - "nauc_ndcg_at_100_max": 0.4859086696461096, - "nauc_ndcg_at_100_std": 0.08683324562972554, - "nauc_ndcg_at_10_diff1": 0.6558132357644398, - "nauc_ndcg_at_10_max": 0.4912776808457643, - "nauc_ndcg_at_10_std": 0.09056604431255127, - "nauc_ndcg_at_1_diff1": 0.6964868193650136, - "nauc_ndcg_at_1_max": 0.4658840817263219, - "nauc_ndcg_at_1_std": 0.06009948824657468, - "nauc_ndcg_at_20_diff1": 0.6611087197877977, - "nauc_ndcg_at_20_max": 0.48954890256829625, - "nauc_ndcg_at_20_std": 0.08876502997875993, - "nauc_ndcg_at_3_diff1": 0.6501669512812934, - "nauc_ndcg_at_3_max": 0.4827750854554256, - "nauc_ndcg_at_3_std": 0.08585341799650138, - "nauc_ndcg_at_5_diff1": 0.6534210015801749, - "nauc_ndcg_at_5_max": 0.49293204109100786, - "nauc_ndcg_at_5_std": 0.09712640857173467, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.6346185533036952, - "nauc_precision_at_100_max": 0.6848052946668934, - "nauc_precision_at_100_std": 0.46640028560445, - "nauc_precision_at_10_diff1": 0.5758345691217664, - "nauc_precision_at_10_max": 0.5616378316032302, - "nauc_precision_at_10_std": 0.1567045641786131, - "nauc_precision_at_1_diff1": 0.6964868193650136, - "nauc_precision_at_1_max": 0.4658840817263219, - "nauc_precision_at_1_std": 0.06009948824657468, - "nauc_precision_at_20_diff1": 0.59426049622128, - "nauc_precision_at_20_max": 0.5904757914561855, - "nauc_precision_at_20_std": 0.18555826889160457, - "nauc_precision_at_3_diff1": 0.6016933324728304, - "nauc_precision_at_3_max": 0.49967144441250244, - "nauc_precision_at_3_std": 0.1114439040226548, - "nauc_precision_at_5_diff1": 0.5963268545815941, - "nauc_precision_at_5_max": 0.5435354775022362, - "nauc_precision_at_5_std": 0.16471325854088018, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.6346185533036849, - "nauc_recall_at_100_max": 0.6848052946668853, - "nauc_recall_at_100_std": 0.4664002856044359, - "nauc_recall_at_10_diff1": 0.5758345691217669, - "nauc_recall_at_10_max": 0.5616378316032299, - "nauc_recall_at_10_std": 0.15670456417861245, - "nauc_recall_at_1_diff1": 0.6964868193650136, - "nauc_recall_at_1_max": 0.4658840817263219, - "nauc_recall_at_1_std": 0.06009948824657468, - "nauc_recall_at_20_diff1": 0.5942604962212811, - "nauc_recall_at_20_max": 0.5904757914561845, - "nauc_recall_at_20_std": 0.18555826889159976, - "nauc_recall_at_3_diff1": 0.6016933324728311, - "nauc_recall_at_3_max": 0.4996714444125018, - "nauc_recall_at_3_std": 0.11144390402265514, - "nauc_recall_at_5_diff1": 0.596326854581594, - "nauc_recall_at_5_max": 0.5435354775022344, - "nauc_recall_at_5_std": 0.1647132585408802, - "ndcg_at_1": 0.6214, - "ndcg_at_10": 0.76748, - "ndcg_at_100": 0.78475, - "ndcg_at_1000": 0.78565, - "ndcg_at_20": 0.77717, - "ndcg_at_3": 0.7233, - "ndcg_at_5": 0.74763, - "precision_at_1": 0.6214, - "precision_at_10": 0.0915, - "precision_at_100": 0.00993, - "precision_at_1000": 0.001, - "precision_at_20": 0.04766, - "precision_at_3": 0.2648, - "precision_at_5": 0.1708, - "recall_at_1": 0.6214, - "recall_at_10": 0.915, - "recall_at_100": 0.9932, - "recall_at_1000": 1.0, - "recall_at_20": 0.9532, - "recall_at_3": 0.7944, - "recall_at_5": 0.854 - } - ] - }, - "task_name": "Flickr30kT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesI2TRetrieval.json deleted file mode 100644 index e7c0e0194b..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", - "evaluation_time": 13.294805765151978, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.343, - "cv_recall_at_10": 0.678, - "cv_recall_at_100": 0.798, - "cv_recall_at_1000": 0.886, - "cv_recall_at_20": 0.721, - "cv_recall_at_3": 0.557, - "cv_recall_at_5": 0.617, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.51779, - "map_at_1": 0.344, - "map_at_10": 0.46588, - "map_at_100": 0.47092, - "map_at_1000": 0.47125, - "map_at_20": 0.46902, - "map_at_3": 0.44583, - "map_at_5": 0.45763, - "mrr_at_1": 0.343, - "mrr_at_10": 0.46419642857142834, - "mrr_at_100": 0.469239410916577, - "mrr_at_1000": 0.4695660695365724, - "mrr_at_20": 0.4672927183884223, - "mrr_at_3": 0.4421666666666665, - "mrr_at_5": 0.4561666666666663, - "nauc_cv_recall_at_1000_diff1": 0.02351124577554979, - "nauc_cv_recall_at_1000_max": 0.6100895666505729, - "nauc_cv_recall_at_1000_std": 0.2871501822964356, - "nauc_cv_recall_at_100_diff1": 0.025343623449170807, - "nauc_cv_recall_at_100_max": 0.5897113711728056, - "nauc_cv_recall_at_100_std": 0.21345050979692376, - "nauc_cv_recall_at_10_diff1": 0.07681508015980319, - "nauc_cv_recall_at_10_max": 0.5990426933969243, - "nauc_cv_recall_at_10_std": 0.1857973138981508, - "nauc_cv_recall_at_1_diff1": 0.4267100244615221, - "nauc_cv_recall_at_1_max": 0.3618982353042344, - "nauc_cv_recall_at_1_std": 0.01272687102864061, - "nauc_cv_recall_at_20_diff1": 0.07174907627632691, - "nauc_cv_recall_at_20_max": 0.60599143151816, - "nauc_cv_recall_at_20_std": 0.19146863931819508, - "nauc_cv_recall_at_3_diff1": 0.15770090027886977, - "nauc_cv_recall_at_3_max": 0.547799106088024, - "nauc_cv_recall_at_3_std": 0.11460834402735758, - "nauc_cv_recall_at_5_diff1": 0.11310377114566393, - "nauc_cv_recall_at_5_max": 0.590776612134555, - "nauc_cv_recall_at_5_std": 0.1591778721919892, - "nauc_map_at_1000_diff1": 0.2931757212219033, - "nauc_map_at_1000_max": 0.47769835956247375, - "nauc_map_at_1000_std": 0.07142405784187013, - "nauc_map_at_100_diff1": 0.29296015626770694, - "nauc_map_at_100_max": 0.47772176532812277, - "nauc_map_at_100_std": 0.07134522164076948, - "nauc_map_at_10_diff1": 0.2922630116381693, - "nauc_map_at_10_max": 0.47864171579326814, - "nauc_map_at_10_std": 0.0714917138525951, - "nauc_map_at_1_diff1": 0.4251515375908431, - "nauc_map_at_1_max": 0.4001200664332334, - "nauc_map_at_1_std": 0.02200997556357167, - "nauc_map_at_20_diff1": 0.29275131026457724, - "nauc_map_at_20_max": 0.47842933833844037, - "nauc_map_at_20_std": 0.0715990326462406, - "nauc_map_at_3_diff1": 0.2996338562673293, - "nauc_map_at_3_max": 0.4770168797044735, - "nauc_map_at_3_std": 0.06405282046759102, - "nauc_map_at_5_diff1": 0.2923679786791551, - "nauc_map_at_5_max": 0.47682673138378284, - "nauc_map_at_5_std": 0.06873373261928388, - "nauc_mrr_at_1000_diff1": 0.294675550005923, - "nauc_mrr_at_1000_max": 0.4532811868558267, - "nauc_mrr_at_1000_std": 0.06531584821645785, - "nauc_mrr_at_100_diff1": 0.2944595453301148, - "nauc_mrr_at_100_max": 0.45332600827827807, - "nauc_mrr_at_100_std": 0.06524317581183853, - "nauc_mrr_at_10_diff1": 0.2927959210952178, - "nauc_mrr_at_10_max": 0.45399459720760665, - "nauc_mrr_at_10_std": 0.06574167283682725, - "nauc_mrr_at_1_diff1": 0.4267100244615221, - "nauc_mrr_at_1_max": 0.3618982353042344, - "nauc_mrr_at_1_std": 0.01272687102864061, - "nauc_mrr_at_20_diff1": 0.29404544778933245, - "nauc_mrr_at_20_max": 0.4541045696997424, - "nauc_mrr_at_20_std": 0.06560225616529214, - "nauc_mrr_at_3_diff1": 0.3017349546285236, - "nauc_mrr_at_3_max": 0.44735428178122444, - "nauc_mrr_at_3_std": 0.05620321719877866, - "nauc_mrr_at_5_diff1": 0.2944869030132802, - "nauc_mrr_at_5_max": 0.4544780453588209, - "nauc_mrr_at_5_std": 0.06394477273129469, - "nauc_ndcg_at_1000_diff1": 0.25563741681319463, - "nauc_ndcg_at_1000_max": 0.49813064292479103, - "nauc_ndcg_at_1000_std": 0.0945483999782709, - "nauc_ndcg_at_100_diff1": 0.2487088648098237, - "nauc_ndcg_at_100_max": 0.4997081881373343, - "nauc_ndcg_at_100_std": 0.09413719831820086, - "nauc_ndcg_at_10_diff1": 0.24794669752558104, - "nauc_ndcg_at_10_max": 0.506417052208621, - "nauc_ndcg_at_10_std": 0.09468363744731645, - "nauc_ndcg_at_1_diff1": 0.4251515375908431, - "nauc_ndcg_at_1_max": 0.4001200664332334, - "nauc_ndcg_at_1_std": 0.02200997556357167, - "nauc_ndcg_at_20_diff1": 0.24876495274829946, - "nauc_ndcg_at_20_max": 0.5049386724625328, - "nauc_ndcg_at_20_std": 0.09476708189800392, - "nauc_ndcg_at_3_diff1": 0.2628548139798052, - "nauc_ndcg_at_3_max": 0.5008682261479349, - "nauc_ndcg_at_3_std": 0.07843376700906236, - "nauc_ndcg_at_5_diff1": 0.24906238387697463, - "nauc_ndcg_at_5_max": 0.5015432023723324, - "nauc_ndcg_at_5_std": 0.08771207964905081, - "nauc_precision_at_1000_diff1": 0.02351124577554748, - "nauc_precision_at_1000_max": 0.6100895666505727, - "nauc_precision_at_1000_std": 0.2871501822964333, - "nauc_precision_at_100_diff1": 0.02534362344917089, - "nauc_precision_at_100_max": 0.5897113711728055, - "nauc_precision_at_100_std": 0.2134505097969231, - "nauc_precision_at_10_diff1": 0.09074798611182705, - "nauc_precision_at_10_max": 0.6065579975706014, - "nauc_precision_at_10_std": 0.18167834911065367, - "nauc_precision_at_1_diff1": 0.4251515375908431, - "nauc_precision_at_1_max": 0.4001200664332334, - "nauc_precision_at_1_std": 0.02200997556357167, - "nauc_precision_at_20_diff1": 0.07669661808676746, - "nauc_precision_at_20_max": 0.6072858332725597, - "nauc_precision_at_20_std": 0.18975907082787946, - "nauc_precision_at_3_diff1": 0.15301044037993555, - "nauc_precision_at_3_max": 0.572586687980361, - "nauc_precision_at_3_std": 0.12214031356821216, - "nauc_precision_at_5_diff1": 0.11060646785011727, - "nauc_precision_at_5_max": 0.5803579064397495, - "nauc_precision_at_5_std": 0.15018295439273746, - "nauc_recall_at_1000_diff1": 0.02351124577554979, - "nauc_recall_at_1000_max": 0.6100895666505729, - "nauc_recall_at_1000_std": 0.2871501822964356, - "nauc_recall_at_100_diff1": 0.025343623449170807, - "nauc_recall_at_100_max": 0.5897113711728056, - "nauc_recall_at_100_std": 0.21345050979692376, - "nauc_recall_at_10_diff1": 0.09074798611182784, - "nauc_recall_at_10_max": 0.6065579975706022, - "nauc_recall_at_10_std": 0.18167834911065472, - "nauc_recall_at_1_diff1": 0.4251515375908431, - "nauc_recall_at_1_max": 0.4001200664332334, - "nauc_recall_at_1_std": 0.02200997556357167, - "nauc_recall_at_20_diff1": 0.07669661808676843, - "nauc_recall_at_20_max": 0.6072858332725608, - "nauc_recall_at_20_std": 0.18975907082788046, - "nauc_recall_at_3_diff1": 0.15301044037993589, - "nauc_recall_at_3_max": 0.5725866879803613, - "nauc_recall_at_3_std": 0.12214031356821241, - "nauc_recall_at_5_diff1": 0.11060646785011774, - "nauc_recall_at_5_max": 0.5803579064397502, - "nauc_recall_at_5_std": 0.1501829543927384, - "ndcg_at_1": 0.344, - "ndcg_at_10": 0.51779, - "ndcg_at_100": 0.54301, - "ndcg_at_1000": 0.55369, - "ndcg_at_20": 0.52901, - "ndcg_at_3": 0.47663, - "ndcg_at_5": 0.49776, - "precision_at_1": 0.344, - "precision_at_10": 0.0678, - "precision_at_100": 0.00798, - "precision_at_1000": 0.00089, - "precision_at_20": 0.0361, - "precision_at_3": 0.18833, - "precision_at_5": 0.1232, - "recall_at_1": 0.344, - "recall_at_10": 0.678, - "recall_at_100": 0.798, - "recall_at_1000": 0.886, - "recall_at_20": 0.722, - "recall_at_3": 0.565, - "recall_at_5": 0.616 - } - ] - }, - "task_name": "HatefulMemesI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesT2IRetrieval.json deleted file mode 100644 index 8542a96e83..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/HatefulMemesT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", - "evaluation_time": 64.34664630889893, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.402, - "cv_recall_at_10": 0.706, - "cv_recall_at_100": 0.817, - "cv_recall_at_1000": 0.883, - "cv_recall_at_20": 0.754, - "cv_recall_at_3": 0.593, - "cv_recall_at_5": 0.649, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.55779, - "map_at_1": 0.402, - "map_at_10": 0.50986, - "map_at_100": 0.51494, - "map_at_1000": 0.51523, - "map_at_20": 0.5134, - "map_at_3": 0.48833, - "map_at_5": 0.50178, - "mrr_at_1": 0.402, - "mrr_at_10": 0.5100047619047616, - "mrr_at_100": 0.5150915951115426, - "mrr_at_1000": 0.5153729730152096, - "mrr_at_20": 0.5135412349475424, - "mrr_at_3": 0.4889999999999997, - "mrr_at_5": 0.5019499999999996, - "nauc_cv_recall_at_1000_diff1": 0.25842166489054547, - "nauc_cv_recall_at_1000_max": 0.2840669665736568, - "nauc_cv_recall_at_1000_std": 0.3131370650874093, - "nauc_cv_recall_at_100_diff1": 0.2965754204791379, - "nauc_cv_recall_at_100_max": 0.3454573066555303, - "nauc_cv_recall_at_100_std": 0.2617113134786554, - "nauc_cv_recall_at_10_diff1": 0.36610371955486987, - "nauc_cv_recall_at_10_max": 0.39744200447768313, - "nauc_cv_recall_at_10_std": 0.18884133539018538, - "nauc_cv_recall_at_1_diff1": 0.4244055578313841, - "nauc_cv_recall_at_1_max": 0.23928406722368636, - "nauc_cv_recall_at_1_std": 0.043300432579218215, - "nauc_cv_recall_at_20_diff1": 0.3654705001675286, - "nauc_cv_recall_at_20_max": 0.39659048014895193, - "nauc_cv_recall_at_20_std": 0.19959623548971286, - "nauc_cv_recall_at_3_diff1": 0.4249958405050879, - "nauc_cv_recall_at_3_max": 0.37439871372221056, - "nauc_cv_recall_at_3_std": 0.11358881221021569, - "nauc_cv_recall_at_5_diff1": 0.4179313459067598, - "nauc_cv_recall_at_5_max": 0.40265448632879425, - "nauc_cv_recall_at_5_std": 0.1370580390717256, - "nauc_map_at_1000_diff1": 0.421562102725268, - "nauc_map_at_1000_max": 0.3036341729007964, - "nauc_map_at_1000_std": 0.08116520872126745, - "nauc_map_at_100_diff1": 0.4216137241609162, - "nauc_map_at_100_max": 0.30386546636442807, - "nauc_map_at_100_std": 0.0812544070904664, - "nauc_map_at_10_diff1": 0.4217182579707922, - "nauc_map_at_10_max": 0.3052606025415569, - "nauc_map_at_10_std": 0.08084695393081556, - "nauc_map_at_1_diff1": 0.4244055578313841, - "nauc_map_at_1_max": 0.23928406722368636, - "nauc_map_at_1_std": 0.043300432579218215, - "nauc_map_at_20_diff1": 0.42240694310701743, - "nauc_map_at_20_max": 0.3048275987004854, - "nauc_map_at_20_std": 0.0807437517104306, - "nauc_map_at_3_diff1": 0.4290903919098365, - "nauc_map_at_3_max": 0.3027723173840437, - "nauc_map_at_3_std": 0.07219734359468977, - "nauc_map_at_5_diff1": 0.42728854397252497, - "nauc_map_at_5_max": 0.30693836093800736, - "nauc_map_at_5_std": 0.07621156582526052, - "nauc_mrr_at_1000_diff1": 0.4211232600709866, - "nauc_mrr_at_1000_max": 0.30339937083832536, - "nauc_mrr_at_1000_std": 0.08097110091447081, - "nauc_mrr_at_100_diff1": 0.42117489295099, - "nauc_mrr_at_100_max": 0.3036308757904552, - "nauc_mrr_at_100_std": 0.08106111681094921, - "nauc_mrr_at_10_diff1": 0.4212911582115076, - "nauc_mrr_at_10_max": 0.3050328316378303, - "nauc_mrr_at_10_std": 0.08065462135727981, - "nauc_mrr_at_1_diff1": 0.4244055578313841, - "nauc_mrr_at_1_max": 0.23928406722368636, - "nauc_mrr_at_1_std": 0.043300432579218215, - "nauc_mrr_at_20_diff1": 0.42197662326971863, - "nauc_mrr_at_20_max": 0.30459799740710386, - "nauc_mrr_at_20_std": 0.08054991063228273, - "nauc_mrr_at_3_diff1": 0.4275714649549671, - "nauc_mrr_at_3_max": 0.3015989754499319, - "nauc_mrr_at_3_std": 0.07178001954853169, - "nauc_mrr_at_5_diff1": 0.4268594262944631, - "nauc_mrr_at_5_max": 0.3066866998974272, - "nauc_mrr_at_5_std": 0.07610582583795289, - "nauc_ndcg_at_1000_diff1": 0.4074447487238973, - "nauc_ndcg_at_1000_max": 0.31125512247814136, - "nauc_ndcg_at_1000_std": 0.10429888184923936, - "nauc_ndcg_at_100_diff1": 0.4071583158406819, - "nauc_ndcg_at_100_max": 0.3157770171610164, - "nauc_ndcg_at_100_std": 0.10582027042777999, - "nauc_ndcg_at_10_diff1": 0.41172173847499366, - "nauc_ndcg_at_10_max": 0.3252162868702745, - "nauc_ndcg_at_10_std": 0.10206715227531686, - "nauc_ndcg_at_1_diff1": 0.4244055578313841, - "nauc_ndcg_at_1_max": 0.23928406722368636, - "nauc_ndcg_at_1_std": 0.043300432579218215, - "nauc_ndcg_at_20_diff1": 0.4133242918402585, - "nauc_ndcg_at_20_max": 0.3233602336017414, - "nauc_ndcg_at_20_std": 0.10179080131066064, - "nauc_ndcg_at_3_diff1": 0.42939665072299477, - "nauc_ndcg_at_3_max": 0.32131749101164025, - "nauc_ndcg_at_3_std": 0.08252550533650273, - "nauc_ndcg_at_5_diff1": 0.42550410028071456, - "nauc_ndcg_at_5_max": 0.3289520097105336, - "nauc_ndcg_at_5_std": 0.08989807277937249, - "nauc_precision_at_1000_diff1": 0.2584216648905424, - "nauc_precision_at_1000_max": 0.28406696657365527, - "nauc_precision_at_1000_std": 0.313137065087408, - "nauc_precision_at_100_diff1": 0.2965754204791373, - "nauc_precision_at_100_max": 0.3454573066555294, - "nauc_precision_at_100_std": 0.26171131347865473, - "nauc_precision_at_10_diff1": 0.3661037195548693, - "nauc_precision_at_10_max": 0.3974420044776824, - "nauc_precision_at_10_std": 0.18884133539018413, - "nauc_precision_at_1_diff1": 0.4244055578313841, - "nauc_precision_at_1_max": 0.23928406722368636, - "nauc_precision_at_1_std": 0.043300432579218215, - "nauc_precision_at_20_diff1": 0.36547050016752725, - "nauc_precision_at_20_max": 0.3965904801489509, - "nauc_precision_at_20_std": 0.19959623548971295, - "nauc_precision_at_3_diff1": 0.4299202442181095, - "nauc_precision_at_3_max": 0.378034179370445, - "nauc_precision_at_3_std": 0.114845975410715, - "nauc_precision_at_5_diff1": 0.41793134590675957, - "nauc_precision_at_5_max": 0.4026544863287947, - "nauc_precision_at_5_std": 0.13705803907172523, - "nauc_recall_at_1000_diff1": 0.25842166489054547, - "nauc_recall_at_1000_max": 0.2840669665736568, - "nauc_recall_at_1000_std": 0.3131370650874093, - "nauc_recall_at_100_diff1": 0.2965754204791379, - "nauc_recall_at_100_max": 0.3454573066555303, - "nauc_recall_at_100_std": 0.2617113134786554, - "nauc_recall_at_10_diff1": 0.36610371955486987, - "nauc_recall_at_10_max": 0.39744200447768313, - "nauc_recall_at_10_std": 0.18884133539018538, - "nauc_recall_at_1_diff1": 0.4244055578313841, - "nauc_recall_at_1_max": 0.23928406722368636, - "nauc_recall_at_1_std": 0.043300432579218215, - "nauc_recall_at_20_diff1": 0.3654705001675286, - "nauc_recall_at_20_max": 0.39659048014895193, - "nauc_recall_at_20_std": 0.19959623548971286, - "nauc_recall_at_3_diff1": 0.42992024421810965, - "nauc_recall_at_3_max": 0.3780341793704453, - "nauc_recall_at_3_std": 0.11484597541071496, - "nauc_recall_at_5_diff1": 0.4179313459067598, - "nauc_recall_at_5_max": 0.40265448632879425, - "nauc_recall_at_5_std": 0.1370580390717256, - "ndcg_at_1": 0.402, - "ndcg_at_10": 0.55779, - "ndcg_at_100": 0.58173, - "ndcg_at_1000": 0.58994, - "ndcg_at_20": 0.57019, - "ndcg_at_3": 0.51483, - "ndcg_at_5": 0.53889, - "precision_at_1": 0.402, - "precision_at_10": 0.0706, - "precision_at_100": 0.00817, - "precision_at_1000": 0.00088, - "precision_at_20": 0.0377, - "precision_at_3": 0.197, - "precision_at_5": 0.1298, - "recall_at_1": 0.402, - "recall_at_10": 0.706, - "recall_at_100": 0.817, - "recall_at_1000": 0.883, - "recall_at_20": 0.754, - "recall_at_3": 0.591, - "recall_at_5": 0.649 - } - ] - }, - "task_name": "HatefulMemesT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOI2TRetrieval.json deleted file mode 100644 index d2ffdd60cd..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOI2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", - "evaluation_time": 35.03174376487732, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.46627, - "map_at_1": 0.10555, - "map_at_10": 0.35174, - "map_at_100": 0.39557, - "map_at_1000": 0.39923, - "map_at_20": 0.37565, - "map_at_3": 0.2358, - "map_at_5": 0.30315, - "mrr_at_1": 0.5274, - "mrr_at_10": 0.6307395238095252, - "mrr_at_100": 0.6373119196326027, - "mrr_at_1000": 0.6373802558284773, - "mrr_at_20": 0.6352388099361947, - "mrr_at_3": 0.6044666666666684, - "mrr_at_5": 0.6191566666666672, - "nauc_map_at_1000_diff1": 0.2869664571536092, - "nauc_map_at_1000_max": 0.3837078202810147, - "nauc_map_at_1000_std": -0.3159503626026489, - "nauc_map_at_100_diff1": 0.28703582233761415, - "nauc_map_at_100_max": 0.38340512757594436, - "nauc_map_at_100_std": -0.31691519501613774, - "nauc_map_at_10_diff1": 0.28144600872574277, - "nauc_map_at_10_max": 0.3815317371289302, - "nauc_map_at_10_std": -0.3130948877233665, - "nauc_map_at_1_diff1": 0.42136443963654857, - "nauc_map_at_1_max": 0.34593517564893767, - "nauc_map_at_1_std": -0.3210828465931971, - "nauc_map_at_20_diff1": 0.2839400296739511, - "nauc_map_at_20_max": 0.38270639817217694, - "nauc_map_at_20_std": -0.31693403230799216, - "nauc_map_at_3_diff1": 0.29969868166405594, - "nauc_map_at_3_max": 0.36804745199570993, - "nauc_map_at_3_std": -0.3005060420974389, - "nauc_map_at_5_diff1": 0.28066502811464294, - "nauc_map_at_5_max": 0.38500675433867, - "nauc_map_at_5_std": -0.30277657476472214, - "nauc_mrr_at_1000_diff1": 0.41411392018539944, - "nauc_mrr_at_1000_max": 0.34589612969127426, - "nauc_mrr_at_1000_std": -0.3570352162287009, - "nauc_mrr_at_100_diff1": 0.41413003302489404, - "nauc_mrr_at_100_max": 0.34588621354537913, - "nauc_mrr_at_100_std": -0.3571316999051599, - "nauc_mrr_at_10_diff1": 0.4136538823599215, - "nauc_mrr_at_10_max": 0.34527376482746647, - "nauc_mrr_at_10_std": -0.3592210063673948, - "nauc_mrr_at_1_diff1": 0.42218213910596014, - "nauc_mrr_at_1_max": 0.3463699419397979, - "nauc_mrr_at_1_std": -0.32142245164460015, - "nauc_mrr_at_20_diff1": 0.4140225618106111, - "nauc_mrr_at_20_max": 0.34565328848624477, - "nauc_mrr_at_20_std": -0.357990582740709, - "nauc_mrr_at_3_diff1": 0.41140479122082163, - "nauc_mrr_at_3_max": 0.3468454677490499, - "nauc_mrr_at_3_std": -0.356442538497874, - "nauc_mrr_at_5_diff1": 0.41259544348162186, - "nauc_mrr_at_5_max": 0.34704678498973585, - "nauc_mrr_at_5_std": -0.356703631914488, - "nauc_ndcg_at_1000_diff1": 0.3247556576373652, - "nauc_ndcg_at_1000_max": 0.376393317989791, - "nauc_ndcg_at_1000_std": -0.3324974788562371, - "nauc_ndcg_at_100_diff1": 0.3217721326814981, - "nauc_ndcg_at_100_max": 0.37324015047062437, - "nauc_ndcg_at_100_std": -0.3387959081771918, - "nauc_ndcg_at_10_diff1": 0.3087600262995678, - "nauc_ndcg_at_10_max": 0.36991072734632663, - "nauc_ndcg_at_10_std": -0.3387282627094938, - "nauc_ndcg_at_1_diff1": 0.42218213910596014, - "nauc_ndcg_at_1_max": 0.3463699419397979, - "nauc_ndcg_at_1_std": -0.32142245164460015, - "nauc_ndcg_at_20_diff1": 0.31310134298112313, - "nauc_ndcg_at_20_max": 0.3711638446713194, - "nauc_ndcg_at_20_std": -0.3446387196717444, - "nauc_ndcg_at_3_diff1": 0.32234088999598043, - "nauc_ndcg_at_3_max": 0.3637234182783304, - "nauc_ndcg_at_3_std": -0.3199344660388001, - "nauc_ndcg_at_5_diff1": 0.3087695940641351, - "nauc_ndcg_at_5_max": 0.3752423132579829, - "nauc_ndcg_at_5_std": -0.3254191806337329, - "nauc_precision_at_1000_diff1": 0.0318067226890696, - "nauc_precision_at_1000_max": 0.3495751633986902, - "nauc_precision_at_1000_std": 0.5565662931839553, - "nauc_precision_at_100_diff1": 0.2388655041840271, - "nauc_precision_at_100_max": 0.3006890192309006, - "nauc_precision_at_100_std": -0.26789110526496845, - "nauc_precision_at_10_diff1": 0.25885428174045677, - "nauc_precision_at_10_max": 0.3465562874468837, - "nauc_precision_at_10_std": -0.32815618004174235, - "nauc_precision_at_1_diff1": 0.42218213910596014, - "nauc_precision_at_1_max": 0.3463699419397979, - "nauc_precision_at_1_std": -0.32142245164460015, - "nauc_precision_at_20_diff1": 0.25349909583287034, - "nauc_precision_at_20_max": 0.33188675929576117, - "nauc_precision_at_20_std": -0.3268395370923763, - "nauc_precision_at_3_diff1": 0.2892533844448195, - "nauc_precision_at_3_max": 0.36389248373378874, - "nauc_precision_at_3_std": -0.3153927469706968, - "nauc_precision_at_5_diff1": 0.2692072552423276, - "nauc_precision_at_5_max": 0.37322473714076376, - "nauc_precision_at_5_std": -0.3176984049769347, - "nauc_recall_at_1000_diff1": 0.023861396410433965, - "nauc_recall_at_1000_max": 0.3630736245115491, - "nauc_recall_at_1000_std": 0.5760007706786342, - "nauc_recall_at_100_diff1": 0.23875325449572146, - "nauc_recall_at_100_max": 0.30099776552287444, - "nauc_recall_at_100_std": -0.2687324892664122, - "nauc_recall_at_10_diff1": 0.25859827210476144, - "nauc_recall_at_10_max": 0.3464336452165927, - "nauc_recall_at_10_std": -0.32824547353711414, - "nauc_recall_at_1_diff1": 0.42136443963654857, - "nauc_recall_at_1_max": 0.34593517564893767, - "nauc_recall_at_1_std": -0.3210828465931971, - "nauc_recall_at_20_diff1": 0.2532204432312222, - "nauc_recall_at_20_max": 0.3318583274471809, - "nauc_recall_at_20_std": -0.3269518502721138, - "nauc_recall_at_3_diff1": 0.28903591577358717, - "nauc_recall_at_3_max": 0.3637400527142261, - "nauc_recall_at_3_std": -0.31529036662690074, - "nauc_recall_at_5_diff1": 0.2691830355517802, - "nauc_recall_at_5_max": 0.3732658127670549, - "nauc_recall_at_5_std": -0.31779144735485815, - "ndcg_at_1": 0.5274, - "ndcg_at_10": 0.46627, - "ndcg_at_100": 0.58544, - "ndcg_at_1000": 0.62262, - "ndcg_at_20": 0.51262, - "ndcg_at_3": 0.45724, - "ndcg_at_5": 0.4002, - "precision_at_1": 0.5274, - "precision_at_10": 0.24096, - "precision_at_100": 0.04117, - "precision_at_1000": 0.00496, - "precision_at_20": 0.14745, - "precision_at_3": 0.43627, - "precision_at_5": 0.36168, - "recall_at_1": 0.10555, - "recall_at_10": 0.48216, - "recall_at_100": 0.82383, - "recall_at_1000": 0.99244, - "recall_at_20": 0.59011, - "recall_at_3": 0.2619, - "recall_at_5": 0.36183 - } - ] - }, - "task_name": "MSCOCOI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOT2IRetrieval.json deleted file mode 100644 index caa9409611..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MSCOCOT2IRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", - "evaluation_time": 59.44574952125549, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.51224, - "map_at_1": 0.34778, - "map_at_10": 0.45378, - "map_at_100": 0.46526, - "map_at_1000": 0.46547, - "map_at_20": 0.4607, - "map_at_3": 0.42214, - "map_at_5": 0.43988, - "mrr_at_1": 0.3480591720746503, - "mrr_at_10": 0.4542383812326254, - "mrr_at_100": 0.46563385013928266, - "mrr_at_1000": 0.4658408234293717, - "mrr_at_20": 0.46113137131638215, - "mrr_at_3": 0.4225818587340632, - "mrr_at_5": 0.44035565050318304, - "nauc_map_at_1000_diff1": 0.5300274380820644, - "nauc_map_at_1000_max": 0.2559436002804574, - "nauc_map_at_1000_std": -0.2706192946414834, - "nauc_map_at_100_diff1": 0.5299324120035608, - "nauc_map_at_100_max": 0.2560347704220875, - "nauc_map_at_100_std": -0.27045953970788833, - "nauc_map_at_10_diff1": 0.5278510109426034, - "nauc_map_at_10_max": 0.2539268152655037, - "nauc_map_at_10_std": -0.27321216910868745, - "nauc_map_at_1_diff1": 0.5783541250432945, - "nauc_map_at_1_max": 0.2695828538669502, - "nauc_map_at_1_std": -0.2559433871779334, - "nauc_map_at_20_diff1": 0.5286608424283533, - "nauc_map_at_20_max": 0.25510619043899635, - "nauc_map_at_20_std": -0.27209215189802183, - "nauc_map_at_3_diff1": 0.5324437628964646, - "nauc_map_at_3_max": 0.2553342754483297, - "nauc_map_at_3_std": -0.27130950074984744, - "nauc_map_at_5_diff1": 0.5284529389089929, - "nauc_map_at_5_max": 0.25477174918600354, - "nauc_map_at_5_std": -0.27190725659741016, - "nauc_mrr_at_1000_diff1": 0.5293645653185072, - "nauc_mrr_at_1000_max": 0.2557502693986946, - "nauc_mrr_at_1000_std": -0.26978550875221907, - "nauc_mrr_at_100_diff1": 0.5292685848719979, - "nauc_mrr_at_100_max": 0.25584012455335126, - "nauc_mrr_at_100_std": -0.26961557231523003, - "nauc_mrr_at_10_diff1": 0.5271255164882128, - "nauc_mrr_at_10_max": 0.25379553348651324, - "nauc_mrr_at_10_std": -0.2722279666262872, - "nauc_mrr_at_1_diff1": 0.5776274975462988, - "nauc_mrr_at_1_max": 0.2692774729298938, - "nauc_mrr_at_1_std": -0.25521001402599525, - "nauc_mrr_at_20_diff1": 0.5279751430379428, - "nauc_mrr_at_20_max": 0.2549509074476351, - "nauc_mrr_at_20_std": -0.27112013741544844, - "nauc_mrr_at_3_diff1": 0.5316046890057915, - "nauc_mrr_at_3_max": 0.255056504420029, - "nauc_mrr_at_3_std": -0.27035519360749666, - "nauc_mrr_at_5_diff1": 0.5276492399505774, - "nauc_mrr_at_5_max": 0.25460202269239396, - "nauc_mrr_at_5_std": -0.2708629718337636, - "nauc_ndcg_at_1000_diff1": 0.5212393417971389, - "nauc_ndcg_at_1000_max": 0.2556420055322196, - "nauc_ndcg_at_1000_std": -0.2691695716908475, - "nauc_ndcg_at_100_diff1": 0.5190594914114796, - "nauc_ndcg_at_100_max": 0.2587996367020589, - "nauc_ndcg_at_100_std": -0.26338010950995205, - "nauc_ndcg_at_10_diff1": 0.5074477844063786, - "nauc_ndcg_at_10_max": 0.2469669155787985, - "nauc_ndcg_at_10_std": -0.2809472768896655, - "nauc_ndcg_at_1_diff1": 0.5776274975462988, - "nauc_ndcg_at_1_max": 0.2692774729298938, - "nauc_ndcg_at_1_std": -0.25521001402599525, - "nauc_ndcg_at_20_diff1": 0.5095606734495621, - "nauc_ndcg_at_20_max": 0.25133476467547433, - "nauc_ndcg_at_20_std": -0.27655628582798436, - "nauc_ndcg_at_3_diff1": 0.517781963819374, - "nauc_ndcg_at_3_max": 0.2506704219638103, - "nauc_ndcg_at_3_std": -0.2758702300013848, - "nauc_ndcg_at_5_diff1": 0.5099993994676364, - "nauc_ndcg_at_5_max": 0.2496258742411529, - "nauc_ndcg_at_5_std": -0.27713728292552275, - "nauc_precision_at_1000_diff1": -0.1842058861760832, - "nauc_precision_at_1000_max": 0.09117367390652957, - "nauc_precision_at_1000_std": 0.5620222894753265, - "nauc_precision_at_100_diff1": 0.23210239318675868, - "nauc_precision_at_100_max": 0.37569992320172535, - "nauc_precision_at_100_std": 0.2577714899683604, - "nauc_precision_at_10_diff1": 0.4239940327698815, - "nauc_precision_at_10_max": 0.21854025050104828, - "nauc_precision_at_10_std": -0.3069601314710065, - "nauc_precision_at_1_diff1": 0.5776274975462988, - "nauc_precision_at_1_max": 0.2692774729298938, - "nauc_precision_at_1_std": -0.25521001402599525, - "nauc_precision_at_20_diff1": 0.40073513220011464, - "nauc_precision_at_20_max": 0.2340653951866781, - "nauc_precision_at_20_std": -0.28105232486858844, - "nauc_precision_at_3_diff1": 0.4743236776776763, - "nauc_precision_at_3_max": 0.2369305838402195, - "nauc_precision_at_3_std": -0.2890312678079432, - "nauc_precision_at_5_diff1": 0.44949845794106563, - "nauc_precision_at_5_max": 0.2332288298110243, - "nauc_precision_at_5_std": -0.29233274328970654, - "nauc_recall_at_1000_diff1": 0.5481755072485809, - "nauc_recall_at_1000_max": 0.8579274209326658, - "nauc_recall_at_1000_std": 0.8989772097175315, - "nauc_recall_at_100_diff1": 0.39432018640739835, - "nauc_recall_at_100_max": 0.4852196943081552, - "nauc_recall_at_100_std": 0.16026301353344588, - "nauc_recall_at_10_diff1": 0.4289710956502324, - "nauc_recall_at_10_max": 0.21894642121454272, - "nauc_recall_at_10_std": -0.31340659744516125, - "nauc_recall_at_1_diff1": 0.5783541250432945, - "nauc_recall_at_1_max": 0.2695828538669502, - "nauc_recall_at_1_std": -0.2559433871779334, - "nauc_recall_at_20_diff1": 0.41279018466409206, - "nauc_recall_at_20_max": 0.23609265389037357, - "nauc_recall_at_20_std": -0.2972785442737482, - "nauc_recall_at_3_diff1": 0.4754634956167504, - "nauc_recall_at_3_max": 0.23707374848594062, - "nauc_recall_at_3_std": -0.2902811277316357, - "nauc_recall_at_5_diff1": 0.4515520952519273, - "nauc_recall_at_5_max": 0.23340217997407925, - "nauc_recall_at_5_std": -0.2947192867661317, - "ndcg_at_1": 0.34806, - "ndcg_at_10": 0.51224, - "ndcg_at_100": 0.56914, - "ndcg_at_1000": 0.57354, - "ndcg_at_20": 0.53735, - "ndcg_at_3": 0.44663, - "ndcg_at_5": 0.47858, - "precision_at_1": 0.34806, - "precision_at_10": 0.07004, - "precision_at_100": 0.00972, - "precision_at_1000": 0.00101, - "precision_at_20": 0.04002, - "precision_at_3": 0.17259, - "precision_at_5": 0.11915, - "recall_at_1": 0.34778, - "recall_at_10": 0.69854, - "recall_at_100": 0.96606, - "recall_at_1000": 0.99836, - "recall_at_20": 0.79752, - "recall_at_3": 0.51703, - "recall_at_5": 0.59457 - } - ] - }, - "task_name": "MSCOCOT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionI2TRetrieval.json deleted file mode 100644 index 4571e3c67d..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", - "evaluation_time": 10.613081693649292, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.6198, - "cv_recall_at_10": 0.82496, - "cv_recall_at_100": 0.94118, - "cv_recall_at_1000": 0.98852, - "cv_recall_at_20": 0.87088, - "cv_recall_at_3": 0.74892, - "cv_recall_at_5": 0.7891, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.72207, - "map_at_1": 0.61693, - "map_at_10": 0.68889, - "map_at_100": 0.69395, - "map_at_1000": 0.69417, - "map_at_20": 0.69209, - "map_at_3": 0.6748, - "map_at_5": 0.68412, - "mrr_at_1": 0.6197991391678622, - "mrr_at_10": 0.6908155131971484, - "mrr_at_100": 0.6958826981617491, - "mrr_at_1000": 0.6960943232255445, - "mrr_at_20": 0.6940146437225909, - "mrr_at_3": 0.676709708273553, - "mrr_at_5": 0.686035389765662, - "nauc_cv_recall_at_1000_diff1": 0.7115337193995271, - "nauc_cv_recall_at_1000_max": 0.9162400747714905, - "nauc_cv_recall_at_1000_std": 0.6394650706109806, - "nauc_cv_recall_at_100_diff1": 0.7333454397110322, - "nauc_cv_recall_at_100_max": 0.7444349099565744, - "nauc_cv_recall_at_100_std": 0.4724436931961091, - "nauc_cv_recall_at_10_diff1": 0.6871809913928805, - "nauc_cv_recall_at_10_max": 0.6133230953694784, - "nauc_cv_recall_at_10_std": 0.2527224723281998, - "nauc_cv_recall_at_1_diff1": 0.8178148233019549, - "nauc_cv_recall_at_1_max": 0.6052419836559261, - "nauc_cv_recall_at_1_std": 0.12987371517585286, - "nauc_cv_recall_at_20_diff1": 0.6981768420463863, - "nauc_cv_recall_at_20_max": 0.6659441432074131, - "nauc_cv_recall_at_20_std": 0.3204684923003592, - "nauc_cv_recall_at_3_diff1": 0.7321559190348417, - "nauc_cv_recall_at_3_max": 0.6343014239279325, - "nauc_cv_recall_at_3_std": 0.15185156111493697, - "nauc_cv_recall_at_5_diff1": 0.7072557341340876, - "nauc_cv_recall_at_5_max": 0.6257250342242687, - "nauc_cv_recall_at_5_std": 0.20404936155319015, - "nauc_map_at_1000_diff1": 0.7887351342678683, - "nauc_map_at_1000_max": 0.6168942239184106, - "nauc_map_at_1000_std": 0.1413630951928354, - "nauc_map_at_100_diff1": 0.7887041995063825, - "nauc_map_at_100_max": 0.6170199486176621, - "nauc_map_at_100_std": 0.1416616343081758, - "nauc_map_at_10_diff1": 0.7862784183796454, - "nauc_map_at_10_max": 0.6146388906025935, - "nauc_map_at_10_std": 0.14075448379320352, - "nauc_map_at_1_diff1": 0.8229562595133094, - "nauc_map_at_1_max": 0.6013677818784777, - "nauc_map_at_1_std": 0.12160003122737216, - "nauc_map_at_20_diff1": 0.7879869553581316, - "nauc_map_at_20_max": 0.6168049743962634, - "nauc_map_at_20_std": 0.14202953814505015, - "nauc_map_at_3_diff1": 0.790122035841549, - "nauc_map_at_3_max": 0.6181113964938436, - "nauc_map_at_3_std": 0.12999774139755105, - "nauc_map_at_5_diff1": 0.7867973534260535, - "nauc_map_at_5_max": 0.6154950412048648, - "nauc_map_at_5_std": 0.13770782443201052, - "nauc_mrr_at_1000_diff1": 0.7849851923349892, - "nauc_mrr_at_1000_max": 0.6191346898024174, - "nauc_mrr_at_1000_std": 0.14684978660096312, - "nauc_mrr_at_100_diff1": 0.7849572405285399, - "nauc_mrr_at_100_max": 0.6192591195776083, - "nauc_mrr_at_100_std": 0.1471451120883124, - "nauc_mrr_at_10_diff1": 0.7825858492004815, - "nauc_mrr_at_10_max": 0.6168284552461306, - "nauc_mrr_at_10_std": 0.14613846483667256, - "nauc_mrr_at_1_diff1": 0.8178148233019549, - "nauc_mrr_at_1_max": 0.6052419836559261, - "nauc_mrr_at_1_std": 0.12987371517585286, - "nauc_mrr_at_20_diff1": 0.784261954129683, - "nauc_mrr_at_20_max": 0.6190281187286515, - "nauc_mrr_at_20_std": 0.1474778049919308, - "nauc_mrr_at_3_diff1": 0.7866102375472549, - "nauc_mrr_at_3_max": 0.6202568371440428, - "nauc_mrr_at_3_std": 0.13510628896209134, - "nauc_mrr_at_5_diff1": 0.7831809807419776, - "nauc_mrr_at_5_max": 0.6176841717561736, - "nauc_mrr_at_5_std": 0.14298098309055324, - "nauc_ndcg_at_1000_diff1": 0.7814139580523798, - "nauc_ndcg_at_1000_max": 0.6221268142813257, - "nauc_ndcg_at_1000_std": 0.1552747016000708, - "nauc_ndcg_at_100_diff1": 0.780344212475686, - "nauc_ndcg_at_100_max": 0.6247734986989426, - "nauc_ndcg_at_100_std": 0.16332676375007413, - "nauc_ndcg_at_10_diff1": 0.7687973801203425, - "nauc_ndcg_at_10_max": 0.6150742030451716, - "nauc_ndcg_at_10_std": 0.15909617277981258, - "nauc_ndcg_at_1_diff1": 0.8229562595133094, - "nauc_ndcg_at_1_max": 0.6013677818784777, - "nauc_ndcg_at_1_std": 0.12160003122737216, - "nauc_ndcg_at_20_diff1": 0.7743559187463246, - "nauc_ndcg_at_20_max": 0.6230320610140855, - "nauc_ndcg_at_20_std": 0.1646520543777805, - "nauc_ndcg_at_3_diff1": 0.7772486128568934, - "nauc_ndcg_at_3_max": 0.6219190572302621, - "nauc_ndcg_at_3_std": 0.13473965557594939, - "nauc_ndcg_at_5_diff1": 0.7707816184815659, - "nauc_ndcg_at_5_max": 0.6178140545282956, - "nauc_ndcg_at_5_std": 0.15038650842581236, - "nauc_precision_at_1000_diff1": 0.7115337193995216, - "nauc_precision_at_1000_max": 0.9162400747714883, - "nauc_precision_at_1000_std": 0.6394650706109937, - "nauc_precision_at_100_diff1": 0.7333454397110306, - "nauc_precision_at_100_max": 0.7444349099565769, - "nauc_precision_at_100_std": 0.4724436931961141, - "nauc_precision_at_10_diff1": 0.6871809913928806, - "nauc_precision_at_10_max": 0.6133230953694784, - "nauc_precision_at_10_std": 0.2527224723281986, - "nauc_precision_at_1_diff1": 0.8229562595133094, - "nauc_precision_at_1_max": 0.6013677818784777, - "nauc_precision_at_1_std": 0.12160003122737216, - "nauc_precision_at_20_diff1": 0.6981768420463869, - "nauc_precision_at_20_max": 0.6659441432074145, - "nauc_precision_at_20_std": 0.32046849230036073, - "nauc_precision_at_3_diff1": 0.7321559190348422, - "nauc_precision_at_3_max": 0.6343014239279321, - "nauc_precision_at_3_std": 0.15185156111493722, - "nauc_precision_at_5_diff1": 0.7072557341340879, - "nauc_precision_at_5_max": 0.6257250342242688, - "nauc_precision_at_5_std": 0.20404936155319042, - "nauc_recall_at_1000_diff1": 0.7115337193995271, - "nauc_recall_at_1000_max": 0.9162400747714905, - "nauc_recall_at_1000_std": 0.6394650706109806, - "nauc_recall_at_100_diff1": 0.7333454397110322, - "nauc_recall_at_100_max": 0.7444349099565744, - "nauc_recall_at_100_std": 0.4724436931961091, - "nauc_recall_at_10_diff1": 0.6871809913928805, - "nauc_recall_at_10_max": 0.6133230953694784, - "nauc_recall_at_10_std": 0.2527224723281998, - "nauc_recall_at_1_diff1": 0.8229562595133094, - "nauc_recall_at_1_max": 0.6013677818784777, - "nauc_recall_at_1_std": 0.12160003122737216, - "nauc_recall_at_20_diff1": 0.6981768420463863, - "nauc_recall_at_20_max": 0.6659441432074131, - "nauc_recall_at_20_std": 0.3204684923003592, - "nauc_recall_at_3_diff1": 0.7321559190348417, - "nauc_recall_at_3_max": 0.6343014239279325, - "nauc_recall_at_3_std": 0.15185156111493697, - "nauc_recall_at_5_diff1": 0.7072557341340876, - "nauc_recall_at_5_max": 0.6257250342242687, - "nauc_recall_at_5_std": 0.20404936155319015, - "ndcg_at_1": 0.61693, - "ndcg_at_10": 0.72207, - "ndcg_at_100": 0.74685, - "ndcg_at_1000": 0.75279, - "ndcg_at_20": 0.73369, - "ndcg_at_3": 0.69382, - "ndcg_at_5": 0.71049, - "precision_at_1": 0.61693, - "precision_at_10": 0.0825, - "precision_at_100": 0.00941, - "precision_at_1000": 0.00099, - "precision_at_20": 0.04354, - "precision_at_3": 0.24964, - "precision_at_5": 0.15782, - "recall_at_1": 0.61693, - "recall_at_10": 0.82496, - "recall_at_100": 0.94118, - "recall_at_1000": 0.98852, - "recall_at_20": 0.87088, - "recall_at_3": 0.74892, - "recall_at_5": 0.7891 - } - ] - }, - "task_name": "MemotionI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionT2IRetrieval.json deleted file mode 100644 index 38858558f2..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/MemotionT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", - "evaluation_time": 47.172343254089355, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.7561, - "cv_recall_at_10": 0.89957, - "cv_recall_at_100": 0.95409, - "cv_recall_at_1000": 0.98852, - "cv_recall_at_20": 0.92539, - "cv_recall_at_3": 0.85509, - "cv_recall_at_5": 0.87948, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8304, - "map_at_1": 0.75466, - "map_at_10": 0.80776, - "map_at_100": 0.81037, - "map_at_1000": 0.8105, - "map_at_20": 0.80956, - "map_at_3": 0.79914, - "map_at_5": 0.80495, - "mrr_at_1": 0.7560975609756098, - "mrr_at_10": 0.808956184099656, - "mrr_at_100": 0.8115696597483404, - "mrr_at_1000": 0.811693829214546, - "mrr_at_20": 0.810753790451444, - "mrr_at_3": 0.8003347680535627, - "mrr_at_5": 0.8061453849832614, - "nauc_cv_recall_at_1000_diff1": 0.6706645008987285, - "nauc_cv_recall_at_1000_max": 0.4840668585023845, - "nauc_cv_recall_at_1000_std": 0.5855825248516394, - "nauc_cv_recall_at_100_diff1": 0.7572865775123307, - "nauc_cv_recall_at_100_max": 0.7014046577380366, - "nauc_cv_recall_at_100_std": 0.6971837247747253, - "nauc_cv_recall_at_10_diff1": 0.8240165624741603, - "nauc_cv_recall_at_10_max": 0.6069417503163557, - "nauc_cv_recall_at_10_std": 0.43311682859231987, - "nauc_cv_recall_at_1_diff1": 0.8722966904073866, - "nauc_cv_recall_at_1_max": 0.5299326938244373, - "nauc_cv_recall_at_1_std": 0.22226251845301678, - "nauc_cv_recall_at_20_diff1": 0.7870728454870688, - "nauc_cv_recall_at_20_max": 0.6389615737600375, - "nauc_cv_recall_at_20_std": 0.5199494827002775, - "nauc_cv_recall_at_3_diff1": 0.8326936710385887, - "nauc_cv_recall_at_3_max": 0.5684788343399769, - "nauc_cv_recall_at_3_std": 0.31719170280557873, - "nauc_cv_recall_at_5_diff1": 0.8188729223764644, - "nauc_cv_recall_at_5_max": 0.5606570995953873, - "nauc_cv_recall_at_5_std": 0.3412401813639024, - "nauc_map_at_1000_diff1": 0.8592180617798573, - "nauc_map_at_1000_max": 0.5474455366412145, - "nauc_map_at_1000_std": 0.26314060985977905, - "nauc_map_at_100_diff1": 0.8591578945632715, - "nauc_map_at_100_max": 0.5476108921329297, - "nauc_map_at_100_std": 0.2634699740502252, - "nauc_map_at_10_diff1": 0.85968856552947, - "nauc_map_at_10_max": 0.5474144301133114, - "nauc_map_at_10_std": 0.26240890326221095, - "nauc_map_at_1_diff1": 0.8747133997178218, - "nauc_map_at_1_max": 0.532150450714701, - "nauc_map_at_1_std": 0.22774606041660467, - "nauc_map_at_20_diff1": 0.8590199843105324, - "nauc_map_at_20_max": 0.5476380049340864, - "nauc_map_at_20_std": 0.2630842730031387, - "nauc_map_at_3_diff1": 0.8601043415886783, - "nauc_map_at_3_max": 0.5455420632431127, - "nauc_map_at_3_std": 0.25409460690905056, - "nauc_map_at_5_diff1": 0.8585592346276657, - "nauc_map_at_5_max": 0.5439055718426468, - "nauc_map_at_5_std": 0.25610005430887645, - "nauc_mrr_at_1000_diff1": 0.8571279974501776, - "nauc_mrr_at_1000_max": 0.5466442414674472, - "nauc_mrr_at_1000_std": 0.2620528722535252, - "nauc_mrr_at_100_diff1": 0.8570690034765716, - "nauc_mrr_at_100_max": 0.5468110619980555, - "nauc_mrr_at_100_std": 0.26238479039821916, - "nauc_mrr_at_10_diff1": 0.8576335229981322, - "nauc_mrr_at_10_max": 0.5466254391056721, - "nauc_mrr_at_10_std": 0.2613342084530473, - "nauc_mrr_at_1_diff1": 0.8722966904073866, - "nauc_mrr_at_1_max": 0.5299326938244373, - "nauc_mrr_at_1_std": 0.22226251845301678, - "nauc_mrr_at_20_diff1": 0.8569401410607558, - "nauc_mrr_at_20_max": 0.5468420677776812, - "nauc_mrr_at_20_std": 0.2620021054881723, - "nauc_mrr_at_3_diff1": 0.8581353780119624, - "nauc_mrr_at_3_max": 0.5447757752206631, - "nauc_mrr_at_3_std": 0.25302149099138316, - "nauc_mrr_at_5_diff1": 0.856526428569613, - "nauc_mrr_at_5_max": 0.5431090179640301, - "nauc_mrr_at_5_std": 0.2550070511531312, - "nauc_ndcg_at_1000_diff1": 0.8530331244324216, - "nauc_ndcg_at_1000_max": 0.5523802225859098, - "nauc_ndcg_at_1000_std": 0.2828500300468584, - "nauc_ndcg_at_100_diff1": 0.8510912813706918, - "nauc_ndcg_at_100_max": 0.5586724956727237, - "nauc_ndcg_at_100_std": 0.29552202763851937, - "nauc_ndcg_at_10_diff1": 0.8537108262779001, - "nauc_ndcg_at_10_max": 0.5565012931379493, - "nauc_ndcg_at_10_std": 0.2884764507742189, - "nauc_ndcg_at_1_diff1": 0.8747133997178218, - "nauc_ndcg_at_1_max": 0.532150450714701, - "nauc_ndcg_at_1_std": 0.22774606041660467, - "nauc_ndcg_at_20_diff1": 0.8504706937817262, - "nauc_ndcg_at_20_max": 0.5581650724327955, - "nauc_ndcg_at_20_std": 0.29286385001715937, - "nauc_ndcg_at_3_diff1": 0.8544076459383539, - "nauc_ndcg_at_3_max": 0.5503441549211755, - "nauc_ndcg_at_3_std": 0.2669883605033744, - "nauc_ndcg_at_5_diff1": 0.8512619509841919, - "nauc_ndcg_at_5_max": 0.5472168141098742, - "nauc_ndcg_at_5_std": 0.2715472109001755, - "nauc_precision_at_1000_diff1": 0.67066450089872, - "nauc_precision_at_1000_max": 0.48406685850238695, - "nauc_precision_at_1000_std": 0.5855825248516409, - "nauc_precision_at_100_diff1": 0.7572865775123279, - "nauc_precision_at_100_max": 0.7014046577380433, - "nauc_precision_at_100_std": 0.6971837247747307, - "nauc_precision_at_10_diff1": 0.8240165624741588, - "nauc_precision_at_10_max": 0.6069417503163547, - "nauc_precision_at_10_std": 0.43311682859231926, - "nauc_precision_at_1_diff1": 0.8747133997178218, - "nauc_precision_at_1_max": 0.532150450714701, - "nauc_precision_at_1_std": 0.22774606041660467, - "nauc_precision_at_20_diff1": 0.7870728454870688, - "nauc_precision_at_20_max": 0.6389615737600398, - "nauc_precision_at_20_std": 0.5199494827002775, - "nauc_precision_at_3_diff1": 0.8326936710385894, - "nauc_precision_at_3_max": 0.5684788343399768, - "nauc_precision_at_3_std": 0.31719170280557873, - "nauc_precision_at_5_diff1": 0.8188729223764643, - "nauc_precision_at_5_max": 0.5606570995953888, - "nauc_precision_at_5_std": 0.3412401813639036, - "nauc_recall_at_1000_diff1": 0.6706645008987285, - "nauc_recall_at_1000_max": 0.4840668585023845, - "nauc_recall_at_1000_std": 0.5855825248516394, - "nauc_recall_at_100_diff1": 0.7572865775123307, - "nauc_recall_at_100_max": 0.7014046577380366, - "nauc_recall_at_100_std": 0.6971837247747253, - "nauc_recall_at_10_diff1": 0.8240165624741603, - "nauc_recall_at_10_max": 0.6069417503163557, - "nauc_recall_at_10_std": 0.43311682859231987, - "nauc_recall_at_1_diff1": 0.8747133997178218, - "nauc_recall_at_1_max": 0.532150450714701, - "nauc_recall_at_1_std": 0.22774606041660467, - "nauc_recall_at_20_diff1": 0.7870728454870688, - "nauc_recall_at_20_max": 0.6389615737600375, - "nauc_recall_at_20_std": 0.5199494827002775, - "nauc_recall_at_3_diff1": 0.8326936710385887, - "nauc_recall_at_3_max": 0.5684788343399769, - "nauc_recall_at_3_std": 0.31719170280557873, - "nauc_recall_at_5_diff1": 0.8188729223764644, - "nauc_recall_at_5_max": 0.5606570995953873, - "nauc_recall_at_5_std": 0.3412401813639024, - "ndcg_at_1": 0.75466, - "ndcg_at_10": 0.8304, - "ndcg_at_100": 0.84243, - "ndcg_at_1000": 0.84657, - "ndcg_at_20": 0.83694, - "ndcg_at_3": 0.81352, - "ndcg_at_5": 0.82377, - "precision_at_1": 0.75466, - "precision_at_10": 0.08996, - "precision_at_100": 0.00954, - "precision_at_1000": 0.00099, - "precision_at_20": 0.04627, - "precision_at_3": 0.28503, - "precision_at_5": 0.1759, - "recall_at_1": 0.75466, - "recall_at_10": 0.89957, - "recall_at_100": 0.95409, - "recall_at_1000": 0.98852, - "recall_at_20": 0.92539, - "recall_at_3": 0.85509, - "recall_at_5": 0.87948 - } - ] - }, - "task_name": "MemotionT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/RenderedSST2.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/RenderedSST2.json deleted file mode 100644 index 773aaf9434..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/RenderedSST2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", - "evaluation_time": 12.006900548934937, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "accuracy": 0.6051619989017024, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6051619989017024 - } - ] - }, - "task_name": "RenderedSST2" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS12VisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS12VisualSTS.json deleted file mode 100644 index a1444b9c04..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS12VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "820c25edfba736f3789201b2476208cc62c2ccb9", - "evaluation_time": 36.103975772857666, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.59081142663465, - "cosine_spearman": 0.6381747892969383, - "euclidean_pearson": 0.6340957480798639, - "euclidean_spearman": 0.6676750629657797, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6381747892969383, - "manhattan_pearson": 0.6404929502036195, - "manhattan_spearman": 0.6719835643946208, - "pearson": 0.59081142663465, - "spearman": 0.6381747892969383 - } - ] - }, - "task_name": "STS12VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS13VisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS13VisualSTS.json deleted file mode 100644 index e4453a611c..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS13VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "561ee9ca47ff3e4a657283c59416deca8dc169f2", - "evaluation_time": 16.974658966064453, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.6075975743163279, - "cosine_spearman": 0.6325573553595002, - "euclidean_pearson": 0.6692213417308934, - "euclidean_spearman": 0.6743831446100913, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6325573553595002, - "manhattan_pearson": 0.6754706863972464, - "manhattan_spearman": 0.6786762052263963, - "pearson": 0.6075975743163279, - "spearman": 0.6325573553595002 - } - ] - }, - "task_name": "STS13VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS14VisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS14VisualSTS.json deleted file mode 100644 index cf1e991fb4..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS14VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "824e95e45471024a684b901e0645579ffd9ca288", - "evaluation_time": 40.73365497589111, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.5761138227493786, - "cosine_spearman": 0.5698651220984589, - "euclidean_pearson": 0.6271652197991896, - "euclidean_spearman": 0.6147902301712311, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5698651220984589, - "manhattan_pearson": 0.6340773461052611, - "manhattan_spearman": 0.6220267895742038, - "pearson": 0.5761138227493786, - "spearman": 0.5698651220984589 - } - ] - }, - "task_name": "STS14VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS15VisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS15VisualSTS.json deleted file mode 100644 index 0d385e0c8f..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS15VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "1f8d08d9b9daac7118dfdefeb94b0aac4baf2e5f", - "evaluation_time": 33.43548011779785, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.714887829276996, - "cosine_spearman": 0.7331854831535017, - "euclidean_pearson": 0.7553837552629206, - "euclidean_spearman": 0.7613317705180044, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7331854831535017, - "manhattan_pearson": 0.7598332173640413, - "manhattan_spearman": 0.7656311405677237, - "pearson": 0.714887829276996, - "spearman": 0.7331854831535017 - } - ] - }, - "task_name": "STS15VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS16VisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS16VisualSTS.json deleted file mode 100644 index 032ec34621..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS16VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "fc354f19598af93f32c0af1b94046ffdeaacde15", - "evaluation_time": 13.63206934928894, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.6780449758797138, - "cosine_spearman": 0.6891106167366204, - "euclidean_pearson": 0.7192011457180031, - "euclidean_spearman": 0.7222807032346467, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6891106167366204, - "manhattan_pearson": 0.723392009670844, - "manhattan_spearman": 0.7257618725946727, - "pearson": 0.6780449758797138, - "spearman": 0.6891106167366204 - } - ] - }, - "task_name": "STS16VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS17MultilingualVisualSTS.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS17MultilingualVisualSTS.json deleted file mode 100644 index a7d95e63bd..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/STS17MultilingualVisualSTS.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", - "evaluation_time": 67.66748046875, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.061602218313769115, - "cosine_spearman": 0.10536378343822136, - "euclidean_pearson": 0.09492865294256914, - "euclidean_spearman": 0.1121225391929482, - "hf_subset": "ko-ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.10536378343822136, - "manhattan_pearson": 0.0937650446851091, - "manhattan_spearman": 0.11075594240825876, - "pearson": 0.061602218313769115, - "spearman": 0.10536378343822136 - }, - { - "cosine_pearson": 0.22474277017008987, - "cosine_spearman": 0.3625061930482899, - "euclidean_pearson": 0.34914286152218044, - "euclidean_spearman": 0.35956084669160554, - "hf_subset": "ar-ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.3625061930482899, - "manhattan_pearson": 0.35031626840872193, - "manhattan_spearman": 0.3618855939595577, - "pearson": 0.22474277017008987, - "spearman": 0.3625061930482899 - }, - { - "cosine_pearson": 0.13201283590883084, - "cosine_spearman": 0.1313064418882118, - "euclidean_pearson": 0.13748213800122056, - "euclidean_spearman": 0.12376607271918363, - "hf_subset": "en-ar", - "languages": [ - "eng-Latn", - "ara-Arab" - ], - "main_score": 0.1313064418882118, - "manhattan_pearson": 0.159407808813381, - "manhattan_spearman": 0.15133400989777218, - "pearson": 0.13201283590883084, - "spearman": 0.1313064418882118 - }, - { - "cosine_pearson": 0.4077979312610406, - "cosine_spearman": 0.41573294231401264, - "euclidean_pearson": 0.4486770405893728, - "euclidean_spearman": 0.46190588723590936, - "hf_subset": "en-de", - "languages": [ - "eng-Latn", - "deu-Latn" - ], - "main_score": 0.41573294231401264, - "manhattan_pearson": 0.4515925499294072, - "manhattan_spearman": 0.4601457354285419, - "pearson": 0.4077979312610406, - "spearman": 0.41573294231401264 - }, - { - "cosine_pearson": 0.7487757113443495, - "cosine_spearman": 0.7817926247120496, - "euclidean_pearson": 0.7996271438124265, - "euclidean_spearman": 0.8014141466616177, - "hf_subset": "en-en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7817926247120496, - "manhattan_pearson": 0.8076051409003571, - "manhattan_spearman": 0.8078643207886379, - "pearson": 0.7487757113443495, - "spearman": 0.7817926247120496 - }, - { - "cosine_pearson": 0.34213443280278477, - "cosine_spearman": 0.3541791366106406, - "euclidean_pearson": 0.34915230983683704, - "euclidean_spearman": 0.3607616615863365, - "hf_subset": "en-tr", - "languages": [ - "eng-Latn", - "tur-Latn" - ], - "main_score": 0.3541791366106406, - "manhattan_pearson": 0.3731936697763999, - "manhattan_spearman": 0.3784834597423862, - "pearson": 0.34213443280278477, - "spearman": 0.3541791366106406 - }, - { - "cosine_pearson": 0.20904811500829665, - "cosine_spearman": 0.24627188746430106, - "euclidean_pearson": 0.23028650420666652, - "euclidean_spearman": 0.27021390375087223, - "hf_subset": "es-en", - "languages": [ - "spa-Latn", - "eng-Latn" - ], - "main_score": 0.24627188746430106, - "manhattan_pearson": 0.283890229592864, - "manhattan_spearman": 0.3048350457527673, - "pearson": 0.20904811500829665, - "spearman": 0.24627188746430106 - }, - { - "cosine_pearson": 0.6123145532680564, - "cosine_spearman": 0.6295394162055459, - "euclidean_pearson": 0.6826256087559528, - "euclidean_spearman": 0.6501694437580297, - "hf_subset": "es-es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.6295394162055459, - "manhattan_pearson": 0.6940458973588839, - "manhattan_spearman": 0.6610441303288633, - "pearson": 0.6123145532680564, - "spearman": 0.6295394162055459 - }, - { - "cosine_pearson": 0.3960565459527028, - "cosine_spearman": 0.3872111026208792, - "euclidean_pearson": 0.40972175388780063, - "euclidean_spearman": 0.39840742096400916, - "hf_subset": "fr-en", - "languages": [ - "fra-Latn", - "eng-Latn" - ], - "main_score": 0.3872111026208792, - "manhattan_pearson": 0.42718862954472175, - "manhattan_spearman": 0.41936894929062085, - "pearson": 0.3960565459527028, - "spearman": 0.3872111026208792 - }, - { - "cosine_pearson": 0.3038988006923432, - "cosine_spearman": 0.3139631866240913, - "euclidean_pearson": 0.33027917052156686, - "euclidean_spearman": 0.3509951815614502, - "hf_subset": "it-en", - "languages": [ - "ita-Latn", - "eng-Latn" - ], - "main_score": 0.3139631866240913, - "manhattan_pearson": 0.359871587744587, - "manhattan_spearman": 0.3636218394711262, - "pearson": 0.3038988006923432, - "spearman": 0.3139631866240913 - }, - { - "cosine_pearson": 0.39175159861451403, - "cosine_spearman": 0.38628124736383135, - "euclidean_pearson": 0.4207412968225121, - "euclidean_spearman": 0.4079931177581394, - "hf_subset": "nl-en", - "languages": [ - "nld-Latn", - "eng-Latn" - ], - "main_score": 0.38628124736383135, - "manhattan_pearson": 0.44162755285128313, - "manhattan_spearman": 0.4344688146099064, - "pearson": 0.39175159861451403, - "spearman": 0.38628124736383135 - } - ] - }, - "task_name": "STS17MultilingualVisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRI2TRetrieval.json deleted file mode 100644 index 5391e9d2ac..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", - "evaluation_time": 140.40432357788086, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.08836, - "cv_recall_at_10": 0.21085, - "cv_recall_at_100": 0.38431, - "cv_recall_at_1000": 0.6714, - "cv_recall_at_20": 0.25395, - "cv_recall_at_3": 0.14235, - "cv_recall_at_5": 0.16934, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.14411, - "map_at_1": 0.08836, - "map_at_10": 0.12346, - "map_at_100": 0.1294, - "map_at_1000": 0.13042, - "map_at_20": 0.12639, - "map_at_3": 0.11184, - "map_at_5": 0.11798, - "mrr_at_1": 0.08836008116583656, - "mrr_at_10": 0.12345752213857675, - "mrr_at_100": 0.12940234576740298, - "mrr_at_1000": 0.13042081759294002, - "mrr_at_20": 0.12639283297819012, - "mrr_at_3": 0.1118387341408919, - "mrr_at_5": 0.1179845866896225, - "nauc_cv_recall_at_1000_diff1": 0.1930440666440711, - "nauc_cv_recall_at_1000_max": 0.10121846044999482, - "nauc_cv_recall_at_1000_std": 0.42508401174919186, - "nauc_cv_recall_at_100_diff1": 0.2446500985422135, - "nauc_cv_recall_at_100_max": 0.1467798294776438, - "nauc_cv_recall_at_100_std": 0.414757382289627, - "nauc_cv_recall_at_10_diff1": 0.30827230657058935, - "nauc_cv_recall_at_10_max": 0.22109141920875744, - "nauc_cv_recall_at_10_std": 0.38971247695792116, - "nauc_cv_recall_at_1_diff1": 0.4980711735833261, - "nauc_cv_recall_at_1_max": 0.2971017528223884, - "nauc_cv_recall_at_1_std": 0.35834619013019836, - "nauc_cv_recall_at_20_diff1": 0.2825045381641615, - "nauc_cv_recall_at_20_max": 0.19613741590717187, - "nauc_cv_recall_at_20_std": 0.39649688417982515, - "nauc_cv_recall_at_3_diff1": 0.37694694761984543, - "nauc_cv_recall_at_3_max": 0.2635657508987931, - "nauc_cv_recall_at_3_std": 0.37644697851927716, - "nauc_cv_recall_at_5_diff1": 0.342725203515418, - "nauc_cv_recall_at_5_max": 0.2483062507893586, - "nauc_cv_recall_at_5_std": 0.3796606741574664, - "nauc_map_at_1000_diff1": 0.40552045901217515, - "nauc_map_at_1000_max": 0.26273915141358706, - "nauc_map_at_1000_std": 0.3729451955892119, - "nauc_map_at_100_diff1": 0.4056641617493706, - "nauc_map_at_100_max": 0.2629317107092913, - "nauc_map_at_100_std": 0.37288447789078516, - "nauc_map_at_10_diff1": 0.41197297657520887, - "nauc_map_at_10_max": 0.2684384296464497, - "nauc_map_at_10_std": 0.3709211802013581, - "nauc_map_at_1_diff1": 0.4980711735833261, - "nauc_map_at_1_max": 0.2971017528223884, - "nauc_map_at_1_std": 0.35834619013019836, - "nauc_map_at_20_diff1": 0.408279749615102, - "nauc_map_at_20_max": 0.26530210357036255, - "nauc_map_at_20_std": 0.3720042216340903, - "nauc_map_at_3_diff1": 0.43361382217865935, - "nauc_map_at_3_max": 0.2799005350406905, - "nauc_map_at_3_std": 0.36729622951789603, - "nauc_map_at_5_diff1": 0.42131281614217014, - "nauc_map_at_5_max": 0.2749144272701716, - "nauc_map_at_5_std": 0.36853037133934513, - "nauc_mrr_at_1000_diff1": 0.4055204594804454, - "nauc_mrr_at_1000_max": 0.2627391567916803, - "nauc_mrr_at_1000_std": 0.3729452011303819, - "nauc_mrr_at_100_diff1": 0.4056641617493706, - "nauc_mrr_at_100_max": 0.2629317107092913, - "nauc_mrr_at_100_std": 0.37288447789078516, - "nauc_mrr_at_10_diff1": 0.41197297657520887, - "nauc_mrr_at_10_max": 0.2684384296464497, - "nauc_mrr_at_10_std": 0.3709211802013581, - "nauc_mrr_at_1_diff1": 0.4980711735833261, - "nauc_mrr_at_1_max": 0.2971017528223884, - "nauc_mrr_at_1_std": 0.35834619013019836, - "nauc_mrr_at_20_diff1": 0.408279749615102, - "nauc_mrr_at_20_max": 0.26530210357036255, - "nauc_mrr_at_20_std": 0.3720042216340903, - "nauc_mrr_at_3_diff1": 0.43361382217865935, - "nauc_mrr_at_3_max": 0.2799005350406905, - "nauc_mrr_at_3_std": 0.36729622951789603, - "nauc_mrr_at_5_diff1": 0.42131281614217014, - "nauc_mrr_at_5_max": 0.2749144272701716, - "nauc_mrr_at_5_std": 0.36853037133934513, - "nauc_ndcg_at_1000_diff1": 0.3507824423697204, - "nauc_ndcg_at_1000_max": 0.22522557499324103, - "nauc_ndcg_at_1000_std": 0.385920157570721, - "nauc_ndcg_at_100_diff1": 0.35397834406230494, - "nauc_ndcg_at_100_max": 0.2288619650602957, - "nauc_ndcg_at_100_std": 0.385366160982851, - "nauc_ndcg_at_10_diff1": 0.37884486199522766, - "nauc_ndcg_at_10_max": 0.25371633138876953, - "nauc_ndcg_at_10_std": 0.37680379412540926, - "nauc_ndcg_at_1_diff1": 0.4980711735833261, - "nauc_ndcg_at_1_max": 0.2971017528223884, - "nauc_ndcg_at_1_std": 0.35834619013019836, - "nauc_ndcg_at_20_diff1": 0.3677982770691768, - "nauc_ndcg_at_20_max": 0.24398701478538692, - "nauc_ndcg_at_20_std": 0.37975850307644804, - "nauc_ndcg_at_3_diff1": 0.41691634606030453, - "nauc_ndcg_at_3_max": 0.2751132351903913, - "nauc_ndcg_at_3_std": 0.36996518625696995, - "nauc_ndcg_at_5_diff1": 0.3972950689280354, - "nauc_ndcg_at_5_max": 0.26688816758904627, - "nauc_ndcg_at_5_std": 0.3719255027762222, - "nauc_precision_at_1000_diff1": 0.19304406664407023, - "nauc_precision_at_1000_max": 0.10121846044999426, - "nauc_precision_at_1000_std": 0.42508401174919064, - "nauc_precision_at_100_diff1": 0.2446500985422135, - "nauc_precision_at_100_max": 0.14677982947764376, - "nauc_precision_at_100_std": 0.41475738228962666, - "nauc_precision_at_10_diff1": 0.30827230657058935, - "nauc_precision_at_10_max": 0.2210914192087575, - "nauc_precision_at_10_std": 0.38971247695792105, - "nauc_precision_at_1_diff1": 0.4980711735833261, - "nauc_precision_at_1_max": 0.2971017528223884, - "nauc_precision_at_1_std": 0.35834619013019836, - "nauc_precision_at_20_diff1": 0.2825045381641612, - "nauc_precision_at_20_max": 0.1961374159071717, - "nauc_precision_at_20_std": 0.39649688417982526, - "nauc_precision_at_3_diff1": 0.3769469476198457, - "nauc_precision_at_3_max": 0.2635657508987932, - "nauc_precision_at_3_std": 0.3764469785192774, - "nauc_precision_at_5_diff1": 0.34272520351541796, - "nauc_precision_at_5_max": 0.24830625078935847, - "nauc_precision_at_5_std": 0.37966067415746646, - "nauc_recall_at_1000_diff1": 0.1930440666440711, - "nauc_recall_at_1000_max": 0.10121846044999482, - "nauc_recall_at_1000_std": 0.42508401174919186, - "nauc_recall_at_100_diff1": 0.2446500985422135, - "nauc_recall_at_100_max": 0.1467798294776438, - "nauc_recall_at_100_std": 0.414757382289627, - "nauc_recall_at_10_diff1": 0.30827230657058935, - "nauc_recall_at_10_max": 0.22109141920875744, - "nauc_recall_at_10_std": 0.38971247695792116, - "nauc_recall_at_1_diff1": 0.4980711735833261, - "nauc_recall_at_1_max": 0.2971017528223884, - "nauc_recall_at_1_std": 0.35834619013019836, - "nauc_recall_at_20_diff1": 0.2825045381641615, - "nauc_recall_at_20_max": 0.19613741590717187, - "nauc_recall_at_20_std": 0.39649688417982515, - "nauc_recall_at_3_diff1": 0.37694694761984543, - "nauc_recall_at_3_max": 0.2635657508987931, - "nauc_recall_at_3_std": 0.37644697851927716, - "nauc_recall_at_5_diff1": 0.342725203515418, - "nauc_recall_at_5_max": 0.2483062507893586, - "nauc_recall_at_5_std": 0.3796606741574664, - "ndcg_at_1": 0.08836, - "ndcg_at_10": 0.14411, - "ndcg_at_100": 0.17846, - "ndcg_at_1000": 0.21287, - "ndcg_at_20": 0.15494, - "ndcg_at_3": 0.11966, - "ndcg_at_5": 0.13076, - "precision_at_1": 0.08836, - "precision_at_10": 0.02108, - "precision_at_100": 0.00384, - "precision_at_1000": 0.00067, - "precision_at_20": 0.0127, - "precision_at_3": 0.04745, - "precision_at_5": 0.03387, - "recall_at_1": 0.08836, - "recall_at_10": 0.21085, - "recall_at_100": 0.38431, - "recall_at_1000": 0.6714, - "recall_at_20": 0.25395, - "recall_at_3": 0.14235, - "recall_at_5": 0.16934 - } - ] - }, - "task_name": "SciMMIRI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRT2IRetrieval.json deleted file mode 100644 index 72e576558c..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SciMMIRT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", - "evaluation_time": 137.33402299880981, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.07846, - "cv_recall_at_10": 0.18699, - "cv_recall_at_100": 0.35153, - "cv_recall_at_1000": 0.64207, - "cv_recall_at_20": 0.22603, - "cv_recall_at_3": 0.12593, - "cv_recall_at_5": 0.15046, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.12785, - "map_at_1": 0.07846, - "map_at_10": 0.10952, - "map_at_100": 0.11508, - "map_at_1000": 0.11612, - "map_at_20": 0.11218, - "map_at_3": 0.09912, - "map_at_5": 0.10464, - "mrr_at_1": 0.07846030867613601, - "mrr_at_10": 0.10951404932220245, - "mrr_at_100": 0.1150779301551405, - "mrr_at_1000": 0.11611561738842863, - "mrr_at_20": 0.11217670727748709, - "mrr_at_3": 0.09912070343724981, - "mrr_at_5": 0.10464243989423845, - "nauc_cv_recall_at_1000_diff1": 0.14280409251991044, - "nauc_cv_recall_at_1000_max": 0.18628330671583407, - "nauc_cv_recall_at_1000_std": 0.3870589592403038, - "nauc_cv_recall_at_100_diff1": 0.18605678508837606, - "nauc_cv_recall_at_100_max": 0.17998073338336004, - "nauc_cv_recall_at_100_std": 0.33735207281947194, - "nauc_cv_recall_at_10_diff1": 0.2650604850881054, - "nauc_cv_recall_at_10_max": 0.23823992507714759, - "nauc_cv_recall_at_10_std": 0.27980801514303494, - "nauc_cv_recall_at_1_diff1": 0.4827365014175455, - "nauc_cv_recall_at_1_max": 0.33010914760725707, - "nauc_cv_recall_at_1_std": 0.2512977037559647, - "nauc_cv_recall_at_20_diff1": 0.23941515017415582, - "nauc_cv_recall_at_20_max": 0.22007084090482062, - "nauc_cv_recall_at_20_std": 0.29443736255648456, - "nauc_cv_recall_at_3_diff1": 0.35336896283747304, - "nauc_cv_recall_at_3_max": 0.295773329151323, - "nauc_cv_recall_at_3_std": 0.2681076282273982, - "nauc_cv_recall_at_5_diff1": 0.31570322218368735, - "nauc_cv_recall_at_5_max": 0.2776172685924335, - "nauc_cv_recall_at_5_std": 0.27866786119872977, - "nauc_map_at_1000_diff1": 0.376660313469499, - "nauc_map_at_1000_max": 0.29209998722994507, - "nauc_map_at_1000_std": 0.26716382098614505, - "nauc_map_at_100_diff1": 0.37688184035666233, - "nauc_map_at_100_max": 0.29208878660890697, - "nauc_map_at_100_std": 0.2668800438584805, - "nauc_map_at_10_diff1": 0.3851603824358598, - "nauc_map_at_10_max": 0.29771555101574054, - "nauc_map_at_10_std": 0.26295031900912746, - "nauc_map_at_1_diff1": 0.4827365014175455, - "nauc_map_at_1_max": 0.33010914760725707, - "nauc_map_at_1_std": 0.2512977037559647, - "nauc_map_at_20_diff1": 0.3807333533485515, - "nauc_map_at_20_max": 0.2947438323961747, - "nauc_map_at_20_std": 0.26459412628505796, - "nauc_map_at_3_diff1": 0.41141276578806035, - "nauc_map_at_3_max": 0.3122153310864702, - "nauc_map_at_3_std": 0.25861944245595125, - "nauc_map_at_5_diff1": 0.3982000648043669, - "nauc_map_at_5_max": 0.30657629290318295, - "nauc_map_at_5_std": 0.2624903009814347, - "nauc_mrr_at_1000_diff1": 0.37666471757312453, - "nauc_mrr_at_1000_max": 0.29209929641607063, - "nauc_mrr_at_1000_std": 0.26716424106866365, - "nauc_mrr_at_100_diff1": 0.3768862418339056, - "nauc_mrr_at_100_max": 0.29208808952380083, - "nauc_mrr_at_100_std": 0.2668804612792214, - "nauc_mrr_at_10_diff1": 0.3852100780007191, - "nauc_mrr_at_10_max": 0.29770795553077667, - "nauc_mrr_at_10_std": 0.2629548450361711, - "nauc_mrr_at_1_diff1": 0.4827365014175455, - "nauc_mrr_at_1_max": 0.33010914760725707, - "nauc_mrr_at_1_std": 0.2512977037559647, - "nauc_mrr_at_20_diff1": 0.38073780181367767, - "nauc_mrr_at_20_max": 0.2947431394977995, - "nauc_mrr_at_20_std": 0.2645945387197461, - "nauc_mrr_at_3_diff1": 0.41141276578806035, - "nauc_mrr_at_3_max": 0.3122153310864702, - "nauc_mrr_at_3_std": 0.25861944245595125, - "nauc_mrr_at_5_diff1": 0.3982000648043669, - "nauc_mrr_at_5_max": 0.30657629290318295, - "nauc_mrr_at_5_std": 0.2624903009814347, - "nauc_ndcg_at_1000_diff1": 0.31030920764011694, - "nauc_ndcg_at_1000_max": 0.259488547988665, - "nauc_ndcg_at_1000_std": 0.294399902820702, - "nauc_ndcg_at_100_diff1": 0.31373988925006974, - "nauc_ndcg_at_100_max": 0.25646300051736803, - "nauc_ndcg_at_100_std": 0.2875697891829784, - "nauc_ndcg_at_10_diff1": 0.3465105960548437, - "nauc_ndcg_at_10_max": 0.2792501524340969, - "nauc_ndcg_at_10_std": 0.2683925570046724, - "nauc_ndcg_at_1_diff1": 0.4827365014175455, - "nauc_ndcg_at_1_max": 0.33010914760725707, - "nauc_ndcg_at_1_std": 0.2512977037559647, - "nauc_ndcg_at_20_diff1": 0.3341800450298754, - "nauc_ndcg_at_20_max": 0.270914009208363, - "nauc_ndcg_at_20_std": 0.273906970807198, - "nauc_ndcg_at_3_diff1": 0.3941121974458003, - "nauc_ndcg_at_3_max": 0.3073570658308402, - "nauc_ndcg_at_3_std": 0.2613677969210819, - "nauc_ndcg_at_5_diff1": 0.37280888984298527, - "nauc_ndcg_at_5_max": 0.2978307511070134, - "nauc_ndcg_at_5_std": 0.26738827173898944, - "nauc_precision_at_1000_diff1": 0.14280409251990966, - "nauc_precision_at_1000_max": 0.18628330671583412, - "nauc_precision_at_1000_std": 0.38705895924030337, - "nauc_precision_at_100_diff1": 0.18605678508837586, - "nauc_precision_at_100_max": 0.17998073338336013, - "nauc_precision_at_100_std": 0.33735207281947227, - "nauc_precision_at_10_diff1": 0.2647678271567299, - "nauc_precision_at_10_max": 0.2383047527423176, - "nauc_precision_at_10_std": 0.27977418385850095, - "nauc_precision_at_1_diff1": 0.4827365014175455, - "nauc_precision_at_1_max": 0.33010914760725707, - "nauc_precision_at_1_std": 0.2512977037559647, - "nauc_precision_at_20_diff1": 0.239415150174156, - "nauc_precision_at_20_max": 0.22007084090482062, - "nauc_precision_at_20_std": 0.2944373625564846, - "nauc_precision_at_3_diff1": 0.3533689628374732, - "nauc_precision_at_3_max": 0.295773329151323, - "nauc_precision_at_3_std": 0.26810762822739814, - "nauc_precision_at_5_diff1": 0.31570322218368735, - "nauc_precision_at_5_max": 0.2776172685924336, - "nauc_precision_at_5_std": 0.2786678611987297, - "nauc_recall_at_1000_diff1": 0.14280409251991044, - "nauc_recall_at_1000_max": 0.18628330671583407, - "nauc_recall_at_1000_std": 0.3870589592403038, - "nauc_recall_at_100_diff1": 0.18605678508837606, - "nauc_recall_at_100_max": 0.17998073338336004, - "nauc_recall_at_100_std": 0.33735207281947194, - "nauc_recall_at_10_diff1": 0.2647678271567298, - "nauc_recall_at_10_max": 0.2383047527423176, - "nauc_recall_at_10_std": 0.2797741838585011, - "nauc_recall_at_1_diff1": 0.4827365014175455, - "nauc_recall_at_1_max": 0.33010914760725707, - "nauc_recall_at_1_std": 0.2512977037559647, - "nauc_recall_at_20_diff1": 0.23941515017415582, - "nauc_recall_at_20_max": 0.22007084090482062, - "nauc_recall_at_20_std": 0.29443736255648456, - "nauc_recall_at_3_diff1": 0.35336896283747304, - "nauc_recall_at_3_max": 0.295773329151323, - "nauc_recall_at_3_std": 0.2681076282273982, - "nauc_recall_at_5_diff1": 0.31570322218368735, - "nauc_recall_at_5_max": 0.2776172685924335, - "nauc_recall_at_5_std": 0.27866786119872977, - "ndcg_at_1": 0.07846, - "ndcg_at_10": 0.12785, - "ndcg_at_100": 0.16029, - "ndcg_at_1000": 0.19513, - "ndcg_at_20": 0.13765, - "ndcg_at_3": 0.106, - "ndcg_at_5": 0.11603, - "precision_at_1": 0.07846, - "precision_at_10": 0.01871, - "precision_at_100": 0.00352, - "precision_at_1000": 0.00064, - "precision_at_20": 0.0113, - "precision_at_3": 0.04198, - "precision_at_5": 0.03009, - "recall_at_1": 0.07846, - "recall_at_10": 0.18705, - "recall_at_100": 0.35153, - "recall_at_1000": 0.64207, - "recall_at_20": 0.22603, - "recall_at_3": 0.12593, - "recall_at_5": 0.15046 - } - ] - }, - "task_name": "SciMMIRT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SugarCrepe.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SugarCrepe.json deleted file mode 100644 index 94abe47cb0..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/SugarCrepe.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "134abf9ade6a32f9fdae0e89022ff227a70b87e5", - "evaluation_time": 126.86821699142456, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.7690054774284363, - "text_acc": 0.7690054774284363 - } - ] - }, - "task_name": "SugarCrepe" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TUBerlinT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TUBerlinT2IRetrieval.json deleted file mode 100644 index 81d2509bb5..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TUBerlinT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "0cd78cd1ddbd3cafa9f319c638ebd77836ec9ff6", - "evaluation_time": 213.69540762901306, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.892, - "cv_recall_at_10": 1.0, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 0.988, - "cv_recall_at_5": 0.996, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.86686, - "map_at_1": 0.01115, - "map_at_10": 0.1005, - "map_at_100": 0.52616, - "map_at_1000": 0.5996, - "map_at_20": 0.19016, - "map_at_3": 0.03243, - "map_at_5": 0.05275, - "mrr_at_1": 0.892, - "mrr_at_10": 0.9413333333333334, - "mrr_at_100": 0.9413333333333334, - "mrr_at_1000": 0.9413333333333334, - "mrr_at_20": 0.9413333333333334, - "mrr_at_3": 0.9386666666666666, - "mrr_at_5": 0.9406666666666667, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": NaN, - "nauc_cv_recall_at_10_max": NaN, - "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": 0.04326570520655111, - "nauc_cv_recall_at_1_max": 0.3224325173726293, - "nauc_cv_recall_at_1_std": 0.0345532004050703, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": 0.460939931528164, - "nauc_cv_recall_at_3_max": -0.17086834733893533, - "nauc_cv_recall_at_3_std": -0.10768751945221451, - "nauc_cv_recall_at_5_diff1": 0.5541549953314557, - "nauc_cv_recall_at_5_max": -1.7399626517272968, - "nauc_cv_recall_at_5_std": -1.1517273576096612, - "nauc_map_at_1000_diff1": -0.15908458334668993, - "nauc_map_at_1000_max": 0.2802601284758, - "nauc_map_at_1000_std": 0.5666251153690053, - "nauc_map_at_100_diff1": -0.14422906770688157, - "nauc_map_at_100_max": 0.2646769819295198, - "nauc_map_at_100_std": 0.520091743009256, - "nauc_map_at_10_diff1": 0.02159195828127496, - "nauc_map_at_10_max": 0.2922120128550034, - "nauc_map_at_10_std": 0.2414778043850529, - "nauc_map_at_1_diff1": 0.0432657052065505, - "nauc_map_at_1_max": 0.3224325173726305, - "nauc_map_at_1_std": 0.03455320040506908, - "nauc_map_at_20_diff1": -0.03664228393755642, - "nauc_map_at_20_max": 0.2798113102736121, - "nauc_map_at_20_std": 0.29660406074745105, - "nauc_map_at_3_diff1": -0.04081880084720492, - "nauc_map_at_3_max": 0.29756873651497484, - "nauc_map_at_3_std": 0.1480977454027196, - "nauc_map_at_5_diff1": 0.04000144036378255, - "nauc_map_at_5_max": 0.29959598064961857, - "nauc_map_at_5_std": 0.21234921808246715, - "nauc_mrr_at_1000_diff1": 0.08232431911613593, - "nauc_mrr_at_1000_max": 0.2976966640972928, - "nauc_mrr_at_1000_std": 0.03936761305241755, - "nauc_mrr_at_100_diff1": 0.08232431911613593, - "nauc_mrr_at_100_max": 0.2976966640972928, - "nauc_mrr_at_100_std": 0.03936761305241755, - "nauc_mrr_at_10_diff1": 0.08232431911613593, - "nauc_mrr_at_10_max": 0.2976966640972928, - "nauc_mrr_at_10_std": 0.03936761305241755, - "nauc_mrr_at_1_diff1": 0.04326570520655111, - "nauc_mrr_at_1_max": 0.3224325173726293, - "nauc_mrr_at_1_std": 0.0345532004050703, - "nauc_mrr_at_20_diff1": 0.08232431911613593, - "nauc_mrr_at_20_max": 0.2976966640972928, - "nauc_mrr_at_20_std": 0.03936761305241755, - "nauc_mrr_at_3_diff1": 0.09841645170536006, - "nauc_mrr_at_3_max": 0.28575011259877825, - "nauc_mrr_at_3_std": 0.038641853990093455, - "nauc_mrr_at_5_diff1": 0.08767292202499989, - "nauc_mrr_at_5_max": 0.27459805882788724, - "nauc_mrr_at_5_std": 0.025865536256733646, - "nauc_ndcg_at_1000_diff1": -0.17908397271463292, - "nauc_ndcg_at_1000_max": 0.32950118161580466, - "nauc_ndcg_at_1000_std": 0.6445234074751405, - "nauc_ndcg_at_100_diff1": -0.1622574037823723, - "nauc_ndcg_at_100_max": 0.2902925476213027, - "nauc_ndcg_at_100_std": 0.5668207357528271, - "nauc_ndcg_at_10_diff1": 0.02841530472596724, - "nauc_ndcg_at_10_max": 0.31021101854725536, - "nauc_ndcg_at_10_std": 0.25227525115585936, - "nauc_ndcg_at_1_diff1": 0.04326570520655111, - "nauc_ndcg_at_1_max": 0.3224325173726293, - "nauc_ndcg_at_1_std": 0.0345532004050703, - "nauc_ndcg_at_20_diff1": -0.04345210224440129, - "nauc_ndcg_at_20_max": 0.3060058864100404, - "nauc_ndcg_at_20_std": 0.3203500202379029, - "nauc_ndcg_at_3_diff1": -0.027311698752615493, - "nauc_ndcg_at_3_max": 0.3062961102473417, - "nauc_ndcg_at_3_std": 0.15546986295776838, - "nauc_ndcg_at_5_diff1": 0.054176844078444426, - "nauc_ndcg_at_5_max": 0.29970785169971287, - "nauc_ndcg_at_5_std": 0.20921915554766976, - "nauc_precision_at_1000_diff1": -0.22031096869135272, - "nauc_precision_at_1000_max": 0.2986648920876661, - "nauc_precision_at_1000_std": 0.7259021291856423, - "nauc_precision_at_100_diff1": -0.1885686833994847, - "nauc_precision_at_100_max": 0.27498526176589005, - "nauc_precision_at_100_std": 0.6104621768161939, - "nauc_precision_at_10_diff1": 0.02876131690490984, - "nauc_precision_at_10_max": 0.306889992185712, - "nauc_precision_at_10_std": 0.28382126591470597, - "nauc_precision_at_1_diff1": 0.04326570520655111, - "nauc_precision_at_1_max": 0.3224325173726293, - "nauc_precision_at_1_std": 0.0345532004050703, - "nauc_precision_at_20_diff1": -0.06798464537403294, - "nauc_precision_at_20_max": 0.3008290397432531, - "nauc_precision_at_20_std": 0.3542598081165615, - "nauc_precision_at_3_diff1": -0.0445238095238101, - "nauc_precision_at_3_max": 0.30330827067669053, - "nauc_precision_at_3_std": 0.1895989974937346, - "nauc_precision_at_5_diff1": 0.07960984706120275, - "nauc_precision_at_5_max": 0.2913570788400933, - "nauc_precision_at_5_std": 0.2503577727023061, - "nauc_recall_at_1000_diff1": -0.22031096869135416, - "nauc_recall_at_1000_max": 0.29866489208766495, - "nauc_recall_at_1000_std": 0.7259021291856401, - "nauc_recall_at_100_diff1": -0.18856868339948582, - "nauc_recall_at_100_max": 0.2749852617658886, - "nauc_recall_at_100_std": 0.610462176816193, - "nauc_recall_at_10_diff1": 0.02876131690490984, - "nauc_recall_at_10_max": 0.306889992185712, - "nauc_recall_at_10_std": 0.28382126591470597, - "nauc_recall_at_1_diff1": 0.0432657052065505, - "nauc_recall_at_1_max": 0.3224325173726305, - "nauc_recall_at_1_std": 0.03455320040506908, - "nauc_recall_at_20_diff1": -0.06798464537403294, - "nauc_recall_at_20_max": 0.3008290397432531, - "nauc_recall_at_20_std": 0.3542598081165615, - "nauc_recall_at_3_diff1": -0.04452380952380912, - "nauc_recall_at_3_max": 0.303308270676692, - "nauc_recall_at_3_std": 0.18959899749373313, - "nauc_recall_at_5_diff1": 0.07960984706120275, - "nauc_recall_at_5_max": 0.2913570788400933, - "nauc_recall_at_5_std": 0.2503577727023061, - "ndcg_at_1": 0.892, - "ndcg_at_10": 0.86686, - "ndcg_at_100": 0.68398, - "ndcg_at_1000": 0.83983, - "ndcg_at_20": 0.843, - "ndcg_at_3": 0.89368, - "ndcg_at_5": 0.88679, - "precision_at_1": 0.892, - "precision_at_10": 0.8564, - "precision_at_100": 0.51112, - "precision_at_1000": 0.07312, - "precision_at_20": 0.827, - "precision_at_3": 0.89333, - "precision_at_5": 0.8832, - "recall_at_1": 0.01115, - "recall_at_10": 0.10705, - "recall_at_100": 0.6389, - "recall_at_1000": 0.914, - "recall_at_20": 0.20675, - "recall_at_3": 0.0335, - "recall_at_5": 0.0552 - } - ] - }, - "task_name": "TUBerlinT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TinyImageNetClustering.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TinyImageNetClustering.json deleted file mode 100644 index ec2d58ec81..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/TinyImageNetClustering.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "5a77092c28e51558c5586e9c5eb71a7e17a5e43f", - "evaluation_time": 401.41785073280334, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "valid": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6290288786644828, - "v_measure": 0.6290288786644828 - } - ] - }, - "task_name": "TinyImageNetClustering" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json deleted file mode 100644 index 50150f4e71..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 65.64880728721619, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "accuracy": 0.523546042003231, - "f1": 0.6886687471443521, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "lrap": 0.6682884019924772, - "main_score": 0.523546042003231, - "scores_per_experiment": [ - { - "accuracy": 0.5351373182552503, - "f1": 0.7049899989401454, - "lrap": 0.6795329608688017 - }, - { - "accuracy": 0.5300888529886915, - "f1": 0.6875495899200211, - "lrap": 0.6688218003949185 - }, - { - "accuracy": 0.5119143780290791, - "f1": 0.673686551264662, - "lrap": 0.6658084275713686 - }, - { - "accuracy": 0.5216074313408724, - "f1": 0.6802113731239721, - "lrap": 0.6663693681565408 - }, - { - "accuracy": 0.5189822294022617, - "f1": 0.6969062224729599, - "lrap": 0.6609094529707563 - } - ] - } - ] - }, - "task_name": "VOC2007" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreArxivQARetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreArxivQARetrieval.json deleted file mode 100644 index 8a05dc0907..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreArxivQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", - "evaluation_time": 11.539915561676025, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.198, - "cv_recall_at_10": 0.366, - "cv_recall_at_100": 0.716, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.444, - "cv_recall_at_3": 0.28, - "cv_recall_at_5": 0.318, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.262, - "map_at_1": 0.198, - "map_at_10": 0.25016, - "map_at_100": 0.26189, - "map_at_1000": 0.26345, - "map_at_20": 0.25574, - "map_at_3": 0.23467, - "map_at_5": 0.24337, - "mrr_at_1": 0.198, - "mrr_at_10": 0.2501595238095238, - "mrr_at_100": 0.2618894833637717, - "mrr_at_1000": 0.2634476289251727, - "mrr_at_20": 0.2557436838874919, - "mrr_at_3": 0.23466666666666666, - "mrr_at_5": 0.24336666666666668, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2685481985413355, - "nauc_cv_recall_at_100_max": 0.25195531990315806, - "nauc_cv_recall_at_100_std": 0.06048858485371907, - "nauc_cv_recall_at_10_diff1": 0.4109315183914633, - "nauc_cv_recall_at_10_max": 0.32723380983407496, - "nauc_cv_recall_at_10_std": -0.01634389694395826, - "nauc_cv_recall_at_1_diff1": 0.5591586841257287, - "nauc_cv_recall_at_1_max": 0.30669556981183665, - "nauc_cv_recall_at_1_std": -0.06708263760728826, - "nauc_cv_recall_at_20_diff1": 0.3963498460589525, - "nauc_cv_recall_at_20_max": 0.3259290041660787, - "nauc_cv_recall_at_20_std": 0.0020150856999482376, - "nauc_cv_recall_at_3_diff1": 0.4703111905693055, - "nauc_cv_recall_at_3_max": 0.3366472987317727, - "nauc_cv_recall_at_3_std": 0.0015140510643050556, - "nauc_cv_recall_at_5_diff1": 0.4413490717899683, - "nauc_cv_recall_at_5_max": 0.34234892983316073, - "nauc_cv_recall_at_5_std": -0.00039372925890506795, - "nauc_map_at_1000_diff1": 0.5023400105417865, - "nauc_map_at_1000_max": 0.321188547000857, - "nauc_map_at_1000_std": -0.034383841529415766, - "nauc_map_at_100_diff1": 0.5016743216328943, - "nauc_map_at_100_max": 0.3209011408009578, - "nauc_map_at_100_std": -0.034423076879957805, - "nauc_map_at_10_diff1": 0.5044594445998427, - "nauc_map_at_10_max": 0.3232179253193846, - "nauc_map_at_10_std": -0.03552375018397788, - "nauc_map_at_1_diff1": 0.5591586841257287, - "nauc_map_at_1_max": 0.30669556981183665, - "nauc_map_at_1_std": -0.06708263760728826, - "nauc_map_at_20_diff1": 0.5034779946433506, - "nauc_map_at_20_max": 0.3231314045091189, - "nauc_map_at_20_std": -0.03407625748520336, - "nauc_map_at_3_diff1": 0.5189393862869124, - "nauc_map_at_3_max": 0.3252757143116887, - "nauc_map_at_3_std": -0.034379690779757784, - "nauc_map_at_5_diff1": 0.5103116101493478, - "nauc_map_at_5_max": 0.3256348361969301, - "nauc_map_at_5_std": -0.033658858852439, - "nauc_mrr_at_1000_diff1": 0.5023400105417865, - "nauc_mrr_at_1000_max": 0.321188547000857, - "nauc_mrr_at_1000_std": -0.034383841529415766, - "nauc_mrr_at_100_diff1": 0.5016743216328943, - "nauc_mrr_at_100_max": 0.3209011408009578, - "nauc_mrr_at_100_std": -0.034423076879957805, - "nauc_mrr_at_10_diff1": 0.5044594445998427, - "nauc_mrr_at_10_max": 0.3232179253193846, - "nauc_mrr_at_10_std": -0.03552375018397788, - "nauc_mrr_at_1_diff1": 0.5591586841257287, - "nauc_mrr_at_1_max": 0.30669556981183665, - "nauc_mrr_at_1_std": -0.06708263760728826, - "nauc_mrr_at_20_diff1": 0.5034779946433506, - "nauc_mrr_at_20_max": 0.3231314045091189, - "nauc_mrr_at_20_std": -0.03407625748520336, - "nauc_mrr_at_3_diff1": 0.5189393862869124, - "nauc_mrr_at_3_max": 0.3252757143116887, - "nauc_mrr_at_3_std": -0.034379690779757784, - "nauc_mrr_at_5_diff1": 0.5103116101493478, - "nauc_mrr_at_5_max": 0.3256348361969301, - "nauc_mrr_at_5_std": -0.033658858852439, - "nauc_ndcg_at_1000_diff1": 0.4823895239759644, - "nauc_ndcg_at_1000_max": 0.3190692347281697, - "nauc_ndcg_at_1000_std": -0.026762242805733943, - "nauc_ndcg_at_100_diff1": 0.4613737853231418, - "nauc_ndcg_at_100_max": 0.3119611918607358, - "nauc_ndcg_at_100_std": -0.019659627911515334, - "nauc_ndcg_at_10_diff1": 0.47935455871116944, - "nauc_ndcg_at_10_max": 0.3247619541581183, - "nauc_ndcg_at_10_std": -0.02933370691404022, - "nauc_ndcg_at_1_diff1": 0.5591586841257287, - "nauc_ndcg_at_1_max": 0.30669556981183665, - "nauc_ndcg_at_1_std": -0.06708263760728826, - "nauc_ndcg_at_20_diff1": 0.4754922425300419, - "nauc_ndcg_at_20_max": 0.3244000467022679, - "nauc_ndcg_at_20_std": -0.024220056034281272, - "nauc_ndcg_at_3_diff1": 0.5057760785119431, - "nauc_ndcg_at_3_max": 0.3285522047251961, - "nauc_ndcg_at_3_std": -0.02459832888889593, - "nauc_ndcg_at_5_diff1": 0.4915665657301162, - "nauc_ndcg_at_5_max": 0.33021385983758084, - "nauc_ndcg_at_5_std": -0.024268471559681672, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.26854819854133494, - "nauc_precision_at_100_max": 0.25195531990315767, - "nauc_precision_at_100_std": 0.060488584853718386, - "nauc_precision_at_10_diff1": 0.41093151839146325, - "nauc_precision_at_10_max": 0.327233809834075, - "nauc_precision_at_10_std": -0.016343896943957985, - "nauc_precision_at_1_diff1": 0.5591586841257287, - "nauc_precision_at_1_max": 0.30669556981183665, - "nauc_precision_at_1_std": -0.06708263760728826, - "nauc_precision_at_20_diff1": 0.3963498460589526, - "nauc_precision_at_20_max": 0.32592900416607873, - "nauc_precision_at_20_std": 0.002015085699948558, - "nauc_precision_at_3_diff1": 0.4703111905693055, - "nauc_precision_at_3_max": 0.3366472987317729, - "nauc_precision_at_3_std": 0.0015140510643054945, - "nauc_precision_at_5_diff1": 0.44134907178996824, - "nauc_precision_at_5_max": 0.3423489298331607, - "nauc_precision_at_5_std": -0.00039372925890499764, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2685481985413355, - "nauc_recall_at_100_max": 0.25195531990315806, - "nauc_recall_at_100_std": 0.06048858485371907, - "nauc_recall_at_10_diff1": 0.4109315183914633, - "nauc_recall_at_10_max": 0.32723380983407496, - "nauc_recall_at_10_std": -0.01634389694395826, - "nauc_recall_at_1_diff1": 0.5591586841257287, - "nauc_recall_at_1_max": 0.30669556981183665, - "nauc_recall_at_1_std": -0.06708263760728826, - "nauc_recall_at_20_diff1": 0.3963498460589525, - "nauc_recall_at_20_max": 0.3259290041660787, - "nauc_recall_at_20_std": 0.0020150856999482376, - "nauc_recall_at_3_diff1": 0.4703111905693055, - "nauc_recall_at_3_max": 0.3366472987317727, - "nauc_recall_at_3_std": 0.0015140510643050556, - "nauc_recall_at_5_diff1": 0.4413490717899683, - "nauc_recall_at_5_max": 0.34234892983316073, - "nauc_recall_at_5_std": -0.00039372925890506795, - "ndcg_at_1": 0.198, - "ndcg_at_10": 0.27792, - "ndcg_at_100": 0.34666, - "ndcg_at_1000": 0.38404, - "ndcg_at_20": 0.29785, - "ndcg_at_3": 0.24633, - "ndcg_at_5": 0.262, - "precision_at_1": 0.198, - "precision_at_10": 0.0366, - "precision_at_100": 0.00716, - "precision_at_1000": 0.001, - "precision_at_20": 0.0222, - "precision_at_3": 0.09333, - "precision_at_5": 0.0636, - "recall_at_1": 0.198, - "recall_at_10": 0.366, - "recall_at_100": 0.716, - "recall_at_1000": 1.0, - "recall_at_20": 0.444, - "recall_at_3": 0.28, - "recall_at_5": 0.318 - } - ] - }, - "task_name": "VidoreArxivQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreDocVQARetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreDocVQARetrieval.json deleted file mode 100644 index ad62f9a1d4..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreDocVQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "b1d89eda849e636676df6ead8002602fb1858600", - "evaluation_time": 16.344624996185303, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.098, - "cv_recall_at_10": 0.246, - "cv_recall_at_100": 0.608, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.332, - "cv_recall_at_3": 0.16, - "cv_recall_at_5": 0.19, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.14602, - "map_at_1": 0.098, - "map_at_10": 0.13877, - "map_at_100": 0.15117, - "map_at_1000": 0.15306, - "map_at_20": 0.14451, - "map_at_3": 0.12467, - "map_at_5": 0.13147, - "mrr_at_1": 0.098, - "mrr_at_10": 0.13876746031746037, - "mrr_at_100": 0.15116731214168472, - "mrr_at_1000": 0.1530579249837991, - "mrr_at_20": 0.14450965011081118, - "mrr_at_3": 0.1246666666666667, - "mrr_at_5": 0.13146666666666668, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.18735699318384363, - "nauc_cv_recall_at_100_max": 0.14111611551337905, - "nauc_cv_recall_at_100_std": 0.42355006141766416, - "nauc_cv_recall_at_10_diff1": 0.3071288237751918, - "nauc_cv_recall_at_10_max": 0.1993356269317047, - "nauc_cv_recall_at_10_std": 0.26737311867149083, - "nauc_cv_recall_at_1_diff1": 0.4351971000366878, - "nauc_cv_recall_at_1_max": 0.19066601687629722, - "nauc_cv_recall_at_1_std": 0.22446789287220115, - "nauc_cv_recall_at_20_diff1": 0.22081974452146985, - "nauc_cv_recall_at_20_max": 0.14423736839025114, - "nauc_cv_recall_at_20_std": 0.2758524374737312, - "nauc_cv_recall_at_3_diff1": 0.3795897987402058, - "nauc_cv_recall_at_3_max": 0.22304501459517587, - "nauc_cv_recall_at_3_std": 0.23199416192963584, - "nauc_cv_recall_at_5_diff1": 0.3728078287626318, - "nauc_cv_recall_at_5_max": 0.23064920215896276, - "nauc_cv_recall_at_5_std": 0.2503690750424564, - "nauc_map_at_1000_diff1": 0.38116462145375835, - "nauc_map_at_1000_max": 0.19729004096849725, - "nauc_map_at_1000_std": 0.23852591704752155, - "nauc_map_at_100_diff1": 0.3800813688282042, - "nauc_map_at_100_max": 0.1968220239656418, - "nauc_map_at_100_std": 0.2396456603690272, - "nauc_map_at_10_diff1": 0.3932756999618319, - "nauc_map_at_10_max": 0.2036729162444215, - "nauc_map_at_10_std": 0.23368952621988348, - "nauc_map_at_1_diff1": 0.4351971000366878, - "nauc_map_at_1_max": 0.19066601687629722, - "nauc_map_at_1_std": 0.22446789287220115, - "nauc_map_at_20_diff1": 0.3830786234108412, - "nauc_map_at_20_max": 0.19759003746990553, - "nauc_map_at_20_std": 0.23508413618482943, - "nauc_map_at_3_diff1": 0.4092011738960796, - "nauc_map_at_3_max": 0.20665475087093785, - "nauc_map_at_3_std": 0.2255308517840538, - "nauc_map_at_5_diff1": 0.4066964428867831, - "nauc_map_at_5_max": 0.21030017113953264, - "nauc_map_at_5_std": 0.23113798288129628, - "nauc_mrr_at_1000_diff1": 0.38116462145375835, - "nauc_mrr_at_1000_max": 0.19729004096849725, - "nauc_mrr_at_1000_std": 0.23852591704752155, - "nauc_mrr_at_100_diff1": 0.3800813688282042, - "nauc_mrr_at_100_max": 0.1968220239656418, - "nauc_mrr_at_100_std": 0.2396456603690272, - "nauc_mrr_at_10_diff1": 0.3932756999618319, - "nauc_mrr_at_10_max": 0.2036729162444215, - "nauc_mrr_at_10_std": 0.23368952621988348, - "nauc_mrr_at_1_diff1": 0.4351971000366878, - "nauc_mrr_at_1_max": 0.19066601687629722, - "nauc_mrr_at_1_std": 0.22446789287220115, - "nauc_mrr_at_20_diff1": 0.3830786234108412, - "nauc_mrr_at_20_max": 0.19759003746990553, - "nauc_mrr_at_20_std": 0.23508413618482943, - "nauc_mrr_at_3_diff1": 0.4092011738960796, - "nauc_mrr_at_3_max": 0.20665475087093785, - "nauc_mrr_at_3_std": 0.2255308517840538, - "nauc_mrr_at_5_diff1": 0.4066964428867831, - "nauc_mrr_at_5_max": 0.21030017113953264, - "nauc_mrr_at_5_std": 0.23113798288129628, - "nauc_ndcg_at_1000_diff1": 0.3523903000677059, - "nauc_ndcg_at_1000_max": 0.19100960870984232, - "nauc_ndcg_at_1000_std": 0.2535966606069548, - "nauc_ndcg_at_100_diff1": 0.3262818126767828, - "nauc_ndcg_at_100_max": 0.18243597433523048, - "nauc_ndcg_at_100_std": 0.28083778118767516, - "nauc_ndcg_at_10_diff1": 0.36743814573481537, - "nauc_ndcg_at_10_max": 0.2030744688835133, - "nauc_ndcg_at_10_std": 0.24356124176904942, - "nauc_ndcg_at_1_diff1": 0.4351971000366878, - "nauc_ndcg_at_1_max": 0.19066601687629722, - "nauc_ndcg_at_1_std": 0.22446789287220115, - "nauc_ndcg_at_20_diff1": 0.334829441547401, - "nauc_ndcg_at_20_max": 0.1833490765926271, - "nauc_ndcg_at_20_std": 0.2475227625096276, - "nauc_ndcg_at_3_diff1": 0.40063489059083385, - "nauc_ndcg_at_3_max": 0.21142675686936951, - "nauc_ndcg_at_3_std": 0.22731281521503882, - "nauc_ndcg_at_5_diff1": 0.3964257193277851, - "nauc_ndcg_at_5_max": 0.21658505356493776, - "nauc_ndcg_at_5_std": 0.23672134038325543, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.18735699318384358, - "nauc_precision_at_100_max": 0.14111611551337946, - "nauc_precision_at_100_std": 0.42355006141766455, - "nauc_precision_at_10_diff1": 0.3071288237751922, - "nauc_precision_at_10_max": 0.19933562693170523, - "nauc_precision_at_10_std": 0.26737311867149116, - "nauc_precision_at_1_diff1": 0.4351971000366878, - "nauc_precision_at_1_max": 0.19066601687629722, - "nauc_precision_at_1_std": 0.22446789287220115, - "nauc_precision_at_20_diff1": 0.22081974452147027, - "nauc_precision_at_20_max": 0.14423736839025153, - "nauc_precision_at_20_std": 0.2758524374737314, - "nauc_precision_at_3_diff1": 0.3795897987402061, - "nauc_precision_at_3_max": 0.2230450145951759, - "nauc_precision_at_3_std": 0.231994161929636, - "nauc_precision_at_5_diff1": 0.372807828762632, - "nauc_precision_at_5_max": 0.23064920215896323, - "nauc_precision_at_5_std": 0.25036907504245653, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.18735699318384363, - "nauc_recall_at_100_max": 0.14111611551337905, - "nauc_recall_at_100_std": 0.42355006141766416, - "nauc_recall_at_10_diff1": 0.3071288237751918, - "nauc_recall_at_10_max": 0.1993356269317047, - "nauc_recall_at_10_std": 0.26737311867149083, - "nauc_recall_at_1_diff1": 0.4351971000366878, - "nauc_recall_at_1_max": 0.19066601687629722, - "nauc_recall_at_1_std": 0.22446789287220115, - "nauc_recall_at_20_diff1": 0.22081974452146985, - "nauc_recall_at_20_max": 0.14423736839025114, - "nauc_recall_at_20_std": 0.2758524374737312, - "nauc_recall_at_3_diff1": 0.3795897987402058, - "nauc_recall_at_3_max": 0.22304501459517587, - "nauc_recall_at_3_std": 0.23199416192963584, - "nauc_recall_at_5_diff1": 0.3728078287626318, - "nauc_recall_at_5_max": 0.23064920215896276, - "nauc_recall_at_5_std": 0.2503690750424564, - "ndcg_at_1": 0.098, - "ndcg_at_10": 0.16395, - "ndcg_at_100": 0.23576, - "ndcg_at_1000": 0.28608, - "ndcg_at_20": 0.18543, - "ndcg_at_3": 0.13371, - "ndcg_at_5": 0.14602, - "precision_at_1": 0.098, - "precision_at_10": 0.0246, - "precision_at_100": 0.00608, - "precision_at_1000": 0.001, - "precision_at_20": 0.0166, - "precision_at_3": 0.05333, - "precision_at_5": 0.038, - "recall_at_1": 0.098, - "recall_at_10": 0.246, - "recall_at_100": 0.608, - "recall_at_1000": 1.0, - "recall_at_20": 0.332, - "recall_at_3": 0.16, - "recall_at_5": 0.19 - } - ] - }, - "task_name": "VidoreDocVQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreInfoVQARetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreInfoVQARetrieval.json deleted file mode 100644 index e29196c1c3..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreInfoVQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", - "evaluation_time": 17.864064693450928, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.392, - "cv_recall_at_10": 0.716, - "cv_recall_at_100": 0.922, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.784, - "cv_recall_at_3": 0.564, - "cv_recall_at_5": 0.624, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5185, - "map_at_1": 0.392, - "map_at_10": 0.4957, - "map_at_100": 0.50427, - "map_at_1000": 0.50467, - "map_at_20": 0.5007, - "map_at_3": 0.46933, - "map_at_5": 0.48313, - "mrr_at_1": 0.392, - "mrr_at_10": 0.4957047619047617, - "mrr_at_100": 0.5042708815113673, - "mrr_at_1000": 0.5046663761343821, - "mrr_at_20": 0.5007034324722403, - "mrr_at_3": 0.4693333333333335, - "mrr_at_5": 0.4831333333333333, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.43401805166511004, - "nauc_cv_recall_at_100_max": 0.4512916277622163, - "nauc_cv_recall_at_100_std": 0.5630012688836221, - "nauc_cv_recall_at_10_diff1": 0.4999622889121859, - "nauc_cv_recall_at_10_max": 0.3837631140307873, - "nauc_cv_recall_at_10_std": 0.21789089428073674, - "nauc_cv_recall_at_1_diff1": 0.6361117715774349, - "nauc_cv_recall_at_1_max": 0.38531576607776064, - "nauc_cv_recall_at_1_std": 0.059550515336028, - "nauc_cv_recall_at_20_diff1": 0.44793391539966904, - "nauc_cv_recall_at_20_max": 0.3685975839229269, - "nauc_cv_recall_at_20_std": 0.2803279768177026, - "nauc_cv_recall_at_3_diff1": 0.5806573026979864, - "nauc_cv_recall_at_3_max": 0.3603360079382735, - "nauc_cv_recall_at_3_std": 0.13183122045245887, - "nauc_cv_recall_at_5_diff1": 0.5356571704849744, - "nauc_cv_recall_at_5_max": 0.3265211246652782, - "nauc_cv_recall_at_5_std": 0.10480821680055523, - "nauc_map_at_1000_diff1": 0.5943168242782374, - "nauc_map_at_1000_max": 0.37326847099761423, - "nauc_map_at_1000_std": 0.10119065227206148, - "nauc_map_at_100_diff1": 0.594054309790089, - "nauc_map_at_100_max": 0.3733454665922043, - "nauc_map_at_100_std": 0.10165465316992746, - "nauc_map_at_10_diff1": 0.5946248202240149, - "nauc_map_at_10_max": 0.37243402579793333, - "nauc_map_at_10_std": 0.09948573630126686, - "nauc_map_at_1_diff1": 0.6361117715774349, - "nauc_map_at_1_max": 0.38531576607776064, - "nauc_map_at_1_std": 0.059550515336028, - "nauc_map_at_20_diff1": 0.592838076133879, - "nauc_map_at_20_max": 0.37203415673242324, - "nauc_map_at_20_std": 0.10081523194886777, - "nauc_map_at_3_diff1": 0.6067771499819062, - "nauc_map_at_3_max": 0.3721820842902545, - "nauc_map_at_3_std": 0.09344892818883076, - "nauc_map_at_5_diff1": 0.5979515676651626, - "nauc_map_at_5_max": 0.3662453984051962, - "nauc_map_at_5_std": 0.08812192427868058, - "nauc_mrr_at_1000_diff1": 0.5943168242782374, - "nauc_mrr_at_1000_max": 0.37326847099761423, - "nauc_mrr_at_1000_std": 0.10119065227206148, - "nauc_mrr_at_100_diff1": 0.594054309790089, - "nauc_mrr_at_100_max": 0.3733454665922043, - "nauc_mrr_at_100_std": 0.10165465316992746, - "nauc_mrr_at_10_diff1": 0.5946248202240149, - "nauc_mrr_at_10_max": 0.37243402579793333, - "nauc_mrr_at_10_std": 0.09948573630126686, - "nauc_mrr_at_1_diff1": 0.6361117715774349, - "nauc_mrr_at_1_max": 0.38531576607776064, - "nauc_mrr_at_1_std": 0.059550515336028, - "nauc_mrr_at_20_diff1": 0.592838076133879, - "nauc_mrr_at_20_max": 0.37203415673242324, - "nauc_mrr_at_20_std": 0.10081523194886777, - "nauc_mrr_at_3_diff1": 0.6067771499819062, - "nauc_mrr_at_3_max": 0.3721820842902545, - "nauc_mrr_at_3_std": 0.09344892818883076, - "nauc_mrr_at_5_diff1": 0.5979515676651626, - "nauc_mrr_at_5_max": 0.3662453984051962, - "nauc_mrr_at_5_std": 0.08812192427868058, - "nauc_ndcg_at_1000_diff1": 0.5844068303992299, - "nauc_ndcg_at_1000_max": 0.37439145340533564, - "nauc_ndcg_at_1000_std": 0.11647849865946318, - "nauc_ndcg_at_100_diff1": 0.5785820083831996, - "nauc_ndcg_at_100_max": 0.3770927197960715, - "nauc_ndcg_at_100_std": 0.1320926286188547, - "nauc_ndcg_at_10_diff1": 0.5760468027551768, - "nauc_ndcg_at_10_max": 0.37332452932466365, - "nauc_ndcg_at_10_std": 0.12141024137245926, - "nauc_ndcg_at_1_diff1": 0.6361117715774349, - "nauc_ndcg_at_1_max": 0.38531576607776064, - "nauc_ndcg_at_1_std": 0.059550515336028, - "nauc_ndcg_at_20_diff1": 0.5696852943787447, - "nauc_ndcg_at_20_max": 0.37086868649526816, - "nauc_ndcg_at_20_std": 0.12776127821364855, - "nauc_ndcg_at_3_diff1": 0.60009384166449, - "nauc_ndcg_at_3_max": 0.3691552827726549, - "nauc_ndcg_at_3_std": 0.10303937890816356, - "nauc_ndcg_at_5_diff1": 0.5837010291294129, - "nauc_ndcg_at_5_max": 0.35739896590967324, - "nauc_ndcg_at_5_std": 0.09270634380545034, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.4340180516651088, - "nauc_precision_at_100_max": 0.45129162776221715, - "nauc_precision_at_100_std": 0.5630012688836226, - "nauc_precision_at_10_diff1": 0.49996228891218547, - "nauc_precision_at_10_max": 0.3837631140307867, - "nauc_precision_at_10_std": 0.21789089428073644, - "nauc_precision_at_1_diff1": 0.6361117715774349, - "nauc_precision_at_1_max": 0.38531576607776064, - "nauc_precision_at_1_std": 0.059550515336028, - "nauc_precision_at_20_diff1": 0.4479339153996688, - "nauc_precision_at_20_max": 0.3685975839229259, - "nauc_precision_at_20_std": 0.2803279768177021, - "nauc_precision_at_3_diff1": 0.5806573026979863, - "nauc_precision_at_3_max": 0.36033600793827303, - "nauc_precision_at_3_std": 0.13183122045245868, - "nauc_precision_at_5_diff1": 0.5356571704849745, - "nauc_precision_at_5_max": 0.326521124665278, - "nauc_precision_at_5_std": 0.10480821680055544, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.43401805166511004, - "nauc_recall_at_100_max": 0.4512916277622163, - "nauc_recall_at_100_std": 0.5630012688836221, - "nauc_recall_at_10_diff1": 0.4999622889121859, - "nauc_recall_at_10_max": 0.3837631140307873, - "nauc_recall_at_10_std": 0.21789089428073674, - "nauc_recall_at_1_diff1": 0.6361117715774349, - "nauc_recall_at_1_max": 0.38531576607776064, - "nauc_recall_at_1_std": 0.059550515336028, - "nauc_recall_at_20_diff1": 0.44793391539966904, - "nauc_recall_at_20_max": 0.3685975839229269, - "nauc_recall_at_20_std": 0.2803279768177026, - "nauc_recall_at_3_diff1": 0.5806573026979864, - "nauc_recall_at_3_max": 0.3603360079382735, - "nauc_recall_at_3_std": 0.13183122045245887, - "nauc_recall_at_5_diff1": 0.5356571704849744, - "nauc_recall_at_5_max": 0.3265211246652782, - "nauc_recall_at_5_std": 0.10480821680055523, - "ndcg_at_1": 0.392, - "ndcg_at_10": 0.54854, - "ndcg_at_100": 0.59176, - "ndcg_at_1000": 0.60188, - "ndcg_at_20": 0.56608, - "ndcg_at_3": 0.49371, - "ndcg_at_5": 0.5185, - "precision_at_1": 0.392, - "precision_at_10": 0.0716, - "precision_at_100": 0.00922, - "precision_at_1000": 0.001, - "precision_at_20": 0.0392, - "precision_at_3": 0.188, - "precision_at_5": 0.1248, - "recall_at_1": 0.392, - "recall_at_10": 0.716, - "recall_at_100": 0.922, - "recall_at_1000": 1.0, - "recall_at_20": 0.784, - "recall_at_3": 0.564, - "recall_at_5": 0.624 - } - ] - }, - "task_name": "VidoreInfoVQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreShiftProjectRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreShiftProjectRetrieval.json deleted file mode 100644 index 840a9706ec..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreShiftProjectRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", - "evaluation_time": 31.411139011383057, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.06, - "cv_recall_at_10": 0.12, - "cv_recall_at_100": 0.4, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.16, - "cv_recall_at_3": 0.08, - "cv_recall_at_5": 0.09, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.07518, - "map_at_1": 0.06, - "map_at_10": 0.07425, - "map_at_100": 0.08209, - "map_at_1000": 0.08411, - "map_at_20": 0.07705, - "map_at_3": 0.06833, - "map_at_5": 0.07033, - "mrr_at_1": 0.06, - "mrr_at_10": 0.07425, - "mrr_at_100": 0.08209457915336821, - "mrr_at_1000": 0.08410506965250342, - "mrr_at_20": 0.07704554655870446, - "mrr_at_3": 0.06833333333333333, - "mrr_at_5": 0.07033333333333333, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.08204170957775479, - "nauc_cv_recall_at_100_max": -0.16466271884655007, - "nauc_cv_recall_at_100_std": 0.041310504634397276, - "nauc_cv_recall_at_10_diff1": 0.2861577101696523, - "nauc_cv_recall_at_10_max": -0.02165671500331717, - "nauc_cv_recall_at_10_std": 0.16344422329637, - "nauc_cv_recall_at_1_diff1": 0.4233968339146767, - "nauc_cv_recall_at_1_max": 0.10759323852964853, - "nauc_cv_recall_at_1_std": 0.23982649136928724, - "nauc_cv_recall_at_20_diff1": 0.2683976033184821, - "nauc_cv_recall_at_20_max": -0.05797741588569673, - "nauc_cv_recall_at_20_std": 0.05742049469964647, - "nauc_cv_recall_at_3_diff1": 0.4125972632143814, - "nauc_cv_recall_at_3_max": -0.06298631607190777, - "nauc_cv_recall_at_3_std": 0.0455795546015561, - "nauc_cv_recall_at_5_diff1": 0.3028947917598306, - "nauc_cv_recall_at_5_max": -0.06382851861789338, - "nauc_cv_recall_at_5_std": 0.1516262707569388, - "nauc_map_at_1000_diff1": 0.37410025658803814, - "nauc_map_at_1000_max": 0.01858867554088589, - "nauc_map_at_1000_std": 0.15072481017384068, - "nauc_map_at_100_diff1": 0.37370530168851285, - "nauc_map_at_100_max": 0.017909234398799562, - "nauc_map_at_100_std": 0.14943430467788055, - "nauc_map_at_10_diff1": 0.39419977929478384, - "nauc_map_at_10_max": 0.029271888806436316, - "nauc_map_at_10_std": 0.16444181569745317, - "nauc_map_at_1_diff1": 0.4233968339146767, - "nauc_map_at_1_max": 0.10759323852964853, - "nauc_map_at_1_std": 0.23982649136928724, - "nauc_map_at_20_diff1": 0.38728572779256304, - "nauc_map_at_20_max": 0.024433234986002847, - "nauc_map_at_20_std": 0.1509822547622029, - "nauc_map_at_3_diff1": 0.4250001636050705, - "nauc_map_at_3_max": 0.024383699699621193, - "nauc_map_at_3_std": 0.14415569967344424, - "nauc_map_at_5_diff1": 0.39657196047289084, - "nauc_map_at_5_max": 0.021683704286766203, - "nauc_map_at_5_std": 0.16849250442206679, - "nauc_mrr_at_1000_diff1": 0.37410025658803814, - "nauc_mrr_at_1000_max": 0.01858867554088589, - "nauc_mrr_at_1000_std": 0.15072481017384068, - "nauc_mrr_at_100_diff1": 0.37370530168851285, - "nauc_mrr_at_100_max": 0.017909234398799562, - "nauc_mrr_at_100_std": 0.14943430467788055, - "nauc_mrr_at_10_diff1": 0.39419977929478384, - "nauc_mrr_at_10_max": 0.029271888806436316, - "nauc_mrr_at_10_std": 0.16444181569745317, - "nauc_mrr_at_1_diff1": 0.4233968339146767, - "nauc_mrr_at_1_max": 0.10759323852964853, - "nauc_mrr_at_1_std": 0.23982649136928724, - "nauc_mrr_at_20_diff1": 0.38728572779256304, - "nauc_mrr_at_20_max": 0.024433234986002847, - "nauc_mrr_at_20_std": 0.1509822547622029, - "nauc_mrr_at_3_diff1": 0.4250001636050705, - "nauc_mrr_at_3_max": 0.024383699699621193, - "nauc_mrr_at_3_std": 0.14415569967344424, - "nauc_mrr_at_5_diff1": 0.39657196047289084, - "nauc_mrr_at_5_max": 0.021683704286766203, - "nauc_mrr_at_5_std": 0.16849250442206679, - "nauc_ndcg_at_1000_diff1": 0.31537358926976555, - "nauc_ndcg_at_1000_max": -0.016070596404182663, - "nauc_ndcg_at_1000_std": 0.12905512978048142, - "nauc_ndcg_at_100_diff1": 0.26988366255315605, - "nauc_ndcg_at_100_max": -0.04728349318783632, - "nauc_ndcg_at_100_std": 0.1074580962286981, - "nauc_ndcg_at_10_diff1": 0.3613398039023956, - "nauc_ndcg_at_10_max": 0.011330093365220005, - "nauc_ndcg_at_10_std": 0.16061426691048258, - "nauc_ndcg_at_1_diff1": 0.4233968339146767, - "nauc_ndcg_at_1_max": 0.10759323852964853, - "nauc_ndcg_at_1_std": 0.23982649136928724, - "nauc_ndcg_at_20_diff1": 0.3466254194336954, - "nauc_ndcg_at_20_max": -0.004343325835300405, - "nauc_ndcg_at_20_std": 0.11916220300381282, - "nauc_ndcg_at_3_diff1": 0.4217185598163835, - "nauc_ndcg_at_3_max": -0.0006190144676542667, - "nauc_ndcg_at_3_std": 0.11591042417905126, - "nauc_ndcg_at_5_diff1": 0.3704432033581334, - "nauc_ndcg_at_5_max": -0.004218379208381653, - "nauc_ndcg_at_5_std": 0.16140422923218145, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.08204170957775474, - "nauc_precision_at_100_max": -0.16466271884655004, - "nauc_precision_at_100_std": 0.041310504634397435, - "nauc_precision_at_10_diff1": 0.2861577101696519, - "nauc_precision_at_10_max": -0.02165671500331726, - "nauc_precision_at_10_std": 0.16344422329636993, - "nauc_precision_at_1_diff1": 0.4233968339146767, - "nauc_precision_at_1_max": 0.10759323852964853, - "nauc_precision_at_1_std": 0.23982649136928724, - "nauc_precision_at_20_diff1": 0.2683976033184821, - "nauc_precision_at_20_max": -0.05797741588569691, - "nauc_precision_at_20_std": 0.057420494699646496, - "nauc_precision_at_3_diff1": 0.41259726321438156, - "nauc_precision_at_3_max": -0.06298631607190772, - "nauc_precision_at_3_std": 0.04557955460155622, - "nauc_precision_at_5_diff1": 0.30289479175983075, - "nauc_precision_at_5_max": -0.0638285186178934, - "nauc_precision_at_5_std": 0.15162627075693896, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.08204170957775479, - "nauc_recall_at_100_max": -0.16466271884655007, - "nauc_recall_at_100_std": 0.041310504634397276, - "nauc_recall_at_10_diff1": 0.2861577101696523, - "nauc_recall_at_10_max": -0.02165671500331717, - "nauc_recall_at_10_std": 0.16344422329637, - "nauc_recall_at_1_diff1": 0.4233968339146767, - "nauc_recall_at_1_max": 0.10759323852964853, - "nauc_recall_at_1_std": 0.23982649136928724, - "nauc_recall_at_20_diff1": 0.2683976033184821, - "nauc_recall_at_20_max": -0.05797741588569673, - "nauc_recall_at_20_std": 0.05742049469964647, - "nauc_recall_at_3_diff1": 0.4125972632143814, - "nauc_recall_at_3_max": -0.06298631607190777, - "nauc_recall_at_3_std": 0.0455795546015561, - "nauc_recall_at_5_diff1": 0.3028947917598306, - "nauc_recall_at_5_max": -0.06382851861789338, - "nauc_recall_at_5_std": 0.1516262707569388, - "ndcg_at_1": 0.06, - "ndcg_at_10": 0.08479, - "ndcg_at_100": 0.13719, - "ndcg_at_1000": 0.20844, - "ndcg_at_20": 0.09493, - "ndcg_at_3": 0.07131, - "ndcg_at_5": 0.07518, - "precision_at_1": 0.06, - "precision_at_10": 0.012, - "precision_at_100": 0.004, - "precision_at_1000": 0.001, - "precision_at_20": 0.008, - "precision_at_3": 0.02667, - "precision_at_5": 0.018, - "recall_at_1": 0.06, - "recall_at_10": 0.12, - "recall_at_100": 0.4, - "recall_at_1000": 1.0, - "recall_at_20": 0.16, - "recall_at_3": 0.08, - "recall_at_5": 0.09 - } - ] - }, - "task_name": "VidoreShiftProjectRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAAIRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAAIRetrieval.json deleted file mode 100644 index 0884644b3a..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAAIRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", - "evaluation_time": 29.046104431152344, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.16, - "cv_recall_at_10": 0.38, - "cv_recall_at_100": 0.7, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.44, - "cv_recall_at_3": 0.23, - "cv_recall_at_5": 0.27, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.21534, - "map_at_1": 0.15, - "map_at_10": 0.21204, - "map_at_100": 0.22219, - "map_at_1000": 0.22351, - "map_at_20": 0.21614, - "map_at_3": 0.19, - "map_at_5": 0.197, - "mrr_at_1": 0.16, - "mrr_at_10": 0.21411904761904757, - "mrr_at_100": 0.22427953818055787, - "mrr_at_1000": 0.22560495232991717, - "mrr_at_20": 0.21819018889607122, - "mrr_at_3": 0.19, - "mrr_at_5": 0.19949999999999998, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.04487479733381322, - "nauc_cv_recall_at_100_max": 0.38578634480273827, - "nauc_cv_recall_at_100_std": 0.21624932444604553, - "nauc_cv_recall_at_10_diff1": 0.053816962545161474, - "nauc_cv_recall_at_10_max": 0.3068385621084916, - "nauc_cv_recall_at_10_std": 0.11257245340888422, - "nauc_cv_recall_at_1_diff1": -0.04307497311414982, - "nauc_cv_recall_at_1_max": 0.32458903057305266, - "nauc_cv_recall_at_1_std": 0.02273774773390678, - "nauc_cv_recall_at_20_diff1": 0.011758104087278764, - "nauc_cv_recall_at_20_max": 0.27744427624367063, - "nauc_cv_recall_at_20_std": 0.13502897113326723, - "nauc_cv_recall_at_3_diff1": -0.0619880721770566, - "nauc_cv_recall_at_3_max": 0.3106114915879866, - "nauc_cv_recall_at_3_std": 0.06766217720901142, - "nauc_cv_recall_at_5_diff1": 0.04367159641421238, - "nauc_cv_recall_at_5_max": 0.37758740712327205, - "nauc_cv_recall_at_5_std": 0.11848442228189042, - "nauc_map_at_1000_diff1": -0.011906178432101485, - "nauc_map_at_1000_max": 0.34347834826166185, - "nauc_map_at_1000_std": 0.06868788719712396, - "nauc_map_at_100_diff1": -0.011461416682847331, - "nauc_map_at_100_max": 0.3439169246976087, - "nauc_map_at_100_std": 0.06902969086073914, - "nauc_map_at_10_diff1": -0.006776007026008395, - "nauc_map_at_10_max": 0.34593482568066836, - "nauc_map_at_10_std": 0.06713240165892259, - "nauc_map_at_1_diff1": -0.002036085072069493, - "nauc_map_at_1_max": 0.3561939320633002, - "nauc_map_at_1_std": 0.029170446527567652, - "nauc_map_at_20_diff1": -0.011132446959779519, - "nauc_map_at_20_max": 0.3416196002890793, - "nauc_map_at_20_std": 0.06811661532008675, - "nauc_map_at_3_diff1": -0.0345761769966832, - "nauc_map_at_3_max": 0.34568362190229734, - "nauc_map_at_3_std": 0.05712912954633583, - "nauc_map_at_5_diff1": -0.008473691506968436, - "nauc_map_at_5_max": 0.3577332062251081, - "nauc_map_at_5_std": 0.0680679378630188, - "nauc_mrr_at_1000_diff1": -0.04170970645246049, - "nauc_mrr_at_1000_max": 0.31914921880257574, - "nauc_mrr_at_1000_std": 0.0616313297999736, - "nauc_mrr_at_100_diff1": -0.04115911433035388, - "nauc_mrr_at_100_max": 0.31967350861649435, - "nauc_mrr_at_100_std": 0.06200175938599041, - "nauc_mrr_at_10_diff1": -0.03629641083965268, - "nauc_mrr_at_10_max": 0.32191326930625896, - "nauc_mrr_at_10_std": 0.0601941819647312, - "nauc_mrr_at_1_diff1": -0.04307497311414982, - "nauc_mrr_at_1_max": 0.32458903057305266, - "nauc_mrr_at_1_std": 0.02273774773390678, - "nauc_mrr_at_20_diff1": -0.0405265668427798, - "nauc_mrr_at_20_max": 0.31763501020716245, - "nauc_mrr_at_20_std": 0.06120366174527031, - "nauc_mrr_at_3_diff1": -0.07018191812825324, - "nauc_mrr_at_3_max": 0.3105577791902151, - "nauc_mrr_at_3_std": 0.04388481381592876, - "nauc_mrr_at_5_diff1": -0.03854217469274165, - "nauc_mrr_at_5_max": 0.33172955029216, - "nauc_mrr_at_5_std": 0.059700767552818744, - "nauc_ndcg_at_1000_diff1": -0.008833068456348002, - "nauc_ndcg_at_1000_max": 0.3380747213218557, - "nauc_ndcg_at_1000_std": 0.08270631330777752, - "nauc_ndcg_at_100_diff1": -0.0014110765231264717, - "nauc_ndcg_at_100_max": 0.34496092616016066, - "nauc_ndcg_at_100_std": 0.09604559110919922, - "nauc_ndcg_at_10_diff1": 0.009681017763689211, - "nauc_ndcg_at_10_max": 0.3360390307609726, - "nauc_ndcg_at_10_std": 0.08059365276563066, - "nauc_ndcg_at_1_diff1": -0.002036085072069493, - "nauc_ndcg_at_1_max": 0.3561939320633002, - "nauc_ndcg_at_1_std": 0.029170446527567652, - "nauc_ndcg_at_20_diff1": -0.0038663194029288084, - "nauc_ndcg_at_20_max": 0.3251206918661345, - "nauc_ndcg_at_20_std": 0.08619257062666767, - "nauc_ndcg_at_3_diff1": -0.03739285196077174, - "nauc_ndcg_at_3_max": 0.344486029969986, - "nauc_ndcg_at_3_std": 0.06494039425865014, - "nauc_ndcg_at_5_diff1": 0.0052680993713773135, - "nauc_ndcg_at_5_max": 0.3631429134715458, - "nauc_ndcg_at_5_std": 0.0824333133139938, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.044874797333814595, - "nauc_precision_at_100_max": 0.3857863448027387, - "nauc_precision_at_100_std": 0.21624932444604655, - "nauc_precision_at_10_diff1": 0.05381696254516126, - "nauc_precision_at_10_max": 0.3068385621084915, - "nauc_precision_at_10_std": 0.11257245340888383, - "nauc_precision_at_1_diff1": -0.002036085072069493, - "nauc_precision_at_1_max": 0.3561939320633002, - "nauc_precision_at_1_std": 0.029170446527567652, - "nauc_precision_at_20_diff1": 0.01175810408727945, - "nauc_precision_at_20_max": 0.27744427624367113, - "nauc_precision_at_20_std": 0.1350289711332675, - "nauc_precision_at_3_diff1": -0.042789959261921945, - "nauc_precision_at_3_max": 0.34194524323028996, - "nauc_precision_at_3_std": 0.08451653486700217, - "nauc_precision_at_5_diff1": 0.04367159641421233, - "nauc_precision_at_5_max": 0.3775874071232719, - "nauc_precision_at_5_std": 0.11848442228189054, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.04487479733381322, - "nauc_recall_at_100_max": 0.38578634480273827, - "nauc_recall_at_100_std": 0.21624932444604553, - "nauc_recall_at_10_diff1": 0.053816962545161474, - "nauc_recall_at_10_max": 0.3068385621084916, - "nauc_recall_at_10_std": 0.11257245340888422, - "nauc_recall_at_1_diff1": -0.002036085072069493, - "nauc_recall_at_1_max": 0.3561939320633002, - "nauc_recall_at_1_std": 0.029170446527567652, - "nauc_recall_at_20_diff1": 0.011758104087278764, - "nauc_recall_at_20_max": 0.27744427624367063, - "nauc_recall_at_20_std": 0.13502897113326723, - "nauc_recall_at_3_diff1": -0.04278995926192189, - "nauc_recall_at_3_max": 0.3419452432302902, - "nauc_recall_at_3_std": 0.08451653486700222, - "nauc_recall_at_5_diff1": 0.04367159641421238, - "nauc_recall_at_5_max": 0.37758740712327205, - "nauc_recall_at_5_std": 0.11848442228189042, - "ndcg_at_1": 0.15, - "ndcg_at_10": 0.25127, - "ndcg_at_100": 0.31342, - "ndcg_at_1000": 0.35084, - "ndcg_at_20": 0.26638, - "ndcg_at_3": 0.20286, - "ndcg_at_5": 0.21534, - "precision_at_1": 0.15, - "precision_at_10": 0.038, - "precision_at_100": 0.007, - "precision_at_1000": 0.001, - "precision_at_20": 0.022, - "precision_at_3": 0.08, - "precision_at_5": 0.054, - "recall_at_1": 0.15, - "recall_at_10": 0.38, - "recall_at_100": 0.7, - "recall_at_1000": 1.0, - "recall_at_20": 0.44, - "recall_at_3": 0.24, - "recall_at_5": 0.27 - } - ] - }, - "task_name": "VidoreSyntheticDocQAAIRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAEnergyRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAEnergyRetrieval.json deleted file mode 100644 index 5e1e99143f..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAEnergyRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", - "evaluation_time": 28.03231382369995, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.16, - "cv_recall_at_10": 0.51, - "cv_recall_at_100": 0.83, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.59, - "cv_recall_at_3": 0.34, - "cv_recall_at_5": 0.4, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.27804, - "map_at_1": 0.13, - "map_at_10": 0.25134, - "map_at_100": 0.26306, - "map_at_1000": 0.26398, - "map_at_20": 0.2567, - "map_at_3": 0.22, - "map_at_5": 0.237, - "mrr_at_1": 0.16, - "mrr_at_10": 0.26677380952380947, - "mrr_at_100": 0.2784318900512844, - "mrr_at_1000": 0.27935058562216325, - "mrr_at_20": 0.2720564384445963, - "mrr_at_3": 0.2383333333333333, - "mrr_at_5": 0.2523333333333333, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.048383351715331105, - "nauc_cv_recall_at_100_max": 0.2678934231148774, - "nauc_cv_recall_at_100_std": 0.3938294537644398, - "nauc_cv_recall_at_10_diff1": -0.21793680922956815, - "nauc_cv_recall_at_10_max": 0.2884105380773526, - "nauc_cv_recall_at_10_std": 0.04301118694721539, - "nauc_cv_recall_at_1_diff1": -0.1065447841450301, - "nauc_cv_recall_at_1_max": 0.4035566139191886, - "nauc_cv_recall_at_1_std": -0.08908818558918431, - "nauc_cv_recall_at_20_diff1": -0.17946498011651307, - "nauc_cv_recall_at_20_max": 0.2712518526127342, - "nauc_cv_recall_at_20_std": 0.0032283152596590085, - "nauc_cv_recall_at_3_diff1": -0.24440234240440936, - "nauc_cv_recall_at_3_max": 0.3217626328201594, - "nauc_cv_recall_at_3_std": 0.022841092768753537, - "nauc_cv_recall_at_5_diff1": -0.17510298661174065, - "nauc_cv_recall_at_5_max": 0.33388259526261593, - "nauc_cv_recall_at_5_std": 0.047528321318228595, - "nauc_map_at_1000_diff1": -0.12118347269446686, - "nauc_map_at_1000_max": 0.40798998686985033, - "nauc_map_at_1000_std": 0.033248252345893674, - "nauc_map_at_100_diff1": -0.12178214546037262, - "nauc_map_at_100_max": 0.4069924113722096, - "nauc_map_at_100_std": 0.034050458694374126, - "nauc_map_at_10_diff1": -0.12829595983501216, - "nauc_map_at_10_max": 0.4085183545597175, - "nauc_map_at_10_std": 0.03090773118576432, - "nauc_map_at_1_diff1": 0.013386760985874355, - "nauc_map_at_1_max": 0.5047570011864522, - "nauc_map_at_1_std": 0.05936737480692162, - "nauc_map_at_20_diff1": -0.12576195523349035, - "nauc_map_at_20_max": 0.40697775682648785, - "nauc_map_at_20_std": 0.028255055376397914, - "nauc_map_at_3_diff1": -0.15234727486013372, - "nauc_map_at_3_max": 0.4206173299043492, - "nauc_map_at_3_std": 0.02386471304818614, - "nauc_map_at_5_diff1": -0.12167568813457902, - "nauc_map_at_5_max": 0.4171512252509577, - "nauc_map_at_5_std": 0.030444733533239992, - "nauc_mrr_at_1000_diff1": -0.1568288547366136, - "nauc_mrr_at_1000_max": 0.3562663780279398, - "nauc_mrr_at_1000_std": -0.025261925833143874, - "nauc_mrr_at_100_diff1": -0.1572874396536292, - "nauc_mrr_at_100_max": 0.3554807097494433, - "nauc_mrr_at_100_std": -0.024336608137159148, - "nauc_mrr_at_10_diff1": -0.16286454153286034, - "nauc_mrr_at_10_max": 0.3577889522492966, - "nauc_mrr_at_10_std": -0.026007433851469176, - "nauc_mrr_at_1_diff1": -0.1065447841450301, - "nauc_mrr_at_1_max": 0.4035566139191886, - "nauc_mrr_at_1_std": -0.08908818558918431, - "nauc_mrr_at_20_diff1": -0.16059994761425353, - "nauc_mrr_at_20_max": 0.3561607628353277, - "nauc_mrr_at_20_std": -0.029019937409929492, - "nauc_mrr_at_3_diff1": -0.17650657774505316, - "nauc_mrr_at_3_max": 0.36178524794668954, - "nauc_mrr_at_3_std": -0.03691059193084039, - "nauc_mrr_at_5_diff1": -0.15646123751391736, - "nauc_mrr_at_5_max": 0.36587355422576173, - "nauc_mrr_at_5_std": -0.027926371058717044, - "nauc_ndcg_at_1000_diff1": -0.12540293991394347, - "nauc_ndcg_at_1000_max": 0.388934244167919, - "nauc_ndcg_at_1000_std": 0.042555564548196194, - "nauc_ndcg_at_100_diff1": -0.12334124629330959, - "nauc_ndcg_at_100_max": 0.3765402027977492, - "nauc_ndcg_at_100_std": 0.06951668374021025, - "nauc_ndcg_at_10_diff1": -0.1554041086320955, - "nauc_ndcg_at_10_max": 0.3738909303116026, - "nauc_ndcg_at_10_std": 0.034099449761324147, - "nauc_ndcg_at_1_diff1": 0.013386760985874355, - "nauc_ndcg_at_1_max": 0.5047570011864522, - "nauc_ndcg_at_1_std": 0.05936737480692162, - "nauc_ndcg_at_20_diff1": -0.14450745932102255, - "nauc_ndcg_at_20_max": 0.3700666109489038, - "nauc_ndcg_at_20_std": 0.023554961081632526, - "nauc_ndcg_at_3_diff1": -0.19126335236855005, - "nauc_ndcg_at_3_max": 0.3961688610945783, - "nauc_ndcg_at_3_std": 0.024002257480038775, - "nauc_ndcg_at_5_diff1": -0.139901062146425, - "nauc_ndcg_at_5_max": 0.3917329472525927, - "nauc_ndcg_at_5_std": 0.0347569625654395, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.04838335171533055, - "nauc_precision_at_100_max": 0.26789342311487824, - "nauc_precision_at_100_std": 0.3938294537644401, - "nauc_precision_at_10_diff1": -0.21793680922956724, - "nauc_precision_at_10_max": 0.2884105380773535, - "nauc_precision_at_10_std": 0.04301118694721595, - "nauc_precision_at_1_diff1": 0.013386760985874355, - "nauc_precision_at_1_max": 0.5047570011864522, - "nauc_precision_at_1_std": 0.05936737480692162, - "nauc_precision_at_20_diff1": -0.17946498011651218, - "nauc_precision_at_20_max": 0.27125185261273455, - "nauc_precision_at_20_std": 0.003228315259659776, - "nauc_precision_at_3_diff1": -0.2801586984596132, - "nauc_precision_at_3_max": 0.3391380756480404, - "nauc_precision_at_3_std": 0.026454181145238276, - "nauc_precision_at_5_diff1": -0.17510298661174045, - "nauc_precision_at_5_max": 0.33388259526261604, - "nauc_precision_at_5_std": 0.04752832131822858, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.048383351715331105, - "nauc_recall_at_100_max": 0.2678934231148774, - "nauc_recall_at_100_std": 0.3938294537644398, - "nauc_recall_at_10_diff1": -0.21793680922956815, - "nauc_recall_at_10_max": 0.2884105380773526, - "nauc_recall_at_10_std": 0.04301118694721539, - "nauc_recall_at_1_diff1": 0.013386760985874355, - "nauc_recall_at_1_max": 0.5047570011864522, - "nauc_recall_at_1_std": 0.05936737480692162, - "nauc_recall_at_20_diff1": -0.17946498011651307, - "nauc_recall_at_20_max": 0.2712518526127342, - "nauc_recall_at_20_std": 0.0032283152596590085, - "nauc_recall_at_3_diff1": -0.28015869845961333, - "nauc_recall_at_3_max": 0.3391380756480409, - "nauc_recall_at_3_std": 0.026454181145238197, - "nauc_recall_at_5_diff1": -0.17510298661174065, - "nauc_recall_at_5_max": 0.33388259526261593, - "nauc_recall_at_5_std": 0.047528321318228595, - "ndcg_at_1": 0.13, - "ndcg_at_10": 0.31327, - "ndcg_at_100": 0.37801, - "ndcg_at_1000": 0.40025, - "ndcg_at_20": 0.33322, - "ndcg_at_3": 0.24833, - "ndcg_at_5": 0.27804, - "precision_at_1": 0.13, - "precision_at_10": 0.051, - "precision_at_100": 0.0083, - "precision_at_1000": 0.001, - "precision_at_20": 0.0295, - "precision_at_3": 0.11, - "precision_at_5": 0.08, - "recall_at_1": 0.13, - "recall_at_10": 0.51, - "recall_at_100": 0.83, - "recall_at_1000": 1.0, - "recall_at_20": 0.59, - "recall_at_3": 0.33, - "recall_at_5": 0.4 - } - ] - }, - "task_name": "VidoreSyntheticDocQAEnergyRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json deleted file mode 100644 index 3b7e8c92eb..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAGovernmentReportsRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", - "evaluation_time": 29.03953790664673, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.17, - "cv_recall_at_10": 0.53, - "cv_recall_at_100": 0.88, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.65, - "cv_recall_at_3": 0.41, - "cv_recall_at_5": 0.47, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.34996, - "map_at_1": 0.19, - "map_at_10": 0.3125, - "map_at_100": 0.3268, - "map_at_1000": 0.32732, - "map_at_20": 0.32092, - "map_at_3": 0.28667, - "map_at_5": 0.30617, - "mrr_at_1": 0.17, - "mrr_at_10": 0.3043015873015873, - "mrr_at_100": 0.31867453920623073, - "mrr_at_1000": 0.31919269880635714, - "mrr_at_20": 0.3127636329958311, - "mrr_at_3": 0.2816666666666666, - "mrr_at_5": 0.29616666666666663, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.3516544702513509, - "nauc_cv_recall_at_100_max": 0.6257954183900725, - "nauc_cv_recall_at_100_std": -0.08725739739102838, - "nauc_cv_recall_at_10_diff1": 0.11492941483731742, - "nauc_cv_recall_at_10_max": 0.6216305378768097, - "nauc_cv_recall_at_10_std": 0.1785041718786511, - "nauc_cv_recall_at_1_diff1": 0.33847338601734545, - "nauc_cv_recall_at_1_max": 0.42578705146775714, - "nauc_cv_recall_at_1_std": -0.028346687691376937, - "nauc_cv_recall_at_20_diff1": 0.045601757669460355, - "nauc_cv_recall_at_20_max": 0.550069167548214, - "nauc_cv_recall_at_20_std": 0.09712751240947172, - "nauc_cv_recall_at_3_diff1": -0.004456385548393712, - "nauc_cv_recall_at_3_max": 0.5421160726972114, - "nauc_cv_recall_at_3_std": 0.12001240037717818, - "nauc_cv_recall_at_5_diff1": 0.08508814725932576, - "nauc_cv_recall_at_5_max": 0.6091588760400324, - "nauc_cv_recall_at_5_std": 0.19719405582879526, - "nauc_map_at_1000_diff1": 0.1425413131213182, - "nauc_map_at_1000_max": 0.49699745466836864, - "nauc_map_at_1000_std": 0.09344073064154622, - "nauc_map_at_100_diff1": 0.14260967351190185, - "nauc_map_at_100_max": 0.4972447058805711, - "nauc_map_at_100_std": 0.09313436868733994, - "nauc_map_at_10_diff1": 0.1409243269612414, - "nauc_map_at_10_max": 0.502989362838475, - "nauc_map_at_10_std": 0.10005581107730499, - "nauc_map_at_1_diff1": 0.23836026432162497, - "nauc_map_at_1_max": 0.3865115262387973, - "nauc_map_at_1_std": 0.04561733896053668, - "nauc_map_at_20_diff1": 0.1371717127110203, - "nauc_map_at_20_max": 0.4980711323828558, - "nauc_map_at_20_std": 0.09531305443499136, - "nauc_map_at_3_diff1": 0.11010341190945848, - "nauc_map_at_3_max": 0.4888800460910493, - "nauc_map_at_3_std": 0.07477345077436877, - "nauc_map_at_5_diff1": 0.13706143128873957, - "nauc_map_at_5_max": 0.5032336982522804, - "nauc_map_at_5_std": 0.10533815409735035, - "nauc_mrr_at_1000_diff1": 0.18678624669182986, - "nauc_mrr_at_1000_max": 0.5091271845519432, - "nauc_mrr_at_1000_std": 0.06389258559560088, - "nauc_mrr_at_100_diff1": 0.18678259303086617, - "nauc_mrr_at_100_max": 0.5093634612617053, - "nauc_mrr_at_100_std": 0.06362384958529906, - "nauc_mrr_at_10_diff1": 0.18388219199565162, - "nauc_mrr_at_10_max": 0.515153193106594, - "nauc_mrr_at_10_std": 0.07149646969980394, - "nauc_mrr_at_1_diff1": 0.33847338601734545, - "nauc_mrr_at_1_max": 0.42578705146775714, - "nauc_mrr_at_1_std": -0.028346687691376937, - "nauc_mrr_at_20_diff1": 0.18056301217995185, - "nauc_mrr_at_20_max": 0.5100760746918761, - "nauc_mrr_at_20_std": 0.06626317479181122, - "nauc_mrr_at_3_diff1": 0.15186627578465683, - "nauc_mrr_at_3_max": 0.4937757023080455, - "nauc_mrr_at_3_std": 0.054727704395902466, - "nauc_mrr_at_5_diff1": 0.17992256895127598, - "nauc_mrr_at_5_max": 0.5144597679413234, - "nauc_mrr_at_5_std": 0.07481339912415413, - "nauc_ndcg_at_1000_diff1": 0.14018428326853596, - "nauc_ndcg_at_1000_max": 0.5095848675975442, - "nauc_ndcg_at_1000_std": 0.09683580225656489, - "nauc_ndcg_at_100_diff1": 0.14984320552095684, - "nauc_ndcg_at_100_max": 0.5158634936681875, - "nauc_ndcg_at_100_std": 0.08711396818198522, - "nauc_ndcg_at_10_diff1": 0.13101689912462652, - "nauc_ndcg_at_10_max": 0.53491224497677, - "nauc_ndcg_at_10_std": 0.12134567341471547, - "nauc_ndcg_at_1_diff1": 0.23836026432162497, - "nauc_ndcg_at_1_max": 0.3865115262387973, - "nauc_ndcg_at_1_std": 0.04561733896053668, - "nauc_ndcg_at_20_diff1": 0.11588990092348003, - "nauc_ndcg_at_20_max": 0.5156627793963243, - "nauc_ndcg_at_20_std": 0.10148201064617794, - "nauc_ndcg_at_3_diff1": 0.07592271092042169, - "nauc_ndcg_at_3_max": 0.510637228334191, - "nauc_ndcg_at_3_std": 0.08301779806797376, - "nauc_ndcg_at_5_diff1": 0.1204085206167412, - "nauc_ndcg_at_5_max": 0.5345240398050349, - "nauc_ndcg_at_5_std": 0.13407080761621448, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3516544702513525, - "nauc_precision_at_100_max": 0.6257954183900729, - "nauc_precision_at_100_std": -0.08725739739102806, - "nauc_precision_at_10_diff1": 0.11492941483731785, - "nauc_precision_at_10_max": 0.6216305378768097, - "nauc_precision_at_10_std": 0.17850417187865175, - "nauc_precision_at_1_diff1": 0.23836026432162497, - "nauc_precision_at_1_max": 0.3865115262387973, - "nauc_precision_at_1_std": 0.04561733896053668, - "nauc_precision_at_20_diff1": 0.04560175766946091, - "nauc_precision_at_20_max": 0.5500691675482139, - "nauc_precision_at_20_std": 0.09712751240947225, - "nauc_precision_at_3_diff1": -0.01078784757981495, - "nauc_precision_at_3_max": 0.5642507723995873, - "nauc_precision_at_3_std": 0.10406797116374848, - "nauc_precision_at_5_diff1": 0.08190430824600467, - "nauc_precision_at_5_max": 0.6161347517730504, - "nauc_precision_at_5_std": 0.2138509579760776, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.3516544702513509, - "nauc_recall_at_100_max": 0.6257954183900725, - "nauc_recall_at_100_std": -0.08725739739102838, - "nauc_recall_at_10_diff1": 0.11492941483731742, - "nauc_recall_at_10_max": 0.6216305378768097, - "nauc_recall_at_10_std": 0.1785041718786511, - "nauc_recall_at_1_diff1": 0.23836026432162497, - "nauc_recall_at_1_max": 0.3865115262387973, - "nauc_recall_at_1_std": 0.04561733896053668, - "nauc_recall_at_20_diff1": 0.045601757669460355, - "nauc_recall_at_20_max": 0.550069167548214, - "nauc_recall_at_20_std": 0.09712751240947172, - "nauc_recall_at_3_diff1": -0.010787847579814863, - "nauc_recall_at_3_max": 0.5642507723995882, - "nauc_recall_at_3_std": 0.10406797116374862, - "nauc_recall_at_5_diff1": 0.08190430824600382, - "nauc_recall_at_5_max": 0.6161347517730497, - "nauc_recall_at_5_std": 0.21385095797607692, - "ndcg_at_1": 0.19, - "ndcg_at_10": 0.36581, - "ndcg_at_100": 0.43883, - "ndcg_at_1000": 0.4539, - "ndcg_at_20": 0.39628, - "ndcg_at_3": 0.31595, - "ndcg_at_5": 0.34996, - "precision_at_1": 0.19, - "precision_at_10": 0.053, - "precision_at_100": 0.0088, - "precision_at_1000": 0.001, - "precision_at_20": 0.0325, - "precision_at_3": 0.13333, - "precision_at_5": 0.096, - "recall_at_1": 0.19, - "recall_at_10": 0.53, - "recall_at_100": 0.88, - "recall_at_1000": 1.0, - "recall_at_20": 0.65, - "recall_at_3": 0.4, - "recall_at_5": 0.48 - } - ] - }, - "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json deleted file mode 100644 index 956d9f38be..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", - "evaluation_time": 27.550007104873657, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.19, - "cv_recall_at_10": 0.54, - "cv_recall_at_100": 0.82, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.65, - "cv_recall_at_3": 0.38, - "cv_recall_at_5": 0.45, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.33111, - "map_at_1": 0.19, - "map_at_10": 0.30356, - "map_at_100": 0.31497, - "map_at_1000": 0.31591, - "map_at_20": 0.3098, - "map_at_3": 0.27833, - "map_at_5": 0.29133, - "mrr_at_1": 0.19, - "mrr_at_10": 0.3056468253968253, - "mrr_at_100": 0.3170461377031122, - "mrr_at_1000": 0.31798498793135255, - "mrr_at_20": 0.3123555641554867, - "mrr_at_3": 0.2783333333333333, - "mrr_at_5": 0.2938333333333333, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.15676285934574333, - "nauc_cv_recall_at_100_max": 0.45654807798215596, - "nauc_cv_recall_at_100_std": -0.14313250357968876, - "nauc_cv_recall_at_10_diff1": -0.09325942596760317, - "nauc_cv_recall_at_10_max": 0.31298102303353603, - "nauc_cv_recall_at_10_std": -0.25989830791031104, - "nauc_cv_recall_at_1_diff1": 0.002084979738704942, - "nauc_cv_recall_at_1_max": 0.38790711751551116, - "nauc_cv_recall_at_1_std": -0.11869251593159925, - "nauc_cv_recall_at_20_diff1": 0.06229961754414476, - "nauc_cv_recall_at_20_max": 0.31618520628204094, - "nauc_cv_recall_at_20_std": -0.27548213849784403, - "nauc_cv_recall_at_3_diff1": -0.2096405271229176, - "nauc_cv_recall_at_3_max": 0.3323369635848518, - "nauc_cv_recall_at_3_std": -0.21148597717880088, - "nauc_cv_recall_at_5_diff1": -0.13304524323970407, - "nauc_cv_recall_at_5_max": 0.37726707261179854, - "nauc_cv_recall_at_5_std": -0.24580632488705587, - "nauc_map_at_1000_diff1": -0.09235421018350352, - "nauc_map_at_1000_max": 0.3155831508446477, - "nauc_map_at_1000_std": -0.21978739885425327, - "nauc_map_at_100_diff1": -0.09169127502452307, - "nauc_map_at_100_max": 0.31590394921168813, - "nauc_map_at_100_std": -0.21997561809528432, - "nauc_map_at_10_diff1": -0.09858535432501385, - "nauc_map_at_10_max": 0.31377485989975207, - "nauc_map_at_10_std": -0.21996261057713298, - "nauc_map_at_1_diff1": 0.002084979738704942, - "nauc_map_at_1_max": 0.27975720074655724, - "nauc_map_at_1_std": -0.2348460646007433, - "nauc_map_at_20_diff1": -0.08808591850158354, - "nauc_map_at_20_max": 0.3147501333995878, - "nauc_map_at_20_std": -0.22056529702822625, - "nauc_map_at_3_diff1": -0.12443445016908257, - "nauc_map_at_3_max": 0.31066801703822294, - "nauc_map_at_3_std": -0.2136006611262196, - "nauc_map_at_5_diff1": -0.10518620663878896, - "nauc_map_at_5_max": 0.32470654018637624, - "nauc_map_at_5_std": -0.20637715997684672, - "nauc_mrr_at_1000_diff1": -0.08252339841089382, - "nauc_mrr_at_1000_max": 0.37186520512978016, - "nauc_mrr_at_1000_std": -0.16387062255497364, - "nauc_mrr_at_100_diff1": -0.08188638538702586, - "nauc_mrr_at_100_max": 0.37203061930902376, - "nauc_mrr_at_100_std": -0.1642176357964918, - "nauc_mrr_at_10_diff1": -0.08858050769501902, - "nauc_mrr_at_10_max": 0.36905843834344043, - "nauc_mrr_at_10_std": -0.16523125576816933, - "nauc_mrr_at_1_diff1": 0.002084979738704942, - "nauc_mrr_at_1_max": 0.38790711751551116, - "nauc_mrr_at_1_std": -0.11869251593159925, - "nauc_mrr_at_20_diff1": -0.07936545859048873, - "nauc_mrr_at_20_max": 0.3691865273941238, - "nauc_mrr_at_20_std": -0.16584492656687405, - "nauc_mrr_at_3_diff1": -0.11139584441954048, - "nauc_mrr_at_3_max": 0.36619428184673924, - "nauc_mrr_at_3_std": -0.15469519776158758, - "nauc_mrr_at_5_diff1": -0.09444148817962006, - "nauc_mrr_at_5_max": 0.37868655568518383, - "nauc_mrr_at_5_std": -0.1628485012637727, - "nauc_ndcg_at_1000_diff1": -0.08845801918452714, - "nauc_ndcg_at_1000_max": 0.321947402125894, - "nauc_ndcg_at_1000_std": -0.2209402250110133, - "nauc_ndcg_at_100_diff1": -0.06956430063277862, - "nauc_ndcg_at_100_max": 0.33184409702496304, - "nauc_ndcg_at_100_std": -0.21689400558975963, - "nauc_ndcg_at_10_diff1": -0.10082683668530741, - "nauc_ndcg_at_10_max": 0.31499392517072217, - "nauc_ndcg_at_10_std": -0.22898881208713334, - "nauc_ndcg_at_1_diff1": 0.002084979738704942, - "nauc_ndcg_at_1_max": 0.27975720074655724, - "nauc_ndcg_at_1_std": -0.2348460646007433, - "nauc_ndcg_at_20_diff1": -0.06058784979969239, - "nauc_ndcg_at_20_max": 0.3204646041250894, - "nauc_ndcg_at_20_std": -0.23011656634574992, - "nauc_ndcg_at_3_diff1": -0.15159027688467022, - "nauc_ndcg_at_3_max": 0.31394762477267585, - "nauc_ndcg_at_3_std": -0.2132489721131679, - "nauc_ndcg_at_5_diff1": -0.11540331552795374, - "nauc_ndcg_at_5_max": 0.3388623148185459, - "nauc_ndcg_at_5_std": -0.20140518922691222, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.15676285934574297, - "nauc_precision_at_100_max": 0.4565480779821573, - "nauc_precision_at_100_std": -0.14313250357968957, - "nauc_precision_at_10_diff1": -0.09325942596760275, - "nauc_precision_at_10_max": 0.3129810230335366, - "nauc_precision_at_10_std": -0.25989830791031043, - "nauc_precision_at_1_diff1": 0.002084979738704942, - "nauc_precision_at_1_max": 0.27975720074655724, - "nauc_precision_at_1_std": -0.2348460646007433, - "nauc_precision_at_20_diff1": 0.0800708902797296, - "nauc_precision_at_20_max": 0.3399220845574146, - "nauc_precision_at_20_std": -0.2660620768936003, - "nauc_precision_at_3_diff1": -0.21856608849740672, - "nauc_precision_at_3_max": 0.32083866878791373, - "nauc_precision_at_3_std": -0.21388382290171126, - "nauc_precision_at_5_diff1": -0.13304524323970335, - "nauc_precision_at_5_max": 0.3772670726117992, - "nauc_precision_at_5_std": -0.1891180514633659, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.15676285934574333, - "nauc_recall_at_100_max": 0.45654807798215596, - "nauc_recall_at_100_std": -0.14313250357968876, - "nauc_recall_at_10_diff1": -0.09325942596760317, - "nauc_recall_at_10_max": 0.31298102303353603, - "nauc_recall_at_10_std": -0.25989830791031104, - "nauc_recall_at_1_diff1": 0.002084979738704942, - "nauc_recall_at_1_max": 0.27975720074655724, - "nauc_recall_at_1_std": -0.2348460646007433, - "nauc_recall_at_20_diff1": 0.08007089027972884, - "nauc_recall_at_20_max": 0.3399220845574144, - "nauc_recall_at_20_std": -0.2660620768936013, - "nauc_recall_at_3_diff1": -0.2185660884974068, - "nauc_recall_at_3_max": 0.3208386687879141, - "nauc_recall_at_3_std": -0.21388382290171132, - "nauc_recall_at_5_diff1": -0.13304524323970407, - "nauc_recall_at_5_max": 0.37726707261179854, - "nauc_recall_at_5_std": -0.1891180514633668, - "ndcg_at_1": 0.19, - "ndcg_at_10": 0.36044, - "ndcg_at_100": 0.41927, - "ndcg_at_1000": 0.44269, - "ndcg_at_20": 0.38488, - "ndcg_at_3": 0.30702, - "ndcg_at_5": 0.33111, - "precision_at_1": 0.19, - "precision_at_10": 0.054, - "precision_at_100": 0.0082, - "precision_at_1000": 0.001, - "precision_at_20": 0.032, - "precision_at_3": 0.13, - "precision_at_5": 0.09, - "recall_at_1": 0.19, - "recall_at_10": 0.54, - "recall_at_100": 0.82, - "recall_at_1000": 1.0, - "recall_at_20": 0.64, - "recall_at_3": 0.39, - "recall_at_5": 0.45 - } - ] - }, - "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTabfquadRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTabfquadRetrieval.json deleted file mode 100644 index d165b2f16a..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTabfquadRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", - "evaluation_time": 4.353839874267578, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.02143, - "cv_recall_at_10": 0.175, - "cv_recall_at_100": 0.64643, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.24643, - "cv_recall_at_3": 0.07857, - "cv_recall_at_5": 0.11786, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.06876, - "map_at_1": 0.01429, - "map_at_10": 0.05641, - "map_at_100": 0.06963, - "map_at_1000": 0.07195, - "map_at_20": 0.06128, - "map_at_3": 0.03988, - "map_at_5": 0.05042, - "mrr_at_1": 0.02142857142857143, - "mrr_at_10": 0.06293792517006803, - "mrr_at_100": 0.07592145747016359, - "mrr_at_1000": 0.07823686336276094, - "mrr_at_20": 0.06762745758958719, - "mrr_at_3": 0.04642857142857143, - "mrr_at_5": 0.055535714285714285, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.07013675223542849, - "nauc_cv_recall_at_100_max": 0.11614992524518984, - "nauc_cv_recall_at_100_std": 0.265093833161118, - "nauc_cv_recall_at_10_diff1": -0.062031339049582676, - "nauc_cv_recall_at_10_max": 0.21408400197229802, - "nauc_cv_recall_at_10_std": 0.20956948829108032, - "nauc_cv_recall_at_1_diff1": -0.3614614077452824, - "nauc_cv_recall_at_1_max": 0.3357928628924066, - "nauc_cv_recall_at_1_std": 0.37492174224130215, - "nauc_cv_recall_at_20_diff1": -0.04239040201350441, - "nauc_cv_recall_at_20_max": 0.16552719658634857, - "nauc_cv_recall_at_20_std": 0.21981990556410375, - "nauc_cv_recall_at_3_diff1": -0.19656316315827993, - "nauc_cv_recall_at_3_max": 0.37258823816376824, - "nauc_cv_recall_at_3_std": 0.19223357806668773, - "nauc_cv_recall_at_5_diff1": -0.1539878825273585, - "nauc_cv_recall_at_5_max": 0.33965772853090626, - "nauc_cv_recall_at_5_std": 0.2498221799453256, - "nauc_map_at_1000_diff1": -0.11477836932696846, - "nauc_map_at_1000_max": 0.2906056774186307, - "nauc_map_at_1000_std": 0.23287433742435157, - "nauc_map_at_100_diff1": -0.1113113159044201, - "nauc_map_at_100_max": 0.28669249289561527, - "nauc_map_at_100_std": 0.2326195062209629, - "nauc_map_at_10_diff1": -0.12493110967755462, - "nauc_map_at_10_max": 0.29585346508604804, - "nauc_map_at_10_std": 0.2185152261808974, - "nauc_map_at_1_diff1": -0.25248188891870144, - "nauc_map_at_1_max": 0.5492353099007244, - "nauc_map_at_1_std": 0.47719345317950096, - "nauc_map_at_20_diff1": -0.12048170875102637, - "nauc_map_at_20_max": 0.2853357651323181, - "nauc_map_at_20_std": 0.22407682393584646, - "nauc_map_at_3_diff1": -0.1865611571869657, - "nauc_map_at_3_max": 0.31550725043951156, - "nauc_map_at_3_std": 0.21905898465814205, - "nauc_map_at_5_diff1": -0.14819065296774128, - "nauc_map_at_5_max": 0.31966766311722844, - "nauc_map_at_5_std": 0.2253066719087251, - "nauc_mrr_at_1000_diff1": -0.1731260258556916, - "nauc_mrr_at_1000_max": 0.2870994898422976, - "nauc_mrr_at_1000_std": 0.23989941101230317, - "nauc_mrr_at_100_diff1": -0.16901002627023246, - "nauc_mrr_at_100_max": 0.283614353441143, - "nauc_mrr_at_100_std": 0.23956904850637092, - "nauc_mrr_at_10_diff1": -0.1833916598209548, - "nauc_mrr_at_10_max": 0.2930079237890623, - "nauc_mrr_at_10_std": 0.23234705273326456, - "nauc_mrr_at_1_diff1": -0.3614614077452824, - "nauc_mrr_at_1_max": 0.3357928628924066, - "nauc_mrr_at_1_std": 0.37492174224130215, - "nauc_mrr_at_20_diff1": -0.177786791550534, - "nauc_mrr_at_20_max": 0.2824709513782039, - "nauc_mrr_at_20_std": 0.23149134244179065, - "nauc_mrr_at_3_diff1": -0.2519659036965182, - "nauc_mrr_at_3_max": 0.35485335699985543, - "nauc_mrr_at_3_std": 0.22600152731625767, - "nauc_mrr_at_5_diff1": -0.22520366189697133, - "nauc_mrr_at_5_max": 0.3373958293786979, - "nauc_mrr_at_5_std": 0.2458651801062478, - "nauc_ndcg_at_1000_diff1": -0.08616081793508518, - "nauc_ndcg_at_1000_max": 0.2636258033034474, - "nauc_ndcg_at_1000_std": 0.2400451291595649, - "nauc_ndcg_at_100_diff1": -0.043638278763048495, - "nauc_ndcg_at_100_max": 0.22150656020775003, - "nauc_ndcg_at_100_std": 0.2443691153022479, - "nauc_ndcg_at_10_diff1": -0.0959049965441373, - "nauc_ndcg_at_10_max": 0.25740388210300413, - "nauc_ndcg_at_10_std": 0.20599782546699, - "nauc_ndcg_at_1_diff1": -0.25248188891870144, - "nauc_ndcg_at_1_max": 0.5492353099007244, - "nauc_ndcg_at_1_std": 0.47719345317950096, - "nauc_ndcg_at_20_diff1": -0.08485754868327161, - "nauc_ndcg_at_20_max": 0.23109618682790573, - "nauc_ndcg_at_20_std": 0.22020858825779455, - "nauc_ndcg_at_3_diff1": -0.18560257748100978, - "nauc_ndcg_at_3_max": 0.28745044081818694, - "nauc_ndcg_at_3_std": 0.20898634536396835, - "nauc_ndcg_at_5_diff1": -0.13078634593787122, - "nauc_ndcg_at_5_max": 0.29665194805795475, - "nauc_ndcg_at_5_std": 0.2203277736803905, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.07013675223542876, - "nauc_precision_at_100_max": 0.11614992524519006, - "nauc_precision_at_100_std": 0.2650938331611187, - "nauc_precision_at_10_diff1": -0.05865474703310428, - "nauc_precision_at_10_max": 0.20959556527170534, - "nauc_precision_at_10_std": 0.1914168228190715, - "nauc_precision_at_1_diff1": -0.25248188891870144, - "nauc_precision_at_1_max": 0.5492353099007244, - "nauc_precision_at_1_std": 0.47719345317950096, - "nauc_precision_at_20_diff1": -0.0423904020135042, - "nauc_precision_at_20_max": 0.16552719658634873, - "nauc_precision_at_20_std": 0.219819905564104, - "nauc_precision_at_3_diff1": -0.18520867380540046, - "nauc_precision_at_3_max": 0.24521306437056364, - "nauc_precision_at_3_std": 0.1969778276459253, - "nauc_precision_at_5_diff1": -0.10594039054470711, - "nauc_precision_at_5_max": 0.26743679342240495, - "nauc_precision_at_5_std": 0.21703597122302157, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.07013675223542849, - "nauc_recall_at_100_max": 0.11614992524518984, - "nauc_recall_at_100_std": 0.265093833161118, - "nauc_recall_at_10_diff1": -0.058654747033104304, - "nauc_recall_at_10_max": 0.209595565271705, - "nauc_recall_at_10_std": 0.19141682281907146, - "nauc_recall_at_1_diff1": -0.25248188891870144, - "nauc_recall_at_1_max": 0.5492353099007244, - "nauc_recall_at_1_std": 0.47719345317950096, - "nauc_recall_at_20_diff1": -0.04239040201350441, - "nauc_recall_at_20_max": 0.16552719658634857, - "nauc_recall_at_20_std": 0.21981990556410375, - "nauc_recall_at_3_diff1": -0.18520867380540046, - "nauc_recall_at_3_max": 0.2452130643705637, - "nauc_recall_at_3_std": 0.19697782764592536, - "nauc_recall_at_5_diff1": -0.10594039054470714, - "nauc_recall_at_5_max": 0.26743679342240495, - "nauc_recall_at_5_std": 0.21703597122302157, - "ndcg_at_1": 0.01429, - "ndcg_at_10": 0.08358, - "ndcg_at_100": 0.17244, - "ndcg_at_1000": 0.22086, - "ndcg_at_20": 0.10211, - "ndcg_at_3": 0.0497, - "ndcg_at_5": 0.06876, - "precision_at_1": 0.01429, - "precision_at_10": 0.01714, - "precision_at_100": 0.00646, - "precision_at_1000": 0.001, - "precision_at_20": 0.01232, - "precision_at_3": 0.02619, - "precision_at_5": 0.025, - "recall_at_1": 0.01429, - "recall_at_10": 0.17143, - "recall_at_100": 0.64643, - "recall_at_1000": 1.0, - "recall_at_20": 0.24643, - "recall_at_3": 0.07857, - "recall_at_5": 0.125 - } - ] - }, - "task_name": "VidoreTabfquadRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTatdqaRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTatdqaRetrieval.json deleted file mode 100644 index 0053f94100..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VidoreTatdqaRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", - "evaluation_time": 56.91961908340454, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.00301, - "cv_recall_at_10": 0.03548, - "cv_recall_at_100": 0.14251, - "cv_recall_at_1000": 0.73121, - "cv_recall_at_20": 0.05412, - "cv_recall_at_3": 0.01263, - "cv_recall_at_5": 0.02345, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.01095, - "map_at_1": 0.0018, - "map_at_10": 0.00933, - "map_at_100": 0.01243, - "map_at_1000": 0.01412, - "map_at_20": 0.0105, - "map_at_3": 0.00471, - "map_at_5": 0.00754, - "mrr_at_1": 0.003006614552014432, - "mrr_at_10": 0.010695673338487527, - "mrr_at_100": 0.013802284461660666, - "mrr_at_1000": 0.01548110419231038, - "mrr_at_20": 0.011912744158405977, - "mrr_at_3": 0.006714772499498898, - "mrr_at_5": 0.009089997995590298, - "nauc_cv_recall_at_1000_diff1": -0.014566063340413188, - "nauc_cv_recall_at_1000_max": 0.05409691540398872, - "nauc_cv_recall_at_1000_std": 0.11465856299781937, - "nauc_cv_recall_at_100_diff1": -0.06896710247347743, - "nauc_cv_recall_at_100_max": 0.08860267679149006, - "nauc_cv_recall_at_100_std": 0.17026666742389082, - "nauc_cv_recall_at_10_diff1": 0.10177862892497945, - "nauc_cv_recall_at_10_max": 0.2097063707163848, - "nauc_cv_recall_at_10_std": 0.38240798275194327, - "nauc_cv_recall_at_1_diff1": 0.03526380003576067, - "nauc_cv_recall_at_1_max": 0.4020982414202427, - "nauc_cv_recall_at_1_std": 0.6686073305917339, - "nauc_cv_recall_at_20_diff1": 0.009867753517689027, - "nauc_cv_recall_at_20_max": 0.18905020883601725, - "nauc_cv_recall_at_20_std": 0.2702661332401687, - "nauc_cv_recall_at_3_diff1": 0.04287210481309383, - "nauc_cv_recall_at_3_max": 0.3989496498413514, - "nauc_cv_recall_at_3_std": 0.6033952370820275, - "nauc_cv_recall_at_5_diff1": 0.07819466098025143, - "nauc_cv_recall_at_5_max": 0.33271295458325373, - "nauc_cv_recall_at_5_std": 0.5060975464431268, - "nauc_map_at_1000_diff1": 0.002376257165185258, - "nauc_map_at_1000_max": 0.2819036286133092, - "nauc_map_at_1000_std": 0.41437523372664636, - "nauc_map_at_100_diff1": -0.0012297099110992408, - "nauc_map_at_100_max": 0.28939605934379836, - "nauc_map_at_100_std": 0.4281412770351935, - "nauc_map_at_10_diff1": 0.039876616061044146, - "nauc_map_at_10_max": 0.3563897574654661, - "nauc_map_at_10_std": 0.5389632898745574, - "nauc_map_at_1_diff1": -0.02568513786845483, - "nauc_map_at_1_max": 0.6995235045831298, - "nauc_map_at_1_std": 1.0, - "nauc_map_at_20_diff1": 0.021159229553580298, - "nauc_map_at_20_max": 0.3323683590354517, - "nauc_map_at_20_std": 0.4834271083888167, - "nauc_map_at_3_diff1": -0.06702435727404064, - "nauc_map_at_3_max": 0.5885596535419393, - "nauc_map_at_3_std": 0.7767529197688164, - "nauc_map_at_5_diff1": 0.034734102998953825, - "nauc_map_at_5_max": 0.44997314380901593, - "nauc_map_at_5_std": 0.6196877602314378, - "nauc_mrr_at_1000_diff1": 0.022796670054829382, - "nauc_mrr_at_1000_max": 0.25534052859583023, - "nauc_mrr_at_1000_std": 0.3901775596025565, - "nauc_mrr_at_100_diff1": 0.020228157587260442, - "nauc_mrr_at_100_max": 0.2612247454263831, - "nauc_mrr_at_100_std": 0.4016538436087405, - "nauc_mrr_at_10_diff1": 0.0699129110691725, - "nauc_mrr_at_10_max": 0.3166979674099861, - "nauc_mrr_at_10_std": 0.4943672231606845, - "nauc_mrr_at_1_diff1": 0.03526380003576067, - "nauc_mrr_at_1_max": 0.4020982414202427, - "nauc_mrr_at_1_std": 0.6686073305917339, - "nauc_mrr_at_20_diff1": 0.045078949690099346, - "nauc_mrr_at_20_max": 0.29840241395898265, - "nauc_mrr_at_20_std": 0.4477899188338933, - "nauc_mrr_at_3_diff1": 0.03638945853763958, - "nauc_mrr_at_3_max": 0.414453160995652, - "nauc_mrr_at_3_std": 0.6142739363580281, - "nauc_mrr_at_5_diff1": 0.05644557824415222, - "nauc_mrr_at_5_max": 0.37386357142111964, - "nauc_mrr_at_5_std": 0.5528710632147819, - "nauc_ndcg_at_1000_diff1": -0.012699300979067456, - "nauc_ndcg_at_1000_max": 0.13586894754526818, - "nauc_ndcg_at_1000_std": 0.217056132320633, - "nauc_ndcg_at_100_diff1": -0.043325013809925546, - "nauc_ndcg_at_100_max": 0.15772228815085845, - "nauc_ndcg_at_100_std": 0.25872834420062885, - "nauc_ndcg_at_10_diff1": 0.0562938241735581, - "nauc_ndcg_at_10_max": 0.2794645518226739, - "nauc_ndcg_at_10_std": 0.46133489790312227, - "nauc_ndcg_at_1_diff1": -0.02568513786845483, - "nauc_ndcg_at_1_max": 0.6995235045831298, - "nauc_ndcg_at_1_std": 1.0, - "nauc_ndcg_at_20_diff1": 0.012327678642763356, - "nauc_ndcg_at_20_max": 0.2450981482790461, - "nauc_ndcg_at_20_std": 0.36548222828664273, - "nauc_ndcg_at_3_diff1": -0.07882852915289862, - "nauc_ndcg_at_3_max": 0.5781964658113339, - "nauc_ndcg_at_3_std": 0.7570757867717248, - "nauc_ndcg_at_5_diff1": 0.05284682509921638, - "nauc_ndcg_at_5_max": 0.408496056286339, - "nauc_ndcg_at_5_std": 0.5689798228201582, - "nauc_precision_at_1000_diff1": -0.014566063340413198, - "nauc_precision_at_1000_max": 0.05409691540398878, - "nauc_precision_at_1000_std": 0.11465856299781925, - "nauc_precision_at_100_diff1": -0.07070737778348025, - "nauc_precision_at_100_max": 0.09083326876918361, - "nauc_precision_at_100_std": 0.17229679067374504, - "nauc_precision_at_10_diff1": 0.07230484504920205, - "nauc_precision_at_10_max": 0.20179917541371403, - "nauc_precision_at_10_std": 0.38483848287524763, - "nauc_precision_at_1_diff1": -0.02568513786845483, - "nauc_precision_at_1_max": 0.6995235045831298, - "nauc_precision_at_1_std": 1.0, - "nauc_precision_at_20_diff1": -0.001104068362403356, - "nauc_precision_at_20_max": 0.1744671111021371, - "nauc_precision_at_20_std": 0.2658456862734212, - "nauc_precision_at_3_diff1": -0.09810504569367917, - "nauc_precision_at_3_max": 0.5627605752906821, - "nauc_precision_at_3_std": 0.7280169253981901, - "nauc_precision_at_5_diff1": 0.07493463474156249, - "nauc_precision_at_5_max": 0.36014761171237997, - "nauc_precision_at_5_std": 0.5109657682867575, - "nauc_recall_at_1000_diff1": -0.014566063340413188, - "nauc_recall_at_1000_max": 0.05409691540398872, - "nauc_recall_at_1000_std": 0.11465856299781937, - "nauc_recall_at_100_diff1": -0.07070737778348046, - "nauc_recall_at_100_max": 0.09083326876918348, - "nauc_recall_at_100_std": 0.17229679067374493, - "nauc_recall_at_10_diff1": 0.07230484504920151, - "nauc_recall_at_10_max": 0.2017991754137134, - "nauc_recall_at_10_std": 0.3848384828752471, - "nauc_recall_at_1_diff1": -0.02568513786845483, - "nauc_recall_at_1_max": 0.6995235045831298, - "nauc_recall_at_1_std": 1.0, - "nauc_recall_at_20_diff1": -0.0011040683624037103, - "nauc_recall_at_20_max": 0.17446711110213683, - "nauc_recall_at_20_std": 0.2658456862734206, - "nauc_recall_at_3_diff1": -0.09810504569367938, - "nauc_recall_at_3_max": 0.5627605752906818, - "nauc_recall_at_3_std": 0.7280169253981897, - "nauc_recall_at_5_diff1": 0.07493463474156249, - "nauc_recall_at_5_max": 0.3601476117123801, - "nauc_recall_at_5_std": 0.5109657682867574, - "ndcg_at_1": 0.0018, - "ndcg_at_10": 0.01526, - "ndcg_at_100": 0.03553, - "ndcg_at_1000": 0.10362, - "ndcg_at_20": 0.01972, - "ndcg_at_3": 0.00581, - "ndcg_at_5": 0.01095, - "precision_at_1": 0.0018, - "precision_at_10": 0.00349, - "precision_at_100": 0.00142, - "precision_at_1000": 0.00073, - "precision_at_20": 0.00265, - "precision_at_3": 0.00301, - "precision_at_5": 0.00433, - "recall_at_1": 0.0018, - "recall_at_10": 0.03488, - "recall_at_100": 0.14191, - "recall_at_1000": 0.73121, - "recall_at_20": 0.05292, - "recall_at_3": 0.00902, - "recall_at_5": 0.02165 - } - ] - }, - "task_name": "VidoreTatdqaRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Winoground.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Winoground.json deleted file mode 100644 index d077401528..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Winoground.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "521ec2ba6f9a5d7380f7cca5a7b44aea5c1d677c", - "evaluation_time": 48.52479600906372, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "accuracy": 0.07000000029802322, - "hf_subset": "default", - "image_acc": 0.10249999910593033, - "languages": [ - "eng-Latn" - ], - "main_score": 0.07000000029802322, - "text_acc": 0.25 - } - ] - }, - "task_name": "Winoground" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json deleted file mode 100644 index 2bf553db27..0000000000 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "openai/clip-vit-base-patch16", "revision": "57c216476eefef5ab752ec549e440a49ae4ae5f3", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "CLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROCocoOrder.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROCocoOrder.json deleted file mode 100644 index ed54c38473..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROCocoOrder.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "853ec8757226585a38a80886c51fe0f3f268787c", - "evaluation_time": 158.77182126045227, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.46009597182273865, - "text_acc": 0.46009597182273865 - } - ] - }, - "task_name": "AROCocoOrder" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROFlickrOrder.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROFlickrOrder.json deleted file mode 100644 index 6cdefe46fb..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROFlickrOrder.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "1f9485f69c87947812378a1aedf86410c86a0aa8", - "evaluation_time": 29.562106609344482, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.5612000226974487, - "text_acc": 0.5612000226974487 - } - ] - }, - "task_name": "AROFlickrOrder" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualAttribution.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualAttribution.json deleted file mode 100644 index 4966616b48..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualAttribution.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "18f7e01358d91df599d723f00e16a18640e19398", - "evaluation_time": 280.1469085216522, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.6139209866523743, - "text_acc": 0.6139209866523743 - } - ] - }, - "task_name": "AROVisualAttribution" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualRelation.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualRelation.json deleted file mode 100644 index 949b1aea3b..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/AROVisualRelation.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "3867ad4f46a1ac2e63be034d1fc77dd8c2ef7209", - "evaluation_time": 238.80001974105835, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.5179429054260254, - "text_acc": 0.5179429054260254 - } - ] - }, - "task_name": "AROVisualRelation" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json deleted file mode 100644 index 6b300b8219..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IMultiChoice.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "dataset_revision": "b7b46b72d1ed1fa44d25e2b9c4726afab4a7ce53", - "evaluation_time": 30.517717123031616, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.6161, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6161, - "mrr_at_1": 0.6161048689138576, - "mrr_at_10": 0.7762172284644191, - "mrr_at_100": 0.7762172284644191, - "mrr_at_1000": 0.7762172284644191, - "mrr_at_20": 0.7762172284644191, - "mrr_at_3": 0.7762172284644191, - "mrr_at_5": 0.7762172284644191, - "ndcg_at_1": 0.6161, - "ndcg_at_10": 0.81993, - "ndcg_at_100": 0.81993, - "ndcg_at_1000": 0.81993, - "ndcg_at_20": 0.81993, - "ndcg_at_3": 0.81993, - "ndcg_at_5": 0.81993 - } - ] - }, - "task_name": "BLINKIT2IMultiChoice" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IRetrieval.json deleted file mode 100644 index 1032c117ac..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26", - "evaluation_time": 17.393863439559937, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.31592, - "cv_recall_at_10": 0.56965, - "cv_recall_at_100": 0.87065, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.6393, - "cv_recall_at_3": 0.48507, - "cv_recall_at_5": 0.52736, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.44916, - "map_at_1": 0.31592, - "map_at_10": 0.40997, - "map_at_100": 0.42021, - "map_at_1000": 0.42102, - "map_at_20": 0.41454, - "map_at_3": 0.39469, - "map_at_5": 0.40427, - "mrr_at_1": 0.31592039800995025, - "mrr_at_10": 0.40997295269683326, - "mrr_at_100": 0.4202098745202131, - "mrr_at_1000": 0.42101657433711814, - "mrr_at_20": 0.41453745832471395, - "mrr_at_3": 0.3946932006633499, - "mrr_at_5": 0.4042703150912106, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.17659898624319145, - "nauc_cv_recall_at_100_max": 0.2661970196010441, - "nauc_cv_recall_at_100_std": -0.2790205688665586, - "nauc_cv_recall_at_10_diff1": 0.3814466970679155, - "nauc_cv_recall_at_10_max": 0.45289099967438606, - "nauc_cv_recall_at_10_std": -0.6696974920638539, - "nauc_cv_recall_at_1_diff1": 0.4348231455825518, - "nauc_cv_recall_at_1_max": 0.3562517088470133, - "nauc_cv_recall_at_1_std": -0.4716841092376067, - "nauc_cv_recall_at_20_diff1": 0.34765332655935005, - "nauc_cv_recall_at_20_max": 0.40507962086487265, - "nauc_cv_recall_at_20_std": -0.60412556449698, - "nauc_cv_recall_at_3_diff1": 0.4000259511772347, - "nauc_cv_recall_at_3_max": 0.46517944743370765, - "nauc_cv_recall_at_3_std": -0.6578600193275457, - "nauc_cv_recall_at_5_diff1": 0.4149222268518541, - "nauc_cv_recall_at_5_max": 0.4447326055964038, - "nauc_cv_recall_at_5_std": -0.6819130948957665, - "nauc_map_at_1000_diff1": 0.41844540783755707, - "nauc_map_at_1000_max": 0.40386353368016625, - "nauc_map_at_1000_std": -0.5528994661016456, - "nauc_map_at_100_diff1": 0.41795524243083654, - "nauc_map_at_100_max": 0.40346749605366333, - "nauc_map_at_100_std": -0.5524674902038105, - "nauc_map_at_10_diff1": 0.42120255585772387, - "nauc_map_at_10_max": 0.40787732867834814, - "nauc_map_at_10_std": -0.5613450956416107, - "nauc_map_at_1_diff1": 0.4348231455825518, - "nauc_map_at_1_max": 0.3562517088470133, - "nauc_map_at_1_std": -0.4716841092376067, - "nauc_map_at_20_diff1": 0.4188754182377973, - "nauc_map_at_20_max": 0.4046917634724885, - "nauc_map_at_20_std": -0.5563336557097699, - "nauc_map_at_3_diff1": 0.4214894251666346, - "nauc_map_at_3_max": 0.41188287008473307, - "nauc_map_at_3_std": -0.5590768607537941, - "nauc_map_at_5_diff1": 0.4246473310855139, - "nauc_map_at_5_max": 0.4070463857404686, - "nauc_map_at_5_std": -0.5636557349011302, - "nauc_mrr_at_1000_diff1": 0.41844540783755707, - "nauc_mrr_at_1000_max": 0.40386353368016625, - "nauc_mrr_at_1000_std": -0.5528994661016456, - "nauc_mrr_at_100_diff1": 0.41795524243083654, - "nauc_mrr_at_100_max": 0.40346749605366333, - "nauc_mrr_at_100_std": -0.5524674902038105, - "nauc_mrr_at_10_diff1": 0.42120255585772387, - "nauc_mrr_at_10_max": 0.40787732867834814, - "nauc_mrr_at_10_std": -0.5613450956416107, - "nauc_mrr_at_1_diff1": 0.4348231455825518, - "nauc_mrr_at_1_max": 0.3562517088470133, - "nauc_mrr_at_1_std": -0.4716841092376067, - "nauc_mrr_at_20_diff1": 0.4188754182377973, - "nauc_mrr_at_20_max": 0.4046917634724885, - "nauc_mrr_at_20_std": -0.5563336557097699, - "nauc_mrr_at_3_diff1": 0.4214894251666346, - "nauc_mrr_at_3_max": 0.41188287008473307, - "nauc_mrr_at_3_std": -0.5590768607537941, - "nauc_mrr_at_5_diff1": 0.4246473310855139, - "nauc_mrr_at_5_max": 0.4070463857404686, - "nauc_mrr_at_5_std": -0.5636557349011302, - "nauc_ndcg_at_1000_diff1": 0.410108945334173, - "nauc_ndcg_at_1000_max": 0.4059872490668924, - "nauc_ndcg_at_1000_std": -0.5567117452158448, - "nauc_ndcg_at_100_diff1": 0.3974622734711721, - "nauc_ndcg_at_100_max": 0.39776281190453455, - "nauc_ndcg_at_100_std": -0.5422948577574471, - "nauc_ndcg_at_10_diff1": 0.41250730473823416, - "nauc_ndcg_at_10_max": 0.4193748171136443, - "nauc_ndcg_at_10_std": -0.5882737032592944, - "nauc_ndcg_at_1_diff1": 0.4348231455825518, - "nauc_ndcg_at_1_max": 0.3562517088470133, - "nauc_ndcg_at_1_std": -0.4716841092376067, - "nauc_ndcg_at_20_diff1": 0.4050302095189553, - "nauc_ndcg_at_20_max": 0.40785434428016737, - "nauc_ndcg_at_20_std": -0.5711108827183825, - "nauc_ndcg_at_3_diff1": 0.41605079061886857, - "nauc_ndcg_at_3_max": 0.4259171131235778, - "nauc_ndcg_at_3_std": -0.5847193268248356, - "nauc_ndcg_at_5_diff1": 0.42200838628819426, - "nauc_ndcg_at_5_max": 0.41721735277614064, - "nauc_ndcg_at_5_std": -0.5934002799239759, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.17659898624319106, - "nauc_precision_at_100_max": 0.26619701960104514, - "nauc_precision_at_100_std": -0.279020568866558, - "nauc_precision_at_10_diff1": 0.38144669706791556, - "nauc_precision_at_10_max": 0.4528909996743864, - "nauc_precision_at_10_std": -0.6696974920638536, - "nauc_precision_at_1_diff1": 0.4348231455825518, - "nauc_precision_at_1_max": 0.3562517088470133, - "nauc_precision_at_1_std": -0.4716841092376067, - "nauc_precision_at_20_diff1": 0.34765332655935016, - "nauc_precision_at_20_max": 0.40507962086487265, - "nauc_precision_at_20_std": -0.60412556449698, - "nauc_precision_at_3_diff1": 0.4000259511772348, - "nauc_precision_at_3_max": 0.4651794474337073, - "nauc_precision_at_3_std": -0.6578600193275455, - "nauc_precision_at_5_diff1": 0.41492222685185387, - "nauc_precision_at_5_max": 0.444732605596404, - "nauc_precision_at_5_std": -0.6819130948957663, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.17659898624319145, - "nauc_recall_at_100_max": 0.2661970196010441, - "nauc_recall_at_100_std": -0.2790205688665586, - "nauc_recall_at_10_diff1": 0.3814466970679155, - "nauc_recall_at_10_max": 0.45289099967438606, - "nauc_recall_at_10_std": -0.6696974920638539, - "nauc_recall_at_1_diff1": 0.4348231455825518, - "nauc_recall_at_1_max": 0.3562517088470133, - "nauc_recall_at_1_std": -0.4716841092376067, - "nauc_recall_at_20_diff1": 0.34765332655935005, - "nauc_recall_at_20_max": 0.40507962086487265, - "nauc_recall_at_20_std": -0.60412556449698, - "nauc_recall_at_3_diff1": 0.4000259511772347, - "nauc_recall_at_3_max": 0.46517944743370765, - "nauc_recall_at_3_std": -0.6578600193275457, - "nauc_recall_at_5_diff1": 0.4149222268518541, - "nauc_recall_at_5_max": 0.4447326055964038, - "nauc_recall_at_5_std": -0.6819130948957665, - "ndcg_at_1": 0.31592, - "ndcg_at_10": 0.44916, - "ndcg_at_100": 0.50879, - "ndcg_at_1000": 0.52627, - "ndcg_at_20": 0.46643, - "ndcg_at_3": 0.41809, - "ndcg_at_5": 0.43543, - "precision_at_1": 0.31592, - "precision_at_10": 0.05697, - "precision_at_100": 0.00871, - "precision_at_1000": 0.001, - "precision_at_20": 0.03197, - "precision_at_3": 0.16169, - "precision_at_5": 0.10547, - "recall_at_1": 0.31592, - "recall_at_10": 0.56965, - "recall_at_100": 0.87065, - "recall_at_1000": 1.0, - "recall_at_20": 0.6393, - "recall_at_3": 0.48507, - "recall_at_5": 0.52736 - } - ] - }, - "task_name": "BLINKIT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json deleted file mode 100644 index 55aca4a70e..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TMultiChoice.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "dataset_revision": "ae713b03ae68e343f16c3bcdbd1b1ee760975d55", - "evaluation_time": 10.223464965820312, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.36945, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.36945, - "mrr_at_1": 0.3694474539544962, - "mrr_at_10": 0.6365113759479949, - "mrr_at_100": 0.6365113759479949, - "mrr_at_1000": 0.6365113759479949, - "mrr_at_20": 0.6365113759479949, - "mrr_at_3": 0.61213434452871, - "mrr_at_5": 0.6365113759479949, - "ndcg_at_1": 0.36945, - "ndcg_at_10": 0.72903, - "ndcg_at_100": 0.72903, - "ndcg_at_1000": 0.72903, - "ndcg_at_20": 0.72903, - "ndcg_at_3": 0.68704, - "ndcg_at_5": 0.72903 - } - ] - }, - "task_name": "BLINKIT2TMultiChoice" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json deleted file mode 100644 index 21d3b7f0dc..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "302cf2008f204285985099dcd46425b00356c610", - "evaluation_time": 11.171209335327148, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.08947, - "cv_recall_at_10": 0.64678, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.96645, - "cv_recall_at_3": 0.2479, - "cv_recall_at_5": 0.37651, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.32202, - "map_at_1": 0.08947, - "map_at_10": 0.22464, - "map_at_100": 0.24864, - "map_at_1000": 0.24864, - "map_at_20": 0.24715, - "map_at_3": 0.16061, - "map_at_5": 0.18987, - "mrr_at_1": 0.08946877912395154, - "mrr_at_10": 0.2246397875708215, - "mrr_at_100": 0.24864178823066596, - "mrr_at_1000": 0.24864178823066596, - "mrr_at_20": 0.24714520392298733, - "mrr_at_3": 0.16060888474681598, - "mrr_at_5": 0.18987263125194162, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": 0.6643245257587762, - "nauc_cv_recall_at_10_max": 0.3394378418139761, - "nauc_cv_recall_at_10_std": 0.5631230256620064, - "nauc_cv_recall_at_1_diff1": 0.3501554820571643, - "nauc_cv_recall_at_1_max": 0.3599782655731798, - "nauc_cv_recall_at_1_std": 0.7041481187202615, - "nauc_cv_recall_at_20_diff1": 0.5587451940825555, - "nauc_cv_recall_at_20_max": 0.47434883444027837, - "nauc_cv_recall_at_20_std": 0.6875358512675467, - "nauc_cv_recall_at_3_diff1": 0.5321534332531633, - "nauc_cv_recall_at_3_max": 0.3540044845277292, - "nauc_cv_recall_at_3_std": 0.7076694823976849, - "nauc_cv_recall_at_5_diff1": 0.6570524014466579, - "nauc_cv_recall_at_5_max": 0.3001299949877772, - "nauc_cv_recall_at_5_std": 0.6030407237879231, - "nauc_map_at_1000_diff1": 0.4915782753126093, - "nauc_map_at_1000_max": 0.3523024760071231, - "nauc_map_at_1000_std": 0.6868659279458145, - "nauc_map_at_100_diff1": 0.4915782753126093, - "nauc_map_at_100_max": 0.3523024760071231, - "nauc_map_at_100_std": 0.6868659279458145, - "nauc_map_at_10_diff1": 0.507291819494973, - "nauc_map_at_10_max": 0.34661422520261387, - "nauc_map_at_10_std": 0.6748191769380557, - "nauc_map_at_1_diff1": 0.3501554820571643, - "nauc_map_at_1_max": 0.3599782655731798, - "nauc_map_at_1_std": 0.7041481187202615, - "nauc_map_at_20_diff1": 0.49198266429986276, - "nauc_map_at_20_max": 0.35311180756517996, - "nauc_map_at_20_std": 0.6868445762814864, - "nauc_map_at_3_diff1": 0.4648021929978972, - "nauc_map_at_3_max": 0.3663517861281038, - "nauc_map_at_3_std": 0.7205919836574776, - "nauc_map_at_5_diff1": 0.5087022522085294, - "nauc_map_at_5_max": 0.3467565269949981, - "nauc_map_at_5_std": 0.683363887437365, - "nauc_mrr_at_1000_diff1": 0.4915782753126093, - "nauc_mrr_at_1000_max": 0.3523024760071231, - "nauc_mrr_at_1000_std": 0.6868659279458145, - "nauc_mrr_at_100_diff1": 0.4915782753126093, - "nauc_mrr_at_100_max": 0.3523024760071231, - "nauc_mrr_at_100_std": 0.6868659279458145, - "nauc_mrr_at_10_diff1": 0.507291819494973, - "nauc_mrr_at_10_max": 0.34661422520261387, - "nauc_mrr_at_10_std": 0.6748191769380557, - "nauc_mrr_at_1_diff1": 0.3501554820571643, - "nauc_mrr_at_1_max": 0.3599782655731798, - "nauc_mrr_at_1_std": 0.7041481187202615, - "nauc_mrr_at_20_diff1": 0.49198266429986276, - "nauc_mrr_at_20_max": 0.35311180756517996, - "nauc_mrr_at_20_std": 0.6868445762814864, - "nauc_mrr_at_3_diff1": 0.4648021929978972, - "nauc_mrr_at_3_max": 0.3663517861281038, - "nauc_mrr_at_3_std": 0.7205919836574776, - "nauc_mrr_at_5_diff1": 0.5087022522085294, - "nauc_mrr_at_5_max": 0.3467565269949981, - "nauc_mrr_at_5_std": 0.683363887437365, - "nauc_ndcg_at_1000_diff1": 0.5054976057073902, - "nauc_ndcg_at_1000_max": 0.3504516506472678, - "nauc_ndcg_at_1000_std": 0.6795326611960923, - "nauc_ndcg_at_100_diff1": 0.5054976057073902, - "nauc_ndcg_at_100_max": 0.3504516506472678, - "nauc_ndcg_at_100_std": 0.6795326611960923, - "nauc_ndcg_at_10_diff1": 0.5529277638560192, - "nauc_ndcg_at_10_max": 0.3418676893611852, - "nauc_ndcg_at_10_std": 0.6438431211574437, - "nauc_ndcg_at_1_diff1": 0.3501554820571643, - "nauc_ndcg_at_1_max": 0.3599782655731798, - "nauc_ndcg_at_1_std": 0.7041481187202615, - "nauc_ndcg_at_20_diff1": 0.5075033561523076, - "nauc_ndcg_at_20_max": 0.35530204332453247, - "nauc_ndcg_at_20_std": 0.6798053401234159, - "nauc_ndcg_at_3_diff1": 0.48649313476516426, - "nauc_ndcg_at_3_max": 0.36277467680108344, - "nauc_ndcg_at_3_std": 0.7169907526506151, - "nauc_ndcg_at_5_diff1": 0.5565020379741223, - "nauc_ndcg_at_5_max": 0.33223528863871, - "nauc_ndcg_at_5_std": 0.6584725779305518, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": NaN, - "nauc_precision_at_100_max": NaN, - "nauc_precision_at_100_std": NaN, - "nauc_precision_at_10_diff1": 0.6643245257587758, - "nauc_precision_at_10_max": 0.3394378418139761, - "nauc_precision_at_10_std": 0.563123025662006, - "nauc_precision_at_1_diff1": 0.3501554820571643, - "nauc_precision_at_1_max": 0.3599782655731798, - "nauc_precision_at_1_std": 0.7041481187202615, - "nauc_precision_at_20_diff1": 0.5587451940825576, - "nauc_precision_at_20_max": 0.47434883444027603, - "nauc_precision_at_20_std": 0.6875358512675491, - "nauc_precision_at_3_diff1": 0.5321534332531633, - "nauc_precision_at_3_max": 0.3540044845277291, - "nauc_precision_at_3_std": 0.7076694823976849, - "nauc_precision_at_5_diff1": 0.6570524014466578, - "nauc_precision_at_5_max": 0.30012999498777754, - "nauc_precision_at_5_std": 0.6030407237879232, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": NaN, - "nauc_recall_at_100_max": NaN, - "nauc_recall_at_100_std": NaN, - "nauc_recall_at_10_diff1": 0.6643245257587762, - "nauc_recall_at_10_max": 0.3394378418139761, - "nauc_recall_at_10_std": 0.5631230256620064, - "nauc_recall_at_1_diff1": 0.3501554820571643, - "nauc_recall_at_1_max": 0.3599782655731798, - "nauc_recall_at_1_std": 0.7041481187202615, - "nauc_recall_at_20_diff1": 0.5587451940825555, - "nauc_recall_at_20_max": 0.47434883444027837, - "nauc_recall_at_20_std": 0.6875358512675467, - "nauc_recall_at_3_diff1": 0.5321534332531633, - "nauc_recall_at_3_max": 0.3540044845277292, - "nauc_recall_at_3_std": 0.7076694823976849, - "nauc_recall_at_5_diff1": 0.6570524014466579, - "nauc_recall_at_5_max": 0.3001299949877772, - "nauc_recall_at_5_std": 0.6030407237879231, - "ndcg_at_1": 0.08947, - "ndcg_at_10": 0.32202, - "ndcg_at_100": 0.41063, - "ndcg_at_1000": 0.41063, - "ndcg_at_20": 0.40326, - "ndcg_at_3": 0.18308, - "ndcg_at_5": 0.23594, - "precision_at_1": 0.08947, - "precision_at_10": 0.06468, - "precision_at_100": 0.01, - "precision_at_1000": 0.001, - "precision_at_20": 0.04832, - "precision_at_3": 0.08263, - "precision_at_5": 0.0753, - "recall_at_1": 0.08947, - "recall_at_10": 0.64678, - "recall_at_100": 1.0, - "recall_at_1000": 1.0, - "recall_at_20": 0.96645, - "recall_at_3": 0.2479, - "recall_at_5": 0.37651 - } - ] - }, - "task_name": "BLINKIT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Birdsnap.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Birdsnap.json deleted file mode 100644 index 7f46636f26..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Birdsnap.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "fd23015508be94f0b5b59d61630e4ea2536509e4", - "evaluation_time": 1263.4543855190277, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.46904376012965965, - "f1": 0.4359369143749953, - "f1_weighted": 0.46064550492427314, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.46904376012965965, - "scores_per_experiment": [ - { - "accuracy": 0.473257698541329, - "f1": 0.44226358144193817, - "f1_weighted": 0.46374933668288604 - }, - { - "accuracy": 0.4662344678552134, - "f1": 0.43198733439697296, - "f1_weighted": 0.45760840995849095 - }, - { - "accuracy": 0.4737979470556456, - "f1": 0.43945327318821287, - "f1_weighted": 0.4665146936621815 - }, - { - "accuracy": 0.4705564559697461, - "f1": 0.44085583091607183, - "f1_weighted": 0.46342910945828286 - }, - { - "accuracy": 0.4613722312263641, - "f1": 0.42512455193178084, - "f1_weighted": 0.4519259748595243 - } - ] - } - ] - }, - "task_name": "Birdsnap" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BirdsnapZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BirdsnapZeroShot.json deleted file mode 100644 index a3b1e48e41..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BirdsnapZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "e09b9dea248d579376684268cbedba28cd66b9b4", - "evaluation_time": 235.52802658081055, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.4057266342517558, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4057266342517558 - } - ] - }, - "task_name": "BirdsnapZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10.json deleted file mode 100644 index 7f2341a9d9..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", - "evaluation_time": 81.32431101799011, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.8985799999999999, - "f1": 0.899066397223649, - "f1_weighted": 0.899066397223649, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8985799999999999, - "scores_per_experiment": [ - { - "accuracy": 0.9015, - "f1": 0.9021178664975237, - "f1_weighted": 0.9021178664975236 - }, - { - "accuracy": 0.9052, - "f1": 0.9056178385863303, - "f1_weighted": 0.9056178385863304 - }, - { - "accuracy": 0.8925, - "f1": 0.8929011491758369, - "f1_weighted": 0.8929011491758371 - }, - { - "accuracy": 0.8964, - "f1": 0.8968836634711416, - "f1_weighted": 0.8968836634711413 - }, - { - "accuracy": 0.8973, - "f1": 0.897811468387413, - "f1_weighted": 0.8978114683874128 - } - ] - } - ] - }, - "task_name": "CIFAR10" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100.json deleted file mode 100644 index 3fd76447e6..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", - "evaluation_time": 112.59889626502991, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.67418, - "f1": 0.6752114591311454, - "f1_weighted": 0.6752114591311454, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.67418, - "scores_per_experiment": [ - { - "accuracy": 0.6725, - "f1": 0.6726574672039856, - "f1_weighted": 0.6726574672039854 - }, - { - "accuracy": 0.6809, - "f1": 0.6821528552320336, - "f1_weighted": 0.6821528552320336 - }, - { - "accuracy": 0.6778, - "f1": 0.6793950138107854, - "f1_weighted": 0.6793950138107854 - }, - { - "accuracy": 0.6707, - "f1": 0.6711676254605934, - "f1_weighted": 0.6711676254605933 - }, - { - "accuracy": 0.669, - "f1": 0.6706843339483296, - "f1_weighted": 0.6706843339483296 - } - ] - } - ] - }, - "task_name": "CIFAR100" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100Clustering.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100Clustering.json deleted file mode 100644 index 6ae63282d9..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100Clustering.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", - "evaluation_time": 42.84400296211243, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.0149, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.0149, - "v_measure": 0.5807028840317499 - } - ] - }, - "task_name": "CIFAR100Clustering" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100ZeroShot.json deleted file mode 100644 index f076bff887..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR100ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", - "evaluation_time": 41.24559950828552, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.6165, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6165 - } - ] - }, - "task_name": "CIFAR100ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10Clustering.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10Clustering.json deleted file mode 100644 index 2233e39698..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10Clustering.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", - "evaluation_time": 42.9359176158905, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.137, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.137, - "v_measure": 0.7384717195071175 - } - ] - }, - "task_name": "CIFAR10Clustering" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10ZeroShot.json deleted file mode 100644 index 04c094ec7e..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIFAR10ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", - "evaluation_time": 37.38677930831909, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.883, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.883 - } - ] - }, - "task_name": "CIFAR10ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2IRetrieval.json deleted file mode 100644 index 0dbe5112b4..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CIRRIT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", - "evaluation_time": 88.57046842575073, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.00815, - "cv_recall_at_10": 0.21295, - "cv_recall_at_100": 0.53357, - "cv_recall_at_1000": 0.86643, - "cv_recall_at_20": 0.28921, - "cv_recall_at_3": 0.09065, - "cv_recall_at_5": 0.13861, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.10026, - "map_at_1": 0.00408, - "map_at_10": 0.06541, - "map_at_100": 0.07611, - "map_at_1000": 0.07756, - "map_at_20": 0.07065, - "map_at_3": 0.04442, - "map_at_5": 0.05543, - "mrr_at_1": 0.00815347721822542, - "mrr_at_10": 0.06562540443835395, - "mrr_at_100": 0.0763215755779849, - "mrr_at_1000": 0.07777876289048734, - "mrr_at_20": 0.07086673474032301, - "mrr_at_3": 0.04468425259792174, - "mrr_at_5": 0.0556314948041567, - "nauc_cv_recall_at_1000_diff1": -0.1429646050677094, - "nauc_cv_recall_at_1000_max": 0.1924743520635352, - "nauc_cv_recall_at_1000_std": 0.5456463071997951, - "nauc_cv_recall_at_100_diff1": -0.042346564842556896, - "nauc_cv_recall_at_100_max": 0.1612547792092197, - "nauc_cv_recall_at_100_std": 0.2725592737752734, - "nauc_cv_recall_at_10_diff1": -0.05093849270187104, - "nauc_cv_recall_at_10_max": 0.11611298165158403, - "nauc_cv_recall_at_10_std": 0.12039571371818639, - "nauc_cv_recall_at_1_diff1": -0.516982591265645, - "nauc_cv_recall_at_1_max": 0.17146735270443028, - "nauc_cv_recall_at_1_std": -0.044358338988936034, - "nauc_cv_recall_at_20_diff1": -0.04728747828920591, - "nauc_cv_recall_at_20_max": 0.13714658323389076, - "nauc_cv_recall_at_20_std": 0.13725377667220118, - "nauc_cv_recall_at_3_diff1": -0.06302926023881217, - "nauc_cv_recall_at_3_max": 0.1113801332475869, - "nauc_cv_recall_at_3_std": 0.061263225738675126, - "nauc_cv_recall_at_5_diff1": -0.04626447590534941, - "nauc_cv_recall_at_5_max": 0.10547386265685323, - "nauc_cv_recall_at_5_std": 0.08783936831294922, - "nauc_map_at_1000_diff1": -0.08655636473413195, - "nauc_map_at_1000_max": 0.11676320143589596, - "nauc_map_at_1000_std": 0.09629099312167273, - "nauc_map_at_100_diff1": -0.08516530982285186, - "nauc_map_at_100_max": 0.11685859248705843, - "nauc_map_at_100_std": 0.09453524737415578, - "nauc_map_at_10_diff1": -0.08765926721523226, - "nauc_map_at_10_max": 0.10978450131161009, - "nauc_map_at_10_std": 0.08130576460840168, - "nauc_map_at_1_diff1": -0.5318107924683155, - "nauc_map_at_1_max": 0.20301772439590268, - "nauc_map_at_1_std": -0.021059886256622317, - "nauc_map_at_20_diff1": -0.08592919050491332, - "nauc_map_at_20_max": 0.11502979273505931, - "nauc_map_at_20_std": 0.08621208379307907, - "nauc_map_at_3_diff1": -0.10585711421716114, - "nauc_map_at_3_max": 0.10824597492299746, - "nauc_map_at_3_std": 0.04942655635325061, - "nauc_map_at_5_diff1": -0.0903941122012634, - "nauc_map_at_5_max": 0.106932714398111, - "nauc_map_at_5_std": 0.06656522457054902, - "nauc_mrr_at_1000_diff1": -0.0878544053521387, - "nauc_mrr_at_1000_max": 0.11835254552849916, - "nauc_mrr_at_1000_std": 0.09563928573449795, - "nauc_mrr_at_100_diff1": -0.08646539076860782, - "nauc_mrr_at_100_max": 0.11844073105135247, - "nauc_mrr_at_100_std": 0.0938876335450531, - "nauc_mrr_at_10_diff1": -0.08899929752404169, - "nauc_mrr_at_10_max": 0.11141118460268691, - "nauc_mrr_at_10_std": 0.08072528807311119, - "nauc_mrr_at_1_diff1": -0.516982591265645, - "nauc_mrr_at_1_max": 0.17146735270443028, - "nauc_mrr_at_1_std": -0.044358338988936034, - "nauc_mrr_at_20_diff1": -0.08723030616085735, - "nauc_mrr_at_20_max": 0.11660026484773199, - "nauc_mrr_at_20_std": 0.08564973677674821, - "nauc_mrr_at_3_diff1": -0.10793762866631498, - "nauc_mrr_at_3_max": 0.11188839551635246, - "nauc_mrr_at_3_std": 0.04886788042390466, - "nauc_mrr_at_5_diff1": -0.09176834345451616, - "nauc_mrr_at_5_max": 0.10901080188380181, - "nauc_mrr_at_5_std": 0.06597097726127213, - "nauc_ndcg_at_1000_diff1": -0.08344132954414758, - "nauc_ndcg_at_1000_max": 0.1323215007612839, - "nauc_ndcg_at_1000_std": 0.17697700366712762, - "nauc_ndcg_at_100_diff1": -0.06471556789805906, - "nauc_ndcg_at_100_max": 0.13367762668005959, - "nauc_ndcg_at_100_std": 0.1585535458458699, - "nauc_ndcg_at_10_diff1": -0.07011937821033006, - "nauc_ndcg_at_10_max": 0.11288519695312943, - "nauc_ndcg_at_10_std": 0.09866561563226296, - "nauc_ndcg_at_1_diff1": -0.5186323737973859, - "nauc_ndcg_at_1_max": 0.19513971405573238, - "nauc_ndcg_at_1_std": -0.03383801602207826, - "nauc_ndcg_at_20_diff1": -0.06694529638460721, - "nauc_ndcg_at_20_max": 0.12469099422056072, - "nauc_ndcg_at_20_std": 0.1092102137002337, - "nauc_ndcg_at_3_diff1": -0.08867212008892655, - "nauc_ndcg_at_3_max": 0.10990739346950268, - "nauc_ndcg_at_3_std": 0.0543947490857072, - "nauc_ndcg_at_5_diff1": -0.07145417341479289, - "nauc_ndcg_at_5_max": 0.10696288747042368, - "nauc_ndcg_at_5_std": 0.07571492220327662, - "nauc_precision_at_1000_diff1": -0.18314342593006072, - "nauc_precision_at_1000_max": 0.19346644885136122, - "nauc_precision_at_1000_std": 0.47116734223208256, - "nauc_precision_at_100_diff1": -0.06370954016671407, - "nauc_precision_at_100_max": 0.16241220181965255, - "nauc_precision_at_100_std": 0.258016779609776, - "nauc_precision_at_10_diff1": -0.07421873744815492, - "nauc_precision_at_10_max": 0.12076870044196765, - "nauc_precision_at_10_std": 0.11212464069366684, - "nauc_precision_at_1_diff1": -0.5186323737973859, - "nauc_precision_at_1_max": 0.19513971405573238, - "nauc_precision_at_1_std": -0.03383801602207826, - "nauc_precision_at_20_diff1": -0.06793713334562095, - "nauc_precision_at_20_max": 0.1402319646787195, - "nauc_precision_at_20_std": 0.1291706714805028, - "nauc_precision_at_3_diff1": -0.10265385272093064, - "nauc_precision_at_3_max": 0.11808887171461854, - "nauc_precision_at_3_std": 0.052103940853181874, - "nauc_precision_at_5_diff1": -0.07759274159246009, - "nauc_precision_at_5_max": 0.1124438194370848, - "nauc_precision_at_5_std": 0.07919386090270071, - "nauc_recall_at_1000_diff1": -0.14300824800294304, - "nauc_recall_at_1000_max": 0.19179659255731416, - "nauc_recall_at_1000_std": 0.545720347238759, - "nauc_recall_at_100_diff1": -0.041986859938095734, - "nauc_recall_at_100_max": 0.16127782528121198, - "nauc_recall_at_100_std": 0.27283270613912575, - "nauc_recall_at_10_diff1": -0.05065793884544601, - "nauc_recall_at_10_max": 0.11649110613991276, - "nauc_recall_at_10_std": 0.12019520504934413, - "nauc_recall_at_1_diff1": -0.5318107924683155, - "nauc_recall_at_1_max": 0.20301772439590268, - "nauc_recall_at_1_std": -0.021059886256622317, - "nauc_recall_at_20_diff1": -0.04704172155925913, - "nauc_recall_at_20_max": 0.1374810167985767, - "nauc_recall_at_20_std": 0.137082012873361, - "nauc_recall_at_3_diff1": -0.062451024117901044, - "nauc_recall_at_3_max": 0.11020315329162346, - "nauc_recall_at_3_std": 0.061213944422035795, - "nauc_recall_at_5_diff1": -0.04626447590534941, - "nauc_recall_at_5_max": 0.10547386265685323, - "nauc_recall_at_5_std": 0.08783936831294922, - "ndcg_at_1": 0.00839, - "ndcg_at_10": 0.10026, - "ndcg_at_100": 0.16319, - "ndcg_at_1000": 0.20471, - "ndcg_at_20": 0.11947, - "ndcg_at_3": 0.05637, - "ndcg_at_5": 0.07618, - "precision_at_1": 0.00839, - "precision_at_10": 0.02228, - "precision_at_100": 0.00544, - "precision_at_1000": 0.00088, - "precision_at_20": 0.01495, - "precision_at_3": 0.03341, - "precision_at_5": 0.02969, - "recall_at_1": 0.00408, - "recall_at_10": 0.21283, - "recall_at_100": 0.53345, - "recall_at_1000": 0.86631, - "recall_at_20": 0.28909, - "recall_at_3": 0.09053, - "recall_at_5": 0.13861 - } - ] - }, - "task_name": "CIRRIT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVR.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVR.json deleted file mode 100644 index 468ad76812..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVR.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "ec9c04224a95836ca0344a6000ec8d8bc8a6d4f2", - "evaluation_time": 64.04479908943176, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "accuracy": 0.1634, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.1634 - } - ] - }, - "task_name": "CLEVER" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVRCount.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVRCount.json deleted file mode 100644 index 565041de13..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CLEVRCount.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "8b5dce4d5393a04fb58b9261b22a881b02e379b1", - "evaluation_time": 67.00646996498108, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "accuracy": 0.232, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.232 - } - ] - }, - "task_name": "CLEVRCount" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CUB200I2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CUB200I2IRetrieval.json deleted file mode 100644 index a46d52dd1e..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CUB200I2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "ad08c1307b15a226bf1b64e62656a17f1f85f7ec", - "evaluation_time": 101.24021887779236, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.5195, - "cv_recall_at_10": 0.8835, - "cv_recall_at_100": 0.99275, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.94184, - "cv_recall_at_3": 0.72489, - "cv_recall_at_5": 0.8029, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5195, - "map_at_1": 0.0, - "map_at_10": 0.06775, - "map_at_100": 0.17109, - "map_at_1000": 0.19946, - "map_at_20": 0.10637, - "map_at_3": 0.01856, - "map_at_5": 0.03526, - "mrr_at_1": 0.0, - "mrr_at_10": 0.345117417580307, - "mrr_at_100": 0.3515074205178541, - "mrr_at_1000": 0.3515481539320064, - "mrr_at_20": 0.3499334718788236, - "mrr_at_3": 0.30405016683925506, - "mrr_at_5": 0.33131975606949415, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.29829479476891246, - "nauc_cv_recall_at_100_max": -0.35475926675448466, - "nauc_cv_recall_at_100_std": 0.16066304907385462, - "nauc_cv_recall_at_10_diff1": -0.2786391142334684, - "nauc_cv_recall_at_10_max": -0.04765779245271946, - "nauc_cv_recall_at_10_std": 0.2248031704570365, - "nauc_cv_recall_at_1_diff1": -0.1779435022557211, - "nauc_cv_recall_at_1_max": -0.02718179344833177, - "nauc_cv_recall_at_1_std": 0.03655854432763116, - "nauc_cv_recall_at_20_diff1": -0.31932337225079194, - "nauc_cv_recall_at_20_max": -0.037830258082760186, - "nauc_cv_recall_at_20_std": 0.26690155092620627, - "nauc_cv_recall_at_3_diff1": -0.20332785145569104, - "nauc_cv_recall_at_3_max": -0.04697587564711363, - "nauc_cv_recall_at_3_std": 0.1258221057180091, - "nauc_cv_recall_at_5_diff1": -0.23347006556105487, - "nauc_cv_recall_at_5_max": -0.031142268954388815, - "nauc_cv_recall_at_5_std": 0.1516526846178296, - "nauc_map_at_1000_diff1": -0.2062688124757473, - "nauc_map_at_1000_max": -0.0035555471459080617, - "nauc_map_at_1000_std": 0.23080503561089502, - "nauc_map_at_100_diff1": -0.19641682305588465, - "nauc_map_at_100_max": -0.0026445524622137747, - "nauc_map_at_100_std": 0.20444263793369596, - "nauc_map_at_10_diff1": -0.18598512115387061, - "nauc_map_at_10_max": -0.005392449194288058, - "nauc_map_at_10_std": 0.1014171349773814, - "nauc_map_at_1_diff1": NaN, - "nauc_map_at_1_max": NaN, - "nauc_map_at_1_std": NaN, - "nauc_map_at_20_diff1": -0.18851990685363146, - "nauc_map_at_20_max": 0.0023419122967712775, - "nauc_map_at_20_std": 0.139746932457822, - "nauc_map_at_3_diff1": -0.15825409921589528, - "nauc_map_at_3_max": -0.011497137589137481, - "nauc_map_at_3_std": 0.056414830301161185, - "nauc_map_at_5_diff1": -0.173932631810766, - "nauc_map_at_5_max": -0.004713040702303579, - "nauc_map_at_5_std": 0.07084949917645643, - "nauc_mrr_at_1000_diff1": -0.19955607289570382, - "nauc_mrr_at_1000_max": -0.03238896690902184, - "nauc_mrr_at_1000_std": 0.08628495504725432, - "nauc_mrr_at_100_diff1": -0.1996164211591153, - "nauc_mrr_at_100_max": -0.03249899847707879, - "nauc_mrr_at_100_std": 0.08628990870899618, - "nauc_mrr_at_10_diff1": -0.20015919600929724, - "nauc_mrr_at_10_max": -0.03390649579122204, - "nauc_mrr_at_10_std": 0.09021220925342546, - "nauc_mrr_at_1_diff1": NaN, - "nauc_mrr_at_1_max": NaN, - "nauc_mrr_at_1_std": NaN, - "nauc_mrr_at_20_diff1": -0.2007342426577916, - "nauc_mrr_at_20_max": -0.032351629887101385, - "nauc_mrr_at_20_std": 0.08872643632144, - "nauc_mrr_at_3_diff1": -0.1896169258292634, - "nauc_mrr_at_3_max": -0.02905559729986968, - "nauc_mrr_at_3_std": 0.07153826033664305, - "nauc_mrr_at_5_diff1": -0.20025097945245238, - "nauc_mrr_at_5_max": -0.03291097332957811, - "nauc_mrr_at_5_std": 0.08662818761457174, - "nauc_ndcg_at_1000_diff1": -0.25364131167732773, - "nauc_ndcg_at_1000_max": -0.00969178999184677, - "nauc_ndcg_at_1000_std": 0.3093148634981729, - "nauc_ndcg_at_100_diff1": -0.21304640197927943, - "nauc_ndcg_at_100_max": -0.005994303743098639, - "nauc_ndcg_at_100_std": 0.24287453393772768, - "nauc_ndcg_at_10_diff1": -0.20032000568929073, - "nauc_ndcg_at_10_max": -0.00992656456191809, - "nauc_ndcg_at_10_std": 0.10587882848001441, - "nauc_ndcg_at_1_diff1": NaN, - "nauc_ndcg_at_1_max": NaN, - "nauc_ndcg_at_1_std": NaN, - "nauc_ndcg_at_20_diff1": -0.20182484259143996, - "nauc_ndcg_at_20_max": -0.004590142857157939, - "nauc_ndcg_at_20_std": 0.1429465836051686, - "nauc_ndcg_at_3_diff1": -0.17996517359211764, - "nauc_ndcg_at_3_max": -0.015207435524616678, - "nauc_ndcg_at_3_std": 0.060135755734251166, - "nauc_ndcg_at_5_diff1": -0.1920019943385643, - "nauc_ndcg_at_5_max": -0.01018908132779398, - "nauc_ndcg_at_5_std": 0.07561645521061498, - "nauc_precision_at_1000_diff1": -0.29703869433778635, - "nauc_precision_at_1000_max": -0.01298049022298684, - "nauc_precision_at_1000_std": 0.4308144194919067, - "nauc_precision_at_100_diff1": -0.21818017775568854, - "nauc_precision_at_100_max": -0.006397344189188299, - "nauc_precision_at_100_std": 0.2962068061740296, - "nauc_precision_at_10_diff1": -0.19845507492338102, - "nauc_precision_at_10_max": -0.008714316194455004, - "nauc_precision_at_10_std": 0.11541882266156184, - "nauc_precision_at_1_diff1": NaN, - "nauc_precision_at_1_max": NaN, - "nauc_precision_at_1_std": NaN, - "nauc_precision_at_20_diff1": -0.20066511163855275, - "nauc_precision_at_20_max": -0.0021458092054351886, - "nauc_precision_at_20_std": 0.16126858774816208, - "nauc_precision_at_3_diff1": -0.1802516939125809, - "nauc_precision_at_3_max": -0.01351037586517892, - "nauc_precision_at_3_std": 0.06347722377914423, - "nauc_precision_at_5_diff1": -0.19340808404985577, - "nauc_precision_at_5_max": -0.008276455491399038, - "nauc_precision_at_5_std": 0.08018404870245943, - "nauc_recall_at_1000_diff1": -0.2915096272448527, - "nauc_recall_at_1000_max": -0.013349971323190575, - "nauc_recall_at_1000_std": 0.5447028375955068, - "nauc_recall_at_100_diff1": -0.20185480358135358, - "nauc_recall_at_100_max": -0.004341021867975908, - "nauc_recall_at_100_std": 0.30547878506571907, - "nauc_recall_at_10_diff1": -0.18095986501038946, - "nauc_recall_at_10_max": -0.0073341347622717015, - "nauc_recall_at_10_std": 0.11791560298106384, - "nauc_recall_at_1_diff1": NaN, - "nauc_recall_at_1_max": NaN, - "nauc_recall_at_1_std": NaN, - "nauc_recall_at_20_diff1": -0.1845657988866603, - "nauc_recall_at_20_max": -0.0002001744986707724, - "nauc_recall_at_20_std": 0.16384027707020302, - "nauc_recall_at_3_diff1": -0.15968586730811474, - "nauc_recall_at_3_max": -0.01273857066006781, - "nauc_recall_at_3_std": 0.06752451141789033, - "nauc_recall_at_5_diff1": -0.1755997981980233, - "nauc_recall_at_5_max": -0.00769746739652761, - "nauc_recall_at_5_std": 0.08423476571748854, - "ndcg_at_1": 0.0, - "ndcg_at_10": 0.3074, - "ndcg_at_100": 0.37587, - "ndcg_at_1000": 0.53891, - "ndcg_at_20": 0.2812, - "ndcg_at_3": 0.26012, - "ndcg_at_5": 0.29826, - "precision_at_1": 0.0, - "precision_at_10": 0.33804, - "precision_at_100": 0.13023, - "precision_at_1000": 0.02473, - "precision_at_20": 0.28365, - "precision_at_3": 0.32419, - "precision_at_5": 0.35333, - "recall_at_1": 0.0, - "recall_at_10": 0.12092, - "recall_at_100": 0.46229, - "recall_at_1000": 0.87518, - "recall_at_20": 0.20234, - "recall_at_3": 0.03487, - "recall_at_5": 0.06331 - } - ] - }, - "task_name": "CUB200I2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchCount.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchCount.json deleted file mode 100644 index a888119c3f..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchCount.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 11.933090686798096, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.06598984771573604, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.06598984771573604 - } - ] - }, - "task_name": "CVBenchCount" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDepth.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDepth.json deleted file mode 100644 index cb647a3405..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDepth.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 8.507234334945679, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.4533333333333333, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4533333333333333 - } - ] - }, - "task_name": "CVBenchDepth" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDistance.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDistance.json deleted file mode 100644 index d8246da01d..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchDistance.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 8.697332382202148, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.46166666666666667, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.46166666666666667 - } - ] - }, - "task_name": "CVBenchDistance" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchRelation.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchRelation.json deleted file mode 100644 index 02145addc1..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/CVBenchRelation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 4.9446187019348145, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.4907692307692308, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4907692307692308 - } - ] - }, - "task_name": "CVBenchRelation" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101.json deleted file mode 100644 index 95f11f5a9e..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", - "evaluation_time": 118.63952732086182, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.9175542406311636, - "f1": 0.8767202332660267, - "f1_weighted": 0.9187633243329824, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9175542406311636, - "scores_per_experiment": [ - { - "accuracy": 0.9119000657462196, - "f1": 0.8710825099573519, - "f1_weighted": 0.9133422092543803 - }, - { - "accuracy": 0.9275147928994083, - "f1": 0.8813945266330111, - "f1_weighted": 0.9291005925647753 - }, - { - "accuracy": 0.913379355687048, - "f1": 0.8738935806115447, - "f1_weighted": 0.9148291144424647 - }, - { - "accuracy": 0.9155161078238001, - "f1": 0.8709633024188326, - "f1_weighted": 0.9167341340128863 - }, - { - "accuracy": 0.9194608809993425, - "f1": 0.8862672467093923, - "f1_weighted": 0.9198105713904057 - } - ] - } - ] - }, - "task_name": "Caltech101" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101ZeroShot.json deleted file mode 100644 index d914345212..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Caltech101ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", - "evaluation_time": 39.51401495933533, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.7899408284023669, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7899408284023669 - } - ] - }, - "task_name": "Caltech101ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211.json deleted file mode 100644 index e59ca3c0c2..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "dataset_revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", - "evaluation_time": 516.9000871181488, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "accuracy": 0.1495260663507109, - "f1": 0.1497060161021076, - "f1_weighted": 0.1497060161021076, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.1495260663507109, - "scores_per_experiment": [ - { - "accuracy": 0.1495260663507109, - "f1": 0.1497060161021076, - "f1_weighted": 0.1497060161021076 - } - ] - } - ] - }, - "task_name": "Country211" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211ZeroShot.json deleted file mode 100644 index b14c6829c9..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Country211ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", - "evaluation_time": 167.4767632484436, - "kg_co2_emissions": null, - "mteb_version": "1.12.49", - "scores": { - "test": [ - { - "accuracy": 0.15962085308056873, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.15962085308056873 - } - ] - }, - "task_name": "Country211ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTD.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTD.json deleted file mode 100644 index 22f21ae589..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTD.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", - "evaluation_time": 124.488609790802, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.6753191489361702, - "f1": 0.6745404047233853, - "f1_weighted": 0.6745404047233852, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6753191489361702, - "scores_per_experiment": [ - { - "accuracy": 0.6808510638297872, - "f1": 0.6808368640933851, - "f1_weighted": 0.680836864093385 - }, - { - "accuracy": 0.676063829787234, - "f1": 0.6723410903725255, - "f1_weighted": 0.6723410903725255 - }, - { - "accuracy": 0.676595744680851, - "f1": 0.6772514855600662, - "f1_weighted": 0.6772514855600661 - }, - { - "accuracy": 0.6734042553191489, - "f1": 0.6732365886117732, - "f1_weighted": 0.6732365886117734 - }, - { - "accuracy": 0.6696808510638298, - "f1": 0.6690359949791764, - "f1_weighted": 0.6690359949791764 - } - ] - } - ] - }, - "task_name": "DTD" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTDZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTDZeroShot.json deleted file mode 100644 index c9dd323257..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/DTDZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", - "evaluation_time": 21.39889645576477, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.41914893617021276, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.41914893617021276 - } - ] - }, - "task_name": "DTDZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EDIST2ITRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EDIST2ITRetrieval.json deleted file mode 100644 index b8c457c9ae..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EDIST2ITRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "68c47ef3e49ef883073b3358bd4243eeca0aee9a", - "evaluation_time": 3594.926486968994, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.2058, - "cv_recall_at_10": 0.50231, - "cv_recall_at_100": 0.76797, - "cv_recall_at_1000": 0.92873, - "cv_recall_at_20": 0.5992, - "cv_recall_at_3": 0.34002, - "cv_recall_at_5": 0.415, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.24494, - "map_at_1": 0.07615, - "map_at_10": 0.17644, - "map_at_100": 0.19953, - "map_at_1000": 0.20149, - "map_at_20": 0.18958, - "map_at_3": 0.1292, - "map_at_5": 0.15486, - "mrr_at_1": 0.20580067880283864, - "mrr_at_10": 0.2925150477757707, - "mrr_at_100": 0.3034272424612011, - "mrr_at_1000": 0.30417101259312895, - "mrr_at_20": 0.29926672507651825, - "mrr_at_3": 0.2639103157461694, - "mrr_at_5": 0.28098837807261257, - "nauc_cv_recall_at_1000_diff1": 0.08692340997106167, - "nauc_cv_recall_at_1000_max": -0.08260050261557855, - "nauc_cv_recall_at_1000_std": 0.3207360002955632, - "nauc_cv_recall_at_100_diff1": 0.06500932146729019, - "nauc_cv_recall_at_100_max": 0.03548329590032714, - "nauc_cv_recall_at_100_std": 0.06632350606013385, - "nauc_cv_recall_at_10_diff1": 0.12947130572805773, - "nauc_cv_recall_at_10_max": 0.04226853926632297, - "nauc_cv_recall_at_10_std": -0.017992685315635543, - "nauc_cv_recall_at_1_diff1": 0.2158323552712354, - "nauc_cv_recall_at_1_max": 0.04444742333087822, - "nauc_cv_recall_at_1_std": 0.011407565122854697, - "nauc_cv_recall_at_20_diff1": 0.11043702121642354, - "nauc_cv_recall_at_20_max": 0.07178164802818354, - "nauc_cv_recall_at_20_std": -0.012838329235348646, - "nauc_cv_recall_at_3_diff1": 0.16385487158792883, - "nauc_cv_recall_at_3_max": 0.038761705606404406, - "nauc_cv_recall_at_3_std": -0.015444146140702468, - "nauc_cv_recall_at_5_diff1": 0.1342474969284102, - "nauc_cv_recall_at_5_max": 0.05126831131276464, - "nauc_cv_recall_at_5_std": -0.017244011947458535, - "nauc_map_at_1000_diff1": 0.16203592786234705, - "nauc_map_at_1000_max": 0.030620033890621814, - "nauc_map_at_1000_std": -0.040264811401896076, - "nauc_map_at_100_diff1": 0.16217564027873344, - "nauc_map_at_100_max": 0.030335963317346233, - "nauc_map_at_100_std": -0.04191441091203249, - "nauc_map_at_10_diff1": 0.1668748719138034, - "nauc_map_at_10_max": 0.019665458769663652, - "nauc_map_at_10_std": -0.061030326723274915, - "nauc_map_at_1_diff1": 0.20075357538267108, - "nauc_map_at_1_max": -0.013665387037587046, - "nauc_map_at_1_std": -0.07766759966282027, - "nauc_map_at_20_diff1": 0.16386357584503752, - "nauc_map_at_20_max": 0.027477956932007105, - "nauc_map_at_20_std": -0.05360888917496779, - "nauc_map_at_3_diff1": 0.1828162464833454, - "nauc_map_at_3_max": -0.00042884175525836654, - "nauc_map_at_3_std": -0.07592876118464074, - "nauc_map_at_5_diff1": 0.17337455887715922, - "nauc_map_at_5_max": 0.012236544553715457, - "nauc_map_at_5_std": -0.07362737289120426, - "nauc_mrr_at_1000_diff1": 0.182198101426749, - "nauc_mrr_at_1000_max": 0.0448381948571804, - "nauc_mrr_at_1000_std": -0.0001618378292135322, - "nauc_mrr_at_100_diff1": 0.18191433551703926, - "nauc_mrr_at_100_max": 0.04500572110663768, - "nauc_mrr_at_100_std": -0.0003744867725387041, - "nauc_mrr_at_10_diff1": 0.1826285591048027, - "nauc_mrr_at_10_max": 0.044086235201035766, - "nauc_mrr_at_10_std": -0.00239517797969451, - "nauc_mrr_at_1_diff1": 0.2158323552712354, - "nauc_mrr_at_1_max": 0.04444742333087822, - "nauc_mrr_at_1_std": 0.011407565122854697, - "nauc_mrr_at_20_diff1": 0.18160479504101013, - "nauc_mrr_at_20_max": 0.045950432193568896, - "nauc_mrr_at_20_std": -0.0017360413667605174, - "nauc_mrr_at_3_diff1": 0.19059323121455957, - "nauc_mrr_at_3_max": 0.042015172128788573, - "nauc_mrr_at_3_std": -0.0016791794293602408, - "nauc_mrr_at_5_diff1": 0.18283458974812974, - "nauc_mrr_at_5_max": 0.045060291442748604, - "nauc_mrr_at_5_std": -0.002104815863062162, - "nauc_ndcg_at_1000_diff1": 0.15156096151374038, - "nauc_ndcg_at_1000_max": 0.0406493878022788, - "nauc_ndcg_at_1000_std": 0.016921787353154537, - "nauc_ndcg_at_100_diff1": 0.15054121374308563, - "nauc_ndcg_at_100_max": 0.041154172886530165, - "nauc_ndcg_at_100_std": -0.001532780455810332, - "nauc_ndcg_at_10_diff1": 0.16153464028430178, - "nauc_ndcg_at_10_max": 0.02481835763026216, - "nauc_ndcg_at_10_std": -0.05078346238124407, - "nauc_ndcg_at_1_diff1": 0.21820190922791943, - "nauc_ndcg_at_1_max": 0.04420174217766181, - "nauc_ndcg_at_1_std": 0.009546631680834063, - "nauc_ndcg_at_20_diff1": 0.15686132177509635, - "nauc_ndcg_at_20_max": 0.037578777094029674, - "nauc_ndcg_at_20_std": -0.041502780883555516, - "nauc_ndcg_at_3_diff1": 0.17126079900331548, - "nauc_ndcg_at_3_max": 0.03401910693600456, - "nauc_ndcg_at_3_std": -0.031094315658336043, - "nauc_ndcg_at_5_diff1": 0.16453188548209727, - "nauc_ndcg_at_5_max": 0.029388384612040384, - "nauc_ndcg_at_5_std": -0.04943641033736309, - "nauc_precision_at_1000_diff1": -0.007222168267252968, - "nauc_precision_at_1000_max": 0.1160441913815496, - "nauc_precision_at_1000_std": 0.17313093891904213, - "nauc_precision_at_100_diff1": 0.0325067497088795, - "nauc_precision_at_100_max": 0.11942644169349934, - "nauc_precision_at_100_std": 0.15151622423514086, - "nauc_precision_at_10_diff1": 0.09750520702159386, - "nauc_precision_at_10_max": 0.08979528068901292, - "nauc_precision_at_10_std": 0.04192968105120749, - "nauc_precision_at_1_diff1": 0.21820190922791943, - "nauc_precision_at_1_max": 0.04420174217766181, - "nauc_precision_at_1_std": 0.009546631680834063, - "nauc_precision_at_20_diff1": 0.07494638242908558, - "nauc_precision_at_20_max": 0.11107312948815577, - "nauc_precision_at_20_std": 0.0678622239159027, - "nauc_precision_at_3_diff1": 0.15301239695643, - "nauc_precision_at_3_max": 0.06573995876968261, - "nauc_precision_at_3_std": 0.006095465119184258, - "nauc_precision_at_5_diff1": 0.12418883915813934, - "nauc_precision_at_5_max": 0.08324108054680586, - "nauc_precision_at_5_std": 0.009395259541333896, - "nauc_recall_at_1000_diff1": 0.09463810369253761, - "nauc_recall_at_1000_max": -0.0560747556388996, - "nauc_recall_at_1000_std": 0.3572969720013806, - "nauc_recall_at_100_diff1": 0.08317532906551336, - "nauc_recall_at_100_max": 0.009897416890410404, - "nauc_recall_at_100_std": 0.05893746815666451, - "nauc_recall_at_10_diff1": 0.11874898395838694, - "nauc_recall_at_10_max": -0.004616050285708542, - "nauc_recall_at_10_std": -0.08463423097463776, - "nauc_recall_at_1_diff1": 0.20075357538267108, - "nauc_recall_at_1_max": -0.013665387037587046, - "nauc_recall_at_1_std": -0.07766759966282027, - "nauc_recall_at_20_diff1": 0.10907207863790737, - "nauc_recall_at_20_max": 0.019150773341658055, - "nauc_recall_at_20_std": -0.07091348769187986, - "nauc_recall_at_3_diff1": 0.15218311570784662, - "nauc_recall_at_3_max": -0.01125829888252278, - "nauc_recall_at_3_std": -0.08844619980086296, - "nauc_recall_at_5_diff1": 0.1273697162111514, - "nauc_recall_at_5_max": 0.0008835763863017466, - "nauc_recall_at_5_std": -0.09530587120686718, - "ndcg_at_1": 0.20518, - "ndcg_at_10": 0.24494, - "ndcg_at_100": 0.32936, - "ndcg_at_1000": 0.36533, - "ndcg_at_20": 0.27892, - "ndcg_at_3": 0.19917, - "ndcg_at_5": 0.21463, - "precision_at_1": 0.20518, - "precision_at_10": 0.09327, - "precision_at_100": 0.01747, - "precision_at_1000": 0.00229, - "precision_at_20": 0.06034, - "precision_at_3": 0.16034, - "precision_at_5": 0.13533, - "recall_at_1": 0.07615, - "recall_at_10": 0.31954, - "recall_at_100": 0.64044, - "recall_at_1000": 0.8719, - "recall_at_20": 0.42323, - "recall_at_3": 0.16623, - "recall_at_5": 0.2331 - } - ] - }, - "task_name": "EDIST2ITRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSAT.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSAT.json deleted file mode 100644 index 98c522b3fb..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSAT.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", - "evaluation_time": 46.82566833496094, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.8501481481481481, - "f1": 0.8469797204361242, - "f1_weighted": 0.8496265755865398, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8501481481481481, - "scores_per_experiment": [ - { - "accuracy": 0.8518518518518519, - "f1": 0.8495295548783852, - "f1_weighted": 0.8511665064639554 - }, - { - "accuracy": 0.8501851851851852, - "f1": 0.8471340637143676, - "f1_weighted": 0.8499896449882506 - }, - { - "accuracy": 0.8475925925925926, - "f1": 0.8422738482428335, - "f1_weighted": 0.8460446242004349 - }, - { - "accuracy": 0.8605555555555555, - "f1": 0.8580738304964756, - "f1_weighted": 0.8608069947542264 - }, - { - "accuracy": 0.8405555555555555, - "f1": 0.8378873048485588, - "f1_weighted": 0.840125107525832 - } - ] - } - ] - }, - "task_name": "EuroSAT" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json deleted file mode 100644 index af34ef21d1..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/EuroSATZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", - "evaluation_time": 24.88299822807312, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.4974074074074074, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4974074074074074 - } - ] - }, - "task_name": "EuroSATZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013.json deleted file mode 100644 index 03d577f7b4..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", - "evaluation_time": 49.89526414871216, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.4285594873223739, - "f1": 0.3843822780757347, - "f1_weighted": 0.43760737475533196, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4285594873223739, - "scores_per_experiment": [ - { - "accuracy": 0.4179437169127891, - "f1": 0.37722976261634705, - "f1_weighted": 0.43062554573695727 - }, - { - "accuracy": 0.45388687656728893, - "f1": 0.3994675689562795, - "f1_weighted": 0.459751540669273 - }, - { - "accuracy": 0.4360546113123433, - "f1": 0.39754111862374936, - "f1_weighted": 0.444949254019621 - }, - { - "accuracy": 0.45723042630259125, - "f1": 0.3974530741571466, - "f1_weighted": 0.45463208715209835 - }, - { - "accuracy": 0.37768180551685704, - "f1": 0.3502198660251512, - "f1_weighted": 0.3980784461987101 - } - ] - } - ] - }, - "task_name": "FER2013" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013ZeroShot.json deleted file mode 100644 index 3366d7e8e7..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FER2013ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", - "evaluation_time": 30.47609853744507, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.4352187238785177, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4352187238785177 - } - ] - }, - "task_name": "FER2013ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraft.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraft.json deleted file mode 100644 index 9243b849f8..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraft.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "dataset_revision": "91860adfc9a09aabca5cddb5247442109b38e213", - "evaluation_time": 779.7828733921051, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.3149414941494149, - "f1": 0.312693783815878, - "f1_weighted": 0.312678675983208, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.3149414941494149, - "scores_per_experiment": [ - { - "accuracy": 0.3114311431143114, - "f1": 0.31106599523075695, - "f1_weighted": 0.31096314031434213 - }, - { - "accuracy": 0.3069306930693069, - "f1": 0.3038393838854474, - "f1_weighted": 0.30395665785927795 - }, - { - "accuracy": 0.3096309630963096, - "f1": 0.3087612511688636, - "f1_weighted": 0.3086759599811729 - }, - { - "accuracy": 0.3171317131713171, - "f1": 0.31397818189887455, - "f1_weighted": 0.313804173735766 - }, - { - "accuracy": 0.32163216321632165, - "f1": 0.31683692491491705, - "f1_weighted": 0.3168587733660707 - }, - { - "accuracy": 0.3126312631263126, - "f1": 0.3114657147965915, - "f1_weighted": 0.31150939456099336 - }, - { - "accuracy": 0.32283228322832286, - "f1": 0.32236568456369397, - "f1_weighted": 0.32249530202709764 - }, - { - "accuracy": 0.3102310231023102, - "f1": 0.30481253675964476, - "f1_weighted": 0.3047149182274533 - }, - { - "accuracy": 0.3114311431143114, - "f1": 0.3087445080505303, - "f1_weighted": 0.3087629926503773 - }, - { - "accuracy": 0.32553255325532554, - "f1": 0.3250676568894592, - "f1_weighted": 0.3250454471095288 - } - ] - } - ] - }, - "task_name": "FGVCAircraft" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraftZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraftZeroShot.json deleted file mode 100644 index b78af9c07b..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FGVCAircraftZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "91860adfc9a09aabca5cddb5247442109b38e213", - "evaluation_time": 91.58129215240479, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.18871887188718872, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.18871887188718872 - } - ] - }, - "task_name": "FGVCAircraftZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json deleted file mode 100644 index 9e86b31861..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", - "evaluation_time": 1013.2197253704071, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.2997, - "cv_recall_at_10": 0.43985, - "cv_recall_at_100": 0.57849, - "cv_recall_at_1000": 0.70921, - "cv_recall_at_20": 0.47864, - "cv_recall_at_3": 0.37042, - "cv_recall_at_5": 0.40053, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.2997, - "map_at_1": 0.29872, - "map_at_10": 0.34317, - "map_at_100": 0.34826, - "map_at_1000": 0.34882, - "map_at_20": 0.34587, - "map_at_3": 0.33092, - "map_at_5": 0.33784, - "mrr_at_1": 0.2996981132075472, - "mrr_at_10": 0.34371009284216936, - "mrr_at_100": 0.3488068311925895, - "mrr_at_1000": 0.3493644373926243, - "mrr_at_20": 0.3464172969474597, - "mrr_at_3": 0.3314716981132089, - "mrr_at_5": 0.33839622641509476, - "nauc_cv_recall_at_1000_diff1": 0.8843727921961846, - "nauc_cv_recall_at_1000_max": -1.1122059360557914, - "nauc_cv_recall_at_1000_std": -0.9675872247776091, - "nauc_cv_recall_at_100_diff1": 0.7785055286507491, - "nauc_cv_recall_at_100_max": -0.7106703205453045, - "nauc_cv_recall_at_100_std": -0.6128728702791083, - "nauc_cv_recall_at_10_diff1": 0.7465191137528097, - "nauc_cv_recall_at_10_max": -0.45743970495195085, - "nauc_cv_recall_at_10_std": -0.44231729089667093, - "nauc_cv_recall_at_1_diff1": 0.809005297864194, - "nauc_cv_recall_at_1_max": -0.2993329488342445, - "nauc_cv_recall_at_1_std": -0.32620741505098666, - "nauc_cv_recall_at_20_diff1": 0.7469876391343502, - "nauc_cv_recall_at_20_max": -0.5177087431978706, - "nauc_cv_recall_at_20_std": -0.4798695500368616, - "nauc_cv_recall_at_3_diff1": 0.7660283014554723, - "nauc_cv_recall_at_3_max": -0.37414600202429155, - "nauc_cv_recall_at_3_std": -0.3812954637897764, - "nauc_cv_recall_at_5_diff1": 0.7506021664679934, - "nauc_cv_recall_at_5_max": -0.40594245769432563, - "nauc_cv_recall_at_5_std": -0.40001908508400597, - "nauc_map_at_1000_diff1": 0.787800072995429, - "nauc_map_at_1000_max": -0.3566392040651437, - "nauc_map_at_1000_std": -0.36714262152547383, - "nauc_map_at_100_diff1": 0.7874967433485706, - "nauc_map_at_100_max": -0.355689418488149, - "nauc_map_at_100_std": -0.3663899874287131, - "nauc_map_at_10_diff1": 0.7868312493681175, - "nauc_map_at_10_max": -0.34775384038463175, - "nauc_map_at_10_std": -0.36125695985470363, - "nauc_map_at_1_diff1": 0.8126115518742145, - "nauc_map_at_1_max": -0.29913151762869594, - "nauc_map_at_1_std": -0.32523258687122936, - "nauc_map_at_20_diff1": 0.7869431680525095, - "nauc_map_at_20_max": -0.35188890045617494, - "nauc_map_at_20_std": -0.36378010134017036, - "nauc_map_at_3_diff1": 0.7911567236759326, - "nauc_map_at_3_max": -0.333149500020068, - "nauc_map_at_3_std": -0.35085539025987095, - "nauc_map_at_5_diff1": 0.787330722378602, - "nauc_map_at_5_max": -0.34076571407782574, - "nauc_map_at_5_std": -0.35536942851206027, - "nauc_mrr_at_1000_diff1": 0.7860249244782958, - "nauc_mrr_at_1000_max": -0.35671253205293696, - "nauc_mrr_at_1000_std": -0.36767791330321364, - "nauc_mrr_at_100_diff1": 0.7857232346858175, - "nauc_mrr_at_100_max": -0.35576342624777996, - "nauc_mrr_at_100_std": -0.3669254208694941, - "nauc_mrr_at_10_diff1": 0.7850630778152726, - "nauc_mrr_at_10_max": -0.347832581099885, - "nauc_mrr_at_10_std": -0.3617931109970315, - "nauc_mrr_at_1_diff1": 0.809005297864194, - "nauc_mrr_at_1_max": -0.2993329488342445, - "nauc_mrr_at_1_std": -0.32620741505098666, - "nauc_mrr_at_20_diff1": 0.7851670975927381, - "nauc_mrr_at_20_max": -0.3519692193968048, - "nauc_mrr_at_20_std": -0.36432365017895374, - "nauc_mrr_at_3_diff1": 0.7893467497060561, - "nauc_mrr_at_3_max": -0.3332399381221576, - "nauc_mrr_at_3_std": -0.35142740018662944, - "nauc_mrr_at_5_diff1": 0.7855592591112479, - "nauc_mrr_at_5_max": -0.34085078593741425, - "nauc_mrr_at_5_std": -0.3559324773796398, - "nauc_ndcg_at_1000_diff1": 0.7921287476026665, - "nauc_ndcg_at_1000_max": -0.4495754958173373, - "nauc_ndcg_at_1000_std": -0.4379750085804611, - "nauc_ndcg_at_100_diff1": 0.7823270946335801, - "nauc_ndcg_at_100_max": -0.41962827268592534, - "nauc_ndcg_at_100_std": -0.4109985043880115, - "nauc_ndcg_at_10_diff1": 0.7766582486969618, - "nauc_ndcg_at_10_max": -0.37418523977947854, - "nauc_ndcg_at_10_std": -0.38075972176781303, - "nauc_ndcg_at_1_diff1": 0.8126115518742145, - "nauc_ndcg_at_1_max": -0.29913151762869594, - "nauc_ndcg_at_1_std": -0.32523258687122936, - "nauc_ndcg_at_20_diff1": 0.776952347684857, - "nauc_ndcg_at_20_max": -0.38921670424824983, - "nauc_ndcg_at_20_std": -0.3900253543363332, - "nauc_ndcg_at_3_diff1": 0.7846298919901769, - "nauc_ndcg_at_3_max": -0.3438430556258442, - "nauc_ndcg_at_3_std": -0.35876507125063356, - "nauc_ndcg_at_5_diff1": 0.7779001958221061, - "nauc_ndcg_at_5_max": -0.35740366313832533, - "nauc_ndcg_at_5_std": -0.36678964402271136, - "nauc_precision_at_1000_diff1": 0.8843727921961839, - "nauc_precision_at_1000_max": -1.1122059360557912, - "nauc_precision_at_1000_std": -0.967587224777609, - "nauc_precision_at_100_diff1": 0.778505528650749, - "nauc_precision_at_100_max": -0.7106703205453052, - "nauc_precision_at_100_std": -0.6128728702791086, - "nauc_precision_at_10_diff1": 0.7465191137528097, - "nauc_precision_at_10_max": -0.4574397049519504, - "nauc_precision_at_10_std": -0.44231729089667066, - "nauc_precision_at_1_diff1": 0.8126115518742145, - "nauc_precision_at_1_max": -0.29913151762869594, - "nauc_precision_at_1_std": -0.32523258687122936, - "nauc_precision_at_20_diff1": 0.7471224397052061, - "nauc_precision_at_20_max": -0.5176185970858633, - "nauc_precision_at_20_std": -0.47970254372347515, - "nauc_precision_at_3_diff1": 0.7662084419673741, - "nauc_precision_at_3_max": -0.3741336169388402, - "nauc_precision_at_3_std": -0.3811257749890647, - "nauc_precision_at_5_diff1": 0.7506575148707284, - "nauc_precision_at_5_max": -0.40590652706796215, - "nauc_precision_at_5_std": -0.39985853688778367, - "nauc_recall_at_1000_diff1": 0.8843727921961846, - "nauc_recall_at_1000_max": -1.1122059360557914, - "nauc_recall_at_1000_std": -0.9675872247776091, - "nauc_recall_at_100_diff1": 0.7785055286507491, - "nauc_recall_at_100_max": -0.7106703205453045, - "nauc_recall_at_100_std": -0.6128728702791083, - "nauc_recall_at_10_diff1": 0.7465191137528097, - "nauc_recall_at_10_max": -0.45743970495195085, - "nauc_recall_at_10_std": -0.44231729089667093, - "nauc_recall_at_1_diff1": 0.8126115518742145, - "nauc_recall_at_1_max": -0.29913151762869594, - "nauc_recall_at_1_std": -0.32523258687122936, - "nauc_recall_at_20_diff1": 0.7471224397052065, - "nauc_recall_at_20_max": -0.5176185970858634, - "nauc_recall_at_20_std": -0.47970254372347554, - "nauc_recall_at_3_diff1": 0.7662084419673746, - "nauc_recall_at_3_max": -0.37413361693884, - "nauc_recall_at_3_std": -0.38112577498906464, - "nauc_recall_at_5_diff1": 0.750657514870729, - "nauc_recall_at_5_max": -0.4059065270679627, - "nauc_recall_at_5_std": -0.39985853688778417, - "ndcg_at_1": 0.29872, - "ndcg_at_10": 0.36632, - "ndcg_at_100": 0.39432, - "ndcg_at_1000": 0.41057, - "ndcg_at_20": 0.37613, - "ndcg_at_3": 0.34107, - "ndcg_at_5": 0.35351, - "precision_at_1": 0.29872, - "precision_at_10": 0.04398, - "precision_at_100": 0.00578, - "precision_at_1000": 0.00071, - "precision_at_20": 0.02393, - "precision_at_3": 0.12345, - "precision_at_5": 0.08009, - "recall_at_1": 0.29872, - "recall_at_10": 0.43985, - "recall_at_100": 0.57849, - "recall_at_1000": 0.70921, - "recall_at_20": 0.47857, - "recall_at_3": 0.37034, - "recall_at_5": 0.40045 - } - ] - }, - "task_name": "FORBI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kI2TRetrieval.json deleted file mode 100644 index ab412718ee..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kI2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "96a313715ecf67f5dfe70c4fa52406bc7bdfbeee", - "evaluation_time": 34.811583518981934, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.02127, - "map_at_1": 0.00614, - "map_at_10": 0.01488, - "map_at_100": 0.01919, - "map_at_1000": 0.02032, - "map_at_20": 0.01661, - "map_at_3": 0.01087, - "map_at_5": 0.01255, - "mrr_at_1": 0.006136224176723257, - "mrr_at_10": 0.014879856626635128, - "mrr_at_100": 0.019185020550758353, - "mrr_at_1000": 0.020315066221173354, - "mrr_at_20": 0.01661308681252838, - "mrr_at_3": 0.010874752846526219, - "mrr_at_5": 0.012551987454830582, - "nauc_map_at_1000_diff1": 0.07142806850553053, - "nauc_map_at_1000_max": 0.17906242598039115, - "nauc_map_at_1000_std": 0.3168950516726266, - "nauc_map_at_100_diff1": 0.07266830335912308, - "nauc_map_at_100_max": 0.17944952380795798, - "nauc_map_at_100_std": 0.316030703623398, - "nauc_map_at_10_diff1": 0.09164462564312548, - "nauc_map_at_10_max": 0.20279243640349265, - "nauc_map_at_10_std": 0.33321717908166465, - "nauc_map_at_1_diff1": 0.11815857703103975, - "nauc_map_at_1_max": 0.26408283394287474, - "nauc_map_at_1_std": 0.4157890704329977, - "nauc_map_at_20_diff1": 0.08132288730033431, - "nauc_map_at_20_max": 0.1912704962920035, - "nauc_map_at_20_std": 0.3217909720943865, - "nauc_map_at_3_diff1": 0.09133842779733792, - "nauc_map_at_3_max": 0.2256728647948325, - "nauc_map_at_3_std": 0.3398655747948414, - "nauc_map_at_5_diff1": 0.0823469463229123, - "nauc_map_at_5_max": 0.22508575065063585, - "nauc_map_at_5_std": 0.34776718853366784, - "nauc_mrr_at_1000_diff1": 0.07142806021147871, - "nauc_mrr_at_1000_max": 0.17906243579920242, - "nauc_mrr_at_1000_std": 0.316895066554052, - "nauc_mrr_at_100_diff1": 0.07266830335912308, - "nauc_mrr_at_100_max": 0.17944952380795798, - "nauc_mrr_at_100_std": 0.316030703623398, - "nauc_mrr_at_10_diff1": 0.09164462564312548, - "nauc_mrr_at_10_max": 0.20279243640349265, - "nauc_mrr_at_10_std": 0.33321717908166465, - "nauc_mrr_at_1_diff1": 0.11815857703103975, - "nauc_mrr_at_1_max": 0.26408283394287474, - "nauc_mrr_at_1_std": 0.4157890704329977, - "nauc_mrr_at_20_diff1": 0.08132288730033431, - "nauc_mrr_at_20_max": 0.1912704962920035, - "nauc_mrr_at_20_std": 0.3217909720943865, - "nauc_mrr_at_3_diff1": 0.09133842779733792, - "nauc_mrr_at_3_max": 0.2256728647948325, - "nauc_mrr_at_3_std": 0.3398655747948414, - "nauc_mrr_at_5_diff1": 0.0823469463229123, - "nauc_mrr_at_5_max": 0.22508575065063585, - "nauc_mrr_at_5_std": 0.34776718853366784, - "nauc_ndcg_at_1000_diff1": 0.041457607781455685, - "nauc_ndcg_at_1000_max": 0.14174752587899012, - "nauc_ndcg_at_1000_std": 0.30584900382090274, - "nauc_ndcg_at_100_diff1": 0.04809582600060403, - "nauc_ndcg_at_100_max": 0.14021333277597503, - "nauc_ndcg_at_100_std": 0.29106027529194606, - "nauc_ndcg_at_10_diff1": 0.09329799574658992, - "nauc_ndcg_at_10_max": 0.18548222751408638, - "nauc_ndcg_at_10_std": 0.3196240966651513, - "nauc_ndcg_at_1_diff1": 0.11815857703103975, - "nauc_ndcg_at_1_max": 0.26408283394287474, - "nauc_ndcg_at_1_std": 0.4157890704329977, - "nauc_ndcg_at_20_diff1": 0.07002390545831393, - "nauc_ndcg_at_20_max": 0.16326790533898922, - "nauc_ndcg_at_20_std": 0.296988919048661, - "nauc_ndcg_at_3_diff1": 0.08864420643603425, - "nauc_ndcg_at_3_max": 0.2246812088071025, - "nauc_ndcg_at_3_std": 0.3324288293295176, - "nauc_ndcg_at_5_diff1": 0.07762634832410889, - "nauc_ndcg_at_5_max": 0.22473611455942216, - "nauc_ndcg_at_5_std": 0.34600164677283235, - "nauc_precision_at_1000_diff1": 0.02402277161780163, - "nauc_precision_at_1000_max": 0.12476362384000816, - "nauc_precision_at_1000_std": 0.3107702224242301, - "nauc_precision_at_100_diff1": 0.032135602055711115, - "nauc_precision_at_100_max": 0.11791356699033088, - "nauc_precision_at_100_std": 0.27889601162358074, - "nauc_precision_at_10_diff1": 0.09648404787315049, - "nauc_precision_at_10_max": 0.1628069283520906, - "nauc_precision_at_10_std": 0.3024701967127499, - "nauc_precision_at_1_diff1": 0.11815857703103975, - "nauc_precision_at_1_max": 0.26408283394287474, - "nauc_precision_at_1_std": 0.4157890704329977, - "nauc_precision_at_20_diff1": 0.05755102665376828, - "nauc_precision_at_20_max": 0.134828413427724, - "nauc_precision_at_20_std": 0.2718561150616516, - "nauc_precision_at_3_diff1": 0.08406873162770592, - "nauc_precision_at_3_max": 0.2240122829921389, - "nauc_precision_at_3_std": 0.31986667568664484, - "nauc_precision_at_5_diff1": 0.07086103530059168, - "nauc_precision_at_5_max": 0.2253351962128875, - "nauc_precision_at_5_std": 0.3453532111047287, - "nauc_recall_at_1000_diff1": 0.02402277161780195, - "nauc_recall_at_1000_max": 0.12476362384000846, - "nauc_recall_at_1000_std": 0.31077022242423036, - "nauc_recall_at_100_diff1": 0.032135602055711136, - "nauc_recall_at_100_max": 0.11791356699033084, - "nauc_recall_at_100_std": 0.2788960116235806, - "nauc_recall_at_10_diff1": 0.09648404787315036, - "nauc_recall_at_10_max": 0.16280692835209049, - "nauc_recall_at_10_std": 0.30247019671274983, - "nauc_recall_at_1_diff1": 0.11815857703103975, - "nauc_recall_at_1_max": 0.26408283394287474, - "nauc_recall_at_1_std": 0.4157890704329977, - "nauc_recall_at_20_diff1": 0.05755102665376812, - "nauc_recall_at_20_max": 0.13482841342772403, - "nauc_recall_at_20_std": 0.27185611506165164, - "nauc_recall_at_3_diff1": 0.0840687316277059, - "nauc_recall_at_3_max": 0.22401228299213888, - "nauc_recall_at_3_std": 0.3198666756866447, - "nauc_recall_at_5_diff1": 0.07086103530059157, - "nauc_recall_at_5_max": 0.22533519621288747, - "nauc_recall_at_5_std": 0.34535321110472866, - "ndcg_at_1": 0.00614, - "ndcg_at_10": 0.02127, - "ndcg_at_100": 0.04791, - "ndcg_at_1000": 0.0874, - "ndcg_at_20": 0.02764, - "ndcg_at_3": 0.01249, - "ndcg_at_5": 0.01556, - "precision_at_1": 0.00614, - "precision_at_10": 0.00425, - "precision_at_100": 0.00181, - "precision_at_1000": 0.00051, - "precision_at_20": 0.0034, - "precision_at_3": 0.00573, - "precision_at_5": 0.00495, - "recall_at_1": 0.00614, - "recall_at_10": 0.04254, - "recall_at_100": 0.18061, - "recall_at_1000": 0.51237, - "recall_at_20": 0.06791, - "recall_at_3": 0.01718, - "recall_at_5": 0.02475 - } - ] - }, - "task_name": "Fashion200kI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kT2IRetrieval.json deleted file mode 100644 index bd95aed3da..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Fashion200kT2IRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "1b86e2dde50e671d5c83d07a79e8b1d8c696964b", - "evaluation_time": 787.4428789615631, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.01519, - "map_at_1": 0.00463, - "map_at_10": 0.0098, - "map_at_100": 0.01313, - "map_at_1000": 0.0145, - "map_at_20": 0.01107, - "map_at_3": 0.00718, - "map_at_5": 0.00837, - "mrr_at_1": 0.009307737056428156, - "mrr_at_10": 0.017149136171823776, - "mrr_at_100": 0.021056239330545873, - "mrr_at_1000": 0.022229945965150693, - "mrr_at_20": 0.01876052565704011, - "mrr_at_3": 0.012798138452588712, - "mrr_at_5": 0.015154159394997088, - "nauc_map_at_1000_diff1": 0.15317823848588835, - "nauc_map_at_1000_max": 0.3707778175232726, - "nauc_map_at_1000_std": 0.5513529953370644, - "nauc_map_at_100_diff1": 0.15847808580536737, - "nauc_map_at_100_max": 0.37859910108283945, - "nauc_map_at_100_std": 0.5591702162310715, - "nauc_map_at_10_diff1": 0.21041130697622099, - "nauc_map_at_10_max": 0.42580269159038553, - "nauc_map_at_10_std": 0.6098544986773251, - "nauc_map_at_1_diff1": 0.24460437684065364, - "nauc_map_at_1_max": 0.5398908169554762, - "nauc_map_at_1_std": 0.7089495012532402, - "nauc_map_at_20_diff1": 0.18101652805238996, - "nauc_map_at_20_max": 0.41197121831300915, - "nauc_map_at_20_std": 0.5931332163584082, - "nauc_map_at_3_diff1": 0.29239402842906054, - "nauc_map_at_3_max": 0.48418913696284516, - "nauc_map_at_3_std": 0.668531890584055, - "nauc_map_at_5_diff1": 0.2567059698570642, - "nauc_map_at_5_max": 0.45341283736125376, - "nauc_map_at_5_std": 0.6253794092756182, - "nauc_mrr_at_1000_diff1": 0.1307719512664129, - "nauc_mrr_at_1000_max": 0.326784704037806, - "nauc_mrr_at_1000_std": 0.4531970552488093, - "nauc_mrr_at_100_diff1": 0.13123410546023803, - "nauc_mrr_at_100_max": 0.3290602289954133, - "nauc_mrr_at_100_std": 0.4537819119706659, - "nauc_mrr_at_10_diff1": 0.1593237928308708, - "nauc_mrr_at_10_max": 0.3583121338641627, - "nauc_mrr_at_10_std": 0.4928534978178621, - "nauc_mrr_at_1_diff1": 0.20730859868154203, - "nauc_mrr_at_1_max": 0.468788203939954, - "nauc_mrr_at_1_std": 0.633374523001285, - "nauc_mrr_at_20_diff1": 0.14047651995233532, - "nauc_mrr_at_20_max": 0.3435170904987213, - "nauc_mrr_at_20_std": 0.4690886379389067, - "nauc_mrr_at_3_diff1": 0.22240178945439196, - "nauc_mrr_at_3_max": 0.4191234439740691, - "nauc_mrr_at_3_std": 0.5456838549504663, - "nauc_mrr_at_5_diff1": 0.19036504815271832, - "nauc_mrr_at_5_max": 0.38841387275222106, - "nauc_mrr_at_5_std": 0.5025611673628193, - "nauc_ndcg_at_1000_diff1": 0.06271178728730444, - "nauc_ndcg_at_1000_max": 0.25980377843991076, - "nauc_ndcg_at_1000_std": 0.42238971980153833, - "nauc_ndcg_at_100_diff1": 0.06892927634047888, - "nauc_ndcg_at_100_max": 0.26550937963515786, - "nauc_ndcg_at_100_std": 0.4232852028631629, - "nauc_ndcg_at_10_diff1": 0.1525986987396939, - "nauc_ndcg_at_10_max": 0.35518195150540305, - "nauc_ndcg_at_10_std": 0.5325073933143113, - "nauc_ndcg_at_1_diff1": 0.20730859868154203, - "nauc_ndcg_at_1_max": 0.468788203939954, - "nauc_ndcg_at_1_std": 0.633374523001285, - "nauc_ndcg_at_20_diff1": 0.11002233895191663, - "nauc_ndcg_at_20_max": 0.33186733998731466, - "nauc_ndcg_at_20_std": 0.49977699059530195, - "nauc_ndcg_at_3_diff1": 0.2510876023881896, - "nauc_ndcg_at_3_max": 0.4428573475936426, - "nauc_ndcg_at_3_std": 0.6108361525357452, - "nauc_ndcg_at_5_diff1": 0.21887224034279398, - "nauc_ndcg_at_5_max": 0.40876119909845304, - "nauc_ndcg_at_5_std": 0.5573993419624723, - "nauc_precision_at_1000_diff1": 0.031115583194632115, - "nauc_precision_at_1000_max": 0.1809611010103693, - "nauc_precision_at_1000_std": 0.2832925196998456, - "nauc_precision_at_100_diff1": 0.019172601222170217, - "nauc_precision_at_100_max": 0.20235981496845598, - "nauc_precision_at_100_std": 0.29677588925162657, - "nauc_precision_at_10_diff1": 0.06807082214371594, - "nauc_precision_at_10_max": 0.267528890941748, - "nauc_precision_at_10_std": 0.4088806442998267, - "nauc_precision_at_1_diff1": 0.20730859868154203, - "nauc_precision_at_1_max": 0.468788203939954, - "nauc_precision_at_1_std": 0.633374523001285, - "nauc_precision_at_20_diff1": 0.026159835125366106, - "nauc_precision_at_20_max": 0.2561727193975215, - "nauc_precision_at_20_std": 0.3703395011665188, - "nauc_precision_at_3_diff1": 0.23535261382907086, - "nauc_precision_at_3_max": 0.36679733902634626, - "nauc_precision_at_3_std": 0.5191545837207172, - "nauc_precision_at_5_diff1": 0.15226141418229086, - "nauc_precision_at_5_max": 0.3120957551523761, - "nauc_precision_at_5_std": 0.4277907224452402, - "nauc_recall_at_1000_diff1": 0.028915172975529115, - "nauc_recall_at_1000_max": 0.22768242611982248, - "nauc_recall_at_1000_std": 0.38989994849780985, - "nauc_recall_at_100_diff1": 0.020589093891737117, - "nauc_recall_at_100_max": 0.20835600133847137, - "nauc_recall_at_100_std": 0.37729386277822163, - "nauc_recall_at_10_diff1": 0.1037459686807867, - "nauc_recall_at_10_max": 0.29877440064977856, - "nauc_recall_at_10_std": 0.5155308636468926, - "nauc_recall_at_1_diff1": 0.24460437684065364, - "nauc_recall_at_1_max": 0.5398908169554762, - "nauc_recall_at_1_std": 0.7089495012532402, - "nauc_recall_at_20_diff1": 0.054939369566234304, - "nauc_recall_at_20_max": 0.28198381141530837, - "nauc_recall_at_20_std": 0.4785510498944293, - "nauc_recall_at_3_diff1": 0.2912211204998551, - "nauc_recall_at_3_max": 0.4274023242423866, - "nauc_recall_at_3_std": 0.6124458691991229, - "nauc_recall_at_5_diff1": 0.21817377658775405, - "nauc_recall_at_5_max": 0.3836703721551013, - "nauc_recall_at_5_std": 0.537587926520719, - "ndcg_at_1": 0.00931, - "ndcg_at_10": 0.01519, - "ndcg_at_100": 0.03573, - "ndcg_at_1000": 0.07765, - "ndcg_at_20": 0.0196, - "ndcg_at_3": 0.00981, - "ndcg_at_5": 0.01174, - "precision_at_1": 0.00931, - "precision_at_10": 0.00547, - "precision_at_100": 0.00267, - "precision_at_1000": 0.00096, - "precision_at_20": 0.00439, - "precision_at_3": 0.00679, - "precision_at_5": 0.00675, - "recall_at_1": 0.00463, - "recall_at_10": 0.02274, - "recall_at_100": 0.10759, - "recall_at_1000": 0.3845, - "recall_at_20": 0.03638, - "recall_at_3": 0.00971, - "recall_at_5": 0.01394 - } - ] - }, - "task_name": "Fashion200kT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FashionIQIT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FashionIQIT2IRetrieval.json deleted file mode 100644 index a22e7f02e6..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FashionIQIT2IRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "e6f0ec70becc413d940cd62b2cfa3b1d3a08c31a", - "evaluation_time": 238.13813376426697, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.02256, - "map_at_1": 0.00083, - "map_at_10": 0.01391, - "map_at_100": 0.01746, - "map_at_1000": 0.01834, - "map_at_20": 0.01542, - "map_at_3": 0.00858, - "map_at_5": 0.01143, - "mrr_at_1": 0.0011660836248542396, - "mrr_at_10": 0.014161966635729764, - "mrr_at_100": 0.017712377694095203, - "mrr_at_1000": 0.018584663737948794, - "mrr_at_20": 0.01566681657909266, - "mrr_at_3": 0.008801154978066527, - "mrr_at_5": 0.01172469320895109, - "nauc_map_at_1000_diff1": -0.0598912640286578, - "nauc_map_at_1000_max": 0.0859123853979979, - "nauc_map_at_1000_std": 0.22907437716306384, - "nauc_map_at_100_diff1": -0.05872727439663456, - "nauc_map_at_100_max": 0.08223964815666353, - "nauc_map_at_100_std": 0.23237931413225082, - "nauc_map_at_10_diff1": -0.060422008312022546, - "nauc_map_at_10_max": 0.08379801841169691, - "nauc_map_at_10_std": 0.23414021829196138, - "nauc_map_at_1_diff1": -0.5746793234443401, - "nauc_map_at_1_max": -0.11897126039760639, - "nauc_map_at_1_std": 0.34186303568634, - "nauc_map_at_20_diff1": -0.057420791623517736, - "nauc_map_at_20_max": 0.07986494217564785, - "nauc_map_at_20_std": 0.23768374054528263, - "nauc_map_at_3_diff1": -0.05308375080151235, - "nauc_map_at_3_max": 0.06643197364615645, - "nauc_map_at_3_std": 0.2153150428142786, - "nauc_map_at_5_diff1": -0.04557552118534602, - "nauc_map_at_5_max": 0.08569842305795435, - "nauc_map_at_5_std": 0.22520034277664164, - "nauc_mrr_at_1000_diff1": -0.06692849899536789, - "nauc_mrr_at_1000_max": 0.07959932284987632, - "nauc_mrr_at_1000_std": 0.23483241832540724, - "nauc_mrr_at_100_diff1": -0.06593632580506924, - "nauc_mrr_at_100_max": 0.07585456666993269, - "nauc_mrr_at_100_std": 0.23821922385835617, - "nauc_mrr_at_10_diff1": -0.06928288793645844, - "nauc_mrr_at_10_max": 0.07594893566631603, - "nauc_mrr_at_10_std": 0.2411048130518643, - "nauc_mrr_at_1_diff1": -0.5746793234443403, - "nauc_mrr_at_1_max": -0.24917356412524466, - "nauc_mrr_at_1_std": 0.4815856420359519, - "nauc_mrr_at_20_diff1": -0.06542436073391363, - "nauc_mrr_at_20_max": 0.07256823322137557, - "nauc_mrr_at_20_std": 0.24438604121380278, - "nauc_mrr_at_3_diff1": -0.06574346236975877, - "nauc_mrr_at_3_max": 0.05465660858795071, - "nauc_mrr_at_3_std": 0.22718655887292788, - "nauc_mrr_at_5_diff1": -0.05700786192665374, - "nauc_mrr_at_5_max": 0.07490019732032167, - "nauc_mrr_at_5_std": 0.2363561699002534, - "nauc_ndcg_at_1000_diff1": -0.07572396773256204, - "nauc_ndcg_at_1000_max": 0.12148459423561712, - "nauc_ndcg_at_1000_std": 0.1883299605315341, - "nauc_ndcg_at_100_diff1": -0.058616309315249894, - "nauc_ndcg_at_100_max": 0.08313219905874299, - "nauc_ndcg_at_100_std": 0.22773559257768028, - "nauc_ndcg_at_10_diff1": -0.0612109564493037, - "nauc_ndcg_at_10_max": 0.08768057389756329, - "nauc_ndcg_at_10_std": 0.2352208258944654, - "nauc_ndcg_at_1_diff1": -0.5746793234443401, - "nauc_ndcg_at_1_max": -0.11897126039760639, - "nauc_ndcg_at_1_std": 0.34186303568634, - "nauc_ndcg_at_20_diff1": -0.0520605133348174, - "nauc_ndcg_at_20_max": 0.07900368270245652, - "nauc_ndcg_at_20_std": 0.24274695093730195, - "nauc_ndcg_at_3_diff1": -0.045307157277366054, - "nauc_ndcg_at_3_max": 0.0737406770633018, - "nauc_ndcg_at_3_std": 0.2103745920700806, - "nauc_ndcg_at_5_diff1": -0.03635638958328183, - "nauc_ndcg_at_5_max": 0.09290557427909102, - "nauc_ndcg_at_5_std": 0.22452211139249575, - "nauc_precision_at_1000_diff1": -0.09528976019730388, - "nauc_precision_at_1000_max": 0.15453704642183053, - "nauc_precision_at_1000_std": 0.1476052191408992, - "nauc_precision_at_100_diff1": -0.06103098984068198, - "nauc_precision_at_100_max": 0.08062010746375872, - "nauc_precision_at_100_std": 0.2240150722195767, - "nauc_precision_at_10_diff1": -0.06356559296982019, - "nauc_precision_at_10_max": 0.08804802619647352, - "nauc_precision_at_10_std": 0.23838131471222793, - "nauc_precision_at_1_diff1": -0.5746793234443401, - "nauc_precision_at_1_max": -0.11897126039760639, - "nauc_precision_at_1_std": 0.34186303568634, - "nauc_precision_at_20_diff1": -0.04729671581693161, - "nauc_precision_at_20_max": 0.07471004104058977, - "nauc_precision_at_20_std": 0.25043638477905833, - "nauc_precision_at_3_diff1": -0.036268500912781894, - "nauc_precision_at_3_max": 0.08353291703652879, - "nauc_precision_at_3_std": 0.2037605481596069, - "nauc_precision_at_5_diff1": -0.02651744785994212, - "nauc_precision_at_5_max": 0.09965707310149198, - "nauc_precision_at_5_std": 0.2260845282965032, - "nauc_recall_at_1000_diff1": -0.09421844317230951, - "nauc_recall_at_1000_max": 0.15558854134225528, - "nauc_recall_at_1000_std": 0.14658974505101816, - "nauc_recall_at_100_diff1": -0.05982995232397422, - "nauc_recall_at_100_max": 0.0820516416081276, - "nauc_recall_at_100_std": 0.2234736572809348, - "nauc_recall_at_10_diff1": -0.06399862735766773, - "nauc_recall_at_10_max": 0.09020731094094402, - "nauc_recall_at_10_std": 0.23574022327133567, - "nauc_recall_at_1_diff1": -0.5746793234443401, - "nauc_recall_at_1_max": -0.11897126039760639, - "nauc_recall_at_1_std": 0.34186303568634, - "nauc_recall_at_20_diff1": -0.046939197170608, - "nauc_recall_at_20_max": 0.076141622480933, - "nauc_recall_at_20_std": 0.24777983661827613, - "nauc_recall_at_3_diff1": -0.03626850091278174, - "nauc_recall_at_3_max": 0.08353291703652911, - "nauc_recall_at_3_std": 0.20376054815960698, - "nauc_recall_at_5_diff1": -0.026002358838177916, - "nauc_recall_at_5_max": 0.10141617506386659, - "nauc_recall_at_5_std": 0.22293448855893305, - "ndcg_at_1": 0.00083, - "ndcg_at_10": 0.02256, - "ndcg_at_100": 0.04358, - "ndcg_at_1000": 0.07302, - "ndcg_at_20": 0.0281, - "ndcg_at_3": 0.01136, - "ndcg_at_5": 0.01652, - "precision_at_1": 0.00083, - "precision_at_10": 0.0051, - "precision_at_100": 0.00158, - "precision_at_1000": 0.0004, - "precision_at_20": 0.00366, - "precision_at_3": 0.0065, - "precision_at_5": 0.00643, - "recall_at_1": 0.00083, - "recall_at_10": 0.05072, - "recall_at_100": 0.15775, - "recall_at_1000": 0.40346, - "recall_at_20": 0.0728, - "recall_at_3": 0.01949, - "recall_at_5": 0.03198 - } - ] - }, - "task_name": "FashionIQIT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json deleted file mode 100644 index c27e7fd151..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb", - "evaluation_time": 11.527727365493774, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.782, - "cv_recall_at_10": 0.979, - "cv_recall_at_100": 0.998, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.994, - "cv_recall_at_3": 0.919, - "cv_recall_at_5": 0.951, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.71535, - "map_at_1": 0.1564, - "map_at_10": 0.6016, - "map_at_100": 0.64142, - "map_at_1000": 0.64252, - "map_at_20": 0.62777, - "map_at_3": 0.38783, - "map_at_5": 0.52369, - "mrr_at_1": 0.782, - "mrr_at_10": 0.8533876984126986, - "mrr_at_100": 0.8545914967755422, - "mrr_at_1000": 0.8546010863340449, - "mrr_at_20": 0.8544995892995896, - "mrr_at_3": 0.842166666666667, - "mrr_at_5": 0.849416666666667, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.6790382819794335, - "nauc_cv_recall_at_100_max": 0.6136788048552808, - "nauc_cv_recall_at_100_std": 0.5401493930905519, - "nauc_cv_recall_at_10_diff1": 0.5072695744964671, - "nauc_cv_recall_at_10_max": 0.6689342403628125, - "nauc_cv_recall_at_10_std": 0.3829754123871726, - "nauc_cv_recall_at_1_diff1": 0.488735446449004, - "nauc_cv_recall_at_1_max": 0.5141133834291504, - "nauc_cv_recall_at_1_std": -0.03618247455901551, - "nauc_cv_recall_at_20_diff1": 0.34002489884843273, - "nauc_cv_recall_at_20_max": 0.6156240273887368, - "nauc_cv_recall_at_20_std": 0.7568471833177612, - "nauc_cv_recall_at_3_diff1": 0.5087895240400684, - "nauc_cv_recall_at_3_max": 0.5634459545134927, - "nauc_cv_recall_at_3_std": 0.10033890099249525, - "nauc_cv_recall_at_5_diff1": 0.5434650050496416, - "nauc_cv_recall_at_5_max": 0.5627870195697343, - "nauc_cv_recall_at_5_std": 0.2778158882600648, - "nauc_map_at_1000_diff1": 0.15465269435050452, - "nauc_map_at_1000_max": 0.46711312406477384, - "nauc_map_at_1000_std": 0.048575938256388396, - "nauc_map_at_100_diff1": 0.15431140329951903, - "nauc_map_at_100_max": 0.46719979617191026, - "nauc_map_at_100_std": 0.05052363203080754, - "nauc_map_at_10_diff1": 0.1529787812866786, - "nauc_map_at_10_max": 0.45711233128671735, - "nauc_map_at_10_std": 0.02449973700621325, - "nauc_map_at_1_diff1": 0.4887354464490032, - "nauc_map_at_1_max": 0.5141133834291495, - "nauc_map_at_1_std": -0.03618247455901751, - "nauc_map_at_20_diff1": 0.15076154146537132, - "nauc_map_at_20_max": 0.46444349461535234, - "nauc_map_at_20_std": 0.04479449662456778, - "nauc_map_at_3_diff1": 0.21351903584890045, - "nauc_map_at_3_max": 0.45698690208877696, - "nauc_map_at_3_std": -0.010887987588539814, - "nauc_map_at_5_diff1": 0.14578035683987364, - "nauc_map_at_5_max": 0.4515880427226945, - "nauc_map_at_5_std": -0.0026874220062081747, - "nauc_mrr_at_1000_diff1": 0.4893310611891473, - "nauc_mrr_at_1000_max": 0.5261133355559909, - "nauc_mrr_at_1000_std": 0.0020315610859939267, - "nauc_mrr_at_100_diff1": 0.4893377672062905, - "nauc_mrr_at_100_max": 0.5261248596328446, - "nauc_mrr_at_100_std": 0.0020657882998518138, - "nauc_mrr_at_10_diff1": 0.48988261824797846, - "nauc_mrr_at_10_max": 0.5277240085344459, - "nauc_mrr_at_10_std": 0.00453451162925253, - "nauc_mrr_at_1_diff1": 0.488735446449004, - "nauc_mrr_at_1_max": 0.5141133834291504, - "nauc_mrr_at_1_std": -0.03618247455901551, - "nauc_mrr_at_20_diff1": 0.48916248000131224, - "nauc_mrr_at_20_max": 0.5262057614175848, - "nauc_mrr_at_20_std": 0.0026394141831601174, - "nauc_mrr_at_3_diff1": 0.48985167550576125, - "nauc_mrr_at_3_max": 0.5300441674581229, - "nauc_mrr_at_3_std": 0.0027730733995076284, - "nauc_mrr_at_5_diff1": 0.4918762189399869, - "nauc_mrr_at_5_max": 0.52702946776199, - "nauc_mrr_at_5_std": 0.011169701853700587, - "nauc_ndcg_at_1000_diff1": 0.22766089836536801, - "nauc_ndcg_at_1000_max": 0.492189505568212, - "nauc_ndcg_at_1000_std": 0.06504915436760614, - "nauc_ndcg_at_100_diff1": 0.2199335774609376, - "nauc_ndcg_at_100_max": 0.4953948669765448, - "nauc_ndcg_at_100_std": 0.0980744945605265, - "nauc_ndcg_at_10_diff1": 0.2044319581978452, - "nauc_ndcg_at_10_max": 0.4763899911130591, - "nauc_ndcg_at_10_std": 0.04503381614955524, - "nauc_ndcg_at_1_diff1": 0.488735446449004, - "nauc_ndcg_at_1_max": 0.5141133834291504, - "nauc_ndcg_at_1_std": -0.03618247455901551, - "nauc_ndcg_at_20_diff1": 0.200495407478228, - "nauc_ndcg_at_20_max": 0.4889831247980535, - "nauc_ndcg_at_20_std": 0.0867421312607528, - "nauc_ndcg_at_3_diff1": 0.25423026936814114, - "nauc_ndcg_at_3_max": 0.4728682169678516, - "nauc_ndcg_at_3_std": 0.001298277045082591, - "nauc_ndcg_at_5_diff1": 0.19418768049527063, - "nauc_ndcg_at_5_max": 0.46699616414823136, - "nauc_ndcg_at_5_std": 0.01093649505873361, - "nauc_precision_at_1000_diff1": 0.12759103641456715, - "nauc_precision_at_1000_max": 0.4844070961717251, - "nauc_precision_at_1000_std": 0.7185340802988052, - "nauc_precision_at_100_diff1": 0.08522333406731375, - "nauc_precision_at_100_max": 0.48179115797715183, - "nauc_precision_at_100_std": 0.5722441057258686, - "nauc_precision_at_10_diff1": 0.13469873984556588, - "nauc_precision_at_10_max": 0.4428990975706562, - "nauc_precision_at_10_std": 0.08769758273258854, - "nauc_precision_at_1_diff1": 0.488735446449004, - "nauc_precision_at_1_max": 0.5141133834291504, - "nauc_precision_at_1_std": -0.03618247455901551, - "nauc_precision_at_20_diff1": 0.09427425323646658, - "nauc_precision_at_20_max": 0.4578914983843951, - "nauc_precision_at_20_std": 0.2235341685316957, - "nauc_precision_at_3_diff1": 0.2010575428533153, - "nauc_precision_at_3_max": 0.46236331299632105, - "nauc_precision_at_3_std": 0.013547375486496331, - "nauc_precision_at_5_diff1": 0.1317349745546741, - "nauc_precision_at_5_max": 0.4500305482173576, - "nauc_precision_at_5_std": 0.022958692069647763, - "nauc_recall_at_1000_diff1": 0.12759103641454075, - "nauc_recall_at_1000_max": 0.48440709617180283, - "nauc_recall_at_1000_std": 0.7185340802988699, - "nauc_recall_at_100_diff1": 0.0852233340673131, - "nauc_recall_at_100_max": 0.48179115797715144, - "nauc_recall_at_100_std": 0.5722441057258676, - "nauc_recall_at_10_diff1": 0.13469873984556588, - "nauc_recall_at_10_max": 0.4428990975706562, - "nauc_recall_at_10_std": 0.08769758273258854, - "nauc_recall_at_1_diff1": 0.4887354464490032, - "nauc_recall_at_1_max": 0.5141133834291495, - "nauc_recall_at_1_std": -0.03618247455901751, - "nauc_recall_at_20_diff1": 0.09427425323646658, - "nauc_recall_at_20_max": 0.4578914983843951, - "nauc_recall_at_20_std": 0.2235341685316957, - "nauc_recall_at_3_diff1": 0.20105754285331554, - "nauc_recall_at_3_max": 0.4623633129963219, - "nauc_recall_at_3_std": 0.013547375486496333, - "nauc_recall_at_5_diff1": 0.1317349745546741, - "nauc_recall_at_5_max": 0.4500305482173576, - "nauc_recall_at_5_std": 0.022958692069647763, - "ndcg_at_1": 0.782, - "ndcg_at_10": 0.71535, - "ndcg_at_100": 0.79531, - "ndcg_at_1000": 0.80538, - "ndcg_at_20": 0.75694, - "ndcg_at_3": 0.70872, - "ndcg_at_5": 0.63009, - "precision_at_1": 0.782, - "precision_at_10": 0.3677, - "precision_at_100": 0.04762, - "precision_at_1000": 0.00499, - "precision_at_20": 0.2081, - "precision_at_3": 0.68767, - "precision_at_5": 0.5814, - "recall_at_1": 0.1564, - "recall_at_10": 0.7354, - "recall_at_100": 0.9524, - "recall_at_1000": 0.998, - "recall_at_20": 0.8324, - "recall_at_3": 0.4126, - "recall_at_5": 0.5814 - } - ] - }, - "task_name": "Flickr30kI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json deleted file mode 100644 index 8ab9b00d39..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "e819702b287bfbe084e129a61f308a802b7c108e", - "evaluation_time": 19.96243405342102, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.582, - "cv_recall_at_10": 0.892, - "cv_recall_at_100": 0.99, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.94, - "cv_recall_at_3": 0.7656, - "cv_recall_at_5": 0.8288, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.73691, - "map_at_1": 0.582, - "map_at_10": 0.6872, - "map_at_100": 0.69194, - "map_at_1000": 0.692, - "map_at_20": 0.69052, - "map_at_3": 0.664, - "map_at_5": 0.6786, - "mrr_at_1": 0.582, - "mrr_at_10": 0.6872000793650814, - "mrr_at_100": 0.6919418796590151, - "mrr_at_1000": 0.6920043726572235, - "mrr_at_20": 0.6905210107419258, - "mrr_at_3": 0.6640000000000026, - "mrr_at_5": 0.6786000000000014, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5202707749766565, - "nauc_cv_recall_at_100_max": 0.5908309990662858, - "nauc_cv_recall_at_100_std": 0.2939682539682632, - "nauc_cv_recall_at_10_diff1": 0.5617933093550299, - "nauc_cv_recall_at_10_max": 0.5589071830149802, - "nauc_cv_recall_at_10_std": 0.2244342982854361, - "nauc_cv_recall_at_1_diff1": 0.696381939657304, - "nauc_cv_recall_at_1_max": 0.47579049630166126, - "nauc_cv_recall_at_1_std": 0.0856787171723342, - "nauc_cv_recall_at_20_diff1": 0.5454154995331454, - "nauc_cv_recall_at_20_max": 0.575666044195457, - "nauc_cv_recall_at_20_std": 0.27880018674136237, - "nauc_cv_recall_at_3_diff1": 0.6071203712031832, - "nauc_cv_recall_at_3_max": 0.5034522530911553, - "nauc_cv_recall_at_3_std": 0.1294339186732062, - "nauc_cv_recall_at_5_diff1": 0.5786903531517902, - "nauc_cv_recall_at_5_max": 0.5299681868096371, - "nauc_cv_recall_at_5_std": 0.18341665359380663, - "nauc_map_at_1000_diff1": 0.6637636015118767, - "nauc_map_at_1000_max": 0.48813186871534664, - "nauc_map_at_1000_std": 0.10620231745962286, - "nauc_map_at_100_diff1": 0.663725501715657, - "nauc_map_at_100_max": 0.48815298386872974, - "nauc_map_at_100_std": 0.10623045894817788, - "nauc_map_at_10_diff1": 0.6621084063953249, - "nauc_map_at_10_max": 0.48930142276760086, - "nauc_map_at_10_std": 0.10756855111381708, - "nauc_map_at_1_diff1": 0.696381939657304, - "nauc_map_at_1_max": 0.47579049630166126, - "nauc_map_at_1_std": 0.0856787171723342, - "nauc_map_at_20_diff1": 0.6630269532116994, - "nauc_map_at_20_max": 0.48862647994909303, - "nauc_map_at_20_std": 0.10717682940880707, - "nauc_map_at_3_diff1": 0.662292666771372, - "nauc_map_at_3_max": 0.48547341513990666, - "nauc_map_at_3_std": 0.10081169237632849, - "nauc_map_at_5_diff1": 0.6602863231828925, - "nauc_map_at_5_max": 0.4883925967971498, - "nauc_map_at_5_std": 0.10706113722330816, - "nauc_mrr_at_1000_diff1": 0.6637636015118767, - "nauc_mrr_at_1000_max": 0.48813186871534664, - "nauc_mrr_at_1000_std": 0.10620231745962286, - "nauc_mrr_at_100_diff1": 0.663725501715657, - "nauc_mrr_at_100_max": 0.48815298386872974, - "nauc_mrr_at_100_std": 0.10623045894817788, - "nauc_mrr_at_10_diff1": 0.6621084063953249, - "nauc_mrr_at_10_max": 0.48930142276760086, - "nauc_mrr_at_10_std": 0.10756855111381708, - "nauc_mrr_at_1_diff1": 0.696381939657304, - "nauc_mrr_at_1_max": 0.47579049630166126, - "nauc_mrr_at_1_std": 0.0856787171723342, - "nauc_mrr_at_20_diff1": 0.6630269532116994, - "nauc_mrr_at_20_max": 0.48862647994909303, - "nauc_mrr_at_20_std": 0.10717682940880707, - "nauc_mrr_at_3_diff1": 0.662292666771372, - "nauc_mrr_at_3_max": 0.48547341513990666, - "nauc_mrr_at_3_std": 0.10081169237632849, - "nauc_mrr_at_5_diff1": 0.6602863231828925, - "nauc_mrr_at_5_max": 0.4883925967971498, - "nauc_mrr_at_5_std": 0.10706113722330816, - "nauc_ndcg_at_1000_diff1": 0.6588020949056204, - "nauc_ndcg_at_1000_max": 0.4911070818710271, - "nauc_ndcg_at_1000_std": 0.11159626078786719, - "nauc_ndcg_at_100_diff1": 0.6578430110166047, - "nauc_ndcg_at_100_max": 0.49175784358270214, - "nauc_ndcg_at_100_std": 0.11271907006804692, - "nauc_ndcg_at_10_diff1": 0.6491199341823203, - "nauc_ndcg_at_10_max": 0.49780129062969325, - "nauc_ndcg_at_10_std": 0.1217823152584022, - "nauc_ndcg_at_1_diff1": 0.696381939657304, - "nauc_ndcg_at_1_max": 0.47579049630166126, - "nauc_ndcg_at_1_std": 0.0856787171723342, - "nauc_ndcg_at_20_diff1": 0.6527562952947522, - "nauc_ndcg_at_20_max": 0.4954501005931264, - "nauc_ndcg_at_20_std": 0.12020249054744403, - "nauc_ndcg_at_3_diff1": 0.6504859796432384, - "nauc_ndcg_at_3_max": 0.4892786954353797, - "nauc_ndcg_at_3_std": 0.10686134169700189, - "nauc_ndcg_at_5_diff1": 0.6458666500847972, - "nauc_ndcg_at_5_max": 0.4954671444369836, - "nauc_ndcg_at_5_std": 0.11996492669439957, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.5202707749766482, - "nauc_precision_at_100_max": 0.5908309990662891, - "nauc_precision_at_100_std": 0.29396825396825177, - "nauc_precision_at_10_diff1": 0.5617933093550302, - "nauc_precision_at_10_max": 0.5589071830149824, - "nauc_precision_at_10_std": 0.22443429828543696, - "nauc_precision_at_1_diff1": 0.696381939657304, - "nauc_precision_at_1_max": 0.47579049630166126, - "nauc_precision_at_1_std": 0.0856787171723342, - "nauc_precision_at_20_diff1": 0.545415499533147, - "nauc_precision_at_20_max": 0.5756660441954571, - "nauc_precision_at_20_std": 0.27880018674136353, - "nauc_precision_at_3_diff1": 0.6071203712031833, - "nauc_precision_at_3_max": 0.5034522530911553, - "nauc_precision_at_3_std": 0.1294339186732061, - "nauc_precision_at_5_diff1": 0.5786903531517906, - "nauc_precision_at_5_max": 0.5299681868096384, - "nauc_precision_at_5_std": 0.183416653593807, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5202707749766565, - "nauc_recall_at_100_max": 0.5908309990662858, - "nauc_recall_at_100_std": 0.2939682539682632, - "nauc_recall_at_10_diff1": 0.5617933093550299, - "nauc_recall_at_10_max": 0.5589071830149802, - "nauc_recall_at_10_std": 0.2244342982854361, - "nauc_recall_at_1_diff1": 0.696381939657304, - "nauc_recall_at_1_max": 0.47579049630166126, - "nauc_recall_at_1_std": 0.0856787171723342, - "nauc_recall_at_20_diff1": 0.5454154995331454, - "nauc_recall_at_20_max": 0.575666044195457, - "nauc_recall_at_20_std": 0.27880018674136237, - "nauc_recall_at_3_diff1": 0.6071203712031832, - "nauc_recall_at_3_max": 0.5034522530911553, - "nauc_recall_at_3_std": 0.1294339186732062, - "nauc_recall_at_5_diff1": 0.5786903531517902, - "nauc_recall_at_5_max": 0.5299681868096371, - "nauc_recall_at_5_std": 0.18341665359380663, - "ndcg_at_1": 0.582, - "ndcg_at_10": 0.73691, - "ndcg_at_100": 0.75858, - "ndcg_at_1000": 0.75994, - "ndcg_at_20": 0.74904, - "ndcg_at_3": 0.69014, - "ndcg_at_5": 0.71631, - "precision_at_1": 0.582, - "precision_at_10": 0.0892, - "precision_at_100": 0.0099, - "precision_at_1000": 0.001, - "precision_at_20": 0.047, - "precision_at_3": 0.2552, - "precision_at_5": 0.16576, - "recall_at_1": 0.582, - "recall_at_10": 0.892, - "recall_at_100": 0.99, - "recall_at_1000": 1.0, - "recall_at_20": 0.94, - "recall_at_3": 0.7656, - "recall_at_5": 0.8288 - } - ] - }, - "task_name": "Flickr30kT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101Classification.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101Classification.json deleted file mode 100644 index d741fb416a..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101Classification.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", - "evaluation_time": 1405.0914344787598, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "validation": [ - { - "accuracy": 0.7963564356435644, - "f1": 0.7966444203871001, - "f1_weighted": 0.7966444203871001, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7963564356435644, - "scores_per_experiment": [ - { - "accuracy": 0.7967524752475248, - "f1": 0.796877338175403, - "f1_weighted": 0.7968773381754028 - }, - { - "accuracy": 0.802930693069307, - "f1": 0.8032004030944163, - "f1_weighted": 0.8032004030944164 - }, - { - "accuracy": 0.7946930693069307, - "f1": 0.7948653096928756, - "f1_weighted": 0.7948653096928756 - }, - { - "accuracy": 0.7951683168316832, - "f1": 0.7952119577172861, - "f1_weighted": 0.7952119577172861 - }, - { - "accuracy": 0.7922376237623763, - "f1": 0.7930670932555197, - "f1_weighted": 0.7930670932555199 - } - ] - } - ] - }, - "task_name": "Food101Classification" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101ZeroShot.json deleted file mode 100644 index 5d14882a38..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Food101ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", - "evaluation_time": 264.61052894592285, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "validation": [ - { - "accuracy": 0.827089108910891, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.827089108910891 - } - ] - }, - "task_name": "Food101ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2IRetrieval.json deleted file mode 100644 index cd8dcc6a44..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c6b162ee349adb293901128a18c0b446f7b43457", - "evaluation_time": 3894.6362721920013, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.23206, - "cv_recall_at_10": 0.39681, - "cv_recall_at_100": 0.54119, - "cv_recall_at_1000": 0.66873, - "cv_recall_at_20": 0.4349, - "cv_recall_at_3": 0.31709, - "cv_recall_at_5": 0.35164, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.15119, - "map_at_1": 0.02397, - "map_at_10": 0.06479, - "map_at_100": 0.08024, - "map_at_1000": 0.08268, - "map_at_20": 0.07251, - "map_at_3": 0.04501, - "map_at_5": 0.0545, - "mrr_at_1": 0.2320637732506643, - "mrr_at_10": 0.2828837853417125, - "mrr_at_100": 0.28837169422300607, - "mrr_at_1000": 0.2888825334937045, - "mrr_at_20": 0.28556164461936007, - "mrr_at_3": 0.2692648361381754, - "mrr_at_5": 0.2770593445527013, - "nauc_cv_recall_at_1000_diff1": 0.12259919578063702, - "nauc_cv_recall_at_1000_max": 0.2470948622454878, - "nauc_cv_recall_at_1000_std": 0.28799480962162954, - "nauc_cv_recall_at_100_diff1": 0.19420908123263017, - "nauc_cv_recall_at_100_max": 0.29687778489700106, - "nauc_cv_recall_at_100_std": 0.3231354986765902, - "nauc_cv_recall_at_10_diff1": 0.15306314553147926, - "nauc_cv_recall_at_10_max": 0.32002948607067444, - "nauc_cv_recall_at_10_std": 0.33790292791220566, - "nauc_cv_recall_at_1_diff1": 0.27344631062903946, - "nauc_cv_recall_at_1_max": 0.3959578109654247, - "nauc_cv_recall_at_1_std": 0.31118842144952946, - "nauc_cv_recall_at_20_diff1": 0.18529666847332407, - "nauc_cv_recall_at_20_max": 0.3300665371997956, - "nauc_cv_recall_at_20_std": 0.34546242556909446, - "nauc_cv_recall_at_3_diff1": 0.2064027555036601, - "nauc_cv_recall_at_3_max": 0.3741850802826935, - "nauc_cv_recall_at_3_std": 0.31988594991760555, - "nauc_cv_recall_at_5_diff1": 0.1704464042473904, - "nauc_cv_recall_at_5_max": 0.36195054598654247, - "nauc_cv_recall_at_5_std": 0.3283984149202229, - "nauc_map_at_1000_diff1": 0.1557976801642832, - "nauc_map_at_1000_max": 0.40047760186355585, - "nauc_map_at_1000_std": 0.3430030328191707, - "nauc_map_at_100_diff1": 0.15789089377828042, - "nauc_map_at_100_max": 0.39699246549039235, - "nauc_map_at_100_std": 0.3351785250251348, - "nauc_map_at_10_diff1": 0.17418470196289046, - "nauc_map_at_10_max": 0.358989373523996, - "nauc_map_at_10_std": 0.2748470090808481, - "nauc_map_at_1_diff1": 0.2670595538891123, - "nauc_map_at_1_max": 0.22266498241056235, - "nauc_map_at_1_std": 0.16770237726987589, - "nauc_map_at_20_diff1": 0.16473144746616353, - "nauc_map_at_20_max": 0.382369999843966, - "nauc_map_at_20_std": 0.30453408973596896, - "nauc_map_at_3_diff1": 0.2129585099540166, - "nauc_map_at_3_max": 0.2993400684456921, - "nauc_map_at_3_std": 0.218588537928361, - "nauc_map_at_5_diff1": 0.19694693530356244, - "nauc_map_at_5_max": 0.3342893645915572, - "nauc_map_at_5_std": 0.24672942576337248, - "nauc_mrr_at_1000_diff1": 0.23236391056686204, - "nauc_mrr_at_1000_max": 0.3756143254195909, - "nauc_mrr_at_1000_std": 0.31925253283001154, - "nauc_mrr_at_100_diff1": 0.23265291802648855, - "nauc_mrr_at_100_max": 0.3757140545121209, - "nauc_mrr_at_100_std": 0.3192347113370852, - "nauc_mrr_at_10_diff1": 0.22970198669237163, - "nauc_mrr_at_10_max": 0.37553145595069753, - "nauc_mrr_at_10_std": 0.31870821442127495, - "nauc_mrr_at_1_diff1": 0.27344631062903946, - "nauc_mrr_at_1_max": 0.3959578109654247, - "nauc_mrr_at_1_std": 0.31118842144952946, - "nauc_mrr_at_20_diff1": 0.23247830829777727, - "nauc_mrr_at_20_max": 0.37634277791741844, - "nauc_mrr_at_20_std": 0.31953209478801003, - "nauc_mrr_at_3_diff1": 0.24061890025051894, - "nauc_mrr_at_3_max": 0.3855162215701596, - "nauc_mrr_at_3_std": 0.31578819749904946, - "nauc_mrr_at_5_diff1": 0.2319139806137186, - "nauc_mrr_at_5_max": 0.3819090578458763, - "nauc_mrr_at_5_std": 0.3175158789349314, - "nauc_ndcg_at_1000_diff1": 0.15244014180873733, - "nauc_ndcg_at_1000_max": 0.37532662485358, - "nauc_ndcg_at_1000_std": 0.3547988725977299, - "nauc_ndcg_at_100_diff1": 0.15885667952123475, - "nauc_ndcg_at_100_max": 0.3729866051910131, - "nauc_ndcg_at_100_std": 0.33820898925869763, - "nauc_ndcg_at_10_diff1": 0.16388891530934463, - "nauc_ndcg_at_10_max": 0.40598231749041, - "nauc_ndcg_at_10_std": 0.3721247127572905, - "nauc_ndcg_at_1_diff1": 0.27344631062903946, - "nauc_ndcg_at_1_max": 0.3959578109654247, - "nauc_ndcg_at_1_std": 0.31118842144952946, - "nauc_ndcg_at_20_diff1": 0.16172703481778403, - "nauc_ndcg_at_20_max": 0.38288168356456037, - "nauc_ndcg_at_20_std": 0.3474159684723018, - "nauc_ndcg_at_3_diff1": 0.19679763706202097, - "nauc_ndcg_at_3_max": 0.4230963505688227, - "nauc_ndcg_at_3_std": 0.347768230242545, - "nauc_ndcg_at_5_diff1": 0.17995417866426847, - "nauc_ndcg_at_5_max": 0.4227315421417604, - "nauc_ndcg_at_5_std": 0.36635238151784116, - "nauc_precision_at_1000_diff1": 0.08337368153007142, - "nauc_precision_at_1000_max": 0.36980358553307513, - "nauc_precision_at_1000_std": 0.4450461238965513, - "nauc_precision_at_100_diff1": 0.08704233522482205, - "nauc_precision_at_100_max": 0.41699203263093915, - "nauc_precision_at_100_std": 0.4858353623514438, - "nauc_precision_at_10_diff1": 0.11122338641251278, - "nauc_precision_at_10_max": 0.43299919237102896, - "nauc_precision_at_10_std": 0.4184487772738942, - "nauc_precision_at_1_diff1": 0.27344631062903946, - "nauc_precision_at_1_max": 0.3959578109654247, - "nauc_precision_at_1_std": 0.31118842144952946, - "nauc_precision_at_20_diff1": 0.10990731374977268, - "nauc_precision_at_20_max": 0.4433545232195496, - "nauc_precision_at_20_std": 0.4496808942169916, - "nauc_precision_at_3_diff1": 0.17030870918458044, - "nauc_precision_at_3_max": 0.42980656332418254, - "nauc_precision_at_3_std": 0.3571964991226058, - "nauc_precision_at_5_diff1": 0.14154739179456016, - "nauc_precision_at_5_max": 0.4355121325859214, - "nauc_precision_at_5_std": 0.38477132537722974, - "nauc_recall_at_1000_diff1": 0.07754568718176012, - "nauc_recall_at_1000_max": 0.2569226744646939, - "nauc_recall_at_1000_std": 0.286809873936029, - "nauc_recall_at_100_diff1": 0.09581782819772992, - "nauc_recall_at_100_max": 0.2854307707331638, - "nauc_recall_at_100_std": 0.2766737038359882, - "nauc_recall_at_10_diff1": 0.114367230618245, - "nauc_recall_at_10_max": 0.2879608088269033, - "nauc_recall_at_10_std": 0.2502157880276106, - "nauc_recall_at_1_diff1": 0.2670595538891123, - "nauc_recall_at_1_max": 0.22266498241056235, - "nauc_recall_at_1_std": 0.16770237726987589, - "nauc_recall_at_20_diff1": 0.10103125164181147, - "nauc_recall_at_20_max": 0.29720895768918904, - "nauc_recall_at_20_std": 0.2590143736205232, - "nauc_recall_at_3_diff1": 0.18358542215665785, - "nauc_recall_at_3_max": 0.2756115962160671, - "nauc_recall_at_3_std": 0.20050115218597597, - "nauc_recall_at_5_diff1": 0.1504626011770959, - "nauc_recall_at_5_max": 0.2922839977079823, - "nauc_recall_at_5_std": 0.22274453322721552, - "ndcg_at_1": 0.23206, - "ndcg_at_10": 0.15119, - "ndcg_at_100": 0.15473, - "ndcg_at_1000": 0.18542, - "ndcg_at_20": 0.14062, - "ndcg_at_3": 0.19526, - "ndcg_at_5": 0.17429, - "precision_at_1": 0.23206, - "precision_at_10": 0.11001, - "precision_at_100": 0.02504, - "precision_at_1000": 0.00415, - "precision_at_20": 0.07516, - "precision_at_3": 0.18364, - "precision_at_5": 0.15182, - "recall_at_1": 0.02397, - "recall_at_10": 0.08901, - "recall_at_100": 0.17505, - "recall_at_1000": 0.28531, - "recall_at_20": 0.11334, - "recall_at_3": 0.05164, - "recall_at_5": 0.06666 - } - ] - }, - "task_name": "GLDv2I2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2TRetrieval.json deleted file mode 100644 index 5454017683..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GLDv2I2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "d8c3e53160860f76de73ed3041a8593672fe5928", - "evaluation_time": 7.942275285720825, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.45833, - "cv_recall_at_10": 0.75939, - "cv_recall_at_100": 0.94894, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.82923, - "cv_recall_at_3": 0.61737, - "cv_recall_at_5": 0.67547, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.59091, - "map_at_1": 0.41862, - "map_at_10": 0.53693, - "map_at_100": 0.54643, - "map_at_1000": 0.54678, - "map_at_20": 0.54267, - "map_at_3": 0.50797, - "map_at_5": 0.52367, - "mrr_at_1": 0.4583333333333333, - "mrr_at_10": 0.5535262500931519, - "mrr_at_100": 0.5614522827750004, - "mrr_at_1000": 0.5617553026170138, - "mrr_at_20": 0.5583628555240837, - "mrr_at_3": 0.528951486697966, - "mrr_at_5": 0.542243740219093, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4805358684051926, - "nauc_cv_recall_at_100_max": 0.5649594006452747, - "nauc_cv_recall_at_100_std": 0.32360974466378195, - "nauc_cv_recall_at_10_diff1": 0.5013034837210644, - "nauc_cv_recall_at_10_max": 0.4802128509459776, - "nauc_cv_recall_at_10_std": 0.09412450554170432, - "nauc_cv_recall_at_1_diff1": 0.636898011304436, - "nauc_cv_recall_at_1_max": 0.44436198490104667, - "nauc_cv_recall_at_1_std": -0.06652018253703539, - "nauc_cv_recall_at_20_diff1": 0.47564828778050955, - "nauc_cv_recall_at_20_max": 0.5259584288342394, - "nauc_cv_recall_at_20_std": 0.1768940709215821, - "nauc_cv_recall_at_3_diff1": 0.5205869375457675, - "nauc_cv_recall_at_3_max": 0.450051437202706, - "nauc_cv_recall_at_3_std": -0.029414267124329728, - "nauc_cv_recall_at_5_diff1": 0.5001251296481215, - "nauc_cv_recall_at_5_max": 0.4693890552265276, - "nauc_cv_recall_at_5_std": 0.013796336174200773, - "nauc_map_at_1000_diff1": 0.5818434859115754, - "nauc_map_at_1000_max": 0.4430387835643193, - "nauc_map_at_1000_std": -0.04099022306215756, - "nauc_map_at_100_diff1": 0.5817200322199132, - "nauc_map_at_100_max": 0.44314688497711546, - "nauc_map_at_100_std": -0.040679605719069825, - "nauc_map_at_10_diff1": 0.5806678508477698, - "nauc_map_at_10_max": 0.4427501241681824, - "nauc_map_at_10_std": -0.04102488719045883, - "nauc_map_at_1_diff1": 0.6363475951973468, - "nauc_map_at_1_max": 0.3978201419843627, - "nauc_map_at_1_std": -0.07535068057179833, - "nauc_map_at_20_diff1": 0.5811344454158822, - "nauc_map_at_20_max": 0.4440615143544749, - "nauc_map_at_20_std": -0.03955530709812017, - "nauc_map_at_3_diff1": 0.583539903148956, - "nauc_map_at_3_max": 0.4347198346189301, - "nauc_map_at_3_std": -0.05914956536629059, - "nauc_map_at_5_diff1": 0.5808229323614098, - "nauc_map_at_5_max": 0.44138202029415413, - "nauc_map_at_5_std": -0.05072257931247744, - "nauc_mrr_at_1000_diff1": 0.5881203904403035, - "nauc_mrr_at_1000_max": 0.4484481284666203, - "nauc_mrr_at_1000_std": -0.041015672924594425, - "nauc_mrr_at_100_diff1": 0.5880202802699318, - "nauc_mrr_at_100_max": 0.4485244130022475, - "nauc_mrr_at_100_std": -0.04073803891469471, - "nauc_mrr_at_10_diff1": 0.5872948949505256, - "nauc_mrr_at_10_max": 0.44803331196596113, - "nauc_mrr_at_10_std": -0.040942756000374045, - "nauc_mrr_at_1_diff1": 0.636898011304436, - "nauc_mrr_at_1_max": 0.44436198490104667, - "nauc_mrr_at_1_std": -0.06652018253703539, - "nauc_mrr_at_20_diff1": 0.5873601429560038, - "nauc_mrr_at_20_max": 0.4491342783654011, - "nauc_mrr_at_20_std": -0.039490970673790156, - "nauc_mrr_at_3_diff1": 0.5876254651867967, - "nauc_mrr_at_3_max": 0.4441519579742525, - "nauc_mrr_at_3_std": -0.05530094979107773, - "nauc_mrr_at_5_diff1": 0.5853848397651132, - "nauc_mrr_at_5_max": 0.44757418986482955, - "nauc_mrr_at_5_std": -0.047728167237815616, - "nauc_ndcg_at_1000_diff1": 0.5746984338116712, - "nauc_ndcg_at_1000_max": 0.44886091973812225, - "nauc_ndcg_at_1000_std": -0.02776328674880611, - "nauc_ndcg_at_100_diff1": 0.5718934034391514, - "nauc_ndcg_at_100_max": 0.4519133095623923, - "nauc_ndcg_at_100_std": -0.01859333456676269, - "nauc_ndcg_at_10_diff1": 0.5648965674150932, - "nauc_ndcg_at_10_max": 0.4509288057994942, - "nauc_ndcg_at_10_std": -0.01760326906724774, - "nauc_ndcg_at_1_diff1": 0.636898011304436, - "nauc_ndcg_at_1_max": 0.44436198490104667, - "nauc_ndcg_at_1_std": -0.06652018253703539, - "nauc_ndcg_at_20_diff1": 0.5653913906886788, - "nauc_ndcg_at_20_max": 0.4567155165752001, - "nauc_ndcg_at_20_std": -0.010364885098699228, - "nauc_ndcg_at_3_diff1": 0.5691469740706386, - "nauc_ndcg_at_3_max": 0.4377098844996301, - "nauc_ndcg_at_3_std": -0.05395678784871217, - "nauc_ndcg_at_5_diff1": 0.5637652891437743, - "nauc_ndcg_at_5_max": 0.44804388763273256, - "nauc_ndcg_at_5_std": -0.03867023871561297, - "nauc_precision_at_1000_diff1": -0.1269295688966779, - "nauc_precision_at_1000_max": 0.09576039350822045, - "nauc_precision_at_1000_std": 0.09159920254296734, - "nauc_precision_at_100_diff1": -0.007437154335484724, - "nauc_precision_at_100_max": 0.20039321410244404, - "nauc_precision_at_100_std": 0.1426867944558165, - "nauc_precision_at_10_diff1": 0.2452553975868521, - "nauc_precision_at_10_max": 0.37544079604903624, - "nauc_precision_at_10_std": 0.10584035906560733, - "nauc_precision_at_1_diff1": 0.636898011304436, - "nauc_precision_at_1_max": 0.44436198490104667, - "nauc_precision_at_1_std": -0.06652018253703539, - "nauc_precision_at_20_diff1": 0.16916910475445357, - "nauc_precision_at_20_max": 0.34642136478127744, - "nauc_precision_at_20_std": 0.14261345610270082, - "nauc_precision_at_3_diff1": 0.377439212377142, - "nauc_precision_at_3_max": 0.4175722537265246, - "nauc_precision_at_3_std": -0.001966043546048812, - "nauc_precision_at_5_diff1": 0.32001958998919133, - "nauc_precision_at_5_max": 0.4091917985758222, - "nauc_precision_at_5_std": 0.037734817146087056, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.46741327337617883, - "nauc_recall_at_100_max": 0.5601227994679975, - "nauc_recall_at_100_std": 0.32393562317395125, - "nauc_recall_at_10_diff1": 0.48441585851399666, - "nauc_recall_at_10_max": 0.46628557295962697, - "nauc_recall_at_10_std": 0.08341081487340105, - "nauc_recall_at_1_diff1": 0.6363475951973468, - "nauc_recall_at_1_max": 0.3978201419843627, - "nauc_recall_at_1_std": -0.07535068057179833, - "nauc_recall_at_20_diff1": 0.4621173128401141, - "nauc_recall_at_20_max": 0.5117026275838669, - "nauc_recall_at_20_std": 0.1626250437132122, - "nauc_recall_at_3_diff1": 0.5164962617526161, - "nauc_recall_at_3_max": 0.43204598915930476, - "nauc_recall_at_3_std": -0.04234979378579455, - "nauc_recall_at_5_diff1": 0.49384971068639183, - "nauc_recall_at_5_max": 0.4557447504137923, - "nauc_recall_at_5_std": -0.0012052087838667518, - "ndcg_at_1": 0.45833, - "ndcg_at_10": 0.59091, - "ndcg_at_100": 0.6353, - "ndcg_at_1000": 0.64262, - "ndcg_at_20": 0.61038, - "ndcg_at_3": 0.5348, - "ndcg_at_5": 0.56109, - "precision_at_1": 0.45833, - "precision_at_10": 0.08474, - "precision_at_100": 0.01079, - "precision_at_1000": 0.00114, - "precision_at_20": 0.0466, - "precision_at_3": 0.22418, - "precision_at_5": 0.14906, - "recall_at_1": 0.41862, - "recall_at_10": 0.74169, - "recall_at_100": 0.9464, - "recall_at_1000": 1.0, - "recall_at_20": 0.81641, - "recall_at_3": 0.59165, - "recall_at_5": 0.65278 - } - ] - }, - "task_name": "GLDv2I2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRB.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRB.json deleted file mode 100644 index c3d7abe851..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRB.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "dataset_revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", - "evaluation_time": 161.53320479393005, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "accuracy": 0.6921615201900237, - "f1": 0.6132213777667673, - "f1_weighted": 0.7046823400547864, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6921615201900237, - "scores_per_experiment": [ - { - "accuracy": 0.6921615201900237, - "f1": 0.6132213777667673, - "f1_weighted": 0.7046823400547864 - } - ] - } - ] - }, - "task_name": "GTSRB" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRBZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRBZeroShot.json deleted file mode 100644 index e341bc14a9..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/GTSRBZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", - "evaluation_time": 32.170368909835815, - "kg_co2_emissions": null, - "mteb_version": "1.12.49", - "scores": { - "test": [ - { - "accuracy": 0.34418052256532067, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.34418052256532067 - } - ] - }, - "task_name": "GTSRBZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesI2TRetrieval.json deleted file mode 100644 index 1730af37d4..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", - "evaluation_time": 11.598620414733887, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.275, - "cv_recall_at_10": 0.573, - "cv_recall_at_100": 0.725, - "cv_recall_at_1000": 0.842, - "cv_recall_at_20": 0.621, - "cv_recall_at_3": 0.464, - "cv_recall_at_5": 0.515, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.43121, - "map_at_1": 0.279, - "map_at_10": 0.38507, - "map_at_100": 0.39114, - "map_at_1000": 0.39162, - "map_at_20": 0.3885, - "map_at_3": 0.3645, - "map_at_5": 0.37645, - "mrr_at_1": 0.275, - "mrr_at_10": 0.3829083333333332, - "mrr_at_100": 0.389097149571975, - "mrr_at_1000": 0.38958301943944784, - "mrr_at_20": 0.3864570007177203, - "mrr_at_3": 0.3629999999999999, - "mrr_at_5": 0.37474999999999964, - "nauc_cv_recall_at_1000_diff1": 0.031303926802318874, - "nauc_cv_recall_at_1000_max": 0.5619684912289413, - "nauc_cv_recall_at_1000_std": 0.37050428931457885, - "nauc_cv_recall_at_100_diff1": 0.042561843442901065, - "nauc_cv_recall_at_100_max": 0.5743835019605944, - "nauc_cv_recall_at_100_std": 0.32809410853463755, - "nauc_cv_recall_at_10_diff1": 0.11190351744901811, - "nauc_cv_recall_at_10_max": 0.5729760161495386, - "nauc_cv_recall_at_10_std": 0.24052820049848195, - "nauc_cv_recall_at_1_diff1": 0.3829883438391379, - "nauc_cv_recall_at_1_max": 0.4329626368982641, - "nauc_cv_recall_at_1_std": 0.08369053068986149, - "nauc_cv_recall_at_20_diff1": 0.09076044156560661, - "nauc_cv_recall_at_20_max": 0.5839339901644851, - "nauc_cv_recall_at_20_std": 0.26806324195952075, - "nauc_cv_recall_at_3_diff1": 0.12719746094572382, - "nauc_cv_recall_at_3_max": 0.5563392142827067, - "nauc_cv_recall_at_3_std": 0.20532706640279597, - "nauc_cv_recall_at_5_diff1": 0.11012794403209765, - "nauc_cv_recall_at_5_max": 0.5633962340746552, - "nauc_cv_recall_at_5_std": 0.2285752948260813, - "nauc_map_at_1000_diff1": 0.24302513335039774, - "nauc_map_at_1000_max": 0.4961908063983587, - "nauc_map_at_1000_std": 0.1429811933626351, - "nauc_map_at_100_diff1": 0.242827802636344, - "nauc_map_at_100_max": 0.4961973841716296, - "nauc_map_at_100_std": 0.1428596727473324, - "nauc_map_at_10_diff1": 0.24405062651754694, - "nauc_map_at_10_max": 0.49637204783980043, - "nauc_map_at_10_std": 0.14114302546813653, - "nauc_map_at_1_diff1": 0.3699809584569773, - "nauc_map_at_1_max": 0.4374252364103405, - "nauc_map_at_1_std": 0.0701724931549778, - "nauc_map_at_20_diff1": 0.2432461326883906, - "nauc_map_at_20_max": 0.49697515480820853, - "nauc_map_at_20_std": 0.14257792437817154, - "nauc_map_at_3_diff1": 0.24748123481892303, - "nauc_map_at_3_max": 0.4924971968626506, - "nauc_map_at_3_std": 0.13419322516706664, - "nauc_map_at_5_diff1": 0.24242153690056423, - "nauc_map_at_5_max": 0.4961183870348337, - "nauc_map_at_5_std": 0.13967033379573657, - "nauc_mrr_at_1000_diff1": 0.25115301640443705, - "nauc_mrr_at_1000_max": 0.4956146751056607, - "nauc_mrr_at_1000_std": 0.15010693213347828, - "nauc_mrr_at_100_diff1": 0.2509461328160055, - "nauc_mrr_at_100_max": 0.49562173640059265, - "nauc_mrr_at_100_std": 0.14997675975447758, - "nauc_mrr_at_10_diff1": 0.25258481872032185, - "nauc_mrr_at_10_max": 0.4961071238767585, - "nauc_mrr_at_10_std": 0.14802888717908258, - "nauc_mrr_at_1_diff1": 0.3829883438391379, - "nauc_mrr_at_1_max": 0.4329626368982641, - "nauc_mrr_at_1_std": 0.08369053068986149, - "nauc_mrr_at_20_diff1": 0.2513171572614364, - "nauc_mrr_at_20_max": 0.4964005836134918, - "nauc_mrr_at_20_std": 0.14965489299578572, - "nauc_mrr_at_3_diff1": 0.25549351845818574, - "nauc_mrr_at_3_max": 0.49385058969179646, - "nauc_mrr_at_3_std": 0.14122783678315726, - "nauc_mrr_at_5_diff1": 0.2515091731083041, - "nauc_mrr_at_5_max": 0.49544845579986435, - "nauc_mrr_at_5_std": 0.14676174165262207, - "nauc_ndcg_at_1000_diff1": 0.20883775445147526, - "nauc_ndcg_at_1000_max": 0.5104248735532774, - "nauc_ndcg_at_1000_std": 0.17388519073185565, - "nauc_ndcg_at_100_diff1": 0.20378718412019636, - "nauc_ndcg_at_100_max": 0.5127636544849964, - "nauc_ndcg_at_100_std": 0.17537566417326647, - "nauc_ndcg_at_10_diff1": 0.2093610810884053, - "nauc_ndcg_at_10_max": 0.5144665835322124, - "nauc_ndcg_at_10_std": 0.16593790274578846, - "nauc_ndcg_at_1_diff1": 0.3699809584569773, - "nauc_ndcg_at_1_max": 0.4374252364103405, - "nauc_ndcg_at_1_std": 0.0701724931549778, - "nauc_ndcg_at_20_diff1": 0.2068660199084453, - "nauc_ndcg_at_20_max": 0.516929381210776, - "nauc_ndcg_at_20_std": 0.17094758749205533, - "nauc_ndcg_at_3_diff1": 0.21501967088991664, - "nauc_ndcg_at_3_max": 0.5071457194184146, - "nauc_ndcg_at_3_std": 0.15279761060645786, - "nauc_ndcg_at_5_diff1": 0.20621710050182515, - "nauc_ndcg_at_5_max": 0.5135055727441509, - "nauc_ndcg_at_5_std": 0.16245661298705746, - "nauc_precision_at_1000_diff1": 0.03130392680231766, - "nauc_precision_at_1000_max": 0.5619684912289395, - "nauc_precision_at_1000_std": 0.37050428931457763, - "nauc_precision_at_100_diff1": 0.042561843442900246, - "nauc_precision_at_100_max": 0.5743835019605946, - "nauc_precision_at_100_std": 0.3280941085346374, - "nauc_precision_at_10_diff1": 0.10631618094334747, - "nauc_precision_at_10_max": 0.5696137757264226, - "nauc_precision_at_10_std": 0.24271559167529141, - "nauc_precision_at_1_diff1": 0.3699809584569773, - "nauc_precision_at_1_max": 0.4374252364103405, - "nauc_precision_at_1_std": 0.0701724931549778, - "nauc_precision_at_20_diff1": 0.09076044156560646, - "nauc_precision_at_20_max": 0.5839339901644841, - "nauc_precision_at_20_std": 0.26806324195951986, - "nauc_precision_at_3_diff1": 0.12543951694556468, - "nauc_precision_at_3_max": 0.5475888041194277, - "nauc_precision_at_3_std": 0.2047169339929046, - "nauc_precision_at_5_diff1": 0.10323701029939387, - "nauc_precision_at_5_max": 0.5633364797327312, - "nauc_precision_at_5_std": 0.22856093414302459, - "nauc_recall_at_1000_diff1": 0.031303926802318874, - "nauc_recall_at_1000_max": 0.5619684912289413, - "nauc_recall_at_1000_std": 0.37050428931457885, - "nauc_recall_at_100_diff1": 0.042561843442901065, - "nauc_recall_at_100_max": 0.5743835019605944, - "nauc_recall_at_100_std": 0.32809410853463755, - "nauc_recall_at_10_diff1": 0.1063161809433481, - "nauc_recall_at_10_max": 0.5696137757264229, - "nauc_recall_at_10_std": 0.24271559167529194, - "nauc_recall_at_1_diff1": 0.3699809584569773, - "nauc_recall_at_1_max": 0.4374252364103405, - "nauc_recall_at_1_std": 0.0701724931549778, - "nauc_recall_at_20_diff1": 0.09076044156560661, - "nauc_recall_at_20_max": 0.5839339901644851, - "nauc_recall_at_20_std": 0.26806324195952075, - "nauc_recall_at_3_diff1": 0.12543951694556424, - "nauc_recall_at_3_max": 0.5475888041194277, - "nauc_recall_at_3_std": 0.20471693399290464, - "nauc_recall_at_5_diff1": 0.10323701029939407, - "nauc_recall_at_5_max": 0.5633364797327317, - "nauc_recall_at_5_std": 0.22856093414302503, - "ndcg_at_1": 0.279, - "ndcg_at_10": 0.43121, - "ndcg_at_100": 0.46258, - "ndcg_at_1000": 0.47698, - "ndcg_at_20": 0.44331, - "ndcg_at_3": 0.38975, - "ndcg_at_5": 0.41101, - "precision_at_1": 0.279, - "precision_at_10": 0.0574, - "precision_at_100": 0.00725, - "precision_at_1000": 0.00084, - "precision_at_20": 0.03105, - "precision_at_3": 0.154, - "precision_at_5": 0.1026, - "recall_at_1": 0.279, - "recall_at_10": 0.574, - "recall_at_100": 0.725, - "recall_at_1000": 0.842, - "recall_at_20": 0.621, - "recall_at_3": 0.462, - "recall_at_5": 0.513 - } - ] - }, - "task_name": "HatefulMemesI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesT2IRetrieval.json deleted file mode 100644 index 2555d35599..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/HatefulMemesT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", - "evaluation_time": 48.356528520584106, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.329, - "cv_recall_at_10": 0.599, - "cv_recall_at_100": 0.751, - "cv_recall_at_1000": 0.851, - "cv_recall_at_20": 0.657, - "cv_recall_at_3": 0.492, - "cv_recall_at_5": 0.542, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.46492, - "map_at_1": 0.329, - "map_at_10": 0.42189, - "map_at_100": 0.42825, - "map_at_1000": 0.42866, - "map_at_20": 0.42593, - "map_at_3": 0.4025, - "map_at_5": 0.41405, - "mrr_at_1": 0.329, - "mrr_at_10": 0.4219277777777777, - "mrr_at_100": 0.42830345620240934, - "mrr_at_1000": 0.4286957438222649, - "mrr_at_20": 0.42596962569164554, - "mrr_at_3": 0.4024999999999998, - "mrr_at_5": 0.4140499999999997, - "nauc_cv_recall_at_1000_diff1": 0.22118073062290955, - "nauc_cv_recall_at_1000_max": 0.3484340520825837, - "nauc_cv_recall_at_1000_std": 0.271289099588132, - "nauc_cv_recall_at_100_diff1": 0.3151051127355348, - "nauc_cv_recall_at_100_max": 0.3956778936713964, - "nauc_cv_recall_at_100_std": 0.2495175376887586, - "nauc_cv_recall_at_10_diff1": 0.3598026585943281, - "nauc_cv_recall_at_10_max": 0.36412098427583917, - "nauc_cv_recall_at_10_std": 0.0729252987471962, - "nauc_cv_recall_at_1_diff1": 0.48014966506514223, - "nauc_cv_recall_at_1_max": 0.2804395112575141, - "nauc_cv_recall_at_1_std": 0.04746098198717796, - "nauc_cv_recall_at_20_diff1": 0.3692325712994173, - "nauc_cv_recall_at_20_max": 0.37522127689761997, - "nauc_cv_recall_at_20_std": 0.14494647285037632, - "nauc_cv_recall_at_3_diff1": 0.4240862392440114, - "nauc_cv_recall_at_3_max": 0.35920484079276727, - "nauc_cv_recall_at_3_std": 0.04660884077150774, - "nauc_cv_recall_at_5_diff1": 0.3993186558000575, - "nauc_cv_recall_at_5_max": 0.3607991802708112, - "nauc_cv_recall_at_5_std": 0.033385030475325865, - "nauc_map_at_1000_diff1": 0.4437952396317114, - "nauc_map_at_1000_max": 0.31816421599945327, - "nauc_map_at_1000_std": 0.056598746601127375, - "nauc_map_at_100_diff1": 0.44391951463638446, - "nauc_map_at_100_max": 0.3183891752169483, - "nauc_map_at_100_std": 0.056738797910687, - "nauc_map_at_10_diff1": 0.4433494101595439, - "nauc_map_at_10_max": 0.3179603154080035, - "nauc_map_at_10_std": 0.05128662739139575, - "nauc_map_at_1_diff1": 0.48014966506514223, - "nauc_map_at_1_max": 0.2804395112575141, - "nauc_map_at_1_std": 0.04746098198717796, - "nauc_map_at_20_diff1": 0.444778154649432, - "nauc_map_at_20_max": 0.3182550979373629, - "nauc_map_at_20_std": 0.055205200384806766, - "nauc_map_at_3_diff1": 0.45290708022813964, - "nauc_map_at_3_max": 0.31707367227066335, - "nauc_map_at_3_std": 0.04922208476639303, - "nauc_map_at_5_diff1": 0.4474035304498997, - "nauc_map_at_5_max": 0.31734789204809866, - "nauc_map_at_5_std": 0.04660491752104569, - "nauc_mrr_at_1000_diff1": 0.44370331964018916, - "nauc_mrr_at_1000_max": 0.31808399595668846, - "nauc_mrr_at_1000_std": 0.05655108354905683, - "nauc_mrr_at_100_diff1": 0.4438120282477109, - "nauc_mrr_at_100_max": 0.3182912799047865, - "nauc_mrr_at_100_std": 0.05668195802059791, - "nauc_mrr_at_10_diff1": 0.4432637013304661, - "nauc_mrr_at_10_max": 0.31788692981966776, - "nauc_mrr_at_10_std": 0.051245648964999085, - "nauc_mrr_at_1_diff1": 0.48014966506514223, - "nauc_mrr_at_1_max": 0.2804395112575141, - "nauc_mrr_at_1_std": 0.04746098198717796, - "nauc_mrr_at_20_diff1": 0.44469195301952164, - "nauc_mrr_at_20_max": 0.3181812500366557, - "nauc_mrr_at_20_std": 0.05516412700119084, - "nauc_mrr_at_3_diff1": 0.45290708022813964, - "nauc_mrr_at_3_max": 0.31707367227066335, - "nauc_mrr_at_3_std": 0.04922208476639303, - "nauc_mrr_at_5_diff1": 0.4474035304498997, - "nauc_mrr_at_5_max": 0.31734789204809866, - "nauc_mrr_at_5_std": 0.04660491752104569, - "nauc_ndcg_at_1000_diff1": 0.4208294347119966, - "nauc_ndcg_at_1000_max": 0.32675359943049515, - "nauc_ndcg_at_1000_std": 0.0778020901079404, - "nauc_ndcg_at_100_diff1": 0.42354921239544435, - "nauc_ndcg_at_100_max": 0.33216360569482184, - "nauc_ndcg_at_100_std": 0.08196916027497256, - "nauc_ndcg_at_10_diff1": 0.4247919690071033, - "nauc_ndcg_at_10_max": 0.3292634407930656, - "nauc_ndcg_at_10_std": 0.0554815525525049, - "nauc_ndcg_at_1_diff1": 0.48014966506514223, - "nauc_ndcg_at_1_max": 0.2804395112575141, - "nauc_ndcg_at_1_std": 0.04746098198717796, - "nauc_ndcg_at_20_diff1": 0.42888603859808855, - "nauc_ndcg_at_20_max": 0.3307635310552114, - "nauc_ndcg_at_20_std": 0.07056726430076309, - "nauc_ndcg_at_3_diff1": 0.44540872613954735, - "nauc_ndcg_at_3_max": 0.32795961422684516, - "nauc_ndcg_at_3_std": 0.04864217823353319, - "nauc_ndcg_at_5_diff1": 0.43559223952131837, - "nauc_ndcg_at_5_max": 0.32832423954086903, - "nauc_ndcg_at_5_std": 0.04362509890485802, - "nauc_precision_at_1000_diff1": 0.22118073062290727, - "nauc_precision_at_1000_max": 0.3484340520825814, - "nauc_precision_at_1000_std": 0.2712890995881306, - "nauc_precision_at_100_diff1": 0.3179822082679228, - "nauc_precision_at_100_max": 0.39829408686551576, - "nauc_precision_at_100_std": 0.2503150183150184, - "nauc_precision_at_10_diff1": 0.35980265859432753, - "nauc_precision_at_10_max": 0.3641209842758386, - "nauc_precision_at_10_std": 0.07292529874719615, - "nauc_precision_at_1_diff1": 0.48014966506514223, - "nauc_precision_at_1_max": 0.2804395112575141, - "nauc_precision_at_1_std": 0.04746098198717796, - "nauc_precision_at_20_diff1": 0.3692325712994169, - "nauc_precision_at_20_max": 0.375221276897619, - "nauc_precision_at_20_std": 0.14494647285037554, - "nauc_precision_at_3_diff1": 0.42408623924401156, - "nauc_precision_at_3_max": 0.35920484079276777, - "nauc_precision_at_3_std": 0.046608840771507926, - "nauc_precision_at_5_diff1": 0.39931865580005693, - "nauc_precision_at_5_max": 0.360799180270811, - "nauc_precision_at_5_std": 0.033385030475325955, - "nauc_recall_at_1000_diff1": 0.22118073062290955, - "nauc_recall_at_1000_max": 0.3484340520825837, - "nauc_recall_at_1000_std": 0.271289099588132, - "nauc_recall_at_100_diff1": 0.3179822082679226, - "nauc_recall_at_100_max": 0.3982940868655151, - "nauc_recall_at_100_std": 0.25031501831501846, - "nauc_recall_at_10_diff1": 0.3598026585943281, - "nauc_recall_at_10_max": 0.36412098427583917, - "nauc_recall_at_10_std": 0.0729252987471962, - "nauc_recall_at_1_diff1": 0.48014966506514223, - "nauc_recall_at_1_max": 0.2804395112575141, - "nauc_recall_at_1_std": 0.04746098198717796, - "nauc_recall_at_20_diff1": 0.3692325712994173, - "nauc_recall_at_20_max": 0.37522127689761997, - "nauc_recall_at_20_std": 0.14494647285037632, - "nauc_recall_at_3_diff1": 0.4240862392440114, - "nauc_recall_at_3_max": 0.35920484079276727, - "nauc_recall_at_3_std": 0.04660884077150774, - "nauc_recall_at_5_diff1": 0.3993186558000575, - "nauc_recall_at_5_max": 0.3607991802708112, - "nauc_recall_at_5_std": 0.033385030475325865, - "ndcg_at_1": 0.329, - "ndcg_at_10": 0.46492, - "ndcg_at_100": 0.49677, - "ndcg_at_1000": 0.50914, - "ndcg_at_20": 0.4796, - "ndcg_at_3": 0.42556, - "ndcg_at_5": 0.44626, - "precision_at_1": 0.329, - "precision_at_10": 0.0599, - "precision_at_100": 0.0075, - "precision_at_1000": 0.00085, - "precision_at_20": 0.03285, - "precision_at_3": 0.164, - "precision_at_5": 0.1084, - "recall_at_1": 0.329, - "recall_at_10": 0.599, - "recall_at_100": 0.75, - "recall_at_1000": 0.851, - "recall_at_20": 0.657, - "recall_at_3": 0.492, - "recall_at_5": 0.542 - } - ] - }, - "task_name": "HatefulMemesT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IMultiChoice.json deleted file mode 100644 index 1f3e5fbd9e..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IMultiChoice.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "dataset_revision": "d28adfd8b34fefa546fdf94bdc352622b2575f6c", - "evaluation_time": 459.3987202644348, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.13206, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.13206, - "mrr_at_1": 0.13205907906168549, - "mrr_at_10": 0.32183470550108295, - "mrr_at_100": 0.32183470550108295, - "mrr_at_1000": 0.32183470550108295, - "mrr_at_20": 0.32183470550108295, - "mrr_at_3": 0.2158268172603526, - "mrr_at_5": 0.2607225600926729, - "ndcg_at_1": 0.13206, - "ndcg_at_10": 0.47717, - "ndcg_at_100": 0.47717, - "ndcg_at_1000": 0.47717, - "ndcg_at_20": 0.47717, - "ndcg_at_3": 0.24566, - "ndcg_at_5": 0.32738 - } - ] - }, - "task_name": "ImageCoDeT2IMultiChoice" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IRetrieval.json deleted file mode 100644 index 13200b7196..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageCoDeT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "a424cd523ffb157b69a875fb5e71c1d51be54089", - "evaluation_time": 82.47818303108215, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.0404, - "cv_recall_at_10": 0.19939, - "cv_recall_at_100": 0.4748, - "cv_recall_at_1000": 0.67811, - "cv_recall_at_20": 0.29018, - "cv_recall_at_3": 0.08905, - "cv_recall_at_5": 0.12858, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.1073, - "map_at_1": 0.0404, - "map_at_10": 0.07947, - "map_at_100": 0.09013, - "map_at_1000": 0.09115, - "map_at_20": 0.0858, - "map_at_3": 0.06118, - "map_at_5": 0.07015, - "mrr_at_1": 0.04039965247610773, - "mrr_at_10": 0.07948695406340925, - "mrr_at_100": 0.09015226912980688, - "mrr_at_1000": 0.09117557368465125, - "mrr_at_20": 0.08582291845210348, - "mrr_at_3": 0.06117868520127424, - "mrr_at_5": 0.07017086591369824, - "nauc_cv_recall_at_1000_diff1": 0.10207641080088928, - "nauc_cv_recall_at_1000_max": 0.47551950475797095, - "nauc_cv_recall_at_1000_std": 0.667878501495164, - "nauc_cv_recall_at_100_diff1": 0.10625659946342164, - "nauc_cv_recall_at_100_max": 0.429394897766257, - "nauc_cv_recall_at_100_std": 0.6428953991022838, - "nauc_cv_recall_at_10_diff1": 0.1561354794756864, - "nauc_cv_recall_at_10_max": 0.3230948830435383, - "nauc_cv_recall_at_10_std": 0.4849191311442782, - "nauc_cv_recall_at_1_diff1": 0.31664206223566443, - "nauc_cv_recall_at_1_max": 0.17966444353568511, - "nauc_cv_recall_at_1_std": 0.39566281510890866, - "nauc_cv_recall_at_20_diff1": 0.12129977218912101, - "nauc_cv_recall_at_20_max": 0.38830333447601323, - "nauc_cv_recall_at_20_std": 0.5541829899702385, - "nauc_cv_recall_at_3_diff1": 0.19875202658849309, - "nauc_cv_recall_at_3_max": 0.28302032898711493, - "nauc_cv_recall_at_3_std": 0.4420405932660536, - "nauc_cv_recall_at_5_diff1": 0.1777398492807893, - "nauc_cv_recall_at_5_max": 0.28188387839125345, - "nauc_cv_recall_at_5_std": 0.4596852170744158, - "nauc_map_at_1000_diff1": 0.21083155153769165, - "nauc_map_at_1000_max": 0.2761233937186182, - "nauc_map_at_1000_std": 0.4596747781481127, - "nauc_map_at_100_diff1": 0.21045007881515884, - "nauc_map_at_100_max": 0.2758409491731245, - "nauc_map_at_100_std": 0.45980619206042345, - "nauc_map_at_10_diff1": 0.2198233112203957, - "nauc_map_at_10_max": 0.25983030622113884, - "nauc_map_at_10_std": 0.4388620519482371, - "nauc_map_at_1_diff1": 0.31664206223566443, - "nauc_map_at_1_max": 0.17966444353568511, - "nauc_map_at_1_std": 0.39566281510890866, - "nauc_map_at_20_diff1": 0.21187076846238379, - "nauc_map_at_20_max": 0.2726996242735469, - "nauc_map_at_20_std": 0.4532884201759696, - "nauc_map_at_3_diff1": 0.2424283784615785, - "nauc_map_at_3_max": 0.2405242743804034, - "nauc_map_at_3_std": 0.4204628045064827, - "nauc_map_at_5_diff1": 0.22933322343545606, - "nauc_map_at_5_max": 0.24325667796160028, - "nauc_map_at_5_std": 0.4279703891935171, - "nauc_mrr_at_1000_diff1": 0.2111114841525623, - "nauc_mrr_at_1000_max": 0.27633286230256027, - "nauc_mrr_at_1000_std": 0.4599361510805012, - "nauc_mrr_at_100_diff1": 0.2107295407122133, - "nauc_mrr_at_100_max": 0.27604998501829475, - "nauc_mrr_at_100_std": 0.46006698826843867, - "nauc_mrr_at_10_diff1": 0.22007340887962168, - "nauc_mrr_at_10_max": 0.25997366670711813, - "nauc_mrr_at_10_std": 0.4390697248762385, - "nauc_mrr_at_1_diff1": 0.31664206223566443, - "nauc_mrr_at_1_max": 0.17966444353568511, - "nauc_mrr_at_1_std": 0.39566281510890866, - "nauc_mrr_at_20_diff1": 0.21215153596461336, - "nauc_mrr_at_20_max": 0.2729107668453846, - "nauc_mrr_at_20_std": 0.45355197906522926, - "nauc_mrr_at_3_diff1": 0.2424283784615785, - "nauc_mrr_at_3_max": 0.2405242743804034, - "nauc_mrr_at_3_std": 0.4204628045064827, - "nauc_mrr_at_5_diff1": 0.22960397137313956, - "nauc_mrr_at_5_max": 0.24341652796565305, - "nauc_mrr_at_5_std": 0.42819917818577374, - "nauc_ndcg_at_1000_diff1": 0.17173880474374906, - "nauc_ndcg_at_1000_max": 0.33895563139815, - "nauc_ndcg_at_1000_std": 0.5246453040907454, - "nauc_ndcg_at_100_diff1": 0.1683122269839326, - "nauc_ndcg_at_100_max": 0.33447412438250224, - "nauc_ndcg_at_100_std": 0.5261247976469097, - "nauc_ndcg_at_10_diff1": 0.1941087589715255, - "nauc_ndcg_at_10_max": 0.2852009961647385, - "nauc_ndcg_at_10_std": 0.4571235834817824, - "nauc_ndcg_at_1_diff1": 0.31664206223566443, - "nauc_ndcg_at_1_max": 0.17966444353568511, - "nauc_ndcg_at_1_std": 0.39566281510890866, - "nauc_ndcg_at_20_diff1": 0.17466391431078573, - "nauc_ndcg_at_20_max": 0.31877860873395164, - "nauc_ndcg_at_20_std": 0.49309984249311956, - "nauc_ndcg_at_3_diff1": 0.22782771989085718, - "nauc_ndcg_at_3_max": 0.25457117569804816, - "nauc_ndcg_at_3_std": 0.4275096555899013, - "nauc_ndcg_at_5_diff1": 0.21040054798962565, - "nauc_ndcg_at_5_max": 0.25741771300479144, - "nauc_ndcg_at_5_std": 0.4391783586085749, - "nauc_precision_at_1000_diff1": 0.10207641080088917, - "nauc_precision_at_1000_max": 0.47551950475797033, - "nauc_precision_at_1000_std": 0.6678785014951635, - "nauc_precision_at_100_diff1": 0.10625659946342192, - "nauc_precision_at_100_max": 0.4293948977662575, - "nauc_precision_at_100_std": 0.6428953991022842, - "nauc_precision_at_10_diff1": 0.15613547947568646, - "nauc_precision_at_10_max": 0.32309488304353845, - "nauc_precision_at_10_std": 0.4849191311442785, - "nauc_precision_at_1_diff1": 0.31664206223566443, - "nauc_precision_at_1_max": 0.17966444353568511, - "nauc_precision_at_1_std": 0.39566281510890866, - "nauc_precision_at_20_diff1": 0.12129977218912126, - "nauc_precision_at_20_max": 0.3883033344760135, - "nauc_precision_at_20_std": 0.5541829899702387, - "nauc_precision_at_3_diff1": 0.19875202658849306, - "nauc_precision_at_3_max": 0.283020328987115, - "nauc_precision_at_3_std": 0.44204059326605394, - "nauc_precision_at_5_diff1": 0.17773984928078942, - "nauc_precision_at_5_max": 0.2818838783912535, - "nauc_precision_at_5_std": 0.45968521707441584, - "nauc_recall_at_1000_diff1": 0.10207641080088928, - "nauc_recall_at_1000_max": 0.47551950475797095, - "nauc_recall_at_1000_std": 0.667878501495164, - "nauc_recall_at_100_diff1": 0.10625659946342164, - "nauc_recall_at_100_max": 0.429394897766257, - "nauc_recall_at_100_std": 0.6428953991022838, - "nauc_recall_at_10_diff1": 0.1561354794756864, - "nauc_recall_at_10_max": 0.3230948830435383, - "nauc_recall_at_10_std": 0.4849191311442782, - "nauc_recall_at_1_diff1": 0.31664206223566443, - "nauc_recall_at_1_max": 0.17966444353568511, - "nauc_recall_at_1_std": 0.39566281510890866, - "nauc_recall_at_20_diff1": 0.12129977218912101, - "nauc_recall_at_20_max": 0.38830333447601323, - "nauc_recall_at_20_std": 0.5541829899702385, - "nauc_recall_at_3_diff1": 0.19875202658849309, - "nauc_recall_at_3_max": 0.28302032898711493, - "nauc_recall_at_3_std": 0.4420405932660536, - "nauc_recall_at_5_diff1": 0.1777398492807893, - "nauc_recall_at_5_max": 0.28188387839125345, - "nauc_recall_at_5_std": 0.4596852170744158, - "ndcg_at_1": 0.0404, - "ndcg_at_10": 0.1073, - "ndcg_at_100": 0.16373, - "ndcg_at_1000": 0.19005, - "ndcg_at_20": 0.13029, - "ndcg_at_3": 0.06831, - "ndcg_at_5": 0.08453, - "precision_at_1": 0.0404, - "precision_at_10": 0.01994, - "precision_at_100": 0.00475, - "precision_at_1000": 0.00068, - "precision_at_20": 0.01451, - "precision_at_3": 0.02968, - "precision_at_5": 0.02572, - "recall_at_1": 0.0404, - "recall_at_10": 0.19939, - "recall_at_100": 0.4748, - "recall_at_1000": 0.67811, - "recall_at_20": 0.29018, - "recall_at_3": 0.08905, - "recall_at_5": 0.12858 - } - ] - }, - "task_name": "ImageCoDeT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNet10Clustering.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNet10Clustering.json deleted file mode 100644 index d502635992..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNet10Clustering.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "dataset_revision": "88f8a6d47c257895094c5ad81e67ba751771fc99", - "evaluation_time": 33.32936453819275, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.1993076923076923, - "ari": 0.9672782515730578, - "cluster_accuracy": 0.985, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.1993076923076923, - "nmi": 0.9644473066207006, - "v_measure": 0.9644473066207006 - } - ] - }, - "task_name": "ImageNet10Clustering" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNetDog15Clustering.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNetDog15Clustering.json deleted file mode 100644 index fe53c8ed7e..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ImageNetDog15Clustering.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "dataset_revision": "bfb6ad3b2109d26c9daddf14f98d315daa35ee72", - "evaluation_time": 4.18316650390625, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.026022304832713755, - "ari": 0.36465670607270784, - "cluster_accuracy": 0.4656133828996282, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.026022304832713755, - "nmi": 0.5160500208664386, - "v_measure": 0.5160500208664386 - } - ] - }, - "task_name": "ImageNetDog15Clustering" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1k.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1k.json deleted file mode 100644 index 57d150dd7e..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1k.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "dataset_revision": "b24c7a5a3ef12df09089055d1795e2ce7c7e7397", - "evaluation_time": 4185.878092765808, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "accuracy": 0.6102, - "f1": 0.607661279811048, - "f1_weighted": 0.6076612798110479, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6102, - "scores_per_experiment": [ - { - "accuracy": 0.6102, - "f1": 0.607661279811048, - "f1_weighted": 0.6076612798110479 - } - ] - } - ] - }, - "task_name": "Imagenet1k" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1kZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1kZeroShot.json deleted file mode 100644 index 93f2b00122..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Imagenet1kZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "b24c7a5a3ef12df09089055d1795e2ce7c7e7397", - "evaluation_time": 234.19821047782898, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "accuracy": 0.58842, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.58842 - } - ] - }, - "task_name": "Imagenet1kZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2ITRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2ITRetrieval.json deleted file mode 100644 index c7dc42a9cd..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2ITRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "78ee7f7708aac75d3afac5dcab1c9e03cb62664c", - "evaluation_time": 1977.6153802871704, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.02302, - "cv_recall_at_10": 0.1159, - "cv_recall_at_100": 0.33792, - "cv_recall_at_1000": 0.64213, - "cv_recall_at_20": 0.17109, - "cv_recall_at_3": 0.05281, - "cv_recall_at_5": 0.07401, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.03561, - "map_at_1": 0.00363, - "map_at_10": 0.0159, - "map_at_100": 0.03009, - "map_at_1000": 0.03503, - "map_at_20": 0.02049, - "map_at_3": 0.00821, - "map_at_5": 0.01103, - "mrr_at_1": 0.023020519524811003, - "mrr_at_10": 0.045806435459990955, - "mrr_at_100": 0.05351402487304762, - "mrr_at_1000": 0.05465984282946314, - "mrr_at_20": 0.04955566772605788, - "mrr_at_3": 0.03543075844559389, - "mrr_at_5": 0.040216752875196694, - "nauc_cv_recall_at_1000_diff1": 0.02015559920712523, - "nauc_cv_recall_at_1000_max": 0.24843256485471527, - "nauc_cv_recall_at_1000_std": 0.551074214857492, - "nauc_cv_recall_at_100_diff1": -0.013440078556748954, - "nauc_cv_recall_at_100_max": 0.08259437484215827, - "nauc_cv_recall_at_100_std": 0.3728506524892255, - "nauc_cv_recall_at_10_diff1": -0.010908853697860047, - "nauc_cv_recall_at_10_max": 0.03265302320587433, - "nauc_cv_recall_at_10_std": 0.2912474705012547, - "nauc_cv_recall_at_1_diff1": -0.07459591752948071, - "nauc_cv_recall_at_1_max": 0.1041413767622372, - "nauc_cv_recall_at_1_std": 0.35757281008095554, - "nauc_cv_recall_at_20_diff1": -0.012006431352003777, - "nauc_cv_recall_at_20_max": 0.03928626639631902, - "nauc_cv_recall_at_20_std": 0.3109439121223359, - "nauc_cv_recall_at_3_diff1": -0.017901157268070282, - "nauc_cv_recall_at_3_max": 0.04809033608912219, - "nauc_cv_recall_at_3_std": 0.33002426201564206, - "nauc_cv_recall_at_5_diff1": -0.0005855123496424908, - "nauc_cv_recall_at_5_max": 0.04584585149215492, - "nauc_cv_recall_at_5_std": 0.3091096240828216, - "nauc_map_at_1000_diff1": -0.024361704214272237, - "nauc_map_at_1000_max": 0.09242420909635184, - "nauc_map_at_1000_std": 0.4217296486267511, - "nauc_map_at_100_diff1": -0.021467259941218432, - "nauc_map_at_100_max": 0.09288371297474067, - "nauc_map_at_100_std": 0.39700462700388234, - "nauc_map_at_10_diff1": -0.01621091913919308, - "nauc_map_at_10_max": 0.10737421656077656, - "nauc_map_at_10_std": 0.2883212294123066, - "nauc_map_at_1_diff1": -0.11122157215775409, - "nauc_map_at_1_max": 0.11283092055308541, - "nauc_map_at_1_std": 0.3101815193459182, - "nauc_map_at_20_diff1": -0.0132414014004414, - "nauc_map_at_20_max": 0.10305956628322452, - "nauc_map_at_20_std": 0.32172528986885807, - "nauc_map_at_3_diff1": -0.04703922992329759, - "nauc_map_at_3_max": 0.09191500246547152, - "nauc_map_at_3_std": 0.2776262040824528, - "nauc_map_at_5_diff1": -0.014823044890154886, - "nauc_map_at_5_max": 0.11547342983958507, - "nauc_map_at_5_std": 0.2786259705268041, - "nauc_mrr_at_1000_diff1": -0.025295693852781534, - "nauc_mrr_at_1000_max": 0.06665675577708217, - "nauc_mrr_at_1000_std": 0.3306288852027639, - "nauc_mrr_at_100_diff1": -0.025881583161203855, - "nauc_mrr_at_100_max": 0.06461279535497533, - "nauc_mrr_at_100_std": 0.32831954836747484, - "nauc_mrr_at_10_diff1": -0.026092451126092788, - "nauc_mrr_at_10_max": 0.061697627022831616, - "nauc_mrr_at_10_std": 0.3212451183051902, - "nauc_mrr_at_1_diff1": -0.07459591752948071, - "nauc_mrr_at_1_max": 0.1041413767622372, - "nauc_mrr_at_1_std": 0.35757281008095554, - "nauc_mrr_at_20_diff1": -0.025821452727717447, - "nauc_mrr_at_20_max": 0.061755410936926804, - "nauc_mrr_at_20_std": 0.32393209801181, - "nauc_mrr_at_3_diff1": -0.03487019953781371, - "nauc_mrr_at_3_max": 0.07256664246126977, - "nauc_mrr_at_3_std": 0.3399856220056522, - "nauc_mrr_at_5_diff1": -0.025993114561723766, - "nauc_mrr_at_5_max": 0.06852797283441574, - "nauc_mrr_at_5_std": 0.3305774742662973, - "nauc_ndcg_at_1000_diff1": -0.009096662902559042, - "nauc_ndcg_at_1000_max": 0.11414272124177291, - "nauc_ndcg_at_1000_std": 0.44903775651649097, - "nauc_ndcg_at_100_diff1": -0.0134914554783792, - "nauc_ndcg_at_100_max": 0.08546139791128299, - "nauc_ndcg_at_100_std": 0.39209880947251335, - "nauc_ndcg_at_10_diff1": -0.0170068734005736, - "nauc_ndcg_at_10_max": 0.07471253375985419, - "nauc_ndcg_at_10_std": 0.33715358748076135, - "nauc_ndcg_at_1_diff1": -0.07459591752948071, - "nauc_ndcg_at_1_max": 0.1041413767622372, - "nauc_ndcg_at_1_std": 0.35757281008095554, - "nauc_ndcg_at_20_diff1": -0.01646794403371852, - "nauc_ndcg_at_20_max": 0.07852251935576997, - "nauc_ndcg_at_20_std": 0.3362314720735628, - "nauc_ndcg_at_3_diff1": -0.040316335382471105, - "nauc_ndcg_at_3_max": 0.06681861033932048, - "nauc_ndcg_at_3_std": 0.34481018084001014, - "nauc_ndcg_at_5_diff1": -0.01906600466599756, - "nauc_ndcg_at_5_max": 0.07855526607330436, - "nauc_ndcg_at_5_std": 0.35145622550348826, - "nauc_precision_at_1000_diff1": -0.04214581973882055, - "nauc_precision_at_1000_max": 0.054818857222616854, - "nauc_precision_at_1000_std": 0.4210937554531033, - "nauc_precision_at_100_diff1": -0.04404957111240489, - "nauc_precision_at_100_max": 0.048214060241998896, - "nauc_precision_at_100_std": 0.5022884634473646, - "nauc_precision_at_10_diff1": -0.02670920513820409, - "nauc_precision_at_10_max": 0.0524986708658414, - "nauc_precision_at_10_std": 0.38931841492231234, - "nauc_precision_at_1_diff1": -0.07459591752948071, - "nauc_precision_at_1_max": 0.1041413767622372, - "nauc_precision_at_1_std": 0.35757281008095554, - "nauc_precision_at_20_diff1": -0.03507168094166081, - "nauc_precision_at_20_max": 0.03854551370879106, - "nauc_precision_at_20_std": 0.4348216482299613, - "nauc_precision_at_3_diff1": -0.03182756465463996, - "nauc_precision_at_3_max": 0.06796130945038668, - "nauc_precision_at_3_std": 0.3375923259123569, - "nauc_precision_at_5_diff1": -0.017097663262845838, - "nauc_precision_at_5_max": 0.06997218653624815, - "nauc_precision_at_5_std": 0.3613890823731953, - "nauc_recall_at_1000_diff1": 0.01878544994568831, - "nauc_recall_at_1000_max": 0.19492400747658695, - "nauc_recall_at_1000_std": 0.4947445992310574, - "nauc_recall_at_100_diff1": 0.0004998682480571065, - "nauc_recall_at_100_max": 0.10307683961504077, - "nauc_recall_at_100_std": 0.3729176792809507, - "nauc_recall_at_10_diff1": 0.012273865896795341, - "nauc_recall_at_10_max": 0.0923729011236101, - "nauc_recall_at_10_std": 0.2597789820846874, - "nauc_recall_at_1_diff1": -0.11122157215775409, - "nauc_recall_at_1_max": 0.11283092055308541, - "nauc_recall_at_1_std": 0.3101815193459182, - "nauc_recall_at_20_diff1": 0.009083054016587643, - "nauc_recall_at_20_max": 0.09732442344740161, - "nauc_recall_at_20_std": 0.28782323808504057, - "nauc_recall_at_3_diff1": -0.020078445600854207, - "nauc_recall_at_3_max": 0.05741724146915211, - "nauc_recall_at_3_std": 0.27282304342784597, - "nauc_recall_at_5_diff1": 0.029128946277282092, - "nauc_recall_at_5_max": 0.10730405854943821, - "nauc_recall_at_5_std": 0.26884834591228696, - "ndcg_at_1": 0.02302, - "ndcg_at_10": 0.03561, - "ndcg_at_100": 0.08069, - "ndcg_at_1000": 0.13787, - "ndcg_at_20": 0.04519, - "ndcg_at_3": 0.0251, - "ndcg_at_5": 0.02823, - "precision_at_1": 0.02302, - "precision_at_10": 0.02506, - "precision_at_100": 0.01596, - "precision_at_1000": 0.00403, - "precision_at_20": 0.02346, - "precision_at_3": 0.02384, - "precision_at_5": 0.02445, - "recall_at_1": 0.00363, - "recall_at_10": 0.04202, - "recall_at_100": 0.19135, - "recall_at_1000": 0.45742, - "recall_at_20": 0.07132, - "recall_at_3": 0.01285, - "recall_at_5": 0.02134 - } - ] - }, - "task_name": "InfoSeekIT2ITRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2TRetrieval.json deleted file mode 100644 index 1819e46926..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/InfoSeekIT2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "d4f4606f7a42bbf311c2957419ef3734fe81c47f", - "evaluation_time": 556.8317360877991, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.00185, - "cv_recall_at_10": 0.00742, - "cv_recall_at_100": 0.02994, - "cv_recall_at_1000": 0.08938, - "cv_recall_at_20": 0.01263, - "cv_recall_at_3": 0.00344, - "cv_recall_at_5": 0.00468, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.00178, - "map_at_1": 0.00031, - "map_at_10": 0.00101, - "map_at_100": 0.00121, - "map_at_1000": 0.00136, - "map_at_20": 0.00108, - "map_at_3": 0.00072, - "map_at_5": 0.0009, - "mrr_at_1": 0.0018546321646206836, - "mrr_at_10": 0.0031619866292656195, - "mrr_at_100": 0.0038811488094678136, - "mrr_at_1000": 0.004054925968264824, - "mrr_at_20": 0.0035053863113972037, - "mrr_at_3": 0.002546439400629986, - "mrr_at_5": 0.0028246342253230882, - "nauc_cv_recall_at_1000_diff1": 0.016795257748445515, - "nauc_cv_recall_at_1000_max": -0.11000237649698653, - "nauc_cv_recall_at_1000_std": -0.02941444476048924, - "nauc_cv_recall_at_100_diff1": 0.02698413117107532, - "nauc_cv_recall_at_100_max": -0.17015023428711334, - "nauc_cv_recall_at_100_std": -0.07555477868108984, - "nauc_cv_recall_at_10_diff1": -0.08516873412427065, - "nauc_cv_recall_at_10_max": -0.34819048962459037, - "nauc_cv_recall_at_10_std": -0.20008171938905003, - "nauc_cv_recall_at_1_diff1": -0.22629056772533634, - "nauc_cv_recall_at_1_max": -0.4457500290615541, - "nauc_cv_recall_at_1_std": -0.2617919564222382, - "nauc_cv_recall_at_20_diff1": -0.009662351956441516, - "nauc_cv_recall_at_20_max": -0.3111702270408559, - "nauc_cv_recall_at_20_std": -0.2084163850625874, - "nauc_cv_recall_at_3_diff1": -0.0562154615947782, - "nauc_cv_recall_at_3_max": -0.3912950585084963, - "nauc_cv_recall_at_3_std": -0.25332095011701883, - "nauc_cv_recall_at_5_diff1": -0.05391581003426526, - "nauc_cv_recall_at_5_max": -0.3124748627958558, - "nauc_cv_recall_at_5_std": -0.19528751853375279, - "nauc_map_at_1000_diff1": -0.010046715926208967, - "nauc_map_at_1000_max": -0.3727128923537285, - "nauc_map_at_1000_std": -0.10214863303623575, - "nauc_map_at_100_diff1": -0.011514482774858219, - "nauc_map_at_100_max": -0.39681497563871815, - "nauc_map_at_100_std": -0.11134665897764658, - "nauc_map_at_10_diff1": -0.044739667147889156, - "nauc_map_at_10_max": -0.4372186067013952, - "nauc_map_at_10_std": -0.0916840677417323, - "nauc_map_at_1_diff1": -0.1417401389289008, - "nauc_map_at_1_max": -0.4509225939541273, - "nauc_map_at_1_std": -0.1417396365832301, - "nauc_map_at_20_diff1": -0.025195053611041466, - "nauc_map_at_20_max": -0.4215612727662873, - "nauc_map_at_20_std": -0.1051435759810444, - "nauc_map_at_3_diff1": -0.11893535191042216, - "nauc_map_at_3_max": -0.4712135550112047, - "nauc_map_at_3_std": -0.0842650004835489, - "nauc_map_at_5_diff1": -0.059516500712779793, - "nauc_map_at_5_max": -0.4302726508985725, - "nauc_map_at_5_std": -0.07873538453894163, - "nauc_mrr_at_1000_diff1": -0.07607485688565918, - "nauc_mrr_at_1000_max": -0.3316597602298558, - "nauc_mrr_at_1000_std": -0.19792342004324956, - "nauc_mrr_at_100_diff1": -0.08219089640081807, - "nauc_mrr_at_100_max": -0.34407366018720503, - "nauc_mrr_at_100_std": -0.20666808516931123, - "nauc_mrr_at_10_diff1": -0.12069870481367113, - "nauc_mrr_at_10_max": -0.3800294117787652, - "nauc_mrr_at_10_std": -0.22619327527291744, - "nauc_mrr_at_1_diff1": -0.22629056772533634, - "nauc_mrr_at_1_max": -0.4457500290615541, - "nauc_mrr_at_1_std": -0.2617919564222382, - "nauc_mrr_at_20_diff1": -0.09926861453018676, - "nauc_mrr_at_20_max": -0.36925466427380327, - "nauc_mrr_at_20_std": -0.225543528899351, - "nauc_mrr_at_3_diff1": -0.12586977633057814, - "nauc_mrr_at_3_max": -0.4071395819966693, - "nauc_mrr_at_3_std": -0.2501530490148159, - "nauc_mrr_at_5_diff1": -0.11766245961586667, - "nauc_mrr_at_5_max": -0.37517494900469095, - "nauc_mrr_at_5_std": -0.22731118499071057, - "nauc_ndcg_at_1000_diff1": 0.02009277943675779, - "nauc_ndcg_at_1000_max": -0.18904948738068383, - "nauc_ndcg_at_1000_std": -0.03387648550118827, - "nauc_ndcg_at_100_diff1": 0.026232139318861668, - "nauc_ndcg_at_100_max": -0.27860403299468667, - "nauc_ndcg_at_100_std": -0.11772519727613279, - "nauc_ndcg_at_10_diff1": -0.04947234436473754, - "nauc_ndcg_at_10_max": -0.40364486097929114, - "nauc_ndcg_at_10_std": -0.1533786803793439, - "nauc_ndcg_at_1_diff1": -0.22629056772533634, - "nauc_ndcg_at_1_max": -0.4457500290615541, - "nauc_ndcg_at_1_std": -0.2617919564222382, - "nauc_ndcg_at_20_diff1": -0.00461252186000012, - "nauc_ndcg_at_20_max": -0.36649311208018615, - "nauc_ndcg_at_20_std": -0.16353687300636874, - "nauc_ndcg_at_3_diff1": -0.11267773936384334, - "nauc_ndcg_at_3_max": -0.42899411497666556, - "nauc_ndcg_at_3_std": -0.19653902566677903, - "nauc_ndcg_at_5_diff1": -0.06838328486298777, - "nauc_ndcg_at_5_max": -0.38111958871878415, - "nauc_ndcg_at_5_std": -0.1571783232595063, - "nauc_precision_at_1000_diff1": -0.03249085084774534, - "nauc_precision_at_1000_max": -0.16446381467440538, - "nauc_precision_at_1000_std": -0.10709319311891546, - "nauc_precision_at_100_diff1": -0.02883688252039862, - "nauc_precision_at_100_max": -0.2305183260721622, - "nauc_precision_at_100_std": -0.15283039340337035, - "nauc_precision_at_10_diff1": -0.06352844247941265, - "nauc_precision_at_10_max": -0.4007107530138077, - "nauc_precision_at_10_std": -0.17770052560462707, - "nauc_precision_at_1_diff1": -0.22629056772533634, - "nauc_precision_at_1_max": -0.4457500290615541, - "nauc_precision_at_1_std": -0.2617919564222382, - "nauc_precision_at_20_diff1": -0.02366990094318863, - "nauc_precision_at_20_max": -0.3483225640082979, - "nauc_precision_at_20_std": -0.21523236052685726, - "nauc_precision_at_3_diff1": -0.07889917386247514, - "nauc_precision_at_3_max": -0.43053862302789736, - "nauc_precision_at_3_std": -0.1807336392649264, - "nauc_precision_at_5_diff1": -0.049298631452126984, - "nauc_precision_at_5_max": -0.3765150851533156, - "nauc_precision_at_5_std": -0.14011440498471459, - "nauc_recall_at_1000_diff1": 0.052127753913729535, - "nauc_recall_at_1000_max": -0.13241816139211318, - "nauc_recall_at_1000_std": 0.034748550997898996, - "nauc_recall_at_100_diff1": 0.12778745661018148, - "nauc_recall_at_100_max": -0.1777150772680246, - "nauc_recall_at_100_std": -0.07989548943143837, - "nauc_recall_at_10_diff1": 0.026765977685626496, - "nauc_recall_at_10_max": -0.3841766539530331, - "nauc_recall_at_10_std": -0.12630049632674936, - "nauc_recall_at_1_diff1": -0.1417401389289008, - "nauc_recall_at_1_max": -0.4509225939541273, - "nauc_recall_at_1_std": -0.1417396365832301, - "nauc_recall_at_20_diff1": 0.10846163997925674, - "nauc_recall_at_20_max": -0.309999991338878, - "nauc_recall_at_20_std": -0.1682396965569195, - "nauc_recall_at_3_diff1": -0.0712472400268593, - "nauc_recall_at_3_max": -0.4618660243341875, - "nauc_recall_at_3_std": -0.10049121379738826, - "nauc_recall_at_5_diff1": 0.03522780744618103, - "nauc_recall_at_5_max": -0.34111771775889277, - "nauc_recall_at_5_std": -0.07960384930087543, - "ndcg_at_1": 0.00185, - "ndcg_at_10": 0.00178, - "ndcg_at_100": 0.00335, - "ndcg_at_1000": 0.00812, - "ndcg_at_20": 0.0021, - "ndcg_at_3": 0.00169, - "ndcg_at_5": 0.00165, - "precision_at_1": 0.00185, - "precision_at_10": 0.0011, - "precision_at_100": 0.00056, - "precision_at_1000": 0.00026, - "precision_at_20": 0.00091, - "precision_at_3": 0.00165, - "precision_at_5": 0.0014, - "recall_at_1": 0.00031, - "recall_at_10": 0.00183, - "recall_at_100": 0.00666, - "recall_at_1000": 0.02982, - "recall_at_20": 0.00261, - "recall_at_3": 0.00086, - "recall_at_5": 0.00127 - } - ] - }, - "task_name": "InfoSeekIT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNIST.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNIST.json deleted file mode 100644 index 8617dc7bad..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 69.79996991157532, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.89482, - "f1": 0.8938189564908757, - "f1_weighted": 0.8946237575172749, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89482, - "scores_per_experiment": [ - { - "accuracy": 0.8937, - "f1": 0.8927176424361276, - "f1_weighted": 0.893609834767085 - }, - { - "accuracy": 0.9019, - "f1": 0.9009269794036706, - "f1_weighted": 0.9015503965412863 - }, - { - "accuracy": 0.8788, - "f1": 0.878248230427133, - "f1_weighted": 0.8789836029821599 - }, - { - "accuracy": 0.8807, - "f1": 0.8791930140887333, - "f1_weighted": 0.8803275669342271 - }, - { - "accuracy": 0.919, - "f1": 0.9180089160987143, - "f1_weighted": 0.918647386361616 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNISTZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNISTZeroShot.json deleted file mode 100644 index 46d3432848..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MNISTZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 41.33473753929138, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.4899, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4899 - } - ] - }, - "task_name": "MNISTZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json deleted file mode 100644 index d71a300c97..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", - "evaluation_time": 31.064056158065796, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.5016, - "cv_recall_at_10": 0.8362, - "cv_recall_at_100": 0.9864, - "cv_recall_at_1000": 0.9998, - "cv_recall_at_20": 0.9078, - "cv_recall_at_3": 0.6744, - "cv_recall_at_5": 0.747, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.44382, - "map_at_1": 0.10038, - "map_at_10": 0.33055, - "map_at_100": 0.3751, - "map_at_1000": 0.37878, - "map_at_20": 0.35503, - "map_at_3": 0.22144, - "map_at_5": 0.28304, - "mrr_at_1": 0.5016, - "mrr_at_10": 0.6070182539682555, - "mrr_at_100": 0.6142642221262432, - "mrr_at_1000": 0.6143593027903639, - "mrr_at_20": 0.6120301861073234, - "mrr_at_3": 0.5781000000000018, - "mrr_at_5": 0.5949800000000011, - "nauc_cv_recall_at_1000_diff1": 1.0, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": 0.4323474487834336, - "nauc_cv_recall_at_100_max": 0.3997912890646433, - "nauc_cv_recall_at_100_std": -0.1196037238424793, - "nauc_cv_recall_at_10_diff1": 0.37501696719115774, - "nauc_cv_recall_at_10_max": 0.3127110388245365, - "nauc_cv_recall_at_10_std": -0.3305046673289374, - "nauc_cv_recall_at_1_diff1": 0.4269635838909405, - "nauc_cv_recall_at_1_max": 0.3359596131075358, - "nauc_cv_recall_at_1_std": -0.29597124038134076, - "nauc_cv_recall_at_20_diff1": 0.3767446240969275, - "nauc_cv_recall_at_20_max": 0.33553088625182514, - "nauc_cv_recall_at_20_std": -0.37383008156263287, - "nauc_cv_recall_at_3_diff1": 0.3859951165456735, - "nauc_cv_recall_at_3_max": 0.3344155089983177, - "nauc_cv_recall_at_3_std": -0.33527359778317994, - "nauc_cv_recall_at_5_diff1": 0.3725036329450592, - "nauc_cv_recall_at_5_max": 0.3198504947231606, - "nauc_cv_recall_at_5_std": -0.35528274781245933, - "nauc_map_at_1000_diff1": 0.28351782466117825, - "nauc_map_at_1000_max": 0.3661635018482055, - "nauc_map_at_1000_std": -0.2823328432610589, - "nauc_map_at_100_diff1": 0.2836697696803503, - "nauc_map_at_100_max": 0.3658508272945635, - "nauc_map_at_100_std": -0.2831750575480581, - "nauc_map_at_10_diff1": 0.2751925036810592, - "nauc_map_at_10_max": 0.362845180918207, - "nauc_map_at_10_std": -0.2796511485424725, - "nauc_map_at_1_diff1": 0.42704782267482194, - "nauc_map_at_1_max": 0.3354423674380913, - "nauc_map_at_1_std": -0.295768978541717, - "nauc_map_at_20_diff1": 0.27970330738728366, - "nauc_map_at_20_max": 0.3631059439197211, - "nauc_map_at_20_std": -0.28526313274442944, - "nauc_map_at_3_diff1": 0.30553729889530606, - "nauc_map_at_3_max": 0.3491175423733284, - "nauc_map_at_3_std": -0.2785635560793982, - "nauc_map_at_5_diff1": 0.28121998853700064, - "nauc_map_at_5_max": 0.3642230800059721, - "nauc_map_at_5_std": -0.27715004713761787, - "nauc_mrr_at_1000_diff1": 0.41157652946293, - "nauc_mrr_at_1000_max": 0.33317963379765403, - "nauc_mrr_at_1000_std": -0.3099069801037426, - "nauc_mrr_at_100_diff1": 0.4115850028024918, - "nauc_mrr_at_100_max": 0.33318274824006155, - "nauc_mrr_at_100_std": -0.30988539187877806, - "nauc_mrr_at_10_diff1": 0.41067364563442044, - "nauc_mrr_at_10_max": 0.33216357042070327, - "nauc_mrr_at_10_std": -0.31053927104989015, - "nauc_mrr_at_1_diff1": 0.4269635838909405, - "nauc_mrr_at_1_max": 0.3359596131075358, - "nauc_mrr_at_1_std": -0.29597124038134076, - "nauc_mrr_at_20_diff1": 0.4111282203637099, - "nauc_mrr_at_20_max": 0.3329553429417173, - "nauc_mrr_at_20_std": -0.3108973352234124, - "nauc_mrr_at_3_diff1": 0.4106584396861111, - "nauc_mrr_at_3_max": 0.33525947044408083, - "nauc_mrr_at_3_std": -0.3108151055129786, - "nauc_mrr_at_5_diff1": 0.40928356526564613, - "nauc_mrr_at_5_max": 0.33241615563794474, - "nauc_mrr_at_5_std": -0.31330790888317744, - "nauc_ndcg_at_1000_diff1": 0.32243874847713305, - "nauc_ndcg_at_1000_max": 0.3624742162317057, - "nauc_ndcg_at_1000_std": -0.2869514637432924, - "nauc_ndcg_at_100_diff1": 0.32058430384405767, - "nauc_ndcg_at_100_max": 0.361149409268888, - "nauc_ndcg_at_100_std": -0.28782778350008814, - "nauc_ndcg_at_10_diff1": 0.300464637566366, - "nauc_ndcg_at_10_max": 0.35240653924539134, - "nauc_ndcg_at_10_std": -0.2926815629512613, - "nauc_ndcg_at_1_diff1": 0.4269635838909405, - "nauc_ndcg_at_1_max": 0.3359596131075358, - "nauc_ndcg_at_1_std": -0.29597124038134076, - "nauc_ndcg_at_20_diff1": 0.3069332270693507, - "nauc_ndcg_at_20_max": 0.35335596836456534, - "nauc_ndcg_at_20_std": -0.3019191697870882, - "nauc_ndcg_at_3_diff1": 0.32651722328846816, - "nauc_ndcg_at_3_max": 0.3454350647220891, - "nauc_ndcg_at_3_std": -0.29031057373881475, - "nauc_ndcg_at_5_diff1": 0.3080772824134777, - "nauc_ndcg_at_5_max": 0.35452785631989464, - "nauc_ndcg_at_5_std": -0.2931716669816145, - "nauc_precision_at_1000_diff1": 0.09001395234899717, - "nauc_precision_at_1000_max": 0.43535411629797816, - "nauc_precision_at_1000_std": 0.5700954653090525, - "nauc_precision_at_100_diff1": 0.247818689232099, - "nauc_precision_at_100_max": 0.3133668190372178, - "nauc_precision_at_100_std": -0.17862978392429757, - "nauc_precision_at_10_diff1": 0.24140633693516517, - "nauc_precision_at_10_max": 0.331551710651199, - "nauc_precision_at_10_std": -0.27078528866090024, - "nauc_precision_at_1_diff1": 0.4269635838909405, - "nauc_precision_at_1_max": 0.3359596131075358, - "nauc_precision_at_1_std": -0.29597124038134076, - "nauc_precision_at_20_diff1": 0.2415524145841425, - "nauc_precision_at_20_max": 0.3178040334889773, - "nauc_precision_at_20_std": -0.27782664848315153, - "nauc_precision_at_3_diff1": 0.2911132728042898, - "nauc_precision_at_3_max": 0.3430986077114997, - "nauc_precision_at_3_std": -0.28545786591563965, - "nauc_precision_at_5_diff1": 0.2627202382806198, - "nauc_precision_at_5_max": 0.3493999886119583, - "nauc_precision_at_5_std": -0.28416900168712844, - "nauc_recall_at_1000_diff1": 0.11234864316098189, - "nauc_recall_at_1000_max": 0.4494890548812042, - "nauc_recall_at_1000_std": 0.5881259911372041, - "nauc_recall_at_100_diff1": 0.24940335817964793, - "nauc_recall_at_100_max": 0.3135186357535106, - "nauc_recall_at_100_std": -0.17935112790907157, - "nauc_recall_at_10_diff1": 0.24176763561476253, - "nauc_recall_at_10_max": 0.3313976857716855, - "nauc_recall_at_10_std": -0.27091127395840575, - "nauc_recall_at_1_diff1": 0.42704782267482194, - "nauc_recall_at_1_max": 0.3354423674380913, - "nauc_recall_at_1_std": -0.295768978541717, - "nauc_recall_at_20_diff1": 0.24212418741953098, - "nauc_recall_at_20_max": 0.3177591546517272, - "nauc_recall_at_20_std": -0.277952723894512, - "nauc_recall_at_3_diff1": 0.29145311226379, - "nauc_recall_at_3_max": 0.34287227460196085, - "nauc_recall_at_3_std": -0.2854556141136081, - "nauc_recall_at_5_diff1": 0.26299935501097627, - "nauc_recall_at_5_max": 0.3492528893515414, - "nauc_recall_at_5_std": -0.2842098794048584, - "ndcg_at_1": 0.5016, - "ndcg_at_10": 0.44382, - "ndcg_at_100": 0.56713, - "ndcg_at_1000": 0.60567, - "ndcg_at_20": 0.49235, - "ndcg_at_3": 0.43251, - "ndcg_at_5": 0.37825, - "precision_at_1": 0.5016, - "precision_at_10": 0.23016, - "precision_at_100": 0.04069, - "precision_at_1000": 0.00495, - "precision_at_20": 0.14332, - "precision_at_3": 0.41153, - "precision_at_5": 0.3408, - "recall_at_1": 0.10038, - "recall_at_10": 0.46052, - "recall_at_100": 0.81409, - "recall_at_1000": 0.98992, - "recall_at_20": 0.57355, - "recall_at_3": 0.24704, - "recall_at_5": 0.34096 - } - ] - }, - "task_name": "MSCOCOI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json deleted file mode 100644 index d3eb8cb7bb..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MSCOCOT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", - "evaluation_time": 64.15870261192322, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.3219, - "cv_recall_at_10": 0.69043, - "cv_recall_at_100": 0.96699, - "cv_recall_at_1000": 0.99802, - "cv_recall_at_20": 0.78975, - "cv_recall_at_3": 0.49833, - "cv_recall_at_5": 0.57894, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.49354, - "map_at_1": 0.32148, - "map_at_10": 0.4321, - "map_at_100": 0.44382, - "map_at_1000": 0.44402, - "map_at_20": 0.43906, - "map_at_3": 0.39864, - "map_at_5": 0.41705, - "mrr_at_1": 0.3218993107340078, - "mrr_at_10": 0.43262941572023605, - "mrr_at_100": 0.44425445210134723, - "mrr_at_1000": 0.44445143330526093, - "mrr_at_20": 0.43955600062406863, - "mrr_at_3": 0.3991898101495421, - "mrr_at_5": 0.41756015961949616, - "nauc_cv_recall_at_1000_diff1": 0.6908755779988776, - "nauc_cv_recall_at_1000_max": 0.8673002265484557, - "nauc_cv_recall_at_1000_std": 0.9557913979496387, - "nauc_cv_recall_at_100_diff1": 0.3857607701244185, - "nauc_cv_recall_at_100_max": 0.515578931619228, - "nauc_cv_recall_at_100_std": 0.26327215392208675, - "nauc_cv_recall_at_10_diff1": 0.4056733846252647, - "nauc_cv_recall_at_10_max": 0.2300820556651551, - "nauc_cv_recall_at_10_std": -0.24485753427069776, - "nauc_cv_recall_at_1_diff1": 0.5511594051722437, - "nauc_cv_recall_at_1_max": 0.25830642727292813, - "nauc_cv_recall_at_1_std": -0.21261765560734627, - "nauc_cv_recall_at_20_diff1": 0.4094495789026411, - "nauc_cv_recall_at_20_max": 0.24534623596758162, - "nauc_cv_recall_at_20_std": -0.23697708049579708, - "nauc_cv_recall_at_3_diff1": 0.43694665288793116, - "nauc_cv_recall_at_3_max": 0.2252768452995081, - "nauc_cv_recall_at_3_std": -0.2370109213521609, - "nauc_cv_recall_at_5_diff1": 0.4117629943006741, - "nauc_cv_recall_at_5_max": 0.21644561968880394, - "nauc_cv_recall_at_5_std": -0.24928700760880004, - "nauc_map_at_1000_diff1": 0.49930616061028704, - "nauc_map_at_1000_max": 0.24685433707080706, - "nauc_map_at_1000_std": -0.22328503812884298, - "nauc_map_at_100_diff1": 0.4992272277806359, - "nauc_map_at_100_max": 0.24697921089776936, - "nauc_map_at_100_std": -0.22308864169984505, - "nauc_map_at_10_diff1": 0.49690012441978665, - "nauc_map_at_10_max": 0.24536246677307505, - "nauc_map_at_10_std": -0.22543751253448477, - "nauc_map_at_1_diff1": 0.5515444068560961, - "nauc_map_at_1_max": 0.25822323169218364, - "nauc_map_at_1_std": -0.21358954191507926, - "nauc_map_at_20_diff1": 0.49827147970901364, - "nauc_map_at_20_max": 0.24612833477596713, - "nauc_map_at_20_std": -0.2247932396699695, - "nauc_map_at_3_diff1": 0.5000595542031279, - "nauc_map_at_3_max": 0.24476462058347448, - "nauc_map_at_3_std": -0.2240119884784554, - "nauc_map_at_5_diff1": 0.49580203095577297, - "nauc_map_at_5_max": 0.24334951100406244, - "nauc_map_at_5_std": -0.2262968722182906, - "nauc_mrr_at_1000_diff1": 0.499078624321733, - "nauc_mrr_at_1000_max": 0.24705331263803362, - "nauc_mrr_at_1000_std": -0.22227497999784016, - "nauc_mrr_at_100_diff1": 0.49900036301502043, - "nauc_mrr_at_100_max": 0.2471775251171831, - "nauc_mrr_at_100_std": -0.2220739366647331, - "nauc_mrr_at_10_diff1": 0.4966506098098704, - "nauc_mrr_at_10_max": 0.2456726135596894, - "nauc_mrr_at_10_std": -0.22420370232311077, - "nauc_mrr_at_1_diff1": 0.5511594051722437, - "nauc_mrr_at_1_max": 0.25830642727292813, - "nauc_mrr_at_1_std": -0.21261765560734627, - "nauc_mrr_at_20_diff1": 0.49803403767882, - "nauc_mrr_at_20_max": 0.246377103277424, - "nauc_mrr_at_20_std": -0.22364539619421414, - "nauc_mrr_at_3_diff1": 0.49979377716896667, - "nauc_mrr_at_3_max": 0.24503825278773872, - "nauc_mrr_at_3_std": -0.22279957760963745, - "nauc_mrr_at_5_diff1": 0.4955529709824597, - "nauc_mrr_at_5_max": 0.24364677021673845, - "nauc_mrr_at_5_std": -0.22507060373310997, - "nauc_ndcg_at_1000_diff1": 0.49112188758844744, - "nauc_ndcg_at_1000_max": 0.24757056662063803, - "nauc_ndcg_at_1000_std": -0.22064747313445351, - "nauc_ndcg_at_100_diff1": 0.48918551868023125, - "nauc_ndcg_at_100_max": 0.2512228254623345, - "nauc_ndcg_at_100_std": -0.2143001857039737, - "nauc_ndcg_at_10_diff1": 0.477453906201686, - "nauc_ndcg_at_10_max": 0.24164298975701132, - "nauc_ndcg_at_10_std": -0.22983343834132502, - "nauc_ndcg_at_1_diff1": 0.5511594051722437, - "nauc_ndcg_at_1_max": 0.25830642727292813, - "nauc_ndcg_at_1_std": -0.21261765560734627, - "nauc_ndcg_at_20_diff1": 0.48210010047151386, - "nauc_ndcg_at_20_max": 0.24473128548110354, - "nauc_ndcg_at_20_std": -0.2275562152465016, - "nauc_ndcg_at_3_diff1": 0.48380834514144605, - "nauc_ndcg_at_3_max": 0.23983301673015683, - "nauc_ndcg_at_3_std": -0.22727992582610385, - "nauc_ndcg_at_5_diff1": 0.47554641186612, - "nauc_ndcg_at_5_max": 0.2368921319067536, - "nauc_ndcg_at_5_std": -0.2318280497980852, - "nauc_precision_at_1000_diff1": -0.08156277108785304, - "nauc_precision_at_1000_max": 0.18948123029517247, - "nauc_precision_at_1000_std": 0.5497574779729605, - "nauc_precision_at_100_diff1": 0.2394769875392392, - "nauc_precision_at_100_max": 0.41698415214593487, - "nauc_precision_at_100_std": 0.328556989104763, - "nauc_precision_at_10_diff1": 0.40269735640645404, - "nauc_precision_at_10_max": 0.22952547239545323, - "nauc_precision_at_10_std": -0.24130163140327587, - "nauc_precision_at_1_diff1": 0.5511594051722437, - "nauc_precision_at_1_max": 0.25830642727292813, - "nauc_precision_at_1_std": -0.21261765560734627, - "nauc_precision_at_20_diff1": 0.3999531202089017, - "nauc_precision_at_20_max": 0.24431433894217264, - "nauc_precision_at_20_std": -0.22380219125785444, - "nauc_precision_at_3_diff1": 0.4364725818613955, - "nauc_precision_at_3_max": 0.22523007776906476, - "nauc_precision_at_3_std": -0.23673214595260939, - "nauc_precision_at_5_diff1": 0.41023560618565763, - "nauc_precision_at_5_max": 0.21622732648955392, - "nauc_precision_at_5_std": -0.24767741784863434, - "nauc_recall_at_1000_diff1": 0.6939980469079802, - "nauc_recall_at_1000_max": 0.8686406283004262, - "nauc_recall_at_1000_std": 0.9562379494854356, - "nauc_recall_at_100_diff1": 0.38475736215115286, - "nauc_recall_at_100_max": 0.5134058933554025, - "nauc_recall_at_100_std": 0.2577215977974941, - "nauc_recall_at_10_diff1": 0.40577007213243543, - "nauc_recall_at_10_max": 0.22880905921602546, - "nauc_recall_at_10_std": -0.24719647545185264, - "nauc_recall_at_1_diff1": 0.5515444068560961, - "nauc_recall_at_1_max": 0.25822323169218364, - "nauc_recall_at_1_std": -0.21358954191507926, - "nauc_recall_at_20_diff1": 0.4096257462985691, - "nauc_recall_at_20_max": 0.24395998446094397, - "nauc_recall_at_20_std": -0.23983127974593152, - "nauc_recall_at_3_diff1": 0.4371184473204451, - "nauc_recall_at_3_max": 0.2247648394958751, - "nauc_recall_at_3_std": -0.23841868685719025, - "nauc_recall_at_5_diff1": 0.41182244342055924, - "nauc_recall_at_5_max": 0.21563596886073882, - "nauc_recall_at_5_std": -0.2510594326845239, - "ndcg_at_1": 0.3219, - "ndcg_at_10": 0.49354, - "ndcg_at_100": 0.5522, - "ndcg_at_1000": 0.55645, - "ndcg_at_20": 0.51874, - "ndcg_at_3": 0.4242, - "ndcg_at_5": 0.45735, - "precision_at_1": 0.3219, - "precision_at_10": 0.06912, - "precision_at_100": 0.00973, - "precision_at_1000": 0.00101, - "precision_at_20": 0.03958, - "precision_at_3": 0.16615, - "precision_at_5": 0.11589, - "recall_at_1": 0.32148, - "recall_at_10": 0.68941, - "recall_at_100": 0.96663, - "recall_at_1000": 0.998, - "recall_at_20": 0.78867, - "recall_at_3": 0.49757, - "recall_at_5": 0.57818 - } - ] - }, - "task_name": "MSCOCOT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionI2TRetrieval.json deleted file mode 100644 index 6f097133d0..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", - "evaluation_time": 9.379658937454224, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.54089, - "cv_recall_at_10": 0.76327, - "cv_recall_at_100": 0.90961, - "cv_recall_at_1000": 0.97561, - "cv_recall_at_20": 0.81492, - "cv_recall_at_3": 0.66858, - "cv_recall_at_5": 0.70014, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.64846, - "map_at_1": 0.53945, - "map_at_10": 0.61219, - "map_at_100": 0.61816, - "map_at_1000": 0.61842, - "map_at_20": 0.61573, - "map_at_3": 0.5966, - "map_at_5": 0.60385, - "mrr_at_1": 0.5408895265423243, - "mrr_at_10": 0.6129033727312059, - "mrr_at_100": 0.6188781791949352, - "mrr_at_1000": 0.6191415311661694, - "mrr_at_20": 0.6164480169192975, - "mrr_at_3": 0.5973218555714968, - "mrr_at_5": 0.6045671927307505, - "nauc_cv_recall_at_1000_diff1": 0.6031068526863032, - "nauc_cv_recall_at_1000_max": 0.6187947052677972, - "nauc_cv_recall_at_1000_std": 0.13614884941799502, - "nauc_cv_recall_at_100_diff1": 0.6384413822343304, - "nauc_cv_recall_at_100_max": 0.6568440212607971, - "nauc_cv_recall_at_100_std": 0.36389752289108457, - "nauc_cv_recall_at_10_diff1": 0.6436032470889269, - "nauc_cv_recall_at_10_max": 0.5770073527151407, - "nauc_cv_recall_at_10_std": 0.1248391391939967, - "nauc_cv_recall_at_1_diff1": 0.7550158265678641, - "nauc_cv_recall_at_1_max": 0.5433476926735126, - "nauc_cv_recall_at_1_std": 0.08333119803942535, - "nauc_cv_recall_at_20_diff1": 0.6545130104125019, - "nauc_cv_recall_at_20_max": 0.626723137941904, - "nauc_cv_recall_at_20_std": 0.26198168043372594, - "nauc_cv_recall_at_3_diff1": 0.6830756998324176, - "nauc_cv_recall_at_3_max": 0.602332850939447, - "nauc_cv_recall_at_3_std": 0.14321032322791108, - "nauc_cv_recall_at_5_diff1": 0.6562142711894405, - "nauc_cv_recall_at_5_max": 0.5739787707563608, - "nauc_cv_recall_at_5_std": 0.12240833670188953, - "nauc_map_at_1000_diff1": 0.7273224554410895, - "nauc_map_at_1000_max": 0.5611946300973509, - "nauc_map_at_1000_std": 0.10422567634452316, - "nauc_map_at_100_diff1": 0.7272509146321907, - "nauc_map_at_100_max": 0.5612908433321292, - "nauc_map_at_100_std": 0.10454572772341748, - "nauc_map_at_10_diff1": 0.725549268371285, - "nauc_map_at_10_max": 0.5592550980691888, - "nauc_map_at_10_std": 0.0990385207650767, - "nauc_map_at_1_diff1": 0.758129112404582, - "nauc_map_at_1_max": 0.537597678667553, - "nauc_map_at_1_std": 0.08080106485075698, - "nauc_map_at_20_diff1": 0.726514594912638, - "nauc_map_at_20_max": 0.5609091488732393, - "nauc_map_at_20_std": 0.10421116489530967, - "nauc_map_at_3_diff1": 0.729828848291311, - "nauc_map_at_3_max": 0.5651513147787425, - "nauc_map_at_3_std": 0.10316360192499197, - "nauc_map_at_5_diff1": 0.7253689641307103, - "nauc_map_at_5_max": 0.5590662677071627, - "nauc_map_at_5_std": 0.09856405589506938, - "nauc_mrr_at_1000_diff1": 0.7255248516916601, - "nauc_mrr_at_1000_max": 0.5644856932022566, - "nauc_mrr_at_1000_std": 0.10568520167017303, - "nauc_mrr_at_100_diff1": 0.7254548437423526, - "nauc_mrr_at_100_max": 0.5645790694557368, - "nauc_mrr_at_100_std": 0.10600420119309457, - "nauc_mrr_at_10_diff1": 0.7237804426229136, - "nauc_mrr_at_10_max": 0.5624890366615415, - "nauc_mrr_at_10_std": 0.10046897577029043, - "nauc_mrr_at_1_diff1": 0.7550158265678641, - "nauc_mrr_at_1_max": 0.5433476926735126, - "nauc_mrr_at_1_std": 0.08333119803942535, - "nauc_mrr_at_20_diff1": 0.7247307286147782, - "nauc_mrr_at_20_max": 0.5641735376688276, - "nauc_mrr_at_20_std": 0.10565892278323516, - "nauc_mrr_at_3_diff1": 0.7281207899682096, - "nauc_mrr_at_3_max": 0.5682855124046605, - "nauc_mrr_at_3_std": 0.10455119493617923, - "nauc_mrr_at_5_diff1": 0.7236347661753463, - "nauc_mrr_at_5_max": 0.5622364615091957, - "nauc_mrr_at_5_std": 0.09996598700114626, - "nauc_ndcg_at_1000_diff1": 0.719295351046238, - "nauc_ndcg_at_1000_max": 0.5672932873311153, - "nauc_ndcg_at_1000_std": 0.11522678717518038, - "nauc_ndcg_at_100_diff1": 0.7170409644450718, - "nauc_ndcg_at_100_max": 0.5705269633295453, - "nauc_ndcg_at_100_std": 0.1259050314954282, - "nauc_ndcg_at_10_diff1": 0.709841562609803, - "nauc_ndcg_at_10_max": 0.5633088061734607, - "nauc_ndcg_at_10_std": 0.10457845445884532, - "nauc_ndcg_at_1_diff1": 0.758129112404582, - "nauc_ndcg_at_1_max": 0.537597678667553, - "nauc_ndcg_at_1_std": 0.08080106485075698, - "nauc_ndcg_at_20_diff1": 0.7137629800374607, - "nauc_ndcg_at_20_max": 0.5706475244876037, - "nauc_ndcg_at_20_std": 0.12599765657967227, - "nauc_ndcg_at_3_diff1": 0.7188796629698468, - "nauc_ndcg_at_3_max": 0.5739922976173544, - "nauc_ndcg_at_3_std": 0.11249256157795794, - "nauc_ndcg_at_5_diff1": 0.7101347667270254, - "nauc_ndcg_at_5_max": 0.5629182818624199, - "nauc_ndcg_at_5_std": 0.10415403042320356, - "nauc_precision_at_1000_diff1": 0.6031068526863047, - "nauc_precision_at_1000_max": 0.6187947052677932, - "nauc_precision_at_1000_std": 0.13614884941798686, - "nauc_precision_at_100_diff1": 0.6384413822343313, - "nauc_precision_at_100_max": 0.6568440212607987, - "nauc_precision_at_100_std": 0.3638975228910859, - "nauc_precision_at_10_diff1": 0.6436032470889286, - "nauc_precision_at_10_max": 0.5770073527151411, - "nauc_precision_at_10_std": 0.12483913919399606, - "nauc_precision_at_1_diff1": 0.758129112404582, - "nauc_precision_at_1_max": 0.537597678667553, - "nauc_precision_at_1_std": 0.08080106485075698, - "nauc_precision_at_20_diff1": 0.6545130104125023, - "nauc_precision_at_20_max": 0.6267231379419045, - "nauc_precision_at_20_std": 0.2619816804337271, - "nauc_precision_at_3_diff1": 0.6830756998324173, - "nauc_precision_at_3_max": 0.6023328509394469, - "nauc_precision_at_3_std": 0.14321032322791036, - "nauc_precision_at_5_diff1": 0.656214271189441, - "nauc_precision_at_5_max": 0.5739787707563612, - "nauc_precision_at_5_std": 0.12240833670189052, - "nauc_recall_at_1000_diff1": 0.6031068526863032, - "nauc_recall_at_1000_max": 0.6187947052677972, - "nauc_recall_at_1000_std": 0.13614884941799502, - "nauc_recall_at_100_diff1": 0.6384413822343304, - "nauc_recall_at_100_max": 0.6568440212607971, - "nauc_recall_at_100_std": 0.36389752289108457, - "nauc_recall_at_10_diff1": 0.6436032470889269, - "nauc_recall_at_10_max": 0.5770073527151407, - "nauc_recall_at_10_std": 0.1248391391939967, - "nauc_recall_at_1_diff1": 0.758129112404582, - "nauc_recall_at_1_max": 0.537597678667553, - "nauc_recall_at_1_std": 0.08080106485075698, - "nauc_recall_at_20_diff1": 0.6545130104125019, - "nauc_recall_at_20_max": 0.626723137941904, - "nauc_recall_at_20_std": 0.26198168043372594, - "nauc_recall_at_3_diff1": 0.6830756998324176, - "nauc_recall_at_3_max": 0.602332850939447, - "nauc_recall_at_3_std": 0.14321032322791108, - "nauc_recall_at_5_diff1": 0.6562142711894405, - "nauc_recall_at_5_max": 0.5739787707563608, - "nauc_recall_at_5_std": 0.12240833670188953, - "ndcg_at_1": 0.53945, - "ndcg_at_10": 0.64846, - "ndcg_at_100": 0.67903, - "ndcg_at_1000": 0.68716, - "ndcg_at_20": 0.66146, - "ndcg_at_3": 0.6151, - "ndcg_at_5": 0.62813, - "precision_at_1": 0.53945, - "precision_at_10": 0.07633, - "precision_at_100": 0.0091, - "precision_at_1000": 0.00098, - "precision_at_20": 0.04075, - "precision_at_3": 0.22286, - "precision_at_5": 0.14003, - "recall_at_1": 0.53945, - "recall_at_10": 0.76327, - "recall_at_100": 0.90961, - "recall_at_1000": 0.97561, - "recall_at_20": 0.81492, - "recall_at_3": 0.66858, - "recall_at_5": 0.70014 - } - ] - }, - "task_name": "MemotionI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionT2IRetrieval.json deleted file mode 100644 index df3428cbb9..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/MemotionT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", - "evaluation_time": 35.66201996803284, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.66571, - "cv_recall_at_10": 0.8594, - "cv_recall_at_100": 0.93831, - "cv_recall_at_1000": 0.97561, - "cv_recall_at_20": 0.89813, - "cv_recall_at_3": 0.77618, - "cv_recall_at_5": 0.80344, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.75862, - "map_at_1": 0.66284, - "map_at_10": 0.72675, - "map_at_100": 0.73055, - "map_at_1000": 0.73072, - "map_at_20": 0.7296, - "map_at_3": 0.71258, - "map_at_5": 0.71911, - "mrr_at_1": 0.6657101865136298, - "mrr_at_10": 0.7285418232333586, - "mrr_at_100": 0.7323483961958512, - "mrr_at_1000": 0.732515458568796, - "mrr_at_20": 0.7313936571835862, - "mrr_at_3": 0.7147297943567668, - "mrr_at_5": 0.7208990913438542, - "nauc_cv_recall_at_1000_diff1": 0.7275510355633449, - "nauc_cv_recall_at_1000_max": 0.6270510775474021, - "nauc_cv_recall_at_1000_std": 0.6977342597863756, - "nauc_cv_recall_at_100_diff1": 0.67409025551175, - "nauc_cv_recall_at_100_max": 0.7187642379876352, - "nauc_cv_recall_at_100_std": 0.7412802975591734, - "nauc_cv_recall_at_10_diff1": 0.7440079572260425, - "nauc_cv_recall_at_10_max": 0.6116559774821306, - "nauc_cv_recall_at_10_std": 0.39207495561469424, - "nauc_cv_recall_at_1_diff1": 0.8221009762961692, - "nauc_cv_recall_at_1_max": 0.5448218971848086, - "nauc_cv_recall_at_1_std": 0.2320609220538416, - "nauc_cv_recall_at_20_diff1": 0.7031663816757069, - "nauc_cv_recall_at_20_max": 0.6291439507988603, - "nauc_cv_recall_at_20_std": 0.5077566395661355, - "nauc_cv_recall_at_3_diff1": 0.7702920446965164, - "nauc_cv_recall_at_3_max": 0.5372157631138539, - "nauc_cv_recall_at_3_std": 0.2549429305075186, - "nauc_cv_recall_at_5_diff1": 0.7650835866602818, - "nauc_cv_recall_at_5_max": 0.5442226414009286, - "nauc_cv_recall_at_5_std": 0.28724135516500227, - "nauc_map_at_1000_diff1": 0.8062791119801845, - "nauc_map_at_1000_max": 0.5506991188965215, - "nauc_map_at_1000_std": 0.25596886109538064, - "nauc_map_at_100_diff1": 0.8061675355941809, - "nauc_map_at_100_max": 0.550882192093261, - "nauc_map_at_100_std": 0.2563613670042926, - "nauc_map_at_10_diff1": 0.8066919819754693, - "nauc_map_at_10_max": 0.5503929884485892, - "nauc_map_at_10_std": 0.2528355504090529, - "nauc_map_at_1_diff1": 0.8276857924244969, - "nauc_map_at_1_max": 0.54640376432564, - "nauc_map_at_1_std": 0.23200436630899549, - "nauc_map_at_20_diff1": 0.8059529978723045, - "nauc_map_at_20_max": 0.5505629223268083, - "nauc_map_at_20_std": 0.25567849411462323, - "nauc_map_at_3_diff1": 0.8069803190393084, - "nauc_map_at_3_max": 0.5415664121870912, - "nauc_map_at_3_std": 0.23722873621401488, - "nauc_map_at_5_diff1": 0.8066155055231402, - "nauc_map_at_5_max": 0.543970624015228, - "nauc_map_at_5_std": 0.2448945089486877, - "nauc_mrr_at_1000_diff1": 0.8023950467283947, - "nauc_mrr_at_1000_max": 0.5512541672454327, - "nauc_mrr_at_1000_std": 0.25907486182158634, - "nauc_mrr_at_100_diff1": 0.8022857703546528, - "nauc_mrr_at_100_max": 0.5514376745671156, - "nauc_mrr_at_100_std": 0.259466912692116, - "nauc_mrr_at_10_diff1": 0.8028693166599788, - "nauc_mrr_at_10_max": 0.5509381510440614, - "nauc_mrr_at_10_std": 0.25587984389286694, - "nauc_mrr_at_1_diff1": 0.8221009762961692, - "nauc_mrr_at_1_max": 0.5448218971848086, - "nauc_mrr_at_1_std": 0.2320609220538416, - "nauc_mrr_at_20_diff1": 0.8020861161859157, - "nauc_mrr_at_20_max": 0.5511146841530047, - "nauc_mrr_at_20_std": 0.2587682482356528, - "nauc_mrr_at_3_diff1": 0.8029147313834037, - "nauc_mrr_at_3_max": 0.5434426114809214, - "nauc_mrr_at_3_std": 0.24285713945064882, - "nauc_mrr_at_5_diff1": 0.8028978569938292, - "nauc_mrr_at_5_max": 0.5444726737717525, - "nauc_mrr_at_5_std": 0.24782017249190733, - "nauc_ndcg_at_1000_diff1": 0.7990426525026079, - "nauc_ndcg_at_1000_max": 0.5568838813048612, - "nauc_ndcg_at_1000_std": 0.2780511118290086, - "nauc_ndcg_at_100_diff1": 0.7952466149552354, - "nauc_ndcg_at_100_max": 0.562318000567148, - "nauc_ndcg_at_100_std": 0.2899863705321415, - "nauc_ndcg_at_10_diff1": 0.7967440189968154, - "nauc_ndcg_at_10_max": 0.5591866658895446, - "nauc_ndcg_at_10_std": 0.2735502422966261, - "nauc_ndcg_at_1_diff1": 0.8276857924244969, - "nauc_ndcg_at_1_max": 0.54640376432564, - "nauc_ndcg_at_1_std": 0.23200436630899549, - "nauc_ndcg_at_20_diff1": 0.7935903921913575, - "nauc_ndcg_at_20_max": 0.5595600940092564, - "nauc_ndcg_at_20_std": 0.2842931614515233, - "nauc_ndcg_at_3_diff1": 0.7992628243453683, - "nauc_ndcg_at_3_max": 0.5391546646696291, - "nauc_ndcg_at_3_std": 0.23825763951943946, - "nauc_ndcg_at_5_diff1": 0.7980409986597947, - "nauc_ndcg_at_5_max": 0.5438585983101174, - "nauc_ndcg_at_5_std": 0.25319042848202133, - "nauc_precision_at_1000_diff1": 0.727551035563347, - "nauc_precision_at_1000_max": 0.6270510775474053, - "nauc_precision_at_1000_std": 0.697734259786382, - "nauc_precision_at_100_diff1": 0.6740902555117545, - "nauc_precision_at_100_max": 0.7187642379876428, - "nauc_precision_at_100_std": 0.7412802975591787, - "nauc_precision_at_10_diff1": 0.7440079572260412, - "nauc_precision_at_10_max": 0.6116559774821304, - "nauc_precision_at_10_std": 0.39207495561469397, - "nauc_precision_at_1_diff1": 0.8276857924244969, - "nauc_precision_at_1_max": 0.54640376432564, - "nauc_precision_at_1_std": 0.23200436630899549, - "nauc_precision_at_20_diff1": 0.7031663816757052, - "nauc_precision_at_20_max": 0.6291439507988585, - "nauc_precision_at_20_std": 0.5077566395661335, - "nauc_precision_at_3_diff1": 0.7724802552153682, - "nauc_precision_at_3_max": 0.5305632143015199, - "nauc_precision_at_3_std": 0.24149371118183455, - "nauc_precision_at_5_diff1": 0.7650835866602826, - "nauc_precision_at_5_max": 0.5442226414009279, - "nauc_precision_at_5_std": 0.28724135516500265, - "nauc_recall_at_1000_diff1": 0.7275510355633449, - "nauc_recall_at_1000_max": 0.6270510775474021, - "nauc_recall_at_1000_std": 0.6977342597863756, - "nauc_recall_at_100_diff1": 0.67409025551175, - "nauc_recall_at_100_max": 0.7187642379876352, - "nauc_recall_at_100_std": 0.7412802975591734, - "nauc_recall_at_10_diff1": 0.7440079572260425, - "nauc_recall_at_10_max": 0.6116559774821306, - "nauc_recall_at_10_std": 0.39207495561469424, - "nauc_recall_at_1_diff1": 0.8276857924244969, - "nauc_recall_at_1_max": 0.54640376432564, - "nauc_recall_at_1_std": 0.23200436630899549, - "nauc_recall_at_20_diff1": 0.7031663816757069, - "nauc_recall_at_20_max": 0.6291439507988603, - "nauc_recall_at_20_std": 0.5077566395661355, - "nauc_recall_at_3_diff1": 0.772480255215368, - "nauc_recall_at_3_max": 0.5305632143015196, - "nauc_recall_at_3_std": 0.24149371118183494, - "nauc_recall_at_5_diff1": 0.7650835866602818, - "nauc_recall_at_5_max": 0.5442226414009286, - "nauc_recall_at_5_std": 0.28724135516500227, - "ndcg_at_1": 0.66284, - "ndcg_at_10": 0.75862, - "ndcg_at_100": 0.77596, - "ndcg_at_1000": 0.78066, - "ndcg_at_20": 0.76862, - "ndcg_at_3": 0.72856, - "ndcg_at_5": 0.74036, - "precision_at_1": 0.66284, - "precision_at_10": 0.08594, - "precision_at_100": 0.00938, - "precision_at_1000": 0.00098, - "precision_at_20": 0.04491, - "precision_at_3": 0.25825, - "precision_at_5": 0.16069, - "recall_at_1": 0.66284, - "recall_at_10": 0.8594, - "recall_at_100": 0.93831, - "recall_at_1000": 0.97561, - "recall_at_20": 0.89813, - "recall_at_3": 0.77475, - "recall_at_5": 0.80344 - } - ] - }, - "task_name": "MemotionT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json deleted file mode 100644 index 0f91c2dce8..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/NIGHTSI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", - "evaluation_time": 125.03077864646912, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.07736, - "cv_recall_at_10": 0.44057, - "cv_recall_at_100": 0.91085, - "cv_recall_at_1000": 0.98396, - "cv_recall_at_20": 0.6283, - "cv_recall_at_3": 0.18962, - "cv_recall_at_5": 0.28255, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.23006, - "map_at_1": 0.07736, - "map_at_10": 0.16639, - "map_at_100": 0.18754, - "map_at_1000": 0.18791, - "map_at_20": 0.17939, - "map_at_3": 0.12469, - "map_at_5": 0.14561, - "mrr_at_1": 0.07735849056603773, - "mrr_at_10": 0.16638982479784364, - "mrr_at_100": 0.18754037945219226, - "mrr_at_1000": 0.18790557532212301, - "mrr_at_20": 0.179393828269821, - "mrr_at_3": 0.12468553459119512, - "mrr_at_5": 0.14560534591194946, - "nauc_cv_recall_at_1000_diff1": -0.03953149887405362, - "nauc_cv_recall_at_1000_max": 0.5072636897896413, - "nauc_cv_recall_at_1000_std": 0.7267946394243926, - "nauc_cv_recall_at_100_diff1": -0.0815659597172199, - "nauc_cv_recall_at_100_max": 0.4456795063704503, - "nauc_cv_recall_at_100_std": 0.5793601391173749, - "nauc_cv_recall_at_10_diff1": 0.00024134988468162648, - "nauc_cv_recall_at_10_max": 0.0714549580777712, - "nauc_cv_recall_at_10_std": 0.001055290057000327, - "nauc_cv_recall_at_1_diff1": 0.10866321569038064, - "nauc_cv_recall_at_1_max": -0.04068530891909402, - "nauc_cv_recall_at_1_std": -0.09376533797535455, - "nauc_cv_recall_at_20_diff1": -0.04957649222390187, - "nauc_cv_recall_at_20_max": 0.1409364325732594, - "nauc_cv_recall_at_20_std": 0.04479103118396537, - "nauc_cv_recall_at_3_diff1": 0.08256318842375683, - "nauc_cv_recall_at_3_max": -0.021108881570062934, - "nauc_cv_recall_at_3_std": -0.03800758467622141, - "nauc_cv_recall_at_5_diff1": 0.07592926310375617, - "nauc_cv_recall_at_5_max": 0.0008620050605076114, - "nauc_cv_recall_at_5_std": -0.026056300607536588, - "nauc_map_at_1000_diff1": 0.07235874903557624, - "nauc_map_at_1000_max": 0.0012495691831918834, - "nauc_map_at_1000_std": -0.04266110409531828, - "nauc_map_at_100_diff1": 0.07206445308000643, - "nauc_map_at_100_max": 0.0019260185802715023, - "nauc_map_at_100_std": -0.04168632470080018, - "nauc_map_at_10_diff1": 0.07403831515427159, - "nauc_map_at_10_max": -0.0046099372581098605, - "nauc_map_at_10_std": -0.049713387866956435, - "nauc_map_at_1_diff1": 0.10866321569038064, - "nauc_map_at_1_max": -0.04068530891909402, - "nauc_map_at_1_std": -0.09376533797535455, - "nauc_map_at_20_diff1": 0.07037909843337932, - "nauc_map_at_20_max": 0.00034557195128515213, - "nauc_map_at_20_std": -0.04702226280108443, - "nauc_map_at_3_diff1": 0.09486061153270883, - "nauc_map_at_3_max": -0.02683999938507012, - "nauc_map_at_3_std": -0.06125207210727061, - "nauc_map_at_5_diff1": 0.09173894865921334, - "nauc_map_at_5_max": -0.019068286969482498, - "nauc_map_at_5_std": -0.055410455100472364, - "nauc_mrr_at_1000_diff1": 0.07235874903557624, - "nauc_mrr_at_1000_max": 0.0012495691831918834, - "nauc_mrr_at_1000_std": -0.04266110409531828, - "nauc_mrr_at_100_diff1": 0.07206445308000643, - "nauc_mrr_at_100_max": 0.0019260185802715023, - "nauc_mrr_at_100_std": -0.04168632470080018, - "nauc_mrr_at_10_diff1": 0.07403831515427159, - "nauc_mrr_at_10_max": -0.0046099372581098605, - "nauc_mrr_at_10_std": -0.049713387866956435, - "nauc_mrr_at_1_diff1": 0.10866321569038064, - "nauc_mrr_at_1_max": -0.04068530891909402, - "nauc_mrr_at_1_std": -0.09376533797535455, - "nauc_mrr_at_20_diff1": 0.07037909843337932, - "nauc_mrr_at_20_max": 0.00034557195128515213, - "nauc_mrr_at_20_std": -0.04702226280108443, - "nauc_mrr_at_3_diff1": 0.09486061153270883, - "nauc_mrr_at_3_max": -0.02683999938507012, - "nauc_mrr_at_3_std": -0.06125207210727061, - "nauc_mrr_at_5_diff1": 0.09173894865921334, - "nauc_mrr_at_5_max": -0.019068286969482498, - "nauc_mrr_at_5_std": -0.055410455100472364, - "nauc_ndcg_at_1000_diff1": 0.056957712662548704, - "nauc_ndcg_at_1000_max": 0.028286697913508196, - "nauc_ndcg_at_1000_std": -0.012334573800565744, - "nauc_ndcg_at_100_diff1": 0.049742644817624616, - "nauc_ndcg_at_100_max": 0.047234136999433886, - "nauc_ndcg_at_100_std": 0.01417643393391879, - "nauc_ndcg_at_10_diff1": 0.050417918199136984, - "nauc_ndcg_at_10_max": 0.01942596425857416, - "nauc_ndcg_at_10_std": -0.03280260244397793, - "nauc_ndcg_at_1_diff1": 0.10866321569038064, - "nauc_ndcg_at_1_max": -0.04068530891909402, - "nauc_ndcg_at_1_std": -0.09376533797535455, - "nauc_ndcg_at_20_diff1": 0.03740429843564625, - "nauc_ndcg_at_20_max": 0.03803214514992617, - "nauc_ndcg_at_20_std": -0.021565488128833982, - "nauc_ndcg_at_3_diff1": 0.09090936216756011, - "nauc_ndcg_at_3_max": -0.02488929764403297, - "nauc_ndcg_at_3_std": -0.053688777047987864, - "nauc_ndcg_at_5_diff1": 0.08633721570987683, - "nauc_ndcg_at_5_max": -0.01238793829885936, - "nauc_ndcg_at_5_std": -0.04527442021920161, - "nauc_precision_at_1000_diff1": -0.03953149887405619, - "nauc_precision_at_1000_max": 0.507263689789647, - "nauc_precision_at_1000_std": 0.7267946394243914, - "nauc_precision_at_100_diff1": -0.08156595971721974, - "nauc_precision_at_100_max": 0.4456795063704518, - "nauc_precision_at_100_std": 0.5793601391173772, - "nauc_precision_at_10_diff1": 0.00024134988468130694, - "nauc_precision_at_10_max": 0.07145495807777137, - "nauc_precision_at_10_std": 0.0010552900570005093, - "nauc_precision_at_1_diff1": 0.10866321569038064, - "nauc_precision_at_1_max": -0.04068530891909402, - "nauc_precision_at_1_std": -0.09376533797535455, - "nauc_precision_at_20_diff1": -0.04957649222390242, - "nauc_precision_at_20_max": 0.140936432573259, - "nauc_precision_at_20_std": 0.04479103118396489, - "nauc_precision_at_3_diff1": 0.08256318842375679, - "nauc_precision_at_3_max": -0.021108881570062847, - "nauc_precision_at_3_std": -0.038007584676221444, - "nauc_precision_at_5_diff1": 0.07592926310375633, - "nauc_precision_at_5_max": 0.0008620050605078056, - "nauc_precision_at_5_std": -0.026056300607536234, - "nauc_recall_at_1000_diff1": -0.03953149887405362, - "nauc_recall_at_1000_max": 0.5072636897896413, - "nauc_recall_at_1000_std": 0.7267946394243926, - "nauc_recall_at_100_diff1": -0.0815659597172199, - "nauc_recall_at_100_max": 0.4456795063704503, - "nauc_recall_at_100_std": 0.5793601391173749, - "nauc_recall_at_10_diff1": 0.00024134988468162648, - "nauc_recall_at_10_max": 0.0714549580777712, - "nauc_recall_at_10_std": 0.001055290057000327, - "nauc_recall_at_1_diff1": 0.10866321569038064, - "nauc_recall_at_1_max": -0.04068530891909402, - "nauc_recall_at_1_std": -0.09376533797535455, - "nauc_recall_at_20_diff1": -0.04957649222390187, - "nauc_recall_at_20_max": 0.1409364325732594, - "nauc_recall_at_20_std": 0.04479103118396537, - "nauc_recall_at_3_diff1": 0.08256318842375683, - "nauc_recall_at_3_max": -0.021108881570062934, - "nauc_recall_at_3_std": -0.03800758467622141, - "nauc_recall_at_5_diff1": 0.07592926310375617, - "nauc_recall_at_5_max": 0.0008620050605076114, - "nauc_recall_at_5_std": -0.026056300607536588, - "ndcg_at_1": 0.07736, - "ndcg_at_10": 0.23006, - "ndcg_at_100": 0.33167, - "ndcg_at_1000": 0.34104, - "ndcg_at_20": 0.2775, - "ndcg_at_3": 0.14127, - "ndcg_at_5": 0.17927, - "precision_at_1": 0.07736, - "precision_at_10": 0.04406, - "precision_at_100": 0.00911, - "precision_at_1000": 0.00098, - "precision_at_20": 0.03142, - "precision_at_3": 0.06321, - "precision_at_5": 0.05651, - "recall_at_1": 0.07736, - "recall_at_10": 0.44057, - "recall_at_100": 0.91085, - "recall_at_1000": 0.98396, - "recall_at_20": 0.6283, - "recall_at_3": 0.18962, - "recall_at_5": 0.28255 - } - ] - }, - "task_name": "NIGHTSI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2ITRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2ITRetrieval.json deleted file mode 100644 index 5bf8b100e7..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2ITRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "350d14b7258189654e26a2be93dc0bd6bee09b76", - "evaluation_time": 1292.0344169139862, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.035, - "map_at_1": 0.01112, - "map_at_10": 0.01862, - "map_at_100": 0.02022, - "map_at_1000": 0.02041, - "map_at_20": 0.01946, - "map_at_3": 0.0157, - "map_at_5": 0.0173, - "mrr_at_1": 0.04097415372091446, - "mrr_at_10": 0.06899569067162872, - "mrr_at_100": 0.0753954947792008, - "mrr_at_1000": 0.07629663251285479, - "mrr_at_20": 0.07228274812120399, - "mrr_at_3": 0.05770752775704968, - "mrr_at_5": 0.06368066390792163, - "nauc_map_at_1000_diff1": 0.18464620105157786, - "nauc_map_at_1000_max": 0.09517461783393011, - "nauc_map_at_1000_std": 0.2040040891002083, - "nauc_map_at_100_diff1": 0.18507972397574288, - "nauc_map_at_100_max": 0.09527206757614491, - "nauc_map_at_100_std": 0.20298383306893306, - "nauc_map_at_10_diff1": 0.19701268529905605, - "nauc_map_at_10_max": 0.10283536792759419, - "nauc_map_at_10_std": 0.19184524055330834, - "nauc_map_at_1_diff1": 0.29584333547408176, - "nauc_map_at_1_max": 0.0916453292778421, - "nauc_map_at_1_std": 0.158857181905166, - "nauc_map_at_20_diff1": 0.18875941083901676, - "nauc_map_at_20_max": 0.09781871339391238, - "nauc_map_at_20_std": 0.1973870257544205, - "nauc_map_at_3_diff1": 0.22949511019464636, - "nauc_map_at_3_max": 0.11329425489178281, - "nauc_map_at_3_std": 0.18181067833411912, - "nauc_map_at_5_diff1": 0.20746294345468097, - "nauc_map_at_5_max": 0.1090653010040002, - "nauc_map_at_5_std": 0.18710071896768143, - "nauc_mrr_at_1000_diff1": 0.20282098428667883, - "nauc_mrr_at_1000_max": 0.06325911499231403, - "nauc_mrr_at_1000_std": 0.26265911763626243, - "nauc_mrr_at_100_diff1": 0.20294920425657323, - "nauc_mrr_at_100_max": 0.06321419720347958, - "nauc_mrr_at_100_std": 0.26095342846451874, - "nauc_mrr_at_10_diff1": 0.21082571157251911, - "nauc_mrr_at_10_max": 0.06496216667032441, - "nauc_mrr_at_10_std": 0.2456448673566505, - "nauc_mrr_at_1_diff1": 0.2888956197536302, - "nauc_mrr_at_1_max": 0.051945770424599746, - "nauc_mrr_at_1_std": 0.19366663227841058, - "nauc_mrr_at_20_diff1": 0.20578680515172237, - "nauc_mrr_at_20_max": 0.0643691629331365, - "nauc_mrr_at_20_std": 0.2526530831331751, - "nauc_mrr_at_3_diff1": 0.22957524087977552, - "nauc_mrr_at_3_max": 0.06920027956142867, - "nauc_mrr_at_3_std": 0.22811278430845391, - "nauc_mrr_at_5_diff1": 0.2164768879713726, - "nauc_mrr_at_5_max": 0.06709195985714196, - "nauc_mrr_at_5_std": 0.23643709402910953, - "nauc_ndcg_at_1000_diff1": 0.12300707877619335, - "nauc_ndcg_at_1000_max": 0.06750648751687409, - "nauc_ndcg_at_1000_std": 0.25758578795921744, - "nauc_ndcg_at_100_diff1": 0.131778013283211, - "nauc_ndcg_at_100_max": 0.06705475282373502, - "nauc_ndcg_at_100_std": 0.24966571751929542, - "nauc_ndcg_at_10_diff1": 0.1762416783033131, - "nauc_ndcg_at_10_max": 0.08358197089345713, - "nauc_ndcg_at_10_std": 0.23310907898873637, - "nauc_ndcg_at_1_diff1": 0.2888956197536302, - "nauc_ndcg_at_1_max": 0.051945770424599746, - "nauc_ndcg_at_1_std": 0.19366663227841058, - "nauc_ndcg_at_20_diff1": 0.15417408875873417, - "nauc_ndcg_at_20_max": 0.07760562462777675, - "nauc_ndcg_at_20_std": 0.23462305257844196, - "nauc_ndcg_at_3_diff1": 0.2171101478271009, - "nauc_ndcg_at_3_max": 0.08391708221792189, - "nauc_ndcg_at_3_std": 0.22973205684239975, - "nauc_ndcg_at_5_diff1": 0.19286258630447398, - "nauc_ndcg_at_5_max": 0.0858785578759677, - "nauc_ndcg_at_5_std": 0.23456022322849668, - "nauc_precision_at_1000_diff1": 0.09567903231176829, - "nauc_precision_at_1000_max": 0.0572561397518354, - "nauc_precision_at_1000_std": 0.565307711225196, - "nauc_precision_at_100_diff1": 0.10342329710952293, - "nauc_precision_at_100_max": 0.04018894578484745, - "nauc_precision_at_100_std": 0.4354605906817404, - "nauc_precision_at_10_diff1": 0.14942098424596895, - "nauc_precision_at_10_max": 0.06288408623104451, - "nauc_precision_at_10_std": 0.2973117922065519, - "nauc_precision_at_1_diff1": 0.2888956197536302, - "nauc_precision_at_1_max": 0.051945770424599746, - "nauc_precision_at_1_std": 0.19366663227841058, - "nauc_precision_at_20_diff1": 0.12836410396823858, - "nauc_precision_at_20_max": 0.059383711439420894, - "nauc_precision_at_20_std": 0.3294161555092745, - "nauc_precision_at_3_diff1": 0.1858984302659261, - "nauc_precision_at_3_max": 0.08228617635694004, - "nauc_precision_at_3_std": 0.2525153498411853, - "nauc_precision_at_5_diff1": 0.15962892065976547, - "nauc_precision_at_5_max": 0.07259812245110689, - "nauc_precision_at_5_std": 0.2693630038131517, - "nauc_recall_at_1000_diff1": 0.03339051340747988, - "nauc_recall_at_1000_max": 0.027844058563400814, - "nauc_recall_at_1000_std": 0.22858279511428947, - "nauc_recall_at_100_diff1": 0.05325867308988818, - "nauc_recall_at_100_max": 0.029638320585807856, - "nauc_recall_at_100_std": 0.25965889999647557, - "nauc_recall_at_10_diff1": 0.11051583333025253, - "nauc_recall_at_10_max": 0.08798717299067295, - "nauc_recall_at_10_std": 0.21440114661530302, - "nauc_recall_at_1_diff1": 0.29584333547408176, - "nauc_recall_at_1_max": 0.0916453292778421, - "nauc_recall_at_1_std": 0.158857181905166, - "nauc_recall_at_20_diff1": 0.07959909782666622, - "nauc_recall_at_20_max": 0.060615013502609366, - "nauc_recall_at_20_std": 0.23701031660389618, - "nauc_recall_at_3_diff1": 0.18117525603122883, - "nauc_recall_at_3_max": 0.12812464133898993, - "nauc_recall_at_3_std": 0.19577533427777125, - "nauc_recall_at_5_diff1": 0.1329832130842022, - "nauc_recall_at_5_max": 0.11245638518254758, - "nauc_recall_at_5_std": 0.2052166509941484, - "ndcg_at_1": 0.04097, - "ndcg_at_10": 0.035, - "ndcg_at_100": 0.0428, - "ndcg_at_1000": 0.05186, - "ndcg_at_20": 0.03662, - "ndcg_at_3": 0.03592, - "ndcg_at_5": 0.03512, - "precision_at_1": 0.04097, - "precision_at_10": 0.0147, - "precision_at_100": 0.00326, - "precision_at_1000": 0.00057, - "precision_at_20": 0.00972, - "precision_at_3": 0.02684, - "precision_at_5": 0.02136, - "recall_at_1": 0.01112, - "recall_at_10": 0.0387, - "recall_at_100": 0.08325, - "recall_at_1000": 0.13449, - "recall_at_20": 0.05102, - "recall_at_3": 0.02186, - "recall_at_5": 0.02891 - } - ] - }, - "task_name": "OVENIT2ITRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2TRetrieval.json deleted file mode 100644 index 31fc4f04f4..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OVENIT2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "2192074af29422bc1dc41cf07936f198b8c69bd0", - "evaluation_time": 1350.504133939743, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.00094, - "map_at_1": 0.00021, - "map_at_10": 0.0006, - "map_at_100": 0.00081, - "map_at_1000": 0.0009, - "map_at_20": 0.00068, - "map_at_3": 0.00046, - "map_at_5": 0.00052, - "mrr_at_1": 0.0002599792016638669, - "mrr_at_10": 0.0008833817104155483, - "mrr_at_100": 0.0012310911072741933, - "mrr_at_1000": 0.0013783242698433686, - "mrr_at_20": 0.0010165745852014807, - "mrr_at_3": 0.0006266165373436792, - "mrr_at_5": 0.0007476068581180172, - "nauc_map_at_1000_diff1": 0.3356025486850719, - "nauc_map_at_1000_max": -0.4169882172024213, - "nauc_map_at_1000_std": 0.14538374737049836, - "nauc_map_at_100_diff1": 0.3734562909349873, - "nauc_map_at_100_max": -0.43756259083510546, - "nauc_map_at_100_std": 0.14703254584037814, - "nauc_map_at_10_diff1": 0.4757961007140568, - "nauc_map_at_10_max": -0.457305835140058, - "nauc_map_at_10_std": 0.12131750979365002, - "nauc_map_at_1_diff1": 0.447023862894289, - "nauc_map_at_1_max": -0.5350799140255992, - "nauc_map_at_1_std": -0.1337349259821173, - "nauc_map_at_20_diff1": 0.4261599927770859, - "nauc_map_at_20_max": -0.4553914497074392, - "nauc_map_at_20_std": 0.14240410328361042, - "nauc_map_at_3_diff1": 0.48316789955802897, - "nauc_map_at_3_max": -0.44933732920300423, - "nauc_map_at_3_std": 0.07520438823121707, - "nauc_map_at_5_diff1": 0.4953864048755024, - "nauc_map_at_5_max": -0.45095276202437423, - "nauc_map_at_5_std": 0.08797594420893395, - "nauc_mrr_at_1000_diff1": 0.3233793837746254, - "nauc_mrr_at_1000_max": -0.40215331933996257, - "nauc_mrr_at_1000_std": 0.13645754139971128, - "nauc_mrr_at_100_diff1": 0.3631435594964905, - "nauc_mrr_at_100_max": -0.42486989417384147, - "nauc_mrr_at_100_std": 0.1353533277431553, - "nauc_mrr_at_10_diff1": 0.4571705267558742, - "nauc_mrr_at_10_max": -0.4452088246780154, - "nauc_mrr_at_10_std": 0.09358311806660385, - "nauc_mrr_at_1_diff1": 0.4390801123428319, - "nauc_mrr_at_1_max": -0.5385950516645776, - "nauc_mrr_at_1_std": -0.1013774439293419, - "nauc_mrr_at_20_diff1": 0.4123398002197073, - "nauc_mrr_at_20_max": -0.4436660692753614, - "nauc_mrr_at_20_std": 0.11881121462963219, - "nauc_mrr_at_3_diff1": 0.4566504542958695, - "nauc_mrr_at_3_max": -0.4392625768386175, - "nauc_mrr_at_3_std": 0.032253694253006206, - "nauc_mrr_at_5_diff1": 0.48230049629222144, - "nauc_mrr_at_5_max": -0.4428045340735895, - "nauc_mrr_at_5_std": 0.05184288616601152, - "nauc_ndcg_at_1000_diff1": 0.0804865683285438, - "nauc_ndcg_at_1000_max": -0.2846453341018818, - "nauc_ndcg_at_1000_std": 0.14182083062836548, - "nauc_ndcg_at_100_diff1": 0.22666071609718713, - "nauc_ndcg_at_100_max": -0.39369103560568086, - "nauc_ndcg_at_100_std": 0.17908268911241257, - "nauc_ndcg_at_10_diff1": 0.4669999755097327, - "nauc_ndcg_at_10_max": -0.4527155457158219, - "nauc_ndcg_at_10_std": 0.16289182127074892, - "nauc_ndcg_at_1_diff1": 0.4390801123428319, - "nauc_ndcg_at_1_max": -0.5385950516645776, - "nauc_ndcg_at_1_std": -0.1013774439293419, - "nauc_ndcg_at_20_diff1": 0.3635078941824615, - "nauc_ndcg_at_20_max": -0.44848318122661257, - "nauc_ndcg_at_20_std": 0.20109287009889576, - "nauc_ndcg_at_3_diff1": 0.47922450040931114, - "nauc_ndcg_at_3_max": -0.4322498148581518, - "nauc_ndcg_at_3_std": 0.08155045223711481, - "nauc_ndcg_at_5_diff1": 0.503620610286048, - "nauc_ndcg_at_5_max": -0.44037040269154254, - "nauc_ndcg_at_5_std": 0.10619497578100863, - "nauc_precision_at_1000_diff1": 0.019035664196766656, - "nauc_precision_at_1000_max": -0.22871440079272565, - "nauc_precision_at_1000_std": 0.14038857900152193, - "nauc_precision_at_100_diff1": 0.1566858798466843, - "nauc_precision_at_100_max": -0.3579552164851941, - "nauc_precision_at_100_std": 0.19897068898487488, - "nauc_precision_at_10_diff1": 0.4468305221392101, - "nauc_precision_at_10_max": -0.44023350004181505, - "nauc_precision_at_10_std": 0.18697492500703727, - "nauc_precision_at_1_diff1": 0.4390801123428319, - "nauc_precision_at_1_max": -0.5385950516645776, - "nauc_precision_at_1_std": -0.1013774439293419, - "nauc_precision_at_20_diff1": 0.3048328179348221, - "nauc_precision_at_20_max": -0.4341435816581416, - "nauc_precision_at_20_std": 0.23561115232580918, - "nauc_precision_at_3_diff1": 0.47986106476963836, - "nauc_precision_at_3_max": -0.4099968370570377, - "nauc_precision_at_3_std": 0.08388081246556922, - "nauc_precision_at_5_diff1": 0.5257145247079027, - "nauc_precision_at_5_max": -0.42785043345887885, - "nauc_precision_at_5_std": 0.11188569056651354, - "nauc_recall_at_1000_diff1": 0.01763504437122254, - "nauc_recall_at_1000_max": -0.25041869469339395, - "nauc_recall_at_1000_std": 0.12900976977544382, - "nauc_recall_at_100_diff1": 0.13782698515403186, - "nauc_recall_at_100_max": -0.3683545629769233, - "nauc_recall_at_100_std": 0.18322328259085513, - "nauc_recall_at_10_diff1": 0.46517127103809264, - "nauc_recall_at_10_max": -0.4578770140642293, - "nauc_recall_at_10_std": 0.2376092789995941, - "nauc_recall_at_1_diff1": 0.447023862894289, - "nauc_recall_at_1_max": -0.5350799140255992, - "nauc_recall_at_1_std": -0.1337349259821173, - "nauc_recall_at_20_diff1": 0.29085723394404345, - "nauc_recall_at_20_max": -0.44744217769546446, - "nauc_recall_at_20_std": 0.2712484276298942, - "nauc_recall_at_3_diff1": 0.5196230856918127, - "nauc_recall_at_3_max": -0.4244201041308224, - "nauc_recall_at_3_std": 0.16691297737481914, - "nauc_recall_at_5_diff1": 0.5373957061055848, - "nauc_recall_at_5_max": -0.4355135518338287, - "nauc_recall_at_5_std": 0.17644746841227857, - "ndcg_at_1": 0.00026, - "ndcg_at_10": 0.00094, - "ndcg_at_100": 0.00245, - "ndcg_at_1000": 0.00603, - "ndcg_at_20": 0.00126, - "ndcg_at_3": 0.00061, - "ndcg_at_5": 0.00074, - "precision_at_1": 0.00026, - "precision_at_10": 0.00027, - "precision_at_100": 0.00015, - "precision_at_1000": 6e-05, - "precision_at_20": 0.00023, - "precision_at_3": 0.00037, - "precision_at_5": 0.00034, - "recall_at_1": 0.00021, - "recall_at_10": 0.00166, - "recall_at_100": 0.00893, - "recall_at_1000": 0.03645, - "recall_at_20": 0.0028, - "recall_at_3": 0.00079, - "recall_at_5": 0.00109 - } - ] - }, - "task_name": "OVENIT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPets.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPets.json deleted file mode 100644 index e786b5adde..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPets.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "dataset_revision": "557b480fae8d69247be74d9503b378a09425096f", - "evaluation_time": 231.90859651565552, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.7442354865085855, - "f1": 0.7417313700884481, - "f1_weighted": 0.7420306227198182, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7442354865085855, - "scores_per_experiment": [ - { - "accuracy": 0.7391659852820932, - "f1": 0.7358160134713131, - "f1_weighted": 0.7358830566044606 - }, - { - "accuracy": 0.7320795857181793, - "f1": 0.7282184271306233, - "f1_weighted": 0.728373282427556 - }, - { - "accuracy": 0.7465249386753884, - "f1": 0.7452129062431734, - "f1_weighted": 0.7456542760421347 - }, - { - "accuracy": 0.7285363859362224, - "f1": 0.7259459516246249, - "f1_weighted": 0.7263860554430688 - }, - { - "accuracy": 0.7399836467702371, - "f1": 0.7391860302875606, - "f1_weighted": 0.7395108556167312 - }, - { - "accuracy": 0.7424366312346689, - "f1": 0.7384044479611193, - "f1_weighted": 0.738786323176385 - }, - { - "accuracy": 0.7519760152630145, - "f1": 0.751101111638505, - "f1_weighted": 0.751437383241543 - }, - { - "accuracy": 0.7467974925047697, - "f1": 0.7442267562085082, - "f1_weighted": 0.7445854619774289 - }, - { - "accuracy": 0.753338784409921, - "f1": 0.749093788861857, - "f1_weighted": 0.7492934083075202 - }, - { - "accuracy": 0.76151539929136, - "f1": 0.7601082674571955, - "f1_weighted": 0.7603961243613546 - } - ] - } - ] - }, - "task_name": "OxfordPets" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPetsZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPetsZeroShot.json deleted file mode 100644 index b0a0146b92..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/OxfordPetsZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "557b480fae8d69247be74d9503b378a09425096f", - "evaluation_time": 36.26844310760498, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.874897792313982, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.874897792313982 - } - ] - }, - "task_name": "OxfordPetsZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyon.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyon.json deleted file mode 100644 index 09c443ccac..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyon.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "dataset_revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", - "evaluation_time": 210.66001987457275, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "accuracy": 0.701171875, - "ap": 0.6377405975860886, - "ap_weighted": 0.6377405975860886, - "f1": 0.700650138300302, - "f1_weighted": 0.7006447988896325, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.701171875, - "scores_per_experiment": [ - { - "accuracy": 0.701171875, - "ap": 0.6377405975860886, - "ap_weighted": 0.6377405975860886, - "f1": 0.700650138300302, - "f1_weighted": 0.7006447988896325 - } - ] - } - ] - }, - "task_name": "PatchCamelyon" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyonZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyonZeroShot.json deleted file mode 100644 index edff52f341..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/PatchCamelyonZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", - "evaluation_time": 104.11877298355103, - "kg_co2_emissions": null, - "mteb_version": "1.12.49", - "scores": { - "test": [ - { - "accuracy": 0.618804931640625, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.618804931640625 - } - ] - }, - "task_name": "PatchCamelyonZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45.json deleted file mode 100644 index 05a8755f9b..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", - "evaluation_time": 124.30631566047668, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.8280000000000001, - "f1": 0.8274150272493415, - "f1_weighted": 0.8282609883607547, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8280000000000001, - "scores_per_experiment": [ - { - "accuracy": 0.8301587301587302, - "f1": 0.8298896051930552, - "f1_weighted": 0.8302023220255628 - }, - { - "accuracy": 0.8342857142857143, - "f1": 0.833295417662022, - "f1_weighted": 0.834228803768538 - }, - { - "accuracy": 0.8223809523809524, - "f1": 0.8212130407841506, - "f1_weighted": 0.8225055946019044 - }, - { - "accuracy": 0.8285714285714286, - "f1": 0.8283929971104949, - "f1_weighted": 0.8293046793995597 - }, - { - "accuracy": 0.8246031746031746, - "f1": 0.8242840754969853, - "f1_weighted": 0.8250635420082086 - } - ] - } - ] - }, - "task_name": "RESISC45" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45ZeroShot.json deleted file mode 100644 index 807ddc393a..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RESISC45ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", - "evaluation_time": 37.90462064743042, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.533968253968254, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.533968253968254 - } - ] - }, - "task_name": "RESISC45ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IMultiChoice.json deleted file mode 100644 index 2b9fb1f38e..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IMultiChoice.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "dataset_revision": "4c167c3ce529f19457c9b8e694258cc6cf8e7cc7", - "evaluation_time": 13.940337419509888, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.12011, - "cv_recall_at_1": 0.72857, - "cv_recall_at_10": 0.85714, - "cv_recall_at_100": 0.94286, - "cv_recall_at_1000": 0.97143, - "cv_recall_at_20": 0.87143, - "cv_recall_at_3": 0.78571, - "cv_recall_at_5": 0.84286, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.12011, - "map_at_1": 0.12011, - "map_at_10": 0.27377, - "map_at_100": 0.42059, - "map_at_1000": 0.48363, - "map_at_20": 0.32913, - "map_at_3": 0.1815, - "map_at_5": 0.21622, - "mrr_at_1": 0.7285714285714285, - "mrr_at_10": 0.7696598639455783, - "mrr_at_100": 0.7720894555929709, - "mrr_at_1000": 0.7721998445218422, - "mrr_at_20": 0.7705527210884354, - "mrr_at_3": 0.7547619047619047, - "mrr_at_5": 0.7676190476190478, - "nauc_cv_recall_at_1000_diff1": 0.24042950513538902, - "nauc_cv_recall_at_1000_max": -0.004668534080296818, - "nauc_cv_recall_at_1000_std": -0.5144724556489252, - "nauc_cv_recall_at_100_diff1": 0.22934173669467725, - "nauc_cv_recall_at_100_max": 0.3862044817927167, - "nauc_cv_recall_at_100_std": 0.06185807656395899, - "nauc_cv_recall_at_10_diff1": 0.46904761904761855, - "nauc_cv_recall_at_10_max": 0.45544217687074784, - "nauc_cv_recall_at_10_std": -0.11797862001943518, - "nauc_cv_recall_at_1_diff1": 0.37913486005089103, - "nauc_cv_recall_at_1_max": 0.6452758437491262, - "nauc_cv_recall_at_1_std": 0.13670553365209895, - "nauc_cv_recall_at_20_diff1": 0.4747302062186126, - "nauc_cv_recall_at_20_max": 0.38631264023934025, - "nauc_cv_recall_at_20_std": -0.1437653595469617, - "nauc_cv_recall_at_3_diff1": 0.2941673710904481, - "nauc_cv_recall_at_3_max": 0.5864074387151308, - "nauc_cv_recall_at_3_std": 0.08314454775993227, - "nauc_cv_recall_at_5_diff1": 0.48302290348453764, - "nauc_cv_recall_at_5_max": 0.4865876481597005, - "nauc_cv_recall_at_5_std": -0.012743962213706547, - "nauc_map_at_1000_diff1": 0.039056418726859325, - "nauc_map_at_1000_max": 0.3108646441695373, - "nauc_map_at_1000_std": 0.18387643315553479, - "nauc_map_at_100_diff1": 0.17039421636019683, - "nauc_map_at_100_max": 0.09905795512067436, - "nauc_map_at_100_std": -0.047296000166814706, - "nauc_map_at_10_diff1": 0.39318659870946504, - "nauc_map_at_10_max": -0.2567134684719764, - "nauc_map_at_10_std": -0.3372313247661854, - "nauc_map_at_1_diff1": 0.4052522690135478, - "nauc_map_at_1_max": -0.3133714030581606, - "nauc_map_at_1_std": -0.3279366241134995, - "nauc_map_at_20_diff1": 0.31128533550252463, - "nauc_map_at_20_max": -0.1668988523200872, - "nauc_map_at_20_std": -0.2950570027817496, - "nauc_map_at_3_diff1": 0.47350551731133683, - "nauc_map_at_3_max": -0.3190738829133001, - "nauc_map_at_3_std": -0.34746748342150613, - "nauc_map_at_5_diff1": 0.4297189568601358, - "nauc_map_at_5_max": -0.306734565163091, - "nauc_map_at_5_std": -0.3356128846263281, - "nauc_mrr_at_1000_diff1": 0.3765086703362948, - "nauc_mrr_at_1000_max": 0.6070793223986851, - "nauc_mrr_at_1000_std": 0.09762189861063993, - "nauc_mrr_at_100_diff1": 0.3765699744452178, - "nauc_mrr_at_100_max": 0.6072126617329957, - "nauc_mrr_at_100_std": 0.0978972735979146, - "nauc_mrr_at_10_diff1": 0.3788805046207156, - "nauc_mrr_at_10_max": 0.6082629986581378, - "nauc_mrr_at_10_std": 0.0963034659332654, - "nauc_mrr_at_1_diff1": 0.37913486005089103, - "nauc_mrr_at_1_max": 0.6452758437491262, - "nauc_mrr_at_1_std": 0.13670553365209895, - "nauc_mrr_at_20_diff1": 0.3787562524948725, - "nauc_mrr_at_20_max": 0.6062244307479707, - "nauc_mrr_at_20_std": 0.09613446360209521, - "nauc_mrr_at_3_diff1": 0.34401526617095446, - "nauc_mrr_at_3_max": 0.6246273302929221, - "nauc_mrr_at_3_std": 0.12242294784901903, - "nauc_mrr_at_5_diff1": 0.38107615384083393, - "nauc_mrr_at_5_max": 0.6101526396341715, - "nauc_mrr_at_5_std": 0.10525288237443857, - "nauc_ndcg_at_1000_diff1": 0.12762070579347834, - "nauc_ndcg_at_1000_max": 0.4065392036326618, - "nauc_ndcg_at_1000_std": 0.13600118977631057, - "nauc_ndcg_at_100_diff1": 0.08905540963562808, - "nauc_ndcg_at_100_max": 0.3340876494383974, - "nauc_ndcg_at_100_std": 0.16389116328814335, - "nauc_ndcg_at_10_diff1": 0.031817431716988705, - "nauc_ndcg_at_10_max": 0.48755851455947774, - "nauc_ndcg_at_10_std": 0.1880385361511959, - "nauc_ndcg_at_1_diff1": 0.37913486005089103, - "nauc_ndcg_at_1_max": 0.6452758437491262, - "nauc_ndcg_at_1_std": 0.13670553365209895, - "nauc_ndcg_at_20_diff1": 0.03212844503807327, - "nauc_ndcg_at_20_max": 0.40893860255098513, - "nauc_ndcg_at_20_std": 0.2191836350931408, - "nauc_ndcg_at_3_diff1": 0.10952929105103347, - "nauc_ndcg_at_3_max": 0.5868226539522446, - "nauc_ndcg_at_3_std": 0.20418399857463668, - "nauc_ndcg_at_5_diff1": 0.05132083453784389, - "nauc_ndcg_at_5_max": 0.5305326791254221, - "nauc_ndcg_at_5_std": 0.2280586632195748, - "nauc_precision_at_1000_diff1": -0.3414619911663539, - "nauc_precision_at_1000_max": 0.5560000366090037, - "nauc_precision_at_1000_std": 0.5508816734665237, - "nauc_precision_at_100_diff1": -0.3787961838502515, - "nauc_precision_at_100_max": 0.7160900928470757, - "nauc_precision_at_100_std": 0.6714675209553157, - "nauc_precision_at_10_diff1": -0.3066362781491847, - "nauc_precision_at_10_max": 0.6670893640902829, - "nauc_precision_at_10_std": 0.4078566027551021, - "nauc_precision_at_1_diff1": 0.37913486005089103, - "nauc_precision_at_1_max": 0.6452758437491262, - "nauc_precision_at_1_std": 0.13670553365209895, - "nauc_precision_at_20_diff1": -0.3569525663518488, - "nauc_precision_at_20_max": 0.6799005999378203, - "nauc_precision_at_20_std": 0.47997442491768877, - "nauc_precision_at_3_diff1": -0.022372098222603304, - "nauc_precision_at_3_max": 0.6144576517494506, - "nauc_precision_at_3_std": 0.2747400976264113, - "nauc_precision_at_5_diff1": -0.2176180661349585, - "nauc_precision_at_5_max": 0.6350255399229319, - "nauc_precision_at_5_std": 0.3850972309346714, - "nauc_recall_at_1000_diff1": 0.21587695470985352, - "nauc_recall_at_1000_max": 0.11425941926240299, - "nauc_recall_at_1000_std": 0.05546332909555691, - "nauc_recall_at_100_diff1": 0.21066338253500203, - "nauc_recall_at_100_max": -0.23122605362849036, - "nauc_recall_at_100_std": -0.17781012181323355, - "nauc_recall_at_10_diff1": 0.42259978636740364, - "nauc_recall_at_10_max": -0.34951875178863107, - "nauc_recall_at_10_std": -0.39276608756700837, - "nauc_recall_at_1_diff1": 0.4052522690135478, - "nauc_recall_at_1_max": -0.3133714030581606, - "nauc_recall_at_1_std": -0.3279366241134995, - "nauc_recall_at_20_diff1": 0.3429964665906505, - "nauc_recall_at_20_max": -0.32830343914659427, - "nauc_recall_at_20_std": -0.37518353692962664, - "nauc_recall_at_3_diff1": 0.4793068578557846, - "nauc_recall_at_3_max": -0.3280804979129421, - "nauc_recall_at_3_std": -0.3517807268312443, - "nauc_recall_at_5_diff1": 0.43717782790927456, - "nauc_recall_at_5_max": -0.36275023715800275, - "nauc_recall_at_5_std": -0.34949269884261674, - "ndcg_at_1": 0.72857, - "ndcg_at_10": 0.62501, - "ndcg_at_100": 0.61893, - "ndcg_at_1000": 0.69861, - "ndcg_at_20": 0.60364, - "ndcg_at_3": 0.64316, - "ndcg_at_5": 0.63774, - "precision_at_1": 0.72857, - "precision_at_10": 0.47286, - "precision_at_100": 0.19714, - "precision_at_1000": 0.0413, - "precision_at_20": 0.38857, - "precision_at_3": 0.57619, - "precision_at_5": 0.53429, - "recall_at_1": 0.12011, - "recall_at_10": 0.3365, - "recall_at_100": 0.67186, - "recall_at_1000": 0.915, - "recall_at_20": 0.43313, - "recall_at_3": 0.18464, - "recall_at_5": 0.24446 - } - ] - }, - "task_name": "ROxfordEasyI2IMultiChoice" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json deleted file mode 100644 index 738473a5e4..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "b71b5f67a93aa63761b79a67bcf28bd2ae590902", - "evaluation_time": 18.838356256484985, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.88235, - "cv_recall_at_10": 0.97059, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.98529, - "cv_recall_at_3": 0.91176, - "cv_recall_at_5": 0.94118, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.88235, - "map_at_1": 0.18698, - "map_at_10": 0.44691, - "map_at_100": 0.66159, - "map_at_1000": 0.73348, - "map_at_20": 0.53479, - "map_at_3": 0.30295, - "map_at_5": 0.36197, - "mrr_at_1": 0.8823529411764706, - "mrr_at_10": 0.908700980392157, - "mrr_at_100": 0.9098240302312877, - "mrr_at_1000": 0.9098240302312877, - "mrr_at_20": 0.9095660322952712, - "mrr_at_3": 0.8970588235294118, - "mrr_at_5": 0.9044117647058824, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": 0.45647666859182273, - "nauc_cv_recall_at_10_max": 0.7947783849423156, - "nauc_cv_recall_at_10_std": 0.07401479495347633, - "nauc_cv_recall_at_1_diff1": 0.15992222589126076, - "nauc_cv_recall_at_1_max": 0.7066438838407989, - "nauc_cv_recall_at_1_std": 0.34516803717873823, - "nauc_cv_recall_at_20_diff1": 0.5540376442015769, - "nauc_cv_recall_at_20_max": 0.8688524590163946, - "nauc_cv_recall_at_20_std": 0.7207043108682435, - "nauc_cv_recall_at_3_diff1": 0.02979739943425187, - "nauc_cv_recall_at_3_max": 0.6677398031346646, - "nauc_cv_recall_at_3_std": 0.19367311898769965, - "nauc_cv_recall_at_5_diff1": 0.33506965405558836, - "nauc_cv_recall_at_5_max": 0.8646023072252566, - "nauc_cv_recall_at_5_std": 0.3439494354763536, - "nauc_map_at_1000_diff1": -0.0748418292398556, - "nauc_map_at_1000_max": 0.1642828370088438, - "nauc_map_at_1000_std": 0.05154074134715094, - "nauc_map_at_100_diff1": 0.18300297522101533, - "nauc_map_at_100_max": -0.09630611530306471, - "nauc_map_at_100_std": -0.27588270039938373, - "nauc_map_at_10_diff1": 0.3759801235280281, - "nauc_map_at_10_max": -0.4332228813048274, - "nauc_map_at_10_std": -0.513562318981569, - "nauc_map_at_1_diff1": 0.3294708101134731, - "nauc_map_at_1_max": -0.42544232201495896, - "nauc_map_at_1_std": -0.5038005227813385, - "nauc_map_at_20_diff1": 0.32605775541629023, - "nauc_map_at_20_max": -0.3650620149830886, - "nauc_map_at_20_std": -0.48968947212775454, - "nauc_map_at_3_diff1": 0.3734075945845939, - "nauc_map_at_3_max": -0.46014380098464513, - "nauc_map_at_3_std": -0.5246559323671462, - "nauc_map_at_5_diff1": 0.3703663478894585, - "nauc_map_at_5_max": -0.45530691941081636, - "nauc_map_at_5_std": -0.5435081911059385, - "nauc_mrr_at_1000_diff1": 0.1563665959264749, - "nauc_mrr_at_1000_max": 0.7110551871683163, - "nauc_mrr_at_1000_std": 0.3006510045832456, - "nauc_mrr_at_100_diff1": 0.1563665959264749, - "nauc_mrr_at_100_max": 0.7110551871683163, - "nauc_mrr_at_100_std": 0.3006510045832456, - "nauc_mrr_at_10_diff1": 0.15944197884390138, - "nauc_mrr_at_10_max": 0.7115982604675308, - "nauc_mrr_at_10_std": 0.2934876531933919, - "nauc_mrr_at_1_diff1": 0.15992222589126076, - "nauc_mrr_at_1_max": 0.7066438838407989, - "nauc_mrr_at_1_std": 0.34516803717873823, - "nauc_mrr_at_20_diff1": 0.15751333465270692, - "nauc_mrr_at_20_max": 0.7115102171316658, - "nauc_mrr_at_20_std": 0.3018622856043013, - "nauc_mrr_at_3_diff1": 0.10362708820418606, - "nauc_mrr_at_3_max": 0.6898130404147407, - "nauc_mrr_at_3_std": 0.2796276825003452, - "nauc_mrr_at_5_diff1": 0.15680775836822267, - "nauc_mrr_at_5_max": 0.7221234773231954, - "nauc_mrr_at_5_std": 0.30966199967981883, - "nauc_ndcg_at_1000_diff1": -0.08504950934995431, - "nauc_ndcg_at_1000_max": 0.3562828960467138, - "nauc_ndcg_at_1000_std": 0.12738028206826132, - "nauc_ndcg_at_100_diff1": 0.042015479832883486, - "nauc_ndcg_at_100_max": 0.267061759431482, - "nauc_ndcg_at_100_std": 0.07290512182104139, - "nauc_ndcg_at_10_diff1": -0.12122120870754685, - "nauc_ndcg_at_10_max": 0.48196302566814775, - "nauc_ndcg_at_10_std": 0.20073536709572107, - "nauc_ndcg_at_1_diff1": 0.15992222589126076, - "nauc_ndcg_at_1_max": 0.7066438838407989, - "nauc_ndcg_at_1_std": 0.34516803717873823, - "nauc_ndcg_at_20_diff1": -0.10661851294694442, - "nauc_ndcg_at_20_max": 0.33981330679892496, - "nauc_ndcg_at_20_std": 0.20675699745130993, - "nauc_ndcg_at_3_diff1": -0.06822895395133349, - "nauc_ndcg_at_3_max": 0.5801948973031311, - "nauc_ndcg_at_3_std": 0.2644877411998497, - "nauc_ndcg_at_5_diff1": -0.08300096492046799, - "nauc_ndcg_at_5_max": 0.5620424257737597, - "nauc_ndcg_at_5_std": 0.25135047052009385, - "nauc_precision_at_1000_diff1": -0.5039877076914758, - "nauc_precision_at_1000_max": 0.5273281460558504, - "nauc_precision_at_1000_std": 0.6496446422377248, - "nauc_precision_at_100_diff1": -0.4796643779471086, - "nauc_precision_at_100_max": 0.6151414562917783, - "nauc_precision_at_100_std": 0.6678831725250701, - "nauc_precision_at_10_diff1": -0.42975726364512384, - "nauc_precision_at_10_max": 0.7033828091076564, - "nauc_precision_at_10_std": 0.6296497019716161, - "nauc_precision_at_1_diff1": 0.15992222589126076, - "nauc_precision_at_1_max": 0.7066438838407989, - "nauc_precision_at_1_std": 0.34516803717873823, - "nauc_precision_at_20_diff1": -0.4946390997396103, - "nauc_precision_at_20_max": 0.6858303123733283, - "nauc_precision_at_20_std": 0.6298812155793545, - "nauc_precision_at_3_diff1": -0.24876059895537342, - "nauc_precision_at_3_max": 0.6456537706466767, - "nauc_precision_at_3_std": 0.5045016679013927, - "nauc_precision_at_5_diff1": -0.32421116538126843, - "nauc_precision_at_5_max": 0.7016255386813988, - "nauc_precision_at_5_std": 0.5649338219757292, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5157172488761371, - "nauc_recall_at_100_max": -0.2912343469081552, - "nauc_recall_at_100_std": -0.45545038271617866, - "nauc_recall_at_10_diff1": 0.430025555269766, - "nauc_recall_at_10_max": -0.49271570163117323, - "nauc_recall_at_10_std": -0.5498160938181346, - "nauc_recall_at_1_diff1": 0.3294708101134731, - "nauc_recall_at_1_max": -0.42544232201495896, - "nauc_recall_at_1_std": -0.5038005227813385, - "nauc_recall_at_20_diff1": 0.4176823227802248, - "nauc_recall_at_20_max": -0.5026073945178894, - "nauc_recall_at_20_std": -0.5407548592660978, - "nauc_recall_at_3_diff1": 0.36561954570571, - "nauc_recall_at_3_max": -0.4642250997167962, - "nauc_recall_at_3_std": -0.5342346748455666, - "nauc_recall_at_5_diff1": 0.3529315499791077, - "nauc_recall_at_5_max": -0.47879447841104084, - "nauc_recall_at_5_std": -0.5554339290951641, - "ndcg_at_1": 0.88235, - "ndcg_at_10": 0.83499, - "ndcg_at_100": 0.82462, - "ndcg_at_1000": 0.87986, - "ndcg_at_20": 0.82108, - "ndcg_at_3": 0.85596, - "ndcg_at_5": 0.84619, - "precision_at_1": 0.88235, - "precision_at_10": 0.61176, - "precision_at_100": 0.24838, - "precision_at_1000": 0.04454, - "precision_at_20": 0.50882, - "precision_at_3": 0.76471, - "precision_at_5": 0.69412, - "recall_at_1": 0.18698, - "recall_at_10": 0.48422, - "recall_at_100": 0.80718, - "recall_at_1000": 1.0, - "recall_at_20": 0.60501, - "recall_at_3": 0.30707, - "recall_at_5": 0.3807 - } - ] - }, - "task_name": "ROxfordEasyI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IMultiChoice.json deleted file mode 100644 index 0c110c6ead..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IMultiChoice.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "dataset_revision": "fc7c4ae6655b1e6b132f3b262a359acef42dfce8", - "evaluation_time": 15.003882884979248, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.01148, - "cv_recall_at_1": 0.25714, - "cv_recall_at_10": 0.65714, - "cv_recall_at_100": 0.84286, - "cv_recall_at_1000": 0.92857, - "cv_recall_at_20": 0.68571, - "cv_recall_at_3": 0.45714, - "cv_recall_at_5": 0.57143, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.01148, - "map_at_1": 0.01148, - "map_at_10": 0.05335, - "map_at_100": 0.07243, - "map_at_1000": 0.08784, - "map_at_20": 0.0581, - "map_at_3": 0.03866, - "map_at_5": 0.04476, - "mrr_at_1": 0.2571428571428571, - "mrr_at_10": 0.38481859410430835, - "mrr_at_100": 0.39253963385655055, - "mrr_at_1000": 0.393048468885359, - "mrr_at_20": 0.38730777159348584, - "mrr_at_3": 0.3476190476190476, - "mrr_at_5": 0.3726190476190476, - "nauc_cv_recall_at_1000_diff1": 0.26685340802987884, - "nauc_cv_recall_at_1000_max": 0.07301587301587338, - "nauc_cv_recall_at_1000_std": -0.1830999066293196, - "nauc_cv_recall_at_100_diff1": 0.21040905445147495, - "nauc_cv_recall_at_100_max": 0.11148738971571162, - "nauc_cv_recall_at_100_std": 0.02419570448266679, - "nauc_cv_recall_at_10_diff1": 0.17949926917817913, - "nauc_cv_recall_at_10_max": 0.06129473336791022, - "nauc_cv_recall_at_10_std": 0.20811919468150294, - "nauc_cv_recall_at_1_diff1": 0.4553397252313951, - "nauc_cv_recall_at_1_max": -0.00898601253476105, - "nauc_cv_recall_at_1_std": 0.15209811978583004, - "nauc_cv_recall_at_20_diff1": 0.12448806313055567, - "nauc_cv_recall_at_20_max": 0.16716611727100145, - "nauc_cv_recall_at_20_std": 0.15480471481370445, - "nauc_cv_recall_at_3_diff1": 0.30826644662816005, - "nauc_cv_recall_at_3_max": -0.013483609631685515, - "nauc_cv_recall_at_3_std": 0.12409046583152049, - "nauc_cv_recall_at_5_diff1": 0.1614784394250518, - "nauc_cv_recall_at_5_max": -0.02459958932238193, - "nauc_cv_recall_at_5_std": 0.15406570841889153, - "nauc_map_at_1000_diff1": 0.31728785205951265, - "nauc_map_at_1000_max": -0.21424686509817786, - "nauc_map_at_1000_std": 0.0221906403447802, - "nauc_map_at_100_diff1": 0.31755174177151774, - "nauc_map_at_100_max": -0.19078874190305894, - "nauc_map_at_100_std": -0.023681508955016594, - "nauc_map_at_10_diff1": 0.2799136913282966, - "nauc_map_at_10_max": -0.1520026777231754, - "nauc_map_at_10_std": -0.07629896780635309, - "nauc_map_at_1_diff1": 0.4868843030130222, - "nauc_map_at_1_max": -0.15511847387665895, - "nauc_map_at_1_std": -0.02716611321363879, - "nauc_map_at_20_diff1": 0.2902764596993506, - "nauc_map_at_20_max": -0.12983705173414045, - "nauc_map_at_20_std": -0.04242912291533665, - "nauc_map_at_3_diff1": 0.2199130409925843, - "nauc_map_at_3_max": -0.1300575199918267, - "nauc_map_at_3_std": -0.12177697907382404, - "nauc_map_at_5_diff1": 0.2526425553708014, - "nauc_map_at_5_max": -0.1506194670949868, - "nauc_map_at_5_std": -0.11862027451443419, - "nauc_mrr_at_1000_diff1": 0.3597607815149087, - "nauc_mrr_at_1000_max": 0.008898477350336038, - "nauc_mrr_at_1000_std": 0.14259842374421733, - "nauc_mrr_at_100_diff1": 0.3595466281641209, - "nauc_mrr_at_100_max": 0.008904925581822543, - "nauc_mrr_at_100_std": 0.1429363715769275, - "nauc_mrr_at_10_diff1": 0.3577159847817333, - "nauc_mrr_at_10_max": 0.00585201676533247, - "nauc_mrr_at_10_std": 0.1493154319727965, - "nauc_mrr_at_1_diff1": 0.4553397252313951, - "nauc_mrr_at_1_max": -0.00898601253476105, - "nauc_mrr_at_1_std": 0.15209811978583004, - "nauc_mrr_at_20_diff1": 0.354629000714685, - "nauc_mrr_at_20_max": 0.013433224395754593, - "nauc_mrr_at_20_std": 0.14517705322833865, - "nauc_mrr_at_3_diff1": 0.38374642485486704, - "nauc_mrr_at_3_max": -0.007022664947542781, - "nauc_mrr_at_3_std": 0.13390513839165924, - "nauc_mrr_at_5_diff1": 0.3519534083953614, - "nauc_mrr_at_5_max": -0.004425335185700782, - "nauc_mrr_at_5_std": 0.14223014611578186, - "nauc_ndcg_at_1000_diff1": 0.3270873952168286, - "nauc_ndcg_at_1000_max": -0.23540909631297202, - "nauc_ndcg_at_1000_std": 0.056615205005261696, - "nauc_ndcg_at_100_diff1": 0.3574410299969898, - "nauc_ndcg_at_100_max": -0.2459841410933895, - "nauc_ndcg_at_100_std": -0.004941432018533064, - "nauc_ndcg_at_10_diff1": 0.27284914278868305, - "nauc_ndcg_at_10_max": -0.09860237927268649, - "nauc_ndcg_at_10_std": 0.17143394924414862, - "nauc_ndcg_at_1_diff1": 0.4553397252313951, - "nauc_ndcg_at_1_max": -0.00898601253476105, - "nauc_ndcg_at_1_std": 0.15209811978583004, - "nauc_ndcg_at_20_diff1": 0.24219624937954304, - "nauc_ndcg_at_20_max": -0.09686643301083671, - "nauc_ndcg_at_20_std": 0.1612277453319146, - "nauc_ndcg_at_3_diff1": 0.32690150790533234, - "nauc_ndcg_at_3_max": -0.047784701430556326, - "nauc_ndcg_at_3_std": 0.16012561788746316, - "nauc_ndcg_at_5_diff1": 0.31332748247111847, - "nauc_ndcg_at_5_max": -0.10699011157518731, - "nauc_ndcg_at_5_std": 0.15645669379446075, - "nauc_precision_at_1000_diff1": 0.0819429686636851, - "nauc_precision_at_1000_max": -0.1290271513175996, - "nauc_precision_at_1000_std": 0.2830467579249119, - "nauc_precision_at_100_diff1": 0.1086615495942521, - "nauc_precision_at_100_max": -0.22937147878224026, - "nauc_precision_at_100_std": 0.24534165503695654, - "nauc_precision_at_10_diff1": 0.23400746254303456, - "nauc_precision_at_10_max": -0.12882503413921087, - "nauc_precision_at_10_std": 0.24371550016348392, - "nauc_precision_at_1_diff1": 0.4553397252313951, - "nauc_precision_at_1_max": -0.00898601253476105, - "nauc_precision_at_1_std": 0.15209811978583004, - "nauc_precision_at_20_diff1": 0.12509716785790417, - "nauc_precision_at_20_max": -0.08597893607214858, - "nauc_precision_at_20_std": 0.3218130662392427, - "nauc_precision_at_3_diff1": 0.2960017153008923, - "nauc_precision_at_3_max": -0.05259235057482985, - "nauc_precision_at_3_std": 0.18596720508055797, - "nauc_precision_at_5_diff1": 0.2793466731531475, - "nauc_precision_at_5_max": -0.14856965124532454, - "nauc_precision_at_5_std": 0.18446594198853172, - "nauc_recall_at_1000_diff1": 0.23767465676853808, - "nauc_recall_at_1000_max": -0.21102175240994991, - "nauc_recall_at_1000_std": -0.1566324123319451, - "nauc_recall_at_100_diff1": 0.3128178199504678, - "nauc_recall_at_100_max": -0.2698286972760149, - "nauc_recall_at_100_std": -0.06876008993571361, - "nauc_recall_at_10_diff1": 0.19422816254098801, - "nauc_recall_at_10_max": -0.13751003581798485, - "nauc_recall_at_10_std": -0.07386552580291213, - "nauc_recall_at_1_diff1": 0.4868843030130222, - "nauc_recall_at_1_max": -0.15511847387665895, - "nauc_recall_at_1_std": -0.02716611321363879, - "nauc_recall_at_20_diff1": 0.19903944694380463, - "nauc_recall_at_20_max": -0.07262351965705542, - "nauc_recall_at_20_std": -0.04461273334236608, - "nauc_recall_at_3_diff1": 0.13345389226212073, - "nauc_recall_at_3_max": -0.1026168555200003, - "nauc_recall_at_3_std": -0.13519897599044758, - "nauc_recall_at_5_diff1": 0.16489673880783298, - "nauc_recall_at_5_max": -0.13503944945681817, - "nauc_recall_at_5_std": -0.13677876300364675, - "ndcg_at_1": 0.25714, - "ndcg_at_10": 0.19342, - "ndcg_at_100": 0.19552, - "ndcg_at_1000": 0.32441, - "ndcg_at_20": 0.17295, - "ndcg_at_3": 0.23726, - "ndcg_at_5": 0.21753, - "precision_at_1": 0.25714, - "precision_at_10": 0.15429, - "precision_at_100": 0.05786, - "precision_at_1000": 0.01964, - "precision_at_20": 0.10571, - "precision_at_3": 0.21905, - "precision_at_5": 0.19143, - "recall_at_1": 0.01148, - "recall_at_10": 0.10105, - "recall_at_100": 0.26388, - "recall_at_1000": 0.62162, - "recall_at_20": 0.11767, - "recall_at_3": 0.06125, - "recall_at_5": 0.07694 - } - ] - }, - "task_name": "ROxfordHardI2IMultiChoice" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json deleted file mode 100644 index a944a1a301..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "f71ab9d4aabcda93d55a7e65edfb3a34767d89e6", - "evaluation_time": 24.67104411125183, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.27143, - "cv_recall_at_10": 0.61429, - "cv_recall_at_100": 0.85714, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.74286, - "cv_recall_at_3": 0.42857, - "cv_recall_at_5": 0.57143, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.27143, - "map_at_1": 0.06428, - "map_at_10": 0.10339, - "map_at_100": 0.1463, - "map_at_1000": 0.18944, - "map_at_20": 0.11828, - "map_at_3": 0.07918, - "map_at_5": 0.09165, - "mrr_at_1": 0.2714285714285714, - "mrr_at_10": 0.3802551020408163, - "mrr_at_100": 0.3931760616047423, - "mrr_at_1000": 0.39374600845359353, - "mrr_at_20": 0.38933244509925174, - "mrr_at_3": 0.34047619047619043, - "mrr_at_5": 0.374047619047619, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.3957240038872689, - "nauc_cv_recall_at_100_max": -0.45918367346938554, - "nauc_cv_recall_at_100_std": -0.7625364431486866, - "nauc_cv_recall_at_10_diff1": 0.3501381065268928, - "nauc_cv_recall_at_10_max": -0.3230170295134737, - "nauc_cv_recall_at_10_std": -0.28669610039365817, - "nauc_cv_recall_at_1_diff1": 0.31133966351507303, - "nauc_cv_recall_at_1_max": -0.36652967019618166, - "nauc_cv_recall_at_1_std": -0.39039944931468024, - "nauc_cv_recall_at_20_diff1": 0.48971241524433046, - "nauc_cv_recall_at_20_max": -0.35202829085807796, - "nauc_cv_recall_at_20_std": -0.4874327799859716, - "nauc_cv_recall_at_3_diff1": 0.20616527390900677, - "nauc_cv_recall_at_3_max": -0.3163788300835654, - "nauc_cv_recall_at_3_std": -0.34913649025069615, - "nauc_cv_recall_at_5_diff1": 0.24162217659137575, - "nauc_cv_recall_at_5_max": -0.40519507186858295, - "nauc_cv_recall_at_5_std": -0.349342915811088, - "nauc_map_at_1000_diff1": 0.33483050942379805, - "nauc_map_at_1000_max": -0.3354071059234048, - "nauc_map_at_1000_std": -0.325502994839205, - "nauc_map_at_100_diff1": 0.36421675112335855, - "nauc_map_at_100_max": -0.37056372968494616, - "nauc_map_at_100_std": -0.39098657201855935, - "nauc_map_at_10_diff1": 0.43724421619965775, - "nauc_map_at_10_max": -0.3475499176287484, - "nauc_map_at_10_std": -0.3087709853113486, - "nauc_map_at_1_diff1": 0.5328003645974971, - "nauc_map_at_1_max": -0.30820864556110844, - "nauc_map_at_1_std": -0.2525466251963858, - "nauc_map_at_20_diff1": 0.40424948920352965, - "nauc_map_at_20_max": -0.36561679338161, - "nauc_map_at_20_std": -0.34598603785503834, - "nauc_map_at_3_diff1": 0.47569665621843843, - "nauc_map_at_3_max": -0.3350766680518147, - "nauc_map_at_3_std": -0.29588688611880093, - "nauc_map_at_5_diff1": 0.44497494504359997, - "nauc_map_at_5_max": -0.35371869827402724, - "nauc_map_at_5_std": -0.2890820614425645, - "nauc_mrr_at_1000_diff1": 0.2682845806139734, - "nauc_mrr_at_1000_max": -0.36456025639429424, - "nauc_mrr_at_1000_std": -0.37575466392348833, - "nauc_mrr_at_100_diff1": 0.26897531827173626, - "nauc_mrr_at_100_max": -0.3659176034789706, - "nauc_mrr_at_100_std": -0.37740763588526954, - "nauc_mrr_at_10_diff1": 0.2686668033979706, - "nauc_mrr_at_10_max": -0.3628633448193816, - "nauc_mrr_at_10_std": -0.36335706237767257, - "nauc_mrr_at_1_diff1": 0.31133966351507303, - "nauc_mrr_at_1_max": -0.36652967019618166, - "nauc_mrr_at_1_std": -0.39039944931468024, - "nauc_mrr_at_20_diff1": 0.2731473760001198, - "nauc_mrr_at_20_max": -0.36635207234744227, - "nauc_mrr_at_20_std": -0.3759359230442535, - "nauc_mrr_at_3_diff1": 0.24452621954568515, - "nauc_mrr_at_3_max": -0.35272889400073, - "nauc_mrr_at_3_std": -0.3683045738415161, - "nauc_mrr_at_5_diff1": 0.25344916447455645, - "nauc_mrr_at_5_max": -0.3740602466954703, - "nauc_mrr_at_5_std": -0.3711781326955965, - "nauc_ndcg_at_1000_diff1": 0.15289382130510762, - "nauc_ndcg_at_1000_max": -0.11235365434908824, - "nauc_ndcg_at_1000_std": -0.16901273056125712, - "nauc_ndcg_at_100_diff1": 0.33546453963254314, - "nauc_ndcg_at_100_max": -0.39506763446494053, - "nauc_ndcg_at_100_std": -0.47504197799159326, - "nauc_ndcg_at_10_diff1": 0.2602337772209344, - "nauc_ndcg_at_10_max": -0.35707200508700326, - "nauc_ndcg_at_10_std": -0.32627013894830836, - "nauc_ndcg_at_1_diff1": 0.31133966351507303, - "nauc_ndcg_at_1_max": -0.36652967019618166, - "nauc_ndcg_at_1_std": -0.39039944931468024, - "nauc_ndcg_at_20_diff1": 0.2951622910865746, - "nauc_ndcg_at_20_max": -0.4075808329991618, - "nauc_ndcg_at_20_std": -0.4033601223158598, - "nauc_ndcg_at_3_diff1": 0.2583369568946569, - "nauc_ndcg_at_3_max": -0.36460761087963334, - "nauc_ndcg_at_3_std": -0.37460677189461866, - "nauc_ndcg_at_5_diff1": 0.24895791797858352, - "nauc_ndcg_at_5_max": -0.38343316697513324, - "nauc_ndcg_at_5_std": -0.35014891765983175, - "nauc_precision_at_1000_diff1": -0.15221736880156744, - "nauc_precision_at_1000_max": 0.24012671254256857, - "nauc_precision_at_1000_std": 0.31990023300446896, - "nauc_precision_at_100_diff1": -0.0025063624250149666, - "nauc_precision_at_100_max": -0.23039752243674347, - "nauc_precision_at_100_std": -0.2687557194001006, - "nauc_precision_at_10_diff1": 0.04266939397452653, - "nauc_precision_at_10_max": -0.2921849379730692, - "nauc_precision_at_10_std": -0.32849429418487175, - "nauc_precision_at_1_diff1": 0.31133966351507303, - "nauc_precision_at_1_max": -0.36652967019618166, - "nauc_precision_at_1_std": -0.39039944931468024, - "nauc_precision_at_20_diff1": -0.012177956618922228, - "nauc_precision_at_20_max": -0.3166595426664091, - "nauc_precision_at_20_std": -0.38852936131749555, - "nauc_precision_at_3_diff1": 0.151720996554046, - "nauc_precision_at_3_max": -0.3436744959797202, - "nauc_precision_at_3_std": -0.37392561492454535, - "nauc_precision_at_5_diff1": 0.09040511727078884, - "nauc_precision_at_5_max": -0.35160602517367096, - "nauc_precision_at_5_std": -0.35003095123461053, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.26559603138781285, - "nauc_recall_at_100_max": -0.3073116185520707, - "nauc_recall_at_100_std": -0.449657488606374, - "nauc_recall_at_10_diff1": 0.38683534388556096, - "nauc_recall_at_10_max": -0.3358496548040101, - "nauc_recall_at_10_std": -0.2154853289653661, - "nauc_recall_at_1_diff1": 0.5328003645974971, - "nauc_recall_at_1_max": -0.30820864556110844, - "nauc_recall_at_1_std": -0.2525466251963858, - "nauc_recall_at_20_diff1": 0.30135439280777626, - "nauc_recall_at_20_max": -0.33860982229647496, - "nauc_recall_at_20_std": -0.2973578871821516, - "nauc_recall_at_3_diff1": 0.4514271986825242, - "nauc_recall_at_3_max": -0.33100045995309413, - "nauc_recall_at_3_std": -0.2898936824323878, - "nauc_recall_at_5_diff1": 0.3748793011771698, - "nauc_recall_at_5_max": -0.3504567142206052, - "nauc_recall_at_5_std": -0.17845054202949487, - "ndcg_at_1": 0.27143, - "ndcg_at_10": 0.22624, - "ndcg_at_100": 0.29318, - "ndcg_at_1000": 0.52204, - "ndcg_at_20": 0.24135, - "ndcg_at_3": 0.24796, - "ndcg_at_5": 0.24666, - "precision_at_1": 0.27143, - "precision_at_10": 0.15714, - "precision_at_100": 0.07357, - "precision_at_1000": 0.03567, - "precision_at_20": 0.13357, - "precision_at_3": 0.20952, - "precision_at_5": 0.2, - "recall_at_1": 0.06428, - "recall_at_10": 0.14214, - "recall_at_100": 0.38739, - "recall_at_1000": 1.0, - "recall_at_20": 0.2058, - "recall_at_3": 0.08351, - "recall_at_5": 0.11676 - } - ] - }, - "task_name": "ROxfordHardI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IMultiChoice.json deleted file mode 100644 index 2579f6ef68..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IMultiChoice.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "dataset_revision": "83bd440268e200a4f60313070618e3f45000fa94", - "evaluation_time": 14.132107257843018, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.04963, - "cv_recall_at_1": 0.77143, - "cv_recall_at_10": 0.87143, - "cv_recall_at_100": 0.94286, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.88571, - "cv_recall_at_3": 0.82857, - "cv_recall_at_5": 0.87143, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.04963, - "map_at_1": 0.04963, - "map_at_10": 0.15207, - "map_at_100": 0.2724, - "map_at_1000": 0.34306, - "map_at_20": 0.19547, - "map_at_3": 0.087, - "map_at_5": 0.11245, - "mrr_at_1": 0.7714285714285715, - "mrr_at_10": 0.81, - "mrr_at_100": 0.8126116140632085, - "mrr_at_1000": 0.8129819154176776, - "mrr_at_20": 0.8108928571428572, - "mrr_at_3": 0.8, - "mrr_at_5": 0.81, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.22934173669467725, - "nauc_cv_recall_at_100_max": 0.3862044817927167, - "nauc_cv_recall_at_100_std": 0.09862278244631179, - "nauc_cv_recall_at_10_diff1": 0.40127150336574263, - "nauc_cv_recall_at_10_max": 0.5091355914093374, - "nauc_cv_recall_at_10_std": -0.04840260711614463, - "nauc_cv_recall_at_1_diff1": 0.33182314761262127, - "nauc_cv_recall_at_1_max": 0.6395797185270868, - "nauc_cv_recall_at_1_std": 0.17595270226849136, - "nauc_cv_recall_at_20_diff1": 0.40009492168960564, - "nauc_cv_recall_at_20_max": 0.4383009017560512, - "nauc_cv_recall_at_20_std": -0.0693521594684388, - "nauc_cv_recall_at_3_diff1": 0.1894338380513495, - "nauc_cv_recall_at_3_max": 0.6075543120473995, - "nauc_cv_recall_at_3_std": 0.12532916392363366, - "nauc_cv_recall_at_5_diff1": 0.40127150336574263, - "nauc_cv_recall_at_5_max": 0.5091355914093374, - "nauc_cv_recall_at_5_std": -0.04840260711614463, - "nauc_map_at_1000_diff1": -0.13224573449005503, - "nauc_map_at_1000_max": 0.4227560714777927, - "nauc_map_at_1000_std": 0.31351087144984363, - "nauc_map_at_100_diff1": -0.02836497538559071, - "nauc_map_at_100_max": 0.19639808649260623, - "nauc_map_at_100_std": 0.04954005369447823, - "nauc_map_at_10_diff1": 0.25121232316392567, - "nauc_map_at_10_max": -0.15556227534649655, - "nauc_map_at_10_std": -0.2488378494247982, - "nauc_map_at_1_diff1": 0.272345967565727, - "nauc_map_at_1_max": -0.2643088054590121, - "nauc_map_at_1_std": -0.27101749212632714, - "nauc_map_at_20_diff1": 0.13842997076502075, - "nauc_map_at_20_max": -0.06672772445238266, - "nauc_map_at_20_std": -0.19809071582803736, - "nauc_map_at_3_diff1": 0.3224142774768474, - "nauc_map_at_3_max": -0.2326757410552881, - "nauc_map_at_3_std": -0.2675429832114887, - "nauc_map_at_5_diff1": 0.3167741526994269, - "nauc_map_at_5_max": -0.22646890969571978, - "nauc_map_at_5_std": -0.26391954191876693, - "nauc_mrr_at_1000_diff1": 0.3082925432291351, - "nauc_mrr_at_1000_max": 0.6087298059076731, - "nauc_mrr_at_1000_std": 0.128068145081543, - "nauc_mrr_at_100_diff1": 0.30825889635587544, - "nauc_mrr_at_100_max": 0.6080934109778549, - "nauc_mrr_at_100_std": 0.12852823805463925, - "nauc_mrr_at_10_diff1": 0.31165182357930454, - "nauc_mrr_at_10_max": 0.6097725826972008, - "nauc_mrr_at_10_std": 0.12443013146734498, - "nauc_mrr_at_1_diff1": 0.33182314761262127, - "nauc_mrr_at_1_max": 0.6395797185270868, - "nauc_mrr_at_1_std": 0.17595270226849136, - "nauc_mrr_at_20_diff1": 0.31120802031702144, - "nauc_mrr_at_20_max": 0.6073798478505396, - "nauc_mrr_at_20_std": 0.12435570379822256, - "nauc_mrr_at_3_diff1": 0.26938620863854473, - "nauc_mrr_at_3_max": 0.6255367517049751, - "nauc_mrr_at_3_std": 0.1537545556237145, - "nauc_mrr_at_5_diff1": 0.31165182357930454, - "nauc_mrr_at_5_max": 0.6097725826972008, - "nauc_mrr_at_5_std": 0.12443013146734498, - "nauc_ndcg_at_1000_diff1": 0.004789789538060326, - "nauc_ndcg_at_1000_max": 0.4304267036999359, - "nauc_ndcg_at_1000_std": 0.20516350588416282, - "nauc_ndcg_at_100_diff1": -0.0985051399330637, - "nauc_ndcg_at_100_max": 0.4047840878665377, - "nauc_ndcg_at_100_std": 0.3189929050444598, - "nauc_ndcg_at_10_diff1": -0.12943461724680122, - "nauc_ndcg_at_10_max": 0.5959856882638876, - "nauc_ndcg_at_10_std": 0.3322657775788708, - "nauc_ndcg_at_1_diff1": 0.33182314761262127, - "nauc_ndcg_at_1_max": 0.6395797185270868, - "nauc_ndcg_at_1_std": 0.17595270226849136, - "nauc_ndcg_at_20_diff1": -0.15227287405763856, - "nauc_ndcg_at_20_max": 0.577403391414749, - "nauc_ndcg_at_20_std": 0.36970889043336586, - "nauc_ndcg_at_3_diff1": 0.022411758177576204, - "nauc_ndcg_at_3_max": 0.575626107585026, - "nauc_ndcg_at_3_std": 0.24755287374915227, - "nauc_ndcg_at_5_diff1": -0.034454400308166065, - "nauc_ndcg_at_5_max": 0.5655036501426706, - "nauc_ndcg_at_5_std": 0.280337402240637, - "nauc_precision_at_1000_diff1": -0.1979455862208862, - "nauc_precision_at_1000_max": 0.49452629750592836, - "nauc_precision_at_1000_std": 0.5170286173407959, - "nauc_precision_at_100_diff1": -0.31426081091366387, - "nauc_precision_at_100_max": 0.6695443542749885, - "nauc_precision_at_100_std": 0.6261360571210748, - "nauc_precision_at_10_diff1": -0.2527118526563905, - "nauc_precision_at_10_max": 0.6263582372380011, - "nauc_precision_at_10_std": 0.3992628197721442, - "nauc_precision_at_1_diff1": 0.33182314761262127, - "nauc_precision_at_1_max": 0.6395797185270868, - "nauc_precision_at_1_std": 0.17595270226849136, - "nauc_precision_at_20_diff1": -0.31590964166066615, - "nauc_precision_at_20_max": 0.645482829549644, - "nauc_precision_at_20_std": 0.4668678815358941, - "nauc_precision_at_3_diff1": -0.05580043594090598, - "nauc_precision_at_3_max": 0.5759748123032211, - "nauc_precision_at_3_std": 0.2806167756518931, - "nauc_precision_at_5_diff1": -0.14313081953662518, - "nauc_precision_at_5_max": 0.5793499136946008, - "nauc_precision_at_5_std": 0.3373884715435296, - "nauc_recall_at_1000_diff1": 0.07995145169939738, - "nauc_recall_at_1000_max": 0.1842438166265837, - "nauc_recall_at_1000_std": 0.043545956333912506, - "nauc_recall_at_100_diff1": 0.03299050541578269, - "nauc_recall_at_100_max": -0.11358015050633144, - "nauc_recall_at_100_std": -0.03558993365235519, - "nauc_recall_at_10_diff1": 0.31258822262912317, - "nauc_recall_at_10_max": -0.21448543830771524, - "nauc_recall_at_10_std": -0.2702885296246692, - "nauc_recall_at_1_diff1": 0.272345967565727, - "nauc_recall_at_1_max": -0.2643088054590121, - "nauc_recall_at_1_std": -0.27101749212632714, - "nauc_recall_at_20_diff1": 0.20536080239790513, - "nauc_recall_at_20_max": -0.1828385547433498, - "nauc_recall_at_20_std": -0.23751709286812595, - "nauc_recall_at_3_diff1": 0.3339569204499953, - "nauc_recall_at_3_max": -0.23145861780156063, - "nauc_recall_at_3_std": -0.26721917195809763, - "nauc_recall_at_5_diff1": 0.3584675101320892, - "nauc_recall_at_5_max": -0.2639487514014815, - "nauc_recall_at_5_std": -0.2733945840450043, - "ndcg_at_1": 0.77143, - "ndcg_at_10": 0.60443, - "ndcg_at_100": 0.49539, - "ndcg_at_1000": 0.60683, - "ndcg_at_20": 0.54202, - "ndcg_at_3": 0.67149, - "ndcg_at_5": 0.65383, - "precision_at_1": 0.77143, - "precision_at_10": 0.53286, - "precision_at_100": 0.229, - "precision_at_1000": 0.05983, - "precision_at_20": 0.435, - "precision_at_3": 0.62857, - "precision_at_5": 0.6, - "recall_at_1": 0.04963, - "recall_at_10": 0.18302, - "recall_at_100": 0.4477, - "recall_at_1000": 0.76971, - "recall_at_20": 0.25156, - "recall_at_3": 0.09291, - "recall_at_5": 0.12898 - } - ] - }, - "task_name": "ROxfordMediumI2IMultiChoice" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json deleted file mode 100644 index f1e28194b9..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "1dfb86730ee4b3f49b441f4896d473c83eb5ff0d", - "evaluation_time": 26.38901996612549, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.91429, - "cv_recall_at_10": 0.94286, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.97143, - "cv_recall_at_3": 0.91429, - "cv_recall_at_5": 0.94286, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.91429, - "map_at_1": 0.08083, - "map_at_10": 0.24687, - "map_at_100": 0.42618, - "map_at_1000": 0.5327, - "map_at_20": 0.31214, - "map_at_3": 0.14225, - "map_at_5": 0.18461, - "mrr_at_1": 0.9142857142857143, - "mrr_at_10": 0.92, - "mrr_at_100": 0.9224459377246834, - "mrr_at_1000": 0.9224459377246834, - "mrr_at_20": 0.9215873015873017, - "mrr_at_3": 0.9142857142857143, - "mrr_at_5": 0.92, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": -0.4013772175536871, - "nauc_cv_recall_at_10_max": 0.42938842203548166, - "nauc_cv_recall_at_10_std": 0.6586134453781514, - "nauc_cv_recall_at_1_diff1": -0.11554621848739352, - "nauc_cv_recall_at_1_max": 0.5269996887643944, - "nauc_cv_recall_at_1_std": 0.21856520385932235, - "nauc_cv_recall_at_20_diff1": 0.3489729225023323, - "nauc_cv_recall_at_20_max": 0.9346405228758171, - "nauc_cv_recall_at_20_std": 0.5401493930905691, - "nauc_cv_recall_at_3_diff1": -0.11554621848739352, - "nauc_cv_recall_at_3_max": 0.5269996887643944, - "nauc_cv_recall_at_3_std": 0.21856520385932235, - "nauc_cv_recall_at_5_diff1": -0.4013772175536871, - "nauc_cv_recall_at_5_max": 0.42938842203548166, - "nauc_cv_recall_at_5_std": 0.6586134453781514, - "nauc_map_at_1000_diff1": -0.23125835300876216, - "nauc_map_at_1000_max": 0.23519514137370254, - "nauc_map_at_1000_std": 0.2182230501859235, - "nauc_map_at_100_diff1": -0.05015810647551921, - "nauc_map_at_100_max": -0.04327639525818628, - "nauc_map_at_100_std": -0.15828452131450388, - "nauc_map_at_10_diff1": 0.1893778857001354, - "nauc_map_at_10_max": -0.34441772024823814, - "nauc_map_at_10_std": -0.3848968165397017, - "nauc_map_at_1_diff1": 0.1529533401602949, - "nauc_map_at_1_max": -0.4190116365256113, - "nauc_map_at_1_std": -0.3330481576960033, - "nauc_map_at_20_diff1": 0.09589668436288233, - "nauc_map_at_20_max": -0.2673998395876724, - "nauc_map_at_20_std": -0.3314998258254241, - "nauc_map_at_3_diff1": 0.22783602569587666, - "nauc_map_at_3_max": -0.375611479696183, - "nauc_map_at_3_std": -0.3593266287200665, - "nauc_map_at_5_diff1": 0.23171354107352043, - "nauc_map_at_5_max": -0.3834575082863232, - "nauc_map_at_5_std": -0.38241308439234356, - "nauc_mrr_at_1000_diff1": -0.14268770122670912, - "nauc_mrr_at_1000_max": 0.5205774182168708, - "nauc_mrr_at_1000_std": 0.2680405904560385, - "nauc_mrr_at_100_diff1": -0.14268770122670912, - "nauc_mrr_at_100_max": 0.5205774182168708, - "nauc_mrr_at_100_std": 0.2680405904560385, - "nauc_mrr_at_10_diff1": -0.15637921835400717, - "nauc_mrr_at_10_max": 0.5130552220888355, - "nauc_mrr_at_10_std": 0.2814292383620098, - "nauc_mrr_at_1_diff1": -0.11554621848739352, - "nauc_mrr_at_1_max": 0.5269996887643944, - "nauc_mrr_at_1_std": 0.21856520385932235, - "nauc_mrr_at_20_diff1": -0.13623047059579352, - "nauc_mrr_at_20_max": 0.524976657329598, - "nauc_mrr_at_20_std": 0.2713958727890603, - "nauc_mrr_at_3_diff1": -0.11554621848739352, - "nauc_mrr_at_3_max": 0.5269996887643944, - "nauc_mrr_at_3_std": 0.21856520385932235, - "nauc_mrr_at_5_diff1": -0.15637921835400717, - "nauc_mrr_at_5_max": 0.5130552220888355, - "nauc_mrr_at_5_std": 0.2814292383620098, - "nauc_ndcg_at_1000_diff1": -0.24370615505516585, - "nauc_ndcg_at_1000_max": 0.3585208209406902, - "nauc_ndcg_at_1000_std": 0.263969492188972, - "nauc_ndcg_at_100_diff1": -0.14186676166246207, - "nauc_ndcg_at_100_max": 0.2275659208451184, - "nauc_ndcg_at_100_std": 0.16048415696096283, - "nauc_ndcg_at_10_diff1": -0.30589363585428303, - "nauc_ndcg_at_10_max": 0.500142114454778, - "nauc_ndcg_at_10_std": 0.4141408373889108, - "nauc_ndcg_at_1_diff1": -0.11554621848739352, - "nauc_ndcg_at_1_max": 0.5269996887643944, - "nauc_ndcg_at_1_std": 0.21856520385932235, - "nauc_ndcg_at_20_diff1": -0.2982156974982523, - "nauc_ndcg_at_20_max": 0.487494996769566, - "nauc_ndcg_at_20_std": 0.4663494007016063, - "nauc_ndcg_at_3_diff1": -0.13605929183456947, - "nauc_ndcg_at_3_max": 0.5394295944368543, - "nauc_ndcg_at_3_std": 0.2841764519265793, - "nauc_ndcg_at_5_diff1": -0.18517240726502945, - "nauc_ndcg_at_5_max": 0.5018241088290002, - "nauc_ndcg_at_5_std": 0.32986920973664985, - "nauc_precision_at_1000_diff1": -0.2610146663608417, - "nauc_precision_at_1000_max": 0.42090687761865053, - "nauc_precision_at_1000_std": 0.5453516915872082, - "nauc_precision_at_100_diff1": -0.31550298388759973, - "nauc_precision_at_100_max": 0.5382316505653624, - "nauc_precision_at_100_std": 0.5864596701910575, - "nauc_precision_at_10_diff1": -0.3396134608542791, - "nauc_precision_at_10_max": 0.5841486511939392, - "nauc_precision_at_10_std": 0.4649483079756401, - "nauc_precision_at_1_diff1": -0.11554621848739352, - "nauc_precision_at_1_max": 0.5269996887643944, - "nauc_precision_at_1_std": 0.21856520385932235, - "nauc_precision_at_20_diff1": -0.3739204789385173, - "nauc_precision_at_20_max": 0.5974041684159593, - "nauc_precision_at_20_std": 0.5452135797286048, - "nauc_precision_at_3_diff1": -0.16477444647626113, - "nauc_precision_at_3_max": 0.5728888479602177, - "nauc_precision_at_3_std": 0.3239051214923765, - "nauc_precision_at_5_diff1": -0.21896064507078017, - "nauc_precision_at_5_max": 0.5663742826794222, - "nauc_precision_at_5_std": 0.378912278123309, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.18668259021675102, - "nauc_recall_at_100_max": -0.13326671980523305, - "nauc_recall_at_100_std": -0.28507500423807525, - "nauc_recall_at_10_diff1": 0.17850980669329247, - "nauc_recall_at_10_max": -0.40096190909138624, - "nauc_recall_at_10_std": -0.3570134850979618, - "nauc_recall_at_1_diff1": 0.1529533401602949, - "nauc_recall_at_1_max": -0.4190116365256113, - "nauc_recall_at_1_std": -0.3330481576960033, - "nauc_recall_at_20_diff1": 0.1509705328425781, - "nauc_recall_at_20_max": -0.3011359312987517, - "nauc_recall_at_20_std": -0.2929059943600196, - "nauc_recall_at_3_diff1": 0.22113145253276656, - "nauc_recall_at_3_max": -0.3819326194584587, - "nauc_recall_at_3_std": -0.3572099966902034, - "nauc_recall_at_5_diff1": 0.1981700707835856, - "nauc_recall_at_5_max": -0.42018635579579555, - "nauc_recall_at_5_std": -0.3263923639383356, - "ndcg_at_1": 0.91429, - "ndcg_at_10": 0.77153, - "ndcg_at_100": 0.65508, - "ndcg_at_1000": 0.81056, - "ndcg_at_20": 0.70656, - "ndcg_at_3": 0.85518, - "ndcg_at_5": 0.83659, - "precision_at_1": 0.91429, - "precision_at_10": 0.66143, - "precision_at_100": 0.29157, - "precision_at_1000": 0.07894, - "precision_at_20": 0.55, - "precision_at_3": 0.81429, - "precision_at_5": 0.76571, - "recall_at_1": 0.08083, - "recall_at_10": 0.26885, - "recall_at_100": 0.57158, - "recall_at_1000": 1.0, - "recall_at_20": 0.35986, - "recall_at_3": 0.14512, - "recall_at_5": 0.20079 - } - ] - }, - "task_name": "ROxfordMediumI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RP2kI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RP2kI2IRetrieval.json deleted file mode 100644 index eafeff620f..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RP2kI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "f8f82d4eb1aa4dc4dbf2c768596c8110a3703765", - "evaluation_time": 324.14615392684937, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.55887, - "cv_recall_at_10": 0.77913, - "cv_recall_at_100": 0.90418, - "cv_recall_at_1000": 0.95933, - "cv_recall_at_20": 0.82554, - "cv_recall_at_3": 0.6808, - "cv_recall_at_5": 0.72537, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.55887, - "map_at_1": 0.06083, - "map_at_10": 0.12431, - "map_at_100": 0.17952, - "map_at_1000": 0.2009, - "map_at_20": 0.14322, - "map_at_3": 0.09136, - "map_at_5": 0.10532, - "mrr_at_1": 0.9998166867438328, - "mrr_at_10": 0.9998952495679044, - "mrr_at_100": 0.9998952495679044, - "mrr_at_1000": 0.9998952495679044, - "mrr_at_20": 0.9998952495679044, - "mrr_at_3": 0.9998952495679044, - "mrr_at_5": 0.9998952495679044, - "nauc_cv_recall_at_1000_diff1": -0.6534775051605857, - "nauc_cv_recall_at_1000_max": 0.016875483355911193, - "nauc_cv_recall_at_1000_std": 0.1834914123790724, - "nauc_cv_recall_at_100_diff1": -0.46190450882692813, - "nauc_cv_recall_at_100_max": 0.0044555049072486745, - "nauc_cv_recall_at_100_std": 0.3217134902493942, - "nauc_cv_recall_at_10_diff1": -0.3197493491364096, - "nauc_cv_recall_at_10_max": 0.004081705009985084, - "nauc_cv_recall_at_10_std": 0.3833079134677583, - "nauc_cv_recall_at_1_diff1": -0.2732166234186543, - "nauc_cv_recall_at_1_max": -0.010034973300874108, - "nauc_cv_recall_at_1_std": 0.3494947305628685, - "nauc_cv_recall_at_20_diff1": -0.3517887547295718, - "nauc_cv_recall_at_20_max": 0.006552844210252164, - "nauc_cv_recall_at_20_std": 0.3793238219190742, - "nauc_cv_recall_at_3_diff1": -0.28039644458970836, - "nauc_cv_recall_at_3_max": 0.00026932680904395343, - "nauc_cv_recall_at_3_std": 0.3777753345990772, - "nauc_cv_recall_at_5_diff1": -0.29514123944001847, - "nauc_cv_recall_at_5_max": 0.0002103622905348435, - "nauc_cv_recall_at_5_std": 0.3862948975183731, - "nauc_map_at_1000_diff1": 0.08734717040767775, - "nauc_map_at_1000_max": -0.010675299588909455, - "nauc_map_at_1000_std": 0.41490480554200815, - "nauc_map_at_100_diff1": 0.135072420725819, - "nauc_map_at_100_max": -0.00837091280878698, - "nauc_map_at_100_std": 0.35005558541793835, - "nauc_map_at_10_diff1": 0.26950863371478956, - "nauc_map_at_10_max": -0.004745789557209113, - "nauc_map_at_10_std": 0.14275927317472537, - "nauc_map_at_1_diff1": 0.40347502895694837, - "nauc_map_at_1_max": -0.006070639716117982, - "nauc_map_at_1_std": 0.01609734274601474, - "nauc_map_at_20_diff1": 0.22773030861533414, - "nauc_map_at_20_max": -0.005347115187225529, - "nauc_map_at_20_std": 0.2096685956925957, - "nauc_map_at_3_diff1": 0.3350402994842465, - "nauc_map_at_3_max": -0.002361981770631997, - "nauc_map_at_3_std": 0.05928879397905913, - "nauc_map_at_5_diff1": 0.30786980093898875, - "nauc_map_at_5_max": -0.0033167682834259605, - "nauc_map_at_5_std": 0.08902137115070434, - "nauc_mrr_at_1000_diff1": 0.7693741473921198, - "nauc_mrr_at_1000_max": -0.07329673170966303, - "nauc_mrr_at_1000_std": -0.41562237775388855, - "nauc_mrr_at_100_diff1": 0.7693741473921198, - "nauc_mrr_at_100_max": -0.07329673170966303, - "nauc_mrr_at_100_std": -0.41562237775388855, - "nauc_mrr_at_10_diff1": 0.7693741473921198, - "nauc_mrr_at_10_max": -0.07329673170966303, - "nauc_mrr_at_10_std": -0.41562237775388855, - "nauc_mrr_at_1_diff1": 0.7976521093473791, - "nauc_mrr_at_1_max": -0.05649010070984422, - "nauc_mrr_at_1_std": -0.43184586481189485, - "nauc_mrr_at_20_diff1": 0.7693741473921198, - "nauc_mrr_at_20_max": -0.07329673170966303, - "nauc_mrr_at_20_std": -0.41562237775388855, - "nauc_mrr_at_3_diff1": 0.7693741473921198, - "nauc_mrr_at_3_max": -0.07329673170966303, - "nauc_mrr_at_3_std": -0.41562237775388855, - "nauc_mrr_at_5_diff1": 0.7693741473921198, - "nauc_mrr_at_5_max": -0.07329673170966303, - "nauc_mrr_at_5_std": -0.41562237775388855, - "nauc_ndcg_at_1000_diff1": 0.009920350828696455, - "nauc_ndcg_at_1000_max": -0.010774540419665588, - "nauc_ndcg_at_1000_std": 0.43694481975295574, - "nauc_ndcg_at_100_diff1": 0.018084517333648395, - "nauc_ndcg_at_100_max": -0.0147514814443795, - "nauc_ndcg_at_100_std": 0.46744646393565903, - "nauc_ndcg_at_10_diff1": -0.16407446859272273, - "nauc_ndcg_at_10_max": -0.01784761939283373, - "nauc_ndcg_at_10_std": 0.45860760438003595, - "nauc_ndcg_at_1_diff1": 0.7976521093473791, - "nauc_ndcg_at_1_max": -0.11251153162885205, - "nauc_ndcg_at_1_std": -0.43184586481189485, - "nauc_ndcg_at_20_diff1": -0.1180606939919965, - "nauc_ndcg_at_20_max": -0.017164859556287968, - "nauc_ndcg_at_20_std": 0.48562601519497933, - "nauc_ndcg_at_3_diff1": -0.21591401825959575, - "nauc_ndcg_at_3_max": -0.013782741144658254, - "nauc_ndcg_at_3_std": 0.38832357405024914, - "nauc_ndcg_at_5_diff1": -0.19548651074158818, - "nauc_ndcg_at_5_max": -0.017383534701624292, - "nauc_ndcg_at_5_std": 0.42227867362587224, - "nauc_precision_at_1000_diff1": -0.26457930829821386, - "nauc_precision_at_1000_max": -0.019625850952636456, - "nauc_precision_at_1000_std": 0.44487310862894697, - "nauc_precision_at_100_diff1": -0.28070930263126453, - "nauc_precision_at_100_max": -0.0183983445570901, - "nauc_precision_at_100_std": 0.5216852457299462, - "nauc_precision_at_10_diff1": -0.2642289169634242, - "nauc_precision_at_10_max": -0.016037018440131376, - "nauc_precision_at_10_std": 0.43775176956816353, - "nauc_precision_at_1_diff1": 0.7976521093473791, - "nauc_precision_at_1_max": -0.11251153162885205, - "nauc_precision_at_1_std": -0.43184586481189485, - "nauc_precision_at_20_diff1": -0.26818857979333577, - "nauc_precision_at_20_max": -0.015097011478949736, - "nauc_precision_at_20_std": 0.47166450463929854, - "nauc_precision_at_3_diff1": -0.2643174309717343, - "nauc_precision_at_3_max": -0.01229104802996264, - "nauc_precision_at_3_std": 0.37315288392656526, - "nauc_precision_at_5_diff1": -0.2618738011679915, - "nauc_precision_at_5_max": -0.016386565682558307, - "nauc_precision_at_5_std": 0.4022744328150495, - "nauc_recall_at_1000_diff1": 0.02988142348682087, - "nauc_recall_at_1000_max": -0.006736994916843787, - "nauc_recall_at_1000_std": 0.38393510595562075, - "nauc_recall_at_100_diff1": 0.12754685099331003, - "nauc_recall_at_100_max": -0.005988478824620124, - "nauc_recall_at_100_std": 0.3383849668377813, - "nauc_recall_at_10_diff1": 0.27252371808491316, - "nauc_recall_at_10_max": -0.00514889254539631, - "nauc_recall_at_10_std": 0.1385644816305786, - "nauc_recall_at_1_diff1": 0.40347502895694837, - "nauc_recall_at_1_max": -0.006070639716117982, - "nauc_recall_at_1_std": 0.01609734274601474, - "nauc_recall_at_20_diff1": 0.23198661607389173, - "nauc_recall_at_20_max": -0.0036289993571175373, - "nauc_recall_at_20_std": 0.20018947547570426, - "nauc_recall_at_3_diff1": 0.3355751433312504, - "nauc_recall_at_3_max": -0.001792243329487425, - "nauc_recall_at_3_std": 0.0593931739832833, - "nauc_recall_at_5_diff1": 0.30950726685875857, - "nauc_recall_at_5_max": -0.003205556081532953, - "nauc_recall_at_5_std": 0.08819687899167265, - "ndcg_at_1": 0.99982, - "ndcg_at_10": 0.5575, - "ndcg_at_100": 0.36797, - "ndcg_at_1000": 0.41673, - "ndcg_at_20": 0.475, - "ndcg_at_3": 0.7517, - "ndcg_at_5": 0.66112, - "precision_at_1": 0.99982, - "precision_at_10": 0.42989, - "precision_at_100": 0.15119, - "precision_at_1000": 0.03396, - "precision_at_20": 0.32737, - "precision_at_3": 0.67203, - "precision_at_5": 0.557, - "recall_at_1": 0.06083, - "recall_at_10": 0.13643, - "recall_at_100": 0.26298, - "recall_at_1000": 0.44653, - "recall_at_20": 0.16902, - "recall_at_3": 0.09249, - "recall_at_5": 0.10951 - } - ] - }, - "task_name": "RP2kI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IMultiChoice.json deleted file mode 100644 index f13cd60fba..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IMultiChoice.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "dataset_revision": "db94b5afd0014ab8c978f20a0fbcc52da1612a08", - "evaluation_time": 18.75471258163452, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.02814, - "cv_recall_at_1": 0.97143, - "cv_recall_at_10": 1.0, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 1.0, - "cv_recall_at_5": 1.0, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.02814, - "map_at_1": 0.02814, - "map_at_10": 0.14604, - "map_at_100": 0.613, - "map_at_1000": 0.71993, - "map_at_20": 0.24425, - "map_at_3": 0.06669, - "map_at_5": 0.09219, - "mrr_at_1": 0.9714285714285714, - "mrr_at_10": 0.9857142857142858, - "mrr_at_100": 0.9857142857142858, - "mrr_at_1000": 0.9857142857142858, - "mrr_at_20": 0.9857142857142858, - "mrr_at_3": 0.9857142857142858, - "mrr_at_5": 0.9857142857142858, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": NaN, - "nauc_cv_recall_at_10_max": NaN, - "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": 0.15289449112978365, - "nauc_cv_recall_at_1_max": 0.4960317460317484, - "nauc_cv_recall_at_1_std": 0.8692810457516342, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": NaN, - "nauc_cv_recall_at_3_max": NaN, - "nauc_cv_recall_at_3_std": NaN, - "nauc_cv_recall_at_5_diff1": NaN, - "nauc_cv_recall_at_5_max": NaN, - "nauc_cv_recall_at_5_std": NaN, - "nauc_map_at_1000_diff1": -0.5162392334753443, - "nauc_map_at_1000_max": 0.4031740557038615, - "nauc_map_at_1000_std": 0.41563960286590534, - "nauc_map_at_100_diff1": -0.23225006961960362, - "nauc_map_at_100_max": 0.3085695443992811, - "nauc_map_at_100_std": 0.12110491855395035, - "nauc_map_at_10_diff1": 0.5381554521493018, - "nauc_map_at_10_max": -0.10493092334419908, - "nauc_map_at_10_std": -0.2680755265314605, - "nauc_map_at_1_diff1": 0.8594916968198884, - "nauc_map_at_1_max": -0.24256237866427466, - "nauc_map_at_1_std": -0.4444380497243242, - "nauc_map_at_20_diff1": 0.3590188805543108, - "nauc_map_at_20_max": -0.013558846615686978, - "nauc_map_at_20_std": -0.16797583975226438, - "nauc_map_at_3_diff1": 0.7548580604028502, - "nauc_map_at_3_max": -0.2047628239453275, - "nauc_map_at_3_std": -0.4210073714397216, - "nauc_map_at_5_diff1": 0.6593773157320483, - "nauc_map_at_5_max": -0.18318576348363613, - "nauc_map_at_5_std": -0.3667442070438189, - "nauc_mrr_at_1000_diff1": 0.15289449112977452, - "nauc_mrr_at_1000_max": 0.49603174603173467, - "nauc_mrr_at_1000_std": 0.8692810457516342, - "nauc_mrr_at_100_diff1": 0.15289449112977452, - "nauc_mrr_at_100_max": 0.49603174603173467, - "nauc_mrr_at_100_std": 0.8692810457516342, - "nauc_mrr_at_10_diff1": 0.15289449112977452, - "nauc_mrr_at_10_max": 0.49603174603173467, - "nauc_mrr_at_10_std": 0.8692810457516342, - "nauc_mrr_at_1_diff1": 0.15289449112978365, - "nauc_mrr_at_1_max": 0.4960317460317484, - "nauc_mrr_at_1_std": 0.8692810457516342, - "nauc_mrr_at_20_diff1": 0.15289449112977452, - "nauc_mrr_at_20_max": 0.49603174603173467, - "nauc_mrr_at_20_std": 0.8692810457516342, - "nauc_mrr_at_3_diff1": 0.15289449112977452, - "nauc_mrr_at_3_max": 0.49603174603173467, - "nauc_mrr_at_3_std": 0.8692810457516342, - "nauc_mrr_at_5_diff1": 0.15289449112977452, - "nauc_mrr_at_5_max": 0.49603174603173467, - "nauc_mrr_at_5_std": 0.8692810457516342, - "nauc_ndcg_at_1000_diff1": -0.6430087042511464, - "nauc_ndcg_at_1000_max": 0.42655378925643106, - "nauc_ndcg_at_1000_std": 0.5044627676548901, - "nauc_ndcg_at_100_diff1": -0.5428592210544468, - "nauc_ndcg_at_100_max": 0.446739791744679, - "nauc_ndcg_at_100_std": 0.465672473136578, - "nauc_ndcg_at_10_diff1": -0.9208561880491332, - "nauc_ndcg_at_10_max": 0.5967384668371186, - "nauc_ndcg_at_10_std": 0.7022870489893225, - "nauc_ndcg_at_1_diff1": 0.15289449112978365, - "nauc_ndcg_at_1_max": 0.4960317460317484, - "nauc_ndcg_at_1_std": 0.8692810457516342, - "nauc_ndcg_at_20_diff1": -0.8251725302548746, - "nauc_ndcg_at_20_max": 0.5377716300435347, - "nauc_ndcg_at_20_std": 0.5566860440051336, - "nauc_ndcg_at_3_diff1": -0.9569729064943051, - "nauc_ndcg_at_3_max": 0.6190215986665834, - "nauc_ndcg_at_3_std": 0.9222091686879381, - "nauc_ndcg_at_5_diff1": -1.1226587238391097, - "nauc_ndcg_at_5_max": 0.5998868316343212, - "nauc_ndcg_at_5_std": 0.8858441072536013, - "nauc_precision_at_1000_diff1": -0.4777678980183267, - "nauc_precision_at_1000_max": 0.19411501983394858, - "nauc_precision_at_1000_std": 0.4317886018119784, - "nauc_precision_at_100_diff1": -0.6193565917597147, - "nauc_precision_at_100_max": 0.31610227934661556, - "nauc_precision_at_100_std": 0.41899007291320295, - "nauc_precision_at_10_diff1": -1.0942053142464947, - "nauc_precision_at_10_max": 0.57491910971664, - "nauc_precision_at_10_std": 0.7286596725169142, - "nauc_precision_at_1_diff1": 0.15289449112978365, - "nauc_precision_at_1_max": 0.4960317460317484, - "nauc_precision_at_1_std": 0.8692810457516342, - "nauc_precision_at_20_diff1": -1.006987444474548, - "nauc_precision_at_20_max": 0.5295046147956847, - "nauc_precision_at_20_std": 0.6468370068581858, - "nauc_precision_at_3_diff1": -1.188608776844072, - "nauc_precision_at_3_max": 0.6297333748314129, - "nauc_precision_at_3_std": 0.9273783587509071, - "nauc_precision_at_5_diff1": -1.304355295936774, - "nauc_precision_at_5_max": 0.5779142685336303, - "nauc_precision_at_5_std": 0.8799716519199388, - "nauc_recall_at_1000_diff1": -0.1671984561020124, - "nauc_recall_at_1000_max": 0.006203927434206601, - "nauc_recall_at_1000_std": 0.14154184625248917, - "nauc_recall_at_100_diff1": 0.07725532133003771, - "nauc_recall_at_100_max": 0.13911681142723956, - "nauc_recall_at_100_std": -0.13776713676284705, - "nauc_recall_at_10_diff1": 0.5586543206392892, - "nauc_recall_at_10_max": -0.1405893884569057, - "nauc_recall_at_10_std": -0.30419927459760987, - "nauc_recall_at_1_diff1": 0.8594916968198884, - "nauc_recall_at_1_max": -0.24256237866427466, - "nauc_recall_at_1_std": -0.4444380497243242, - "nauc_recall_at_20_diff1": 0.3955687201747704, - "nauc_recall_at_20_max": -0.084623109059799, - "nauc_recall_at_20_std": -0.24589251462208517, - "nauc_recall_at_3_diff1": 0.7237068204895556, - "nauc_recall_at_3_max": -0.21361841875189405, - "nauc_recall_at_3_std": -0.4269612949915014, - "nauc_recall_at_5_diff1": 0.6397655259145639, - "nauc_recall_at_5_max": -0.20024408587215523, - "nauc_recall_at_5_std": -0.38234727563640214, - "ndcg_at_1": 0.97143, - "ndcg_at_10": 0.92763, - "ndcg_at_100": 0.82979, - "ndcg_at_1000": 0.88963, - "ndcg_at_20": 0.90969, - "ndcg_at_3": 0.96255, - "ndcg_at_5": 0.95149, - "precision_at_1": 0.97143, - "precision_at_10": 0.89286, - "precision_at_100": 0.61286, - "precision_at_1000": 0.09256, - "precision_at_20": 0.85429, - "precision_at_3": 0.95714, - "precision_at_5": 0.93714, - "recall_at_1": 0.02814, - "recall_at_10": 0.15323, - "recall_at_100": 0.6921, - "recall_at_1000": 0.94803, - "recall_at_20": 0.26283, - "recall_at_3": 0.06802, - "recall_at_5": 0.09552 - } - ] - }, - "task_name": "RParisEasyI2IMultiChoice" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json deleted file mode 100644 index 1c5827ef01..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "7d821ddebcb30ad343133e3a81e23347ac2a08a8", - "evaluation_time": 44.682684659957886, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.97143, - "cv_recall_at_10": 1.0, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 1.0, - "cv_recall_at_5": 1.0, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.97143, - "map_at_1": 0.02925, - "map_at_10": 0.14539, - "map_at_100": 0.63054, - "map_at_1000": 0.75513, - "map_at_20": 0.24788, - "map_at_3": 0.06378, - "map_at_5": 0.08742, - "mrr_at_1": 0.9714285714285714, - "mrr_at_10": 0.9857142857142858, - "mrr_at_100": 0.9857142857142858, - "mrr_at_1000": 0.9857142857142858, - "mrr_at_20": 0.9857142857142858, - "mrr_at_3": 0.9857142857142858, - "mrr_at_5": 0.9857142857142858, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": NaN, - "nauc_cv_recall_at_10_max": NaN, - "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": -0.6615312791783367, - "nauc_cv_recall_at_1_max": 0.2754435107376266, - "nauc_cv_recall_at_1_std": 0.7117180205415492, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": NaN, - "nauc_cv_recall_at_3_max": NaN, - "nauc_cv_recall_at_3_std": NaN, - "nauc_cv_recall_at_5_diff1": NaN, - "nauc_cv_recall_at_5_max": NaN, - "nauc_cv_recall_at_5_std": NaN, - "nauc_map_at_1000_diff1": -0.40591219852099064, - "nauc_map_at_1000_max": 0.26094130889238604, - "nauc_map_at_1000_std": 0.09665505089407778, - "nauc_map_at_100_diff1": -0.12855307032951752, - "nauc_map_at_100_max": 0.15818953768862642, - "nauc_map_at_100_std": -0.2617923598733903, - "nauc_map_at_10_diff1": 0.4267948633570509, - "nauc_map_at_10_max": -0.08091999723228532, - "nauc_map_at_10_std": -0.48144180579841844, - "nauc_map_at_1_diff1": 0.7246477183930717, - "nauc_map_at_1_max": -0.2595633654811421, - "nauc_map_at_1_std": -0.473439121428067, - "nauc_map_at_20_diff1": 0.28638200450299545, - "nauc_map_at_20_max": 0.022722341991735208, - "nauc_map_at_20_std": -0.4223664868261228, - "nauc_map_at_3_diff1": 0.599669109747631, - "nauc_map_at_3_max": -0.18294743939161057, - "nauc_map_at_3_std": -0.500291623756148, - "nauc_map_at_5_diff1": 0.5435589386638962, - "nauc_map_at_5_max": -0.1414910554361904, - "nauc_map_at_5_std": -0.5064288331914394, - "nauc_mrr_at_1000_diff1": -0.6615312791783412, - "nauc_mrr_at_1000_max": 0.2754435107376266, - "nauc_mrr_at_1000_std": 0.7117180205415492, - "nauc_mrr_at_100_diff1": -0.6615312791783412, - "nauc_mrr_at_100_max": 0.2754435107376266, - "nauc_mrr_at_100_std": 0.7117180205415492, - "nauc_mrr_at_10_diff1": -0.6615312791783412, - "nauc_mrr_at_10_max": 0.2754435107376266, - "nauc_mrr_at_10_std": 0.7117180205415492, - "nauc_mrr_at_1_diff1": -0.6615312791783367, - "nauc_mrr_at_1_max": 0.2754435107376266, - "nauc_mrr_at_1_std": 0.7117180205415492, - "nauc_mrr_at_20_diff1": -0.6615312791783412, - "nauc_mrr_at_20_max": 0.2754435107376266, - "nauc_mrr_at_20_std": 0.7117180205415492, - "nauc_mrr_at_3_diff1": -0.6615312791783412, - "nauc_mrr_at_3_max": 0.2754435107376266, - "nauc_mrr_at_3_std": 0.7117180205415492, - "nauc_mrr_at_5_diff1": -0.6615312791783412, - "nauc_mrr_at_5_max": 0.2754435107376266, - "nauc_mrr_at_5_std": 0.7117180205415492, - "nauc_ndcg_at_1000_diff1": -0.6596495665503245, - "nauc_ndcg_at_1000_max": 0.35984465648987496, - "nauc_ndcg_at_1000_std": 0.31222271915833927, - "nauc_ndcg_at_100_diff1": -0.42172915492454077, - "nauc_ndcg_at_100_max": 0.1660609412830679, - "nauc_ndcg_at_100_std": 0.03890643898659772, - "nauc_ndcg_at_10_diff1": -0.795470804963755, - "nauc_ndcg_at_10_max": 0.62218300778039, - "nauc_ndcg_at_10_std": 0.41853096184140837, - "nauc_ndcg_at_1_diff1": -0.6615312791783367, - "nauc_ndcg_at_1_max": 0.2754435107376266, - "nauc_ndcg_at_1_std": 0.7117180205415492, - "nauc_ndcg_at_20_diff1": -0.8166236637349379, - "nauc_ndcg_at_20_max": 0.5512278088652715, - "nauc_ndcg_at_20_std": 0.4393916095824728, - "nauc_ndcg_at_3_diff1": -0.9341977827431088, - "nauc_ndcg_at_3_max": 0.6672676444157623, - "nauc_ndcg_at_3_std": 0.39642947946164325, - "nauc_ndcg_at_5_diff1": -0.9441446617385689, - "nauc_ndcg_at_5_max": 0.680574940437725, - "nauc_ndcg_at_5_std": 0.45573053150824994, - "nauc_precision_at_1000_diff1": -0.4449935554869243, - "nauc_precision_at_1000_max": 0.16030281251691458, - "nauc_precision_at_1000_std": 0.5733632485324373, - "nauc_precision_at_100_diff1": -0.5302407180791104, - "nauc_precision_at_100_max": 0.1551769884391311, - "nauc_precision_at_100_std": 0.4184977473765995, - "nauc_precision_at_10_diff1": -0.9456434208407176, - "nauc_precision_at_10_max": 0.5868765770118642, - "nauc_precision_at_10_std": 0.50484511730284, - "nauc_precision_at_1_diff1": -0.6615312791783367, - "nauc_precision_at_1_max": 0.2754435107376266, - "nauc_precision_at_1_std": 0.7117180205415492, - "nauc_precision_at_20_diff1": -0.926778796672683, - "nauc_precision_at_20_max": 0.5280684464415885, - "nauc_precision_at_20_std": 0.5682403269442342, - "nauc_precision_at_3_diff1": -1.0055244319950187, - "nauc_precision_at_3_max": 0.7123404917522567, - "nauc_precision_at_3_std": 0.37741207594148934, - "nauc_precision_at_5_diff1": -1.0297986387840703, - "nauc_precision_at_5_max": 0.6765172620516983, - "nauc_precision_at_5_std": 0.4879394858412396, - "nauc_recall_at_1000_diff1": -0.033581200345988084, - "nauc_recall_at_1000_max": -0.25140608933946496, - "nauc_recall_at_1000_std": 0.10256335172342733, - "nauc_recall_at_100_diff1": 0.18293184373343338, - "nauc_recall_at_100_max": -0.0562892444564702, - "nauc_recall_at_100_std": -0.4925077606843601, - "nauc_recall_at_10_diff1": 0.48352650007752274, - "nauc_recall_at_10_max": -0.13513020550162758, - "nauc_recall_at_10_std": -0.48966303958842, - "nauc_recall_at_1_diff1": 0.7246477183930717, - "nauc_recall_at_1_max": -0.2595633654811421, - "nauc_recall_at_1_std": -0.473439121428067, - "nauc_recall_at_20_diff1": 0.34593183103928304, - "nauc_recall_at_20_max": -0.054714009747642164, - "nauc_recall_at_20_std": -0.45184646419179664, - "nauc_recall_at_3_diff1": 0.6064436631162091, - "nauc_recall_at_3_max": -0.18329351534591007, - "nauc_recall_at_3_std": -0.5063835059924418, - "nauc_recall_at_5_diff1": 0.5499812724878281, - "nauc_recall_at_5_max": -0.14351977259114093, - "nauc_recall_at_5_std": -0.510558046605686, - "ndcg_at_1": 0.97143, - "ndcg_at_10": 0.92622, - "ndcg_at_100": 0.84274, - "ndcg_at_1000": 0.92213, - "ndcg_at_20": 0.91612, - "ndcg_at_3": 0.95074, - "ndcg_at_5": 0.939, - "precision_at_1": 0.97143, - "precision_at_10": 0.89571, - "precision_at_100": 0.62029, - "precision_at_1000": 0.09697, - "precision_at_20": 0.86929, - "precision_at_3": 0.94286, - "precision_at_5": 0.92286, - "recall_at_1": 0.02925, - "recall_at_10": 0.15836, - "recall_at_100": 0.7168, - "recall_at_1000": 0.98961, - "recall_at_20": 0.26733, - "recall_at_3": 0.06435, - "recall_at_5": 0.08842 - } - ] - }, - "task_name": "RParisEasyI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IMultiChoice.json deleted file mode 100644 index e6162419e8..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IMultiChoice.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "dataset_revision": "4e5997e48fb2f2f8bf1c8973851dedeb17e09a83", - "evaluation_time": 18.11166262626648, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.00907, - "cv_recall_at_1": 0.81429, - "cv_recall_at_10": 0.95714, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.97143, - "cv_recall_at_3": 0.9, - "cv_recall_at_5": 0.9, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.00907, - "map_at_1": 0.00907, - "map_at_10": 0.06456, - "map_at_100": 0.23858, - "map_at_1000": 0.33338, - "map_at_20": 0.10802, - "map_at_3": 0.0239, - "map_at_5": 0.03746, - "mrr_at_1": 0.8142857142857143, - "mrr_at_10": 0.8544047619047621, - "mrr_at_100": 0.8560692239858908, - "mrr_at_1000": 0.8560692239858908, - "mrr_at_20": 0.8555952380952383, - "mrr_at_3": 0.8476190476190477, - "mrr_at_5": 0.8476190476190477, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": 0.12651727357610024, - "nauc_cv_recall_at_10_max": 0.742452536570181, - "nauc_cv_recall_at_10_std": 0.5659819483348865, - "nauc_cv_recall_at_1_diff1": 0.07804020484598333, - "nauc_cv_recall_at_1_max": 0.60700909577314, - "nauc_cv_recall_at_1_std": 0.041771764885728854, - "nauc_cv_recall_at_20_diff1": -0.171335200746965, - "nauc_cv_recall_at_20_max": 0.6790382819794577, - "nauc_cv_recall_at_20_std": 0.3489729225023323, - "nauc_cv_recall_at_3_diff1": 0.12058156595971639, - "nauc_cv_recall_at_3_max": 0.6089102307589694, - "nauc_cv_recall_at_3_std": 0.36261171135120646, - "nauc_cv_recall_at_5_diff1": 0.12058156595971639, - "nauc_cv_recall_at_5_max": 0.6089102307589694, - "nauc_cv_recall_at_5_std": 0.36261171135120646, - "nauc_map_at_1000_diff1": -0.14730057025035195, - "nauc_map_at_1000_max": 0.35443918991459294, - "nauc_map_at_1000_std": 0.2592666345486196, - "nauc_map_at_100_diff1": -0.02007929644055961, - "nauc_map_at_100_max": 0.12631888830781082, - "nauc_map_at_100_std": -0.04309297943451873, - "nauc_map_at_10_diff1": 0.316050298469623, - "nauc_map_at_10_max": -0.13310958861916383, - "nauc_map_at_10_std": -0.3020901743993513, - "nauc_map_at_1_diff1": 0.5378248883416507, - "nauc_map_at_1_max": -0.13238109598299855, - "nauc_map_at_1_std": -0.2002514420418722, - "nauc_map_at_20_diff1": 0.21121038948324744, - "nauc_map_at_20_max": -0.06818178937655603, - "nauc_map_at_20_std": -0.2902145910447691, - "nauc_map_at_3_diff1": 0.45725171311121, - "nauc_map_at_3_max": -0.19287806549614692, - "nauc_map_at_3_std": -0.25735181540630836, - "nauc_map_at_5_diff1": 0.42067337504189933, - "nauc_map_at_5_max": -0.18761920119844047, - "nauc_map_at_5_std": -0.2766021627006926, - "nauc_mrr_at_1000_diff1": 0.11587681189811185, - "nauc_mrr_at_1000_max": 0.6097767662452036, - "nauc_mrr_at_1000_std": 0.08966689534414451, - "nauc_mrr_at_100_diff1": 0.11587681189811185, - "nauc_mrr_at_100_max": 0.6097767662452036, - "nauc_mrr_at_100_std": 0.08966689534414451, - "nauc_mrr_at_10_diff1": 0.12009817077420695, - "nauc_mrr_at_10_max": 0.6119594659009971, - "nauc_mrr_at_10_std": 0.09790880388517216, - "nauc_mrr_at_1_diff1": 0.07804020484598333, - "nauc_mrr_at_1_max": 0.60700909577314, - "nauc_mrr_at_1_std": 0.041771764885728854, - "nauc_mrr_at_20_diff1": 0.11488715765282757, - "nauc_mrr_at_20_max": 0.6097325059817074, - "nauc_mrr_at_20_std": 0.09010175972487022, - "nauc_mrr_at_3_diff1": 0.11806489062690863, - "nauc_mrr_at_3_max": 0.6070359281437123, - "nauc_mrr_at_3_std": 0.101521446902114, - "nauc_mrr_at_5_diff1": 0.11806489062690863, - "nauc_mrr_at_5_max": 0.6070359281437123, - "nauc_mrr_at_5_std": 0.101521446902114, - "nauc_ndcg_at_1000_diff1": 0.025969018525589978, - "nauc_ndcg_at_1000_max": 0.36217088530709435, - "nauc_ndcg_at_1000_std": 0.25309337496879, - "nauc_ndcg_at_100_diff1": -0.17968384395623355, - "nauc_ndcg_at_100_max": 0.4239707948301639, - "nauc_ndcg_at_100_std": 0.3667281666093518, - "nauc_ndcg_at_10_diff1": -0.2397422113161351, - "nauc_ndcg_at_10_max": 0.3812994195515857, - "nauc_ndcg_at_10_std": 0.04827273248313771, - "nauc_ndcg_at_1_diff1": 0.07804020484598333, - "nauc_ndcg_at_1_max": 0.60700909577314, - "nauc_ndcg_at_1_std": 0.041771764885728854, - "nauc_ndcg_at_20_diff1": -0.28132858513315473, - "nauc_ndcg_at_20_max": 0.4204685927011596, - "nauc_ndcg_at_20_std": 0.10721091043225721, - "nauc_ndcg_at_3_diff1": -0.09344111682157266, - "nauc_ndcg_at_3_max": 0.379692511012279, - "nauc_ndcg_at_3_std": 0.04006405355649845, - "nauc_ndcg_at_5_diff1": -0.12981305098158163, - "nauc_ndcg_at_5_max": 0.3878036317455262, - "nauc_ndcg_at_5_std": 0.028676296191065903, - "nauc_precision_at_1000_diff1": -0.42027963870829504, - "nauc_precision_at_1000_max": 0.5131562832922288, - "nauc_precision_at_1000_std": 0.6312618988329652, - "nauc_precision_at_100_diff1": -0.3568206317095141, - "nauc_precision_at_100_max": 0.5107076773140476, - "nauc_precision_at_100_std": 0.4725107993998001, - "nauc_precision_at_10_diff1": -0.2953003666954273, - "nauc_precision_at_10_max": 0.361267795513373, - "nauc_precision_at_10_std": 0.055085202761000866, - "nauc_precision_at_1_diff1": 0.07804020484598333, - "nauc_precision_at_1_max": 0.60700909577314, - "nauc_precision_at_1_std": 0.041771764885728854, - "nauc_precision_at_20_diff1": -0.3210375121807198, - "nauc_precision_at_20_max": 0.42343158585680635, - "nauc_precision_at_20_std": 0.13184322004366664, - "nauc_precision_at_3_diff1": -0.15055311012827152, - "nauc_precision_at_3_max": 0.3240710586307241, - "nauc_precision_at_3_std": 0.04530985723140268, - "nauc_precision_at_5_diff1": -0.1735214934469398, - "nauc_precision_at_5_max": 0.3602496967545344, - "nauc_precision_at_5_std": 0.026229993195467694, - "nauc_recall_at_1000_diff1": 0.29022911927159833, - "nauc_recall_at_1000_max": 0.2125654628634255, - "nauc_recall_at_1000_std": 0.28260953057103566, - "nauc_recall_at_100_diff1": 0.19480792831160149, - "nauc_recall_at_100_max": -0.003561859262334835, - "nauc_recall_at_100_std": -0.12640776822732094, - "nauc_recall_at_10_diff1": 0.35131981594744904, - "nauc_recall_at_10_max": -0.17286224306644377, - "nauc_recall_at_10_std": -0.3177220315856104, - "nauc_recall_at_1_diff1": 0.5378248883416507, - "nauc_recall_at_1_max": -0.13238109598299855, - "nauc_recall_at_1_std": -0.2002514420418722, - "nauc_recall_at_20_diff1": 0.276802747494477, - "nauc_recall_at_20_max": -0.13421078554955512, - "nauc_recall_at_20_std": -0.3214777307000227, - "nauc_recall_at_3_diff1": 0.4639750095588438, - "nauc_recall_at_3_max": -0.2163484225516375, - "nauc_recall_at_3_std": -0.2587991550499656, - "nauc_recall_at_5_diff1": 0.45052599994510506, - "nauc_recall_at_5_max": -0.20428456806687892, - "nauc_recall_at_5_std": -0.28400516448621627, - "ndcg_at_1": 0.81429, - "ndcg_at_10": 0.70636, - "ndcg_at_100": 0.50703, - "ndcg_at_1000": 0.62521, - "ndcg_at_20": 0.6597, - "ndcg_at_3": 0.76457, - "ndcg_at_5": 0.74394, - "precision_at_1": 0.81429, - "precision_at_10": 0.68143, - "precision_at_100": 0.39386, - "precision_at_1000": 0.10154, - "precision_at_20": 0.62643, - "precision_at_3": 0.75238, - "precision_at_5": 0.72857, - "recall_at_1": 0.00907, - "recall_at_10": 0.06983, - "recall_at_100": 0.33142, - "recall_at_1000": 0.70794, - "recall_at_20": 0.12346, - "recall_at_3": 0.02464, - "recall_at_5": 0.03943 - } - ] - }, - "task_name": "RParisHardI2IMultiChoice" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json deleted file mode 100644 index 2fd705aca4..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "d3e0adf4e942446c04427511ccce281c86861248", - "evaluation_time": 60.67172384262085, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.41429, - "cv_recall_at_10": 0.68571, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.77143, - "cv_recall_at_3": 0.51429, - "cv_recall_at_5": 0.6, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.41429, - "map_at_1": 0.00597, - "map_at_10": 0.05664, - "map_at_100": 0.23596, - "map_at_1000": 0.3628, - "map_at_20": 0.10243, - "map_at_3": 0.01731, - "map_at_5": 0.02872, - "mrr_at_1": 0.4142857142857143, - "mrr_at_10": 0.49003968253968255, - "mrr_at_100": 0.502828543277437, - "mrr_at_1000": 0.502828543277437, - "mrr_at_20": 0.49544343814080655, - "mrr_at_3": 0.4619047619047618, - "mrr_at_5": 0.4804761904761904, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": 0.13932174607931247, - "nauc_cv_recall_at_10_max": -0.8963390270702233, - "nauc_cv_recall_at_10_std": -0.8396014384177408, - "nauc_cv_recall_at_1_diff1": 0.2060503021455104, - "nauc_cv_recall_at_1_max": -0.5281725278583708, - "nauc_cv_recall_at_1_std": -0.7573041598137233, - "nauc_cv_recall_at_20_diff1": 0.0628172996594046, - "nauc_cv_recall_at_20_max": -0.7762997236681444, - "nauc_cv_recall_at_20_std": -0.7122935543988169, - "nauc_cv_recall_at_3_diff1": 0.35040951939422, - "nauc_cv_recall_at_3_max": -0.6603886570854577, - "nauc_cv_recall_at_3_std": -0.80932236130428, - "nauc_cv_recall_at_5_diff1": 0.345291099520841, - "nauc_cv_recall_at_5_max": -0.6727939617521097, - "nauc_cv_recall_at_5_std": -0.7926260441843702, - "nauc_map_at_1000_diff1": 0.2637485190424332, - "nauc_map_at_1000_max": -0.3214602514655989, - "nauc_map_at_1000_std": -0.3732986822642579, - "nauc_map_at_100_diff1": 0.31321538209822075, - "nauc_map_at_100_max": -0.4434862100043841, - "nauc_map_at_100_std": -0.5807650790699361, - "nauc_map_at_10_diff1": 0.3581935661941152, - "nauc_map_at_10_max": -0.4787594793701487, - "nauc_map_at_10_std": -0.5695584780209748, - "nauc_map_at_1_diff1": 0.35392215263704485, - "nauc_map_at_1_max": -0.47033776664547916, - "nauc_map_at_1_std": -0.581374357285355, - "nauc_map_at_20_diff1": 0.33452504163824237, - "nauc_map_at_20_max": -0.4689043535361316, - "nauc_map_at_20_std": -0.5815981719022251, - "nauc_map_at_3_diff1": 0.3418415128674503, - "nauc_map_at_3_max": -0.48587780235367684, - "nauc_map_at_3_std": -0.5797047003739161, - "nauc_map_at_5_diff1": 0.3476248106096462, - "nauc_map_at_5_max": -0.4810067114931804, - "nauc_map_at_5_std": -0.5755673287021957, - "nauc_mrr_at_1000_diff1": 0.25879474150167453, - "nauc_mrr_at_1000_max": -0.5954028468975692, - "nauc_mrr_at_1000_std": -0.7782793535054984, - "nauc_mrr_at_100_diff1": 0.25879474150167453, - "nauc_mrr_at_100_max": -0.5954028468975692, - "nauc_mrr_at_100_std": -0.7782793535054984, - "nauc_mrr_at_10_diff1": 0.25457966886240146, - "nauc_mrr_at_10_max": -0.6072896469946196, - "nauc_mrr_at_10_std": -0.7834200101118244, - "nauc_mrr_at_1_diff1": 0.2060503021455104, - "nauc_mrr_at_1_max": -0.5281725278583708, - "nauc_mrr_at_1_std": -0.7573041598137233, - "nauc_mrr_at_20_diff1": 0.25380616623495234, - "nauc_mrr_at_20_max": -0.5999473178413075, - "nauc_mrr_at_20_std": -0.778340949953845, - "nauc_mrr_at_3_diff1": 0.276790929933105, - "nauc_mrr_at_3_max": -0.5865299456675351, - "nauc_mrr_at_3_std": -0.7820043340347346, - "nauc_mrr_at_5_diff1": 0.2734971388552989, - "nauc_mrr_at_5_max": -0.5883165176933902, - "nauc_mrr_at_5_std": -0.7787421973872966, - "nauc_ndcg_at_1000_diff1": 0.29912959985776905, - "nauc_ndcg_at_1000_max": -0.28569354071871356, - "nauc_ndcg_at_1000_std": -0.3302978581904054, - "nauc_ndcg_at_100_diff1": 0.33339067606556777, - "nauc_ndcg_at_100_max": -0.3794361926736525, - "nauc_ndcg_at_100_std": -0.4883736139930127, - "nauc_ndcg_at_10_diff1": 0.26397022259588276, - "nauc_ndcg_at_10_max": -0.5817578270842201, - "nauc_ndcg_at_10_std": -0.7307563120549173, - "nauc_ndcg_at_1_diff1": 0.2060503021455104, - "nauc_ndcg_at_1_max": -0.5281725278583708, - "nauc_ndcg_at_1_std": -0.7573041598137233, - "nauc_ndcg_at_20_diff1": 0.2623409031073529, - "nauc_ndcg_at_20_max": -0.5359230413673359, - "nauc_ndcg_at_20_std": -0.6975403451344717, - "nauc_ndcg_at_3_diff1": 0.23259798801223266, - "nauc_ndcg_at_3_max": -0.573906976133018, - "nauc_ndcg_at_3_std": -0.7515043175758601, - "nauc_ndcg_at_5_diff1": 0.2390793878627882, - "nauc_ndcg_at_5_max": -0.5712510242428932, - "nauc_ndcg_at_5_std": -0.7286974701355182, - "nauc_precision_at_1000_diff1": -0.24245498329943332, - "nauc_precision_at_1000_max": 0.3291646825531801, - "nauc_precision_at_1000_std": 0.5255011870667545, - "nauc_precision_at_100_diff1": 0.13623543103107302, - "nauc_precision_at_100_max": -0.07853246653158767, - "nauc_precision_at_100_std": -0.14296193871521895, - "nauc_precision_at_10_diff1": 0.2775400329964219, - "nauc_precision_at_10_max": -0.5859260716532688, - "nauc_precision_at_10_std": -0.7212562014378959, - "nauc_precision_at_1_diff1": 0.2060503021455104, - "nauc_precision_at_1_max": -0.5281725278583708, - "nauc_precision_at_1_std": -0.7573041598137233, - "nauc_precision_at_20_diff1": 0.26665797590030293, - "nauc_precision_at_20_max": -0.5123924265023908, - "nauc_precision_at_20_std": -0.6708786566904752, - "nauc_precision_at_3_diff1": 0.2363702775381167, - "nauc_precision_at_3_max": -0.5803676577092874, - "nauc_precision_at_3_std": -0.7427529614771804, - "nauc_precision_at_5_diff1": 0.2456529314233418, - "nauc_precision_at_5_max": -0.573319088810008, - "nauc_precision_at_5_std": -0.7135127683771583, - "nauc_recall_at_1000_diff1": 0.38214489204648355, - "nauc_recall_at_1000_max": -0.09155063706261089, - "nauc_recall_at_1000_std": -0.022172073157980317, - "nauc_recall_at_100_diff1": 0.3802641372927062, - "nauc_recall_at_100_max": -0.3842566138245613, - "nauc_recall_at_100_std": -0.5385141624854306, - "nauc_recall_at_10_diff1": 0.3783267103081628, - "nauc_recall_at_10_max": -0.488856095520999, - "nauc_recall_at_10_std": -0.58247849954068, - "nauc_recall_at_1_diff1": 0.35392215263704485, - "nauc_recall_at_1_max": -0.47033776664547916, - "nauc_recall_at_1_std": -0.581374357285355, - "nauc_recall_at_20_diff1": 0.3567752100934632, - "nauc_recall_at_20_max": -0.4784863097401327, - "nauc_recall_at_20_std": -0.5895318668296887, - "nauc_recall_at_3_diff1": 0.35482016307799724, - "nauc_recall_at_3_max": -0.492800686077188, - "nauc_recall_at_3_std": -0.5845744958893286, - "nauc_recall_at_5_diff1": 0.36822693817594376, - "nauc_recall_at_5_max": -0.4820944786320667, - "nauc_recall_at_5_std": -0.5821080266388661, - "ndcg_at_1": 0.41429, - "ndcg_at_10": 0.43851, - "ndcg_at_100": 0.44725, - "ndcg_at_1000": 0.67745, - "ndcg_at_20": 0.42956, - "ndcg_at_3": 0.42275, - "ndcg_at_5": 0.43186, - "precision_at_1": 0.41429, - "precision_at_10": 0.44429, - "precision_at_100": 0.36357, - "precision_at_1000": 0.12124, - "precision_at_20": 0.42857, - "precision_at_3": 0.42381, - "precision_at_5": 0.43714, - "recall_at_1": 0.00597, - "recall_at_10": 0.06062, - "recall_at_100": 0.35778, - "recall_at_1000": 0.83615, - "recall_at_20": 0.11218, - "recall_at_3": 0.01773, - "recall_at_5": 0.03016 - } - ] - }, - "task_name": "RParisHardI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IMultiChoice.json deleted file mode 100644 index 229344b315..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IMultiChoice.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "dataset_revision": "372c79fc823e1cebc1d55f8e0039aa239285e177", - "evaluation_time": 17.42084789276123, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "accuracy": 0.00556, - "cv_recall_at_1": 1.0, - "cv_recall_at_10": 1.0, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 1.0, - "cv_recall_at_5": 1.0, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.00556, - "map_at_1": 0.00556, - "map_at_10": 0.05258, - "map_at_100": 0.35085, - "map_at_1000": 0.57431, - "map_at_20": 0.1003, - "map_at_3": 0.01627, - "map_at_5": 0.02709, - "mrr_at_1": 1.0, - "mrr_at_10": 1.0, - "mrr_at_100": 1.0, - "mrr_at_1000": 1.0, - "mrr_at_20": 1.0, - "mrr_at_3": 1.0, - "mrr_at_5": 1.0, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": NaN, - "nauc_cv_recall_at_10_max": NaN, - "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": NaN, - "nauc_cv_recall_at_1_max": NaN, - "nauc_cv_recall_at_1_std": NaN, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": NaN, - "nauc_cv_recall_at_3_max": NaN, - "nauc_cv_recall_at_3_std": NaN, - "nauc_cv_recall_at_5_diff1": NaN, - "nauc_cv_recall_at_5_max": NaN, - "nauc_cv_recall_at_5_std": NaN, - "nauc_map_at_1000_diff1": -0.6053563345663608, - "nauc_map_at_1000_max": 0.1905977136179189, - "nauc_map_at_1000_std": 0.32921704776067334, - "nauc_map_at_100_diff1": -0.22879583528362765, - "nauc_map_at_100_max": -0.021084881609830797, - "nauc_map_at_100_std": 0.040852018876228476, - "nauc_map_at_10_diff1": 0.07357334898390998, - "nauc_map_at_10_max": -0.07446530454222411, - "nauc_map_at_10_std": 0.030841288385805625, - "nauc_map_at_1_diff1": 0.17749829551090057, - "nauc_map_at_1_max": -0.1265328999786591, - "nauc_map_at_1_std": -0.04029967299581773, - "nauc_map_at_20_diff1": 0.031681429904680244, - "nauc_map_at_20_max": -0.04579935466000076, - "nauc_map_at_20_std": 0.0600822959918182, - "nauc_map_at_3_diff1": 0.1040211006608559, - "nauc_map_at_3_max": -0.10556625686545555, - "nauc_map_at_3_std": 0.004385201213601456, - "nauc_map_at_5_diff1": 0.10349991026936102, - "nauc_map_at_5_max": -0.10412867049469088, - "nauc_map_at_5_std": 0.0066619256709396935, - "nauc_mrr_at_1000_diff1": NaN, - "nauc_mrr_at_1000_max": NaN, - "nauc_mrr_at_1000_std": NaN, - "nauc_mrr_at_100_diff1": NaN, - "nauc_mrr_at_100_max": NaN, - "nauc_mrr_at_100_std": NaN, - "nauc_mrr_at_10_diff1": NaN, - "nauc_mrr_at_10_max": NaN, - "nauc_mrr_at_10_std": NaN, - "nauc_mrr_at_1_diff1": NaN, - "nauc_mrr_at_1_max": NaN, - "nauc_mrr_at_1_std": NaN, - "nauc_mrr_at_20_diff1": NaN, - "nauc_mrr_at_20_max": NaN, - "nauc_mrr_at_20_std": NaN, - "nauc_mrr_at_3_diff1": NaN, - "nauc_mrr_at_3_max": NaN, - "nauc_mrr_at_3_std": NaN, - "nauc_mrr_at_5_diff1": NaN, - "nauc_mrr_at_5_max": NaN, - "nauc_mrr_at_5_std": NaN, - "nauc_ndcg_at_1000_diff1": -0.48417134785087246, - "nauc_ndcg_at_1000_max": 0.18405070377983077, - "nauc_ndcg_at_1000_std": 0.31559100185273364, - "nauc_ndcg_at_100_diff1": -0.7053306583324787, - "nauc_ndcg_at_100_max": 0.3110977036980108, - "nauc_ndcg_at_100_std": 0.30984291735802094, - "nauc_ndcg_at_10_diff1": -1.0871954957472245, - "nauc_ndcg_at_10_max": 0.5259679929438033, - "nauc_ndcg_at_10_std": 0.6817427956237951, - "nauc_ndcg_at_1_diff1": NaN, - "nauc_ndcg_at_1_max": NaN, - "nauc_ndcg_at_1_std": NaN, - "nauc_ndcg_at_20_diff1": -0.8880065469302778, - "nauc_ndcg_at_20_max": 0.4873208891288291, - "nauc_ndcg_at_20_std": 0.561373434731217, - "nauc_ndcg_at_3_diff1": -1.4590964058433729, - "nauc_ndcg_at_3_max": 0.49010909822533216, - "nauc_ndcg_at_3_std": 0.911235002385481, - "nauc_ndcg_at_5_diff1": -1.4171604522672352, - "nauc_ndcg_at_5_max": 0.5204958630957071, - "nauc_ndcg_at_5_std": 0.9186533100767078, - "nauc_precision_at_1000_diff1": -0.38234481360274825, - "nauc_precision_at_1000_max": 0.24781406894305885, - "nauc_precision_at_1000_std": 0.26403818342266816, - "nauc_precision_at_100_diff1": -0.6128971684961878, - "nauc_precision_at_100_max": 0.31904847150594623, - "nauc_precision_at_100_std": 0.24419530291834549, - "nauc_precision_at_10_diff1": -1.0307736255745676, - "nauc_precision_at_10_max": 0.5403562988547558, - "nauc_precision_at_10_std": 0.650531072272575, - "nauc_precision_at_1_diff1": NaN, - "nauc_precision_at_1_max": NaN, - "nauc_precision_at_1_std": NaN, - "nauc_precision_at_20_diff1": -0.836691736727432, - "nauc_precision_at_20_max": 0.4856534423121902, - "nauc_precision_at_20_std": 0.5336271166968242, - "nauc_precision_at_3_diff1": -1.4785247432306272, - "nauc_precision_at_3_max": 0.5099595393713005, - "nauc_precision_at_3_std": 0.912854030501091, - "nauc_precision_at_5_diff1": -1.4262371615312837, - "nauc_precision_at_5_max": 0.5442577030812298, - "nauc_precision_at_5_std": 0.9215686274509778, - "nauc_recall_at_1000_diff1": -0.25506002472395295, - "nauc_recall_at_1000_max": 0.10322366296428723, - "nauc_recall_at_1000_std": 0.2469529706691035, - "nauc_recall_at_100_diff1": -0.13201603693344977, - "nauc_recall_at_100_max": -0.0624401543533192, - "nauc_recall_at_100_std": -0.018309376923327982, - "nauc_recall_at_10_diff1": 0.09463286260272151, - "nauc_recall_at_10_max": -0.08130077043564286, - "nauc_recall_at_10_std": 0.017648859246839073, - "nauc_recall_at_1_diff1": 0.17749829551090057, - "nauc_recall_at_1_max": -0.1265328999786591, - "nauc_recall_at_1_std": -0.04029967299581773, - "nauc_recall_at_20_diff1": 0.05416706502986838, - "nauc_recall_at_20_max": -0.05925356554363863, - "nauc_recall_at_20_std": 0.04363713977519275, - "nauc_recall_at_3_diff1": 0.11034320172647584, - "nauc_recall_at_3_max": -0.1059689684621208, - "nauc_recall_at_3_std": -0.0006734851708733446, - "nauc_recall_at_5_diff1": 0.11748996520995737, - "nauc_recall_at_5_max": -0.10649893198419375, - "nauc_recall_at_5_std": -0.0026859671591820122, - "ndcg_at_1": 1.0, - "ndcg_at_10": 0.95692, - "ndcg_at_100": 0.81534, - "ndcg_at_1000": 0.7841, - "ndcg_at_20": 0.93009, - "ndcg_at_3": 0.97638, - "ndcg_at_5": 0.975, - "precision_at_1": 1.0, - "precision_at_10": 0.94714, - "precision_at_100": 0.76243, - "precision_at_1000": 0.18964, - "precision_at_20": 0.91357, - "precision_at_3": 0.97143, - "precision_at_5": 0.97143, - "recall_at_1": 0.00556, - "recall_at_10": 0.05313, - "recall_at_100": 0.38141, - "recall_at_1000": 0.79881, - "recall_at_20": 0.10199, - "recall_at_3": 0.01632, - "recall_at_5": 0.02723 - } - ] - }, - "task_name": "RParisMediumI2IMultiChoice" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json deleted file mode 100644 index 3f2c633d86..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "3d959815e102785efd628170281f1e65561b03d2", - "evaluation_time": 79.89404535293579, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 1.0, - "cv_recall_at_10": 1.0, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 1.0, - "cv_recall_at_5": 1.0, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 1.0, - "map_at_1": 0.00556, - "map_at_10": 0.05371, - "map_at_100": 0.40433, - "map_at_1000": 0.65468, - "map_at_20": 0.10419, - "map_at_3": 0.01659, - "map_at_5": 0.02755, - "mrr_at_1": 1.0, - "mrr_at_10": 1.0, - "mrr_at_100": 1.0, - "mrr_at_1000": 1.0, - "mrr_at_20": 1.0, - "mrr_at_3": 1.0, - "mrr_at_5": 1.0, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": NaN, - "nauc_cv_recall_at_10_max": NaN, - "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": NaN, - "nauc_cv_recall_at_1_max": NaN, - "nauc_cv_recall_at_1_std": NaN, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": NaN, - "nauc_cv_recall_at_3_max": NaN, - "nauc_cv_recall_at_3_std": NaN, - "nauc_cv_recall_at_5_diff1": NaN, - "nauc_cv_recall_at_5_max": NaN, - "nauc_cv_recall_at_5_std": NaN, - "nauc_map_at_1000_diff1": -0.37033768465220357, - "nauc_map_at_1000_max": 0.2394188102952743, - "nauc_map_at_1000_std": 0.35189302913259624, - "nauc_map_at_100_diff1": -0.0008350440689574251, - "nauc_map_at_100_max": 0.06810001616925912, - "nauc_map_at_100_std": -0.08292235454963914, - "nauc_map_at_10_diff1": 0.137914576743156, - "nauc_map_at_10_max": -0.09262735632646613, - "nauc_map_at_10_std": -0.19844704585800624, - "nauc_map_at_1_diff1": 0.18523999970603441, - "nauc_map_at_1_max": -0.1265328999786591, - "nauc_map_at_1_std": -0.230231416885371, - "nauc_map_at_20_diff1": 0.08800700279626224, - "nauc_map_at_20_max": -0.05754868196581078, - "nauc_map_at_20_std": -0.1590299889904529, - "nauc_map_at_3_diff1": 0.1740453429583345, - "nauc_map_at_3_max": -0.11523868645571757, - "nauc_map_at_3_std": -0.22138850054902404, - "nauc_map_at_5_diff1": 0.1634612763199569, - "nauc_map_at_5_max": -0.11473486161026644, - "nauc_map_at_5_std": -0.21287472489660672, - "nauc_mrr_at_1000_diff1": NaN, - "nauc_mrr_at_1000_max": NaN, - "nauc_mrr_at_1000_std": NaN, - "nauc_mrr_at_100_diff1": NaN, - "nauc_mrr_at_100_max": NaN, - "nauc_mrr_at_100_std": NaN, - "nauc_mrr_at_10_diff1": NaN, - "nauc_mrr_at_10_max": NaN, - "nauc_mrr_at_10_std": NaN, - "nauc_mrr_at_1_diff1": NaN, - "nauc_mrr_at_1_max": NaN, - "nauc_mrr_at_1_std": NaN, - "nauc_mrr_at_20_diff1": NaN, - "nauc_mrr_at_20_max": NaN, - "nauc_mrr_at_20_std": NaN, - "nauc_mrr_at_3_diff1": NaN, - "nauc_mrr_at_3_max": NaN, - "nauc_mrr_at_3_std": NaN, - "nauc_mrr_at_5_diff1": NaN, - "nauc_mrr_at_5_max": NaN, - "nauc_mrr_at_5_std": NaN, - "nauc_ndcg_at_1000_diff1": -0.22205760084759807, - "nauc_ndcg_at_1000_max": 0.10796195890373474, - "nauc_ndcg_at_1000_std": 0.26231183916047035, - "nauc_ndcg_at_100_diff1": -0.6398984813885491, - "nauc_ndcg_at_100_max": 0.4842652819900039, - "nauc_ndcg_at_100_std": 0.5107149500380104, - "nauc_ndcg_at_10_diff1": -0.9371088266265549, - "nauc_ndcg_at_10_max": 0.6084341828001775, - "nauc_ndcg_at_10_std": 0.5430117586762949, - "nauc_ndcg_at_1_diff1": NaN, - "nauc_ndcg_at_1_max": NaN, - "nauc_ndcg_at_1_std": NaN, - "nauc_ndcg_at_20_diff1": -0.9268541652920639, - "nauc_ndcg_at_20_max": 0.597237167140367, - "nauc_ndcg_at_20_std": 0.5649258862662437, - "nauc_ndcg_at_3_diff1": -0.8999624476329307, - "nauc_ndcg_at_3_max": 1.0, - "nauc_ndcg_at_3_std": 0.6408995549373118, - "nauc_ndcg_at_5_diff1": -1.1761404982018615, - "nauc_ndcg_at_5_max": 0.8247853976989885, - "nauc_ndcg_at_5_std": 0.7965821461413944, - "nauc_precision_at_1000_diff1": -0.31783317612679535, - "nauc_precision_at_1000_max": 0.1930712065318718, - "nauc_precision_at_1000_std": 0.4208950298526623, - "nauc_precision_at_100_diff1": -0.539592973425393, - "nauc_precision_at_100_max": 0.4460055144887294, - "nauc_precision_at_100_std": 0.4485043771568464, - "nauc_precision_at_10_diff1": -0.9108685686074025, - "nauc_precision_at_10_max": 0.5802094720880185, - "nauc_precision_at_10_std": 0.5333427372430083, - "nauc_precision_at_1_diff1": NaN, - "nauc_precision_at_1_max": NaN, - "nauc_precision_at_1_std": NaN, - "nauc_precision_at_20_diff1": -0.9101954663166522, - "nauc_precision_at_20_max": 0.5867536982510865, - "nauc_precision_at_20_std": 0.5645109794600713, - "nauc_precision_at_3_diff1": -0.8266417678182514, - "nauc_precision_at_3_max": 1.0, - "nauc_precision_at_3_std": 0.6095549330843374, - "nauc_precision_at_5_diff1": -1.1852629940865032, - "nauc_precision_at_5_max": 0.8047774665421796, - "nauc_precision_at_5_std": 0.8047774665421796, - "nauc_recall_at_1000_diff1": -0.01696359441028296, - "nauc_recall_at_1000_max": -0.06214643338578423, - "nauc_recall_at_1000_std": 0.12422436451417722, - "nauc_recall_at_100_diff1": 0.05923389523513676, - "nauc_recall_at_100_max": 0.023919554920948157, - "nauc_recall_at_100_std": -0.13407391685199146, - "nauc_recall_at_10_diff1": 0.14007744935778899, - "nauc_recall_at_10_max": -0.09598247717834782, - "nauc_recall_at_10_std": -0.19995201976434931, - "nauc_recall_at_1_diff1": 0.18523999970603441, - "nauc_recall_at_1_max": -0.1265328999786591, - "nauc_recall_at_1_std": -0.230231416885371, - "nauc_recall_at_20_diff1": 0.09732767803178609, - "nauc_recall_at_20_max": -0.06812531189724261, - "nauc_recall_at_20_std": -0.16830905580000974, - "nauc_recall_at_3_diff1": 0.1740453429583345, - "nauc_recall_at_3_max": -0.11523868645571757, - "nauc_recall_at_3_std": -0.22138850054902404, - "nauc_recall_at_5_diff1": 0.16407715576816015, - "nauc_recall_at_5_max": -0.11553420065887736, - "nauc_recall_at_5_std": -0.2140734184274355, - "ndcg_at_1": 1.0, - "ndcg_at_10": 0.97012, - "ndcg_at_100": 0.87092, - "ndcg_at_1000": 0.84355, - "ndcg_at_20": 0.95286, - "ndcg_at_3": 0.98907, - "ndcg_at_5": 0.98605, - "precision_at_1": 1.0, - "precision_at_10": 0.96143, - "precision_at_100": 0.821, - "precision_at_1000": 0.20586, - "precision_at_20": 0.94143, - "precision_at_3": 0.98571, - "precision_at_5": 0.98286, - "recall_at_1": 0.00556, - "recall_at_10": 0.05391, - "recall_at_100": 0.42614, - "recall_at_1000": 0.86053, - "recall_at_20": 0.10535, - "recall_at_3": 0.01659, - "recall_at_5": 0.02758 - } - ] - }, - "task_name": "RParisMediumI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RenderedSST2.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RenderedSST2.json deleted file mode 100644 index b293d8e5ff..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RenderedSST2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", - "evaluation_time": 8.718077659606934, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "accuracy": 0.5848434925864909, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5848434925864909 - } - ] - }, - "task_name": "RenderedSST2" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json deleted file mode 100644 index 77f92463bd..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SOPI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "4ac3894bdabee3c3938cf0133ab991c4b501891d", - "evaluation_time": 1411.4596996307373, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.4856, - "cv_recall_at_10": 0.65892, - "cv_recall_at_100": 0.80265, - "cv_recall_at_1000": 0.91906, - "cv_recall_at_20": 0.70484, - "cv_recall_at_3": 0.57481, - "cv_recall_at_5": 0.61136, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4856, - "map_at_1": 0.18847, - "map_at_10": 0.37878, - "map_at_100": 0.39701, - "map_at_1000": 0.39955, - "map_at_20": 0.38884, - "map_at_3": 0.31574, - "map_at_5": 0.35297, - "mrr_at_1": 0.9996334952062839, - "mrr_at_10": 0.9998132769138076, - "mrr_at_100": 0.9998132769138076, - "mrr_at_1000": 0.9998132769138076, - "mrr_at_20": 0.9998132769138076, - "mrr_at_3": 0.999811194500207, - "mrr_at_5": 0.9998132769138076, - "nauc_cv_recall_at_1000_diff1": -0.4588738282028226, - "nauc_cv_recall_at_1000_max": -0.00045154026330132926, - "nauc_cv_recall_at_1000_std": 0.13389839300845494, - "nauc_cv_recall_at_100_diff1": -0.40349410035175093, - "nauc_cv_recall_at_100_max": -0.0015857946226738907, - "nauc_cv_recall_at_100_std": 0.1564430560934902, - "nauc_cv_recall_at_10_diff1": -0.4005181992520635, - "nauc_cv_recall_at_10_max": -0.003957796532778069, - "nauc_cv_recall_at_10_std": 0.14566052153014458, - "nauc_cv_recall_at_1_diff1": -0.40652510568056555, - "nauc_cv_recall_at_1_max": -0.00544324116729814, - "nauc_cv_recall_at_1_std": 0.11520889475416408, - "nauc_cv_recall_at_20_diff1": -0.3973805526446551, - "nauc_cv_recall_at_20_max": -0.0046404467731546205, - "nauc_cv_recall_at_20_std": 0.14736528921128944, - "nauc_cv_recall_at_3_diff1": -0.4068997028240004, - "nauc_cv_recall_at_3_max": -0.00581379693442588, - "nauc_cv_recall_at_3_std": 0.13232980933178856, - "nauc_cv_recall_at_5_diff1": -0.4027824500878659, - "nauc_cv_recall_at_5_max": -0.00622124305658329, - "nauc_cv_recall_at_5_std": 0.13795375991899358, - "nauc_map_at_1000_diff1": -0.2815134589062327, - "nauc_map_at_1000_max": -0.009309150035044157, - "nauc_map_at_1000_std": 0.21821074035964452, - "nauc_map_at_100_diff1": -0.2802916020865774, - "nauc_map_at_100_max": -0.00929273233548749, - "nauc_map_at_100_std": 0.21738215325378304, - "nauc_map_at_10_diff1": -0.2714998577314571, - "nauc_map_at_10_max": -0.00972009151554314, - "nauc_map_at_10_std": 0.20451609552118097, - "nauc_map_at_1_diff1": 0.006605172707482505, - "nauc_map_at_1_max": -0.00535293182577402, - "nauc_map_at_1_std": 0.08912323354604411, - "nauc_map_at_20_diff1": -0.276631619640432, - "nauc_map_at_20_max": -0.009275901369492237, - "nauc_map_at_20_std": 0.21273445874298258, - "nauc_map_at_3_diff1": -0.2144365987666858, - "nauc_map_at_3_max": -0.008565184161123921, - "nauc_map_at_3_std": 0.1490404208758112, - "nauc_map_at_5_diff1": -0.2525988859250142, - "nauc_map_at_5_max": -0.00899732355635905, - "nauc_map_at_5_std": 0.18031989165161078, - "nauc_mrr_at_1000_diff1": 1.0, - "nauc_mrr_at_1000_max": 0.17398421274750922, - "nauc_mrr_at_1000_std": -0.3793244588442443, - "nauc_mrr_at_100_diff1": 1.0, - "nauc_mrr_at_100_max": 0.17398421274750922, - "nauc_mrr_at_100_std": -0.3793244588442443, - "nauc_mrr_at_10_diff1": 1.0, - "nauc_mrr_at_10_max": 0.17398421274750922, - "nauc_mrr_at_10_std": -0.3793244588442443, - "nauc_mrr_at_1_diff1": 1.0, - "nauc_mrr_at_1_max": 0.17317138790943362, - "nauc_mrr_at_1_std": -0.36469583799829147, - "nauc_mrr_at_20_diff1": 1.0, - "nauc_mrr_at_20_max": 0.17398421274750922, - "nauc_mrr_at_20_std": -0.3793244588442443, - "nauc_mrr_at_3_diff1": 1.0, - "nauc_mrr_at_3_max": 0.17341951004597636, - "nauc_mrr_at_3_std": -0.3878434792194687, - "nauc_mrr_at_5_diff1": 1.0, - "nauc_mrr_at_5_max": 0.17398421274750922, - "nauc_mrr_at_5_std": -0.3793244588442443, - "nauc_ndcg_at_1000_diff1": -0.3092675026331808, - "nauc_ndcg_at_1000_max": -0.009597931782047127, - "nauc_ndcg_at_1000_std": 0.23456527611896652, - "nauc_ndcg_at_100_diff1": -0.29597233628106967, - "nauc_ndcg_at_100_max": -0.009309685183289425, - "nauc_ndcg_at_100_std": 0.22752615017083713, - "nauc_ndcg_at_10_diff1": -0.2930380020320867, - "nauc_ndcg_at_10_max": -0.01000547366692856, - "nauc_ndcg_at_10_std": 0.20753291775628463, - "nauc_ndcg_at_1_diff1": 1.0, - "nauc_ndcg_at_1_max": 0.17610023993447146, - "nauc_ndcg_at_1_std": -0.36469583799829147, - "nauc_ndcg_at_20_diff1": -0.28823218777084936, - "nauc_ndcg_at_20_max": -0.00923331182370813, - "nauc_ndcg_at_20_std": 0.215531645591675, - "nauc_ndcg_at_3_diff1": -0.3656155045971337, - "nauc_ndcg_at_3_max": -0.007681435647815146, - "nauc_ndcg_at_3_std": 0.14037876964394946, - "nauc_ndcg_at_5_diff1": -0.3365107912821576, - "nauc_ndcg_at_5_max": -0.008614755602549906, - "nauc_ndcg_at_5_std": 0.18007814023447868, - "nauc_precision_at_1000_diff1": -0.19806317219535188, - "nauc_precision_at_1000_max": -0.0015221284698821193, - "nauc_precision_at_1000_std": 0.10530172591311254, - "nauc_precision_at_100_diff1": -0.2572749977303199, - "nauc_precision_at_100_max": -0.003787347697600317, - "nauc_precision_at_100_std": 0.1656157857683021, - "nauc_precision_at_10_diff1": -0.3101437993491914, - "nauc_precision_at_10_max": -0.007420244456215511, - "nauc_precision_at_10_std": 0.17668167529658718, - "nauc_precision_at_1_diff1": 1.0, - "nauc_precision_at_1_max": 0.17610023993447146, - "nauc_precision_at_1_std": -0.36469583799829147, - "nauc_precision_at_20_diff1": -0.29472889748829945, - "nauc_precision_at_20_max": -0.005520841437776455, - "nauc_precision_at_20_std": 0.18138281023915606, - "nauc_precision_at_3_diff1": -0.35841782500948777, - "nauc_precision_at_3_max": -0.00748476584740033, - "nauc_precision_at_3_std": 0.12872707729510002, - "nauc_precision_at_5_diff1": -0.3295916303527328, - "nauc_precision_at_5_max": -0.007145664789792643, - "nauc_precision_at_5_std": 0.15297931876842116, - "nauc_recall_at_1000_diff1": -0.25792280904882936, - "nauc_recall_at_1000_max": -0.00797884188493623, - "nauc_recall_at_1000_std": 0.24244025939864458, - "nauc_recall_at_100_diff1": -0.2524975921657061, - "nauc_recall_at_100_max": -0.007586553109911903, - "nauc_recall_at_100_std": 0.23439228371650886, - "nauc_recall_at_10_diff1": -0.2562100752922837, - "nauc_recall_at_10_max": -0.009867846823865567, - "nauc_recall_at_10_std": 0.20691424138363798, - "nauc_recall_at_1_diff1": 0.006605172707482505, - "nauc_recall_at_1_max": -0.00535293182577402, - "nauc_recall_at_1_std": 0.08912323354604411, - "nauc_recall_at_20_diff1": -0.2560836034546459, - "nauc_recall_at_20_max": -0.007754724440480409, - "nauc_recall_at_20_std": 0.21910103605653863, - "nauc_recall_at_3_diff1": -0.21039105024165025, - "nauc_recall_at_3_max": -0.008374958963214659, - "nauc_recall_at_3_std": 0.1489992487923535, - "nauc_recall_at_5_diff1": -0.242995017130598, - "nauc_recall_at_5_max": -0.00909414443674345, - "nauc_recall_at_5_std": 0.18059416206675072, - "ndcg_at_1": 0.99963, - "ndcg_at_10": 0.52441, - "ndcg_at_100": 0.5623, - "ndcg_at_1000": 0.59803, - "ndcg_at_20": 0.53321, - "ndcg_at_3": 0.69421, - "ndcg_at_5": 0.59023, - "precision_at_1": 0.99963, - "precision_at_10": 0.24637, - "precision_at_100": 0.03343, - "precision_at_1000": 0.00451, - "precision_at_20": 0.13636, - "precision_at_3": 0.59241, - "precision_at_5": 0.42271, - "recall_at_1": 0.18847, - "recall_at_10": 0.41136, - "recall_at_100": 0.52984, - "recall_at_1000": 0.6822, - "recall_at_20": 0.44681, - "recall_at_3": 0.31926, - "recall_at_5": 0.3657 - } - ] - }, - "task_name": "SOPI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10.json deleted file mode 100644 index b11a0d7f99..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "49ae7f94508f7feae62baf836db284306eab0b0f", - "evaluation_time": 72.2589225769043, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.9773, - "f1": 0.9772715792109945, - "f1_weighted": 0.9772715792109942, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9773, - "scores_per_experiment": [ - { - "accuracy": 0.977, - "f1": 0.9769671771353311, - "f1_weighted": 0.9769671771353311 - }, - { - "accuracy": 0.97825, - "f1": 0.9781918264450462, - "f1_weighted": 0.9781918264450461 - }, - { - "accuracy": 0.97675, - "f1": 0.9767550808626206, - "f1_weighted": 0.9767550808626204 - }, - { - "accuracy": 0.978375, - "f1": 0.9783827689066221, - "f1_weighted": 0.978382768906622 - }, - { - "accuracy": 0.976125, - "f1": 0.976061042705352, - "f1_weighted": 0.9760610427053519 - } - ] - } - ] - }, - "task_name": "STL10" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10ZeroShot.json deleted file mode 100644 index a703124990..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STL10ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "49ae7f94508f7feae62baf836db284306eab0b0f", - "evaluation_time": 34.58206105232239, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.973, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.973 - } - ] - }, - "task_name": "STL10ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS12VisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS12VisualSTS.json deleted file mode 100644 index 76c2dea8f9..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS12VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "820c25edfba736f3789201b2476208cc62c2ccb9", - "evaluation_time": 24.613746643066406, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.4927841964614398, - "cosine_spearman": 0.5381410964527167, - "euclidean_pearson": 0.5376663970806768, - "euclidean_spearman": 0.5680000317402323, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5381410964527167, - "manhattan_pearson": 0.5482646153361562, - "manhattan_spearman": 0.5781763600741524, - "pearson": 0.4927841964614398, - "spearman": 0.5381410964527167 - } - ] - }, - "task_name": "STS12VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS13VisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS13VisualSTS.json deleted file mode 100644 index 1bbbf70ea2..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS13VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "561ee9ca47ff3e4a657283c59416deca8dc169f2", - "evaluation_time": 12.163323640823364, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.5190223767985211, - "cosine_spearman": 0.5250273114471853, - "euclidean_pearson": 0.5926139149679672, - "euclidean_spearman": 0.5939351043456808, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5250273114471853, - "manhattan_pearson": 0.5985603216705444, - "manhattan_spearman": 0.5996096714526287, - "pearson": 0.5190223767985211, - "spearman": 0.5250273114471853 - } - ] - }, - "task_name": "STS13VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS14VisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS14VisualSTS.json deleted file mode 100644 index 798c057d23..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS14VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "824e95e45471024a684b901e0645579ffd9ca288", - "evaluation_time": 29.798357725143433, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.4329815257276047, - "cosine_spearman": 0.43691481964575507, - "euclidean_pearson": 0.4900456996545259, - "euclidean_spearman": 0.48614470617248245, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.43691481964575507, - "manhattan_pearson": 0.4968577536111171, - "manhattan_spearman": 0.4933346299127321, - "pearson": 0.4329815257276047, - "spearman": 0.43691481964575507 - } - ] - }, - "task_name": "STS14VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS15VisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS15VisualSTS.json deleted file mode 100644 index a9ee924256..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS15VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "1f8d08d9b9daac7118dfdefeb94b0aac4baf2e5f", - "evaluation_time": 24.508333921432495, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.5863061376001493, - "cosine_spearman": 0.5956425557069953, - "euclidean_pearson": 0.6310648275006788, - "euclidean_spearman": 0.6394200753868249, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5956425557069953, - "manhattan_pearson": 0.6352520023800736, - "manhattan_spearman": 0.644021485552327, - "pearson": 0.5863061376001493, - "spearman": 0.5956425557069953 - } - ] - }, - "task_name": "STS15VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS16VisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS16VisualSTS.json deleted file mode 100644 index 5f8705554d..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS16VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "fc354f19598af93f32c0af1b94046ffdeaacde15", - "evaluation_time": 10.809600830078125, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.516953879631008, - "cosine_spearman": 0.5301267122042563, - "euclidean_pearson": 0.5692230389664796, - "euclidean_spearman": 0.5712672683254012, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5301267122042563, - "manhattan_pearson": 0.572549410410549, - "manhattan_spearman": 0.5746695336316615, - "pearson": 0.516953879631008, - "spearman": 0.5301267122042563 - } - ] - }, - "task_name": "STS16VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS17MultilingualVisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS17MultilingualVisualSTS.json deleted file mode 100644 index 8fe733385b..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STS17MultilingualVisualSTS.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", - "evaluation_time": 52.30275201797485, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.04942380420066665, - "cosine_spearman": 0.18100901146250903, - "euclidean_pearson": 0.10583208785240501, - "euclidean_spearman": 0.18141227558130985, - "hf_subset": "ko-ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.18100901146250903, - "manhattan_pearson": 0.10771252455967777, - "manhattan_spearman": 0.18258653950535597, - "pearson": 0.04942380420066665, - "spearman": 0.18100901146250903 - }, - { - "cosine_pearson": 0.232334349420455, - "cosine_spearman": 0.28303067418377864, - "euclidean_pearson": 0.3164072647785108, - "euclidean_spearman": 0.2888667664816579, - "hf_subset": "ar-ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.28303067418377864, - "manhattan_pearson": 0.30513106540241797, - "manhattan_spearman": 0.2763182836682906, - "pearson": 0.232334349420455, - "spearman": 0.28303067418377864 - }, - { - "cosine_pearson": 0.09559095557157546, - "cosine_spearman": 0.08247355658408094, - "euclidean_pearson": 0.08598225245654229, - "euclidean_spearman": 0.07747906144678571, - "hf_subset": "en-ar", - "languages": [ - "eng-Latn", - "ara-Arab" - ], - "main_score": 0.08247355658408094, - "manhattan_pearson": 0.07391796156253816, - "manhattan_spearman": 0.07205903670762905, - "pearson": 0.09559095557157546, - "spearman": 0.08247355658408094 - }, - { - "cosine_pearson": 0.2489932748844445, - "cosine_spearman": 0.2215127409184059, - "euclidean_pearson": 0.25256785548840116, - "euclidean_spearman": 0.20598504223777503, - "hf_subset": "en-de", - "languages": [ - "eng-Latn", - "deu-Latn" - ], - "main_score": 0.2215127409184059, - "manhattan_pearson": 0.25919799382395436, - "manhattan_spearman": 0.21472583541264484, - "pearson": 0.2489932748844445, - "spearman": 0.2215127409184059 - }, - { - "cosine_pearson": 0.7018077649163341, - "cosine_spearman": 0.710059231126493, - "euclidean_pearson": 0.7476029594905387, - "euclidean_spearman": 0.7415144009008715, - "hf_subset": "en-en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.710059231126493, - "manhattan_pearson": 0.75001718011639, - "manhattan_spearman": 0.7459299640830576, - "pearson": 0.7018077649163341, - "spearman": 0.710059231126493 - }, - { - "cosine_pearson": 0.20500049148877042, - "cosine_spearman": 0.1797047388493705, - "euclidean_pearson": 0.15313682625154354, - "euclidean_spearman": 0.11880071563811015, - "hf_subset": "en-tr", - "languages": [ - "eng-Latn", - "tur-Latn" - ], - "main_score": 0.1797047388493705, - "manhattan_pearson": 0.15802674968664282, - "manhattan_spearman": 0.12885078297440422, - "pearson": 0.20500049148877042, - "spearman": 0.1797047388493705 - }, - { - "cosine_pearson": 0.12063182375204703, - "cosine_spearman": 0.12152613636208974, - "euclidean_pearson": 0.11799595218875049, - "euclidean_spearman": 0.12272133423994501, - "hf_subset": "es-en", - "languages": [ - "spa-Latn", - "eng-Latn" - ], - "main_score": 0.12152613636208974, - "manhattan_pearson": 0.13792614785112425, - "manhattan_spearman": 0.14357713355532858, - "pearson": 0.12063182375204703, - "spearman": 0.12152613636208974 - }, - { - "cosine_pearson": 0.44396659087851953, - "cosine_spearman": 0.4756388184990541, - "euclidean_pearson": 0.5318750782307595, - "euclidean_spearman": 0.4929346496915896, - "hf_subset": "es-es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.4756388184990541, - "manhattan_pearson": 0.5387926098506205, - "manhattan_spearman": 0.5017886694568034, - "pearson": 0.44396659087851953, - "spearman": 0.4756388184990541 - }, - { - "cosine_pearson": 0.20645976694323198, - "cosine_spearman": 0.19479103027395156, - "euclidean_pearson": 0.2831839920669268, - "euclidean_spearman": 0.2661811577393992, - "hf_subset": "fr-en", - "languages": [ - "fra-Latn", - "eng-Latn" - ], - "main_score": 0.19479103027395156, - "manhattan_pearson": 0.2876400969391301, - "manhattan_spearman": 0.2737037982439158, - "pearson": 0.20645976694323198, - "spearman": 0.19479103027395156 - }, - { - "cosine_pearson": 0.23084128283452515, - "cosine_spearman": 0.2273932393567347, - "euclidean_pearson": 0.3150620968342629, - "euclidean_spearman": 0.29362092114125965, - "hf_subset": "it-en", - "languages": [ - "ita-Latn", - "eng-Latn" - ], - "main_score": 0.2273932393567347, - "manhattan_pearson": 0.3143620843939853, - "manhattan_spearman": 0.2921852000229986, - "pearson": 0.23084128283452515, - "spearman": 0.2273932393567347 - }, - { - "cosine_pearson": 0.24590072426750986, - "cosine_spearman": 0.2504966252659302, - "euclidean_pearson": 0.2797988081265482, - "euclidean_spearman": 0.25981516645706415, - "hf_subset": "nl-en", - "languages": [ - "nld-Latn", - "eng-Latn" - ], - "main_score": 0.2504966252659302, - "manhattan_pearson": 0.30580381481568664, - "manhattan_spearman": 0.2951646576903725, - "pearson": 0.24590072426750986, - "spearman": 0.2504966252659302 - } - ] - }, - "task_name": "STS17MultilingualVisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STSBenchmarkMultilingualVisualSTS.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STSBenchmarkMultilingualVisualSTS.json deleted file mode 100644 index 90c2bf295e..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/STSBenchmarkMultilingualVisualSTS.json +++ /dev/null @@ -1,313 +0,0 @@ -{ - "dataset_revision": "9f1ab21f17f497974996ab74b3ff911165a7dbf9", - "evaluation_time": 227.7307116985321, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "dev": [ - { - "cosine_pearson": 0.5272718426731863, - "cosine_spearman": 0.537680296325815, - "euclidean_pearson": 0.6007770904077094, - "euclidean_spearman": 0.6010636781456551, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.537680296325815, - "manhattan_pearson": 0.6100414071389302, - "manhattan_spearman": 0.6089001105999241, - "pearson": 0.5272718426731863, - "spearman": 0.537680296325815 - }, - { - "cosine_pearson": 0.2676685900225375, - "cosine_spearman": 0.29324273637437337, - "euclidean_pearson": 0.3597775948582906, - "euclidean_spearman": 0.34396994265808895, - "hf_subset": "de", - "languages": [ - "deu-Latn" - ], - "main_score": 0.29324273637437337, - "manhattan_pearson": 0.36893705209216576, - "manhattan_spearman": 0.3506676102021138, - "pearson": 0.2676685900225375, - "spearman": 0.29324273637437337 - }, - { - "cosine_pearson": 0.17206470202776458, - "cosine_spearman": 0.20995341198163026, - "euclidean_pearson": 0.26850806204192573, - "euclidean_spearman": 0.2644582230111772, - "hf_subset": "es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.20995341198163026, - "manhattan_pearson": 0.2776953897517844, - "manhattan_spearman": 0.27065854111745996, - "pearson": 0.17206470202776458, - "spearman": 0.20995341198163026 - }, - { - "cosine_pearson": 0.23513332207947069, - "cosine_spearman": 0.2548454596994308, - "euclidean_pearson": 0.3185683710098647, - "euclidean_spearman": 0.2921945764011797, - "hf_subset": "fr", - "languages": [ - "fra-Latn" - ], - "main_score": 0.2548454596994308, - "manhattan_pearson": 0.3263260541428151, - "manhattan_spearman": 0.30069700086416457, - "pearson": 0.23513332207947069, - "spearman": 0.2548454596994308 - }, - { - "cosine_pearson": 0.17437752754674288, - "cosine_spearman": 0.2022199564598799, - "euclidean_pearson": 0.25458208115505687, - "euclidean_spearman": 0.24701419915645423, - "hf_subset": "it", - "languages": [ - "ita-Latn" - ], - "main_score": 0.2022199564598799, - "manhattan_pearson": 0.2643892014665793, - "manhattan_spearman": 0.25371535347866575, - "pearson": 0.17437752754674288, - "spearman": 0.2022199564598799 - }, - { - "cosine_pearson": 0.20169834707998577, - "cosine_spearman": 0.2315465811887652, - "euclidean_pearson": 0.2856781367609151, - "euclidean_spearman": 0.27242351901545375, - "hf_subset": "nl", - "languages": [ - "nld-Latn" - ], - "main_score": 0.2315465811887652, - "manhattan_pearson": 0.28888139925688416, - "manhattan_spearman": 0.27547663872028577, - "pearson": 0.20169834707998577, - "spearman": 0.2315465811887652 - }, - { - "cosine_pearson": 0.2812070785959071, - "cosine_spearman": 0.2957475897374985, - "euclidean_pearson": 0.3611073985811877, - "euclidean_spearman": 0.34699149889488523, - "hf_subset": "pl", - "languages": [ - "pol-Latn" - ], - "main_score": 0.2957475897374985, - "manhattan_pearson": 0.37331542624423614, - "manhattan_spearman": 0.3583753122357121, - "pearson": 0.2812070785959071, - "spearman": 0.2957475897374985 - }, - { - "cosine_pearson": 0.13670550978570914, - "cosine_spearman": 0.14557883052048473, - "euclidean_pearson": 0.20315124815132346, - "euclidean_spearman": 0.19102861115778288, - "hf_subset": "pt", - "languages": [ - "por-Latn" - ], - "main_score": 0.14557883052048473, - "manhattan_pearson": 0.20680263501202406, - "manhattan_spearman": 0.19173115079101838, - "pearson": 0.13670550978570914, - "spearman": 0.14557883052048473 - }, - { - "cosine_pearson": 0.12452547178134374, - "cosine_spearman": 0.18614027821316195, - "euclidean_pearson": 0.20199310870295473, - "euclidean_spearman": 0.19732036928609661, - "hf_subset": "ru", - "languages": [ - "rus-Cyrl" - ], - "main_score": 0.18614027821316195, - "manhattan_pearson": 0.20622105611449384, - "manhattan_spearman": 0.1996718640479634, - "pearson": 0.12452547178134374, - "spearman": 0.18614027821316195 - }, - { - "cosine_pearson": 0.07938008146809811, - "cosine_spearman": 0.18403175659612492, - "euclidean_pearson": 0.14422004233703312, - "euclidean_spearman": 0.18924749482955278, - "hf_subset": "zh", - "languages": [ - "cmn-Hans" - ], - "main_score": 0.18403175659612492, - "manhattan_pearson": 0.13883666866119507, - "manhattan_spearman": 0.18413299674895794, - "pearson": 0.07938008146809811, - "spearman": 0.18403175659612492 - } - ], - "test": [ - { - "cosine_pearson": 0.45584849857229814, - "cosine_spearman": 0.47171211498989196, - "euclidean_pearson": 0.5493184633612522, - "euclidean_spearman": 0.542186090785119, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.47171211498989196, - "manhattan_pearson": 0.558028494795885, - "manhattan_spearman": 0.5508031599036809, - "pearson": 0.45584849857229814, - "spearman": 0.47171211498989196 - }, - { - "cosine_pearson": 0.2736341916716048, - "cosine_spearman": 0.294077550139935, - "euclidean_pearson": 0.34400702776748815, - "euclidean_spearman": 0.33227797643753454, - "hf_subset": "de", - "languages": [ - "deu-Latn" - ], - "main_score": 0.294077550139935, - "manhattan_pearson": 0.35534149649541, - "manhattan_spearman": 0.33973816803083806, - "pearson": 0.2736341916716048, - "spearman": 0.294077550139935 - }, - { - "cosine_pearson": 0.1787373679481151, - "cosine_spearman": 0.23429932660594835, - "euclidean_pearson": 0.2730687899228438, - "euclidean_spearman": 0.27917737631797584, - "hf_subset": "es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.23429932660594835, - "manhattan_pearson": 0.28368410011635753, - "manhattan_spearman": 0.28781260934323166, - "pearson": 0.1787373679481151, - "spearman": 0.23429932660594835 - }, - { - "cosine_pearson": 0.24359460439610567, - "cosine_spearman": 0.2684595434289802, - "euclidean_pearson": 0.31871428107193633, - "euclidean_spearman": 0.29373936931462125, - "hf_subset": "fr", - "languages": [ - "fra-Latn" - ], - "main_score": 0.2684595434289802, - "manhattan_pearson": 0.32834491701105845, - "manhattan_spearman": 0.3020391568058281, - "pearson": 0.24359460439610567, - "spearman": 0.2684595434289802 - }, - { - "cosine_pearson": 0.17151725238870894, - "cosine_spearman": 0.20546160529404361, - "euclidean_pearson": 0.2528804508579695, - "euclidean_spearman": 0.24624455796714606, - "hf_subset": "it", - "languages": [ - "ita-Latn" - ], - "main_score": 0.20546160529404361, - "manhattan_pearson": 0.264170451940936, - "manhattan_spearman": 0.25410292788867, - "pearson": 0.17151725238870894, - "spearman": 0.20546160529404361 - }, - { - "cosine_pearson": 0.13346252416405158, - "cosine_spearman": 0.1905489071972136, - "euclidean_pearson": 0.21314765423638377, - "euclidean_spearman": 0.2125972363673462, - "hf_subset": "nl", - "languages": [ - "nld-Latn" - ], - "main_score": 0.1905489071972136, - "manhattan_pearson": 0.2222995027779861, - "manhattan_spearman": 0.2185386986595845, - "pearson": 0.13346252416405158, - "spearman": 0.1905489071972136 - }, - { - "cosine_pearson": 0.29193603521517864, - "cosine_spearman": 0.30368819951978826, - "euclidean_pearson": 0.3540877734657154, - "euclidean_spearman": 0.3343842768900143, - "hf_subset": "pl", - "languages": [ - "pol-Latn" - ], - "main_score": 0.30368819951978826, - "manhattan_pearson": 0.36455550272672194, - "manhattan_spearman": 0.3449415822927682, - "pearson": 0.29193603521517864, - "spearman": 0.30368819951978826 - }, - { - "cosine_pearson": 0.10018100867037781, - "cosine_spearman": 0.140073723116761, - "euclidean_pearson": 0.1815827550610446, - "euclidean_spearman": 0.17476484700015202, - "hf_subset": "pt", - "languages": [ - "por-Latn" - ], - "main_score": 0.140073723116761, - "manhattan_pearson": 0.19128411004281437, - "manhattan_spearman": 0.18039207613584918, - "pearson": 0.10018100867037781, - "spearman": 0.140073723116761 - }, - { - "cosine_pearson": 0.11943191616611451, - "cosine_spearman": 0.18592383434972357, - "euclidean_pearson": 0.19435454798231933, - "euclidean_spearman": 0.19704910739182657, - "hf_subset": "ru", - "languages": [ - "rus-Cyrl" - ], - "main_score": 0.18592383434972357, - "manhattan_pearson": 0.1994075652457647, - "manhattan_spearman": 0.20203574788795442, - "pearson": 0.11943191616611451, - "spearman": 0.18592383434972357 - }, - { - "cosine_pearson": 0.02043380644449274, - "cosine_spearman": 0.10518089601052212, - "euclidean_pearson": 0.0834029290349068, - "euclidean_spearman": 0.11379865455499177, - "hf_subset": "zh", - "languages": [ - "cmn-Hans" - ], - "main_score": 0.10518089601052212, - "manhattan_pearson": 0.07858627139925128, - "manhattan_spearman": 0.11183544342388632, - "pearson": 0.02043380644449274, - "spearman": 0.10518089601052212 - } - ] - }, - "task_name": "STSBenchmarkMultilingualVisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397.json deleted file mode 100644 index 84753b28c9..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "7e6af6a2499ad708618be868e1471eac0aca1168", - "evaluation_time": 2724.93390917778, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.6952091954022988, - "f1": 0.6758912178278318, - "f1_weighted": 0.7047976563694092, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6952091954022988, - "scores_per_experiment": [ - { - "accuracy": 0.6940689655172414, - "f1": 0.6720733245789509, - "f1_weighted": 0.7035562570363177 - }, - { - "accuracy": 0.6872183908045977, - "f1": 0.6717815223371335, - "f1_weighted": 0.69741352860682 - }, - { - "accuracy": 0.6966436781609195, - "f1": 0.6783298988455696, - "f1_weighted": 0.7056108128599463 - }, - { - "accuracy": 0.6977471264367816, - "f1": 0.6759683891558872, - "f1_weighted": 0.7076021298138837 - }, - { - "accuracy": 0.700367816091954, - "f1": 0.6813029542216178, - "f1_weighted": 0.7098055535300781 - } - ] - } - ] - }, - "task_name": "SUN397" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json deleted file mode 100644 index 525fe2dbbe..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SUN397ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "7e6af6a2499ad708618be868e1471eac0aca1168", - "evaluation_time": 211.204514503479, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.6106206896551725, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6106206896551725 - } - ] - }, - "task_name": "SUN397ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIR.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIR.json deleted file mode 100644 index b491c5cfea..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIR.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", - "evaluation_time": 94.48416376113892, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "accuracy": 0.4564963413884277, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4564963413884277 - } - ] - }, - "task_name": "SciMMIR" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRI2TRetrieval.json deleted file mode 100644 index 300c9a1314..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", - "evaluation_time": 114.45678329467773, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.05817, - "cv_recall_at_10": 0.14579, - "cv_recall_at_100": 0.30197, - "cv_recall_at_1000": 0.60315, - "cv_recall_at_20": 0.18434, - "cv_recall_at_3": 0.09291, - "cv_recall_at_5": 0.11369, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.09715, - "map_at_1": 0.05817, - "map_at_10": 0.08221, - "map_at_100": 0.08759, - "map_at_1000": 0.08863, - "map_at_20": 0.08488, - "map_at_3": 0.0733, - "map_at_5": 0.07796, - "mrr_at_1": 0.05816884953575601, - "mrr_at_10": 0.08220717687144535, - "mrr_at_100": 0.08758833735797077, - "mrr_at_1000": 0.08862821378958152, - "mrr_at_20": 0.08488199256182855, - "mrr_at_3": 0.07329520998585749, - "mrr_at_5": 0.07796224558814507, - "nauc_cv_recall_at_1000_diff1": 0.14684226988015606, - "nauc_cv_recall_at_1000_max": 0.134802691044856, - "nauc_cv_recall_at_1000_std": 0.389831948813318, - "nauc_cv_recall_at_100_diff1": 0.1806306486342646, - "nauc_cv_recall_at_100_max": 0.14414724950638658, - "nauc_cv_recall_at_100_std": 0.39548007814807806, - "nauc_cv_recall_at_10_diff1": 0.27063104148951167, - "nauc_cv_recall_at_10_max": 0.21400035358270772, - "nauc_cv_recall_at_10_std": 0.41381715017793386, - "nauc_cv_recall_at_1_diff1": 0.48484300258564605, - "nauc_cv_recall_at_1_max": 0.3213938518834158, - "nauc_cv_recall_at_1_std": 0.4507495622214212, - "nauc_cv_recall_at_20_diff1": 0.23455604464471214, - "nauc_cv_recall_at_20_max": 0.18500843309451123, - "nauc_cv_recall_at_20_std": 0.40437096458544736, - "nauc_cv_recall_at_3_diff1": 0.3408987613305649, - "nauc_cv_recall_at_3_max": 0.2664202105680823, - "nauc_cv_recall_at_3_std": 0.40308964378875, - "nauc_cv_recall_at_5_diff1": 0.3075078599041494, - "nauc_cv_recall_at_5_max": 0.24843826848027892, - "nauc_cv_recall_at_5_std": 0.4110419152792929, - "nauc_map_at_1000_diff1": 0.3681055550235476, - "nauc_map_at_1000_max": 0.2670263591078058, - "nauc_map_at_1000_std": 0.42310319341347463, - "nauc_map_at_100_diff1": 0.368442314606953, - "nauc_map_at_100_max": 0.2672938366227504, - "nauc_map_at_100_std": 0.4230389418150572, - "nauc_map_at_10_diff1": 0.38017416191050696, - "nauc_map_at_10_max": 0.27588764093276885, - "nauc_map_at_10_std": 0.4252284193888985, - "nauc_map_at_1_diff1": 0.48484300258564605, - "nauc_map_at_1_max": 0.3213938518834158, - "nauc_map_at_1_std": 0.4507495622214212, - "nauc_map_at_20_diff1": 0.37306566641438116, - "nauc_map_at_20_max": 0.27061824646863875, - "nauc_map_at_20_std": 0.42367498722326263, - "nauc_map_at_3_diff1": 0.40484680089911823, - "nauc_map_at_3_max": 0.292228620983057, - "nauc_map_at_3_std": 0.42377965324762107, - "nauc_map_at_5_diff1": 0.3909036060274109, - "nauc_map_at_5_max": 0.28493006995037873, - "nauc_map_at_5_std": 0.425098683635399, - "nauc_mrr_at_1000_diff1": 0.36810555556946195, - "nauc_mrr_at_1000_max": 0.26702636048225037, - "nauc_mrr_at_1000_std": 0.42310319409545427, - "nauc_mrr_at_100_diff1": 0.368442314606953, - "nauc_mrr_at_100_max": 0.2672938366227504, - "nauc_mrr_at_100_std": 0.4230389418150572, - "nauc_mrr_at_10_diff1": 0.38017416191050696, - "nauc_mrr_at_10_max": 0.27588764093276885, - "nauc_mrr_at_10_std": 0.4252284193888985, - "nauc_mrr_at_1_diff1": 0.48484300258564605, - "nauc_mrr_at_1_max": 0.3213938518834158, - "nauc_mrr_at_1_std": 0.4507495622214212, - "nauc_mrr_at_20_diff1": 0.37306566641438116, - "nauc_mrr_at_20_max": 0.27061824646863875, - "nauc_mrr_at_20_std": 0.42367498722326263, - "nauc_mrr_at_3_diff1": 0.40484680089911823, - "nauc_mrr_at_3_max": 0.292228620983057, - "nauc_mrr_at_3_std": 0.42377965324762107, - "nauc_mrr_at_5_diff1": 0.3909036060274109, - "nauc_mrr_at_5_max": 0.28493006995037873, - "nauc_mrr_at_5_std": 0.425098683635399, - "nauc_ndcg_at_1000_diff1": 0.2943590352399392, - "nauc_ndcg_at_1000_max": 0.2217944789510595, - "nauc_ndcg_at_1000_std": 0.41258004401152276, - "nauc_ndcg_at_100_diff1": 0.2995056724630319, - "nauc_ndcg_at_100_max": 0.2231781872995249, - "nauc_ndcg_at_100_std": 0.4130007443807324, - "nauc_ndcg_at_10_diff1": 0.3432523114481406, - "nauc_ndcg_at_10_max": 0.2554435336092466, - "nauc_ndcg_at_10_std": 0.42092555323891523, - "nauc_ndcg_at_1_diff1": 0.48484300258564605, - "nauc_ndcg_at_1_max": 0.3213938518834158, - "nauc_ndcg_at_1_std": 0.4507495622214212, - "nauc_ndcg_at_20_diff1": 0.32406899107370635, - "nauc_ndcg_at_20_max": 0.24102103282365073, - "nauc_ndcg_at_20_std": 0.4166142389037552, - "nauc_ndcg_at_3_diff1": 0.3853732743990311, - "nauc_ndcg_at_3_max": 0.2844243743500187, - "nauc_ndcg_at_3_std": 0.41745995065792757, - "nauc_ndcg_at_5_diff1": 0.3641878536983579, - "nauc_ndcg_at_5_max": 0.27331942089003486, - "nauc_ndcg_at_5_std": 0.42032916876913506, - "nauc_precision_at_1000_diff1": 0.14684226988015617, - "nauc_precision_at_1000_max": 0.13480269104485515, - "nauc_precision_at_1000_std": 0.3898319488133176, - "nauc_precision_at_100_diff1": 0.18063064863426476, - "nauc_precision_at_100_max": 0.14414724950638685, - "nauc_precision_at_100_std": 0.39548007814807845, - "nauc_precision_at_10_diff1": 0.27063104148951156, - "nauc_precision_at_10_max": 0.21400035358270783, - "nauc_precision_at_10_std": 0.4138171501779341, - "nauc_precision_at_1_diff1": 0.48484300258564605, - "nauc_precision_at_1_max": 0.3213938518834158, - "nauc_precision_at_1_std": 0.4507495622214212, - "nauc_precision_at_20_diff1": 0.23455604464471222, - "nauc_precision_at_20_max": 0.1850084330945111, - "nauc_precision_at_20_std": 0.40437096458544713, - "nauc_precision_at_3_diff1": 0.3408987613305649, - "nauc_precision_at_3_max": 0.2664202105680823, - "nauc_precision_at_3_std": 0.40308964378875006, - "nauc_precision_at_5_diff1": 0.30750785990414936, - "nauc_precision_at_5_max": 0.24843826848027897, - "nauc_precision_at_5_std": 0.41104191527929257, - "nauc_recall_at_1000_diff1": 0.14684226988015606, - "nauc_recall_at_1000_max": 0.134802691044856, - "nauc_recall_at_1000_std": 0.389831948813318, - "nauc_recall_at_100_diff1": 0.1806306486342646, - "nauc_recall_at_100_max": 0.14414724950638658, - "nauc_recall_at_100_std": 0.39548007814807806, - "nauc_recall_at_10_diff1": 0.27063104148951167, - "nauc_recall_at_10_max": 0.21400035358270772, - "nauc_recall_at_10_std": 0.41381715017793386, - "nauc_recall_at_1_diff1": 0.48484300258564605, - "nauc_recall_at_1_max": 0.3213938518834158, - "nauc_recall_at_1_std": 0.4507495622214212, - "nauc_recall_at_20_diff1": 0.23455604464471214, - "nauc_recall_at_20_max": 0.18500843309451123, - "nauc_recall_at_20_std": 0.40437096458544736, - "nauc_recall_at_3_diff1": 0.3408987613305649, - "nauc_recall_at_3_max": 0.2664202105680823, - "nauc_recall_at_3_std": 0.40308964378875, - "nauc_recall_at_5_diff1": 0.3075078599041494, - "nauc_recall_at_5_max": 0.24843826848027892, - "nauc_recall_at_5_std": 0.4110419152792929, - "ndcg_at_1": 0.05817, - "ndcg_at_10": 0.09715, - "ndcg_at_100": 0.12811, - "ndcg_at_1000": 0.16402, - "ndcg_at_20": 0.1069, - "ndcg_at_3": 0.07833, - "ndcg_at_5": 0.08681, - "precision_at_1": 0.05817, - "precision_at_10": 0.01458, - "precision_at_100": 0.00302, - "precision_at_1000": 0.0006, - "precision_at_20": 0.00922, - "precision_at_3": 0.03097, - "precision_at_5": 0.02274, - "recall_at_1": 0.05817, - "recall_at_10": 0.14579, - "recall_at_100": 0.30197, - "recall_at_1000": 0.60315, - "recall_at_20": 0.18434, - "recall_at_3": 0.09291, - "recall_at_5": 0.11369 - } - ] - }, - "task_name": "SciMMIRI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRT2IRetrieval.json deleted file mode 100644 index d46edfa129..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SciMMIRT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", - "evaluation_time": 109.91747164726257, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.04895, - "cv_recall_at_10": 0.1207, - "cv_recall_at_100": 0.2657, - "cv_recall_at_1000": 0.56601, - "cv_recall_at_20": 0.15544, - "cv_recall_at_3": 0.07963, - "cv_recall_at_5": 0.0966, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.08144, - "map_at_1": 0.04895, - "map_at_10": 0.0693, - "map_at_100": 0.07419, - "map_at_1000": 0.0752, - "map_at_20": 0.07168, - "map_at_3": 0.06228, - "map_at_5": 0.06612, - "mrr_at_1": 0.048945459017401466, - "mrr_at_10": 0.06929704099967894, - "mrr_at_100": 0.07418728836197862, - "mrr_at_1000": 0.07520233371125491, - "mrr_at_20": 0.07167806562939899, - "mrr_at_3": 0.062278382422267485, - "mrr_at_5": 0.06612146180491527, - "nauc_cv_recall_at_1000_diff1": 0.08565868923582504, - "nauc_cv_recall_at_1000_max": 0.10138064328519379, - "nauc_cv_recall_at_1000_std": 0.3288030738840006, - "nauc_cv_recall_at_100_diff1": 0.15064453104841738, - "nauc_cv_recall_at_100_max": 0.143295352377065, - "nauc_cv_recall_at_100_std": 0.27989257851203553, - "nauc_cv_recall_at_10_diff1": 0.248178863037523, - "nauc_cv_recall_at_10_max": 0.20552793051112445, - "nauc_cv_recall_at_10_std": 0.23682614089723797, - "nauc_cv_recall_at_1_diff1": 0.4931665319076531, - "nauc_cv_recall_at_1_max": 0.34046617367806836, - "nauc_cv_recall_at_1_std": 0.22520394629166143, - "nauc_cv_recall_at_20_diff1": 0.20448507514630834, - "nauc_cv_recall_at_20_max": 0.17905271116474095, - "nauc_cv_recall_at_20_std": 0.24941512163240517, - "nauc_cv_recall_at_3_diff1": 0.33601488304743465, - "nauc_cv_recall_at_3_max": 0.2672543870064712, - "nauc_cv_recall_at_3_std": 0.22899599659848674, - "nauc_cv_recall_at_5_diff1": 0.2836379140928627, - "nauc_cv_recall_at_5_max": 0.2313767233406441, - "nauc_cv_recall_at_5_std": 0.21663135292170893, - "nauc_map_at_1000_diff1": 0.3568321384391532, - "nauc_map_at_1000_max": 0.26999090632663203, - "nauc_map_at_1000_std": 0.23150834718439864, - "nauc_map_at_100_diff1": 0.357872984312932, - "nauc_map_at_100_max": 0.2707815994312529, - "nauc_map_at_100_std": 0.2310206556257606, - "nauc_map_at_10_diff1": 0.37342184657421573, - "nauc_map_at_10_max": 0.27996073103876784, - "nauc_map_at_10_std": 0.2267919116158103, - "nauc_map_at_1_diff1": 0.4931665319076531, - "nauc_map_at_1_max": 0.34046617367806836, - "nauc_map_at_1_std": 0.22520394629166143, - "nauc_map_at_20_diff1": 0.36463069830617245, - "nauc_map_at_20_max": 0.2747543332241589, - "nauc_map_at_20_std": 0.2289897405088031, - "nauc_map_at_3_diff1": 0.40611981629392646, - "nauc_map_at_3_max": 0.3014379705164891, - "nauc_map_at_3_std": 0.22655778598644558, - "nauc_map_at_5_diff1": 0.38456853729603835, - "nauc_map_at_5_max": 0.28753313941804676, - "nauc_map_at_5_std": 0.2226991023456713, - "nauc_mrr_at_1000_diff1": 0.356832161495167, - "nauc_mrr_at_1000_max": 0.2699909040097771, - "nauc_mrr_at_1000_std": 0.23150833671626686, - "nauc_mrr_at_100_diff1": 0.357872984312932, - "nauc_mrr_at_100_max": 0.2707815994312529, - "nauc_mrr_at_100_std": 0.2310206556257606, - "nauc_mrr_at_10_diff1": 0.37342184657421573, - "nauc_mrr_at_10_max": 0.27996073103876784, - "nauc_mrr_at_10_std": 0.2267919116158103, - "nauc_mrr_at_1_diff1": 0.4931665319076531, - "nauc_mrr_at_1_max": 0.34046617367806836, - "nauc_mrr_at_1_std": 0.22520394629166143, - "nauc_mrr_at_20_diff1": 0.36463069830617245, - "nauc_mrr_at_20_max": 0.2747543332241589, - "nauc_mrr_at_20_std": 0.2289897405088031, - "nauc_mrr_at_3_diff1": 0.40611981629392646, - "nauc_mrr_at_3_max": 0.3014379705164891, - "nauc_mrr_at_3_std": 0.22655778598644558, - "nauc_mrr_at_5_diff1": 0.38456853729603835, - "nauc_mrr_at_5_max": 0.28753313941804676, - "nauc_mrr_at_5_std": 0.2226991023456713, - "nauc_ndcg_at_1000_diff1": 0.26011437580815167, - "nauc_ndcg_at_1000_max": 0.21035983312198475, - "nauc_ndcg_at_1000_std": 0.25938262677666973, - "nauc_ndcg_at_100_diff1": 0.27692777560932097, - "nauc_ndcg_at_100_max": 0.22175842905077686, - "nauc_ndcg_at_100_std": 0.2476316343627969, - "nauc_ndcg_at_10_diff1": 0.3306820261454687, - "nauc_ndcg_at_10_max": 0.2548593820745378, - "nauc_ndcg_at_10_std": 0.2297778323461325, - "nauc_ndcg_at_1_diff1": 0.4931665319076531, - "nauc_ndcg_at_1_max": 0.34046617367806836, - "nauc_ndcg_at_1_std": 0.22520394629166143, - "nauc_ndcg_at_20_diff1": 0.3072405340031752, - "nauc_ndcg_at_20_max": 0.24080054220757713, - "nauc_ndcg_at_20_std": 0.23564872251630792, - "nauc_ndcg_at_3_diff1": 0.3846824095684635, - "nauc_ndcg_at_3_max": 0.29104440148011873, - "nauc_ndcg_at_3_std": 0.22727424904293786, - "nauc_ndcg_at_5_diff1": 0.3518141711145917, - "nauc_ndcg_at_5_max": 0.26947351331044006, - "nauc_ndcg_at_5_std": 0.22086120599448097, - "nauc_precision_at_1000_diff1": 0.08565868923582474, - "nauc_precision_at_1000_max": 0.10138064328519283, - "nauc_precision_at_1000_std": 0.328803073884, - "nauc_precision_at_100_diff1": 0.15064453104841719, - "nauc_precision_at_100_max": 0.1432953523770651, - "nauc_precision_at_100_std": 0.2798925785120355, - "nauc_precision_at_10_diff1": 0.24817886303752315, - "nauc_precision_at_10_max": 0.2055279305111246, - "nauc_precision_at_10_std": 0.23682614089723822, - "nauc_precision_at_1_diff1": 0.4931665319076531, - "nauc_precision_at_1_max": 0.34046617367806836, - "nauc_precision_at_1_std": 0.22520394629166143, - "nauc_precision_at_20_diff1": 0.20448507514630834, - "nauc_precision_at_20_max": 0.17905271116474109, - "nauc_precision_at_20_std": 0.24941512163240534, - "nauc_precision_at_3_diff1": 0.33601488304743476, - "nauc_precision_at_3_max": 0.26725438700647136, - "nauc_precision_at_3_std": 0.22899599659848702, - "nauc_precision_at_5_diff1": 0.28363791409286265, - "nauc_precision_at_5_max": 0.23137672334064413, - "nauc_precision_at_5_std": 0.21663135292170888, - "nauc_recall_at_1000_diff1": 0.08565868923582504, - "nauc_recall_at_1000_max": 0.10138064328519379, - "nauc_recall_at_1000_std": 0.3288030738840006, - "nauc_recall_at_100_diff1": 0.15064453104841738, - "nauc_recall_at_100_max": 0.143295352377065, - "nauc_recall_at_100_std": 0.27989257851203553, - "nauc_recall_at_10_diff1": 0.248178863037523, - "nauc_recall_at_10_max": 0.20552793051112445, - "nauc_recall_at_10_std": 0.23682614089723797, - "nauc_recall_at_1_diff1": 0.4931665319076531, - "nauc_recall_at_1_max": 0.34046617367806836, - "nauc_recall_at_1_std": 0.22520394629166143, - "nauc_recall_at_20_diff1": 0.20448507514630834, - "nauc_recall_at_20_max": 0.17905271116474095, - "nauc_recall_at_20_std": 0.24941512163240517, - "nauc_recall_at_3_diff1": 0.33601488304743465, - "nauc_recall_at_3_max": 0.2672543870064712, - "nauc_recall_at_3_std": 0.22899599659848674, - "nauc_recall_at_5_diff1": 0.2836379140928627, - "nauc_recall_at_5_max": 0.2313767233406441, - "nauc_recall_at_5_std": 0.21663135292170893, - "ndcg_at_1": 0.04895, - "ndcg_at_10": 0.08144, - "ndcg_at_100": 0.10998, - "ndcg_at_1000": 0.14567, - "ndcg_at_20": 0.09018, - "ndcg_at_3": 0.06673, - "ndcg_at_5": 0.07369, - "precision_at_1": 0.04895, - "precision_at_10": 0.01207, - "precision_at_100": 0.00266, - "precision_at_1000": 0.00057, - "precision_at_20": 0.00777, - "precision_at_3": 0.02654, - "precision_at_5": 0.01932, - "recall_at_1": 0.04895, - "recall_at_10": 0.1207, - "recall_at_100": 0.2657, - "recall_at_1000": 0.56601, - "recall_at_20": 0.15544, - "recall_at_3": 0.07963, - "recall_at_5": 0.0966 - } - ] - }, - "task_name": "SciMMIRT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCars.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCars.json deleted file mode 100644 index c7a9dd1630..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCars.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", - "evaluation_time": 660.5333795547485, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.7398830991170253, - "f1": 0.7403225810264707, - "f1_weighted": 0.7400357184409913, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7398830991170253, - "scores_per_experiment": [ - { - "accuracy": 0.7430667827384654, - "f1": 0.7436627162291564, - "f1_weighted": 0.7435113191449505 - }, - { - "accuracy": 0.7363512000994901, - "f1": 0.7365198863893867, - "f1_weighted": 0.7363691751162225 - }, - { - "accuracy": 0.734112672553165, - "f1": 0.7341057453057904, - "f1_weighted": 0.7342243485571237 - }, - { - "accuracy": 0.7404551672677527, - "f1": 0.7408828281148372, - "f1_weighted": 0.7406406001011914 - }, - { - "accuracy": 0.745429672926253, - "f1": 0.7464417290931831, - "f1_weighted": 0.7454331492854683 - } - ] - } - ] - }, - "task_name": "StanfordCars" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json deleted file mode 100644 index 5b68c93a9f..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "b27a0612211af3598bd11fe28af20928f20cce06", - "evaluation_time": 211.27373456954956, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.62418, - "cv_recall_at_10": 0.94242, - "cv_recall_at_100": 0.99689, - "cv_recall_at_1000": 0.99988, - "cv_recall_at_20": 0.97364, - "cv_recall_at_3": 0.8188, - "cv_recall_at_5": 0.88534, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.62418, - "map_at_1": 0.0, - "map_at_10": 0.06072, - "map_at_100": 0.17896, - "map_at_1000": 0.21707, - "map_at_20": 0.09946, - "map_at_3": 0.01597, - "map_at_5": 0.03084, - "mrr_at_1": 0.00012436264146250465, - "mrr_at_10": 0.3901051259122414, - "mrr_at_100": 0.3934181724945363, - "mrr_at_1000": 0.39343998588915124, - "mrr_at_20": 0.3926309438221267, - "mrr_at_3": 0.35457861791651374, - "mrr_at_5": 0.3791464577374308, - "nauc_cv_recall_at_1000_diff1": -1.7401334875861458, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 0.5541703308201117, - "nauc_cv_recall_at_100_diff1": -0.9997097911342215, - "nauc_cv_recall_at_100_max": 0.06743574922100379, - "nauc_cv_recall_at_100_std": 0.7587532067319325, - "nauc_cv_recall_at_10_diff1": -0.4239485200991442, - "nauc_cv_recall_at_10_max": -0.0024729549630218292, - "nauc_cv_recall_at_10_std": 0.35822657204730457, - "nauc_cv_recall_at_1_diff1": -0.22768537577324857, - "nauc_cv_recall_at_1_max": 0.012943261327137467, - "nauc_cv_recall_at_1_std": 0.06964226599453852, - "nauc_cv_recall_at_20_diff1": -0.5260289144807463, - "nauc_cv_recall_at_20_max": -0.015992094159549268, - "nauc_cv_recall_at_20_std": 0.47448921254663456, - "nauc_cv_recall_at_3_diff1": -0.30953137539602743, - "nauc_cv_recall_at_3_max": 0.013214322506575777, - "nauc_cv_recall_at_3_std": 0.18959869944763055, - "nauc_cv_recall_at_5_diff1": -0.3383429761997499, - "nauc_cv_recall_at_5_max": -0.014524830990478535, - "nauc_cv_recall_at_5_std": 0.22152839165884303, - "nauc_map_at_1000_diff1": -0.20058034455775214, - "nauc_map_at_1000_max": -0.009541604862555151, - "nauc_map_at_1000_std": 0.36415192557738685, - "nauc_map_at_100_diff1": -0.1859159754121815, - "nauc_map_at_100_max": -0.00915256217544372, - "nauc_map_at_100_std": 0.3179916778555873, - "nauc_map_at_10_diff1": -0.17016138595951308, - "nauc_map_at_10_max": -0.008401015865521977, - "nauc_map_at_10_std": 0.13541784388452338, - "nauc_map_at_1_diff1": -0.5746685568283685, - "nauc_map_at_1_max": -0.4995495220405232, - "nauc_map_at_1_std": 1.0, - "nauc_map_at_20_diff1": -0.17026308934464754, - "nauc_map_at_20_max": -0.006465085107102209, - "nauc_map_at_20_std": 0.19410962638554546, - "nauc_map_at_3_diff1": -0.19618028548625588, - "nauc_map_at_3_max": 0.0015533961447322393, - "nauc_map_at_3_std": 0.08037524782358588, - "nauc_map_at_5_diff1": -0.17897720472394818, - "nauc_map_at_5_max": -0.0022562794225319126, - "nauc_map_at_5_std": 0.09354551152212666, - "nauc_mrr_at_1000_diff1": -0.26756915105219325, - "nauc_mrr_at_1000_max": 0.007219560942946209, - "nauc_mrr_at_1000_std": 0.1277621843185587, - "nauc_mrr_at_100_diff1": -0.26768624224236, - "nauc_mrr_at_100_max": 0.007215515428976401, - "nauc_mrr_at_100_std": 0.12791181627808856, - "nauc_mrr_at_10_diff1": -0.2689209194339867, - "nauc_mrr_at_10_max": 0.00554788070506897, - "nauc_mrr_at_10_std": 0.13204137790528978, - "nauc_mrr_at_1_diff1": -0.5746685568283684, - "nauc_mrr_at_1_max": -0.49954952204052305, - "nauc_mrr_at_1_std": 1.0, - "nauc_mrr_at_20_diff1": -0.26916203739306344, - "nauc_mrr_at_20_max": 0.0067661125465101725, - "nauc_mrr_at_20_std": 0.1304515877065281, - "nauc_mrr_at_3_diff1": -0.25798593374810697, - "nauc_mrr_at_3_max": 0.00846034337944975, - "nauc_mrr_at_3_std": 0.10915550317935098, - "nauc_mrr_at_5_diff1": -0.26767632665665964, - "nauc_mrr_at_5_max": 0.007533563370578295, - "nauc_mrr_at_5_std": 0.1314283638264671, - "nauc_ndcg_at_1000_diff1": -0.26848867699612694, - "nauc_ndcg_at_1000_max": -0.01219681131327712, - "nauc_ndcg_at_1000_std": 0.47014178875035684, - "nauc_ndcg_at_100_diff1": -0.21785067520456688, - "nauc_ndcg_at_100_max": -0.012635572435124356, - "nauc_ndcg_at_100_std": 0.3658418059055116, - "nauc_ndcg_at_10_diff1": -0.18666608258490472, - "nauc_ndcg_at_10_max": -0.006380936466543662, - "nauc_ndcg_at_10_std": 0.1570549915957106, - "nauc_ndcg_at_1_diff1": -0.5746685568283684, - "nauc_ndcg_at_1_max": -0.49954952204052305, - "nauc_ndcg_at_1_std": 1.0, - "nauc_ndcg_at_20_diff1": -0.1892271528105937, - "nauc_ndcg_at_20_max": -0.0045807039352219504, - "nauc_ndcg_at_20_std": 0.2166402891227918, - "nauc_ndcg_at_3_diff1": -0.2156887442186972, - "nauc_ndcg_at_3_max": 0.005069652317788833, - "nauc_ndcg_at_3_std": 0.09725117443347137, - "nauc_ndcg_at_5_diff1": -0.1949431835234068, - "nauc_ndcg_at_5_max": 0.0020311591774530764, - "nauc_ndcg_at_5_std": 0.11416484694774583, - "nauc_precision_at_1000_diff1": -0.2311605207327101, - "nauc_precision_at_1000_max": -0.011274296784979255, - "nauc_precision_at_1000_std": 0.5429105568395621, - "nauc_precision_at_100_diff1": -0.20218396795823576, - "nauc_precision_at_100_max": -0.013867657753386877, - "nauc_precision_at_100_std": 0.43295422350606705, - "nauc_precision_at_10_diff1": -0.17588408513054, - "nauc_precision_at_10_max": -0.009047036150405124, - "nauc_precision_at_10_std": 0.16785493183646077, - "nauc_precision_at_1_diff1": -0.5746685568283684, - "nauc_precision_at_1_max": -0.49954952204052305, - "nauc_precision_at_1_std": 1.0, - "nauc_precision_at_20_diff1": -0.1795363254595728, - "nauc_precision_at_20_max": -0.005308651370902496, - "nauc_precision_at_20_std": 0.23936954535875124, - "nauc_precision_at_3_diff1": -0.2138380176200969, - "nauc_precision_at_3_max": 0.004063308534515615, - "nauc_precision_at_3_std": 0.10034463101558477, - "nauc_precision_at_5_diff1": -0.18747136042736523, - "nauc_precision_at_5_max": 0.0007804267760293301, - "nauc_precision_at_5_std": 0.11857533805458287, - "nauc_recall_at_1000_diff1": -0.3090612097064354, - "nauc_recall_at_1000_max": -0.014613854656747456, - "nauc_recall_at_1000_std": 0.6433331116323254, - "nauc_recall_at_100_diff1": -0.21428842666117542, - "nauc_recall_at_100_max": -0.0157915060538706, - "nauc_recall_at_100_std": 0.42610561659215096, - "nauc_recall_at_10_diff1": -0.1808720571796819, - "nauc_recall_at_10_max": -0.012426875194332561, - "nauc_recall_at_10_std": 0.1595995698351979, - "nauc_recall_at_1_diff1": -0.5746685568283685, - "nauc_recall_at_1_max": -0.4995495220405232, - "nauc_recall_at_1_std": 1.0, - "nauc_recall_at_20_diff1": -0.18610900567530295, - "nauc_recall_at_20_max": -0.009094327927607726, - "nauc_recall_at_20_std": 0.22959563262774454, - "nauc_recall_at_3_diff1": -0.20479580828524666, - "nauc_recall_at_3_max": 0.0011739339718611925, - "nauc_recall_at_3_std": 0.08948104084187246, - "nauc_recall_at_5_diff1": -0.18850109914331253, - "nauc_recall_at_5_max": -0.0027946570149449752, - "nauc_recall_at_5_std": 0.1101593382209862, - "ndcg_at_1": 0.00012, - "ndcg_at_10": 0.37362, - "ndcg_at_100": 0.3811, - "ndcg_at_1000": 0.56344, - "ndcg_at_20": 0.34466, - "ndcg_at_3": 0.31444, - "ndcg_at_5": 0.36135, - "precision_at_1": 0.00012, - "precision_at_10": 0.41114, - "precision_at_100": 0.17081, - "precision_at_1000": 0.03375, - "precision_at_20": 0.35062, - "precision_at_3": 0.39216, - "precision_at_5": 0.42838, - "recall_at_1": 0.0, - "recall_at_10": 0.10276, - "recall_at_100": 0.42446, - "recall_at_1000": 0.83495, - "recall_at_20": 0.17504, - "recall_at_3": 0.02944, - "recall_at_5": 0.05356 - } - ] - }, - "task_name": "StanfordCarsI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsZeroShot.json deleted file mode 100644 index 2762ca15c7..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/StanfordCarsZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", - "evaluation_time": 131.17899537086487, - "kg_co2_emissions": null, - "mteb_version": "1.12.80", - "scores": { - "test": [ - { - "accuracy": 0.5861211292127845, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5861211292127845 - } - ] - }, - "task_name": "StanfordCarsZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SugarCrepe.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SugarCrepe.json deleted file mode 100644 index 722f2da7e1..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/SugarCrepe.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "134abf9ade6a32f9fdae0e89022ff227a70b87e5", - "evaluation_time": 113.4538505077362, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.7660764455795288, - "text_acc": 0.7660764455795288 - } - ] - }, - "task_name": "SugarCrepe" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/TUBerlinT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/TUBerlinT2IRetrieval.json deleted file mode 100644 index d1daf9a0aa..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/TUBerlinT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "dcd8328b8b27cd39ed6c066862532a5dcd35f012", - "evaluation_time": 175.85956716537476, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.856, - "cv_recall_at_10": 0.992, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.996, - "cv_recall_at_3": 0.964, - "cv_recall_at_5": 0.976, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.81945, - "map_at_1": 0.0107, - "map_at_10": 0.09306, - "map_at_100": 0.46335, - "map_at_1000": 0.54192, - "map_at_20": 0.17108, - "map_at_3": 0.03029, - "map_at_5": 0.04878, - "mrr_at_1": 0.856, - "mrr_at_10": 0.908388888888889, - "mrr_at_100": 0.9086424259573667, - "mrr_at_1000": 0.9086424259573667, - "mrr_at_20": 0.9085994152046785, - "mrr_at_3": 0.9033333333333334, - "mrr_at_5": 0.9063333333333334, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": 0.5541549953314484, - "nauc_cv_recall_at_10_max": 1.0, - "nauc_cv_recall_at_10_std": 1.0, - "nauc_cv_recall_at_1_diff1": 0.13113248325048668, - "nauc_cv_recall_at_1_max": 0.43477145018370433, - "nauc_cv_recall_at_1_std": 0.23836989409984835, - "nauc_cv_recall_at_20_diff1": 0.5541549953314557, - "nauc_cv_recall_at_20_max": 1.0, - "nauc_cv_recall_at_20_std": 1.0, - "nauc_cv_recall_at_3_diff1": 0.04160182591555137, - "nauc_cv_recall_at_3_max": 0.4733893557422951, - "nauc_cv_recall_at_3_std": 0.4679427326486138, - "nauc_cv_recall_at_5_diff1": 0.09337068160597826, - "nauc_cv_recall_at_5_max": 0.3998599439775951, - "nauc_cv_recall_at_5_std": 0.39713663243075265, - "nauc_map_at_1000_diff1": -0.1659607050128913, - "nauc_map_at_1000_max": 0.4002951420709613, - "nauc_map_at_1000_std": 0.6008649703703033, - "nauc_map_at_100_diff1": -0.159320455241414, - "nauc_map_at_100_max": 0.37907101423707096, - "nauc_map_at_100_std": 0.5501993811365806, - "nauc_map_at_10_diff1": -0.03247236891366951, - "nauc_map_at_10_max": 0.39401250753640266, - "nauc_map_at_10_std": 0.2912329253162918, - "nauc_map_at_1_diff1": 0.13113248325048504, - "nauc_map_at_1_max": 0.43477145018370394, - "nauc_map_at_1_std": 0.23836989409984927, - "nauc_map_at_20_diff1": -0.07440129133273242, - "nauc_map_at_20_max": 0.3846356021740391, - "nauc_map_at_20_std": 0.34792679433137946, - "nauc_map_at_3_diff1": 0.08322035015793686, - "nauc_map_at_3_max": 0.423486714736962, - "nauc_map_at_3_std": 0.24833821828229646, - "nauc_map_at_5_diff1": 0.00919846951370846, - "nauc_map_at_5_max": 0.41473543872025803, - "nauc_map_at_5_std": 0.25080776636776136, - "nauc_mrr_at_1000_diff1": 0.11207727768308348, - "nauc_mrr_at_1000_max": 0.4261596090309955, - "nauc_mrr_at_1000_std": 0.24872225471043358, - "nauc_mrr_at_100_diff1": 0.11207727768308348, - "nauc_mrr_at_100_max": 0.4261596090309955, - "nauc_mrr_at_100_std": 0.24872225471043358, - "nauc_mrr_at_10_diff1": 0.11334039502197703, - "nauc_mrr_at_10_max": 0.4277992025202272, - "nauc_mrr_at_10_std": 0.25086882737971083, - "nauc_mrr_at_1_diff1": 0.13113248325048668, - "nauc_mrr_at_1_max": 0.43477145018370433, - "nauc_mrr_at_1_std": 0.23836989409984835, - "nauc_mrr_at_20_diff1": 0.11229206610732925, - "nauc_mrr_at_20_max": 0.4264384158299444, - "nauc_mrr_at_20_std": 0.24908727144083842, - "nauc_mrr_at_3_diff1": 0.10013605442176822, - "nauc_mrr_at_3_max": 0.42878048780487743, - "nauc_mrr_at_3_std": 0.26161937945910047, - "nauc_mrr_at_5_diff1": 0.10549139387956116, - "nauc_mrr_at_5_max": 0.4224472165945086, - "nauc_mrr_at_5_std": 0.2501225528005465, - "nauc_ndcg_at_1000_diff1": -0.15473148465972003, - "nauc_ndcg_at_1000_max": 0.46638630840908135, - "nauc_ndcg_at_1000_std": 0.6706615415351543, - "nauc_ndcg_at_100_diff1": -0.15091091689134778, - "nauc_ndcg_at_100_max": 0.41430352035241735, - "nauc_ndcg_at_100_std": 0.5945250349662444, - "nauc_ndcg_at_10_diff1": -0.02231970983029379, - "nauc_ndcg_at_10_max": 0.4180194276810624, - "nauc_ndcg_at_10_std": 0.30351802916358955, - "nauc_ndcg_at_1_diff1": 0.13113248325048668, - "nauc_ndcg_at_1_max": 0.43477145018370433, - "nauc_ndcg_at_1_std": 0.23836989409984835, - "nauc_ndcg_at_20_diff1": -0.05816593196928681, - "nauc_ndcg_at_20_max": 0.4138040923822943, - "nauc_ndcg_at_20_std": 0.3652054221361988, - "nauc_ndcg_at_3_diff1": 0.08447472013550292, - "nauc_ndcg_at_3_max": 0.4259852943789907, - "nauc_ndcg_at_3_std": 0.25416061971191134, - "nauc_ndcg_at_5_diff1": 0.00766324120279822, - "nauc_ndcg_at_5_max": 0.4151433003158982, - "nauc_ndcg_at_5_std": 0.2495238554669031, - "nauc_precision_at_1000_diff1": -0.1809233410743444, - "nauc_precision_at_1000_max": 0.42887321721343796, - "nauc_precision_at_1000_std": 0.7621261043128252, - "nauc_precision_at_100_diff1": -0.17465674567829512, - "nauc_precision_at_100_max": 0.3924712011609377, - "nauc_precision_at_100_std": 0.642807625809328, - "nauc_precision_at_10_diff1": -0.053504093361584064, - "nauc_precision_at_10_max": 0.41072806234342696, - "nauc_precision_at_10_std": 0.32022830196389684, - "nauc_precision_at_1_diff1": 0.13113248325048668, - "nauc_precision_at_1_max": 0.43477145018370433, - "nauc_precision_at_1_std": 0.23836989409984835, - "nauc_precision_at_20_diff1": -0.08543376918703843, - "nauc_precision_at_20_max": 0.4026196702671966, - "nauc_precision_at_20_std": 0.3896424104604883, - "nauc_precision_at_3_diff1": 0.07327497513632078, - "nauc_precision_at_3_max": 0.426716451181454, - "nauc_precision_at_3_std": 0.26320775746767655, - "nauc_precision_at_5_diff1": -0.031485413419408415, - "nauc_precision_at_5_max": 0.40567508192295, - "nauc_precision_at_5_std": 0.25015525472821193, - "nauc_recall_at_1000_diff1": -0.18092334107434152, - "nauc_recall_at_1000_max": 0.4288732172134386, - "nauc_recall_at_1000_std": 0.7621261043128265, - "nauc_recall_at_100_diff1": -0.1746567456782966, - "nauc_recall_at_100_max": 0.3924712011609363, - "nauc_recall_at_100_std": 0.6428076258093276, - "nauc_recall_at_10_diff1": -0.053504093361584064, - "nauc_recall_at_10_max": 0.41072806234342696, - "nauc_recall_at_10_std": 0.32022830196389684, - "nauc_recall_at_1_diff1": 0.13113248325048504, - "nauc_recall_at_1_max": 0.43477145018370394, - "nauc_recall_at_1_std": 0.23836989409984927, - "nauc_recall_at_20_diff1": -0.08543376918703843, - "nauc_recall_at_20_max": 0.4026196702671966, - "nauc_recall_at_20_std": 0.3896424104604883, - "nauc_recall_at_3_diff1": 0.07327497513632164, - "nauc_recall_at_3_max": 0.42671645118145396, - "nauc_recall_at_3_std": 0.2632077574676772, - "nauc_recall_at_5_diff1": -0.031485413419408415, - "nauc_recall_at_5_max": 0.40567508192295, - "nauc_recall_at_5_std": 0.25015525472821193, - "ndcg_at_1": 0.856, - "ndcg_at_10": 0.81945, - "ndcg_at_100": 0.63488, - "ndcg_at_1000": 0.81053, - "ndcg_at_20": 0.7844, - "ndcg_at_3": 0.84208, - "ndcg_at_5": 0.83105, - "precision_at_1": 0.856, - "precision_at_10": 0.8108, - "precision_at_100": 0.4738, - "precision_at_1000": 0.07223, - "precision_at_20": 0.7644, - "precision_at_3": 0.83867, - "precision_at_5": 0.824, - "recall_at_1": 0.0107, - "recall_at_10": 0.10135, - "recall_at_100": 0.59225, - "recall_at_1000": 0.9029, - "recall_at_20": 0.1911, - "recall_at_3": 0.03145, - "recall_at_5": 0.0515 - } - ] - }, - "task_name": "TUBerlinT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101.json deleted file mode 100644 index 4ff30ca9fa..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "dataset_revision": "1098eed48f2929443f47c39f3b5c814e16369c11", - "evaluation_time": 3175.5142464637756, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "accuracy": 0.7968753137451199, - "f1": 0.7654702260165476, - "f1_weighted": 0.7939622985652318, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7968753137451199, - "scores_per_experiment": [ - { - "accuracy": 0.7968753137451199, - "f1": 0.7654702260165476, - "f1_weighted": 0.7939622985652318 - } - ] - } - ] - }, - "task_name": "UCF101" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101ZeroShot.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101ZeroShot.json deleted file mode 100644 index 146378a6c6..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/UCF101ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "1098eed48f2929443f47c39f3b5c814e16369c11", - "evaluation_time": 2312.6607372760773, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "accuracy": 0.5814977152183953, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5814977152183953 - } - ] - }, - "task_name": "UCF101ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json deleted file mode 100644 index 44558b6f3d..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 48.62162375450134, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "accuracy": 0.5166801292407108, - "f1": 0.6851483326294392, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "lrap": 0.6602331269072138, - "main_score": 0.5166801292407108, - "scores_per_experiment": [ - { - "accuracy": 0.52140549273021, - "f1": 0.6981947346229871, - "lrap": 0.6660473882606517 - }, - { - "accuracy": 0.5357431340872375, - "f1": 0.6897556454047876, - "lrap": 0.6734304882427027 - }, - { - "accuracy": 0.5102988691437803, - "f1": 0.6665291163888459, - "lrap": 0.6582500336564507 - }, - { - "accuracy": 0.5149434571890146, - "f1": 0.6906546111504063, - "lrap": 0.6572852158499527 - }, - { - "accuracy": 0.5010096930533118, - "f1": 0.6806075555801697, - "lrap": 0.6461525085263116 - } - ] - } - ] - }, - "task_name": "VOC2007" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VQA2IT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VQA2IT2TRetrieval.json deleted file mode 100644 index 63970704c1..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VQA2IT2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "69882b6ba0b443dd62e633e546725b0f13b7e3aa", - "evaluation_time": 1096.5730390548706, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.00224, - "cv_recall_at_10": 0.01687, - "cv_recall_at_100": 0.07996, - "cv_recall_at_1000": 0.2859, - "cv_recall_at_20": 0.02905, - "cv_recall_at_3": 0.00618, - "cv_recall_at_5": 0.00962, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.0082, - "map_at_1": 0.00224, - "map_at_10": 0.00561, - "map_at_100": 0.0076, - "map_at_1000": 0.00818, - "map_at_20": 0.00644, - "map_at_3": 0.00389, - "map_at_5": 0.00467, - "mrr_at_1": 0.0022392864140627186, - "mrr_at_10": 0.005608912626509676, - "mrr_at_100": 0.007598006845600478, - "mrr_at_1000": 0.008184917799157795, - "mrr_at_20": 0.006437587783214147, - "mrr_at_3": 0.003891537674438876, - "mrr_at_5": 0.00466502452329641, - "nauc_cv_recall_at_1000_diff1": 0.008825837565798213, - "nauc_cv_recall_at_1000_max": 0.06850469648818859, - "nauc_cv_recall_at_1000_std": -0.0651814166780974, - "nauc_cv_recall_at_100_diff1": 0.045034516026494435, - "nauc_cv_recall_at_100_max": 0.004398660634268218, - "nauc_cv_recall_at_100_std": -0.03975561918800372, - "nauc_cv_recall_at_10_diff1": 0.06821726635054984, - "nauc_cv_recall_at_10_max": -0.01603124686173661, - "nauc_cv_recall_at_10_std": -0.08733387779771963, - "nauc_cv_recall_at_1_diff1": 0.17890721176822888, - "nauc_cv_recall_at_1_max": -0.08979014629444568, - "nauc_cv_recall_at_1_std": -0.07645555550866222, - "nauc_cv_recall_at_20_diff1": 0.06573644523017931, - "nauc_cv_recall_at_20_max": -0.008446727739883213, - "nauc_cv_recall_at_20_std": -0.10159027094805619, - "nauc_cv_recall_at_3_diff1": 0.10725987074701412, - "nauc_cv_recall_at_3_max": -0.04337993473508564, - "nauc_cv_recall_at_3_std": -0.06088188555814698, - "nauc_cv_recall_at_5_diff1": 0.08502413140032329, - "nauc_cv_recall_at_5_max": -0.018521868603006558, - "nauc_cv_recall_at_5_std": -0.07681198564756143, - "nauc_map_at_1000_diff1": 0.0857862921438581, - "nauc_map_at_1000_max": -0.022301252824256254, - "nauc_map_at_1000_std": -0.07002071019417072, - "nauc_map_at_100_diff1": 0.09059194638791825, - "nauc_map_at_100_max": -0.02730484396719757, - "nauc_map_at_100_std": -0.07112702467096156, - "nauc_map_at_10_diff1": 0.10561558040018261, - "nauc_map_at_10_max": -0.03887680003815409, - "nauc_map_at_10_std": -0.07384905648271843, - "nauc_map_at_1_diff1": 0.17890721176822888, - "nauc_map_at_1_max": -0.08979014629444568, - "nauc_map_at_1_std": -0.07645555550866222, - "nauc_map_at_20_diff1": 0.09999678477824628, - "nauc_map_at_20_max": -0.033642117064318564, - "nauc_map_at_20_std": -0.08006217604646698, - "nauc_map_at_3_diff1": 0.13264050928045265, - "nauc_map_at_3_max": -0.05878443289453297, - "nauc_map_at_3_std": -0.06106821971711554, - "nauc_map_at_5_diff1": 0.11792502863936245, - "nauc_map_at_5_max": -0.04439031923384928, - "nauc_map_at_5_std": -0.06823912896230401, - "nauc_mrr_at_1000_diff1": 0.08578620852479406, - "nauc_mrr_at_1000_max": -0.022301287770317674, - "nauc_mrr_at_1000_std": -0.07002051705337815, - "nauc_mrr_at_100_diff1": 0.09059184648214072, - "nauc_mrr_at_100_max": -0.02730487433721151, - "nauc_mrr_at_100_std": -0.07112681168567145, - "nauc_mrr_at_10_diff1": 0.10561558040018261, - "nauc_mrr_at_10_max": -0.03887680003815409, - "nauc_mrr_at_10_std": -0.07384905648271843, - "nauc_mrr_at_1_diff1": 0.17890721176822888, - "nauc_mrr_at_1_max": -0.08979014629444568, - "nauc_mrr_at_1_std": -0.07645555550866222, - "nauc_mrr_at_20_diff1": 0.09999678477824628, - "nauc_mrr_at_20_max": -0.033642117064318564, - "nauc_mrr_at_20_std": -0.08006217604646698, - "nauc_mrr_at_3_diff1": 0.13264050928045265, - "nauc_mrr_at_3_max": -0.05878443289453297, - "nauc_mrr_at_3_std": -0.06106821971711554, - "nauc_mrr_at_5_diff1": 0.11792502863936245, - "nauc_mrr_at_5_max": -0.04439031923384928, - "nauc_mrr_at_5_std": -0.06823912896230401, - "nauc_ndcg_at_1000_diff1": 0.03027392253878934, - "nauc_ndcg_at_1000_max": 0.03872756726005947, - "nauc_ndcg_at_1000_std": -0.06339954815558649, - "nauc_ndcg_at_100_diff1": 0.060190435328092255, - "nauc_ndcg_at_100_max": -0.006149217698640977, - "nauc_ndcg_at_100_std": -0.05423336919167363, - "nauc_ndcg_at_10_diff1": 0.08801115068128108, - "nauc_ndcg_at_10_max": -0.027958780559606267, - "nauc_ndcg_at_10_std": -0.0798528780495694, - "nauc_ndcg_at_1_diff1": 0.17890721176822888, - "nauc_ndcg_at_1_max": -0.08979014629444568, - "nauc_ndcg_at_1_std": -0.07645555550866222, - "nauc_ndcg_at_20_diff1": 0.08098580648408388, - "nauc_ndcg_at_20_max": -0.019828494542212433, - "nauc_ndcg_at_20_std": -0.09119555503912527, - "nauc_ndcg_at_3_diff1": 0.12374455660074443, - "nauc_ndcg_at_3_max": -0.05335042807216969, - "nauc_ndcg_at_3_std": -0.06082434994775198, - "nauc_ndcg_at_5_diff1": 0.10480672753283997, - "nauc_ndcg_at_5_max": -0.03413521896711701, - "nauc_ndcg_at_5_std": -0.07135709274825974, - "nauc_precision_at_1000_diff1": 0.00882583756579781, - "nauc_precision_at_1000_max": 0.06850469648818812, - "nauc_precision_at_1000_std": -0.06518141667809778, - "nauc_precision_at_100_diff1": 0.045034516026494456, - "nauc_precision_at_100_max": 0.004398660634268171, - "nauc_precision_at_100_std": -0.039755619188003716, - "nauc_precision_at_10_diff1": 0.06821726635055005, - "nauc_precision_at_10_max": -0.01603124686173649, - "nauc_precision_at_10_std": -0.08733387779771956, - "nauc_precision_at_1_diff1": 0.17890721176822888, - "nauc_precision_at_1_max": -0.08979014629444568, - "nauc_precision_at_1_std": -0.07645555550866222, - "nauc_precision_at_20_diff1": 0.06573644523017923, - "nauc_precision_at_20_max": -0.008446727739883295, - "nauc_precision_at_20_std": -0.10159027094805619, - "nauc_precision_at_3_diff1": 0.10725987074701417, - "nauc_precision_at_3_max": -0.04337993473508582, - "nauc_precision_at_3_std": -0.060881885558147046, - "nauc_precision_at_5_diff1": 0.0850241314003234, - "nauc_precision_at_5_max": -0.01852186860300637, - "nauc_precision_at_5_std": -0.0768119856475613, - "nauc_recall_at_1000_diff1": 0.008825837565798213, - "nauc_recall_at_1000_max": 0.06850469648818859, - "nauc_recall_at_1000_std": -0.0651814166780974, - "nauc_recall_at_100_diff1": 0.045034516026494435, - "nauc_recall_at_100_max": 0.004398660634268218, - "nauc_recall_at_100_std": -0.03975561918800372, - "nauc_recall_at_10_diff1": 0.06821726635054984, - "nauc_recall_at_10_max": -0.01603124686173661, - "nauc_recall_at_10_std": -0.08733387779771963, - "nauc_recall_at_1_diff1": 0.17890721176822888, - "nauc_recall_at_1_max": -0.08979014629444568, - "nauc_recall_at_1_std": -0.07645555550866222, - "nauc_recall_at_20_diff1": 0.06573644523017931, - "nauc_recall_at_20_max": -0.008446727739883213, - "nauc_recall_at_20_std": -0.10159027094805619, - "nauc_recall_at_3_diff1": 0.10725987074701412, - "nauc_recall_at_3_max": -0.04337993473508564, - "nauc_recall_at_3_std": -0.06088188555814698, - "nauc_recall_at_5_diff1": 0.08502413140032329, - "nauc_recall_at_5_max": -0.018521868603006558, - "nauc_recall_at_5_std": -0.07681198564756143, - "ndcg_at_1": 0.00224, - "ndcg_at_10": 0.0082, - "ndcg_at_100": 0.02041, - "ndcg_at_1000": 0.04417, - "ndcg_at_20": 0.01126, - "ndcg_at_3": 0.00448, - "ndcg_at_5": 0.00588, - "precision_at_1": 0.00224, - "precision_at_10": 0.00169, - "precision_at_100": 0.0008, - "precision_at_1000": 0.00029, - "precision_at_20": 0.00145, - "precision_at_3": 0.00206, - "precision_at_5": 0.00192, - "recall_at_1": 0.00224, - "recall_at_10": 0.01687, - "recall_at_100": 0.07996, - "recall_at_1000": 0.2859, - "recall_at_20": 0.02905, - "recall_at_3": 0.00618, - "recall_at_5": 0.00962 - } - ] - }, - "task_name": "VQA2IT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreArxivQARetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreArxivQARetrieval.json deleted file mode 100644 index 6c91af01bf..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreArxivQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", - "evaluation_time": 11.5097815990448, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.118, - "cv_recall_at_10": 0.276, - "cv_recall_at_100": 0.65, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.366, - "cv_recall_at_3": 0.178, - "cv_recall_at_5": 0.216, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.16803, - "map_at_1": 0.118, - "map_at_10": 0.16063, - "map_at_100": 0.17306, - "map_at_1000": 0.17487, - "map_at_20": 0.16669, - "map_at_3": 0.14367, - "map_at_5": 0.15227, - "mrr_at_1": 0.118, - "mrr_at_10": 0.16062539682539684, - "mrr_at_100": 0.17306236569997144, - "mrr_at_1000": 0.17487418229364743, - "mrr_at_20": 0.16669178075966007, - "mrr_at_3": 0.1436666666666667, - "mrr_at_5": 0.15226666666666672, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.18296688095044317, - "nauc_cv_recall_at_100_max": 0.13051346732850505, - "nauc_cv_recall_at_100_std": 0.0815916673447794, - "nauc_cv_recall_at_10_diff1": 0.21002008066013042, - "nauc_cv_recall_at_10_max": 0.11474944173514898, - "nauc_cv_recall_at_10_std": -0.08737055848984453, - "nauc_cv_recall_at_1_diff1": 0.485761413165958, - "nauc_cv_recall_at_1_max": 0.27947811827209385, - "nauc_cv_recall_at_1_std": -0.14702257939228888, - "nauc_cv_recall_at_20_diff1": 0.15293554287329153, - "nauc_cv_recall_at_20_max": 0.06147670746099138, - "nauc_cv_recall_at_20_std": -0.0882638469126888, - "nauc_cv_recall_at_3_diff1": 0.32625290643148597, - "nauc_cv_recall_at_3_max": 0.1682279703995215, - "nauc_cv_recall_at_3_std": -0.11057711023736298, - "nauc_cv_recall_at_5_diff1": 0.25332857375468115, - "nauc_cv_recall_at_5_max": 0.14352947807462518, - "nauc_cv_recall_at_5_std": -0.11418604942446668, - "nauc_map_at_1000_diff1": 0.36779035073667693, - "nauc_map_at_1000_max": 0.20151349002499713, - "nauc_map_at_1000_std": -0.11756000190443665, - "nauc_map_at_100_diff1": 0.3669486782669981, - "nauc_map_at_100_max": 0.20130857725652113, - "nauc_map_at_100_std": -0.11682548553478848, - "nauc_map_at_10_diff1": 0.37227377992149197, - "nauc_map_at_10_max": 0.20513150531011887, - "nauc_map_at_10_std": -0.12196059276209226, - "nauc_map_at_1_diff1": 0.485761413165958, - "nauc_map_at_1_max": 0.27947811827209385, - "nauc_map_at_1_std": -0.14702257939228888, - "nauc_map_at_20_diff1": 0.36526766556678214, - "nauc_map_at_20_max": 0.1986343132176448, - "nauc_map_at_20_std": -0.12266526320796677, - "nauc_map_at_3_diff1": 0.4086126400407416, - "nauc_map_at_3_max": 0.22074868328572508, - "nauc_map_at_3_std": -0.12634281959678811, - "nauc_map_at_5_diff1": 0.384865760968298, - "nauc_map_at_5_max": 0.2118497650368132, - "nauc_map_at_5_std": -0.12708878494906578, - "nauc_mrr_at_1000_diff1": 0.36779035073667693, - "nauc_mrr_at_1000_max": 0.20151349002499713, - "nauc_mrr_at_1000_std": -0.11756000190443665, - "nauc_mrr_at_100_diff1": 0.3669486782669981, - "nauc_mrr_at_100_max": 0.20130857725652113, - "nauc_mrr_at_100_std": -0.11682548553478848, - "nauc_mrr_at_10_diff1": 0.37227377992149197, - "nauc_mrr_at_10_max": 0.20513150531011887, - "nauc_mrr_at_10_std": -0.12196059276209226, - "nauc_mrr_at_1_diff1": 0.485761413165958, - "nauc_mrr_at_1_max": 0.27947811827209385, - "nauc_mrr_at_1_std": -0.14702257939228888, - "nauc_mrr_at_20_diff1": 0.36526766556678214, - "nauc_mrr_at_20_max": 0.1986343132176448, - "nauc_mrr_at_20_std": -0.12266526320796677, - "nauc_mrr_at_3_diff1": 0.4086126400407416, - "nauc_mrr_at_3_max": 0.22074868328572508, - "nauc_mrr_at_3_std": -0.12634281959678811, - "nauc_mrr_at_5_diff1": 0.384865760968298, - "nauc_mrr_at_5_max": 0.2118497650368132, - "nauc_mrr_at_5_std": -0.12708878494906578, - "nauc_ndcg_at_1000_diff1": 0.3339421857835854, - "nauc_ndcg_at_1000_max": 0.18217616137925813, - "nauc_ndcg_at_1000_std": -0.10173651860955586, - "nauc_ndcg_at_100_diff1": 0.3131503412000536, - "nauc_ndcg_at_100_max": 0.17582437121383968, - "nauc_ndcg_at_100_std": -0.07731985140025688, - "nauc_ndcg_at_10_diff1": 0.32246312193822635, - "nauc_ndcg_at_10_max": 0.1773668049229422, - "nauc_ndcg_at_10_std": -0.11172669555093219, - "nauc_ndcg_at_1_diff1": 0.485761413165958, - "nauc_ndcg_at_1_max": 0.27947811827209385, - "nauc_ndcg_at_1_std": -0.14702257939228888, - "nauc_ndcg_at_20_diff1": 0.3005922257636208, - "nauc_ndcg_at_20_max": 0.1575132660989431, - "nauc_ndcg_at_20_std": -0.11244207879437416, - "nauc_ndcg_at_3_diff1": 0.38501728903920196, - "nauc_ndcg_at_3_max": 0.2055141399629287, - "nauc_ndcg_at_3_std": -0.12170959892325568, - "nauc_ndcg_at_5_diff1": 0.34602861150963293, - "nauc_ndcg_at_5_max": 0.19130504508028406, - "nauc_ndcg_at_5_std": -0.12304138641095921, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.18296688095044372, - "nauc_precision_at_100_max": 0.13051346732850508, - "nauc_precision_at_100_std": 0.0815916673447802, - "nauc_precision_at_10_diff1": 0.210020080660131, - "nauc_precision_at_10_max": 0.11474944173514932, - "nauc_precision_at_10_std": -0.08737055848984408, - "nauc_precision_at_1_diff1": 0.485761413165958, - "nauc_precision_at_1_max": 0.27947811827209385, - "nauc_precision_at_1_std": -0.14702257939228888, - "nauc_precision_at_20_diff1": 0.15293554287329167, - "nauc_precision_at_20_max": 0.06147670746099171, - "nauc_precision_at_20_std": -0.0882638469126885, - "nauc_precision_at_3_diff1": 0.32625290643148613, - "nauc_precision_at_3_max": 0.1682279703995217, - "nauc_precision_at_3_std": -0.11057711023736295, - "nauc_precision_at_5_diff1": 0.2533285737546817, - "nauc_precision_at_5_max": 0.14352947807462554, - "nauc_precision_at_5_std": -0.11418604942446653, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.18296688095044317, - "nauc_recall_at_100_max": 0.13051346732850505, - "nauc_recall_at_100_std": 0.0815916673447794, - "nauc_recall_at_10_diff1": 0.21002008066013042, - "nauc_recall_at_10_max": 0.11474944173514898, - "nauc_recall_at_10_std": -0.08737055848984453, - "nauc_recall_at_1_diff1": 0.485761413165958, - "nauc_recall_at_1_max": 0.27947811827209385, - "nauc_recall_at_1_std": -0.14702257939228888, - "nauc_recall_at_20_diff1": 0.15293554287329153, - "nauc_recall_at_20_max": 0.06147670746099138, - "nauc_recall_at_20_std": -0.0882638469126888, - "nauc_recall_at_3_diff1": 0.32625290643148597, - "nauc_recall_at_3_max": 0.1682279703995215, - "nauc_recall_at_3_std": -0.11057711023736298, - "nauc_recall_at_5_diff1": 0.25332857375468115, - "nauc_recall_at_5_max": 0.14352947807462518, - "nauc_recall_at_5_std": -0.11418604942446668, - "ndcg_at_1": 0.118, - "ndcg_at_10": 0.18779, - "ndcg_at_100": 0.26119, - "ndcg_at_1000": 0.3067, - "ndcg_at_20": 0.21032, - "ndcg_at_3": 0.15245, - "ndcg_at_5": 0.16803, - "precision_at_1": 0.118, - "precision_at_10": 0.0276, - "precision_at_100": 0.0065, - "precision_at_1000": 0.001, - "precision_at_20": 0.0183, - "precision_at_3": 0.05933, - "precision_at_5": 0.0432, - "recall_at_1": 0.118, - "recall_at_10": 0.276, - "recall_at_100": 0.65, - "recall_at_1000": 1.0, - "recall_at_20": 0.366, - "recall_at_3": 0.178, - "recall_at_5": 0.216 - } - ] - }, - "task_name": "VidoreArxivQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreDocVQARetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreDocVQARetrieval.json deleted file mode 100644 index e07536809f..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreDocVQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "b1d89eda849e636676df6ead8002602fb1858600", - "evaluation_time": 16.02955412864685, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.058, - "cv_recall_at_10": 0.172, - "cv_recall_at_100": 0.54, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.262, - "cv_recall_at_3": 0.1, - "cv_recall_at_5": 0.13, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.0948, - "map_at_1": 0.058, - "map_at_10": 0.08928, - "map_at_100": 0.10155, - "map_at_1000": 0.10376, - "map_at_20": 0.09538, - "map_at_3": 0.07667, - "map_at_5": 0.08327, - "mrr_at_1": 0.058, - "mrr_at_10": 0.08927777777777782, - "mrr_at_100": 0.10155225471194389, - "mrr_at_1000": 0.10375791922323688, - "mrr_at_20": 0.09537714343619923, - "mrr_at_3": 0.07666666666666667, - "mrr_at_5": 0.0832666666666667, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.032680392320302315, - "nauc_cv_recall_at_100_max": 0.008841099163679788, - "nauc_cv_recall_at_100_std": 0.25023478091745127, - "nauc_cv_recall_at_10_diff1": 0.05107536651136432, - "nauc_cv_recall_at_10_max": 0.09401370828820849, - "nauc_cv_recall_at_10_std": 0.14515449080905074, - "nauc_cv_recall_at_1_diff1": 0.14420399137699735, - "nauc_cv_recall_at_1_max": 0.17870525429531003, - "nauc_cv_recall_at_1_std": 0.07417447702228848, - "nauc_cv_recall_at_20_diff1": 0.06076049285975611, - "nauc_cv_recall_at_20_max": 0.07384829898852548, - "nauc_cv_recall_at_20_std": 0.17957633881371404, - "nauc_cv_recall_at_3_diff1": 0.10039173598068159, - "nauc_cv_recall_at_3_max": 0.2013791253018513, - "nauc_cv_recall_at_3_std": 0.1424953045344781, - "nauc_cv_recall_at_5_diff1": 0.027530164983994042, - "nauc_cv_recall_at_5_max": 0.12327013050972653, - "nauc_cv_recall_at_5_std": 0.10974009984106002, - "nauc_map_at_1000_diff1": 0.10296205110535936, - "nauc_map_at_1000_max": 0.15862975165487792, - "nauc_map_at_1000_std": 0.12497468408354453, - "nauc_map_at_100_diff1": 0.10261504617825842, - "nauc_map_at_100_max": 0.15700943002591616, - "nauc_map_at_100_std": 0.12577033745669342, - "nauc_map_at_10_diff1": 0.10233479631218023, - "nauc_map_at_10_max": 0.1655998866614918, - "nauc_map_at_10_std": 0.11614651176369198, - "nauc_map_at_1_diff1": 0.14420399137699735, - "nauc_map_at_1_max": 0.17870525429531003, - "nauc_map_at_1_std": 0.07417447702228848, - "nauc_map_at_20_diff1": 0.10286768495886522, - "nauc_map_at_20_max": 0.16077790911145834, - "nauc_map_at_20_std": 0.12225269367497407, - "nauc_map_at_3_diff1": 0.12587697297045058, - "nauc_map_at_3_max": 0.19920439565567363, - "nauc_map_at_3_std": 0.11490533241562752, - "nauc_map_at_5_diff1": 0.1008074992224546, - "nauc_map_at_5_max": 0.17466259834098666, - "nauc_map_at_5_std": 0.10658693949141812, - "nauc_mrr_at_1000_diff1": 0.10296205110535936, - "nauc_mrr_at_1000_max": 0.15862975165487792, - "nauc_mrr_at_1000_std": 0.12497468408354453, - "nauc_mrr_at_100_diff1": 0.10261504617825842, - "nauc_mrr_at_100_max": 0.15700943002591616, - "nauc_mrr_at_100_std": 0.12577033745669342, - "nauc_mrr_at_10_diff1": 0.10233479631218023, - "nauc_mrr_at_10_max": 0.1655998866614918, - "nauc_mrr_at_10_std": 0.11614651176369198, - "nauc_mrr_at_1_diff1": 0.14420399137699735, - "nauc_mrr_at_1_max": 0.17870525429531003, - "nauc_mrr_at_1_std": 0.07417447702228848, - "nauc_mrr_at_20_diff1": 0.10286768495886522, - "nauc_mrr_at_20_max": 0.16077790911145834, - "nauc_mrr_at_20_std": 0.12225269367497407, - "nauc_mrr_at_3_diff1": 0.12587697297045058, - "nauc_mrr_at_3_max": 0.19920439565567363, - "nauc_mrr_at_3_std": 0.11490533241562752, - "nauc_mrr_at_5_diff1": 0.1008074992224546, - "nauc_mrr_at_5_max": 0.17466259834098666, - "nauc_mrr_at_5_std": 0.10658693949141812, - "nauc_ndcg_at_1000_diff1": 0.08993939510748966, - "nauc_ndcg_at_1000_max": 0.13722131092451595, - "nauc_ndcg_at_1000_std": 0.14394044997396738, - "nauc_ndcg_at_100_diff1": 0.07881330019181253, - "nauc_ndcg_at_100_max": 0.10857797405282689, - "nauc_ndcg_at_100_std": 0.16533729610286177, - "nauc_ndcg_at_10_diff1": 0.08389649942218956, - "nauc_ndcg_at_10_max": 0.14191890938892987, - "nauc_ndcg_at_10_std": 0.12645751380780731, - "nauc_ndcg_at_1_diff1": 0.14420399137699735, - "nauc_ndcg_at_1_max": 0.17870525429531003, - "nauc_ndcg_at_1_std": 0.07417447702228848, - "nauc_ndcg_at_20_diff1": 0.08576639195224166, - "nauc_ndcg_at_20_max": 0.12935318913387056, - "nauc_ndcg_at_20_std": 0.14299439642394743, - "nauc_ndcg_at_3_diff1": 0.11823031088822708, - "nauc_ndcg_at_3_max": 0.2001720221732457, - "nauc_ndcg_at_3_std": 0.12353115467979815, - "nauc_ndcg_at_5_diff1": 0.07772393650891742, - "nauc_ndcg_at_5_max": 0.1591303206130916, - "nauc_ndcg_at_5_std": 0.10837344904056297, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.03268039232030201, - "nauc_precision_at_100_max": 0.008841099163679927, - "nauc_precision_at_100_std": 0.25023478091745144, - "nauc_precision_at_10_diff1": 0.051075366511364524, - "nauc_precision_at_10_max": 0.09401370828820864, - "nauc_precision_at_10_std": 0.1451544908090511, - "nauc_precision_at_1_diff1": 0.14420399137699735, - "nauc_precision_at_1_max": 0.17870525429531003, - "nauc_precision_at_1_std": 0.07417447702228848, - "nauc_precision_at_20_diff1": 0.060760492859756446, - "nauc_precision_at_20_max": 0.07384829898852595, - "nauc_precision_at_20_std": 0.17957633881371443, - "nauc_precision_at_3_diff1": 0.1003917359806816, - "nauc_precision_at_3_max": 0.20137912530185134, - "nauc_precision_at_3_std": 0.14249530453447826, - "nauc_precision_at_5_diff1": 0.027530164983994375, - "nauc_precision_at_5_max": 0.12327013050972686, - "nauc_precision_at_5_std": 0.10974009984106038, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.032680392320302315, - "nauc_recall_at_100_max": 0.008841099163679788, - "nauc_recall_at_100_std": 0.25023478091745127, - "nauc_recall_at_10_diff1": 0.05107536651136432, - "nauc_recall_at_10_max": 0.09401370828820849, - "nauc_recall_at_10_std": 0.14515449080905074, - "nauc_recall_at_1_diff1": 0.14420399137699735, - "nauc_recall_at_1_max": 0.17870525429531003, - "nauc_recall_at_1_std": 0.07417447702228848, - "nauc_recall_at_20_diff1": 0.06076049285975611, - "nauc_recall_at_20_max": 0.07384829898852548, - "nauc_recall_at_20_std": 0.17957633881371404, - "nauc_recall_at_3_diff1": 0.10039173598068159, - "nauc_recall_at_3_max": 0.2013791253018513, - "nauc_recall_at_3_std": 0.1424953045344781, - "nauc_recall_at_5_diff1": 0.027530164983994042, - "nauc_recall_at_5_max": 0.12327013050972653, - "nauc_recall_at_5_std": 0.10974009984106002, - "ndcg_at_1": 0.058, - "ndcg_at_10": 0.10879, - "ndcg_at_100": 0.18103, - "ndcg_at_1000": 0.23998, - "ndcg_at_20": 0.13135, - "ndcg_at_3": 0.08267, - "ndcg_at_5": 0.0948, - "precision_at_1": 0.058, - "precision_at_10": 0.0172, - "precision_at_100": 0.0054, - "precision_at_1000": 0.001, - "precision_at_20": 0.0131, - "precision_at_3": 0.03333, - "precision_at_5": 0.026, - "recall_at_1": 0.058, - "recall_at_10": 0.172, - "recall_at_100": 0.54, - "recall_at_1000": 1.0, - "recall_at_20": 0.262, - "recall_at_3": 0.1, - "recall_at_5": 0.13 - } - ] - }, - "task_name": "VidoreDocVQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreInfoVQARetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreInfoVQARetrieval.json deleted file mode 100644 index 79c911b0de..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreInfoVQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", - "evaluation_time": 15.756592273712158, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.274, - "cv_recall_at_10": 0.548, - "cv_recall_at_100": 0.834, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.628, - "cv_recall_at_3": 0.392, - "cv_recall_at_5": 0.458, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.36997, - "map_at_1": 0.274, - "map_at_10": 0.35329, - "map_at_100": 0.36413, - "map_at_1000": 0.36503, - "map_at_20": 0.35891, - "map_at_3": 0.32567, - "map_at_5": 0.34087, - "mrr_at_1": 0.274, - "mrr_at_10": 0.35328888888888876, - "mrr_at_100": 0.3641305092820594, - "mrr_at_1000": 0.3650337866097008, - "mrr_at_20": 0.3589137667801754, - "mrr_at_3": 0.3256666666666667, - "mrr_at_5": 0.34086666666666665, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4108195982105783, - "nauc_cv_recall_at_100_max": 0.3513699524165519, - "nauc_cv_recall_at_100_std": 0.3340927699263115, - "nauc_cv_recall_at_10_diff1": 0.3707691359200985, - "nauc_cv_recall_at_10_max": 0.2438081840142132, - "nauc_cv_recall_at_10_std": 0.15858559250312437, - "nauc_cv_recall_at_1_diff1": 0.4619750027514469, - "nauc_cv_recall_at_1_max": 0.2773571434618564, - "nauc_cv_recall_at_1_std": 0.09954311870165566, - "nauc_cv_recall_at_20_diff1": 0.4034132420804166, - "nauc_cv_recall_at_20_max": 0.2844867960110151, - "nauc_cv_recall_at_20_std": 0.15804197550908236, - "nauc_cv_recall_at_3_diff1": 0.4065778999927638, - "nauc_cv_recall_at_3_max": 0.2496459326186513, - "nauc_cv_recall_at_3_std": 0.13659247619736797, - "nauc_cv_recall_at_5_diff1": 0.39779343598416905, - "nauc_cv_recall_at_5_max": 0.25945904777912304, - "nauc_cv_recall_at_5_std": 0.17263041234541332, - "nauc_map_at_1000_diff1": 0.4289400319756985, - "nauc_map_at_1000_max": 0.2689506737673011, - "nauc_map_at_1000_std": 0.12803419134610242, - "nauc_map_at_100_diff1": 0.42885442258511014, - "nauc_map_at_100_max": 0.2692582970619958, - "nauc_map_at_100_std": 0.12843203024529296, - "nauc_map_at_10_diff1": 0.4262212910098287, - "nauc_map_at_10_max": 0.26616493787234696, - "nauc_map_at_10_std": 0.1265953804223453, - "nauc_map_at_1_diff1": 0.4619750027514469, - "nauc_map_at_1_max": 0.2773571434618564, - "nauc_map_at_1_std": 0.09954311870165566, - "nauc_map_at_20_diff1": 0.4284448343118344, - "nauc_map_at_20_max": 0.2690760269120174, - "nauc_map_at_20_std": 0.12663495864569618, - "nauc_map_at_3_diff1": 0.43187220501498585, - "nauc_map_at_3_max": 0.2654933953846897, - "nauc_map_at_3_std": 0.11997884284630515, - "nauc_map_at_5_diff1": 0.4296497686024672, - "nauc_map_at_5_max": 0.2687269018079568, - "nauc_map_at_5_std": 0.12984081889984647, - "nauc_mrr_at_1000_diff1": 0.4289400319756985, - "nauc_mrr_at_1000_max": 0.2689506737673011, - "nauc_mrr_at_1000_std": 0.12803419134610242, - "nauc_mrr_at_100_diff1": 0.42885442258511014, - "nauc_mrr_at_100_max": 0.2692582970619958, - "nauc_mrr_at_100_std": 0.12843203024529296, - "nauc_mrr_at_10_diff1": 0.4262212910098287, - "nauc_mrr_at_10_max": 0.26616493787234696, - "nauc_mrr_at_10_std": 0.1265953804223453, - "nauc_mrr_at_1_diff1": 0.4619750027514469, - "nauc_mrr_at_1_max": 0.2773571434618564, - "nauc_mrr_at_1_std": 0.09954311870165566, - "nauc_mrr_at_20_diff1": 0.4284448343118344, - "nauc_mrr_at_20_max": 0.2690760269120174, - "nauc_mrr_at_20_std": 0.12663495864569618, - "nauc_mrr_at_3_diff1": 0.43187220501498585, - "nauc_mrr_at_3_max": 0.2654933953846897, - "nauc_mrr_at_3_std": 0.11997884284630515, - "nauc_mrr_at_5_diff1": 0.4296497686024672, - "nauc_mrr_at_5_max": 0.2687269018079568, - "nauc_mrr_at_5_std": 0.12984081889984647, - "nauc_ndcg_at_1000_diff1": 0.42490331218837485, - "nauc_ndcg_at_1000_max": 0.26923686243472, - "nauc_ndcg_at_1000_std": 0.13661973225278048, - "nauc_ndcg_at_100_diff1": 0.4237501775481191, - "nauc_ndcg_at_100_max": 0.2754123277695727, - "nauc_ndcg_at_100_std": 0.14932637461067172, - "nauc_ndcg_at_10_diff1": 0.4129110395057173, - "nauc_ndcg_at_10_max": 0.2608695707815544, - "nauc_ndcg_at_10_std": 0.1347290633153979, - "nauc_ndcg_at_1_diff1": 0.4619750027514469, - "nauc_ndcg_at_1_max": 0.2773571434618564, - "nauc_ndcg_at_1_std": 0.09954311870165566, - "nauc_ndcg_at_20_diff1": 0.4212506902996852, - "nauc_ndcg_at_20_max": 0.2709472172227333, - "nauc_ndcg_at_20_std": 0.13434839262447948, - "nauc_ndcg_at_3_diff1": 0.4249611000998995, - "nauc_ndcg_at_3_max": 0.2613207228374443, - "nauc_ndcg_at_3_std": 0.12453307166402308, - "nauc_ndcg_at_5_diff1": 0.42116772431902333, - "nauc_ndcg_at_5_max": 0.26628491565434564, - "nauc_ndcg_at_5_std": 0.14088721346867775, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.4108195982105766, - "nauc_precision_at_100_max": 0.3513699524165505, - "nauc_precision_at_100_std": 0.3340927699263106, - "nauc_precision_at_10_diff1": 0.37076913592009925, - "nauc_precision_at_10_max": 0.24380818401421336, - "nauc_precision_at_10_std": 0.1585855925031248, - "nauc_precision_at_1_diff1": 0.4619750027514469, - "nauc_precision_at_1_max": 0.2773571434618564, - "nauc_precision_at_1_std": 0.09954311870165566, - "nauc_precision_at_20_diff1": 0.40341324208041696, - "nauc_precision_at_20_max": 0.2844867960110156, - "nauc_precision_at_20_std": 0.1580419755090828, - "nauc_precision_at_3_diff1": 0.4065778999927632, - "nauc_precision_at_3_max": 0.2496459326186512, - "nauc_precision_at_3_std": 0.136592476197368, - "nauc_precision_at_5_diff1": 0.397793435984169, - "nauc_precision_at_5_max": 0.25945904777912304, - "nauc_precision_at_5_std": 0.1726304123454132, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4108195982105783, - "nauc_recall_at_100_max": 0.3513699524165519, - "nauc_recall_at_100_std": 0.3340927699263115, - "nauc_recall_at_10_diff1": 0.3707691359200985, - "nauc_recall_at_10_max": 0.2438081840142132, - "nauc_recall_at_10_std": 0.15858559250312437, - "nauc_recall_at_1_diff1": 0.4619750027514469, - "nauc_recall_at_1_max": 0.2773571434618564, - "nauc_recall_at_1_std": 0.09954311870165566, - "nauc_recall_at_20_diff1": 0.4034132420804166, - "nauc_recall_at_20_max": 0.2844867960110151, - "nauc_recall_at_20_std": 0.15804197550908236, - "nauc_recall_at_3_diff1": 0.4065778999927638, - "nauc_recall_at_3_max": 0.2496459326186513, - "nauc_recall_at_3_std": 0.13659247619736797, - "nauc_recall_at_5_diff1": 0.39779343598416905, - "nauc_recall_at_5_max": 0.25945904777912304, - "nauc_recall_at_5_std": 0.17263041234541332, - "ndcg_at_1": 0.274, - "ndcg_at_10": 0.39949, - "ndcg_at_100": 0.45791, - "ndcg_at_1000": 0.47973, - "ndcg_at_20": 0.41982, - "ndcg_at_3": 0.34269, - "ndcg_at_5": 0.36997, - "precision_at_1": 0.274, - "precision_at_10": 0.0548, - "precision_at_100": 0.00834, - "precision_at_1000": 0.001, - "precision_at_20": 0.0314, - "precision_at_3": 0.13067, - "precision_at_5": 0.0916, - "recall_at_1": 0.274, - "recall_at_10": 0.548, - "recall_at_100": 0.834, - "recall_at_1000": 1.0, - "recall_at_20": 0.628, - "recall_at_3": 0.392, - "recall_at_5": 0.458 - } - ] - }, - "task_name": "VidoreInfoVQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreShiftProjectRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreShiftProjectRetrieval.json deleted file mode 100644 index 65d569d994..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreShiftProjectRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", - "evaluation_time": 30.601240396499634, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.01, - "cv_recall_at_10": 0.03, - "cv_recall_at_100": 0.19, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.05, - "cv_recall_at_3": 0.01, - "cv_recall_at_5": 0.01, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.01, - "map_at_1": 0.01, - "map_at_10": 0.01243, - "map_at_100": 0.01593, - "map_at_1000": 0.01858, - "map_at_20": 0.01345, - "map_at_3": 0.01, - "map_at_5": 0.01, - "mrr_at_1": 0.01, - "mrr_at_10": 0.01242857142857143, - "mrr_at_100": 0.01592918539143402, - "mrr_at_1000": 0.018579131316456693, - "mrr_at_20": 0.013454887218045114, - "mrr_at_3": 0.01, - "mrr_at_5": 0.01, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.03801725152590248, - "nauc_cv_recall_at_100_max": -0.08736737679283049, - "nauc_cv_recall_at_100_std": -0.27136683873354306, - "nauc_cv_recall_at_10_diff1": 0.04588140595653343, - "nauc_cv_recall_at_10_max": -0.35032644665056795, - "nauc_cv_recall_at_10_std": -0.1259279134245595, - "nauc_cv_recall_at_1_diff1": 0.6619264824255433, - "nauc_cv_recall_at_1_max": -0.5747249798765763, - "nauc_cv_recall_at_1_std": -0.5747249798765763, - "nauc_cv_recall_at_20_diff1": 0.05999463375368926, - "nauc_cv_recall_at_20_max": -0.2735175744566677, - "nauc_cv_recall_at_20_std": -0.20461497182720695, - "nauc_cv_recall_at_3_diff1": 0.6619264824255433, - "nauc_cv_recall_at_3_max": -0.5747249798765763, - "nauc_cv_recall_at_3_std": -0.5747249798765763, - "nauc_cv_recall_at_5_diff1": 0.6619264824255433, - "nauc_cv_recall_at_5_max": -0.5747249798765763, - "nauc_cv_recall_at_5_std": -0.5747249798765763, - "nauc_map_at_1000_diff1": 0.37055532373093525, - "nauc_map_at_1000_max": -0.4409673650509871, - "nauc_map_at_1000_std": -0.39942667927544195, - "nauc_map_at_100_diff1": 0.38208823611071047, - "nauc_map_at_100_max": -0.44308890084107116, - "nauc_map_at_100_std": -0.4127580396465241, - "nauc_map_at_10_diff1": 0.47941859496868144, - "nauc_map_at_10_max": -0.5205598166840916, - "nauc_map_at_10_std": -0.443181012123399, - "nauc_map_at_1_diff1": 0.6619264824255433, - "nauc_map_at_1_max": -0.5747249798765763, - "nauc_map_at_1_std": -0.5747249798765763, - "nauc_map_at_20_diff1": 0.45017633719278727, - "nauc_map_at_20_max": -0.4924253686557989, - "nauc_map_at_20_std": -0.4344798043441707, - "nauc_map_at_3_diff1": 0.6619264824255433, - "nauc_map_at_3_max": -0.5747249798765763, - "nauc_map_at_3_std": -0.5747249798765763, - "nauc_map_at_5_diff1": 0.6619264824255433, - "nauc_map_at_5_max": -0.5747249798765763, - "nauc_map_at_5_std": -0.5747249798765763, - "nauc_mrr_at_1000_diff1": 0.37055532373093525, - "nauc_mrr_at_1000_max": -0.4409673650509871, - "nauc_mrr_at_1000_std": -0.39942667927544195, - "nauc_mrr_at_100_diff1": 0.38208823611071047, - "nauc_mrr_at_100_max": -0.44308890084107116, - "nauc_mrr_at_100_std": -0.4127580396465241, - "nauc_mrr_at_10_diff1": 0.47941859496868144, - "nauc_mrr_at_10_max": -0.5205598166840916, - "nauc_mrr_at_10_std": -0.443181012123399, - "nauc_mrr_at_1_diff1": 0.6619264824255433, - "nauc_mrr_at_1_max": -0.5747249798765763, - "nauc_mrr_at_1_std": -0.5747249798765763, - "nauc_mrr_at_20_diff1": 0.45017633719278727, - "nauc_mrr_at_20_max": -0.4924253686557989, - "nauc_mrr_at_20_std": -0.4344798043441707, - "nauc_mrr_at_3_diff1": 0.6619264824255433, - "nauc_mrr_at_3_max": -0.5747249798765763, - "nauc_mrr_at_3_std": -0.5747249798765763, - "nauc_mrr_at_5_diff1": 0.6619264824255433, - "nauc_mrr_at_5_max": -0.5747249798765763, - "nauc_mrr_at_5_std": -0.5747249798765763, - "nauc_ndcg_at_1000_diff1": 0.1597681443227891, - "nauc_ndcg_at_1000_max": -0.3000341630497847, - "nauc_ndcg_at_1000_std": -0.2766624383259684, - "nauc_ndcg_at_100_diff1": 0.13848351210393903, - "nauc_ndcg_at_100_max": -0.21869254918959663, - "nauc_ndcg_at_100_std": -0.30606324798461026, - "nauc_ndcg_at_10_diff1": 0.3058903797405613, - "nauc_ndcg_at_10_max": -0.454780829335424, - "nauc_ndcg_at_10_std": -0.3164679670512058, - "nauc_ndcg_at_1_diff1": 0.6619264824255433, - "nauc_ndcg_at_1_max": -0.5747249798765763, - "nauc_ndcg_at_1_std": -0.5747249798765763, - "nauc_ndcg_at_20_diff1": 0.2573632473234855, - "nauc_ndcg_at_20_max": -0.38893757347888275, - "nauc_ndcg_at_20_std": -0.31827945516426087, - "nauc_ndcg_at_3_diff1": 0.6619264824255433, - "nauc_ndcg_at_3_max": -0.5747249798765763, - "nauc_ndcg_at_3_std": -0.5747249798765763, - "nauc_ndcg_at_5_diff1": 0.6619264824255433, - "nauc_ndcg_at_5_max": -0.5747249798765763, - "nauc_ndcg_at_5_std": -0.5747249798765763, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.03801725152590239, - "nauc_precision_at_100_max": -0.08736737679283053, - "nauc_precision_at_100_std": -0.2713668387335432, - "nauc_precision_at_10_diff1": 0.04588140595653325, - "nauc_precision_at_10_max": -0.3503264466505679, - "nauc_precision_at_10_std": -0.12592791342455947, - "nauc_precision_at_1_diff1": 0.6619264824255433, - "nauc_precision_at_1_max": -0.5747249798765763, - "nauc_precision_at_1_std": -0.5747249798765763, - "nauc_precision_at_20_diff1": 0.0599946337536892, - "nauc_precision_at_20_max": -0.2735175744566675, - "nauc_precision_at_20_std": -0.20461497182720684, - "nauc_precision_at_3_diff1": 0.6619264824255435, - "nauc_precision_at_3_max": -0.5747249798765764, - "nauc_precision_at_3_std": -0.5747249798765764, - "nauc_precision_at_5_diff1": 0.6619264824255435, - "nauc_precision_at_5_max": -0.5747249798765764, - "nauc_precision_at_5_std": -0.5747249798765764, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.03801725152590248, - "nauc_recall_at_100_max": -0.08736737679283049, - "nauc_recall_at_100_std": -0.27136683873354306, - "nauc_recall_at_10_diff1": 0.04588140595653343, - "nauc_recall_at_10_max": -0.35032644665056795, - "nauc_recall_at_10_std": -0.1259279134245595, - "nauc_recall_at_1_diff1": 0.6619264824255433, - "nauc_recall_at_1_max": -0.5747249798765763, - "nauc_recall_at_1_std": -0.5747249798765763, - "nauc_recall_at_20_diff1": 0.05999463375368926, - "nauc_recall_at_20_max": -0.2735175744566677, - "nauc_recall_at_20_std": -0.20461497182720695, - "nauc_recall_at_3_diff1": 0.6619264824255433, - "nauc_recall_at_3_max": -0.5747249798765763, - "nauc_recall_at_3_std": -0.5747249798765763, - "nauc_recall_at_5_diff1": 0.6619264824255433, - "nauc_recall_at_5_max": -0.5747249798765763, - "nauc_recall_at_5_std": -0.5747249798765763, - "ndcg_at_1": 0.01, - "ndcg_at_10": 0.01622, - "ndcg_at_100": 0.04451, - "ndcg_at_1000": 0.14068, - "ndcg_at_20": 0.02081, - "ndcg_at_3": 0.01, - "ndcg_at_5": 0.01, - "precision_at_1": 0.01, - "precision_at_10": 0.003, - "precision_at_100": 0.0019, - "precision_at_1000": 0.001, - "precision_at_20": 0.0025, - "precision_at_3": 0.00333, - "precision_at_5": 0.002, - "recall_at_1": 0.01, - "recall_at_10": 0.03, - "recall_at_100": 0.19, - "recall_at_1000": 1.0, - "recall_at_20": 0.05, - "recall_at_3": 0.01, - "recall_at_5": 0.01 - } - ] - }, - "task_name": "VidoreShiftProjectRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAAIRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAAIRetrieval.json deleted file mode 100644 index a33ab39de5..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAAIRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", - "evaluation_time": 29.120087385177612, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.08, - "cv_recall_at_10": 0.16, - "cv_recall_at_100": 0.48, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.23, - "cv_recall_at_3": 0.11, - "cv_recall_at_5": 0.12, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.10323, - "map_at_1": 0.08, - "map_at_10": 0.10186, - "map_at_100": 0.11176, - "map_at_1000": 0.11365, - "map_at_20": 0.10621, - "map_at_3": 0.095, - "map_at_5": 0.0975, - "mrr_at_1": 0.08, - "mrr_at_10": 0.10019444444444443, - "mrr_at_100": 0.11011084461491963, - "mrr_at_1000": 0.11200037839646133, - "mrr_at_20": 0.1045761630010856, - "mrr_at_3": 0.09333333333333332, - "mrr_at_5": 0.09583333333333333, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.04469672912035571, - "nauc_cv_recall_at_100_max": 0.06390917751667213, - "nauc_cv_recall_at_100_std": 0.16791044776119407, - "nauc_cv_recall_at_10_diff1": 0.07295667537256094, - "nauc_cv_recall_at_10_max": 0.32518436011676144, - "nauc_cv_recall_at_10_std": 0.1959402365954832, - "nauc_cv_recall_at_1_diff1": 0.23302924604239317, - "nauc_cv_recall_at_1_max": 0.4590823718808693, - "nauc_cv_recall_at_1_std": 0.150757982291387, - "nauc_cv_recall_at_20_diff1": -0.024984365705221116, - "nauc_cv_recall_at_20_max": 0.20837083021918498, - "nauc_cv_recall_at_20_std": 0.10311008068821408, - "nauc_cv_recall_at_3_diff1": 0.16787880313058365, - "nauc_cv_recall_at_3_max": 0.48519012507235065, - "nauc_cv_recall_at_3_std": 0.11757304275612139, - "nauc_cv_recall_at_5_diff1": 0.12994029001990343, - "nauc_cv_recall_at_5_max": 0.42870344043218656, - "nauc_cv_recall_at_5_std": 0.19900957255236476, - "nauc_map_at_1000_diff1": 0.1675620920313474, - "nauc_map_at_1000_max": 0.4200132379612108, - "nauc_map_at_1000_std": 0.14740733681367998, - "nauc_map_at_100_diff1": 0.16626251906483952, - "nauc_map_at_100_max": 0.4170654384966692, - "nauc_map_at_100_std": 0.14678438936186355, - "nauc_map_at_10_diff1": 0.17286908282947616, - "nauc_map_at_10_max": 0.43939582397261495, - "nauc_map_at_10_std": 0.1561951662475019, - "nauc_map_at_1_diff1": 0.23302924604239317, - "nauc_map_at_1_max": 0.4590823718808693, - "nauc_map_at_1_std": 0.150757982291387, - "nauc_map_at_20_diff1": 0.1595907524800354, - "nauc_map_at_20_max": 0.42461718707559404, - "nauc_map_at_20_std": 0.1458996645672452, - "nauc_map_at_3_diff1": 0.1958075136945926, - "nauc_map_at_3_max": 0.47399824594194356, - "nauc_map_at_3_std": 0.13179877215936048, - "nauc_map_at_5_diff1": 0.18401107555819254, - "nauc_map_at_5_max": 0.45735415651289435, - "nauc_map_at_5_std": 0.1558277355145401, - "nauc_mrr_at_1000_diff1": 0.15244811169195893, - "nauc_mrr_at_1000_max": 0.40811312724788, - "nauc_mrr_at_1000_std": 0.14373520730250974, - "nauc_mrr_at_100_diff1": 0.1512175820307389, - "nauc_mrr_at_100_max": 0.40519414293681005, - "nauc_mrr_at_100_std": 0.1431171513061672, - "nauc_mrr_at_10_diff1": 0.15761340785428135, - "nauc_mrr_at_10_max": 0.4271793359905257, - "nauc_mrr_at_10_std": 0.15239727016735158, - "nauc_mrr_at_1_diff1": 0.23302924604239317, - "nauc_mrr_at_1_max": 0.4590823718808693, - "nauc_mrr_at_1_std": 0.150757982291387, - "nauc_mrr_at_20_diff1": 0.14432706409099771, - "nauc_mrr_at_20_max": 0.41254188560602034, - "nauc_mrr_at_20_std": 0.14202270786213608, - "nauc_mrr_at_3_diff1": 0.18007488166369212, - "nauc_mrr_at_3_max": 0.4615940482656942, - "nauc_mrr_at_3_std": 0.12752192131605505, - "nauc_mrr_at_5_diff1": 0.16841716593017853, - "nauc_mrr_at_5_max": 0.44497302190406995, - "nauc_mrr_at_5_std": 0.15190871517562562, - "nauc_ndcg_at_1000_diff1": 0.1490204225947559, - "nauc_ndcg_at_1000_max": 0.37537786682774904, - "nauc_ndcg_at_1000_std": 0.1497468010938436, - "nauc_ndcg_at_100_diff1": 0.12563396873682928, - "nauc_ndcg_at_100_max": 0.31099215070250275, - "nauc_ndcg_at_100_std": 0.14960003826861942, - "nauc_ndcg_at_10_diff1": 0.14300628393164974, - "nauc_ndcg_at_10_max": 0.40712871839693465, - "nauc_ndcg_at_10_std": 0.16798748927277138, - "nauc_ndcg_at_1_diff1": 0.23302924604239317, - "nauc_ndcg_at_1_max": 0.4590823718808693, - "nauc_ndcg_at_1_std": 0.150757982291387, - "nauc_ndcg_at_20_diff1": 0.10129856953452061, - "nauc_ndcg_at_20_max": 0.3585785212661761, - "nauc_ndcg_at_20_std": 0.134777637721659, - "nauc_ndcg_at_3_diff1": 0.1877518282172568, - "nauc_ndcg_at_3_max": 0.4772264027867892, - "nauc_ndcg_at_3_std": 0.12769553934109137, - "nauc_ndcg_at_5_diff1": 0.16861518269930026, - "nauc_ndcg_at_5_max": 0.44980817963915065, - "nauc_ndcg_at_5_std": 0.16726310730787342, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.044696729120355265, - "nauc_precision_at_100_max": 0.06390917751667215, - "nauc_precision_at_100_std": 0.1679104477611941, - "nauc_precision_at_10_diff1": 0.07295667537256093, - "nauc_precision_at_10_max": 0.3251843601167612, - "nauc_precision_at_10_std": 0.195940236595483, - "nauc_precision_at_1_diff1": 0.23302924604239317, - "nauc_precision_at_1_max": 0.4590823718808693, - "nauc_precision_at_1_std": 0.150757982291387, - "nauc_precision_at_20_diff1": -0.024984365705221136, - "nauc_precision_at_20_max": 0.20837083021918515, - "nauc_precision_at_20_std": 0.10311008068821399, - "nauc_precision_at_3_diff1": 0.16787880313058348, - "nauc_precision_at_3_max": 0.48519012507235065, - "nauc_precision_at_3_std": 0.11757304275612139, - "nauc_precision_at_5_diff1": 0.12994029001990304, - "nauc_precision_at_5_max": 0.4287034404321862, - "nauc_precision_at_5_std": 0.19900957255236448, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.04469672912035571, - "nauc_recall_at_100_max": 0.06390917751667213, - "nauc_recall_at_100_std": 0.16791044776119407, - "nauc_recall_at_10_diff1": 0.07295667537256094, - "nauc_recall_at_10_max": 0.32518436011676144, - "nauc_recall_at_10_std": 0.1959402365954832, - "nauc_recall_at_1_diff1": 0.23302924604239317, - "nauc_recall_at_1_max": 0.4590823718808693, - "nauc_recall_at_1_std": 0.150757982291387, - "nauc_recall_at_20_diff1": -0.024984365705221116, - "nauc_recall_at_20_max": 0.20837083021918498, - "nauc_recall_at_20_std": 0.10311008068821408, - "nauc_recall_at_3_diff1": 0.16787880313058365, - "nauc_recall_at_3_max": 0.48519012507235065, - "nauc_recall_at_3_std": 0.11757304275612139, - "nauc_recall_at_5_diff1": 0.12994029001990343, - "nauc_recall_at_5_max": 0.42870344043218656, - "nauc_recall_at_5_std": 0.19900957255236476, - "ndcg_at_1": 0.08, - "ndcg_at_10": 0.11518, - "ndcg_at_100": 0.17682, - "ndcg_at_1000": 0.23957, - "ndcg_at_20": 0.13223, - "ndcg_at_3": 0.09893, - "ndcg_at_5": 0.10323, - "precision_at_1": 0.08, - "precision_at_10": 0.016, - "precision_at_100": 0.0048, - "precision_at_1000": 0.001, - "precision_at_20": 0.0115, - "precision_at_3": 0.03667, - "precision_at_5": 0.024, - "recall_at_1": 0.08, - "recall_at_10": 0.16, - "recall_at_100": 0.48, - "recall_at_1000": 1.0, - "recall_at_20": 0.23, - "recall_at_3": 0.11, - "recall_at_5": 0.12 - } - ] - }, - "task_name": "VidoreSyntheticDocQAAIRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAEnergyRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAEnergyRetrieval.json deleted file mode 100644 index 289c1e6c15..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAEnergyRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", - "evaluation_time": 27.4805850982666, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.09, - "cv_recall_at_10": 0.22, - "cv_recall_at_100": 0.61, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.3, - "cv_recall_at_3": 0.18, - "cv_recall_at_5": 0.22, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.15421, - "map_at_1": 0.08, - "map_at_10": 0.13233, - "map_at_100": 0.14599, - "map_at_1000": 0.14767, - "map_at_20": 0.13794, - "map_at_3": 0.12333, - "map_at_5": 0.13233, - "mrr_at_1": 0.09, - "mrr_at_10": 0.13733333333333334, - "mrr_at_100": 0.1508502521768428, - "mrr_at_1000": 0.15253465640562683, - "mrr_at_20": 0.14279348326252353, - "mrr_at_3": 0.12833333333333333, - "mrr_at_5": 0.13733333333333334, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.4314655628238636, - "nauc_cv_recall_at_100_max": -0.06355656942577056, - "nauc_cv_recall_at_100_std": 0.27635852733672706, - "nauc_cv_recall_at_10_diff1": -0.2979457376386859, - "nauc_cv_recall_at_10_max": 0.1128445784255835, - "nauc_cv_recall_at_10_std": 0.12398607701152999, - "nauc_cv_recall_at_1_diff1": -0.19631517753331543, - "nauc_cv_recall_at_1_max": 0.236860149658647, - "nauc_cv_recall_at_1_std": -0.01994454878812264, - "nauc_cv_recall_at_20_diff1": -0.3405350628293475, - "nauc_cv_recall_at_20_max": 0.03631941629509504, - "nauc_cv_recall_at_20_std": 0.10158086745034434, - "nauc_cv_recall_at_3_diff1": -0.34073606287057506, - "nauc_cv_recall_at_3_max": 0.12274146581061644, - "nauc_cv_recall_at_3_std": 0.26448093183173693, - "nauc_cv_recall_at_5_diff1": -0.2979457376386859, - "nauc_cv_recall_at_5_max": 0.1128445784255835, - "nauc_cv_recall_at_5_std": 0.12398607701152999, - "nauc_map_at_1000_diff1": -0.2604944977994006, - "nauc_map_at_1000_max": 0.17830635109139642, - "nauc_map_at_1000_std": 0.12090084293431434, - "nauc_map_at_100_diff1": -0.2621590656772411, - "nauc_map_at_100_max": 0.17648762923000524, - "nauc_map_at_100_std": 0.121268509592713, - "nauc_map_at_10_diff1": -0.2616638520052834, - "nauc_map_at_10_max": 0.18996089626586538, - "nauc_map_at_10_std": 0.11845610393114783, - "nauc_map_at_1_diff1": -0.15840488328414284, - "nauc_map_at_1_max": 0.30346122887040494, - "nauc_map_at_1_std": 0.049771934531795, - "nauc_map_at_20_diff1": -0.2646839857229457, - "nauc_map_at_20_max": 0.18237211042993076, - "nauc_map_at_20_std": 0.1192844198516302, - "nauc_map_at_3_diff1": -0.2756486484329484, - "nauc_map_at_3_max": 0.1953966113057568, - "nauc_map_at_3_std": 0.16172914388782036, - "nauc_map_at_5_diff1": -0.2616638520052834, - "nauc_map_at_5_max": 0.18996089626586538, - "nauc_map_at_5_std": 0.11845610393114783, - "nauc_mrr_at_1000_diff1": -0.27452629073850626, - "nauc_mrr_at_1000_max": 0.1574894915596897, - "nauc_mrr_at_1000_std": 0.09317346271772621, - "nauc_mrr_at_100_diff1": -0.27604407977587014, - "nauc_mrr_at_100_max": 0.15586164065749025, - "nauc_mrr_at_100_std": 0.09368786780755935, - "nauc_mrr_at_10_diff1": -0.2754707494354536, - "nauc_mrr_at_10_max": 0.16860312970455407, - "nauc_mrr_at_10_std": 0.09050459282085065, - "nauc_mrr_at_1_diff1": -0.19631517753331543, - "nauc_mrr_at_1_max": 0.236860149658647, - "nauc_mrr_at_1_std": -0.01994454878812264, - "nauc_mrr_at_20_diff1": -0.27839677071526, - "nauc_mrr_at_20_max": 0.1615180503347429, - "nauc_mrr_at_20_std": 0.09183806368518946, - "nauc_mrr_at_3_diff1": -0.2852138636025044, - "nauc_mrr_at_3_max": 0.17441047601220877, - "nauc_mrr_at_3_std": 0.1301479492243458, - "nauc_mrr_at_5_diff1": -0.2754707494354536, - "nauc_mrr_at_5_max": 0.16860312970455407, - "nauc_mrr_at_5_std": 0.09050459282085065, - "nauc_ndcg_at_1000_diff1": -0.2708524099375955, - "nauc_ndcg_at_1000_max": 0.1462212343162209, - "nauc_ndcg_at_1000_std": 0.13236351721468018, - "nauc_ndcg_at_100_diff1": -0.2999893850603643, - "nauc_ndcg_at_100_max": 0.1105185531422724, - "nauc_ndcg_at_100_std": 0.15311561566657694, - "nauc_ndcg_at_10_diff1": -0.27466649873010107, - "nauc_ndcg_at_10_max": 0.16425858048656408, - "nauc_ndcg_at_10_std": 0.12216702114512916, - "nauc_ndcg_at_1_diff1": -0.15840488328414284, - "nauc_ndcg_at_1_max": 0.30346122887040494, - "nauc_ndcg_at_1_std": 0.049771934531795, - "nauc_ndcg_at_20_diff1": -0.2886967979788647, - "nauc_ndcg_at_20_max": 0.13646757078462018, - "nauc_ndcg_at_20_std": 0.11774155369915566, - "nauc_ndcg_at_3_diff1": -0.29680730637047387, - "nauc_ndcg_at_3_max": 0.17213099438008325, - "nauc_ndcg_at_3_std": 0.19399951682309413, - "nauc_ndcg_at_5_diff1": -0.27466649873010107, - "nauc_ndcg_at_5_max": 0.16425858048656408, - "nauc_ndcg_at_5_std": 0.12216702114512916, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.43146556282386345, - "nauc_precision_at_100_max": -0.06355656942577005, - "nauc_precision_at_100_std": 0.27635852733672717, - "nauc_precision_at_10_diff1": -0.29794573763868604, - "nauc_precision_at_10_max": 0.11284457842558356, - "nauc_precision_at_10_std": 0.12398607701152996, - "nauc_precision_at_1_diff1": -0.15840488328414284, - "nauc_precision_at_1_max": 0.30346122887040494, - "nauc_precision_at_1_std": 0.049771934531795, - "nauc_precision_at_20_diff1": -0.34053506282934753, - "nauc_precision_at_20_max": 0.036319416295095025, - "nauc_precision_at_20_std": 0.1015808674503442, - "nauc_precision_at_3_diff1": -0.3407360628705749, - "nauc_precision_at_3_max": 0.1227414658106165, - "nauc_precision_at_3_std": 0.26448093183173715, - "nauc_precision_at_5_diff1": -0.29794573763868604, - "nauc_precision_at_5_max": 0.11284457842558356, - "nauc_precision_at_5_std": 0.12398607701152996, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.4314655628238636, - "nauc_recall_at_100_max": -0.06355656942577056, - "nauc_recall_at_100_std": 0.27635852733672706, - "nauc_recall_at_10_diff1": -0.2979457376386859, - "nauc_recall_at_10_max": 0.1128445784255835, - "nauc_recall_at_10_std": 0.12398607701152999, - "nauc_recall_at_1_diff1": -0.15840488328414284, - "nauc_recall_at_1_max": 0.30346122887040494, - "nauc_recall_at_1_std": 0.049771934531795, - "nauc_recall_at_20_diff1": -0.3405350628293475, - "nauc_recall_at_20_max": 0.03631941629509504, - "nauc_recall_at_20_std": 0.10158086745034434, - "nauc_recall_at_3_diff1": -0.34073606287057506, - "nauc_recall_at_3_max": 0.12274146581061644, - "nauc_recall_at_3_std": 0.26448093183173693, - "nauc_recall_at_5_diff1": -0.2979457376386859, - "nauc_recall_at_5_max": 0.1128445784255835, - "nauc_recall_at_5_std": 0.12398607701152999, - "ndcg_at_1": 0.08, - "ndcg_at_10": 0.15421, - "ndcg_at_100": 0.2321, - "ndcg_at_1000": 0.28117, - "ndcg_at_20": 0.17448, - "ndcg_at_3": 0.13786, - "ndcg_at_5": 0.15421, - "precision_at_1": 0.08, - "precision_at_10": 0.022, - "precision_at_100": 0.0061, - "precision_at_1000": 0.001, - "precision_at_20": 0.015, - "precision_at_3": 0.06, - "precision_at_5": 0.044, - "recall_at_1": 0.08, - "recall_at_10": 0.22, - "recall_at_100": 0.61, - "recall_at_1000": 1.0, - "recall_at_20": 0.3, - "recall_at_3": 0.18, - "recall_at_5": 0.22 - } - ] - }, - "task_name": "VidoreSyntheticDocQAEnergyRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAGovernmentReportsRetrieval.json deleted file mode 100644 index 1f71d1c048..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAGovernmentReportsRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", - "evaluation_time": 28.769615650177002, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.06, - "cv_recall_at_10": 0.16, - "cv_recall_at_100": 0.42, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.23, - "cv_recall_at_3": 0.09, - "cv_recall_at_5": 0.09, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.08123, - "map_at_1": 0.06, - "map_at_10": 0.08121, - "map_at_100": 0.09109, - "map_at_1000": 0.0932, - "map_at_20": 0.08728, - "map_at_3": 0.07, - "map_at_5": 0.075, - "mrr_at_1": 0.06, - "mrr_at_10": 0.08202777777777777, - "mrr_at_100": 0.09121283057539557, - "mrr_at_1000": 0.09323295120835003, - "mrr_at_20": 0.08731412520228309, - "mrr_at_3": 0.07333333333333333, - "mrr_at_5": 0.07333333333333333, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.16483915286310502, - "nauc_cv_recall_at_100_max": 0.43260232781190855, - "nauc_cv_recall_at_100_std": 0.2827721181014594, - "nauc_cv_recall_at_10_diff1": 0.058591949608234696, - "nauc_cv_recall_at_10_max": 0.34500307266861274, - "nauc_cv_recall_at_10_std": 0.20777001075434018, - "nauc_cv_recall_at_1_diff1": 0.1739558179053752, - "nauc_cv_recall_at_1_max": 0.33887845452106247, - "nauc_cv_recall_at_1_std": 0.3941507915213307, - "nauc_cv_recall_at_20_diff1": 0.19142477997590027, - "nauc_cv_recall_at_20_max": 0.3804243376397552, - "nauc_cv_recall_at_20_std": 0.09292109638352061, - "nauc_cv_recall_at_3_diff1": 0.37417643025370423, - "nauc_cv_recall_at_3_max": 0.5592523030140416, - "nauc_cv_recall_at_3_std": 0.42455952061532964, - "nauc_cv_recall_at_5_diff1": 0.37417643025370423, - "nauc_cv_recall_at_5_max": 0.5592523030140416, - "nauc_cv_recall_at_5_std": 0.42455952061532964, - "nauc_map_at_1000_diff1": 0.22211997448946572, - "nauc_map_at_1000_max": 0.4226703021544981, - "nauc_map_at_1000_std": 0.35509700498369534, - "nauc_map_at_100_diff1": 0.22279977014716346, - "nauc_map_at_100_max": 0.4252031145284334, - "nauc_map_at_100_std": 0.3563523572828241, - "nauc_map_at_10_diff1": 0.22192935627611637, - "nauc_map_at_10_max": 0.42415004690257796, - "nauc_map_at_10_std": 0.3662952971659471, - "nauc_map_at_1_diff1": 0.1739558179053752, - "nauc_map_at_1_max": 0.33887845452106247, - "nauc_map_at_1_std": 0.3941507915213307, - "nauc_map_at_20_diff1": 0.2235370957493086, - "nauc_map_at_20_max": 0.42363627165734064, - "nauc_map_at_20_std": 0.3493697046949435, - "nauc_map_at_3_diff1": 0.2436659128368275, - "nauc_map_at_3_max": 0.433324389589482, - "nauc_map_at_3_std": 0.45655257004867944, - "nauc_map_at_5_diff1": 0.24691888024326974, - "nauc_map_at_5_max": 0.44856452911188616, - "nauc_map_at_5_std": 0.408639656560236, - "nauc_mrr_at_1000_diff1": 0.24970464634892656, - "nauc_mrr_at_1000_max": 0.4293471710105555, - "nauc_mrr_at_1000_std": 0.3328740535666148, - "nauc_mrr_at_100_diff1": 0.24973309243013533, - "nauc_mrr_at_100_max": 0.43118294840439664, - "nauc_mrr_at_100_std": 0.3336076391475648, - "nauc_mrr_at_10_diff1": 0.23498849214811451, - "nauc_mrr_at_10_max": 0.424442907589755, - "nauc_mrr_at_10_std": 0.3482117751660398, - "nauc_mrr_at_1_diff1": 0.1739558179053752, - "nauc_mrr_at_1_max": 0.33887845452106247, - "nauc_mrr_at_1_std": 0.3941507915213307, - "nauc_mrr_at_20_diff1": 0.25119430080323674, - "nauc_mrr_at_20_max": 0.43033456107778606, - "nauc_mrr_at_20_std": 0.3270902377687649, - "nauc_mrr_at_3_diff1": 0.29341171305217445, - "nauc_mrr_at_3_max": 0.4590823718808692, - "nauc_mrr_at_3_std": 0.3990413932726785, - "nauc_mrr_at_5_diff1": 0.29341171305217445, - "nauc_mrr_at_5_max": 0.4590823718808692, - "nauc_mrr_at_5_std": 0.3990413932726785, - "nauc_ndcg_at_1000_diff1": 0.2138824549051445, - "nauc_ndcg_at_1000_max": 0.4161443897516234, - "nauc_ndcg_at_1000_std": 0.3158045285204317, - "nauc_ndcg_at_100_diff1": 0.21009464889600732, - "nauc_ndcg_at_100_max": 0.4347151408518815, - "nauc_ndcg_at_100_std": 0.3206930265791543, - "nauc_ndcg_at_10_diff1": 0.2036918358650856, - "nauc_ndcg_at_10_max": 0.41509671780130775, - "nauc_ndcg_at_10_std": 0.30358830108810564, - "nauc_ndcg_at_1_diff1": 0.1739558179053752, - "nauc_ndcg_at_1_max": 0.33887845452106247, - "nauc_ndcg_at_1_std": 0.3941507915213307, - "nauc_ndcg_at_20_diff1": 0.2130246179818501, - "nauc_ndcg_at_20_max": 0.4125222189697398, - "nauc_ndcg_at_20_std": 0.25841208063962756, - "nauc_ndcg_at_3_diff1": 0.25874820773692664, - "nauc_ndcg_at_3_max": 0.45375846655090646, - "nauc_ndcg_at_3_std": 0.47005365661247767, - "nauc_ndcg_at_5_diff1": 0.26232291688461556, - "nauc_ndcg_at_5_max": 0.4758317739319828, - "nauc_ndcg_at_5_std": 0.3924146371401771, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.1842618546314115, - "nauc_precision_at_100_max": 0.4588010385315887, - "nauc_precision_at_100_std": 0.30947983001149604, - "nauc_precision_at_10_diff1": 0.155266606188892, - "nauc_precision_at_10_max": 0.3810704566071967, - "nauc_precision_at_10_std": 0.16837012397943735, - "nauc_precision_at_1_diff1": 0.1739558179053752, - "nauc_precision_at_1_max": 0.33887845452106247, - "nauc_precision_at_1_std": 0.3941507915213307, - "nauc_precision_at_20_diff1": 0.19142477997590027, - "nauc_precision_at_20_max": 0.3804243376397553, - "nauc_precision_at_20_std": 0.0929210963835207, - "nauc_precision_at_3_diff1": 0.2959484840354171, - "nauc_precision_at_3_max": 0.504158840890797, - "nauc_precision_at_3_std": 0.5033539039441909, - "nauc_precision_at_5_diff1": 0.2952508720150253, - "nauc_precision_at_5_max": 0.5357123691977462, - "nauc_precision_at_5_std": 0.35025489669975846, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.18426185463141187, - "nauc_recall_at_100_max": 0.4588010385315888, - "nauc_recall_at_100_std": 0.30947983001149604, - "nauc_recall_at_10_diff1": 0.15526660618889215, - "nauc_recall_at_10_max": 0.38107045660719685, - "nauc_recall_at_10_std": 0.16837012397943757, - "nauc_recall_at_1_diff1": 0.1739558179053752, - "nauc_recall_at_1_max": 0.33887845452106247, - "nauc_recall_at_1_std": 0.3941507915213307, - "nauc_recall_at_20_diff1": 0.19142477997590027, - "nauc_recall_at_20_max": 0.3804243376397552, - "nauc_recall_at_20_std": 0.09292109638352061, - "nauc_recall_at_3_diff1": 0.29594848403541707, - "nauc_recall_at_3_max": 0.5041588408907967, - "nauc_recall_at_3_std": 0.503353903944191, - "nauc_recall_at_5_diff1": 0.2952508720150256, - "nauc_recall_at_5_max": 0.5357123691977462, - "nauc_recall_at_5_std": 0.3502548966997586, - "ndcg_at_1": 0.06, - "ndcg_at_10": 0.09692, - "ndcg_at_100": 0.14955, - "ndcg_at_1000": 0.22068, - "ndcg_at_20": 0.11779, - "ndcg_at_3": 0.07262, - "ndcg_at_5": 0.08123, - "precision_at_1": 0.06, - "precision_at_10": 0.015, - "precision_at_100": 0.0041, - "precision_at_1000": 0.001, - "precision_at_20": 0.0115, - "precision_at_3": 0.02667, - "precision_at_5": 0.02, - "recall_at_1": 0.06, - "recall_at_10": 0.15, - "recall_at_100": 0.41, - "recall_at_1000": 1.0, - "recall_at_20": 0.23, - "recall_at_3": 0.08, - "recall_at_5": 0.1 - } - ] - }, - "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json deleted file mode 100644 index f817e7bf63..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", - "evaluation_time": 27.634584188461304, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.07, - "cv_recall_at_10": 0.24, - "cv_recall_at_100": 0.49, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.26, - "cv_recall_at_3": 0.13, - "cv_recall_at_5": 0.16, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.1179, - "map_at_1": 0.07, - "map_at_10": 0.1134, - "map_at_100": 0.11961, - "map_at_1000": 0.12196, - "map_at_20": 0.11461, - "map_at_3": 0.095, - "map_at_5": 0.104, - "mrr_at_1": 0.07, - "mrr_at_10": 0.11423412698412698, - "mrr_at_100": 0.1204383669219322, - "mrr_at_1000": 0.1227870262150322, - "mrr_at_20": 0.11548412698412697, - "mrr_at_3": 0.09833333333333333, - "mrr_at_5": 0.10483333333333335, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.0033593137970868337, - "nauc_cv_recall_at_100_max": 0.13656340870765996, - "nauc_cv_recall_at_100_std": 0.0730551166465286, - "nauc_cv_recall_at_10_diff1": 0.0789300263599329, - "nauc_cv_recall_at_10_max": 0.32840582314881384, - "nauc_cv_recall_at_10_std": -0.11701713395638631, - "nauc_cv_recall_at_1_diff1": 0.13193299858177768, - "nauc_cv_recall_at_1_max": 0.7504695465521868, - "nauc_cv_recall_at_1_std": -0.1696883744106713, - "nauc_cv_recall_at_20_diff1": 0.05911942412766328, - "nauc_cv_recall_at_20_max": 0.3305917724263536, - "nauc_cv_recall_at_20_std": -0.08836113439912116, - "nauc_cv_recall_at_3_diff1": 0.15529090461373152, - "nauc_cv_recall_at_3_max": 0.7226388484699247, - "nauc_cv_recall_at_3_std": -0.008103691432920732, - "nauc_cv_recall_at_5_diff1": 0.12473114149638939, - "nauc_cv_recall_at_5_max": 0.5643916116146874, - "nauc_cv_recall_at_5_std": -0.03466354278691037, - "nauc_map_at_1000_diff1": 0.10475800103203144, - "nauc_map_at_1000_max": 0.5801798373932328, - "nauc_map_at_1000_std": -0.11603267202556924, - "nauc_map_at_100_diff1": 0.10355861111113708, - "nauc_map_at_100_max": 0.5772673044620018, - "nauc_map_at_100_std": -0.1162521804291923, - "nauc_map_at_10_diff1": 0.11088361581323901, - "nauc_map_at_10_max": 0.5871221018084627, - "nauc_map_at_10_std": -0.1280790298966015, - "nauc_map_at_1_diff1": 0.13193299858177768, - "nauc_map_at_1_max": 0.6216796350952509, - "nauc_map_at_1_std": -0.2189045191460004, - "nauc_map_at_20_diff1": 0.10849886911360809, - "nauc_map_at_20_max": 0.5864430025825756, - "nauc_map_at_20_std": -0.12474158742824958, - "nauc_map_at_3_diff1": 0.11585374782188503, - "nauc_map_at_3_max": 0.6925014276719429, - "nauc_map_at_3_std": -0.14349933375309332, - "nauc_map_at_5_diff1": 0.1203639033404092, - "nauc_map_at_5_max": 0.6400549402202009, - "nauc_map_at_5_std": -0.11654861027090663, - "nauc_mrr_at_1000_diff1": 0.10924678698408471, - "nauc_mrr_at_1000_max": 0.6234840055556127, - "nauc_mrr_at_1000_std": -0.09032182856834724, - "nauc_mrr_at_100_diff1": 0.10801566961185796, - "nauc_mrr_at_100_max": 0.6202581345282157, - "nauc_mrr_at_100_std": -0.09072222954701539, - "nauc_mrr_at_10_diff1": 0.11531417186020634, - "nauc_mrr_at_10_max": 0.630524081718047, - "nauc_mrr_at_10_std": -0.10245821572444079, - "nauc_mrr_at_1_diff1": 0.13193299858177768, - "nauc_mrr_at_1_max": 0.7504695465521868, - "nauc_mrr_at_1_std": -0.1696883744106713, - "nauc_mrr_at_20_diff1": 0.112947332125841, - "nauc_mrr_at_20_max": 0.6295858860926723, - "nauc_mrr_at_20_std": -0.09929781553533608, - "nauc_mrr_at_3_diff1": 0.13664548907836976, - "nauc_mrr_at_3_max": 0.7357647047609058, - "nauc_mrr_at_3_std": -0.08623869826941129, - "nauc_mrr_at_5_diff1": 0.12530857755601577, - "nauc_mrr_at_5_max": 0.6851435714471256, - "nauc_mrr_at_5_std": -0.0898469382407397, - "nauc_ndcg_at_1000_diff1": 0.09128770677873728, - "nauc_ndcg_at_1000_max": 0.5149398611965009, - "nauc_ndcg_at_1000_std": -0.08505448929111248, - "nauc_ndcg_at_100_diff1": 0.07184603454814685, - "nauc_ndcg_at_100_max": 0.443191212813763, - "nauc_ndcg_at_100_std": -0.06142226399285712, - "nauc_ndcg_at_10_diff1": 0.10049597269892058, - "nauc_ndcg_at_10_max": 0.5040530206853383, - "nauc_ndcg_at_10_std": -0.12066788476859182, - "nauc_ndcg_at_1_diff1": 0.13193299858177768, - "nauc_ndcg_at_1_max": 0.6216796350952509, - "nauc_ndcg_at_1_std": -0.2189045191460004, - "nauc_ndcg_at_20_diff1": 0.09281328000703318, - "nauc_ndcg_at_20_max": 0.5026776186328519, - "nauc_ndcg_at_20_std": -0.1098844443956209, - "nauc_ndcg_at_3_diff1": 0.11282399978682046, - "nauc_ndcg_at_3_max": 0.7058460909341273, - "nauc_ndcg_at_3_std": -0.12929104036372097, - "nauc_ndcg_at_5_diff1": 0.12132540415555444, - "nauc_ndcg_at_5_max": 0.6182882597405424, - "nauc_ndcg_at_5_std": -0.08996362964673281, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.003359313797086568, - "nauc_precision_at_100_max": 0.1365634087076601, - "nauc_precision_at_100_std": 0.07305511664652851, - "nauc_precision_at_10_diff1": 0.07893002635993276, - "nauc_precision_at_10_max": 0.3284058231488137, - "nauc_precision_at_10_std": -0.1170171339563864, - "nauc_precision_at_1_diff1": 0.13193299858177768, - "nauc_precision_at_1_max": 0.6216796350952509, - "nauc_precision_at_1_std": -0.2189045191460004, - "nauc_precision_at_20_diff1": 0.05911942412766315, - "nauc_precision_at_20_max": 0.3305917724263538, - "nauc_precision_at_20_std": -0.08836113439912127, - "nauc_precision_at_3_diff1": 0.10591413136195617, - "nauc_precision_at_3_max": 0.7362809212396925, - "nauc_precision_at_3_std": -0.09688655103781642, - "nauc_precision_at_5_diff1": 0.12473114149638945, - "nauc_precision_at_5_max": 0.5643916116146872, - "nauc_precision_at_5_std": -0.03466354278691048, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.0033593137970868337, - "nauc_recall_at_100_max": 0.13656340870765996, - "nauc_recall_at_100_std": 0.0730551166465286, - "nauc_recall_at_10_diff1": 0.0789300263599329, - "nauc_recall_at_10_max": 0.32840582314881384, - "nauc_recall_at_10_std": -0.11701713395638631, - "nauc_recall_at_1_diff1": 0.13193299858177768, - "nauc_recall_at_1_max": 0.6216796350952509, - "nauc_recall_at_1_std": -0.2189045191460004, - "nauc_recall_at_20_diff1": 0.05911942412766328, - "nauc_recall_at_20_max": 0.3305917724263536, - "nauc_recall_at_20_std": -0.08836113439912116, - "nauc_recall_at_3_diff1": 0.10591413136195621, - "nauc_recall_at_3_max": 0.736280921239693, - "nauc_recall_at_3_std": -0.09688655103781627, - "nauc_recall_at_5_diff1": 0.12473114149638939, - "nauc_recall_at_5_max": 0.5643916116146874, - "nauc_recall_at_5_std": -0.03466354278691037, - "ndcg_at_1": 0.07, - "ndcg_at_10": 0.1425, - "ndcg_at_100": 0.1882, - "ndcg_at_1000": 0.25263, - "ndcg_at_20": 0.14734, - "ndcg_at_3": 0.10155, - "ndcg_at_5": 0.1179, - "precision_at_1": 0.07, - "precision_at_10": 0.024, - "precision_at_100": 0.0049, - "precision_at_1000": 0.001, - "precision_at_20": 0.013, - "precision_at_3": 0.04, - "precision_at_5": 0.032, - "recall_at_1": 0.07, - "recall_at_10": 0.24, - "recall_at_100": 0.49, - "recall_at_1000": 1.0, - "recall_at_20": 0.26, - "recall_at_3": 0.12, - "recall_at_5": 0.16 - } - ] - }, - "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTabfquadRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTabfquadRetrieval.json deleted file mode 100644 index fc0eead64f..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTabfquadRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", - "evaluation_time": 4.014543533325195, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.01429, - "cv_recall_at_10": 0.12857, - "cv_recall_at_100": 0.51429, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.21071, - "cv_recall_at_3": 0.05714, - "cv_recall_at_5": 0.08571, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.05553, - "map_at_1": 0.02143, - "map_at_10": 0.04813, - "map_at_100": 0.06068, - "map_at_1000": 0.06382, - "map_at_20": 0.05355, - "map_at_3": 0.03631, - "map_at_5": 0.04345, - "mrr_at_1": 0.014285714285714285, - "mrr_at_10": 0.0428798185941043, - "mrr_at_100": 0.05584756483687463, - "mrr_at_1000": 0.058994237118739384, - "mrr_at_20": 0.048727962983132145, - "mrr_at_3": 0.03095238095238095, - "mrr_at_5": 0.03755952380952381, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.14435075722453997, - "nauc_cv_recall_at_100_max": 0.04018409055787388, - "nauc_cv_recall_at_100_std": 0.35451437181270307, - "nauc_cv_recall_at_10_diff1": -0.14477967959256025, - "nauc_cv_recall_at_10_max": 0.10806498404409637, - "nauc_cv_recall_at_10_std": 0.2212874319053606, - "nauc_cv_recall_at_1_diff1": -0.3902602629460694, - "nauc_cv_recall_at_1_max": 0.07713979071639386, - "nauc_cv_recall_at_1_std": 0.3971022269922189, - "nauc_cv_recall_at_20_diff1": -0.13442302587360852, - "nauc_cv_recall_at_20_max": 0.07322101291703624, - "nauc_cv_recall_at_20_std": 0.2198268315137276, - "nauc_cv_recall_at_3_diff1": -0.17822645559431183, - "nauc_cv_recall_at_3_max": 0.08101354977193453, - "nauc_cv_recall_at_3_std": 0.2634156157767643, - "nauc_cv_recall_at_5_diff1": -0.16759458009122633, - "nauc_cv_recall_at_5_max": 0.10308782756461854, - "nauc_cv_recall_at_5_std": 0.24713800196762364, - "nauc_map_at_1000_diff1": -0.19046764108103575, - "nauc_map_at_1000_max": 0.08958064836736834, - "nauc_map_at_1000_std": 0.25301683753256515, - "nauc_map_at_100_diff1": -0.18963136394230734, - "nauc_map_at_100_max": 0.08780243681091121, - "nauc_map_at_100_std": 0.25502426729338673, - "nauc_map_at_10_diff1": -0.1859454096855409, - "nauc_map_at_10_max": 0.0922310989379099, - "nauc_map_at_10_std": 0.25011689318446306, - "nauc_map_at_1_diff1": -0.29433861014220575, - "nauc_map_at_1_max": 0.1435470888113763, - "nauc_map_at_1_std": 0.28816742688489394, - "nauc_map_at_20_diff1": -0.18776838654510583, - "nauc_map_at_20_max": 0.08945737330394286, - "nauc_map_at_20_std": 0.24766535799913875, - "nauc_map_at_3_diff1": -0.21081430588483685, - "nauc_map_at_3_max": 0.06450931835476176, - "nauc_map_at_3_std": 0.27496734067306816, - "nauc_map_at_5_diff1": -0.2022784493753467, - "nauc_map_at_5_max": 0.09233031083797995, - "nauc_map_at_5_std": 0.2453719801081337, - "nauc_mrr_at_1000_diff1": -0.20178163215368516, - "nauc_mrr_at_1000_max": 0.06804656436386369, - "nauc_mrr_at_1000_std": 0.26218897036305217, - "nauc_mrr_at_100_diff1": -0.2005749976351999, - "nauc_mrr_at_100_max": 0.06662175737366387, - "nauc_mrr_at_100_std": 0.2641890624642852, - "nauc_mrr_at_10_diff1": -0.20573494584516494, - "nauc_mrr_at_10_max": 0.07332293876449567, - "nauc_mrr_at_10_std": 0.2588373545696161, - "nauc_mrr_at_1_diff1": -0.3902602629460694, - "nauc_mrr_at_1_max": 0.07713979071639386, - "nauc_mrr_at_1_std": 0.3971022269922189, - "nauc_mrr_at_20_diff1": -0.19903140504273983, - "nauc_mrr_at_20_max": 0.06777812049985836, - "nauc_mrr_at_20_std": 0.256438663304796, - "nauc_mrr_at_3_diff1": -0.23416957338341837, - "nauc_mrr_at_3_max": 0.0601277579410126, - "nauc_mrr_at_3_std": 0.2875121256527214, - "nauc_mrr_at_5_diff1": -0.21939272971424953, - "nauc_mrr_at_5_max": 0.07277131754103458, - "nauc_mrr_at_5_std": 0.27610187703812117, - "nauc_ndcg_at_1000_diff1": -0.17606361925913822, - "nauc_ndcg_at_1000_max": 0.08433762641379804, - "nauc_ndcg_at_1000_std": 0.2549701974882767, - "nauc_ndcg_at_100_diff1": -0.1661933244126871, - "nauc_ndcg_at_100_max": 0.06822003904242817, - "nauc_ndcg_at_100_std": 0.2860804140807971, - "nauc_ndcg_at_10_diff1": -0.1543556954363095, - "nauc_ndcg_at_10_max": 0.09069581700525794, - "nauc_ndcg_at_10_std": 0.24111594833834304, - "nauc_ndcg_at_1_diff1": -0.29433861014220575, - "nauc_ndcg_at_1_max": 0.1435470888113763, - "nauc_ndcg_at_1_std": 0.28816742688489394, - "nauc_ndcg_at_20_diff1": -0.1613278895889277, - "nauc_ndcg_at_20_max": 0.08190405817102135, - "nauc_ndcg_at_20_std": 0.23431193680408277, - "nauc_ndcg_at_3_diff1": -0.19150484931331083, - "nauc_ndcg_at_3_max": 0.04565509051495962, - "nauc_ndcg_at_3_std": 0.272087289057183, - "nauc_ndcg_at_5_diff1": -0.18240460530220248, - "nauc_ndcg_at_5_max": 0.08876459284421898, - "nauc_ndcg_at_5_std": 0.22815212336675553, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.1443507572245399, - "nauc_precision_at_100_max": 0.04018409055787383, - "nauc_precision_at_100_std": 0.3545143718127031, - "nauc_precision_at_10_diff1": -0.11212161916576463, - "nauc_precision_at_10_max": 0.09143561500960012, - "nauc_precision_at_10_std": 0.2314019548194914, - "nauc_precision_at_1_diff1": -0.29433861014220575, - "nauc_precision_at_1_max": 0.1435470888113763, - "nauc_precision_at_1_std": 0.28816742688489394, - "nauc_precision_at_20_diff1": -0.1344230258736084, - "nauc_precision_at_20_max": 0.0732210129170362, - "nauc_precision_at_20_std": 0.21982683151372764, - "nauc_precision_at_3_diff1": -0.15702583689767838, - "nauc_precision_at_3_max": 0.011900440347858922, - "nauc_precision_at_3_std": 0.26697075395760667, - "nauc_precision_at_5_diff1": -0.15362944005283702, - "nauc_precision_at_5_max": 0.08550907102020577, - "nauc_precision_at_5_std": 0.20099688344925784, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.14435075722453997, - "nauc_recall_at_100_max": 0.04018409055787388, - "nauc_recall_at_100_std": 0.35451437181270307, - "nauc_recall_at_10_diff1": -0.1121216191657646, - "nauc_recall_at_10_max": 0.09143561500960022, - "nauc_recall_at_10_std": 0.2314019548194913, - "nauc_recall_at_1_diff1": -0.29433861014220575, - "nauc_recall_at_1_max": 0.1435470888113763, - "nauc_recall_at_1_std": 0.28816742688489394, - "nauc_recall_at_20_diff1": -0.13442302587360852, - "nauc_recall_at_20_max": 0.07322101291703624, - "nauc_recall_at_20_std": 0.2198268315137276, - "nauc_recall_at_3_diff1": -0.1570258368976784, - "nauc_recall_at_3_max": 0.011900440347858999, - "nauc_recall_at_3_std": 0.2669707539576066, - "nauc_recall_at_5_diff1": -0.1536294400528369, - "nauc_recall_at_5_max": 0.08550907102020605, - "nauc_recall_at_5_std": 0.20099688344925803, - "ndcg_at_1": 0.02143, - "ndcg_at_10": 0.06767, - "ndcg_at_100": 0.14238, - "ndcg_at_1000": 0.20878, - "ndcg_at_20": 0.0875, - "ndcg_at_3": 0.04247, - "ndcg_at_5": 0.05553, - "precision_at_1": 0.02143, - "precision_at_10": 0.01321, - "precision_at_100": 0.00514, - "precision_at_1000": 0.001, - "precision_at_20": 0.01054, - "precision_at_3": 0.02024, - "precision_at_5": 0.01857, - "recall_at_1": 0.02143, - "recall_at_10": 0.13214, - "recall_at_100": 0.51429, - "recall_at_1000": 1.0, - "recall_at_20": 0.21071, - "recall_at_3": 0.06071, - "recall_at_5": 0.09286 - } - ] - }, - "task_name": "VidoreTabfquadRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTatdqaRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTatdqaRetrieval.json deleted file mode 100644 index 6e2e9d79d4..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VidoreTatdqaRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", - "evaluation_time": 57.08458185195923, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.0018, - "cv_recall_at_10": 0.01443, - "cv_recall_at_100": 0.089, - "cv_recall_at_1000": 0.64702, - "cv_recall_at_20": 0.02044, - "cv_recall_at_3": 0.00481, - "cv_recall_at_5": 0.00782, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.00494, - "map_at_1": 0.0018, - "map_at_10": 0.00453, - "map_at_100": 0.0064, - "map_at_1000": 0.00793, - "map_at_20": 0.00509, - "map_at_3": 0.00301, - "map_at_5": 0.00382, - "mrr_at_1": 0.0018039687312086591, - "mrr_at_10": 0.004618494020177725, - "mrr_at_100": 0.006420684518018152, - "mrr_at_1000": 0.007956473201255023, - "mrr_at_20": 0.005077383723695083, - "mrr_at_3": 0.003106835037081579, - "mrr_at_5": 0.00373822409300461, - "nauc_cv_recall_at_1000_diff1": 0.08556738813980604, - "nauc_cv_recall_at_1000_max": 0.04177557742932173, - "nauc_cv_recall_at_1000_std": 0.09542880648474937, - "nauc_cv_recall_at_100_diff1": -0.03785066090099726, - "nauc_cv_recall_at_100_max": 0.16260798258815512, - "nauc_cv_recall_at_100_std": 0.16515780334855118, - "nauc_cv_recall_at_10_diff1": -0.04756502325362798, - "nauc_cv_recall_at_10_max": 0.37839103716358846, - "nauc_cv_recall_at_10_std": 0.5507149715917287, - "nauc_cv_recall_at_1_diff1": 0.22229315572956015, - "nauc_cv_recall_at_1_max": 0.3351554829783855, - "nauc_cv_recall_at_1_std": 0.7742753455023496, - "nauc_cv_recall_at_20_diff1": -0.05830832523955373, - "nauc_cv_recall_at_20_max": 0.2981752507205586, - "nauc_cv_recall_at_20_std": 0.43391229719546554, - "nauc_cv_recall_at_3_diff1": -0.00034359410196027406, - "nauc_cv_recall_at_3_max": 0.406924879634949, - "nauc_cv_recall_at_3_std": 0.7463139581208836, - "nauc_cv_recall_at_5_diff1": 0.0003485640898375281, - "nauc_cv_recall_at_5_max": 0.5266418196067134, - "nauc_cv_recall_at_5_std": 0.7917952078057013, - "nauc_map_at_1000_diff1": 0.03434531985076361, - "nauc_map_at_1000_max": 0.31700764572606704, - "nauc_map_at_1000_std": 0.513352390609002, - "nauc_map_at_100_diff1": 0.03249238585044048, - "nauc_map_at_100_max": 0.344444287197186, - "nauc_map_at_100_std": 0.5550391228334784, - "nauc_map_at_10_diff1": 0.058183011455032234, - "nauc_map_at_10_max": 0.4398961980755496, - "nauc_map_at_10_std": 0.7447981009503175, - "nauc_map_at_1_diff1": 0.10976975438838904, - "nauc_map_at_1_max": 0.47413777599313356, - "nauc_map_at_1_std": 0.8871376727511748, - "nauc_map_at_20_diff1": 0.04397629778875386, - "nauc_map_at_20_max": 0.39610041742289226, - "nauc_map_at_20_std": 0.6896304269055668, - "nauc_map_at_3_diff1": 0.16301622153895354, - "nauc_map_at_3_max": 0.4091613868613357, - "nauc_map_at_3_std": 0.8758514400262921, - "nauc_map_at_5_diff1": 0.08874006709245287, - "nauc_map_at_5_max": 0.4728023227574577, - "nauc_map_at_5_std": 0.8436644205016807, - "nauc_mrr_at_1000_diff1": 0.01740259977732265, - "nauc_mrr_at_1000_max": 0.29799538890335353, - "nauc_mrr_at_1000_std": 0.471596671070976, - "nauc_mrr_at_100_diff1": 0.013919150972653904, - "nauc_mrr_at_100_max": 0.3236918247489658, - "nauc_mrr_at_100_std": 0.5110336712019508, - "nauc_mrr_at_10_diff1": 0.027150120334202502, - "nauc_mrr_at_10_max": 0.4057588635600142, - "nauc_mrr_at_10_std": 0.6752326785079271, - "nauc_mrr_at_1_diff1": 0.22229315572956015, - "nauc_mrr_at_1_max": 0.3351554829783855, - "nauc_mrr_at_1_std": 0.7742753455023496, - "nauc_mrr_at_20_diff1": 0.016326440037051686, - "nauc_mrr_at_20_max": 0.37567945892777016, - "nauc_mrr_at_20_std": 0.6305420118844842, - "nauc_mrr_at_3_diff1": 0.06993606678947581, - "nauc_mrr_at_3_max": 0.38672053784189653, - "nauc_mrr_at_3_std": 0.7380649388523338, - "nauc_mrr_at_5_diff1": 0.05434084587610187, - "nauc_mrr_at_5_max": 0.4449723918087936, - "nauc_mrr_at_5_std": 0.7605202238131582, - "nauc_ndcg_at_1000_diff1": 0.050164927261478795, - "nauc_ndcg_at_1000_max": 0.117452719821097, - "nauc_ndcg_at_1000_std": 0.1867385570636576, - "nauc_ndcg_at_100_diff1": -0.016674132560073465, - "nauc_ndcg_at_100_max": 0.21961935814992392, - "nauc_ndcg_at_100_std": 0.27453278925206187, - "nauc_ndcg_at_10_diff1": 0.020330391687192594, - "nauc_ndcg_at_10_max": 0.4189996415078504, - "nauc_ndcg_at_10_std": 0.6632133533180021, - "nauc_ndcg_at_1_diff1": 0.10976975438838904, - "nauc_ndcg_at_1_max": 0.47413777599313356, - "nauc_ndcg_at_1_std": 0.8871376727511748, - "nauc_ndcg_at_20_diff1": 0.004206349712176916, - "nauc_ndcg_at_20_max": 0.3318577423266575, - "nauc_ndcg_at_20_std": 0.5628695087328316, - "nauc_ndcg_at_3_diff1": 0.17023097167757442, - "nauc_ndcg_at_3_max": 0.38994775524514685, - "nauc_ndcg_at_3_std": 0.8747791276963487, - "nauc_ndcg_at_5_diff1": 0.06334559288223734, - "nauc_ndcg_at_5_max": 0.4816104113947934, - "nauc_ndcg_at_5_std": 0.828927419007028, - "nauc_precision_at_1000_diff1": 0.08556738813980541, - "nauc_precision_at_1000_max": 0.04177557742932134, - "nauc_precision_at_1000_std": 0.09542880648474839, - "nauc_precision_at_100_diff1": -0.035702893863874686, - "nauc_precision_at_100_max": 0.16550643303366588, - "nauc_precision_at_100_std": 0.1588177693069593, - "nauc_precision_at_10_diff1": -0.02465221955110585, - "nauc_precision_at_10_max": 0.390557146877249, - "nauc_precision_at_10_std": 0.5605789784335422, - "nauc_precision_at_1_diff1": 0.10976975438838904, - "nauc_precision_at_1_max": 0.47413777599313356, - "nauc_precision_at_1_std": 0.8871376727511748, - "nauc_precision_at_20_diff1": -0.02927689545315649, - "nauc_precision_at_20_max": 0.26369251710115277, - "nauc_precision_at_20_std": 0.43725836390939754, - "nauc_precision_at_3_diff1": 0.18284774781549482, - "nauc_precision_at_3_max": 0.35347854339802, - "nauc_precision_at_3_std": 0.8730298818450715, - "nauc_precision_at_5_diff1": 0.023455296242707004, - "nauc_precision_at_5_max": 0.4960652977597613, - "nauc_precision_at_5_std": 0.8066669786767225, - "nauc_recall_at_1000_diff1": 0.08556738813980604, - "nauc_recall_at_1000_max": 0.04177557742932173, - "nauc_recall_at_1000_std": 0.09542880648474937, - "nauc_recall_at_100_diff1": -0.03570289386387487, - "nauc_recall_at_100_max": 0.16550643303366577, - "nauc_recall_at_100_std": 0.15881776930695907, - "nauc_recall_at_10_diff1": -0.024652219551105883, - "nauc_recall_at_10_max": 0.39055714687724913, - "nauc_recall_at_10_std": 0.5605789784335424, - "nauc_recall_at_1_diff1": 0.10976975438838904, - "nauc_recall_at_1_max": 0.47413777599313356, - "nauc_recall_at_1_std": 0.8871376727511748, - "nauc_recall_at_20_diff1": -0.029276895453156492, - "nauc_recall_at_20_max": 0.26369251710115293, - "nauc_recall_at_20_std": 0.43725836390939793, - "nauc_recall_at_3_diff1": 0.18284774781549484, - "nauc_recall_at_3_max": 0.35347854339801976, - "nauc_recall_at_3_std": 0.8730298818450716, - "nauc_recall_at_5_diff1": 0.023455296242707074, - "nauc_recall_at_5_max": 0.4960652977597618, - "nauc_recall_at_5_std": 0.8066669786767229, - "ndcg_at_1": 0.0018, - "ndcg_at_10": 0.00668, - "ndcg_at_100": 0.02049, - "ndcg_at_1000": 0.08436, - "ndcg_at_20": 0.00868, - "ndcg_at_3": 0.00346, - "ndcg_at_5": 0.00494, - "precision_at_1": 0.0018, - "precision_at_10": 0.00138, - "precision_at_100": 0.0009, - "precision_at_1000": 0.00065, - "precision_at_20": 0.00108, - "precision_at_3": 0.0016, - "precision_at_5": 0.00168, - "recall_at_1": 0.0018, - "recall_at_10": 0.01383, - "recall_at_100": 0.0902, - "recall_at_1000": 0.64702, - "recall_at_20": 0.02165, - "recall_at_3": 0.00481, - "recall_at_5": 0.00842 - } - ] - }, - "task_name": "VidoreTatdqaRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsI2TRetrieval.json deleted file mode 100644 index 22a50384e2..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "aaee58895a66e4d619168849267ed2bb40d37043", - "evaluation_time": 637.0611886978149, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.14225, - "cv_recall_at_10": 0.3711, - "cv_recall_at_100": 0.63325, - "cv_recall_at_1000": 0.8308, - "cv_recall_at_20": 0.4508, - "cv_recall_at_3": 0.2426, - "cv_recall_at_5": 0.2962, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.24665, - "map_at_1": 0.14215, - "map_at_10": 0.20805, - "map_at_100": 0.21816, - "map_at_1000": 0.21898, - "map_at_20": 0.2136, - "map_at_3": 0.18584, - "map_at_5": 0.19808, - "mrr_at_1": 0.14225, - "mrr_at_10": 0.20809535714285854, - "mrr_at_100": 0.21821117637519633, - "mrr_at_1000": 0.21903244279521142, - "mrr_at_20": 0.21365230710680844, - "mrr_at_3": 0.18589166666666862, - "mrr_at_5": 0.19812916666666722, - "nauc_cv_recall_at_1000_diff1": 0.1374166886265108, - "nauc_cv_recall_at_1000_max": 0.4326023231571799, - "nauc_cv_recall_at_1000_std": 0.5371027114383197, - "nauc_cv_recall_at_100_diff1": 0.1859457721091746, - "nauc_cv_recall_at_100_max": 0.37897087843161015, - "nauc_cv_recall_at_100_std": 0.3801615793616046, - "nauc_cv_recall_at_10_diff1": 0.24411979633870634, - "nauc_cv_recall_at_10_max": 0.31955568154632447, - "nauc_cv_recall_at_10_std": 0.15577379453785636, - "nauc_cv_recall_at_1_diff1": 0.43018478049486225, - "nauc_cv_recall_at_1_max": 0.3132376136367806, - "nauc_cv_recall_at_1_std": 0.04846489768168653, - "nauc_cv_recall_at_20_diff1": 0.22602718014627493, - "nauc_cv_recall_at_20_max": 0.3316118262662824, - "nauc_cv_recall_at_20_std": 0.21054304754429856, - "nauc_cv_recall_at_3_diff1": 0.3032334244956384, - "nauc_cv_recall_at_3_max": 0.308301563672542, - "nauc_cv_recall_at_3_std": 0.08475358897537363, - "nauc_cv_recall_at_5_diff1": 0.27115644663985733, - "nauc_cv_recall_at_5_max": 0.3128647962090153, - "nauc_cv_recall_at_5_std": 0.11554840469255882, - "nauc_map_at_1000_diff1": 0.3436467125060594, - "nauc_map_at_1000_max": 0.31323974428346524, - "nauc_map_at_1000_std": 0.09113461601378964, - "nauc_map_at_100_diff1": 0.34347632151089674, - "nauc_map_at_100_max": 0.31324681175801156, - "nauc_map_at_100_std": 0.0911844476249971, - "nauc_map_at_10_diff1": 0.34583327161961397, - "nauc_map_at_10_max": 0.31087876312089413, - "nauc_map_at_10_std": 0.0819667272939276, - "nauc_map_at_1_diff1": 0.4308567698038146, - "nauc_map_at_1_max": 0.31313330076808726, - "nauc_map_at_1_std": 0.048642072248903705, - "nauc_map_at_20_diff1": 0.3440655791308437, - "nauc_map_at_20_max": 0.3119640515148501, - "nauc_map_at_20_std": 0.08682389789331668, - "nauc_map_at_3_diff1": 0.36217978931296785, - "nauc_map_at_3_max": 0.30861261807449575, - "nauc_map_at_3_std": 0.06565290770639244, - "nauc_map_at_5_diff1": 0.3511167742174239, - "nauc_map_at_5_max": 0.3097889051312141, - "nauc_map_at_5_std": 0.07493591815578858, - "nauc_mrr_at_1000_diff1": 0.3433908476595087, - "nauc_mrr_at_1000_max": 0.3132831686651032, - "nauc_mrr_at_1000_std": 0.09105210117570409, - "nauc_mrr_at_100_diff1": 0.3432209969084996, - "nauc_mrr_at_100_max": 0.31329014350682705, - "nauc_mrr_at_100_std": 0.09110207533826178, - "nauc_mrr_at_10_diff1": 0.34557725180410864, - "nauc_mrr_at_10_max": 0.3109217636298925, - "nauc_mrr_at_10_std": 0.08188747150218671, - "nauc_mrr_at_1_diff1": 0.43018478049486225, - "nauc_mrr_at_1_max": 0.3132376136367806, - "nauc_mrr_at_1_std": 0.04846489768168653, - "nauc_mrr_at_20_diff1": 0.3438109296410951, - "nauc_mrr_at_20_max": 0.31200750537885574, - "nauc_mrr_at_20_std": 0.08674305106857981, - "nauc_mrr_at_3_diff1": 0.36191163048605396, - "nauc_mrr_at_3_max": 0.30865755294985336, - "nauc_mrr_at_3_std": 0.06557455228533765, - "nauc_mrr_at_5_diff1": 0.35085778529185385, - "nauc_mrr_at_5_max": 0.30983244732505644, - "nauc_mrr_at_5_std": 0.07485758391758571, - "nauc_ndcg_at_1000_diff1": 0.309216176636288, - "nauc_ndcg_at_1000_max": 0.32619689821644354, - "nauc_ndcg_at_1000_std": 0.14728501394403046, - "nauc_ndcg_at_100_diff1": 0.3055060583350281, - "nauc_ndcg_at_100_max": 0.3255442418423197, - "nauc_ndcg_at_100_std": 0.14920089955599194, - "nauc_ndcg_at_10_diff1": 0.31576995334315505, - "nauc_ndcg_at_10_max": 0.3131255785064659, - "nauc_ndcg_at_10_std": 0.10267909300234637, - "nauc_ndcg_at_1_diff1": 0.4308567698038146, - "nauc_ndcg_at_1_max": 0.31313330076808726, - "nauc_ndcg_at_1_std": 0.048642072248903705, - "nauc_ndcg_at_20_diff1": 0.3101266740624065, - "nauc_ndcg_at_20_max": 0.31681047684725033, - "nauc_ndcg_at_20_std": 0.11932009857835636, - "nauc_ndcg_at_3_diff1": 0.34498997548370164, - "nauc_ndcg_at_3_max": 0.3084426214505432, - "nauc_ndcg_at_3_std": 0.07112303928163569, - "nauc_ndcg_at_5_diff1": 0.32728459943297505, - "nauc_ndcg_at_5_max": 0.3105551753088202, - "nauc_ndcg_at_5_std": 0.08666800363917986, - "nauc_precision_at_1000_diff1": 0.13741668862651063, - "nauc_precision_at_1000_max": 0.43260232315717956, - "nauc_precision_at_1000_std": 0.5371027114383187, - "nauc_precision_at_100_diff1": 0.1859457721091744, - "nauc_precision_at_100_max": 0.37897087843161026, - "nauc_precision_at_100_std": 0.38016157936160416, - "nauc_precision_at_10_diff1": 0.24411979633870629, - "nauc_precision_at_10_max": 0.3195556815463242, - "nauc_precision_at_10_std": 0.15577379453785628, - "nauc_precision_at_1_diff1": 0.4308567698038146, - "nauc_precision_at_1_max": 0.31313330076808726, - "nauc_precision_at_1_std": 0.048642072248903705, - "nauc_precision_at_20_diff1": 0.22602718014627443, - "nauc_precision_at_20_max": 0.3316118262662818, - "nauc_precision_at_20_std": 0.21054304754429795, - "nauc_precision_at_3_diff1": 0.3032334244956383, - "nauc_precision_at_3_max": 0.3083015636725422, - "nauc_precision_at_3_std": 0.08475358897537373, - "nauc_precision_at_5_diff1": 0.2711564466398571, - "nauc_precision_at_5_max": 0.31286479620901536, - "nauc_precision_at_5_std": 0.1155484046925588, - "nauc_recall_at_1000_diff1": 0.1374166886265108, - "nauc_recall_at_1000_max": 0.4326023231571799, - "nauc_recall_at_1000_std": 0.5371027114383197, - "nauc_recall_at_100_diff1": 0.1859457721091746, - "nauc_recall_at_100_max": 0.37897087843161015, - "nauc_recall_at_100_std": 0.3801615793616046, - "nauc_recall_at_10_diff1": 0.24411979633870634, - "nauc_recall_at_10_max": 0.31955568154632447, - "nauc_recall_at_10_std": 0.15577379453785636, - "nauc_recall_at_1_diff1": 0.4308567698038146, - "nauc_recall_at_1_max": 0.31313330076808726, - "nauc_recall_at_1_std": 0.048642072248903705, - "nauc_recall_at_20_diff1": 0.22602718014627493, - "nauc_recall_at_20_max": 0.3316118262662824, - "nauc_recall_at_20_std": 0.21054304754429856, - "nauc_recall_at_3_diff1": 0.3032334244956384, - "nauc_recall_at_3_max": 0.308301563672542, - "nauc_recall_at_3_std": 0.08475358897537363, - "nauc_recall_at_5_diff1": 0.27115644663985733, - "nauc_recall_at_5_max": 0.3128647962090153, - "nauc_recall_at_5_std": 0.11554840469255882, - "ndcg_at_1": 0.14215, - "ndcg_at_10": 0.24665, - "ndcg_at_100": 0.30043, - "ndcg_at_1000": 0.32483, - "ndcg_at_20": 0.26683, - "ndcg_at_3": 0.20039, - "ndcg_at_5": 0.22246, - "precision_at_1": 0.14215, - "precision_at_10": 0.03711, - "precision_at_100": 0.00633, - "precision_at_1000": 0.00083, - "precision_at_20": 0.02254, - "precision_at_3": 0.08087, - "precision_at_5": 0.05924, - "recall_at_1": 0.14215, - "recall_at_10": 0.3711, - "recall_at_100": 0.63325, - "recall_at_1000": 0.8308, - "recall_at_20": 0.4508, - "recall_at_3": 0.2426, - "recall_at_5": 0.2962 - } - ] - }, - "task_name": "VisualNewsI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsT2IRetrieval.json deleted file mode 100644 index f6e3f67041..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VisualNewsT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "94c519d850dba2b0058c2fc9b5da6142a59aa285", - "evaluation_time": 2032.3265976905823, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.14214, - "cv_recall_at_10": 0.38615, - "cv_recall_at_100": 0.64896, - "cv_recall_at_1000": 0.83731, - "cv_recall_at_20": 0.47277, - "cv_recall_at_3": 0.24846, - "cv_recall_at_5": 0.30413, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.25322, - "map_at_1": 0.14189, - "map_at_10": 0.21201, - "map_at_100": 0.22241, - "map_at_1000": 0.22322, - "map_at_20": 0.21802, - "map_at_3": 0.18826, - "map_at_5": 0.20104, - "mrr_at_1": 0.14213553388347086, - "mrr_at_10": 0.21213452172567082, - "mrr_at_100": 0.22253810455058598, - "mrr_at_1000": 0.22334262930582421, - "mrr_at_20": 0.21815238887195898, - "mrr_at_3": 0.18843044094357164, - "mrr_at_5": 0.2011486204884563, - "nauc_cv_recall_at_1000_diff1": 0.2738151451376307, - "nauc_cv_recall_at_1000_max": 0.5931214969636025, - "nauc_cv_recall_at_1000_std": 0.7501777008032257, - "nauc_cv_recall_at_100_diff1": 0.2743373648460172, - "nauc_cv_recall_at_100_max": 0.46575688991474795, - "nauc_cv_recall_at_100_std": 0.576698437786742, - "nauc_cv_recall_at_10_diff1": 0.28264363675897997, - "nauc_cv_recall_at_10_max": 0.35455403543397274, - "nauc_cv_recall_at_10_std": 0.3409080895780153, - "nauc_cv_recall_at_1_diff1": 0.43574527435374394, - "nauc_cv_recall_at_1_max": 0.30366809286647123, - "nauc_cv_recall_at_1_std": 0.19297182230123075, - "nauc_cv_recall_at_20_diff1": 0.27117097023604686, - "nauc_cv_recall_at_20_max": 0.38504583058673747, - "nauc_cv_recall_at_20_std": 0.4083714618245761, - "nauc_cv_recall_at_3_diff1": 0.3290103271052618, - "nauc_cv_recall_at_3_max": 0.3175734967043012, - "nauc_cv_recall_at_3_std": 0.2572722720243564, - "nauc_cv_recall_at_5_diff1": 0.30524189503214394, - "nauc_cv_recall_at_5_max": 0.3267082717032173, - "nauc_cv_recall_at_5_std": 0.28295915343267924, - "nauc_map_at_1000_diff1": 0.36629495836246867, - "nauc_map_at_1000_max": 0.32247650003298395, - "nauc_map_at_1000_std": 0.254260785723814, - "nauc_map_at_100_diff1": 0.3661110091807752, - "nauc_map_at_100_max": 0.32235486608870173, - "nauc_map_at_100_std": 0.25427301752910086, - "nauc_map_at_10_diff1": 0.3666371005486745, - "nauc_map_at_10_max": 0.3179934234214588, - "nauc_map_at_10_std": 0.2447703790002406, - "nauc_map_at_1_diff1": 0.43743514555841434, - "nauc_map_at_1_max": 0.30350123194006623, - "nauc_map_at_1_std": 0.1933581414080973, - "nauc_map_at_20_diff1": 0.3656297660643177, - "nauc_map_at_20_max": 0.3206298829457058, - "nauc_map_at_20_std": 0.2506027371516703, - "nauc_map_at_3_diff1": 0.37914731231852516, - "nauc_map_at_3_max": 0.31014982499900495, - "nauc_map_at_3_std": 0.22582773571427692, - "nauc_map_at_5_diff1": 0.37107276727666866, - "nauc_map_at_5_max": 0.3129587682028005, - "nauc_map_at_5_std": 0.23406545136024842, - "nauc_mrr_at_1000_diff1": 0.365694119656496, - "nauc_mrr_at_1000_max": 0.3225064538045288, - "nauc_mrr_at_1000_std": 0.25372446071677807, - "nauc_mrr_at_100_diff1": 0.36550943256415147, - "nauc_mrr_at_100_max": 0.32238352603484055, - "nauc_mrr_at_100_std": 0.2537390594412823, - "nauc_mrr_at_10_diff1": 0.36604292281022704, - "nauc_mrr_at_10_max": 0.318023509003043, - "nauc_mrr_at_10_std": 0.2442321947644308, - "nauc_mrr_at_1_diff1": 0.43574527435374394, - "nauc_mrr_at_1_max": 0.30366809286647123, - "nauc_mrr_at_1_std": 0.19297182230123075, - "nauc_mrr_at_20_diff1": 0.36503648073161493, - "nauc_mrr_at_20_max": 0.32065493374494664, - "nauc_mrr_at_20_std": 0.2500569829241322, - "nauc_mrr_at_3_diff1": 0.3783635531911808, - "nauc_mrr_at_3_max": 0.3099868426780373, - "nauc_mrr_at_3_std": 0.2250798843091142, - "nauc_mrr_at_5_diff1": 0.37029885694565495, - "nauc_mrr_at_5_max": 0.3128479822671293, - "nauc_mrr_at_5_std": 0.23345888219471445, - "nauc_ndcg_at_1000_diff1": 0.34587426274769756, - "nauc_ndcg_at_1000_max": 0.35296786555677756, - "nauc_ndcg_at_1000_std": 0.31690613840921944, - "nauc_ndcg_at_100_diff1": 0.34177689609578826, - "nauc_ndcg_at_100_max": 0.3503176936120679, - "nauc_ndcg_at_100_std": 0.31898929198056136, - "nauc_ndcg_at_10_diff1": 0.34181459809724857, - "nauc_ndcg_at_10_max": 0.328190780924741, - "nauc_ndcg_at_10_std": 0.2720579741726425, - "nauc_ndcg_at_1_diff1": 0.43743514555841434, - "nauc_ndcg_at_1_max": 0.30350123194006623, - "nauc_ndcg_at_1_std": 0.1933581414080973, - "nauc_ndcg_at_20_diff1": 0.33852580543057287, - "nauc_ndcg_at_20_max": 0.33728315846227197, - "nauc_ndcg_at_20_std": 0.29215613689258596, - "nauc_ndcg_at_3_diff1": 0.3646193686577447, - "nauc_ndcg_at_3_max": 0.31244948573813575, - "nauc_ndcg_at_3_std": 0.23509499886693777, - "nauc_ndcg_at_5_diff1": 0.35161503224004037, - "nauc_ndcg_at_5_max": 0.31712039236583617, - "nauc_ndcg_at_5_std": 0.24853289096733608, - "nauc_precision_at_1000_diff1": 0.27273524762384516, - "nauc_precision_at_1000_max": 0.5914908390552021, - "nauc_precision_at_1000_std": 0.7492366648909354, - "nauc_precision_at_100_diff1": 0.2743373648460168, - "nauc_precision_at_100_max": 0.46575688991474795, - "nauc_precision_at_100_std": 0.5766984377867421, - "nauc_precision_at_10_diff1": 0.28267830445192166, - "nauc_precision_at_10_max": 0.35462752640006245, - "nauc_precision_at_10_std": 0.34090808957801505, - "nauc_precision_at_1_diff1": 0.43743514555841434, - "nauc_precision_at_1_max": 0.30350123194006623, - "nauc_precision_at_1_std": 0.1933581414080973, - "nauc_precision_at_20_diff1": 0.2711325060911637, - "nauc_precision_at_20_max": 0.3851281672816333, - "nauc_precision_at_20_std": 0.4085615041031749, - "nauc_precision_at_3_diff1": 0.32949775513819174, - "nauc_precision_at_3_max": 0.3181893480026203, - "nauc_precision_at_3_std": 0.2578284990482501, - "nauc_precision_at_5_diff1": 0.30599236424623105, - "nauc_precision_at_5_max": 0.32728294271527764, - "nauc_precision_at_5_std": 0.28316104447138657, - "nauc_recall_at_1000_diff1": 0.2735240225347363, - "nauc_recall_at_1000_max": 0.5932571091213296, - "nauc_recall_at_1000_std": 0.7502902830528209, - "nauc_recall_at_100_diff1": 0.274464026101684, - "nauc_recall_at_100_max": 0.4658591197876539, - "nauc_recall_at_100_std": 0.5765276833211775, - "nauc_recall_at_10_diff1": 0.2826783044519219, - "nauc_recall_at_10_max": 0.3546275264000627, - "nauc_recall_at_10_std": 0.3409080895780153, - "nauc_recall_at_1_diff1": 0.43743514555841434, - "nauc_recall_at_1_max": 0.30350123194006623, - "nauc_recall_at_1_std": 0.1933581414080973, - "nauc_recall_at_20_diff1": 0.27116932602660276, - "nauc_recall_at_20_max": 0.385175879145329, - "nauc_recall_at_20_std": 0.4085181212052684, - "nauc_recall_at_3_diff1": 0.3294977551381917, - "nauc_recall_at_3_max": 0.3181893480026203, - "nauc_recall_at_3_std": 0.25782849904825034, - "nauc_recall_at_5_diff1": 0.3059923642462314, - "nauc_recall_at_5_max": 0.3272829427152781, - "nauc_recall_at_5_std": 0.2831610444713866, - "ndcg_at_1": 0.14189, - "ndcg_at_10": 0.25322, - "ndcg_at_100": 0.30753, - "ndcg_at_1000": 0.33093, - "ndcg_at_20": 0.2751, - "ndcg_at_3": 0.20363, - "ndcg_at_5": 0.22668, - "precision_at_1": 0.14189, - "precision_at_10": 0.03861, - "precision_at_100": 0.00649, - "precision_at_1000": 0.00084, - "precision_at_20": 0.02363, - "precision_at_3": 0.08274, - "precision_at_5": 0.06084, - "recall_at_1": 0.14189, - "recall_at_10": 0.38615, - "recall_at_100": 0.64889, - "recall_at_1000": 0.83726, - "recall_at_20": 0.47264, - "recall_at_3": 0.24821, - "recall_at_5": 0.30418 - } - ] - }, - "task_name": "VisualNewsT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VizWizIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VizWizIT2TRetrieval.json deleted file mode 100644 index 03ec1474f1..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VizWizIT2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "044af162d55f82ab603fa16ffcf7f1e4dbf300e9", - "evaluation_time": 30.84842872619629, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.01343, - "cv_recall_at_10": 0.07131, - "cv_recall_at_100": 0.39569, - "cv_recall_at_1000": 0.91433, - "cv_recall_at_20": 0.12063, - "cv_recall_at_3": 0.0345, - "cv_recall_at_5": 0.04561, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.03809, - "map_at_1": 0.01343, - "map_at_10": 0.02804, - "map_at_100": 0.03742, - "map_at_1000": 0.03951, - "map_at_20": 0.0312, - "map_at_3": 0.02227, - "map_at_5": 0.0247, - "mrr_at_1": 0.013429034498726558, - "mrr_at_10": 0.028036417160056885, - "mrr_at_100": 0.037420008046085314, - "mrr_at_1000": 0.039505486644354056, - "mrr_at_20": 0.031201242210305073, - "mrr_at_3": 0.022265956625762126, - "mrr_at_5": 0.024697074940186765, - "nauc_cv_recall_at_1000_diff1": 0.050583394811374374, - "nauc_cv_recall_at_1000_max": -0.18376667036878835, - "nauc_cv_recall_at_1000_std": 0.1298217845164567, - "nauc_cv_recall_at_100_diff1": 0.027698576038987024, - "nauc_cv_recall_at_100_max": -0.009117749373629837, - "nauc_cv_recall_at_100_std": 0.006829941177572575, - "nauc_cv_recall_at_10_diff1": 0.16070351829818008, - "nauc_cv_recall_at_10_max": -0.06294639384504126, - "nauc_cv_recall_at_10_std": -0.008037027217108009, - "nauc_cv_recall_at_1_diff1": 0.38564376924174965, - "nauc_cv_recall_at_1_max": -0.14158948678291924, - "nauc_cv_recall_at_1_std": -0.04672599956819156, - "nauc_cv_recall_at_20_diff1": 0.08493892106912637, - "nauc_cv_recall_at_20_max": -0.015163802751554916, - "nauc_cv_recall_at_20_std": 0.021963421659727856, - "nauc_cv_recall_at_3_diff1": 0.24266018554188226, - "nauc_cv_recall_at_3_max": -0.09291021576633819, - "nauc_cv_recall_at_3_std": -0.02148263688905047, - "nauc_cv_recall_at_5_diff1": 0.21381393860129796, - "nauc_cv_recall_at_5_max": -0.11435865792008686, - "nauc_cv_recall_at_5_std": -0.019942360767102027, - "nauc_map_at_1000_diff1": 0.20370724207547133, - "nauc_map_at_1000_max": -0.07702226966924787, - "nauc_map_at_1000_std": -0.014549893563493205, - "nauc_map_at_100_diff1": 0.20105115841633822, - "nauc_map_at_100_max": -0.0738589428173749, - "nauc_map_at_100_std": -0.015012934920928511, - "nauc_map_at_10_diff1": 0.2501026724956375, - "nauc_map_at_10_max": -0.09835226805301363, - "nauc_map_at_10_std": -0.024470727792863956, - "nauc_map_at_1_diff1": 0.38564376924174965, - "nauc_map_at_1_max": -0.14158948678291924, - "nauc_map_at_1_std": -0.04672599956819156, - "nauc_map_at_20_diff1": 0.2244173819284951, - "nauc_map_at_20_max": -0.08444395428873437, - "nauc_map_at_20_std": -0.016020286898738696, - "nauc_map_at_3_diff1": 0.2910544924422312, - "nauc_map_at_3_max": -0.10878289402033367, - "nauc_map_at_3_std": -0.03140611440353479, - "nauc_map_at_5_diff1": 0.2743756933858256, - "nauc_map_at_5_max": -0.11562364386231114, - "nauc_map_at_5_std": -0.02997097324144088, - "nauc_mrr_at_1000_diff1": 0.20370724207547133, - "nauc_mrr_at_1000_max": -0.07702226966924787, - "nauc_mrr_at_1000_std": -0.014549893563493205, - "nauc_mrr_at_100_diff1": 0.20105115841633822, - "nauc_mrr_at_100_max": -0.0738589428173749, - "nauc_mrr_at_100_std": -0.015012934920928511, - "nauc_mrr_at_10_diff1": 0.2501026724956375, - "nauc_mrr_at_10_max": -0.09835226805301363, - "nauc_mrr_at_10_std": -0.024470727792863956, - "nauc_mrr_at_1_diff1": 0.38564376924174965, - "nauc_mrr_at_1_max": -0.14158948678291924, - "nauc_mrr_at_1_std": -0.04672599956819156, - "nauc_mrr_at_20_diff1": 0.2244173819284951, - "nauc_mrr_at_20_max": -0.08444395428873437, - "nauc_mrr_at_20_std": -0.016020286898738696, - "nauc_mrr_at_3_diff1": 0.2910544924422312, - "nauc_mrr_at_3_max": -0.10878289402033367, - "nauc_mrr_at_3_std": -0.03140611440353479, - "nauc_mrr_at_5_diff1": 0.2743756933858256, - "nauc_mrr_at_5_max": -0.11562364386231114, - "nauc_mrr_at_5_std": -0.02997097324144088, - "nauc_ndcg_at_1000_diff1": 0.13406325540576058, - "nauc_ndcg_at_1000_max": -0.07106041428299001, - "nauc_ndcg_at_1000_std": 0.012519681774848902, - "nauc_ndcg_at_100_diff1": 0.10175722112103643, - "nauc_ndcg_at_100_max": -0.03450295253160438, - "nauc_ndcg_at_100_std": -0.0014070304908460896, - "nauc_ndcg_at_10_diff1": 0.21195276364011353, - "nauc_ndcg_at_10_max": -0.08376357345515494, - "nauc_ndcg_at_10_std": -0.017452778189587872, - "nauc_ndcg_at_1_diff1": 0.38564376924174965, - "nauc_ndcg_at_1_max": -0.14158948678291924, - "nauc_ndcg_at_1_std": -0.04672599956819156, - "nauc_ndcg_at_20_diff1": 0.15751370600120412, - "nauc_ndcg_at_20_max": -0.05224769346714892, - "nauc_ndcg_at_20_std": 0.0017388040866098441, - "nauc_ndcg_at_3_diff1": 0.27420783124692033, - "nauc_ndcg_at_3_max": -0.10323505279682847, - "nauc_ndcg_at_3_std": -0.028002766346549326, - "nauc_ndcg_at_5_diff1": 0.2515210601839793, - "nauc_ndcg_at_5_max": -0.11470427341623049, - "nauc_ndcg_at_5_std": -0.026194743288170313, - "nauc_precision_at_1000_diff1": 0.050583394811373666, - "nauc_precision_at_1000_max": -0.18376667036878533, - "nauc_precision_at_1000_std": 0.1298217845164564, - "nauc_precision_at_100_diff1": 0.027698576038987212, - "nauc_precision_at_100_max": -0.009117749373629547, - "nauc_precision_at_100_std": 0.006829941177572595, - "nauc_precision_at_10_diff1": 0.16070351829818025, - "nauc_precision_at_10_max": -0.06294639384504118, - "nauc_precision_at_10_std": -0.008037027217107877, - "nauc_precision_at_1_diff1": 0.38564376924174965, - "nauc_precision_at_1_max": -0.14158948678291924, - "nauc_precision_at_1_std": -0.04672599956819156, - "nauc_precision_at_20_diff1": 0.08493892106912652, - "nauc_precision_at_20_max": -0.015163802751554795, - "nauc_precision_at_20_std": 0.021963421659728005, - "nauc_precision_at_3_diff1": 0.24266018554188226, - "nauc_precision_at_3_max": -0.09291021576633826, - "nauc_precision_at_3_std": -0.02148263688905037, - "nauc_precision_at_5_diff1": 0.2138139386012981, - "nauc_precision_at_5_max": -0.11435865792008668, - "nauc_precision_at_5_std": -0.019942360767101795, - "nauc_recall_at_1000_diff1": 0.050583394811374374, - "nauc_recall_at_1000_max": -0.18376667036878835, - "nauc_recall_at_1000_std": 0.1298217845164567, - "nauc_recall_at_100_diff1": 0.027698576038987024, - "nauc_recall_at_100_max": -0.009117749373629837, - "nauc_recall_at_100_std": 0.006829941177572575, - "nauc_recall_at_10_diff1": 0.16070351829818008, - "nauc_recall_at_10_max": -0.06294639384504126, - "nauc_recall_at_10_std": -0.008037027217108009, - "nauc_recall_at_1_diff1": 0.38564376924174965, - "nauc_recall_at_1_max": -0.14158948678291924, - "nauc_recall_at_1_std": -0.04672599956819156, - "nauc_recall_at_20_diff1": 0.08493892106912637, - "nauc_recall_at_20_max": -0.015163802751554916, - "nauc_recall_at_20_std": 0.021963421659727856, - "nauc_recall_at_3_diff1": 0.24266018554188226, - "nauc_recall_at_3_max": -0.09291021576633819, - "nauc_recall_at_3_std": -0.02148263688905047, - "nauc_recall_at_5_diff1": 0.21381393860129796, - "nauc_recall_at_5_max": -0.11435865792008686, - "nauc_recall_at_5_std": -0.019942360767102027, - "ndcg_at_1": 0.01343, - "ndcg_at_10": 0.03809, - "ndcg_at_100": 0.09961, - "ndcg_at_1000": 0.16333, - "ndcg_at_20": 0.05024, - "ndcg_at_3": 0.02539, - "ndcg_at_5": 0.02987, - "precision_at_1": 0.01343, - "precision_at_10": 0.00713, - "precision_at_100": 0.00396, - "precision_at_1000": 0.00091, - "precision_at_20": 0.00603, - "precision_at_3": 0.0115, - "precision_at_5": 0.00912, - "recall_at_1": 0.01343, - "recall_at_10": 0.07131, - "recall_at_100": 0.39569, - "recall_at_1000": 0.91433, - "recall_at_20": 0.12063, - "recall_at_3": 0.0345, - "recall_at_5": 0.04561 - } - ] - }, - "task_name": "VizWizIT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WITT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WITT2IRetrieval.json deleted file mode 100644 index 3211c5c5af..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WITT2IRetrieval.json +++ /dev/null @@ -1,1936 +0,0 @@ -{ - "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", - "evaluation_time": 50.179383754730225, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.03933, - "cv_recall_at_10": 0.09888, - "cv_recall_at_100": 0.29775, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.12921, - "cv_recall_at_3": 0.06292, - "cv_recall_at_5": 0.07191, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.06583, - "map_at_1": 0.03933, - "map_at_10": 0.05581, - "map_at_100": 0.06157, - "map_at_1000": 0.06398, - "map_at_20": 0.05799, - "map_at_3": 0.05037, - "map_at_5": 0.05245, - "mrr_at_1": 0.03932584269662921, - "mrr_at_10": 0.05581148564294632, - "mrr_at_100": 0.06157040381700697, - "mrr_at_1000": 0.06397873885256421, - "mrr_at_20": 0.057929903295724096, - "mrr_at_3": 0.050374531835205995, - "mrr_at_5": 0.05245318352059926, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.10651632318768792, - "nauc_cv_recall_at_100_max": 0.28469315454649335, - "nauc_cv_recall_at_100_std": 0.3307774941906975, - "nauc_cv_recall_at_10_diff1": 0.24737175890918847, - "nauc_cv_recall_at_10_max": 0.37047222967534205, - "nauc_cv_recall_at_10_std": 0.35490096836353874, - "nauc_cv_recall_at_1_diff1": 0.6811989727471347, - "nauc_cv_recall_at_1_max": 0.7112499520870863, - "nauc_cv_recall_at_1_std": 0.5198436122503737, - "nauc_cv_recall_at_20_diff1": 0.2177190567014866, - "nauc_cv_recall_at_20_max": 0.29381449809900323, - "nauc_cv_recall_at_20_std": 0.33120286981015296, - "nauc_cv_recall_at_3_diff1": 0.45237935528383605, - "nauc_cv_recall_at_3_max": 0.5901481467285061, - "nauc_cv_recall_at_3_std": 0.5043744490014948, - "nauc_cv_recall_at_5_diff1": 0.36561326133619537, - "nauc_cv_recall_at_5_max": 0.5235444056882212, - "nauc_cv_recall_at_5_std": 0.47758334451301304, - "nauc_map_at_1000_diff1": 0.4572039928639722, - "nauc_map_at_1000_max": 0.556407100014074, - "nauc_map_at_1000_std": 0.4681785109479179, - "nauc_map_at_100_diff1": 0.4562262777878306, - "nauc_map_at_100_max": 0.5547399837702551, - "nauc_map_at_100_std": 0.4669314989959343, - "nauc_map_at_10_diff1": 0.4839164521569286, - "nauc_map_at_10_max": 0.581580308737203, - "nauc_map_at_10_std": 0.47985609505140064, - "nauc_map_at_1_diff1": 0.6811989727471347, - "nauc_map_at_1_max": 0.7112499520870863, - "nauc_map_at_1_std": 0.5198436122503737, - "nauc_map_at_20_diff1": 0.47296162994063123, - "nauc_map_at_20_max": 0.5646353533007439, - "nauc_map_at_20_std": 0.4719863696175586, - "nauc_map_at_3_diff1": 0.549209376368368, - "nauc_map_at_3_max": 0.6432214234915912, - "nauc_map_at_3_std": 0.5172682414970432, - "nauc_map_at_5_diff1": 0.5185926257506827, - "nauc_map_at_5_max": 0.6214937035692051, - "nauc_map_at_5_std": 0.5080343780782037, - "nauc_mrr_at_1000_diff1": 0.45723407696403573, - "nauc_mrr_at_1000_max": 0.5564216652366148, - "nauc_mrr_at_1000_std": 0.4681993792193247, - "nauc_mrr_at_100_diff1": 0.45625615571648387, - "nauc_mrr_at_100_max": 0.554754423070155, - "nauc_mrr_at_100_std": 0.46695221181507013, - "nauc_mrr_at_10_diff1": 0.4839164521569286, - "nauc_mrr_at_10_max": 0.581580308737203, - "nauc_mrr_at_10_std": 0.47985609505140064, - "nauc_mrr_at_1_diff1": 0.6811989727471347, - "nauc_mrr_at_1_max": 0.7112499520870863, - "nauc_mrr_at_1_std": 0.5198436122503737, - "nauc_mrr_at_20_diff1": 0.4738328510486203, - "nauc_mrr_at_20_max": 0.5652040793743005, - "nauc_mrr_at_20_std": 0.4724949134770215, - "nauc_mrr_at_3_diff1": 0.549209376368368, - "nauc_mrr_at_3_max": 0.6432214234915912, - "nauc_mrr_at_3_std": 0.5172682414970432, - "nauc_mrr_at_5_diff1": 0.5185926257506827, - "nauc_mrr_at_5_max": 0.6214937035692051, - "nauc_mrr_at_5_std": 0.5080343780782037, - "nauc_ndcg_at_1000_diff1": 0.36375745046225905, - "nauc_ndcg_at_1000_max": 0.4830894657999479, - "nauc_ndcg_at_1000_std": 0.4325926006081789, - "nauc_ndcg_at_100_diff1": 0.30353191564763154, - "nauc_ndcg_at_100_max": 0.43301860060603764, - "nauc_ndcg_at_100_std": 0.4049675638021951, - "nauc_ndcg_at_10_diff1": 0.4021193676211634, - "nauc_ndcg_at_10_max": 0.5101132679389379, - "nauc_ndcg_at_10_std": 0.4388204382824301, - "nauc_ndcg_at_1_diff1": 0.6811989727471347, - "nauc_ndcg_at_1_max": 0.7112499520870863, - "nauc_ndcg_at_1_std": 0.5198436122503737, - "nauc_ndcg_at_20_diff1": 0.3753829158483357, - "nauc_ndcg_at_20_max": 0.46551460951632206, - "nauc_ndcg_at_20_std": 0.4201020394377109, - "nauc_ndcg_at_3_diff1": 0.5198565864811237, - "nauc_ndcg_at_3_max": 0.6272080086003348, - "nauc_ndcg_at_3_std": 0.5136205735200569, - "nauc_ndcg_at_5_diff1": 0.4709900963415891, - "nauc_ndcg_at_5_max": 0.5914010152437607, - "nauc_ndcg_at_5_std": 0.4989203141368066, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.10651632318768808, - "nauc_precision_at_100_max": 0.2846931545464938, - "nauc_precision_at_100_std": 0.33077749419069763, - "nauc_precision_at_10_diff1": 0.24737175890918878, - "nauc_precision_at_10_max": 0.37047222967534255, - "nauc_precision_at_10_std": 0.3549009683635391, - "nauc_precision_at_1_diff1": 0.6811989727471347, - "nauc_precision_at_1_max": 0.7112499520870863, - "nauc_precision_at_1_std": 0.5198436122503737, - "nauc_precision_at_20_diff1": 0.21099288999578236, - "nauc_precision_at_20_max": 0.2900690915664105, - "nauc_precision_at_20_std": 0.3272258932695979, - "nauc_precision_at_3_diff1": 0.45237935528383627, - "nauc_precision_at_3_max": 0.5901481467285062, - "nauc_precision_at_3_std": 0.5043744490014948, - "nauc_precision_at_5_diff1": 0.36561326133619565, - "nauc_precision_at_5_max": 0.5235444056882217, - "nauc_precision_at_5_std": 0.47758334451301343, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.10651632318768792, - "nauc_recall_at_100_max": 0.28469315454649335, - "nauc_recall_at_100_std": 0.3307774941906975, - "nauc_recall_at_10_diff1": 0.24737175890918847, - "nauc_recall_at_10_max": 0.37047222967534205, - "nauc_recall_at_10_std": 0.35490096836353874, - "nauc_recall_at_1_diff1": 0.6811989727471347, - "nauc_recall_at_1_max": 0.7112499520870863, - "nauc_recall_at_1_std": 0.5198436122503737, - "nauc_recall_at_20_diff1": 0.21099288999578225, - "nauc_recall_at_20_max": 0.29006909156641014, - "nauc_recall_at_20_std": 0.3272258932695978, - "nauc_recall_at_3_diff1": 0.45237935528383605, - "nauc_recall_at_3_max": 0.5901481467285061, - "nauc_recall_at_3_std": 0.5043744490014948, - "nauc_recall_at_5_diff1": 0.36561326133619537, - "nauc_recall_at_5_max": 0.5235444056882212, - "nauc_recall_at_5_std": 0.47758334451301304, - "ndcg_at_1": 0.03933, - "ndcg_at_10": 0.06583, - "ndcg_at_100": 0.10342, - "ndcg_at_1000": 0.18752, - "ndcg_at_20": 0.07377, - "ndcg_at_3": 0.05362, - "ndcg_at_5": 0.05735, - "precision_at_1": 0.03933, - "precision_at_10": 0.00989, - "precision_at_100": 0.00298, - "precision_at_1000": 0.001, - "precision_at_20": 0.00652, - "precision_at_3": 0.02097, - "precision_at_5": 0.01438, - "recall_at_1": 0.03933, - "recall_at_10": 0.09888, - "recall_at_100": 0.29775, - "recall_at_1000": 1.0, - "recall_at_20": 0.13034, - "recall_at_3": 0.06292, - "recall_at_5": 0.07191 - }, - { - "cv_recall_at_1": 0.02558, - "cv_recall_at_10": 0.07093, - "cv_recall_at_100": 0.28256, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.1, - "cv_recall_at_3": 0.03837, - "cv_recall_at_5": 0.04419, - "hf_subset": "bg", - "languages": [ - "bul-Cyrl" - ], - "main_score": 0.04462, - "map_at_1": 0.02558, - "map_at_10": 0.03675, - "map_at_100": 0.04235, - "map_at_1000": 0.04478, - "map_at_20": 0.03855, - "map_at_3": 0.03178, - "map_at_5": 0.03306, - "mrr_at_1": 0.02558139534883721, - "mrr_at_10": 0.036754798818752314, - "mrr_at_100": 0.0423459890570135, - "mrr_at_1000": 0.044784919706231274, - "mrr_at_20": 0.03855251028621143, - "mrr_at_3": 0.03178294573643411, - "mrr_at_5": 0.03306201550387596, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2277560274329255, - "nauc_cv_recall_at_100_max": 0.3552178996443939, - "nauc_cv_recall_at_100_std": 0.43804062543965594, - "nauc_cv_recall_at_10_diff1": 0.35780107061012467, - "nauc_cv_recall_at_10_max": 0.39030644785284174, - "nauc_cv_recall_at_10_std": 0.5091951950102706, - "nauc_cv_recall_at_1_diff1": 0.7235285508695759, - "nauc_cv_recall_at_1_max": 0.6937824718881873, - "nauc_cv_recall_at_1_std": 0.5971534502524575, - "nauc_cv_recall_at_20_diff1": 0.26377596545634924, - "nauc_cv_recall_at_20_max": 0.37452031373821454, - "nauc_cv_recall_at_20_std": 0.4336176611901837, - "nauc_cv_recall_at_3_diff1": 0.5328845200055291, - "nauc_cv_recall_at_3_max": 0.6126545844817913, - "nauc_cv_recall_at_3_std": 0.5657893667016287, - "nauc_cv_recall_at_5_diff1": 0.4982630307994294, - "nauc_cv_recall_at_5_max": 0.493165096804259, - "nauc_cv_recall_at_5_std": 0.5330447799132926, - "nauc_map_at_1000_diff1": 0.5045425403634579, - "nauc_map_at_1000_max": 0.5414648881911014, - "nauc_map_at_1000_std": 0.5392505548104458, - "nauc_map_at_100_diff1": 0.5054810695846258, - "nauc_map_at_100_max": 0.5437676463628387, - "nauc_map_at_100_std": 0.5382516698611334, - "nauc_map_at_10_diff1": 0.5455350487951657, - "nauc_map_at_10_max": 0.5617908913164679, - "nauc_map_at_10_std": 0.5565122107198375, - "nauc_map_at_1_diff1": 0.7235285508695759, - "nauc_map_at_1_max": 0.6937824718881873, - "nauc_map_at_1_std": 0.5971534502524575, - "nauc_map_at_20_diff1": 0.5221237722219191, - "nauc_map_at_20_max": 0.5511832677721797, - "nauc_map_at_20_std": 0.5425866236786927, - "nauc_map_at_3_diff1": 0.6067588526703618, - "nauc_map_at_3_max": 0.6429417500507175, - "nauc_map_at_3_std": 0.5757638720739231, - "nauc_map_at_5_diff1": 0.5925839482550422, - "nauc_map_at_5_max": 0.6056554448369696, - "nauc_map_at_5_std": 0.5653784005755035, - "nauc_mrr_at_1000_diff1": 0.5045447401485028, - "nauc_mrr_at_1000_max": 0.5414661483330887, - "nauc_mrr_at_1000_std": 0.5392502061789687, - "nauc_mrr_at_100_diff1": 0.5054810695846258, - "nauc_mrr_at_100_max": 0.5437676463628387, - "nauc_mrr_at_100_std": 0.5382516698611334, - "nauc_mrr_at_10_diff1": 0.5455350487951657, - "nauc_mrr_at_10_max": 0.5617908913164679, - "nauc_mrr_at_10_std": 0.5565122107198375, - "nauc_mrr_at_1_diff1": 0.7235285508695759, - "nauc_mrr_at_1_max": 0.6937824718881873, - "nauc_mrr_at_1_std": 0.5971534502524575, - "nauc_mrr_at_20_diff1": 0.5221237722219191, - "nauc_mrr_at_20_max": 0.5511832677721797, - "nauc_mrr_at_20_std": 0.5425866236786927, - "nauc_mrr_at_3_diff1": 0.6067588526703618, - "nauc_mrr_at_3_max": 0.6429417500507175, - "nauc_mrr_at_3_std": 0.5757638720739231, - "nauc_mrr_at_5_diff1": 0.5925839482550422, - "nauc_mrr_at_5_max": 0.6056554448369696, - "nauc_mrr_at_5_std": 0.5653784005755035, - "nauc_ndcg_at_1000_diff1": 0.40681047732259257, - "nauc_ndcg_at_1000_max": 0.4677020272069214, - "nauc_ndcg_at_1000_std": 0.5095760841854711, - "nauc_ndcg_at_100_diff1": 0.362276233440421, - "nauc_ndcg_at_100_max": 0.4464708274109496, - "nauc_ndcg_at_100_std": 0.48668924230421035, - "nauc_ndcg_at_10_diff1": 0.47689523663253697, - "nauc_ndcg_at_10_max": 0.49869013966045334, - "nauc_ndcg_at_10_std": 0.5393206487389861, - "nauc_ndcg_at_1_diff1": 0.7235285508695759, - "nauc_ndcg_at_1_max": 0.6937824718881873, - "nauc_ndcg_at_1_std": 0.5971534502524575, - "nauc_ndcg_at_20_diff1": 0.41674993791448256, - "nauc_ndcg_at_20_max": 0.47634689041203726, - "nauc_ndcg_at_20_std": 0.49999937386166915, - "nauc_ndcg_at_3_diff1": 0.5844922883447777, - "nauc_ndcg_at_3_max": 0.6337591090258609, - "nauc_ndcg_at_3_std": 0.572655587911338, - "nauc_ndcg_at_5_diff1": 0.5629327805950496, - "nauc_ndcg_at_5_max": 0.5720266485267931, - "nauc_ndcg_at_5_std": 0.5555844489319993, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.22775602743292586, - "nauc_precision_at_100_max": 0.3552178996443944, - "nauc_precision_at_100_std": 0.43804062543965655, - "nauc_precision_at_10_diff1": 0.3578010706101251, - "nauc_precision_at_10_max": 0.3903064478528421, - "nauc_precision_at_10_std": 0.5091951950102711, - "nauc_precision_at_1_diff1": 0.7235285508695759, - "nauc_precision_at_1_max": 0.6937824718881873, - "nauc_precision_at_1_std": 0.5971534502524575, - "nauc_precision_at_20_diff1": 0.26377596545634974, - "nauc_precision_at_20_max": 0.3745203137382151, - "nauc_precision_at_20_std": 0.433617661190184, - "nauc_precision_at_3_diff1": 0.5328845200055288, - "nauc_precision_at_3_max": 0.612654584481791, - "nauc_precision_at_3_std": 0.5657893667016284, - "nauc_precision_at_5_diff1": 0.4982630307994295, - "nauc_precision_at_5_max": 0.4931650968042591, - "nauc_precision_at_5_std": 0.5330447799132926, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2277560274329255, - "nauc_recall_at_100_max": 0.3552178996443939, - "nauc_recall_at_100_std": 0.43804062543965594, - "nauc_recall_at_10_diff1": 0.35780107061012467, - "nauc_recall_at_10_max": 0.39030644785284174, - "nauc_recall_at_10_std": 0.5091951950102706, - "nauc_recall_at_1_diff1": 0.7235285508695759, - "nauc_recall_at_1_max": 0.6937824718881873, - "nauc_recall_at_1_std": 0.5971534502524575, - "nauc_recall_at_20_diff1": 0.26377596545634924, - "nauc_recall_at_20_max": 0.37452031373821454, - "nauc_recall_at_20_std": 0.4336176611901837, - "nauc_recall_at_3_diff1": 0.5328845200055291, - "nauc_recall_at_3_max": 0.6126545844817913, - "nauc_recall_at_3_std": 0.5657893667016287, - "nauc_recall_at_5_diff1": 0.4982630307994294, - "nauc_recall_at_5_max": 0.493165096804259, - "nauc_recall_at_5_std": 0.5330447799132926, - "ndcg_at_1": 0.02558, - "ndcg_at_10": 0.04462, - "ndcg_at_100": 0.08375, - "ndcg_at_1000": 0.1694, - "ndcg_at_20": 0.05169, - "ndcg_at_3": 0.0335, - "ndcg_at_5": 0.03585, - "precision_at_1": 0.02558, - "precision_at_10": 0.00709, - "precision_at_100": 0.00283, - "precision_at_1000": 0.001, - "precision_at_20": 0.005, - "precision_at_3": 0.01279, - "precision_at_5": 0.00884, - "recall_at_1": 0.02558, - "recall_at_10": 0.07093, - "recall_at_100": 0.28256, - "recall_at_1000": 1.0, - "recall_at_20": 0.1, - "recall_at_3": 0.03837, - "recall_at_5": 0.04419 - }, - { - "cv_recall_at_1": 0.29068, - "cv_recall_at_10": 0.52189, - "cv_recall_at_100": 0.77441, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.61167, - "cv_recall_at_3": 0.40067, - "cv_recall_at_5": 0.4624, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.39951, - "map_at_1": 0.29068, - "map_at_10": 0.36108, - "map_at_100": 0.37108, - "map_at_1000": 0.37212, - "map_at_20": 0.3673, - "map_at_3": 0.33895, - "map_at_5": 0.3532, - "mrr_at_1": 0.29068462401795736, - "mrr_at_10": 0.36107726292911485, - "mrr_at_100": 0.3710819986806254, - "mrr_at_1000": 0.3721214599242405, - "mrr_at_20": 0.36730148814055474, - "mrr_at_3": 0.33894500561167235, - "mrr_at_5": 0.35319865319865307, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4387577827219246, - "nauc_cv_recall_at_100_max": 0.6008487386838273, - "nauc_cv_recall_at_100_std": 0.533621040315399, - "nauc_cv_recall_at_10_diff1": 0.49950907077991114, - "nauc_cv_recall_at_10_max": 0.539881136077154, - "nauc_cv_recall_at_10_std": 0.3437086563088045, - "nauc_cv_recall_at_1_diff1": 0.7014840285137456, - "nauc_cv_recall_at_1_max": 0.5261619493150632, - "nauc_cv_recall_at_1_std": 0.1801679569228979, - "nauc_cv_recall_at_20_diff1": 0.4645735138658558, - "nauc_cv_recall_at_20_max": 0.576285247683802, - "nauc_cv_recall_at_20_std": 0.45524140316029943, - "nauc_cv_recall_at_3_diff1": 0.5749082348993855, - "nauc_cv_recall_at_3_max": 0.5187893702843904, - "nauc_cv_recall_at_3_std": 0.2655757143100922, - "nauc_cv_recall_at_5_diff1": 0.5501054370979939, - "nauc_cv_recall_at_5_max": 0.5141499178844948, - "nauc_cv_recall_at_5_std": 0.3210478752976409, - "nauc_map_at_1000_diff1": 0.631552054636958, - "nauc_map_at_1000_max": 0.5218272252876711, - "nauc_map_at_1000_std": 0.24075680609784053, - "nauc_map_at_100_diff1": 0.6309965116020844, - "nauc_map_at_100_max": 0.5219786854993752, - "nauc_map_at_100_std": 0.24136011393137455, - "nauc_map_at_10_diff1": 0.6318556518207006, - "nauc_map_at_10_max": 0.5199789751965203, - "nauc_map_at_10_std": 0.23535067632308174, - "nauc_map_at_1_diff1": 0.7014840285137456, - "nauc_map_at_1_max": 0.5261619493150632, - "nauc_map_at_1_std": 0.1801679569228979, - "nauc_map_at_20_diff1": 0.6303065708342569, - "nauc_map_at_20_max": 0.5221526521785125, - "nauc_map_at_20_std": 0.24159207985057737, - "nauc_map_at_3_diff1": 0.6440257440499185, - "nauc_map_at_3_max": 0.5181108294605662, - "nauc_map_at_3_std": 0.22082207870157222, - "nauc_map_at_5_diff1": 0.6384163044898216, - "nauc_map_at_5_max": 0.5165030019975038, - "nauc_map_at_5_std": 0.23301582972395324, - "nauc_mrr_at_1000_diff1": 0.631552054636958, - "nauc_mrr_at_1000_max": 0.5218272252876711, - "nauc_mrr_at_1000_std": 0.24075680609784053, - "nauc_mrr_at_100_diff1": 0.6309965116020844, - "nauc_mrr_at_100_max": 0.5219786854993752, - "nauc_mrr_at_100_std": 0.24136011393137455, - "nauc_mrr_at_10_diff1": 0.6318556518207006, - "nauc_mrr_at_10_max": 0.5199789751965203, - "nauc_mrr_at_10_std": 0.23535067632308174, - "nauc_mrr_at_1_diff1": 0.7014840285137456, - "nauc_mrr_at_1_max": 0.5261619493150632, - "nauc_mrr_at_1_std": 0.1801679569228979, - "nauc_mrr_at_20_diff1": 0.6303065708342569, - "nauc_mrr_at_20_max": 0.5221526521785125, - "nauc_mrr_at_20_std": 0.24159207985057737, - "nauc_mrr_at_3_diff1": 0.6440257440499185, - "nauc_mrr_at_3_max": 0.5181108294605662, - "nauc_mrr_at_3_std": 0.22082207870157222, - "nauc_mrr_at_5_diff1": 0.6384163044898216, - "nauc_mrr_at_5_max": 0.5165030019975038, - "nauc_mrr_at_5_std": 0.23301582972395324, - "nauc_ndcg_at_1000_diff1": 0.6127710409024325, - "nauc_ndcg_at_1000_max": 0.5263563070902447, - "nauc_ndcg_at_1000_std": 0.2637975600245607, - "nauc_ndcg_at_100_diff1": 0.5985083818914605, - "nauc_ndcg_at_100_max": 0.5320434101562074, - "nauc_ndcg_at_100_std": 0.28454891775164165, - "nauc_ndcg_at_10_diff1": 0.6000563468060606, - "nauc_ndcg_at_10_max": 0.5240039855705034, - "nauc_ndcg_at_10_std": 0.2613431364632493, - "nauc_ndcg_at_1_diff1": 0.7014840285137456, - "nauc_ndcg_at_1_max": 0.5261619493150632, - "nauc_ndcg_at_1_std": 0.1801679569228979, - "nauc_ndcg_at_20_diff1": 0.5941994895044833, - "nauc_ndcg_at_20_max": 0.5320728148915203, - "nauc_ndcg_at_20_std": 0.28502456673907356, - "nauc_ndcg_at_3_diff1": 0.6258340313166566, - "nauc_ndcg_at_3_max": 0.5180926268377164, - "nauc_ndcg_at_3_std": 0.23268002915426508, - "nauc_ndcg_at_5_diff1": 0.6157796708166019, - "nauc_ndcg_at_5_max": 0.5156928199908123, - "nauc_ndcg_at_5_std": 0.25510446898549743, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.43875778272192617, - "nauc_precision_at_100_max": 0.6008487386838282, - "nauc_precision_at_100_std": 0.5336210403154, - "nauc_precision_at_10_diff1": 0.49950907077991147, - "nauc_precision_at_10_max": 0.539881136077154, - "nauc_precision_at_10_std": 0.34370865630880415, - "nauc_precision_at_1_diff1": 0.7014840285137456, - "nauc_precision_at_1_max": 0.5261619493150632, - "nauc_precision_at_1_std": 0.1801679569228979, - "nauc_precision_at_20_diff1": 0.4645735138658555, - "nauc_precision_at_20_max": 0.5762852476838015, - "nauc_precision_at_20_std": 0.4552414031602992, - "nauc_precision_at_3_diff1": 0.5749082348993859, - "nauc_precision_at_3_max": 0.5187893702843903, - "nauc_precision_at_3_std": 0.2655757143100918, - "nauc_precision_at_5_diff1": 0.5501054370979936, - "nauc_precision_at_5_max": 0.5141499178844949, - "nauc_precision_at_5_std": 0.3210478752976408, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4387577827219246, - "nauc_recall_at_100_max": 0.6008487386838273, - "nauc_recall_at_100_std": 0.533621040315399, - "nauc_recall_at_10_diff1": 0.49950907077991114, - "nauc_recall_at_10_max": 0.539881136077154, - "nauc_recall_at_10_std": 0.3437086563088045, - "nauc_recall_at_1_diff1": 0.7014840285137456, - "nauc_recall_at_1_max": 0.5261619493150632, - "nauc_recall_at_1_std": 0.1801679569228979, - "nauc_recall_at_20_diff1": 0.4645735138658558, - "nauc_recall_at_20_max": 0.576285247683802, - "nauc_recall_at_20_std": 0.45524140316029943, - "nauc_recall_at_3_diff1": 0.5749082348993855, - "nauc_recall_at_3_max": 0.5187893702843904, - "nauc_recall_at_3_std": 0.2655757143100922, - "nauc_recall_at_5_diff1": 0.5501054370979939, - "nauc_recall_at_5_max": 0.5141499178844948, - "nauc_recall_at_5_std": 0.3210478752976409, - "ndcg_at_1": 0.29068, - "ndcg_at_10": 0.39951, - "ndcg_at_100": 0.45161, - "ndcg_at_1000": 0.48018, - "ndcg_at_20": 0.4222, - "ndcg_at_3": 0.35479, - "ndcg_at_5": 0.38034, - "precision_at_1": 0.29068, - "precision_at_10": 0.05219, - "precision_at_100": 0.00774, - "precision_at_1000": 0.001, - "precision_at_20": 0.03058, - "precision_at_3": 0.13356, - "precision_at_5": 0.09248, - "recall_at_1": 0.29068, - "recall_at_10": 0.52189, - "recall_at_100": 0.77441, - "recall_at_1000": 1.0, - "recall_at_20": 0.61167, - "recall_at_3": 0.40067, - "recall_at_5": 0.4624 - }, - { - "cv_recall_at_1": 0.04211, - "cv_recall_at_10": 0.08596, - "cv_recall_at_100": 0.32982, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.11404, - "cv_recall_at_3": 0.05789, - "cv_recall_at_5": 0.06842, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.0613, - "map_at_1": 0.04211, - "map_at_10": 0.05372, - "map_at_100": 0.0604, - "map_at_1000": 0.06307, - "map_at_20": 0.05556, - "map_at_3": 0.04883, - "map_at_5": 0.05129, - "mrr_at_1": 0.042105263157894736, - "mrr_at_10": 0.053719715956558056, - "mrr_at_100": 0.06039952708701682, - "mrr_at_1000": 0.06306899201660603, - "mrr_at_20": 0.05555911552041582, - "mrr_at_3": 0.04883040935672514, - "mrr_at_5": 0.05128654970760233, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.09380606219165781, - "nauc_cv_recall_at_100_max": 0.15657469352668574, - "nauc_cv_recall_at_100_std": 0.11667009268949023, - "nauc_cv_recall_at_10_diff1": 0.3468949694178718, - "nauc_cv_recall_at_10_max": 0.24647497850763583, - "nauc_cv_recall_at_10_std": 0.029695054839751628, - "nauc_cv_recall_at_1_diff1": 0.8321035685537966, - "nauc_cv_recall_at_1_max": 0.37766076379572494, - "nauc_cv_recall_at_1_std": -0.042046775780341786, - "nauc_cv_recall_at_20_diff1": 0.2297577258533434, - "nauc_cv_recall_at_20_max": 0.18198772477656933, - "nauc_cv_recall_at_20_std": 0.06291375040379016, - "nauc_cv_recall_at_3_diff1": 0.5682041775414459, - "nauc_cv_recall_at_3_max": 0.3961753299021879, - "nauc_cv_recall_at_3_std": 0.05106877739021546, - "nauc_cv_recall_at_5_diff1": 0.46971854726080636, - "nauc_cv_recall_at_5_max": 0.35717185059819867, - "nauc_cv_recall_at_5_std": 0.06804813110152522, - "nauc_map_at_1000_diff1": 0.5748181453726331, - "nauc_map_at_1000_max": 0.32974875316499175, - "nauc_map_at_1000_std": 0.020406796197221914, - "nauc_map_at_100_diff1": 0.5707707528476451, - "nauc_map_at_100_max": 0.32830437398801354, - "nauc_map_at_100_std": 0.021232965580687834, - "nauc_map_at_10_diff1": 0.6244299171747286, - "nauc_map_at_10_max": 0.34840847762878835, - "nauc_map_at_10_std": 0.007780667702480225, - "nauc_map_at_1_diff1": 0.8321035685537966, - "nauc_map_at_1_max": 0.37766076379572494, - "nauc_map_at_1_std": -0.042046775780341786, - "nauc_map_at_20_diff1": 0.601702734154098, - "nauc_map_at_20_max": 0.3375386968560606, - "nauc_map_at_20_std": 0.01285666527928734, - "nauc_map_at_3_diff1": 0.6993102606164114, - "nauc_map_at_3_max": 0.3881081411097846, - "nauc_map_at_3_std": 0.0075432713858572635, - "nauc_map_at_5_diff1": 0.6621187678192878, - "nauc_map_at_5_max": 0.3756754846678021, - "nauc_map_at_5_std": 0.014107047741541544, - "nauc_mrr_at_1000_diff1": 0.5748181453726331, - "nauc_mrr_at_1000_max": 0.32974875316499175, - "nauc_mrr_at_1000_std": 0.020406796197221914, - "nauc_mrr_at_100_diff1": 0.5707707528476451, - "nauc_mrr_at_100_max": 0.32830437398801354, - "nauc_mrr_at_100_std": 0.021232965580687834, - "nauc_mrr_at_10_diff1": 0.6244299171747286, - "nauc_mrr_at_10_max": 0.34840847762878835, - "nauc_mrr_at_10_std": 0.007780667702480225, - "nauc_mrr_at_1_diff1": 0.8321035685537966, - "nauc_mrr_at_1_max": 0.37766076379572494, - "nauc_mrr_at_1_std": -0.042046775780341786, - "nauc_mrr_at_20_diff1": 0.601702734154098, - "nauc_mrr_at_20_max": 0.3375386968560606, - "nauc_mrr_at_20_std": 0.01285666527928734, - "nauc_mrr_at_3_diff1": 0.6993102606164114, - "nauc_mrr_at_3_max": 0.3881081411097846, - "nauc_mrr_at_3_std": 0.0075432713858572635, - "nauc_mrr_at_5_diff1": 0.6621187678192878, - "nauc_mrr_at_5_max": 0.3756754846678021, - "nauc_mrr_at_5_std": 0.014107047741541544, - "nauc_ndcg_at_1000_diff1": 0.45496062445451224, - "nauc_ndcg_at_1000_max": 0.2865060854068369, - "nauc_ndcg_at_1000_std": 0.04176564515853639, - "nauc_ndcg_at_100_diff1": 0.35624810191070644, - "nauc_ndcg_at_100_max": 0.2512968056518267, - "nauc_ndcg_at_100_std": 0.06261282949917163, - "nauc_ndcg_at_10_diff1": 0.5326357528972646, - "nauc_ndcg_at_10_max": 0.3164974797165294, - "nauc_ndcg_at_10_std": 0.01633397875986684, - "nauc_ndcg_at_1_diff1": 0.8321035685537966, - "nauc_ndcg_at_1_max": 0.37766076379572494, - "nauc_ndcg_at_1_std": -0.042046775780341786, - "nauc_ndcg_at_20_diff1": 0.4695413875225568, - "nauc_ndcg_at_20_max": 0.28473639847072607, - "nauc_ndcg_at_20_std": 0.03105480362242873, - "nauc_ndcg_at_3_diff1": 0.6613777696246801, - "nauc_ndcg_at_3_max": 0.39048664131910743, - "nauc_ndcg_at_3_std": 0.020243838343380078, - "nauc_ndcg_at_5_diff1": 0.6033311906370703, - "nauc_ndcg_at_5_max": 0.370390668135692, - "nauc_ndcg_at_5_std": 0.030727359090790975, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.09380606219165794, - "nauc_precision_at_100_max": 0.1565746935266858, - "nauc_precision_at_100_std": 0.11667009268949041, - "nauc_precision_at_10_diff1": 0.346894969417872, - "nauc_precision_at_10_max": 0.24647497850763628, - "nauc_precision_at_10_std": 0.029695054839751957, - "nauc_precision_at_1_diff1": 0.8321035685537966, - "nauc_precision_at_1_max": 0.37766076379572494, - "nauc_precision_at_1_std": -0.042046775780341786, - "nauc_precision_at_20_diff1": 0.2297577258533437, - "nauc_precision_at_20_max": 0.1819877247765697, - "nauc_precision_at_20_std": 0.0629137504037904, - "nauc_precision_at_3_diff1": 0.5682041775414457, - "nauc_precision_at_3_max": 0.3961753299021877, - "nauc_precision_at_3_std": 0.051068777390215286, - "nauc_precision_at_5_diff1": 0.4697185472608067, - "nauc_precision_at_5_max": 0.3571718505981989, - "nauc_precision_at_5_std": 0.06804813110152524, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.09380606219165781, - "nauc_recall_at_100_max": 0.15657469352668574, - "nauc_recall_at_100_std": 0.11667009268949023, - "nauc_recall_at_10_diff1": 0.3468949694178718, - "nauc_recall_at_10_max": 0.24647497850763583, - "nauc_recall_at_10_std": 0.029695054839751628, - "nauc_recall_at_1_diff1": 0.8321035685537966, - "nauc_recall_at_1_max": 0.37766076379572494, - "nauc_recall_at_1_std": -0.042046775780341786, - "nauc_recall_at_20_diff1": 0.2297577258533434, - "nauc_recall_at_20_max": 0.18198772477656933, - "nauc_recall_at_20_std": 0.06291375040379016, - "nauc_recall_at_3_diff1": 0.5682041775414459, - "nauc_recall_at_3_max": 0.3961753299021879, - "nauc_recall_at_3_std": 0.05106877739021546, - "nauc_recall_at_5_diff1": 0.46971854726080636, - "nauc_recall_at_5_max": 0.35717185059819867, - "nauc_recall_at_5_std": 0.06804813110152522, - "ndcg_at_1": 0.04211, - "ndcg_at_10": 0.0613, - "ndcg_at_100": 0.10684, - "ndcg_at_1000": 0.18972, - "ndcg_at_20": 0.06826, - "ndcg_at_3": 0.05115, - "ndcg_at_5": 0.05553, - "precision_at_1": 0.04211, - "precision_at_10": 0.0086, - "precision_at_100": 0.0033, - "precision_at_1000": 0.001, - "precision_at_20": 0.0057, - "precision_at_3": 0.0193, - "precision_at_5": 0.01368, - "recall_at_1": 0.04211, - "recall_at_10": 0.08596, - "recall_at_100": 0.32982, - "recall_at_1000": 1.0, - "recall_at_20": 0.11404, - "recall_at_3": 0.05789, - "recall_at_5": 0.06842 - }, - { - "cv_recall_at_1": 0.14989, - "cv_recall_at_10": 0.31922, - "cv_recall_at_100": 0.58352, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.37643, - "cv_recall_at_3": 0.21053, - "cv_recall_at_5": 0.24256, - "hf_subset": "et", - "languages": [ - "est-Latn" - ], - "main_score": 0.22233, - "map_at_1": 0.14989, - "map_at_10": 0.193, - "map_at_100": 0.20171, - "map_at_1000": 0.20348, - "map_at_20": 0.19694, - "map_at_3": 0.17582, - "map_at_5": 0.18291, - "mrr_at_1": 0.14988558352402745, - "mrr_at_10": 0.1929978932839344, - "mrr_at_100": 0.20170528803138663, - "mrr_at_1000": 0.20348456836129458, - "mrr_at_20": 0.19694152960335984, - "mrr_at_3": 0.17581998474446983, - "mrr_at_5": 0.18291380625476733, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4882250360254275, - "nauc_cv_recall_at_100_max": 0.6143355183282306, - "nauc_cv_recall_at_100_std": 0.4515863451976158, - "nauc_cv_recall_at_10_diff1": 0.5174096468546779, - "nauc_cv_recall_at_10_max": 0.549513338317607, - "nauc_cv_recall_at_10_std": 0.30730731800583605, - "nauc_cv_recall_at_1_diff1": 0.6808299232010442, - "nauc_cv_recall_at_1_max": 0.5687023501836552, - "nauc_cv_recall_at_1_std": 0.1388330409326975, - "nauc_cv_recall_at_20_diff1": 0.5197372472881798, - "nauc_cv_recall_at_20_max": 0.5625030680575044, - "nauc_cv_recall_at_20_std": 0.36834068406172427, - "nauc_cv_recall_at_3_diff1": 0.5134289510178096, - "nauc_cv_recall_at_3_max": 0.49314510685331053, - "nauc_cv_recall_at_3_std": 0.14642685289874202, - "nauc_cv_recall_at_5_diff1": 0.5291372580273346, - "nauc_cv_recall_at_5_max": 0.5050836393241586, - "nauc_cv_recall_at_5_std": 0.18841237458492802, - "nauc_map_at_1000_diff1": 0.6011728516457758, - "nauc_map_at_1000_max": 0.5469848342131521, - "nauc_map_at_1000_std": 0.18180128661413641, - "nauc_map_at_100_diff1": 0.6007245774464908, - "nauc_map_at_100_max": 0.547240345099142, - "nauc_map_at_100_std": 0.18293810395039678, - "nauc_map_at_10_diff1": 0.5995452269902336, - "nauc_map_at_10_max": 0.5435080550775966, - "nauc_map_at_10_std": 0.17191793396850194, - "nauc_map_at_1_diff1": 0.6808299232010442, - "nauc_map_at_1_max": 0.5687023501836552, - "nauc_map_at_1_std": 0.1388330409326975, - "nauc_map_at_20_diff1": 0.5999647161769283, - "nauc_map_at_20_max": 0.5448508731630719, - "nauc_map_at_20_std": 0.1784683306043517, - "nauc_map_at_3_diff1": 0.5989549491315777, - "nauc_map_at_3_max": 0.5323565312763833, - "nauc_map_at_3_std": 0.13864403101476822, - "nauc_map_at_5_diff1": 0.6026925769676897, - "nauc_map_at_5_max": 0.5361290665852161, - "nauc_map_at_5_std": 0.1510969096382901, - "nauc_mrr_at_1000_diff1": 0.6011728516457758, - "nauc_mrr_at_1000_max": 0.5469848342131521, - "nauc_mrr_at_1000_std": 0.18180128661413641, - "nauc_mrr_at_100_diff1": 0.6007245774464908, - "nauc_mrr_at_100_max": 0.547240345099142, - "nauc_mrr_at_100_std": 0.18293810395039678, - "nauc_mrr_at_10_diff1": 0.5995452269902336, - "nauc_mrr_at_10_max": 0.5435080550775966, - "nauc_mrr_at_10_std": 0.17191793396850194, - "nauc_mrr_at_1_diff1": 0.6808299232010442, - "nauc_mrr_at_1_max": 0.5687023501836552, - "nauc_mrr_at_1_std": 0.1388330409326975, - "nauc_mrr_at_20_diff1": 0.5999647161769283, - "nauc_mrr_at_20_max": 0.5448508731630719, - "nauc_mrr_at_20_std": 0.1784683306043517, - "nauc_mrr_at_3_diff1": 0.5989549491315777, - "nauc_mrr_at_3_max": 0.5323565312763833, - "nauc_mrr_at_3_std": 0.13864403101476822, - "nauc_mrr_at_5_diff1": 0.6026925769676897, - "nauc_mrr_at_5_max": 0.5361290665852161, - "nauc_mrr_at_5_std": 0.1510969096382901, - "nauc_ndcg_at_1000_diff1": 0.5858679688974716, - "nauc_ndcg_at_1000_max": 0.5516847995363141, - "nauc_ndcg_at_1000_std": 0.2156924796629923, - "nauc_ndcg_at_100_diff1": 0.5728007301638065, - "nauc_ndcg_at_100_max": 0.5600893766197521, - "nauc_ndcg_at_100_std": 0.24765875357756895, - "nauc_ndcg_at_10_diff1": 0.5747391302933329, - "nauc_ndcg_at_10_max": 0.5438454792535534, - "nauc_ndcg_at_10_std": 0.2084742509312004, - "nauc_ndcg_at_1_diff1": 0.6808299232010442, - "nauc_ndcg_at_1_max": 0.5687023501836552, - "nauc_ndcg_at_1_std": 0.1388330409326975, - "nauc_ndcg_at_20_diff1": 0.5754484900044904, - "nauc_ndcg_at_20_max": 0.5480596643830482, - "nauc_ndcg_at_20_std": 0.22899177987152664, - "nauc_ndcg_at_3_diff1": 0.5748875227084549, - "nauc_ndcg_at_3_max": 0.5213466125963717, - "nauc_ndcg_at_3_std": 0.14067796344801495, - "nauc_ndcg_at_5_diff1": 0.581243380549002, - "nauc_ndcg_at_5_max": 0.5271206231905041, - "nauc_ndcg_at_5_std": 0.16123828946207489, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.488225036025428, - "nauc_precision_at_100_max": 0.6143355183282307, - "nauc_precision_at_100_std": 0.4515863451976166, - "nauc_precision_at_10_diff1": 0.517409646854678, - "nauc_precision_at_10_max": 0.5495133383176073, - "nauc_precision_at_10_std": 0.3073073180058359, - "nauc_precision_at_1_diff1": 0.6808299232010442, - "nauc_precision_at_1_max": 0.5687023501836552, - "nauc_precision_at_1_std": 0.1388330409326975, - "nauc_precision_at_20_diff1": 0.5197372472881798, - "nauc_precision_at_20_max": 0.5625030680575044, - "nauc_precision_at_20_std": 0.36834068406172404, - "nauc_precision_at_3_diff1": 0.5134289510178094, - "nauc_precision_at_3_max": 0.4931451068533103, - "nauc_precision_at_3_std": 0.14642685289874197, - "nauc_precision_at_5_diff1": 0.5291372580273341, - "nauc_precision_at_5_max": 0.5050836393241585, - "nauc_precision_at_5_std": 0.1884123745849278, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4882250360254275, - "nauc_recall_at_100_max": 0.6143355183282306, - "nauc_recall_at_100_std": 0.4515863451976158, - "nauc_recall_at_10_diff1": 0.5174096468546779, - "nauc_recall_at_10_max": 0.549513338317607, - "nauc_recall_at_10_std": 0.30730731800583605, - "nauc_recall_at_1_diff1": 0.6808299232010442, - "nauc_recall_at_1_max": 0.5687023501836552, - "nauc_recall_at_1_std": 0.1388330409326975, - "nauc_recall_at_20_diff1": 0.5197372472881798, - "nauc_recall_at_20_max": 0.5625030680575044, - "nauc_recall_at_20_std": 0.36834068406172427, - "nauc_recall_at_3_diff1": 0.5134289510178096, - "nauc_recall_at_3_max": 0.49314510685331053, - "nauc_recall_at_3_std": 0.14642685289874202, - "nauc_recall_at_5_diff1": 0.5291372580273346, - "nauc_recall_at_5_max": 0.5050836393241586, - "nauc_recall_at_5_std": 0.18841237458492802, - "ndcg_at_1": 0.14989, - "ndcg_at_10": 0.22233, - "ndcg_at_100": 0.27403, - "ndcg_at_1000": 0.32579, - "ndcg_at_20": 0.23674, - "ndcg_at_3": 0.1847, - "ndcg_at_5": 0.1977, - "precision_at_1": 0.14989, - "precision_at_10": 0.03192, - "precision_at_100": 0.00584, - "precision_at_1000": 0.001, - "precision_at_20": 0.01882, - "precision_at_3": 0.07018, - "precision_at_5": 0.04851, - "recall_at_1": 0.14989, - "recall_at_10": 0.31922, - "recall_at_100": 0.58352, - "recall_at_1000": 1.0, - "recall_at_20": 0.37643, - "recall_at_3": 0.21053, - "recall_at_5": 0.24256 - }, - { - "cv_recall_at_1": 0.29412, - "cv_recall_at_10": 0.58713, - "cv_recall_at_100": 0.84351, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.66371, - "cv_recall_at_3": 0.43507, - "cv_recall_at_5": 0.50055, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.43171, - "map_at_1": 0.29412, - "map_at_10": 0.3831, - "map_at_100": 0.39302, - "map_at_1000": 0.39377, - "map_at_20": 0.3884, - "map_at_3": 0.35664, - "map_at_5": 0.37151, - "mrr_at_1": 0.29411764705882354, - "mrr_at_10": 0.38309726406285766, - "mrr_at_100": 0.3930220896763865, - "mrr_at_1000": 0.39377684475076513, - "mrr_at_20": 0.3884071483670995, - "mrr_at_3": 0.3566407695153533, - "mrr_at_5": 0.3715131335553087, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4843906969178279, - "nauc_cv_recall_at_100_max": 0.43792503492328166, - "nauc_cv_recall_at_100_std": 0.39718120153890746, - "nauc_cv_recall_at_10_diff1": 0.512287603657462, - "nauc_cv_recall_at_10_max": 0.3918793835337743, - "nauc_cv_recall_at_10_std": 0.1831530751852317, - "nauc_cv_recall_at_1_diff1": 0.6952445853768835, - "nauc_cv_recall_at_1_max": 0.4071562479248181, - "nauc_cv_recall_at_1_std": 0.0314163160006179, - "nauc_cv_recall_at_20_diff1": 0.505832405815506, - "nauc_cv_recall_at_20_max": 0.4045125508969871, - "nauc_cv_recall_at_20_std": 0.2514925229932094, - "nauc_cv_recall_at_3_diff1": 0.590516922258488, - "nauc_cv_recall_at_3_max": 0.39106348574742505, - "nauc_cv_recall_at_3_std": 0.10230223861586185, - "nauc_cv_recall_at_5_diff1": 0.5608122121866207, - "nauc_cv_recall_at_5_max": 0.38005227936377856, - "nauc_cv_recall_at_5_std": 0.16948919006007676, - "nauc_map_at_1000_diff1": 0.6369696516571739, - "nauc_map_at_1000_max": 0.3998724277397566, - "nauc_map_at_1000_std": 0.08593593767386237, - "nauc_map_at_100_diff1": 0.6366365082322767, - "nauc_map_at_100_max": 0.39986303867167483, - "nauc_map_at_100_std": 0.08651676401231734, - "nauc_map_at_10_diff1": 0.6356948139146141, - "nauc_map_at_10_max": 0.3981409041198343, - "nauc_map_at_10_std": 0.0825337878498932, - "nauc_map_at_1_diff1": 0.6952445853768835, - "nauc_map_at_1_max": 0.4071562479248181, - "nauc_map_at_1_std": 0.0314163160006179, - "nauc_map_at_20_diff1": 0.6361343896322116, - "nauc_map_at_20_max": 0.3990389829224659, - "nauc_map_at_20_std": 0.08591182877461828, - "nauc_map_at_3_diff1": 0.6467022333496839, - "nauc_map_at_3_max": 0.3975578898342642, - "nauc_map_at_3_std": 0.0659588019641901, - "nauc_map_at_5_diff1": 0.6402241056103767, - "nauc_map_at_5_max": 0.39585631490830975, - "nauc_map_at_5_std": 0.08224554824992186, - "nauc_mrr_at_1000_diff1": 0.6369672954076999, - "nauc_mrr_at_1000_max": 0.3998832679414343, - "nauc_mrr_at_1000_std": 0.08592558754217373, - "nauc_mrr_at_100_diff1": 0.6366341543959031, - "nauc_mrr_at_100_max": 0.3998738564086212, - "nauc_mrr_at_100_std": 0.08650643954365476, - "nauc_mrr_at_10_diff1": 0.6356948139146141, - "nauc_mrr_at_10_max": 0.3981409041198343, - "nauc_mrr_at_10_std": 0.0825337878498932, - "nauc_mrr_at_1_diff1": 0.6952445853768835, - "nauc_mrr_at_1_max": 0.4071562479248181, - "nauc_mrr_at_1_std": 0.0314163160006179, - "nauc_mrr_at_20_diff1": 0.6361316614271182, - "nauc_mrr_at_20_max": 0.39904960011386953, - "nauc_mrr_at_20_std": 0.08590071405171072, - "nauc_mrr_at_3_diff1": 0.6467022333496839, - "nauc_mrr_at_3_max": 0.3975578898342642, - "nauc_mrr_at_3_std": 0.0659588019641901, - "nauc_mrr_at_5_diff1": 0.6402241056103767, - "nauc_mrr_at_5_max": 0.39585631490830975, - "nauc_mrr_at_5_std": 0.08224554824992186, - "nauc_ndcg_at_1000_diff1": 0.6227303173383151, - "nauc_ndcg_at_1000_max": 0.4014729497517295, - "nauc_ndcg_at_1000_std": 0.10451577191092994, - "nauc_ndcg_at_100_diff1": 0.6141022926671437, - "nauc_ndcg_at_100_max": 0.4033225984556337, - "nauc_ndcg_at_100_std": 0.12240480538607998, - "nauc_ndcg_at_10_diff1": 0.6076778215477938, - "nauc_ndcg_at_10_max": 0.3965483537094308, - "nauc_ndcg_at_10_std": 0.10579698991001121, - "nauc_ndcg_at_1_diff1": 0.6952445853768835, - "nauc_ndcg_at_1_max": 0.4071562479248181, - "nauc_ndcg_at_1_std": 0.0314163160006179, - "nauc_ndcg_at_20_diff1": 0.6089840297925048, - "nauc_ndcg_at_20_max": 0.39959392769169655, - "nauc_ndcg_at_20_std": 0.11850236148706383, - "nauc_ndcg_at_3_diff1": 0.6319939826876647, - "nauc_ndcg_at_3_max": 0.3957659622947088, - "nauc_ndcg_at_3_std": 0.07554519519230592, - "nauc_ndcg_at_5_diff1": 0.6201611375960527, - "nauc_ndcg_at_5_max": 0.39195675369259475, - "nauc_ndcg_at_5_std": 0.10395844860266833, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.48439069691782943, - "nauc_precision_at_100_max": 0.43792503492328155, - "nauc_precision_at_100_std": 0.39718120153891034, - "nauc_precision_at_10_diff1": 0.5122876036574616, - "nauc_precision_at_10_max": 0.39187938353377433, - "nauc_precision_at_10_std": 0.18315307518523186, - "nauc_precision_at_1_diff1": 0.6952445853768835, - "nauc_precision_at_1_max": 0.4071562479248181, - "nauc_precision_at_1_std": 0.0314163160006179, - "nauc_precision_at_20_diff1": 0.5058324058155061, - "nauc_precision_at_20_max": 0.4045125508969867, - "nauc_precision_at_20_std": 0.2514925229932092, - "nauc_precision_at_3_diff1": 0.5905169222584882, - "nauc_precision_at_3_max": 0.39106348574742467, - "nauc_precision_at_3_std": 0.10230223861586156, - "nauc_precision_at_5_diff1": 0.5608122121866198, - "nauc_precision_at_5_max": 0.380052279363778, - "nauc_precision_at_5_std": 0.16948919006007634, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4843906969178279, - "nauc_recall_at_100_max": 0.43792503492328166, - "nauc_recall_at_100_std": 0.39718120153890746, - "nauc_recall_at_10_diff1": 0.512287603657462, - "nauc_recall_at_10_max": 0.3918793835337743, - "nauc_recall_at_10_std": 0.1831530751852317, - "nauc_recall_at_1_diff1": 0.6952445853768835, - "nauc_recall_at_1_max": 0.4071562479248181, - "nauc_recall_at_1_std": 0.0314163160006179, - "nauc_recall_at_20_diff1": 0.505832405815506, - "nauc_recall_at_20_max": 0.4045125508969871, - "nauc_recall_at_20_std": 0.2514925229932094, - "nauc_recall_at_3_diff1": 0.590516922258488, - "nauc_recall_at_3_max": 0.39106348574742505, - "nauc_recall_at_3_std": 0.10230223861586185, - "nauc_recall_at_5_diff1": 0.5608122121866207, - "nauc_recall_at_5_max": 0.38005227936377856, - "nauc_recall_at_5_std": 0.16948919006007676, - "ndcg_at_1": 0.29412, - "ndcg_at_10": 0.43171, - "ndcg_at_100": 0.48443, - "ndcg_at_1000": 0.50446, - "ndcg_at_20": 0.45106, - "ndcg_at_3": 0.3768, - "ndcg_at_5": 0.40369, - "precision_at_1": 0.29412, - "precision_at_10": 0.05871, - "precision_at_100": 0.00844, - "precision_at_1000": 0.001, - "precision_at_20": 0.03319, - "precision_at_3": 0.14502, - "precision_at_5": 0.10011, - "recall_at_1": 0.29412, - "recall_at_10": 0.58713, - "recall_at_100": 0.84351, - "recall_at_1000": 1.0, - "recall_at_20": 0.66371, - "recall_at_3": 0.43507, - "recall_at_5": 0.50055 - }, - { - "cv_recall_at_1": 0.03222, - "cv_recall_at_10": 0.06767, - "cv_recall_at_100": 0.22879, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.0913, - "cv_recall_at_3": 0.04941, - "cv_recall_at_5": 0.05693, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.04892, - "map_at_1": 0.03222, - "map_at_10": 0.04304, - "map_at_100": 0.04719, - "map_at_1000": 0.04959, - "map_at_20": 0.0446, - "map_at_3": 0.03992, - "map_at_5": 0.04153, - "mrr_at_1": 0.0322234156820623, - "mrr_at_10": 0.043041276661040365, - "mrr_at_100": 0.04718814194903709, - "mrr_at_1000": 0.04959093601368786, - "mrr_at_20": 0.0445958309537952, - "mrr_at_3": 0.03992123165055496, - "mrr_at_5": 0.04153240243465807, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.10562240979373716, - "nauc_cv_recall_at_100_max": 0.2105611626269996, - "nauc_cv_recall_at_100_std": 0.25837760713964875, - "nauc_cv_recall_at_10_diff1": 0.29553317613513297, - "nauc_cv_recall_at_10_max": 0.47472584291680914, - "nauc_cv_recall_at_10_std": 0.4991933445782172, - "nauc_cv_recall_at_1_diff1": 0.6407063004327759, - "nauc_cv_recall_at_1_max": 0.7093671143201729, - "nauc_cv_recall_at_1_std": 0.6154096090896026, - "nauc_cv_recall_at_20_diff1": 0.19368466542337776, - "nauc_cv_recall_at_20_max": 0.36592455501695004, - "nauc_cv_recall_at_20_std": 0.46422209858794283, - "nauc_cv_recall_at_3_diff1": 0.44919576693768404, - "nauc_cv_recall_at_3_max": 0.6201389699342573, - "nauc_cv_recall_at_3_std": 0.5796657253268006, - "nauc_cv_recall_at_5_diff1": 0.37643444575546825, - "nauc_cv_recall_at_5_max": 0.5566219987238744, - "nauc_cv_recall_at_5_std": 0.5375968609046953, - "nauc_map_at_1000_diff1": 0.4518544643890885, - "nauc_map_at_1000_max": 0.5903397782167271, - "nauc_map_at_1000_std": 0.5543340345108702, - "nauc_map_at_100_diff1": 0.4527133546593165, - "nauc_map_at_100_max": 0.5902421317414485, - "nauc_map_at_100_std": 0.5539003872018832, - "nauc_map_at_10_diff1": 0.4873787589645924, - "nauc_map_at_10_max": 0.6303591553353276, - "nauc_map_at_10_std": 0.5814079879764974, - "nauc_map_at_1_diff1": 0.6407063004327759, - "nauc_map_at_1_max": 0.7093671143201729, - "nauc_map_at_1_std": 0.6154096090896026, - "nauc_map_at_20_diff1": 0.4687980014608742, - "nauc_map_at_20_max": 0.6102232719890351, - "nauc_map_at_20_std": 0.574625041111532, - "nauc_map_at_3_diff1": 0.5359507784595627, - "nauc_map_at_3_max": 0.6735009660285682, - "nauc_map_at_3_std": 0.6051326156020952, - "nauc_map_at_5_diff1": 0.5099922656257474, - "nauc_map_at_5_max": 0.6520574953798828, - "nauc_map_at_5_std": 0.5912656863022154, - "nauc_mrr_at_1000_diff1": 0.4518419414636818, - "nauc_mrr_at_1000_max": 0.5903318134461414, - "nauc_mrr_at_1000_std": 0.5543324172527735, - "nauc_mrr_at_100_diff1": 0.4527008589324545, - "nauc_mrr_at_100_max": 0.5902341876107777, - "nauc_mrr_at_100_std": 0.5538987762820369, - "nauc_mrr_at_10_diff1": 0.4873787589645924, - "nauc_mrr_at_10_max": 0.6303591553353276, - "nauc_mrr_at_10_std": 0.5814079879764974, - "nauc_mrr_at_1_diff1": 0.6407063004327759, - "nauc_mrr_at_1_max": 0.7093671143201729, - "nauc_mrr_at_1_std": 0.6154096090896026, - "nauc_mrr_at_20_diff1": 0.4687980014608742, - "nauc_mrr_at_20_max": 0.6102232719890351, - "nauc_mrr_at_20_std": 0.574625041111532, - "nauc_mrr_at_3_diff1": 0.5359507784595627, - "nauc_mrr_at_3_max": 0.6735009660285682, - "nauc_mrr_at_3_std": 0.6051326156020952, - "nauc_mrr_at_5_diff1": 0.5099922656257474, - "nauc_mrr_at_5_max": 0.6520574953798828, - "nauc_mrr_at_5_std": 0.5912656863022154, - "nauc_ndcg_at_1000_diff1": 0.35206988723325433, - "nauc_ndcg_at_1000_max": 0.4933585210026129, - "nauc_ndcg_at_1000_std": 0.487200281897772, - "nauc_ndcg_at_100_diff1": 0.2956713638094448, - "nauc_ndcg_at_100_max": 0.42359186477415095, - "nauc_ndcg_at_100_std": 0.4286119018031178, - "nauc_ndcg_at_10_diff1": 0.4247419124081797, - "nauc_ndcg_at_10_max": 0.5805727332491769, - "nauc_ndcg_at_10_std": 0.555289148310664, - "nauc_ndcg_at_1_diff1": 0.6407063004327759, - "nauc_ndcg_at_1_max": 0.7093671143201729, - "nauc_ndcg_at_1_std": 0.6154096090896026, - "nauc_ndcg_at_20_diff1": 0.3705827146953171, - "nauc_ndcg_at_20_max": 0.5241929646948014, - "nauc_ndcg_at_20_std": 0.5356342066296511, - "nauc_ndcg_at_3_diff1": 0.5099647348339971, - "nauc_ndcg_at_3_max": 0.6580326655970231, - "nauc_ndcg_at_3_std": 0.597873810995354, - "nauc_ndcg_at_5_diff1": 0.46849727115361317, - "nauc_ndcg_at_5_max": 0.6231169257330541, - "nauc_ndcg_at_5_std": 0.5751661711855892, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.10562240979373733, - "nauc_precision_at_100_max": 0.21056116262699995, - "nauc_precision_at_100_std": 0.2583776071396491, - "nauc_precision_at_10_diff1": 0.29553317613513325, - "nauc_precision_at_10_max": 0.47472584291680964, - "nauc_precision_at_10_std": 0.49919334457821773, - "nauc_precision_at_1_diff1": 0.6407063004327759, - "nauc_precision_at_1_max": 0.7093671143201729, - "nauc_precision_at_1_std": 0.6154096090896026, - "nauc_precision_at_20_diff1": 0.1936846654233782, - "nauc_precision_at_20_max": 0.3659245550169505, - "nauc_precision_at_20_std": 0.4642220985879432, - "nauc_precision_at_3_diff1": 0.4491957669376836, - "nauc_precision_at_3_max": 0.6201389699342572, - "nauc_precision_at_3_std": 0.5796657253268003, - "nauc_precision_at_5_diff1": 0.3764344457554687, - "nauc_precision_at_5_max": 0.556621998723875, - "nauc_precision_at_5_std": 0.5375968609046958, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.10562240979373716, - "nauc_recall_at_100_max": 0.2105611626269996, - "nauc_recall_at_100_std": 0.25837760713964875, - "nauc_recall_at_10_diff1": 0.29553317613513297, - "nauc_recall_at_10_max": 0.47472584291680914, - "nauc_recall_at_10_std": 0.4991933445782172, - "nauc_recall_at_1_diff1": 0.6407063004327759, - "nauc_recall_at_1_max": 0.7093671143201729, - "nauc_recall_at_1_std": 0.6154096090896026, - "nauc_recall_at_20_diff1": 0.19368466542337776, - "nauc_recall_at_20_max": 0.36592455501695004, - "nauc_recall_at_20_std": 0.46422209858794283, - "nauc_recall_at_3_diff1": 0.44919576693768404, - "nauc_recall_at_3_max": 0.6201389699342573, - "nauc_recall_at_3_std": 0.5796657253268006, - "nauc_recall_at_5_diff1": 0.37643444575546825, - "nauc_recall_at_5_max": 0.5566219987238744, - "nauc_recall_at_5_std": 0.5375968609046953, - "ndcg_at_1": 0.03222, - "ndcg_at_10": 0.04892, - "ndcg_at_100": 0.0784, - "ndcg_at_1000": 0.16893, - "ndcg_at_20": 0.05478, - "ndcg_at_3": 0.04236, - "ndcg_at_5": 0.04537, - "precision_at_1": 0.03222, - "precision_at_10": 0.00677, - "precision_at_100": 0.00229, - "precision_at_1000": 0.001, - "precision_at_20": 0.00456, - "precision_at_3": 0.01647, - "precision_at_5": 0.01139, - "recall_at_1": 0.03222, - "recall_at_10": 0.06767, - "recall_at_100": 0.22879, - "recall_at_1000": 1.0, - "recall_at_20": 0.0913, - "recall_at_3": 0.04941, - "recall_at_5": 0.05693 - }, - { - "cv_recall_at_1": 0.046, - "cv_recall_at_10": 0.183, - "cv_recall_at_100": 0.468, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.231, - "cv_recall_at_3": 0.088, - "cv_recall_at_5": 0.127, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.10243, - "map_at_1": 0.046, - "map_at_10": 0.07809, - "map_at_100": 0.0867, - "map_at_1000": 0.08868, - "map_at_20": 0.08144, - "map_at_3": 0.06217, - "map_at_5": 0.07087, - "mrr_at_1": 0.046, - "mrr_at_10": 0.07808690476190482, - "mrr_at_100": 0.0867040477420345, - "mrr_at_1000": 0.0886765889783381, - "mrr_at_20": 0.08143502048776828, - "mrr_at_3": 0.06216666666666669, - "mrr_at_5": 0.07086666666666668, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.20749990776317787, - "nauc_cv_recall_at_100_max": 0.436570055183973, - "nauc_cv_recall_at_100_std": 0.41855883918641856, - "nauc_cv_recall_at_10_diff1": 0.21364906028285896, - "nauc_cv_recall_at_10_max": 0.38371942001278025, - "nauc_cv_recall_at_10_std": 0.2998176500231185, - "nauc_cv_recall_at_1_diff1": 0.5176152868025337, - "nauc_cv_recall_at_1_max": 0.5964349459292354, - "nauc_cv_recall_at_1_std": 0.3364344792991215, - "nauc_cv_recall_at_20_diff1": 0.21708481966818757, - "nauc_cv_recall_at_20_max": 0.36739457109057766, - "nauc_cv_recall_at_20_std": 0.29985064332987743, - "nauc_cv_recall_at_3_diff1": 0.3207155401614752, - "nauc_cv_recall_at_3_max": 0.4184940361489865, - "nauc_cv_recall_at_3_std": 0.23899004805229662, - "nauc_cv_recall_at_5_diff1": 0.2426378699275454, - "nauc_cv_recall_at_5_max": 0.41560723773016267, - "nauc_cv_recall_at_5_std": 0.28245722259275363, - "nauc_map_at_1000_diff1": 0.35475021236774645, - "nauc_map_at_1000_max": 0.4769723827541295, - "nauc_map_at_1000_std": 0.306061384977434, - "nauc_map_at_100_diff1": 0.3527062886798938, - "nauc_map_at_100_max": 0.475180844474526, - "nauc_map_at_100_std": 0.3057957381898707, - "nauc_map_at_10_diff1": 0.3553754944785794, - "nauc_map_at_10_max": 0.47785190792936655, - "nauc_map_at_10_std": 0.2996686156665722, - "nauc_map_at_1_diff1": 0.5176152868025337, - "nauc_map_at_1_max": 0.5964349459292354, - "nauc_map_at_1_std": 0.3364344792991215, - "nauc_map_at_20_diff1": 0.35322811248117314, - "nauc_map_at_20_max": 0.47292251789868756, - "nauc_map_at_20_std": 0.2995973460187708, - "nauc_map_at_3_diff1": 0.40677657640678727, - "nauc_map_at_3_max": 0.49858974183751503, - "nauc_map_at_3_std": 0.2837240886193138, - "nauc_map_at_5_diff1": 0.3693536652542208, - "nauc_map_at_5_max": 0.4893966665377987, - "nauc_map_at_5_std": 0.29508609208289105, - "nauc_mrr_at_1000_diff1": 0.35475021236774645, - "nauc_mrr_at_1000_max": 0.4769723827541295, - "nauc_mrr_at_1000_std": 0.306061384977434, - "nauc_mrr_at_100_diff1": 0.3527062886798938, - "nauc_mrr_at_100_max": 0.475180844474526, - "nauc_mrr_at_100_std": 0.3057957381898707, - "nauc_mrr_at_10_diff1": 0.3553754944785794, - "nauc_mrr_at_10_max": 0.47785190792936655, - "nauc_mrr_at_10_std": 0.2996686156665722, - "nauc_mrr_at_1_diff1": 0.5176152868025337, - "nauc_mrr_at_1_max": 0.5964349459292354, - "nauc_mrr_at_1_std": 0.3364344792991215, - "nauc_mrr_at_20_diff1": 0.35322811248117314, - "nauc_mrr_at_20_max": 0.47292251789868756, - "nauc_mrr_at_20_std": 0.2995973460187708, - "nauc_mrr_at_3_diff1": 0.40677657640678727, - "nauc_mrr_at_3_max": 0.49858974183751503, - "nauc_mrr_at_3_std": 0.2837240886193138, - "nauc_mrr_at_5_diff1": 0.3693536652542208, - "nauc_mrr_at_5_max": 0.4893966665377987, - "nauc_mrr_at_5_std": 0.29508609208289105, - "nauc_ndcg_at_1000_diff1": 0.3216112624721145, - "nauc_ndcg_at_1000_max": 0.4624439762191791, - "nauc_ndcg_at_1000_std": 0.3221750936476479, - "nauc_ndcg_at_100_diff1": 0.29198957430111977, - "nauc_ndcg_at_100_max": 0.45015053169593955, - "nauc_ndcg_at_100_std": 0.33914260134993834, - "nauc_ndcg_at_10_diff1": 0.3001737819671239, - "nauc_ndcg_at_10_max": 0.4410967156973652, - "nauc_ndcg_at_10_std": 0.29866148848726565, - "nauc_ndcg_at_1_diff1": 0.5176152868025337, - "nauc_ndcg_at_1_max": 0.5964349459292354, - "nauc_ndcg_at_1_std": 0.3364344792991215, - "nauc_ndcg_at_20_diff1": 0.29716029150230633, - "nauc_ndcg_at_20_max": 0.43019187042701645, - "nauc_ndcg_at_20_std": 0.29866122236808, - "nauc_ndcg_at_3_diff1": 0.3787649140925966, - "nauc_ndcg_at_3_max": 0.472604051756553, - "nauc_ndcg_at_3_std": 0.2692416496275881, - "nauc_ndcg_at_5_diff1": 0.32430704106536345, - "nauc_ndcg_at_5_max": 0.46253507332181965, - "nauc_ndcg_at_5_std": 0.2898110469273884, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.20749990776317812, - "nauc_precision_at_100_max": 0.43657005518397296, - "nauc_precision_at_100_std": 0.41855883918641856, - "nauc_precision_at_10_diff1": 0.21364906028285893, - "nauc_precision_at_10_max": 0.3837194200127799, - "nauc_precision_at_10_std": 0.2998176500231184, - "nauc_precision_at_1_diff1": 0.5176152868025337, - "nauc_precision_at_1_max": 0.5964349459292354, - "nauc_precision_at_1_std": 0.3364344792991215, - "nauc_precision_at_20_diff1": 0.2170848196681876, - "nauc_precision_at_20_max": 0.36739457109057777, - "nauc_precision_at_20_std": 0.29985064332987715, - "nauc_precision_at_3_diff1": 0.3207155401614752, - "nauc_precision_at_3_max": 0.41849403614898656, - "nauc_precision_at_3_std": 0.23899004805229648, - "nauc_precision_at_5_diff1": 0.2426378699275453, - "nauc_precision_at_5_max": 0.41560723773016267, - "nauc_precision_at_5_std": 0.2824572225927536, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.20749990776317787, - "nauc_recall_at_100_max": 0.436570055183973, - "nauc_recall_at_100_std": 0.41855883918641856, - "nauc_recall_at_10_diff1": 0.21364906028285896, - "nauc_recall_at_10_max": 0.38371942001278025, - "nauc_recall_at_10_std": 0.2998176500231185, - "nauc_recall_at_1_diff1": 0.5176152868025337, - "nauc_recall_at_1_max": 0.5964349459292354, - "nauc_recall_at_1_std": 0.3364344792991215, - "nauc_recall_at_20_diff1": 0.21708481966818757, - "nauc_recall_at_20_max": 0.36739457109057766, - "nauc_recall_at_20_std": 0.29985064332987743, - "nauc_recall_at_3_diff1": 0.3207155401614752, - "nauc_recall_at_3_max": 0.4184940361489865, - "nauc_recall_at_3_std": 0.23899004805229662, - "nauc_recall_at_5_diff1": 0.2426378699275454, - "nauc_recall_at_5_max": 0.41560723773016267, - "nauc_recall_at_5_std": 0.28245722259275363, - "ndcg_at_1": 0.046, - "ndcg_at_10": 0.10243, - "ndcg_at_100": 0.15691, - "ndcg_at_1000": 0.22153, - "ndcg_at_20": 0.11458, - "ndcg_at_3": 0.0687, - "ndcg_at_5": 0.08458, - "precision_at_1": 0.046, - "precision_at_10": 0.0183, - "precision_at_100": 0.00468, - "precision_at_1000": 0.001, - "precision_at_20": 0.01155, - "precision_at_3": 0.02933, - "precision_at_5": 0.0254, - "recall_at_1": 0.046, - "recall_at_10": 0.183, - "recall_at_100": 0.468, - "recall_at_1000": 1.0, - "recall_at_20": 0.231, - "recall_at_3": 0.088, - "recall_at_5": 0.127 - }, - { - "cv_recall_at_1": 0.28433, - "cv_recall_at_10": 0.49653, - "cv_recall_at_100": 0.74202, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.55617, - "cv_recall_at_3": 0.38696, - "cv_recall_at_5": 0.42996, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.38454, - "map_at_1": 0.28433, - "map_at_10": 0.34956, - "map_at_100": 0.3582, - "map_at_1000": 0.3594, - "map_at_20": 0.35373, - "map_at_3": 0.33056, - "map_at_5": 0.34068, - "mrr_at_1": 0.2843273231622746, - "mrr_at_10": 0.3496736234947052, - "mrr_at_100": 0.35831473713940976, - "mrr_at_1000": 0.3595171973886647, - "mrr_at_20": 0.3538465479813173, - "mrr_at_3": 0.3310217290799815, - "mrr_at_5": 0.3407998150716597, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.23761896041641875, - "nauc_cv_recall_at_100_max": 0.43470547280407584, - "nauc_cv_recall_at_100_std": 0.4975354980788784, - "nauc_cv_recall_at_10_diff1": 0.4150816022911774, - "nauc_cv_recall_at_10_max": 0.5209102105032801, - "nauc_cv_recall_at_10_std": 0.4249162231453765, - "nauc_cv_recall_at_1_diff1": 0.6280767235329688, - "nauc_cv_recall_at_1_max": 0.5388327846189178, - "nauc_cv_recall_at_1_std": 0.3137589617260528, - "nauc_cv_recall_at_20_diff1": 0.38530925823819695, - "nauc_cv_recall_at_20_max": 0.5028179354666593, - "nauc_cv_recall_at_20_std": 0.4521541760355578, - "nauc_cv_recall_at_3_diff1": 0.4767108005085613, - "nauc_cv_recall_at_3_max": 0.5112648355110674, - "nauc_cv_recall_at_3_std": 0.37223839089566285, - "nauc_cv_recall_at_5_diff1": 0.45511911048701437, - "nauc_cv_recall_at_5_max": 0.5115753542928264, - "nauc_cv_recall_at_5_std": 0.37491215163806885, - "nauc_map_at_1000_diff1": 0.5431805028682132, - "nauc_map_at_1000_max": 0.5260011879130603, - "nauc_map_at_1000_std": 0.3491217816227124, - "nauc_map_at_100_diff1": 0.5423833566859384, - "nauc_map_at_100_max": 0.5257651543217859, - "nauc_map_at_100_std": 0.34955714965846457, - "nauc_map_at_10_diff1": 0.5451356885776881, - "nauc_map_at_10_max": 0.5286083509975998, - "nauc_map_at_10_std": 0.3479502149045429, - "nauc_map_at_1_diff1": 0.6280767235329688, - "nauc_map_at_1_max": 0.5388327846189178, - "nauc_map_at_1_std": 0.3137589617260528, - "nauc_map_at_20_diff1": 0.5435138517432606, - "nauc_map_at_20_max": 0.5272449970946642, - "nauc_map_at_20_std": 0.34943383178671283, - "nauc_map_at_3_diff1": 0.5566611680382575, - "nauc_map_at_3_max": 0.5277712012619854, - "nauc_map_at_3_std": 0.34333294348863985, - "nauc_map_at_5_diff1": 0.5505014537298937, - "nauc_map_at_5_max": 0.527187496253836, - "nauc_map_at_5_std": 0.3417995414399236, - "nauc_mrr_at_1000_diff1": 0.5431073194186685, - "nauc_mrr_at_1000_max": 0.5260286122417211, - "nauc_mrr_at_1000_std": 0.34880756967791593, - "nauc_mrr_at_100_diff1": 0.5423104335156301, - "nauc_mrr_at_100_max": 0.5257925119636787, - "nauc_mrr_at_100_std": 0.3492438326638032, - "nauc_mrr_at_10_diff1": 0.5450633720446688, - "nauc_mrr_at_10_max": 0.5286352598125965, - "nauc_mrr_at_10_std": 0.34764027112219087, - "nauc_mrr_at_1_diff1": 0.6280767235329688, - "nauc_mrr_at_1_max": 0.5388327846189178, - "nauc_mrr_at_1_std": 0.3137589617260528, - "nauc_mrr_at_20_diff1": 0.5434413327263256, - "nauc_mrr_at_20_max": 0.5272720939865494, - "nauc_mrr_at_20_std": 0.3491225367203654, - "nauc_mrr_at_3_diff1": 0.5563704316641994, - "nauc_mrr_at_3_max": 0.5278785986431076, - "nauc_mrr_at_3_std": 0.3420991513795755, - "nauc_mrr_at_5_diff1": 0.5504290598830466, - "nauc_mrr_at_5_max": 0.5272144010756797, - "nauc_mrr_at_5_std": 0.3414909249262113, - "nauc_ndcg_at_1000_diff1": 0.5192096295937189, - "nauc_ndcg_at_1000_max": 0.5201595378932853, - "nauc_ndcg_at_1000_std": 0.36109934162118995, - "nauc_ndcg_at_100_diff1": 0.49485884979265127, - "nauc_ndcg_at_100_max": 0.5128032894094785, - "nauc_ndcg_at_100_std": 0.37306503846461936, - "nauc_ndcg_at_10_diff1": 0.5127616820408724, - "nauc_ndcg_at_10_max": 0.5262872359686687, - "nauc_ndcg_at_10_std": 0.36597912477976347, - "nauc_ndcg_at_1_diff1": 0.6280767235329688, - "nauc_ndcg_at_1_max": 0.5388327846189178, - "nauc_ndcg_at_1_std": 0.3137589617260528, - "nauc_ndcg_at_20_diff1": 0.5067474987898707, - "nauc_ndcg_at_20_max": 0.5217141516107013, - "nauc_ndcg_at_20_std": 0.3717175599106268, - "nauc_ndcg_at_3_diff1": 0.5356901062750342, - "nauc_ndcg_at_3_max": 0.5234068589888287, - "nauc_ndcg_at_3_std": 0.3519483341630906, - "nauc_ndcg_at_5_diff1": 0.5255266240188277, - "nauc_ndcg_at_5_max": 0.5230445458245795, - "nauc_ndcg_at_5_std": 0.35040124016865504, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.23761896041641897, - "nauc_precision_at_100_max": 0.43470547280407584, - "nauc_precision_at_100_std": 0.49753549807887865, - "nauc_precision_at_10_diff1": 0.41508160229117763, - "nauc_precision_at_10_max": 0.5209102105032803, - "nauc_precision_at_10_std": 0.4249162231453769, - "nauc_precision_at_1_diff1": 0.6280767235329688, - "nauc_precision_at_1_max": 0.5388327846189178, - "nauc_precision_at_1_std": 0.3137589617260528, - "nauc_precision_at_20_diff1": 0.3853092582381967, - "nauc_precision_at_20_max": 0.5028179354666594, - "nauc_precision_at_20_std": 0.45215417603555763, - "nauc_precision_at_3_diff1": 0.4775004668645391, - "nauc_precision_at_3_max": 0.5109427978783841, - "nauc_precision_at_3_std": 0.3758320895768744, - "nauc_precision_at_5_diff1": 0.45511911048701426, - "nauc_precision_at_5_max": 0.5115753542928256, - "nauc_precision_at_5_std": 0.3749121516380687, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.23761896041641875, - "nauc_recall_at_100_max": 0.43470547280407584, - "nauc_recall_at_100_std": 0.4975354980788784, - "nauc_recall_at_10_diff1": 0.4150816022911774, - "nauc_recall_at_10_max": 0.5209102105032801, - "nauc_recall_at_10_std": 0.4249162231453765, - "nauc_recall_at_1_diff1": 0.6280767235329688, - "nauc_recall_at_1_max": 0.5388327846189178, - "nauc_recall_at_1_std": 0.3137589617260528, - "nauc_recall_at_20_diff1": 0.38530925823819695, - "nauc_recall_at_20_max": 0.5028179354666593, - "nauc_recall_at_20_std": 0.4521541760355578, - "nauc_recall_at_3_diff1": 0.4775004668645395, - "nauc_recall_at_3_max": 0.5109427978783846, - "nauc_recall_at_3_std": 0.37583208957687475, - "nauc_recall_at_5_diff1": 0.45511911048701437, - "nauc_recall_at_5_max": 0.5115753542928264, - "nauc_recall_at_5_std": 0.37491215163806885, - "ndcg_at_1": 0.28433, - "ndcg_at_10": 0.38454, - "ndcg_at_100": 0.43357, - "ndcg_at_1000": 0.46632, - "ndcg_at_20": 0.39967, - "ndcg_at_3": 0.34476, - "ndcg_at_5": 0.36302, - "precision_at_1": 0.28433, - "precision_at_10": 0.04965, - "precision_at_100": 0.00742, - "precision_at_1000": 0.001, - "precision_at_20": 0.02781, - "precision_at_3": 0.12853, - "precision_at_5": 0.08599, - "recall_at_1": 0.28433, - "recall_at_10": 0.49653, - "recall_at_100": 0.74202, - "recall_at_1000": 1.0, - "recall_at_20": 0.55617, - "recall_at_3": 0.38558, - "recall_at_5": 0.42996 - }, - { - "cv_recall_at_1": 0.21036, - "cv_recall_at_10": 0.41121, - "cv_recall_at_100": 0.66385, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.48626, - "cv_recall_at_3": 0.31818, - "cv_recall_at_5": 0.35729, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.30654, - "map_at_1": 0.2093, - "map_at_10": 0.27352, - "map_at_100": 0.28284, - "map_at_1000": 0.28417, - "map_at_20": 0.27883, - "map_at_3": 0.2574, - "map_at_5": 0.26638, - "mrr_at_1": 0.21035940803382663, - "mrr_at_10": 0.2740461089298299, - "mrr_at_100": 0.2833713578803151, - "mrr_at_1000": 0.2846954029031251, - "mrr_at_20": 0.2793545398253834, - "mrr_at_3": 0.25792811839323476, - "mrr_at_5": 0.2669133192389007, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.08412810930470697, - "nauc_cv_recall_at_100_max": -0.02893784821948192, - "nauc_cv_recall_at_100_std": 0.47651441730821087, - "nauc_cv_recall_at_10_diff1": 0.09489042949352793, - "nauc_cv_recall_at_10_max": 0.04818954906149728, - "nauc_cv_recall_at_10_std": 0.3784429655388849, - "nauc_cv_recall_at_1_diff1": 0.3793938217491008, - "nauc_cv_recall_at_1_max": 0.19745064963008244, - "nauc_cv_recall_at_1_std": 0.2226905726319812, - "nauc_cv_recall_at_20_diff1": -0.012006572717603949, - "nauc_cv_recall_at_20_max": -0.005659777804928359, - "nauc_cv_recall_at_20_std": 0.4477195025551248, - "nauc_cv_recall_at_3_diff1": 0.20273059052660355, - "nauc_cv_recall_at_3_max": 0.10868493007535936, - "nauc_cv_recall_at_3_std": 0.28516507041481937, - "nauc_cv_recall_at_5_diff1": 0.16823261516669927, - "nauc_cv_recall_at_5_max": 0.08532235034723983, - "nauc_cv_recall_at_5_std": 0.3037949441825064, - "nauc_map_at_1000_diff1": 0.2660247127381813, - "nauc_map_at_1000_max": 0.13870549236132962, - "nauc_map_at_1000_std": 0.27802370987169195, - "nauc_map_at_100_diff1": 0.26492589150670853, - "nauc_map_at_100_max": 0.1383701574159222, - "nauc_map_at_100_std": 0.27874235749011944, - "nauc_map_at_10_diff1": 0.2734729386436431, - "nauc_map_at_10_max": 0.1410624776447832, - "nauc_map_at_10_std": 0.2736177637841688, - "nauc_map_at_1_diff1": 0.3837519718710883, - "nauc_map_at_1_max": 0.19829121524445215, - "nauc_map_at_1_std": 0.22626211058018178, - "nauc_map_at_20_diff1": 0.2659316827657333, - "nauc_map_at_20_max": 0.13786512872317508, - "nauc_map_at_20_std": 0.2785818431897626, - "nauc_map_at_3_diff1": 0.2942392073848912, - "nauc_map_at_3_max": 0.1532573637149286, - "nauc_map_at_3_std": 0.25733381482426193, - "nauc_map_at_5_diff1": 0.284716350245029, - "nauc_map_at_5_max": 0.14692619982324098, - "nauc_map_at_5_std": 0.26223728974597177, - "nauc_mrr_at_1000_diff1": 0.26411706149667585, - "nauc_mrr_at_1000_max": 0.13885635065484084, - "nauc_mrr_at_1000_std": 0.2763953765129113, - "nauc_mrr_at_100_diff1": 0.26302522415884505, - "nauc_mrr_at_100_max": 0.13852085679095028, - "nauc_mrr_at_100_std": 0.2771184420870135, - "nauc_mrr_at_10_diff1": 0.2715777825185273, - "nauc_mrr_at_10_max": 0.14121035656644584, - "nauc_mrr_at_10_std": 0.27200881631607793, - "nauc_mrr_at_1_diff1": 0.3793938217491008, - "nauc_mrr_at_1_max": 0.19745064963008244, - "nauc_mrr_at_1_std": 0.2226905726319812, - "nauc_mrr_at_20_diff1": 0.26403967098494374, - "nauc_mrr_at_20_max": 0.1380162722929516, - "nauc_mrr_at_20_std": 0.27696502679263457, - "nauc_mrr_at_3_diff1": 0.292310398478914, - "nauc_mrr_at_3_max": 0.15339498925691814, - "nauc_mrr_at_3_std": 0.2557279132809406, - "nauc_mrr_at_5_diff1": 0.2828056401282569, - "nauc_mrr_at_5_max": 0.14706903825940273, - "nauc_mrr_at_5_std": 0.26063455175574307, - "nauc_ndcg_at_1000_diff1": 0.22736974070140006, - "nauc_ndcg_at_1000_max": 0.11943544918999241, - "nauc_ndcg_at_1000_std": 0.29876642126214314, - "nauc_ndcg_at_100_diff1": 0.19421368450323065, - "nauc_ndcg_at_100_max": 0.1048189143932704, - "nauc_ndcg_at_100_std": 0.3182914348541533, - "nauc_ndcg_at_10_diff1": 0.22660181037495555, - "nauc_ndcg_at_10_max": 0.11665523286258694, - "nauc_ndcg_at_10_std": 0.3001276869199722, - "nauc_ndcg_at_1_diff1": 0.3837519718710883, - "nauc_ndcg_at_1_max": 0.19829121524445215, - "nauc_ndcg_at_1_std": 0.22626211058018178, - "nauc_ndcg_at_20_diff1": 0.19891196886667445, - "nauc_ndcg_at_20_max": 0.10344495204601216, - "nauc_ndcg_at_20_std": 0.3181706232827814, - "nauc_ndcg_at_3_diff1": 0.2690721795734909, - "nauc_ndcg_at_3_max": 0.1409682595554232, - "nauc_ndcg_at_3_std": 0.2650702824215696, - "nauc_ndcg_at_5_diff1": 0.2528960713191041, - "nauc_ndcg_at_5_max": 0.130149050767249, - "nauc_ndcg_at_5_std": 0.2735581019527345, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": -0.0841281093047071, - "nauc_precision_at_100_max": -0.028937848219481813, - "nauc_precision_at_100_std": 0.47651441730821126, - "nauc_precision_at_10_diff1": 0.09489042949352763, - "nauc_precision_at_10_max": 0.04818954906149731, - "nauc_precision_at_10_std": 0.37844296553888496, - "nauc_precision_at_1_diff1": 0.3837519718710883, - "nauc_precision_at_1_max": 0.19829121524445215, - "nauc_precision_at_1_std": 0.22626211058018178, - "nauc_precision_at_20_diff1": -0.012006572717604604, - "nauc_precision_at_20_max": -0.00565977780492901, - "nauc_precision_at_20_std": 0.44771950255512455, - "nauc_precision_at_3_diff1": 0.20273059052660364, - "nauc_precision_at_3_max": 0.10868493007535932, - "nauc_precision_at_3_std": 0.2851650704148193, - "nauc_precision_at_5_diff1": 0.16823261516669877, - "nauc_precision_at_5_max": 0.08532235034723942, - "nauc_precision_at_5_std": 0.30379494418250624, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.08412810930470697, - "nauc_recall_at_100_max": -0.02893784821948192, - "nauc_recall_at_100_std": 0.47651441730821087, - "nauc_recall_at_10_diff1": 0.09489042949352793, - "nauc_recall_at_10_max": 0.04818954906149728, - "nauc_recall_at_10_std": 0.3784429655388849, - "nauc_recall_at_1_diff1": 0.3837519718710883, - "nauc_recall_at_1_max": 0.19829121524445215, - "nauc_recall_at_1_std": 0.22626211058018178, - "nauc_recall_at_20_diff1": -0.012006572717603949, - "nauc_recall_at_20_max": -0.005659777804928359, - "nauc_recall_at_20_std": 0.4477195025551248, - "nauc_recall_at_3_diff1": 0.20273059052660355, - "nauc_recall_at_3_max": 0.10868493007535936, - "nauc_recall_at_3_std": 0.28516507041481937, - "nauc_recall_at_5_diff1": 0.16823261516669927, - "nauc_recall_at_5_max": 0.08532235034723983, - "nauc_recall_at_5_std": 0.3037949441825064, - "ndcg_at_1": 0.2093, - "ndcg_at_10": 0.30654, - "ndcg_at_100": 0.35751, - "ndcg_at_1000": 0.39881, - "ndcg_at_20": 0.32565, - "ndcg_at_3": 0.27302, - "ndcg_at_5": 0.28916, - "precision_at_1": 0.2093, - "precision_at_10": 0.04112, - "precision_at_100": 0.00664, - "precision_at_1000": 0.001, - "precision_at_20": 0.02431, - "precision_at_3": 0.10606, - "precision_at_5": 0.07146, - "recall_at_1": 0.2093, - "recall_at_10": 0.41121, - "recall_at_100": 0.66385, - "recall_at_1000": 1.0, - "recall_at_20": 0.48626, - "recall_at_3": 0.31818, - "recall_at_5": 0.35729 - }, - { - "cv_recall_at_1": 0.594, - "cv_recall_at_10": 0.85, - "cv_recall_at_100": 0.996, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.904, - "cv_recall_at_3": 0.732, - "cv_recall_at_5": 0.789, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.71959, - "map_at_1": 0.594, - "map_at_10": 0.67823, - "map_at_100": 0.68461, - "map_at_1000": 0.68464, - "map_at_20": 0.6821, - "map_at_3": 0.657, - "map_at_5": 0.6702, - "mrr_at_1": 0.594, - "mrr_at_10": 0.6782289682539687, - "mrr_at_100": 0.684613528313254, - "mrr_at_1000": 0.6846440027954259, - "mrr_at_20": 0.6821040953886158, - "mrr_at_3": 0.657, - "mrr_at_5": 0.6702000000000002, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.7537348272642541, - "nauc_cv_recall_at_100_max": 0.967320261437891, - "nauc_cv_recall_at_100_std": 0.9346405228758147, - "nauc_cv_recall_at_10_diff1": 0.8035242434103493, - "nauc_cv_recall_at_10_max": 0.3245916042954772, - "nauc_cv_recall_at_10_std": -0.7203904978848026, - "nauc_cv_recall_at_1_diff1": 0.8321732542062084, - "nauc_cv_recall_at_1_max": 0.4253716822768802, - "nauc_cv_recall_at_1_std": -0.3308525033829497, - "nauc_cv_recall_at_20_diff1": 0.8025550497976981, - "nauc_cv_recall_at_20_max": 0.24774354186118897, - "nauc_cv_recall_at_20_std": -0.7750252878929338, - "nauc_cv_recall_at_3_diff1": 0.7885327415564996, - "nauc_cv_recall_at_3_max": 0.3675701548271744, - "nauc_cv_recall_at_3_std": -0.5087263348602445, - "nauc_cv_recall_at_5_diff1": 0.7794579521198232, - "nauc_cv_recall_at_5_max": 0.3440420490780201, - "nauc_cv_recall_at_5_std": -0.617837797693913, - "nauc_map_at_1000_diff1": 0.8144087540259416, - "nauc_map_at_1000_max": 0.4032590686849337, - "nauc_map_at_1000_std": -0.40569870835620436, - "nauc_map_at_100_diff1": 0.8144007244122586, - "nauc_map_at_100_max": 0.4033259669845395, - "nauc_map_at_100_std": -0.40553996166294154, - "nauc_map_at_10_diff1": 0.8141070805369576, - "nauc_map_at_10_max": 0.40229735186651844, - "nauc_map_at_10_std": -0.4127291380825015, - "nauc_map_at_1_diff1": 0.8321732542062084, - "nauc_map_at_1_max": 0.4253716822768802, - "nauc_map_at_1_std": -0.3308525033829497, - "nauc_map_at_20_diff1": 0.8143371892155288, - "nauc_map_at_20_max": 0.4014071959812905, - "nauc_map_at_20_std": -0.41027294967571937, - "nauc_map_at_3_diff1": 0.8116345917610723, - "nauc_map_at_3_max": 0.40462408831787977, - "nauc_map_at_3_std": -0.3966193654010294, - "nauc_map_at_5_diff1": 0.8111413316486993, - "nauc_map_at_5_max": 0.401958381543243, - "nauc_map_at_5_std": -0.411090564130863, - "nauc_mrr_at_1000_diff1": 0.81440362259585, - "nauc_mrr_at_1000_max": 0.4032546495314979, - "nauc_mrr_at_1000_std": -0.40568753541777147, - "nauc_mrr_at_100_diff1": 0.8143955935310161, - "nauc_mrr_at_100_max": 0.403321548820474, - "nauc_mrr_at_100_std": -0.4055287889306838, - "nauc_mrr_at_10_diff1": 0.8141070805369576, - "nauc_mrr_at_10_max": 0.40229735186651844, - "nauc_mrr_at_10_std": -0.4127291380825015, - "nauc_mrr_at_1_diff1": 0.8321732542062084, - "nauc_mrr_at_1_max": 0.4253716822768802, - "nauc_mrr_at_1_std": -0.3308525033829497, - "nauc_mrr_at_20_diff1": 0.8143321072331481, - "nauc_mrr_at_20_max": 0.40140280700044934, - "nauc_mrr_at_20_std": -0.4102619171951776, - "nauc_mrr_at_3_diff1": 0.8116345917610723, - "nauc_mrr_at_3_max": 0.40462408831787977, - "nauc_mrr_at_3_std": -0.3966193654010294, - "nauc_mrr_at_5_diff1": 0.8111413316486993, - "nauc_mrr_at_5_max": 0.401958381543243, - "nauc_mrr_at_5_std": -0.411090564130863, - "nauc_ndcg_at_1000_diff1": 0.8129387588004101, - "nauc_ndcg_at_1000_max": 0.3989000675515646, - "nauc_ndcg_at_1000_std": -0.41972484200714394, - "nauc_ndcg_at_100_diff1": 0.8127717229000625, - "nauc_ndcg_at_100_max": 0.40046696209809696, - "nauc_ndcg_at_100_std": -0.41599250354747924, - "nauc_ndcg_at_10_diff1": 0.811699922432113, - "nauc_ndcg_at_10_max": 0.3904109417859162, - "nauc_ndcg_at_10_std": -0.4591140511511065, - "nauc_ndcg_at_1_diff1": 0.8321732542062084, - "nauc_ndcg_at_1_max": 0.4253716822768802, - "nauc_ndcg_at_1_std": -0.3308525033829497, - "nauc_ndcg_at_20_diff1": 0.812209351956681, - "nauc_ndcg_at_20_max": 0.38571822662873284, - "nauc_ndcg_at_20_std": -0.45109142559304494, - "nauc_ndcg_at_3_diff1": 0.8062362518327737, - "nauc_ndcg_at_3_max": 0.396305885742386, - "nauc_ndcg_at_3_std": -0.42186974764971025, - "nauc_ndcg_at_5_diff1": 0.8047198234130771, - "nauc_ndcg_at_5_max": 0.390662297645285, - "nauc_ndcg_at_5_std": -0.45119130174831534, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.7537348272642461, - "nauc_precision_at_100_max": 0.9673202614378978, - "nauc_precision_at_100_std": 0.9346405228758466, - "nauc_precision_at_10_diff1": 0.8035242434103476, - "nauc_precision_at_10_max": 0.3245916042954751, - "nauc_precision_at_10_std": -0.7203904978848067, - "nauc_precision_at_1_diff1": 0.8321732542062084, - "nauc_precision_at_1_max": 0.4253716822768802, - "nauc_precision_at_1_std": -0.3308525033829497, - "nauc_precision_at_20_diff1": 0.802555049797697, - "nauc_precision_at_20_max": 0.2477435418611877, - "nauc_precision_at_20_std": -0.7750252878929389, - "nauc_precision_at_3_diff1": 0.7885327415565007, - "nauc_precision_at_3_max": 0.36757015482717503, - "nauc_precision_at_3_std": -0.5087263348602433, - "nauc_precision_at_5_diff1": 0.7794579521198226, - "nauc_precision_at_5_max": 0.3440420490780189, - "nauc_precision_at_5_std": -0.6178377976939148, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.7537348272642541, - "nauc_recall_at_100_max": 0.967320261437891, - "nauc_recall_at_100_std": 0.9346405228758147, - "nauc_recall_at_10_diff1": 0.8035242434103493, - "nauc_recall_at_10_max": 0.3245916042954772, - "nauc_recall_at_10_std": -0.7203904978848026, - "nauc_recall_at_1_diff1": 0.8321732542062084, - "nauc_recall_at_1_max": 0.4253716822768802, - "nauc_recall_at_1_std": -0.3308525033829497, - "nauc_recall_at_20_diff1": 0.8025550497976981, - "nauc_recall_at_20_max": 0.24774354186118897, - "nauc_recall_at_20_std": -0.7750252878929338, - "nauc_recall_at_3_diff1": 0.7885327415564996, - "nauc_recall_at_3_max": 0.3675701548271744, - "nauc_recall_at_3_std": -0.5087263348602445, - "nauc_recall_at_5_diff1": 0.7794579521198232, - "nauc_recall_at_5_max": 0.3440420490780201, - "nauc_recall_at_5_std": -0.617837797693913, - "ndcg_at_1": 0.594, - "ndcg_at_10": 0.71959, - "ndcg_at_100": 0.75084, - "ndcg_at_1000": 0.75141, - "ndcg_at_20": 0.7334, - "ndcg_at_3": 0.67635, - "ndcg_at_5": 0.69998, - "precision_at_1": 0.594, - "precision_at_10": 0.085, - "precision_at_100": 0.00996, - "precision_at_1000": 0.001, - "precision_at_20": 0.0452, - "precision_at_3": 0.244, - "precision_at_5": 0.1578, - "recall_at_1": 0.594, - "recall_at_10": 0.85, - "recall_at_100": 0.996, - "recall_at_1000": 1.0, - "recall_at_20": 0.904, - "recall_at_3": 0.732, - "recall_at_5": 0.789 - } - ] - }, - "task_name": "WITT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2ITRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2ITRetrieval.json deleted file mode 100644 index 4c6ba8a3eb..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2ITRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "53db4c9f9c93cb74926a1c9d04dea7d7acac2f21", - "evaluation_time": 1701.370358467102, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.24052, - "map_at_1": 0.13361, - "map_at_10": 0.19599, - "map_at_100": 0.20547, - "map_at_1000": 0.20624, - "map_at_20": 0.20136, - "map_at_3": 0.17145, - "map_at_5": 0.18571, - "mrr_at_1": 0.16367980884109917, - "mrr_at_10": 0.23035169065634997, - "mrr_at_100": 0.2389849253733174, - "mrr_at_1000": 0.23958364576706842, - "mrr_at_20": 0.2352058042930738, - "mrr_at_3": 0.20483207221558458, - "mrr_at_5": 0.21974644895791826, - "nauc_map_at_1000_diff1": 0.36381862473180293, - "nauc_map_at_1000_max": -0.059600161024810354, - "nauc_map_at_1000_std": 0.13655474912550186, - "nauc_map_at_100_diff1": 0.36367553777720224, - "nauc_map_at_100_max": -0.059461863914186626, - "nauc_map_at_100_std": 0.13590446748202067, - "nauc_map_at_10_diff1": 0.36720760002091996, - "nauc_map_at_10_max": -0.05534352171819153, - "nauc_map_at_10_std": 0.12347691221697427, - "nauc_map_at_1_diff1": 0.45326896661199906, - "nauc_map_at_1_max": -0.02764769512868723, - "nauc_map_at_1_std": 0.0652048653787392, - "nauc_map_at_20_diff1": 0.36423569015898755, - "nauc_map_at_20_max": -0.05757039949977667, - "nauc_map_at_20_std": 0.12988237600501815, - "nauc_map_at_3_diff1": 0.3880170796310493, - "nauc_map_at_3_max": -0.046602902370452, - "nauc_map_at_3_std": 0.09557679001411416, - "nauc_map_at_5_diff1": 0.3737522906429374, - "nauc_map_at_5_max": -0.05285458276228204, - "nauc_map_at_5_std": 0.10793344879301892, - "nauc_mrr_at_1000_diff1": 0.3573332666479495, - "nauc_mrr_at_1000_max": -0.08009727757992803, - "nauc_mrr_at_1000_std": 0.13850009234942326, - "nauc_mrr_at_100_diff1": 0.35730299243761976, - "nauc_mrr_at_100_max": -0.08005647033421684, - "nauc_mrr_at_100_std": 0.13832388992979938, - "nauc_mrr_at_10_diff1": 0.3588306944358961, - "nauc_mrr_at_10_max": -0.0765992981251009, - "nauc_mrr_at_10_std": 0.1311338745029867, - "nauc_mrr_at_1_diff1": 0.4365726498076461, - "nauc_mrr_at_1_max": -0.0487958782703477, - "nauc_mrr_at_1_std": 0.07704304358420916, - "nauc_mrr_at_20_diff1": 0.357610013030439, - "nauc_mrr_at_20_max": -0.07903837125426354, - "nauc_mrr_at_20_std": 0.13393741454859437, - "nauc_mrr_at_3_diff1": 0.37733345474150354, - "nauc_mrr_at_3_max": -0.07021652088221061, - "nauc_mrr_at_3_std": 0.10434506999102883, - "nauc_mrr_at_5_diff1": 0.3642816751404842, - "nauc_mrr_at_5_max": -0.07777365989547423, - "nauc_mrr_at_5_std": 0.1146242099830958, - "nauc_ndcg_at_1000_diff1": 0.3236304186893325, - "nauc_ndcg_at_1000_max": -0.09260496608113483, - "nauc_ndcg_at_1000_std": 0.22257429357826702, - "nauc_ndcg_at_100_diff1": 0.32154613705297064, - "nauc_ndcg_at_100_max": -0.08750441934383102, - "nauc_ndcg_at_100_std": 0.2125151063859874, - "nauc_ndcg_at_10_diff1": 0.33334659337391326, - "nauc_ndcg_at_10_max": -0.07039991614529735, - "nauc_ndcg_at_10_std": 0.16000053754583443, - "nauc_ndcg_at_1_diff1": 0.43899642735143635, - "nauc_ndcg_at_1_max": -0.047721861162555626, - "nauc_ndcg_at_1_std": 0.07437915878107718, - "nauc_ndcg_at_20_diff1": 0.3255145799542806, - "nauc_ndcg_at_20_max": -0.07740934864013048, - "nauc_ndcg_at_20_std": 0.17623988789589784, - "nauc_ndcg_at_3_diff1": 0.3683119776482275, - "nauc_ndcg_at_3_max": -0.058131293476559404, - "nauc_ndcg_at_3_std": 0.10461441678807436, - "nauc_ndcg_at_5_diff1": 0.3469661061624626, - "nauc_ndcg_at_5_max": -0.0682112902662825, - "nauc_ndcg_at_5_std": 0.12503579582342164, - "nauc_precision_at_1000_diff1": 0.0828430329174661, - "nauc_precision_at_1000_max": -0.159449938520161, - "nauc_precision_at_1000_std": 0.4042474323720823, - "nauc_precision_at_100_diff1": 0.1296372825312727, - "nauc_precision_at_100_max": -0.14496151170022903, - "nauc_precision_at_100_std": 0.3906087071646669, - "nauc_precision_at_10_diff1": 0.22409492077465515, - "nauc_precision_at_10_max": -0.10925589146862036, - "nauc_precision_at_10_std": 0.2540176909879479, - "nauc_precision_at_1_diff1": 0.43899642735143635, - "nauc_precision_at_1_max": -0.047721861162555626, - "nauc_precision_at_1_std": 0.07437915878107718, - "nauc_precision_at_20_diff1": 0.18273565087641105, - "nauc_precision_at_20_max": -0.12370069538638741, - "nauc_precision_at_20_std": 0.28690928298508195, - "nauc_precision_at_3_diff1": 0.30938374042197886, - "nauc_precision_at_3_max": -0.08990275443773689, - "nauc_precision_at_3_std": 0.13718837605008682, - "nauc_precision_at_5_diff1": 0.26069624103883665, - "nauc_precision_at_5_max": -0.11012349271655618, - "nauc_precision_at_5_std": 0.17092390048598666, - "nauc_recall_at_1000_diff1": 0.17306295937133223, - "nauc_recall_at_1000_max": -0.2040107326619824, - "nauc_recall_at_1000_std": 0.5680970730286881, - "nauc_recall_at_100_diff1": 0.18974615129600783, - "nauc_recall_at_100_max": -0.14329014714781005, - "nauc_recall_at_100_std": 0.4331187602298659, - "nauc_recall_at_10_diff1": 0.24572008013380212, - "nauc_recall_at_10_max": -0.0817630066289336, - "nauc_recall_at_10_std": 0.23340550568856921, - "nauc_recall_at_1_diff1": 0.45326896661199906, - "nauc_recall_at_1_max": -0.02764769512868723, - "nauc_recall_at_1_std": 0.0652048653787392, - "nauc_recall_at_20_diff1": 0.22026766806324424, - "nauc_recall_at_20_max": -0.10008085428330954, - "nauc_recall_at_20_std": 0.2763531788296686, - "nauc_recall_at_3_diff1": 0.3237039549334668, - "nauc_recall_at_3_max": -0.06308781198222387, - "nauc_recall_at_3_std": 0.11812305027912111, - "nauc_recall_at_5_diff1": 0.27886114226778713, - "nauc_recall_at_5_max": -0.07965627595402587, - "nauc_recall_at_5_std": 0.15601728348224048, - "ndcg_at_1": 0.16328, - "ndcg_at_10": 0.24052, - "ndcg_at_100": 0.28761, - "ndcg_at_1000": 0.30845, - "ndcg_at_20": 0.25886, - "ndcg_at_3": 0.19115, - "ndcg_at_5": 0.21613, - "precision_at_1": 0.16328, - "precision_at_10": 0.04456, - "precision_at_100": 0.00744, - "precision_at_1000": 0.00098, - "precision_at_20": 0.02708, - "precision_at_3": 0.09067, - "precision_at_5": 0.06969, - "recall_at_1": 0.13361, - "recall_at_10": 0.3409, - "recall_at_100": 0.55078, - "recall_at_1000": 0.70291, - "recall_at_20": 0.40721, - "recall_at_3": 0.21545, - "recall_at_5": 0.27141 - } - ] - }, - "task_name": "WebQAT2ITRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2TRetrieval.json deleted file mode 100644 index 58915fa1e1..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/WebQAT2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "468b42a2b2e767d80d2d93f5ae5d42f135a10478", - "evaluation_time": 318.56735610961914, - "kg_co2_emissions": null, - "mteb_version": "1.12.84", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.21064, - "map_at_1": 0.0961, - "map_at_10": 0.15535, - "map_at_100": 0.16454, - "map_at_1000": 0.16555, - "map_at_20": 0.16023, - "map_at_3": 0.13528, - "map_at_5": 0.14522, - "mrr_at_1": 0.19389002036659878, - "mrr_at_10": 0.2656623993793037, - "mrr_at_100": 0.27305022327360007, - "mrr_at_1000": 0.2736181031441951, - "mrr_at_20": 0.26991093217110423, - "mrr_at_3": 0.24372029871011533, - "mrr_at_5": 0.2552274270196877, - "nauc_map_at_1000_diff1": 0.3577072821087506, - "nauc_map_at_1000_max": 0.19011419837219326, - "nauc_map_at_1000_std": 0.1988610404714397, - "nauc_map_at_100_diff1": 0.3578630736764876, - "nauc_map_at_100_max": 0.189817834110483, - "nauc_map_at_100_std": 0.19750527765359563, - "nauc_map_at_10_diff1": 0.35972052505814145, - "nauc_map_at_10_max": 0.18765097035594713, - "nauc_map_at_10_std": 0.18349339512654916, - "nauc_map_at_1_diff1": 0.41104840600884934, - "nauc_map_at_1_max": 0.2132812008152571, - "nauc_map_at_1_std": 0.10415969608061279, - "nauc_map_at_20_diff1": 0.3585489861669943, - "nauc_map_at_20_max": 0.1888402615468673, - "nauc_map_at_20_std": 0.18977266178160795, - "nauc_map_at_3_diff1": 0.36352191826392466, - "nauc_map_at_3_max": 0.20092109988099888, - "nauc_map_at_3_std": 0.15110133509151957, - "nauc_map_at_5_diff1": 0.36223938564701375, - "nauc_map_at_5_max": 0.1942064221941677, - "nauc_map_at_5_std": 0.16623308333665782, - "nauc_mrr_at_1000_diff1": 0.3761532158040219, - "nauc_mrr_at_1000_max": 0.18223027885145637, - "nauc_mrr_at_1000_std": 0.17085813127223065, - "nauc_mrr_at_100_diff1": 0.3761733367847822, - "nauc_mrr_at_100_max": 0.18193717387598093, - "nauc_mrr_at_100_std": 0.17076467545929344, - "nauc_mrr_at_10_diff1": 0.375757427090388, - "nauc_mrr_at_10_max": 0.1803972311772891, - "nauc_mrr_at_10_std": 0.1648971880152693, - "nauc_mrr_at_1_diff1": 0.40955729844120853, - "nauc_mrr_at_1_max": 0.2121400747392042, - "nauc_mrr_at_1_std": 0.10527479271269759, - "nauc_mrr_at_20_diff1": 0.3755902098099904, - "nauc_mrr_at_20_max": 0.18151951134518635, - "nauc_mrr_at_20_std": 0.1678452516111775, - "nauc_mrr_at_3_diff1": 0.38254901314999595, - "nauc_mrr_at_3_max": 0.18245118202851954, - "nauc_mrr_at_3_std": 0.14070604452851126, - "nauc_mrr_at_5_diff1": 0.3795585112521173, - "nauc_mrr_at_5_max": 0.18014590457007112, - "nauc_mrr_at_5_std": 0.15327419153894142, - "nauc_ndcg_at_1000_diff1": 0.34599492975328583, - "nauc_ndcg_at_1000_max": 0.1891205366172212, - "nauc_ndcg_at_1000_std": 0.28791695656426014, - "nauc_ndcg_at_100_diff1": 0.34912910612774883, - "nauc_ndcg_at_100_max": 0.18109853020411557, - "nauc_ndcg_at_100_std": 0.26671467734778753, - "nauc_ndcg_at_10_diff1": 0.3517277732735262, - "nauc_ndcg_at_10_max": 0.17272550706941595, - "nauc_ndcg_at_10_std": 0.21315749304817078, - "nauc_ndcg_at_1_diff1": 0.40955729844120853, - "nauc_ndcg_at_1_max": 0.2121400747392042, - "nauc_ndcg_at_1_std": 0.10527479271269759, - "nauc_ndcg_at_20_diff1": 0.34901296756219047, - "nauc_ndcg_at_20_max": 0.1758313664925407, - "nauc_ndcg_at_20_std": 0.2285421938559212, - "nauc_ndcg_at_3_diff1": 0.3614228822890276, - "nauc_ndcg_at_3_max": 0.1860348188440428, - "nauc_ndcg_at_3_std": 0.15802875724589438, - "nauc_ndcg_at_5_diff1": 0.358612584433648, - "nauc_ndcg_at_5_max": 0.1783684698176462, - "nauc_ndcg_at_5_std": 0.1806675247484184, - "nauc_precision_at_1000_diff1": 0.22394818622056822, - "nauc_precision_at_1000_max": 0.1745345166780263, - "nauc_precision_at_1000_std": 0.5296236186874234, - "nauc_precision_at_100_diff1": 0.2619757063234864, - "nauc_precision_at_100_max": 0.14566238107492874, - "nauc_precision_at_100_std": 0.4219600671015512, - "nauc_precision_at_10_diff1": 0.29206784963413884, - "nauc_precision_at_10_max": 0.1326876541958677, - "nauc_precision_at_10_std": 0.28378504557731654, - "nauc_precision_at_1_diff1": 0.40955729844120853, - "nauc_precision_at_1_max": 0.2121400747392042, - "nauc_precision_at_1_std": 0.10527479271269759, - "nauc_precision_at_20_diff1": 0.2768467143465054, - "nauc_precision_at_20_max": 0.13614135922924672, - "nauc_precision_at_20_std": 0.3133650863423826, - "nauc_precision_at_3_diff1": 0.33410422371740034, - "nauc_precision_at_3_max": 0.16730664435836548, - "nauc_precision_at_3_std": 0.18585733891306555, - "nauc_precision_at_5_diff1": 0.32446847913096094, - "nauc_precision_at_5_max": 0.1506483395267056, - "nauc_precision_at_5_std": 0.22620462778951125, - "nauc_recall_at_1000_diff1": 0.22464019658813997, - "nauc_recall_at_1000_max": 0.1835604946338646, - "nauc_recall_at_1000_std": 0.5264229949341164, - "nauc_recall_at_100_diff1": 0.26141381285720783, - "nauc_recall_at_100_max": 0.14837280463323577, - "nauc_recall_at_100_std": 0.41701657242690526, - "nauc_recall_at_10_diff1": 0.2915143454748924, - "nauc_recall_at_10_max": 0.13415867035178677, - "nauc_recall_at_10_std": 0.28071339987465604, - "nauc_recall_at_1_diff1": 0.41104840600884934, - "nauc_recall_at_1_max": 0.2132812008152571, - "nauc_recall_at_1_std": 0.10415969608061279, - "nauc_recall_at_20_diff1": 0.2757155375999912, - "nauc_recall_at_20_max": 0.13846791944824233, - "nauc_recall_at_20_std": 0.3096311569155981, - "nauc_recall_at_3_diff1": 0.33382352673735427, - "nauc_recall_at_3_max": 0.16883784484659523, - "nauc_recall_at_3_std": 0.1837381252088242, - "nauc_recall_at_5_diff1": 0.3248394719989662, - "nauc_recall_at_5_max": 0.1517889072443636, - "nauc_recall_at_5_std": 0.2240342787050381, - "ndcg_at_1": 0.19389, - "ndcg_at_10": 0.21064, - "ndcg_at_100": 0.25331, - "ndcg_at_1000": 0.27728, - "ndcg_at_20": 0.22613, - "ndcg_at_3": 0.17078, - "ndcg_at_5": 0.18803, - "precision_at_1": 0.19389, - "precision_at_10": 0.05263, - "precision_at_100": 0.00872, - "precision_at_1000": 0.00119, - "precision_at_20": 0.03134, - "precision_at_3": 0.11432, - "precision_at_5": 0.08236, - "recall_at_1": 0.0961, - "recall_at_10": 0.2609, - "recall_at_100": 0.43113, - "recall_at_1000": 0.58955, - "recall_at_20": 0.31039, - "recall_at_3": 0.16999, - "recall_at_5": 0.20418 - } - ] - }, - "task_name": "WebQAT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Winoground.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Winoground.json deleted file mode 100644 index 13d93f9d05..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Winoground.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "521ec2ba6f9a5d7380f7cca5a7b44aea5c1d677c", - "evaluation_time": 46.73370671272278, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "accuracy": 0.08250000327825546, - "hf_subset": "default", - "image_acc": 0.10249999910593033, - "languages": [ - "eng-Latn" - ], - "main_score": 0.08250000327825546, - "text_acc": 0.3075000047683716 - } - ] - }, - "task_name": "Winoground" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XFlickr30kCoT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XFlickr30kCoT2IRetrieval.json deleted file mode 100644 index 81671f5ed3..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XFlickr30kCoT2IRetrieval.json +++ /dev/null @@ -1,1411 +0,0 @@ -{ - "dataset_revision": "0af2c2eba58b27a71898787e286be04befdd7a20", - "evaluation_time": 99.99835991859436, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.0965, - "cv_recall_at_10": 0.3335, - "cv_recall_at_100": 0.645, - "cv_recall_at_1000": 0.935, - "cv_recall_at_20": 0.4295, - "cv_recall_at_3": 0.189, - "cv_recall_at_5": 0.2405, - "hf_subset": "de", - "languages": [ - "deu-Latn" - ], - "main_score": 0.20111, - "map_at_1": 0.0965, - "map_at_10": 0.16057, - "map_at_100": 0.17239, - "map_at_1000": 0.1735, - "map_at_20": 0.1672, - "map_at_3": 0.13667, - "map_at_5": 0.14822, - "mrr_at_1": 0.0965, - "mrr_at_10": 0.1605702380952379, - "mrr_at_100": 0.1723897851951646, - "mrr_at_1000": 0.17350047917826844, - "mrr_at_20": 0.1672007281246442, - "mrr_at_3": 0.13666666666666671, - "mrr_at_5": 0.14821666666666652, - "nauc_cv_recall_at_1000_diff1": -0.13237807943690208, - "nauc_cv_recall_at_1000_max": 0.6101845866551727, - "nauc_cv_recall_at_1000_std": 0.7255440637793579, - "nauc_cv_recall_at_100_diff1": -0.08269381110117098, - "nauc_cv_recall_at_100_max": 0.5258267043851802, - "nauc_cv_recall_at_100_std": 0.6066611861988931, - "nauc_cv_recall_at_10_diff1": 0.05244456199507041, - "nauc_cv_recall_at_10_max": 0.40815755564401374, - "nauc_cv_recall_at_10_std": 0.3812155724158006, - "nauc_cv_recall_at_1_diff1": 0.23466483899175744, - "nauc_cv_recall_at_1_max": 0.38016379563220937, - "nauc_cv_recall_at_1_std": 0.23962792171953437, - "nauc_cv_recall_at_20_diff1": 0.02385501952755473, - "nauc_cv_recall_at_20_max": 0.4362241829602174, - "nauc_cv_recall_at_20_std": 0.4356448904252889, - "nauc_cv_recall_at_3_diff1": 0.12159462941294655, - "nauc_cv_recall_at_3_max": 0.3403158731847171, - "nauc_cv_recall_at_3_std": 0.26052530716928135, - "nauc_cv_recall_at_5_diff1": 0.09419131251931254, - "nauc_cv_recall_at_5_max": 0.3499742250107925, - "nauc_cv_recall_at_5_std": 0.312421300241589, - "nauc_map_at_1000_diff1": 0.1461081483396935, - "nauc_map_at_1000_max": 0.37703355044770975, - "nauc_map_at_1000_std": 0.29448647756773627, - "nauc_map_at_100_diff1": 0.14547433469922097, - "nauc_map_at_100_max": 0.3772992484744435, - "nauc_map_at_100_std": 0.29507724913928524, - "nauc_map_at_10_diff1": 0.1518516384994729, - "nauc_map_at_10_max": 0.37112018036702726, - "nauc_map_at_10_std": 0.28302886799037996, - "nauc_map_at_1_diff1": 0.23466483899175744, - "nauc_map_at_1_max": 0.38016379563220937, - "nauc_map_at_1_std": 0.23962792171953437, - "nauc_map_at_20_diff1": 0.14803098009106574, - "nauc_map_at_20_max": 0.37413032174506977, - "nauc_map_at_20_std": 0.2893847241629189, - "nauc_map_at_3_diff1": 0.1730919004523251, - "nauc_map_at_3_max": 0.35789019817695994, - "nauc_map_at_3_std": 0.2519022240220413, - "nauc_map_at_5_diff1": 0.16283563503461893, - "nauc_map_at_5_max": 0.3603487503123098, - "nauc_map_at_5_std": 0.26862429359407647, - "nauc_mrr_at_1000_diff1": 0.1461081483396935, - "nauc_mrr_at_1000_max": 0.37703355044770975, - "nauc_mrr_at_1000_std": 0.29448647756773627, - "nauc_mrr_at_100_diff1": 0.14547433469922097, - "nauc_mrr_at_100_max": 0.3772992484744435, - "nauc_mrr_at_100_std": 0.29507724913928524, - "nauc_mrr_at_10_diff1": 0.1518516384994729, - "nauc_mrr_at_10_max": 0.37112018036702726, - "nauc_mrr_at_10_std": 0.28302886799037996, - "nauc_mrr_at_1_diff1": 0.23466483899175744, - "nauc_mrr_at_1_max": 0.38016379563220937, - "nauc_mrr_at_1_std": 0.23962792171953437, - "nauc_mrr_at_20_diff1": 0.14803098009106574, - "nauc_mrr_at_20_max": 0.37413032174506977, - "nauc_mrr_at_20_std": 0.2893847241629189, - "nauc_mrr_at_3_diff1": 0.1730919004523251, - "nauc_mrr_at_3_max": 0.35789019817695994, - "nauc_mrr_at_3_std": 0.2519022240220413, - "nauc_mrr_at_5_diff1": 0.16283563503461893, - "nauc_mrr_at_5_max": 0.3603487503123098, - "nauc_mrr_at_5_std": 0.26862429359407647, - "nauc_ndcg_at_1000_diff1": 0.1086985272853948, - "nauc_ndcg_at_1000_max": 0.39926536348764186, - "nauc_ndcg_at_1000_std": 0.3436142168990749, - "nauc_ndcg_at_100_diff1": 0.0903687422592335, - "nauc_ndcg_at_100_max": 0.40946033268315823, - "nauc_ndcg_at_100_std": 0.36639710955282306, - "nauc_ndcg_at_10_diff1": 0.11977407519483287, - "nauc_ndcg_at_10_max": 0.3812462038075193, - "nauc_ndcg_at_10_std": 0.31303685757698146, - "nauc_ndcg_at_1_diff1": 0.23466483899175744, - "nauc_ndcg_at_1_max": 0.38016379563220937, - "nauc_ndcg_at_1_std": 0.23962792171953437, - "nauc_ndcg_at_20_diff1": 0.10880436162955412, - "nauc_ndcg_at_20_max": 0.3910719259043727, - "nauc_ndcg_at_20_std": 0.33271214860697695, - "nauc_ndcg_at_3_diff1": 0.15734556970000205, - "nauc_ndcg_at_3_max": 0.3524947490091569, - "nauc_ndcg_at_3_std": 0.2545727797832577, - "nauc_ndcg_at_5_diff1": 0.14116730424839466, - "nauc_ndcg_at_5_max": 0.35683485916594976, - "nauc_ndcg_at_5_std": 0.281957030568669, - "nauc_precision_at_1000_diff1": -0.1323780794369055, - "nauc_precision_at_1000_max": 0.6101845866551743, - "nauc_precision_at_1000_std": 0.725544063779355, - "nauc_precision_at_100_diff1": -0.08269381110117134, - "nauc_precision_at_100_max": 0.5258267043851799, - "nauc_precision_at_100_std": 0.6066611861988925, - "nauc_precision_at_10_diff1": 0.05244456199507014, - "nauc_precision_at_10_max": 0.4081575556440135, - "nauc_precision_at_10_std": 0.3812155724158001, - "nauc_precision_at_1_diff1": 0.23466483899175744, - "nauc_precision_at_1_max": 0.38016379563220937, - "nauc_precision_at_1_std": 0.23962792171953437, - "nauc_precision_at_20_diff1": 0.023855019527554727, - "nauc_precision_at_20_max": 0.4362241829602172, - "nauc_precision_at_20_std": 0.4356448904252892, - "nauc_precision_at_3_diff1": 0.12159462941294678, - "nauc_precision_at_3_max": 0.34031587318471695, - "nauc_precision_at_3_std": 0.26052530716928124, - "nauc_precision_at_5_diff1": 0.09419131251931243, - "nauc_precision_at_5_max": 0.34997422501079256, - "nauc_precision_at_5_std": 0.31242130024158893, - "nauc_recall_at_1000_diff1": -0.13237807943690208, - "nauc_recall_at_1000_max": 0.6101845866551727, - "nauc_recall_at_1000_std": 0.7255440637793579, - "nauc_recall_at_100_diff1": -0.08269381110117098, - "nauc_recall_at_100_max": 0.5258267043851802, - "nauc_recall_at_100_std": 0.6066611861988931, - "nauc_recall_at_10_diff1": 0.05244456199507041, - "nauc_recall_at_10_max": 0.40815755564401374, - "nauc_recall_at_10_std": 0.3812155724158006, - "nauc_recall_at_1_diff1": 0.23466483899175744, - "nauc_recall_at_1_max": 0.38016379563220937, - "nauc_recall_at_1_std": 0.23962792171953437, - "nauc_recall_at_20_diff1": 0.02385501952755473, - "nauc_recall_at_20_max": 0.4362241829602174, - "nauc_recall_at_20_std": 0.4356448904252889, - "nauc_recall_at_3_diff1": 0.12159462941294655, - "nauc_recall_at_3_max": 0.3403158731847171, - "nauc_recall_at_3_std": 0.26052530716928135, - "nauc_recall_at_5_diff1": 0.09419131251931254, - "nauc_recall_at_5_max": 0.3499742250107925, - "nauc_recall_at_5_std": 0.312421300241589, - "ndcg_at_1": 0.0965, - "ndcg_at_10": 0.20111, - "ndcg_at_100": 0.2646, - "ndcg_at_1000": 0.29981, - "ndcg_at_20": 0.22534, - "ndcg_at_3": 0.15008, - "ndcg_at_5": 0.1711, - "precision_at_1": 0.0965, - "precision_at_10": 0.03335, - "precision_at_100": 0.00645, - "precision_at_1000": 0.00093, - "precision_at_20": 0.02147, - "precision_at_3": 0.063, - "precision_at_5": 0.0481, - "recall_at_1": 0.0965, - "recall_at_10": 0.3335, - "recall_at_100": 0.645, - "recall_at_1000": 0.935, - "recall_at_20": 0.4295, - "recall_at_3": 0.189, - "recall_at_5": 0.2405 - }, - { - "cv_recall_at_1": 0.5075, - "cv_recall_at_10": 0.8545, - "cv_recall_at_100": 0.9855, - "cv_recall_at_1000": 0.998, - "cv_recall_at_20": 0.9225, - "cv_recall_at_3": 0.6945, - "cv_recall_at_5": 0.756, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6747, - "map_at_1": 0.5075, - "map_at_10": 0.61801, - "map_at_100": 0.6245, - "map_at_1000": 0.62457, - "map_at_20": 0.62272, - "map_at_3": 0.59067, - "map_at_5": 0.60464, - "mrr_at_1": 0.5075, - "mrr_at_10": 0.6180101190476197, - "mrr_at_100": 0.6244980889716759, - "mrr_at_1000": 0.6245731391573968, - "mrr_at_20": 0.622716377877369, - "mrr_at_3": 0.5906666666666675, - "mrr_at_5": 0.604641666666668, - "nauc_cv_recall_at_1000_diff1": 0.6095938375349927, - "nauc_cv_recall_at_1000_max": 0.967320261437891, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": 0.4820824881676851, - "nauc_cv_recall_at_100_max": 0.5075823432821354, - "nauc_cv_recall_at_100_std": 0.3155285102546801, - "nauc_cv_recall_at_10_diff1": 0.4956058624184979, - "nauc_cv_recall_at_10_max": 0.47403160032249275, - "nauc_cv_recall_at_10_std": 0.07836853215398068, - "nauc_cv_recall_at_1_diff1": 0.6850281471236896, - "nauc_cv_recall_at_1_max": 0.44724621432096096, - "nauc_cv_recall_at_1_std": 0.029021916824158608, - "nauc_cv_recall_at_20_diff1": 0.46023011355079824, - "nauc_cv_recall_at_20_max": 0.46001927652781566, - "nauc_cv_recall_at_20_std": 0.14276678413300842, - "nauc_cv_recall_at_3_diff1": 0.5766142565770788, - "nauc_cv_recall_at_3_max": 0.46684329587974965, - "nauc_cv_recall_at_3_std": 0.046707921695569345, - "nauc_cv_recall_at_5_diff1": 0.5441108037261881, - "nauc_cv_recall_at_5_max": 0.4786111324572869, - "nauc_cv_recall_at_5_std": 0.06760226952534688, - "nauc_map_at_1000_diff1": 0.6401875478503286, - "nauc_map_at_1000_max": 0.45045885067932817, - "nauc_map_at_1000_std": 0.03892977625614155, - "nauc_map_at_100_diff1": 0.6401443049412445, - "nauc_map_at_100_max": 0.4504404663927708, - "nauc_map_at_100_std": 0.03894741190570511, - "nauc_map_at_10_diff1": 0.6377847159976536, - "nauc_map_at_10_max": 0.4508785914698639, - "nauc_map_at_10_std": 0.03819626760294528, - "nauc_map_at_1_diff1": 0.6850281471236896, - "nauc_map_at_1_max": 0.44724621432096096, - "nauc_map_at_1_std": 0.029021916824158608, - "nauc_map_at_20_diff1": 0.6390353557528677, - "nauc_map_at_20_max": 0.45026491456488166, - "nauc_map_at_20_std": 0.039003664779637244, - "nauc_map_at_3_diff1": 0.6409558441584462, - "nauc_map_at_3_max": 0.4510104885839783, - "nauc_map_at_3_std": 0.035494081481255046, - "nauc_map_at_5_diff1": 0.6377571454432578, - "nauc_map_at_5_max": 0.45223368623817506, - "nauc_map_at_5_std": 0.03851371505521922, - "nauc_mrr_at_1000_diff1": 0.6401875478503286, - "nauc_mrr_at_1000_max": 0.45045885067932817, - "nauc_mrr_at_1000_std": 0.03892977625614155, - "nauc_mrr_at_100_diff1": 0.6401443049412445, - "nauc_mrr_at_100_max": 0.4504404663927708, - "nauc_mrr_at_100_std": 0.03894741190570511, - "nauc_mrr_at_10_diff1": 0.6377847159976536, - "nauc_mrr_at_10_max": 0.4508785914698639, - "nauc_mrr_at_10_std": 0.03819626760294528, - "nauc_mrr_at_1_diff1": 0.6850281471236896, - "nauc_mrr_at_1_max": 0.44724621432096096, - "nauc_mrr_at_1_std": 0.029021916824158608, - "nauc_mrr_at_20_diff1": 0.6390353557528677, - "nauc_mrr_at_20_max": 0.45026491456488166, - "nauc_mrr_at_20_std": 0.039003664779637244, - "nauc_mrr_at_3_diff1": 0.6409558441584462, - "nauc_mrr_at_3_max": 0.4510104885839783, - "nauc_mrr_at_3_std": 0.035494081481255046, - "nauc_mrr_at_5_diff1": 0.6377571454432578, - "nauc_mrr_at_5_max": 0.45223368623817506, - "nauc_mrr_at_5_std": 0.03851371505521922, - "nauc_ndcg_at_1000_diff1": 0.6324611400318849, - "nauc_ndcg_at_1000_max": 0.45241723555814617, - "nauc_ndcg_at_1000_std": 0.0438352694520655, - "nauc_ndcg_at_100_diff1": 0.6312415529234053, - "nauc_ndcg_at_100_max": 0.452206191978489, - "nauc_ndcg_at_100_std": 0.04483598941004991, - "nauc_ndcg_at_10_diff1": 0.6176965168162838, - "nauc_ndcg_at_10_max": 0.45412985687060525, - "nauc_ndcg_at_10_std": 0.04369653927305166, - "nauc_ndcg_at_1_diff1": 0.6850281471236896, - "nauc_ndcg_at_1_max": 0.44724621432096096, - "nauc_ndcg_at_1_std": 0.029021916824158608, - "nauc_ndcg_at_20_diff1": 0.6224722379002166, - "nauc_ndcg_at_20_max": 0.45166656786354653, - "nauc_ndcg_at_20_std": 0.04699416807281372, - "nauc_ndcg_at_3_diff1": 0.6262669267161668, - "nauc_ndcg_at_3_max": 0.45442926321108096, - "nauc_ndcg_at_3_std": 0.03802072419464461, - "nauc_ndcg_at_5_diff1": 0.6190724921092656, - "nauc_ndcg_at_5_max": 0.4572192055170717, - "nauc_ndcg_at_5_std": 0.044100513069060335, - "nauc_precision_at_1000_diff1": 0.6095938375350154, - "nauc_precision_at_1000_max": 0.9673202614379109, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.48208248816768146, - "nauc_precision_at_100_max": 0.5075823432821327, - "nauc_precision_at_100_std": 0.3155285102546789, - "nauc_precision_at_10_diff1": 0.49560586241849763, - "nauc_precision_at_10_max": 0.47403160032249225, - "nauc_precision_at_10_std": 0.07836853215397865, - "nauc_precision_at_1_diff1": 0.6850281471236896, - "nauc_precision_at_1_max": 0.44724621432096096, - "nauc_precision_at_1_std": 0.029021916824158608, - "nauc_precision_at_20_diff1": 0.4602301135507949, - "nauc_precision_at_20_max": 0.46001927652781444, - "nauc_precision_at_20_std": 0.14276678413300534, - "nauc_precision_at_3_diff1": 0.5766142565770792, - "nauc_precision_at_3_max": 0.4668432958797501, - "nauc_precision_at_3_std": 0.04670792169556997, - "nauc_precision_at_5_diff1": 0.544110803726187, - "nauc_precision_at_5_max": 0.4786111324572858, - "nauc_precision_at_5_std": 0.06760226952534575, - "nauc_recall_at_1000_diff1": 0.6095938375349927, - "nauc_recall_at_1000_max": 0.967320261437891, - "nauc_recall_at_1000_std": 1.0, - "nauc_recall_at_100_diff1": 0.4820824881676851, - "nauc_recall_at_100_max": 0.5075823432821354, - "nauc_recall_at_100_std": 0.3155285102546801, - "nauc_recall_at_10_diff1": 0.4956058624184979, - "nauc_recall_at_10_max": 0.47403160032249275, - "nauc_recall_at_10_std": 0.07836853215398068, - "nauc_recall_at_1_diff1": 0.6850281471236896, - "nauc_recall_at_1_max": 0.44724621432096096, - "nauc_recall_at_1_std": 0.029021916824158608, - "nauc_recall_at_20_diff1": 0.46023011355079824, - "nauc_recall_at_20_max": 0.46001927652781566, - "nauc_recall_at_20_std": 0.14276678413300842, - "nauc_recall_at_3_diff1": 0.5766142565770788, - "nauc_recall_at_3_max": 0.46684329587974965, - "nauc_recall_at_3_std": 0.046707921695569345, - "nauc_recall_at_5_diff1": 0.5441108037261881, - "nauc_recall_at_5_max": 0.4786111324572869, - "nauc_recall_at_5_std": 0.06760226952534688, - "ndcg_at_1": 0.5075, - "ndcg_at_10": 0.6747, - "ndcg_at_100": 0.70392, - "ndcg_at_1000": 0.70559, - "ndcg_at_20": 0.69188, - "ndcg_at_3": 0.61737, - "ndcg_at_5": 0.64263, - "precision_at_1": 0.5075, - "precision_at_10": 0.08545, - "precision_at_100": 0.00986, - "precision_at_1000": 0.001, - "precision_at_20": 0.04612, - "precision_at_3": 0.2315, - "precision_at_5": 0.1512, - "recall_at_1": 0.5075, - "recall_at_10": 0.8545, - "recall_at_100": 0.9855, - "recall_at_1000": 0.998, - "recall_at_20": 0.9225, - "recall_at_3": 0.6945, - "recall_at_5": 0.756 - }, - { - "cv_recall_at_1": 0.178, - "cv_recall_at_10": 0.499, - "cv_recall_at_100": 0.806, - "cv_recall_at_1000": 0.9815, - "cv_recall_at_20": 0.6035, - "cv_recall_at_3": 0.3085, - "cv_recall_at_5": 0.386, - "hf_subset": "es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.32156, - "map_at_1": 0.178, - "map_at_10": 0.26683, - "map_at_100": 0.27926, - "map_at_1000": 0.28003, - "map_at_20": 0.27412, - "map_at_3": 0.23392, - "map_at_5": 0.25179, - "mrr_at_1": 0.178, - "mrr_at_10": 0.2668319444444442, - "mrr_at_100": 0.27925731838360435, - "mrr_at_1000": 0.28003361566253326, - "mrr_at_20": 0.2741153201610047, - "mrr_at_3": 0.2339166666666664, - "mrr_at_5": 0.2517916666666662, - "nauc_cv_recall_at_1000_diff1": 0.13902137431548597, - "nauc_cv_recall_at_1000_max": 0.6893027481262831, - "nauc_cv_recall_at_1000_std": 0.8832235596941536, - "nauc_cv_recall_at_100_diff1": 0.17696568592187603, - "nauc_cv_recall_at_100_max": 0.4787900664905917, - "nauc_cv_recall_at_100_std": 0.5594677155805847, - "nauc_cv_recall_at_10_diff1": 0.20089162625222612, - "nauc_cv_recall_at_10_max": 0.35988054818897564, - "nauc_cv_recall_at_10_std": 0.308875104393797, - "nauc_cv_recall_at_1_diff1": 0.39869161921412205, - "nauc_cv_recall_at_1_max": 0.3995277159429934, - "nauc_cv_recall_at_1_std": 0.19832125932977787, - "nauc_cv_recall_at_20_diff1": 0.20381530227560704, - "nauc_cv_recall_at_20_max": 0.39670392767516544, - "nauc_cv_recall_at_20_std": 0.36613850164004713, - "nauc_cv_recall_at_3_diff1": 0.25113374985451803, - "nauc_cv_recall_at_3_max": 0.3840655418755832, - "nauc_cv_recall_at_3_std": 0.25083235840824936, - "nauc_cv_recall_at_5_diff1": 0.2354295015175038, - "nauc_cv_recall_at_5_max": 0.3670337531198894, - "nauc_cv_recall_at_5_std": 0.25613869803100303, - "nauc_map_at_1000_diff1": 0.3176858451051277, - "nauc_map_at_1000_max": 0.39011754631587403, - "nauc_map_at_1000_std": 0.236531571726055, - "nauc_map_at_100_diff1": 0.3173466063685225, - "nauc_map_at_100_max": 0.39012399997496483, - "nauc_map_at_100_std": 0.23699486254878585, - "nauc_map_at_10_diff1": 0.31498941632282373, - "nauc_map_at_10_max": 0.3854108768250645, - "nauc_map_at_10_std": 0.23161137771783466, - "nauc_map_at_1_diff1": 0.39869161921412205, - "nauc_map_at_1_max": 0.3995277159429934, - "nauc_map_at_1_std": 0.19832125932977787, - "nauc_map_at_20_diff1": 0.31655878548331357, - "nauc_map_at_20_max": 0.3884827665912284, - "nauc_map_at_20_std": 0.2352857571016366, - "nauc_map_at_3_diff1": 0.32501618220535744, - "nauc_map_at_3_max": 0.3911459304357299, - "nauc_map_at_3_std": 0.22201571734296388, - "nauc_map_at_5_diff1": 0.3201466900694881, - "nauc_map_at_5_max": 0.38699141592725994, - "nauc_map_at_5_std": 0.22417441689275736, - "nauc_mrr_at_1000_diff1": 0.3176858451051277, - "nauc_mrr_at_1000_max": 0.39011754631587403, - "nauc_mrr_at_1000_std": 0.236531571726055, - "nauc_mrr_at_100_diff1": 0.3173466063685225, - "nauc_mrr_at_100_max": 0.39012399997496483, - "nauc_mrr_at_100_std": 0.23699486254878585, - "nauc_mrr_at_10_diff1": 0.31498941632282373, - "nauc_mrr_at_10_max": 0.3854108768250645, - "nauc_mrr_at_10_std": 0.23161137771783466, - "nauc_mrr_at_1_diff1": 0.39869161921412205, - "nauc_mrr_at_1_max": 0.3995277159429934, - "nauc_mrr_at_1_std": 0.19832125932977787, - "nauc_mrr_at_20_diff1": 0.31655878548331357, - "nauc_mrr_at_20_max": 0.3884827665912284, - "nauc_mrr_at_20_std": 0.2352857571016366, - "nauc_mrr_at_3_diff1": 0.32501618220535744, - "nauc_mrr_at_3_max": 0.3911459304357299, - "nauc_mrr_at_3_std": 0.22201571734296388, - "nauc_mrr_at_5_diff1": 0.3201466900694881, - "nauc_mrr_at_5_max": 0.38699141592725994, - "nauc_mrr_at_5_std": 0.22417441689275736, - "nauc_ndcg_at_1000_diff1": 0.3003864567165558, - "nauc_ndcg_at_1000_max": 0.3956273045582767, - "nauc_ndcg_at_1000_std": 0.2619661173143108, - "nauc_ndcg_at_100_diff1": 0.29160769223913835, - "nauc_ndcg_at_100_max": 0.3990188203836822, - "nauc_ndcg_at_100_std": 0.27953787343260206, - "nauc_ndcg_at_10_diff1": 0.2838945792321021, - "nauc_ndcg_at_10_max": 0.37855000399640953, - "nauc_ndcg_at_10_std": 0.25146628957781897, - "nauc_ndcg_at_1_diff1": 0.39869161921412205, - "nauc_ndcg_at_1_max": 0.3995277159429934, - "nauc_ndcg_at_1_std": 0.19832125932977787, - "nauc_ndcg_at_20_diff1": 0.2876980324046296, - "nauc_ndcg_at_20_max": 0.3888791107654943, - "nauc_ndcg_at_20_std": 0.2647289947812142, - "nauc_ndcg_at_3_diff1": 0.3040117361499693, - "nauc_ndcg_at_3_max": 0.38910655776005576, - "nauc_ndcg_at_3_std": 0.2301072351567442, - "nauc_ndcg_at_5_diff1": 0.29588345621924833, - "nauc_ndcg_at_5_max": 0.3815210586999878, - "nauc_ndcg_at_5_std": 0.23330023029333213, - "nauc_precision_at_1000_diff1": 0.13902137431548026, - "nauc_precision_at_1000_max": 0.6893027481262702, - "nauc_precision_at_1000_std": 0.8832235596941468, - "nauc_precision_at_100_diff1": 0.1769656859218761, - "nauc_precision_at_100_max": 0.4787900664905905, - "nauc_precision_at_100_std": 0.5594677155805846, - "nauc_precision_at_10_diff1": 0.200891626252226, - "nauc_precision_at_10_max": 0.3598805481889754, - "nauc_precision_at_10_std": 0.30887510439379645, - "nauc_precision_at_1_diff1": 0.39869161921412205, - "nauc_precision_at_1_max": 0.3995277159429934, - "nauc_precision_at_1_std": 0.19832125932977787, - "nauc_precision_at_20_diff1": 0.20381530227560674, - "nauc_precision_at_20_max": 0.39670392767516544, - "nauc_precision_at_20_std": 0.36613850164004663, - "nauc_precision_at_3_diff1": 0.2511337498545181, - "nauc_precision_at_3_max": 0.3840655418755833, - "nauc_precision_at_3_std": 0.2508323584082494, - "nauc_precision_at_5_diff1": 0.23542950151750375, - "nauc_precision_at_5_max": 0.36703375311988906, - "nauc_precision_at_5_std": 0.25613869803100286, - "nauc_recall_at_1000_diff1": 0.13902137431548597, - "nauc_recall_at_1000_max": 0.6893027481262831, - "nauc_recall_at_1000_std": 0.8832235596941536, - "nauc_recall_at_100_diff1": 0.17696568592187603, - "nauc_recall_at_100_max": 0.4787900664905917, - "nauc_recall_at_100_std": 0.5594677155805847, - "nauc_recall_at_10_diff1": 0.20089162625222612, - "nauc_recall_at_10_max": 0.35988054818897564, - "nauc_recall_at_10_std": 0.308875104393797, - "nauc_recall_at_1_diff1": 0.39869161921412205, - "nauc_recall_at_1_max": 0.3995277159429934, - "nauc_recall_at_1_std": 0.19832125932977787, - "nauc_recall_at_20_diff1": 0.20381530227560704, - "nauc_recall_at_20_max": 0.39670392767516544, - "nauc_recall_at_20_std": 0.36613850164004713, - "nauc_recall_at_3_diff1": 0.25113374985451803, - "nauc_recall_at_3_max": 0.3840655418755832, - "nauc_recall_at_3_std": 0.25083235840824936, - "nauc_recall_at_5_diff1": 0.2354295015175038, - "nauc_recall_at_5_max": 0.3670337531198894, - "nauc_recall_at_5_std": 0.25613869803100303, - "ndcg_at_1": 0.178, - "ndcg_at_10": 0.32156, - "ndcg_at_100": 0.38552, - "ndcg_at_1000": 0.40745, - "ndcg_at_20": 0.34802, - "ndcg_at_3": 0.253, - "ndcg_at_5": 0.28507, - "precision_at_1": 0.178, - "precision_at_10": 0.0499, - "precision_at_100": 0.00806, - "precision_at_1000": 0.00098, - "precision_at_20": 0.03017, - "precision_at_3": 0.10283, - "precision_at_5": 0.0772, - "recall_at_1": 0.178, - "recall_at_10": 0.499, - "recall_at_100": 0.806, - "recall_at_1000": 0.9815, - "recall_at_20": 0.6035, - "recall_at_3": 0.3085, - "recall_at_5": 0.386 - }, - { - "cv_recall_at_1": 0.03, - "cv_recall_at_10": 0.15, - "cv_recall_at_100": 0.449, - "cv_recall_at_1000": 0.8745, - "cv_recall_at_20": 0.216, - "cv_recall_at_3": 0.071, - "cv_recall_at_5": 0.095, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.08105, - "map_at_1": 0.03, - "map_at_10": 0.06018, - "map_at_100": 0.07, - "map_at_1000": 0.07157, - "map_at_20": 0.06471, - "map_at_3": 0.04742, - "map_at_5": 0.05287, - "mrr_at_1": 0.03, - "mrr_at_10": 0.06017539682539684, - "mrr_at_100": 0.06999993552071689, - "mrr_at_1000": 0.07157353469443933, - "mrr_at_20": 0.06470514698150148, - "mrr_at_3": 0.047416666666666635, - "mrr_at_5": 0.05286666666666666, - "nauc_cv_recall_at_1000_diff1": 0.0031622471635306543, - "nauc_cv_recall_at_1000_max": 0.1230754238939771, - "nauc_cv_recall_at_1000_std": 0.5455420913249333, - "nauc_cv_recall_at_100_diff1": 0.004456004806830121, - "nauc_cv_recall_at_100_max": 0.14632687230090322, - "nauc_cv_recall_at_100_std": 0.48531704526562575, - "nauc_cv_recall_at_10_diff1": 0.02403991533111579, - "nauc_cv_recall_at_10_max": 0.17673319221852626, - "nauc_cv_recall_at_10_std": 0.3569922386856162, - "nauc_cv_recall_at_1_diff1": 0.1459753152669708, - "nauc_cv_recall_at_1_max": 0.2915481620606387, - "nauc_cv_recall_at_1_std": 0.2551202933547983, - "nauc_cv_recall_at_20_diff1": 0.00797385989303483, - "nauc_cv_recall_at_20_max": 0.17344280364233017, - "nauc_cv_recall_at_20_std": 0.3785712496399003, - "nauc_cv_recall_at_3_diff1": 0.02156324045696237, - "nauc_cv_recall_at_3_max": 0.14944807022980386, - "nauc_cv_recall_at_3_std": 0.25913112158326945, - "nauc_cv_recall_at_5_diff1": 0.02042421589256204, - "nauc_cv_recall_at_5_max": 0.1657464024967167, - "nauc_cv_recall_at_5_std": 0.31874514566534384, - "nauc_map_at_1000_diff1": 0.05222864258329805, - "nauc_map_at_1000_max": 0.19626197508061652, - "nauc_map_at_1000_std": 0.31100022728590426, - "nauc_map_at_100_diff1": 0.05216544996314628, - "nauc_map_at_100_max": 0.19629835874395282, - "nauc_map_at_100_std": 0.3112490909791947, - "nauc_map_at_10_diff1": 0.05857667764737536, - "nauc_map_at_10_max": 0.20089763876749644, - "nauc_map_at_10_std": 0.294255605599976, - "nauc_map_at_1_diff1": 0.1459753152669708, - "nauc_map_at_1_max": 0.2915481620606387, - "nauc_map_at_1_std": 0.2551202933547983, - "nauc_map_at_20_diff1": 0.05318000546518001, - "nauc_map_at_20_max": 0.19925942267648766, - "nauc_map_at_20_std": 0.3021914243564095, - "nauc_map_at_3_diff1": 0.06724170695673946, - "nauc_map_at_3_max": 0.2011644471563847, - "nauc_map_at_3_std": 0.2575835953190111, - "nauc_map_at_5_diff1": 0.06244812656762225, - "nauc_map_at_5_max": 0.20323435608935306, - "nauc_map_at_5_std": 0.2823908454409408, - "nauc_mrr_at_1000_diff1": 0.05222864258329805, - "nauc_mrr_at_1000_max": 0.19626197508061652, - "nauc_mrr_at_1000_std": 0.31100022728590426, - "nauc_mrr_at_100_diff1": 0.05216544996314628, - "nauc_mrr_at_100_max": 0.19629835874395282, - "nauc_mrr_at_100_std": 0.3112490909791947, - "nauc_mrr_at_10_diff1": 0.05857667764737536, - "nauc_mrr_at_10_max": 0.20089763876749644, - "nauc_mrr_at_10_std": 0.294255605599976, - "nauc_mrr_at_1_diff1": 0.1459753152669708, - "nauc_mrr_at_1_max": 0.2915481620606387, - "nauc_mrr_at_1_std": 0.2551202933547983, - "nauc_mrr_at_20_diff1": 0.05318000546518001, - "nauc_mrr_at_20_max": 0.19925942267648766, - "nauc_mrr_at_20_std": 0.3021914243564095, - "nauc_mrr_at_3_diff1": 0.06724170695673946, - "nauc_mrr_at_3_max": 0.2011644471563847, - "nauc_mrr_at_3_std": 0.2575835953190111, - "nauc_mrr_at_5_diff1": 0.06244812656762225, - "nauc_mrr_at_5_max": 0.20323435608935306, - "nauc_mrr_at_5_std": 0.2823908454409408, - "nauc_ndcg_at_1000_diff1": 0.03313709100866658, - "nauc_ndcg_at_1000_max": 0.17555839361410389, - "nauc_ndcg_at_1000_std": 0.36791242069420094, - "nauc_ndcg_at_100_diff1": 0.029382656610136004, - "nauc_ndcg_at_100_max": 0.17392712886916573, - "nauc_ndcg_at_100_std": 0.3798645355648723, - "nauc_ndcg_at_10_diff1": 0.04341832341400407, - "nauc_ndcg_at_10_max": 0.1895060849776859, - "nauc_ndcg_at_10_std": 0.3185087090048503, - "nauc_ndcg_at_1_diff1": 0.1459753152669708, - "nauc_ndcg_at_1_max": 0.2915481620606387, - "nauc_ndcg_at_1_std": 0.2551202933547983, - "nauc_ndcg_at_20_diff1": 0.032429813463278674, - "nauc_ndcg_at_20_max": 0.18647413650324113, - "nauc_ndcg_at_20_std": 0.3346764917048847, - "nauc_ndcg_at_3_diff1": 0.05174277111392793, - "nauc_ndcg_at_3_max": 0.1836002702068179, - "nauc_ndcg_at_3_std": 0.2581059616380073, - "nauc_ndcg_at_5_diff1": 0.04662102581978742, - "nauc_ndcg_at_5_max": 0.18887628288344824, - "nauc_ndcg_at_5_std": 0.29523558230070585, - "nauc_precision_at_1000_diff1": 0.0031622471635302882, - "nauc_precision_at_1000_max": 0.12307542389397594, - "nauc_precision_at_1000_std": 0.5455420913249317, - "nauc_precision_at_100_diff1": 0.004456004806830124, - "nauc_precision_at_100_max": 0.14632687230090355, - "nauc_precision_at_100_std": 0.48531704526562597, - "nauc_precision_at_10_diff1": 0.024039915331115713, - "nauc_precision_at_10_max": 0.17673319221852626, - "nauc_precision_at_10_std": 0.3569922386856165, - "nauc_precision_at_1_diff1": 0.1459753152669708, - "nauc_precision_at_1_max": 0.2915481620606387, - "nauc_precision_at_1_std": 0.2551202933547983, - "nauc_precision_at_20_diff1": 0.007973859893034724, - "nauc_precision_at_20_max": 0.17344280364233036, - "nauc_precision_at_20_std": 0.3785712496399004, - "nauc_precision_at_3_diff1": 0.021563240456962503, - "nauc_precision_at_3_max": 0.1494480702298039, - "nauc_precision_at_3_std": 0.25913112158326945, - "nauc_precision_at_5_diff1": 0.020424215892562138, - "nauc_precision_at_5_max": 0.16574640249671674, - "nauc_precision_at_5_std": 0.31874514566534407, - "nauc_recall_at_1000_diff1": 0.0031622471635306543, - "nauc_recall_at_1000_max": 0.1230754238939771, - "nauc_recall_at_1000_std": 0.5455420913249333, - "nauc_recall_at_100_diff1": 0.004456004806830121, - "nauc_recall_at_100_max": 0.14632687230090322, - "nauc_recall_at_100_std": 0.48531704526562575, - "nauc_recall_at_10_diff1": 0.02403991533111579, - "nauc_recall_at_10_max": 0.17673319221852626, - "nauc_recall_at_10_std": 0.3569922386856162, - "nauc_recall_at_1_diff1": 0.1459753152669708, - "nauc_recall_at_1_max": 0.2915481620606387, - "nauc_recall_at_1_std": 0.2551202933547983, - "nauc_recall_at_20_diff1": 0.00797385989303483, - "nauc_recall_at_20_max": 0.17344280364233017, - "nauc_recall_at_20_std": 0.3785712496399003, - "nauc_recall_at_3_diff1": 0.02156324045696237, - "nauc_recall_at_3_max": 0.14944807022980386, - "nauc_recall_at_3_std": 0.25913112158326945, - "nauc_recall_at_5_diff1": 0.02042421589256204, - "nauc_recall_at_5_max": 0.1657464024967167, - "nauc_recall_at_5_std": 0.31874514566534384, - "ndcg_at_1": 0.03, - "ndcg_at_10": 0.08105, - "ndcg_at_100": 0.13952, - "ndcg_at_1000": 0.19087, - "ndcg_at_20": 0.09768, - "ndcg_at_3": 0.05345, - "ndcg_at_5": 0.0633, - "precision_at_1": 0.03, - "precision_at_10": 0.015, - "precision_at_100": 0.00449, - "precision_at_1000": 0.00087, - "precision_at_20": 0.0108, - "precision_at_3": 0.02367, - "precision_at_5": 0.019, - "recall_at_1": 0.03, - "recall_at_10": 0.15, - "recall_at_100": 0.449, - "recall_at_1000": 0.8745, - "recall_at_20": 0.216, - "recall_at_3": 0.071, - "recall_at_5": 0.095 - }, - { - "cv_recall_at_1": 0.016, - "cv_recall_at_10": 0.091, - "cv_recall_at_100": 0.3185, - "cv_recall_at_1000": 0.8135, - "cv_recall_at_20": 0.1365, - "cv_recall_at_3": 0.041, - "cv_recall_at_5": 0.059, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.04778, - "map_at_1": 0.016, - "map_at_10": 0.03469, - "map_at_100": 0.0421, - "map_at_1000": 0.0438, - "map_at_20": 0.03791, - "map_at_3": 0.02642, - "map_at_5": 0.03054, - "mrr_at_1": 0.016, - "mrr_at_10": 0.03468730158730162, - "mrr_at_100": 0.042098286930249154, - "mrr_at_1000": 0.0438005310087541, - "mrr_at_20": 0.03790599534839091, - "mrr_at_3": 0.026416666666666675, - "mrr_at_5": 0.030541666666666682, - "nauc_cv_recall_at_1000_diff1": 0.07106216173723574, - "nauc_cv_recall_at_1000_max": 0.3177797971022986, - "nauc_cv_recall_at_1000_std": 0.5407727380530455, - "nauc_cv_recall_at_100_diff1": 0.0713599535138732, - "nauc_cv_recall_at_100_max": 0.297997227166803, - "nauc_cv_recall_at_100_std": 0.48667707873788535, - "nauc_cv_recall_at_10_diff1": 0.05669645621349406, - "nauc_cv_recall_at_10_max": 0.2794030493252388, - "nauc_cv_recall_at_10_std": 0.40729514649557574, - "nauc_cv_recall_at_1_diff1": 0.14172759592165277, - "nauc_cv_recall_at_1_max": 0.19727662999731693, - "nauc_cv_recall_at_1_std": 0.400665749932922, - "nauc_cv_recall_at_20_diff1": 0.09607008604081496, - "nauc_cv_recall_at_20_max": 0.2698632401611685, - "nauc_cv_recall_at_20_std": 0.40934703849091997, - "nauc_cv_recall_at_3_diff1": 0.14946304815878855, - "nauc_cv_recall_at_3_max": 0.2802096762582865, - "nauc_cv_recall_at_3_std": 0.42041922163251666, - "nauc_cv_recall_at_5_diff1": 0.08865220811940362, - "nauc_cv_recall_at_5_max": 0.2713910856643915, - "nauc_cv_recall_at_5_std": 0.442433365318587, - "nauc_map_at_1000_diff1": 0.10616900969923737, - "nauc_map_at_1000_max": 0.2520076462657044, - "nauc_map_at_1000_std": 0.41663431232817416, - "nauc_map_at_100_diff1": 0.1055771817335392, - "nauc_map_at_100_max": 0.25226679270899655, - "nauc_map_at_100_std": 0.4159028968396924, - "nauc_map_at_10_diff1": 0.1051404762247085, - "nauc_map_at_10_max": 0.2502761119493815, - "nauc_map_at_10_std": 0.4074511071618613, - "nauc_map_at_1_diff1": 0.14172759592165277, - "nauc_map_at_1_max": 0.19727662999731693, - "nauc_map_at_1_std": 0.400665749932922, - "nauc_map_at_20_diff1": 0.11168787671435364, - "nauc_map_at_20_max": 0.24958328967583604, - "nauc_map_at_20_std": 0.4069329483284715, - "nauc_map_at_3_diff1": 0.14520944019216925, - "nauc_map_at_3_max": 0.24336265583486172, - "nauc_map_at_3_std": 0.40580671864195006, - "nauc_map_at_5_diff1": 0.12029564869169368, - "nauc_map_at_5_max": 0.24414012125666815, - "nauc_map_at_5_std": 0.41652591556544544, - "nauc_mrr_at_1000_diff1": 0.10616900969923737, - "nauc_mrr_at_1000_max": 0.2520076462657044, - "nauc_mrr_at_1000_std": 0.41663431232817416, - "nauc_mrr_at_100_diff1": 0.1055771817335392, - "nauc_mrr_at_100_max": 0.25226679270899655, - "nauc_mrr_at_100_std": 0.4159028968396924, - "nauc_mrr_at_10_diff1": 0.1051404762247085, - "nauc_mrr_at_10_max": 0.2502761119493815, - "nauc_mrr_at_10_std": 0.4074511071618613, - "nauc_mrr_at_1_diff1": 0.14172759592165277, - "nauc_mrr_at_1_max": 0.19727662999731693, - "nauc_mrr_at_1_std": 0.400665749932922, - "nauc_mrr_at_20_diff1": 0.11168787671435364, - "nauc_mrr_at_20_max": 0.24958328967583604, - "nauc_mrr_at_20_std": 0.4069329483284715, - "nauc_mrr_at_3_diff1": 0.14520944019216925, - "nauc_mrr_at_3_max": 0.24336265583486172, - "nauc_mrr_at_3_std": 0.40580671864195006, - "nauc_mrr_at_5_diff1": 0.12029564869169368, - "nauc_mrr_at_5_max": 0.24414012125666815, - "nauc_mrr_at_5_std": 0.41652591556544544, - "nauc_ndcg_at_1000_diff1": 0.09118363577195546, - "nauc_ndcg_at_1000_max": 0.2734412591888149, - "nauc_ndcg_at_1000_std": 0.45381717258062526, - "nauc_ndcg_at_100_diff1": 0.08635669876740394, - "nauc_ndcg_at_100_max": 0.2745828115432325, - "nauc_ndcg_at_100_std": 0.4479316944010123, - "nauc_ndcg_at_10_diff1": 0.08469399333358935, - "nauc_ndcg_at_10_max": 0.2633158764122716, - "nauc_ndcg_at_10_std": 0.40804473149726966, - "nauc_ndcg_at_1_diff1": 0.14172759592165277, - "nauc_ndcg_at_1_max": 0.19727662999731693, - "nauc_ndcg_at_1_std": 0.400665749932922, - "nauc_ndcg_at_20_diff1": 0.10288612513091308, - "nauc_ndcg_at_20_max": 0.2601636991744163, - "nauc_ndcg_at_20_std": 0.4082859431626536, - "nauc_ndcg_at_3_diff1": 0.14663138399660128, - "nauc_ndcg_at_3_max": 0.2558808244090084, - "nauc_ndcg_at_3_std": 0.41060566375848273, - "nauc_ndcg_at_5_diff1": 0.10881705441852337, - "nauc_ndcg_at_5_max": 0.2547659569303373, - "nauc_ndcg_at_5_std": 0.4260939220880413, - "nauc_precision_at_1000_diff1": 0.07106216173723427, - "nauc_precision_at_1000_max": 0.31777979710229853, - "nauc_precision_at_1000_std": 0.5407727380530446, - "nauc_precision_at_100_diff1": 0.0713599535138732, - "nauc_precision_at_100_max": 0.2979972271668032, - "nauc_precision_at_100_std": 0.4866770787378858, - "nauc_precision_at_10_diff1": 0.0566964562134939, - "nauc_precision_at_10_max": 0.27940304932523885, - "nauc_precision_at_10_std": 0.40729514649557574, - "nauc_precision_at_1_diff1": 0.14172759592165277, - "nauc_precision_at_1_max": 0.19727662999731693, - "nauc_precision_at_1_std": 0.400665749932922, - "nauc_precision_at_20_diff1": 0.09607008604081504, - "nauc_precision_at_20_max": 0.2698632401611685, - "nauc_precision_at_20_std": 0.40934703849091997, - "nauc_precision_at_3_diff1": 0.14946304815878844, - "nauc_precision_at_3_max": 0.2802096762582866, - "nauc_precision_at_3_std": 0.4204192216325167, - "nauc_precision_at_5_diff1": 0.0886522081194035, - "nauc_precision_at_5_max": 0.2713910856643913, - "nauc_precision_at_5_std": 0.4424333653185869, - "nauc_recall_at_1000_diff1": 0.07106216173723574, - "nauc_recall_at_1000_max": 0.3177797971022986, - "nauc_recall_at_1000_std": 0.5407727380530455, - "nauc_recall_at_100_diff1": 0.0713599535138732, - "nauc_recall_at_100_max": 0.297997227166803, - "nauc_recall_at_100_std": 0.48667707873788535, - "nauc_recall_at_10_diff1": 0.05669645621349406, - "nauc_recall_at_10_max": 0.2794030493252388, - "nauc_recall_at_10_std": 0.40729514649557574, - "nauc_recall_at_1_diff1": 0.14172759592165277, - "nauc_recall_at_1_max": 0.19727662999731693, - "nauc_recall_at_1_std": 0.400665749932922, - "nauc_recall_at_20_diff1": 0.09607008604081496, - "nauc_recall_at_20_max": 0.2698632401611685, - "nauc_recall_at_20_std": 0.40934703849091997, - "nauc_recall_at_3_diff1": 0.14946304815878855, - "nauc_recall_at_3_max": 0.2802096762582865, - "nauc_recall_at_3_std": 0.42041922163251666, - "nauc_recall_at_5_diff1": 0.08865220811940362, - "nauc_recall_at_5_max": 0.2713910856643915, - "nauc_recall_at_5_std": 0.442433365318587, - "ndcg_at_1": 0.016, - "ndcg_at_10": 0.04778, - "ndcg_at_100": 0.09216, - "ndcg_at_1000": 0.15112, - "ndcg_at_20": 0.05937, - "ndcg_at_3": 0.03014, - "ndcg_at_5": 0.03756, - "precision_at_1": 0.016, - "precision_at_10": 0.0091, - "precision_at_100": 0.00318, - "precision_at_1000": 0.00081, - "precision_at_20": 0.00683, - "precision_at_3": 0.01367, - "precision_at_5": 0.0118, - "recall_at_1": 0.016, - "recall_at_10": 0.091, - "recall_at_100": 0.3185, - "recall_at_1000": 0.8135, - "recall_at_20": 0.1365, - "recall_at_3": 0.041, - "recall_at_5": 0.059 - }, - { - "cv_recall_at_1": 0.005, - "cv_recall_at_10": 0.018, - "cv_recall_at_100": 0.087, - "cv_recall_at_1000": 0.5615, - "cv_recall_at_20": 0.028, - "cv_recall_at_3": 0.0085, - "cv_recall_at_5": 0.0135, - "hf_subset": "ru", - "languages": [ - "rus-Cyrl" - ], - "main_score": 0.01038, - "map_at_1": 0.005, - "map_at_10": 0.00807, - "map_at_100": 0.00989, - "map_at_1000": 0.01114, - "map_at_20": 0.00875, - "map_at_3": 0.00642, - "map_at_5": 0.00749, - "mrr_at_1": 0.005, - "mrr_at_10": 0.00806984126984127, - "mrr_at_100": 0.009885263099824498, - "mrr_at_1000": 0.011144283116471717, - "mrr_at_20": 0.008748844853546866, - "mrr_at_3": 0.006416666666666667, - "mrr_at_5": 0.0074916666666666664, - "nauc_cv_recall_at_1000_diff1": 0.01866337186197954, - "nauc_cv_recall_at_1000_max": 0.1432479371333068, - "nauc_cv_recall_at_1000_std": 0.1353675624298429, - "nauc_cv_recall_at_100_diff1": 0.09785535190548013, - "nauc_cv_recall_at_100_max": 0.26174174785427257, - "nauc_cv_recall_at_100_std": 0.32808582293237604, - "nauc_cv_recall_at_10_diff1": 0.15284112929672355, - "nauc_cv_recall_at_10_max": 0.5223668127478165, - "nauc_cv_recall_at_10_std": 0.5570759920102556, - "nauc_cv_recall_at_1_diff1": 0.12621411322779696, - "nauc_cv_recall_at_1_max": 0.5159109203112423, - "nauc_cv_recall_at_1_std": 0.5785350147571774, - "nauc_cv_recall_at_20_diff1": 0.18863314040400164, - "nauc_cv_recall_at_20_max": 0.43482885507301905, - "nauc_cv_recall_at_20_std": 0.46413718425389994, - "nauc_cv_recall_at_3_diff1": 0.10757745545226408, - "nauc_cv_recall_at_3_max": 0.48136807714768237, - "nauc_cv_recall_at_3_std": 0.5222304644959674, - "nauc_cv_recall_at_5_diff1": 0.14783014836677316, - "nauc_cv_recall_at_5_max": 0.5114529608760894, - "nauc_cv_recall_at_5_std": 0.5787099146369339, - "nauc_map_at_1000_diff1": 0.13484125064851551, - "nauc_map_at_1000_max": 0.43525690428320235, - "nauc_map_at_1000_std": 0.4860746717149452, - "nauc_map_at_100_diff1": 0.14322051241470532, - "nauc_map_at_100_max": 0.4553782899063439, - "nauc_map_at_100_std": 0.510862684307652, - "nauc_map_at_10_diff1": 0.14185029312751027, - "nauc_map_at_10_max": 0.5083078127015377, - "nauc_map_at_10_std": 0.5587400256321647, - "nauc_map_at_1_diff1": 0.12621411322779696, - "nauc_map_at_1_max": 0.5159109203112423, - "nauc_map_at_1_std": 0.5785350147571774, - "nauc_map_at_20_diff1": 0.1542361916482395, - "nauc_map_at_20_max": 0.4926807911639361, - "nauc_map_at_20_std": 0.5426787801006763, - "nauc_map_at_3_diff1": 0.12486976398969958, - "nauc_map_at_3_max": 0.48833538342526805, - "nauc_map_at_3_std": 0.5397990793751459, - "nauc_map_at_5_diff1": 0.14057476139155894, - "nauc_map_at_5_max": 0.503370319046921, - "nauc_map_at_5_std": 0.5624163389366535, - "nauc_mrr_at_1000_diff1": 0.13484125064851551, - "nauc_mrr_at_1000_max": 0.43525690428320235, - "nauc_mrr_at_1000_std": 0.4860746717149452, - "nauc_mrr_at_100_diff1": 0.14322051241470532, - "nauc_mrr_at_100_max": 0.4553782899063439, - "nauc_mrr_at_100_std": 0.510862684307652, - "nauc_mrr_at_10_diff1": 0.14185029312751027, - "nauc_mrr_at_10_max": 0.5083078127015377, - "nauc_mrr_at_10_std": 0.5587400256321647, - "nauc_mrr_at_1_diff1": 0.12621411322779696, - "nauc_mrr_at_1_max": 0.5159109203112423, - "nauc_mrr_at_1_std": 0.5785350147571774, - "nauc_mrr_at_20_diff1": 0.1542361916482395, - "nauc_mrr_at_20_max": 0.4926807911639361, - "nauc_mrr_at_20_std": 0.5426787801006763, - "nauc_mrr_at_3_diff1": 0.12486976398969958, - "nauc_mrr_at_3_max": 0.48833538342526805, - "nauc_mrr_at_3_std": 0.5397990793751459, - "nauc_mrr_at_5_diff1": 0.14057476139155894, - "nauc_mrr_at_5_max": 0.503370319046921, - "nauc_mrr_at_5_std": 0.5624163389366535, - "nauc_ndcg_at_1000_diff1": 0.06105664555017735, - "nauc_ndcg_at_1000_max": 0.2326125051202298, - "nauc_ndcg_at_1000_std": 0.24874451461947863, - "nauc_ndcg_at_100_diff1": 0.11970760208087682, - "nauc_ndcg_at_100_max": 0.34084016686829804, - "nauc_ndcg_at_100_std": 0.4020989248226949, - "nauc_ndcg_at_10_diff1": 0.14611669901831748, - "nauc_ndcg_at_10_max": 0.513929921408022, - "nauc_ndcg_at_10_std": 0.558476215872956, - "nauc_ndcg_at_1_diff1": 0.12621411322779696, - "nauc_ndcg_at_1_max": 0.5159109203112423, - "nauc_ndcg_at_1_std": 0.5785350147571774, - "nauc_ndcg_at_20_diff1": 0.17016830197419816, - "nauc_ndcg_at_20_max": 0.4698563274930774, - "nauc_ndcg_at_20_std": 0.5111408910283253, - "nauc_ndcg_at_3_diff1": 0.11980779651940149, - "nauc_ndcg_at_3_max": 0.4858343012706748, - "nauc_ndcg_at_3_std": 0.5340182129958824, - "nauc_ndcg_at_5_diff1": 0.14327180898186234, - "nauc_ndcg_at_5_max": 0.5062088605443746, - "nauc_ndcg_at_5_std": 0.5678637638968992, - "nauc_precision_at_1000_diff1": 0.01866337186197888, - "nauc_precision_at_1000_max": 0.14324793713330597, - "nauc_precision_at_1000_std": 0.1353675624298418, - "nauc_precision_at_100_diff1": 0.09785535190547996, - "nauc_precision_at_100_max": 0.26174174785427246, - "nauc_precision_at_100_std": 0.3280858229323761, - "nauc_precision_at_10_diff1": 0.15284112929672372, - "nauc_precision_at_10_max": 0.5223668127478164, - "nauc_precision_at_10_std": 0.5570759920102557, - "nauc_precision_at_1_diff1": 0.12621411322779696, - "nauc_precision_at_1_max": 0.5159109203112423, - "nauc_precision_at_1_std": 0.5785350147571774, - "nauc_precision_at_20_diff1": 0.18863314040400164, - "nauc_precision_at_20_max": 0.4348288550730195, - "nauc_precision_at_20_std": 0.4641371842539002, - "nauc_precision_at_3_diff1": 0.10757745545226413, - "nauc_precision_at_3_max": 0.48136807714768237, - "nauc_precision_at_3_std": 0.5222304644959676, - "nauc_precision_at_5_diff1": 0.147830148366773, - "nauc_precision_at_5_max": 0.5114529608760895, - "nauc_precision_at_5_std": 0.5787099146369338, - "nauc_recall_at_1000_diff1": 0.01866337186197954, - "nauc_recall_at_1000_max": 0.1432479371333068, - "nauc_recall_at_1000_std": 0.1353675624298429, - "nauc_recall_at_100_diff1": 0.09785535190548013, - "nauc_recall_at_100_max": 0.26174174785427257, - "nauc_recall_at_100_std": 0.32808582293237604, - "nauc_recall_at_10_diff1": 0.15284112929672355, - "nauc_recall_at_10_max": 0.5223668127478165, - "nauc_recall_at_10_std": 0.5570759920102556, - "nauc_recall_at_1_diff1": 0.12621411322779696, - "nauc_recall_at_1_max": 0.5159109203112423, - "nauc_recall_at_1_std": 0.5785350147571774, - "nauc_recall_at_20_diff1": 0.18863314040400164, - "nauc_recall_at_20_max": 0.43482885507301905, - "nauc_recall_at_20_std": 0.46413718425389994, - "nauc_recall_at_3_diff1": 0.10757745545226408, - "nauc_recall_at_3_max": 0.48136807714768237, - "nauc_recall_at_3_std": 0.5222304644959674, - "nauc_recall_at_5_diff1": 0.14783014836677316, - "nauc_recall_at_5_max": 0.5114529608760894, - "nauc_recall_at_5_std": 0.5787099146369339, - "ndcg_at_1": 0.005, - "ndcg_at_10": 0.01038, - "ndcg_at_100": 0.02307, - "ndcg_at_1000": 0.07725, - "ndcg_at_20": 0.01289, - "ndcg_at_3": 0.00695, - "ndcg_at_5": 0.00895, - "precision_at_1": 0.005, - "precision_at_10": 0.0018, - "precision_at_100": 0.00087, - "precision_at_1000": 0.00056, - "precision_at_20": 0.0014, - "precision_at_3": 0.00283, - "precision_at_5": 0.0027, - "recall_at_1": 0.005, - "recall_at_10": 0.018, - "recall_at_100": 0.087, - "recall_at_1000": 0.5615, - "recall_at_20": 0.028, - "recall_at_3": 0.0085, - "recall_at_5": 0.0135 - }, - { - "cv_recall_at_1": 0.0155, - "cv_recall_at_10": 0.073, - "cv_recall_at_100": 0.245, - "cv_recall_at_1000": 0.6955, - "cv_recall_at_20": 0.105, - "cv_recall_at_3": 0.0325, - "cv_recall_at_5": 0.0515, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.03951, - "map_at_1": 0.0155, - "map_at_10": 0.02936, - "map_at_100": 0.03458, - "map_at_1000": 0.03597, - "map_at_20": 0.03162, - "map_at_3": 0.02233, - "map_at_5": 0.02666, - "mrr_at_1": 0.0155, - "mrr_at_10": 0.029362896825396858, - "mrr_at_100": 0.03457677859623816, - "mrr_at_1000": 0.035970307503409196, - "mrr_at_20": 0.03161513074343963, - "mrr_at_3": 0.02233333333333334, - "mrr_at_5": 0.026658333333333357, - "nauc_cv_recall_at_1000_diff1": 0.10103909150196763, - "nauc_cv_recall_at_1000_max": 0.321531328425893, - "nauc_cv_recall_at_1000_std": 0.3990844717958969, - "nauc_cv_recall_at_100_diff1": 0.1785332076619936, - "nauc_cv_recall_at_100_max": 0.437202674721507, - "nauc_cv_recall_at_100_std": 0.515561476291904, - "nauc_cv_recall_at_10_diff1": 0.14435937678032573, - "nauc_cv_recall_at_10_max": 0.4617783593253233, - "nauc_cv_recall_at_10_std": 0.48276920362699444, - "nauc_cv_recall_at_1_diff1": 0.3766499043596423, - "nauc_cv_recall_at_1_max": 0.5343223382985537, - "nauc_cv_recall_at_1_std": 0.47144204886746255, - "nauc_cv_recall_at_20_diff1": 0.1347877465684938, - "nauc_cv_recall_at_20_max": 0.44873712779357744, - "nauc_cv_recall_at_20_std": 0.486303842507824, - "nauc_cv_recall_at_3_diff1": 0.2304802790448082, - "nauc_cv_recall_at_3_max": 0.5613134919815895, - "nauc_cv_recall_at_3_std": 0.49969660068935606, - "nauc_cv_recall_at_5_diff1": 0.18932690078435777, - "nauc_cv_recall_at_5_max": 0.4822874797137655, - "nauc_cv_recall_at_5_std": 0.49071978034859776, - "nauc_map_at_1000_diff1": 0.23946320151573663, - "nauc_map_at_1000_max": 0.4999034385390828, - "nauc_map_at_1000_std": 0.49153269673252975, - "nauc_map_at_100_diff1": 0.23981703740687996, - "nauc_map_at_100_max": 0.500941148105614, - "nauc_map_at_100_std": 0.49212619471780217, - "nauc_map_at_10_diff1": 0.24737658303862772, - "nauc_map_at_10_max": 0.5085696956319696, - "nauc_map_at_10_std": 0.4840574549189613, - "nauc_map_at_1_diff1": 0.3766499043596423, - "nauc_map_at_1_max": 0.5343223382985537, - "nauc_map_at_1_std": 0.47144204886746255, - "nauc_map_at_20_diff1": 0.23875732622736212, - "nauc_map_at_20_max": 0.5025212214500192, - "nauc_map_at_20_std": 0.4861404954526816, - "nauc_map_at_3_diff1": 0.2963179140519566, - "nauc_map_at_3_max": 0.5525251392621011, - "nauc_map_at_3_std": 0.4902626657429246, - "nauc_map_at_5_diff1": 0.26780211115410124, - "nauc_map_at_5_max": 0.5187074240818313, - "nauc_map_at_5_std": 0.48668004230259443, - "nauc_mrr_at_1000_diff1": 0.23946245396171295, - "nauc_mrr_at_1000_max": 0.49990313744248865, - "nauc_mrr_at_1000_std": 0.4915329100203209, - "nauc_mrr_at_100_diff1": 0.23981703740687996, - "nauc_mrr_at_100_max": 0.500941148105614, - "nauc_mrr_at_100_std": 0.49212619471780217, - "nauc_mrr_at_10_diff1": 0.24737658303862772, - "nauc_mrr_at_10_max": 0.5085696956319696, - "nauc_mrr_at_10_std": 0.4840574549189613, - "nauc_mrr_at_1_diff1": 0.3766499043596423, - "nauc_mrr_at_1_max": 0.5343223382985537, - "nauc_mrr_at_1_std": 0.47144204886746255, - "nauc_mrr_at_20_diff1": 0.23875732622736212, - "nauc_mrr_at_20_max": 0.5025212214500192, - "nauc_mrr_at_20_std": 0.4861404954526816, - "nauc_mrr_at_3_diff1": 0.2963179140519566, - "nauc_mrr_at_3_max": 0.5525251392621011, - "nauc_mrr_at_3_std": 0.4902626657429246, - "nauc_mrr_at_5_diff1": 0.26780211115410124, - "nauc_mrr_at_5_max": 0.5187074240818313, - "nauc_mrr_at_5_std": 0.48668004230259443, - "nauc_ndcg_at_1000_diff1": 0.18188268853902662, - "nauc_ndcg_at_1000_max": 0.4366951694757732, - "nauc_ndcg_at_1000_std": 0.4718476543887696, - "nauc_ndcg_at_100_diff1": 0.20134563640609476, - "nauc_ndcg_at_100_max": 0.46779496202101484, - "nauc_ndcg_at_100_std": 0.5044989040265137, - "nauc_ndcg_at_10_diff1": 0.20378966242608526, - "nauc_ndcg_at_10_max": 0.4891954411104291, - "nauc_ndcg_at_10_std": 0.4836847893872883, - "nauc_ndcg_at_1_diff1": 0.3766499043596423, - "nauc_ndcg_at_1_max": 0.5343223382985537, - "nauc_ndcg_at_1_std": 0.47144204886746255, - "nauc_ndcg_at_20_diff1": 0.18929143990336425, - "nauc_ndcg_at_20_max": 0.4776004370098488, - "nauc_ndcg_at_20_std": 0.4866188784612998, - "nauc_ndcg_at_3_diff1": 0.2746821234810608, - "nauc_ndcg_at_3_max": 0.5555235609881958, - "nauc_ndcg_at_3_std": 0.49347073414047715, - "nauc_ndcg_at_5_diff1": 0.237699805726334, - "nauc_ndcg_at_5_max": 0.5054343509360476, - "nauc_ndcg_at_5_std": 0.48831749596962815, - "nauc_precision_at_1000_diff1": 0.10103909150196676, - "nauc_precision_at_1000_max": 0.3215313284258918, - "nauc_precision_at_1000_std": 0.3990844717958955, - "nauc_precision_at_100_diff1": 0.17853320766199374, - "nauc_precision_at_100_max": 0.4372026747215069, - "nauc_precision_at_100_std": 0.5155614762919041, - "nauc_precision_at_10_diff1": 0.14435937678032562, - "nauc_precision_at_10_max": 0.46177835932532324, - "nauc_precision_at_10_std": 0.4827692036269943, - "nauc_precision_at_1_diff1": 0.3766499043596423, - "nauc_precision_at_1_max": 0.5343223382985537, - "nauc_precision_at_1_std": 0.47144204886746255, - "nauc_precision_at_20_diff1": 0.13478774656849388, - "nauc_precision_at_20_max": 0.4487371277935772, - "nauc_precision_at_20_std": 0.48630384250782405, - "nauc_precision_at_3_diff1": 0.23048027904480814, - "nauc_precision_at_3_max": 0.5613134919815899, - "nauc_precision_at_3_std": 0.49969660068935623, - "nauc_precision_at_5_diff1": 0.18932690078435782, - "nauc_precision_at_5_max": 0.48228747971376534, - "nauc_precision_at_5_std": 0.4907197803485978, - "nauc_recall_at_1000_diff1": 0.10103909150196763, - "nauc_recall_at_1000_max": 0.321531328425893, - "nauc_recall_at_1000_std": 0.3990844717958969, - "nauc_recall_at_100_diff1": 0.1785332076619936, - "nauc_recall_at_100_max": 0.437202674721507, - "nauc_recall_at_100_std": 0.515561476291904, - "nauc_recall_at_10_diff1": 0.14435937678032573, - "nauc_recall_at_10_max": 0.4617783593253233, - "nauc_recall_at_10_std": 0.48276920362699444, - "nauc_recall_at_1_diff1": 0.3766499043596423, - "nauc_recall_at_1_max": 0.5343223382985537, - "nauc_recall_at_1_std": 0.47144204886746255, - "nauc_recall_at_20_diff1": 0.1347877465684938, - "nauc_recall_at_20_max": 0.44873712779357744, - "nauc_recall_at_20_std": 0.486303842507824, - "nauc_recall_at_3_diff1": 0.2304802790448082, - "nauc_recall_at_3_max": 0.5613134919815895, - "nauc_recall_at_3_std": 0.49969660068935606, - "nauc_recall_at_5_diff1": 0.18932690078435777, - "nauc_recall_at_5_max": 0.4822874797137655, - "nauc_recall_at_5_std": 0.49071978034859776, - "ndcg_at_1": 0.0155, - "ndcg_at_10": 0.03951, - "ndcg_at_100": 0.07233, - "ndcg_at_1000": 0.125, - "ndcg_at_20": 0.04764, - "ndcg_at_3": 0.02492, - "ndcg_at_5": 0.03273, - "precision_at_1": 0.0155, - "precision_at_10": 0.0073, - "precision_at_100": 0.00245, - "precision_at_1000": 0.0007, - "precision_at_20": 0.00525, - "precision_at_3": 0.01083, - "precision_at_5": 0.0103, - "recall_at_1": 0.0155, - "recall_at_10": 0.073, - "recall_at_100": 0.245, - "recall_at_1000": 0.6955, - "recall_at_20": 0.105, - "recall_at_3": 0.0325, - "recall_at_5": 0.0515 - }, - { - "cv_recall_at_1": 0.004, - "cv_recall_at_10": 0.0285, - "cv_recall_at_100": 0.1545, - "cv_recall_at_1000": 0.676, - "cv_recall_at_20": 0.0525, - "cv_recall_at_3": 0.0125, - "cv_recall_at_5": 0.0185, - "hf_subset": "zh", - "languages": [ - "zho-Hans" - ], - "main_score": 0.01454, - "map_at_1": 0.004, - "map_at_10": 0.01031, - "map_at_100": 0.01413, - "map_at_1000": 0.01568, - "map_at_20": 0.01189, - "map_at_3": 0.00767, - "map_at_5": 0.00907, - "mrr_at_1": 0.004, - "mrr_at_10": 0.010310912698412694, - "mrr_at_100": 0.014128261725250548, - "mrr_at_1000": 0.015682993790635865, - "mrr_at_20": 0.011885865834801591, - "mrr_at_3": 0.007666666666666668, - "mrr_at_5": 0.009066666666666664, - "nauc_cv_recall_at_1000_diff1": 0.1463224055326704, - "nauc_cv_recall_at_1000_max": 0.20534263898503774, - "nauc_cv_recall_at_1000_std": 0.28937880833738816, - "nauc_cv_recall_at_100_diff1": 0.11011108120551756, - "nauc_cv_recall_at_100_max": 0.24934265204776085, - "nauc_cv_recall_at_100_std": 0.2998186932307211, - "nauc_cv_recall_at_10_diff1": 0.14244559614759988, - "nauc_cv_recall_at_10_max": 0.3921737534068603, - "nauc_cv_recall_at_10_std": 0.3703933835124436, - "nauc_cv_recall_at_1_diff1": 0.27867587872283334, - "nauc_cv_recall_at_1_max": 0.7182720686879527, - "nauc_cv_recall_at_1_std": 0.5737858867722029, - "nauc_cv_recall_at_20_diff1": 0.08256608787867177, - "nauc_cv_recall_at_20_max": 0.3323674089974063, - "nauc_cv_recall_at_20_std": 0.31922010553617747, - "nauc_cv_recall_at_3_diff1": 0.1741239602897772, - "nauc_cv_recall_at_3_max": 0.5191628655755299, - "nauc_cv_recall_at_3_std": 0.419125301851355, - "nauc_cv_recall_at_5_diff1": 0.13886250081581455, - "nauc_cv_recall_at_5_max": 0.43471671295658415, - "nauc_cv_recall_at_5_std": 0.3732079275411715, - "nauc_map_at_1000_diff1": 0.15194835205582552, - "nauc_map_at_1000_max": 0.40877748630514626, - "nauc_map_at_1000_std": 0.3731482014043823, - "nauc_map_at_100_diff1": 0.1513790258426513, - "nauc_map_at_100_max": 0.4168178083848067, - "nauc_map_at_100_std": 0.37797865889115057, - "nauc_map_at_10_diff1": 0.17883793655783767, - "nauc_map_at_10_max": 0.48375802812964486, - "nauc_map_at_10_std": 0.41700647583883665, - "nauc_map_at_1_diff1": 0.27867587872283334, - "nauc_map_at_1_max": 0.7182720686879527, - "nauc_map_at_1_std": 0.5737858867722029, - "nauc_map_at_20_diff1": 0.15818833869847163, - "nauc_map_at_20_max": 0.45599751664213256, - "nauc_map_at_20_std": 0.3962545579279782, - "nauc_map_at_3_diff1": 0.2025670489145016, - "nauc_map_at_3_max": 0.5618430722926705, - "nauc_map_at_3_std": 0.45777872399995345, - "nauc_map_at_5_diff1": 0.18630911748922815, - "nauc_map_at_5_max": 0.5126387924367496, - "nauc_map_at_5_std": 0.4302770226013669, - "nauc_mrr_at_1000_diff1": 0.15194835205582552, - "nauc_mrr_at_1000_max": 0.40877748630514626, - "nauc_mrr_at_1000_std": 0.3731482014043823, - "nauc_mrr_at_100_diff1": 0.1513790258426513, - "nauc_mrr_at_100_max": 0.4168178083848067, - "nauc_mrr_at_100_std": 0.37797865889115057, - "nauc_mrr_at_10_diff1": 0.17883793655783767, - "nauc_mrr_at_10_max": 0.48375802812964486, - "nauc_mrr_at_10_std": 0.41700647583883665, - "nauc_mrr_at_1_diff1": 0.27867587872283334, - "nauc_mrr_at_1_max": 0.7182720686879527, - "nauc_mrr_at_1_std": 0.5737858867722029, - "nauc_mrr_at_20_diff1": 0.15818833869847163, - "nauc_mrr_at_20_max": 0.45599751664213256, - "nauc_mrr_at_20_std": 0.3962545579279782, - "nauc_mrr_at_3_diff1": 0.2025670489145016, - "nauc_mrr_at_3_max": 0.5618430722926705, - "nauc_mrr_at_3_std": 0.45777872399995345, - "nauc_mrr_at_5_diff1": 0.18630911748922815, - "nauc_mrr_at_5_max": 0.5126387924367496, - "nauc_mrr_at_5_std": 0.4302770226013669, - "nauc_ndcg_at_1000_diff1": 0.13663974265837234, - "nauc_ndcg_at_1000_max": 0.27448743546385457, - "nauc_ndcg_at_1000_std": 0.3102772052076174, - "nauc_ndcg_at_100_diff1": 0.12070721039046628, - "nauc_ndcg_at_100_max": 0.30620942623287684, - "nauc_ndcg_at_100_std": 0.3236602531782627, - "nauc_ndcg_at_10_diff1": 0.16174756306693933, - "nauc_ndcg_at_10_max": 0.4415918844721654, - "nauc_ndcg_at_10_std": 0.3945274124410024, - "nauc_ndcg_at_1_diff1": 0.27867587872283334, - "nauc_ndcg_at_1_max": 0.7182720686879527, - "nauc_ndcg_at_1_std": 0.5737858867722029, - "nauc_ndcg_at_20_diff1": 0.11917990049729905, - "nauc_ndcg_at_20_max": 0.39048500817778287, - "nauc_ndcg_at_20_std": 0.3551788561055616, - "nauc_ndcg_at_3_diff1": 0.1922478271939251, - "nauc_ndcg_at_3_max": 0.5459087060547988, - "nauc_ndcg_at_3_std": 0.4436211622584078, - "nauc_ndcg_at_5_diff1": 0.16790385395307483, - "nauc_ndcg_at_5_max": 0.48136065712524156, - "nauc_ndcg_at_5_std": 0.40748577321050217, - "nauc_precision_at_1000_diff1": 0.1463224055326696, - "nauc_precision_at_1000_max": 0.20534263898503682, - "nauc_precision_at_1000_std": 0.289378808337388, - "nauc_precision_at_100_diff1": 0.11011108120551759, - "nauc_precision_at_100_max": 0.24934265204776102, - "nauc_precision_at_100_std": 0.29981869323072124, - "nauc_precision_at_10_diff1": 0.14244559614760016, - "nauc_precision_at_10_max": 0.3921737534068608, - "nauc_precision_at_10_std": 0.3703933835124439, - "nauc_precision_at_1_diff1": 0.27867587872283334, - "nauc_precision_at_1_max": 0.7182720686879527, - "nauc_precision_at_1_std": 0.5737858867722029, - "nauc_precision_at_20_diff1": 0.08256608787867174, - "nauc_precision_at_20_max": 0.3323674089974065, - "nauc_precision_at_20_std": 0.31922010553617763, - "nauc_precision_at_3_diff1": 0.1741239602897773, - "nauc_precision_at_3_max": 0.51916286557553, - "nauc_precision_at_3_std": 0.419125301851355, - "nauc_precision_at_5_diff1": 0.13886250081581444, - "nauc_precision_at_5_max": 0.43471671295658404, - "nauc_precision_at_5_std": 0.3732079275411712, - "nauc_recall_at_1000_diff1": 0.1463224055326704, - "nauc_recall_at_1000_max": 0.20534263898503774, - "nauc_recall_at_1000_std": 0.28937880833738816, - "nauc_recall_at_100_diff1": 0.11011108120551756, - "nauc_recall_at_100_max": 0.24934265204776085, - "nauc_recall_at_100_std": 0.2998186932307211, - "nauc_recall_at_10_diff1": 0.14244559614759988, - "nauc_recall_at_10_max": 0.3921737534068603, - "nauc_recall_at_10_std": 0.3703933835124436, - "nauc_recall_at_1_diff1": 0.27867587872283334, - "nauc_recall_at_1_max": 0.7182720686879527, - "nauc_recall_at_1_std": 0.5737858867722029, - "nauc_recall_at_20_diff1": 0.08256608787867177, - "nauc_recall_at_20_max": 0.3323674089974063, - "nauc_recall_at_20_std": 0.31922010553617747, - "nauc_recall_at_3_diff1": 0.1741239602897772, - "nauc_recall_at_3_max": 0.5191628655755299, - "nauc_recall_at_3_std": 0.419125301851355, - "nauc_recall_at_5_diff1": 0.13886250081581455, - "nauc_recall_at_5_max": 0.43471671295658415, - "nauc_recall_at_5_std": 0.3732079275411715, - "ndcg_at_1": 0.004, - "ndcg_at_10": 0.01454, - "ndcg_at_100": 0.03865, - "ndcg_at_1000": 0.09934, - "ndcg_at_20": 0.0205, - "ndcg_at_3": 0.0089, - "ndcg_at_5": 0.0114, - "precision_at_1": 0.004, - "precision_at_10": 0.00285, - "precision_at_100": 0.00154, - "precision_at_1000": 0.00068, - "precision_at_20": 0.00262, - "precision_at_3": 0.00417, - "precision_at_5": 0.0037, - "recall_at_1": 0.004, - "recall_at_10": 0.0285, - "recall_at_100": 0.1545, - "recall_at_1000": 0.676, - "recall_at_20": 0.0525, - "recall_at_3": 0.0125, - "recall_at_5": 0.0185 - } - ] - }, - "task_name": "XFlickr30kCoT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XM3600T2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XM3600T2IRetrieval.json deleted file mode 100644 index 2ed56d8198..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/XM3600T2IRetrieval.json +++ /dev/null @@ -1,6311 +0,0 @@ -{ - "dataset_revision": "8d3e5665526c55a5855cd6ddfbaba2032bc7cee4", - "evaluation_time": 1117.7640960216522, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.00231, - "cv_recall_at_10": 0.0076, - "cv_recall_at_100": 0.03909, - "cv_recall_at_1000": 0.31682, - "cv_recall_at_20": 0.01167, - "cv_recall_at_3": 0.00421, - "cv_recall_at_5": 0.00611, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.00466, - "map_at_1": 0.00231, - "map_at_10": 0.00375, - "map_at_100": 0.0046, - "map_at_1000": 0.00532, - "map_at_20": 0.00401, - "map_at_3": 0.0031, - "map_at_5": 0.00355, - "mrr_at_1": 0.0023075878919505905, - "mrr_at_10": 0.0037512308212729, - "mrr_at_100": 0.004595156575724333, - "mrr_at_1000": 0.005322854437046951, - "mrr_at_20": 0.004008086271771789, - "mrr_at_3": 0.003099407266639519, - "mrr_at_5": 0.0035473507986063975, - "nauc_cv_recall_at_1000_diff1": 0.02086085561171304, - "nauc_cv_recall_at_1000_max": 0.0659162316936909, - "nauc_cv_recall_at_1000_std": 0.043105470750601366, - "nauc_cv_recall_at_100_diff1": 0.121895728673165, - "nauc_cv_recall_at_100_max": 0.1638896678397787, - "nauc_cv_recall_at_100_std": 0.1526763117197299, - "nauc_cv_recall_at_10_diff1": 0.368207794124593, - "nauc_cv_recall_at_10_max": 0.3689105982750756, - "nauc_cv_recall_at_10_std": 0.33971394675418326, - "nauc_cv_recall_at_1_diff1": 0.6257674859330593, - "nauc_cv_recall_at_1_max": 0.691941523846895, - "nauc_cv_recall_at_1_std": 0.44349508908902135, - "nauc_cv_recall_at_20_diff1": 0.31845657498616187, - "nauc_cv_recall_at_20_max": 0.3064984576105181, - "nauc_cv_recall_at_20_std": 0.30547319280104007, - "nauc_cv_recall_at_3_diff1": 0.3879895446339794, - "nauc_cv_recall_at_3_max": 0.5360946810241998, - "nauc_cv_recall_at_3_std": 0.47829712213481257, - "nauc_cv_recall_at_5_diff1": 0.32946195996894806, - "nauc_cv_recall_at_5_max": 0.41995735231054454, - "nauc_cv_recall_at_5_std": 0.4275124824163301, - "nauc_map_at_1000_diff1": 0.3611149618443007, - "nauc_map_at_1000_max": 0.42087811795849395, - "nauc_map_at_1000_std": 0.33039782609147533, - "nauc_map_at_100_diff1": 0.40341655896005185, - "nauc_map_at_100_max": 0.46470868833845236, - "nauc_map_at_100_std": 0.3648103174068322, - "nauc_map_at_10_diff1": 0.4722863678408031, - "nauc_map_at_10_max": 0.5333535587266557, - "nauc_map_at_10_std": 0.40955214535310597, - "nauc_map_at_1_diff1": 0.6257674859330593, - "nauc_map_at_1_max": 0.691941523846895, - "nauc_map_at_1_std": 0.44349508908902135, - "nauc_map_at_20_diff1": 0.45771386169128625, - "nauc_map_at_20_max": 0.5132111217657302, - "nauc_map_at_20_std": 0.3993673769704931, - "nauc_map_at_3_diff1": 0.5049492976183295, - "nauc_map_at_3_max": 0.6133226050659266, - "nauc_map_at_3_std": 0.4539495984666943, - "nauc_map_at_5_diff1": 0.46805712656168214, - "nauc_map_at_5_max": 0.5564098464493543, - "nauc_map_at_5_std": 0.4345108623452712, - "nauc_mrr_at_1000_diff1": 0.3611149618443007, - "nauc_mrr_at_1000_max": 0.42087811795849395, - "nauc_mrr_at_1000_std": 0.33039782609147533, - "nauc_mrr_at_100_diff1": 0.40341655896005185, - "nauc_mrr_at_100_max": 0.46470868833845236, - "nauc_mrr_at_100_std": 0.3648103174068322, - "nauc_mrr_at_10_diff1": 0.4722863678408031, - "nauc_mrr_at_10_max": 0.5333535587266557, - "nauc_mrr_at_10_std": 0.40955214535310597, - "nauc_mrr_at_1_diff1": 0.6257674859330593, - "nauc_mrr_at_1_max": 0.691941523846895, - "nauc_mrr_at_1_std": 0.44349508908902135, - "nauc_mrr_at_20_diff1": 0.45771386169128625, - "nauc_mrr_at_20_max": 0.5132111217657302, - "nauc_mrr_at_20_std": 0.3993673769704931, - "nauc_mrr_at_3_diff1": 0.5049492976183295, - "nauc_mrr_at_3_max": 0.6133226050659266, - "nauc_mrr_at_3_std": 0.4539495984666943, - "nauc_mrr_at_5_diff1": 0.46805712656168214, - "nauc_mrr_at_5_max": 0.5564098464493543, - "nauc_mrr_at_5_std": 0.4345108623452712, - "nauc_ndcg_at_1000_diff1": 0.08034972935111898, - "nauc_ndcg_at_1000_max": 0.12690504164043506, - "nauc_ndcg_at_1000_std": 0.09785564212119391, - "nauc_ndcg_at_100_diff1": 0.23054253166440147, - "nauc_ndcg_at_100_max": 0.2790563706353781, - "nauc_ndcg_at_100_std": 0.23914173820141837, - "nauc_ndcg_at_10_diff1": 0.43064078578103404, - "nauc_ndcg_at_10_max": 0.47043726798904806, - "nauc_ndcg_at_10_std": 0.3846310774648549, - "nauc_ndcg_at_1_diff1": 0.6257674859330593, - "nauc_ndcg_at_1_max": 0.691941523846895, - "nauc_ndcg_at_1_std": 0.44349508908902135, - "nauc_ndcg_at_20_diff1": 0.39563088229431864, - "nauc_ndcg_at_20_max": 0.42274654860387567, - "nauc_ndcg_at_20_std": 0.3600233392323016, - "nauc_ndcg_at_3_diff1": 0.46834083139804267, - "nauc_ndcg_at_3_max": 0.5891689419024891, - "nauc_ndcg_at_3_std": 0.46133255794941463, - "nauc_ndcg_at_5_diff1": 0.4184780599766308, - "nauc_ndcg_at_5_max": 0.5080341676704876, - "nauc_ndcg_at_5_std": 0.43208940846013444, - "nauc_precision_at_1000_diff1": 0.02086085561171273, - "nauc_precision_at_1000_max": 0.06591623169369042, - "nauc_precision_at_1000_std": 0.04310547075060104, - "nauc_precision_at_100_diff1": 0.1218957286731651, - "nauc_precision_at_100_max": 0.1638896678397788, - "nauc_precision_at_100_std": 0.15267631171972992, - "nauc_precision_at_10_diff1": 0.36820779412459315, - "nauc_precision_at_10_max": 0.36891059827507583, - "nauc_precision_at_10_std": 0.3397139467541833, - "nauc_precision_at_1_diff1": 0.6257674859330593, - "nauc_precision_at_1_max": 0.691941523846895, - "nauc_precision_at_1_std": 0.44349508908902135, - "nauc_precision_at_20_diff1": 0.3184565749861623, - "nauc_precision_at_20_max": 0.3064984576105185, - "nauc_precision_at_20_std": 0.30547319280104024, - "nauc_precision_at_3_diff1": 0.3879895446339794, - "nauc_precision_at_3_max": 0.5360946810241999, - "nauc_precision_at_3_std": 0.4782971221348123, - "nauc_precision_at_5_diff1": 0.32946195996894834, - "nauc_precision_at_5_max": 0.4199573523105448, - "nauc_precision_at_5_std": 0.4275124824163303, - "nauc_recall_at_1000_diff1": 0.02086085561171304, - "nauc_recall_at_1000_max": 0.0659162316936909, - "nauc_recall_at_1000_std": 0.043105470750601366, - "nauc_recall_at_100_diff1": 0.121895728673165, - "nauc_recall_at_100_max": 0.1638896678397787, - "nauc_recall_at_100_std": 0.1526763117197299, - "nauc_recall_at_10_diff1": 0.368207794124593, - "nauc_recall_at_10_max": 0.3689105982750756, - "nauc_recall_at_10_std": 0.33971394675418326, - "nauc_recall_at_1_diff1": 0.6257674859330593, - "nauc_recall_at_1_max": 0.691941523846895, - "nauc_recall_at_1_std": 0.44349508908902135, - "nauc_recall_at_20_diff1": 0.31845657498616187, - "nauc_recall_at_20_max": 0.3064984576105181, - "nauc_recall_at_20_std": 0.30547319280104007, - "nauc_recall_at_3_diff1": 0.3879895446339794, - "nauc_recall_at_3_max": 0.5360946810241998, - "nauc_recall_at_3_std": 0.47829712213481257, - "nauc_recall_at_5_diff1": 0.32946195996894806, - "nauc_recall_at_5_max": 0.41995735231054454, - "nauc_recall_at_5_std": 0.4275124824163301, - "ndcg_at_1": 0.00231, - "ndcg_at_10": 0.00466, - "ndcg_at_100": 0.01051, - "ndcg_at_1000": 0.04216, - "ndcg_at_20": 0.00566, - "ndcg_at_3": 0.00338, - "ndcg_at_5": 0.00418, - "precision_at_1": 0.00231, - "precision_at_10": 0.00076, - "precision_at_100": 0.00039, - "precision_at_1000": 0.00032, - "precision_at_20": 0.00058, - "precision_at_3": 0.0014, - "precision_at_5": 0.00122, - "recall_at_1": 0.00231, - "recall_at_10": 0.0076, - "recall_at_100": 0.03909, - "recall_at_1000": 0.31682, - "recall_at_20": 0.01167, - "recall_at_3": 0.00421, - "recall_at_5": 0.00611 - }, - { - "cv_recall_at_1": 0.00028, - "cv_recall_at_10": 0.00333, - "cv_recall_at_100": 0.02972, - "cv_recall_at_1000": 0.28056, - "cv_recall_at_20": 0.00528, - "cv_recall_at_3": 0.00056, - "cv_recall_at_5": 0.00194, - "hf_subset": "bn", - "languages": [ - "ben-Beng" - ], - "main_score": 0.00141, - "map_at_1": 0.00028, - "map_at_10": 0.00085, - "map_at_100": 0.00146, - "map_at_1000": 0.00208, - "map_at_20": 0.00098, - "map_at_3": 0.00037, - "map_at_5": 0.00069, - "mrr_at_1": 0.0002777777777777778, - "mrr_at_10": 0.000849867724867725, - "mrr_at_100": 0.0014617241741730137, - "mrr_at_1000": 0.002083443888849066, - "mrr_at_20": 0.0009788853969388029, - "mrr_at_3": 0.00037037037037037035, - "mrr_at_5": 0.0006898148148148149, - "nauc_cv_recall_at_1000_diff1": 0.016552384422470947, - "nauc_cv_recall_at_1000_max": 0.10185474461891487, - "nauc_cv_recall_at_1000_std": -0.03476725571443525, - "nauc_cv_recall_at_100_diff1": 0.03830095614472818, - "nauc_cv_recall_at_100_max": 0.10712431887539524, - "nauc_cv_recall_at_100_std": -0.027134650153339247, - "nauc_cv_recall_at_10_diff1": -0.1874161524013952, - "nauc_cv_recall_at_10_max": -0.2595027278418746, - "nauc_cv_recall_at_10_std": -0.029380198551113474, - "nauc_cv_recall_at_1_diff1": -0.2057955460155622, - "nauc_cv_recall_at_1_max": -0.4995975315266971, - "nauc_cv_recall_at_1_std": -0.3184867185403811, - "nauc_cv_recall_at_20_diff1": 0.069747080338356, - "nauc_cv_recall_at_20_max": -0.09340092920791382, - "nauc_cv_recall_at_20_std": 0.01042181520342312, - "nauc_cv_recall_at_3_diff1": 0.22806546820499068, - "nauc_cv_recall_at_3_max": -0.40904212503353915, - "nauc_cv_recall_at_3_std": 0.0026831231553529154, - "nauc_cv_recall_at_5_diff1": -0.0941392924220936, - "nauc_cv_recall_at_5_max": -0.22921537812871332, - "nauc_cv_recall_at_5_std": -0.0891946797500862, - "nauc_map_at_1000_diff1": -0.0202846438557245, - "nauc_map_at_1000_max": -0.06063474992068149, - "nauc_map_at_1000_std": -0.06367760673418538, - "nauc_map_at_100_diff1": -0.027108134687678215, - "nauc_map_at_100_max": -0.11170282021618887, - "nauc_map_at_100_std": -0.07244311982233366, - "nauc_map_at_10_diff1": -0.13723183123677368, - "nauc_map_at_10_max": -0.3065024497854023, - "nauc_map_at_10_std": -0.11363684293498172, - "nauc_map_at_1_diff1": -0.2057955460155622, - "nauc_map_at_1_max": -0.4995975315266971, - "nauc_map_at_1_std": -0.3184867185403811, - "nauc_map_at_20_diff1": -0.057597244728021066, - "nauc_map_at_20_max": -0.23585465837190886, - "nauc_map_at_20_std": -0.08564537656879898, - "nauc_map_at_3_diff1": 0.011134961094714114, - "nauc_map_at_3_max": -0.4543198282801179, - "nauc_map_at_3_std": -0.15790179769251397, - "nauc_map_at_5_diff1": -0.09377965616407036, - "nauc_map_at_5_max": -0.3051161216085054, - "nauc_map_at_5_std": -0.14828307129364351, - "nauc_mrr_at_1000_diff1": -0.0202846438557245, - "nauc_mrr_at_1000_max": -0.06063474992068149, - "nauc_mrr_at_1000_std": -0.06367760673418538, - "nauc_mrr_at_100_diff1": -0.027108134687678215, - "nauc_mrr_at_100_max": -0.11170282021618887, - "nauc_mrr_at_100_std": -0.07244311982233366, - "nauc_mrr_at_10_diff1": -0.13723183123677368, - "nauc_mrr_at_10_max": -0.3065024497854023, - "nauc_mrr_at_10_std": -0.11363684293498172, - "nauc_mrr_at_1_diff1": -0.2057955460155622, - "nauc_mrr_at_1_max": -0.4995975315266971, - "nauc_mrr_at_1_std": -0.3184867185403811, - "nauc_mrr_at_20_diff1": -0.057597244728021066, - "nauc_mrr_at_20_max": -0.23585465837190886, - "nauc_mrr_at_20_std": -0.08564537656879898, - "nauc_mrr_at_3_diff1": 0.011134961094714114, - "nauc_mrr_at_3_max": -0.4543198282801179, - "nauc_mrr_at_3_std": -0.15790179769251397, - "nauc_mrr_at_5_diff1": -0.09377965616407036, - "nauc_mrr_at_5_max": -0.3051161216085054, - "nauc_mrr_at_5_std": -0.14828307129364351, - "nauc_ndcg_at_1000_diff1": 0.01392467040584063, - "nauc_ndcg_at_1000_max": 0.08760122688186933, - "nauc_ndcg_at_1000_std": -0.03651546233738003, - "nauc_ndcg_at_100_diff1": 0.023885256557399884, - "nauc_ndcg_at_100_max": 0.053667739728740935, - "nauc_ndcg_at_100_std": -0.03498338816742055, - "nauc_ndcg_at_10_diff1": -0.16040888984827062, - "nauc_ndcg_at_10_max": -0.27991923638992355, - "nauc_ndcg_at_10_std": -0.06988154388981663, - "nauc_ndcg_at_1_diff1": -0.2057955460155622, - "nauc_ndcg_at_1_max": -0.4995975315266971, - "nauc_ndcg_at_1_std": -0.3184867185403811, - "nauc_ndcg_at_20_diff1": 0.007121822135668618, - "nauc_ndcg_at_20_max": -0.1563107432414096, - "nauc_ndcg_at_20_std": -0.030328522176883817, - "nauc_ndcg_at_3_diff1": 0.08344513013147309, - "nauc_ndcg_at_3_max": -0.43922726053125843, - "nauc_ndcg_at_3_std": -0.1043734907432251, - "nauc_ndcg_at_5_diff1": -0.09195109198698515, - "nauc_ndcg_at_5_max": -0.268777979300934, - "nauc_ndcg_at_5_std": -0.12003272556917093, - "nauc_precision_at_1000_diff1": 0.016552384422470486, - "nauc_precision_at_1000_max": 0.10185474461891456, - "nauc_precision_at_1000_std": -0.03476725571443556, - "nauc_precision_at_100_diff1": 0.038300956144728254, - "nauc_precision_at_100_max": 0.10712431887539543, - "nauc_precision_at_100_std": -0.027134650153339105, - "nauc_precision_at_10_diff1": -0.1874161524013952, - "nauc_precision_at_10_max": -0.25950272784187467, - "nauc_precision_at_10_std": -0.0293801985511136, - "nauc_precision_at_1_diff1": -0.2057955460155622, - "nauc_precision_at_1_max": -0.4995975315266971, - "nauc_precision_at_1_std": -0.3184867185403811, - "nauc_precision_at_20_diff1": 0.06974708033835594, - "nauc_precision_at_20_max": -0.09340092920791385, - "nauc_precision_at_20_std": 0.010421815203423114, - "nauc_precision_at_3_diff1": 0.22806546820499055, - "nauc_precision_at_3_max": -0.409042125033539, - "nauc_precision_at_3_std": 0.0026831231553529475, - "nauc_precision_at_5_diff1": -0.09413929242209361, - "nauc_precision_at_5_max": -0.22921537812871337, - "nauc_precision_at_5_std": -0.08919467975008624, - "nauc_recall_at_1000_diff1": 0.016552384422470947, - "nauc_recall_at_1000_max": 0.10185474461891487, - "nauc_recall_at_1000_std": -0.03476725571443525, - "nauc_recall_at_100_diff1": 0.03830095614472818, - "nauc_recall_at_100_max": 0.10712431887539524, - "nauc_recall_at_100_std": -0.027134650153339247, - "nauc_recall_at_10_diff1": -0.1874161524013952, - "nauc_recall_at_10_max": -0.2595027278418746, - "nauc_recall_at_10_std": -0.029380198551113474, - "nauc_recall_at_1_diff1": -0.2057955460155622, - "nauc_recall_at_1_max": -0.4995975315266971, - "nauc_recall_at_1_std": -0.3184867185403811, - "nauc_recall_at_20_diff1": 0.069747080338356, - "nauc_recall_at_20_max": -0.09340092920791382, - "nauc_recall_at_20_std": 0.01042181520342312, - "nauc_recall_at_3_diff1": 0.22806546820499068, - "nauc_recall_at_3_max": -0.40904212503353915, - "nauc_recall_at_3_std": 0.0026831231553529154, - "nauc_recall_at_5_diff1": -0.0941392924220936, - "nauc_recall_at_5_max": -0.22921537812871332, - "nauc_recall_at_5_std": -0.0891946797500862, - "ndcg_at_1": 0.00028, - "ndcg_at_10": 0.00141, - "ndcg_at_100": 0.00615, - "ndcg_at_1000": 0.03451, - "ndcg_at_20": 0.0019, - "ndcg_at_3": 0.00042, - "ndcg_at_5": 0.00099, - "precision_at_1": 0.00028, - "precision_at_10": 0.00033, - "precision_at_100": 0.0003, - "precision_at_1000": 0.00028, - "precision_at_20": 0.00026, - "precision_at_3": 0.00019, - "precision_at_5": 0.00039, - "recall_at_1": 0.00028, - "recall_at_10": 0.00333, - "recall_at_100": 0.02972, - "recall_at_1000": 0.28056, - "recall_at_20": 0.00528, - "recall_at_3": 0.00056, - "recall_at_5": 0.00194 - }, - { - "cv_recall_at_1": 0.01762, - "cv_recall_at_10": 0.05162, - "cv_recall_at_100": 0.1221, - "cv_recall_at_1000": 0.42098, - "cv_recall_at_20": 0.06771, - "cv_recall_at_3": 0.03122, - "cv_recall_at_5": 0.03857, - "hf_subset": "cs", - "languages": [ - "ces-Latn" - ], - "main_score": 0.03255, - "map_at_1": 0.01762, - "map_at_10": 0.02672, - "map_at_100": 0.02906, - "map_at_1000": 0.02993, - "map_at_20": 0.02784, - "map_at_3": 0.02338, - "map_at_5": 0.02506, - "mrr_at_1": 0.01762175662550298, - "mrr_at_10": 0.026723522765565222, - "mrr_at_100": 0.029056966856710383, - "mrr_at_1000": 0.029929262097212285, - "mrr_at_20": 0.027837486156330166, - "mrr_at_3": 0.023380047176356323, - "mrr_at_5": 0.025058970445400314, - "nauc_cv_recall_at_1000_diff1": 0.130452831771586, - "nauc_cv_recall_at_1000_max": 0.2814459080268943, - "nauc_cv_recall_at_1000_std": 0.3298758134400155, - "nauc_cv_recall_at_100_diff1": 0.20707211860438848, - "nauc_cv_recall_at_100_max": 0.43864028814097267, - "nauc_cv_recall_at_100_std": 0.5045396386605944, - "nauc_cv_recall_at_10_diff1": 0.3018499527889426, - "nauc_cv_recall_at_10_max": 0.5588325128414509, - "nauc_cv_recall_at_10_std": 0.6018154506928893, - "nauc_cv_recall_at_1_diff1": 0.5227812130926891, - "nauc_cv_recall_at_1_max": 0.6728450969099916, - "nauc_cv_recall_at_1_std": 0.5969588549628254, - "nauc_cv_recall_at_20_diff1": 0.25978593693778945, - "nauc_cv_recall_at_20_max": 0.5488017653895927, - "nauc_cv_recall_at_20_std": 0.5966074067105238, - "nauc_cv_recall_at_3_diff1": 0.37876172952921305, - "nauc_cv_recall_at_3_max": 0.6228517843168137, - "nauc_cv_recall_at_3_std": 0.6315853508334851, - "nauc_cv_recall_at_5_diff1": 0.3407870427930846, - "nauc_cv_recall_at_5_max": 0.5891937579864478, - "nauc_cv_recall_at_5_std": 0.6205552352479096, - "nauc_map_at_1000_diff1": 0.3868841569445223, - "nauc_map_at_1000_max": 0.6000870613766743, - "nauc_map_at_1000_std": 0.5952722337740114, - "nauc_map_at_100_diff1": 0.39070161741982057, - "nauc_map_at_100_max": 0.6059801916711482, - "nauc_map_at_100_std": 0.6006444295906668, - "nauc_map_at_10_diff1": 0.4125970705509705, - "nauc_map_at_10_max": 0.6225128813748829, - "nauc_map_at_10_std": 0.609612837662559, - "nauc_map_at_1_diff1": 0.5227812130926891, - "nauc_map_at_1_max": 0.6728450969099916, - "nauc_map_at_1_std": 0.5969588549628254, - "nauc_map_at_20_diff1": 0.4011252779432377, - "nauc_map_at_20_max": 0.6183272754038512, - "nauc_map_at_20_std": 0.6084821498947195, - "nauc_map_at_3_diff1": 0.4438398201237769, - "nauc_map_at_3_max": 0.6437136529111982, - "nauc_map_at_3_std": 0.6152275089611549, - "nauc_map_at_5_diff1": 0.4264761762444746, - "nauc_map_at_5_max": 0.6316489699199355, - "nauc_map_at_5_std": 0.6130112758334513, - "nauc_mrr_at_1000_diff1": 0.3868841569445223, - "nauc_mrr_at_1000_max": 0.6000870613766743, - "nauc_mrr_at_1000_std": 0.5952722337740114, - "nauc_mrr_at_100_diff1": 0.39070161741982057, - "nauc_mrr_at_100_max": 0.6059801916711482, - "nauc_mrr_at_100_std": 0.6006444295906668, - "nauc_mrr_at_10_diff1": 0.4125970705509705, - "nauc_mrr_at_10_max": 0.6225128813748829, - "nauc_mrr_at_10_std": 0.609612837662559, - "nauc_mrr_at_1_diff1": 0.5227812130926891, - "nauc_mrr_at_1_max": 0.6728450969099916, - "nauc_mrr_at_1_std": 0.5969588549628254, - "nauc_mrr_at_20_diff1": 0.4011252779432377, - "nauc_mrr_at_20_max": 0.6183272754038512, - "nauc_mrr_at_20_std": 0.6084821498947195, - "nauc_mrr_at_3_diff1": 0.4438398201237769, - "nauc_mrr_at_3_max": 0.6437136529111982, - "nauc_mrr_at_3_std": 0.6152275089611549, - "nauc_mrr_at_5_diff1": 0.4264761762444746, - "nauc_mrr_at_5_max": 0.6316489699199355, - "nauc_mrr_at_5_std": 0.6130112758334513, - "nauc_ndcg_at_1000_diff1": 0.2525598663486497, - "nauc_ndcg_at_1000_max": 0.445635242025813, - "nauc_ndcg_at_1000_std": 0.474868093318983, - "nauc_ndcg_at_100_diff1": 0.30412139279929973, - "nauc_ndcg_at_100_max": 0.5326396104845605, - "nauc_ndcg_at_100_std": 0.5622182232718004, - "nauc_ndcg_at_10_diff1": 0.3717663380113828, - "nauc_ndcg_at_10_max": 0.5996211807967179, - "nauc_ndcg_at_10_std": 0.6075999987669697, - "nauc_ndcg_at_1_diff1": 0.5227812130926891, - "nauc_ndcg_at_1_max": 0.6728450969099916, - "nauc_ndcg_at_1_std": 0.5969588549628254, - "nauc_ndcg_at_20_diff1": 0.3443968913567033, - "nauc_ndcg_at_20_max": 0.5904537522253336, - "nauc_ndcg_at_20_std": 0.604579390794644, - "nauc_ndcg_at_3_diff1": 0.42347904622269117, - "nauc_ndcg_at_3_max": 0.6371236760291885, - "nauc_ndcg_at_3_std": 0.6203194196263287, - "nauc_ndcg_at_5_diff1": 0.3977540111389816, - "nauc_ndcg_at_5_max": 0.6176889358407618, - "nauc_ndcg_at_5_std": 0.615791094672046, - "nauc_precision_at_1000_diff1": 0.13045283177158534, - "nauc_precision_at_1000_max": 0.28144590802689373, - "nauc_precision_at_1000_std": 0.3298758134400153, - "nauc_precision_at_100_diff1": 0.2070721186043888, - "nauc_precision_at_100_max": 0.43864028814097294, - "nauc_precision_at_100_std": 0.5045396386605943, - "nauc_precision_at_10_diff1": 0.3018499527889427, - "nauc_precision_at_10_max": 0.558832512841451, - "nauc_precision_at_10_std": 0.6018154506928893, - "nauc_precision_at_1_diff1": 0.5227812130926891, - "nauc_precision_at_1_max": 0.6728450969099916, - "nauc_precision_at_1_std": 0.5969588549628254, - "nauc_precision_at_20_diff1": 0.2597859369377895, - "nauc_precision_at_20_max": 0.5488017653895926, - "nauc_precision_at_20_std": 0.596607406710524, - "nauc_precision_at_3_diff1": 0.3787617295292129, - "nauc_precision_at_3_max": 0.6228517843168137, - "nauc_precision_at_3_std": 0.6315853508334855, - "nauc_precision_at_5_diff1": 0.34078704279308475, - "nauc_precision_at_5_max": 0.5891937579864481, - "nauc_precision_at_5_std": 0.6205552352479098, - "nauc_recall_at_1000_diff1": 0.130452831771586, - "nauc_recall_at_1000_max": 0.2814459080268943, - "nauc_recall_at_1000_std": 0.3298758134400155, - "nauc_recall_at_100_diff1": 0.20707211860438848, - "nauc_recall_at_100_max": 0.43864028814097267, - "nauc_recall_at_100_std": 0.5045396386605944, - "nauc_recall_at_10_diff1": 0.3018499527889426, - "nauc_recall_at_10_max": 0.5588325128414509, - "nauc_recall_at_10_std": 0.6018154506928893, - "nauc_recall_at_1_diff1": 0.5227812130926891, - "nauc_recall_at_1_max": 0.6728450969099916, - "nauc_recall_at_1_std": 0.5969588549628254, - "nauc_recall_at_20_diff1": 0.25978593693778945, - "nauc_recall_at_20_max": 0.5488017653895927, - "nauc_recall_at_20_std": 0.5966074067105238, - "nauc_recall_at_3_diff1": 0.37876172952921305, - "nauc_recall_at_3_max": 0.6228517843168137, - "nauc_recall_at_3_std": 0.6315853508334851, - "nauc_recall_at_5_diff1": 0.3407870427930846, - "nauc_recall_at_5_max": 0.5891937579864478, - "nauc_recall_at_5_std": 0.6205552352479096, - "ndcg_at_1": 0.01762, - "ndcg_at_10": 0.03255, - "ndcg_at_100": 0.04634, - "ndcg_at_1000": 0.08096, - "ndcg_at_20": 0.03662, - "ndcg_at_3": 0.02538, - "ndcg_at_5": 0.02841, - "precision_at_1": 0.01762, - "precision_at_10": 0.00516, - "precision_at_100": 0.00122, - "precision_at_1000": 0.00042, - "precision_at_20": 0.00339, - "precision_at_3": 0.01041, - "precision_at_5": 0.00771, - "recall_at_1": 0.01762, - "recall_at_10": 0.05162, - "recall_at_100": 0.1221, - "recall_at_1000": 0.42098, - "recall_at_20": 0.06771, - "recall_at_3": 0.03122, - "recall_at_5": 0.03857 - }, - { - "cv_recall_at_1": 0.03373, - "cv_recall_at_10": 0.12252, - "cv_recall_at_100": 0.28621, - "cv_recall_at_1000": 0.64262, - "cv_recall_at_20": 0.1579, - "cv_recall_at_3": 0.06814, - "cv_recall_at_5": 0.08783, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.07269, - "map_at_1": 0.03373, - "map_at_10": 0.05742, - "map_at_100": 0.06286, - "map_at_1000": 0.06404, - "map_at_20": 0.05987, - "map_at_3": 0.04825, - "map_at_5": 0.05279, - "mrr_at_1": 0.03372797356828194, - "mrr_at_10": 0.057418679200755156, - "mrr_at_100": 0.0628598667569856, - "mrr_at_1000": 0.06403826011730678, - "mrr_at_20": 0.05987001474381017, - "mrr_at_3": 0.04825165198237881, - "mrr_at_5": 0.05279460352422895, - "nauc_cv_recall_at_1000_diff1": 0.11049894267922444, - "nauc_cv_recall_at_1000_max": 0.22451665754549788, - "nauc_cv_recall_at_1000_std": 0.48731589245366713, - "nauc_cv_recall_at_100_diff1": 0.12244855581388936, - "nauc_cv_recall_at_100_max": 0.23170326856395815, - "nauc_cv_recall_at_100_std": 0.4746093691675264, - "nauc_cv_recall_at_10_diff1": 0.17701327797996758, - "nauc_cv_recall_at_10_max": 0.3015191441210729, - "nauc_cv_recall_at_10_std": 0.4539356653217957, - "nauc_cv_recall_at_1_diff1": 0.3855984094356238, - "nauc_cv_recall_at_1_max": 0.4732925626042209, - "nauc_cv_recall_at_1_std": 0.4691654722763578, - "nauc_cv_recall_at_20_diff1": 0.15264152564379713, - "nauc_cv_recall_at_20_max": 0.28317527597564235, - "nauc_cv_recall_at_20_std": 0.45334028568180784, - "nauc_cv_recall_at_3_diff1": 0.24145222405286212, - "nauc_cv_recall_at_3_max": 0.39572480799685716, - "nauc_cv_recall_at_3_std": 0.4564294029992678, - "nauc_cv_recall_at_5_diff1": 0.2013629050318432, - "nauc_cv_recall_at_5_max": 0.33924257352135545, - "nauc_cv_recall_at_5_std": 0.4370634014365294, - "nauc_map_at_1000_diff1": 0.2571905858974858, - "nauc_map_at_1000_max": 0.375801399606642, - "nauc_map_at_1000_std": 0.4583349631543094, - "nauc_map_at_100_diff1": 0.25762509405172734, - "nauc_map_at_100_max": 0.37631945451142995, - "nauc_map_at_100_std": 0.4581332450756916, - "nauc_map_at_10_diff1": 0.2690954142123744, - "nauc_map_at_10_max": 0.38870643006079597, - "nauc_map_at_10_std": 0.45732740396293864, - "nauc_map_at_1_diff1": 0.3855984094356238, - "nauc_map_at_1_max": 0.4732925626042209, - "nauc_map_at_1_std": 0.4691654722763578, - "nauc_map_at_20_diff1": 0.2626524057901557, - "nauc_map_at_20_max": 0.3830007580042664, - "nauc_map_at_20_std": 0.4572309000202074, - "nauc_map_at_3_diff1": 0.3014548144513771, - "nauc_map_at_3_max": 0.42829192851781556, - "nauc_map_at_3_std": 0.4630579329096384, - "nauc_map_at_5_diff1": 0.28178452153706796, - "nauc_map_at_5_max": 0.40429588937500843, - "nauc_map_at_5_std": 0.4551840498859357, - "nauc_mrr_at_1000_diff1": 0.2571905858974858, - "nauc_mrr_at_1000_max": 0.375801399606642, - "nauc_mrr_at_1000_std": 0.4583349631543094, - "nauc_mrr_at_100_diff1": 0.25762509405172734, - "nauc_mrr_at_100_max": 0.37631945451142995, - "nauc_mrr_at_100_std": 0.4581332450756916, - "nauc_mrr_at_10_diff1": 0.2690954142123744, - "nauc_mrr_at_10_max": 0.38870643006079597, - "nauc_mrr_at_10_std": 0.45732740396293864, - "nauc_mrr_at_1_diff1": 0.3855984094356238, - "nauc_mrr_at_1_max": 0.4732925626042209, - "nauc_mrr_at_1_std": 0.4691654722763578, - "nauc_mrr_at_20_diff1": 0.2626524057901557, - "nauc_mrr_at_20_max": 0.3830007580042664, - "nauc_mrr_at_20_std": 0.4572309000202074, - "nauc_mrr_at_3_diff1": 0.3014548144513771, - "nauc_mrr_at_3_max": 0.42829192851781556, - "nauc_mrr_at_3_std": 0.4630579329096384, - "nauc_mrr_at_5_diff1": 0.28178452153706796, - "nauc_mrr_at_5_max": 0.40429588937500843, - "nauc_mrr_at_5_std": 0.4551840498859357, - "nauc_ndcg_at_1000_diff1": 0.19627469020943997, - "nauc_ndcg_at_1000_max": 0.31360426152518983, - "nauc_ndcg_at_1000_std": 0.4658878490099195, - "nauc_ndcg_at_100_diff1": 0.19799771617987885, - "nauc_ndcg_at_100_max": 0.31445442190760137, - "nauc_ndcg_at_100_std": 0.46270076157765944, - "nauc_ndcg_at_10_diff1": 0.23346397488496937, - "nauc_ndcg_at_10_max": 0.3556214902489086, - "nauc_ndcg_at_10_std": 0.455413729729024, - "nauc_ndcg_at_1_diff1": 0.3855984094356238, - "nauc_ndcg_at_1_max": 0.4732925626042209, - "nauc_ndcg_at_1_std": 0.4691654722763578, - "nauc_ndcg_at_20_diff1": 0.21803715738452054, - "nauc_ndcg_at_20_max": 0.34277709628744024, - "nauc_ndcg_at_20_std": 0.4551131267039593, - "nauc_ndcg_at_3_diff1": 0.28193549803328094, - "nauc_ndcg_at_3_max": 0.41770762655518223, - "nauc_ndcg_at_3_std": 0.4609494373853862, - "nauc_ndcg_at_5_diff1": 0.2535360527632714, - "nauc_ndcg_at_5_max": 0.3817724923731276, - "nauc_ndcg_at_5_std": 0.4490004569904931, - "nauc_precision_at_1000_diff1": 0.11049894267922375, - "nauc_precision_at_1000_max": 0.2245166575454968, - "nauc_precision_at_1000_std": 0.4873158924536665, - "nauc_precision_at_100_diff1": 0.12244855581388976, - "nauc_precision_at_100_max": 0.2317032685639583, - "nauc_precision_at_100_std": 0.47460936916752666, - "nauc_precision_at_10_diff1": 0.17701327797996744, - "nauc_precision_at_10_max": 0.30151914412107284, - "nauc_precision_at_10_std": 0.4539356653217955, - "nauc_precision_at_1_diff1": 0.3855984094356238, - "nauc_precision_at_1_max": 0.4732925626042209, - "nauc_precision_at_1_std": 0.4691654722763578, - "nauc_precision_at_20_diff1": 0.15264152564379704, - "nauc_precision_at_20_max": 0.2831752759756425, - "nauc_precision_at_20_std": 0.453340285681808, - "nauc_precision_at_3_diff1": 0.24145222405286218, - "nauc_precision_at_3_max": 0.3957248079968571, - "nauc_precision_at_3_std": 0.4564294029992675, - "nauc_precision_at_5_diff1": 0.20136290503184318, - "nauc_precision_at_5_max": 0.3392425735213555, - "nauc_precision_at_5_std": 0.4370634014365295, - "nauc_recall_at_1000_diff1": 0.11049894267922444, - "nauc_recall_at_1000_max": 0.22451665754549788, - "nauc_recall_at_1000_std": 0.48731589245366713, - "nauc_recall_at_100_diff1": 0.12244855581388936, - "nauc_recall_at_100_max": 0.23170326856395815, - "nauc_recall_at_100_std": 0.4746093691675264, - "nauc_recall_at_10_diff1": 0.17701327797996758, - "nauc_recall_at_10_max": 0.3015191441210729, - "nauc_recall_at_10_std": 0.4539356653217957, - "nauc_recall_at_1_diff1": 0.3855984094356238, - "nauc_recall_at_1_max": 0.4732925626042209, - "nauc_recall_at_1_std": 0.4691654722763578, - "nauc_recall_at_20_diff1": 0.15264152564379713, - "nauc_recall_at_20_max": 0.28317527597564235, - "nauc_recall_at_20_std": 0.45334028568180784, - "nauc_recall_at_3_diff1": 0.24145222405286212, - "nauc_recall_at_3_max": 0.39572480799685716, - "nauc_recall_at_3_std": 0.4564294029992678, - "nauc_recall_at_5_diff1": 0.2013629050318432, - "nauc_recall_at_5_max": 0.33924257352135545, - "nauc_recall_at_5_std": 0.4370634014365294, - "ndcg_at_1": 0.03373, - "ndcg_at_10": 0.07269, - "ndcg_at_100": 0.10483, - "ndcg_at_1000": 0.14699, - "ndcg_at_20": 0.08163, - "ndcg_at_3": 0.05333, - "ndcg_at_5": 0.06148, - "precision_at_1": 0.03373, - "precision_at_10": 0.01225, - "precision_at_100": 0.00286, - "precision_at_1000": 0.00064, - "precision_at_20": 0.0079, - "precision_at_3": 0.02271, - "precision_at_5": 0.01757, - "recall_at_1": 0.03373, - "recall_at_10": 0.12252, - "recall_at_100": 0.28621, - "recall_at_1000": 0.64262, - "recall_at_20": 0.1579, - "recall_at_3": 0.06814, - "recall_at_5": 0.08783 - }, - { - "cv_recall_at_1": 0.0995, - "cv_recall_at_10": 0.28694, - "cv_recall_at_100": 0.56878, - "cv_recall_at_1000": 0.86683, - "cv_recall_at_20": 0.36307, - "cv_recall_at_3": 0.17691, - "cv_recall_at_5": 0.22319, - "hf_subset": "de", - "languages": [ - "deu-Latn" - ], - "main_score": 0.184, - "map_at_1": 0.0995, - "map_at_10": 0.15218, - "map_at_100": 0.16243, - "map_at_1000": 0.16364, - "map_at_20": 0.1574, - "map_at_3": 0.13311, - "map_at_5": 0.14368, - "mrr_at_1": 0.09950248756218906, - "mrr_at_10": 0.1521787518663603, - "mrr_at_100": 0.16243341041355344, - "mrr_at_1000": 0.16363593463741677, - "mrr_at_20": 0.15740114488028334, - "mrr_at_3": 0.13311350225616123, - "mrr_at_5": 0.1436769640171242, - "nauc_cv_recall_at_1000_diff1": 0.20657275023501265, - "nauc_cv_recall_at_1000_max": 0.5421799124337883, - "nauc_cv_recall_at_1000_std": 0.688794162507746, - "nauc_cv_recall_at_100_diff1": 0.18604652154929796, - "nauc_cv_recall_at_100_max": 0.37685297250936006, - "nauc_cv_recall_at_100_std": 0.4528221311140118, - "nauc_cv_recall_at_10_diff1": 0.22549313028612575, - "nauc_cv_recall_at_10_max": 0.3161762107844544, - "nauc_cv_recall_at_10_std": 0.26299428936782887, - "nauc_cv_recall_at_1_diff1": 0.3948317473041426, - "nauc_cv_recall_at_1_max": 0.375502342242583, - "nauc_cv_recall_at_1_std": 0.2023848866968725, - "nauc_cv_recall_at_20_diff1": 0.22005941352512404, - "nauc_cv_recall_at_20_max": 0.33182507120198707, - "nauc_cv_recall_at_20_std": 0.3187275546938682, - "nauc_cv_recall_at_3_diff1": 0.27936582648185887, - "nauc_cv_recall_at_3_max": 0.3486223843295035, - "nauc_cv_recall_at_3_std": 0.23765876004992492, - "nauc_cv_recall_at_5_diff1": 0.24286332116512424, - "nauc_cv_recall_at_5_max": 0.32743243092200136, - "nauc_cv_recall_at_5_std": 0.2431073992347361, - "nauc_map_at_1000_diff1": 0.3120378155628561, - "nauc_map_at_1000_max": 0.3549682164024318, - "nauc_map_at_1000_std": 0.2387843964547122, - "nauc_map_at_100_diff1": 0.3116763503762316, - "nauc_map_at_100_max": 0.3545353494643112, - "nauc_map_at_100_std": 0.23852763645291886, - "nauc_map_at_10_diff1": 0.3142076734004048, - "nauc_map_at_10_max": 0.3517818416606607, - "nauc_map_at_10_std": 0.22774463525698088, - "nauc_map_at_1_diff1": 0.3948317473041426, - "nauc_map_at_1_max": 0.375502342242583, - "nauc_map_at_1_std": 0.2023848866968725, - "nauc_map_at_20_diff1": 0.3127906116286042, - "nauc_map_at_20_max": 0.35330490228525435, - "nauc_map_at_20_std": 0.23389994953259716, - "nauc_map_at_3_diff1": 0.3335696361095251, - "nauc_map_at_3_max": 0.36261707635010537, - "nauc_map_at_3_std": 0.22179461625406885, - "nauc_map_at_5_diff1": 0.3192034566172024, - "nauc_map_at_5_max": 0.3546817463773264, - "nauc_map_at_5_std": 0.22391567292597164, - "nauc_mrr_at_1000_diff1": 0.3120378155628561, - "nauc_mrr_at_1000_max": 0.3549682164024318, - "nauc_mrr_at_1000_std": 0.2387843964547122, - "nauc_mrr_at_100_diff1": 0.3116763503762316, - "nauc_mrr_at_100_max": 0.3545353494643112, - "nauc_mrr_at_100_std": 0.23852763645291886, - "nauc_mrr_at_10_diff1": 0.3142076734004048, - "nauc_mrr_at_10_max": 0.3517818416606607, - "nauc_mrr_at_10_std": 0.22774463525698088, - "nauc_mrr_at_1_diff1": 0.3948317473041426, - "nauc_mrr_at_1_max": 0.375502342242583, - "nauc_mrr_at_1_std": 0.2023848866968725, - "nauc_mrr_at_20_diff1": 0.3127906116286042, - "nauc_mrr_at_20_max": 0.35330490228525435, - "nauc_mrr_at_20_std": 0.23389994953259716, - "nauc_mrr_at_3_diff1": 0.3335696361095251, - "nauc_mrr_at_3_max": 0.36261707635010537, - "nauc_mrr_at_3_std": 0.22179461625406885, - "nauc_mrr_at_5_diff1": 0.3192034566172024, - "nauc_mrr_at_5_max": 0.3546817463773264, - "nauc_mrr_at_5_std": 0.22391567292597164, - "nauc_ndcg_at_1000_diff1": 0.28551352733572516, - "nauc_ndcg_at_1000_max": 0.3656732372768745, - "nauc_ndcg_at_1000_std": 0.2905152999979401, - "nauc_ndcg_at_100_diff1": 0.2752630378426891, - "nauc_ndcg_at_100_max": 0.35575148947476287, - "nauc_ndcg_at_100_std": 0.2889703869897625, - "nauc_ndcg_at_10_diff1": 0.2855811374048819, - "nauc_ndcg_at_10_max": 0.34051231293075496, - "nauc_ndcg_at_10_std": 0.2386260521492016, - "nauc_ndcg_at_1_diff1": 0.3948317473041426, - "nauc_ndcg_at_1_max": 0.375502342242583, - "nauc_ndcg_at_1_std": 0.2023848866968725, - "nauc_ndcg_at_20_diff1": 0.2822639305782844, - "nauc_ndcg_at_20_max": 0.3455752287482097, - "nauc_ndcg_at_20_std": 0.258428457487018, - "nauc_ndcg_at_3_diff1": 0.31738521509897943, - "nauc_ndcg_at_3_max": 0.3584920009725775, - "nauc_ndcg_at_3_std": 0.22655842034674972, - "nauc_ndcg_at_5_diff1": 0.29541169832324277, - "nauc_ndcg_at_5_max": 0.3463061961496835, - "nauc_ndcg_at_5_std": 0.22992848838017527, - "nauc_precision_at_1000_diff1": 0.20657275023500984, - "nauc_precision_at_1000_max": 0.5421799124337869, - "nauc_precision_at_1000_std": 0.688794162507745, - "nauc_precision_at_100_diff1": 0.1860465215492983, - "nauc_precision_at_100_max": 0.37685297250936, - "nauc_precision_at_100_std": 0.4528221311140119, - "nauc_precision_at_10_diff1": 0.2254931302861256, - "nauc_precision_at_10_max": 0.3161762107844541, - "nauc_precision_at_10_std": 0.2629942893678289, - "nauc_precision_at_1_diff1": 0.3948317473041426, - "nauc_precision_at_1_max": 0.375502342242583, - "nauc_precision_at_1_std": 0.2023848866968725, - "nauc_precision_at_20_diff1": 0.22005941352512381, - "nauc_precision_at_20_max": 0.33182507120198695, - "nauc_precision_at_20_std": 0.31872755469386793, - "nauc_precision_at_3_diff1": 0.27936582648185876, - "nauc_precision_at_3_max": 0.3486223843295034, - "nauc_precision_at_3_std": 0.2376587600499251, - "nauc_precision_at_5_diff1": 0.24286332116512424, - "nauc_precision_at_5_max": 0.3274324309220014, - "nauc_precision_at_5_std": 0.24310739923473612, - "nauc_recall_at_1000_diff1": 0.20657275023501265, - "nauc_recall_at_1000_max": 0.5421799124337883, - "nauc_recall_at_1000_std": 0.688794162507746, - "nauc_recall_at_100_diff1": 0.18604652154929796, - "nauc_recall_at_100_max": 0.37685297250936006, - "nauc_recall_at_100_std": 0.4528221311140118, - "nauc_recall_at_10_diff1": 0.22549313028612575, - "nauc_recall_at_10_max": 0.3161762107844544, - "nauc_recall_at_10_std": 0.26299428936782887, - "nauc_recall_at_1_diff1": 0.3948317473041426, - "nauc_recall_at_1_max": 0.375502342242583, - "nauc_recall_at_1_std": 0.2023848866968725, - "nauc_recall_at_20_diff1": 0.22005941352512404, - "nauc_recall_at_20_max": 0.33182507120198707, - "nauc_recall_at_20_std": 0.3187275546938682, - "nauc_recall_at_3_diff1": 0.27936582648185887, - "nauc_recall_at_3_max": 0.3486223843295035, - "nauc_recall_at_3_std": 0.23765876004992492, - "nauc_recall_at_5_diff1": 0.24286332116512424, - "nauc_recall_at_5_max": 0.32743243092200136, - "nauc_recall_at_5_std": 0.2431073992347361, - "ndcg_at_1": 0.0995, - "ndcg_at_10": 0.184, - "ndcg_at_100": 0.24082, - "ndcg_at_1000": 0.2774, - "ndcg_at_20": 0.20317, - "ndcg_at_3": 0.14434, - "ndcg_at_5": 0.16339, - "precision_at_1": 0.0995, - "precision_at_10": 0.02869, - "precision_at_100": 0.00569, - "precision_at_1000": 0.00087, - "precision_at_20": 0.01815, - "precision_at_3": 0.05897, - "precision_at_5": 0.04464, - "recall_at_1": 0.0995, - "recall_at_10": 0.28694, - "recall_at_100": 0.56878, - "recall_at_1000": 0.86683, - "recall_at_20": 0.36307, - "recall_at_3": 0.17691, - "recall_at_5": 0.22319 - }, - { - "cv_recall_at_1": 0.00083, - "cv_recall_at_10": 0.00514, - "cv_recall_at_100": 0.03123, - "cv_recall_at_1000": 0.28415, - "cv_recall_at_20": 0.00805, - "cv_recall_at_3": 0.00236, - "cv_recall_at_5": 0.00292, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.00267, - "map_at_1": 0.00083, - "map_at_10": 0.00193, - "map_at_100": 0.00259, - "map_at_1000": 0.00325, - "map_at_20": 0.00212, - "map_at_3": 0.0015, - "map_at_5": 0.00164, - "mrr_at_1": 0.000832870627429206, - "mrr_at_10": 0.0019262887461110668, - "mrr_at_100": 0.0025882539691765576, - "mrr_at_1000": 0.003250035238441143, - "mrr_at_20": 0.002115760796454829, - "mrr_at_3": 0.001503794188413844, - "mrr_at_5": 0.001635665371090135, - "nauc_cv_recall_at_1000_diff1": -0.007626276133075307, - "nauc_cv_recall_at_1000_max": 0.012534558684651335, - "nauc_cv_recall_at_1000_std": 0.04417646187985337, - "nauc_cv_recall_at_100_diff1": 0.00847795729585559, - "nauc_cv_recall_at_100_max": 0.017779803226628355, - "nauc_cv_recall_at_100_std": 0.15477340945105372, - "nauc_cv_recall_at_10_diff1": 0.07691737696626728, - "nauc_cv_recall_at_10_max": 0.17591848148934253, - "nauc_cv_recall_at_10_std": 0.5213689918625686, - "nauc_cv_recall_at_1_diff1": 0.49849303801480216, - "nauc_cv_recall_at_1_max": 0.6149456523898669, - "nauc_cv_recall_at_1_std": 0.7802588913641183, - "nauc_cv_recall_at_20_diff1": 0.05901771001094816, - "nauc_cv_recall_at_20_max": 0.12694926843059573, - "nauc_cv_recall_at_20_std": 0.3765085181438246, - "nauc_cv_recall_at_3_diff1": 0.10624149252697057, - "nauc_cv_recall_at_3_max": 0.34846165139330765, - "nauc_cv_recall_at_3_std": 0.6460134983154191, - "nauc_cv_recall_at_5_diff1": 0.14246132052081384, - "nauc_cv_recall_at_5_max": 0.35420359563849213, - "nauc_cv_recall_at_5_std": 0.6560222993317575, - "nauc_map_at_1000_diff1": 0.11399284167745291, - "nauc_map_at_1000_max": 0.2162294325349823, - "nauc_map_at_1000_std": 0.4174091620453698, - "nauc_map_at_100_diff1": 0.13790384373461875, - "nauc_map_at_100_max": 0.2584907115040497, - "nauc_map_at_100_std": 0.49596050136186354, - "nauc_map_at_10_diff1": 0.18358668703366798, - "nauc_map_at_10_max": 0.3476649659861242, - "nauc_map_at_10_std": 0.6301256971193019, - "nauc_map_at_1_diff1": 0.49849303801480216, - "nauc_map_at_1_max": 0.6149456523898669, - "nauc_map_at_1_std": 0.7802588913641183, - "nauc_map_at_20_diff1": 0.16775456657418533, - "nauc_map_at_20_max": 0.319778921800124, - "nauc_map_at_20_std": 0.5843430754256731, - "nauc_map_at_3_diff1": 0.20590715961779868, - "nauc_map_at_3_max": 0.4189163654652013, - "nauc_map_at_3_std": 0.6778753784305693, - "nauc_map_at_5_diff1": 0.2140425392356912, - "nauc_map_at_5_max": 0.41903410657851026, - "nauc_map_at_5_std": 0.6833802853679313, - "nauc_mrr_at_1000_diff1": 0.11399287345428381, - "nauc_mrr_at_1000_max": 0.21622944086170898, - "nauc_mrr_at_1000_std": 0.4174091508705687, - "nauc_mrr_at_100_diff1": 0.13790384373461875, - "nauc_mrr_at_100_max": 0.2584907115040497, - "nauc_mrr_at_100_std": 0.49596050136186354, - "nauc_mrr_at_10_diff1": 0.18358668703366798, - "nauc_mrr_at_10_max": 0.3476649659861242, - "nauc_mrr_at_10_std": 0.6301256971193019, - "nauc_mrr_at_1_diff1": 0.49849303801480216, - "nauc_mrr_at_1_max": 0.6149456523898669, - "nauc_mrr_at_1_std": 0.7802588913641183, - "nauc_mrr_at_20_diff1": 0.16775456657418533, - "nauc_mrr_at_20_max": 0.319778921800124, - "nauc_mrr_at_20_std": 0.5843430754256731, - "nauc_mrr_at_3_diff1": 0.20590715961779868, - "nauc_mrr_at_3_max": 0.4189163654652013, - "nauc_mrr_at_3_std": 0.6778753784305693, - "nauc_mrr_at_5_diff1": 0.2140425392356912, - "nauc_mrr_at_5_max": 0.41903410657851026, - "nauc_mrr_at_5_std": 0.6833802853679313, - "nauc_ndcg_at_1000_diff1": 0.006198937508097668, - "nauc_ndcg_at_1000_max": 0.033653742379016145, - "nauc_ndcg_at_1000_std": 0.09181517437244709, - "nauc_ndcg_at_100_diff1": 0.04743129405503746, - "nauc_ndcg_at_100_max": 0.09152470836559619, - "nauc_ndcg_at_100_std": 0.26640497705006577, - "nauc_ndcg_at_10_diff1": 0.1360284394048584, - "nauc_ndcg_at_10_max": 0.2740083558852659, - "nauc_ndcg_at_10_std": 0.5837315004075809, - "nauc_ndcg_at_1_diff1": 0.49849303801480216, - "nauc_ndcg_at_1_max": 0.6149456523898669, - "nauc_ndcg_at_1_std": 0.7802588913641183, - "nauc_ndcg_at_20_diff1": 0.11145099981068309, - "nauc_ndcg_at_20_max": 0.2240742297112962, - "nauc_ndcg_at_20_std": 0.4852180698957463, - "nauc_ndcg_at_3_diff1": 0.1700577557184439, - "nauc_ndcg_at_3_max": 0.3936978605948217, - "nauc_ndcg_at_3_std": 0.6663133613132409, - "nauc_ndcg_at_5_diff1": 0.18572114707059503, - "nauc_ndcg_at_5_max": 0.3943678490772386, - "nauc_ndcg_at_5_std": 0.6729657879243428, - "nauc_precision_at_1000_diff1": -0.007626276133075511, - "nauc_precision_at_1000_max": 0.012534558684651226, - "nauc_precision_at_1000_std": 0.04417646187985327, - "nauc_precision_at_100_diff1": 0.008477957295855518, - "nauc_precision_at_100_max": 0.01777980322662837, - "nauc_precision_at_100_std": 0.15477340945105367, - "nauc_precision_at_10_diff1": 0.07691737696626709, - "nauc_precision_at_10_max": 0.1759184814893425, - "nauc_precision_at_10_std": 0.5213689918625685, - "nauc_precision_at_1_diff1": 0.49849303801480216, - "nauc_precision_at_1_max": 0.6149456523898669, - "nauc_precision_at_1_std": 0.7802588913641183, - "nauc_precision_at_20_diff1": 0.05901771001094845, - "nauc_precision_at_20_max": 0.12694926843059612, - "nauc_precision_at_20_std": 0.3765085181438252, - "nauc_precision_at_3_diff1": 0.10624149252697063, - "nauc_precision_at_3_max": 0.3484616513933078, - "nauc_precision_at_3_std": 0.6460134983154195, - "nauc_precision_at_5_diff1": 0.14246132052081387, - "nauc_precision_at_5_max": 0.3542035956384923, - "nauc_precision_at_5_std": 0.6560222993317575, - "nauc_recall_at_1000_diff1": -0.007626276133075307, - "nauc_recall_at_1000_max": 0.012534558684651335, - "nauc_recall_at_1000_std": 0.04417646187985337, - "nauc_recall_at_100_diff1": 0.00847795729585559, - "nauc_recall_at_100_max": 0.017779803226628355, - "nauc_recall_at_100_std": 0.15477340945105372, - "nauc_recall_at_10_diff1": 0.07691737696626728, - "nauc_recall_at_10_max": 0.17591848148934253, - "nauc_recall_at_10_std": 0.5213689918625686, - "nauc_recall_at_1_diff1": 0.49849303801480216, - "nauc_recall_at_1_max": 0.6149456523898669, - "nauc_recall_at_1_std": 0.7802588913641183, - "nauc_recall_at_20_diff1": 0.05901771001094816, - "nauc_recall_at_20_max": 0.12694926843059573, - "nauc_recall_at_20_std": 0.3765085181438246, - "nauc_recall_at_3_diff1": 0.10624149252697057, - "nauc_recall_at_3_max": 0.34846165139330765, - "nauc_recall_at_3_std": 0.6460134983154191, - "nauc_recall_at_5_diff1": 0.14246132052081384, - "nauc_recall_at_5_max": 0.35420359563849213, - "nauc_recall_at_5_std": 0.6560222993317575, - "ndcg_at_1": 0.00083, - "ndcg_at_10": 0.00267, - "ndcg_at_100": 0.00744, - "ndcg_at_1000": 0.03624, - "ndcg_at_20": 0.00339, - "ndcg_at_3": 0.00172, - "ndcg_at_5": 0.00196, - "precision_at_1": 0.00083, - "precision_at_10": 0.00051, - "precision_at_100": 0.00031, - "precision_at_1000": 0.00028, - "precision_at_20": 0.0004, - "precision_at_3": 0.00079, - "precision_at_5": 0.00058, - "recall_at_1": 0.00083, - "recall_at_10": 0.00514, - "recall_at_100": 0.03123, - "recall_at_1000": 0.28415, - "recall_at_20": 0.00805, - "recall_at_3": 0.00236, - "recall_at_5": 0.00292 - }, - { - "cv_recall_at_1": 0.39056, - "cv_recall_at_10": 0.76833, - "cv_recall_at_100": 0.95611, - "cv_recall_at_1000": 0.99917, - "cv_recall_at_20": 0.84819, - "cv_recall_at_3": 0.59597, - "cv_recall_at_5": 0.67222, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57352, - "map_at_1": 0.39056, - "map_at_10": 0.51189, - "map_at_100": 0.52037, - "map_at_1000": 0.52062, - "map_at_20": 0.5175, - "map_at_3": 0.48157, - "map_at_5": 0.49905, - "mrr_at_1": 0.39055555555555554, - "mrr_at_10": 0.5118883377425072, - "mrr_at_100": 0.5203659494810378, - "mrr_at_1000": 0.5206184290057001, - "mrr_at_20": 0.5175045521828792, - "mrr_at_3": 0.4815740740740779, - "mrr_at_5": 0.49905324074074237, - "nauc_cv_recall_at_1000_diff1": 0.14394646747577447, - "nauc_cv_recall_at_1000_max": 0.423591658885699, - "nauc_cv_recall_at_1000_std": 0.2710862122626985, - "nauc_cv_recall_at_100_diff1": 0.46606596224987895, - "nauc_cv_recall_at_100_max": 0.46268866196267544, - "nauc_cv_recall_at_100_std": 0.08952800529494662, - "nauc_cv_recall_at_10_diff1": 0.4379063393575716, - "nauc_cv_recall_at_10_max": 0.4216913978135716, - "nauc_cv_recall_at_10_std": -0.06510608864641969, - "nauc_cv_recall_at_1_diff1": 0.5635180937370017, - "nauc_cv_recall_at_1_max": 0.3921407024851276, - "nauc_cv_recall_at_1_std": -0.05834478567184907, - "nauc_cv_recall_at_20_diff1": 0.4242996361711508, - "nauc_cv_recall_at_20_max": 0.4116836539897339, - "nauc_cv_recall_at_20_std": -0.03936486874353415, - "nauc_cv_recall_at_3_diff1": 0.4587991676794355, - "nauc_cv_recall_at_3_max": 0.422630444047248, - "nauc_cv_recall_at_3_std": -0.0698509911455654, - "nauc_cv_recall_at_5_diff1": 0.4454212445561298, - "nauc_cv_recall_at_5_max": 0.41885208732865375, - "nauc_cv_recall_at_5_std": -0.07339352687217181, - "nauc_map_at_1000_diff1": 0.5211156484047063, - "nauc_map_at_1000_max": 0.4059355660415839, - "nauc_map_at_1000_std": -0.06225085095320318, - "nauc_map_at_100_diff1": 0.5210906627423454, - "nauc_map_at_100_max": 0.40600689920338445, - "nauc_map_at_100_std": -0.06217505336158905, - "nauc_map_at_10_diff1": 0.5194607406726276, - "nauc_map_at_10_max": 0.40661441290393446, - "nauc_map_at_10_std": -0.06299795452615677, - "nauc_map_at_1_diff1": 0.5635180937370017, - "nauc_map_at_1_max": 0.3921407024851276, - "nauc_map_at_1_std": -0.05834478567184907, - "nauc_map_at_20_diff1": 0.5202557335360662, - "nauc_map_at_20_max": 0.40603319833528057, - "nauc_map_at_20_std": -0.06228822965887942, - "nauc_map_at_3_diff1": 0.5191755804934863, - "nauc_map_at_3_max": 0.40734379586175073, - "nauc_map_at_3_std": -0.06333142407599146, - "nauc_map_at_5_diff1": 0.5184830281353593, - "nauc_map_at_5_max": 0.40646787276964275, - "nauc_map_at_5_std": -0.06391705205112995, - "nauc_mrr_at_1000_diff1": 0.5211156484047063, - "nauc_mrr_at_1000_max": 0.4059355660415839, - "nauc_mrr_at_1000_std": -0.06225085095320318, - "nauc_mrr_at_100_diff1": 0.5210906627423454, - "nauc_mrr_at_100_max": 0.40600689920338445, - "nauc_mrr_at_100_std": -0.06217505336158905, - "nauc_mrr_at_10_diff1": 0.5194607406726276, - "nauc_mrr_at_10_max": 0.40661441290393446, - "nauc_mrr_at_10_std": -0.06299795452615677, - "nauc_mrr_at_1_diff1": 0.5635180937370017, - "nauc_mrr_at_1_max": 0.3921407024851276, - "nauc_mrr_at_1_std": -0.05834478567184907, - "nauc_mrr_at_20_diff1": 0.5202557335360662, - "nauc_mrr_at_20_max": 0.40603319833528057, - "nauc_mrr_at_20_std": -0.06228822965887942, - "nauc_mrr_at_3_diff1": 0.5191755804934863, - "nauc_mrr_at_3_max": 0.40734379586175073, - "nauc_mrr_at_3_std": -0.06333142407599146, - "nauc_mrr_at_5_diff1": 0.5184830281353593, - "nauc_mrr_at_5_max": 0.40646787276964275, - "nauc_mrr_at_5_std": -0.06391705205112995, - "nauc_ndcg_at_1000_diff1": 0.5142678476811551, - "nauc_ndcg_at_1000_max": 0.4071623435820476, - "nauc_ndcg_at_1000_std": -0.0612249688199367, - "nauc_ndcg_at_100_diff1": 0.5134288832740771, - "nauc_ndcg_at_100_max": 0.4085381492895844, - "nauc_ndcg_at_100_std": -0.058269328250549, - "nauc_ndcg_at_10_diff1": 0.5042774894693469, - "nauc_ndcg_at_10_max": 0.40968828510708827, - "nauc_ndcg_at_10_std": -0.06365206462233498, - "nauc_ndcg_at_1_diff1": 0.5635180937370017, - "nauc_ndcg_at_1_max": 0.3921407024851276, - "nauc_ndcg_at_1_std": -0.05834478567184907, - "nauc_ndcg_at_20_diff1": 0.5064832037799302, - "nauc_ndcg_at_20_max": 0.40761011384041623, - "nauc_ndcg_at_20_std": -0.06039784256595193, - "nauc_ndcg_at_3_diff1": 0.5045191285336305, - "nauc_ndcg_at_3_max": 0.4111560098071898, - "nauc_ndcg_at_3_std": -0.06491911626024735, - "nauc_ndcg_at_5_diff1": 0.502323225972798, - "nauc_ndcg_at_5_max": 0.40944904239400765, - "nauc_ndcg_at_5_std": -0.06600665556081452, - "nauc_precision_at_1000_diff1": 0.143946467475703, - "nauc_precision_at_1000_max": 0.4235916588856638, - "nauc_precision_at_1000_std": 0.27108621226244745, - "nauc_precision_at_100_diff1": 0.4660659622498764, - "nauc_precision_at_100_max": 0.46268866196267266, - "nauc_precision_at_100_std": 0.08952800529494184, - "nauc_precision_at_10_diff1": 0.43790633935757056, - "nauc_precision_at_10_max": 0.421691397813571, - "nauc_precision_at_10_std": -0.06510608864641966, - "nauc_precision_at_1_diff1": 0.5635180937370017, - "nauc_precision_at_1_max": 0.3921407024851276, - "nauc_precision_at_1_std": -0.05834478567184907, - "nauc_precision_at_20_diff1": 0.424299636171151, - "nauc_precision_at_20_max": 0.4116836539897328, - "nauc_precision_at_20_std": -0.03936486874353411, - "nauc_precision_at_3_diff1": 0.4587991676794361, - "nauc_precision_at_3_max": 0.4226304440472487, - "nauc_precision_at_3_std": -0.06985099114556456, - "nauc_precision_at_5_diff1": 0.44542124455612886, - "nauc_precision_at_5_max": 0.4188520873286538, - "nauc_precision_at_5_std": -0.07339352687217225, - "nauc_recall_at_1000_diff1": 0.14394646747577447, - "nauc_recall_at_1000_max": 0.423591658885699, - "nauc_recall_at_1000_std": 0.2710862122626985, - "nauc_recall_at_100_diff1": 0.46606596224987895, - "nauc_recall_at_100_max": 0.46268866196267544, - "nauc_recall_at_100_std": 0.08952800529494662, - "nauc_recall_at_10_diff1": 0.4379063393575716, - "nauc_recall_at_10_max": 0.4216913978135716, - "nauc_recall_at_10_std": -0.06510608864641969, - "nauc_recall_at_1_diff1": 0.5635180937370017, - "nauc_recall_at_1_max": 0.3921407024851276, - "nauc_recall_at_1_std": -0.05834478567184907, - "nauc_recall_at_20_diff1": 0.4242996361711508, - "nauc_recall_at_20_max": 0.4116836539897339, - "nauc_recall_at_20_std": -0.03936486874353415, - "nauc_recall_at_3_diff1": 0.4587991676794355, - "nauc_recall_at_3_max": 0.422630444047248, - "nauc_recall_at_3_std": -0.0698509911455654, - "nauc_recall_at_5_diff1": 0.4454212445561298, - "nauc_recall_at_5_max": 0.41885208732865375, - "nauc_recall_at_5_std": -0.07339352687217181, - "ndcg_at_1": 0.39056, - "ndcg_at_10": 0.57352, - "ndcg_at_100": 0.614, - "ndcg_at_1000": 0.61973, - "ndcg_at_20": 0.5938, - "ndcg_at_3": 0.51098, - "ndcg_at_5": 0.54243, - "precision_at_1": 0.39056, - "precision_at_10": 0.07683, - "precision_at_100": 0.00956, - "precision_at_1000": 0.001, - "precision_at_20": 0.04241, - "precision_at_3": 0.19866, - "precision_at_5": 0.13444, - "recall_at_1": 0.39056, - "recall_at_10": 0.76833, - "recall_at_100": 0.95611, - "recall_at_1000": 0.99917, - "recall_at_20": 0.84819, - "recall_at_3": 0.59597, - "recall_at_5": 0.67222 - }, - { - "cv_recall_at_1": 0.15277, - "cv_recall_at_10": 0.43302, - "cv_recall_at_100": 0.74983, - "cv_recall_at_1000": 0.95774, - "cv_recall_at_20": 0.52624, - "cv_recall_at_3": 0.26759, - "cv_recall_at_5": 0.33237, - "hf_subset": "es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.27852, - "map_at_1": 0.15277, - "map_at_10": 0.23092, - "map_at_100": 0.24302, - "map_at_1000": 0.24395, - "map_at_20": 0.2374, - "map_at_3": 0.20283, - "map_at_5": 0.21761, - "mrr_at_1": 0.15277455305316925, - "mrr_at_10": 0.23091626403676438, - "mrr_at_100": 0.24301997050327284, - "mrr_at_1000": 0.24395317098725908, - "mrr_at_20": 0.2374002759347962, - "mrr_at_3": 0.20282872842659158, - "mrr_at_5": 0.21760699636251063, - "nauc_cv_recall_at_1000_diff1": 0.28871615017386665, - "nauc_cv_recall_at_1000_max": 0.4184062793922356, - "nauc_cv_recall_at_1000_std": 0.7481146617611558, - "nauc_cv_recall_at_100_diff1": 0.24780216183232057, - "nauc_cv_recall_at_100_max": 0.2647257211285579, - "nauc_cv_recall_at_100_std": 0.4019682823243577, - "nauc_cv_recall_at_10_diff1": 0.23715268833907996, - "nauc_cv_recall_at_10_max": 0.25727352781077684, - "nauc_cv_recall_at_10_std": 0.1883518291743347, - "nauc_cv_recall_at_1_diff1": 0.3951332983445778, - "nauc_cv_recall_at_1_max": 0.28679458916310435, - "nauc_cv_recall_at_1_std": 0.09567202538938674, - "nauc_cv_recall_at_20_diff1": 0.23858513986687463, - "nauc_cv_recall_at_20_max": 0.2509656308028913, - "nauc_cv_recall_at_20_std": 0.2432511638351338, - "nauc_cv_recall_at_3_diff1": 0.2612068631489378, - "nauc_cv_recall_at_3_max": 0.2554936434215696, - "nauc_cv_recall_at_3_std": 0.11764445499701637, - "nauc_cv_recall_at_5_diff1": 0.24864032378071985, - "nauc_cv_recall_at_5_max": 0.25485945507513963, - "nauc_cv_recall_at_5_std": 0.1447814692586401, - "nauc_map_at_1000_diff1": 0.31803830658466875, - "nauc_map_at_1000_max": 0.270883669496194, - "nauc_map_at_1000_std": 0.12967884875912955, - "nauc_map_at_100_diff1": 0.3177685807545578, - "nauc_map_at_100_max": 0.27073040716077845, - "nauc_map_at_100_std": 0.12997750284027942, - "nauc_map_at_10_diff1": 0.3167483618348145, - "nauc_map_at_10_max": 0.2711432056874237, - "nauc_map_at_10_std": 0.12326759319526, - "nauc_map_at_1_diff1": 0.3951332983445778, - "nauc_map_at_1_max": 0.28679458916310435, - "nauc_map_at_1_std": 0.09567202538938674, - "nauc_map_at_20_diff1": 0.31721758152947716, - "nauc_map_at_20_max": 0.27060117498496455, - "nauc_map_at_20_std": 0.12773963278126685, - "nauc_map_at_3_diff1": 0.324870380303839, - "nauc_map_at_3_max": 0.271729787112544, - "nauc_map_at_3_std": 0.10750958295910185, - "nauc_map_at_5_diff1": 0.319389243314658, - "nauc_map_at_5_max": 0.2711176076197904, - "nauc_map_at_5_std": 0.11561201659260371, - "nauc_mrr_at_1000_diff1": 0.31803830658466875, - "nauc_mrr_at_1000_max": 0.270883669496194, - "nauc_mrr_at_1000_std": 0.12967884875912955, - "nauc_mrr_at_100_diff1": 0.3177685807545578, - "nauc_mrr_at_100_max": 0.27073040716077845, - "nauc_mrr_at_100_std": 0.12997750284027942, - "nauc_mrr_at_10_diff1": 0.3167483618348145, - "nauc_mrr_at_10_max": 0.2711432056874237, - "nauc_mrr_at_10_std": 0.12326759319526, - "nauc_mrr_at_1_diff1": 0.3951332983445778, - "nauc_mrr_at_1_max": 0.28679458916310435, - "nauc_mrr_at_1_std": 0.09567202538938674, - "nauc_mrr_at_20_diff1": 0.31721758152947716, - "nauc_mrr_at_20_max": 0.27060117498496455, - "nauc_mrr_at_20_std": 0.12773963278126685, - "nauc_mrr_at_3_diff1": 0.324870380303839, - "nauc_mrr_at_3_max": 0.271729787112544, - "nauc_mrr_at_3_std": 0.10750958295910185, - "nauc_mrr_at_5_diff1": 0.319389243314658, - "nauc_mrr_at_5_max": 0.2711176076197904, - "nauc_mrr_at_5_std": 0.11561201659260371, - "nauc_ndcg_at_1000_diff1": 0.3049322282875534, - "nauc_ndcg_at_1000_max": 0.27115908501182373, - "nauc_ndcg_at_1000_std": 0.16206072135256805, - "nauc_ndcg_at_100_diff1": 0.2992437168898892, - "nauc_ndcg_at_100_max": 0.2674940082163727, - "nauc_ndcg_at_100_std": 0.1736118577818984, - "nauc_ndcg_at_10_diff1": 0.2930997019668874, - "nauc_ndcg_at_10_max": 0.266813494365646, - "nauc_ndcg_at_10_std": 0.14098806944662287, - "nauc_ndcg_at_1_diff1": 0.3951332983445778, - "nauc_ndcg_at_1_max": 0.28679458916310435, - "nauc_ndcg_at_1_std": 0.09567202538938674, - "nauc_ndcg_at_20_diff1": 0.2943634186381247, - "nauc_ndcg_at_20_max": 0.2650448328834689, - "nauc_ndcg_at_20_std": 0.15633248515225112, - "nauc_ndcg_at_3_diff1": 0.30640418992551965, - "nauc_ndcg_at_3_max": 0.2670751781101042, - "nauc_ndcg_at_3_std": 0.11046182320159958, - "nauc_ndcg_at_5_diff1": 0.29831489333197114, - "nauc_ndcg_at_5_max": 0.2662957662223934, - "nauc_ndcg_at_5_std": 0.12394467057830935, - "nauc_precision_at_1000_diff1": 0.2887161501738626, - "nauc_precision_at_1000_max": 0.4184062793922289, - "nauc_precision_at_1000_std": 0.7481146617611505, - "nauc_precision_at_100_diff1": 0.24780216183231976, - "nauc_precision_at_100_max": 0.26472572112855824, - "nauc_precision_at_100_std": 0.4019682823243572, - "nauc_precision_at_10_diff1": 0.23715268833907965, - "nauc_precision_at_10_max": 0.2572735278107764, - "nauc_precision_at_10_std": 0.1883518291743345, - "nauc_precision_at_1_diff1": 0.3951332983445778, - "nauc_precision_at_1_max": 0.28679458916310435, - "nauc_precision_at_1_std": 0.09567202538938674, - "nauc_precision_at_20_diff1": 0.2385851398668743, - "nauc_precision_at_20_max": 0.25096563080289136, - "nauc_precision_at_20_std": 0.2432511638351335, - "nauc_precision_at_3_diff1": 0.26120686314893776, - "nauc_precision_at_3_max": 0.25549364342156955, - "nauc_precision_at_3_std": 0.11764445499701617, - "nauc_precision_at_5_diff1": 0.2486403237807199, - "nauc_precision_at_5_max": 0.2548594550751396, - "nauc_precision_at_5_std": 0.1447814692586399, - "nauc_recall_at_1000_diff1": 0.28871615017386665, - "nauc_recall_at_1000_max": 0.4184062793922356, - "nauc_recall_at_1000_std": 0.7481146617611558, - "nauc_recall_at_100_diff1": 0.24780216183232057, - "nauc_recall_at_100_max": 0.2647257211285579, - "nauc_recall_at_100_std": 0.4019682823243577, - "nauc_recall_at_10_diff1": 0.23715268833907996, - "nauc_recall_at_10_max": 0.25727352781077684, - "nauc_recall_at_10_std": 0.1883518291743347, - "nauc_recall_at_1_diff1": 0.3951332983445778, - "nauc_recall_at_1_max": 0.28679458916310435, - "nauc_recall_at_1_std": 0.09567202538938674, - "nauc_recall_at_20_diff1": 0.23858513986687463, - "nauc_recall_at_20_max": 0.2509656308028913, - "nauc_recall_at_20_std": 0.2432511638351338, - "nauc_recall_at_3_diff1": 0.2612068631489378, - "nauc_recall_at_3_max": 0.2554936434215696, - "nauc_recall_at_3_std": 0.11764445499701637, - "nauc_recall_at_5_diff1": 0.24864032378071985, - "nauc_recall_at_5_max": 0.25485945507513963, - "nauc_recall_at_5_std": 0.1447814692586401, - "ndcg_at_1": 0.15277, - "ndcg_at_10": 0.27852, - "ndcg_at_100": 0.34335, - "ndcg_at_1000": 0.36944, - "ndcg_at_20": 0.30211, - "ndcg_at_3": 0.21944, - "ndcg_at_5": 0.24609, - "precision_at_1": 0.15277, - "precision_at_10": 0.0433, - "precision_at_100": 0.0075, - "precision_at_1000": 0.00096, - "precision_at_20": 0.02631, - "precision_at_3": 0.0892, - "precision_at_5": 0.06647, - "recall_at_1": 0.15277, - "recall_at_10": 0.43302, - "recall_at_100": 0.74983, - "recall_at_1000": 0.95774, - "recall_at_20": 0.52624, - "recall_at_3": 0.26759, - "recall_at_5": 0.33237 - }, - { - "cv_recall_at_1": 0.00028, - "cv_recall_at_10": 0.00373, - "cv_recall_at_100": 0.03271, - "cv_recall_at_1000": 0.29896, - "cv_recall_at_20": 0.00759, - "cv_recall_at_3": 0.0011, - "cv_recall_at_5": 0.00166, - "hf_subset": "fa", - "languages": [ - "fas-Arab" - ], - "main_score": 0.00162, - "map_at_1": 0.00028, - "map_at_10": 0.001, - "map_at_100": 0.00175, - "map_at_1000": 0.00244, - "map_at_20": 0.00124, - "map_at_3": 0.0006, - "map_at_5": 0.00072, - "mrr_at_1": 0.00027605244996549346, - "mrr_at_10": 0.001000032863386901, - "mrr_at_100": 0.0017508881788116945, - "mrr_at_1000": 0.0024357808783905394, - "mrr_at_20": 0.0012443930416931972, - "mrr_at_3": 0.0005981136415919026, - "mrr_at_5": 0.0007223372440763745, - "nauc_cv_recall_at_1000_diff1": 0.027557638498875554, - "nauc_cv_recall_at_1000_max": 0.03097565502070411, - "nauc_cv_recall_at_1000_std": 0.026461907925695066, - "nauc_cv_recall_at_100_diff1": -0.002745721981439647, - "nauc_cv_recall_at_100_max": 0.06018066655766445, - "nauc_cv_recall_at_100_std": 0.15216477829100694, - "nauc_cv_recall_at_10_diff1": -0.07750927677273792, - "nauc_cv_recall_at_10_max": 0.21309126607147533, - "nauc_cv_recall_at_10_std": 0.3776404599665452, - "nauc_cv_recall_at_1_diff1": 0.5492375204561938, - "nauc_cv_recall_at_1_max": 0.12346348800082675, - "nauc_cv_recall_at_1_std": 0.1717739449950981, - "nauc_cv_recall_at_20_diff1": 0.08359618948954435, - "nauc_cv_recall_at_20_max": 0.1892765672248886, - "nauc_cv_recall_at_20_std": 0.3033865628259771, - "nauc_cv_recall_at_3_diff1": 0.06821325088738682, - "nauc_cv_recall_at_3_max": 0.11353499778022445, - "nauc_cv_recall_at_3_std": 0.2872444300231949, - "nauc_cv_recall_at_5_diff1": -0.0650688349346964, - "nauc_cv_recall_at_5_max": 0.19116218917300562, - "nauc_cv_recall_at_5_std": 0.34639713863337207, - "nauc_map_at_1000_diff1": 0.04255967091580391, - "nauc_map_at_1000_max": 0.11000969859685499, - "nauc_map_at_1000_std": 0.20010807009438789, - "nauc_map_at_100_diff1": 0.04612963512398783, - "nauc_map_at_100_max": 0.13125045693129808, - "nauc_map_at_100_std": 0.25357706080615683, - "nauc_map_at_10_diff1": 0.04690727472110029, - "nauc_map_at_10_max": 0.18274617339818786, - "nauc_map_at_10_std": 0.3173527679515929, - "nauc_map_at_1_diff1": 0.5492375204561938, - "nauc_map_at_1_max": 0.12346348800082675, - "nauc_map_at_1_std": 0.1717739449950981, - "nauc_map_at_20_diff1": 0.08227213092585944, - "nauc_map_at_20_max": 0.17655587928495695, - "nauc_map_at_20_std": 0.3020484016197641, - "nauc_map_at_3_diff1": 0.18614921270224682, - "nauc_map_at_3_max": 0.12106983686463088, - "nauc_map_at_3_std": 0.2558291149549853, - "nauc_map_at_5_diff1": 0.09670003344083386, - "nauc_map_at_5_max": 0.16079189981757416, - "nauc_map_at_5_std": 0.2917500251732401, - "nauc_mrr_at_1000_diff1": 0.042559355600840974, - "nauc_mrr_at_1000_max": 0.11000950541871306, - "nauc_mrr_at_1000_std": 0.20010802605611105, - "nauc_mrr_at_100_diff1": 0.04612963512398783, - "nauc_mrr_at_100_max": 0.13125045693129808, - "nauc_mrr_at_100_std": 0.25357706080615683, - "nauc_mrr_at_10_diff1": 0.04690727472110029, - "nauc_mrr_at_10_max": 0.18274617339818786, - "nauc_mrr_at_10_std": 0.3173527679515929, - "nauc_mrr_at_1_diff1": 0.5492375204561938, - "nauc_mrr_at_1_max": 0.12346348800082675, - "nauc_mrr_at_1_std": 0.1717739449950981, - "nauc_mrr_at_20_diff1": 0.08227213092585944, - "nauc_mrr_at_20_max": 0.17655587928495695, - "nauc_mrr_at_20_std": 0.3020484016197641, - "nauc_mrr_at_3_diff1": 0.18614921270224682, - "nauc_mrr_at_3_max": 0.12106983686463088, - "nauc_mrr_at_3_std": 0.2558291149549853, - "nauc_mrr_at_5_diff1": 0.09670003344083386, - "nauc_mrr_at_5_max": 0.16079189981757416, - "nauc_mrr_at_5_std": 0.2917500251732401, - "nauc_ndcg_at_1000_diff1": 0.026082342552705223, - "nauc_ndcg_at_1000_max": 0.04196522349909082, - "nauc_ndcg_at_1000_std": 0.05286649907971509, - "nauc_ndcg_at_100_diff1": 0.007152590164762058, - "nauc_ndcg_at_100_max": 0.0817202656013075, - "nauc_ndcg_at_100_std": 0.18461686140719014, - "nauc_ndcg_at_10_diff1": -0.018790372369411834, - "nauc_ndcg_at_10_max": 0.19847190477287552, - "nauc_ndcg_at_10_std": 0.3478122953194488, - "nauc_ndcg_at_1_diff1": 0.5492375204561938, - "nauc_ndcg_at_1_max": 0.12346348800082675, - "nauc_ndcg_at_1_std": 0.1717739449950981, - "nauc_ndcg_at_20_diff1": 0.07476889360503366, - "nauc_ndcg_at_20_max": 0.18476999136430186, - "nauc_ndcg_at_20_std": 0.30587666233259075, - "nauc_ndcg_at_3_diff1": 0.14017557586032753, - "nauc_ndcg_at_3_max": 0.11832017157679414, - "nauc_ndcg_at_3_std": 0.26796122915840065, - "nauc_ndcg_at_5_diff1": 0.02805491053532623, - "nauc_ndcg_at_5_max": 0.173039085748848, - "nauc_ndcg_at_5_std": 0.31460287587606217, - "nauc_precision_at_1000_diff1": 0.027557638498875506, - "nauc_precision_at_1000_max": 0.03097565502070383, - "nauc_precision_at_1000_std": 0.026461907925694854, - "nauc_precision_at_100_diff1": -0.0027457219814395932, - "nauc_precision_at_100_max": 0.06018066655766441, - "nauc_precision_at_100_std": 0.1521647782910069, - "nauc_precision_at_10_diff1": -0.07750927677273797, - "nauc_precision_at_10_max": 0.21309126607147505, - "nauc_precision_at_10_std": 0.377640459966545, - "nauc_precision_at_1_diff1": 0.5492375204561938, - "nauc_precision_at_1_max": 0.12346348800082675, - "nauc_precision_at_1_std": 0.1717739449950981, - "nauc_precision_at_20_diff1": 0.0835961894895447, - "nauc_precision_at_20_max": 0.18927656722488895, - "nauc_precision_at_20_std": 0.3033865628259774, - "nauc_precision_at_3_diff1": 0.06821325088738678, - "nauc_precision_at_3_max": 0.11353499778022445, - "nauc_precision_at_3_std": 0.28724443002319483, - "nauc_precision_at_5_diff1": -0.0650688349346966, - "nauc_precision_at_5_max": 0.1911621891730056, - "nauc_precision_at_5_std": 0.34639713863337207, - "nauc_recall_at_1000_diff1": 0.027557638498875554, - "nauc_recall_at_1000_max": 0.03097565502070411, - "nauc_recall_at_1000_std": 0.026461907925695066, - "nauc_recall_at_100_diff1": -0.002745721981439647, - "nauc_recall_at_100_max": 0.06018066655766445, - "nauc_recall_at_100_std": 0.15216477829100694, - "nauc_recall_at_10_diff1": -0.07750927677273792, - "nauc_recall_at_10_max": 0.21309126607147533, - "nauc_recall_at_10_std": 0.3776404599665452, - "nauc_recall_at_1_diff1": 0.5492375204561938, - "nauc_recall_at_1_max": 0.12346348800082675, - "nauc_recall_at_1_std": 0.1717739449950981, - "nauc_recall_at_20_diff1": 0.08359618948954435, - "nauc_recall_at_20_max": 0.1892765672248886, - "nauc_recall_at_20_std": 0.3033865628259771, - "nauc_recall_at_3_diff1": 0.06821325088738682, - "nauc_recall_at_3_max": 0.11353499778022445, - "nauc_recall_at_3_std": 0.2872444300231949, - "nauc_recall_at_5_diff1": -0.0650688349346964, - "nauc_recall_at_5_max": 0.19116218917300562, - "nauc_recall_at_5_std": 0.34639713863337207, - "ndcg_at_1": 0.00028, - "ndcg_at_10": 0.00162, - "ndcg_at_100": 0.00695, - "ndcg_at_1000": 0.03721, - "ndcg_at_20": 0.00257, - "ndcg_at_3": 0.00073, - "ndcg_at_5": 0.00095, - "precision_at_1": 0.00028, - "precision_at_10": 0.00037, - "precision_at_100": 0.00033, - "precision_at_1000": 0.0003, - "precision_at_20": 0.00038, - "precision_at_3": 0.00037, - "precision_at_5": 0.00033, - "recall_at_1": 0.00028, - "recall_at_10": 0.00373, - "recall_at_100": 0.03271, - "recall_at_1000": 0.29896, - "recall_at_20": 0.00759, - "recall_at_3": 0.0011, - "recall_at_5": 0.00166 - }, - { - "cv_recall_at_1": 0.01389, - "cv_recall_at_10": 0.04546, - "cv_recall_at_100": 0.12165, - "cv_recall_at_1000": 0.41981, - "cv_recall_at_20": 0.06075, - "cv_recall_at_3": 0.02652, - "cv_recall_at_5": 0.03283, - "hf_subset": "fi", - "languages": [ - "fin-Latn" - ], - "main_score": 0.02784, - "map_at_1": 0.01389, - "map_at_10": 0.02244, - "map_at_100": 0.02485, - "map_at_1000": 0.02572, - "map_at_20": 0.02351, - "map_at_3": 0.01934, - "map_at_5": 0.02073, - "mrr_at_1": 0.013890837659604322, - "mrr_at_10": 0.022435918850960225, - "mrr_at_100": 0.024847508933185956, - "mrr_at_1000": 0.02572312343545724, - "mrr_at_20": 0.023512049607711832, - "mrr_at_3": 0.01933960057995416, - "mrr_at_5": 0.020728684345914582, - "nauc_cv_recall_at_1000_diff1": 0.08707531313449987, - "nauc_cv_recall_at_1000_max": 0.11996333114376777, - "nauc_cv_recall_at_1000_std": 0.30826029993220766, - "nauc_cv_recall_at_100_diff1": 0.10043742076486982, - "nauc_cv_recall_at_100_max": 0.1306432263472824, - "nauc_cv_recall_at_100_std": 0.48927315499614593, - "nauc_cv_recall_at_10_diff1": 0.1592530475063388, - "nauc_cv_recall_at_10_max": 0.20862327017350116, - "nauc_cv_recall_at_10_std": 0.5878117132491872, - "nauc_cv_recall_at_1_diff1": 0.350182350009386, - "nauc_cv_recall_at_1_max": 0.3046495935419812, - "nauc_cv_recall_at_1_std": 0.636084527606397, - "nauc_cv_recall_at_20_diff1": 0.1437524685823495, - "nauc_cv_recall_at_20_max": 0.17332115309790147, - "nauc_cv_recall_at_20_std": 0.5524658081664272, - "nauc_cv_recall_at_3_diff1": 0.22918673154466432, - "nauc_cv_recall_at_3_max": 0.24654796039781401, - "nauc_cv_recall_at_3_std": 0.623762262234279, - "nauc_cv_recall_at_5_diff1": 0.19788756712790903, - "nauc_cv_recall_at_5_max": 0.23318835036456873, - "nauc_cv_recall_at_5_std": 0.5975157204466219, - "nauc_map_at_1000_diff1": 0.2285067575085296, - "nauc_map_at_1000_max": 0.2343738982187907, - "nauc_map_at_1000_std": 0.5922158608918494, - "nauc_map_at_100_diff1": 0.2312473374345219, - "nauc_map_at_100_max": 0.23634822390739973, - "nauc_map_at_100_std": 0.5988199988119439, - "nauc_map_at_10_diff1": 0.24829523040656082, - "nauc_map_at_10_max": 0.2521165587215269, - "nauc_map_at_10_std": 0.6155248532278808, - "nauc_map_at_1_diff1": 0.350182350009386, - "nauc_map_at_1_max": 0.3046495935419812, - "nauc_map_at_1_std": 0.636084527606397, - "nauc_map_at_20_diff1": 0.24103792357800025, - "nauc_map_at_20_max": 0.24410149380833931, - "nauc_map_at_20_std": 0.6084242871609905, - "nauc_map_at_3_diff1": 0.2784663707486378, - "nauc_map_at_3_max": 0.26713667795648827, - "nauc_map_at_3_std": 0.6286305327651562, - "nauc_map_at_5_diff1": 0.26364300508564076, - "nauc_map_at_5_max": 0.26084655501595433, - "nauc_map_at_5_std": 0.6192192010943397, - "nauc_mrr_at_1000_diff1": 0.2285067575085296, - "nauc_mrr_at_1000_max": 0.2343738982187907, - "nauc_mrr_at_1000_std": 0.5922158608918494, - "nauc_mrr_at_100_diff1": 0.2312473374345219, - "nauc_mrr_at_100_max": 0.23634822390739973, - "nauc_mrr_at_100_std": 0.5988199988119439, - "nauc_mrr_at_10_diff1": 0.24829523040656082, - "nauc_mrr_at_10_max": 0.2521165587215269, - "nauc_mrr_at_10_std": 0.6155248532278808, - "nauc_mrr_at_1_diff1": 0.350182350009386, - "nauc_mrr_at_1_max": 0.3046495935419812, - "nauc_mrr_at_1_std": 0.636084527606397, - "nauc_mrr_at_20_diff1": 0.24103792357800025, - "nauc_mrr_at_20_max": 0.24410149380833931, - "nauc_mrr_at_20_std": 0.6084242871609905, - "nauc_mrr_at_3_diff1": 0.2784663707486378, - "nauc_mrr_at_3_max": 0.26713667795648827, - "nauc_mrr_at_3_std": 0.6286305327651562, - "nauc_mrr_at_5_diff1": 0.26364300508564076, - "nauc_mrr_at_5_max": 0.26084655501595433, - "nauc_mrr_at_5_std": 0.6192192010943397, - "nauc_ndcg_at_1000_diff1": 0.14334326058182448, - "nauc_ndcg_at_1000_max": 0.16778475378997937, - "nauc_ndcg_at_1000_std": 0.45092880887103054, - "nauc_ndcg_at_100_diff1": 0.16462476841543675, - "nauc_ndcg_at_100_max": 0.18467248424197302, - "nauc_ndcg_at_100_std": 0.5468753796421475, - "nauc_ndcg_at_10_diff1": 0.2143706326811892, - "nauc_ndcg_at_10_max": 0.2356068704911122, - "nauc_ndcg_at_10_std": 0.6052599730660827, - "nauc_ndcg_at_1_diff1": 0.350182350009386, - "nauc_ndcg_at_1_max": 0.3046495935419812, - "nauc_ndcg_at_1_std": 0.636084527606397, - "nauc_ndcg_at_20_diff1": 0.19970850602275628, - "nauc_ndcg_at_20_max": 0.2154759511247703, - "nauc_ndcg_at_20_std": 0.5864551079981218, - "nauc_ndcg_at_3_diff1": 0.2626559898455386, - "nauc_ndcg_at_3_max": 0.26041373891905834, - "nauc_ndcg_at_3_std": 0.627062901348483, - "nauc_ndcg_at_5_diff1": 0.24107407016068239, - "nauc_ndcg_at_5_max": 0.25125248027494673, - "nauc_ndcg_at_5_std": 0.6120704833304749, - "nauc_precision_at_1000_diff1": 0.08707531313449945, - "nauc_precision_at_1000_max": 0.11996333114376724, - "nauc_precision_at_1000_std": 0.3082602999322072, - "nauc_precision_at_100_diff1": 0.10043742076487006, - "nauc_precision_at_100_max": 0.13064322634728254, - "nauc_precision_at_100_std": 0.48927315499614615, - "nauc_precision_at_10_diff1": 0.15925304750633867, - "nauc_precision_at_10_max": 0.20862327017350096, - "nauc_precision_at_10_std": 0.587811713249187, - "nauc_precision_at_1_diff1": 0.350182350009386, - "nauc_precision_at_1_max": 0.3046495935419812, - "nauc_precision_at_1_std": 0.636084527606397, - "nauc_precision_at_20_diff1": 0.1437524685823496, - "nauc_precision_at_20_max": 0.1733211530979015, - "nauc_precision_at_20_std": 0.5524658081664269, - "nauc_precision_at_3_diff1": 0.22918673154466415, - "nauc_precision_at_3_max": 0.2465479603978137, - "nauc_precision_at_3_std": 0.6237622622342787, - "nauc_precision_at_5_diff1": 0.19788756712790923, - "nauc_precision_at_5_max": 0.23318835036456873, - "nauc_precision_at_5_std": 0.5975157204466223, - "nauc_recall_at_1000_diff1": 0.08707531313449987, - "nauc_recall_at_1000_max": 0.11996333114376777, - "nauc_recall_at_1000_std": 0.30826029993220766, - "nauc_recall_at_100_diff1": 0.10043742076486982, - "nauc_recall_at_100_max": 0.1306432263472824, - "nauc_recall_at_100_std": 0.48927315499614593, - "nauc_recall_at_10_diff1": 0.1592530475063388, - "nauc_recall_at_10_max": 0.20862327017350116, - "nauc_recall_at_10_std": 0.5878117132491872, - "nauc_recall_at_1_diff1": 0.350182350009386, - "nauc_recall_at_1_max": 0.3046495935419812, - "nauc_recall_at_1_std": 0.636084527606397, - "nauc_recall_at_20_diff1": 0.1437524685823495, - "nauc_recall_at_20_max": 0.17332115309790147, - "nauc_recall_at_20_std": 0.5524658081664272, - "nauc_recall_at_3_diff1": 0.22918673154466432, - "nauc_recall_at_3_max": 0.24654796039781401, - "nauc_recall_at_3_std": 0.623762262234279, - "nauc_recall_at_5_diff1": 0.19788756712790903, - "nauc_recall_at_5_max": 0.23318835036456873, - "nauc_recall_at_5_std": 0.5975157204466219, - "ndcg_at_1": 0.01389, - "ndcg_at_10": 0.02784, - "ndcg_at_100": 0.04256, - "ndcg_at_1000": 0.07711, - "ndcg_at_20": 0.03172, - "ndcg_at_3": 0.02118, - "ndcg_at_5": 0.02373, - "precision_at_1": 0.01389, - "precision_at_10": 0.00455, - "precision_at_100": 0.00122, - "precision_at_1000": 0.00042, - "precision_at_20": 0.00304, - "precision_at_3": 0.00884, - "precision_at_5": 0.00657, - "recall_at_1": 0.01389, - "recall_at_10": 0.04546, - "recall_at_100": 0.12165, - "recall_at_1000": 0.41981, - "recall_at_20": 0.06075, - "recall_at_3": 0.02652, - "recall_at_5": 0.03283 - }, - { - "cv_recall_at_1": 0.03798, - "cv_recall_at_10": 0.11197, - "cv_recall_at_100": 0.24926, - "cv_recall_at_1000": 0.57237, - "cv_recall_at_20": 0.14953, - "cv_recall_at_3": 0.06879, - "cv_recall_at_5": 0.08693, - "hf_subset": "fil", - "languages": [ - "fil-Latn" - ], - "main_score": 0.07134, - "map_at_1": 0.03798, - "map_at_10": 0.05877, - "map_at_100": 0.06375, - "map_at_1000": 0.06482, - "map_at_20": 0.06137, - "map_at_3": 0.0513, - "map_at_5": 0.05543, - "mrr_at_1": 0.03798002532001688, - "mrr_at_10": 0.058774591564013344, - "mrr_at_100": 0.06374552093791727, - "mrr_at_1000": 0.06482382251457994, - "mrr_at_20": 0.061370797279852826, - "mrr_at_3": 0.05129647864209684, - "mrr_at_5": 0.05543208139916527, - "nauc_cv_recall_at_1000_diff1": 0.20853496875406827, - "nauc_cv_recall_at_1000_max": 0.3881570464510469, - "nauc_cv_recall_at_1000_std": 0.49387254350944787, - "nauc_cv_recall_at_100_diff1": 0.23205637728352363, - "nauc_cv_recall_at_100_max": 0.3954508542415302, - "nauc_cv_recall_at_100_std": 0.4519956658040415, - "nauc_cv_recall_at_10_diff1": 0.3064742275003843, - "nauc_cv_recall_at_10_max": 0.4429589802430824, - "nauc_cv_recall_at_10_std": 0.40852835518045455, - "nauc_cv_recall_at_1_diff1": 0.5101715108551284, - "nauc_cv_recall_at_1_max": 0.5394259119087396, - "nauc_cv_recall_at_1_std": 0.37869340143950214, - "nauc_cv_recall_at_20_diff1": 0.28407658168080185, - "nauc_cv_recall_at_20_max": 0.43327377918876614, - "nauc_cv_recall_at_20_std": 0.4286340582660606, - "nauc_cv_recall_at_3_diff1": 0.3877289427470475, - "nauc_cv_recall_at_3_max": 0.5078791693039313, - "nauc_cv_recall_at_3_std": 0.4042514668481585, - "nauc_cv_recall_at_5_diff1": 0.3482188187657036, - "nauc_cv_recall_at_5_max": 0.4746387236244124, - "nauc_cv_recall_at_5_std": 0.4024974999774052, - "nauc_map_at_1000_diff1": 0.3928899992476722, - "nauc_map_at_1000_max": 0.4912539176811793, - "nauc_map_at_1000_std": 0.4005048908372273, - "nauc_map_at_100_diff1": 0.39371100428208494, - "nauc_map_at_100_max": 0.4917173814614342, - "nauc_map_at_100_std": 0.3997058234821897, - "nauc_map_at_10_diff1": 0.4057686405129143, - "nauc_map_at_10_max": 0.4988172418114415, - "nauc_map_at_10_std": 0.3942126514110771, - "nauc_map_at_1_diff1": 0.5101715108551284, - "nauc_map_at_1_max": 0.5394259119087396, - "nauc_map_at_1_std": 0.37869340143950214, - "nauc_map_at_20_diff1": 0.399727617762414, - "nauc_map_at_20_max": 0.49596156476295106, - "nauc_map_at_20_std": 0.3977598921807418, - "nauc_map_at_3_diff1": 0.437893265390524, - "nauc_map_at_3_max": 0.52071099682199, - "nauc_map_at_3_std": 0.39242515380387855, - "nauc_map_at_5_diff1": 0.42035957346223796, - "nauc_map_at_5_max": 0.5080647449043068, - "nauc_map_at_5_std": 0.39237505333497585, - "nauc_mrr_at_1000_diff1": 0.39288998966076943, - "nauc_mrr_at_1000_max": 0.49125389950626924, - "nauc_mrr_at_1000_std": 0.4005048696979699, - "nauc_mrr_at_100_diff1": 0.39371100428208494, - "nauc_mrr_at_100_max": 0.4917173814614342, - "nauc_mrr_at_100_std": 0.3997058234821897, - "nauc_mrr_at_10_diff1": 0.4057686405129143, - "nauc_mrr_at_10_max": 0.4988172418114415, - "nauc_mrr_at_10_std": 0.3942126514110771, - "nauc_mrr_at_1_diff1": 0.5101715108551284, - "nauc_mrr_at_1_max": 0.5394259119087396, - "nauc_mrr_at_1_std": 0.37869340143950214, - "nauc_mrr_at_20_diff1": 0.399727617762414, - "nauc_mrr_at_20_max": 0.49596156476295106, - "nauc_mrr_at_20_std": 0.3977598921807418, - "nauc_mrr_at_3_diff1": 0.437893265390524, - "nauc_mrr_at_3_max": 0.52071099682199, - "nauc_mrr_at_3_std": 0.39242515380387855, - "nauc_mrr_at_5_diff1": 0.42035957346223796, - "nauc_mrr_at_5_max": 0.5080647449043068, - "nauc_mrr_at_5_std": 0.39237505333497585, - "nauc_ndcg_at_1000_diff1": 0.31930736032054235, - "nauc_ndcg_at_1000_max": 0.44969017603521383, - "nauc_ndcg_at_1000_std": 0.4321298154567702, - "nauc_ndcg_at_100_diff1": 0.3274105911114975, - "nauc_ndcg_at_100_max": 0.4530516882928912, - "nauc_ndcg_at_100_std": 0.4196524139305105, - "nauc_ndcg_at_10_diff1": 0.3698265574948488, - "nauc_ndcg_at_10_max": 0.47919476729829397, - "nauc_ndcg_at_10_std": 0.3993731393346552, - "nauc_ndcg_at_1_diff1": 0.5101715108551284, - "nauc_ndcg_at_1_max": 0.5394259119087396, - "nauc_ndcg_at_1_std": 0.37869340143950214, - "nauc_ndcg_at_20_diff1": 0.35501548299345015, - "nauc_ndcg_at_20_max": 0.4721907577788585, - "nauc_ndcg_at_20_std": 0.4090085995177488, - "nauc_ndcg_at_3_diff1": 0.4219935975403448, - "nauc_ndcg_at_3_max": 0.5166402700042355, - "nauc_ndcg_at_3_std": 0.3961200978609036, - "nauc_ndcg_at_5_diff1": 0.3958647270105075, - "nauc_ndcg_at_5_max": 0.4969596142383925, - "nauc_ndcg_at_5_std": 0.39585335475850914, - "nauc_precision_at_1000_diff1": 0.2085349687540682, - "nauc_precision_at_1000_max": 0.38815704645104643, - "nauc_precision_at_1000_std": 0.4938725435094476, - "nauc_precision_at_100_diff1": 0.23205637728352368, - "nauc_precision_at_100_max": 0.39545085424153065, - "nauc_precision_at_100_std": 0.45199566580404166, - "nauc_precision_at_10_diff1": 0.3064742275003845, - "nauc_precision_at_10_max": 0.44295898024308256, - "nauc_precision_at_10_std": 0.4085283551804548, - "nauc_precision_at_1_diff1": 0.5101715108551284, - "nauc_precision_at_1_max": 0.5394259119087396, - "nauc_precision_at_1_std": 0.37869340143950214, - "nauc_precision_at_20_diff1": 0.2840765816808022, - "nauc_precision_at_20_max": 0.4332737791887661, - "nauc_precision_at_20_std": 0.4286340582660607, - "nauc_precision_at_3_diff1": 0.3877289427470472, - "nauc_precision_at_3_max": 0.5078791693039312, - "nauc_precision_at_3_std": 0.4042514668481582, - "nauc_precision_at_5_diff1": 0.34821881876570365, - "nauc_precision_at_5_max": 0.4746387236244124, - "nauc_precision_at_5_std": 0.40249749997740514, - "nauc_recall_at_1000_diff1": 0.20853496875406827, - "nauc_recall_at_1000_max": 0.3881570464510469, - "nauc_recall_at_1000_std": 0.49387254350944787, - "nauc_recall_at_100_diff1": 0.23205637728352363, - "nauc_recall_at_100_max": 0.3954508542415302, - "nauc_recall_at_100_std": 0.4519956658040415, - "nauc_recall_at_10_diff1": 0.3064742275003843, - "nauc_recall_at_10_max": 0.4429589802430824, - "nauc_recall_at_10_std": 0.40852835518045455, - "nauc_recall_at_1_diff1": 0.5101715108551284, - "nauc_recall_at_1_max": 0.5394259119087396, - "nauc_recall_at_1_std": 0.37869340143950214, - "nauc_recall_at_20_diff1": 0.28407658168080185, - "nauc_recall_at_20_max": 0.43327377918876614, - "nauc_recall_at_20_std": 0.4286340582660606, - "nauc_recall_at_3_diff1": 0.3877289427470475, - "nauc_recall_at_3_max": 0.5078791693039313, - "nauc_recall_at_3_std": 0.4042514668481585, - "nauc_recall_at_5_diff1": 0.3482188187657036, - "nauc_recall_at_5_max": 0.4746387236244124, - "nauc_recall_at_5_std": 0.4024974999774052, - "ndcg_at_1": 0.03798, - "ndcg_at_10": 0.07134, - "ndcg_at_100": 0.09895, - "ndcg_at_1000": 0.13722, - "ndcg_at_20": 0.08082, - "ndcg_at_3": 0.05578, - "ndcg_at_5": 0.06324, - "precision_at_1": 0.03798, - "precision_at_10": 0.0112, - "precision_at_100": 0.00249, - "precision_at_1000": 0.00057, - "precision_at_20": 0.00748, - "precision_at_3": 0.02293, - "precision_at_5": 0.01739, - "recall_at_1": 0.03798, - "recall_at_10": 0.11197, - "recall_at_100": 0.24926, - "recall_at_1000": 0.57237, - "recall_at_20": 0.14953, - "recall_at_3": 0.06879, - "recall_at_5": 0.08693 - }, - { - "cv_recall_at_1": 0.19224, - "cv_recall_at_10": 0.47349, - "cv_recall_at_100": 0.76337, - "cv_recall_at_1000": 0.95457, - "cv_recall_at_20": 0.564, - "cv_recall_at_3": 0.31873, - "cv_recall_at_5": 0.38636, - "hf_subset": "fr", - "languages": [ - "fra-Latn" - ], - "main_score": 0.32113, - "map_at_1": 0.19224, - "map_at_10": 0.27376, - "map_at_100": 0.28501, - "map_at_1000": 0.28586, - "map_at_20": 0.28005, - "map_at_3": 0.24689, - "map_at_5": 0.26217, - "mrr_at_1": 0.19224480261621116, - "mrr_at_10": 0.2737565488704229, - "mrr_at_100": 0.2850083273545978, - "mrr_at_1000": 0.28586338845015846, - "mrr_at_20": 0.28004661355585186, - "mrr_at_3": 0.24688546289807567, - "mrr_at_5": 0.26216810714007593, - "nauc_cv_recall_at_1000_diff1": 0.18221512084554836, - "nauc_cv_recall_at_1000_max": 0.46557401193098863, - "nauc_cv_recall_at_1000_std": 0.7395760302380457, - "nauc_cv_recall_at_100_diff1": 0.21033825061479602, - "nauc_cv_recall_at_100_max": 0.3345035605929724, - "nauc_cv_recall_at_100_std": 0.43863742408636125, - "nauc_cv_recall_at_10_diff1": 0.2360019555784446, - "nauc_cv_recall_at_10_max": 0.3163427257486659, - "nauc_cv_recall_at_10_std": 0.2284968215737257, - "nauc_cv_recall_at_1_diff1": 0.4095398577757327, - "nauc_cv_recall_at_1_max": 0.3589766108920974, - "nauc_cv_recall_at_1_std": 0.12674851925413028, - "nauc_cv_recall_at_20_diff1": 0.21034976734280142, - "nauc_cv_recall_at_20_max": 0.3025692961549244, - "nauc_cv_recall_at_20_std": 0.2732621004173901, - "nauc_cv_recall_at_3_diff1": 0.2922105553144808, - "nauc_cv_recall_at_3_max": 0.3357950932758947, - "nauc_cv_recall_at_3_std": 0.16981489233169808, - "nauc_cv_recall_at_5_diff1": 0.2650865302094447, - "nauc_cv_recall_at_5_max": 0.3296761807327087, - "nauc_cv_recall_at_5_std": 0.17915999497804208, - "nauc_map_at_1000_diff1": 0.3397297903402744, - "nauc_map_at_1000_max": 0.34529149765324946, - "nauc_map_at_1000_std": 0.1621579787020462, - "nauc_map_at_100_diff1": 0.33946264347274546, - "nauc_map_at_100_max": 0.34515605960451845, - "nauc_map_at_100_std": 0.1624126192568733, - "nauc_map_at_10_diff1": 0.3396071563022263, - "nauc_map_at_10_max": 0.34531124141719866, - "nauc_map_at_10_std": 0.15790447070890276, - "nauc_map_at_1_diff1": 0.4095398577757327, - "nauc_map_at_1_max": 0.3589766108920974, - "nauc_map_at_1_std": 0.12674851925413028, - "nauc_map_at_20_diff1": 0.3383203366998559, - "nauc_map_at_20_max": 0.34438683549639915, - "nauc_map_at_20_std": 0.16050069735291433, - "nauc_map_at_3_diff1": 0.35137520059375393, - "nauc_map_at_3_max": 0.3486786651913938, - "nauc_map_at_3_std": 0.14744928173306157, - "nauc_map_at_5_diff1": 0.34400402411460196, - "nauc_map_at_5_max": 0.3471475789241504, - "nauc_map_at_5_std": 0.1501588391853353, - "nauc_mrr_at_1000_diff1": 0.3397297903402744, - "nauc_mrr_at_1000_max": 0.34529149765324946, - "nauc_mrr_at_1000_std": 0.1621579787020462, - "nauc_mrr_at_100_diff1": 0.33946264347274546, - "nauc_mrr_at_100_max": 0.34515605960451845, - "nauc_mrr_at_100_std": 0.1624126192568733, - "nauc_mrr_at_10_diff1": 0.3396071563022263, - "nauc_mrr_at_10_max": 0.34531124141719866, - "nauc_mrr_at_10_std": 0.15790447070890276, - "nauc_mrr_at_1_diff1": 0.4095398577757327, - "nauc_mrr_at_1_max": 0.3589766108920974, - "nauc_mrr_at_1_std": 0.12674851925413028, - "nauc_mrr_at_20_diff1": 0.3383203366998559, - "nauc_mrr_at_20_max": 0.34438683549639915, - "nauc_mrr_at_20_std": 0.16050069735291433, - "nauc_mrr_at_3_diff1": 0.35137520059375393, - "nauc_mrr_at_3_max": 0.3486786651913938, - "nauc_mrr_at_3_std": 0.14744928173306157, - "nauc_mrr_at_5_diff1": 0.34400402411460196, - "nauc_mrr_at_5_max": 0.3471475789241504, - "nauc_mrr_at_5_std": 0.1501588391853353, - "nauc_ndcg_at_1000_diff1": 0.320807196056081, - "nauc_ndcg_at_1000_max": 0.3441773496778746, - "nauc_ndcg_at_1000_std": 0.1916687643243846, - "nauc_ndcg_at_100_diff1": 0.3136149630871213, - "nauc_ndcg_at_100_max": 0.3408155054521133, - "nauc_ndcg_at_100_std": 0.20215146786074856, - "nauc_ndcg_at_10_diff1": 0.3119374437666259, - "nauc_ndcg_at_10_max": 0.3378532278503251, - "nauc_ndcg_at_10_std": 0.17590910749683075, - "nauc_ndcg_at_1_diff1": 0.4095398577757327, - "nauc_ndcg_at_1_max": 0.3589766108920974, - "nauc_ndcg_at_1_std": 0.12674851925413028, - "nauc_ndcg_at_20_diff1": 0.30681576682350986, - "nauc_ndcg_at_20_max": 0.3346076731018436, - "nauc_ndcg_at_20_std": 0.1861829240387525, - "nauc_ndcg_at_3_diff1": 0.3348439226129218, - "nauc_ndcg_at_3_max": 0.3451279423006363, - "nauc_ndcg_at_3_std": 0.15367180839364783, - "nauc_ndcg_at_5_diff1": 0.32205950130045446, - "nauc_ndcg_at_5_max": 0.34235607549196584, - "nauc_ndcg_at_5_std": 0.15822721650377838, - "nauc_precision_at_1000_diff1": 0.18221512084554412, - "nauc_precision_at_1000_max": 0.4655740119309881, - "nauc_precision_at_1000_std": 0.7395760302380422, - "nauc_precision_at_100_diff1": 0.21033825061479558, - "nauc_precision_at_100_max": 0.3345035605929725, - "nauc_precision_at_100_std": 0.4386374240863622, - "nauc_precision_at_10_diff1": 0.23600195557844436, - "nauc_precision_at_10_max": 0.3163427257486658, - "nauc_precision_at_10_std": 0.2284968215737256, - "nauc_precision_at_1_diff1": 0.4095398577757327, - "nauc_precision_at_1_max": 0.3589766108920974, - "nauc_precision_at_1_std": 0.12674851925413028, - "nauc_precision_at_20_diff1": 0.21034976734280128, - "nauc_precision_at_20_max": 0.3025692961549245, - "nauc_precision_at_20_std": 0.2732621004173904, - "nauc_precision_at_3_diff1": 0.29221055531448104, - "nauc_precision_at_3_max": 0.33579509327589446, - "nauc_precision_at_3_std": 0.16981489233169803, - "nauc_precision_at_5_diff1": 0.26508653020944456, - "nauc_precision_at_5_max": 0.3296761807327088, - "nauc_precision_at_5_std": 0.17915999497804247, - "nauc_recall_at_1000_diff1": 0.18221512084554836, - "nauc_recall_at_1000_max": 0.46557401193098863, - "nauc_recall_at_1000_std": 0.7395760302380457, - "nauc_recall_at_100_diff1": 0.21033825061479602, - "nauc_recall_at_100_max": 0.3345035605929724, - "nauc_recall_at_100_std": 0.43863742408636125, - "nauc_recall_at_10_diff1": 0.2360019555784446, - "nauc_recall_at_10_max": 0.3163427257486659, - "nauc_recall_at_10_std": 0.2284968215737257, - "nauc_recall_at_1_diff1": 0.4095398577757327, - "nauc_recall_at_1_max": 0.3589766108920974, - "nauc_recall_at_1_std": 0.12674851925413028, - "nauc_recall_at_20_diff1": 0.21034976734280142, - "nauc_recall_at_20_max": 0.3025692961549244, - "nauc_recall_at_20_std": 0.2732621004173901, - "nauc_recall_at_3_diff1": 0.2922105553144808, - "nauc_recall_at_3_max": 0.3357950932758947, - "nauc_recall_at_3_std": 0.16981489233169808, - "nauc_recall_at_5_diff1": 0.2650865302094447, - "nauc_recall_at_5_max": 0.3296761807327087, - "nauc_recall_at_5_std": 0.17915999497804208, - "ndcg_at_1": 0.19224, - "ndcg_at_10": 0.32113, - "ndcg_at_100": 0.38071, - "ndcg_at_1000": 0.40466, - "ndcg_at_20": 0.34402, - "ndcg_at_3": 0.26529, - "ndcg_at_5": 0.29299, - "precision_at_1": 0.19224, - "precision_at_10": 0.04735, - "precision_at_100": 0.00763, - "precision_at_1000": 0.00095, - "precision_at_20": 0.0282, - "precision_at_3": 0.10624, - "precision_at_5": 0.07727, - "recall_at_1": 0.19224, - "recall_at_10": 0.47349, - "recall_at_100": 0.76337, - "recall_at_1000": 0.95457, - "recall_at_20": 0.564, - "recall_at_3": 0.31873, - "recall_at_5": 0.38636 - }, - { - "cv_recall_at_1": 0.00097, - "cv_recall_at_10": 0.00403, - "cv_recall_at_100": 0.03208, - "cv_recall_at_1000": 0.28361, - "cv_recall_at_20": 0.00722, - "cv_recall_at_3": 0.0025, - "cv_recall_at_5": 0.00306, - "hf_subset": "he", - "languages": [ - "heb-Hebr" - ], - "main_score": 0.00243, - "map_at_1": 0.00097, - "map_at_10": 0.00193, - "map_at_100": 0.00264, - "map_at_1000": 0.00328, - "map_at_20": 0.00214, - "map_at_3": 0.00167, - "map_at_5": 0.0018, - "mrr_at_1": 0.0009722222222222222, - "mrr_at_10": 0.001931437389770723, - "mrr_at_100": 0.0026372004826096546, - "mrr_at_1000": 0.003280471256251679, - "mrr_at_20": 0.0021394673002483886, - "mrr_at_3": 0.0016666666666666668, - "mrr_at_5": 0.001798611111111111, - "nauc_cv_recall_at_1000_diff1": -0.014688330434024493, - "nauc_cv_recall_at_1000_max": -0.040394933260628846, - "nauc_cv_recall_at_1000_std": 0.016267299925534613, - "nauc_cv_recall_at_100_diff1": 0.050751681017310306, - "nauc_cv_recall_at_100_max": 0.0670188410998714, - "nauc_cv_recall_at_100_std": 0.12506954631988185, - "nauc_cv_recall_at_10_diff1": 0.0967034593784406, - "nauc_cv_recall_at_10_max": 0.23868693504066313, - "nauc_cv_recall_at_10_std": 0.38581460544211377, - "nauc_cv_recall_at_1_diff1": 0.7857717812104719, - "nauc_cv_recall_at_1_max": 0.589482157231017, - "nauc_cv_recall_at_1_std": 0.7263214381540114, - "nauc_cv_recall_at_20_diff1": 0.06092237518317478, - "nauc_cv_recall_at_20_max": 0.16503787331530825, - "nauc_cv_recall_at_20_std": 0.2861860436317104, - "nauc_cv_recall_at_3_diff1": 0.1659660733983243, - "nauc_cv_recall_at_3_max": 0.2803267447753629, - "nauc_cv_recall_at_3_std": 0.3652923113615357, - "nauc_cv_recall_at_5_diff1": 0.15365758470131952, - "nauc_cv_recall_at_5_max": 0.2923384637900333, - "nauc_cv_recall_at_5_std": 0.3567334195185012, - "nauc_map_at_1000_diff1": 0.1962660554666269, - "nauc_map_at_1000_max": 0.22297679269240125, - "nauc_map_at_1000_std": 0.3176544914119965, - "nauc_map_at_100_diff1": 0.23879431747758573, - "nauc_map_at_100_max": 0.27444983312462656, - "nauc_map_at_100_std": 0.3793285530962337, - "nauc_map_at_10_diff1": 0.3146894224461796, - "nauc_map_at_10_max": 0.36308819726862906, - "nauc_map_at_10_std": 0.4853605633939986, - "nauc_map_at_1_diff1": 0.7857717812104719, - "nauc_map_at_1_max": 0.589482157231017, - "nauc_map_at_1_std": 0.7263214381540114, - "nauc_map_at_20_diff1": 0.28606662763634433, - "nauc_map_at_20_max": 0.33238947614972497, - "nauc_map_at_20_std": 0.4516059480170954, - "nauc_map_at_3_diff1": 0.3631309065975018, - "nauc_map_at_3_max": 0.38325060370271, - "nauc_map_at_3_std": 0.496590197656739, - "nauc_map_at_5_diff1": 0.3448749067208954, - "nauc_map_at_5_max": 0.38267862711114653, - "nauc_map_at_5_std": 0.485963329268937, - "nauc_mrr_at_1000_diff1": 0.1962660554666269, - "nauc_mrr_at_1000_max": 0.22297679269240125, - "nauc_mrr_at_1000_std": 0.3176544914119965, - "nauc_mrr_at_100_diff1": 0.23879431747758573, - "nauc_mrr_at_100_max": 0.27444983312462656, - "nauc_mrr_at_100_std": 0.3793285530962337, - "nauc_mrr_at_10_diff1": 0.3146894224461796, - "nauc_mrr_at_10_max": 0.36308819726862906, - "nauc_mrr_at_10_std": 0.4853605633939986, - "nauc_mrr_at_1_diff1": 0.7857717812104719, - "nauc_mrr_at_1_max": 0.589482157231017, - "nauc_mrr_at_1_std": 0.7263214381540114, - "nauc_mrr_at_20_diff1": 0.28606662763634433, - "nauc_mrr_at_20_max": 0.33238947614972497, - "nauc_mrr_at_20_std": 0.4516059480170954, - "nauc_mrr_at_3_diff1": 0.3631309065975018, - "nauc_mrr_at_3_max": 0.38325060370271, - "nauc_mrr_at_3_std": 0.496590197656739, - "nauc_mrr_at_5_diff1": 0.3448749067208954, - "nauc_mrr_at_5_max": 0.38267862711114653, - "nauc_mrr_at_5_std": 0.485963329268937, - "nauc_ndcg_at_1000_diff1": 0.009107777849244263, - "nauc_ndcg_at_1000_max": -0.005747819977854507, - "nauc_ndcg_at_1000_std": 0.053727994473211596, - "nauc_ndcg_at_100_diff1": 0.1010042139450688, - "nauc_ndcg_at_100_max": 0.12816600315656998, - "nauc_ndcg_at_100_std": 0.20279351156528072, - "nauc_ndcg_at_10_diff1": 0.2274399109660982, - "nauc_ndcg_at_10_max": 0.31451605513713937, - "nauc_ndcg_at_10_std": 0.44420690102477517, - "nauc_ndcg_at_1_diff1": 0.7857717812104719, - "nauc_ndcg_at_1_max": 0.589482157231017, - "nauc_ndcg_at_1_std": 0.7263214381540114, - "nauc_ndcg_at_20_diff1": 0.17585608075751075, - "nauc_ndcg_at_20_max": 0.25298732196021045, - "nauc_ndcg_at_20_std": 0.3727442874544738, - "nauc_ndcg_at_3_diff1": 0.29556646211677523, - "nauc_ndcg_at_3_max": 0.34815707856545386, - "nauc_ndcg_at_3_std": 0.4522308753168862, - "nauc_ndcg_at_5_diff1": 0.2746872655009156, - "nauc_ndcg_at_5_max": 0.3495516499369595, - "nauc_ndcg_at_5_std": 0.43928694839903326, - "nauc_precision_at_1000_diff1": -0.014688330434025001, - "nauc_precision_at_1000_max": -0.04039493326062917, - "nauc_precision_at_1000_std": 0.016267299925534114, - "nauc_precision_at_100_diff1": 0.05075168101731036, - "nauc_precision_at_100_max": 0.06701884109987172, - "nauc_precision_at_100_std": 0.12506954631988182, - "nauc_precision_at_10_diff1": 0.0967034593784405, - "nauc_precision_at_10_max": 0.23868693504066316, - "nauc_precision_at_10_std": 0.3858146054421139, - "nauc_precision_at_1_diff1": 0.7857717812104719, - "nauc_precision_at_1_max": 0.589482157231017, - "nauc_precision_at_1_std": 0.7263214381540114, - "nauc_precision_at_20_diff1": 0.060922375183175, - "nauc_precision_at_20_max": 0.16503787331530848, - "nauc_precision_at_20_std": 0.28618604363171074, - "nauc_precision_at_3_diff1": 0.16596607339832442, - "nauc_precision_at_3_max": 0.28032674477536285, - "nauc_precision_at_3_std": 0.36529231136153584, - "nauc_precision_at_5_diff1": 0.1536575847013195, - "nauc_precision_at_5_max": 0.2923384637900333, - "nauc_precision_at_5_std": 0.35673341951850135, - "nauc_recall_at_1000_diff1": -0.014688330434024493, - "nauc_recall_at_1000_max": -0.040394933260628846, - "nauc_recall_at_1000_std": 0.016267299925534613, - "nauc_recall_at_100_diff1": 0.050751681017310306, - "nauc_recall_at_100_max": 0.0670188410998714, - "nauc_recall_at_100_std": 0.12506954631988185, - "nauc_recall_at_10_diff1": 0.0967034593784406, - "nauc_recall_at_10_max": 0.23868693504066313, - "nauc_recall_at_10_std": 0.38581460544211377, - "nauc_recall_at_1_diff1": 0.7857717812104719, - "nauc_recall_at_1_max": 0.589482157231017, - "nauc_recall_at_1_std": 0.7263214381540114, - "nauc_recall_at_20_diff1": 0.06092237518317478, - "nauc_recall_at_20_max": 0.16503787331530825, - "nauc_recall_at_20_std": 0.2861860436317104, - "nauc_recall_at_3_diff1": 0.1659660733983243, - "nauc_recall_at_3_max": 0.2803267447753629, - "nauc_recall_at_3_std": 0.3652923113615357, - "nauc_recall_at_5_diff1": 0.15365758470131952, - "nauc_recall_at_5_max": 0.2923384637900333, - "nauc_recall_at_5_std": 0.3567334195185012, - "ndcg_at_1": 0.00097, - "ndcg_at_10": 0.00243, - "ndcg_at_100": 0.00755, - "ndcg_at_1000": 0.0361, - "ndcg_at_20": 0.00322, - "ndcg_at_3": 0.00188, - "ndcg_at_5": 0.00211, - "precision_at_1": 0.00097, - "precision_at_10": 0.0004, - "precision_at_100": 0.00032, - "precision_at_1000": 0.00028, - "precision_at_20": 0.00036, - "precision_at_3": 0.00083, - "precision_at_5": 0.00061, - "recall_at_1": 0.00097, - "recall_at_10": 0.00403, - "recall_at_100": 0.03208, - "recall_at_1000": 0.28361, - "recall_at_20": 0.00722, - "recall_at_3": 0.0025, - "recall_at_5": 0.00306 - }, - { - "cv_recall_at_1": 0.00047, - "cv_recall_at_10": 0.00212, - "cv_recall_at_100": 0.02975, - "cv_recall_at_1000": 0.28167, - "cv_recall_at_20": 0.006, - "cv_recall_at_3": 0.00082, - "cv_recall_at_5": 0.00106, - "hf_subset": "hi", - "languages": [ - "hin-Deva" - ], - "main_score": 0.00108, - "map_at_1": 0.00047, - "map_at_10": 0.00077, - "map_at_100": 0.00156, - "map_at_1000": 0.00218, - "map_at_20": 0.00104, - "map_at_3": 0.00059, - "map_at_5": 0.00065, - "mrr_at_1": 0.0004704222039280254, - "mrr_at_10": 0.0007740498684871258, - "mrr_at_100": 0.0015571687071699366, - "mrr_at_1000": 0.0021844098865008567, - "mrr_at_20": 0.0010434503539331948, - "mrr_at_3": 0.0005880277549100317, - "mrr_at_5": 0.000646830530401035, - "nauc_cv_recall_at_1000_diff1": 0.023466105738380243, - "nauc_cv_recall_at_1000_max": 0.02982050639025681, - "nauc_cv_recall_at_1000_std": 0.020550417425237402, - "nauc_cv_recall_at_100_diff1": 0.03784573719978466, - "nauc_cv_recall_at_100_max": -0.007253568714786409, - "nauc_cv_recall_at_100_std": 0.006491902302095944, - "nauc_cv_recall_at_10_diff1": -0.06717912818256067, - "nauc_cv_recall_at_10_max": 0.09469709173925962, - "nauc_cv_recall_at_10_std": 0.12422926632555775, - "nauc_cv_recall_at_1_diff1": 0.147582993880789, - "nauc_cv_recall_at_1_max": 0.08980850020969633, - "nauc_cv_recall_at_1_std": 0.20248893816056948, - "nauc_cv_recall_at_20_diff1": 0.055127169766363955, - "nauc_cv_recall_at_20_max": 0.1518853718623151, - "nauc_cv_recall_at_20_std": 0.1327060259635255, - "nauc_cv_recall_at_3_diff1": -0.0498531255070009, - "nauc_cv_recall_at_3_max": 0.025913192697909856, - "nauc_cv_recall_at_3_std": 0.011191172961024218, - "nauc_cv_recall_at_5_diff1": -0.12966772945517036, - "nauc_cv_recall_at_5_max": 0.16724926662252462, - "nauc_cv_recall_at_5_std": 0.1933515924059369, - "nauc_map_at_1000_diff1": 0.026214389853673414, - "nauc_map_at_1000_max": 0.05082203380750064, - "nauc_map_at_1000_std": 0.07111822507222437, - "nauc_map_at_100_diff1": 0.032677935166499354, - "nauc_map_at_100_max": 0.0576271300372991, - "nauc_map_at_100_std": 0.08628569066070878, - "nauc_map_at_10_diff1": 0.004674473643573394, - "nauc_map_at_10_max": 0.08824978358169214, - "nauc_map_at_10_std": 0.15106284212743892, - "nauc_map_at_1_diff1": 0.147582993880789, - "nauc_map_at_1_max": 0.08980850020969633, - "nauc_map_at_1_std": 0.20248893816056948, - "nauc_map_at_20_diff1": 0.03385553500941615, - "nauc_map_at_20_max": 0.10920469708203091, - "nauc_map_at_20_std": 0.14686018180633145, - "nauc_map_at_3_diff1": 0.055446138166487, - "nauc_map_at_3_max": 0.05999069003752935, - "nauc_map_at_3_std": 0.1132166477341151, - "nauc_map_at_5_diff1": 0.013222048944646094, - "nauc_map_at_5_max": 0.11471203870308802, - "nauc_map_at_5_std": 0.1784617761640255, - "nauc_mrr_at_1000_diff1": 0.026214308205751255, - "nauc_mrr_at_1000_max": 0.05082201653104815, - "nauc_mrr_at_1000_std": 0.07111832244532566, - "nauc_mrr_at_100_diff1": 0.032677935166499354, - "nauc_mrr_at_100_max": 0.0576271300372991, - "nauc_mrr_at_100_std": 0.08628569066070878, - "nauc_mrr_at_10_diff1": 0.004674473643573394, - "nauc_mrr_at_10_max": 0.08824978358169214, - "nauc_mrr_at_10_std": 0.15106284212743892, - "nauc_mrr_at_1_diff1": 0.147582993880789, - "nauc_mrr_at_1_max": 0.08980850020969633, - "nauc_mrr_at_1_std": 0.20248893816056948, - "nauc_mrr_at_20_diff1": 0.03385553500941615, - "nauc_mrr_at_20_max": 0.10920469708203091, - "nauc_mrr_at_20_std": 0.14686018180633145, - "nauc_mrr_at_3_diff1": 0.055446138166487, - "nauc_mrr_at_3_max": 0.05999069003752935, - "nauc_mrr_at_3_std": 0.1132166477341151, - "nauc_mrr_at_5_diff1": 0.013222048944646094, - "nauc_mrr_at_5_max": 0.11471203870308802, - "nauc_mrr_at_5_std": 0.1784617761640255, - "nauc_ndcg_at_1000_diff1": 0.022629861488434374, - "nauc_ndcg_at_1000_max": 0.029349628076607032, - "nauc_ndcg_at_1000_std": 0.023801430605690604, - "nauc_ndcg_at_100_diff1": 0.0353565114670267, - "nauc_ndcg_at_100_max": 0.011691000348249958, - "nauc_ndcg_at_100_std": 0.026352383777968207, - "nauc_ndcg_at_10_diff1": -0.03086303226289459, - "nauc_ndcg_at_10_max": 0.09027620565935654, - "nauc_ndcg_at_10_std": 0.13825251774960307, - "nauc_ndcg_at_1_diff1": 0.147582993880789, - "nauc_ndcg_at_1_max": 0.08980850020969633, - "nauc_ndcg_at_1_std": 0.20248893816056948, - "nauc_ndcg_at_20_diff1": 0.041403752773278076, - "nauc_ndcg_at_20_max": 0.13243074970635393, - "nauc_ndcg_at_20_std": 0.13752833968262493, - "nauc_ndcg_at_3_diff1": 0.021941826997649856, - "nauc_ndcg_at_3_max": 0.049147849974923004, - "nauc_ndcg_at_3_std": 0.08075399666994976, - "nauc_ndcg_at_5_diff1": -0.036412002682510894, - "nauc_ndcg_at_5_max": 0.1321204425287366, - "nauc_ndcg_at_5_std": 0.18232858864848792, - "nauc_precision_at_1000_diff1": 0.023466105738379983, - "nauc_precision_at_1000_max": 0.029820506390256532, - "nauc_precision_at_1000_std": 0.02055041742523727, - "nauc_precision_at_100_diff1": 0.037845737199784756, - "nauc_precision_at_100_max": -0.007253568714786299, - "nauc_precision_at_100_std": 0.006491902302096105, - "nauc_precision_at_10_diff1": -0.06717912818256085, - "nauc_precision_at_10_max": 0.09469709173925928, - "nauc_precision_at_10_std": 0.12422926632555738, - "nauc_precision_at_1_diff1": 0.147582993880789, - "nauc_precision_at_1_max": 0.08980850020969633, - "nauc_precision_at_1_std": 0.20248893816056948, - "nauc_precision_at_20_diff1": 0.055127169766364476, - "nauc_precision_at_20_max": 0.15188537186231557, - "nauc_precision_at_20_std": 0.13270602596352585, - "nauc_precision_at_3_diff1": -0.04985312550700086, - "nauc_precision_at_3_max": 0.025913192697909995, - "nauc_precision_at_3_std": 0.01119117296102431, - "nauc_precision_at_5_diff1": -0.1296677294551703, - "nauc_precision_at_5_max": 0.16724926662252482, - "nauc_precision_at_5_std": 0.193351592405937, - "nauc_recall_at_1000_diff1": 0.023466105738380243, - "nauc_recall_at_1000_max": 0.02982050639025681, - "nauc_recall_at_1000_std": 0.020550417425237402, - "nauc_recall_at_100_diff1": 0.03784573719978466, - "nauc_recall_at_100_max": -0.007253568714786409, - "nauc_recall_at_100_std": 0.006491902302095944, - "nauc_recall_at_10_diff1": -0.06717912818256067, - "nauc_recall_at_10_max": 0.09469709173925962, - "nauc_recall_at_10_std": 0.12422926632555775, - "nauc_recall_at_1_diff1": 0.147582993880789, - "nauc_recall_at_1_max": 0.08980850020969633, - "nauc_recall_at_1_std": 0.20248893816056948, - "nauc_recall_at_20_diff1": 0.055127169766363955, - "nauc_recall_at_20_max": 0.1518853718623151, - "nauc_recall_at_20_std": 0.1327060259635255, - "nauc_recall_at_3_diff1": -0.0498531255070009, - "nauc_recall_at_3_max": 0.025913192697909856, - "nauc_recall_at_3_std": 0.011191172961024218, - "nauc_recall_at_5_diff1": -0.12966772945517036, - "nauc_recall_at_5_max": 0.16724926662252462, - "nauc_recall_at_5_std": 0.1933515924059369, - "ndcg_at_1": 0.00047, - "ndcg_at_10": 0.00108, - "ndcg_at_100": 0.00627, - "ndcg_at_1000": 0.03476, - "ndcg_at_20": 0.00206, - "ndcg_at_3": 0.00065, - "ndcg_at_5": 0.00075, - "precision_at_1": 0.00047, - "precision_at_10": 0.00021, - "precision_at_100": 0.0003, - "precision_at_1000": 0.00028, - "precision_at_20": 0.0003, - "precision_at_3": 0.00027, - "precision_at_5": 0.00021, - "recall_at_1": 0.00047, - "recall_at_10": 0.00212, - "recall_at_100": 0.02975, - "recall_at_1000": 0.28167, - "recall_at_20": 0.006, - "recall_at_3": 0.00082, - "recall_at_5": 0.00106 - }, - { - "cv_recall_at_1": 0.01484, - "cv_recall_at_10": 0.05137, - "cv_recall_at_100": 0.14038, - "cv_recall_at_1000": 0.45989, - "cv_recall_at_20": 0.07019, - "cv_recall_at_3": 0.02816, - "cv_recall_at_5": 0.03777, - "hf_subset": "hr", - "languages": [ - "hrv-Latn" - ], - "main_score": 0.03088, - "map_at_1": 0.01484, - "map_at_10": 0.02461, - "map_at_100": 0.02739, - "map_at_1000": 0.02837, - "map_at_20": 0.02585, - "map_at_3": 0.02067, - "map_at_5": 0.02286, - "mrr_at_1": 0.014835164835164835, - "mrr_at_10": 0.02460895255538113, - "mrr_at_100": 0.027393640089861407, - "mrr_at_1000": 0.02836957940781095, - "mrr_at_20": 0.02585265651115727, - "mrr_at_3": 0.020673076923076916, - "mrr_at_5": 0.02285714285714286, - "nauc_cv_recall_at_1000_diff1": 0.09582476902539795, - "nauc_cv_recall_at_1000_max": 0.27929155755972995, - "nauc_cv_recall_at_1000_std": 0.3520941312199978, - "nauc_cv_recall_at_100_diff1": 0.16728310795758913, - "nauc_cv_recall_at_100_max": 0.37389641110769223, - "nauc_cv_recall_at_100_std": 0.45023838508861586, - "nauc_cv_recall_at_10_diff1": 0.19929435295839448, - "nauc_cv_recall_at_10_max": 0.46857517551499467, - "nauc_cv_recall_at_10_std": 0.4969495615891551, - "nauc_cv_recall_at_1_diff1": 0.4321169841695732, - "nauc_cv_recall_at_1_max": 0.5775298373232366, - "nauc_cv_recall_at_1_std": 0.5090480875294399, - "nauc_cv_recall_at_20_diff1": 0.1913912177336063, - "nauc_cv_recall_at_20_max": 0.4393900331688629, - "nauc_cv_recall_at_20_std": 0.4817797035122659, - "nauc_cv_recall_at_3_diff1": 0.2756104105178428, - "nauc_cv_recall_at_3_max": 0.5093405406820369, - "nauc_cv_recall_at_3_std": 0.526119876707219, - "nauc_cv_recall_at_5_diff1": 0.22791033490255386, - "nauc_cv_recall_at_5_max": 0.4876834890357833, - "nauc_cv_recall_at_5_std": 0.512941922579701, - "nauc_map_at_1000_diff1": 0.29128633313057856, - "nauc_map_at_1000_max": 0.5021598434492848, - "nauc_map_at_1000_std": 0.49977560680857286, - "nauc_map_at_100_diff1": 0.29528377391075455, - "nauc_map_at_100_max": 0.5065295243303385, - "nauc_map_at_100_std": 0.5028370765094626, - "nauc_map_at_10_diff1": 0.3082125854119504, - "nauc_map_at_10_max": 0.5252052250217453, - "nauc_map_at_10_std": 0.5108376017093625, - "nauc_map_at_1_diff1": 0.4321169841695732, - "nauc_map_at_1_max": 0.5775298373232366, - "nauc_map_at_1_std": 0.5090480875294399, - "nauc_map_at_20_diff1": 0.3023319120607131, - "nauc_map_at_20_max": 0.5179141232025187, - "nauc_map_at_20_std": 0.5080217864768897, - "nauc_map_at_3_diff1": 0.34768224809470033, - "nauc_map_at_3_max": 0.5429651809057904, - "nauc_map_at_3_std": 0.5195759536304023, - "nauc_map_at_5_diff1": 0.3226213994981184, - "nauc_map_at_5_max": 0.5323292959639635, - "nauc_map_at_5_std": 0.5155839093018375, - "nauc_mrr_at_1000_diff1": 0.29128634518308727, - "nauc_mrr_at_1000_max": 0.5021598494729698, - "nauc_mrr_at_1000_std": 0.4997756092105584, - "nauc_mrr_at_100_diff1": 0.29528377391075455, - "nauc_mrr_at_100_max": 0.5065295243303385, - "nauc_mrr_at_100_std": 0.5028370765094626, - "nauc_mrr_at_10_diff1": 0.3082125854119504, - "nauc_mrr_at_10_max": 0.5252052250217453, - "nauc_mrr_at_10_std": 0.5108376017093625, - "nauc_mrr_at_1_diff1": 0.4321169841695732, - "nauc_mrr_at_1_max": 0.5775298373232366, - "nauc_mrr_at_1_std": 0.5090480875294399, - "nauc_mrr_at_20_diff1": 0.3023319120607131, - "nauc_mrr_at_20_max": 0.5179141232025187, - "nauc_mrr_at_20_std": 0.5080217864768897, - "nauc_mrr_at_3_diff1": 0.34768224809470033, - "nauc_mrr_at_3_max": 0.5429651809057904, - "nauc_mrr_at_3_std": 0.5195759536304023, - "nauc_mrr_at_5_diff1": 0.3226213994981184, - "nauc_mrr_at_5_max": 0.5323292959639635, - "nauc_mrr_at_5_std": 0.5155839093018375, - "nauc_ndcg_at_1000_diff1": 0.18573460599963715, - "nauc_ndcg_at_1000_max": 0.38794287119574855, - "nauc_ndcg_at_1000_std": 0.43063491189756353, - "nauc_ndcg_at_100_diff1": 0.2290343646224961, - "nauc_ndcg_at_100_max": 0.441780450426168, - "nauc_ndcg_at_100_std": 0.47789482167649283, - "nauc_ndcg_at_10_diff1": 0.26575026349242603, - "nauc_ndcg_at_10_max": 0.503623708130739, - "nauc_ndcg_at_10_std": 0.505926394732924, - "nauc_ndcg_at_1_diff1": 0.4321169841695732, - "nauc_ndcg_at_1_max": 0.5775298373232366, - "nauc_ndcg_at_1_std": 0.5090480875294399, - "nauc_ndcg_at_20_diff1": 0.2538508507635941, - "nauc_ndcg_at_20_max": 0.48529750684276435, - "nauc_ndcg_at_20_std": 0.49782418515803956, - "nauc_ndcg_at_3_diff1": 0.32497674728583975, - "nauc_ndcg_at_3_max": 0.5324472658517485, - "nauc_ndcg_at_3_std": 0.5216821129287026, - "nauc_ndcg_at_5_diff1": 0.28955414564042403, - "nauc_ndcg_at_5_max": 0.5168867804137073, - "nauc_ndcg_at_5_std": 0.5148896151162353, - "nauc_precision_at_1000_diff1": 0.0958247690253974, - "nauc_precision_at_1000_max": 0.2792915575597295, - "nauc_precision_at_1000_std": 0.35209413121999733, - "nauc_precision_at_100_diff1": 0.16728310795758933, - "nauc_precision_at_100_max": 0.3738964111076925, - "nauc_precision_at_100_std": 0.45023838508861597, - "nauc_precision_at_10_diff1": 0.19929435295839446, - "nauc_precision_at_10_max": 0.46857517551499467, - "nauc_precision_at_10_std": 0.49694956158915504, - "nauc_precision_at_1_diff1": 0.4321169841695732, - "nauc_precision_at_1_max": 0.5775298373232366, - "nauc_precision_at_1_std": 0.5090480875294399, - "nauc_precision_at_20_diff1": 0.19139121773360634, - "nauc_precision_at_20_max": 0.4393900331688631, - "nauc_precision_at_20_std": 0.48177970351226623, - "nauc_precision_at_3_diff1": 0.2756104105178425, - "nauc_precision_at_3_max": 0.5093405406820367, - "nauc_precision_at_3_std": 0.5261198767072187, - "nauc_precision_at_5_diff1": 0.227910334902554, - "nauc_precision_at_5_max": 0.48768348903578324, - "nauc_precision_at_5_std": 0.5129419225797013, - "nauc_recall_at_1000_diff1": 0.09582476902539795, - "nauc_recall_at_1000_max": 0.27929155755972995, - "nauc_recall_at_1000_std": 0.3520941312199978, - "nauc_recall_at_100_diff1": 0.16728310795758913, - "nauc_recall_at_100_max": 0.37389641110769223, - "nauc_recall_at_100_std": 0.45023838508861586, - "nauc_recall_at_10_diff1": 0.19929435295839448, - "nauc_recall_at_10_max": 0.46857517551499467, - "nauc_recall_at_10_std": 0.4969495615891551, - "nauc_recall_at_1_diff1": 0.4321169841695732, - "nauc_recall_at_1_max": 0.5775298373232366, - "nauc_recall_at_1_std": 0.5090480875294399, - "nauc_recall_at_20_diff1": 0.1913912177336063, - "nauc_recall_at_20_max": 0.4393900331688629, - "nauc_recall_at_20_std": 0.4817797035122659, - "nauc_recall_at_3_diff1": 0.2756104105178428, - "nauc_recall_at_3_max": 0.5093405406820369, - "nauc_recall_at_3_std": 0.526119876707219, - "nauc_recall_at_5_diff1": 0.22791033490255386, - "nauc_recall_at_5_max": 0.4876834890357833, - "nauc_recall_at_5_std": 0.512941922579701, - "ndcg_at_1": 0.01484, - "ndcg_at_10": 0.03088, - "ndcg_at_100": 0.04805, - "ndcg_at_1000": 0.08534, - "ndcg_at_20": 0.03556, - "ndcg_at_3": 0.02259, - "ndcg_at_5": 0.02654, - "precision_at_1": 0.01484, - "precision_at_10": 0.00514, - "precision_at_100": 0.0014, - "precision_at_1000": 0.00046, - "precision_at_20": 0.00351, - "precision_at_3": 0.00939, - "precision_at_5": 0.00755, - "recall_at_1": 0.01484, - "recall_at_10": 0.05137, - "recall_at_100": 0.14038, - "recall_at_1000": 0.45989, - "recall_at_20": 0.07019, - "recall_at_3": 0.02816, - "recall_at_5": 0.03777 - }, - { - "cv_recall_at_1": 0.01677, - "cv_recall_at_10": 0.05058, - "cv_recall_at_100": 0.11572, - "cv_recall_at_1000": 0.40133, - "cv_recall_at_20": 0.06569, - "cv_recall_at_3": 0.03063, - "cv_recall_at_5": 0.03922, - "hf_subset": "hu", - "languages": [ - "hun-Latn" - ], - "main_score": 0.03203, - "map_at_1": 0.01677, - "map_at_10": 0.02629, - "map_at_100": 0.02846, - "map_at_1000": 0.02922, - "map_at_20": 0.02736, - "map_at_3": 0.02284, - "map_at_5": 0.02477, - "mrr_at_1": 0.01676829268292683, - "mrr_at_10": 0.02629199222187025, - "mrr_at_100": 0.028456210816696165, - "mrr_at_1000": 0.029224241038230778, - "mrr_at_20": 0.027362670619909213, - "mrr_at_3": 0.022842756836659275, - "mrr_at_5": 0.024769031781226896, - "nauc_cv_recall_at_1000_diff1": 0.07926305037393254, - "nauc_cv_recall_at_1000_max": 0.20067504942111922, - "nauc_cv_recall_at_1000_std": 0.2843937312557254, - "nauc_cv_recall_at_100_diff1": 0.1787803597914015, - "nauc_cv_recall_at_100_max": 0.40116950225577536, - "nauc_cv_recall_at_100_std": 0.524557608239799, - "nauc_cv_recall_at_10_diff1": 0.24628444574025682, - "nauc_cv_recall_at_10_max": 0.5619038337044497, - "nauc_cv_recall_at_10_std": 0.6709807774465408, - "nauc_cv_recall_at_1_diff1": 0.4976826031932956, - "nauc_cv_recall_at_1_max": 0.6802883947403277, - "nauc_cv_recall_at_1_std": 0.7427794453984465, - "nauc_cv_recall_at_20_diff1": 0.2206405756030709, - "nauc_cv_recall_at_20_max": 0.51654787140732, - "nauc_cv_recall_at_20_std": 0.6405806955555178, - "nauc_cv_recall_at_3_diff1": 0.3431538514634977, - "nauc_cv_recall_at_3_max": 0.6245233178374265, - "nauc_cv_recall_at_3_std": 0.7011957839961755, - "nauc_cv_recall_at_5_diff1": 0.29096232825573, - "nauc_cv_recall_at_5_max": 0.6048474626697937, - "nauc_cv_recall_at_5_std": 0.7052704818228328, - "nauc_map_at_1000_diff1": 0.34279126451861025, - "nauc_map_at_1000_max": 0.5960966964779011, - "nauc_map_at_1000_std": 0.6841950163672862, - "nauc_map_at_100_diff1": 0.34720535115453793, - "nauc_map_at_100_max": 0.6029773348188909, - "nauc_map_at_100_std": 0.6913385789082904, - "nauc_map_at_10_diff1": 0.36511963754672155, - "nauc_map_at_10_max": 0.6268488465681822, - "nauc_map_at_10_std": 0.7095996529439429, - "nauc_map_at_1_diff1": 0.4976826031932956, - "nauc_map_at_1_max": 0.6802883947403277, - "nauc_map_at_1_std": 0.7427794453984465, - "nauc_map_at_20_diff1": 0.3564859418420348, - "nauc_map_at_20_max": 0.6173916874697682, - "nauc_map_at_20_std": 0.7031393212610639, - "nauc_map_at_3_diff1": 0.4065773968949888, - "nauc_map_at_3_max": 0.6474856419723101, - "nauc_map_at_3_std": 0.7184473993852785, - "nauc_map_at_5_diff1": 0.381601324684726, - "nauc_map_at_5_max": 0.6388029077117123, - "nauc_map_at_5_std": 0.7182565082647873, - "nauc_mrr_at_1000_diff1": 0.34279126451861025, - "nauc_mrr_at_1000_max": 0.5960966964779011, - "nauc_mrr_at_1000_std": 0.6841950163672862, - "nauc_mrr_at_100_diff1": 0.34720535115453793, - "nauc_mrr_at_100_max": 0.6029773348188909, - "nauc_mrr_at_100_std": 0.6913385789082904, - "nauc_mrr_at_10_diff1": 0.36511963754672155, - "nauc_mrr_at_10_max": 0.6268488465681822, - "nauc_mrr_at_10_std": 0.7095996529439429, - "nauc_mrr_at_1_diff1": 0.4976826031932956, - "nauc_mrr_at_1_max": 0.6802883947403277, - "nauc_mrr_at_1_std": 0.7427794453984465, - "nauc_mrr_at_20_diff1": 0.3564859418420348, - "nauc_mrr_at_20_max": 0.6173916874697682, - "nauc_mrr_at_20_std": 0.7031393212610639, - "nauc_mrr_at_3_diff1": 0.4065773968949888, - "nauc_mrr_at_3_max": 0.6474856419723101, - "nauc_mrr_at_3_std": 0.7184473993852785, - "nauc_mrr_at_5_diff1": 0.381601324684726, - "nauc_mrr_at_5_max": 0.6388029077117123, - "nauc_mrr_at_5_std": 0.7182565082647873, - "nauc_ndcg_at_1000_diff1": 0.2043031642322968, - "nauc_ndcg_at_1000_max": 0.40307981800072007, - "nauc_ndcg_at_1000_std": 0.4961372678753577, - "nauc_ndcg_at_100_diff1": 0.2667995253569192, - "nauc_ndcg_at_100_max": 0.5144267445806551, - "nauc_ndcg_at_100_std": 0.620689438403104, - "nauc_ndcg_at_10_diff1": 0.3210474233046296, - "nauc_ndcg_at_10_max": 0.6036012095205587, - "nauc_ndcg_at_10_std": 0.6958951414348914, - "nauc_ndcg_at_1_diff1": 0.4976826031932956, - "nauc_ndcg_at_1_max": 0.6802883947403277, - "nauc_ndcg_at_1_std": 0.7427794453984465, - "nauc_ndcg_at_20_diff1": 0.3014415857234393, - "nauc_ndcg_at_20_max": 0.5787442865570296, - "nauc_ndcg_at_20_std": 0.6792556751813358, - "nauc_ndcg_at_3_diff1": 0.3864969421729367, - "nauc_ndcg_at_3_max": 0.640218475645307, - "nauc_ndcg_at_3_std": 0.7129925772036456, - "nauc_ndcg_at_5_diff1": 0.3505090436279808, - "nauc_ndcg_at_5_max": 0.6272360189711365, - "nauc_ndcg_at_5_std": 0.7136074643615019, - "nauc_precision_at_1000_diff1": 0.07926305037393205, - "nauc_precision_at_1000_max": 0.20067504942111877, - "nauc_precision_at_1000_std": 0.284393731255725, - "nauc_precision_at_100_diff1": 0.1787803597914016, - "nauc_precision_at_100_max": 0.40116950225577563, - "nauc_precision_at_100_std": 0.524557608239799, - "nauc_precision_at_10_diff1": 0.24628444574025693, - "nauc_precision_at_10_max": 0.5619038337044496, - "nauc_precision_at_10_std": 0.6709807774465409, - "nauc_precision_at_1_diff1": 0.4976826031932956, - "nauc_precision_at_1_max": 0.6802883947403277, - "nauc_precision_at_1_std": 0.7427794453984465, - "nauc_precision_at_20_diff1": 0.22064057560307082, - "nauc_precision_at_20_max": 0.51654787140732, - "nauc_precision_at_20_std": 0.6405806955555178, - "nauc_precision_at_3_diff1": 0.34315385146349775, - "nauc_precision_at_3_max": 0.6245233178374265, - "nauc_precision_at_3_std": 0.7011957839961754, - "nauc_precision_at_5_diff1": 0.2909623282557299, - "nauc_precision_at_5_max": 0.6048474626697936, - "nauc_precision_at_5_std": 0.7052704818228327, - "nauc_recall_at_1000_diff1": 0.07926305037393254, - "nauc_recall_at_1000_max": 0.20067504942111922, - "nauc_recall_at_1000_std": 0.2843937312557254, - "nauc_recall_at_100_diff1": 0.1787803597914015, - "nauc_recall_at_100_max": 0.40116950225577536, - "nauc_recall_at_100_std": 0.524557608239799, - "nauc_recall_at_10_diff1": 0.24628444574025682, - "nauc_recall_at_10_max": 0.5619038337044497, - "nauc_recall_at_10_std": 0.6709807774465408, - "nauc_recall_at_1_diff1": 0.4976826031932956, - "nauc_recall_at_1_max": 0.6802883947403277, - "nauc_recall_at_1_std": 0.7427794453984465, - "nauc_recall_at_20_diff1": 0.2206405756030709, - "nauc_recall_at_20_max": 0.51654787140732, - "nauc_recall_at_20_std": 0.6405806955555178, - "nauc_recall_at_3_diff1": 0.3431538514634977, - "nauc_recall_at_3_max": 0.6245233178374265, - "nauc_recall_at_3_std": 0.7011957839961755, - "nauc_recall_at_5_diff1": 0.29096232825573, - "nauc_recall_at_5_max": 0.6048474626697937, - "nauc_recall_at_5_std": 0.7052704818228328, - "ndcg_at_1": 0.01677, - "ndcg_at_10": 0.03203, - "ndcg_at_100": 0.04477, - "ndcg_at_1000": 0.07744, - "ndcg_at_20": 0.03587, - "ndcg_at_3": 0.02484, - "ndcg_at_5": 0.02835, - "precision_at_1": 0.01677, - "precision_at_10": 0.00506, - "precision_at_100": 0.00116, - "precision_at_1000": 0.0004, - "precision_at_20": 0.00328, - "precision_at_3": 0.01021, - "precision_at_5": 0.00784, - "recall_at_1": 0.01677, - "recall_at_10": 0.05058, - "recall_at_100": 0.11572, - "recall_at_1000": 0.40133, - "recall_at_20": 0.06569, - "recall_at_3": 0.03063, - "recall_at_5": 0.03922 - }, - { - "cv_recall_at_1": 0.05726, - "cv_recall_at_10": 0.1691, - "cv_recall_at_100": 0.38675, - "cv_recall_at_1000": 0.76172, - "cv_recall_at_20": 0.22214, - "cv_recall_at_3": 0.10342, - "cv_recall_at_5": 0.12939, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.10767, - "map_at_1": 0.05726, - "map_at_10": 0.08873, - "map_at_100": 0.09617, - "map_at_1000": 0.09752, - "map_at_20": 0.09236, - "map_at_3": 0.07758, - "map_at_5": 0.08348, - "mrr_at_1": 0.05725512208812798, - "mrr_at_10": 0.08872622722959524, - "mrr_at_100": 0.09617481516990756, - "mrr_at_1000": 0.09752354332896494, - "mrr_at_20": 0.09236356409778489, - "mrr_at_3": 0.07757975488820275, - "mrr_at_5": 0.08348068107400132, - "nauc_cv_recall_at_1000_diff1": 0.15030287323588018, - "nauc_cv_recall_at_1000_max": 0.3191062141405888, - "nauc_cv_recall_at_1000_std": 0.4750670986169835, - "nauc_cv_recall_at_100_diff1": 0.16397068588078975, - "nauc_cv_recall_at_100_max": 0.28952305269069245, - "nauc_cv_recall_at_100_std": 0.3689023094107915, - "nauc_cv_recall_at_10_diff1": 0.19309923383240687, - "nauc_cv_recall_at_10_max": 0.3141285805092174, - "nauc_cv_recall_at_10_std": 0.29483488469475855, - "nauc_cv_recall_at_1_diff1": 0.40092103200855395, - "nauc_cv_recall_at_1_max": 0.4284863429779795, - "nauc_cv_recall_at_1_std": 0.28322408485037104, - "nauc_cv_recall_at_20_diff1": 0.1615483611160991, - "nauc_cv_recall_at_20_max": 0.2894060506254236, - "nauc_cv_recall_at_20_std": 0.3123395488443825, - "nauc_cv_recall_at_3_diff1": 0.2672418202383049, - "nauc_cv_recall_at_3_max": 0.34414828955915494, - "nauc_cv_recall_at_3_std": 0.2533006977882842, - "nauc_cv_recall_at_5_diff1": 0.22732741825710243, - "nauc_cv_recall_at_5_max": 0.3295891147074863, - "nauc_cv_recall_at_5_std": 0.28159355610020537, - "nauc_map_at_1000_diff1": 0.2868239638816492, - "nauc_map_at_1000_max": 0.3617561081540899, - "nauc_map_at_1000_std": 0.2831757076010107, - "nauc_map_at_100_diff1": 0.2868253956883315, - "nauc_map_at_100_max": 0.36157898123651255, - "nauc_map_at_100_std": 0.28267515257161274, - "nauc_map_at_10_diff1": 0.29513212149528517, - "nauc_map_at_10_max": 0.3676838581354323, - "nauc_map_at_10_std": 0.2768791058387735, - "nauc_map_at_1_diff1": 0.40092103200855395, - "nauc_map_at_1_max": 0.4284863429779795, - "nauc_map_at_1_std": 0.28322408485037104, - "nauc_map_at_20_diff1": 0.2884731293833199, - "nauc_map_at_20_max": 0.3628246225023507, - "nauc_map_at_20_std": 0.2794205983087937, - "nauc_map_at_3_diff1": 0.3220621170390292, - "nauc_map_at_3_max": 0.38001448681516115, - "nauc_map_at_3_std": 0.2653154290950081, - "nauc_map_at_5_diff1": 0.306102269865566, - "nauc_map_at_5_max": 0.3727343689500744, - "nauc_map_at_5_std": 0.2740171214843683, - "nauc_mrr_at_1000_diff1": 0.28682396238682445, - "nauc_mrr_at_1000_max": 0.3617561074098036, - "nauc_mrr_at_1000_std": 0.28317571035510314, - "nauc_mrr_at_100_diff1": 0.2868253956883315, - "nauc_mrr_at_100_max": 0.36157898123651255, - "nauc_mrr_at_100_std": 0.28267515257161274, - "nauc_mrr_at_10_diff1": 0.29513212149528517, - "nauc_mrr_at_10_max": 0.3676838581354323, - "nauc_mrr_at_10_std": 0.2768791058387735, - "nauc_mrr_at_1_diff1": 0.40092103200855395, - "nauc_mrr_at_1_max": 0.4284863429779795, - "nauc_mrr_at_1_std": 0.28322408485037104, - "nauc_mrr_at_20_diff1": 0.2884731293833199, - "nauc_mrr_at_20_max": 0.3628246225023507, - "nauc_mrr_at_20_std": 0.2794205983087937, - "nauc_mrr_at_3_diff1": 0.3220621170390292, - "nauc_mrr_at_3_max": 0.38001448681516115, - "nauc_mrr_at_3_std": 0.2653154290950081, - "nauc_mrr_at_5_diff1": 0.306102269865566, - "nauc_mrr_at_5_max": 0.3727343689500744, - "nauc_mrr_at_5_std": 0.2740171214843683, - "nauc_ndcg_at_1000_diff1": 0.24161348000786495, - "nauc_ndcg_at_1000_max": 0.34037403947611655, - "nauc_ndcg_at_1000_std": 0.32039258837486695, - "nauc_ndcg_at_100_diff1": 0.23785912449290542, - "nauc_ndcg_at_100_max": 0.3328042419327395, - "nauc_ndcg_at_100_std": 0.3098300951168714, - "nauc_ndcg_at_10_diff1": 0.25966152812590537, - "nauc_ndcg_at_10_max": 0.348885270202277, - "nauc_ndcg_at_10_std": 0.28238321943430694, - "nauc_ndcg_at_1_diff1": 0.40092103200855395, - "nauc_ndcg_at_1_max": 0.4284863429779795, - "nauc_ndcg_at_1_std": 0.28322408485037104, - "nauc_ndcg_at_20_diff1": 0.24229561650784148, - "nauc_ndcg_at_20_max": 0.3361781384678798, - "nauc_ndcg_at_20_std": 0.2901134461141482, - "nauc_ndcg_at_3_diff1": 0.30481794493109293, - "nauc_ndcg_at_3_max": 0.3687831617083144, - "nauc_ndcg_at_3_std": 0.2615259423663829, - "nauc_ndcg_at_5_diff1": 0.2803326002641599, - "nauc_ndcg_at_5_max": 0.3584572010028402, - "nauc_ndcg_at_5_std": 0.27604262728435447, - "nauc_precision_at_1000_diff1": 0.15030287323587857, - "nauc_precision_at_1000_max": 0.3191062141405875, - "nauc_precision_at_1000_std": 0.47506709861698254, - "nauc_precision_at_100_diff1": 0.16397068588079, - "nauc_precision_at_100_max": 0.2895230526906928, - "nauc_precision_at_100_std": 0.3689023094107917, - "nauc_precision_at_10_diff1": 0.19309923383240699, - "nauc_precision_at_10_max": 0.31412858050921744, - "nauc_precision_at_10_std": 0.29483488469475866, - "nauc_precision_at_1_diff1": 0.40092103200855395, - "nauc_precision_at_1_max": 0.4284863429779795, - "nauc_precision_at_1_std": 0.28322408485037104, - "nauc_precision_at_20_diff1": 0.16154836111609902, - "nauc_precision_at_20_max": 0.28940605062542357, - "nauc_precision_at_20_std": 0.3123395488443823, - "nauc_precision_at_3_diff1": 0.2672418202383047, - "nauc_precision_at_3_max": 0.3441482895591548, - "nauc_precision_at_3_std": 0.2533006977882841, - "nauc_precision_at_5_diff1": 0.22732741825710256, - "nauc_precision_at_5_max": 0.32958911470748636, - "nauc_precision_at_5_std": 0.28159355610020526, - "nauc_recall_at_1000_diff1": 0.15030287323588018, - "nauc_recall_at_1000_max": 0.3191062141405888, - "nauc_recall_at_1000_std": 0.4750670986169835, - "nauc_recall_at_100_diff1": 0.16397068588078975, - "nauc_recall_at_100_max": 0.28952305269069245, - "nauc_recall_at_100_std": 0.3689023094107915, - "nauc_recall_at_10_diff1": 0.19309923383240687, - "nauc_recall_at_10_max": 0.3141285805092174, - "nauc_recall_at_10_std": 0.29483488469475855, - "nauc_recall_at_1_diff1": 0.40092103200855395, - "nauc_recall_at_1_max": 0.4284863429779795, - "nauc_recall_at_1_std": 0.28322408485037104, - "nauc_recall_at_20_diff1": 0.1615483611160991, - "nauc_recall_at_20_max": 0.2894060506254236, - "nauc_recall_at_20_std": 0.3123395488443825, - "nauc_recall_at_3_diff1": 0.2672418202383049, - "nauc_recall_at_3_max": 0.34414828955915494, - "nauc_recall_at_3_std": 0.2533006977882842, - "nauc_recall_at_5_diff1": 0.22732741825710243, - "nauc_recall_at_5_max": 0.3295891147074863, - "nauc_recall_at_5_std": 0.28159355610020537, - "ndcg_at_1": 0.05726, - "ndcg_at_10": 0.10767, - "ndcg_at_100": 0.15073, - "ndcg_at_1000": 0.19579, - "ndcg_at_20": 0.12102, - "ndcg_at_3": 0.08422, - "ndcg_at_5": 0.09488, - "precision_at_1": 0.05726, - "precision_at_10": 0.01691, - "precision_at_100": 0.00387, - "precision_at_1000": 0.00076, - "precision_at_20": 0.01111, - "precision_at_3": 0.03447, - "precision_at_5": 0.02588, - "recall_at_1": 0.05726, - "recall_at_10": 0.1691, - "recall_at_100": 0.38675, - "recall_at_1000": 0.76172, - "recall_at_20": 0.22214, - "recall_at_3": 0.10342, - "recall_at_5": 0.12939 - }, - { - "cv_recall_at_1": 0.08346, - "cv_recall_at_10": 0.25369, - "cv_recall_at_100": 0.52331, - "cv_recall_at_1000": 0.85763, - "cv_recall_at_20": 0.32523, - "cv_recall_at_3": 0.1504, - "cv_recall_at_5": 0.18841, - "hf_subset": "it", - "languages": [ - "ita-Latn" - ], - "main_score": 0.15843, - "map_at_1": 0.08346, - "map_at_10": 0.12922, - "map_at_100": 0.13889, - "map_at_1000": 0.14021, - "map_at_20": 0.13422, - "map_at_3": 0.11191, - "map_at_5": 0.12055, - "mrr_at_1": 0.08346122063510801, - "mrr_at_10": 0.12921868822293828, - "mrr_at_100": 0.13888512076542645, - "mrr_at_1000": 0.14020972512942267, - "mrr_at_20": 0.13422039590574278, - "mrr_at_3": 0.11191122653759927, - "mrr_at_5": 0.12055247314366743, - "nauc_cv_recall_at_1000_diff1": 0.06115378612995843, - "nauc_cv_recall_at_1000_max": 0.2198677467029866, - "nauc_cv_recall_at_1000_std": 0.5562687441684824, - "nauc_cv_recall_at_100_diff1": 0.09190227414493322, - "nauc_cv_recall_at_100_max": 0.14206614111453056, - "nauc_cv_recall_at_100_std": 0.35163639320895096, - "nauc_cv_recall_at_10_diff1": 0.13034473302850733, - "nauc_cv_recall_at_10_max": 0.15694257987261587, - "nauc_cv_recall_at_10_std": 0.22673453590977974, - "nauc_cv_recall_at_1_diff1": 0.28469477006474864, - "nauc_cv_recall_at_1_max": 0.24277500552741196, - "nauc_cv_recall_at_1_std": 0.14989170009299607, - "nauc_cv_recall_at_20_diff1": 0.1139533115734007, - "nauc_cv_recall_at_20_max": 0.15001100076217513, - "nauc_cv_recall_at_20_std": 0.24987924114844598, - "nauc_cv_recall_at_3_diff1": 0.1723560568226035, - "nauc_cv_recall_at_3_max": 0.1899520873785719, - "nauc_cv_recall_at_3_std": 0.16313589784115362, - "nauc_cv_recall_at_5_diff1": 0.14723328888291767, - "nauc_cv_recall_at_5_max": 0.17217984968931596, - "nauc_cv_recall_at_5_std": 0.18099972391070385, - "nauc_map_at_1000_diff1": 0.2032600423187777, - "nauc_map_at_1000_max": 0.20174194367014686, - "nauc_map_at_1000_std": 0.17851924731958435, - "nauc_map_at_100_diff1": 0.2029155467410285, - "nauc_map_at_100_max": 0.20116705824454936, - "nauc_map_at_100_std": 0.17824653925030834, - "nauc_map_at_10_diff1": 0.20648651750802421, - "nauc_map_at_10_max": 0.203185940142513, - "nauc_map_at_10_std": 0.1707974532854906, - "nauc_map_at_1_diff1": 0.28469477006474864, - "nauc_map_at_1_max": 0.24277500552741196, - "nauc_map_at_1_std": 0.14989170009299607, - "nauc_map_at_20_diff1": 0.2034704607134265, - "nauc_map_at_20_max": 0.20159871772025179, - "nauc_map_at_20_std": 0.1739055372628376, - "nauc_map_at_3_diff1": 0.22302423083706688, - "nauc_map_at_3_max": 0.21488035339192169, - "nauc_map_at_3_std": 0.15477364907074237, - "nauc_map_at_5_diff1": 0.2121914715975426, - "nauc_map_at_5_max": 0.20762170454855294, - "nauc_map_at_5_std": 0.1610095680454263, - "nauc_mrr_at_1000_diff1": 0.2032600423187777, - "nauc_mrr_at_1000_max": 0.20174194367014686, - "nauc_mrr_at_1000_std": 0.17851924731958435, - "nauc_mrr_at_100_diff1": 0.2029155467410285, - "nauc_mrr_at_100_max": 0.20116705824454936, - "nauc_mrr_at_100_std": 0.17824653925030834, - "nauc_mrr_at_10_diff1": 0.20648651750802421, - "nauc_mrr_at_10_max": 0.203185940142513, - "nauc_mrr_at_10_std": 0.1707974532854906, - "nauc_mrr_at_1_diff1": 0.28469477006474864, - "nauc_mrr_at_1_max": 0.24277500552741196, - "nauc_mrr_at_1_std": 0.14989170009299607, - "nauc_mrr_at_20_diff1": 0.2034704607134265, - "nauc_mrr_at_20_max": 0.20159871772025179, - "nauc_mrr_at_20_std": 0.1739055372628376, - "nauc_mrr_at_3_diff1": 0.22302423083706688, - "nauc_mrr_at_3_max": 0.21488035339192169, - "nauc_mrr_at_3_std": 0.15477364907074237, - "nauc_mrr_at_5_diff1": 0.2121914715975426, - "nauc_mrr_at_5_max": 0.20762170454855294, - "nauc_mrr_at_5_std": 0.1610095680454263, - "nauc_ndcg_at_1000_diff1": 0.17430403953515997, - "nauc_ndcg_at_1000_max": 0.19342891016794667, - "nauc_ndcg_at_1000_std": 0.22824905385496075, - "nauc_ndcg_at_100_diff1": 0.1676830441466766, - "nauc_ndcg_at_100_max": 0.1817532485004325, - "nauc_ndcg_at_100_std": 0.2249159052021771, - "nauc_ndcg_at_10_diff1": 0.18074395377365038, - "nauc_ndcg_at_10_max": 0.1877903243250276, - "nauc_ndcg_at_10_std": 0.1882740651781376, - "nauc_ndcg_at_1_diff1": 0.28469477006474864, - "nauc_ndcg_at_1_max": 0.24277500552741196, - "nauc_ndcg_at_1_std": 0.14989170009299607, - "nauc_ndcg_at_20_diff1": 0.17271785778675547, - "nauc_ndcg_at_20_max": 0.18391476422554845, - "nauc_ndcg_at_20_std": 0.19791156758163084, - "nauc_ndcg_at_3_diff1": 0.20761307528172798, - "nauc_ndcg_at_3_max": 0.2073413273262493, - "nauc_ndcg_at_3_std": 0.15722376432472251, - "nauc_ndcg_at_5_diff1": 0.1912682684291303, - "nauc_ndcg_at_5_max": 0.19628937163214283, - "nauc_ndcg_at_5_std": 0.1672009773360082, - "nauc_precision_at_1000_diff1": 0.06115378612995736, - "nauc_precision_at_1000_max": 0.21986774670298528, - "nauc_precision_at_1000_std": 0.556268744168481, - "nauc_precision_at_100_diff1": 0.09190227414493372, - "nauc_precision_at_100_max": 0.14206614111453086, - "nauc_precision_at_100_std": 0.35163639320895096, - "nauc_precision_at_10_diff1": 0.13034473302850733, - "nauc_precision_at_10_max": 0.15694257987261595, - "nauc_precision_at_10_std": 0.22673453590977952, - "nauc_precision_at_1_diff1": 0.28469477006474864, - "nauc_precision_at_1_max": 0.24277500552741196, - "nauc_precision_at_1_std": 0.14989170009299607, - "nauc_precision_at_20_diff1": 0.11395331157340065, - "nauc_precision_at_20_max": 0.15001100076217486, - "nauc_precision_at_20_std": 0.24987924114844567, - "nauc_precision_at_3_diff1": 0.1723560568226034, - "nauc_precision_at_3_max": 0.1899520873785718, - "nauc_precision_at_3_std": 0.16313589784115368, - "nauc_precision_at_5_diff1": 0.1472332888829176, - "nauc_precision_at_5_max": 0.17217984968931588, - "nauc_precision_at_5_std": 0.1809997239107037, - "nauc_recall_at_1000_diff1": 0.06115378612995843, - "nauc_recall_at_1000_max": 0.2198677467029866, - "nauc_recall_at_1000_std": 0.5562687441684824, - "nauc_recall_at_100_diff1": 0.09190227414493322, - "nauc_recall_at_100_max": 0.14206614111453056, - "nauc_recall_at_100_std": 0.35163639320895096, - "nauc_recall_at_10_diff1": 0.13034473302850733, - "nauc_recall_at_10_max": 0.15694257987261587, - "nauc_recall_at_10_std": 0.22673453590977974, - "nauc_recall_at_1_diff1": 0.28469477006474864, - "nauc_recall_at_1_max": 0.24277500552741196, - "nauc_recall_at_1_std": 0.14989170009299607, - "nauc_recall_at_20_diff1": 0.1139533115734007, - "nauc_recall_at_20_max": 0.15001100076217513, - "nauc_recall_at_20_std": 0.24987924114844598, - "nauc_recall_at_3_diff1": 0.1723560568226035, - "nauc_recall_at_3_max": 0.1899520873785719, - "nauc_recall_at_3_std": 0.16313589784115362, - "nauc_recall_at_5_diff1": 0.14723328888291767, - "nauc_recall_at_5_max": 0.17217984968931596, - "nauc_recall_at_5_std": 0.18099972391070385, - "ndcg_at_1": 0.08346, - "ndcg_at_10": 0.15843, - "ndcg_at_100": 0.21249, - "ndcg_at_1000": 0.25338, - "ndcg_at_20": 0.17656, - "ndcg_at_3": 0.12175, - "ndcg_at_5": 0.13737, - "precision_at_1": 0.08346, - "precision_at_10": 0.02537, - "precision_at_100": 0.00523, - "precision_at_1000": 0.00086, - "precision_at_20": 0.01626, - "precision_at_3": 0.05013, - "precision_at_5": 0.03768, - "recall_at_1": 0.08346, - "recall_at_10": 0.25369, - "recall_at_100": 0.52331, - "recall_at_1000": 0.85763, - "recall_at_20": 0.32523, - "recall_at_3": 0.1504, - "recall_at_5": 0.18841 - }, - { - "cv_recall_at_1": 0.01754, - "cv_recall_at_10": 0.07474, - "cv_recall_at_100": 0.25678, - "cv_recall_at_1000": 0.67794, - "cv_recall_at_20": 0.11162, - "cv_recall_at_3": 0.03674, - "cv_recall_at_5": 0.05177, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.04196, - "map_at_1": 0.01754, - "map_at_10": 0.03201, - "map_at_100": 0.03789, - "map_at_1000": 0.03937, - "map_at_20": 0.03456, - "map_at_3": 0.02568, - "map_at_5": 0.02904, - "mrr_at_1": 0.01753653444676409, - "mrr_at_10": 0.03200936695717485, - "mrr_at_100": 0.03788702657630412, - "mrr_at_1000": 0.03936756710182338, - "mrr_at_20": 0.0345623431663912, - "mrr_at_3": 0.025678496868476012, - "mrr_at_5": 0.029039665970772448, - "nauc_cv_recall_at_1000_diff1": 0.10757708129019053, - "nauc_cv_recall_at_1000_max": 0.2961920238707485, - "nauc_cv_recall_at_1000_std": 0.4776809922444095, - "nauc_cv_recall_at_100_diff1": 0.09135429300688364, - "nauc_cv_recall_at_100_max": 0.22092085853045743, - "nauc_cv_recall_at_100_std": 0.39170988106526494, - "nauc_cv_recall_at_10_diff1": 0.11416323917396812, - "nauc_cv_recall_at_10_max": 0.21204992762585068, - "nauc_cv_recall_at_10_std": 0.32864092293250496, - "nauc_cv_recall_at_1_diff1": 0.29466646698439264, - "nauc_cv_recall_at_1_max": 0.29818456527762005, - "nauc_cv_recall_at_1_std": 0.29694254864600017, - "nauc_cv_recall_at_20_diff1": 0.09575980053421915, - "nauc_cv_recall_at_20_max": 0.20513323624093097, - "nauc_cv_recall_at_20_std": 0.35369955523262436, - "nauc_cv_recall_at_3_diff1": 0.16073635724839264, - "nauc_cv_recall_at_3_max": 0.2627261073098647, - "nauc_cv_recall_at_3_std": 0.3374073837314188, - "nauc_cv_recall_at_5_diff1": 0.16451856690252917, - "nauc_cv_recall_at_5_max": 0.2463348129018922, - "nauc_cv_recall_at_5_std": 0.34402122105642435, - "nauc_map_at_1000_diff1": 0.17404810974414622, - "nauc_map_at_1000_max": 0.2511168768175765, - "nauc_map_at_1000_std": 0.32791949498308215, - "nauc_map_at_100_diff1": 0.17408071817208495, - "nauc_map_at_100_max": 0.25012756041114087, - "nauc_map_at_100_std": 0.3262846189521321, - "nauc_map_at_10_diff1": 0.18821696482519734, - "nauc_map_at_10_max": 0.257032447064839, - "nauc_map_at_10_std": 0.3172818065044762, - "nauc_map_at_1_diff1": 0.29466646698439264, - "nauc_map_at_1_max": 0.29818456527762005, - "nauc_map_at_1_std": 0.29694254864600017, - "nauc_map_at_20_diff1": 0.17937160474901026, - "nauc_map_at_20_max": 0.2520240041798804, - "nauc_map_at_20_std": 0.32251538668153346, - "nauc_map_at_3_diff1": 0.21001530584534686, - "nauc_map_at_3_max": 0.278535322608543, - "nauc_map_at_3_std": 0.31651830882096854, - "nauc_map_at_5_diff1": 0.20592982486101274, - "nauc_map_at_5_max": 0.2700919615800627, - "nauc_map_at_5_std": 0.32095831129341523, - "nauc_mrr_at_1000_diff1": 0.17404810757470401, - "nauc_mrr_at_1000_max": 0.25111687217927603, - "nauc_mrr_at_1000_std": 0.32791949494354067, - "nauc_mrr_at_100_diff1": 0.17408071817208495, - "nauc_mrr_at_100_max": 0.25012756041114087, - "nauc_mrr_at_100_std": 0.3262846189521321, - "nauc_mrr_at_10_diff1": 0.18821696482519734, - "nauc_mrr_at_10_max": 0.257032447064839, - "nauc_mrr_at_10_std": 0.3172818065044762, - "nauc_mrr_at_1_diff1": 0.29466646698439264, - "nauc_mrr_at_1_max": 0.29818456527762005, - "nauc_mrr_at_1_std": 0.29694254864600017, - "nauc_mrr_at_20_diff1": 0.17937160474901026, - "nauc_mrr_at_20_max": 0.2520240041798804, - "nauc_mrr_at_20_std": 0.32251538668153346, - "nauc_mrr_at_3_diff1": 0.21001530584534686, - "nauc_mrr_at_3_max": 0.278535322608543, - "nauc_mrr_at_3_std": 0.31651830882096854, - "nauc_mrr_at_5_diff1": 0.20592982486101274, - "nauc_mrr_at_5_max": 0.2700919615800627, - "nauc_mrr_at_5_std": 0.32095831129341523, - "nauc_ndcg_at_1000_diff1": 0.1344421999066752, - "nauc_ndcg_at_1000_max": 0.25423268532569104, - "nauc_ndcg_at_1000_std": 0.3822515103732003, - "nauc_ndcg_at_100_diff1": 0.12834258133928364, - "nauc_ndcg_at_100_max": 0.23140922516545348, - "nauc_ndcg_at_100_std": 0.35678364039747357, - "nauc_ndcg_at_10_diff1": 0.1581123115368676, - "nauc_ndcg_at_10_max": 0.23894108805699588, - "nauc_ndcg_at_10_std": 0.32222987084202304, - "nauc_ndcg_at_1_diff1": 0.29466646698439264, - "nauc_ndcg_at_1_max": 0.29818456527762005, - "nauc_ndcg_at_1_std": 0.29694254864600017, - "nauc_ndcg_at_20_diff1": 0.1409982120405537, - "nauc_ndcg_at_20_max": 0.23033385189500785, - "nauc_ndcg_at_20_std": 0.3360002313034926, - "nauc_ndcg_at_3_diff1": 0.19365403664588626, - "nauc_ndcg_at_3_max": 0.2733960002514536, - "nauc_ndcg_at_3_std": 0.3232158137802028, - "nauc_ndcg_at_5_diff1": 0.19021691859943332, - "nauc_ndcg_at_5_max": 0.2614640537553007, - "nauc_ndcg_at_5_std": 0.3293040490494083, - "nauc_precision_at_1000_diff1": 0.10757708129018899, - "nauc_precision_at_1000_max": 0.29619202387074844, - "nauc_precision_at_1000_std": 0.4776809922444093, - "nauc_precision_at_100_diff1": 0.09135429300688377, - "nauc_precision_at_100_max": 0.2209208585304576, - "nauc_precision_at_100_std": 0.39170988106526494, - "nauc_precision_at_10_diff1": 0.11416323917396817, - "nauc_precision_at_10_max": 0.2120499276258509, - "nauc_precision_at_10_std": 0.3286409229325053, - "nauc_precision_at_1_diff1": 0.29466646698439264, - "nauc_precision_at_1_max": 0.29818456527762005, - "nauc_precision_at_1_std": 0.29694254864600017, - "nauc_precision_at_20_diff1": 0.09575980053421912, - "nauc_precision_at_20_max": 0.20513323624093116, - "nauc_precision_at_20_std": 0.3536995552326241, - "nauc_precision_at_3_diff1": 0.16073635724839253, - "nauc_precision_at_3_max": 0.2627261073098646, - "nauc_precision_at_3_std": 0.33740738373141876, - "nauc_precision_at_5_diff1": 0.16451856690252925, - "nauc_precision_at_5_max": 0.24633481290189227, - "nauc_precision_at_5_std": 0.34402122105642446, - "nauc_recall_at_1000_diff1": 0.10757708129019053, - "nauc_recall_at_1000_max": 0.2961920238707485, - "nauc_recall_at_1000_std": 0.4776809922444095, - "nauc_recall_at_100_diff1": 0.09135429300688364, - "nauc_recall_at_100_max": 0.22092085853045743, - "nauc_recall_at_100_std": 0.39170988106526494, - "nauc_recall_at_10_diff1": 0.11416323917396812, - "nauc_recall_at_10_max": 0.21204992762585068, - "nauc_recall_at_10_std": 0.32864092293250496, - "nauc_recall_at_1_diff1": 0.29466646698439264, - "nauc_recall_at_1_max": 0.29818456527762005, - "nauc_recall_at_1_std": 0.29694254864600017, - "nauc_recall_at_20_diff1": 0.09575980053421915, - "nauc_recall_at_20_max": 0.20513323624093097, - "nauc_recall_at_20_std": 0.35369955523262436, - "nauc_recall_at_3_diff1": 0.16073635724839264, - "nauc_recall_at_3_max": 0.2627261073098647, - "nauc_recall_at_3_std": 0.3374073837314188, - "nauc_recall_at_5_diff1": 0.16451856690252917, - "nauc_recall_at_5_max": 0.2463348129018922, - "nauc_recall_at_5_std": 0.34402122105642435, - "ndcg_at_1": 0.01754, - "ndcg_at_10": 0.04196, - "ndcg_at_100": 0.0774, - "ndcg_at_1000": 0.12779, - "ndcg_at_20": 0.05128, - "ndcg_at_3": 0.02851, - "ndcg_at_5": 0.03463, - "precision_at_1": 0.01754, - "precision_at_10": 0.00747, - "precision_at_100": 0.00257, - "precision_at_1000": 0.00068, - "precision_at_20": 0.00558, - "precision_at_3": 0.01225, - "precision_at_5": 0.01035, - "recall_at_1": 0.01754, - "recall_at_10": 0.07474, - "recall_at_100": 0.25678, - "recall_at_1000": 0.67794, - "recall_at_20": 0.11162, - "recall_at_3": 0.03674, - "recall_at_5": 0.05177 - }, - { - "cv_recall_at_1": 0.00078, - "cv_recall_at_10": 0.00405, - "cv_recall_at_100": 0.03608, - "cv_recall_at_1000": 0.30301, - "cv_recall_at_20": 0.00797, - "cv_recall_at_3": 0.0017, - "cv_recall_at_5": 0.00235, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.0021, - "map_at_1": 0.00078, - "map_at_10": 0.00152, - "map_at_100": 0.00235, - "map_at_1000": 0.00305, - "map_at_20": 0.00179, - "map_at_3": 0.00115, - "map_at_5": 0.00131, - "mrr_at_1": 0.000784313725490196, - "mrr_at_10": 0.0015218902375765124, - "mrr_at_100": 0.002352320429387841, - "mrr_at_1000": 0.0030529639414161465, - "mrr_at_20": 0.001791004357188416, - "mrr_at_3": 0.0011546840958605665, - "mrr_at_5": 0.0013050108932461874, - "nauc_cv_recall_at_1000_diff1": 0.008327387047313645, - "nauc_cv_recall_at_1000_max": 0.020073873859256595, - "nauc_cv_recall_at_1000_std": 0.044956744075964845, - "nauc_cv_recall_at_100_diff1": -0.01001893740545887, - "nauc_cv_recall_at_100_max": 0.05164234357197578, - "nauc_cv_recall_at_100_std": 0.18444138542480845, - "nauc_cv_recall_at_10_diff1": 0.04428884253529171, - "nauc_cv_recall_at_10_max": 0.18272068687952775, - "nauc_cv_recall_at_10_std": 0.28539775136969114, - "nauc_cv_recall_at_1_diff1": 0.17909847061980153, - "nauc_cv_recall_at_1_max": 0.7746176549503622, - "nauc_cv_recall_at_1_std": 0.9436544137375904, - "nauc_cv_recall_at_20_diff1": 0.09493417551144288, - "nauc_cv_recall_at_20_max": 0.15466225637461684, - "nauc_cv_recall_at_20_std": 0.23296546688542194, - "nauc_cv_recall_at_3_diff1": -0.002621204928690842, - "nauc_cv_recall_at_3_max": 0.5246331345070279, - "nauc_cv_recall_at_3_std": 0.5633320261707704, - "nauc_cv_recall_at_5_diff1": 0.012744834987925951, - "nauc_cv_recall_at_5_max": 0.3523238827773307, - "nauc_cv_recall_at_5_std": 0.5068270578063978, - "nauc_map_at_1000_diff1": 0.04187726128428896, - "nauc_map_at_1000_max": 0.25894178858660327, - "nauc_map_at_1000_std": 0.3620957970584589, - "nauc_map_at_100_diff1": 0.04651947833130733, - "nauc_map_at_100_max": 0.31442613006940473, - "nauc_map_at_100_std": 0.4353382498271757, - "nauc_map_at_10_diff1": 0.058577903756969585, - "nauc_map_at_10_max": 0.4602039446126341, - "nauc_map_at_10_std": 0.5739097636756447, - "nauc_map_at_1_diff1": 0.17909847061980153, - "nauc_map_at_1_max": 0.7746176549503622, - "nauc_map_at_1_std": 0.9436544137375904, - "nauc_map_at_20_diff1": 0.07270252474050658, - "nauc_map_at_20_max": 0.41226148400652984, - "nauc_map_at_20_std": 0.5140015311159577, - "nauc_map_at_3_diff1": 0.05353590069406826, - "nauc_map_at_3_max": 0.6316628782317711, - "nauc_map_at_3_std": 0.7147688210964356, - "nauc_map_at_5_diff1": 0.051837133080668826, - "nauc_map_at_5_max": 0.54565183991027, - "nauc_map_at_5_std": 0.6708672400517273, - "nauc_mrr_at_1000_diff1": 0.041877343672373425, - "nauc_mrr_at_1000_max": 0.2589420300194208, - "nauc_mrr_at_1000_std": 0.36209599021851346, - "nauc_mrr_at_100_diff1": 0.04651947833130733, - "nauc_mrr_at_100_max": 0.31442613006940473, - "nauc_mrr_at_100_std": 0.4353382498271757, - "nauc_mrr_at_10_diff1": 0.058577903756969585, - "nauc_mrr_at_10_max": 0.4602039446126341, - "nauc_mrr_at_10_std": 0.5739097636756447, - "nauc_mrr_at_1_diff1": 0.17909847061980153, - "nauc_mrr_at_1_max": 0.7746176549503622, - "nauc_mrr_at_1_std": 0.9436544137375904, - "nauc_mrr_at_20_diff1": 0.07270252474050658, - "nauc_mrr_at_20_max": 0.41226148400652984, - "nauc_mrr_at_20_std": 0.5140015311159577, - "nauc_mrr_at_3_diff1": 0.05353590069406826, - "nauc_mrr_at_3_max": 0.6316628782317711, - "nauc_mrr_at_3_std": 0.7147688210964356, - "nauc_mrr_at_5_diff1": 0.051837133080668826, - "nauc_mrr_at_5_max": 0.54565183991027, - "nauc_mrr_at_5_std": 0.6708672400517273, - "nauc_ndcg_at_1000_diff1": 0.011827898354997757, - "nauc_ndcg_at_1000_max": 0.0436259867857809, - "nauc_ndcg_at_1000_std": 0.08536272462093923, - "nauc_ndcg_at_100_diff1": 0.007417244212828439, - "nauc_ndcg_at_100_max": 0.11683503091805404, - "nauc_ndcg_at_100_std": 0.24690930985418497, - "nauc_ndcg_at_10_diff1": 0.05072202541744854, - "nauc_ndcg_at_10_max": 0.33967388199412657, - "nauc_ndcg_at_10_std": 0.4487116052117022, - "nauc_ndcg_at_1_diff1": 0.17909847061980153, - "nauc_ndcg_at_1_max": 0.7746176549503622, - "nauc_ndcg_at_1_std": 0.9436544137375904, - "nauc_ndcg_at_20_diff1": 0.08209430337287296, - "nauc_ndcg_at_20_max": 0.27299828714362034, - "nauc_ndcg_at_20_std": 0.36199893473900846, - "nauc_ndcg_at_3_diff1": 0.03415592930056411, - "nauc_ndcg_at_3_max": 0.5961127440833953, - "nauc_ndcg_at_3_std": 0.6640858782599793, - "nauc_ndcg_at_5_diff1": 0.03618641095466082, - "nauc_ndcg_at_5_max": 0.4749818186573876, - "nauc_ndcg_at_5_std": 0.6093292134220444, - "nauc_precision_at_1000_diff1": 0.008327387047313468, - "nauc_precision_at_1000_max": 0.02007387385925629, - "nauc_precision_at_1000_std": 0.044956744075964394, - "nauc_precision_at_100_diff1": -0.01001893740545882, - "nauc_precision_at_100_max": 0.051642343571975946, - "nauc_precision_at_100_std": 0.18444138542480845, - "nauc_precision_at_10_diff1": 0.0442888425352916, - "nauc_precision_at_10_max": 0.18272068687952764, - "nauc_precision_at_10_std": 0.28539775136969103, - "nauc_precision_at_1_diff1": 0.17909847061980153, - "nauc_precision_at_1_max": 0.7746176549503622, - "nauc_precision_at_1_std": 0.9436544137375904, - "nauc_precision_at_20_diff1": 0.09493417551144295, - "nauc_precision_at_20_max": 0.15466225637461695, - "nauc_precision_at_20_std": 0.23296546688542216, - "nauc_precision_at_3_diff1": -0.0026212049286907556, - "nauc_precision_at_3_max": 0.5246331345070279, - "nauc_precision_at_3_std": 0.5633320261707707, - "nauc_precision_at_5_diff1": 0.012744834987925856, - "nauc_precision_at_5_max": 0.35232388277733073, - "nauc_precision_at_5_std": 0.5068270578063978, - "nauc_recall_at_1000_diff1": 0.008327387047313645, - "nauc_recall_at_1000_max": 0.020073873859256595, - "nauc_recall_at_1000_std": 0.044956744075964845, - "nauc_recall_at_100_diff1": -0.01001893740545887, - "nauc_recall_at_100_max": 0.05164234357197578, - "nauc_recall_at_100_std": 0.18444138542480845, - "nauc_recall_at_10_diff1": 0.04428884253529171, - "nauc_recall_at_10_max": 0.18272068687952775, - "nauc_recall_at_10_std": 0.28539775136969114, - "nauc_recall_at_1_diff1": 0.17909847061980153, - "nauc_recall_at_1_max": 0.7746176549503622, - "nauc_recall_at_1_std": 0.9436544137375904, - "nauc_recall_at_20_diff1": 0.09493417551144288, - "nauc_recall_at_20_max": 0.15466225637461684, - "nauc_recall_at_20_std": 0.23296546688542194, - "nauc_recall_at_3_diff1": -0.002621204928690842, - "nauc_recall_at_3_max": 0.5246331345070279, - "nauc_recall_at_3_std": 0.5633320261707704, - "nauc_recall_at_5_diff1": 0.012744834987925951, - "nauc_recall_at_5_max": 0.3523238827773307, - "nauc_recall_at_5_std": 0.5068270578063978, - "ndcg_at_1": 0.00078, - "ndcg_at_10": 0.0021, - "ndcg_at_100": 0.00799, - "ndcg_at_1000": 0.03843, - "ndcg_at_20": 0.00309, - "ndcg_at_3": 0.00129, - "ndcg_at_5": 0.00156, - "precision_at_1": 0.00078, - "precision_at_10": 0.00041, - "precision_at_100": 0.00036, - "precision_at_1000": 0.0003, - "precision_at_20": 0.0004, - "precision_at_3": 0.00057, - "precision_at_5": 0.00047, - "recall_at_1": 0.00078, - "recall_at_10": 0.00405, - "recall_at_100": 0.03608, - "recall_at_1000": 0.30301, - "recall_at_20": 0.00797, - "recall_at_3": 0.0017, - "recall_at_5": 0.00235 - }, - { - "cv_recall_at_1": 0.00148, - "cv_recall_at_10": 0.00972, - "cv_recall_at_100": 0.04501, - "cv_recall_at_1000": 0.30917, - "cv_recall_at_20": 0.01479, - "cv_recall_at_3": 0.00528, - "cv_recall_at_5": 0.00676, - "hf_subset": "mi", - "languages": [ - "mri-Latn" - ], - "main_score": 0.00529, - "map_at_1": 0.00148, - "map_at_10": 0.00393, - "map_at_100": 0.0049, - "map_at_1000": 0.00561, - "map_at_20": 0.00429, - "map_at_3": 0.00321, - "map_at_5": 0.00355, - "mrr_at_1": 0.0014792899408284023, - "mrr_at_10": 0.0039252338016074265, - "mrr_at_100": 0.004904245024487178, - "mrr_at_1000": 0.005613644475039326, - "mrr_at_20": 0.00429167584331425, - "mrr_at_3": 0.0032051282051282055, - "mrr_at_5": 0.003553817976894901, - "nauc_cv_recall_at_1000_diff1": 0.004355331507438031, - "nauc_cv_recall_at_1000_max": 0.009966506525083177, - "nauc_cv_recall_at_1000_std": 0.04879354673716684, - "nauc_cv_recall_at_100_diff1": 0.031237245978576855, - "nauc_cv_recall_at_100_max": 0.10106260594284133, - "nauc_cv_recall_at_100_std": 0.1590760683957908, - "nauc_cv_recall_at_10_diff1": 0.1597156391580622, - "nauc_cv_recall_at_10_max": 0.32647292273706424, - "nauc_cv_recall_at_10_std": 0.3601123873766218, - "nauc_cv_recall_at_1_diff1": 0.18771392552872676, - "nauc_cv_recall_at_1_max": 0.03441764330158385, - "nauc_cv_recall_at_1_std": 0.15518249989400212, - "nauc_cv_recall_at_20_diff1": 0.09262905856022437, - "nauc_cv_recall_at_20_max": 0.29146045930012626, - "nauc_cv_recall_at_20_std": 0.36969914504123136, - "nauc_cv_recall_at_3_diff1": 0.17869623110167043, - "nauc_cv_recall_at_3_max": 0.3614519530503182, - "nauc_cv_recall_at_3_std": 0.3343149914188446, - "nauc_cv_recall_at_5_diff1": 0.2355772518637709, - "nauc_cv_recall_at_5_max": 0.37012544836471284, - "nauc_cv_recall_at_5_std": 0.3971712510046319, - "nauc_map_at_1000_diff1": 0.13344744017464022, - "nauc_map_at_1000_max": 0.21873607901173248, - "nauc_map_at_1000_std": 0.2599947155714711, - "nauc_map_at_100_diff1": 0.14674367263273896, - "nauc_map_at_100_max": 0.24234073023274486, - "nauc_map_at_100_std": 0.2833404271031057, - "nauc_map_at_10_diff1": 0.18487856478608633, - "nauc_map_at_10_max": 0.27962716852110747, - "nauc_map_at_10_std": 0.30848428841801184, - "nauc_map_at_1_diff1": 0.18771392552872676, - "nauc_map_at_1_max": 0.03441764330158385, - "nauc_map_at_1_std": 0.15518249989400212, - "nauc_map_at_20_diff1": 0.16789492237246179, - "nauc_map_at_20_max": 0.27544213437309595, - "nauc_map_at_20_std": 0.3146802255655496, - "nauc_map_at_3_diff1": 0.18426767070717054, - "nauc_map_at_3_max": 0.27566821370593203, - "nauc_map_at_3_std": 0.28178798599583, - "nauc_map_at_5_diff1": 0.20690189219095959, - "nauc_map_at_5_max": 0.28441408056847484, - "nauc_map_at_5_std": 0.3128851793422813, - "nauc_mrr_at_1000_diff1": 0.13344751137747338, - "nauc_mrr_at_1000_max": 0.21873628661038808, - "nauc_mrr_at_1000_std": 0.25999493540900376, - "nauc_mrr_at_100_diff1": 0.14674367263273896, - "nauc_mrr_at_100_max": 0.24234073023274486, - "nauc_mrr_at_100_std": 0.2833404271031057, - "nauc_mrr_at_10_diff1": 0.18487856478608633, - "nauc_mrr_at_10_max": 0.27962716852110747, - "nauc_mrr_at_10_std": 0.30848428841801184, - "nauc_mrr_at_1_diff1": 0.18771392552872676, - "nauc_mrr_at_1_max": 0.03441764330158385, - "nauc_mrr_at_1_std": 0.15518249989400212, - "nauc_mrr_at_20_diff1": 0.16789492237246179, - "nauc_mrr_at_20_max": 0.27544213437309595, - "nauc_mrr_at_20_std": 0.3146802255655496, - "nauc_mrr_at_3_diff1": 0.18426767070717054, - "nauc_mrr_at_3_max": 0.27566821370593203, - "nauc_mrr_at_3_std": 0.28178798599583, - "nauc_mrr_at_5_diff1": 0.20690189219095959, - "nauc_mrr_at_5_max": 0.28441408056847484, - "nauc_mrr_at_5_std": 0.3128851793422813, - "nauc_ndcg_at_1000_diff1": 0.02906027104190531, - "nauc_ndcg_at_1000_max": 0.055826066557204464, - "nauc_ndcg_at_1000_std": 0.09776181492411479, - "nauc_ndcg_at_100_diff1": 0.07528148312906342, - "nauc_ndcg_at_100_max": 0.16335625399710246, - "nauc_ndcg_at_100_std": 0.21626420423589257, - "nauc_ndcg_at_10_diff1": 0.17550276864424455, - "nauc_ndcg_at_10_max": 0.3020510586250383, - "nauc_ndcg_at_10_std": 0.33171510525118086, - "nauc_ndcg_at_1_diff1": 0.18771392552872676, - "nauc_ndcg_at_1_max": 0.03441764330158385, - "nauc_ndcg_at_1_std": 0.15518249989400212, - "nauc_ndcg_at_20_diff1": 0.13533400680107543, - "nauc_ndcg_at_20_max": 0.28723325888796997, - "nauc_ndcg_at_20_std": 0.34232923635660023, - "nauc_ndcg_at_3_diff1": 0.18236157753867158, - "nauc_ndcg_at_3_max": 0.30678150196283976, - "nauc_ndcg_at_3_std": 0.3006411799252318, - "nauc_ndcg_at_5_diff1": 0.21707062747772668, - "nauc_ndcg_at_5_max": 0.3176201891977177, - "nauc_ndcg_at_5_std": 0.344583002486531, - "nauc_precision_at_1000_diff1": 0.004355331507438161, - "nauc_precision_at_1000_max": 0.009966506525083213, - "nauc_precision_at_1000_std": 0.048793546737166654, - "nauc_precision_at_100_diff1": 0.031237245978576973, - "nauc_precision_at_100_max": 0.10106260594284158, - "nauc_precision_at_100_std": 0.15907606839579083, - "nauc_precision_at_10_diff1": 0.15971563915806258, - "nauc_precision_at_10_max": 0.3264729227370646, - "nauc_precision_at_10_std": 0.3601123873766221, - "nauc_precision_at_1_diff1": 0.18771392552872676, - "nauc_precision_at_1_max": 0.03441764330158385, - "nauc_precision_at_1_std": 0.15518249989400212, - "nauc_precision_at_20_diff1": 0.09262905856022462, - "nauc_precision_at_20_max": 0.2914604593001266, - "nauc_precision_at_20_std": 0.36969914504123164, - "nauc_precision_at_3_diff1": 0.1786962311016705, - "nauc_precision_at_3_max": 0.36145195305031846, - "nauc_precision_at_3_std": 0.3343149914188446, - "nauc_precision_at_5_diff1": 0.23557725186377082, - "nauc_precision_at_5_max": 0.37012544836471256, - "nauc_precision_at_5_std": 0.3971712510046316, - "nauc_recall_at_1000_diff1": 0.004355331507438031, - "nauc_recall_at_1000_max": 0.009966506525083177, - "nauc_recall_at_1000_std": 0.04879354673716684, - "nauc_recall_at_100_diff1": 0.031237245978576855, - "nauc_recall_at_100_max": 0.10106260594284133, - "nauc_recall_at_100_std": 0.1590760683957908, - "nauc_recall_at_10_diff1": 0.1597156391580622, - "nauc_recall_at_10_max": 0.32647292273706424, - "nauc_recall_at_10_std": 0.3601123873766218, - "nauc_recall_at_1_diff1": 0.18771392552872676, - "nauc_recall_at_1_max": 0.03441764330158385, - "nauc_recall_at_1_std": 0.15518249989400212, - "nauc_recall_at_20_diff1": 0.09262905856022437, - "nauc_recall_at_20_max": 0.29146045930012626, - "nauc_recall_at_20_std": 0.36969914504123136, - "nauc_recall_at_3_diff1": 0.17869623110167043, - "nauc_recall_at_3_max": 0.3614519530503182, - "nauc_recall_at_3_std": 0.3343149914188446, - "nauc_recall_at_5_diff1": 0.2355772518637709, - "nauc_recall_at_5_max": 0.37012544836471284, - "nauc_recall_at_5_std": 0.3971712510046319, - "ndcg_at_1": 0.00148, - "ndcg_at_10": 0.00529, - "ndcg_at_100": 0.01187, - "ndcg_at_1000": 0.04206, - "ndcg_at_20": 0.00659, - "ndcg_at_3": 0.00374, - "ndcg_at_5": 0.00436, - "precision_at_1": 0.00148, - "precision_at_10": 0.00097, - "precision_at_100": 0.00045, - "precision_at_1000": 0.00031, - "precision_at_20": 0.00074, - "precision_at_3": 0.00176, - "precision_at_5": 0.00135, - "recall_at_1": 0.00148, - "recall_at_10": 0.00972, - "recall_at_100": 0.04501, - "recall_at_1000": 0.30917, - "recall_at_20": 0.01479, - "recall_at_3": 0.00528, - "recall_at_5": 0.00676 - }, - { - "cv_recall_at_1": 0.07879, - "cv_recall_at_10": 0.24271, - "cv_recall_at_100": 0.49671, - "cv_recall_at_1000": 0.8341, - "cv_recall_at_20": 0.30686, - "cv_recall_at_3": 0.14915, - "cv_recall_at_5": 0.187, - "hf_subset": "nl", - "languages": [ - "nld-Latn" - ], - "main_score": 0.15295, - "map_at_1": 0.07879, - "map_at_10": 0.12519, - "map_at_100": 0.13416, - "map_at_1000": 0.13546, - "map_at_20": 0.12963, - "map_at_3": 0.10919, - "map_at_5": 0.11781, - "mrr_at_1": 0.07879389502419655, - "mrr_at_10": 0.12519405495581232, - "mrr_at_100": 0.13416070786638226, - "mrr_at_1000": 0.13545788899692368, - "mrr_at_20": 0.12963349463435273, - "mrr_at_3": 0.10919468916739085, - "mrr_at_5": 0.11780617942672854, - "nauc_cv_recall_at_1000_diff1": 0.08930409968796703, - "nauc_cv_recall_at_1000_max": 0.30663134466327585, - "nauc_cv_recall_at_1000_std": 0.5460102145860056, - "nauc_cv_recall_at_100_diff1": 0.09605523895794219, - "nauc_cv_recall_at_100_max": 0.2487062961121653, - "nauc_cv_recall_at_100_std": 0.39344458008528294, - "nauc_cv_recall_at_10_diff1": 0.13602886643850298, - "nauc_cv_recall_at_10_max": 0.29325465769882425, - "nauc_cv_recall_at_10_std": 0.290467383536801, - "nauc_cv_recall_at_1_diff1": 0.3277831672915696, - "nauc_cv_recall_at_1_max": 0.38358513881662565, - "nauc_cv_recall_at_1_std": 0.26455104901466153, - "nauc_cv_recall_at_20_diff1": 0.11704118112102552, - "nauc_cv_recall_at_20_max": 0.26934457082677205, - "nauc_cv_recall_at_20_std": 0.3076089089674812, - "nauc_cv_recall_at_3_diff1": 0.2015815540924677, - "nauc_cv_recall_at_3_max": 0.33519097609046333, - "nauc_cv_recall_at_3_std": 0.26655658100966784, - "nauc_cv_recall_at_5_diff1": 0.1604036961954942, - "nauc_cv_recall_at_5_max": 0.3133802482025979, - "nauc_cv_recall_at_5_std": 0.273309526612517, - "nauc_map_at_1000_diff1": 0.22963830503228963, - "nauc_map_at_1000_max": 0.338849193361561, - "nauc_map_at_1000_std": 0.27744821129645536, - "nauc_map_at_100_diff1": 0.22923942148213955, - "nauc_map_at_100_max": 0.3383619423952214, - "nauc_map_at_100_std": 0.27691713466932893, - "nauc_map_at_10_diff1": 0.2338906235270105, - "nauc_map_at_10_max": 0.34335803851093954, - "nauc_map_at_10_std": 0.27054230976216415, - "nauc_map_at_1_diff1": 0.3277831672915696, - "nauc_map_at_1_max": 0.38358513881662565, - "nauc_map_at_1_std": 0.26455104901466153, - "nauc_map_at_20_diff1": 0.23039337286476344, - "nauc_map_at_20_max": 0.33966045852700694, - "nauc_map_at_20_std": 0.27296817740708507, - "nauc_map_at_3_diff1": 0.2581545114721057, - "nauc_map_at_3_max": 0.3569969229868415, - "nauc_map_at_3_std": 0.2643599427994337, - "nauc_map_at_5_diff1": 0.24144220774040495, - "nauc_map_at_5_max": 0.34874293922106625, - "nauc_map_at_5_std": 0.2669519907389868, - "nauc_mrr_at_1000_diff1": 0.22963830503228963, - "nauc_mrr_at_1000_max": 0.338849193361561, - "nauc_mrr_at_1000_std": 0.27744821129645536, - "nauc_mrr_at_100_diff1": 0.22923942148213955, - "nauc_mrr_at_100_max": 0.3383619423952214, - "nauc_mrr_at_100_std": 0.27691713466932893, - "nauc_mrr_at_10_diff1": 0.2338906235270105, - "nauc_mrr_at_10_max": 0.34335803851093954, - "nauc_mrr_at_10_std": 0.27054230976216415, - "nauc_mrr_at_1_diff1": 0.3277831672915696, - "nauc_mrr_at_1_max": 0.38358513881662565, - "nauc_mrr_at_1_std": 0.26455104901466153, - "nauc_mrr_at_20_diff1": 0.23039337286476344, - "nauc_mrr_at_20_max": 0.33966045852700694, - "nauc_mrr_at_20_std": 0.27296817740708507, - "nauc_mrr_at_3_diff1": 0.2581545114721057, - "nauc_mrr_at_3_max": 0.3569969229868415, - "nauc_mrr_at_3_std": 0.2643599427994337, - "nauc_mrr_at_5_diff1": 0.24144220774040495, - "nauc_mrr_at_5_max": 0.34874293922106625, - "nauc_mrr_at_5_std": 0.2669519907389868, - "nauc_ndcg_at_1000_diff1": 0.1941231777187904, - "nauc_ndcg_at_1000_max": 0.3219632581148484, - "nauc_ndcg_at_1000_std": 0.3149786501614155, - "nauc_ndcg_at_100_diff1": 0.18513247666713342, - "nauc_ndcg_at_100_max": 0.30990446132316557, - "nauc_ndcg_at_100_std": 0.30800382186794323, - "nauc_ndcg_at_10_diff1": 0.20094888060006244, - "nauc_ndcg_at_10_max": 0.32679891787939697, - "nauc_ndcg_at_10_std": 0.27674862360276026, - "nauc_ndcg_at_1_diff1": 0.3277831672915696, - "nauc_ndcg_at_1_max": 0.38358513881662565, - "nauc_ndcg_at_1_std": 0.26455104901466153, - "nauc_ndcg_at_20_diff1": 0.19136917742345397, - "nauc_ndcg_at_20_max": 0.31632500954065434, - "nauc_ndcg_at_20_std": 0.2838398423680135, - "nauc_ndcg_at_3_diff1": 0.24077335740756323, - "nauc_ndcg_at_3_max": 0.35030162729309505, - "nauc_ndcg_at_3_std": 0.2649857421079281, - "nauc_ndcg_at_5_diff1": 0.215227244408146, - "nauc_ndcg_at_5_max": 0.33738550021643154, - "nauc_ndcg_at_5_std": 0.26893681994359964, - "nauc_precision_at_1000_diff1": 0.0893040996879657, - "nauc_precision_at_1000_max": 0.30663134466327596, - "nauc_precision_at_1000_std": 0.5460102145860033, - "nauc_precision_at_100_diff1": 0.09605523895794259, - "nauc_precision_at_100_max": 0.24870629611216516, - "nauc_precision_at_100_std": 0.3934445800852835, - "nauc_precision_at_10_diff1": 0.13602886643850304, - "nauc_precision_at_10_max": 0.29325465769882403, - "nauc_precision_at_10_std": 0.2904673835368009, - "nauc_precision_at_1_diff1": 0.3277831672915696, - "nauc_precision_at_1_max": 0.38358513881662565, - "nauc_precision_at_1_std": 0.26455104901466153, - "nauc_precision_at_20_diff1": 0.11704118112102543, - "nauc_precision_at_20_max": 0.2693445708267721, - "nauc_precision_at_20_std": 0.3076089089674813, - "nauc_precision_at_3_diff1": 0.20158155409246764, - "nauc_precision_at_3_max": 0.33519097609046333, - "nauc_precision_at_3_std": 0.2665565810096679, - "nauc_precision_at_5_diff1": 0.1604036961954938, - "nauc_precision_at_5_max": 0.31338024820259774, - "nauc_precision_at_5_std": 0.2733095266125168, - "nauc_recall_at_1000_diff1": 0.08930409968796703, - "nauc_recall_at_1000_max": 0.30663134466327585, - "nauc_recall_at_1000_std": 0.5460102145860056, - "nauc_recall_at_100_diff1": 0.09605523895794219, - "nauc_recall_at_100_max": 0.2487062961121653, - "nauc_recall_at_100_std": 0.39344458008528294, - "nauc_recall_at_10_diff1": 0.13602886643850298, - "nauc_recall_at_10_max": 0.29325465769882425, - "nauc_recall_at_10_std": 0.290467383536801, - "nauc_recall_at_1_diff1": 0.3277831672915696, - "nauc_recall_at_1_max": 0.38358513881662565, - "nauc_recall_at_1_std": 0.26455104901466153, - "nauc_recall_at_20_diff1": 0.11704118112102552, - "nauc_recall_at_20_max": 0.26934457082677205, - "nauc_recall_at_20_std": 0.3076089089674812, - "nauc_recall_at_3_diff1": 0.2015815540924677, - "nauc_recall_at_3_max": 0.33519097609046333, - "nauc_recall_at_3_std": 0.26655658100966784, - "nauc_recall_at_5_diff1": 0.1604036961954942, - "nauc_recall_at_5_max": 0.3133802482025979, - "nauc_recall_at_5_std": 0.273309526612517, - "ndcg_at_1": 0.07879, - "ndcg_at_10": 0.15295, - "ndcg_at_100": 0.20368, - "ndcg_at_1000": 0.24471, - "ndcg_at_20": 0.16916, - "ndcg_at_3": 0.11943, - "ndcg_at_5": 0.13499, - "precision_at_1": 0.07879, - "precision_at_10": 0.02427, - "precision_at_100": 0.00497, - "precision_at_1000": 0.00083, - "precision_at_20": 0.01534, - "precision_at_3": 0.04972, - "precision_at_5": 0.0374, - "recall_at_1": 0.07879, - "recall_at_10": 0.24271, - "recall_at_100": 0.49671, - "recall_at_1000": 0.8341, - "recall_at_20": 0.30686, - "recall_at_3": 0.14915, - "recall_at_5": 0.187 - }, - { - "cv_recall_at_1": 0.0305, - "cv_recall_at_10": 0.11438, - "cv_recall_at_100": 0.29197, - "cv_recall_at_1000": 0.65937, - "cv_recall_at_20": 0.15694, - "cv_recall_at_3": 0.06447, - "cv_recall_at_5": 0.08277, - "hf_subset": "no", - "languages": [ - "nor-Latn" - ], - "main_score": 0.06762, - "map_at_1": 0.0305, - "map_at_10": 0.05326, - "map_at_100": 0.05935, - "map_at_1000": 0.06061, - "map_at_20": 0.0562, - "map_at_3": 0.04497, - "map_at_5": 0.04907, - "mrr_at_1": 0.030500485234992374, - "mrr_at_10": 0.05326449818339451, - "mrr_at_100": 0.05935235207055774, - "mrr_at_1000": 0.06061448878521882, - "mrr_at_20": 0.05619837034936023, - "mrr_at_3": 0.04496510929340537, - "mrr_at_5": 0.04906881094320427, - "nauc_cv_recall_at_1000_diff1": 0.11112518129848828, - "nauc_cv_recall_at_1000_max": 0.17993525311133474, - "nauc_cv_recall_at_1000_std": 0.4386071263187699, - "nauc_cv_recall_at_100_diff1": 0.11926601259822187, - "nauc_cv_recall_at_100_max": 0.17193302058599702, - "nauc_cv_recall_at_100_std": 0.37419523754131717, - "nauc_cv_recall_at_10_diff1": 0.1517786413891697, - "nauc_cv_recall_at_10_max": 0.2442149898435886, - "nauc_cv_recall_at_10_std": 0.36620202281025915, - "nauc_cv_recall_at_1_diff1": 0.44248156992731186, - "nauc_cv_recall_at_1_max": 0.4793737235035624, - "nauc_cv_recall_at_1_std": 0.3974568606637231, - "nauc_cv_recall_at_20_diff1": 0.14704354887179238, - "nauc_cv_recall_at_20_max": 0.21188556513826365, - "nauc_cv_recall_at_20_std": 0.35708153462693637, - "nauc_cv_recall_at_3_diff1": 0.2398255875343225, - "nauc_cv_recall_at_3_max": 0.3218596579776256, - "nauc_cv_recall_at_3_std": 0.3679462784006674, - "nauc_cv_recall_at_5_diff1": 0.19407458813228443, - "nauc_cv_recall_at_5_max": 0.2903569698107385, - "nauc_cv_recall_at_5_std": 0.3734450090214543, - "nauc_map_at_1000_diff1": 0.2670329419285782, - "nauc_map_at_1000_max": 0.3327567801100508, - "nauc_map_at_1000_std": 0.37732823515389075, - "nauc_map_at_100_diff1": 0.2670297580050445, - "nauc_map_at_100_max": 0.33294280925431413, - "nauc_map_at_100_std": 0.37662222184363, - "nauc_map_at_10_diff1": 0.2779145274337388, - "nauc_map_at_10_max": 0.34846692225241066, - "nauc_map_at_10_std": 0.3773006555270141, - "nauc_map_at_1_diff1": 0.44248156992731186, - "nauc_map_at_1_max": 0.4793737235035624, - "nauc_map_at_1_std": 0.3974568606637231, - "nauc_map_at_20_diff1": 0.27307450094883884, - "nauc_map_at_20_max": 0.3398105447803915, - "nauc_map_at_20_std": 0.3757032305314629, - "nauc_map_at_3_diff1": 0.3209398158909183, - "nauc_map_at_3_max": 0.38457816609234413, - "nauc_map_at_3_std": 0.37935958905552525, - "nauc_map_at_5_diff1": 0.2966734685382508, - "nauc_map_at_5_max": 0.3669004169612993, - "nauc_map_at_5_std": 0.38033458614308907, - "nauc_mrr_at_1000_diff1": 0.2670329419285782, - "nauc_mrr_at_1000_max": 0.3327567801100508, - "nauc_mrr_at_1000_std": 0.37732823515389075, - "nauc_mrr_at_100_diff1": 0.2670297580050445, - "nauc_mrr_at_100_max": 0.33294280925431413, - "nauc_mrr_at_100_std": 0.37662222184363, - "nauc_mrr_at_10_diff1": 0.2779145274337388, - "nauc_mrr_at_10_max": 0.34846692225241066, - "nauc_mrr_at_10_std": 0.3773006555270141, - "nauc_mrr_at_1_diff1": 0.44248156992731186, - "nauc_mrr_at_1_max": 0.4793737235035624, - "nauc_mrr_at_1_std": 0.3974568606637231, - "nauc_mrr_at_20_diff1": 0.27307450094883884, - "nauc_mrr_at_20_max": 0.3398105447803915, - "nauc_mrr_at_20_std": 0.3757032305314629, - "nauc_mrr_at_3_diff1": 0.3209398158909183, - "nauc_mrr_at_3_max": 0.38457816609234413, - "nauc_mrr_at_3_std": 0.37935958905552525, - "nauc_mrr_at_5_diff1": 0.2966734685382508, - "nauc_mrr_at_5_max": 0.3669004169612993, - "nauc_mrr_at_5_std": 0.38033458614308907, - "nauc_ndcg_at_1000_diff1": 0.19975186137524023, - "nauc_ndcg_at_1000_max": 0.2650715883370969, - "nauc_ndcg_at_1000_std": 0.3899538430343936, - "nauc_ndcg_at_100_diff1": 0.19805283041177202, - "nauc_ndcg_at_100_max": 0.26046936636798446, - "nauc_ndcg_at_100_std": 0.3741037394930803, - "nauc_ndcg_at_10_diff1": 0.2288349265356686, - "nauc_ndcg_at_10_max": 0.3080841122158018, - "nauc_ndcg_at_10_std": 0.3728326724105545, - "nauc_ndcg_at_1_diff1": 0.44248156992731186, - "nauc_ndcg_at_1_max": 0.4793737235035624, - "nauc_ndcg_at_1_std": 0.3974568606637231, - "nauc_ndcg_at_20_diff1": 0.21954849145263852, - "nauc_ndcg_at_20_max": 0.28716962341639507, - "nauc_ndcg_at_20_std": 0.3682131493631691, - "nauc_ndcg_at_3_diff1": 0.2942502076310742, - "nauc_ndcg_at_3_max": 0.36392958959880967, - "nauc_ndcg_at_3_std": 0.3755897658861957, - "nauc_ndcg_at_5_diff1": 0.26012575265309584, - "nauc_ndcg_at_5_max": 0.33953946005404073, - "nauc_ndcg_at_5_std": 0.3777176090535697, - "nauc_precision_at_1000_diff1": 0.11112518129848781, - "nauc_precision_at_1000_max": 0.1799352531113345, - "nauc_precision_at_1000_std": 0.43860712631876997, - "nauc_precision_at_100_diff1": 0.11926601259822206, - "nauc_precision_at_100_max": 0.17193302058599713, - "nauc_precision_at_100_std": 0.3741952375413173, - "nauc_precision_at_10_diff1": 0.15177864138916966, - "nauc_precision_at_10_max": 0.24421498984358866, - "nauc_precision_at_10_std": 0.36620202281025893, - "nauc_precision_at_1_diff1": 0.44248156992731186, - "nauc_precision_at_1_max": 0.4793737235035624, - "nauc_precision_at_1_std": 0.3974568606637231, - "nauc_precision_at_20_diff1": 0.14704354887179236, - "nauc_precision_at_20_max": 0.21188556513826365, - "nauc_precision_at_20_std": 0.3570815346269364, - "nauc_precision_at_3_diff1": 0.23982558753432265, - "nauc_precision_at_3_max": 0.32185965797762567, - "nauc_precision_at_3_std": 0.3679462784006674, - "nauc_precision_at_5_diff1": 0.1940745881322843, - "nauc_precision_at_5_max": 0.29035696981073855, - "nauc_precision_at_5_std": 0.3734450090214543, - "nauc_recall_at_1000_diff1": 0.11112518129848828, - "nauc_recall_at_1000_max": 0.17993525311133474, - "nauc_recall_at_1000_std": 0.4386071263187699, - "nauc_recall_at_100_diff1": 0.11926601259822187, - "nauc_recall_at_100_max": 0.17193302058599702, - "nauc_recall_at_100_std": 0.37419523754131717, - "nauc_recall_at_10_diff1": 0.1517786413891697, - "nauc_recall_at_10_max": 0.2442149898435886, - "nauc_recall_at_10_std": 0.36620202281025915, - "nauc_recall_at_1_diff1": 0.44248156992731186, - "nauc_recall_at_1_max": 0.4793737235035624, - "nauc_recall_at_1_std": 0.3974568606637231, - "nauc_recall_at_20_diff1": 0.14704354887179238, - "nauc_recall_at_20_max": 0.21188556513826365, - "nauc_recall_at_20_std": 0.35708153462693637, - "nauc_recall_at_3_diff1": 0.2398255875343225, - "nauc_recall_at_3_max": 0.3218596579776256, - "nauc_recall_at_3_std": 0.3679462784006674, - "nauc_recall_at_5_diff1": 0.19407458813228443, - "nauc_recall_at_5_max": 0.2903569698107385, - "nauc_recall_at_5_std": 0.3734450090214543, - "ndcg_at_1": 0.0305, - "ndcg_at_10": 0.06762, - "ndcg_at_100": 0.10277, - "ndcg_at_1000": 0.14652, - "ndcg_at_20": 0.07836, - "ndcg_at_3": 0.04995, - "ndcg_at_5": 0.05742, - "precision_at_1": 0.0305, - "precision_at_10": 0.01144, - "precision_at_100": 0.00292, - "precision_at_1000": 0.00066, - "precision_at_20": 0.00785, - "precision_at_3": 0.02149, - "precision_at_5": 0.01655, - "recall_at_1": 0.0305, - "recall_at_10": 0.11438, - "recall_at_100": 0.29197, - "recall_at_1000": 0.65937, - "recall_at_20": 0.15694, - "recall_at_3": 0.06447, - "recall_at_5": 0.08277 - }, - { - "cv_recall_at_1": 0.01694, - "cv_recall_at_10": 0.06162, - "cv_recall_at_100": 0.1413, - "cv_recall_at_1000": 0.44602, - "cv_recall_at_20": 0.0815, - "cv_recall_at_3": 0.03375, - "cv_recall_at_5": 0.04397, - "hf_subset": "pl", - "languages": [ - "pol-Latn" - ], - "main_score": 0.03658, - "map_at_1": 0.01694, - "map_at_10": 0.02894, - "map_at_100": 0.03165, - "map_at_1000": 0.03254, - "map_at_20": 0.03029, - "map_at_3": 0.02439, - "map_at_5": 0.02669, - "mrr_at_1": 0.016944405545441816, - "mrr_at_10": 0.02894219163649213, - "mrr_at_100": 0.03165311711940447, - "mrr_at_1000": 0.0325447626187637, - "mrr_at_20": 0.03028990356706552, - "mrr_at_3": 0.024389674648742008, - "mrr_at_5": 0.02669327358446528, - "nauc_cv_recall_at_1000_diff1": 0.1377259253556431, - "nauc_cv_recall_at_1000_max": 0.3263785888666531, - "nauc_cv_recall_at_1000_std": 0.3424801457124148, - "nauc_cv_recall_at_100_diff1": 0.2423337950404231, - "nauc_cv_recall_at_100_max": 0.5241164351239225, - "nauc_cv_recall_at_100_std": 0.52779401007404, - "nauc_cv_recall_at_10_diff1": 0.3255266166865096, - "nauc_cv_recall_at_10_max": 0.6390632499675445, - "nauc_cv_recall_at_10_std": 0.6161215849797735, - "nauc_cv_recall_at_1_diff1": 0.5378727151451592, - "nauc_cv_recall_at_1_max": 0.7425241822187479, - "nauc_cv_recall_at_1_std": 0.6231210907908145, - "nauc_cv_recall_at_20_diff1": 0.2983831125608907, - "nauc_cv_recall_at_20_max": 0.6014093798271857, - "nauc_cv_recall_at_20_std": 0.590690017609021, - "nauc_cv_recall_at_3_diff1": 0.4092151134545967, - "nauc_cv_recall_at_3_max": 0.7160824725563123, - "nauc_cv_recall_at_3_std": 0.6627713649654221, - "nauc_cv_recall_at_5_diff1": 0.3815391729351457, - "nauc_cv_recall_at_5_max": 0.6942264335856264, - "nauc_cv_recall_at_5_std": 0.6582818015126044, - "nauc_map_at_1000_diff1": 0.40117554070848055, - "nauc_map_at_1000_max": 0.6732502978922448, - "nauc_map_at_1000_std": 0.6187333045457905, - "nauc_map_at_100_diff1": 0.4051924506529917, - "nauc_map_at_100_max": 0.67979233659632, - "nauc_map_at_100_std": 0.6244531722043362, - "nauc_map_at_10_diff1": 0.4237179846418767, - "nauc_map_at_10_max": 0.6997202433553061, - "nauc_map_at_10_std": 0.6377480420470368, - "nauc_map_at_1_diff1": 0.5378727151451592, - "nauc_map_at_1_max": 0.7425241822187479, - "nauc_map_at_1_std": 0.6231210907908145, - "nauc_map_at_20_diff1": 0.41474178486365765, - "nauc_map_at_20_max": 0.6907510609941931, - "nauc_map_at_20_std": 0.6326295788794856, - "nauc_map_at_3_diff1": 0.4575569051255311, - "nauc_map_at_3_max": 0.7238338855927117, - "nauc_map_at_3_std": 0.6481336001436105, - "nauc_map_at_5_diff1": 0.4429423480805897, - "nauc_map_at_5_max": 0.7148428260410172, - "nauc_map_at_5_std": 0.6477392623063669, - "nauc_mrr_at_1000_diff1": 0.40117554070848055, - "nauc_mrr_at_1000_max": 0.6732502978922448, - "nauc_mrr_at_1000_std": 0.6187333045457905, - "nauc_mrr_at_100_diff1": 0.4051924506529917, - "nauc_mrr_at_100_max": 0.67979233659632, - "nauc_mrr_at_100_std": 0.6244531722043362, - "nauc_mrr_at_10_diff1": 0.4237179846418767, - "nauc_mrr_at_10_max": 0.6997202433553061, - "nauc_mrr_at_10_std": 0.6377480420470368, - "nauc_mrr_at_1_diff1": 0.5378727151451592, - "nauc_mrr_at_1_max": 0.7425241822187479, - "nauc_mrr_at_1_std": 0.6231210907908145, - "nauc_mrr_at_20_diff1": 0.41474178486365765, - "nauc_mrr_at_20_max": 0.6907510609941931, - "nauc_mrr_at_20_std": 0.6326295788794856, - "nauc_mrr_at_3_diff1": 0.4575569051255311, - "nauc_mrr_at_3_max": 0.7238338855927117, - "nauc_mrr_at_3_std": 0.6481336001436105, - "nauc_mrr_at_5_diff1": 0.4429423480805897, - "nauc_mrr_at_5_max": 0.7148428260410172, - "nauc_mrr_at_5_std": 0.6477392623063669, - "nauc_ndcg_at_1000_diff1": 0.27181028944517127, - "nauc_ndcg_at_1000_max": 0.5132561629976885, - "nauc_ndcg_at_1000_std": 0.4959066930758658, - "nauc_ndcg_at_100_diff1": 0.32845283023793725, - "nauc_ndcg_at_100_max": 0.6096509413496082, - "nauc_ndcg_at_100_std": 0.5825751332255994, - "nauc_ndcg_at_10_diff1": 0.3858193527140121, - "nauc_ndcg_at_10_max": 0.6772961496311672, - "nauc_ndcg_at_10_std": 0.6306167576871402, - "nauc_ndcg_at_1_diff1": 0.5378727151451592, - "nauc_ndcg_at_1_max": 0.7425241822187479, - "nauc_ndcg_at_1_std": 0.6231210907908145, - "nauc_ndcg_at_20_diff1": 0.365546336861465, - "nauc_ndcg_at_20_max": 0.6547149548674186, - "nauc_ndcg_at_20_std": 0.6167961341010798, - "nauc_ndcg_at_3_diff1": 0.4418525175146829, - "nauc_ndcg_at_3_max": 0.7212285747452951, - "nauc_ndcg_at_3_std": 0.6528992424951905, - "nauc_ndcg_at_5_diff1": 0.42124824904040953, - "nauc_ndcg_at_5_max": 0.707684892712741, - "nauc_ndcg_at_5_std": 0.6516317812025026, - "nauc_precision_at_1000_diff1": 0.1377259253556427, - "nauc_precision_at_1000_max": 0.3263785888666525, - "nauc_precision_at_1000_std": 0.3424801457124145, - "nauc_precision_at_100_diff1": 0.24233379504042352, - "nauc_precision_at_100_max": 0.5241164351239226, - "nauc_precision_at_100_std": 0.5277940100740403, - "nauc_precision_at_10_diff1": 0.32552661668650973, - "nauc_precision_at_10_max": 0.6390632499675446, - "nauc_precision_at_10_std": 0.6161215849797738, - "nauc_precision_at_1_diff1": 0.5378727151451592, - "nauc_precision_at_1_max": 0.7425241822187479, - "nauc_precision_at_1_std": 0.6231210907908145, - "nauc_precision_at_20_diff1": 0.29838311256089073, - "nauc_precision_at_20_max": 0.601409379827186, - "nauc_precision_at_20_std": 0.5906900176090211, - "nauc_precision_at_3_diff1": 0.4092151134545966, - "nauc_precision_at_3_max": 0.7160824725563122, - "nauc_precision_at_3_std": 0.6627713649654222, - "nauc_precision_at_5_diff1": 0.3815391729351459, - "nauc_precision_at_5_max": 0.6942264335856266, - "nauc_precision_at_5_std": 0.6582818015126047, - "nauc_recall_at_1000_diff1": 0.1377259253556431, - "nauc_recall_at_1000_max": 0.3263785888666531, - "nauc_recall_at_1000_std": 0.3424801457124148, - "nauc_recall_at_100_diff1": 0.2423337950404231, - "nauc_recall_at_100_max": 0.5241164351239225, - "nauc_recall_at_100_std": 0.52779401007404, - "nauc_recall_at_10_diff1": 0.3255266166865096, - "nauc_recall_at_10_max": 0.6390632499675445, - "nauc_recall_at_10_std": 0.6161215849797735, - "nauc_recall_at_1_diff1": 0.5378727151451592, - "nauc_recall_at_1_max": 0.7425241822187479, - "nauc_recall_at_1_std": 0.6231210907908145, - "nauc_recall_at_20_diff1": 0.2983831125608907, - "nauc_recall_at_20_max": 0.6014093798271857, - "nauc_recall_at_20_std": 0.590690017609021, - "nauc_recall_at_3_diff1": 0.4092151134545967, - "nauc_recall_at_3_max": 0.7160824725563123, - "nauc_recall_at_3_std": 0.6627713649654221, - "nauc_recall_at_5_diff1": 0.3815391729351457, - "nauc_recall_at_5_max": 0.6942264335856264, - "nauc_recall_at_5_std": 0.6582818015126044, - "ndcg_at_1": 0.01694, - "ndcg_at_10": 0.03658, - "ndcg_at_100": 0.05231, - "ndcg_at_1000": 0.08767, - "ndcg_at_20": 0.04156, - "ndcg_at_3": 0.0268, - "ndcg_at_5": 0.03098, - "precision_at_1": 0.01694, - "precision_at_10": 0.00616, - "precision_at_100": 0.00141, - "precision_at_1000": 0.00045, - "precision_at_20": 0.00408, - "precision_at_3": 0.01125, - "precision_at_5": 0.00879, - "recall_at_1": 0.01694, - "recall_at_10": 0.06162, - "recall_at_100": 0.1413, - "recall_at_1000": 0.44602, - "recall_at_20": 0.0815, - "recall_at_3": 0.03375, - "recall_at_5": 0.04397 - }, - { - "cv_recall_at_1": 0.10534, - "cv_recall_at_10": 0.31092, - "cv_recall_at_100": 0.6206, - "cv_recall_at_1000": 0.90708, - "cv_recall_at_20": 0.39832, - "cv_recall_at_3": 0.18749, - "cv_recall_at_5": 0.2365, - "hf_subset": "pt", - "languages": [ - "por-Latn" - ], - "main_score": 0.19699, - "map_at_1": 0.10534, - "map_at_10": 0.16193, - "map_at_100": 0.17328, - "map_at_1000": 0.17446, - "map_at_20": 0.16791, - "map_at_3": 0.14094, - "map_at_5": 0.15208, - "mrr_at_1": 0.10534308987988403, - "mrr_at_10": 0.16193067636185143, - "mrr_at_100": 0.17327529387246055, - "mrr_at_1000": 0.17445652475890244, - "mrr_at_20": 0.16791378478806276, - "mrr_at_3": 0.14094067835611443, - "mrr_at_5": 0.15207556721432272, - "nauc_cv_recall_at_1000_diff1": 0.188634082401284, - "nauc_cv_recall_at_1000_max": 0.4168648718922161, - "nauc_cv_recall_at_1000_std": 0.6453755575883677, - "nauc_cv_recall_at_100_diff1": 0.1751770945181202, - "nauc_cv_recall_at_100_max": 0.28556121125098244, - "nauc_cv_recall_at_100_std": 0.4132481269734251, - "nauc_cv_recall_at_10_diff1": 0.22174110466323108, - "nauc_cv_recall_at_10_max": 0.2843735838808934, - "nauc_cv_recall_at_10_std": 0.2187110961376596, - "nauc_cv_recall_at_1_diff1": 0.39297845070777243, - "nauc_cv_recall_at_1_max": 0.36221897239007084, - "nauc_cv_recall_at_1_std": 0.14795166958801662, - "nauc_cv_recall_at_20_diff1": 0.19741468067443296, - "nauc_cv_recall_at_20_max": 0.2793650184105488, - "nauc_cv_recall_at_20_std": 0.26820124036829546, - "nauc_cv_recall_at_3_diff1": 0.2781889669980125, - "nauc_cv_recall_at_3_max": 0.30514260823551453, - "nauc_cv_recall_at_3_std": 0.16817074987638658, - "nauc_cv_recall_at_5_diff1": 0.25413255473223123, - "nauc_cv_recall_at_5_max": 0.29288529852666495, - "nauc_cv_recall_at_5_std": 0.19417874420021805, - "nauc_map_at_1000_diff1": 0.3121601858930789, - "nauc_map_at_1000_max": 0.3271421192958233, - "nauc_map_at_1000_std": 0.1822483099304241, - "nauc_map_at_100_diff1": 0.3117424613332504, - "nauc_map_at_100_max": 0.32669216375125915, - "nauc_map_at_100_std": 0.1822541691667279, - "nauc_map_at_10_diff1": 0.31517810279756303, - "nauc_map_at_10_max": 0.32688790799889034, - "nauc_map_at_10_std": 0.17248238517072223, - "nauc_map_at_1_diff1": 0.39297845070777243, - "nauc_map_at_1_max": 0.36221897239007084, - "nauc_map_at_1_std": 0.14795166958801662, - "nauc_map_at_20_diff1": 0.31187388094996743, - "nauc_map_at_20_max": 0.3263220697760968, - "nauc_map_at_20_std": 0.1780222958748876, - "nauc_map_at_3_diff1": 0.332594494915826, - "nauc_map_at_3_max": 0.33455296392040723, - "nauc_map_at_3_std": 0.15889529679138556, - "nauc_map_at_5_diff1": 0.32306905528238067, - "nauc_map_at_5_max": 0.3297156250885414, - "nauc_map_at_5_std": 0.16704299359039254, - "nauc_mrr_at_1000_diff1": 0.3121601858930789, - "nauc_mrr_at_1000_max": 0.3271421192958233, - "nauc_mrr_at_1000_std": 0.1822483099304241, - "nauc_mrr_at_100_diff1": 0.3117424613332504, - "nauc_mrr_at_100_max": 0.32669216375125915, - "nauc_mrr_at_100_std": 0.1822541691667279, - "nauc_mrr_at_10_diff1": 0.31517810279756303, - "nauc_mrr_at_10_max": 0.32688790799889034, - "nauc_mrr_at_10_std": 0.17248238517072223, - "nauc_mrr_at_1_diff1": 0.39297845070777243, - "nauc_mrr_at_1_max": 0.36221897239007084, - "nauc_mrr_at_1_std": 0.14795166958801662, - "nauc_mrr_at_20_diff1": 0.31187388094996743, - "nauc_mrr_at_20_max": 0.3263220697760968, - "nauc_mrr_at_20_std": 0.1780222958748876, - "nauc_mrr_at_3_diff1": 0.332594494915826, - "nauc_mrr_at_3_max": 0.33455296392040723, - "nauc_mrr_at_3_std": 0.15889529679138556, - "nauc_mrr_at_5_diff1": 0.32306905528238067, - "nauc_mrr_at_5_max": 0.3297156250885414, - "nauc_mrr_at_5_std": 0.16704299359039254, - "nauc_ndcg_at_1000_diff1": 0.2856756508689451, - "nauc_ndcg_at_1000_max": 0.324123956897226, - "nauc_ndcg_at_1000_std": 0.22731027234399237, - "nauc_ndcg_at_100_diff1": 0.27435431907395347, - "nauc_ndcg_at_100_max": 0.31385085918658157, - "nauc_ndcg_at_100_std": 0.23333306719606933, - "nauc_ndcg_at_10_diff1": 0.28555354474720956, - "nauc_ndcg_at_10_max": 0.3132068665418705, - "nauc_ndcg_at_10_std": 0.18660288218305618, - "nauc_ndcg_at_1_diff1": 0.39297845070777243, - "nauc_ndcg_at_1_max": 0.36221897239007084, - "nauc_ndcg_at_1_std": 0.14795166958801662, - "nauc_ndcg_at_20_diff1": 0.2759260943286604, - "nauc_ndcg_at_20_max": 0.3112481235577066, - "nauc_ndcg_at_20_std": 0.2041925668301508, - "nauc_ndcg_at_3_diff1": 0.31634053385890426, - "nauc_ndcg_at_3_max": 0.3258569771631374, - "nauc_ndcg_at_3_std": 0.16167819626007002, - "nauc_ndcg_at_5_diff1": 0.30167602085250383, - "nauc_ndcg_at_5_max": 0.3183719831610955, - "nauc_ndcg_at_5_std": 0.17518979370699594, - "nauc_precision_at_1000_diff1": 0.18863408240128252, - "nauc_precision_at_1000_max": 0.4168648718922157, - "nauc_precision_at_1000_std": 0.6453755575883675, - "nauc_precision_at_100_diff1": 0.17517709451812002, - "nauc_precision_at_100_max": 0.28556121125098244, - "nauc_precision_at_100_std": 0.4132481269734251, - "nauc_precision_at_10_diff1": 0.22174110466323108, - "nauc_precision_at_10_max": 0.2843735838808934, - "nauc_precision_at_10_std": 0.21871109613765954, - "nauc_precision_at_1_diff1": 0.39297845070777243, - "nauc_precision_at_1_max": 0.36221897239007084, - "nauc_precision_at_1_std": 0.14795166958801662, - "nauc_precision_at_20_diff1": 0.197414680674433, - "nauc_precision_at_20_max": 0.27936501841054867, - "nauc_precision_at_20_std": 0.2682012403682955, - "nauc_precision_at_3_diff1": 0.27818896699801243, - "nauc_precision_at_3_max": 0.30514260823551453, - "nauc_precision_at_3_std": 0.16817074987638667, - "nauc_precision_at_5_diff1": 0.2541325547322312, - "nauc_precision_at_5_max": 0.2928852985266648, - "nauc_precision_at_5_std": 0.19417874420021808, - "nauc_recall_at_1000_diff1": 0.188634082401284, - "nauc_recall_at_1000_max": 0.4168648718922161, - "nauc_recall_at_1000_std": 0.6453755575883677, - "nauc_recall_at_100_diff1": 0.1751770945181202, - "nauc_recall_at_100_max": 0.28556121125098244, - "nauc_recall_at_100_std": 0.4132481269734251, - "nauc_recall_at_10_diff1": 0.22174110466323108, - "nauc_recall_at_10_max": 0.2843735838808934, - "nauc_recall_at_10_std": 0.2187110961376596, - "nauc_recall_at_1_diff1": 0.39297845070777243, - "nauc_recall_at_1_max": 0.36221897239007084, - "nauc_recall_at_1_std": 0.14795166958801662, - "nauc_recall_at_20_diff1": 0.19741468067443296, - "nauc_recall_at_20_max": 0.2793650184105488, - "nauc_recall_at_20_std": 0.26820124036829546, - "nauc_recall_at_3_diff1": 0.2781889669980125, - "nauc_recall_at_3_max": 0.30514260823551453, - "nauc_recall_at_3_std": 0.16817074987638658, - "nauc_recall_at_5_diff1": 0.25413255473223123, - "nauc_recall_at_5_max": 0.29288529852666495, - "nauc_recall_at_5_std": 0.19417874420021805, - "ndcg_at_1": 0.10534, - "ndcg_at_10": 0.19699, - "ndcg_at_100": 0.25954, - "ndcg_at_1000": 0.29488, - "ndcg_at_20": 0.21898, - "ndcg_at_3": 0.15287, - "ndcg_at_5": 0.173, - "precision_at_1": 0.10534, - "precision_at_10": 0.03109, - "precision_at_100": 0.00621, - "precision_at_1000": 0.00091, - "precision_at_20": 0.01992, - "precision_at_3": 0.0625, - "precision_at_5": 0.0473, - "recall_at_1": 0.10534, - "recall_at_10": 0.31092, - "recall_at_100": 0.6206, - "recall_at_1000": 0.90708, - "recall_at_20": 0.39832, - "recall_at_3": 0.18749, - "recall_at_5": 0.2365 - }, - { - "cv_recall_at_1": 0.01181, - "cv_recall_at_10": 0.04556, - "cv_recall_at_100": 0.11806, - "cv_recall_at_1000": 0.40236, - "cv_recall_at_20": 0.05708, - "cv_recall_at_3": 0.02333, - "cv_recall_at_5": 0.03236, - "hf_subset": "quz", - "languages": [ - "quz-Latn" - ], - "main_score": 0.02625, - "map_at_1": 0.01181, - "map_at_10": 0.02037, - "map_at_100": 0.02254, - "map_at_1000": 0.02335, - "map_at_20": 0.02118, - "map_at_3": 0.01667, - "map_at_5": 0.01865, - "mrr_at_1": 0.011805555555555555, - "mrr_at_10": 0.020373126102292763, - "mrr_at_100": 0.0225429811211209, - "mrr_at_1000": 0.023349026771023774, - "mrr_at_20": 0.021176072890936776, - "mrr_at_3": 0.016666666666666656, - "mrr_at_5": 0.01865277777777777, - "nauc_cv_recall_at_1000_diff1": 0.08667190720067616, - "nauc_cv_recall_at_1000_max": 0.16811377816078713, - "nauc_cv_recall_at_1000_std": 0.1776941853155657, - "nauc_cv_recall_at_100_diff1": 0.15789524491120618, - "nauc_cv_recall_at_100_max": 0.31809563437245164, - "nauc_cv_recall_at_100_std": 0.3395186658455805, - "nauc_cv_recall_at_10_diff1": 0.26798755947044317, - "nauc_cv_recall_at_10_max": 0.48983767104910125, - "nauc_cv_recall_at_10_std": 0.463757386768931, - "nauc_cv_recall_at_1_diff1": 0.5396391988509922, - "nauc_cv_recall_at_1_max": 0.7001152164649064, - "nauc_cv_recall_at_1_std": 0.5476285926229898, - "nauc_cv_recall_at_20_diff1": 0.23274755075248219, - "nauc_cv_recall_at_20_max": 0.43348759659406566, - "nauc_cv_recall_at_20_std": 0.4368803438053475, - "nauc_cv_recall_at_3_diff1": 0.37070540585431927, - "nauc_cv_recall_at_3_max": 0.5930676402570686, - "nauc_cv_recall_at_3_std": 0.5005478043108846, - "nauc_cv_recall_at_5_diff1": 0.33045137501425054, - "nauc_cv_recall_at_5_max": 0.5399457156971915, - "nauc_cv_recall_at_5_std": 0.4736610582099537, - "nauc_map_at_1000_diff1": 0.36268114588829053, - "nauc_map_at_1000_max": 0.5462730411169001, - "nauc_map_at_1000_std": 0.4733374126425002, - "nauc_map_at_100_diff1": 0.3684690660506013, - "nauc_map_at_100_max": 0.5545265516609, - "nauc_map_at_100_std": 0.4802315274638474, - "nauc_map_at_10_diff1": 0.39593461221974763, - "nauc_map_at_10_max": 0.5894828816343469, - "nauc_map_at_10_std": 0.4992925577248018, - "nauc_map_at_1_diff1": 0.5396391988509922, - "nauc_map_at_1_max": 0.7001152164649064, - "nauc_map_at_1_std": 0.5476285926229898, - "nauc_map_at_20_diff1": 0.38501958259804275, - "nauc_map_at_20_max": 0.5756154448581887, - "nauc_map_at_20_std": 0.4932341583371749, - "nauc_map_at_3_diff1": 0.44470157708016583, - "nauc_map_at_3_max": 0.6355059177771817, - "nauc_map_at_3_std": 0.5158617297200611, - "nauc_map_at_5_diff1": 0.4213217255125721, - "nauc_map_at_5_max": 0.6099601340775752, - "nauc_map_at_5_std": 0.5039017165794833, - "nauc_mrr_at_1000_diff1": 0.36268111735207437, - "nauc_mrr_at_1000_max": 0.5462730296720244, - "nauc_mrr_at_1000_std": 0.47333742053027184, - "nauc_mrr_at_100_diff1": 0.3684690660506013, - "nauc_mrr_at_100_max": 0.5545265516609, - "nauc_mrr_at_100_std": 0.4802315274638474, - "nauc_mrr_at_10_diff1": 0.39593461221974763, - "nauc_mrr_at_10_max": 0.5894828816343469, - "nauc_mrr_at_10_std": 0.4992925577248018, - "nauc_mrr_at_1_diff1": 0.5396391988509922, - "nauc_mrr_at_1_max": 0.7001152164649064, - "nauc_mrr_at_1_std": 0.5476285926229898, - "nauc_mrr_at_20_diff1": 0.38501958259804275, - "nauc_mrr_at_20_max": 0.5756154448581887, - "nauc_mrr_at_20_std": 0.4932341583371749, - "nauc_mrr_at_3_diff1": 0.44470157708016583, - "nauc_mrr_at_3_max": 0.6355059177771817, - "nauc_mrr_at_3_std": 0.5158617297200611, - "nauc_mrr_at_5_diff1": 0.4213217255125721, - "nauc_mrr_at_5_max": 0.6099601340775752, - "nauc_mrr_at_5_std": 0.5039017165794833, - "nauc_ndcg_at_1000_diff1": 0.2016366246175423, - "nauc_ndcg_at_1000_max": 0.33638671153257416, - "nauc_ndcg_at_1000_std": 0.31838659406129766, - "nauc_ndcg_at_100_diff1": 0.26037541254677204, - "nauc_ndcg_at_100_max": 0.4360310644584217, - "nauc_ndcg_at_100_std": 0.41292617667752046, - "nauc_ndcg_at_10_diff1": 0.34486121056368324, - "nauc_ndcg_at_10_max": 0.5497655909742565, - "nauc_ndcg_at_10_std": 0.48505160980469963, - "nauc_ndcg_at_1_diff1": 0.5396391988509922, - "nauc_ndcg_at_1_max": 0.7001152164649064, - "nauc_ndcg_at_1_std": 0.5476285926229898, - "nauc_ndcg_at_20_diff1": 0.32022522388215546, - "nauc_ndcg_at_20_max": 0.5162912607985376, - "nauc_ndcg_at_20_std": 0.4698969645910983, - "nauc_ndcg_at_3_diff1": 0.42087129551165053, - "nauc_ndcg_at_3_max": 0.6216838926839852, - "nauc_ndcg_at_3_std": 0.5107451829129399, - "nauc_ndcg_at_5_diff1": 0.3885307148680432, - "nauc_ndcg_at_5_max": 0.5848124141719698, - "nauc_ndcg_at_5_std": 0.4930263498076343, - "nauc_precision_at_1000_diff1": 0.08667190720067593, - "nauc_precision_at_1000_max": 0.16811377816078696, - "nauc_precision_at_1000_std": 0.17769418531556536, - "nauc_precision_at_100_diff1": 0.1578952449112062, - "nauc_precision_at_100_max": 0.3180956343724515, - "nauc_precision_at_100_std": 0.33951866584558027, - "nauc_precision_at_10_diff1": 0.26798755947044317, - "nauc_precision_at_10_max": 0.4898376710491013, - "nauc_precision_at_10_std": 0.463757386768931, - "nauc_precision_at_1_diff1": 0.5396391988509922, - "nauc_precision_at_1_max": 0.7001152164649064, - "nauc_precision_at_1_std": 0.5476285926229898, - "nauc_precision_at_20_diff1": 0.23274755075248224, - "nauc_precision_at_20_max": 0.43348759659406555, - "nauc_precision_at_20_std": 0.4368803438053475, - "nauc_precision_at_3_diff1": 0.37070540585431927, - "nauc_precision_at_3_max": 0.5930676402570686, - "nauc_precision_at_3_std": 0.5005478043108843, - "nauc_precision_at_5_diff1": 0.3304513750142505, - "nauc_precision_at_5_max": 0.539945715697192, - "nauc_precision_at_5_std": 0.47366105820995386, - "nauc_recall_at_1000_diff1": 0.08667190720067616, - "nauc_recall_at_1000_max": 0.16811377816078713, - "nauc_recall_at_1000_std": 0.1776941853155657, - "nauc_recall_at_100_diff1": 0.15789524491120618, - "nauc_recall_at_100_max": 0.31809563437245164, - "nauc_recall_at_100_std": 0.3395186658455805, - "nauc_recall_at_10_diff1": 0.26798755947044317, - "nauc_recall_at_10_max": 0.48983767104910125, - "nauc_recall_at_10_std": 0.463757386768931, - "nauc_recall_at_1_diff1": 0.5396391988509922, - "nauc_recall_at_1_max": 0.7001152164649064, - "nauc_recall_at_1_std": 0.5476285926229898, - "nauc_recall_at_20_diff1": 0.23274755075248219, - "nauc_recall_at_20_max": 0.43348759659406566, - "nauc_recall_at_20_std": 0.4368803438053475, - "nauc_recall_at_3_diff1": 0.37070540585431927, - "nauc_recall_at_3_max": 0.5930676402570686, - "nauc_recall_at_3_std": 0.5005478043108846, - "nauc_recall_at_5_diff1": 0.33045137501425054, - "nauc_recall_at_5_max": 0.5399457156971915, - "nauc_recall_at_5_std": 0.4736610582099537, - "ndcg_at_1": 0.01181, - "ndcg_at_10": 0.02625, - "ndcg_at_100": 0.04007, - "ndcg_at_1000": 0.07286, - "ndcg_at_20": 0.02917, - "ndcg_at_3": 0.01837, - "ndcg_at_5": 0.02202, - "precision_at_1": 0.01181, - "precision_at_10": 0.00456, - "precision_at_100": 0.00118, - "precision_at_1000": 0.0004, - "precision_at_20": 0.00285, - "precision_at_3": 0.00778, - "precision_at_5": 0.00647, - "recall_at_1": 0.01181, - "recall_at_10": 0.04556, - "recall_at_100": 0.11806, - "recall_at_1000": 0.40236, - "recall_at_20": 0.05708, - "recall_at_3": 0.02333, - "recall_at_5": 0.03236 - }, - { - "cv_recall_at_1": 0.03369, - "cv_recall_at_10": 0.11091, - "cv_recall_at_100": 0.26295, - "cv_recall_at_1000": 0.62488, - "cv_recall_at_20": 0.1446, - "cv_recall_at_3": 0.05981, - "cv_recall_at_5": 0.08044, - "hf_subset": "ro", - "languages": [ - "ron-Latn" - ], - "main_score": 0.06699, - "map_at_1": 0.03369, - "map_at_10": 0.05358, - "map_at_100": 0.05857, - "map_at_1000": 0.05976, - "map_at_20": 0.05587, - "map_at_3": 0.04478, - "map_at_5": 0.04947, - "mrr_at_1": 0.033693668398146845, - "mrr_at_10": 0.05357894947955309, - "mrr_at_100": 0.058565448685346744, - "mrr_at_1000": 0.059755076195600644, - "mrr_at_20": 0.05587114896195827, - "mrr_at_3": 0.04478450091253684, - "mrr_at_5": 0.04947353643127886, - "nauc_cv_recall_at_1000_diff1": -0.02750735436367819, - "nauc_cv_recall_at_1000_max": 0.2381781077979006, - "nauc_cv_recall_at_1000_std": 0.47236792234255026, - "nauc_cv_recall_at_100_diff1": 0.05755823909457514, - "nauc_cv_recall_at_100_max": 0.2768187610206619, - "nauc_cv_recall_at_100_std": 0.45525879717090767, - "nauc_cv_recall_at_10_diff1": 0.1244953534412135, - "nauc_cv_recall_at_10_max": 0.3349391617413094, - "nauc_cv_recall_at_10_std": 0.4056744665816779, - "nauc_cv_recall_at_1_diff1": 0.35164541350606676, - "nauc_cv_recall_at_1_max": 0.4524196178438682, - "nauc_cv_recall_at_1_std": 0.3814439266603003, - "nauc_cv_recall_at_20_diff1": 0.10413983945287474, - "nauc_cv_recall_at_20_max": 0.33140833112110973, - "nauc_cv_recall_at_20_std": 0.4386125720428651, - "nauc_cv_recall_at_3_diff1": 0.20245552150328452, - "nauc_cv_recall_at_3_max": 0.39518574373629967, - "nauc_cv_recall_at_3_std": 0.39401446691465314, - "nauc_cv_recall_at_5_diff1": 0.1570894857492748, - "nauc_cv_recall_at_5_max": 0.37154853140642485, - "nauc_cv_recall_at_5_std": 0.41356064041724844, - "nauc_map_at_1000_diff1": 0.21603653091241862, - "nauc_map_at_1000_max": 0.38683705895047166, - "nauc_map_at_1000_std": 0.4015751814700608, - "nauc_map_at_100_diff1": 0.21725138314361314, - "nauc_map_at_100_max": 0.3875455996624328, - "nauc_map_at_100_std": 0.4009878482275901, - "nauc_map_at_10_diff1": 0.2306561663320846, - "nauc_map_at_10_max": 0.3961959068993542, - "nauc_map_at_10_std": 0.39365347040267035, - "nauc_map_at_1_diff1": 0.35164541350606676, - "nauc_map_at_1_max": 0.4524196178438682, - "nauc_map_at_1_std": 0.3814439266603003, - "nauc_map_at_20_diff1": 0.22448119703894479, - "nauc_map_at_20_max": 0.394177621020983, - "nauc_map_at_20_std": 0.39904273371471993, - "nauc_map_at_3_diff1": 0.26782441090742687, - "nauc_map_at_3_max": 0.4198874910551511, - "nauc_map_at_3_std": 0.38756137620331704, - "nauc_map_at_5_diff1": 0.2454570039859637, - "nauc_map_at_5_max": 0.40897423169559444, - "nauc_map_at_5_std": 0.39559603455162656, - "nauc_mrr_at_1000_diff1": 0.2160365268297375, - "nauc_mrr_at_1000_max": 0.3868370537295254, - "nauc_mrr_at_1000_std": 0.4015751831776443, - "nauc_mrr_at_100_diff1": 0.21725138314361314, - "nauc_mrr_at_100_max": 0.3875455996624328, - "nauc_mrr_at_100_std": 0.4009878482275901, - "nauc_mrr_at_10_diff1": 0.2306561663320846, - "nauc_mrr_at_10_max": 0.3961959068993542, - "nauc_mrr_at_10_std": 0.39365347040267035, - "nauc_mrr_at_1_diff1": 0.35164541350606676, - "nauc_mrr_at_1_max": 0.4524196178438682, - "nauc_mrr_at_1_std": 0.3814439266603003, - "nauc_mrr_at_20_diff1": 0.22448119703894479, - "nauc_mrr_at_20_max": 0.394177621020983, - "nauc_mrr_at_20_std": 0.39904273371471993, - "nauc_mrr_at_3_diff1": 0.26782441090742687, - "nauc_mrr_at_3_max": 0.4198874910551511, - "nauc_mrr_at_3_std": 0.38756137620331704, - "nauc_mrr_at_5_diff1": 0.2454570039859637, - "nauc_mrr_at_5_max": 0.40897423169559444, - "nauc_mrr_at_5_std": 0.39559603455162656, - "nauc_ndcg_at_1000_diff1": 0.1256340994577069, - "nauc_ndcg_at_1000_max": 0.33073808738849186, - "nauc_ndcg_at_1000_std": 0.42844771829601985, - "nauc_ndcg_at_100_diff1": 0.1466673284994856, - "nauc_ndcg_at_100_max": 0.34043465035066833, - "nauc_ndcg_at_100_std": 0.4230647078541618, - "nauc_ndcg_at_10_diff1": 0.19009024385248605, - "nauc_ndcg_at_10_max": 0.3732493794963218, - "nauc_ndcg_at_10_std": 0.39822924971822304, - "nauc_ndcg_at_1_diff1": 0.35164541350606676, - "nauc_ndcg_at_1_max": 0.4524196178438682, - "nauc_ndcg_at_1_std": 0.3814439266603003, - "nauc_ndcg_at_20_diff1": 0.17574199361072354, - "nauc_ndcg_at_20_max": 0.3688816786833972, - "nauc_ndcg_at_20_std": 0.41355952693089276, - "nauc_ndcg_at_3_diff1": 0.24730011344565822, - "nauc_ndcg_at_3_max": 0.41211762593417495, - "nauc_ndcg_at_3_std": 0.38955204033843227, - "nauc_ndcg_at_5_diff1": 0.21484158800987502, - "nauc_ndcg_at_5_max": 0.3960497699441697, - "nauc_ndcg_at_5_std": 0.4016771024352979, - "nauc_precision_at_1000_diff1": -0.027507354363678547, - "nauc_precision_at_1000_max": 0.23817810779789986, - "nauc_precision_at_1000_std": 0.4723679223425496, - "nauc_precision_at_100_diff1": 0.05755823909457536, - "nauc_precision_at_100_max": 0.276818761020662, - "nauc_precision_at_100_std": 0.4552587971709079, - "nauc_precision_at_10_diff1": 0.12449535344121339, - "nauc_precision_at_10_max": 0.33493916174130933, - "nauc_precision_at_10_std": 0.4056744665816778, - "nauc_precision_at_1_diff1": 0.35164541350606676, - "nauc_precision_at_1_max": 0.4524196178438682, - "nauc_precision_at_1_std": 0.3814439266603003, - "nauc_precision_at_20_diff1": 0.10413983945287494, - "nauc_precision_at_20_max": 0.3314083311211099, - "nauc_precision_at_20_std": 0.43861257204286525, - "nauc_precision_at_3_diff1": 0.20245552150328444, - "nauc_precision_at_3_max": 0.3951857437362995, - "nauc_precision_at_3_std": 0.3940144669146529, - "nauc_precision_at_5_diff1": 0.15708948574927478, - "nauc_precision_at_5_max": 0.371548531406425, - "nauc_precision_at_5_std": 0.4135606404172485, - "nauc_recall_at_1000_diff1": -0.02750735436367819, - "nauc_recall_at_1000_max": 0.2381781077979006, - "nauc_recall_at_1000_std": 0.47236792234255026, - "nauc_recall_at_100_diff1": 0.05755823909457514, - "nauc_recall_at_100_max": 0.2768187610206619, - "nauc_recall_at_100_std": 0.45525879717090767, - "nauc_recall_at_10_diff1": 0.1244953534412135, - "nauc_recall_at_10_max": 0.3349391617413094, - "nauc_recall_at_10_std": 0.4056744665816779, - "nauc_recall_at_1_diff1": 0.35164541350606676, - "nauc_recall_at_1_max": 0.4524196178438682, - "nauc_recall_at_1_std": 0.3814439266603003, - "nauc_recall_at_20_diff1": 0.10413983945287474, - "nauc_recall_at_20_max": 0.33140833112110973, - "nauc_recall_at_20_std": 0.4386125720428651, - "nauc_recall_at_3_diff1": 0.20245552150328452, - "nauc_recall_at_3_max": 0.39518574373629967, - "nauc_recall_at_3_std": 0.39401446691465314, - "nauc_recall_at_5_diff1": 0.1570894857492748, - "nauc_recall_at_5_max": 0.37154853140642485, - "nauc_recall_at_5_std": 0.41356064041724844, - "ndcg_at_1": 0.03369, - "ndcg_at_10": 0.06699, - "ndcg_at_100": 0.09672, - "ndcg_at_1000": 0.1395, - "ndcg_at_20": 0.07545, - "ndcg_at_3": 0.04862, - "ndcg_at_5": 0.0571, - "precision_at_1": 0.03369, - "precision_at_10": 0.01109, - "precision_at_100": 0.00263, - "precision_at_1000": 0.00062, - "precision_at_20": 0.00723, - "precision_at_3": 0.01994, - "precision_at_5": 0.01609, - "recall_at_1": 0.03369, - "recall_at_10": 0.11091, - "recall_at_100": 0.26295, - "recall_at_1000": 0.62488, - "recall_at_20": 0.1446, - "recall_at_3": 0.05981, - "recall_at_5": 0.08044 - }, - { - "cv_recall_at_1": 0.00653, - "cv_recall_at_10": 0.01736, - "cv_recall_at_100": 0.07069, - "cv_recall_at_1000": 0.37639, - "cv_recall_at_20": 0.02528, - "cv_recall_at_3": 0.01097, - "cv_recall_at_5": 0.01333, - "hf_subset": "ru", - "languages": [ - "rus-Cyrl" - ], - "main_score": 0.01126, - "map_at_1": 0.00653, - "map_at_10": 0.0094, - "map_at_100": 0.01089, - "map_at_1000": 0.01175, - "map_at_20": 0.00995, - "map_at_3": 0.00836, - "map_at_5": 0.00889, - "mrr_at_1": 0.006527777777777778, - "mrr_at_10": 0.00940018738977073, - "mrr_at_100": 0.01088995737556746, - "mrr_at_1000": 0.011745707743364286, - "mrr_at_20": 0.009950996655443904, - "mrr_at_3": 0.008356481481481486, - "mrr_at_5": 0.00889120370370371, - "nauc_cv_recall_at_1000_diff1": 0.024532552702524496, - "nauc_cv_recall_at_1000_max": 0.14192208036422993, - "nauc_cv_recall_at_1000_std": 0.2185378722265516, - "nauc_cv_recall_at_100_diff1": 0.08880346939948125, - "nauc_cv_recall_at_100_max": 0.26940032461045954, - "nauc_cv_recall_at_100_std": 0.402973996899385, - "nauc_cv_recall_at_10_diff1": 0.3576538771129596, - "nauc_cv_recall_at_10_max": 0.5124894016635365, - "nauc_cv_recall_at_10_std": 0.6625961899651195, - "nauc_cv_recall_at_1_diff1": 0.5424532879676199, - "nauc_cv_recall_at_1_max": 0.5945287122721484, - "nauc_cv_recall_at_1_std": 0.7447093949271848, - "nauc_cv_recall_at_20_diff1": 0.2560805172825565, - "nauc_cv_recall_at_20_max": 0.4391653423045964, - "nauc_cv_recall_at_20_std": 0.5729942180170247, - "nauc_cv_recall_at_3_diff1": 0.4817462716475395, - "nauc_cv_recall_at_3_max": 0.6084202518060134, - "nauc_cv_recall_at_3_std": 0.7518212972051367, - "nauc_cv_recall_at_5_diff1": 0.45935627403631146, - "nauc_cv_recall_at_5_max": 0.6203604328772024, - "nauc_cv_recall_at_5_std": 0.7455197433145516, - "nauc_map_at_1000_diff1": 0.39858061256455646, - "nauc_map_at_1000_max": 0.5120458013119253, - "nauc_map_at_1000_std": 0.6542261352485719, - "nauc_map_at_100_diff1": 0.4183137782320978, - "nauc_map_at_100_max": 0.532310812827618, - "nauc_map_at_100_std": 0.6788068616607587, - "nauc_map_at_10_diff1": 0.4800763361335301, - "nauc_map_at_10_max": 0.581327495038239, - "nauc_map_at_10_std": 0.7316489298319947, - "nauc_map_at_1_diff1": 0.5424532879676199, - "nauc_map_at_1_max": 0.5945287122721484, - "nauc_map_at_1_std": 0.7447093949271848, - "nauc_map_at_20_diff1": 0.4564129368416336, - "nauc_map_at_20_max": 0.5651552122786756, - "nauc_map_at_20_std": 0.712024497097081, - "nauc_map_at_3_diff1": 0.5150370100048537, - "nauc_map_at_3_max": 0.6003811372725942, - "nauc_map_at_3_std": 0.7522800972464915, - "nauc_map_at_5_diff1": 0.5053000588806172, - "nauc_map_at_5_max": 0.6048658623537564, - "nauc_map_at_5_std": 0.7502147162144954, - "nauc_mrr_at_1000_diff1": 0.3985808585466461, - "nauc_mrr_at_1000_max": 0.5120460518793085, - "nauc_mrr_at_1000_std": 0.654226196079803, - "nauc_mrr_at_100_diff1": 0.4183137782320978, - "nauc_mrr_at_100_max": 0.532310812827618, - "nauc_mrr_at_100_std": 0.6788068616607587, - "nauc_mrr_at_10_diff1": 0.4800763361335301, - "nauc_mrr_at_10_max": 0.581327495038239, - "nauc_mrr_at_10_std": 0.7316489298319947, - "nauc_mrr_at_1_diff1": 0.5424532879676199, - "nauc_mrr_at_1_max": 0.5945287122721484, - "nauc_mrr_at_1_std": 0.7447093949271848, - "nauc_mrr_at_20_diff1": 0.4564129368416336, - "nauc_mrr_at_20_max": 0.5651552122786756, - "nauc_mrr_at_20_std": 0.712024497097081, - "nauc_mrr_at_3_diff1": 0.5150370100048537, - "nauc_mrr_at_3_max": 0.6003811372725942, - "nauc_mrr_at_3_std": 0.7522800972464915, - "nauc_mrr_at_5_diff1": 0.5053000588806172, - "nauc_mrr_at_5_max": 0.6048658623537564, - "nauc_mrr_at_5_std": 0.7502147162144954, - "nauc_ndcg_at_1000_diff1": 0.12888500419156781, - "nauc_ndcg_at_1000_max": 0.2535818040636635, - "nauc_ndcg_at_1000_std": 0.35265764685265366, - "nauc_ndcg_at_100_diff1": 0.23620062158291075, - "nauc_ndcg_at_100_max": 0.38965171262518355, - "nauc_ndcg_at_100_std": 0.529019771584767, - "nauc_ndcg_at_10_diff1": 0.4380991139616366, - "nauc_ndcg_at_10_max": 0.5591425095770631, - "nauc_ndcg_at_10_std": 0.7089649644315631, - "nauc_ndcg_at_1_diff1": 0.5424532879676199, - "nauc_ndcg_at_1_max": 0.5945287122721484, - "nauc_ndcg_at_1_std": 0.7447093949271848, - "nauc_ndcg_at_20_diff1": 0.3780217086880114, - "nauc_ndcg_at_20_max": 0.5173384500681644, - "nauc_ndcg_at_20_std": 0.6588354395696102, - "nauc_ndcg_at_3_diff1": 0.5049257793161163, - "nauc_ndcg_at_3_max": 0.6028096366542133, - "nauc_ndcg_at_3_std": 0.7522771466939655, - "nauc_ndcg_at_5_diff1": 0.490324978851244, - "nauc_ndcg_at_5_max": 0.6098454030674927, - "nauc_ndcg_at_5_std": 0.7488643686971155, - "nauc_precision_at_1000_diff1": 0.024532552702524055, - "nauc_precision_at_1000_max": 0.14192208036422974, - "nauc_precision_at_1000_std": 0.21853787222655116, - "nauc_precision_at_100_diff1": 0.08880346939948128, - "nauc_precision_at_100_max": 0.2694003246104596, - "nauc_precision_at_100_std": 0.40297399689938496, - "nauc_precision_at_10_diff1": 0.35765387711295954, - "nauc_precision_at_10_max": 0.5124894016635364, - "nauc_precision_at_10_std": 0.6625961899651195, - "nauc_precision_at_1_diff1": 0.5424532879676199, - "nauc_precision_at_1_max": 0.5945287122721484, - "nauc_precision_at_1_std": 0.7447093949271848, - "nauc_precision_at_20_diff1": 0.2560805172825565, - "nauc_precision_at_20_max": 0.4391653423045967, - "nauc_precision_at_20_std": 0.5729942180170249, - "nauc_precision_at_3_diff1": 0.4817462716475393, - "nauc_precision_at_3_max": 0.6084202518060132, - "nauc_precision_at_3_std": 0.7518212972051364, - "nauc_precision_at_5_diff1": 0.45935627403631163, - "nauc_precision_at_5_max": 0.6203604328772026, - "nauc_precision_at_5_std": 0.7455197433145517, - "nauc_recall_at_1000_diff1": 0.024532552702524496, - "nauc_recall_at_1000_max": 0.14192208036422993, - "nauc_recall_at_1000_std": 0.2185378722265516, - "nauc_recall_at_100_diff1": 0.08880346939948125, - "nauc_recall_at_100_max": 0.26940032461045954, - "nauc_recall_at_100_std": 0.402973996899385, - "nauc_recall_at_10_diff1": 0.3576538771129596, - "nauc_recall_at_10_max": 0.5124894016635365, - "nauc_recall_at_10_std": 0.6625961899651195, - "nauc_recall_at_1_diff1": 0.5424532879676199, - "nauc_recall_at_1_max": 0.5945287122721484, - "nauc_recall_at_1_std": 0.7447093949271848, - "nauc_recall_at_20_diff1": 0.2560805172825565, - "nauc_recall_at_20_max": 0.4391653423045964, - "nauc_recall_at_20_std": 0.5729942180170247, - "nauc_recall_at_3_diff1": 0.4817462716475395, - "nauc_recall_at_3_max": 0.6084202518060134, - "nauc_recall_at_3_std": 0.7518212972051367, - "nauc_recall_at_5_diff1": 0.45935627403631146, - "nauc_recall_at_5_max": 0.6203604328772024, - "nauc_recall_at_5_std": 0.7455197433145516, - "ndcg_at_1": 0.00653, - "ndcg_at_10": 0.01126, - "ndcg_at_100": 0.02124, - "ndcg_at_1000": 0.05644, - "ndcg_at_20": 0.01327, - "ndcg_at_3": 0.00902, - "ndcg_at_5": 0.00999, - "precision_at_1": 0.00653, - "precision_at_10": 0.00174, - "precision_at_100": 0.00071, - "precision_at_1000": 0.00038, - "precision_at_20": 0.00126, - "precision_at_3": 0.00366, - "precision_at_5": 0.00267, - "recall_at_1": 0.00653, - "recall_at_10": 0.01736, - "recall_at_100": 0.07069, - "recall_at_1000": 0.37639, - "recall_at_20": 0.02528, - "recall_at_3": 0.01097, - "recall_at_5": 0.01333 - }, - { - "cv_recall_at_1": 0.03231, - "cv_recall_at_10": 0.11646, - "cv_recall_at_100": 0.28654, - "cv_recall_at_1000": 0.66011, - "cv_recall_at_20": 0.15124, - "cv_recall_at_3": 0.06572, - "cv_recall_at_5": 0.08456, - "hf_subset": "sv", - "languages": [ - "swe-Latn" - ], - "main_score": 0.06925, - "map_at_1": 0.03231, - "map_at_10": 0.05478, - "map_at_100": 0.06028, - "map_at_1000": 0.06155, - "map_at_20": 0.05716, - "map_at_3": 0.04634, - "map_at_5": 0.05056, - "mrr_at_1": 0.03231128832668775, - "mrr_at_10": 0.05477609073786709, - "mrr_at_100": 0.06027529180422331, - "mrr_at_1000": 0.06154752807105916, - "mrr_at_20": 0.05716054677837161, - "mrr_at_3": 0.046335762408909624, - "mrr_at_5": 0.05056372886016759, - "nauc_cv_recall_at_1000_diff1": 0.06548065830489194, - "nauc_cv_recall_at_1000_max": 0.06237750952655973, - "nauc_cv_recall_at_1000_std": 0.4660590815935642, - "nauc_cv_recall_at_100_diff1": 0.0851957419101093, - "nauc_cv_recall_at_100_max": 0.09897022897570883, - "nauc_cv_recall_at_100_std": 0.42192223874633566, - "nauc_cv_recall_at_10_diff1": 0.14987876127649138, - "nauc_cv_recall_at_10_max": 0.1678938870971709, - "nauc_cv_recall_at_10_std": 0.3794087013292822, - "nauc_cv_recall_at_1_diff1": 0.36422330937694863, - "nauc_cv_recall_at_1_max": 0.2951200582537345, - "nauc_cv_recall_at_1_std": 0.3413342988414905, - "nauc_cv_recall_at_20_diff1": 0.1290383181182195, - "nauc_cv_recall_at_20_max": 0.13366207759059023, - "nauc_cv_recall_at_20_std": 0.3742757190843819, - "nauc_cv_recall_at_3_diff1": 0.2059850506998725, - "nauc_cv_recall_at_3_max": 0.23042996884778455, - "nauc_cv_recall_at_3_std": 0.3839113945804567, - "nauc_cv_recall_at_5_diff1": 0.17099262193583747, - "nauc_cv_recall_at_5_max": 0.19722962011318293, - "nauc_cv_recall_at_5_std": 0.38142108667220237, - "nauc_map_at_1000_diff1": 0.2249431356126328, - "nauc_map_at_1000_max": 0.2206935159089587, - "nauc_map_at_1000_std": 0.3697545426309123, - "nauc_map_at_100_diff1": 0.22568599685467652, - "nauc_map_at_100_max": 0.22145061764922253, - "nauc_map_at_100_std": 0.36889370714450276, - "nauc_map_at_10_diff1": 0.23874524889661422, - "nauc_map_at_10_max": 0.23508221440558008, - "nauc_map_at_10_std": 0.366064655472392, - "nauc_map_at_1_diff1": 0.36422330937694863, - "nauc_map_at_1_max": 0.2951200582537345, - "nauc_map_at_1_std": 0.3413342988414905, - "nauc_map_at_20_diff1": 0.23263970425299382, - "nauc_map_at_20_max": 0.22750740972522127, - "nauc_map_at_20_std": 0.3658400097951212, - "nauc_map_at_3_diff1": 0.26758268789384415, - "nauc_map_at_3_max": 0.2598950633798191, - "nauc_map_at_3_std": 0.36573044558195744, - "nauc_map_at_5_diff1": 0.24956071499912286, - "nauc_map_at_5_max": 0.24540260206801137, - "nauc_map_at_5_std": 0.36597438240167834, - "nauc_mrr_at_1000_diff1": 0.2249431249999408, - "nauc_mrr_at_1000_max": 0.22069352402740708, - "nauc_mrr_at_1000_std": 0.36975454576408934, - "nauc_mrr_at_100_diff1": 0.22568599685467652, - "nauc_mrr_at_100_max": 0.22145061764922253, - "nauc_mrr_at_100_std": 0.36889370714450276, - "nauc_mrr_at_10_diff1": 0.23874524889661422, - "nauc_mrr_at_10_max": 0.23508221440558008, - "nauc_mrr_at_10_std": 0.366064655472392, - "nauc_mrr_at_1_diff1": 0.36422330937694863, - "nauc_mrr_at_1_max": 0.2951200582537345, - "nauc_mrr_at_1_std": 0.3413342988414905, - "nauc_mrr_at_20_diff1": 0.23263970425299382, - "nauc_mrr_at_20_max": 0.22750740972522127, - "nauc_mrr_at_20_std": 0.3658400097951212, - "nauc_mrr_at_3_diff1": 0.26758268789384415, - "nauc_mrr_at_3_max": 0.2598950633798191, - "nauc_mrr_at_3_std": 0.36573044558195744, - "nauc_mrr_at_5_diff1": 0.24956071499912286, - "nauc_mrr_at_5_max": 0.24540260206801137, - "nauc_mrr_at_5_std": 0.36597438240167834, - "nauc_ndcg_at_1000_diff1": 0.1590560410499789, - "nauc_ndcg_at_1000_max": 0.16025781184624488, - "nauc_ndcg_at_1000_std": 0.4001104098966938, - "nauc_ndcg_at_100_diff1": 0.16297583355014392, - "nauc_ndcg_at_100_max": 0.16781657473650827, - "nauc_ndcg_at_100_std": 0.3889778490410969, - "nauc_ndcg_at_10_diff1": 0.2041448033747642, - "nauc_ndcg_at_10_max": 0.20982886824766298, - "nauc_ndcg_at_10_std": 0.37141720258175265, - "nauc_ndcg_at_1_diff1": 0.36422330937694863, - "nauc_ndcg_at_1_max": 0.2951200582537345, - "nauc_ndcg_at_1_std": 0.3413342988414905, - "nauc_ndcg_at_20_diff1": 0.19013064317884823, - "nauc_ndcg_at_20_max": 0.1907506207789357, - "nauc_ndcg_at_20_std": 0.369922491987889, - "nauc_ndcg_at_3_diff1": 0.24734016858623997, - "nauc_ndcg_at_3_max": 0.2503765220804391, - "nauc_ndcg_at_3_std": 0.3716434714330145, - "nauc_ndcg_at_5_diff1": 0.2216607014157429, - "nauc_ndcg_at_5_max": 0.22866568745631413, - "nauc_ndcg_at_5_std": 0.3715288106986452, - "nauc_precision_at_1000_diff1": 0.0654806583048908, - "nauc_precision_at_1000_max": 0.062377509526559224, - "nauc_precision_at_1000_std": 0.4660590815935634, - "nauc_precision_at_100_diff1": 0.0851957419101095, - "nauc_precision_at_100_max": 0.09897022897570915, - "nauc_precision_at_100_std": 0.4219222387463361, - "nauc_precision_at_10_diff1": 0.14987876127649127, - "nauc_precision_at_10_max": 0.16789388709717096, - "nauc_precision_at_10_std": 0.3794087013292824, - "nauc_precision_at_1_diff1": 0.36422330937694863, - "nauc_precision_at_1_max": 0.2951200582537345, - "nauc_precision_at_1_std": 0.3413342988414905, - "nauc_precision_at_20_diff1": 0.12903831811821959, - "nauc_precision_at_20_max": 0.13366207759059015, - "nauc_precision_at_20_std": 0.37427571908438173, - "nauc_precision_at_3_diff1": 0.20598505069987239, - "nauc_precision_at_3_max": 0.2304299688477843, - "nauc_precision_at_3_std": 0.38391139458045653, - "nauc_precision_at_5_diff1": 0.1709926219358375, - "nauc_precision_at_5_max": 0.19722962011318282, - "nauc_precision_at_5_std": 0.3814210866722025, - "nauc_recall_at_1000_diff1": 0.06548065830489194, - "nauc_recall_at_1000_max": 0.06237750952655973, - "nauc_recall_at_1000_std": 0.4660590815935642, - "nauc_recall_at_100_diff1": 0.0851957419101093, - "nauc_recall_at_100_max": 0.09897022897570883, - "nauc_recall_at_100_std": 0.42192223874633566, - "nauc_recall_at_10_diff1": 0.14987876127649138, - "nauc_recall_at_10_max": 0.1678938870971709, - "nauc_recall_at_10_std": 0.3794087013292822, - "nauc_recall_at_1_diff1": 0.36422330937694863, - "nauc_recall_at_1_max": 0.2951200582537345, - "nauc_recall_at_1_std": 0.3413342988414905, - "nauc_recall_at_20_diff1": 0.1290383181182195, - "nauc_recall_at_20_max": 0.13366207759059023, - "nauc_recall_at_20_std": 0.3742757190843819, - "nauc_recall_at_3_diff1": 0.2059850506998725, - "nauc_recall_at_3_max": 0.23042996884778455, - "nauc_recall_at_3_std": 0.3839113945804567, - "nauc_recall_at_5_diff1": 0.17099262193583747, - "nauc_recall_at_5_max": 0.19722962011318293, - "nauc_recall_at_5_std": 0.38142108667220237, - "ndcg_at_1": 0.03231, - "ndcg_at_10": 0.06925, - "ndcg_at_100": 0.10239, - "ndcg_at_1000": 0.14682, - "ndcg_at_20": 0.078, - "ndcg_at_3": 0.05129, - "ndcg_at_5": 0.05898, - "precision_at_1": 0.03231, - "precision_at_10": 0.01165, - "precision_at_100": 0.00287, - "precision_at_1000": 0.00066, - "precision_at_20": 0.00756, - "precision_at_3": 0.02191, - "precision_at_5": 0.01691, - "recall_at_1": 0.03231, - "recall_at_10": 0.11646, - "recall_at_100": 0.28654, - "recall_at_1000": 0.66011, - "recall_at_20": 0.15124, - "recall_at_3": 0.06572, - "recall_at_5": 0.08456 - }, - { - "cv_recall_at_1": 0.01093, - "cv_recall_at_10": 0.04116, - "cv_recall_at_100": 0.09523, - "cv_recall_at_1000": 0.34601, - "cv_recall_at_20": 0.05436, - "cv_recall_at_3": 0.02285, - "cv_recall_at_5": 0.03051, - "hf_subset": "sw", - "languages": [ - "swa-Latn" - ], - "main_score": 0.02425, - "map_at_1": 0.01093, - "map_at_10": 0.01906, - "map_at_100": 0.02086, - "map_at_1000": 0.02153, - "map_at_20": 0.01995, - "map_at_3": 0.01599, - "map_at_5": 0.01771, - "mrr_at_1": 0.010928186204938973, - "mrr_at_10": 0.01906333211683765, - "mrr_at_100": 0.020855486004402696, - "mrr_at_1000": 0.0215304990410364, - "mrr_at_20": 0.019949391755870887, - "mrr_at_3": 0.01599015990159901, - "mrr_at_5": 0.017707446305232282, - "nauc_cv_recall_at_1000_diff1": -0.029066888331591732, - "nauc_cv_recall_at_1000_max": 0.11440054004195425, - "nauc_cv_recall_at_1000_std": 0.17961864663480073, - "nauc_cv_recall_at_100_diff1": -0.027853539828646603, - "nauc_cv_recall_at_100_max": 0.28190708885837534, - "nauc_cv_recall_at_100_std": 0.41941312173899065, - "nauc_cv_recall_at_10_diff1": -0.030576462166714554, - "nauc_cv_recall_at_10_max": 0.45854089032862744, - "nauc_cv_recall_at_10_std": 0.5998171835323256, - "nauc_cv_recall_at_1_diff1": 0.0507230754331051, - "nauc_cv_recall_at_1_max": 0.5739380056956582, - "nauc_cv_recall_at_1_std": 0.640247504575004, - "nauc_cv_recall_at_20_diff1": -0.038133309010667873, - "nauc_cv_recall_at_20_max": 0.4169107885199648, - "nauc_cv_recall_at_20_std": 0.5630257929220277, - "nauc_cv_recall_at_3_diff1": -0.046575756766060704, - "nauc_cv_recall_at_3_max": 0.4955809528793124, - "nauc_cv_recall_at_3_std": 0.6080999130990666, - "nauc_cv_recall_at_5_diff1": -0.049400486753644265, - "nauc_cv_recall_at_5_max": 0.4755393479810073, - "nauc_cv_recall_at_5_std": 0.5964315119056258, - "nauc_map_at_1000_diff1": -0.015275603948263626, - "nauc_map_at_1000_max": 0.47149663581085133, - "nauc_map_at_1000_std": 0.5803479917322986, - "nauc_map_at_100_diff1": -0.015009992508063967, - "nauc_map_at_100_max": 0.48032511689420504, - "nauc_map_at_100_std": 0.5899444038260839, - "nauc_map_at_10_diff1": -0.012038249666452248, - "nauc_map_at_10_max": 0.5073521391195545, - "nauc_map_at_10_std": 0.6118480534729175, - "nauc_map_at_1_diff1": 0.0507230754331051, - "nauc_map_at_1_max": 0.5739380056956582, - "nauc_map_at_1_std": 0.640247504575004, - "nauc_map_at_20_diff1": -0.015037961413821672, - "nauc_map_at_20_max": 0.49703156017996536, - "nauc_map_at_20_std": 0.6049687714113716, - "nauc_map_at_3_diff1": -0.00927438069788099, - "nauc_map_at_3_max": 0.5256291385247881, - "nauc_map_at_3_std": 0.6179931686874017, - "nauc_map_at_5_diff1": -0.014151180614339051, - "nauc_map_at_5_max": 0.5137793474810228, - "nauc_map_at_5_std": 0.6122338853382189, - "nauc_mrr_at_1000_diff1": -0.015275754819867498, - "nauc_mrr_at_1000_max": 0.4714939865282889, - "nauc_mrr_at_1000_std": 0.5803448405218806, - "nauc_mrr_at_100_diff1": -0.015010146863816942, - "nauc_mrr_at_100_max": 0.48032239486348116, - "nauc_mrr_at_100_std": 0.5899411685976717, - "nauc_mrr_at_10_diff1": -0.012038249666452248, - "nauc_mrr_at_10_max": 0.5073521391195545, - "nauc_mrr_at_10_std": 0.6118480534729175, - "nauc_mrr_at_1_diff1": 0.0507230754331051, - "nauc_mrr_at_1_max": 0.5739380056956582, - "nauc_mrr_at_1_std": 0.640247504575004, - "nauc_mrr_at_20_diff1": -0.015037961413821672, - "nauc_mrr_at_20_max": 0.49703156017996536, - "nauc_mrr_at_20_std": 0.6049687714113716, - "nauc_mrr_at_3_diff1": -0.00927438069788099, - "nauc_mrr_at_3_max": 0.5256291385247881, - "nauc_mrr_at_3_std": 0.6179931686874017, - "nauc_mrr_at_5_diff1": -0.014151180614339051, - "nauc_mrr_at_5_max": 0.5137793474810228, - "nauc_mrr_at_5_std": 0.6122338853382189, - "nauc_ndcg_at_1000_diff1": -0.025189927360568656, - "nauc_ndcg_at_1000_max": 0.2755200466099936, - "nauc_ndcg_at_1000_std": 0.3708203542848814, - "nauc_ndcg_at_100_diff1": -0.023820824475258018, - "nauc_ndcg_at_100_max": 0.38842037833035764, - "nauc_ndcg_at_100_std": 0.5143521973344823, - "nauc_ndcg_at_10_diff1": -0.020566808342009864, - "nauc_ndcg_at_10_max": 0.4880795737018556, - "nauc_ndcg_at_10_std": 0.6067670926354549, - "nauc_ndcg_at_1_diff1": 0.0507230754331051, - "nauc_ndcg_at_1_max": 0.5739380056956582, - "nauc_ndcg_at_1_std": 0.640247504575004, - "nauc_ndcg_at_20_diff1": -0.026200186616833147, - "nauc_ndcg_at_20_max": 0.4635014088103323, - "nauc_ndcg_at_20_std": 0.5881263187112693, - "nauc_ndcg_at_3_diff1": -0.021590017901252544, - "nauc_ndcg_at_3_max": 0.5157085805179946, - "nauc_ndcg_at_3_std": 0.6146341160023117, - "nauc_ndcg_at_5_diff1": -0.0271432345961738, - "nauc_ndcg_at_5_max": 0.49984923150505894, - "nauc_ndcg_at_5_std": 0.6064767042528324, - "nauc_precision_at_1000_diff1": -0.02906688833159206, - "nauc_precision_at_1000_max": 0.11440054004195374, - "nauc_precision_at_1000_std": 0.17961864663480062, - "nauc_precision_at_100_diff1": -0.027853539828646398, - "nauc_precision_at_100_max": 0.2819070888583754, - "nauc_precision_at_100_std": 0.4194131217389907, - "nauc_precision_at_10_diff1": -0.030576462166714447, - "nauc_precision_at_10_max": 0.4585408903286276, - "nauc_precision_at_10_std": 0.5998171835323257, - "nauc_precision_at_1_diff1": 0.0507230754331051, - "nauc_precision_at_1_max": 0.5739380056956582, - "nauc_precision_at_1_std": 0.640247504575004, - "nauc_precision_at_20_diff1": -0.03813330901066789, - "nauc_precision_at_20_max": 0.4169107885199649, - "nauc_precision_at_20_std": 0.5630257929220278, - "nauc_precision_at_3_diff1": -0.04657575676606086, - "nauc_precision_at_3_max": 0.4955809528793123, - "nauc_precision_at_3_std": 0.6080999130990669, - "nauc_precision_at_5_diff1": -0.04940048675364416, - "nauc_precision_at_5_max": 0.4755393479810075, - "nauc_precision_at_5_std": 0.5964315119056262, - "nauc_recall_at_1000_diff1": -0.029066888331591732, - "nauc_recall_at_1000_max": 0.11440054004195425, - "nauc_recall_at_1000_std": 0.17961864663480073, - "nauc_recall_at_100_diff1": -0.027853539828646603, - "nauc_recall_at_100_max": 0.28190708885837534, - "nauc_recall_at_100_std": 0.41941312173899065, - "nauc_recall_at_10_diff1": -0.030576462166714554, - "nauc_recall_at_10_max": 0.45854089032862744, - "nauc_recall_at_10_std": 0.5998171835323256, - "nauc_recall_at_1_diff1": 0.0507230754331051, - "nauc_recall_at_1_max": 0.5739380056956582, - "nauc_recall_at_1_std": 0.640247504575004, - "nauc_recall_at_20_diff1": -0.038133309010667873, - "nauc_recall_at_20_max": 0.4169107885199648, - "nauc_recall_at_20_std": 0.5630257929220277, - "nauc_recall_at_3_diff1": -0.046575756766060704, - "nauc_recall_at_3_max": 0.4955809528793124, - "nauc_recall_at_3_std": 0.6080999130990666, - "nauc_recall_at_5_diff1": -0.049400486753644265, - "nauc_recall_at_5_max": 0.4755393479810073, - "nauc_recall_at_5_std": 0.5964315119056258, - "ndcg_at_1": 0.01093, - "ndcg_at_10": 0.02425, - "ndcg_at_100": 0.03484, - "ndcg_at_1000": 0.06352, - "ndcg_at_20": 0.02755, - "ndcg_at_3": 0.01774, - "ndcg_at_5": 0.02087, - "precision_at_1": 0.01093, - "precision_at_10": 0.00412, - "precision_at_100": 0.00095, - "precision_at_1000": 0.00035, - "precision_at_20": 0.00272, - "precision_at_3": 0.00762, - "precision_at_5": 0.0061, - "recall_at_1": 0.01093, - "recall_at_10": 0.04116, - "recall_at_100": 0.09523, - "recall_at_1000": 0.34601, - "recall_at_20": 0.05436, - "recall_at_3": 0.02285, - "recall_at_5": 0.03051 - }, - { - "cv_recall_at_1": 0.00028, - "cv_recall_at_10": 0.00306, - "cv_recall_at_100": 0.02806, - "cv_recall_at_1000": 0.28208, - "cv_recall_at_20": 0.00542, - "cv_recall_at_3": 0.00083, - "cv_recall_at_5": 0.00125, - "hf_subset": "te", - "languages": [ - "tel-Telu" - ], - "main_score": 0.00133, - "map_at_1": 0.00028, - "map_at_10": 0.00082, - "map_at_100": 0.00143, - "map_at_1000": 0.00209, - "map_at_20": 0.00097, - "map_at_3": 0.00051, - "map_at_5": 0.0006, - "mrr_at_1": 0.0002777777777777778, - "mrr_at_10": 0.0008230268959435623, - "mrr_at_100": 0.0014292767932411472, - "mrr_at_1000": 0.0020886165708559946, - "mrr_at_20": 0.0009712024307773786, - "mrr_at_3": 0.0005092592592592592, - "mrr_at_5": 0.000599537037037037, - "nauc_cv_recall_at_1000_diff1": -0.006836242757369639, - "nauc_cv_recall_at_1000_max": 0.049066917816044116, - "nauc_cv_recall_at_1000_std": 0.037085901708809384, - "nauc_cv_recall_at_100_diff1": 0.04301099549182174, - "nauc_cv_recall_at_100_max": 0.10517311457467186, - "nauc_cv_recall_at_100_std": 0.07455761674906419, - "nauc_cv_recall_at_10_diff1": -0.07094421543039736, - "nauc_cv_recall_at_10_max": -0.06462668000097577, - "nauc_cv_recall_at_10_std": -0.020635656267531922, - "nauc_cv_recall_at_1_diff1": 0.22806546820499068, - "nauc_cv_recall_at_1_max": 0.22806546820499068, - "nauc_cv_recall_at_1_std": 0.17171988194258117, - "nauc_cv_recall_at_20_diff1": -0.07966123850212935, - "nauc_cv_recall_at_20_max": -0.08551595082316851, - "nauc_cv_recall_at_20_std": 0.00919141675782415, - "nauc_cv_recall_at_3_diff1": -0.17789106519989273, - "nauc_cv_recall_at_3_max": -0.03063232269027806, - "nauc_cv_recall_at_3_std": 0.003353903944191056, - "nauc_cv_recall_at_5_diff1": -0.06949288972363836, - "nauc_cv_recall_at_5_max": -0.02900754255731451, - "nauc_cv_recall_at_5_std": -0.020033986226634583, - "nauc_map_at_1000_diff1": -0.011257585036496497, - "nauc_map_at_1000_max": 0.029375661768316776, - "nauc_map_at_1000_std": 0.04410516975714857, - "nauc_map_at_100_diff1": -0.011517552807642387, - "nauc_map_at_100_max": 0.025626619594148595, - "nauc_map_at_100_std": 0.04467982209656915, - "nauc_map_at_10_diff1": -0.03690924911344003, - "nauc_map_at_10_max": 0.0012078815651148187, - "nauc_map_at_10_std": 0.029082751539292167, - "nauc_map_at_1_diff1": 0.22806546820499068, - "nauc_map_at_1_max": 0.22806546820499068, - "nauc_map_at_1_std": 0.17171988194258117, - "nauc_map_at_20_diff1": -0.03897001588097872, - "nauc_map_at_20_max": -0.009199473447509016, - "nauc_map_at_20_std": 0.03966215121731595, - "nauc_map_at_3_diff1": -0.046161914286411206, - "nauc_map_at_3_max": 0.054821084469595376, - "nauc_map_at_3_std": 0.07165158426226309, - "nauc_map_at_5_diff1": -0.014985087429412576, - "nauc_map_at_5_max": 0.04412649837924862, - "nauc_map_at_5_std": 0.05532724260923882, - "nauc_mrr_at_1000_diff1": -0.011257585036496497, - "nauc_mrr_at_1000_max": 0.029375661768316776, - "nauc_mrr_at_1000_std": 0.04410516975714857, - "nauc_mrr_at_100_diff1": -0.011517552807642387, - "nauc_mrr_at_100_max": 0.025626619594148595, - "nauc_mrr_at_100_std": 0.04467982209656915, - "nauc_mrr_at_10_diff1": -0.03690924911344003, - "nauc_mrr_at_10_max": 0.0012078815651148187, - "nauc_mrr_at_10_std": 0.029082751539292167, - "nauc_mrr_at_1_diff1": 0.22806546820499068, - "nauc_mrr_at_1_max": 0.22806546820499068, - "nauc_mrr_at_1_std": 0.17171988194258117, - "nauc_mrr_at_20_diff1": -0.03897001588097872, - "nauc_mrr_at_20_max": -0.009199473447509016, - "nauc_mrr_at_20_std": 0.03966215121731595, - "nauc_mrr_at_3_diff1": -0.046161914286411206, - "nauc_mrr_at_3_max": 0.054821084469595376, - "nauc_mrr_at_3_std": 0.07165158426226309, - "nauc_mrr_at_5_diff1": -0.014985087429412576, - "nauc_mrr_at_5_max": 0.04412649837924862, - "nauc_mrr_at_5_std": 0.05532724260923882, - "nauc_ndcg_at_1000_diff1": -0.005808457844089288, - "nauc_ndcg_at_1000_max": 0.04742256068385995, - "nauc_ndcg_at_1000_std": 0.039097539233417467, - "nauc_ndcg_at_100_diff1": 0.024104271564372164, - "nauc_ndcg_at_100_max": 0.0760880166616617, - "nauc_ndcg_at_100_std": 0.06196169194032705, - "nauc_ndcg_at_10_diff1": -0.057071756465906585, - "nauc_ndcg_at_10_max": -0.034206271856895745, - "nauc_ndcg_at_10_std": 0.0021729394642180655, - "nauc_ndcg_at_1_diff1": 0.22806546820499068, - "nauc_ndcg_at_1_max": 0.22806546820499068, - "nauc_ndcg_at_1_std": 0.17171988194258117, - "nauc_ndcg_at_20_diff1": -0.06339489290133342, - "nauc_ndcg_at_20_max": -0.05349972674229658, - "nauc_ndcg_at_20_std": 0.02098634745170093, - "nauc_ndcg_at_3_diff1": -0.09337817648407122, - "nauc_ndcg_at_3_max": 0.024243382226872005, - "nauc_ndcg_at_3_std": 0.047640069545597234, - "nauc_ndcg_at_5_diff1": -0.038933138628667374, - "nauc_ndcg_at_5_max": 0.014222495494588525, - "nauc_ndcg_at_5_std": 0.025555236284224053, - "nauc_precision_at_1000_diff1": -0.006836242757369853, - "nauc_precision_at_1000_max": 0.04906691781604409, - "nauc_precision_at_1000_std": 0.037085901708809176, - "nauc_precision_at_100_diff1": 0.04301099549182199, - "nauc_precision_at_100_max": 0.10517311457467209, - "nauc_precision_at_100_std": 0.07455761674906439, - "nauc_precision_at_10_diff1": -0.07094421543039747, - "nauc_precision_at_10_max": -0.06462668000097574, - "nauc_precision_at_10_std": -0.020635656267531922, - "nauc_precision_at_1_diff1": 0.22806546820499068, - "nauc_precision_at_1_max": 0.22806546820499068, - "nauc_precision_at_1_std": 0.17171988194258117, - "nauc_precision_at_20_diff1": -0.07966123850212924, - "nauc_precision_at_20_max": -0.0855159508231683, - "nauc_precision_at_20_std": 0.009191416757824122, - "nauc_precision_at_3_diff1": -0.17789106519989278, - "nauc_precision_at_3_max": -0.030632322690278217, - "nauc_precision_at_3_std": 0.0033539039441908805, - "nauc_precision_at_5_diff1": -0.06949288972363828, - "nauc_precision_at_5_max": -0.029007542557314442, - "nauc_precision_at_5_std": -0.020033986226634496, - "nauc_recall_at_1000_diff1": -0.006836242757369639, - "nauc_recall_at_1000_max": 0.049066917816044116, - "nauc_recall_at_1000_std": 0.037085901708809384, - "nauc_recall_at_100_diff1": 0.04301099549182174, - "nauc_recall_at_100_max": 0.10517311457467186, - "nauc_recall_at_100_std": 0.07455761674906419, - "nauc_recall_at_10_diff1": -0.07094421543039736, - "nauc_recall_at_10_max": -0.06462668000097577, - "nauc_recall_at_10_std": -0.020635656267531922, - "nauc_recall_at_1_diff1": 0.22806546820499068, - "nauc_recall_at_1_max": 0.22806546820499068, - "nauc_recall_at_1_std": 0.17171988194258117, - "nauc_recall_at_20_diff1": -0.07966123850212935, - "nauc_recall_at_20_max": -0.08551595082316851, - "nauc_recall_at_20_std": 0.00919141675782415, - "nauc_recall_at_3_diff1": -0.17789106519989273, - "nauc_recall_at_3_max": -0.03063232269027806, - "nauc_recall_at_3_std": 0.003353903944191056, - "nauc_recall_at_5_diff1": -0.06949288972363836, - "nauc_recall_at_5_max": -0.02900754255731451, - "nauc_recall_at_5_std": -0.020033986226634583, - "ndcg_at_1": 0.00028, - "ndcg_at_10": 0.00133, - "ndcg_at_100": 0.00585, - "ndcg_at_1000": 0.03475, - "ndcg_at_20": 0.0019, - "ndcg_at_3": 0.00059, - "ndcg_at_5": 0.00076, - "precision_at_1": 0.00028, - "precision_at_10": 0.00031, - "precision_at_100": 0.00028, - "precision_at_1000": 0.00028, - "precision_at_20": 0.00027, - "precision_at_3": 0.00028, - "precision_at_5": 0.00025, - "recall_at_1": 0.00028, - "recall_at_10": 0.00306, - "recall_at_100": 0.02806, - "recall_at_1000": 0.28208, - "recall_at_20": 0.00542, - "recall_at_3": 0.00083, - "recall_at_5": 0.00125 - }, - { - "cv_recall_at_1": 0.01264, - "cv_recall_at_10": 0.03583, - "cv_recall_at_100": 0.11236, - "cv_recall_at_1000": 0.44097, - "cv_recall_at_20": 0.04778, - "cv_recall_at_3": 0.02125, - "cv_recall_at_5": 0.02681, - "hf_subset": "th", - "languages": [ - "tha-Thai" - ], - "main_score": 0.0227, - "map_at_1": 0.01264, - "map_at_10": 0.0187, - "map_at_100": 0.02089, - "map_at_1000": 0.02184, - "map_at_20": 0.01949, - "map_at_3": 0.01627, - "map_at_5": 0.01753, - "mrr_at_1": 0.012638888888888889, - "mrr_at_10": 0.018695436507936505, - "mrr_at_100": 0.02088866848548745, - "mrr_at_1000": 0.021838592527878096, - "mrr_at_20": 0.019494748513082492, - "mrr_at_3": 0.016273148148148144, - "mrr_at_5": 0.017530092592592594, - "nauc_cv_recall_at_1000_diff1": 0.022292591165605398, - "nauc_cv_recall_at_1000_max": 0.13098130226865806, - "nauc_cv_recall_at_1000_std": 0.2720168619540829, - "nauc_cv_recall_at_100_diff1": 0.09501884772423592, - "nauc_cv_recall_at_100_max": 0.2599535722147032, - "nauc_cv_recall_at_100_std": 0.3801763208649803, - "nauc_cv_recall_at_10_diff1": 0.2632091816890365, - "nauc_cv_recall_at_10_max": 0.47505631438715595, - "nauc_cv_recall_at_10_std": 0.506678688722355, - "nauc_cv_recall_at_1_diff1": 0.6077745704791587, - "nauc_cv_recall_at_1_max": 0.6941858785164394, - "nauc_cv_recall_at_1_std": 0.5627334833130381, - "nauc_cv_recall_at_20_diff1": 0.19312636886079568, - "nauc_cv_recall_at_20_max": 0.40874885343283757, - "nauc_cv_recall_at_20_std": 0.4715815139054417, - "nauc_cv_recall_at_3_diff1": 0.3669705785900801, - "nauc_cv_recall_at_3_max": 0.5861063323460141, - "nauc_cv_recall_at_3_std": 0.5506084376331697, - "nauc_cv_recall_at_5_diff1": 0.33259049284662684, - "nauc_cv_recall_at_5_max": 0.5517516067458997, - "nauc_cv_recall_at_5_std": 0.5278314943049669, - "nauc_map_at_1000_diff1": 0.3787159713673133, - "nauc_map_at_1000_max": 0.5428421168181844, - "nauc_map_at_1000_std": 0.5131407062499223, - "nauc_map_at_100_diff1": 0.3880841736654061, - "nauc_map_at_100_max": 0.5539388799369834, - "nauc_map_at_100_std": 0.5200669870443412, - "nauc_map_at_10_diff1": 0.431272322825696, - "nauc_map_at_10_max": 0.5984353858644095, - "nauc_map_at_10_std": 0.5422788490060564, - "nauc_map_at_1_diff1": 0.6077745704791587, - "nauc_map_at_1_max": 0.6941858785164394, - "nauc_map_at_1_std": 0.5627334833130381, - "nauc_map_at_20_diff1": 0.41359649406159166, - "nauc_map_at_20_max": 0.5820965759060719, - "nauc_map_at_20_std": 0.5346552811096857, - "nauc_map_at_3_diff1": 0.47600932948256175, - "nauc_map_at_3_max": 0.6379062326699059, - "nauc_map_at_3_std": 0.5565259241985268, - "nauc_map_at_5_diff1": 0.45640314603896, - "nauc_map_at_5_max": 0.6221316525944712, - "nauc_map_at_5_std": 0.5488202814490963, - "nauc_mrr_at_1000_diff1": 0.37871597713247773, - "nauc_mrr_at_1000_max": 0.5428421179864462, - "nauc_mrr_at_1000_std": 0.5131406973059689, - "nauc_mrr_at_100_diff1": 0.3880841736654061, - "nauc_mrr_at_100_max": 0.5539388799369834, - "nauc_mrr_at_100_std": 0.5200669870443412, - "nauc_mrr_at_10_diff1": 0.431272322825696, - "nauc_mrr_at_10_max": 0.5984353858644095, - "nauc_mrr_at_10_std": 0.5422788490060564, - "nauc_mrr_at_1_diff1": 0.6077745704791587, - "nauc_mrr_at_1_max": 0.6941858785164394, - "nauc_mrr_at_1_std": 0.5627334833130381, - "nauc_mrr_at_20_diff1": 0.41359649406159166, - "nauc_mrr_at_20_max": 0.5820965759060719, - "nauc_mrr_at_20_std": 0.5346552811096857, - "nauc_mrr_at_3_diff1": 0.47600932948256175, - "nauc_mrr_at_3_max": 0.6379062326699059, - "nauc_mrr_at_3_std": 0.5565259241985268, - "nauc_mrr_at_5_diff1": 0.45640314603896, - "nauc_mrr_at_5_max": 0.6221316525944712, - "nauc_mrr_at_5_std": 0.5488202814490963, - "nauc_ndcg_at_1000_diff1": 0.15985113642093954, - "nauc_ndcg_at_1000_max": 0.3001669091845696, - "nauc_ndcg_at_1000_std": 0.3763836071659746, - "nauc_ndcg_at_100_diff1": 0.23393916616983274, - "nauc_ndcg_at_100_max": 0.40375379090698243, - "nauc_ndcg_at_100_std": 0.450200694745624, - "nauc_ndcg_at_10_diff1": 0.36912578430047643, - "nauc_ndcg_at_10_max": 0.5538603225353859, - "nauc_ndcg_at_10_std": 0.5296354727215236, - "nauc_ndcg_at_1_diff1": 0.6077745704791587, - "nauc_ndcg_at_1_max": 0.6941858785164394, - "nauc_ndcg_at_1_std": 0.5627334833130381, - "nauc_ndcg_at_20_diff1": 0.32493358843185, - "nauc_ndcg_at_20_max": 0.5134958869109029, - "nauc_ndcg_at_20_std": 0.5102637742103265, - "nauc_ndcg_at_3_diff1": 0.4423071127667642, - "nauc_ndcg_at_3_max": 0.6220036053763677, - "nauc_ndcg_at_3_std": 0.5547130370697566, - "nauc_ndcg_at_5_diff1": 0.41467932527755136, - "nauc_ndcg_at_5_max": 0.5987024010782739, - "nauc_ndcg_at_5_std": 0.5420809148948331, - "nauc_precision_at_1000_diff1": 0.022292591165604964, - "nauc_precision_at_1000_max": 0.13098130226865767, - "nauc_precision_at_1000_std": 0.2720168619540827, - "nauc_precision_at_100_diff1": 0.0950188477242359, - "nauc_precision_at_100_max": 0.2599535722147032, - "nauc_precision_at_100_std": 0.3801763208649802, - "nauc_precision_at_10_diff1": 0.2632091816890364, - "nauc_precision_at_10_max": 0.475056314387156, - "nauc_precision_at_10_std": 0.5066786887223551, - "nauc_precision_at_1_diff1": 0.6077745704791587, - "nauc_precision_at_1_max": 0.6941858785164394, - "nauc_precision_at_1_std": 0.5627334833130381, - "nauc_precision_at_20_diff1": 0.19312636886079576, - "nauc_precision_at_20_max": 0.4087488534328376, - "nauc_precision_at_20_std": 0.47158151390544195, - "nauc_precision_at_3_diff1": 0.3669705785900801, - "nauc_precision_at_3_max": 0.5861063323460142, - "nauc_precision_at_3_std": 0.5506084376331697, - "nauc_precision_at_5_diff1": 0.33259049284662695, - "nauc_precision_at_5_max": 0.5517516067459, - "nauc_precision_at_5_std": 0.5278314943049669, - "nauc_recall_at_1000_diff1": 0.022292591165605398, - "nauc_recall_at_1000_max": 0.13098130226865806, - "nauc_recall_at_1000_std": 0.2720168619540829, - "nauc_recall_at_100_diff1": 0.09501884772423592, - "nauc_recall_at_100_max": 0.2599535722147032, - "nauc_recall_at_100_std": 0.3801763208649803, - "nauc_recall_at_10_diff1": 0.2632091816890365, - "nauc_recall_at_10_max": 0.47505631438715595, - "nauc_recall_at_10_std": 0.506678688722355, - "nauc_recall_at_1_diff1": 0.6077745704791587, - "nauc_recall_at_1_max": 0.6941858785164394, - "nauc_recall_at_1_std": 0.5627334833130381, - "nauc_recall_at_20_diff1": 0.19312636886079568, - "nauc_recall_at_20_max": 0.40874885343283757, - "nauc_recall_at_20_std": 0.4715815139054417, - "nauc_recall_at_3_diff1": 0.3669705785900801, - "nauc_recall_at_3_max": 0.5861063323460141, - "nauc_recall_at_3_std": 0.5506084376331697, - "nauc_recall_at_5_diff1": 0.33259049284662684, - "nauc_recall_at_5_max": 0.5517516067458997, - "nauc_recall_at_5_std": 0.5278314943049669, - "ndcg_at_1": 0.01264, - "ndcg_at_10": 0.0227, - "ndcg_at_100": 0.03713, - "ndcg_at_1000": 0.07513, - "ndcg_at_20": 0.02568, - "ndcg_at_3": 0.01754, - "ndcg_at_5": 0.01982, - "precision_at_1": 0.01264, - "precision_at_10": 0.00358, - "precision_at_100": 0.00112, - "precision_at_1000": 0.00044, - "precision_at_20": 0.00239, - "precision_at_3": 0.00708, - "precision_at_5": 0.00536, - "recall_at_1": 0.01264, - "recall_at_10": 0.03583, - "recall_at_100": 0.11236, - "recall_at_1000": 0.44097, - "recall_at_20": 0.04778, - "recall_at_3": 0.02125, - "recall_at_5": 0.02681 - }, - { - "cv_recall_at_1": 0.01452, - "cv_recall_at_10": 0.04853, - "cv_recall_at_100": 0.13563, - "cv_recall_at_1000": 0.44546, - "cv_recall_at_20": 0.06622, - "cv_recall_at_3": 0.02696, - "cv_recall_at_5": 0.03567, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.02939, - "map_at_1": 0.01452, - "map_at_10": 0.02354, - "map_at_100": 0.02629, - "map_at_1000": 0.02721, - "map_at_20": 0.02477, - "map_at_3": 0.01993, - "map_at_5": 0.02188, - "mrr_at_1": 0.014516798009124844, - "mrr_at_10": 0.02354140743813079, - "mrr_at_100": 0.02629381239415381, - "mrr_at_1000": 0.027214393245717273, - "mrr_at_20": 0.024767747558012896, - "mrr_at_3": 0.01993179409189363, - "mrr_at_5": 0.021881192681690413, - "nauc_cv_recall_at_1000_diff1": 0.012975759729651388, - "nauc_cv_recall_at_1000_max": 0.17599164905484435, - "nauc_cv_recall_at_1000_std": 0.29982526861184217, - "nauc_cv_recall_at_100_diff1": 0.04097170057080594, - "nauc_cv_recall_at_100_max": 0.2975198043678887, - "nauc_cv_recall_at_100_std": 0.4599486909929725, - "nauc_cv_recall_at_10_diff1": 0.15247718510875016, - "nauc_cv_recall_at_10_max": 0.4477304911588335, - "nauc_cv_recall_at_10_std": 0.5574011165041363, - "nauc_cv_recall_at_1_diff1": 0.37003413985130795, - "nauc_cv_recall_at_1_max": 0.5920701629678448, - "nauc_cv_recall_at_1_std": 0.6075379204919302, - "nauc_cv_recall_at_20_diff1": 0.10275027990254477, - "nauc_cv_recall_at_20_max": 0.3951312064574037, - "nauc_cv_recall_at_20_std": 0.5439696855040402, - "nauc_cv_recall_at_3_diff1": 0.26504280766216726, - "nauc_cv_recall_at_3_max": 0.5667941191367358, - "nauc_cv_recall_at_3_std": 0.576336612293182, - "nauc_cv_recall_at_5_diff1": 0.22435904340469076, - "nauc_cv_recall_at_5_max": 0.5158004698221501, - "nauc_cv_recall_at_5_std": 0.5663764917385143, - "nauc_map_at_1000_diff1": 0.23186302755386246, - "nauc_map_at_1000_max": 0.5002654710152582, - "nauc_map_at_1000_std": 0.561532936580193, - "nauc_map_at_100_diff1": 0.23567961199945364, - "nauc_map_at_100_max": 0.5068296025584254, - "nauc_map_at_100_std": 0.5675416011527985, - "nauc_map_at_10_diff1": 0.26566349671711437, - "nauc_map_at_10_max": 0.5390175619472563, - "nauc_map_at_10_std": 0.5826401711776862, - "nauc_map_at_1_diff1": 0.37003413985130795, - "nauc_map_at_1_max": 0.5920701629678448, - "nauc_map_at_1_std": 0.6075379204919302, - "nauc_map_at_20_diff1": 0.2504929196056003, - "nauc_map_at_20_max": 0.5243658129963331, - "nauc_map_at_20_std": 0.5787047908123439, - "nauc_map_at_3_diff1": 0.30807275588101796, - "nauc_map_at_3_max": 0.5785385671839364, - "nauc_map_at_3_std": 0.5898595966081458, - "nauc_map_at_5_diff1": 0.2890780673741935, - "nauc_map_at_5_max": 0.5587200102194135, - "nauc_map_at_5_std": 0.5851700030151188, - "nauc_mrr_at_1000_diff1": 0.23186302528567013, - "nauc_mrr_at_1000_max": 0.5002654647368839, - "nauc_mrr_at_1000_std": 0.561532932436037, - "nauc_mrr_at_100_diff1": 0.23567961199945364, - "nauc_mrr_at_100_max": 0.5068296025584254, - "nauc_mrr_at_100_std": 0.5675416011527985, - "nauc_mrr_at_10_diff1": 0.26566349671711437, - "nauc_mrr_at_10_max": 0.5390175619472563, - "nauc_mrr_at_10_std": 0.5826401711776862, - "nauc_mrr_at_1_diff1": 0.37003413985130795, - "nauc_mrr_at_1_max": 0.5920701629678448, - "nauc_mrr_at_1_std": 0.6075379204919302, - "nauc_mrr_at_20_diff1": 0.2504929196056003, - "nauc_mrr_at_20_max": 0.5243658129963331, - "nauc_mrr_at_20_std": 0.5787047908123439, - "nauc_mrr_at_3_diff1": 0.30807275588101796, - "nauc_mrr_at_3_max": 0.5785385671839364, - "nauc_mrr_at_3_std": 0.5898595966081458, - "nauc_mrr_at_5_diff1": 0.2890780673741935, - "nauc_mrr_at_5_max": 0.5587200102194135, - "nauc_mrr_at_5_std": 0.5851700030151188, - "nauc_ndcg_at_1000_diff1": 0.1054369793354991, - "nauc_ndcg_at_1000_max": 0.3286790257113219, - "nauc_ndcg_at_1000_std": 0.4337873128983138, - "nauc_ndcg_at_100_diff1": 0.13702123082185066, - "nauc_ndcg_at_100_max": 0.404377005984407, - "nauc_ndcg_at_100_std": 0.5170717047904392, - "nauc_ndcg_at_10_diff1": 0.2230863223743629, - "nauc_ndcg_at_10_max": 0.5055905424621192, - "nauc_ndcg_at_10_std": 0.5731628679097766, - "nauc_ndcg_at_1_diff1": 0.37003413985130795, - "nauc_ndcg_at_1_max": 0.5920701629678448, - "nauc_ndcg_at_1_std": 0.6075379204919302, - "nauc_ndcg_at_20_diff1": 0.1888436713819155, - "nauc_ndcg_at_20_max": 0.47161431523098796, - "nauc_ndcg_at_20_std": 0.5642877406496789, - "nauc_ndcg_at_3_diff1": 0.2943800217921929, - "nauc_ndcg_at_3_max": 0.5748548662193177, - "nauc_ndcg_at_3_std": 0.585584818460011, - "nauc_ndcg_at_5_diff1": 0.26661831436553673, - "nauc_ndcg_at_5_max": 0.5442655392840073, - "nauc_ndcg_at_5_std": 0.5786701870420777, - "nauc_precision_at_1000_diff1": 0.012975759729651133, - "nauc_precision_at_1000_max": 0.17599164905484405, - "nauc_precision_at_1000_std": 0.29982526861184183, - "nauc_precision_at_100_diff1": 0.040971700570806076, - "nauc_precision_at_100_max": 0.29751980436788894, - "nauc_precision_at_100_std": 0.4599486909929728, - "nauc_precision_at_10_diff1": 0.15247718510875025, - "nauc_precision_at_10_max": 0.4477304911588337, - "nauc_precision_at_10_std": 0.5574011165041366, - "nauc_precision_at_1_diff1": 0.37003413985130795, - "nauc_precision_at_1_max": 0.5920701629678448, - "nauc_precision_at_1_std": 0.6075379204919302, - "nauc_precision_at_20_diff1": 0.1027502799025449, - "nauc_precision_at_20_max": 0.3951312064574038, - "nauc_precision_at_20_std": 0.5439696855040405, - "nauc_precision_at_3_diff1": 0.26504280766216703, - "nauc_precision_at_3_max": 0.5667941191367358, - "nauc_precision_at_3_std": 0.5763366122931819, - "nauc_precision_at_5_diff1": 0.2243590434046909, - "nauc_precision_at_5_max": 0.5158004698221502, - "nauc_precision_at_5_std": 0.5663764917385142, - "nauc_recall_at_1000_diff1": 0.012975759729651388, - "nauc_recall_at_1000_max": 0.17599164905484435, - "nauc_recall_at_1000_std": 0.29982526861184217, - "nauc_recall_at_100_diff1": 0.04097170057080594, - "nauc_recall_at_100_max": 0.2975198043678887, - "nauc_recall_at_100_std": 0.4599486909929725, - "nauc_recall_at_10_diff1": 0.15247718510875016, - "nauc_recall_at_10_max": 0.4477304911588335, - "nauc_recall_at_10_std": 0.5574011165041363, - "nauc_recall_at_1_diff1": 0.37003413985130795, - "nauc_recall_at_1_max": 0.5920701629678448, - "nauc_recall_at_1_std": 0.6075379204919302, - "nauc_recall_at_20_diff1": 0.10275027990254477, - "nauc_recall_at_20_max": 0.3951312064574037, - "nauc_recall_at_20_std": 0.5439696855040402, - "nauc_recall_at_3_diff1": 0.26504280766216726, - "nauc_recall_at_3_max": 0.5667941191367358, - "nauc_recall_at_3_std": 0.576336612293182, - "nauc_recall_at_5_diff1": 0.22435904340469076, - "nauc_recall_at_5_max": 0.5158004698221501, - "nauc_recall_at_5_std": 0.5663764917385143, - "ndcg_at_1": 0.01452, - "ndcg_at_10": 0.02939, - "ndcg_at_100": 0.04623, - "ndcg_at_1000": 0.08222, - "ndcg_at_20": 0.03386, - "ndcg_at_3": 0.02173, - "ndcg_at_5": 0.02529, - "precision_at_1": 0.01452, - "precision_at_10": 0.00485, - "precision_at_100": 0.00136, - "precision_at_1000": 0.00045, - "precision_at_20": 0.00331, - "precision_at_3": 0.00899, - "precision_at_5": 0.00713, - "recall_at_1": 0.01452, - "recall_at_10": 0.04853, - "recall_at_100": 0.13563, - "recall_at_1000": 0.44546, - "recall_at_20": 0.06622, - "recall_at_3": 0.02696, - "recall_at_5": 0.03567 - }, - { - "cv_recall_at_1": 0.00236, - "cv_recall_at_10": 0.01067, - "cv_recall_at_100": 0.05295, - "cv_recall_at_1000": 0.33638, - "cv_recall_at_20": 0.01733, - "cv_recall_at_3": 0.00513, - "cv_recall_at_5": 0.00693, - "hf_subset": "uk", - "languages": [ - "ukr-Cyrl" - ], - "main_score": 0.00586, - "map_at_1": 0.00236, - "map_at_10": 0.00441, - "map_at_100": 0.00555, - "map_at_1000": 0.00632, - "map_at_20": 0.00486, - "map_at_3": 0.00351, - "map_at_5": 0.00393, - "mrr_at_1": 0.002356202356202356, - "mrr_at_10": 0.004410014410014408, - "mrr_at_100": 0.005552806872394621, - "mrr_at_1000": 0.006320104674748412, - "mrr_at_20": 0.0048556029825379664, - "mrr_at_3": 0.00351120351120351, - "mrr_at_5": 0.003927003927003925, - "nauc_cv_recall_at_1000_diff1": -0.08406797065643612, - "nauc_cv_recall_at_1000_max": -0.001525952193718598, - "nauc_cv_recall_at_1000_std": 0.14984248217962143, - "nauc_cv_recall_at_100_diff1": -0.07298074982572675, - "nauc_cv_recall_at_100_max": 0.04679622553449316, - "nauc_cv_recall_at_100_std": 0.2517498038656578, - "nauc_cv_recall_at_10_diff1": 0.08726660901806763, - "nauc_cv_recall_at_10_max": 0.2364171653297631, - "nauc_cv_recall_at_10_std": 0.5057754862103674, - "nauc_cv_recall_at_1_diff1": 0.15584461979096606, - "nauc_cv_recall_at_1_max": 0.28341285856642234, - "nauc_cv_recall_at_1_std": 0.5876382323289251, - "nauc_cv_recall_at_20_diff1": -0.012332708752283775, - "nauc_cv_recall_at_20_max": 0.1682570311545627, - "nauc_cv_recall_at_20_std": 0.42095305867101396, - "nauc_cv_recall_at_3_diff1": 0.14922454775842056, - "nauc_cv_recall_at_3_max": 0.3453267670397321, - "nauc_cv_recall_at_3_std": 0.616668865370652, - "nauc_cv_recall_at_5_diff1": 0.11501170879346388, - "nauc_cv_recall_at_5_max": 0.33071772697417046, - "nauc_cv_recall_at_5_std": 0.5981901963197561, - "nauc_map_at_1000_diff1": 0.0569558703850738, - "nauc_map_at_1000_max": 0.22432832656300244, - "nauc_map_at_1000_std": 0.4825080785510095, - "nauc_map_at_100_diff1": 0.07037303142604615, - "nauc_map_at_100_max": 0.2451547614749494, - "nauc_map_at_100_std": 0.5161914580691662, - "nauc_map_at_10_diff1": 0.12231154578051895, - "nauc_map_at_10_max": 0.300438045334167, - "nauc_map_at_10_std": 0.5873229977719833, - "nauc_map_at_1_diff1": 0.15584461979096606, - "nauc_map_at_1_max": 0.28341285856642234, - "nauc_map_at_1_std": 0.5876382323289251, - "nauc_map_at_20_diff1": 0.09475071247578104, - "nauc_map_at_20_max": 0.2790261663013074, - "nauc_map_at_20_std": 0.5613443798754721, - "nauc_map_at_3_diff1": 0.14692091393411147, - "nauc_map_at_3_max": 0.33111931125429245, - "nauc_map_at_3_std": 0.6209682999618753, - "nauc_map_at_5_diff1": 0.13374610850404176, - "nauc_map_at_5_max": 0.32939615249119725, - "nauc_map_at_5_std": 0.6172151637277775, - "nauc_mrr_at_1000_diff1": 0.05695589575990401, - "nauc_mrr_at_1000_max": 0.22432832213015588, - "nauc_mrr_at_1000_std": 0.48250801343703764, - "nauc_mrr_at_100_diff1": 0.07037303142604615, - "nauc_mrr_at_100_max": 0.2451547614749494, - "nauc_mrr_at_100_std": 0.5161914580691662, - "nauc_mrr_at_10_diff1": 0.12231154578051895, - "nauc_mrr_at_10_max": 0.300438045334167, - "nauc_mrr_at_10_std": 0.5873229977719833, - "nauc_mrr_at_1_diff1": 0.15584461979096606, - "nauc_mrr_at_1_max": 0.28341285856642234, - "nauc_mrr_at_1_std": 0.5876382323289251, - "nauc_mrr_at_20_diff1": 0.09475071247578104, - "nauc_mrr_at_20_max": 0.2790261663013074, - "nauc_mrr_at_20_std": 0.5613443798754721, - "nauc_mrr_at_3_diff1": 0.14692091393411147, - "nauc_mrr_at_3_max": 0.33111931125429245, - "nauc_mrr_at_3_std": 0.6209682999618753, - "nauc_mrr_at_5_diff1": 0.13374610850404176, - "nauc_mrr_at_5_max": 0.32939615249119725, - "nauc_mrr_at_5_std": 0.6172151637277775, - "nauc_ndcg_at_1000_diff1": -0.058503630663267324, - "nauc_ndcg_at_1000_max": 0.04506885875721012, - "nauc_ndcg_at_1000_std": 0.2191973371313843, - "nauc_ndcg_at_100_diff1": -0.02062968232376607, - "nauc_ndcg_at_100_max": 0.12258667087293763, - "nauc_ndcg_at_100_std": 0.35313324716473754, - "nauc_ndcg_at_10_diff1": 0.10804545321905491, - "nauc_ndcg_at_10_max": 0.2762775018336161, - "nauc_ndcg_at_10_std": 0.5558309811414158, - "nauc_ndcg_at_1_diff1": 0.15584461979096606, - "nauc_ndcg_at_1_max": 0.28341285856642234, - "nauc_ndcg_at_1_std": 0.5876382323289251, - "nauc_ndcg_at_20_diff1": 0.0457669467001654, - "nauc_ndcg_at_20_max": 0.2290101438279385, - "nauc_ndcg_at_20_std": 0.4974691435985739, - "nauc_ndcg_at_3_diff1": 0.14751080948533413, - "nauc_ndcg_at_3_max": 0.3362208528847951, - "nauc_ndcg_at_3_std": 0.6201048482107948, - "nauc_ndcg_at_5_diff1": 0.12715063122699638, - "nauc_ndcg_at_5_max": 0.3307088007409043, - "nauc_ndcg_at_5_std": 0.6113230989942977, - "nauc_precision_at_1000_diff1": -0.08406797065643624, - "nauc_precision_at_1000_max": -0.0015259521937187298, - "nauc_precision_at_1000_std": 0.14984248217962123, - "nauc_precision_at_100_diff1": -0.07298074982572668, - "nauc_precision_at_100_max": 0.04679622553449324, - "nauc_precision_at_100_std": 0.2517498038656581, - "nauc_precision_at_10_diff1": 0.08726660901806796, - "nauc_precision_at_10_max": 0.2364171653297635, - "nauc_precision_at_10_std": 0.5057754862103677, - "nauc_precision_at_1_diff1": 0.15584461979096606, - "nauc_precision_at_1_max": 0.28341285856642234, - "nauc_precision_at_1_std": 0.5876382323289251, - "nauc_precision_at_20_diff1": -0.012332708752283828, - "nauc_precision_at_20_max": 0.16825703115456306, - "nauc_precision_at_20_std": 0.4209530586710141, - "nauc_precision_at_3_diff1": 0.14922454775842028, - "nauc_precision_at_3_max": 0.3453267670397317, - "nauc_precision_at_3_std": 0.6166688653706518, - "nauc_precision_at_5_diff1": 0.11501170879346426, - "nauc_precision_at_5_max": 0.33071772697417073, - "nauc_precision_at_5_std": 0.5981901963197568, - "nauc_recall_at_1000_diff1": -0.08406797065643612, - "nauc_recall_at_1000_max": -0.001525952193718598, - "nauc_recall_at_1000_std": 0.14984248217962143, - "nauc_recall_at_100_diff1": -0.07298074982572675, - "nauc_recall_at_100_max": 0.04679622553449316, - "nauc_recall_at_100_std": 0.2517498038656578, - "nauc_recall_at_10_diff1": 0.08726660901806763, - "nauc_recall_at_10_max": 0.2364171653297631, - "nauc_recall_at_10_std": 0.5057754862103674, - "nauc_recall_at_1_diff1": 0.15584461979096606, - "nauc_recall_at_1_max": 0.28341285856642234, - "nauc_recall_at_1_std": 0.5876382323289251, - "nauc_recall_at_20_diff1": -0.012332708752283775, - "nauc_recall_at_20_max": 0.1682570311545627, - "nauc_recall_at_20_std": 0.42095305867101396, - "nauc_recall_at_3_diff1": 0.14922454775842056, - "nauc_recall_at_3_max": 0.3453267670397321, - "nauc_recall_at_3_std": 0.616668865370652, - "nauc_recall_at_5_diff1": 0.11501170879346388, - "nauc_recall_at_5_max": 0.33071772697417046, - "nauc_recall_at_5_std": 0.5981901963197561, - "ndcg_at_1": 0.00236, - "ndcg_at_10": 0.00586, - "ndcg_at_100": 0.01369, - "ndcg_at_1000": 0.04614, - "ndcg_at_20": 0.00752, - "ndcg_at_3": 0.00392, - "ndcg_at_5": 0.00467, - "precision_at_1": 0.00236, - "precision_at_10": 0.00107, - "precision_at_100": 0.00053, - "precision_at_1000": 0.00034, - "precision_at_20": 0.00087, - "precision_at_3": 0.00171, - "precision_at_5": 0.00139, - "recall_at_1": 0.00236, - "recall_at_10": 0.01067, - "recall_at_100": 0.05295, - "recall_at_1000": 0.33638, - "recall_at_20": 0.01733, - "recall_at_3": 0.00513, - "recall_at_5": 0.00693 - }, - { - "cv_recall_at_1": 0.00857, - "cv_recall_at_10": 0.03007, - "cv_recall_at_100": 0.07469, - "cv_recall_at_1000": 0.34544, - "cv_recall_at_20": 0.03905, - "cv_recall_at_3": 0.0166, - "cv_recall_at_5": 0.02095, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.01787, - "map_at_1": 0.00857, - "map_at_10": 0.01415, - "map_at_100": 0.01555, - "map_at_1000": 0.01628, - "map_at_20": 0.01476, - "map_at_3": 0.01195, - "map_at_5": 0.01294, - "mrr_at_1": 0.008571428571428572, - "mrr_at_10": 0.014153925062088327, - "mrr_at_100": 0.015554419065062603, - "mrr_at_1000": 0.016277461294299248, - "mrr_at_20": 0.014763666787850148, - "mrr_at_3": 0.011950113378684804, - "mrr_at_5": 0.012936507936507937, - "nauc_cv_recall_at_1000_diff1": 0.05577697623272914, - "nauc_cv_recall_at_1000_max": 0.1081174324422234, - "nauc_cv_recall_at_1000_std": 0.2069110458185291, - "nauc_cv_recall_at_100_diff1": 0.18158341409582918, - "nauc_cv_recall_at_100_max": 0.3224112137931103, - "nauc_cv_recall_at_100_std": 0.5094058372834869, - "nauc_cv_recall_at_10_diff1": 0.3869245702450141, - "nauc_cv_recall_at_10_max": 0.5598828167208352, - "nauc_cv_recall_at_10_std": 0.8236957411186805, - "nauc_cv_recall_at_1_diff1": 0.609533179160225, - "nauc_cv_recall_at_1_max": 0.6525099978279478, - "nauc_cv_recall_at_1_std": 0.9003666934978981, - "nauc_cv_recall_at_20_diff1": 0.3237865879367905, - "nauc_cv_recall_at_20_max": 0.5040690918235797, - "nauc_cv_recall_at_20_std": 0.7479556377839834, - "nauc_cv_recall_at_3_diff1": 0.42102380941908174, - "nauc_cv_recall_at_3_max": 0.5894469687306189, - "nauc_cv_recall_at_3_std": 0.8605149837033257, - "nauc_cv_recall_at_5_diff1": 0.43539074287665624, - "nauc_cv_recall_at_5_max": 0.5926827398520448, - "nauc_cv_recall_at_5_std": 0.8704922659846193, - "nauc_map_at_1000_diff1": 0.4336332621953282, - "nauc_map_at_1000_max": 0.5629566075355275, - "nauc_map_at_1000_std": 0.8006440442716728, - "nauc_map_at_100_diff1": 0.4465754563450948, - "nauc_map_at_100_max": 0.5796866198941129, - "nauc_map_at_100_std": 0.8225647482382251, - "nauc_map_at_10_diff1": 0.4852007113910197, - "nauc_map_at_10_max": 0.6142575371011583, - "nauc_map_at_10_std": 0.8658948991412293, - "nauc_map_at_1_diff1": 0.609533179160225, - "nauc_map_at_1_max": 0.6525099978279478, - "nauc_map_at_1_std": 0.9003666934978981, - "nauc_map_at_20_diff1": 0.46978800543511556, - "nauc_map_at_20_max": 0.6036917861122867, - "nauc_map_at_20_std": 0.8516885712299876, - "nauc_map_at_3_diff1": 0.5060955772253246, - "nauc_map_at_3_max": 0.6266019187130784, - "nauc_map_at_3_std": 0.8757357587001666, - "nauc_map_at_5_diff1": 0.5043082586342598, - "nauc_map_at_5_max": 0.6267534421460095, - "nauc_map_at_5_std": 0.8788762487633768, - "nauc_mrr_at_1000_diff1": 0.4336332621953282, - "nauc_mrr_at_1000_max": 0.5629566075355275, - "nauc_mrr_at_1000_std": 0.8006440442716728, - "nauc_mrr_at_100_diff1": 0.4465754563450948, - "nauc_mrr_at_100_max": 0.5796866198941129, - "nauc_mrr_at_100_std": 0.8225647482382251, - "nauc_mrr_at_10_diff1": 0.4852007113910197, - "nauc_mrr_at_10_max": 0.6142575371011583, - "nauc_mrr_at_10_std": 0.8658948991412293, - "nauc_mrr_at_1_diff1": 0.609533179160225, - "nauc_mrr_at_1_max": 0.6525099978279478, - "nauc_mrr_at_1_std": 0.9003666934978981, - "nauc_mrr_at_20_diff1": 0.46978800543511556, - "nauc_mrr_at_20_max": 0.6036917861122867, - "nauc_mrr_at_20_std": 0.8516885712299876, - "nauc_mrr_at_3_diff1": 0.5060955772253246, - "nauc_mrr_at_3_max": 0.6266019187130784, - "nauc_mrr_at_3_std": 0.8757357587001666, - "nauc_mrr_at_5_diff1": 0.5043082586342598, - "nauc_mrr_at_5_max": 0.6267534421460095, - "nauc_mrr_at_5_std": 0.8788762487633768, - "nauc_ndcg_at_1000_diff1": 0.19463394578176046, - "nauc_ndcg_at_1000_max": 0.2847639002989628, - "nauc_ndcg_at_1000_std": 0.441408571736638, - "nauc_ndcg_at_100_diff1": 0.3161729621111146, - "nauc_ndcg_at_100_max": 0.4586921287973759, - "nauc_ndcg_at_100_std": 0.6779271906930151, - "nauc_ndcg_at_10_diff1": 0.44654032280824246, - "nauc_ndcg_at_10_max": 0.5934056697433198, - "nauc_ndcg_at_10_std": 0.8502453582865719, - "nauc_ndcg_at_1_diff1": 0.609533179160225, - "nauc_ndcg_at_1_max": 0.6525099978279478, - "nauc_ndcg_at_1_std": 0.9003666934978981, - "nauc_ndcg_at_20_diff1": 0.40910476157401005, - "nauc_ndcg_at_20_max": 0.56404265530875, - "nauc_ndcg_at_20_std": 0.8114472859077251, - "nauc_ndcg_at_3_diff1": 0.4788979735449278, - "nauc_ndcg_at_3_max": 0.6150215956603844, - "nauc_ndcg_at_3_std": 0.8707748731872657, - "nauc_ndcg_at_5_diff1": 0.4798110373153733, - "nauc_ndcg_at_5_max": 0.6151902535439208, - "nauc_ndcg_at_5_std": 0.8757788094218073, - "nauc_precision_at_1000_diff1": 0.05577697623272853, - "nauc_precision_at_1000_max": 0.10811743244222277, - "nauc_precision_at_1000_std": 0.2069110458185286, - "nauc_precision_at_100_diff1": 0.18158341409582915, - "nauc_precision_at_100_max": 0.3224112137931104, - "nauc_precision_at_100_std": 0.5094058372834871, - "nauc_precision_at_10_diff1": 0.38692457024501414, - "nauc_precision_at_10_max": 0.5598828167208352, - "nauc_precision_at_10_std": 0.8236957411186803, - "nauc_precision_at_1_diff1": 0.609533179160225, - "nauc_precision_at_1_max": 0.6525099978279478, - "nauc_precision_at_1_std": 0.9003666934978981, - "nauc_precision_at_20_diff1": 0.32378658793679044, - "nauc_precision_at_20_max": 0.5040690918235794, - "nauc_precision_at_20_std": 0.7479556377839834, - "nauc_precision_at_3_diff1": 0.42102380941908174, - "nauc_precision_at_3_max": 0.5894469687306185, - "nauc_precision_at_3_std": 0.8605149837033252, - "nauc_precision_at_5_diff1": 0.4353907428766564, - "nauc_precision_at_5_max": 0.5926827398520451, - "nauc_precision_at_5_std": 0.8704922659846194, - "nauc_recall_at_1000_diff1": 0.05577697623272914, - "nauc_recall_at_1000_max": 0.1081174324422234, - "nauc_recall_at_1000_std": 0.2069110458185291, - "nauc_recall_at_100_diff1": 0.18158341409582918, - "nauc_recall_at_100_max": 0.3224112137931103, - "nauc_recall_at_100_std": 0.5094058372834869, - "nauc_recall_at_10_diff1": 0.3869245702450141, - "nauc_recall_at_10_max": 0.5598828167208352, - "nauc_recall_at_10_std": 0.8236957411186805, - "nauc_recall_at_1_diff1": 0.609533179160225, - "nauc_recall_at_1_max": 0.6525099978279478, - "nauc_recall_at_1_std": 0.9003666934978981, - "nauc_recall_at_20_diff1": 0.3237865879367905, - "nauc_recall_at_20_max": 0.5040690918235797, - "nauc_recall_at_20_std": 0.7479556377839834, - "nauc_recall_at_3_diff1": 0.42102380941908174, - "nauc_recall_at_3_max": 0.5894469687306189, - "nauc_recall_at_3_std": 0.8605149837033257, - "nauc_recall_at_5_diff1": 0.43539074287665624, - "nauc_recall_at_5_max": 0.5926827398520448, - "nauc_recall_at_5_std": 0.8704922659846193, - "ndcg_at_1": 0.00857, - "ndcg_at_10": 0.01787, - "ndcg_at_100": 0.02649, - "ndcg_at_1000": 0.0574, - "ndcg_at_20": 0.02013, - "ndcg_at_3": 0.01314, - "ndcg_at_5": 0.01492, - "precision_at_1": 0.00857, - "precision_at_10": 0.00301, - "precision_at_100": 0.00075, - "precision_at_1000": 0.00035, - "precision_at_20": 0.00195, - "precision_at_3": 0.00553, - "precision_at_5": 0.00419, - "recall_at_1": 0.00857, - "recall_at_10": 0.03007, - "recall_at_100": 0.07469, - "recall_at_1000": 0.34544, - "recall_at_20": 0.03905, - "recall_at_3": 0.0166, - "recall_at_5": 0.02095 - }, - { - "cv_recall_at_1": 0.00362, - "cv_recall_at_10": 0.01979, - "cv_recall_at_100": 0.11221, - "cv_recall_at_1000": 0.49582, - "cv_recall_at_20": 0.03415, - "cv_recall_at_3": 0.0085, - "cv_recall_at_5": 0.01213, - "hf_subset": "zh", - "languages": [ - "zho-Hans" - ], - "main_score": 0.0104, - "map_at_1": 0.00362, - "map_at_10": 0.00757, - "map_at_100": 0.01013, - "map_at_1000": 0.0113, - "map_at_20": 0.00851, - "map_at_3": 0.00574, - "map_at_5": 0.00656, - "mrr_at_1": 0.0036241984945637024, - "mrr_at_10": 0.007570050136958424, - "mrr_at_100": 0.010130797363797312, - "mrr_at_1000": 0.011298712371181118, - "mrr_at_20": 0.008506887845520186, - "mrr_at_3": 0.005738314283059196, - "mrr_at_5": 0.006560728556825576, - "nauc_cv_recall_at_1000_diff1": 0.06310131748808638, - "nauc_cv_recall_at_1000_max": 0.19115897801802068, - "nauc_cv_recall_at_1000_std": 0.3494729582464983, - "nauc_cv_recall_at_100_diff1": 0.06462051707840712, - "nauc_cv_recall_at_100_max": 0.20466155072653078, - "nauc_cv_recall_at_100_std": 0.36358143329435333, - "nauc_cv_recall_at_10_diff1": 0.11464586708775552, - "nauc_cv_recall_at_10_max": 0.3011761761750302, - "nauc_cv_recall_at_10_std": 0.44072601631113906, - "nauc_cv_recall_at_1_diff1": 0.3314490401058548, - "nauc_cv_recall_at_1_max": 0.576029066981452, - "nauc_cv_recall_at_1_std": 0.5506135812699169, - "nauc_cv_recall_at_20_diff1": 0.09732639529181819, - "nauc_cv_recall_at_20_max": 0.2762623071349517, - "nauc_cv_recall_at_20_std": 0.4160697700973022, - "nauc_cv_recall_at_3_diff1": 0.20311571079691665, - "nauc_cv_recall_at_3_max": 0.4890582156975279, - "nauc_cv_recall_at_3_std": 0.5122791391441451, - "nauc_cv_recall_at_5_diff1": 0.16597217434441056, - "nauc_cv_recall_at_5_max": 0.38658520019136333, - "nauc_cv_recall_at_5_std": 0.48447201176385574, - "nauc_map_at_1000_diff1": 0.16156489121310788, - "nauc_map_at_1000_max": 0.36931173274986095, - "nauc_map_at_1000_std": 0.4596945612412111, - "nauc_map_at_100_diff1": 0.16719444274697842, - "nauc_map_at_100_max": 0.37904946654327765, - "nauc_map_at_100_std": 0.4657260813330009, - "nauc_map_at_10_diff1": 0.20219953418305472, - "nauc_map_at_10_max": 0.43623717234748305, - "nauc_map_at_10_std": 0.4999483552034321, - "nauc_map_at_1_diff1": 0.3314490401058548, - "nauc_map_at_1_max": 0.576029066981452, - "nauc_map_at_1_std": 0.5506135812699169, - "nauc_map_at_20_diff1": 0.18651020724728606, - "nauc_map_at_20_max": 0.41297186154186527, - "nauc_map_at_20_std": 0.4871155315109689, - "nauc_map_at_3_diff1": 0.24822480464649008, - "nauc_map_at_3_max": 0.5220554202941391, - "nauc_map_at_3_std": 0.5308037116682419, - "nauc_map_at_5_diff1": 0.22892372186142276, - "nauc_map_at_5_max": 0.4760368182138216, - "nauc_map_at_5_std": 0.5172233572837502, - "nauc_mrr_at_1000_diff1": 0.16156489121310788, - "nauc_mrr_at_1000_max": 0.36931173274986095, - "nauc_mrr_at_1000_std": 0.4596945612412111, - "nauc_mrr_at_100_diff1": 0.16719444274697842, - "nauc_mrr_at_100_max": 0.37904946654327765, - "nauc_mrr_at_100_std": 0.4657260813330009, - "nauc_mrr_at_10_diff1": 0.20219953418305472, - "nauc_mrr_at_10_max": 0.43623717234748305, - "nauc_mrr_at_10_std": 0.4999483552034321, - "nauc_mrr_at_1_diff1": 0.3314490401058548, - "nauc_mrr_at_1_max": 0.576029066981452, - "nauc_mrr_at_1_std": 0.5506135812699169, - "nauc_mrr_at_20_diff1": 0.18651020724728606, - "nauc_mrr_at_20_max": 0.41297186154186527, - "nauc_mrr_at_20_std": 0.4871155315109689, - "nauc_mrr_at_3_diff1": 0.24822480464649008, - "nauc_mrr_at_3_max": 0.5220554202941391, - "nauc_mrr_at_3_std": 0.5308037116682419, - "nauc_mrr_at_5_diff1": 0.22892372186142276, - "nauc_mrr_at_5_max": 0.4760368182138216, - "nauc_mrr_at_5_std": 0.5172233572837502, - "nauc_ndcg_at_1000_diff1": 0.08511107747492294, - "nauc_ndcg_at_1000_max": 0.23496356891655748, - "nauc_ndcg_at_1000_std": 0.37818549814470126, - "nauc_ndcg_at_100_diff1": 0.09804413340784086, - "nauc_ndcg_at_100_max": 0.26285200146189763, - "nauc_ndcg_at_100_std": 0.3985480067269638, - "nauc_ndcg_at_10_diff1": 0.16419473245341326, - "nauc_ndcg_at_10_max": 0.3787350408792725, - "nauc_ndcg_at_10_std": 0.47506805045248357, - "nauc_ndcg_at_1_diff1": 0.3314490401058548, - "nauc_ndcg_at_1_max": 0.576029066981452, - "nauc_ndcg_at_1_std": 0.5506135812699169, - "nauc_ndcg_at_20_diff1": 0.1399569918152696, - "nauc_ndcg_at_20_max": 0.34241103145134444, - "nauc_ndcg_at_20_std": 0.45143958624730546, - "nauc_ndcg_at_3_diff1": 0.23294671622092797, - "nauc_ndcg_at_3_max": 0.5109715994212655, - "nauc_ndcg_at_3_std": 0.524721159753844, - "nauc_ndcg_at_5_diff1": 0.20545348119843998, - "nauc_ndcg_at_5_max": 0.4433749890215097, - "nauc_ndcg_at_5_std": 0.5052378279790504, - "nauc_precision_at_1000_diff1": 0.0631013174880857, - "nauc_precision_at_1000_max": 0.19115897801802026, - "nauc_precision_at_1000_std": 0.34947295824649804, - "nauc_precision_at_100_diff1": 0.06462051707840721, - "nauc_precision_at_100_max": 0.20466155072653078, - "nauc_precision_at_100_std": 0.3635814332943533, - "nauc_precision_at_10_diff1": 0.11464586708775586, - "nauc_precision_at_10_max": 0.30117617617503045, - "nauc_precision_at_10_std": 0.44072601631113917, - "nauc_precision_at_1_diff1": 0.3314490401058548, - "nauc_precision_at_1_max": 0.576029066981452, - "nauc_precision_at_1_std": 0.5506135812699169, - "nauc_precision_at_20_diff1": 0.09732639529181827, - "nauc_precision_at_20_max": 0.2762623071349518, - "nauc_precision_at_20_std": 0.4160697700973024, - "nauc_precision_at_3_diff1": 0.20311571079691687, - "nauc_precision_at_3_max": 0.4890582156975278, - "nauc_precision_at_3_std": 0.5122791391441456, - "nauc_precision_at_5_diff1": 0.16597217434441086, - "nauc_precision_at_5_max": 0.386585200191364, - "nauc_precision_at_5_std": 0.48447201176385596, - "nauc_recall_at_1000_diff1": 0.06310131748808638, - "nauc_recall_at_1000_max": 0.19115897801802068, - "nauc_recall_at_1000_std": 0.3494729582464983, - "nauc_recall_at_100_diff1": 0.06462051707840712, - "nauc_recall_at_100_max": 0.20466155072653078, - "nauc_recall_at_100_std": 0.36358143329435333, - "nauc_recall_at_10_diff1": 0.11464586708775552, - "nauc_recall_at_10_max": 0.3011761761750302, - "nauc_recall_at_10_std": 0.44072601631113906, - "nauc_recall_at_1_diff1": 0.3314490401058548, - "nauc_recall_at_1_max": 0.576029066981452, - "nauc_recall_at_1_std": 0.5506135812699169, - "nauc_recall_at_20_diff1": 0.09732639529181819, - "nauc_recall_at_20_max": 0.2762623071349517, - "nauc_recall_at_20_std": 0.4160697700973022, - "nauc_recall_at_3_diff1": 0.20311571079691665, - "nauc_recall_at_3_max": 0.4890582156975279, - "nauc_recall_at_3_std": 0.5122791391441451, - "nauc_recall_at_5_diff1": 0.16597217434441056, - "nauc_recall_at_5_max": 0.38658520019136333, - "nauc_recall_at_5_std": 0.48447201176385574, - "ndcg_at_1": 0.00362, - "ndcg_at_10": 0.0104, - "ndcg_at_100": 0.02767, - "ndcg_at_1000": 0.07249, - "ndcg_at_20": 0.01395, - "ndcg_at_3": 0.00645, - "ndcg_at_5": 0.00793, - "precision_at_1": 0.00362, - "precision_at_10": 0.00198, - "precision_at_100": 0.00112, - "precision_at_1000": 0.0005, - "precision_at_20": 0.00171, - "precision_at_3": 0.00283, - "precision_at_5": 0.00243, - "recall_at_1": 0.00362, - "recall_at_10": 0.01979, - "recall_at_100": 0.11221, - "recall_at_1000": 0.49582, - "recall_at_20": 0.03415, - "recall_at_3": 0.0085, - "recall_at_5": 0.01213 - } - ] - }, - "task_name": "XM3600T2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json deleted file mode 100644 index 0b09a17fae..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "openai/clip-vit-base-patch32", "revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "CLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROCocoOrder.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROCocoOrder.json deleted file mode 100644 index 5d7cf611c6..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROCocoOrder.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "853ec8757226585a38a80886c51fe0f3f268787c", - "evaluation_time": 432.3775689601898, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.4538184702396393, - "text_acc": 0.4538184702396393 - } - ] - }, - "task_name": "AROCocoOrder" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROFlickrOrder.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROFlickrOrder.json deleted file mode 100644 index b67b51a5cd..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROFlickrOrder.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "1f9485f69c87947812378a1aedf86410c86a0aa8", - "evaluation_time": 84.40965294837952, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.5440000295639038, - "text_acc": 0.5440000295639038 - } - ] - }, - "task_name": "AROFlickrOrder" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualAttribution.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualAttribution.json deleted file mode 100644 index d8cd1fe3e8..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualAttribution.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "18f7e01358d91df599d723f00e16a18640e19398", - "evaluation_time": 570.5447673797607, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.616216778755188, - "text_acc": 0.616216778755188 - } - ] - }, - "task_name": "AROVisualAttribution" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualRelation.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualRelation.json deleted file mode 100644 index c92ce1a787..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/AROVisualRelation.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "3867ad4f46a1ac2e63be034d1fc77dd8c2ef7209", - "evaluation_time": 476.9429078102112, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.5326064229011536, - "text_acc": 0.5326064229011536 - } - ] - }, - "task_name": "AROVisualRelation" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CIRRIT2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CIRRIT2TRetrieval.json deleted file mode 100644 index c812f33795..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CIRRIT2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", - "evaluation_time": 334.5525403022766, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.09282, - "map_at_1": 0.00396, - "map_at_10": 0.0602, - "map_at_100": 0.07143, - "map_at_1000": 0.07288, - "map_at_20": 0.06567, - "map_at_3": 0.04097, - "map_at_5": 0.05122, - "mrr_at_1": 0.00815347721822542, - "mrr_at_10": 0.06053176506413904, - "mrr_at_100": 0.07175520833724132, - "mrr_at_1000": 0.07319771612683665, - "mrr_at_20": 0.06600739568371021, - "mrr_at_3": 0.04128697042366111, - "mrr_at_5": 0.05150279776179057, - "nauc_map_at_1000_diff1": -0.12353623400385476, - "nauc_map_at_1000_max": 0.0912971224497447, - "nauc_map_at_1000_std": 0.08525451600633958, - "nauc_map_at_100_diff1": -0.12289322987256437, - "nauc_map_at_100_max": 0.09144178386642453, - "nauc_map_at_100_std": 0.08296012542760436, - "nauc_map_at_10_diff1": -0.12494376185642647, - "nauc_map_at_10_max": 0.08674812682258538, - "nauc_map_at_10_std": 0.06449719501276227, - "nauc_map_at_1_diff1": -0.5463082664585215, - "nauc_map_at_1_max": 0.26114918977811374, - "nauc_map_at_1_std": 0.031162171757824993, - "nauc_map_at_20_diff1": -0.1211136576549465, - "nauc_map_at_20_max": 0.08760779427458243, - "nauc_map_at_20_std": 0.07233682566935275, - "nauc_map_at_3_diff1": -0.1519614939106193, - "nauc_map_at_3_max": 0.08140281531605219, - "nauc_map_at_3_std": 0.0465481293396245, - "nauc_map_at_5_diff1": -0.1306452402075584, - "nauc_map_at_5_max": 0.07638304265278853, - "nauc_map_at_5_std": 0.05717760292127242, - "nauc_mrr_at_1000_diff1": -0.12585046105692907, - "nauc_mrr_at_1000_max": 0.09142429242891584, - "nauc_mrr_at_1000_std": 0.08671902673955421, - "nauc_mrr_at_100_diff1": -0.1251947063565411, - "nauc_mrr_at_100_max": 0.09156836634919437, - "nauc_mrr_at_100_std": 0.08442756201074159, - "nauc_mrr_at_10_diff1": -0.12735264435622265, - "nauc_mrr_at_10_max": 0.08687944105221006, - "nauc_mrr_at_10_std": 0.06615580242111993, - "nauc_mrr_at_1_diff1": -0.5415016019823545, - "nauc_mrr_at_1_max": 0.25890560141416386, - "nauc_mrr_at_1_std": 0.014078505026910177, - "nauc_mrr_at_20_diff1": -0.12354316783713191, - "nauc_mrr_at_20_max": 0.0877260670527992, - "nauc_mrr_at_20_std": 0.07382572978906658, - "nauc_mrr_at_3_diff1": -0.15456867301767724, - "nauc_mrr_at_3_max": 0.08127863156043737, - "nauc_mrr_at_3_std": 0.049917519287707324, - "nauc_mrr_at_5_diff1": -0.13341097010600572, - "nauc_mrr_at_5_max": 0.07607293298892381, - "nauc_mrr_at_5_std": 0.058415510075163025, - "nauc_ndcg_at_1000_diff1": -0.1197947190208689, - "nauc_ndcg_at_1000_max": 0.10761964394885151, - "nauc_ndcg_at_1000_std": 0.17544190656064562, - "nauc_ndcg_at_100_diff1": -0.11055695160977044, - "nauc_ndcg_at_100_max": 0.10882941855322274, - "nauc_ndcg_at_100_std": 0.14514994054402142, - "nauc_ndcg_at_10_diff1": -0.10490212058705983, - "nauc_ndcg_at_10_max": 0.0877933413318513, - "nauc_ndcg_at_10_std": 0.07407721145352236, - "nauc_ndcg_at_1_diff1": -0.5415016019823545, - "nauc_ndcg_at_1_max": 0.25890560141416386, - "nauc_ndcg_at_1_std": 0.014078505026910177, - "nauc_ndcg_at_20_diff1": -0.09739288742447834, - "nauc_ndcg_at_20_max": 0.08898965589119323, - "nauc_ndcg_at_20_std": 0.09116950469122251, - "nauc_ndcg_at_3_diff1": -0.13592258833726864, - "nauc_ndcg_at_3_max": 0.07511923847763913, - "nauc_ndcg_at_3_std": 0.05019467063066953, - "nauc_ndcg_at_5_diff1": -0.11148549384385414, - "nauc_ndcg_at_5_max": 0.07112590862724855, - "nauc_ndcg_at_5_std": 0.06408420927682207, - "nauc_precision_at_1000_diff1": -0.2052249721287206, - "nauc_precision_at_1000_max": 0.1810837917883838, - "nauc_precision_at_1000_std": 0.4888332886224504, - "nauc_precision_at_100_diff1": -0.12904477641430562, - "nauc_precision_at_100_max": 0.1472874080458991, - "nauc_precision_at_100_std": 0.2367939277259602, - "nauc_precision_at_10_diff1": -0.10753940868940802, - "nauc_precision_at_10_max": 0.09962493139437552, - "nauc_precision_at_10_std": 0.08001071204177711, - "nauc_precision_at_1_diff1": -0.5415016019823545, - "nauc_precision_at_1_max": 0.25890560141416386, - "nauc_precision_at_1_std": 0.014078505026910177, - "nauc_precision_at_20_diff1": -0.09444210001342718, - "nauc_precision_at_20_max": 0.09951342399887461, - "nauc_precision_at_20_std": 0.10807933554562363, - "nauc_precision_at_3_diff1": -0.1543208082621269, - "nauc_precision_at_3_max": 0.08611811554190649, - "nauc_precision_at_3_std": 0.048213763137672686, - "nauc_precision_at_5_diff1": -0.1195619408248877, - "nauc_precision_at_5_max": 0.0791887243073344, - "nauc_precision_at_5_std": 0.06718837781985118, - "nauc_recall_at_1000_diff1": -0.1684682707344514, - "nauc_recall_at_1000_max": 0.17380982864946737, - "nauc_recall_at_1000_std": 0.5550045573571595, - "nauc_recall_at_100_diff1": -0.10849369522179253, - "nauc_recall_at_100_max": 0.1424542327994153, - "nauc_recall_at_100_std": 0.2500201617623722, - "nauc_recall_at_10_diff1": -0.0824038257749242, - "nauc_recall_at_10_max": 0.09070695218424778, - "nauc_recall_at_10_std": 0.08439981274339317, - "nauc_recall_at_1_diff1": -0.5463082664585215, - "nauc_recall_at_1_max": 0.26114918977811374, - "nauc_recall_at_1_std": 0.031162171757824993, - "nauc_recall_at_20_diff1": -0.07172879582348955, - "nauc_recall_at_20_max": 0.09175525207752894, - "nauc_recall_at_20_std": 0.1135769541413721, - "nauc_recall_at_3_diff1": -0.1121549242083564, - "nauc_recall_at_3_max": 0.06611057402653997, - "nauc_recall_at_3_std": 0.05347844526372312, - "nauc_recall_at_5_diff1": -0.08645717088592683, - "nauc_recall_at_5_max": 0.06479040069227433, - "nauc_recall_at_5_std": 0.07221167912917376, - "ndcg_at_1": 0.00815, - "ndcg_at_10": 0.09282, - "ndcg_at_100": 0.15648, - "ndcg_at_1000": 0.19752, - "ndcg_at_20": 0.11286, - "ndcg_at_3": 0.05247, - "ndcg_at_5": 0.07096, - "precision_at_1": 0.00815, - "precision_at_10": 0.02082, - "precision_at_100": 0.00528, - "precision_at_1000": 0.00086, - "precision_at_20": 0.01439, - "precision_at_3": 0.03173, - "precision_at_5": 0.02806, - "recall_at_1": 0.00396, - "recall_at_10": 0.1982, - "recall_at_100": 0.51703, - "recall_at_1000": 0.84604, - "recall_at_20": 0.2777, - "recall_at_3": 0.08549, - "recall_at_5": 0.13046 - } - ] - }, - "task_name": "CIRRIT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchCount.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchCount.json deleted file mode 100644 index d69a7b62ff..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchCount.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 18.0276620388031, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.027918781725888325, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.027918781725888325 - } - ] - }, - "task_name": "CVBenchCount" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDepth.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDepth.json deleted file mode 100644 index f6e87d0e07..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDepth.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 14.530367136001587, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5266666666666666, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5266666666666666 - } - ] - }, - "task_name": "CVBenchDepth" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDistance.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDistance.json deleted file mode 100644 index f8d0a0fc67..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchDistance.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 14.808384895324707, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.47, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.47 - } - ] - }, - "task_name": "CVBenchDistance" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchRelation.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchRelation.json deleted file mode 100644 index 21cf37be5b..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/CVBenchRelation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "default", - "evaluation_time": 11.672912359237671, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5092307692307693, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5092307692307693 - } - ] - }, - "task_name": "CVBenchRelation" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/FORBI2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/FORBI2IRetrieval.json deleted file mode 100644 index 67200bf463..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/FORBI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", - "evaluation_time": 2148.242924451828, - "kg_co2_emissions": null, - "mteb_version": "1.14.21", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.47955, - "cv_recall_at_10": 0.64309, - "cv_recall_at_100": 0.72468, - "cv_recall_at_1000": 0.75774, - "cv_recall_at_20": 0.67275, - "cv_recall_at_3": 0.56981, - "cv_recall_at_5": 0.60325, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.47955, - "map_at_1": 0.47826, - "map_at_10": 0.53199, - "map_at_100": 0.53545, - "map_at_1000": 0.53562, - "map_at_20": 0.53409, - "map_at_3": 0.51886, - "map_at_5": 0.52653, - "mrr_at_1": 0.47954716981132073, - "mrr_at_10": 0.5327790955375838, - "mrr_at_100": 0.536231063428097, - "mrr_at_1000": 0.5363963198824263, - "mrr_at_20": 0.5348666258075478, - "mrr_at_3": 0.5196729559748401, - "mrr_at_5": 0.527325786163518, - "nauc_cv_recall_at_1000_diff1": 0.9783622452451709, - "nauc_cv_recall_at_1000_max": -1.3700380127097909, - "nauc_cv_recall_at_1000_std": -1.3622593419555964, - "nauc_cv_recall_at_100_diff1": 0.9367206943403367, - "nauc_cv_recall_at_100_max": -1.1415729064069615, - "nauc_cv_recall_at_100_std": -1.1585751691509931, - "nauc_cv_recall_at_10_diff1": 0.8585716756688088, - "nauc_cv_recall_at_10_max": -0.7579079603295624, - "nauc_cv_recall_at_10_std": -0.8979314486065754, - "nauc_cv_recall_at_1_diff1": 0.8389554763575047, - "nauc_cv_recall_at_1_max": -0.3861612888474326, - "nauc_cv_recall_at_1_std": -0.6155125338004539, - "nauc_cv_recall_at_20_diff1": 0.8841302115144418, - "nauc_cv_recall_at_20_max": -0.8720455440695557, - "nauc_cv_recall_at_20_std": -0.9664298519627592, - "nauc_cv_recall_at_3_diff1": 0.8259560459790033, - "nauc_cv_recall_at_3_max": -0.546319243284802, - "nauc_cv_recall_at_3_std": -0.7634227610142231, - "nauc_cv_recall_at_5_diff1": 0.8299350588708171, - "nauc_cv_recall_at_5_max": -0.6287235649657446, - "nauc_cv_recall_at_5_std": -0.8161938597529815, - "nauc_map_at_1000_diff1": 0.840854892934761, - "nauc_map_at_1000_max": -0.4924278647794887, - "nauc_map_at_1000_std": -0.7022277605230981, - "nauc_map_at_100_diff1": 0.8407441848445416, - "nauc_map_at_100_max": -0.49192560404032804, - "nauc_map_at_100_std": -0.7017634806879928, - "nauc_map_at_10_diff1": 0.8385260397871099, - "nauc_map_at_10_max": -0.4821878658539431, - "nauc_map_at_10_std": -0.6956217111863439, - "nauc_map_at_1_diff1": 0.8418694895112901, - "nauc_map_at_1_max": -0.38442688222770177, - "nauc_map_at_1_std": -0.6125805636780283, - "nauc_map_at_20_diff1": 0.8399206633399918, - "nauc_map_at_20_max": -0.4878232150574702, - "nauc_map_at_20_std": -0.6989058410509833, - "nauc_map_at_3_diff1": 0.8341959402895627, - "nauc_map_at_3_max": -0.45174026022682956, - "nauc_map_at_3_std": -0.676617769201114, - "nauc_map_at_5_diff1": 0.8350923824381429, - "nauc_map_at_5_max": -0.46831637558370504, - "nauc_map_at_5_std": -0.6870541683615817, - "nauc_mrr_at_1000_diff1": 0.8393074209785932, - "nauc_mrr_at_1000_max": -0.4936213541118185, - "nauc_mrr_at_1000_std": -0.7041826758357784, - "nauc_mrr_at_100_diff1": 0.8391972194458248, - "nauc_mrr_at_100_max": -0.4931184490320341, - "nauc_mrr_at_100_std": -0.7037174874420373, - "nauc_mrr_at_10_diff1": 0.8369860440129665, - "nauc_mrr_at_10_max": -0.4833850308584579, - "nauc_mrr_at_10_std": -0.6975876349014124, - "nauc_mrr_at_1_diff1": 0.8389554763575047, - "nauc_mrr_at_1_max": -0.3861612888474326, - "nauc_mrr_at_1_std": -0.6155125338004539, - "nauc_mrr_at_20_diff1": 0.8383772686177878, - "nauc_mrr_at_20_max": -0.4890112340146167, - "nauc_mrr_at_20_std": -0.7008535231923642, - "nauc_mrr_at_3_diff1": 0.8326910963164327, - "nauc_mrr_at_3_max": -0.45292910686532045, - "nauc_mrr_at_3_std": -0.6786157396873091, - "nauc_mrr_at_5_diff1": 0.8335794976849904, - "nauc_mrr_at_5_max": -0.46951576427639513, - "nauc_mrr_at_5_std": -0.689014845411203, - "nauc_ndcg_at_1000_diff1": 0.8571833725542346, - "nauc_ndcg_at_1000_max": -0.6105533656454696, - "nauc_ndcg_at_1000_std": -0.7896000730053025, - "nauc_ndcg_at_100_diff1": 0.854211660299563, - "nauc_ndcg_at_100_max": -0.5955309613288091, - "nauc_ndcg_at_100_std": -0.7753517475583974, - "nauc_ndcg_at_10_diff1": 0.8421946492855936, - "nauc_ndcg_at_10_max": -0.5410015654361832, - "nauc_ndcg_at_10_std": -0.7393165425948717, - "nauc_ndcg_at_1_diff1": 0.8418694895112901, - "nauc_ndcg_at_1_max": -0.38442688222770177, - "nauc_ndcg_at_1_std": -0.6125805636780283, - "nauc_ndcg_at_20_diff1": 0.8475183312428272, - "nauc_ndcg_at_20_max": -0.563016555679603, - "nauc_ndcg_at_20_std": -0.7522312081819034, - "nauc_ndcg_at_3_diff1": 0.8321161349170787, - "nauc_ndcg_at_3_max": -0.47507879358661737, - "nauc_ndcg_at_3_std": -0.6980061182251038, - "nauc_ndcg_at_5_diff1": 0.8337229410549184, - "nauc_ndcg_at_5_max": -0.5059407346638893, - "nauc_ndcg_at_5_std": -0.7175685317009618, - "nauc_precision_at_1000_diff1": 0.9783622452451708, - "nauc_precision_at_1000_max": -1.370038012709789, - "nauc_precision_at_1000_std": -1.362259341955595, - "nauc_precision_at_100_diff1": 0.9367206943403371, - "nauc_precision_at_100_max": -1.141572906406963, - "nauc_precision_at_100_std": -1.1585751691509945, - "nauc_precision_at_10_diff1": 0.8585920210973836, - "nauc_precision_at_10_max": -0.7576550732857764, - "nauc_precision_at_10_std": -0.8975426783303393, - "nauc_precision_at_1_diff1": 0.8418694895112901, - "nauc_precision_at_1_max": -0.38442688222770177, - "nauc_precision_at_1_std": -0.6125805636780283, - "nauc_precision_at_20_diff1": 0.8841302115144408, - "nauc_precision_at_20_max": -0.8720455440695557, - "nauc_precision_at_20_std": -0.9664298519627592, - "nauc_precision_at_3_diff1": 0.8259800230617228, - "nauc_precision_at_3_max": -0.5459658712057691, - "nauc_precision_at_3_std": -0.7627160737347208, - "nauc_precision_at_5_diff1": 0.8298537192402428, - "nauc_precision_at_5_max": -0.6284652527439748, - "nauc_precision_at_5_std": -0.8158750015206129, - "nauc_recall_at_1000_diff1": 0.9783622452451709, - "nauc_recall_at_1000_max": -1.3700380127097909, - "nauc_recall_at_1000_std": -1.3622593419555964, - "nauc_recall_at_100_diff1": 0.9367206943403367, - "nauc_recall_at_100_max": -1.1415729064069615, - "nauc_recall_at_100_std": -1.1585751691509931, - "nauc_recall_at_10_diff1": 0.8585920210973839, - "nauc_recall_at_10_max": -0.7576550732857767, - "nauc_recall_at_10_std": -0.8975426783303397, - "nauc_recall_at_1_diff1": 0.8418694895112901, - "nauc_recall_at_1_max": -0.38442688222770177, - "nauc_recall_at_1_std": -0.6125805636780283, - "nauc_recall_at_20_diff1": 0.8841302115144418, - "nauc_recall_at_20_max": -0.8720455440695557, - "nauc_recall_at_20_std": -0.9664298519627592, - "nauc_recall_at_3_diff1": 0.8259800230617225, - "nauc_recall_at_3_max": -0.5459658712057688, - "nauc_recall_at_3_std": -0.76271607373472, - "nauc_recall_at_5_diff1": 0.829853719240243, - "nauc_recall_at_5_max": -0.628465252743975, - "nauc_recall_at_5_std": -0.8158750015206132, - "ndcg_at_1": 0.47826, - "ndcg_at_10": 0.55876, - "ndcg_at_100": 0.57601, - "ndcg_at_1000": 0.58025, - "ndcg_at_20": 0.56632, - "ndcg_at_3": 0.5319, - "ndcg_at_5": 0.54573, - "precision_at_1": 0.47826, - "precision_at_10": 0.0643, - "precision_at_100": 0.00725, - "precision_at_1000": 0.00076, - "precision_at_20": 0.03364, - "precision_at_3": 0.18986, - "precision_at_5": 0.12063, - "recall_at_1": 0.47826, - "recall_at_10": 0.64302, - "recall_at_100": 0.72468, - "recall_at_1000": 0.75774, - "recall_at_20": 0.67275, - "recall_at_3": 0.56958, - "recall_at_5": 0.60317 - } - ] - }, - "task_name": "FORBI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Fashion200kI2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Fashion200kI2TRetrieval.json deleted file mode 100644 index d8b3f67a4e..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Fashion200kI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "96a313715ecf67f5dfe70c4fa52406bc7bdfbeee", - "evaluation_time": 165.14273023605347, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.01657, - "cv_recall_at_10": 0.07486, - "cv_recall_at_100": 0.26652, - "cv_recall_at_1000": 0.62487, - "cv_recall_at_20": 0.11638, - "cv_recall_at_3": 0.03559, - "cv_recall_at_5": 0.05011, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.04138, - "map_at_1": 0.01657, - "map_at_10": 0.03121, - "map_at_100": 0.03747, - "map_at_1000": 0.03879, - "map_at_20": 0.03402, - "map_at_3": 0.02461, - "map_at_5": 0.02794, - "mrr_at_1": 0.01656780527715279, - "mrr_at_10": 0.031209923800433102, - "mrr_at_100": 0.03746585906947475, - "mrr_at_1000": 0.03878613232811227, - "mrr_at_20": 0.0340226319794799, - "mrr_at_3": 0.02461307697552327, - "mrr_at_5": 0.027936865071248384, - "nauc_cv_recall_at_1000_diff1": 0.05215643497408927, - "nauc_cv_recall_at_1000_max": 0.16019254559848906, - "nauc_cv_recall_at_1000_std": 0.3630252561689271, - "nauc_cv_recall_at_100_diff1": 0.09934080212830879, - "nauc_cv_recall_at_100_max": 0.11775386806178802, - "nauc_cv_recall_at_100_std": 0.2626429228260664, - "nauc_cv_recall_at_10_diff1": 0.14732289617853372, - "nauc_cv_recall_at_10_max": 0.1300838307363312, - "nauc_cv_recall_at_10_std": 0.22261060186404014, - "nauc_cv_recall_at_1_diff1": 0.43658812029041394, - "nauc_cv_recall_at_1_max": 0.20106424998008995, - "nauc_cv_recall_at_1_std": 0.21594406605223246, - "nauc_cv_recall_at_20_diff1": 0.12199658363483398, - "nauc_cv_recall_at_20_max": 0.13309204661129767, - "nauc_cv_recall_at_20_std": 0.24973972652762613, - "nauc_cv_recall_at_3_diff1": 0.24662246338856247, - "nauc_cv_recall_at_3_max": 0.15878837526882447, - "nauc_cv_recall_at_3_std": 0.18589391029851615, - "nauc_cv_recall_at_5_diff1": 0.19081976581565688, - "nauc_cv_recall_at_5_max": 0.15164531948619475, - "nauc_cv_recall_at_5_std": 0.21248017451063386, - "nauc_map_at_1000_diff1": 0.2406708231108197, - "nauc_map_at_1000_max": 0.16014121809590698, - "nauc_map_at_1000_std": 0.22704010525299115, - "nauc_map_at_100_diff1": 0.24219026893873466, - "nauc_map_at_100_max": 0.15936399671152812, - "nauc_map_at_100_std": 0.22429926115695684, - "nauc_map_at_10_diff1": 0.2674661720376913, - "nauc_map_at_10_max": 0.16657906217191382, - "nauc_map_at_10_std": 0.2149240991921386, - "nauc_map_at_1_diff1": 0.43658812029041394, - "nauc_map_at_1_max": 0.20106424998008995, - "nauc_map_at_1_std": 0.21594406605223246, - "nauc_map_at_20_diff1": 0.25261446616694033, - "nauc_map_at_20_max": 0.16383021382320623, - "nauc_map_at_20_std": 0.22094405123394173, - "nauc_map_at_3_diff1": 0.32374852815286687, - "nauc_map_at_3_max": 0.18065670256553057, - "nauc_map_at_3_std": 0.20149692020784588, - "nauc_map_at_5_diff1": 0.29208437849888963, - "nauc_map_at_5_max": 0.17535941533844787, - "nauc_map_at_5_std": 0.21150556697054126, - "nauc_mrr_at_1000_diff1": 0.2406708231108197, - "nauc_mrr_at_1000_max": 0.16014121809590698, - "nauc_mrr_at_1000_std": 0.22704010525299115, - "nauc_mrr_at_100_diff1": 0.24219026893873466, - "nauc_mrr_at_100_max": 0.15936399671152812, - "nauc_mrr_at_100_std": 0.22429926115695684, - "nauc_mrr_at_10_diff1": 0.2674661720376913, - "nauc_mrr_at_10_max": 0.16657906217191382, - "nauc_mrr_at_10_std": 0.2149240991921386, - "nauc_mrr_at_1_diff1": 0.43658812029041394, - "nauc_mrr_at_1_max": 0.20106424998008995, - "nauc_mrr_at_1_std": 0.21594406605223246, - "nauc_mrr_at_20_diff1": 0.25261446616694033, - "nauc_mrr_at_20_max": 0.16383021382320623, - "nauc_mrr_at_20_std": 0.22094405123394173, - "nauc_mrr_at_3_diff1": 0.32374852815286687, - "nauc_mrr_at_3_max": 0.18065670256553057, - "nauc_mrr_at_3_std": 0.20149692020784588, - "nauc_mrr_at_5_diff1": 0.29208437849888963, - "nauc_mrr_at_5_max": 0.17535941533844787, - "nauc_mrr_at_5_std": 0.21150556697054126, - "nauc_ndcg_at_1000_diff1": 0.1473174010573472, - "nauc_ndcg_at_1000_max": 0.1509480108109463, - "nauc_ndcg_at_1000_std": 0.27887845367426434, - "nauc_ndcg_at_100_diff1": 0.1627532520071558, - "nauc_ndcg_at_100_max": 0.1363846085513206, - "nauc_ndcg_at_100_std": 0.24358663136522796, - "nauc_ndcg_at_10_diff1": 0.21697906584798596, - "nauc_ndcg_at_10_max": 0.15163132959233716, - "nauc_ndcg_at_10_std": 0.21785059479581834, - "nauc_ndcg_at_1_diff1": 0.43658812029041394, - "nauc_ndcg_at_1_max": 0.20106424998008995, - "nauc_ndcg_at_1_std": 0.21594406605223246, - "nauc_ndcg_at_20_diff1": 0.19023028222093716, - "nauc_ndcg_at_20_max": 0.14880588752163815, - "nauc_ndcg_at_20_std": 0.233151196984513, - "nauc_ndcg_at_3_diff1": 0.29829003226909856, - "nauc_ndcg_at_3_max": 0.17360427192148528, - "nauc_ndcg_at_3_std": 0.19646665255827292, - "nauc_ndcg_at_5_diff1": 0.2548364192942451, - "nauc_ndcg_at_5_max": 0.16670644633246848, - "nauc_ndcg_at_5_std": 0.21170762231114157, - "nauc_precision_at_1000_diff1": 0.05215643497408827, - "nauc_precision_at_1000_max": 0.1601925455984888, - "nauc_precision_at_1000_std": 0.3630252561689268, - "nauc_precision_at_100_diff1": 0.09934080212830884, - "nauc_precision_at_100_max": 0.11775386806178809, - "nauc_precision_at_100_std": 0.26264292282606644, - "nauc_precision_at_10_diff1": 0.14732289617853392, - "nauc_precision_at_10_max": 0.13008383073633142, - "nauc_precision_at_10_std": 0.22261060186404052, - "nauc_precision_at_1_diff1": 0.43658812029041394, - "nauc_precision_at_1_max": 0.20106424998008995, - "nauc_precision_at_1_std": 0.21594406605223246, - "nauc_precision_at_20_diff1": 0.12199658363483433, - "nauc_precision_at_20_max": 0.13309204661129775, - "nauc_precision_at_20_std": 0.24973972652762635, - "nauc_precision_at_3_diff1": 0.24662246338856234, - "nauc_precision_at_3_max": 0.15878837526882422, - "nauc_precision_at_3_std": 0.18589391029851624, - "nauc_precision_at_5_diff1": 0.19081976581565693, - "nauc_precision_at_5_max": 0.1516453194861949, - "nauc_precision_at_5_std": 0.21248017451063406, - "nauc_recall_at_1000_diff1": 0.05215643497408927, - "nauc_recall_at_1000_max": 0.16019254559848906, - "nauc_recall_at_1000_std": 0.3630252561689271, - "nauc_recall_at_100_diff1": 0.09934080212830879, - "nauc_recall_at_100_max": 0.11775386806178802, - "nauc_recall_at_100_std": 0.2626429228260664, - "nauc_recall_at_10_diff1": 0.14732289617853372, - "nauc_recall_at_10_max": 0.1300838307363312, - "nauc_recall_at_10_std": 0.22261060186404014, - "nauc_recall_at_1_diff1": 0.43658812029041394, - "nauc_recall_at_1_max": 0.20106424998008995, - "nauc_recall_at_1_std": 0.21594406605223246, - "nauc_recall_at_20_diff1": 0.12199658363483398, - "nauc_recall_at_20_max": 0.13309204661129767, - "nauc_recall_at_20_std": 0.24973972652762613, - "nauc_recall_at_3_diff1": 0.24662246338856247, - "nauc_recall_at_3_max": 0.15878837526882447, - "nauc_recall_at_3_std": 0.18589391029851615, - "nauc_recall_at_5_diff1": 0.19081976581565688, - "nauc_recall_at_5_max": 0.15164531948619475, - "nauc_recall_at_5_std": 0.21248017451063386, - "ndcg_at_1": 0.01657, - "ndcg_at_10": 0.04138, - "ndcg_at_100": 0.07884, - "ndcg_at_1000": 0.1221, - "ndcg_at_20": 0.05179, - "ndcg_at_3": 0.02742, - "ndcg_at_5": 0.0334, - "precision_at_1": 0.01657, - "precision_at_10": 0.00749, - "precision_at_100": 0.00267, - "precision_at_1000": 0.00062, - "precision_at_20": 0.00582, - "precision_at_3": 0.01186, - "precision_at_5": 0.01002, - "recall_at_1": 0.01657, - "recall_at_10": 0.07486, - "recall_at_100": 0.26652, - "recall_at_1000": 0.62487, - "recall_at_20": 0.11638, - "recall_at_3": 0.03559, - "recall_at_5": 0.05011 - } - ] - }, - "task_name": "Fashion200kI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesI2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesI2TRetrieval.json deleted file mode 100644 index 52a575b78b..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", - "evaluation_time": 24.870660543441772, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.39, - "cv_recall_at_10": 0.724, - "cv_recall_at_100": 0.834, - "cv_recall_at_1000": 0.911, - "cv_recall_at_20": 0.758, - "cv_recall_at_3": 0.621, - "cv_recall_at_5": 0.678, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56877, - "map_at_1": 0.392, - "map_at_10": 0.51737, - "map_at_100": 0.52151, - "map_at_1000": 0.52183, - "map_at_20": 0.51954, - "map_at_3": 0.498, - "map_at_5": 0.5108, - "mrr_at_1": 0.39, - "mrr_at_10": 0.5160158730158726, - "mrr_at_100": 0.5203345965174644, - "mrr_at_1000": 0.5206527081735791, - "mrr_at_20": 0.518364903616335, - "mrr_at_3": 0.496833333333333, - "mrr_at_5": 0.5095833333333328, - "nauc_cv_recall_at_1000_diff1": 0.10263955769573768, - "nauc_cv_recall_at_1000_max": 0.6129732791993203, - "nauc_cv_recall_at_1000_std": 0.3754445598464104, - "nauc_cv_recall_at_100_diff1": 0.1065109821651072, - "nauc_cv_recall_at_100_max": 0.670416384844493, - "nauc_cv_recall_at_100_std": 0.3493289665729246, - "nauc_cv_recall_at_10_diff1": 0.1539117944599323, - "nauc_cv_recall_at_10_max": 0.6446834964052115, - "nauc_cv_recall_at_10_std": 0.236339029908953, - "nauc_cv_recall_at_1_diff1": 0.4762368553800783, - "nauc_cv_recall_at_1_max": 0.398969125502826, - "nauc_cv_recall_at_1_std": 0.021837207196719798, - "nauc_cv_recall_at_20_diff1": 0.1302823105717122, - "nauc_cv_recall_at_20_max": 0.6618096412777686, - "nauc_cv_recall_at_20_std": 0.28837236287373114, - "nauc_cv_recall_at_3_diff1": 0.22523976186403152, - "nauc_cv_recall_at_3_max": 0.6245459435051234, - "nauc_cv_recall_at_3_std": 0.15245202065735497, - "nauc_cv_recall_at_5_diff1": 0.16951071378184274, - "nauc_cv_recall_at_5_max": 0.6226364127498648, - "nauc_cv_recall_at_5_std": 0.19889403902865374, - "nauc_map_at_1000_diff1": 0.3561706156286026, - "nauc_map_at_1000_max": 0.49921751700988676, - "nauc_map_at_1000_std": 0.09758022590827435, - "nauc_map_at_100_diff1": 0.35594508335415453, - "nauc_map_at_100_max": 0.4994175748948008, - "nauc_map_at_100_std": 0.09772300134657558, - "nauc_map_at_10_diff1": 0.35503732771075197, - "nauc_map_at_10_max": 0.5003076952556401, - "nauc_map_at_10_std": 0.09670690179240216, - "nauc_map_at_1_diff1": 0.47169657717635144, - "nauc_map_at_1_max": 0.41488426907053433, - "nauc_map_at_1_std": 0.04491745319591042, - "nauc_map_at_20_diff1": 0.35511537035310814, - "nauc_map_at_20_max": 0.500525711086734, - "nauc_map_at_20_std": 0.09838319969822605, - "nauc_map_at_3_diff1": 0.36183536801796407, - "nauc_map_at_3_max": 0.5003507757379961, - "nauc_map_at_3_std": 0.08623036096074559, - "nauc_map_at_5_diff1": 0.3547091654543549, - "nauc_map_at_5_max": 0.4997381766720002, - "nauc_map_at_5_std": 0.09477623407261138, - "nauc_mrr_at_1000_diff1": 0.3591123682228567, - "nauc_mrr_at_1000_max": 0.49027183863310125, - "nauc_mrr_at_1000_std": 0.08439711102794764, - "nauc_mrr_at_100_diff1": 0.35888388249516046, - "nauc_mrr_at_100_max": 0.49047961412452795, - "nauc_mrr_at_100_std": 0.08455100972634555, - "nauc_mrr_at_10_diff1": 0.35816060914379055, - "nauc_mrr_at_10_max": 0.4917050030422624, - "nauc_mrr_at_10_std": 0.08386230024828238, - "nauc_mrr_at_1_diff1": 0.4762368553800783, - "nauc_mrr_at_1_max": 0.398969125502826, - "nauc_mrr_at_1_std": 0.021837207196719798, - "nauc_mrr_at_20_diff1": 0.35803929783268085, - "nauc_mrr_at_20_max": 0.49163331748152517, - "nauc_mrr_at_20_std": 0.08527826344844179, - "nauc_mrr_at_3_diff1": 0.3647354992093332, - "nauc_mrr_at_3_max": 0.49377814932302555, - "nauc_mrr_at_3_std": 0.07502909231738003, - "nauc_mrr_at_5_diff1": 0.3576362645252059, - "nauc_mrr_at_5_max": 0.4907831433335126, - "nauc_mrr_at_5_std": 0.08143862051903414, - "nauc_ndcg_at_1000_diff1": 0.32390740934623524, - "nauc_ndcg_at_1000_max": 0.5192713520679167, - "nauc_ndcg_at_1000_std": 0.12475783601928456, - "nauc_ndcg_at_100_diff1": 0.31759934966550735, - "nauc_ndcg_at_100_max": 0.5257628613485602, - "nauc_ndcg_at_100_std": 0.1300015379219445, - "nauc_ndcg_at_10_diff1": 0.31327913630562454, - "nauc_ndcg_at_10_max": 0.5294161890178075, - "nauc_ndcg_at_10_std": 0.12336856511354594, - "nauc_ndcg_at_1_diff1": 0.47169657717635144, - "nauc_ndcg_at_1_max": 0.41488426907053433, - "nauc_ndcg_at_1_std": 0.04491745319591042, - "nauc_ndcg_at_20_diff1": 0.3128923127075764, - "nauc_ndcg_at_20_max": 0.530861917752235, - "nauc_ndcg_at_20_std": 0.13092750080735302, - "nauc_ndcg_at_3_diff1": 0.3286920241406717, - "nauc_ndcg_at_3_max": 0.5277702997215936, - "nauc_ndcg_at_3_std": 0.1002345470431284, - "nauc_ndcg_at_5_diff1": 0.3136910095539595, - "nauc_ndcg_at_5_max": 0.5273652484740478, - "nauc_ndcg_at_5_std": 0.11725163266241763, - "nauc_precision_at_1000_diff1": 0.10263955769573535, - "nauc_precision_at_1000_max": 0.6129732791993188, - "nauc_precision_at_1000_std": 0.37544455984641006, - "nauc_precision_at_100_diff1": 0.10651098216510617, - "nauc_precision_at_100_max": 0.6704163848444932, - "nauc_precision_at_100_std": 0.34932896657292334, - "nauc_precision_at_10_diff1": 0.14952668399115804, - "nauc_precision_at_10_max": 0.64193653642222, - "nauc_precision_at_10_std": 0.23424384028837889, - "nauc_precision_at_1_diff1": 0.47169657717635144, - "nauc_precision_at_1_max": 0.41488426907053433, - "nauc_precision_at_1_std": 0.04491745319591042, - "nauc_precision_at_20_diff1": 0.13028231057171152, - "nauc_precision_at_20_max": 0.6618096412777668, - "nauc_precision_at_20_std": 0.28837236287373114, - "nauc_precision_at_3_diff1": 0.22446919505049567, - "nauc_precision_at_3_max": 0.6146379954166691, - "nauc_precision_at_3_std": 0.1448788900139171, - "nauc_precision_at_5_diff1": 0.16951071378184204, - "nauc_precision_at_5_max": 0.622636412749864, - "nauc_precision_at_5_std": 0.19889403902865282, - "nauc_recall_at_1000_diff1": 0.10263955769573768, - "nauc_recall_at_1000_max": 0.6129732791993203, - "nauc_recall_at_1000_std": 0.3754445598464104, - "nauc_recall_at_100_diff1": 0.1065109821651072, - "nauc_recall_at_100_max": 0.670416384844493, - "nauc_recall_at_100_std": 0.3493289665729246, - "nauc_recall_at_10_diff1": 0.14952668399115937, - "nauc_recall_at_10_max": 0.641936536422221, - "nauc_recall_at_10_std": 0.2342438402883794, - "nauc_recall_at_1_diff1": 0.47169657717635144, - "nauc_recall_at_1_max": 0.41488426907053433, - "nauc_recall_at_1_std": 0.04491745319591042, - "nauc_recall_at_20_diff1": 0.1302823105717122, - "nauc_recall_at_20_max": 0.6618096412777686, - "nauc_recall_at_20_std": 0.28837236287373114, - "nauc_recall_at_3_diff1": 0.2244691950504948, - "nauc_recall_at_3_max": 0.6146379954166685, - "nauc_recall_at_3_std": 0.14487889001391666, - "nauc_recall_at_5_diff1": 0.16951071378184274, - "nauc_recall_at_5_max": 0.6226364127498648, - "nauc_recall_at_5_std": 0.19889403902865374, - "ndcg_at_1": 0.392, - "ndcg_at_10": 0.56877, - "ndcg_at_100": 0.59092, - "ndcg_at_1000": 0.60039, - "ndcg_at_20": 0.57679, - "ndcg_at_3": 0.52981, - "ndcg_at_5": 0.55308, - "precision_at_1": 0.392, - "precision_at_10": 0.0726, - "precision_at_100": 0.00834, - "precision_at_1000": 0.00091, - "precision_at_20": 0.0379, - "precision_at_3": 0.207, - "precision_at_5": 0.1356, - "recall_at_1": 0.392, - "recall_at_10": 0.726, - "recall_at_100": 0.834, - "recall_at_1000": 0.911, - "recall_at_20": 0.758, - "recall_at_3": 0.621, - "recall_at_5": 0.678 - } - ] - }, - "task_name": "HatefulMemesI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesT2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesT2IRetrieval.json deleted file mode 100644 index 2614fcd742..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/HatefulMemesT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", - "evaluation_time": 147.79884696006775, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.457, - "cv_recall_at_10": 0.761, - "cv_recall_at_100": 0.839, - "cv_recall_at_1000": 0.895, - "cv_recall_at_20": 0.793, - "cv_recall_at_3": 0.671, - "cv_recall_at_5": 0.724, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.61855, - "map_at_1": 0.457, - "map_at_10": 0.57175, - "map_at_100": 0.5752, - "map_at_1000": 0.5754, - "map_at_20": 0.57406, - "map_at_3": 0.5545, - "map_at_5": 0.5666, - "mrr_at_1": 0.457, - "mrr_at_10": 0.5717476190476187, - "mrr_at_100": 0.5751927754695753, - "mrr_at_1000": 0.5753988882730213, - "mrr_at_20": 0.5740526036656837, - "mrr_at_3": 0.5544999999999997, - "mrr_at_5": 0.5665999999999995, - "nauc_cv_recall_at_1000_diff1": 0.09881437072166871, - "nauc_cv_recall_at_1000_max": 0.3361460337345095, - "nauc_cv_recall_at_1000_std": 0.3314348351304203, - "nauc_cv_recall_at_100_diff1": 0.21713821991445592, - "nauc_cv_recall_at_100_max": 0.32597885179784103, - "nauc_cv_recall_at_100_std": 0.29378672410321527, - "nauc_cv_recall_at_10_diff1": 0.31827978460409795, - "nauc_cv_recall_at_10_max": 0.3669826124892289, - "nauc_cv_recall_at_10_std": 0.15130021075112898, - "nauc_cv_recall_at_1_diff1": 0.459490502212497, - "nauc_cv_recall_at_1_max": 0.19915884427724026, - "nauc_cv_recall_at_1_std": -0.0022919130648232137, - "nauc_cv_recall_at_20_diff1": 0.30115823543573067, - "nauc_cv_recall_at_20_max": 0.3577860510810276, - "nauc_cv_recall_at_20_std": 0.17380097220769894, - "nauc_cv_recall_at_3_diff1": 0.3679652871898125, - "nauc_cv_recall_at_3_max": 0.33460272118305795, - "nauc_cv_recall_at_3_std": 0.09750594972545565, - "nauc_cv_recall_at_5_diff1": 0.34694925594433695, - "nauc_cv_recall_at_5_max": 0.38095109388152243, - "nauc_cv_recall_at_5_std": 0.13474859644922935, - "nauc_map_at_1000_diff1": 0.41712920668130243, - "nauc_map_at_1000_max": 0.25742819455695226, - "nauc_map_at_1000_std": 0.043159224520371416, - "nauc_map_at_100_diff1": 0.41723483378100945, - "nauc_map_at_100_max": 0.25750564533870984, - "nauc_map_at_100_std": 0.04322917723700672, - "nauc_map_at_10_diff1": 0.41823666093016765, - "nauc_map_at_10_max": 0.259394302781376, - "nauc_map_at_10_std": 0.04212208767183869, - "nauc_map_at_1_diff1": 0.459490502212497, - "nauc_map_at_1_max": 0.19915884427724026, - "nauc_map_at_1_std": -0.0022919130648232137, - "nauc_map_at_20_diff1": 0.4179471732526579, - "nauc_map_at_20_max": 0.2581972993364297, - "nauc_map_at_20_std": 0.04229858410142156, - "nauc_map_at_3_diff1": 0.4228403396208564, - "nauc_map_at_3_max": 0.25461398394576956, - "nauc_map_at_3_std": 0.035903781369377205, - "nauc_map_at_5_diff1": 0.4204871609033417, - "nauc_map_at_5_max": 0.26207000402583563, - "nauc_map_at_5_std": 0.04155080339322991, - "nauc_mrr_at_1000_diff1": 0.4171205757774428, - "nauc_mrr_at_1000_max": 0.25744646575786206, - "nauc_mrr_at_1000_std": 0.04313917421147163, - "nauc_mrr_at_100_diff1": 0.4172264345923441, - "nauc_mrr_at_100_max": 0.2575237455468791, - "nauc_mrr_at_100_std": 0.04320914712121044, - "nauc_mrr_at_10_diff1": 0.41823666093016765, - "nauc_mrr_at_10_max": 0.259394302781376, - "nauc_mrr_at_10_std": 0.04212208767183869, - "nauc_mrr_at_1_diff1": 0.459490502212497, - "nauc_mrr_at_1_max": 0.19915884427724026, - "nauc_mrr_at_1_std": -0.0022919130648232137, - "nauc_mrr_at_20_diff1": 0.4179381540751488, - "nauc_mrr_at_20_max": 0.2582088368426858, - "nauc_mrr_at_20_std": 0.04227994399955885, - "nauc_mrr_at_3_diff1": 0.4228403396208564, - "nauc_mrr_at_3_max": 0.25461398394576956, - "nauc_mrr_at_3_std": 0.035903781369377205, - "nauc_mrr_at_5_diff1": 0.4204871609033417, - "nauc_mrr_at_5_max": 0.26207000402583563, - "nauc_mrr_at_5_std": 0.04155080339322991, - "nauc_ndcg_at_1000_diff1": 0.38960988590535767, - "nauc_ndcg_at_1000_max": 0.27080662641082004, - "nauc_ndcg_at_1000_std": 0.07097161120143067, - "nauc_ndcg_at_100_diff1": 0.39184562939358347, - "nauc_ndcg_at_100_max": 0.27174824704307293, - "nauc_ndcg_at_100_std": 0.07372029959444318, - "nauc_ndcg_at_10_diff1": 0.39942366314570593, - "nauc_ndcg_at_10_max": 0.28076942681445477, - "nauc_ndcg_at_10_std": 0.06280659045704115, - "nauc_ndcg_at_1_diff1": 0.459490502212497, - "nauc_ndcg_at_1_max": 0.19915884427724026, - "nauc_ndcg_at_1_std": -0.0022919130648232137, - "nauc_ndcg_at_20_diff1": 0.3980577456506629, - "nauc_ndcg_at_20_max": 0.2770711582775575, - "nauc_ndcg_at_20_std": 0.0645484759837998, - "nauc_ndcg_at_3_diff1": 0.41020207771802114, - "nauc_ndcg_at_3_max": 0.2730916384556409, - "nauc_ndcg_at_3_std": 0.050010910356776794, - "nauc_ndcg_at_5_diff1": 0.40510394288119245, - "nauc_ndcg_at_5_max": 0.2869883283095438, - "nauc_ndcg_at_5_std": 0.060887275366932087, - "nauc_precision_at_1000_diff1": 0.09881437072166596, - "nauc_precision_at_1000_max": 0.33614603373450663, - "nauc_precision_at_1000_std": 0.33143483513041805, - "nauc_precision_at_100_diff1": 0.21713821991445537, - "nauc_precision_at_100_max": 0.3259788517978413, - "nauc_precision_at_100_std": 0.2937867241032148, - "nauc_precision_at_10_diff1": 0.31827978460409695, - "nauc_precision_at_10_max": 0.36698261248922853, - "nauc_precision_at_10_std": 0.15130021075112876, - "nauc_precision_at_1_diff1": 0.459490502212497, - "nauc_precision_at_1_max": 0.19915884427724026, - "nauc_precision_at_1_std": -0.0022919130648232137, - "nauc_precision_at_20_diff1": 0.3011582354357301, - "nauc_precision_at_20_max": 0.3577860510810275, - "nauc_precision_at_20_std": 0.17380097220769716, - "nauc_precision_at_3_diff1": 0.3679652871898132, - "nauc_precision_at_3_max": 0.3346027211830583, - "nauc_precision_at_3_std": 0.09750594972545619, - "nauc_precision_at_5_diff1": 0.3469492559443356, - "nauc_precision_at_5_max": 0.38095109388152204, - "nauc_precision_at_5_std": 0.13474859644922846, - "nauc_recall_at_1000_diff1": 0.09881437072166871, - "nauc_recall_at_1000_max": 0.3361460337345095, - "nauc_recall_at_1000_std": 0.3314348351304203, - "nauc_recall_at_100_diff1": 0.21713821991445592, - "nauc_recall_at_100_max": 0.32597885179784103, - "nauc_recall_at_100_std": 0.29378672410321527, - "nauc_recall_at_10_diff1": 0.31827978460409795, - "nauc_recall_at_10_max": 0.3669826124892289, - "nauc_recall_at_10_std": 0.15130021075112898, - "nauc_recall_at_1_diff1": 0.459490502212497, - "nauc_recall_at_1_max": 0.19915884427724026, - "nauc_recall_at_1_std": -0.0022919130648232137, - "nauc_recall_at_20_diff1": 0.30115823543573067, - "nauc_recall_at_20_max": 0.3577860510810276, - "nauc_recall_at_20_std": 0.17380097220769894, - "nauc_recall_at_3_diff1": 0.3679652871898125, - "nauc_recall_at_3_max": 0.33460272118305795, - "nauc_recall_at_3_std": 0.09750594972545565, - "nauc_recall_at_5_diff1": 0.34694925594433695, - "nauc_recall_at_5_max": 0.38095109388152243, - "nauc_recall_at_5_std": 0.13474859644922935, - "ndcg_at_1": 0.457, - "ndcg_at_10": 0.61855, - "ndcg_at_100": 0.63523, - "ndcg_at_1000": 0.64198, - "ndcg_at_20": 0.62676, - "ndcg_at_3": 0.58456, - "ndcg_at_5": 0.60637, - "precision_at_1": 0.457, - "precision_at_10": 0.0761, - "precision_at_100": 0.00839, - "precision_at_1000": 0.0009, - "precision_at_20": 0.03965, - "precision_at_3": 0.22367, - "precision_at_5": 0.1448, - "recall_at_1": 0.457, - "recall_at_10": 0.761, - "recall_at_100": 0.839, - "recall_at_1000": 0.895, - "recall_at_20": 0.793, - "recall_at_3": 0.671, - "recall_at_5": 0.724 - } - ] - }, - "task_name": "HatefulMemesT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MNIST.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MNIST.json deleted file mode 100644 index a77358f9a5..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", - "evaluation_time": 162.98263120651245, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "accuracy": 0.9369799999999999, - "f1": 0.9364180161805986, - "f1_weighted": 0.9369031495544664, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9369799999999999, - "scores_per_experiment": [ - { - "accuracy": 0.9396, - "f1": 0.9390095708059392, - "f1_weighted": 0.9393176829760612 - }, - { - "accuracy": 0.9176, - "f1": 0.9161560756580137, - "f1_weighted": 0.9172825619473937 - }, - { - "accuracy": 0.9493, - "f1": 0.9490741086414646, - "f1_weighted": 0.9492979321844203 - }, - { - "accuracy": 0.9303, - "f1": 0.9302718818210677, - "f1_weighted": 0.9305303676361367 - }, - { - "accuracy": 0.9481, - "f1": 0.9475784439765083, - "f1_weighted": 0.9480872030283201 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOI2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOI2TRetrieval.json deleted file mode 100644 index 49b5d4ec5b..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOI2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", - "evaluation_time": 78.54339241981506, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.48795, - "map_at_1": 0.11196, - "map_at_10": 0.37316, - "map_at_100": 0.4174, - "map_at_1000": 0.42082, - "map_at_20": 0.39776, - "map_at_3": 0.25241, - "map_at_5": 0.32523, - "mrr_at_1": 0.5596, - "mrr_at_10": 0.6581655555555556, - "mrr_at_100": 0.6643201403638, - "mrr_at_1000": 0.6643768592817573, - "mrr_at_20": 0.6623790460837746, - "mrr_at_3": 0.633833333333335, - "mrr_at_5": 0.6485333333333335, - "nauc_map_at_1000_diff1": 0.27585543400346596, - "nauc_map_at_1000_max": 0.39045174229638263, - "nauc_map_at_1000_std": -0.31907954307019376, - "nauc_map_at_100_diff1": 0.2756901587956695, - "nauc_map_at_100_max": 0.3901590379869748, - "nauc_map_at_100_std": -0.31982716652037885, - "nauc_map_at_10_diff1": 0.2695596721841738, - "nauc_map_at_10_max": 0.38999760608673667, - "nauc_map_at_10_std": -0.3152421322220176, - "nauc_map_at_1_diff1": 0.4535097760243967, - "nauc_map_at_1_max": 0.3847697525357455, - "nauc_map_at_1_std": -0.3475010070412092, - "nauc_map_at_20_diff1": 0.27324757934344923, - "nauc_map_at_20_max": 0.3887973382937881, - "nauc_map_at_20_std": -0.3206806647695715, - "nauc_map_at_3_diff1": 0.30732032953704846, - "nauc_map_at_3_max": 0.3835772487850095, - "nauc_map_at_3_std": -0.31666427177733625, - "nauc_map_at_5_diff1": 0.27329624635564115, - "nauc_map_at_5_max": 0.3924020258946327, - "nauc_map_at_5_std": -0.30667966704337424, - "nauc_mrr_at_1000_diff1": 0.4409454696413627, - "nauc_mrr_at_1000_max": 0.38085560117885164, - "nauc_mrr_at_1000_std": -0.37551707053781974, - "nauc_mrr_at_100_diff1": 0.4409658468757943, - "nauc_mrr_at_100_max": 0.3808934906228772, - "nauc_mrr_at_100_std": -0.375496240900825, - "nauc_mrr_at_10_diff1": 0.43980000162065963, - "nauc_mrr_at_10_max": 0.3798372891177225, - "nauc_mrr_at_10_std": -0.3781076819730193, - "nauc_mrr_at_1_diff1": 0.453795105129665, - "nauc_mrr_at_1_max": 0.38541783132147206, - "nauc_mrr_at_1_std": -0.34743272717366425, - "nauc_mrr_at_20_diff1": 0.4406709449591336, - "nauc_mrr_at_20_max": 0.38057516342819747, - "nauc_mrr_at_20_std": -0.376140377873905, - "nauc_mrr_at_3_diff1": 0.4382924266992488, - "nauc_mrr_at_3_max": 0.3805844116331195, - "nauc_mrr_at_3_std": -0.3769284278433346, - "nauc_mrr_at_5_diff1": 0.4397654551355224, - "nauc_mrr_at_5_max": 0.3802589096353254, - "nauc_mrr_at_5_std": -0.3774278739567175, - "nauc_ndcg_at_1000_diff1": 0.32227128701745533, - "nauc_ndcg_at_1000_max": 0.38730359922065866, - "nauc_ndcg_at_1000_std": -0.33970752042293817, - "nauc_ndcg_at_100_diff1": 0.31453883156173185, - "nauc_ndcg_at_100_max": 0.3835118980277628, - "nauc_ndcg_at_100_std": -0.342167313367944, - "nauc_ndcg_at_10_diff1": 0.3018499732014452, - "nauc_ndcg_at_10_max": 0.38168035853082877, - "nauc_ndcg_at_10_std": -0.3432797044973395, - "nauc_ndcg_at_1_diff1": 0.453795105129665, - "nauc_ndcg_at_1_max": 0.38541783132147206, - "nauc_ndcg_at_1_std": -0.34743272717366425, - "nauc_ndcg_at_20_diff1": 0.3081597647895536, - "nauc_ndcg_at_20_max": 0.3790139927724731, - "nauc_ndcg_at_20_std": -0.3496269372983925, - "nauc_ndcg_at_3_diff1": 0.33292580807819666, - "nauc_ndcg_at_3_max": 0.38163227948458556, - "nauc_ndcg_at_3_std": -0.33633460176008967, - "nauc_ndcg_at_5_diff1": 0.30594814905303774, - "nauc_ndcg_at_5_max": 0.38448832915244796, - "nauc_ndcg_at_5_std": -0.33317544529631354, - "nauc_precision_at_1000_diff1": 0.11133163483146782, - "nauc_precision_at_1000_max": 0.3092422898807732, - "nauc_precision_at_1000_std": 0.4494367261142218, - "nauc_precision_at_100_diff1": 0.20407789253226402, - "nauc_precision_at_100_max": 0.29949976205541556, - "nauc_precision_at_100_std": -0.25144385212643716, - "nauc_precision_at_10_diff1": 0.24180001061503437, - "nauc_precision_at_10_max": 0.35458118880548456, - "nauc_precision_at_10_std": -0.3254216181230694, - "nauc_precision_at_1_diff1": 0.453795105129665, - "nauc_precision_at_1_max": 0.38541783132147206, - "nauc_precision_at_1_std": -0.34743272717366425, - "nauc_precision_at_20_diff1": 0.23979773773669036, - "nauc_precision_at_20_max": 0.33042944980132744, - "nauc_precision_at_20_std": -0.3243391967972116, - "nauc_precision_at_3_diff1": 0.2972609148045276, - "nauc_precision_at_3_max": 0.3788843739773271, - "nauc_precision_at_3_std": -0.33157111929715655, - "nauc_precision_at_5_diff1": 0.25833278252283215, - "nauc_precision_at_5_max": 0.37596098373156045, - "nauc_precision_at_5_std": -0.3212248673681845, - "nauc_recall_at_1000_diff1": 0.12173947593223243, - "nauc_recall_at_1000_max": 0.31204928780024443, - "nauc_recall_at_1000_std": 0.45505045990026477, - "nauc_recall_at_100_diff1": 0.20458582828997776, - "nauc_recall_at_100_max": 0.29902721698634105, - "nauc_recall_at_100_std": -0.2531368692860601, - "nauc_recall_at_10_diff1": 0.24178112080035385, - "nauc_recall_at_10_max": 0.354323008010846, - "nauc_recall_at_10_std": -0.32570952801324166, - "nauc_recall_at_1_diff1": 0.4535097760243967, - "nauc_recall_at_1_max": 0.3847697525357455, - "nauc_recall_at_1_std": -0.3475010070412092, - "nauc_recall_at_20_diff1": 0.2397438401654046, - "nauc_recall_at_20_max": 0.33015488732418014, - "nauc_recall_at_20_std": -0.32472267420389017, - "nauc_recall_at_3_diff1": 0.297280489755177, - "nauc_recall_at_3_max": 0.3786095196183429, - "nauc_recall_at_3_std": -0.33164997286157694, - "nauc_recall_at_5_diff1": 0.2582794667970118, - "nauc_recall_at_5_max": 0.375682734001994, - "nauc_recall_at_5_std": -0.32143238857573325, - "ndcg_at_1": 0.5596, - "ndcg_at_10": 0.48795, - "ndcg_at_100": 0.60605, - "ndcg_at_1000": 0.64058, - "ndcg_at_20": 0.53528, - "ndcg_at_3": 0.48541, - "ndcg_at_5": 0.4244, - "precision_at_1": 0.5596, - "precision_at_10": 0.2492, - "precision_at_100": 0.04177, - "precision_at_1000": 0.00496, - "precision_at_20": 0.15213, - "precision_at_3": 0.46293, - "precision_at_5": 0.38288, - "recall_at_1": 0.11196, - "recall_at_10": 0.4986, - "recall_at_100": 0.83586, - "recall_at_1000": 0.99248, - "recall_at_20": 0.6088, - "recall_at_3": 0.27789, - "recall_at_5": 0.38304 - } - ] - }, - "task_name": "MSCOCOI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOT2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOT2IRetrieval.json deleted file mode 100644 index cfb8d64243..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MSCOCOT2IRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", - "evaluation_time": 102.51849317550659, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.52318, - "map_at_1": 0.35829, - "map_at_10": 0.46471, - "map_at_100": 0.4758, - "map_at_1000": 0.47599, - "map_at_20": 0.47131, - "map_at_3": 0.43292, - "map_at_5": 0.45109, - "mrr_at_1": 0.35853923979201097, - "mrr_at_10": 0.4651022222478256, - "mrr_at_100": 0.47611295706758416, - "mrr_at_1000": 0.47629480544722647, - "mrr_at_20": 0.4716961247971015, - "mrr_at_3": 0.43328362019160016, - "mrr_at_5": 0.45149273785051924, - "nauc_map_at_1000_diff1": 0.5364389960050693, - "nauc_map_at_1000_max": 0.27063297333846825, - "nauc_map_at_1000_std": -0.27269894766508856, - "nauc_map_at_100_diff1": 0.5363642248904831, - "nauc_map_at_100_max": 0.2707039659874514, - "nauc_map_at_100_std": -0.27255201916852684, - "nauc_map_at_10_diff1": 0.5335239245537733, - "nauc_map_at_10_max": 0.26812050699895346, - "nauc_map_at_10_std": -0.2755012154401604, - "nauc_map_at_1_diff1": 0.5905593031641301, - "nauc_map_at_1_max": 0.2895208438254399, - "nauc_map_at_1_std": -0.253486375459467, - "nauc_map_at_20_diff1": 0.5348655481247181, - "nauc_map_at_20_max": 0.2693117996470176, - "nauc_map_at_20_std": -0.274396735120252, - "nauc_map_at_3_diff1": 0.5393950231886427, - "nauc_map_at_3_max": 0.27217487369108845, - "nauc_map_at_3_std": -0.27233322179292724, - "nauc_map_at_5_diff1": 0.5344534524377408, - "nauc_map_at_5_max": 0.2687999532811149, - "nauc_map_at_5_std": -0.27534165682431466, - "nauc_mrr_at_1000_diff1": 0.5360368110159093, - "nauc_mrr_at_1000_max": 0.27076830927500367, - "nauc_mrr_at_1000_std": -0.27230173708749605, - "nauc_mrr_at_100_diff1": 0.5359631708332794, - "nauc_mrr_at_100_max": 0.27084424913172367, - "nauc_mrr_at_100_std": -0.27214703103659804, - "nauc_mrr_at_10_diff1": 0.5330918181125404, - "nauc_mrr_at_10_max": 0.26837418551663855, - "nauc_mrr_at_10_std": -0.27488930900694575, - "nauc_mrr_at_1_diff1": 0.5901385036140094, - "nauc_mrr_at_1_max": 0.28945880832060467, - "nauc_mrr_at_1_std": -0.2534654694762397, - "nauc_mrr_at_20_diff1": 0.5344539765635075, - "nauc_mrr_at_20_max": 0.26958226442134897, - "nauc_mrr_at_20_std": -0.27382433915008214, - "nauc_mrr_at_3_diff1": 0.538923422488798, - "nauc_mrr_at_3_max": 0.2723350955816106, - "nauc_mrr_at_3_std": -0.2718935761838524, - "nauc_mrr_at_5_diff1": 0.533959843918952, - "nauc_mrr_at_5_max": 0.26905895384706296, - "nauc_mrr_at_5_std": -0.2747297894605437, - "nauc_ndcg_at_1000_diff1": 0.5269109631077971, - "nauc_ndcg_at_1000_max": 0.2691664483433101, - "nauc_ndcg_at_1000_std": -0.27198727370439346, - "nauc_ndcg_at_100_diff1": 0.5250688847438519, - "nauc_ndcg_at_100_max": 0.2716629727071101, - "nauc_ndcg_at_100_std": -0.2666531261354938, - "nauc_ndcg_at_10_diff1": 0.5100643637960702, - "nauc_ndcg_at_10_max": 0.25728575887213956, - "nauc_ndcg_at_10_std": -0.2851559836697289, - "nauc_ndcg_at_1_diff1": 0.5901385036140094, - "nauc_ndcg_at_1_max": 0.28945880832060467, - "nauc_ndcg_at_1_std": -0.2534654694762397, - "nauc_ndcg_at_20_diff1": 0.5141405251743207, - "nauc_ndcg_at_20_max": 0.2612554258422001, - "nauc_ndcg_at_20_std": -0.2812613383940013, - "nauc_ndcg_at_3_diff1": 0.5234248839202699, - "nauc_ndcg_at_3_max": 0.26623604986271715, - "nauc_ndcg_at_3_std": -0.2785040097130991, - "nauc_ndcg_at_5_diff1": 0.5137575676531829, - "nauc_ndcg_at_5_max": 0.2598174091166936, - "nauc_ndcg_at_5_std": -0.28407927219401274, - "nauc_precision_at_1000_diff1": -0.138201035126385, - "nauc_precision_at_1000_max": 0.2679227808873947, - "nauc_precision_at_1000_std": 0.5849397960968612, - "nauc_precision_at_100_diff1": 0.23962363566765063, - "nauc_precision_at_100_max": 0.4075927257460724, - "nauc_precision_at_100_std": 0.2742652644802499, - "nauc_precision_at_10_diff1": 0.4117501094295695, - "nauc_precision_at_10_max": 0.2126322069047596, - "nauc_precision_at_10_std": -0.31894957843005256, - "nauc_precision_at_1_diff1": 0.5901385036140094, - "nauc_precision_at_1_max": 0.28945880832060467, - "nauc_precision_at_1_std": -0.2534654694762397, - "nauc_precision_at_20_diff1": 0.3961829922935417, - "nauc_precision_at_20_max": 0.2209834925530759, - "nauc_precision_at_20_std": -0.2985831435827799, - "nauc_precision_at_3_diff1": 0.4756800519614237, - "nauc_precision_at_3_max": 0.24830374451491583, - "nauc_precision_at_3_std": -0.29648946830298284, - "nauc_precision_at_5_diff1": 0.44476767482754076, - "nauc_precision_at_5_max": 0.2298515223070273, - "nauc_precision_at_5_std": -0.31137543613145, - "nauc_recall_at_1000_diff1": 0.4494321491065472, - "nauc_recall_at_1000_max": 0.9332926952790045, - "nauc_recall_at_1000_std": 0.9655409418898824, - "nauc_recall_at_100_diff1": 0.4033274574532679, - "nauc_recall_at_100_max": 0.48101784678152426, - "nauc_recall_at_100_std": 0.17496166411367567, - "nauc_recall_at_10_diff1": 0.41658984626823475, - "nauc_recall_at_10_max": 0.21186835042904034, - "nauc_recall_at_10_std": -0.3250128096915894, - "nauc_recall_at_1_diff1": 0.5905593031641301, - "nauc_recall_at_1_max": 0.2895208438254399, - "nauc_recall_at_1_std": -0.253486375459467, - "nauc_recall_at_20_diff1": 0.4079171828509436, - "nauc_recall_at_20_max": 0.2192969350421769, - "nauc_recall_at_20_std": -0.31531173454663586, - "nauc_recall_at_3_diff1": 0.4765351052198576, - "nauc_recall_at_3_max": 0.2481479402236204, - "nauc_recall_at_3_std": -0.2973941516041928, - "nauc_recall_at_5_diff1": 0.4467425425914727, - "nauc_recall_at_5_max": 0.22955437013140248, - "nauc_recall_at_5_std": -0.3134480016683754, - "ndcg_at_1": 0.35854, - "ndcg_at_10": 0.52318, - "ndcg_at_100": 0.57825, - "ndcg_at_1000": 0.58224, - "ndcg_at_20": 0.54712, - "ndcg_at_3": 0.45743, - "ndcg_at_5": 0.49017, - "precision_at_1": 0.35854, - "precision_at_10": 0.07112, - "precision_at_100": 0.00975, - "precision_at_1000": 0.001, - "precision_at_20": 0.04033, - "precision_at_3": 0.17621, - "precision_at_5": 0.1217, - "recall_at_1": 0.35829, - "recall_at_10": 0.70943, - "recall_at_100": 0.96845, - "recall_at_1000": 0.99797, - "recall_at_20": 0.80373, - "recall_at_3": 0.52801, - "recall_at_5": 0.6074 - } - ] - }, - "task_name": "MSCOCOT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionI2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionI2TRetrieval.json deleted file mode 100644 index 54da1c305d..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", - "evaluation_time": 19.325831651687622, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.66571, - "cv_recall_at_10": 0.86514, - "cv_recall_at_100": 0.9627, - "cv_recall_at_1000": 0.99139, - "cv_recall_at_20": 0.90387, - "cv_recall_at_3": 0.77762, - "cv_recall_at_5": 0.81492, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7633, - "map_at_1": 0.66571, - "map_at_10": 0.73104, - "map_at_100": 0.73511, - "map_at_1000": 0.73523, - "map_at_20": 0.73364, - "map_at_3": 0.71593, - "map_at_5": 0.72439, - "mrr_at_1": 0.6657101865136298, - "mrr_at_10": 0.7310355036323473, - "mrr_at_100": 0.7351134678303535, - "mrr_at_1000": 0.7352377709339848, - "mrr_at_20": 0.7336445586812597, - "mrr_at_3": 0.7159253945480629, - "mrr_at_5": 0.7243902439024387, - "nauc_cv_recall_at_1000_diff1": 0.6899284883121061, - "nauc_cv_recall_at_1000_max": 0.7332667407845029, - "nauc_cv_recall_at_1000_std": 0.9039084682819682, - "nauc_cv_recall_at_100_diff1": 0.7201839624155785, - "nauc_cv_recall_at_100_max": 0.6907472572724536, - "nauc_cv_recall_at_100_std": 0.3332725327931793, - "nauc_cv_recall_at_10_diff1": 0.7077842881191173, - "nauc_cv_recall_at_10_max": 0.6440495623955013, - "nauc_cv_recall_at_10_std": 0.22920450783080562, - "nauc_cv_recall_at_1_diff1": 0.8247920662804782, - "nauc_cv_recall_at_1_max": 0.6475202783168332, - "nauc_cv_recall_at_1_std": 0.18738440780550997, - "nauc_cv_recall_at_20_diff1": 0.7505468149526092, - "nauc_cv_recall_at_20_max": 0.71884925093909, - "nauc_cv_recall_at_20_std": 0.33390742592073563, - "nauc_cv_recall_at_3_diff1": 0.7864899607118706, - "nauc_cv_recall_at_3_max": 0.6854512548125032, - "nauc_cv_recall_at_3_std": 0.2128102726462487, - "nauc_cv_recall_at_5_diff1": 0.7639985062373839, - "nauc_cv_recall_at_5_max": 0.6612189478923202, - "nauc_cv_recall_at_5_std": 0.22878287705869715, - "nauc_map_at_1000_diff1": 0.8061418496180208, - "nauc_map_at_1000_max": 0.6558110507203014, - "nauc_map_at_1000_std": 0.19895557515971368, - "nauc_map_at_100_diff1": 0.8060998402466953, - "nauc_map_at_100_max": 0.6558378199002864, - "nauc_map_at_100_std": 0.19895388464990604, - "nauc_map_at_10_diff1": 0.8039242682913358, - "nauc_map_at_10_max": 0.6543112939934062, - "nauc_map_at_10_std": 0.19680737714854116, - "nauc_map_at_1_diff1": 0.8247920662804782, - "nauc_map_at_1_max": 0.6475202783168332, - "nauc_map_at_1_std": 0.18738440780550997, - "nauc_map_at_20_diff1": 0.8060382107450633, - "nauc_map_at_20_max": 0.6563399027924791, - "nauc_map_at_20_std": 0.19964630412672676, - "nauc_map_at_3_diff1": 0.8102967459949728, - "nauc_map_at_3_max": 0.6604941579272117, - "nauc_map_at_3_std": 0.1949003281855533, - "nauc_map_at_5_diff1": 0.80709000272496, - "nauc_map_at_5_max": 0.656058091855951, - "nauc_map_at_5_std": 0.19791335976271474, - "nauc_mrr_at_1000_diff1": 0.8061349415976898, - "nauc_mrr_at_1000_max": 0.6558011266170611, - "nauc_mrr_at_1000_std": 0.19893648517694507, - "nauc_mrr_at_100_diff1": 0.8060929351394873, - "nauc_mrr_at_100_max": 0.6558279017289936, - "nauc_mrr_at_100_std": 0.19893480501137517, - "nauc_mrr_at_10_diff1": 0.8039242682913358, - "nauc_mrr_at_10_max": 0.6543112939934062, - "nauc_mrr_at_10_std": 0.19680737714854116, - "nauc_mrr_at_1_diff1": 0.8247920662804782, - "nauc_mrr_at_1_max": 0.6475202783168332, - "nauc_mrr_at_1_std": 0.18738440780550997, - "nauc_mrr_at_20_diff1": 0.8060313484873587, - "nauc_mrr_at_20_max": 0.6563300579354437, - "nauc_mrr_at_20_std": 0.19962736007518378, - "nauc_mrr_at_3_diff1": 0.8102967459949728, - "nauc_mrr_at_3_max": 0.6604941579272117, - "nauc_mrr_at_3_std": 0.1949003281855533, - "nauc_mrr_at_5_diff1": 0.80709000272496, - "nauc_mrr_at_5_max": 0.656058091855951, - "nauc_mrr_at_5_std": 0.19791335976271474, - "nauc_ndcg_at_1000_diff1": 0.8006827413362921, - "nauc_ndcg_at_1000_max": 0.6578663386654083, - "nauc_ndcg_at_1000_std": 0.20815645113293776, - "nauc_ndcg_at_100_diff1": 0.7993042173321111, - "nauc_ndcg_at_100_max": 0.6583569899337858, - "nauc_ndcg_at_100_std": 0.2073560131798155, - "nauc_ndcg_at_10_diff1": 0.7895163176975294, - "nauc_ndcg_at_10_max": 0.6533321610894864, - "nauc_ndcg_at_10_std": 0.2020008736673157, - "nauc_ndcg_at_1_diff1": 0.8247920662804782, - "nauc_ndcg_at_1_max": 0.6475202783168332, - "nauc_ndcg_at_1_std": 0.18738440780550997, - "nauc_ndcg_at_20_diff1": 0.7981126769738102, - "nauc_ndcg_at_20_max": 0.6625089082782497, - "nauc_ndcg_at_20_std": 0.21380178270542483, - "nauc_ndcg_at_3_diff1": 0.8049686594776644, - "nauc_ndcg_at_3_max": 0.6660170615545122, - "nauc_ndcg_at_3_std": 0.19881314216111726, - "nauc_ndcg_at_5_diff1": 0.7986014653829432, - "nauc_ndcg_at_5_max": 0.6574369041289396, - "nauc_ndcg_at_5_std": 0.20400546764153435, - "nauc_precision_at_1000_diff1": 0.6899284883120989, - "nauc_precision_at_1000_max": 0.7332667407844798, - "nauc_precision_at_1000_std": 0.9039084682819483, - "nauc_precision_at_100_diff1": 0.7201839624155809, - "nauc_precision_at_100_max": 0.6907472572724539, - "nauc_precision_at_100_std": 0.33327253279318164, - "nauc_precision_at_10_diff1": 0.7077842881191184, - "nauc_precision_at_10_max": 0.6440495623955018, - "nauc_precision_at_10_std": 0.22920450783080423, - "nauc_precision_at_1_diff1": 0.8247920662804782, - "nauc_precision_at_1_max": 0.6475202783168332, - "nauc_precision_at_1_std": 0.18738440780550997, - "nauc_precision_at_20_diff1": 0.7505468149526078, - "nauc_precision_at_20_max": 0.7188492509390902, - "nauc_precision_at_20_std": 0.33390742592073597, - "nauc_precision_at_3_diff1": 0.786489960711871, - "nauc_precision_at_3_max": 0.6854512548125032, - "nauc_precision_at_3_std": 0.21281027264624763, - "nauc_precision_at_5_diff1": 0.763998506237385, - "nauc_precision_at_5_max": 0.661218947892321, - "nauc_precision_at_5_std": 0.22878287705869702, - "nauc_recall_at_1000_diff1": 0.6899284883121061, - "nauc_recall_at_1000_max": 0.7332667407845029, - "nauc_recall_at_1000_std": 0.9039084682819682, - "nauc_recall_at_100_diff1": 0.7201839624155785, - "nauc_recall_at_100_max": 0.6907472572724536, - "nauc_recall_at_100_std": 0.3332725327931793, - "nauc_recall_at_10_diff1": 0.7077842881191173, - "nauc_recall_at_10_max": 0.6440495623955013, - "nauc_recall_at_10_std": 0.22920450783080562, - "nauc_recall_at_1_diff1": 0.8247920662804782, - "nauc_recall_at_1_max": 0.6475202783168332, - "nauc_recall_at_1_std": 0.18738440780550997, - "nauc_recall_at_20_diff1": 0.7505468149526092, - "nauc_recall_at_20_max": 0.71884925093909, - "nauc_recall_at_20_std": 0.33390742592073563, - "nauc_recall_at_3_diff1": 0.7864899607118706, - "nauc_recall_at_3_max": 0.6854512548125032, - "nauc_recall_at_3_std": 0.2128102726462487, - "nauc_recall_at_5_diff1": 0.7639985062373839, - "nauc_recall_at_5_max": 0.6612189478923202, - "nauc_recall_at_5_std": 0.22878287705869715, - "ndcg_at_1": 0.66571, - "ndcg_at_10": 0.7633, - "ndcg_at_100": 0.78382, - "ndcg_at_1000": 0.78738, - "ndcg_at_20": 0.77298, - "ndcg_at_3": 0.73181, - "ndcg_at_5": 0.74712, - "precision_at_1": 0.66571, - "precision_at_10": 0.08651, - "precision_at_100": 0.00963, - "precision_at_1000": 0.00099, - "precision_at_20": 0.04519, - "precision_at_3": 0.25921, - "precision_at_5": 0.16298, - "recall_at_1": 0.66571, - "recall_at_10": 0.86514, - "recall_at_100": 0.9627, - "recall_at_1000": 0.99139, - "recall_at_20": 0.90387, - "recall_at_3": 0.77762, - "recall_at_5": 0.81492 - } - ] - }, - "task_name": "MemotionI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionT2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionT2IRetrieval.json deleted file mode 100644 index cc31450ae6..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/MemotionT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cdb15b61d84d56db73e0e59535dfea81ea3c22f4", - "evaluation_time": 104.71015620231628, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.80057, - "cv_recall_at_10": 0.92539, - "cv_recall_at_100": 0.97131, - "cv_recall_at_1000": 0.99139, - "cv_recall_at_20": 0.94692, - "cv_recall_at_3": 0.88235, - "cv_recall_at_5": 0.90387, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.86489, - "map_at_1": 0.79914, - "map_at_10": 0.84527, - "map_at_100": 0.84754, - "map_at_1000": 0.84764, - "map_at_20": 0.84696, - "map_at_3": 0.83716, - "map_at_5": 0.8424, - "mrr_at_1": 0.8005738880918221, - "mrr_at_10": 0.8461034820432242, - "mrr_at_100": 0.8483797957589674, - "mrr_at_1000": 0.8484791559369318, - "mrr_at_20": 0.8477932437401593, - "mrr_at_3": 0.8383548541367767, - "mrr_at_5": 0.8432329029172645, - "nauc_cv_recall_at_1000_diff1": 0.8175387148222494, - "nauc_cv_recall_at_1000_max": 0.7813301097103972, - "nauc_cv_recall_at_1000_std": 0.6193570292722026, - "nauc_cv_recall_at_100_diff1": 0.8263292349023955, - "nauc_cv_recall_at_100_max": 0.5635539288551284, - "nauc_cv_recall_at_100_std": 0.3522543346587812, - "nauc_cv_recall_at_10_diff1": 0.8295473346563693, - "nauc_cv_recall_at_10_max": 0.6923738450403997, - "nauc_cv_recall_at_10_std": 0.389947956274333, - "nauc_cv_recall_at_1_diff1": 0.893555032247032, - "nauc_cv_recall_at_1_max": 0.5814690622558578, - "nauc_cv_recall_at_1_std": 0.23603882918725352, - "nauc_cv_recall_at_20_diff1": 0.8490190393318788, - "nauc_cv_recall_at_20_max": 0.6828827824898038, - "nauc_cv_recall_at_20_std": 0.4875361417563047, - "nauc_cv_recall_at_3_diff1": 0.848738389948679, - "nauc_cv_recall_at_3_max": 0.6746014172072109, - "nauc_cv_recall_at_3_std": 0.28114809635552956, - "nauc_cv_recall_at_5_diff1": 0.8356168637564215, - "nauc_cv_recall_at_5_max": 0.6556750572467132, - "nauc_cv_recall_at_5_std": 0.2804939414519074, - "nauc_map_at_1000_diff1": 0.8805921183146161, - "nauc_map_at_1000_max": 0.613148597740671, - "nauc_map_at_1000_std": 0.2610442510570384, - "nauc_map_at_100_diff1": 0.8805784135077193, - "nauc_map_at_100_max": 0.6129179080060736, - "nauc_map_at_100_std": 0.26086055894944565, - "nauc_map_at_10_diff1": 0.8794830794153383, - "nauc_map_at_10_max": 0.6148311342515017, - "nauc_map_at_10_std": 0.2610332258087086, - "nauc_map_at_1_diff1": 0.8962682754647417, - "nauc_map_at_1_max": 0.5837819476683158, - "nauc_map_at_1_std": 0.24138500178899677, - "nauc_map_at_20_diff1": 0.8806350178255765, - "nauc_map_at_20_max": 0.6136795206917912, - "nauc_map_at_20_std": 0.26241957265256216, - "nauc_map_at_3_diff1": 0.8806059823643008, - "nauc_map_at_3_max": 0.6163799755044372, - "nauc_map_at_3_std": 0.25270335150389284, - "nauc_map_at_5_diff1": 0.8790830385851995, - "nauc_map_at_5_max": 0.6127564220549455, - "nauc_map_at_5_std": 0.2534054339713012, - "nauc_mrr_at_1000_diff1": 0.8785846990485243, - "nauc_mrr_at_1000_max": 0.6119022395966969, - "nauc_mrr_at_1000_std": 0.2583100845365849, - "nauc_mrr_at_100_diff1": 0.8785723188754336, - "nauc_mrr_at_100_max": 0.6116714134549016, - "nauc_mrr_at_100_std": 0.25812748443787986, - "nauc_mrr_at_10_diff1": 0.8775035197699672, - "nauc_mrr_at_10_max": 0.6136121471258943, - "nauc_mrr_at_10_std": 0.2583433756438642, - "nauc_mrr_at_1_diff1": 0.893555032247032, - "nauc_mrr_at_1_max": 0.5814690622558578, - "nauc_mrr_at_1_std": 0.23603882918725352, - "nauc_mrr_at_20_diff1": 0.8786372968530105, - "nauc_mrr_at_20_max": 0.612441351461196, - "nauc_mrr_at_20_std": 0.25970427563020676, - "nauc_mrr_at_3_diff1": 0.8781764404997089, - "nauc_mrr_at_3_max": 0.6156394406227684, - "nauc_mrr_at_3_std": 0.25193373541652625, - "nauc_mrr_at_5_diff1": 0.877139876269722, - "nauc_mrr_at_5_max": 0.6115522294704427, - "nauc_mrr_at_5_std": 0.2507355224549089, - "nauc_ndcg_at_1000_diff1": 0.8768801815238092, - "nauc_ndcg_at_1000_max": 0.6192014315122151, - "nauc_ndcg_at_1000_std": 0.27264225668380687, - "nauc_ndcg_at_100_diff1": 0.8760097922537312, - "nauc_ndcg_at_100_max": 0.6151797528361204, - "nauc_ndcg_at_100_std": 0.2708666025109306, - "nauc_ndcg_at_10_diff1": 0.8720122547537932, - "nauc_ndcg_at_10_max": 0.6265374448079035, - "nauc_ndcg_at_10_std": 0.2783746570432192, - "nauc_ndcg_at_1_diff1": 0.8962682754647417, - "nauc_ndcg_at_1_max": 0.5837819476683158, - "nauc_ndcg_at_1_std": 0.24138500178899677, - "nauc_ndcg_at_20_diff1": 0.8761941265114012, - "nauc_ndcg_at_20_max": 0.6224642007356991, - "nauc_ndcg_at_20_std": 0.2844421174644763, - "nauc_ndcg_at_3_diff1": 0.8746287968004955, - "nauc_ndcg_at_3_max": 0.6280145743162838, - "nauc_ndcg_at_3_std": 0.25660331622091054, - "nauc_ndcg_at_5_diff1": 0.8712648393471466, - "nauc_ndcg_at_5_max": 0.6210730988223663, - "nauc_ndcg_at_5_std": 0.25831577923580146, - "nauc_precision_at_1000_diff1": 0.8175387148222356, - "nauc_precision_at_1000_max": 0.7813301097103712, - "nauc_precision_at_1000_std": 0.6193570292721997, - "nauc_precision_at_100_diff1": 0.8263292349023921, - "nauc_precision_at_100_max": 0.5635539288551251, - "nauc_precision_at_100_std": 0.3522543346587806, - "nauc_precision_at_10_diff1": 0.8295473346563669, - "nauc_precision_at_10_max": 0.6923738450404027, - "nauc_precision_at_10_std": 0.3899479562743347, - "nauc_precision_at_1_diff1": 0.8962682754647417, - "nauc_precision_at_1_max": 0.5837819476683158, - "nauc_precision_at_1_std": 0.24138500178899677, - "nauc_precision_at_20_diff1": 0.8490190393318748, - "nauc_precision_at_20_max": 0.6828827824898049, - "nauc_precision_at_20_std": 0.4875361417562997, - "nauc_precision_at_3_diff1": 0.8519642374715442, - "nauc_precision_at_3_max": 0.6717930971542201, - "nauc_precision_at_3_std": 0.2712153905918118, - "nauc_precision_at_5_diff1": 0.8356168637564203, - "nauc_precision_at_5_max": 0.6556750572467118, - "nauc_precision_at_5_std": 0.2804939414519041, - "nauc_recall_at_1000_diff1": 0.8175387148222494, - "nauc_recall_at_1000_max": 0.7813301097103972, - "nauc_recall_at_1000_std": 0.6193570292722026, - "nauc_recall_at_100_diff1": 0.8263292349023955, - "nauc_recall_at_100_max": 0.5635539288551284, - "nauc_recall_at_100_std": 0.3522543346587812, - "nauc_recall_at_10_diff1": 0.8295473346563693, - "nauc_recall_at_10_max": 0.6923738450403997, - "nauc_recall_at_10_std": 0.389947956274333, - "nauc_recall_at_1_diff1": 0.8962682754647417, - "nauc_recall_at_1_max": 0.5837819476683158, - "nauc_recall_at_1_std": 0.24138500178899677, - "nauc_recall_at_20_diff1": 0.8490190393318788, - "nauc_recall_at_20_max": 0.6828827824898038, - "nauc_recall_at_20_std": 0.4875361417563047, - "nauc_recall_at_3_diff1": 0.8519642374715442, - "nauc_recall_at_3_max": 0.6717930971542223, - "nauc_recall_at_3_std": 0.27121539059181127, - "nauc_recall_at_5_diff1": 0.8356168637564215, - "nauc_recall_at_5_max": 0.6556750572467132, - "nauc_recall_at_5_std": 0.2804939414519074, - "ndcg_at_1": 0.79914, - "ndcg_at_10": 0.86489, - "ndcg_at_100": 0.87503, - "ndcg_at_1000": 0.87761, - "ndcg_at_20": 0.87058, - "ndcg_at_3": 0.84848, - "ndcg_at_5": 0.85793, - "precision_at_1": 0.79914, - "precision_at_10": 0.09254, - "precision_at_100": 0.00971, - "precision_at_1000": 0.00099, - "precision_at_20": 0.04735, - "precision_at_3": 0.29364, - "precision_at_5": 0.18077, - "recall_at_1": 0.79914, - "recall_at_10": 0.92539, - "recall_at_100": 0.97131, - "recall_at_1000": 0.99139, - "recall_at_20": 0.94692, - "recall_at_3": 0.88092, - "recall_at_5": 0.90387 - } - ] - }, - "task_name": "MemotionT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/RenderedSST2.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/RenderedSST2.json deleted file mode 100644 index dafe3400a8..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/RenderedSST2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", - "evaluation_time": 27.908196687698364, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "accuracy": 0.6979681493684788, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6979681493684788 - } - ] - }, - "task_name": "RenderedSST2" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS12VisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS12VisualSTS.json deleted file mode 100644 index 9c16b4a119..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS12VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "820c25edfba736f3789201b2476208cc62c2ccb9", - "evaluation_time": 87.90539646148682, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.4405211392977615, - "cosine_spearman": 0.5389032781902258, - "euclidean_pearson": 0.5399657862990608, - "euclidean_spearman": 0.5974928829513158, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5389032781902258, - "manhattan_pearson": 0.5436282415345026, - "manhattan_spearman": 0.5999630768327725, - "pearson": 0.4405211392977615, - "spearman": 0.5389032781902258 - } - ] - }, - "task_name": "STS12VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS13VisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS13VisualSTS.json deleted file mode 100644 index c0b48421cf..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS13VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "561ee9ca47ff3e4a657283c59416deca8dc169f2", - "evaluation_time": 42.32037687301636, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.6245913368723488, - "cosine_spearman": 0.6677606349866827, - "euclidean_pearson": 0.6979935416065853, - "euclidean_spearman": 0.7086329137634516, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6677606349866827, - "manhattan_pearson": 0.7114180697383635, - "manhattan_spearman": 0.719152354710675, - "pearson": 0.6245913368723488, - "spearman": 0.6677606349866827 - } - ] - }, - "task_name": "STS13VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS14VisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS14VisualSTS.json deleted file mode 100644 index 8d36ba6127..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS14VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "824e95e45471024a684b901e0645579ffd9ca288", - "evaluation_time": 106.82659244537354, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.5453585198353686, - "cosine_spearman": 0.5598248304719118, - "euclidean_pearson": 0.6150781984784081, - "euclidean_spearman": 0.6101318963139828, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5598248304719118, - "manhattan_pearson": 0.6264295102013351, - "manhattan_spearman": 0.6204680830805613, - "pearson": 0.5453585198353686, - "spearman": 0.5598248304719118 - } - ] - }, - "task_name": "STS14VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS15VisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS15VisualSTS.json deleted file mode 100644 index 007949f7e9..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS15VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "1f8d08d9b9daac7118dfdefeb94b0aac4baf2e5f", - "evaluation_time": 85.88353157043457, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.6929759616177938, - "cosine_spearman": 0.7202791619453712, - "euclidean_pearson": 0.7518673935267203, - "euclidean_spearman": 0.7550769421651201, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7202791619453712, - "manhattan_pearson": 0.756112007453974, - "manhattan_spearman": 0.7594251311725545, - "pearson": 0.6929759616177938, - "spearman": 0.7202791619453712 - } - ] - }, - "task_name": "STS15VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS16VisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS16VisualSTS.json deleted file mode 100644 index ddfcfad75d..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS16VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "fc354f19598af93f32c0af1b94046ffdeaacde15", - "evaluation_time": 33.86677646636963, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.6849728559170047, - "cosine_spearman": 0.7049303888061984, - "euclidean_pearson": 0.7334630202312067, - "euclidean_spearman": 0.7352518330560981, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7049303888061984, - "manhattan_pearson": 0.7392596027406768, - "manhattan_spearman": 0.7411917628510096, - "pearson": 0.6849728559170047, - "spearman": 0.7049303888061984 - } - ] - }, - "task_name": "STS16VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS17MultilingualVisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS17MultilingualVisualSTS.json deleted file mode 100644 index 1a8e2b87dd..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STS17MultilingualVisualSTS.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", - "evaluation_time": 157.03906059265137, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.07658049577791673, - "cosine_spearman": 0.11072720883928548, - "euclidean_pearson": 0.1078991145581461, - "euclidean_spearman": 0.11751897940369754, - "hf_subset": "ko-ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.11072720883928548, - "manhattan_pearson": 0.10677695356608186, - "manhattan_spearman": 0.11595152029180494, - "pearson": 0.07658049577791673, - "spearman": 0.11072720883928548 - }, - { - "cosine_pearson": 0.29431080534890053, - "cosine_spearman": 0.39115449274164804, - "euclidean_pearson": 0.39214676108210245, - "euclidean_spearman": 0.3865876678489144, - "hf_subset": "ar-ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.39115449274164804, - "manhattan_pearson": 0.3971512172702036, - "manhattan_spearman": 0.39399986650659113, - "pearson": 0.29431080534890053, - "spearman": 0.39115449274164804 - }, - { - "cosine_pearson": 0.21213474813505678, - "cosine_spearman": 0.18952058277809578, - "euclidean_pearson": 0.19447082478701705, - "euclidean_spearman": 0.1638631082203305, - "hf_subset": "en-ar", - "languages": [ - "eng-Latn", - "ara-Arab" - ], - "main_score": 0.18952058277809578, - "manhattan_pearson": 0.2078261530517104, - "manhattan_spearman": 0.18518725377919415, - "pearson": 0.21213474813505678, - "spearman": 0.18952058277809578 - }, - { - "cosine_pearson": 0.44553015622322445, - "cosine_spearman": 0.457078250950972, - "euclidean_pearson": 0.4972439157265751, - "euclidean_spearman": 0.5086700340533576, - "hf_subset": "en-de", - "languages": [ - "eng-Latn", - "deu-Latn" - ], - "main_score": 0.457078250950972, - "manhattan_pearson": 0.49637286566496275, - "manhattan_spearman": 0.5067761122462164, - "pearson": 0.44553015622322445, - "spearman": 0.457078250950972 - }, - { - "cosine_pearson": 0.7227717533500028, - "cosine_spearman": 0.7526100073476201, - "euclidean_pearson": 0.7691084686390968, - "euclidean_spearman": 0.7745390617932849, - "hf_subset": "en-en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7526100073476201, - "manhattan_pearson": 0.7737367747407056, - "manhattan_spearman": 0.7768901802649685, - "pearson": 0.7227717533500028, - "spearman": 0.7526100073476201 - }, - { - "cosine_pearson": 0.3868402813159765, - "cosine_spearman": 0.39701995157424336, - "euclidean_pearson": 0.40798422293969827, - "euclidean_spearman": 0.41023229143697265, - "hf_subset": "en-tr", - "languages": [ - "eng-Latn", - "tur-Latn" - ], - "main_score": 0.39701995157424336, - "manhattan_pearson": 0.41772293269264277, - "manhattan_spearman": 0.42169259775526863, - "pearson": 0.3868402813159765, - "spearman": 0.39701995157424336 - }, - { - "cosine_pearson": 0.3733755575936537, - "cosine_spearman": 0.3675712091760419, - "euclidean_pearson": 0.40908137483711915, - "euclidean_spearman": 0.3963751320439048, - "hf_subset": "es-en", - "languages": [ - "spa-Latn", - "eng-Latn" - ], - "main_score": 0.3675712091760419, - "manhattan_pearson": 0.4485532114750566, - "manhattan_spearman": 0.43086210741364434, - "pearson": 0.3733755575936537, - "spearman": 0.3675712091760419 - }, - { - "cosine_pearson": 0.685661509838329, - "cosine_spearman": 0.7010571783071657, - "euclidean_pearson": 0.7446173298221099, - "euclidean_spearman": 0.7165276802273365, - "hf_subset": "es-es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.7010571783071657, - "manhattan_pearson": 0.7504667158699667, - "manhattan_spearman": 0.7237947747962055, - "pearson": 0.685661509838329, - "spearman": 0.7010571783071657 - }, - { - "cosine_pearson": 0.4394450113967274, - "cosine_spearman": 0.4406322289098256, - "euclidean_pearson": 0.4661773590688362, - "euclidean_spearman": 0.4746951448866896, - "hf_subset": "fr-en", - "languages": [ - "fra-Latn", - "eng-Latn" - ], - "main_score": 0.4406322289098256, - "manhattan_pearson": 0.47363965221268445, - "manhattan_spearman": 0.4882408949499833, - "pearson": 0.4394450113967274, - "spearman": 0.4406322289098256 - }, - { - "cosine_pearson": 0.40548839627224753, - "cosine_spearman": 0.40166825690676916, - "euclidean_pearson": 0.4454998522066273, - "euclidean_spearman": 0.439994899428145, - "hf_subset": "it-en", - "languages": [ - "ita-Latn", - "eng-Latn" - ], - "main_score": 0.40166825690676916, - "manhattan_pearson": 0.45497014049754414, - "manhattan_spearman": 0.4528287470788499, - "pearson": 0.40548839627224753, - "spearman": 0.40166825690676916 - }, - { - "cosine_pearson": 0.4129668284215636, - "cosine_spearman": 0.41634720800262875, - "euclidean_pearson": 0.44891021370876105, - "euclidean_spearman": 0.44611833648675253, - "hf_subset": "nl-en", - "languages": [ - "nld-Latn", - "eng-Latn" - ], - "main_score": 0.41634720800262875, - "manhattan_pearson": 0.460244992998331, - "manhattan_spearman": 0.4585112812933756, - "pearson": 0.4129668284215636, - "spearman": 0.41634720800262875 - } - ] - }, - "task_name": "STS17MultilingualVisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STSBenchmarkMultilingualVisualSTS.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STSBenchmarkMultilingualVisualSTS.json deleted file mode 100644 index 254c21fe5f..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/STSBenchmarkMultilingualVisualSTS.json +++ /dev/null @@ -1,313 +0,0 @@ -{ - "dataset_revision": "9f1ab21f17f497974996ab74b3ff911165a7dbf9", - "evaluation_time": 803.8532636165619, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "dev": [ - { - "cosine_pearson": 0.6064690420220407, - "cosine_spearman": 0.6197864969896985, - "euclidean_pearson": 0.6942088223171196, - "euclidean_spearman": 0.6870127125489691, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6197864969896985, - "manhattan_pearson": 0.6990269458595981, - "manhattan_spearman": 0.6915353669156528, - "pearson": 0.6064690420220407, - "spearman": 0.6197864969896985 - }, - { - "cosine_pearson": 0.47849293571967316, - "cosine_spearman": 0.48272931892528914, - "euclidean_pearson": 0.5514231774961209, - "euclidean_spearman": 0.5427657874720075, - "hf_subset": "de", - "languages": [ - "deu-Latn" - ], - "main_score": 0.48272931892528914, - "manhattan_pearson": 0.5563708024997025, - "manhattan_spearman": 0.547306668508022, - "pearson": 0.47849293571967316, - "spearman": 0.48272931892528914 - }, - { - "cosine_pearson": 0.5296312367505166, - "cosine_spearman": 0.5415773190695478, - "euclidean_pearson": 0.6074187893745651, - "euclidean_spearman": 0.6023347969959988, - "hf_subset": "es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.5415773190695478, - "manhattan_pearson": 0.6164718955947857, - "manhattan_spearman": 0.6112736003257329, - "pearson": 0.5296312367505166, - "spearman": 0.5415773190695478 - }, - { - "cosine_pearson": 0.5471814676359532, - "cosine_spearman": 0.5599694424985604, - "euclidean_pearson": 0.6252995994474644, - "euclidean_spearman": 0.6210724060203441, - "hf_subset": "fr", - "languages": [ - "fra-Latn" - ], - "main_score": 0.5599694424985604, - "manhattan_pearson": 0.6312980999587994, - "manhattan_spearman": 0.6269701559513498, - "pearson": 0.5471814676359532, - "spearman": 0.5599694424985604 - }, - { - "cosine_pearson": 0.4535418234458108, - "cosine_spearman": 0.4697046074838273, - "euclidean_pearson": 0.5456851627302479, - "euclidean_spearman": 0.5362763883603435, - "hf_subset": "it", - "languages": [ - "ita-Latn" - ], - "main_score": 0.4697046074838273, - "manhattan_pearson": 0.5555002671807903, - "manhattan_spearman": 0.5443832355991821, - "pearson": 0.4535418234458108, - "spearman": 0.4697046074838273 - }, - { - "cosine_pearson": 0.4444109218978589, - "cosine_spearman": 0.45599958771514254, - "euclidean_pearson": 0.5267559388120012, - "euclidean_spearman": 0.5176849288935731, - "hf_subset": "nl", - "languages": [ - "nld-Latn" - ], - "main_score": 0.45599958771514254, - "manhattan_pearson": 0.5308716107005126, - "manhattan_spearman": 0.5222130897679905, - "pearson": 0.4444109218978589, - "spearman": 0.45599958771514254 - }, - { - "cosine_pearson": 0.3063101737717443, - "cosine_spearman": 0.3367199070808206, - "euclidean_pearson": 0.3987785895110705, - "euclidean_spearman": 0.3913098100074221, - "hf_subset": "pl", - "languages": [ - "pol-Latn" - ], - "main_score": 0.3367199070808206, - "manhattan_pearson": 0.3998630367465825, - "manhattan_spearman": 0.3923105563563955, - "pearson": 0.3063101737717443, - "spearman": 0.3367199070808206 - }, - { - "cosine_pearson": 0.4632062714190491, - "cosine_spearman": 0.47564584417433386, - "euclidean_pearson": 0.5393215479135585, - "euclidean_spearman": 0.5319489388628749, - "hf_subset": "pt", - "languages": [ - "por-Latn" - ], - "main_score": 0.47564584417433386, - "manhattan_pearson": 0.5485030654666205, - "manhattan_spearman": 0.5401675722048989, - "pearson": 0.4632062714190491, - "spearman": 0.47564584417433386 - }, - { - "cosine_pearson": 0.1800744555427423, - "cosine_spearman": 0.2183834305276247, - "euclidean_pearson": 0.25850374844167806, - "euclidean_spearman": 0.2451706003193563, - "hf_subset": "ru", - "languages": [ - "rus-Cyrl" - ], - "main_score": 0.2183834305276247, - "manhattan_pearson": 0.2605585663635114, - "manhattan_spearman": 0.24598330621017067, - "pearson": 0.1800744555427423, - "spearman": 0.2183834305276247 - }, - { - "cosine_pearson": 0.10264590141987182, - "cosine_spearman": 0.19151948112441222, - "euclidean_pearson": 0.16974507374451053, - "euclidean_spearman": 0.20056773053396862, - "hf_subset": "zh", - "languages": [ - "cmn-Hans" - ], - "main_score": 0.19151948112441222, - "manhattan_pearson": 0.17342254049511108, - "manhattan_spearman": 0.20243639613422254, - "pearson": 0.10264590141987182, - "spearman": 0.19151948112441222 - } - ], - "test": [ - { - "cosine_pearson": 0.5270768320200866, - "cosine_spearman": 0.5673533844224959, - "euclidean_pearson": 0.6443058202600288, - "euclidean_spearman": 0.645019056836489, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5673533844224959, - "manhattan_pearson": 0.6530080093827622, - "manhattan_spearman": 0.6525615100978652, - "pearson": 0.5270768320200866, - "spearman": 0.5673533844224959 - }, - { - "cosine_pearson": 0.3521438273165664, - "cosine_spearman": 0.3749686752380741, - "euclidean_pearson": 0.45537539942154803, - "euclidean_spearman": 0.44106091613516796, - "hf_subset": "de", - "languages": [ - "deu-Latn" - ], - "main_score": 0.3749686752380741, - "manhattan_pearson": 0.46237570766054453, - "manhattan_spearman": 0.4471662661275312, - "pearson": 0.3521438273165664, - "spearman": 0.3749686752380741 - }, - { - "cosine_pearson": 0.4096357505353805, - "cosine_spearman": 0.44184121082999916, - "euclidean_pearson": 0.51269336299542, - "euclidean_spearman": 0.5063138290175788, - "hf_subset": "es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.44184121082999916, - "manhattan_pearson": 0.5215447226048767, - "manhattan_spearman": 0.5134022523664801, - "pearson": 0.4096357505353805, - "spearman": 0.44184121082999916 - }, - { - "cosine_pearson": 0.4471367141662311, - "cosine_spearman": 0.4753040189693729, - "euclidean_pearson": 0.5453656618359074, - "euclidean_spearman": 0.5399611013119631, - "hf_subset": "fr", - "languages": [ - "fra-Latn" - ], - "main_score": 0.4753040189693729, - "manhattan_pearson": 0.5502662673469212, - "manhattan_spearman": 0.5436264459052782, - "pearson": 0.4471367141662311, - "spearman": 0.4753040189693729 - }, - { - "cosine_pearson": 0.316221141637704, - "cosine_spearman": 0.3688720060740888, - "euclidean_pearson": 0.4439254366261284, - "euclidean_spearman": 0.4427471555272557, - "hf_subset": "it", - "languages": [ - "ita-Latn" - ], - "main_score": 0.3688720060740888, - "manhattan_pearson": 0.45081804481326126, - "manhattan_spearman": 0.44797869193706497, - "pearson": 0.316221141637704, - "spearman": 0.3688720060740888 - }, - { - "cosine_pearson": 0.287912292278141, - "cosine_spearman": 0.3250557807992596, - "euclidean_pearson": 0.39531965148425147, - "euclidean_spearman": 0.3852550773539582, - "hf_subset": "nl", - "languages": [ - "nld-Latn" - ], - "main_score": 0.3250557807992596, - "manhattan_pearson": 0.4010420046702985, - "manhattan_spearman": 0.39002819841596115, - "pearson": 0.287912292278141, - "spearman": 0.3250557807992596 - }, - { - "cosine_pearson": 0.198384733263269, - "cosine_spearman": 0.23406616776235958, - "euclidean_pearson": 0.30015046447501326, - "euclidean_spearman": 0.28485294933685945, - "hf_subset": "pl", - "languages": [ - "pol-Latn" - ], - "main_score": 0.23406616776235958, - "manhattan_pearson": 0.3048021063858739, - "manhattan_spearman": 0.29003810288565623, - "pearson": 0.198384733263269, - "spearman": 0.23406616776235958 - }, - { - "cosine_pearson": 0.31449680168072336, - "cosine_spearman": 0.3548920430387672, - "euclidean_pearson": 0.41260364320400034, - "euclidean_spearman": 0.4094693280962031, - "hf_subset": "pt", - "languages": [ - "por-Latn" - ], - "main_score": 0.3548920430387672, - "manhattan_pearson": 0.4236962018619726, - "manhattan_spearman": 0.41740681554571923, - "pearson": 0.31449680168072336, - "spearman": 0.3548920430387672 - }, - { - "cosine_pearson": 0.07318840017959619, - "cosine_spearman": 0.14060633900590963, - "euclidean_pearson": 0.15202005150811604, - "euclidean_spearman": 0.16390557574378006, - "hf_subset": "ru", - "languages": [ - "rus-Cyrl" - ], - "main_score": 0.14060633900590963, - "manhattan_pearson": 0.1551360292588062, - "manhattan_spearman": 0.16695666252040703, - "pearson": 0.07318840017959619, - "spearman": 0.14060633900590963 - }, - { - "cosine_pearson": 0.010396153007584142, - "cosine_spearman": 0.12123555650303279, - "euclidean_pearson": 0.07498960212905466, - "euclidean_spearman": 0.13153048990121405, - "hf_subset": "zh", - "languages": [ - "cmn-Hans" - ], - "main_score": 0.12123555650303279, - "manhattan_pearson": 0.07621513791751623, - "manhattan_spearman": 0.1314945508379666, - "pearson": 0.010396153007584142, - "spearman": 0.12123555650303279 - } - ] - }, - "task_name": "STSBenchmarkMultilingualVisualSTS" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRI2TRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRI2TRetrieval.json deleted file mode 100644 index 92da9c357d..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", - "evaluation_time": 283.1252419948578, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.10533, - "cv_recall_at_10": 0.24633, - "cv_recall_at_100": 0.4378, - "cv_recall_at_1000": 0.70805, - "cv_recall_at_20": 0.2957, - "cv_recall_at_3": 0.17137, - "cv_recall_at_5": 0.20138, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.17056, - "map_at_1": 0.10533, - "map_at_10": 0.14695, - "map_at_100": 0.15368, - "map_at_1000": 0.15468, - "map_at_20": 0.15037, - "map_at_3": 0.13406, - "map_at_5": 0.14093, - "mrr_at_1": 0.10533111971960893, - "mrr_at_10": 0.1469541826855974, - "mrr_at_100": 0.15368428460896544, - "mrr_at_1000": 0.15467910679952085, - "mrr_at_20": 0.15036538821910328, - "mrr_at_3": 0.1340568570784395, - "mrr_at_5": 0.1409252085511078, - "nauc_cv_recall_at_1000_diff1": 0.23028863328141164, - "nauc_cv_recall_at_1000_max": 0.092046541316207, - "nauc_cv_recall_at_1000_std": 0.4571359824346522, - "nauc_cv_recall_at_100_diff1": 0.2559857960706079, - "nauc_cv_recall_at_100_max": 0.14529317511624498, - "nauc_cv_recall_at_100_std": 0.4106083321055665, - "nauc_cv_recall_at_10_diff1": 0.31671357009704, - "nauc_cv_recall_at_10_max": 0.2194327767019012, - "nauc_cv_recall_at_10_std": 0.35938923946956497, - "nauc_cv_recall_at_1_diff1": 0.5069311041965368, - "nauc_cv_recall_at_1_max": 0.3013928019939395, - "nauc_cv_recall_at_1_std": 0.2965515266383622, - "nauc_cv_recall_at_20_diff1": 0.2913557570660773, - "nauc_cv_recall_at_20_max": 0.1915193655073638, - "nauc_cv_recall_at_20_std": 0.37274328079150615, - "nauc_cv_recall_at_3_diff1": 0.38300334076067827, - "nauc_cv_recall_at_3_max": 0.27189162202407996, - "nauc_cv_recall_at_3_std": 0.3332969427195455, - "nauc_cv_recall_at_5_diff1": 0.34020077997092296, - "nauc_cv_recall_at_5_max": 0.2493568151718566, - "nauc_cv_recall_at_5_std": 0.33961563695736396, - "nauc_map_at_1000_diff1": 0.41636126447481425, - "nauc_map_at_1000_max": 0.26831410429735236, - "nauc_map_at_1000_std": 0.3272211690775936, - "nauc_map_at_100_diff1": 0.4163608476581626, - "nauc_map_at_100_max": 0.2684712221457793, - "nauc_map_at_100_std": 0.327070263065989, - "nauc_map_at_10_diff1": 0.4217520673857625, - "nauc_map_at_10_max": 0.2739379505279472, - "nauc_map_at_10_std": 0.3236386178749276, - "nauc_map_at_1_diff1": 0.5069311041965368, - "nauc_map_at_1_max": 0.3013928019939395, - "nauc_map_at_1_std": 0.2965515266383622, - "nauc_map_at_20_diff1": 0.4181470924103441, - "nauc_map_at_20_max": 0.270497163275252, - "nauc_map_at_20_std": 0.32538970678533347, - "nauc_map_at_3_diff1": 0.44235606110109965, - "nauc_map_at_3_max": 0.2868603836336605, - "nauc_map_at_3_std": 0.3166462314562751, - "nauc_map_at_5_diff1": 0.4273046860941681, - "nauc_map_at_5_max": 0.27968527276930205, - "nauc_map_at_5_std": 0.3192981186456483, - "nauc_mrr_at_1000_diff1": 0.4163612644311463, - "nauc_mrr_at_1000_max": 0.2683141057970173, - "nauc_mrr_at_1000_std": 0.32722116697250775, - "nauc_mrr_at_100_diff1": 0.4163608476581626, - "nauc_mrr_at_100_max": 0.2684712221457793, - "nauc_mrr_at_100_std": 0.327070263065989, - "nauc_mrr_at_10_diff1": 0.4217520673857625, - "nauc_mrr_at_10_max": 0.2739379505279472, - "nauc_mrr_at_10_std": 0.3236386178749276, - "nauc_mrr_at_1_diff1": 0.5069311041965368, - "nauc_mrr_at_1_max": 0.3013928019939395, - "nauc_mrr_at_1_std": 0.2965515266383622, - "nauc_mrr_at_20_diff1": 0.4181470924103441, - "nauc_mrr_at_20_max": 0.270497163275252, - "nauc_mrr_at_20_std": 0.32538970678533347, - "nauc_mrr_at_3_diff1": 0.44235606110109965, - "nauc_mrr_at_3_max": 0.2868603836336605, - "nauc_mrr_at_3_std": 0.3166462314562751, - "nauc_mrr_at_5_diff1": 0.4273046860941681, - "nauc_mrr_at_5_max": 0.27968527276930205, - "nauc_mrr_at_5_std": 0.3192981186456483, - "nauc_ndcg_at_1000_diff1": 0.3701149218651449, - "nauc_ndcg_at_1000_max": 0.23194294078168726, - "nauc_ndcg_at_1000_std": 0.3532591641869934, - "nauc_ndcg_at_100_diff1": 0.3683136220697507, - "nauc_ndcg_at_100_max": 0.23463564798854547, - "nauc_ndcg_at_100_std": 0.35003745669206504, - "nauc_ndcg_at_10_diff1": 0.3893629910934037, - "nauc_ndcg_at_10_max": 0.257886567721901, - "nauc_ndcg_at_10_std": 0.3345022150293404, - "nauc_ndcg_at_1_diff1": 0.5069311041965368, - "nauc_ndcg_at_1_max": 0.3013928019939395, - "nauc_ndcg_at_1_std": 0.2965515266383622, - "nauc_ndcg_at_20_diff1": 0.3786251554746303, - "nauc_ndcg_at_20_max": 0.24722749345684428, - "nauc_ndcg_at_20_std": 0.33980295740359023, - "nauc_ndcg_at_3_diff1": 0.4249771061750892, - "nauc_ndcg_at_3_max": 0.2825108192119049, - "nauc_ndcg_at_3_std": 0.3215594005999232, - "nauc_ndcg_at_5_diff1": 0.40095083808527676, - "nauc_ndcg_at_5_max": 0.27067795407149536, - "nauc_ndcg_at_5_std": 0.32555508887366974, - "nauc_precision_at_1000_diff1": 0.23028863328141058, - "nauc_precision_at_1000_max": 0.09204654131620628, - "nauc_precision_at_1000_std": 0.4571359824346514, - "nauc_precision_at_100_diff1": 0.25598579607060845, - "nauc_precision_at_100_max": 0.1452931751162452, - "nauc_precision_at_100_std": 0.4106083321055668, - "nauc_precision_at_10_diff1": 0.31671357009703993, - "nauc_precision_at_10_max": 0.21943277670190117, - "nauc_precision_at_10_std": 0.35938923946956486, - "nauc_precision_at_1_diff1": 0.5069311041965368, - "nauc_precision_at_1_max": 0.3013928019939395, - "nauc_precision_at_1_std": 0.2965515266383622, - "nauc_precision_at_20_diff1": 0.29135575706607697, - "nauc_precision_at_20_max": 0.19151936550736376, - "nauc_precision_at_20_std": 0.37274328079150626, - "nauc_precision_at_3_diff1": 0.3830033407606782, - "nauc_precision_at_3_max": 0.27189162202407974, - "nauc_precision_at_3_std": 0.33329694271954546, - "nauc_precision_at_5_diff1": 0.3402007799709232, - "nauc_precision_at_5_max": 0.24935681517185637, - "nauc_precision_at_5_std": 0.33961563695736424, - "nauc_recall_at_1000_diff1": 0.23028863328141164, - "nauc_recall_at_1000_max": 0.092046541316207, - "nauc_recall_at_1000_std": 0.4571359824346522, - "nauc_recall_at_100_diff1": 0.2559857960706079, - "nauc_recall_at_100_max": 0.14529317511624498, - "nauc_recall_at_100_std": 0.4106083321055665, - "nauc_recall_at_10_diff1": 0.31671357009704, - "nauc_recall_at_10_max": 0.2194327767019012, - "nauc_recall_at_10_std": 0.35938923946956497, - "nauc_recall_at_1_diff1": 0.5069311041965368, - "nauc_recall_at_1_max": 0.3013928019939395, - "nauc_recall_at_1_std": 0.2965515266383622, - "nauc_recall_at_20_diff1": 0.2913557570660773, - "nauc_recall_at_20_max": 0.1915193655073638, - "nauc_recall_at_20_std": 0.37274328079150615, - "nauc_recall_at_3_diff1": 0.38300334076067827, - "nauc_recall_at_3_max": 0.27189162202407996, - "nauc_recall_at_3_std": 0.3332969427195455, - "nauc_recall_at_5_diff1": 0.34020077997092296, - "nauc_recall_at_5_max": 0.2493568151718566, - "nauc_recall_at_5_std": 0.33961563695736396, - "ndcg_at_1": 0.10533, - "ndcg_at_10": 0.17056, - "ndcg_at_100": 0.20873, - "ndcg_at_1000": 0.24132, - "ndcg_at_20": 0.18302, - "ndcg_at_3": 0.14362, - "ndcg_at_5": 0.15599, - "precision_at_1": 0.10533, - "precision_at_10": 0.02463, - "precision_at_100": 0.00438, - "precision_at_1000": 0.00071, - "precision_at_20": 0.01479, - "precision_at_3": 0.05712, - "precision_at_5": 0.04028, - "recall_at_1": 0.10533, - "recall_at_10": 0.24633, - "recall_at_100": 0.4378, - "recall_at_1000": 0.70805, - "recall_at_20": 0.2957, - "recall_at_3": 0.17137, - "recall_at_5": 0.20138 - } - ] - }, - "task_name": "SciMMIRI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRT2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRT2IRetrieval.json deleted file mode 100644 index 586fd3ba1d..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SciMMIRT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "eea276dc58c52eab33e9476acb137ff5530b78e9", - "evaluation_time": 278.578586101532, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.09592, - "cv_recall_at_10": 0.22597, - "cv_recall_at_100": 0.40638, - "cv_recall_at_1000": 0.69796, - "cv_recall_at_20": 0.27104, - "cv_recall_at_3": 0.15255, - "cv_recall_at_5": 0.1833, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.15526, - "map_at_1": 0.09592, - "map_at_10": 0.13334, - "map_at_100": 0.13961, - "map_at_1000": 0.14066, - "map_at_20": 0.13642, - "map_at_3": 0.1207, - "map_at_5": 0.12773, - "mrr_at_1": 0.09592326139088729, - "mrr_at_10": 0.1333431667364518, - "mrr_at_100": 0.13960663411040436, - "mrr_at_1000": 0.14065717797325847, - "mrr_at_20": 0.13641789220130707, - "mrr_at_3": 0.12070343725019922, - "mrr_at_5": 0.12772858636167966, - "nauc_cv_recall_at_1000_diff1": 0.16617922291887638, - "nauc_cv_recall_at_1000_max": 0.16984497409382054, - "nauc_cv_recall_at_1000_std": 0.42084793152355615, - "nauc_cv_recall_at_100_diff1": 0.21608124742748158, - "nauc_cv_recall_at_100_max": 0.20246475499276076, - "nauc_cv_recall_at_100_std": 0.35902357900333665, - "nauc_cv_recall_at_10_diff1": 0.2896832447371392, - "nauc_cv_recall_at_10_max": 0.2679650963958003, - "nauc_cv_recall_at_10_std": 0.2892498692096226, - "nauc_cv_recall_at_1_diff1": 0.4832228454253711, - "nauc_cv_recall_at_1_max": 0.3539648181240017, - "nauc_cv_recall_at_1_std": 0.24684187585451275, - "nauc_cv_recall_at_20_diff1": 0.2572076651086662, - "nauc_cv_recall_at_20_max": 0.23989562009918416, - "nauc_cv_recall_at_20_std": 0.3134865111627789, - "nauc_cv_recall_at_3_diff1": 0.37048797651088045, - "nauc_cv_recall_at_3_max": 0.32499007199890534, - "nauc_cv_recall_at_3_std": 0.26812218154000894, - "nauc_cv_recall_at_5_diff1": 0.33336929117413305, - "nauc_cv_recall_at_5_max": 0.298643405877764, - "nauc_cv_recall_at_5_std": 0.2752327353435506, - "nauc_map_at_1000_diff1": 0.3946988397633675, - "nauc_map_at_1000_max": 0.3183753649842543, - "nauc_map_at_1000_std": 0.26834083843885537, - "nauc_map_at_100_diff1": 0.39476113058528917, - "nauc_map_at_100_max": 0.3184429744599427, - "nauc_map_at_100_std": 0.26806520363704717, - "nauc_map_at_10_diff1": 0.40163266202098985, - "nauc_map_at_10_max": 0.32402854649481405, - "nauc_map_at_10_std": 0.2635706102905207, - "nauc_map_at_1_diff1": 0.4832228454253711, - "nauc_map_at_1_max": 0.3539648181240017, - "nauc_map_at_1_std": 0.24684187585451275, - "nauc_map_at_20_diff1": 0.3971216898413058, - "nauc_map_at_20_max": 0.3203661920396288, - "nauc_map_at_20_std": 0.26622152413508027, - "nauc_map_at_3_diff1": 0.42552280749830074, - "nauc_map_at_3_max": 0.33981729678252126, - "nauc_map_at_3_std": 0.25792014034448607, - "nauc_map_at_5_diff1": 0.41199461734235987, - "nauc_map_at_5_max": 0.33105191712818083, - "nauc_map_at_5_std": 0.2603994992627051, - "nauc_mrr_at_1000_diff1": 0.3946988543879861, - "nauc_mrr_at_1000_max": 0.31837534752269314, - "nauc_mrr_at_1000_std": 0.26834083921874824, - "nauc_mrr_at_100_diff1": 0.39476113058528917, - "nauc_mrr_at_100_max": 0.3184429744599427, - "nauc_mrr_at_100_std": 0.26806520363704717, - "nauc_mrr_at_10_diff1": 0.40163266202098985, - "nauc_mrr_at_10_max": 0.32402854649481405, - "nauc_mrr_at_10_std": 0.2635706102905207, - "nauc_mrr_at_1_diff1": 0.4832228454253711, - "nauc_mrr_at_1_max": 0.3539648181240017, - "nauc_mrr_at_1_std": 0.24684187585451275, - "nauc_mrr_at_20_diff1": 0.3971216898413058, - "nauc_mrr_at_20_max": 0.3203661920396288, - "nauc_mrr_at_20_std": 0.26622152413508027, - "nauc_mrr_at_3_diff1": 0.42552280749830074, - "nauc_mrr_at_3_max": 0.33981729678252126, - "nauc_mrr_at_3_std": 0.25792014034448607, - "nauc_mrr_at_5_diff1": 0.41199461734235987, - "nauc_mrr_at_5_max": 0.33105191712818083, - "nauc_mrr_at_5_std": 0.2603994992627051, - "nauc_ndcg_at_1000_diff1": 0.3388507558240032, - "nauc_ndcg_at_1000_max": 0.28376369750843505, - "nauc_ndcg_at_1000_std": 0.2984960891174002, - "nauc_ndcg_at_100_diff1": 0.3398066680113286, - "nauc_ndcg_at_100_max": 0.284463506938026, - "nauc_ndcg_at_100_std": 0.29289785303561827, - "nauc_ndcg_at_10_diff1": 0.3668889675479959, - "nauc_ndcg_at_10_max": 0.30698993721798895, - "nauc_ndcg_at_10_std": 0.2714358842514923, - "nauc_ndcg_at_1_diff1": 0.4832228454253711, - "nauc_ndcg_at_1_max": 0.3539648181240017, - "nauc_ndcg_at_1_std": 0.24684187585451275, - "nauc_ndcg_at_20_diff1": 0.35340066661737557, - "nauc_ndcg_at_20_max": 0.295972831155534, - "nauc_ndcg_at_20_std": 0.28017707571664463, - "nauc_ndcg_at_3_diff1": 0.409522030418501, - "nauc_ndcg_at_3_max": 0.33553325712974746, - "nauc_ndcg_at_3_std": 0.26089360395464967, - "nauc_ndcg_at_5_diff1": 0.3881318919670405, - "nauc_ndcg_at_5_max": 0.32138515823147434, - "nauc_ndcg_at_5_std": 0.26489458853923464, - "nauc_precision_at_1000_diff1": 0.16617922291887616, - "nauc_precision_at_1000_max": 0.1698449740938206, - "nauc_precision_at_1000_std": 0.42084793152355576, - "nauc_precision_at_100_diff1": 0.21608124742748158, - "nauc_precision_at_100_max": 0.20246475499276068, - "nauc_precision_at_100_std": 0.3590235790033364, - "nauc_precision_at_10_diff1": 0.289683244737139, - "nauc_precision_at_10_max": 0.26796509639579996, - "nauc_precision_at_10_std": 0.2892498692096226, - "nauc_precision_at_1_diff1": 0.4832228454253711, - "nauc_precision_at_1_max": 0.3539648181240017, - "nauc_precision_at_1_std": 0.24684187585451275, - "nauc_precision_at_20_diff1": 0.2572076651086662, - "nauc_precision_at_20_max": 0.23989562009918394, - "nauc_precision_at_20_std": 0.31348651116277904, - "nauc_precision_at_3_diff1": 0.3704879765108802, - "nauc_precision_at_3_max": 0.32499007199890523, - "nauc_precision_at_3_std": 0.2681221815400088, - "nauc_precision_at_5_diff1": 0.333369291174133, - "nauc_precision_at_5_max": 0.29864340587776383, - "nauc_precision_at_5_std": 0.27523273534355047, - "nauc_recall_at_1000_diff1": 0.16617922291887638, - "nauc_recall_at_1000_max": 0.16984497409382054, - "nauc_recall_at_1000_std": 0.42084793152355615, - "nauc_recall_at_100_diff1": 0.21608124742748158, - "nauc_recall_at_100_max": 0.20246475499276076, - "nauc_recall_at_100_std": 0.35902357900333665, - "nauc_recall_at_10_diff1": 0.2896832447371392, - "nauc_recall_at_10_max": 0.2679650963958003, - "nauc_recall_at_10_std": 0.2892498692096226, - "nauc_recall_at_1_diff1": 0.4832228454253711, - "nauc_recall_at_1_max": 0.3539648181240017, - "nauc_recall_at_1_std": 0.24684187585451275, - "nauc_recall_at_20_diff1": 0.2572076651086662, - "nauc_recall_at_20_max": 0.23989562009918416, - "nauc_recall_at_20_std": 0.3134865111627789, - "nauc_recall_at_3_diff1": 0.37048797651088045, - "nauc_recall_at_3_max": 0.32499007199890534, - "nauc_recall_at_3_std": 0.26812218154000894, - "nauc_recall_at_5_diff1": 0.33336929117413305, - "nauc_recall_at_5_max": 0.298643405877764, - "nauc_recall_at_5_std": 0.2752327353435506, - "ndcg_at_1": 0.09592, - "ndcg_at_10": 0.15526, - "ndcg_at_100": 0.19112, - "ndcg_at_1000": 0.22617, - "ndcg_at_20": 0.16658, - "ndcg_at_3": 0.12888, - "ndcg_at_5": 0.14154, - "precision_at_1": 0.09592, - "precision_at_10": 0.0226, - "precision_at_100": 0.00406, - "precision_at_1000": 0.0007, - "precision_at_20": 0.01355, - "precision_at_3": 0.05085, - "precision_at_5": 0.03666, - "recall_at_1": 0.09592, - "recall_at_10": 0.22597, - "recall_at_100": 0.40638, - "recall_at_1000": 0.69796, - "recall_at_20": 0.27104, - "recall_at_3": 0.15255, - "recall_at_5": 0.1833 - } - ] - }, - "task_name": "SciMMIRT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SugarCrepe.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SugarCrepe.json deleted file mode 100644 index 74c0442e9f..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/SugarCrepe.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "134abf9ade6a32f9fdae0e89022ff227a70b87e5", - "evaluation_time": 189.15511345863342, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.7671415209770203, - "text_acc": 0.7671415209770203 - } - ] - }, - "task_name": "SugarCrepe" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/TUBerlinT2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/TUBerlinT2IRetrieval.json deleted file mode 100644 index 355e3b33e2..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/TUBerlinT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "0cd78cd1ddbd3cafa9f319c638ebd77836ec9ff6", - "evaluation_time": 385.1187882423401, - "kg_co2_emissions": null, - "mteb_version": "1.14.1", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.908, - "cv_recall_at_10": 1.0, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 0.992, - "cv_recall_at_5": 0.992, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8924, - "map_at_1": 0.01135, - "map_at_10": 0.10502, - "map_at_100": 0.62701, - "map_at_1000": 0.6943, - "map_at_20": 0.20153, - "map_at_3": 0.03284, - "map_at_5": 0.05367, - "mrr_at_1": 0.908, - "mrr_at_10": 0.9484047619047621, - "mrr_at_100": 0.9484047619047621, - "mrr_at_1000": 0.9484047619047621, - "mrr_at_20": 0.9484047619047621, - "mrr_at_3": 0.9473333333333334, - "mrr_at_5": 0.9473333333333334, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": NaN, - "nauc_cv_recall_at_10_max": NaN, - "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": 0.11582024113993439, - "nauc_cv_recall_at_1_max": 0.373929281857669, - "nauc_cv_recall_at_1_std": 0.23413713311411463, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": 0.4561157796451851, - "nauc_cv_recall_at_3_max": 0.24042950513537323, - "nauc_cv_recall_at_3_std": -0.5144724556489358, - "nauc_cv_recall_at_5_diff1": 0.4561157796451851, - "nauc_cv_recall_at_5_max": 0.24042950513537323, - "nauc_cv_recall_at_5_std": -0.5144724556489358, - "nauc_map_at_1000_diff1": -0.06530005170454266, - "nauc_map_at_1000_max": 0.1502471022239646, - "nauc_map_at_1000_std": 0.4724049267752983, - "nauc_map_at_100_diff1": -0.0704630599295298, - "nauc_map_at_100_max": 0.13854288149819008, - "nauc_map_at_100_std": 0.43057914277907994, - "nauc_map_at_10_diff1": 0.006402040467115649, - "nauc_map_at_10_max": 0.255115488651325, - "nauc_map_at_10_std": 0.15962216044023922, - "nauc_map_at_1_diff1": 0.11582024113993612, - "nauc_map_at_1_max": 0.37392928185767066, - "nauc_map_at_1_std": 0.23413713311411455, - "nauc_map_at_20_diff1": -0.04061394388241786, - "nauc_map_at_20_max": 0.2114603378673776, - "nauc_map_at_20_std": 0.19804307292662585, - "nauc_map_at_3_diff1": 0.015403428628263226, - "nauc_map_at_3_max": 0.2868909673477487, - "nauc_map_at_3_std": 0.16916419193497864, - "nauc_map_at_5_diff1": 0.018750084565292698, - "nauc_map_at_5_max": 0.23531345421895392, - "nauc_map_at_5_std": 0.11997890223687437, - "nauc_mrr_at_1000_diff1": 0.110313776333482, - "nauc_mrr_at_1000_max": 0.3707442983346247, - "nauc_mrr_at_1000_std": 0.22493264341576957, - "nauc_mrr_at_100_diff1": 0.110313776333482, - "nauc_mrr_at_100_max": 0.3707442983346247, - "nauc_mrr_at_100_std": 0.22493264341576957, - "nauc_mrr_at_10_diff1": 0.110313776333482, - "nauc_mrr_at_10_max": 0.3707442983346247, - "nauc_mrr_at_10_std": 0.22493264341576957, - "nauc_mrr_at_1_diff1": 0.11582024113993439, - "nauc_mrr_at_1_max": 0.373929281857669, - "nauc_mrr_at_1_std": 0.23413713311411463, - "nauc_mrr_at_20_diff1": 0.110313776333482, - "nauc_mrr_at_20_max": 0.3707442983346247, - "nauc_mrr_at_20_std": 0.22493264341576957, - "nauc_mrr_at_3_diff1": 0.11721566263636403, - "nauc_mrr_at_3_max": 0.36825278634660735, - "nauc_mrr_at_3_std": 0.2090262265243671, - "nauc_mrr_at_5_diff1": 0.11721566263636403, - "nauc_mrr_at_5_max": 0.36825278634660735, - "nauc_mrr_at_5_std": 0.2090262265243671, - "nauc_ndcg_at_1000_diff1": -0.037292624079711435, - "nauc_ndcg_at_1000_max": 0.21046169942509146, - "nauc_ndcg_at_1000_std": 0.54841970255689, - "nauc_ndcg_at_100_diff1": -0.06417993720042019, - "nauc_ndcg_at_100_max": 0.15400863056231898, - "nauc_ndcg_at_100_std": 0.4723216905906371, - "nauc_ndcg_at_10_diff1": 0.01494953591412352, - "nauc_ndcg_at_10_max": 0.2680953405148115, - "nauc_ndcg_at_10_std": 0.16994841614578113, - "nauc_ndcg_at_1_diff1": 0.11582024113993439, - "nauc_ndcg_at_1_max": 0.373929281857669, - "nauc_ndcg_at_1_std": 0.23413713311411463, - "nauc_ndcg_at_20_diff1": -0.025310686952678893, - "nauc_ndcg_at_20_max": 0.2248683312661876, - "nauc_ndcg_at_20_std": 0.19959181596096173, - "nauc_ndcg_at_3_diff1": 0.0248235494869043, - "nauc_ndcg_at_3_max": 0.29864496147398095, - "nauc_ndcg_at_3_std": 0.16810172850714117, - "nauc_ndcg_at_5_diff1": 0.032312983699494084, - "nauc_ndcg_at_5_max": 0.2447447927622062, - "nauc_ndcg_at_5_std": 0.11460242029652785, - "nauc_precision_at_1000_diff1": -0.004062035055989619, - "nauc_precision_at_1000_max": 0.20159639972746157, - "nauc_precision_at_1000_std": 0.6755051245483236, - "nauc_precision_at_100_diff1": -0.0731911806199345, - "nauc_precision_at_100_max": 0.12152536286318481, - "nauc_precision_at_100_std": 0.5067829123595188, - "nauc_precision_at_10_diff1": 0.0010081351446029666, - "nauc_precision_at_10_max": 0.2530613833638247, - "nauc_precision_at_10_std": 0.17313261453427123, - "nauc_precision_at_1_diff1": 0.11582024113993439, - "nauc_precision_at_1_max": 0.373929281857669, - "nauc_precision_at_1_std": 0.23413713311411463, - "nauc_precision_at_20_diff1": -0.04926658558983306, - "nauc_precision_at_20_max": 0.19675521296851267, - "nauc_precision_at_20_std": 0.20747986966249193, - "nauc_precision_at_3_diff1": 0.0017235227188574813, - "nauc_precision_at_3_max": 0.28269138844566605, - "nauc_precision_at_3_std": 0.14890966990500618, - "nauc_precision_at_5_diff1": 0.023111522328179257, - "nauc_precision_at_5_max": 0.20941877145445104, - "nauc_precision_at_5_std": 0.08089615948310987, - "nauc_recall_at_1000_diff1": -0.004062035055987913, - "nauc_recall_at_1000_max": 0.20159639972746426, - "nauc_recall_at_1000_std": 0.6755051245483176, - "nauc_recall_at_100_diff1": -0.07319118061993815, - "nauc_recall_at_100_max": 0.1215253628631808, - "nauc_recall_at_100_std": 0.5067829123595168, - "nauc_recall_at_10_diff1": 0.0010081351446029666, - "nauc_recall_at_10_max": 0.2530613833638247, - "nauc_recall_at_10_std": 0.17313261453427123, - "nauc_recall_at_1_diff1": 0.11582024113993612, - "nauc_recall_at_1_max": 0.37392928185767066, - "nauc_recall_at_1_std": 0.23413713311411455, - "nauc_recall_at_20_diff1": -0.04926658558983306, - "nauc_recall_at_20_max": 0.19675521296851267, - "nauc_recall_at_20_std": 0.20747986966249193, - "nauc_recall_at_3_diff1": 0.001723522718856776, - "nauc_recall_at_3_max": 0.2826913884456665, - "nauc_recall_at_3_std": 0.1489096699050044, - "nauc_recall_at_5_diff1": 0.023111522328179257, - "nauc_recall_at_5_max": 0.20941877145445104, - "nauc_recall_at_5_std": 0.08089615948310987, - "ndcg_at_1": 0.908, - "ndcg_at_10": 0.8924, - "ndcg_at_100": 0.76172, - "ndcg_at_1000": 0.88769, - "ndcg_at_20": 0.87619, - "ndcg_at_3": 0.90306, - "ndcg_at_5": 0.89731, - "precision_at_1": 0.908, - "precision_at_10": 0.888, - "precision_at_100": 0.58372, - "precision_at_1000": 0.07593, - "precision_at_20": 0.8672, - "precision_at_3": 0.90133, - "precision_at_5": 0.8936, - "recall_at_1": 0.01135, - "recall_at_10": 0.111, - "recall_at_100": 0.72965, - "recall_at_1000": 0.9491, - "recall_at_20": 0.2168, - "recall_at_3": 0.0338, - "recall_at_5": 0.05585 - } - ] - }, - "task_name": "TUBerlinT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json deleted file mode 100644 index 151085b676..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 143.64798212051392, - "kg_co2_emissions": null, - "mteb_version": "1.16.5", - "scores": { - "test": [ - { - "accuracy": 0.5198707592891761, - "f1": 0.6818774389774342, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "lrap": 0.6617288749775786, - "main_score": 0.5198707592891761, - "scores_per_experiment": [ - { - "accuracy": 0.5310985460420032, - "f1": 0.696133632301483, - "lrap": 0.6760691527553577 - }, - { - "accuracy": 0.5274636510500808, - "f1": 0.6754522344136807, - "lrap": 0.6636039310716377 - }, - { - "accuracy": 0.507875605815832, - "f1": 0.6621408823160164, - "lrap": 0.6531808135882394 - }, - { - "accuracy": 0.5359450726978998, - "f1": 0.6914673370418792, - "lrap": 0.673578576557188 - }, - { - "accuracy": 0.4969709208400646, - "f1": 0.684193108814112, - "lrap": 0.6422119009154702 - } - ] - } - ] - }, - "task_name": "VOC2007" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreArxivQARetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreArxivQARetrieval.json deleted file mode 100644 index 2c60f777b4..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreArxivQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", - "evaluation_time": 15.369404315948486, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.218, - "cv_recall_at_10": 0.426, - "cv_recall_at_100": 0.766, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.526, - "cv_recall_at_3": 0.304, - "cv_recall_at_5": 0.352, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.28621, - "map_at_1": 0.218, - "map_at_10": 0.27448, - "map_at_100": 0.28695, - "map_at_1000": 0.28821, - "map_at_20": 0.28155, - "map_at_3": 0.254, - "map_at_5": 0.2646, - "mrr_at_1": 0.218, - "mrr_at_10": 0.27447857142857146, - "mrr_at_100": 0.2869511549000328, - "mrr_at_1000": 0.28820781348047175, - "mrr_at_20": 0.28154940605490136, - "mrr_at_3": 0.2539999999999999, - "mrr_at_5": 0.26459999999999995, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.24209378407851656, - "nauc_cv_recall_at_100_max": 0.2393564651499565, - "nauc_cv_recall_at_100_std": 0.2226853098689175, - "nauc_cv_recall_at_10_diff1": 0.3765744283484118, - "nauc_cv_recall_at_10_max": 0.3556539153880319, - "nauc_cv_recall_at_10_std": 0.05377148482652398, - "nauc_cv_recall_at_1_diff1": 0.5788284986102574, - "nauc_cv_recall_at_1_max": 0.5023162373698774, - "nauc_cv_recall_at_1_std": 0.06590280225727854, - "nauc_cv_recall_at_20_diff1": 0.29657056660798986, - "nauc_cv_recall_at_20_max": 0.31536989534027415, - "nauc_cv_recall_at_20_std": 0.09941715784373036, - "nauc_cv_recall_at_3_diff1": 0.475473591093467, - "nauc_cv_recall_at_3_max": 0.4033426469319063, - "nauc_cv_recall_at_3_std": 0.026381936653145647, - "nauc_cv_recall_at_5_diff1": 0.4553467933992285, - "nauc_cv_recall_at_5_max": 0.39863326940504556, - "nauc_cv_recall_at_5_std": 0.028559611372210154, - "nauc_map_at_1000_diff1": 0.510784304760984, - "nauc_map_at_1000_max": 0.44976634773529395, - "nauc_map_at_1000_std": 0.05570654098630355, - "nauc_map_at_100_diff1": 0.5097698322258946, - "nauc_map_at_100_max": 0.44894547649673194, - "nauc_map_at_100_std": 0.05591543784058362, - "nauc_map_at_10_diff1": 0.5142652560397565, - "nauc_map_at_10_max": 0.4518266232736721, - "nauc_map_at_10_std": 0.052221764538752394, - "nauc_map_at_1_diff1": 0.5788284986102574, - "nauc_map_at_1_max": 0.5023162373698774, - "nauc_map_at_1_std": 0.06590280225727854, - "nauc_map_at_20_diff1": 0.5082674529304618, - "nauc_map_at_20_max": 0.4483239199812672, - "nauc_map_at_20_std": 0.05508802163522663, - "nauc_map_at_3_diff1": 0.5326564812078188, - "nauc_map_at_3_max": 0.4609798017337326, - "nauc_map_at_3_std": 0.04817950286142923, - "nauc_map_at_5_diff1": 0.5274072636450247, - "nauc_map_at_5_max": 0.4593382703749472, - "nauc_map_at_5_std": 0.04884229953125181, - "nauc_mrr_at_1000_diff1": 0.510784304760984, - "nauc_mrr_at_1000_max": 0.44976634773529395, - "nauc_mrr_at_1000_std": 0.05570654098630355, - "nauc_mrr_at_100_diff1": 0.5097698322258946, - "nauc_mrr_at_100_max": 0.44894547649673194, - "nauc_mrr_at_100_std": 0.05591543784058362, - "nauc_mrr_at_10_diff1": 0.5142652560397565, - "nauc_mrr_at_10_max": 0.4518266232736721, - "nauc_mrr_at_10_std": 0.052221764538752394, - "nauc_mrr_at_1_diff1": 0.5788284986102574, - "nauc_mrr_at_1_max": 0.5023162373698774, - "nauc_mrr_at_1_std": 0.06590280225727854, - "nauc_mrr_at_20_diff1": 0.5082674529304618, - "nauc_mrr_at_20_max": 0.4483239199812672, - "nauc_mrr_at_20_std": 0.05508802163522663, - "nauc_mrr_at_3_diff1": 0.5326564812078188, - "nauc_mrr_at_3_max": 0.4609798017337326, - "nauc_mrr_at_3_std": 0.04817950286142923, - "nauc_mrr_at_5_diff1": 0.5274072636450247, - "nauc_mrr_at_5_max": 0.4593382703749472, - "nauc_mrr_at_5_std": 0.04884229953125181, - "nauc_ndcg_at_1000_diff1": 0.4874518894504884, - "nauc_ndcg_at_1000_max": 0.4332576903427314, - "nauc_ndcg_at_1000_std": 0.061892632637893304, - "nauc_ndcg_at_100_diff1": 0.4647491919732108, - "nauc_ndcg_at_100_max": 0.41517598386589954, - "nauc_ndcg_at_100_std": 0.07455149159361915, - "nauc_ndcg_at_10_diff1": 0.4788542200767202, - "nauc_ndcg_at_10_max": 0.42663150440804193, - "nauc_ndcg_at_10_std": 0.051773329518349155, - "nauc_ndcg_at_1_diff1": 0.5788284986102574, - "nauc_ndcg_at_1_max": 0.5023162373698774, - "nauc_ndcg_at_1_std": 0.06590280225727854, - "nauc_ndcg_at_20_diff1": 0.4584286443534341, - "nauc_ndcg_at_20_max": 0.41589497776504764, - "nauc_ndcg_at_20_std": 0.06316090723521364, - "nauc_ndcg_at_3_diff1": 0.5171896708549963, - "nauc_ndcg_at_3_max": 0.4455034140521195, - "nauc_ndcg_at_3_std": 0.042281025973323616, - "nauc_ndcg_at_5_diff1": 0.5079788124522473, - "nauc_ndcg_at_5_max": 0.4428814009365794, - "nauc_ndcg_at_5_std": 0.04328737509334253, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.24209378407851703, - "nauc_precision_at_100_max": 0.23935646514995684, - "nauc_precision_at_100_std": 0.2226853098689183, - "nauc_precision_at_10_diff1": 0.37657442834841215, - "nauc_precision_at_10_max": 0.3556539153880323, - "nauc_precision_at_10_std": 0.05377148482652421, - "nauc_precision_at_1_diff1": 0.5788284986102574, - "nauc_precision_at_1_max": 0.5023162373698774, - "nauc_precision_at_1_std": 0.06590280225727854, - "nauc_precision_at_20_diff1": 0.29657056660798986, - "nauc_precision_at_20_max": 0.3153698953402739, - "nauc_precision_at_20_std": 0.09941715784373033, - "nauc_precision_at_3_diff1": 0.4754735910934663, - "nauc_precision_at_3_max": 0.4033426469319066, - "nauc_precision_at_3_std": 0.026381936653145834, - "nauc_precision_at_5_diff1": 0.45534679339922834, - "nauc_precision_at_5_max": 0.3986332694050452, - "nauc_precision_at_5_std": 0.028559611372210206, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.24209378407851656, - "nauc_recall_at_100_max": 0.2393564651499565, - "nauc_recall_at_100_std": 0.2226853098689175, - "nauc_recall_at_10_diff1": 0.3765744283484118, - "nauc_recall_at_10_max": 0.3556539153880319, - "nauc_recall_at_10_std": 0.05377148482652398, - "nauc_recall_at_1_diff1": 0.5788284986102574, - "nauc_recall_at_1_max": 0.5023162373698774, - "nauc_recall_at_1_std": 0.06590280225727854, - "nauc_recall_at_20_diff1": 0.29657056660798986, - "nauc_recall_at_20_max": 0.31536989534027415, - "nauc_recall_at_20_std": 0.09941715784373036, - "nauc_recall_at_3_diff1": 0.475473591093467, - "nauc_recall_at_3_max": 0.4033426469319063, - "nauc_recall_at_3_std": 0.026381936653145647, - "nauc_recall_at_5_diff1": 0.4553467933992285, - "nauc_recall_at_5_max": 0.39863326940504556, - "nauc_recall_at_5_std": 0.028559611372210154, - "ndcg_at_1": 0.218, - "ndcg_at_10": 0.31015, - "ndcg_at_100": 0.37861, - "ndcg_at_1000": 0.40927, - "ndcg_at_20": 0.3356, - "ndcg_at_3": 0.26676, - "ndcg_at_5": 0.28621, - "precision_at_1": 0.218, - "precision_at_10": 0.0426, - "precision_at_100": 0.00766, - "precision_at_1000": 0.001, - "precision_at_20": 0.0263, - "precision_at_3": 0.10133, - "precision_at_5": 0.0704, - "recall_at_1": 0.218, - "recall_at_10": 0.426, - "recall_at_100": 0.766, - "recall_at_1000": 1.0, - "recall_at_20": 0.526, - "recall_at_3": 0.304, - "recall_at_5": 0.352 - } - ] - }, - "task_name": "VidoreArxivQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreDocVQARetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreDocVQARetrieval.json deleted file mode 100644 index 61c6949996..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreDocVQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "b1d89eda849e636676df6ead8002602fb1858600", - "evaluation_time": 20.387259244918823, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.106, - "cv_recall_at_10": 0.308, - "cv_recall_at_100": 0.618, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.388, - "cv_recall_at_3": 0.174, - "cv_recall_at_5": 0.228, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.16692, - "map_at_1": 0.106, - "map_at_10": 0.15761, - "map_at_100": 0.16868, - "map_at_1000": 0.17047, - "map_at_20": 0.16321, - "map_at_3": 0.135, - "map_at_5": 0.147, - "mrr_at_1": 0.106, - "mrr_at_10": 0.15760873015873023, - "mrr_at_100": 0.16867595360076532, - "mrr_at_1000": 0.17047065030110975, - "mrr_at_20": 0.16321167112900875, - "mrr_at_3": 0.13500000000000004, - "mrr_at_5": 0.14700000000000008, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.18647709959934627, - "nauc_cv_recall_at_100_max": 0.15991502588767584, - "nauc_cv_recall_at_100_std": 0.3570537403017011, - "nauc_cv_recall_at_10_diff1": 0.2136343326568294, - "nauc_cv_recall_at_10_max": 0.12466152380234613, - "nauc_cv_recall_at_10_std": 0.24616034754709049, - "nauc_cv_recall_at_1_diff1": 0.3991979396235373, - "nauc_cv_recall_at_1_max": 0.3656858545246218, - "nauc_cv_recall_at_1_std": 0.33274666969460814, - "nauc_cv_recall_at_20_diff1": 0.22020310825017214, - "nauc_cv_recall_at_20_max": 0.12771428127540788, - "nauc_cv_recall_at_20_std": 0.23684483088985434, - "nauc_cv_recall_at_3_diff1": 0.32715242283767193, - "nauc_cv_recall_at_3_max": 0.2647489995641667, - "nauc_cv_recall_at_3_std": 0.2599008028700173, - "nauc_cv_recall_at_5_diff1": 0.258351245261759, - "nauc_cv_recall_at_5_max": 0.21310908211317878, - "nauc_cv_recall_at_5_std": 0.27940771223338784, - "nauc_map_at_1000_diff1": 0.3320639485473742, - "nauc_map_at_1000_max": 0.28235594429927185, - "nauc_map_at_1000_std": 0.29583968480955875, - "nauc_map_at_100_diff1": 0.33145311782359116, - "nauc_map_at_100_max": 0.28158165538707064, - "nauc_map_at_100_std": 0.2958828545201129, - "nauc_map_at_10_diff1": 0.33156624683420866, - "nauc_map_at_10_max": 0.279553774946279, - "nauc_map_at_10_std": 0.2941832501123395, - "nauc_map_at_1_diff1": 0.3991979396235373, - "nauc_map_at_1_max": 0.3656858545246218, - "nauc_map_at_1_std": 0.33274666969460814, - "nauc_map_at_20_diff1": 0.3316839346518803, - "nauc_map_at_20_max": 0.27915064588763655, - "nauc_map_at_20_std": 0.29232482968981444, - "nauc_map_at_3_diff1": 0.36732346501976904, - "nauc_map_at_3_max": 0.31561466377660846, - "nauc_map_at_3_std": 0.2963195329433932, - "nauc_map_at_5_diff1": 0.3434991787671332, - "nauc_map_at_5_max": 0.2997608630800974, - "nauc_map_at_5_std": 0.3022477486176086, - "nauc_mrr_at_1000_diff1": 0.3320639485473742, - "nauc_mrr_at_1000_max": 0.28235594429927185, - "nauc_mrr_at_1000_std": 0.29583968480955875, - "nauc_mrr_at_100_diff1": 0.33145311782359116, - "nauc_mrr_at_100_max": 0.28158165538707064, - "nauc_mrr_at_100_std": 0.2958828545201129, - "nauc_mrr_at_10_diff1": 0.33156624683420866, - "nauc_mrr_at_10_max": 0.279553774946279, - "nauc_mrr_at_10_std": 0.2941832501123395, - "nauc_mrr_at_1_diff1": 0.3991979396235373, - "nauc_mrr_at_1_max": 0.3656858545246218, - "nauc_mrr_at_1_std": 0.33274666969460814, - "nauc_mrr_at_20_diff1": 0.3316839346518803, - "nauc_mrr_at_20_max": 0.27915064588763655, - "nauc_mrr_at_20_std": 0.29232482968981444, - "nauc_mrr_at_3_diff1": 0.36732346501976904, - "nauc_mrr_at_3_max": 0.31561466377660846, - "nauc_mrr_at_3_std": 0.2963195329433932, - "nauc_mrr_at_5_diff1": 0.3434991787671332, - "nauc_mrr_at_5_max": 0.2997608630800974, - "nauc_mrr_at_5_std": 0.3022477486176086, - "nauc_ndcg_at_1000_diff1": 0.30916637131727903, - "nauc_ndcg_at_1000_max": 0.259081036791414, - "nauc_ndcg_at_1000_std": 0.29352066034827357, - "nauc_ndcg_at_100_diff1": 0.29165565271392513, - "nauc_ndcg_at_100_max": 0.24382913622899394, - "nauc_ndcg_at_100_std": 0.3018042533076364, - "nauc_ndcg_at_10_diff1": 0.29439259518708877, - "nauc_ndcg_at_10_max": 0.23166412777636708, - "nauc_ndcg_at_10_std": 0.2789810354221307, - "nauc_ndcg_at_1_diff1": 0.3991979396235373, - "nauc_ndcg_at_1_max": 0.3656858545246218, - "nauc_ndcg_at_1_std": 0.33274666969460814, - "nauc_ndcg_at_20_diff1": 0.29550536417143825, - "nauc_ndcg_at_20_max": 0.23119634036974532, - "nauc_ndcg_at_20_std": 0.2745837194582885, - "nauc_ndcg_at_3_diff1": 0.355735107849091, - "nauc_ndcg_at_3_max": 0.30075662999643216, - "nauc_ndcg_at_3_std": 0.2856705283412383, - "nauc_ndcg_at_5_diff1": 0.31771622814966893, - "nauc_ndcg_at_5_max": 0.2735109839698581, - "nauc_ndcg_at_5_std": 0.2949705422143009, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.18647709959934627, - "nauc_precision_at_100_max": 0.15991502588767598, - "nauc_precision_at_100_std": 0.35705374030170134, - "nauc_precision_at_10_diff1": 0.21363433265682943, - "nauc_precision_at_10_max": 0.1246615238023463, - "nauc_precision_at_10_std": 0.2461603475470906, - "nauc_precision_at_1_diff1": 0.3991979396235373, - "nauc_precision_at_1_max": 0.3656858545246218, - "nauc_precision_at_1_std": 0.33274666969460814, - "nauc_precision_at_20_diff1": 0.22020310825017242, - "nauc_precision_at_20_max": 0.12771428127540807, - "nauc_precision_at_20_std": 0.23684483088985492, - "nauc_precision_at_3_diff1": 0.32715242283767215, - "nauc_precision_at_3_max": 0.2647489995641667, - "nauc_precision_at_3_std": 0.2599008028700174, - "nauc_precision_at_5_diff1": 0.2583512452617593, - "nauc_precision_at_5_max": 0.21310908211317917, - "nauc_precision_at_5_std": 0.27940771223338806, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.18647709959934627, - "nauc_recall_at_100_max": 0.15991502588767584, - "nauc_recall_at_100_std": 0.3570537403017011, - "nauc_recall_at_10_diff1": 0.2136343326568294, - "nauc_recall_at_10_max": 0.12466152380234613, - "nauc_recall_at_10_std": 0.24616034754709049, - "nauc_recall_at_1_diff1": 0.3991979396235373, - "nauc_recall_at_1_max": 0.3656858545246218, - "nauc_recall_at_1_std": 0.33274666969460814, - "nauc_recall_at_20_diff1": 0.22020310825017214, - "nauc_recall_at_20_max": 0.12771428127540788, - "nauc_recall_at_20_std": 0.23684483088985434, - "nauc_recall_at_3_diff1": 0.32715242283767193, - "nauc_recall_at_3_max": 0.2647489995641667, - "nauc_recall_at_3_std": 0.2599008028700173, - "nauc_recall_at_5_diff1": 0.258351245261759, - "nauc_recall_at_5_max": 0.21310908211317878, - "nauc_recall_at_5_std": 0.27940771223338784, - "ndcg_at_1": 0.106, - "ndcg_at_10": 0.1927, - "ndcg_at_100": 0.25482, - "ndcg_at_1000": 0.30362, - "ndcg_at_20": 0.213, - "ndcg_at_3": 0.14498, - "ndcg_at_5": 0.16692, - "precision_at_1": 0.106, - "precision_at_10": 0.0308, - "precision_at_100": 0.00618, - "precision_at_1000": 0.001, - "precision_at_20": 0.0194, - "precision_at_3": 0.058, - "precision_at_5": 0.0456, - "recall_at_1": 0.106, - "recall_at_10": 0.308, - "recall_at_100": 0.618, - "recall_at_1000": 1.0, - "recall_at_20": 0.388, - "recall_at_3": 0.174, - "recall_at_5": 0.228 - } - ] - }, - "task_name": "VidoreDocVQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreInfoVQARetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreInfoVQARetrieval.json deleted file mode 100644 index 8bb3c53692..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreInfoVQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", - "evaluation_time": 19.91568922996521, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.514, - "cv_recall_at_10": 0.78, - "cv_recall_at_100": 0.952, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.85, - "cv_recall_at_3": 0.666, - "cv_recall_at_5": 0.722, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.62564, - "map_at_1": 0.514, - "map_at_10": 0.60106, - "map_at_100": 0.60859, - "map_at_1000": 0.60883, - "map_at_20": 0.6059, - "map_at_3": 0.58033, - "map_at_5": 0.59343, - "mrr_at_1": 0.514, - "mrr_at_10": 0.6010595238095238, - "mrr_at_100": 0.6085922116441255, - "mrr_at_1000": 0.608826236252889, - "mrr_at_20": 0.6058996230893289, - "mrr_at_3": 0.5803333333333334, - "mrr_at_5": 0.5934333333333331, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5994786803610316, - "nauc_cv_recall_at_100_max": 0.4174447556800473, - "nauc_cv_recall_at_100_std": 0.7542600373482705, - "nauc_cv_recall_at_10_diff1": 0.6065006723234561, - "nauc_cv_recall_at_10_max": 0.41218528307135877, - "nauc_cv_recall_at_10_std": 0.13799323039829378, - "nauc_cv_recall_at_1_diff1": 0.7065975273128384, - "nauc_cv_recall_at_1_max": 0.4003963436889161, - "nauc_cv_recall_at_1_std": 0.05878106742438773, - "nauc_cv_recall_at_20_diff1": 0.5701204035144813, - "nauc_cv_recall_at_20_max": 0.4218939147412955, - "nauc_cv_recall_at_20_std": 0.23806703547022504, - "nauc_cv_recall_at_3_diff1": 0.643071383708003, - "nauc_cv_recall_at_3_max": 0.3869760755586455, - "nauc_cv_recall_at_3_std": 0.140661474025036, - "nauc_cv_recall_at_5_diff1": 0.6101338410105434, - "nauc_cv_recall_at_5_max": 0.3596857211946283, - "nauc_cv_recall_at_5_std": 0.1005976210307328, - "nauc_map_at_1000_diff1": 0.6767499194189067, - "nauc_map_at_1000_max": 0.39488883206709513, - "nauc_map_at_1000_std": 0.0840509486527422, - "nauc_map_at_100_diff1": 0.6767211533807833, - "nauc_map_at_100_max": 0.3949524090616767, - "nauc_map_at_100_std": 0.08452792320253084, - "nauc_map_at_10_diff1": 0.676363167757303, - "nauc_map_at_10_max": 0.3950306149617231, - "nauc_map_at_10_std": 0.08043584269066946, - "nauc_map_at_1_diff1": 0.7065975273128384, - "nauc_map_at_1_max": 0.4003963436889161, - "nauc_map_at_1_std": 0.05878106742438773, - "nauc_map_at_20_diff1": 0.6757578583739451, - "nauc_map_at_20_max": 0.3948965844015537, - "nauc_map_at_20_std": 0.08276739928464345, - "nauc_map_at_3_diff1": 0.6814372098114474, - "nauc_map_at_3_max": 0.39504568161146403, - "nauc_map_at_3_std": 0.08606245763290074, - "nauc_map_at_5_diff1": 0.6757588337318032, - "nauc_map_at_5_max": 0.3903524306747273, - "nauc_map_at_5_std": 0.07749784338609057, - "nauc_mrr_at_1000_diff1": 0.6767499194189067, - "nauc_mrr_at_1000_max": 0.39488883206709513, - "nauc_mrr_at_1000_std": 0.0840509486527422, - "nauc_mrr_at_100_diff1": 0.6767211533807833, - "nauc_mrr_at_100_max": 0.3949524090616767, - "nauc_mrr_at_100_std": 0.08452792320253084, - "nauc_mrr_at_10_diff1": 0.676363167757303, - "nauc_mrr_at_10_max": 0.3950306149617231, - "nauc_mrr_at_10_std": 0.08043584269066946, - "nauc_mrr_at_1_diff1": 0.7065975273128384, - "nauc_mrr_at_1_max": 0.4003963436889161, - "nauc_mrr_at_1_std": 0.05878106742438773, - "nauc_mrr_at_20_diff1": 0.6757578583739451, - "nauc_mrr_at_20_max": 0.3948965844015537, - "nauc_mrr_at_20_std": 0.08276739928464345, - "nauc_mrr_at_3_diff1": 0.6814372098114474, - "nauc_mrr_at_3_max": 0.39504568161146403, - "nauc_mrr_at_3_std": 0.08606245763290074, - "nauc_mrr_at_5_diff1": 0.6757588337318032, - "nauc_mrr_at_5_max": 0.3903524306747273, - "nauc_mrr_at_5_std": 0.07749784338609057, - "nauc_ndcg_at_1000_diff1": 0.67013446159532, - "nauc_ndcg_at_1000_max": 0.3951244287190329, - "nauc_ndcg_at_1000_std": 0.09644033556766034, - "nauc_ndcg_at_100_diff1": 0.6686093666822559, - "nauc_ndcg_at_100_max": 0.39599855847861803, - "nauc_ndcg_at_100_std": 0.11257804884700358, - "nauc_ndcg_at_10_diff1": 0.6633460215679075, - "nauc_ndcg_at_10_max": 0.3969814632280433, - "nauc_ndcg_at_10_std": 0.09075371352008957, - "nauc_ndcg_at_1_diff1": 0.7065975273128384, - "nauc_ndcg_at_1_max": 0.4003963436889161, - "nauc_ndcg_at_1_std": 0.05878106742438773, - "nauc_ndcg_at_20_diff1": 0.6610383819053706, - "nauc_ndcg_at_20_max": 0.39712671597648946, - "nauc_ndcg_at_20_std": 0.10152856644348263, - "nauc_ndcg_at_3_diff1": 0.6724945628522271, - "nauc_ndcg_at_3_max": 0.3931631826532289, - "nauc_ndcg_at_3_std": 0.09856146625556418, - "nauc_ndcg_at_5_diff1": 0.6619718384007528, - "nauc_ndcg_at_5_max": 0.3841315571764587, - "nauc_ndcg_at_5_std": 0.08283508024988509, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.5994786803610277, - "nauc_precision_at_100_max": 0.4174447556800509, - "nauc_precision_at_100_std": 0.7542600373482679, - "nauc_precision_at_10_diff1": 0.606500672323457, - "nauc_precision_at_10_max": 0.4121852830713588, - "nauc_precision_at_10_std": 0.1379932303982931, - "nauc_precision_at_1_diff1": 0.7065975273128384, - "nauc_precision_at_1_max": 0.4003963436889161, - "nauc_precision_at_1_std": 0.05878106742438773, - "nauc_precision_at_20_diff1": 0.5701204035144817, - "nauc_precision_at_20_max": 0.42189391474129495, - "nauc_precision_at_20_std": 0.23806703547022473, - "nauc_precision_at_3_diff1": 0.6430713837080028, - "nauc_precision_at_3_max": 0.38697607555864566, - "nauc_precision_at_3_std": 0.14066147402503643, - "nauc_precision_at_5_diff1": 0.6101338410105439, - "nauc_precision_at_5_max": 0.3596857211946276, - "nauc_precision_at_5_std": 0.10059762103073269, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5994786803610316, - "nauc_recall_at_100_max": 0.4174447556800473, - "nauc_recall_at_100_std": 0.7542600373482705, - "nauc_recall_at_10_diff1": 0.6065006723234561, - "nauc_recall_at_10_max": 0.41218528307135877, - "nauc_recall_at_10_std": 0.13799323039829378, - "nauc_recall_at_1_diff1": 0.7065975273128384, - "nauc_recall_at_1_max": 0.4003963436889161, - "nauc_recall_at_1_std": 0.05878106742438773, - "nauc_recall_at_20_diff1": 0.5701204035144813, - "nauc_recall_at_20_max": 0.4218939147412955, - "nauc_recall_at_20_std": 0.23806703547022504, - "nauc_recall_at_3_diff1": 0.643071383708003, - "nauc_recall_at_3_max": 0.3869760755586455, - "nauc_recall_at_3_std": 0.140661474025036, - "nauc_recall_at_5_diff1": 0.6101338410105434, - "nauc_recall_at_5_max": 0.3596857211946283, - "nauc_recall_at_5_std": 0.1005976210307328, - "ndcg_at_1": 0.514, - "ndcg_at_10": 0.64427, - "ndcg_at_100": 0.68104, - "ndcg_at_1000": 0.68721, - "ndcg_at_20": 0.66194, - "ndcg_at_3": 0.60231, - "ndcg_at_5": 0.62564, - "precision_at_1": 0.514, - "precision_at_10": 0.078, - "precision_at_100": 0.00952, - "precision_at_1000": 0.001, - "precision_at_20": 0.0425, - "precision_at_3": 0.222, - "precision_at_5": 0.1444, - "recall_at_1": 0.514, - "recall_at_10": 0.78, - "recall_at_100": 0.952, - "recall_at_1000": 1.0, - "recall_at_20": 0.85, - "recall_at_3": 0.666, - "recall_at_5": 0.722 - } - ] - }, - "task_name": "VidoreInfoVQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreShiftProjectRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreShiftProjectRetrieval.json deleted file mode 100644 index 1217f26233..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreShiftProjectRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", - "evaluation_time": 36.15191340446472, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.11, - "cv_recall_at_10": 0.31, - "cv_recall_at_100": 0.59, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.37, - "cv_recall_at_3": 0.18, - "cv_recall_at_5": 0.24, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.17302, - "map_at_1": 0.11, - "map_at_10": 0.16052, - "map_at_100": 0.16994, - "map_at_1000": 0.17162, - "map_at_20": 0.16487, - "map_at_3": 0.13833, - "map_at_5": 0.15133, - "mrr_at_1": 0.11, - "mrr_at_10": 0.1605238095238095, - "mrr_at_100": 0.16993682614552097, - "mrr_at_1000": 0.171616148007682, - "mrr_at_20": 0.16487272857861085, - "mrr_at_3": 0.13833333333333334, - "mrr_at_5": 0.15133333333333332, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.09664401954598185, - "nauc_cv_recall_at_100_max": 0.2321892379708571, - "nauc_cv_recall_at_100_std": 0.29961700441692257, - "nauc_cv_recall_at_10_diff1": 0.08412546889495404, - "nauc_cv_recall_at_10_max": 0.21329174341530308, - "nauc_cv_recall_at_10_std": 0.15528994178307814, - "nauc_cv_recall_at_1_diff1": 0.11281677026448891, - "nauc_cv_recall_at_1_max": 0.3319324558975262, - "nauc_cv_recall_at_1_std": 0.26501748999672853, - "nauc_cv_recall_at_20_diff1": -0.002768382977056451, - "nauc_cv_recall_at_20_max": 0.14356416249885756, - "nauc_cv_recall_at_20_std": 0.08242468561877275, - "nauc_cv_recall_at_3_diff1": 0.11991720169806679, - "nauc_cv_recall_at_3_max": 0.3230712556572991, - "nauc_cv_recall_at_3_std": 0.3017752517278882, - "nauc_cv_recall_at_5_diff1": 0.07033309369757972, - "nauc_cv_recall_at_5_max": 0.2757458662832493, - "nauc_cv_recall_at_5_std": 0.2587766594775941, - "nauc_map_at_1000_diff1": 0.07924737253621075, - "nauc_map_at_1000_max": 0.2907190868329696, - "nauc_map_at_1000_std": 0.24198829664232294, - "nauc_map_at_100_diff1": 0.07793045130849639, - "nauc_map_at_100_max": 0.29059223848457494, - "nauc_map_at_100_std": 0.24230159395580023, - "nauc_map_at_10_diff1": 0.0896249074215925, - "nauc_map_at_10_max": 0.29429957243051497, - "nauc_map_at_10_std": 0.24209507427102459, - "nauc_map_at_1_diff1": 0.11281677026448891, - "nauc_map_at_1_max": 0.3319324558975262, - "nauc_map_at_1_std": 0.26501748999672853, - "nauc_map_at_20_diff1": 0.08024968159849469, - "nauc_map_at_20_max": 0.2872360074786645, - "nauc_map_at_20_std": 0.23456233639857504, - "nauc_map_at_3_diff1": 0.10023862092088413, - "nauc_map_at_3_max": 0.3218243621616418, - "nauc_map_at_3_std": 0.2780094235646185, - "nauc_map_at_5_diff1": 0.08321524099693559, - "nauc_map_at_5_max": 0.30548488940425605, - "nauc_map_at_5_std": 0.26643845322657783, - "nauc_mrr_at_1000_diff1": 0.07924737253621075, - "nauc_mrr_at_1000_max": 0.2907190868329696, - "nauc_mrr_at_1000_std": 0.24198829664232294, - "nauc_mrr_at_100_diff1": 0.07793045130849639, - "nauc_mrr_at_100_max": 0.29059223848457494, - "nauc_mrr_at_100_std": 0.24230159395580023, - "nauc_mrr_at_10_diff1": 0.0896249074215925, - "nauc_mrr_at_10_max": 0.29429957243051497, - "nauc_mrr_at_10_std": 0.24209507427102459, - "nauc_mrr_at_1_diff1": 0.11281677026448891, - "nauc_mrr_at_1_max": 0.3319324558975262, - "nauc_mrr_at_1_std": 0.26501748999672853, - "nauc_mrr_at_20_diff1": 0.08024968159849469, - "nauc_mrr_at_20_max": 0.2872360074786645, - "nauc_mrr_at_20_std": 0.23456233639857504, - "nauc_mrr_at_3_diff1": 0.10023862092088413, - "nauc_mrr_at_3_max": 0.3218243621616418, - "nauc_mrr_at_3_std": 0.2780094235646185, - "nauc_mrr_at_5_diff1": 0.08321524099693559, - "nauc_mrr_at_5_max": 0.30548488940425605, - "nauc_mrr_at_5_std": 0.26643845322657783, - "nauc_ndcg_at_1000_diff1": 0.06615008028731253, - "nauc_ndcg_at_1000_max": 0.2748275557288526, - "nauc_ndcg_at_1000_std": 0.23349999155014844, - "nauc_ndcg_at_100_diff1": 0.038763196864327186, - "nauc_ndcg_at_100_max": 0.26890993061914903, - "nauc_ndcg_at_100_std": 0.2432662566601855, - "nauc_ndcg_at_10_diff1": 0.08806371511158409, - "nauc_ndcg_at_10_max": 0.2702528801580752, - "nauc_ndcg_at_10_std": 0.21641774547281548, - "nauc_ndcg_at_1_diff1": 0.11281677026448891, - "nauc_ndcg_at_1_max": 0.3319324558975262, - "nauc_ndcg_at_1_std": 0.26501748999672853, - "nauc_ndcg_at_20_diff1": 0.0580246437643237, - "nauc_ndcg_at_20_max": 0.24637629255838372, - "nauc_ndcg_at_20_std": 0.19127929130989058, - "nauc_ndcg_at_3_diff1": 0.10541266833562696, - "nauc_ndcg_at_3_max": 0.3219743407161455, - "nauc_ndcg_at_3_std": 0.28479647710896433, - "nauc_ndcg_at_5_diff1": 0.07896740745121697, - "nauc_ndcg_at_5_max": 0.29633957889754337, - "nauc_ndcg_at_5_std": 0.26451689568166686, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.0966440195459817, - "nauc_precision_at_100_max": 0.23218923797085725, - "nauc_precision_at_100_std": 0.2996170044169224, - "nauc_precision_at_10_diff1": 0.08412546889495406, - "nauc_precision_at_10_max": 0.21329174341530296, - "nauc_precision_at_10_std": 0.15528994178307795, - "nauc_precision_at_1_diff1": 0.11281677026448891, - "nauc_precision_at_1_max": 0.3319324558975262, - "nauc_precision_at_1_std": 0.26501748999672853, - "nauc_precision_at_20_diff1": -0.0027683829770564048, - "nauc_precision_at_20_max": 0.14356416249885745, - "nauc_precision_at_20_std": 0.082424685618773, - "nauc_precision_at_3_diff1": 0.11991720169806704, - "nauc_precision_at_3_max": 0.32307125565729955, - "nauc_precision_at_3_std": 0.3017752517278884, - "nauc_precision_at_5_diff1": 0.07033309369757954, - "nauc_precision_at_5_max": 0.27574586628324943, - "nauc_precision_at_5_std": 0.2587766594775939, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.09664401954598185, - "nauc_recall_at_100_max": 0.2321892379708571, - "nauc_recall_at_100_std": 0.29961700441692257, - "nauc_recall_at_10_diff1": 0.08412546889495404, - "nauc_recall_at_10_max": 0.21329174341530308, - "nauc_recall_at_10_std": 0.15528994178307814, - "nauc_recall_at_1_diff1": 0.11281677026448891, - "nauc_recall_at_1_max": 0.3319324558975262, - "nauc_recall_at_1_std": 0.26501748999672853, - "nauc_recall_at_20_diff1": -0.002768382977056451, - "nauc_recall_at_20_max": 0.14356416249885756, - "nauc_recall_at_20_std": 0.08242468561877275, - "nauc_recall_at_3_diff1": 0.11991720169806679, - "nauc_recall_at_3_max": 0.3230712556572991, - "nauc_recall_at_3_std": 0.3017752517278882, - "nauc_recall_at_5_diff1": 0.07033309369757972, - "nauc_recall_at_5_max": 0.2757458662832493, - "nauc_recall_at_5_std": 0.2587766594775941, - "ndcg_at_1": 0.11, - "ndcg_at_10": 0.19548, - "ndcg_at_100": 0.25056, - "ndcg_at_1000": 0.30114, - "ndcg_at_20": 0.21089, - "ndcg_at_3": 0.14893, - "ndcg_at_5": 0.17302, - "precision_at_1": 0.11, - "precision_at_10": 0.031, - "precision_at_100": 0.0059, - "precision_at_1000": 0.001, - "precision_at_20": 0.0185, - "precision_at_3": 0.06, - "precision_at_5": 0.048, - "recall_at_1": 0.11, - "recall_at_10": 0.31, - "recall_at_100": 0.59, - "recall_at_1000": 1.0, - "recall_at_20": 0.37, - "recall_at_3": 0.18, - "recall_at_5": 0.24 - } - ] - }, - "task_name": "VidoreShiftProjectRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAAIRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAAIRetrieval.json deleted file mode 100644 index 34b30646b3..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAAIRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", - "evaluation_time": 34.92671084403992, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.22, - "cv_recall_at_10": 0.53, - "cv_recall_at_100": 0.88, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.61, - "cv_recall_at_3": 0.39, - "cv_recall_at_5": 0.44, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.34875, - "map_at_1": 0.24, - "map_at_10": 0.33169, - "map_at_100": 0.34394, - "map_at_1000": 0.34446, - "map_at_20": 0.33755, - "map_at_3": 0.30667, - "map_at_5": 0.31817, - "mrr_at_1": 0.22, - "mrr_at_10": 0.3172222222222222, - "mrr_at_100": 0.3304223584351638, - "mrr_at_1000": 0.3309363490911448, - "mrr_at_20": 0.32343802396433974, - "mrr_at_3": 0.29333333333333333, - "mrr_at_5": 0.3048333333333333, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.25588609608654145, - "nauc_cv_recall_at_100_max": 0.34254692968501516, - "nauc_cv_recall_at_100_std": -0.030862233534839804, - "nauc_cv_recall_at_10_diff1": 0.015945236360637088, - "nauc_cv_recall_at_10_max": 0.5467978941291292, - "nauc_cv_recall_at_10_std": 0.1269708174682812, - "nauc_cv_recall_at_1_diff1": 0.2607763309196009, - "nauc_cv_recall_at_1_max": 0.31143363271902275, - "nauc_cv_recall_at_1_std": -0.02879385896758557, - "nauc_cv_recall_at_20_diff1": 0.0002417174021419322, - "nauc_cv_recall_at_20_max": 0.5049627604127324, - "nauc_cv_recall_at_20_std": 0.06206094300001453, - "nauc_cv_recall_at_3_diff1": -0.01054156480798838, - "nauc_cv_recall_at_3_max": 0.5327823263875415, - "nauc_cv_recall_at_3_std": -0.0017849520779171265, - "nauc_cv_recall_at_5_diff1": 0.09731429764576903, - "nauc_cv_recall_at_5_max": 0.5683040141984653, - "nauc_cv_recall_at_5_std": 0.06458474708983654, - "nauc_map_at_1000_diff1": 0.10443462676821944, - "nauc_map_at_1000_max": 0.4090433812678768, - "nauc_map_at_1000_std": -0.016759060369677045, - "nauc_map_at_100_diff1": 0.10470542206159071, - "nauc_map_at_100_max": 0.4090673255951132, - "nauc_map_at_100_std": -0.016784069113126018, - "nauc_map_at_10_diff1": 0.10170960093274041, - "nauc_map_at_10_max": 0.41526994097301995, - "nauc_map_at_10_std": -0.013590212809351342, - "nauc_map_at_1_diff1": 0.19137910376228143, - "nauc_map_at_1_max": 0.28789240354660905, - "nauc_map_at_1_std": -0.04003414809489571, - "nauc_map_at_20_diff1": 0.10300773263289635, - "nauc_map_at_20_max": 0.4121548149249788, - "nauc_map_at_20_std": -0.0161035883846474, - "nauc_map_at_3_diff1": 0.09119254288953736, - "nauc_map_at_3_max": 0.4142859798145018, - "nauc_map_at_3_std": -0.027759707068641875, - "nauc_map_at_5_diff1": 0.1003239990473492, - "nauc_map_at_5_max": 0.4226964161286666, - "nauc_map_at_5_std": -0.024585700506800114, - "nauc_mrr_at_1000_diff1": 0.13095044354935767, - "nauc_mrr_at_1000_max": 0.4088406531555248, - "nauc_mrr_at_1000_std": -0.006095326008767425, - "nauc_mrr_at_100_diff1": 0.13118474764897708, - "nauc_mrr_at_100_max": 0.4088652545299387, - "nauc_mrr_at_100_std": -0.0061360067675142686, - "nauc_mrr_at_10_diff1": 0.1271792759098105, - "nauc_mrr_at_10_max": 0.4164808103965064, - "nauc_mrr_at_10_std": -0.0011619168678242003, - "nauc_mrr_at_1_diff1": 0.2607763309196009, - "nauc_mrr_at_1_max": 0.31143363271902275, - "nauc_mrr_at_1_std": -0.02879385896758557, - "nauc_mrr_at_20_diff1": 0.1273336734023317, - "nauc_mrr_at_20_max": 0.4119379927819901, - "nauc_mrr_at_20_std": -0.006324913135721497, - "nauc_mrr_at_3_diff1": 0.11325609802287717, - "nauc_mrr_at_3_max": 0.4132688811453676, - "nauc_mrr_at_3_std": -0.020022157412317262, - "nauc_mrr_at_5_diff1": 0.13761856788427215, - "nauc_mrr_at_5_max": 0.4200430161805268, - "nauc_mrr_at_5_std": -0.006370825854209519, - "nauc_ndcg_at_1000_diff1": 0.09617174046670891, - "nauc_ndcg_at_1000_max": 0.4203264186005738, - "nauc_ndcg_at_1000_std": -0.0068412291888518975, - "nauc_ndcg_at_100_diff1": 0.10442963683136622, - "nauc_ndcg_at_100_max": 0.4169754620603158, - "nauc_ndcg_at_100_std": -0.007949442426679318, - "nauc_ndcg_at_10_diff1": 0.07882991025385962, - "nauc_ndcg_at_10_max": 0.4457128887951948, - "nauc_ndcg_at_10_std": 0.013698345756966162, - "nauc_ndcg_at_1_diff1": 0.19137910376228143, - "nauc_ndcg_at_1_max": 0.28789240354660905, - "nauc_ndcg_at_1_std": -0.04003414809489571, - "nauc_ndcg_at_20_diff1": 0.0853973025330627, - "nauc_ndcg_at_20_max": 0.4381885352579316, - "nauc_ndcg_at_20_std": 0.005313883337564515, - "nauc_ndcg_at_3_diff1": 0.06342948608957481, - "nauc_ndcg_at_3_max": 0.44683332935967496, - "nauc_ndcg_at_3_std": -0.020959173735207975, - "nauc_ndcg_at_5_diff1": 0.0800650733044633, - "nauc_ndcg_at_5_max": 0.463683639664684, - "nauc_ndcg_at_5_std": -0.014099286510659888, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.25588609608654206, - "nauc_precision_at_100_max": 0.34254692968501455, - "nauc_precision_at_100_std": -0.03086223353483784, - "nauc_precision_at_10_diff1": 0.012461448695507674, - "nauc_precision_at_10_max": 0.5267566891722935, - "nauc_precision_at_10_std": 0.10683226362146125, - "nauc_precision_at_1_diff1": 0.19137910376228143, - "nauc_precision_at_1_max": 0.28789240354660905, - "nauc_precision_at_1_std": -0.04003414809489571, - "nauc_precision_at_20_diff1": 0.03819647417161561, - "nauc_precision_at_20_max": 0.5101375257668525, - "nauc_precision_at_20_std": 0.08176168353690481, - "nauc_precision_at_3_diff1": -0.01054156480798847, - "nauc_precision_at_3_max": 0.5327823263875415, - "nauc_precision_at_3_std": -0.001784952077917081, - "nauc_precision_at_5_diff1": 0.028488281046093265, - "nauc_precision_at_5_max": 0.5765255520175396, - "nauc_precision_at_5_std": 0.016899827739208093, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.25588609608654145, - "nauc_recall_at_100_max": 0.34254692968501516, - "nauc_recall_at_100_std": -0.030862233534839804, - "nauc_recall_at_10_diff1": 0.012461448695507143, - "nauc_recall_at_10_max": 0.5267566891722932, - "nauc_recall_at_10_std": 0.10683226362146066, - "nauc_recall_at_1_diff1": 0.19137910376228143, - "nauc_recall_at_1_max": 0.28789240354660905, - "nauc_recall_at_1_std": -0.04003414809489571, - "nauc_recall_at_20_diff1": 0.03819647417161486, - "nauc_recall_at_20_max": 0.5101375257668519, - "nauc_recall_at_20_std": 0.08176168353690391, - "nauc_recall_at_3_diff1": -0.01054156480798838, - "nauc_recall_at_3_max": 0.5327823263875415, - "nauc_recall_at_3_std": -0.0017849520779171265, - "nauc_recall_at_5_diff1": 0.028488281046092762, - "nauc_recall_at_5_max": 0.5765255520175391, - "nauc_recall_at_5_std": 0.0168998277392075, - "ndcg_at_1": 0.24, - "ndcg_at_10": 0.38125, - "ndcg_at_100": 0.44966, - "ndcg_at_1000": 0.4647, - "ndcg_at_20": 0.40182, - "ndcg_at_3": 0.32809, - "ndcg_at_5": 0.34875, - "precision_at_1": 0.24, - "precision_at_10": 0.054, - "precision_at_100": 0.0088, - "precision_at_1000": 0.001, - "precision_at_20": 0.031, - "precision_at_3": 0.13, - "precision_at_5": 0.088, - "recall_at_1": 0.24, - "recall_at_10": 0.54, - "recall_at_100": 0.88, - "recall_at_1000": 1.0, - "recall_at_20": 0.62, - "recall_at_3": 0.39, - "recall_at_5": 0.44 - } - ] - }, - "task_name": "VidoreSyntheticDocQAAIRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAEnergyRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAEnergyRetrieval.json deleted file mode 100644 index ab864d679f..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAEnergyRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", - "evaluation_time": 34.53333258628845, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.37, - "cv_recall_at_10": 0.76, - "cv_recall_at_100": 0.95, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.83, - "cv_recall_at_3": 0.64, - "cv_recall_at_5": 0.68, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.53058, - "map_at_1": 0.32, - "map_at_10": 0.4915, - "map_at_100": 0.4978, - "map_at_1000": 0.4981, - "map_at_20": 0.49541, - "map_at_3": 0.46667, - "map_at_5": 0.47917, - "mrr_at_1": 0.37, - "mrr_at_10": 0.5126746031746032, - "mrr_at_100": 0.5198506079345971, - "mrr_at_1000": 0.5201440447016661, - "mrr_at_20": 0.5174591278604437, - "mrr_at_3": 0.49166666666666664, - "mrr_at_5": 0.5011666666666666, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.21447245564892614, - "nauc_cv_recall_at_100_max": 0.30924369747899044, - "nauc_cv_recall_at_100_std": 0.4916900093370688, - "nauc_cv_recall_at_10_diff1": -0.03153932681491796, - "nauc_cv_recall_at_10_max": 0.47771913126243837, - "nauc_cv_recall_at_10_std": -0.2724971878515191, - "nauc_cv_recall_at_1_diff1": 0.13080609566591367, - "nauc_cv_recall_at_1_max": 0.2643805743088836, - "nauc_cv_recall_at_1_std": -0.02580341869180856, - "nauc_cv_recall_at_20_diff1": -0.19083415568584217, - "nauc_cv_recall_at_20_max": 0.34146978580135867, - "nauc_cv_recall_at_20_std": -0.23689557090613594, - "nauc_cv_recall_at_3_diff1": -0.1465704432239113, - "nauc_cv_recall_at_3_max": 0.3345095159024136, - "nauc_cv_recall_at_3_std": -0.2043045088772514, - "nauc_cv_recall_at_5_diff1": -0.10203164950590858, - "nauc_cv_recall_at_5_max": 0.3911270817497059, - "nauc_cv_recall_at_5_std": -0.19333494575357663, - "nauc_map_at_1000_diff1": 0.12011397834904365, - "nauc_map_at_1000_max": 0.28323827692841, - "nauc_map_at_1000_std": -0.1905001367686242, - "nauc_map_at_100_diff1": 0.11966738540821883, - "nauc_map_at_100_max": 0.28323118429537103, - "nauc_map_at_100_std": -0.1898627518331248, - "nauc_map_at_10_diff1": 0.12065719889856055, - "nauc_map_at_10_max": 0.29021890060830574, - "nauc_map_at_10_std": -0.19420513505740164, - "nauc_map_at_1_diff1": 0.2819891897677404, - "nauc_map_at_1_max": 0.2161778871882692, - "nauc_map_at_1_std": -0.16618056298833367, - "nauc_map_at_20_diff1": 0.11805338930333743, - "nauc_map_at_20_max": 0.2847181951397561, - "nauc_map_at_20_std": -0.1908796432928892, - "nauc_map_at_3_diff1": 0.09382723677706396, - "nauc_map_at_3_max": 0.27825677782425196, - "nauc_map_at_3_std": -0.19836780866192627, - "nauc_map_at_5_diff1": 0.10911512170369399, - "nauc_map_at_5_max": 0.28478822185507247, - "nauc_map_at_5_std": -0.18782586739147047, - "nauc_mrr_at_1000_diff1": 0.010318386026540968, - "nauc_mrr_at_1000_max": 0.28476023988898364, - "nauc_mrr_at_1000_std": -0.11479441647303681, - "nauc_mrr_at_100_diff1": 0.009982532144116743, - "nauc_mrr_at_100_max": 0.2847509614579322, - "nauc_mrr_at_100_std": -0.11424005887976545, - "nauc_mrr_at_10_diff1": 0.01425407904727165, - "nauc_mrr_at_10_max": 0.29299876743388203, - "nauc_mrr_at_10_std": -0.11910098135182312, - "nauc_mrr_at_1_diff1": 0.13080609566591367, - "nauc_mrr_at_1_max": 0.2643805743088836, - "nauc_mrr_at_1_std": -0.02580341869180856, - "nauc_mrr_at_20_diff1": 0.00914945533225785, - "nauc_mrr_at_20_max": 0.2862026177193902, - "nauc_mrr_at_20_std": -0.11575968447469559, - "nauc_mrr_at_3_diff1": -0.006611838316696035, - "nauc_mrr_at_3_max": 0.2755136338315525, - "nauc_mrr_at_3_std": -0.11688827568051624, - "nauc_mrr_at_5_diff1": 0.004146095220944025, - "nauc_mrr_at_5_max": 0.2862054857241672, - "nauc_mrr_at_5_std": -0.11479469794696757, - "nauc_ndcg_at_1000_diff1": 0.09693517981423239, - "nauc_ndcg_at_1000_max": 0.2911411346799712, - "nauc_ndcg_at_1000_std": -0.19237592779877516, - "nauc_ndcg_at_100_diff1": 0.08831100693949918, - "nauc_ndcg_at_100_max": 0.29170420800681346, - "nauc_ndcg_at_100_std": -0.17476915296633083, - "nauc_ndcg_at_10_diff1": 0.08365375220406525, - "nauc_ndcg_at_10_max": 0.32139912966631373, - "nauc_ndcg_at_10_std": -0.20956662213502888, - "nauc_ndcg_at_1_diff1": 0.2819891897677404, - "nauc_ndcg_at_1_max": 0.2161778871882692, - "nauc_ndcg_at_1_std": -0.16618056298833367, - "nauc_ndcg_at_20_diff1": 0.07085280370170333, - "nauc_ndcg_at_20_max": 0.29872845359082356, - "nauc_ndcg_at_20_std": -0.197926152444139, - "nauc_ndcg_at_3_diff1": 0.03496567870813315, - "nauc_ndcg_at_3_max": 0.29633703404607786, - "nauc_ndcg_at_3_std": -0.20824883400477262, - "nauc_ndcg_at_5_diff1": 0.06009325276331838, - "nauc_ndcg_at_5_max": 0.30874738372680843, - "nauc_ndcg_at_5_std": -0.18973188186410675, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.21447245564892456, - "nauc_precision_at_100_max": 0.30924369747899483, - "nauc_precision_at_100_std": 0.491690009337072, - "nauc_precision_at_10_diff1": -0.059197421722878354, - "nauc_precision_at_10_max": 0.45963608692732916, - "nauc_precision_at_10_std": -0.28781808822541777, - "nauc_precision_at_1_diff1": 0.2819891897677404, - "nauc_precision_at_1_max": 0.2161778871882692, - "nauc_precision_at_1_std": -0.16618056298833367, - "nauc_precision_at_20_diff1": -0.19083415568583906, - "nauc_precision_at_20_max": 0.3414697858013586, - "nauc_precision_at_20_std": -0.23689557090613314, - "nauc_precision_at_3_diff1": -0.14959494821291394, - "nauc_precision_at_3_max": 0.35252824393989307, - "nauc_precision_at_3_std": -0.23916075150033597, - "nauc_precision_at_5_diff1": -0.10203164950590708, - "nauc_precision_at_5_max": 0.3911270817497069, - "nauc_precision_at_5_std": -0.1933349457535751, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.21447245564892614, - "nauc_recall_at_100_max": 0.30924369747899044, - "nauc_recall_at_100_std": 0.4916900093370688, - "nauc_recall_at_10_diff1": -0.059197421722880034, - "nauc_recall_at_10_max": 0.4596360869273292, - "nauc_recall_at_10_std": -0.28781808822542065, - "nauc_recall_at_1_diff1": 0.2819891897677404, - "nauc_recall_at_1_max": 0.2161778871882692, - "nauc_recall_at_1_std": -0.16618056298833367, - "nauc_recall_at_20_diff1": -0.19083415568584217, - "nauc_recall_at_20_max": 0.34146978580135867, - "nauc_recall_at_20_std": -0.23689557090613594, - "nauc_recall_at_3_diff1": -0.1495949482129152, - "nauc_recall_at_3_max": 0.3525282439398921, - "nauc_recall_at_3_std": -0.23916075150033683, - "nauc_recall_at_5_diff1": -0.10203164950590858, - "nauc_recall_at_5_max": 0.3911270817497059, - "nauc_recall_at_5_std": -0.19333494575357663, - "ndcg_at_1": 0.32, - "ndcg_at_10": 0.55999, - "ndcg_at_100": 0.59578, - "ndcg_at_1000": 0.6024, - "ndcg_at_20": 0.57487, - "ndcg_at_3": 0.50904, - "ndcg_at_5": 0.53058, - "precision_at_1": 0.32, - "precision_at_10": 0.077, - "precision_at_100": 0.0095, - "precision_at_1000": 0.001, - "precision_at_20": 0.0415, - "precision_at_3": 0.21, - "precision_at_5": 0.136, - "recall_at_1": 0.32, - "recall_at_10": 0.77, - "recall_at_100": 0.95, - "recall_at_1000": 1.0, - "recall_at_20": 0.83, - "recall_at_3": 0.63, - "recall_at_5": 0.68 - } - ] - }, - "task_name": "VidoreSyntheticDocQAEnergyRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAGovernmentReportsRetrieval.json deleted file mode 100644 index b846e73a5f..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAGovernmentReportsRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", - "evaluation_time": 35.05716419219971, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.29, - "cv_recall_at_10": 0.68, - "cv_recall_at_100": 0.92, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.77, - "cv_recall_at_3": 0.59, - "cv_recall_at_5": 0.61, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.47815, - "map_at_1": 0.29, - "map_at_10": 0.43789, - "map_at_100": 0.44824, - "map_at_1000": 0.44875, - "map_at_20": 0.44399, - "map_at_3": 0.41833, - "map_at_5": 0.42983, - "mrr_at_1": 0.29, - "mrr_at_10": 0.4382341269841271, - "mrr_at_100": 0.4484619359551061, - "mrr_at_1000": 0.44896652588458674, - "mrr_at_20": 0.4437531526780754, - "mrr_at_3": 0.42333333333333345, - "mrr_at_5": 0.42833333333333345, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.14869281045751423, - "nauc_cv_recall_at_100_max": 0.21381886087768426, - "nauc_cv_recall_at_100_std": 0.2429971988795516, - "nauc_cv_recall_at_10_diff1": 0.08117960058047077, - "nauc_cv_recall_at_10_max": 0.5350701402805611, - "nauc_cv_recall_at_10_std": 0.12226176490912774, - "nauc_cv_recall_at_1_diff1": 0.19073389832836563, - "nauc_cv_recall_at_1_max": 0.19907146683515795, - "nauc_cv_recall_at_1_std": -0.10344353941458466, - "nauc_cv_recall_at_20_diff1": 0.1277948121125314, - "nauc_cv_recall_at_20_max": 0.45941227814954944, - "nauc_cv_recall_at_20_std": 0.01828517714464683, - "nauc_cv_recall_at_3_diff1": 0.07260774502179126, - "nauc_cv_recall_at_3_max": 0.49208329053369904, - "nauc_cv_recall_at_3_std": -0.014791553553347924, - "nauc_cv_recall_at_5_diff1": 0.07505325336515935, - "nauc_cv_recall_at_5_max": 0.5005665251612711, - "nauc_cv_recall_at_5_std": 0.04027616213194748, - "nauc_map_at_1000_diff1": 0.11877256344149041, - "nauc_map_at_1000_max": 0.2720490218954375, - "nauc_map_at_1000_std": -0.0901853515205092, - "nauc_map_at_100_diff1": 0.11885317591762472, - "nauc_map_at_100_max": 0.2721652636804168, - "nauc_map_at_100_std": -0.08947323825938826, - "nauc_map_at_10_diff1": 0.1172708279300533, - "nauc_map_at_10_max": 0.282532047752819, - "nauc_map_at_10_std": -0.08414817563361583, - "nauc_map_at_1_diff1": 0.20611787975770224, - "nauc_map_at_1_max": 0.12768704586349439, - "nauc_map_at_1_std": -0.19106355507328007, - "nauc_map_at_20_diff1": 0.12091348717143217, - "nauc_map_at_20_max": 0.27613935786590416, - "nauc_map_at_20_std": -0.08964440713899123, - "nauc_map_at_3_diff1": 0.09281666092911399, - "nauc_map_at_3_max": 0.27717924962051305, - "nauc_map_at_3_std": -0.09299918442034921, - "nauc_map_at_5_diff1": 0.12036062962477545, - "nauc_map_at_5_max": 0.2796734132716185, - "nauc_map_at_5_std": -0.09220654472338634, - "nauc_mrr_at_1000_diff1": 0.12026493721452002, - "nauc_mrr_at_1000_max": 0.2935938299086023, - "nauc_mrr_at_1000_std": -0.06270575154374189, - "nauc_mrr_at_100_diff1": 0.1203394458874957, - "nauc_mrr_at_100_max": 0.2936737480473231, - "nauc_mrr_at_100_std": -0.06204189363139713, - "nauc_mrr_at_10_diff1": 0.11879442086655934, - "nauc_mrr_at_10_max": 0.3035665197689515, - "nauc_mrr_at_10_std": -0.05735455948583704, - "nauc_mrr_at_1_diff1": 0.19073389832836563, - "nauc_mrr_at_1_max": 0.19907146683515795, - "nauc_mrr_at_1_std": -0.10344353941458466, - "nauc_mrr_at_20_diff1": 0.12052296633532071, - "nauc_mrr_at_20_max": 0.2972690785119719, - "nauc_mrr_at_20_std": -0.06412275539703116, - "nauc_mrr_at_3_diff1": 0.11476782259264771, - "nauc_mrr_at_3_max": 0.30347475442774224, - "nauc_mrr_at_3_std": -0.0737158617353772, - "nauc_mrr_at_5_diff1": 0.11562569511923856, - "nauc_mrr_at_5_max": 0.3041525691607059, - "nauc_mrr_at_5_std": -0.06131800461266482, - "nauc_ndcg_at_1000_diff1": 0.11377699350015925, - "nauc_ndcg_at_1000_max": 0.28899299562034736, - "nauc_ndcg_at_1000_std": -0.07328852794331638, - "nauc_ndcg_at_100_diff1": 0.11528044837168852, - "nauc_ndcg_at_100_max": 0.2870892189870652, - "nauc_ndcg_at_100_std": -0.05994758581865143, - "nauc_ndcg_at_10_diff1": 0.10730189616771577, - "nauc_ndcg_at_10_max": 0.33739176053782455, - "nauc_ndcg_at_10_std": -0.041229286834314025, - "nauc_ndcg_at_1_diff1": 0.20611787975770224, - "nauc_ndcg_at_1_max": 0.12768704586349439, - "nauc_ndcg_at_1_std": -0.19106355507328007, - "nauc_ndcg_at_20_diff1": 0.12519909516253155, - "nauc_ndcg_at_20_max": 0.3147205255038511, - "nauc_ndcg_at_20_std": -0.05945710494891262, - "nauc_ndcg_at_3_diff1": 0.06440362309001588, - "nauc_ndcg_at_3_max": 0.32086799317238685, - "nauc_ndcg_at_3_std": -0.06743828423819527, - "nauc_ndcg_at_5_diff1": 0.11596750866018518, - "nauc_ndcg_at_5_max": 0.3279860926092747, - "nauc_ndcg_at_5_std": -0.06638587402429165, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.14869281045751595, - "nauc_precision_at_100_max": 0.21381886087768318, - "nauc_precision_at_100_std": 0.2429971988795515, - "nauc_precision_at_10_diff1": 0.08117960058047159, - "nauc_precision_at_10_max": 0.5350701402805608, - "nauc_precision_at_10_std": 0.12226176490912925, - "nauc_precision_at_1_diff1": 0.20611787975770224, - "nauc_precision_at_1_max": 0.12768704586349439, - "nauc_precision_at_1_std": -0.19106355507328007, - "nauc_precision_at_20_diff1": 0.18957203134418324, - "nauc_precision_at_20_max": 0.46979181156396266, - "nauc_precision_at_20_std": 0.07613483562850694, - "nauc_precision_at_3_diff1": -0.017881488497581227, - "nauc_precision_at_3_max": 0.44996483976980833, - "nauc_precision_at_3_std": 0.006988992695283658, - "nauc_precision_at_5_diff1": 0.11437405777928256, - "nauc_precision_at_5_max": 0.4810632864658649, - "nauc_precision_at_5_std": 0.012368089099468108, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.14869281045751423, - "nauc_recall_at_100_max": 0.21381886087768426, - "nauc_recall_at_100_std": 0.2429971988795516, - "nauc_recall_at_10_diff1": 0.08117960058047077, - "nauc_recall_at_10_max": 0.5350701402805611, - "nauc_recall_at_10_std": 0.12226176490912774, - "nauc_recall_at_1_diff1": 0.20611787975770224, - "nauc_recall_at_1_max": 0.12768704586349439, - "nauc_recall_at_1_std": -0.19106355507328007, - "nauc_recall_at_20_diff1": 0.1895720313441832, - "nauc_recall_at_20_max": 0.46979181156396255, - "nauc_recall_at_20_std": 0.07613483562850668, - "nauc_recall_at_3_diff1": -0.017881488497581748, - "nauc_recall_at_3_max": 0.4499648397698083, - "nauc_recall_at_3_std": 0.006988992695282862, - "nauc_recall_at_5_diff1": 0.1143740577792816, - "nauc_recall_at_5_max": 0.4810632864658645, - "nauc_recall_at_5_std": 0.012368089099467303, - "ndcg_at_1": 0.29, - "ndcg_at_10": 0.49761, - "ndcg_at_100": 0.54922, - "ndcg_at_1000": 0.56002, - "ndcg_at_20": 0.52184, - "ndcg_at_3": 0.4575, - "ndcg_at_5": 0.47815, - "precision_at_1": 0.29, - "precision_at_10": 0.068, - "precision_at_100": 0.0092, - "precision_at_1000": 0.001, - "precision_at_20": 0.039, - "precision_at_3": 0.19, - "precision_at_5": 0.124, - "recall_at_1": 0.29, - "recall_at_10": 0.68, - "recall_at_100": 0.92, - "recall_at_1000": 1.0, - "recall_at_20": 0.78, - "recall_at_3": 0.57, - "recall_at_5": 0.62 - } - ] - }, - "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json deleted file mode 100644 index 8570c3f0ee..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", - "evaluation_time": 33.74739336967468, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.29, - "cv_recall_at_10": 0.85, - "cv_recall_at_100": 0.93, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.87, - "cv_recall_at_3": 0.6, - "cv_recall_at_5": 0.72, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.52595, - "map_at_1": 0.31, - "map_at_10": 0.47932, - "map_at_100": 0.48276, - "map_at_1000": 0.48316, - "map_at_20": 0.48087, - "map_at_3": 0.43833, - "map_at_5": 0.46133, - "mrr_at_1": 0.29, - "mrr_at_10": 0.4757420634920635, - "mrr_at_100": 0.4791640089167549, - "mrr_at_1000": 0.4795730200133059, - "mrr_at_20": 0.47728968253968257, - "mrr_at_3": 0.43, - "mrr_at_5": 0.4575000000000001, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.19654528478058125, - "nauc_cv_recall_at_100_max": 0.42957182873149247, - "nauc_cv_recall_at_100_std": -0.07476323862878458, - "nauc_cv_recall_at_10_diff1": -0.2242434103481935, - "nauc_cv_recall_at_10_max": 0.44301985030914476, - "nauc_cv_recall_at_10_std": -0.05893263911487101, - "nauc_cv_recall_at_1_diff1": -0.13760421960633507, - "nauc_cv_recall_at_1_max": 0.07348598272049224, - "nauc_cv_recall_at_1_std": -0.00712882710877292, - "nauc_cv_recall_at_20_diff1": -0.30885729097237585, - "nauc_cv_recall_at_20_max": 0.4254239798563282, - "nauc_cv_recall_at_20_std": -0.1016811079019473, - "nauc_cv_recall_at_3_diff1": -0.3601513802315228, - "nauc_cv_recall_at_3_max": 0.33622736717126733, - "nauc_cv_recall_at_3_std": -0.06294152567527489, - "nauc_cv_recall_at_5_diff1": -0.39176968864468903, - "nauc_cv_recall_at_5_max": 0.27016559829059883, - "nauc_cv_recall_at_5_std": -0.07577838827838879, - "nauc_map_at_1000_diff1": -0.2638744951639972, - "nauc_map_at_1000_max": 0.16549033577792002, - "nauc_map_at_1000_std": -0.09178021864822189, - "nauc_map_at_100_diff1": -0.2638803838632255, - "nauc_map_at_100_max": 0.165875694571672, - "nauc_map_at_100_std": -0.09179531209394068, - "nauc_map_at_10_diff1": -0.2613495470274504, - "nauc_map_at_10_max": 0.16861705649237943, - "nauc_map_at_10_std": -0.08941924748102244, - "nauc_map_at_1_diff1": -0.1842908425991907, - "nauc_map_at_1_max": 0.04688277290022446, - "nauc_map_at_1_std": -0.09131855277841287, - "nauc_map_at_20_diff1": -0.2643001111188259, - "nauc_map_at_20_max": 0.16699166558456383, - "nauc_map_at_20_std": -0.0906144307284239, - "nauc_map_at_3_diff1": -0.2878495963843492, - "nauc_map_at_3_max": 0.17777726007291886, - "nauc_map_at_3_std": -0.0957122389311466, - "nauc_map_at_5_diff1": -0.2680322640463455, - "nauc_map_at_5_max": 0.1687211954163152, - "nauc_map_at_5_std": -0.0887669041098019, - "nauc_mrr_at_1000_diff1": -0.23448658723913543, - "nauc_mrr_at_1000_max": 0.18899168193132979, - "nauc_mrr_at_1000_std": -0.03492752674627976, - "nauc_mrr_at_100_diff1": -0.2345317236965136, - "nauc_mrr_at_100_max": 0.1893521317106688, - "nauc_mrr_at_100_std": -0.03501909631084891, - "nauc_mrr_at_10_diff1": -0.232273562005334, - "nauc_mrr_at_10_max": 0.19185458070516728, - "nauc_mrr_at_10_std": -0.033212636577323054, - "nauc_mrr_at_1_diff1": -0.13760421960633507, - "nauc_mrr_at_1_max": 0.07348598272049224, - "nauc_mrr_at_1_std": -0.00712882710877292, - "nauc_mrr_at_20_diff1": -0.23513440790266207, - "nauc_mrr_at_20_max": 0.1903108519693569, - "nauc_mrr_at_20_std": -0.034173102757696694, - "nauc_mrr_at_3_diff1": -0.24945724900105762, - "nauc_mrr_at_3_max": 0.20986215996182014, - "nauc_mrr_at_3_std": -0.03673862796290598, - "nauc_mrr_at_5_diff1": -0.2496203340274323, - "nauc_mrr_at_5_max": 0.18933274365636593, - "nauc_mrr_at_5_std": -0.03722532416877058, - "nauc_ndcg_at_1000_diff1": -0.26887041578092546, - "nauc_ndcg_at_1000_max": 0.18120174826484248, - "nauc_ndcg_at_1000_std": -0.09304740019552571, - "nauc_ndcg_at_100_diff1": -0.26678413669480144, - "nauc_ndcg_at_100_max": 0.1904171225816268, - "nauc_ndcg_at_100_std": -0.0923756902429794, - "nauc_ndcg_at_10_diff1": -0.2614009801076366, - "nauc_ndcg_at_10_max": 0.20542522951307385, - "nauc_ndcg_at_10_std": -0.08560121250285908, - "nauc_ndcg_at_1_diff1": -0.1842908425991907, - "nauc_ndcg_at_1_max": 0.04688277290022446, - "nauc_ndcg_at_1_std": -0.09131855277841287, - "nauc_ndcg_at_20_diff1": -0.2727678331984996, - "nauc_ndcg_at_20_max": 0.19931315260195256, - "nauc_ndcg_at_20_std": -0.09099972088577328, - "nauc_ndcg_at_3_diff1": -0.316753883236658, - "nauc_ndcg_at_3_max": 0.21041750335678436, - "nauc_ndcg_at_3_std": -0.09777860382526975, - "nauc_ndcg_at_5_diff1": -0.28017894422744943, - "nauc_ndcg_at_5_max": 0.1933173642946018, - "nauc_ndcg_at_5_std": -0.08364682820592939, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": -0.19654528478057492, - "nauc_precision_at_100_max": 0.42957182873149663, - "nauc_precision_at_100_std": -0.07476323862878233, - "nauc_precision_at_10_diff1": -0.2242434103481926, - "nauc_precision_at_10_max": 0.4430198503091455, - "nauc_precision_at_10_std": -0.058932639114869526, - "nauc_precision_at_1_diff1": -0.1842908425991907, - "nauc_precision_at_1_max": 0.04688277290022446, - "nauc_precision_at_1_std": -0.09131855277841287, - "nauc_precision_at_20_diff1": -0.3088572909723734, - "nauc_precision_at_20_max": 0.42542397985632724, - "nauc_precision_at_20_std": -0.1016811079019465, - "nauc_precision_at_3_diff1": -0.4065932375473025, - "nauc_precision_at_3_max": 0.3102790511645081, - "nauc_precision_at_3_std": -0.10454419592037588, - "nauc_precision_at_5_diff1": -0.3156478937728926, - "nauc_precision_at_5_max": 0.276175213675214, - "nauc_precision_at_5_std": -0.06089743589743516, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": -0.19654528478058125, - "nauc_recall_at_100_max": 0.42957182873149247, - "nauc_recall_at_100_std": -0.07476323862878458, - "nauc_recall_at_10_diff1": -0.2242434103481935, - "nauc_recall_at_10_max": 0.44301985030914476, - "nauc_recall_at_10_std": -0.05893263911487101, - "nauc_recall_at_1_diff1": -0.1842908425991907, - "nauc_recall_at_1_max": 0.04688277290022446, - "nauc_recall_at_1_std": -0.09131855277841287, - "nauc_recall_at_20_diff1": -0.30885729097237585, - "nauc_recall_at_20_max": 0.4254239798563282, - "nauc_recall_at_20_std": -0.1016811079019473, - "nauc_recall_at_3_diff1": -0.4065932375473035, - "nauc_recall_at_3_max": 0.3102790511645078, - "nauc_recall_at_3_std": -0.10454419592037704, - "nauc_recall_at_5_diff1": -0.315647893772894, - "nauc_recall_at_5_max": 0.2761752136752141, - "nauc_recall_at_5_std": -0.06089743589743614, - "ndcg_at_1": 0.31, - "ndcg_at_10": 0.56863, - "ndcg_at_100": 0.58571, - "ndcg_at_1000": 0.59506, - "ndcg_at_20": 0.57389, - "ndcg_at_3": 0.48464, - "ndcg_at_5": 0.52595, - "precision_at_1": 0.31, - "precision_at_10": 0.085, - "precision_at_100": 0.0093, - "precision_at_1000": 0.001, - "precision_at_20": 0.0435, - "precision_at_3": 0.20667, - "precision_at_5": 0.144, - "recall_at_1": 0.31, - "recall_at_10": 0.85, - "recall_at_100": 0.93, - "recall_at_1000": 1.0, - "recall_at_20": 0.87, - "recall_at_3": 0.62, - "recall_at_5": 0.72 - } - ] - }, - "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTabfquadRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTabfquadRetrieval.json deleted file mode 100644 index 96af80c3d8..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTabfquadRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", - "evaluation_time": 6.725373029708862, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.04643, - "cv_recall_at_10": 0.29643, - "cv_recall_at_100": 0.73929, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.4, - "cv_recall_at_3": 0.16071, - "cv_recall_at_5": 0.24286, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.14514, - "map_at_1": 0.05, - "map_at_10": 0.12193, - "map_at_100": 0.13702, - "map_at_1000": 0.1386, - "map_at_20": 0.1291, - "map_at_3": 0.09762, - "map_at_5": 0.11423, - "mrr_at_1": 0.04642857142857143, - "mrr_at_10": 0.12090561224489796, - "mrr_at_100": 0.1360089000569665, - "mrr_at_1000": 0.13759077042079312, - "mrr_at_20": 0.1280908305844728, - "mrr_at_3": 0.09464285714285713, - "mrr_at_5": 0.11374999999999996, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.12803173158202777, - "nauc_cv_recall_at_100_max": 0.0654788997984262, - "nauc_cv_recall_at_100_std": 0.4255586622450532, - "nauc_cv_recall_at_10_diff1": 0.014522791365928616, - "nauc_cv_recall_at_10_max": 0.3343785342270934, - "nauc_cv_recall_at_10_std": 0.2151285548560583, - "nauc_cv_recall_at_1_diff1": -0.08249571732265588, - "nauc_cv_recall_at_1_max": 0.5310313512620999, - "nauc_cv_recall_at_1_std": 0.2604487007492106, - "nauc_cv_recall_at_20_diff1": 0.00324591731646297, - "nauc_cv_recall_at_20_max": 0.25144824922760045, - "nauc_cv_recall_at_20_std": 0.23772436368986324, - "nauc_cv_recall_at_3_diff1": -0.0041901637103114115, - "nauc_cv_recall_at_3_max": 0.3307836904883966, - "nauc_cv_recall_at_3_std": 0.14303974845346734, - "nauc_cv_recall_at_5_diff1": 0.029390791630302882, - "nauc_cv_recall_at_5_max": 0.3490070031290577, - "nauc_cv_recall_at_5_std": 0.15756508227080204, - "nauc_map_at_1000_diff1": 0.07024249062149376, - "nauc_map_at_1000_max": 0.3532682565434852, - "nauc_map_at_1000_std": 0.11365370017831007, - "nauc_map_at_100_diff1": 0.07061643502220898, - "nauc_map_at_100_max": 0.3507764960731763, - "nauc_map_at_100_std": 0.11587548078896048, - "nauc_map_at_10_diff1": 0.07629220002560363, - "nauc_map_at_10_max": 0.36801613967093916, - "nauc_map_at_10_std": 0.10835899650696748, - "nauc_map_at_1_diff1": 0.13227797155889448, - "nauc_map_at_1_max": 0.3912185212158381, - "nauc_map_at_1_std": -0.04321744796657597, - "nauc_map_at_20_diff1": 0.067463263265385, - "nauc_map_at_20_max": 0.35562412424426926, - "nauc_map_at_20_std": 0.11282823760206927, - "nauc_map_at_3_diff1": 0.09211535320973883, - "nauc_map_at_3_max": 0.3902951639128921, - "nauc_map_at_3_std": 0.08363924252859689, - "nauc_map_at_5_diff1": 0.07955415447338117, - "nauc_map_at_5_max": 0.36905360018748185, - "nauc_map_at_5_std": 0.09305798236023993, - "nauc_mrr_at_1000_diff1": -0.0011983533381621932, - "nauc_mrr_at_1000_max": 0.3807218888272681, - "nauc_mrr_at_1000_std": 0.20305842835780713, - "nauc_mrr_at_100_diff1": -0.00023939715846331688, - "nauc_mrr_at_100_max": 0.37798336319707565, - "nauc_mrr_at_100_std": 0.20460984418849354, - "nauc_mrr_at_10_diff1": -0.0012782569765985967, - "nauc_mrr_at_10_max": 0.39359082264653156, - "nauc_mrr_at_10_std": 0.1979779592188029, - "nauc_mrr_at_1_diff1": -0.08249571732265588, - "nauc_mrr_at_1_max": 0.5310313512620999, - "nauc_mrr_at_1_std": 0.2604487007492106, - "nauc_mrr_at_20_diff1": -0.0021143734778184373, - "nauc_mrr_at_20_max": 0.3826280903094471, - "nauc_mrr_at_20_std": 0.20021923515610932, - "nauc_mrr_at_3_diff1": -0.017440468685185215, - "nauc_mrr_at_3_max": 0.3941611708358232, - "nauc_mrr_at_3_std": 0.1815424142915358, - "nauc_mrr_at_5_diff1": 0.0009224287977095485, - "nauc_mrr_at_5_max": 0.39848050580147903, - "nauc_mrr_at_5_std": 0.1819622237637238, - "nauc_ndcg_at_1000_diff1": 0.06292042013878293, - "nauc_ndcg_at_1000_max": 0.3324384230789723, - "nauc_ndcg_at_1000_std": 0.14549363875421784, - "nauc_ndcg_at_100_diff1": 0.07282815521201402, - "nauc_ndcg_at_100_max": 0.2898769800095464, - "nauc_ndcg_at_100_std": 0.18831946927410187, - "nauc_ndcg_at_10_diff1": 0.06810378455819563, - "nauc_ndcg_at_10_max": 0.3597482951290414, - "nauc_ndcg_at_10_std": 0.14694597443117657, - "nauc_ndcg_at_1_diff1": 0.13227797155889448, - "nauc_ndcg_at_1_max": 0.3912185212158381, - "nauc_ndcg_at_1_std": -0.04321744796657597, - "nauc_ndcg_at_20_diff1": 0.0447225718184246, - "nauc_ndcg_at_20_max": 0.32188100789256113, - "nauc_ndcg_at_20_std": 0.15945765527960085, - "nauc_ndcg_at_3_diff1": 0.08841552608454324, - "nauc_ndcg_at_3_max": 0.3946137737235812, - "nauc_ndcg_at_3_std": 0.10748417874736685, - "nauc_ndcg_at_5_diff1": 0.0714709503254886, - "nauc_ndcg_at_5_max": 0.36215797961625046, - "nauc_ndcg_at_5_std": 0.12058489753937601, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.12803173158202766, - "nauc_precision_at_100_max": 0.06547889979842647, - "nauc_precision_at_100_std": 0.42555866224505257, - "nauc_precision_at_10_diff1": 0.05654763349367328, - "nauc_precision_at_10_max": 0.34480950971017543, - "nauc_precision_at_10_std": 0.2104931050718133, - "nauc_precision_at_1_diff1": 0.13227797155889448, - "nauc_precision_at_1_max": 0.3912185212158381, - "nauc_precision_at_1_std": -0.04321744796657597, - "nauc_precision_at_20_diff1": 0.003245917316463151, - "nauc_precision_at_20_max": 0.2514482492276001, - "nauc_precision_at_20_std": 0.23772436368986333, - "nauc_precision_at_3_diff1": 0.08234531736637818, - "nauc_precision_at_3_max": 0.40334195160981035, - "nauc_precision_at_3_std": 0.15114164611284883, - "nauc_precision_at_5_diff1": 0.05837426155139814, - "nauc_precision_at_5_max": 0.34929757001537187, - "nauc_precision_at_5_std": 0.1668692992839126, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.12803173158202777, - "nauc_recall_at_100_max": 0.0654788997984262, - "nauc_recall_at_100_std": 0.4255586622450532, - "nauc_recall_at_10_diff1": 0.05654763349367305, - "nauc_recall_at_10_max": 0.34480950971017527, - "nauc_recall_at_10_std": 0.21049310507181299, - "nauc_recall_at_1_diff1": 0.13227797155889448, - "nauc_recall_at_1_max": 0.3912185212158381, - "nauc_recall_at_1_std": -0.04321744796657597, - "nauc_recall_at_20_diff1": 0.00324591731646297, - "nauc_recall_at_20_max": 0.25144824922760045, - "nauc_recall_at_20_std": 0.23772436368986324, - "nauc_recall_at_3_diff1": 0.08234531736637819, - "nauc_recall_at_3_max": 0.40334195160981046, - "nauc_recall_at_3_std": 0.1511416461128487, - "nauc_recall_at_5_diff1": 0.05837426155139821, - "nauc_recall_at_5_max": 0.3492975700153716, - "nauc_recall_at_5_std": 0.16686929928391245, - "ndcg_at_1": 0.05, - "ndcg_at_10": 0.16371, - "ndcg_at_100": 0.25125, - "ndcg_at_1000": 0.28643, - "ndcg_at_20": 0.18987, - "ndcg_at_3": 0.11548, - "ndcg_at_5": 0.14514, - "precision_at_1": 0.05, - "precision_at_10": 0.02964, - "precision_at_100": 0.00739, - "precision_at_1000": 0.001, - "precision_at_20": 0.02, - "precision_at_3": 0.05595, - "precision_at_5": 0.04786, - "recall_at_1": 0.05, - "recall_at_10": 0.29643, - "recall_at_100": 0.73929, - "recall_at_1000": 1.0, - "recall_at_20": 0.4, - "recall_at_3": 0.16786, - "recall_at_5": 0.23929 - } - ] - }, - "task_name": "VidoreTabfquadRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTatdqaRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTatdqaRetrieval.json deleted file mode 100644 index b76decc1ed..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VidoreTatdqaRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", - "evaluation_time": 59.205832719802856, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.01022, - "cv_recall_at_10": 0.08539, - "cv_recall_at_100": 0.25256, - "cv_recall_at_1000": 0.80878, - "cv_recall_at_20": 0.12267, - "cv_recall_at_3": 0.03307, - "cv_recall_at_5": 0.05171, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.02729, - "map_at_1": 0.00722, - "map_at_10": 0.02527, - "map_at_100": 0.0308, - "map_at_1000": 0.03258, - "map_at_20": 0.02783, - "map_at_3": 0.01523, - "map_at_5": 0.02019, - "mrr_at_1": 0.010222489476849068, - "mrr_at_10": 0.02842921093071426, - "mrr_at_100": 0.03400601407873887, - "mrr_at_1000": 0.03579084760325294, - "mrr_at_20": 0.030996115929951835, - "mrr_at_3": 0.01964321507316095, - "mrr_at_5": 0.02382240930046101, - "nauc_cv_recall_at_1000_diff1": 0.062245112432012605, - "nauc_cv_recall_at_1000_max": 0.0824049719625231, - "nauc_cv_recall_at_1000_std": 0.1926981253481964, - "nauc_cv_recall_at_100_diff1": -0.0016464566256863254, - "nauc_cv_recall_at_100_max": 0.13593753936259395, - "nauc_cv_recall_at_100_std": 0.20986830714623048, - "nauc_cv_recall_at_10_diff1": -0.07914455925364039, - "nauc_cv_recall_at_10_max": 0.221889572021891, - "nauc_cv_recall_at_10_std": 0.22793307551587869, - "nauc_cv_recall_at_1_diff1": -0.07096976801316256, - "nauc_cv_recall_at_1_max": 0.1406929737474586, - "nauc_cv_recall_at_1_std": 0.3511188157780023, - "nauc_cv_recall_at_20_diff1": -0.04273344090166819, - "nauc_cv_recall_at_20_max": 0.19950094761618484, - "nauc_cv_recall_at_20_std": 0.23339685125761483, - "nauc_cv_recall_at_3_diff1": -0.029092895639082766, - "nauc_cv_recall_at_3_max": 0.3024227478832245, - "nauc_cv_recall_at_3_std": 0.3651209878641501, - "nauc_cv_recall_at_5_diff1": -0.06245270317059542, - "nauc_cv_recall_at_5_max": 0.2780656666101435, - "nauc_cv_recall_at_5_std": 0.2916408018352315, - "nauc_map_at_1000_diff1": -0.07915406935803154, - "nauc_map_at_1000_max": 0.28295299157468146, - "nauc_map_at_1000_std": 0.3338660650388641, - "nauc_map_at_100_diff1": -0.08057721702418681, - "nauc_map_at_100_max": 0.2851732241233053, - "nauc_map_at_100_std": 0.33535597233102077, - "nauc_map_at_10_diff1": -0.10153434963322266, - "nauc_map_at_10_max": 0.3179042357801275, - "nauc_map_at_10_std": 0.35665721016705615, - "nauc_map_at_1_diff1": -0.19014926326861104, - "nauc_map_at_1_max": 0.390026592747675, - "nauc_map_at_1_std": 0.6195628746702934, - "nauc_map_at_20_diff1": -0.08822801352495452, - "nauc_map_at_20_max": 0.3006216111365114, - "nauc_map_at_20_std": 0.344373361564743, - "nauc_map_at_3_diff1": -0.12322204243809397, - "nauc_map_at_3_max": 0.4422719884723983, - "nauc_map_at_3_std": 0.4833438290593387, - "nauc_map_at_5_diff1": -0.09916631427213753, - "nauc_map_at_5_max": 0.367562851560209, - "nauc_map_at_5_std": 0.41793713089980417, - "nauc_mrr_at_1000_diff1": -0.05131706920876975, - "nauc_mrr_at_1000_max": 0.210064636684804, - "nauc_mrr_at_1000_std": 0.2869649829077407, - "nauc_mrr_at_100_diff1": -0.052613232566971556, - "nauc_mrr_at_100_max": 0.21221535560957858, - "nauc_mrr_at_100_std": 0.28811513687379225, - "nauc_mrr_at_10_diff1": -0.06605985271574645, - "nauc_mrr_at_10_max": 0.23039009119491277, - "nauc_mrr_at_10_std": 0.2967871584573022, - "nauc_mrr_at_1_diff1": -0.07096976801316256, - "nauc_mrr_at_1_max": 0.1406929737474586, - "nauc_mrr_at_1_std": 0.3511188157780023, - "nauc_mrr_at_20_diff1": -0.05738441359665133, - "nauc_mrr_at_20_max": 0.22292825397434873, - "nauc_mrr_at_20_std": 0.2933713427853174, - "nauc_mrr_at_3_diff1": -0.041994622737094436, - "nauc_mrr_at_3_max": 0.2490547970067084, - "nauc_mrr_at_3_std": 0.3664687256018941, - "nauc_mrr_at_5_diff1": -0.055577165183634286, - "nauc_mrr_at_5_max": 0.24481872513306857, - "nauc_mrr_at_5_std": 0.3301532065538261, - "nauc_ndcg_at_1000_diff1": -0.027271831337053593, - "nauc_ndcg_at_1000_max": 0.19320311929839745, - "nauc_ndcg_at_1000_std": 0.26230675769834827, - "nauc_ndcg_at_100_diff1": -0.03857256879121206, - "nauc_ndcg_at_100_max": 0.20053651893359686, - "nauc_ndcg_at_100_std": 0.2634249219923024, - "nauc_ndcg_at_10_diff1": -0.09208853881933711, - "nauc_ndcg_at_10_max": 0.2741867263963051, - "nauc_ndcg_at_10_std": 0.3018685154562808, - "nauc_ndcg_at_1_diff1": -0.19014926326861104, - "nauc_ndcg_at_1_max": 0.390026592747675, - "nauc_ndcg_at_1_std": 0.6195628746702934, - "nauc_ndcg_at_20_diff1": -0.0662883493156438, - "nauc_ndcg_at_20_max": 0.2455835992143969, - "nauc_ndcg_at_20_std": 0.2862069918150026, - "nauc_ndcg_at_3_diff1": -0.115158159385534, - "nauc_ndcg_at_3_max": 0.4486641436064538, - "nauc_ndcg_at_3_std": 0.46223973106257327, - "nauc_ndcg_at_5_diff1": -0.08719685556011549, - "nauc_ndcg_at_5_max": 0.34464533880726306, - "nauc_ndcg_at_5_std": 0.382579857386717, - "nauc_precision_at_1000_diff1": 0.06224511243200888, - "nauc_precision_at_1000_max": 0.08240497196252142, - "nauc_precision_at_1000_std": 0.19269812534819536, - "nauc_precision_at_100_diff1": -0.0008778967359981768, - "nauc_precision_at_100_max": 0.13264567779183334, - "nauc_precision_at_100_std": 0.21307362293607623, - "nauc_precision_at_10_diff1": -0.08303221130704748, - "nauc_precision_at_10_max": 0.22554733634346374, - "nauc_precision_at_10_std": 0.2456046858526632, - "nauc_precision_at_1_diff1": -0.19014926326861104, - "nauc_precision_at_1_max": 0.390026592747675, - "nauc_precision_at_1_std": 0.6195628746702934, - "nauc_precision_at_20_diff1": -0.04471530791368069, - "nauc_precision_at_20_max": 0.19301794526935728, - "nauc_precision_at_20_std": 0.23610474317523472, - "nauc_precision_at_3_diff1": -0.10217447389281792, - "nauc_precision_at_3_max": 0.458977611263278, - "nauc_precision_at_3_std": 0.42722577233039744, - "nauc_precision_at_5_diff1": -0.07232585588874545, - "nauc_precision_at_5_max": 0.3115793713279293, - "nauc_precision_at_5_std": 0.3367719489231344, - "nauc_recall_at_1000_diff1": 0.062245112432012605, - "nauc_recall_at_1000_max": 0.0824049719625231, - "nauc_recall_at_1000_std": 0.1926981253481964, - "nauc_recall_at_100_diff1": -0.0008778967359985224, - "nauc_recall_at_100_max": 0.13264567779183312, - "nauc_recall_at_100_std": 0.2130736229360761, - "nauc_recall_at_10_diff1": -0.08303221130704774, - "nauc_recall_at_10_max": 0.2255473363434636, - "nauc_recall_at_10_std": 0.24560468585266304, - "nauc_recall_at_1_diff1": -0.19014926326861104, - "nauc_recall_at_1_max": 0.390026592747675, - "nauc_recall_at_1_std": 0.6195628746702934, - "nauc_recall_at_20_diff1": -0.044715307913680905, - "nauc_recall_at_20_max": 0.19301794526935703, - "nauc_recall_at_20_std": 0.2361047431752346, - "nauc_recall_at_3_diff1": -0.10217447389281765, - "nauc_recall_at_3_max": 0.45897761126327813, - "nauc_recall_at_3_std": 0.42722577233039744, - "nauc_recall_at_5_diff1": -0.07232585588874574, - "nauc_recall_at_5_max": 0.31157937132792907, - "nauc_recall_at_5_std": 0.3367719489231341, - "ndcg_at_1": 0.00722, - "ndcg_at_10": 0.03933, - "ndcg_at_100": 0.07208, - "ndcg_at_1000": 0.13746, - "ndcg_at_20": 0.04872, - "ndcg_at_3": 0.01824, - "ndcg_at_5": 0.02729, - "precision_at_1": 0.00722, - "precision_at_10": 0.0086, - "precision_at_100": 0.00253, - "precision_at_1000": 0.00081, - "precision_at_20": 0.00616, - "precision_at_3": 0.00902, - "precision_at_5": 0.00986, - "recall_at_1": 0.00722, - "recall_at_10": 0.08599, - "recall_at_100": 0.25316, - "recall_at_1000": 0.80878, - "recall_at_20": 0.12327, - "recall_at_3": 0.02706, - "recall_at_5": 0.04931 - } - ] - }, - "task_name": "VidoreTatdqaRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Winoground.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Winoground.json deleted file mode 100644 index bc891ebea0..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/Winoground.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "521ec2ba6f9a5d7380f7cca5a7b44aea5c1d677c", - "evaluation_time": 54.006736040115356, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "accuracy": 0.07999999821186066, - "hf_subset": "default", - "image_acc": 0.10999999940395355, - "languages": [ - "eng-Latn" - ], - "main_score": 0.07999999821186066, - "text_acc": 0.2849999964237213 - } - ] - }, - "task_name": "Winoground" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json deleted file mode 100644 index 69b52c701c..0000000000 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "openai/clip-vit-large-patch14", "revision": "32bd64288804d66eefd0ccbe215aa642df71cc41", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "CLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/CIRRIT2TRetrieval.json b/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/CIRRIT2TRetrieval.json deleted file mode 100644 index a91d4da661..0000000000 --- a/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/CIRRIT2TRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", - "evaluation_time": 13414.161827325821, - "kg_co2_emissions": null, - "mteb_version": "1.12.67", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.2768, - "map_at_1": 0.12414, - "map_at_10": 0.22102, - "map_at_100": 0.23505, - "map_at_1000": 0.23583, - "map_at_20": 0.22904, - "map_at_3": 0.18959, - "map_at_5": 0.20727, - "mrr_at_1": 0.12541966426858514, - "mrr_at_10": 0.2214481747934993, - "mrr_at_100": 0.23534746672978193, - "mrr_at_1000": 0.23613361179255168, - "mrr_at_20": 0.229363862062874, - "mrr_at_3": 0.19044764188649146, - "mrr_at_5": 0.20782174260591652, - "nauc_map_at_1000_diff1": 0.051644929630880985, - "nauc_map_at_1000_max": -0.028815405528502636, - "nauc_map_at_1000_std": -0.020198524699442843, - "nauc_map_at_100_diff1": 0.05124473143253365, - "nauc_map_at_100_max": -0.028632119897176594, - "nauc_map_at_100_std": -0.019628560449452832, - "nauc_map_at_10_diff1": 0.05562870137097447, - "nauc_map_at_10_max": -0.028495628556100824, - "nauc_map_at_10_std": -0.02536320038491407, - "nauc_map_at_1_diff1": 0.0842856352950554, - "nauc_map_at_1_max": -0.06453708554638175, - "nauc_map_at_1_std": -0.05526923569827222, - "nauc_map_at_20_diff1": 0.052613598616759794, - "nauc_map_at_20_max": -0.029475951982590367, - "nauc_map_at_20_std": -0.023762823555117794, - "nauc_map_at_3_diff1": 0.05880862286832309, - "nauc_map_at_3_max": -0.03736131717268504, - "nauc_map_at_3_std": -0.042634580624996886, - "nauc_map_at_5_diff1": 0.05860660116822883, - "nauc_map_at_5_max": -0.03508785748513017, - "nauc_map_at_5_std": -0.03541676336950194, - "nauc_mrr_at_1000_diff1": 0.05207102176098774, - "nauc_mrr_at_1000_max": -0.027027039710112655, - "nauc_mrr_at_1000_std": -0.018340925011646518, - "nauc_mrr_at_100_diff1": 0.05167035757467952, - "nauc_mrr_at_100_max": -0.026832081188300435, - "nauc_mrr_at_100_std": -0.017754235557587542, - "nauc_mrr_at_10_diff1": 0.05607174629690726, - "nauc_mrr_at_10_max": -0.026650963919061615, - "nauc_mrr_at_10_std": -0.023439519375587647, - "nauc_mrr_at_1_diff1": 0.07953246778996732, - "nauc_mrr_at_1_max": -0.06299248593212067, - "nauc_mrr_at_1_std": -0.054162643797493096, - "nauc_mrr_at_20_diff1": 0.052842729664946166, - "nauc_mrr_at_20_max": -0.02756510867227389, - "nauc_mrr_at_20_std": -0.021727381415748443, - "nauc_mrr_at_3_diff1": 0.05837293843551822, - "nauc_mrr_at_3_max": -0.03597640902892992, - "nauc_mrr_at_3_std": -0.03964104042267918, - "nauc_mrr_at_5_diff1": 0.05833672640078505, - "nauc_mrr_at_5_max": -0.03310995104869811, - "nauc_mrr_at_5_std": -0.03382008427172164, - "nauc_ndcg_at_1000_diff1": 0.03606123281509481, - "nauc_ndcg_at_1000_max": -0.015599943386736242, - "nauc_ndcg_at_1000_std": 0.012256741773024273, - "nauc_ndcg_at_100_diff1": 0.02695005113788398, - "nauc_ndcg_at_100_max": -0.0070994293461191915, - "nauc_ndcg_at_100_std": 0.03257925388016779, - "nauc_ndcg_at_10_diff1": 0.04716531655680059, - "nauc_ndcg_at_10_max": -0.011574560261841896, - "nauc_ndcg_at_10_std": -0.002845570474735191, - "nauc_ndcg_at_1_diff1": 0.07696384973044536, - "nauc_ndcg_at_1_max": -0.0645711978776169, - "nauc_ndcg_at_1_std": -0.05613824052026943, - "nauc_ndcg_at_20_diff1": 0.037171068658800745, - "nauc_ndcg_at_20_max": -0.014733746284082054, - "nauc_ndcg_at_20_std": 0.0025827030969972587, - "nauc_ndcg_at_3_diff1": 0.05393062096446941, - "nauc_ndcg_at_3_max": -0.03021653468760562, - "nauc_ndcg_at_3_std": -0.038667943978515695, - "nauc_ndcg_at_5_diff1": 0.0530747472819436, - "nauc_ndcg_at_5_max": -0.02632200815887294, - "nauc_ndcg_at_5_std": -0.02619673280345486, - "nauc_precision_at_1000_diff1": -0.31337858572216665, - "nauc_precision_at_1000_max": 0.040106460630914706, - "nauc_precision_at_1000_std": 0.39471123050414453, - "nauc_precision_at_100_diff1": -0.13314928522696645, - "nauc_precision_at_100_max": 0.09704344396619104, - "nauc_precision_at_100_std": 0.32833150871848465, - "nauc_precision_at_10_diff1": 0.016706080957278685, - "nauc_precision_at_10_max": 0.028890104977194225, - "nauc_precision_at_10_std": 0.054075958829982056, - "nauc_precision_at_1_diff1": 0.07696384973044536, - "nauc_precision_at_1_max": -0.0645711978776169, - "nauc_precision_at_1_std": -0.05613824052026943, - "nauc_precision_at_20_diff1": -0.01980517203283842, - "nauc_precision_at_20_max": 0.02074496687832344, - "nauc_precision_at_20_std": 0.07551452450628639, - "nauc_precision_at_3_diff1": 0.033455817663258815, - "nauc_precision_at_3_max": -0.01298097522003606, - "nauc_precision_at_3_std": -0.02933261669672727, - "nauc_precision_at_5_diff1": 0.030996712811149657, - "nauc_precision_at_5_max": -0.00630530526298891, - "nauc_precision_at_5_std": -0.004748881667681358, - "nauc_recall_at_1000_diff1": -0.3250596534910208, - "nauc_recall_at_1000_max": 0.12736884877407081, - "nauc_recall_at_1000_std": 0.7188677940311853, - "nauc_recall_at_100_diff1": -0.1174097854649418, - "nauc_recall_at_100_max": 0.11271080733878815, - "nauc_recall_at_100_std": 0.36247172343121803, - "nauc_recall_at_10_diff1": 0.02773456006330092, - "nauc_recall_at_10_max": 0.030773925835657637, - "nauc_recall_at_10_std": 0.05648260910488339, - "nauc_recall_at_1_diff1": 0.0842856352950554, - "nauc_recall_at_1_max": -0.06453708554638175, - "nauc_recall_at_1_std": -0.05526923569827222, - "nauc_recall_at_20_diff1": -0.007102304486688877, - "nauc_recall_at_20_max": 0.0238251830397759, - "nauc_recall_at_20_std": 0.07965089580706912, - "nauc_recall_at_3_diff1": 0.043901759927141515, - "nauc_recall_at_3_max": -0.013952110738630563, - "nauc_recall_at_3_std": -0.028953041657461525, - "nauc_recall_at_5_diff1": 0.04156482887300926, - "nauc_recall_at_5_max": -0.006295432031863492, - "nauc_recall_at_5_std": -0.0038079303663845496, - "ndcg_at_1": 0.1259, - "ndcg_at_10": 0.2768, - "ndcg_at_100": 0.34982, - "ndcg_at_1000": 0.37092, - "ndcg_at_20": 0.30577, - "ndcg_at_3": 0.21148, - "ndcg_at_5": 0.24328, - "precision_at_1": 0.1259, - "precision_at_10": 0.04624, - "precision_at_100": 0.00818, - "precision_at_1000": 0.00099, - "precision_at_20": 0.02891, - "precision_at_3": 0.09297, - "precision_at_5": 0.07141, - "recall_at_1": 0.12414, - "recall_at_10": 0.45544, - "recall_at_100": 0.80791, - "recall_at_1000": 0.9741, - "recall_at_20": 0.56954, - "recall_at_3": 0.2741, - "recall_at_5": 0.35132 - } - ] - }, - "task_name": "CIRRIT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json b/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json deleted file mode 100644 index da7a8a862e..0000000000 --- a/results-mieb/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "royokong/e5-v", "revision": "0c1f22679417b3ae925d779442221c40cd1861ab", "release_date": "2024-07-17", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "E5VWrapper"} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/AROCocoOrder.json b/results-mieb/voyage-multimodal-3/1/AROCocoOrder.json deleted file mode 100644 index b2ccd7997b..0000000000 --- a/results-mieb/voyage-multimodal-3/1/AROCocoOrder.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "853ec8757226585a38a80886c51fe0f3f268787c", - "evaluation_time": 7601.064672470093, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.4089964032173157, - "text_acc": 0.4089964032173157 - } - ] - }, - "task_name": "AROCocoOrder" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/AROFlickrOrder.json b/results-mieb/voyage-multimodal-3/1/AROFlickrOrder.json deleted file mode 100644 index 75cc95cea6..0000000000 --- a/results-mieb/voyage-multimodal-3/1/AROFlickrOrder.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "1f9485f69c87947812378a1aedf86410c86a0aa8", - "evaluation_time": 1100.6516120433807, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.3718000054359436, - "text_acc": 0.3718000054359436 - } - ] - }, - "task_name": "AROFlickrOrder" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/AROVisualAttribution.json b/results-mieb/voyage-multimodal-3/1/AROVisualAttribution.json deleted file mode 100644 index 29c10acf9c..0000000000 --- a/results-mieb/voyage-multimodal-3/1/AROVisualAttribution.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "18f7e01358d91df599d723f00e16a18640e19398", - "evaluation_time": 6468.914260625839, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.6587588787078857, - "text_acc": 0.6587588787078857 - } - ] - }, - "task_name": "AROVisualAttribution" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/AROVisualRelation.json b/results-mieb/voyage-multimodal-3/1/AROVisualRelation.json deleted file mode 100644 index 5d5d540699..0000000000 --- a/results-mieb/voyage-multimodal-3/1/AROVisualRelation.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "3867ad4f46a1ac2e63be034d1fc77dd8c2ef7209", - "evaluation_time": 5647.357409238815, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.5194051265716553, - "text_acc": 0.5194051265716553 - } - ] - }, - "task_name": "AROVisualRelation" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR10.json b/results-mieb/voyage-multimodal-3/1/CIFAR10.json deleted file mode 100644 index 887d06ad49..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CIFAR10.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", - "evaluation_time": 490.16403460502625, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.9553800000000001, - "f1": 0.9555334014223762, - "f1_weighted": 0.955533401422376, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9553800000000001, - "scores_per_experiment": [ - { - "accuracy": 0.9583, - "f1": 0.9585482615112164, - "f1_weighted": 0.9585482615112163 - }, - { - "accuracy": 0.956, - "f1": 0.9561424473749524, - "f1_weighted": 0.9561424473749522 - }, - { - "accuracy": 0.956, - "f1": 0.9561017525171834, - "f1_weighted": 0.9561017525171835 - }, - { - "accuracy": 0.9544, - "f1": 0.9545053818964286, - "f1_weighted": 0.9545053818964285 - }, - { - "accuracy": 0.9522, - "f1": 0.9523691638120997, - "f1_weighted": 0.9523691638120996 - } - ] - } - ] - }, - "task_name": "CIFAR10" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR100.json b/results-mieb/voyage-multimodal-3/1/CIFAR100.json deleted file mode 100644 index 1ed53469ef..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CIFAR100.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", - "evaluation_time": 746.6555774211884, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.7813399999999999, - "f1": 0.7742194484494215, - "f1_weighted": 0.7742194484494215, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7813399999999999, - "scores_per_experiment": [ - { - "accuracy": 0.7747, - "f1": 0.7663694868474199, - "f1_weighted": 0.76636948684742 - }, - { - "accuracy": 0.7852, - "f1": 0.7783034822861626, - "f1_weighted": 0.7783034822861625 - }, - { - "accuracy": 0.7808, - "f1": 0.7731842063509314, - "f1_weighted": 0.7731842063509313 - }, - { - "accuracy": 0.7803, - "f1": 0.7736245004512284, - "f1_weighted": 0.7736245004512283 - }, - { - "accuracy": 0.7857, - "f1": 0.7796155663113651, - "f1_weighted": 0.7796155663113651 - } - ] - } - ] - }, - "task_name": "CIFAR100" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR100Clustering.json b/results-mieb/voyage-multimodal-3/1/CIFAR100Clustering.json deleted file mode 100644 index 221465417f..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CIFAR100Clustering.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", - "evaluation_time": 231.98164081573486, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "ari": 0.46531392836213886, - "cluster_accuracy": 0.6042, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7515067668506068, - "nmi": 0.7515067668506068, - "v_measure": 0.7515067668506068 - } - ] - }, - "task_name": "CIFAR100Clustering" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR100ZeroShot.json b/results-mieb/voyage-multimodal-3/1/CIFAR100ZeroShot.json deleted file mode 100644 index 9697091e6f..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CIFAR100ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", - "evaluation_time": 364.0606734752655, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.7192, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7192 - } - ] - }, - "task_name": "CIFAR100ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR10Clustering.json b/results-mieb/voyage-multimodal-3/1/CIFAR10Clustering.json deleted file mode 100644 index 5459074f45..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CIFAR10Clustering.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", - "evaluation_time": 236.98327565193176, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "ari": 0.7986159784876974, - "cluster_accuracy": 0.8244, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8622309717386565, - "nmi": 0.8622309717386565, - "v_measure": 0.8622309717386566 - } - ] - }, - "task_name": "CIFAR10Clustering" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR10ZeroShot.json b/results-mieb/voyage-multimodal-3/1/CIFAR10ZeroShot.json deleted file mode 100644 index 8463987f5d..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CIFAR10ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", - "evaluation_time": 356.5223488807678, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.9456, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9456 - } - ] - }, - "task_name": "CIFAR10ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIRRIT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/CIRRIT2IRetrieval.json deleted file mode 100644 index d53cb3d9c3..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CIRRIT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "503301cd99348035b9675883a543aa1ded0cf07c", - "evaluation_time": 4035.3517696857452, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.1729, - "cv_recall_at_10": 0.55516, - "cv_recall_at_100": 0.87146, - "cv_recall_at_1000": 0.98513, - "cv_recall_at_20": 0.65803, - "cv_recall_at_3": 0.35276, - "cv_recall_at_5": 0.43933, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.35005, - "map_at_1": 0.17026, - "map_at_10": 0.28592, - "map_at_100": 0.29878, - "map_at_1000": 0.29934, - "map_at_20": 0.29326, - "map_at_3": 0.25032, - "map_at_5": 0.27043, - "mrr_at_1": 0.17290167865707434, - "mrr_at_10": 0.28662517604963694, - "mrr_at_100": 0.2994069042767556, - "mrr_at_1000": 0.29996016856575075, - "mrr_at_20": 0.29391993573720654, - "mrr_at_3": 0.2514788169464439, - "mrr_at_5": 0.27127498001598865, - "nauc_cv_recall_at_1000_diff1": -0.3170009939459746, - "nauc_cv_recall_at_1000_max": 0.09882985452245488, - "nauc_cv_recall_at_1000_std": 0.7541941507785804, - "nauc_cv_recall_at_100_diff1": -0.12703984241006452, - "nauc_cv_recall_at_100_max": 0.07035034338244234, - "nauc_cv_recall_at_100_std": 0.26590969436450945, - "nauc_cv_recall_at_10_diff1": -0.06282729604578144, - "nauc_cv_recall_at_10_max": 0.05042200343697883, - "nauc_cv_recall_at_10_std": 0.008458354144268477, - "nauc_cv_recall_at_1_diff1": 3.515431224046151e-05, - "nauc_cv_recall_at_1_max": -0.08628430637720942, - "nauc_cv_recall_at_1_std": -0.07433487824238864, - "nauc_cv_recall_at_20_diff1": -0.09258740959337054, - "nauc_cv_recall_at_20_max": 0.06248701680718945, - "nauc_cv_recall_at_20_std": 0.0493702259511631, - "nauc_cv_recall_at_3_diff1": -0.01803245197203495, - "nauc_cv_recall_at_3_max": 0.021054911702306108, - "nauc_cv_recall_at_3_std": -0.03882970308722248, - "nauc_cv_recall_at_5_diff1": -0.03739685944600389, - "nauc_cv_recall_at_5_max": 0.0390507377060466, - "nauc_cv_recall_at_5_std": -0.01965818597776538, - "nauc_map_at_1000_diff1": -0.013888056487772901, - "nauc_map_at_1000_max": -0.025047113581432034, - "nauc_map_at_1000_std": -0.04615687795133258, - "nauc_map_at_100_diff1": -0.013929180383153755, - "nauc_map_at_100_max": -0.024852376888687074, - "nauc_map_at_100_std": -0.045846793015922024, - "nauc_map_at_10_diff1": -0.013282752194913467, - "nauc_map_at_10_max": -0.023507328813512387, - "nauc_map_at_10_std": -0.04880002828070644, - "nauc_map_at_1_diff1": 0.006972875008966575, - "nauc_map_at_1_max": -0.09078674038972498, - "nauc_map_at_1_std": -0.07738315349064624, - "nauc_map_at_20_diff1": -0.014642392810803973, - "nauc_map_at_20_max": -0.023616316784522622, - "nauc_map_at_20_std": -0.046676108387409335, - "nauc_map_at_3_diff1": -0.004152872482186347, - "nauc_map_at_3_max": -0.030055151218383824, - "nauc_map_at_3_std": -0.05893383119480629, - "nauc_map_at_5_diff1": -0.009966154532398824, - "nauc_map_at_5_max": -0.024600214946042556, - "nauc_map_at_5_std": -0.05267076625586234, - "nauc_mrr_at_1000_diff1": -0.015342812781242809, - "nauc_mrr_at_1000_max": -0.02446839675351005, - "nauc_mrr_at_1000_std": -0.04471699907025565, - "nauc_mrr_at_100_diff1": -0.015387382226896854, - "nauc_mrr_at_100_max": -0.024279183900671086, - "nauc_mrr_at_100_std": -0.04441731123645614, - "nauc_mrr_at_10_diff1": -0.01482075150830492, - "nauc_mrr_at_10_max": -0.023072871766092212, - "nauc_mrr_at_10_std": -0.047457658821462065, - "nauc_mrr_at_1_diff1": 3.515431224046151e-05, - "nauc_mrr_at_1_max": -0.08628430637720942, - "nauc_mrr_at_1_std": -0.07433487824238864, - "nauc_mrr_at_20_diff1": -0.01614543662658322, - "nauc_mrr_at_20_max": -0.02310471528284919, - "nauc_mrr_at_20_std": -0.04526342848718061, - "nauc_mrr_at_3_diff1": -0.006574670211446688, - "nauc_mrr_at_3_max": -0.029183469428734166, - "nauc_mrr_at_3_std": -0.0562752472060744, - "nauc_mrr_at_5_diff1": -0.011737536762471967, - "nauc_mrr_at_5_max": -0.024300128014767088, - "nauc_mrr_at_5_std": -0.05139431149525923, - "nauc_ndcg_at_1000_diff1": -0.024295226300813422, - "nauc_ndcg_at_1000_max": -0.014020784847656736, - "nauc_ndcg_at_1000_std": -0.026752128393981025, - "nauc_ndcg_at_100_diff1": -0.027543699842724954, - "nauc_ndcg_at_100_max": -0.009617940963043888, - "nauc_ndcg_at_100_std": -0.01567309309677566, - "nauc_ndcg_at_10_diff1": -0.025739220828095582, - "nauc_ndcg_at_10_max": -0.0033833300342746863, - "nauc_ndcg_at_10_std": -0.03391736967026064, - "nauc_ndcg_at_1_diff1": 3.515431224046151e-05, - "nauc_ndcg_at_1_max": -0.08628430637720942, - "nauc_ndcg_at_1_std": -0.07433487824238864, - "nauc_ndcg_at_20_diff1": -0.03129848142341674, - "nauc_ndcg_at_20_max": -0.0028210068185106894, - "nauc_ndcg_at_20_std": -0.025766809747319912, - "nauc_ndcg_at_3_diff1": -0.007831804331826687, - "nauc_ndcg_at_3_max": -0.015209020075707921, - "nauc_ndcg_at_3_std": -0.05331550077340847, - "nauc_ndcg_at_5_diff1": -0.01753602339841263, - "nauc_ndcg_at_5_max": -0.006182671781771981, - "nauc_ndcg_at_5_std": -0.04310939231430975, - "nauc_precision_at_1000_diff1": -0.3268111988325224, - "nauc_precision_at_1000_max": 0.041709669149306676, - "nauc_precision_at_1000_std": 0.32473064559600107, - "nauc_precision_at_100_diff1": -0.15269962228413458, - "nauc_precision_at_100_max": 0.06541595153406797, - "nauc_precision_at_100_std": 0.23656402842386526, - "nauc_precision_at_10_diff1": -0.07325353438185137, - "nauc_precision_at_10_max": 0.05403993597313785, - "nauc_precision_at_10_std": 0.010807791672858408, - "nauc_precision_at_1_diff1": 3.515431224046151e-05, - "nauc_precision_at_1_max": -0.08628430637720942, - "nauc_precision_at_1_std": -0.07433487824238864, - "nauc_precision_at_20_diff1": -0.10452717120685504, - "nauc_precision_at_20_max": 0.06487413760673558, - "nauc_precision_at_20_std": 0.049045265681017115, - "nauc_precision_at_3_diff1": -0.025008316851500872, - "nauc_precision_at_3_max": 0.026180552745518643, - "nauc_precision_at_3_std": -0.03899051183927358, - "nauc_precision_at_5_diff1": -0.04617245194188806, - "nauc_precision_at_5_max": 0.04457879931255899, - "nauc_precision_at_5_std": -0.01624571071222153, - "nauc_recall_at_1000_diff1": -0.3170009939459746, - "nauc_recall_at_1000_max": 0.09882985452245488, - "nauc_recall_at_1000_std": 0.7541941507785804, - "nauc_recall_at_100_diff1": -0.12609939922364324, - "nauc_recall_at_100_max": 0.07040950128302399, - "nauc_recall_at_100_std": 0.2665222461130575, - "nauc_recall_at_10_diff1": -0.06210442381973989, - "nauc_recall_at_10_max": 0.051164767833312974, - "nauc_recall_at_10_std": 0.008895461448149669, - "nauc_recall_at_1_diff1": 0.006972875008966575, - "nauc_recall_at_1_max": -0.09078674038972498, - "nauc_recall_at_1_std": -0.07738315349064624, - "nauc_recall_at_20_diff1": -0.09152161662419589, - "nauc_recall_at_20_max": 0.06302964517499607, - "nauc_recall_at_20_std": 0.049771567847766096, - "nauc_recall_at_3_diff1": -0.015476344901671572, - "nauc_recall_at_3_max": 0.020647503145370293, - "nauc_recall_at_3_std": -0.04128457833864728, - "nauc_recall_at_5_diff1": -0.036186390273891984, - "nauc_recall_at_5_max": 0.03974749178499069, - "nauc_recall_at_5_std": -0.0193040791268312, - "ndcg_at_1": 0.1729, - "ndcg_at_10": 0.35005, - "ndcg_at_100": 0.41605, - "ndcg_at_1000": 0.43061, - "ndcg_at_20": 0.37634, - "ndcg_at_3": 0.27647, - "ndcg_at_5": 0.31251, - "precision_at_1": 0.1729, - "precision_at_10": 0.05626, - "precision_at_100": 0.00882, - "precision_at_1000": 0.001, - "precision_at_20": 0.03332, - "precision_at_3": 0.11902, - "precision_at_5": 0.08906, - "recall_at_1": 0.17026, - "recall_at_10": 0.55468, - "recall_at_100": 0.87134, - "recall_at_1000": 0.98513, - "recall_at_20": 0.65767, - "recall_at_3": 0.3512, - "recall_at_5": 0.43849 - } - ] - }, - "task_name": "CIRRIT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CLEVRCountZeroShot.json b/results-mieb/voyage-multimodal-3/1/CLEVRCountZeroShot.json deleted file mode 100644 index 7fc596a170..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CLEVRCountZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "8b5dce4d5393a04fb58b9261b22a881b02e379b1", - "evaluation_time": 2673.3230113983154, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.45653333333333335, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.45653333333333335 - } - ] - }, - "task_name": "CLEVRCountZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CLEVRZeroShot.json b/results-mieb/voyage-multimodal-3/1/CLEVRZeroShot.json deleted file mode 100644 index 334c77fbcf..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CLEVRZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "ec9c04224a95836ca0344a6000ec8d8bc8a6d4f2", - "evaluation_time": 2703.7519657611847, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.13906666666666667, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.13906666666666667 - } - ] - }, - "task_name": "CLEVRZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CUB200I2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/CUB200I2IRetrieval.json deleted file mode 100644 index 0fd1e59bcf..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CUB200I2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "ad08c1307b15a226bf1b64e62656a17f1f85f7ec", - "evaluation_time": 2163.535472869873, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.64429, - "cv_recall_at_10": 0.91146, - "cv_recall_at_100": 0.98826, - "cv_recall_at_1000": 0.99983, - "cv_recall_at_20": 0.94546, - "cv_recall_at_3": 0.79651, - "cv_recall_at_5": 0.8533, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.64429, - "map_at_1": 0.0, - "map_at_10": 0.1002, - "map_at_100": 0.24338, - "map_at_1000": 0.26897, - "map_at_20": 0.1634, - "map_at_3": 0.02418, - "map_at_5": 0.04884, - "mrr_at_1": 0.0, - "mrr_at_10": 0.38507918700790505, - "mrr_at_100": 0.3891901905104831, - "mrr_at_1000": 0.38925599889084067, - "mrr_at_20": 0.38798629740580254, - "mrr_at_3": 0.3548786100563762, - "mrr_at_5": 0.3750460246231719, - "nauc_cv_recall_at_1000_diff1": 0.12277185076635803, - "nauc_cv_recall_at_1000_max": 0.8692910721335301, - "nauc_cv_recall_at_1000_std": 0.5541669634720343, - "nauc_cv_recall_at_100_diff1": -0.6321515418902193, - "nauc_cv_recall_at_100_max": 0.07387669514437997, - "nauc_cv_recall_at_100_std": -0.04785958499869987, - "nauc_cv_recall_at_10_diff1": -0.5677044574472964, - "nauc_cv_recall_at_10_max": 0.0009713859993077379, - "nauc_cv_recall_at_10_std": 0.05011589213768149, - "nauc_cv_recall_at_1_diff1": -0.4046106762756965, - "nauc_cv_recall_at_1_max": 0.0004226548426512101, - "nauc_cv_recall_at_1_std": 0.027432944700061697, - "nauc_cv_recall_at_20_diff1": -0.603899401030247, - "nauc_cv_recall_at_20_max": 0.001148593758454481, - "nauc_cv_recall_at_20_std": -0.019548921899248613, - "nauc_cv_recall_at_3_diff1": -0.4455725103414982, - "nauc_cv_recall_at_3_max": 0.008991726540476934, - "nauc_cv_recall_at_3_std": 0.038874900776692395, - "nauc_cv_recall_at_5_diff1": -0.5017947639447603, - "nauc_cv_recall_at_5_max": 0.000468670147831035, - "nauc_cv_recall_at_5_std": 0.03870377084563178, - "nauc_map_at_1000_diff1": -0.2876895130339865, - "nauc_map_at_1000_max": -0.012896517311254253, - "nauc_map_at_1000_std": 0.14461249150690794, - "nauc_map_at_100_diff1": -0.28681032009565455, - "nauc_map_at_100_max": -0.01236690630903875, - "nauc_map_at_100_std": 0.12302542192767915, - "nauc_map_at_10_diff1": -0.3374463371796672, - "nauc_map_at_10_max": -0.00903324753451852, - "nauc_map_at_10_std": 0.047063840155417105, - "nauc_map_at_1_diff1": NaN, - "nauc_map_at_1_max": NaN, - "nauc_map_at_1_std": NaN, - "nauc_map_at_20_diff1": -0.3155064667819531, - "nauc_map_at_20_max": -0.01380755932826781, - "nauc_map_at_20_std": 0.07916016466857276, - "nauc_map_at_3_diff1": -0.3565761066694584, - "nauc_map_at_3_max": 0.003993550400848401, - "nauc_map_at_3_std": 0.03848458302964767, - "nauc_map_at_5_diff1": -0.356426360214968, - "nauc_map_at_5_max": -0.0028489029065050923, - "nauc_map_at_5_std": 0.03850371509343761, - "nauc_mrr_at_1000_diff1": -0.43800684377365345, - "nauc_mrr_at_1000_max": -0.0005539340869123282, - "nauc_mrr_at_1000_std": 0.02817514259134202, - "nauc_mrr_at_100_diff1": -0.4381292853682621, - "nauc_mrr_at_100_max": -0.0004748461079494744, - "nauc_mrr_at_100_std": 0.02809781745809359, - "nauc_mrr_at_10_diff1": -0.4391379705707844, - "nauc_mrr_at_10_max": -0.0017781045577203935, - "nauc_mrr_at_10_std": 0.030221395566769144, - "nauc_mrr_at_1_diff1": NaN, - "nauc_mrr_at_1_max": NaN, - "nauc_mrr_at_1_std": NaN, - "nauc_mrr_at_20_diff1": -0.4399000080783371, - "nauc_mrr_at_20_max": -0.0006053657448768487, - "nauc_mrr_at_20_std": 0.028534324700874168, - "nauc_mrr_at_3_diff1": -0.42519681466953113, - "nauc_mrr_at_3_max": -0.002367553551530187, - "nauc_mrr_at_3_std": 0.029485330664110954, - "nauc_mrr_at_5_diff1": -0.43346713040751733, - "nauc_mrr_at_5_max": -0.001378564156485059, - "nauc_mrr_at_5_std": 0.0273542041859884, - "nauc_ndcg_at_1000_diff1": -0.3070099919423358, - "nauc_ndcg_at_1000_max": -0.009340238003473103, - "nauc_ndcg_at_1000_std": 0.23271373655751904, - "nauc_ndcg_at_100_diff1": -0.2955390340514443, - "nauc_ndcg_at_100_max": -0.004923830214676451, - "nauc_ndcg_at_100_std": 0.1223181178316258, - "nauc_ndcg_at_10_diff1": -0.3656129326908537, - "nauc_ndcg_at_10_max": -0.006791901707602765, - "nauc_ndcg_at_10_std": 0.03661090493229275, - "nauc_ndcg_at_1_diff1": NaN, - "nauc_ndcg_at_1_max": NaN, - "nauc_ndcg_at_1_std": NaN, - "nauc_ndcg_at_20_diff1": -0.34169185864420143, - "nauc_ndcg_at_20_max": -0.010944008239357693, - "nauc_ndcg_at_20_std": 0.059350958926926344, - "nauc_ndcg_at_3_diff1": -0.3955340807216821, - "nauc_ndcg_at_3_max": 0.0028371458265729526, - "nauc_ndcg_at_3_std": 0.03183977196015828, - "nauc_ndcg_at_5_diff1": -0.3877160890591247, - "nauc_ndcg_at_5_max": -0.0029189946975532806, - "nauc_ndcg_at_5_std": 0.03016904642545801, - "nauc_precision_at_1000_diff1": -0.15317446430623788, - "nauc_precision_at_1000_max": -0.012330932399289705, - "nauc_precision_at_1000_std": 0.46901467409707714, - "nauc_precision_at_100_diff1": -0.2317631455616716, - "nauc_precision_at_100_max": -0.0030020431834504513, - "nauc_precision_at_100_std": 0.15385722543416624, - "nauc_precision_at_10_diff1": -0.35247656600482197, - "nauc_precision_at_10_max": -0.00888510351612272, - "nauc_precision_at_10_std": 0.037710119874103415, - "nauc_precision_at_1_diff1": NaN, - "nauc_precision_at_1_max": NaN, - "nauc_precision_at_1_std": NaN, - "nauc_precision_at_20_diff1": -0.31990309054950195, - "nauc_precision_at_20_max": -0.013165426936379529, - "nauc_precision_at_20_std": 0.06634509720427917, - "nauc_precision_at_3_diff1": -0.39434779116425517, - "nauc_precision_at_3_max": 0.0031527140838212694, - "nauc_precision_at_3_std": 0.032415733853461234, - "nauc_precision_at_5_diff1": -0.38367802159695974, - "nauc_precision_at_5_max": -0.0043498860313263225, - "nauc_precision_at_5_std": 0.029773528967274605, - "nauc_recall_at_1000_diff1": -0.14659919440214333, - "nauc_recall_at_1000_max": -0.01572890369746866, - "nauc_recall_at_1000_std": 0.5701773317964056, - "nauc_recall_at_100_diff1": -0.22574095098196198, - "nauc_recall_at_100_max": -0.0028660455635216874, - "nauc_recall_at_100_std": 0.16027905776692558, - "nauc_recall_at_10_diff1": -0.3380962218041837, - "nauc_recall_at_10_max": -0.009633461116730245, - "nauc_recall_at_10_std": 0.04188238545019402, - "nauc_recall_at_1_diff1": NaN, - "nauc_recall_at_1_max": NaN, - "nauc_recall_at_1_std": NaN, - "nauc_recall_at_20_diff1": -0.3097175381529999, - "nauc_recall_at_20_max": -0.013468091696335054, - "nauc_recall_at_20_std": 0.07041109319858846, - "nauc_recall_at_3_diff1": -0.3605831363960561, - "nauc_recall_at_3_max": 0.0027883037507332492, - "nauc_recall_at_3_std": 0.03930001152121736, - "nauc_recall_at_5_diff1": -0.3607461144071461, - "nauc_recall_at_5_max": -0.003960285973988208, - "nauc_recall_at_5_std": 0.03577635038665928, - "ndcg_at_1": 0.0, - "ndcg_at_10": 0.3976, - "ndcg_at_100": 0.44182, - "ndcg_at_1000": 0.58205, - "ndcg_at_20": 0.36201, - "ndcg_at_3": 0.32687, - "ndcg_at_5": 0.38173, - "precision_at_1": 0.0, - "precision_at_10": 0.43961, - "precision_at_100": 0.14667, - "precision_at_1000": 0.02471, - "precision_at_20": 0.36555, - "precision_at_3": 0.40812, - "precision_at_5": 0.45437, - "recall_at_1": 0.0, - "recall_at_10": 0.15711, - "recall_at_100": 0.52043, - "recall_at_1000": 0.87486, - "recall_at_20": 0.26077, - "recall_at_3": 0.04378, - "recall_at_5": 0.08123 - } - ] - }, - "task_name": "CUB200I2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchCount.json b/results-mieb/voyage-multimodal-3/1/CVBenchCount.json deleted file mode 100644 index ff443295ee..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CVBenchCount.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", - "evaluation_time": 444.4619266986847, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.2639593908629442, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.2639593908629442 - } - ] - }, - "task_name": "CVBenchCount" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchDepth.json b/results-mieb/voyage-multimodal-3/1/CVBenchDepth.json deleted file mode 100644 index 185e34679a..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CVBenchDepth.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", - "evaluation_time": 413.3635849952698, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5316666666666666, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5316666666666666 - } - ] - }, - "task_name": "CVBenchDepth" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchDistance.json b/results-mieb/voyage-multimodal-3/1/CVBenchDistance.json deleted file mode 100644 index a950524030..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CVBenchDistance.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", - "evaluation_time": 402.95056080818176, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.475, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.475 - } - ] - }, - "task_name": "CVBenchDistance" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchRelation.json b/results-mieb/voyage-multimodal-3/1/CVBenchRelation.json deleted file mode 100644 index 9a75e5c540..0000000000 --- a/results-mieb/voyage-multimodal-3/1/CVBenchRelation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", - "evaluation_time": 201.59757685661316, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5353846153846153, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5353846153846153 - } - ] - }, - "task_name": "CVBenchRelation" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Caltech101.json b/results-mieb/voyage-multimodal-3/1/Caltech101.json deleted file mode 100644 index a219d45f7e..0000000000 --- a/results-mieb/voyage-multimodal-3/1/Caltech101.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", - "evaluation_time": 1512.7276003360748, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.913905325443787, - "f1": 0.8977839451811815, - "f1_weighted": 0.9124464844882695, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.913905325443787, - "scores_per_experiment": [ - { - "accuracy": 0.9077909270216963, - "f1": 0.894273201231098, - "f1_weighted": 0.907344141460267 - }, - { - "accuracy": 0.9095989480604865, - "f1": 0.8941508848751586, - "f1_weighted": 0.9101502765347295 - }, - { - "accuracy": 0.9045036160420776, - "f1": 0.8959083825916897, - "f1_weighted": 0.902335674287055 - }, - { - "accuracy": 0.9294871794871795, - "f1": 0.8999318659786849, - "f1_weighted": 0.9275837214048807 - }, - { - "accuracy": 0.9181459566074951, - "f1": 0.9046553912292769, - "f1_weighted": 0.9148186087544156 - } - ] - } - ] - }, - "task_name": "Caltech101" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Caltech101ZeroShot.json b/results-mieb/voyage-multimodal-3/1/Caltech101ZeroShot.json deleted file mode 100644 index 9dbefa140a..0000000000 --- a/results-mieb/voyage-multimodal-3/1/Caltech101ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", - "evaluation_time": 643.5588819980621, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.7817225509533202, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7817225509533202 - } - ] - }, - "task_name": "Caltech101ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Country211.json b/results-mieb/voyage-multimodal-3/1/Country211.json deleted file mode 100644 index a6442dcb2d..0000000000 --- a/results-mieb/voyage-multimodal-3/1/Country211.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", - "evaluation_time": 7564.615749359131, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.08897630331753555, - "f1": 0.06644398101423767, - "f1_weighted": 0.06644398101423768, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.08897630331753555, - "scores_per_experiment": [ - { - "accuracy": 0.08729857819905214, - "f1": 0.06449707199222236, - "f1_weighted": 0.06449707199222238 - }, - { - "accuracy": 0.08725118483412322, - "f1": 0.06513743277476837, - "f1_weighted": 0.06513743277476838 - }, - { - "accuracy": 0.09251184834123223, - "f1": 0.06917324902817186, - "f1_weighted": 0.06917324902817186 - }, - { - "accuracy": 0.09104265402843602, - "f1": 0.06930671211306509, - "f1_weighted": 0.06930671211306509 - }, - { - "accuracy": 0.08677725118483412, - "f1": 0.06410543916296063, - "f1_weighted": 0.06410543916296063 - } - ] - } - ] - }, - "task_name": "Country211" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Country211ZeroShot.json b/results-mieb/voyage-multimodal-3/1/Country211ZeroShot.json deleted file mode 100644 index 4873804f03..0000000000 --- a/results-mieb/voyage-multimodal-3/1/Country211ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", - "evaluation_time": 4199.378444910049, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.11071090047393364, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.11071090047393364 - } - ] - }, - "task_name": "Country211ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/DTD.json b/results-mieb/voyage-multimodal-3/1/DTD.json deleted file mode 100644 index 653786bec4..0000000000 --- a/results-mieb/voyage-multimodal-3/1/DTD.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", - "evaluation_time": 1314.5107147693634, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.7256382978723404, - "f1": 0.7125223958188174, - "f1_weighted": 0.7125223958188174, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7256382978723404, - "scores_per_experiment": [ - { - "accuracy": 0.726063829787234, - "f1": 0.7128435451309848, - "f1_weighted": 0.7128435451309848 - }, - { - "accuracy": 0.7303191489361702, - "f1": 0.7175258840982487, - "f1_weighted": 0.7175258840982487 - }, - { - "accuracy": 0.7351063829787234, - "f1": 0.7246822849042761, - "f1_weighted": 0.7246822849042759 - }, - { - "accuracy": 0.7186170212765958, - "f1": 0.7066549466550418, - "f1_weighted": 0.7066549466550417 - }, - { - "accuracy": 0.7180851063829787, - "f1": 0.7009053183055356, - "f1_weighted": 0.7009053183055356 - } - ] - } - ] - }, - "task_name": "DTD" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/DTDZeroShot.json b/results-mieb/voyage-multimodal-3/1/DTDZeroShot.json deleted file mode 100644 index 97b89a1b72..0000000000 --- a/results-mieb/voyage-multimodal-3/1/DTDZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", - "evaluation_time": 437.40010952949524, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5904255319148937, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5904255319148937 - } - ] - }, - "task_name": "DTDZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/EuroSAT.json b/results-mieb/voyage-multimodal-3/1/EuroSAT.json deleted file mode 100644 index bb3b8ece72..0000000000 --- a/results-mieb/voyage-multimodal-3/1/EuroSAT.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", - "evaluation_time": 264.42366337776184, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.7926666666666666, - "f1": 0.7917939849512137, - "f1_weighted": 0.7920934779674137, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7926666666666666, - "scores_per_experiment": [ - { - "accuracy": 0.8044444444444444, - "f1": 0.8037094408204952, - "f1_weighted": 0.8041734004823994 - }, - { - "accuracy": 0.7892592592592592, - "f1": 0.7892676858734221, - "f1_weighted": 0.7890248894183581 - }, - { - "accuracy": 0.7870370370370371, - "f1": 0.7872003711470628, - "f1_weighted": 0.7865408192601453 - }, - { - "accuracy": 0.805, - "f1": 0.8040550271257725, - "f1_weighted": 0.8038929843599751 - }, - { - "accuracy": 0.7775925925925926, - "f1": 0.7747373997893159, - "f1_weighted": 0.7768352963161905 - } - ] - } - ] - }, - "task_name": "EuroSAT" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/EuroSATZeroShot.json b/results-mieb/voyage-multimodal-3/1/EuroSATZeroShot.json deleted file mode 100644 index 02d4005da2..0000000000 --- a/results-mieb/voyage-multimodal-3/1/EuroSATZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", - "evaluation_time": 233.39685010910034, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5235185185185185, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5235185185185185 - } - ] - }, - "task_name": "EuroSATZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/FER2013.json b/results-mieb/voyage-multimodal-3/1/FER2013.json deleted file mode 100644 index 6426e2e752..0000000000 --- a/results-mieb/voyage-multimodal-3/1/FER2013.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", - "evaluation_time": 272.1406137943268, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.4668988576205072, - "f1": 0.3944239157839031, - "f1_weighted": 0.46477203838065845, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.4668988576205072, - "scores_per_experiment": [ - { - "accuracy": 0.4821677347450543, - "f1": 0.3961622657921301, - "f1_weighted": 0.46406292230733487 - }, - { - "accuracy": 0.4701866815268877, - "f1": 0.39728952325640365, - "f1_weighted": 0.4745316385258285 - }, - { - "accuracy": 0.4742268041237113, - "f1": 0.3946468406633246, - "f1_weighted": 0.47228603834416566 - }, - { - "accuracy": 0.5050153246029535, - "f1": 0.4180255073056839, - "f1_weighted": 0.48855025062664625 - }, - { - "accuracy": 0.4028977431039287, - "f1": 0.3659954419019734, - "f1_weighted": 0.42442934209931704 - } - ] - } - ] - }, - "task_name": "FER2013" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/FER2013ZeroShot.json b/results-mieb/voyage-multimodal-3/1/FER2013ZeroShot.json deleted file mode 100644 index 26e1c82a1c..0000000000 --- a/results-mieb/voyage-multimodal-3/1/FER2013ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", - "evaluation_time": 269.1961929798126, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5135135135135135, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5135135135135135 - } - ] - }, - "task_name": "FER2013ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/FGVCAircraft.json b/results-mieb/voyage-multimodal-3/1/FGVCAircraft.json deleted file mode 100644 index fdd75070bd..0000000000 --- a/results-mieb/voyage-multimodal-3/1/FGVCAircraft.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "91860adfc9a09aabca5cddb5247442109b38e213", - "evaluation_time": 8436.217485189438, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.2252025202520252, - "f1": 0.20703858415613344, - "f1_weighted": 0.20687263986710783, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.2252025202520252, - "scores_per_experiment": [ - { - "accuracy": 0.23072307230723071, - "f1": 0.2122745870540116, - "f1_weighted": 0.2122443878373498 - }, - { - "accuracy": 0.22652265226522653, - "f1": 0.21175395338696293, - "f1_weighted": 0.21152301004474827 - }, - { - "accuracy": 0.22082208220822083, - "f1": 0.20328120444347306, - "f1_weighted": 0.20296849188917818 - }, - { - "accuracy": 0.22532253225322532, - "f1": 0.20487362177158278, - "f1_weighted": 0.2046900408124526 - }, - { - "accuracy": 0.2226222622262226, - "f1": 0.20300955412463678, - "f1_weighted": 0.20293726875181017 - } - ] - } - ] - }, - "task_name": "FGVCAircraft" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/FGVCAircraftZeroShot.json b/results-mieb/voyage-multimodal-3/1/FGVCAircraftZeroShot.json deleted file mode 100644 index a5a381b43b..0000000000 --- a/results-mieb/voyage-multimodal-3/1/FGVCAircraftZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "91860adfc9a09aabca5cddb5247442109b38e213", - "evaluation_time": 2428.0195784568787, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.11671167116711671, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.11671167116711671 - } - ] - }, - "task_name": "FGVCAircraftZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Fashion200kI2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/Fashion200kI2TRetrieval.json deleted file mode 100644 index 9aa4af6e7e..0000000000 --- a/results-mieb/voyage-multimodal-3/1/Fashion200kI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "96a313715ecf67f5dfe70c4fa52406bc7bdfbeee", - "evaluation_time": 2920.238646507263, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.03191, - "cv_recall_at_10": 0.12722, - "cv_recall_at_100": 0.36265, - "cv_recall_at_1000": 0.71364, - "cv_recall_at_20": 0.17611, - "cv_recall_at_3": 0.06545, - "cv_recall_at_5": 0.08836, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.07318, - "map_at_1": 0.03191, - "map_at_10": 0.05672, - "map_at_100": 0.06447, - "map_at_1000": 0.06583, - "map_at_20": 0.06006, - "map_at_3": 0.04646, - "map_at_5": 0.05162, - "mrr_at_1": 0.031908365718960933, - "mrr_at_10": 0.056717298632823265, - "mrr_at_100": 0.06447384392155588, - "mrr_at_1000": 0.06582834414070698, - "mrr_at_20": 0.06005622816900517, - "mrr_at_3": 0.046464853071521144, - "mrr_at_5": 0.05161928137996866, - "nauc_cv_recall_at_1000_diff1": 0.07064142096733762, - "nauc_cv_recall_at_1000_max": 0.14933892098071952, - "nauc_cv_recall_at_1000_std": 0.1860120425733161, - "nauc_cv_recall_at_100_diff1": 0.06431278693555113, - "nauc_cv_recall_at_100_max": 0.099638842754024, - "nauc_cv_recall_at_100_std": 0.11930789207654446, - "nauc_cv_recall_at_10_diff1": 0.12327773856112313, - "nauc_cv_recall_at_10_max": 0.054975009177794736, - "nauc_cv_recall_at_10_std": 0.066868711233558, - "nauc_cv_recall_at_1_diff1": 0.3139494374020095, - "nauc_cv_recall_at_1_max": 0.13028777354452473, - "nauc_cv_recall_at_1_std": 0.04248164918312027, - "nauc_cv_recall_at_20_diff1": 0.09857331248582667, - "nauc_cv_recall_at_20_max": 0.07617887127857093, - "nauc_cv_recall_at_20_std": 0.07353148089369, - "nauc_cv_recall_at_3_diff1": 0.16925343333548418, - "nauc_cv_recall_at_3_max": 0.06310232394286991, - "nauc_cv_recall_at_3_std": 0.021711082664363137, - "nauc_cv_recall_at_5_diff1": 0.14716519740601317, - "nauc_cv_recall_at_5_max": 0.04865909257707613, - "nauc_cv_recall_at_5_std": 0.04051251752627315, - "nauc_map_at_1000_diff1": 0.18891923417066206, - "nauc_map_at_1000_max": 0.08785537204131776, - "nauc_map_at_1000_std": 0.054198501963607405, - "nauc_map_at_100_diff1": 0.18861241659777042, - "nauc_map_at_100_max": 0.08744669384239624, - "nauc_map_at_100_std": 0.053693641168022804, - "nauc_map_at_10_diff1": 0.2016071032113213, - "nauc_map_at_10_max": 0.0832784376344867, - "nauc_map_at_10_std": 0.047034075026690034, - "nauc_map_at_1_diff1": 0.3139494374020095, - "nauc_map_at_1_max": 0.13028777354452473, - "nauc_map_at_1_std": 0.04248164918312027, - "nauc_map_at_20_diff1": 0.19416427337317085, - "nauc_map_at_20_max": 0.08585800930572945, - "nauc_map_at_20_std": 0.048891821597338306, - "nauc_map_at_3_diff1": 0.22786718131396266, - "nauc_map_at_3_max": 0.09363713840892451, - "nauc_map_at_3_std": 0.03247132719959962, - "nauc_map_at_5_diff1": 0.21265602213173257, - "nauc_map_at_5_max": 0.084718050116957, - "nauc_map_at_5_std": 0.03963097845753907, - "nauc_mrr_at_1000_diff1": 0.18891923417066206, - "nauc_mrr_at_1000_max": 0.08785537204131776, - "nauc_mrr_at_1000_std": 0.054198501963607405, - "nauc_mrr_at_100_diff1": 0.18861241659777042, - "nauc_mrr_at_100_max": 0.08744669384239624, - "nauc_mrr_at_100_std": 0.053693641168022804, - "nauc_mrr_at_10_diff1": 0.2016071032113213, - "nauc_mrr_at_10_max": 0.0832784376344867, - "nauc_mrr_at_10_std": 0.047034075026690034, - "nauc_mrr_at_1_diff1": 0.3139494374020095, - "nauc_mrr_at_1_max": 0.13028777354452473, - "nauc_mrr_at_1_std": 0.04248164918312027, - "nauc_mrr_at_20_diff1": 0.19416427337317085, - "nauc_mrr_at_20_max": 0.08585800930572945, - "nauc_mrr_at_20_std": 0.048891821597338306, - "nauc_mrr_at_3_diff1": 0.22786718131396266, - "nauc_mrr_at_3_max": 0.09363713840892451, - "nauc_mrr_at_3_std": 0.03247132719959962, - "nauc_mrr_at_5_diff1": 0.21265602213173257, - "nauc_mrr_at_5_max": 0.084718050116957, - "nauc_mrr_at_5_std": 0.03963097845753907, - "nauc_ndcg_at_1000_diff1": 0.13976312621484005, - "nauc_ndcg_at_1000_max": 0.0988780200747641, - "nauc_ndcg_at_1000_std": 0.09099886566875824, - "nauc_ndcg_at_100_diff1": 0.13274169983767356, - "nauc_ndcg_at_100_max": 0.08923727522653309, - "nauc_ndcg_at_100_std": 0.07994532547221007, - "nauc_ndcg_at_10_diff1": 0.17057951444650982, - "nauc_ndcg_at_10_max": 0.07155992086707413, - "nauc_ndcg_at_10_std": 0.05413784909723341, - "nauc_ndcg_at_1_diff1": 0.3139494374020095, - "nauc_ndcg_at_1_max": 0.13028777354452473, - "nauc_ndcg_at_1_std": 0.04248164918312027, - "nauc_ndcg_at_20_diff1": 0.15371757602855127, - "nauc_ndcg_at_20_max": 0.07996117756779385, - "nauc_ndcg_at_20_std": 0.05859145892477607, - "nauc_ndcg_at_3_diff1": 0.20879984851659117, - "nauc_ndcg_at_3_max": 0.08382482722194272, - "nauc_ndcg_at_3_std": 0.029056373224856612, - "nauc_ndcg_at_5_diff1": 0.1889200636596775, - "nauc_ndcg_at_5_max": 0.07180390421297643, - "nauc_ndcg_at_5_std": 0.039851440751330514, - "nauc_precision_at_1000_diff1": 0.07064142096733839, - "nauc_precision_at_1000_max": 0.14933892098071938, - "nauc_precision_at_1000_std": 0.18601204257331688, - "nauc_precision_at_100_diff1": 0.06431278693555126, - "nauc_precision_at_100_max": 0.0996388427540241, - "nauc_precision_at_100_std": 0.11930789207654464, - "nauc_precision_at_10_diff1": 0.1232777385611232, - "nauc_precision_at_10_max": 0.054975009177794666, - "nauc_precision_at_10_std": 0.06686871123355793, - "nauc_precision_at_1_diff1": 0.3139494374020095, - "nauc_precision_at_1_max": 0.13028777354452473, - "nauc_precision_at_1_std": 0.04248164918312027, - "nauc_precision_at_20_diff1": 0.09857331248582687, - "nauc_precision_at_20_max": 0.07617887127857105, - "nauc_precision_at_20_std": 0.07353148089369012, - "nauc_precision_at_3_diff1": 0.16925343333548437, - "nauc_precision_at_3_max": 0.06310232394286996, - "nauc_precision_at_3_std": 0.021711082664362967, - "nauc_precision_at_5_diff1": 0.14716519740601297, - "nauc_precision_at_5_max": 0.048659092577076006, - "nauc_precision_at_5_std": 0.04051251752627308, - "nauc_recall_at_1000_diff1": 0.07064142096733762, - "nauc_recall_at_1000_max": 0.14933892098071952, - "nauc_recall_at_1000_std": 0.1860120425733161, - "nauc_recall_at_100_diff1": 0.06431278693555113, - "nauc_recall_at_100_max": 0.099638842754024, - "nauc_recall_at_100_std": 0.11930789207654446, - "nauc_recall_at_10_diff1": 0.12327773856112313, - "nauc_recall_at_10_max": 0.054975009177794736, - "nauc_recall_at_10_std": 0.066868711233558, - "nauc_recall_at_1_diff1": 0.3139494374020095, - "nauc_recall_at_1_max": 0.13028777354452473, - "nauc_recall_at_1_std": 0.04248164918312027, - "nauc_recall_at_20_diff1": 0.09857331248582667, - "nauc_recall_at_20_max": 0.07617887127857093, - "nauc_recall_at_20_std": 0.07353148089369, - "nauc_recall_at_3_diff1": 0.16925343333548418, - "nauc_recall_at_3_max": 0.06310232394286991, - "nauc_recall_at_3_std": 0.021711082664363137, - "nauc_recall_at_5_diff1": 0.14716519740601317, - "nauc_recall_at_5_max": 0.04865909257707613, - "nauc_recall_at_5_std": 0.04051251752627315, - "ndcg_at_1": 0.03191, - "ndcg_at_10": 0.07318, - "ndcg_at_100": 0.11934, - "ndcg_at_1000": 0.16206, - "ndcg_at_20": 0.08547, - "ndcg_at_3": 0.05133, - "ndcg_at_5": 0.0607, - "precision_at_1": 0.03191, - "precision_at_10": 0.01272, - "precision_at_100": 0.00363, - "precision_at_1000": 0.00071, - "precision_at_20": 0.00881, - "precision_at_3": 0.02182, - "precision_at_5": 0.01767, - "recall_at_1": 0.03191, - "recall_at_10": 0.12722, - "recall_at_100": 0.36265, - "recall_at_1000": 0.71364, - "recall_at_20": 0.17611, - "recall_at_3": 0.06545, - "recall_at_5": 0.08836 - } - ] - }, - "task_name": "Fashion200kI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Flickr30kI2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/Flickr30kI2TRetrieval.json deleted file mode 100644 index 9759b31060..0000000000 --- a/results-mieb/voyage-multimodal-3/1/Flickr30kI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb", - "evaluation_time": 207.93018054962158, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.958, - "cv_recall_at_10": 1.0, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 0.99, - "cv_recall_at_5": 1.0, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.92171, - "map_at_1": 0.1916, - "map_at_10": 0.86479, - "map_at_100": 0.88173, - "map_at_1000": 0.88183, - "map_at_20": 0.87868, - "map_at_3": 0.53913, - "map_at_5": 0.79289, - "mrr_at_1": 0.958, - "mrr_at_10": 0.9750166666666669, - "mrr_at_100": 0.9750166666666669, - "mrr_at_1000": 0.9750166666666669, - "mrr_at_20": 0.9750166666666669, - "mrr_at_3": 0.9726666666666668, - "mrr_at_5": 0.9750166666666669, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": NaN, - "nauc_cv_recall_at_10_max": NaN, - "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": 0.4916966786714666, - "nauc_cv_recall_at_1_max": 0.6713240851896326, - "nauc_cv_recall_at_1_std": -0.7526010404161642, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": 0.5285714285714299, - "nauc_cv_recall_at_3_max": 0.6155929038281921, - "nauc_cv_recall_at_3_std": -0.9986461251167119, - "nauc_cv_recall_at_5_diff1": NaN, - "nauc_cv_recall_at_5_max": NaN, - "nauc_cv_recall_at_5_std": NaN, - "nauc_map_at_1000_diff1": -0.02045249189608823, - "nauc_map_at_1000_max": 0.48444705367590757, - "nauc_map_at_1000_std": -0.3509317395111607, - "nauc_map_at_100_diff1": -0.02058308054045463, - "nauc_map_at_100_max": 0.4846637323698576, - "nauc_map_at_100_std": -0.35009368037401234, - "nauc_map_at_10_diff1": -0.029243897097944935, - "nauc_map_at_10_max": 0.48560114666485193, - "nauc_map_at_10_std": -0.3487188193186465, - "nauc_map_at_1_diff1": 0.4916966786714697, - "nauc_map_at_1_max": 0.6713240851896303, - "nauc_map_at_1_std": -0.7526010404161716, - "nauc_map_at_20_diff1": -0.020918191278483735, - "nauc_map_at_20_max": 0.4907827196529326, - "nauc_map_at_20_std": -0.3437478022987934, - "nauc_map_at_3_diff1": 0.06404734735826337, - "nauc_map_at_3_max": 0.49496142191054154, - "nauc_map_at_3_std": -0.5206501004530918, - "nauc_map_at_5_diff1": -0.03213800540833977, - "nauc_map_at_5_max": 0.4343110150737827, - "nauc_map_at_5_std": -0.3532847395080826, - "nauc_mrr_at_1000_diff1": 0.5147667695052229, - "nauc_mrr_at_1000_max": 0.6689794441236546, - "nauc_mrr_at_1000_std": -0.7744101420866352, - "nauc_mrr_at_100_diff1": 0.5147667695052229, - "nauc_mrr_at_100_max": 0.6689794441236546, - "nauc_mrr_at_100_std": -0.7744101420866352, - "nauc_mrr_at_10_diff1": 0.5147667695052229, - "nauc_mrr_at_10_max": 0.6689794441236546, - "nauc_mrr_at_10_std": -0.7744101420866352, - "nauc_mrr_at_1_diff1": 0.4916966786714666, - "nauc_mrr_at_1_max": 0.6713240851896326, - "nauc_mrr_at_1_std": -0.7526010404161642, - "nauc_mrr_at_20_diff1": 0.5147667695052229, - "nauc_mrr_at_20_max": 0.6689794441236546, - "nauc_mrr_at_20_std": -0.7744101420866352, - "nauc_mrr_at_3_diff1": 0.5158644758716511, - "nauc_mrr_at_3_max": 0.6622799526314594, - "nauc_mrr_at_3_std": -0.7943937737696769, - "nauc_mrr_at_5_diff1": 0.5147667695052229, - "nauc_mrr_at_5_max": 0.6689794441236546, - "nauc_mrr_at_5_std": -0.7744101420866352, - "nauc_ndcg_at_1000_diff1": 0.03770546246624843, - "nauc_ndcg_at_1000_max": 0.5150471462106003, - "nauc_ndcg_at_1000_std": -0.40462586903272596, - "nauc_ndcg_at_100_diff1": 0.036148575249940015, - "nauc_ndcg_at_100_max": 0.5184447889732504, - "nauc_ndcg_at_100_std": -0.39038829419383936, - "nauc_ndcg_at_10_diff1": 0.0008624674014364987, - "nauc_ndcg_at_10_max": 0.5191981043207383, - "nauc_ndcg_at_10_std": -0.3598665352164584, - "nauc_ndcg_at_1_diff1": 0.4916966786714666, - "nauc_ndcg_at_1_max": 0.6713240851896326, - "nauc_ndcg_at_1_std": -0.7526010404161642, - "nauc_ndcg_at_20_diff1": 0.026377554690408925, - "nauc_ndcg_at_20_max": 0.5383192158414997, - "nauc_ndcg_at_20_std": -0.35091054444218184, - "nauc_ndcg_at_3_diff1": 0.09388531218398928, - "nauc_ndcg_at_3_max": 0.5082388274289871, - "nauc_ndcg_at_3_std": -0.5390436547567337, - "nauc_ndcg_at_5_diff1": -0.014690636108521834, - "nauc_ndcg_at_5_max": 0.44758188335638405, - "nauc_ndcg_at_5_std": -0.37355202546595306, - "nauc_precision_at_1000_diff1": -0.6909430438838531, - "nauc_precision_at_1000_max": -0.29878618113860494, - "nauc_precision_at_1000_std": -0.00466853408005154, - "nauc_precision_at_100_diff1": -0.11266728913794523, - "nauc_precision_at_100_max": 0.5945637514264249, - "nauc_precision_at_100_std": 0.45455960161837233, - "nauc_precision_at_10_diff1": -0.07461466511359147, - "nauc_precision_at_10_max": 0.5239693994783068, - "nauc_precision_at_10_std": -0.20969420193093674, - "nauc_precision_at_1_diff1": 0.4916966786714666, - "nauc_precision_at_1_max": 0.6713240851896326, - "nauc_precision_at_1_std": -0.7526010404161642, - "nauc_precision_at_20_diff1": -0.06213897988573373, - "nauc_precision_at_20_max": 0.6271463387614928, - "nauc_precision_at_20_std": 0.04860814382249958, - "nauc_precision_at_3_diff1": 0.04620363145654133, - "nauc_precision_at_3_max": 0.48641878979090936, - "nauc_precision_at_3_std": -0.5070883890641662, - "nauc_precision_at_5_diff1": -0.05513998536331827, - "nauc_precision_at_5_max": 0.42244786124143063, - "nauc_precision_at_5_std": -0.3237443035379046, - "nauc_recall_at_1000_diff1": -0.69094304388462, - "nauc_recall_at_1000_max": -0.2987861811394452, - "nauc_recall_at_1000_std": -0.0046685340802375035, - "nauc_recall_at_100_diff1": -0.11266728913786052, - "nauc_recall_at_100_max": 0.5945637514265448, - "nauc_recall_at_100_std": 0.45455960161845066, - "nauc_recall_at_10_diff1": -0.07461466511359147, - "nauc_recall_at_10_max": 0.5239693994783068, - "nauc_recall_at_10_std": -0.20969420193093674, - "nauc_recall_at_1_diff1": 0.4916966786714697, - "nauc_recall_at_1_max": 0.6713240851896303, - "nauc_recall_at_1_std": -0.7526010404161716, - "nauc_recall_at_20_diff1": -0.06213897988573373, - "nauc_recall_at_20_max": 0.6271463387614928, - "nauc_recall_at_20_std": 0.04860814382249958, - "nauc_recall_at_3_diff1": 0.046203631456540835, - "nauc_recall_at_3_max": 0.48641878979091213, - "nauc_recall_at_3_std": -0.5070883890641666, - "nauc_recall_at_5_diff1": -0.05513998536331827, - "nauc_recall_at_5_max": 0.42244786124143063, - "nauc_recall_at_5_std": -0.3237443035379046, - "ndcg_at_1": 0.958, - "ndcg_at_10": 0.92171, - "ndcg_at_100": 0.946, - "ndcg_at_1000": 0.94674, - "ndcg_at_20": 0.9393, - "ndcg_at_3": 0.92272, - "ndcg_at_5": 0.85616, - "precision_at_1": 0.958, - "precision_at_10": 0.4682, - "precision_at_100": 0.04982, - "precision_at_1000": 0.005, - "precision_at_20": 0.2441, - "precision_at_3": 0.91233, - "precision_at_5": 0.8186, - "recall_at_1": 0.1916, - "recall_at_10": 0.9364, - "recall_at_100": 0.9964, - "recall_at_1000": 0.9996, - "recall_at_20": 0.9764, - "recall_at_3": 0.5474, - "recall_at_5": 0.8186 - } - ] - }, - "task_name": "Flickr30kI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Flickr30kT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/Flickr30kT2IRetrieval.json deleted file mode 100644 index 8d5d6c98af..0000000000 --- a/results-mieb/voyage-multimodal-3/1/Flickr30kT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "e819702b287bfbe084e129a61f308a802b7c108e", - "evaluation_time": 211.6292426586151, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.8684, - "cv_recall_at_10": 0.9932, - "cv_recall_at_100": 0.9994, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.9968, - "cv_recall_at_3": 0.9638, - "cv_recall_at_5": 0.9824, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.93746, - "map_at_1": 0.8684, - "map_at_10": 0.91887, - "map_at_100": 0.91921, - "map_at_1000": 0.91921, - "map_at_20": 0.91913, - "map_at_3": 0.91313, - "map_at_5": 0.91739, - "mrr_at_1": 0.8684, - "mrr_at_10": 0.9188720634920634, - "mrr_at_100": 0.9192070964776662, - "mrr_at_1000": 0.9192105992294939, - "mrr_at_20": 0.9191285436579243, - "mrr_at_3": 0.9131333333333338, - "mrr_at_5": 0.9173933333333328, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.9074074074072582, - "nauc_cv_recall_at_100_max": 1.0, - "nauc_cv_recall_at_100_std": 0.043106131341511, - "nauc_cv_recall_at_10_diff1": 0.8142884604822204, - "nauc_cv_recall_at_10_max": 0.8789201955291783, - "nauc_cv_recall_at_10_std": -0.24512550118084836, - "nauc_cv_recall_at_1_diff1": 0.8696890409875206, - "nauc_cv_recall_at_1_max": 0.6075508571662983, - "nauc_cv_recall_at_1_std": -0.2787834127846835, - "nauc_cv_recall_at_20_diff1": 0.832954014939291, - "nauc_cv_recall_at_20_max": 0.9302637721755362, - "nauc_cv_recall_at_20_std": -0.30966970121385284, - "nauc_cv_recall_at_3_diff1": 0.8315897261298597, - "nauc_cv_recall_at_3_max": 0.7461736075645744, - "nauc_cv_recall_at_3_std": -0.42221603190078993, - "nauc_cv_recall_at_5_diff1": 0.8321449792038011, - "nauc_cv_recall_at_5_max": 0.8269140989729185, - "nauc_cv_recall_at_5_std": -0.4130485527544341, - "nauc_map_at_1000_diff1": 0.8642366527631479, - "nauc_map_at_1000_max": 0.6351029904336999, - "nauc_map_at_1000_std": -0.2980863362503044, - "nauc_map_at_100_diff1": 0.8642395287230616, - "nauc_map_at_100_max": 0.635119225458973, - "nauc_map_at_100_std": -0.2980619656916447, - "nauc_map_at_10_diff1": 0.8639983629217577, - "nauc_map_at_10_max": 0.6360145331039044, - "nauc_map_at_10_std": -0.2976928255578576, - "nauc_map_at_1_diff1": 0.8696890409875206, - "nauc_map_at_1_max": 0.6075508571662983, - "nauc_map_at_1_std": -0.2787834127846835, - "nauc_map_at_20_diff1": 0.8642205241547971, - "nauc_map_at_20_max": 0.6354286591080467, - "nauc_map_at_20_std": -0.2979018073376867, - "nauc_map_at_3_diff1": 0.8618419652170195, - "nauc_map_at_3_max": 0.6401380939902368, - "nauc_map_at_3_std": -0.3073303296624219, - "nauc_map_at_5_diff1": 0.8634868750280645, - "nauc_map_at_5_max": 0.6387016700954582, - "nauc_map_at_5_std": -0.3019303288726223, - "nauc_mrr_at_1000_diff1": 0.8642366527631479, - "nauc_mrr_at_1000_max": 0.6351029904336999, - "nauc_mrr_at_1000_std": -0.2980863362503044, - "nauc_mrr_at_100_diff1": 0.8642395287230616, - "nauc_mrr_at_100_max": 0.635119225458973, - "nauc_mrr_at_100_std": -0.2980619656916447, - "nauc_mrr_at_10_diff1": 0.8639983629217577, - "nauc_mrr_at_10_max": 0.6360145331039044, - "nauc_mrr_at_10_std": -0.2976928255578576, - "nauc_mrr_at_1_diff1": 0.8696890409875206, - "nauc_mrr_at_1_max": 0.6075508571662983, - "nauc_mrr_at_1_std": -0.2787834127846835, - "nauc_mrr_at_20_diff1": 0.8642205241547971, - "nauc_mrr_at_20_max": 0.6354286591080467, - "nauc_mrr_at_20_std": -0.2979018073376867, - "nauc_mrr_at_3_diff1": 0.8618419652170195, - "nauc_mrr_at_3_max": 0.6401380939902368, - "nauc_mrr_at_3_std": -0.3073303296624219, - "nauc_mrr_at_5_diff1": 0.8634868750280645, - "nauc_mrr_at_5_max": 0.6387016700954582, - "nauc_mrr_at_5_std": -0.3019303288726223, - "nauc_ndcg_at_1000_diff1": 0.8635726458401701, - "nauc_ndcg_at_1000_max": 0.6388763171254515, - "nauc_ndcg_at_1000_std": -0.30013610087668907, - "nauc_ndcg_at_100_diff1": 0.8636378174190988, - "nauc_ndcg_at_100_max": 0.6393648885077013, - "nauc_ndcg_at_100_std": -0.29961527282803385, - "nauc_ndcg_at_10_diff1": 0.8623106702131149, - "nauc_ndcg_at_10_max": 0.6445380433892065, - "nauc_ndcg_at_10_std": -0.2983077230796854, - "nauc_ndcg_at_1_diff1": 0.8696890409875206, - "nauc_ndcg_at_1_max": 0.6075508571662983, - "nauc_ndcg_at_1_std": -0.2787834127846835, - "nauc_ndcg_at_20_diff1": 0.863308876326781, - "nauc_ndcg_at_20_max": 0.64174443168976, - "nauc_ndcg_at_20_std": -0.29975245292051106, - "nauc_ndcg_at_3_diff1": 0.857902077747856, - "nauc_ndcg_at_3_max": 0.6541603888741332, - "nauc_ndcg_at_3_std": -0.3222399570836121, - "nauc_ndcg_at_5_diff1": 0.8611092724054866, - "nauc_ndcg_at_5_max": 0.6524486999880342, - "nauc_ndcg_at_5_std": -0.3105729128710701, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.9074074074072626, - "nauc_precision_at_100_max": 1.0, - "nauc_precision_at_100_std": 0.04310613134107902, - "nauc_precision_at_10_diff1": 0.814288460482237, - "nauc_precision_at_10_max": 0.8789201955291973, - "nauc_precision_at_10_std": -0.2451255011808053, - "nauc_precision_at_1_diff1": 0.8696890409875206, - "nauc_precision_at_1_max": 0.6075508571662983, - "nauc_precision_at_1_std": -0.2787834127846835, - "nauc_precision_at_20_diff1": 0.8329540149393047, - "nauc_precision_at_20_max": 0.9302637721755742, - "nauc_precision_at_20_std": -0.3096697012137737, - "nauc_precision_at_3_diff1": 0.8315897261298597, - "nauc_precision_at_3_max": 0.7461736075645716, - "nauc_precision_at_3_std": -0.42221603190079987, - "nauc_precision_at_5_diff1": 0.8321449792038005, - "nauc_precision_at_5_max": 0.8269140989729217, - "nauc_precision_at_5_std": -0.4130485527544209, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.9074074074072582, - "nauc_recall_at_100_max": 1.0, - "nauc_recall_at_100_std": 0.043106131341511, - "nauc_recall_at_10_diff1": 0.8142884604822204, - "nauc_recall_at_10_max": 0.8789201955291783, - "nauc_recall_at_10_std": -0.24512550118084836, - "nauc_recall_at_1_diff1": 0.8696890409875206, - "nauc_recall_at_1_max": 0.6075508571662983, - "nauc_recall_at_1_std": -0.2787834127846835, - "nauc_recall_at_20_diff1": 0.832954014939291, - "nauc_recall_at_20_max": 0.9302637721755362, - "nauc_recall_at_20_std": -0.30966970121385284, - "nauc_recall_at_3_diff1": 0.8315897261298597, - "nauc_recall_at_3_max": 0.7461736075645744, - "nauc_recall_at_3_std": -0.42221603190078993, - "nauc_recall_at_5_diff1": 0.8321449792038011, - "nauc_recall_at_5_max": 0.8269140989729185, - "nauc_recall_at_5_std": -0.4130485527544341, - "ndcg_at_1": 0.8684, - "ndcg_at_10": 0.93746, - "ndcg_at_100": 0.93888, - "ndcg_at_1000": 0.93896, - "ndcg_at_20": 0.93838, - "ndcg_at_3": 0.92626, - "ndcg_at_5": 0.93393, - "precision_at_1": 0.8684, - "precision_at_10": 0.09932, - "precision_at_100": 0.00999, - "precision_at_1000": 0.001, - "precision_at_20": 0.04984, - "precision_at_3": 0.32127, - "precision_at_5": 0.19648, - "recall_at_1": 0.8684, - "recall_at_10": 0.9932, - "recall_at_100": 0.9994, - "recall_at_1000": 1.0, - "recall_at_20": 0.9968, - "recall_at_3": 0.9638, - "recall_at_5": 0.9824 - } - ] - }, - "task_name": "Flickr30kT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Food101Classification.json b/results-mieb/voyage-multimodal-3/1/Food101Classification.json deleted file mode 100644 index 3012cfd20e..0000000000 --- a/results-mieb/voyage-multimodal-3/1/Food101Classification.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", - "evaluation_time": 7259.677006721497, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "validation": [ - { - "accuracy": 0.8738376237623763, - "f1": 0.8720096472259975, - "f1_weighted": 0.8720096472259975, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8738376237623763, - "scores_per_experiment": [ - { - "accuracy": 0.8794851485148515, - "f1": 0.8778787574787363, - "f1_weighted": 0.8778787574787366 - }, - { - "accuracy": 0.8722772277227723, - "f1": 0.8703819262029309, - "f1_weighted": 0.8703819262029308 - }, - { - "accuracy": 0.8743762376237624, - "f1": 0.8726621943156567, - "f1_weighted": 0.8726621943156568 - }, - { - "accuracy": 0.8725148514851485, - "f1": 0.87059138861965, - "f1_weighted": 0.8705913886196499 - }, - { - "accuracy": 0.8705346534653465, - "f1": 0.8685339695130132, - "f1_weighted": 0.8685339695130132 - } - ] - } - ] - }, - "task_name": "Food101Classification" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Food101ZeroShot.json b/results-mieb/voyage-multimodal-3/1/Food101ZeroShot.json deleted file mode 100644 index d17bf8f3d0..0000000000 --- a/results-mieb/voyage-multimodal-3/1/Food101ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", - "evaluation_time": 5834.1527643203735, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "validation": [ - { - "accuracy": 0.7442376237623762, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7442376237623762 - } - ] - }, - "task_name": "Food101ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/GTSRB.json b/results-mieb/voyage-multimodal-3/1/GTSRB.json deleted file mode 100644 index acdcbd69f5..0000000000 --- a/results-mieb/voyage-multimodal-3/1/GTSRB.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", - "evaluation_time": 652.1987547874451, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.755249406175772, - "f1": 0.6720088924349847, - "f1_weighted": 0.7683155357055661, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.755249406175772, - "scores_per_experiment": [ - { - "accuracy": 0.7548693586698337, - "f1": 0.6718940774766187, - "f1_weighted": 0.7714544889259504 - }, - { - "accuracy": 0.7595407759303247, - "f1": 0.6797291490699253, - "f1_weighted": 0.7732718458370778 - }, - { - "accuracy": 0.7379255740300871, - "f1": 0.6516528744630659, - "f1_weighted": 0.7463279498290893 - }, - { - "accuracy": 0.7685669041963579, - "f1": 0.6830864810804418, - "f1_weighted": 0.7791283842787832 - }, - { - "accuracy": 0.7553444180522565, - "f1": 0.6736818800848718, - "f1_weighted": 0.7713950096569298 - } - ] - } - ] - }, - "task_name": "GTSRB" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/GTSRBZeroShot.json b/results-mieb/voyage-multimodal-3/1/GTSRBZeroShot.json deleted file mode 100644 index 6bbaeadc54..0000000000 --- a/results-mieb/voyage-multimodal-3/1/GTSRBZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", - "evaluation_time": 535.8853387832642, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5511480601741885, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5511480601741885 - } - ] - }, - "task_name": "GTSRBZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/HatefulMemesI2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/HatefulMemesI2TRetrieval.json deleted file mode 100644 index 62c7d29be8..0000000000 --- a/results-mieb/voyage-multimodal-3/1/HatefulMemesI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9a9a6c3ef0765622a6de0af6ebb68f323ad73ba", - "evaluation_time": 650.209198474884, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.351, - "cv_recall_at_10": 0.748, - "cv_recall_at_100": 0.928, - "cv_recall_at_1000": 0.992, - "cv_recall_at_20": 0.817, - "cv_recall_at_3": 0.602, - "cv_recall_at_5": 0.674, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.55506, - "map_at_1": 0.355, - "map_at_10": 0.49261, - "map_at_100": 0.50018, - "map_at_1000": 0.5005, - "map_at_20": 0.49749, - "map_at_3": 0.46617, - "map_at_5": 0.48247, - "mrr_at_1": 0.351, - "mrr_at_10": 0.49080277777777725, - "mrr_at_100": 0.49846476493880365, - "mrr_at_1000": 0.49878931223846307, - "mrr_at_20": 0.49577409491952756, - "mrr_at_3": 0.4644999999999996, - "mrr_at_5": 0.4807999999999994, - "nauc_cv_recall_at_1000_diff1": 0.07621381886086587, - "nauc_cv_recall_at_1000_max": 0.8082983193277192, - "nauc_cv_recall_at_1000_std": 0.17851307189543436, - "nauc_cv_recall_at_100_diff1": 0.09988717709305933, - "nauc_cv_recall_at_100_max": 0.6147032887228979, - "nauc_cv_recall_at_100_std": 0.24981196182176554, - "nauc_cv_recall_at_10_diff1": 0.06197436174726118, - "nauc_cv_recall_at_10_max": 0.5141043664889241, - "nauc_cv_recall_at_10_std": 0.041529061400370854, - "nauc_cv_recall_at_1_diff1": 0.3850282742742979, - "nauc_cv_recall_at_1_max": 0.3175284686558156, - "nauc_cv_recall_at_1_std": -0.10299980100603963, - "nauc_cv_recall_at_20_diff1": 0.07301231742174892, - "nauc_cv_recall_at_20_max": 0.4605582162247186, - "nauc_cv_recall_at_20_std": 0.01682458928368046, - "nauc_cv_recall_at_3_diff1": 0.08599160592529945, - "nauc_cv_recall_at_3_max": 0.45713749542019533, - "nauc_cv_recall_at_3_std": -0.007640454793230996, - "nauc_cv_recall_at_5_diff1": 0.07195517798082428, - "nauc_cv_recall_at_5_max": 0.49091067338837635, - "nauc_cv_recall_at_5_std": 0.005269966311808489, - "nauc_map_at_1000_diff1": 0.2533107101417881, - "nauc_map_at_1000_max": 0.38972900961589796, - "nauc_map_at_1000_std": -0.06353846682756097, - "nauc_map_at_100_diff1": 0.2531591611847431, - "nauc_map_at_100_max": 0.38989054771157217, - "nauc_map_at_100_std": -0.06323456068673915, - "nauc_map_at_10_diff1": 0.24912377115374532, - "nauc_map_at_10_max": 0.3926449653117099, - "nauc_map_at_10_std": -0.062216164680505115, - "nauc_map_at_1_diff1": 0.3730153823894987, - "nauc_map_at_1_max": 0.3316677298644587, - "nauc_map_at_1_std": -0.10641009621391966, - "nauc_map_at_20_diff1": 0.25148518715541224, - "nauc_map_at_20_max": 0.38937799165745557, - "nauc_map_at_20_std": -0.06397118323619402, - "nauc_map_at_3_diff1": 0.24482055059226032, - "nauc_map_at_3_max": 0.38720954729341184, - "nauc_map_at_3_std": -0.06613319316079966, - "nauc_map_at_5_diff1": 0.24685220069553152, - "nauc_map_at_5_max": 0.3920841183102921, - "nauc_map_at_5_std": -0.06464680615951957, - "nauc_mrr_at_1000_diff1": 0.2585096359144679, - "nauc_mrr_at_1000_max": 0.38199432135718614, - "nauc_mrr_at_1000_std": -0.05978602963891485, - "nauc_mrr_at_100_diff1": 0.258352622577515, - "nauc_mrr_at_100_max": 0.38216389801366024, - "nauc_mrr_at_100_std": -0.059485839290383584, - "nauc_mrr_at_10_diff1": 0.25416421261734345, - "nauc_mrr_at_10_max": 0.38504436738237263, - "nauc_mrr_at_10_std": -0.05867002311891825, - "nauc_mrr_at_1_diff1": 0.3850282742742979, - "nauc_mrr_at_1_max": 0.3175284686558156, - "nauc_mrr_at_1_std": -0.10299980100603963, - "nauc_mrr_at_20_diff1": 0.25663329766992704, - "nauc_mrr_at_20_max": 0.38169566832125074, - "nauc_mrr_at_20_std": -0.060261518241809524, - "nauc_mrr_at_3_diff1": 0.2500928845703025, - "nauc_mrr_at_3_max": 0.37966197148104364, - "nauc_mrr_at_3_std": -0.06330230125980266, - "nauc_mrr_at_5_diff1": 0.25167385837686956, - "nauc_mrr_at_5_max": 0.384322454672087, - "nauc_mrr_at_5_std": -0.06189888851605895, - "nauc_ndcg_at_1000_diff1": 0.23587589947343313, - "nauc_ndcg_at_1000_max": 0.4022544236196415, - "nauc_ndcg_at_1000_std": -0.05176148282936083, - "nauc_ndcg_at_100_diff1": 0.2318767422114613, - "nauc_ndcg_at_100_max": 0.40775700069666154, - "nauc_ndcg_at_100_std": -0.04240628044464809, - "nauc_ndcg_at_10_diff1": 0.21177187255605082, - "nauc_ndcg_at_10_max": 0.41614606723035347, - "nauc_ndcg_at_10_std": -0.04210221725280141, - "nauc_ndcg_at_1_diff1": 0.3730153823894987, - "nauc_ndcg_at_1_max": 0.3316677298644587, - "nauc_ndcg_at_1_std": -0.10641009621391966, - "nauc_ndcg_at_20_diff1": 0.22009221253620218, - "nauc_ndcg_at_20_max": 0.4035476208118783, - "nauc_ndcg_at_20_std": -0.04956172046512765, - "nauc_ndcg_at_3_diff1": 0.2054629176787454, - "nauc_ndcg_at_3_max": 0.4044812807732281, - "nauc_ndcg_at_3_std": -0.05122687481333138, - "nauc_ndcg_at_5_diff1": 0.20751438759573293, - "nauc_ndcg_at_5_max": 0.41420346702045324, - "nauc_ndcg_at_5_std": -0.04793996715762123, - "nauc_precision_at_1000_diff1": 0.07621381886085157, - "nauc_precision_at_1000_max": 0.8082983193277065, - "nauc_precision_at_1000_std": 0.17851307189538645, - "nauc_precision_at_100_diff1": 0.09988717709305533, - "nauc_precision_at_100_max": 0.6147032887228965, - "nauc_precision_at_100_std": 0.2498119618217622, - "nauc_precision_at_10_diff1": 0.06162567259966079, - "nauc_precision_at_10_max": 0.5152149919569698, - "nauc_precision_at_10_std": 0.04418137397062698, - "nauc_precision_at_1_diff1": 0.3730153823894987, - "nauc_precision_at_1_max": 0.3316677298644587, - "nauc_precision_at_1_std": -0.10641009621391966, - "nauc_precision_at_20_diff1": 0.07301231742174608, - "nauc_precision_at_20_max": 0.46055821622471715, - "nauc_precision_at_20_std": 0.016824589283678037, - "nauc_precision_at_3_diff1": 0.08407925817588253, - "nauc_precision_at_3_max": 0.45783005126402626, - "nauc_precision_at_3_std": -0.0043206548450612, - "nauc_precision_at_5_diff1": 0.07281530201617145, - "nauc_precision_at_5_max": 0.49198582843256056, - "nauc_precision_at_5_std": 0.011004126547455258, - "nauc_recall_at_1000_diff1": 0.07621381886086587, - "nauc_recall_at_1000_max": 0.8082983193277192, - "nauc_recall_at_1000_std": 0.17851307189543436, - "nauc_recall_at_100_diff1": 0.09988717709305933, - "nauc_recall_at_100_max": 0.6147032887228979, - "nauc_recall_at_100_std": 0.24981196182176554, - "nauc_recall_at_10_diff1": 0.061625672599661785, - "nauc_recall_at_10_max": 0.5152149919569704, - "nauc_recall_at_10_std": 0.044181373970627394, - "nauc_recall_at_1_diff1": 0.3730153823894987, - "nauc_recall_at_1_max": 0.3316677298644587, - "nauc_recall_at_1_std": -0.10641009621391966, - "nauc_recall_at_20_diff1": 0.07301231742174892, - "nauc_recall_at_20_max": 0.4605582162247186, - "nauc_recall_at_20_std": 0.01682458928368046, - "nauc_recall_at_3_diff1": 0.08407925817588281, - "nauc_recall_at_3_max": 0.4578300512640267, - "nauc_recall_at_3_std": -0.004320654845060782, - "nauc_recall_at_5_diff1": 0.07281530201617166, - "nauc_recall_at_5_max": 0.49198582843256045, - "nauc_recall_at_5_std": 0.011004126547455015, - "ndcg_at_1": 0.355, - "ndcg_at_10": 0.55506, - "ndcg_at_100": 0.59275, - "ndcg_at_1000": 0.60097, - "ndcg_at_20": 0.57245, - "ndcg_at_3": 0.50115, - "ndcg_at_5": 0.53067, - "precision_at_1": 0.355, - "precision_at_10": 0.0749, - "precision_at_100": 0.00928, - "precision_at_1000": 0.00099, - "precision_at_20": 0.04085, - "precision_at_3": 0.20067, - "precision_at_5": 0.1348, - "recall_at_1": 0.355, - "recall_at_10": 0.749, - "recall_at_100": 0.928, - "recall_at_1000": 0.992, - "recall_at_20": 0.817, - "recall_at_3": 0.602, - "recall_at_5": 0.674 - } - ] - }, - "task_name": "HatefulMemesI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/ImageNet10Clustering.json b/results-mieb/voyage-multimodal-3/1/ImageNet10Clustering.json deleted file mode 100644 index 750953aea8..0000000000 --- a/results-mieb/voyage-multimodal-3/1/ImageNet10Clustering.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "dataset_revision": "88f8a6d47c257895094c5ad81e67ba751771fc99", - "evaluation_time": 914.053968667984, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "ari": 0.9808613040028806, - "cluster_accuracy": 0.9913076923076923, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9758070118084365, - "nmi": 0.9758070118084365, - "v_measure": 0.9758070118084365 - } - ] - }, - "task_name": "ImageNet10Clustering" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/ImageNetDog15Clustering.json b/results-mieb/voyage-multimodal-3/1/ImageNetDog15Clustering.json deleted file mode 100644 index 96f915915f..0000000000 --- a/results-mieb/voyage-multimodal-3/1/ImageNetDog15Clustering.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "dataset_revision": "bfb6ad3b2109d26c9daddf14f98d315daa35ee72", - "evaluation_time": 78.63793706893921, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "ari": 0.8119121646758497, - "cluster_accuracy": 0.820631970260223, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8381839891894807, - "nmi": 0.8381839891894807, - "v_measure": 0.8381839891894807 - } - ] - }, - "task_name": "ImageNetDog15Clustering" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/InfoSeekIT2ITRetrieval.json b/results-mieb/voyage-multimodal-3/1/InfoSeekIT2ITRetrieval.json deleted file mode 100644 index 66f630de1e..0000000000 --- a/results-mieb/voyage-multimodal-3/1/InfoSeekIT2ITRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "78ee7f7708aac75d3afac5dcab1c9e03cb62664c", - "evaluation_time": 65952.82875466347, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.28403, - "cv_recall_at_10": 0.65719, - "cv_recall_at_100": 0.89257, - "cv_recall_at_1000": 0.97874, - "cv_recall_at_20": 0.75001, - "cv_recall_at_3": 0.47144, - "cv_recall_at_5": 0.55545, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.33953, - "map_at_1": 0.08145, - "map_at_10": 0.21206, - "map_at_100": 0.28257, - "map_at_1000": 0.28611, - "map_at_20": 0.24427, - "map_at_3": 0.14147, - "map_at_5": 0.17264, - "mrr_at_1": 0.28403342238390267, - "mrr_at_10": 0.3996135051910026, - "mrr_at_100": 0.41000322037538744, - "mrr_at_1000": 0.41041189646994997, - "mrr_at_20": 0.4061427559913534, - "mrr_at_3": 0.36668940298224206, - "mrr_at_5": 0.3859271680024156, - "nauc_cv_recall_at_1000_diff1": 0.006255086097224733, - "nauc_cv_recall_at_1000_max": 0.7865445077141644, - "nauc_cv_recall_at_1000_std": 0.8924394061114759, - "nauc_cv_recall_at_100_diff1": -0.013434457267599696, - "nauc_cv_recall_at_100_max": 0.593648393742511, - "nauc_cv_recall_at_100_std": 0.6319873871338207, - "nauc_cv_recall_at_10_diff1": 0.022263238930525824, - "nauc_cv_recall_at_10_max": 0.39839916458848434, - "nauc_cv_recall_at_10_std": 0.2557897885542902, - "nauc_cv_recall_at_1_diff1": 0.01822031583099016, - "nauc_cv_recall_at_1_max": 0.2281715481495325, - "nauc_cv_recall_at_1_std": 0.1716855759958051, - "nauc_cv_recall_at_20_diff1": 0.010652852309121942, - "nauc_cv_recall_at_20_max": 0.4417588169147453, - "nauc_cv_recall_at_20_std": 0.3408777750831783, - "nauc_cv_recall_at_3_diff1": 0.03860765592580623, - "nauc_cv_recall_at_3_max": 0.3102534078441316, - "nauc_cv_recall_at_3_std": 0.1910136003216648, - "nauc_cv_recall_at_5_diff1": 0.025886440425177976, - "nauc_cv_recall_at_5_max": 0.3427393218302827, - "nauc_cv_recall_at_5_std": 0.203679095765542, - "nauc_map_at_1000_diff1": 0.04518541449257291, - "nauc_map_at_1000_max": 0.3083486535985005, - "nauc_map_at_1000_std": 0.19210317667174406, - "nauc_map_at_100_diff1": 0.04724029064314546, - "nauc_map_at_100_max": 0.311711329509263, - "nauc_map_at_100_std": 0.18956658646818914, - "nauc_map_at_10_diff1": 0.08946183195431161, - "nauc_map_at_10_max": 0.2622402614931031, - "nauc_map_at_10_std": 0.024433285370009744, - "nauc_map_at_1_diff1": 0.15993382767564207, - "nauc_map_at_1_max": 0.1073051172298578, - "nauc_map_at_1_std": -0.03657308891148322, - "nauc_map_at_20_diff1": 0.06913273401030226, - "nauc_map_at_20_max": 0.29230128274413936, - "nauc_map_at_20_std": 0.08624682957270653, - "nauc_map_at_3_diff1": 0.13700226942401322, - "nauc_map_at_3_max": 0.18402074614649278, - "nauc_map_at_3_std": -0.017710823650508613, - "nauc_map_at_5_diff1": 0.11491927496912738, - "nauc_map_at_5_max": 0.2200920168842469, - "nauc_map_at_5_std": -0.011576798777768498, - "nauc_mrr_at_1000_diff1": 0.025121567896100157, - "nauc_mrr_at_1000_max": 0.27882633572887006, - "nauc_mrr_at_1000_std": 0.19289373369702298, - "nauc_mrr_at_100_diff1": 0.025086948333758748, - "nauc_mrr_at_100_max": 0.2791026960702822, - "nauc_mrr_at_100_std": 0.19325678818734734, - "nauc_mrr_at_10_diff1": 0.02555860944367029, - "nauc_mrr_at_10_max": 0.2784044740210458, - "nauc_mrr_at_10_std": 0.188423836265687, - "nauc_mrr_at_1_diff1": 0.01822031583099016, - "nauc_mrr_at_1_max": 0.2281715481495325, - "nauc_mrr_at_1_std": 0.1716855759958051, - "nauc_mrr_at_20_diff1": 0.025081365238416566, - "nauc_mrr_at_20_max": 0.278957933480566, - "nauc_mrr_at_20_std": 0.191588318130598, - "nauc_mrr_at_3_diff1": 0.028847425808651184, - "nauc_mrr_at_3_max": 0.2667595353588465, - "nauc_mrr_at_3_std": 0.1806448364151367, - "nauc_mrr_at_5_diff1": 0.02610524326290375, - "nauc_mrr_at_5_max": 0.2732974796911029, - "nauc_mrr_at_5_std": 0.18325469633434754, - "nauc_ndcg_at_1000_diff1": 0.028503470874592905, - "nauc_ndcg_at_1000_max": 0.3216882468647445, - "nauc_ndcg_at_1000_std": 0.2528746110996414, - "nauc_ndcg_at_100_diff1": 0.03207688712074031, - "nauc_ndcg_at_100_max": 0.34667674488041317, - "nauc_ndcg_at_100_std": 0.27229705056106845, - "nauc_ndcg_at_10_diff1": 0.038192209224090994, - "nauc_ndcg_at_10_max": 0.30684379429067615, - "nauc_ndcg_at_10_std": 0.16647699668688587, - "nauc_ndcg_at_1_diff1": 0.01822031583099016, - "nauc_ndcg_at_1_max": 0.2281715481495325, - "nauc_ndcg_at_1_std": 0.1716855759958051, - "nauc_ndcg_at_20_diff1": 0.034938762725658365, - "nauc_ndcg_at_20_max": 0.3264420320738274, - "nauc_ndcg_at_20_std": 0.19022825198564514, - "nauc_ndcg_at_3_diff1": 0.038729342507024075, - "nauc_ndcg_at_3_max": 0.2687825534865985, - "nauc_ndcg_at_3_std": 0.17768045830564053, - "nauc_ndcg_at_5_diff1": 0.03748423929351206, - "nauc_ndcg_at_5_max": 0.28150109846155363, - "nauc_ndcg_at_5_std": 0.16672113376409234, - "nauc_precision_at_1000_diff1": -0.08799871966231278, - "nauc_precision_at_1000_max": 0.12781925314854187, - "nauc_precision_at_1000_std": 0.37277258907442795, - "nauc_precision_at_100_diff1": -0.08320462963055977, - "nauc_precision_at_100_max": 0.2051136071484949, - "nauc_precision_at_100_std": 0.45246451743688454, - "nauc_precision_at_10_diff1": -0.05976693193763614, - "nauc_precision_at_10_max": 0.2968576435115502, - "nauc_precision_at_10_std": 0.2844011395283614, - "nauc_precision_at_1_diff1": 0.01822031583099016, - "nauc_precision_at_1_max": 0.2281715481495325, - "nauc_precision_at_1_std": 0.1716855759958051, - "nauc_precision_at_20_diff1": -0.07361102895671524, - "nauc_precision_at_20_max": 0.2754586684079379, - "nauc_precision_at_20_std": 0.35559497309161087, - "nauc_precision_at_3_diff1": -0.010106575994914213, - "nauc_precision_at_3_max": 0.29340247188852653, - "nauc_precision_at_3_std": 0.22318287453672572, - "nauc_precision_at_5_diff1": -0.03836290413957423, - "nauc_precision_at_5_max": 0.3018368815002715, - "nauc_precision_at_5_std": 0.23624481906440406, - "nauc_recall_at_1000_diff1": -0.008059356781186513, - "nauc_recall_at_1000_max": 0.2304098551929707, - "nauc_recall_at_1000_std": 0.3354339325308451, - "nauc_recall_at_100_diff1": 0.01083728828629382, - "nauc_recall_at_100_max": 0.3836673140413695, - "nauc_recall_at_100_std": 0.42905365137980256, - "nauc_recall_at_10_diff1": 0.06556309837226998, - "nauc_recall_at_10_max": 0.2599183481627226, - "nauc_recall_at_10_std": 0.012531473925899825, - "nauc_recall_at_1_diff1": 0.15993382767564207, - "nauc_recall_at_1_max": 0.1073051172298578, - "nauc_recall_at_1_std": -0.03657308891148322, - "nauc_recall_at_20_diff1": 0.040179941335428404, - "nauc_recall_at_20_max": 0.3090495656432163, - "nauc_recall_at_20_std": 0.11374639642332243, - "nauc_recall_at_3_diff1": 0.1258408660747518, - "nauc_recall_at_3_max": 0.1745820829297796, - "nauc_recall_at_3_std": -0.0251053671266273, - "nauc_recall_at_5_diff1": 0.09408305297650396, - "nauc_recall_at_5_max": 0.20631551260094838, - "nauc_recall_at_5_std": -0.026805180940384123, - "ndcg_at_1": 0.28403, - "ndcg_at_10": 0.33953, - "ndcg_at_100": 0.43251, - "ndcg_at_1000": 0.45665, - "ndcg_at_20": 0.37096, - "ndcg_at_3": 0.29659, - "ndcg_at_5": 0.31142, - "precision_at_1": 0.28403, - "precision_at_10": 0.19173, - "precision_at_100": 0.05487, - "precision_at_1000": 0.00649, - "precision_at_20": 0.14789, - "precision_at_3": 0.25429, - "precision_at_5": 0.23093, - "recall_at_1": 0.08145, - "recall_at_10": 0.36199, - "recall_at_100": 0.68958, - "recall_at_1000": 0.79688, - "recall_at_20": 0.47438, - "recall_at_3": 0.18495, - "recall_at_5": 0.25407 - } - ] - }, - "task_name": "InfoSeekIT2ITRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/InfoSeekIT2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/InfoSeekIT2TRetrieval.json deleted file mode 100644 index 9aac076ec1..0000000000 --- a/results-mieb/voyage-multimodal-3/1/InfoSeekIT2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "d4f4606f7a42bbf311c2957419ef3734fe81c47f", - "evaluation_time": 6618.2249002456665, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.21046, - "cv_recall_at_10": 0.50799, - "cv_recall_at_100": 0.74565, - "cv_recall_at_1000": 0.9048, - "cv_recall_at_20": 0.58792, - "cv_recall_at_3": 0.35741, - "cv_recall_at_5": 0.4233, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.22647, - "map_at_1": 0.06088, - "map_at_10": 0.14196, - "map_at_100": 0.17037, - "map_at_1000": 0.17435, - "map_at_20": 0.15631, - "map_at_3": 0.10603, - "map_at_5": 0.12422, - "mrr_at_1": 0.21045659277576614, - "mrr_at_10": 0.3008787774287208, - "mrr_at_100": 0.31040670679487925, - "mrr_at_1000": 0.311117781969453, - "mrr_at_20": 0.30655042035402164, - "mrr_at_3": 0.274662191998589, - "mrr_at_5": 0.28961847566899324, - "nauc_cv_recall_at_1000_diff1": 0.17824996811865748, - "nauc_cv_recall_at_1000_max": 0.47987402979389815, - "nauc_cv_recall_at_1000_std": 0.6816462895154791, - "nauc_cv_recall_at_100_diff1": 0.1843143682914582, - "nauc_cv_recall_at_100_max": 0.37905438003510206, - "nauc_cv_recall_at_100_std": 0.5699429810427198, - "nauc_cv_recall_at_10_diff1": 0.15012587648149436, - "nauc_cv_recall_at_10_max": 0.32368974301707476, - "nauc_cv_recall_at_10_std": 0.37409116505337014, - "nauc_cv_recall_at_1_diff1": 0.2097167948031123, - "nauc_cv_recall_at_1_max": 0.2497542239386446, - "nauc_cv_recall_at_1_std": 0.19016398114295133, - "nauc_cv_recall_at_20_diff1": 0.1618159923376429, - "nauc_cv_recall_at_20_max": 0.35277316434660777, - "nauc_cv_recall_at_20_std": 0.4508148652376852, - "nauc_cv_recall_at_3_diff1": 0.17779863645022098, - "nauc_cv_recall_at_3_max": 0.3111112573039595, - "nauc_cv_recall_at_3_std": 0.27563634786514596, - "nauc_cv_recall_at_5_diff1": 0.16305905635229093, - "nauc_cv_recall_at_5_max": 0.3162609882047919, - "nauc_cv_recall_at_5_std": 0.317778809950664, - "nauc_map_at_1000_diff1": 0.18062629663019164, - "nauc_map_at_1000_max": 0.30887386534256817, - "nauc_map_at_1000_std": 0.27501630146891326, - "nauc_map_at_100_diff1": 0.18194687531246523, - "nauc_map_at_100_max": 0.3088618001377212, - "nauc_map_at_100_std": 0.26949909365789393, - "nauc_map_at_10_diff1": 0.19995134632470404, - "nauc_map_at_10_max": 0.2866006053364188, - "nauc_map_at_10_std": 0.19399104469893683, - "nauc_map_at_1_diff1": 0.2387998551076001, - "nauc_map_at_1_max": 0.19279908750721483, - "nauc_map_at_1_std": 0.07437905413912614, - "nauc_map_at_20_diff1": 0.19175528798673863, - "nauc_map_at_20_max": 0.3029648366966638, - "nauc_map_at_20_std": 0.23512831620110114, - "nauc_map_at_3_diff1": 0.21024928339969556, - "nauc_map_at_3_max": 0.2514882852013589, - "nauc_map_at_3_std": 0.12580882754076886, - "nauc_map_at_5_diff1": 0.20305860756214764, - "nauc_map_at_5_max": 0.2736987803243673, - "nauc_map_at_5_std": 0.15805778659737624, - "nauc_mrr_at_1000_diff1": 0.18931473336796248, - "nauc_mrr_at_1000_max": 0.2822758075273321, - "nauc_mrr_at_1000_std": 0.2530585297752362, - "nauc_mrr_at_100_diff1": 0.18933364810876388, - "nauc_mrr_at_100_max": 0.28237055263016825, - "nauc_mrr_at_100_std": 0.2533934194623867, - "nauc_mrr_at_10_diff1": 0.1876819617265076, - "nauc_mrr_at_10_max": 0.28034735163378566, - "nauc_mrr_at_10_std": 0.2477265488055595, - "nauc_mrr_at_1_diff1": 0.2097167948031123, - "nauc_mrr_at_1_max": 0.2497542239386446, - "nauc_mrr_at_1_std": 0.19016398114295133, - "nauc_mrr_at_20_diff1": 0.18886764488043434, - "nauc_mrr_at_20_max": 0.28232620395091035, - "nauc_mrr_at_20_std": 0.2524704902573781, - "nauc_mrr_at_3_diff1": 0.19305391522666082, - "nauc_mrr_at_3_max": 0.2780885313791518, - "nauc_mrr_at_3_std": 0.22948759954151343, - "nauc_mrr_at_5_diff1": 0.18917066668515167, - "nauc_mrr_at_5_max": 0.27927262304030254, - "nauc_mrr_at_5_std": 0.23992936705688953, - "nauc_ndcg_at_1000_diff1": 0.1717625966155328, - "nauc_ndcg_at_1000_max": 0.32055194498333217, - "nauc_ndcg_at_1000_std": 0.3628761510517062, - "nauc_ndcg_at_100_diff1": 0.17792896235260067, - "nauc_ndcg_at_100_max": 0.3242135904641071, - "nauc_ndcg_at_100_std": 0.35207271116750893, - "nauc_ndcg_at_10_diff1": 0.17208049806241, - "nauc_ndcg_at_10_max": 0.3142723749184195, - "nauc_ndcg_at_10_std": 0.28137422414888263, - "nauc_ndcg_at_1_diff1": 0.2097167948031123, - "nauc_ndcg_at_1_max": 0.2497542239386446, - "nauc_ndcg_at_1_std": 0.19016398114295133, - "nauc_ndcg_at_20_diff1": 0.1747475140985028, - "nauc_ndcg_at_20_max": 0.3230887123924791, - "nauc_ndcg_at_20_std": 0.30773262192430567, - "nauc_ndcg_at_3_diff1": 0.1742869872978061, - "nauc_ndcg_at_3_max": 0.29735164173310114, - "nauc_ndcg_at_3_std": 0.23763176754395007, - "nauc_ndcg_at_5_diff1": 0.16989028753503907, - "nauc_ndcg_at_5_max": 0.3073545116664261, - "nauc_ndcg_at_5_std": 0.25833572648743863, - "nauc_precision_at_1000_diff1": -0.021454119112792058, - "nauc_precision_at_1000_max": 0.11979743941369668, - "nauc_precision_at_1000_std": 0.2915837925495167, - "nauc_precision_at_100_diff1": 0.012985694748761932, - "nauc_precision_at_100_max": 0.2099562509077403, - "nauc_precision_at_100_std": 0.407278331288373, - "nauc_precision_at_10_diff1": 0.09491876075712896, - "nauc_precision_at_10_max": 0.3032449465495486, - "nauc_precision_at_10_std": 0.35930676023805985, - "nauc_precision_at_1_diff1": 0.2097167948031123, - "nauc_precision_at_1_max": 0.2497542239386446, - "nauc_precision_at_1_std": 0.19016398114295133, - "nauc_precision_at_20_diff1": 0.0642121578016976, - "nauc_precision_at_20_max": 0.2990014577834674, - "nauc_precision_at_20_std": 0.4111485581458398, - "nauc_precision_at_3_diff1": 0.14574681544852458, - "nauc_precision_at_3_max": 0.31432409778474146, - "nauc_precision_at_3_std": 0.27721588283927534, - "nauc_precision_at_5_diff1": 0.1190395919590016, - "nauc_precision_at_5_max": 0.31560001369555274, - "nauc_precision_at_5_std": 0.3122611999979494, - "nauc_recall_at_1000_diff1": 0.11212176965567214, - "nauc_recall_at_1000_max": 0.2670091990423596, - "nauc_recall_at_1000_std": 0.486655313646483, - "nauc_recall_at_100_diff1": 0.14036796429967138, - "nauc_recall_at_100_max": 0.30035112693144195, - "nauc_recall_at_100_std": 0.4466945003332598, - "nauc_recall_at_10_diff1": 0.14160363251091304, - "nauc_recall_at_10_max": 0.2703810658821937, - "nauc_recall_at_10_std": 0.24857153753984734, - "nauc_recall_at_1_diff1": 0.2387998551076001, - "nauc_recall_at_1_max": 0.19279908750721483, - "nauc_recall_at_1_std": 0.07437905413912614, - "nauc_recall_at_20_diff1": 0.13385820717289768, - "nauc_recall_at_20_max": 0.2951208204679184, - "nauc_recall_at_20_std": 0.3320829918125117, - "nauc_recall_at_3_diff1": 0.17579515035676807, - "nauc_recall_at_3_max": 0.25216176892291103, - "nauc_recall_at_3_std": 0.14486396283814276, - "nauc_recall_at_5_diff1": 0.15470956333707822, - "nauc_recall_at_5_max": 0.26649716278842084, - "nauc_recall_at_5_std": 0.19097762883436376, - "ndcg_at_1": 0.21046, - "ndcg_at_10": 0.22647, - "ndcg_at_100": 0.29013, - "ndcg_at_1000": 0.33478, - "ndcg_at_20": 0.24561, - "ndcg_at_3": 0.2057, - "ndcg_at_5": 0.21249, - "precision_at_1": 0.21046, - "precision_at_10": 0.10974, - "precision_at_100": 0.02777, - "precision_at_1000": 0.00449, - "precision_at_20": 0.0786, - "precision_at_3": 0.1741, - "precision_at_5": 0.14828, - "recall_at_1": 0.06088, - "recall_at_10": 0.24851, - "recall_at_100": 0.48191, - "recall_at_1000": 0.68299, - "recall_at_20": 0.32062, - "recall_at_3": 0.14024, - "recall_at_5": 0.18554 - } - ] - }, - "task_name": "InfoSeekIT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/MNIST.json b/results-mieb/voyage-multimodal-3/1/MNIST.json deleted file mode 100644 index 3c45e466be..0000000000 --- a/results-mieb/voyage-multimodal-3/1/MNIST.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 355.2941155433655, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.94478, - "f1": 0.9455916630851033, - "f1_weighted": 0.9451557749481427, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94478, - "scores_per_experiment": [ - { - "accuracy": 0.9443, - "f1": 0.945215965609092, - "f1_weighted": 0.9446586990733822 - }, - { - "accuracy": 0.9403, - "f1": 0.9402495134915263, - "f1_weighted": 0.9403240114833532 - }, - { - "accuracy": 0.9468, - "f1": 0.947810464896407, - "f1_weighted": 0.9472892702769644 - }, - { - "accuracy": 0.9484, - "f1": 0.9494772820351903, - "f1_weighted": 0.9489311434850805 - }, - { - "accuracy": 0.9441, - "f1": 0.9452050893933018, - "f1_weighted": 0.9445757504219333 - } - ] - } - ] - }, - "task_name": "MNIST" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/MNISTZeroShot.json b/results-mieb/voyage-multimodal-3/1/MNISTZeroShot.json deleted file mode 100644 index 4f6ea48d74..0000000000 --- a/results-mieb/voyage-multimodal-3/1/MNISTZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "77f3279092a1c1579b2250db8eafed0ad422088c", - "evaluation_time": 305.5367000102997, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.8883, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8883 - } - ] - }, - "task_name": "MNISTZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/MSCOCOI2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/MSCOCOI2TRetrieval.json deleted file mode 100644 index d42c9cc67d..0000000000 --- a/results-mieb/voyage-multimodal-3/1/MSCOCOI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cca3a3e223763e6519a4d68936bc9279034d75d2", - "evaluation_time": 774.8769903182983, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.6884, - "cv_recall_at_10": 0.9402, - "cv_recall_at_100": 0.9994, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.978, - "cv_recall_at_3": 0.8396, - "cv_recall_at_5": 0.89, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.62155, - "map_at_1": 0.13774, - "map_at_10": 0.50639, - "map_at_100": 0.55195, - "map_at_1000": 0.55375, - "map_at_20": 0.53359, - "map_at_3": 0.33166, - "map_at_5": 0.44173, - "mrr_at_1": 0.6884, - "mrr_at_10": 0.7739456349206367, - "mrr_at_100": 0.777367826450281, - "mrr_at_1000": 0.777372600703197, - "mrr_at_20": 0.7767063895117622, - "mrr_at_3": 0.7555333333333364, - "mrr_at_5": 0.7670633333333351, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": -0.38702147525650704, - "nauc_cv_recall_at_100_max": 0.8513849984437857, - "nauc_cv_recall_at_100_std": 0.6095549330843056, - "nauc_cv_recall_at_10_diff1": 0.32301727826024573, - "nauc_cv_recall_at_10_max": 0.5230288324917483, - "nauc_cv_recall_at_10_std": -0.8189186488419226, - "nauc_cv_recall_at_1_diff1": 0.39010303247213973, - "nauc_cv_recall_at_1_max": 0.40557934934779377, - "nauc_cv_recall_at_1_std": -0.5562807759674807, - "nauc_cv_recall_at_20_diff1": 0.32278669043374947, - "nauc_cv_recall_at_20_max": 0.6116246498599451, - "nauc_cv_recall_at_20_std": -0.8675367116543555, - "nauc_cv_recall_at_3_diff1": 0.3461162635215054, - "nauc_cv_recall_at_3_max": 0.4234316032826311, - "nauc_cv_recall_at_3_std": -0.7107640615089226, - "nauc_cv_recall_at_5_diff1": 0.29777940039515494, - "nauc_cv_recall_at_5_max": 0.4556661798814545, - "nauc_cv_recall_at_5_std": -0.7423692122669878, - "nauc_map_at_1000_diff1": 0.15389759883779974, - "nauc_map_at_1000_max": 0.4039851793741281, - "nauc_map_at_1000_std": -0.5002106432329312, - "nauc_map_at_100_diff1": 0.15400793732497264, - "nauc_map_at_100_max": 0.4046850658511939, - "nauc_map_at_100_std": -0.5006753334767017, - "nauc_map_at_10_diff1": 0.14417566577977628, - "nauc_map_at_10_max": 0.3943356064629755, - "nauc_map_at_10_std": -0.4933375568345805, - "nauc_map_at_1_diff1": 0.39061782604629997, - "nauc_map_at_1_max": 0.40501007654030197, - "nauc_map_at_1_std": -0.5557214169508682, - "nauc_map_at_20_diff1": 0.15204311998140219, - "nauc_map_at_20_max": 0.4010186472342661, - "nauc_map_at_20_std": -0.5030459911389683, - "nauc_map_at_3_diff1": 0.19751394364383174, - "nauc_map_at_3_max": 0.39371898829057755, - "nauc_map_at_3_std": -0.48573229813847163, - "nauc_map_at_5_diff1": 0.1446331422469198, - "nauc_map_at_5_max": 0.38494293310597494, - "nauc_map_at_5_std": -0.46770254575698766, - "nauc_mrr_at_1000_diff1": 0.3763516616711763, - "nauc_mrr_at_1000_max": 0.4150495687962729, - "nauc_mrr_at_1000_std": -0.5983744526758652, - "nauc_mrr_at_100_diff1": 0.37633456843660706, - "nauc_mrr_at_100_max": 0.4150600564683398, - "nauc_mrr_at_100_std": -0.5983453993911649, - "nauc_mrr_at_10_diff1": 0.3754618101299754, - "nauc_mrr_at_10_max": 0.41645403348086485, - "nauc_mrr_at_10_std": -0.6020064260805211, - "nauc_mrr_at_1_diff1": 0.39010303247213973, - "nauc_mrr_at_1_max": 0.40557934934779377, - "nauc_mrr_at_1_std": -0.5562807759674807, - "nauc_mrr_at_20_diff1": 0.3761781415674808, - "nauc_mrr_at_20_max": 0.41572849241686777, - "nauc_mrr_at_20_std": -0.5993859021710495, - "nauc_mrr_at_3_diff1": 0.3763720703975814, - "nauc_mrr_at_3_max": 0.41205472109725566, - "nauc_mrr_at_3_std": -0.6039347601343806, - "nauc_mrr_at_5_diff1": 0.37176138944258397, - "nauc_mrr_at_5_max": 0.4152519913506332, - "nauc_mrr_at_5_std": -0.6027741075483298, - "nauc_ndcg_at_1000_diff1": 0.20766514945642406, - "nauc_ndcg_at_1000_max": 0.41934210758284596, - "nauc_ndcg_at_1000_std": -0.5364815413635738, - "nauc_ndcg_at_100_diff1": 0.20424384491896133, - "nauc_ndcg_at_100_max": 0.4273533185423817, - "nauc_ndcg_at_100_std": -0.535102542515517, - "nauc_ndcg_at_10_diff1": 0.1791208806960087, - "nauc_ndcg_at_10_max": 0.4073289016660168, - "nauc_ndcg_at_10_std": -0.5368173592120352, - "nauc_ndcg_at_1_diff1": 0.39010303247213973, - "nauc_ndcg_at_1_max": 0.40557934934779377, - "nauc_ndcg_at_1_std": -0.5562807759674807, - "nauc_ndcg_at_20_diff1": 0.19275176413625172, - "nauc_ndcg_at_20_max": 0.41722170756908916, - "nauc_ndcg_at_20_std": -0.5492816531157821, - "nauc_ndcg_at_3_diff1": 0.225181694942575, - "nauc_ndcg_at_3_max": 0.399396123599283, - "nauc_ndcg_at_3_std": -0.5164886537494516, - "nauc_ndcg_at_5_diff1": 0.1794399278507418, - "nauc_ndcg_at_5_max": 0.39482220207783547, - "nauc_ndcg_at_5_std": -0.5061581424107108, - "nauc_precision_at_1000_diff1": -0.051082798710932215, - "nauc_precision_at_1000_max": 0.40066564260112725, - "nauc_precision_at_1000_std": 0.28745118118929563, - "nauc_precision_at_100_diff1": 0.112982571814616, - "nauc_precision_at_100_max": 0.44836567702541963, - "nauc_precision_at_100_std": -0.37237152343030927, - "nauc_precision_at_10_diff1": 0.11378576600186772, - "nauc_precision_at_10_max": 0.38720465164652307, - "nauc_precision_at_10_std": -0.5159158443916068, - "nauc_precision_at_1_diff1": 0.39010303247213973, - "nauc_precision_at_1_max": 0.40557934934779377, - "nauc_precision_at_1_std": -0.5562807759674807, - "nauc_precision_at_20_diff1": 0.12756821213828962, - "nauc_precision_at_20_max": 0.39267144771467427, - "nauc_precision_at_20_std": -0.5230524294306809, - "nauc_precision_at_3_diff1": 0.18112368731828446, - "nauc_precision_at_3_max": 0.39285170555391796, - "nauc_precision_at_3_std": -0.5018078606210227, - "nauc_precision_at_5_diff1": 0.12367663772807914, - "nauc_precision_at_5_max": 0.38035879461638944, - "nauc_precision_at_5_std": -0.4811535038733751, - "nauc_recall_at_1000_diff1": 0.01710277606974907, - "nauc_recall_at_1000_max": 0.46253786067273645, - "nauc_recall_at_1000_std": 0.29927580788410824, - "nauc_recall_at_100_diff1": 0.11703851756088672, - "nauc_recall_at_100_max": 0.45101915757917294, - "nauc_recall_at_100_std": -0.37542345400577626, - "nauc_recall_at_10_diff1": 0.11430396402305408, - "nauc_recall_at_10_max": 0.3869821152574072, - "nauc_recall_at_10_std": -0.5162572501669658, - "nauc_recall_at_1_diff1": 0.39061782604629997, - "nauc_recall_at_1_max": 0.40501007654030197, - "nauc_recall_at_1_std": -0.5557214169508682, - "nauc_recall_at_20_diff1": 0.12861124356686862, - "nauc_recall_at_20_max": 0.3930253578243864, - "nauc_recall_at_20_std": -0.5239274880060881, - "nauc_recall_at_3_diff1": 0.18169641263240677, - "nauc_recall_at_3_max": 0.3925592388545264, - "nauc_recall_at_3_std": -0.5019990921945948, - "nauc_recall_at_5_diff1": 0.1240600740311391, - "nauc_recall_at_5_max": 0.38014532371394416, - "nauc_recall_at_5_std": -0.48140440409252105, - "ndcg_at_1": 0.6884, - "ndcg_at_10": 0.62155, - "ndcg_at_100": 0.72631, - "ndcg_at_1000": 0.7415, - "ndcg_at_20": 0.66921, - "ndcg_at_3": 0.61503, - "ndcg_at_5": 0.54496, - "precision_at_1": 0.6884, - "precision_at_10": 0.31916, - "precision_at_100": 0.04645, - "precision_at_1000": 0.00498, - "precision_at_20": 0.18725, - "precision_at_3": 0.593, - "precision_at_5": 0.4996, - "recall_at_1": 0.13774, - "recall_at_10": 0.63858, - "recall_at_100": 0.92951, - "recall_at_1000": 0.99672, - "recall_at_20": 0.74932, - "recall_at_3": 0.35597, - "recall_at_5": 0.49981 - } - ] - }, - "task_name": "MSCOCOI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/MSCOCOT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/MSCOCOT2IRetrieval.json deleted file mode 100644 index 1fda5ba6ae..0000000000 --- a/results-mieb/voyage-multimodal-3/1/MSCOCOT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "cfe15bd2791dde5f8f20aebecf0b4eb3812972d6", - "evaluation_time": 798.1402442455292, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.54553, - "cv_recall_at_10": 0.8692, - "cv_recall_at_100": 0.99222, - "cv_recall_at_1000": 0.99851, - "cv_recall_at_20": 0.9287, - "cv_recall_at_3": 0.72784, - "cv_recall_at_5": 0.7952, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.70362, - "map_at_1": 0.54513, - "map_at_10": 0.65107, - "map_at_100": 0.65729, - "map_at_1000": 0.65733, - "map_at_20": 0.65538, - "map_at_3": 0.62561, - "map_at_5": 0.64106, - "mrr_at_1": 0.5455278326413802, - "mrr_at_10": 0.6515759449815842, - "mrr_at_100": 0.6576602730040336, - "mrr_at_1000": 0.6576998696355355, - "mrr_at_20": 0.655811964604866, - "mrr_at_3": 0.6261638921359333, - "mrr_at_5": 0.6415957918497558, - "nauc_cv_recall_at_1000_diff1": 0.5913883028915704, - "nauc_cv_recall_at_1000_max": 0.9645439962109348, - "nauc_cv_recall_at_1000_std": 0.9018481347746059, - "nauc_cv_recall_at_100_diff1": 0.5534699359027497, - "nauc_cv_recall_at_100_max": 0.6861774550140799, - "nauc_cv_recall_at_100_std": 0.09057132259190011, - "nauc_cv_recall_at_10_diff1": 0.5839186752981996, - "nauc_cv_recall_at_10_max": 0.3798440454573067, - "nauc_cv_recall_at_10_std": -0.6611604991204363, - "nauc_cv_recall_at_1_diff1": 0.7167795833336132, - "nauc_cv_recall_at_1_max": 0.3455331044071751, - "nauc_cv_recall_at_1_std": -0.47493901255939786, - "nauc_cv_recall_at_20_diff1": 0.5748112965125677, - "nauc_cv_recall_at_20_max": 0.41011759638021583, - "nauc_cv_recall_at_20_std": -0.6670384673426152, - "nauc_cv_recall_at_3_diff1": 0.6251011207741684, - "nauc_cv_recall_at_3_max": 0.35786814470274086, - "nauc_cv_recall_at_3_std": -0.5911605866207055, - "nauc_cv_recall_at_5_diff1": 0.5980744554890782, - "nauc_cv_recall_at_5_max": 0.35849158393841046, - "nauc_cv_recall_at_5_std": -0.627844625782769, - "nauc_map_at_1000_diff1": 0.6828832422396731, - "nauc_map_at_1000_max": 0.35112835281277105, - "nauc_map_at_1000_std": -0.5195228699055376, - "nauc_map_at_100_diff1": 0.6828661396882304, - "nauc_map_at_100_max": 0.351159422258553, - "nauc_map_at_100_std": -0.519495008275733, - "nauc_map_at_10_diff1": 0.6810098053755158, - "nauc_map_at_10_max": 0.35100061273843497, - "nauc_map_at_10_std": -0.5234568934205572, - "nauc_map_at_1_diff1": 0.7171262914350959, - "nauc_map_at_1_max": 0.3453959690825254, - "nauc_map_at_1_std": -0.4759877819944269, - "nauc_map_at_20_diff1": 0.6821066111194438, - "nauc_map_at_20_max": 0.3512342628943979, - "nauc_map_at_20_std": -0.5213520952561683, - "nauc_map_at_3_diff1": 0.6814472223356598, - "nauc_map_at_3_max": 0.35014496482511315, - "nauc_map_at_3_std": -0.5211132052373082, - "nauc_map_at_5_diff1": 0.6794310461672968, - "nauc_map_at_5_max": 0.3500457603735776, - "nauc_map_at_5_std": -0.524235589581454, - "nauc_mrr_at_1000_diff1": 0.6824923157391112, - "nauc_mrr_at_1000_max": 0.35120387598639335, - "nauc_mrr_at_1000_std": -0.5181617693570029, - "nauc_mrr_at_100_diff1": 0.6824742872964704, - "nauc_mrr_at_100_max": 0.3512335430834624, - "nauc_mrr_at_100_std": -0.5181247949793334, - "nauc_mrr_at_10_diff1": 0.6805412633139878, - "nauc_mrr_at_10_max": 0.3511525454141477, - "nauc_mrr_at_10_std": -0.5216875823342743, - "nauc_mrr_at_1_diff1": 0.7167795833336132, - "nauc_mrr_at_1_max": 0.3455331044071751, - "nauc_mrr_at_1_std": -0.47493901255939786, - "nauc_mrr_at_20_diff1": 0.6816926085638637, - "nauc_mrr_at_20_max": 0.3513213303961809, - "nauc_mrr_at_20_std": -0.5197619427467188, - "nauc_mrr_at_3_diff1": 0.6808933173811453, - "nauc_mrr_at_3_max": 0.3502752099018471, - "nauc_mrr_at_3_std": -0.5194553383921099, - "nauc_mrr_at_5_diff1": 0.6789105553784537, - "nauc_mrr_at_5_max": 0.35021487576792626, - "nauc_mrr_at_5_std": -0.5224301407248029, - "nauc_ndcg_at_1000_diff1": 0.6774926006858535, - "nauc_ndcg_at_1000_max": 0.3536992083970472, - "nauc_ndcg_at_1000_std": -0.5230396169325919, - "nauc_ndcg_at_100_diff1": 0.6769870449597041, - "nauc_ndcg_at_100_max": 0.3547231738920983, - "nauc_ndcg_at_100_std": -0.521607000189851, - "nauc_ndcg_at_10_diff1": 0.6673294220619244, - "nauc_ndcg_at_10_max": 0.3546008945512635, - "nauc_ndcg_at_10_std": -0.5429692294158105, - "nauc_ndcg_at_1_diff1": 0.7167795833336132, - "nauc_ndcg_at_1_max": 0.3455331044071751, - "nauc_ndcg_at_1_std": -0.47493901255939786, - "nauc_ndcg_at_20_diff1": 0.6715391142197418, - "nauc_ndcg_at_20_max": 0.3556597729668483, - "nauc_ndcg_at_20_std": -0.5355709965917, - "nauc_ndcg_at_3_diff1": 0.6689984648994431, - "nauc_ndcg_at_3_max": 0.35184351638444844, - "nauc_ndcg_at_3_std": -0.536740668922552, - "nauc_ndcg_at_5_diff1": 0.6641807917915493, - "nauc_ndcg_at_5_max": 0.3516502816268339, - "nauc_ndcg_at_5_std": -0.5439790194614311, - "nauc_precision_at_1000_diff1": -0.29051394786528345, - "nauc_precision_at_1000_max": 0.007168455625121431, - "nauc_precision_at_1000_std": 0.7814796737105516, - "nauc_precision_at_100_diff1": -0.01673301799858297, - "nauc_precision_at_100_max": 0.213340763947866, - "nauc_precision_at_100_std": 0.4911809823345084, - "nauc_precision_at_10_diff1": 0.5604152027291861, - "nauc_precision_at_10_max": 0.36862480817292415, - "nauc_precision_at_10_std": -0.6305299231273225, - "nauc_precision_at_1_diff1": 0.7167795833336132, - "nauc_precision_at_1_max": 0.3455331044071751, - "nauc_precision_at_1_std": -0.47493901255939786, - "nauc_precision_at_20_diff1": 0.5008270292935011, - "nauc_precision_at_20_max": 0.3736600387723433, - "nauc_precision_at_20_std": -0.5638978451960841, - "nauc_precision_at_3_diff1": 0.6231722745154151, - "nauc_precision_at_3_max": 0.3570527652098802, - "nauc_precision_at_3_std": -0.5889077632335837, - "nauc_precision_at_5_diff1": 0.591025634855629, - "nauc_precision_at_5_max": 0.35518613527368537, - "nauc_precision_at_5_std": -0.6201665718909292, - "nauc_recall_at_1000_diff1": 0.5913883028915704, - "nauc_recall_at_1000_max": 0.9645439962109348, - "nauc_recall_at_1000_std": 0.9018481347746059, - "nauc_recall_at_100_diff1": 0.5555338594243105, - "nauc_recall_at_100_max": 0.6846457428695056, - "nauc_recall_at_100_std": 0.07067487943143345, - "nauc_recall_at_10_diff1": 0.585589719395042, - "nauc_recall_at_10_max": 0.37847409422814565, - "nauc_recall_at_10_std": -0.6673654162116813, - "nauc_recall_at_1_diff1": 0.7171262914350959, - "nauc_recall_at_1_max": 0.3453959690825254, - "nauc_recall_at_1_std": -0.4759877819944269, - "nauc_recall_at_20_diff1": 0.5761021751313068, - "nauc_recall_at_20_max": 0.4087068955598216, - "nauc_recall_at_20_std": -0.6760273988990346, - "nauc_recall_at_3_diff1": 0.6259496325614421, - "nauc_recall_at_3_max": 0.3575239405717769, - "nauc_recall_at_3_std": -0.5937210978019355, - "nauc_recall_at_5_diff1": 0.5993470174348666, - "nauc_recall_at_5_max": 0.3578894732514209, - "nauc_recall_at_5_std": -0.6316698946209255, - "ndcg_at_1": 0.54553, - "ndcg_at_10": 0.70362, - "ndcg_at_100": 0.7314, - "ndcg_at_1000": 0.73227, - "ndcg_at_20": 0.71892, - "ndcg_at_3": 0.65181, - "ndcg_at_5": 0.67958, - "precision_at_1": 0.54553, - "precision_at_10": 0.0871, - "precision_at_100": 0.00999, - "precision_at_1000": 0.00101, - "precision_at_20": 0.0466, - "precision_at_3": 0.24271, - "precision_at_5": 0.1592, - "recall_at_1": 0.54513, - "recall_at_10": 0.86799, - "recall_at_100": 0.99213, - "recall_at_1000": 0.99851, - "recall_at_20": 0.9277, - "recall_at_3": 0.72695, - "recall_at_5": 0.79419 - } - ] - }, - "task_name": "MSCOCOT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/NIGHTSI2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/NIGHTSI2IRetrieval.json deleted file mode 100644 index 7dfde434cd..0000000000 --- a/results-mieb/voyage-multimodal-3/1/NIGHTSI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "c9583e052be7ad52d870c62a207a2e887ba9b8aa", - "evaluation_time": 4715.950295448303, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.08255, - "cv_recall_at_10": 0.48113, - "cv_recall_at_100": 0.94387, - "cv_recall_at_1000": 0.99245, - "cv_recall_at_20": 0.70236, - "cv_recall_at_3": 0.20047, - "cv_recall_at_5": 0.29575, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.24863, - "map_at_1": 0.08255, - "map_at_10": 0.17858, - "map_at_100": 0.20121, - "map_at_1000": 0.20146, - "map_at_20": 0.19397, - "map_at_3": 0.13255, - "map_at_5": 0.1542, - "mrr_at_1": 0.08254716981132075, - "mrr_at_10": 0.1785770440251568, - "mrr_at_100": 0.201205823403381, - "mrr_at_1000": 0.20145930114431737, - "mrr_at_20": 0.19396875697904437, - "mrr_at_3": 0.1325471698113209, - "mrr_at_5": 0.15419811320754678, - "nauc_cv_recall_at_1000_diff1": 0.038077731092443015, - "nauc_cv_recall_at_1000_max": 0.8064309056956054, - "nauc_cv_recall_at_1000_std": 0.8878092903828085, - "nauc_cv_recall_at_100_diff1": -0.06317821246145294, - "nauc_cv_recall_at_100_max": 0.39503644595092996, - "nauc_cv_recall_at_100_std": 0.3795557438661738, - "nauc_cv_recall_at_10_diff1": 0.022939846901184988, - "nauc_cv_recall_at_10_max": 0.07836573883290258, - "nauc_cv_recall_at_10_std": -0.022496534672011292, - "nauc_cv_recall_at_1_diff1": 0.056265859174364684, - "nauc_cv_recall_at_1_max": 0.0379362949902258, - "nauc_cv_recall_at_1_std": -0.05939974701981684, - "nauc_cv_recall_at_20_diff1": -0.013337017151137259, - "nauc_cv_recall_at_20_max": 0.1534706439353441, - "nauc_cv_recall_at_20_std": -0.03275147830296917, - "nauc_cv_recall_at_3_diff1": 0.0775182887163471, - "nauc_cv_recall_at_3_max": 0.05816325366059642, - "nauc_cv_recall_at_3_std": -0.03549171253904066, - "nauc_cv_recall_at_5_diff1": 0.07408983351399633, - "nauc_cv_recall_at_5_max": 0.0633687792374156, - "nauc_cv_recall_at_5_std": -0.04857622730560427, - "nauc_map_at_1000_diff1": 0.05657786707298797, - "nauc_map_at_1000_max": 0.06293274475496655, - "nauc_map_at_1000_std": -0.04234689425652198, - "nauc_map_at_100_diff1": 0.056474202269853196, - "nauc_map_at_100_max": 0.06318118028967631, - "nauc_map_at_100_std": -0.042139181358562774, - "nauc_map_at_10_diff1": 0.057573823422659154, - "nauc_map_at_10_max": 0.05735556955687778, - "nauc_map_at_10_std": -0.04449564938916585, - "nauc_map_at_1_diff1": 0.056265859174364684, - "nauc_map_at_1_max": 0.0379362949902258, - "nauc_map_at_1_std": -0.05939974701981684, - "nauc_map_at_20_diff1": 0.055360882192033056, - "nauc_map_at_20_max": 0.06295645186107313, - "nauc_map_at_20_std": -0.046234789714899104, - "nauc_map_at_3_diff1": 0.06704928481972888, - "nauc_map_at_3_max": 0.05150166325545375, - "nauc_map_at_3_std": -0.04372918706365568, - "nauc_map_at_5_diff1": 0.0674125668851068, - "nauc_map_at_5_max": 0.05419732008304022, - "nauc_map_at_5_std": -0.0486370059907162, - "nauc_mrr_at_1000_diff1": 0.05657786707298797, - "nauc_mrr_at_1000_max": 0.06293274475496655, - "nauc_mrr_at_1000_std": -0.04234689425652198, - "nauc_mrr_at_100_diff1": 0.056474202269853196, - "nauc_mrr_at_100_max": 0.06318118028967631, - "nauc_mrr_at_100_std": -0.042139181358562774, - "nauc_mrr_at_10_diff1": 0.057573823422659154, - "nauc_mrr_at_10_max": 0.05735556955687778, - "nauc_mrr_at_10_std": -0.04449564938916585, - "nauc_mrr_at_1_diff1": 0.056265859174364684, - "nauc_mrr_at_1_max": 0.0379362949902258, - "nauc_mrr_at_1_std": -0.05939974701981684, - "nauc_mrr_at_20_diff1": 0.055360882192033056, - "nauc_mrr_at_20_max": 0.06295645186107313, - "nauc_mrr_at_20_std": -0.046234789714899104, - "nauc_mrr_at_3_diff1": 0.06704928481972888, - "nauc_mrr_at_3_max": 0.05150166325545375, - "nauc_mrr_at_3_std": -0.04372918706365568, - "nauc_mrr_at_5_diff1": 0.0674125668851068, - "nauc_mrr_at_5_max": 0.05419732008304022, - "nauc_mrr_at_5_std": -0.0486370059907162, - "nauc_ndcg_at_1000_diff1": 0.05077518917317683, - "nauc_ndcg_at_1000_max": 0.07723869390595439, - "nauc_ndcg_at_1000_std": -0.028531105281549857, - "nauc_ndcg_at_100_diff1": 0.046878435984643675, - "nauc_ndcg_at_100_max": 0.08518688575096077, - "nauc_ndcg_at_100_std": -0.018849997307570036, - "nauc_ndcg_at_10_diff1": 0.04776145512022865, - "nauc_ndcg_at_10_max": 0.06415267011217751, - "nauc_ndcg_at_10_std": -0.03795328556951406, - "nauc_ndcg_at_1_diff1": 0.056265859174364684, - "nauc_ndcg_at_1_max": 0.0379362949902258, - "nauc_ndcg_at_1_std": -0.05939974701981684, - "nauc_ndcg_at_20_diff1": 0.039820501697777864, - "nauc_ndcg_at_20_max": 0.08391303154450644, - "nauc_ndcg_at_20_std": -0.042736635918916216, - "nauc_ndcg_at_3_diff1": 0.07038185031455026, - "nauc_ndcg_at_3_max": 0.05372259050957707, - "nauc_ndcg_at_3_std": -0.040999397625074596, - "nauc_ndcg_at_5_diff1": 0.06984543170279325, - "nauc_ndcg_at_5_max": 0.0574112976802138, - "nauc_ndcg_at_5_std": -0.048435863215603926, - "nauc_precision_at_1000_diff1": 0.03807773109245917, - "nauc_precision_at_1000_max": 0.806430905695622, - "nauc_precision_at_1000_std": 0.8878092903828303, - "nauc_precision_at_100_diff1": -0.06317821246145515, - "nauc_precision_at_100_max": 0.3950364459509304, - "nauc_precision_at_100_std": 0.3795557438661729, - "nauc_precision_at_10_diff1": 0.022939846901185168, - "nauc_precision_at_10_max": 0.07836573883290233, - "nauc_precision_at_10_std": -0.02249653467201138, - "nauc_precision_at_1_diff1": 0.056265859174364684, - "nauc_precision_at_1_max": 0.0379362949902258, - "nauc_precision_at_1_std": -0.05939974701981684, - "nauc_precision_at_20_diff1": -0.013337017151137004, - "nauc_precision_at_20_max": 0.15347064393534382, - "nauc_precision_at_20_std": -0.03275147830296866, - "nauc_precision_at_3_diff1": 0.07751828871634699, - "nauc_precision_at_3_max": 0.05816325366059655, - "nauc_precision_at_3_std": -0.03549171253904067, - "nauc_precision_at_5_diff1": 0.07408983351399642, - "nauc_precision_at_5_max": 0.06336877923741566, - "nauc_precision_at_5_std": -0.048576227305604174, - "nauc_recall_at_1000_diff1": 0.038077731092443015, - "nauc_recall_at_1000_max": 0.8064309056956054, - "nauc_recall_at_1000_std": 0.8878092903828085, - "nauc_recall_at_100_diff1": -0.06317821246145294, - "nauc_recall_at_100_max": 0.39503644595092996, - "nauc_recall_at_100_std": 0.3795557438661738, - "nauc_recall_at_10_diff1": 0.022939846901184988, - "nauc_recall_at_10_max": 0.07836573883290258, - "nauc_recall_at_10_std": -0.022496534672011292, - "nauc_recall_at_1_diff1": 0.056265859174364684, - "nauc_recall_at_1_max": 0.0379362949902258, - "nauc_recall_at_1_std": -0.05939974701981684, - "nauc_recall_at_20_diff1": -0.013337017151137259, - "nauc_recall_at_20_max": 0.1534706439353441, - "nauc_recall_at_20_std": -0.03275147830296917, - "nauc_recall_at_3_diff1": 0.0775182887163471, - "nauc_recall_at_3_max": 0.05816325366059642, - "nauc_recall_at_3_std": -0.03549171253904066, - "nauc_recall_at_5_diff1": 0.07408983351399633, - "nauc_recall_at_5_max": 0.0633687792374156, - "nauc_recall_at_5_std": -0.04857622730560427, - "ndcg_at_1": 0.08255, - "ndcg_at_10": 0.24863, - "ndcg_at_100": 0.35147, - "ndcg_at_1000": 0.35776, - "ndcg_at_20": 0.30462, - "ndcg_at_3": 0.14991, - "ndcg_at_5": 0.18904, - "precision_at_1": 0.08255, - "precision_at_10": 0.04811, - "precision_at_100": 0.00944, - "precision_at_1000": 0.00099, - "precision_at_20": 0.03512, - "precision_at_3": 0.06682, - "precision_at_5": 0.05915, - "recall_at_1": 0.08255, - "recall_at_10": 0.48113, - "recall_at_100": 0.94387, - "recall_at_1000": 0.99245, - "recall_at_20": 0.70236, - "recall_at_3": 0.20047, - "recall_at_5": 0.29575 - } - ] - }, - "task_name": "NIGHTSI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/OVENIT2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/OVENIT2TRetrieval.json deleted file mode 100644 index cea3ced09d..0000000000 --- a/results-mieb/voyage-multimodal-3/1/OVENIT2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "2192074af29422bc1dc41cf07936f198b8c69bd0", - "evaluation_time": 12739.220294475555, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.21298, - "cv_recall_at_10": 0.44988, - "cv_recall_at_100": 0.67133, - "cv_recall_at_1000": 0.84835, - "cv_recall_at_20": 0.51906, - "cv_recall_at_3": 0.32843, - "cv_recall_at_5": 0.37901, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.16397, - "map_at_1": 0.08389, - "map_at_10": 0.11264, - "map_at_100": 0.11625, - "map_at_1000": 0.11655, - "map_at_20": 0.1147, - "map_at_3": 0.10413, - "map_at_5": 0.10875, - "mrr_at_1": 0.21298296136309094, - "mrr_at_10": 0.28446166370183723, - "mrr_at_100": 0.29304291594034704, - "mrr_at_1000": 0.29376949339180636, - "mrr_at_20": 0.2892705097899374, - "mrr_at_3": 0.2634555902194559, - "mrr_at_5": 0.2750026664533632, - "nauc_cv_recall_at_1000_diff1": 0.23906549629288248, - "nauc_cv_recall_at_1000_max": 0.2807966191767593, - "nauc_cv_recall_at_1000_std": 0.4505710122579727, - "nauc_cv_recall_at_100_diff1": 0.2906996072526638, - "nauc_cv_recall_at_100_max": 0.24695181642790698, - "nauc_cv_recall_at_100_std": 0.3413980973312369, - "nauc_cv_recall_at_10_diff1": 0.3323472647610039, - "nauc_cv_recall_at_10_max": 0.2507787156281116, - "nauc_cv_recall_at_10_std": 0.18034362446270058, - "nauc_cv_recall_at_1_diff1": 0.46901333074242085, - "nauc_cv_recall_at_1_max": 0.26293584739784476, - "nauc_cv_recall_at_1_std": 0.09393200644418588, - "nauc_cv_recall_at_20_diff1": 0.3171006136473147, - "nauc_cv_recall_at_20_max": 0.2492534303712411, - "nauc_cv_recall_at_20_std": 0.22178558628756295, - "nauc_cv_recall_at_3_diff1": 0.3719133094010799, - "nauc_cv_recall_at_3_max": 0.2579938533946551, - "nauc_cv_recall_at_3_std": 0.1326339382457508, - "nauc_cv_recall_at_5_diff1": 0.35066227486275997, - "nauc_cv_recall_at_5_max": 0.2540836536889673, - "nauc_cv_recall_at_5_std": 0.14999446316316967, - "nauc_map_at_1000_diff1": 0.348010615372264, - "nauc_map_at_1000_max": 0.15014507094893012, - "nauc_map_at_1000_std": 0.1717617324425307, - "nauc_map_at_100_diff1": 0.34798249017698873, - "nauc_map_at_100_max": 0.15043936623182197, - "nauc_map_at_100_std": 0.1715964239960731, - "nauc_map_at_10_diff1": 0.34956012147948906, - "nauc_map_at_10_max": 0.15358517354268658, - "nauc_map_at_10_std": 0.166532234878434, - "nauc_map_at_1_diff1": 0.4229161887175077, - "nauc_map_at_1_max": 0.18563915762564656, - "nauc_map_at_1_std": 0.13930048980022755, - "nauc_map_at_20_diff1": 0.34840795188919627, - "nauc_map_at_20_max": 0.15201542141647068, - "nauc_map_at_20_std": 0.16963394984544367, - "nauc_map_at_3_diff1": 0.36108195136505084, - "nauc_map_at_3_max": 0.16270134475236106, - "nauc_map_at_3_std": 0.157948835875016, - "nauc_map_at_5_diff1": 0.3537755602811475, - "nauc_map_at_5_max": 0.15750835781201705, - "nauc_map_at_5_std": 0.162398077711519, - "nauc_mrr_at_1000_diff1": 0.41148171312784426, - "nauc_mrr_at_1000_max": 0.2589681398234147, - "nauc_mrr_at_1000_std": 0.1250147526328914, - "nauc_mrr_at_100_diff1": 0.4114161400120679, - "nauc_mrr_at_100_max": 0.25888484323358824, - "nauc_mrr_at_100_std": 0.12503103420716968, - "nauc_mrr_at_10_diff1": 0.4125753728196354, - "nauc_mrr_at_10_max": 0.2591621686708534, - "nauc_mrr_at_10_std": 0.1200428951411931, - "nauc_mrr_at_1_diff1": 0.46901333074242085, - "nauc_mrr_at_1_max": 0.26293584739784476, - "nauc_mrr_at_1_std": 0.09393200644418588, - "nauc_mrr_at_20_diff1": 0.41153820316953676, - "nauc_mrr_at_20_max": 0.25899728906177827, - "nauc_mrr_at_20_std": 0.12291882408761697, - "nauc_mrr_at_3_diff1": 0.4211323790196329, - "nauc_mrr_at_3_max": 0.26058816436358323, - "nauc_mrr_at_3_std": 0.11128421021474276, - "nauc_mrr_at_5_diff1": 0.4154901347685823, - "nauc_mrr_at_5_max": 0.2596997935959254, - "nauc_mrr_at_5_std": 0.1158003575154667, - "nauc_ndcg_at_1000_diff1": 0.31991443787696583, - "nauc_ndcg_at_1000_max": 0.13581124148871299, - "nauc_ndcg_at_1000_std": 0.19355178755325522, - "nauc_ndcg_at_100_diff1": 0.3224029160648352, - "nauc_ndcg_at_100_max": 0.14466078255754752, - "nauc_ndcg_at_100_std": 0.19269940771241528, - "nauc_ndcg_at_10_diff1": 0.349283174454767, - "nauc_ndcg_at_10_max": 0.17453802215625974, - "nauc_ndcg_at_10_std": 0.16998673115401805, - "nauc_ndcg_at_1_diff1": 0.46901333074242085, - "nauc_ndcg_at_1_max": 0.26293584739784476, - "nauc_ndcg_at_1_std": 0.09393200644418588, - "nauc_ndcg_at_20_diff1": 0.33437010159127156, - "nauc_ndcg_at_20_max": 0.1596751444117642, - "nauc_ndcg_at_20_std": 0.18048101211742304, - "nauc_ndcg_at_3_diff1": 0.3921220985778911, - "nauc_ndcg_at_3_max": 0.2239505034815312, - "nauc_ndcg_at_3_std": 0.14162546946540802, - "nauc_ndcg_at_5_diff1": 0.3715600153430542, - "nauc_ndcg_at_5_max": 0.1999499608847638, - "nauc_ndcg_at_5_std": 0.15656366009111194, - "nauc_precision_at_1000_diff1": 0.239065496292881, - "nauc_precision_at_1000_max": 0.28079661917675847, - "nauc_precision_at_1000_std": 0.45057101225797214, - "nauc_precision_at_100_diff1": 0.29069960725266336, - "nauc_precision_at_100_max": 0.24695181642790687, - "nauc_precision_at_100_std": 0.34139809733123694, - "nauc_precision_at_10_diff1": 0.3323472647610031, - "nauc_precision_at_10_max": 0.25077871562811144, - "nauc_precision_at_10_std": 0.1803436244627001, - "nauc_precision_at_1_diff1": 0.46901333074242085, - "nauc_precision_at_1_max": 0.26293584739784476, - "nauc_precision_at_1_std": 0.09393200644418588, - "nauc_precision_at_20_diff1": 0.31710061364731434, - "nauc_precision_at_20_max": 0.2492534303712408, - "nauc_precision_at_20_std": 0.22178558628756279, - "nauc_precision_at_3_diff1": 0.37191330940108014, - "nauc_precision_at_3_max": 0.257993853394655, - "nauc_precision_at_3_std": 0.13263393824575104, - "nauc_precision_at_5_diff1": 0.3506622748627595, - "nauc_precision_at_5_max": 0.25408365368896674, - "nauc_precision_at_5_std": 0.1499944631631691, - "nauc_recall_at_1000_diff1": 0.10001578043238427, - "nauc_recall_at_1000_max": -0.04662176275850385, - "nauc_recall_at_1000_std": 0.1900591135811226, - "nauc_recall_at_100_diff1": 0.15271776326483577, - "nauc_recall_at_100_max": 0.017407724603021528, - "nauc_recall_at_100_std": 0.21645494602605883, - "nauc_recall_at_10_diff1": 0.22719647788274416, - "nauc_recall_at_10_max": 0.08859833048166742, - "nauc_recall_at_10_std": 0.18795701765526512, - "nauc_recall_at_1_diff1": 0.4229161887175077, - "nauc_recall_at_1_max": 0.18563915762564656, - "nauc_recall_at_1_std": 0.13930048980022755, - "nauc_recall_at_20_diff1": 0.20317559817494252, - "nauc_recall_at_20_max": 0.069132885674504, - "nauc_recall_at_20_std": 0.2030970361116893, - "nauc_recall_at_3_diff1": 0.2881138434807705, - "nauc_recall_at_3_max": 0.1332165142075036, - "nauc_recall_at_3_std": 0.1689131041304317, - "nauc_recall_at_5_diff1": 0.2575729755739856, - "nauc_recall_at_5_max": 0.11253154184070889, - "nauc_recall_at_5_std": 0.1761186732999266, - "ndcg_at_1": 0.21298, - "ndcg_at_10": 0.16397, - "ndcg_at_100": 0.17741, - "ndcg_at_1000": 0.18846, - "ndcg_at_20": 0.16623, - "ndcg_at_3": 0.17523, - "ndcg_at_5": 0.16706, - "precision_at_1": 0.21298, - "precision_at_10": 0.04499, - "precision_at_100": 0.00671, - "precision_at_1000": 0.00085, - "precision_at_20": 0.02595, - "precision_at_3": 0.10948, - "precision_at_5": 0.0758, - "recall_at_1": 0.08389, - "recall_at_10": 0.17962, - "recall_at_100": 0.27225, - "recall_at_1000": 0.34652, - "recall_at_20": 0.20928, - "recall_at_3": 0.13013, - "recall_at_5": 0.15041 - } - ] - }, - "task_name": "OVENIT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/OxfordFlowersClassification.json b/results-mieb/voyage-multimodal-3/1/OxfordFlowersClassification.json deleted file mode 100644 index 1615c93265..0000000000 --- a/results-mieb/voyage-multimodal-3/1/OxfordFlowersClassification.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", - "evaluation_time": 2976.4298408031464, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.9264705882352942, - "f1": 0.9260382675718866, - "f1_weighted": 0.9255751571683284, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9264705882352942, - "scores_per_experiment": [ - { - "accuracy": 0.9333333333333333, - "f1": 0.9328345649295149, - "f1_weighted": 0.9322880465495 - }, - { - "accuracy": 0.9245098039215687, - "f1": 0.923887629877471, - "f1_weighted": 0.9234416217263346 - }, - { - "accuracy": 0.9245098039215687, - "f1": 0.9245404681244752, - "f1_weighted": 0.924177246664839 - }, - { - "accuracy": 0.9235294117647059, - "f1": 0.9228630696130874, - "f1_weighted": 0.9222446908845601 - }, - { - "accuracy": 0.9264705882352942, - "f1": 0.9260656053148842, - "f1_weighted": 0.9257241800164078 - } - ] - } - ] - }, - "task_name": "OxfordFlowersClassification" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/OxfordPets.json b/results-mieb/voyage-multimodal-3/1/OxfordPets.json deleted file mode 100644 index 344b6ab2f8..0000000000 --- a/results-mieb/voyage-multimodal-3/1/OxfordPets.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "557b480fae8d69247be74d9503b378a09425096f", - "evaluation_time": 1211.356342792511, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.8655764513491414, - "f1": 0.8620427475542456, - "f1_weighted": 0.8627648602390383, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8655764513491414, - "scores_per_experiment": [ - { - "accuracy": 0.860452439356773, - "f1": 0.8575450302018354, - "f1_weighted": 0.8581765441792053 - }, - { - "accuracy": 0.8689016080675933, - "f1": 0.8639469162042649, - "f1_weighted": 0.8648596012758134 - }, - { - "accuracy": 0.8639956391387299, - "f1": 0.8603811509947239, - "f1_weighted": 0.8611082432838298 - }, - { - "accuracy": 0.8659035159443991, - "f1": 0.8628250437566548, - "f1_weighted": 0.8635045209980706 - }, - { - "accuracy": 0.868629054238212, - "f1": 0.8655155966137498, - "f1_weighted": 0.866175391458273 - } - ] - } - ] - }, - "task_name": "OxfordPets" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/OxfordPetsZeroShot.json b/results-mieb/voyage-multimodal-3/1/OxfordPetsZeroShot.json deleted file mode 100644 index d405698708..0000000000 --- a/results-mieb/voyage-multimodal-3/1/OxfordPetsZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "557b480fae8d69247be74d9503b378a09425096f", - "evaluation_time": 677.260498046875, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.7050967566094304, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7050967566094304 - } - ] - }, - "task_name": "OxfordPetsZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/PatchCamelyon.json b/results-mieb/voyage-multimodal-3/1/PatchCamelyon.json deleted file mode 100644 index 565761adc2..0000000000 --- a/results-mieb/voyage-multimodal-3/1/PatchCamelyon.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "dataset_revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", - "evaluation_time": 1878.3961374759674, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.677899169921875, - "ap": 0.6171958412862929, - "ap_weighted": 0.6171958412862929, - "f1": 0.6716262837899631, - "f1_weighted": 0.6716104944451556, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.677899169921875, - "scores_per_experiment": [ - { - "accuracy": 0.7061767578125, - "ap": 0.6366412143481036, - "ap_weighted": 0.6366412143481036, - "f1": 0.7010964063919244, - "f1_weighted": 0.7010797573001903 - }, - { - "accuracy": 0.635162353515625, - "ap": 0.5809307261507256, - "ap_weighted": 0.5809307261507256, - "f1": 0.6234755309662483, - "f1_weighted": 0.6234471894599793 - }, - { - "accuracy": 0.7178955078125, - "ap": 0.6544682544786682, - "ap_weighted": 0.6544682544786682, - "f1": 0.7177918531207379, - "f1_weighted": 0.7177895423473547 - }, - { - "accuracy": 0.6851806640625, - "ap": 0.6264741993887054, - "ap_weighted": 0.6264741993887054, - "f1": 0.6851780733405203, - "f1_weighted": 0.6851776874883105 - }, - { - "accuracy": 0.64508056640625, - "ap": 0.5874648120652616, - "ap_weighted": 0.5874648120652616, - "f1": 0.6305895551303842, - "f1_weighted": 0.6305582956299431 - } - ] - } - ] - }, - "task_name": "PatchCamelyon" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/PatchCamelyonZeroShot.json b/results-mieb/voyage-multimodal-3/1/PatchCamelyonZeroShot.json deleted file mode 100644 index faad802aa7..0000000000 --- a/results-mieb/voyage-multimodal-3/1/PatchCamelyonZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", - "evaluation_time": 1639.0224986076355, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.611114501953125, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.611114501953125 - } - ] - }, - "task_name": "PatchCamelyonZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/RESISC45.json b/results-mieb/voyage-multimodal-3/1/RESISC45.json deleted file mode 100644 index 7b5e2ebe51..0000000000 --- a/results-mieb/voyage-multimodal-3/1/RESISC45.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", - "evaluation_time": 938.5634450912476, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.8733650793650793, - "f1": 0.8719221769133947, - "f1_weighted": 0.8726249328515951, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8733650793650793, - "scores_per_experiment": [ - { - "accuracy": 0.878095238095238, - "f1": 0.8766732208112427, - "f1_weighted": 0.8773890989875517 - }, - { - "accuracy": 0.8687301587301587, - "f1": 0.8665918615541792, - "f1_weighted": 0.8676431359172864 - }, - { - "accuracy": 0.8744444444444445, - "f1": 0.8729636786793165, - "f1_weighted": 0.8737471540962463 - }, - { - "accuracy": 0.8714285714285714, - "f1": 0.8704211254047678, - "f1_weighted": 0.8707897264966017 - }, - { - "accuracy": 0.8741269841269841, - "f1": 0.8729609981174671, - "f1_weighted": 0.8735555487602894 - } - ] - } - ] - }, - "task_name": "RESISC45" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/RESISC45ZeroShot.json b/results-mieb/voyage-multimodal-3/1/RESISC45ZeroShot.json deleted file mode 100644 index 27bf07b0fe..0000000000 --- a/results-mieb/voyage-multimodal-3/1/RESISC45ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", - "evaluation_time": 669.4255020618439, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.6173015873015874, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6173015873015874 - } - ] - }, - "task_name": "RESISC45ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/RP2kI2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/RP2kI2IRetrieval.json deleted file mode 100644 index 7ae814b4ed..0000000000 --- a/results-mieb/voyage-multimodal-3/1/RP2kI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "f8f82d4eb1aa4dc4dbf2c768596c8110a3703765", - "evaluation_time": 5771.473861217499, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.5352, - "cv_recall_at_10": 0.76819, - "cv_recall_at_100": 0.90682, - "cv_recall_at_1000": 0.96452, - "cv_recall_at_20": 0.81752, - "cv_recall_at_3": 0.65718, - "cv_recall_at_5": 0.70872, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5352, - "map_at_1": 0.06083, - "map_at_10": 0.13284, - "map_at_100": 0.2051, - "map_at_1000": 0.23144, - "map_at_20": 0.15717, - "map_at_3": 0.09271, - "map_at_5": 0.10891, - "mrr_at_1": 0.9998166867438328, - "mrr_at_10": 0.9998952495679044, - "mrr_at_100": 0.9998952495679044, - "mrr_at_1000": 0.9998952495679044, - "mrr_at_20": 0.9998952495679044, - "mrr_at_3": 0.9998952495679044, - "mrr_at_5": 0.9998952495679044, - "nauc_cv_recall_at_1000_diff1": -0.766019391968589, - "nauc_cv_recall_at_1000_max": -0.06295290555454099, - "nauc_cv_recall_at_1000_std": 0.3959283270329479, - "nauc_cv_recall_at_100_diff1": -0.5678400545768518, - "nauc_cv_recall_at_100_max": -0.02212835197434516, - "nauc_cv_recall_at_100_std": 0.419964891717186, - "nauc_cv_recall_at_10_diff1": -0.4646819473475005, - "nauc_cv_recall_at_10_max": -0.0077422960251187075, - "nauc_cv_recall_at_10_std": 0.3988736082851318, - "nauc_cv_recall_at_1_diff1": -0.3815580121104423, - "nauc_cv_recall_at_1_max": -0.006154143668930801, - "nauc_cv_recall_at_1_std": 0.3183472656015551, - "nauc_cv_recall_at_20_diff1": -0.49646521246826764, - "nauc_cv_recall_at_20_max": -0.013337728131588886, - "nauc_cv_recall_at_20_std": 0.40877102450803326, - "nauc_cv_recall_at_3_diff1": -0.4151287956971939, - "nauc_cv_recall_at_3_max": -0.0016536980365809229, - "nauc_cv_recall_at_3_std": 0.36740802264805184, - "nauc_cv_recall_at_5_diff1": -0.43025372385530897, - "nauc_cv_recall_at_5_max": -0.003428009132052308, - "nauc_cv_recall_at_5_std": 0.381203098652696, - "nauc_map_at_1000_diff1": -0.15120985670983805, - "nauc_map_at_1000_max": 0.009746992690475763, - "nauc_map_at_1000_std": 0.29094231039826757, - "nauc_map_at_100_diff1": -0.10235809229014188, - "nauc_map_at_100_max": 0.01064839570938484, - "nauc_map_at_100_std": 0.20613516282109914, - "nauc_map_at_10_diff1": 0.09725844480380152, - "nauc_map_at_10_max": 0.011271028569452779, - "nauc_map_at_10_std": -0.06270213969677413, - "nauc_map_at_1_diff1": 0.3425746405724122, - "nauc_map_at_1_max": 0.02365490366793686, - "nauc_map_at_1_std": -0.18605605415952398, - "nauc_map_at_20_diff1": 0.02314585270469293, - "nauc_map_at_20_max": 0.009229531572517024, - "nauc_map_at_20_std": 0.01713455805688018, - "nauc_map_at_3_diff1": 0.22558432744395535, - "nauc_map_at_3_max": 0.018412296426926624, - "nauc_map_at_3_std": -0.15025269476059971, - "nauc_map_at_5_diff1": 0.17255685699656403, - "nauc_map_at_5_max": 0.014538356234491148, - "nauc_map_at_5_std": -0.12065309682842128, - "nauc_mrr_at_1000_diff1": 0.8746298450050553, - "nauc_mrr_at_1000_max": -0.08822922720082992, - "nauc_mrr_at_1000_std": -0.26587692280465064, - "nauc_mrr_at_100_diff1": 0.8746298450050553, - "nauc_mrr_at_100_max": -0.08822922720082992, - "nauc_mrr_at_100_std": -0.26587692280465064, - "nauc_mrr_at_10_diff1": 0.8746298450050553, - "nauc_mrr_at_10_max": -0.08822922720082992, - "nauc_mrr_at_10_std": -0.26587692280465064, - "nauc_mrr_at_1_diff1": 0.8779506238559595, - "nauc_mrr_at_1_max": -0.03254161762624195, - "nauc_mrr_at_1_std": -0.26357582606359653, - "nauc_mrr_at_20_diff1": 0.8746298450050553, - "nauc_mrr_at_20_max": -0.08822922720082992, - "nauc_mrr_at_20_std": -0.26587692280465064, - "nauc_mrr_at_3_diff1": 0.8746298450050553, - "nauc_mrr_at_3_max": -0.08822922720082992, - "nauc_mrr_at_3_std": -0.26587692280465064, - "nauc_mrr_at_5_diff1": 0.8746298450050553, - "nauc_mrr_at_5_max": -0.08822922720082992, - "nauc_mrr_at_5_std": -0.26587692280465064, - "nauc_ndcg_at_1000_diff1": -0.21974184350852993, - "nauc_ndcg_at_1000_max": 0.0012162062863128136, - "nauc_ndcg_at_1000_std": 0.34775895764952175, - "nauc_ndcg_at_100_diff1": -0.20146467973477145, - "nauc_ndcg_at_100_max": 0.007725844480498439, - "nauc_ndcg_at_100_std": 0.3486766114714179, - "nauc_ndcg_at_10_diff1": -0.3173040774339957, - "nauc_ndcg_at_10_max": 0.005709132027725912, - "nauc_ndcg_at_10_std": 0.38631272877540723, - "nauc_ndcg_at_1_diff1": 0.8779506238559595, - "nauc_ndcg_at_1_max": -0.03254161762624195, - "nauc_ndcg_at_1_std": -0.26357582606359653, - "nauc_ndcg_at_20_diff1": -0.2909080633079741, - "nauc_ndcg_at_20_max": 0.006541878226556776, - "nauc_ndcg_at_20_std": 0.40254942311031094, - "nauc_ndcg_at_3_diff1": -0.34084820150248, - "nauc_ndcg_at_3_max": 0.001684052550435759, - "nauc_ndcg_at_3_std": 0.33703978667617646, - "nauc_ndcg_at_5_diff1": -0.33192655529088805, - "nauc_ndcg_at_5_max": 0.0035293920900730095, - "nauc_ndcg_at_5_std": 0.36066658957673187, - "nauc_precision_at_1000_diff1": -0.31636500140446766, - "nauc_precision_at_1000_max": -0.002192204079137363, - "nauc_precision_at_1000_std": 0.5138508071474455, - "nauc_precision_at_100_diff1": -0.3834205130450664, - "nauc_precision_at_100_max": 0.002633365597297572, - "nauc_precision_at_100_std": 0.5708202512526964, - "nauc_precision_at_10_diff1": -0.3959065384681834, - "nauc_precision_at_10_max": -0.001494129511746914, - "nauc_precision_at_10_std": 0.43210713213550545, - "nauc_precision_at_1_diff1": 0.8779506238559595, - "nauc_precision_at_1_max": -0.03254161762624195, - "nauc_precision_at_1_std": -0.26357582606359653, - "nauc_precision_at_20_diff1": -0.40001518210288856, - "nauc_precision_at_20_max": -0.0009124698708636982, - "nauc_precision_at_20_std": 0.48403989980888124, - "nauc_precision_at_3_diff1": -0.38229287317154065, - "nauc_precision_at_3_max": -0.0033741789883080554, - "nauc_precision_at_3_std": 0.3523027667471958, - "nauc_precision_at_5_diff1": -0.3863143496226523, - "nauc_precision_at_5_max": -0.00291350156539194, - "nauc_precision_at_5_std": 0.38593312972103216, - "nauc_recall_at_1000_diff1": -0.17969847236289574, - "nauc_recall_at_1000_max": -0.00016524020492468333, - "nauc_recall_at_1000_std": 0.31391704126449405, - "nauc_recall_at_100_diff1": -0.1010639403425878, - "nauc_recall_at_100_max": 0.009107897691813319, - "nauc_recall_at_100_std": 0.20625697881855873, - "nauc_recall_at_10_diff1": 0.10199050893201908, - "nauc_recall_at_10_max": 0.010931865384178865, - "nauc_recall_at_10_std": -0.06471258207066122, - "nauc_recall_at_1_diff1": 0.3425746405724122, - "nauc_recall_at_1_max": 0.02365490366793686, - "nauc_recall_at_1_std": -0.18605605415952398, - "nauc_recall_at_20_diff1": 0.02928013978860769, - "nauc_recall_at_20_max": 0.007750531130897131, - "nauc_recall_at_20_std": 0.01251388074417554, - "nauc_recall_at_3_diff1": 0.2268665131612476, - "nauc_recall_at_3_max": 0.018239264908074454, - "nauc_recall_at_3_std": -0.14999317739606058, - "nauc_recall_at_5_diff1": 0.17448687702754684, - "nauc_recall_at_5_max": 0.014284892663234853, - "nauc_recall_at_5_std": -0.12052944586207395, - "ndcg_at_1": 0.99982, - "ndcg_at_10": 0.56417, - "ndcg_at_100": 0.39403, - "ndcg_at_1000": 0.45364, - "ndcg_at_20": 0.48913, - "ndcg_at_3": 0.74355, - "ndcg_at_5": 0.65817, - "precision_at_1": 0.99982, - "precision_at_10": 0.44004, - "precision_at_100": 0.1663, - "precision_at_1000": 0.03803, - "precision_at_20": 0.34382, - "precision_at_3": 0.66234, - "precision_at_5": 0.55514, - "recall_at_1": 0.06083, - "recall_at_10": 0.14578, - "recall_at_100": 0.29716, - "recall_at_1000": 0.49997, - "recall_at_20": 0.18462, - "recall_at_3": 0.09383, - "recall_at_5": 0.1132 - } - ] - }, - "task_name": "RP2kI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/RenderedSST2.json b/results-mieb/voyage-multimodal-3/1/RenderedSST2.json deleted file mode 100644 index c7be2dccee..0000000000 --- a/results-mieb/voyage-multimodal-3/1/RenderedSST2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "66b9a461eda025201dd147e5f390f5984c33643a", - "evaluation_time": 130.35417199134827, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.8544755628775398, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8544755628775398 - } - ] - }, - "task_name": "RenderedSST2" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STL10.json b/results-mieb/voyage-multimodal-3/1/STL10.json deleted file mode 100644 index d5a02be6ea..0000000000 --- a/results-mieb/voyage-multimodal-3/1/STL10.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "49ae7f94508f7feae62baf836db284306eab0b0f", - "evaluation_time": 403.07147884368896, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.9878500000000001, - "f1": 0.9878558133843768, - "f1_weighted": 0.9878558133843767, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9878500000000001, - "scores_per_experiment": [ - { - "accuracy": 0.988, - "f1": 0.9880014064377992, - "f1_weighted": 0.9880014064377991 - }, - { - "accuracy": 0.9885, - "f1": 0.9885067563274645, - "f1_weighted": 0.9885067563274644 - }, - { - "accuracy": 0.987375, - "f1": 0.9873823425288343, - "f1_weighted": 0.9873823425288343 - }, - { - "accuracy": 0.98775, - "f1": 0.9877593004896299, - "f1_weighted": 0.9877593004896299 - }, - { - "accuracy": 0.987625, - "f1": 0.9876292611381554, - "f1_weighted": 0.9876292611381554 - } - ] - } - ] - }, - "task_name": "STL10" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STL10ZeroShot.json b/results-mieb/voyage-multimodal-3/1/STL10ZeroShot.json deleted file mode 100644 index bf6f845917..0000000000 --- a/results-mieb/voyage-multimodal-3/1/STL10ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "49ae7f94508f7feae62baf836db284306eab0b0f", - "evaluation_time": 383.49997425079346, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.987125, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.987125 - } - ] - }, - "task_name": "STL10ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS12VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS12VisualSTS.json deleted file mode 100644 index 0d5231319d..0000000000 --- a/results-mieb/voyage-multimodal-3/1/STS12VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "820c25edfba736f3789201b2476208cc62c2ccb9", - "evaluation_time": 381.2039792537689, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.7530408956307458, - "cosine_spearman": 0.7161988759864429, - "euclidean_pearson": 0.7141309522927283, - "euclidean_spearman": 0.7162724788414607, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7161988759864429, - "manhattan_pearson": 0.7140835255725306, - "manhattan_spearman": 0.7154340341352178, - "pearson": 0.7530408956307458, - "spearman": 0.7161988759864429 - } - ] - }, - "task_name": "STS12VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS13VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS13VisualSTS.json deleted file mode 100644 index f422f81ad5..0000000000 --- a/results-mieb/voyage-multimodal-3/1/STS13VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "561ee9ca47ff3e4a657283c59416deca8dc169f2", - "evaluation_time": 178.80085134506226, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.8041898098977802, - "cosine_spearman": 0.8160271504089486, - "euclidean_pearson": 0.8138453622908126, - "euclidean_spearman": 0.8160367132733977, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8160271504089486, - "manhattan_pearson": 0.8141249912528841, - "manhattan_spearman": 0.816315080890506, - "pearson": 0.8041898098977802, - "spearman": 0.8160271504089486 - } - ] - }, - "task_name": "STS13VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS14VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS14VisualSTS.json deleted file mode 100644 index 54abe0a859..0000000000 --- a/results-mieb/voyage-multimodal-3/1/STS14VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "824e95e45471024a684b901e0645579ffd9ca288", - "evaluation_time": 477.0949454307556, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.8126428329540769, - "cosine_spearman": 0.7798398073449294, - "euclidean_pearson": 0.8019773377625052, - "euclidean_spearman": 0.779773229647281, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7798398073449294, - "manhattan_pearson": 0.8019854029161337, - "manhattan_spearman": 0.7797635347309039, - "pearson": 0.8126428329540769, - "spearman": 0.7798398073449294 - } - ] - }, - "task_name": "STS14VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS15VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS15VisualSTS.json deleted file mode 100644 index d2a32dd340..0000000000 --- a/results-mieb/voyage-multimodal-3/1/STS15VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "1f8d08d9b9daac7118dfdefeb94b0aac4baf2e5f", - "evaluation_time": 385.04992389678955, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.8596747384607906, - "cosine_spearman": 0.8684963727234637, - "euclidean_pearson": 0.8648294270010821, - "euclidean_spearman": 0.8684429467437428, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8684963727234637, - "manhattan_pearson": 0.8645923546880342, - "manhattan_spearman": 0.8682092035635413, - "pearson": 0.8596747384607906, - "spearman": 0.8684963727234637 - } - ] - }, - "task_name": "STS15VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS16VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS16VisualSTS.json deleted file mode 100644 index 653d939404..0000000000 --- a/results-mieb/voyage-multimodal-3/1/STS16VisualSTS.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "fc354f19598af93f32c0af1b94046ffdeaacde15", - "evaluation_time": 150.6804370880127, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.8169368709648801, - "cosine_spearman": 0.8261532729608647, - "euclidean_pearson": 0.8216160911211441, - "euclidean_spearman": 0.8260077161971415, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8261532729608647, - "manhattan_pearson": 0.8212353007123708, - "manhattan_spearman": 0.8258613312454957, - "pearson": 0.8169368709648801, - "spearman": 0.8261532729608647 - } - ] - }, - "task_name": "STS16VisualSTS" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS17MultilingualVisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS17MultilingualVisualSTS.json deleted file mode 100644 index 8bde9d8217..0000000000 --- a/results-mieb/voyage-multimodal-3/1/STS17MultilingualVisualSTS.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", - "evaluation_time": 693.5000240802765, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cosine_pearson": 0.6295478701037069, - "cosine_spearman": 0.6279910477771019, - "euclidean_pearson": 0.6276223489798609, - "euclidean_spearman": 0.6280662784030709, - "hf_subset": "ko-ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.6279910477771019, - "manhattan_pearson": 0.6272252575072031, - "manhattan_spearman": 0.6275841741159163, - "pearson": 0.6295478701037069, - "spearman": 0.6279910477771019 - }, - { - "cosine_pearson": 0.6640588532698395, - "cosine_spearman": 0.6575414606953666, - "euclidean_pearson": 0.6455970584249845, - "euclidean_spearman": 0.6568954839487309, - "hf_subset": "ar-ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.6575414606953666, - "manhattan_pearson": 0.6456300855443649, - "manhattan_spearman": 0.6581063058387047, - "pearson": 0.6640588532698395, - "spearman": 0.6575414606953666 - }, - { - "cosine_pearson": 0.3785792513239742, - "cosine_spearman": 0.3440485632827937, - "euclidean_pearson": 0.3818330414668889, - "euclidean_spearman": 0.3425467167114543, - "hf_subset": "en-ar", - "languages": [ - "eng-Latn", - "ara-Arab" - ], - "main_score": 0.3440485632827937, - "manhattan_pearson": 0.37280091876138727, - "manhattan_spearman": 0.33497021404416516, - "pearson": 0.3785792513239742, - "spearman": 0.3440485632827937 - }, - { - "cosine_pearson": 0.797388779176213, - "cosine_spearman": 0.8041814174327988, - "euclidean_pearson": 0.8018372869118563, - "euclidean_spearman": 0.8046465372595982, - "hf_subset": "en-de", - "languages": [ - "eng-Latn", - "deu-Latn" - ], - "main_score": 0.8041814174327988, - "manhattan_pearson": 0.8036367691934629, - "manhattan_spearman": 0.806153756136359, - "pearson": 0.797388779176213, - "spearman": 0.8041814174327988 - }, - { - "cosine_pearson": 0.889324389401427, - "cosine_spearman": 0.8968348245174512, - "euclidean_pearson": 0.8988351349236602, - "euclidean_spearman": 0.8966841410694303, - "hf_subset": "en-en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8968348245174512, - "manhattan_pearson": 0.8987261929400799, - "manhattan_spearman": 0.8980541303774577, - "pearson": 0.889324389401427, - "spearman": 0.8968348245174512 - }, - { - "cosine_pearson": 0.46493619455982876, - "cosine_spearman": 0.4498243279387666, - "euclidean_pearson": 0.46435399844913355, - "euclidean_spearman": 0.451091543774284, - "hf_subset": "en-tr", - "languages": [ - "eng-Latn", - "tur-Latn" - ], - "main_score": 0.4498243279387666, - "manhattan_pearson": 0.4687785223213941, - "manhattan_spearman": 0.4568005510381185, - "pearson": 0.46493619455982876, - "spearman": 0.4498243279387666 - }, - { - "cosine_pearson": 0.7413496252477598, - "cosine_spearman": 0.7471866607172193, - "euclidean_pearson": 0.7483417196429633, - "euclidean_spearman": 0.7466169330700719, - "hf_subset": "es-en", - "languages": [ - "spa-Latn", - "eng-Latn" - ], - "main_score": 0.7471866607172193, - "manhattan_pearson": 0.7497169457615618, - "manhattan_spearman": 0.749720065032133, - "pearson": 0.7413496252477598, - "spearman": 0.7471866607172193 - }, - { - "cosine_pearson": 0.8401635646321107, - "cosine_spearman": 0.8370332502012957, - "euclidean_pearson": 0.850259304444977, - "euclidean_spearman": 0.8371101547419473, - "hf_subset": "es-es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.8370332502012957, - "manhattan_pearson": 0.8511002525095128, - "manhattan_spearman": 0.8384332973638572, - "pearson": 0.8401635646321107, - "spearman": 0.8370332502012957 - }, - { - "cosine_pearson": 0.7447131067366383, - "cosine_spearman": 0.750669573557799, - "euclidean_pearson": 0.7452718819569142, - "euclidean_spearman": 0.7508660001953976, - "hf_subset": "fr-en", - "languages": [ - "fra-Latn", - "eng-Latn" - ], - "main_score": 0.750669573557799, - "manhattan_pearson": 0.7480003355590945, - "manhattan_spearman": 0.7524451012272094, - "pearson": 0.7447131067366383, - "spearman": 0.750669573557799 - }, - { - "cosine_pearson": 0.7685526370938169, - "cosine_spearman": 0.7776349916156341, - "euclidean_pearson": 0.7692583708827287, - "euclidean_spearman": 0.7780436051494257, - "hf_subset": "it-en", - "languages": [ - "ita-Latn", - "eng-Latn" - ], - "main_score": 0.7776349916156341, - "manhattan_pearson": 0.7680758399408012, - "manhattan_spearman": 0.776599427307041, - "pearson": 0.7685526370938169, - "spearman": 0.7776349916156341 - }, - { - "cosine_pearson": 0.7848622824203596, - "cosine_spearman": 0.7824649342798778, - "euclidean_pearson": 0.7849080671012388, - "euclidean_spearman": 0.7823646067800476, - "hf_subset": "nl-en", - "languages": [ - "nld-Latn", - "eng-Latn" - ], - "main_score": 0.7824649342798778, - "manhattan_pearson": 0.7863829683396525, - "manhattan_spearman": 0.7856565788434446, - "pearson": 0.7848622824203596, - "spearman": 0.7824649342798778 - } - ] - }, - "task_name": "STS17MultilingualVisualSTS" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STSBenchmarkMultilingualVisualSTS.json b/results-mieb/voyage-multimodal-3/1/STSBenchmarkMultilingualVisualSTS.json deleted file mode 100644 index 2a7ee75282..0000000000 --- a/results-mieb/voyage-multimodal-3/1/STSBenchmarkMultilingualVisualSTS.json +++ /dev/null @@ -1,313 +0,0 @@ -{ - "dataset_revision": "9f1ab21f17f497974996ab74b3ff911165a7dbf9", - "evaluation_time": 5075.43874335289, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "dev": [ - { - "cosine_pearson": 0.8563893474191902, - "cosine_spearman": 0.858058860838852, - "euclidean_pearson": 0.8602263464877766, - "euclidean_spearman": 0.8579783803562652, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.858058860838852, - "manhattan_pearson": 0.8598337346826477, - "manhattan_spearman": 0.8575681612119586, - "pearson": 0.8563893474191902, - "spearman": 0.858058860838852 - }, - { - "cosine_pearson": 0.789854359195163, - "cosine_spearman": 0.7899033365671463, - "euclidean_pearson": 0.7916778007557717, - "euclidean_spearman": 0.7898486229980857, - "hf_subset": "de", - "languages": [ - "deu-Latn" - ], - "main_score": 0.7899033365671463, - "manhattan_pearson": 0.7910013245869385, - "manhattan_spearman": 0.7894274945306929, - "pearson": 0.789854359195163, - "spearman": 0.7899033365671463 - }, - { - "cosine_pearson": 0.796760464557315, - "cosine_spearman": 0.8013993811687563, - "euclidean_pearson": 0.8032555282023144, - "euclidean_spearman": 0.8015704115360485, - "hf_subset": "es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.8013993811687563, - "manhattan_pearson": 0.8020719661864791, - "manhattan_spearman": 0.8003944783391376, - "pearson": 0.796760464557315, - "spearman": 0.8013993811687563 - }, - { - "cosine_pearson": 0.7985932474785588, - "cosine_spearman": 0.8021648661142508, - "euclidean_pearson": 0.8031042634558174, - "euclidean_spearman": 0.8021820102550952, - "hf_subset": "fr", - "languages": [ - "fra-Latn" - ], - "main_score": 0.8021648661142508, - "manhattan_pearson": 0.8029724285823261, - "manhattan_spearman": 0.8019900652727078, - "pearson": 0.7985932474785588, - "spearman": 0.8021648661142508 - }, - { - "cosine_pearson": 0.7869557821233382, - "cosine_spearman": 0.7890330736912589, - "euclidean_pearson": 0.7906461233218647, - "euclidean_spearman": 0.7890663388388003, - "hf_subset": "it", - "languages": [ - "ita-Latn" - ], - "main_score": 0.7890330736912589, - "manhattan_pearson": 0.7903840939372646, - "manhattan_spearman": 0.7888588814000735, - "pearson": 0.7869557821233382, - "spearman": 0.7890330736912589 - }, - { - "cosine_pearson": 0.7791772215249965, - "cosine_spearman": 0.781520955767216, - "euclidean_pearson": 0.7801465349415607, - "euclidean_spearman": 0.7815041664138321, - "hf_subset": "nl", - "languages": [ - "nld-Latn" - ], - "main_score": 0.781520955767216, - "manhattan_pearson": 0.7789332520700933, - "manhattan_spearman": 0.7801130952911972, - "pearson": 0.7791772215249965, - "spearman": 0.781520955767216 - }, - { - "cosine_pearson": 0.7448849902931678, - "cosine_spearman": 0.7448043158631151, - "euclidean_pearson": 0.7405744069410145, - "euclidean_spearman": 0.7447120862522622, - "hf_subset": "pl", - "languages": [ - "pol-Latn" - ], - "main_score": 0.7448043158631151, - "manhattan_pearson": 0.7406202109132378, - "manhattan_spearman": 0.7445824918429544, - "pearson": 0.7448849902931678, - "spearman": 0.7448043158631151 - }, - { - "cosine_pearson": 0.779680602143197, - "cosine_spearman": 0.7826858098266277, - "euclidean_pearson": 0.7852151538285848, - "euclidean_spearman": 0.7826847689006386, - "hf_subset": "pt", - "languages": [ - "por-Latn" - ], - "main_score": 0.7826858098266277, - "manhattan_pearson": 0.7839441476775062, - "manhattan_spearman": 0.7813235864617706, - "pearson": 0.779680602143197, - "spearman": 0.7826858098266277 - }, - { - "cosine_pearson": 0.7884272004913433, - "cosine_spearman": 0.7892230784770122, - "euclidean_pearson": 0.7805909078732953, - "euclidean_spearman": 0.7893512299219531, - "hf_subset": "ru", - "languages": [ - "rus-Cyrl" - ], - "main_score": 0.7892230784770122, - "manhattan_pearson": 0.7794459614806362, - "manhattan_spearman": 0.7878490798979422, - "pearson": 0.7884272004913433, - "spearman": 0.7892230784770122 - }, - { - "cosine_pearson": 0.7654432441204759, - "cosine_spearman": 0.7647755317149096, - "euclidean_pearson": 0.7593503194362672, - "euclidean_spearman": 0.7647270138720049, - "hf_subset": "zh", - "languages": [ - "cmn-Hans" - ], - "main_score": 0.7647755317149096, - "manhattan_pearson": 0.7589361597204412, - "manhattan_spearman": 0.764541990552287, - "pearson": 0.7654432441204759, - "spearman": 0.7647755317149096 - } - ], - "test": [ - { - "cosine_pearson": 0.823492051147191, - "cosine_spearman": 0.8255498065512306, - "euclidean_pearson": 0.826698523592022, - "euclidean_spearman": 0.8254643124686717, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8255498065512306, - "manhattan_pearson": 0.8263307385343645, - "manhattan_spearman": 0.8247261005744897, - "pearson": 0.823492051147191, - "spearman": 0.8255498065512306 - }, - { - "cosine_pearson": 0.7492177185806835, - "cosine_spearman": 0.7413351070642012, - "euclidean_pearson": 0.7439940554472478, - "euclidean_spearman": 0.7414010247347829, - "hf_subset": "de", - "languages": [ - "deu-Latn" - ], - "main_score": 0.7413351070642012, - "manhattan_pearson": 0.7436240717647824, - "manhattan_spearman": 0.7409998672149616, - "pearson": 0.7492177185806835, - "spearman": 0.7413351070642012 - }, - { - "cosine_pearson": 0.7607906213948373, - "cosine_spearman": 0.7599250153963639, - "euclidean_pearson": 0.7564171074765642, - "euclidean_spearman": 0.7598909226716342, - "hf_subset": "es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.7599250153963639, - "manhattan_pearson": 0.7558868227763365, - "manhattan_spearman": 0.7597894599923053, - "pearson": 0.7607906213948373, - "spearman": 0.7599250153963639 - }, - { - "cosine_pearson": 0.7520573823123218, - "cosine_spearman": 0.7442880594186052, - "euclidean_pearson": 0.7420168403959864, - "euclidean_spearman": 0.7442472350562676, - "hf_subset": "fr", - "languages": [ - "fra-Latn" - ], - "main_score": 0.7442880594186052, - "manhattan_pearson": 0.7415021156781515, - "manhattan_spearman": 0.7434403372096718, - "pearson": 0.7520573823123218, - "spearman": 0.7442880594186052 - }, - { - "cosine_pearson": 0.7467690218563255, - "cosine_spearman": 0.7395905416647525, - "euclidean_pearson": 0.7357731015973681, - "euclidean_spearman": 0.7394434799461104, - "hf_subset": "it", - "languages": [ - "ita-Latn" - ], - "main_score": 0.7395905416647525, - "manhattan_pearson": 0.7350784821793559, - "manhattan_spearman": 0.7386311414238852, - "pearson": 0.7467690218563255, - "spearman": 0.7395905416647525 - }, - { - "cosine_pearson": 0.7202011364978524, - "cosine_spearman": 0.7134491712781116, - "euclidean_pearson": 0.7118458785635648, - "euclidean_spearman": 0.7134129390083591, - "hf_subset": "nl", - "languages": [ - "nld-Latn" - ], - "main_score": 0.7134491712781116, - "manhattan_pearson": 0.7109658343515404, - "manhattan_spearman": 0.7126032622007037, - "pearson": 0.7202011364978524, - "spearman": 0.7134491712781116 - }, - { - "cosine_pearson": 0.6985464798474408, - "cosine_spearman": 0.6883224080503167, - "euclidean_pearson": 0.6874216559416312, - "euclidean_spearman": 0.6881582413464393, - "hf_subset": "pl", - "languages": [ - "pol-Latn" - ], - "main_score": 0.6883224080503167, - "manhattan_pearson": 0.6871835930951957, - "manhattan_spearman": 0.6879319205502966, - "pearson": 0.6985464798474408, - "spearman": 0.6883224080503167 - }, - { - "cosine_pearson": 0.7362293011231412, - "cosine_spearman": 0.7348219968439652, - "euclidean_pearson": 0.7298573693202373, - "euclidean_spearman": 0.7346749511887691, - "hf_subset": "pt", - "languages": [ - "por-Latn" - ], - "main_score": 0.7348219968439652, - "manhattan_pearson": 0.7295003107001472, - "manhattan_spearman": 0.73423890877604, - "pearson": 0.7362293011231412, - "spearman": 0.7348219968439652 - }, - { - "cosine_pearson": 0.7368742675667894, - "cosine_spearman": 0.7267810144277265, - "euclidean_pearson": 0.7227644385927088, - "euclidean_spearman": 0.726788301200454, - "hf_subset": "ru", - "languages": [ - "rus-Cyrl" - ], - "main_score": 0.7267810144277265, - "manhattan_pearson": 0.721898680516144, - "manhattan_spearman": 0.7256356052253394, - "pearson": 0.7368742675667894, - "spearman": 0.7267810144277265 - }, - { - "cosine_pearson": 0.7345816344328195, - "cosine_spearman": 0.7259656655754602, - "euclidean_pearson": 0.7169100386731838, - "euclidean_spearman": 0.7260215086810657, - "hf_subset": "zh", - "languages": [ - "cmn-Hans" - ], - "main_score": 0.7259656655754602, - "manhattan_pearson": 0.7164561544415515, - "manhattan_spearman": 0.7256700966168421, - "pearson": 0.7345816344328195, - "spearman": 0.7259656655754602 - } - ] - }, - "task_name": "STSBenchmarkMultilingualVisualSTS" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/SUN397.json b/results-mieb/voyage-multimodal-3/1/SUN397.json deleted file mode 100644 index aa51b639cc..0000000000 --- a/results-mieb/voyage-multimodal-3/1/SUN397.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "7e6af6a2499ad708618be868e1471eac0aca1168", - "evaluation_time": 9430.040058612823, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.7631080459770115, - "f1": 0.7352522662342722, - "f1_weighted": 0.7637320144217425, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.7631080459770115, - "scores_per_experiment": [ - { - "accuracy": 0.7626206896551724, - "f1": 0.7355455453428275, - "f1_weighted": 0.7629772013685338 - }, - { - "accuracy": 0.7569655172413793, - "f1": 0.7319488897671674, - "f1_weighted": 0.7583476931412039 - }, - { - "accuracy": 0.7645057471264368, - "f1": 0.7373019095808092, - "f1_weighted": 0.7650115777182991 - }, - { - "accuracy": 0.7662528735632184, - "f1": 0.7359482181635232, - "f1_weighted": 0.7658961973253269 - }, - { - "accuracy": 0.7651954022988505, - "f1": 0.7355167683170333, - "f1_weighted": 0.7664274025553492 - } - ] - } - ] - }, - "task_name": "SUN397" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/SUN397ZeroShot.json b/results-mieb/voyage-multimodal-3/1/SUN397ZeroShot.json deleted file mode 100644 index ed71ad10e9..0000000000 --- a/results-mieb/voyage-multimodal-3/1/SUN397ZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "7e6af6a2499ad708618be868e1471eac0aca1168", - "evaluation_time": 3248.6113753318787, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.6864367816091954, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6864367816091954 - } - ] - }, - "task_name": "SUN397ZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/StanfordCars.json b/results-mieb/voyage-multimodal-3/1/StanfordCars.json deleted file mode 100644 index 5033cbad06..0000000000 --- a/results-mieb/voyage-multimodal-3/1/StanfordCars.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "dataset_revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", - "evaluation_time": 9452.947076797485, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.5290635493097874, - "f1": 0.531423599021726, - "f1_weighted": 0.5296608613945841, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5290635493097874, - "scores_per_experiment": [ - { - "accuracy": 0.531525929610745, - "f1": 0.5346764534490921, - "f1_weighted": 0.5336051434306501 - }, - { - "accuracy": 0.53276955602537, - "f1": 0.5323535520016426, - "f1_weighted": 0.5307497038657789 - }, - { - "accuracy": 0.5271732371595572, - "f1": 0.5312445144487528, - "f1_weighted": 0.5289222489776181 - }, - { - "accuracy": 0.5245616216888447, - "f1": 0.5265087065369574, - "f1_weighted": 0.5247431253688526 - }, - { - "accuracy": 0.5292874020644198, - "f1": 0.532334768672185, - "f1_weighted": 0.5302840853300209 - } - ] - } - ] - }, - "task_name": "StanfordCars" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/StanfordCarsI2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/StanfordCarsI2IRetrieval.json deleted file mode 100644 index af74f4743c..0000000000 --- a/results-mieb/voyage-multimodal-3/1/StanfordCarsI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "b27a0612211af3598bd11fe28af20928f20cce06", - "evaluation_time": 6306.916451454163, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.46785, - "cv_recall_at_10": 0.73672, - "cv_recall_at_100": 0.95075, - "cv_recall_at_1000": 0.9995, - "cv_recall_at_20": 0.81831, - "cv_recall_at_3": 0.60167, - "cv_recall_at_5": 0.65912, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.46785, - "map_at_1": 0.0, - "map_at_10": 0.03844, - "map_at_100": 0.08637, - "map_at_1000": 0.10395, - "map_at_20": 0.05772, - "map_at_3": 0.01142, - "map_at_5": 0.02093, - "mrr_at_1": 0.00012436264146250465, - "mrr_at_10": 0.2937613875712362, - "mrr_at_100": 0.3032439165995846, - "mrr_at_1000": 0.3035436772085266, - "mrr_at_20": 0.29964394708226216, - "mrr_at_3": 0.2634829830452242, - "mrr_at_5": 0.28110516934046254, - "nauc_cv_recall_at_1000_diff1": -1.151825029874656, - "nauc_cv_recall_at_1000_max": -0.2298295685244738, - "nauc_cv_recall_at_1000_std": -0.13833406665530235, - "nauc_cv_recall_at_100_diff1": -0.3065583434061352, - "nauc_cv_recall_at_100_max": 0.08250669884887286, - "nauc_cv_recall_at_100_std": -0.348791493134295, - "nauc_cv_recall_at_10_diff1": -0.395575993710386, - "nauc_cv_recall_at_10_max": -0.006248115883899727, - "nauc_cv_recall_at_10_std": -0.08906794871950323, - "nauc_cv_recall_at_1_diff1": -0.435307778833609, - "nauc_cv_recall_at_1_max": -0.007866420116790255, - "nauc_cv_recall_at_1_std": -0.030415830440148624, - "nauc_cv_recall_at_20_diff1": -0.36737862331436555, - "nauc_cv_recall_at_20_max": -0.006505738205243344, - "nauc_cv_recall_at_20_std": -0.15938166843054247, - "nauc_cv_recall_at_3_diff1": -0.4251008432692228, - "nauc_cv_recall_at_3_max": -0.009223330865472642, - "nauc_cv_recall_at_3_std": -0.051319053449302245, - "nauc_cv_recall_at_5_diff1": -0.4149559790404437, - "nauc_cv_recall_at_5_max": -0.011111361572481566, - "nauc_cv_recall_at_5_std": -0.07638029271951365, - "nauc_map_at_1000_diff1": -0.33739502802420707, - "nauc_map_at_1000_max": 0.0005840703968449165, - "nauc_map_at_1000_std": 0.05069004792828951, - "nauc_map_at_100_diff1": -0.34132369612264735, - "nauc_map_at_100_max": 0.0008966411708180106, - "nauc_map_at_100_std": 0.05068205970232099, - "nauc_map_at_10_diff1": -0.3619601668272867, - "nauc_map_at_10_max": 0.00014890087230641323, - "nauc_map_at_10_std": 0.012024432555931297, - "nauc_map_at_1_diff1": -0.5746685568283685, - "nauc_map_at_1_max": 0.09844700703211207, - "nauc_map_at_1_std": 0.661917627637042, - "nauc_map_at_20_diff1": -0.35182572786027133, - "nauc_map_at_20_max": 0.0013720545342691007, - "nauc_map_at_20_std": 0.02669559294518995, - "nauc_map_at_3_diff1": -0.38399345622920933, - "nauc_map_at_3_max": 0.0022945953431132045, - "nauc_map_at_3_std": -0.023320109114295547, - "nauc_map_at_5_diff1": -0.37108426221595764, - "nauc_map_at_5_max": 0.0038153853032365675, - "nauc_map_at_5_std": -0.009807037280071264, - "nauc_mrr_at_1000_diff1": -0.42454422892630644, - "nauc_mrr_at_1000_max": -0.006784315618668348, - "nauc_mrr_at_1000_std": -0.05235860206269823, - "nauc_mrr_at_100_diff1": -0.42426889507485727, - "nauc_mrr_at_100_max": -0.006583638725419789, - "nauc_mrr_at_100_std": -0.05289654089881731, - "nauc_mrr_at_10_diff1": -0.4252515611778805, - "nauc_mrr_at_10_max": -0.00828493016316975, - "nauc_mrr_at_10_std": -0.049487627586929324, - "nauc_mrr_at_1_diff1": -0.5746685568283684, - "nauc_mrr_at_1_max": 0.32383525527408397, - "nauc_mrr_at_1_std": 0.6619176276370419, - "nauc_mrr_at_20_diff1": -0.42374558442388, - "nauc_mrr_at_20_max": -0.007498321718482753, - "nauc_mrr_at_20_std": -0.05306656104216916, - "nauc_mrr_at_3_diff1": -0.4349144250848074, - "nauc_mrr_at_3_max": -0.005834866917902523, - "nauc_mrr_at_3_std": -0.030070112760466665, - "nauc_mrr_at_5_diff1": -0.4305631366028515, - "nauc_mrr_at_5_max": -0.005579537855220841, - "nauc_mrr_at_5_std": -0.044730291358109804, - "nauc_ndcg_at_1000_diff1": -0.3245598189947247, - "nauc_ndcg_at_1000_max": -0.001956981765244245, - "nauc_ndcg_at_1000_std": -0.005711298478346799, - "nauc_ndcg_at_100_diff1": -0.3384631852576036, - "nauc_ndcg_at_100_max": 0.0013101096062173095, - "nauc_ndcg_at_100_std": 0.010073931380352638, - "nauc_ndcg_at_10_diff1": -0.3680955726064528, - "nauc_ndcg_at_10_max": -0.000348791327047919, - "nauc_ndcg_at_10_std": -0.005616112265719418, - "nauc_ndcg_at_1_diff1": -0.5746685568283684, - "nauc_ndcg_at_1_max": 0.09844700703211205, - "nauc_ndcg_at_1_std": 0.6619176276370419, - "nauc_ndcg_at_20_diff1": -0.3541664733286353, - "nauc_ndcg_at_20_max": 0.0009538962638639399, - "nauc_ndcg_at_20_std": 0.001578883560078981, - "nauc_ndcg_at_3_diff1": -0.4051504575351157, - "nauc_ndcg_at_3_max": 0.0013420721598884624, - "nauc_ndcg_at_3_std": -0.0241111780824001, - "nauc_ndcg_at_5_diff1": -0.3845192804422571, - "nauc_ndcg_at_5_max": 0.0036776018462837904, - "nauc_ndcg_at_5_std": -0.019678161155368832, - "nauc_precision_at_1000_diff1": -0.23443308943299598, - "nauc_precision_at_1000_max": -0.005920538114289565, - "nauc_precision_at_1000_std": -0.00887291260859431, - "nauc_precision_at_100_diff1": -0.3029942937622413, - "nauc_precision_at_100_max": 0.002100537907211134, - "nauc_precision_at_100_std": 0.024000880512538438, - "nauc_precision_at_10_diff1": -0.35274373994854663, - "nauc_precision_at_10_max": -0.0008779745401321433, - "nauc_precision_at_10_std": -0.000516369982727631, - "nauc_precision_at_1_diff1": -0.5746685568283684, - "nauc_precision_at_1_max": 0.09844700703211205, - "nauc_precision_at_1_std": 0.6619176276370419, - "nauc_precision_at_20_diff1": -0.3334397092483834, - "nauc_precision_at_20_max": 0.0009749871462993896, - "nauc_precision_at_20_std": 0.00780023573483833, - "nauc_precision_at_3_diff1": -0.40082900870258426, - "nauc_precision_at_3_max": 0.0025889621419443207, - "nauc_precision_at_3_std": -0.023423965100093654, - "nauc_precision_at_5_diff1": -0.3751583392422432, - "nauc_precision_at_5_max": 0.005117377241896628, - "nauc_precision_at_5_std": -0.017923373435618145, - "nauc_recall_at_1000_diff1": -0.25844392309517555, - "nauc_recall_at_1000_max": -0.004435575371902964, - "nauc_recall_at_1000_std": -0.018739155834388607, - "nauc_recall_at_100_diff1": -0.30620642698455364, - "nauc_recall_at_100_max": 0.0019769607998182674, - "nauc_recall_at_100_std": 0.01672304677306232, - "nauc_recall_at_10_diff1": -0.35096276881889504, - "nauc_recall_at_10_max": -0.002378536023237739, - "nauc_recall_at_10_std": -0.00525163144320875, - "nauc_recall_at_1_diff1": -0.5746685568283685, - "nauc_recall_at_1_max": 0.09844700703211207, - "nauc_recall_at_1_std": 0.661917627637042, - "nauc_recall_at_20_diff1": -0.3327188709289407, - "nauc_recall_at_20_max": -0.00018314878513959012, - "nauc_recall_at_20_std": 0.002013745169352921, - "nauc_recall_at_3_diff1": -0.3866363742979928, - "nauc_recall_at_3_max": 0.0017557406432875135, - "nauc_recall_at_3_std": -0.0241880762963367, - "nauc_recall_at_5_diff1": -0.36821866944017695, - "nauc_recall_at_5_max": 0.003778984799888619, - "nauc_recall_at_5_std": -0.02061686291660384, - "ndcg_at_1": 0.00012, - "ndcg_at_10": 0.24122, - "ndcg_at_100": 0.21367, - "ndcg_at_1000": 0.35718, - "ndcg_at_20": 0.21098, - "ndcg_at_3": 0.22568, - "ndcg_at_5": 0.24666, - "precision_at_1": 0.00012, - "precision_at_10": 0.25882, - "precision_at_100": 0.09024, - "precision_at_1000": 0.02253, - "precision_at_20": 0.2067, - "precision_at_3": 0.27973, - "precision_at_5": 0.28807, - "recall_at_1": 0.0, - "recall_at_10": 0.06538, - "recall_at_100": 0.22602, - "recall_at_1000": 0.5597, - "recall_at_20": 0.1043, - "recall_at_3": 0.02118, - "recall_at_5": 0.03639 - } - ] - }, - "task_name": "StanfordCarsI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/StanfordCarsZeroShot.json b/results-mieb/voyage-multimodal-3/1/StanfordCarsZeroShot.json deleted file mode 100644 index 032e15909d..0000000000 --- a/results-mieb/voyage-multimodal-3/1/StanfordCarsZeroShot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", - "evaluation_time": 3249.768146753311, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.49396841188906854, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.49396841188906854 - } - ] - }, - "task_name": "StanfordCarsZeroShot" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/SugarCrepe.json b/results-mieb/voyage-multimodal-3/1/SugarCrepe.json deleted file mode 100644 index 402dbaac7c..0000000000 --- a/results-mieb/voyage-multimodal-3/1/SugarCrepe.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "134abf9ade6a32f9fdae0e89022ff227a70b87e5", - "evaluation_time": 2256.938088417053, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0, - "hf_subset": "default", - "image_acc": 0.0, - "languages": [ - "eng-Latn" - ], - "main_score": 0.8961523175239563, - "text_acc": 0.8961523175239563 - } - ] - }, - "task_name": "SugarCrepe" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/TUBerlinT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/TUBerlinT2IRetrieval.json deleted file mode 100644 index f989165017..0000000000 --- a/results-mieb/voyage-multimodal-3/1/TUBerlinT2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "0cd78cd1ddbd3cafa9f319c638ebd77836ec9ff6", - "evaluation_time": 6954.40220117569, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.632, - "cv_recall_at_10": 0.688, - "cv_recall_at_100": 0.768, - "cv_recall_at_1000": 0.88, - "cv_recall_at_20": 0.7, - "cv_recall_at_3": 0.672, - "cv_recall_at_5": 0.676, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.60246, - "map_at_1": 0.0079, - "map_at_10": 0.07075, - "map_at_100": 0.35832, - "map_at_1000": 0.39369, - "map_at_20": 0.13453, - "map_at_3": 0.02261, - "map_at_5": 0.03665, - "mrr_at_1": 0.632, - "mrr_at_10": 0.6523111111111112, - "mrr_at_100": 0.6544858405118993, - "mrr_at_1000": 0.6548076054198555, - "mrr_at_20": 0.6531414141414141, - "mrr_at_3": 0.65, - "mrr_at_5": 0.6507999999999999, - "nauc_cv_recall_at_1000_diff1": 0.43452115812917536, - "nauc_cv_recall_at_1000_max": 0.7407254215717467, - "nauc_cv_recall_at_1000_std": 0.9309576837416487, - "nauc_cv_recall_at_100_diff1": 0.4753903054873923, - "nauc_cv_recall_at_100_max": 0.7488708702300939, - "nauc_cv_recall_at_100_std": 0.9500337849852408, - "nauc_cv_recall_at_10_diff1": 0.4894545301258994, - "nauc_cv_recall_at_10_max": 0.7497962567446042, - "nauc_cv_recall_at_10_std": 0.9646751348920867, - "nauc_cv_recall_at_1_diff1": 0.41400656793789425, - "nauc_cv_recall_at_1_max": 0.6853743850422137, - "nauc_cv_recall_at_1_std": 0.820853831926218, - "nauc_cv_recall_at_20_diff1": 0.5039740587281571, - "nauc_cv_recall_at_20_max": 0.7425761124121779, - "nauc_cv_recall_at_20_std": 0.9761556476310572, - "nauc_cv_recall_at_3_diff1": 0.49802833754861175, - "nauc_cv_recall_at_3_max": 0.7442549835468169, - "nauc_cv_recall_at_3_std": 0.9699967365587014, - "nauc_cv_recall_at_5_diff1": 0.4873986442490248, - "nauc_cv_recall_at_5_max": 0.7356285598744322, - "nauc_cv_recall_at_5_std": 0.9651260820990151, - "nauc_map_at_1000_diff1": 0.18517532377370824, - "nauc_map_at_1000_max": 0.5397099906148431, - "nauc_map_at_1000_std": 0.7018854603300901, - "nauc_map_at_100_diff1": 0.16384249487832003, - "nauc_map_at_100_max": 0.5133118473344895, - "nauc_map_at_100_std": 0.6644666463401745, - "nauc_map_at_10_diff1": 0.33706560106522576, - "nauc_map_at_10_max": 0.644029000779149, - "nauc_map_at_10_std": 0.7258322218404971, - "nauc_map_at_1_diff1": 0.41400656793789353, - "nauc_map_at_1_max": 0.6853743850422134, - "nauc_map_at_1_std": 0.8208538319262177, - "nauc_map_at_20_diff1": 0.2937231437358452, - "nauc_map_at_20_max": 0.6247902961302264, - "nauc_map_at_20_std": 0.71105971054528, - "nauc_map_at_3_diff1": 0.3688228653223755, - "nauc_map_at_3_max": 0.6485263366734154, - "nauc_map_at_3_std": 0.7622131083978515, - "nauc_map_at_5_diff1": 0.35873005431899013, - "nauc_map_at_5_max": 0.6408516884302472, - "nauc_map_at_5_std": 0.7351246510760714, - "nauc_mrr_at_1000_diff1": 0.44699780409031015, - "nauc_mrr_at_1000_max": 0.7147526104841861, - "nauc_mrr_at_1000_std": 0.8853212411559411, - "nauc_mrr_at_100_diff1": 0.4470661493736612, - "nauc_mrr_at_100_max": 0.7148642247167679, - "nauc_mrr_at_100_std": 0.8854262060790729, - "nauc_mrr_at_10_diff1": 0.4473669698354353, - "nauc_mrr_at_10_max": 0.7148720726802916, - "nauc_mrr_at_10_std": 0.8854679886170576, - "nauc_mrr_at_1_diff1": 0.41400656793789425, - "nauc_mrr_at_1_max": 0.6853743850422137, - "nauc_mrr_at_1_std": 0.820853831926218, - "nauc_mrr_at_20_diff1": 0.4477358270590955, - "nauc_mrr_at_20_max": 0.7143799945198087, - "nauc_mrr_at_20_std": 0.8859265713576927, - "nauc_mrr_at_3_diff1": 0.44895760436162346, - "nauc_mrr_at_3_max": 0.7143570130468984, - "nauc_mrr_at_3_std": 0.8866628692326469, - "nauc_mrr_at_5_diff1": 0.4468593663508612, - "nauc_mrr_at_5_max": 0.7126695706582533, - "nauc_mrr_at_5_std": 0.8856072465478423, - "nauc_ndcg_at_1000_diff1": 0.3022533037187929, - "nauc_ndcg_at_1000_max": 0.6408175973611855, - "nauc_ndcg_at_1000_std": 0.8298954905844009, - "nauc_ndcg_at_100_diff1": 0.2540055991335015, - "nauc_ndcg_at_100_max": 0.5881314675701598, - "nauc_ndcg_at_100_std": 0.7623167279503071, - "nauc_ndcg_at_10_diff1": 0.3851681326373794, - "nauc_ndcg_at_10_max": 0.6781755638355185, - "nauc_ndcg_at_10_std": 0.7960845072732069, - "nauc_ndcg_at_1_diff1": 0.41400656793789425, - "nauc_ndcg_at_1_max": 0.6853743850422137, - "nauc_ndcg_at_1_std": 0.820853831926218, - "nauc_ndcg_at_20_diff1": 0.354832824475887, - "nauc_ndcg_at_20_max": 0.6678366209429181, - "nauc_ndcg_at_20_std": 0.7900276393440292, - "nauc_ndcg_at_3_diff1": 0.39433539441866544, - "nauc_ndcg_at_3_max": 0.6710511733456289, - "nauc_ndcg_at_3_std": 0.8067166269026869, - "nauc_ndcg_at_5_diff1": 0.39262145769837264, - "nauc_ndcg_at_5_max": 0.6688008888875651, - "nauc_ndcg_at_5_std": 0.7955553159332548, - "nauc_precision_at_1000_diff1": 0.3013794038214159, - "nauc_precision_at_1000_max": 0.6376415183291911, - "nauc_precision_at_1000_std": 0.84650127290728, - "nauc_precision_at_100_diff1": 0.2207309788254333, - "nauc_precision_at_100_max": 0.5578314150677521, - "nauc_precision_at_100_std": 0.747955127810586, - "nauc_precision_at_10_diff1": 0.37965193897786503, - "nauc_precision_at_10_max": 0.6763534352967545, - "nauc_precision_at_10_std": 0.7888648245686696, - "nauc_precision_at_1_diff1": 0.41400656793789425, - "nauc_precision_at_1_max": 0.6853743850422137, - "nauc_precision_at_1_std": 0.820853831926218, - "nauc_precision_at_20_diff1": 0.3384274323417243, - "nauc_precision_at_20_max": 0.6597557047868738, - "nauc_precision_at_20_std": 0.7803763157657103, - "nauc_precision_at_3_diff1": 0.3895862418304692, - "nauc_precision_at_3_max": 0.6625175539384656, - "nauc_precision_at_3_std": 0.8014294362560389, - "nauc_precision_at_5_diff1": 0.38931076445156515, - "nauc_precision_at_5_max": 0.6617444000758278, - "nauc_precision_at_5_std": 0.7868194632144762, - "nauc_recall_at_1000_diff1": 0.30137940382141565, - "nauc_recall_at_1000_max": 0.6376415183291911, - "nauc_recall_at_1000_std": 0.8465012729072791, - "nauc_recall_at_100_diff1": 0.2207309788254334, - "nauc_recall_at_100_max": 0.5578314150677524, - "nauc_recall_at_100_std": 0.7479551278105863, - "nauc_recall_at_10_diff1": 0.37965193897786503, - "nauc_recall_at_10_max": 0.6763534352967545, - "nauc_recall_at_10_std": 0.7888648245686696, - "nauc_recall_at_1_diff1": 0.41400656793789353, - "nauc_recall_at_1_max": 0.6853743850422134, - "nauc_recall_at_1_std": 0.8208538319262177, - "nauc_recall_at_20_diff1": 0.3384274323417243, - "nauc_recall_at_20_max": 0.6597557047868738, - "nauc_recall_at_20_std": 0.7803763157657103, - "nauc_recall_at_3_diff1": 0.38958624183046886, - "nauc_recall_at_3_max": 0.6625175539384649, - "nauc_recall_at_3_std": 0.8014294362560384, - "nauc_recall_at_5_diff1": 0.38931076445156515, - "nauc_recall_at_5_max": 0.6617444000758278, - "nauc_recall_at_5_std": 0.7868194632144762, - "ndcg_at_1": 0.632, - "ndcg_at_10": 0.60246, - "ndcg_at_100": 0.45777, - "ndcg_at_1000": 0.55125, - "ndcg_at_20": 0.58608, - "ndcg_at_3": 0.61902, - "ndcg_at_5": 0.61088, - "precision_at_1": 0.632, - "precision_at_10": 0.596, - "precision_at_100": 0.33596, - "precision_at_1000": 0.04708, - "precision_at_20": 0.5756, - "precision_at_3": 0.616, - "precision_at_5": 0.6056, - "recall_at_1": 0.0079, - "recall_at_10": 0.0745, - "recall_at_100": 0.41995, - "recall_at_1000": 0.58845, - "recall_at_20": 0.1439, - "recall_at_3": 0.0231, - "recall_at_5": 0.03785 - } - ] - }, - "task_name": "TUBerlinT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/TinyImageNetClustering.json b/results-mieb/voyage-multimodal-3/1/TinyImageNetClustering.json deleted file mode 100644 index 0aa996e4ba..0000000000 --- a/results-mieb/voyage-multimodal-3/1/TinyImageNetClustering.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "dataset_revision": "5a77092c28e51558c5586e9c5eb71a7e17a5e43f", - "evaluation_time": 368.8909206390381, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "valid": [ - { - "ari": 0.31776897199787596, - "cluster_accuracy": 0.4675, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6929420627528728, - "nmi": 0.6929420627528728, - "v_measure": 0.6929420627528728 - } - ] - }, - "task_name": "TinyImageNetClustering" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreArxivQARetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreArxivQARetrieval.json deleted file mode 100644 index 4203c356be..0000000000 --- a/results-mieb/voyage-multimodal-3/1/VidoreArxivQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "fe2b0e055eaac82d8f6801ebc8e85d8832248133", - "evaluation_time": 503.60697174072266, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.774, - "cv_recall_at_10": 0.93, - "cv_recall_at_100": 0.994, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.958, - "cv_recall_at_3": 0.872, - "cv_recall_at_5": 0.908, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8461, - "map_at_1": 0.774, - "map_at_10": 0.82827, - "map_at_100": 0.83144, - "map_at_1000": 0.83148, - "map_at_20": 0.83034, - "map_at_3": 0.81733, - "map_at_5": 0.82543, - "mrr_at_1": 0.774, - "mrr_at_10": 0.8282658730158727, - "mrr_at_100": 0.8314401062878024, - "mrr_at_1000": 0.8314832333872403, - "mrr_at_20": 0.8303449179475492, - "mrr_at_3": 0.8173333333333332, - "mrr_at_5": 0.8254333333333331, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.8078120136943575, - "nauc_cv_recall_at_100_max": 0.9074074074074102, - "nauc_cv_recall_at_100_std": 0.6934329287270279, - "nauc_cv_recall_at_10_diff1": 0.8166599973322675, - "nauc_cv_recall_at_10_max": 0.7872615712951853, - "nauc_cv_recall_at_10_std": -0.2622248899559833, - "nauc_cv_recall_at_1_diff1": 0.914007925160341, - "nauc_cv_recall_at_1_max": 0.8011901357626264, - "nauc_cv_recall_at_1_std": -0.39925832119140625, - "nauc_cv_recall_at_20_diff1": 0.8250411275621354, - "nauc_cv_recall_at_20_max": 0.8003868213952258, - "nauc_cv_recall_at_20_std": -0.30123160375261115, - "nauc_cv_recall_at_3_diff1": 0.8521416556530105, - "nauc_cv_recall_at_3_max": 0.7492265409107289, - "nauc_cv_recall_at_3_std": -0.3856557130842251, - "nauc_cv_recall_at_5_diff1": 0.8248792270531401, - "nauc_cv_recall_at_5_max": 0.7668980635732549, - "nauc_cv_recall_at_5_std": -0.30081597856534015, - "nauc_map_at_1000_diff1": 0.8962125696737271, - "nauc_map_at_1000_max": 0.7910425116794585, - "nauc_map_at_1000_std": -0.3820849643364578, - "nauc_map_at_100_diff1": 0.8961830489825202, - "nauc_map_at_100_max": 0.7910662571939857, - "nauc_map_at_100_std": -0.38181788077602036, - "nauc_map_at_10_diff1": 0.8944475891541415, - "nauc_map_at_10_max": 0.790785383308978, - "nauc_map_at_10_std": -0.3803408069574615, - "nauc_map_at_1_diff1": 0.914007925160341, - "nauc_map_at_1_max": 0.8011901357626264, - "nauc_map_at_1_std": -0.39925832119140625, - "nauc_map_at_20_diff1": 0.8956877931446563, - "nauc_map_at_20_max": 0.7911494373989493, - "nauc_map_at_20_std": -0.38239572816275114, - "nauc_map_at_3_diff1": 0.8949812790326085, - "nauc_map_at_3_max": 0.7857879975203541, - "nauc_map_at_3_std": -0.38926151758415656, - "nauc_map_at_5_diff1": 0.893513670583005, - "nauc_map_at_5_max": 0.7891425005671933, - "nauc_map_at_5_std": -0.3810974151707644, - "nauc_mrr_at_1000_diff1": 0.8962125696737271, - "nauc_mrr_at_1000_max": 0.7910425116794585, - "nauc_mrr_at_1000_std": -0.3820849643364578, - "nauc_mrr_at_100_diff1": 0.8961830489825202, - "nauc_mrr_at_100_max": 0.7910662571939857, - "nauc_mrr_at_100_std": -0.38181788077602036, - "nauc_mrr_at_10_diff1": 0.8944475891541415, - "nauc_mrr_at_10_max": 0.790785383308978, - "nauc_mrr_at_10_std": -0.3803408069574615, - "nauc_mrr_at_1_diff1": 0.914007925160341, - "nauc_mrr_at_1_max": 0.8011901357626264, - "nauc_mrr_at_1_std": -0.39925832119140625, - "nauc_mrr_at_20_diff1": 0.8956877931446563, - "nauc_mrr_at_20_max": 0.7911494373989493, - "nauc_mrr_at_20_std": -0.38239572816275114, - "nauc_mrr_at_3_diff1": 0.8949812790326085, - "nauc_mrr_at_3_max": 0.7857879975203541, - "nauc_mrr_at_3_std": -0.38926151758415656, - "nauc_mrr_at_5_diff1": 0.893513670583005, - "nauc_mrr_at_5_max": 0.7891425005671933, - "nauc_mrr_at_5_std": -0.3810974151707644, - "nauc_ndcg_at_1000_diff1": 0.8929451112766938, - "nauc_ndcg_at_1000_max": 0.7902064276004215, - "nauc_ndcg_at_1000_std": -0.37646127760601994, - "nauc_ndcg_at_100_diff1": 0.8923458106503991, - "nauc_ndcg_at_100_max": 0.7909366860251333, - "nauc_ndcg_at_100_std": -0.3695077693697132, - "nauc_ndcg_at_10_diff1": 0.8843225087145591, - "nauc_ndcg_at_10_max": 0.7895468920172003, - "nauc_ndcg_at_10_std": -0.3661802754316691, - "nauc_ndcg_at_1_diff1": 0.914007925160341, - "nauc_ndcg_at_1_max": 0.8011901357626264, - "nauc_ndcg_at_1_std": -0.39925832119140625, - "nauc_ndcg_at_20_diff1": 0.8888415250006587, - "nauc_ndcg_at_20_max": 0.7908558372598717, - "nauc_ndcg_at_20_std": -0.37481576312176695, - "nauc_ndcg_at_3_diff1": 0.8863782969130156, - "nauc_ndcg_at_3_max": 0.7784715680451962, - "nauc_ndcg_at_3_std": -0.38827854477910334, - "nauc_ndcg_at_5_diff1": 0.8825150872002235, - "nauc_ndcg_at_5_max": 0.7851998944708328, - "nauc_ndcg_at_5_std": -0.368947880668478, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.8078120136943184, - "nauc_precision_at_100_max": 0.9074074074073771, - "nauc_precision_at_100_std": 0.6934329287270289, - "nauc_precision_at_10_diff1": 0.8166599973322659, - "nauc_precision_at_10_max": 0.7872615712951837, - "nauc_precision_at_10_std": -0.2622248899559834, - "nauc_precision_at_1_diff1": 0.914007925160341, - "nauc_precision_at_1_max": 0.8011901357626264, - "nauc_precision_at_1_std": -0.39925832119140625, - "nauc_precision_at_20_diff1": 0.8250411275621338, - "nauc_precision_at_20_max": 0.8003868213952249, - "nauc_precision_at_20_std": -0.3012316037526172, - "nauc_precision_at_3_diff1": 0.8521416556530094, - "nauc_precision_at_3_max": 0.7492265409107284, - "nauc_precision_at_3_std": -0.3856557130842243, - "nauc_precision_at_5_diff1": 0.8248792270531391, - "nauc_precision_at_5_max": 0.7668980635732537, - "nauc_precision_at_5_std": -0.3008159785653395, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.8078120136943575, - "nauc_recall_at_100_max": 0.9074074074074102, - "nauc_recall_at_100_std": 0.6934329287270279, - "nauc_recall_at_10_diff1": 0.8166599973322675, - "nauc_recall_at_10_max": 0.7872615712951853, - "nauc_recall_at_10_std": -0.2622248899559833, - "nauc_recall_at_1_diff1": 0.914007925160341, - "nauc_recall_at_1_max": 0.8011901357626264, - "nauc_recall_at_1_std": -0.39925832119140625, - "nauc_recall_at_20_diff1": 0.8250411275621354, - "nauc_recall_at_20_max": 0.8003868213952258, - "nauc_recall_at_20_std": -0.30123160375261115, - "nauc_recall_at_3_diff1": 0.8521416556530105, - "nauc_recall_at_3_max": 0.7492265409107289, - "nauc_recall_at_3_std": -0.3856557130842251, - "nauc_recall_at_5_diff1": 0.8248792270531401, - "nauc_recall_at_5_max": 0.7668980635732549, - "nauc_recall_at_5_std": -0.30081597856534015, - "ndcg_at_1": 0.774, - "ndcg_at_10": 0.85311, - "ndcg_at_100": 0.86738, - "ndcg_at_1000": 0.86821, - "ndcg_at_20": 0.86035, - "ndcg_at_3": 0.83138, - "ndcg_at_5": 0.8461, - "precision_at_1": 0.774, - "precision_at_10": 0.093, - "precision_at_100": 0.00994, - "precision_at_1000": 0.001, - "precision_at_20": 0.0479, - "precision_at_3": 0.29067, - "precision_at_5": 0.1816, - "recall_at_1": 0.774, - "recall_at_10": 0.93, - "recall_at_100": 0.994, - "recall_at_1000": 1.0, - "recall_at_20": 0.958, - "recall_at_3": 0.872, - "recall_at_5": 0.908 - } - ] - }, - "task_name": "VidoreArxivQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreDocVQARetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreDocVQARetrieval.json deleted file mode 100644 index c376dd98c2..0000000000 --- a/results-mieb/voyage-multimodal-3/1/VidoreDocVQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "b1d89eda849e636676df6ead8002602fb1858600", - "evaluation_time": 636.6930768489838, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.4, - "cv_recall_at_10": 0.678, - "cv_recall_at_100": 0.904, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.742, - "cv_recall_at_3": 0.526, - "cv_recall_at_5": 0.586, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.49756, - "map_at_1": 0.4, - "map_at_10": 0.48082, - "map_at_100": 0.48939, - "map_at_1000": 0.48988, - "map_at_20": 0.48533, - "map_at_3": 0.455, - "map_at_5": 0.4683, - "mrr_at_1": 0.4, - "mrr_at_10": 0.4808214285714284, - "mrr_at_100": 0.48938796963272635, - "mrr_at_1000": 0.48987689381415356, - "mrr_at_20": 0.48532962232985455, - "mrr_at_3": 0.45500000000000024, - "mrr_at_5": 0.46830000000000005, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5545440398381588, - "nauc_cv_recall_at_100_max": 0.6224031279178338, - "nauc_cv_recall_at_100_std": 0.7042483660130732, - "nauc_cv_recall_at_10_diff1": 0.5922844253579569, - "nauc_cv_recall_at_10_max": 0.5432800999287697, - "nauc_cv_recall_at_10_std": 0.3076353976332295, - "nauc_cv_recall_at_1_diff1": 0.7808676622039137, - "nauc_cv_recall_at_1_max": 0.42874356333676633, - "nauc_cv_recall_at_1_std": 0.023313594232749747, - "nauc_cv_recall_at_20_diff1": 0.6226725372659271, - "nauc_cv_recall_at_20_max": 0.5880120622044127, - "nauc_cv_recall_at_20_std": 0.38371357569278075, - "nauc_cv_recall_at_3_diff1": 0.6621787729379458, - "nauc_cv_recall_at_3_max": 0.47943079125818877, - "nauc_cv_recall_at_3_std": 0.14791707160827738, - "nauc_cv_recall_at_5_diff1": 0.6327712997671421, - "nauc_cv_recall_at_5_max": 0.5113726338055739, - "nauc_cv_recall_at_5_std": 0.19278695996423892, - "nauc_map_at_1000_diff1": 0.7223029853211699, - "nauc_map_at_1000_max": 0.4573500912745452, - "nauc_map_at_1000_std": 0.0938975472934603, - "nauc_map_at_100_diff1": 0.7220311079046942, - "nauc_map_at_100_max": 0.4575388060411859, - "nauc_map_at_100_std": 0.09469610080340737, - "nauc_map_at_10_diff1": 0.7194427550991311, - "nauc_map_at_10_max": 0.4566455594885925, - "nauc_map_at_10_std": 0.09238228683223873, - "nauc_map_at_1_diff1": 0.7808676622039137, - "nauc_map_at_1_max": 0.42874356333676633, - "nauc_map_at_1_std": 0.023313594232749747, - "nauc_map_at_20_diff1": 0.7219933633150782, - "nauc_map_at_20_max": 0.4582682569401601, - "nauc_map_at_20_std": 0.09425310386243768, - "nauc_map_at_3_diff1": 0.7271767768253766, - "nauc_map_at_3_max": 0.44831239354747643, - "nauc_map_at_3_std": 0.07368394666258558, - "nauc_map_at_5_diff1": 0.7221060762465903, - "nauc_map_at_5_max": 0.45374468132339285, - "nauc_map_at_5_std": 0.08090647086207076, - "nauc_mrr_at_1000_diff1": 0.7223029853211699, - "nauc_mrr_at_1000_max": 0.4573500912745452, - "nauc_mrr_at_1000_std": 0.0938975472934603, - "nauc_mrr_at_100_diff1": 0.7220311079046942, - "nauc_mrr_at_100_max": 0.4575388060411859, - "nauc_mrr_at_100_std": 0.09469610080340737, - "nauc_mrr_at_10_diff1": 0.7194427550991311, - "nauc_mrr_at_10_max": 0.4566455594885925, - "nauc_mrr_at_10_std": 0.09238228683223873, - "nauc_mrr_at_1_diff1": 0.7808676622039137, - "nauc_mrr_at_1_max": 0.42874356333676633, - "nauc_mrr_at_1_std": 0.023313594232749747, - "nauc_mrr_at_20_diff1": 0.7219933633150782, - "nauc_mrr_at_20_max": 0.4582682569401601, - "nauc_mrr_at_20_std": 0.09425310386243768, - "nauc_mrr_at_3_diff1": 0.7271767768253766, - "nauc_mrr_at_3_max": 0.44831239354747643, - "nauc_mrr_at_3_std": 0.07368394666258558, - "nauc_mrr_at_5_diff1": 0.7221060762465903, - "nauc_mrr_at_5_max": 0.45374468132339285, - "nauc_mrr_at_5_std": 0.08090647086207076, - "nauc_ndcg_at_1000_diff1": 0.710686523220205, - "nauc_ndcg_at_1000_max": 0.46774398996740346, - "nauc_ndcg_at_1000_std": 0.12081898918084653, - "nauc_ndcg_at_100_diff1": 0.7038461631864035, - "nauc_ndcg_at_100_max": 0.4740669291705547, - "nauc_ndcg_at_100_std": 0.1453284101083725, - "nauc_ndcg_at_10_diff1": 0.6931946908889765, - "nauc_ndcg_at_10_max": 0.4740638945720107, - "nauc_ndcg_at_10_std": 0.13525061573977884, - "nauc_ndcg_at_1_diff1": 0.7808676622039137, - "nauc_ndcg_at_1_max": 0.42874356333676633, - "nauc_ndcg_at_1_std": 0.023313594232749747, - "nauc_ndcg_at_20_diff1": 0.7022453664871185, - "nauc_ndcg_at_20_max": 0.48027538874049586, - "nauc_ndcg_at_20_std": 0.14356378032475198, - "nauc_ndcg_at_3_diff1": 0.710802049583976, - "nauc_ndcg_at_3_max": 0.4560014006504877, - "nauc_ndcg_at_3_std": 0.09211662046447473, - "nauc_ndcg_at_5_diff1": 0.7008949703273786, - "nauc_ndcg_at_5_max": 0.46696217106895704, - "nauc_ndcg_at_5_std": 0.10693699002475192, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.5545440398381573, - "nauc_precision_at_100_max": 0.6224031279178333, - "nauc_precision_at_100_std": 0.7042483660130705, - "nauc_precision_at_10_diff1": 0.5922844253579582, - "nauc_precision_at_10_max": 0.5432800999287695, - "nauc_precision_at_10_std": 0.30763539763322983, - "nauc_precision_at_1_diff1": 0.7808676622039137, - "nauc_precision_at_1_max": 0.42874356333676633, - "nauc_precision_at_1_std": 0.023313594232749747, - "nauc_precision_at_20_diff1": 0.6226725372659271, - "nauc_precision_at_20_max": 0.5880120622044128, - "nauc_precision_at_20_std": 0.3837135756927807, - "nauc_precision_at_3_diff1": 0.6621787729379459, - "nauc_precision_at_3_max": 0.47943079125818866, - "nauc_precision_at_3_std": 0.14791707160827736, - "nauc_precision_at_5_diff1": 0.6327712997671422, - "nauc_precision_at_5_max": 0.5113726338055742, - "nauc_precision_at_5_std": 0.19278695996423872, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5545440398381588, - "nauc_recall_at_100_max": 0.6224031279178338, - "nauc_recall_at_100_std": 0.7042483660130732, - "nauc_recall_at_10_diff1": 0.5922844253579569, - "nauc_recall_at_10_max": 0.5432800999287697, - "nauc_recall_at_10_std": 0.3076353976332295, - "nauc_recall_at_1_diff1": 0.7808676622039137, - "nauc_recall_at_1_max": 0.42874356333676633, - "nauc_recall_at_1_std": 0.023313594232749747, - "nauc_recall_at_20_diff1": 0.6226725372659271, - "nauc_recall_at_20_max": 0.5880120622044127, - "nauc_recall_at_20_std": 0.38371357569278075, - "nauc_recall_at_3_diff1": 0.6621787729379458, - "nauc_recall_at_3_max": 0.47943079125818877, - "nauc_recall_at_3_std": 0.14791707160827738, - "nauc_recall_at_5_diff1": 0.6327712997671421, - "nauc_recall_at_5_max": 0.5113726338055739, - "nauc_recall_at_5_std": 0.19278695996423892, - "ndcg_at_1": 0.4, - "ndcg_at_10": 0.52756, - "ndcg_at_100": 0.57365, - "ndcg_at_1000": 0.58612, - "ndcg_at_20": 0.54381, - "ndcg_at_3": 0.47321, - "ndcg_at_5": 0.49756, - "precision_at_1": 0.4, - "precision_at_10": 0.0678, - "precision_at_100": 0.00904, - "precision_at_1000": 0.001, - "precision_at_20": 0.0371, - "precision_at_3": 0.17533, - "precision_at_5": 0.1172, - "recall_at_1": 0.4, - "recall_at_10": 0.678, - "recall_at_100": 0.904, - "recall_at_1000": 1.0, - "recall_at_20": 0.742, - "recall_at_3": 0.526, - "recall_at_5": 0.586 - } - ] - }, - "task_name": "VidoreDocVQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreInfoVQARetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreInfoVQARetrieval.json deleted file mode 100644 index ebe098be22..0000000000 --- a/results-mieb/voyage-multimodal-3/1/VidoreInfoVQARetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "fec9c59496ddf4a34e01ca8080515722bd3cf970", - "evaluation_time": 992.0337262153625, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.804, - "cv_recall_at_10": 0.942, - "cv_recall_at_100": 0.98, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.964, - "cv_recall_at_3": 0.878, - "cv_recall_at_5": 0.912, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.86114, - "map_at_1": 0.804, - "map_at_10": 0.8487, - "map_at_100": 0.85081, - "map_at_1000": 0.85089, - "map_at_20": 0.85029, - "map_at_3": 0.83667, - "map_at_5": 0.84427, - "mrr_at_1": 0.804, - "mrr_at_10": 0.8486992063492061, - "mrr_at_100": 0.8508131647390718, - "mrr_at_1000": 0.8508934544613491, - "mrr_at_20": 0.8502916295692607, - "mrr_at_3": 0.8366666666666666, - "mrr_at_5": 0.8442666666666663, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.8513071895424827, - "nauc_cv_recall_at_100_max": 0.8503734827264224, - "nauc_cv_recall_at_100_std": 0.7185340802987861, - "nauc_cv_recall_at_10_diff1": 0.834991467851509, - "nauc_cv_recall_at_10_max": 0.7839112656556875, - "nauc_cv_recall_at_10_std": 0.29012524550049995, - "nauc_cv_recall_at_1_diff1": 0.8625328284370615, - "nauc_cv_recall_at_1_max": 0.4626162991917587, - "nauc_cv_recall_at_1_std": -0.20730267310002226, - "nauc_cv_recall_at_20_diff1": 0.826097105508873, - "nauc_cv_recall_at_20_max": 0.7789189749974079, - "nauc_cv_recall_at_20_std": 0.5447660545699772, - "nauc_cv_recall_at_3_diff1": 0.8494693608816293, - "nauc_cv_recall_at_3_max": 0.5553683121443455, - "nauc_cv_recall_at_3_std": -0.17134862284648175, - "nauc_cv_recall_at_5_diff1": 0.8283783210253801, - "nauc_cv_recall_at_5_max": 0.6534462269756393, - "nauc_cv_recall_at_5_std": 0.020870469399881346, - "nauc_map_at_1000_diff1": 0.8581318188677318, - "nauc_map_at_1000_max": 0.5029151761338965, - "nauc_map_at_1000_std": -0.170525470223046, - "nauc_map_at_100_diff1": 0.8581332599209811, - "nauc_map_at_100_max": 0.503110087695966, - "nauc_map_at_100_std": -0.17004814783619687, - "nauc_map_at_10_diff1": 0.8577013332981014, - "nauc_map_at_10_max": 0.5067936179071768, - "nauc_map_at_10_std": -0.16767133793646685, - "nauc_map_at_1_diff1": 0.8625328284370615, - "nauc_map_at_1_max": 0.4626162991917587, - "nauc_map_at_1_std": -0.20730267310002226, - "nauc_map_at_20_diff1": 0.8579526180444351, - "nauc_map_at_20_max": 0.5035773415391347, - "nauc_map_at_20_std": -0.16871880221874927, - "nauc_map_at_3_diff1": 0.8590778618599331, - "nauc_map_at_3_max": 0.49468576246010537, - "nauc_map_at_3_std": -0.19930147132620046, - "nauc_map_at_5_diff1": 0.856560029179217, - "nauc_map_at_5_max": 0.5033649175384577, - "nauc_map_at_5_std": -0.17793292309864372, - "nauc_mrr_at_1000_diff1": 0.8581318188677318, - "nauc_mrr_at_1000_max": 0.5029151761338965, - "nauc_mrr_at_1000_std": -0.170525470223046, - "nauc_mrr_at_100_diff1": 0.8581332599209811, - "nauc_mrr_at_100_max": 0.503110087695966, - "nauc_mrr_at_100_std": -0.17004814783619687, - "nauc_mrr_at_10_diff1": 0.8577013332981014, - "nauc_mrr_at_10_max": 0.5067936179071768, - "nauc_mrr_at_10_std": -0.16767133793646685, - "nauc_mrr_at_1_diff1": 0.8625328284370615, - "nauc_mrr_at_1_max": 0.4626162991917587, - "nauc_mrr_at_1_std": -0.20730267310002226, - "nauc_mrr_at_20_diff1": 0.8579526180444351, - "nauc_mrr_at_20_max": 0.5035773415391347, - "nauc_mrr_at_20_std": -0.16871880221874927, - "nauc_mrr_at_3_diff1": 0.8590778618599331, - "nauc_mrr_at_3_max": 0.49468576246010537, - "nauc_mrr_at_3_std": -0.19930147132620046, - "nauc_mrr_at_5_diff1": 0.856560029179217, - "nauc_mrr_at_5_max": 0.5033649175384577, - "nauc_mrr_at_5_std": -0.17793292309864372, - "nauc_ndcg_at_1000_diff1": 0.8570009197459093, - "nauc_ndcg_at_1000_max": 0.515089390761954, - "nauc_ndcg_at_1000_std": -0.14762192315949124, - "nauc_ndcg_at_100_diff1": 0.8568954592274121, - "nauc_ndcg_at_100_max": 0.5223246104075928, - "nauc_ndcg_at_100_std": -0.12904095017909736, - "nauc_ndcg_at_10_diff1": 0.8548433259646303, - "nauc_ndcg_at_10_max": 0.5385901444924547, - "nauc_ndcg_at_10_std": -0.11731864860745544, - "nauc_ndcg_at_1_diff1": 0.8625328284370615, - "nauc_ndcg_at_1_max": 0.4626162991917587, - "nauc_ndcg_at_1_std": -0.20730267310002226, - "nauc_ndcg_at_20_diff1": 0.855337897660986, - "nauc_ndcg_at_20_max": 0.5263685689351201, - "nauc_ndcg_at_20_std": -0.11775218712386955, - "nauc_ndcg_at_3_diff1": 0.8571055573097612, - "nauc_ndcg_at_3_max": 0.5074477037297324, - "nauc_ndcg_at_3_std": -0.19362485217798167, - "nauc_ndcg_at_5_diff1": 0.8519901570958626, - "nauc_ndcg_at_5_max": 0.5278319529214093, - "nauc_ndcg_at_5_std": -0.14676364377526235, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.8513071895424774, - "nauc_precision_at_100_max": 0.8503734827264159, - "nauc_precision_at_100_std": 0.7185340802987755, - "nauc_precision_at_10_diff1": 0.8349914678515092, - "nauc_precision_at_10_max": 0.7839112656556898, - "nauc_precision_at_10_std": 0.2901252455005006, - "nauc_precision_at_1_diff1": 0.8625328284370615, - "nauc_precision_at_1_max": 0.4626162991917587, - "nauc_precision_at_1_std": -0.20730267310002226, - "nauc_precision_at_20_diff1": 0.8260971055088702, - "nauc_precision_at_20_max": 0.7789189749974024, - "nauc_precision_at_20_std": 0.5447660545699731, - "nauc_precision_at_3_diff1": 0.8494693608816277, - "nauc_precision_at_3_max": 0.5553683121443457, - "nauc_precision_at_3_std": -0.1713486228464822, - "nauc_precision_at_5_diff1": 0.8283783210253806, - "nauc_precision_at_5_max": 0.6534462269756387, - "nauc_precision_at_5_std": 0.020870469399880635, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.8513071895424827, - "nauc_recall_at_100_max": 0.8503734827264224, - "nauc_recall_at_100_std": 0.7185340802987861, - "nauc_recall_at_10_diff1": 0.834991467851509, - "nauc_recall_at_10_max": 0.7839112656556875, - "nauc_recall_at_10_std": 0.29012524550049995, - "nauc_recall_at_1_diff1": 0.8625328284370615, - "nauc_recall_at_1_max": 0.4626162991917587, - "nauc_recall_at_1_std": -0.20730267310002226, - "nauc_recall_at_20_diff1": 0.826097105508873, - "nauc_recall_at_20_max": 0.7789189749974079, - "nauc_recall_at_20_std": 0.5447660545699772, - "nauc_recall_at_3_diff1": 0.8494693608816293, - "nauc_recall_at_3_max": 0.5553683121443455, - "nauc_recall_at_3_std": -0.17134862284648175, - "nauc_recall_at_5_diff1": 0.8283783210253801, - "nauc_recall_at_5_max": 0.6534462269756393, - "nauc_recall_at_5_std": 0.020870469399881346, - "ndcg_at_1": 0.804, - "ndcg_at_10": 0.87127, - "ndcg_at_100": 0.88009, - "ndcg_at_1000": 0.88257, - "ndcg_at_20": 0.87692, - "ndcg_at_3": 0.84728, - "ndcg_at_5": 0.86114, - "precision_at_1": 0.804, - "precision_at_10": 0.0942, - "precision_at_100": 0.0098, - "precision_at_1000": 0.001, - "precision_at_20": 0.0482, - "precision_at_3": 0.29267, - "precision_at_5": 0.1824, - "recall_at_1": 0.804, - "recall_at_10": 0.942, - "recall_at_100": 0.98, - "recall_at_1000": 1.0, - "recall_at_20": 0.964, - "recall_at_3": 0.878, - "recall_at_5": 0.912 - } - ] - }, - "task_name": "VidoreInfoVQARetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreShiftProjectRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreShiftProjectRetrieval.json deleted file mode 100644 index 30cf8399b9..0000000000 --- a/results-mieb/voyage-multimodal-3/1/VidoreShiftProjectRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "9e7df4c35994683a7ba88002fb22917ffa15067e", - "evaluation_time": 1577.1782314777374, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.56, - "cv_recall_at_10": 0.98, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.99, - "cv_recall_at_3": 0.87, - "cv_recall_at_5": 0.94, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.77482, - "map_at_1": 0.56, - "map_at_10": 0.72388, - "map_at_100": 0.72486, - "map_at_1000": 0.72486, - "map_at_20": 0.7246, - "map_at_3": 0.70167, - "map_at_5": 0.71867, - "mrr_at_1": 0.56, - "mrr_at_10": 0.7238849206349208, - "mrr_at_100": 0.7248623642439433, - "mrr_at_1000": 0.7248623642439433, - "mrr_at_20": 0.7245992063492065, - "mrr_at_3": 0.7016666666666669, - "mrr_at_5": 0.7186666666666668, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": -0.5634920634920563, - "nauc_cv_recall_at_10_max": 0.3384687208216605, - "nauc_cv_recall_at_10_std": -0.5634920634920563, - "nauc_cv_recall_at_1_diff1": 0.6556609130138538, - "nauc_cv_recall_at_1_max": 0.17394390188507777, - "nauc_cv_recall_at_1_std": -0.47217806041335475, - "nauc_cv_recall_at_20_diff1": -0.5634920634920657, - "nauc_cv_recall_at_20_max": 0.12278244631185359, - "nauc_cv_recall_at_20_std": -0.5634920634920657, - "nauc_cv_recall_at_3_diff1": 0.5655409908909134, - "nauc_cv_recall_at_3_max": 0.4225357328001189, - "nauc_cv_recall_at_3_std": -0.6214544915944596, - "nauc_cv_recall_at_5_diff1": 0.30376595082477537, - "nauc_cv_recall_at_5_max": 0.25676937441643427, - "nauc_cv_recall_at_5_std": -0.7432306255835657, - "nauc_map_at_1000_diff1": 0.6230116173866328, - "nauc_map_at_1000_max": 0.20589579326708154, - "nauc_map_at_1000_std": -0.5150012670469309, - "nauc_map_at_100_diff1": 0.6230116173866328, - "nauc_map_at_100_max": 0.20589579326708154, - "nauc_map_at_100_std": -0.5150012670469309, - "nauc_map_at_10_diff1": 0.617681710216042, - "nauc_map_at_10_max": 0.2069385042265956, - "nauc_map_at_10_std": -0.5152190931252199, - "nauc_map_at_1_diff1": 0.6556609130138538, - "nauc_map_at_1_max": 0.17394390188507777, - "nauc_map_at_1_std": -0.47217806041335475, - "nauc_map_at_20_diff1": 0.6215719162765422, - "nauc_map_at_20_max": 0.20579494370394566, - "nauc_map_at_20_std": -0.515060105678952, - "nauc_map_at_3_diff1": 0.6329090417439073, - "nauc_map_at_3_max": 0.23321919704961555, - "nauc_map_at_3_std": -0.5188096812763278, - "nauc_map_at_5_diff1": 0.620811050355049, - "nauc_map_at_5_max": 0.20777525487629664, - "nauc_map_at_5_std": -0.522137936670128, - "nauc_mrr_at_1000_diff1": 0.6230116173866328, - "nauc_mrr_at_1000_max": 0.20589579326708154, - "nauc_mrr_at_1000_std": -0.5150012670469309, - "nauc_mrr_at_100_diff1": 0.6230116173866328, - "nauc_mrr_at_100_max": 0.20589579326708154, - "nauc_mrr_at_100_std": -0.5150012670469309, - "nauc_mrr_at_10_diff1": 0.617681710216042, - "nauc_mrr_at_10_max": 0.2069385042265956, - "nauc_mrr_at_10_std": -0.5152190931252199, - "nauc_mrr_at_1_diff1": 0.6556609130138538, - "nauc_mrr_at_1_max": 0.17394390188507777, - "nauc_mrr_at_1_std": -0.47217806041335475, - "nauc_mrr_at_20_diff1": 0.6215719162765422, - "nauc_mrr_at_20_max": 0.20579494370394566, - "nauc_mrr_at_20_std": -0.515060105678952, - "nauc_mrr_at_3_diff1": 0.6329090417439073, - "nauc_mrr_at_3_max": 0.23321919704961555, - "nauc_mrr_at_3_std": -0.5188096812763278, - "nauc_mrr_at_5_diff1": 0.620811050355049, - "nauc_mrr_at_5_max": 0.20777525487629664, - "nauc_mrr_at_5_std": -0.522137936670128, - "nauc_ndcg_at_1000_diff1": 0.6144858168068215, - "nauc_ndcg_at_1000_max": 0.20826882361687157, - "nauc_ndcg_at_1000_std": -0.519398085858273, - "nauc_ndcg_at_100_diff1": 0.6144858168068215, - "nauc_ndcg_at_100_max": 0.20826882361687157, - "nauc_ndcg_at_100_std": -0.519398085858273, - "nauc_ndcg_at_10_diff1": 0.5834552836723182, - "nauc_ndcg_at_10_max": 0.2125506181876858, - "nauc_ndcg_at_10_std": -0.5205596183937152, - "nauc_ndcg_at_1_diff1": 0.6556609130138538, - "nauc_ndcg_at_1_max": 0.17394390188507777, - "nauc_ndcg_at_1_std": -0.47217806041335475, - "nauc_ndcg_at_20_diff1": 0.6010944498115274, - "nauc_ndcg_at_20_max": 0.20729700617383273, - "nauc_ndcg_at_20_std": -0.5198993504801755, - "nauc_ndcg_at_3_diff1": 0.6224241742598228, - "nauc_ndcg_at_3_max": 0.26251029455353375, - "nauc_ndcg_at_3_std": -0.5352297029710446, - "nauc_ndcg_at_5_diff1": 0.5947188948668818, - "nauc_ndcg_at_5_max": 0.21356792656360682, - "nauc_ndcg_at_5_std": -0.5419445301313862, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": NaN, - "nauc_precision_at_100_max": NaN, - "nauc_precision_at_100_std": NaN, - "nauc_precision_at_10_diff1": -0.5634920634920675, - "nauc_precision_at_10_max": 0.3384687208216692, - "nauc_precision_at_10_std": -0.5634920634920675, - "nauc_precision_at_1_diff1": 0.6556609130138538, - "nauc_precision_at_1_max": 0.17394390188507777, - "nauc_precision_at_1_std": -0.47217806041335475, - "nauc_precision_at_20_diff1": -0.5634920634920767, - "nauc_precision_at_20_max": 0.12278244631185926, - "nauc_precision_at_20_std": -0.5634920634920767, - "nauc_precision_at_3_diff1": 0.5655409908909123, - "nauc_precision_at_3_max": 0.42253573280011736, - "nauc_precision_at_3_std": -0.6214544915944612, - "nauc_precision_at_5_diff1": 0.3037659508247733, - "nauc_precision_at_5_max": 0.2567693744164348, - "nauc_precision_at_5_std": -0.7432306255835679, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": NaN, - "nauc_recall_at_100_max": NaN, - "nauc_recall_at_100_std": NaN, - "nauc_recall_at_10_diff1": -0.5634920634920563, - "nauc_recall_at_10_max": 0.3384687208216605, - "nauc_recall_at_10_std": -0.5634920634920563, - "nauc_recall_at_1_diff1": 0.6556609130138538, - "nauc_recall_at_1_max": 0.17394390188507777, - "nauc_recall_at_1_std": -0.47217806041335475, - "nauc_recall_at_20_diff1": -0.5634920634920657, - "nauc_recall_at_20_max": 0.12278244631185359, - "nauc_recall_at_20_std": -0.5634920634920657, - "nauc_recall_at_3_diff1": 0.5655409908909134, - "nauc_recall_at_3_max": 0.4225357328001189, - "nauc_recall_at_3_std": -0.6214544915944596, - "nauc_recall_at_5_diff1": 0.30376595082477537, - "nauc_recall_at_5_max": 0.25676937441643427, - "nauc_recall_at_5_std": -0.7432306255835657, - "ndcg_at_1": 0.56, - "ndcg_at_10": 0.78765, - "ndcg_at_100": 0.79211, - "ndcg_at_1000": 0.79211, - "ndcg_at_20": 0.79021, - "ndcg_at_3": 0.74511, - "ndcg_at_5": 0.77482, - "precision_at_1": 0.56, - "precision_at_10": 0.098, - "precision_at_100": 0.01, - "precision_at_1000": 0.001, - "precision_at_20": 0.0495, - "precision_at_3": 0.29, - "precision_at_5": 0.188, - "recall_at_1": 0.56, - "recall_at_10": 0.98, - "recall_at_100": 1.0, - "recall_at_1000": 1.0, - "recall_at_20": 0.99, - "recall_at_3": 0.87, - "recall_at_5": 0.94 - } - ] - }, - "task_name": "VidoreShiftProjectRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAAIRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAAIRetrieval.json deleted file mode 100644 index f53016df4c..0000000000 --- a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAAIRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "5fe59d7e52732b86d11ee0e9c4a8cdb0e8ba7a6e", - "evaluation_time": 947.4307334423065, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.59, - "cv_recall_at_10": 1.0, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 0.99, - "cv_recall_at_5": 0.99, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.83559, - "map_at_1": 0.6, - "map_at_10": 0.78333, - "map_at_100": 0.78333, - "map_at_1000": 0.78333, - "map_at_20": 0.78333, - "map_at_3": 0.78167, - "map_at_5": 0.78167, - "mrr_at_1": 0.59, - "mrr_at_10": 0.7814285714285714, - "mrr_at_100": 0.7814285714285714, - "mrr_at_1000": 0.7814285714285714, - "mrr_at_20": 0.7814285714285714, - "mrr_at_3": 0.78, - "mrr_at_5": 0.78, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": NaN, - "nauc_cv_recall_at_10_max": NaN, - "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": 0.5448662450291281, - "nauc_cv_recall_at_1_max": 0.12269065402732315, - "nauc_cv_recall_at_1_std": -0.08235138744185386, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": -0.17133520074696149, - "nauc_cv_recall_at_3_max": 1.0, - "nauc_cv_recall_at_3_std": 0.554154995331463, - "nauc_cv_recall_at_5_diff1": -0.17133520074696149, - "nauc_cv_recall_at_5_max": 1.0, - "nauc_cv_recall_at_5_std": 0.554154995331463, - "nauc_map_at_1000_diff1": 0.4738056897477185, - "nauc_map_at_1000_max": 0.15407944176060157, - "nauc_map_at_1000_std": -0.08807478976561002, - "nauc_map_at_100_diff1": 0.4738056897477185, - "nauc_map_at_100_max": 0.15407944176060157, - "nauc_map_at_100_std": -0.08807478976561002, - "nauc_map_at_10_diff1": 0.4738056897477185, - "nauc_map_at_10_max": 0.15407944176060157, - "nauc_map_at_10_std": -0.08807478976561002, - "nauc_map_at_1_diff1": 0.5112644701691897, - "nauc_map_at_1_max": 0.1607153457999401, - "nauc_map_at_1_std": -0.09436034431582087, - "nauc_map_at_20_diff1": 0.4738056897477185, - "nauc_map_at_20_max": 0.15407944176060157, - "nauc_map_at_20_std": -0.08807478976561002, - "nauc_map_at_3_diff1": 0.4676830511027818, - "nauc_map_at_3_max": 0.16210755775314364, - "nauc_map_at_3_std": -0.08197977864617986, - "nauc_map_at_5_diff1": 0.4676830511027818, - "nauc_map_at_5_max": 0.16210755775314364, - "nauc_map_at_5_std": -0.08197977864617986, - "nauc_mrr_at_1000_diff1": 0.5025737502583414, - "nauc_mrr_at_1000_max": 0.15191339579868746, - "nauc_mrr_at_1000_std": -0.04260546581499429, - "nauc_mrr_at_100_diff1": 0.5025737502583414, - "nauc_mrr_at_100_max": 0.15191339579868746, - "nauc_mrr_at_100_std": -0.04260546581499429, - "nauc_mrr_at_10_diff1": 0.5025737502583414, - "nauc_mrr_at_10_max": 0.15191339579868746, - "nauc_mrr_at_10_std": -0.04260546581499429, - "nauc_mrr_at_1_diff1": 0.5448662450291281, - "nauc_mrr_at_1_max": 0.12269065402732315, - "nauc_mrr_at_1_std": -0.08235138744185386, - "nauc_mrr_at_20_diff1": 0.5025737502583414, - "nauc_mrr_at_20_max": 0.15191339579868746, - "nauc_mrr_at_20_std": -0.04260546581499429, - "nauc_mrr_at_3_diff1": 0.4970431657570428, - "nauc_mrr_at_3_max": 0.15887340815934356, - "nauc_mrr_at_3_std": -0.03770801764713768, - "nauc_mrr_at_5_diff1": 0.4970431657570428, - "nauc_mrr_at_5_max": 0.15887340815934356, - "nauc_mrr_at_5_std": -0.03770801764713768, - "nauc_ndcg_at_1000_diff1": 0.4710027496985591, - "nauc_ndcg_at_1000_max": 0.15453799577065144, - "nauc_ndcg_at_1000_std": -0.0870733233833494, - "nauc_ndcg_at_100_diff1": 0.4710027496985591, - "nauc_ndcg_at_100_max": 0.15453799577065144, - "nauc_ndcg_at_100_std": -0.0870733233833494, - "nauc_ndcg_at_10_diff1": 0.4710027496985591, - "nauc_ndcg_at_10_max": 0.15453799577065144, - "nauc_ndcg_at_10_std": -0.0870733233833494, - "nauc_ndcg_at_1_diff1": 0.5112644701691897, - "nauc_ndcg_at_1_max": 0.1607153457999401, - "nauc_ndcg_at_1_std": -0.09436034431582087, - "nauc_ndcg_at_20_diff1": 0.4710027496985591, - "nauc_ndcg_at_20_max": 0.15453799577065144, - "nauc_ndcg_at_20_std": -0.0870733233833494, - "nauc_ndcg_at_3_diff1": 0.45378387388053215, - "nauc_ndcg_at_3_max": 0.17720193030789522, - "nauc_ndcg_at_3_std": -0.06988419298354469, - "nauc_ndcg_at_5_diff1": 0.45378387388053215, - "nauc_ndcg_at_5_max": 0.17720193030789522, - "nauc_ndcg_at_5_std": -0.06988419298354469, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": NaN, - "nauc_precision_at_100_max": NaN, - "nauc_precision_at_100_std": NaN, - "nauc_precision_at_10_diff1": 1.0, - "nauc_precision_at_10_max": 1.0, - "nauc_precision_at_10_std": 1.0, - "nauc_precision_at_1_diff1": 0.5112644701691897, - "nauc_precision_at_1_max": 0.1607153457999401, - "nauc_precision_at_1_std": -0.09436034431582087, - "nauc_precision_at_20_diff1": 1.0, - "nauc_precision_at_20_max": 1.0, - "nauc_precision_at_20_std": 1.0, - "nauc_precision_at_3_diff1": -0.1713352007469878, - "nauc_precision_at_3_max": 1.0, - "nauc_precision_at_3_std": 0.5541549953314585, - "nauc_precision_at_5_diff1": -0.1713352007469681, - "nauc_precision_at_5_max": 1.0, - "nauc_precision_at_5_std": 0.5541549953314738, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": NaN, - "nauc_recall_at_100_max": NaN, - "nauc_recall_at_100_std": NaN, - "nauc_recall_at_10_diff1": NaN, - "nauc_recall_at_10_max": NaN, - "nauc_recall_at_10_std": NaN, - "nauc_recall_at_1_diff1": 0.5112644701691897, - "nauc_recall_at_1_max": 0.1607153457999401, - "nauc_recall_at_1_std": -0.09436034431582087, - "nauc_recall_at_20_diff1": NaN, - "nauc_recall_at_20_max": NaN, - "nauc_recall_at_20_std": NaN, - "nauc_recall_at_3_diff1": -0.17133520074696149, - "nauc_recall_at_3_max": 1.0, - "nauc_recall_at_3_std": 0.554154995331463, - "nauc_recall_at_5_diff1": -0.17133520074696149, - "nauc_recall_at_5_max": 1.0, - "nauc_recall_at_5_std": 0.554154995331463, - "ndcg_at_1": 0.6, - "ndcg_at_10": 0.83915, - "ndcg_at_100": 0.83915, - "ndcg_at_1000": 0.83915, - "ndcg_at_20": 0.83915, - "ndcg_at_3": 0.83559, - "ndcg_at_5": 0.83559, - "precision_at_1": 0.6, - "precision_at_10": 0.1, - "precision_at_100": 0.01, - "precision_at_1000": 0.001, - "precision_at_20": 0.05, - "precision_at_3": 0.33, - "precision_at_5": 0.198, - "recall_at_1": 0.6, - "recall_at_10": 1.0, - "recall_at_100": 1.0, - "recall_at_1000": 1.0, - "recall_at_20": 1.0, - "recall_at_3": 0.99, - "recall_at_5": 0.99 - } - ] - }, - "task_name": "VidoreSyntheticDocQAAIRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAEnergyRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAEnergyRetrieval.json deleted file mode 100644 index ca9fa98ef9..0000000000 --- a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAEnergyRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "0821bc71310cfa51d5c8131d4d8b9c4d537bd8c8", - "evaluation_time": 974.9262182712555, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.62, - "cv_recall_at_10": 0.97, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.99, - "cv_recall_at_3": 0.91, - "cv_recall_at_5": 0.93, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.79422, - "map_at_1": 0.59, - "map_at_10": 0.75287, - "map_at_100": 0.75479, - "map_at_1000": 0.75479, - "map_at_20": 0.75442, - "map_at_3": 0.745, - "map_at_5": 0.747, - "mrr_at_1": 0.62, - "mrr_at_10": 0.7637063492063493, - "mrr_at_100": 0.7656243386243387, - "mrr_at_1000": 0.7656243386243387, - "mrr_at_20": 0.7652539682539683, - "mrr_at_3": 0.7533333333333333, - "mrr_at_5": 0.7578333333333334, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": 0.27964519140989424, - "nauc_cv_recall_at_10_max": 0.13414254590725253, - "nauc_cv_recall_at_10_std": -0.9229691876750729, - "nauc_cv_recall_at_1_diff1": 0.6289419438205704, - "nauc_cv_recall_at_1_max": -0.04647263329538835, - "nauc_cv_recall_at_1_std": -0.4368212165030922, - "nauc_cv_recall_at_20_diff1": 0.35807656395891085, - "nauc_cv_recall_at_20_max": 1.0, - "nauc_cv_recall_at_20_std": -1.151727357609709, - "nauc_cv_recall_at_3_diff1": 0.2426081543728594, - "nauc_cv_recall_at_3_max": 0.39864093785662147, - "nauc_cv_recall_at_3_std": -0.8358232181761602, - "nauc_cv_recall_at_5_diff1": 0.3188608776844068, - "nauc_cv_recall_at_5_max": 0.4858610110710928, - "nauc_cv_recall_at_5_std": -0.9136321195144755, - "nauc_map_at_1000_diff1": 0.6050487509101241, - "nauc_map_at_1000_max": 0.08450426872364926, - "nauc_map_at_1000_std": -0.529759315257296, - "nauc_map_at_100_diff1": 0.6050487509101241, - "nauc_map_at_100_max": 0.08450426872364926, - "nauc_map_at_100_std": -0.529759315257296, - "nauc_map_at_10_diff1": 0.6018452314830903, - "nauc_map_at_10_max": 0.08272890897551674, - "nauc_map_at_10_std": -0.5336167734838453, - "nauc_map_at_1_diff1": 0.6693031241287217, - "nauc_map_at_1_max": 0.031432051300864135, - "nauc_map_at_1_std": -0.445859685679487, - "nauc_map_at_20_diff1": 0.6045885907957769, - "nauc_map_at_20_max": 0.08621002606252022, - "nauc_map_at_20_std": -0.53091816998107, - "nauc_map_at_3_diff1": 0.5869736640493909, - "nauc_map_at_3_max": 0.10354995309831401, - "nauc_map_at_3_std": -0.5425945788378098, - "nauc_map_at_5_diff1": 0.5943286796830018, - "nauc_map_at_5_max": 0.101081239769278, - "nauc_map_at_5_std": -0.5423918892300909, - "nauc_mrr_at_1000_diff1": 0.5629855995499428, - "nauc_mrr_at_1000_max": 0.025525805506282775, - "nauc_mrr_at_1000_std": -0.519994655773919, - "nauc_mrr_at_100_diff1": 0.5629855995499428, - "nauc_mrr_at_100_max": 0.025525805506282775, - "nauc_mrr_at_100_std": -0.519994655773919, - "nauc_mrr_at_10_diff1": 0.5601335343154699, - "nauc_mrr_at_10_max": 0.024292956739306003, - "nauc_mrr_at_10_std": -0.5240179228031371, - "nauc_mrr_at_1_diff1": 0.6289419438205704, - "nauc_mrr_at_1_max": -0.04647263329538835, - "nauc_mrr_at_1_std": -0.4368212165030922, - "nauc_mrr_at_20_diff1": 0.5625967679302477, - "nauc_mrr_at_20_max": 0.027374949826847985, - "nauc_mrr_at_20_std": -0.5211934201376318, - "nauc_mrr_at_3_diff1": 0.5414891458713736, - "nauc_mrr_at_3_max": 0.04508464642366023, - "nauc_mrr_at_3_std": -0.5338848688412485, - "nauc_mrr_at_5_diff1": 0.5536925278417223, - "nauc_mrr_at_5_max": 0.04466542695249133, - "nauc_mrr_at_5_std": -0.5332272586570691, - "nauc_ndcg_at_1000_diff1": 0.5975908372870623, - "nauc_ndcg_at_1000_max": 0.09253310457244465, - "nauc_ndcg_at_1000_std": -0.53968784190258, - "nauc_ndcg_at_100_diff1": 0.5975908372870623, - "nauc_ndcg_at_100_max": 0.09253310457244465, - "nauc_ndcg_at_100_std": -0.53968784190258, - "nauc_ndcg_at_10_diff1": 0.5826233093198272, - "nauc_ndcg_at_10_max": 0.09067392180823283, - "nauc_ndcg_at_10_std": -0.5575736421728364, - "nauc_ndcg_at_1_diff1": 0.6693031241287217, - "nauc_ndcg_at_1_max": 0.031432051300864135, - "nauc_ndcg_at_1_std": -0.445859685679487, - "nauc_ndcg_at_20_diff1": 0.5943230562558864, - "nauc_ndcg_at_20_max": 0.1049140080822575, - "nauc_ndcg_at_20_std": -0.5480381213743085, - "nauc_ndcg_at_3_diff1": 0.5475794752772646, - "nauc_ndcg_at_3_max": 0.14652511249455774, - "nauc_ndcg_at_3_std": -0.5823793030749803, - "nauc_ndcg_at_5_diff1": 0.5638248903434933, - "nauc_ndcg_at_5_max": 0.14174465488043955, - "nauc_ndcg_at_5_std": -0.582806100658493, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": NaN, - "nauc_precision_at_100_max": NaN, - "nauc_precision_at_100_std": NaN, - "nauc_precision_at_10_diff1": 0.2796451914099003, - "nauc_precision_at_10_max": 0.13414254590725072, - "nauc_precision_at_10_std": -0.9229691876750711, - "nauc_precision_at_1_diff1": 0.6693031241287217, - "nauc_precision_at_1_max": 0.031432051300864135, - "nauc_precision_at_1_std": -0.445859685679487, - "nauc_precision_at_20_diff1": 0.35807656395891135, - "nauc_precision_at_20_max": 1.0, - "nauc_precision_at_20_std": -1.1517273576097316, - "nauc_precision_at_3_diff1": 0.25758636788048667, - "nauc_precision_at_3_max": 0.469887955182072, - "nauc_precision_at_3_std": -0.86986461251167, - "nauc_precision_at_5_diff1": 0.3188608776844103, - "nauc_precision_at_5_max": 0.48586101107109614, - "nauc_precision_at_5_std": -0.9136321195144689, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": NaN, - "nauc_recall_at_100_max": NaN, - "nauc_recall_at_100_std": NaN, - "nauc_recall_at_10_diff1": 0.27964519140989424, - "nauc_recall_at_10_max": 0.13414254590725253, - "nauc_recall_at_10_std": -0.9229691876750729, - "nauc_recall_at_1_diff1": 0.6693031241287217, - "nauc_recall_at_1_max": 0.031432051300864135, - "nauc_recall_at_1_std": -0.445859685679487, - "nauc_recall_at_20_diff1": 0.35807656395891085, - "nauc_recall_at_20_max": 1.0, - "nauc_recall_at_20_std": -1.151727357609709, - "nauc_recall_at_3_diff1": 0.2575863678804853, - "nauc_recall_at_3_max": 0.4698879551820732, - "nauc_recall_at_3_std": -0.869864612511674, - "nauc_recall_at_5_diff1": 0.3188608776844068, - "nauc_recall_at_5_max": 0.4858610110710928, - "nauc_recall_at_5_std": -0.9136321195144755, - "ndcg_at_1": 0.59, - "ndcg_at_10": 0.80769, - "ndcg_at_100": 0.81503, - "ndcg_at_1000": 0.81503, - "ndcg_at_20": 0.81295, - "ndcg_at_3": 0.79035, - "ndcg_at_5": 0.79422, - "precision_at_1": 0.59, - "precision_at_10": 0.097, - "precision_at_100": 0.01, - "precision_at_1000": 0.001, - "precision_at_20": 0.0495, - "precision_at_3": 0.30667, - "precision_at_5": 0.186, - "recall_at_1": 0.59, - "recall_at_10": 0.97, - "recall_at_100": 1.0, - "recall_at_1000": 1.0, - "recall_at_20": 0.99, - "recall_at_3": 0.92, - "recall_at_5": 0.93 - } - ] - }, - "task_name": "VidoreSyntheticDocQAEnergyRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAGovernmentReportsRetrieval.json deleted file mode 100644 index 6db6c0dec9..0000000000 --- a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAGovernmentReportsRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "8270b3751ce6b95bec362fb38fbcd2a4aa400cfc", - "evaluation_time": 986.1590669155121, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.61, - "cv_recall_at_10": 1.0, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 0.99, - "cv_recall_at_5": 1.0, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.83922, - "map_at_1": 0.61, - "map_at_10": 0.78367, - "map_at_100": 0.78367, - "map_at_1000": 0.78367, - "map_at_20": 0.78367, - "map_at_3": 0.78167, - "map_at_5": 0.78367, - "mrr_at_1": 0.61, - "mrr_at_10": 0.7825, - "mrr_at_100": 0.7825, - "mrr_at_1000": 0.7825, - "mrr_at_20": 0.7825, - "mrr_at_3": 0.78, - "mrr_at_5": 0.7825, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": NaN, - "nauc_cv_recall_at_10_max": NaN, - "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": 0.6026921275663585, - "nauc_cv_recall_at_1_max": 0.1182149169851797, - "nauc_cv_recall_at_1_std": -0.09300077047421948, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": 0.12278244631185359, - "nauc_cv_recall_at_3_max": 1.0, - "nauc_cv_recall_at_3_std": 0.35807656395891085, - "nauc_cv_recall_at_5_diff1": NaN, - "nauc_cv_recall_at_5_max": NaN, - "nauc_cv_recall_at_5_std": NaN, - "nauc_map_at_1000_diff1": 0.5422695338733823, - "nauc_map_at_1000_max": 0.26261751641037917, - "nauc_map_at_1000_std": -0.12059887347484721, - "nauc_map_at_100_diff1": 0.5422695338733823, - "nauc_map_at_100_max": 0.26261751641037917, - "nauc_map_at_100_std": -0.12059887347484721, - "nauc_map_at_10_diff1": 0.5422695338733823, - "nauc_map_at_10_max": 0.26261751641037917, - "nauc_map_at_10_std": -0.12059887347484721, - "nauc_map_at_1_diff1": 0.5832036620186425, - "nauc_map_at_1_max": 0.24653664284743093, - "nauc_map_at_1_std": -0.11910624990557943, - "nauc_map_at_20_diff1": 0.5422695338733823, - "nauc_map_at_20_max": 0.26261751641037917, - "nauc_map_at_20_std": -0.12059887347484721, - "nauc_map_at_3_diff1": 0.537540514107023, - "nauc_map_at_3_max": 0.2709302784563632, - "nauc_map_at_3_std": -0.11520260342884224, - "nauc_map_at_5_diff1": 0.5422695338733823, - "nauc_map_at_5_max": 0.26261751641037917, - "nauc_map_at_5_std": -0.12059887347484721, - "nauc_mrr_at_1000_diff1": 0.5704040484258083, - "nauc_mrr_at_1000_max": 0.10875070591557207, - "nauc_mrr_at_1000_std": -0.10805140830156752, - "nauc_mrr_at_100_diff1": 0.5704040484258083, - "nauc_mrr_at_100_max": 0.10875070591557207, - "nauc_mrr_at_100_std": -0.10805140830156752, - "nauc_mrr_at_10_diff1": 0.5704040484258083, - "nauc_mrr_at_10_max": 0.10875070591557207, - "nauc_mrr_at_10_std": -0.10805140830156752, - "nauc_mrr_at_1_diff1": 0.6026921275663585, - "nauc_mrr_at_1_max": 0.1182149169851797, - "nauc_mrr_at_1_std": -0.09300077047421948, - "nauc_mrr_at_20_diff1": 0.5704040484258083, - "nauc_mrr_at_20_max": 0.10875070591557207, - "nauc_mrr_at_20_std": -0.10805140830156752, - "nauc_mrr_at_3_diff1": 0.5641474056962875, - "nauc_mrr_at_3_max": 0.1212081663904222, - "nauc_mrr_at_3_std": -0.10153609229604636, - "nauc_mrr_at_5_diff1": 0.5704040484258083, - "nauc_mrr_at_5_max": 0.10875070591557207, - "nauc_mrr_at_5_std": -0.10805140830156752, - "nauc_ndcg_at_1000_diff1": 0.5397013077011296, - "nauc_ndcg_at_1000_max": 0.26399500098098233, - "nauc_ndcg_at_1000_std": -0.12037456199431834, - "nauc_ndcg_at_100_diff1": 0.5397013077011296, - "nauc_ndcg_at_100_max": 0.26399500098098233, - "nauc_ndcg_at_100_std": -0.12037456199431834, - "nauc_ndcg_at_10_diff1": 0.5397013077011296, - "nauc_ndcg_at_10_max": 0.26399500098098233, - "nauc_ndcg_at_10_std": -0.12037456199431834, - "nauc_ndcg_at_1_diff1": 0.5832036620186425, - "nauc_ndcg_at_1_max": 0.24653664284743093, - "nauc_ndcg_at_1_std": -0.11910624990557943, - "nauc_ndcg_at_20_diff1": 0.5397013077011296, - "nauc_ndcg_at_20_max": 0.26399500098098233, - "nauc_ndcg_at_20_std": -0.12037456199431834, - "nauc_ndcg_at_3_diff1": 0.5277027131062747, - "nauc_ndcg_at_3_max": 0.28517664187290104, - "nauc_ndcg_at_3_std": -0.10660511756420861, - "nauc_ndcg_at_5_diff1": 0.5397013077011296, - "nauc_ndcg_at_5_max": 0.26399500098098233, - "nauc_ndcg_at_5_std": -0.12037456199431834, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": NaN, - "nauc_precision_at_100_max": NaN, - "nauc_precision_at_100_std": NaN, - "nauc_precision_at_10_diff1": 1.0, - "nauc_precision_at_10_max": 1.0, - "nauc_precision_at_10_std": 1.0, - "nauc_precision_at_1_diff1": 0.5832036620186425, - "nauc_precision_at_1_max": 0.24653664284743093, - "nauc_precision_at_1_std": -0.11910624990557943, - "nauc_precision_at_20_diff1": 1.0, - "nauc_precision_at_20_max": 1.0, - "nauc_precision_at_20_std": 1.0, - "nauc_precision_at_3_diff1": 0.12278244631183229, - "nauc_precision_at_3_max": 1.0, - "nauc_precision_at_3_std": 0.35807656395889226, - "nauc_precision_at_5_diff1": 1.0, - "nauc_precision_at_5_max": 1.0, - "nauc_precision_at_5_std": 1.0, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": NaN, - "nauc_recall_at_100_max": NaN, - "nauc_recall_at_100_std": NaN, - "nauc_recall_at_10_diff1": NaN, - "nauc_recall_at_10_max": NaN, - "nauc_recall_at_10_std": NaN, - "nauc_recall_at_1_diff1": 0.5832036620186425, - "nauc_recall_at_1_max": 0.24653664284743093, - "nauc_recall_at_1_std": -0.11910624990557943, - "nauc_recall_at_20_diff1": NaN, - "nauc_recall_at_20_max": NaN, - "nauc_recall_at_20_std": NaN, - "nauc_recall_at_3_diff1": 0.12278244631185359, - "nauc_recall_at_3_max": 1.0, - "nauc_recall_at_3_std": 0.35807656395891085, - "nauc_recall_at_5_diff1": NaN, - "nauc_recall_at_5_max": NaN, - "nauc_recall_at_5_std": NaN, - "ndcg_at_1": 0.61, - "ndcg_at_10": 0.83922, - "ndcg_at_100": 0.83922, - "ndcg_at_1000": 0.83922, - "ndcg_at_20": 0.83922, - "ndcg_at_3": 0.83535, - "ndcg_at_5": 0.83922, - "precision_at_1": 0.61, - "precision_at_10": 0.1, - "precision_at_100": 0.01, - "precision_at_1000": 0.001, - "precision_at_20": 0.05, - "precision_at_3": 0.33, - "precision_at_5": 0.2, - "recall_at_1": 0.61, - "recall_at_10": 1.0, - "recall_at_100": 1.0, - "recall_at_1000": 1.0, - "recall_at_20": 1.0, - "recall_at_3": 0.99, - "recall_at_5": 1.0 - } - ] - }, - "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json deleted file mode 100644 index 59efeb2991..0000000000 --- a/results-mieb/voyage-multimodal-3/1/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "86f09ebc1703516c76e5f931465e2ed7626a5e52", - "evaluation_time": 1046.7588300704956, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.56, - "cv_recall_at_10": 1.0, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 0.99, - "cv_recall_at_5": 1.0, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.82394, - "map_at_1": 0.57, - "map_at_10": 0.76317, - "map_at_100": 0.76317, - "map_at_1000": 0.76317, - "map_at_20": 0.76317, - "map_at_3": 0.75667, - "map_at_5": 0.76317, - "mrr_at_1": 0.56, - "mrr_at_10": 0.757, - "mrr_at_100": 0.757, - "mrr_at_1000": 0.757, - "mrr_at_20": 0.757, - "mrr_at_3": 0.755, - "mrr_at_5": 0.757, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": NaN, - "nauc_cv_recall_at_10_max": NaN, - "nauc_cv_recall_at_10_std": NaN, - "nauc_cv_recall_at_1_diff1": 0.6210680218033159, - "nauc_cv_recall_at_1_max": 0.17181467181467122, - "nauc_cv_recall_at_1_std": -0.34974733136497865, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": 1.0, - "nauc_cv_recall_at_3_max": 1.0, - "nauc_cv_recall_at_3_std": -1.739962651727339, - "nauc_cv_recall_at_5_diff1": NaN, - "nauc_cv_recall_at_5_max": NaN, - "nauc_cv_recall_at_5_std": NaN, - "nauc_map_at_1000_diff1": 0.5566705457911219, - "nauc_map_at_1000_max": 0.2256523009570009, - "nauc_map_at_1000_std": -0.28558097348570577, - "nauc_map_at_100_diff1": 0.5566705457911219, - "nauc_map_at_100_max": 0.2256523009570009, - "nauc_map_at_100_std": -0.28558097348570577, - "nauc_map_at_10_diff1": 0.5566705457911219, - "nauc_map_at_10_max": 0.2256523009570009, - "nauc_map_at_10_std": -0.28558097348570577, - "nauc_map_at_1_diff1": 0.591997818630617, - "nauc_map_at_1_max": 0.19318035045421283, - "nauc_map_at_1_std": -0.2656247757638381, - "nauc_map_at_20_diff1": 0.5566705457911219, - "nauc_map_at_20_max": 0.2256523009570009, - "nauc_map_at_20_std": -0.28558097348570577, - "nauc_map_at_3_diff1": 0.5472291446587567, - "nauc_map_at_3_max": 0.23894981992601236, - "nauc_map_at_3_std": -0.2930617472275123, - "nauc_map_at_5_diff1": 0.5566705457911219, - "nauc_map_at_5_max": 0.2256523009570009, - "nauc_map_at_5_std": -0.28558097348570577, - "nauc_mrr_at_1000_diff1": 0.6054587201058389, - "nauc_mrr_at_1000_max": 0.18683215614528823, - "nauc_mrr_at_1000_std": -0.3503804825054683, - "nauc_mrr_at_100_diff1": 0.6054587201058389, - "nauc_mrr_at_100_max": 0.18683215614528823, - "nauc_mrr_at_100_std": -0.3503804825054683, - "nauc_mrr_at_10_diff1": 0.6054587201058389, - "nauc_mrr_at_10_max": 0.18683215614528823, - "nauc_mrr_at_10_std": -0.3503804825054683, - "nauc_mrr_at_1_diff1": 0.6210680218033159, - "nauc_mrr_at_1_max": 0.17181467181467122, - "nauc_mrr_at_1_std": -0.34974733136497865, - "nauc_mrr_at_20_diff1": 0.6054587201058389, - "nauc_mrr_at_20_max": 0.18683215614528823, - "nauc_mrr_at_20_std": -0.3503804825054683, - "nauc_mrr_at_3_diff1": 0.609638385387044, - "nauc_mrr_at_3_max": 0.19544663979491714, - "nauc_mrr_at_3_std": -0.3651013460603219, - "nauc_mrr_at_5_diff1": 0.6054587201058389, - "nauc_mrr_at_5_max": 0.18683215614528823, - "nauc_mrr_at_5_std": -0.3503804825054683, - "nauc_ndcg_at_1000_diff1": 0.5544855949125493, - "nauc_ndcg_at_1000_max": 0.22819031621405042, - "nauc_ndcg_at_1000_std": -0.2877622655219696, - "nauc_ndcg_at_100_diff1": 0.5544855949125493, - "nauc_ndcg_at_100_max": 0.22819031621405042, - "nauc_ndcg_at_100_std": -0.2877622655219696, - "nauc_ndcg_at_10_diff1": 0.5544855949125493, - "nauc_ndcg_at_10_max": 0.22819031621405042, - "nauc_ndcg_at_10_std": -0.2877622655219696, - "nauc_ndcg_at_1_diff1": 0.591997818630617, - "nauc_ndcg_at_1_max": 0.19318035045421283, - "nauc_ndcg_at_1_std": -0.2656247757638381, - "nauc_ndcg_at_20_diff1": 0.5544855949125493, - "nauc_ndcg_at_20_max": 0.22819031621405042, - "nauc_ndcg_at_20_std": -0.2877622655219696, - "nauc_ndcg_at_3_diff1": 0.5339051421424691, - "nauc_ndcg_at_3_max": 0.26038319275348876, - "nauc_ndcg_at_3_std": -0.3106305076373389, - "nauc_ndcg_at_5_diff1": 0.5544855949125493, - "nauc_ndcg_at_5_max": 0.22819031621405042, - "nauc_ndcg_at_5_std": -0.2877622655219696, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": NaN, - "nauc_precision_at_100_max": NaN, - "nauc_precision_at_100_std": NaN, - "nauc_precision_at_10_diff1": 1.0, - "nauc_precision_at_10_max": 1.0, - "nauc_precision_at_10_std": 1.0, - "nauc_precision_at_1_diff1": 0.591997818630617, - "nauc_precision_at_1_max": 0.19318035045421283, - "nauc_precision_at_1_std": -0.2656247757638381, - "nauc_precision_at_20_diff1": 1.0, - "nauc_precision_at_20_max": 1.0, - "nauc_precision_at_20_std": 1.0, - "nauc_precision_at_3_diff1": 0.31714908185496393, - "nauc_precision_at_3_max": 0.6374105197634555, - "nauc_precision_at_3_std": -0.6305633364456867, - "nauc_precision_at_5_diff1": 1.0, - "nauc_precision_at_5_max": 1.0, - "nauc_precision_at_5_std": 1.0, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": NaN, - "nauc_recall_at_100_max": NaN, - "nauc_recall_at_100_std": NaN, - "nauc_recall_at_10_diff1": NaN, - "nauc_recall_at_10_max": NaN, - "nauc_recall_at_10_std": NaN, - "nauc_recall_at_1_diff1": 0.591997818630617, - "nauc_recall_at_1_max": 0.19318035045421283, - "nauc_recall_at_1_std": -0.2656247757638381, - "nauc_recall_at_20_diff1": NaN, - "nauc_recall_at_20_max": NaN, - "nauc_recall_at_20_std": NaN, - "nauc_recall_at_3_diff1": 0.3171490818549668, - "nauc_recall_at_3_max": 0.6374105197634592, - "nauc_recall_at_3_std": -0.6305633364456882, - "nauc_recall_at_5_diff1": NaN, - "nauc_recall_at_5_max": NaN, - "nauc_recall_at_5_std": NaN, - "ndcg_at_1": 0.57, - "ndcg_at_10": 0.82394, - "ndcg_at_100": 0.82394, - "ndcg_at_1000": 0.82394, - "ndcg_at_20": 0.82394, - "ndcg_at_3": 0.8119, - "ndcg_at_5": 0.82394, - "precision_at_1": 0.57, - "precision_at_10": 0.1, - "precision_at_100": 0.01, - "precision_at_1000": 0.001, - "precision_at_20": 0.05, - "precision_at_3": 0.32333, - "precision_at_5": 0.2, - "recall_at_1": 0.57, - "recall_at_10": 1.0, - "recall_at_100": 1.0, - "recall_at_1000": 1.0, - "recall_at_20": 1.0, - "recall_at_3": 0.97, - "recall_at_5": 1.0 - } - ] - }, - "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreTabfquadRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreTabfquadRetrieval.json deleted file mode 100644 index fb3a8f4c35..0000000000 --- a/results-mieb/voyage-multimodal-3/1/VidoreTabfquadRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "501f02a80aff50c90045b0feaa81565c4e8f889e", - "evaluation_time": 274.94982719421387, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.22857, - "cv_recall_at_10": 0.95714, - "cv_recall_at_100": 0.99286, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.97143, - "cv_recall_at_3": 0.65, - "cv_recall_at_5": 0.89643, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56361, - "map_at_1": 0.21071, - "map_at_10": 0.4602, - "map_at_100": 0.46294, - "map_at_1000": 0.463, - "map_at_20": 0.46228, - "map_at_3": 0.39286, - "map_at_5": 0.45339, - "mrr_at_1": 0.22857142857142856, - "mrr_at_10": 0.4707539682539679, - "mrr_at_100": 0.47251577376236964, - "mrr_at_1000": 0.47257600444588754, - "mrr_at_20": 0.47187420218670195, - "mrr_at_3": 0.4023809523809521, - "mrr_at_5": 0.46291666666666637, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.35807656395892135, - "nauc_cv_recall_at_100_max": 1.0, - "nauc_cv_recall_at_100_std": 1.0, - "nauc_cv_recall_at_10_diff1": -0.27170868347338767, - "nauc_cv_recall_at_10_max": 0.9564270152505449, - "nauc_cv_recall_at_10_std": 0.5902583255524414, - "nauc_cv_recall_at_1_diff1": -0.02582364764484841, - "nauc_cv_recall_at_1_max": 0.0113715808791401, - "nauc_cv_recall_at_1_std": -0.027112645285327273, - "nauc_cv_recall_at_20_diff1": -0.2777777777777773, - "nauc_cv_recall_at_20_max": 0.9673202614379085, - "nauc_cv_recall_at_20_std": 0.6473506069094338, - "nauc_cv_recall_at_3_diff1": -0.07400346418980988, - "nauc_cv_recall_at_3_max": 0.29113725399021206, - "nauc_cv_recall_at_3_std": 0.11649210094974545, - "nauc_cv_recall_at_5_diff1": -0.17426178078204416, - "nauc_cv_recall_at_5_max": 0.7885927746692977, - "nauc_cv_recall_at_5_std": 0.24504350076005102, - "nauc_map_at_1000_diff1": 0.13553846809082123, - "nauc_map_at_1000_max": 0.22199520700174338, - "nauc_map_at_1000_std": 0.05778807691192794, - "nauc_map_at_100_diff1": 0.13559236781767062, - "nauc_map_at_100_max": 0.22218364325876425, - "nauc_map_at_100_std": 0.05801628487376212, - "nauc_map_at_10_diff1": 0.1344768398789461, - "nauc_map_at_10_max": 0.22892455842544213, - "nauc_map_at_10_std": 0.060278324955635704, - "nauc_map_at_1_diff1": 0.23368144002078475, - "nauc_map_at_1_max": 0.16144318367946314, - "nauc_map_at_1_std": 0.024416417682840807, - "nauc_map_at_20_diff1": 0.13384108210812262, - "nauc_map_at_20_max": 0.22410132807912986, - "nauc_map_at_20_std": 0.05917900593894214, - "nauc_map_at_3_diff1": 0.07766973381953121, - "nauc_map_at_3_max": 0.21138604876278055, - "nauc_map_at_3_std": 0.06812727676058795, - "nauc_map_at_5_diff1": 0.13235687982693659, - "nauc_map_at_5_max": 0.2386465804104747, - "nauc_map_at_5_std": 0.05423220277094575, - "nauc_mrr_at_1000_diff1": -0.0392865823395486, - "nauc_mrr_at_1000_max": 0.10096326261753061, - "nauc_mrr_at_1000_std": 0.004056299571522236, - "nauc_mrr_at_100_diff1": -0.039192201090724105, - "nauc_mrr_at_100_max": 0.10117680081503029, - "nauc_mrr_at_100_std": 0.004292855002193332, - "nauc_mrr_at_10_diff1": -0.04146431336462882, - "nauc_mrr_at_10_max": 0.10699792280650171, - "nauc_mrr_at_10_std": 0.007644556829231765, - "nauc_mrr_at_1_diff1": -0.02582364764484841, - "nauc_mrr_at_1_max": 0.0113715808791401, - "nauc_mrr_at_1_std": -0.027112645285327273, - "nauc_mrr_at_20_diff1": -0.04028570429277086, - "nauc_mrr_at_20_max": 0.10331896246767898, - "nauc_mrr_at_20_std": 0.005529622938990385, - "nauc_mrr_at_3_diff1": -0.041454567100425964, - "nauc_mrr_at_3_max": 0.10482829405207283, - "nauc_mrr_at_3_std": 0.012387757089927653, - "nauc_mrr_at_5_diff1": -0.045969795381389594, - "nauc_mrr_at_5_max": 0.12346001607950009, - "nauc_mrr_at_5_std": 0.006284038497855093, - "nauc_ndcg_at_1000_diff1": 0.12651591808080723, - "nauc_ndcg_at_1000_max": 0.2367262598374307, - "nauc_ndcg_at_1000_std": 0.06640769582000103, - "nauc_ndcg_at_100_diff1": 0.12774994597272604, - "nauc_ndcg_at_100_max": 0.24079388143590194, - "nauc_ndcg_at_100_std": 0.07138297535711573, - "nauc_ndcg_at_10_diff1": 0.11834485636734318, - "nauc_ndcg_at_10_max": 0.2740394382500109, - "nauc_ndcg_at_10_std": 0.08438885306140487, - "nauc_ndcg_at_1_diff1": 0.23368144002078475, - "nauc_ndcg_at_1_max": 0.16144318367946314, - "nauc_ndcg_at_1_std": 0.024416417682840807, - "nauc_ndcg_at_20_diff1": 0.11455084278957535, - "nauc_ndcg_at_20_max": 0.2558954677030979, - "nauc_ndcg_at_20_std": 0.0809048502571133, - "nauc_ndcg_at_3_diff1": 0.01996065575006399, - "nauc_ndcg_at_3_max": 0.2368828569659345, - "nauc_ndcg_at_3_std": 0.08883647180660824, - "nauc_ndcg_at_5_diff1": 0.11396014515287547, - "nauc_ndcg_at_5_max": 0.29797120024144946, - "nauc_ndcg_at_5_std": 0.06725907831809537, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.35807656395888193, - "nauc_precision_at_100_max": 1.0, - "nauc_precision_at_100_std": 1.0, - "nauc_precision_at_10_diff1": -0.03588546529723173, - "nauc_precision_at_10_max": 0.892623716153129, - "nauc_precision_at_10_std": 0.43106131341425424, - "nauc_precision_at_1_diff1": 0.23368144002078475, - "nauc_precision_at_1_max": 0.16144318367946314, - "nauc_precision_at_1_std": 0.024416417682840807, - "nauc_precision_at_20_diff1": -0.277777777777766, - "nauc_precision_at_20_max": 0.9673202614379075, - "nauc_precision_at_20_std": 0.647350606909435, - "nauc_precision_at_3_diff1": -0.15931785685223693, - "nauc_precision_at_3_max": 0.31803703660648913, - "nauc_precision_at_3_std": 0.15442962928519124, - "nauc_precision_at_5_diff1": 0.011255215239820097, - "nauc_precision_at_5_max": 0.7531129726058416, - "nauc_precision_at_5_std": 0.15925482712894967, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.35807656395892135, - "nauc_recall_at_100_max": 1.0, - "nauc_recall_at_100_std": 1.0, - "nauc_recall_at_10_diff1": -0.035885465297229795, - "nauc_recall_at_10_max": 0.8926237161531256, - "nauc_recall_at_10_std": 0.43106131341425596, - "nauc_recall_at_1_diff1": 0.23368144002078475, - "nauc_recall_at_1_max": 0.16144318367946314, - "nauc_recall_at_1_std": 0.024416417682840807, - "nauc_recall_at_20_diff1": -0.2777777777777773, - "nauc_recall_at_20_max": 0.9673202614379085, - "nauc_recall_at_20_std": 0.6473506069094338, - "nauc_recall_at_3_diff1": -0.15931785685223723, - "nauc_recall_at_3_max": 0.31803703660648813, - "nauc_recall_at_3_std": 0.15442962928519088, - "nauc_recall_at_5_diff1": 0.011255215239821375, - "nauc_recall_at_5_max": 0.7531129726058414, - "nauc_recall_at_5_std": 0.15925482712894984, - "ndcg_at_1": 0.21071, - "ndcg_at_10": 0.57991, - "ndcg_at_100": 0.59086, - "ndcg_at_1000": 0.59189, - "ndcg_at_20": 0.58666, - "ndcg_at_3": 0.45841, - "ndcg_at_5": 0.56361, - "precision_at_1": 0.21071, - "precision_at_10": 0.09464, - "precision_at_100": 0.00993, - "precision_at_1000": 0.001, - "precision_at_20": 0.04857, - "precision_at_3": 0.21667, - "precision_at_5": 0.17929, - "recall_at_1": 0.21071, - "recall_at_10": 0.94643, - "recall_at_100": 0.99286, - "recall_at_1000": 1.0, - "recall_at_20": 0.97143, - "recall_at_3": 0.65, - "recall_at_5": 0.89643 - } - ] - }, - "task_name": "VidoreTabfquadRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VidoreTatdqaRetrieval.json b/results-mieb/voyage-multimodal-3/1/VidoreTatdqaRetrieval.json deleted file mode 100644 index 37adcd3e71..0000000000 --- a/results-mieb/voyage-multimodal-3/1/VidoreTatdqaRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "9c3a626c16c811f15514689c3e7e95a4f2b9b8c3", - "evaluation_time": 1544.8007860183716, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.09321, - "cv_recall_at_10": 0.635, - "cv_recall_at_100": 0.94227, - "cv_recall_at_1000": 0.99579, - "cv_recall_at_20": 0.75646, - "cv_recall_at_3": 0.28443, - "cv_recall_at_5": 0.46603, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.27633, - "map_at_1": 0.09501, - "map_at_10": 0.23984, - "map_at_100": 0.25355, - "map_at_1000": 0.25382, - "map_at_20": 0.24829, - "map_at_3": 0.17619, - "map_at_5": 0.21563, - "mrr_at_1": 0.09320505111244738, - "mrr_at_10": 0.2381229180387323, - "mrr_at_100": 0.25191031872372854, - "mrr_at_1000": 0.252176402965878, - "mrr_at_20": 0.246719851990479, - "mrr_at_3": 0.1733814391661656, - "mrr_at_5": 0.2141511324914806, - "nauc_cv_recall_at_1000_diff1": 0.31050486909740643, - "nauc_cv_recall_at_1000_max": 0.5565879699515275, - "nauc_cv_recall_at_1000_std": 0.7932362042056588, - "nauc_cv_recall_at_100_diff1": 0.017229634643007606, - "nauc_cv_recall_at_100_max": 0.7245328143477694, - "nauc_cv_recall_at_100_std": 0.554035254223751, - "nauc_cv_recall_at_10_diff1": 0.032786105058676726, - "nauc_cv_recall_at_10_max": 0.4898125717664449, - "nauc_cv_recall_at_10_std": -0.04286901530782105, - "nauc_cv_recall_at_1_diff1": -0.030595244757954252, - "nauc_cv_recall_at_1_max": 0.17229432194888586, - "nauc_cv_recall_at_1_std": -0.022750385688563874, - "nauc_cv_recall_at_20_diff1": 0.042499064344320193, - "nauc_cv_recall_at_20_max": 0.5315043626694177, - "nauc_cv_recall_at_20_std": -0.01091688306662299, - "nauc_cv_recall_at_3_diff1": -0.008924384137994422, - "nauc_cv_recall_at_3_max": 0.3078286274347898, - "nauc_cv_recall_at_3_std": 0.017083055748262896, - "nauc_cv_recall_at_5_diff1": 0.024757025565364448, - "nauc_cv_recall_at_5_max": 0.4073620550301354, - "nauc_cv_recall_at_5_std": -0.06215327720340901, - "nauc_map_at_1000_diff1": -0.0003925171649634412, - "nauc_map_at_1000_max": 0.2898040228378256, - "nauc_map_at_1000_std": -0.011252721627844083, - "nauc_map_at_100_diff1": -0.000347784297757475, - "nauc_map_at_100_max": 0.2902793635938703, - "nauc_map_at_100_std": -0.010735880636550706, - "nauc_map_at_10_diff1": -0.0002285351889445233, - "nauc_map_at_10_max": 0.2959599078926981, - "nauc_map_at_10_std": -0.017859607333720634, - "nauc_map_at_1_diff1": -0.027194095074582824, - "nauc_map_at_1_max": 0.1916225301721836, - "nauc_map_at_1_std": -0.001399713359798536, - "nauc_map_at_20_diff1": 0.0017744924766137128, - "nauc_map_at_20_max": 0.29305441547884215, - "nauc_map_at_20_std": -0.015289326952853316, - "nauc_map_at_3_diff1": 0.001522630982425383, - "nauc_map_at_3_max": 0.24227359001382884, - "nauc_map_at_3_std": -0.0027546423271664284, - "nauc_map_at_5_diff1": -0.001963766600180219, - "nauc_map_at_5_max": 0.281071243656603, - "nauc_map_at_5_std": -0.019853852764696617, - "nauc_mrr_at_1000_diff1": -0.008039463989689176, - "nauc_mrr_at_1000_max": 0.28401646367520217, - "nauc_mrr_at_1000_std": -0.013541290133244552, - "nauc_mrr_at_100_diff1": -0.00799442245722196, - "nauc_mrr_at_100_max": 0.2845070670650448, - "nauc_mrr_at_100_std": -0.01303344586909849, - "nauc_mrr_at_10_diff1": -0.00614564411167257, - "nauc_mrr_at_10_max": 0.289135604430979, - "nauc_mrr_at_10_std": -0.019937260545786917, - "nauc_mrr_at_1_diff1": -0.030595244757954252, - "nauc_mrr_at_1_max": 0.17229432194888586, - "nauc_mrr_at_1_std": -0.022750385688563874, - "nauc_mrr_at_20_diff1": -0.006644806046490563, - "nauc_mrr_at_20_max": 0.28733189413446436, - "nauc_mrr_at_20_std": -0.01754671101514299, - "nauc_mrr_at_3_diff1": -0.017339059829432638, - "nauc_mrr_at_3_max": 0.24425253352221082, - "nauc_mrr_at_3_std": 0.0013361903711106577, - "nauc_mrr_at_5_diff1": -0.007130039960877325, - "nauc_mrr_at_5_max": 0.2757222346163175, - "nauc_mrr_at_5_std": -0.023769414978505286, - "nauc_ndcg_at_1000_diff1": 0.003468485965427646, - "nauc_ndcg_at_1000_max": 0.31731576935974565, - "nauc_ndcg_at_1000_std": -0.0024293118052454063, - "nauc_ndcg_at_100_diff1": 0.0035673138731956147, - "nauc_ndcg_at_100_max": 0.3309317704266328, - "nauc_ndcg_at_100_std": 0.014880579527387122, - "nauc_ndcg_at_10_diff1": 0.005795685643303614, - "nauc_ndcg_at_10_max": 0.3528635747427016, - "nauc_ndcg_at_10_std": -0.02502551479265232, - "nauc_ndcg_at_1_diff1": -0.027194095074582824, - "nauc_ndcg_at_1_max": 0.1916225301721836, - "nauc_ndcg_at_1_std": -0.001399713359798536, - "nauc_ndcg_at_20_diff1": 0.014598223077526385, - "nauc_ndcg_at_20_max": 0.34721854443722916, - "nauc_ndcg_at_20_std": -0.016046354479508872, - "nauc_ndcg_at_3_diff1": 0.007765400458555541, - "nauc_ndcg_at_3_max": 0.2575186815309078, - "nauc_ndcg_at_3_std": -0.002035068546740743, - "nauc_ndcg_at_5_diff1": 0.002649179955746284, - "nauc_ndcg_at_5_max": 0.321154170908288, - "nauc_ndcg_at_5_std": -0.030304143245058396, - "nauc_precision_at_1000_diff1": 0.3105048690973943, - "nauc_precision_at_1000_max": 0.556587969951549, - "nauc_precision_at_1000_std": 0.7932362042056358, - "nauc_precision_at_100_diff1": 0.020743590857352306, - "nauc_precision_at_100_max": 0.7187339631723969, - "nauc_precision_at_100_std": 0.5557689545945156, - "nauc_precision_at_10_diff1": 0.02015203520063033, - "nauc_precision_at_10_max": 0.5039818563028681, - "nauc_precision_at_10_std": -0.04236193373091959, - "nauc_precision_at_1_diff1": -0.027194095074582824, - "nauc_precision_at_1_max": 0.1916225301721836, - "nauc_precision_at_1_std": -0.001399713359798536, - "nauc_precision_at_20_diff1": 0.06509303733464025, - "nauc_precision_at_20_max": 0.5326931548142373, - "nauc_precision_at_20_std": -0.009687706719512464, - "nauc_precision_at_3_diff1": 0.020858108362361155, - "nauc_precision_at_3_max": 0.2904392930383802, - "nauc_precision_at_3_std": -0.00029894316449353845, - "nauc_precision_at_5_diff1": 0.011899104360575728, - "nauc_precision_at_5_max": 0.410610008372957, - "nauc_precision_at_5_std": -0.05446107727275674, - "nauc_recall_at_1000_diff1": 0.31050486909740643, - "nauc_recall_at_1000_max": 0.5565879699515275, - "nauc_recall_at_1000_std": 0.7932362042056588, - "nauc_recall_at_100_diff1": 0.020743590857350565, - "nauc_recall_at_100_max": 0.7187339631723954, - "nauc_recall_at_100_std": 0.5557689545945156, - "nauc_recall_at_10_diff1": 0.020152035200631094, - "nauc_recall_at_10_max": 0.5039818563028683, - "nauc_recall_at_10_std": -0.042361933730918684, - "nauc_recall_at_1_diff1": -0.027194095074582824, - "nauc_recall_at_1_max": 0.1916225301721836, - "nauc_recall_at_1_std": -0.001399713359798536, - "nauc_recall_at_20_diff1": 0.06509303733464074, - "nauc_recall_at_20_max": 0.5326931548142366, - "nauc_recall_at_20_std": -0.009687706719513361, - "nauc_recall_at_3_diff1": 0.02085810836236144, - "nauc_recall_at_3_max": 0.2904392930383805, - "nauc_recall_at_3_std": -0.0002989431644934902, - "nauc_recall_at_5_diff1": 0.011899104360575681, - "nauc_recall_at_5_max": 0.41061000837295714, - "nauc_recall_at_5_std": -0.05446107727275678, - "ndcg_at_1": 0.09501, - "ndcg_at_10": 0.33305, - "ndcg_at_100": 0.39919, - "ndcg_at_1000": 0.40611, - "ndcg_at_20": 0.36358, - "ndcg_at_3": 0.2046, - "ndcg_at_5": 0.27633, - "precision_at_1": 0.09501, - "precision_at_10": 0.06344, - "precision_at_100": 0.00942, - "precision_at_1000": 0.001, - "precision_at_20": 0.03773, - "precision_at_3": 0.09581, - "precision_at_5": 0.0926, - "recall_at_1": 0.09501, - "recall_at_10": 0.6344, - "recall_at_100": 0.94167, - "recall_at_1000": 0.99579, - "recall_at_20": 0.75466, - "recall_at_3": 0.28743, - "recall_at_5": 0.46302 - } - ] - }, - "task_name": "VidoreTatdqaRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/VisualNewsI2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/VisualNewsI2TRetrieval.json deleted file mode 100644 index 6dd46a1e7a..0000000000 --- a/results-mieb/voyage-multimodal-3/1/VisualNewsI2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "aaee58895a66e4d619168849267ed2bb40d37043", - "evaluation_time": 6107.169147491455, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.06825, - "cv_recall_at_10": 0.19545, - "cv_recall_at_100": 0.39875, - "cv_recall_at_1000": 0.6308, - "cv_recall_at_20": 0.2496, - "cv_recall_at_3": 0.1178, - "cv_recall_at_5": 0.1496, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.12474, - "map_at_1": 0.06825, - "map_at_10": 0.103, - "map_at_100": 0.11027, - "map_at_1000": 0.11118, - "map_at_20": 0.10672, - "map_at_3": 0.0897, - "map_at_5": 0.09691, - "mrr_at_1": 0.06825, - "mrr_at_10": 0.10299886904761851, - "mrr_at_100": 0.11027488430144886, - "mrr_at_1000": 0.11117591475965252, - "mrr_at_20": 0.10671955475724743, - "mrr_at_3": 0.0896999999999995, - "mrr_at_5": 0.09691249999999985, - "nauc_cv_recall_at_1000_diff1": 0.07071177533627818, - "nauc_cv_recall_at_1000_max": 0.26562236167072856, - "nauc_cv_recall_at_1000_std": 0.3641313388165596, - "nauc_cv_recall_at_100_diff1": 0.0879483088139398, - "nauc_cv_recall_at_100_max": 0.19499859114790075, - "nauc_cv_recall_at_100_std": 0.24543736404577246, - "nauc_cv_recall_at_10_diff1": 0.14309800392206096, - "nauc_cv_recall_at_10_max": 0.18275171223289868, - "nauc_cv_recall_at_10_std": 0.14117281838957083, - "nauc_cv_recall_at_1_diff1": 0.3392937980541951, - "nauc_cv_recall_at_1_max": 0.2444093246883695, - "nauc_cv_recall_at_1_std": 0.060403490615457335, - "nauc_cv_recall_at_20_diff1": 0.12719265702804436, - "nauc_cv_recall_at_20_max": 0.18740195316364447, - "nauc_cv_recall_at_20_std": 0.17863831788138756, - "nauc_cv_recall_at_3_diff1": 0.20773411557883753, - "nauc_cv_recall_at_3_max": 0.20086790461930343, - "nauc_cv_recall_at_3_std": 0.08982097516973798, - "nauc_cv_recall_at_5_diff1": 0.17471486468798317, - "nauc_cv_recall_at_5_max": 0.19173329789004723, - "nauc_cv_recall_at_5_std": 0.11246321681756993, - "nauc_map_at_1000_diff1": 0.23557151715228627, - "nauc_map_at_1000_max": 0.21360085336214968, - "nauc_map_at_1000_std": 0.10179157596177511, - "nauc_map_at_100_diff1": 0.23558150842830375, - "nauc_map_at_100_max": 0.2131076975340969, - "nauc_map_at_100_std": 0.10101384389492717, - "nauc_map_at_10_diff1": 0.24205448446889496, - "nauc_map_at_10_max": 0.21288563308201747, - "nauc_map_at_10_std": 0.09145715995583001, - "nauc_map_at_1_diff1": 0.3392937980541951, - "nauc_map_at_1_max": 0.2444093246883695, - "nauc_map_at_1_std": 0.060403490615457335, - "nauc_map_at_20_diff1": 0.23871992868144357, - "nauc_map_at_20_max": 0.21330652626665822, - "nauc_map_at_20_std": 0.09696116661274039, - "nauc_map_at_3_diff1": 0.2657615794348174, - "nauc_map_at_3_max": 0.21969159446682562, - "nauc_map_at_3_std": 0.07592966888480318, - "nauc_map_at_5_diff1": 0.2523244135153835, - "nauc_map_at_5_max": 0.21603781978533362, - "nauc_map_at_5_std": 0.08404111409954902, - "nauc_mrr_at_1000_diff1": 0.2355715148963118, - "nauc_mrr_at_1000_max": 0.21360085206378832, - "nauc_mrr_at_1000_std": 0.10179157561911444, - "nauc_mrr_at_100_diff1": 0.23558150842830375, - "nauc_mrr_at_100_max": 0.2131076975340969, - "nauc_mrr_at_100_std": 0.10101384389492717, - "nauc_mrr_at_10_diff1": 0.24205448446889496, - "nauc_mrr_at_10_max": 0.21288563308201747, - "nauc_mrr_at_10_std": 0.09145715995583001, - "nauc_mrr_at_1_diff1": 0.3392937980541951, - "nauc_mrr_at_1_max": 0.2444093246883695, - "nauc_mrr_at_1_std": 0.060403490615457335, - "nauc_mrr_at_20_diff1": 0.23871992868144357, - "nauc_mrr_at_20_max": 0.21330652626665822, - "nauc_mrr_at_20_std": 0.09696116661274039, - "nauc_mrr_at_3_diff1": 0.2657615794348174, - "nauc_mrr_at_3_max": 0.21969159446682562, - "nauc_mrr_at_3_std": 0.07592966888480318, - "nauc_mrr_at_5_diff1": 0.2523244135153835, - "nauc_mrr_at_5_max": 0.21603781978533362, - "nauc_mrr_at_5_std": 0.08404111409954902, - "nauc_ndcg_at_1000_diff1": 0.18371223467656492, - "nauc_ndcg_at_1000_max": 0.21760781116915476, - "nauc_ndcg_at_1000_std": 0.1651354537417224, - "nauc_ndcg_at_100_diff1": 0.18351509037449174, - "nauc_ndcg_at_100_max": 0.2047563926427989, - "nauc_ndcg_at_100_std": 0.14708224364063993, - "nauc_ndcg_at_10_diff1": 0.20798943664064767, - "nauc_ndcg_at_10_max": 0.2024610973940482, - "nauc_ndcg_at_10_std": 0.10797693655625268, - "nauc_ndcg_at_1_diff1": 0.3392937980541951, - "nauc_ndcg_at_1_max": 0.2444093246883695, - "nauc_ndcg_at_1_std": 0.060403490615457335, - "nauc_ndcg_at_20_diff1": 0.199135794702939, - "nauc_ndcg_at_20_max": 0.2037165138514619, - "nauc_ndcg_at_20_std": 0.12405620210520404, - "nauc_ndcg_at_3_diff1": 0.24804200545089336, - "nauc_ndcg_at_3_max": 0.2139285891161409, - "nauc_ndcg_at_3_std": 0.08013578418891945, - "nauc_ndcg_at_5_diff1": 0.22734280862143041, - "nauc_ndcg_at_5_max": 0.2082046891841067, - "nauc_ndcg_at_5_std": 0.09300248180293258, - "nauc_precision_at_1000_diff1": 0.07071177533627745, - "nauc_precision_at_1000_max": 0.2656223616707277, - "nauc_precision_at_1000_std": 0.364131338816559, - "nauc_precision_at_100_diff1": 0.08794830881393986, - "nauc_precision_at_100_max": 0.19499859114790064, - "nauc_precision_at_100_std": 0.2454373640457725, - "nauc_precision_at_10_diff1": 0.1430980039220609, - "nauc_precision_at_10_max": 0.18275171223289863, - "nauc_precision_at_10_std": 0.1411728183895707, - "nauc_precision_at_1_diff1": 0.3392937980541951, - "nauc_precision_at_1_max": 0.2444093246883695, - "nauc_precision_at_1_std": 0.060403490615457335, - "nauc_precision_at_20_diff1": 0.1271926570280442, - "nauc_precision_at_20_max": 0.18740195316364444, - "nauc_precision_at_20_std": 0.17863831788138737, - "nauc_precision_at_3_diff1": 0.2077341155788376, - "nauc_precision_at_3_max": 0.2008679046193036, - "nauc_precision_at_3_std": 0.08982097516973812, - "nauc_precision_at_5_diff1": 0.1747148646879832, - "nauc_precision_at_5_max": 0.1917332978900472, - "nauc_precision_at_5_std": 0.11246321681756989, - "nauc_recall_at_1000_diff1": 0.07071177533627818, - "nauc_recall_at_1000_max": 0.26562236167072856, - "nauc_recall_at_1000_std": 0.3641313388165596, - "nauc_recall_at_100_diff1": 0.0879483088139398, - "nauc_recall_at_100_max": 0.19499859114790075, - "nauc_recall_at_100_std": 0.24543736404577246, - "nauc_recall_at_10_diff1": 0.14309800392206096, - "nauc_recall_at_10_max": 0.18275171223289868, - "nauc_recall_at_10_std": 0.14117281838957083, - "nauc_recall_at_1_diff1": 0.3392937980541951, - "nauc_recall_at_1_max": 0.2444093246883695, - "nauc_recall_at_1_std": 0.060403490615457335, - "nauc_recall_at_20_diff1": 0.12719265702804436, - "nauc_recall_at_20_max": 0.18740195316364447, - "nauc_recall_at_20_std": 0.17863831788138756, - "nauc_recall_at_3_diff1": 0.20773411557883753, - "nauc_recall_at_3_max": 0.20086790461930343, - "nauc_recall_at_3_std": 0.08982097516973798, - "nauc_recall_at_5_diff1": 0.17471486468798317, - "nauc_recall_at_5_max": 0.19173329789004723, - "nauc_recall_at_5_std": 0.11246321681756993, - "ndcg_at_1": 0.06825, - "ndcg_at_10": 0.12474, - "ndcg_at_100": 0.16551, - "ndcg_at_1000": 0.19379, - "ndcg_at_20": 0.13839, - "ndcg_at_3": 0.0969, - "ndcg_at_5": 0.10995, - "precision_at_1": 0.06825, - "precision_at_10": 0.01955, - "precision_at_100": 0.00399, - "precision_at_1000": 0.00063, - "precision_at_20": 0.01248, - "precision_at_3": 0.03927, - "precision_at_5": 0.02992, - "recall_at_1": 0.06825, - "recall_at_10": 0.19545, - "recall_at_100": 0.39875, - "recall_at_1000": 0.6308, - "recall_at_20": 0.2496, - "recall_at_3": 0.1178, - "recall_at_5": 0.1496 - } - ] - }, - "task_name": "VisualNewsI2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/WITT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/WITT2IRetrieval.json deleted file mode 100644 index c68c3a9c67..0000000000 --- a/results-mieb/voyage-multimodal-3/1/WITT2IRetrieval.json +++ /dev/null @@ -1,1936 +0,0 @@ -{ - "dataset_revision": "91ac153f1371a98b209ed763205e25e115ecd06e", - "evaluation_time": 1137.1250095367432, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.39775, - "cv_recall_at_10": 0.68764, - "cv_recall_at_100": 0.90337, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.76067, - "cv_recall_at_3": 0.53708, - "cv_recall_at_5": 0.59775, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.53332, - "map_at_1": 0.39775, - "map_at_10": 0.48513, - "map_at_100": 0.49387, - "map_at_1000": 0.49438, - "map_at_20": 0.49008, - "map_at_3": 0.45936, - "map_at_5": 0.47307, - "mrr_at_1": 0.39775280898876403, - "mrr_at_10": 0.4851252898163007, - "mrr_at_100": 0.4938677598278257, - "mrr_at_1000": 0.4943758260759537, - "mrr_at_20": 0.49007998369444744, - "mrr_at_3": 0.4593632958801497, - "mrr_at_5": 0.47307116104868874, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5000271426400019, - "nauc_cv_recall_at_100_max": 0.33645473693353406, - "nauc_cv_recall_at_100_std": 0.2653898768809851, - "nauc_cv_recall_at_10_diff1": 0.5174089324309398, - "nauc_cv_recall_at_10_max": 0.3901568933693027, - "nauc_cv_recall_at_10_std": 0.14601968857967695, - "nauc_cv_recall_at_1_diff1": 0.6433933086018607, - "nauc_cv_recall_at_1_max": 0.4016098624710042, - "nauc_cv_recall_at_1_std": 0.041856713841805254, - "nauc_cv_recall_at_20_diff1": 0.5004080379828553, - "nauc_cv_recall_at_20_max": 0.40080048526069356, - "nauc_cv_recall_at_20_std": 0.21059144801108873, - "nauc_cv_recall_at_3_diff1": 0.5769084735634901, - "nauc_cv_recall_at_3_max": 0.44111111456910174, - "nauc_cv_recall_at_3_std": 0.1138455170439162, - "nauc_cv_recall_at_5_diff1": 0.5616616989567813, - "nauc_cv_recall_at_5_max": 0.4342382770917608, - "nauc_cv_recall_at_5_std": 0.14279892750691942, - "nauc_map_at_1000_diff1": 0.6062730714611361, - "nauc_map_at_1000_max": 0.4110539637034493, - "nauc_map_at_1000_std": 0.0791359300689424, - "nauc_map_at_100_diff1": 0.6061385539345461, - "nauc_map_at_100_max": 0.4110626956856239, - "nauc_map_at_100_std": 0.07934859334144191, - "nauc_map_at_10_diff1": 0.6047402412710131, - "nauc_map_at_10_max": 0.4099565332479481, - "nauc_map_at_10_std": 0.077238302183022, - "nauc_map_at_1_diff1": 0.6433933086018607, - "nauc_map_at_1_max": 0.4016098624710042, - "nauc_map_at_1_std": 0.041856713841805254, - "nauc_map_at_20_diff1": 0.6051415598634673, - "nauc_map_at_20_max": 0.41124616862006846, - "nauc_map_at_20_std": 0.08010265866099041, - "nauc_map_at_3_diff1": 0.6108768857240254, - "nauc_map_at_3_max": 0.41615469088166257, - "nauc_map_at_3_std": 0.07301219530664803, - "nauc_map_at_5_diff1": 0.6082690683739146, - "nauc_map_at_5_max": 0.41430708987714243, - "nauc_map_at_5_std": 0.07783192900428913, - "nauc_mrr_at_1000_diff1": 0.6062730714611361, - "nauc_mrr_at_1000_max": 0.4110539637034493, - "nauc_mrr_at_1000_std": 0.0791359300689424, - "nauc_mrr_at_100_diff1": 0.6061385539345461, - "nauc_mrr_at_100_max": 0.4110626956856239, - "nauc_mrr_at_100_std": 0.07934859334144191, - "nauc_mrr_at_10_diff1": 0.6047402412710131, - "nauc_mrr_at_10_max": 0.4099565332479481, - "nauc_mrr_at_10_std": 0.077238302183022, - "nauc_mrr_at_1_diff1": 0.6433933086018607, - "nauc_mrr_at_1_max": 0.4016098624710042, - "nauc_mrr_at_1_std": 0.041856713841805254, - "nauc_mrr_at_20_diff1": 0.6051415598634673, - "nauc_mrr_at_20_max": 0.41124616862006846, - "nauc_mrr_at_20_std": 0.08010265866099041, - "nauc_mrr_at_3_diff1": 0.6108768857240254, - "nauc_mrr_at_3_max": 0.41615469088166257, - "nauc_mrr_at_3_std": 0.07301219530664803, - "nauc_mrr_at_5_diff1": 0.6082690683739146, - "nauc_mrr_at_5_max": 0.41430708987714243, - "nauc_mrr_at_5_std": 0.07783192900428913, - "nauc_ndcg_at_1000_diff1": 0.5985728701025478, - "nauc_ndcg_at_1000_max": 0.4105306381695453, - "nauc_ndcg_at_1000_std": 0.08997211615828361, - "nauc_ndcg_at_100_diff1": 0.5943985687790737, - "nauc_ndcg_at_100_max": 0.40799386515039965, - "nauc_ndcg_at_100_std": 0.09701143431709665, - "nauc_ndcg_at_10_diff1": 0.5873207409203248, - "nauc_ndcg_at_10_max": 0.4072659270468375, - "nauc_ndcg_at_10_std": 0.0916668746640074, - "nauc_ndcg_at_1_diff1": 0.6433933086018607, - "nauc_ndcg_at_1_max": 0.4016098624710042, - "nauc_ndcg_at_1_std": 0.041856713841805254, - "nauc_ndcg_at_20_diff1": 0.5875622071363173, - "nauc_ndcg_at_20_max": 0.4106690124705196, - "nauc_ndcg_at_20_std": 0.10170412297234428, - "nauc_ndcg_at_3_diff1": 0.6022457729043741, - "nauc_ndcg_at_3_max": 0.42226539179035844, - "nauc_ndcg_at_3_std": 0.08319193818224027, - "nauc_ndcg_at_5_diff1": 0.5971440686463333, - "nauc_ndcg_at_5_max": 0.41907996722877244, - "nauc_ndcg_at_5_std": 0.09291896572761014, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.5000271426400036, - "nauc_precision_at_100_max": 0.33645473693353495, - "nauc_precision_at_100_std": 0.26538987688098664, - "nauc_precision_at_10_diff1": 0.5174089324309394, - "nauc_precision_at_10_max": 0.3901568933693025, - "nauc_precision_at_10_std": 0.14601968857967695, - "nauc_precision_at_1_diff1": 0.6433933086018607, - "nauc_precision_at_1_max": 0.4016098624710042, - "nauc_precision_at_1_std": 0.041856713841805254, - "nauc_precision_at_20_diff1": 0.5004080379828557, - "nauc_precision_at_20_max": 0.4008004852606927, - "nauc_precision_at_20_std": 0.21059144801108778, - "nauc_precision_at_3_diff1": 0.5769084735634901, - "nauc_precision_at_3_max": 0.4411111145691011, - "nauc_precision_at_3_std": 0.11384551704391571, - "nauc_precision_at_5_diff1": 0.5616616989567803, - "nauc_precision_at_5_max": 0.4342382770917606, - "nauc_precision_at_5_std": 0.14279892750691883, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5000271426400019, - "nauc_recall_at_100_max": 0.33645473693353406, - "nauc_recall_at_100_std": 0.2653898768809851, - "nauc_recall_at_10_diff1": 0.5174089324309398, - "nauc_recall_at_10_max": 0.3901568933693027, - "nauc_recall_at_10_std": 0.14601968857967695, - "nauc_recall_at_1_diff1": 0.6433933086018607, - "nauc_recall_at_1_max": 0.4016098624710042, - "nauc_recall_at_1_std": 0.041856713841805254, - "nauc_recall_at_20_diff1": 0.5004080379828553, - "nauc_recall_at_20_max": 0.40080048526069356, - "nauc_recall_at_20_std": 0.21059144801108873, - "nauc_recall_at_3_diff1": 0.5769084735634901, - "nauc_recall_at_3_max": 0.44111111456910174, - "nauc_recall_at_3_std": 0.1138455170439162, - "nauc_recall_at_5_diff1": 0.5616616989567813, - "nauc_recall_at_5_max": 0.4342382770917608, - "nauc_recall_at_5_std": 0.14279892750691942, - "ndcg_at_1": 0.39775, - "ndcg_at_10": 0.53332, - "ndcg_at_100": 0.57838, - "ndcg_at_1000": 0.59093, - "ndcg_at_20": 0.55163, - "ndcg_at_3": 0.47933, - "ndcg_at_5": 0.50418, - "precision_at_1": 0.39775, - "precision_at_10": 0.06876, - "precision_at_100": 0.00903, - "precision_at_1000": 0.001, - "precision_at_20": 0.03803, - "precision_at_3": 0.17903, - "precision_at_5": 0.11955, - "recall_at_1": 0.39775, - "recall_at_10": 0.68764, - "recall_at_100": 0.90337, - "recall_at_1000": 1.0, - "recall_at_20": 0.76067, - "recall_at_3": 0.53708, - "recall_at_5": 0.59775 - }, - { - "cv_recall_at_1": 0.32209, - "cv_recall_at_10": 0.6, - "cv_recall_at_100": 0.87558, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.68953, - "cv_recall_at_3": 0.4407, - "cv_recall_at_5": 0.5093, - "hf_subset": "bg", - "languages": [ - "bul-Cyrl" - ], - "main_score": 0.44843, - "map_at_1": 0.32209, - "map_at_10": 0.4015, - "map_at_100": 0.41219, - "map_at_1000": 0.41284, - "map_at_20": 0.40769, - "map_at_3": 0.37403, - "map_at_5": 0.38961, - "mrr_at_1": 0.32209302325581396, - "mrr_at_10": 0.4015014765596159, - "mrr_at_100": 0.41219135100183496, - "mrr_at_1000": 0.41283779969855594, - "mrr_at_20": 0.4076860035219613, - "mrr_at_3": 0.3740310077519379, - "mrr_at_5": 0.3896124031007749, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.4348271939958975, - "nauc_cv_recall_at_100_max": 0.5359090786963637, - "nauc_cv_recall_at_100_std": 0.24929023706529907, - "nauc_cv_recall_at_10_diff1": 0.4647400719280445, - "nauc_cv_recall_at_10_max": 0.46673155377000985, - "nauc_cv_recall_at_10_std": -0.014356271614653547, - "nauc_cv_recall_at_1_diff1": 0.6469130123286798, - "nauc_cv_recall_at_1_max": 0.4996837405335848, - "nauc_cv_recall_at_1_std": 0.021401518977899162, - "nauc_cv_recall_at_20_diff1": 0.46356765129520633, - "nauc_cv_recall_at_20_max": 0.44654751684700356, - "nauc_cv_recall_at_20_std": -0.04457976228770894, - "nauc_cv_recall_at_3_diff1": 0.5278920774752104, - "nauc_cv_recall_at_3_max": 0.47376992697805126, - "nauc_cv_recall_at_3_std": -0.02027797214207337, - "nauc_cv_recall_at_5_diff1": 0.5037572697381799, - "nauc_cv_recall_at_5_max": 0.4908852578000109, - "nauc_cv_recall_at_5_std": 0.001618973430065162, - "nauc_map_at_1000_diff1": 0.5876636565567583, - "nauc_map_at_1000_max": 0.4868422238450373, - "nauc_map_at_1000_std": 0.006172855129180589, - "nauc_map_at_100_diff1": 0.5872173827802959, - "nauc_map_at_100_max": 0.48681846044840055, - "nauc_map_at_100_std": 0.006174527620908233, - "nauc_map_at_10_diff1": 0.5843873762953701, - "nauc_map_at_10_max": 0.48569620079470033, - "nauc_map_at_10_std": 0.0048121712540381174, - "nauc_map_at_1_diff1": 0.6469130123286798, - "nauc_map_at_1_max": 0.4996837405335848, - "nauc_map_at_1_std": 0.021401518977899162, - "nauc_map_at_20_diff1": 0.5860522812282696, - "nauc_map_at_20_max": 0.4853116062444229, - "nauc_map_at_20_std": 0.003802028482319083, - "nauc_map_at_3_diff1": 0.5930592769877978, - "nauc_map_at_3_max": 0.48448560795502427, - "nauc_map_at_3_std": 0.0019450780457777837, - "nauc_map_at_5_diff1": 0.5876191275019375, - "nauc_map_at_5_max": 0.48860167161632034, - "nauc_map_at_5_std": 0.0071168197778243335, - "nauc_mrr_at_1000_diff1": 0.5876636565567583, - "nauc_mrr_at_1000_max": 0.4868422238450373, - "nauc_mrr_at_1000_std": 0.006172855129180589, - "nauc_mrr_at_100_diff1": 0.5872173827802959, - "nauc_mrr_at_100_max": 0.48681846044840055, - "nauc_mrr_at_100_std": 0.006174527620908233, - "nauc_mrr_at_10_diff1": 0.5843873762953701, - "nauc_mrr_at_10_max": 0.48569620079470033, - "nauc_mrr_at_10_std": 0.0048121712540381174, - "nauc_mrr_at_1_diff1": 0.6469130123286798, - "nauc_mrr_at_1_max": 0.4996837405335848, - "nauc_mrr_at_1_std": 0.021401518977899162, - "nauc_mrr_at_20_diff1": 0.5860522812282696, - "nauc_mrr_at_20_max": 0.4853116062444229, - "nauc_mrr_at_20_std": 0.003802028482319083, - "nauc_mrr_at_3_diff1": 0.5930592769877978, - "nauc_mrr_at_3_max": 0.48448560795502427, - "nauc_mrr_at_3_std": 0.0019450780457777837, - "nauc_mrr_at_5_diff1": 0.5876191275019375, - "nauc_mrr_at_5_max": 0.48860167161632034, - "nauc_mrr_at_5_std": 0.0071168197778243335, - "nauc_ndcg_at_1000_diff1": 0.5764346198771939, - "nauc_ndcg_at_1000_max": 0.48685121856010233, - "nauc_ndcg_at_1000_std": 0.008790975817410052, - "nauc_ndcg_at_100_diff1": 0.5681028972323554, - "nauc_ndcg_at_100_max": 0.48868765751875193, - "nauc_ndcg_at_100_std": 0.018708262636436764, - "nauc_ndcg_at_10_diff1": 0.557269842812324, - "nauc_ndcg_at_10_max": 0.4815575861378846, - "nauc_ndcg_at_10_std": 0.0002687978334403188, - "nauc_ndcg_at_1_diff1": 0.6469130123286798, - "nauc_ndcg_at_1_max": 0.4996837405335848, - "nauc_ndcg_at_1_std": 0.021401518977899162, - "nauc_ndcg_at_20_diff1": 0.5612535119375365, - "nauc_ndcg_at_20_max": 0.4786432157571462, - "nauc_ndcg_at_20_std": -0.004814012882018789, - "nauc_ndcg_at_3_diff1": 0.5761744796564754, - "nauc_ndcg_at_3_max": 0.48156121083060116, - "nauc_ndcg_at_3_std": -0.0038383683766595052, - "nauc_ndcg_at_5_diff1": 0.5665516875226873, - "nauc_ndcg_at_5_max": 0.48879365280033604, - "nauc_ndcg_at_5_std": 0.005361216092415931, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.43482719399589986, - "nauc_precision_at_100_max": 0.5359090786963644, - "nauc_precision_at_100_std": 0.2492902370652986, - "nauc_precision_at_10_diff1": 0.4647400719280439, - "nauc_precision_at_10_max": 0.4667315537700093, - "nauc_precision_at_10_std": -0.01435627161465371, - "nauc_precision_at_1_diff1": 0.6469130123286798, - "nauc_precision_at_1_max": 0.4996837405335848, - "nauc_precision_at_1_std": 0.021401518977899162, - "nauc_precision_at_20_diff1": 0.4635676512952064, - "nauc_precision_at_20_max": 0.4465475168470034, - "nauc_precision_at_20_std": -0.04457976228771008, - "nauc_precision_at_3_diff1": 0.5278920774752098, - "nauc_precision_at_3_max": 0.47376992697805037, - "nauc_precision_at_3_std": -0.020277972142073643, - "nauc_precision_at_5_diff1": 0.5037572697381797, - "nauc_precision_at_5_max": 0.49088525780001036, - "nauc_precision_at_5_std": 0.0016189734300648799, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.4348271939958975, - "nauc_recall_at_100_max": 0.5359090786963637, - "nauc_recall_at_100_std": 0.24929023706529907, - "nauc_recall_at_10_diff1": 0.4647400719280445, - "nauc_recall_at_10_max": 0.46673155377000985, - "nauc_recall_at_10_std": -0.014356271614653547, - "nauc_recall_at_1_diff1": 0.6469130123286798, - "nauc_recall_at_1_max": 0.4996837405335848, - "nauc_recall_at_1_std": 0.021401518977899162, - "nauc_recall_at_20_diff1": 0.46356765129520633, - "nauc_recall_at_20_max": 0.44654751684700356, - "nauc_recall_at_20_std": -0.04457976228770894, - "nauc_recall_at_3_diff1": 0.5278920774752104, - "nauc_recall_at_3_max": 0.47376992697805126, - "nauc_recall_at_3_std": -0.02027797214207337, - "nauc_recall_at_5_diff1": 0.5037572697381799, - "nauc_recall_at_5_max": 0.4908852578000109, - "nauc_recall_at_5_std": 0.001618973430065162, - "ndcg_at_1": 0.32209, - "ndcg_at_10": 0.44843, - "ndcg_at_100": 0.50502, - "ndcg_at_1000": 0.52107, - "ndcg_at_20": 0.47104, - "ndcg_at_3": 0.39114, - "ndcg_at_5": 0.41931, - "precision_at_1": 0.32209, - "precision_at_10": 0.06, - "precision_at_100": 0.00876, - "precision_at_1000": 0.001, - "precision_at_20": 0.03448, - "precision_at_3": 0.1469, - "precision_at_5": 0.10186, - "recall_at_1": 0.32209, - "recall_at_10": 0.6, - "recall_at_100": 0.87558, - "recall_at_1000": 1.0, - "recall_at_20": 0.68953, - "recall_at_3": 0.4407, - "recall_at_5": 0.5093 - }, - { - "cv_recall_at_1": 0.39843, - "cv_recall_at_10": 0.66667, - "cv_recall_at_100": 0.8945, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.73625, - "cv_recall_at_3": 0.53872, - "cv_recall_at_5": 0.59484, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.52595, - "map_at_1": 0.39843, - "map_at_10": 0.4817, - "map_at_100": 0.49033, - "map_at_1000": 0.49091, - "map_at_20": 0.48638, - "map_at_3": 0.45978, - "map_at_5": 0.47213, - "mrr_at_1": 0.3984287317620651, - "mrr_at_10": 0.48169704095629967, - "mrr_at_100": 0.49033374059196033, - "mrr_at_1000": 0.4909059829649581, - "mrr_at_20": 0.4863823772433859, - "mrr_at_3": 0.4597830153385704, - "mrr_at_5": 0.47212869435091576, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.35457465941244737, - "nauc_cv_recall_at_100_max": 0.5576807085732769, - "nauc_cv_recall_at_100_std": 0.39068957310710517, - "nauc_cv_recall_at_10_diff1": 0.49672487714012176, - "nauc_cv_recall_at_10_max": 0.6096103856426884, - "nauc_cv_recall_at_10_std": 0.18319856486677483, - "nauc_cv_recall_at_1_diff1": 0.6873254674015392, - "nauc_cv_recall_at_1_max": 0.5296868024822796, - "nauc_cv_recall_at_1_std": -0.0357131227009029, - "nauc_cv_recall_at_20_diff1": 0.45066929394701233, - "nauc_cv_recall_at_20_max": 0.6132154485168014, - "nauc_cv_recall_at_20_std": 0.24783736517811059, - "nauc_cv_recall_at_3_diff1": 0.5782535988484254, - "nauc_cv_recall_at_3_max": 0.5585680345701475, - "nauc_cv_recall_at_3_std": 0.03651369037074517, - "nauc_cv_recall_at_5_diff1": 0.5350363821120877, - "nauc_cv_recall_at_5_max": 0.5901419254871935, - "nauc_cv_recall_at_5_std": 0.1290002567018423, - "nauc_map_at_1000_diff1": 0.6326492352687968, - "nauc_map_at_1000_max": 0.5476258801331528, - "nauc_map_at_1000_std": 0.013729566821103458, - "nauc_map_at_100_diff1": 0.632252036265804, - "nauc_map_at_100_max": 0.5476452208181938, - "nauc_map_at_100_std": 0.014276296948681754, - "nauc_map_at_10_diff1": 0.6324263142199975, - "nauc_map_at_10_max": 0.5489231706174229, - "nauc_map_at_10_std": 0.013307014661679977, - "nauc_map_at_1_diff1": 0.6873254674015392, - "nauc_map_at_1_max": 0.5296868024822796, - "nauc_map_at_1_std": -0.0357131227009029, - "nauc_map_at_20_diff1": 0.631412253292453, - "nauc_map_at_20_max": 0.5484171024649589, - "nauc_map_at_20_std": 0.014782813145197712, - "nauc_map_at_3_diff1": 0.6430234238893437, - "nauc_map_at_3_max": 0.5411032167398862, - "nauc_map_at_3_std": -0.008678979459466659, - "nauc_map_at_5_diff1": 0.63526153323913, - "nauc_map_at_5_max": 0.5477376679115433, - "nauc_map_at_5_std": 0.009739363758753661, - "nauc_mrr_at_1000_diff1": 0.6326492352687968, - "nauc_mrr_at_1000_max": 0.5476258801331528, - "nauc_mrr_at_1000_std": 0.013729566821103458, - "nauc_mrr_at_100_diff1": 0.632252036265804, - "nauc_mrr_at_100_max": 0.5476452208181938, - "nauc_mrr_at_100_std": 0.014276296948681754, - "nauc_mrr_at_10_diff1": 0.6324263142199975, - "nauc_mrr_at_10_max": 0.5489231706174229, - "nauc_mrr_at_10_std": 0.013307014661679977, - "nauc_mrr_at_1_diff1": 0.6873254674015392, - "nauc_mrr_at_1_max": 0.5296868024822796, - "nauc_mrr_at_1_std": -0.0357131227009029, - "nauc_mrr_at_20_diff1": 0.631412253292453, - "nauc_mrr_at_20_max": 0.5484171024649589, - "nauc_mrr_at_20_std": 0.014782813145197712, - "nauc_mrr_at_3_diff1": 0.6430234238893437, - "nauc_mrr_at_3_max": 0.5411032167398862, - "nauc_mrr_at_3_std": -0.008678979459466659, - "nauc_mrr_at_5_diff1": 0.63526153323913, - "nauc_mrr_at_5_max": 0.5477376679115433, - "nauc_mrr_at_5_std": 0.009739363758753661, - "nauc_ndcg_at_1000_diff1": 0.6165259932868026, - "nauc_ndcg_at_1000_max": 0.5525071476684726, - "nauc_ndcg_at_1000_std": 0.03385620702194307, - "nauc_ndcg_at_100_diff1": 0.604737973648432, - "nauc_ndcg_at_100_max": 0.5527301799243698, - "nauc_ndcg_at_100_std": 0.049956953429283336, - "nauc_ndcg_at_10_diff1": 0.6042269549971561, - "nauc_ndcg_at_10_max": 0.5612708687432685, - "nauc_ndcg_at_10_std": 0.04763297007687358, - "nauc_ndcg_at_1_diff1": 0.6873254674015392, - "nauc_ndcg_at_1_max": 0.5296868024822796, - "nauc_ndcg_at_1_std": -0.0357131227009029, - "nauc_ndcg_at_20_diff1": 0.5994153801773628, - "nauc_ndcg_at_20_max": 0.5600367471910729, - "nauc_ndcg_at_20_std": 0.05474081561828843, - "nauc_ndcg_at_3_diff1": 0.6270236848909636, - "nauc_ndcg_at_3_max": 0.5454043787504821, - "nauc_ndcg_at_3_std": 0.0023889403899353366, - "nauc_ndcg_at_5_diff1": 0.6119886766768379, - "nauc_ndcg_at_5_max": 0.5574494515120727, - "nauc_ndcg_at_5_std": 0.03679944430425355, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.35457465941244803, - "nauc_precision_at_100_max": 0.5576807085732789, - "nauc_precision_at_100_std": 0.39068957310710556, - "nauc_precision_at_10_diff1": 0.4967248771401208, - "nauc_precision_at_10_max": 0.6096103856426881, - "nauc_precision_at_10_std": 0.1831985648667747, - "nauc_precision_at_1_diff1": 0.6873254674015392, - "nauc_precision_at_1_max": 0.5296868024822796, - "nauc_precision_at_1_std": -0.0357131227009029, - "nauc_precision_at_20_diff1": 0.4506692939470118, - "nauc_precision_at_20_max": 0.6132154485168013, - "nauc_precision_at_20_std": 0.24783736517811034, - "nauc_precision_at_3_diff1": 0.5782535988484254, - "nauc_precision_at_3_max": 0.5585680345701476, - "nauc_precision_at_3_std": 0.03651369037074524, - "nauc_precision_at_5_diff1": 0.5350363821120875, - "nauc_precision_at_5_max": 0.5901419254871931, - "nauc_precision_at_5_std": 0.12900025670184193, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.35457465941244737, - "nauc_recall_at_100_max": 0.5576807085732769, - "nauc_recall_at_100_std": 0.39068957310710517, - "nauc_recall_at_10_diff1": 0.49672487714012176, - "nauc_recall_at_10_max": 0.6096103856426884, - "nauc_recall_at_10_std": 0.18319856486677483, - "nauc_recall_at_1_diff1": 0.6873254674015392, - "nauc_recall_at_1_max": 0.5296868024822796, - "nauc_recall_at_1_std": -0.0357131227009029, - "nauc_recall_at_20_diff1": 0.45066929394701233, - "nauc_recall_at_20_max": 0.6132154485168014, - "nauc_recall_at_20_std": 0.24783736517811059, - "nauc_recall_at_3_diff1": 0.5782535988484254, - "nauc_recall_at_3_max": 0.5585680345701475, - "nauc_recall_at_3_std": 0.03651369037074517, - "nauc_recall_at_5_diff1": 0.5350363821120877, - "nauc_recall_at_5_max": 0.5901419254871935, - "nauc_recall_at_5_std": 0.1290002567018423, - "ndcg_at_1": 0.39843, - "ndcg_at_10": 0.52595, - "ndcg_at_100": 0.57245, - "ndcg_at_1000": 0.58623, - "ndcg_at_20": 0.54336, - "ndcg_at_3": 0.48004, - "ndcg_at_5": 0.50273, - "precision_at_1": 0.39843, - "precision_at_10": 0.06667, - "precision_at_100": 0.00895, - "precision_at_1000": 0.001, - "precision_at_20": 0.03681, - "precision_at_3": 0.17957, - "precision_at_5": 0.11897, - "recall_at_1": 0.39843, - "recall_at_10": 0.66667, - "recall_at_100": 0.8945, - "recall_at_1000": 1.0, - "recall_at_20": 0.73625, - "recall_at_3": 0.53872, - "recall_at_5": 0.59484 - }, - { - "cv_recall_at_1": 0.27895, - "cv_recall_at_10": 0.54035, - "cv_recall_at_100": 0.85789, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.65965, - "cv_recall_at_3": 0.36667, - "cv_recall_at_5": 0.44912, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.39322, - "map_at_1": 0.27895, - "map_at_10": 0.348, - "map_at_100": 0.36124, - "map_at_1000": 0.36205, - "map_at_20": 0.35658, - "map_at_3": 0.31667, - "map_at_5": 0.33579, - "mrr_at_1": 0.2789473684210526, - "mrr_at_10": 0.34799986076301853, - "mrr_at_100": 0.3612379451067614, - "mrr_at_1000": 0.3620538102805267, - "mrr_at_20": 0.35657692868239554, - "mrr_at_3": 0.31666666666666665, - "mrr_at_5": 0.33578947368421047, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.27402837236632377, - "nauc_cv_recall_at_100_max": 0.5046647719777929, - "nauc_cv_recall_at_100_std": 0.43405763211857484, - "nauc_cv_recall_at_10_diff1": 0.36232640243426695, - "nauc_cv_recall_at_10_max": 0.41406189045798525, - "nauc_cv_recall_at_10_std": 0.18209701958336588, - "nauc_cv_recall_at_1_diff1": 0.578766879493732, - "nauc_cv_recall_at_1_max": 0.4193896011763175, - "nauc_cv_recall_at_1_std": 0.023251418233412462, - "nauc_cv_recall_at_20_diff1": 0.27230672547627616, - "nauc_cv_recall_at_20_max": 0.45335866491324034, - "nauc_cv_recall_at_20_std": 0.3078805264934194, - "nauc_cv_recall_at_3_diff1": 0.46300966262279075, - "nauc_cv_recall_at_3_max": 0.4259355718065274, - "nauc_cv_recall_at_3_std": 0.07531384593497516, - "nauc_cv_recall_at_5_diff1": 0.40499343150854816, - "nauc_cv_recall_at_5_max": 0.4071683830486986, - "nauc_cv_recall_at_5_std": 0.1010146711620259, - "nauc_map_at_1000_diff1": 0.5071629288831934, - "nauc_map_at_1000_max": 0.42123422925176146, - "nauc_map_at_1000_std": 0.06497914953890994, - "nauc_map_at_100_diff1": 0.506574340833974, - "nauc_map_at_100_max": 0.4214497059406688, - "nauc_map_at_100_std": 0.06587010366751947, - "nauc_map_at_10_diff1": 0.5082446345792789, - "nauc_map_at_10_max": 0.4189152236357482, - "nauc_map_at_10_std": 0.06010066860993658, - "nauc_map_at_1_diff1": 0.578766879493732, - "nauc_map_at_1_max": 0.4193896011763175, - "nauc_map_at_1_std": 0.023251418233412462, - "nauc_map_at_20_diff1": 0.5041720899117745, - "nauc_map_at_20_max": 0.42110178848378027, - "nauc_map_at_20_std": 0.06636052994372635, - "nauc_map_at_3_diff1": 0.528685422161208, - "nauc_map_at_3_max": 0.42343211512633144, - "nauc_map_at_3_std": 0.04403801855881142, - "nauc_map_at_5_diff1": 0.513704999469193, - "nauc_map_at_5_max": 0.4189232291850256, - "nauc_map_at_5_std": 0.050220770372886234, - "nauc_mrr_at_1000_diff1": 0.5071629288831934, - "nauc_mrr_at_1000_max": 0.42123422925176146, - "nauc_mrr_at_1000_std": 0.06497914953890994, - "nauc_mrr_at_100_diff1": 0.506574340833974, - "nauc_mrr_at_100_max": 0.4214497059406688, - "nauc_mrr_at_100_std": 0.06587010366751947, - "nauc_mrr_at_10_diff1": 0.5082446345792789, - "nauc_mrr_at_10_max": 0.4189152236357482, - "nauc_mrr_at_10_std": 0.06010066860993658, - "nauc_mrr_at_1_diff1": 0.578766879493732, - "nauc_mrr_at_1_max": 0.4193896011763175, - "nauc_mrr_at_1_std": 0.023251418233412462, - "nauc_mrr_at_20_diff1": 0.5041720899117745, - "nauc_mrr_at_20_max": 0.42110178848378027, - "nauc_mrr_at_20_std": 0.06636052994372635, - "nauc_mrr_at_3_diff1": 0.528685422161208, - "nauc_mrr_at_3_max": 0.42343211512633144, - "nauc_mrr_at_3_std": 0.04403801855881142, - "nauc_mrr_at_5_diff1": 0.513704999469193, - "nauc_mrr_at_5_max": 0.4189232291850256, - "nauc_mrr_at_5_std": 0.050220770372886234, - "nauc_ndcg_at_1000_diff1": 0.48697231270265845, - "nauc_ndcg_at_1000_max": 0.4231368217605475, - "nauc_ndcg_at_1000_std": 0.08523789915262146, - "nauc_ndcg_at_100_diff1": 0.47393152314008596, - "nauc_ndcg_at_100_max": 0.42813566600077263, - "nauc_ndcg_at_100_std": 0.10638039759966585, - "nauc_ndcg_at_10_diff1": 0.47317884996832, - "nauc_ndcg_at_10_max": 0.4175207394457012, - "nauc_ndcg_at_10_std": 0.08784415774604273, - "nauc_ndcg_at_1_diff1": 0.578766879493732, - "nauc_ndcg_at_1_max": 0.4193896011763175, - "nauc_ndcg_at_1_std": 0.023251418233412462, - "nauc_ndcg_at_20_diff1": 0.4575951570176458, - "nauc_ndcg_at_20_max": 0.42601612367918157, - "nauc_ndcg_at_20_std": 0.11166824539675238, - "nauc_ndcg_at_3_diff1": 0.5114809115902081, - "nauc_ndcg_at_3_max": 0.42413650929139324, - "nauc_ndcg_at_3_std": 0.05216132426187068, - "nauc_ndcg_at_5_diff1": 0.4858911567361287, - "nauc_ndcg_at_5_max": 0.41614631893275955, - "nauc_ndcg_at_5_std": 0.06310315441584341, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.2740283723663231, - "nauc_precision_at_100_max": 0.5046647719777917, - "nauc_precision_at_100_std": 0.4340576321185744, - "nauc_precision_at_10_diff1": 0.36232640243426684, - "nauc_precision_at_10_max": 0.414061890457985, - "nauc_precision_at_10_std": 0.18209701958336594, - "nauc_precision_at_1_diff1": 0.578766879493732, - "nauc_precision_at_1_max": 0.4193896011763175, - "nauc_precision_at_1_std": 0.023251418233412462, - "nauc_precision_at_20_diff1": 0.2723067254762764, - "nauc_precision_at_20_max": 0.45335866491324045, - "nauc_precision_at_20_std": 0.3078805264934188, - "nauc_precision_at_3_diff1": 0.4630096626227905, - "nauc_precision_at_3_max": 0.42593557180652697, - "nauc_precision_at_3_std": 0.07531384593497523, - "nauc_precision_at_5_diff1": 0.4049934315085483, - "nauc_precision_at_5_max": 0.40716838304869857, - "nauc_precision_at_5_std": 0.1010146711620258, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.27402837236632377, - "nauc_recall_at_100_max": 0.5046647719777929, - "nauc_recall_at_100_std": 0.43405763211857484, - "nauc_recall_at_10_diff1": 0.36232640243426695, - "nauc_recall_at_10_max": 0.41406189045798525, - "nauc_recall_at_10_std": 0.18209701958336588, - "nauc_recall_at_1_diff1": 0.578766879493732, - "nauc_recall_at_1_max": 0.4193896011763175, - "nauc_recall_at_1_std": 0.023251418233412462, - "nauc_recall_at_20_diff1": 0.27230672547627616, - "nauc_recall_at_20_max": 0.45335866491324034, - "nauc_recall_at_20_std": 0.3078805264934194, - "nauc_recall_at_3_diff1": 0.46300966262279075, - "nauc_recall_at_3_max": 0.4259355718065274, - "nauc_recall_at_3_std": 0.07531384593497516, - "nauc_recall_at_5_diff1": 0.40499343150854816, - "nauc_recall_at_5_max": 0.4071683830486986, - "nauc_recall_at_5_std": 0.1010146711620259, - "ndcg_at_1": 0.27895, - "ndcg_at_10": 0.39322, - "ndcg_at_100": 0.45969, - "ndcg_at_1000": 0.47856, - "ndcg_at_20": 0.42374, - "ndcg_at_3": 0.32947, - "ndcg_at_5": 0.36367, - "precision_at_1": 0.27895, - "precision_at_10": 0.05404, - "precision_at_100": 0.00858, - "precision_at_1000": 0.001, - "precision_at_20": 0.03298, - "precision_at_3": 0.12222, - "precision_at_5": 0.08982, - "recall_at_1": 0.27895, - "recall_at_10": 0.54035, - "recall_at_100": 0.85789, - "recall_at_1000": 1.0, - "recall_at_20": 0.65965, - "recall_at_3": 0.36667, - "recall_at_5": 0.44912 - }, - { - "cv_recall_at_1": 0.19108, - "cv_recall_at_10": 0.42906, - "cv_recall_at_100": 0.76659, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.50572, - "cv_recall_at_3": 0.29519, - "cv_recall_at_5": 0.35469, - "hf_subset": "et", - "languages": [ - "est-Latn" - ], - "main_score": 0.2994, - "map_at_1": 0.19108, - "map_at_10": 0.25916, - "map_at_100": 0.27058, - "map_at_1000": 0.2716, - "map_at_20": 0.26439, - "map_at_3": 0.2357, - "map_at_5": 0.24937, - "mrr_at_1": 0.19107551487414187, - "mrr_at_10": 0.25915967454869054, - "mrr_at_100": 0.27057718507760353, - "mrr_at_1000": 0.271598792636963, - "mrr_at_20": 0.2643949666341024, - "mrr_at_3": 0.2356979405034326, - "mrr_at_5": 0.2493707093821511, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.008516376183806164, - "nauc_cv_recall_at_100_max": 0.5283066591673302, - "nauc_cv_recall_at_100_std": 0.5829740280471483, - "nauc_cv_recall_at_10_diff1": 0.2220847558311369, - "nauc_cv_recall_at_10_max": 0.43731525006271427, - "nauc_cv_recall_at_10_std": 0.20381677609332463, - "nauc_cv_recall_at_1_diff1": 0.5605226558512929, - "nauc_cv_recall_at_1_max": 0.5336666192190369, - "nauc_cv_recall_at_1_std": 0.0588461370327258, - "nauc_cv_recall_at_20_diff1": 0.15852763873374953, - "nauc_cv_recall_at_20_max": 0.4520595706685236, - "nauc_cv_recall_at_20_std": 0.31579778409402076, - "nauc_cv_recall_at_3_diff1": 0.3743596257745175, - "nauc_cv_recall_at_3_max": 0.46859264567363157, - "nauc_cv_recall_at_3_std": 0.06856746069956918, - "nauc_cv_recall_at_5_diff1": 0.2959683249078792, - "nauc_cv_recall_at_5_max": 0.42952579478833075, - "nauc_cv_recall_at_5_std": 0.09896414249666792, - "nauc_map_at_1000_diff1": 0.4317225812730425, - "nauc_map_at_1000_max": 0.4951111047823308, - "nauc_map_at_1000_std": 0.09733934003605733, - "nauc_map_at_100_diff1": 0.4308421866301083, - "nauc_map_at_100_max": 0.4953343634107362, - "nauc_map_at_100_std": 0.09853358301236757, - "nauc_map_at_10_diff1": 0.43592345198798077, - "nauc_map_at_10_max": 0.49290073487004293, - "nauc_map_at_10_std": 0.08683872112304801, - "nauc_map_at_1_diff1": 0.5605226558512929, - "nauc_map_at_1_max": 0.5336666192190369, - "nauc_map_at_1_std": 0.0588461370327258, - "nauc_map_at_20_diff1": 0.4316814136771351, - "nauc_map_at_20_max": 0.49457613365867314, - "nauc_map_at_20_std": 0.09548935501608316, - "nauc_map_at_3_diff1": 0.46875777456087836, - "nauc_map_at_3_max": 0.5034907821086785, - "nauc_map_at_3_std": 0.06354789645788499, - "nauc_map_at_5_diff1": 0.44783010602361834, - "nauc_map_at_5_max": 0.49306058616671694, - "nauc_map_at_5_std": 0.07173277743503308, - "nauc_mrr_at_1000_diff1": 0.4317225812730425, - "nauc_mrr_at_1000_max": 0.4951111047823308, - "nauc_mrr_at_1000_std": 0.09733934003605733, - "nauc_mrr_at_100_diff1": 0.4308421866301083, - "nauc_mrr_at_100_max": 0.4953343634107362, - "nauc_mrr_at_100_std": 0.09853358301236757, - "nauc_mrr_at_10_diff1": 0.43592345198798077, - "nauc_mrr_at_10_max": 0.49290073487004293, - "nauc_mrr_at_10_std": 0.08683872112304801, - "nauc_mrr_at_1_diff1": 0.5605226558512929, - "nauc_mrr_at_1_max": 0.5336666192190369, - "nauc_mrr_at_1_std": 0.0588461370327258, - "nauc_mrr_at_20_diff1": 0.4316814136771351, - "nauc_mrr_at_20_max": 0.49457613365867314, - "nauc_mrr_at_20_std": 0.09548935501608316, - "nauc_mrr_at_3_diff1": 0.46875777456087836, - "nauc_mrr_at_3_max": 0.5034907821086785, - "nauc_mrr_at_3_std": 0.06354789645788499, - "nauc_mrr_at_5_diff1": 0.44783010602361834, - "nauc_mrr_at_5_max": 0.49306058616671694, - "nauc_mrr_at_5_std": 0.07173277743503308, - "nauc_ndcg_at_1000_diff1": 0.3902812726294797, - "nauc_ndcg_at_1000_max": 0.4883252404313833, - "nauc_ndcg_at_1000_std": 0.12918188527168667, - "nauc_ndcg_at_100_diff1": 0.3583473582903687, - "nauc_ndcg_at_100_max": 0.4926795176760152, - "nauc_ndcg_at_100_std": 0.16837332429694266, - "nauc_ndcg_at_10_diff1": 0.37884464737747714, - "nauc_ndcg_at_10_max": 0.4771396846330027, - "nauc_ndcg_at_10_std": 0.11567882387987487, - "nauc_ndcg_at_1_diff1": 0.5605226558512929, - "nauc_ndcg_at_1_max": 0.5336666192190369, - "nauc_ndcg_at_1_std": 0.0588461370327258, - "nauc_ndcg_at_20_diff1": 0.3629320685513456, - "nauc_ndcg_at_20_max": 0.48203568883383097, - "nauc_ndcg_at_20_std": 0.14601766668739585, - "nauc_ndcg_at_3_diff1": 0.44244034795034554, - "nauc_ndcg_at_3_max": 0.4938399382557547, - "nauc_ndcg_at_3_std": 0.06494358833093328, - "nauc_ndcg_at_5_diff1": 0.4060786418896771, - "nauc_ndcg_at_5_max": 0.47576035017023, - "nauc_ndcg_at_5_std": 0.07898953790711814, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.008516376183805786, - "nauc_precision_at_100_max": 0.5283066591673299, - "nauc_precision_at_100_std": 0.5829740280471479, - "nauc_precision_at_10_diff1": 0.2220847558311367, - "nauc_precision_at_10_max": 0.437315250062714, - "nauc_precision_at_10_std": 0.20381677609332435, - "nauc_precision_at_1_diff1": 0.5605226558512929, - "nauc_precision_at_1_max": 0.5336666192190369, - "nauc_precision_at_1_std": 0.0588461370327258, - "nauc_precision_at_20_diff1": 0.15852763873374898, - "nauc_precision_at_20_max": 0.45205957066852337, - "nauc_precision_at_20_std": 0.3157977840940208, - "nauc_precision_at_3_diff1": 0.3743596257745172, - "nauc_precision_at_3_max": 0.4685926456736312, - "nauc_precision_at_3_std": 0.06856746069956916, - "nauc_precision_at_5_diff1": 0.2959683249078796, - "nauc_precision_at_5_max": 0.429525794788331, - "nauc_precision_at_5_std": 0.09896414249666852, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.008516376183806164, - "nauc_recall_at_100_max": 0.5283066591673302, - "nauc_recall_at_100_std": 0.5829740280471483, - "nauc_recall_at_10_diff1": 0.2220847558311369, - "nauc_recall_at_10_max": 0.43731525006271427, - "nauc_recall_at_10_std": 0.20381677609332463, - "nauc_recall_at_1_diff1": 0.5605226558512929, - "nauc_recall_at_1_max": 0.5336666192190369, - "nauc_recall_at_1_std": 0.0588461370327258, - "nauc_recall_at_20_diff1": 0.15852763873374953, - "nauc_recall_at_20_max": 0.4520595706685236, - "nauc_recall_at_20_std": 0.31579778409402076, - "nauc_recall_at_3_diff1": 0.3743596257745175, - "nauc_recall_at_3_max": 0.46859264567363157, - "nauc_recall_at_3_std": 0.06856746069956918, - "nauc_recall_at_5_diff1": 0.2959683249078792, - "nauc_recall_at_5_max": 0.42952579478833075, - "nauc_recall_at_5_std": 0.09896414249666792, - "ndcg_at_1": 0.19108, - "ndcg_at_10": 0.2994, - "ndcg_at_100": 0.36608, - "ndcg_at_1000": 0.3953, - "ndcg_at_20": 0.31868, - "ndcg_at_3": 0.25092, - "ndcg_at_5": 0.2755, - "precision_at_1": 0.19108, - "precision_at_10": 0.04291, - "precision_at_100": 0.00767, - "precision_at_1000": 0.001, - "precision_at_20": 0.02529, - "precision_at_3": 0.0984, - "precision_at_5": 0.07094, - "recall_at_1": 0.19108, - "recall_at_10": 0.42906, - "recall_at_100": 0.76659, - "recall_at_1000": 1.0, - "recall_at_20": 0.50572, - "recall_at_3": 0.29519, - "recall_at_5": 0.35469 - }, - { - "cv_recall_at_1": 0.43507, - "cv_recall_at_10": 0.73141, - "cv_recall_at_100": 0.94007, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.81021, - "cv_recall_at_3": 0.56715, - "cv_recall_at_5": 0.62819, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.57023, - "map_at_1": 0.43507, - "map_at_10": 0.52031, - "map_at_100": 0.52893, - "map_at_1000": 0.52923, - "map_at_20": 0.52575, - "map_at_3": 0.4926, - "map_at_5": 0.50653, - "mrr_at_1": 0.43507214206437295, - "mrr_at_10": 0.5203103253880169, - "mrr_at_100": 0.5289324885399981, - "mrr_at_1000": 0.5292325136237984, - "mrr_at_20": 0.5257454133953585, - "mrr_at_3": 0.4926008139104697, - "mrr_at_5": 0.5065297817240101, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.6978733265275376, - "nauc_cv_recall_at_100_max": 0.5949865305257486, - "nauc_cv_recall_at_100_std": 0.5166498497442625, - "nauc_cv_recall_at_10_diff1": 0.5870400374929791, - "nauc_cv_recall_at_10_max": 0.4192351294053403, - "nauc_cv_recall_at_10_std": 0.15748320564522547, - "nauc_cv_recall_at_1_diff1": 0.7306432624255598, - "nauc_cv_recall_at_1_max": 0.514466513106884, - "nauc_cv_recall_at_1_std": 0.067281314126223, - "nauc_cv_recall_at_20_diff1": 0.5648059742557373, - "nauc_cv_recall_at_20_max": 0.4139453010958553, - "nauc_cv_recall_at_20_std": 0.22809417514046965, - "nauc_cv_recall_at_3_diff1": 0.6414846050634647, - "nauc_cv_recall_at_3_max": 0.47781893080666504, - "nauc_cv_recall_at_3_std": 0.09515707826623646, - "nauc_cv_recall_at_5_diff1": 0.6142780982117747, - "nauc_cv_recall_at_5_max": 0.43914771938140457, - "nauc_cv_recall_at_5_std": 0.09945784932203011, - "nauc_map_at_1000_diff1": 0.6895559381936318, - "nauc_map_at_1000_max": 0.49563279664453025, - "nauc_map_at_1000_std": 0.08673011670743808, - "nauc_map_at_100_diff1": 0.689562613982922, - "nauc_map_at_100_max": 0.4957215240433665, - "nauc_map_at_100_std": 0.08710243504689028, - "nauc_map_at_10_diff1": 0.6874136307446468, - "nauc_map_at_10_max": 0.49376751093121124, - "nauc_map_at_10_std": 0.08555924018421786, - "nauc_map_at_1_diff1": 0.7306432624255598, - "nauc_map_at_1_max": 0.514466513106884, - "nauc_map_at_1_std": 0.067281314126223, - "nauc_map_at_20_diff1": 0.6875938630174272, - "nauc_map_at_20_max": 0.49392152305217696, - "nauc_map_at_20_std": 0.08669227777785457, - "nauc_map_at_3_diff1": 0.6929406454047352, - "nauc_map_at_3_max": 0.5020126401504487, - "nauc_map_at_3_std": 0.0800419124746124, - "nauc_map_at_5_diff1": 0.6882651563349085, - "nauc_map_at_5_max": 0.4944791856163838, - "nauc_map_at_5_std": 0.08138282547856554, - "nauc_mrr_at_1000_diff1": 0.6895559381936318, - "nauc_mrr_at_1000_max": 0.49563279664453025, - "nauc_mrr_at_1000_std": 0.08673011670743808, - "nauc_mrr_at_100_diff1": 0.689562613982922, - "nauc_mrr_at_100_max": 0.4957215240433665, - "nauc_mrr_at_100_std": 0.08710243504689028, - "nauc_mrr_at_10_diff1": 0.6874136307446468, - "nauc_mrr_at_10_max": 0.49376751093121124, - "nauc_mrr_at_10_std": 0.08555924018421786, - "nauc_mrr_at_1_diff1": 0.7306432624255598, - "nauc_mrr_at_1_max": 0.514466513106884, - "nauc_mrr_at_1_std": 0.067281314126223, - "nauc_mrr_at_20_diff1": 0.6875938630174272, - "nauc_mrr_at_20_max": 0.49392152305217696, - "nauc_mrr_at_20_std": 0.08669227777785457, - "nauc_mrr_at_3_diff1": 0.6929406454047352, - "nauc_mrr_at_3_max": 0.5020126401504487, - "nauc_mrr_at_3_std": 0.0800419124746124, - "nauc_mrr_at_5_diff1": 0.6882651563349085, - "nauc_mrr_at_5_max": 0.4944791856163838, - "nauc_mrr_at_5_std": 0.08138282547856554, - "nauc_ndcg_at_1000_diff1": 0.6822856007923828, - "nauc_ndcg_at_1000_max": 0.49173586713551026, - "nauc_ndcg_at_1000_std": 0.09620762284631088, - "nauc_ndcg_at_100_diff1": 0.6827195804493991, - "nauc_ndcg_at_100_max": 0.494646191162205, - "nauc_ndcg_at_100_std": 0.10798696761972172, - "nauc_ndcg_at_10_diff1": 0.6681409781657833, - "nauc_ndcg_at_10_max": 0.47984251929370647, - "nauc_ndcg_at_10_std": 0.09819022365147305, - "nauc_ndcg_at_1_diff1": 0.7306432624255598, - "nauc_ndcg_at_1_max": 0.514466513106884, - "nauc_ndcg_at_1_std": 0.067281314126223, - "nauc_ndcg_at_20_diff1": 0.6684980553980436, - "nauc_ndcg_at_20_max": 0.48128478811001296, - "nauc_ndcg_at_20_std": 0.10510187950480612, - "nauc_ndcg_at_3_diff1": 0.6802566039971267, - "nauc_ndcg_at_3_max": 0.49618043172313236, - "nauc_ndcg_at_3_std": 0.08380990206914761, - "nauc_ndcg_at_5_diff1": 0.6712860737099376, - "nauc_ndcg_at_5_max": 0.48218117316614384, - "nauc_ndcg_at_5_std": 0.08570823218004223, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.697873326527537, - "nauc_precision_at_100_max": 0.5949865305257479, - "nauc_precision_at_100_std": 0.5166498497442638, - "nauc_precision_at_10_diff1": 0.5870400374929786, - "nauc_precision_at_10_max": 0.41923512940533914, - "nauc_precision_at_10_std": 0.15748320564522525, - "nauc_precision_at_1_diff1": 0.7306432624255598, - "nauc_precision_at_1_max": 0.514466513106884, - "nauc_precision_at_1_std": 0.067281314126223, - "nauc_precision_at_20_diff1": 0.5648059742557369, - "nauc_precision_at_20_max": 0.4139453010958548, - "nauc_precision_at_20_std": 0.22809417514046984, - "nauc_precision_at_3_diff1": 0.6414846050634644, - "nauc_precision_at_3_max": 0.4778189308066647, - "nauc_precision_at_3_std": 0.09515707826623591, - "nauc_precision_at_5_diff1": 0.614278098211774, - "nauc_precision_at_5_max": 0.4391477193814045, - "nauc_precision_at_5_std": 0.09945784932202985, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.6978733265275376, - "nauc_recall_at_100_max": 0.5949865305257486, - "nauc_recall_at_100_std": 0.5166498497442625, - "nauc_recall_at_10_diff1": 0.5870400374929791, - "nauc_recall_at_10_max": 0.4192351294053403, - "nauc_recall_at_10_std": 0.15748320564522547, - "nauc_recall_at_1_diff1": 0.7306432624255598, - "nauc_recall_at_1_max": 0.514466513106884, - "nauc_recall_at_1_std": 0.067281314126223, - "nauc_recall_at_20_diff1": 0.5648059742557373, - "nauc_recall_at_20_max": 0.4139453010958553, - "nauc_recall_at_20_std": 0.22809417514046965, - "nauc_recall_at_3_diff1": 0.6414846050634647, - "nauc_recall_at_3_max": 0.47781893080666504, - "nauc_recall_at_3_std": 0.09515707826623646, - "nauc_recall_at_5_diff1": 0.6142780982117747, - "nauc_recall_at_5_max": 0.43914771938140457, - "nauc_recall_at_5_std": 0.09945784932203011, - "ndcg_at_1": 0.43507, - "ndcg_at_10": 0.57023, - "ndcg_at_100": 0.61394, - "ndcg_at_1000": 0.62169, - "ndcg_at_20": 0.59012, - "ndcg_at_3": 0.51172, - "ndcg_at_5": 0.53684, - "precision_at_1": 0.43507, - "precision_at_10": 0.07314, - "precision_at_100": 0.0094, - "precision_at_1000": 0.001, - "precision_at_20": 0.04051, - "precision_at_3": 0.18905, - "precision_at_5": 0.12564, - "recall_at_1": 0.43507, - "recall_at_10": 0.73141, - "recall_at_100": 0.94007, - "recall_at_1000": 1.0, - "recall_at_20": 0.81021, - "recall_at_3": 0.56715, - "recall_at_5": 0.62819 - }, - { - "cv_recall_at_1": 0.25242, - "cv_recall_at_10": 0.49087, - "cv_recall_at_100": 0.75295, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.55854, - "cv_recall_at_3": 0.35661, - "cv_recall_at_5": 0.41998, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.36077, - "map_at_1": 0.25242, - "map_at_10": 0.32029, - "map_at_100": 0.32975, - "map_at_1000": 0.33084, - "map_at_20": 0.32497, - "map_at_3": 0.29646, - "map_at_5": 0.31074, - "mrr_at_1": 0.25241675617615467, - "mrr_at_10": 0.3202913746952415, - "mrr_at_100": 0.32974560595937136, - "mrr_at_1000": 0.3308396030425244, - "mrr_at_20": 0.3249678191109887, - "mrr_at_3": 0.29645542427497334, - "mrr_at_5": 0.3107411385606875, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5126757026738264, - "nauc_cv_recall_at_100_max": 0.18510105009062797, - "nauc_cv_recall_at_100_std": 0.5888298680979758, - "nauc_cv_recall_at_10_diff1": 0.5370004026652652, - "nauc_cv_recall_at_10_max": 0.36735113267523206, - "nauc_cv_recall_at_10_std": 0.3517130623928045, - "nauc_cv_recall_at_1_diff1": 0.7038058763022518, - "nauc_cv_recall_at_1_max": 0.4635485682447222, - "nauc_cv_recall_at_1_std": 0.23949641642304445, - "nauc_cv_recall_at_20_diff1": 0.5231357787700573, - "nauc_cv_recall_at_20_max": 0.3204339578087355, - "nauc_cv_recall_at_20_std": 0.3921653472617399, - "nauc_cv_recall_at_3_diff1": 0.5830257867486661, - "nauc_cv_recall_at_3_max": 0.40642348260668365, - "nauc_cv_recall_at_3_std": 0.2693784583886975, - "nauc_cv_recall_at_5_diff1": 0.5673927390985861, - "nauc_cv_recall_at_5_max": 0.4032859392730803, - "nauc_cv_recall_at_5_std": 0.3144014157382476, - "nauc_map_at_1000_diff1": 0.6411599264391882, - "nauc_map_at_1000_max": 0.4289680674918197, - "nauc_map_at_1000_std": 0.2784491957470269, - "nauc_map_at_100_diff1": 0.6407788736651042, - "nauc_map_at_100_max": 0.4285537008083134, - "nauc_map_at_100_std": 0.27905660807460186, - "nauc_map_at_10_diff1": 0.6411016894025999, - "nauc_map_at_10_max": 0.43353902064107813, - "nauc_map_at_10_std": 0.2737477676391273, - "nauc_map_at_1_diff1": 0.7038058763022518, - "nauc_map_at_1_max": 0.4635485682447222, - "nauc_map_at_1_std": 0.23949641642304445, - "nauc_map_at_20_diff1": 0.6406624642001553, - "nauc_map_at_20_max": 0.43069966751305555, - "nauc_map_at_20_std": 0.27629602423470784, - "nauc_map_at_3_diff1": 0.6488533514998196, - "nauc_map_at_3_max": 0.4386996076405115, - "nauc_map_at_3_std": 0.2573084499673646, - "nauc_map_at_5_diff1": 0.6447966801619837, - "nauc_map_at_5_max": 0.4383966725195242, - "nauc_map_at_5_std": 0.26829757773957064, - "nauc_mrr_at_1000_diff1": 0.6411599264391882, - "nauc_mrr_at_1000_max": 0.4289680674918197, - "nauc_mrr_at_1000_std": 0.2784491957470269, - "nauc_mrr_at_100_diff1": 0.6407788736651042, - "nauc_mrr_at_100_max": 0.4285537008083134, - "nauc_mrr_at_100_std": 0.27905660807460186, - "nauc_mrr_at_10_diff1": 0.6411016894025999, - "nauc_mrr_at_10_max": 0.43353902064107813, - "nauc_mrr_at_10_std": 0.2737477676391273, - "nauc_mrr_at_1_diff1": 0.7038058763022518, - "nauc_mrr_at_1_max": 0.4635485682447222, - "nauc_mrr_at_1_std": 0.23949641642304445, - "nauc_mrr_at_20_diff1": 0.6406624642001553, - "nauc_mrr_at_20_max": 0.43069966751305555, - "nauc_mrr_at_20_std": 0.27629602423470784, - "nauc_mrr_at_3_diff1": 0.6488533514998196, - "nauc_mrr_at_3_max": 0.4386996076405115, - "nauc_mrr_at_3_std": 0.2573084499673646, - "nauc_mrr_at_5_diff1": 0.6447966801619837, - "nauc_mrr_at_5_max": 0.4383966725195242, - "nauc_mrr_at_5_std": 0.26829757773957064, - "nauc_ndcg_at_1000_diff1": 0.6256559860728634, - "nauc_ndcg_at_1000_max": 0.4109741897935761, - "nauc_ndcg_at_1000_std": 0.2988253387764826, - "nauc_ndcg_at_100_diff1": 0.6157338383229767, - "nauc_ndcg_at_100_max": 0.3930122773391341, - "nauc_ndcg_at_100_std": 0.3224691211574583, - "nauc_ndcg_at_10_diff1": 0.6148314521713852, - "nauc_ndcg_at_10_max": 0.4173940060757511, - "nauc_ndcg_at_10_std": 0.29279317802765203, - "nauc_ndcg_at_1_diff1": 0.7038058763022518, - "nauc_ndcg_at_1_max": 0.4635485682447222, - "nauc_ndcg_at_1_std": 0.23949641642304445, - "nauc_ndcg_at_20_diff1": 0.6127351090974947, - "nauc_ndcg_at_20_max": 0.4066455428917083, - "nauc_ndcg_at_20_std": 0.3020330912108028, - "nauc_ndcg_at_3_diff1": 0.6312889534711793, - "nauc_ndcg_at_3_max": 0.43013404859610777, - "nauc_ndcg_at_3_std": 0.260700720898194, - "nauc_ndcg_at_5_diff1": 0.6242322616276245, - "nauc_ndcg_at_5_max": 0.4291320833572419, - "nauc_ndcg_at_5_std": 0.28018555183852706, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.5126757026738261, - "nauc_precision_at_100_max": 0.18510105009062786, - "nauc_precision_at_100_std": 0.5888298680979758, - "nauc_precision_at_10_diff1": 0.5370004026652653, - "nauc_precision_at_10_max": 0.367351132675232, - "nauc_precision_at_10_std": 0.3517130623928041, - "nauc_precision_at_1_diff1": 0.7038058763022518, - "nauc_precision_at_1_max": 0.4635485682447222, - "nauc_precision_at_1_std": 0.23949641642304445, - "nauc_precision_at_20_diff1": 0.5231357787700573, - "nauc_precision_at_20_max": 0.3204339578087353, - "nauc_precision_at_20_std": 0.3921653472617399, - "nauc_precision_at_3_diff1": 0.5830257867486658, - "nauc_precision_at_3_max": 0.40642348260668354, - "nauc_precision_at_3_std": 0.2693784583886974, - "nauc_precision_at_5_diff1": 0.567392739098586, - "nauc_precision_at_5_max": 0.40328593927308015, - "nauc_precision_at_5_std": 0.31440141573824765, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5126757026738264, - "nauc_recall_at_100_max": 0.18510105009062797, - "nauc_recall_at_100_std": 0.5888298680979758, - "nauc_recall_at_10_diff1": 0.5370004026652652, - "nauc_recall_at_10_max": 0.36735113267523206, - "nauc_recall_at_10_std": 0.3517130623928045, - "nauc_recall_at_1_diff1": 0.7038058763022518, - "nauc_recall_at_1_max": 0.4635485682447222, - "nauc_recall_at_1_std": 0.23949641642304445, - "nauc_recall_at_20_diff1": 0.5231357787700573, - "nauc_recall_at_20_max": 0.3204339578087355, - "nauc_recall_at_20_std": 0.3921653472617399, - "nauc_recall_at_3_diff1": 0.5830257867486661, - "nauc_recall_at_3_max": 0.40642348260668365, - "nauc_recall_at_3_std": 0.2693784583886975, - "nauc_recall_at_5_diff1": 0.5673927390985861, - "nauc_recall_at_5_max": 0.4032859392730803, - "nauc_recall_at_5_std": 0.3144014157382476, - "ndcg_at_1": 0.25242, - "ndcg_at_10": 0.36077, - "ndcg_at_100": 0.41347, - "ndcg_at_1000": 0.4444, - "ndcg_at_20": 0.37786, - "ndcg_at_3": 0.31182, - "ndcg_at_5": 0.33775, - "precision_at_1": 0.25242, - "precision_at_10": 0.04909, - "precision_at_100": 0.00753, - "precision_at_1000": 0.001, - "precision_at_20": 0.02793, - "precision_at_3": 0.11887, - "precision_at_5": 0.084, - "recall_at_1": 0.25242, - "recall_at_10": 0.49087, - "recall_at_100": 0.75295, - "recall_at_1000": 1.0, - "recall_at_20": 0.55854, - "recall_at_3": 0.35661, - "recall_at_5": 0.41998 - }, - { - "cv_recall_at_1": 0.3, - "cv_recall_at_10": 0.604, - "cv_recall_at_100": 0.911, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.694, - "cv_recall_at_3": 0.443, - "cv_recall_at_5": 0.497, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.43895, - "map_at_1": 0.3, - "map_at_10": 0.38779, - "map_at_100": 0.3999, - "map_at_1000": 0.4004, - "map_at_20": 0.39416, - "map_at_3": 0.36117, - "map_at_5": 0.37342, - "mrr_at_1": 0.3, - "mrr_at_10": 0.387792857142857, - "mrr_at_100": 0.39990229179090303, - "mrr_at_1000": 0.40040463142649246, - "mrr_at_20": 0.3941621968459814, - "mrr_at_3": 0.36116666666666647, - "mrr_at_5": 0.3734166666666663, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.36345849201103536, - "nauc_cv_recall_at_100_max": 0.544057323303853, - "nauc_cv_recall_at_100_std": 0.4570022765660564, - "nauc_cv_recall_at_10_diff1": 0.4760467048899043, - "nauc_cv_recall_at_10_max": 0.14560452700238558, - "nauc_cv_recall_at_10_std": -0.18205012465398782, - "nauc_cv_recall_at_1_diff1": 0.618448858262397, - "nauc_cv_recall_at_1_max": 0.2139305499256856, - "nauc_cv_recall_at_1_std": -0.15225645183083375, - "nauc_cv_recall_at_20_diff1": 0.4649302232925785, - "nauc_cv_recall_at_20_max": 0.16381429399774458, - "nauc_cv_recall_at_20_std": -0.13788681339399703, - "nauc_cv_recall_at_3_diff1": 0.5061653239554741, - "nauc_cv_recall_at_3_max": 0.23292893058180955, - "nauc_cv_recall_at_3_std": -0.13830338958916558, - "nauc_cv_recall_at_5_diff1": 0.48439149248824326, - "nauc_cv_recall_at_5_max": 0.2202481335411537, - "nauc_cv_recall_at_5_std": -0.1419893010545874, - "nauc_map_at_1000_diff1": 0.5687810962778976, - "nauc_map_at_1000_max": 0.21659951806752817, - "nauc_map_at_1000_std": -0.1486960241996032, - "nauc_map_at_100_diff1": 0.5685672958859157, - "nauc_map_at_100_max": 0.21699988255049374, - "nauc_map_at_100_std": -0.148001824259214, - "nauc_map_at_10_diff1": 0.5668997355688797, - "nauc_map_at_10_max": 0.2117857851360944, - "nauc_map_at_10_std": -0.15404347629409737, - "nauc_map_at_1_diff1": 0.618448858262397, - "nauc_map_at_1_max": 0.2139305499256856, - "nauc_map_at_1_std": -0.15225645183083375, - "nauc_map_at_20_diff1": 0.5676571870804599, - "nauc_map_at_20_max": 0.21318841167959945, - "nauc_map_at_20_std": -0.15183481227253412, - "nauc_map_at_3_diff1": 0.5712598391235426, - "nauc_map_at_3_max": 0.22358983160989943, - "nauc_map_at_3_std": -0.14827136439311678, - "nauc_map_at_5_diff1": 0.5668118056008307, - "nauc_map_at_5_max": 0.22092235324327933, - "nauc_map_at_5_std": -0.1487400368543234, - "nauc_mrr_at_1000_diff1": 0.5687810962778976, - "nauc_mrr_at_1000_max": 0.21659951806752817, - "nauc_mrr_at_1000_std": -0.1486960241996032, - "nauc_mrr_at_100_diff1": 0.5685672958859157, - "nauc_mrr_at_100_max": 0.21699988255049374, - "nauc_mrr_at_100_std": -0.148001824259214, - "nauc_mrr_at_10_diff1": 0.5668997355688797, - "nauc_mrr_at_10_max": 0.2117857851360944, - "nauc_mrr_at_10_std": -0.15404347629409737, - "nauc_mrr_at_1_diff1": 0.618448858262397, - "nauc_mrr_at_1_max": 0.2139305499256856, - "nauc_mrr_at_1_std": -0.15225645183083375, - "nauc_mrr_at_20_diff1": 0.5676571870804599, - "nauc_mrr_at_20_max": 0.21318841167959945, - "nauc_mrr_at_20_std": -0.15183481227253412, - "nauc_mrr_at_3_diff1": 0.5712598391235426, - "nauc_mrr_at_3_max": 0.22358983160989943, - "nauc_mrr_at_3_std": -0.14827136439311678, - "nauc_mrr_at_5_diff1": 0.5668118056008307, - "nauc_mrr_at_5_max": 0.22092235324327933, - "nauc_mrr_at_5_std": -0.1487400368543234, - "nauc_ndcg_at_1000_diff1": 0.5577030015084231, - "nauc_ndcg_at_1000_max": 0.21916185280007355, - "nauc_ndcg_at_1000_std": -0.14041310480318414, - "nauc_ndcg_at_100_diff1": 0.5505621433052558, - "nauc_ndcg_at_100_max": 0.2316041550327303, - "nauc_ndcg_at_100_std": -0.11784099812524322, - "nauc_ndcg_at_10_diff1": 0.5457623705560521, - "nauc_ndcg_at_10_max": 0.1986485025025215, - "nauc_ndcg_at_10_std": -0.1595026539038139, - "nauc_ndcg_at_1_diff1": 0.618448858262397, - "nauc_ndcg_at_1_max": 0.2139305499256856, - "nauc_ndcg_at_1_std": -0.15225645183083375, - "nauc_ndcg_at_20_diff1": 0.5467444705576753, - "nauc_ndcg_at_20_max": 0.2041154912575809, - "nauc_ndcg_at_20_std": -0.15017440615387506, - "nauc_ndcg_at_3_diff1": 0.5544831112496268, - "nauc_ndcg_at_3_max": 0.226063809860779, - "nauc_ndcg_at_3_std": -0.1457775189902536, - "nauc_ndcg_at_5_diff1": 0.5461086332262867, - "nauc_ndcg_at_5_max": 0.22102516107083806, - "nauc_ndcg_at_5_std": -0.1469887491451484, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.3634584920110345, - "nauc_precision_at_100_max": 0.5440573233038528, - "nauc_precision_at_100_std": 0.45700227656605796, - "nauc_precision_at_10_diff1": 0.4760467048899036, - "nauc_precision_at_10_max": 0.145604527002385, - "nauc_precision_at_10_std": -0.18205012465398845, - "nauc_precision_at_1_diff1": 0.618448858262397, - "nauc_precision_at_1_max": 0.2139305499256856, - "nauc_precision_at_1_std": -0.15225645183083375, - "nauc_precision_at_20_diff1": 0.46493022329257805, - "nauc_precision_at_20_max": 0.16381429399774428, - "nauc_precision_at_20_std": -0.1378868133939973, - "nauc_precision_at_3_diff1": 0.5061653239554742, - "nauc_precision_at_3_max": 0.23292893058180972, - "nauc_precision_at_3_std": -0.13830338958916566, - "nauc_precision_at_5_diff1": 0.4843914924882435, - "nauc_precision_at_5_max": 0.22024813354115405, - "nauc_precision_at_5_std": -0.14198930105458732, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.36345849201103536, - "nauc_recall_at_100_max": 0.544057323303853, - "nauc_recall_at_100_std": 0.4570022765660564, - "nauc_recall_at_10_diff1": 0.4760467048899043, - "nauc_recall_at_10_max": 0.14560452700238558, - "nauc_recall_at_10_std": -0.18205012465398782, - "nauc_recall_at_1_diff1": 0.618448858262397, - "nauc_recall_at_1_max": 0.2139305499256856, - "nauc_recall_at_1_std": -0.15225645183083375, - "nauc_recall_at_20_diff1": 0.4649302232925785, - "nauc_recall_at_20_max": 0.16381429399774458, - "nauc_recall_at_20_std": -0.13788681339399703, - "nauc_recall_at_3_diff1": 0.5061653239554741, - "nauc_recall_at_3_max": 0.23292893058180955, - "nauc_recall_at_3_std": -0.13830338958916558, - "nauc_recall_at_5_diff1": 0.48439149248824326, - "nauc_recall_at_5_max": 0.2202481335411537, - "nauc_recall_at_5_std": -0.1419893010545874, - "ndcg_at_1": 0.3, - "ndcg_at_10": 0.43895, - "ndcg_at_100": 0.50257, - "ndcg_at_1000": 0.51426, - "ndcg_at_20": 0.46186, - "ndcg_at_3": 0.38211, - "ndcg_at_5": 0.40427, - "precision_at_1": 0.3, - "precision_at_10": 0.0604, - "precision_at_100": 0.00911, - "precision_at_1000": 0.001, - "precision_at_20": 0.0347, - "precision_at_3": 0.14767, - "precision_at_5": 0.0994, - "recall_at_1": 0.3, - "recall_at_10": 0.604, - "recall_at_100": 0.911, - "recall_at_1000": 1.0, - "recall_at_20": 0.694, - "recall_at_3": 0.443, - "recall_at_5": 0.497 - }, - { - "cv_recall_at_1": 0.36616, - "cv_recall_at_10": 0.681, - "cv_recall_at_100": 0.90291, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.75312, - "cv_recall_at_3": 0.51456, - "cv_recall_at_5": 0.58114, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.5121, - "map_at_1": 0.36616, - "map_at_10": 0.45946, - "map_at_100": 0.46798, - "map_at_1000": 0.46848, - "map_at_20": 0.46429, - "map_at_3": 0.43088, - "map_at_5": 0.44607, - "mrr_at_1": 0.36615811373092927, - "mrr_at_10": 0.459455562160139, - "mrr_at_100": 0.46798337728849837, - "mrr_at_1000": 0.4684780705303624, - "mrr_at_20": 0.46429064697191796, - "mrr_at_3": 0.4308830328247804, - "mrr_at_5": 0.4460702727693017, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5412854653754147, - "nauc_cv_recall_at_100_max": 0.6184196906683408, - "nauc_cv_recall_at_100_std": 0.5350992879255435, - "nauc_cv_recall_at_10_diff1": 0.5393654107724762, - "nauc_cv_recall_at_10_max": 0.5480476304207523, - "nauc_cv_recall_at_10_std": 0.31205880886496196, - "nauc_cv_recall_at_1_diff1": 0.6942671543678366, - "nauc_cv_recall_at_1_max": 0.5683430205647082, - "nauc_cv_recall_at_1_std": 0.11179006846134261, - "nauc_cv_recall_at_20_diff1": 0.5364223430251741, - "nauc_cv_recall_at_20_max": 0.547757810534148, - "nauc_cv_recall_at_20_std": 0.37293722690682907, - "nauc_cv_recall_at_3_diff1": 0.5898032624856843, - "nauc_cv_recall_at_3_max": 0.5381986930768126, - "nauc_cv_recall_at_3_std": 0.20251897425233206, - "nauc_cv_recall_at_5_diff1": 0.5594173118504862, - "nauc_cv_recall_at_5_max": 0.5450843125766591, - "nauc_cv_recall_at_5_std": 0.26971020091218706, - "nauc_map_at_1000_diff1": 0.6418669842366795, - "nauc_map_at_1000_max": 0.555669417389876, - "nauc_map_at_1000_std": 0.16327461338547425, - "nauc_map_at_100_diff1": 0.641654783339766, - "nauc_map_at_100_max": 0.5557261032650354, - "nauc_map_at_100_std": 0.16371740784880098, - "nauc_map_at_10_diff1": 0.6399662792835568, - "nauc_map_at_10_max": 0.5551146107747434, - "nauc_map_at_10_std": 0.16320993121786914, - "nauc_map_at_1_diff1": 0.6942671543678366, - "nauc_map_at_1_max": 0.5683430205647082, - "nauc_map_at_1_std": 0.11179006846134261, - "nauc_map_at_20_diff1": 0.6407387457589852, - "nauc_map_at_20_max": 0.5550219526965132, - "nauc_map_at_20_std": 0.16405741588426367, - "nauc_map_at_3_diff1": 0.646333023545593, - "nauc_map_at_3_max": 0.5532091489280436, - "nauc_map_at_3_std": 0.1476639087107855, - "nauc_map_at_5_diff1": 0.6412698510592656, - "nauc_map_at_5_max": 0.5553110808740862, - "nauc_map_at_5_std": 0.1611907197031228, - "nauc_mrr_at_1000_diff1": 0.6418669842366795, - "nauc_mrr_at_1000_max": 0.555669417389876, - "nauc_mrr_at_1000_std": 0.16327461338547425, - "nauc_mrr_at_100_diff1": 0.641654783339766, - "nauc_mrr_at_100_max": 0.5557261032650354, - "nauc_mrr_at_100_std": 0.16371740784880098, - "nauc_mrr_at_10_diff1": 0.6399662792835568, - "nauc_mrr_at_10_max": 0.5551146107747434, - "nauc_mrr_at_10_std": 0.16320993121786914, - "nauc_mrr_at_1_diff1": 0.6942671543678366, - "nauc_mrr_at_1_max": 0.5683430205647082, - "nauc_mrr_at_1_std": 0.11179006846134261, - "nauc_mrr_at_20_diff1": 0.6407387457589852, - "nauc_mrr_at_20_max": 0.5550219526965132, - "nauc_mrr_at_20_std": 0.16405741588426367, - "nauc_mrr_at_3_diff1": 0.646333023545593, - "nauc_mrr_at_3_max": 0.5532091489280436, - "nauc_mrr_at_3_std": 0.1476639087107855, - "nauc_mrr_at_5_diff1": 0.6412698510592656, - "nauc_mrr_at_5_max": 0.5553110808740862, - "nauc_mrr_at_5_std": 0.1611907197031228, - "nauc_ndcg_at_1000_diff1": 0.6323074138798798, - "nauc_ndcg_at_1000_max": 0.5556067762911258, - "nauc_ndcg_at_1000_std": 0.1816086842456634, - "nauc_ndcg_at_100_diff1": 0.6279765940181444, - "nauc_ndcg_at_100_max": 0.5580944570852119, - "nauc_ndcg_at_100_std": 0.1962220514043483, - "nauc_ndcg_at_10_diff1": 0.6187016187672618, - "nauc_ndcg_at_10_max": 0.553114482711755, - "nauc_ndcg_at_10_std": 0.1932531729558633, - "nauc_ndcg_at_1_diff1": 0.6942671543678366, - "nauc_ndcg_at_1_max": 0.5683430205647082, - "nauc_ndcg_at_1_std": 0.11179006846134261, - "nauc_ndcg_at_20_diff1": 0.6211872614610904, - "nauc_ndcg_at_20_max": 0.5530597590465678, - "nauc_ndcg_at_20_std": 0.19883341794091794, - "nauc_ndcg_at_3_diff1": 0.6319328518327908, - "nauc_ndcg_at_3_max": 0.549327389660898, - "nauc_ndcg_at_3_std": 0.16138071986818298, - "nauc_ndcg_at_5_diff1": 0.621779492307214, - "nauc_ndcg_at_5_max": 0.5526865020578606, - "nauc_ndcg_at_5_std": 0.18642976010409754, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.5412854653754162, - "nauc_precision_at_100_max": 0.6184196906683425, - "nauc_precision_at_100_std": 0.5350992879255447, - "nauc_precision_at_10_diff1": 0.5393654107724762, - "nauc_precision_at_10_max": 0.5480476304207519, - "nauc_precision_at_10_std": 0.31205880886496185, - "nauc_precision_at_1_diff1": 0.6942671543678366, - "nauc_precision_at_1_max": 0.5683430205647082, - "nauc_precision_at_1_std": 0.11179006846134261, - "nauc_precision_at_20_diff1": 0.5364223430251746, - "nauc_precision_at_20_max": 0.5477578105341474, - "nauc_precision_at_20_std": 0.3729372269068284, - "nauc_precision_at_3_diff1": 0.589803262485684, - "nauc_precision_at_3_max": 0.5381986930768128, - "nauc_precision_at_3_std": 0.20251897425233165, - "nauc_precision_at_5_diff1": 0.559417311850486, - "nauc_precision_at_5_max": 0.5450843125766585, - "nauc_precision_at_5_std": 0.26971020091218756, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5412854653754147, - "nauc_recall_at_100_max": 0.6184196906683408, - "nauc_recall_at_100_std": 0.5350992879255435, - "nauc_recall_at_10_diff1": 0.5393654107724762, - "nauc_recall_at_10_max": 0.5480476304207523, - "nauc_recall_at_10_std": 0.31205880886496196, - "nauc_recall_at_1_diff1": 0.6942671543678366, - "nauc_recall_at_1_max": 0.5683430205647082, - "nauc_recall_at_1_std": 0.11179006846134261, - "nauc_recall_at_20_diff1": 0.5364223430251741, - "nauc_recall_at_20_max": 0.547757810534148, - "nauc_recall_at_20_std": 0.37293722690682907, - "nauc_recall_at_3_diff1": 0.5898032624856843, - "nauc_recall_at_3_max": 0.5381986930768126, - "nauc_recall_at_3_std": 0.20251897425233206, - "nauc_recall_at_5_diff1": 0.5594173118504862, - "nauc_recall_at_5_max": 0.5450843125766591, - "nauc_recall_at_5_std": 0.26971020091218706, - "ndcg_at_1": 0.36616, - "ndcg_at_10": 0.5121, - "ndcg_at_100": 0.55758, - "ndcg_at_1000": 0.57013, - "ndcg_at_20": 0.53012, - "ndcg_at_3": 0.45235, - "ndcg_at_5": 0.47974, - "precision_at_1": 0.36616, - "precision_at_10": 0.0681, - "precision_at_100": 0.00903, - "precision_at_1000": 0.001, - "precision_at_20": 0.03766, - "precision_at_3": 0.17152, - "precision_at_5": 0.11623, - "recall_at_1": 0.36616, - "recall_at_10": 0.681, - "recall_at_100": 0.90291, - "recall_at_1000": 1.0, - "recall_at_20": 0.75312, - "recall_at_3": 0.51456, - "recall_at_5": 0.58114 - }, - { - "cv_recall_at_1": 0.44926, - "cv_recall_at_10": 0.72304, - "cv_recall_at_100": 0.92495, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.78541, - "cv_recall_at_3": 0.59408, - "cv_recall_at_5": 0.65328, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.58154, - "map_at_1": 0.44926, - "map_at_10": 0.53678, - "map_at_100": 0.5448, - "map_at_1000": 0.54521, - "map_at_20": 0.5411, - "map_at_3": 0.51374, - "map_at_5": 0.52733, - "mrr_at_1": 0.4492600422832981, - "mrr_at_10": 0.5367776603241715, - "mrr_at_100": 0.5448034191893545, - "mrr_at_1000": 0.5452066948845224, - "mrr_at_20": 0.5411048749935305, - "mrr_at_3": 0.5137420718816064, - "mrr_at_5": 0.5273255813953482, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.5604890359223188, - "nauc_cv_recall_at_100_max": 0.600611877989979, - "nauc_cv_recall_at_100_std": 0.5128565005478855, - "nauc_cv_recall_at_10_diff1": 0.5478845958212548, - "nauc_cv_recall_at_10_max": 0.5569566649688461, - "nauc_cv_recall_at_10_std": 0.2380854206882242, - "nauc_cv_recall_at_1_diff1": 0.7076954521901431, - "nauc_cv_recall_at_1_max": 0.6081804572874958, - "nauc_cv_recall_at_1_std": 0.18855092140607752, - "nauc_cv_recall_at_20_diff1": 0.519804538332009, - "nauc_cv_recall_at_20_max": 0.5699259306994402, - "nauc_cv_recall_at_20_std": 0.3267690220002922, - "nauc_cv_recall_at_3_diff1": 0.6196183890984953, - "nauc_cv_recall_at_3_max": 0.5625131608580892, - "nauc_cv_recall_at_3_std": 0.18026450762612506, - "nauc_cv_recall_at_5_diff1": 0.5837624234626474, - "nauc_cv_recall_at_5_max": 0.5955172595825997, - "nauc_cv_recall_at_5_std": 0.2345211583004191, - "nauc_map_at_1000_diff1": 0.662767027607681, - "nauc_map_at_1000_max": 0.5891614953909932, - "nauc_map_at_1000_std": 0.19271138919076922, - "nauc_map_at_100_diff1": 0.6626694252918228, - "nauc_map_at_100_max": 0.5891853733373635, - "nauc_map_at_100_std": 0.1931273640612953, - "nauc_map_at_10_diff1": 0.6611057630068342, - "nauc_map_at_10_max": 0.5885808385362452, - "nauc_map_at_10_std": 0.19038392512862137, - "nauc_map_at_1_diff1": 0.7076954521901431, - "nauc_map_at_1_max": 0.6081804572874958, - "nauc_map_at_1_std": 0.18855092140607752, - "nauc_map_at_20_diff1": 0.6611052423433593, - "nauc_map_at_20_max": 0.589097186728387, - "nauc_map_at_20_std": 0.19301781252750633, - "nauc_map_at_3_diff1": 0.6698031561319879, - "nauc_map_at_3_max": 0.5864773804656612, - "nauc_map_at_3_std": 0.18035627546412655, - "nauc_map_at_5_diff1": 0.6634047508034983, - "nauc_map_at_5_max": 0.5925821858489451, - "nauc_map_at_5_std": 0.19006959491476397, - "nauc_mrr_at_1000_diff1": 0.662767027607681, - "nauc_mrr_at_1000_max": 0.5891614953909932, - "nauc_mrr_at_1000_std": 0.19271138919076922, - "nauc_mrr_at_100_diff1": 0.6626694252918228, - "nauc_mrr_at_100_max": 0.5891853733373635, - "nauc_mrr_at_100_std": 0.1931273640612953, - "nauc_mrr_at_10_diff1": 0.6611057630068342, - "nauc_mrr_at_10_max": 0.5885808385362452, - "nauc_mrr_at_10_std": 0.19038392512862137, - "nauc_mrr_at_1_diff1": 0.7076954521901431, - "nauc_mrr_at_1_max": 0.6081804572874958, - "nauc_mrr_at_1_std": 0.18855092140607752, - "nauc_mrr_at_20_diff1": 0.6611052423433593, - "nauc_mrr_at_20_max": 0.589097186728387, - "nauc_mrr_at_20_std": 0.19301781252750633, - "nauc_mrr_at_3_diff1": 0.6698031561319879, - "nauc_mrr_at_3_max": 0.5864773804656612, - "nauc_mrr_at_3_std": 0.18035627546412655, - "nauc_mrr_at_5_diff1": 0.6634047508034983, - "nauc_mrr_at_5_max": 0.5925821858489451, - "nauc_mrr_at_5_std": 0.19006959491476397, - "nauc_ndcg_at_1000_diff1": 0.6527791339329123, - "nauc_ndcg_at_1000_max": 0.5866662271791159, - "nauc_ndcg_at_1000_std": 0.2011792752392176, - "nauc_ndcg_at_100_diff1": 0.6496398779949115, - "nauc_ndcg_at_100_max": 0.587267185991599, - "nauc_ndcg_at_100_std": 0.2124728126641152, - "nauc_ndcg_at_10_diff1": 0.6389497640400255, - "nauc_ndcg_at_10_max": 0.5823909084092497, - "nauc_ndcg_at_10_std": 0.1992511948037301, - "nauc_ndcg_at_1_diff1": 0.7076954521901431, - "nauc_ndcg_at_1_max": 0.6081804572874958, - "nauc_ndcg_at_1_std": 0.18855092140607752, - "nauc_ndcg_at_20_diff1": 0.6378501985977086, - "nauc_ndcg_at_20_max": 0.5850990776613686, - "nauc_ndcg_at_20_std": 0.2118455632925891, - "nauc_ndcg_at_3_diff1": 0.6574992976045798, - "nauc_ndcg_at_3_max": 0.5805105572521726, - "nauc_ndcg_at_3_std": 0.18012778697797263, - "nauc_ndcg_at_5_diff1": 0.6454889254857838, - "nauc_ndcg_at_5_max": 0.5925209317588707, - "nauc_ndcg_at_5_std": 0.19914914490304186, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.5604890359223192, - "nauc_precision_at_100_max": 0.6006118779899806, - "nauc_precision_at_100_std": 0.5128565005478841, - "nauc_precision_at_10_diff1": 0.5478845958212546, - "nauc_precision_at_10_max": 0.5569566649688457, - "nauc_precision_at_10_std": 0.23808542068822366, - "nauc_precision_at_1_diff1": 0.7076954521901431, - "nauc_precision_at_1_max": 0.6081804572874958, - "nauc_precision_at_1_std": 0.18855092140607752, - "nauc_precision_at_20_diff1": 0.5198045383320089, - "nauc_precision_at_20_max": 0.5699259306994408, - "nauc_precision_at_20_std": 0.32676902200029057, - "nauc_precision_at_3_diff1": 0.6196183890984959, - "nauc_precision_at_3_max": 0.5625131608580891, - "nauc_precision_at_3_std": 0.18026450762612492, - "nauc_precision_at_5_diff1": 0.5837624234626474, - "nauc_precision_at_5_max": 0.5955172595825996, - "nauc_precision_at_5_std": 0.23452115830041848, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.5604890359223188, - "nauc_recall_at_100_max": 0.600611877989979, - "nauc_recall_at_100_std": 0.5128565005478855, - "nauc_recall_at_10_diff1": 0.5478845958212548, - "nauc_recall_at_10_max": 0.5569566649688461, - "nauc_recall_at_10_std": 0.2380854206882242, - "nauc_recall_at_1_diff1": 0.7076954521901431, - "nauc_recall_at_1_max": 0.6081804572874958, - "nauc_recall_at_1_std": 0.18855092140607752, - "nauc_recall_at_20_diff1": 0.519804538332009, - "nauc_recall_at_20_max": 0.5699259306994402, - "nauc_recall_at_20_std": 0.3267690220002922, - "nauc_recall_at_3_diff1": 0.6196183890984953, - "nauc_recall_at_3_max": 0.5625131608580892, - "nauc_recall_at_3_std": 0.18026450762612506, - "nauc_recall_at_5_diff1": 0.5837624234626474, - "nauc_recall_at_5_max": 0.5955172595825997, - "nauc_recall_at_5_std": 0.2345211583004191, - "ndcg_at_1": 0.44926, - "ndcg_at_10": 0.58154, - "ndcg_at_100": 0.62338, - "ndcg_at_1000": 0.63315, - "ndcg_at_20": 0.59731, - "ndcg_at_3": 0.5344, - "ndcg_at_5": 0.55883, - "precision_at_1": 0.44926, - "precision_at_10": 0.0723, - "precision_at_100": 0.00925, - "precision_at_1000": 0.001, - "precision_at_20": 0.03927, - "precision_at_3": 0.19803, - "precision_at_5": 0.13066, - "recall_at_1": 0.44926, - "recall_at_10": 0.72304, - "recall_at_100": 0.92495, - "recall_at_1000": 1.0, - "recall_at_20": 0.78541, - "recall_at_3": 0.59408, - "recall_at_5": 0.65328 - }, - { - "cv_recall_at_1": 0.517, - "cv_recall_at_10": 0.781, - "cv_recall_at_100": 0.971, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.827, - "cv_recall_at_3": 0.659, - "cv_recall_at_5": 0.722, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.64447, - "map_at_1": 0.517, - "map_at_10": 0.60113, - "map_at_100": 0.608, - "map_at_1000": 0.60816, - "map_at_20": 0.60429, - "map_at_3": 0.57867, - "map_at_5": 0.59307, - "mrr_at_1": 0.517, - "mrr_at_10": 0.6011313492063491, - "mrr_at_100": 0.6079956742603368, - "mrr_at_1000": 0.6081601475225663, - "mrr_at_20": 0.6042893008694089, - "mrr_at_3": 0.5786666666666664, - "mrr_at_5": 0.5930666666666664, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.2455004990501933, - "nauc_cv_recall_at_100_max": 0.777117743649181, - "nauc_cv_recall_at_100_std": 0.580137802247331, - "nauc_cv_recall_at_10_diff1": 0.6447662014039314, - "nauc_cv_recall_at_10_max": 0.09860095441293794, - "nauc_cv_recall_at_10_std": -0.5034772553298917, - "nauc_cv_recall_at_1_diff1": 0.751608559639086, - "nauc_cv_recall_at_1_max": 0.312891122377997, - "nauc_cv_recall_at_1_std": -0.21080843597152954, - "nauc_cv_recall_at_20_diff1": 0.6256890376607283, - "nauc_cv_recall_at_20_max": 0.13114136053877856, - "nauc_cv_recall_at_20_std": -0.4660522212003668, - "nauc_cv_recall_at_3_diff1": 0.6906157507780316, - "nauc_cv_recall_at_3_max": 0.23405883395415814, - "nauc_cv_recall_at_3_std": -0.35375070184359253, - "nauc_cv_recall_at_5_diff1": 0.6616474688427189, - "nauc_cv_recall_at_5_max": 0.15739656780522973, - "nauc_cv_recall_at_5_std": -0.42995129215876704, - "nauc_map_at_1000_diff1": 0.7218586514551526, - "nauc_map_at_1000_max": 0.276378975392984, - "nauc_map_at_1000_std": -0.2741734105052388, - "nauc_map_at_100_diff1": 0.7215779428420441, - "nauc_map_at_100_max": 0.2766641973079267, - "nauc_map_at_100_std": -0.27366533367406815, - "nauc_map_at_10_diff1": 0.7217438447391757, - "nauc_map_at_10_max": 0.2703046067353741, - "nauc_map_at_10_std": -0.2820292800514403, - "nauc_map_at_1_diff1": 0.751608559639086, - "nauc_map_at_1_max": 0.312891122377997, - "nauc_map_at_1_std": -0.21080843597152954, - "nauc_map_at_20_diff1": 0.7217143483142994, - "nauc_map_at_20_max": 0.27281254198769883, - "nauc_map_at_20_std": -0.2790188913967356, - "nauc_map_at_3_diff1": 0.7261837934787818, - "nauc_map_at_3_max": 0.2859687992340636, - "nauc_map_at_3_std": -0.2664707654182384, - "nauc_map_at_5_diff1": 0.7220012985109051, - "nauc_map_at_5_max": 0.2736003653205686, - "nauc_map_at_5_std": -0.2780137338090851, - "nauc_mrr_at_1000_diff1": 0.7218586514551526, - "nauc_mrr_at_1000_max": 0.276378975392984, - "nauc_mrr_at_1000_std": -0.2741734105052388, - "nauc_mrr_at_100_diff1": 0.7215779428420441, - "nauc_mrr_at_100_max": 0.2766641973079267, - "nauc_mrr_at_100_std": -0.27366533367406815, - "nauc_mrr_at_10_diff1": 0.7217438447391757, - "nauc_mrr_at_10_max": 0.2703046067353741, - "nauc_mrr_at_10_std": -0.2820292800514403, - "nauc_mrr_at_1_diff1": 0.751608559639086, - "nauc_mrr_at_1_max": 0.312891122377997, - "nauc_mrr_at_1_std": -0.21080843597152954, - "nauc_mrr_at_20_diff1": 0.7217143483142994, - "nauc_mrr_at_20_max": 0.27281254198769883, - "nauc_mrr_at_20_std": -0.2790188913967356, - "nauc_mrr_at_3_diff1": 0.7261837934787818, - "nauc_mrr_at_3_max": 0.2859687992340636, - "nauc_mrr_at_3_std": -0.2664707654182384, - "nauc_mrr_at_5_diff1": 0.7220012985109051, - "nauc_mrr_at_5_max": 0.2736003653205686, - "nauc_mrr_at_5_std": -0.2780137338090851, - "nauc_ndcg_at_1000_diff1": 0.7137999194741065, - "nauc_ndcg_at_1000_max": 0.2702799148084682, - "nauc_ndcg_at_1000_std": -0.28243336701775806, - "nauc_ndcg_at_100_diff1": 0.7063295481485343, - "nauc_ndcg_at_100_max": 0.27827257821700485, - "nauc_ndcg_at_100_std": -0.26866795728820436, - "nauc_ndcg_at_10_diff1": 0.7078987305730228, - "nauc_ndcg_at_10_max": 0.2404606593292584, - "nauc_ndcg_at_10_std": -0.32119201147324616, - "nauc_ndcg_at_1_diff1": 0.751608559639086, - "nauc_ndcg_at_1_max": 0.312891122377997, - "nauc_ndcg_at_1_std": -0.21080843597152954, - "nauc_ndcg_at_20_diff1": 0.7072336259985014, - "nauc_ndcg_at_20_max": 0.25013100556604273, - "nauc_ndcg_at_20_std": -0.30950587073132424, - "nauc_ndcg_at_3_diff1": 0.7177845158741234, - "nauc_ndcg_at_3_max": 0.2739795946205384, - "nauc_ndcg_at_3_std": -0.28690383831825556, - "nauc_ndcg_at_5_diff1": 0.7093666031080256, - "nauc_ndcg_at_5_max": 0.24987529421961602, - "nauc_ndcg_at_5_std": -0.3096330840540708, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.24550049905018675, - "nauc_precision_at_100_max": 0.7771177436491825, - "nauc_precision_at_100_std": 0.5801378022473334, - "nauc_precision_at_10_diff1": 0.6447662014039321, - "nauc_precision_at_10_max": 0.09860095441293745, - "nauc_precision_at_10_std": -0.5034772553298922, - "nauc_precision_at_1_diff1": 0.751608559639086, - "nauc_precision_at_1_max": 0.312891122377997, - "nauc_precision_at_1_std": -0.21080843597152954, - "nauc_precision_at_20_diff1": 0.6256890376607267, - "nauc_precision_at_20_max": 0.13114136053877531, - "nauc_precision_at_20_std": -0.46605222120036965, - "nauc_precision_at_3_diff1": 0.6906157507780325, - "nauc_precision_at_3_max": 0.23405883395415797, - "nauc_precision_at_3_std": -0.3537507018435914, - "nauc_precision_at_5_diff1": 0.661647468842718, - "nauc_precision_at_5_max": 0.15739656780522981, - "nauc_precision_at_5_std": -0.42995129215876715, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.2455004990501933, - "nauc_recall_at_100_max": 0.777117743649181, - "nauc_recall_at_100_std": 0.580137802247331, - "nauc_recall_at_10_diff1": 0.6447662014039314, - "nauc_recall_at_10_max": 0.09860095441293794, - "nauc_recall_at_10_std": -0.5034772553298917, - "nauc_recall_at_1_diff1": 0.751608559639086, - "nauc_recall_at_1_max": 0.312891122377997, - "nauc_recall_at_1_std": -0.21080843597152954, - "nauc_recall_at_20_diff1": 0.6256890376607283, - "nauc_recall_at_20_max": 0.13114136053877856, - "nauc_recall_at_20_std": -0.4660522212003668, - "nauc_recall_at_3_diff1": 0.6906157507780316, - "nauc_recall_at_3_max": 0.23405883395415814, - "nauc_recall_at_3_std": -0.35375070184359253, - "nauc_recall_at_5_diff1": 0.6616474688427189, - "nauc_recall_at_5_max": 0.15739656780522973, - "nauc_recall_at_5_std": -0.42995129215876704, - "ndcg_at_1": 0.517, - "ndcg_at_10": 0.64447, - "ndcg_at_100": 0.68284, - "ndcg_at_1000": 0.68668, - "ndcg_at_20": 0.65606, - "ndcg_at_3": 0.59926, - "ndcg_at_5": 0.62521, - "precision_at_1": 0.517, - "precision_at_10": 0.0781, - "precision_at_100": 0.00971, - "precision_at_1000": 0.001, - "precision_at_20": 0.04135, - "precision_at_3": 0.21967, - "precision_at_5": 0.1444, - "recall_at_1": 0.517, - "recall_at_10": 0.781, - "recall_at_100": 0.971, - "recall_at_1000": 1.0, - "recall_at_20": 0.827, - "recall_at_3": 0.659, - "recall_at_5": 0.722 - } - ] - }, - "task_name": "WITT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/WebQAT2ITRetrieval.json b/results-mieb/voyage-multimodal-3/1/WebQAT2ITRetrieval.json deleted file mode 100644 index bc7a9a6655..0000000000 --- a/results-mieb/voyage-multimodal-3/1/WebQAT2ITRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "53db4c9f9c93cb74926a1c9d04dea7d7acac2f21", - "evaluation_time": 53264.06916642189, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.51971, - "cv_recall_at_10": 0.88371, - "cv_recall_at_100": 0.98606, - "cv_recall_at_1000": 0.99801, - "cv_recall_at_20": 0.93986, - "cv_recall_at_3": 0.71446, - "cv_recall_at_5": 0.7965, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.65641, - "map_at_1": 0.41219, - "map_at_10": 0.5806, - "map_at_100": 0.59162, - "map_at_1000": 0.59181, - "map_at_20": 0.58823, - "map_at_3": 0.53322, - "map_at_5": 0.5607, - "mrr_at_1": 0.5197132616487455, - "mrr_at_10": 0.6376906689929384, - "mrr_at_100": 0.6430317621020902, - "mrr_at_1000": 0.6430981364740875, - "mrr_at_20": 0.6417126087487943, - "mrr_at_3": 0.6069958847736621, - "mrr_at_5": 0.6257334395327226, - "nauc_cv_recall_at_1000_diff1": -0.16359871061056505, - "nauc_cv_recall_at_1000_max": 0.9738573659551857, - "nauc_cv_recall_at_1000_std": 0.8716355426212542, - "nauc_cv_recall_at_100_diff1": 0.37040443651434385, - "nauc_cv_recall_at_100_max": 0.5536562443724028, - "nauc_cv_recall_at_100_std": -0.2029133108392488, - "nauc_cv_recall_at_10_diff1": 0.4373814649783142, - "nauc_cv_recall_at_10_max": 0.2623278235230551, - "nauc_cv_recall_at_10_std": -0.4777458069030308, - "nauc_cv_recall_at_1_diff1": 0.6296700135608259, - "nauc_cv_recall_at_1_max": 0.18595989336420365, - "nauc_cv_recall_at_1_std": -0.308000860424921, - "nauc_cv_recall_at_20_diff1": 0.44396751777858534, - "nauc_cv_recall_at_20_max": 0.37139002202609384, - "nauc_cv_recall_at_20_std": -0.46416365148674804, - "nauc_cv_recall_at_3_diff1": 0.5250228583228617, - "nauc_cv_recall_at_3_max": 0.21452132324025366, - "nauc_cv_recall_at_3_std": -0.376613676516204, - "nauc_cv_recall_at_5_diff1": 0.48969921934185057, - "nauc_cv_recall_at_5_max": 0.2201012342193106, - "nauc_cv_recall_at_5_std": -0.3966907506361567, - "nauc_map_at_1000_diff1": 0.5606927411639652, - "nauc_map_at_1000_max": 0.23695502946687824, - "nauc_map_at_1000_std": -0.3219725297843542, - "nauc_map_at_100_diff1": 0.560601290642645, - "nauc_map_at_100_max": 0.2369761010127848, - "nauc_map_at_100_std": -0.322177283851876, - "nauc_map_at_10_diff1": 0.557362556735293, - "nauc_map_at_10_max": 0.23837454920688683, - "nauc_map_at_10_std": -0.32505856270238065, - "nauc_map_at_1_diff1": 0.5956803415962526, - "nauc_map_at_1_max": 0.2511508848721171, - "nauc_map_at_1_std": -0.2533533014832028, - "nauc_map_at_20_diff1": 0.5593378056108329, - "nauc_map_at_20_max": 0.2378486279109965, - "nauc_map_at_20_std": -0.3236293166425821, - "nauc_map_at_3_diff1": 0.5639181501834886, - "nauc_map_at_3_max": 0.24187972587351358, - "nauc_map_at_3_std": -0.31665507581977087, - "nauc_map_at_5_diff1": 0.5580745238384635, - "nauc_map_at_5_max": 0.23962606930376834, - "nauc_map_at_5_std": -0.31997561410739733, - "nauc_mrr_at_1000_diff1": 0.5885577140835274, - "nauc_mrr_at_1000_max": 0.19740465284088993, - "nauc_mrr_at_1000_std": -0.33687489687989824, - "nauc_mrr_at_100_diff1": 0.5885256709976912, - "nauc_mrr_at_100_max": 0.19746031034257694, - "nauc_mrr_at_100_std": -0.3369123512013644, - "nauc_mrr_at_10_diff1": 0.5857089403879366, - "nauc_mrr_at_10_max": 0.1971397736486086, - "nauc_mrr_at_10_std": -0.3400377337570375, - "nauc_mrr_at_1_diff1": 0.6296700135608259, - "nauc_mrr_at_1_max": 0.18595989336420365, - "nauc_mrr_at_1_std": -0.308000860424921, - "nauc_mrr_at_20_diff1": 0.5878546914502075, - "nauc_mrr_at_20_max": 0.1979369859133071, - "nauc_mrr_at_20_std": -0.33782493585182194, - "nauc_mrr_at_3_diff1": 0.5876131256705309, - "nauc_mrr_at_3_max": 0.1952677167723346, - "nauc_mrr_at_3_std": -0.33690000733845876, - "nauc_mrr_at_5_diff1": 0.5848840257463417, - "nauc_mrr_at_5_max": 0.19573848094590507, - "nauc_mrr_at_5_std": -0.33754136097831156, - "nauc_ndcg_at_1000_diff1": 0.5578263978615521, - "nauc_ndcg_at_1000_max": 0.2287106730052526, - "nauc_ndcg_at_1000_std": -0.32790750604660857, - "nauc_ndcg_at_100_diff1": 0.5553967772122925, - "nauc_ndcg_at_100_max": 0.23045027433227894, - "nauc_ndcg_at_100_std": -0.33087731960853317, - "nauc_ndcg_at_10_diff1": 0.5423985101785965, - "nauc_ndcg_at_10_max": 0.23676469280404897, - "nauc_ndcg_at_10_std": -0.34577355251853725, - "nauc_ndcg_at_1_diff1": 0.6206472732725669, - "nauc_ndcg_at_1_max": 0.1884184377108565, - "nauc_ndcg_at_1_std": -0.30647209320694563, - "nauc_ndcg_at_20_diff1": 0.5488316930039923, - "nauc_ndcg_at_20_max": 0.23729811515756516, - "nauc_ndcg_at_20_std": -0.3397743302343221, - "nauc_ndcg_at_3_diff1": 0.5554620170205623, - "nauc_ndcg_at_3_max": 0.23676777748450337, - "nauc_ndcg_at_3_std": -0.32826473567801495, - "nauc_ndcg_at_5_diff1": 0.5453608942107281, - "nauc_ndcg_at_5_max": 0.2362765393024237, - "nauc_ndcg_at_5_std": -0.33441704181011644, - "nauc_precision_at_1000_diff1": -0.2490286905418845, - "nauc_precision_at_1000_max": -0.2823172572380373, - "nauc_precision_at_1000_std": 0.06940935099459479, - "nauc_precision_at_100_diff1": -0.20322438082025343, - "nauc_precision_at_100_max": -0.23944899930888527, - "nauc_precision_at_100_std": 0.014691925313461496, - "nauc_precision_at_10_diff1": 0.04685689049403343, - "nauc_precision_at_10_max": -0.048833759034091796, - "nauc_precision_at_10_std": -0.19287228031286877, - "nauc_precision_at_1_diff1": 0.6206472732725669, - "nauc_precision_at_1_max": 0.1884184377108565, - "nauc_precision_at_1_std": -0.30647209320694563, - "nauc_precision_at_20_diff1": -0.059356183920344925, - "nauc_precision_at_20_max": -0.11647543513941302, - "nauc_precision_at_20_std": -0.11040894004972752, - "nauc_precision_at_3_diff1": 0.2963034493624533, - "nauc_precision_at_3_max": 0.06193015454029306, - "nauc_precision_at_3_std": -0.2847934074807809, - "nauc_precision_at_5_diff1": 0.1818704510358003, - "nauc_precision_at_5_max": 0.012838216995005054, - "nauc_precision_at_5_std": -0.23678360333260248, - "nauc_recall_at_1000_diff1": -0.031658167426407764, - "nauc_recall_at_1000_max": 0.761294904963372, - "nauc_recall_at_1000_std": 0.589836641693236, - "nauc_recall_at_100_diff1": 0.30495269315830603, - "nauc_recall_at_100_max": 0.40319937864466737, - "nauc_recall_at_100_std": -0.3076823635453479, - "nauc_recall_at_10_diff1": 0.40320059333569697, - "nauc_recall_at_10_max": 0.2820181081363864, - "nauc_recall_at_10_std": -0.4051693238202271, - "nauc_recall_at_1_diff1": 0.5956803415962526, - "nauc_recall_at_1_max": 0.2511508848721171, - "nauc_recall_at_1_std": -0.2533533014832028, - "nauc_recall_at_20_diff1": 0.38601321807115213, - "nauc_recall_at_20_max": 0.33457449244668686, - "nauc_recall_at_20_std": -0.4021984802798079, - "nauc_recall_at_3_diff1": 0.49455357752002116, - "nauc_recall_at_3_max": 0.254980097750559, - "nauc_recall_at_3_std": -0.33207921509986293, - "nauc_recall_at_5_diff1": 0.4489604939387345, - "nauc_recall_at_5_max": 0.2583193566941697, - "nauc_recall_at_5_std": -0.34658019377545485, - "ndcg_at_1": 0.5237, - "ndcg_at_10": 0.65641, - "ndcg_at_100": 0.69529, - "ndcg_at_1000": 0.69876, - "ndcg_at_20": 0.67833, - "ndcg_at_3": 0.57495, - "ndcg_at_5": 0.61754, - "precision_at_1": 0.5237, - "precision_at_10": 0.11489, - "precision_at_100": 0.014, - "precision_at_1000": 0.00144, - "precision_at_20": 0.06356, - "precision_at_3": 0.28169, - "precision_at_5": 0.19681, - "recall_at_1": 0.41219, - "recall_at_10": 0.82059, - "recall_at_100": 0.97292, - "recall_at_1000": 0.99642, - "recall_at_20": 0.89626, - "recall_at_3": 0.62186, - "recall_at_5": 0.71505 - } - ] - }, - "task_name": "WebQAT2ITRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/WebQAT2TRetrieval.json b/results-mieb/voyage-multimodal-3/1/WebQAT2TRetrieval.json deleted file mode 100644 index 5a5b13bc1d..0000000000 --- a/results-mieb/voyage-multimodal-3/1/WebQAT2TRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "468b42a2b2e767d80d2d93f5ae5d42f135a10478", - "evaluation_time": 2916.9777975082397, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.589, - "cv_recall_at_10": 0.92912, - "cv_recall_at_100": 0.99267, - "cv_recall_at_1000": 0.99959, - "cv_recall_at_20": 0.96375, - "cv_recall_at_3": 0.8167, - "cv_recall_at_5": 0.87413, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.6262, - "map_at_1": 0.29219, - "map_at_10": 0.51981, - "map_at_100": 0.53586, - "map_at_1000": 0.53636, - "map_at_20": 0.53029, - "map_at_3": 0.45758, - "map_at_5": 0.49438, - "mrr_at_1": 0.5890020366598778, - "mrr_at_10": 0.7122235153396038, - "mrr_at_100": 0.7155789027275217, - "mrr_at_1000": 0.7156207060411495, - "mrr_at_20": 0.7146892001931302, - "mrr_at_3": 0.6912423625254568, - "mrr_at_5": 0.7045417515274939, - "nauc_cv_recall_at_1000_diff1": -1.741603926146025, - "nauc_cv_recall_at_1000_max": -1.741603926146025, - "nauc_cv_recall_at_1000_std": -0.17150958406100894, - "nauc_cv_recall_at_100_diff1": 0.32561412897936853, - "nauc_cv_recall_at_100_max": 0.46515820318395235, - "nauc_cv_recall_at_100_std": 0.3833298186528627, - "nauc_cv_recall_at_10_diff1": 0.4575429865422298, - "nauc_cv_recall_at_10_max": 0.515849278224097, - "nauc_cv_recall_at_10_std": 0.04406300734960047, - "nauc_cv_recall_at_1_diff1": 0.4966656191251561, - "nauc_cv_recall_at_1_max": 0.27772683782666246, - "nauc_cv_recall_at_1_std": -0.06175533593483985, - "nauc_cv_recall_at_20_diff1": 0.5055617524351645, - "nauc_cv_recall_at_20_max": 0.5636112160370098, - "nauc_cv_recall_at_20_std": 0.14768827018120337, - "nauc_cv_recall_at_3_diff1": 0.4427478637599672, - "nauc_cv_recall_at_3_max": 0.37453726047706076, - "nauc_cv_recall_at_3_std": -0.10573679542784171, - "nauc_cv_recall_at_5_diff1": 0.467397733168413, - "nauc_cv_recall_at_5_max": 0.44894693050871626, - "nauc_cv_recall_at_5_std": -0.07286478058455781, - "nauc_map_at_1000_diff1": 0.29202796941658243, - "nauc_map_at_1000_max": 0.25787082263263295, - "nauc_map_at_1000_std": -0.03977740340077741, - "nauc_map_at_100_diff1": 0.29188967264538324, - "nauc_map_at_100_max": 0.25805786714594836, - "nauc_map_at_100_std": -0.039356656861644475, - "nauc_map_at_10_diff1": 0.28739222005560244, - "nauc_map_at_10_max": 0.2564964868236401, - "nauc_map_at_10_std": -0.046076737174336496, - "nauc_map_at_1_diff1": 0.49075731029230757, - "nauc_map_at_1_max": 0.27779701957024744, - "nauc_map_at_1_std": -0.05748861166572812, - "nauc_map_at_20_diff1": 0.2902754739335047, - "nauc_map_at_20_max": 0.25793116771651126, - "nauc_map_at_20_std": -0.04153755449200511, - "nauc_map_at_3_diff1": 0.27680439152954867, - "nauc_map_at_3_max": 0.22708615796427953, - "nauc_map_at_3_std": -0.06030884464319331, - "nauc_map_at_5_diff1": 0.2819460660292493, - "nauc_map_at_5_max": 0.24510310548741399, - "nauc_map_at_5_std": -0.05476259007222099, - "nauc_mrr_at_1000_diff1": 0.4851985065498464, - "nauc_mrr_at_1000_max": 0.30846768529245516, - "nauc_mrr_at_1000_std": -0.06793412217064555, - "nauc_mrr_at_100_diff1": 0.48518671596935053, - "nauc_mrr_at_100_max": 0.30851404606767807, - "nauc_mrr_at_100_std": -0.06786147285634107, - "nauc_mrr_at_10_diff1": 0.48435566911818745, - "nauc_mrr_at_10_max": 0.31096037838142404, - "nauc_mrr_at_10_std": -0.06727771912167632, - "nauc_mrr_at_1_diff1": 0.4966656191251561, - "nauc_mrr_at_1_max": 0.27772683782666246, - "nauc_mrr_at_1_std": -0.06175533593483985, - "nauc_mrr_at_20_diff1": 0.4853842400252949, - "nauc_mrr_at_20_max": 0.3094884777639313, - "nauc_mrr_at_20_std": -0.06731324685156856, - "nauc_mrr_at_3_diff1": 0.47912398040972315, - "nauc_mrr_at_3_max": 0.3067047656670131, - "nauc_mrr_at_3_std": -0.07596845472211204, - "nauc_mrr_at_5_diff1": 0.48407057547709265, - "nauc_mrr_at_5_max": 0.31223132247632523, - "nauc_mrr_at_5_std": -0.07178600314134528, - "nauc_ndcg_at_1000_diff1": 0.3371883578414304, - "nauc_ndcg_at_1000_max": 0.2850891815328272, - "nauc_ndcg_at_1000_std": -0.0260621294242751, - "nauc_ndcg_at_100_diff1": 0.33162470243382997, - "nauc_ndcg_at_100_max": 0.2896799083593092, - "nauc_ndcg_at_100_std": -0.013478968978344635, - "nauc_ndcg_at_10_diff1": 0.3165447269095804, - "nauc_ndcg_at_10_max": 0.2891288151784892, - "nauc_ndcg_at_10_std": -0.03508962570767255, - "nauc_ndcg_at_1_diff1": 0.4966656191251561, - "nauc_ndcg_at_1_max": 0.27772683782666246, - "nauc_ndcg_at_1_std": -0.06175533593483985, - "nauc_ndcg_at_20_diff1": 0.32393047592260726, - "nauc_ndcg_at_20_max": 0.2915271314676446, - "nauc_ndcg_at_20_std": -0.022416010844775933, - "nauc_ndcg_at_3_diff1": 0.3060273038108367, - "nauc_ndcg_at_3_max": 0.2483764803867023, - "nauc_ndcg_at_3_std": -0.06481599681457599, - "nauc_ndcg_at_5_diff1": 0.31042685093662403, - "nauc_ndcg_at_5_max": 0.2712696876035037, - "nauc_ndcg_at_5_std": -0.054821583327499285, - "nauc_precision_at_1000_diff1": -0.05502022885141667, - "nauc_precision_at_1000_max": 0.018828263504736552, - "nauc_precision_at_1000_std": 0.10707652007203783, - "nauc_precision_at_100_diff1": 0.04617918393088877, - "nauc_precision_at_100_max": 0.21887404884172, - "nauc_precision_at_100_std": 0.24637836543125177, - "nauc_precision_at_10_diff1": 0.16899025415585744, - "nauc_precision_at_10_max": 0.27988248388439646, - "nauc_precision_at_10_std": 0.020501892255375364, - "nauc_precision_at_1_diff1": 0.4966656191251561, - "nauc_precision_at_1_max": 0.27772683782666246, - "nauc_precision_at_1_std": -0.06175533593483985, - "nauc_precision_at_20_diff1": 0.1414296996662419, - "nauc_precision_at_20_max": 0.26899190108516663, - "nauc_precision_at_20_std": 0.08692554067249968, - "nauc_precision_at_3_diff1": 0.20830973485598295, - "nauc_precision_at_3_max": 0.2243587062044391, - "nauc_precision_at_3_std": -0.06454460552554597, - "nauc_precision_at_5_diff1": 0.19572312072504147, - "nauc_precision_at_5_max": 0.25367040781044475, - "nauc_precision_at_5_std": -0.041281502254642145, - "nauc_recall_at_1000_diff1": 0.1623570160659502, - "nauc_recall_at_1000_max": 0.33549516337261165, - "nauc_recall_at_1000_std": 0.43557417309518925, - "nauc_recall_at_100_diff1": 0.12776533061990886, - "nauc_recall_at_100_max": 0.33731010803929673, - "nauc_recall_at_100_std": 0.34635643583394, - "nauc_recall_at_10_diff1": 0.18871296693272535, - "nauc_recall_at_10_max": 0.2989969596246417, - "nauc_recall_at_10_std": 0.02753041423599101, - "nauc_recall_at_1_diff1": 0.49075731029230757, - "nauc_recall_at_1_max": 0.27779701957024744, - "nauc_recall_at_1_std": -0.05748861166572812, - "nauc_recall_at_20_diff1": 0.17416536192966864, - "nauc_recall_at_20_max": 0.3061574059562882, - "nauc_recall_at_20_std": 0.10364372354738594, - "nauc_recall_at_3_diff1": 0.21276002628565463, - "nauc_recall_at_3_max": 0.22813299646040489, - "nauc_recall_at_3_std": -0.06009924853869038, - "nauc_recall_at_5_diff1": 0.2030057553831004, - "nauc_recall_at_5_max": 0.25995381724385386, - "nauc_recall_at_5_std": -0.0356346062273783, - "ndcg_at_1": 0.589, - "ndcg_at_10": 0.6262, - "ndcg_at_100": 0.67704, - "ndcg_at_1000": 0.6853, - "ndcg_at_20": 0.65185, - "ndcg_at_3": 0.53576, - "ndcg_at_5": 0.58335, - "precision_at_1": 0.589, - "precision_at_10": 0.14982, - "precision_at_100": 0.01886, - "precision_at_1000": 0.002, - "precision_at_20": 0.08322, - "precision_at_3": 0.36307, - "precision_at_5": 0.25589, - "recall_at_1": 0.29219, - "recall_at_10": 0.73836, - "recall_at_100": 0.9278, - "recall_at_1000": 0.98001, - "recall_at_20": 0.82006, - "recall_at_3": 0.53856, - "recall_at_5": 0.63208 - } - ] - }, - "task_name": "WebQAT2TRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/Winoground.json b/results-mieb/voyage-multimodal-3/1/Winoground.json deleted file mode 100644 index b008641bfc..0000000000 --- a/results-mieb/voyage-multimodal-3/1/Winoground.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dataset_revision": "b400e173549071916ad1b3d449293bc8d8b4b763", - "evaluation_time": 1458.447455406189, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "accuracy": 0.0625, - "hf_subset": "default", - "image_acc": 0.0925000011920929, - "languages": [ - "eng-Latn" - ], - "main_score": 0.0625, - "text_acc": 0.27000001072883606 - } - ] - }, - "task_name": "Winoground" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/XFlickr30kCoT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/XFlickr30kCoT2IRetrieval.json deleted file mode 100644 index 8088fb6cce..0000000000 --- a/results-mieb/voyage-multimodal-3/1/XFlickr30kCoT2IRetrieval.json +++ /dev/null @@ -1,1411 +0,0 @@ -{ - "dataset_revision": "0af2c2eba58b27a71898787e286be04befdd7a20", - "evaluation_time": 3659.7448897361755, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.6755, - "cv_recall_at_10": 0.929, - "cv_recall_at_100": 0.9865, - "cv_recall_at_1000": 0.9985, - "cv_recall_at_20": 0.9585, - "cv_recall_at_3": 0.8335, - "cv_recall_at_5": 0.887, - "hf_subset": "de", - "languages": [ - "deu-Latn" - ], - "main_score": 0.80482, - "map_at_1": 0.6755, - "map_at_10": 0.76463, - "map_at_100": 0.7676, - "map_at_1000": 0.76767, - "map_at_20": 0.76683, - "map_at_3": 0.7465, - "map_at_5": 0.75875, - "mrr_at_1": 0.6755, - "mrr_at_10": 0.764634126984127, - "mrr_at_100": 0.7675982824740856, - "mrr_at_1000": 0.767672981003628, - "mrr_at_20": 0.7668271411380156, - "mrr_at_3": 0.7464999999999994, - "mrr_at_5": 0.7587499999999999, - "nauc_cv_recall_at_1000_diff1": 0.8513849984438709, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": 0.7822561123214679, - "nauc_cv_recall_at_100_max": 0.9683577134557472, - "nauc_cv_recall_at_100_std": 0.8572120206107111, - "nauc_cv_recall_at_10_diff1": 0.6544594363567012, - "nauc_cv_recall_at_10_max": 0.7857274365145103, - "nauc_cv_recall_at_10_std": 0.30422403703265355, - "nauc_cv_recall_at_1_diff1": 0.7463322614632609, - "nauc_cv_recall_at_1_max": 0.5806226318361674, - "nauc_cv_recall_at_1_std": 0.10438451063582097, - "nauc_cv_recall_at_20_diff1": 0.6694002902365792, - "nauc_cv_recall_at_20_max": 0.8863802549132098, - "nauc_cv_recall_at_20_std": 0.5763220951031016, - "nauc_cv_recall_at_3_diff1": 0.6924928185360167, - "nauc_cv_recall_at_3_max": 0.6660863314034848, - "nauc_cv_recall_at_3_std": 0.15626155616629717, - "nauc_cv_recall_at_5_diff1": 0.6826023673900833, - "nauc_cv_recall_at_5_max": 0.7265547008045103, - "nauc_cv_recall_at_5_std": 0.21227234213903992, - "nauc_map_at_1000_diff1": 0.7290871254473019, - "nauc_map_at_1000_max": 0.6120752280056199, - "nauc_map_at_1000_std": 0.1339926740717831, - "nauc_map_at_100_diff1": 0.729116189000848, - "nauc_map_at_100_max": 0.6122070042437942, - "nauc_map_at_100_std": 0.13424625550910035, - "nauc_map_at_10_diff1": 0.7275587072696231, - "nauc_map_at_10_max": 0.6131427813295653, - "nauc_map_at_10_std": 0.13247460462832822, - "nauc_map_at_1_diff1": 0.7463322614632609, - "nauc_map_at_1_max": 0.5806226318361674, - "nauc_map_at_1_std": 0.10438451063582097, - "nauc_map_at_20_diff1": 0.7287129282021074, - "nauc_map_at_20_max": 0.6129948863589249, - "nauc_map_at_20_std": 0.13516965016715282, - "nauc_map_at_3_diff1": 0.7276166067316586, - "nauc_map_at_3_max": 0.6078307595298069, - "nauc_map_at_3_std": 0.12362603641270917, - "nauc_map_at_5_diff1": 0.7281564633744259, - "nauc_map_at_5_max": 0.6130927975568831, - "nauc_map_at_5_std": 0.12986554015100024, - "nauc_mrr_at_1000_diff1": 0.7290871254473019, - "nauc_mrr_at_1000_max": 0.6120752280056199, - "nauc_mrr_at_1000_std": 0.1339926740717831, - "nauc_mrr_at_100_diff1": 0.729116189000848, - "nauc_mrr_at_100_max": 0.6122070042437942, - "nauc_mrr_at_100_std": 0.13424625550910035, - "nauc_mrr_at_10_diff1": 0.7275587072696231, - "nauc_mrr_at_10_max": 0.6131427813295653, - "nauc_mrr_at_10_std": 0.13247460462832822, - "nauc_mrr_at_1_diff1": 0.7463322614632609, - "nauc_mrr_at_1_max": 0.5806226318361674, - "nauc_mrr_at_1_std": 0.10438451063582097, - "nauc_mrr_at_20_diff1": 0.7287129282021074, - "nauc_mrr_at_20_max": 0.6129948863589249, - "nauc_mrr_at_20_std": 0.13516965016715282, - "nauc_mrr_at_3_diff1": 0.7276166067316586, - "nauc_mrr_at_3_max": 0.6078307595298069, - "nauc_mrr_at_3_std": 0.12362603641270917, - "nauc_mrr_at_5_diff1": 0.7281564633744259, - "nauc_mrr_at_5_max": 0.6130927975568831, - "nauc_mrr_at_5_std": 0.12986554015100024, - "nauc_ndcg_at_1000_diff1": 0.7267113760326085, - "nauc_ndcg_at_1000_max": 0.6205977818229345, - "nauc_ndcg_at_1000_std": 0.14558789880807216, - "nauc_ndcg_at_100_diff1": 0.727248678404592, - "nauc_ndcg_at_100_max": 0.6240844070904952, - "nauc_ndcg_at_100_std": 0.15255848317430523, - "nauc_ndcg_at_10_diff1": 0.7195813943287325, - "nauc_ndcg_at_10_max": 0.6312960344952803, - "nauc_ndcg_at_10_std": 0.1500285742121337, - "nauc_ndcg_at_1_diff1": 0.7463322614632609, - "nauc_ndcg_at_1_max": 0.5806226318361674, - "nauc_ndcg_at_1_std": 0.10438451063582097, - "nauc_ndcg_at_20_diff1": 0.7237711267067128, - "nauc_ndcg_at_20_max": 0.6309783007730462, - "nauc_ndcg_at_20_std": 0.16121216549752765, - "nauc_ndcg_at_3_diff1": 0.7206010395502354, - "nauc_ndcg_at_3_max": 0.619345909718247, - "nauc_ndcg_at_3_std": 0.13020291795367483, - "nauc_ndcg_at_5_diff1": 0.7210205657545173, - "nauc_ndcg_at_5_max": 0.6303298538413662, - "nauc_ndcg_at_5_std": 0.14244007171848722, - "nauc_precision_at_1000_diff1": 0.8513849984438441, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.7822561123214696, - "nauc_precision_at_100_max": 0.968357713455748, - "nauc_precision_at_100_std": 0.8572120206107152, - "nauc_precision_at_10_diff1": 0.6544594363567019, - "nauc_precision_at_10_max": 0.7857274365145102, - "nauc_precision_at_10_std": 0.30422403703265183, - "nauc_precision_at_1_diff1": 0.7463322614632609, - "nauc_precision_at_1_max": 0.5806226318361674, - "nauc_precision_at_1_std": 0.10438451063582097, - "nauc_precision_at_20_diff1": 0.6694002902365723, - "nauc_precision_at_20_max": 0.8863802549132097, - "nauc_precision_at_20_std": 0.5763220951030971, - "nauc_precision_at_3_diff1": 0.6924928185360182, - "nauc_precision_at_3_max": 0.6660863314034838, - "nauc_precision_at_3_std": 0.15626155616629744, - "nauc_precision_at_5_diff1": 0.6826023673900811, - "nauc_precision_at_5_max": 0.7265547008045097, - "nauc_precision_at_5_std": 0.21227234213903803, - "nauc_recall_at_1000_diff1": 0.8513849984438709, - "nauc_recall_at_1000_max": 1.0, - "nauc_recall_at_1000_std": 1.0, - "nauc_recall_at_100_diff1": 0.7822561123214679, - "nauc_recall_at_100_max": 0.9683577134557472, - "nauc_recall_at_100_std": 0.8572120206107111, - "nauc_recall_at_10_diff1": 0.6544594363567012, - "nauc_recall_at_10_max": 0.7857274365145103, - "nauc_recall_at_10_std": 0.30422403703265355, - "nauc_recall_at_1_diff1": 0.7463322614632609, - "nauc_recall_at_1_max": 0.5806226318361674, - "nauc_recall_at_1_std": 0.10438451063582097, - "nauc_recall_at_20_diff1": 0.6694002902365792, - "nauc_recall_at_20_max": 0.8863802549132098, - "nauc_recall_at_20_std": 0.5763220951031016, - "nauc_recall_at_3_diff1": 0.6924928185360167, - "nauc_recall_at_3_max": 0.6660863314034848, - "nauc_recall_at_3_std": 0.15626155616629717, - "nauc_recall_at_5_diff1": 0.6826023673900833, - "nauc_recall_at_5_max": 0.7265547008045103, - "nauc_recall_at_5_std": 0.21227234213903992, - "ndcg_at_1": 0.6755, - "ndcg_at_10": 0.80482, - "ndcg_at_100": 0.81775, - "ndcg_at_1000": 0.81937, - "ndcg_at_20": 0.81246, - "ndcg_at_3": 0.7689, - "ndcg_at_5": 0.79096, - "precision_at_1": 0.6755, - "precision_at_10": 0.0929, - "precision_at_100": 0.00987, - "precision_at_1000": 0.001, - "precision_at_20": 0.04792, - "precision_at_3": 0.27783, - "precision_at_5": 0.1774, - "recall_at_1": 0.6755, - "recall_at_10": 0.929, - "recall_at_100": 0.9865, - "recall_at_1000": 0.9985, - "recall_at_20": 0.9585, - "recall_at_3": 0.8335, - "recall_at_5": 0.887 - }, - { - "cv_recall_at_1": 0.8125, - "cv_recall_at_10": 0.979, - "cv_recall_at_100": 0.997, - "cv_recall_at_1000": 0.9995, - "cv_recall_at_20": 0.9905, - "cv_recall_at_3": 0.9245, - "cv_recall_at_5": 0.9505, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89914, - "map_at_1": 0.8125, - "map_at_10": 0.87317, - "map_at_100": 0.87417, - "map_at_1000": 0.87417, - "map_at_20": 0.87394, - "map_at_3": 0.86325, - "map_at_5": 0.86925, - "mrr_at_1": 0.8125, - "mrr_at_10": 0.873170634920634, - "mrr_at_100": 0.874165591919412, - "mrr_at_1000": 0.8741744359007023, - "mrr_at_20": 0.8739376106628999, - "mrr_at_3": 0.8632499999999987, - "mrr_at_5": 0.8692499999999992, - "nauc_cv_recall_at_1000_diff1": 0.869281045751564, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": 0.8175381263616397, - "nauc_cv_recall_at_100_max": 0.9782135076252607, - "nauc_cv_recall_at_100_std": 0.09025832555244799, - "nauc_cv_recall_at_10_diff1": 0.8532746431906091, - "nauc_cv_recall_at_10_max": 0.8753390244986885, - "nauc_cv_recall_at_10_std": -0.21299630963496155, - "nauc_cv_recall_at_1_diff1": 0.8630334769638712, - "nauc_cv_recall_at_1_max": 0.636707988067617, - "nauc_cv_recall_at_1_std": -0.35566721909181354, - "nauc_cv_recall_at_20_diff1": 0.8380755811096317, - "nauc_cv_recall_at_20_max": 0.9094550100742143, - "nauc_cv_recall_at_20_std": -0.16393925991450467, - "nauc_cv_recall_at_3_diff1": 0.7887318282721475, - "nauc_cv_recall_at_3_max": 0.6857210875520191, - "nauc_cv_recall_at_3_std": -0.40794021802981484, - "nauc_cv_recall_at_5_diff1": 0.8004838298955943, - "nauc_cv_recall_at_5_max": 0.7276311197879799, - "nauc_cv_recall_at_5_std": -0.4019230587858072, - "nauc_map_at_1000_diff1": 0.8475728663389887, - "nauc_map_at_1000_max": 0.6502781025937084, - "nauc_map_at_1000_std": -0.36506027547060665, - "nauc_map_at_100_diff1": 0.8475672508185738, - "nauc_map_at_100_max": 0.6503002284229283, - "nauc_map_at_100_std": -0.36505931191358815, - "nauc_map_at_10_diff1": 0.8476975751166981, - "nauc_map_at_10_max": 0.6519234592343327, - "nauc_map_at_10_std": -0.3643551754692845, - "nauc_map_at_1_diff1": 0.8630334769638712, - "nauc_map_at_1_max": 0.636707988067617, - "nauc_map_at_1_std": -0.35566721909181354, - "nauc_map_at_20_diff1": 0.8475754154618236, - "nauc_map_at_20_max": 0.6507046372651616, - "nauc_map_at_20_std": -0.36490811352349883, - "nauc_map_at_3_diff1": 0.8411035099411375, - "nauc_map_at_3_max": 0.6487267627438167, - "nauc_map_at_3_std": -0.3728350091644064, - "nauc_map_at_5_diff1": 0.8448914569338772, - "nauc_map_at_5_max": 0.6511609163961992, - "nauc_map_at_5_std": -0.36969059260196196, - "nauc_mrr_at_1000_diff1": 0.8475728663389887, - "nauc_mrr_at_1000_max": 0.6502781025937084, - "nauc_mrr_at_1000_std": -0.36506027547060665, - "nauc_mrr_at_100_diff1": 0.8475672508185738, - "nauc_mrr_at_100_max": 0.6503002284229283, - "nauc_mrr_at_100_std": -0.36505931191358815, - "nauc_mrr_at_10_diff1": 0.8476975751166981, - "nauc_mrr_at_10_max": 0.6519234592343327, - "nauc_mrr_at_10_std": -0.3643551754692845, - "nauc_mrr_at_1_diff1": 0.8630334769638712, - "nauc_mrr_at_1_max": 0.636707988067617, - "nauc_mrr_at_1_std": -0.35566721909181354, - "nauc_mrr_at_20_diff1": 0.8475754154618236, - "nauc_mrr_at_20_max": 0.6507046372651616, - "nauc_mrr_at_20_std": -0.36490811352349883, - "nauc_mrr_at_3_diff1": 0.8411035099411375, - "nauc_mrr_at_3_max": 0.6487267627438167, - "nauc_mrr_at_3_std": -0.3728350091644064, - "nauc_mrr_at_5_diff1": 0.8448914569338772, - "nauc_mrr_at_5_max": 0.6511609163961992, - "nauc_mrr_at_5_std": -0.36969059260196196, - "nauc_ndcg_at_1000_diff1": 0.8463031464137469, - "nauc_ndcg_at_1000_max": 0.6541835788530691, - "nauc_ndcg_at_1000_std": -0.3627001519386867, - "nauc_ndcg_at_100_diff1": 0.8461539015610023, - "nauc_ndcg_at_100_max": 0.6552171012623584, - "nauc_ndcg_at_100_std": -0.3619261331869923, - "nauc_ndcg_at_10_diff1": 0.8467822882186511, - "nauc_ndcg_at_10_max": 0.663866200724262, - "nauc_ndcg_at_10_std": -0.35788015610205864, - "nauc_ndcg_at_1_diff1": 0.8630334769638712, - "nauc_ndcg_at_1_max": 0.636707988067617, - "nauc_ndcg_at_1_std": -0.35566721909181354, - "nauc_ndcg_at_20_diff1": 0.8461883899116863, - "nauc_ndcg_at_20_max": 0.6582925692614663, - "nauc_ndcg_at_20_std": -0.36084713873893653, - "nauc_ndcg_at_3_diff1": 0.8322438678494426, - "nauc_ndcg_at_3_max": 0.6548607059305832, - "nauc_ndcg_at_3_std": -0.3788620814328986, - "nauc_ndcg_at_5_diff1": 0.8391437301727366, - "nauc_ndcg_at_5_max": 0.6602246180383025, - "nauc_ndcg_at_5_std": -0.3738041857189095, - "nauc_precision_at_1000_diff1": 0.8692810457516023, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.8175381263616008, - "nauc_precision_at_100_max": 0.9782135076252637, - "nauc_precision_at_100_std": 0.09025832555238163, - "nauc_precision_at_10_diff1": 0.8532746431906064, - "nauc_precision_at_10_max": 0.8753390244986764, - "nauc_precision_at_10_std": -0.21299630963496555, - "nauc_precision_at_1_diff1": 0.8630334769638712, - "nauc_precision_at_1_max": 0.636707988067617, - "nauc_precision_at_1_std": -0.35566721909181354, - "nauc_precision_at_20_diff1": 0.8380755811096442, - "nauc_precision_at_20_max": 0.9094550100741999, - "nauc_precision_at_20_std": -0.16393925991451266, - "nauc_precision_at_3_diff1": 0.7887318282721495, - "nauc_precision_at_3_max": 0.6857210875520209, - "nauc_precision_at_3_std": -0.40794021802981506, - "nauc_precision_at_5_diff1": 0.800483829895592, - "nauc_precision_at_5_max": 0.7276311197879809, - "nauc_precision_at_5_std": -0.40192305878580603, - "nauc_recall_at_1000_diff1": 0.869281045751564, - "nauc_recall_at_1000_max": 1.0, - "nauc_recall_at_1000_std": 1.0, - "nauc_recall_at_100_diff1": 0.8175381263616397, - "nauc_recall_at_100_max": 0.9782135076252607, - "nauc_recall_at_100_std": 0.09025832555244799, - "nauc_recall_at_10_diff1": 0.8532746431906091, - "nauc_recall_at_10_max": 0.8753390244986885, - "nauc_recall_at_10_std": -0.21299630963496155, - "nauc_recall_at_1_diff1": 0.8630334769638712, - "nauc_recall_at_1_max": 0.636707988067617, - "nauc_recall_at_1_std": -0.35566721909181354, - "nauc_recall_at_20_diff1": 0.8380755811096317, - "nauc_recall_at_20_max": 0.9094550100742143, - "nauc_recall_at_20_std": -0.16393925991450467, - "nauc_recall_at_3_diff1": 0.7887318282721475, - "nauc_recall_at_3_max": 0.6857210875520191, - "nauc_recall_at_3_std": -0.40794021802981484, - "nauc_recall_at_5_diff1": 0.8004838298955943, - "nauc_recall_at_5_max": 0.7276311197879799, - "nauc_recall_at_5_std": -0.4019230587858072, - "ndcg_at_1": 0.8125, - "ndcg_at_10": 0.89914, - "ndcg_at_100": 0.90333, - "ndcg_at_1000": 0.90363, - "ndcg_at_20": 0.90201, - "ndcg_at_3": 0.87904, - "ndcg_at_5": 0.8898, - "precision_at_1": 0.8125, - "precision_at_10": 0.0979, - "precision_at_100": 0.00997, - "precision_at_1000": 0.001, - "precision_at_20": 0.04952, - "precision_at_3": 0.30817, - "precision_at_5": 0.1901, - "recall_at_1": 0.8125, - "recall_at_10": 0.979, - "recall_at_100": 0.997, - "recall_at_1000": 0.9995, - "recall_at_20": 0.9905, - "recall_at_3": 0.9245, - "recall_at_5": 0.9505 - }, - { - "cv_recall_at_1": 0.738, - "cv_recall_at_10": 0.9655, - "cv_recall_at_100": 0.999, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.9835, - "cv_recall_at_3": 0.902, - "cv_recall_at_5": 0.936, - "hf_subset": "es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.85832, - "map_at_1": 0.738, - "map_at_10": 0.82313, - "map_at_100": 0.82485, - "map_at_1000": 0.82485, - "map_at_20": 0.82436, - "map_at_3": 0.81117, - "map_at_5": 0.81909, - "mrr_at_1": 0.738, - "mrr_at_10": 0.8231287698412685, - "mrr_at_100": 0.8248457880053297, - "mrr_at_1000": 0.8248512406319713, - "mrr_at_20": 0.8243573382285052, - "mrr_at_3": 0.8111666666666654, - "mrr_at_5": 0.8190916666666657, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.8611111111111184, - "nauc_cv_recall_at_100_max": 1.0, - "nauc_cv_recall_at_100_std": 0.8692810457516946, - "nauc_cv_recall_at_10_diff1": 0.7032774462441969, - "nauc_cv_recall_at_10_max": 0.847054763934563, - "nauc_cv_recall_at_10_std": 0.21517882515325232, - "nauc_cv_recall_at_1_diff1": 0.8228027015155729, - "nauc_cv_recall_at_1_max": 0.657963506973408, - "nauc_cv_recall_at_1_std": 0.07059039237257138, - "nauc_cv_recall_at_20_diff1": 0.6693687576040491, - "nauc_cv_recall_at_20_max": 0.820360467419293, - "nauc_cv_recall_at_20_std": 0.33644568938686364, - "nauc_cv_recall_at_3_diff1": 0.7436331675527346, - "nauc_cv_recall_at_3_max": 0.7652084643381168, - "nauc_cv_recall_at_3_std": 0.13750738390594427, - "nauc_cv_recall_at_5_diff1": 0.7467903828197938, - "nauc_cv_recall_at_5_max": 0.7998694269374397, - "nauc_cv_recall_at_5_std": 0.16819852941176452, - "nauc_map_at_1000_diff1": 0.8055201985370324, - "nauc_map_at_1000_max": 0.6868428387023021, - "nauc_map_at_1000_std": 0.09455811602935846, - "nauc_map_at_100_diff1": 0.8055226699057262, - "nauc_map_at_100_max": 0.686853584905029, - "nauc_map_at_100_std": 0.09458470117793472, - "nauc_map_at_10_diff1": 0.804476460367567, - "nauc_map_at_10_max": 0.6886359840382733, - "nauc_map_at_10_std": 0.09505039704659991, - "nauc_map_at_1_diff1": 0.8228027015155729, - "nauc_map_at_1_max": 0.657963506973408, - "nauc_map_at_1_std": 0.07059039237257138, - "nauc_map_at_20_diff1": 0.8050762144661993, - "nauc_map_at_20_max": 0.687217723852823, - "nauc_map_at_20_std": 0.09495729170023198, - "nauc_map_at_3_diff1": 0.8013479649548041, - "nauc_map_at_3_max": 0.6900708586722066, - "nauc_map_at_3_std": 0.09348015452204224, - "nauc_map_at_5_diff1": 0.8044246497031612, - "nauc_map_at_5_max": 0.689862580590495, - "nauc_map_at_5_std": 0.09487116744284162, - "nauc_mrr_at_1000_diff1": 0.8055201985370324, - "nauc_mrr_at_1000_max": 0.6868428387023021, - "nauc_mrr_at_1000_std": 0.09455811602935846, - "nauc_mrr_at_100_diff1": 0.8055226699057262, - "nauc_mrr_at_100_max": 0.686853584905029, - "nauc_mrr_at_100_std": 0.09458470117793472, - "nauc_mrr_at_10_diff1": 0.804476460367567, - "nauc_mrr_at_10_max": 0.6886359840382733, - "nauc_mrr_at_10_std": 0.09505039704659991, - "nauc_mrr_at_1_diff1": 0.8228027015155729, - "nauc_mrr_at_1_max": 0.657963506973408, - "nauc_mrr_at_1_std": 0.07059039237257138, - "nauc_mrr_at_20_diff1": 0.8050762144661993, - "nauc_mrr_at_20_max": 0.687217723852823, - "nauc_mrr_at_20_std": 0.09495729170023198, - "nauc_mrr_at_3_diff1": 0.8013479649548041, - "nauc_mrr_at_3_max": 0.6900708586722066, - "nauc_mrr_at_3_std": 0.09348015452204224, - "nauc_mrr_at_5_diff1": 0.8044246497031612, - "nauc_mrr_at_5_max": 0.689862580590495, - "nauc_mrr_at_5_std": 0.09487116744284162, - "nauc_ndcg_at_1000_diff1": 0.8029525661469813, - "nauc_ndcg_at_1000_max": 0.6908828889360142, - "nauc_ndcg_at_1000_std": 0.09881132089648269, - "nauc_ndcg_at_100_diff1": 0.8030192391414134, - "nauc_ndcg_at_100_max": 0.6912190957005266, - "nauc_ndcg_at_100_std": 0.09964931117290805, - "nauc_ndcg_at_10_diff1": 0.7971382738540258, - "nauc_ndcg_at_10_max": 0.7003088811485032, - "nauc_ndcg_at_10_std": 0.10401499946098114, - "nauc_ndcg_at_1_diff1": 0.8228027015155729, - "nauc_ndcg_at_1_max": 0.657963506973408, - "nauc_ndcg_at_1_std": 0.07059039237257138, - "nauc_ndcg_at_20_diff1": 0.7995066079500553, - "nauc_ndcg_at_20_max": 0.6940947622326022, - "nauc_ndcg_at_20_std": 0.10398558209170257, - "nauc_ndcg_at_3_diff1": 0.7917995978691086, - "nauc_ndcg_at_3_max": 0.7026512611495505, - "nauc_ndcg_at_3_std": 0.10099772846271363, - "nauc_ndcg_at_5_diff1": 0.7973058952477493, - "nauc_ndcg_at_5_max": 0.7032016467149081, - "nauc_ndcg_at_5_std": 0.10391207396070531, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.8611111111109864, - "nauc_precision_at_100_max": 1.0, - "nauc_precision_at_100_std": 0.8692810457515646, - "nauc_precision_at_10_diff1": 0.703277446244195, - "nauc_precision_at_10_max": 0.8470547639345555, - "nauc_precision_at_10_std": 0.21517882515324474, - "nauc_precision_at_1_diff1": 0.8228027015155729, - "nauc_precision_at_1_max": 0.657963506973408, - "nauc_precision_at_1_std": 0.07059039237257138, - "nauc_precision_at_20_diff1": 0.6693687576040411, - "nauc_precision_at_20_max": 0.82036046741928, - "nauc_precision_at_20_std": 0.33644568938685376, - "nauc_precision_at_3_diff1": 0.7436331675527355, - "nauc_precision_at_3_max": 0.7652084643381156, - "nauc_precision_at_3_std": 0.13750738390594283, - "nauc_precision_at_5_diff1": 0.7467903828197912, - "nauc_precision_at_5_max": 0.7998694269374391, - "nauc_precision_at_5_std": 0.1681985294117573, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.8611111111111184, - "nauc_recall_at_100_max": 1.0, - "nauc_recall_at_100_std": 0.8692810457516946, - "nauc_recall_at_10_diff1": 0.7032774462441969, - "nauc_recall_at_10_max": 0.847054763934563, - "nauc_recall_at_10_std": 0.21517882515325232, - "nauc_recall_at_1_diff1": 0.8228027015155729, - "nauc_recall_at_1_max": 0.657963506973408, - "nauc_recall_at_1_std": 0.07059039237257138, - "nauc_recall_at_20_diff1": 0.6693687576040491, - "nauc_recall_at_20_max": 0.820360467419293, - "nauc_recall_at_20_std": 0.33644568938686364, - "nauc_recall_at_3_diff1": 0.7436331675527346, - "nauc_recall_at_3_max": 0.7652084643381168, - "nauc_recall_at_3_std": 0.13750738390594427, - "nauc_recall_at_5_diff1": 0.7467903828197938, - "nauc_recall_at_5_max": 0.7998694269374397, - "nauc_recall_at_5_std": 0.16819852941176452, - "ndcg_at_1": 0.738, - "ndcg_at_10": 0.85832, - "ndcg_at_100": 0.86589, - "ndcg_at_1000": 0.86603, - "ndcg_at_20": 0.86284, - "ndcg_at_3": 0.83453, - "ndcg_at_5": 0.84867, - "precision_at_1": 0.738, - "precision_at_10": 0.09655, - "precision_at_100": 0.00999, - "precision_at_1000": 0.001, - "precision_at_20": 0.04917, - "precision_at_3": 0.30067, - "precision_at_5": 0.1872, - "recall_at_1": 0.738, - "recall_at_10": 0.9655, - "recall_at_100": 0.999, - "recall_at_1000": 1.0, - "recall_at_20": 0.9835, - "recall_at_3": 0.902, - "recall_at_5": 0.936 - }, - { - "cv_recall_at_1": 0.57, - "cv_recall_at_10": 0.89, - "cv_recall_at_100": 0.9885, - "cv_recall_at_1000": 0.998, - "cv_recall_at_20": 0.936, - "cv_recall_at_3": 0.756, - "cv_recall_at_5": 0.8245, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.72935, - "map_at_1": 0.57, - "map_at_10": 0.67785, - "map_at_100": 0.68246, - "map_at_1000": 0.68253, - "map_at_20": 0.68098, - "map_at_3": 0.65292, - "map_at_5": 0.66864, - "mrr_at_1": 0.57, - "mrr_at_10": 0.6778513888888882, - "mrr_at_100": 0.682463976758287, - "mrr_at_1000": 0.6825288641722557, - "mrr_at_20": 0.6809768672568393, - "mrr_at_3": 0.6529166666666663, - "mrr_at_5": 0.6686416666666668, - "nauc_cv_recall_at_1000_diff1": 0.057773109243720255, - "nauc_cv_recall_at_1000_max": 0.07119514472454563, - "nauc_cv_recall_at_1000_std": 0.3162931839401999, - "nauc_cv_recall_at_100_diff1": 0.5459343157552812, - "nauc_cv_recall_at_100_max": 0.6831283237932791, - "nauc_cv_recall_at_100_std": 0.4349449924897459, - "nauc_cv_recall_at_10_diff1": 0.506069066231423, - "nauc_cv_recall_at_10_max": 0.6634932565930765, - "nauc_cv_recall_at_10_std": 0.18059230306674634, - "nauc_cv_recall_at_1_diff1": 0.7044065096654757, - "nauc_cv_recall_at_1_max": 0.5750721143496796, - "nauc_cv_recall_at_1_std": 0.046992006429299085, - "nauc_cv_recall_at_20_diff1": 0.5504201680672272, - "nauc_cv_recall_at_20_max": 0.7070786647992531, - "nauc_cv_recall_at_20_std": 0.2757972980858999, - "nauc_cv_recall_at_3_diff1": 0.5681348565963951, - "nauc_cv_recall_at_3_max": 0.6170784343861266, - "nauc_cv_recall_at_3_std": 0.09851473313011817, - "nauc_cv_recall_at_5_diff1": 0.5156420439651506, - "nauc_cv_recall_at_5_max": 0.6434716064276265, - "nauc_cv_recall_at_5_std": 0.146063346120565, - "nauc_map_at_1000_diff1": 0.6561740158631391, - "nauc_map_at_1000_max": 0.5903652066345317, - "nauc_map_at_1000_std": 0.06752440959391125, - "nauc_map_at_100_diff1": 0.6561738489049953, - "nauc_map_at_100_max": 0.5904283966716123, - "nauc_map_at_100_std": 0.0676291858616124, - "nauc_map_at_10_diff1": 0.65333134434392, - "nauc_map_at_10_max": 0.5913969782544157, - "nauc_map_at_10_std": 0.06836479826993805, - "nauc_map_at_1_diff1": 0.7044065096654757, - "nauc_map_at_1_max": 0.5750721143496796, - "nauc_map_at_1_std": 0.046992006429299085, - "nauc_map_at_20_diff1": 0.6556880678927408, - "nauc_map_at_20_max": 0.5909962160263641, - "nauc_map_at_20_std": 0.06838331967377713, - "nauc_map_at_3_diff1": 0.653394169001542, - "nauc_map_at_3_max": 0.5888483426159291, - "nauc_map_at_3_std": 0.06312086141745396, - "nauc_map_at_5_diff1": 0.6494305220897459, - "nauc_map_at_5_max": 0.5912390189784108, - "nauc_map_at_5_std": 0.06860456750834616, - "nauc_mrr_at_1000_diff1": 0.6561740158631391, - "nauc_mrr_at_1000_max": 0.5903652066345317, - "nauc_mrr_at_1000_std": 0.06752440959391125, - "nauc_mrr_at_100_diff1": 0.6561738489049953, - "nauc_mrr_at_100_max": 0.5904283966716123, - "nauc_mrr_at_100_std": 0.0676291858616124, - "nauc_mrr_at_10_diff1": 0.65333134434392, - "nauc_mrr_at_10_max": 0.5913969782544157, - "nauc_mrr_at_10_std": 0.06836479826993805, - "nauc_mrr_at_1_diff1": 0.7044065096654757, - "nauc_mrr_at_1_max": 0.5750721143496796, - "nauc_mrr_at_1_std": 0.046992006429299085, - "nauc_mrr_at_20_diff1": 0.6556880678927408, - "nauc_mrr_at_20_max": 0.5909962160263641, - "nauc_mrr_at_20_std": 0.06838331967377713, - "nauc_mrr_at_3_diff1": 0.653394169001542, - "nauc_mrr_at_3_max": 0.5888483426159291, - "nauc_mrr_at_3_std": 0.06312086141745396, - "nauc_mrr_at_5_diff1": 0.6494305220897459, - "nauc_mrr_at_5_max": 0.5912390189784108, - "nauc_mrr_at_5_std": 0.06860456750834616, - "nauc_ndcg_at_1000_diff1": 0.6483006530425621, - "nauc_ndcg_at_1000_max": 0.5933424088703956, - "nauc_ndcg_at_1000_std": 0.07405067389762883, - "nauc_ndcg_at_100_diff1": 0.6483197398489603, - "nauc_ndcg_at_100_max": 0.5947897242846063, - "nauc_ndcg_at_100_std": 0.07655442331368115, - "nauc_ndcg_at_10_diff1": 0.6340715926512107, - "nauc_ndcg_at_10_max": 0.600450440109288, - "nauc_ndcg_at_10_std": 0.08208340679131955, - "nauc_ndcg_at_1_diff1": 0.7044065096654757, - "nauc_ndcg_at_1_max": 0.5750721143496796, - "nauc_ndcg_at_1_std": 0.046992006429299085, - "nauc_ndcg_at_20_diff1": 0.6440067539607353, - "nauc_ndcg_at_20_max": 0.5999888493818042, - "nauc_ndcg_at_20_std": 0.083672066995679, - "nauc_ndcg_at_3_diff1": 0.6350601872400334, - "nauc_ndcg_at_3_max": 0.5948277431038169, - "nauc_ndcg_at_3_std": 0.07058416162338209, - "nauc_ndcg_at_5_diff1": 0.6259232001622383, - "nauc_ndcg_at_5_max": 0.6001942413051717, - "nauc_ndcg_at_5_std": 0.08181067673621384, - "nauc_precision_at_1000_diff1": 0.05777310924364636, - "nauc_precision_at_1000_max": 0.07119514472449866, - "nauc_precision_at_1000_std": 0.31629318394019806, - "nauc_precision_at_100_diff1": 0.5459343157552841, - "nauc_precision_at_100_max": 0.6831283237932635, - "nauc_precision_at_100_std": 0.4349449924897548, - "nauc_precision_at_10_diff1": 0.5060690662314247, - "nauc_precision_at_10_max": 0.6634932565930743, - "nauc_precision_at_10_std": 0.18059230306674617, - "nauc_precision_at_1_diff1": 0.7044065096654757, - "nauc_precision_at_1_max": 0.5750721143496796, - "nauc_precision_at_1_std": 0.046992006429299085, - "nauc_precision_at_20_diff1": 0.5504201680672255, - "nauc_precision_at_20_max": 0.7070786647992503, - "nauc_precision_at_20_std": 0.2757972980858991, - "nauc_precision_at_3_diff1": 0.5681348565963955, - "nauc_precision_at_3_max": 0.617078434386128, - "nauc_precision_at_3_std": 0.09851473313011903, - "nauc_precision_at_5_diff1": 0.5156420439651481, - "nauc_precision_at_5_max": 0.6434716064276249, - "nauc_precision_at_5_std": 0.14606334612056285, - "nauc_recall_at_1000_diff1": 0.057773109243720255, - "nauc_recall_at_1000_max": 0.07119514472454563, - "nauc_recall_at_1000_std": 0.3162931839401999, - "nauc_recall_at_100_diff1": 0.5459343157552812, - "nauc_recall_at_100_max": 0.6831283237932791, - "nauc_recall_at_100_std": 0.4349449924897459, - "nauc_recall_at_10_diff1": 0.506069066231423, - "nauc_recall_at_10_max": 0.6634932565930765, - "nauc_recall_at_10_std": 0.18059230306674634, - "nauc_recall_at_1_diff1": 0.7044065096654757, - "nauc_recall_at_1_max": 0.5750721143496796, - "nauc_recall_at_1_std": 0.046992006429299085, - "nauc_recall_at_20_diff1": 0.5504201680672272, - "nauc_recall_at_20_max": 0.7070786647992531, - "nauc_recall_at_20_std": 0.2757972980858999, - "nauc_recall_at_3_diff1": 0.5681348565963951, - "nauc_recall_at_3_max": 0.6170784343861266, - "nauc_recall_at_3_std": 0.09851473313011817, - "nauc_recall_at_5_diff1": 0.5156420439651506, - "nauc_recall_at_5_max": 0.6434716064276265, - "nauc_recall_at_5_std": 0.146063346120565, - "ndcg_at_1": 0.57, - "ndcg_at_10": 0.72935, - "ndcg_at_100": 0.75091, - "ndcg_at_1000": 0.75221, - "ndcg_at_20": 0.7409, - "ndcg_at_3": 0.67943, - "ndcg_at_5": 0.70771, - "precision_at_1": 0.57, - "precision_at_10": 0.089, - "precision_at_100": 0.00989, - "precision_at_1000": 0.001, - "precision_at_20": 0.0468, - "precision_at_3": 0.252, - "precision_at_5": 0.1649, - "recall_at_1": 0.57, - "recall_at_10": 0.89, - "recall_at_100": 0.9885, - "recall_at_1000": 0.998, - "recall_at_20": 0.936, - "recall_at_3": 0.756, - "recall_at_5": 0.8245 - }, - { - "cv_recall_at_1": 0.6515, - "cv_recall_at_10": 0.9325, - "cv_recall_at_100": 0.9915, - "cv_recall_at_1000": 0.999, - "cv_recall_at_20": 0.962, - "cv_recall_at_3": 0.822, - "cv_recall_at_5": 0.879, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.79245, - "map_at_1": 0.6515, - "map_at_10": 0.74747, - "map_at_100": 0.75038, - "map_at_1000": 0.75042, - "map_at_20": 0.74956, - "map_at_3": 0.72733, - "map_at_5": 0.74018, - "mrr_at_1": 0.6515, - "mrr_at_10": 0.7474712301587296, - "mrr_at_100": 0.7503837624364388, - "mrr_at_1000": 0.7504248142935365, - "mrr_at_20": 0.7495552861139542, - "mrr_at_3": 0.7273333333333322, - "mrr_at_5": 0.740183333333333, - "nauc_cv_recall_at_1000_diff1": 0.7770774976656952, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": 0.8009007524578522, - "nauc_cv_recall_at_100_max": 0.9682814302191535, - "nauc_cv_recall_at_100_std": 0.729856648541773, - "nauc_cv_recall_at_10_diff1": 0.5883217484524672, - "nauc_cv_recall_at_10_max": 0.7557526714389453, - "nauc_cv_recall_at_10_std": 0.3104955562471906, - "nauc_cv_recall_at_1_diff1": 0.7329375789067353, - "nauc_cv_recall_at_1_max": 0.5361300015750745, - "nauc_cv_recall_at_1_std": 0.06148747407860173, - "nauc_cv_recall_at_20_diff1": 0.623163300407883, - "nauc_cv_recall_at_20_max": 0.8448633839500713, - "nauc_cv_recall_at_20_std": 0.42984298982750796, - "nauc_cv_recall_at_3_diff1": 0.6402739291556773, - "nauc_cv_recall_at_3_max": 0.6452442159383034, - "nauc_cv_recall_at_3_std": 0.17984982027398486, - "nauc_cv_recall_at_5_diff1": 0.6157551065526523, - "nauc_cv_recall_at_5_max": 0.6875468807984149, - "nauc_cv_recall_at_5_std": 0.28306329993446605, - "nauc_map_at_1000_diff1": 0.7039322369949266, - "nauc_map_at_1000_max": 0.5719568116443199, - "nauc_map_at_1000_std": 0.10175653818610049, - "nauc_map_at_100_diff1": 0.7039442745246894, - "nauc_map_at_100_max": 0.5720300068067742, - "nauc_map_at_100_std": 0.10188202291296097, - "nauc_map_at_10_diff1": 0.7021694520509648, - "nauc_map_at_10_max": 0.5735604207840197, - "nauc_map_at_10_std": 0.10327802325986768, - "nauc_map_at_1_diff1": 0.7329375789067353, - "nauc_map_at_1_max": 0.5361300015750745, - "nauc_map_at_1_std": 0.06148747407860173, - "nauc_map_at_20_diff1": 0.7035120770713532, - "nauc_map_at_20_max": 0.5729577824792697, - "nauc_map_at_20_std": 0.10260589720880942, - "nauc_map_at_3_diff1": 0.7005784793460021, - "nauc_map_at_3_max": 0.5730788546799835, - "nauc_map_at_3_std": 0.10061225940427539, - "nauc_map_at_5_diff1": 0.7005878046876627, - "nauc_map_at_5_max": 0.5743062534656194, - "nauc_map_at_5_std": 0.10807883793605706, - "nauc_mrr_at_1000_diff1": 0.7039322369949266, - "nauc_mrr_at_1000_max": 0.5719568116443199, - "nauc_mrr_at_1000_std": 0.10175653818610049, - "nauc_mrr_at_100_diff1": 0.7039442745246894, - "nauc_mrr_at_100_max": 0.5720300068067742, - "nauc_mrr_at_100_std": 0.10188202291296097, - "nauc_mrr_at_10_diff1": 0.7021694520509648, - "nauc_mrr_at_10_max": 0.5735604207840197, - "nauc_mrr_at_10_std": 0.10327802325986768, - "nauc_mrr_at_1_diff1": 0.7329375789067353, - "nauc_mrr_at_1_max": 0.5361300015750745, - "nauc_mrr_at_1_std": 0.06148747407860173, - "nauc_mrr_at_20_diff1": 0.7035120770713532, - "nauc_mrr_at_20_max": 0.5729577824792697, - "nauc_mrr_at_20_std": 0.10260589720880942, - "nauc_mrr_at_3_diff1": 0.7005784793460021, - "nauc_mrr_at_3_max": 0.5730788546799835, - "nauc_mrr_at_3_std": 0.10061225940427539, - "nauc_mrr_at_5_diff1": 0.7005878046876627, - "nauc_mrr_at_5_max": 0.5743062534656194, - "nauc_mrr_at_5_std": 0.10807883793605706, - "nauc_ndcg_at_1000_diff1": 0.700087104488326, - "nauc_ndcg_at_1000_max": 0.5795021097985256, - "nauc_ndcg_at_1000_std": 0.11154540403613405, - "nauc_ndcg_at_100_diff1": 0.7006550596100719, - "nauc_ndcg_at_100_max": 0.5817496273763957, - "nauc_ndcg_at_100_std": 0.11502416071470394, - "nauc_ndcg_at_10_diff1": 0.6905913110087234, - "nauc_ndcg_at_10_max": 0.5912371201845855, - "nauc_ndcg_at_10_std": 0.1235839893326974, - "nauc_ndcg_at_1_diff1": 0.7329375789067353, - "nauc_ndcg_at_1_max": 0.5361300015750745, - "nauc_ndcg_at_1_std": 0.06148747407860173, - "nauc_ndcg_at_20_diff1": 0.6966587387995632, - "nauc_ndcg_at_20_max": 0.5891614061644422, - "nauc_ndcg_at_20_std": 0.121914444846387, - "nauc_ndcg_at_3_diff1": 0.6884729340233343, - "nauc_ndcg_at_3_max": 0.587509909021513, - "nauc_ndcg_at_3_std": 0.11641242005242053, - "nauc_ndcg_at_5_diff1": 0.6873311264384971, - "nauc_ndcg_at_5_max": 0.5917469354526042, - "nauc_ndcg_at_5_std": 0.13434677089258235, - "nauc_precision_at_1000_diff1": 0.7770774976657188, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.8009007524578535, - "nauc_precision_at_100_max": 0.9682814302191545, - "nauc_precision_at_100_std": 0.7298566485417957, - "nauc_precision_at_10_diff1": 0.5883217484524637, - "nauc_precision_at_10_max": 0.7557526714389439, - "nauc_precision_at_10_std": 0.31049555624718805, - "nauc_precision_at_1_diff1": 0.7329375789067353, - "nauc_precision_at_1_max": 0.5361300015750745, - "nauc_precision_at_1_std": 0.06148747407860173, - "nauc_precision_at_20_diff1": 0.6231633004078801, - "nauc_precision_at_20_max": 0.8448633839500712, - "nauc_precision_at_20_std": 0.4298429898275066, - "nauc_precision_at_3_diff1": 0.6402739291556786, - "nauc_precision_at_3_max": 0.6452442159383038, - "nauc_precision_at_3_std": 0.17984982027398597, - "nauc_precision_at_5_diff1": 0.6157551065526519, - "nauc_precision_at_5_max": 0.6875468807984131, - "nauc_precision_at_5_std": 0.2830632999344636, - "nauc_recall_at_1000_diff1": 0.7770774976656952, - "nauc_recall_at_1000_max": 1.0, - "nauc_recall_at_1000_std": 1.0, - "nauc_recall_at_100_diff1": 0.8009007524578522, - "nauc_recall_at_100_max": 0.9682814302191535, - "nauc_recall_at_100_std": 0.729856648541773, - "nauc_recall_at_10_diff1": 0.5883217484524672, - "nauc_recall_at_10_max": 0.7557526714389453, - "nauc_recall_at_10_std": 0.3104955562471906, - "nauc_recall_at_1_diff1": 0.7329375789067353, - "nauc_recall_at_1_max": 0.5361300015750745, - "nauc_recall_at_1_std": 0.06148747407860173, - "nauc_recall_at_20_diff1": 0.623163300407883, - "nauc_recall_at_20_max": 0.8448633839500713, - "nauc_recall_at_20_std": 0.42984298982750796, - "nauc_recall_at_3_diff1": 0.6402739291556773, - "nauc_recall_at_3_max": 0.6452442159383034, - "nauc_recall_at_3_std": 0.17984982027398486, - "nauc_recall_at_5_diff1": 0.6157551065526523, - "nauc_recall_at_5_max": 0.6875468807984149, - "nauc_recall_at_5_std": 0.28306329993446605, - "ndcg_at_1": 0.6515, - "ndcg_at_10": 0.79245, - "ndcg_at_100": 0.80558, - "ndcg_at_1000": 0.80656, - "ndcg_at_20": 0.79995, - "ndcg_at_3": 0.75168, - "ndcg_at_5": 0.775, - "precision_at_1": 0.6515, - "precision_at_10": 0.09325, - "precision_at_100": 0.00992, - "precision_at_1000": 0.001, - "precision_at_20": 0.0481, - "precision_at_3": 0.274, - "precision_at_5": 0.1758, - "recall_at_1": 0.6515, - "recall_at_10": 0.9325, - "recall_at_100": 0.9915, - "recall_at_1000": 0.999, - "recall_at_20": 0.962, - "recall_at_3": 0.822, - "recall_at_5": 0.879 - }, - { - "cv_recall_at_1": 0.733, - "cv_recall_at_10": 0.963, - "cv_recall_at_100": 0.9985, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.984, - "cv_recall_at_3": 0.88, - "cv_recall_at_5": 0.9275, - "hf_subset": "ru", - "languages": [ - "rus-Cyrl" - ], - "main_score": 0.85232, - "map_at_1": 0.733, - "map_at_10": 0.81632, - "map_at_100": 0.81832, - "map_at_1000": 0.81834, - "map_at_20": 0.81786, - "map_at_3": 0.8005, - "map_at_5": 0.81127, - "mrr_at_1": 0.733, - "mrr_at_10": 0.8163236111111105, - "mrr_at_100": 0.8183248956975758, - "mrr_at_1000": 0.8183350849805338, - "mrr_at_20": 0.8178556814509012, - "mrr_at_3": 0.8004999999999989, - "mrr_at_5": 0.8112749999999994, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.8513849984438709, - "nauc_cv_recall_at_100_max": 0.8638344226580088, - "nauc_cv_recall_at_100_std": -0.10768751945235333, - "nauc_cv_recall_at_10_diff1": 0.7755759961642286, - "nauc_cv_recall_at_10_max": 0.7924899689605557, - "nauc_cv_recall_at_10_std": 0.03805486158427406, - "nauc_cv_recall_at_1_diff1": 0.8133792547946724, - "nauc_cv_recall_at_1_max": 0.6252031052536563, - "nauc_cv_recall_at_1_std": 0.09226384736404669, - "nauc_cv_recall_at_20_diff1": 0.7742471988795507, - "nauc_cv_recall_at_20_max": 0.8285189075630218, - "nauc_cv_recall_at_20_std": 0.1130222922502361, - "nauc_cv_recall_at_3_diff1": 0.7380508272351263, - "nauc_cv_recall_at_3_max": 0.7140669742284433, - "nauc_cv_recall_at_3_std": 0.09508232580337304, - "nauc_cv_recall_at_5_diff1": 0.7234907756205932, - "nauc_cv_recall_at_5_max": 0.753079622653659, - "nauc_cv_recall_at_5_std": 0.051019028300975704, - "nauc_map_at_1000_diff1": 0.7952082628577973, - "nauc_map_at_1000_max": 0.6478391020353417, - "nauc_map_at_1000_std": 0.09081338024626912, - "nauc_map_at_100_diff1": 0.7952086285167334, - "nauc_map_at_100_max": 0.6478494920991844, - "nauc_map_at_100_std": 0.09077674916537626, - "nauc_map_at_10_diff1": 0.7951467350309017, - "nauc_map_at_10_max": 0.6496051997722657, - "nauc_map_at_10_std": 0.09048990915348447, - "nauc_map_at_1_diff1": 0.8133792547946724, - "nauc_map_at_1_max": 0.6252031052536563, - "nauc_map_at_1_std": 0.09226384736404669, - "nauc_map_at_20_diff1": 0.7952202923863493, - "nauc_map_at_20_max": 0.6483931037134688, - "nauc_map_at_20_std": 0.09102570610885713, - "nauc_map_at_3_diff1": 0.790142614160109, - "nauc_map_at_3_max": 0.6517459403239515, - "nauc_map_at_3_std": 0.09487554955026746, - "nauc_map_at_5_diff1": 0.7914810063107847, - "nauc_map_at_5_max": 0.651685378725342, - "nauc_map_at_5_std": 0.09063688167845714, - "nauc_mrr_at_1000_diff1": 0.7952082628577973, - "nauc_mrr_at_1000_max": 0.6478391020353417, - "nauc_mrr_at_1000_std": 0.09081338024626912, - "nauc_mrr_at_100_diff1": 0.7952086285167334, - "nauc_mrr_at_100_max": 0.6478494920991844, - "nauc_mrr_at_100_std": 0.09077674916537626, - "nauc_mrr_at_10_diff1": 0.7951467350309017, - "nauc_mrr_at_10_max": 0.6496051997722657, - "nauc_mrr_at_10_std": 0.09048990915348447, - "nauc_mrr_at_1_diff1": 0.8133792547946724, - "nauc_mrr_at_1_max": 0.6252031052536563, - "nauc_mrr_at_1_std": 0.09226384736404669, - "nauc_mrr_at_20_diff1": 0.7952202923863493, - "nauc_mrr_at_20_max": 0.6483931037134688, - "nauc_mrr_at_20_std": 0.09102570610885713, - "nauc_mrr_at_3_diff1": 0.790142614160109, - "nauc_mrr_at_3_max": 0.6517459403239515, - "nauc_mrr_at_3_std": 0.09487554955026746, - "nauc_mrr_at_5_diff1": 0.7914810063107847, - "nauc_mrr_at_5_max": 0.651685378725342, - "nauc_mrr_at_5_std": 0.09063688167845714, - "nauc_ndcg_at_1000_diff1": 0.793184615751275, - "nauc_ndcg_at_1000_max": 0.6512997647156443, - "nauc_ndcg_at_1000_std": 0.0896376656750505, - "nauc_ndcg_at_100_diff1": 0.7932576505906453, - "nauc_ndcg_at_100_max": 0.6516218237126856, - "nauc_ndcg_at_100_std": 0.08916329598012869, - "nauc_ndcg_at_10_diff1": 0.7923981235421121, - "nauc_ndcg_at_10_max": 0.6601701521909391, - "nauc_ndcg_at_10_std": 0.08679745362754197, - "nauc_ndcg_at_1_diff1": 0.8133792547946724, - "nauc_ndcg_at_1_max": 0.6252031052536563, - "nauc_ndcg_at_1_std": 0.09226384736404669, - "nauc_ndcg_at_20_diff1": 0.7928995512374133, - "nauc_ndcg_at_20_max": 0.6555697297862627, - "nauc_ndcg_at_20_std": 0.09046056892469329, - "nauc_ndcg_at_3_diff1": 0.7805316637468971, - "nauc_ndcg_at_3_max": 0.6632016786911084, - "nauc_ndcg_at_3_std": 0.0949989221869438, - "nauc_ndcg_at_5_diff1": 0.7825497211406456, - "nauc_ndcg_at_5_max": 0.664619339432301, - "nauc_ndcg_at_5_std": 0.08604866742140158, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.851384998443846, - "nauc_precision_at_100_max": 0.8638344226578644, - "nauc_precision_at_100_std": -0.10768751945219786, - "nauc_precision_at_10_diff1": 0.7755759961642249, - "nauc_precision_at_10_max": 0.7924899689605511, - "nauc_precision_at_10_std": 0.0380548615842704, - "nauc_precision_at_1_diff1": 0.8133792547946724, - "nauc_precision_at_1_max": 0.6252031052536563, - "nauc_precision_at_1_std": 0.09226384736404669, - "nauc_precision_at_20_diff1": 0.7742471988795416, - "nauc_precision_at_20_max": 0.8285189075630202, - "nauc_precision_at_20_std": 0.11302229225024674, - "nauc_precision_at_3_diff1": 0.7380508272351264, - "nauc_precision_at_3_max": 0.7140669742284463, - "nauc_precision_at_3_std": 0.0950823258033741, - "nauc_precision_at_5_diff1": 0.7234907756205926, - "nauc_precision_at_5_max": 0.7530796226536576, - "nauc_precision_at_5_std": 0.05101902830097392, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.8513849984438709, - "nauc_recall_at_100_max": 0.8638344226580088, - "nauc_recall_at_100_std": -0.10768751945235333, - "nauc_recall_at_10_diff1": 0.7755759961642286, - "nauc_recall_at_10_max": 0.7924899689605557, - "nauc_recall_at_10_std": 0.03805486158427406, - "nauc_recall_at_1_diff1": 0.8133792547946724, - "nauc_recall_at_1_max": 0.6252031052536563, - "nauc_recall_at_1_std": 0.09226384736404669, - "nauc_recall_at_20_diff1": 0.7742471988795507, - "nauc_recall_at_20_max": 0.8285189075630218, - "nauc_recall_at_20_std": 0.1130222922502361, - "nauc_recall_at_3_diff1": 0.7380508272351263, - "nauc_recall_at_3_max": 0.7140669742284433, - "nauc_recall_at_3_std": 0.09508232580337304, - "nauc_recall_at_5_diff1": 0.7234907756205932, - "nauc_recall_at_5_max": 0.753079622653659, - "nauc_recall_at_5_std": 0.051019028300975704, - "ndcg_at_1": 0.733, - "ndcg_at_10": 0.85232, - "ndcg_at_100": 0.86059, - "ndcg_at_1000": 0.8608, - "ndcg_at_20": 0.85772, - "ndcg_at_3": 0.82103, - "ndcg_at_5": 0.84053, - "precision_at_1": 0.733, - "precision_at_10": 0.0963, - "precision_at_100": 0.00999, - "precision_at_1000": 0.001, - "precision_at_20": 0.0492, - "precision_at_3": 0.29333, - "precision_at_5": 0.1855, - "recall_at_1": 0.733, - "recall_at_10": 0.963, - "recall_at_100": 0.9985, - "recall_at_1000": 1.0, - "recall_at_20": 0.984, - "recall_at_3": 0.88, - "recall_at_5": 0.9275 - }, - { - "cv_recall_at_1": 0.329, - "cv_recall_at_10": 0.6625, - "cv_recall_at_100": 0.898, - "cv_recall_at_1000": 0.9935, - "cv_recall_at_20": 0.753, - "cv_recall_at_3": 0.488, - "cv_recall_at_5": 0.566, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.48558, - "map_at_1": 0.329, - "map_at_10": 0.43015, - "map_at_100": 0.4402, - "map_at_1000": 0.44068, - "map_at_20": 0.43662, - "map_at_3": 0.39908, - "map_at_5": 0.41716, - "mrr_at_1": 0.329, - "mrr_at_10": 0.4301498015873023, - "mrr_at_100": 0.44020367057678783, - "mrr_at_1000": 0.44068107415161467, - "mrr_at_20": 0.43662267512329517, - "mrr_at_3": 0.3990833333333339, - "mrr_at_5": 0.4171583333333347, - "nauc_cv_recall_at_1000_diff1": 0.570746247216831, - "nauc_cv_recall_at_1000_max": 0.9355383178912546, - "nauc_cv_recall_at_1000_std": 0.8143719026071998, - "nauc_cv_recall_at_100_diff1": 0.28561103871074306, - "nauc_cv_recall_at_100_max": 0.6815610029184481, - "nauc_cv_recall_at_100_std": 0.5854242763532237, - "nauc_cv_recall_at_10_diff1": 0.34808730638537383, - "nauc_cv_recall_at_10_max": 0.6230498863849571, - "nauc_cv_recall_at_10_std": 0.38972169526152306, - "nauc_cv_recall_at_1_diff1": 0.4897268439399369, - "nauc_cv_recall_at_1_max": 0.48143439232866647, - "nauc_cv_recall_at_1_std": 0.22002136550057733, - "nauc_cv_recall_at_20_diff1": 0.3732459116186757, - "nauc_cv_recall_at_20_max": 0.6731810217095842, - "nauc_cv_recall_at_20_std": 0.46552168739127814, - "nauc_cv_recall_at_3_diff1": 0.39113799929941523, - "nauc_cv_recall_at_3_max": 0.5270269194433536, - "nauc_cv_recall_at_3_std": 0.28440694428227187, - "nauc_cv_recall_at_5_diff1": 0.3756919063036706, - "nauc_cv_recall_at_5_max": 0.5573107027224745, - "nauc_cv_recall_at_5_std": 0.3082772997579581, - "nauc_map_at_1000_diff1": 0.4414051282408367, - "nauc_map_at_1000_max": 0.514311675803377, - "nauc_map_at_1000_std": 0.2621652878648845, - "nauc_map_at_100_diff1": 0.4411597090934524, - "nauc_map_at_100_max": 0.5145432570532894, - "nauc_map_at_100_std": 0.2625758722812923, - "nauc_map_at_10_diff1": 0.43856234721121634, - "nauc_map_at_10_max": 0.5144546481309277, - "nauc_map_at_10_std": 0.26100856265121153, - "nauc_map_at_1_diff1": 0.4897268439399369, - "nauc_map_at_1_max": 0.48143439232866647, - "nauc_map_at_1_std": 0.22002136550057733, - "nauc_map_at_20_diff1": 0.44113261525244246, - "nauc_map_at_20_max": 0.5157374897438756, - "nauc_map_at_20_std": 0.2631221937442976, - "nauc_map_at_3_diff1": 0.4430321264763373, - "nauc_map_at_3_max": 0.5016682101912822, - "nauc_map_at_3_std": 0.24791585529023333, - "nauc_map_at_5_diff1": 0.44076808631336867, - "nauc_map_at_5_max": 0.5079389409486393, - "nauc_map_at_5_std": 0.25294942024853534, - "nauc_mrr_at_1000_diff1": 0.4414051282408367, - "nauc_mrr_at_1000_max": 0.514311675803377, - "nauc_mrr_at_1000_std": 0.2621652878648845, - "nauc_mrr_at_100_diff1": 0.4411597090934524, - "nauc_mrr_at_100_max": 0.5145432570532894, - "nauc_mrr_at_100_std": 0.2625758722812923, - "nauc_mrr_at_10_diff1": 0.43856234721121634, - "nauc_mrr_at_10_max": 0.5144546481309277, - "nauc_mrr_at_10_std": 0.26100856265121153, - "nauc_mrr_at_1_diff1": 0.4897268439399369, - "nauc_mrr_at_1_max": 0.48143439232866647, - "nauc_mrr_at_1_std": 0.22002136550057733, - "nauc_mrr_at_20_diff1": 0.44113261525244246, - "nauc_mrr_at_20_max": 0.5157374897438756, - "nauc_mrr_at_20_std": 0.2631221937442976, - "nauc_mrr_at_3_diff1": 0.4430321264763373, - "nauc_mrr_at_3_max": 0.5016682101912822, - "nauc_mrr_at_3_std": 0.24791585529023333, - "nauc_mrr_at_5_diff1": 0.44076808631336867, - "nauc_mrr_at_5_max": 0.5079389409486393, - "nauc_mrr_at_5_std": 0.25294942024853534, - "nauc_ndcg_at_1000_diff1": 0.4330018707890607, - "nauc_ndcg_at_1000_max": 0.5266139850395504, - "nauc_ndcg_at_1000_std": 0.2796469830342363, - "nauc_ndcg_at_100_diff1": 0.4260189469019916, - "nauc_ndcg_at_100_max": 0.5325043437016203, - "nauc_ndcg_at_100_std": 0.2916521593222054, - "nauc_ndcg_at_10_diff1": 0.4190897586950866, - "nauc_ndcg_at_10_max": 0.5363395029642329, - "nauc_ndcg_at_10_std": 0.2870156955251191, - "nauc_ndcg_at_1_diff1": 0.4897268439399369, - "nauc_ndcg_at_1_max": 0.48143439232866647, - "nauc_ndcg_at_1_std": 0.22002136550057733, - "nauc_ndcg_at_20_diff1": 0.4274167281601817, - "nauc_ndcg_at_20_max": 0.5416035681782816, - "nauc_ndcg_at_20_std": 0.2959927566759077, - "nauc_ndcg_at_3_diff1": 0.42958629116833824, - "nauc_ndcg_at_3_max": 0.508177698679707, - "nauc_ndcg_at_3_std": 0.25725414826981835, - "nauc_ndcg_at_5_diff1": 0.42485040568184196, - "nauc_ndcg_at_5_max": 0.5196752086294824, - "nauc_ndcg_at_5_std": 0.26626862328931283, - "nauc_precision_at_1000_diff1": 0.5707462472168311, - "nauc_precision_at_1000_max": 0.9355383178912654, - "nauc_precision_at_1000_std": 0.8143719026071755, - "nauc_precision_at_100_diff1": 0.28561103871074167, - "nauc_precision_at_100_max": 0.6815610029184485, - "nauc_precision_at_100_std": 0.5854242763532232, - "nauc_precision_at_10_diff1": 0.3480873063853732, - "nauc_precision_at_10_max": 0.6230498863849561, - "nauc_precision_at_10_std": 0.3897216952615221, - "nauc_precision_at_1_diff1": 0.4897268439399369, - "nauc_precision_at_1_max": 0.48143439232866647, - "nauc_precision_at_1_std": 0.22002136550057733, - "nauc_precision_at_20_diff1": 0.3732459116186754, - "nauc_precision_at_20_max": 0.6731810217095847, - "nauc_precision_at_20_std": 0.4655216873912767, - "nauc_precision_at_3_diff1": 0.3911379992994151, - "nauc_precision_at_3_max": 0.5270269194433539, - "nauc_precision_at_3_std": 0.28440694428227226, - "nauc_precision_at_5_diff1": 0.37569190630366955, - "nauc_precision_at_5_max": 0.557310702722474, - "nauc_precision_at_5_std": 0.3082772997579572, - "nauc_recall_at_1000_diff1": 0.570746247216831, - "nauc_recall_at_1000_max": 0.9355383178912546, - "nauc_recall_at_1000_std": 0.8143719026071998, - "nauc_recall_at_100_diff1": 0.28561103871074306, - "nauc_recall_at_100_max": 0.6815610029184481, - "nauc_recall_at_100_std": 0.5854242763532237, - "nauc_recall_at_10_diff1": 0.34808730638537383, - "nauc_recall_at_10_max": 0.6230498863849571, - "nauc_recall_at_10_std": 0.38972169526152306, - "nauc_recall_at_1_diff1": 0.4897268439399369, - "nauc_recall_at_1_max": 0.48143439232866647, - "nauc_recall_at_1_std": 0.22002136550057733, - "nauc_recall_at_20_diff1": 0.3732459116186757, - "nauc_recall_at_20_max": 0.6731810217095842, - "nauc_recall_at_20_std": 0.46552168739127814, - "nauc_recall_at_3_diff1": 0.39113799929941523, - "nauc_recall_at_3_max": 0.5270269194433536, - "nauc_recall_at_3_std": 0.28440694428227187, - "nauc_recall_at_5_diff1": 0.3756919063036706, - "nauc_recall_at_5_max": 0.5573107027224745, - "nauc_recall_at_5_std": 0.3082772997579581, - "ndcg_at_1": 0.329, - "ndcg_at_10": 0.48558, - "ndcg_at_100": 0.53536, - "ndcg_at_1000": 0.5476, - "ndcg_at_20": 0.5087, - "ndcg_at_3": 0.42192, - "ndcg_at_5": 0.45426, - "precision_at_1": 0.329, - "precision_at_10": 0.06625, - "precision_at_100": 0.00898, - "precision_at_1000": 0.00099, - "precision_at_20": 0.03765, - "precision_at_3": 0.16267, - "precision_at_5": 0.1132, - "recall_at_1": 0.329, - "recall_at_10": 0.6625, - "recall_at_100": 0.898, - "recall_at_1000": 0.9935, - "recall_at_20": 0.753, - "recall_at_3": 0.488, - "recall_at_5": 0.566 - }, - { - "cv_recall_at_1": 0.7535, - "cv_recall_at_10": 0.9685, - "cv_recall_at_100": 0.9985, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.986, - "cv_recall_at_3": 0.8905, - "cv_recall_at_5": 0.939, - "hf_subset": "zh", - "languages": [ - "zho-Hans" - ], - "main_score": 0.8647, - "map_at_1": 0.7535, - "map_at_10": 0.83094, - "map_at_100": 0.83264, - "map_at_1000": 0.83265, - "map_at_20": 0.83224, - "map_at_3": 0.81583, - "map_at_5": 0.82686, - "mrr_at_1": 0.7535, - "mrr_at_10": 0.8309436507936507, - "mrr_at_100": 0.8326428461698397, - "mrr_at_1000": 0.8326528711689501, - "mrr_at_20": 0.8322375589172607, - "mrr_at_3": 0.815833333333333, - "mrr_at_5": 0.8268583333333337, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.9074074074074042, - "nauc_cv_recall_at_100_max": 0.9564270152506046, - "nauc_cv_recall_at_100_std": 0.08667911609083245, - "nauc_cv_recall_at_10_diff1": 0.7159604582573755, - "nauc_cv_recall_at_10_max": 0.7671216634802024, - "nauc_cv_recall_at_10_std": -0.3054925673973297, - "nauc_cv_recall_at_1_diff1": 0.810180475530634, - "nauc_cv_recall_at_1_max": 0.5523023359455304, - "nauc_cv_recall_at_1_std": -0.18382166913136913, - "nauc_cv_recall_at_20_diff1": 0.7574196345204801, - "nauc_cv_recall_at_20_max": 0.8418867547018848, - "nauc_cv_recall_at_20_std": -0.17697078831532298, - "nauc_cv_recall_at_3_diff1": 0.7287258127811141, - "nauc_cv_recall_at_3_max": 0.670665706437131, - "nauc_cv_recall_at_3_std": -0.1443513922609512, - "nauc_cv_recall_at_5_diff1": 0.718185088242946, - "nauc_cv_recall_at_5_max": 0.7166850346696072, - "nauc_cv_recall_at_5_std": -0.17488252131453574, - "nauc_map_at_1000_diff1": 0.7907323043547354, - "nauc_map_at_1000_max": 0.5861408086781905, - "nauc_map_at_1000_std": -0.1787536887590416, - "nauc_map_at_100_diff1": 0.7907384415401878, - "nauc_map_at_100_max": 0.5861665619023456, - "nauc_map_at_100_std": -0.17870425477150786, - "nauc_map_at_10_diff1": 0.7898007655152364, - "nauc_map_at_10_max": 0.58760606658666, - "nauc_map_at_10_std": -0.18104252909636068, - "nauc_map_at_1_diff1": 0.810180475530634, - "nauc_map_at_1_max": 0.5523023359455304, - "nauc_map_at_1_std": -0.18382166913136913, - "nauc_map_at_20_diff1": 0.7905669610329621, - "nauc_map_at_20_max": 0.586723738010071, - "nauc_map_at_20_std": -0.17897834402449417, - "nauc_map_at_3_diff1": 0.7848945769498309, - "nauc_map_at_3_max": 0.5899769317245516, - "nauc_map_at_3_std": -0.17212119267453208, - "nauc_map_at_5_diff1": 0.7879028117425809, - "nauc_map_at_5_max": 0.5892120425613406, - "nauc_map_at_5_std": -0.17707379971722223, - "nauc_mrr_at_1000_diff1": 0.7907323043547354, - "nauc_mrr_at_1000_max": 0.5861408086781905, - "nauc_mrr_at_1000_std": -0.1787536887590416, - "nauc_mrr_at_100_diff1": 0.7907384415401878, - "nauc_mrr_at_100_max": 0.5861665619023456, - "nauc_mrr_at_100_std": -0.17870425477150786, - "nauc_mrr_at_10_diff1": 0.7898007655152364, - "nauc_mrr_at_10_max": 0.58760606658666, - "nauc_mrr_at_10_std": -0.18104252909636068, - "nauc_mrr_at_1_diff1": 0.810180475530634, - "nauc_mrr_at_1_max": 0.5523023359455304, - "nauc_mrr_at_1_std": -0.18382166913136913, - "nauc_mrr_at_20_diff1": 0.7905669610329621, - "nauc_mrr_at_20_max": 0.586723738010071, - "nauc_mrr_at_20_std": -0.17897834402449417, - "nauc_mrr_at_3_diff1": 0.7848945769498309, - "nauc_mrr_at_3_max": 0.5899769317245516, - "nauc_mrr_at_3_std": -0.17212119267453208, - "nauc_mrr_at_5_diff1": 0.7879028117425809, - "nauc_mrr_at_5_max": 0.5892120425613406, - "nauc_mrr_at_5_std": -0.17707379971722223, - "nauc_ndcg_at_1000_diff1": 0.7886545545175407, - "nauc_ndcg_at_1000_max": 0.5912553799694619, - "nauc_ndcg_at_1000_std": -0.17878692861317544, - "nauc_ndcg_at_100_diff1": 0.7888518971892833, - "nauc_ndcg_at_100_max": 0.5919000390709643, - "nauc_ndcg_at_100_std": -0.1781301914289305, - "nauc_ndcg_at_10_diff1": 0.7841553474017766, - "nauc_ndcg_at_10_max": 0.6004233460965622, - "nauc_ndcg_at_10_std": -0.18770318177002843, - "nauc_ndcg_at_1_diff1": 0.810180475530634, - "nauc_ndcg_at_1_max": 0.5523023359455304, - "nauc_ndcg_at_1_std": -0.18382166913136913, - "nauc_ndcg_at_20_diff1": 0.7877499835627467, - "nauc_ndcg_at_20_max": 0.5966008749588094, - "nauc_ndcg_at_20_std": -0.17918935976242298, - "nauc_ndcg_at_3_diff1": 0.7746259970314816, - "nauc_ndcg_at_3_max": 0.6047740676129029, - "nauc_ndcg_at_3_std": -0.1670716308750744, - "nauc_ndcg_at_5_diff1": 0.779426755716452, - "nauc_ndcg_at_5_max": 0.6045320925860991, - "nauc_ndcg_at_5_std": -0.17646919507329487, - "nauc_precision_at_1000_diff1": NaN, - "nauc_precision_at_1000_max": NaN, - "nauc_precision_at_1000_std": NaN, - "nauc_precision_at_100_diff1": 0.9074074074073368, - "nauc_precision_at_100_max": 0.9564270152505274, - "nauc_precision_at_100_std": 0.0866791160907527, - "nauc_precision_at_10_diff1": 0.7159604582573762, - "nauc_precision_at_10_max": 0.7671216634802053, - "nauc_precision_at_10_std": -0.30549256739733754, - "nauc_precision_at_1_diff1": 0.810180475530634, - "nauc_precision_at_1_max": 0.5523023359455304, - "nauc_precision_at_1_std": -0.18382166913136913, - "nauc_precision_at_20_diff1": 0.7574196345204598, - "nauc_precision_at_20_max": 0.8418867547018706, - "nauc_precision_at_20_std": -0.1769707883153585, - "nauc_precision_at_3_diff1": 0.7287258127811156, - "nauc_precision_at_3_max": 0.6706657064371305, - "nauc_precision_at_3_std": -0.14435139226094928, - "nauc_precision_at_5_diff1": 0.7181850882429455, - "nauc_precision_at_5_max": 0.7166850346696044, - "nauc_precision_at_5_std": -0.17488252131453935, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.9074074074074042, - "nauc_recall_at_100_max": 0.9564270152506046, - "nauc_recall_at_100_std": 0.08667911609083245, - "nauc_recall_at_10_diff1": 0.7159604582573755, - "nauc_recall_at_10_max": 0.7671216634802024, - "nauc_recall_at_10_std": -0.3054925673973297, - "nauc_recall_at_1_diff1": 0.810180475530634, - "nauc_recall_at_1_max": 0.5523023359455304, - "nauc_recall_at_1_std": -0.18382166913136913, - "nauc_recall_at_20_diff1": 0.7574196345204801, - "nauc_recall_at_20_max": 0.8418867547018848, - "nauc_recall_at_20_std": -0.17697078831532298, - "nauc_recall_at_3_diff1": 0.7287258127811141, - "nauc_recall_at_3_max": 0.670665706437131, - "nauc_recall_at_3_std": -0.1443513922609512, - "nauc_recall_at_5_diff1": 0.718185088242946, - "nauc_recall_at_5_max": 0.7166850346696072, - "nauc_recall_at_5_std": -0.17488252131453574, - "ndcg_at_1": 0.7535, - "ndcg_at_10": 0.8647, - "ndcg_at_100": 0.87171, - "ndcg_at_1000": 0.87191, - "ndcg_at_20": 0.86922, - "ndcg_at_3": 0.83509, - "ndcg_at_5": 0.85502, - "precision_at_1": 0.7535, - "precision_at_10": 0.09685, - "precision_at_100": 0.00999, - "precision_at_1000": 0.001, - "precision_at_20": 0.0493, - "precision_at_3": 0.29683, - "precision_at_5": 0.1878, - "recall_at_1": 0.7535, - "recall_at_10": 0.9685, - "recall_at_100": 0.9985, - "recall_at_1000": 1.0, - "recall_at_20": 0.986, - "recall_at_3": 0.8905, - "recall_at_5": 0.939 - } - ] - }, - "task_name": "XFlickr30kCoT2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/XM3600T2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/XM3600T2IRetrieval.json deleted file mode 100644 index e68e5e1d02..0000000000 --- a/results-mieb/voyage-multimodal-3/1/XM3600T2IRetrieval.json +++ /dev/null @@ -1,6311 +0,0 @@ -{ - "dataset_revision": "8d3e5665526c55a5855cd6ddfbaba2032bc7cee4", - "evaluation_time": 37511.4420838356, - "kg_co2_emissions": null, - "mteb_version": "1.14.15", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.34125, - "cv_recall_at_10": 0.67287, - "cv_recall_at_100": 0.91421, - "cv_recall_at_1000": 0.99525, - "cv_recall_at_20": 0.76164, - "cv_recall_at_3": 0.50821, - "cv_recall_at_5": 0.58681, - "hf_subset": "ar", - "languages": [ - "ara-Arab" - ], - "main_score": 0.49827, - "map_at_1": 0.34125, - "map_at_10": 0.44329, - "map_at_100": 0.45355, - "map_at_1000": 0.45395, - "map_at_20": 0.44954, - "map_at_3": 0.41378, - "map_at_5": 0.43164, - "mrr_at_1": 0.3412515270802226, - "mrr_at_10": 0.44329409787534, - "mrr_at_100": 0.4535499457119047, - "mrr_at_1000": 0.45395489734902766, - "mrr_at_20": 0.4495370855277087, - "mrr_at_3": 0.41378218180173004, - "mrr_at_5": 0.43163883987149987, - "nauc_cv_recall_at_1000_diff1": 0.4337496830686869, - "nauc_cv_recall_at_1000_max": 0.8412941148071966, - "nauc_cv_recall_at_1000_std": 0.947111521299572, - "nauc_cv_recall_at_100_diff1": 0.3580251362565319, - "nauc_cv_recall_at_100_max": 0.6640640610501624, - "nauc_cv_recall_at_100_std": 0.5675327555710422, - "nauc_cv_recall_at_10_diff1": 0.4196819928283325, - "nauc_cv_recall_at_10_max": 0.4847935456565573, - "nauc_cv_recall_at_10_std": 0.17822153608684996, - "nauc_cv_recall_at_1_diff1": 0.5748737887556816, - "nauc_cv_recall_at_1_max": 0.4046463836442685, - "nauc_cv_recall_at_1_std": 0.03730800730770686, - "nauc_cv_recall_at_20_diff1": 0.3998000878261237, - "nauc_cv_recall_at_20_max": 0.5260038074914021, - "nauc_cv_recall_at_20_std": 0.2575834089930464, - "nauc_cv_recall_at_3_diff1": 0.4671406343027723, - "nauc_cv_recall_at_3_max": 0.43526122324655075, - "nauc_cv_recall_at_3_std": 0.09393815190552963, - "nauc_cv_recall_at_5_diff1": 0.4436903813254511, - "nauc_cv_recall_at_5_max": 0.44982329761040435, - "nauc_cv_recall_at_5_std": 0.1137201501321779, - "nauc_map_at_1000_diff1": 0.5233639336100439, - "nauc_map_at_1000_max": 0.42625971848572086, - "nauc_map_at_1000_std": 0.0741080415578734, - "nauc_map_at_100_diff1": 0.5231536531540483, - "nauc_map_at_100_max": 0.4264913079765343, - "nauc_map_at_100_std": 0.07459475731986948, - "nauc_map_at_10_diff1": 0.5222653578421395, - "nauc_map_at_10_max": 0.42578248644685257, - "nauc_map_at_10_std": 0.0722300615883244, - "nauc_map_at_1_diff1": 0.5748737887556816, - "nauc_map_at_1_max": 0.4046463836442685, - "nauc_map_at_1_std": 0.03730800730770686, - "nauc_map_at_20_diff1": 0.5224573371553266, - "nauc_map_at_20_max": 0.42670066441524435, - "nauc_map_at_20_std": 0.07428043686934636, - "nauc_map_at_3_diff1": 0.5272012215426869, - "nauc_map_at_3_max": 0.41957225459462183, - "nauc_map_at_3_std": 0.062243752690080166, - "nauc_map_at_5_diff1": 0.523499038800135, - "nauc_map_at_5_max": 0.42241837980697394, - "nauc_map_at_5_std": 0.06596759643358041, - "nauc_mrr_at_1000_diff1": 0.5233639336100439, - "nauc_mrr_at_1000_max": 0.42625971848572086, - "nauc_mrr_at_1000_std": 0.0741080415578734, - "nauc_mrr_at_100_diff1": 0.5231536531540483, - "nauc_mrr_at_100_max": 0.4264913079765343, - "nauc_mrr_at_100_std": 0.07459475731986948, - "nauc_mrr_at_10_diff1": 0.5222653578421395, - "nauc_mrr_at_10_max": 0.42578248644685257, - "nauc_mrr_at_10_std": 0.0722300615883244, - "nauc_mrr_at_1_diff1": 0.5748737887556816, - "nauc_mrr_at_1_max": 0.4046463836442685, - "nauc_mrr_at_1_std": 0.03730800730770686, - "nauc_mrr_at_20_diff1": 0.5224573371553266, - "nauc_mrr_at_20_max": 0.42670066441524435, - "nauc_mrr_at_20_std": 0.07428043686934636, - "nauc_mrr_at_3_diff1": 0.5272012215426869, - "nauc_mrr_at_3_max": 0.41957225459462183, - "nauc_mrr_at_3_std": 0.062243752690080166, - "nauc_mrr_at_5_diff1": 0.523499038800135, - "nauc_mrr_at_5_max": 0.42241837980697394, - "nauc_mrr_at_5_std": 0.06596759643358041, - "nauc_ndcg_at_1000_diff1": 0.5121095899758535, - "nauc_ndcg_at_1000_max": 0.4350773199360631, - "nauc_ndcg_at_1000_std": 0.09165177551394452, - "nauc_ndcg_at_100_diff1": 0.5063075594145833, - "nauc_ndcg_at_100_max": 0.44272557540194885, - "nauc_ndcg_at_100_std": 0.10759226879109529, - "nauc_ndcg_at_10_diff1": 0.5005565980631268, - "nauc_ndcg_at_10_max": 0.4376288011730624, - "nauc_ndcg_at_10_std": 0.09344467085604088, - "nauc_ndcg_at_1_diff1": 0.5748737887556816, - "nauc_ndcg_at_1_max": 0.4046463836442685, - "nauc_ndcg_at_1_std": 0.03730800730770686, - "nauc_ndcg_at_20_diff1": 0.5007512854572422, - "nauc_ndcg_at_20_max": 0.44231034997606367, - "nauc_ndcg_at_20_std": 0.10301397573730728, - "nauc_ndcg_at_3_diff1": 0.5118924631476841, - "nauc_ndcg_at_3_max": 0.42363017314963436, - "nauc_ndcg_at_3_std": 0.07031701275239649, - "nauc_ndcg_at_5_diff1": 0.5044496190290501, - "nauc_ndcg_at_5_max": 0.42893401043585694, - "nauc_ndcg_at_5_std": 0.07729988422114781, - "nauc_precision_at_1000_diff1": 0.43374968306866063, - "nauc_precision_at_1000_max": 0.8412941148072187, - "nauc_precision_at_1000_std": 0.9471115212995683, - "nauc_precision_at_100_diff1": 0.35802513625653104, - "nauc_precision_at_100_max": 0.6640640610501625, - "nauc_precision_at_100_std": 0.5675327555710422, - "nauc_precision_at_10_diff1": 0.41968199282833213, - "nauc_precision_at_10_max": 0.48479354565655736, - "nauc_precision_at_10_std": 0.1782215360868503, - "nauc_precision_at_1_diff1": 0.5748737887556816, - "nauc_precision_at_1_max": 0.4046463836442685, - "nauc_precision_at_1_std": 0.03730800730770686, - "nauc_precision_at_20_diff1": 0.3998000878261242, - "nauc_precision_at_20_max": 0.526003807491402, - "nauc_precision_at_20_std": 0.25758340899304577, - "nauc_precision_at_3_diff1": 0.46714063430277264, - "nauc_precision_at_3_max": 0.4352612232465508, - "nauc_precision_at_3_std": 0.0939381519055294, - "nauc_precision_at_5_diff1": 0.44369038132545086, - "nauc_precision_at_5_max": 0.44982329761040407, - "nauc_precision_at_5_std": 0.11372015013217776, - "nauc_recall_at_1000_diff1": 0.4337496830686869, - "nauc_recall_at_1000_max": 0.8412941148071966, - "nauc_recall_at_1000_std": 0.947111521299572, - "nauc_recall_at_100_diff1": 0.3580251362565319, - "nauc_recall_at_100_max": 0.6640640610501624, - "nauc_recall_at_100_std": 0.5675327555710422, - "nauc_recall_at_10_diff1": 0.4196819928283325, - "nauc_recall_at_10_max": 0.4847935456565573, - "nauc_recall_at_10_std": 0.17822153608684996, - "nauc_recall_at_1_diff1": 0.5748737887556816, - "nauc_recall_at_1_max": 0.4046463836442685, - "nauc_recall_at_1_std": 0.03730800730770686, - "nauc_recall_at_20_diff1": 0.3998000878261237, - "nauc_recall_at_20_max": 0.5260038074914021, - "nauc_recall_at_20_std": 0.2575834089930464, - "nauc_recall_at_3_diff1": 0.4671406343027723, - "nauc_recall_at_3_max": 0.43526122324655075, - "nauc_recall_at_3_std": 0.09393815190552963, - "nauc_recall_at_5_diff1": 0.4436903813254511, - "nauc_recall_at_5_max": 0.44982329761040435, - "nauc_recall_at_5_std": 0.1137201501321779, - "ndcg_at_1": 0.34125, - "ndcg_at_10": 0.49827, - "ndcg_at_100": 0.54933, - "ndcg_at_1000": 0.55972, - "ndcg_at_20": 0.52082, - "ndcg_at_3": 0.43799, - "ndcg_at_5": 0.47027, - "precision_at_1": 0.34125, - "precision_at_10": 0.06729, - "precision_at_100": 0.00914, - "precision_at_1000": 0.001, - "precision_at_20": 0.03808, - "precision_at_3": 0.1694, - "precision_at_5": 0.11736, - "recall_at_1": 0.34125, - "recall_at_10": 0.67287, - "recall_at_100": 0.91421, - "recall_at_1000": 0.99525, - "recall_at_20": 0.76164, - "recall_at_3": 0.50821, - "recall_at_5": 0.58681 - }, - { - "cv_recall_at_1": 0.09139, - "cv_recall_at_10": 0.30556, - "cv_recall_at_100": 0.65111, - "cv_recall_at_1000": 0.94139, - "cv_recall_at_20": 0.40417, - "cv_recall_at_3": 0.175, - "cv_recall_at_5": 0.2225, - "hf_subset": "bn", - "languages": [ - "ben-Beng" - ], - "main_score": 0.18545, - "map_at_1": 0.09139, - "map_at_10": 0.14866, - "map_at_100": 0.16133, - "map_at_1000": 0.16255, - "map_at_20": 0.15547, - "map_at_3": 0.12685, - "map_at_5": 0.13764, - "mrr_at_1": 0.0913888888888889, - "mrr_at_10": 0.14865861992945317, - "mrr_at_100": 0.16133118432056448, - "mrr_at_1000": 0.16254998543628785, - "mrr_at_20": 0.15547410346433166, - "mrr_at_3": 0.12685185185185158, - "mrr_at_5": 0.13764351851851805, - "nauc_cv_recall_at_1000_diff1": 0.1463176107725881, - "nauc_cv_recall_at_1000_max": 0.432051809665412, - "nauc_cv_recall_at_1000_std": 0.6566857390665582, - "nauc_cv_recall_at_100_diff1": 0.09256493988960171, - "nauc_cv_recall_at_100_max": 0.31923931441163583, - "nauc_cv_recall_at_100_std": 0.43131748148846194, - "nauc_cv_recall_at_10_diff1": 0.14427143766141398, - "nauc_cv_recall_at_10_max": 0.23871243028783165, - "nauc_cv_recall_at_10_std": 0.23834337687614615, - "nauc_cv_recall_at_1_diff1": 0.2843768018313742, - "nauc_cv_recall_at_1_max": 0.22642949706528298, - "nauc_cv_recall_at_1_std": 0.10465485168200821, - "nauc_cv_recall_at_20_diff1": 0.12648614968120367, - "nauc_cv_recall_at_20_max": 0.25141099395352257, - "nauc_cv_recall_at_20_std": 0.2806707883697071, - "nauc_cv_recall_at_3_diff1": 0.165651630383353, - "nauc_cv_recall_at_3_max": 0.2255198450052544, - "nauc_cv_recall_at_3_std": 0.16683700150910694, - "nauc_cv_recall_at_5_diff1": 0.14574711280727373, - "nauc_cv_recall_at_5_max": 0.22042896416792016, - "nauc_cv_recall_at_5_std": 0.19751873920296673, - "nauc_map_at_1000_diff1": 0.20534403160355588, - "nauc_map_at_1000_max": 0.2315760856254332, - "nauc_map_at_1000_std": 0.1682451290346834, - "nauc_map_at_100_diff1": 0.2049094254754938, - "nauc_map_at_100_max": 0.2316968894689101, - "nauc_map_at_100_std": 0.16869208240463784, - "nauc_map_at_10_diff1": 0.20802098159538998, - "nauc_map_at_10_max": 0.22787432381186667, - "nauc_map_at_10_std": 0.1587117311493885, - "nauc_map_at_1_diff1": 0.2843768018313742, - "nauc_map_at_1_max": 0.22642949706528298, - "nauc_map_at_1_std": 0.10465485168200821, - "nauc_map_at_20_diff1": 0.20541601683277563, - "nauc_map_at_20_max": 0.22918516814373516, - "nauc_map_at_20_std": 0.16411208132577196, - "nauc_map_at_3_diff1": 0.21836768475010676, - "nauc_map_at_3_max": 0.2269940196497221, - "nauc_map_at_3_std": 0.13857795813754806, - "nauc_map_at_5_diff1": 0.21000082961031727, - "nauc_map_at_5_max": 0.22530968187638267, - "nauc_map_at_5_std": 0.14928697575884342, - "nauc_mrr_at_1000_diff1": 0.20534403160355588, - "nauc_mrr_at_1000_max": 0.2315760856254332, - "nauc_mrr_at_1000_std": 0.1682451290346834, - "nauc_mrr_at_100_diff1": 0.2049094254754938, - "nauc_mrr_at_100_max": 0.2316968894689101, - "nauc_mrr_at_100_std": 0.16869208240463784, - "nauc_mrr_at_10_diff1": 0.20802098159538998, - "nauc_mrr_at_10_max": 0.22787432381186667, - "nauc_mrr_at_10_std": 0.1587117311493885, - "nauc_mrr_at_1_diff1": 0.2843768018313742, - "nauc_mrr_at_1_max": 0.22642949706528298, - "nauc_mrr_at_1_std": 0.10465485168200821, - "nauc_mrr_at_20_diff1": 0.20541601683277563, - "nauc_mrr_at_20_max": 0.22918516814373516, - "nauc_mrr_at_20_std": 0.16411208132577196, - "nauc_mrr_at_3_diff1": 0.21836768475010676, - "nauc_mrr_at_3_max": 0.2269940196497221, - "nauc_mrr_at_3_std": 0.13857795813754806, - "nauc_mrr_at_5_diff1": 0.21000082961031727, - "nauc_mrr_at_5_max": 0.22530968187638267, - "nauc_mrr_at_5_std": 0.14928697575884342, - "nauc_ndcg_at_1000_diff1": 0.18662594605709457, - "nauc_ndcg_at_1000_max": 0.2457099584894618, - "nauc_ndcg_at_1000_std": 0.21375918674731395, - "nauc_ndcg_at_100_diff1": 0.17441894991358925, - "nauc_ndcg_at_100_max": 0.25014125671538306, - "nauc_ndcg_at_100_std": 0.2301861852193149, - "nauc_ndcg_at_10_diff1": 0.1862184690915828, - "nauc_ndcg_at_10_max": 0.23081719080160418, - "nauc_ndcg_at_10_std": 0.1842803191599018, - "nauc_ndcg_at_1_diff1": 0.2843768018313742, - "nauc_ndcg_at_1_max": 0.22642949706528298, - "nauc_ndcg_at_1_std": 0.10465485168200821, - "nauc_ndcg_at_20_diff1": 0.1788396032268059, - "nauc_ndcg_at_20_max": 0.2353158331926509, - "nauc_ndcg_at_20_std": 0.20060602016675325, - "nauc_ndcg_at_3_diff1": 0.2021741614985555, - "nauc_ndcg_at_3_max": 0.22658302834150937, - "nauc_ndcg_at_3_std": 0.14723330002930776, - "nauc_ndcg_at_5_diff1": 0.1893465371489313, - "nauc_ndcg_at_5_max": 0.22383697227093177, - "nauc_ndcg_at_5_std": 0.16449380903854632, - "nauc_precision_at_1000_diff1": 0.14631761077258681, - "nauc_precision_at_1000_max": 0.4320518096654112, - "nauc_precision_at_1000_std": 0.6566857390665592, - "nauc_precision_at_100_diff1": 0.09256493988960134, - "nauc_precision_at_100_max": 0.31923931441163605, - "nauc_precision_at_100_std": 0.431317481488462, - "nauc_precision_at_10_diff1": 0.1442714376614143, - "nauc_precision_at_10_max": 0.23871243028783182, - "nauc_precision_at_10_std": 0.2383433768761465, - "nauc_precision_at_1_diff1": 0.2843768018313742, - "nauc_precision_at_1_max": 0.22642949706528298, - "nauc_precision_at_1_std": 0.10465485168200821, - "nauc_precision_at_20_diff1": 0.1264861496812037, - "nauc_precision_at_20_max": 0.25141099395352234, - "nauc_precision_at_20_std": 0.28067078836970705, - "nauc_precision_at_3_diff1": 0.165651630383353, - "nauc_precision_at_3_max": 0.2255198450052545, - "nauc_precision_at_3_std": 0.16683700150910682, - "nauc_precision_at_5_diff1": 0.1457471128072736, - "nauc_precision_at_5_max": 0.22042896416792016, - "nauc_precision_at_5_std": 0.1975187392029669, - "nauc_recall_at_1000_diff1": 0.1463176107725881, - "nauc_recall_at_1000_max": 0.432051809665412, - "nauc_recall_at_1000_std": 0.6566857390665582, - "nauc_recall_at_100_diff1": 0.09256493988960171, - "nauc_recall_at_100_max": 0.31923931441163583, - "nauc_recall_at_100_std": 0.43131748148846194, - "nauc_recall_at_10_diff1": 0.14427143766141398, - "nauc_recall_at_10_max": 0.23871243028783165, - "nauc_recall_at_10_std": 0.23834337687614615, - "nauc_recall_at_1_diff1": 0.2843768018313742, - "nauc_recall_at_1_max": 0.22642949706528298, - "nauc_recall_at_1_std": 0.10465485168200821, - "nauc_recall_at_20_diff1": 0.12648614968120367, - "nauc_recall_at_20_max": 0.25141099395352257, - "nauc_recall_at_20_std": 0.2806707883697071, - "nauc_recall_at_3_diff1": 0.165651630383353, - "nauc_recall_at_3_max": 0.2255198450052544, - "nauc_recall_at_3_std": 0.16683700150910694, - "nauc_recall_at_5_diff1": 0.14574711280727373, - "nauc_recall_at_5_max": 0.22042896416792016, - "nauc_recall_at_5_std": 0.19751873920296673, - "ndcg_at_1": 0.09139, - "ndcg_at_10": 0.18545, - "ndcg_at_100": 0.25521, - "ndcg_at_1000": 0.29117, - "ndcg_at_20": 0.21035, - "ndcg_at_3": 0.13916, - "ndcg_at_5": 0.15867, - "precision_at_1": 0.09139, - "precision_at_10": 0.03056, - "precision_at_100": 0.00651, - "precision_at_1000": 0.00094, - "precision_at_20": 0.02021, - "precision_at_3": 0.05833, - "precision_at_5": 0.0445, - "recall_at_1": 0.09139, - "recall_at_10": 0.30556, - "recall_at_100": 0.65111, - "recall_at_1000": 0.94139, - "recall_at_20": 0.40417, - "recall_at_3": 0.175, - "recall_at_5": 0.2225 - }, - { - "cv_recall_at_1": 0.35202, - "cv_recall_at_10": 0.69807, - "cv_recall_at_100": 0.91716, - "cv_recall_at_1000": 0.99043, - "cv_recall_at_20": 0.7791, - "cv_recall_at_3": 0.52033, - "cv_recall_at_5": 0.59609, - "hf_subset": "cs", - "languages": [ - "ces-Latn" - ], - "main_score": 0.51423, - "map_at_1": 0.35202, - "map_at_10": 0.45667, - "map_at_100": 0.46613, - "map_at_1000": 0.46649, - "map_at_20": 0.46242, - "map_at_3": 0.4256, - "map_at_5": 0.44292, - "mrr_at_1": 0.3520188705425281, - "mrr_at_10": 0.4566733510850353, - "mrr_at_100": 0.4661325189110714, - "mrr_at_1000": 0.4664901648332512, - "mrr_at_20": 0.46242210594362715, - "mrr_at_3": 0.42560473613616545, - "mrr_at_5": 0.44292123398547745, - "nauc_cv_recall_at_1000_diff1": 0.43984952545943873, - "nauc_cv_recall_at_1000_max": 0.9333946385512961, - "nauc_cv_recall_at_1000_std": 0.9622112512663857, - "nauc_cv_recall_at_100_diff1": 0.4365013750273154, - "nauc_cv_recall_at_100_max": 0.781445962038063, - "nauc_cv_recall_at_100_std": 0.6866576843293807, - "nauc_cv_recall_at_10_diff1": 0.4440013860687523, - "nauc_cv_recall_at_10_max": 0.5457172777528433, - "nauc_cv_recall_at_10_std": 0.24115786357421992, - "nauc_cv_recall_at_1_diff1": 0.5917764418026163, - "nauc_cv_recall_at_1_max": 0.432157993286181, - "nauc_cv_recall_at_1_std": 0.06054202201021012, - "nauc_cv_recall_at_20_diff1": 0.43832825010501836, - "nauc_cv_recall_at_20_max": 0.6175497235851238, - "nauc_cv_recall_at_20_std": 0.3565597779245211, - "nauc_cv_recall_at_3_diff1": 0.4759945496272966, - "nauc_cv_recall_at_3_max": 0.4690612568061717, - "nauc_cv_recall_at_3_std": 0.11757529811594443, - "nauc_cv_recall_at_5_diff1": 0.45499226706325, - "nauc_cv_recall_at_5_max": 0.48788633462122094, - "nauc_cv_recall_at_5_std": 0.14809518063666652, - "nauc_map_at_1000_diff1": 0.5399112943616813, - "nauc_map_at_1000_max": 0.4600690969544165, - "nauc_map_at_1000_std": 0.10168618302358416, - "nauc_map_at_100_diff1": 0.5397977611488858, - "nauc_map_at_100_max": 0.46034560041005235, - "nauc_map_at_100_std": 0.10218056859319692, - "nauc_map_at_10_diff1": 0.5380519968078198, - "nauc_map_at_10_max": 0.45838195646878466, - "nauc_map_at_10_std": 0.09868025608798509, - "nauc_map_at_1_diff1": 0.5917764418026163, - "nauc_map_at_1_max": 0.432157993286181, - "nauc_map_at_1_std": 0.06054202201021012, - "nauc_map_at_20_diff1": 0.5389459887198278, - "nauc_map_at_20_max": 0.4605616437478957, - "nauc_map_at_20_std": 0.10203876555994912, - "nauc_map_at_3_diff1": 0.5406680525792159, - "nauc_map_at_3_max": 0.4494874340157258, - "nauc_map_at_3_std": 0.08437790118672613, - "nauc_map_at_5_diff1": 0.5371655685684823, - "nauc_map_at_5_max": 0.45302351717025974, - "nauc_map_at_5_std": 0.09022498915722132, - "nauc_mrr_at_1000_diff1": 0.5399112943616813, - "nauc_mrr_at_1000_max": 0.4600690969544165, - "nauc_mrr_at_1000_std": 0.10168618302358416, - "nauc_mrr_at_100_diff1": 0.5397977611488858, - "nauc_mrr_at_100_max": 0.46034560041005235, - "nauc_mrr_at_100_std": 0.10218056859319692, - "nauc_mrr_at_10_diff1": 0.5380519968078198, - "nauc_mrr_at_10_max": 0.45838195646878466, - "nauc_mrr_at_10_std": 0.09868025608798509, - "nauc_mrr_at_1_diff1": 0.5917764418026163, - "nauc_mrr_at_1_max": 0.432157993286181, - "nauc_mrr_at_1_std": 0.06054202201021012, - "nauc_mrr_at_20_diff1": 0.5389459887198278, - "nauc_mrr_at_20_max": 0.4605616437478957, - "nauc_mrr_at_20_std": 0.10203876555994912, - "nauc_mrr_at_3_diff1": 0.5406680525792159, - "nauc_mrr_at_3_max": 0.4494874340157258, - "nauc_mrr_at_3_std": 0.08437790118672613, - "nauc_mrr_at_5_diff1": 0.5371655685684823, - "nauc_mrr_at_5_max": 0.45302351717025974, - "nauc_mrr_at_5_std": 0.09022498915722132, - "nauc_ndcg_at_1000_diff1": 0.530352272874443, - "nauc_ndcg_at_1000_max": 0.47375450416916254, - "nauc_ndcg_at_1000_std": 0.12512049368825368, - "nauc_ndcg_at_100_diff1": 0.5272470054380564, - "nauc_ndcg_at_100_max": 0.48297319995546817, - "nauc_ndcg_at_100_std": 0.1418827176716715, - "nauc_ndcg_at_10_diff1": 0.5184077234967509, - "nauc_ndcg_at_10_max": 0.4751084362503045, - "nauc_ndcg_at_10_std": 0.12585944973703592, - "nauc_ndcg_at_1_diff1": 0.5917764418026163, - "nauc_ndcg_at_1_max": 0.432157993286181, - "nauc_ndcg_at_1_std": 0.06054202201021012, - "nauc_ndcg_at_20_diff1": 0.5210283417403792, - "nauc_ndcg_at_20_max": 0.48403106565510695, - "nauc_ndcg_at_20_std": 0.13995236921918247, - "nauc_ndcg_at_3_diff1": 0.5242927568943412, - "nauc_ndcg_at_3_max": 0.4544883574193214, - "nauc_ndcg_at_3_std": 0.09272567738930695, - "nauc_ndcg_at_5_diff1": 0.5175607308176516, - "nauc_ndcg_at_5_max": 0.46120957487887876, - "nauc_ndcg_at_5_std": 0.10370762506625461, - "nauc_precision_at_1000_diff1": 0.43984952545942857, - "nauc_precision_at_1000_max": 0.9333946385512796, - "nauc_precision_at_1000_std": 0.9622112512663694, - "nauc_precision_at_100_diff1": 0.4365013750273164, - "nauc_precision_at_100_max": 0.7814459620380634, - "nauc_precision_at_100_std": 0.6866576843293785, - "nauc_precision_at_10_diff1": 0.4440013860687516, - "nauc_precision_at_10_max": 0.5457172777528427, - "nauc_precision_at_10_std": 0.24115786357421895, - "nauc_precision_at_1_diff1": 0.5917764418026163, - "nauc_precision_at_1_max": 0.432157993286181, - "nauc_precision_at_1_std": 0.06054202201021012, - "nauc_precision_at_20_diff1": 0.4383282501050174, - "nauc_precision_at_20_max": 0.6175497235851231, - "nauc_precision_at_20_std": 0.3565597779245206, - "nauc_precision_at_3_diff1": 0.4759945496272968, - "nauc_precision_at_3_max": 0.4690612568061714, - "nauc_precision_at_3_std": 0.11757529811594437, - "nauc_precision_at_5_diff1": 0.45499226706324974, - "nauc_precision_at_5_max": 0.4878863346212209, - "nauc_precision_at_5_std": 0.14809518063666594, - "nauc_recall_at_1000_diff1": 0.43984952545943873, - "nauc_recall_at_1000_max": 0.9333946385512961, - "nauc_recall_at_1000_std": 0.9622112512663857, - "nauc_recall_at_100_diff1": 0.4365013750273154, - "nauc_recall_at_100_max": 0.781445962038063, - "nauc_recall_at_100_std": 0.6866576843293807, - "nauc_recall_at_10_diff1": 0.4440013860687523, - "nauc_recall_at_10_max": 0.5457172777528433, - "nauc_recall_at_10_std": 0.24115786357421992, - "nauc_recall_at_1_diff1": 0.5917764418026163, - "nauc_recall_at_1_max": 0.432157993286181, - "nauc_recall_at_1_std": 0.06054202201021012, - "nauc_recall_at_20_diff1": 0.43832825010501836, - "nauc_recall_at_20_max": 0.6175497235851238, - "nauc_recall_at_20_std": 0.3565597779245211, - "nauc_recall_at_3_diff1": 0.4759945496272966, - "nauc_recall_at_3_max": 0.4690612568061717, - "nauc_recall_at_3_std": 0.11757529811594443, - "nauc_recall_at_5_diff1": 0.45499226706325, - "nauc_recall_at_5_max": 0.48788633462122094, - "nauc_recall_at_5_std": 0.14809518063666652, - "ndcg_at_1": 0.35202, - "ndcg_at_10": 0.51423, - "ndcg_at_100": 0.56081, - "ndcg_at_1000": 0.57017, - "ndcg_at_20": 0.53488, - "ndcg_at_3": 0.44991, - "ndcg_at_5": 0.48111, - "precision_at_1": 0.35202, - "precision_at_10": 0.06981, - "precision_at_100": 0.00917, - "precision_at_1000": 0.00099, - "precision_at_20": 0.03896, - "precision_at_3": 0.17344, - "precision_at_5": 0.11922, - "recall_at_1": 0.35202, - "recall_at_10": 0.69807, - "recall_at_100": 0.91716, - "recall_at_1000": 0.99043, - "recall_at_20": 0.7791, - "recall_at_3": 0.52033, - "recall_at_5": 0.59609 - }, - { - "cv_recall_at_1": 0.39868, - "cv_recall_at_10": 0.72205, - "cv_recall_at_100": 0.91534, - "cv_recall_at_1000": 0.99023, - "cv_recall_at_20": 0.79447, - "cv_recall_at_3": 0.56911, - "cv_recall_at_5": 0.6378, - "hf_subset": "da", - "languages": [ - "dan-Latn" - ], - "main_score": 0.55393, - "map_at_1": 0.39868, - "map_at_10": 0.50082, - "map_at_100": 0.50903, - "map_at_1000": 0.50939, - "map_at_20": 0.50592, - "map_at_3": 0.47359, - "map_at_5": 0.48935, - "mrr_at_1": 0.3986784140969163, - "mrr_at_10": 0.5008177968323918, - "mrr_at_100": 0.5090339944872555, - "mrr_at_1000": 0.5093904636645038, - "mrr_at_20": 0.505920593279936, - "mrr_at_3": 0.473591226138035, - "mrr_at_5": 0.48934700807635967, - "nauc_cv_recall_at_1000_diff1": 0.5263074522842123, - "nauc_cv_recall_at_1000_max": 0.9194863959419592, - "nauc_cv_recall_at_1000_std": 0.9357992764288658, - "nauc_cv_recall_at_100_diff1": 0.47055254295863874, - "nauc_cv_recall_at_100_max": 0.7334811069618354, - "nauc_cv_recall_at_100_std": 0.6566165938186991, - "nauc_cv_recall_at_10_diff1": 0.47823304517462406, - "nauc_cv_recall_at_10_max": 0.5706203482207864, - "nauc_cv_recall_at_10_std": 0.30137527176231743, - "nauc_cv_recall_at_1_diff1": 0.627152858930617, - "nauc_cv_recall_at_1_max": 0.4658527019227254, - "nauc_cv_recall_at_1_std": 0.10458940196560304, - "nauc_cv_recall_at_20_diff1": 0.48853535388484476, - "nauc_cv_recall_at_20_max": 0.6242157780019971, - "nauc_cv_recall_at_20_std": 0.40844620006462734, - "nauc_cv_recall_at_3_diff1": 0.5282633012642057, - "nauc_cv_recall_at_3_max": 0.5080606004209856, - "nauc_cv_recall_at_3_std": 0.1679506090067389, - "nauc_cv_recall_at_5_diff1": 0.511101454871359, - "nauc_cv_recall_at_5_max": 0.5367492026393453, - "nauc_cv_recall_at_5_std": 0.22728158638352508, - "nauc_map_at_1000_diff1": 0.5818138831820504, - "nauc_map_at_1000_max": 0.4918722831473778, - "nauc_map_at_1000_std": 0.14951051959313508, - "nauc_map_at_100_diff1": 0.5816662994279276, - "nauc_map_at_100_max": 0.49207753096794643, - "nauc_map_at_100_std": 0.14995210436964093, - "nauc_map_at_10_diff1": 0.5796002758151477, - "nauc_map_at_10_max": 0.4910260270825821, - "nauc_map_at_10_std": 0.14736297265232623, - "nauc_map_at_1_diff1": 0.627152858930617, - "nauc_map_at_1_max": 0.4658527019227254, - "nauc_map_at_1_std": 0.10458940196560304, - "nauc_map_at_20_diff1": 0.5811449028241016, - "nauc_map_at_20_max": 0.49239153644280603, - "nauc_map_at_20_std": 0.15016380990114506, - "nauc_map_at_3_diff1": 0.5842145172396421, - "nauc_map_at_3_max": 0.4836508860144376, - "nauc_map_at_3_std": 0.13078463307068863, - "nauc_map_at_5_diff1": 0.5817855309317947, - "nauc_map_at_5_max": 0.4887406792626373, - "nauc_map_at_5_std": 0.1416335599236373, - "nauc_mrr_at_1000_diff1": 0.5818138831820504, - "nauc_mrr_at_1000_max": 0.4918722831473778, - "nauc_mrr_at_1000_std": 0.14951051959313508, - "nauc_mrr_at_100_diff1": 0.5816662994279276, - "nauc_mrr_at_100_max": 0.49207753096794643, - "nauc_mrr_at_100_std": 0.14995210436964093, - "nauc_mrr_at_10_diff1": 0.5796002758151477, - "nauc_mrr_at_10_max": 0.4910260270825821, - "nauc_mrr_at_10_std": 0.14736297265232623, - "nauc_mrr_at_1_diff1": 0.627152858930617, - "nauc_mrr_at_1_max": 0.4658527019227254, - "nauc_mrr_at_1_std": 0.10458940196560304, - "nauc_mrr_at_20_diff1": 0.5811449028241016, - "nauc_mrr_at_20_max": 0.49239153644280603, - "nauc_mrr_at_20_std": 0.15016380990114506, - "nauc_mrr_at_3_diff1": 0.5842145172396421, - "nauc_mrr_at_3_max": 0.4836508860144376, - "nauc_mrr_at_3_std": 0.13078463307068863, - "nauc_mrr_at_5_diff1": 0.5817855309317947, - "nauc_mrr_at_5_max": 0.4887406792626373, - "nauc_mrr_at_5_std": 0.1416335599236373, - "nauc_ndcg_at_1000_diff1": 0.5729456825600561, - "nauc_ndcg_at_1000_max": 0.503419434860686, - "nauc_ndcg_at_1000_std": 0.17228177959800048, - "nauc_ndcg_at_100_diff1": 0.5690964391255721, - "nauc_ndcg_at_100_max": 0.5101963241609828, - "nauc_ndcg_at_100_std": 0.1870203735628991, - "nauc_ndcg_at_10_diff1": 0.5597447480902777, - "nauc_ndcg_at_10_max": 0.5059287479367308, - "nauc_ndcg_at_10_std": 0.1759310934875448, - "nauc_ndcg_at_1_diff1": 0.627152858930617, - "nauc_ndcg_at_1_max": 0.4658527019227254, - "nauc_ndcg_at_1_std": 0.10458940196560304, - "nauc_ndcg_at_20_diff1": 0.5650278051813951, - "nauc_ndcg_at_20_max": 0.5118452737194403, - "nauc_ndcg_at_20_std": 0.18793319055399638, - "nauc_ndcg_at_3_diff1": 0.5704529161273854, - "nauc_ndcg_at_3_max": 0.4896311500930583, - "nauc_ndcg_at_3_std": 0.13986740335109915, - "nauc_ndcg_at_5_diff1": 0.5656134964841482, - "nauc_ndcg_at_5_max": 0.49939996963906663, - "nauc_ndcg_at_5_std": 0.16050641016164127, - "nauc_precision_at_1000_diff1": 0.5263074522841861, - "nauc_precision_at_1000_max": 0.9194863959419753, - "nauc_precision_at_1000_std": 0.9357992764288523, - "nauc_precision_at_100_diff1": 0.47055254295863985, - "nauc_precision_at_100_max": 0.7334811069618357, - "nauc_precision_at_100_std": 0.656616593818699, - "nauc_precision_at_10_diff1": 0.4782330451746229, - "nauc_precision_at_10_max": 0.5706203482207856, - "nauc_precision_at_10_std": 0.30137527176231704, - "nauc_precision_at_1_diff1": 0.627152858930617, - "nauc_precision_at_1_max": 0.4658527019227254, - "nauc_precision_at_1_std": 0.10458940196560304, - "nauc_precision_at_20_diff1": 0.48853535388484326, - "nauc_precision_at_20_max": 0.624215778001996, - "nauc_precision_at_20_std": 0.4084462000646254, - "nauc_precision_at_3_diff1": 0.5282633012642054, - "nauc_precision_at_3_max": 0.5080606004209852, - "nauc_precision_at_3_std": 0.16795060900673922, - "nauc_precision_at_5_diff1": 0.5111014548713589, - "nauc_precision_at_5_max": 0.536749202639345, - "nauc_precision_at_5_std": 0.2272815863835243, - "nauc_recall_at_1000_diff1": 0.5263074522842123, - "nauc_recall_at_1000_max": 0.9194863959419592, - "nauc_recall_at_1000_std": 0.9357992764288658, - "nauc_recall_at_100_diff1": 0.47055254295863874, - "nauc_recall_at_100_max": 0.7334811069618354, - "nauc_recall_at_100_std": 0.6566165938186991, - "nauc_recall_at_10_diff1": 0.47823304517462406, - "nauc_recall_at_10_max": 0.5706203482207864, - "nauc_recall_at_10_std": 0.30137527176231743, - "nauc_recall_at_1_diff1": 0.627152858930617, - "nauc_recall_at_1_max": 0.4658527019227254, - "nauc_recall_at_1_std": 0.10458940196560304, - "nauc_recall_at_20_diff1": 0.48853535388484476, - "nauc_recall_at_20_max": 0.6242157780019971, - "nauc_recall_at_20_std": 0.40844620006462734, - "nauc_recall_at_3_diff1": 0.5282633012642057, - "nauc_recall_at_3_max": 0.5080606004209856, - "nauc_recall_at_3_std": 0.1679506090067389, - "nauc_recall_at_5_diff1": 0.511101454871359, - "nauc_recall_at_5_max": 0.5367492026393453, - "nauc_recall_at_5_std": 0.22728158638352508, - "ndcg_at_1": 0.39868, - "ndcg_at_10": 0.55393, - "ndcg_at_100": 0.5948, - "ndcg_at_1000": 0.6043, - "ndcg_at_20": 0.57234, - "ndcg_at_3": 0.49811, - "ndcg_at_5": 0.52646, - "precision_at_1": 0.39868, - "precision_at_10": 0.07221, - "precision_at_100": 0.00915, - "precision_at_1000": 0.00099, - "precision_at_20": 0.03972, - "precision_at_3": 0.1897, - "precision_at_5": 0.12756, - "recall_at_1": 0.39868, - "recall_at_10": 0.72205, - "recall_at_100": 0.91534, - "recall_at_1000": 0.99023, - "recall_at_20": 0.79447, - "recall_at_3": 0.56911, - "recall_at_5": 0.6378 - }, - { - "cv_recall_at_1": 0.74048, - "cv_recall_at_10": 0.96344, - "cv_recall_at_100": 0.99803, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.98288, - "cv_recall_at_3": 0.88985, - "cv_recall_at_5": 0.92861, - "hf_subset": "de", - "languages": [ - "deu-Latn" - ], - "main_score": 0.85632, - "map_at_1": 0.74048, - "map_at_10": 0.82147, - "map_at_100": 0.82329, - "map_at_1000": 0.8233, - "map_at_20": 0.82284, - "map_at_3": 0.80778, - "map_at_5": 0.81671, - "mrr_at_1": 0.740483628369779, - "mrr_at_10": 0.8214715000119333, - "mrr_at_100": 0.8232923058252596, - "mrr_at_1000": 0.8233045665997936, - "mrr_at_20": 0.8228402912251109, - "mrr_at_3": 0.8077827914690076, - "mrr_at_5": 0.8167090902078693, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.6080298926938646, - "nauc_cv_recall_at_100_max": 0.8094408328267139, - "nauc_cv_recall_at_100_std": 0.15159729318007661, - "nauc_cv_recall_at_10_diff1": 0.7262385280719563, - "nauc_cv_recall_at_10_max": 0.6053348863156468, - "nauc_cv_recall_at_10_std": -0.3586274961626783, - "nauc_cv_recall_at_1_diff1": 0.813003513000172, - "nauc_cv_recall_at_1_max": 0.45234481733298193, - "nauc_cv_recall_at_1_std": -0.3639220449247165, - "nauc_cv_recall_at_20_diff1": 0.7458693329791404, - "nauc_cv_recall_at_20_max": 0.7318334192674248, - "nauc_cv_recall_at_20_std": -0.14523783296087528, - "nauc_cv_recall_at_3_diff1": 0.7358042843081358, - "nauc_cv_recall_at_3_max": 0.525921917798875, - "nauc_cv_recall_at_3_std": -0.4153844768358637, - "nauc_cv_recall_at_5_diff1": 0.7216698778631316, - "nauc_cv_recall_at_5_max": 0.542507469966757, - "nauc_cv_recall_at_5_std": -0.4223158051137424, - "nauc_map_at_1000_diff1": 0.7934790235857491, - "nauc_map_at_1000_max": 0.47296201032733654, - "nauc_map_at_1000_std": -0.37694405501566375, - "nauc_map_at_100_diff1": 0.7934662202964456, - "nauc_map_at_100_max": 0.472987098543922, - "nauc_map_at_100_std": -0.3769046329700942, - "nauc_map_at_10_diff1": 0.7926637368909875, - "nauc_map_at_10_max": 0.47381616393015213, - "nauc_map_at_10_std": -0.3778287683589719, - "nauc_map_at_1_diff1": 0.813003513000172, - "nauc_map_at_1_max": 0.45234481733298193, - "nauc_map_at_1_std": -0.3639220449247165, - "nauc_map_at_20_diff1": 0.7933995647470824, - "nauc_map_at_20_max": 0.4736339018818771, - "nauc_map_at_20_std": -0.37649670320119927, - "nauc_map_at_3_diff1": 0.789140185230195, - "nauc_map_at_3_max": 0.4745045467764235, - "nauc_map_at_3_std": -0.38230865280163173, - "nauc_map_at_5_diff1": 0.7905349301308817, - "nauc_map_at_5_max": 0.4736720556975195, - "nauc_map_at_5_std": -0.3812223296792622, - "nauc_mrr_at_1000_diff1": 0.7934790235857491, - "nauc_mrr_at_1000_max": 0.47296201032733654, - "nauc_mrr_at_1000_std": -0.37694405501566375, - "nauc_mrr_at_100_diff1": 0.7934662202964456, - "nauc_mrr_at_100_max": 0.472987098543922, - "nauc_mrr_at_100_std": -0.3769046329700942, - "nauc_mrr_at_10_diff1": 0.7926637368909875, - "nauc_mrr_at_10_max": 0.47381616393015213, - "nauc_mrr_at_10_std": -0.3778287683589719, - "nauc_mrr_at_1_diff1": 0.813003513000172, - "nauc_mrr_at_1_max": 0.45234481733298193, - "nauc_mrr_at_1_std": -0.3639220449247165, - "nauc_mrr_at_20_diff1": 0.7933995647470824, - "nauc_mrr_at_20_max": 0.4736339018818771, - "nauc_mrr_at_20_std": -0.37649670320119927, - "nauc_mrr_at_3_diff1": 0.789140185230195, - "nauc_mrr_at_3_max": 0.4745045467764235, - "nauc_mrr_at_3_std": -0.38230865280163173, - "nauc_mrr_at_5_diff1": 0.7905349301308817, - "nauc_mrr_at_5_max": 0.4736720556975195, - "nauc_mrr_at_5_std": -0.3812223296792622, - "nauc_ndcg_at_1000_diff1": 0.7910361385308725, - "nauc_ndcg_at_1000_max": 0.47684137941910887, - "nauc_ndcg_at_1000_std": -0.3765196296277837, - "nauc_ndcg_at_100_diff1": 0.7906493843256589, - "nauc_ndcg_at_100_max": 0.4775492752123655, - "nauc_ndcg_at_100_std": -0.3754033971076211, - "nauc_ndcg_at_10_diff1": 0.7869969737209641, - "nauc_ndcg_at_10_max": 0.48352106502398234, - "nauc_ndcg_at_10_std": -0.37766974970717637, - "nauc_ndcg_at_1_diff1": 0.813003513000172, - "nauc_ndcg_at_1_max": 0.45234481733298193, - "nauc_ndcg_at_1_std": -0.3639220449247165, - "nauc_ndcg_at_20_diff1": 0.790077876058056, - "nauc_ndcg_at_20_max": 0.48309721524984295, - "nauc_ndcg_at_20_std": -0.3712496025867674, - "nauc_ndcg_at_3_diff1": 0.7796023612066416, - "nauc_ndcg_at_3_max": 0.4836702178151078, - "nauc_ndcg_at_3_std": -0.3883468622656064, - "nauc_ndcg_at_5_diff1": 0.7814603879909108, - "nauc_ndcg_at_5_max": 0.4827126092034757, - "nauc_ndcg_at_5_std": -0.386787692131109, - "nauc_precision_at_1000_diff1": 1.0, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.6080298926938649, - "nauc_precision_at_100_max": 0.8094408328266878, - "nauc_precision_at_100_std": 0.15159729317989815, - "nauc_precision_at_10_diff1": 0.7262385280719623, - "nauc_precision_at_10_max": 0.6053348863156497, - "nauc_precision_at_10_std": -0.3586274961626875, - "nauc_precision_at_1_diff1": 0.813003513000172, - "nauc_precision_at_1_max": 0.45234481733298193, - "nauc_precision_at_1_std": -0.3639220449247165, - "nauc_precision_at_20_diff1": 0.7458693329791394, - "nauc_precision_at_20_max": 0.7318334192674198, - "nauc_precision_at_20_std": -0.14523783296088458, - "nauc_precision_at_3_diff1": 0.7358042843081359, - "nauc_precision_at_3_max": 0.5259219177988752, - "nauc_precision_at_3_std": -0.4153844768358635, - "nauc_precision_at_5_diff1": 0.7216698778631329, - "nauc_precision_at_5_max": 0.5425074699667538, - "nauc_precision_at_5_std": -0.4223158051137469, - "nauc_recall_at_1000_diff1": NaN, - "nauc_recall_at_1000_max": NaN, - "nauc_recall_at_1000_std": NaN, - "nauc_recall_at_100_diff1": 0.6080298926938646, - "nauc_recall_at_100_max": 0.8094408328267139, - "nauc_recall_at_100_std": 0.15159729318007661, - "nauc_recall_at_10_diff1": 0.7262385280719563, - "nauc_recall_at_10_max": 0.6053348863156468, - "nauc_recall_at_10_std": -0.3586274961626783, - "nauc_recall_at_1_diff1": 0.813003513000172, - "nauc_recall_at_1_max": 0.45234481733298193, - "nauc_recall_at_1_std": -0.3639220449247165, - "nauc_recall_at_20_diff1": 0.7458693329791404, - "nauc_recall_at_20_max": 0.7318334192674248, - "nauc_recall_at_20_std": -0.14523783296087528, - "nauc_recall_at_3_diff1": 0.7358042843081358, - "nauc_recall_at_3_max": 0.525921917798875, - "nauc_recall_at_3_std": -0.4153844768358637, - "nauc_recall_at_5_diff1": 0.7216698778631316, - "nauc_recall_at_5_max": 0.542507469966757, - "nauc_recall_at_5_std": -0.4223158051137424, - "ndcg_at_1": 0.74048, - "ndcg_at_10": 0.85632, - "ndcg_at_100": 0.8642, - "ndcg_at_1000": 0.86447, - "ndcg_at_20": 0.86126, - "ndcg_at_3": 0.82892, - "ndcg_at_5": 0.84495, - "precision_at_1": 0.74048, - "precision_at_10": 0.09634, - "precision_at_100": 0.00998, - "precision_at_1000": 0.001, - "precision_at_20": 0.04914, - "precision_at_3": 0.29662, - "precision_at_5": 0.18572, - "recall_at_1": 0.74048, - "recall_at_10": 0.96344, - "recall_at_100": 0.99803, - "recall_at_1000": 1.0, - "recall_at_20": 0.98288, - "recall_at_3": 0.88985, - "recall_at_5": 0.92861 - }, - { - "cv_recall_at_1": 0.13631, - "cv_recall_at_10": 0.37882, - "cv_recall_at_100": 0.70086, - "cv_recall_at_1000": 0.94142, - "cv_recall_at_20": 0.47043, - "cv_recall_at_3": 0.23778, - "cv_recall_at_5": 0.29511, - "hf_subset": "el", - "languages": [ - "ell-Grek" - ], - "main_score": 0.24596, - "map_at_1": 0.13631, - "map_at_10": 0.2049, - "map_at_100": 0.21693, - "map_at_1000": 0.21796, - "map_at_20": 0.21121, - "map_at_3": 0.18069, - "map_at_5": 0.19379, - "mrr_at_1": 0.13631315935591337, - "mrr_at_10": 0.2049025893903735, - "mrr_at_100": 0.2169267661146188, - "mrr_at_1000": 0.2179567802285265, - "mrr_at_20": 0.21121163783023364, - "mrr_at_3": 0.18068665556172492, - "mrr_at_5": 0.19379048676661156, - "nauc_cv_recall_at_1000_diff1": 0.12315048052987966, - "nauc_cv_recall_at_1000_max": 0.684113262807908, - "nauc_cv_recall_at_1000_std": 0.8107300733807016, - "nauc_cv_recall_at_100_diff1": 0.12615226785532357, - "nauc_cv_recall_at_100_max": 0.49369246006412104, - "nauc_cv_recall_at_100_std": 0.5612294690418357, - "nauc_cv_recall_at_10_diff1": 0.2041646423498855, - "nauc_cv_recall_at_10_max": 0.3876832008855094, - "nauc_cv_recall_at_10_std": 0.3430556378240179, - "nauc_cv_recall_at_1_diff1": 0.38364736725904475, - "nauc_cv_recall_at_1_max": 0.38427880845344464, - "nauc_cv_recall_at_1_std": 0.24976199043925765, - "nauc_cv_recall_at_20_diff1": 0.17985737581721026, - "nauc_cv_recall_at_20_max": 0.41034482476852835, - "nauc_cv_recall_at_20_std": 0.39534016418479334, - "nauc_cv_recall_at_3_diff1": 0.2550695105023992, - "nauc_cv_recall_at_3_max": 0.3720164931669786, - "nauc_cv_recall_at_3_std": 0.2852382529047927, - "nauc_cv_recall_at_5_diff1": 0.23154248441240807, - "nauc_cv_recall_at_5_max": 0.37603218061608035, - "nauc_cv_recall_at_5_std": 0.29973737854224664, - "nauc_map_at_1000_diff1": 0.29537867204369916, - "nauc_map_at_1000_max": 0.3836994455065988, - "nauc_map_at_1000_std": 0.2875121245790976, - "nauc_map_at_100_diff1": 0.2948369157392871, - "nauc_map_at_100_max": 0.38363691195770255, - "nauc_map_at_100_std": 0.28777583177941146, - "nauc_map_at_10_diff1": 0.297555214002202, - "nauc_map_at_10_max": 0.3794912846856662, - "nauc_map_at_10_std": 0.2795066005348579, - "nauc_map_at_1_diff1": 0.38364736725904475, - "nauc_map_at_1_max": 0.38427880845344464, - "nauc_map_at_1_std": 0.24976199043925765, - "nauc_map_at_20_diff1": 0.2953827791586574, - "nauc_map_at_20_max": 0.3817202452937388, - "nauc_map_at_20_std": 0.2843268638555573, - "nauc_map_at_3_diff1": 0.3109444517252625, - "nauc_map_at_3_max": 0.3759194965449022, - "nauc_map_at_3_std": 0.26660162570747864, - "nauc_map_at_5_diff1": 0.3026825934896626, - "nauc_map_at_5_max": 0.3771281027413023, - "nauc_map_at_5_std": 0.27143091554943366, - "nauc_mrr_at_1000_diff1": 0.29537867204369916, - "nauc_mrr_at_1000_max": 0.3836994455065988, - "nauc_mrr_at_1000_std": 0.2875121245790976, - "nauc_mrr_at_100_diff1": 0.2948369157392871, - "nauc_mrr_at_100_max": 0.38363691195770255, - "nauc_mrr_at_100_std": 0.28777583177941146, - "nauc_mrr_at_10_diff1": 0.297555214002202, - "nauc_mrr_at_10_max": 0.3794912846856662, - "nauc_mrr_at_10_std": 0.2795066005348579, - "nauc_mrr_at_1_diff1": 0.38364736725904475, - "nauc_mrr_at_1_max": 0.38427880845344464, - "nauc_mrr_at_1_std": 0.24976199043925765, - "nauc_mrr_at_20_diff1": 0.2953827791586574, - "nauc_mrr_at_20_max": 0.3817202452937388, - "nauc_mrr_at_20_std": 0.2843268638555573, - "nauc_mrr_at_3_diff1": 0.3109444517252625, - "nauc_mrr_at_3_max": 0.3759194965449022, - "nauc_mrr_at_3_std": 0.26660162570747864, - "nauc_mrr_at_5_diff1": 0.3026825934896626, - "nauc_mrr_at_5_max": 0.3771281027413023, - "nauc_mrr_at_5_std": 0.27143091554943366, - "nauc_ndcg_at_1000_diff1": 0.27099643551398866, - "nauc_ndcg_at_1000_max": 0.4000380546347878, - "nauc_ndcg_at_1000_std": 0.3252095778830688, - "nauc_ndcg_at_100_diff1": 0.2578745995168681, - "nauc_ndcg_at_100_max": 0.40221959905473226, - "nauc_ndcg_at_100_std": 0.33799821839115624, - "nauc_ndcg_at_10_diff1": 0.2696423737390834, - "nauc_ndcg_at_10_max": 0.38158703798004134, - "nauc_ndcg_at_10_std": 0.29754569894191735, - "nauc_ndcg_at_1_diff1": 0.38364736725904475, - "nauc_ndcg_at_1_max": 0.38427880845344464, - "nauc_ndcg_at_1_std": 0.24976199043925765, - "nauc_ndcg_at_20_diff1": 0.2623574521934335, - "nauc_ndcg_at_20_max": 0.38873844027747434, - "nauc_ndcg_at_20_std": 0.31360247763855964, - "nauc_ndcg_at_3_diff1": 0.2944299126805184, - "nauc_ndcg_at_3_max": 0.3747043268401905, - "nauc_ndcg_at_3_std": 0.2719708163834449, - "nauc_ndcg_at_5_diff1": 0.2811396024950695, - "nauc_ndcg_at_5_max": 0.37667440905687843, - "nauc_ndcg_at_5_std": 0.2797929568888234, - "nauc_precision_at_1000_diff1": 0.12315048052987512, - "nauc_precision_at_1000_max": 0.684113262807911, - "nauc_precision_at_1000_std": 0.8107300733807009, - "nauc_precision_at_100_diff1": 0.1261522678553237, - "nauc_precision_at_100_max": 0.49369246006412154, - "nauc_precision_at_100_std": 0.5612294690418358, - "nauc_precision_at_10_diff1": 0.20416464234988557, - "nauc_precision_at_10_max": 0.3876832008855091, - "nauc_precision_at_10_std": 0.3430556378240175, - "nauc_precision_at_1_diff1": 0.38364736725904475, - "nauc_precision_at_1_max": 0.38427880845344464, - "nauc_precision_at_1_std": 0.24976199043925765, - "nauc_precision_at_20_diff1": 0.17985737581721012, - "nauc_precision_at_20_max": 0.4103448247685279, - "nauc_precision_at_20_std": 0.3953401641847935, - "nauc_precision_at_3_diff1": 0.255069510502399, - "nauc_precision_at_3_max": 0.37201649316697843, - "nauc_precision_at_3_std": 0.28523825290479277, - "nauc_precision_at_5_diff1": 0.23154248441240788, - "nauc_precision_at_5_max": 0.3760321806160802, - "nauc_precision_at_5_std": 0.29973737854224647, - "nauc_recall_at_1000_diff1": 0.12315048052987966, - "nauc_recall_at_1000_max": 0.684113262807908, - "nauc_recall_at_1000_std": 0.8107300733807016, - "nauc_recall_at_100_diff1": 0.12615226785532357, - "nauc_recall_at_100_max": 0.49369246006412104, - "nauc_recall_at_100_std": 0.5612294690418357, - "nauc_recall_at_10_diff1": 0.2041646423498855, - "nauc_recall_at_10_max": 0.3876832008855094, - "nauc_recall_at_10_std": 0.3430556378240179, - "nauc_recall_at_1_diff1": 0.38364736725904475, - "nauc_recall_at_1_max": 0.38427880845344464, - "nauc_recall_at_1_std": 0.24976199043925765, - "nauc_recall_at_20_diff1": 0.17985737581721026, - "nauc_recall_at_20_max": 0.41034482476852835, - "nauc_recall_at_20_std": 0.39534016418479334, - "nauc_recall_at_3_diff1": 0.2550695105023992, - "nauc_recall_at_3_max": 0.3720164931669786, - "nauc_recall_at_3_std": 0.2852382529047927, - "nauc_recall_at_5_diff1": 0.23154248441240807, - "nauc_recall_at_5_max": 0.37603218061608035, - "nauc_recall_at_5_std": 0.29973737854224664, - "ndcg_at_1": 0.13631, - "ndcg_at_10": 0.24596, - "ndcg_at_100": 0.31142, - "ndcg_at_1000": 0.34132, - "ndcg_at_20": 0.26906, - "ndcg_at_3": 0.19534, - "ndcg_at_5": 0.21895, - "precision_at_1": 0.13631, - "precision_at_10": 0.03788, - "precision_at_100": 0.00701, - "precision_at_1000": 0.00094, - "precision_at_20": 0.02352, - "precision_at_3": 0.07926, - "precision_at_5": 0.05902, - "recall_at_1": 0.13631, - "recall_at_10": 0.37882, - "recall_at_100": 0.70086, - "recall_at_1000": 0.94142, - "recall_at_20": 0.47043, - "recall_at_3": 0.23778, - "recall_at_5": 0.29511 - }, - { - "cv_recall_at_1": 0.59639, - "cv_recall_at_10": 0.90028, - "cv_recall_at_100": 0.9925, - "cv_recall_at_1000": 0.99972, - "cv_recall_at_20": 0.94361, - "cv_recall_at_3": 0.77264, - "cv_recall_at_5": 0.83444, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "main_score": 0.74696, - "map_at_1": 0.59639, - "map_at_10": 0.698, - "map_at_100": 0.7025, - "map_at_1000": 0.70255, - "map_at_20": 0.7011, - "map_at_3": 0.67491, - "map_at_5": 0.68905, - "mrr_at_1": 0.5963888888888889, - "mrr_at_10": 0.6979968033509697, - "mrr_at_100": 0.7025036513629882, - "mrr_at_1000": 0.7025546795050475, - "mrr_at_20": 0.7011014931156818, - "mrr_at_3": 0.6749074074074076, - "mrr_at_5": 0.6890462962962937, - "nauc_cv_recall_at_1000_diff1": -0.8576097105504197, - "nauc_cv_recall_at_1000_max": 0.7957516339868551, - "nauc_cv_recall_at_1000_std": 0.638188608776782, - "nauc_cv_recall_at_100_diff1": 0.5582529308019509, - "nauc_cv_recall_at_100_max": 0.5609157243144177, - "nauc_cv_recall_at_100_std": -0.4681156413182839, - "nauc_cv_recall_at_10_diff1": 0.5970066763938622, - "nauc_cv_recall_at_10_max": 0.3365284832595995, - "nauc_cv_recall_at_10_std": -0.6122197254017663, - "nauc_cv_recall_at_1_diff1": 0.7377487579296046, - "nauc_cv_recall_at_1_max": 0.3024684160415429, - "nauc_cv_recall_at_1_std": -0.3981561808321467, - "nauc_cv_recall_at_20_diff1": 0.6089677250210449, - "nauc_cv_recall_at_20_max": 0.3989549842925695, - "nauc_cv_recall_at_20_std": -0.6511236678579452, - "nauc_cv_recall_at_3_diff1": 0.6471234666158875, - "nauc_cv_recall_at_3_max": 0.32663422677411136, - "nauc_cv_recall_at_3_std": -0.53340168067965, - "nauc_cv_recall_at_5_diff1": 0.6179094330520296, - "nauc_cv_recall_at_5_max": 0.3198350834990974, - "nauc_cv_recall_at_5_std": -0.5785430943899972, - "nauc_map_at_1000_diff1": 0.7053641370418658, - "nauc_map_at_1000_max": 0.30906898400039495, - "nauc_map_at_1000_std": -0.44711110357925826, - "nauc_map_at_100_diff1": 0.7053426977769025, - "nauc_map_at_100_max": 0.30912545216268755, - "nauc_map_at_100_std": -0.44712648955253825, - "nauc_map_at_10_diff1": 0.7033110878186942, - "nauc_map_at_10_max": 0.3087413776346512, - "nauc_map_at_10_std": -0.4495718484061258, - "nauc_map_at_1_diff1": 0.7377487579296046, - "nauc_map_at_1_max": 0.3024684160415429, - "nauc_map_at_1_std": -0.3981561808321467, - "nauc_map_at_20_diff1": 0.704843842130606, - "nauc_map_at_20_max": 0.30953620116106784, - "nauc_map_at_20_std": -0.44821432469327055, - "nauc_map_at_3_diff1": 0.703340423649336, - "nauc_map_at_3_max": 0.31031179232153905, - "nauc_map_at_3_std": -0.4480133791974383, - "nauc_map_at_5_diff1": 0.7015623635558061, - "nauc_map_at_5_max": 0.3083947625804306, - "nauc_map_at_5_std": -0.4508028481773592, - "nauc_mrr_at_1000_diff1": 0.7053641370418658, - "nauc_mrr_at_1000_max": 0.30906898400039495, - "nauc_mrr_at_1000_std": -0.44711110357925826, - "nauc_mrr_at_100_diff1": 0.7053426977769025, - "nauc_mrr_at_100_max": 0.30912545216268755, - "nauc_mrr_at_100_std": -0.44712648955253825, - "nauc_mrr_at_10_diff1": 0.7033110878186942, - "nauc_mrr_at_10_max": 0.3087413776346512, - "nauc_mrr_at_10_std": -0.4495718484061258, - "nauc_mrr_at_1_diff1": 0.7377487579296046, - "nauc_mrr_at_1_max": 0.3024684160415429, - "nauc_mrr_at_1_std": -0.3981561808321467, - "nauc_mrr_at_20_diff1": 0.704843842130606, - "nauc_mrr_at_20_max": 0.30953620116106784, - "nauc_mrr_at_20_std": -0.44821432469327055, - "nauc_mrr_at_3_diff1": 0.703340423649336, - "nauc_mrr_at_3_max": 0.31031179232153905, - "nauc_mrr_at_3_std": -0.4480133791974383, - "nauc_mrr_at_5_diff1": 0.7015623635558061, - "nauc_mrr_at_5_max": 0.3083947625804306, - "nauc_mrr_at_5_std": -0.4508028481773592, - "nauc_ndcg_at_1000_diff1": 0.7003430081375407, - "nauc_ndcg_at_1000_max": 0.31097470498087065, - "nauc_ndcg_at_1000_std": -0.45496562187963535, - "nauc_ndcg_at_100_diff1": 0.6998734859936994, - "nauc_ndcg_at_100_max": 0.31226100914742533, - "nauc_ndcg_at_100_std": -0.4552630761942781, - "nauc_ndcg_at_10_diff1": 0.6901158602766978, - "nauc_ndcg_at_10_max": 0.3119355925051725, - "nauc_ndcg_at_10_std": -0.47001381290428546, - "nauc_ndcg_at_1_diff1": 0.7377487579296046, - "nauc_ndcg_at_1_max": 0.3024684160415429, - "nauc_ndcg_at_1_std": -0.3981561808321467, - "nauc_ndcg_at_20_diff1": 0.6960519472233067, - "nauc_ndcg_at_20_max": 0.315223759324874, - "nauc_ndcg_at_20_std": -0.4650794295364529, - "nauc_ndcg_at_3_diff1": 0.6913575792745672, - "nauc_ndcg_at_3_max": 0.31372725712292077, - "nauc_ndcg_at_3_std": -0.4661427966293675, - "nauc_ndcg_at_5_diff1": 0.6869440581528756, - "nauc_ndcg_at_5_max": 0.31045414986716, - "nauc_ndcg_at_5_std": -0.47303632455277805, - "nauc_precision_at_1000_diff1": -0.857609710551402, - "nauc_precision_at_1000_max": 0.7957516339864209, - "nauc_precision_at_1000_std": 0.638188608776549, - "nauc_precision_at_100_diff1": 0.5582529308019456, - "nauc_precision_at_100_max": 0.5609157243144026, - "nauc_precision_at_100_std": -0.468115641318239, - "nauc_precision_at_10_diff1": 0.597006676393863, - "nauc_precision_at_10_max": 0.3365284832595971, - "nauc_precision_at_10_std": -0.6122197254017686, - "nauc_precision_at_1_diff1": 0.7377487579296046, - "nauc_precision_at_1_max": 0.3024684160415429, - "nauc_precision_at_1_std": -0.3981561808321467, - "nauc_precision_at_20_diff1": 0.6089677250210448, - "nauc_precision_at_20_max": 0.3989549842925696, - "nauc_precision_at_20_std": -0.6511236678579463, - "nauc_precision_at_3_diff1": 0.6471234666158877, - "nauc_precision_at_3_max": 0.326634226774111, - "nauc_precision_at_3_std": -0.5334016806796505, - "nauc_precision_at_5_diff1": 0.6179094330520296, - "nauc_precision_at_5_max": 0.3198350834990983, - "nauc_precision_at_5_std": -0.5785430943899962, - "nauc_recall_at_1000_diff1": -0.8576097105504197, - "nauc_recall_at_1000_max": 0.7957516339868551, - "nauc_recall_at_1000_std": 0.638188608776782, - "nauc_recall_at_100_diff1": 0.5582529308019509, - "nauc_recall_at_100_max": 0.5609157243144177, - "nauc_recall_at_100_std": -0.4681156413182839, - "nauc_recall_at_10_diff1": 0.5970066763938622, - "nauc_recall_at_10_max": 0.3365284832595995, - "nauc_recall_at_10_std": -0.6122197254017663, - "nauc_recall_at_1_diff1": 0.7377487579296046, - "nauc_recall_at_1_max": 0.3024684160415429, - "nauc_recall_at_1_std": -0.3981561808321467, - "nauc_recall_at_20_diff1": 0.6089677250210449, - "nauc_recall_at_20_max": 0.3989549842925695, - "nauc_recall_at_20_std": -0.6511236678579452, - "nauc_recall_at_3_diff1": 0.6471234666158875, - "nauc_recall_at_3_max": 0.32663422677411136, - "nauc_recall_at_3_std": -0.53340168067965, - "nauc_recall_at_5_diff1": 0.6179094330520296, - "nauc_recall_at_5_max": 0.3198350834990974, - "nauc_recall_at_5_std": -0.5785430943899972, - "ndcg_at_1": 0.59639, - "ndcg_at_10": 0.74696, - "ndcg_at_100": 0.7674, - "ndcg_at_1000": 0.76841, - "ndcg_at_20": 0.75804, - "ndcg_at_3": 0.70004, - "ndcg_at_5": 0.72551, - "precision_at_1": 0.59639, - "precision_at_10": 0.09003, - "precision_at_100": 0.00992, - "precision_at_1000": 0.001, - "precision_at_20": 0.04718, - "precision_at_3": 0.25755, - "precision_at_5": 0.16689, - "recall_at_1": 0.59639, - "recall_at_10": 0.90028, - "recall_at_100": 0.9925, - "recall_at_1000": 0.99972, - "recall_at_20": 0.94361, - "recall_at_3": 0.77264, - "recall_at_5": 0.83444 - }, - { - "cv_recall_at_1": 0.60169, - "cv_recall_at_10": 0.9127, - "cv_recall_at_100": 0.99512, - "cv_recall_at_1000": 0.99977, - "cv_recall_at_20": 0.9538, - "cv_recall_at_3": 0.79057, - "cv_recall_at_5": 0.84711, - "hf_subset": "es", - "languages": [ - "spa-Latn" - ], - "main_score": 0.75655, - "map_at_1": 0.60169, - "map_at_10": 0.70649, - "map_at_100": 0.71061, - "map_at_1000": 0.71064, - "map_at_20": 0.70942, - "map_at_3": 0.68441, - "map_at_5": 0.69742, - "mrr_at_1": 0.6016949152542372, - "mrr_at_10": 0.7064896661396504, - "mrr_at_100": 0.7106148496171836, - "mrr_at_1000": 0.7106424374843844, - "mrr_at_20": 0.709417724462585, - "mrr_at_3": 0.684409101462732, - "mrr_at_5": 0.6974228000928664, - "nauc_cv_recall_at_1000_diff1": 0.6136786139686762, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 0.8611123491271107, - "nauc_cv_recall_at_100_diff1": 0.6200398503013391, - "nauc_cv_recall_at_100_max": 0.6930038548827888, - "nauc_cv_recall_at_100_std": 0.24702230513887494, - "nauc_cv_recall_at_10_diff1": 0.590287713177001, - "nauc_cv_recall_at_10_max": 0.43903135906364865, - "nauc_cv_recall_at_10_std": -0.2547022176587369, - "nauc_cv_recall_at_1_diff1": 0.7212591196929298, - "nauc_cv_recall_at_1_max": 0.3424258071570245, - "nauc_cv_recall_at_1_std": -0.2610397341026975, - "nauc_cv_recall_at_20_diff1": 0.5829385232149739, - "nauc_cv_recall_at_20_max": 0.5086830360137641, - "nauc_cv_recall_at_20_std": -0.14825110719122006, - "nauc_cv_recall_at_3_diff1": 0.6246144891401507, - "nauc_cv_recall_at_3_max": 0.4017382109009011, - "nauc_cv_recall_at_3_std": -0.28904230971819406, - "nauc_cv_recall_at_5_diff1": 0.6093640088946269, - "nauc_cv_recall_at_5_max": 0.42715394711549926, - "nauc_cv_recall_at_5_std": -0.26842751256748754, - "nauc_map_at_1000_diff1": 0.691433722593808, - "nauc_map_at_1000_max": 0.3614960036233303, - "nauc_map_at_1000_std": -0.2678824364782566, - "nauc_map_at_100_diff1": 0.6914272321601181, - "nauc_map_at_100_max": 0.3615230313975254, - "nauc_map_at_100_std": -0.2678474806057638, - "nauc_map_at_10_diff1": 0.6896377884065275, - "nauc_map_at_10_max": 0.36170263160915467, - "nauc_map_at_10_std": -0.2692104736323017, - "nauc_map_at_1_diff1": 0.7212591196929298, - "nauc_map_at_1_max": 0.3424258071570245, - "nauc_map_at_1_std": -0.2610397341026975, - "nauc_map_at_20_diff1": 0.6907849900168619, - "nauc_map_at_20_max": 0.361939109058859, - "nauc_map_at_20_std": -0.2676976848761723, - "nauc_map_at_3_diff1": 0.6875917923426406, - "nauc_map_at_3_max": 0.36170270426098095, - "nauc_map_at_3_std": -0.27333141519414944, - "nauc_map_at_5_diff1": 0.6882704724520071, - "nauc_map_at_5_max": 0.36360481691410973, - "nauc_map_at_5_std": -0.269604778112949, - "nauc_mrr_at_1000_diff1": 0.691433722593808, - "nauc_mrr_at_1000_max": 0.3614960036233303, - "nauc_mrr_at_1000_std": -0.2678824364782566, - "nauc_mrr_at_100_diff1": 0.6914272321601181, - "nauc_mrr_at_100_max": 0.3615230313975254, - "nauc_mrr_at_100_std": -0.2678474806057638, - "nauc_mrr_at_10_diff1": 0.6896377884065275, - "nauc_mrr_at_10_max": 0.36170263160915467, - "nauc_mrr_at_10_std": -0.2692104736323017, - "nauc_mrr_at_1_diff1": 0.7212591196929298, - "nauc_mrr_at_1_max": 0.3424258071570245, - "nauc_mrr_at_1_std": -0.2610397341026975, - "nauc_mrr_at_20_diff1": 0.6907849900168619, - "nauc_mrr_at_20_max": 0.361939109058859, - "nauc_mrr_at_20_std": -0.2676976848761723, - "nauc_mrr_at_3_diff1": 0.6875917923426406, - "nauc_mrr_at_3_max": 0.36170270426098095, - "nauc_mrr_at_3_std": -0.27333141519414944, - "nauc_mrr_at_5_diff1": 0.6882704724520071, - "nauc_mrr_at_5_max": 0.36360481691410973, - "nauc_mrr_at_5_std": -0.269604778112949, - "nauc_ndcg_at_1000_diff1": 0.6872101463698389, - "nauc_ndcg_at_1000_max": 0.3658212859632228, - "nauc_ndcg_at_1000_std": -0.26604171691781336, - "nauc_ndcg_at_100_diff1": 0.6870017228600985, - "nauc_ndcg_at_100_max": 0.36679058513416024, - "nauc_ndcg_at_100_std": -0.26458444374726653, - "nauc_ndcg_at_10_diff1": 0.6780702361160329, - "nauc_ndcg_at_10_max": 0.37034357646106747, - "nauc_ndcg_at_10_std": -0.26829335037077523, - "nauc_ndcg_at_1_diff1": 0.7212591196929298, - "nauc_ndcg_at_1_max": 0.3424258071570245, - "nauc_ndcg_at_1_std": -0.2610397341026975, - "nauc_ndcg_at_20_diff1": 0.6823980035535215, - "nauc_ndcg_at_20_max": 0.3712004190242446, - "nauc_ndcg_at_20_std": -0.26223920912492443, - "nauc_ndcg_at_3_diff1": 0.6746910152153665, - "nauc_ndcg_at_3_max": 0.3698392560465834, - "nauc_ndcg_at_3_std": -0.2766681307398376, - "nauc_ndcg_at_5_diff1": 0.6749598180573865, - "nauc_ndcg_at_5_max": 0.3741053208981384, - "nauc_ndcg_at_5_std": -0.26977667295231167, - "nauc_precision_at_1000_diff1": 0.6136786139684929, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 0.8611123491272562, - "nauc_precision_at_100_diff1": 0.6200398503013244, - "nauc_precision_at_100_max": 0.6930038548827688, - "nauc_precision_at_100_std": 0.24702230513882242, - "nauc_precision_at_10_diff1": 0.5902877131770019, - "nauc_precision_at_10_max": 0.4390313590636452, - "nauc_precision_at_10_std": -0.25470221765874135, - "nauc_precision_at_1_diff1": 0.7212591196929298, - "nauc_precision_at_1_max": 0.3424258071570245, - "nauc_precision_at_1_std": -0.2610397341026975, - "nauc_precision_at_20_diff1": 0.5829385232149772, - "nauc_precision_at_20_max": 0.5086830360137653, - "nauc_precision_at_20_std": -0.1482511071912294, - "nauc_precision_at_3_diff1": 0.6246144891401509, - "nauc_precision_at_3_max": 0.40173821090090217, - "nauc_precision_at_3_std": -0.28904230971819217, - "nauc_precision_at_5_diff1": 0.6093640088946262, - "nauc_precision_at_5_max": 0.42715394711549876, - "nauc_precision_at_5_std": -0.26842751256748654, - "nauc_recall_at_1000_diff1": 0.6136786139686762, - "nauc_recall_at_1000_max": 1.0, - "nauc_recall_at_1000_std": 0.8611123491271107, - "nauc_recall_at_100_diff1": 0.6200398503013391, - "nauc_recall_at_100_max": 0.6930038548827888, - "nauc_recall_at_100_std": 0.24702230513887494, - "nauc_recall_at_10_diff1": 0.590287713177001, - "nauc_recall_at_10_max": 0.43903135906364865, - "nauc_recall_at_10_std": -0.2547022176587369, - "nauc_recall_at_1_diff1": 0.7212591196929298, - "nauc_recall_at_1_max": 0.3424258071570245, - "nauc_recall_at_1_std": -0.2610397341026975, - "nauc_recall_at_20_diff1": 0.5829385232149739, - "nauc_recall_at_20_max": 0.5086830360137641, - "nauc_recall_at_20_std": -0.14825110719122006, - "nauc_recall_at_3_diff1": 0.6246144891401507, - "nauc_recall_at_3_max": 0.4017382109009011, - "nauc_recall_at_3_std": -0.28904230971819406, - "nauc_recall_at_5_diff1": 0.6093640088946269, - "nauc_recall_at_5_max": 0.42715394711549926, - "nauc_recall_at_5_std": -0.26842751256748754, - "ndcg_at_1": 0.60169, - "ndcg_at_10": 0.75655, - "ndcg_at_100": 0.77497, - "ndcg_at_1000": 0.77559, - "ndcg_at_20": 0.76703, - "ndcg_at_3": 0.71165, - "ndcg_at_5": 0.73502, - "precision_at_1": 0.60169, - "precision_at_10": 0.09127, - "precision_at_100": 0.00995, - "precision_at_1000": 0.001, - "precision_at_20": 0.04769, - "precision_at_3": 0.26352, - "precision_at_5": 0.16942, - "recall_at_1": 0.60169, - "recall_at_10": 0.9127, - "recall_at_100": 0.99512, - "recall_at_1000": 0.99977, - "recall_at_20": 0.9538, - "recall_at_3": 0.79057, - "recall_at_5": 0.84711 - }, - { - "cv_recall_at_1": 0.23188, - "cv_recall_at_10": 0.53333, - "cv_recall_at_100": 0.82374, - "cv_recall_at_1000": 0.9793, - "cv_recall_at_20": 0.62692, - "cv_recall_at_3": 0.36729, - "cv_recall_at_5": 0.43547, - "hf_subset": "fa", - "languages": [ - "fas-Arab" - ], - "main_score": 0.36969, - "map_at_1": 0.23188, - "map_at_10": 0.31889, - "map_at_100": 0.33036, - "map_at_1000": 0.33108, - "map_at_20": 0.32538, - "map_at_3": 0.29043, - "map_at_5": 0.30588, - "mrr_at_1": 0.2318840579710145, - "mrr_at_10": 0.318891025009036, - "mrr_at_100": 0.3303591182389066, - "mrr_at_1000": 0.33107825961711584, - "mrr_at_20": 0.3253773380134024, - "mrr_at_3": 0.2904301817345277, - "mrr_at_5": 0.3058753163100979, - "nauc_cv_recall_at_1000_diff1": 0.32211872213818943, - "nauc_cv_recall_at_1000_max": 0.7659603007995041, - "nauc_cv_recall_at_1000_std": 0.830699251839353, - "nauc_cv_recall_at_100_diff1": 0.25863051315437807, - "nauc_cv_recall_at_100_max": 0.5732638373083003, - "nauc_cv_recall_at_100_std": 0.5865637398670023, - "nauc_cv_recall_at_10_diff1": 0.2869595891354477, - "nauc_cv_recall_at_10_max": 0.4643134653282664, - "nauc_cv_recall_at_10_std": 0.3382897937481642, - "nauc_cv_recall_at_1_diff1": 0.457735843069892, - "nauc_cv_recall_at_1_max": 0.36856484473201834, - "nauc_cv_recall_at_1_std": 0.14545186619790604, - "nauc_cv_recall_at_20_diff1": 0.2724295353745255, - "nauc_cv_recall_at_20_max": 0.4988732153083199, - "nauc_cv_recall_at_20_std": 0.39954793525470816, - "nauc_cv_recall_at_3_diff1": 0.3506172985202539, - "nauc_cv_recall_at_3_max": 0.414967683433225, - "nauc_cv_recall_at_3_std": 0.23232033769502783, - "nauc_cv_recall_at_5_diff1": 0.31547657274500124, - "nauc_cv_recall_at_5_max": 0.4216833529163241, - "nauc_cv_recall_at_5_std": 0.2677498638456819, - "nauc_map_at_1000_diff1": 0.3947235492658944, - "nauc_map_at_1000_max": 0.3984217284223239, - "nauc_map_at_1000_std": 0.20562945107832167, - "nauc_map_at_100_diff1": 0.39447621630777163, - "nauc_map_at_100_max": 0.3986690453603573, - "nauc_map_at_100_std": 0.20621371414865497, - "nauc_map_at_10_diff1": 0.39359027833782984, - "nauc_map_at_10_max": 0.39713016437567766, - "nauc_map_at_10_std": 0.20280200415718957, - "nauc_map_at_1_diff1": 0.457735843069892, - "nauc_map_at_1_max": 0.36856484473201834, - "nauc_map_at_1_std": 0.14545186619790604, - "nauc_map_at_20_diff1": 0.3935234674955701, - "nauc_map_at_20_max": 0.3985802315211193, - "nauc_map_at_20_std": 0.20542407715586428, - "nauc_map_at_3_diff1": 0.40614112697680893, - "nauc_map_at_3_max": 0.39010031302332215, - "nauc_map_at_3_std": 0.18497814770151996, - "nauc_map_at_5_diff1": 0.3969890334451716, - "nauc_map_at_5_max": 0.3913816377867295, - "nauc_map_at_5_std": 0.19352248177676157, - "nauc_mrr_at_1000_diff1": 0.3947235492658944, - "nauc_mrr_at_1000_max": 0.3984217284223239, - "nauc_mrr_at_1000_std": 0.20562945107832167, - "nauc_mrr_at_100_diff1": 0.39447621630777163, - "nauc_mrr_at_100_max": 0.3986690453603573, - "nauc_mrr_at_100_std": 0.20621371414865497, - "nauc_mrr_at_10_diff1": 0.39359027833782984, - "nauc_mrr_at_10_max": 0.39713016437567766, - "nauc_mrr_at_10_std": 0.20280200415718957, - "nauc_mrr_at_1_diff1": 0.457735843069892, - "nauc_mrr_at_1_max": 0.36856484473201834, - "nauc_mrr_at_1_std": 0.14545186619790604, - "nauc_mrr_at_20_diff1": 0.3935234674955701, - "nauc_mrr_at_20_max": 0.3985802315211193, - "nauc_mrr_at_20_std": 0.20542407715586428, - "nauc_mrr_at_3_diff1": 0.40614112697680893, - "nauc_mrr_at_3_max": 0.39010031302332215, - "nauc_mrr_at_3_std": 0.18497814770151996, - "nauc_mrr_at_5_diff1": 0.3969890334451716, - "nauc_mrr_at_5_max": 0.3913816377867295, - "nauc_mrr_at_5_std": 0.19352248177676157, - "nauc_ndcg_at_1000_diff1": 0.3797599734049316, - "nauc_ndcg_at_1000_max": 0.41316958890282335, - "nauc_ndcg_at_1000_std": 0.23518241317437694, - "nauc_ndcg_at_100_diff1": 0.3720305312998685, - "nauc_ndcg_at_100_max": 0.42098508813898644, - "nauc_ndcg_at_100_std": 0.2538830227370447, - "nauc_ndcg_at_10_diff1": 0.3669227076502561, - "nauc_ndcg_at_10_max": 0.4131996641700761, - "nauc_ndcg_at_10_std": 0.23542173233196248, - "nauc_ndcg_at_1_diff1": 0.457735843069892, - "nauc_ndcg_at_1_max": 0.36856484473201834, - "nauc_ndcg_at_1_std": 0.14545186619790604, - "nauc_ndcg_at_20_diff1": 0.3659279059662814, - "nauc_ndcg_at_20_max": 0.41951719608107246, - "nauc_ndcg_at_20_std": 0.24647748233394168, - "nauc_ndcg_at_3_diff1": 0.39103472570202114, - "nauc_ndcg_at_3_max": 0.39683204553553547, - "nauc_ndcg_at_3_std": 0.19776213687706054, - "nauc_ndcg_at_5_diff1": 0.3752911647015171, - "nauc_ndcg_at_5_max": 0.39944245600088607, - "nauc_ndcg_at_5_std": 0.21304738210858173, - "nauc_precision_at_1000_diff1": 0.3221187221381867, - "nauc_precision_at_1000_max": 0.7659603007994962, - "nauc_precision_at_1000_std": 0.8306992518393461, - "nauc_precision_at_100_diff1": 0.2586305131543791, - "nauc_precision_at_100_max": 0.5732638373083007, - "nauc_precision_at_100_std": 0.5865637398670032, - "nauc_precision_at_10_diff1": 0.2869595891354475, - "nauc_precision_at_10_max": 0.4643134653282665, - "nauc_precision_at_10_std": 0.3382897937481639, - "nauc_precision_at_1_diff1": 0.457735843069892, - "nauc_precision_at_1_max": 0.36856484473201834, - "nauc_precision_at_1_std": 0.14545186619790604, - "nauc_precision_at_20_diff1": 0.2724295353745257, - "nauc_precision_at_20_max": 0.4988732153083194, - "nauc_precision_at_20_std": 0.3995479352547079, - "nauc_precision_at_3_diff1": 0.35061729852025403, - "nauc_precision_at_3_max": 0.41496768343322493, - "nauc_precision_at_3_std": 0.23232033769502788, - "nauc_precision_at_5_diff1": 0.3154765727450011, - "nauc_precision_at_5_max": 0.4216833529163243, - "nauc_precision_at_5_std": 0.26774986384568156, - "nauc_recall_at_1000_diff1": 0.32211872213818943, - "nauc_recall_at_1000_max": 0.7659603007995041, - "nauc_recall_at_1000_std": 0.830699251839353, - "nauc_recall_at_100_diff1": 0.25863051315437807, - "nauc_recall_at_100_max": 0.5732638373083003, - "nauc_recall_at_100_std": 0.5865637398670023, - "nauc_recall_at_10_diff1": 0.2869595891354477, - "nauc_recall_at_10_max": 0.4643134653282664, - "nauc_recall_at_10_std": 0.3382897937481642, - "nauc_recall_at_1_diff1": 0.457735843069892, - "nauc_recall_at_1_max": 0.36856484473201834, - "nauc_recall_at_1_std": 0.14545186619790604, - "nauc_recall_at_20_diff1": 0.2724295353745255, - "nauc_recall_at_20_max": 0.4988732153083199, - "nauc_recall_at_20_std": 0.39954793525470816, - "nauc_recall_at_3_diff1": 0.3506172985202539, - "nauc_recall_at_3_max": 0.414967683433225, - "nauc_recall_at_3_std": 0.23232033769502783, - "nauc_recall_at_5_diff1": 0.31547657274500124, - "nauc_recall_at_5_max": 0.4216833529163241, - "nauc_recall_at_5_std": 0.2677498638456819, - "ndcg_at_1": 0.23188, - "ndcg_at_10": 0.36969, - "ndcg_at_100": 0.42974, - "ndcg_at_1000": 0.44938, - "ndcg_at_20": 0.39334, - "ndcg_at_3": 0.31012, - "ndcg_at_5": 0.33808, - "precision_at_1": 0.23188, - "precision_at_10": 0.05333, - "precision_at_100": 0.00824, - "precision_at_1000": 0.00098, - "precision_at_20": 0.03135, - "precision_at_3": 0.12243, - "precision_at_5": 0.08709, - "recall_at_1": 0.23188, - "recall_at_10": 0.53333, - "recall_at_100": 0.82374, - "recall_at_1000": 0.9793, - "recall_at_20": 0.62692, - "recall_at_3": 0.36729, - "recall_at_5": 0.43547 - }, - { - "cv_recall_at_1": 0.12726, - "cv_recall_at_10": 0.33717, - "cv_recall_at_100": 0.62537, - "cv_recall_at_1000": 0.90908, - "cv_recall_at_20": 0.41336, - "cv_recall_at_3": 0.21454, - "cv_recall_at_5": 0.26365, - "hf_subset": "fi", - "languages": [ - "fin-Latn" - ], - "main_score": 0.22208, - "map_at_1": 0.12726, - "map_at_10": 0.18652, - "map_at_100": 0.19685, - "map_at_1000": 0.19799, - "map_at_20": 0.19179, - "map_at_3": 0.1655, - "map_at_5": 0.17668, - "mrr_at_1": 0.12726252280061737, - "mrr_at_10": 0.18652129950712795, - "mrr_at_100": 0.1968498342359693, - "mrr_at_1000": 0.19799300280423807, - "mrr_at_20": 0.19179013391721447, - "mrr_at_3": 0.1654974042374072, - "mrr_at_5": 0.1766802301108467, - "nauc_cv_recall_at_1000_diff1": 0.1723409475154773, - "nauc_cv_recall_at_1000_max": 0.5806492335928906, - "nauc_cv_recall_at_1000_std": 0.6978656115925028, - "nauc_cv_recall_at_100_diff1": 0.15949208699029174, - "nauc_cv_recall_at_100_max": 0.5081647501763306, - "nauc_cv_recall_at_100_std": 0.5648517323131209, - "nauc_cv_recall_at_10_diff1": 0.20978800071637904, - "nauc_cv_recall_at_10_max": 0.4590187077320782, - "nauc_cv_recall_at_10_std": 0.40395526036896534, - "nauc_cv_recall_at_1_diff1": 0.4194797367320295, - "nauc_cv_recall_at_1_max": 0.48084040784838245, - "nauc_cv_recall_at_1_std": 0.3249433866361136, - "nauc_cv_recall_at_20_diff1": 0.19576055879244425, - "nauc_cv_recall_at_20_max": 0.48531777680211063, - "nauc_cv_recall_at_20_std": 0.4645460165139727, - "nauc_cv_recall_at_3_diff1": 0.2660061870437314, - "nauc_cv_recall_at_3_max": 0.43734745197707886, - "nauc_cv_recall_at_3_std": 0.33147816691693927, - "nauc_cv_recall_at_5_diff1": 0.24040063953680255, - "nauc_cv_recall_at_5_max": 0.43914265734822067, - "nauc_cv_recall_at_5_std": 0.35044859851275406, - "nauc_map_at_1000_diff1": 0.3179538832843002, - "nauc_map_at_1000_max": 0.46396708740385206, - "nauc_map_at_1000_std": 0.35230253784996296, - "nauc_map_at_100_diff1": 0.317458246872419, - "nauc_map_at_100_max": 0.46384790020380545, - "nauc_map_at_100_std": 0.35241899135604876, - "nauc_map_at_10_diff1": 0.3197482521656431, - "nauc_map_at_10_max": 0.4608621677461234, - "nauc_map_at_10_std": 0.34404064722784683, - "nauc_map_at_1_diff1": 0.4194797367320295, - "nauc_map_at_1_max": 0.48084040784838245, - "nauc_map_at_1_std": 0.3249433866361136, - "nauc_map_at_20_diff1": 0.3180910840899835, - "nauc_map_at_20_max": 0.46339777801996374, - "nauc_map_at_20_std": 0.34997322868109837, - "nauc_map_at_3_diff1": 0.33613678125759255, - "nauc_map_at_3_max": 0.45845354566336505, - "nauc_map_at_3_std": 0.329036739176186, - "nauc_map_at_5_diff1": 0.32697875909516155, - "nauc_map_at_5_max": 0.4581622492089542, - "nauc_map_at_5_std": 0.33448880015691235, - "nauc_mrr_at_1000_diff1": 0.3179538832843002, - "nauc_mrr_at_1000_max": 0.46396708740385206, - "nauc_mrr_at_1000_std": 0.35230253784996296, - "nauc_mrr_at_100_diff1": 0.317458246872419, - "nauc_mrr_at_100_max": 0.46384790020380545, - "nauc_mrr_at_100_std": 0.35241899135604876, - "nauc_mrr_at_10_diff1": 0.3197482521656431, - "nauc_mrr_at_10_max": 0.4608621677461234, - "nauc_mrr_at_10_std": 0.34404064722784683, - "nauc_mrr_at_1_diff1": 0.4194797367320295, - "nauc_mrr_at_1_max": 0.48084040784838245, - "nauc_mrr_at_1_std": 0.3249433866361136, - "nauc_mrr_at_20_diff1": 0.3180910840899835, - "nauc_mrr_at_20_max": 0.46339777801996374, - "nauc_mrr_at_20_std": 0.34997322868109837, - "nauc_mrr_at_3_diff1": 0.33613678125759255, - "nauc_mrr_at_3_max": 0.45845354566336505, - "nauc_mrr_at_3_std": 0.329036739176186, - "nauc_mrr_at_5_diff1": 0.32697875909516155, - "nauc_mrr_at_5_max": 0.4581622492089542, - "nauc_mrr_at_5_std": 0.33448880015691235, - "nauc_ndcg_at_1000_diff1": 0.2891655168605115, - "nauc_ndcg_at_1000_max": 0.4719452565725258, - "nauc_ndcg_at_1000_std": 0.38864092349062784, - "nauc_ndcg_at_100_diff1": 0.2766580840757124, - "nauc_ndcg_at_100_max": 0.47154980205352626, - "nauc_ndcg_at_100_std": 0.397782913341144, - "nauc_ndcg_at_10_diff1": 0.2861483545191091, - "nauc_ndcg_at_10_max": 0.45935636504902333, - "nauc_ndcg_at_10_std": 0.3607452554429696, - "nauc_ndcg_at_1_diff1": 0.4194797367320295, - "nauc_ndcg_at_1_max": 0.48084040784838245, - "nauc_ndcg_at_1_std": 0.3249433866361136, - "nauc_ndcg_at_20_diff1": 0.2809672887972353, - "nauc_ndcg_at_20_max": 0.46783014778517384, - "nauc_ndcg_at_20_std": 0.3805501614476263, - "nauc_ndcg_at_3_diff1": 0.3154473467289752, - "nauc_ndcg_at_3_max": 0.4522775531898738, - "nauc_ndcg_at_3_std": 0.32977935791371676, - "nauc_ndcg_at_5_diff1": 0.30081531429951247, - "nauc_ndcg_at_5_max": 0.45229813529152296, - "nauc_ndcg_at_5_std": 0.3390725044860204, - "nauc_precision_at_1000_diff1": 0.17234094751547313, - "nauc_precision_at_1000_max": 0.5806492335928894, - "nauc_precision_at_1000_std": 0.697865611592502, - "nauc_precision_at_100_diff1": 0.15949208699029158, - "nauc_precision_at_100_max": 0.5081647501763312, - "nauc_precision_at_100_std": 0.5648517323131214, - "nauc_precision_at_10_diff1": 0.20978800071637896, - "nauc_precision_at_10_max": 0.45901870773207815, - "nauc_precision_at_10_std": 0.40395526036896495, - "nauc_precision_at_1_diff1": 0.4194797367320295, - "nauc_precision_at_1_max": 0.48084040784838245, - "nauc_precision_at_1_std": 0.3249433866361136, - "nauc_precision_at_20_diff1": 0.19576055879244383, - "nauc_precision_at_20_max": 0.4853177768021108, - "nauc_precision_at_20_std": 0.4645460165139723, - "nauc_precision_at_3_diff1": 0.26600618704373113, - "nauc_precision_at_3_max": 0.4373474519770788, - "nauc_precision_at_3_std": 0.3314781669169393, - "nauc_precision_at_5_diff1": 0.24040063953680246, - "nauc_precision_at_5_max": 0.43914265734822083, - "nauc_precision_at_5_std": 0.350448598512754, - "nauc_recall_at_1000_diff1": 0.1723409475154773, - "nauc_recall_at_1000_max": 0.5806492335928906, - "nauc_recall_at_1000_std": 0.6978656115925028, - "nauc_recall_at_100_diff1": 0.15949208699029174, - "nauc_recall_at_100_max": 0.5081647501763306, - "nauc_recall_at_100_std": 0.5648517323131209, - "nauc_recall_at_10_diff1": 0.20978800071637904, - "nauc_recall_at_10_max": 0.4590187077320782, - "nauc_recall_at_10_std": 0.40395526036896534, - "nauc_recall_at_1_diff1": 0.4194797367320295, - "nauc_recall_at_1_max": 0.48084040784838245, - "nauc_recall_at_1_std": 0.3249433866361136, - "nauc_recall_at_20_diff1": 0.19576055879244425, - "nauc_recall_at_20_max": 0.48531777680211063, - "nauc_recall_at_20_std": 0.4645460165139727, - "nauc_recall_at_3_diff1": 0.2660061870437314, - "nauc_recall_at_3_max": 0.43734745197707886, - "nauc_recall_at_3_std": 0.33147816691693927, - "nauc_recall_at_5_diff1": 0.24040063953680255, - "nauc_recall_at_5_max": 0.43914265734822067, - "nauc_recall_at_5_std": 0.35044859851275406, - "ndcg_at_1": 0.12726, - "ndcg_at_10": 0.22208, - "ndcg_at_100": 0.27988, - "ndcg_at_1000": 0.3147, - "ndcg_at_20": 0.24132, - "ndcg_at_3": 0.17808, - "ndcg_at_5": 0.19827, - "precision_at_1": 0.12726, - "precision_at_10": 0.03372, - "precision_at_100": 0.00625, - "precision_at_1000": 0.00091, - "precision_at_20": 0.02067, - "precision_at_3": 0.07151, - "precision_at_5": 0.05273, - "recall_at_1": 0.12726, - "recall_at_10": 0.33717, - "recall_at_100": 0.62537, - "recall_at_1000": 0.90908, - "recall_at_20": 0.41336, - "recall_at_3": 0.21454, - "recall_at_5": 0.26365 - }, - { - "cv_recall_at_1": 0.10409, - "cv_recall_at_10": 0.26994, - "cv_recall_at_100": 0.48052, - "cv_recall_at_1000": 0.77761, - "cv_recall_at_20": 0.32747, - "cv_recall_at_3": 0.17626, - "cv_recall_at_5": 0.21691, - "hf_subset": "fil", - "languages": [ - "fil-Latn" - ], - "main_score": 0.17979, - "map_at_1": 0.10409, - "map_at_10": 0.15181, - "map_at_100": 0.15935, - "map_at_1000": 0.16042, - "map_at_20": 0.15576, - "map_at_3": 0.13551, - "map_at_5": 0.14481, - "mrr_at_1": 0.10409340272893515, - "mrr_at_10": 0.15181348480687365, - "mrr_at_100": 0.15934884398364807, - "mrr_at_1000": 0.16041844070172975, - "mrr_at_20": 0.15576377193412375, - "mrr_at_3": 0.1355089792282088, - "mrr_at_5": 0.14481408543161325, - "nauc_cv_recall_at_1000_diff1": 0.21424855009198726, - "nauc_cv_recall_at_1000_max": 0.6396145976080426, - "nauc_cv_recall_at_1000_std": 0.6832780714438885, - "nauc_cv_recall_at_100_diff1": 0.2408891924307149, - "nauc_cv_recall_at_100_max": 0.6415385547577763, - "nauc_cv_recall_at_100_std": 0.6241230779955738, - "nauc_cv_recall_at_10_diff1": 0.31301465223645725, - "nauc_cv_recall_at_10_max": 0.6259268235668424, - "nauc_cv_recall_at_10_std": 0.5067559222657122, - "nauc_cv_recall_at_1_diff1": 0.48340507367292174, - "nauc_cv_recall_at_1_max": 0.5918060803445095, - "nauc_cv_recall_at_1_std": 0.3803959830733999, - "nauc_cv_recall_at_20_diff1": 0.2842973728738141, - "nauc_cv_recall_at_20_max": 0.6290968057402018, - "nauc_cv_recall_at_20_std": 0.5382641411507488, - "nauc_cv_recall_at_3_diff1": 0.36960796666684165, - "nauc_cv_recall_at_3_max": 0.6189674816818908, - "nauc_cv_recall_at_3_std": 0.4484934591367724, - "nauc_cv_recall_at_5_diff1": 0.3386517993718759, - "nauc_cv_recall_at_5_max": 0.6155180401590951, - "nauc_cv_recall_at_5_std": 0.46547602875489463, - "nauc_map_at_1000_diff1": 0.39924658085767, - "nauc_map_at_1000_max": 0.6091345749330558, - "nauc_map_at_1000_std": 0.437122064911765, - "nauc_map_at_100_diff1": 0.3990805059906888, - "nauc_map_at_100_max": 0.609073942129352, - "nauc_map_at_100_std": 0.43701505321298145, - "nauc_map_at_10_diff1": 0.4047246467831402, - "nauc_map_at_10_max": 0.6080048359197399, - "nauc_map_at_10_std": 0.4297648942019917, - "nauc_map_at_1_diff1": 0.48340507367292174, - "nauc_map_at_1_max": 0.5918060803445095, - "nauc_map_at_1_std": 0.3803959830733999, - "nauc_map_at_20_diff1": 0.4010306354033571, - "nauc_map_at_20_max": 0.6084782287906838, - "nauc_map_at_20_std": 0.43363590478649666, - "nauc_map_at_3_diff1": 0.4226552701229157, - "nauc_map_at_3_max": 0.6069539099308938, - "nauc_map_at_3_std": 0.4149357350267575, - "nauc_map_at_5_diff1": 0.41071077410712703, - "nauc_map_at_5_max": 0.6059960903085639, - "nauc_map_at_5_std": 0.421177696907373, - "nauc_mrr_at_1000_diff1": 0.39924658085767, - "nauc_mrr_at_1000_max": 0.6091345749330558, - "nauc_mrr_at_1000_std": 0.437122064911765, - "nauc_mrr_at_100_diff1": 0.3990805059906888, - "nauc_mrr_at_100_max": 0.609073942129352, - "nauc_mrr_at_100_std": 0.43701505321298145, - "nauc_mrr_at_10_diff1": 0.4047246467831402, - "nauc_mrr_at_10_max": 0.6080048359197399, - "nauc_mrr_at_10_std": 0.4297648942019917, - "nauc_mrr_at_1_diff1": 0.48340507367292174, - "nauc_mrr_at_1_max": 0.5918060803445095, - "nauc_mrr_at_1_std": 0.3803959830733999, - "nauc_mrr_at_20_diff1": 0.4010306354033571, - "nauc_mrr_at_20_max": 0.6084782287906838, - "nauc_mrr_at_20_std": 0.43363590478649666, - "nauc_mrr_at_3_diff1": 0.4226552701229157, - "nauc_mrr_at_3_max": 0.6069539099308938, - "nauc_mrr_at_3_std": 0.4149357350267575, - "nauc_mrr_at_5_diff1": 0.41071077410712703, - "nauc_mrr_at_5_max": 0.6059960903085639, - "nauc_mrr_at_5_std": 0.421177696907373, - "nauc_ndcg_at_1000_diff1": 0.358454534190773, - "nauc_ndcg_at_1000_max": 0.6170885217468562, - "nauc_ndcg_at_1000_std": 0.48463916900891485, - "nauc_ndcg_at_100_diff1": 0.35325035842887503, - "nauc_ndcg_at_100_max": 0.6182234443263771, - "nauc_ndcg_at_100_std": 0.4869165041620259, - "nauc_ndcg_at_10_diff1": 0.3759730083967871, - "nauc_ndcg_at_10_max": 0.6135123636072259, - "nauc_ndcg_at_10_std": 0.4531881757711014, - "nauc_ndcg_at_1_diff1": 0.48340507367292174, - "nauc_ndcg_at_1_max": 0.5918060803445095, - "nauc_ndcg_at_1_std": 0.3803959830733999, - "nauc_ndcg_at_20_diff1": 0.36471685952311034, - "nauc_ndcg_at_20_max": 0.6148799438274387, - "nauc_ndcg_at_20_std": 0.4651709626901215, - "nauc_ndcg_at_3_diff1": 0.4069453017377879, - "nauc_ndcg_at_3_max": 0.6105376936705731, - "nauc_ndcg_at_3_std": 0.4248011868786331, - "nauc_ndcg_at_5_diff1": 0.38850935119559227, - "nauc_ndcg_at_5_max": 0.6090221015664716, - "nauc_ndcg_at_5_std": 0.43472779438740256, - "nauc_precision_at_1000_diff1": 0.2142485500919868, - "nauc_precision_at_1000_max": 0.6396145976080425, - "nauc_precision_at_1000_std": 0.6832780714438873, - "nauc_precision_at_100_diff1": 0.2408891924307154, - "nauc_precision_at_100_max": 0.641538554757777, - "nauc_precision_at_100_std": 0.6241230779955743, - "nauc_precision_at_10_diff1": 0.313014652236457, - "nauc_precision_at_10_max": 0.6259268235668421, - "nauc_precision_at_10_std": 0.5067559222657121, - "nauc_precision_at_1_diff1": 0.48340507367292174, - "nauc_precision_at_1_max": 0.5918060803445095, - "nauc_precision_at_1_std": 0.3803959830733999, - "nauc_precision_at_20_diff1": 0.2842973728738142, - "nauc_precision_at_20_max": 0.6290968057402014, - "nauc_precision_at_20_std": 0.5382641411507488, - "nauc_precision_at_3_diff1": 0.3696079666668415, - "nauc_precision_at_3_max": 0.6189674816818908, - "nauc_precision_at_3_std": 0.4484934591367722, - "nauc_precision_at_5_diff1": 0.3386517993718761, - "nauc_precision_at_5_max": 0.6155180401590954, - "nauc_precision_at_5_std": 0.465476028754895, - "nauc_recall_at_1000_diff1": 0.21424855009198726, - "nauc_recall_at_1000_max": 0.6396145976080426, - "nauc_recall_at_1000_std": 0.6832780714438885, - "nauc_recall_at_100_diff1": 0.2408891924307149, - "nauc_recall_at_100_max": 0.6415385547577763, - "nauc_recall_at_100_std": 0.6241230779955738, - "nauc_recall_at_10_diff1": 0.31301465223645725, - "nauc_recall_at_10_max": 0.6259268235668424, - "nauc_recall_at_10_std": 0.5067559222657122, - "nauc_recall_at_1_diff1": 0.48340507367292174, - "nauc_recall_at_1_max": 0.5918060803445095, - "nauc_recall_at_1_std": 0.3803959830733999, - "nauc_recall_at_20_diff1": 0.2842973728738141, - "nauc_recall_at_20_max": 0.6290968057402018, - "nauc_recall_at_20_std": 0.5382641411507488, - "nauc_recall_at_3_diff1": 0.36960796666684165, - "nauc_recall_at_3_max": 0.6189674816818908, - "nauc_recall_at_3_std": 0.4484934591367724, - "nauc_recall_at_5_diff1": 0.3386517993718759, - "nauc_recall_at_5_max": 0.6155180401590951, - "nauc_recall_at_5_std": 0.46547602875489463, - "ndcg_at_1": 0.10409, - "ndcg_at_10": 0.17979, - "ndcg_at_100": 0.22198, - "ndcg_at_1000": 0.25766, - "ndcg_at_20": 0.19428, - "ndcg_at_3": 0.14596, - "ndcg_at_5": 0.16271, - "precision_at_1": 0.10409, - "precision_at_10": 0.02699, - "precision_at_100": 0.00481, - "precision_at_1000": 0.00078, - "precision_at_20": 0.01637, - "precision_at_3": 0.05875, - "precision_at_5": 0.04338, - "recall_at_1": 0.10409, - "recall_at_10": 0.26994, - "recall_at_100": 0.48052, - "recall_at_1000": 0.77761, - "recall_at_20": 0.32747, - "recall_at_3": 0.17626, - "recall_at_5": 0.21691 - }, - { - "cv_recall_at_1": 0.68862, - "cv_recall_at_10": 0.94441, - "cv_recall_at_100": 0.99696, - "cv_recall_at_1000": 0.99988, - "cv_recall_at_20": 0.97384, - "cv_recall_at_3": 0.8491, - "cv_recall_at_5": 0.89757, - "hf_subset": "fr", - "languages": [ - "fra-Latn" - ], - "main_score": 0.81861, - "map_at_1": 0.68862, - "map_at_10": 0.77802, - "map_at_100": 0.78078, - "map_at_1000": 0.78079, - "map_at_20": 0.7801, - "map_at_3": 0.76039, - "map_at_5": 0.77162, - "mrr_at_1": 0.6886241532352254, - "mrr_at_10": 0.7780242989510651, - "mrr_at_100": 0.780775407295502, - "mrr_at_1000": 0.780794622085721, - "mrr_at_20": 0.7800977472520112, - "mrr_at_3": 0.7603947675776656, - "mrr_at_5": 0.7716187806587189, - "nauc_cv_recall_at_1000_diff1": 0.8692844384055428, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 0.8692844384055428, - "nauc_cv_recall_at_100_diff1": 0.6691478259630901, - "nauc_cv_recall_at_100_max": 0.7475953505943645, - "nauc_cv_recall_at_100_std": 0.27056257097105973, - "nauc_cv_recall_at_10_diff1": 0.6880948148322499, - "nauc_cv_recall_at_10_max": 0.5467060331144916, - "nauc_cv_recall_at_10_std": -0.2425628667761323, - "nauc_cv_recall_at_1_diff1": 0.7838184527245065, - "nauc_cv_recall_at_1_max": 0.4390541428215865, - "nauc_cv_recall_at_1_std": -0.2511815664212096, - "nauc_cv_recall_at_20_diff1": 0.6995952401305364, - "nauc_cv_recall_at_20_max": 0.649355920590757, - "nauc_cv_recall_at_20_std": -0.07975196053310503, - "nauc_cv_recall_at_3_diff1": 0.7060942351703845, - "nauc_cv_recall_at_3_max": 0.5002462807404169, - "nauc_cv_recall_at_3_std": -0.26503056328244606, - "nauc_cv_recall_at_5_diff1": 0.6879778818833843, - "nauc_cv_recall_at_5_max": 0.5076987150481848, - "nauc_cv_recall_at_5_std": -0.27728822139617615, - "nauc_map_at_1000_diff1": 0.7607228052600944, - "nauc_map_at_1000_max": 0.4572493121820054, - "nauc_map_at_1000_std": -0.25597521847200916, - "nauc_map_at_100_diff1": 0.7607113854693041, - "nauc_map_at_100_max": 0.4572754191998793, - "nauc_map_at_100_std": -0.255932878139392, - "nauc_map_at_10_diff1": 0.7596199821431895, - "nauc_map_at_10_max": 0.4577533173217279, - "nauc_map_at_10_std": -0.25703026926053885, - "nauc_map_at_1_diff1": 0.7838184527245065, - "nauc_map_at_1_max": 0.4390541428215865, - "nauc_map_at_1_std": -0.2511815664212096, - "nauc_map_at_20_diff1": 0.7605122214228532, - "nauc_map_at_20_max": 0.4578668309208731, - "nauc_map_at_20_std": -0.2556437602241119, - "nauc_map_at_3_diff1": 0.7571372016948014, - "nauc_map_at_3_max": 0.45891292471001566, - "nauc_map_at_3_std": -0.2575630818052683, - "nauc_map_at_5_diff1": 0.7572888701559919, - "nauc_map_at_5_max": 0.4577447733020841, - "nauc_map_at_5_std": -0.25873803768931697, - "nauc_mrr_at_1000_diff1": 0.7607228052600944, - "nauc_mrr_at_1000_max": 0.4572493121820054, - "nauc_mrr_at_1000_std": -0.25597521847200916, - "nauc_mrr_at_100_diff1": 0.7607113854693041, - "nauc_mrr_at_100_max": 0.4572754191998793, - "nauc_mrr_at_100_std": -0.255932878139392, - "nauc_mrr_at_10_diff1": 0.7596199821431895, - "nauc_mrr_at_10_max": 0.4577533173217279, - "nauc_mrr_at_10_std": -0.25703026926053885, - "nauc_mrr_at_1_diff1": 0.7838184527245065, - "nauc_mrr_at_1_max": 0.4390541428215865, - "nauc_mrr_at_1_std": -0.2511815664212096, - "nauc_mrr_at_20_diff1": 0.7605122214228532, - "nauc_mrr_at_20_max": 0.4578668309208731, - "nauc_mrr_at_20_std": -0.2556437602241119, - "nauc_mrr_at_3_diff1": 0.7571372016948014, - "nauc_mrr_at_3_max": 0.45891292471001566, - "nauc_mrr_at_3_std": -0.2575630818052683, - "nauc_mrr_at_5_diff1": 0.7572888701559919, - "nauc_mrr_at_5_max": 0.4577447733020841, - "nauc_mrr_at_5_std": -0.25873803768931697, - "nauc_ndcg_at_1000_diff1": 0.7578805872601987, - "nauc_ndcg_at_1000_max": 0.46077069695263406, - "nauc_ndcg_at_1000_std": -0.2547039501696542, - "nauc_ndcg_at_100_diff1": 0.7576123162409897, - "nauc_ndcg_at_100_max": 0.4614898486574226, - "nauc_ndcg_at_100_std": -0.25343030205722045, - "nauc_ndcg_at_10_diff1": 0.7524321328014326, - "nauc_ndcg_at_10_max": 0.4658254372938121, - "nauc_ndcg_at_10_std": -0.2564521956378803, - "nauc_ndcg_at_1_diff1": 0.7838184527245065, - "nauc_ndcg_at_1_max": 0.4390541428215865, - "nauc_ndcg_at_1_std": -0.2511815664212096, - "nauc_ndcg_at_20_diff1": 0.7560684300061722, - "nauc_ndcg_at_20_max": 0.46645964477127183, - "nauc_ndcg_at_20_std": -0.25013633353776266, - "nauc_ndcg_at_3_diff1": 0.7472704480142333, - "nauc_ndcg_at_3_max": 0.46684629344125356, - "nauc_ndcg_at_3_std": -0.25908669463469447, - "nauc_ndcg_at_5_diff1": 0.7468090942832256, - "nauc_ndcg_at_5_max": 0.46528854262350067, - "nauc_ndcg_at_5_std": -0.2615309959446548, - "nauc_precision_at_1000_diff1": 0.8692844384061307, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 0.8692844384061307, - "nauc_precision_at_100_diff1": 0.6691478259631217, - "nauc_precision_at_100_max": 0.7475953505944476, - "nauc_precision_at_100_std": 0.27056257097103237, - "nauc_precision_at_10_diff1": 0.6880948148322517, - "nauc_precision_at_10_max": 0.5467060331144964, - "nauc_precision_at_10_std": -0.24256286677613534, - "nauc_precision_at_1_diff1": 0.7838184527245065, - "nauc_precision_at_1_max": 0.4390541428215865, - "nauc_precision_at_1_std": -0.2511815664212096, - "nauc_precision_at_20_diff1": 0.6995952401305368, - "nauc_precision_at_20_max": 0.6493559205907617, - "nauc_precision_at_20_std": -0.07975196053311341, - "nauc_precision_at_3_diff1": 0.7060942351703826, - "nauc_precision_at_3_max": 0.5002462807404171, - "nauc_precision_at_3_std": -0.26503056328244434, - "nauc_precision_at_5_diff1": 0.687977881883383, - "nauc_precision_at_5_max": 0.5076987150481839, - "nauc_precision_at_5_std": -0.2772882213961815, - "nauc_recall_at_1000_diff1": 0.8692844384055428, - "nauc_recall_at_1000_max": 1.0, - "nauc_recall_at_1000_std": 0.8692844384055428, - "nauc_recall_at_100_diff1": 0.6691478259630901, - "nauc_recall_at_100_max": 0.7475953505943645, - "nauc_recall_at_100_std": 0.27056257097105973, - "nauc_recall_at_10_diff1": 0.6880948148322499, - "nauc_recall_at_10_max": 0.5467060331144916, - "nauc_recall_at_10_std": -0.2425628667761323, - "nauc_recall_at_1_diff1": 0.7838184527245065, - "nauc_recall_at_1_max": 0.4390541428215865, - "nauc_recall_at_1_std": -0.2511815664212096, - "nauc_recall_at_20_diff1": 0.6995952401305364, - "nauc_recall_at_20_max": 0.649355920590757, - "nauc_recall_at_20_std": -0.07975196053310503, - "nauc_recall_at_3_diff1": 0.7060942351703845, - "nauc_recall_at_3_max": 0.5002462807404169, - "nauc_recall_at_3_std": -0.26503056328244606, - "nauc_recall_at_5_diff1": 0.6879778818833843, - "nauc_recall_at_5_max": 0.5076987150481848, - "nauc_recall_at_5_std": -0.27728822139617615, - "ndcg_at_1": 0.68862, - "ndcg_at_10": 0.81861, - "ndcg_at_100": 0.83055, - "ndcg_at_1000": 0.83095, - "ndcg_at_20": 0.82609, - "ndcg_at_3": 0.78322, - "ndcg_at_5": 0.80331, - "precision_at_1": 0.68862, - "precision_at_10": 0.09444, - "precision_at_100": 0.00997, - "precision_at_1000": 0.001, - "precision_at_20": 0.04869, - "precision_at_3": 0.28303, - "precision_at_5": 0.17951, - "recall_at_1": 0.68862, - "recall_at_10": 0.94441, - "recall_at_100": 0.99696, - "recall_at_1000": 0.99988, - "recall_at_20": 0.97384, - "recall_at_3": 0.8491, - "recall_at_5": 0.89757 - }, - { - "cv_recall_at_1": 0.26514, - "cv_recall_at_10": 0.54569, - "cv_recall_at_100": 0.8125, - "cv_recall_at_1000": 0.97514, - "cv_recall_at_20": 0.62986, - "cv_recall_at_3": 0.39319, - "cv_recall_at_5": 0.45528, - "hf_subset": "he", - "languages": [ - "heb-Hebr" - ], - "main_score": 0.39416, - "map_at_1": 0.26514, - "map_at_10": 0.34707, - "map_at_100": 0.35737, - "map_at_1000": 0.35809, - "map_at_20": 0.35287, - "map_at_3": 0.32095, - "map_at_5": 0.33505, - "mrr_at_1": 0.26513888888888887, - "mrr_at_10": 0.34706889329806007, - "mrr_at_100": 0.3573730842851282, - "mrr_at_1000": 0.35808713808956977, - "mrr_at_20": 0.35286717176979276, - "mrr_at_3": 0.3209490740740735, - "mrr_at_5": 0.3350532407407405, - "nauc_cv_recall_at_1000_diff1": 0.3269851702319663, - "nauc_cv_recall_at_1000_max": 0.8007266221199839, - "nauc_cv_recall_at_1000_std": 0.8604030066402758, - "nauc_cv_recall_at_100_diff1": 0.30351047766360917, - "nauc_cv_recall_at_100_max": 0.6370308989798541, - "nauc_cv_recall_at_100_std": 0.6387209516443855, - "nauc_cv_recall_at_10_diff1": 0.37820827033091164, - "nauc_cv_recall_at_10_max": 0.5316589717810623, - "nauc_cv_recall_at_10_std": 0.4018630792445418, - "nauc_cv_recall_at_1_diff1": 0.5573653194549761, - "nauc_cv_recall_at_1_max": 0.4823866545239, - "nauc_cv_recall_at_1_std": 0.23133760983158186, - "nauc_cv_recall_at_20_diff1": 0.35068725924169186, - "nauc_cv_recall_at_20_max": 0.5735204752668261, - "nauc_cv_recall_at_20_std": 0.4857388404002906, - "nauc_cv_recall_at_3_diff1": 0.44625816393502604, - "nauc_cv_recall_at_3_max": 0.5034472787959484, - "nauc_cv_recall_at_3_std": 0.30912719460546667, - "nauc_cv_recall_at_5_diff1": 0.4136262213740317, - "nauc_cv_recall_at_5_max": 0.5175696018757037, - "nauc_cv_recall_at_5_std": 0.3483951091470087, - "nauc_map_at_1000_diff1": 0.4927579225630324, - "nauc_map_at_1000_max": 0.5000060861294572, - "nauc_map_at_1000_std": 0.2886077873016723, - "nauc_map_at_100_diff1": 0.4923931506441432, - "nauc_map_at_100_max": 0.5001400591264515, - "nauc_map_at_100_std": 0.289072191366851, - "nauc_map_at_10_diff1": 0.4927509866960801, - "nauc_map_at_10_max": 0.49742219248009734, - "nauc_map_at_10_std": 0.2840858256437432, - "nauc_map_at_1_diff1": 0.5573653194549761, - "nauc_map_at_1_max": 0.4823866545239, - "nauc_map_at_1_std": 0.23133760983158186, - "nauc_map_at_20_diff1": 0.49190107685036644, - "nauc_map_at_20_max": 0.49970774159254117, - "nauc_map_at_20_std": 0.2883899724055493, - "nauc_map_at_3_diff1": 0.5049691154013799, - "nauc_map_at_3_max": 0.49270595560391245, - "nauc_map_at_3_std": 0.26857534912218023, - "nauc_map_at_5_diff1": 0.49706923863072766, - "nauc_map_at_5_max": 0.49588840312255467, - "nauc_map_at_5_std": 0.27777587422769684, - "nauc_mrr_at_1000_diff1": 0.4927578897775547, - "nauc_mrr_at_1000_max": 0.5000060857020273, - "nauc_mrr_at_1000_std": 0.2886077752540086, - "nauc_mrr_at_100_diff1": 0.49239311795880597, - "nauc_mrr_at_100_max": 0.5001400586862619, - "nauc_mrr_at_100_std": 0.2890721792953119, - "nauc_mrr_at_10_diff1": 0.4927509866960801, - "nauc_mrr_at_10_max": 0.49742219248009734, - "nauc_mrr_at_10_std": 0.2840858256437432, - "nauc_mrr_at_1_diff1": 0.5573653194549761, - "nauc_mrr_at_1_max": 0.4823866545239, - "nauc_mrr_at_1_std": 0.23133760983158186, - "nauc_mrr_at_20_diff1": 0.49190107685036644, - "nauc_mrr_at_20_max": 0.49970774159254117, - "nauc_mrr_at_20_std": 0.2883899724055493, - "nauc_mrr_at_3_diff1": 0.5049691154013799, - "nauc_mrr_at_3_max": 0.49270595560391245, - "nauc_mrr_at_3_std": 0.26857534912218023, - "nauc_mrr_at_5_diff1": 0.49706923863072766, - "nauc_mrr_at_5_max": 0.49588840312255467, - "nauc_mrr_at_5_std": 0.27777587422769684, - "nauc_ndcg_at_1000_diff1": 0.4744650521139254, - "nauc_ndcg_at_1000_max": 0.5116627297111898, - "nauc_ndcg_at_1000_std": 0.31760456896239264, - "nauc_ndcg_at_100_diff1": 0.4638063088866722, - "nauc_ndcg_at_100_max": 0.5171831990090189, - "nauc_ndcg_at_100_std": 0.3341316651561419, - "nauc_ndcg_at_10_diff1": 0.46499963403197564, - "nauc_ndcg_at_10_max": 0.505441465700467, - "nauc_ndcg_at_10_std": 0.31177303332403883, - "nauc_ndcg_at_1_diff1": 0.5573653194549761, - "nauc_ndcg_at_1_max": 0.4823866545239, - "nauc_ndcg_at_1_std": 0.23133760983158186, - "nauc_ndcg_at_20_diff1": 0.46109580163026737, - "nauc_ndcg_at_20_max": 0.5142500463015515, - "nauc_ndcg_at_20_std": 0.3284962350243677, - "nauc_ndcg_at_3_diff1": 0.48924758859092904, - "nauc_ndcg_at_3_max": 0.49559857295157583, - "nauc_ndcg_at_3_std": 0.2794575762925081, - "nauc_ndcg_at_5_diff1": 0.4753797790260948, - "nauc_ndcg_at_5_max": 0.5014202054503818, - "nauc_ndcg_at_5_std": 0.29593140631395104, - "nauc_precision_at_1000_diff1": 0.3269851702319637, - "nauc_precision_at_1000_max": 0.8007266221199862, - "nauc_precision_at_1000_std": 0.8604030066402799, - "nauc_precision_at_100_diff1": 0.3035104776636093, - "nauc_precision_at_100_max": 0.6370308989798543, - "nauc_precision_at_100_std": 0.6387209516443859, - "nauc_precision_at_10_diff1": 0.3782082703309115, - "nauc_precision_at_10_max": 0.5316589717810623, - "nauc_precision_at_10_std": 0.4018630792445415, - "nauc_precision_at_1_diff1": 0.5573653194549761, - "nauc_precision_at_1_max": 0.4823866545239, - "nauc_precision_at_1_std": 0.23133760983158186, - "nauc_precision_at_20_diff1": 0.3506872592416918, - "nauc_precision_at_20_max": 0.5735204752668259, - "nauc_precision_at_20_std": 0.4857388404002907, - "nauc_precision_at_3_diff1": 0.4462581639350264, - "nauc_precision_at_3_max": 0.5034472787959483, - "nauc_precision_at_3_std": 0.30912719460546667, - "nauc_precision_at_5_diff1": 0.4136262213740317, - "nauc_precision_at_5_max": 0.5175696018757038, - "nauc_precision_at_5_std": 0.34839510914700866, - "nauc_recall_at_1000_diff1": 0.3269851702319663, - "nauc_recall_at_1000_max": 0.8007266221199839, - "nauc_recall_at_1000_std": 0.8604030066402758, - "nauc_recall_at_100_diff1": 0.30351047766360917, - "nauc_recall_at_100_max": 0.6370308989798541, - "nauc_recall_at_100_std": 0.6387209516443855, - "nauc_recall_at_10_diff1": 0.37820827033091164, - "nauc_recall_at_10_max": 0.5316589717810623, - "nauc_recall_at_10_std": 0.4018630792445418, - "nauc_recall_at_1_diff1": 0.5573653194549761, - "nauc_recall_at_1_max": 0.4823866545239, - "nauc_recall_at_1_std": 0.23133760983158186, - "nauc_recall_at_20_diff1": 0.35068725924169186, - "nauc_recall_at_20_max": 0.5735204752668261, - "nauc_recall_at_20_std": 0.4857388404002906, - "nauc_recall_at_3_diff1": 0.44625816393502604, - "nauc_recall_at_3_max": 0.5034472787959484, - "nauc_recall_at_3_std": 0.30912719460546667, - "nauc_recall_at_5_diff1": 0.4136262213740317, - "nauc_recall_at_5_max": 0.5175696018757037, - "nauc_recall_at_5_std": 0.3483951091470087, - "ndcg_at_1": 0.26514, - "ndcg_at_10": 0.39416, - "ndcg_at_100": 0.44892, - "ndcg_at_1000": 0.46923, - "ndcg_at_20": 0.41539, - "ndcg_at_3": 0.33948, - "ndcg_at_5": 0.36497, - "precision_at_1": 0.26514, - "precision_at_10": 0.05457, - "precision_at_100": 0.00812, - "precision_at_1000": 0.00098, - "precision_at_20": 0.03149, - "precision_at_3": 0.13106, - "precision_at_5": 0.09106, - "recall_at_1": 0.26514, - "recall_at_10": 0.54569, - "recall_at_100": 0.8125, - "recall_at_1000": 0.97514, - "recall_at_20": 0.62986, - "recall_at_3": 0.39319, - "recall_at_5": 0.45528 - }, - { - "cv_recall_at_1": 0.05833, - "cv_recall_at_10": 0.21804, - "cv_recall_at_100": 0.54463, - "cv_recall_at_1000": 0.90909, - "cv_recall_at_20": 0.29989, - "cv_recall_at_3": 0.11372, - "cv_recall_at_5": 0.15242, - "hf_subset": "hi", - "languages": [ - "hin-Deva" - ], - "main_score": 0.12728, - "map_at_1": 0.05833, - "map_at_10": 0.09962, - "map_at_100": 0.11096, - "map_at_1000": 0.11246, - "map_at_20": 0.10526, - "map_at_3": 0.08211, - "map_at_5": 0.09091, - "mrr_at_1": 0.05833235328707515, - "mrr_at_10": 0.09962170214434136, - "mrr_at_100": 0.11096139209081642, - "mrr_at_1000": 0.11246454103870365, - "mrr_at_20": 0.1052608397121846, - "mrr_at_3": 0.08210827551060423, - "mrr_at_5": 0.09091105100160765, - "nauc_cv_recall_at_1000_diff1": 0.05707411396874622, - "nauc_cv_recall_at_1000_max": 0.4305314431765789, - "nauc_cv_recall_at_1000_std": 0.6174045190761287, - "nauc_cv_recall_at_100_diff1": 0.10092964522892446, - "nauc_cv_recall_at_100_max": 0.29425618904662393, - "nauc_cv_recall_at_100_std": 0.3670656043747271, - "nauc_cv_recall_at_10_diff1": 0.14125562730402333, - "nauc_cv_recall_at_10_max": 0.22871058991291543, - "nauc_cv_recall_at_10_std": 0.19129187889176197, - "nauc_cv_recall_at_1_diff1": 0.3300486422524422, - "nauc_cv_recall_at_1_max": 0.3317286526321265, - "nauc_cv_recall_at_1_std": 0.18794368017024868, - "nauc_cv_recall_at_20_diff1": 0.12612567776569172, - "nauc_cv_recall_at_20_max": 0.2318086938059483, - "nauc_cv_recall_at_20_std": 0.2284786449365189, - "nauc_cv_recall_at_3_diff1": 0.20819459246454827, - "nauc_cv_recall_at_3_max": 0.27691548648139885, - "nauc_cv_recall_at_3_std": 0.18434800757157196, - "nauc_cv_recall_at_5_diff1": 0.17383631683254794, - "nauc_cv_recall_at_5_max": 0.2524853389049951, - "nauc_cv_recall_at_5_std": 0.18646658605239053, - "nauc_map_at_1000_diff1": 0.2234782052076568, - "nauc_map_at_1000_max": 0.2848397578636281, - "nauc_map_at_1000_std": 0.19887907283255718, - "nauc_map_at_100_diff1": 0.22313787879826344, - "nauc_map_at_100_max": 0.2843108110506845, - "nauc_map_at_100_std": 0.1985558390711766, - "nauc_map_at_10_diff1": 0.22840032784693742, - "nauc_map_at_10_max": 0.28147844267923133, - "nauc_map_at_10_std": 0.18601453853576044, - "nauc_map_at_1_diff1": 0.3300486422524422, - "nauc_map_at_1_max": 0.3317286526321265, - "nauc_map_at_1_std": 0.18794368017024868, - "nauc_map_at_20_diff1": 0.22437804015760093, - "nauc_map_at_20_max": 0.2812076570563848, - "nauc_map_at_20_std": 0.19193729091168843, - "nauc_map_at_3_diff1": 0.25646565525415654, - "nauc_map_at_3_max": 0.2999563478016542, - "nauc_map_at_3_std": 0.18478880360825953, - "nauc_map_at_5_diff1": 0.2409823131635885, - "nauc_map_at_5_max": 0.2895968259498882, - "nauc_map_at_5_std": 0.18532356610291725, - "nauc_mrr_at_1000_diff1": 0.2234782052076568, - "nauc_mrr_at_1000_max": 0.2848397578636281, - "nauc_mrr_at_1000_std": 0.19887907283255718, - "nauc_mrr_at_100_diff1": 0.22313787879826344, - "nauc_mrr_at_100_max": 0.2843108110506845, - "nauc_mrr_at_100_std": 0.1985558390711766, - "nauc_mrr_at_10_diff1": 0.22840032784693742, - "nauc_mrr_at_10_max": 0.28147844267923133, - "nauc_mrr_at_10_std": 0.18601453853576044, - "nauc_mrr_at_1_diff1": 0.3300486422524422, - "nauc_mrr_at_1_max": 0.3317286526321265, - "nauc_mrr_at_1_std": 0.18794368017024868, - "nauc_mrr_at_20_diff1": 0.22437804015760093, - "nauc_mrr_at_20_max": 0.2812076570563848, - "nauc_mrr_at_20_std": 0.19193729091168843, - "nauc_mrr_at_3_diff1": 0.25646565525415654, - "nauc_mrr_at_3_max": 0.2999563478016542, - "nauc_mrr_at_3_std": 0.18478880360825953, - "nauc_mrr_at_5_diff1": 0.2409823131635885, - "nauc_mrr_at_5_max": 0.2895968259498882, - "nauc_mrr_at_5_std": 0.18532356610291725, - "nauc_ndcg_at_1000_diff1": 0.1890100975712789, - "nauc_ndcg_at_1000_max": 0.28911610663847015, - "nauc_ndcg_at_1000_std": 0.24423313156730872, - "nauc_ndcg_at_100_diff1": 0.17960027600642617, - "nauc_ndcg_at_100_max": 0.28107378010359907, - "nauc_ndcg_at_100_std": 0.24509423028735777, - "nauc_ndcg_at_10_diff1": 0.19636121499921674, - "nauc_ndcg_at_10_max": 0.2623750702242693, - "nauc_ndcg_at_10_std": 0.18766127725556314, - "nauc_ndcg_at_1_diff1": 0.3300486422524422, - "nauc_ndcg_at_1_max": 0.3317286526321265, - "nauc_ndcg_at_1_std": 0.18794368017024868, - "nauc_ndcg_at_20_diff1": 0.18666766339752958, - "nauc_ndcg_at_20_max": 0.2620930821244513, - "nauc_ndcg_at_20_std": 0.2041134409992863, - "nauc_ndcg_at_3_diff1": 0.24103897094024151, - "nauc_ndcg_at_3_max": 0.2926455989044745, - "nauc_ndcg_at_3_std": 0.18460883817440596, - "nauc_ndcg_at_5_diff1": 0.2181687546993421, - "nauc_ndcg_at_5_max": 0.2771137878870709, - "nauc_ndcg_at_5_std": 0.1856287474451102, - "nauc_precision_at_1000_diff1": 0.05707411396874407, - "nauc_precision_at_1000_max": 0.4305314431765774, - "nauc_precision_at_1000_std": 0.6174045190761295, - "nauc_precision_at_100_diff1": 0.10092964522892454, - "nauc_precision_at_100_max": 0.2942561890466242, - "nauc_precision_at_100_std": 0.36706560437472713, - "nauc_precision_at_10_diff1": 0.14125562730402325, - "nauc_precision_at_10_max": 0.2287105899129151, - "nauc_precision_at_10_std": 0.19129187889176166, - "nauc_precision_at_1_diff1": 0.3300486422524422, - "nauc_precision_at_1_max": 0.3317286526321265, - "nauc_precision_at_1_std": 0.18794368017024868, - "nauc_precision_at_20_diff1": 0.12612567776569164, - "nauc_precision_at_20_max": 0.23180869380594815, - "nauc_precision_at_20_std": 0.22847864493651887, - "nauc_precision_at_3_diff1": 0.20819459246454838, - "nauc_precision_at_3_max": 0.27691548648139863, - "nauc_precision_at_3_std": 0.18434800757157188, - "nauc_precision_at_5_diff1": 0.1738363168325479, - "nauc_precision_at_5_max": 0.2524853389049951, - "nauc_precision_at_5_std": 0.1864665860523905, - "nauc_recall_at_1000_diff1": 0.05707411396874622, - "nauc_recall_at_1000_max": 0.4305314431765789, - "nauc_recall_at_1000_std": 0.6174045190761287, - "nauc_recall_at_100_diff1": 0.10092964522892446, - "nauc_recall_at_100_max": 0.29425618904662393, - "nauc_recall_at_100_std": 0.3670656043747271, - "nauc_recall_at_10_diff1": 0.14125562730402333, - "nauc_recall_at_10_max": 0.22871058991291543, - "nauc_recall_at_10_std": 0.19129187889176197, - "nauc_recall_at_1_diff1": 0.3300486422524422, - "nauc_recall_at_1_max": 0.3317286526321265, - "nauc_recall_at_1_std": 0.18794368017024868, - "nauc_recall_at_20_diff1": 0.12612567776569172, - "nauc_recall_at_20_max": 0.2318086938059483, - "nauc_recall_at_20_std": 0.2284786449365189, - "nauc_recall_at_3_diff1": 0.20819459246454827, - "nauc_recall_at_3_max": 0.27691548648139885, - "nauc_recall_at_3_std": 0.18434800757157196, - "nauc_recall_at_5_diff1": 0.17383631683254794, - "nauc_recall_at_5_max": 0.2524853389049951, - "nauc_recall_at_5_std": 0.18646658605239053, - "ndcg_at_1": 0.05833, - "ndcg_at_10": 0.12728, - "ndcg_at_100": 0.19216, - "ndcg_at_1000": 0.23713, - "ndcg_at_20": 0.14792, - "ndcg_at_3": 0.0902, - "ndcg_at_5": 0.1061, - "precision_at_1": 0.05833, - "precision_at_10": 0.0218, - "precision_at_100": 0.00545, - "precision_at_1000": 0.00091, - "precision_at_20": 0.01499, - "precision_at_3": 0.03791, - "precision_at_5": 0.03048, - "recall_at_1": 0.05833, - "recall_at_10": 0.21804, - "recall_at_100": 0.54463, - "recall_at_1000": 0.90909, - "recall_at_20": 0.29989, - "recall_at_3": 0.11372, - "recall_at_5": 0.15242 - }, - { - "cv_recall_at_1": 0.30014, - "cv_recall_at_10": 0.61264, - "cv_recall_at_100": 0.86044, - "cv_recall_at_1000": 0.97569, - "cv_recall_at_20": 0.69876, - "cv_recall_at_3": 0.45371, - "cv_recall_at_5": 0.51909, - "hf_subset": "hr", - "languages": [ - "hrv-Latn" - ], - "main_score": 0.44666, - "map_at_1": 0.30014, - "map_at_10": 0.39473, - "map_at_100": 0.40487, - "map_at_1000": 0.4054, - "map_at_20": 0.40078, - "map_at_3": 0.36726, - "map_at_5": 0.38221, - "mrr_at_1": 0.3001373626373626, - "mrr_at_10": 0.39472549276120766, - "mrr_at_100": 0.4048696521882351, - "mrr_at_1000": 0.4054039192989718, - "mrr_at_20": 0.4007758807961287, - "mrr_at_3": 0.3672619047619057, - "mrr_at_5": 0.38220695970695995, - "nauc_cv_recall_at_1000_diff1": 0.3586726592708677, - "nauc_cv_recall_at_1000_max": 0.8499844382197388, - "nauc_cv_recall_at_1000_std": 0.9014728301866921, - "nauc_cv_recall_at_100_diff1": 0.3620273025034925, - "nauc_cv_recall_at_100_max": 0.7129467387403892, - "nauc_cv_recall_at_100_std": 0.6609581262359034, - "nauc_cv_recall_at_10_diff1": 0.3842176703377357, - "nauc_cv_recall_at_10_max": 0.5471484927211087, - "nauc_cv_recall_at_10_std": 0.3463565927031771, - "nauc_cv_recall_at_1_diff1": 0.5663570758468127, - "nauc_cv_recall_at_1_max": 0.4682186108963969, - "nauc_cv_recall_at_1_std": 0.17349940936789837, - "nauc_cv_recall_at_20_diff1": 0.37088773390466634, - "nauc_cv_recall_at_20_max": 0.5907304165964303, - "nauc_cv_recall_at_20_std": 0.4283686973553312, - "nauc_cv_recall_at_3_diff1": 0.4475332706202563, - "nauc_cv_recall_at_3_max": 0.49930415860465016, - "nauc_cv_recall_at_3_std": 0.24217892102168642, - "nauc_cv_recall_at_5_diff1": 0.41165724944078125, - "nauc_cv_recall_at_5_max": 0.5111652414669552, - "nauc_cv_recall_at_5_std": 0.2742803051020646, - "nauc_map_at_1000_diff1": 0.5042398721002027, - "nauc_map_at_1000_max": 0.4905559146440463, - "nauc_map_at_1000_std": 0.22119873498521436, - "nauc_map_at_100_diff1": 0.5040238826821266, - "nauc_map_at_100_max": 0.49077281353278235, - "nauc_map_at_100_std": 0.22164398487988576, - "nauc_map_at_10_diff1": 0.5025445743495411, - "nauc_map_at_10_max": 0.4881575618691484, - "nauc_map_at_10_std": 0.2174519067162409, - "nauc_map_at_1_diff1": 0.5663570758468127, - "nauc_map_at_1_max": 0.4682186108963969, - "nauc_map_at_1_std": 0.17349940936789837, - "nauc_map_at_20_diff1": 0.5028888029066095, - "nauc_map_at_20_max": 0.49009005537891887, - "nauc_map_at_20_std": 0.22069609562067313, - "nauc_map_at_3_diff1": 0.5123125544273271, - "nauc_map_at_3_max": 0.4813389503783199, - "nauc_map_at_3_std": 0.20237276561219283, - "nauc_map_at_5_diff1": 0.504540343021302, - "nauc_map_at_5_max": 0.4839795117311297, - "nauc_map_at_5_std": 0.20949681898134187, - "nauc_mrr_at_1000_diff1": 0.5042398721002027, - "nauc_mrr_at_1000_max": 0.4905559146440463, - "nauc_mrr_at_1000_std": 0.22119873498521436, - "nauc_mrr_at_100_diff1": 0.5040238826821266, - "nauc_mrr_at_100_max": 0.49077281353278235, - "nauc_mrr_at_100_std": 0.22164398487988576, - "nauc_mrr_at_10_diff1": 0.5025445743495411, - "nauc_mrr_at_10_max": 0.4881575618691484, - "nauc_mrr_at_10_std": 0.2174519067162409, - "nauc_mrr_at_1_diff1": 0.5663570758468127, - "nauc_mrr_at_1_max": 0.4682186108963969, - "nauc_mrr_at_1_std": 0.17349940936789837, - "nauc_mrr_at_20_diff1": 0.5028888029066095, - "nauc_mrr_at_20_max": 0.49009005537891887, - "nauc_mrr_at_20_std": 0.22069609562067313, - "nauc_mrr_at_3_diff1": 0.5123125544273271, - "nauc_mrr_at_3_max": 0.4813389503783199, - "nauc_mrr_at_3_std": 0.20237276561219283, - "nauc_mrr_at_5_diff1": 0.504540343021302, - "nauc_mrr_at_5_max": 0.4839795117311297, - "nauc_mrr_at_5_std": 0.20949681898134187, - "nauc_ndcg_at_1000_diff1": 0.48946212025389374, - "nauc_ndcg_at_1000_max": 0.5051410930621354, - "nauc_ndcg_at_1000_std": 0.25004833305588986, - "nauc_ndcg_at_100_diff1": 0.48360996856535743, - "nauc_ndcg_at_100_max": 0.5128779666752031, - "nauc_ndcg_at_100_std": 0.2657005413472347, - "nauc_ndcg_at_10_diff1": 0.4756950853648613, - "nauc_ndcg_at_10_max": 0.5008947200817642, - "nauc_ndcg_at_10_std": 0.24531428168736577, - "nauc_ndcg_at_1_diff1": 0.5663570758468127, - "nauc_ndcg_at_1_max": 0.4682186108963969, - "nauc_ndcg_at_1_std": 0.17349940936789837, - "nauc_ndcg_at_20_diff1": 0.47626368707300626, - "nauc_ndcg_at_20_max": 0.5082725864196117, - "nauc_ndcg_at_20_std": 0.25838814733070764, - "nauc_ndcg_at_3_diff1": 0.49549491996306794, - "nauc_ndcg_at_3_max": 0.4859596037574002, - "nauc_ndcg_at_3_std": 0.21260631844181213, - "nauc_ndcg_at_5_diff1": 0.4814150548165864, - "nauc_ndcg_at_5_max": 0.49068965409631315, - "nauc_ndcg_at_5_std": 0.22542235043204617, - "nauc_precision_at_1000_diff1": 0.35867265927085856, - "nauc_precision_at_1000_max": 0.8499844382197332, - "nauc_precision_at_1000_std": 0.9014728301866906, - "nauc_precision_at_100_diff1": 0.3620273025034949, - "nauc_precision_at_100_max": 0.712946738740391, - "nauc_precision_at_100_std": 0.6609581262359061, - "nauc_precision_at_10_diff1": 0.38421767033773546, - "nauc_precision_at_10_max": 0.5471484927211082, - "nauc_precision_at_10_std": 0.3463565927031767, - "nauc_precision_at_1_diff1": 0.5663570758468127, - "nauc_precision_at_1_max": 0.4682186108963969, - "nauc_precision_at_1_std": 0.17349940936789837, - "nauc_precision_at_20_diff1": 0.3708877339046656, - "nauc_precision_at_20_max": 0.5907304165964296, - "nauc_precision_at_20_std": 0.4283686973553297, - "nauc_precision_at_3_diff1": 0.4475332706202558, - "nauc_precision_at_3_max": 0.49930415860465016, - "nauc_precision_at_3_std": 0.24217892102168648, - "nauc_precision_at_5_diff1": 0.4116572494407812, - "nauc_precision_at_5_max": 0.5111652414669557, - "nauc_precision_at_5_std": 0.2742803051020649, - "nauc_recall_at_1000_diff1": 0.3586726592708677, - "nauc_recall_at_1000_max": 0.8499844382197388, - "nauc_recall_at_1000_std": 0.9014728301866921, - "nauc_recall_at_100_diff1": 0.3620273025034925, - "nauc_recall_at_100_max": 0.7129467387403892, - "nauc_recall_at_100_std": 0.6609581262359034, - "nauc_recall_at_10_diff1": 0.3842176703377357, - "nauc_recall_at_10_max": 0.5471484927211087, - "nauc_recall_at_10_std": 0.3463565927031771, - "nauc_recall_at_1_diff1": 0.5663570758468127, - "nauc_recall_at_1_max": 0.4682186108963969, - "nauc_recall_at_1_std": 0.17349940936789837, - "nauc_recall_at_20_diff1": 0.37088773390466634, - "nauc_recall_at_20_max": 0.5907304165964303, - "nauc_recall_at_20_std": 0.4283686973553312, - "nauc_recall_at_3_diff1": 0.4475332706202563, - "nauc_recall_at_3_max": 0.49930415860465016, - "nauc_recall_at_3_std": 0.24217892102168642, - "nauc_recall_at_5_diff1": 0.41165724944078125, - "nauc_recall_at_5_max": 0.5111652414669552, - "nauc_recall_at_5_std": 0.2742803051020646, - "ndcg_at_1": 0.30014, - "ndcg_at_10": 0.44666, - "ndcg_at_100": 0.4984, - "ndcg_at_1000": 0.51294, - "ndcg_at_20": 0.46853, - "ndcg_at_3": 0.38944, - "ndcg_at_5": 0.41637, - "precision_at_1": 0.30014, - "precision_at_10": 0.06126, - "precision_at_100": 0.0086, - "precision_at_1000": 0.00098, - "precision_at_20": 0.03494, - "precision_at_3": 0.15124, - "precision_at_5": 0.10382, - "recall_at_1": 0.30014, - "recall_at_10": 0.61264, - "recall_at_100": 0.86044, - "recall_at_1000": 0.97569, - "recall_at_20": 0.69876, - "recall_at_3": 0.45371, - "recall_at_5": 0.51909 - }, - { - "cv_recall_at_1": 0.12957, - "cv_recall_at_10": 0.34202, - "cv_recall_at_100": 0.61572, - "cv_recall_at_1000": 0.89676, - "cv_recall_at_20": 0.41602, - "cv_recall_at_3": 0.21605, - "cv_recall_at_5": 0.264, - "hf_subset": "hu", - "languages": [ - "hun-Latn" - ], - "main_score": 0.22415, - "map_at_1": 0.12957, - "map_at_10": 0.18788, - "map_at_100": 0.1978, - "map_at_1000": 0.19892, - "map_at_20": 0.19295, - "map_at_3": 0.16655, - "map_at_5": 0.1775, - "mrr_at_1": 0.12957317073170732, - "mrr_at_10": 0.1878756533101041, - "mrr_at_100": 0.1978017747317187, - "mrr_at_1000": 0.19891865771814843, - "mrr_at_20": 0.1929485400897659, - "mrr_at_3": 0.16655118255728013, - "mrr_at_5": 0.17749907612712537, - "nauc_cv_recall_at_1000_diff1": 0.1926685407697348, - "nauc_cv_recall_at_1000_max": 0.6660348086677469, - "nauc_cv_recall_at_1000_std": 0.7471546962869257, - "nauc_cv_recall_at_100_diff1": 0.21347335173344084, - "nauc_cv_recall_at_100_max": 0.5658371871490846, - "nauc_cv_recall_at_100_std": 0.6158093204180968, - "nauc_cv_recall_at_10_diff1": 0.24759927370543935, - "nauc_cv_recall_at_10_max": 0.48185439575113903, - "nauc_cv_recall_at_10_std": 0.4256128948881968, - "nauc_cv_recall_at_1_diff1": 0.46692725912228233, - "nauc_cv_recall_at_1_max": 0.48419673225706406, - "nauc_cv_recall_at_1_std": 0.32274242950108506, - "nauc_cv_recall_at_20_diff1": 0.22756116239806018, - "nauc_cv_recall_at_20_max": 0.5193004941893448, - "nauc_cv_recall_at_20_std": 0.49708893210813393, - "nauc_cv_recall_at_3_diff1": 0.32531876464137166, - "nauc_cv_recall_at_3_max": 0.46145983642570054, - "nauc_cv_recall_at_3_std": 0.35095283380672254, - "nauc_cv_recall_at_5_diff1": 0.28815505370161415, - "nauc_cv_recall_at_5_max": 0.4686756489676717, - "nauc_cv_recall_at_5_std": 0.38257551233467973, - "nauc_map_at_1000_diff1": 0.3702845352984357, - "nauc_map_at_1000_max": 0.4817317513479292, - "nauc_map_at_1000_std": 0.3647306036257812, - "nauc_map_at_100_diff1": 0.3698677927453357, - "nauc_map_at_100_max": 0.4816904952589892, - "nauc_map_at_100_std": 0.36499745995427846, - "nauc_map_at_10_diff1": 0.3718401888534464, - "nauc_map_at_10_max": 0.47669317984440557, - "nauc_map_at_10_std": 0.35466092452935705, - "nauc_map_at_1_diff1": 0.46692725912228233, - "nauc_map_at_1_max": 0.48419673225706406, - "nauc_map_at_1_std": 0.32274242950108506, - "nauc_map_at_20_diff1": 0.3694959663934745, - "nauc_map_at_20_max": 0.4804261018834059, - "nauc_map_at_20_std": 0.3618955962942498, - "nauc_map_at_3_diff1": 0.39376791566150743, - "nauc_map_at_3_max": 0.4729620605043809, - "nauc_map_at_3_std": 0.33694067222910934, - "nauc_map_at_5_diff1": 0.38131559850812324, - "nauc_map_at_5_max": 0.47484262669734634, - "nauc_map_at_5_std": 0.3465880790071037, - "nauc_mrr_at_1000_diff1": 0.3702845352984357, - "nauc_mrr_at_1000_max": 0.4817317513479292, - "nauc_mrr_at_1000_std": 0.3647306036257812, - "nauc_mrr_at_100_diff1": 0.3698677927453357, - "nauc_mrr_at_100_max": 0.4816904952589892, - "nauc_mrr_at_100_std": 0.36499745995427846, - "nauc_mrr_at_10_diff1": 0.3718401888534464, - "nauc_mrr_at_10_max": 0.47669317984440557, - "nauc_mrr_at_10_std": 0.35466092452935705, - "nauc_mrr_at_1_diff1": 0.46692725912228233, - "nauc_mrr_at_1_max": 0.48419673225706406, - "nauc_mrr_at_1_std": 0.32274242950108506, - "nauc_mrr_at_20_diff1": 0.3694959663934745, - "nauc_mrr_at_20_max": 0.4804261018834059, - "nauc_mrr_at_20_std": 0.3618955962942498, - "nauc_mrr_at_3_diff1": 0.39376791566150743, - "nauc_mrr_at_3_max": 0.4729620605043809, - "nauc_mrr_at_3_std": 0.33694067222910934, - "nauc_mrr_at_5_diff1": 0.38131559850812324, - "nauc_mrr_at_5_max": 0.47484262669734634, - "nauc_mrr_at_5_std": 0.3465880790071037, - "nauc_ndcg_at_1000_diff1": 0.33817509632392545, - "nauc_ndcg_at_1000_max": 0.49769845184107936, - "nauc_ndcg_at_1000_std": 0.40885320217687693, - "nauc_ndcg_at_100_diff1": 0.32797720523689905, - "nauc_ndcg_at_100_max": 0.49853128913777867, - "nauc_ndcg_at_100_std": 0.4199722248423589, - "nauc_ndcg_at_10_diff1": 0.3343020690828065, - "nauc_ndcg_at_10_max": 0.47766588097578827, - "nauc_ndcg_at_10_std": 0.37506780176899107, - "nauc_ndcg_at_1_diff1": 0.46692725912228233, - "nauc_ndcg_at_1_max": 0.48419673225706406, - "nauc_ndcg_at_1_std": 0.32274242950108506, - "nauc_ndcg_at_20_diff1": 0.3270780139351324, - "nauc_ndcg_at_20_max": 0.4898620209866191, - "nauc_ndcg_at_20_std": 0.39862570492352634, - "nauc_ndcg_at_3_diff1": 0.37372291907645316, - "nauc_ndcg_at_3_max": 0.4696137532695068, - "nauc_ndcg_at_3_std": 0.34102932389855384, - "nauc_ndcg_at_5_diff1": 0.3534976171106256, - "nauc_ndcg_at_5_max": 0.47287581533045453, - "nauc_ndcg_at_5_std": 0.35706313498705516, - "nauc_precision_at_1000_diff1": 0.19266854076973416, - "nauc_precision_at_1000_max": 0.6660348086677451, - "nauc_precision_at_1000_std": 0.7471546962869254, - "nauc_precision_at_100_diff1": 0.21347335173344126, - "nauc_precision_at_100_max": 0.5658371871490853, - "nauc_precision_at_100_std": 0.6158093204180971, - "nauc_precision_at_10_diff1": 0.24759927370543952, - "nauc_precision_at_10_max": 0.4818543957511388, - "nauc_precision_at_10_std": 0.42561289488819687, - "nauc_precision_at_1_diff1": 0.46692725912228233, - "nauc_precision_at_1_max": 0.48419673225706406, - "nauc_precision_at_1_std": 0.32274242950108506, - "nauc_precision_at_20_diff1": 0.22756116239805987, - "nauc_precision_at_20_max": 0.5193004941893443, - "nauc_precision_at_20_std": 0.49708893210813343, - "nauc_precision_at_3_diff1": 0.32531876464137177, - "nauc_precision_at_3_max": 0.4614598364257008, - "nauc_precision_at_3_std": 0.35095283380672265, - "nauc_precision_at_5_diff1": 0.2881550537016138, - "nauc_precision_at_5_max": 0.4686756489676716, - "nauc_precision_at_5_std": 0.38257551233467957, - "nauc_recall_at_1000_diff1": 0.1926685407697348, - "nauc_recall_at_1000_max": 0.6660348086677469, - "nauc_recall_at_1000_std": 0.7471546962869257, - "nauc_recall_at_100_diff1": 0.21347335173344084, - "nauc_recall_at_100_max": 0.5658371871490846, - "nauc_recall_at_100_std": 0.6158093204180968, - "nauc_recall_at_10_diff1": 0.24759927370543935, - "nauc_recall_at_10_max": 0.48185439575113903, - "nauc_recall_at_10_std": 0.4256128948881968, - "nauc_recall_at_1_diff1": 0.46692725912228233, - "nauc_recall_at_1_max": 0.48419673225706406, - "nauc_recall_at_1_std": 0.32274242950108506, - "nauc_recall_at_20_diff1": 0.22756116239806018, - "nauc_recall_at_20_max": 0.5193004941893448, - "nauc_recall_at_20_std": 0.49708893210813393, - "nauc_recall_at_3_diff1": 0.32531876464137166, - "nauc_recall_at_3_max": 0.46145983642570054, - "nauc_recall_at_3_std": 0.35095283380672254, - "nauc_recall_at_5_diff1": 0.28815505370161415, - "nauc_recall_at_5_max": 0.4686756489676717, - "nauc_recall_at_5_std": 0.38257551233467973, - "ndcg_at_1": 0.12957, - "ndcg_at_10": 0.22415, - "ndcg_at_100": 0.27927, - "ndcg_at_1000": 0.31368, - "ndcg_at_20": 0.24278, - "ndcg_at_3": 0.17922, - "ndcg_at_5": 0.19895, - "precision_at_1": 0.12957, - "precision_at_10": 0.0342, - "precision_at_100": 0.00616, - "precision_at_1000": 0.0009, - "precision_at_20": 0.0208, - "precision_at_3": 0.07202, - "precision_at_5": 0.0528, - "recall_at_1": 0.12957, - "recall_at_10": 0.34202, - "recall_at_100": 0.61572, - "recall_at_1000": 0.89676, - "recall_at_20": 0.41602, - "recall_at_3": 0.21605, - "recall_at_5": 0.264 - }, - { - "cv_recall_at_1": 0.62686, - "cv_recall_at_10": 0.90668, - "cv_recall_at_100": 0.98976, - "cv_recall_at_1000": 0.99958, - "cv_recall_at_20": 0.94583, - "cv_recall_at_3": 0.79006, - "cv_recall_at_5": 0.8462, - "hf_subset": "id", - "languages": [ - "ind-Latn" - ], - "main_score": 0.76524, - "map_at_1": 0.62686, - "map_at_10": 0.72007, - "map_at_100": 0.72415, - "map_at_1000": 0.72421, - "map_at_20": 0.72287, - "map_at_3": 0.69894, - "map_at_5": 0.71199, - "mrr_at_1": 0.6268593881560482, - "mrr_at_10": 0.7200742084653106, - "mrr_at_100": 0.7241543067524624, - "mrr_at_1000": 0.7242108276755566, - "mrr_at_20": 0.7228741256671652, - "mrr_at_3": 0.6989428384320333, - "mrr_at_5": 0.7119866217606866, - "nauc_cv_recall_at_1000_diff1": 0.8078060515091644, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 0.9074064093008918, - "nauc_cv_recall_at_100_diff1": 0.6432568018093953, - "nauc_cv_recall_at_100_max": 0.8536611954812929, - "nauc_cv_recall_at_100_std": 0.6027490773953507, - "nauc_cv_recall_at_10_diff1": 0.6657814723494743, - "nauc_cv_recall_at_10_max": 0.6957225433890553, - "nauc_cv_recall_at_10_std": 0.0022072977408623567, - "nauc_cv_recall_at_1_diff1": 0.7792706869820992, - "nauc_cv_recall_at_1_max": 0.5719054031807366, - "nauc_cv_recall_at_1_std": -0.09248351745761402, - "nauc_cv_recall_at_20_diff1": 0.6667407251770318, - "nauc_cv_recall_at_20_max": 0.7637506812219734, - "nauc_cv_recall_at_20_std": 0.11928232129370707, - "nauc_cv_recall_at_3_diff1": 0.6908769200523394, - "nauc_cv_recall_at_3_max": 0.6350382272098436, - "nauc_cv_recall_at_3_std": -0.08527690593618807, - "nauc_cv_recall_at_5_diff1": 0.6860261523249013, - "nauc_cv_recall_at_5_max": 0.6741805654950536, - "nauc_cv_recall_at_5_std": -0.03915985834128877, - "nauc_map_at_1000_diff1": 0.7512516676979538, - "nauc_map_at_1000_max": 0.5969291195432564, - "nauc_map_at_1000_std": -0.08207223388574669, - "nauc_map_at_100_diff1": 0.7512142978388329, - "nauc_map_at_100_max": 0.5969853338639733, - "nauc_map_at_100_std": -0.08191885206401718, - "nauc_map_at_10_diff1": 0.7497104407031654, - "nauc_map_at_10_max": 0.5977098586582251, - "nauc_map_at_10_std": -0.08277371394360927, - "nauc_map_at_1_diff1": 0.7792706869820992, - "nauc_map_at_1_max": 0.5719054031807366, - "nauc_map_at_1_std": -0.09248351745761402, - "nauc_map_at_20_diff1": 0.7507293057842352, - "nauc_map_at_20_max": 0.5976806632240285, - "nauc_map_at_20_std": -0.08176207226315944, - "nauc_map_at_3_diff1": 0.7473253850027027, - "nauc_map_at_3_max": 0.595246565048443, - "nauc_map_at_3_std": -0.08948415715129338, - "nauc_map_at_5_diff1": 0.7490489669338255, - "nauc_map_at_5_max": 0.5987530936259463, - "nauc_map_at_5_std": -0.08367155879418912, - "nauc_mrr_at_1000_diff1": 0.7512516676979538, - "nauc_mrr_at_1000_max": 0.5969291195432564, - "nauc_mrr_at_1000_std": -0.08207223388574669, - "nauc_mrr_at_100_diff1": 0.7512142978388329, - "nauc_mrr_at_100_max": 0.5969853338639733, - "nauc_mrr_at_100_std": -0.08191885206401718, - "nauc_mrr_at_10_diff1": 0.7497104407031654, - "nauc_mrr_at_10_max": 0.5977098586582251, - "nauc_mrr_at_10_std": -0.08277371394360927, - "nauc_mrr_at_1_diff1": 0.7792706869820992, - "nauc_mrr_at_1_max": 0.5719054031807366, - "nauc_mrr_at_1_std": -0.09248351745761402, - "nauc_mrr_at_20_diff1": 0.7507293057842352, - "nauc_mrr_at_20_max": 0.5976806632240285, - "nauc_mrr_at_20_std": -0.08176207226315944, - "nauc_mrr_at_3_diff1": 0.7473253850027027, - "nauc_mrr_at_3_max": 0.595246565048443, - "nauc_mrr_at_3_std": -0.08948415715129338, - "nauc_mrr_at_5_diff1": 0.7490489669338255, - "nauc_mrr_at_5_max": 0.5987530936259463, - "nauc_mrr_at_5_std": -0.08367155879418912, - "nauc_ndcg_at_1000_diff1": 0.7472767756637274, - "nauc_ndcg_at_1000_max": 0.6023894445439884, - "nauc_ndcg_at_1000_std": -0.07606050765907078, - "nauc_ndcg_at_100_diff1": 0.746430499512041, - "nauc_ndcg_at_100_max": 0.6040946792686888, - "nauc_ndcg_at_100_std": -0.07141106721599925, - "nauc_ndcg_at_10_diff1": 0.7391483459861079, - "nauc_ndcg_at_10_max": 0.6095221940049876, - "nauc_ndcg_at_10_std": -0.07343255503981704, - "nauc_ndcg_at_1_diff1": 0.7792706869820992, - "nauc_ndcg_at_1_max": 0.5719054031807366, - "nauc_ndcg_at_1_std": -0.09248351745761402, - "nauc_ndcg_at_20_diff1": 0.7430368326541804, - "nauc_ndcg_at_20_max": 0.6099604556357914, - "nauc_ndcg_at_20_std": -0.0687887602859378, - "nauc_ndcg_at_3_diff1": 0.735431504924604, - "nauc_ndcg_at_3_max": 0.6036553110308045, - "nauc_ndcg_at_3_std": -0.08857924253599371, - "nauc_ndcg_at_5_diff1": 0.7379351281643888, - "nauc_ndcg_at_5_max": 0.6115526644589184, - "nauc_ndcg_at_5_std": -0.07649475310316721, - "nauc_precision_at_1000_diff1": 0.8078060515090997, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 0.9074064093009346, - "nauc_precision_at_100_diff1": 0.643256801809399, - "nauc_precision_at_100_max": 0.8536611954813077, - "nauc_precision_at_100_std": 0.6027490773953564, - "nauc_precision_at_10_diff1": 0.665781472349474, - "nauc_precision_at_10_max": 0.6957225433890558, - "nauc_precision_at_10_std": 0.002207297740855719, - "nauc_precision_at_1_diff1": 0.7792706869820992, - "nauc_precision_at_1_max": 0.5719054031807366, - "nauc_precision_at_1_std": -0.09248351745761402, - "nauc_precision_at_20_diff1": 0.6667407251770338, - "nauc_precision_at_20_max": 0.7637506812219743, - "nauc_precision_at_20_std": 0.11928232129370485, - "nauc_precision_at_3_diff1": 0.690876920052339, - "nauc_precision_at_3_max": 0.6350382272098447, - "nauc_precision_at_3_std": -0.08527690593618771, - "nauc_precision_at_5_diff1": 0.6860261523249014, - "nauc_precision_at_5_max": 0.6741805654950541, - "nauc_precision_at_5_std": -0.03915985834128767, - "nauc_recall_at_1000_diff1": 0.8078060515091644, - "nauc_recall_at_1000_max": 1.0, - "nauc_recall_at_1000_std": 0.9074064093008918, - "nauc_recall_at_100_diff1": 0.6432568018093953, - "nauc_recall_at_100_max": 0.8536611954812929, - "nauc_recall_at_100_std": 0.6027490773953507, - "nauc_recall_at_10_diff1": 0.6657814723494743, - "nauc_recall_at_10_max": 0.6957225433890553, - "nauc_recall_at_10_std": 0.0022072977408623567, - "nauc_recall_at_1_diff1": 0.7792706869820992, - "nauc_recall_at_1_max": 0.5719054031807366, - "nauc_recall_at_1_std": -0.09248351745761402, - "nauc_recall_at_20_diff1": 0.6667407251770318, - "nauc_recall_at_20_max": 0.7637506812219734, - "nauc_recall_at_20_std": 0.11928232129370707, - "nauc_recall_at_3_diff1": 0.6908769200523394, - "nauc_recall_at_3_max": 0.6350382272098436, - "nauc_recall_at_3_std": -0.08527690593618807, - "nauc_recall_at_5_diff1": 0.6860261523249013, - "nauc_recall_at_5_max": 0.6741805654950536, - "nauc_recall_at_5_std": -0.03915985834128877, - "ndcg_at_1": 0.62686, - "ndcg_at_10": 0.76524, - "ndcg_at_100": 0.78368, - "ndcg_at_1000": 0.78498, - "ndcg_at_20": 0.77524, - "ndcg_at_3": 0.72235, - "ndcg_at_5": 0.74566, - "precision_at_1": 0.62686, - "precision_at_10": 0.09067, - "precision_at_100": 0.0099, - "precision_at_1000": 0.001, - "precision_at_20": 0.04729, - "precision_at_3": 0.26335, - "precision_at_5": 0.16924, - "recall_at_1": 0.62686, - "recall_at_10": 0.90668, - "recall_at_100": 0.98976, - "recall_at_1000": 0.99958, - "recall_at_20": 0.94583, - "recall_at_3": 0.79006, - "recall_at_5": 0.8462 - }, - { - "cv_recall_at_1": 0.67631, - "cv_recall_at_10": 0.93873, - "cv_recall_at_100": 0.99469, - "cv_recall_at_1000": 0.99988, - "cv_recall_at_20": 0.96777, - "cv_recall_at_3": 0.8391, - "cv_recall_at_5": 0.89045, - "hf_subset": "it", - "languages": [ - "ita-Latn" - ], - "main_score": 0.8093, - "map_at_1": 0.67631, - "map_at_10": 0.76757, - "map_at_100": 0.77043, - "map_at_1000": 0.77046, - "map_at_20": 0.76965, - "map_at_3": 0.7491, - "map_at_5": 0.76093, - "mrr_at_1": 0.6763074017235273, - "mrr_at_10": 0.7675706846701964, - "mrr_at_100": 0.7704339580269138, - "mrr_at_1000": 0.7704636853468644, - "mrr_at_20": 0.7696493226681896, - "mrr_at_3": 0.7491047888875737, - "mrr_at_5": 0.7609274780624032, - "nauc_cv_recall_at_1000_diff1": 0.7222282767327224, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": 0.6081278002284253, - "nauc_cv_recall_at_100_max": 0.8488622992394228, - "nauc_cv_recall_at_100_std": 0.5571362488972622, - "nauc_cv_recall_at_10_diff1": 0.6628892757554061, - "nauc_cv_recall_at_10_max": 0.6565974764765896, - "nauc_cv_recall_at_10_std": -0.08043915108181712, - "nauc_cv_recall_at_1_diff1": 0.7623652046686186, - "nauc_cv_recall_at_1_max": 0.4681979463974185, - "nauc_cv_recall_at_1_std": -0.21274061728203167, - "nauc_cv_recall_at_20_diff1": 0.6662127485138235, - "nauc_cv_recall_at_20_max": 0.7161591219904855, - "nauc_cv_recall_at_20_std": 0.018820578206631276, - "nauc_cv_recall_at_3_diff1": 0.6943259930732723, - "nauc_cv_recall_at_3_max": 0.5333482867271512, - "nauc_cv_recall_at_3_std": -0.22973946875310663, - "nauc_cv_recall_at_5_diff1": 0.6836306948389111, - "nauc_cv_recall_at_5_max": 0.5859049187975784, - "nauc_cv_recall_at_5_std": -0.16199264140836703, - "nauc_map_at_1000_diff1": 0.7411736126116106, - "nauc_map_at_1000_max": 0.4917828756376569, - "nauc_map_at_1000_std": -0.20683083313104692, - "nauc_map_at_100_diff1": 0.7411562255890868, - "nauc_map_at_100_max": 0.4918349144235231, - "nauc_map_at_100_std": -0.20672737625036824, - "nauc_map_at_10_diff1": 0.7400330232348077, - "nauc_map_at_10_max": 0.49357475828973923, - "nauc_map_at_10_std": -0.20606748745004916, - "nauc_map_at_1_diff1": 0.7623652046686186, - "nauc_map_at_1_max": 0.4681979463974185, - "nauc_map_at_1_std": -0.21274061728203167, - "nauc_map_at_20_diff1": 0.7408903405171129, - "nauc_map_at_20_max": 0.49254548529020414, - "nauc_map_at_20_std": -0.20637267290486805, - "nauc_map_at_3_diff1": 0.7382633309068931, - "nauc_map_at_3_max": 0.4892418217193733, - "nauc_map_at_3_std": -0.21665425397028518, - "nauc_map_at_5_diff1": 0.7391457643885568, - "nauc_map_at_5_max": 0.4935053812177111, - "nauc_map_at_5_std": -0.2079626656780284, - "nauc_mrr_at_1000_diff1": 0.7411736126116106, - "nauc_mrr_at_1000_max": 0.4917828756376569, - "nauc_mrr_at_1000_std": -0.20683083313104692, - "nauc_mrr_at_100_diff1": 0.7411562255890868, - "nauc_mrr_at_100_max": 0.4918349144235231, - "nauc_mrr_at_100_std": -0.20672737625036824, - "nauc_mrr_at_10_diff1": 0.7400330232348077, - "nauc_mrr_at_10_max": 0.49357475828973923, - "nauc_mrr_at_10_std": -0.20606748745004916, - "nauc_mrr_at_1_diff1": 0.7623652046686186, - "nauc_mrr_at_1_max": 0.4681979463974185, - "nauc_mrr_at_1_std": -0.21274061728203167, - "nauc_mrr_at_20_diff1": 0.7408903405171129, - "nauc_mrr_at_20_max": 0.49254548529020414, - "nauc_mrr_at_20_std": -0.20637267290486805, - "nauc_mrr_at_3_diff1": 0.7382633309068931, - "nauc_mrr_at_3_max": 0.4892418217193733, - "nauc_mrr_at_3_std": -0.21665425397028518, - "nauc_mrr_at_5_diff1": 0.7391457643885568, - "nauc_mrr_at_5_max": 0.4935053812177111, - "nauc_mrr_at_5_std": -0.2079626656780284, - "nauc_ndcg_at_1000_diff1": 0.7382481720449412, - "nauc_ndcg_at_1000_max": 0.4972038613490746, - "nauc_ndcg_at_1000_std": -0.20261425964460525, - "nauc_ndcg_at_100_diff1": 0.7376880004808867, - "nauc_ndcg_at_100_max": 0.4987282147380828, - "nauc_ndcg_at_100_std": -0.1993796792055653, - "nauc_ndcg_at_10_diff1": 0.7323219789374692, - "nauc_ndcg_at_10_max": 0.508551124757385, - "nauc_ndcg_at_10_std": -0.19555820350978895, - "nauc_ndcg_at_1_diff1": 0.7623652046686186, - "nauc_ndcg_at_1_max": 0.4681979463974185, - "nauc_ndcg_at_1_std": -0.21274061728203167, - "nauc_ndcg_at_20_diff1": 0.7357179194450744, - "nauc_ndcg_at_20_max": 0.5047377774302884, - "nauc_ndcg_at_20_std": -0.19602191052106646, - "nauc_ndcg_at_3_diff1": 0.7295970107460835, - "nauc_ndcg_at_3_max": 0.4978410741544815, - "nauc_ndcg_at_3_std": -0.21913117291610995, - "nauc_ndcg_at_5_diff1": 0.7305310266263622, - "nauc_ndcg_at_5_max": 0.5072305632059156, - "nauc_ndcg_at_5_std": -0.20161380941344245, - "nauc_precision_at_1000_diff1": 0.7222282767317681, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.6081278002284122, - "nauc_precision_at_100_max": 0.8488622992394487, - "nauc_precision_at_100_std": 0.5571362488972942, - "nauc_precision_at_10_diff1": 0.6628892757554079, - "nauc_precision_at_10_max": 0.6565974764765872, - "nauc_precision_at_10_std": -0.08043915108182106, - "nauc_precision_at_1_diff1": 0.7623652046686186, - "nauc_precision_at_1_max": 0.4681979463974185, - "nauc_precision_at_1_std": -0.21274061728203167, - "nauc_precision_at_20_diff1": 0.6662127485138202, - "nauc_precision_at_20_max": 0.7161591219904893, - "nauc_precision_at_20_std": 0.018820578206631296, - "nauc_precision_at_3_diff1": 0.6943259930732729, - "nauc_precision_at_3_max": 0.5333482867271515, - "nauc_precision_at_3_std": -0.22973946875310539, - "nauc_precision_at_5_diff1": 0.6836306948389095, - "nauc_precision_at_5_max": 0.5859049187975767, - "nauc_precision_at_5_std": -0.16199264140836858, - "nauc_recall_at_1000_diff1": 0.7222282767327224, - "nauc_recall_at_1000_max": 1.0, - "nauc_recall_at_1000_std": 1.0, - "nauc_recall_at_100_diff1": 0.6081278002284253, - "nauc_recall_at_100_max": 0.8488622992394228, - "nauc_recall_at_100_std": 0.5571362488972622, - "nauc_recall_at_10_diff1": 0.6628892757554061, - "nauc_recall_at_10_max": 0.6565974764765896, - "nauc_recall_at_10_std": -0.08043915108181712, - "nauc_recall_at_1_diff1": 0.7623652046686186, - "nauc_recall_at_1_max": 0.4681979463974185, - "nauc_recall_at_1_std": -0.21274061728203167, - "nauc_recall_at_20_diff1": 0.6662127485138235, - "nauc_recall_at_20_max": 0.7161591219904855, - "nauc_recall_at_20_std": 0.018820578206631276, - "nauc_recall_at_3_diff1": 0.6943259930732723, - "nauc_recall_at_3_max": 0.5333482867271512, - "nauc_recall_at_3_std": -0.22973946875310663, - "nauc_recall_at_5_diff1": 0.6836306948389111, - "nauc_recall_at_5_max": 0.5859049187975784, - "nauc_recall_at_5_std": -0.16199264140836703, - "ndcg_at_1": 0.67631, - "ndcg_at_10": 0.8093, - "ndcg_at_100": 0.82191, - "ndcg_at_1000": 0.82259, - "ndcg_at_20": 0.81672, - "ndcg_at_3": 0.77226, - "ndcg_at_5": 0.79349, - "precision_at_1": 0.67631, - "precision_at_10": 0.09387, - "precision_at_100": 0.00995, - "precision_at_1000": 0.001, - "precision_at_20": 0.04839, - "precision_at_3": 0.2797, - "precision_at_5": 0.17809, - "recall_at_1": 0.67631, - "recall_at_10": 0.93873, - "recall_at_100": 0.99469, - "recall_at_1000": 0.99988, - "recall_at_20": 0.96777, - "recall_at_3": 0.8391, - "recall_at_5": 0.89045 - }, - { - "cv_recall_at_1": 0.6366, - "cv_recall_at_10": 0.91496, - "cv_recall_at_100": 0.99165, - "cv_recall_at_1000": 0.9993, - "cv_recall_at_20": 0.95254, - "cv_recall_at_3": 0.80264, - "cv_recall_at_5": 0.8611, - "hf_subset": "ja", - "languages": [ - "jpn-Jpan" - ], - "main_score": 0.77708, - "map_at_1": 0.6366, - "map_at_10": 0.7328, - "map_at_100": 0.73663, - "map_at_1000": 0.73667, - "map_at_20": 0.7355, - "map_at_3": 0.71211, - "map_at_5": 0.72556, - "mrr_at_1": 0.6366040361864996, - "mrr_at_10": 0.7327972186322893, - "mrr_at_100": 0.736627017488815, - "mrr_at_1000": 0.7366699895317544, - "mrr_at_20": 0.7355008701473986, - "mrr_at_3": 0.7121085594989554, - "mrr_at_5": 0.7255601948503794, - "nauc_cv_recall_at_1000_diff1": 0.4870112277730652, - "nauc_cv_recall_at_1000_max": 0.9738582304731981, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": 0.7360194174964291, - "nauc_cv_recall_at_100_max": 0.8283919638695952, - "nauc_cv_recall_at_100_std": 0.6650081686379116, - "nauc_cv_recall_at_10_diff1": 0.6654660906057306, - "nauc_cv_recall_at_10_max": 0.5986711490902606, - "nauc_cv_recall_at_10_std": -0.044115384650931296, - "nauc_cv_recall_at_1_diff1": 0.7705764177641267, - "nauc_cv_recall_at_1_max": 0.46467207153346846, - "nauc_cv_recall_at_1_std": -0.15797317602015953, - "nauc_cv_recall_at_20_diff1": 0.6725663450558479, - "nauc_cv_recall_at_20_max": 0.671566686556366, - "nauc_cv_recall_at_20_std": 0.1500791693559291, - "nauc_cv_recall_at_3_diff1": 0.682922952048957, - "nauc_cv_recall_at_3_max": 0.5255304200904859, - "nauc_cv_recall_at_3_std": -0.14608349755474395, - "nauc_cv_recall_at_5_diff1": 0.6707935677999222, - "nauc_cv_recall_at_5_max": 0.5559591575751662, - "nauc_cv_recall_at_5_std": -0.10345018603810813, - "nauc_map_at_1000_diff1": 0.7441881259546695, - "nauc_map_at_1000_max": 0.48932443569077916, - "nauc_map_at_1000_std": -0.14707622380005525, - "nauc_map_at_100_diff1": 0.7441879554181916, - "nauc_map_at_100_max": 0.489378471692625, - "nauc_map_at_100_std": -0.14693806269457682, - "nauc_map_at_10_diff1": 0.7427115484863952, - "nauc_map_at_10_max": 0.4901723060092325, - "nauc_map_at_10_std": -0.1479163009747065, - "nauc_map_at_1_diff1": 0.7705764177641267, - "nauc_map_at_1_max": 0.46467207153346846, - "nauc_map_at_1_std": -0.15797317602015953, - "nauc_map_at_20_diff1": 0.7437556217723685, - "nauc_map_at_20_max": 0.49013412945893137, - "nauc_map_at_20_std": -0.14605028470717732, - "nauc_map_at_3_diff1": 0.739929554116238, - "nauc_map_at_3_max": 0.4878653280639865, - "nauc_map_at_3_std": -0.15494221973189937, - "nauc_map_at_5_diff1": 0.7410872828829704, - "nauc_map_at_5_max": 0.48992805135002115, - "nauc_map_at_5_std": -0.1497645067760917, - "nauc_mrr_at_1000_diff1": 0.7441881259546695, - "nauc_mrr_at_1000_max": 0.48932443569077916, - "nauc_mrr_at_1000_std": -0.14707622380005525, - "nauc_mrr_at_100_diff1": 0.7441879554181916, - "nauc_mrr_at_100_max": 0.489378471692625, - "nauc_mrr_at_100_std": -0.14693806269457682, - "nauc_mrr_at_10_diff1": 0.7427115484863952, - "nauc_mrr_at_10_max": 0.4901723060092325, - "nauc_mrr_at_10_std": -0.1479163009747065, - "nauc_mrr_at_1_diff1": 0.7705764177641267, - "nauc_mrr_at_1_max": 0.46467207153346846, - "nauc_mrr_at_1_std": -0.15797317602015953, - "nauc_mrr_at_20_diff1": 0.7437556217723685, - "nauc_mrr_at_20_max": 0.49013412945893137, - "nauc_mrr_at_20_std": -0.14605028470717732, - "nauc_mrr_at_3_diff1": 0.739929554116238, - "nauc_mrr_at_3_max": 0.4878653280639865, - "nauc_mrr_at_3_std": -0.15494221973189937, - "nauc_mrr_at_5_diff1": 0.7410872828829704, - "nauc_mrr_at_5_max": 0.48992805135002115, - "nauc_mrr_at_5_std": -0.1497645067760917, - "nauc_ndcg_at_1000_diff1": 0.740475731829745, - "nauc_ndcg_at_1000_max": 0.4947914358930953, - "nauc_ndcg_at_1000_std": -0.14023034742460014, - "nauc_ndcg_at_100_diff1": 0.7405599849365426, - "nauc_ndcg_at_100_max": 0.49657575192060943, - "nauc_ndcg_at_100_std": -0.13586349043153753, - "nauc_ndcg_at_10_diff1": 0.7331826599258588, - "nauc_ndcg_at_10_max": 0.5023841859205426, - "nauc_ndcg_at_10_std": -0.13705141672240842, - "nauc_ndcg_at_1_diff1": 0.7705764177641267, - "nauc_ndcg_at_1_max": 0.46467207153346846, - "nauc_ndcg_at_1_std": -0.15797317602015953, - "nauc_ndcg_at_20_diff1": 0.7372168346745342, - "nauc_ndcg_at_20_max": 0.5024041914908172, - "nauc_ndcg_at_20_std": -0.12880153641013473, - "nauc_ndcg_at_3_diff1": 0.728071926540345, - "nauc_ndcg_at_3_max": 0.495789424094431, - "nauc_ndcg_at_3_std": -0.15315224992624585, - "nauc_ndcg_at_5_diff1": 0.7293058439381539, - "nauc_ndcg_at_5_max": 0.5008014824465146, - "nauc_ndcg_at_5_std": -0.14255901932569748, - "nauc_precision_at_1000_diff1": 0.4870112277732604, - "nauc_precision_at_1000_max": 0.9738582304732849, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.7360194174964301, - "nauc_precision_at_100_max": 0.8283919638695951, - "nauc_precision_at_100_std": 0.6650081686379157, - "nauc_precision_at_10_diff1": 0.6654660906057299, - "nauc_precision_at_10_max": 0.5986711490902608, - "nauc_precision_at_10_std": -0.044115384650932864, - "nauc_precision_at_1_diff1": 0.7705764177641267, - "nauc_precision_at_1_max": 0.46467207153346846, - "nauc_precision_at_1_std": -0.15797317602015953, - "nauc_precision_at_20_diff1": 0.672566345055847, - "nauc_precision_at_20_max": 0.671566686556363, - "nauc_precision_at_20_std": 0.15007916935592425, - "nauc_precision_at_3_diff1": 0.6829229520489568, - "nauc_precision_at_3_max": 0.5255304200904857, - "nauc_precision_at_3_std": -0.14608349755474373, - "nauc_precision_at_5_diff1": 0.6707935677999219, - "nauc_precision_at_5_max": 0.5559591575751665, - "nauc_precision_at_5_std": -0.10345018603810868, - "nauc_recall_at_1000_diff1": 0.4870112277730652, - "nauc_recall_at_1000_max": 0.9738582304731981, - "nauc_recall_at_1000_std": 1.0, - "nauc_recall_at_100_diff1": 0.7360194174964291, - "nauc_recall_at_100_max": 0.8283919638695952, - "nauc_recall_at_100_std": 0.6650081686379116, - "nauc_recall_at_10_diff1": 0.6654660906057306, - "nauc_recall_at_10_max": 0.5986711490902606, - "nauc_recall_at_10_std": -0.044115384650931296, - "nauc_recall_at_1_diff1": 0.7705764177641267, - "nauc_recall_at_1_max": 0.46467207153346846, - "nauc_recall_at_1_std": -0.15797317602015953, - "nauc_recall_at_20_diff1": 0.6725663450558479, - "nauc_recall_at_20_max": 0.671566686556366, - "nauc_recall_at_20_std": 0.1500791693559291, - "nauc_recall_at_3_diff1": 0.682922952048957, - "nauc_recall_at_3_max": 0.5255304200904859, - "nauc_recall_at_3_std": -0.14608349755474395, - "nauc_recall_at_5_diff1": 0.6707935677999222, - "nauc_recall_at_5_max": 0.5559591575751662, - "nauc_recall_at_5_std": -0.10345018603810813, - "ndcg_at_1": 0.6366, - "ndcg_at_10": 0.77708, - "ndcg_at_100": 0.7942, - "ndcg_at_1000": 0.79521, - "ndcg_at_20": 0.7867, - "ndcg_at_3": 0.73546, - "ndcg_at_5": 0.75962, - "precision_at_1": 0.6366, - "precision_at_10": 0.0915, - "precision_at_100": 0.00992, - "precision_at_1000": 0.001, - "precision_at_20": 0.04763, - "precision_at_3": 0.26755, - "precision_at_5": 0.17222, - "recall_at_1": 0.6366, - "recall_at_10": 0.91496, - "recall_at_100": 0.99165, - "recall_at_1000": 0.9993, - "recall_at_20": 0.95254, - "recall_at_3": 0.80264, - "recall_at_5": 0.8611 - }, - { - "cv_recall_at_1": 0.32824, - "cv_recall_at_10": 0.66092, - "cv_recall_at_100": 0.90157, - "cv_recall_at_1000": 0.98967, - "cv_recall_at_20": 0.74654, - "cv_recall_at_3": 0.49673, - "cv_recall_at_5": 0.5698, - "hf_subset": "ko", - "languages": [ - "kor-Hang" - ], - "main_score": 0.48615, - "map_at_1": 0.32824, - "map_at_10": 0.43119, - "map_at_100": 0.44096, - "map_at_1000": 0.44138, - "map_at_20": 0.43704, - "map_at_3": 0.40227, - "map_at_5": 0.41891, - "mrr_at_1": 0.32823529411764707, - "mrr_at_10": 0.4311886087768451, - "mrr_at_100": 0.44095616656130077, - "mrr_at_1000": 0.44137718862191744, - "mrr_at_20": 0.43703863946848, - "mrr_at_3": 0.4022657952069736, - "mrr_at_5": 0.4189128540305019, - "nauc_cv_recall_at_1000_diff1": 0.31606566677303044, - "nauc_cv_recall_at_1000_max": 0.7567280076587494, - "nauc_cv_recall_at_1000_std": 0.8719876136108337, - "nauc_cv_recall_at_100_diff1": 0.36533294149886736, - "nauc_cv_recall_at_100_max": 0.6483074858982988, - "nauc_cv_recall_at_100_std": 0.6143704050898802, - "nauc_cv_recall_at_10_diff1": 0.41043823786876577, - "nauc_cv_recall_at_10_max": 0.501184849572785, - "nauc_cv_recall_at_10_std": 0.24836789796558764, - "nauc_cv_recall_at_1_diff1": 0.5724479741203551, - "nauc_cv_recall_at_1_max": 0.4093246568520635, - "nauc_cv_recall_at_1_std": 0.08187266516484734, - "nauc_cv_recall_at_20_diff1": 0.4008308929393305, - "nauc_cv_recall_at_20_max": 0.5438192664450267, - "nauc_cv_recall_at_20_std": 0.34315894629094834, - "nauc_cv_recall_at_3_diff1": 0.4557066058908472, - "nauc_cv_recall_at_3_max": 0.4461926964630562, - "nauc_cv_recall_at_3_std": 0.15308129925712008, - "nauc_cv_recall_at_5_diff1": 0.4327341488634024, - "nauc_cv_recall_at_5_max": 0.47990161573683054, - "nauc_cv_recall_at_5_std": 0.19948489678898848, - "nauc_map_at_1000_diff1": 0.5148975188038908, - "nauc_map_at_1000_max": 0.4355824398849164, - "nauc_map_at_1000_std": 0.13065522167678348, - "nauc_map_at_100_diff1": 0.514698319934614, - "nauc_map_at_100_max": 0.4357570122065423, - "nauc_map_at_100_std": 0.1311025115276141, - "nauc_map_at_10_diff1": 0.513418485624799, - "nauc_map_at_10_max": 0.43463205566792706, - "nauc_map_at_10_std": 0.12744454519859516, - "nauc_map_at_1_diff1": 0.5724479741203551, - "nauc_map_at_1_max": 0.4093246568520635, - "nauc_map_at_1_std": 0.08187266516484734, - "nauc_map_at_20_diff1": 0.5140149448972925, - "nauc_map_at_20_max": 0.43588645221616207, - "nauc_map_at_20_std": 0.13058737794323919, - "nauc_map_at_3_diff1": 0.5189536861901586, - "nauc_map_at_3_max": 0.4260398579272175, - "nauc_map_at_3_std": 0.11384666123814757, - "nauc_map_at_5_diff1": 0.5147617743556974, - "nauc_map_at_5_max": 0.4329202598408649, - "nauc_map_at_5_std": 0.12324129047164137, - "nauc_mrr_at_1000_diff1": 0.5148975188038908, - "nauc_mrr_at_1000_max": 0.4355824398849164, - "nauc_mrr_at_1000_std": 0.13065522167678348, - "nauc_mrr_at_100_diff1": 0.514698319934614, - "nauc_mrr_at_100_max": 0.4357570122065423, - "nauc_mrr_at_100_std": 0.1311025115276141, - "nauc_mrr_at_10_diff1": 0.513418485624799, - "nauc_mrr_at_10_max": 0.43463205566792706, - "nauc_mrr_at_10_std": 0.12744454519859516, - "nauc_mrr_at_1_diff1": 0.5724479741203551, - "nauc_mrr_at_1_max": 0.4093246568520635, - "nauc_mrr_at_1_std": 0.08187266516484734, - "nauc_mrr_at_20_diff1": 0.5140149448972925, - "nauc_mrr_at_20_max": 0.43588645221616207, - "nauc_mrr_at_20_std": 0.13058737794323919, - "nauc_mrr_at_3_diff1": 0.5189536861901586, - "nauc_mrr_at_3_max": 0.4260398579272175, - "nauc_mrr_at_3_std": 0.11384666123814757, - "nauc_mrr_at_5_diff1": 0.5147617743556974, - "nauc_mrr_at_5_max": 0.4329202598408649, - "nauc_mrr_at_5_std": 0.12324129047164137, - "nauc_ndcg_at_1000_diff1": 0.5028633000738268, - "nauc_ndcg_at_1000_max": 0.4467397582711455, - "nauc_ndcg_at_1000_std": 0.15358645176436325, - "nauc_ndcg_at_100_diff1": 0.49762609351535697, - "nauc_ndcg_at_100_max": 0.4535299823361077, - "nauc_ndcg_at_100_std": 0.16943724112556677, - "nauc_ndcg_at_10_diff1": 0.491020691200883, - "nauc_ndcg_at_10_max": 0.4485124450008812, - "nauc_ndcg_at_10_std": 0.15250408970563734, - "nauc_ndcg_at_1_diff1": 0.5724479741203551, - "nauc_ndcg_at_1_max": 0.4093246568520635, - "nauc_ndcg_at_1_std": 0.08187266516484734, - "nauc_ndcg_at_20_diff1": 0.49270587037633035, - "nauc_ndcg_at_20_max": 0.453954581379322, - "nauc_ndcg_at_20_std": 0.16561882754616208, - "nauc_ndcg_at_3_diff1": 0.5026632707669226, - "nauc_ndcg_at_3_max": 0.431222781113809, - "nauc_ndcg_at_3_std": 0.12392363092639624, - "nauc_ndcg_at_5_diff1": 0.49479976602682435, - "nauc_ndcg_at_5_max": 0.4439816644355392, - "nauc_ndcg_at_5_std": 0.1413280531203506, - "nauc_precision_at_1000_diff1": 0.31606566677303394, - "nauc_precision_at_1000_max": 0.7567280076587406, - "nauc_precision_at_1000_std": 0.8719876136108287, - "nauc_precision_at_100_diff1": 0.3653329414988664, - "nauc_precision_at_100_max": 0.6483074858983007, - "nauc_precision_at_100_std": 0.6143704050898817, - "nauc_precision_at_10_diff1": 0.41043823786876493, - "nauc_precision_at_10_max": 0.5011848495727845, - "nauc_precision_at_10_std": 0.2483678979655874, - "nauc_precision_at_1_diff1": 0.5724479741203551, - "nauc_precision_at_1_max": 0.4093246568520635, - "nauc_precision_at_1_std": 0.08187266516484734, - "nauc_precision_at_20_diff1": 0.4008308929393299, - "nauc_precision_at_20_max": 0.5438192664450257, - "nauc_precision_at_20_std": 0.3431589462909468, - "nauc_precision_at_3_diff1": 0.4557066058908471, - "nauc_precision_at_3_max": 0.44619269646305626, - "nauc_precision_at_3_std": 0.15308129925712013, - "nauc_precision_at_5_diff1": 0.4327341488634025, - "nauc_precision_at_5_max": 0.47990161573683066, - "nauc_precision_at_5_std": 0.1994848967889882, - "nauc_recall_at_1000_diff1": 0.31606566677303044, - "nauc_recall_at_1000_max": 0.7567280076587494, - "nauc_recall_at_1000_std": 0.8719876136108337, - "nauc_recall_at_100_diff1": 0.36533294149886736, - "nauc_recall_at_100_max": 0.6483074858982988, - "nauc_recall_at_100_std": 0.6143704050898802, - "nauc_recall_at_10_diff1": 0.41043823786876577, - "nauc_recall_at_10_max": 0.501184849572785, - "nauc_recall_at_10_std": 0.24836789796558764, - "nauc_recall_at_1_diff1": 0.5724479741203551, - "nauc_recall_at_1_max": 0.4093246568520635, - "nauc_recall_at_1_std": 0.08187266516484734, - "nauc_recall_at_20_diff1": 0.4008308929393305, - "nauc_recall_at_20_max": 0.5438192664450267, - "nauc_recall_at_20_std": 0.34315894629094834, - "nauc_recall_at_3_diff1": 0.4557066058908472, - "nauc_recall_at_3_max": 0.4461926964630562, - "nauc_recall_at_3_std": 0.15308129925712008, - "nauc_recall_at_5_diff1": 0.4327341488634024, - "nauc_recall_at_5_max": 0.47990161573683054, - "nauc_recall_at_5_std": 0.19948489678898848, - "ndcg_at_1": 0.32824, - "ndcg_at_10": 0.48615, - "ndcg_at_100": 0.53635, - "ndcg_at_1000": 0.54755, - "ndcg_at_20": 0.50768, - "ndcg_at_3": 0.42652, - "ndcg_at_5": 0.45657, - "precision_at_1": 0.32824, - "precision_at_10": 0.06609, - "precision_at_100": 0.00902, - "precision_at_1000": 0.00099, - "precision_at_20": 0.03733, - "precision_at_3": 0.16558, - "precision_at_5": 0.11396, - "recall_at_1": 0.32824, - "recall_at_10": 0.66092, - "recall_at_100": 0.90157, - "recall_at_1000": 0.98967, - "recall_at_20": 0.74654, - "recall_at_3": 0.49673, - "recall_at_5": 0.5698 - }, - { - "cv_recall_at_1": 0.00254, - "cv_recall_at_10": 0.01205, - "cv_recall_at_100": 0.07101, - "cv_recall_at_1000": 0.38821, - "cv_recall_at_20": 0.01902, - "cv_recall_at_3": 0.00592, - "cv_recall_at_5": 0.00824, - "hf_subset": "mi", - "languages": [ - "mri-Latn" - ], - "main_score": 0.00661, - "map_at_1": 0.00254, - "map_at_10": 0.00496, - "map_at_100": 0.00656, - "map_at_1000": 0.00746, - "map_at_20": 0.00544, - "map_at_3": 0.00394, - "map_at_5": 0.00446, - "mrr_at_1": 0.00253592561284869, - "mrr_at_10": 0.004960233600343489, - "mrr_at_100": 0.006561841673598242, - "mrr_at_1000": 0.007462404192643936, - "mrr_at_20": 0.005441873817226138, - "mrr_at_3": 0.0039447731755424065, - "mrr_at_5": 0.004462524654832346, - "nauc_cv_recall_at_1000_diff1": -0.001754921851847047, - "nauc_cv_recall_at_1000_max": 0.0813775510944442, - "nauc_cv_recall_at_1000_std": 0.1703090139503763, - "nauc_cv_recall_at_100_diff1": 0.01629770853022363, - "nauc_cv_recall_at_100_max": 0.15059830692876489, - "nauc_cv_recall_at_100_std": 0.2651021745283781, - "nauc_cv_recall_at_10_diff1": 0.16620091601208944, - "nauc_cv_recall_at_10_max": 0.44683741032627683, - "nauc_cv_recall_at_10_std": 0.5502644434148982, - "nauc_cv_recall_at_1_diff1": 0.38191644037549527, - "nauc_cv_recall_at_1_max": 0.8121549617389423, - "nauc_cv_recall_at_1_std": 0.7411893196695267, - "nauc_cv_recall_at_20_diff1": 0.09707278061979692, - "nauc_cv_recall_at_20_max": 0.3273986828212527, - "nauc_cv_recall_at_20_std": 0.47419581902757957, - "nauc_cv_recall_at_3_diff1": 0.24906706569566026, - "nauc_cv_recall_at_3_max": 0.6659004348431463, - "nauc_cv_recall_at_3_std": 0.6936494424405377, - "nauc_cv_recall_at_5_diff1": 0.22487748592748064, - "nauc_cv_recall_at_5_max": 0.5868842812113284, - "nauc_cv_recall_at_5_std": 0.638464710612695, - "nauc_map_at_1000_diff1": 0.1766150827944164, - "nauc_map_at_1000_max": 0.46607818255706707, - "nauc_map_at_1000_std": 0.515329367279344, - "nauc_map_at_100_diff1": 0.1927671971676861, - "nauc_map_at_100_max": 0.5007461423878604, - "nauc_map_at_100_std": 0.5474677908875887, - "nauc_map_at_10_diff1": 0.2611240007883676, - "nauc_map_at_10_max": 0.6300118926644409, - "nauc_map_at_10_std": 0.6460166321923634, - "nauc_map_at_1_diff1": 0.38191644037549527, - "nauc_map_at_1_max": 0.8121549617389423, - "nauc_map_at_1_std": 0.7411893196695267, - "nauc_map_at_20_diff1": 0.2374761310339923, - "nauc_map_at_20_max": 0.5851741900653458, - "nauc_map_at_20_std": 0.619475842174224, - "nauc_map_at_3_diff1": 0.2967462852583165, - "nauc_map_at_3_max": 0.7195420182534291, - "nauc_map_at_3_std": 0.7092403257642388, - "nauc_map_at_5_diff1": 0.28388313400981, - "nauc_map_at_5_max": 0.6826019446607527, - "nauc_map_at_5_std": 0.6809347447189633, - "nauc_mrr_at_1000_diff1": 0.1766150827944164, - "nauc_mrr_at_1000_max": 0.46607818255706707, - "nauc_mrr_at_1000_std": 0.515329367279344, - "nauc_mrr_at_100_diff1": 0.1927671971676861, - "nauc_mrr_at_100_max": 0.5007461423878604, - "nauc_mrr_at_100_std": 0.5474677908875887, - "nauc_mrr_at_10_diff1": 0.2611240007883676, - "nauc_mrr_at_10_max": 0.6300118926644409, - "nauc_mrr_at_10_std": 0.6460166321923634, - "nauc_mrr_at_1_diff1": 0.38191644037549527, - "nauc_mrr_at_1_max": 0.8121549617389423, - "nauc_mrr_at_1_std": 0.7411893196695267, - "nauc_mrr_at_20_diff1": 0.2374761310339923, - "nauc_mrr_at_20_max": 0.5851741900653458, - "nauc_mrr_at_20_std": 0.619475842174224, - "nauc_mrr_at_3_diff1": 0.2967462852583165, - "nauc_mrr_at_3_max": 0.7195420182534291, - "nauc_mrr_at_3_std": 0.7092403257642388, - "nauc_mrr_at_5_diff1": 0.28388313400981, - "nauc_mrr_at_5_max": 0.6826019446607527, - "nauc_mrr_at_5_std": 0.6809347447189633, - "nauc_ndcg_at_1000_diff1": 0.0322965161207195, - "nauc_ndcg_at_1000_max": 0.15979964836745464, - "nauc_ndcg_at_1000_std": 0.2455940665585908, - "nauc_ndcg_at_100_diff1": 0.0732874730991388, - "nauc_ndcg_at_100_max": 0.26639849711214036, - "nauc_ndcg_at_100_std": 0.3626258542492295, - "nauc_ndcg_at_10_diff1": 0.22212352710492203, - "nauc_ndcg_at_10_max": 0.5555024514690303, - "nauc_ndcg_at_10_std": 0.6066786205790419, - "nauc_ndcg_at_1_diff1": 0.38191644037549527, - "nauc_ndcg_at_1_max": 0.8121549617389423, - "nauc_ndcg_at_1_std": 0.7411893196695267, - "nauc_ndcg_at_20_diff1": 0.17186485474703883, - "nauc_ndcg_at_20_max": 0.46433618491655965, - "nauc_ndcg_at_20_std": 0.5514430613917896, - "nauc_ndcg_at_3_diff1": 0.28048949881588553, - "nauc_ndcg_at_3_max": 0.7012956749530924, - "nauc_ndcg_at_3_std": 0.7038690040172996, - "nauc_ndcg_at_5_diff1": 0.2618439126388269, - "nauc_ndcg_at_5_max": 0.647270534174721, - "nauc_ndcg_at_5_std": 0.6650220312022473, - "nauc_precision_at_1000_diff1": -0.0017549218518471566, - "nauc_precision_at_1000_max": 0.08137755109444397, - "nauc_precision_at_1000_std": 0.1703090139503762, - "nauc_precision_at_100_diff1": 0.016297708530223557, - "nauc_precision_at_100_max": 0.15059830692876505, - "nauc_precision_at_100_std": 0.2651021745283781, - "nauc_precision_at_10_diff1": 0.16620091601208986, - "nauc_precision_at_10_max": 0.44683741032627705, - "nauc_precision_at_10_std": 0.5502644434148986, - "nauc_precision_at_1_diff1": 0.38191644037549527, - "nauc_precision_at_1_max": 0.8121549617389423, - "nauc_precision_at_1_std": 0.7411893196695267, - "nauc_precision_at_20_diff1": 0.09707278061979728, - "nauc_precision_at_20_max": 0.32739868282125306, - "nauc_precision_at_20_std": 0.47419581902757973, - "nauc_precision_at_3_diff1": 0.24906706569565998, - "nauc_precision_at_3_max": 0.6659004348431463, - "nauc_precision_at_3_std": 0.6936494424405376, - "nauc_precision_at_5_diff1": 0.2248774859274806, - "nauc_precision_at_5_max": 0.5868842812113281, - "nauc_precision_at_5_std": 0.6384647106126947, - "nauc_recall_at_1000_diff1": -0.001754921851847047, - "nauc_recall_at_1000_max": 0.0813775510944442, - "nauc_recall_at_1000_std": 0.1703090139503763, - "nauc_recall_at_100_diff1": 0.01629770853022363, - "nauc_recall_at_100_max": 0.15059830692876489, - "nauc_recall_at_100_std": 0.2651021745283781, - "nauc_recall_at_10_diff1": 0.16620091601208944, - "nauc_recall_at_10_max": 0.44683741032627683, - "nauc_recall_at_10_std": 0.5502644434148982, - "nauc_recall_at_1_diff1": 0.38191644037549527, - "nauc_recall_at_1_max": 0.8121549617389423, - "nauc_recall_at_1_std": 0.7411893196695267, - "nauc_recall_at_20_diff1": 0.09707278061979692, - "nauc_recall_at_20_max": 0.3273986828212527, - "nauc_recall_at_20_std": 0.47419581902757957, - "nauc_recall_at_3_diff1": 0.24906706569566026, - "nauc_recall_at_3_max": 0.6659004348431463, - "nauc_recall_at_3_std": 0.6936494424405377, - "nauc_recall_at_5_diff1": 0.22487748592748064, - "nauc_recall_at_5_max": 0.5868842812113284, - "nauc_recall_at_5_std": 0.638464710612695, - "ndcg_at_1": 0.00254, - "ndcg_at_10": 0.00661, - "ndcg_at_100": 0.01758, - "ndcg_at_1000": 0.0542, - "ndcg_at_20": 0.00837, - "ndcg_at_3": 0.00445, - "ndcg_at_5": 0.00539, - "precision_at_1": 0.00254, - "precision_at_10": 0.0012, - "precision_at_100": 0.00071, - "precision_at_1000": 0.00039, - "precision_at_20": 0.00095, - "precision_at_3": 0.00197, - "precision_at_5": 0.00165, - "recall_at_1": 0.00254, - "recall_at_10": 0.01205, - "recall_at_100": 0.07101, - "recall_at_1000": 0.38821, - "recall_at_20": 0.01902, - "recall_at_3": 0.00592, - "recall_at_5": 0.00824 - }, - { - "cv_recall_at_1": 0.52488, - "cv_recall_at_10": 0.84874, - "cv_recall_at_100": 0.9732, - "cv_recall_at_1000": 0.99752, - "cv_recall_at_20": 0.90135, - "cv_recall_at_3": 0.70852, - "cv_recall_at_5": 0.77503, - "hf_subset": "nl", - "languages": [ - "nld-Latn" - ], - "main_score": 0.68389, - "map_at_1": 0.52488, - "map_at_10": 0.63139, - "map_at_100": 0.63708, - "map_at_1000": 0.63722, - "map_at_20": 0.63508, - "map_at_3": 0.60613, - "map_at_5": 0.6214, - "mrr_at_1": 0.5248790172477975, - "mrr_at_10": 0.6313939655359169, - "mrr_at_100": 0.6370779085742311, - "mrr_at_1000": 0.6372166028814777, - "mrr_at_20": 0.6350809687459603, - "mrr_at_3": 0.6061339289407283, - "mrr_at_5": 0.621396368449349, - "nauc_cv_recall_at_1000_diff1": 0.5744083171531605, - "nauc_cv_recall_at_1000_max": 0.9317200295261857, - "nauc_cv_recall_at_1000_std": 0.8472486786395506, - "nauc_cv_recall_at_100_diff1": 0.6148330048914848, - "nauc_cv_recall_at_100_max": 0.8057075120537615, - "nauc_cv_recall_at_100_std": 0.5279928320652953, - "nauc_cv_recall_at_10_diff1": 0.5678154941414436, - "nauc_cv_recall_at_10_max": 0.5581864497311916, - "nauc_cv_recall_at_10_std": 0.06369656555600225, - "nauc_cv_recall_at_1_diff1": 0.704020495398862, - "nauc_cv_recall_at_1_max": 0.43716341122428626, - "nauc_cv_recall_at_1_std": -0.08216752278968253, - "nauc_cv_recall_at_20_diff1": 0.5613842322724212, - "nauc_cv_recall_at_20_max": 0.6061462124421776, - "nauc_cv_recall_at_20_std": 0.1645691248854774, - "nauc_cv_recall_at_3_diff1": 0.6111622811567877, - "nauc_cv_recall_at_3_max": 0.4913209733378915, - "nauc_cv_recall_at_3_std": -0.05312478344784343, - "nauc_cv_recall_at_5_diff1": 0.5802937631290497, - "nauc_cv_recall_at_5_max": 0.5161516497709878, - "nauc_cv_recall_at_5_std": -0.00560283531961672, - "nauc_map_at_1000_diff1": 0.6686512103954919, - "nauc_map_at_1000_max": 0.46228001523632817, - "nauc_map_at_1000_std": -0.061803739790496234, - "nauc_map_at_100_diff1": 0.6686285521104891, - "nauc_map_at_100_max": 0.4624367851182591, - "nauc_map_at_100_std": -0.06156955965185607, - "nauc_map_at_10_diff1": 0.666741377527361, - "nauc_map_at_10_max": 0.4624129783306096, - "nauc_map_at_10_std": -0.06258848985899317, - "nauc_map_at_1_diff1": 0.704020495398862, - "nauc_map_at_1_max": 0.43716341122428626, - "nauc_map_at_1_std": -0.08216752278968253, - "nauc_map_at_20_diff1": 0.6678504831149329, - "nauc_map_at_20_max": 0.46272393004272905, - "nauc_map_at_20_std": -0.061428251168442145, - "nauc_map_at_3_diff1": 0.6678284118591633, - "nauc_map_at_3_max": 0.4581214019816706, - "nauc_map_at_3_std": -0.07277191030264109, - "nauc_map_at_5_diff1": 0.6651378761592399, - "nauc_map_at_5_max": 0.461044446009575, - "nauc_map_at_5_std": -0.06575410993244067, - "nauc_mrr_at_1000_diff1": 0.6686512103954919, - "nauc_mrr_at_1000_max": 0.46228001523632817, - "nauc_mrr_at_1000_std": -0.061803739790496234, - "nauc_mrr_at_100_diff1": 0.6686285521104891, - "nauc_mrr_at_100_max": 0.4624367851182591, - "nauc_mrr_at_100_std": -0.06156955965185607, - "nauc_mrr_at_10_diff1": 0.666741377527361, - "nauc_mrr_at_10_max": 0.4624129783306096, - "nauc_mrr_at_10_std": -0.06258848985899317, - "nauc_mrr_at_1_diff1": 0.704020495398862, - "nauc_mrr_at_1_max": 0.43716341122428626, - "nauc_mrr_at_1_std": -0.08216752278968253, - "nauc_mrr_at_20_diff1": 0.6678504831149329, - "nauc_mrr_at_20_max": 0.46272393004272905, - "nauc_mrr_at_20_std": -0.061428251168442145, - "nauc_mrr_at_3_diff1": 0.6678284118591633, - "nauc_mrr_at_3_max": 0.4581214019816706, - "nauc_mrr_at_3_std": -0.07277191030264109, - "nauc_mrr_at_5_diff1": 0.6651378761592399, - "nauc_mrr_at_5_max": 0.461044446009575, - "nauc_mrr_at_5_std": -0.06575410993244067, - "nauc_ndcg_at_1000_diff1": 0.6626036419000162, - "nauc_ndcg_at_1000_max": 0.47013464056151005, - "nauc_ndcg_at_1000_std": -0.05030975463253996, - "nauc_ndcg_at_100_diff1": 0.6620050846517636, - "nauc_ndcg_at_100_max": 0.47467890036858557, - "nauc_ndcg_at_100_std": -0.04281287475600159, - "nauc_ndcg_at_10_diff1": 0.651813517049161, - "nauc_ndcg_at_10_max": 0.4760935858527636, - "nauc_ndcg_at_10_std": -0.04531223138002542, - "nauc_ndcg_at_1_diff1": 0.704020495398862, - "nauc_ndcg_at_1_max": 0.43716341122428626, - "nauc_ndcg_at_1_std": -0.08216752278968253, - "nauc_ndcg_at_20_diff1": 0.6554275763205546, - "nauc_ndcg_at_20_max": 0.4774537795303503, - "nauc_ndcg_at_20_std": -0.03987783535410952, - "nauc_ndcg_at_3_diff1": 0.6550643982877136, - "nauc_ndcg_at_3_max": 0.46559271959208537, - "nauc_ndcg_at_3_std": -0.06843432861398872, - "nauc_ndcg_at_5_diff1": 0.6487639755957474, - "nauc_ndcg_at_5_max": 0.471639310185695, - "nauc_ndcg_at_5_std": -0.05459590263752243, - "nauc_precision_at_1000_diff1": 0.5744083171532188, - "nauc_precision_at_1000_max": 0.9317200295261956, - "nauc_precision_at_1000_std": 0.8472486786394815, - "nauc_precision_at_100_diff1": 0.6148330048914886, - "nauc_precision_at_100_max": 0.8057075120537643, - "nauc_precision_at_100_std": 0.5279928320652951, - "nauc_precision_at_10_diff1": 0.567815494141444, - "nauc_precision_at_10_max": 0.5581864497311915, - "nauc_precision_at_10_std": 0.06369656555600109, - "nauc_precision_at_1_diff1": 0.704020495398862, - "nauc_precision_at_1_max": 0.43716341122428626, - "nauc_precision_at_1_std": -0.08216752278968253, - "nauc_precision_at_20_diff1": 0.5613842322724212, - "nauc_precision_at_20_max": 0.6061462124421773, - "nauc_precision_at_20_std": 0.16456912488547504, - "nauc_precision_at_3_diff1": 0.6111622811567875, - "nauc_precision_at_3_max": 0.49132097333789176, - "nauc_precision_at_3_std": -0.05312478344784239, - "nauc_precision_at_5_diff1": 0.5802937631290486, - "nauc_precision_at_5_max": 0.5161516497709874, - "nauc_precision_at_5_std": -0.005602835319617839, - "nauc_recall_at_1000_diff1": 0.5744083171531605, - "nauc_recall_at_1000_max": 0.9317200295261857, - "nauc_recall_at_1000_std": 0.8472486786395506, - "nauc_recall_at_100_diff1": 0.6148330048914848, - "nauc_recall_at_100_max": 0.8057075120537615, - "nauc_recall_at_100_std": 0.5279928320652953, - "nauc_recall_at_10_diff1": 0.5678154941414436, - "nauc_recall_at_10_max": 0.5581864497311916, - "nauc_recall_at_10_std": 0.06369656555600225, - "nauc_recall_at_1_diff1": 0.704020495398862, - "nauc_recall_at_1_max": 0.43716341122428626, - "nauc_recall_at_1_std": -0.08216752278968253, - "nauc_recall_at_20_diff1": 0.5613842322724212, - "nauc_recall_at_20_max": 0.6061462124421776, - "nauc_recall_at_20_std": 0.1645691248854774, - "nauc_recall_at_3_diff1": 0.6111622811567877, - "nauc_recall_at_3_max": 0.4913209733378915, - "nauc_recall_at_3_std": -0.05312478344784343, - "nauc_recall_at_5_diff1": 0.5802937631290497, - "nauc_recall_at_5_max": 0.5161516497709878, - "nauc_recall_at_5_std": -0.00560283531961672, - "ndcg_at_1": 0.52488, - "ndcg_at_10": 0.68389, - "ndcg_at_100": 0.71087, - "ndcg_at_1000": 0.71408, - "ndcg_at_20": 0.69723, - "ndcg_at_3": 0.63244, - "ndcg_at_5": 0.65989, - "precision_at_1": 0.52488, - "precision_at_10": 0.08487, - "precision_at_100": 0.00973, - "precision_at_1000": 0.001, - "precision_at_20": 0.04507, - "precision_at_3": 0.23617, - "precision_at_5": 0.15501, - "recall_at_1": 0.52488, - "recall_at_10": 0.84874, - "recall_at_100": 0.9732, - "recall_at_1000": 0.99752, - "recall_at_20": 0.90135, - "recall_at_3": 0.70852, - "recall_at_5": 0.77503 - }, - { - "cv_recall_at_1": 0.42659, - "cv_recall_at_10": 0.75863, - "cv_recall_at_100": 0.94011, - "cv_recall_at_1000": 0.9939, - "cv_recall_at_20": 0.83169, - "cv_recall_at_3": 0.60003, - "cv_recall_at_5": 0.67184, - "hf_subset": "no", - "languages": [ - "nor-Latn" - ], - "main_score": 0.58599, - "map_at_1": 0.42659, - "map_at_10": 0.53152, - "map_at_100": 0.53948, - "map_at_1000": 0.53977, - "map_at_20": 0.53661, - "map_at_3": 0.50344, - "map_at_5": 0.51986, - "mrr_at_1": 0.42659087758214337, - "mrr_at_10": 0.531517718229213, - "mrr_at_100": 0.5394811572813161, - "mrr_at_1000": 0.5397675971072246, - "mrr_at_20": 0.5366131694602159, - "mrr_at_3": 0.5034428578030439, - "mrr_at_5": 0.5198576644022371, - "nauc_cv_recall_at_1000_diff1": 0.5365388365819015, - "nauc_cv_recall_at_1000_max": 0.8820883601096524, - "nauc_cv_recall_at_1000_std": 0.8560327455019514, - "nauc_cv_recall_at_100_diff1": 0.4291196561694799, - "nauc_cv_recall_at_100_max": 0.7317432023038329, - "nauc_cv_recall_at_100_std": 0.6364378846573594, - "nauc_cv_recall_at_10_diff1": 0.4727513594062488, - "nauc_cv_recall_at_10_max": 0.565976388070829, - "nauc_cv_recall_at_10_std": 0.2629214710555805, - "nauc_cv_recall_at_1_diff1": 0.6612644123585855, - "nauc_cv_recall_at_1_max": 0.4708878245975592, - "nauc_cv_recall_at_1_std": 0.048464967034370754, - "nauc_cv_recall_at_20_diff1": 0.44156738177753563, - "nauc_cv_recall_at_20_max": 0.6029237011588209, - "nauc_cv_recall_at_20_std": 0.3588193861745839, - "nauc_cv_recall_at_3_diff1": 0.5273771194696363, - "nauc_cv_recall_at_3_max": 0.5060320514918131, - "nauc_cv_recall_at_3_std": 0.1341634219173162, - "nauc_cv_recall_at_5_diff1": 0.5064775255421811, - "nauc_cv_recall_at_5_max": 0.5328714909701933, - "nauc_cv_recall_at_5_std": 0.18744484006449222, - "nauc_map_at_1000_diff1": 0.604318482617079, - "nauc_map_at_1000_max": 0.4925757726192787, - "nauc_map_at_1000_std": 0.09783991699344406, - "nauc_map_at_100_diff1": 0.6041514366932171, - "nauc_map_at_100_max": 0.4927559851221211, - "nauc_map_at_100_std": 0.09826761490615679, - "nauc_map_at_10_diff1": 0.6024278938638531, - "nauc_map_at_10_max": 0.4927211325536286, - "nauc_map_at_10_std": 0.09765222129343051, - "nauc_map_at_1_diff1": 0.6612644123585855, - "nauc_map_at_1_max": 0.4708878245975592, - "nauc_map_at_1_std": 0.048464967034370754, - "nauc_map_at_20_diff1": 0.6029216146779116, - "nauc_map_at_20_max": 0.4929855196311677, - "nauc_map_at_20_std": 0.09885380406817676, - "nauc_map_at_3_diff1": 0.6050308340493391, - "nauc_map_at_3_max": 0.48646198146939745, - "nauc_map_at_3_std": 0.08468646272239062, - "nauc_map_at_5_diff1": 0.6033105252155057, - "nauc_map_at_5_max": 0.4909150228364483, - "nauc_map_at_5_std": 0.09326129242686755, - "nauc_mrr_at_1000_diff1": 0.604318482617079, - "nauc_mrr_at_1000_max": 0.4925757726192787, - "nauc_mrr_at_1000_std": 0.09783991699344406, - "nauc_mrr_at_100_diff1": 0.6041514366932171, - "nauc_mrr_at_100_max": 0.4927559851221211, - "nauc_mrr_at_100_std": 0.09826761490615679, - "nauc_mrr_at_10_diff1": 0.6024278938638531, - "nauc_mrr_at_10_max": 0.4927211325536286, - "nauc_mrr_at_10_std": 0.09765222129343051, - "nauc_mrr_at_1_diff1": 0.6612644123585855, - "nauc_mrr_at_1_max": 0.4708878245975592, - "nauc_mrr_at_1_std": 0.048464967034370754, - "nauc_mrr_at_20_diff1": 0.6029216146779116, - "nauc_mrr_at_20_max": 0.4929855196311677, - "nauc_mrr_at_20_std": 0.09885380406817676, - "nauc_mrr_at_3_diff1": 0.6050308340493391, - "nauc_mrr_at_3_max": 0.48646198146939745, - "nauc_mrr_at_3_std": 0.08468646272239062, - "nauc_mrr_at_5_diff1": 0.6033105252155057, - "nauc_mrr_at_5_max": 0.4909150228364483, - "nauc_mrr_at_5_std": 0.09326129242686755, - "nauc_ndcg_at_1000_diff1": 0.5930751206638605, - "nauc_ndcg_at_1000_max": 0.5008525904823516, - "nauc_ndcg_at_1000_std": 0.11665718259501222, - "nauc_ndcg_at_100_diff1": 0.5884476368333909, - "nauc_ndcg_at_100_max": 0.5063649050617395, - "nauc_ndcg_at_100_std": 0.12946071575730214, - "nauc_ndcg_at_10_diff1": 0.5784819770863457, - "nauc_ndcg_at_10_max": 0.5055103201730918, - "nauc_ndcg_at_10_std": 0.12658901227383149, - "nauc_ndcg_at_1_diff1": 0.6612644123585855, - "nauc_ndcg_at_1_max": 0.4708878245975592, - "nauc_ndcg_at_1_std": 0.048464967034370754, - "nauc_ndcg_at_20_diff1": 0.5794062043265107, - "nauc_ndcg_at_20_max": 0.5075170784122032, - "nauc_ndcg_at_20_std": 0.13313497986974157, - "nauc_ndcg_at_3_diff1": 0.5861725439494837, - "nauc_ndcg_at_3_max": 0.4912503969392426, - "nauc_ndcg_at_3_std": 0.09671201364172016, - "nauc_ndcg_at_5_diff1": 0.5818577136220319, - "nauc_ndcg_at_5_max": 0.49996681584677094, - "nauc_ndcg_at_5_std": 0.11362132485163863, - "nauc_precision_at_1000_diff1": 0.5365388365818697, - "nauc_precision_at_1000_max": 0.8820883601096454, - "nauc_precision_at_1000_std": 0.8560327455019321, - "nauc_precision_at_100_diff1": 0.42911965616948045, - "nauc_precision_at_100_max": 0.7317432023038345, - "nauc_precision_at_100_std": 0.6364378846573614, - "nauc_precision_at_10_diff1": 0.4727513594062487, - "nauc_precision_at_10_max": 0.565976388070829, - "nauc_precision_at_10_std": 0.2629214710555803, - "nauc_precision_at_1_diff1": 0.6612644123585855, - "nauc_precision_at_1_max": 0.4708878245975592, - "nauc_precision_at_1_std": 0.048464967034370754, - "nauc_precision_at_20_diff1": 0.4415673817775351, - "nauc_precision_at_20_max": 0.6029237011588202, - "nauc_precision_at_20_std": 0.358819386174583, - "nauc_precision_at_3_diff1": 0.5273771194696367, - "nauc_precision_at_3_max": 0.506032051491813, - "nauc_precision_at_3_std": 0.13416342191731678, - "nauc_precision_at_5_diff1": 0.5064775255421813, - "nauc_precision_at_5_max": 0.5328714909701934, - "nauc_precision_at_5_std": 0.18744484006449183, - "nauc_recall_at_1000_diff1": 0.5365388365819015, - "nauc_recall_at_1000_max": 0.8820883601096524, - "nauc_recall_at_1000_std": 0.8560327455019514, - "nauc_recall_at_100_diff1": 0.4291196561694799, - "nauc_recall_at_100_max": 0.7317432023038329, - "nauc_recall_at_100_std": 0.6364378846573594, - "nauc_recall_at_10_diff1": 0.4727513594062488, - "nauc_recall_at_10_max": 0.565976388070829, - "nauc_recall_at_10_std": 0.2629214710555805, - "nauc_recall_at_1_diff1": 0.6612644123585855, - "nauc_recall_at_1_max": 0.4708878245975592, - "nauc_recall_at_1_std": 0.048464967034370754, - "nauc_recall_at_20_diff1": 0.44156738177753563, - "nauc_recall_at_20_max": 0.6029237011588209, - "nauc_recall_at_20_std": 0.3588193861745839, - "nauc_recall_at_3_diff1": 0.5273771194696363, - "nauc_recall_at_3_max": 0.5060320514918131, - "nauc_recall_at_3_std": 0.1341634219173162, - "nauc_recall_at_5_diff1": 0.5064775255421811, - "nauc_recall_at_5_max": 0.5328714909701933, - "nauc_recall_at_5_std": 0.18744484006449222, - "ndcg_at_1": 0.42659, - "ndcg_at_10": 0.58599, - "ndcg_at_100": 0.62479, - "ndcg_at_1000": 0.63179, - "ndcg_at_20": 0.60449, - "ndcg_at_3": 0.52827, - "ndcg_at_5": 0.55785, - "precision_at_1": 0.42659, - "precision_at_10": 0.07586, - "precision_at_100": 0.0094, - "precision_at_1000": 0.00099, - "precision_at_20": 0.04158, - "precision_at_3": 0.20001, - "precision_at_5": 0.13437, - "recall_at_1": 0.42659, - "recall_at_10": 0.75863, - "recall_at_100": 0.94011, - "recall_at_1000": 0.9939, - "recall_at_20": 0.83169, - "recall_at_3": 0.60003, - "recall_at_5": 0.67184 - }, - { - "cv_recall_at_1": 0.50217, - "cv_recall_at_10": 0.8384, - "cv_recall_at_100": 0.97325, - "cv_recall_at_1000": 0.99762, - "cv_recall_at_20": 0.89693, - "cv_recall_at_3": 0.68758, - "cv_recall_at_5": 0.76012, - "hf_subset": "pl", - "languages": [ - "pol-Latn" - ], - "main_score": 0.66649, - "map_at_1": 0.50217, - "map_at_10": 0.61184, - "map_at_100": 0.61806, - "map_at_1000": 0.61819, - "map_at_20": 0.61596, - "map_at_3": 0.58461, - "map_at_5": 0.60119, - "mrr_at_1": 0.5021705643467301, - "mrr_at_10": 0.611844746300706, - "mrr_at_100": 0.6180557676786029, - "mrr_at_1000": 0.6181902933760485, - "mrr_at_20": 0.6159593318440516, - "mrr_at_3": 0.5846053307193233, - "mrr_at_5": 0.6011926434206231, - "nauc_cv_recall_at_1000_diff1": 0.6238997805255307, - "nauc_cv_recall_at_1000_max": 0.9737748390295698, - "nauc_cv_recall_at_1000_std": 0.9221778368717609, - "nauc_cv_recall_at_100_diff1": 0.4493973192301335, - "nauc_cv_recall_at_100_max": 0.7550416988642245, - "nauc_cv_recall_at_100_std": 0.5710271382364848, - "nauc_cv_recall_at_10_diff1": 0.5346408354506577, - "nauc_cv_recall_at_10_max": 0.5740556765410139, - "nauc_cv_recall_at_10_std": 0.11800066290994378, - "nauc_cv_recall_at_1_diff1": 0.6800424383383243, - "nauc_cv_recall_at_1_max": 0.44862302660287423, - "nauc_cv_recall_at_1_std": -0.04130418695693082, - "nauc_cv_recall_at_20_diff1": 0.5260408734169024, - "nauc_cv_recall_at_20_max": 0.6071704973496403, - "nauc_cv_recall_at_20_std": 0.20817459838843785, - "nauc_cv_recall_at_3_diff1": 0.5690289851202682, - "nauc_cv_recall_at_3_max": 0.49770044477603553, - "nauc_cv_recall_at_3_std": -0.005964362544451287, - "nauc_cv_recall_at_5_diff1": 0.5393013571222156, - "nauc_cv_recall_at_5_max": 0.5175617234123552, - "nauc_cv_recall_at_5_std": 0.035400452964310235, - "nauc_map_at_1000_diff1": 0.6376403336847224, - "nauc_map_at_1000_max": 0.47264358864353895, - "nauc_map_at_1000_std": -0.019520330304244982, - "nauc_map_at_100_diff1": 0.6375513504358796, - "nauc_map_at_100_max": 0.4727485339786326, - "nauc_map_at_100_std": -0.019298350612240715, - "nauc_map_at_10_diff1": 0.6358558627801074, - "nauc_map_at_10_max": 0.47356484765601586, - "nauc_map_at_10_std": -0.019397646543862737, - "nauc_map_at_1_diff1": 0.6800424383383243, - "nauc_map_at_1_max": 0.44862302660287423, - "nauc_map_at_1_std": -0.04130418695693082, - "nauc_map_at_20_diff1": 0.6369311067316696, - "nauc_map_at_20_max": 0.47322037710465403, - "nauc_map_at_20_std": -0.018869432047767736, - "nauc_map_at_3_diff1": 0.6357415873057528, - "nauc_map_at_3_max": 0.46874622110718456, - "nauc_map_at_3_std": -0.0291487680219996, - "nauc_map_at_5_diff1": 0.6335952409143635, - "nauc_map_at_5_max": 0.4711353116370656, - "nauc_map_at_5_std": -0.023090682569859827, - "nauc_mrr_at_1000_diff1": 0.6376403336847224, - "nauc_mrr_at_1000_max": 0.47264358864353895, - "nauc_mrr_at_1000_std": -0.019520330304244982, - "nauc_mrr_at_100_diff1": 0.6375513504358796, - "nauc_mrr_at_100_max": 0.4727485339786326, - "nauc_mrr_at_100_std": -0.019298350612240715, - "nauc_mrr_at_10_diff1": 0.6358558627801074, - "nauc_mrr_at_10_max": 0.47356484765601586, - "nauc_mrr_at_10_std": -0.019397646543862737, - "nauc_mrr_at_1_diff1": 0.6800424383383243, - "nauc_mrr_at_1_max": 0.44862302660287423, - "nauc_mrr_at_1_std": -0.04130418695693082, - "nauc_mrr_at_20_diff1": 0.6369311067316696, - "nauc_mrr_at_20_max": 0.47322037710465403, - "nauc_mrr_at_20_std": -0.018869432047767736, - "nauc_mrr_at_3_diff1": 0.6357415873057528, - "nauc_mrr_at_3_max": 0.46874622110718456, - "nauc_mrr_at_3_std": -0.0291487680219996, - "nauc_mrr_at_5_diff1": 0.6335952409143635, - "nauc_mrr_at_5_max": 0.4711353116370656, - "nauc_mrr_at_5_std": -0.023090682569859827, - "nauc_ndcg_at_1000_diff1": 0.6305642442540332, - "nauc_ndcg_at_1000_max": 0.4797823361804251, - "nauc_ndcg_at_1000_std": -0.008377140285703567, - "nauc_ndcg_at_100_diff1": 0.6278877828219978, - "nauc_ndcg_at_100_max": 0.48314104583429296, - "nauc_ndcg_at_100_std": -0.0011767182837512037, - "nauc_ndcg_at_10_diff1": 0.6199082824561567, - "nauc_ndcg_at_10_max": 0.48793329890789217, - "nauc_ndcg_at_10_std": -0.000376817961762273, - "nauc_ndcg_at_1_diff1": 0.6800424383383243, - "nauc_ndcg_at_1_max": 0.44862302660287423, - "nauc_ndcg_at_1_std": -0.04130418695693082, - "nauc_ndcg_at_20_diff1": 0.6235155170852444, - "nauc_ndcg_at_20_max": 0.4870249224862057, - "nauc_ndcg_at_20_std": 0.0028991542134987796, - "nauc_ndcg_at_3_diff1": 0.6204971834552783, - "nauc_ndcg_at_3_max": 0.4753871652584329, - "nauc_ndcg_at_3_std": -0.023940094636684603, - "nauc_ndcg_at_5_diff1": 0.6150442795355727, - "nauc_ndcg_at_5_max": 0.4802621376780359, - "nauc_ndcg_at_5_std": -0.011994226824793612, - "nauc_precision_at_1000_diff1": 0.6238997805255041, - "nauc_precision_at_1000_max": 0.9737748390295885, - "nauc_precision_at_1000_std": 0.922177836871779, - "nauc_precision_at_100_diff1": 0.44939731923013837, - "nauc_precision_at_100_max": 0.755041698864231, - "nauc_precision_at_100_std": 0.5710271382364832, - "nauc_precision_at_10_diff1": 0.534640835450658, - "nauc_precision_at_10_max": 0.5740556765410155, - "nauc_precision_at_10_std": 0.1180006629099428, - "nauc_precision_at_1_diff1": 0.6800424383383243, - "nauc_precision_at_1_max": 0.44862302660287423, - "nauc_precision_at_1_std": -0.04130418695693082, - "nauc_precision_at_20_diff1": 0.5260408734169023, - "nauc_precision_at_20_max": 0.6071704973496397, - "nauc_precision_at_20_std": 0.20817459838843586, - "nauc_precision_at_3_diff1": 0.5690289851202678, - "nauc_precision_at_3_max": 0.49770044477603614, - "nauc_precision_at_3_std": -0.005964362544451902, - "nauc_precision_at_5_diff1": 0.539301357122216, - "nauc_precision_at_5_max": 0.5175617234123555, - "nauc_precision_at_5_std": 0.03540045296431024, - "nauc_recall_at_1000_diff1": 0.6238997805255307, - "nauc_recall_at_1000_max": 0.9737748390295698, - "nauc_recall_at_1000_std": 0.9221778368717609, - "nauc_recall_at_100_diff1": 0.4493973192301335, - "nauc_recall_at_100_max": 0.7550416988642245, - "nauc_recall_at_100_std": 0.5710271382364848, - "nauc_recall_at_10_diff1": 0.5346408354506577, - "nauc_recall_at_10_max": 0.5740556765410139, - "nauc_recall_at_10_std": 0.11800066290994378, - "nauc_recall_at_1_diff1": 0.6800424383383243, - "nauc_recall_at_1_max": 0.44862302660287423, - "nauc_recall_at_1_std": -0.04130418695693082, - "nauc_recall_at_20_diff1": 0.5260408734169024, - "nauc_recall_at_20_max": 0.6071704973496403, - "nauc_recall_at_20_std": 0.20817459838843785, - "nauc_recall_at_3_diff1": 0.5690289851202682, - "nauc_recall_at_3_max": 0.49770044477603553, - "nauc_recall_at_3_std": -0.005964362544451287, - "nauc_recall_at_5_diff1": 0.5393013571222156, - "nauc_recall_at_5_max": 0.5175617234123552, - "nauc_recall_at_5_std": 0.035400452964310235, - "ndcg_at_1": 0.50217, - "ndcg_at_10": 0.66649, - "ndcg_at_100": 0.69579, - "ndcg_at_1000": 0.69899, - "ndcg_at_20": 0.68135, - "ndcg_at_3": 0.61108, - "ndcg_at_5": 0.64097, - "precision_at_1": 0.50217, - "precision_at_10": 0.08384, - "precision_at_100": 0.00973, - "precision_at_1000": 0.001, - "precision_at_20": 0.04485, - "precision_at_3": 0.22919, - "precision_at_5": 0.15202, - "recall_at_1": 0.50217, - "recall_at_10": 0.8384, - "recall_at_100": 0.97325, - "recall_at_1000": 0.99762, - "recall_at_20": 0.89693, - "recall_at_3": 0.68758, - "recall_at_5": 0.76012 - }, - { - "cv_recall_at_1": 0.62502, - "cv_recall_at_10": 0.90156, - "cv_recall_at_100": 0.98826, - "cv_recall_at_1000": 0.99945, - "cv_recall_at_20": 0.93953, - "cv_recall_at_3": 0.78421, - "cv_recall_at_5": 0.84123, - "hf_subset": "pt", - "languages": [ - "por-Latn" - ], - "main_score": 0.76169, - "map_at_1": 0.62502, - "map_at_10": 0.71706, - "map_at_100": 0.72117, - "map_at_1000": 0.72123, - "map_at_20": 0.71975, - "map_at_3": 0.69582, - "map_at_5": 0.70887, - "mrr_at_1": 0.6250172580422477, - "mrr_at_10": 0.7170563696968486, - "mrr_at_100": 0.721167438819844, - "mrr_at_1000": 0.721228572929272, - "mrr_at_20": 0.7197508171057015, - "mrr_at_3": 0.69582125270376, - "mrr_at_5": 0.7088683326430089, - "nauc_cv_recall_at_1000_diff1": 0.6038682984684438, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": 0.6397914759225447, - "nauc_cv_recall_at_100_max": 0.7941256297154522, - "nauc_cv_recall_at_100_std": 0.5105784983752181, - "nauc_cv_recall_at_10_diff1": 0.6213519862367571, - "nauc_cv_recall_at_10_max": 0.5579690033478077, - "nauc_cv_recall_at_10_std": -0.04098643332655127, - "nauc_cv_recall_at_1_diff1": 0.7490694455433466, - "nauc_cv_recall_at_1_max": 0.441556203916839, - "nauc_cv_recall_at_1_std": -0.1558626182312551, - "nauc_cv_recall_at_20_diff1": 0.6381809212766827, - "nauc_cv_recall_at_20_max": 0.601919642260997, - "nauc_cv_recall_at_20_std": 0.06712229461141428, - "nauc_cv_recall_at_3_diff1": 0.6620309591987041, - "nauc_cv_recall_at_3_max": 0.4771822776364749, - "nauc_cv_recall_at_3_std": -0.15353925517434355, - "nauc_cv_recall_at_5_diff1": 0.6476318184903693, - "nauc_cv_recall_at_5_max": 0.5013983365902959, - "nauc_cv_recall_at_5_std": -0.13494507379350235, - "nauc_map_at_1000_diff1": 0.7192701129566936, - "nauc_map_at_1000_max": 0.4583892756334078, - "nauc_map_at_1000_std": -0.14998938985883367, - "nauc_map_at_100_diff1": 0.7192547571330423, - "nauc_map_at_100_max": 0.4584642094024353, - "nauc_map_at_100_std": -0.14984408132811972, - "nauc_map_at_10_diff1": 0.7175606048461799, - "nauc_map_at_10_max": 0.4592566978388754, - "nauc_map_at_10_std": -0.14999184568623453, - "nauc_map_at_1_diff1": 0.7490694455433466, - "nauc_map_at_1_max": 0.441556203916839, - "nauc_map_at_1_std": -0.1558626182312551, - "nauc_map_at_20_diff1": 0.7188548018807867, - "nauc_map_at_20_max": 0.4589323914725497, - "nauc_map_at_20_std": -0.1493844447508069, - "nauc_map_at_3_diff1": 0.7168356111328161, - "nauc_map_at_3_max": 0.45474550225239724, - "nauc_map_at_3_std": -0.15736836775108023, - "nauc_map_at_5_diff1": 0.7170041475021451, - "nauc_map_at_5_max": 0.4571011403468147, - "nauc_map_at_5_std": -0.1550665804562101, - "nauc_mrr_at_1000_diff1": 0.7192701129566936, - "nauc_mrr_at_1000_max": 0.4583892756334078, - "nauc_mrr_at_1000_std": -0.14998938985883367, - "nauc_mrr_at_100_diff1": 0.7192547571330423, - "nauc_mrr_at_100_max": 0.4584642094024353, - "nauc_mrr_at_100_std": -0.14984408132811972, - "nauc_mrr_at_10_diff1": 0.7175606048461799, - "nauc_mrr_at_10_max": 0.4592566978388754, - "nauc_mrr_at_10_std": -0.14999184568623453, - "nauc_mrr_at_1_diff1": 0.7490694455433466, - "nauc_mrr_at_1_max": 0.441556203916839, - "nauc_mrr_at_1_std": -0.1558626182312551, - "nauc_mrr_at_20_diff1": 0.7188548018807867, - "nauc_mrr_at_20_max": 0.4589323914725497, - "nauc_mrr_at_20_std": -0.1493844447508069, - "nauc_mrr_at_3_diff1": 0.7168356111328161, - "nauc_mrr_at_3_max": 0.45474550225239724, - "nauc_mrr_at_3_std": -0.15736836775108023, - "nauc_mrr_at_5_diff1": 0.7170041475021451, - "nauc_mrr_at_5_max": 0.4571011403468147, - "nauc_mrr_at_5_std": -0.1550665804562101, - "nauc_ndcg_at_1000_diff1": 0.7145970154353453, - "nauc_ndcg_at_1000_max": 0.4635310945979437, - "nauc_ndcg_at_1000_std": -0.14364931096142944, - "nauc_ndcg_at_100_diff1": 0.7140530995986156, - "nauc_ndcg_at_100_max": 0.46598217186904245, - "nauc_ndcg_at_100_std": -0.13883751906403827, - "nauc_ndcg_at_10_diff1": 0.705282613937573, - "nauc_ndcg_at_10_max": 0.4708566499849579, - "nauc_ndcg_at_10_std": -0.13809732008953995, - "nauc_ndcg_at_1_diff1": 0.7490694455433466, - "nauc_ndcg_at_1_max": 0.441556203916839, - "nauc_ndcg_at_1_std": -0.1558626182312551, - "nauc_ndcg_at_20_diff1": 0.7106077672375423, - "nauc_ndcg_at_20_max": 0.4700208803914439, - "nauc_ndcg_at_20_std": -0.13454089853888754, - "nauc_ndcg_at_3_diff1": 0.7051459214080644, - "nauc_ndcg_at_3_max": 0.4595309984418196, - "nauc_ndcg_at_3_std": -0.1566627942812548, - "nauc_ndcg_at_5_diff1": 0.7046798864341236, - "nauc_ndcg_at_5_max": 0.46471324477089654, - "nauc_ndcg_at_5_std": -0.15186183615748197, - "nauc_precision_at_1000_diff1": 0.6038682984685291, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.6397914759225701, - "nauc_precision_at_100_max": 0.794125629715474, - "nauc_precision_at_100_std": 0.5105784983752383, - "nauc_precision_at_10_diff1": 0.6213519862367571, - "nauc_precision_at_10_max": 0.5579690033478083, - "nauc_precision_at_10_std": -0.04098643332655233, - "nauc_precision_at_1_diff1": 0.7490694455433466, - "nauc_precision_at_1_max": 0.441556203916839, - "nauc_precision_at_1_std": -0.1558626182312551, - "nauc_precision_at_20_diff1": 0.6381809212766826, - "nauc_precision_at_20_max": 0.6019196422609968, - "nauc_precision_at_20_std": 0.06712229461141557, - "nauc_precision_at_3_diff1": 0.6620309591987041, - "nauc_precision_at_3_max": 0.4771822776364753, - "nauc_precision_at_3_std": -0.15353925517434228, - "nauc_precision_at_5_diff1": 0.6476318184903682, - "nauc_precision_at_5_max": 0.5013983365902952, - "nauc_precision_at_5_std": -0.13494507379350357, - "nauc_recall_at_1000_diff1": 0.6038682984684438, - "nauc_recall_at_1000_max": 1.0, - "nauc_recall_at_1000_std": 1.0, - "nauc_recall_at_100_diff1": 0.6397914759225447, - "nauc_recall_at_100_max": 0.7941256297154522, - "nauc_recall_at_100_std": 0.5105784983752181, - "nauc_recall_at_10_diff1": 0.6213519862367571, - "nauc_recall_at_10_max": 0.5579690033478077, - "nauc_recall_at_10_std": -0.04098643332655127, - "nauc_recall_at_1_diff1": 0.7490694455433466, - "nauc_recall_at_1_max": 0.441556203916839, - "nauc_recall_at_1_std": -0.1558626182312551, - "nauc_recall_at_20_diff1": 0.6381809212766827, - "nauc_recall_at_20_max": 0.601919642260997, - "nauc_recall_at_20_std": 0.06712229461141428, - "nauc_recall_at_3_diff1": 0.6620309591987041, - "nauc_recall_at_3_max": 0.4771822776364749, - "nauc_recall_at_3_std": -0.15353925517434355, - "nauc_recall_at_5_diff1": 0.6476318184903693, - "nauc_recall_at_5_max": 0.5013983365902959, - "nauc_recall_at_5_std": -0.13494507379350235, - "ndcg_at_1": 0.62502, - "ndcg_at_10": 0.76169, - "ndcg_at_100": 0.78073, - "ndcg_at_1000": 0.78219, - "ndcg_at_20": 0.77137, - "ndcg_at_3": 0.71855, - "ndcg_at_5": 0.74205, - "precision_at_1": 0.62502, - "precision_at_10": 0.09016, - "precision_at_100": 0.00988, - "precision_at_1000": 0.001, - "precision_at_20": 0.04698, - "precision_at_3": 0.2614, - "precision_at_5": 0.16825, - "recall_at_1": 0.62502, - "recall_at_10": 0.90156, - "recall_at_100": 0.98826, - "recall_at_1000": 0.99945, - "recall_at_20": 0.93953, - "recall_at_3": 0.78421, - "recall_at_5": 0.84123 - }, - { - "cv_recall_at_1": 0.02292, - "cv_recall_at_10": 0.08417, - "cv_recall_at_100": 0.20569, - "cv_recall_at_1000": 0.52736, - "cv_recall_at_20": 0.11222, - "cv_recall_at_3": 0.04569, - "cv_recall_at_5": 0.05917, - "hf_subset": "quz", - "languages": [ - "quz-Latn" - ], - "main_score": 0.0496, - "map_at_1": 0.02292, - "map_at_10": 0.03905, - "map_at_100": 0.04313, - "map_at_1000": 0.04414, - "map_at_20": 0.04097, - "map_at_3": 0.03266, - "map_at_5": 0.03575, - "mrr_at_1": 0.022916666666666665, - "mrr_at_10": 0.03905042989417988, - "mrr_at_100": 0.04313296486782967, - "mrr_at_1000": 0.044140873349093286, - "mrr_at_20": 0.0409663441169826, - "mrr_at_3": 0.03266203703703707, - "mrr_at_5": 0.03574537037037037, - "nauc_cv_recall_at_1000_diff1": 0.0556925167648709, - "nauc_cv_recall_at_1000_max": 0.41275860775657136, - "nauc_cv_recall_at_1000_std": 0.4770003851498549, - "nauc_cv_recall_at_100_diff1": 0.10493534615151756, - "nauc_cv_recall_at_100_max": 0.5412833456822512, - "nauc_cv_recall_at_100_std": 0.5932599080938897, - "nauc_cv_recall_at_10_diff1": 0.19215412284453554, - "nauc_cv_recall_at_10_max": 0.6037828494413702, - "nauc_cv_recall_at_10_std": 0.6119495502005258, - "nauc_cv_recall_at_1_diff1": 0.43177793497085165, - "nauc_cv_recall_at_1_max": 0.7547918140351733, - "nauc_cv_recall_at_1_std": 0.7091445715540161, - "nauc_cv_recall_at_20_diff1": 0.15027188078341383, - "nauc_cv_recall_at_20_max": 0.5905053410817863, - "nauc_cv_recall_at_20_std": 0.6199595523034361, - "nauc_cv_recall_at_3_diff1": 0.2763078594304439, - "nauc_cv_recall_at_3_max": 0.6678081493545414, - "nauc_cv_recall_at_3_std": 0.642089312932898, - "nauc_cv_recall_at_5_diff1": 0.23357027956127785, - "nauc_cv_recall_at_5_max": 0.644227947058075, - "nauc_cv_recall_at_5_std": 0.6378463968679261, - "nauc_map_at_1000_diff1": 0.2796256774634396, - "nauc_map_at_1000_max": 0.6650125561397447, - "nauc_map_at_1000_std": 0.6516817769669179, - "nauc_map_at_100_diff1": 0.2814989691212026, - "nauc_map_at_100_max": 0.6673002299701124, - "nauc_map_at_100_std": 0.6531316581428467, - "nauc_map_at_10_diff1": 0.3016853183868804, - "nauc_map_at_10_max": 0.6788300049355528, - "nauc_map_at_10_std": 0.6563158970332853, - "nauc_map_at_1_diff1": 0.43177793497085165, - "nauc_map_at_1_max": 0.7547918140351733, - "nauc_map_at_1_std": 0.7091445715540161, - "nauc_map_at_20_diff1": 0.28995599574333136, - "nauc_map_at_20_max": 0.6727288716216021, - "nauc_map_at_20_std": 0.6551859095502353, - "nauc_map_at_3_diff1": 0.3416224280952469, - "nauc_map_at_3_max": 0.7058933820795898, - "nauc_map_at_3_std": 0.6695571249470176, - "nauc_map_at_5_diff1": 0.32038815315465746, - "nauc_map_at_5_max": 0.6941412662777497, - "nauc_map_at_5_std": 0.6659616445790018, - "nauc_mrr_at_1000_diff1": 0.27962567785184494, - "nauc_mrr_at_1000_max": 0.6650125299359211, - "nauc_mrr_at_1000_std": 0.6516817509750019, - "nauc_mrr_at_100_diff1": 0.2814989691212026, - "nauc_mrr_at_100_max": 0.6673002299701124, - "nauc_mrr_at_100_std": 0.6531316581428467, - "nauc_mrr_at_10_diff1": 0.3016853183868804, - "nauc_mrr_at_10_max": 0.6788300049355528, - "nauc_mrr_at_10_std": 0.6563158970332853, - "nauc_mrr_at_1_diff1": 0.43177793497085165, - "nauc_mrr_at_1_max": 0.7547918140351733, - "nauc_mrr_at_1_std": 0.7091445715540161, - "nauc_mrr_at_20_diff1": 0.28995599574333136, - "nauc_mrr_at_20_max": 0.6727288716216021, - "nauc_mrr_at_20_std": 0.6551859095502353, - "nauc_mrr_at_3_diff1": 0.3416224280952469, - "nauc_mrr_at_3_max": 0.7058933820795898, - "nauc_mrr_at_3_std": 0.6695571249470176, - "nauc_mrr_at_5_diff1": 0.32038815315465746, - "nauc_mrr_at_5_max": 0.6941412662777497, - "nauc_mrr_at_5_std": 0.6659616445790018, - "nauc_ndcg_at_1000_diff1": 0.17514521567782645, - "nauc_ndcg_at_1000_max": 0.5643365826227588, - "nauc_ndcg_at_1000_std": 0.5877111863477217, - "nauc_ndcg_at_100_diff1": 0.19774191621612744, - "nauc_ndcg_at_100_max": 0.6098229146580955, - "nauc_ndcg_at_100_std": 0.625863941031239, - "nauc_ndcg_at_10_diff1": 0.2585250624624412, - "nauc_ndcg_at_10_max": 0.6496629866590963, - "nauc_ndcg_at_10_std": 0.6389591527927075, - "nauc_ndcg_at_1_diff1": 0.43177793497085165, - "nauc_ndcg_at_1_max": 0.7547918140351733, - "nauc_ndcg_at_1_std": 0.7091445715540161, - "nauc_ndcg_at_20_diff1": 0.23119336697129492, - "nauc_ndcg_at_20_max": 0.637617837219863, - "nauc_ndcg_at_20_std": 0.6392094102892021, - "nauc_ndcg_at_3_diff1": 0.32051666816022684, - "nauc_ndcg_at_3_max": 0.6936419572495052, - "nauc_ndcg_at_3_std": 0.660655911138741, - "nauc_ndcg_at_5_diff1": 0.2899701948259364, - "nauc_ndcg_at_5_max": 0.6767013412913847, - "nauc_ndcg_at_5_std": 0.6559718938617654, - "nauc_precision_at_1000_diff1": 0.055692516764870165, - "nauc_precision_at_1000_max": 0.4127586077565709, - "nauc_precision_at_1000_std": 0.4770003851498547, - "nauc_precision_at_100_diff1": 0.1049353461515177, - "nauc_precision_at_100_max": 0.5412833456822514, - "nauc_precision_at_100_std": 0.59325990809389, - "nauc_precision_at_10_diff1": 0.19215412284453554, - "nauc_precision_at_10_max": 0.6037828494413703, - "nauc_precision_at_10_std": 0.6119495502005261, - "nauc_precision_at_1_diff1": 0.43177793497085165, - "nauc_precision_at_1_max": 0.7547918140351733, - "nauc_precision_at_1_std": 0.7091445715540161, - "nauc_precision_at_20_diff1": 0.15027188078341375, - "nauc_precision_at_20_max": 0.5905053410817863, - "nauc_precision_at_20_std": 0.619959552303436, - "nauc_precision_at_3_diff1": 0.2763078594304441, - "nauc_precision_at_3_max": 0.6678081493545418, - "nauc_precision_at_3_std": 0.6420893129328985, - "nauc_precision_at_5_diff1": 0.2335702795612778, - "nauc_precision_at_5_max": 0.6442279470580751, - "nauc_precision_at_5_std": 0.6378463968679261, - "nauc_recall_at_1000_diff1": 0.0556925167648709, - "nauc_recall_at_1000_max": 0.41275860775657136, - "nauc_recall_at_1000_std": 0.4770003851498549, - "nauc_recall_at_100_diff1": 0.10493534615151756, - "nauc_recall_at_100_max": 0.5412833456822512, - "nauc_recall_at_100_std": 0.5932599080938897, - "nauc_recall_at_10_diff1": 0.19215412284453554, - "nauc_recall_at_10_max": 0.6037828494413702, - "nauc_recall_at_10_std": 0.6119495502005258, - "nauc_recall_at_1_diff1": 0.43177793497085165, - "nauc_recall_at_1_max": 0.7547918140351733, - "nauc_recall_at_1_std": 0.7091445715540161, - "nauc_recall_at_20_diff1": 0.15027188078341383, - "nauc_recall_at_20_max": 0.5905053410817863, - "nauc_recall_at_20_std": 0.6199595523034361, - "nauc_recall_at_3_diff1": 0.2763078594304439, - "nauc_recall_at_3_max": 0.6678081493545414, - "nauc_recall_at_3_std": 0.642089312932898, - "nauc_recall_at_5_diff1": 0.23357027956127785, - "nauc_recall_at_5_max": 0.644227947058075, - "nauc_recall_at_5_std": 0.6378463968679261, - "ndcg_at_1": 0.02292, - "ndcg_at_10": 0.0496, - "ndcg_at_100": 0.07353, - "ndcg_at_1000": 0.11124, - "ndcg_at_20": 0.05666, - "ndcg_at_3": 0.036, - "ndcg_at_5": 0.04155, - "precision_at_1": 0.02292, - "precision_at_10": 0.00842, - "precision_at_100": 0.00206, - "precision_at_1000": 0.00053, - "precision_at_20": 0.00561, - "precision_at_3": 0.01523, - "precision_at_5": 0.01183, - "recall_at_1": 0.02292, - "recall_at_10": 0.08417, - "recall_at_100": 0.20569, - "recall_at_1000": 0.52736, - "recall_at_20": 0.11222, - "recall_at_3": 0.04569, - "recall_at_5": 0.05917 - }, - { - "cv_recall_at_1": 0.36768, - "cv_recall_at_10": 0.67556, - "cv_recall_at_100": 0.89723, - "cv_recall_at_1000": 0.98708, - "cv_recall_at_20": 0.75516, - "cv_recall_at_3": 0.52352, - "cv_recall_at_5": 0.58964, - "hf_subset": "ro", - "languages": [ - "ron-Latn" - ], - "main_score": 0.51408, - "map_at_1": 0.36768, - "map_at_10": 0.46332, - "map_at_100": 0.47262, - "map_at_1000": 0.47304, - "map_at_20": 0.46897, - "map_at_3": 0.43661, - "map_at_5": 0.45175, - "mrr_at_1": 0.36768215639477747, - "mrr_at_10": 0.4633156842689346, - "mrr_at_100": 0.4726189598420947, - "mrr_at_1000": 0.47304219633255584, - "mrr_at_20": 0.46896763103749506, - "mrr_at_3": 0.4366137863259877, - "mrr_at_5": 0.4517478590481541, - "nauc_cv_recall_at_1000_diff1": 0.4610798974007453, - "nauc_cv_recall_at_1000_max": 0.8955736220454759, - "nauc_cv_recall_at_1000_std": 0.9412424160278959, - "nauc_cv_recall_at_100_diff1": 0.4108601422390299, - "nauc_cv_recall_at_100_max": 0.6956175426234881, - "nauc_cv_recall_at_100_std": 0.6343823723345061, - "nauc_cv_recall_at_10_diff1": 0.4540269841800468, - "nauc_cv_recall_at_10_max": 0.5812748871363784, - "nauc_cv_recall_at_10_std": 0.330293834235662, - "nauc_cv_recall_at_1_diff1": 0.6328181504850688, - "nauc_cv_recall_at_1_max": 0.516415613700369, - "nauc_cv_recall_at_1_std": 0.1613423219822137, - "nauc_cv_recall_at_20_diff1": 0.4215794118456787, - "nauc_cv_recall_at_20_max": 0.6175782422623985, - "nauc_cv_recall_at_20_std": 0.42480252175436806, - "nauc_cv_recall_at_3_diff1": 0.5112768505535457, - "nauc_cv_recall_at_3_max": 0.5433888785986725, - "nauc_cv_recall_at_3_std": 0.21539500930972913, - "nauc_cv_recall_at_5_diff1": 0.4767674258764573, - "nauc_cv_recall_at_5_max": 0.5603999867601904, - "nauc_cv_recall_at_5_std": 0.26000574623796224, - "nauc_map_at_1000_diff1": 0.5725380617205134, - "nauc_map_at_1000_max": 0.5348171726552217, - "nauc_map_at_1000_std": 0.2024265935259748, - "nauc_map_at_100_diff1": 0.5723428685740998, - "nauc_map_at_100_max": 0.5349390652281885, - "nauc_map_at_100_std": 0.20281321546350395, - "nauc_map_at_10_diff1": 0.571185933778101, - "nauc_map_at_10_max": 0.5338641080299099, - "nauc_map_at_10_std": 0.19995969194038202, - "nauc_map_at_1_diff1": 0.6328181504850688, - "nauc_map_at_1_max": 0.516415613700369, - "nauc_map_at_1_std": 0.1613423219822137, - "nauc_map_at_20_diff1": 0.5710745751223294, - "nauc_map_at_20_max": 0.5350518763447834, - "nauc_map_at_20_std": 0.20302780619493885, - "nauc_map_at_3_diff1": 0.5781262316460313, - "nauc_map_at_3_max": 0.5287022746286781, - "nauc_map_at_3_std": 0.18451064634106332, - "nauc_map_at_5_diff1": 0.5720343476770393, - "nauc_map_at_5_max": 0.5322655936266794, - "nauc_map_at_5_std": 0.19351069901578477, - "nauc_mrr_at_1000_diff1": 0.5725380514865761, - "nauc_mrr_at_1000_max": 0.5348171727699603, - "nauc_mrr_at_1000_std": 0.20242659444987554, - "nauc_mrr_at_100_diff1": 0.5723428685740998, - "nauc_mrr_at_100_max": 0.5349390652281885, - "nauc_mrr_at_100_std": 0.20281321546350395, - "nauc_mrr_at_10_diff1": 0.571185933778101, - "nauc_mrr_at_10_max": 0.5338641080299099, - "nauc_mrr_at_10_std": 0.19995969194038202, - "nauc_mrr_at_1_diff1": 0.6328181504850688, - "nauc_mrr_at_1_max": 0.516415613700369, - "nauc_mrr_at_1_std": 0.1613423219822137, - "nauc_mrr_at_20_diff1": 0.5710745751223294, - "nauc_mrr_at_20_max": 0.5350518763447834, - "nauc_mrr_at_20_std": 0.20302780619493885, - "nauc_mrr_at_3_diff1": 0.5781262316460313, - "nauc_mrr_at_3_max": 0.5287022746286781, - "nauc_mrr_at_3_std": 0.18451064634106332, - "nauc_mrr_at_5_diff1": 0.5720343476770393, - "nauc_mrr_at_5_max": 0.5322655936266794, - "nauc_mrr_at_5_std": 0.19351069901578477, - "nauc_ndcg_at_1000_diff1": 0.55932571244852, - "nauc_ndcg_at_1000_max": 0.5438659750081213, - "nauc_ndcg_at_1000_std": 0.22493836703581904, - "nauc_ndcg_at_100_diff1": 0.5533220616630024, - "nauc_ndcg_at_100_max": 0.5483763183365132, - "nauc_ndcg_at_100_std": 0.23857994788319828, - "nauc_ndcg_at_10_diff1": 0.5464225493257165, - "nauc_ndcg_at_10_max": 0.5434379756441989, - "nauc_ndcg_at_10_std": 0.225836309928415, - "nauc_ndcg_at_1_diff1": 0.6328181504850688, - "nauc_ndcg_at_1_max": 0.516415613700369, - "nauc_ndcg_at_1_std": 0.1613423219822137, - "nauc_ndcg_at_20_diff1": 0.5447313710226348, - "nauc_ndcg_at_20_max": 0.5483598729493525, - "nauc_ndcg_at_20_std": 0.23846248552590435, - "nauc_ndcg_at_3_diff1": 0.5612078893755854, - "nauc_ndcg_at_3_max": 0.5324257577340359, - "nauc_ndcg_at_3_std": 0.1922759753998782, - "nauc_ndcg_at_5_diff1": 0.5494157617079797, - "nauc_ndcg_at_5_max": 0.5388763844572947, - "nauc_ndcg_at_5_std": 0.20893951759575607, - "nauc_precision_at_1000_diff1": 0.4610798974007486, - "nauc_precision_at_1000_max": 0.8955736220454706, - "nauc_precision_at_1000_std": 0.9412424160278849, - "nauc_precision_at_100_diff1": 0.4108601422390304, - "nauc_precision_at_100_max": 0.6956175426234907, - "nauc_precision_at_100_std": 0.6343823723345081, - "nauc_precision_at_10_diff1": 0.45402698418004667, - "nauc_precision_at_10_max": 0.5812748871363789, - "nauc_precision_at_10_std": 0.33029383423566167, - "nauc_precision_at_1_diff1": 0.6328181504850688, - "nauc_precision_at_1_max": 0.516415613700369, - "nauc_precision_at_1_std": 0.1613423219822137, - "nauc_precision_at_20_diff1": 0.42157941184567777, - "nauc_precision_at_20_max": 0.6175782422623981, - "nauc_precision_at_20_std": 0.42480252175436756, - "nauc_precision_at_3_diff1": 0.511276850553546, - "nauc_precision_at_3_max": 0.5433888785986725, - "nauc_precision_at_3_std": 0.21539500930972955, - "nauc_precision_at_5_diff1": 0.4767674258764569, - "nauc_precision_at_5_max": 0.5603999867601905, - "nauc_precision_at_5_std": 0.2600057462379625, - "nauc_recall_at_1000_diff1": 0.4610798974007453, - "nauc_recall_at_1000_max": 0.8955736220454759, - "nauc_recall_at_1000_std": 0.9412424160278959, - "nauc_recall_at_100_diff1": 0.4108601422390299, - "nauc_recall_at_100_max": 0.6956175426234881, - "nauc_recall_at_100_std": 0.6343823723345061, - "nauc_recall_at_10_diff1": 0.4540269841800468, - "nauc_recall_at_10_max": 0.5812748871363784, - "nauc_recall_at_10_std": 0.330293834235662, - "nauc_recall_at_1_diff1": 0.6328181504850688, - "nauc_recall_at_1_max": 0.516415613700369, - "nauc_recall_at_1_std": 0.1613423219822137, - "nauc_recall_at_20_diff1": 0.4215794118456787, - "nauc_recall_at_20_max": 0.6175782422623985, - "nauc_recall_at_20_std": 0.42480252175436806, - "nauc_recall_at_3_diff1": 0.5112768505535457, - "nauc_recall_at_3_max": 0.5433888785986725, - "nauc_recall_at_3_std": 0.21539500930972913, - "nauc_recall_at_5_diff1": 0.4767674258764573, - "nauc_recall_at_5_max": 0.5603999867601904, - "nauc_recall_at_5_std": 0.26000574623796224, - "ndcg_at_1": 0.36768, - "ndcg_at_10": 0.51408, - "ndcg_at_100": 0.56075, - "ndcg_at_1000": 0.57214, - "ndcg_at_20": 0.53437, - "ndcg_at_3": 0.45894, - "ndcg_at_5": 0.4862, - "precision_at_1": 0.36768, - "precision_at_10": 0.06756, - "precision_at_100": 0.00897, - "precision_at_1000": 0.00099, - "precision_at_20": 0.03776, - "precision_at_3": 0.17451, - "precision_at_5": 0.11793, - "recall_at_1": 0.36768, - "recall_at_10": 0.67556, - "recall_at_100": 0.89723, - "recall_at_1000": 0.98708, - "recall_at_20": 0.75516, - "recall_at_3": 0.52352, - "recall_at_5": 0.58964 - }, - { - "cv_recall_at_1": 0.6875, - "cv_recall_at_10": 0.94847, - "cv_recall_at_100": 0.99667, - "cv_recall_at_1000": 0.99958, - "cv_recall_at_20": 0.97556, - "cv_recall_at_3": 0.8525, - "cv_recall_at_5": 0.905, - "hf_subset": "ru", - "languages": [ - "rus-Cyrl" - ], - "main_score": 0.82082, - "map_at_1": 0.6875, - "map_at_10": 0.77952, - "map_at_100": 0.78212, - "map_at_1000": 0.78214, - "map_at_20": 0.78149, - "map_at_3": 0.76155, - "map_at_5": 0.77351, - "mrr_at_1": 0.6875, - "mrr_at_10": 0.7795203373015837, - "mrr_at_100": 0.7821174627343088, - "mrr_at_1000": 0.7821387642208056, - "mrr_at_20": 0.7814853532093554, - "mrr_at_3": 0.7615509259259242, - "mrr_at_5": 0.7735092592592552, - "nauc_cv_recall_at_1000_diff1": 0.22455648926279748, - "nauc_cv_recall_at_1000_max": 0.7424525365703915, - "nauc_cv_recall_at_1000_std": 0.28275754746343273, - "nauc_cv_recall_at_100_diff1": 0.6293183940242689, - "nauc_cv_recall_at_100_max": 0.6672307812013722, - "nauc_cv_recall_at_100_std": 0.021241830065380926, - "nauc_cv_recall_at_10_diff1": 0.6660374841760586, - "nauc_cv_recall_at_10_max": 0.5524939535562631, - "nauc_cv_recall_at_10_std": -0.2843640600894453, - "nauc_cv_recall_at_1_diff1": 0.7775443213802683, - "nauc_cv_recall_at_1_max": 0.4261422426923034, - "nauc_cv_recall_at_1_std": -0.2797555774877853, - "nauc_cv_recall_at_20_diff1": 0.6828128978864284, - "nauc_cv_recall_at_20_max": 0.6325571895424784, - "nauc_cv_recall_at_20_std": -0.14440094219505392, - "nauc_cv_recall_at_3_diff1": 0.6895588157008296, - "nauc_cv_recall_at_3_max": 0.4706256505146292, - "nauc_cv_recall_at_3_std": -0.350748309809257, - "nauc_cv_recall_at_5_diff1": 0.6537804205502865, - "nauc_cv_recall_at_5_max": 0.48176268558105567, - "nauc_cv_recall_at_5_std": -0.35012299266685093, - "nauc_map_at_1000_diff1": 0.7523940563023707, - "nauc_map_at_1000_max": 0.4397824911340208, - "nauc_map_at_1000_std": -0.2945689733295087, - "nauc_map_at_100_diff1": 0.7523891794383645, - "nauc_map_at_100_max": 0.4398054057499954, - "nauc_map_at_100_std": -0.2945364909899665, - "nauc_map_at_10_diff1": 0.7511612685448151, - "nauc_map_at_10_max": 0.44062613325240885, - "nauc_map_at_10_std": -0.2953424639709468, - "nauc_map_at_1_diff1": 0.7775443213802683, - "nauc_map_at_1_max": 0.4261422426923034, - "nauc_map_at_1_std": -0.2797555774877853, - "nauc_map_at_20_diff1": 0.7521905452599742, - "nauc_map_at_20_max": 0.4403356639427312, - "nauc_map_at_20_std": -0.2942524185462377, - "nauc_map_at_3_diff1": 0.7485436882265807, - "nauc_map_at_3_max": 0.43994611267483863, - "nauc_map_at_3_std": -0.30159040020254646, - "nauc_map_at_5_diff1": 0.747667991627285, - "nauc_map_at_5_max": 0.4393237311004251, - "nauc_map_at_5_std": -0.29926015528317973, - "nauc_mrr_at_1000_diff1": 0.7523940563023707, - "nauc_mrr_at_1000_max": 0.4397824911340208, - "nauc_mrr_at_1000_std": -0.2945689733295087, - "nauc_mrr_at_100_diff1": 0.7523891794383645, - "nauc_mrr_at_100_max": 0.4398054057499954, - "nauc_mrr_at_100_std": -0.2945364909899665, - "nauc_mrr_at_10_diff1": 0.7511612685448151, - "nauc_mrr_at_10_max": 0.44062613325240885, - "nauc_mrr_at_10_std": -0.2953424639709468, - "nauc_mrr_at_1_diff1": 0.7775443213802683, - "nauc_mrr_at_1_max": 0.4261422426923034, - "nauc_mrr_at_1_std": -0.2797555774877853, - "nauc_mrr_at_20_diff1": 0.7521905452599742, - "nauc_mrr_at_20_max": 0.4403356639427312, - "nauc_mrr_at_20_std": -0.2942524185462377, - "nauc_mrr_at_3_diff1": 0.7485436882265807, - "nauc_mrr_at_3_max": 0.43994611267483863, - "nauc_mrr_at_3_std": -0.30159040020254646, - "nauc_mrr_at_5_diff1": 0.747667991627285, - "nauc_mrr_at_5_max": 0.4393237311004251, - "nauc_mrr_at_5_std": -0.29926015528317973, - "nauc_ndcg_at_1000_diff1": 0.7489550956775304, - "nauc_ndcg_at_1000_max": 0.44306305389667855, - "nauc_ndcg_at_1000_std": -0.29462236952032134, - "nauc_ndcg_at_100_diff1": 0.7487981693510771, - "nauc_ndcg_at_100_max": 0.4436446927443267, - "nauc_ndcg_at_100_std": -0.2938447251654681, - "nauc_ndcg_at_10_diff1": 0.7430649301150828, - "nauc_ndcg_at_10_max": 0.44957064968735844, - "nauc_ndcg_at_10_std": -0.295938010067524, - "nauc_ndcg_at_1_diff1": 0.7775443213802683, - "nauc_ndcg_at_1_max": 0.4261422426923034, - "nauc_ndcg_at_1_std": -0.2797555774877853, - "nauc_ndcg_at_20_diff1": 0.747269951253347, - "nauc_ndcg_at_20_max": 0.44835823024848326, - "nauc_ndcg_at_20_std": -0.2908477349842589, - "nauc_ndcg_at_3_diff1": 0.7373188355954172, - "nauc_ndcg_at_3_max": 0.44574261091161155, - "nauc_ndcg_at_3_std": -0.31086889192437706, - "nauc_ndcg_at_5_diff1": 0.7343963293135684, - "nauc_ndcg_at_5_max": 0.4453075054682102, - "nauc_ndcg_at_5_std": -0.30669292975040996, - "nauc_precision_at_1000_diff1": 0.2245564892621984, - "nauc_precision_at_1000_max": 0.7424525365701791, - "nauc_precision_at_1000_std": 0.2827575474633392, - "nauc_precision_at_100_diff1": 0.6293183940242882, - "nauc_precision_at_100_max": 0.6672307812013673, - "nauc_precision_at_100_std": 0.021241830065376183, - "nauc_precision_at_10_diff1": 0.6660374841760579, - "nauc_precision_at_10_max": 0.5524939535562643, - "nauc_precision_at_10_std": -0.2843640600894466, - "nauc_precision_at_1_diff1": 0.7775443213802683, - "nauc_precision_at_1_max": 0.4261422426923034, - "nauc_precision_at_1_std": -0.2797555774877853, - "nauc_precision_at_20_diff1": 0.6828128978864274, - "nauc_precision_at_20_max": 0.6325571895424831, - "nauc_precision_at_20_std": -0.1444009421950585, - "nauc_precision_at_3_diff1": 0.6895588157008311, - "nauc_precision_at_3_max": 0.47062565051463034, - "nauc_precision_at_3_std": -0.3507483098092554, - "nauc_precision_at_5_diff1": 0.6537804205502861, - "nauc_precision_at_5_max": 0.4817626855810538, - "nauc_precision_at_5_std": -0.35012299266685065, - "nauc_recall_at_1000_diff1": 0.22455648926279748, - "nauc_recall_at_1000_max": 0.7424525365703915, - "nauc_recall_at_1000_std": 0.28275754746343273, - "nauc_recall_at_100_diff1": 0.6293183940242689, - "nauc_recall_at_100_max": 0.6672307812013722, - "nauc_recall_at_100_std": 0.021241830065380926, - "nauc_recall_at_10_diff1": 0.6660374841760586, - "nauc_recall_at_10_max": 0.5524939535562631, - "nauc_recall_at_10_std": -0.2843640600894453, - "nauc_recall_at_1_diff1": 0.7775443213802683, - "nauc_recall_at_1_max": 0.4261422426923034, - "nauc_recall_at_1_std": -0.2797555774877853, - "nauc_recall_at_20_diff1": 0.6828128978864284, - "nauc_recall_at_20_max": 0.6325571895424784, - "nauc_recall_at_20_std": -0.14440094219505392, - "nauc_recall_at_3_diff1": 0.6895588157008296, - "nauc_recall_at_3_max": 0.4706256505146292, - "nauc_recall_at_3_std": -0.350748309809257, - "nauc_recall_at_5_diff1": 0.6537804205502865, - "nauc_recall_at_5_max": 0.48176268558105567, - "nauc_recall_at_5_std": -0.35012299266685093, - "ndcg_at_1": 0.6875, - "ndcg_at_10": 0.82082, - "ndcg_at_100": 0.83187, - "ndcg_at_1000": 0.83228, - "ndcg_at_20": 0.82778, - "ndcg_at_3": 0.78497, - "ndcg_at_5": 0.80655, - "precision_at_1": 0.6875, - "precision_at_10": 0.09485, - "precision_at_100": 0.00997, - "precision_at_1000": 0.001, - "precision_at_20": 0.04878, - "precision_at_3": 0.28417, - "precision_at_5": 0.181, - "recall_at_1": 0.6875, - "recall_at_10": 0.94847, - "recall_at_100": 0.99667, - "recall_at_1000": 0.99958, - "recall_at_20": 0.97556, - "recall_at_3": 0.8525, - "recall_at_5": 0.905 - }, - { - "cv_recall_at_1": 0.44191, - "cv_recall_at_10": 0.75375, - "cv_recall_at_100": 0.93249, - "cv_recall_at_1000": 0.99161, - "cv_recall_at_20": 0.82511, - "cv_recall_at_3": 0.60608, - "cv_recall_at_5": 0.67042, - "hf_subset": "sv", - "languages": [ - "swe-Latn" - ], - "main_score": 0.59198, - "map_at_1": 0.44191, - "map_at_10": 0.5409, - "map_at_100": 0.54873, - "map_at_1000": 0.54902, - "map_at_20": 0.54593, - "map_at_3": 0.51476, - "map_at_5": 0.52958, - "mrr_at_1": 0.44190842843393374, - "mrr_at_10": 0.5408954406273279, - "mrr_at_100": 0.5487300488511478, - "mrr_at_1000": 0.5490215246024334, - "mrr_at_20": 0.5459287610986369, - "mrr_at_3": 0.5147577799165894, - "mrr_at_5": 0.5295797240936813, - "nauc_cv_recall_at_1000_diff1": 0.6018887730673989, - "nauc_cv_recall_at_1000_max": 0.9335011023870879, - "nauc_cv_recall_at_1000_std": 0.9682392039474562, - "nauc_cv_recall_at_100_diff1": 0.5028102786422908, - "nauc_cv_recall_at_100_max": 0.7927753764073203, - "nauc_cv_recall_at_100_std": 0.7094087560669563, - "nauc_cv_recall_at_10_diff1": 0.5276450066271778, - "nauc_cv_recall_at_10_max": 0.6308089434631886, - "nauc_cv_recall_at_10_std": 0.30694843082843076, - "nauc_cv_recall_at_1_diff1": 0.6637448075588903, - "nauc_cv_recall_at_1_max": 0.5210610346172718, - "nauc_cv_recall_at_1_std": 0.10086716825348936, - "nauc_cv_recall_at_20_diff1": 0.51394371240035, - "nauc_cv_recall_at_20_max": 0.6816926536113006, - "nauc_cv_recall_at_20_std": 0.4167650335983907, - "nauc_cv_recall_at_3_diff1": 0.5634036392278551, - "nauc_cv_recall_at_3_max": 0.562466498612833, - "nauc_cv_recall_at_3_std": 0.17652617589734698, - "nauc_cv_recall_at_5_diff1": 0.5507036238602347, - "nauc_cv_recall_at_5_max": 0.590022693778581, - "nauc_cv_recall_at_5_std": 0.22280541751586838, - "nauc_map_at_1000_diff1": 0.6205079039809748, - "nauc_map_at_1000_max": 0.5475945312783034, - "nauc_map_at_1000_std": 0.14706532562026975, - "nauc_map_at_100_diff1": 0.6203751026229967, - "nauc_map_at_100_max": 0.5477794610615384, - "nauc_map_at_100_std": 0.1474929591918372, - "nauc_map_at_10_diff1": 0.6191456711326916, - "nauc_map_at_10_max": 0.547395199632512, - "nauc_map_at_10_std": 0.14573244740440383, - "nauc_map_at_1_diff1": 0.6637448075588903, - "nauc_map_at_1_max": 0.5210610346172718, - "nauc_map_at_1_std": 0.10086716825348936, - "nauc_map_at_20_diff1": 0.6196065865981142, - "nauc_map_at_20_max": 0.5481024012855527, - "nauc_map_at_20_std": 0.1477613806133426, - "nauc_map_at_3_diff1": 0.6209787798337106, - "nauc_map_at_3_max": 0.5398502671603651, - "nauc_map_at_3_std": 0.13226616364245533, - "nauc_map_at_5_diff1": 0.6200512056953228, - "nauc_map_at_5_max": 0.5447501900175638, - "nauc_map_at_5_std": 0.1398100581454376, - "nauc_mrr_at_1000_diff1": 0.6205079039809748, - "nauc_mrr_at_1000_max": 0.5475945312783034, - "nauc_mrr_at_1000_std": 0.14706532562026975, - "nauc_mrr_at_100_diff1": 0.6203751026229967, - "nauc_mrr_at_100_max": 0.5477794610615384, - "nauc_mrr_at_100_std": 0.1474929591918372, - "nauc_mrr_at_10_diff1": 0.6191456711326916, - "nauc_mrr_at_10_max": 0.547395199632512, - "nauc_mrr_at_10_std": 0.14573244740440383, - "nauc_mrr_at_1_diff1": 0.6637448075588903, - "nauc_mrr_at_1_max": 0.5210610346172718, - "nauc_mrr_at_1_std": 0.10086716825348936, - "nauc_mrr_at_20_diff1": 0.6196065865981142, - "nauc_mrr_at_20_max": 0.5481024012855527, - "nauc_mrr_at_20_std": 0.1477613806133426, - "nauc_mrr_at_3_diff1": 0.6209787798337106, - "nauc_mrr_at_3_max": 0.5398502671603651, - "nauc_mrr_at_3_std": 0.13226616364245533, - "nauc_mrr_at_5_diff1": 0.6200512056953228, - "nauc_mrr_at_5_max": 0.5447501900175638, - "nauc_mrr_at_5_std": 0.1398100581454376, - "nauc_ndcg_at_1000_diff1": 0.6123639151635346, - "nauc_ndcg_at_1000_max": 0.5579031740640773, - "nauc_ndcg_at_1000_std": 0.16845132021900988, - "nauc_ndcg_at_100_diff1": 0.6087985522272299, - "nauc_ndcg_at_100_max": 0.5638215386490731, - "nauc_ndcg_at_100_std": 0.1823456244021935, - "nauc_ndcg_at_10_diff1": 0.6018289811050774, - "nauc_ndcg_at_10_max": 0.5622477594382406, - "nauc_ndcg_at_10_std": 0.1742020463281658, - "nauc_ndcg_at_1_diff1": 0.6637448075588903, - "nauc_ndcg_at_1_max": 0.5210610346172718, - "nauc_ndcg_at_1_std": 0.10086716825348936, - "nauc_ndcg_at_20_diff1": 0.6032490714691485, - "nauc_ndcg_at_20_max": 0.5661797134702593, - "nauc_ndcg_at_20_std": 0.18382793567563085, - "nauc_ndcg_at_3_diff1": 0.6069939876443473, - "nauc_ndcg_at_3_max": 0.5453946451688797, - "nauc_ndcg_at_3_std": 0.1429789340404351, - "nauc_ndcg_at_5_diff1": 0.6045274257458881, - "nauc_ndcg_at_5_max": 0.5546680908480792, - "nauc_ndcg_at_5_std": 0.15784285604573, - "nauc_precision_at_1000_diff1": 0.6018887730673832, - "nauc_precision_at_1000_max": 0.9335011023870675, - "nauc_precision_at_1000_std": 0.9682392039474488, - "nauc_precision_at_100_diff1": 0.5028102786422958, - "nauc_precision_at_100_max": 0.7927753764073243, - "nauc_precision_at_100_std": 0.7094087560669561, - "nauc_precision_at_10_diff1": 0.5276450066271775, - "nauc_precision_at_10_max": 0.6308089434631875, - "nauc_precision_at_10_std": 0.3069484308284308, - "nauc_precision_at_1_diff1": 0.6637448075588903, - "nauc_precision_at_1_max": 0.5210610346172718, - "nauc_precision_at_1_std": 0.10086716825348936, - "nauc_precision_at_20_diff1": 0.5139437124003483, - "nauc_precision_at_20_max": 0.6816926536113007, - "nauc_precision_at_20_std": 0.4167650335983901, - "nauc_precision_at_3_diff1": 0.563403639227856, - "nauc_precision_at_3_max": 0.5624664986128333, - "nauc_precision_at_3_std": 0.17652617589734707, - "nauc_precision_at_5_diff1": 0.5507036238602353, - "nauc_precision_at_5_max": 0.5900226937785804, - "nauc_precision_at_5_std": 0.22280541751586855, - "nauc_recall_at_1000_diff1": 0.6018887730673989, - "nauc_recall_at_1000_max": 0.9335011023870879, - "nauc_recall_at_1000_std": 0.9682392039474562, - "nauc_recall_at_100_diff1": 0.5028102786422908, - "nauc_recall_at_100_max": 0.7927753764073203, - "nauc_recall_at_100_std": 0.7094087560669563, - "nauc_recall_at_10_diff1": 0.5276450066271778, - "nauc_recall_at_10_max": 0.6308089434631886, - "nauc_recall_at_10_std": 0.30694843082843076, - "nauc_recall_at_1_diff1": 0.6637448075588903, - "nauc_recall_at_1_max": 0.5210610346172718, - "nauc_recall_at_1_std": 0.10086716825348936, - "nauc_recall_at_20_diff1": 0.51394371240035, - "nauc_recall_at_20_max": 0.6816926536113006, - "nauc_recall_at_20_std": 0.4167650335983907, - "nauc_recall_at_3_diff1": 0.5634036392278551, - "nauc_recall_at_3_max": 0.562466498612833, - "nauc_recall_at_3_std": 0.17652617589734698, - "nauc_recall_at_5_diff1": 0.5507036238602347, - "nauc_recall_at_5_max": 0.590022693778581, - "nauc_recall_at_5_std": 0.22280541751586838, - "ndcg_at_1": 0.44191, - "ndcg_at_10": 0.59198, - "ndcg_at_100": 0.63016, - "ndcg_at_1000": 0.63771, - "ndcg_at_20": 0.61012, - "ndcg_at_3": 0.53823, - "ndcg_at_5": 0.56484, - "precision_at_1": 0.44191, - "precision_at_10": 0.07537, - "precision_at_100": 0.00932, - "precision_at_1000": 0.00099, - "precision_at_20": 0.04126, - "precision_at_3": 0.20203, - "precision_at_5": 0.13408, - "recall_at_1": 0.44191, - "recall_at_10": 0.75375, - "recall_at_100": 0.93249, - "recall_at_1000": 0.99161, - "recall_at_20": 0.82511, - "recall_at_3": 0.60608, - "recall_at_5": 0.67042 - }, - { - "cv_recall_at_1": 0.03321, - "cv_recall_at_10": 0.10914, - "cv_recall_at_100": 0.27462, - "cv_recall_at_1000": 0.65995, - "cv_recall_at_20": 0.14647, - "cv_recall_at_3": 0.06117, - "cv_recall_at_5": 0.07934, - "hf_subset": "sw", - "languages": [ - "swa-Latn" - ], - "main_score": 0.06635, - "map_at_1": 0.03321, - "map_at_10": 0.05328, - "map_at_100": 0.05867, - "map_at_1000": 0.05996, - "map_at_20": 0.05579, - "map_at_3": 0.0452, - "map_at_5": 0.04938, - "mrr_at_1": 0.033210332103321034, - "mrr_at_10": 0.05327907537317126, - "mrr_at_100": 0.05867479582017056, - "mrr_at_1000": 0.059961352964132415, - "mrr_at_20": 0.05578620053772376, - "mrr_at_3": 0.045202952029520266, - "mrr_at_5": 0.04938262844166893, - "nauc_cv_recall_at_1000_diff1": 0.09433291988275253, - "nauc_cv_recall_at_1000_max": 0.3899071785651053, - "nauc_cv_recall_at_1000_std": 0.5473068856604867, - "nauc_cv_recall_at_100_diff1": 0.13672128250729124, - "nauc_cv_recall_at_100_max": 0.4333965593081663, - "nauc_cv_recall_at_100_std": 0.5290505913112106, - "nauc_cv_recall_at_10_diff1": 0.20374907353063168, - "nauc_cv_recall_at_10_max": 0.528916904832453, - "nauc_cv_recall_at_10_std": 0.5137024217892807, - "nauc_cv_recall_at_1_diff1": 0.4569530640746109, - "nauc_cv_recall_at_1_max": 0.6778468188971734, - "nauc_cv_recall_at_1_std": 0.5496011931528505, - "nauc_cv_recall_at_20_diff1": 0.18314325393687095, - "nauc_cv_recall_at_20_max": 0.5104320443662624, - "nauc_cv_recall_at_20_std": 0.5336469498387514, - "nauc_cv_recall_at_3_diff1": 0.3050165529276982, - "nauc_cv_recall_at_3_max": 0.6152380149146236, - "nauc_cv_recall_at_3_std": 0.5439195330633968, - "nauc_cv_recall_at_5_diff1": 0.26018121898972457, - "nauc_cv_recall_at_5_max": 0.5685319189687142, - "nauc_cv_recall_at_5_std": 0.5204816360035446, - "nauc_map_at_1000_diff1": 0.31171531791929985, - "nauc_map_at_1000_max": 0.5920984209133583, - "nauc_map_at_1000_std": 0.5351020291315202, - "nauc_map_at_100_diff1": 0.31233497958791867, - "nauc_map_at_100_max": 0.5928125136050744, - "nauc_map_at_100_std": 0.5347639542407587, - "nauc_map_at_10_diff1": 0.32661816730495424, - "nauc_map_at_10_max": 0.6063397910374565, - "nauc_map_at_10_std": 0.5326493130326762, - "nauc_map_at_1_diff1": 0.4569530640746109, - "nauc_map_at_1_max": 0.6778468188971734, - "nauc_map_at_1_std": 0.5496011931528505, - "nauc_map_at_20_diff1": 0.31983437901763073, - "nauc_map_at_20_max": 0.6018794532035682, - "nauc_map_at_20_std": 0.5357468910437008, - "nauc_map_at_3_diff1": 0.3703688368182306, - "nauc_map_at_3_max": 0.6405968692379383, - "nauc_map_at_3_std": 0.5451280846967744, - "nauc_map_at_5_diff1": 0.3484623067657344, - "nauc_map_at_5_max": 0.6212741811071596, - "nauc_map_at_5_std": 0.5368538853770519, - "nauc_mrr_at_1000_diff1": 0.31171531791929985, - "nauc_mrr_at_1000_max": 0.5920984209133583, - "nauc_mrr_at_1000_std": 0.5351020291315202, - "nauc_mrr_at_100_diff1": 0.31233497958791867, - "nauc_mrr_at_100_max": 0.5928125136050744, - "nauc_mrr_at_100_std": 0.5347639542407587, - "nauc_mrr_at_10_diff1": 0.32661816730495424, - "nauc_mrr_at_10_max": 0.6063397910374565, - "nauc_mrr_at_10_std": 0.5326493130326762, - "nauc_mrr_at_1_diff1": 0.4569530640746109, - "nauc_mrr_at_1_max": 0.6778468188971734, - "nauc_mrr_at_1_std": 0.5496011931528505, - "nauc_mrr_at_20_diff1": 0.31983437901763073, - "nauc_mrr_at_20_max": 0.6018794532035682, - "nauc_mrr_at_20_std": 0.5357468910437008, - "nauc_mrr_at_3_diff1": 0.3703688368182306, - "nauc_mrr_at_3_max": 0.6405968692379383, - "nauc_mrr_at_3_std": 0.5451280846967744, - "nauc_mrr_at_5_diff1": 0.3484623067657344, - "nauc_mrr_at_5_max": 0.6212741811071596, - "nauc_mrr_at_5_std": 0.5368538853770519, - "nauc_ndcg_at_1000_diff1": 0.22668553918732787, - "nauc_ndcg_at_1000_max": 0.5173234138399592, - "nauc_ndcg_at_1000_std": 0.5363578196999412, - "nauc_ndcg_at_100_diff1": 0.23407015585595642, - "nauc_ndcg_at_100_max": 0.5254226946529976, - "nauc_ndcg_at_100_std": 0.5311337513021962, - "nauc_ndcg_at_10_diff1": 0.28018176043152276, - "nauc_ndcg_at_10_max": 0.5773268253715648, - "nauc_ndcg_at_10_std": 0.5253766421907764, - "nauc_ndcg_at_1_diff1": 0.4569530640746109, - "nauc_ndcg_at_1_max": 0.6778468188971734, - "nauc_ndcg_at_1_std": 0.5496011931528505, - "nauc_ndcg_at_20_diff1": 0.26444563897191153, - "nauc_ndcg_at_20_max": 0.5655565090014336, - "nauc_ndcg_at_20_std": 0.5339288064473993, - "nauc_ndcg_at_3_diff1": 0.34967108251727236, - "nauc_ndcg_at_3_max": 0.6325070835957037, - "nauc_ndcg_at_3_std": 0.5446994265508098, - "nauc_ndcg_at_5_diff1": 0.3180407180415853, - "nauc_ndcg_at_5_max": 0.6033328171533652, - "nauc_ndcg_at_5_std": 0.531442568868304, - "nauc_precision_at_1000_diff1": 0.09433291988275161, - "nauc_precision_at_1000_max": 0.3899071785651051, - "nauc_precision_at_1000_std": 0.5473068856604861, - "nauc_precision_at_100_diff1": 0.13672128250729124, - "nauc_precision_at_100_max": 0.4333965593081666, - "nauc_precision_at_100_std": 0.5290505913112108, - "nauc_precision_at_10_diff1": 0.20374907353063154, - "nauc_precision_at_10_max": 0.5289169048324531, - "nauc_precision_at_10_std": 0.513702421789281, - "nauc_precision_at_1_diff1": 0.4569530640746109, - "nauc_precision_at_1_max": 0.6778468188971734, - "nauc_precision_at_1_std": 0.5496011931528505, - "nauc_precision_at_20_diff1": 0.18314325393687095, - "nauc_precision_at_20_max": 0.5104320443662624, - "nauc_precision_at_20_std": 0.5336469498387516, - "nauc_precision_at_3_diff1": 0.3050165529276982, - "nauc_precision_at_3_max": 0.6152380149146237, - "nauc_precision_at_3_std": 0.5439195330633968, - "nauc_precision_at_5_diff1": 0.2601812189897246, - "nauc_precision_at_5_max": 0.5685319189687144, - "nauc_precision_at_5_std": 0.5204816360035447, - "nauc_recall_at_1000_diff1": 0.09433291988275253, - "nauc_recall_at_1000_max": 0.3899071785651053, - "nauc_recall_at_1000_std": 0.5473068856604867, - "nauc_recall_at_100_diff1": 0.13672128250729124, - "nauc_recall_at_100_max": 0.4333965593081663, - "nauc_recall_at_100_std": 0.5290505913112106, - "nauc_recall_at_10_diff1": 0.20374907353063168, - "nauc_recall_at_10_max": 0.528916904832453, - "nauc_recall_at_10_std": 0.5137024217892807, - "nauc_recall_at_1_diff1": 0.4569530640746109, - "nauc_recall_at_1_max": 0.6778468188971734, - "nauc_recall_at_1_std": 0.5496011931528505, - "nauc_recall_at_20_diff1": 0.18314325393687095, - "nauc_recall_at_20_max": 0.5104320443662624, - "nauc_recall_at_20_std": 0.5336469498387514, - "nauc_recall_at_3_diff1": 0.3050165529276982, - "nauc_recall_at_3_max": 0.6152380149146236, - "nauc_recall_at_3_std": 0.5439195330633968, - "nauc_recall_at_5_diff1": 0.26018121898972457, - "nauc_recall_at_5_max": 0.5685319189687142, - "nauc_recall_at_5_std": 0.5204816360035446, - "ndcg_at_1": 0.03321, - "ndcg_at_10": 0.06635, - "ndcg_at_100": 0.09866, - "ndcg_at_1000": 0.14433, - "ndcg_at_20": 0.07569, - "ndcg_at_3": 0.04929, - "ndcg_at_5": 0.0568, - "precision_at_1": 0.03321, - "precision_at_10": 0.01091, - "precision_at_100": 0.00275, - "precision_at_1000": 0.00066, - "precision_at_20": 0.00732, - "precision_at_3": 0.02039, - "precision_at_5": 0.01587, - "recall_at_1": 0.03321, - "recall_at_10": 0.10914, - "recall_at_100": 0.27462, - "recall_at_1000": 0.65995, - "recall_at_20": 0.14647, - "recall_at_3": 0.06117, - "recall_at_5": 0.07934 - }, - { - "cv_recall_at_1": 0.0175, - "cv_recall_at_10": 0.07792, - "cv_recall_at_100": 0.27306, - "cv_recall_at_1000": 0.69653, - "cv_recall_at_20": 0.11569, - "cv_recall_at_3": 0.03625, - "cv_recall_at_5": 0.05181, - "hf_subset": "te", - "languages": [ - "tel-Telu" - ], - "main_score": 0.04301, - "map_at_1": 0.0175, - "map_at_10": 0.03245, - "map_at_100": 0.03862, - "map_at_1000": 0.0401, - "map_at_20": 0.03506, - "map_at_3": 0.02551, - "map_at_5": 0.02903, - "mrr_at_1": 0.0175, - "mrr_at_10": 0.032452270723104, - "mrr_at_100": 0.03861571397773774, - "mrr_at_1000": 0.040102337223021915, - "mrr_at_20": 0.035063106461487484, - "mrr_at_3": 0.02550925925925927, - "mrr_at_5": 0.02903009259259258, - "nauc_cv_recall_at_1000_diff1": 0.05792005006960345, - "nauc_cv_recall_at_1000_max": 0.3388544839738773, - "nauc_cv_recall_at_1000_std": 0.4881706837009283, - "nauc_cv_recall_at_100_diff1": 0.04207486335429808, - "nauc_cv_recall_at_100_max": 0.26806131714561543, - "nauc_cv_recall_at_100_std": 0.40190030075205696, - "nauc_cv_recall_at_10_diff1": 0.04582783914844086, - "nauc_cv_recall_at_10_max": 0.2544246422622029, - "nauc_cv_recall_at_10_std": 0.3615142571407665, - "nauc_cv_recall_at_1_diff1": 0.2181847607122626, - "nauc_cv_recall_at_1_max": 0.36452996367136414, - "nauc_cv_recall_at_1_std": 0.3431075676849756, - "nauc_cv_recall_at_20_diff1": 0.056574970520330975, - "nauc_cv_recall_at_20_max": 0.2636871580046122, - "nauc_cv_recall_at_20_std": 0.3689271290937749, - "nauc_cv_recall_at_3_diff1": 0.06931709889621863, - "nauc_cv_recall_at_3_max": 0.26279597881052325, - "nauc_cv_recall_at_3_std": 0.3422328724735209, - "nauc_cv_recall_at_5_diff1": 0.06066304073383788, - "nauc_cv_recall_at_5_max": 0.27006102127004533, - "nauc_cv_recall_at_5_std": 0.36088581908268846, - "nauc_map_at_1000_diff1": 0.10326350163816736, - "nauc_map_at_1000_max": 0.2928593760781644, - "nauc_map_at_1000_std": 0.36085285884092044, - "nauc_map_at_100_diff1": 0.10282362577640569, - "nauc_map_at_100_max": 0.29160155095442103, - "nauc_map_at_100_std": 0.3589087984267018, - "nauc_map_at_10_diff1": 0.10783350494599624, - "nauc_map_at_10_max": 0.29312380079082967, - "nauc_map_at_10_std": 0.35285556380874783, - "nauc_map_at_1_diff1": 0.2181847607122626, - "nauc_map_at_1_max": 0.36452996367136414, - "nauc_map_at_1_std": 0.3431075676849756, - "nauc_map_at_20_diff1": 0.10668443333534292, - "nauc_map_at_20_max": 0.29228742620962267, - "nauc_map_at_20_std": 0.3541220639866919, - "nauc_map_at_3_diff1": 0.12890605027228108, - "nauc_map_at_3_max": 0.3036582022490514, - "nauc_map_at_3_std": 0.3447869254476457, - "nauc_map_at_5_diff1": 0.1187492842098791, - "nauc_map_at_5_max": 0.3028427904848807, - "nauc_map_at_5_std": 0.35328503511968795, - "nauc_mrr_at_1000_diff1": 0.10326350163816736, - "nauc_mrr_at_1000_max": 0.2928593760781644, - "nauc_mrr_at_1000_std": 0.36085285884092044, - "nauc_mrr_at_100_diff1": 0.10282362577640569, - "nauc_mrr_at_100_max": 0.29160155095442103, - "nauc_mrr_at_100_std": 0.3589087984267018, - "nauc_mrr_at_10_diff1": 0.10783350494599624, - "nauc_mrr_at_10_max": 0.29312380079082967, - "nauc_mrr_at_10_std": 0.35285556380874783, - "nauc_mrr_at_1_diff1": 0.2181847607122626, - "nauc_mrr_at_1_max": 0.36452996367136414, - "nauc_mrr_at_1_std": 0.3431075676849756, - "nauc_mrr_at_20_diff1": 0.10668443333534292, - "nauc_mrr_at_20_max": 0.29228742620962267, - "nauc_mrr_at_20_std": 0.3541220639866919, - "nauc_mrr_at_3_diff1": 0.12890605027228108, - "nauc_mrr_at_3_max": 0.3036582022490514, - "nauc_mrr_at_3_std": 0.3447869254476457, - "nauc_mrr_at_5_diff1": 0.1187492842098791, - "nauc_mrr_at_5_max": 0.3028427904848807, - "nauc_mrr_at_5_std": 0.35328503511968795, - "nauc_ndcg_at_1000_diff1": 0.07655630184733145, - "nauc_ndcg_at_1000_max": 0.2993228122542535, - "nauc_ndcg_at_1000_std": 0.40555622921563655, - "nauc_ndcg_at_100_diff1": 0.06938793828736407, - "nauc_ndcg_at_100_max": 0.2772667362111841, - "nauc_ndcg_at_100_std": 0.37949298578727964, - "nauc_ndcg_at_10_diff1": 0.0813050607259139, - "nauc_ndcg_at_10_max": 0.2764629145165488, - "nauc_ndcg_at_10_std": 0.35630363144556443, - "nauc_ndcg_at_1_diff1": 0.2181847607122626, - "nauc_ndcg_at_1_max": 0.36452996367136414, - "nauc_ndcg_at_1_std": 0.3431075676849756, - "nauc_ndcg_at_20_diff1": 0.08195382927419637, - "nauc_ndcg_at_20_max": 0.2776948814786933, - "nauc_ndcg_at_20_std": 0.36038885937927667, - "nauc_ndcg_at_3_diff1": 0.10939001881105939, - "nauc_ndcg_at_3_max": 0.2902805163223652, - "nauc_ndcg_at_3_std": 0.344031325078653, - "nauc_ndcg_at_5_diff1": 0.09712085095822343, - "nauc_ndcg_at_5_max": 0.2905608812025033, - "nauc_ndcg_at_5_std": 0.35607370989095327, - "nauc_precision_at_1000_diff1": 0.057920050069601936, - "nauc_precision_at_1000_max": 0.33885448397387585, - "nauc_precision_at_1000_std": 0.4881706837009277, - "nauc_precision_at_100_diff1": 0.04207486335429823, - "nauc_precision_at_100_max": 0.26806131714561515, - "nauc_precision_at_100_std": 0.4019003007520569, - "nauc_precision_at_10_diff1": 0.04582783914844074, - "nauc_precision_at_10_max": 0.25442464226220274, - "nauc_precision_at_10_std": 0.36151425714076646, - "nauc_precision_at_1_diff1": 0.2181847607122626, - "nauc_precision_at_1_max": 0.36452996367136414, - "nauc_precision_at_1_std": 0.3431075676849756, - "nauc_precision_at_20_diff1": 0.05657497052033086, - "nauc_precision_at_20_max": 0.26368715800461207, - "nauc_precision_at_20_std": 0.368927129093775, - "nauc_precision_at_3_diff1": 0.06931709889621865, - "nauc_precision_at_3_max": 0.2627959788105232, - "nauc_precision_at_3_std": 0.3422328724735208, - "nauc_precision_at_5_diff1": 0.060663040733837666, - "nauc_precision_at_5_max": 0.2700610212700452, - "nauc_precision_at_5_std": 0.36088581908268846, - "nauc_recall_at_1000_diff1": 0.05792005006960345, - "nauc_recall_at_1000_max": 0.3388544839738773, - "nauc_recall_at_1000_std": 0.4881706837009283, - "nauc_recall_at_100_diff1": 0.04207486335429808, - "nauc_recall_at_100_max": 0.26806131714561543, - "nauc_recall_at_100_std": 0.40190030075205696, - "nauc_recall_at_10_diff1": 0.04582783914844086, - "nauc_recall_at_10_max": 0.2544246422622029, - "nauc_recall_at_10_std": 0.3615142571407665, - "nauc_recall_at_1_diff1": 0.2181847607122626, - "nauc_recall_at_1_max": 0.36452996367136414, - "nauc_recall_at_1_std": 0.3431075676849756, - "nauc_recall_at_20_diff1": 0.056574970520330975, - "nauc_recall_at_20_max": 0.2636871580046122, - "nauc_recall_at_20_std": 0.3689271290937749, - "nauc_recall_at_3_diff1": 0.06931709889621863, - "nauc_recall_at_3_max": 0.26279597881052325, - "nauc_recall_at_3_std": 0.3422328724735209, - "nauc_recall_at_5_diff1": 0.06066304073383788, - "nauc_recall_at_5_max": 0.27006102127004533, - "nauc_recall_at_5_std": 0.36088581908268846, - "ndcg_at_1": 0.0175, - "ndcg_at_10": 0.04301, - "ndcg_at_100": 0.08077, - "ndcg_at_1000": 0.13143, - "ndcg_at_20": 0.05255, - "ndcg_at_3": 0.02826, - "ndcg_at_5": 0.03463, - "precision_at_1": 0.0175, - "precision_at_10": 0.00779, - "precision_at_100": 0.00273, - "precision_at_1000": 0.0007, - "precision_at_20": 0.00578, - "precision_at_3": 0.01208, - "precision_at_5": 0.01036, - "recall_at_1": 0.0175, - "recall_at_10": 0.07792, - "recall_at_100": 0.27306, - "recall_at_1000": 0.69653, - "recall_at_20": 0.11569, - "recall_at_3": 0.03625, - "recall_at_5": 0.05181 - }, - { - "cv_recall_at_1": 0.40139, - "cv_recall_at_10": 0.74375, - "cv_recall_at_100": 0.94611, - "cv_recall_at_1000": 0.99708, - "cv_recall_at_20": 0.82389, - "cv_recall_at_3": 0.57778, - "cv_recall_at_5": 0.65111, - "hf_subset": "th", - "languages": [ - "tha-Thai" - ], - "main_score": 0.56517, - "map_at_1": 0.40139, - "map_at_10": 0.50891, - "map_at_100": 0.51779, - "map_at_1000": 0.51806, - "map_at_20": 0.51453, - "map_at_3": 0.4797, - "map_at_5": 0.4964, - "mrr_at_1": 0.4013888888888889, - "mrr_at_10": 0.5089053681657874, - "mrr_at_100": 0.5177867851592552, - "mrr_at_1000": 0.5180590992722832, - "mrr_at_20": 0.514527151608116, - "mrr_at_3": 0.4796990740740768, - "mrr_at_5": 0.4964004629629642, - "nauc_cv_recall_at_1000_diff1": 0.5401271619758887, - "nauc_cv_recall_at_1000_max": 0.961095549330813, - "nauc_cv_recall_at_1000_std": 0.9379974211906545, - "nauc_cv_recall_at_100_diff1": 0.448168442634786, - "nauc_cv_recall_at_100_max": 0.7295438794074325, - "nauc_cv_recall_at_100_std": 0.6025585491928738, - "nauc_cv_recall_at_10_diff1": 0.47379036107157435, - "nauc_cv_recall_at_10_max": 0.5529269123201439, - "nauc_cv_recall_at_10_std": 0.24143568087558764, - "nauc_cv_recall_at_1_diff1": 0.6164133423250504, - "nauc_cv_recall_at_1_max": 0.43162943869145476, - "nauc_cv_recall_at_1_std": 0.05381972083937815, - "nauc_cv_recall_at_20_diff1": 0.4589103478632773, - "nauc_cv_recall_at_20_max": 0.5918795480867991, - "nauc_cv_recall_at_20_std": 0.3251142582423592, - "nauc_cv_recall_at_3_diff1": 0.5199180551981475, - "nauc_cv_recall_at_3_max": 0.48830316360010284, - "nauc_cv_recall_at_3_std": 0.12179900604734932, - "nauc_cv_recall_at_5_diff1": 0.4933589969881374, - "nauc_cv_recall_at_5_max": 0.5115624626235463, - "nauc_cv_recall_at_5_std": 0.17317824898789116, - "nauc_map_at_1000_diff1": 0.5728540706664647, - "nauc_map_at_1000_max": 0.4592079702642643, - "nauc_map_at_1000_std": 0.09332696144631226, - "nauc_map_at_100_diff1": 0.5727418480818869, - "nauc_map_at_100_max": 0.45939586065791144, - "nauc_map_at_100_std": 0.09367322367533219, - "nauc_map_at_10_diff1": 0.571122576444961, - "nauc_map_at_10_max": 0.4597304926570427, - "nauc_map_at_10_std": 0.09303375964132092, - "nauc_map_at_1_diff1": 0.6164133423250504, - "nauc_map_at_1_max": 0.43162943869145476, - "nauc_map_at_1_std": 0.05381972083937815, - "nauc_map_at_20_diff1": 0.5718838115024688, - "nauc_map_at_20_max": 0.4598623581030198, - "nauc_map_at_20_std": 0.09418505747711978, - "nauc_map_at_3_diff1": 0.5753151106225249, - "nauc_map_at_3_max": 0.45362185426138985, - "nauc_map_at_3_std": 0.07996167733667833, - "nauc_map_at_5_diff1": 0.5714216051987776, - "nauc_map_at_5_max": 0.4574917315602048, - "nauc_map_at_5_std": 0.08896721117497133, - "nauc_mrr_at_1000_diff1": 0.5728540706664647, - "nauc_mrr_at_1000_max": 0.4592079702642643, - "nauc_mrr_at_1000_std": 0.09332696144631226, - "nauc_mrr_at_100_diff1": 0.5727418480818869, - "nauc_mrr_at_100_max": 0.45939586065791144, - "nauc_mrr_at_100_std": 0.09367322367533219, - "nauc_mrr_at_10_diff1": 0.571122576444961, - "nauc_mrr_at_10_max": 0.4597304926570427, - "nauc_mrr_at_10_std": 0.09303375964132092, - "nauc_mrr_at_1_diff1": 0.6164133423250504, - "nauc_mrr_at_1_max": 0.43162943869145476, - "nauc_mrr_at_1_std": 0.05381972083937815, - "nauc_mrr_at_20_diff1": 0.5718838115024688, - "nauc_mrr_at_20_max": 0.4598623581030198, - "nauc_mrr_at_20_std": 0.09418505747711978, - "nauc_mrr_at_3_diff1": 0.5753151106225249, - "nauc_mrr_at_3_max": 0.45362185426138985, - "nauc_mrr_at_3_std": 0.07996167733667833, - "nauc_mrr_at_5_diff1": 0.5714216051987776, - "nauc_mrr_at_5_max": 0.4574917315602048, - "nauc_mrr_at_5_std": 0.08896721117497133, - "nauc_ndcg_at_1000_diff1": 0.5644879483816287, - "nauc_ndcg_at_1000_max": 0.46864246103483886, - "nauc_ndcg_at_1000_std": 0.10961745828253958, - "nauc_ndcg_at_100_diff1": 0.5614769460541409, - "nauc_ndcg_at_100_max": 0.47454605148821766, - "nauc_ndcg_at_100_std": 0.12083764507191204, - "nauc_ndcg_at_10_diff1": 0.552642154001218, - "nauc_ndcg_at_10_max": 0.4764490613074466, - "nauc_ndcg_at_10_std": 0.11939195000827524, - "nauc_ndcg_at_1_diff1": 0.6164133423250504, - "nauc_ndcg_at_1_max": 0.43162943869145476, - "nauc_ndcg_at_1_std": 0.05381972083937815, - "nauc_ndcg_at_20_diff1": 0.5547211132622595, - "nauc_ndcg_at_20_max": 0.47794734067506456, - "nauc_ndcg_at_20_std": 0.1251463269383499, - "nauc_ndcg_at_3_diff1": 0.5617637090081964, - "nauc_ndcg_at_3_max": 0.46201121733709233, - "nauc_ndcg_at_3_std": 0.09007225455387947, - "nauc_ndcg_at_5_diff1": 0.5539623110541856, - "nauc_ndcg_at_5_max": 0.46943015360238455, - "nauc_ndcg_at_5_std": 0.10731330075560759, - "nauc_precision_at_1000_diff1": 0.5401271619758738, - "nauc_precision_at_1000_max": 0.961095549330848, - "nauc_precision_at_1000_std": 0.9379974211907273, - "nauc_precision_at_100_diff1": 0.4481684426347872, - "nauc_precision_at_100_max": 0.7295438794074368, - "nauc_precision_at_100_std": 0.6025585491928742, - "nauc_precision_at_10_diff1": 0.4737903610715738, - "nauc_precision_at_10_max": 0.5529269123201438, - "nauc_precision_at_10_std": 0.24143568087558764, - "nauc_precision_at_1_diff1": 0.6164133423250504, - "nauc_precision_at_1_max": 0.43162943869145476, - "nauc_precision_at_1_std": 0.05381972083937815, - "nauc_precision_at_20_diff1": 0.4589103478632771, - "nauc_precision_at_20_max": 0.5918795480867984, - "nauc_precision_at_20_std": 0.3251142582423566, - "nauc_precision_at_3_diff1": 0.5199180551981472, - "nauc_precision_at_3_max": 0.4883031636001032, - "nauc_precision_at_3_std": 0.12179900604734974, - "nauc_precision_at_5_diff1": 0.49335899698813696, - "nauc_precision_at_5_max": 0.5115624626235458, - "nauc_precision_at_5_std": 0.17317824898789044, - "nauc_recall_at_1000_diff1": 0.5401271619758887, - "nauc_recall_at_1000_max": 0.961095549330813, - "nauc_recall_at_1000_std": 0.9379974211906545, - "nauc_recall_at_100_diff1": 0.448168442634786, - "nauc_recall_at_100_max": 0.7295438794074325, - "nauc_recall_at_100_std": 0.6025585491928738, - "nauc_recall_at_10_diff1": 0.47379036107157435, - "nauc_recall_at_10_max": 0.5529269123201439, - "nauc_recall_at_10_std": 0.24143568087558764, - "nauc_recall_at_1_diff1": 0.6164133423250504, - "nauc_recall_at_1_max": 0.43162943869145476, - "nauc_recall_at_1_std": 0.05381972083937815, - "nauc_recall_at_20_diff1": 0.4589103478632773, - "nauc_recall_at_20_max": 0.5918795480867991, - "nauc_recall_at_20_std": 0.3251142582423592, - "nauc_recall_at_3_diff1": 0.5199180551981475, - "nauc_recall_at_3_max": 0.48830316360010284, - "nauc_recall_at_3_std": 0.12179900604734932, - "nauc_recall_at_5_diff1": 0.4933589969881374, - "nauc_recall_at_5_max": 0.5115624626235463, - "nauc_recall_at_5_std": 0.17317824898789116, - "ndcg_at_1": 0.40139, - "ndcg_at_10": 0.56517, - "ndcg_at_100": 0.60842, - "ndcg_at_1000": 0.61505, - "ndcg_at_20": 0.5855, - "ndcg_at_3": 0.50491, - "ndcg_at_5": 0.53507, - "precision_at_1": 0.40139, - "precision_at_10": 0.07438, - "precision_at_100": 0.00946, - "precision_at_1000": 0.001, - "precision_at_20": 0.04119, - "precision_at_3": 0.19259, - "precision_at_5": 0.13022, - "recall_at_1": 0.40139, - "recall_at_10": 0.74375, - "recall_at_100": 0.94611, - "recall_at_1000": 0.99708, - "recall_at_20": 0.82389, - "recall_at_3": 0.57778, - "recall_at_5": 0.65111 - }, - { - "cv_recall_at_1": 0.24679, - "cv_recall_at_10": 0.55095, - "cv_recall_at_100": 0.8399, - "cv_recall_at_1000": 0.98092, - "cv_recall_at_20": 0.65464, - "cv_recall_at_3": 0.38366, - "cv_recall_at_5": 0.45279, - "hf_subset": "tr", - "languages": [ - "tur-Latn" - ], - "main_score": 0.38614, - "map_at_1": 0.24679, - "map_at_10": 0.33501, - "map_at_100": 0.34688, - "map_at_1000": 0.34754, - "map_at_20": 0.3422, - "map_at_3": 0.3064, - "map_at_5": 0.3221, - "mrr_at_1": 0.24678556615512234, - "mrr_at_10": 0.335005617989856, - "mrr_at_100": 0.3468761477353053, - "mrr_at_1000": 0.34754482431159706, - "mrr_at_20": 0.3422046159949083, - "mrr_at_3": 0.30639660813862274, - "mrr_at_5": 0.32210240103230525, - "nauc_cv_recall_at_1000_diff1": 0.28818656533654663, - "nauc_cv_recall_at_1000_max": 0.7881388928706085, - "nauc_cv_recall_at_1000_std": 0.8586308516367452, - "nauc_cv_recall_at_100_diff1": 0.3077133849819943, - "nauc_cv_recall_at_100_max": 0.6228904388563352, - "nauc_cv_recall_at_100_std": 0.5832133669363783, - "nauc_cv_recall_at_10_diff1": 0.3333065129762372, - "nauc_cv_recall_at_10_max": 0.4456554564461219, - "nauc_cv_recall_at_10_std": 0.2561096026594872, - "nauc_cv_recall_at_1_diff1": 0.519216885422269, - "nauc_cv_recall_at_1_max": 0.41701938956923496, - "nauc_cv_recall_at_1_std": 0.12515186524794, - "nauc_cv_recall_at_20_diff1": 0.33771403630592267, - "nauc_cv_recall_at_20_max": 0.4986739517512237, - "nauc_cv_recall_at_20_std": 0.34355706815645887, - "nauc_cv_recall_at_3_diff1": 0.3983883376536369, - "nauc_cv_recall_at_3_max": 0.42284870112868844, - "nauc_cv_recall_at_3_std": 0.16867702576877472, - "nauc_cv_recall_at_5_diff1": 0.37441532248421466, - "nauc_cv_recall_at_5_max": 0.43958306723412155, - "nauc_cv_recall_at_5_std": 0.20772423785543728, - "nauc_map_at_1000_diff1": 0.4525338584708267, - "nauc_map_at_1000_max": 0.4293850432847313, - "nauc_map_at_1000_std": 0.166782289898531, - "nauc_map_at_100_diff1": 0.45224768945865956, - "nauc_map_at_100_max": 0.42965326556217975, - "nauc_map_at_100_std": 0.16737932916906623, - "nauc_map_at_10_diff1": 0.4504471657225726, - "nauc_map_at_10_max": 0.4255150942788997, - "nauc_map_at_10_std": 0.16125390697135564, - "nauc_map_at_1_diff1": 0.519216885422269, - "nauc_map_at_1_max": 0.41701938956923496, - "nauc_map_at_1_std": 0.12515186524794, - "nauc_map_at_20_diff1": 0.45163270802450295, - "nauc_map_at_20_max": 0.4285784046915898, - "nauc_map_at_20_std": 0.165738020819407, - "nauc_map_at_3_diff1": 0.4608529827096026, - "nauc_map_at_3_max": 0.4206307446552375, - "nauc_map_at_3_std": 0.14550345666637035, - "nauc_map_at_5_diff1": 0.45511022081691893, - "nauc_map_at_5_max": 0.42481744042818403, - "nauc_map_at_5_std": 0.15509451723166393, - "nauc_mrr_at_1000_diff1": 0.4525338584708267, - "nauc_mrr_at_1000_max": 0.4293850432847313, - "nauc_mrr_at_1000_std": 0.166782289898531, - "nauc_mrr_at_100_diff1": 0.45224768945865956, - "nauc_mrr_at_100_max": 0.42965326556217975, - "nauc_mrr_at_100_std": 0.16737932916906623, - "nauc_mrr_at_10_diff1": 0.4504471657225726, - "nauc_mrr_at_10_max": 0.4255150942788997, - "nauc_mrr_at_10_std": 0.16125390697135564, - "nauc_mrr_at_1_diff1": 0.519216885422269, - "nauc_mrr_at_1_max": 0.41701938956923496, - "nauc_mrr_at_1_std": 0.12515186524794, - "nauc_mrr_at_20_diff1": 0.45163270802450295, - "nauc_mrr_at_20_max": 0.4285784046915898, - "nauc_mrr_at_20_std": 0.165738020819407, - "nauc_mrr_at_3_diff1": 0.4608529827096026, - "nauc_mrr_at_3_max": 0.4206307446552375, - "nauc_mrr_at_3_std": 0.14550345666637035, - "nauc_mrr_at_5_diff1": 0.45511022081691893, - "nauc_mrr_at_5_max": 0.42481744042818403, - "nauc_mrr_at_5_std": 0.15509451723166393, - "nauc_ndcg_at_1000_diff1": 0.4373081282043892, - "nauc_ndcg_at_1000_max": 0.4409003063361399, - "nauc_ndcg_at_1000_std": 0.19381797221676839, - "nauc_ndcg_at_100_diff1": 0.429992205334297, - "nauc_ndcg_at_100_max": 0.44949420342200636, - "nauc_ndcg_at_100_std": 0.21286903391313913, - "nauc_ndcg_at_10_diff1": 0.42196328661774485, - "nauc_ndcg_at_10_max": 0.4303021596534981, - "nauc_ndcg_at_10_std": 0.18350106704037714, - "nauc_ndcg_at_1_diff1": 0.519216885422269, - "nauc_ndcg_at_1_max": 0.41701938956923496, - "nauc_ndcg_at_1_std": 0.12515186524794, - "nauc_ndcg_at_20_diff1": 0.4259642730438616, - "nauc_ndcg_at_20_max": 0.44168870941829563, - "nauc_ndcg_at_20_std": 0.2005691635345835, - "nauc_ndcg_at_3_diff1": 0.4439390916850469, - "nauc_ndcg_at_3_max": 0.4212649609693536, - "nauc_ndcg_at_3_std": 0.1517498527473349, - "nauc_ndcg_at_5_diff1": 0.43375968280368277, - "nauc_ndcg_at_5_max": 0.428572455649159, - "nauc_ndcg_at_5_std": 0.16862594213907914, - "nauc_precision_at_1000_diff1": 0.28818656533653536, - "nauc_precision_at_1000_max": 0.788138892870599, - "nauc_precision_at_1000_std": 0.8586308516367483, - "nauc_precision_at_100_diff1": 0.30771338498199347, - "nauc_precision_at_100_max": 0.6228904388563357, - "nauc_precision_at_100_std": 0.5832133669363774, - "nauc_precision_at_10_diff1": 0.33330651297623737, - "nauc_precision_at_10_max": 0.4456554564461219, - "nauc_precision_at_10_std": 0.2561096026594872, - "nauc_precision_at_1_diff1": 0.519216885422269, - "nauc_precision_at_1_max": 0.41701938956923496, - "nauc_precision_at_1_std": 0.12515186524794, - "nauc_precision_at_20_diff1": 0.3377140363059223, - "nauc_precision_at_20_max": 0.49867395175122375, - "nauc_precision_at_20_std": 0.3435570681564585, - "nauc_precision_at_3_diff1": 0.3983883376536368, - "nauc_precision_at_3_max": 0.4228487011286886, - "nauc_precision_at_3_std": 0.16867702576877458, - "nauc_precision_at_5_diff1": 0.37441532248421455, - "nauc_precision_at_5_max": 0.43958306723412105, - "nauc_precision_at_5_std": 0.20772423785543706, - "nauc_recall_at_1000_diff1": 0.28818656533654663, - "nauc_recall_at_1000_max": 0.7881388928706085, - "nauc_recall_at_1000_std": 0.8586308516367452, - "nauc_recall_at_100_diff1": 0.3077133849819943, - "nauc_recall_at_100_max": 0.6228904388563352, - "nauc_recall_at_100_std": 0.5832133669363783, - "nauc_recall_at_10_diff1": 0.3333065129762372, - "nauc_recall_at_10_max": 0.4456554564461219, - "nauc_recall_at_10_std": 0.2561096026594872, - "nauc_recall_at_1_diff1": 0.519216885422269, - "nauc_recall_at_1_max": 0.41701938956923496, - "nauc_recall_at_1_std": 0.12515186524794, - "nauc_recall_at_20_diff1": 0.33771403630592267, - "nauc_recall_at_20_max": 0.4986739517512237, - "nauc_recall_at_20_std": 0.34355706815645887, - "nauc_recall_at_3_diff1": 0.3983883376536369, - "nauc_recall_at_3_max": 0.42284870112868844, - "nauc_recall_at_3_std": 0.16867702576877472, - "nauc_recall_at_5_diff1": 0.37441532248421466, - "nauc_recall_at_5_max": 0.43958306723412155, - "nauc_recall_at_5_std": 0.20772423785543728, - "ndcg_at_1": 0.24679, - "ndcg_at_10": 0.38614, - "ndcg_at_100": 0.44657, - "ndcg_at_1000": 0.46446, - "ndcg_at_20": 0.41236, - "ndcg_at_3": 0.32621, - "ndcg_at_5": 0.3546, - "precision_at_1": 0.24679, - "precision_at_10": 0.05509, - "precision_at_100": 0.0084, - "precision_at_1000": 0.00098, - "precision_at_20": 0.03273, - "precision_at_3": 0.12789, - "precision_at_5": 0.09056, - "recall_at_1": 0.24679, - "recall_at_10": 0.55095, - "recall_at_100": 0.8399, - "recall_at_1000": 0.98092, - "recall_at_20": 0.65464, - "recall_at_3": 0.38366, - "recall_at_5": 0.45279 - }, - { - "cv_recall_at_1": 0.47997, - "cv_recall_at_10": 0.81816, - "cv_recall_at_100": 0.96798, - "cv_recall_at_1000": 0.99861, - "cv_recall_at_20": 0.87845, - "cv_recall_at_3": 0.66778, - "cv_recall_at_5": 0.73818, - "hf_subset": "uk", - "languages": [ - "ukr-Cyrl" - ], - "main_score": 0.64496, - "map_at_1": 0.47997, - "map_at_10": 0.58998, - "map_at_100": 0.59669, - "map_at_1000": 0.59685, - "map_at_20": 0.5942, - "map_at_3": 0.56322, - "map_at_5": 0.57925, - "mrr_at_1": 0.47997227997228, - "mrr_at_10": 0.5899797599797617, - "mrr_at_100": 0.5966925968730221, - "mrr_at_1000": 0.5968492732003331, - "mrr_at_20": 0.5942022639319625, - "mrr_at_3": 0.563224763224767, - "mrr_at_5": 0.5792469392469406, - "nauc_cv_recall_at_1000_diff1": 0.359619386307682, - "nauc_cv_recall_at_1000_max": 0.8289359591762638, - "nauc_cv_recall_at_1000_std": 0.8567147435685029, - "nauc_cv_recall_at_100_diff1": 0.46590847395195073, - "nauc_cv_recall_at_100_max": 0.6662723523534553, - "nauc_cv_recall_at_100_std": 0.5424559953823549, - "nauc_cv_recall_at_10_diff1": 0.48640162038929124, - "nauc_cv_recall_at_10_max": 0.5388902567020947, - "nauc_cv_recall_at_10_std": 0.14897840060568757, - "nauc_cv_recall_at_1_diff1": 0.6461339577495517, - "nauc_cv_recall_at_1_max": 0.4259055740159931, - "nauc_cv_recall_at_1_std": -0.033787027088644075, - "nauc_cv_recall_at_20_diff1": 0.47921561047789607, - "nauc_cv_recall_at_20_max": 0.5846110809233728, - "nauc_cv_recall_at_20_std": 0.24702918047480954, - "nauc_cv_recall_at_3_diff1": 0.533961427938337, - "nauc_cv_recall_at_3_max": 0.4791315812220909, - "nauc_cv_recall_at_3_std": 0.01770929535518152, - "nauc_cv_recall_at_5_diff1": 0.5062471209011927, - "nauc_cv_recall_at_5_max": 0.5035739906001356, - "nauc_cv_recall_at_5_std": 0.062482575188251704, - "nauc_map_at_1000_diff1": 0.5998859220426689, - "nauc_map_at_1000_max": 0.44981533649256034, - "nauc_map_at_1000_std": -0.0034501414561704364, - "nauc_map_at_100_diff1": 0.5998140517633854, - "nauc_map_at_100_max": 0.4498957671854623, - "nauc_map_at_100_std": -0.0032166715611345533, - "nauc_map_at_10_diff1": 0.5977066123970523, - "nauc_map_at_10_max": 0.45028403262092415, - "nauc_map_at_10_std": -0.0035025904796239958, - "nauc_map_at_1_diff1": 0.6461339577495517, - "nauc_map_at_1_max": 0.4259055740159931, - "nauc_map_at_1_std": -0.033787027088644075, - "nauc_map_at_20_diff1": 0.5989064282726737, - "nauc_map_at_20_max": 0.45060898042681835, - "nauc_map_at_20_std": -0.0024880705803015054, - "nauc_map_at_3_diff1": 0.5997550415090089, - "nauc_map_at_3_max": 0.44650429192658436, - "nauc_map_at_3_std": -0.01452171041883708, - "nauc_map_at_5_diff1": 0.5970255575961725, - "nauc_map_at_5_max": 0.4491223043095994, - "nauc_map_at_5_std": -0.008338990928095574, - "nauc_mrr_at_1000_diff1": 0.5998859220426689, - "nauc_mrr_at_1000_max": 0.44981533649256034, - "nauc_mrr_at_1000_std": -0.0034501414561704364, - "nauc_mrr_at_100_diff1": 0.5998140517633854, - "nauc_mrr_at_100_max": 0.4498957671854623, - "nauc_mrr_at_100_std": -0.0032166715611345533, - "nauc_mrr_at_10_diff1": 0.5977066123970523, - "nauc_mrr_at_10_max": 0.45028403262092415, - "nauc_mrr_at_10_std": -0.0035025904796239958, - "nauc_mrr_at_1_diff1": 0.6461339577495517, - "nauc_mrr_at_1_max": 0.4259055740159931, - "nauc_mrr_at_1_std": -0.033787027088644075, - "nauc_mrr_at_20_diff1": 0.5989064282726737, - "nauc_mrr_at_20_max": 0.45060898042681835, - "nauc_mrr_at_20_std": -0.0024880705803015054, - "nauc_mrr_at_3_diff1": 0.5997550415090089, - "nauc_mrr_at_3_max": 0.44650429192658436, - "nauc_mrr_at_3_std": -0.01452171041883708, - "nauc_mrr_at_5_diff1": 0.5970255575961725, - "nauc_mrr_at_5_max": 0.4491223043095994, - "nauc_mrr_at_5_std": -0.008338990928095574, - "nauc_ndcg_at_1000_diff1": 0.5920290475390783, - "nauc_ndcg_at_1000_max": 0.4568058683689263, - "nauc_ndcg_at_1000_std": 0.009398550737737928, - "nauc_ndcg_at_100_diff1": 0.5900339801932035, - "nauc_ndcg_at_100_max": 0.45991286069602905, - "nauc_ndcg_at_100_std": 0.01755545805425444, - "nauc_ndcg_at_10_diff1": 0.5793414880989026, - "nauc_ndcg_at_10_max": 0.46396882476782636, - "nauc_ndcg_at_10_std": 0.019337122970605566, - "nauc_ndcg_at_1_diff1": 0.6461339577495517, - "nauc_ndcg_at_1_max": 0.4259055740159931, - "nauc_ndcg_at_1_std": -0.033787027088644075, - "nauc_ndcg_at_20_diff1": 0.5833380066387772, - "nauc_ndcg_at_20_max": 0.4656530069396211, - "nauc_ndcg_at_20_std": 0.02432957786471851, - "nauc_ndcg_at_3_diff1": 0.5844841161839118, - "nauc_ndcg_at_3_max": 0.45401230876657694, - "nauc_ndcg_at_3_std": -0.007134162372538832, - "nauc_ndcg_at_5_diff1": 0.5784652389857332, - "nauc_ndcg_at_5_max": 0.4599763929238898, - "nauc_ndcg_at_5_std": 0.005591487123884887, - "nauc_precision_at_1000_diff1": 0.3596193863074833, - "nauc_precision_at_1000_max": 0.8289359591760609, - "nauc_precision_at_1000_std": 0.8567147435683159, - "nauc_precision_at_100_diff1": 0.465908473951957, - "nauc_precision_at_100_max": 0.6662723523534598, - "nauc_precision_at_100_std": 0.5424559953823621, - "nauc_precision_at_10_diff1": 0.4864016203892922, - "nauc_precision_at_10_max": 0.5388902567020952, - "nauc_precision_at_10_std": 0.14897840060568837, - "nauc_precision_at_1_diff1": 0.6461339577495517, - "nauc_precision_at_1_max": 0.4259055740159931, - "nauc_precision_at_1_std": -0.033787027088644075, - "nauc_precision_at_20_diff1": 0.47921561047789474, - "nauc_precision_at_20_max": 0.584611080923373, - "nauc_precision_at_20_std": 0.24702918047481107, - "nauc_precision_at_3_diff1": 0.5339614279383376, - "nauc_precision_at_3_max": 0.479131581222091, - "nauc_precision_at_3_std": 0.017709295355182442, - "nauc_precision_at_5_diff1": 0.506247120901192, - "nauc_precision_at_5_max": 0.5035739906001366, - "nauc_precision_at_5_std": 0.06248257518825092, - "nauc_recall_at_1000_diff1": 0.359619386307682, - "nauc_recall_at_1000_max": 0.8289359591762638, - "nauc_recall_at_1000_std": 0.8567147435685029, - "nauc_recall_at_100_diff1": 0.46590847395195073, - "nauc_recall_at_100_max": 0.6662723523534553, - "nauc_recall_at_100_std": 0.5424559953823549, - "nauc_recall_at_10_diff1": 0.48640162038929124, - "nauc_recall_at_10_max": 0.5388902567020947, - "nauc_recall_at_10_std": 0.14897840060568757, - "nauc_recall_at_1_diff1": 0.6461339577495517, - "nauc_recall_at_1_max": 0.4259055740159931, - "nauc_recall_at_1_std": -0.033787027088644075, - "nauc_recall_at_20_diff1": 0.47921561047789607, - "nauc_recall_at_20_max": 0.5846110809233728, - "nauc_recall_at_20_std": 0.24702918047480954, - "nauc_recall_at_3_diff1": 0.533961427938337, - "nauc_recall_at_3_max": 0.4791315812220909, - "nauc_recall_at_3_std": 0.01770929535518152, - "nauc_recall_at_5_diff1": 0.5062471209011927, - "nauc_recall_at_5_max": 0.5035739906001356, - "nauc_recall_at_5_std": 0.062482575188251704, - "ndcg_at_1": 0.47997, - "ndcg_at_10": 0.64496, - "ndcg_at_100": 0.67724, - "ndcg_at_1000": 0.68119, - "ndcg_at_20": 0.66025, - "ndcg_at_3": 0.5901, - "ndcg_at_5": 0.61904, - "precision_at_1": 0.47997, - "precision_at_10": 0.08182, - "precision_at_100": 0.00968, - "precision_at_1000": 0.001, - "precision_at_20": 0.04392, - "precision_at_3": 0.22259, - "precision_at_5": 0.14764, - "recall_at_1": 0.47997, - "recall_at_10": 0.81816, - "recall_at_100": 0.96798, - "recall_at_1000": 0.99861, - "recall_at_20": 0.87845, - "recall_at_3": 0.66778, - "recall_at_5": 0.73818 - }, - { - "cv_recall_at_1": 0.56735, - "cv_recall_at_10": 0.88136, - "cv_recall_at_100": 0.98449, - "cv_recall_at_1000": 0.99973, - "cv_recall_at_20": 0.93048, - "cv_recall_at_3": 0.75116, - "cv_recall_at_5": 0.81429, - "hf_subset": "vi", - "languages": [ - "vie-Latn" - ], - "main_score": 0.7231, - "map_at_1": 0.56735, - "map_at_10": 0.67252, - "map_at_100": 0.67759, - "map_at_1000": 0.67768, - "map_at_20": 0.67607, - "map_at_3": 0.64896, - "map_at_5": 0.66347, - "mrr_at_1": 0.5673469387755102, - "mrr_at_10": 0.6725237555339588, - "mrr_at_100": 0.6775858165821529, - "mrr_at_1000": 0.67767650718611, - "mrr_at_20": 0.676070937221595, - "mrr_at_3": 0.6489569160997744, - "mrr_at_5": 0.663467120181404, - "nauc_cv_recall_at_1000_diff1": 0.9346405228757856, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": 0.6620923223090417, - "nauc_cv_recall_at_100_max": 0.8255770144315029, - "nauc_cv_recall_at_100_std": 0.4641382868937102, - "nauc_cv_recall_at_10_diff1": 0.6058370101844033, - "nauc_cv_recall_at_10_max": 0.6163717897701185, - "nauc_cv_recall_at_10_std": 0.041965843547617905, - "nauc_cv_recall_at_1_diff1": 0.7259662689257877, - "nauc_cv_recall_at_1_max": 0.43676348337684495, - "nauc_cv_recall_at_1_std": -0.1449503651311885, - "nauc_cv_recall_at_20_diff1": 0.5987335573498791, - "nauc_cv_recall_at_20_max": 0.6816936820390254, - "nauc_cv_recall_at_20_std": 0.1540250803517749, - "nauc_cv_recall_at_3_diff1": 0.6377585424746157, - "nauc_cv_recall_at_3_max": 0.5040415194409423, - "nauc_cv_recall_at_3_std": -0.11346754850123454, - "nauc_cv_recall_at_5_diff1": 0.6239492907917592, - "nauc_cv_recall_at_5_max": 0.5487095212616887, - "nauc_cv_recall_at_5_std": -0.052386412028695956, - "nauc_map_at_1000_diff1": 0.6928319955996259, - "nauc_map_at_1000_max": 0.4662204138391279, - "nauc_map_at_1000_std": -0.12257467776910304, - "nauc_map_at_100_diff1": 0.6928192827730435, - "nauc_map_at_100_max": 0.4663371019670773, - "nauc_map_at_100_std": -0.12240526964653446, - "nauc_map_at_10_diff1": 0.6911981783560863, - "nauc_map_at_10_max": 0.46793659396050186, - "nauc_map_at_10_std": -0.12145300386378471, - "nauc_map_at_1_diff1": 0.7259662689257877, - "nauc_map_at_1_max": 0.43676348337684495, - "nauc_map_at_1_std": -0.1449503651311885, - "nauc_map_at_20_diff1": 0.6921305592473023, - "nauc_map_at_20_max": 0.46723871180782983, - "nauc_map_at_20_std": -0.12153973319496549, - "nauc_map_at_3_diff1": 0.6905332223021816, - "nauc_map_at_3_max": 0.46149727185955913, - "nauc_map_at_3_std": -0.13361290779652502, - "nauc_map_at_5_diff1": 0.6904150526689351, - "nauc_map_at_5_max": 0.46635245248076457, - "nauc_map_at_5_std": -0.12489992753838132, - "nauc_mrr_at_1000_diff1": 0.6928319955996259, - "nauc_mrr_at_1000_max": 0.4662204138391279, - "nauc_mrr_at_1000_std": -0.12257467776910304, - "nauc_mrr_at_100_diff1": 0.6928192827730435, - "nauc_mrr_at_100_max": 0.4663371019670773, - "nauc_mrr_at_100_std": -0.12240526964653446, - "nauc_mrr_at_10_diff1": 0.6911981783560863, - "nauc_mrr_at_10_max": 0.46793659396050186, - "nauc_mrr_at_10_std": -0.12145300386378471, - "nauc_mrr_at_1_diff1": 0.7259662689257877, - "nauc_mrr_at_1_max": 0.43676348337684495, - "nauc_mrr_at_1_std": -0.1449503651311885, - "nauc_mrr_at_20_diff1": 0.6921305592473023, - "nauc_mrr_at_20_max": 0.46723871180782983, - "nauc_mrr_at_20_std": -0.12153973319496549, - "nauc_mrr_at_3_diff1": 0.6905332223021816, - "nauc_mrr_at_3_max": 0.46149727185955913, - "nauc_mrr_at_3_std": -0.13361290779652502, - "nauc_mrr_at_5_diff1": 0.6904150526689351, - "nauc_mrr_at_5_max": 0.46635245248076457, - "nauc_mrr_at_5_std": -0.12489992753838132, - "nauc_ndcg_at_1000_diff1": 0.6883667434948514, - "nauc_ndcg_at_1000_max": 0.4738703537940675, - "nauc_ndcg_at_1000_std": -0.1136077195297271, - "nauc_ndcg_at_100_diff1": 0.6880647036947598, - "nauc_ndcg_at_100_max": 0.4772551525584301, - "nauc_ndcg_at_100_std": -0.10821757655795085, - "nauc_ndcg_at_10_diff1": 0.6794314751585946, - "nauc_ndcg_at_10_max": 0.4866239032204006, - "nauc_ndcg_at_10_std": -0.10157943275930026, - "nauc_ndcg_at_1_diff1": 0.7259662689257877, - "nauc_ndcg_at_1_max": 0.43676348337684495, - "nauc_ndcg_at_1_std": -0.1449503651311885, - "nauc_ndcg_at_20_diff1": 0.6828716896544922, - "nauc_ndcg_at_20_max": 0.48476789923178737, - "nauc_ndcg_at_20_std": -0.10049463526827918, - "nauc_ndcg_at_3_diff1": 0.6790131486012451, - "nauc_ndcg_at_3_max": 0.4706728176618378, - "nauc_ndcg_at_3_std": -0.12927902751301362, - "nauc_ndcg_at_5_diff1": 0.6780737481870208, - "nauc_ndcg_at_5_max": 0.48101881603227314, - "nauc_ndcg_at_5_std": -0.11223564267542065, - "nauc_precision_at_1000_diff1": 0.9346405228756913, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.6620923223090409, - "nauc_precision_at_100_max": 0.8255770144315049, - "nauc_precision_at_100_std": 0.4641382868936997, - "nauc_precision_at_10_diff1": 0.6058370101844027, - "nauc_precision_at_10_max": 0.6163717897701152, - "nauc_precision_at_10_std": 0.04196584354761412, - "nauc_precision_at_1_diff1": 0.7259662689257877, - "nauc_precision_at_1_max": 0.43676348337684495, - "nauc_precision_at_1_std": -0.1449503651311885, - "nauc_precision_at_20_diff1": 0.5987335573498817, - "nauc_precision_at_20_max": 0.6816936820390233, - "nauc_precision_at_20_std": 0.15402508035177356, - "nauc_precision_at_3_diff1": 0.6377585424746148, - "nauc_precision_at_3_max": 0.504041519440942, - "nauc_precision_at_3_std": -0.11346754850123358, - "nauc_precision_at_5_diff1": 0.6239492907917575, - "nauc_precision_at_5_max": 0.5487095212616888, - "nauc_precision_at_5_std": -0.0523864120286966, - "nauc_recall_at_1000_diff1": 0.9346405228757856, - "nauc_recall_at_1000_max": 1.0, - "nauc_recall_at_1000_std": 1.0, - "nauc_recall_at_100_diff1": 0.6620923223090417, - "nauc_recall_at_100_max": 0.8255770144315029, - "nauc_recall_at_100_std": 0.4641382868937102, - "nauc_recall_at_10_diff1": 0.6058370101844033, - "nauc_recall_at_10_max": 0.6163717897701185, - "nauc_recall_at_10_std": 0.041965843547617905, - "nauc_recall_at_1_diff1": 0.7259662689257877, - "nauc_recall_at_1_max": 0.43676348337684495, - "nauc_recall_at_1_std": -0.1449503651311885, - "nauc_recall_at_20_diff1": 0.5987335573498791, - "nauc_recall_at_20_max": 0.6816936820390254, - "nauc_recall_at_20_std": 0.1540250803517749, - "nauc_recall_at_3_diff1": 0.6377585424746157, - "nauc_recall_at_3_max": 0.5040415194409423, - "nauc_recall_at_3_std": -0.11346754850123454, - "nauc_recall_at_5_diff1": 0.6239492907917592, - "nauc_recall_at_5_max": 0.5487095212616887, - "nauc_recall_at_5_std": -0.052386412028695956, - "ndcg_at_1": 0.56735, - "ndcg_at_10": 0.7231, - "ndcg_at_100": 0.74596, - "ndcg_at_1000": 0.74799, - "ndcg_at_20": 0.73569, - "ndcg_at_3": 0.67523, - "ndcg_at_5": 0.7013, - "precision_at_1": 0.56735, - "precision_at_10": 0.08814, - "precision_at_100": 0.00984, - "precision_at_1000": 0.001, - "precision_at_20": 0.04652, - "precision_at_3": 0.25039, - "precision_at_5": 0.16286, - "recall_at_1": 0.56735, - "recall_at_10": 0.88136, - "recall_at_100": 0.98449, - "recall_at_1000": 0.99973, - "recall_at_20": 0.93048, - "recall_at_3": 0.75116, - "recall_at_5": 0.81429 - }, - { - "cv_recall_at_1": 0.71536, - "cv_recall_at_10": 0.94271, - "cv_recall_at_100": 0.99345, - "cv_recall_at_1000": 0.99902, - "cv_recall_at_20": 0.97031, - "cv_recall_at_3": 0.86117, - "cv_recall_at_5": 0.90131, - "hf_subset": "zh", - "languages": [ - "zho-Hans" - ], - "main_score": 0.83161, - "map_at_1": 0.71536, - "map_at_10": 0.79571, - "map_at_100": 0.79833, - "map_at_1000": 0.79836, - "map_at_20": 0.79769, - "map_at_3": 0.78076, - "map_at_5": 0.79005, - "mrr_at_1": 0.7153610259269585, - "mrr_at_10": 0.7957073824790565, - "mrr_at_100": 0.7983271878223961, - "mrr_at_1000": 0.7983598682574501, - "mrr_at_20": 0.7976864767200028, - "mrr_at_3": 0.7807592231205266, - "mrr_at_5": 0.7900497165690891, - "nauc_cv_recall_at_1000_diff1": 0.8312673556411718, - "nauc_cv_recall_at_1000_max": 1.0, - "nauc_cv_recall_at_1000_std": 1.0, - "nauc_cv_recall_at_100_diff1": 0.7479815757407692, - "nauc_cv_recall_at_100_max": 0.9122042907617619, - "nauc_cv_recall_at_100_std": 0.5432500070608071, - "nauc_cv_recall_at_10_diff1": 0.7198905770773609, - "nauc_cv_recall_at_10_max": 0.645919725401898, - "nauc_cv_recall_at_10_std": -0.11963773343079089, - "nauc_cv_recall_at_1_diff1": 0.8099670140993427, - "nauc_cv_recall_at_1_max": 0.5193603728956219, - "nauc_cv_recall_at_1_std": -0.1858820928055574, - "nauc_cv_recall_at_20_diff1": 0.7224495327813171, - "nauc_cv_recall_at_20_max": 0.7747247259994867, - "nauc_cv_recall_at_20_std": 0.1537137334369623, - "nauc_cv_recall_at_3_diff1": 0.7284515534007231, - "nauc_cv_recall_at_3_max": 0.5667370329867204, - "nauc_cv_recall_at_3_std": -0.2152454936985054, - "nauc_cv_recall_at_5_diff1": 0.7158676556641865, - "nauc_cv_recall_at_5_max": 0.5848997160398609, - "nauc_cv_recall_at_5_std": -0.21527788827804065, - "nauc_map_at_1000_diff1": 0.7868766395441894, - "nauc_map_at_1000_max": 0.5354946244045167, - "nauc_map_at_1000_std": -0.19143339710059779, - "nauc_map_at_100_diff1": 0.7868643565417248, - "nauc_map_at_100_max": 0.5355539423405813, - "nauc_map_at_100_std": -0.19134157963282333, - "nauc_map_at_10_diff1": 0.785880091957294, - "nauc_map_at_10_max": 0.5359410279488298, - "nauc_map_at_10_std": -0.19304472971847336, - "nauc_map_at_1_diff1": 0.8099670140993427, - "nauc_map_at_1_max": 0.5193603728956219, - "nauc_map_at_1_std": -0.1858820928055574, - "nauc_map_at_20_diff1": 0.7866387909363147, - "nauc_map_at_20_max": 0.536205919161459, - "nauc_map_at_20_std": -0.19059204221385653, - "nauc_map_at_3_diff1": 0.7829163122175201, - "nauc_map_at_3_max": 0.5346738369730953, - "nauc_map_at_3_std": -0.19793512002963232, - "nauc_map_at_5_diff1": 0.7836488568017643, - "nauc_map_at_5_max": 0.5351251307032907, - "nauc_map_at_5_std": -0.19758825403953903, - "nauc_mrr_at_1000_diff1": 0.7868766395441894, - "nauc_mrr_at_1000_max": 0.5354946244045167, - "nauc_mrr_at_1000_std": -0.19143339710059779, - "nauc_mrr_at_100_diff1": 0.7868643565417248, - "nauc_mrr_at_100_max": 0.5355539423405813, - "nauc_mrr_at_100_std": -0.19134157963282333, - "nauc_mrr_at_10_diff1": 0.785880091957294, - "nauc_mrr_at_10_max": 0.5359410279488298, - "nauc_mrr_at_10_std": -0.19304472971847336, - "nauc_mrr_at_1_diff1": 0.8099670140993427, - "nauc_mrr_at_1_max": 0.5193603728956219, - "nauc_mrr_at_1_std": -0.1858820928055574, - "nauc_mrr_at_20_diff1": 0.7866387909363147, - "nauc_mrr_at_20_max": 0.536205919161459, - "nauc_mrr_at_20_std": -0.19059204221385653, - "nauc_mrr_at_3_diff1": 0.7829163122175201, - "nauc_mrr_at_3_max": 0.5346738369730953, - "nauc_mrr_at_3_std": -0.19793512002963232, - "nauc_mrr_at_5_diff1": 0.7836488568017643, - "nauc_mrr_at_5_max": 0.5351251307032907, - "nauc_mrr_at_5_std": -0.19758825403953903, - "nauc_ndcg_at_1000_diff1": 0.7839030308467452, - "nauc_ndcg_at_1000_max": 0.5403658515167975, - "nauc_ndcg_at_1000_std": -0.18649224306281725, - "nauc_ndcg_at_100_diff1": 0.783623949402799, - "nauc_ndcg_at_100_max": 0.5422041371843155, - "nauc_ndcg_at_100_std": -0.18323455028455227, - "nauc_ndcg_at_10_diff1": 0.7786406797093983, - "nauc_ndcg_at_10_max": 0.5460790489887855, - "nauc_ndcg_at_10_std": -0.18765922119721604, - "nauc_ndcg_at_1_diff1": 0.8099670140993427, - "nauc_ndcg_at_1_max": 0.5193603728956219, - "nauc_ndcg_at_1_std": -0.1858820928055574, - "nauc_ndcg_at_20_diff1": 0.7816313221107242, - "nauc_ndcg_at_20_max": 0.5478768608659305, - "nauc_ndcg_at_20_std": -0.1766526940959122, - "nauc_ndcg_at_3_diff1": 0.7724560802243667, - "nauc_ndcg_at_3_max": 0.5408118753435333, - "nauc_ndcg_at_3_std": -0.20137195399903116, - "nauc_ndcg_at_5_diff1": 0.7731218817284778, - "nauc_ndcg_at_5_max": 0.5426458166494701, - "nauc_ndcg_at_5_std": -0.20055915517570486, - "nauc_precision_at_1000_diff1": 0.8312673556410209, - "nauc_precision_at_1000_max": 1.0, - "nauc_precision_at_1000_std": 1.0, - "nauc_precision_at_100_diff1": 0.7479815757407771, - "nauc_precision_at_100_max": 0.9122042907617612, - "nauc_precision_at_100_std": 0.5432500070608429, - "nauc_precision_at_10_diff1": 0.7198905770773566, - "nauc_precision_at_10_max": 0.6459197254018942, - "nauc_precision_at_10_std": -0.11963773343079387, - "nauc_precision_at_1_diff1": 0.8099670140993427, - "nauc_precision_at_1_max": 0.5193603728956219, - "nauc_precision_at_1_std": -0.1858820928055574, - "nauc_precision_at_20_diff1": 0.7224495327813097, - "nauc_precision_at_20_max": 0.774724725999475, - "nauc_precision_at_20_std": 0.15371373343695663, - "nauc_precision_at_3_diff1": 0.7284515534007242, - "nauc_precision_at_3_max": 0.5667370329867211, - "nauc_precision_at_3_std": -0.21524549369850432, - "nauc_precision_at_5_diff1": 0.7158676556641869, - "nauc_precision_at_5_max": 0.5848997160398612, - "nauc_precision_at_5_std": -0.2152778882780402, - "nauc_recall_at_1000_diff1": 0.8312673556411718, - "nauc_recall_at_1000_max": 1.0, - "nauc_recall_at_1000_std": 1.0, - "nauc_recall_at_100_diff1": 0.7479815757407692, - "nauc_recall_at_100_max": 0.9122042907617619, - "nauc_recall_at_100_std": 0.5432500070608071, - "nauc_recall_at_10_diff1": 0.7198905770773609, - "nauc_recall_at_10_max": 0.645919725401898, - "nauc_recall_at_10_std": -0.11963773343079089, - "nauc_recall_at_1_diff1": 0.8099670140993427, - "nauc_recall_at_1_max": 0.5193603728956219, - "nauc_recall_at_1_std": -0.1858820928055574, - "nauc_recall_at_20_diff1": 0.7224495327813171, - "nauc_recall_at_20_max": 0.7747247259994867, - "nauc_recall_at_20_std": 0.1537137334369623, - "nauc_recall_at_3_diff1": 0.7284515534007231, - "nauc_recall_at_3_max": 0.5667370329867204, - "nauc_recall_at_3_std": -0.2152454936985054, - "nauc_recall_at_5_diff1": 0.7158676556641865, - "nauc_recall_at_5_max": 0.5848997160398609, - "nauc_recall_at_5_std": -0.21527788827804065, - "ndcg_at_1": 0.71536, - "ndcg_at_10": 0.83161, - "ndcg_at_100": 0.84306, - "ndcg_at_1000": 0.8438, - "ndcg_at_20": 0.83867, - "ndcg_at_3": 0.80146, - "ndcg_at_5": 0.81809, - "precision_at_1": 0.71536, - "precision_at_10": 0.09427, - "precision_at_100": 0.00993, - "precision_at_1000": 0.001, - "precision_at_20": 0.04852, - "precision_at_3": 0.28706, - "precision_at_5": 0.18026, - "recall_at_1": 0.71536, - "recall_at_10": 0.94271, - "recall_at_100": 0.99345, - "recall_at_1000": 0.99902, - "recall_at_20": 0.97031, - "recall_at_3": 0.86117, - "recall_at_5": 0.90131 - } - ] - }, - "task_name": "XM3600T2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/model_meta.json b/results-mieb/voyage-multimodal-3/1/model_meta.json deleted file mode 100644 index 528c812ae0..0000000000 --- a/results-mieb/voyage-multimodal-3/1/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "voyage-multimodal-3", "revision": "1", "release_date": "2024-11-10", "languages": [], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": 1024, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": "cosine", "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "voyage_v_loader"} \ No newline at end of file From 1a6f743aba67e10bf244575462bd42c429464b82 Mon Sep 17 00:00:00 2001 From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Date: Sun, 19 Jan 2025 21:10:26 +0800 Subject: [PATCH 135/154] [mieb] fixing lrap computation for multi-label classification (#1834) multi-label cls lrap computation fix --- .../Image/AbsTaskImageMultilabelClassification.py | 8 +++++++- .../ImageMultilabelClassification/eng/PascalVOC2007.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py index 6635e89828..dc779d5e69 100644 --- a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py @@ -36,7 +36,13 @@ def evaluate_classifier( f1 = f1_score(y_test, y_pred, average="macro") scores["accuracy"] = accuracy scores["f1"] = f1 - lrap = label_ranking_average_precision_score(y_test, y_pred) + all_probs = [] + for estimator in classifier.estimators_: + probs = estimator.predict_proba(embeddings_test)[:, 1] + all_probs.append(probs) + + y_score = np.stack(all_probs, axis=1) # shape: (n_samples, n_labels) + lrap = label_ranking_average_precision_score(y_test, y_score) scores["lrap"] = lrap return scores diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index 0ce54da8eb..ce32f85f93 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -21,7 +21,7 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], - main_score="accuracy", + main_score="lrap", date=( "2005-01-01", "2014-01-01", From 668d3da7d05e40197c225021e047c1e285c6b025 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Thu, 23 Jan 2025 21:23:01 +0900 Subject: [PATCH 136/154] [mieb] Merge from main (#1853) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update tasks table * 1.19.0 Automatically generated by python-semantic-release * fix: Add the_ugly_duckling.txt for speedtask to Python wheel (#1402) Add the_ugly_duckling.txt for speedtask to Python wheel * 1.19.1 Automatically generated by python-semantic-release * fix: Added the necessary trust_remote_code (#1406) * 1.19.2 Automatically generated by python-semantic-release * docs: Update recommendation for pushing results (#1401) fix: Update recommendation for pushing results * docs: Fix a typo in README (#1430) Fix typo in readme * fix: add logging for RetrievalEvaluator NaN values for similarity scores (#1398) Fixes #1389 * 1.19.3 Automatically generated by python-semantic-release * fix: make samples_per_label a task attribute (#1419) make samples_per_label a task attr * fix: Add Korean AutoRAGRetrieval (#1388) * feat: add AutoRAG Korean embedding retrieval benchmark * fix: run --- 🧹 Running linters --- ruff format . # running ruff formatting 716 files left unchanged ruff check . --fix # running ruff linting All checks passed! * fix: add metadata for AutoRAGRetrieval * change link for markers_bm * add AutoRAGRetrieval to init.py and update metadata * add precise metadata * update metadata: description and license * delete descriptive_stats in AutoRAGRetrieval.py and run calculate_matadata_metrics.py * fix: Add missing benchmarks in benchmarks.py (#1431) Fixes #1423 * Update tasks table * 1.19.4 Automatically generated by python-semantic-release * Leaderboard 2.0: added performance x n_parameters plot + more benchmark info (#1437) * Added elementary speed/performance plot * Refactored table formatting code * Bumped Gradio version * Added more general info to benchmark description markdown block * Adjusted margin an range on plot * Made hover information easier to read on plot * Made range scaling dynamic in plot * Moved citation next to benchmark description * Made titles in benchmark info bold * Leaderboard: Fixed code benchmarks (#1441) * fixed code benchmarks * fix: Made n_parameters formatting smarter and more robust * fix: changed jina-embeddings-v3 number of parameters from 572K to 572M * fix: Fixed use_instuctions typo in model overview * fix: Fixed sentence-transformer compatibility switch * Ran linting * Added all languages, tasks, types and domains to options * Removed resetting options when a new benchmark is selected * All results now get displayed, but models that haven't been run on everything get nan values in the table * fix: Count unique texts, data leaks in calculate metrics (#1438) * add more stat * add more stat * update statistics * fix: update task metadata to allow for null (#1448) * Update tasks table * 1.19.5 Automatically generated by python-semantic-release * Fix: Made data parsing in the leaderboard figure more robust (#1450) Bugfixes with data parsing in main figure * Fixed task loading (#1451) * Fixed task result loading from disk * Fixed task result loading from disk * fix: publish (#1452) * 1.19.6 Automatically generated by python-semantic-release * fix: Fix load external results with `None` mteb_version (#1453) * fix * lint * 1.19.7 Automatically generated by python-semantic-release * WIP: Polishing up leaderboard UI (#1461) * fix: Removed column wrapping on the table, so that it remains readable * Added disclaimer to figure * fix: Added links to task info table, switched out license with metric * fix: loading pre 1.11.0 (#1460) * small fix * fix: fix * 1.19.8 Automatically generated by python-semantic-release * fix: swap touche2020 to maintain compatibility (#1469) swap touche2020 for parity * 1.19.9 Automatically generated by python-semantic-release * docs: Add sum per language for task counts (#1468) * add sum per lang * add sort by sum option * make lint * fix: pinned datasets to <3.0.0 (#1470) * 1.19.10 Automatically generated by python-semantic-release * feat: add CUREv1 retrieval dataset (#1459) * feat: add CUREv1 dataset --------- Co-authored-by: nadshe Co-authored-by: olivierr42 Co-authored-by: Daniel Buades Marcos * feat: add missing domains to medical tasks * feat: modify benchmark tasks * chore: benchmark naming --------- Co-authored-by: nadshe Co-authored-by: olivierr42 * Update tasks table * 1.20.0 Automatically generated by python-semantic-release * fix: check if `model` attr of model exists (#1499) * check if model attr of model exists * lint * Fix retrieval evaluator * 1.20.1 Automatically generated by python-semantic-release * fix: Leaderboard demo data loading (#1507) * Made get_scores error tolerant * Added join_revisions, made get_scores failsafe * Fetching metadata fixed fr HF models * Added failsafe metadata fetching to leaderboard code * Added revision joining to leaderboard app * fix * Only show models that have metadata, when filter_models is called * Ran linting * 1.20.2 Automatically generated by python-semantic-release * fix: leaderboard only shows models that have ModelMeta (#1508) Filtering for models that have metadata * 1.20.3 Automatically generated by python-semantic-release * fix: align readme with current mteb (#1493) * align readme with current mteb * align with mieb branch * fix test * 1.20.4 Automatically generated by python-semantic-release * docs: Add lang family mapping and map to task table (#1486) * add lang family mapping and map to task table * make lint * add back some unclassified lang codes * Update tasks table * fix: Ensure that models match the names on embedding-benchmarks/results (#1519) * 1.20.5 Automatically generated by python-semantic-release * fix: Adding missing metadata on models and mathcing names up with the results repo (#1528) * Added Voyage 3 models * Added correct metadata to Cohere models and matched names with the results repo * 1.20.6 Automatically generated by python-semantic-release * feat: Evaluate missing splits (#1525) * fix: evaluate missing splits (#1268) * implement partial evaluation for missing splits * lint * requested changes done from scratch * test for missing split evaluation added * uncomment test * lint * avoid circular import * use TaskResult * skip tests for now --------- Co-authored-by: Isaac Chung * got test_all_splits_evaluated passing * tests passing * address review comments * make lint * handle None cases for kg_co2_emissions * use new results info --------- Co-authored-by: Thivyanth * 1.21.0 Automatically generated by python-semantic-release * fix: Correct typos superseeded -> superseded (#1532) fix typo -> superseded * 1.21.1 Automatically generated by python-semantic-release * fix: Task load data error for SICK-BR-STS and XStance (#1534) * fix task load data for two tasks * correct dataset keys * 1.21.2 Automatically generated by python-semantic-release * fix: Proprietary models now get correctly shown in leaderboard (#1530) * Fixed showing proprietary models in leaderboard * Added links to all OpenAI models * Fixed table formatting issues * Bumped Gradio version * 1.21.3 Automatically generated by python-semantic-release * docs: Add Model Meta parameters and metadata (#1536) * add multi_qa_MiniLM_L6_cos_v1 model meta * add all_mpnet_base_v2 * add parameters to model meta * make lint * add extra params to meta * fix: add more model meta (jina, e5) (#1537) * add e5 model meta * address review comments * 1.21.4 Automatically generated by python-semantic-release * Add cohere models (#1538) * fix: bug cohere names * format * fix: add nomic models (#1543) #1515 * fix: Added all-minilm-l12-v2 (#1542) #1515 * fix: Added arctic models (#1541) #1515 * fix: add sentence trimming to OpenAIWrapper (#1526) * fix: add sentence trimming to OpenAIWrapper * fix: import tiktoken library inside encode function * fix: check tokenizer library installed and update ModelMeta to pass tokenizer_name * fix: pass tokenizer_name, max_tokens to loader * fix: make tokenizer_name None for default * fix: delete changes for ModelMeta * fix: fix revision to 2 for OpenAI models * fix: add docstring for OpenAIWrapper * fix: lint * feat: add openai optional dependency set * fix: add sleep for too many requests * fix: add lint * fix: delete evaluate file * 1.21.5 Automatically generated by python-semantic-release * fix: Fixed metadata errors (#1547) * 1.21.6 Automatically generated by python-semantic-release * fix: remove curev1 from multlingual (#1552) Seems like it was added here: https://github.com/embeddings-benchmark/mteb/commit/1cc6c9e0fe62ca4e77708b641823fa1a121f048b * 1.21.7 Automatically generated by python-semantic-release * fix: Add Model2vec (#1546) * Added Model2Vec wrapper * Added Model2vec models * Added model2vec models to registry * Added model2vec as a dependency * Ran linting * Update mteb/models/model2vec_models.py Co-authored-by: Kenneth Enevoldsen * Update mteb/models/model2vec_models.py Co-authored-by: Kenneth Enevoldsen * Added adapted_from and superseeded_by to model2vec models. * Added missing import * Moved pyproject.toml to optional dependencies * Fixed typos * Added import error and changed model to model_name * Added Numpy to frameworks * Added Numpy to frameworks * Corrected false info on model2vec models * Replaced np.inf with maxint * Update mteb/models/model2vec_models.py Co-authored-by: Isaac Chung * Added option to have infinite max tokens, added it to Model2vec --------- Co-authored-by: Kenneth Enevoldsen Co-authored-by: Isaac Chung * Made result loading more permissive, changed eval splits for HotPotQA and DBPedia (#1554) * Removed train and dev from eval splits on HotpotQA * Removed dev from eval splits on DBPedia * Made task_results validation more permissive * Readded exception in get_score * Ran linting * 1.21.8 Automatically generated by python-semantic-release * docs: Correction of SICK-R metadata (#1558) * Correction of SICK-R metadata * Correction of SICK-R metadata --------- Co-authored-by: rposwiata * feat(google_models): fix issues and add support for `text-embedding-005` and `text-multilingual-embedding-002` (#1562) * fix: google_models batching and prompt * feat: add text-embedding-005 and text-multilingual-embedding-002 * chore: `make lint` errors * fix: address PR comments * 1.22.0 Automatically generated by python-semantic-release * fix(bm25s): search implementation (#1566) fix: bm25s implementation * 1.22.1 Automatically generated by python-semantic-release * docs: Fix dependency library name for bm25s (#1568) * fix: bm25s implementation * correct library name --------- Co-authored-by: Daniel Buades Marcos * fix: Add training dataset to model meta (#1561) * fix: Add training dataset to model meta Adresses #1556 * Added docs * format * feat: (cohere_models) cohere_task_type issue, batch requests and tqdm for visualization (#1564) * feat: batch requests to cohere models * fix: use correct task_type * feat: use tqdm with openai * fix: explicitely set `show_progress_bar` to False * fix(publichealth-qa): ignore rows with `None` values in `question` or `answer` (#1565) * 1.23.0 Automatically generated by python-semantic-release * fix: Added metadata for miscellaneous models (#1557) * Added script for generating metadata, and metadata for the listed models * Added misc models to overview * Fixed misc metas * Removed unnecessary imports * Added logic to retrieve base model information * Added base models to misc meta * Added superseded_by to sentence-croissant models * Added training datasets to mis models * 1.23.1 Automatically generated by python-semantic-release * fix: Added radar chart displaying capabilities on task types (#1570) * Added radar chart displaying capabilities on task types * Fixed table aggregation in leaderboard * Spelled out why instructionretrieval is excluded * 1.23.2 Automatically generated by python-semantic-release * feat: add new arctic v2.0 models (#1574) * feat: add new arctic v2.0 models * chore: make lint * 1.24.0 Automatically generated by python-semantic-release * fix: Add namaa MrTydi reranking dataset (#1573) * Add dataset class and file requirements * pass tests * make lint changes * adjust meta data and remove load_data --------- Co-authored-by: Omar Elshehy * Update tasks table * 1.24.1 Automatically generated by python-semantic-release * fix: Eval langs not correctly passed to monolingual tasks (#1587) * fix SouthAfricanLangClassification.py * add check for langs * lint * 1.24.2 Automatically generated by python-semantic-release * feat: Add ColBert (#1563) * feat: add max_sim operator for IR tasks to support multi-vector models * docs: add doc for Model2VecWrapper.__init__(...) * feat: add ColBERTWrapper to models & add ColBERTv2 * fix: resolve issues * fix: resolve issues * Update README.md Co-authored-by: Roman Solomatin * Update README.md Co-authored-by: Isaac Chung * Update README.md Co-authored-by: Isaac Chung * Update mteb/evaluation/evaluators/RetrievalEvaluator.py Co-authored-by: Isaac Chung * Update README.md Co-authored-by: Isaac Chung * README.md: rm subset * doc: update example for Late Interaction * get colbert running without errors * fix: pass is_query to pylate * fix: max_sim add pad_sequence * feat: integrate Jinja templates for ColBERTv2 and add model prompt handling * feat: add revision & prompt_name * doc: pad_sequence * rm TODO jina colbert v2 * doc: warning: higher resource usage for MaxSim --------- Co-authored-by: sam021313 <40773225+sam021313@users.noreply.github.com> Co-authored-by: Roman Solomatin Co-authored-by: Isaac Chung * 1.25.0 Automatically generated by python-semantic-release * doc: colbert add score_function & doc section (#1592) * doc: colbert add score_function & doc section * doc: Update README.md Co-authored-by: Kenneth Enevoldsen * doc: Update README.md Co-authored-by: Isaac Chung --------- Co-authored-by: sam021313 <40773225+sam021313@users.noreply.github.com> Co-authored-by: Kenneth Enevoldsen Co-authored-by: Isaac Chung * Feat: add support for scoring function (#1594) * add support for scoring function * lint * move similarity to wrapper * remove score function * lint * remove from InstructionRetrievalEvaluator * Update mteb/evaluation/evaluators/RetrievalEvaluator.py Co-authored-by: Kenneth Enevoldsen * remove score function from README.md --------- Co-authored-by: Kenneth Enevoldsen * Add new models nvidia, gte, linq (#1436) * Add new models nvidia, gte, linq * add warning for gte-Qwen and nvidia models re: instruction used in docs as well --------- Co-authored-by: isaac-chung * Leaderboard: Refined plots (#1601) * Added embedding size guide to performance-size plot, removed shading on radar chart * Changed plot names to something more descriptive * Made plots failsafe * fix: Leaderboard refinements (#1603) * Added explanation of aggregate measures * Added download button to result tables * Task info gets sorted by task name * Added custom, shareable links for each benchmark * Moved explanation of aggregate metrics to the summary tab * 1.25.1 Automatically generated by python-semantic-release * Feat: Use similarity scores if available (#1602) * Use similarity scores if available * lint * Add NanoBEIR Datasets (#1588) * add NanoClimateFeverRetrieval task, still requires some debugging * move task to correct place in init file * add all Nano datasets and results * format code * Update mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py Co-authored-by: Roman Solomatin * pin revision to commit and add datasets to benchmark.py * create new benchmark for NanoBEIR * add revision when loading datasets * lint --------- Co-authored-by: Roman Solomatin Co-authored-by: isaac-chung * Update tasks table * Feat: Evaluate missing languages (#1584) * init * fix tests * update mock retrieval * update tests * use subsets instead of langs * Apply suggestions from code review Co-authored-by: Isaac Chung * fix tests * add to readme * rename subset in readme --------- Co-authored-by: Isaac Chung * Add IBM Granite Embedding Models (#1613) * add IBM granite embedding models * lint formatting * add adapted_from and superseded_by to ModelMeta * fix: disable co2_tracker for API models (#1614) * 1.25.2 Automatically generated by python-semantic-release * fix: set `use_instructions` to True in models using prompts (#1616) feat: set `use_instructions` to True in models using prompts * 1.25.3 Automatically generated by python-semantic-release * fix: override existing results (#1617) * fix override existing results * lint * fix tests * add tests with overwrite * lint * update tests * lint * update * lint * 1.25.4 Automatically generated by python-semantic-release * add MSMARCO eval split in MTEB English (classic) benchmark (#1620) * add MSMARCO eval split in MTEB English (classic) benchmark Fixes #1608 * Add co-author Co-authored-by: aashka-trivedi --------- Co-authored-by: aashka-trivedi * fix: GermanDPR Dataset Causes Cross-Encoder Failure Due to Unexpected dict (#1621) Fixes #1609 * fix: properly add mteb_model_meta to model object (#1623) * 1.25.5 Automatically generated by python-semantic-release * Feat: Add jasper (#1591) * init jasper * init jasper * add to overview * add to overview * remove some params * fix max length * return sdpa * add dtype * add dtype * fix convert_to_tensor * change to encode * return whitespace processing * explicitly add instructions * move seq length * try float * fix max_seq_length * add prompt validation to format instruction * don't use instructions only to s2p * fix: Update results_to_dataframe to use BenchmarkResults class (#1628) * 1.25.6 Automatically generated by python-semantic-release * Speed up test_save_predictions (#1631) * fix: Correction of discrepancies for gte-Qweb model (#1637) * 1.25.7 Automatically generated by python-semantic-release * fix: output_folder for co2 evaluation (#1642) * 1.25.8 Automatically generated by python-semantic-release * fix: add missing benchmark to benchmarks.py (#1641) add missing benchmark * 1.25.9 Automatically generated by python-semantic-release * fix: Cast all Model2Vec outputs as floats (#1667) cast all outputs as floats * 1.25.10 Automatically generated by python-semantic-release * fix: Update gritlm kwargs (#1643) * Fix kwarg * format --------- Co-authored-by: Kenneth Enevoldsen * 1.25.11 Automatically generated by python-semantic-release * fix: Use batch size kwargs for openai APIs (#1668) Fixes #1645 * 1.25.12 Automatically generated by python-semantic-release * fix: Pass trust_remote_code=True to CPM model (#1669) Fixes #1651 * 1.25.13 Automatically generated by python-semantic-release * fix: Updated metadata for CPM (#1670) * fix: Pass trust_remote_code=True to CPM model Fixes #1651 * fix: Updated metadata for cpm * 1.25.14 Automatically generated by python-semantic-release * fix: remove model as a parameter for MulticlassClassification (#1666) remove model parameter * fix: Use prompts instead of prompt names for voyage (#1665) * fix prompt names * lint * change input type * 1.25.15 Automatically generated by python-semantic-release * fix: Update BUCC dataset revision (#1674) * trust remote code * Update revision * 1.25.16 Automatically generated by python-semantic-release * fix: Add warning for non-retrieval tasks when using bm25s (#1678) * clean up install instruction * add check for bm25s and skip non-retrieval tasks * add versions * 1.25.17 Automatically generated by python-semantic-release * fix: add check for key error in loader (#1675) * add check for key error * make KeyError everywhere * update error * 1.25.18 Automatically generated by python-semantic-release * fix: trust remote code for snowflake-arctic-embed-m-v2.0 (#1682) trust remote code * 1.25.19 Automatically generated by python-semantic-release * fix: nomic tensor return (#1683) * fix nomic tensor return * add typehint * 1.25.20 Automatically generated by python-semantic-release * feat: add `avsolatorio/NoInstruct-small-Embedding-v0` (#1677) add no_instruct * fix: arg name for openbmb/MiniCPM-Embedding (#1691) fix name * 1.26.0 Automatically generated by python-semantic-release * fix: add trust_remote_code to Snowflake/snowflake-arctic-embed-m-long (#1695) trust remote code * fix: add revision for jinaai/jina-embeddings-v2-small-en (#1692) add revision * 1.26.1 Automatically generated by python-semantic-release * fix: update model loader to trust remote code (#1697) update model loader * 1.26.2 Automatically generated by python-semantic-release * fix: nomic prompts (#1685) * fix nomic prompts * fix variable model name * pass prompts to model * use sentence transformer wrapper * update prompts * lint * update prompts * update list for classification * fix: NanoBeir (#1687) * fix nano beir * lint * 1.26.3 Automatically generated by python-semantic-release * Update RerankingEvaluator.py (#1702) * fix: Register MicroLlama Text Embedding (#1644) Register MicroLlama Text Embedding * fix: GermanDPR (#1703) * fix GermanDPR * lint * 1.26.4 Automatically generated by python-semantic-release * Fix: minicpmv2 (#1705) * updmini cpm * flash_attn implementation * remove flash attn * ci: Refresh the v2 leaderboard daily (#1711) * Create leaderboard_refresh.yaml * Shorten and fix * factory reset instead of normal * Fix: typos in adding a model (#1722) * fix: rollback BUCC revision (#1706) * fix bucc * fix logger * upd evaluator * add comment * lint * 1.26.5 Automatically generated by python-semantic-release * fix: Added zero shot tag to benchmark (#1710) * Added method for determining whether a model is zero shot * Added .items() where intended * Added filtering functions for zero shot models * Added zero-shot filtering button and error message when table is empty.: * Ran linting * Fixed docstring linting error * is_zero_shot returns None when no training data is specified * Added zero-shot emoji column to leaderboard * Added explanation for zero shot column * Added soft and hard zero-shot buttons * Added training data annotations to 24 models from HuggingFace Hub * 1.26.6 Automatically generated by python-semantic-release * feat: reduce logging for load_results() - redacts missing subsets to avoid 100+ subsets printed - reduce to logging.info - removed splits that are commonly never evaluated on and thus also the errors for them being missing The second part removed quite a few warnings (4930 to XX) It also seems like the splits were accidentally included in some of the MMTEB benchmark. This will remove those splits from those benchmarks (which are all in beta). We will have to recompute the tables for the paper though (we should do that anyway) Other potential thing to consider: - Scifact is included in MTEB(Medical). I have removed the "train" split from it as I think that was a mistake. (checked other dataset in benchmark) Here is a count of the current top errors: ```py { "MassiveScenarioClassification: Missing splits {'validation'}": 238, # included in e.g. mteb(fra) "MassiveIntentClassification: Missing splits {'validation'}": 237, # included in e.g. mteb(fra) "MassiveScenarioClassification: Missing subsets {'af', 'da', ...} for split test": 230, "AmazonReviewsClassification: Missing splits {'validation'}": 229, # included in e.g. mteb(deu) "MassiveIntentClassification: Missing subsets {'af', 'da', ...} for split test": 228, "STS22: Missing subsets {'fr-pl', 'de-en', ...} for split test": 223, "AmazonReviewsClassification: Missing subsets {'es', 'ja', ...} for split test": 196, "MTOPDomainClassification: Missing splits {'validation'}": 195, # included in mteb(fra) "MTOPIntentClassification: Missing splits {'validation'}": 194, # included in mteb(fra) "AmazonCounterfactualClassification: Missing splits {'validation'}": 189, # included in mteb(deu) "MTOPDomainClassification: Missing subsets {'es', 'th', ...} for split test": 165, "STS17: Missing subsets {'en-ar', 'es-es', ...} for split test": 164, "MTOPIntentClassification: Missing subsets {'es', 'th', ...} for split test": 164, "AmazonCounterfactualClassification: Missing subsets {'de', 'ja', ...} for split test": 148, } ``` * 1.27.0 Automatically generated by python-semantic-release * feat: Add nomic modern bert (#1684) * add nomic modern bert * use SentenceTransformerWrapper * use SentenceTransformerWrapper * try nomic wrapper * update * use all prompts * pass prompts * use fp16 * lint * change to version * remove commented code * fix: allow kwargs in init for RerankingWrapper (#1676) * allow kwargs in init * fix retrieval * convert corpus_in_pair to list * 1.28.0 Automatically generated by python-semantic-release * Fixed result loading on leaderboard (#1739) * Only main_score gets loaded for leaderboard thereby avoiding OOM errors * Fixed plot failing because of missing embedding dimensions * Ran linting * test: Add script to test model loading below n_parameters threshold (#1698) * add model loading test for models below 2B params * add failure message to include model namne * use the real get_model_meta * use cache folder * teardown per function * fix directory removal * write to file * wip loading from before * wip * Rename model_loading_testing.py to model_loading.py * Delete tests/test_models/test_model_loading.py * checks for models below 2B * try not using cache folder * update script with scan_cache_dir and add args * add github CI: detect changed model files and run model loading test * install all model dependencies * dependecy installations and move file location * should trigger a model load test in CI * find correct commit for diff * explicity fetch base branch * add make command * try to run in python instead and add pytest * fix attribute error and add read mode * separate script calling * let pip install be cached and specify repo path * check ancestry * add cache and rebase * try to merge instead of rebase * try without merge base * check if file exists first * Apply suggestions from code review Co-authored-by: Kenneth Enevoldsen * Update .github/workflows/model_loading.yml Co-authored-by: Kenneth Enevoldsen * address review comments to run test once from CI and not pytest --------- Co-authored-by: Kenneth Enevoldsen * fix: Leaderboard Speedup (#1745) * Added get_scores_fast * Made leaderboard faster with smarter dependency graph and event management and caching * Changed print to logger.info * 1.28.1 Automatically generated by python-semantic-release * fix: Fixed task_type aggregation on leaderboard (#1746) * Fixed task_type aggregation in leaderboard * Fixed an error due to unneccesary indentation in get_score * 1.28.2 Automatically generated by python-semantic-release * fix: Fixed definition of zero-shot in ModelMeta (#1747) * Corrected zero_shot definition to be based on task names, not dataset path * 1.28.3 Automatically generated by python-semantic-release * fix: fixes implementation of similarity() (#1748) * fix(#1594): fixes implementation of similarity() * fix: add similarity to SentenceTransformerWrapper --------- Co-authored-by: sam021313 <40773225+sam021313@users.noreply.github.com> * 1.28.4 Automatically generated by python-semantic-release * fix: Leaderboard: `K` instead of `M` (#1761) Fixes #1752 * other: add script for leaderboard compare (#1758) * add script * remove changes * remove changes * add comment * lint * order like in benchmark object * round results * 1.28.5 Automatically generated by python-semantic-release * fix: added annotations for training data (#1742) * fix: Added annotations for arctic embed models * added google and bge * added cohere * Added e5 * added bge based model2vec * annotated oAI * format and update annotations * 1.28.6 Automatically generated by python-semantic-release * fix: update max tokens for OpenAI (#1772) update max tokens * ci: skip AfriSentiLID for now (#1785) * skip AfriSentiLID for now * skip relevant test case instead --------- Co-authored-by: Isaac Chung * 1.28.7 Automatically generated by python-semantic-release * ci: fix model loading test (#1775) * pass base branch into the make command as an arg * test a file that has custom wrapper * what about overview * just dont check overview * revert instance check * explicitly omit overview and init * remove test change * try on a lot of models * revert test model file --------- Co-authored-by: Isaac Chung * feat: Update task filtering, fixing bug which included cross-lingual tasks in overly many benchmarks (#1787) * feat: Update task filtering, fixing bug on MTEB - Updated task filtering adding exclusive_language_filter and hf_subset - fix bug in MTEB where cross-lingual splits were included - added missing language filtering to MTEB(europe, beta) and MTEB(indic, beta) The following code outlines the problems: ```py import mteb from mteb.benchmarks import MTEB_ENG_CLASSIC task = [t for t in MTEB_ENG_CLASSIC.tasks if t.metadata.name == "STS22"][0] # was eq. to: task = mteb.get_task("STS22", languages=["eng"]) task.hf_subsets # correct filtering to English datasets: # ['en', 'de-en', 'es-en', 'pl-en', 'zh-en'] # However it should be: # ['en'] # with the changes it is: task = [t for t in MTEB_ENG_CLASSIC.tasks if t.metadata.name == "STS22"][0] task.hf_subsets # ['en'] # eq. to task = mteb.get_task("STS22", hf_subsets=["en"]) # which you can also obtain using the exclusive_language_filter (though not if there was multiple english splits): task = mteb.get_task("STS22", languages=["eng"], exclusive_language_filter=True) ``` * format * remove "en-ext" from AmazonCounterfactualClassification * fixed mteb(deu) * fix: simplify in a few areas * 1.29.0 Automatically generated by python-semantic-release * fix: Added C-MTEB (#1786) Added C-MTEB * 1.29.1 Automatically generated by python-semantic-release * docs: Add contact to MMTEB benchmarks (#1796) * Add myself to MMTEB benchmarks * lint * fix: loading pre 11 (#1798) * fix loading pre 11 * add similarity * lint * run all task types * 1.29.2 Automatically generated by python-semantic-release * fix: allow to load no revision available (#1801) * fix allow to load no revision available * lint * add require_model_meta to leaderboard * lint * 1.29.3 Automatically generated by python-semantic-release * fix: Zero shot and aggregation on Leaderboard (#1810) * Made join_revision filter out no_revision_available when other revisions have been run on the task * Fixed zero-shot filtering * Fixed aggregation of task types * Ran linting * fix: Added `ModelMeta` for BGE, GTE Chinese and multilingual models (#1811) * Added BGE Chinese and multilingual-gemma models * Added GTE multilingual and Chinese models * Fixed date format * 1.29.4 Automatically generated by python-semantic-release * fix: Add additional contacts (#1817) add contacts from #1790 * Update points table * 1.29.5 Automatically generated by python-semantic-release * fix: Added more Chinese models' `ModelMeta` (#1814) * Added Multilingual USE models * Added Moka models * Added dmeta models * Added jina-zh * Added piccolo models * 1.29.6 Automatically generated by python-semantic-release * Add model inf-retriever-v1 (#1744) * feat(models): add infly/inf-retriever-v1 model metadata- Add inf_models.py file with metadata for infly/inf-retriever-v1 model - Update overview.py to include inf_models in model imports * Reformat code * Update inf-retriever-v1 ModelMeta * Fill more information for inf-retriever-v1 * Add license information for inf-retriever-v1 --------- Co-authored-by: Samuel Yang * ci: only return 1 model_name per file (#1818) * only return 1 model_name per file * fix args parse * revert test change * fix: add bge-m3 `ModelMeta` (#1821) add bge * 1.29.7 Automatically generated by python-semantic-release * fix: Added Chinese Stella models (#1824) Added Chinese Stella models * fix: bm25s (#1827) Co-authored-by: sam021313 <40773225+sam021313@users.noreply.github.com> * fix: Added way more training dataset annotations (#1765) * fix: Leaderboard: `K` instead of `M` Fixes #1752 * format * fixed existing annotations to refer to task name instead of hf dataset * added annotation to nvidia * added voyage * added uae annotations * Added stella annotations * sentence trf models * added salesforce and e5 * jina * bge + model2vec * added llm2vec annotations * add jasper * format * format * Updated annotations and moved jina models * fix: add even more training dataset annotations (#1793) * fix: update max tokens for OpenAI (#1772) update max tokens * ci: skip AfriSentiLID for now (#1785) * skip AfriSentiLID for now * skip relevant test case instead --------- Co-authored-by: Isaac Chung * 1.28.7 Automatically generated by python-semantic-release * ci: fix model loading test (#1775) * pass base branch into the make command as an arg * test a file that has custom wrapper * what about overview * just dont check overview * revert instance check * explicitly omit overview and init * remove test change * try on a lot of models * revert test model file --------- Co-authored-by: Isaac Chung * feat: Update task filtering, fixing bug which included cross-lingual tasks in overly many benchmarks (#1787) * feat: Update task filtering, fixing bug on MTEB - Updated task filtering adding exclusive_language_filter and hf_subset - fix bug in MTEB where cross-lingual splits were included - added missing language filtering to MTEB(europe, beta) and MTEB(indic, beta) The following code outlines the problems: ```py import mteb from mteb.benchmarks import MTEB_ENG_CLASSIC task = [t for t in MTEB_ENG_CLASSIC.tasks if t.metadata.name == "STS22"][0] # was eq. to: task = mteb.get_task("STS22", languages=["eng"]) task.hf_subsets # correct filtering to English datasets: # ['en', 'de-en', 'es-en', 'pl-en', 'zh-en'] # However it should be: # ['en'] # with the changes it is: task = [t for t in MTEB_ENG_CLASSIC.tasks if t.metadata.name == "STS22"][0] task.hf_subsets # ['en'] # eq. to task = mteb.get_task("STS22", hf_subsets=["en"]) # which you can also obtain using the exclusive_language_filter (though not if there was multiple english splits): task = mteb.get_task("STS22", languages=["eng"], exclusive_language_filter=True) ``` * format * remove "en-ext" from AmazonCounterfactualClassification * fixed mteb(deu) * fix: simplify in a few areas * fix: Add gritlm * 1.29.0 Automatically generated by python-semantic-release * fix: Added more annotations! * fix: Added C-MTEB (#1786) Added C-MTEB * 1.29.1 Automatically generated by python-semantic-release * docs: Add contact to MMTEB benchmarks (#1796) * Add myself to MMTEB benchmarks * lint * fix: loading pre 11 (#1798) * fix loading pre 11 * add similarity * lint * run all task types * 1.29.2 Automatically generated by python-semantic-release * fix: allow to load no revision available (#1801) * fix allow to load no revision available * lint * add require_model_meta to leaderboard * lint * 1.29.3 Automatically generated by python-semantic-release --------- Co-authored-by: Roman Solomatin Co-authored-by: Isaac Chung Co-authored-by: Isaac Chung Co-authored-by: github-actions Co-authored-by: Márton Kardos --------- Co-authored-by: Roman Solomatin Co-authored-by: Isaac Chung Co-authored-by: Isaac Chung Co-authored-by: github-actions Co-authored-by: Márton Kardos * fix: Added Misc Chinese models (#1819) * Added moka and piccolo models to overview file * Added Text2Vec models * Added various Chinese embedding models --------- Co-authored-by: Isaac Chung * 1.29.8 Automatically generated by python-semantic-release * fix: Fixed eval split for MultilingualSentiment in C-MTEB (#1804) * Fixed eval split for MultilingualSentiment in C-MTEB * FIxed splits for atec, bq and stsb in C-MTEB * 1.29.9 Automatically generated by python-semantic-release * fix: subsets to run (#1830) * fix split evals * add test * lint * fix moka * add assert * fix: Remove default params, `public_training_data` and `memory usage` in `ModelMeta` (#1794) * fix: Leaderboard: `K` instead of `M` Fixes #1752 * format * fixed existing annotations to refer to task name instead of hf dataset * added annotation to nvidia * added voyage * added uae annotations * Added stella annotations * sentence trf models * added salesforce and e5 * jina * bge + model2vec * added llm2vec annotations * add jasper * format * format * Updated annotations and moved jina models * make models parameters needed to be filled * fix tests * remove comments * remove model meta from test * fix model meta from split * fix: add even more training dataset annotations (#1793) * fix: update max tokens for OpenAI (#1772) update max tokens * ci: skip AfriSentiLID for now (#1785) * skip AfriSentiLID for now * skip relevant test case instead --------- Co-authored-by: Isaac Chung * 1.28.7 Automatically generated by python-semantic-release * ci: fix model loading test (#1775) * pass base branch into the make command as an arg * test a file that has custom wrapper * what about overview * just dont check overview * revert instance check * explicitly omit overview and init * remove test change * try on a lot of models * revert test model file --------- Co-authored-by: Isaac Chung * feat: Update task filtering, fixing bug which included cross-lingual tasks in overly many benchmarks (#1787) * feat: Update task filtering, fixing bug on MTEB - Updated task filtering adding exclusive_language_filter and hf_subset - fix bug in MTEB where cross-lingual splits were included - added missing language filtering to MTEB(europe, beta) and MTEB(indic, beta) The following code outlines the problems: ```py import mteb from mteb.benchmarks import MTEB_ENG_CLASSIC task = [t for t in MTEB_ENG_CLASSIC.tasks if t.metadata.name == "STS22"][0] # was eq. to: task = mteb.get_task("STS22", languages=["eng"]) task.hf_subsets # correct filtering to English datasets: # ['en', 'de-en', 'es-en', 'pl-en', 'zh-en'] # However it should be: # ['en'] # with the changes it is: task = [t for t in MTEB_ENG_CLASSIC.tasks if t.metadata.name == "STS22"][0] task.hf_subsets # ['en'] # eq. to task = mteb.get_task("STS22", hf_subsets=["en"]) # which you can also obtain using the exclusive_language_filter (though not if there was multiple english splits): task = mteb.get_task("STS22", languages=["eng"], exclusive_language_filter=True) ``` * format * remove "en-ext" from AmazonCounterfactualClassification * fixed mteb(deu) * fix: simplify in a few areas * fix: Add gritlm * 1.29.0 Automatically generated by python-semantic-release * fix: Added more annotations! * fix: Added C-MTEB (#1786) Added C-MTEB * 1.29.1 Automatically generated by python-semantic-release * docs: Add contact to MMTEB benchmarks (#1796) * Add myself to MMTEB benchmarks * lint * fix: loading pre 11 (#1798) * fix loading pre 11 * add similarity * lint * run all task types * 1.29.2 Automatically generated by python-semantic-release * fix: allow to load no revision available (#1801) * fix allow to load no revision available * lint * add require_model_meta to leaderboard * lint * 1.29.3 Automatically generated by python-semantic-release --------- Co-authored-by: Roman Solomatin Co-authored-by: Isaac Chung Co-authored-by: Isaac Chung Co-authored-by: github-actions Co-authored-by: Márton Kardos * fig merges * update models info * change public_training_code to str * change `public_training_code=False` to None * remove annotations * remove annotations * remove changed annotations * remove changed annotations * remove `public_training_data` and `memory usage` * make framework not optional * make framework non-optional * empty frameworks * add framework * fix tests * Update mteb/models/overview.py Co-authored-by: Isaac Chung --------- Co-authored-by: Kenneth Enevoldsen Co-authored-by: Isaac Chung Co-authored-by: Isaac Chung Co-authored-by: github-actions Co-authored-by: Márton Kardos * 1.29.10 Automatically generated by python-semantic-release * fix: Add reported annotation and re-added public_training_data (#1846) * fix: Add additional dataset annotations * fix: readded public training data * update voyage annotations * 1.29.11 Automatically generated by python-semantic-release * fix: Leaderboard Refinements (#1849) * Added better descriptions to benchmarks and removed beta tags * Fixed zero-shot filtering on app loading * Added zero-shot definition in an accordion * NaN values are now filled with blank * Added type hints to filter_models * 1.29.12 Automatically generated by python-semantic-release * rest of the merge conflicts * fix merge conflicts * fill in model meta defaults * fix ModeMeta modalities * fix metadata pydantic errors; * assert model.model instead since it is a wrapper * fix: Fixed leaderboard search bar (#1852) Fixed leaderboard search bar * 1.29.13 Automatically generated by python-semantic-release * fix: Hotfixed public_training_data type annotation (#1857) Fixed public_training_data flag type to include boolean, as this is how all models are annotated * fix: Fix zeta alpha mistral (#1736) * fix zeta alpha mistral * update use_instructions * update training datasets * Update mteb/models/e5_instruct.py Co-authored-by: Kenneth Enevoldsen * update float * Update mteb/models/e5_instruct.py --------- Co-authored-by: Kenneth Enevoldsen * Add more annotations (#1833) * apply additions from #1794 * add annotations for rumodels * add nomic training data * fix metadata * update rest of model meta * fix bge reranker * 1.29.14 Automatically generated by python-semantic-release * fix: Adding missing model meta (#1856) * Added CDE models * Added bge-en-icl * Updated CDE to bge_full_data * Fixed public_training_data flag type to include boolean, as this is how all models are annotated * Added public training data link instead of bool to CDE and BGE * Added GME models * Changed Torch to PyTorch * Added metadata on LENS models * Added ember_v1 * Added metadata for amazon titan * Removed GME implementation * fix Encoder class --------- Co-authored-by: github-actions[bot] Co-authored-by: github-actions Co-authored-by: Helena Kloosterman Co-authored-by: Alexey Vatolin Co-authored-by: Kenneth Enevoldsen Co-authored-by: Elias H <40372306+eherra@users.noreply.github.com> Co-authored-by: Youngjoon Jang <82500463+yjoonjang@users.noreply.github.com> Co-authored-by: Márton Kardos Co-authored-by: Roman Solomatin Co-authored-by: Napuh <55241721+Napuh@users.noreply.github.com> Co-authored-by: Daniel Buades Marcos Co-authored-by: nadshe Co-authored-by: olivierr42 Co-authored-by: Thivyanth Co-authored-by: Rafał Poświata Co-authored-by: Omar Elshehy <41394057+omarelshehy@users.noreply.github.com> Co-authored-by: Omar Elshehy Co-authored-by: Sam <40773225+sam-hey@users.noreply.github.com> Co-authored-by: sam021313 <40773225+sam021313@users.noreply.github.com> Co-authored-by: KGupta10 <92774828+KGupta10@users.noreply.github.com> Co-authored-by: Aashka Trivedi Co-authored-by: Niklas Muennighoff Co-authored-by: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com> Co-authored-by: Ken Wang Co-authored-by: Orion Weller <31665361+orionw@users.noreply.github.com> Co-authored-by: Isaac Chung Co-authored-by: Samuel Yang Co-authored-by: Samuel Yang --- .github/workflows/leaderboard_refresh.yaml | 16 + .github/workflows/model_loading.yml | 24 + .gitignore | 6 +- Makefile | 9 +- README.md | 78 +- docs/adding_a_dataset.md | 7 +- docs/adding_a_model.md | 31 +- docs/create_tasks_table.py | 62 +- docs/mmteb/points_table.md | 103 + docs/tasks.md | 1756 +- mteb/__init__.py | 9 +- mteb/abstasks/AbsTask.py | 116 +- mteb/abstasks/AbsTaskBitextMining.py | 53 +- mteb/abstasks/AbsTaskClassification.py | 90 +- mteb/abstasks/AbsTaskClustering.py | 34 +- mteb/abstasks/AbsTaskClusteringFast.py | 39 +- mteb/abstasks/AbsTaskInstructionRetrieval.py | 119 +- .../AbsTaskMultilabelClassification.py | 82 +- mteb/abstasks/AbsTaskPairClassification.py | 50 +- mteb/abstasks/AbsTaskReranking.py | 99 +- mteb/abstasks/AbsTaskRetrieval.py | 90 +- mteb/abstasks/AbsTaskSTS.py | 42 +- mteb/abstasks/AbsTaskSummarization.py | 87 +- mteb/abstasks/MultilingualTask.py | 4 +- mteb/abstasks/TaskMetadata.py | 89 +- mteb/benchmarks/benchmarks.py | 679 +- mteb/caching.py | 19 + .../BitextMining/BUCC.v2.json | 69 + .../BitextMining/BornholmBitextMining.json | 15 + .../BitextMining/IN22ConvBitextMining.json | 6595 ++ .../BitextMining/IN22GenBitextMining.json | 6595 ++ .../BitextMining/IWSLT2017BitextMining.json | 329 + .../IndicGenBenchFloresBitextMining.json | 1540 + .../BitextMining/NTREXBitextMining.json | 24925 ++++++ .../BitextMining/NollySentiBitextMining.json | 69 + .../NorwegianCourtsBitextMining.json | 15 + .../NusaTranslationBitextMining.json | 160 + .../BitextMining/PhincBitextMining.json | 30 + .../TbilisiCityHallBitextMining.json | 43 + .../BitextMining/VieMedEVBitextMining.json | 15 + .../LanguageClassification.json | 146 + .../SlovakHateSpeechClassification.json | 38 + .../ArXivHierarchicalClusteringP2P.json | 402 + .../Clustering/BiorxivClusteringS2S.json | 94 + .../Clustering/MedrxivClusteringP2P.v2.json | 168 + .../Clustering/MedrxivClusteringS2S.v2.json | 168 + .../Clustering/RedditClusteringP2P.v2.json | 1335 + .../RuSciBenchGRNTIClusteringP2P.json | 99 + .../TwentyNewsgroupsClustering.v2.json | 75 + .../Clustering/WikiClusteringP2P.json | 1948 + .../Core17InstructionRetrieval.json | 30 + .../CEDRClassification.json | 68 + .../PawsXPairClassification.json | 342 + .../PairClassification/TwitterURLCorpus.json | 23 + .../PairClassification/XNLI.json | 636 + .../Reranking/AskUbuntuDupQuestions.json | 20 + .../Reranking/ESCIReranking.json | 76 + .../WikipediaRerankingMultilingual.json | 310 + .../Retrieval/AppsRetrieval.json | 20 + .../Retrieval/AutoRAGRetrieval.json | 11 + .../Retrieval/BelebeleRetrieval.json | 6790 ++ .../Retrieval/COIRCodeSearchNetRetrieval.json | 130 + .../Retrieval/CodeEditSearchRetrieval.json | 256 + .../Retrieval/CodeFeedbackMT.json | 20 + .../Retrieval/CodeFeedbackST.json | 20 + .../Retrieval/CodeSearchNetCCRetrieval.json | 130 + .../Retrieval/CodeSearchNetRetrieval.json | 130 + .../Retrieval/CodeTransOceanContest.json | 20 + .../Retrieval/CodeTransOceanDL.json | 20 + mteb/descriptive_stats/Retrieval/CosQA.json | 20 + .../Retrieval/JaqketRetrieval.json | 20 + .../descriptive_stats/Retrieval/NFCorpus.json | 11 + .../Retrieval/StackOverflowQA.json | 20 + .../Retrieval/SyntheticText2SQL.json | 20 + .../Retrieval/Touche2020.json | 20 + .../Retrieval/Touche2020Retrieval.v3.json | 20 + ...lowIRCrossLingualInstructionRetrieval.json | 116 + .../mFollowIRInstructionRetrieval.json | 116 + mteb/descriptive_stats/STS/STS12.json | 17 + mteb/descriptive_stats/STS/STS17.json | 184 + .../Summarization/SummEval.json | 55 + mteb/encoder_interface.py | 4 +- mteb/evaluation/MTEB.py | 305 +- .../evaluators/BitextMiningEvaluator.py | 43 +- .../evaluators/ClassificationEvaluator.py | 47 +- .../evaluators/ClusteringEvaluator.py | 11 +- .../InstructionRetrievalEvaluator.py | 2 - .../evaluators/PairClassificationEvaluator.py | 11 +- .../evaluators/RerankingEvaluator.py | 34 +- .../evaluators/RetrievalEvaluator.py | 137 +- mteb/evaluation/evaluators/STSEvaluator.py | 21 +- .../evaluators/SummarizationEvaluator.py | 37 +- mteb/language_family.json | 62611 ++++++++++++++++ mteb/languages.py | 22 +- mteb/leaderboard/app.py | 630 +- mteb/leaderboard/figures.py | 271 + mteb/leaderboard/table.py | 274 +- mteb/load_results/benchmark_results.py | 255 +- mteb/load_results/load_results.py | 12 +- mteb/load_results/task_results.py | 103 +- mteb/model_meta.py | 64 +- mteb/models/align_models.py | 13 +- mteb/models/arctic_models.py | 451 +- mteb/models/bge_models.py | 522 +- mteb/models/blip2_models.py | 26 +- mteb/models/blip_models.py | 104 +- mteb/models/bm25.py | 22 +- mteb/models/cde_models.py | 54 + mteb/models/clip_models.py | 39 +- mteb/models/cohere_models.py | 262 +- mteb/models/cohere_v.py | 16 +- mteb/models/colbert_models.py | 228 + mteb/models/dino_models.py | 52 +- mteb/models/e5_instruct.py | 147 +- mteb/models/e5_models.py | 143 +- mteb/models/e5_v.py | 13 +- mteb/models/evaclip_models.py | 52 +- mteb/models/gme_models.py | 471 +- mteb/models/gme_v_models.py | 453 + mteb/models/google_models.py | 153 +- mteb/models/gritlm_models.py | 76 +- mteb/models/gte_models.py | 322 +- mteb/models/ibm_granite_models.py | 185 + mteb/models/inf_models.py | 31 + mteb/models/instruct_wrapper.py | 80 + mteb/models/jasper_models.py | 97 + mteb/models/jina_clip.py | 13 +- mteb/models/jina_models.py | 329 + mteb/models/lens_models.py | 43 + mteb/models/linq_models.py | 45 + mteb/models/llm2vec_models.py | 81 +- mteb/models/misc_models.py | 1777 + mteb/models/moco_models.py | 26 +- mteb/models/model2vec_models.py | 231 + mteb/models/moka_models.py | 147 + mteb/models/mxbai_models.py | 8 +- mteb/models/no_instruct_sentence_models.py | 105 + mteb/models/nomic_models.py | 257 +- mteb/models/nomic_models_vision.py | 13 +- mteb/models/nvidia_models.py | 169 + mteb/models/openai_models.py | 139 +- mteb/models/openclip_models.py | 104 +- mteb/models/overview.py | 149 +- mteb/models/piccolo_models.py | 48 + mteb/models/promptriever_models.py | 24 +- mteb/models/repllama_models.py | 20 +- mteb/models/rerankers_custom.py | 75 +- mteb/models/rerankers_monot5_based.py | 267 +- mteb/models/ru_sentence_models.py | 234 +- mteb/models/salesforce_models.py | 102 +- mteb/models/sentence_transformer_wrapper.py | 104 +- mteb/models/sentence_transformers_models.py | 184 +- mteb/models/siglip_models.py | 130 +- mteb/models/stella_models.py | 168 + mteb/models/text2vec_models.py | 100 + mteb/models/uae_models.py | 87 + mteb/models/utils.py | 16 + mteb/models/vista_models.py | 26 +- mteb/models/vlm2vec_models.py | 26 +- mteb/models/voyage_models.py | 203 +- mteb/models/voyage_v.py | 8 +- mteb/models/wrapper.py | 120 +- mteb/overview.py | 39 +- mteb/task_selection.py | 25 +- .../dan/BornholmskBitextMining.py | 29 +- .../kat/TbilisiCityHallBitextMining.py | 4 - .../multilingual/BUCCBitextMining.py | 14 +- .../multilingual/BUCCBitextMiningFast.py | 4 - .../multilingual/BibleNLPBitextMining.py | 4 - .../multilingual/DiaBLaBitextMining.py | 1 - .../multilingual/FloresBitextMining.py | 4 - .../multilingual/IN22ConvBitextMining.py | 2539 - .../multilingual/IN22GenBitextMining.py | 4 - .../multilingual/IWSLT2017BitextMining.py | 4 - .../IndicGenBenchFloresBitextMining.py | 4 - .../multilingual/LinceMTBitextMining.py | 4 - .../multilingual/NTREXBitextMining.py | 4 - .../multilingual/NollySentiBitextMining.py | 4 - .../NorwegianCourtsBitextMining.py | 5 +- .../NusaTranslationBitextMining.py | 65 - .../multilingual/NusaXBitextMining.py | 4 - .../multilingual/PhincBitextMining.py | 4 - .../multilingual/RomaTalesBitextMining.py | 4 - .../multilingual/TatoebaBitextMining.py | 4 - .../BitextMining/srn/SRNCorpusBitextMining.py | 4 - .../BitextMining/vie/VieMedEVBitextMining.py | 4 - mteb/tasks/Classification/ara/AJGT.py | 4 - .../ara/HotelReviewSentimentClassification.py | 6 - ...nlineStoreReviewSentimentClassification.py | 6 - ...RestaurantReviewSentimentClassification.py | 6 - .../ara/TweetEmotionClassification.py | 6 - .../ara/TweetSarcasmClassification.py | 4 - .../ben/BengaliDocumentClassification.py | 4 - .../ben/BengaliHateSpeechClassification.py | 4 - .../ben/BengaliSentimentAnalysis.py | 4 - ...garianStoreReviewSentimentClassfication.py | 4 - ...SFDCZMovieReviewSentimentClassification.py | 17 +- ...echProductReviewSentimentClassification.py | 12 +- .../ces/CzechSoMeSentimentClassification.py | 12 +- .../ces/CzechSubjectivityClassification.py | 4 - .../dan/AngryTweetsClassification.py | 12 +- .../dan/DKHateClassification.py | 12 +- .../DanishPoliticalCommentsClassification.py | 12 +- .../dan/DdiscoCohesionClassification.py | 1 - .../dan/LccSentimentClassification.py | 12 +- ...liticiansTwitterSentimentClassification.py | 4 - .../deu/TenKGnadClassification.py | 4 - .../ell/GreekLegalCodeClassification.py | 4 - .../eng/AmazonPolarityClassification.py | 5 +- .../Classification/eng/ArxivClassification.py | 1 - .../eng/Banking77Classification.py | 5 +- .../eng/DBpediaClassification.py | 4 - .../eng/EmotionClassification.py | 14 +- .../eng/FinancialPhrasebankClassification.py | 4 - .../eng/FrenkEnClassification.py | 4 - .../Classification/eng/ImdbClassification.py | 5 +- .../eng/LegalBenchClassification.py | 448 - .../Classification/eng/NewsClassification.py | 4 - .../eng/PatentClassification.py | 4 - .../eng/PoemSentimentClassification.py | 4 - .../eng/ToxicChatClassification.py | 4 - .../eng/ToxicConversationsClassification.py | 12 +- .../TweetSentimentExtractionClassification.py | 12 +- .../eng/TweetTopicSingleClassification.py | 4 - .../eng/YahooAnswersTopicsClassification.py | 11 +- .../eng/YelpReviewFullClassification.py | 8 +- .../Classification/est/estonian_valence.py | 14 +- .../fas/PersianFoodSentimentClassification.py | 4 - .../fil/FilipinoHateSpeechClassification.py | 4 - .../FilipinoShopeeReviewsClassification.py | 4 - .../fin/FinToxicityClassification.py | 4 - .../Classification/fra/FrenchBookReviews.py | 4 - .../fra/MovieReviewSentimentClassification.py | 6 - .../guj/GujaratiNewsClassification.py | 4 - .../heb/HebrewSentimentAnalysis.py | 4 - .../hin/HindiDiscourseClassification.py | 4 - .../hin/SentimentAnalysisHindi.py | 4 - .../hrv/FrenkHrClassification.py | 4 - .../IndonesianIdClickbaitClassification.py | 6 - ...esianMongabayConservationClassification.py | 4 - .../ita/ItaCaseholdClassification.py | 4 - ...lianLinguistAcceptabilityClassification.py | 4 - .../jav/JavaneseIMDBClassification.py | 4 - .../Classification/jpn/WRIMEClassification.py | 4 - .../kan/KannadaNewsClassification.py | 4 - .../kat/GeorgianSentimentClassification.py | 4 - mteb/tasks/Classification/kor/KlueTC.py | 4 - mteb/tasks/Classification/kor/KorFin.py | 4 - .../kor/KorHateClassification.py | 4 - .../kor/KorSarcasmClassification.py | 7 +- .../kur/KurdishSentimentClassification.py | 4 - .../mal/MalayalamNewsClassification.py | 4 - .../mar/MarathiNewsClassification.py | 4 - .../MacedonianTweetSentimentClassification.py | 4 - .../multilingual/AfriSentiClassification.py | 4 - .../AfriSentiLangClassification.py | 11 +- .../AmazonCounterfactualClassification.py | 12 +- .../AmazonReviewsClassification.py | 5 +- .../CataloniaTweetClassification.py | 5 - .../CyrillicTurkicLangClassification.py | 4 - .../multilingual/HinDialectClassification.py | 50 +- .../multilingual/IndicLangClassification.py | 4 - .../IndicNLPNewsClassification.py | 5 - .../IndicSentimentClassification.py | 4 - .../multilingual/LanguageClassification.py | 57 - .../multilingual/MTOPDomainClassification.py | 385 +- .../multilingual/MTOPIntentClassification.py | 5 +- .../multilingual/MasakhaNEWSClassification.py | 4 - .../MassiveIntentClassification.py | 5 +- .../MassiveScenarioClassification.py | 5 +- .../multilingual/MultiHateClassification.py | 4 - .../MultilingualSentimentClassification.py | 4 - .../Classification/multilingual/NaijaSenti.py | 4 - .../multilingual/NordicLangClassification.py | 12 +- .../NusaParagraphEmotionClassification.py | 8 - .../NusaParagraphTopicClassification.py | 8 - .../Classification/multilingual/NusaXSenti.py | 4 - .../multilingual/SIB200Classification.py | 8 - .../multilingual/ScalaClassification.py | 12 +- .../SouthAfricanLangClassification.py | 30 +- .../SwissJudgementClassification.py | 5 - .../multilingual/TurkicClassification.py | 4 - .../TweetSentimentClassification.py | 4 - mteb/tasks/Classification/mya/MyanmarNews.py | 4 - .../nep/NepaliNewsClassification.py | 4 - .../DutchBookReviewSentimentClassification.py | 4 - .../Classification/nob/NoRecClassification.py | 5 +- .../nob/NorwegianParliamentClassification.py | 5 +- .../ory/OdiaNewsClassification.py | 4 - .../pan/PunjabiNewsClassification.py | 4 - .../pol/PolishClassification.py | 17 - .../por/HateSpeechPortugueseClassification.py | 4 - mteb/tasks/Classification/ron/Moroco.py | 4 - .../ron/RomanianReviewsSentiment.py | 6 - .../ron/RomanianSentimentClassification.py | 4 - .../rus/GeoreviewClassification.py | 5 +- .../rus/HeadlineClassification.py | 5 +- .../rus/InappropriatenessClassification.py | 5 +- .../rus/KinopoiskClassification.py | 5 +- .../rus/RuReviewsClassification.py | 5 +- .../rus/RuSciBenchGRNTIClassification.py | 5 +- .../rus/RuSciBenchOECDClassification.py | 5 +- .../san/SanskritShlokasClassification.py | 4 - .../sin/SinhalaNewsClassification.py | 4 - .../sin/SinhalaNewsSourceClassification.py | 4 - ...SFDSKMovieReviewSentimentClassification.py | 16 +- .../slk/SlovakHateSpeechClassification.py | 4 - .../slv/FrenkSlClassification.py | 4 - .../spa/SpanishNewsClassification.py | 4 - .../spa/SpanishSentimentClassification.py | 4 - .../ssw/SiswatiNewsClassification.py | 6 - ...lovakMovieReviewSentimentClassification.py | 4 - .../swa/SwahiliNewsClassification.py | 4 - .../Classification/swe/DalajClassification.py | 12 +- .../swe/SweRecClassification.py | 5 +- .../swe/SwedishSentimentClassification.py | 6 - .../tam/TamilNewsClassification.py | 4 - .../TeluguAndhraJyotiNewsClassification.py | 4 - .../tha/WisesightSentimentClassification.py | 4 - .../tha/WongnaiReviewsClassification .py | 4 - .../tsn/TswanaNewsClassification.py | 4 - .../TurkishMovieSentimentClassification.py | 4 - .../TurkishProductSentimentClassification.py | 4 - .../ukr/UkrFormalityClassification.py | 4 - .../urd/UrduRomanSentimentClassification.py | 4 - .../vie/VieStudentFeedbackClassification.py | 4 - .../Classification/zho/CMTEBClassification.py | 46 +- .../zho/YueOpenriceReviewClassification.py | 11 +- .../zul/IsiZuluNewsClassification.py | 6 - .../Clustering/deu/BlurbsClusteringP2P.py | 8 - .../Clustering/deu/BlurbsClusteringS2S.py | 9 - .../Clustering/deu/TenKGnadClusteringP2P.py | 8 - .../Clustering/deu/TenKGnadClusteringS2S.py | 8 - .../eng/ArXivHierarchicalClustering.py | 144 - .../Clustering/eng/ArxivClusteringP2P.py | 10 +- .../Clustering/eng/ArxivClusteringS2S.py | 5 +- .../Clustering/eng/BigPatentClustering.py | 6 - .../Clustering/eng/BiorxivClusteringP2P.py | 10 +- .../Clustering/eng/BiorxivClusteringS2S.py | 43 +- .../Clustering/eng/MedrxivClusteringP2P.py | 10 +- .../Clustering/eng/MedrxivClusteringS2S.py | 10 +- mteb/tasks/Clustering/eng/RedditClustering.py | 11 +- .../Clustering/eng/RedditClusteringP2P.py | 10 +- .../Clustering/eng/StackExchangeClustering.py | 11 +- .../eng/StackExchangeClusteringP2P.py | 11 +- .../eng/TwentyNewsgroupsClustering.py | 10 +- .../Clustering/eng/WikiCitiesClustering.py | 1 - .../Clustering/fra/AlloProfClusteringP2P.py | 5 - .../Clustering/fra/AlloProfClusteringS2S.py | 5 - mteb/tasks/Clustering/fra/HALClusteringS2S.py | 6 - .../Clustering/jpn/LivedoorNewsClustering.py | 8 - .../Clustering/jpn/MewsC16JaClustering.py | 4 - .../multilingual/IndicReviewsClusteringP2P.py | 4 - .../multilingual/MLSUMClusteringP2P.py | 8 - .../multilingual/MLSUMClusteringS2S.py | 9 - .../multilingual/MasakhaNEWSClusteringP2P.py | 1 - .../multilingual/MasakhaNEWSClusteringS2S.py | 1 - .../multilingual/SIB200ClusteringS2S.py | 4 - .../multilingual/WikiClusteringP2P.py | 1174 - .../nob/SNLHierarchicalClustering.py | 10 +- .../nob/VGHierarchicalClustering.py | 10 +- mteb/tasks/Clustering/nob/snl_clustering.py | 4 - mteb/tasks/Clustering/nob/vg_clustering.py | 4 - mteb/tasks/Clustering/pol/PolishClustering.py | 24 - .../Clustering/rom/RomaniBibleClustering.py | 4 - .../Clustering/rus/GeoreviewClusteringP2P.py | 5 +- .../rus/RuSciBenchGRNTIClusteringP2P.py | 41 +- .../rus/RuSciBenchOECDClusteringP2P.py | 6 +- .../spa/SpanishNewsClusteringP2P.py | 1 - mteb/tasks/Clustering/swe/SwednClustering.py | 10 +- mteb/tasks/Clustering/swe/swedn_clustering.py | 4 - mteb/tasks/Clustering/zho/CMTEBClustering.py | 32 +- .../eng/Core17InstructionRetrieval.py | 13 - .../eng/News21InstructionRetrieval.py | 4 - .../eng/Robust04InstructionRetrieval.py | 4 - .../multilingual/mFollowIR.py | 90 - .../kor/KorHateSpeechMLClassification.py | 4 - .../mlt/MalteseNewsClassification.py | 4 - .../MultiEURLEXMultilabelClassification.py | 701 - .../por/BrazilianToxicTweetsClassification.py | 10 +- .../rus/CEDRClassification.py | 18 +- .../rus/SensitiveTopicsClassification.py | 5 +- mteb/tasks/PairClassification/ara/ArEntail.py | 4 - .../PairClassification/ces/CTKFactsNLI.py | 7 - .../deu/FalseFriendsDeEnPC.py | 4 - .../PairClassification/eng/LegalBenchPC.py | 4 - .../eng/SprintDuplicateQuestionsPC.py | 7 +- .../eng/TwitterSemEval2015PC.py | 5 +- .../eng/TwitterURLCorpusPC.py | 11 +- mteb/tasks/PairClassification/fas/FarsTail.py | 4 - .../hye/ArmenianParaphrasePC.py | 4 - mteb/tasks/PairClassification/ind/IndoNLI.py | 4 - mteb/tasks/PairClassification/kor/KlueNLI.py | 4 - .../IndicXnliPairClassification.py | 5 +- .../multilingual/OpusparcusPC.py | 4 - .../multilingual/PawsXPairClassification.py | 119 - .../PairClassification/multilingual/RTE3.py | 4 - .../PairClassification/multilingual/XNLI.py | 222 +- .../multilingual/XStance.py | 11 +- mteb/tasks/PairClassification/pol/PolishPC.py | 4 - .../tasks/PairClassification/por/Assin2RTE.py | 4 - mteb/tasks/PairClassification/por/SickBrPC.py | 7 - mteb/tasks/PairClassification/rus/TERRa.py | 5 +- .../zho/CMTEBPairClassification.py | 4 +- mteb/tasks/Reranking/__init__.py | 1 + .../Reranking/ara/NamaaMrTydiReranking.py | 39 + mteb/tasks/Reranking/ara/__init__.py | 0 .../Reranking/eng/AskUbuntuDupQuestions.py | 12 +- .../tasks/Reranking/eng/MindSmallReranking.py | 5 +- mteb/tasks/Reranking/eng/SciDocsReranking.py | 5 +- .../eng/StackOverflowDupQuestions.py | 5 +- .../eng/WebLINXCandidatesReranking.py | 18 - mteb/tasks/Reranking/fra/AlloprofReranking.py | 4 - mteb/tasks/Reranking/fra/SyntecReranking.py | 1 - mteb/tasks/Reranking/jpn/MMarcoReranking.py | 5 +- .../Reranking/multilingual/ESCIReranking.py | 36 - .../Reranking/multilingual/MIRACLReranking.py | 5 +- .../WikipediaRerankingMultilingual.py | 158 - mteb/tasks/Reranking/rus/RuBQReranking.py | 5 +- mteb/tasks/Reranking/zho/CMTEBReranking.py | 13 +- mteb/tasks/Retrieval/__init__.py | 15 + .../Retrieval/ara/SadeemQuestionRetrieval.py | 4 - mteb/tasks/Retrieval/code/AppsRetrieval.py | 12 - .../code/COIRCodeSearchNetRetrieval.py | 51 - .../Retrieval/code/CodeEditSearchRetrieval.py | 100 - .../Retrieval/code/CodeFeedbackMTRetrieval.py | 14 - .../Retrieval/code/CodeFeedbackSTRetrieval.py | 14 - .../code/CodeSearchNetCCRetrieval.py | 51 - .../Retrieval/code/CodeSearchNetRetrieval.py | 53 +- .../code/CodeTransOceanContestRetrieval.py | 11 - .../code/CodeTransOceanDLRetrieval.py | 11 - mteb/tasks/Retrieval/code/CosQARetrieval.py | 11 - .../code/StackOverflowQARetrieval.py | 14 - .../code/SyntheticText2SqlRetrieval.py | 14 - mteb/tasks/Retrieval/dan/DanFeverRetrieval.py | 26 +- mteb/tasks/Retrieval/dan/TV2Nordretrieval.py | 13 +- .../Retrieval/dan/TwitterHjerneRetrieval.py | 13 +- mteb/tasks/Retrieval/deu/GerDaLIRRetrieval.py | 12 - .../Retrieval/deu/GerDaLIRSmallRetrieval.py | 12 - .../tasks/Retrieval/deu/GermanDPRRetrieval.py | 15 +- .../deu/GermanGovServiceRetrieval.py | 12 - .../Retrieval/deu/GermanQuADRetrieval.py | 12 - .../tasks/Retrieval/deu/LegalQuADRetrieval.py | 12 - mteb/tasks/Retrieval/ell/GreekCivicsQA.py | 12 - .../Retrieval/eng/AILACasedocsRetrieval.py | 12 - .../Retrieval/eng/AILAStatutesRetrieval.py | 12 - .../Retrieval/eng/ARCChallengeRetrieval.py | 13 +- mteb/tasks/Retrieval/eng/AlphaNLIRetrieval.py | 13 +- mteb/tasks/Retrieval/eng/ArguAnaRetrieval.py | 13 +- mteb/tasks/Retrieval/eng/BrightRetrieval.py | 8 +- .../eng/CQADupstackAndroidRetrieval.py | 12 - .../eng/CQADupstackEnglishRetrieval.py | 12 - .../eng/CQADupstackGamingRetrieval.py | 12 - .../Retrieval/eng/CQADupstackGisRetrieval.py | 12 - .../eng/CQADupstackMathematicaRetrieval.py | 12 - .../eng/CQADupstackPhysicsRetrieval.py | 12 - .../eng/CQADupstackProgrammersRetrieval.py | 12 - .../eng/CQADupstackStatsRetrieval.py | 12 - .../Retrieval/eng/CQADupstackTexRetrieval.py | 12 - .../Retrieval/eng/CQADupstackUnixRetrieval.py | 12 - .../eng/CQADupstackWebmastersRetrieval.py | 12 - .../eng/CQADupstackWordpressRetrieval.py | 12 - .../Retrieval/eng/ClimateFEVERRetrieval.py | 25 +- mteb/tasks/Retrieval/eng/DBPediaRetrieval.py | 27 +- mteb/tasks/Retrieval/eng/FEVERRetrieval.py | 41 +- .../tasks/Retrieval/eng/FaithDialRetrieval.py | 12 - .../Retrieval/eng/FeedbackQARetrieval.py | 12 - mteb/tasks/Retrieval/eng/FiQA2018Retrieval.py | 29 +- mteb/tasks/Retrieval/eng/HagridRetrieval.py | 12 - .../tasks/Retrieval/eng/HellaSwagRetrieval.py | 13 +- mteb/tasks/Retrieval/eng/HotpotQARetrieval.py | 41 +- .../Retrieval/eng/LEMBNarrativeQARetrieval.py | 12 - .../Retrieval/eng/LEMBNeedleRetrieval.py | 70 - .../Retrieval/eng/LEMBPasskeyRetrieval.py | 70 - .../tasks/Retrieval/eng/LEMBQMSumRetrieval.py | 12 - .../eng/LEMBSummScreenFDRetrieval.py | 12 - .../Retrieval/eng/LEMBWikimQARetrieval.py | 12 - .../LegalBenchConsumerContractsQARetrieval.py | 12 - .../LegalBenchCorporateLobbyingRetrieval.py | 12 - .../eng/LegalSummarizationRetrieval.py | 12 - .../tasks/Retrieval/eng/LitSearchRetrieval.py | 12 - mteb/tasks/Retrieval/eng/MLQuestions.py | 19 - mteb/tasks/Retrieval/eng/MSMARCORetrieval.py | 41 +- .../tasks/Retrieval/eng/MSMARCOv2Retrieval.py | 1 - .../tasks/Retrieval/eng/MedicalQARetrieval.py | 12 - mteb/tasks/Retrieval/eng/NFCorpusRetrieval.py | 15 +- mteb/tasks/Retrieval/eng/NQRetrieval.py | 25 +- .../Retrieval/eng/NanoArguAnaRetrieval.py | 90 + .../eng/NanoClimateFeverRetrieval.py | 90 + .../Retrieval/eng/NanoDBPediaRetrieval.py | 80 + .../tasks/Retrieval/eng/NanoFEVERRetrieval.py | 104 + .../Retrieval/eng/NanoFiQA2018Retrieval.py | 90 + .../Retrieval/eng/NanoHotpotQARetrieval.py | 107 + .../Retrieval/eng/NanoMSMARCORetrieval.py | 102 + .../Retrieval/eng/NanoNFCorpusRetrieval.py | 92 + mteb/tasks/Retrieval/eng/NanoNQRetrieval.py | 88 + .../tasks/Retrieval/eng/NanoQuoraRetrieval.py | 91 + .../Retrieval/eng/NanoSCIDOCSRetrieval.py | 90 + .../Retrieval/eng/NanoSciFactRetrieval.py | 88 + .../Retrieval/eng/NanoTouche2020Retrieval.py | 99 + .../Retrieval/eng/NarrativeQARetrieval.py | 12 - mteb/tasks/Retrieval/eng/PiqaRetrieval.py | 13 +- mteb/tasks/Retrieval/eng/QuailRetrieval.py | 13 +- mteb/tasks/Retrieval/eng/QuoraRetrieval.py | 34 +- mteb/tasks/Retrieval/eng/RARbCodeRetrieval.py | 13 +- mteb/tasks/Retrieval/eng/RARbMathRetrieval.py | 13 +- mteb/tasks/Retrieval/eng/SCIDOCSRetrieval.py | 13 +- mteb/tasks/Retrieval/eng/SciFactRetrieval.py | 24 +- mteb/tasks/Retrieval/eng/SiqaRetrieval.py | 13 +- mteb/tasks/Retrieval/eng/SpartQARetrieval.py | 13 +- .../tasks/Retrieval/eng/TRECCOVIDRetrieval.py | 15 +- .../Retrieval/eng/TempReasonL1Retrieval.py | 13 +- .../eng/TempReasonL2ContextRetrieval.py | 13 +- .../eng/TempReasonL2FactRetrieval.py | 13 +- .../eng/TempReasonL2PureRetrieval.py | 13 +- .../eng/TempReasonL3ContextRetrieval.py | 13 +- .../eng/TempReasonL3FactRetrieval.py | 13 +- .../eng/TempReasonL3PureRetrieval.py | 13 +- mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py | 24 - .../Retrieval/eng/Touche2020Retrieval.py | 22 +- .../Retrieval/eng/WinoGrandeRetrieval.py | 13 +- mteb/tasks/Retrieval/est/estqa.py | 12 - mteb/tasks/Retrieval/fra/AlloprofRetrieval.py | 12 - mteb/tasks/Retrieval/fra/BSARDRetrieval.py | 12 - mteb/tasks/Retrieval/fra/FQuADRetrieval.py | 19 - mteb/tasks/Retrieval/fra/SyntecRetrieval.py | 12 - mteb/tasks/Retrieval/hun/HunSum2.py | 14 - .../tasks/Retrieval/jpn/JaGovFaqsRetrieval.py | 12 - mteb/tasks/Retrieval/jpn/JaQuADRetrieval.py | 12 - mteb/tasks/Retrieval/jpn/JaqketRetrieval.py | 9 - .../jpn/NLPJournalAbsIntroRetrieval.py | 12 - .../jpn/NLPJournalTitleAbsRetrieval.py | 12 - .../jpn/NLPJournalTitleIntroRetrieval.py | 12 - .../Retrieval/kat/GeorgianFAQRetrieval.py | 12 - mteb/tasks/Retrieval/kor/AutoRAGRetrieval.py | 40 + mteb/tasks/Retrieval/kor/KoStrategyQA.py | 12 - .../multilingual/BelebeleRetrieval.py | 2644 - .../Retrieval/multilingual/CUREv1Retrieval.py | 151 + ...CrossLingualSemanticDiscriminationWMT19.py | 21 - ...CrossLingualSemanticDiscriminationWMT21.py | 21 - .../multilingual/IndicQARetrieval.py | 84 - .../Retrieval/multilingual/MIRACLRetrieval.py | 274 +- .../Retrieval/multilingual/MLQARetrieval.py | 695 - .../multilingual/MintakaRetrieval.py | 63 - .../Retrieval/multilingual/MrTidyRetrieval.py | 1 - .../multilingual/MultiLongDocRetrieval.py | 198 +- .../multilingual/NeuCLIR2022Retrieval.py | 63 - .../multilingual/NeuCLIR2023Retrieval.py | 63 - .../multilingual/PublicHealthQARetrieval.py | 66 +- .../StatcanDialogueDatasetRetrieval.py | 37 - .../WikipediaRetrievalMultilingual.py | 136 - .../multilingual/XMarketRetrieval.py | 32 +- .../Retrieval/multilingual/XPQARetrieval.py | 259 - .../Retrieval/multilingual/XQuADRetrieval.py | 91 - mteb/tasks/Retrieval/nob/norquad.py | 13 +- mteb/tasks/Retrieval/nob/snl_retrieval.py | 13 +- .../tasks/Retrieval/pol/ArguAnaPLRetrieval.py | 12 - .../tasks/Retrieval/pol/DBPediaPLRetrieval.py | 24 - mteb/tasks/Retrieval/pol/FiQAPLRetrieval.py | 12 - .../Retrieval/pol/HotpotQAPLRetrieval.py | 24 - .../tasks/Retrieval/pol/MSMARCOPLRetrieval.py | 24 - .../Retrieval/pol/NFCorpusPLRetrieval.py | 12 - mteb/tasks/Retrieval/pol/NQPLRetrieval.py | 24 - mteb/tasks/Retrieval/pol/QuoraPLRetrieval.py | 31 - .../tasks/Retrieval/pol/SCIDOCSPLRetrieval.py | 12 - .../tasks/Retrieval/pol/SciFactPLRetrieval.py | 14 +- .../Retrieval/pol/TRECCOVIDPLRetrieval.py | 14 +- mteb/tasks/Retrieval/rus/RiaNewsRetrieval.py | 25 +- mteb/tasks/Retrieval/rus/RuBQRetrieval.py | 13 +- mteb/tasks/Retrieval/slk/SKQuadRetrieval.py | 12 - .../tasks/Retrieval/slk/SlovakSumRetrieval.py | 14 +- .../spa/SpanishPassageRetrievalS2P.py | 12 - .../spa/SpanishPassageRetrievalS2S.py | 12 - mteb/tasks/Retrieval/swe/SweFaqRetrieval.py | 13 +- mteb/tasks/Retrieval/swe/SwednRetrieval.py | 13 +- mteb/tasks/Retrieval/tur/TurHistQuad.py | 12 - mteb/tasks/Retrieval/vie/VieQuADRetrieval.py | 12 - mteb/tasks/Retrieval/zho/CMTEBRetrieval.py | 106 +- mteb/tasks/Retrieval/zho/LeCaRDv2Retrieval.py | 12 - mteb/tasks/STS/deu/GermanSTSBenchmarkSTS.py | 1 - mteb/tasks/STS/eng/BiossesSTS.py | 1 - mteb/tasks/STS/eng/STS12STS.py | 9 - mteb/tasks/STS/eng/STS13STS.py | 4 - mteb/tasks/STS/eng/STS14STS.py | 4 - mteb/tasks/STS/eng/STS15STS.py | 4 - mteb/tasks/STS/eng/STS16STS.py | 4 - mteb/tasks/STS/eng/STSBenchmarkSTS.py | 1 - mteb/tasks/STS/eng/SickrSTS.py | 49 +- mteb/tasks/STS/fao/FaroeseSTS.py | 4 - mteb/tasks/STS/fin/FinParaSTS.py | 4 - mteb/tasks/STS/fra/SickFrSTS.py | 1 - mteb/tasks/STS/jpn/JSICK.py | 4 - mteb/tasks/STS/jpn/JSTS.py | 4 - mteb/tasks/STS/kor/KlueSTS.py | 4 - mteb/tasks/STS/kor/KorSTS.py | 4 - .../STS/multilingual/IndicCrosslingualSTS.py | 4 - .../STS/multilingual/STS17CrosslingualSTS.py | 77 - .../STS/multilingual/STS22CrosslingualSTS.py | 8 - .../STSBenchmarkMultilingualSTS.py | 4 - mteb/tasks/STS/multilingual/SemRel24STS.py | 4 - mteb/tasks/STS/pol/PolishSTS.py | 8 - mteb/tasks/STS/por/Assin2STS.py | 4 - mteb/tasks/STS/por/SickBrSTS.py | 18 +- mteb/tasks/STS/ron/RonSTS.py | 4 - mteb/tasks/STS/rus/RUParaPhraserSTS.py | 4 - mteb/tasks/STS/rus/RuSTSBenchmarkSTS.py | 4 - mteb/tasks/STS/spa/STSES.py | 1 - mteb/tasks/STS/zho/CMTEBSTS.py | 7 - mteb/tasks/SpeedTask/CPUSpeedTask.py | 4 - mteb/tasks/SpeedTask/GPUSpeedTask.py | 4 - .../eng/SummEvalSummarization.py | 14 - .../fra/SummEvalFrSummarization.py | 8 - pyproject.toml | 17 +- scripts/compare_leaderboard_results.py | 102 + scripts/create_language_family_mapping.py | 47 + scripts/extract_model_names.py | 101 + scripts/generate_metadata.py | 269 + .../task_selection_eng_lite.ipynb | 5 +- .../task_selection/task_selection_eu.ipynb | 4 +- .../task_selection/task_selection_mult.ipynb | 5 +- tests/test_TaskMetadata.py | 594 +- tests/test_benchmark/mock_models.py | 2 +- tests/test_benchmark/mock_tasks.py | 1475 +- tests/test_benchmark/test_benchmark.py | 4 +- tests/test_cli.py | 6 +- .../test_evaluation/test_split_evaluation.py | 380 + tests/test_models/model_load_failures.json | 197 + tests/test_models/model_loading.py | 127 + tests/test_overview.py | 8 +- tests/test_reproducible_workflow.py | 10 +- tests/test_tasks/test_all_abstasks.py | 6 +- tests/test_tasks/test_metadata.py | 4 +- tests/test_tasks/test_mteb_rerank.py | 15 +- tests/test_tasks/test_retrieval_abstask.py | 6 +- 634 files changed, 135442 insertions(+), 17361 deletions(-) create mode 100644 .github/workflows/leaderboard_refresh.yaml create mode 100644 .github/workflows/model_loading.yml create mode 100644 mteb/caching.py create mode 100644 mteb/descriptive_stats/BitextMining/BUCC.v2.json create mode 100644 mteb/descriptive_stats/BitextMining/BornholmBitextMining.json create mode 100644 mteb/descriptive_stats/BitextMining/IN22ConvBitextMining.json create mode 100644 mteb/descriptive_stats/BitextMining/IN22GenBitextMining.json create mode 100644 mteb/descriptive_stats/BitextMining/IWSLT2017BitextMining.json create mode 100644 mteb/descriptive_stats/BitextMining/IndicGenBenchFloresBitextMining.json create mode 100644 mteb/descriptive_stats/BitextMining/NTREXBitextMining.json create mode 100644 mteb/descriptive_stats/BitextMining/NollySentiBitextMining.json create mode 100644 mteb/descriptive_stats/BitextMining/NorwegianCourtsBitextMining.json create mode 100644 mteb/descriptive_stats/BitextMining/NusaTranslationBitextMining.json create mode 100644 mteb/descriptive_stats/BitextMining/PhincBitextMining.json create mode 100644 mteb/descriptive_stats/BitextMining/TbilisiCityHallBitextMining.json create mode 100644 mteb/descriptive_stats/BitextMining/VieMedEVBitextMining.json create mode 100644 mteb/descriptive_stats/Classification/LanguageClassification.json create mode 100644 mteb/descriptive_stats/Classification/SlovakHateSpeechClassification.json create mode 100644 mteb/descriptive_stats/Clustering/ArXivHierarchicalClusteringP2P.json create mode 100644 mteb/descriptive_stats/Clustering/BiorxivClusteringS2S.json create mode 100644 mteb/descriptive_stats/Clustering/MedrxivClusteringP2P.v2.json create mode 100644 mteb/descriptive_stats/Clustering/MedrxivClusteringS2S.v2.json create mode 100644 mteb/descriptive_stats/Clustering/RedditClusteringP2P.v2.json create mode 100644 mteb/descriptive_stats/Clustering/RuSciBenchGRNTIClusteringP2P.json create mode 100644 mteb/descriptive_stats/Clustering/TwentyNewsgroupsClustering.v2.json create mode 100644 mteb/descriptive_stats/Clustering/WikiClusteringP2P.json create mode 100644 mteb/descriptive_stats/InstructionRetrieval/Core17InstructionRetrieval.json create mode 100644 mteb/descriptive_stats/MultilabelClassification/CEDRClassification.json create mode 100644 mteb/descriptive_stats/PairClassification/PawsXPairClassification.json create mode 100644 mteb/descriptive_stats/PairClassification/TwitterURLCorpus.json create mode 100644 mteb/descriptive_stats/PairClassification/XNLI.json create mode 100644 mteb/descriptive_stats/Reranking/AskUbuntuDupQuestions.json create mode 100644 mteb/descriptive_stats/Reranking/ESCIReranking.json create mode 100644 mteb/descriptive_stats/Reranking/WikipediaRerankingMultilingual.json create mode 100644 mteb/descriptive_stats/Retrieval/AppsRetrieval.json create mode 100644 mteb/descriptive_stats/Retrieval/AutoRAGRetrieval.json create mode 100644 mteb/descriptive_stats/Retrieval/BelebeleRetrieval.json create mode 100644 mteb/descriptive_stats/Retrieval/COIRCodeSearchNetRetrieval.json create mode 100644 mteb/descriptive_stats/Retrieval/CodeEditSearchRetrieval.json create mode 100644 mteb/descriptive_stats/Retrieval/CodeFeedbackMT.json create mode 100644 mteb/descriptive_stats/Retrieval/CodeFeedbackST.json create mode 100644 mteb/descriptive_stats/Retrieval/CodeSearchNetCCRetrieval.json create mode 100644 mteb/descriptive_stats/Retrieval/CodeSearchNetRetrieval.json create mode 100644 mteb/descriptive_stats/Retrieval/CodeTransOceanContest.json create mode 100644 mteb/descriptive_stats/Retrieval/CodeTransOceanDL.json create mode 100644 mteb/descriptive_stats/Retrieval/CosQA.json create mode 100644 mteb/descriptive_stats/Retrieval/JaqketRetrieval.json create mode 100644 mteb/descriptive_stats/Retrieval/NFCorpus.json create mode 100644 mteb/descriptive_stats/Retrieval/StackOverflowQA.json create mode 100644 mteb/descriptive_stats/Retrieval/SyntheticText2SQL.json create mode 100644 mteb/descriptive_stats/Retrieval/Touche2020.json create mode 100644 mteb/descriptive_stats/Retrieval/Touche2020Retrieval.v3.json create mode 100644 mteb/descriptive_stats/Retrieval/mFollowIRCrossLingualInstructionRetrieval.json create mode 100644 mteb/descriptive_stats/Retrieval/mFollowIRInstructionRetrieval.json create mode 100644 mteb/descriptive_stats/STS/STS12.json create mode 100644 mteb/descriptive_stats/STS/STS17.json create mode 100644 mteb/descriptive_stats/Summarization/SummEval.json create mode 100644 mteb/language_family.json create mode 100644 mteb/leaderboard/figures.py create mode 100644 mteb/models/cde_models.py create mode 100644 mteb/models/colbert_models.py create mode 100644 mteb/models/gme_v_models.py create mode 100644 mteb/models/ibm_granite_models.py create mode 100644 mteb/models/inf_models.py create mode 100644 mteb/models/instruct_wrapper.py create mode 100644 mteb/models/jasper_models.py create mode 100644 mteb/models/jina_models.py create mode 100644 mteb/models/lens_models.py create mode 100644 mteb/models/linq_models.py create mode 100644 mteb/models/misc_models.py create mode 100644 mteb/models/model2vec_models.py create mode 100644 mteb/models/moka_models.py create mode 100644 mteb/models/no_instruct_sentence_models.py create mode 100644 mteb/models/nvidia_models.py create mode 100644 mteb/models/piccolo_models.py create mode 100644 mteb/models/stella_models.py create mode 100644 mteb/models/text2vec_models.py create mode 100644 mteb/models/uae_models.py create mode 100644 mteb/models/utils.py create mode 100644 mteb/tasks/Reranking/ara/NamaaMrTydiReranking.py create mode 100644 mteb/tasks/Reranking/ara/__init__.py create mode 100644 mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoNQRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py create mode 100644 mteb/tasks/Retrieval/kor/AutoRAGRetrieval.py create mode 100644 mteb/tasks/Retrieval/multilingual/CUREv1Retrieval.py create mode 100644 scripts/compare_leaderboard_results.py create mode 100644 scripts/create_language_family_mapping.py create mode 100644 scripts/extract_model_names.py create mode 100644 scripts/generate_metadata.py create mode 100644 tests/test_evaluation/test_split_evaluation.py create mode 100644 tests/test_models/model_load_failures.json create mode 100644 tests/test_models/model_loading.py diff --git a/.github/workflows/leaderboard_refresh.yaml b/.github/workflows/leaderboard_refresh.yaml new file mode 100644 index 0000000000..159ba6e86f --- /dev/null +++ b/.github/workflows/leaderboard_refresh.yaml @@ -0,0 +1,16 @@ +name: Daily Space Rebuild +on: + schedule: + # Runs at midnight Pacific Time (8 AM UTC) + - cron: '0 8 * * *' + workflow_dispatch: # Allows manual triggering + +jobs: + rebuild: + runs-on: ubuntu-latest + steps: + - name: Trigger Factory Rebuild + run: | + curl -X POST \ + "https://huggingface.co/api/spaces/mteb/leaderboard_2_demo/restart?factory=true" \ + -H "Authorization: Bearer ${{ secrets.HF_TOKEN }}" diff --git a/.github/workflows/model_loading.yml b/.github/workflows/model_loading.yml new file mode 100644 index 0000000000..c139536321 --- /dev/null +++ b/.github/workflows/model_loading.yml @@ -0,0 +1,24 @@ +name: Model Loading + +on: + pull_request: + paths: + - 'mteb/models/**.py' + +jobs: + extract-and-run: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + + - name: Install dependencies and run tests + run: | + make model-load-test BASE_BRANCH=${{ github.event.pull_request.base.ref }} diff --git a/.gitignore b/.gitignore index 3219560494..977fe8dc1a 100644 --- a/.gitignore +++ b/.gitignore @@ -143,4 +143,8 @@ sb.ipynb tests/create_meta/model_card.md # removed results from mteb repo they are now available at: https://github.com/embeddings-benchmark/results -results/ \ No newline at end of file +results/ +uv.lock + +# model loading tests +model_names.txt \ No newline at end of file diff --git a/Makefile b/Makefile index c1404270d9..9729d080ff 100644 --- a/Makefile +++ b/Makefile @@ -35,4 +35,11 @@ pr: build-docs: @echo "--- 📚 Building documentation ---" # since we do not have a documentation site, this just build tables for the .md files - python docs/create_tasks_table.py \ No newline at end of file + python docs/create_tasks_table.py + + +model-load-test: + @echo "--- 🚀 Running model load test ---" + pip install ".[dev, speedtask, pylate,gritlm,xformers,model2vec]" + python scripts/extract_model_names.py $(BASE_BRANCH) --return_one_model_name_per_file + python tests/test_models/model_loading.py --model_name_file scripts/model_names.txt \ No newline at end of file diff --git a/README.md b/README.md index d20afdbedc..f556cad894 100644 --- a/README.md +++ b/README.md @@ -46,17 +46,15 @@ from sentence_transformers import SentenceTransformer # Define the sentence-transformers model name model_name = "average_word_embeddings_komninos" -# or directly from huggingface: -# model_name = "sentence-transformers/all-MiniLM-L6-v2" -model = SentenceTransformer(model_name) +model = mteb.get_model(model_name) # if the model is not implemented in MTEB it will be eq. to SentenceTransformer(model_name) tasks = mteb.get_tasks(tasks=["Banking77Classification"]) evaluation = mteb.MTEB(tasks=tasks) results = evaluation.run(model, output_folder=f"results/{model_name}") ```
- Running SentneceTransformermer model with prompts + Running SentenceTransformer model with prompts Prompts can be passed to the SentenceTransformer model using the `prompts` parameter. The following code shows how to use prompts with SentenceTransformer: @@ -164,7 +162,7 @@ For instance to select the 56 English datasets that form the "Overall MTEB Engli ```python import mteb -benchmark = mteb.get_benchmark("MTEB(eng)") +benchmark = mteb.get_benchmark("MTEB(eng, classic)") evaluation = mteb.MTEB(tasks=benchmark) ``` @@ -211,6 +209,21 @@ Note that the public leaderboard uses the test splits for all datasets except MS
+ +
+ Selecting evaluation subset + +### Selecting evaluation subset +You can evaluate only on selected subsets. For example, if you want to evaluate only the `subset_name_to_run` subset of all tasks, do the following: + +```python +evaluation.run(model, eval_subsets=["subset_name_to_run"]) +``` + +Monolingual tasks have `default` subset, other tasks have subsets that are specific to the dataset. + +
+
Using a custom model @@ -220,7 +233,10 @@ Note that the public leaderboard uses the test splits for all datasets except MS Models should implement the following interface, implementing an `encode` function taking as inputs a list of sentences, and returning a list of embeddings (embeddings can be `np.array`, `torch.tensor`, etc.). For inspiration, you can look at the [mteb/mtebscripts repo](https://github.com/embeddings-benchmark/mtebscripts) used for running diverse models via SLURM scripts for the paper. ```python +import mteb from mteb.encoder_interface import PromptType +import numpy as np + class CustomModel: def encode( @@ -244,7 +260,7 @@ class CustomModel: pass model = CustomModel() -tasks = mteb.get_task("Banking77Classification") +tasks = mteb.get_tasks(tasks=["Banking77Classification"]) evaluation = MTEB(tasks=tasks) evaluation.run(model) ``` @@ -313,6 +329,34 @@ evaluation.run( ) ``` +
+ +
+ Late Interaction (ColBERT) + +### Using Late Interaction models for retrieval + +```python +from mteb import MTEB +import mteb + + +colbert = mteb.get_model("colbert-ir/colbertv2.0") +tasks = mteb.get_tasks(tasks=["NFCorpus"], languages=["eng"]) + +eval_splits = ["test"] + +evaluation = MTEB(tasks=tasks) + +evaluation.run( + colbert, + eval_splits=eval_splits, + corpus_chunk_size=500, +) +``` +This implementation employs the MaxSim operation to compute the similarity between sentences. While MaxSim provides high-quality results, it processes a larger number of embeddings, potentially leading to increased resource usage. To manage resource consumption, consider lowering the `corpus_chunk_size` parameter. + +
@@ -378,6 +422,28 @@ results = mteb.load_results(models=models, tasks=tasks) df = results_to_dataframe(results) ``` +
+ + +
+ Annotate Contamination in the training data of a model + +### Annotate Contamination + +have your found contamination in the training data of a model? Please let us know, either by opening an issue or ideally by submitting a PR +annotatig the training datasets of the model: + +```py +model_w_contamination = ModelMeta( + name = "model-with-contamination" + ... + training_datasets: {"ArguAna": # name of dataset within MTEB + ["test"]} # the splits that have been trained on + ... +) +``` + +
diff --git a/docs/adding_a_dataset.md b/docs/adding_a_dataset.md index 4dc1b70a2f..f2167f0fd8 100644 --- a/docs/adding_a_dataset.md +++ b/docs/adding_a_dataset.md @@ -37,7 +37,7 @@ class SciDocsReranking(AbsTaskReranking): dataset={ "path": "mteb/scidocs-reranking", "revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab", - } + }, date=("2000-01-01", "2020-12-31"), # best guess domains=["Academic", "Non-fiction", "Domains"], task_subtypes=["Scientific Reranking"], @@ -45,7 +45,6 @@ class SciDocsReranking(AbsTaskReranking): annotations_creators="derived", dialect=[], sample_creation="found", - descriptive_stats={"n_samples": {"test": 19599}, "avg_character_length": {"test": 69.0}}, bibtex_citation=""" @inproceedings{cohan-etal-2020-specter, title = "{SPECTER}: Document-level Representation Learning using Citation-informed Transformers", @@ -73,7 +72,7 @@ class SciDocsReranking(AbsTaskReranking): # testing the task with a model: model = SentenceTransformer("average_word_embeddings_komninos") -evaluation = MTEB(tasks=[MindSmallReranking()]) +evaluation = MTEB(tasks=[SciDocsReranking()]) evaluation.run(model) ``` @@ -109,7 +108,7 @@ class VGClustering(AbsTaskClustering): dialect=[], text_creation="found", bibtex_citation= ... # removed for brevity -) + ) def dataset_transform(self): splits = self.description["eval_splits"] diff --git a/docs/adding_a_model.md b/docs/adding_a_model.md index 5018f49b65..f87d723934 100644 --- a/docs/adding_a_model.md +++ b/docs/adding_a_model.md @@ -14,8 +14,7 @@ model = mteb.get_model("sentence-transformers/paraphrase-multilingual-MiniLM-L12 tasks = mteb.get_tasks(...) # get specific tasks # or -from mteb.benchmarks import MTEB_MAIN_EN -tasks = MTEB_MAIN_EN # or use a specific benchmark +tasks = mteb.get_benchmark("MTEB(eng, classic)") # or use a specific benchmark evaluation = mteb.MTEB(tasks=tasks) evaluation.run(model, output_folder="results") @@ -29,21 +28,38 @@ mteb run -m {model_name} -t {task_names} These will save the results in a folder called `results/{model_name}/{model_revision}`. +<<<<<<< HEAD 1. **Format the results using the CLI:** +======= +2. **Push Results to the Leaderboard** + +To add results to the public leaderboard you can push your results to the [results repository](https://github.com/embeddings-benchmark/results) via a PR. Once merged they will appear on the leaderboard after a day. + + +3. (Optional) **Add results to the model card:** + +`mteb` implements a cli for adding results to the model card: +>>>>>>> main ```bash mteb create_meta --results_folder results/{model_name}/{model_revision} --output_path model_card.md ``` -If readme of model exists: +To add the content to the public model simply copy the content of the `model_card.md` file to the top of a `README.md` file of your model on the Hub. See [here](https://huggingface.co/Muennighoff/SGPT-5.8B-weightedmean-msmarco-specb-bitfit/blob/main/README.md) for an example. + +If the readme already exists: ```bash mteb create_meta --results_folder results/{model_name}/{model_revision} --output_path model_card.md --from_existing your_existing_readme.md ``` +<<<<<<< HEAD 2. **Add the frontmatter to model repository:** Copy the content of the `model_card.md` file to the top of a `README.md` file of your model on the Hub. See [here](https://huggingface.co/Muennighoff/SGPT-5.8B-weightedmean-msmarco-specb-bitfit/blob/main/README.md) for an example. +======= +Note that running the model on many tasks may lead to a huge readme front matter. +>>>>>>> main 3. **Wait for a refresh the leaderboard:** @@ -51,7 +67,10 @@ The leaderboard [automatically refreshes daily](https://github.com/embeddings-be **Notes:** - We remove models with scores that cannot be reproduced, so please ensure that your model is accessible and scores can be reproduced. +<<<<<<< HEAD - An alternative way of submitting to the leaderboard is by opening a PR with your results [here](https://github.com/embeddings-benchmark/results) & checking that they are displayed correctly by [locally running the leaderboard](https://github.com/embeddings-benchmark/leaderboard?tab=readme-ov-file#developer-setup) +======= +>>>>>>> main - ##### Using Prompts with Sentence Transformers @@ -65,4 +84,8 @@ The leaderboard [automatically refreshes daily](https://github.com/embeddings-be ###### Instantiating the Model with Prompts - If you are unable to directly add the prompts in the model configuration, you can instantiate the model using the `sentence_transformers_loader` and pass `prompts` as an argument. For more details, see the `mteb/models/bge_models.py` file. \ No newline at end of file +<<<<<<< HEAD + If you are unable to directly add the prompts in the model configuration, you can instantiate the model using the `sentence_transformers_loader` and pass `prompts` as an argument. For more details, see the `mteb/models/bge_models.py` file. +======= + If you are unable to directly add the prompts in the model configuration, you can instantiate the model using the `sentence_transformers_loader` and pass `prompts` as an argument. For more details, see the `mteb/models/bge_models.py` file. +>>>>>>> main diff --git a/docs/create_tasks_table.py b/docs/create_tasks_table.py index 4a3f85c9f4..4a1be0cd89 100644 --- a/docs/create_tasks_table.py +++ b/docs/create_tasks_table.py @@ -8,6 +8,7 @@ import mteb from mteb.abstasks.TaskMetadata import PROGRAMMING_LANGS, TASK_TYPE +from mteb.languages import ISO_TO_FAM_LEVEL0, ISO_TO_LANGUAGE def author_from_bibtex(bibtex: str | None) -> str: @@ -32,6 +33,17 @@ def author_from_bibtex(bibtex: str | None) -> str: return f" ({author_str_w_et_al}, {year_str})" +def round_floats_in_dict(d: dict, precision: int = 2) -> dict: + if not isinstance(d, dict): + return d + for key, value in d.items(): + if isinstance(value, float): + d[key] = round(value, precision) + elif isinstance(value, dict): + d[key] = round_floats_in_dict(value, precision) + return d + + def task_to_markdown_row(task: mteb.AbsTask) -> str: name = task.metadata.name name_w_reference = ( @@ -40,20 +52,8 @@ def task_to_markdown_row(task: mteb.AbsTask) -> str: domains = ( "[" + ", ".join(task.metadata.domains) + "]" if task.metadata.domains else "" ) - n_samples = ( - task.metadata.descriptive_stats["n_samples"] - if "n_samples" in task.metadata.descriptive_stats - else "" - ) - dataset_statistics = "" - if "avg_character_length" in task.metadata.descriptive_stats: - dataset_statistics = task.metadata.descriptive_stats["avg_character_length"] - elif len(task.metadata.descriptive_stats) > 1: - all_stat = task.metadata.descriptive_stats - all_stat.pop("n_samples") - if len(all_stat) > 0: - dataset_statistics = all_stat - + n_samples = task.metadata.n_samples + dataset_statistics = round_floats_in_dict(task.metadata.descriptive_stats) name_w_reference += author_from_bibtex(task.metadata.bibtex_citation) return f"| {name_w_reference} | {task.metadata.languages} | {task.metadata.type} | {task.metadata.category} | {domains} | {n_samples} | {dataset_statistics} |" @@ -69,7 +69,7 @@ def create_tasks_table(tasks: list[mteb.AbsTask]) -> str: return table -def create_task_lang_table(tasks: list[mteb.AbsTask]) -> str: +def create_task_lang_table(tasks: list[mteb.AbsTask], sort_by_sum=False) -> str: table_dict = {} ## Group by language. If it is a multilingual dataset, 1 is added to all languages present. for task in tasks: @@ -83,22 +83,38 @@ def create_task_lang_table(tasks: list[mteb.AbsTask]) -> str: ## Wrangle for polars pl_table_dict = [] for lang, d in table_dict.items(): - d.update({"lang": lang}) + d.update({"0-lang-code": lang}) # for sorting columns pl_table_dict.append(d) - df = pl.DataFrame(pl_table_dict).sort(by="lang") + df = pl.DataFrame(pl_table_dict).sort(by="0-lang-code") + df = df.with_columns( + pl.col("0-lang-code") + .replace_strict(ISO_TO_LANGUAGE, default="unknown") + .alias("1-lang-name") + ) + df = df.with_columns( + pl.col("0-lang-code") + .replace_strict(ISO_TO_FAM_LEVEL0, default="Unclassified") + .alias("2-lang-fam") + ) + + df = df.with_columns(sum=pl.sum_horizontal(get_args(TASK_TYPE))) + df = df.select(sorted(df.columns)) + if sort_by_sum: + df = df.sort(by="sum", descending=True) + total = df.sum() task_names_md = " | ".join(sorted(get_args(TASK_TYPE))) - horizontal_line_md = "---|---" * len(sorted(get_args(TASK_TYPE))) + horizontal_line_md = "---|---" * (len(sorted(get_args(TASK_TYPE))) + 1) table = f""" -| Language | {task_names_md} | +| ISO Code | Language | Family | {task_names_md} | Sum | |{horizontal_line_md}| """ for row in df.iter_rows(): - table += f"| {row[-1]} " - for num in row[:-1]: + table += f"| {row[0]} " + for num in row[1:]: table += f"| {num} " table += "|\n" @@ -115,14 +131,14 @@ def insert_tables( file_path: str, tables: list[str], tags: list[str] = ["TASKS TABLE"] ) -> None: """Insert tables within and or similar tags.""" - md = Path(file_path).read_text() + md = Path(file_path).read_text(encoding="utf-8") for table, tag in zip(tables, tags): start = f"" end = f"" md = md.replace(md[md.index(start) + len(start) : md.index(end)], table) - Path(file_path).write_text(md) + Path(file_path).write_text(md, encoding="utf-8") def main(): diff --git a/docs/mmteb/points_table.md b/docs/mmteb/points_table.md index 44ee346804..85978dcc00 100644 --- a/docs/mmteb/points_table.md +++ b/docs/mmteb/points_table.md @@ -2,6 +2,7 @@ _Note_: this table is **autogenerated** and should not be edited. It is intended to get an overview of contributions. +<<<<<<< HEAD | GitHub | Paper writing | New dataset | Review PR | Bug fixes | Coordination | Dataset annotations | New task | Running Models | Total | |:------------------|----------------:|--------------:|------------:|------------:|---------------:|----------------------:|-----------:|-----------------:|--------:| | KennethEnevoldsen | 0 | 68 | 326 | 87 | 81 | 35 | 0 | 0 | 597 | @@ -102,3 +103,105 @@ _Note_: this table is **autogenerated** and should not be edited. It is intended | bakrianoo | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | | hanhainebula | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | | monikernemo | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +======= + | GitHub | New dataset | Review PR | Bug fixes | Coordination | Paper writing | Dataset annotations | Running Models | New task | Total | +|:------------------|--------------:|------------:|------------:|---------------:|----------------:|----------------------:|-----------------:|-----------:|--------:| +| KennethEnevoldsen | 68 | 326 | 87 | 81 | 0 | 35 | 0 | 0 | 597 | +| isaac-chung | 120 | 194 | 50 | 54 | 12 | 1 | 0 | 2 | 433 | +| imenelydiaker | 120 | 144 | 24 | 70 | 0 | 0 | 0 | 0 | 358 | +| awinml | 300 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 302 | +| x-tabdeveloping | 144 | 32 | 10 | 41 | 0 | 0 | 0 | 12 | 239 | +| davidstap | 176 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 176 | +| jaygala24 | 149 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 149 | +| wissam-sib | 134 | 6 | 4 | 0 | 0 | 0 | 0 | 0 | 144 | +| Muennighoff | 0 | 48 | 0 | 70 | 0 | 0 | 24 | 0 | 142 | +| orionw | 0 | 20 | 20 | 75 | 0 | 0 | 0 | 10 | 125 | +| dokato | 94 | 6 | 12 | 0 | 0 | 0 | 0 | 0 | 112 | +| gentaiscool | 110 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 110 | +| jupyterjazz | 108 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 108 | +| SaitejaUtpala | 102 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 102 | +| vaibhavad | 6 | 4 | 8 | 75 | 0 | 0 | 0 | 0 | 93 | +| schmarion | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| MathieuCiancone | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| GabrielSequeira | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| digantamisra98 | 71 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 71 | +| shreeya-dhakal | 54 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 62 | +| Rysias | 58 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 58 | +| Samoed | 18 | 2 | 22 | 0 | 0 | 0 | 9 | 0 | 51 | +| sivareddyg | 0 | 0 | 0 | 50 | 0 | 0 | 0 | 0 | 50 | +| gowitheflow-1998 | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 50 | +| asparius | 34 | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 48 | +| Akash190104 | 46 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 46 | +| MartinBernstorff | 2 | 8 | 13 | 20 | 0 | 0 | 0 | 0 | 43 | +| akshita-sukhlecha | 36 | 0 | 4 | 0 | 0 | 0 | 0 | 0 | 40 | +| staoxiao | 40 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 40 | +| bp-high | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | +| rafalposwiata | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | +| KranthiGV | 20 | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 34 | +| loicmagne | 0 | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 28 | +| ShawonAshraf | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| bjoernpl | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| jphme | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| rasdani | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| violenil | 26 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 26 | +| mariyahendriksen | 0 | 0 | 0 | 0 | 24 | 0 | 0 | 0 | 24 | +| dwzhu-pku | 24 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 24 | +| hgissbkh | 0 | 2 | 13 | 0 | 3 | 0 | 0 | 5 | 23 | +| taeminlee | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | +| kwojtasi | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | +| jankounchained | 14 | 0 | 8 | 0 | 0 | 0 | 0 | 0 | 22 | +| tomaarsen | 0 | 2 | 0 | 20 | 0 | 0 | 0 | 0 | 22 | +| crystina-z | 21 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 21 | +| mrshu | 16 | 4 | 0 | 0 | 0 | 1 | 0 | 0 | 21 | +| john-b-yang | 0 | 0 | 0 | 0 | 20 | 0 | 0 | 0 | 20 | +| rbroc | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | +| mmhamdy | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | +| ManuelFay | 2 | 0 | 13 | 0 | 0 | 0 | 0 | 5 | 20 | +| AlexeyVatolin | 0 | 0 | 20 | 0 | 0 | 0 | 0 | 0 | 20 | +| Andrian0s | 14 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 20 | +| thakur-nandan | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | +| manandey | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | +| PranjalChitale | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| dipam7 | 14 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| sted97 | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| Sakshamrzt | 12 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| taidnguyen | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 14 | +| artemsnegirev | 12 | 0 | 0 | 0 | 0 | 2 | 0 | 0 | 14 | +| slvnwhrl | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| anpalmak2003 | 9 | 0 | 0 | 0 | 0 | 3 | 0 | 0 | 12 | +| Art3mis07 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| guenthermi | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| jordiclive | 2 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 12 | +| xhluca | 6 | 2 | 4 | 0 | 0 | 0 | 0 | 0 | 12 | +| henilp105 | 0 | 0 | 2 | 0 | 0 | 9 | 0 | 0 | 11 | +| MariyaTikhonova | 7 | 0 | 0 | 0 | 0 | 4 | 0 | 0 | 11 | +| ab1992ao | 8 | 0 | 0 | 0 | 0 | 3 | 0 | 0 | 11 | +| tmp_handle | 0 | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 10 | +| swj0419 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| Ruqyai | 2 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| ZhengLiu101 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| Alenush | 6 | 0 | 0 | 0 | 0 | 4 | 0 | 0 | 10 | +| ABorghini | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| simon-clematide | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| sarahooker | 0 | 0 | 0 | 0 | 10 | 0 | 0 | 0 | 10 | +| guangyusong | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| HLasse | 0 | 0 | 5 | 0 | 0 | 5 | 0 | 0 | 10 | +| cassanof | 8 | 0 | 1 | 0 | 0 | 0 | 1 | 0 | 10 | +| hongjin-su | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| xiamengzhou | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| xu3kev | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| howard-yen | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| malteos | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| ljvmiranda921 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| marcobellagente93 | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| izhx | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| MexicanLemonade | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| antoniolanza1996 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | +| achibb | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| NouamaneTazi | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| PhilipMay | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| cslizc | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| bakrianoo | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| hanhainebula | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| monikernemo | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +>>>>>>> main diff --git a/docs/tasks.md b/docs/tasks.md index d90ac1816b..d4e6b376ad 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -8,11 +8,12 @@ The following tables give you an overview of the tasks in MTEB. | Name | Languages | Type | Category | Domains | # Samples | Dataset statistics | |------|-----------|------|----------|---------|-----------|--------------------| | [AFQMC](https://aclanthology.org/2021.emnlp-main.357) | ['cmn'] | STS | s2s | | None | None | -| [AILACasedocs](https://zenodo.org/records/4063986) | ['eng'] | Retrieval | p2p | [Legal, Written] | None | {'test': {'average_document_length': 26948.344086021505, 'average_query_length': 3038.42, 'num_documents': 186, 'num_queries': 50, 'average_relevant_docs_per_query': 3.9}} | -| [AILAStatutes](https://zenodo.org/records/4063986) | ['eng'] | Retrieval | p2p | [Legal, Written] | None | {'test': {'average_document_length': 1973.6341463414635, 'average_query_length': 3038.42, 'num_documents': 82, 'num_queries': 50, 'average_relevant_docs_per_query': 4.34}} | -| [AJGT](https://link.springer.com/chapter/10.1007/978-3-319-60042-0_66/) (Alomari et al., 2017) | ['ara'] | Classification | s2s | [Social, Written] | {'train': 1800} | {'train': 46.81} | -| [ARCChallenge](https://allenai.org/data/arc) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 1172} | {'test': {'average_document_length': 30.94235294117647, 'average_query_length': 131.56569965870307, 'num_documents': 9350, 'num_queries': 1172, 'average_relevant_docs_per_query': 1.0}} | +| [AILACasedocs](https://zenodo.org/records/4063986) | ['eng'] | Retrieval | p2p | [Legal, Written] | None | None | +| [AILAStatutes](https://zenodo.org/records/4063986) | ['eng'] | Retrieval | p2p | [Legal, Written] | None | None | +| [AJGT](https://link.springer.com/chapter/10.1007/978-3-319-60042-0_66/) (Alomari et al., 2017) | ['ara'] | Classification | s2s | [Social, Written] | None | None | +| [ARCChallenge](https://allenai.org/data/arc) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | | [ATEC](https://aclanthology.org/2021.emnlp-main.357) | ['cmn'] | STS | s2s | | None | None | +<<<<<<< HEAD | [AfriSentiClassification](https://arxiv.org/abs/2302.08956) | ['amh', 'arq', 'ary', 'hau', 'ibo', 'kin', 'pcm', 'por', 'swa', 'tso', 'twi', 'yor'] | Classification | s2s | [Social, Written] | {'test': 2048} | {'test': 74.77} | | [AfriSentiLangClassification](https://huggingface.co/datasets/HausaNLP/afrisenti-lid-data/) | ['amh', 'arq', 'ary', 'hau', 'ibo', 'kin', 'pcm', 'por', 'swa', 'tso', 'twi', 'yor'] | Classification | s2s | [Social, Written] | {'test': 5754} | {'test': 77.84} | | [AllegroReviews](https://aclanthology.org/2020.acl-main.111.pdf) | ['pol'] | Classification | s2s | | {'test': 1006} | {'test': 477.2} | @@ -36,27 +37,54 @@ The following tables give you an overview of the tasks in MTEB. | [AskUbuntuDupQuestions](https://github.com/taolei87/askubuntu) | ['eng'] | Reranking | s2s | | {'test': 2255} | {'test': {'num_samples': 375, 'num_positive': 375, 'num_negative': 375, 'avg_query_len': 50.205333333333336, 'avg_positive_len': 6.013333333333334, 'avg_negative_len': 13.986666666666666}} | | [Assin2RTE](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) (Real et al., 2020) | ['por'] | PairClassification | s2s | [Written] | {'test': 2448} | {'test': 53.55} | | [Assin2STS](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) (Real et al., 2020) | ['por'] | STS | s2s | [Written] | {'test': 2448} | {'test': 53.55} | +======= +| [AfriSentiClassification](https://arxiv.org/abs/2302.08956) | ['amh', 'arq', 'ary', 'hau', 'ibo', 'kin', 'pcm', 'por', 'swa', 'tso', 'twi', 'yor'] | Classification | s2s | [Social, Written] | None | None | +| [AfriSentiLangClassification](https://huggingface.co/datasets/HausaNLP/afrisenti-lid-data/) | ['amh', 'arq', 'ary', 'hau', 'ibo', 'kin', 'pcm', 'por', 'swa', 'tso', 'twi', 'yor'] | Classification | s2s | [Social, Written] | None | None | +| [AllegroReviews](https://aclanthology.org/2020.acl-main.111.pdf) | ['pol'] | Classification | s2s | | None | None | +| [AlloProfClusteringP2P.v2](https://huggingface.co/datasets/lyon-nlp/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Clustering | p2p | [Encyclopaedic, Written] | None | None | +| [AlloProfClusteringS2S.v2](https://huggingface.co/datasets/lyon-nlp/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Clustering | s2s | [Encyclopaedic, Written] | None | None | +| [AlloprofReranking](https://huggingface.co/datasets/antoinelb7/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Reranking | s2p | [Web, Academic, Written] | None | None | +| [AlloprofRetrieval](https://huggingface.co/datasets/antoinelb7/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [AlphaNLI](https://leaderboard.allenai.org/anli/submissions/get-started) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [AmazonCounterfactualClassification](https://arxiv.org/abs/2104.06893) | ['deu', 'eng', 'jpn'] | Classification | s2s | [Reviews, Written] | None | None | +| [AmazonPolarityClassification](https://huggingface.co/datasets/amazon_polarity) (Julian McAuley, 2013) | ['eng'] | Classification | p2p | [Reviews, Written] | None | None | +| [AmazonReviewsClassification](https://arxiv.org/abs/2010.02573) (Phillip Keung, 2020) | ['cmn', 'deu', 'eng', 'fra', 'jpn', 'spa'] | Classification | s2s | [Reviews, Written] | None | None | +| [AngryTweetsClassification](https://aclanthology.org/2021.nodalida-main.53/) (Pauli et al., 2021) | ['dan'] | Classification | s2s | [Social, Written] | None | None | +| [AppsRetrieval](https://arxiv.org/abs/2105.09938) (Dan Hendrycks, 2021) | ['eng', 'python'] | Retrieval | p2p | [Programming, Written] | {'test': 12530} | {'test': {'number_of_characters': 11335620, 'num_samples': 12530, 'num_queries': 3765, 'num_documents': 8765, 'min_document_length': 152, 'average_document_length': 717.27, 'max_document_length': 5742, 'unique_documents': 8765, 'min_query_length': 6, 'average_query_length': 1340.96, 'max_query_length': 289049, 'unique_queries': 3765, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 3765}} | +| [ArEntail](https://link.springer.com/article/10.1007/s10579-024-09731-1) (Obeidat et al., 2024) | ['ara'] | PairClassification | s2s | [News, Written] | None | None | +| [ArXivHierarchicalClusteringP2P](https://www.kaggle.com/Cornell-University/arxiv) | ['eng'] | Clustering | p2p | [Academic, Written] | {'test': 2048} | {'test': {'num_samples': 2048, 'number_of_characters': 2065284, 'min_text_length': 103, 'average_text_length': 1008.44, 'max_text_length': 2103, 'min_labels_per_text': 1, 'average_labels_per_text': 1.46, 'max_labels_per_text': 381, 'unique_labels': 129, 'labels': {'cs': {'count': 356}, 'math': {'count': 381}, 'OC': {'count': 11}, 'hep-lat': {'count': 13}, 'hep': {'count': 98}, 'astro-ph': {'count': 213}, 'eess': {'count': 76}, 'quant-ph': {'count': 135}, 'DC': {'count': 5}, 'cond-mat': {'count': 274}, 'hep-th': {'count': 66}, 'SP': {'count': 33}, 'hep-ph': {'count': 69}, 'FA': {'count': 6}, 'nucl-th': {'count': 17}, 'q-bio': {'count': 80}, 'HE': {'count': 22}, 'HC': {'count': 2}, 'stat': {'count': 60}, 'ML': {'count': 16}, 'IV': {'count': 13}, 'stat-mech': {'count': 47}, 'DS': {'count': 14}, 'ME': {'count': 12}, 'CC': {'count': 2}, 'mtrl-sci': {'count': 22}, 'PE': {'count': 16}, 'NT': {'count': 11}, 'SC': {'count': 6}, 'AG': {'count': 13}, 'physics': {'count': 81}, 'ins-det': {'count': 9}, 'GA': {'count': 18}, 'BM': {'count': 6}, 'GN': {'count': 17}, 'NA': {'count': 15}, 'app-ph': {'count': 7}, 'RT': {'count': 6}, 'other': {'count': 37}, 'soft': {'count': 15}, 'CO': {'count': 33}, 'supr-con': {'count': 21}, 'chem-ph': {'count': 3}, 'DM': {'count': 2}, 'MN': {'count': 12}, 'q-fin': {'count': 27}, 'PM': {'count': 2}, 'AP': {'count': 27}, 'gr-qc': {'count': 15}, 'quant-gas': {'count': 8}, 'mes-hall': {'count': 33}, 'IT': {'count': 19}, 'SI': {'count': 6}, 'SG': {'count': 3}, 'bio-ph': {'count': 2}, 'SR': {'count': 16}, 'soc-ph': {'count': 5}, 'hep-ex': {'count': 15}, 'DG': {'count': 11}, 'NE': {'count': 5}, 'CR': {'count': 6}, 'CL': {'count': 12}, 'RM': {'count': 3}, 'econ': {'count': 17}, 'nlin': {'count': 5}, 'PS': {'count': 1}, 'LG': {'count': 26}, 'QA': {'count': 9}, 'str-el': {'count': 26}, 'CV': {'count': 34}, 'MF': {'count': 6}, 'IM': {'count': 7}, 'EM': {'count': 6}, 'TH': {'count': 5}, 'PR': {'count': 20}, 'AT': {'count': 4}, 'OA': {'count': 4}, 'CP': {'count': 6}, 'LO': {'count': 14}, 'flu-dyn': {'count': 6}, 'atom-ph': {'count': 8}, 'class-ph': {'count': 1}, 'SY': {'count': 20}, 'IR': {'count': 1}, 'plasm-ph': {'count': 8}, 'CE': {'count': 2}, 'AO': {'count': 1}, 'comp-ph': {'count': 3}, 'optics': {'count': 12}, 'MG': {'count': 4}, 'ST': {'count': 6}, 'nucl-ex': {'count': 6}, 'CY': {'count': 9}, 'ao-ph': {'count': 2}, 'DB': {'count': 1}, 'math-ph': {'count': 10}, 'NC': {'count': 13}, 'GT': {'count': 11}, 'TO': {'count': 2}, 'AI': {'count': 9}, 'NI': {'count': 2}, 'gen-ph': {'count': 4}, 'OT': {'count': 4}, 'SD': {'count': 2}, 'dis-nn': {'count': 4}, 'RO': {'count': 7}, 'CA': {'count': 6}, 'FL': {'count': 1}, 'SE': {'count': 5}, 'EP': {'count': 9}, 'hist-ph': {'count': 1}, 'QM': {'count': 9}, 'ed-ph': {'count': 2}, 'GR': {'count': 4}, 'MS': {'count': 1}, 'CD': {'count': 1}, 'ET': {'count': 1}, 'acc-ph': {'count': 5}, 'AC': {'count': 2}, 'OH': {'count': 1}, 'EC': {'count': 2}, 'DL': {'count': 1}, 'AS': {'count': 3}, 'geo-ph': {'count': 2}, 'CG': {'count': 3}, 'CB': {'count': 1}, 'AR': {'count': 1}, 'TR': {'count': 1}, 'atm-clus': {'count': 1}}}} | +| [ArXivHierarchicalClusteringS2S](https://www.kaggle.com/Cornell-University/arxiv) | ['eng'] | Clustering | p2p | [Academic, Written] | None | None | +| [ArguAna](http://argumentation.bplaced.net/arguana/data) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | [Medical, Written] | None | None | +| [ArguAna-PL](https://huggingface.co/datasets/clarin-knext/arguana-pl) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | None | +| [ArmenianParaphrasePC](https://github.com/ivannikov-lab/arpa-paraphrase-corpus) (Arthur Malajyan, 2020) | ['hye'] | PairClassification | s2s | [News, Written] | None | None | +| [ArxivClassification](https://ieeexplore.ieee.org/document/8675939) (He et al., 2019) | ['eng'] | Classification | s2s | [Academic, Written] | None | None | +| [AskUbuntuDupQuestions](https://github.com/taolei87/askubuntu) | ['eng'] | Reranking | s2s | | {'test': 375} | {'test': {'num_samples': 375, 'number_of_characters': 413674, 'num_positive': 2255, 'num_negative': 5245, 'min_query_length': 17, 'avg_query_length': 50.21, 'max_query_length': 148, 'unique_query': 374, 'min_positive_length': 15, 'avg_positive_length': 52.54, 'max_positive_length': 152, 'unique_positive': 2165, 'min_negative_length': 15, 'avg_negative_length': 52.69, 'max_negative_length': 148, 'unique_negative': 5002}} | +| [Assin2RTE](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) (Real et al., 2020) | ['por'] | PairClassification | s2s | [Written] | None | None | +| [Assin2STS](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) (Real et al., 2020) | ['por'] | STS | s2s | [Written] | None | None | +| [AutoRAGRetrieval](https://arxiv.org/abs/2410.20878) (Dongkyu Kim, 2024) | ['kor'] | Retrieval | s2p | [Government, Medical, Legal, Social] | {'test': 834} | {'test': {'number_of_characters': 894.22, 'num_samples': 834, 'num_queries': 114, 'num_documents': 720, 'average_document_length': 1.15, 'average_query_length': 0.61, 'average_relevant_docs_per_query': 1.0}} | +>>>>>>> main | [BIOSSES](https://tabilab.cmpe.boun.edu.tr/BIOSSES/DataSet.html) (Soğancıoğlu et al., 2017) | ['eng'] | STS | s2s | | None | None | | [BQ](https://aclanthology.org/2021.emnlp-main.357) (Shitao Xiao, 2024) | ['cmn'] | STS | s2s | | None | None | -| [BSARDRetrieval](https://huggingface.co/datasets/maastrichtlawtech/bsard) (Louis et al., 2022) | ['fra'] | Retrieval | s2p | [Legal, Spoken] | {'test': 222} | {'test': {'average_document_length': 880.2900631820793, 'average_query_length': 144.77027027027026, 'num_documents': 22633, 'num_queries': 222, 'average_relevant_docs_per_query': 1.0}} | -| [BUCC.v2](https://comparable.limsi.fr/bucc2018/bucc2018-task.html) | ['cmn', 'deu', 'eng', 'fra', 'rus'] | BitextMining | s2s | [Written] | {'test': 641684} | {'test': 101.3} | -| [Banking77Classification](https://arxiv.org/abs/2003.04807) | ['eng'] | Classification | s2s | [Written] | {'test': 3080} | {'test': 54.2} | -| [BelebeleRetrieval](https://arxiv.org/abs/2308.16884) (Lucas Bandarkar, 2023) | ['acm', 'afr', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'azj', 'bam', 'ben', 'bod', 'bul', 'cat', 'ceb', 'ces', 'ckb', 'dan', 'deu', 'ell', 'eng', 'est', 'eus', 'fin', 'fra', 'fuv', 'gaz', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kac', 'kan', 'kat', 'kaz', 'kea', 'khk', 'khm', 'kin', 'kir', 'kor', 'lao', 'lin', 'lit', 'lug', 'luo', 'lvs', 'mal', 'mar', 'mkd', 'mlt', 'mri', 'mya', 'nld', 'nob', 'npi', 'nso', 'nya', 'ory', 'pan', 'pbt', 'pes', 'plt', 'pol', 'por', 'ron', 'rus', 'shn', 'sin', 'slk', 'slv', 'sna', 'snd', 'som', 'sot', 'spa', 'srp', 'ssw', 'sun', 'swe', 'swh', 'tam', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tsn', 'tso', 'tur', 'ukr', 'urd', 'uzn', 'vie', 'war', 'wol', 'xho', 'yor', 'zho', 'zsm', 'zul'] | Retrieval | s2p | [Web, News, Written] | {'test': 103500} | {'test': {'average_document_length': 487.3975028339728, 'average_query_length': 74.49551684802204, 'num_documents': 183488, 'num_queries': 338378, 'average_relevant_docs_per_query': 1.0, 'hf_subset_descriptive_stats': {'acm_Arab-acm_Arab': {'average_document_length': 416.4733606557377, 'average_query_length': 55.84, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'acm_Arab-eng_Latn': {'average_document_length': 416.4733606557377, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-acm_Arab': {'average_document_length': 475.51024590163934, 'average_query_length': 55.84, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'afr_Latn-afr_Latn': {'average_document_length': 503.6659836065574, 'average_query_length': 78.04555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'afr_Latn-eng_Latn': {'average_document_length': 503.6659836065574, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-afr_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 78.04555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'als_Latn-als_Latn': {'average_document_length': 534.016393442623, 'average_query_length': 76.13555555555556, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'als_Latn-eng_Latn': {'average_document_length': 534.016393442623, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-als_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 76.13555555555556, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'amh_Ethi-amh_Ethi': {'average_document_length': 319.8688524590164, 'average_query_length': 49.16111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'amh_Ethi-eng_Latn': {'average_document_length': 319.8688524590164, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-amh_Ethi': {'average_document_length': 475.51024590163934, 'average_query_length': 49.16111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'apc_Arab-apc_Arab': {'average_document_length': 393.0553278688525, 'average_query_length': 55.85777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'apc_Arab-eng_Latn': {'average_document_length': 393.0553278688525, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-apc_Arab': {'average_document_length': 475.51024590163934, 'average_query_length': 55.85777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'arb_Arab-arb_Arab': {'average_document_length': 421.96311475409834, 'average_query_length': 58.55, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'arb_Arab-eng_Latn': {'average_document_length': 421.96311475409834, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-arb_Arab': {'average_document_length': 475.51024590163934, 'average_query_length': 58.55, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'arb_Latn-arb_Latn': {'average_document_length': 555.6188524590164, 'average_query_length': 67.02444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'arb_Latn-eng_Latn': {'average_document_length': 555.6188524590164, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-arb_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 67.02444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ars_Arab-ars_Arab': {'average_document_length': 422.5553278688525, 'average_query_length': 56.43222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ars_Arab-eng_Latn': {'average_document_length': 422.5553278688525, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ars_Arab': {'average_document_length': 475.51024590163934, 'average_query_length': 56.43222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ary_Arab-ary_Arab': {'average_document_length': 411.1475409836066, 'average_query_length': 66.01893095768374, 'num_documents': 488, 'num_queries': 898, 'average_relevant_docs_per_query': 1.0}, 'ary_Arab-eng_Latn': {'average_document_length': 411.1475409836066, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ary_Arab': {'average_document_length': 475.51024590163934, 'average_query_length': 66.01893095768374, 'num_documents': 488, 'num_queries': 898, 'average_relevant_docs_per_query': 1.0}, 'arz_Arab-arz_Arab': {'average_document_length': 412.05122950819674, 'average_query_length': 57.14111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'arz_Arab-eng_Latn': {'average_document_length': 412.05122950819674, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-arz_Arab': {'average_document_length': 475.51024590163934, 'average_query_length': 57.14111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'asm_Beng-asm_Beng': {'average_document_length': 458.5983606557377, 'average_query_length': 68.26, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'asm_Beng-eng_Latn': {'average_document_length': 458.5983606557377, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-asm_Beng': {'average_document_length': 475.51024590163934, 'average_query_length': 68.26, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'azj_Latn-azj_Latn': {'average_document_length': 519.6127049180328, 'average_query_length': 73.51222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'azj_Latn-eng_Latn': {'average_document_length': 519.6127049180328, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-azj_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 73.51222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'bam_Latn-bam_Latn': {'average_document_length': 457.3114754098361, 'average_query_length': 72.34222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'bam_Latn-eng_Latn': {'average_document_length': 457.3114754098361, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-bam_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 72.34222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ben_Beng-ben_Beng': {'average_document_length': 467.7745901639344, 'average_query_length': 69.48444444444445, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ben_Beng-eng_Latn': {'average_document_length': 467.7745901639344, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ben_Beng': {'average_document_length': 475.51024590163934, 'average_query_length': 69.48444444444445, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ben_Latn-ben_Latn': {'average_document_length': 522.8934426229508, 'average_query_length': 74.78777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ben_Latn-eng_Latn': {'average_document_length': 522.8934426229508, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ben_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 74.78777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'bod_Tibt-bod_Tibt': {'average_document_length': 533.3872950819672, 'average_query_length': 86.90222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'bod_Tibt-eng_Latn': {'average_document_length': 533.3872950819672, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-bod_Tibt': {'average_document_length': 475.51024590163934, 'average_query_length': 86.90222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'bul_Cyrl-bul_Cyrl': {'average_document_length': 496.97131147540983, 'average_query_length': 72.89, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'bul_Cyrl-eng_Latn': {'average_document_length': 496.97131147540983, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-bul_Cyrl': {'average_document_length': 475.51024590163934, 'average_query_length': 72.89, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'cat_Latn-cat_Latn': {'average_document_length': 525.4467213114754, 'average_query_length': 75.40666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'cat_Latn-eng_Latn': {'average_document_length': 525.4467213114754, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-cat_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 75.40666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ceb_Latn-ceb_Latn': {'average_document_length': 570.8483606557377, 'average_query_length': 81.19666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ceb_Latn-eng_Latn': {'average_document_length': 570.8483606557377, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ceb_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 81.19666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ces_Latn-ces_Latn': {'average_document_length': 461.0061475409836, 'average_query_length': 67.73333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ces_Latn-eng_Latn': {'average_document_length': 461.0061475409836, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ces_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 67.73333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ckb_Arab-ckb_Arab': {'average_document_length': 462.98770491803276, 'average_query_length': 71.04555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ckb_Arab-eng_Latn': {'average_document_length': 462.98770491803276, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ckb_Arab': {'average_document_length': 475.51024590163934, 'average_query_length': 71.04555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'dan_Latn-dan_Latn': {'average_document_length': 489.4856557377049, 'average_query_length': 72.96888888888888, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'dan_Latn-eng_Latn': {'average_document_length': 489.4856557377049, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-dan_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 72.96888888888888, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'deu_Latn-deu_Latn': {'average_document_length': 555.1659836065573, 'average_query_length': 75.32444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'deu_Latn-eng_Latn': {'average_document_length': 555.1659836065573, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-deu_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 75.32444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ell_Grek-ell_Grek': {'average_document_length': 568.3872950819672, 'average_query_length': 86.92666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ell_Grek-eng_Latn': {'average_document_length': 568.3872950819672, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ell_Grek': {'average_document_length': 475.51024590163934, 'average_query_length': 86.92666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-eng_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'est_Latn-est_Latn': {'average_document_length': 467.1475409836066, 'average_query_length': 67.55888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'est_Latn-eng_Latn': {'average_document_length': 467.1475409836066, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-est_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 67.55888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eus_Latn-eus_Latn': {'average_document_length': 506.19262295081967, 'average_query_length': 74.44777777777777, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eus_Latn-eng_Latn': {'average_document_length': 506.19262295081967, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-eus_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 74.44777777777777, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'fin_Latn-fin_Latn': {'average_document_length': 507.5, 'average_query_length': 72.50888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'fin_Latn-eng_Latn': {'average_document_length': 507.5, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-fin_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 72.50888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'fra_Latn-fra_Latn': {'average_document_length': 564.8401639344262, 'average_query_length': 90.54222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'fra_Latn-eng_Latn': {'average_document_length': 564.8401639344262, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-fra_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 90.54222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'fuv_Latn-fuv_Latn': {'average_document_length': 443.4733606557377, 'average_query_length': 58.42111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'fuv_Latn-eng_Latn': {'average_document_length': 443.4733606557377, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-fuv_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 58.42111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'gaz_Latn-gaz_Latn': {'average_document_length': 563.5389344262295, 'average_query_length': 85.93222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'gaz_Latn-eng_Latn': {'average_document_length': 563.5389344262295, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-gaz_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 85.93222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'grn_Latn-grn_Latn': {'average_document_length': 480.3299180327869, 'average_query_length': 75.10666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'grn_Latn-eng_Latn': {'average_document_length': 480.3299180327869, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-grn_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 75.10666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'guj_Gujr-guj_Gujr': {'average_document_length': 458.1885245901639, 'average_query_length': 62.25666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'guj_Gujr-eng_Latn': {'average_document_length': 458.1885245901639, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-guj_Gujr': {'average_document_length': 475.51024590163934, 'average_query_length': 62.25666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hat_Latn-hat_Latn': {'average_document_length': 438.6700819672131, 'average_query_length': 70.64666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hat_Latn-eng_Latn': {'average_document_length': 438.6700819672131, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-hat_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 70.64666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hau_Latn-hau_Latn': {'average_document_length': 507.24590163934425, 'average_query_length': 85.8488888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hau_Latn-eng_Latn': {'average_document_length': 507.24590163934425, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-hau_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 85.8488888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'heb_Hebr-heb_Hebr': {'average_document_length': 371.36270491803276, 'average_query_length': 55.135555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'heb_Hebr-eng_Latn': {'average_document_length': 371.36270491803276, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-heb_Hebr': {'average_document_length': 475.51024590163934, 'average_query_length': 55.135555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hin_Deva-hin_Deva': {'average_document_length': 473.55737704918033, 'average_query_length': 72.61777777777777, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hin_Deva-eng_Latn': {'average_document_length': 473.55737704918033, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-hin_Deva': {'average_document_length': 475.51024590163934, 'average_query_length': 72.61777777777777, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hin_Latn-hin_Latn': {'average_document_length': 541.7315573770492, 'average_query_length': 74.81222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hin_Latn-eng_Latn': {'average_document_length': 541.7315573770492, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-hin_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 74.81222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hrv_Latn-hrv_Latn': {'average_document_length': 469.202868852459, 'average_query_length': 68.83555555555556, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hrv_Latn-eng_Latn': {'average_document_length': 469.202868852459, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-hrv_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 68.83555555555556, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hun_Latn-hun_Latn': {'average_document_length': 501.1946721311475, 'average_query_length': 74.40555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hun_Latn-eng_Latn': {'average_document_length': 501.1946721311475, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-hun_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 74.40555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hye_Armn-hye_Armn': {'average_document_length': 527.5102459016393, 'average_query_length': 75.42555555555556, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hye_Armn-eng_Latn': {'average_document_length': 527.5102459016393, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-hye_Armn': {'average_document_length': 475.51024590163934, 'average_query_length': 75.42555555555556, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ibo_Latn-ibo_Latn': {'average_document_length': 482.3483606557377, 'average_query_length': 72.51501668520578, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0011123470522802}, 'ibo_Latn-eng_Latn': {'average_document_length': 482.3483606557377, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ibo_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 72.51501668520578, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0011123470522802}, 'ilo_Latn-ilo_Latn': {'average_document_length': 574.6987704918033, 'average_query_length': 85.7611111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ilo_Latn-eng_Latn': {'average_document_length': 574.6987704918033, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ilo_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 85.7611111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ind_Latn-ind_Latn': {'average_document_length': 516.0573770491803, 'average_query_length': 82.10555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ind_Latn-eng_Latn': {'average_document_length': 516.0573770491803, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ind_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 82.10555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'isl_Latn-isl_Latn': {'average_document_length': 470.73975409836066, 'average_query_length': 77.27333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'isl_Latn-eng_Latn': {'average_document_length': 470.73975409836066, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-isl_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 77.27333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ita_Latn-ita_Latn': {'average_document_length': 560.9344262295082, 'average_query_length': 83.49777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ita_Latn-eng_Latn': {'average_document_length': 560.9344262295082, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ita_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 83.49777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'jav_Latn-jav_Latn': {'average_document_length': 494.1803278688525, 'average_query_length': 78.60666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'jav_Latn-eng_Latn': {'average_document_length': 494.1803278688525, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-jav_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 78.60666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'jpn_Jpan-jpn_Jpan': {'average_document_length': 207.74795081967213, 'average_query_length': 35.79, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'jpn_Jpan-eng_Latn': {'average_document_length': 207.74795081967213, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-jpn_Jpan': {'average_document_length': 475.51024590163934, 'average_query_length': 35.79, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kac_Latn-kac_Latn': {'average_document_length': 605.2889344262295, 'average_query_length': 98.64182424916574, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0}, 'kac_Latn-eng_Latn': {'average_document_length': 605.2889344262295, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-kac_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 98.64182424916574, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0}, 'kan_Knda-kan_Knda': {'average_document_length': 498.9077868852459, 'average_query_length': 72.13666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kan_Knda-eng_Latn': {'average_document_length': 498.9077868852459, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-kan_Knda': {'average_document_length': 475.51024590163934, 'average_query_length': 72.13666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kat_Geor-kat_Geor': {'average_document_length': 521.7766393442623, 'average_query_length': 74.81444444444445, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kat_Geor-eng_Latn': {'average_document_length': 521.7766393442623, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-kat_Geor': {'average_document_length': 475.51024590163934, 'average_query_length': 74.81444444444445, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kaz_Cyrl-kaz_Cyrl': {'average_document_length': 488.2110655737705, 'average_query_length': 70.75666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kaz_Cyrl-eng_Latn': {'average_document_length': 488.2110655737705, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-kaz_Cyrl': {'average_document_length': 475.51024590163934, 'average_query_length': 70.75666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kea_Latn-kea_Latn': {'average_document_length': 471.5594262295082, 'average_query_length': 75.94111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kea_Latn-eng_Latn': {'average_document_length': 471.5594262295082, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-kea_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 75.94111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'khk_Cyrl-khk_Cyrl': {'average_document_length': 496.655737704918, 'average_query_length': 73.33444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'khk_Cyrl-eng_Latn': {'average_document_length': 496.655737704918, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-khk_Cyrl': {'average_document_length': 475.51024590163934, 'average_query_length': 73.33444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'khm_Khmr-khm_Khmr': {'average_document_length': 562.4139344262295, 'average_query_length': 75.74888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'khm_Khmr-eng_Latn': {'average_document_length': 562.4139344262295, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-khm_Khmr': {'average_document_length': 475.51024590163934, 'average_query_length': 75.74888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kin_Latn-kin_Latn': {'average_document_length': 529.2520491803278, 'average_query_length': 79.89655172413794, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0011123470522802}, 'kin_Latn-eng_Latn': {'average_document_length': 529.2520491803278, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-kin_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 79.89655172413794, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0011123470522802}, 'kir_Cyrl-kir_Cyrl': {'average_document_length': 487.80737704918033, 'average_query_length': 74.42333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kir_Cyrl-eng_Latn': {'average_document_length': 487.80737704918033, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-kir_Cyrl': {'average_document_length': 475.51024590163934, 'average_query_length': 74.42333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kor_Hang-kor_Hang': {'average_document_length': 241.32991803278688, 'average_query_length': 35.257777777777775, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'kor_Hang-eng_Latn': {'average_document_length': 241.32991803278688, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-kor_Hang': {'average_document_length': 475.51024590163934, 'average_query_length': 35.257777777777775, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'lao_Laoo-lao_Laoo': {'average_document_length': 471.6495901639344, 'average_query_length': 63.31333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'lao_Laoo-eng_Latn': {'average_document_length': 471.6495901639344, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-lao_Laoo': {'average_document_length': 475.51024590163934, 'average_query_length': 63.31333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'lin_Latn-lin_Latn': {'average_document_length': 512.9016393442623, 'average_query_length': 81.56681514476615, 'num_documents': 488, 'num_queries': 898, 'average_relevant_docs_per_query': 1.0022271714922049}, 'lin_Latn-eng_Latn': {'average_document_length': 512.9016393442623, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-lin_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 81.56681514476615, 'num_documents': 488, 'num_queries': 898, 'average_relevant_docs_per_query': 1.0022271714922049}, 'lit_Latn-lit_Latn': {'average_document_length': 474.0553278688525, 'average_query_length': 68.69888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'lit_Latn-eng_Latn': {'average_document_length': 474.0553278688525, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-lit_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 68.69888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'lug_Latn-lug_Latn': {'average_document_length': 485.73975409836066, 'average_query_length': 78.52057842046719, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0011123470522802}, 'lug_Latn-eng_Latn': {'average_document_length': 485.73975409836066, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-lug_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 78.52057842046719, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0011123470522802}, 'luo_Latn-luo_Latn': {'average_document_length': 497.53688524590166, 'average_query_length': 73.14333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'luo_Latn-eng_Latn': {'average_document_length': 497.53688524590166, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-luo_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 73.14333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'lvs_Latn-lvs_Latn': {'average_document_length': 487.21311475409834, 'average_query_length': 69.97888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'lvs_Latn-eng_Latn': {'average_document_length': 487.21311475409834, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-lvs_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 69.97888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'mal_Mlym-mal_Mlym': {'average_document_length': 539.2827868852459, 'average_query_length': 80.69222222222223, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'mal_Mlym-eng_Latn': {'average_document_length': 539.2827868852459, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-mal_Mlym': {'average_document_length': 475.51024590163934, 'average_query_length': 80.69222222222223, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'mar_Deva-mar_Deva': {'average_document_length': 478.67418032786884, 'average_query_length': 68.62625139043382, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0011123470522802}, 'mar_Deva-eng_Latn': {'average_document_length': 478.67418032786884, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-mar_Deva': {'average_document_length': 475.51024590163934, 'average_query_length': 68.62625139043382, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0011123470522802}, 'mkd_Cyrl-mkd_Cyrl': {'average_document_length': 495.77868852459017, 'average_query_length': 74.01333333333334, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'mkd_Cyrl-eng_Latn': {'average_document_length': 495.77868852459017, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-mkd_Cyrl': {'average_document_length': 475.51024590163934, 'average_query_length': 74.01333333333334, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'mlt_Latn-mlt_Latn': {'average_document_length': 525.8995901639345, 'average_query_length': 75.00444444444445, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'mlt_Latn-eng_Latn': {'average_document_length': 525.8995901639345, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-mlt_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 75.00444444444445, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'mri_Latn-mri_Latn': {'average_document_length': 526.0860655737705, 'average_query_length': 81.71444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'mri_Latn-eng_Latn': {'average_document_length': 526.0860655737705, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-mri_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 81.71444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'mya_Mymr-mya_Mymr': {'average_document_length': 590.389344262295, 'average_query_length': 89.28333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'mya_Mymr-eng_Latn': {'average_document_length': 590.389344262295, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-mya_Mymr': {'average_document_length': 475.51024590163934, 'average_query_length': 89.28333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'nld_Latn-nld_Latn': {'average_document_length': 529.1434426229508, 'average_query_length': 75.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'nld_Latn-eng_Latn': {'average_document_length': 529.1434426229508, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-nld_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 75.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'nob_Latn-nob_Latn': {'average_document_length': 479.13729508196724, 'average_query_length': 71.04555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'nob_Latn-eng_Latn': {'average_document_length': 479.13729508196724, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-nob_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 71.04555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'npi_Deva-npi_Deva': {'average_document_length': 456.9590163934426, 'average_query_length': 66.89666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'npi_Deva-eng_Latn': {'average_document_length': 456.9590163934426, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-npi_Deva': {'average_document_length': 475.51024590163934, 'average_query_length': 66.89666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'npi_Latn-npi_Latn': {'average_document_length': 515.9815573770492, 'average_query_length': 71.89666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'npi_Latn-eng_Latn': {'average_document_length': 515.9815573770492, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-npi_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 71.89666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'nso_Latn-nso_Latn': {'average_document_length': 548.0225409836065, 'average_query_length': 86.77444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'nso_Latn-eng_Latn': {'average_document_length': 548.0225409836065, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-nso_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 86.77444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'nya_Latn-nya_Latn': {'average_document_length': 532.3934426229508, 'average_query_length': 90.78777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'nya_Latn-eng_Latn': {'average_document_length': 532.3934426229508, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-nya_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 90.78777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ory_Orya-ory_Orya': {'average_document_length': 487.78483606557376, 'average_query_length': 72.95777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ory_Orya-eng_Latn': {'average_document_length': 487.78483606557376, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ory_Orya': {'average_document_length': 475.51024590163934, 'average_query_length': 72.95777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'pan_Guru-pan_Guru': {'average_document_length': 480.2438524590164, 'average_query_length': 73.29777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'pan_Guru-eng_Latn': {'average_document_length': 480.2438524590164, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-pan_Guru': {'average_document_length': 475.51024590163934, 'average_query_length': 73.29777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'pbt_Arab-pbt_Arab': {'average_document_length': 453.3299180327869, 'average_query_length': 67.67111111111112, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'pbt_Arab-eng_Latn': {'average_document_length': 453.3299180327869, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-pbt_Arab': {'average_document_length': 475.51024590163934, 'average_query_length': 67.67111111111112, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'pes_Arab-pes_Arab': {'average_document_length': 448.84631147540983, 'average_query_length': 64.75111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'pes_Arab-eng_Latn': {'average_document_length': 448.84631147540983, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-pes_Arab': {'average_document_length': 475.51024590163934, 'average_query_length': 64.75111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'plt_Latn-plt_Latn': {'average_document_length': 581.2745901639345, 'average_query_length': 94.99555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'plt_Latn-eng_Latn': {'average_document_length': 581.2745901639345, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-plt_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 94.99555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'pol_Latn-pol_Latn': {'average_document_length': 504.0409836065574, 'average_query_length': 74.09777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'pol_Latn-eng_Latn': {'average_document_length': 504.0409836065574, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-pol_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 74.09777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'por_Latn-por_Latn': {'average_document_length': 517.2827868852459, 'average_query_length': 78.11666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'por_Latn-eng_Latn': {'average_document_length': 517.2827868852459, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-por_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 78.11666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ron_Latn-ron_Latn': {'average_document_length': 534.8668032786885, 'average_query_length': 78.74222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ron_Latn-eng_Latn': {'average_document_length': 534.8668032786885, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ron_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 78.74222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'rus_Cyrl-rus_Cyrl': {'average_document_length': 520.1700819672132, 'average_query_length': 83.16333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'rus_Cyrl-eng_Latn': {'average_document_length': 520.1700819672132, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-rus_Cyrl': {'average_document_length': 475.51024590163934, 'average_query_length': 83.16333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'shn_Mymr-shn_Mymr': {'average_document_length': 676.172131147541, 'average_query_length': 75.90222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'shn_Mymr-eng_Latn': {'average_document_length': 676.172131147541, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-shn_Mymr': {'average_document_length': 475.51024590163934, 'average_query_length': 75.90222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'sin_Latn-sin_Latn': {'average_document_length': 590.7889344262295, 'average_query_length': 94.46666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'sin_Latn-eng_Latn': {'average_document_length': 590.7889344262295, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-sin_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 94.46666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'sin_Sinh-sin_Sinh': {'average_document_length': 478.66803278688525, 'average_query_length': 69.91777777777777, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'sin_Sinh-eng_Latn': {'average_document_length': 478.66803278688525, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-sin_Sinh': {'average_document_length': 475.51024590163934, 'average_query_length': 69.91777777777777, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'slk_Latn-slk_Latn': {'average_document_length': 476.7766393442623, 'average_query_length': 68.5411111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'slk_Latn-eng_Latn': {'average_document_length': 476.7766393442623, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-slk_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 68.5411111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'slv_Latn-slv_Latn': {'average_document_length': 474.84631147540983, 'average_query_length': 68.79888888888888, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'slv_Latn-eng_Latn': {'average_document_length': 474.84631147540983, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-slv_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 68.79888888888888, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'sna_Latn-sna_Latn': {'average_document_length': 532.5860655737705, 'average_query_length': 81.30700778642937, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0}, 'sna_Latn-eng_Latn': {'average_document_length': 532.5860655737705, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-sna_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 81.30700778642937, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0}, 'snd_Arab-snd_Arab': {'average_document_length': 431.48770491803276, 'average_query_length': 63.42333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'snd_Arab-eng_Latn': {'average_document_length': 431.48770491803276, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-snd_Arab': {'average_document_length': 475.51024590163934, 'average_query_length': 63.42333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'som_Latn-som_Latn': {'average_document_length': 542.0737704918033, 'average_query_length': 90.95777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'som_Latn-eng_Latn': {'average_document_length': 542.0737704918033, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-som_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 90.95777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'sot_Latn-sot_Latn': {'average_document_length': 573.3258196721312, 'average_query_length': 83.13111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'sot_Latn-eng_Latn': {'average_document_length': 573.3258196721312, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-sot_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 83.13111111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'spa_Latn-spa_Latn': {'average_document_length': 564.3319672131148, 'average_query_length': 82.16, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'spa_Latn-eng_Latn': {'average_document_length': 564.3319672131148, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-spa_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 82.16, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'srp_Cyrl-srp_Cyrl': {'average_document_length': 471.84631147540983, 'average_query_length': 67.49833147942158, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0011123470522802}, 'srp_Cyrl-eng_Latn': {'average_document_length': 471.84631147540983, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-srp_Cyrl': {'average_document_length': 475.51024590163934, 'average_query_length': 67.49833147942158, 'num_documents': 488, 'num_queries': 899, 'average_relevant_docs_per_query': 1.0011123470522802}, 'ssw_Latn-ssw_Latn': {'average_document_length': 535.0901639344262, 'average_query_length': 81.09777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ssw_Latn-eng_Latn': {'average_document_length': 535.0901639344262, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ssw_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 81.09777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'sun_Latn-sun_Latn': {'average_document_length': 495.3032786885246, 'average_query_length': 78.16, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'sun_Latn-eng_Latn': {'average_document_length': 495.3032786885246, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-sun_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 78.16, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'swe_Latn-swe_Latn': {'average_document_length': 480.6803278688525, 'average_query_length': 68.67666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'swe_Latn-eng_Latn': {'average_document_length': 480.6803278688525, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-swe_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 68.67666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'swh_Latn-swh_Latn': {'average_document_length': 499.0983606557377, 'average_query_length': 80.56, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'swh_Latn-eng_Latn': {'average_document_length': 499.0983606557377, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-swh_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 80.56, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tam_Taml-tam_Taml': {'average_document_length': 555.5286885245902, 'average_query_length': 81.12777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tam_Taml-eng_Latn': {'average_document_length': 555.5286885245902, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-tam_Taml': {'average_document_length': 475.51024590163934, 'average_query_length': 81.12777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tel_Telu-tel_Telu': {'average_document_length': 481.5245901639344, 'average_query_length': 72.18777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tel_Telu-eng_Latn': {'average_document_length': 481.5245901639344, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-tel_Telu': {'average_document_length': 475.51024590163934, 'average_query_length': 72.18777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tgk_Cyrl-tgk_Cyrl': {'average_document_length': 528.516393442623, 'average_query_length': 74.28111111111112, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tgk_Cyrl-eng_Latn': {'average_document_length': 528.516393442623, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-tgk_Cyrl': {'average_document_length': 475.51024590163934, 'average_query_length': 74.28111111111112, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tgl_Latn-tgl_Latn': {'average_document_length': 597.6270491803278, 'average_query_length': 82.34555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tgl_Latn-eng_Latn': {'average_document_length': 597.6270491803278, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-tgl_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 82.34555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tha_Thai-tha_Thai': {'average_document_length': 456.1659836065574, 'average_query_length': 59.46666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tha_Thai-eng_Latn': {'average_document_length': 456.1659836065574, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-tha_Thai': {'average_document_length': 475.51024590163934, 'average_query_length': 59.46666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tir_Ethi-tir_Ethi': {'average_document_length': 327.6967213114754, 'average_query_length': 51.99888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tir_Ethi-eng_Latn': {'average_document_length': 327.6967213114754, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-tir_Ethi': {'average_document_length': 475.51024590163934, 'average_query_length': 51.99888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tsn_Latn-tsn_Latn': {'average_document_length': 591.7131147540983, 'average_query_length': 87.12777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tsn_Latn-eng_Latn': {'average_document_length': 591.7131147540983, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-tsn_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 87.12777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tso_Latn-tso_Latn': {'average_document_length': 569.6475409836065, 'average_query_length': 91.69444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tso_Latn-eng_Latn': {'average_document_length': 569.6475409836065, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-tso_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 91.69444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tur_Latn-tur_Latn': {'average_document_length': 489.0409836065574, 'average_query_length': 71.56222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'tur_Latn-eng_Latn': {'average_document_length': 489.0409836065574, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-tur_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 71.56222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ukr_Cyrl-ukr_Cyrl': {'average_document_length': 488.11475409836066, 'average_query_length': 72.08222222222223, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ukr_Cyrl-eng_Latn': {'average_document_length': 488.11475409836066, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-ukr_Cyrl': {'average_document_length': 475.51024590163934, 'average_query_length': 72.08222222222223, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'urd_Arab-urd_Arab': {'average_document_length': 470.452868852459, 'average_query_length': 70.52666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'urd_Arab-eng_Latn': {'average_document_length': 470.452868852459, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-urd_Arab': {'average_document_length': 475.51024590163934, 'average_query_length': 70.52666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'urd_Latn-urd_Latn': {'average_document_length': 590.5348360655738, 'average_query_length': 90.07, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'urd_Latn-eng_Latn': {'average_document_length': 590.5348360655738, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-urd_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 90.07, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'uzn_Latn-uzn_Latn': {'average_document_length': 539.2418032786885, 'average_query_length': 77.61333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'uzn_Latn-eng_Latn': {'average_document_length': 539.2418032786885, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-uzn_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 77.61333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'vie_Latn-vie_Latn': {'average_document_length': 499.8360655737705, 'average_query_length': 73.05333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'vie_Latn-eng_Latn': {'average_document_length': 499.8360655737705, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-vie_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 73.05333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'war_Latn-war_Latn': {'average_document_length': 592.8688524590164, 'average_query_length': 86.07555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'war_Latn-eng_Latn': {'average_document_length': 592.8688524590164, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-war_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 86.07555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'wol_Latn-wol_Latn': {'average_document_length': 456.9795081967213, 'average_query_length': 70.60555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'wol_Latn-eng_Latn': {'average_document_length': 456.9795081967213, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-wol_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 70.60555555555555, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'xho_Latn-xho_Latn': {'average_document_length': 505.0655737704918, 'average_query_length': 78.50333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'xho_Latn-eng_Latn': {'average_document_length': 505.0655737704918, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-xho_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 78.50333333333333, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'yor_Latn-yor_Latn': {'average_document_length': 459.5204918032787, 'average_query_length': 68.64, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'yor_Latn-eng_Latn': {'average_document_length': 459.5204918032787, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-yor_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 68.64, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'zho_Hans-zho_Hans': {'average_document_length': 159.76024590163934, 'average_query_length': 21.747777777777777, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'zho_Hans-eng_Latn': {'average_document_length': 159.76024590163934, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-zho_Hans': {'average_document_length': 475.51024590163934, 'average_query_length': 21.747777777777777, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'zho_Hant-zho_Hant': {'average_document_length': 149.77254098360655, 'average_query_length': 21.07888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'zho_Hant-eng_Latn': {'average_document_length': 149.77254098360655, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-zho_Hant': {'average_document_length': 475.51024590163934, 'average_query_length': 21.07888888888889, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'zsm_Latn-zsm_Latn': {'average_document_length': 528.9139344262295, 'average_query_length': 78.92444444444445, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'zsm_Latn-eng_Latn': {'average_document_length': 528.9139344262295, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-zsm_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 78.92444444444445, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'zul_Latn-zul_Latn': {'average_document_length': 532.9713114754098, 'average_query_length': 76.0411111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'zul_Latn-eng_Latn': {'average_document_length': 532.9713114754098, 'average_query_length': 77.34777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'eng_Latn-zul_Latn': {'average_document_length': 475.51024590163934, 'average_query_length': 76.0411111111111, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'arb_Arab-arb_Latn': {'average_document_length': 421.96311475409834, 'average_query_length': 67.02444444444444, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'arb_Latn-arb_Arab': {'average_document_length': 555.6188524590164, 'average_query_length': 58.55, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ben_Beng-ben_Latn': {'average_document_length': 467.7745901639344, 'average_query_length': 74.78777777777778, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'ben_Latn-ben_Beng': {'average_document_length': 522.8934426229508, 'average_query_length': 69.48444444444445, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hin_Deva-hin_Latn': {'average_document_length': 473.55737704918033, 'average_query_length': 74.81222222222222, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'hin_Latn-hin_Deva': {'average_document_length': 541.7315573770492, 'average_query_length': 72.61777777777777, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'npi_Deva-npi_Latn': {'average_document_length': 456.9590163934426, 'average_query_length': 71.89666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'npi_Latn-npi_Deva': {'average_document_length': 515.9815573770492, 'average_query_length': 66.89666666666666, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'sin_Sinh-sin_Latn': {'average_document_length': 478.66803278688525, 'average_query_length': 94.46666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'sin_Latn-sin_Sinh': {'average_document_length': 590.7889344262295, 'average_query_length': 69.91777777777777, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'urd_Arab-urd_Latn': {'average_document_length': 470.452868852459, 'average_query_length': 90.07, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}, 'urd_Latn-urd_Arab': {'average_document_length': 590.5348360655738, 'average_query_length': 70.52666666666667, 'num_documents': 488, 'num_queries': 900, 'average_relevant_docs_per_query': 1.0}}}} | -| [BengaliDocumentClassification](https://aclanthology.org/2023.eacl-main.4) | ['ben'] | Classification | s2s | [News, Written] | {'test': 2048} | {'test': 1658.1} | -| [BengaliHateSpeechClassification](https://huggingface.co/datasets/bn_hate_speech) (Karim et al., 2020) | ['ben'] | Classification | s2s | [News, Written] | {'train': 3418} | {'train': 103.42} | -| [BengaliSentimentAnalysis](https://data.mendeley.com/datasets/p6zc7krs37/4) (Sazzed et al., 2020) | ['ben'] | Classification | s2s | [Reviews, Written] | {'train': 11807} | {'train': 69.66} | -| [BibleNLPBitextMining](https://arxiv.org/abs/2304.09919) (Akerman et al., 2023) | ['aai', 'aak', 'aau', 'aaz', 'abt', 'abx', 'aby', 'acf', 'acr', 'acu', 'adz', 'aer', 'aey', 'agd', 'agg', 'agm', 'agn', 'agr', 'agt', 'agu', 'aia', 'aii', 'aka', 'ake', 'alp', 'alq', 'als', 'aly', 'ame', 'amf', 'amk', 'amm', 'amn', 'amo', 'amp', 'amr', 'amu', 'amx', 'anh', 'anv', 'aoi', 'aoj', 'aom', 'aon', 'apb', 'ape', 'apn', 'apr', 'apu', 'apw', 'apz', 'arb', 'are', 'arl', 'arn', 'arp', 'asm', 'aso', 'ata', 'atb', 'atd', 'atg', 'att', 'auc', 'aui', 'auy', 'avt', 'awb', 'awk', 'awx', 'azb', 'azg', 'azz', 'bao', 'bba', 'bbb', 'bbr', 'bch', 'bco', 'bdd', 'bea', 'bef', 'bel', 'ben', 'beo', 'beu', 'bgs', 'bgt', 'bhg', 'bhl', 'big', 'bjk', 'bjp', 'bjr', 'bjv', 'bjz', 'bkd', 'bki', 'bkq', 'bkx', 'blw', 'blz', 'bmh', 'bmk', 'bmr', 'bmu', 'bnp', 'boa', 'boj', 'bon', 'box', 'bpr', 'bps', 'bqc', 'bqp', 'bre', 'bsj', 'bsn', 'bsp', 'bss', 'buk', 'bus', 'bvd', 'bvr', 'bxh', 'byr', 'byx', 'bzd', 'bzh', 'bzj', 'caa', 'cab', 'cac', 'caf', 'cak', 'cao', 'cap', 'car', 'cav', 'cax', 'cbc', 'cbi', 'cbk', 'cbr', 'cbs', 'cbt', 'cbu', 'cbv', 'cco', 'ceb', 'cek', 'ces', 'cgc', 'cha', 'chd', 'chf', 'chk', 'chq', 'chz', 'cjo', 'cjv', 'ckb', 'cle', 'clu', 'cme', 'cmn', 'cni', 'cnl', 'cnt', 'cof', 'con', 'cop', 'cot', 'cpa', 'cpb', 'cpc', 'cpu', 'cpy', 'crn', 'crx', 'cso', 'csy', 'cta', 'cth', 'ctp', 'ctu', 'cub', 'cuc', 'cui', 'cuk', 'cut', 'cux', 'cwe', 'cya', 'daa', 'dad', 'dah', 'dan', 'ded', 'deu', 'dgc', 'dgr', 'dgz', 'dhg', 'dif', 'dik', 'dji', 'djk', 'djr', 'dob', 'dop', 'dov', 'dwr', 'dww', 'dwy', 'ebk', 'eko', 'emi', 'emp', 'eng', 'enq', 'epo', 'eri', 'ese', 'esk', 'etr', 'ewe', 'faa', 'fai', 'far', 'ffm', 'for', 'fra', 'fue', 'fuf', 'fuh', 'gah', 'gai', 'gam', 'gaw', 'gdn', 'gdr', 'geb', 'gfk', 'ghs', 'glk', 'gmv', 'gng', 'gnn', 'gnw', 'gof', 'grc', 'gub', 'guh', 'gui', 'guj', 'gul', 'gum', 'gun', 'guo', 'gup', 'gux', 'gvc', 'gvf', 'gvn', 'gvs', 'gwi', 'gym', 'gyr', 'hat', 'hau', 'haw', 'hbo', 'hch', 'heb', 'heg', 'hin', 'hix', 'hla', 'hlt', 'hmo', 'hns', 'hop', 'hot', 'hrv', 'hto', 'hub', 'hui', 'hun', 'hus', 'huu', 'huv', 'hvn', 'ian', 'ign', 'ikk', 'ikw', 'ilo', 'imo', 'inb', 'ind', 'ino', 'iou', 'ipi', 'isn', 'ita', 'iws', 'ixl', 'jac', 'jae', 'jao', 'jic', 'jid', 'jiv', 'jni', 'jpn', 'jvn', 'kan', 'kaq', 'kbc', 'kbh', 'kbm', 'kbq', 'kdc', 'kde', 'kdl', 'kek', 'ken', 'kew', 'kgf', 'kgk', 'kgp', 'khs', 'khz', 'kik', 'kiw', 'kiz', 'kje', 'kjs', 'kkc', 'kkl', 'klt', 'klv', 'kmg', 'kmh', 'kmk', 'kmo', 'kms', 'kmu', 'kne', 'knf', 'knj', 'knv', 'kos', 'kpf', 'kpg', 'kpj', 'kpr', 'kpw', 'kpx', 'kqa', 'kqc', 'kqf', 'kql', 'kqw', 'ksd', 'ksj', 'ksr', 'ktm', 'kto', 'kud', 'kue', 'kup', 'kvg', 'kvn', 'kwd', 'kwf', 'kwi', 'kwj', 'kyc', 'kyf', 'kyg', 'kyq', 'kyz', 'kze', 'lac', 'lat', 'lbb', 'lbk', 'lcm', 'leu', 'lex', 'lgl', 'lid', 'lif', 'lin', 'lit', 'llg', 'lug', 'luo', 'lww', 'maa', 'maj', 'mal', 'mam', 'maq', 'mar', 'mau', 'mav', 'maz', 'mbb', 'mbc', 'mbh', 'mbj', 'mbl', 'mbs', 'mbt', 'mca', 'mcb', 'mcd', 'mcf', 'mco', 'mcp', 'mcq', 'mcr', 'mdy', 'med', 'mee', 'mek', 'meq', 'met', 'meu', 'mgc', 'mgh', 'mgw', 'mhl', 'mib', 'mic', 'mie', 'mig', 'mih', 'mil', 'mio', 'mir', 'mit', 'miz', 'mjc', 'mkj', 'mkl', 'mkn', 'mks', 'mle', 'mlh', 'mlp', 'mmo', 'mmx', 'mna', 'mop', 'mox', 'mph', 'mpj', 'mpm', 'mpp', 'mps', 'mpt', 'mpx', 'mqb', 'mqj', 'msb', 'msc', 'msk', 'msm', 'msy', 'mti', 'mto', 'mux', 'muy', 'mva', 'mvn', 'mwc', 'mwe', 'mwf', 'mwp', 'mxb', 'mxp', 'mxq', 'mxt', 'mya', 'myk', 'myu', 'myw', 'myy', 'mzz', 'nab', 'naf', 'nak', 'nas', 'nbq', 'nca', 'nch', 'ncj', 'ncl', 'ncu', 'ndg', 'ndj', 'nfa', 'ngp', 'ngu', 'nhe', 'nhg', 'nhi', 'nho', 'nhr', 'nhu', 'nhw', 'nhy', 'nif', 'nii', 'nin', 'nko', 'nld', 'nlg', 'nna', 'nnq', 'noa', 'nop', 'not', 'nou', 'npi', 'npl', 'nsn', 'nss', 'ntj', 'ntp', 'ntu', 'nuy', 'nvm', 'nwi', 'nya', 'nys', 'nyu', 'obo', 'okv', 'omw', 'ong', 'ons', 'ood', 'opm', 'ory', 'ote', 'otm', 'otn', 'otq', 'ots', 'pab', 'pad', 'pah', 'pan', 'pao', 'pes', 'pib', 'pio', 'pir', 'piu', 'pjt', 'pls', 'plu', 'pma', 'poe', 'poh', 'poi', 'pol', 'pon', 'por', 'poy', 'ppo', 'prf', 'pri', 'ptp', 'ptu', 'pwg', 'qub', 'quc', 'quf', 'quh', 'qul', 'qup', 'qvc', 'qve', 'qvh', 'qvm', 'qvn', 'qvs', 'qvw', 'qvz', 'qwh', 'qxh', 'qxn', 'qxo', 'rai', 'reg', 'rgu', 'rkb', 'rmc', 'rmy', 'ron', 'roo', 'rop', 'row', 'rro', 'ruf', 'rug', 'rus', 'rwo', 'sab', 'san', 'sbe', 'sbk', 'sbs', 'seh', 'sey', 'sgb', 'sgz', 'shj', 'shp', 'sim', 'sja', 'sll', 'smk', 'snc', 'snn', 'snp', 'snx', 'sny', 'som', 'soq', 'soy', 'spa', 'spl', 'spm', 'spp', 'sps', 'spy', 'sri', 'srm', 'srn', 'srp', 'srq', 'ssd', 'ssg', 'ssx', 'stp', 'sua', 'sue', 'sus', 'suz', 'swe', 'swh', 'swp', 'sxb', 'tac', 'taj', 'tam', 'tav', 'taw', 'tbc', 'tbf', 'tbg', 'tbo', 'tbz', 'tca', 'tcs', 'tcz', 'tdt', 'tee', 'tel', 'ter', 'tet', 'tew', 'tfr', 'tgk', 'tgl', 'tgo', 'tgp', 'tha', 'tif', 'tim', 'tiw', 'tiy', 'tke', 'tku', 'tlf', 'tmd', 'tna', 'tnc', 'tnk', 'tnn', 'tnp', 'toc', 'tod', 'tof', 'toj', 'ton', 'too', 'top', 'tos', 'tpa', 'tpi', 'tpt', 'tpz', 'trc', 'tsw', 'ttc', 'tte', 'tuc', 'tue', 'tuf', 'tuo', 'tur', 'tvk', 'twi', 'txq', 'txu', 'tzj', 'tzo', 'ubr', 'ubu', 'udu', 'uig', 'ukr', 'uli', 'ulk', 'upv', 'ura', 'urb', 'urd', 'uri', 'urt', 'urw', 'usa', 'usp', 'uvh', 'uvl', 'vid', 'vie', 'viv', 'vmy', 'waj', 'wal', 'wap', 'wat', 'wbi', 'wbp', 'wed', 'wer', 'wim', 'wiu', 'wiv', 'wmt', 'wmw', 'wnc', 'wnu', 'wol', 'wos', 'wrk', 'wro', 'wrs', 'wsk', 'wuv', 'xav', 'xbi', 'xed', 'xla', 'xnn', 'xon', 'xsi', 'xtd', 'xtm', 'yaa', 'yad', 'yal', 'yap', 'yaq', 'yby', 'ycn', 'yka', 'yle', 'yml', 'yon', 'yor', 'yrb', 'yre', 'yss', 'yuj', 'yut', 'yuw', 'yva', 'zaa', 'zab', 'zac', 'zad', 'zai', 'zaj', 'zam', 'zao', 'zap', 'zar', 'zas', 'zat', 'zav', 'zaw', 'zca', 'zga', 'zia', 'ziw', 'zlm', 'zos', 'zpc', 'zpl', 'zpm', 'zpo', 'zpq', 'zpu', 'zpv', 'zpz', 'zsr', 'ztq', 'zty', 'zyp'] | BitextMining | s2s | [Religious, Written] | {'train': 256} | {'train': 120} | -| [BigPatentClustering.v2](https://huggingface.co/datasets/NortheasternUniversity/big_patent) (Eva Sharma and Chen Li and Lu Wang, 2019) | ['eng'] | Clustering | p2p | [Legal, Written] | {'test': 2048} | {'test': 30995.5} | -| [BiorxivClusteringP2P.v2](https://api.biorxiv.org/) | ['eng'] | Clustering | p2p | [Academic, Written] | {'test': 2151} | {'test': 1664.0} | -| [BiorxivClusteringS2S.v2](https://api.biorxiv.org/) | ['eng'] | Clustering | s2s | [Academic, Written] | {'test': 2151} | {'test': 101.7} | -| [BlurbsClusteringP2P.v2](https://www.inf.uni-hamburg.de/en/inst/ab/lt/resources/data/germeval-2019-hmc.html) (Steffen Remus, 2019) | ['deu'] | Clustering | p2p | [Fiction, Written] | {'test': 2048} | {'test': 664.09} | -| [BlurbsClusteringS2S.v2](https://www.inf.uni-hamburg.de/en/inst/ab/lt/resources/data/germeval-2019-hmc.html) (Steffen Remus, 2019) | ['deu'] | Clustering | s2s | [Fiction, Written] | {'test': 2048} | {'test': 23.02} | -| [BornholmBitextMining](https://aclanthology.org/W19-6138/) | ['dan'] | BitextMining | s2s | [Web, Social, Fiction, Written] | {'test': 500} | {'test': {'average_sentence1_length': 49.834, 'average_sentence2_length': 38.888, 'num_samples': 500}} | -| [BrazilianToxicTweetsClassification](https://paperswithcode.com/dataset/told-br) (Joao Augusto Leite and Diego F. Silva and Kalina Bontcheva and Carolina Scarton, 2020) | ['por'] | MultilabelClassification | s2s | [Constructed, Written] | {'test': 2048} | {'test': 85.05} | -| [BrightRetrieval](https://huggingface.co/datasets/xlangai/BRIGHT) (Hongjin Su, 2024) | ['eng'] | Retrieval | s2p | [Non-fiction] | {'standard': 1334914, 'long': 7048} | {'standard': 800.3994729248476, 'long': 46527.35839954597} | -| [BulgarianStoreReviewSentimentClassfication](https://doi.org/10.7910/DVN/TXIK9P) (Georgieva-Trifonova et al., 2018) | ['bul'] | Classification | s2s | [Reviews, Written] | {'test': 182} | {'test': 316.7} | -| [CBD](http://2019.poleval.pl/files/poleval2019.pdf) | ['pol'] | Classification | s2s | [Written, Social] | {'test': 1000} | {'test': 93.2} | +| [BSARDRetrieval](https://huggingface.co/datasets/maastrichtlawtech/bsard) (Louis et al., 2022) | ['fra'] | Retrieval | s2p | [Legal, Spoken] | None | None | +| [BUCC.v2](https://comparable.limsi.fr/bucc2018/bucc2018-task.html) | ['cmn', 'deu', 'eng', 'fra', 'rus'] | BitextMining | s2s | [Written] | {'test': 35000} | {'test': {'num_samples': 35000, 'number_of_characters': 6640032, 'unique_pairs': 34978, 'min_sentence1_length': 16, 'average_sentence1_length': 99.11, 'max_sentence1_length': 204, 'unique_sentence1': 34978, 'min_sentence2_length': 42, 'average_sentence2_length': 90.61, 'max_sentence2_length': 159, 'unique_sentence2': 25306, 'hf_subset_descriptive_stats': {'de-en': {'num_samples': 9580, 'number_of_characters': 1919197, 'unique_pairs': 9573, 'min_sentence1_length': 50, 'average_sentence1_length': 109.08, 'max_sentence1_length': 204, 'unique_sentence1': 9573, 'min_sentence2_length': 46, 'average_sentence2_length': 91.25, 'max_sentence2_length': 155, 'unique_sentence2': 9570}, 'fr-en': {'num_samples': 9086, 'number_of_characters': 1677545, 'unique_pairs': 9081, 'min_sentence1_length': 43, 'average_sentence1_length': 99.32, 'max_sentence1_length': 174, 'unique_sentence1': 9081, 'min_sentence2_length': 42, 'average_sentence2_length': 85.31, 'max_sentence2_length': 159, 'unique_sentence2': 9076}, 'ru-en': {'num_samples': 14435, 'number_of_characters': 2808206, 'unique_pairs': 14425, 'min_sentence1_length': 40, 'average_sentence1_length': 101.66, 'max_sentence1_length': 186, 'unique_sentence1': 14425, 'min_sentence2_length': 45, 'average_sentence2_length': 92.88, 'max_sentence2_length': 159, 'unique_sentence2': 14424}, 'zh-en': {'num_samples': 1899, 'number_of_characters': 235084, 'unique_pairs': 1899, 'min_sentence1_length': 16, 'average_sentence1_length': 28.43, 'max_sentence1_length': 40, 'unique_sentence1': 1899, 'min_sentence2_length': 48, 'average_sentence2_length': 95.36, 'max_sentence2_length': 159, 'unique_sentence2': 1899}}}} | +| [Banking77Classification](https://arxiv.org/abs/2003.04807) | ['eng'] | Classification | s2s | [Written] | None | None | +| [BelebeleRetrieval](https://arxiv.org/abs/2308.16884) (Lucas Bandarkar, 2023) | ['acm', 'afr', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'azj', 'bam', 'ben', 'bod', 'bul', 'cat', 'ceb', 'ces', 'ckb', 'dan', 'deu', 'ell', 'eng', 'est', 'eus', 'fin', 'fra', 'fuv', 'gaz', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kac', 'kan', 'kat', 'kaz', 'kea', 'khk', 'khm', 'kin', 'kir', 'kor', 'lao', 'lin', 'lit', 'lug', 'luo', 'lvs', 'mal', 'mar', 'mkd', 'mlt', 'mri', 'mya', 'nld', 'nob', 'npi', 'nso', 'nya', 'ory', 'pan', 'pbt', 'pes', 'plt', 'pol', 'por', 'ron', 'rus', 'shn', 'sin', 'slk', 'slv', 'sna', 'snd', 'som', 'sot', 'spa', 'srp', 'ssw', 'sun', 'swe', 'swh', 'tam', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tsn', 'tso', 'tur', 'ukr', 'urd', 'uzn', 'vie', 'war', 'wol', 'xho', 'yor', 'zho', 'zsm', 'zul'] | Retrieval | s2p | [Web, News, Written] | {'test': 521866} | {'test': {'number_of_characters': 25574620, 'num_samples': 521866, 'num_queries': 338378, 'num_documents': 183488, 'min_document_length': 4, 'average_document_length': 137.38, 'max_document_length': 237, 'unique_documents': 183488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 338378, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 183488, 'hf_subset_descriptive_stats': {'acm_Arab-acm_Arab': {'number_of_characters': 51232, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 102.98, 'max_document_length': 129, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'acm_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-acm_Arab': {'number_of_characters': 51232, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 102.98, 'max_document_length': 129, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'afr_Latn-afr_Latn': {'number_of_characters': 71217, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 143.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'afr_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-afr_Latn': {'number_of_characters': 71217, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 143.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'als_Latn-als_Latn': {'number_of_characters': 69498, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 140.41, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'als_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-als_Latn': {'number_of_characters': 69498, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 140.41, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'amh_Ethi-amh_Ethi': {'number_of_characters': 45221, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 90.67, 'max_document_length': 100, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'amh_Ethi-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-amh_Ethi': {'number_of_characters': 45221, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 90.67, 'max_document_length': 100, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'apc_Arab-apc_Arab': {'number_of_characters': 51248, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 103.02, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'apc_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-apc_Arab': {'number_of_characters': 51248, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 103.02, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Arab-arb_Arab': {'number_of_characters': 53671, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 107.98, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-arb_Arab': {'number_of_characters': 53671, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 107.98, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Latn-arb_Latn': {'number_of_characters': 61298, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 123.61, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-arb_Latn': {'number_of_characters': 61298, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 123.61, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ars_Arab-ars_Arab': {'number_of_characters': 51765, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 104.08, 'max_document_length': 119, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ars_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ars_Arab': {'number_of_characters': 51765, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 104.08, 'max_document_length': 119, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ary_Arab-ary_Arab': {'number_of_characters': 60261, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 121.49, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ary_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ary_Arab': {'number_of_characters': 60261, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 121.49, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arz_Arab-arz_Arab': {'number_of_characters': 52403, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 105.38, 'max_document_length': 115, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arz_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-arz_Arab': {'number_of_characters': 52403, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 105.38, 'max_document_length': 115, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'asm_Beng-asm_Beng': {'number_of_characters': 62410, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 4, 'average_document_length': 125.89, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'asm_Beng-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-asm_Beng': {'number_of_characters': 62410, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 4, 'average_document_length': 125.89, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'azj_Latn-azj_Latn': {'number_of_characters': 67137, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.58, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'azj_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-azj_Latn': {'number_of_characters': 67137, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.58, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bam_Latn-bam_Latn': {'number_of_characters': 66084, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 133.42, 'max_document_length': 166, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bam_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-bam_Latn': {'number_of_characters': 66084, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 133.42, 'max_document_length': 166, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Beng-ben_Beng': {'number_of_characters': 63512, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 128.15, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Beng-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ben_Beng': {'number_of_characters': 63512, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 128.15, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Latn-ben_Latn': {'number_of_characters': 68285, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 137.93, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ben_Latn': {'number_of_characters': 68285, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 137.93, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bod_Tibt-bod_Tibt': {'number_of_characters': 79188, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.27, 'max_document_length': 213, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bod_Tibt-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-bod_Tibt': {'number_of_characters': 79188, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.27, 'max_document_length': 213, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bul_Cyrl-bul_Cyrl': {'number_of_characters': 66577, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.43, 'max_document_length': 177, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bul_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-bul_Cyrl': {'number_of_characters': 66577, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.43, 'max_document_length': 177, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'cat_Latn-cat_Latn': {'number_of_characters': 68842, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.07, 'max_document_length': 163, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'cat_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-cat_Latn': {'number_of_characters': 68842, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.07, 'max_document_length': 163, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ceb_Latn-ceb_Latn': {'number_of_characters': 74053, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 149.75, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ceb_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ceb_Latn': {'number_of_characters': 74053, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 149.75, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ces_Latn-ces_Latn': {'number_of_characters': 61936, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 124.92, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ces_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ces_Latn': {'number_of_characters': 61936, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 124.92, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ckb_Arab-ckb_Arab': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 131.03, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ckb_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ckb_Arab': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 131.03, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'dan_Latn-dan_Latn': {'number_of_characters': 66648, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.57, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'dan_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-dan_Latn': {'number_of_characters': 66648, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.57, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'deu_Latn-deu_Latn': {'number_of_characters': 68768, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 138.92, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'deu_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-deu_Latn': {'number_of_characters': 68768, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 138.92, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ell_Grek-ell_Grek': {'number_of_characters': 79210, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.32, 'max_document_length': 212, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ell_Grek-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ell_Grek': {'number_of_characters': 79210, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.32, 'max_document_length': 212, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'est_Latn-est_Latn': {'number_of_characters': 61779, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.6, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'est_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-est_Latn': {'number_of_characters': 61779, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.6, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eus_Latn-eus_Latn': {'number_of_characters': 67979, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 137.3, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eus_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-eus_Latn': {'number_of_characters': 67979, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 137.3, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fin_Latn-fin_Latn': {'number_of_characters': 66234, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-fin_Latn': {'number_of_characters': 66234, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fra_Latn-fra_Latn': {'number_of_characters': 82464, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 166.98, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fra_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-fra_Latn': {'number_of_characters': 82464, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 166.98, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fuv_Latn-fuv_Latn': {'number_of_characters': 53555, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 107.74, 'max_document_length': 122, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fuv_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-fuv_Latn': {'number_of_characters': 53555, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 107.74, 'max_document_length': 122, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'gaz_Latn-gaz_Latn': {'number_of_characters': 78315, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 158.48, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'gaz_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-gaz_Latn': {'number_of_characters': 78315, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 158.48, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'grn_Latn-grn_Latn': {'number_of_characters': 68572, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 138.52, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'grn_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-grn_Latn': {'number_of_characters': 68572, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 138.52, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'guj_Gujr-guj_Gujr': {'number_of_characters': 57007, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 114.82, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'guj_Gujr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-guj_Gujr': {'number_of_characters': 57007, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 114.82, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hat_Latn-hat_Latn': {'number_of_characters': 64558, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.29, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hat_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hat_Latn': {'number_of_characters': 64558, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.29, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hau_Latn-hau_Latn': {'number_of_characters': 78240, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.33, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hau_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hau_Latn': {'number_of_characters': 78240, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.33, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'heb_Hebr-heb_Hebr': {'number_of_characters': 50598, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 101.68, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'heb_Hebr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-heb_Hebr': {'number_of_characters': 50598, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 101.68, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Deva-hin_Deva': {'number_of_characters': 66332, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.93, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Deva-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hin_Deva': {'number_of_characters': 66332, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.93, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Latn-hin_Latn': {'number_of_characters': 68307, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.97, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hin_Latn': {'number_of_characters': 68307, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.97, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hrv_Latn-hrv_Latn': {'number_of_characters': 62928, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.95, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hrv_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hrv_Latn': {'number_of_characters': 62928, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.95, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hun_Latn-hun_Latn': {'number_of_characters': 67941, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 137.22, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hun_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hun_Latn': {'number_of_characters': 67941, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 137.22, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hye_Armn-hye_Armn': {'number_of_characters': 68859, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.1, 'max_document_length': 193, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hye_Armn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hye_Armn': {'number_of_characters': 68859, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.1, 'max_document_length': 193, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ibo_Latn-ibo_Latn': {'number_of_characters': 66167, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 133.59, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'ibo_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ibo_Latn': {'number_of_characters': 66167, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 133.59, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'ilo_Latn-ilo_Latn': {'number_of_characters': 78161, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.17, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ilo_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ilo_Latn': {'number_of_characters': 78161, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.17, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ind_Latn-ind_Latn': {'number_of_characters': 74871, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 151.42, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ind_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ind_Latn': {'number_of_characters': 74871, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 151.42, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'isl_Latn-isl_Latn': {'number_of_characters': 70522, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 142.51, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'isl_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-isl_Latn': {'number_of_characters': 70522, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 142.51, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ita_Latn-ita_Latn': {'number_of_characters': 76124, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 153.99, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ita_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ita_Latn': {'number_of_characters': 76124, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 153.99, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jav_Latn-jav_Latn': {'number_of_characters': 71722, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 144.97, 'max_document_length': 174, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jav_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-jav_Latn': {'number_of_characters': 71722, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 144.97, 'max_document_length': 174, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jpn_Jpan-jpn_Jpan': {'number_of_characters': 33187, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 66.01, 'max_document_length': 76, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jpn_Jpan-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-jpn_Jpan': {'number_of_characters': 33187, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 66.01, 'max_document_length': 76, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kac_Latn-kac_Latn': {'number_of_characters': 89655, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 181.72, 'max_document_length': 195, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kac_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kac_Latn': {'number_of_characters': 89655, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 181.72, 'max_document_length': 195, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kan_Knda-kan_Knda': {'number_of_characters': 65899, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.04, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kan_Knda-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kan_Knda': {'number_of_characters': 65899, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.04, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kat_Geor-kat_Geor': {'number_of_characters': 68309, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.98, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kat_Geor-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kat_Geor': {'number_of_characters': 68309, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.98, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kaz_Cyrl-kaz_Cyrl': {'number_of_characters': 64657, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.49, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kaz_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kaz_Cyrl': {'number_of_characters': 64657, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.49, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kea_Latn-kea_Latn': {'number_of_characters': 69323, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.06, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kea_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kea_Latn': {'number_of_characters': 69323, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.06, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khk_Cyrl-khk_Cyrl': {'number_of_characters': 66977, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 135.25, 'max_document_length': 162, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khk_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-khk_Cyrl': {'number_of_characters': 66977, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 135.25, 'max_document_length': 162, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khm_Khmr-khm_Khmr': {'number_of_characters': 69150, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 139.7, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khm_Khmr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-khm_Khmr': {'number_of_characters': 69150, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 139.7, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kin_Latn-kin_Latn': {'number_of_characters': 72803, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 147.19, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'kin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kin_Latn': {'number_of_characters': 72803, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 147.19, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'kir_Cyrl-kir_Cyrl': {'number_of_characters': 67957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 137.26, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kir_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kir_Cyrl': {'number_of_characters': 67957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 137.26, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kor_Hang-kor_Hang': {'number_of_characters': 32708, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 65.02, 'max_document_length': 88, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kor_Hang-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kor_Hang': {'number_of_characters': 32708, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 65.02, 'max_document_length': 88, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lao_Laoo-lao_Laoo': {'number_of_characters': 57958, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 116.77, 'max_document_length': 142, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lao_Laoo-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lao_Laoo': {'number_of_characters': 57958, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 116.77, 'max_document_length': 142, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lin_Latn-lin_Latn': {'number_of_characters': 74223, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 150.1, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'lin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lin_Latn': {'number_of_characters': 74223, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 150.1, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'lit_Latn-lit_Latn': {'number_of_characters': 62805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 126.7, 'max_document_length': 167, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lit_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lit_Latn': {'number_of_characters': 62805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 126.7, 'max_document_length': 167, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lug_Latn-lug_Latn': {'number_of_characters': 71566, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 144.65, 'max_document_length': 237, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'lug_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lug_Latn': {'number_of_characters': 71566, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 144.65, 'max_document_length': 237, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'luo_Latn-luo_Latn': {'number_of_characters': 66805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 134.9, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'luo_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-luo_Latn': {'number_of_characters': 66805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 134.9, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lvs_Latn-lvs_Latn': {'number_of_characters': 63957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 129.06, 'max_document_length': 172, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lvs_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lvs_Latn': {'number_of_characters': 63957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 129.06, 'max_document_length': 172, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mal_Mlym-mal_Mlym': {'number_of_characters': 73599, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.82, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mal_Mlym-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mal_Mlym': {'number_of_characters': 73599, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.82, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mar_Deva-mar_Deva': {'number_of_characters': 62671, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 126.42, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'mar_Deva-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mar_Deva': {'number_of_characters': 62671, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 126.42, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'mkd_Cyrl-mkd_Cyrl': {'number_of_characters': 67588, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 136.5, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mkd_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mkd_Cyrl': {'number_of_characters': 67588, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 136.5, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mlt_Latn-mlt_Latn': {'number_of_characters': 68480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 138.33, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mlt_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mlt_Latn': {'number_of_characters': 68480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 138.33, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mri_Latn-mri_Latn': {'number_of_characters': 74519, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 150.7, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mri_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mri_Latn': {'number_of_characters': 74519, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 150.7, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mya_Mymr-mya_Mymr': {'number_of_characters': 81331, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 164.66, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mya_Mymr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mya_Mymr': {'number_of_characters': 81331, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 164.66, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nld_Latn-nld_Latn': {'number_of_characters': 68789, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 138.96, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nld_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nld_Latn': {'number_of_characters': 68789, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 138.96, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nob_Latn-nob_Latn': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 131.03, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nob_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nob_Latn': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 131.03, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Deva-npi_Deva': {'number_of_characters': 61183, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 123.38, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Deva-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-npi_Deva': {'number_of_characters': 61183, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 123.38, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Latn-npi_Latn': {'number_of_characters': 65683, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 132.6, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-npi_Latn': {'number_of_characters': 65683, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 132.6, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nso_Latn-nso_Latn': {'number_of_characters': 79073, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 160.03, 'max_document_length': 235, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nso_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nso_Latn': {'number_of_characters': 79073, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 160.03, 'max_document_length': 235, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nya_Latn-nya_Latn': {'number_of_characters': 82685, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.44, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nya_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nya_Latn': {'number_of_characters': 82685, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.44, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ory_Orya-ory_Orya': {'number_of_characters': 66638, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 10, 'average_document_length': 134.55, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ory_Orya-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ory_Orya': {'number_of_characters': 66638, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 10, 'average_document_length': 134.55, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pan_Guru-pan_Guru': {'number_of_characters': 66944, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.18, 'max_document_length': 157, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pan_Guru-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pan_Guru': {'number_of_characters': 66944, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.18, 'max_document_length': 157, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pbt_Arab-pbt_Arab': {'number_of_characters': 61880, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 124.8, 'max_document_length': 155, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pbt_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pbt_Arab': {'number_of_characters': 61880, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 124.8, 'max_document_length': 155, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pes_Arab-pes_Arab': {'number_of_characters': 59252, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 119.42, 'max_document_length': 152, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pes_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pes_Arab': {'number_of_characters': 59252, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 119.42, 'max_document_length': 152, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'plt_Latn-plt_Latn': {'number_of_characters': 86472, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 175.2, 'max_document_length': 222, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'plt_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-plt_Latn': {'number_of_characters': 86472, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 175.2, 'max_document_length': 222, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pol_Latn-pol_Latn': {'number_of_characters': 67664, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 136.66, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pol_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pol_Latn': {'number_of_characters': 67664, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 136.66, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'por_Latn-por_Latn': {'number_of_characters': 71281, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.07, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'por_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-por_Latn': {'number_of_characters': 71281, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.07, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ron_Latn-ron_Latn': {'number_of_characters': 71844, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 145.22, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ron_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ron_Latn': {'number_of_characters': 71844, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 145.22, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'rus_Cyrl-rus_Cyrl': {'number_of_characters': 75823, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 153.38, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'rus_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-rus_Cyrl': {'number_of_characters': 75823, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 153.38, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'shn_Mymr-shn_Mymr': {'number_of_characters': 69288, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 139.98, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'shn_Mymr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-shn_Mymr': {'number_of_characters': 69288, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 139.98, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Latn-sin_Latn': {'number_of_characters': 85996, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 174.22, 'max_document_length': 224, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sin_Latn': {'number_of_characters': 85996, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 174.22, 'max_document_length': 224, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Sinh-sin_Sinh': {'number_of_characters': 63902, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 128.95, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Sinh-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sin_Sinh': {'number_of_characters': 63902, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 128.95, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slk_Latn-slk_Latn': {'number_of_characters': 62663, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 126.41, 'max_document_length': 146, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slk_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-slk_Latn': {'number_of_characters': 62663, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 126.41, 'max_document_length': 146, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slv_Latn-slv_Latn': {'number_of_characters': 62895, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.88, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slv_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-slv_Latn': {'number_of_characters': 62895, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.88, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sna_Latn-sna_Latn': {'number_of_characters': 74071, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.78, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sna_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sna_Latn': {'number_of_characters': 74071, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.78, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'snd_Arab-snd_Arab': {'number_of_characters': 58057, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 116.97, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'snd_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-snd_Arab': {'number_of_characters': 58057, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 116.97, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'som_Latn-som_Latn': {'number_of_characters': 82838, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.75, 'max_document_length': 201, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'som_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-som_Latn': {'number_of_characters': 82838, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.75, 'max_document_length': 201, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sot_Latn-sot_Latn': {'number_of_characters': 75794, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 153.32, 'max_document_length': 186, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sot_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sot_Latn': {'number_of_characters': 75794, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 153.32, 'max_document_length': 186, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'spa_Latn-spa_Latn': {'number_of_characters': 74920, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 151.52, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'spa_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-spa_Latn': {'number_of_characters': 74920, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 151.52, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'srp_Cyrl-srp_Cyrl': {'number_of_characters': 61657, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.35, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'srp_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-srp_Cyrl': {'number_of_characters': 61657, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.35, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'ssw_Latn-ssw_Latn': {'number_of_characters': 73964, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 149.57, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ssw_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ssw_Latn': {'number_of_characters': 73964, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 149.57, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sun_Latn-sun_Latn': {'number_of_characters': 71320, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 144.15, 'max_document_length': 173, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sun_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sun_Latn': {'number_of_characters': 71320, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 144.15, 'max_document_length': 173, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swe_Latn-swe_Latn': {'number_of_characters': 62785, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 126.66, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swe_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-swe_Latn': {'number_of_characters': 62785, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 126.66, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swh_Latn-swh_Latn': {'number_of_characters': 73480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.57, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swh_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-swh_Latn': {'number_of_characters': 73480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.57, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tam_Taml-tam_Taml': {'number_of_characters': 73991, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.62, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tam_Taml-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tam_Taml': {'number_of_characters': 73991, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.62, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tel_Telu-tel_Telu': {'number_of_characters': 65945, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 133.13, 'max_document_length': 149, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tel_Telu-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tel_Telu': {'number_of_characters': 65945, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 133.13, 'max_document_length': 149, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgk_Cyrl-tgk_Cyrl': {'number_of_characters': 67829, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 136.99, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgk_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tgk_Cyrl': {'number_of_characters': 67829, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 136.99, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgl_Latn-tgl_Latn': {'number_of_characters': 75087, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 151.87, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgl_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tgl_Latn': {'number_of_characters': 75087, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 151.87, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tha_Thai-tha_Thai': {'number_of_characters': 54496, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 109.67, 'max_document_length': 123, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tha_Thai-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tha_Thai': {'number_of_characters': 54496, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 109.67, 'max_document_length': 123, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tir_Ethi-tir_Ethi': {'number_of_characters': 47775, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 95.9, 'max_document_length': 110, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tir_Ethi-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tir_Ethi': {'number_of_characters': 47775, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 95.9, 'max_document_length': 110, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tsn_Latn-tsn_Latn': {'number_of_characters': 79391, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 160.69, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tsn_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tsn_Latn': {'number_of_characters': 79391, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 160.69, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tso_Latn-tso_Latn': {'number_of_characters': 83501, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 169.11, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tso_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tso_Latn': {'number_of_characters': 83501, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 169.11, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tur_Latn-tur_Latn': {'number_of_characters': 65382, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 131.98, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tur_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tur_Latn': {'number_of_characters': 65382, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 131.98, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ukr_Cyrl-ukr_Cyrl': {'number_of_characters': 65850, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 132.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ukr_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ukr_Cyrl': {'number_of_characters': 65850, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 132.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Arab-urd_Arab': {'number_of_characters': 64450, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 130.07, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-urd_Arab': {'number_of_characters': 64450, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 130.07, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Latn-urd_Latn': {'number_of_characters': 82039, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 166.11, 'max_document_length': 230, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-urd_Latn': {'number_of_characters': 82039, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 166.11, 'max_document_length': 230, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'uzn_Latn-uzn_Latn': {'number_of_characters': 70828, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 143.14, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'uzn_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-uzn_Latn': {'number_of_characters': 70828, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 143.14, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'vie_Latn-vie_Latn': {'number_of_characters': 66724, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 134.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'vie_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-vie_Latn': {'number_of_characters': 66724, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 134.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'war_Latn-war_Latn': {'number_of_characters': 78444, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 158.75, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'war_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-war_Latn': {'number_of_characters': 78444, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 158.75, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'wol_Latn-wol_Latn': {'number_of_characters': 64521, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 130.22, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'wol_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-wol_Latn': {'number_of_characters': 64521, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 130.22, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'xho_Latn-xho_Latn': {'number_of_characters': 71629, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.78, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'xho_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-xho_Latn': {'number_of_characters': 71629, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.78, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'yor_Latn-yor_Latn': {'number_of_characters': 62752, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 126.59, 'max_document_length': 143, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'yor_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-yor_Latn': {'number_of_characters': 62752, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 126.59, 'max_document_length': 143, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hans-zho_Hans': {'number_of_characters': 20549, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 40.11, 'max_document_length': 64, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hans-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zho_Hans': {'number_of_characters': 20549, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 40.11, 'max_document_length': 64, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hant-zho_Hant': {'number_of_characters': 19947, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 38.88, 'max_document_length': 45, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hant-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zho_Hant': {'number_of_characters': 19947, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 38.88, 'max_document_length': 45, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zsm_Latn-zsm_Latn': {'number_of_characters': 72008, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 145.56, 'max_document_length': 210, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zsm_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zsm_Latn': {'number_of_characters': 72008, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 145.56, 'max_document_length': 210, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zul_Latn-zul_Latn': {'number_of_characters': 69413, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.24, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zul_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zul_Latn': {'number_of_characters': 69413, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.24, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Arab-arb_Latn': {'number_of_characters': 61298, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 123.61, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Latn-arb_Arab': {'number_of_characters': 53671, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 107.98, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Beng-ben_Latn': {'number_of_characters': 68285, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 137.93, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Latn-ben_Beng': {'number_of_characters': 63512, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 128.15, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Deva-hin_Latn': {'number_of_characters': 68307, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.97, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Latn-hin_Deva': {'number_of_characters': 66332, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.93, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Deva-npi_Latn': {'number_of_characters': 65683, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 132.6, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Latn-npi_Deva': {'number_of_characters': 61183, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 123.38, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Sinh-sin_Latn': {'number_of_characters': 85996, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 174.22, 'max_document_length': 224, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Latn-sin_Sinh': {'number_of_characters': 63902, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 128.95, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Arab-urd_Latn': {'number_of_characters': 82039, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 166.11, 'max_document_length': 230, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Latn-urd_Arab': {'number_of_characters': 64450, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 130.07, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}}}} | +| [BengaliDocumentClassification](https://aclanthology.org/2023.eacl-main.4) | ['ben'] | Classification | s2s | [News, Written] | None | None | +| [BengaliHateSpeechClassification](https://huggingface.co/datasets/bn_hate_speech) (Karim et al., 2020) | ['ben'] | Classification | s2s | [News, Written] | None | None | +| [BengaliSentimentAnalysis](https://data.mendeley.com/datasets/p6zc7krs37/4) (Sazzed et al., 2020) | ['ben'] | Classification | s2s | [Reviews, Written] | None | None | +| [BibleNLPBitextMining](https://arxiv.org/abs/2304.09919) (Akerman et al., 2023) | ['aai', 'aak', 'aau', 'aaz', 'abt', 'abx', 'aby', 'acf', 'acr', 'acu', 'adz', 'aer', 'aey', 'agd', 'agg', 'agm', 'agn', 'agr', 'agt', 'agu', 'aia', 'aii', 'aka', 'ake', 'alp', 'alq', 'als', 'aly', 'ame', 'amf', 'amk', 'amm', 'amn', 'amo', 'amp', 'amr', 'amu', 'amx', 'anh', 'anv', 'aoi', 'aoj', 'aom', 'aon', 'apb', 'ape', 'apn', 'apr', 'apu', 'apw', 'apz', 'arb', 'are', 'arl', 'arn', 'arp', 'asm', 'aso', 'ata', 'atb', 'atd', 'atg', 'att', 'auc', 'aui', 'auy', 'avt', 'awb', 'awk', 'awx', 'azb', 'azg', 'azz', 'bao', 'bba', 'bbb', 'bbr', 'bch', 'bco', 'bdd', 'bea', 'bef', 'bel', 'ben', 'beo', 'beu', 'bgs', 'bgt', 'bhg', 'bhl', 'big', 'bjk', 'bjp', 'bjr', 'bjv', 'bjz', 'bkd', 'bki', 'bkq', 'bkx', 'blw', 'blz', 'bmh', 'bmk', 'bmr', 'bmu', 'bnp', 'boa', 'boj', 'bon', 'box', 'bpr', 'bps', 'bqc', 'bqp', 'bre', 'bsj', 'bsn', 'bsp', 'bss', 'buk', 'bus', 'bvd', 'bvr', 'bxh', 'byr', 'byx', 'bzd', 'bzh', 'bzj', 'caa', 'cab', 'cac', 'caf', 'cak', 'cao', 'cap', 'car', 'cav', 'cax', 'cbc', 'cbi', 'cbk', 'cbr', 'cbs', 'cbt', 'cbu', 'cbv', 'cco', 'ceb', 'cek', 'ces', 'cgc', 'cha', 'chd', 'chf', 'chk', 'chq', 'chz', 'cjo', 'cjv', 'ckb', 'cle', 'clu', 'cme', 'cmn', 'cni', 'cnl', 'cnt', 'cof', 'con', 'cop', 'cot', 'cpa', 'cpb', 'cpc', 'cpu', 'cpy', 'crn', 'crx', 'cso', 'csy', 'cta', 'cth', 'ctp', 'ctu', 'cub', 'cuc', 'cui', 'cuk', 'cut', 'cux', 'cwe', 'cya', 'daa', 'dad', 'dah', 'dan', 'ded', 'deu', 'dgc', 'dgr', 'dgz', 'dhg', 'dif', 'dik', 'dji', 'djk', 'djr', 'dob', 'dop', 'dov', 'dwr', 'dww', 'dwy', 'ebk', 'eko', 'emi', 'emp', 'eng', 'enq', 'epo', 'eri', 'ese', 'esk', 'etr', 'ewe', 'faa', 'fai', 'far', 'ffm', 'for', 'fra', 'fue', 'fuf', 'fuh', 'gah', 'gai', 'gam', 'gaw', 'gdn', 'gdr', 'geb', 'gfk', 'ghs', 'glk', 'gmv', 'gng', 'gnn', 'gnw', 'gof', 'grc', 'gub', 'guh', 'gui', 'guj', 'gul', 'gum', 'gun', 'guo', 'gup', 'gux', 'gvc', 'gvf', 'gvn', 'gvs', 'gwi', 'gym', 'gyr', 'hat', 'hau', 'haw', 'hbo', 'hch', 'heb', 'heg', 'hin', 'hix', 'hla', 'hlt', 'hmo', 'hns', 'hop', 'hot', 'hrv', 'hto', 'hub', 'hui', 'hun', 'hus', 'huu', 'huv', 'hvn', 'ian', 'ign', 'ikk', 'ikw', 'ilo', 'imo', 'inb', 'ind', 'ino', 'iou', 'ipi', 'isn', 'ita', 'iws', 'ixl', 'jac', 'jae', 'jao', 'jic', 'jid', 'jiv', 'jni', 'jpn', 'jvn', 'kan', 'kaq', 'kbc', 'kbh', 'kbm', 'kbq', 'kdc', 'kde', 'kdl', 'kek', 'ken', 'kew', 'kgf', 'kgk', 'kgp', 'khs', 'khz', 'kik', 'kiw', 'kiz', 'kje', 'kjs', 'kkc', 'kkl', 'klt', 'klv', 'kmg', 'kmh', 'kmk', 'kmo', 'kms', 'kmu', 'kne', 'knf', 'knj', 'knv', 'kos', 'kpf', 'kpg', 'kpj', 'kpr', 'kpw', 'kpx', 'kqa', 'kqc', 'kqf', 'kql', 'kqw', 'ksd', 'ksj', 'ksr', 'ktm', 'kto', 'kud', 'kue', 'kup', 'kvg', 'kvn', 'kwd', 'kwf', 'kwi', 'kwj', 'kyc', 'kyf', 'kyg', 'kyq', 'kyz', 'kze', 'lac', 'lat', 'lbb', 'lbk', 'lcm', 'leu', 'lex', 'lgl', 'lid', 'lif', 'lin', 'lit', 'llg', 'lug', 'luo', 'lww', 'maa', 'maj', 'mal', 'mam', 'maq', 'mar', 'mau', 'mav', 'maz', 'mbb', 'mbc', 'mbh', 'mbj', 'mbl', 'mbs', 'mbt', 'mca', 'mcb', 'mcd', 'mcf', 'mco', 'mcp', 'mcq', 'mcr', 'mdy', 'med', 'mee', 'mek', 'meq', 'met', 'meu', 'mgc', 'mgh', 'mgw', 'mhl', 'mib', 'mic', 'mie', 'mig', 'mih', 'mil', 'mio', 'mir', 'mit', 'miz', 'mjc', 'mkj', 'mkl', 'mkn', 'mks', 'mle', 'mlh', 'mlp', 'mmo', 'mmx', 'mna', 'mop', 'mox', 'mph', 'mpj', 'mpm', 'mpp', 'mps', 'mpt', 'mpx', 'mqb', 'mqj', 'msb', 'msc', 'msk', 'msm', 'msy', 'mti', 'mto', 'mux', 'muy', 'mva', 'mvn', 'mwc', 'mwe', 'mwf', 'mwp', 'mxb', 'mxp', 'mxq', 'mxt', 'mya', 'myk', 'myu', 'myw', 'myy', 'mzz', 'nab', 'naf', 'nak', 'nas', 'nbq', 'nca', 'nch', 'ncj', 'ncl', 'ncu', 'ndg', 'ndj', 'nfa', 'ngp', 'ngu', 'nhe', 'nhg', 'nhi', 'nho', 'nhr', 'nhu', 'nhw', 'nhy', 'nif', 'nii', 'nin', 'nko', 'nld', 'nlg', 'nna', 'nnq', 'noa', 'nop', 'not', 'nou', 'npi', 'npl', 'nsn', 'nss', 'ntj', 'ntp', 'ntu', 'nuy', 'nvm', 'nwi', 'nya', 'nys', 'nyu', 'obo', 'okv', 'omw', 'ong', 'ons', 'ood', 'opm', 'ory', 'ote', 'otm', 'otn', 'otq', 'ots', 'pab', 'pad', 'pah', 'pan', 'pao', 'pes', 'pib', 'pio', 'pir', 'piu', 'pjt', 'pls', 'plu', 'pma', 'poe', 'poh', 'poi', 'pol', 'pon', 'por', 'poy', 'ppo', 'prf', 'pri', 'ptp', 'ptu', 'pwg', 'qub', 'quc', 'quf', 'quh', 'qul', 'qup', 'qvc', 'qve', 'qvh', 'qvm', 'qvn', 'qvs', 'qvw', 'qvz', 'qwh', 'qxh', 'qxn', 'qxo', 'rai', 'reg', 'rgu', 'rkb', 'rmc', 'rmy', 'ron', 'roo', 'rop', 'row', 'rro', 'ruf', 'rug', 'rus', 'rwo', 'sab', 'san', 'sbe', 'sbk', 'sbs', 'seh', 'sey', 'sgb', 'sgz', 'shj', 'shp', 'sim', 'sja', 'sll', 'smk', 'snc', 'snn', 'snp', 'snx', 'sny', 'som', 'soq', 'soy', 'spa', 'spl', 'spm', 'spp', 'sps', 'spy', 'sri', 'srm', 'srn', 'srp', 'srq', 'ssd', 'ssg', 'ssx', 'stp', 'sua', 'sue', 'sus', 'suz', 'swe', 'swh', 'swp', 'sxb', 'tac', 'taj', 'tam', 'tav', 'taw', 'tbc', 'tbf', 'tbg', 'tbo', 'tbz', 'tca', 'tcs', 'tcz', 'tdt', 'tee', 'tel', 'ter', 'tet', 'tew', 'tfr', 'tgk', 'tgl', 'tgo', 'tgp', 'tha', 'tif', 'tim', 'tiw', 'tiy', 'tke', 'tku', 'tlf', 'tmd', 'tna', 'tnc', 'tnk', 'tnn', 'tnp', 'toc', 'tod', 'tof', 'toj', 'ton', 'too', 'top', 'tos', 'tpa', 'tpi', 'tpt', 'tpz', 'trc', 'tsw', 'ttc', 'tte', 'tuc', 'tue', 'tuf', 'tuo', 'tur', 'tvk', 'twi', 'txq', 'txu', 'tzj', 'tzo', 'ubr', 'ubu', 'udu', 'uig', 'ukr', 'uli', 'ulk', 'upv', 'ura', 'urb', 'urd', 'uri', 'urt', 'urw', 'usa', 'usp', 'uvh', 'uvl', 'vid', 'vie', 'viv', 'vmy', 'waj', 'wal', 'wap', 'wat', 'wbi', 'wbp', 'wed', 'wer', 'wim', 'wiu', 'wiv', 'wmt', 'wmw', 'wnc', 'wnu', 'wol', 'wos', 'wrk', 'wro', 'wrs', 'wsk', 'wuv', 'xav', 'xbi', 'xed', 'xla', 'xnn', 'xon', 'xsi', 'xtd', 'xtm', 'yaa', 'yad', 'yal', 'yap', 'yaq', 'yby', 'ycn', 'yka', 'yle', 'yml', 'yon', 'yor', 'yrb', 'yre', 'yss', 'yuj', 'yut', 'yuw', 'yva', 'zaa', 'zab', 'zac', 'zad', 'zai', 'zaj', 'zam', 'zao', 'zap', 'zar', 'zas', 'zat', 'zav', 'zaw', 'zca', 'zga', 'zia', 'ziw', 'zlm', 'zos', 'zpc', 'zpl', 'zpm', 'zpo', 'zpq', 'zpu', 'zpv', 'zpz', 'zsr', 'ztq', 'zty', 'zyp'] | BitextMining | s2s | [Religious, Written] | None | None | +| [BigPatentClustering.v2](https://huggingface.co/datasets/NortheasternUniversity/big_patent) (Eva Sharma and Chen Li and Lu Wang, 2019) | ['eng'] | Clustering | p2p | [Legal, Written] | None | None | +| [BiorxivClusteringP2P.v2](https://api.biorxiv.org/) | ['eng'] | Clustering | p2p | [Academic, Written] | None | None | +| [BiorxivClusteringS2S.v2](https://api.biorxiv.org/) | ['eng'] | Clustering | s2s | [Academic, Written] | None | None | +| [BlurbsClusteringP2P.v2](https://www.inf.uni-hamburg.de/en/inst/ab/lt/resources/data/germeval-2019-hmc.html) (Steffen Remus, 2019) | ['deu'] | Clustering | p2p | [Fiction, Written] | None | None | +| [BlurbsClusteringS2S.v2](https://www.inf.uni-hamburg.de/en/inst/ab/lt/resources/data/germeval-2019-hmc.html) (Steffen Remus, 2019) | ['deu'] | Clustering | s2s | [Fiction, Written] | None | None | +| [BornholmBitextMining](https://aclanthology.org/W19-6138/) | ['dan'] | BitextMining | s2s | [Web, Social, Fiction, Written] | {'test': 500} | {'test': {'num_samples': 500, 'number_of_characters': 44361, 'unique_pairs': 500, 'min_sentence1_length': 1, 'average_sentence1_length': 49.83, 'max_sentence1_length': 555, 'unique_sentence1': 497, 'min_sentence2_length': 5, 'average_sentence2_length': 38.89, 'max_sentence2_length': 453, 'unique_sentence2': 491}} | +| [BrazilianToxicTweetsClassification](https://paperswithcode.com/dataset/told-br) (Joao Augusto Leite and Diego F. Silva and Kalina Bontcheva and Carolina Scarton, 2020) | ['por'] | MultilabelClassification | s2s | [Constructed, Written] | None | None | +| [BrightRetrieval](https://huggingface.co/datasets/xlangai/BRIGHT) (Hongjin Su, 2024) | ['eng'] | Retrieval | s2p | [Non-fiction] | None | None | +| [BulgarianStoreReviewSentimentClassfication](https://doi.org/10.7910/DVN/TXIK9P) (Georgieva-Trifonova et al., 2018) | ['bul'] | Classification | s2s | [Reviews, Written] | None | None | +| [CBD](http://2019.poleval.pl/files/poleval2019.pdf) | ['pol'] | Classification | s2s | [Written, Social] | None | None | | [CDSC-E](https://aclanthology.org/P17-1073.pdf) | ['pol'] | PairClassification | s2s | [Written] | None | None | +<<<<<<< HEAD | [CDSC-R](https://aclanthology.org/P17-1073.pdf) | ['pol'] | STS | s2s | [Web, Written] | {'test': 1000} | {'test': 75.24} | | [CEDRClassification](https://www.sciencedirect.com/science/article/pii/S1877050921013247) (Sboev et al., 2021) | ['rus'] | MultilabelClassification | s2s | [Web, Social, Blog, Written] | {'test': 1882} | {'test': {'average_text_length': 91.20563230605738, 'average_label_per_text': 0.620616365568544, 'num_samples': 1882, 'unique_labels': 6, 'labels': {'null': {'count': 734}, '3': {'count': 141}, '2': {'count': 170}, '1': {'count': 379}, '0': {'count': 353}, '4': {'count': 125}}}} | | [CLSClusteringP2P.v2](https://arxiv.org/abs/2209.05034) (Yudong Li, 2022) | ['cmn'] | Clustering | p2p | [Academic, Written] | {'test': 2048} | {} | @@ -229,28 +257,204 @@ The following tables give you an overview of the tasks in MTEB. | [HotpotQA-PLHardNegatives](https://hotpotqa.github.io/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | {'test': 1000} | {'test': {'average_document_length': 438.3888210025661, 'average_query_length': 95.161, 'num_documents': 212774, 'num_queries': 1000, 'average_relevant_docs_per_query': 2.0}} | | [HotpotQAHardNegatives](https://hotpotqa.github.io/) | ['eng'] | Retrieval | s2p | [Web, Written] | {'test': 1000} | {'test': {'average_document_length': 373.558822095461, 'average_query_length': 92.584, 'num_documents': 225621, 'num_queries': 1000, 'average_relevant_docs_per_query': 2.0}} | | [HunSum2AbstractiveRetrieval](https://arxiv.org/abs/2404.03555) (Botond Barta, 2024) | ['hun'] | Retrieval | s2p | [News, Written] | {'test': 1998} | {'test': {'average_document_length': 2511.0315315315315, 'average_query_length': 201.2112112112112, 'num_documents': 1998, 'num_queries': 1998, 'average_relevant_docs_per_query': 1.0}} | +======= +| [CDSC-R](https://aclanthology.org/P17-1073.pdf) | ['pol'] | STS | s2s | [Web, Written] | None | None | +| [CEDRClassification](https://www.sciencedirect.com/science/article/pii/S1877050921013247) (Sboev et al., 2021) | ['rus'] | MultilabelClassification | s2s | [Web, Social, Blog, Written] | {'test': 1882, 'train': 7528} | {'test': {'num_samples': 1882, 'number_of_characters': 171649, 'number_texts_in_train': 7, 'min_text_length': 6, 'average_text_length': 91.21, 'max_text_length': 220, 'unique_texts': 1875, 'min_labels_per_text': 0, 'average_label_per_text': 0.62, 'max_labels_per_text': 2, 'unique_labels': 6, 'labels': {'None': {'count': 734}, '3': {'count': 141}, '2': {'count': 170}, '1': {'count': 379}, '0': {'count': 353}, '4': {'count': 125}}}, 'train': {'num_samples': 7528, 'number_of_characters': 697322, 'number_texts_in_train': None, 'min_text_length': 5, 'average_text_length': 92.63, 'max_text_length': 280, 'unique_texts': 7500, 'min_labels_per_text': 0, 'average_label_per_text': 0.61, 'max_labels_per_text': 3, 'unique_labels': 6, 'labels': {'None': {'count': 3043}, '2': {'count': 607}, '0': {'count': 1569}, '3': {'count': 589}, '1': {'count': 1417}, '4': {'count': 411}}}} | +| [CLSClusteringP2P.v2](https://arxiv.org/abs/2209.05034) (Yudong Li, 2022) | ['cmn'] | Clustering | p2p | [Academic, Written] | None | None | +| [CLSClusteringS2S.v2](https://arxiv.org/abs/2209.05034) (Yudong Li, 2022) | ['cmn'] | Clustering | s2s | [Academic, Written] | None | None | +| [CMedQAv1-reranking](https://github.com/zhangsheng93/cMedQA) (Zhang et al., 2017) | ['cmn'] | Reranking | s2s | [Medical, Written] | None | None | +| [CMedQAv2-reranking](https://github.com/zhangsheng93/cMedQA2) (S. Zhang, 2018) | ['cmn'] | Reranking | s2s | [Medical, Written] | None | None | +| [COIRCodeSearchNetRetrieval](https://huggingface.co/datasets/code_search_net/) (Husain et al., 2019) | ['go', 'java', 'javascript', 'php', 'python', 'ruby'] | Retrieval | p2p | [Programming, Written] | {'test': 1056326} | {'test': {'number_of_characters': 36843313, 'num_samples': 1056326, 'num_queries': 52561, 'num_documents': 1003765, 'min_document_length': 54, 'average_document_length': 34.71, 'max_document_length': 334374, 'unique_documents': 1003765, 'min_query_length': 2, 'average_query_length': 38.19, 'max_query_length': 2, 'unique_queries': 52561, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 52561, 'hf_subset_descriptive_stats': {'python': {'number_of_characters': 14574651, 'num_samples': 295228, 'num_queries': 14918, 'num_documents': 280310, 'min_document_length': 95, 'average_document_length': 49.99, 'max_document_length': 14008, 'unique_documents': 280310, 'min_query_length': 2, 'average_query_length': 37.58, 'max_query_length': 2, 'unique_queries': 14918, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 14918}, 'javascript': {'number_of_characters': 2587540, 'num_samples': 68145, 'num_queries': 3291, 'num_documents': 64854, 'min_document_length': 87, 'average_document_length': 37.9, 'max_document_length': 334374, 'unique_documents': 64854, 'min_query_length': 2, 'average_query_length': 39.41, 'max_query_length': 2, 'unique_queries': 3291, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 3291}, 'go': {'number_of_characters': 3641108, 'num_samples': 190562, 'num_queries': 8122, 'num_documents': 182440, 'min_document_length': 54, 'average_document_length': 17.96, 'max_document_length': 5280, 'unique_documents': 182440, 'min_query_length': 2, 'average_query_length': 44.92, 'max_query_length': 2, 'unique_queries': 8122, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 8122}, 'ruby': {'number_of_characters': 629446, 'num_samples': 28831, 'num_queries': 1261, 'num_documents': 27570, 'min_document_length': 83, 'average_document_length': 20.83, 'max_document_length': 3992, 'unique_documents': 27570, 'min_query_length': 2, 'average_query_length': 43.73, 'max_query_length': 2, 'unique_queries': 1261, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1261}, 'java': {'number_of_characters': 6791137, 'num_samples': 191821, 'num_queries': 10955, 'num_documents': 180866, 'min_document_length': 77, 'average_document_length': 35.55, 'max_document_length': 7615, 'unique_documents': 180866, 'min_query_length': 2, 'average_query_length': 33.02, 'max_query_length': 2, 'unique_queries': 10955, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 10955}, 'php': {'number_of_characters': 8619431, 'num_samples': 281739, 'num_queries': 14014, 'num_documents': 267725, 'min_document_length': 94, 'average_document_length': 30.2, 'max_document_length': 4904, 'unique_documents': 267725, 'min_query_length': 2, 'average_query_length': 38.21, 'max_query_length': 2, 'unique_queries': 14014, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 14014}}}} | +| [CPUSpeedTask](https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/blob/c8376f967d1294419be1d3eb41217d04cd3a65d3/src/seb/registered_tasks/speed.py#L83-L96) | ['eng'] | Speed | s2s | [Fiction, Written] | None | None | +| [CQADupstackAndroidRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CQADupstackEnglishRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CQADupstackGamingRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CQADupstackGisRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CQADupstackMathematicaRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CQADupstackPhysicsRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CQADupstackProgrammersRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Programming, Written, Non-fiction] | None | None | +| [CQADupstackStatsRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CQADupstackTexRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CQADupstackUnixRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CQADupstackWebmastersRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CQADupstackWordpressRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CSFDCZMovieReviewSentimentClassification](https://arxiv.org/abs/2304.01922) (Michal Štefánik, 2023) | ['ces'] | Classification | s2s | [Reviews, Written] | None | None | +| [CSFDSKMovieReviewSentimentClassification](https://arxiv.org/abs/2304.01922) (Michal Štefánik, 2023) | ['slk'] | Classification | s2s | [Reviews, Written] | None | None | +| [CTKFactsNLI](https://arxiv.org/abs/2201.11115) (Ullrich et al., 2023) | ['ces'] | PairClassification | s2s | [News, Written] | None | None | +| [CUADAffiliateLicenseLicenseeLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADAffiliateLicenseLicensorLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADAntiAssignmentLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADAuditRightsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADCapOnLiabilityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADChangeOfControlLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADCompetitiveRestrictionExceptionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADCovenantNotToSueLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADEffectiveDateLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADExclusivityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADExpirationDateLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADGoverningLawLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADIPOwnershipAssignmentLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADInsuranceLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADIrrevocableOrPerpetualLicenseLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADJointIPOwnershipLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADLicenseGrantLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADLiquidatedDamagesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADMinimumCommitmentLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADMostFavoredNationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADNoSolicitOfCustomersLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADNoSolicitOfEmployeesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADNonCompeteLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADNonDisparagementLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADNonTransferableLicenseLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADNoticePeriodToTerminateRenewalLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADPostTerminationServicesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADPriceRestrictionsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADRenewalTermLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADRevenueProfitSharingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADRofrRofoRofnLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADSourceCodeEscrowLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADTerminationForConvenienceLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADThirdPartyBeneficiaryLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADUncappedLiabilityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADUnlimitedAllYouCanEatLicenseLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADVolumeRestrictionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUADWarrantyDurationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CUREv1](https://huggingface.co/datasets/clinia/CUREv1) | ['eng', 'fra', 'spa'] | Retrieval | s2p | [Medical, Academic, Written] | None | None | +| [CanadaTaxCourtOutcomesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CataloniaTweetClassification](https://aclanthology.org/2020.lrec-1.171/) | ['cat', 'spa'] | Classification | s2s | [Social, Government, Written] | None | None | +| [ClimateFEVER](https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | | None | None | +| [ClimateFEVERHardNegatives](https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | | None | None | +| [CmedqaRetrieval](https://aclanthology.org/2022.emnlp-main.357.pdf) | ['cmn'] | Retrieval | s2p | [Medical, Written] | None | None | +| [Cmnli](https://huggingface.co/datasets/clue/viewer/cmnli) | ['cmn'] | PairClassification | s2s | | None | None | +| [CodeEditSearchRetrieval](https://huggingface.co/datasets/cassanof/CodeEditSearch/viewer) (Niklas Muennighoff, 2023) | ['c', 'c++', 'go', 'java', 'javascript', 'php', 'python', 'ruby', 'rust', 'scala', 'shell', 'swift', 'typescript'] | Retrieval | p2p | [Programming, Written] | {'train': 26000} | {'train': {'number_of_characters': 935841, 'num_samples': 26000, 'num_queries': 13000, 'num_documents': 13000, 'min_document_length': 18, 'average_document_length': 70.99, 'max_document_length': 2532, 'unique_documents': 13000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 13000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 13000, 'hf_subset_descriptive_stats': {'python': {'number_of_characters': 70519, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 21, 'average_document_length': 69.52, 'max_document_length': 1811, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'javascript': {'number_of_characters': 57880, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 18, 'average_document_length': 56.88, 'max_document_length': 601, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'typescript': {'number_of_characters': 61092, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 19, 'average_document_length': 60.09, 'max_document_length': 659, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'go': {'number_of_characters': 71797, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 19, 'average_document_length': 70.8, 'max_document_length': 1529, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'ruby': {'number_of_characters': 67900, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 20, 'average_document_length': 66.9, 'max_document_length': 751, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'java': {'number_of_characters': 63984, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 23, 'average_document_length': 62.98, 'max_document_length': 807, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'php': {'number_of_characters': 62927, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 21, 'average_document_length': 61.93, 'max_document_length': 766, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'c': {'number_of_characters': 98588, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 20, 'average_document_length': 97.59, 'max_document_length': 1672, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'c++': {'number_of_characters': 115480, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 22, 'average_document_length': 114.48, 'max_document_length': 1856, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'rust': {'number_of_characters': 68503, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 19, 'average_document_length': 67.5, 'max_document_length': 2532, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'swift': {'number_of_characters': 58279, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 19, 'average_document_length': 57.28, 'max_document_length': 727, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'scala': {'number_of_characters': 65833, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 22, 'average_document_length': 64.83, 'max_document_length': 685, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'shell': {'number_of_characters': 73059, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 18, 'average_document_length': 72.06, 'max_document_length': 813, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}}}} | +| [CodeFeedbackMT](https://arxiv.org/abs/2402.14658) (Tianyu Zheng, 2024) | ['eng'] | Retrieval | p2p | [Programming, Written] | {'test': 79660} | {'test': {'number_of_characters': 156266302, 'num_samples': 79660, 'num_queries': 13277, 'num_documents': 66383, 'min_document_length': 127, 'average_document_length': 885.13, 'max_document_length': 32432, 'unique_documents': 66383, 'min_query_length': 2, 'average_query_length': 7344.18, 'max_query_length': 9403, 'unique_queries': 13277, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 13277}} | +| [CodeFeedbackST](https://arxiv.org/abs/2407.02883) (Xiangyang Li, 2024) | ['eng'] | Retrieval | p2p | [Programming, Written] | {'test': 187832} | {'test': {'number_of_characters': 260957682, 'num_samples': 187832, 'num_queries': 31306, 'num_documents': 156526, 'min_document_length': 26, 'average_document_length': 144.85, 'max_document_length': 13851, 'unique_documents': 156526, 'min_query_length': 1, 'average_query_length': 7611.46, 'max_query_length': 11354, 'unique_queries': 31306, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 31306}} | +| [CodeSearchNetCCRetrieval](https://arxiv.org/abs/2407.02883) (Xiangyang Li, 2024) | ['go', 'java', 'javascript', 'php', 'python', 'ruby'] | Retrieval | p2p | [Programming, Written] | {'test': 1058035} | {'test': {'number_of_characters': 22407915, 'num_samples': 1058035, 'num_queries': 52561, 'num_documents': 1005474, 'min_document_length': 23, 'average_document_length': 20.29, 'max_document_length': 214210, 'unique_documents': 1005474, 'min_query_length': 2, 'average_query_length': 38.26, 'max_query_length': 2, 'unique_queries': 52561, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 52561, 'hf_subset_descriptive_stats': {'python': {'number_of_characters': 8792958, 'num_samples': 295570, 'num_queries': 14918, 'num_documents': 280652, 'min_document_length': 38, 'average_document_length': 29.33, 'max_document_length': 8326, 'unique_documents': 280652, 'min_query_length': 2, 'average_query_length': 37.63, 'max_query_length': 2, 'unique_queries': 14918, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 14918}, 'javascript': {'number_of_characters': 1590642, 'num_samples': 68492, 'num_queries': 3291, 'num_documents': 65201, 'min_document_length': 40, 'average_document_length': 22.4, 'max_document_length': 214210, 'unique_documents': 65201, 'min_query_length': 2, 'average_query_length': 39.62, 'max_query_length': 2, 'unique_queries': 3291, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 3291}, 'go': {'number_of_characters': 2264134, 'num_samples': 190857, 'num_queries': 8122, 'num_documents': 182735, 'min_document_length': 23, 'average_document_length': 10.39, 'max_document_length': 3589, 'unique_documents': 182735, 'min_query_length': 2, 'average_query_length': 45.0, 'max_query_length': 2, 'unique_queries': 8122, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 8122}, 'ruby': {'number_of_characters': 391703, 'num_samples': 28849, 'num_queries': 1261, 'num_documents': 27588, 'min_document_length': 36, 'average_document_length': 12.2, 'max_document_length': 2244, 'unique_documents': 27588, 'min_query_length': 2, 'average_query_length': 43.76, 'max_query_length': 2, 'unique_queries': 1261, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1261}, 'java': {'number_of_characters': 4114584, 'num_samples': 192016, 'num_queries': 10955, 'num_documents': 181061, 'min_document_length': 38, 'average_document_length': 20.72, 'max_document_length': 5066, 'unique_documents': 181061, 'min_query_length': 2, 'average_query_length': 33.06, 'max_query_length': 2, 'unique_queries': 10955, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 10955}, 'php': {'number_of_characters': 5253894, 'num_samples': 282251, 'num_queries': 14014, 'num_documents': 268237, 'min_document_length': 40, 'average_document_length': 17.59, 'max_document_length': 2995, 'unique_documents': 268237, 'min_query_length': 2, 'average_query_length': 38.28, 'max_query_length': 2, 'unique_queries': 14014, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 14014}}}} | +| [CodeSearchNetRetrieval](https://huggingface.co/datasets/code_search_net/) (Husain et al., 2019) | ['go', 'java', 'javascript', 'php', 'python', 'ruby'] | Retrieval | p2p | [Programming, Written] | {'test': 12000} | {'test': {'number_of_characters': 1950074, 'num_samples': 12000, 'num_queries': 6000, 'num_documents': 6000, 'min_document_length': 2, 'average_document_length': 324.01, 'max_document_length': 17533, 'unique_documents': 6000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 6000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 6000, 'hf_subset_descriptive_stats': {'python': {'number_of_characters': 467546, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 8, 'average_document_length': 466.55, 'max_document_length': 8636, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'javascript': {'number_of_characters': 187018, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 2, 'average_document_length': 186.02, 'max_document_length': 7657, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'go': {'number_of_characters': 126213, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 14, 'average_document_length': 125.21, 'max_document_length': 1501, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'ruby': {'number_of_characters': 314818, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 5, 'average_document_length': 313.82, 'max_document_length': 17533, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'java': {'number_of_characters': 691360, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 2, 'average_document_length': 690.36, 'max_document_length': 6473, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'php': {'number_of_characters': 163119, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 5, 'average_document_length': 162.12, 'max_document_length': 1240, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}}}} | +| [CodeTransOceanContest](https://arxiv.org/abs/2310.04951) (Weixiang Yan, 2023) | ['c++', 'python'] | Retrieval | p2p | [Programming, Written] | {'test': 1229} | {'test': {'number_of_characters': 1744286, 'num_samples': 1229, 'num_queries': 221, 'num_documents': 1008, 'min_document_length': 8, 'average_document_length': 221.9, 'max_document_length': 4147, 'unique_documents': 1008, 'min_query_length': 8, 'average_query_length': 6880.58, 'max_query_length': 10852, 'unique_queries': 221, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 221}} | +| [CodeTransOceanDL](https://arxiv.org/abs/2310.04951) (Weixiang Yan, 2023) | ['python'] | Retrieval | p2p | [Programming, Written] | {'test': 996} | {'test': {'number_of_characters': 1543912, 'num_samples': 996, 'num_queries': 180, 'num_documents': 816, 'min_document_length': 376, 'average_document_length': 411.98, 'max_document_length': 8285, 'unique_documents': 816, 'min_query_length': 58, 'average_query_length': 6709.67, 'max_query_length': 8469, 'unique_queries': 180, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 180}} | +| [ContractNLIConfidentialityOfAgreementLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLIExplicitIdentificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLIInclusionOfVerballyConveyedInformationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLILimitedUseLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLINoLicensingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLINoticeOnCompelledDisclosureLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLIPermissibleAcquirementOfSimilarInformationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLIPermissibleCopyLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLIPermissibleDevelopmentOfSimilarInformationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLIPermissiblePostAgreementPossessionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLIReturnOfConfidentialInformationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLISharingWithEmployeesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLISharingWithThirdPartiesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ContractNLISurvivalOfObligationsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [Core17InstructionRetrieval](https://arxiv.org/abs/2403.15246) (Orion Weller, 2024) | ['eng'] | InstructionRetrieval | s2p | [News, Written] | {'test': 19919} | {'test': {'num_samples': 19919, 'num_docs': 19899, 'num_queries': 20, 'number_of_characters': 44450333, 'min_document_length': 7, 'average_document_length': 2233.03, 'max_document_length': 2959, 'unique_docs': 19143, 'min_query_length': 55, 'average_query_length': 109.75, 'max_query_length': 278, 'unique_queries': 20, 'min_instruction_length': 102, 'average_instruction_length': 295.55, 'max_instruction_length': 811, 'unique_instructions': 20, 'min_changed_instruction_length': 151, 'average_changed_instruction_length': 355.2, 'max_changed_instruction_length': 837, 'unique_changed_instructions': 20, 'min_average_relevant_docs_per_query': 4, 'average_relevant_docs_per_query': 32.7, 'max_average_relevant_docs_per_query': 55, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}} | +| [CorporateLobbyingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [CosQA](https://arxiv.org/abs/2105.13239) (Junjie Huang, 2021) | ['eng', 'python'] | Retrieval | p2p | [Programming, Written] | {'test': 21104} | {'test': {'number_of_characters': 5728450, 'num_samples': 21104, 'num_queries': 500, 'num_documents': 20604, 'min_document_length': 18, 'average_document_length': 0.89, 'max_document_length': 83, 'unique_documents': 20604, 'min_query_length': 88, 'average_query_length': 11420.09, 'max_query_length': 6396, 'unique_queries': 500, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 500}} | +| [CovidRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | None | +| [CrossLingualSemanticDiscriminationWMT19](https://huggingface.co/datasets/Andrianos/clsd_wmt19_21) | ['deu', 'fra'] | Retrieval | s2s | [News, Written] | None | None | +| [CrossLingualSemanticDiscriminationWMT21](https://huggingface.co/datasets/Andrianos/clsd_wmt19_21) | ['deu', 'fra'] | Retrieval | s2s | [News, Written] | None | None | +| [CyrillicTurkicLangClassification](https://huggingface.co/datasets/tatiana-merz/cyrillic_turkic_langs) (Goldhahn et al., 2012) | ['bak', 'chv', 'kaz', 'kir', 'krc', 'rus', 'sah', 'tat', 'tyv'] | Classification | s2s | [Web, Written] | None | None | +| [CzechProductReviewSentimentClassification](https://aclanthology.org/W13-1609/) | ['ces'] | Classification | s2s | [Reviews, Written] | None | None | +| [CzechSoMeSentimentClassification](https://aclanthology.org/W13-1609/) | ['ces'] | Classification | s2s | [Reviews, Written] | None | None | +| [CzechSubjectivityClassification](https://arxiv.org/abs/2009.08712) | ['ces'] | Classification | s2s | [Reviews, Written] | None | None | +| [DBPedia](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['eng'] | Retrieval | s2p | [Written, Encyclopaedic] | None | None | +| [DBPedia-PL](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['pol'] | Retrieval | s2p | [Written, Encyclopaedic] | None | None | +| [DBPedia-PLHardNegatives](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['pol'] | Retrieval | s2p | [Written, Encyclopaedic] | None | None | +| [DBPediaHardNegatives](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['eng'] | Retrieval | s2p | [Written, Encyclopaedic] | None | None | +| [DBpediaClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [Encyclopaedic, Written] | None | None | +| [DKHateClassification](https://aclanthology.org/2020.lrec-1.430/) | ['dan'] | Classification | s2s | [Social, Written] | None | None | +| [DalajClassification](https://spraakbanken.gu.se/en/resources/superlim) | ['swe'] | Classification | s2s | [Non-fiction, Written] | None | None | +| [DanFeverRetrieval](https://aclanthology.org/2021.nodalida-main.47/) | ['dan'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Spoken] | None | None | +| [DanishPoliticalCommentsClassification](https://huggingface.co/datasets/danish_political_comments) (Mads Guldborg Kjeldgaard Kongsbak, 2019) | ['dan'] | Classification | s2s | [Social, Written] | None | None | +| [DefinitionClassificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [DiaBlaBitextMining](https://inria.hal.science/hal-03021633) (González et al., 2019) | ['eng', 'fra'] | BitextMining | s2s | [Social, Written] | None | None | +| [Diversity1LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [Diversity2LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [Diversity3LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [Diversity4LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [Diversity5LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [Diversity6LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [DuRetrieval](https://aclanthology.org/2022.emnlp-main.357.pdf) (Yifu Qiu, 2022) | ['cmn'] | Retrieval | s2p | | None | None | +| [DutchBookReviewSentimentClassification](https://github.com/benjaminvdb/DBRD) (Benjamin et al., 2019) | ['nld'] | Classification | s2s | [Reviews, Written] | None | None | +| [ESCIReranking](https://github.com/amazon-science/esci-data/) (Chandan K. Reddy, 2022) | ['eng', 'jpn', 'spa'] | Reranking | s2p | [Written] | {'test': 29285} | {'test': {'num_samples': 29285, 'number_of_characters': 254538331, 'num_positive': 271416, 'num_negative': 44235, 'min_query_length': 1, 'avg_query_length': 19.69, 'max_query_length': 151, 'unique_query': 29269, 'min_positive_length': 1, 'avg_positive_length': 803.92, 'max_positive_length': 8640, 'unique_positive': 217712, 'min_negative_length': 1, 'avg_negative_length': 808.5, 'max_negative_length': 4441, 'unique_negative': 39551, 'hf_subset_descriptive_stats': {'us': {'num_samples': 21296, 'number_of_characters': 186915609, 'num_positive': 189375, 'num_negative': 25463, 'min_query_length': 1, 'avg_query_length': 21.44, 'max_query_length': 151, 'unique_query': 21296, 'min_positive_length': 1, 'avg_positive_length': 868.37, 'max_positive_length': 5545, 'unique_positive': 150734, 'min_negative_length': 1, 'avg_negative_length': 864.45, 'max_negative_length': 3779, 'unique_negative': 23073}, 'es': {'num_samples': 3703, 'number_of_characters': 48861389, 'num_positive': 39110, 'num_negative': 10183, 'min_query_length': 3, 'avg_query_length': 20.68, 'max_query_length': 59, 'unique_query': 3703, 'min_positive_length': 1, 'avg_positive_length': 980.96, 'max_positive_length': 8640, 'unique_positive': 32921, 'min_negative_length': 1, 'avg_negative_length': 1023.22, 'max_negative_length': 4441, 'unique_negative': 9285}, 'jp': {'num_samples': 4286, 'number_of_characters': 18761333, 'num_positive': 42931, 'num_negative': 8589, 'min_query_length': 1, 'avg_query_length': 10.15, 'max_query_length': 60, 'unique_query': 4286, 'min_positive_length': 1, 'avg_positive_length': 358.36, 'max_positive_length': 3488, 'unique_positive': 35165, 'min_negative_length': 1, 'avg_negative_length': 388.08, 'max_negative_length': 3940, 'unique_negative': 7289}}}} | +| [EcomRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | None | +| [EightTagsClustering.v2](https://aclanthology.org/2020.lrec-1.207.pdf) | ['pol'] | Clustering | s2s | [Social, Written] | None | None | +| [EmotionClassification](https://www.aclweb.org/anthology/D18-1404) | ['eng'] | Classification | s2s | [Social, Written] | None | None | +| [EstQA](https://www.semanticscholar.org/paper/Extractive-Question-Answering-for-Estonian-Language-182912IAPM-Alum%C3%A4e/ea4f60ab36cadca059c880678bc4c51e293a85d6?utm_source=direct_link) | ['est'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [EstonianValenceClassification](https://figshare.com/articles/dataset/Estonian_Valence_Corpus_Eesti_valentsikorpus/24517054) | ['est'] | Classification | s2s | [News, Written] | None | None | +| [FEVER](https://fever.ai/) | ['eng'] | Retrieval | s2p | | None | None | +| [FEVERHardNegatives](https://fever.ai/) | ['eng'] | Retrieval | s2p | | None | None | +| [FQuADRetrieval](https://huggingface.co/datasets/manu/fquad2_test) | ['fra'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [FaithDial](https://mcgill-nlp.github.io/FaithDial) (Dziri et al., 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [FalseFriendsGermanEnglish](https://drive.google.com/file/d/1jgq0nBnV-UiYNxbKNrrr2gxDEHm-DMKH/view?usp=share_link) | ['deu'] | PairClassification | s2s | [Written] | None | None | +| [FaroeseSTS](https://aclanthology.org/2023.nodalida-1.74.pdf) | ['fao'] | STS | s2s | [News, Web, Written] | None | None | +| [FarsTail](https://link.springer.com/article/10.1007/s00500-023-08959-3) (Amirkhani et al., 2023) | ['fas'] | PairClassification | s2s | [Academic, Written] | None | None | +| [FeedbackQARetrieval](https://arxiv.org/abs/2204.03025) | ['eng'] | Retrieval | s2p | [Web, Government, Medical, Written] | None | None | +| [FiQA-PL](https://sites.google.com/view/fiqa/) (Nandan Thakur, 2021) | ['pol'] | Retrieval | s2p | | None | None | +| [FiQA2018](https://sites.google.com/view/fiqa/) (Nandan Thakur, 2021) | ['eng'] | Retrieval | s2p | | None | None | +| [FilipinoHateSpeechClassification](https://pcj.csp.org.ph/index.php/pcj/issue/download/29/PCJ%20V14%20N1%20pp1-14%202019) (Neil Vicente Cabasag et al., 2019) | ['fil'] | Classification | s2s | [Social, Written] | None | None | +| [FilipinoShopeeReviewsClassification](https://uijrt.com/articles/v4/i8/UIJRTV4I80009.pdf) | ['fil'] | Classification | s2s | [Social, Written] | None | None | +| [FinParaSTS](https://huggingface.co/datasets/TurkuNLP/turku_paraphrase_corpus) | ['fin'] | STS | s2s | [News, Subtitles, Written] | None | None | +| [FinToxicityClassification](https://aclanthology.org/2023.nodalida-1.68) | ['fin'] | Classification | s2s | [News, Written] | None | None | +| [FinancialPhrasebankClassification](https://arxiv.org/abs/1307.5336) (P. Malo, 2014) | ['eng'] | Classification | s2s | [News, Written] | None | None | +| [FloresBitextMining](https://huggingface.co/datasets/facebook/flores) (Goyal et al., 2022) | ['ace', 'acm', 'acq', 'aeb', 'afr', 'ajp', 'aka', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'ast', 'awa', 'ayr', 'azb', 'azj', 'bak', 'bam', 'ban', 'bel', 'bem', 'ben', 'bho', 'bjn', 'bod', 'bos', 'bug', 'bul', 'cat', 'ceb', 'ces', 'cjk', 'ckb', 'crh', 'cym', 'dan', 'deu', 'dik', 'dyu', 'dzo', 'ell', 'eng', 'epo', 'est', 'eus', 'ewe', 'fao', 'fij', 'fin', 'fon', 'fra', 'fur', 'fuv', 'gaz', 'gla', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hne', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kac', 'kam', 'kan', 'kas', 'kat', 'kaz', 'kbp', 'kea', 'khk', 'khm', 'kik', 'kin', 'kir', 'kmb', 'kmr', 'knc', 'kon', 'kor', 'lao', 'lij', 'lim', 'lin', 'lit', 'lmo', 'ltg', 'ltz', 'lua', 'lug', 'luo', 'lus', 'lvs', 'mag', 'mai', 'mal', 'mar', 'min', 'mkd', 'mlt', 'mni', 'mos', 'mri', 'mya', 'nld', 'nno', 'nob', 'npi', 'nso', 'nus', 'nya', 'oci', 'ory', 'pag', 'pan', 'pap', 'pbt', 'pes', 'plt', 'pol', 'por', 'prs', 'quy', 'ron', 'run', 'rus', 'sag', 'san', 'sat', 'scn', 'shn', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'ssw', 'sun', 'swe', 'swh', 'szl', 'tam', 'taq', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tpi', 'tsn', 'tso', 'tuk', 'tum', 'tur', 'twi', 'tzm', 'uig', 'ukr', 'umb', 'urd', 'uzn', 'vec', 'vie', 'war', 'wol', 'xho', 'ydd', 'yor', 'yue', 'zho', 'zsm', 'zul'] | BitextMining | s2s | [Non-fiction, Encyclopaedic, Written] | None | None | +| [FrenchBookReviews](https://huggingface.co/datasets/Abirate/french_book_reviews) | ['fra'] | Classification | s2s | [Reviews, Written] | None | None | +| [FrenkEnClassification](https://arxiv.org/abs/1906.02045) (Nikola Ljubešić, 2019) | ['eng'] | Classification | s2s | [Social, Written] | None | None | +| [FrenkHrClassification](https://arxiv.org/abs/1906.02045) (Nikola Ljubešić, 2019) | ['hrv'] | Classification | s2s | [Social, Written] | None | None | +| [FrenkSlClassification](https://arxiv.org/pdf/1906.02045) (Nikola Ljubešić, 2019) | ['slv'] | Classification | s2s | [Social, Written] | None | None | +| [FunctionOfDecisionSectionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [GPUSpeedTask](https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/blob/c8376f967d1294419be1d3eb41217d04cd3a65d3/src/seb/registered_tasks/speed.py#L83-L96) | ['eng'] | Speed | s2s | [Fiction, Written] | None | None | +| [GeoreviewClassification](https://github.com/yandex/geo-reviews-dataset-2023) | ['rus'] | Classification | p2p | [Reviews, Written] | None | None | +| [GeoreviewClusteringP2P](https://github.com/yandex/geo-reviews-dataset-2023) | ['rus'] | Clustering | p2p | [Reviews, Written] | None | None | +| [GeorgianFAQRetrieval](https://huggingface.co/datasets/jupyterjazz/georgian-faq) | ['kat'] | Retrieval | s2p | [Web, Written] | None | None | +| [GerDaLIR](https://github.com/lavis-nlp/GerDaLIR) | ['deu'] | Retrieval | s2p | | None | None | +| [GerDaLIRSmall](https://github.com/lavis-nlp/GerDaLIR) | ['deu'] | Retrieval | p2p | [Legal, Written] | None | None | +| [GermanDPR](https://huggingface.co/datasets/deepset/germandpr) (Timo Möller, 2021) | ['deu'] | Retrieval | s2p | | None | None | +| [GermanGovServiceRetrieval](https://huggingface.co/datasets/it-at-m/LHM-Dienstleistungen-QA) | ['deu'] | Retrieval | s2p | [Government, Written] | None | None | +| [GermanPoliticiansTwitterSentimentClassification](https://aclanthology.org/2022.konvens-1.9) | ['deu'] | Classification | s2s | [Social, Government, Written] | None | None | +| [GermanQuAD-Retrieval](https://www.kaggle.com/datasets/GermanQuAD) (Timo Möller, 2021) | ['deu'] | Retrieval | s2p | | None | None | +| [GermanSTSBenchmark](https://github.com/t-systems-on-site-services-gmbh/german-STSbenchmark) (Philip May, 2021) | ['deu'] | STS | s2s | | None | None | +| [GreekCivicsQA](https://huggingface.co/datasets/antoinelb7/alloprof) | ['ell'] | Retrieval | s2p | [Academic, Written] | None | None | +| [GreekLegalCodeClassification](https://arxiv.org/abs/2109.15298) | ['ell'] | Classification | s2s | [Legal, Written] | None | None | +| [GujaratiNewsClassification](https://github.com/goru001/nlp-for-gujarati) | ['guj'] | Classification | s2s | [News, Written] | None | None | +| [HALClusteringS2S.v2](https://huggingface.co/datasets/lyon-nlp/clustering-hal-s2s) (Mathieu Ciancone, 2024) | ['fra'] | Clustering | s2s | [Academic, Written] | None | None | +| [HagridRetrieval](https://github.com/project-miracl/hagrid) (Ehsan Kamalloo, 2023) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [HateSpeechPortugueseClassification](https://aclanthology.org/W19-3510) | ['por'] | Classification | s2s | [Social, Written] | None | None | +| [HeadlineClassification](https://aclanthology.org/2020.ngt-1.6/) | ['rus'] | Classification | s2s | [News, Written] | None | None | +| [HebrewSentimentAnalysis](https://huggingface.co/datasets/hebrew_sentiment) | ['heb'] | Classification | s2s | [Reviews, Written] | None | None | +| [HellaSwag](https://rowanzellers.com/hellaswag/) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [HinDialectClassification](https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-4839) (Bafna et al., 2022) | ['anp', 'awa', 'ben', 'bgc', 'bhb', 'bhd', 'bho', 'bjj', 'bns', 'bra', 'gbm', 'guj', 'hne', 'kfg', 'kfy', 'mag', 'mar', 'mup', 'noe', 'pan', 'raj'] | Classification | s2s | [Social, Spoken, Written] | None | None | +| [HindiDiscourseClassification](https://aclanthology.org/2020.lrec-1.149/) | ['hin'] | Classification | s2s | [Fiction, Social, Written] | None | None | +| [HotelReviewSentimentClassification](https://link.springer.com/chapter/10.1007/978-3-319-67056-0_3) (Elnagar et al., 2018) | ['ara'] | Classification | s2s | [Reviews, Written] | None | None | +| [HotpotQA](https://hotpotqa.github.io/) | ['eng'] | Retrieval | s2p | [Web, Written] | None | None | +| [HotpotQA-PL](https://hotpotqa.github.io/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | None | +| [HotpotQA-PLHardNegatives](https://hotpotqa.github.io/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | None | +| [HotpotQAHardNegatives](https://hotpotqa.github.io/) | ['eng'] | Retrieval | s2p | [Web, Written] | None | None | +| [HunSum2AbstractiveRetrieval](https://arxiv.org/abs/2404.03555) (Botond Barta, 2024) | ['hun'] | Retrieval | s2p | [News, Written] | None | None | +>>>>>>> main | [IFlyTek](https://www.cluebenchmarks.com/introduce.html) | ['cmn'] | Classification | s2s | | None | None | -| [IN22ConvBitextMining](https://huggingface.co/datasets/ai4bharat/IN22-Conv) (Jay Gala, 2023) | ['asm', 'ben', 'brx', 'doi', 'eng', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Social, Spoken, Fiction, Spoken] | | | -| [IN22GenBitextMining](https://huggingface.co/datasets/ai4bharat/IN22-Gen) (Jay Gala, 2023) | ['asm', 'ben', 'brx', 'doi', 'eng', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Web, Legal, Government, News, Religious, Non-fiction, Written] | {'test': 1024} | {'test': 156.7} | -| [IWSLT2017BitextMining](https://aclanthology.org/2017.iwslt-1.1/) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'ita', 'jpn', 'kor', 'nld', 'ron'] | BitextMining | s2s | [Non-fiction, Fiction, Written] | {'validation': 21928} | {'validation': 95.4} | -| [ImdbClassification](http://www.aclweb.org/anthology/P11-1015) | ['eng'] | Classification | p2p | [Reviews, Written] | {'test': 25000} | {'test': 1293.8} | -| [InappropriatenessClassification](https://aclanthology.org/2021.bsnlp-1.4) | ['rus'] | Classification | s2s | [Web, Social, Written] | {'test': 2048} | {'test': 97.7} | -| [IndicCrosslingualSTS](https://huggingface.co/datasets/jaygala24/indic_sts) (Ramesh et al., 2022) | ['asm', 'ben', 'eng', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel', 'urd'] | STS | s2s | [News, Non-fiction, Web, Spoken, Government, Written, Spoken] | {'test': 10020} | {'test': 76.22} | -| [IndicGenBenchFloresBitextMining](https://github.com/google-research-datasets/indic-gen-bench/) (Harman Singh, 2024) | ['asm', 'awa', 'ben', 'bgc', 'bho', 'bod', 'boy', 'eng', 'gbm', 'gom', 'guj', 'hin', 'hne', 'kan', 'mai', 'mal', 'mar', 'mni', 'mup', 'mwr', 'nep', 'ory', 'pan', 'pus', 'raj', 'san', 'sat', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Web, News, Written] | {'validation': 997, 'test': 1012} | {'validation': 126.25, 'test': 130.84} | -| [IndicLangClassification](https://arxiv.org/abs/2305.15814) | ['asm', 'ben', 'brx', 'doi', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | Classification | s2s | [Web, Non-fiction, Written] | {'test': 30418} | {'test': 106.5} | -| [IndicNLPNewsClassification](https://github.com/AI4Bharat/indicnlp_corpus#indicnlp-news-article-classification-dataset) (Anoop Kunchukuttan, 2020) | ['guj', 'kan', 'mal', 'mar', 'ori', 'pan', 'tam', 'tel'] | Classification | s2s | [News, Written] | {'test': 2048} | {'test': 1169.053974484789} | -| [IndicQARetrieval](https://arxiv.org/abs/2212.05409) (Sumanth Doddapaneni, 2022) | ['asm', 'ben', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel'] | Retrieval | s2p | [Web, Written] | {'test': 18586} | {'test': {'as': {'average_document_length': 1401.28, 'average_query_length': 56.60504201680672, 'num_documents': 250, 'num_queries': 1785, 'average_relevant_docs_per_query': 1.0016806722689076}, 'bn': {'average_document_length': 2196.012, 'average_query_length': 57.069239500567534, 'num_documents': 250, 'num_queries': 1762, 'average_relevant_docs_per_query': 1.0005675368898979}, 'gu': {'average_document_length': 960.4959677419355, 'average_query_length': 60.3712158808933, 'num_documents': 248, 'num_queries': 2015, 'average_relevant_docs_per_query': 1.0009925558312656}, 'hi': {'average_document_length': 2550.770114942529, 'average_query_length': 52.84909326424871, 'num_documents': 261, 'num_queries': 1544, 'average_relevant_docs_per_query': 1.0019430051813472}, 'kn': {'average_document_length': 882.7354085603113, 'average_query_length': 50.58734344100198, 'num_documents': 257, 'num_queries': 1517, 'average_relevant_docs_per_query': 1.0}, 'ml': {'average_document_length': 2522.6437246963565, 'average_query_length': 75.93635790800252, 'num_documents': 247, 'num_queries': 1587, 'average_relevant_docs_per_query': 1.0}, 'mr': {'average_document_length': 1711.74, 'average_query_length': 58.785, 'num_documents': 250, 'num_queries': 1600, 'average_relevant_docs_per_query': 1.0}, 'or': {'average_document_length': 801.9206349206349, 'average_query_length': 55.072792362768496, 'num_documents': 252, 'num_queries': 1676, 'average_relevant_docs_per_query': 1.0011933174224343}, 'pa': {'average_document_length': 1423.5062240663901, 'average_query_length': 58.394925178919976, 'num_documents': 241, 'num_queries': 1537, 'average_relevant_docs_per_query': 1.0013012361743656}, 'ta': {'average_document_length': 2288.2608695652175, 'average_query_length': 54.06211869107044, 'num_documents': 253, 'num_queries': 1803, 'average_relevant_docs_per_query': 1.0005546311702718}, 'te': {'average_document_length': 2936.176, 'average_query_length': 67.00634371395617, 'num_documents': 250, 'num_queries': 1734, 'average_relevant_docs_per_query': 1.0}}} | -| [IndicReviewsClusteringP2P](https://arxiv.org/abs/2212.05409) (Sumanth Doddapaneni, 2022) | ['asm', 'ben', 'brx', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel', 'urd'] | Clustering | p2p | [Reviews, Written] | {'test': 1000} | {'test': 137.6} | -| [IndicSentimentClassification](https://arxiv.org/abs/2212.05409) (Sumanth Doddapaneni, 2022) | ['asm', 'ben', 'brx', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel', 'urd'] | Classification | s2s | [Reviews, Written] | {'test': 1000} | {'test': 137.6} | -| [IndonesianIdClickbaitClassification](http://www.sciencedirect.com/science/article/pii/S2352340920311252) | ['ind'] | Classification | s2s | [News, Written] | {'train': 2048} | {'train': 64.28} | -| [IndonesianMongabayConservationClassification](https://aclanthology.org/2023.sealp-1.4/) | ['ind'] | Classification | s2s | [Web, Written] | {'validation': 984, 'test': 970} | {'validation': 1675.8, 'test': 1675.5} | -| [InsurancePolicyInterpretationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 133} | {'test': 521.88} | -| [InternationalCitizenshipQuestionsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 2048} | {'test': 206.18} | -| [IsiZuluNewsClassification](https://huggingface.co/datasets/dsfsi/za-isizulu-siswati-news) (Madodonga et al., 2023) | ['zul'] | Classification | s2s | [News, Written] | {'train': 752} | {'train': 43.1} | -| [ItaCaseholdClassification](https://doi.org/10.1145/3594536.3595177) (Licari et al., 2023) | ['ita'] | Classification | s2s | [Legal, Government, Written] | {'test': 221} | {'test': 4207.9} | -| [Itacola](https://aclanthology.org/2021.findings-emnlp.250/) | ['ita'] | Classification | s2s | [Non-fiction, Spoken, Written] | {'train': 7801, 'test': 975} | {'train': 35.95, 'test': 36.67} | -| [JCrewBlockerLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 54} | {'test': 1092.22} | +| [IN22ConvBitextMining](https://huggingface.co/datasets/ai4bharat/IN22-Conv) (Jay Gala, 2023) | ['asm', 'ben', 'brx', 'doi', 'eng', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Social, Spoken, Fiction, Spoken] | {'test': 760518} | {'test': {'num_samples': 760518, 'number_of_characters': 82637104, 'unique_pairs': 759283, 'min_sentence1_length': 3, 'average_sentence1_length': 54.33, 'max_sentence1_length': 239, 'unique_sentence1': 34430, 'min_sentence2_length': 3, 'average_sentence2_length': 54.33, 'max_sentence2_length': 239, 'unique_sentence2': 34430, 'hf_subset_descriptive_stats': {'asm_Beng-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155988, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'asm_Beng-brx_Deva': {'num_samples': 1503, 'number_of_characters': 162044, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'asm_Beng-doi_Deva': {'num_samples': 1503, 'number_of_characters': 167032, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'asm_Beng-eng_Latn': {'num_samples': 1503, 'number_of_characters': 160716, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'asm_Beng-gom_Deva': {'num_samples': 1503, 'number_of_characters': 156282, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'asm_Beng-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 158269, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'asm_Beng-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159964, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'asm_Beng-kan_Knda': {'num_samples': 1503, 'number_of_characters': 165177, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'asm_Beng-kas_Arab': {'num_samples': 1503, 'number_of_characters': 164681, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'asm_Beng-mai_Deva': {'num_samples': 1503, 'number_of_characters': 162408, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'asm_Beng-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 172838, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'asm_Beng-mar_Deva': {'num_samples': 1503, 'number_of_characters': 162747, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'asm_Beng-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 157316, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'asm_Beng-npi_Deva': {'num_samples': 1503, 'number_of_characters': 160906, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'asm_Beng-ory_Orya': {'num_samples': 1503, 'number_of_characters': 164223, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'asm_Beng-pan_Guru': {'num_samples': 1503, 'number_of_characters': 160201, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'asm_Beng-san_Deva': {'num_samples': 1503, 'number_of_characters': 158093, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'asm_Beng-sat_Olck': {'num_samples': 1503, 'number_of_characters': 169379, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'asm_Beng-snd_Deva': {'num_samples': 1503, 'number_of_characters': 162623, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'asm_Beng-tam_Taml': {'num_samples': 1503, 'number_of_characters': 174866, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'asm_Beng-tel_Telu': {'num_samples': 1503, 'number_of_characters': 157690, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'asm_Beng-urd_Arab': {'num_samples': 1503, 'number_of_characters': 161305, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'ben_Beng-asm_Beng': {'num_samples': 1503, 'number_of_characters': 155988, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'ben_Beng-brx_Deva': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'ben_Beng-doi_Deva': {'num_samples': 1503, 'number_of_characters': 161436, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'ben_Beng-eng_Latn': {'num_samples': 1503, 'number_of_characters': 155120, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'ben_Beng-gom_Deva': {'num_samples': 1503, 'number_of_characters': 150686, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'ben_Beng-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 152673, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'ben_Beng-hin_Deva': {'num_samples': 1503, 'number_of_characters': 154368, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'ben_Beng-kan_Knda': {'num_samples': 1503, 'number_of_characters': 159581, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'ben_Beng-kas_Arab': {'num_samples': 1503, 'number_of_characters': 159085, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'ben_Beng-mai_Deva': {'num_samples': 1503, 'number_of_characters': 156812, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'ben_Beng-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 167242, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'ben_Beng-mar_Deva': {'num_samples': 1503, 'number_of_characters': 157151, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'ben_Beng-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 151720, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'ben_Beng-npi_Deva': {'num_samples': 1503, 'number_of_characters': 155310, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'ben_Beng-ory_Orya': {'num_samples': 1503, 'number_of_characters': 158627, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'ben_Beng-pan_Guru': {'num_samples': 1503, 'number_of_characters': 154605, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'ben_Beng-san_Deva': {'num_samples': 1503, 'number_of_characters': 152497, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'ben_Beng-sat_Olck': {'num_samples': 1503, 'number_of_characters': 163783, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'ben_Beng-snd_Deva': {'num_samples': 1503, 'number_of_characters': 157027, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'ben_Beng-tam_Taml': {'num_samples': 1503, 'number_of_characters': 169270, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'ben_Beng-tel_Telu': {'num_samples': 1503, 'number_of_characters': 152094, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'ben_Beng-urd_Arab': {'num_samples': 1503, 'number_of_characters': 155709, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'brx_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162044, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'brx_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'brx_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 167492, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'brx_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161176, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'brx_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 156742, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'brx_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'brx_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 160424, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'brx_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 165637, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'brx_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165141, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'brx_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 162868, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'brx_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 173298, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'brx_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 163207, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'brx_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 157776, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'brx_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'brx_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 164683, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'brx_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 160661, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'brx_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 158553, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'brx_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 169839, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'brx_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 163083, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'brx_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 175326, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'brx_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158150, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'brx_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 161765, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'doi_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 167032, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'doi_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 161436, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'doi_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 167492, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'doi_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 166164, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'doi_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'doi_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 163717, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'doi_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 165412, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'doi_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 170625, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'doi_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 170129, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'doi_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 167856, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'doi_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 178286, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'doi_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 168195, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'doi_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 162764, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'doi_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 166354, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'doi_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 169671, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'doi_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 165649, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'doi_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 163541, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'doi_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 174827, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'doi_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 168071, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'doi_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 180314, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'doi_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 163138, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'doi_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 166753, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'eng_Latn-asm_Beng': {'num_samples': 1503, 'number_of_characters': 160716, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'eng_Latn-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155120, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'eng_Latn-brx_Deva': {'num_samples': 1503, 'number_of_characters': 161176, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'eng_Latn-doi_Deva': {'num_samples': 1503, 'number_of_characters': 166164, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'eng_Latn-gom_Deva': {'num_samples': 1503, 'number_of_characters': 155414, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'eng_Latn-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 157401, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'eng_Latn-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159096, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'eng_Latn-kan_Knda': {'num_samples': 1503, 'number_of_characters': 164309, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'eng_Latn-kas_Arab': {'num_samples': 1503, 'number_of_characters': 163813, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'eng_Latn-mai_Deva': {'num_samples': 1503, 'number_of_characters': 161540, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'eng_Latn-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 171970, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'eng_Latn-mar_Deva': {'num_samples': 1503, 'number_of_characters': 161879, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'eng_Latn-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'eng_Latn-npi_Deva': {'num_samples': 1503, 'number_of_characters': 160038, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'eng_Latn-ory_Orya': {'num_samples': 1503, 'number_of_characters': 163355, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'eng_Latn-pan_Guru': {'num_samples': 1503, 'number_of_characters': 159333, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'eng_Latn-san_Deva': {'num_samples': 1503, 'number_of_characters': 157225, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'eng_Latn-sat_Olck': {'num_samples': 1503, 'number_of_characters': 168511, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'eng_Latn-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161755, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'eng_Latn-tam_Taml': {'num_samples': 1503, 'number_of_characters': 173998, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'eng_Latn-tel_Telu': {'num_samples': 1503, 'number_of_characters': 156822, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'eng_Latn-urd_Arab': {'num_samples': 1503, 'number_of_characters': 160437, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'gom_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 156282, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'gom_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 150686, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'gom_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 156742, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'gom_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'gom_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 155414, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'gom_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 152967, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'gom_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 154662, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'gom_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 159875, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'gom_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 159379, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'gom_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 157106, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'gom_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 167536, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'gom_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 157445, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'gom_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 152014, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'gom_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 155604, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'gom_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 158921, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'gom_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 154899, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'gom_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 152791, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'gom_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 164077, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'gom_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 157321, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'gom_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 169564, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'gom_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 152388, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'gom_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 156003, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'guj_Gujr-asm_Beng': {'num_samples': 1503, 'number_of_characters': 158269, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'guj_Gujr-ben_Beng': {'num_samples': 1503, 'number_of_characters': 152673, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'guj_Gujr-brx_Deva': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'guj_Gujr-doi_Deva': {'num_samples': 1503, 'number_of_characters': 163717, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'guj_Gujr-eng_Latn': {'num_samples': 1503, 'number_of_characters': 157401, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'guj_Gujr-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152967, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'guj_Gujr-hin_Deva': {'num_samples': 1503, 'number_of_characters': 156649, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'guj_Gujr-kan_Knda': {'num_samples': 1503, 'number_of_characters': 161862, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'guj_Gujr-kas_Arab': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'guj_Gujr-mai_Deva': {'num_samples': 1503, 'number_of_characters': 159093, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'guj_Gujr-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 169523, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'guj_Gujr-mar_Deva': {'num_samples': 1503, 'number_of_characters': 159432, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'guj_Gujr-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 154001, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'guj_Gujr-npi_Deva': {'num_samples': 1503, 'number_of_characters': 157591, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'guj_Gujr-ory_Orya': {'num_samples': 1503, 'number_of_characters': 160908, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'guj_Gujr-pan_Guru': {'num_samples': 1503, 'number_of_characters': 156886, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'guj_Gujr-san_Deva': {'num_samples': 1503, 'number_of_characters': 154778, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'guj_Gujr-sat_Olck': {'num_samples': 1503, 'number_of_characters': 166064, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'guj_Gujr-snd_Deva': {'num_samples': 1503, 'number_of_characters': 159308, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'guj_Gujr-tam_Taml': {'num_samples': 1503, 'number_of_characters': 171551, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'guj_Gujr-tel_Telu': {'num_samples': 1503, 'number_of_characters': 154375, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'guj_Gujr-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157990, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'hin_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 159964, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'hin_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 154368, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'hin_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 160424, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'hin_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 165412, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'hin_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 159096, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'hin_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 154662, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'hin_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 156649, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'hin_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 163557, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'hin_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 163061, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'hin_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 160788, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'hin_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 171218, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'hin_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 161127, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'hin_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 155696, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'hin_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 159286, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'hin_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 162603, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'hin_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 158581, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'hin_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 156473, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'hin_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 167759, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'hin_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'hin_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 173246, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'hin_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 156070, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'hin_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 159685, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'kan_Knda-asm_Beng': {'num_samples': 1503, 'number_of_characters': 165177, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'kan_Knda-ben_Beng': {'num_samples': 1503, 'number_of_characters': 159581, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'kan_Knda-brx_Deva': {'num_samples': 1503, 'number_of_characters': 165637, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'kan_Knda-doi_Deva': {'num_samples': 1503, 'number_of_characters': 170625, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'kan_Knda-eng_Latn': {'num_samples': 1503, 'number_of_characters': 164309, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'kan_Knda-gom_Deva': {'num_samples': 1503, 'number_of_characters': 159875, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'kan_Knda-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 161862, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'kan_Knda-hin_Deva': {'num_samples': 1503, 'number_of_characters': 163557, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'kan_Knda-kas_Arab': {'num_samples': 1503, 'number_of_characters': 168274, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'kan_Knda-mai_Deva': {'num_samples': 1503, 'number_of_characters': 166001, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'kan_Knda-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 176431, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'kan_Knda-mar_Deva': {'num_samples': 1503, 'number_of_characters': 166340, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'kan_Knda-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 160909, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'kan_Knda-npi_Deva': {'num_samples': 1503, 'number_of_characters': 164499, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'kan_Knda-ory_Orya': {'num_samples': 1503, 'number_of_characters': 167816, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'kan_Knda-pan_Guru': {'num_samples': 1503, 'number_of_characters': 163794, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'kan_Knda-san_Deva': {'num_samples': 1503, 'number_of_characters': 161686, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'kan_Knda-sat_Olck': {'num_samples': 1503, 'number_of_characters': 172972, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'kan_Knda-snd_Deva': {'num_samples': 1503, 'number_of_characters': 166216, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'kan_Knda-tam_Taml': {'num_samples': 1503, 'number_of_characters': 178459, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'kan_Knda-tel_Telu': {'num_samples': 1503, 'number_of_characters': 161283, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'kan_Knda-urd_Arab': {'num_samples': 1503, 'number_of_characters': 164898, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'kas_Arab-asm_Beng': {'num_samples': 1503, 'number_of_characters': 164681, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'kas_Arab-ben_Beng': {'num_samples': 1503, 'number_of_characters': 159085, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'kas_Arab-brx_Deva': {'num_samples': 1503, 'number_of_characters': 165141, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'kas_Arab-doi_Deva': {'num_samples': 1503, 'number_of_characters': 170129, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'kas_Arab-eng_Latn': {'num_samples': 1503, 'number_of_characters': 163813, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'kas_Arab-gom_Deva': {'num_samples': 1503, 'number_of_characters': 159379, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'kas_Arab-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'kas_Arab-hin_Deva': {'num_samples': 1503, 'number_of_characters': 163061, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'kas_Arab-kan_Knda': {'num_samples': 1503, 'number_of_characters': 168274, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'kas_Arab-mai_Deva': {'num_samples': 1503, 'number_of_characters': 165505, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'kas_Arab-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 175935, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'kas_Arab-mar_Deva': {'num_samples': 1503, 'number_of_characters': 165844, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'kas_Arab-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 160413, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'kas_Arab-npi_Deva': {'num_samples': 1503, 'number_of_characters': 164003, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'kas_Arab-ory_Orya': {'num_samples': 1503, 'number_of_characters': 167320, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'kas_Arab-pan_Guru': {'num_samples': 1503, 'number_of_characters': 163298, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'kas_Arab-san_Deva': {'num_samples': 1503, 'number_of_characters': 161190, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'kas_Arab-sat_Olck': {'num_samples': 1503, 'number_of_characters': 172476, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'kas_Arab-snd_Deva': {'num_samples': 1503, 'number_of_characters': 165720, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'kas_Arab-tam_Taml': {'num_samples': 1503, 'number_of_characters': 177963, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'kas_Arab-tel_Telu': {'num_samples': 1503, 'number_of_characters': 160787, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'kas_Arab-urd_Arab': {'num_samples': 1503, 'number_of_characters': 164402, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mai_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162408, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mai_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 156812, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mai_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 162868, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mai_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 167856, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mai_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161540, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mai_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 157106, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mai_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 159093, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mai_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 160788, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mai_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 166001, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mai_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165505, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mai_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 173662, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'mai_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 163571, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'mai_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 158140, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'mai_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mai_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 165047, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mai_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 161025, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mai_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 158917, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mai_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 170203, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mai_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 163447, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mai_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 175690, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mai_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158514, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mai_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 162129, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mal_Mlym-asm_Beng': {'num_samples': 1503, 'number_of_characters': 172838, 'unique_pairs': 1498, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mal_Mlym-ben_Beng': {'num_samples': 1503, 'number_of_characters': 167242, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mal_Mlym-brx_Deva': {'num_samples': 1503, 'number_of_characters': 173298, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mal_Mlym-doi_Deva': {'num_samples': 1503, 'number_of_characters': 178286, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mal_Mlym-eng_Latn': {'num_samples': 1503, 'number_of_characters': 171970, 'unique_pairs': 1499, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mal_Mlym-gom_Deva': {'num_samples': 1503, 'number_of_characters': 167536, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mal_Mlym-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 169523, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mal_Mlym-hin_Deva': {'num_samples': 1503, 'number_of_characters': 171218, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mal_Mlym-kan_Knda': {'num_samples': 1503, 'number_of_characters': 176431, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mal_Mlym-kas_Arab': {'num_samples': 1503, 'number_of_characters': 175935, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mal_Mlym-mai_Deva': {'num_samples': 1503, 'number_of_characters': 173662, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'mal_Mlym-mar_Deva': {'num_samples': 1503, 'number_of_characters': 174001, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'mal_Mlym-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 168570, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'mal_Mlym-npi_Deva': {'num_samples': 1503, 'number_of_characters': 172160, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mal_Mlym-ory_Orya': {'num_samples': 1503, 'number_of_characters': 175477, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mal_Mlym-pan_Guru': {'num_samples': 1503, 'number_of_characters': 171455, 'unique_pairs': 1498, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mal_Mlym-san_Deva': {'num_samples': 1503, 'number_of_characters': 169347, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mal_Mlym-sat_Olck': {'num_samples': 1503, 'number_of_characters': 180633, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mal_Mlym-snd_Deva': {'num_samples': 1503, 'number_of_characters': 173877, 'unique_pairs': 1499, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mal_Mlym-tam_Taml': {'num_samples': 1503, 'number_of_characters': 186120, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mal_Mlym-tel_Telu': {'num_samples': 1503, 'number_of_characters': 168944, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mal_Mlym-urd_Arab': {'num_samples': 1503, 'number_of_characters': 172559, 'unique_pairs': 1499, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mar_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162747, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mar_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 157151, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mar_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 163207, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mar_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 168195, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mar_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161879, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mar_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 157445, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mar_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 159432, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mar_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 161127, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mar_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 166340, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mar_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165844, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mar_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 163571, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'mar_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 174001, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'mar_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 158479, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'mar_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 162069, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mar_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 165386, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mar_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 161364, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mar_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 159256, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mar_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 170542, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mar_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 163786, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mar_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 176029, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mar_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158853, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mar_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 162468, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mni_Mtei-asm_Beng': {'num_samples': 1503, 'number_of_characters': 157316, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mni_Mtei-ben_Beng': {'num_samples': 1503, 'number_of_characters': 151720, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mni_Mtei-brx_Deva': {'num_samples': 1503, 'number_of_characters': 157776, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mni_Mtei-doi_Deva': {'num_samples': 1503, 'number_of_characters': 162764, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mni_Mtei-eng_Latn': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mni_Mtei-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152014, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mni_Mtei-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 154001, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mni_Mtei-hin_Deva': {'num_samples': 1503, 'number_of_characters': 155696, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mni_Mtei-kan_Knda': {'num_samples': 1503, 'number_of_characters': 160909, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mni_Mtei-kas_Arab': {'num_samples': 1503, 'number_of_characters': 160413, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mni_Mtei-mai_Deva': {'num_samples': 1503, 'number_of_characters': 158140, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'mni_Mtei-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 168570, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'mni_Mtei-mar_Deva': {'num_samples': 1503, 'number_of_characters': 158479, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'mni_Mtei-npi_Deva': {'num_samples': 1503, 'number_of_characters': 156638, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mni_Mtei-ory_Orya': {'num_samples': 1503, 'number_of_characters': 159955, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mni_Mtei-pan_Guru': {'num_samples': 1503, 'number_of_characters': 155933, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mni_Mtei-san_Deva': {'num_samples': 1503, 'number_of_characters': 153825, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mni_Mtei-sat_Olck': {'num_samples': 1503, 'number_of_characters': 165111, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mni_Mtei-snd_Deva': {'num_samples': 1503, 'number_of_characters': 158355, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mni_Mtei-tam_Taml': {'num_samples': 1503, 'number_of_characters': 170598, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mni_Mtei-tel_Telu': {'num_samples': 1503, 'number_of_characters': 153422, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mni_Mtei-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157037, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'npi_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 160906, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'npi_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155310, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'npi_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'npi_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 166354, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'npi_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 160038, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'npi_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 155604, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'npi_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 157591, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'npi_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159286, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'npi_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 164499, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'npi_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 164003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'npi_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'npi_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 172160, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'npi_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 162069, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'npi_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 156638, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'npi_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 163545, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'npi_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 159523, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'npi_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 157415, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'npi_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 168701, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'npi_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161945, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'npi_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 174188, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'npi_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 157012, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'npi_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 160627, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'ory_Orya-asm_Beng': {'num_samples': 1503, 'number_of_characters': 164223, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'ory_Orya-ben_Beng': {'num_samples': 1503, 'number_of_characters': 158627, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'ory_Orya-brx_Deva': {'num_samples': 1503, 'number_of_characters': 164683, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'ory_Orya-doi_Deva': {'num_samples': 1503, 'number_of_characters': 169671, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'ory_Orya-eng_Latn': {'num_samples': 1503, 'number_of_characters': 163355, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'ory_Orya-gom_Deva': {'num_samples': 1503, 'number_of_characters': 158921, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'ory_Orya-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 160908, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'ory_Orya-hin_Deva': {'num_samples': 1503, 'number_of_characters': 162603, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'ory_Orya-kan_Knda': {'num_samples': 1503, 'number_of_characters': 167816, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'ory_Orya-kas_Arab': {'num_samples': 1503, 'number_of_characters': 167320, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'ory_Orya-mai_Deva': {'num_samples': 1503, 'number_of_characters': 165047, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'ory_Orya-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 175477, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'ory_Orya-mar_Deva': {'num_samples': 1503, 'number_of_characters': 165386, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'ory_Orya-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 159955, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'ory_Orya-npi_Deva': {'num_samples': 1503, 'number_of_characters': 163545, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'ory_Orya-pan_Guru': {'num_samples': 1503, 'number_of_characters': 162840, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'ory_Orya-san_Deva': {'num_samples': 1503, 'number_of_characters': 160732, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'ory_Orya-sat_Olck': {'num_samples': 1503, 'number_of_characters': 172018, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'ory_Orya-snd_Deva': {'num_samples': 1503, 'number_of_characters': 165262, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'ory_Orya-tam_Taml': {'num_samples': 1503, 'number_of_characters': 177505, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'ory_Orya-tel_Telu': {'num_samples': 1503, 'number_of_characters': 160329, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'ory_Orya-urd_Arab': {'num_samples': 1503, 'number_of_characters': 163944, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'pan_Guru-asm_Beng': {'num_samples': 1503, 'number_of_characters': 160201, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'pan_Guru-ben_Beng': {'num_samples': 1503, 'number_of_characters': 154605, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'pan_Guru-brx_Deva': {'num_samples': 1503, 'number_of_characters': 160661, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'pan_Guru-doi_Deva': {'num_samples': 1503, 'number_of_characters': 165649, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'pan_Guru-eng_Latn': {'num_samples': 1503, 'number_of_characters': 159333, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'pan_Guru-gom_Deva': {'num_samples': 1503, 'number_of_characters': 154899, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'pan_Guru-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 156886, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'pan_Guru-hin_Deva': {'num_samples': 1503, 'number_of_characters': 158581, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'pan_Guru-kan_Knda': {'num_samples': 1503, 'number_of_characters': 163794, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'pan_Guru-kas_Arab': {'num_samples': 1503, 'number_of_characters': 163298, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'pan_Guru-mai_Deva': {'num_samples': 1503, 'number_of_characters': 161025, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'pan_Guru-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 171455, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'pan_Guru-mar_Deva': {'num_samples': 1503, 'number_of_characters': 161364, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'pan_Guru-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 155933, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'pan_Guru-npi_Deva': {'num_samples': 1503, 'number_of_characters': 159523, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'pan_Guru-ory_Orya': {'num_samples': 1503, 'number_of_characters': 162840, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'pan_Guru-san_Deva': {'num_samples': 1503, 'number_of_characters': 156710, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'pan_Guru-sat_Olck': {'num_samples': 1503, 'number_of_characters': 167996, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'pan_Guru-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161240, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'pan_Guru-tam_Taml': {'num_samples': 1503, 'number_of_characters': 173483, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'pan_Guru-tel_Telu': {'num_samples': 1503, 'number_of_characters': 156307, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'pan_Guru-urd_Arab': {'num_samples': 1503, 'number_of_characters': 159922, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'san_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 158093, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'san_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 152497, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'san_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 158553, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'san_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 163541, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'san_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 157225, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'san_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152791, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'san_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 154778, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'san_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 156473, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'san_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 161686, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'san_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 161190, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'san_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 158917, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'san_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 169347, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'san_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 159256, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'san_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 153825, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'san_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 157415, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'san_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 160732, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'san_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 156710, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'san_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 165888, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'san_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 159132, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'san_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 171375, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'san_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 154199, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'san_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157814, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'sat_Olck-asm_Beng': {'num_samples': 1503, 'number_of_characters': 169379, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'sat_Olck-ben_Beng': {'num_samples': 1503, 'number_of_characters': 163783, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'sat_Olck-brx_Deva': {'num_samples': 1503, 'number_of_characters': 169839, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'sat_Olck-doi_Deva': {'num_samples': 1503, 'number_of_characters': 174827, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'sat_Olck-eng_Latn': {'num_samples': 1503, 'number_of_characters': 168511, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'sat_Olck-gom_Deva': {'num_samples': 1503, 'number_of_characters': 164077, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'sat_Olck-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 166064, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'sat_Olck-hin_Deva': {'num_samples': 1503, 'number_of_characters': 167759, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'sat_Olck-kan_Knda': {'num_samples': 1503, 'number_of_characters': 172972, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'sat_Olck-kas_Arab': {'num_samples': 1503, 'number_of_characters': 172476, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'sat_Olck-mai_Deva': {'num_samples': 1503, 'number_of_characters': 170203, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'sat_Olck-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 180633, 'unique_pairs': 1501, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'sat_Olck-mar_Deva': {'num_samples': 1503, 'number_of_characters': 170542, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'sat_Olck-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 165111, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'sat_Olck-npi_Deva': {'num_samples': 1503, 'number_of_characters': 168701, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'sat_Olck-ory_Orya': {'num_samples': 1503, 'number_of_characters': 172018, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'sat_Olck-pan_Guru': {'num_samples': 1503, 'number_of_characters': 167996, 'unique_pairs': 1501, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'sat_Olck-san_Deva': {'num_samples': 1503, 'number_of_characters': 165888, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'sat_Olck-snd_Deva': {'num_samples': 1503, 'number_of_characters': 170418, 'unique_pairs': 1501, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'sat_Olck-tam_Taml': {'num_samples': 1503, 'number_of_characters': 182661, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'sat_Olck-tel_Telu': {'num_samples': 1503, 'number_of_characters': 165485, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'sat_Olck-urd_Arab': {'num_samples': 1503, 'number_of_characters': 169100, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'snd_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162623, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'snd_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 157027, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'snd_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 163083, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'snd_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 168071, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'snd_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161755, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'snd_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 157321, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'snd_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 159308, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'snd_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 161003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'snd_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 166216, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'snd_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165720, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'snd_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 163447, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'snd_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 173877, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'snd_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 163786, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'snd_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 158355, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'snd_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 161945, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'snd_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 165262, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'snd_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 161240, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'snd_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 159132, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'snd_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 170418, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'snd_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 175905, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'snd_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'snd_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 162344, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'tam_Taml-asm_Beng': {'num_samples': 1503, 'number_of_characters': 174866, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'tam_Taml-ben_Beng': {'num_samples': 1503, 'number_of_characters': 169270, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'tam_Taml-brx_Deva': {'num_samples': 1503, 'number_of_characters': 175326, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'tam_Taml-doi_Deva': {'num_samples': 1503, 'number_of_characters': 180314, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'tam_Taml-eng_Latn': {'num_samples': 1503, 'number_of_characters': 173998, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'tam_Taml-gom_Deva': {'num_samples': 1503, 'number_of_characters': 169564, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'tam_Taml-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 171551, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'tam_Taml-hin_Deva': {'num_samples': 1503, 'number_of_characters': 173246, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'tam_Taml-kan_Knda': {'num_samples': 1503, 'number_of_characters': 178459, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'tam_Taml-kas_Arab': {'num_samples': 1503, 'number_of_characters': 177963, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'tam_Taml-mai_Deva': {'num_samples': 1503, 'number_of_characters': 175690, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'tam_Taml-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 186120, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'tam_Taml-mar_Deva': {'num_samples': 1503, 'number_of_characters': 176029, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'tam_Taml-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 170598, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'tam_Taml-npi_Deva': {'num_samples': 1503, 'number_of_characters': 174188, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'tam_Taml-ory_Orya': {'num_samples': 1503, 'number_of_characters': 177505, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'tam_Taml-pan_Guru': {'num_samples': 1503, 'number_of_characters': 173483, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'tam_Taml-san_Deva': {'num_samples': 1503, 'number_of_characters': 171375, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'tam_Taml-sat_Olck': {'num_samples': 1503, 'number_of_characters': 182661, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'tam_Taml-snd_Deva': {'num_samples': 1503, 'number_of_characters': 175905, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'tam_Taml-tel_Telu': {'num_samples': 1503, 'number_of_characters': 170972, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'tam_Taml-urd_Arab': {'num_samples': 1503, 'number_of_characters': 174587, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'tel_Telu-asm_Beng': {'num_samples': 1503, 'number_of_characters': 157690, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'tel_Telu-ben_Beng': {'num_samples': 1503, 'number_of_characters': 152094, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'tel_Telu-brx_Deva': {'num_samples': 1503, 'number_of_characters': 158150, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'tel_Telu-doi_Deva': {'num_samples': 1503, 'number_of_characters': 163138, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'tel_Telu-eng_Latn': {'num_samples': 1503, 'number_of_characters': 156822, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'tel_Telu-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152388, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'tel_Telu-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 154375, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'tel_Telu-hin_Deva': {'num_samples': 1503, 'number_of_characters': 156070, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'tel_Telu-kan_Knda': {'num_samples': 1503, 'number_of_characters': 161283, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'tel_Telu-kas_Arab': {'num_samples': 1503, 'number_of_characters': 160787, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'tel_Telu-mai_Deva': {'num_samples': 1503, 'number_of_characters': 158514, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'tel_Telu-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 168944, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'tel_Telu-mar_Deva': {'num_samples': 1503, 'number_of_characters': 158853, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'tel_Telu-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 153422, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'tel_Telu-npi_Deva': {'num_samples': 1503, 'number_of_characters': 157012, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'tel_Telu-ory_Orya': {'num_samples': 1503, 'number_of_characters': 160329, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'tel_Telu-pan_Guru': {'num_samples': 1503, 'number_of_characters': 156307, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'tel_Telu-san_Deva': {'num_samples': 1503, 'number_of_characters': 154199, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'tel_Telu-sat_Olck': {'num_samples': 1503, 'number_of_characters': 165485, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'tel_Telu-snd_Deva': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'tel_Telu-tam_Taml': {'num_samples': 1503, 'number_of_characters': 170972, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'tel_Telu-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157411, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'urd_Arab-asm_Beng': {'num_samples': 1503, 'number_of_characters': 161305, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'urd_Arab-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155709, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'urd_Arab-brx_Deva': {'num_samples': 1503, 'number_of_characters': 161765, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'urd_Arab-doi_Deva': {'num_samples': 1503, 'number_of_characters': 166753, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'urd_Arab-eng_Latn': {'num_samples': 1503, 'number_of_characters': 160437, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'urd_Arab-gom_Deva': {'num_samples': 1503, 'number_of_characters': 156003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'urd_Arab-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 157990, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'urd_Arab-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159685, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'urd_Arab-kan_Knda': {'num_samples': 1503, 'number_of_characters': 164898, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'urd_Arab-kas_Arab': {'num_samples': 1503, 'number_of_characters': 164402, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'urd_Arab-mai_Deva': {'num_samples': 1503, 'number_of_characters': 162129, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'urd_Arab-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 172559, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'urd_Arab-mar_Deva': {'num_samples': 1503, 'number_of_characters': 162468, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'urd_Arab-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 157037, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'urd_Arab-npi_Deva': {'num_samples': 1503, 'number_of_characters': 160627, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'urd_Arab-ory_Orya': {'num_samples': 1503, 'number_of_characters': 163944, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'urd_Arab-pan_Guru': {'num_samples': 1503, 'number_of_characters': 159922, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'urd_Arab-san_Deva': {'num_samples': 1503, 'number_of_characters': 157814, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'urd_Arab-sat_Olck': {'num_samples': 1503, 'number_of_characters': 169100, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'urd_Arab-snd_Deva': {'num_samples': 1503, 'number_of_characters': 162344, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'urd_Arab-tam_Taml': {'num_samples': 1503, 'number_of_characters': 174587, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'urd_Arab-tel_Telu': {'num_samples': 1503, 'number_of_characters': 157411, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}}}} | +| [IN22GenBitextMining](https://huggingface.co/datasets/ai4bharat/IN22-Gen) (Jay Gala, 2023) | ['asm', 'ben', 'brx', 'doi', 'eng', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Web, Legal, Government, News, Religious, Non-fiction, Written] | {'test': 518144} | {'test': {'num_samples': 518144, 'number_of_characters': 162367876, 'unique_pairs': 518101, 'min_sentence1_length': 9, 'average_sentence1_length': 156.68, 'max_sentence1_length': 692, 'unique_sentence1': 23550, 'min_sentence2_length': 9, 'average_sentence2_length': 156.68, 'max_sentence2_length': 692, 'unique_sentence2': 23550, 'hf_subset_descriptive_stats': {'asm_Beng-ben_Beng': {'num_samples': 1024, 'number_of_characters': 310622, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'asm_Beng-brx_Deva': {'num_samples': 1024, 'number_of_characters': 323609, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'asm_Beng-doi_Deva': {'num_samples': 1024, 'number_of_characters': 319020, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'asm_Beng-eng_Latn': {'num_samples': 1024, 'number_of_characters': 320098, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'asm_Beng-gom_Deva': {'num_samples': 1024, 'number_of_characters': 312594, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'asm_Beng-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 309440, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'asm_Beng-hin_Deva': {'num_samples': 1024, 'number_of_characters': 320106, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'asm_Beng-kan_Knda': {'num_samples': 1024, 'number_of_characters': 332064, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'asm_Beng-kas_Arab': {'num_samples': 1024, 'number_of_characters': 322764, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'asm_Beng-mai_Deva': {'num_samples': 1024, 'number_of_characters': 308682, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'asm_Beng-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 343636, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'asm_Beng-mar_Deva': {'num_samples': 1024, 'number_of_characters': 321784, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'asm_Beng-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 313134, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'asm_Beng-npi_Deva': {'num_samples': 1024, 'number_of_characters': 313419, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'asm_Beng-ory_Orya': {'num_samples': 1024, 'number_of_characters': 334226, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'asm_Beng-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306863, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'asm_Beng-san_Deva': {'num_samples': 1024, 'number_of_characters': 318079, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'asm_Beng-sat_Olck': {'num_samples': 1024, 'number_of_characters': 326732, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'asm_Beng-snd_Deva': {'num_samples': 1024, 'number_of_characters': 320421, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'asm_Beng-tam_Taml': {'num_samples': 1024, 'number_of_characters': 348346, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'asm_Beng-tel_Telu': {'num_samples': 1024, 'number_of_characters': 319045, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'asm_Beng-urd_Arab': {'num_samples': 1024, 'number_of_characters': 315134, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'ben_Beng-asm_Beng': {'num_samples': 1024, 'number_of_characters': 310622, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'ben_Beng-brx_Deva': {'num_samples': 1024, 'number_of_characters': 313313, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'ben_Beng-doi_Deva': {'num_samples': 1024, 'number_of_characters': 308724, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'ben_Beng-eng_Latn': {'num_samples': 1024, 'number_of_characters': 309802, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'ben_Beng-gom_Deva': {'num_samples': 1024, 'number_of_characters': 302298, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'ben_Beng-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 299144, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'ben_Beng-hin_Deva': {'num_samples': 1024, 'number_of_characters': 309810, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'ben_Beng-kan_Knda': {'num_samples': 1024, 'number_of_characters': 321768, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'ben_Beng-kas_Arab': {'num_samples': 1024, 'number_of_characters': 312468, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'ben_Beng-mai_Deva': {'num_samples': 1024, 'number_of_characters': 298386, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'ben_Beng-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 333340, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'ben_Beng-mar_Deva': {'num_samples': 1024, 'number_of_characters': 311488, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'ben_Beng-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 302838, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'ben_Beng-npi_Deva': {'num_samples': 1024, 'number_of_characters': 303123, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'ben_Beng-ory_Orya': {'num_samples': 1024, 'number_of_characters': 323930, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'ben_Beng-pan_Guru': {'num_samples': 1024, 'number_of_characters': 296567, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'ben_Beng-san_Deva': {'num_samples': 1024, 'number_of_characters': 307783, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'ben_Beng-sat_Olck': {'num_samples': 1024, 'number_of_characters': 316436, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'ben_Beng-snd_Deva': {'num_samples': 1024, 'number_of_characters': 310125, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'ben_Beng-tam_Taml': {'num_samples': 1024, 'number_of_characters': 338050, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'ben_Beng-tel_Telu': {'num_samples': 1024, 'number_of_characters': 308749, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'ben_Beng-urd_Arab': {'num_samples': 1024, 'number_of_characters': 304838, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'brx_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 323609, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'brx_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 313313, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'brx_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 321711, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'brx_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 322789, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'brx_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 315285, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'brx_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 312131, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'brx_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 322797, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'brx_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 334755, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'brx_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 325455, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'brx_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 311373, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'brx_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 346327, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'brx_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 324475, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'brx_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 315825, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'brx_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 316110, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'brx_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 336917, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'brx_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 309554, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'brx_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 320770, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'brx_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 329423, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'brx_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 323112, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'brx_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 351037, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'brx_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 321736, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'brx_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 317825, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'doi_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 319020, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'doi_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 308724, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'doi_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 321711, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'doi_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 318200, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'doi_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 310696, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'doi_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 307542, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'doi_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 318208, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'doi_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 330166, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'doi_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 320866, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'doi_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 306784, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'doi_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 341738, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'doi_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 319886, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'doi_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 311236, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'doi_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 311521, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'doi_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 332328, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'doi_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 304965, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'doi_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 316181, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'doi_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 324834, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'doi_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 318523, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'doi_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 346448, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'doi_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 317147, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'doi_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 313236, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'eng_Latn-asm_Beng': {'num_samples': 1024, 'number_of_characters': 320098, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'eng_Latn-ben_Beng': {'num_samples': 1024, 'number_of_characters': 309802, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'eng_Latn-brx_Deva': {'num_samples': 1024, 'number_of_characters': 322789, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'eng_Latn-doi_Deva': {'num_samples': 1024, 'number_of_characters': 318200, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'eng_Latn-gom_Deva': {'num_samples': 1024, 'number_of_characters': 311774, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'eng_Latn-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 308620, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'eng_Latn-hin_Deva': {'num_samples': 1024, 'number_of_characters': 319286, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'eng_Latn-kan_Knda': {'num_samples': 1024, 'number_of_characters': 331244, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'eng_Latn-kas_Arab': {'num_samples': 1024, 'number_of_characters': 321944, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'eng_Latn-mai_Deva': {'num_samples': 1024, 'number_of_characters': 307862, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'eng_Latn-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 342816, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'eng_Latn-mar_Deva': {'num_samples': 1024, 'number_of_characters': 320964, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'eng_Latn-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 312314, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'eng_Latn-npi_Deva': {'num_samples': 1024, 'number_of_characters': 312599, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'eng_Latn-ory_Orya': {'num_samples': 1024, 'number_of_characters': 333406, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'eng_Latn-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306043, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'eng_Latn-san_Deva': {'num_samples': 1024, 'number_of_characters': 317259, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'eng_Latn-sat_Olck': {'num_samples': 1024, 'number_of_characters': 325912, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'eng_Latn-snd_Deva': {'num_samples': 1024, 'number_of_characters': 319601, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'eng_Latn-tam_Taml': {'num_samples': 1024, 'number_of_characters': 347526, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'eng_Latn-tel_Telu': {'num_samples': 1024, 'number_of_characters': 318225, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'eng_Latn-urd_Arab': {'num_samples': 1024, 'number_of_characters': 314314, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'gom_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 312594, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'gom_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 302298, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'gom_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 315285, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'gom_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 310696, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'gom_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 311774, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'gom_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 301116, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'gom_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 311782, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'gom_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 323740, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'gom_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 314440, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'gom_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 300358, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'gom_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 335312, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'gom_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 313460, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'gom_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 304810, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'gom_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 305095, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'gom_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 325902, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'gom_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 298539, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'gom_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 309755, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'gom_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 318408, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'gom_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 312097, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'gom_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 340022, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'gom_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 310721, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'gom_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 306810, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'guj_Gujr-asm_Beng': {'num_samples': 1024, 'number_of_characters': 309440, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'guj_Gujr-ben_Beng': {'num_samples': 1024, 'number_of_characters': 299144, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'guj_Gujr-brx_Deva': {'num_samples': 1024, 'number_of_characters': 312131, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'guj_Gujr-doi_Deva': {'num_samples': 1024, 'number_of_characters': 307542, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'guj_Gujr-eng_Latn': {'num_samples': 1024, 'number_of_characters': 308620, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'guj_Gujr-gom_Deva': {'num_samples': 1024, 'number_of_characters': 301116, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'guj_Gujr-hin_Deva': {'num_samples': 1024, 'number_of_characters': 308628, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'guj_Gujr-kan_Knda': {'num_samples': 1024, 'number_of_characters': 320586, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'guj_Gujr-kas_Arab': {'num_samples': 1024, 'number_of_characters': 311286, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'guj_Gujr-mai_Deva': {'num_samples': 1024, 'number_of_characters': 297204, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'guj_Gujr-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 332158, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'guj_Gujr-mar_Deva': {'num_samples': 1024, 'number_of_characters': 310306, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'guj_Gujr-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 301656, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'guj_Gujr-npi_Deva': {'num_samples': 1024, 'number_of_characters': 301941, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'guj_Gujr-ory_Orya': {'num_samples': 1024, 'number_of_characters': 322748, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'guj_Gujr-pan_Guru': {'num_samples': 1024, 'number_of_characters': 295385, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'guj_Gujr-san_Deva': {'num_samples': 1024, 'number_of_characters': 306601, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'guj_Gujr-sat_Olck': {'num_samples': 1024, 'number_of_characters': 315254, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'guj_Gujr-snd_Deva': {'num_samples': 1024, 'number_of_characters': 308943, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'guj_Gujr-tam_Taml': {'num_samples': 1024, 'number_of_characters': 336868, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'guj_Gujr-tel_Telu': {'num_samples': 1024, 'number_of_characters': 307567, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'guj_Gujr-urd_Arab': {'num_samples': 1024, 'number_of_characters': 303656, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'hin_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 320106, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'hin_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 309810, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'hin_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 322797, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'hin_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 318208, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'hin_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 319286, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'hin_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 311782, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'hin_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 308628, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'hin_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 331252, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'hin_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 321952, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'hin_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 307870, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'hin_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 342824, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'hin_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 320972, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'hin_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 312322, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'hin_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 312607, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'hin_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 333414, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'hin_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306051, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'hin_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 317267, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'hin_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 325920, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'hin_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 319609, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'hin_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 347534, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'hin_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 318233, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'hin_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 314322, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'kan_Knda-asm_Beng': {'num_samples': 1024, 'number_of_characters': 332064, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'kan_Knda-ben_Beng': {'num_samples': 1024, 'number_of_characters': 321768, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'kan_Knda-brx_Deva': {'num_samples': 1024, 'number_of_characters': 334755, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'kan_Knda-doi_Deva': {'num_samples': 1024, 'number_of_characters': 330166, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'kan_Knda-eng_Latn': {'num_samples': 1024, 'number_of_characters': 331244, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'kan_Knda-gom_Deva': {'num_samples': 1024, 'number_of_characters': 323740, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'kan_Knda-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 320586, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'kan_Knda-hin_Deva': {'num_samples': 1024, 'number_of_characters': 331252, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'kan_Knda-kas_Arab': {'num_samples': 1024, 'number_of_characters': 333910, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'kan_Knda-mai_Deva': {'num_samples': 1024, 'number_of_characters': 319828, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'kan_Knda-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 354782, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'kan_Knda-mar_Deva': {'num_samples': 1024, 'number_of_characters': 332930, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'kan_Knda-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 324280, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'kan_Knda-npi_Deva': {'num_samples': 1024, 'number_of_characters': 324565, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'kan_Knda-ory_Orya': {'num_samples': 1024, 'number_of_characters': 345372, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'kan_Knda-pan_Guru': {'num_samples': 1024, 'number_of_characters': 318009, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'kan_Knda-san_Deva': {'num_samples': 1024, 'number_of_characters': 329225, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'kan_Knda-sat_Olck': {'num_samples': 1024, 'number_of_characters': 337878, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'kan_Knda-snd_Deva': {'num_samples': 1024, 'number_of_characters': 331567, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'kan_Knda-tam_Taml': {'num_samples': 1024, 'number_of_characters': 359492, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'kan_Knda-tel_Telu': {'num_samples': 1024, 'number_of_characters': 330191, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'kan_Knda-urd_Arab': {'num_samples': 1024, 'number_of_characters': 326280, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'kas_Arab-asm_Beng': {'num_samples': 1024, 'number_of_characters': 322764, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'kas_Arab-ben_Beng': {'num_samples': 1024, 'number_of_characters': 312468, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'kas_Arab-brx_Deva': {'num_samples': 1024, 'number_of_characters': 325455, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'kas_Arab-doi_Deva': {'num_samples': 1024, 'number_of_characters': 320866, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'kas_Arab-eng_Latn': {'num_samples': 1024, 'number_of_characters': 321944, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'kas_Arab-gom_Deva': {'num_samples': 1024, 'number_of_characters': 314440, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'kas_Arab-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 311286, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'kas_Arab-hin_Deva': {'num_samples': 1024, 'number_of_characters': 321952, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'kas_Arab-kan_Knda': {'num_samples': 1024, 'number_of_characters': 333910, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'kas_Arab-mai_Deva': {'num_samples': 1024, 'number_of_characters': 310528, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'kas_Arab-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 345482, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'kas_Arab-mar_Deva': {'num_samples': 1024, 'number_of_characters': 323630, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'kas_Arab-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 314980, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'kas_Arab-npi_Deva': {'num_samples': 1024, 'number_of_characters': 315265, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'kas_Arab-ory_Orya': {'num_samples': 1024, 'number_of_characters': 336072, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'kas_Arab-pan_Guru': {'num_samples': 1024, 'number_of_characters': 308709, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'kas_Arab-san_Deva': {'num_samples': 1024, 'number_of_characters': 319925, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'kas_Arab-sat_Olck': {'num_samples': 1024, 'number_of_characters': 328578, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'kas_Arab-snd_Deva': {'num_samples': 1024, 'number_of_characters': 322267, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'kas_Arab-tam_Taml': {'num_samples': 1024, 'number_of_characters': 350192, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'kas_Arab-tel_Telu': {'num_samples': 1024, 'number_of_characters': 320891, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'kas_Arab-urd_Arab': {'num_samples': 1024, 'number_of_characters': 316980, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mai_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 308682, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mai_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 298386, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mai_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 311373, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mai_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 306784, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mai_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 307862, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mai_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 300358, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mai_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 297204, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mai_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 307870, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mai_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 319828, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mai_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 310528, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mai_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 331400, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'mai_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 309548, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'mai_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 300898, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'mai_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 301183, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mai_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 321990, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mai_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 294627, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mai_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 305843, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mai_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 314496, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mai_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 308185, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mai_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 336110, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mai_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 306809, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mai_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 302898, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mal_Mlym-asm_Beng': {'num_samples': 1024, 'number_of_characters': 343636, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mal_Mlym-ben_Beng': {'num_samples': 1024, 'number_of_characters': 333340, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mal_Mlym-brx_Deva': {'num_samples': 1024, 'number_of_characters': 346327, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mal_Mlym-doi_Deva': {'num_samples': 1024, 'number_of_characters': 341738, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mal_Mlym-eng_Latn': {'num_samples': 1024, 'number_of_characters': 342816, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mal_Mlym-gom_Deva': {'num_samples': 1024, 'number_of_characters': 335312, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mal_Mlym-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 332158, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mal_Mlym-hin_Deva': {'num_samples': 1024, 'number_of_characters': 342824, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mal_Mlym-kan_Knda': {'num_samples': 1024, 'number_of_characters': 354782, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mal_Mlym-kas_Arab': {'num_samples': 1024, 'number_of_characters': 345482, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mal_Mlym-mai_Deva': {'num_samples': 1024, 'number_of_characters': 331400, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'mal_Mlym-mar_Deva': {'num_samples': 1024, 'number_of_characters': 344502, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'mal_Mlym-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 335852, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'mal_Mlym-npi_Deva': {'num_samples': 1024, 'number_of_characters': 336137, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mal_Mlym-ory_Orya': {'num_samples': 1024, 'number_of_characters': 356944, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mal_Mlym-pan_Guru': {'num_samples': 1024, 'number_of_characters': 329581, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mal_Mlym-san_Deva': {'num_samples': 1024, 'number_of_characters': 340797, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mal_Mlym-sat_Olck': {'num_samples': 1024, 'number_of_characters': 349450, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mal_Mlym-snd_Deva': {'num_samples': 1024, 'number_of_characters': 343139, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mal_Mlym-tam_Taml': {'num_samples': 1024, 'number_of_characters': 371064, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mal_Mlym-tel_Telu': {'num_samples': 1024, 'number_of_characters': 341763, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mal_Mlym-urd_Arab': {'num_samples': 1024, 'number_of_characters': 337852, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mar_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 321784, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mar_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 311488, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mar_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 324475, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mar_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 319886, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mar_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 320964, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mar_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 313460, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mar_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 310306, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mar_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 320972, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mar_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 332930, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mar_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 323630, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mar_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 309548, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'mar_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 344502, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'mar_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 314000, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'mar_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 314285, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mar_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 335092, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mar_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 307729, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mar_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 318945, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mar_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 327598, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mar_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 321287, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mar_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 349212, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mar_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 319911, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mar_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 316000, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mni_Mtei-asm_Beng': {'num_samples': 1024, 'number_of_characters': 313134, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mni_Mtei-ben_Beng': {'num_samples': 1024, 'number_of_characters': 302838, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mni_Mtei-brx_Deva': {'num_samples': 1024, 'number_of_characters': 315825, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mni_Mtei-doi_Deva': {'num_samples': 1024, 'number_of_characters': 311236, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mni_Mtei-eng_Latn': {'num_samples': 1024, 'number_of_characters': 312314, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mni_Mtei-gom_Deva': {'num_samples': 1024, 'number_of_characters': 304810, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mni_Mtei-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 301656, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mni_Mtei-hin_Deva': {'num_samples': 1024, 'number_of_characters': 312322, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mni_Mtei-kan_Knda': {'num_samples': 1024, 'number_of_characters': 324280, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mni_Mtei-kas_Arab': {'num_samples': 1024, 'number_of_characters': 314980, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mni_Mtei-mai_Deva': {'num_samples': 1024, 'number_of_characters': 300898, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'mni_Mtei-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 335852, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'mni_Mtei-mar_Deva': {'num_samples': 1024, 'number_of_characters': 314000, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'mni_Mtei-npi_Deva': {'num_samples': 1024, 'number_of_characters': 305635, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mni_Mtei-ory_Orya': {'num_samples': 1024, 'number_of_characters': 326442, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mni_Mtei-pan_Guru': {'num_samples': 1024, 'number_of_characters': 299079, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mni_Mtei-san_Deva': {'num_samples': 1024, 'number_of_characters': 310295, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mni_Mtei-sat_Olck': {'num_samples': 1024, 'number_of_characters': 318948, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mni_Mtei-snd_Deva': {'num_samples': 1024, 'number_of_characters': 312637, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mni_Mtei-tam_Taml': {'num_samples': 1024, 'number_of_characters': 340562, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mni_Mtei-tel_Telu': {'num_samples': 1024, 'number_of_characters': 311261, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mni_Mtei-urd_Arab': {'num_samples': 1024, 'number_of_characters': 307350, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'npi_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 313419, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'npi_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 303123, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'npi_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 316110, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'npi_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 311521, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'npi_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 312599, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'npi_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 305095, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'npi_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 301941, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'npi_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 312607, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'npi_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 324565, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'npi_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 315265, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'npi_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 301183, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'npi_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 336137, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'npi_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 314285, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'npi_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 305635, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'npi_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 326727, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'npi_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 299364, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'npi_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 310580, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'npi_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 319233, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'npi_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 312922, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'npi_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 340847, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'npi_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 311546, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'npi_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 307635, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'ory_Orya-asm_Beng': {'num_samples': 1024, 'number_of_characters': 334226, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'ory_Orya-ben_Beng': {'num_samples': 1024, 'number_of_characters': 323930, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'ory_Orya-brx_Deva': {'num_samples': 1024, 'number_of_characters': 336917, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'ory_Orya-doi_Deva': {'num_samples': 1024, 'number_of_characters': 332328, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'ory_Orya-eng_Latn': {'num_samples': 1024, 'number_of_characters': 333406, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'ory_Orya-gom_Deva': {'num_samples': 1024, 'number_of_characters': 325902, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'ory_Orya-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 322748, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'ory_Orya-hin_Deva': {'num_samples': 1024, 'number_of_characters': 333414, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'ory_Orya-kan_Knda': {'num_samples': 1024, 'number_of_characters': 345372, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'ory_Orya-kas_Arab': {'num_samples': 1024, 'number_of_characters': 336072, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'ory_Orya-mai_Deva': {'num_samples': 1024, 'number_of_characters': 321990, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'ory_Orya-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 356944, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'ory_Orya-mar_Deva': {'num_samples': 1024, 'number_of_characters': 335092, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'ory_Orya-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 326442, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'ory_Orya-npi_Deva': {'num_samples': 1024, 'number_of_characters': 326727, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'ory_Orya-pan_Guru': {'num_samples': 1024, 'number_of_characters': 320171, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'ory_Orya-san_Deva': {'num_samples': 1024, 'number_of_characters': 331387, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'ory_Orya-sat_Olck': {'num_samples': 1024, 'number_of_characters': 340040, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'ory_Orya-snd_Deva': {'num_samples': 1024, 'number_of_characters': 333729, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'ory_Orya-tam_Taml': {'num_samples': 1024, 'number_of_characters': 361654, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'ory_Orya-tel_Telu': {'num_samples': 1024, 'number_of_characters': 332353, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'ory_Orya-urd_Arab': {'num_samples': 1024, 'number_of_characters': 328442, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'pan_Guru-asm_Beng': {'num_samples': 1024, 'number_of_characters': 306863, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'pan_Guru-ben_Beng': {'num_samples': 1024, 'number_of_characters': 296567, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'pan_Guru-brx_Deva': {'num_samples': 1024, 'number_of_characters': 309554, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'pan_Guru-doi_Deva': {'num_samples': 1024, 'number_of_characters': 304965, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'pan_Guru-eng_Latn': {'num_samples': 1024, 'number_of_characters': 306043, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'pan_Guru-gom_Deva': {'num_samples': 1024, 'number_of_characters': 298539, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'pan_Guru-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 295385, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'pan_Guru-hin_Deva': {'num_samples': 1024, 'number_of_characters': 306051, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'pan_Guru-kan_Knda': {'num_samples': 1024, 'number_of_characters': 318009, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'pan_Guru-kas_Arab': {'num_samples': 1024, 'number_of_characters': 308709, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'pan_Guru-mai_Deva': {'num_samples': 1024, 'number_of_characters': 294627, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'pan_Guru-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 329581, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'pan_Guru-mar_Deva': {'num_samples': 1024, 'number_of_characters': 307729, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'pan_Guru-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 299079, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'pan_Guru-npi_Deva': {'num_samples': 1024, 'number_of_characters': 299364, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'pan_Guru-ory_Orya': {'num_samples': 1024, 'number_of_characters': 320171, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'pan_Guru-san_Deva': {'num_samples': 1024, 'number_of_characters': 304024, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'pan_Guru-sat_Olck': {'num_samples': 1024, 'number_of_characters': 312677, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'pan_Guru-snd_Deva': {'num_samples': 1024, 'number_of_characters': 306366, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'pan_Guru-tam_Taml': {'num_samples': 1024, 'number_of_characters': 334291, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'pan_Guru-tel_Telu': {'num_samples': 1024, 'number_of_characters': 304990, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'pan_Guru-urd_Arab': {'num_samples': 1024, 'number_of_characters': 301079, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'san_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 318079, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'san_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 307783, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'san_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 320770, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'san_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 316181, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'san_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 317259, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'san_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 309755, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'san_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 306601, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'san_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 317267, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'san_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 329225, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'san_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 319925, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'san_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 305843, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'san_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 340797, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'san_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 318945, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'san_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 310295, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'san_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 310580, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'san_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 331387, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'san_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 304024, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'san_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 323893, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'san_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 317582, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'san_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 345507, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'san_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 316206, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'san_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 312295, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'sat_Olck-asm_Beng': {'num_samples': 1024, 'number_of_characters': 326732, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'sat_Olck-ben_Beng': {'num_samples': 1024, 'number_of_characters': 316436, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'sat_Olck-brx_Deva': {'num_samples': 1024, 'number_of_characters': 329423, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'sat_Olck-doi_Deva': {'num_samples': 1024, 'number_of_characters': 324834, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'sat_Olck-eng_Latn': {'num_samples': 1024, 'number_of_characters': 325912, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'sat_Olck-gom_Deva': {'num_samples': 1024, 'number_of_characters': 318408, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'sat_Olck-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 315254, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'sat_Olck-hin_Deva': {'num_samples': 1024, 'number_of_characters': 325920, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'sat_Olck-kan_Knda': {'num_samples': 1024, 'number_of_characters': 337878, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'sat_Olck-kas_Arab': {'num_samples': 1024, 'number_of_characters': 328578, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'sat_Olck-mai_Deva': {'num_samples': 1024, 'number_of_characters': 314496, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'sat_Olck-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 349450, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'sat_Olck-mar_Deva': {'num_samples': 1024, 'number_of_characters': 327598, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'sat_Olck-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 318948, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'sat_Olck-npi_Deva': {'num_samples': 1024, 'number_of_characters': 319233, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'sat_Olck-ory_Orya': {'num_samples': 1024, 'number_of_characters': 340040, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'sat_Olck-pan_Guru': {'num_samples': 1024, 'number_of_characters': 312677, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'sat_Olck-san_Deva': {'num_samples': 1024, 'number_of_characters': 323893, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'sat_Olck-snd_Deva': {'num_samples': 1024, 'number_of_characters': 326235, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'sat_Olck-tam_Taml': {'num_samples': 1024, 'number_of_characters': 354160, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'sat_Olck-tel_Telu': {'num_samples': 1024, 'number_of_characters': 324859, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'sat_Olck-urd_Arab': {'num_samples': 1024, 'number_of_characters': 320948, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'snd_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 320421, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'snd_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 310125, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'snd_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 323112, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'snd_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 318523, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'snd_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 319601, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'snd_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 312097, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'snd_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 308943, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'snd_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 319609, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'snd_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 331567, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'snd_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 322267, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'snd_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 308185, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'snd_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 343139, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'snd_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 321287, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'snd_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 312637, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'snd_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 312922, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'snd_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 333729, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'snd_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306366, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'snd_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 317582, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'snd_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 326235, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'snd_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 347849, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'snd_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 318548, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'snd_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 314637, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'tam_Taml-asm_Beng': {'num_samples': 1024, 'number_of_characters': 348346, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'tam_Taml-ben_Beng': {'num_samples': 1024, 'number_of_characters': 338050, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'tam_Taml-brx_Deva': {'num_samples': 1024, 'number_of_characters': 351037, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'tam_Taml-doi_Deva': {'num_samples': 1024, 'number_of_characters': 346448, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'tam_Taml-eng_Latn': {'num_samples': 1024, 'number_of_characters': 347526, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'tam_Taml-gom_Deva': {'num_samples': 1024, 'number_of_characters': 340022, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'tam_Taml-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 336868, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'tam_Taml-hin_Deva': {'num_samples': 1024, 'number_of_characters': 347534, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'tam_Taml-kan_Knda': {'num_samples': 1024, 'number_of_characters': 359492, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'tam_Taml-kas_Arab': {'num_samples': 1024, 'number_of_characters': 350192, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'tam_Taml-mai_Deva': {'num_samples': 1024, 'number_of_characters': 336110, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'tam_Taml-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 371064, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'tam_Taml-mar_Deva': {'num_samples': 1024, 'number_of_characters': 349212, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'tam_Taml-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 340562, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'tam_Taml-npi_Deva': {'num_samples': 1024, 'number_of_characters': 340847, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'tam_Taml-ory_Orya': {'num_samples': 1024, 'number_of_characters': 361654, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'tam_Taml-pan_Guru': {'num_samples': 1024, 'number_of_characters': 334291, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'tam_Taml-san_Deva': {'num_samples': 1024, 'number_of_characters': 345507, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'tam_Taml-sat_Olck': {'num_samples': 1024, 'number_of_characters': 354160, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'tam_Taml-snd_Deva': {'num_samples': 1024, 'number_of_characters': 347849, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'tam_Taml-tel_Telu': {'num_samples': 1024, 'number_of_characters': 346473, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'tam_Taml-urd_Arab': {'num_samples': 1024, 'number_of_characters': 342562, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'tel_Telu-asm_Beng': {'num_samples': 1024, 'number_of_characters': 319045, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'tel_Telu-ben_Beng': {'num_samples': 1024, 'number_of_characters': 308749, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'tel_Telu-brx_Deva': {'num_samples': 1024, 'number_of_characters': 321736, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'tel_Telu-doi_Deva': {'num_samples': 1024, 'number_of_characters': 317147, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'tel_Telu-eng_Latn': {'num_samples': 1024, 'number_of_characters': 318225, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'tel_Telu-gom_Deva': {'num_samples': 1024, 'number_of_characters': 310721, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'tel_Telu-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 307567, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'tel_Telu-hin_Deva': {'num_samples': 1024, 'number_of_characters': 318233, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'tel_Telu-kan_Knda': {'num_samples': 1024, 'number_of_characters': 330191, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'tel_Telu-kas_Arab': {'num_samples': 1024, 'number_of_characters': 320891, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'tel_Telu-mai_Deva': {'num_samples': 1024, 'number_of_characters': 306809, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'tel_Telu-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 341763, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'tel_Telu-mar_Deva': {'num_samples': 1024, 'number_of_characters': 319911, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'tel_Telu-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 311261, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'tel_Telu-npi_Deva': {'num_samples': 1024, 'number_of_characters': 311546, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'tel_Telu-ory_Orya': {'num_samples': 1024, 'number_of_characters': 332353, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'tel_Telu-pan_Guru': {'num_samples': 1024, 'number_of_characters': 304990, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'tel_Telu-san_Deva': {'num_samples': 1024, 'number_of_characters': 316206, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'tel_Telu-sat_Olck': {'num_samples': 1024, 'number_of_characters': 324859, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'tel_Telu-snd_Deva': {'num_samples': 1024, 'number_of_characters': 318548, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'tel_Telu-tam_Taml': {'num_samples': 1024, 'number_of_characters': 346473, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'tel_Telu-urd_Arab': {'num_samples': 1024, 'number_of_characters': 313261, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'urd_Arab-asm_Beng': {'num_samples': 1024, 'number_of_characters': 315134, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'urd_Arab-ben_Beng': {'num_samples': 1024, 'number_of_characters': 304838, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'urd_Arab-brx_Deva': {'num_samples': 1024, 'number_of_characters': 317825, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'urd_Arab-doi_Deva': {'num_samples': 1024, 'number_of_characters': 313236, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'urd_Arab-eng_Latn': {'num_samples': 1024, 'number_of_characters': 314314, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'urd_Arab-gom_Deva': {'num_samples': 1024, 'number_of_characters': 306810, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'urd_Arab-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 303656, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'urd_Arab-hin_Deva': {'num_samples': 1024, 'number_of_characters': 314322, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'urd_Arab-kan_Knda': {'num_samples': 1024, 'number_of_characters': 326280, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'urd_Arab-kas_Arab': {'num_samples': 1024, 'number_of_characters': 316980, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'urd_Arab-mai_Deva': {'num_samples': 1024, 'number_of_characters': 302898, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'urd_Arab-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 337852, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'urd_Arab-mar_Deva': {'num_samples': 1024, 'number_of_characters': 316000, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'urd_Arab-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 307350, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'urd_Arab-npi_Deva': {'num_samples': 1024, 'number_of_characters': 307635, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'urd_Arab-ory_Orya': {'num_samples': 1024, 'number_of_characters': 328442, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'urd_Arab-pan_Guru': {'num_samples': 1024, 'number_of_characters': 301079, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'urd_Arab-san_Deva': {'num_samples': 1024, 'number_of_characters': 312295, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'urd_Arab-sat_Olck': {'num_samples': 1024, 'number_of_characters': 320948, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'urd_Arab-snd_Deva': {'num_samples': 1024, 'number_of_characters': 314637, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'urd_Arab-tam_Taml': {'num_samples': 1024, 'number_of_characters': 342562, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'urd_Arab-tel_Telu': {'num_samples': 1024, 'number_of_characters': 313261, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}}}} | +| [IWSLT2017BitextMining](https://aclanthology.org/2017.iwslt-1.1/) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'ita', 'jpn', 'kor', 'nld', 'ron'] | BitextMining | s2s | [Non-fiction, Fiction, Written] | {'validation': 21938} | {'validation': {'num_samples': 21938, 'number_of_characters': 4256244, 'unique_pairs': 21840, 'min_sentence1_length': 2, 'average_sentence1_length': 97.01, 'max_sentence1_length': 521, 'unique_sentence1': 11563, 'min_sentence2_length': 2, 'average_sentence2_length': 97.01, 'max_sentence2_length': 521, 'unique_sentence2': 11563, 'hf_subset_descriptive_stats': {'ar-en': {'num_samples': 888, 'number_of_characters': 172499, 'unique_pairs': 887, 'min_sentence1_length': 4, 'average_sentence1_length': 85.49, 'max_sentence1_length': 369, 'unique_sentence1': 887, 'min_sentence2_length': 10, 'average_sentence2_length': 108.77, 'max_sentence2_length': 462, 'unique_sentence2': 881}, 'de-en': {'num_samples': 888, 'number_of_characters': 202336, 'unique_pairs': 883, 'min_sentence1_length': 6, 'average_sentence1_length': 119.03, 'max_sentence1_length': 521, 'unique_sentence1': 881, 'min_sentence2_length': 10, 'average_sentence2_length': 108.83, 'max_sentence2_length': 462, 'unique_sentence2': 881}, 'en-ar': {'num_samples': 888, 'number_of_characters': 172499, 'unique_pairs': 887, 'min_sentence1_length': 10, 'average_sentence1_length': 108.77, 'max_sentence1_length': 462, 'unique_sentence1': 881, 'min_sentence2_length': 4, 'average_sentence2_length': 85.49, 'max_sentence2_length': 369, 'unique_sentence2': 887}, 'en-de': {'num_samples': 888, 'number_of_characters': 202336, 'unique_pairs': 883, 'min_sentence1_length': 10, 'average_sentence1_length': 108.83, 'max_sentence1_length': 462, 'unique_sentence1': 881, 'min_sentence2_length': 6, 'average_sentence2_length': 119.03, 'max_sentence2_length': 521, 'unique_sentence2': 881}, 'en-fr': {'num_samples': 890, 'number_of_characters': 197619, 'unique_pairs': 883, 'min_sentence1_length': 10, 'average_sentence1_length': 108.41, 'max_sentence1_length': 462, 'unique_sentence1': 883, 'min_sentence2_length': 6, 'average_sentence2_length': 113.63, 'max_sentence2_length': 493, 'unique_sentence2': 881}, 'en-it': {'num_samples': 929, 'number_of_characters': 191803, 'unique_pairs': 924, 'min_sentence1_length': 10, 'average_sentence1_length': 103.0, 'max_sentence1_length': 433, 'unique_sentence1': 922, 'min_sentence2_length': 7, 'average_sentence2_length': 103.46, 'max_sentence2_length': 444, 'unique_sentence2': 918}, 'en-ja': {'num_samples': 871, 'number_of_characters': 132742, 'unique_pairs': 867, 'min_sentence1_length': 10, 'average_sentence1_length': 109.81, 'max_sentence1_length': 462, 'unique_sentence1': 864, 'min_sentence2_length': 5, 'average_sentence2_length': 42.59, 'max_sentence2_length': 225, 'unique_sentence2': 866}, 'en-ko': {'num_samples': 879, 'number_of_characters': 142659, 'unique_pairs': 874, 'min_sentence1_length': 10, 'average_sentence1_length': 107.74, 'max_sentence1_length': 462, 'unique_sentence1': 872, 'min_sentence2_length': 3, 'average_sentence2_length': 54.56, 'max_sentence2_length': 250, 'unique_sentence2': 872}, 'en-nl': {'num_samples': 1003, 'number_of_characters': 189637, 'unique_pairs': 1000, 'min_sentence1_length': 10, 'average_sentence1_length': 95.27, 'max_sentence1_length': 433, 'unique_sentence1': 996, 'min_sentence2_length': 4, 'average_sentence2_length': 93.8, 'max_sentence2_length': 477, 'unique_sentence2': 1000}, 'en-ro': {'num_samples': 914, 'number_of_characters': 194128, 'unique_pairs': 910, 'min_sentence1_length': 10, 'average_sentence1_length': 104.72, 'max_sentence1_length': 433, 'unique_sentence1': 907, 'min_sentence2_length': 9, 'average_sentence2_length': 107.67, 'max_sentence2_length': 448, 'unique_sentence2': 910}, 'en-zh': {'num_samples': 879, 'number_of_characters': 131126, 'unique_pairs': 877, 'min_sentence1_length': 10, 'average_sentence1_length': 109.37, 'max_sentence1_length': 462, 'unique_sentence1': 872, 'min_sentence2_length': 2, 'average_sentence2_length': 39.81, 'max_sentence2_length': 230, 'unique_sentence2': 867}, 'fr-en': {'num_samples': 890, 'number_of_characters': 197619, 'unique_pairs': 883, 'min_sentence1_length': 6, 'average_sentence1_length': 113.63, 'max_sentence1_length': 493, 'unique_sentence1': 881, 'min_sentence2_length': 10, 'average_sentence2_length': 108.41, 'max_sentence2_length': 462, 'unique_sentence2': 883}, 'it-en': {'num_samples': 929, 'number_of_characters': 191803, 'unique_pairs': 924, 'min_sentence1_length': 7, 'average_sentence1_length': 103.46, 'max_sentence1_length': 444, 'unique_sentence1': 918, 'min_sentence2_length': 10, 'average_sentence2_length': 103.0, 'max_sentence2_length': 433, 'unique_sentence2': 922}, 'it-nl': {'num_samples': 1001, 'number_of_characters': 188858, 'unique_pairs': 998, 'min_sentence1_length': 7, 'average_sentence1_length': 94.64, 'max_sentence1_length': 459, 'unique_sentence1': 994, 'min_sentence2_length': 7, 'average_sentence2_length': 94.03, 'max_sentence2_length': 505, 'unique_sentence2': 998}, 'it-ro': {'num_samples': 914, 'number_of_characters': 193339, 'unique_pairs': 911, 'min_sentence1_length': 7, 'average_sentence1_length': 103.91, 'max_sentence1_length': 435, 'unique_sentence1': 907, 'min_sentence2_length': 9, 'average_sentence2_length': 107.62, 'max_sentence2_length': 448, 'unique_sentence2': 910}, 'ja-en': {'num_samples': 871, 'number_of_characters': 132742, 'unique_pairs': 867, 'min_sentence1_length': 5, 'average_sentence1_length': 42.59, 'max_sentence1_length': 225, 'unique_sentence1': 866, 'min_sentence2_length': 10, 'average_sentence2_length': 109.81, 'max_sentence2_length': 462, 'unique_sentence2': 864}, 'ko-en': {'num_samples': 879, 'number_of_characters': 142659, 'unique_pairs': 874, 'min_sentence1_length': 3, 'average_sentence1_length': 54.56, 'max_sentence1_length': 250, 'unique_sentence1': 872, 'min_sentence2_length': 10, 'average_sentence2_length': 107.74, 'max_sentence2_length': 462, 'unique_sentence2': 872}, 'nl-en': {'num_samples': 1003, 'number_of_characters': 189637, 'unique_pairs': 1000, 'min_sentence1_length': 4, 'average_sentence1_length': 93.8, 'max_sentence1_length': 477, 'unique_sentence1': 1000, 'min_sentence2_length': 10, 'average_sentence2_length': 95.27, 'max_sentence2_length': 433, 'unique_sentence2': 996}, 'nl-it': {'num_samples': 1001, 'number_of_characters': 188858, 'unique_pairs': 998, 'min_sentence1_length': 7, 'average_sentence1_length': 94.03, 'max_sentence1_length': 505, 'unique_sentence1': 998, 'min_sentence2_length': 7, 'average_sentence2_length': 94.64, 'max_sentence2_length': 459, 'unique_sentence2': 994}, 'nl-ro': {'num_samples': 913, 'number_of_characters': 191376, 'unique_pairs': 911, 'min_sentence1_length': 7, 'average_sentence1_length': 102.02, 'max_sentence1_length': 478, 'unique_sentence1': 909, 'min_sentence2_length': 9, 'average_sentence2_length': 107.59, 'max_sentence2_length': 515, 'unique_sentence2': 909}, 'ro-en': {'num_samples': 914, 'number_of_characters': 194128, 'unique_pairs': 910, 'min_sentence1_length': 9, 'average_sentence1_length': 107.67, 'max_sentence1_length': 448, 'unique_sentence1': 910, 'min_sentence2_length': 10, 'average_sentence2_length': 104.72, 'max_sentence2_length': 433, 'unique_sentence2': 907}, 'ro-it': {'num_samples': 914, 'number_of_characters': 193339, 'unique_pairs': 911, 'min_sentence1_length': 9, 'average_sentence1_length': 107.62, 'max_sentence1_length': 448, 'unique_sentence1': 910, 'min_sentence2_length': 7, 'average_sentence2_length': 103.91, 'max_sentence2_length': 435, 'unique_sentence2': 907}, 'ro-nl': {'num_samples': 913, 'number_of_characters': 191376, 'unique_pairs': 911, 'min_sentence1_length': 9, 'average_sentence1_length': 107.59, 'max_sentence1_length': 515, 'unique_sentence1': 909, 'min_sentence2_length': 7, 'average_sentence2_length': 102.02, 'max_sentence2_length': 478, 'unique_sentence2': 909}, 'zh-en': {'num_samples': 879, 'number_of_characters': 131126, 'unique_pairs': 877, 'min_sentence1_length': 2, 'average_sentence1_length': 39.81, 'max_sentence1_length': 230, 'unique_sentence1': 867, 'min_sentence2_length': 10, 'average_sentence2_length': 109.37, 'max_sentence2_length': 462, 'unique_sentence2': 872}}}} | +| [ImdbClassification](http://www.aclweb.org/anthology/P11-1015) | ['eng'] | Classification | p2p | [Reviews, Written] | None | None | +| [InappropriatenessClassification](https://aclanthology.org/2021.bsnlp-1.4) | ['rus'] | Classification | s2s | [Web, Social, Written] | None | None | +| [IndicCrosslingualSTS](https://huggingface.co/datasets/jaygala24/indic_sts) (Ramesh et al., 2022) | ['asm', 'ben', 'eng', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel', 'urd'] | STS | s2s | [News, Non-fiction, Web, Spoken, Government, Written, Spoken] | None | None | +| [IndicGenBenchFloresBitextMining](https://github.com/google-research-datasets/indic-gen-bench/) (Harman Singh, 2024) | ['asm', 'awa', 'ben', 'bgc', 'bho', 'bod', 'boy', 'eng', 'gbm', 'gom', 'guj', 'hin', 'hne', 'kan', 'mai', 'mal', 'mar', 'mni', 'mup', 'mwr', 'nep', 'ory', 'pan', 'pus', 'raj', 'san', 'sat', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Web, News, Written] | {'validation': 57826, 'test': 58696} | {'validation': {'num_samples': 57826, 'number_of_characters': 14600950, 'unique_pairs': 57826, 'min_sentence1_length': 24, 'average_sentence1_length': 126.25, 'max_sentence1_length': 368, 'unique_sentence1': 29903, 'min_sentence2_length': 24, 'average_sentence2_length': 126.24, 'max_sentence2_length': 368, 'unique_sentence2': 29903, 'hf_subset_descriptive_stats': {'ben-eng': {'num_samples': 997, 'number_of_characters': 248469, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 123.65, 'max_sentence1_length': 320, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-ben': {'num_samples': 997, 'number_of_characters': 248469, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 123.65, 'max_sentence2_length': 320, 'unique_sentence2': 997}, 'guj-eng': {'num_samples': 997, 'number_of_characters': 245477, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 120.64, 'max_sentence1_length': 368, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-guj': {'num_samples': 997, 'number_of_characters': 245477, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 120.64, 'max_sentence2_length': 368, 'unique_sentence2': 997}, 'hin-eng': {'num_samples': 997, 'number_of_characters': 250573, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 125.76, 'max_sentence1_length': 355, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-hin': {'num_samples': 997, 'number_of_characters': 250564, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 125.75, 'max_sentence2_length': 355, 'unique_sentence2': 997}, 'kan-eng': {'num_samples': 997, 'number_of_characters': 257131, 'unique_pairs': 997, 'min_sentence1_length': 34, 'average_sentence1_length': 132.33, 'max_sentence1_length': 331, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-kan': {'num_samples': 997, 'number_of_characters': 256986, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 34, 'average_sentence2_length': 132.19, 'max_sentence2_length': 331, 'unique_sentence2': 997}, 'mal-eng': {'num_samples': 997, 'number_of_characters': 267295, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 142.53, 'max_sentence1_length': 360, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mal': {'num_samples': 997, 'number_of_characters': 267296, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 142.53, 'max_sentence2_length': 360, 'unique_sentence2': 997}, 'mar-eng': {'num_samples': 997, 'number_of_characters': 251107, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 126.29, 'max_sentence1_length': 321, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mar': {'num_samples': 997, 'number_of_characters': 250897, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 126.08, 'max_sentence2_length': 321, 'unique_sentence2': 997}, 'tam-eng': {'num_samples': 997, 'number_of_characters': 271322, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 146.57, 'max_sentence1_length': 358, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-tam': {'num_samples': 997, 'number_of_characters': 271322, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 146.57, 'max_sentence2_length': 358, 'unique_sentence2': 997}, 'tel-eng': {'num_samples': 997, 'number_of_characters': 252385, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 127.57, 'max_sentence1_length': 317, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-tel': {'num_samples': 997, 'number_of_characters': 252380, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 127.57, 'max_sentence2_length': 317, 'unique_sentence2': 997}, 'urd-eng': {'num_samples': 997, 'number_of_characters': 249824, 'unique_pairs': 997, 'min_sentence1_length': 37, 'average_sentence1_length': 125.01, 'max_sentence1_length': 295, 'unique_sentence1': 996, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-urd': {'num_samples': 997, 'number_of_characters': 249824, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 37, 'average_sentence2_length': 125.01, 'max_sentence2_length': 295, 'unique_sentence2': 996}, 'asm-eng': {'num_samples': 997, 'number_of_characters': 246220, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 121.39, 'max_sentence1_length': 314, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-asm': {'num_samples': 997, 'number_of_characters': 246224, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 121.39, 'max_sentence2_length': 314, 'unique_sentence2': 997}, 'bho-eng': {'num_samples': 997, 'number_of_characters': 246895, 'unique_pairs': 997, 'min_sentence1_length': 25, 'average_sentence1_length': 122.07, 'max_sentence1_length': 326, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-bho': {'num_samples': 997, 'number_of_characters': 246919, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 25, 'average_sentence2_length': 122.09, 'max_sentence2_length': 326, 'unique_sentence2': 997}, 'nep-eng': {'num_samples': 997, 'number_of_characters': 245984, 'unique_pairs': 997, 'min_sentence1_length': 24, 'average_sentence1_length': 121.15, 'max_sentence1_length': 307, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-nep': {'num_samples': 997, 'number_of_characters': 245984, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 24, 'average_sentence2_length': 121.15, 'max_sentence2_length': 307, 'unique_sentence2': 997}, 'ory-eng': {'num_samples': 997, 'number_of_characters': 254206, 'unique_pairs': 997, 'min_sentence1_length': 34, 'average_sentence1_length': 129.4, 'max_sentence1_length': 308, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-ory': {'num_samples': 997, 'number_of_characters': 254206, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 34, 'average_sentence2_length': 129.4, 'max_sentence2_length': 308, 'unique_sentence2': 997}, 'pan-eng': {'num_samples': 997, 'number_of_characters': 251598, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 126.78, 'max_sentence1_length': 309, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-pan': {'num_samples': 997, 'number_of_characters': 251597, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 126.78, 'max_sentence2_length': 309, 'unique_sentence2': 997}, 'pus-eng': {'num_samples': 997, 'number_of_characters': 247450, 'unique_pairs': 997, 'min_sentence1_length': 32, 'average_sentence1_length': 122.62, 'max_sentence1_length': 300, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-pus': {'num_samples': 997, 'number_of_characters': 247450, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 32, 'average_sentence2_length': 122.62, 'max_sentence2_length': 300, 'unique_sentence2': 997}, 'san-eng': {'num_samples': 997, 'number_of_characters': 249042, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 124.22, 'max_sentence1_length': 311, 'unique_sentence1': 994, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-san': {'num_samples': 997, 'number_of_characters': 248877, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 124.06, 'max_sentence2_length': 311, 'unique_sentence2': 994}, 'awa-eng': {'num_samples': 997, 'number_of_characters': 247944, 'unique_pairs': 997, 'min_sentence1_length': 34, 'average_sentence1_length': 123.12, 'max_sentence1_length': 329, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-awa': {'num_samples': 997, 'number_of_characters': 247884, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 34, 'average_sentence2_length': 123.06, 'max_sentence2_length': 329, 'unique_sentence2': 997}, 'bgc-eng': {'num_samples': 997, 'number_of_characters': 245935, 'unique_pairs': 997, 'min_sentence1_length': 27, 'average_sentence1_length': 121.1, 'max_sentence1_length': 303, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-bgc': {'num_samples': 997, 'number_of_characters': 245935, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 27, 'average_sentence2_length': 121.1, 'max_sentence2_length': 303, 'unique_sentence2': 997}, 'bod-eng': {'num_samples': 997, 'number_of_characters': 266515, 'unique_pairs': 997, 'min_sentence1_length': 26, 'average_sentence1_length': 141.75, 'max_sentence1_length': 355, 'unique_sentence1': 996, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-bod': {'num_samples': 997, 'number_of_characters': 266495, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 26, 'average_sentence2_length': 141.73, 'max_sentence2_length': 355, 'unique_sentence2': 996}, 'boy-eng': {'num_samples': 997, 'number_of_characters': 260174, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 135.39, 'max_sentence1_length': 312, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-boy': {'num_samples': 997, 'number_of_characters': 260174, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 135.39, 'max_sentence2_length': 312, 'unique_sentence2': 997}, 'gbm-eng': {'num_samples': 997, 'number_of_characters': 247009, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 122.18, 'max_sentence1_length': 344, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-gbm': {'num_samples': 997, 'number_of_characters': 247009, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 122.18, 'max_sentence2_length': 344, 'unique_sentence2': 997}, 'gom-eng': {'num_samples': 997, 'number_of_characters': 244553, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 119.72, 'max_sentence1_length': 306, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-gom': {'num_samples': 997, 'number_of_characters': 244553, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 119.72, 'max_sentence2_length': 306, 'unique_sentence2': 997}, 'hne-eng': {'num_samples': 997, 'number_of_characters': 246416, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 121.59, 'max_sentence1_length': 321, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-hne': {'num_samples': 997, 'number_of_characters': 246405, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 121.58, 'max_sentence2_length': 321, 'unique_sentence2': 997}, 'raj-eng': {'num_samples': 997, 'number_of_characters': 249541, 'unique_pairs': 997, 'min_sentence1_length': 32, 'average_sentence1_length': 124.72, 'max_sentence1_length': 313, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-raj': {'num_samples': 997, 'number_of_characters': 249541, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 32, 'average_sentence2_length': 124.72, 'max_sentence2_length': 313, 'unique_sentence2': 997}, 'mai-eng': {'num_samples': 997, 'number_of_characters': 247991, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 123.17, 'max_sentence1_length': 312, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mai': {'num_samples': 997, 'number_of_characters': 247994, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 123.17, 'max_sentence2_length': 312, 'unique_sentence2': 997}, 'mni-eng': {'num_samples': 997, 'number_of_characters': 254308, 'unique_pairs': 997, 'min_sentence1_length': 39, 'average_sentence1_length': 129.5, 'max_sentence1_length': 310, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mni': {'num_samples': 997, 'number_of_characters': 254312, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 39, 'average_sentence2_length': 129.51, 'max_sentence2_length': 310, 'unique_sentence2': 997}, 'mup-eng': {'num_samples': 997, 'number_of_characters': 248486, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 123.66, 'max_sentence1_length': 312, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mup': {'num_samples': 997, 'number_of_characters': 248486, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 123.66, 'max_sentence2_length': 312, 'unique_sentence2': 997}, 'mwr-eng': {'num_samples': 997, 'number_of_characters': 248641, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 123.82, 'max_sentence1_length': 324, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mwr': {'num_samples': 997, 'number_of_characters': 248641, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 123.82, 'max_sentence2_length': 324, 'unique_sentence2': 997}, 'sat-eng': {'num_samples': 997, 'number_of_characters': 258279, 'unique_pairs': 997, 'min_sentence1_length': 37, 'average_sentence1_length': 133.49, 'max_sentence1_length': 333, 'unique_sentence1': 995, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-sat': {'num_samples': 997, 'number_of_characters': 258279, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 37, 'average_sentence2_length': 133.49, 'max_sentence2_length': 333, 'unique_sentence2': 995}}}, 'test': {'num_samples': 58696, 'number_of_characters': 15359416, 'unique_pairs': 58690, 'min_sentence1_length': 33, 'average_sentence1_length': 130.84, 'max_sentence1_length': 431, 'unique_sentence1': 30351, 'min_sentence2_length': 33, 'average_sentence2_length': 130.83, 'max_sentence2_length': 431, 'unique_sentence2': 30351, 'hf_subset_descriptive_stats': {'ben-eng': {'num_samples': 1012, 'number_of_characters': 261008, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 127.51, 'max_sentence1_length': 333, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-ben': {'num_samples': 1012, 'number_of_characters': 261008, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 127.51, 'max_sentence2_length': 333, 'unique_sentence2': 1012}, 'guj-eng': {'num_samples': 1012, 'number_of_characters': 258394, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 124.93, 'max_sentence1_length': 349, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-guj': {'num_samples': 1012, 'number_of_characters': 258394, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 124.93, 'max_sentence2_length': 349, 'unique_sentence2': 1012}, 'hin-eng': {'num_samples': 1012, 'number_of_characters': 263040, 'unique_pairs': 1012, 'min_sentence1_length': 41, 'average_sentence1_length': 129.52, 'max_sentence1_length': 381, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-hin': {'num_samples': 1012, 'number_of_characters': 263029, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 41, 'average_sentence2_length': 129.51, 'max_sentence2_length': 381, 'unique_sentence2': 1012}, 'kan-eng': {'num_samples': 1012, 'number_of_characters': 270091, 'unique_pairs': 1012, 'min_sentence1_length': 43, 'average_sentence1_length': 136.49, 'max_sentence1_length': 388, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-kan': {'num_samples': 1012, 'number_of_characters': 270021, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 43, 'average_sentence2_length': 136.42, 'max_sentence2_length': 388, 'unique_sentence2': 1012}, 'mal-eng': {'num_samples': 1012, 'number_of_characters': 281302, 'unique_pairs': 1012, 'min_sentence1_length': 48, 'average_sentence1_length': 147.57, 'max_sentence1_length': 376, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mal': {'num_samples': 1012, 'number_of_characters': 281302, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 48, 'average_sentence2_length': 147.57, 'max_sentence2_length': 376, 'unique_sentence2': 1012}, 'mar-eng': {'num_samples': 1012, 'number_of_characters': 265212, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 131.67, 'max_sentence1_length': 356, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mar': {'num_samples': 1012, 'number_of_characters': 265023, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 131.48, 'max_sentence2_length': 355, 'unique_sentence2': 1012}, 'tam-eng': {'num_samples': 1012, 'number_of_characters': 286099, 'unique_pairs': 1012, 'min_sentence1_length': 48, 'average_sentence1_length': 152.31, 'max_sentence1_length': 404, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-tam': {'num_samples': 1012, 'number_of_characters': 286099, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 48, 'average_sentence2_length': 152.31, 'max_sentence2_length': 404, 'unique_sentence2': 1012}, 'tel-eng': {'num_samples': 1012, 'number_of_characters': 264460, 'unique_pairs': 1012, 'min_sentence1_length': 39, 'average_sentence1_length': 130.92, 'max_sentence1_length': 359, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-tel': {'num_samples': 1012, 'number_of_characters': 264447, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 39, 'average_sentence2_length': 130.91, 'max_sentence2_length': 359, 'unique_sentence2': 1012}, 'urd-eng': {'num_samples': 1012, 'number_of_characters': 261886, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 128.38, 'max_sentence1_length': 348, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-urd': {'num_samples': 1012, 'number_of_characters': 261885, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 128.38, 'max_sentence2_length': 348, 'unique_sentence2': 1012}, 'asm-eng': {'num_samples': 1012, 'number_of_characters': 257902, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 124.44, 'max_sentence1_length': 329, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-asm': {'num_samples': 1012, 'number_of_characters': 257909, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 124.45, 'max_sentence2_length': 329, 'unique_sentence2': 1012}, 'bho-eng': {'num_samples': 1012, 'number_of_characters': 260578, 'unique_pairs': 1012, 'min_sentence1_length': 36, 'average_sentence1_length': 127.09, 'max_sentence1_length': 367, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-bho': {'num_samples': 1012, 'number_of_characters': 260601, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 36, 'average_sentence2_length': 127.11, 'max_sentence2_length': 367, 'unique_sentence2': 1012}, 'nep-eng': {'num_samples': 1012, 'number_of_characters': 258869, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 125.4, 'max_sentence1_length': 362, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-nep': {'num_samples': 1012, 'number_of_characters': 258869, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 125.4, 'max_sentence2_length': 362, 'unique_sentence2': 1012}, 'ory-eng': {'num_samples': 1012, 'number_of_characters': 266805, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 133.24, 'max_sentence1_length': 354, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-ory': {'num_samples': 1012, 'number_of_characters': 266805, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 133.24, 'max_sentence2_length': 354, 'unique_sentence2': 1012}, 'pan-eng': {'num_samples': 1012, 'number_of_characters': 265391, 'unique_pairs': 1012, 'min_sentence1_length': 37, 'average_sentence1_length': 131.84, 'max_sentence1_length': 380, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-pan': {'num_samples': 1012, 'number_of_characters': 265391, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 37, 'average_sentence2_length': 131.84, 'max_sentence2_length': 380, 'unique_sentence2': 1012}, 'pus-eng': {'num_samples': 1012, 'number_of_characters': 254422, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 121.0, 'max_sentence1_length': 325, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-pus': {'num_samples': 1012, 'number_of_characters': 254421, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 121.0, 'max_sentence2_length': 325, 'unique_sentence2': 1012}, 'san-eng': {'num_samples': 1012, 'number_of_characters': 260339, 'unique_pairs': 1012, 'min_sentence1_length': 33, 'average_sentence1_length': 126.85, 'max_sentence1_length': 358, 'unique_sentence1': 1011, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-san': {'num_samples': 1012, 'number_of_characters': 260224, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 33, 'average_sentence2_length': 126.74, 'max_sentence2_length': 358, 'unique_sentence2': 1011}, 'awa-eng': {'num_samples': 1012, 'number_of_characters': 260179, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 126.69, 'max_sentence1_length': 378, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-awa': {'num_samples': 1012, 'number_of_characters': 260137, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 126.65, 'max_sentence2_length': 378, 'unique_sentence2': 1012}, 'bgc-eng': {'num_samples': 1012, 'number_of_characters': 257450, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 124.0, 'max_sentence1_length': 332, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-bgc': {'num_samples': 1012, 'number_of_characters': 257450, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 124.0, 'max_sentence2_length': 332, 'unique_sentence2': 1012}, 'bod-eng': {'num_samples': 1012, 'number_of_characters': 280188, 'unique_pairs': 1012, 'min_sentence1_length': 42, 'average_sentence1_length': 146.46, 'max_sentence1_length': 431, 'unique_sentence1': 1009, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-bod': {'num_samples': 1012, 'number_of_characters': 280126, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 42, 'average_sentence2_length': 146.4, 'max_sentence2_length': 431, 'unique_sentence2': 1009}, 'boy-eng': {'num_samples': 1012, 'number_of_characters': 277538, 'unique_pairs': 1012, 'min_sentence1_length': 36, 'average_sentence1_length': 143.85, 'max_sentence1_length': 396, 'unique_sentence1': 1011, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-boy': {'num_samples': 1012, 'number_of_characters': 277538, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 36, 'average_sentence2_length': 143.85, 'max_sentence2_length': 396, 'unique_sentence2': 1011}, 'gbm-eng': {'num_samples': 1012, 'number_of_characters': 261027, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 127.53, 'max_sentence1_length': 333, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-gbm': {'num_samples': 1012, 'number_of_characters': 261027, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 127.53, 'max_sentence2_length': 333, 'unique_sentence2': 1012}, 'gom-eng': {'num_samples': 1012, 'number_of_characters': 259182, 'unique_pairs': 1012, 'min_sentence1_length': 37, 'average_sentence1_length': 125.71, 'max_sentence1_length': 335, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-gom': {'num_samples': 1012, 'number_of_characters': 259182, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 37, 'average_sentence2_length': 125.71, 'max_sentence2_length': 335, 'unique_sentence2': 1012}, 'hne-eng': {'num_samples': 1012, 'number_of_characters': 258911, 'unique_pairs': 1012, 'min_sentence1_length': 42, 'average_sentence1_length': 125.44, 'max_sentence1_length': 327, 'unique_sentence1': 1011, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-hne': {'num_samples': 1012, 'number_of_characters': 258915, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 42, 'average_sentence2_length': 125.44, 'max_sentence2_length': 326, 'unique_sentence2': 1011}, 'raj-eng': {'num_samples': 1012, 'number_of_characters': 261987, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 128.48, 'max_sentence1_length': 338, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-raj': {'num_samples': 1012, 'number_of_characters': 261987, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 128.48, 'max_sentence2_length': 338, 'unique_sentence2': 1012}, 'mai-eng': {'num_samples': 1012, 'number_of_characters': 261374, 'unique_pairs': 1012, 'min_sentence1_length': 36, 'average_sentence1_length': 127.87, 'max_sentence1_length': 350, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mai': {'num_samples': 1012, 'number_of_characters': 261377, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 36, 'average_sentence2_length': 127.88, 'max_sentence2_length': 350, 'unique_sentence2': 1012}, 'mni-eng': {'num_samples': 1012, 'number_of_characters': 268767, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 135.18, 'max_sentence1_length': 353, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mni': {'num_samples': 1012, 'number_of_characters': 268768, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 135.18, 'max_sentence2_length': 354, 'unique_sentence2': 1012}, 'mup-eng': {'num_samples': 1012, 'number_of_characters': 262034, 'unique_pairs': 1012, 'min_sentence1_length': 40, 'average_sentence1_length': 128.53, 'max_sentence1_length': 340, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mup': {'num_samples': 1012, 'number_of_characters': 262034, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 40, 'average_sentence2_length': 128.53, 'max_sentence2_length': 340, 'unique_sentence2': 1012}, 'mwr-eng': {'num_samples': 1012, 'number_of_characters': 263749, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.22, 'max_sentence1_length': 345, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mwr': {'num_samples': 1012, 'number_of_characters': 263749, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.22, 'max_sentence2_length': 345, 'unique_sentence2': 1012}, 'sat-eng': {'num_samples': 1012, 'number_of_characters': 271757, 'unique_pairs': 1012, 'min_sentence1_length': 43, 'average_sentence1_length': 138.13, 'max_sentence1_length': 366, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-sat': {'num_samples': 1012, 'number_of_characters': 271757, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 43, 'average_sentence2_length': 138.13, 'max_sentence2_length': 366, 'unique_sentence2': 1012}}}} | +| [IndicLangClassification](https://arxiv.org/abs/2305.15814) | ['asm', 'ben', 'brx', 'doi', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | Classification | s2s | [Web, Non-fiction, Written] | None | None | +| [IndicNLPNewsClassification](https://github.com/AI4Bharat/indicnlp_corpus#indicnlp-news-article-classification-dataset) (Anoop Kunchukuttan, 2020) | ['guj', 'kan', 'mal', 'mar', 'ori', 'pan', 'tam', 'tel'] | Classification | s2s | [News, Written] | None | None | +| [IndicQARetrieval](https://arxiv.org/abs/2212.05409) (Sumanth Doddapaneni, 2022) | ['asm', 'ben', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel'] | Retrieval | s2p | [Web, Written] | None | None | +| [IndicReviewsClusteringP2P](https://arxiv.org/abs/2212.05409) (Sumanth Doddapaneni, 2022) | ['asm', 'ben', 'brx', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel', 'urd'] | Clustering | p2p | [Reviews, Written] | None | None | +| [IndicSentimentClassification](https://arxiv.org/abs/2212.05409) (Sumanth Doddapaneni, 2022) | ['asm', 'ben', 'brx', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel', 'urd'] | Classification | s2s | [Reviews, Written] | None | None | +| [IndonesianIdClickbaitClassification](http://www.sciencedirect.com/science/article/pii/S2352340920311252) | ['ind'] | Classification | s2s | [News, Written] | None | None | +| [IndonesianMongabayConservationClassification](https://aclanthology.org/2023.sealp-1.4/) | ['ind'] | Classification | s2s | [Web, Written] | None | None | +| [InsurancePolicyInterpretationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [InternationalCitizenshipQuestionsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [IsiZuluNewsClassification](https://huggingface.co/datasets/dsfsi/za-isizulu-siswati-news) (Madodonga et al., 2023) | ['zul'] | Classification | s2s | [News, Written] | None | None | +| [ItaCaseholdClassification](https://doi.org/10.1145/3594536.3595177) (Licari et al., 2023) | ['ita'] | Classification | s2s | [Legal, Government, Written] | None | None | +| [Itacola](https://aclanthology.org/2021.findings-emnlp.250/) | ['ita'] | Classification | s2s | [Non-fiction, Spoken, Written] | None | None | +| [JCrewBlockerLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [JDReview](https://aclanthology.org/2023.nodalida-1.20/) (Xiao et al., 2023) | ['cmn'] | Classification | s2s | | None | None | +<<<<<<< HEAD | [JSICK](https://github.com/sbintuitions/JMTEB) (Yanaka et al., 2022) | ['jpn'] | STS | s2s | [Web, Written] | {'test': 1986} | {'test': 21.47} | | [JSTS](https://aclanthology.org/2022.lrec-1.317.pdf#page=2.00) | ['jpn'] | STS | s2s | [Web, Written] | {'valudtion': 1457} | {'valudtion': 46.34} | | [JaGovFaqsRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Web, Written] | {'test': 2048} | {'test': {'average_document_length': 210.02601561814512, 'average_query_length': 59.48193359375, 'num_documents': 22794, 'num_queries': 2048, 'average_relevant_docs_per_query': 1.0}} | @@ -318,16 +522,86 @@ The following tables give you an overview of the tasks in MTEB. | [MSMARCO-PL](https://microsoft.github.io/msmarco/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | {'test': {'average_document_length': 349.3574939240471, 'average_query_length': 33.02325581395349, 'num_documents': 8841823, 'num_queries': 43, 'average_relevant_docs_per_query': 95.3953488372093}} | | [MSMARCO-PLHardNegatives](https://microsoft.github.io/msmarco/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | {'test': 43} | {'test': {'average_document_length': 382.3476426537285, 'average_query_length': 33.02325581395349, 'num_documents': 9481, 'num_queries': 43, 'average_relevant_docs_per_query': 95.3953488372093}} | | [MSMARCOHardNegatives](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | {'test': 43} | {'test': {'average_document_length': 355.2909668633681, 'average_query_length': 32.74418604651163, 'num_documents': 8812, 'num_queries': 43, 'average_relevant_docs_per_query': 95.3953488372093}} | +======= +| [JSICK](https://github.com/sbintuitions/JMTEB) (Yanaka et al., 2022) | ['jpn'] | STS | s2s | [Web, Written] | None | None | +| [JSTS](https://aclanthology.org/2022.lrec-1.317.pdf#page=2.00) | ['jpn'] | STS | s2s | [Web, Written] | None | None | +| [JaGovFaqsRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Web, Written] | None | None | +| [JaQuADRetrieval](https://arxiv.org/abs/2202.01764) (ByungHoon So, 2022) | ['jpn'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Written] | None | None | +| [JaqketRetrieval](https://github.com/kumapo/JAQKET-dataset) | ['jpn'] | Retrieval | s2p | [Encyclopaedic, Non-fiction, Written] | {'test': 115226} | {'test': {'number_of_characters': 428294530, 'num_samples': 115226, 'num_queries': 997, 'num_documents': 114229, 'min_document_length': 16, 'average_document_length': 0.44, 'max_document_length': 98, 'unique_documents': 114229, 'min_query_length': 8, 'average_query_length': 429532.57, 'max_query_length': 188424, 'unique_queries': 997, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 989}} | +| [JavaneseIMDBClassification](https://github.com/w11wo/nlp-datasets#javanese-imdb) (Wongso et al., 2021) | ['jav'] | Classification | s2s | [Reviews, Written] | None | None | +| [KLUE-NLI](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | PairClassification | s2s | [News, Encyclopaedic, Written] | None | None | +| [KLUE-STS](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | STS | s2s | [Reviews, News, Spoken, Written, Spoken] | None | None | +| [KLUE-TC](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | Classification | s2s | [News, Written] | None | None | +| [KannadaNewsClassification](https://github.com/goru001/nlp-for-kannada) (Anoop Kunchukuttan, 2020) | ['kan'] | Classification | s2s | [News, Written] | None | None | +| [KinopoiskClassification](https://www.dialog-21.ru/media/1226/blinovpd.pdf) (Blinov et al., 2013) | ['rus'] | Classification | p2p | [Reviews, Written] | None | None | +| Ko-StrategyQA (Geva et al., 2021) | ['kor'] | Retrieval | s2p | | None | None | +| [KorFin](https://huggingface.co/datasets/amphora/korfin-asc) (Son et al., 2023) | ['kor'] | Classification | s2s | [News, Written] | None | None | +| [KorHateClassification](https://paperswithcode.com/dataset/korean-hatespeech-dataset) (Jihyung Moon, 2020) | ['kor'] | Classification | s2s | [Social, Written] | None | None | +| [KorHateSpeechMLClassification](https://paperswithcode.com/dataset/korean-multi-label-hate-speech-dataset) | ['kor'] | MultilabelClassification | s2s | [Social, Written] | None | None | +| [KorSTS](https://arxiv.org/abs/2004.03289) (Ham et al., 2020) | ['kor'] | STS | s2s | [News, Web] | None | None | +| [KorSarcasmClassification](https://github.com/SpellOnYou/korean-sarcasm) (Kim et al., 2019) | ['kor'] | Classification | s2s | [Social, Written] | None | None | +| [KurdishSentimentClassification](https://link.springer.com/article/10.1007/s10579-023-09716-6) (Badawi et al., 2024) | ['kur'] | Classification | s2s | [Web, Written] | None | None | +| [LCQMC](https://aclanthology.org/2021.emnlp-main.357) (Shitao Xiao, 2024) | ['cmn'] | STS | s2s | | None | None | +| [LEMBNarrativeQARetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) | ['eng'] | Retrieval | s2p | [Fiction, Non-fiction, Written] | None | None | +| [LEMBNeedleRetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) (Zhu et al., 2024) | ['eng'] | Retrieval | s2p | [Academic, Blog, Written] | None | None | +| [LEMBPasskeyRetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) (Zhu et al., 2024) | ['eng'] | Retrieval | s2p | [Fiction, Written] | None | None | +| [LEMBQMSumRetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) | ['eng'] | Retrieval | s2p | [Spoken, Written] | None | None | +| [LEMBSummScreenFDRetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) | ['eng'] | Retrieval | s2p | [Spoken, Written] | None | None | +| [LEMBWikimQARetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) (Ho et al., 2020) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [LanguageClassification](https://huggingface.co/datasets/papluca/language-identification) (Conneau et al., 2018) | ['ara', 'bul', 'cmn', 'deu', 'ell', 'eng', 'fra', 'hin', 'ita', 'jpn', 'nld', 'pol', 'por', 'rus', 'spa', 'swa', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Reviews, Web, Non-fiction, Fiction, Government, Written] | {'test': 2048, 'train': 70000} | {'test': {'num_samples': 2048, 'number_of_characters': 224352, 'num_texts_in_train': 31, 'min_text_length': 14, 'average_text_length': 109.55, 'max_text_length': 1270, 'unique_text': 2025, 'unique_labels': 20, 'labels': {'17': {'count': 102}, '0': {'count': 102}, '11': {'count': 102}, '4': {'count': 103}, '3': {'count': 102}, '1': {'count': 102}, '10': {'count': 102}, '2': {'count': 103}, '16': {'count': 103}, '9': {'count': 103}, '5': {'count': 102}, '7': {'count': 102}, '13': {'count': 102}, '14': {'count': 103}, '12': {'count': 102}, '15': {'count': 103}, '19': {'count': 102}, '18': {'count': 102}, '6': {'count': 103}, '8': {'count': 103}}}, 'train': {'num_samples': 70000, 'number_of_characters': 7760299, 'num_texts_in_train': None, 'min_text_length': 2, 'average_text_length': 110.86, 'max_text_length': 2422, 'unique_text': 68978, 'unique_labels': 20, 'labels': {'12': {'count': 3500}, '1': {'count': 3500}, '19': {'count': 3500}, '15': {'count': 3500}, '13': {'count': 3500}, '11': {'count': 3500}, '17': {'count': 3500}, '14': {'count': 3500}, '16': {'count': 3500}, '5': {'count': 3500}, '0': {'count': 3500}, '8': {'count': 3500}, '7': {'count': 3500}, '2': {'count': 3500}, '3': {'count': 3500}, '10': {'count': 3500}, '6': {'count': 3500}, '18': {'count': 3500}, '4': {'count': 3500}, '9': {'count': 3500}}}} | +| [LccSentimentClassification](https://github.com/fnielsen/lcc-sentiment) | ['dan'] | Classification | s2s | [News, Web, Written] | None | None | +| [LeCaRDv2](https://github.com/THUIR/LeCaRDv2) (Haitao Li, 2023) | ['zho'] | Retrieval | p2p | [Legal, Written] | None | None | +| [LearnedHandsBenefitsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsBusinessLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsConsumerLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsCourtsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsCrimeLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsDivorceLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsDomesticViolenceLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsEducationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsEmploymentLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsEstatesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsFamilyLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsHealthLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsHousingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsImmigrationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsTortsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LearnedHandsTrafficLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LegalBenchConsumerContractsQA](https://huggingface.co/datasets/nguha/legalbench/viewer/consumer_contracts_qa) (Koreeda et al., 2021) | ['eng'] | Retrieval | s2p | [Legal, Written] | None | None | +| [LegalBenchCorporateLobbying](https://huggingface.co/datasets/nguha/legalbench/viewer/corporate_lobbying) (Neel Guha, 2023) | ['eng'] | Retrieval | s2p | [Legal, Written] | None | None | +| [LegalBenchPC](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | PairClassification | s2s | [Legal, Written] | None | None | +| [LegalQuAD](https://github.com/Christoph911/AIKE2021_Appendix) (Hoppe et al., 2021) | ['deu'] | Retrieval | s2p | [Legal, Written] | None | None | +| [LegalReasoningCausalityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [LegalSummarization](https://github.com/lauramanor/legal_summarization) | ['eng'] | Retrieval | s2p | [Legal, Written] | None | None | +| [LinceMTBitextMining](https://ritual.uh.edu/lince/) (Aguilar et al., 2020) | ['eng', 'hin'] | BitextMining | s2s | [Social, Written] | None | None | +| [LitSearchRetrieval](https://github.com/princeton-nlp/LitSearch) (Ajith et al., 2024) | ['eng'] | Retrieval | s2p | [Academic, Non-fiction, Written] | None | None | +| [LivedoorNewsClustering.v2](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Clustering | s2s | [News, Written] | None | None | +| [MAUDLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [MIRACLReranking](https://project-miracl.github.io/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Reranking | s2s | [Encyclopaedic, Written] | None | None | +| [MIRACLRetrieval](http://miracl.ai/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [MIRACLRetrievalHardNegatives](http://miracl.ai/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [MLQARetrieval](https://huggingface.co/datasets/mlqa) | ['ara', 'deu', 'eng', 'hin', 'spa', 'vie', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [MLQuestions](https://github.com/McGill-NLP/MLQuestions) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Academic, Written] | None | None | +| [MLSUMClusteringP2P.v2](https://huggingface.co/datasets/mteb/mlsum) (Scialom et al., 2020) | ['deu', 'fra', 'rus', 'spa'] | Clustering | p2p | [News, Written] | None | None | +| [MLSUMClusteringS2S.v2](https://huggingface.co/datasets/mteb/mlsum) (Scialom et al., 2020) | ['deu', 'fra', 'rus', 'spa'] | Clustering | s2s | [News, Written] | None | None | +| [MMarcoReranking](https://github.com/unicamp-dl/mMARCO) (Luiz Henrique Bonifacio, 2021) | ['cmn'] | Reranking | s2s | | None | None | +| [MMarcoRetrieval](https://arxiv.org/abs/2309.07597) (Shitao Xiao, 2024) | ['cmn'] | Retrieval | s2p | | None | None | +| [MSMARCO](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | None | None | +| [MSMARCO-PL](https://microsoft.github.io/msmarco/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | None | +| [MSMARCO-PLHardNegatives](https://microsoft.github.io/msmarco/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | None | +| [MSMARCOHardNegatives](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | None | None | +>>>>>>> main | [MSMARCOv2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | None | None | -| [MTOPDomainClassification](https://arxiv.org/pdf/2008.09335.pdf) | ['deu', 'eng', 'fra', 'hin', 'spa', 'tha'] | Classification | s2s | [Spoken, Spoken] | {'validation': 2235, 'test': 4386} | {'validation': {'num_samples': 10837, 'average_text_length': 39.85374181046415, 'unique_labels': 11, 'labels': {'1': {'count': 1688}, '10': {'count': 754}, '7': {'count': 849}, '3': {'count': 681}, '6': {'count': 985}, '2': {'count': 647}, '9': {'count': 872}, '0': {'count': 833}, '5': {'count': 1182}, '4': {'count': 982}, '8': {'count': 1364}}, 'hf_subset_descriptive_stats': {}, 'en': {'num_samples': 2235, 'average_text_length': 36.53825503355705, 'unique_labels': 11, 'labels': {'1': {'count': 329}, '10': {'count': 185}, '7': {'count': 183}, '3': {'count': 134}, '6': {'count': 186}, '2': {'count': 123}, '9': {'count': 196}, '0': {'count': 176}, '5': {'count': 228}, '4': {'count': 207}, '8': {'count': 288}}}, 'de': {'num_samples': 1815, 'average_text_length': 42.824793388429754, 'unique_labels': 11, 'labels': {'0': {'count': 99}, '1': {'count': 303}, '2': {'count': 104}, '3': {'count': 122}, '6': {'count': 165}, '4': {'count': 157}, '7': {'count': 141}, '5': {'count': 203}, '8': {'count': 220}, '10': {'count': 133}, '9': {'count': 168}}}, 'es': {'num_samples': 1527, 'average_text_length': 44.34839554682384, 'unique_labels': 11, 'labels': {'1': {'count': 197}, '6': {'count': 166}, '4': {'count': 138}, '10': {'count': 103}, '3': {'count': 104}, '5': {'count': 190}, '2': {'count': 115}, '8': {'count': 212}, '7': {'count': 82}, '9': {'count': 76}, '0': {'count': 144}}}, 'fr': {'num_samples': 1577, 'average_text_length': 43.12492073557387, 'unique_labels': 11, 'labels': {'0': {'count': 125}, '1': {'count': 278}, '2': {'count': 92}, '3': {'count': 89}, '4': {'count': 137}, '7': {'count': 145}, '6': {'count': 138}, '5': {'count': 168}, '8': {'count': 203}, '9': {'count': 124}, '10': {'count': 78}}}, 'hi': {'num_samples': 2012, 'average_text_length': 39.139662027833005, 'unique_labels': 11, 'labels': {'0': {'count': 161}, '1': {'count': 304}, '3': {'count': 126}, '4': {'count': 193}, '2': {'count': 109}, '10': {'count': 154}, '5': {'count': 208}, '6': {'count': 167}, '7': {'count': 172}, '8': {'count': 235}, '9': {'count': 183}}}, 'th': {'num_samples': 1671, 'average_text_length': 34.726511071214844, 'unique_labels': 11, 'labels': {'0': {'count': 128}, '1': {'count': 277}, '2': {'count': 104}, '3': {'count': 106}, '4': {'count': 150}, '5': {'count': 185}, '6': {'count': 163}, '7': {'count': 126}, '8': {'count': 206}, '9': {'count': 125}, '10': {'count': 101}}}}, 'test': {'num_samples': 19680, 'average_text_length': 39.71443089430894, 'unique_labels': 11, 'labels': {'2': {'count': 977}, '5': {'count': 2372}, '6': {'count': 2014}, '8': {'count': 2572}, '9': {'count': 1317}, '1': {'count': 3065}, '10': {'count': 1330}, '3': {'count': 1351}, '0': {'count': 1459}, '7': {'count': 1535}, '4': {'count': 1688}}, 'hf_subset_descriptive_stats': {}, 'en': {'num_samples': 4386, 'average_text_length': 36.79343365253078, 'unique_labels': 11, 'labels': {'2': {'count': 197}, '5': {'count': 487}, '6': {'count': 418}, '8': {'count': 613}, '9': {'count': 346}, '1': {'count': 613}, '10': {'count': 358}, '3': {'count': 290}, '0': {'count': 341}, '7': {'count': 354}, '4': {'count': 369}}}, 'de': {'num_samples': 3549, 'average_text_length': 42.67258382642998, 'unique_labels': 11, 'labels': {'0': {'count': 193}, '10': {'count': 264}, '1': {'count': 553}, '2': {'count': 163}, '3': {'count': 256}, '5': {'count': 439}, '4': {'count': 306}, '6': {'count': 353}, '7': {'count': 279}, '8': {'count': 452}, '9': {'count': 291}}}, 'es': {'num_samples': 2998, 'average_text_length': 43.552034689793196, 'unique_labels': 11, 'labels': {'1': {'count': 401}, '6': {'count': 352}, '4': {'count': 246}, '10': {'count': 206}, '3': {'count': 231}, '5': {'count': 404}, '2': {'count': 177}, '8': {'count': 435}, '7': {'count': 156}, '9': {'count': 126}, '0': {'count': 264}}}, 'fr': {'num_samples': 3193, 'average_text_length': 43.854995302223614, 'unique_labels': 11, 'labels': {'0': {'count': 253}, '1': {'count': 551}, '2': {'count': 159}, '3': {'count': 190}, '4': {'count': 280}, '6': {'count': 330}, '5': {'count': 356}, '7': {'count': 272}, '8': {'count': 462}, '10': {'count': 159}, '9': {'count': 181}}}, 'hi': {'num_samples': 2789, 'average_text_length': 37.395123700250984, 'unique_labels': 11, 'labels': {'0': {'count': 208}, '1': {'count': 470}, '5': {'count': 335}, '3': {'count': 195}, '4': {'count': 242}, '2': {'count': 132}, '6': {'count': 267}, '7': {'count': 262}, '8': {'count': 265}, '10': {'count': 186}, '9': {'count': 227}}}, 'th': {'num_samples': 2765, 'average_text_length': 33.94792043399638, 'unique_labels': 11, 'labels': {'0': {'count': 200}, '1': {'count': 477}, '2': {'count': 149}, '3': {'count': 189}, '4': {'count': 245}, '6': {'count': 294}, '5': {'count': 351}, '7': {'count': 212}, '8': {'count': 345}, '9': {'count': 146}, '10': {'count': 157}}}}, 'train': {'num_samples': 73928, 'average_text_length': 39.73095444215994, 'unique_labels': 11, 'labels': {'0': {'count': 5262}, '5': {'count': 8334}, '6': {'count': 6961}, '9': {'count': 5313}, '1': {'count': 11107}, '8': {'count': 9698}, '10': {'count': 5084}, '2': {'count': 4770}, '4': {'count': 6644}, '3': {'count': 5191}, '7': {'count': 5564}}, 'hf_subset_descriptive_stats': {}, 'en': {'num_samples': 15667, 'average_text_length': 36.57222186761984, 'unique_labels': 11, 'labels': {'0': {'count': 1165}, '5': {'count': 1657}, '6': {'count': 1402}, '9': {'count': 1303}, '1': {'count': 2187}, '8': {'count': 2157}, '10': {'count': 1219}, '2': {'count': 929}, '4': {'count': 1353}, '3': {'count': 1064}, '7': {'count': 1231}}}, 'de': {'num_samples': 13424, 'average_text_length': 43.226013110846246, 'unique_labels': 11, 'labels': {'0': {'count': 761}, '10': {'count': 996}, '4': {'count': 1185}, '1': {'count': 2016}, '7': {'count': 1029}, '5': {'count': 1484}, '2': {'count': 814}, '3': {'count': 980}, '6': {'count': 1265}, '8': {'count': 1767}, '9': {'count': 1127}}}, 'es': {'num_samples': 10934, 'average_text_length': 43.60691421254801, 'unique_labels': 11, 'labels': {'1': {'count': 1459}, '6': {'count': 1188}, '4': {'count': 928}, '10': {'count': 743}, '3': {'count': 830}, '5': {'count': 1396}, '2': {'count': 823}, '8': {'count': 1555}, '7': {'count': 525}, '9': {'count': 560}, '0': {'count': 927}}}, 'fr': {'num_samples': 11814, 'average_text_length': 43.594802776367025, 'unique_labels': 11, 'labels': {'0': {'count': 861}, '10': {'count': 668}, '1': {'count': 1968}, '7': {'count': 975}, '5': {'count': 1261}, '2': {'count': 799}, '3': {'count': 734}, '4': {'count': 1082}, '6': {'count': 1113}, '8': {'count': 1656}, '9': {'count': 697}}}, 'hi': {'num_samples': 11330, 'average_text_length': 37.592144748455425, 'unique_labels': 11, 'labels': {'0': {'count': 794}, '1': {'count': 1741}, '7': {'count': 974}, '2': {'count': 670}, '3': {'count': 831}, '5': {'count': 1272}, '6': {'count': 940}, '4': {'count': 1073}, '10': {'count': 786}, '8': {'count': 1281}, '9': {'count': 968}}}, 'th': {'num_samples': 10759, 'average_text_length': 34.04043126684636, 'unique_labels': 11, 'labels': {'0': {'count': 754}, '10': {'count': 672}, '1': {'count': 1736}, '7': {'count': 830}, '2': {'count': 735}, '3': {'count': 752}, '5': {'count': 1264}, '6': {'count': 1053}, '4': {'count': 1023}, '8': {'count': 1282}, '9': {'count': 658}}}}} | -| [MTOPIntentClassification](https://arxiv.org/pdf/2008.09335.pdf) | ['deu', 'eng', 'fra', 'hin', 'spa', 'tha'] | Classification | s2s | [Spoken, Spoken] | {'validation': 2235, 'test': 4386} | {'validation': 36.5, 'test': 36.8} | -| [MacedonianTweetSentimentClassification](https://aclanthology.org/R15-1034/) | ['mkd'] | Classification | s2s | [Social, Written] | {'test': 1139} | {'test': 67.6} | -| [MalayalamNewsClassification](https://github.com/goru001/nlp-for-malyalam) (Anoop Kunchukuttan, 2020) | ['mal'] | Classification | s2s | [News, Written] | {'train': 5036, 'test': 1260} | {'train': 79.48, 'test': 80.44} | -| [MalteseNewsClassification](https://huggingface.co/datasets/MLRS/maltese_news_categories) | ['mlt'] | MultilabelClassification | s2s | [Constructed, Written] | {'train': 10784, 'test': 2297} | {'train': 1595.63, 'test': 1752.1} | -| [MarathiNewsClassification](https://github.com/goru001/nlp-for-marathi) (Anoop Kunchukuttan, 2020) | ['mar'] | Classification | s2s | [News, Written] | {'test': 2048} | {'test': 52.37} | -| [MasakhaNEWSClassification](https://arxiv.org/abs/2304.09972) (David Ifeoluwa Adelani, 2023) | ['amh', 'eng', 'fra', 'hau', 'ibo', 'lin', 'lug', 'orm', 'pcm', 'run', 'sna', 'som', 'swa', 'tir', 'xho', 'yor'] | Classification | s2s | [News, Written] | {'test': 422} | {'test': 5116.6} | +| [MTOPDomainClassification](https://arxiv.org/pdf/2008.09335.pdf) | ['deu', 'eng', 'fra', 'hin', 'spa', 'tha'] | Classification | s2s | [Spoken, Spoken] | None | None | +| [MTOPIntentClassification](https://arxiv.org/pdf/2008.09335.pdf) | ['deu', 'eng', 'fra', 'hin', 'spa', 'tha'] | Classification | s2s | [Spoken, Spoken] | None | None | +| [MacedonianTweetSentimentClassification](https://aclanthology.org/R15-1034/) | ['mkd'] | Classification | s2s | [Social, Written] | None | None | +| [MalayalamNewsClassification](https://github.com/goru001/nlp-for-malyalam) (Anoop Kunchukuttan, 2020) | ['mal'] | Classification | s2s | [News, Written] | None | None | +| [MalteseNewsClassification](https://huggingface.co/datasets/MLRS/maltese_news_categories) | ['mlt'] | MultilabelClassification | s2s | [Constructed, Written] | None | None | +| [MarathiNewsClassification](https://github.com/goru001/nlp-for-marathi) (Anoop Kunchukuttan, 2020) | ['mar'] | Classification | s2s | [News, Written] | None | None | +| [MasakhaNEWSClassification](https://arxiv.org/abs/2304.09972) (David Ifeoluwa Adelani, 2023) | ['amh', 'eng', 'fra', 'hau', 'ibo', 'lin', 'lug', 'orm', 'pcm', 'run', 'sna', 'som', 'swa', 'tir', 'xho', 'yor'] | Classification | s2s | [News, Written] | None | None | | [MasakhaNEWSClusteringP2P](https://huggingface.co/datasets/masakhane/masakhanews) (David Ifeoluwa Adelani, 2023) | ['amh', 'eng', 'fra', 'hau', 'ibo', 'lin', 'lug', 'orm', 'pcm', 'run', 'sna', 'som', 'swa', 'tir', 'xho', 'yor'] | Clustering | p2p | [News, Written, Non-fiction] | None | None | | [MasakhaNEWSClusteringS2S](https://huggingface.co/datasets/masakhane/masakhanews) (David Ifeoluwa Adelani, 2023) | ['amh', 'eng', 'fra', 'hau', 'ibo', 'lin', 'lug', 'orm', 'pcm', 'run', 'sna', 'som', 'swa', 'tir', 'xho', 'yor'] | Clustering | s2s | | None | None | +<<<<<<< HEAD | [MassiveIntentClassification](https://arxiv.org/abs/2204.08582) (Jack FitzGerald, 2022) | ['afr', 'amh', 'ara', 'aze', 'ben', 'cmo', 'cym', 'dan', 'deu', 'ell', 'eng', 'fas', 'fin', 'fra', 'heb', 'hin', 'hun', 'hye', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kan', 'kat', 'khm', 'kor', 'lav', 'mal', 'mon', 'msa', 'mya', 'nld', 'nob', 'pol', 'por', 'ron', 'rus', 'slv', 'spa', 'sqi', 'swa', 'swe', 'tam', 'tel', 'tgl', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Spoken] | {'validation': 2033, 'test': 2974} | {'validation': 34.8, 'test': 34.6} | | [MassiveScenarioClassification](https://arxiv.org/abs/2204.08582) (Jack FitzGerald, 2022) | ['afr', 'amh', 'ara', 'aze', 'ben', 'cmo', 'cym', 'dan', 'deu', 'ell', 'eng', 'fas', 'fin', 'fra', 'heb', 'hin', 'hun', 'hye', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kan', 'kat', 'khm', 'kor', 'lav', 'mal', 'mon', 'msa', 'mya', 'nld', 'nob', 'pol', 'por', 'ron', 'rus', 'slv', 'spa', 'sqi', 'swa', 'swe', 'tam', 'tel', 'tgl', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Spoken] | {'validation': 2033, 'test': 2974} | {'validation': 34.8, 'test': 34.6} | | [MedicalQARetrieval](https://bmcbioinformatics.biomedcentral.com/articles/10.1186/s12859-019-3119-4) (Asma et al., 2019) | ['eng'] | Retrieval | s2s | [Medical, Written] | {'test': 2048} | {'test': {'average_document_length': 1153.482421875, 'average_query_length': 52.4794921875, 'num_documents': 2048, 'num_queries': 2048, 'average_relevant_docs_per_query': 1.0}} | @@ -386,32 +660,107 @@ The following tables give you an overview of the tasks in MTEB. | [OPP115ThirdPartySharingCollectionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 1590} | {'test': 223.64} | | [OPP115UserAccessEditAndDeletionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 462} | {'test': 218.59} | | [OPP115UserChoiceControlLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 1546} | {'test': 210.62} | +======= +| [MassiveIntentClassification](https://arxiv.org/abs/2204.08582) (Jack FitzGerald, 2022) | ['afr', 'amh', 'ara', 'aze', 'ben', 'cmo', 'cym', 'dan', 'deu', 'ell', 'eng', 'fas', 'fin', 'fra', 'heb', 'hin', 'hun', 'hye', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kan', 'kat', 'khm', 'kor', 'lav', 'mal', 'mon', 'msa', 'mya', 'nld', 'nob', 'pol', 'por', 'ron', 'rus', 'slv', 'spa', 'sqi', 'swa', 'swe', 'tam', 'tel', 'tgl', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Spoken] | None | None | +| [MassiveScenarioClassification](https://arxiv.org/abs/2204.08582) (Jack FitzGerald, 2022) | ['afr', 'amh', 'ara', 'aze', 'ben', 'cmo', 'cym', 'dan', 'deu', 'ell', 'eng', 'fas', 'fin', 'fra', 'heb', 'hin', 'hun', 'hye', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kan', 'kat', 'khm', 'kor', 'lav', 'mal', 'mon', 'msa', 'mya', 'nld', 'nob', 'pol', 'por', 'ron', 'rus', 'slv', 'spa', 'sqi', 'swa', 'swe', 'tam', 'tel', 'tgl', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Spoken] | None | None | +| [MedicalQARetrieval](https://bmcbioinformatics.biomedcentral.com/articles/10.1186/s12859-019-3119-4) (Asma et al., 2019) | ['eng'] | Retrieval | s2s | [Medical, Written] | None | None | +| [MedicalRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | None | +| [MedrxivClusteringP2P.v2](https://api.medrxiv.org/) | ['eng'] | Clustering | p2p | [Academic, Medical, Written] | {'test': 37500} | {'test': {'num_samples': 37500, 'number_of_characters': 74294927, 'min_text_length': 148, 'average_text_length': 1981.2, 'max_text_length': 38759, 'min_labels_per_text': 6, 'average_labels_per_text': 1.0, 'max_labels_per_text': 8830, 'unique_labels': 51, 'labels': {'epidemiology': {'count': 6656}, 'public and global health': {'count': 3595}, 'oncology': {'count': 845}, 'allergy and immunology': {'count': 464}, 'orthopedics': {'count': 104}, 'health informatics': {'count': 1107}, 'occupational and environmental health': {'count': 415}, 'infectious diseases': {'count': 8830}, 'genetic and genomic medicine': {'count': 1918}, 'health policy': {'count': 527}, 'gastroenterology': {'count': 343}, 'radiology and imaging': {'count': 541}, 'pain medicine': {'count': 121}, 'neurology': {'count': 1773}, 'primary care research': {'count': 232}, 'rheumatology': {'count': 189}, 'endocrinology': {'count': 419}, 'hematology': {'count': 202}, 'addiction medicine': {'count': 178}, 'pediatrics': {'count': 589}, 'cardiovascular medicine': {'count': 855}, 'obstetrics and gynecology': {'count': 373}, 'health systems and quality improvement': {'count': 491}, 'nephrology': {'count': 241}, 'respiratory medicine': {'count': 482}, 'geriatric medicine': {'count': 169}, 'dentistry and oral medicine': {'count': 159}, 'psychiatry and clinical psychology': {'count': 1781}, 'nutrition': {'count': 240}, 'intensive care and critical care medicine': {'count': 368}, 'rehabilitation medicine and physical therapy': {'count': 322}, 'otolaryngology': {'count': 166}, 'nursing': {'count': 93}, 'transplantation': {'count': 118}, 'health economics': {'count': 327}, 'sports medicine': {'count': 180}, 'hiv aids': {'count': 363}, 'dermatology': {'count': 98}, 'pathology': {'count': 223}, 'emergency medicine': {'count': 191}, 'pharmacology and therapeutics': {'count': 221}, 'ophthalmology': {'count': 220}, 'medical ethics': {'count': 46}, 'palliative medicine': {'count': 45}, 'sexual and reproductive health': {'count': 156}, 'medical education': {'count': 203}, 'surgery': {'count': 162}, 'urology': {'count': 65}, 'anesthesia': {'count': 72}, 'toxicology': {'count': 16}, 'forensic medicine': {'count': 6}}}} | +| [MedrxivClusteringS2S.v2](https://api.medrxiv.org/) | ['eng'] | Clustering | s2s | [Academic, Medical, Written] | {'test': 37500} | {'test': {'num_samples': 37500, 'number_of_characters': 4301276, 'min_text_length': 18, 'average_text_length': 114.7, 'max_text_length': 339, 'min_labels_per_text': 6, 'average_labels_per_text': 1.0, 'max_labels_per_text': 8830, 'unique_labels': 51, 'labels': {'epidemiology': {'count': 6656}, 'public and global health': {'count': 3595}, 'oncology': {'count': 845}, 'allergy and immunology': {'count': 464}, 'orthopedics': {'count': 104}, 'health informatics': {'count': 1107}, 'occupational and environmental health': {'count': 415}, 'infectious diseases': {'count': 8830}, 'genetic and genomic medicine': {'count': 1918}, 'health policy': {'count': 527}, 'gastroenterology': {'count': 343}, 'radiology and imaging': {'count': 541}, 'pain medicine': {'count': 121}, 'neurology': {'count': 1773}, 'primary care research': {'count': 232}, 'rheumatology': {'count': 189}, 'endocrinology': {'count': 419}, 'hematology': {'count': 202}, 'addiction medicine': {'count': 178}, 'pediatrics': {'count': 589}, 'cardiovascular medicine': {'count': 855}, 'obstetrics and gynecology': {'count': 373}, 'health systems and quality improvement': {'count': 491}, 'nephrology': {'count': 241}, 'respiratory medicine': {'count': 482}, 'geriatric medicine': {'count': 169}, 'dentistry and oral medicine': {'count': 159}, 'psychiatry and clinical psychology': {'count': 1781}, 'nutrition': {'count': 240}, 'intensive care and critical care medicine': {'count': 368}, 'rehabilitation medicine and physical therapy': {'count': 322}, 'otolaryngology': {'count': 166}, 'nursing': {'count': 93}, 'transplantation': {'count': 118}, 'health economics': {'count': 327}, 'sports medicine': {'count': 180}, 'hiv aids': {'count': 363}, 'dermatology': {'count': 98}, 'pathology': {'count': 223}, 'emergency medicine': {'count': 191}, 'pharmacology and therapeutics': {'count': 221}, 'ophthalmology': {'count': 220}, 'medical ethics': {'count': 46}, 'palliative medicine': {'count': 45}, 'sexual and reproductive health': {'count': 156}, 'medical education': {'count': 203}, 'surgery': {'count': 162}, 'urology': {'count': 65}, 'anesthesia': {'count': 72}, 'toxicology': {'count': 16}, 'forensic medicine': {'count': 6}}}} | +| [MewsC16JaClustering](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Clustering | s2s | [News, Written] | None | None | +| [MindSmallReranking](https://msnews.github.io/assets/doc/ACL2020_MIND.pdf) | ['eng'] | Reranking | s2s | [News, Written] | None | None | +| MintakaRetrieval | ['ara', 'deu', 'fra', 'hin', 'ita', 'jpn', 'por', 'spa'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [Moroco](https://huggingface.co/datasets/moroco) (Andrei M. Butnaru, 2019) | ['ron'] | Classification | s2s | [News, Written] | None | None | +| [MovieReviewSentimentClassification](https://github.com/TheophileBlard/french-sentiment-analysis-with-bert) (Théophile Blard, 2020) | ['fra'] | Classification | s2s | [Reviews, Written] | None | None | +| [MrTidyRetrieval](https://huggingface.co/datasets/castorini/mr-tydi) (Xinyu Zhang, 2021) | ['ara', 'ben', 'eng', 'fin', 'ind', 'jpn', 'kor', 'rus', 'swa', 'tel', 'tha'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [MultiEURLEXMultilabelClassification](https://huggingface.co/datasets/coastalcph/multi_eurlex) (Chalkidis et al., 2021) | ['bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'est', 'fin', 'fra', 'hrv', 'hun', 'ita', 'lav', 'lit', 'mlt', 'nld', 'pol', 'por', 'ron', 'slk', 'slv', 'spa', 'swe'] | MultilabelClassification | p2p | [Legal, Government, Written] | None | None | +| [MultiHateClassification](https://aclanthology.org/2022.woah-1.15/) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'nld', 'pol', 'por', 'spa'] | Classification | s2s | [Constructed, Written] | None | None | +| [MultiLongDocRetrieval](https://arxiv.org/abs/2402.03216) (Jianlv Chen, 2024) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'jpn', 'kor', 'por', 'rus', 'spa', 'tha'] | Retrieval | s2p | [Encyclopaedic, Written, Web, Non-fiction, Fiction] | None | None | +| [MultilingualSentiment](https://github.com/tyqiangz/multilingual-sentiment-datasets) | ['cmn'] | Classification | s2s | | None | None | +| [MultilingualSentimentClassification](https://huggingface.co/datasets/mteb/multilingual-sentiment-classification) | ['ara', 'bam', 'bul', 'cmn', 'cym', 'deu', 'dza', 'ell', 'eng', 'eus', 'fas', 'fin', 'heb', 'hrv', 'ind', 'jpn', 'kor', 'mlt', 'nor', 'pol', 'rus', 'slk', 'spa', 'tha', 'tur', 'uig', 'urd', 'vie', 'zho'] | Classification | s2s | [Reviews, Written] | None | None | +| [MyanmarNews](https://huggingface.co/datasets/myanmar_news) (A. H. Khine, 2017) | ['mya'] | Classification | p2p | [News, Written] | None | None | +| [NFCorpus](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | [Medical, Academic, Written] | {'test': 3956} | {'test': {'number_of_characters': 1612.55, 'num_samples': 3956, 'num_queries': 323, 'num_documents': 3633, 'average_document_length': 0.44, 'average_query_length': 0.07, 'average_relevant_docs_per_query': 38.19}} | +| [NFCorpus-PL](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | None | +| [NLPJournalAbsIntroRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Academic, Written] | None | None | +| [NLPJournalTitleAbsRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Academic, Written] | None | None | +| [NLPJournalTitleIntroRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Academic, Written] | None | None | +| [NQ](https://ai.google.com/research/NaturalQuestions/) (Tom Kwiatkowski, 2019) | ['eng'] | Retrieval | s2p | | None | None | +| [NQ-PL](https://ai.google.com/research/NaturalQuestions/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | None | +| [NQ-PLHardNegatives](https://ai.google.com/research/NaturalQuestions/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | None | +| [NQHardNegatives](https://ai.google.com/research/NaturalQuestions/) (Tom Kwiatkowski, 2019) | ['eng'] | Retrieval | s2p | | None | None | +| [NTREXBitextMining](https://huggingface.co/datasets/davidstap/NTREX) | ['afr', 'amh', 'arb', 'aze', 'bak', 'bel', 'bem', 'ben', 'bod', 'bos', 'bul', 'cat', 'ces', 'ckb', 'cym', 'dan', 'deu', 'div', 'dzo', 'ell', 'eng', 'eus', 'ewe', 'fao', 'fas', 'fij', 'fil', 'fin', 'fra', 'fuc', 'gle', 'glg', 'guj', 'hau', 'heb', 'hin', 'hmn', 'hrv', 'hun', 'hye', 'ibo', 'ind', 'isl', 'ita', 'jpn', 'kan', 'kat', 'kaz', 'khm', 'kin', 'kir', 'kmr', 'kor', 'lao', 'lav', 'lit', 'ltz', 'mal', 'mar', 'mey', 'mkd', 'mlg', 'mlt', 'mon', 'mri', 'msa', 'mya', 'nde', 'nep', 'nld', 'nno', 'nob', 'nso', 'nya', 'orm', 'pan', 'pol', 'por', 'prs', 'pus', 'ron', 'rus', 'shi', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'spa', 'sqi', 'srp', 'ssw', 'swa', 'swe', 'tah', 'tam', 'tat', 'tel', 'tgk', 'tha', 'tir', 'ton', 'tsn', 'tuk', 'tur', 'uig', 'ukr', 'urd', 'uzb', 'ven', 'vie', 'wol', 'xho', 'yor', 'yue', 'zho', 'zul'] | BitextMining | s2s | [News, Written] | {'test': 3826252} | {'test': {'num_samples': 3826252, 'number_of_characters': 988355274, 'unique_pairs': 3820263, 'min_sentence1_length': 1, 'average_sentence1_length': 129.15, 'max_sentence1_length': 773, 'unique_sentence1': 241259, 'min_sentence2_length': 1, 'average_sentence2_length': 129.15, 'max_sentence2_length': 773, 'unique_sentence2': 241259, 'hf_subset_descriptive_stats': {'afr_Latn-dan_Latn': {'num_samples': 1997, 'number_of_characters': 520490, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 134.38, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 126.26, 'max_sentence2_length': 522, 'unique_sentence2': 1995}, 'afr_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 564002, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 134.38, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'afr_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 516072, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 134.38, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'afr_Latn-fao_Latn': {'num_samples': 1997, 'number_of_characters': 526155, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 134.38, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.1, 'max_sentence2_length': 433, 'unique_sentence2': 1997}, 'afr_Latn-isl_Latn': {'num_samples': 1997, 'number_of_characters': 530560, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 134.38, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 131.3, 'max_sentence2_length': 399, 'unique_sentence2': 1996}, 'afr_Latn-ltz_Latn': {'num_samples': 1997, 'number_of_characters': 549109, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 134.38, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 140.59, 'max_sentence2_length': 543, 'unique_sentence2': 1996}, 'afr_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 560267, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 134.38, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'afr_Latn-nno_Latn': {'num_samples': 1997, 'number_of_characters': 516709, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 134.38, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.37, 'max_sentence2_length': 417, 'unique_sentence2': 1996}, 'afr_Latn-nob_Latn': {'num_samples': 1997, 'number_of_characters': 519796, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 134.38, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.91, 'max_sentence2_length': 482, 'unique_sentence2': 1996}, 'afr_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 520179, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 134.38, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'amh_Ethi-eng_Latn': {'num_samples': 1997, 'number_of_characters': 415227, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'amh_Ethi-hau_Latn': {'num_samples': 1997, 'number_of_characters': 437473, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'amh_Ethi-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 413608, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'amh_Ethi-nso_Latn': {'num_samples': 1997, 'number_of_characters': 459006, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'amh_Ethi-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 404938, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'amh_Ethi-som_Latn': {'num_samples': 1997, 'number_of_characters': 458799, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'amh_Ethi-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 455649, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'amh_Ethi-swa_Latn': {'num_samples': 1997, 'number_of_characters': 440016, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'amh_Ethi-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 332745, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'amh_Ethi-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 501790, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'amh_Ethi-wol_Latn': {'num_samples': 1997, 'number_of_characters': 407310, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'amh_Ethi-xho_Latn': {'num_samples': 1997, 'number_of_characters': 435597, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'amh_Ethi-yor_Latn': {'num_samples': 1997, 'number_of_characters': 483595, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'amh_Ethi-zul_Latn': {'num_samples': 1997, 'number_of_characters': 425239, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 83.88, 'max_sentence1_length': 290, 'unique_sentence1': 1994, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'arb_Arab-ben_Beng': {'num_samples': 1997, 'number_of_characters': 474983, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'arb_Arab-ckb_Arab': {'num_samples': 1997, 'number_of_characters': 483548, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 126.37, 'max_sentence2_length': 399, 'unique_sentence2': 1995}, 'arb_Arab-deu_Latn': {'num_samples': 1997, 'number_of_characters': 526831, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'arb_Arab-ell_Grek': {'num_samples': 1997, 'number_of_characters': 530308, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'arb_Arab-eng_Latn': {'num_samples': 1997, 'number_of_characters': 478901, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'arb_Arab-fas_Arab': {'num_samples': 1997, 'number_of_characters': 474520, 'unique_pairs': 1995, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'arb_Arab-fin_Latn': {'num_samples': 1997, 'number_of_characters': 500981, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'arb_Arab-fra_Latn': {'num_samples': 1997, 'number_of_characters': 524289, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'arb_Arab-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 431477, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'arb_Arab-hin_Deva': {'num_samples': 1997, 'number_of_characters': 492756, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'arb_Arab-hun_Latn': {'num_samples': 1997, 'number_of_characters': 509557, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'arb_Arab-ind_Latn': {'num_samples': 1997, 'number_of_characters': 518153, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'arb_Arab-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 342807, 'unique_pairs': 1995, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'arb_Arab-kmr_Latn': {'num_samples': 1997, 'number_of_characters': 477127, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 123.16, 'max_sentence2_length': 420, 'unique_sentence2': 1996}, 'arb_Arab-kor_Hang': {'num_samples': 1997, 'number_of_characters': 364586, 'unique_pairs': 1995, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'arb_Arab-lit_Latn': {'num_samples': 1997, 'number_of_characters': 490578, 'unique_pairs': 1995, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'arb_Arab-mey_Arab': {'num_samples': 1997, 'number_of_characters': 445016, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 107.08, 'max_sentence2_length': 392, 'unique_sentence2': 1993}, 'arb_Arab-nld_Latn': {'num_samples': 1997, 'number_of_characters': 523096, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'arb_Arab-pol_Latn': {'num_samples': 1997, 'number_of_characters': 509047, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'arb_Arab-por_Latn': {'num_samples': 1997, 'number_of_characters': 508396, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'arb_Arab-prs_Arab': {'num_samples': 1997, 'number_of_characters': 473717, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 121.45, 'max_sentence2_length': 365, 'unique_sentence2': 1997}, 'arb_Arab-pus_Arab': {'num_samples': 1997, 'number_of_characters': 473814, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 121.5, 'max_sentence2_length': 366, 'unique_sentence2': 1996}, 'arb_Arab-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 506074, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'arb_Arab-shi_Arab': {'num_samples': 1997, 'number_of_characters': 446094, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 3, 'average_sentence2_length': 107.62, 'max_sentence2_length': 378, 'unique_sentence2': 1996}, 'arb_Arab-spa_Latn': {'num_samples': 1997, 'number_of_characters': 519381, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'arb_Arab-swa_Latn': {'num_samples': 1997, 'number_of_characters': 503690, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'arb_Arab-swe_Latn': {'num_samples': 1997, 'number_of_characters': 483008, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'arb_Arab-tam_Taml': {'num_samples': 1997, 'number_of_characters': 541142, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'arb_Arab-tgk_Cyrl': {'num_samples': 1997, 'number_of_characters': 505328, 'unique_pairs': 1995, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 3, 'average_sentence2_length': 137.28, 'max_sentence2_length': 451, 'unique_sentence2': 1995}, 'arb_Arab-tur_Latn': {'num_samples': 1997, 'number_of_characters': 496794, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'arb_Arab-vie_Latn': {'num_samples': 1997, 'number_of_characters': 502302, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'arb_Arab-zho_Hant': {'num_samples': 1997, 'number_of_characters': 322659, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'arb_Arab-zul_Latn': {'num_samples': 1997, 'number_of_characters': 488913, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 115.76, 'max_sentence1_length': 362, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'aze_Latn-bak_Cyrl': {'num_samples': 1997, 'number_of_characters': 515960, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 135.02, 'max_sentence1_length': 398, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 123.35, 'max_sentence2_length': 437, 'unique_sentence2': 1995}, 'aze_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 517354, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 135.02, 'max_sentence1_length': 398, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'aze_Latn-kaz_Cyrl': {'num_samples': 1997, 'number_of_characters': 529910, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 135.02, 'max_sentence1_length': 398, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 130.33, 'max_sentence2_length': 473, 'unique_sentence2': 1996}, 'aze_Latn-kir_Cyrl': {'num_samples': 1997, 'number_of_characters': 520498, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 135.02, 'max_sentence1_length': 398, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 125.62, 'max_sentence2_length': 395, 'unique_sentence2': 1996}, 'aze_Latn-tat_Cyrl': {'num_samples': 1997, 'number_of_characters': 515560, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 135.02, 'max_sentence1_length': 398, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 123.15, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'aze_Latn-tuk_Latn': {'num_samples': 1997, 'number_of_characters': 554908, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 135.02, 'max_sentence1_length': 398, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 142.85, 'max_sentence2_length': 576, 'unique_sentence2': 1996}, 'aze_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 535247, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 135.02, 'max_sentence1_length': 398, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'aze_Latn-uig_Arab': {'num_samples': 1997, 'number_of_characters': 580656, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 135.02, 'max_sentence1_length': 398, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 155.74, 'max_sentence2_length': 592, 'unique_sentence2': 1996}, 'aze_Latn-uzb_Latn': {'num_samples': 1997, 'number_of_characters': 563329, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 135.02, 'max_sentence1_length': 398, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 147.07, 'max_sentence2_length': 470, 'unique_sentence2': 1996}, 'bak_Cyrl-aze_Latn': {'num_samples': 1997, 'number_of_characters': 515960, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 123.35, 'max_sentence1_length': 437, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 135.02, 'max_sentence2_length': 398, 'unique_sentence2': 1997}, 'bak_Cyrl-eng_Latn': {'num_samples': 1997, 'number_of_characters': 494046, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 123.35, 'max_sentence1_length': 437, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'bak_Cyrl-kaz_Cyrl': {'num_samples': 1997, 'number_of_characters': 506602, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 123.35, 'max_sentence1_length': 437, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 130.33, 'max_sentence2_length': 473, 'unique_sentence2': 1996}, 'bak_Cyrl-kir_Cyrl': {'num_samples': 1997, 'number_of_characters': 497190, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 123.35, 'max_sentence1_length': 437, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 125.62, 'max_sentence2_length': 395, 'unique_sentence2': 1996}, 'bak_Cyrl-tat_Cyrl': {'num_samples': 1997, 'number_of_characters': 492252, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 123.35, 'max_sentence1_length': 437, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 123.15, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'bak_Cyrl-tuk_Latn': {'num_samples': 1997, 'number_of_characters': 531600, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 123.35, 'max_sentence1_length': 437, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 142.85, 'max_sentence2_length': 576, 'unique_sentence2': 1996}, 'bak_Cyrl-tur_Latn': {'num_samples': 1997, 'number_of_characters': 511939, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 123.35, 'max_sentence1_length': 437, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'bak_Cyrl-uig_Arab': {'num_samples': 1997, 'number_of_characters': 557348, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 123.35, 'max_sentence1_length': 437, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 155.74, 'max_sentence2_length': 592, 'unique_sentence2': 1996}, 'bak_Cyrl-uzb_Latn': {'num_samples': 1997, 'number_of_characters': 540021, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 123.35, 'max_sentence1_length': 437, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 147.07, 'max_sentence2_length': 470, 'unique_sentence2': 1996}, 'bel_Cyrl-bos_Latn': {'num_samples': 1997, 'number_of_characters': 511000, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'bel_Cyrl-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 525979, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'bel_Cyrl-ces_Latn': {'num_samples': 1997, 'number_of_characters': 497408, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'bel_Cyrl-eng_Latn': {'num_samples': 1997, 'number_of_characters': 503810, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'bel_Cyrl-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 512015, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'bel_Cyrl-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 523981, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'bel_Cyrl-pol_Latn': {'num_samples': 1997, 'number_of_characters': 533956, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'bel_Cyrl-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 530983, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'bel_Cyrl-slk_Latn': {'num_samples': 1997, 'number_of_characters': 509059, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'bel_Cyrl-slv_Latn': {'num_samples': 1997, 'number_of_characters': 508986, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'bel_Cyrl-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 508393, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'bel_Cyrl-srp_Latn': {'num_samples': 1997, 'number_of_characters': 512231, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'bel_Cyrl-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 518873, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 128.24, 'max_sentence1_length': 422, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'bem_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 546212, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 149.47, 'max_sentence1_length': 465, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'bem_Latn-ewe_Latn': {'num_samples': 1997, 'number_of_characters': 537470, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 149.47, 'max_sentence1_length': 465, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 119.67, 'max_sentence2_length': 493, 'unique_sentence2': 1994}, 'bem_Latn-fuc_Latn': {'num_samples': 1997, 'number_of_characters': 526972, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 149.47, 'max_sentence1_length': 465, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 114.41, 'max_sentence2_length': 376, 'unique_sentence2': 1996}, 'bem_Latn-kin_Latn': {'num_samples': 1997, 'number_of_characters': 602279, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 149.47, 'max_sentence1_length': 465, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 152.12, 'max_sentence2_length': 541, 'unique_sentence2': 1996}, 'bem_Latn-nde_Latn': {'num_samples': 1997, 'number_of_characters': 596231, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 149.47, 'max_sentence1_length': 465, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 149.09, 'max_sentence2_length': 590, 'unique_sentence2': 1997}, 'bem_Latn-nya_Latn': {'num_samples': 1997, 'number_of_characters': 582774, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 149.47, 'max_sentence1_length': 465, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 142.35, 'max_sentence2_length': 464, 'unique_sentence2': 1993}, 'bem_Latn-sna_Latn': {'num_samples': 1997, 'number_of_characters': 596822, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 149.47, 'max_sentence1_length': 465, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 149.39, 'max_sentence2_length': 511, 'unique_sentence2': 1995}, 'bem_Latn-ven_Latn': {'num_samples': 1997, 'number_of_characters': 598248, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 149.47, 'max_sentence1_length': 465, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 150.1, 'max_sentence2_length': 535, 'unique_sentence2': 1993}, 'ben_Beng-arb_Arab': {'num_samples': 1997, 'number_of_characters': 474983, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'ben_Beng-deu_Latn': {'num_samples': 1997, 'number_of_characters': 539452, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'ben_Beng-div_Thaa': {'num_samples': 1997, 'number_of_characters': 547650, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'ben_Beng-ell_Grek': {'num_samples': 1997, 'number_of_characters': 542929, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'ben_Beng-eng_Latn': {'num_samples': 1997, 'number_of_characters': 491522, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ben_Beng-eus_Latn': {'num_samples': 1997, 'number_of_characters': 519005, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'ben_Beng-fas_Arab': {'num_samples': 1997, 'number_of_characters': 487141, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'ben_Beng-fin_Latn': {'num_samples': 1997, 'number_of_characters': 513602, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'ben_Beng-fra_Latn': {'num_samples': 1997, 'number_of_characters': 536910, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'ben_Beng-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 488733, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'ben_Beng-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 444098, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'ben_Beng-hin_Deva': {'num_samples': 1997, 'number_of_characters': 505377, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'ben_Beng-hun_Latn': {'num_samples': 1997, 'number_of_characters': 522178, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'ben_Beng-ind_Latn': {'num_samples': 1997, 'number_of_characters': 530774, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'ben_Beng-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 355428, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'ben_Beng-kan_Knda': {'num_samples': 1997, 'number_of_characters': 509338, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'ben_Beng-kor_Hang': {'num_samples': 1997, 'number_of_characters': 377207, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'ben_Beng-lit_Latn': {'num_samples': 1997, 'number_of_characters': 503199, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'ben_Beng-mar_Deva': {'num_samples': 1997, 'number_of_characters': 504689, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'ben_Beng-nep_Deva': {'num_samples': 1997, 'number_of_characters': 492025, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'ben_Beng-nld_Latn': {'num_samples': 1997, 'number_of_characters': 535717, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'ben_Beng-pan_Guru': {'num_samples': 1997, 'number_of_characters': 494224, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'ben_Beng-pol_Latn': {'num_samples': 1997, 'number_of_characters': 521668, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'ben_Beng-por_Latn': {'num_samples': 1997, 'number_of_characters': 521017, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'ben_Beng-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 518695, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'ben_Beng-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 502543, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'ben_Beng-snd_Arab': {'num_samples': 1997, 'number_of_characters': 464129, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'ben_Beng-spa_Latn': {'num_samples': 1997, 'number_of_characters': 532002, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'ben_Beng-swa_Latn': {'num_samples': 1997, 'number_of_characters': 516311, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'ben_Beng-swe_Latn': {'num_samples': 1997, 'number_of_characters': 495629, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'ben_Beng-tam_Taml': {'num_samples': 1997, 'number_of_characters': 553763, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'ben_Beng-tel_Telu': {'num_samples': 1997, 'number_of_characters': 491329, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'ben_Beng-tur_Latn': {'num_samples': 1997, 'number_of_characters': 509415, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'ben_Beng-urd_Arab': {'num_samples': 1997, 'number_of_characters': 491800, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'ben_Beng-vie_Latn': {'num_samples': 1997, 'number_of_characters': 514923, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'ben_Beng-zho_Hant': {'num_samples': 1997, 'number_of_characters': 335280, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'ben_Beng-zul_Latn': {'num_samples': 1997, 'number_of_characters': 501534, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 122.08, 'max_sentence1_length': 402, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'bod_Tibt-dzo_Tibt': {'num_samples': 1997, 'number_of_characters': 543850, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 150.54, 'max_sentence1_length': 478, 'unique_sentence1': 1993, 'min_sentence2_length': 5, 'average_sentence2_length': 121.79, 'max_sentence2_length': 411, 'unique_sentence2': 1992}, 'bod_Tibt-eng_Latn': {'num_samples': 1997, 'number_of_characters': 548349, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 150.54, 'max_sentence1_length': 478, 'unique_sentence1': 1993, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'bod_Tibt-khm_Khmr': {'num_samples': 1997, 'number_of_characters': 589120, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 150.54, 'max_sentence1_length': 478, 'unique_sentence1': 1993, 'min_sentence2_length': 6, 'average_sentence2_length': 144.46, 'max_sentence2_length': 517, 'unique_sentence2': 1996}, 'bod_Tibt-lao_Laoo': {'num_samples': 1997, 'number_of_characters': 567609, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 150.54, 'max_sentence1_length': 478, 'unique_sentence1': 1993, 'min_sentence2_length': 7, 'average_sentence2_length': 133.69, 'max_sentence2_length': 507, 'unique_sentence2': 1997}, 'bod_Tibt-mon_Mong': {'num_samples': 1997, 'number_of_characters': 559677, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 150.54, 'max_sentence1_length': 478, 'unique_sentence1': 1993, 'min_sentence2_length': 11, 'average_sentence2_length': 129.72, 'max_sentence2_length': 414, 'unique_sentence2': 1997}, 'bod_Tibt-mya_Mymr': {'num_samples': 1997, 'number_of_characters': 612483, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 150.54, 'max_sentence1_length': 478, 'unique_sentence1': 1993, 'min_sentence2_length': 9, 'average_sentence2_length': 156.16, 'max_sentence2_length': 773, 'unique_sentence2': 1997}, 'bod_Tibt-tha_Thai': {'num_samples': 1997, 'number_of_characters': 538097, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 150.54, 'max_sentence1_length': 478, 'unique_sentence1': 1993, 'min_sentence2_length': 7, 'average_sentence2_length': 118.91, 'max_sentence2_length': 439, 'unique_sentence2': 1996}, 'bos_Latn-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 511000, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'bos_Latn-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 524799, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'bos_Latn-ces_Latn': {'num_samples': 1997, 'number_of_characters': 496228, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'bos_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 502630, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'bos_Latn-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 510835, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'bos_Latn-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 522801, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'bos_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 532776, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'bos_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 529803, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'bos_Latn-slk_Latn': {'num_samples': 1997, 'number_of_characters': 507879, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'bos_Latn-slv_Latn': {'num_samples': 1997, 'number_of_characters': 507806, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'bos_Latn-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 507213, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'bos_Latn-srp_Latn': {'num_samples': 1997, 'number_of_characters': 511051, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'bos_Latn-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 517693, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 127.65, 'max_sentence1_length': 434, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'bul_Cyrl-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 525979, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'bul_Cyrl-bos_Latn': {'num_samples': 1997, 'number_of_characters': 524799, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'bul_Cyrl-ces_Latn': {'num_samples': 1997, 'number_of_characters': 511207, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'bul_Cyrl-eng_Latn': {'num_samples': 1997, 'number_of_characters': 517609, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'bul_Cyrl-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 525814, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'bul_Cyrl-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 537780, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'bul_Cyrl-pol_Latn': {'num_samples': 1997, 'number_of_characters': 547755, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'bul_Cyrl-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 544782, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'bul_Cyrl-slk_Latn': {'num_samples': 1997, 'number_of_characters': 522858, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'bul_Cyrl-slv_Latn': {'num_samples': 1997, 'number_of_characters': 522785, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'bul_Cyrl-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 522192, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'bul_Cyrl-srp_Latn': {'num_samples': 1997, 'number_of_characters': 526030, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'bul_Cyrl-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 532672, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 135.15, 'max_sentence1_length': 493, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'cat_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 530680, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 141.69, 'max_sentence1_length': 460, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'cat_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 576068, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 141.69, 'max_sentence1_length': 460, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'cat_Latn-glg_Latn': {'num_samples': 1997, 'number_of_characters': 554946, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 141.69, 'max_sentence1_length': 460, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 136.2, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'cat_Latn-ita_Latn': {'num_samples': 1997, 'number_of_characters': 572177, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 141.69, 'max_sentence1_length': 460, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 144.83, 'max_sentence2_length': 623, 'unique_sentence2': 1996}, 'cat_Latn-mlt_Latn': {'num_samples': 1997, 'number_of_characters': 560435, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 141.69, 'max_sentence1_length': 460, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 138.95, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'cat_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 560175, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 141.69, 'max_sentence1_length': 460, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'cat_Latn-ron_Latn': {'num_samples': 1997, 'number_of_characters': 575445, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 141.69, 'max_sentence1_length': 460, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 146.46, 'max_sentence2_length': 518, 'unique_sentence2': 1997}, 'cat_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 571160, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 141.69, 'max_sentence1_length': 460, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'ces_Latn-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 497408, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'ces_Latn-bos_Latn': {'num_samples': 1997, 'number_of_characters': 496228, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'ces_Latn-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 511207, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'ces_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 489038, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ces_Latn-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 497243, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'ces_Latn-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 509209, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'ces_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 519184, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'ces_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 516211, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'ces_Latn-slk_Latn': {'num_samples': 1997, 'number_of_characters': 494287, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'ces_Latn-slv_Latn': {'num_samples': 1997, 'number_of_characters': 494214, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'ces_Latn-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 493621, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'ces_Latn-srp_Latn': {'num_samples': 1997, 'number_of_characters': 497459, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'ces_Latn-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 504101, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 120.84, 'max_sentence1_length': 474, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'ckb_Arab-arb_Arab': {'num_samples': 1997, 'number_of_characters': 483548, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 126.37, 'max_sentence1_length': 399, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'ckb_Arab-eng_Latn': {'num_samples': 1997, 'number_of_characters': 500087, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 126.37, 'max_sentence1_length': 399, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ckb_Arab-fas_Arab': {'num_samples': 1997, 'number_of_characters': 495706, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 126.37, 'max_sentence1_length': 399, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'ckb_Arab-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 452663, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 126.37, 'max_sentence1_length': 399, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'ckb_Arab-kmr_Latn': {'num_samples': 1997, 'number_of_characters': 498313, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 126.37, 'max_sentence1_length': 399, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 123.16, 'max_sentence2_length': 420, 'unique_sentence2': 1996}, 'ckb_Arab-mey_Arab': {'num_samples': 1997, 'number_of_characters': 466202, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 126.37, 'max_sentence1_length': 399, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 107.08, 'max_sentence2_length': 392, 'unique_sentence2': 1993}, 'ckb_Arab-prs_Arab': {'num_samples': 1997, 'number_of_characters': 494903, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 126.37, 'max_sentence1_length': 399, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 121.45, 'max_sentence2_length': 365, 'unique_sentence2': 1997}, 'ckb_Arab-pus_Arab': {'num_samples': 1997, 'number_of_characters': 495000, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 126.37, 'max_sentence1_length': 399, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 121.5, 'max_sentence2_length': 366, 'unique_sentence2': 1996}, 'ckb_Arab-shi_Arab': {'num_samples': 1997, 'number_of_characters': 467280, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 126.37, 'max_sentence1_length': 399, 'unique_sentence1': 1995, 'min_sentence2_length': 3, 'average_sentence2_length': 107.62, 'max_sentence2_length': 378, 'unique_sentence2': 1996}, 'ckb_Arab-tgk_Cyrl': {'num_samples': 1997, 'number_of_characters': 526514, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 126.37, 'max_sentence1_length': 399, 'unique_sentence1': 1995, 'min_sentence2_length': 3, 'average_sentence2_length': 137.28, 'max_sentence2_length': 451, 'unique_sentence2': 1995}, 'cym_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 514225, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.45, 'max_sentence1_length': 444, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'cym_Latn-gle_Latn': {'num_samples': 1997, 'number_of_characters': 561314, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.45, 'max_sentence1_length': 444, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 147.63, 'max_sentence2_length': 461, 'unique_sentence2': 1997}, 'dan_Latn-afr_Latn': {'num_samples': 1997, 'number_of_characters': 520490, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 126.26, 'max_sentence1_length': 522, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 134.38, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'dan_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 547788, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 126.26, 'max_sentence1_length': 522, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'dan_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 499858, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 126.26, 'max_sentence1_length': 522, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'dan_Latn-fao_Latn': {'num_samples': 1997, 'number_of_characters': 509941, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 126.26, 'max_sentence1_length': 522, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 129.1, 'max_sentence2_length': 433, 'unique_sentence2': 1997}, 'dan_Latn-isl_Latn': {'num_samples': 1997, 'number_of_characters': 514346, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 126.26, 'max_sentence1_length': 522, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 131.3, 'max_sentence2_length': 399, 'unique_sentence2': 1996}, 'dan_Latn-ltz_Latn': {'num_samples': 1997, 'number_of_characters': 532895, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 126.26, 'max_sentence1_length': 522, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 140.59, 'max_sentence2_length': 543, 'unique_sentence2': 1996}, 'dan_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 544053, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 126.26, 'max_sentence1_length': 522, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'dan_Latn-nno_Latn': {'num_samples': 1997, 'number_of_characters': 500495, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 126.26, 'max_sentence1_length': 522, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.37, 'max_sentence2_length': 417, 'unique_sentence2': 1996}, 'dan_Latn-nob_Latn': {'num_samples': 1997, 'number_of_characters': 503582, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 126.26, 'max_sentence1_length': 522, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 125.91, 'max_sentence2_length': 482, 'unique_sentence2': 1996}, 'dan_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 503965, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 126.26, 'max_sentence1_length': 522, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'deu_Latn-afr_Latn': {'num_samples': 1997, 'number_of_characters': 564002, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 134.38, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'deu_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 526831, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'deu_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 539452, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'deu_Latn-dan_Latn': {'num_samples': 1997, 'number_of_characters': 547788, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 126.26, 'max_sentence2_length': 522, 'unique_sentence2': 1995}, 'deu_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 594777, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'deu_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 543370, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'deu_Latn-fao_Latn': {'num_samples': 1997, 'number_of_characters': 553453, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.1, 'max_sentence2_length': 433, 'unique_sentence2': 1997}, 'deu_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 538989, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'deu_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 565450, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'deu_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 588758, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'deu_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 495946, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'deu_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 557225, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'deu_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 574026, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'deu_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 582622, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'deu_Latn-isl_Latn': {'num_samples': 1997, 'number_of_characters': 557858, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 131.3, 'max_sentence2_length': 399, 'unique_sentence2': 1996}, 'deu_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 407276, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'deu_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 429055, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'deu_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 555047, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'deu_Latn-ltz_Latn': {'num_samples': 1997, 'number_of_characters': 576407, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 140.59, 'max_sentence2_length': 543, 'unique_sentence2': 1996}, 'deu_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 587565, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'deu_Latn-nno_Latn': {'num_samples': 1997, 'number_of_characters': 544007, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.37, 'max_sentence2_length': 417, 'unique_sentence2': 1996}, 'deu_Latn-nob_Latn': {'num_samples': 1997, 'number_of_characters': 547094, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.91, 'max_sentence2_length': 482, 'unique_sentence2': 1996}, 'deu_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 573516, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'deu_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 572865, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'deu_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 570543, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'deu_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 583850, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'deu_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 568159, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'deu_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 547477, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'deu_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 605611, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'deu_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 561263, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'deu_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 566771, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'deu_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 387128, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'deu_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 553382, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 148.05, 'max_sentence1_length': 508, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'div_Thaa-ben_Beng': {'num_samples': 1997, 'number_of_characters': 547650, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'div_Thaa-eng_Latn': {'num_samples': 1997, 'number_of_characters': 551568, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'div_Thaa-eus_Latn': {'num_samples': 1997, 'number_of_characters': 579051, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'div_Thaa-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 548779, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'div_Thaa-hin_Deva': {'num_samples': 1997, 'number_of_characters': 565423, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'div_Thaa-kan_Knda': {'num_samples': 1997, 'number_of_characters': 569384, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'div_Thaa-mar_Deva': {'num_samples': 1997, 'number_of_characters': 564735, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'div_Thaa-nep_Deva': {'num_samples': 1997, 'number_of_characters': 552071, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'div_Thaa-pan_Guru': {'num_samples': 1997, 'number_of_characters': 554270, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'div_Thaa-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 562589, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'div_Thaa-snd_Arab': {'num_samples': 1997, 'number_of_characters': 524175, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'div_Thaa-tam_Taml': {'num_samples': 1997, 'number_of_characters': 613809, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'div_Thaa-tel_Telu': {'num_samples': 1997, 'number_of_characters': 551375, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'div_Thaa-urd_Arab': {'num_samples': 1997, 'number_of_characters': 551846, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 152.15, 'max_sentence1_length': 609, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'dzo_Tibt-bod_Tibt': {'num_samples': 1997, 'number_of_characters': 543850, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 121.79, 'max_sentence1_length': 411, 'unique_sentence1': 1992, 'min_sentence2_length': 8, 'average_sentence2_length': 150.54, 'max_sentence2_length': 478, 'unique_sentence2': 1993}, 'dzo_Tibt-eng_Latn': {'num_samples': 1997, 'number_of_characters': 490941, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 121.79, 'max_sentence1_length': 411, 'unique_sentence1': 1992, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'dzo_Tibt-khm_Khmr': {'num_samples': 1997, 'number_of_characters': 531712, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 121.79, 'max_sentence1_length': 411, 'unique_sentence1': 1992, 'min_sentence2_length': 6, 'average_sentence2_length': 144.46, 'max_sentence2_length': 517, 'unique_sentence2': 1996}, 'dzo_Tibt-lao_Laoo': {'num_samples': 1997, 'number_of_characters': 510201, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 121.79, 'max_sentence1_length': 411, 'unique_sentence1': 1992, 'min_sentence2_length': 7, 'average_sentence2_length': 133.69, 'max_sentence2_length': 507, 'unique_sentence2': 1997}, 'dzo_Tibt-mon_Mong': {'num_samples': 1997, 'number_of_characters': 502269, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 121.79, 'max_sentence1_length': 411, 'unique_sentence1': 1992, 'min_sentence2_length': 11, 'average_sentence2_length': 129.72, 'max_sentence2_length': 414, 'unique_sentence2': 1997}, 'dzo_Tibt-mya_Mymr': {'num_samples': 1997, 'number_of_characters': 555075, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 121.79, 'max_sentence1_length': 411, 'unique_sentence1': 1992, 'min_sentence2_length': 9, 'average_sentence2_length': 156.16, 'max_sentence2_length': 773, 'unique_sentence2': 1997}, 'dzo_Tibt-tha_Thai': {'num_samples': 1997, 'number_of_characters': 480689, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 121.79, 'max_sentence1_length': 411, 'unique_sentence1': 1992, 'min_sentence2_length': 7, 'average_sentence2_length': 118.91, 'max_sentence2_length': 439, 'unique_sentence2': 1996}, 'ell_Grek-arb_Arab': {'num_samples': 1997, 'number_of_characters': 530308, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'ell_Grek-ben_Beng': {'num_samples': 1997, 'number_of_characters': 542929, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'ell_Grek-deu_Latn': {'num_samples': 1997, 'number_of_characters': 594777, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'ell_Grek-eng_Latn': {'num_samples': 1997, 'number_of_characters': 546847, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ell_Grek-fas_Arab': {'num_samples': 1997, 'number_of_characters': 542466, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'ell_Grek-fin_Latn': {'num_samples': 1997, 'number_of_characters': 568927, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'ell_Grek-fra_Latn': {'num_samples': 1997, 'number_of_characters': 592235, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'ell_Grek-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 499423, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'ell_Grek-hin_Deva': {'num_samples': 1997, 'number_of_characters': 560702, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'ell_Grek-hun_Latn': {'num_samples': 1997, 'number_of_characters': 577503, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'ell_Grek-hye_Armn': {'num_samples': 1997, 'number_of_characters': 563842, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 132.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'ell_Grek-ind_Latn': {'num_samples': 1997, 'number_of_characters': 586099, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'ell_Grek-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 410753, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'ell_Grek-kat_Geor': {'num_samples': 1997, 'number_of_characters': 565719, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 133.5, 'max_sentence2_length': 503, 'unique_sentence2': 1995}, 'ell_Grek-kor_Hang': {'num_samples': 1997, 'number_of_characters': 432532, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'ell_Grek-lit_Latn': {'num_samples': 1997, 'number_of_characters': 558524, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'ell_Grek-nld_Latn': {'num_samples': 1997, 'number_of_characters': 591042, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'ell_Grek-pol_Latn': {'num_samples': 1997, 'number_of_characters': 576993, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'ell_Grek-por_Latn': {'num_samples': 1997, 'number_of_characters': 576342, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'ell_Grek-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 574020, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'ell_Grek-spa_Latn': {'num_samples': 1997, 'number_of_characters': 587327, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'ell_Grek-sqi_Latn': {'num_samples': 1997, 'number_of_characters': 582734, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 142.02, 'max_sentence2_length': 461, 'unique_sentence2': 1996}, 'ell_Grek-swa_Latn': {'num_samples': 1997, 'number_of_characters': 571636, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'ell_Grek-swe_Latn': {'num_samples': 1997, 'number_of_characters': 550954, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'ell_Grek-tam_Taml': {'num_samples': 1997, 'number_of_characters': 609088, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'ell_Grek-tur_Latn': {'num_samples': 1997, 'number_of_characters': 564740, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'ell_Grek-vie_Latn': {'num_samples': 1997, 'number_of_characters': 570248, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'ell_Grek-zho_Hant': {'num_samples': 1997, 'number_of_characters': 390605, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'ell_Grek-zul_Latn': {'num_samples': 1997, 'number_of_characters': 556859, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 149.79, 'max_sentence1_length': 584, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'eng_Latn-afr_Latn': {'num_samples': 1997, 'number_of_characters': 516072, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 134.38, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'eng_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 415227, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'eng_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 478901, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'eng_Latn-aze_Latn': {'num_samples': 1997, 'number_of_characters': 517354, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 135.02, 'max_sentence2_length': 398, 'unique_sentence2': 1997}, 'eng_Latn-bak_Cyrl': {'num_samples': 1997, 'number_of_characters': 494046, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 123.35, 'max_sentence2_length': 437, 'unique_sentence2': 1995}, 'eng_Latn-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 503810, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'eng_Latn-bem_Latn': {'num_samples': 1997, 'number_of_characters': 546212, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 149.47, 'max_sentence2_length': 465, 'unique_sentence2': 1997}, 'eng_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 491522, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'eng_Latn-bod_Tibt': {'num_samples': 1997, 'number_of_characters': 548349, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 150.54, 'max_sentence2_length': 478, 'unique_sentence2': 1993}, 'eng_Latn-bos_Latn': {'num_samples': 1997, 'number_of_characters': 502630, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'eng_Latn-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 517609, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'eng_Latn-cat_Latn': {'num_samples': 1997, 'number_of_characters': 530680, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 141.69, 'max_sentence2_length': 460, 'unique_sentence2': 1997}, 'eng_Latn-ces_Latn': {'num_samples': 1997, 'number_of_characters': 489038, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'eng_Latn-ckb_Arab': {'num_samples': 1997, 'number_of_characters': 500087, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 126.37, 'max_sentence2_length': 399, 'unique_sentence2': 1995}, 'eng_Latn-cym_Latn': {'num_samples': 1997, 'number_of_characters': 514225, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 133.45, 'max_sentence2_length': 444, 'unique_sentence2': 1997}, 'eng_Latn-dan_Latn': {'num_samples': 1997, 'number_of_characters': 499858, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 126.26, 'max_sentence2_length': 522, 'unique_sentence2': 1995}, 'eng_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 543370, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'eng_Latn-div_Thaa': {'num_samples': 1997, 'number_of_characters': 551568, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'eng_Latn-dzo_Tibt': {'num_samples': 1997, 'number_of_characters': 490941, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 121.79, 'max_sentence2_length': 411, 'unique_sentence2': 1992}, 'eng_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 546847, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'eng_Latn-eus_Latn': {'num_samples': 1997, 'number_of_characters': 522923, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'eng_Latn-ewe_Latn': {'num_samples': 1997, 'number_of_characters': 486698, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 119.67, 'max_sentence2_length': 493, 'unique_sentence2': 1994}, 'eng_Latn-fao_Latn': {'num_samples': 1997, 'number_of_characters': 505523, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.1, 'max_sentence2_length': 433, 'unique_sentence2': 1997}, 'eng_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 491059, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'eng_Latn-fij_Latn': {'num_samples': 1997, 'number_of_characters': 548225, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 150.48, 'max_sentence2_length': 448, 'unique_sentence2': 1988}, 'eng_Latn-fil_Latn': {'num_samples': 1997, 'number_of_characters': 541140, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 146.93, 'max_sentence2_length': 554, 'unique_sentence2': 1997}, 'eng_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 517520, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'eng_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 540828, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'eng_Latn-fuc_Latn': {'num_samples': 1997, 'number_of_characters': 476200, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 114.41, 'max_sentence2_length': 376, 'unique_sentence2': 1996}, 'eng_Latn-gle_Latn': {'num_samples': 1997, 'number_of_characters': 542529, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 147.63, 'max_sentence2_length': 461, 'unique_sentence2': 1997}, 'eng_Latn-glg_Latn': {'num_samples': 1997, 'number_of_characters': 519706, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 136.2, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'eng_Latn-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 492651, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'eng_Latn-hau_Latn': {'num_samples': 1997, 'number_of_characters': 517686, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'eng_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 448016, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'eng_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 509295, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'eng_Latn-hmn_Latn': {'num_samples': 1997, 'number_of_characters': 578510, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 165.64, 'max_sentence2_length': 643, 'unique_sentence2': 1997}, 'eng_Latn-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 503645, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'eng_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 526096, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'eng_Latn-hye_Armn': {'num_samples': 1997, 'number_of_characters': 512435, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 132.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'eng_Latn-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 493821, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'eng_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 534692, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'eng_Latn-isl_Latn': {'num_samples': 1997, 'number_of_characters': 509928, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 131.3, 'max_sentence2_length': 399, 'unique_sentence2': 1996}, 'eng_Latn-ita_Latn': {'num_samples': 1997, 'number_of_characters': 536937, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 144.83, 'max_sentence2_length': 623, 'unique_sentence2': 1996}, 'eng_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 359346, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'eng_Latn-kan_Knda': {'num_samples': 1997, 'number_of_characters': 513256, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'eng_Latn-kat_Geor': {'num_samples': 1997, 'number_of_characters': 514312, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 133.5, 'max_sentence2_length': 503, 'unique_sentence2': 1995}, 'eng_Latn-kaz_Cyrl': {'num_samples': 1997, 'number_of_characters': 507996, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 130.33, 'max_sentence2_length': 473, 'unique_sentence2': 1996}, 'eng_Latn-khm_Khmr': {'num_samples': 1997, 'number_of_characters': 536211, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 144.46, 'max_sentence2_length': 517, 'unique_sentence2': 1996}, 'eng_Latn-kin_Latn': {'num_samples': 1997, 'number_of_characters': 551507, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 152.12, 'max_sentence2_length': 541, 'unique_sentence2': 1996}, 'eng_Latn-kir_Cyrl': {'num_samples': 1997, 'number_of_characters': 498584, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 125.62, 'max_sentence2_length': 395, 'unique_sentence2': 1996}, 'eng_Latn-kmr_Latn': {'num_samples': 1997, 'number_of_characters': 493666, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 123.16, 'max_sentence2_length': 420, 'unique_sentence2': 1996}, 'eng_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 381125, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'eng_Latn-lao_Laoo': {'num_samples': 1997, 'number_of_characters': 514700, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 133.69, 'max_sentence2_length': 507, 'unique_sentence2': 1997}, 'eng_Latn-lav_Latn': {'num_samples': 1997, 'number_of_characters': 515908, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 134.3, 'max_sentence2_length': 503, 'unique_sentence2': 1994}, 'eng_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 507117, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'eng_Latn-ltz_Latn': {'num_samples': 1997, 'number_of_characters': 528477, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 140.59, 'max_sentence2_length': 543, 'unique_sentence2': 1996}, 'eng_Latn-mal_Mlym': {'num_samples': 1997, 'number_of_characters': 551872, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 152.3, 'max_sentence2_length': 540, 'unique_sentence2': 1996}, 'eng_Latn-mar_Deva': {'num_samples': 1997, 'number_of_characters': 508607, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'eng_Latn-mey_Arab': {'num_samples': 1997, 'number_of_characters': 461555, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 107.08, 'max_sentence2_length': 392, 'unique_sentence2': 1993}, 'eng_Latn-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 515611, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'eng_Latn-mlg_Latn': {'num_samples': 1997, 'number_of_characters': 568028, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 160.39, 'max_sentence2_length': 559, 'unique_sentence2': 1994}, 'eng_Latn-mlt_Latn': {'num_samples': 1997, 'number_of_characters': 525195, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 138.95, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'eng_Latn-mon_Mong': {'num_samples': 1997, 'number_of_characters': 506768, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 129.72, 'max_sentence2_length': 414, 'unique_sentence2': 1997}, 'eng_Latn-mri_Latn': {'num_samples': 1997, 'number_of_characters': 521844, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 137.27, 'max_sentence2_length': 443, 'unique_sentence2': 1997}, 'eng_Latn-msa_Latn': {'num_samples': 1997, 'number_of_characters': 524903, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 138.8, 'max_sentence2_length': 463, 'unique_sentence2': 1997}, 'eng_Latn-mya_Mymr': {'num_samples': 1997, 'number_of_characters': 559574, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 156.16, 'max_sentence2_length': 773, 'unique_sentence2': 1997}, 'eng_Latn-nde_Latn': {'num_samples': 1997, 'number_of_characters': 545459, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 149.09, 'max_sentence2_length': 590, 'unique_sentence2': 1997}, 'eng_Latn-nep_Deva': {'num_samples': 1997, 'number_of_characters': 495943, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'eng_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 539635, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'eng_Latn-nno_Latn': {'num_samples': 1997, 'number_of_characters': 496077, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.37, 'max_sentence2_length': 417, 'unique_sentence2': 1996}, 'eng_Latn-nob_Latn': {'num_samples': 1997, 'number_of_characters': 499164, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 125.91, 'max_sentence2_length': 482, 'unique_sentence2': 1996}, 'eng_Latn-nso_Latn': {'num_samples': 1997, 'number_of_characters': 539219, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'eng_Latn-nya_Latn': {'num_samples': 1997, 'number_of_characters': 532002, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 142.35, 'max_sentence2_length': 464, 'unique_sentence2': 1993}, 'eng_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 485151, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'eng_Latn-pan_Guru': {'num_samples': 1997, 'number_of_characters': 498142, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'eng_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 525586, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'eng_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 524935, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'eng_Latn-prs_Arab': {'num_samples': 1997, 'number_of_characters': 490256, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 121.45, 'max_sentence2_length': 365, 'unique_sentence2': 1997}, 'eng_Latn-pus_Arab': {'num_samples': 1997, 'number_of_characters': 490353, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 121.5, 'max_sentence2_length': 366, 'unique_sentence2': 1996}, 'eng_Latn-ron_Latn': {'num_samples': 1997, 'number_of_characters': 540205, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 146.46, 'max_sentence2_length': 518, 'unique_sentence2': 1997}, 'eng_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 522613, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'eng_Latn-shi_Arab': {'num_samples': 1997, 'number_of_characters': 462633, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 107.62, 'max_sentence2_length': 378, 'unique_sentence2': 1996}, 'eng_Latn-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 506461, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'eng_Latn-slk_Latn': {'num_samples': 1997, 'number_of_characters': 500689, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'eng_Latn-slv_Latn': {'num_samples': 1997, 'number_of_characters': 500616, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'eng_Latn-smo_Latn': {'num_samples': 1997, 'number_of_characters': 525575, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 139.14, 'max_sentence2_length': 431, 'unique_sentence2': 1996}, 'eng_Latn-sna_Latn': {'num_samples': 1997, 'number_of_characters': 546050, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 149.39, 'max_sentence2_length': 511, 'unique_sentence2': 1995}, 'eng_Latn-snd_Arab': {'num_samples': 1997, 'number_of_characters': 468047, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'eng_Latn-som_Latn': {'num_samples': 1997, 'number_of_characters': 539012, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'eng_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 535920, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'eng_Latn-sqi_Latn': {'num_samples': 1997, 'number_of_characters': 531327, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 142.02, 'max_sentence2_length': 461, 'unique_sentence2': 1996}, 'eng_Latn-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 500023, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'eng_Latn-srp_Latn': {'num_samples': 1997, 'number_of_characters': 503861, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'eng_Latn-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 535862, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'eng_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 520229, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'eng_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 499547, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'eng_Latn-tah_Latn': {'num_samples': 1997, 'number_of_characters': 557343, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 155.04, 'max_sentence2_length': 524, 'unique_sentence2': 1997}, 'eng_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 557681, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'eng_Latn-tat_Cyrl': {'num_samples': 1997, 'number_of_characters': 493646, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 123.15, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'eng_Latn-tel_Telu': {'num_samples': 1997, 'number_of_characters': 495247, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'eng_Latn-tgk_Cyrl': {'num_samples': 1997, 'number_of_characters': 521867, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 137.28, 'max_sentence2_length': 451, 'unique_sentence2': 1995}, 'eng_Latn-tha_Thai': {'num_samples': 1997, 'number_of_characters': 485188, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 118.91, 'max_sentence2_length': 439, 'unique_sentence2': 1996}, 'eng_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 412958, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'eng_Latn-ton_Latn': {'num_samples': 1997, 'number_of_characters': 561360, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 157.06, 'max_sentence2_length': 468, 'unique_sentence2': 1997}, 'eng_Latn-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 582003, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'eng_Latn-tuk_Latn': {'num_samples': 1997, 'number_of_characters': 532994, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 142.85, 'max_sentence2_length': 576, 'unique_sentence2': 1996}, 'eng_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 513333, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'eng_Latn-uig_Arab': {'num_samples': 1997, 'number_of_characters': 558742, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 155.74, 'max_sentence2_length': 592, 'unique_sentence2': 1996}, 'eng_Latn-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 510503, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'eng_Latn-urd_Arab': {'num_samples': 1997, 'number_of_characters': 495718, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'eng_Latn-uzb_Latn': {'num_samples': 1997, 'number_of_characters': 541415, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 147.07, 'max_sentence2_length': 470, 'unique_sentence2': 1996}, 'eng_Latn-ven_Latn': {'num_samples': 1997, 'number_of_characters': 547476, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 150.1, 'max_sentence2_length': 535, 'unique_sentence2': 1993}, 'eng_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 518841, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'eng_Latn-wol_Latn': {'num_samples': 1997, 'number_of_characters': 487523, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'eng_Latn-xho_Latn': {'num_samples': 1997, 'number_of_characters': 515810, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'eng_Latn-yor_Latn': {'num_samples': 1997, 'number_of_characters': 563808, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'eng_Latn-yue_Hant': {'num_samples': 1997, 'number_of_characters': 326607, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 4, 'average_sentence2_length': 39.5, 'max_sentence2_length': 133, 'unique_sentence2': 1996}, 'eng_Latn-zho_Hans': {'num_samples': 1997, 'number_of_characters': 332681, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 4, 'average_sentence2_length': 42.54, 'max_sentence2_length': 263, 'unique_sentence2': 1997}, 'eng_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 339198, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'eng_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 505452, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.05, 'max_sentence1_length': 437, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'eus_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 519005, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'eus_Latn-div_Thaa': {'num_samples': 1997, 'number_of_characters': 579051, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'eus_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 522923, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'eus_Latn-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 520134, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'eus_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 536778, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'eus_Latn-kan_Knda': {'num_samples': 1997, 'number_of_characters': 540739, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'eus_Latn-mar_Deva': {'num_samples': 1997, 'number_of_characters': 536090, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'eus_Latn-nep_Deva': {'num_samples': 1997, 'number_of_characters': 523426, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'eus_Latn-pan_Guru': {'num_samples': 1997, 'number_of_characters': 525625, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'eus_Latn-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 533944, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'eus_Latn-snd_Arab': {'num_samples': 1997, 'number_of_characters': 495530, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'eus_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 585164, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'eus_Latn-tel_Telu': {'num_samples': 1997, 'number_of_characters': 522730, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'eus_Latn-urd_Arab': {'num_samples': 1997, 'number_of_characters': 523201, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 137.81, 'max_sentence1_length': 393, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'ewe_Latn-bem_Latn': {'num_samples': 1997, 'number_of_characters': 537470, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 119.67, 'max_sentence1_length': 493, 'unique_sentence1': 1994, 'min_sentence2_length': 8, 'average_sentence2_length': 149.47, 'max_sentence2_length': 465, 'unique_sentence2': 1997}, 'ewe_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 486698, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 119.67, 'max_sentence1_length': 493, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ewe_Latn-fuc_Latn': {'num_samples': 1997, 'number_of_characters': 467458, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 119.67, 'max_sentence1_length': 493, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 114.41, 'max_sentence2_length': 376, 'unique_sentence2': 1996}, 'ewe_Latn-kin_Latn': {'num_samples': 1997, 'number_of_characters': 542765, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 119.67, 'max_sentence1_length': 493, 'unique_sentence1': 1994, 'min_sentence2_length': 10, 'average_sentence2_length': 152.12, 'max_sentence2_length': 541, 'unique_sentence2': 1996}, 'ewe_Latn-nde_Latn': {'num_samples': 1997, 'number_of_characters': 536717, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 119.67, 'max_sentence1_length': 493, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 149.09, 'max_sentence2_length': 590, 'unique_sentence2': 1997}, 'ewe_Latn-nya_Latn': {'num_samples': 1997, 'number_of_characters': 523260, 'unique_pairs': 1995, 'min_sentence1_length': 6, 'average_sentence1_length': 119.67, 'max_sentence1_length': 493, 'unique_sentence1': 1994, 'min_sentence2_length': 10, 'average_sentence2_length': 142.35, 'max_sentence2_length': 464, 'unique_sentence2': 1993}, 'ewe_Latn-sna_Latn': {'num_samples': 1997, 'number_of_characters': 537308, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 119.67, 'max_sentence1_length': 493, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 149.39, 'max_sentence2_length': 511, 'unique_sentence2': 1995}, 'ewe_Latn-ven_Latn': {'num_samples': 1997, 'number_of_characters': 538734, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 119.67, 'max_sentence1_length': 493, 'unique_sentence1': 1994, 'min_sentence2_length': 10, 'average_sentence2_length': 150.1, 'max_sentence2_length': 535, 'unique_sentence2': 1993}, 'fao_Latn-afr_Latn': {'num_samples': 1997, 'number_of_characters': 526155, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.1, 'max_sentence1_length': 433, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 134.38, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'fao_Latn-dan_Latn': {'num_samples': 1997, 'number_of_characters': 509941, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.1, 'max_sentence1_length': 433, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 126.26, 'max_sentence2_length': 522, 'unique_sentence2': 1995}, 'fao_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 553453, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.1, 'max_sentence1_length': 433, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'fao_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 505523, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.1, 'max_sentence1_length': 433, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'fao_Latn-isl_Latn': {'num_samples': 1997, 'number_of_characters': 520011, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.1, 'max_sentence1_length': 433, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 131.3, 'max_sentence2_length': 399, 'unique_sentence2': 1996}, 'fao_Latn-ltz_Latn': {'num_samples': 1997, 'number_of_characters': 538560, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.1, 'max_sentence1_length': 433, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 140.59, 'max_sentence2_length': 543, 'unique_sentence2': 1996}, 'fao_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 549718, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.1, 'max_sentence1_length': 433, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'fao_Latn-nno_Latn': {'num_samples': 1997, 'number_of_characters': 506160, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.1, 'max_sentence1_length': 433, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.37, 'max_sentence2_length': 417, 'unique_sentence2': 1996}, 'fao_Latn-nob_Latn': {'num_samples': 1997, 'number_of_characters': 509247, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.1, 'max_sentence1_length': 433, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 125.91, 'max_sentence2_length': 482, 'unique_sentence2': 1996}, 'fao_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 509630, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.1, 'max_sentence1_length': 433, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'fas_Arab-arb_Arab': {'num_samples': 1997, 'number_of_characters': 474520, 'unique_pairs': 1995, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'fas_Arab-ben_Beng': {'num_samples': 1997, 'number_of_characters': 487141, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'fas_Arab-ckb_Arab': {'num_samples': 1997, 'number_of_characters': 495706, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 126.37, 'max_sentence2_length': 399, 'unique_sentence2': 1995}, 'fas_Arab-deu_Latn': {'num_samples': 1997, 'number_of_characters': 538989, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'fas_Arab-ell_Grek': {'num_samples': 1997, 'number_of_characters': 542466, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'fas_Arab-eng_Latn': {'num_samples': 1997, 'number_of_characters': 491059, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'fas_Arab-fin_Latn': {'num_samples': 1997, 'number_of_characters': 513139, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'fas_Arab-fra_Latn': {'num_samples': 1997, 'number_of_characters': 536447, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'fas_Arab-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 443635, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'fas_Arab-hin_Deva': {'num_samples': 1997, 'number_of_characters': 504914, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'fas_Arab-hun_Latn': {'num_samples': 1997, 'number_of_characters': 521715, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'fas_Arab-ind_Latn': {'num_samples': 1997, 'number_of_characters': 530311, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'fas_Arab-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 354965, 'unique_pairs': 1995, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'fas_Arab-kmr_Latn': {'num_samples': 1997, 'number_of_characters': 489285, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 123.16, 'max_sentence2_length': 420, 'unique_sentence2': 1996}, 'fas_Arab-kor_Hang': {'num_samples': 1997, 'number_of_characters': 376744, 'unique_pairs': 1995, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'fas_Arab-lit_Latn': {'num_samples': 1997, 'number_of_characters': 502736, 'unique_pairs': 1995, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'fas_Arab-mey_Arab': {'num_samples': 1997, 'number_of_characters': 457174, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 107.08, 'max_sentence2_length': 392, 'unique_sentence2': 1993}, 'fas_Arab-nld_Latn': {'num_samples': 1997, 'number_of_characters': 535254, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'fas_Arab-pol_Latn': {'num_samples': 1997, 'number_of_characters': 521205, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'fas_Arab-por_Latn': {'num_samples': 1997, 'number_of_characters': 520554, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'fas_Arab-prs_Arab': {'num_samples': 1997, 'number_of_characters': 485875, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 121.45, 'max_sentence2_length': 365, 'unique_sentence2': 1997}, 'fas_Arab-pus_Arab': {'num_samples': 1997, 'number_of_characters': 485972, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 121.5, 'max_sentence2_length': 366, 'unique_sentence2': 1996}, 'fas_Arab-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 518232, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'fas_Arab-shi_Arab': {'num_samples': 1997, 'number_of_characters': 458252, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 3, 'average_sentence2_length': 107.62, 'max_sentence2_length': 378, 'unique_sentence2': 1996}, 'fas_Arab-spa_Latn': {'num_samples': 1997, 'number_of_characters': 531539, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'fas_Arab-swa_Latn': {'num_samples': 1997, 'number_of_characters': 515848, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'fas_Arab-swe_Latn': {'num_samples': 1997, 'number_of_characters': 495166, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'fas_Arab-tam_Taml': {'num_samples': 1997, 'number_of_characters': 553300, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'fas_Arab-tgk_Cyrl': {'num_samples': 1997, 'number_of_characters': 517486, 'unique_pairs': 1995, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 3, 'average_sentence2_length': 137.28, 'max_sentence2_length': 451, 'unique_sentence2': 1995}, 'fas_Arab-tur_Latn': {'num_samples': 1997, 'number_of_characters': 508952, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'fas_Arab-vie_Latn': {'num_samples': 1997, 'number_of_characters': 514460, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'fas_Arab-zho_Hant': {'num_samples': 1997, 'number_of_characters': 334817, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'fas_Arab-zul_Latn': {'num_samples': 1997, 'number_of_characters': 501071, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 121.85, 'max_sentence1_length': 389, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'fij_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 548225, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.48, 'max_sentence1_length': 448, 'unique_sentence1': 1988, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'fij_Latn-fil_Latn': {'num_samples': 1997, 'number_of_characters': 593925, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.48, 'max_sentence1_length': 448, 'unique_sentence1': 1988, 'min_sentence2_length': 10, 'average_sentence2_length': 146.93, 'max_sentence2_length': 554, 'unique_sentence2': 1997}, 'fij_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 587477, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.48, 'max_sentence1_length': 448, 'unique_sentence1': 1988, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'fij_Latn-mal_Mlym': {'num_samples': 1997, 'number_of_characters': 604657, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.48, 'max_sentence1_length': 448, 'unique_sentence1': 1988, 'min_sentence2_length': 7, 'average_sentence2_length': 152.3, 'max_sentence2_length': 540, 'unique_sentence2': 1996}, 'fij_Latn-mlg_Latn': {'num_samples': 1997, 'number_of_characters': 620813, 'unique_pairs': 1995, 'min_sentence1_length': 10, 'average_sentence1_length': 150.48, 'max_sentence1_length': 448, 'unique_sentence1': 1988, 'min_sentence2_length': 6, 'average_sentence2_length': 160.39, 'max_sentence2_length': 559, 'unique_sentence2': 1994}, 'fij_Latn-mri_Latn': {'num_samples': 1997, 'number_of_characters': 574629, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.48, 'max_sentence1_length': 448, 'unique_sentence1': 1988, 'min_sentence2_length': 9, 'average_sentence2_length': 137.27, 'max_sentence2_length': 443, 'unique_sentence2': 1997}, 'fij_Latn-msa_Latn': {'num_samples': 1997, 'number_of_characters': 577688, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.48, 'max_sentence1_length': 448, 'unique_sentence1': 1988, 'min_sentence2_length': 6, 'average_sentence2_length': 138.8, 'max_sentence2_length': 463, 'unique_sentence2': 1997}, 'fij_Latn-smo_Latn': {'num_samples': 1997, 'number_of_characters': 578360, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.48, 'max_sentence1_length': 448, 'unique_sentence1': 1988, 'min_sentence2_length': 8, 'average_sentence2_length': 139.14, 'max_sentence2_length': 431, 'unique_sentence2': 1996}, 'fij_Latn-tah_Latn': {'num_samples': 1997, 'number_of_characters': 610128, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.48, 'max_sentence1_length': 448, 'unique_sentence1': 1988, 'min_sentence2_length': 9, 'average_sentence2_length': 155.04, 'max_sentence2_length': 524, 'unique_sentence2': 1997}, 'fij_Latn-ton_Latn': {'num_samples': 1997, 'number_of_characters': 614145, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.48, 'max_sentence1_length': 448, 'unique_sentence1': 1988, 'min_sentence2_length': 7, 'average_sentence2_length': 157.06, 'max_sentence2_length': 468, 'unique_sentence2': 1997}, 'fil_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 541140, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 146.93, 'max_sentence1_length': 554, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'fil_Latn-fij_Latn': {'num_samples': 1997, 'number_of_characters': 593925, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 146.93, 'max_sentence1_length': 554, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 150.48, 'max_sentence2_length': 448, 'unique_sentence2': 1988}, 'fil_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 580392, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 146.93, 'max_sentence1_length': 554, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'fil_Latn-mal_Mlym': {'num_samples': 1997, 'number_of_characters': 597572, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 146.93, 'max_sentence1_length': 554, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 152.3, 'max_sentence2_length': 540, 'unique_sentence2': 1996}, 'fil_Latn-mlg_Latn': {'num_samples': 1997, 'number_of_characters': 613728, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 146.93, 'max_sentence1_length': 554, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 160.39, 'max_sentence2_length': 559, 'unique_sentence2': 1994}, 'fil_Latn-mri_Latn': {'num_samples': 1997, 'number_of_characters': 567544, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 146.93, 'max_sentence1_length': 554, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 137.27, 'max_sentence2_length': 443, 'unique_sentence2': 1997}, 'fil_Latn-msa_Latn': {'num_samples': 1997, 'number_of_characters': 570603, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 146.93, 'max_sentence1_length': 554, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 138.8, 'max_sentence2_length': 463, 'unique_sentence2': 1997}, 'fil_Latn-smo_Latn': {'num_samples': 1997, 'number_of_characters': 571275, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 146.93, 'max_sentence1_length': 554, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 139.14, 'max_sentence2_length': 431, 'unique_sentence2': 1996}, 'fil_Latn-tah_Latn': {'num_samples': 1997, 'number_of_characters': 603043, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 146.93, 'max_sentence1_length': 554, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 155.04, 'max_sentence2_length': 524, 'unique_sentence2': 1997}, 'fil_Latn-ton_Latn': {'num_samples': 1997, 'number_of_characters': 607060, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 146.93, 'max_sentence1_length': 554, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 157.06, 'max_sentence2_length': 468, 'unique_sentence2': 1997}, 'fin_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 500981, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'fin_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 513602, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'fin_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 565450, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'fin_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 568927, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'fin_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 517520, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'fin_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 513139, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'fin_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 562908, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'fin_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 470096, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'fin_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 531375, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'fin_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 548176, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'fin_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 556772, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'fin_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 381426, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'fin_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 403205, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'fin_Latn-lav_Latn': {'num_samples': 1997, 'number_of_characters': 537988, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 134.3, 'max_sentence2_length': 503, 'unique_sentence2': 1994}, 'fin_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 529197, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'fin_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 561715, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'fin_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 547666, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'fin_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 547015, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'fin_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 544693, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'fin_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 558000, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'fin_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 542309, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'fin_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 521627, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'fin_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 579761, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'fin_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 535413, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'fin_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 540921, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'fin_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 361278, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'fin_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 527532, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.1, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'fra_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 524289, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'fra_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 536910, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'fra_Latn-cat_Latn': {'num_samples': 1997, 'number_of_characters': 576068, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 141.69, 'max_sentence2_length': 460, 'unique_sentence2': 1997}, 'fra_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 588758, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'fra_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 592235, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'fra_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 540828, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'fra_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 536447, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'fra_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 562908, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'fra_Latn-glg_Latn': {'num_samples': 1997, 'number_of_characters': 565094, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 136.2, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'fra_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 493404, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'fra_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 554683, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'fra_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 571484, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'fra_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 580080, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'fra_Latn-ita_Latn': {'num_samples': 1997, 'number_of_characters': 582325, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 144.83, 'max_sentence2_length': 623, 'unique_sentence2': 1996}, 'fra_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 404734, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'fra_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 426513, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'fra_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 552505, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'fra_Latn-mlt_Latn': {'num_samples': 1997, 'number_of_characters': 570583, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 138.95, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'fra_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 585023, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'fra_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 570974, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'fra_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 570323, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'fra_Latn-ron_Latn': {'num_samples': 1997, 'number_of_characters': 585593, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 146.46, 'max_sentence2_length': 518, 'unique_sentence2': 1997}, 'fra_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 568001, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'fra_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 581308, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'fra_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 565617, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'fra_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 544935, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'fra_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 603069, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'fra_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 558721, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'fra_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 564229, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'fra_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 384586, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'fra_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 550840, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.77, 'max_sentence1_length': 512, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'fuc_Latn-bem_Latn': {'num_samples': 1997, 'number_of_characters': 526972, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 114.41, 'max_sentence1_length': 376, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 149.47, 'max_sentence2_length': 465, 'unique_sentence2': 1997}, 'fuc_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 476200, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 114.41, 'max_sentence1_length': 376, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'fuc_Latn-ewe_Latn': {'num_samples': 1997, 'number_of_characters': 467458, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 114.41, 'max_sentence1_length': 376, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 119.67, 'max_sentence2_length': 493, 'unique_sentence2': 1994}, 'fuc_Latn-kin_Latn': {'num_samples': 1997, 'number_of_characters': 532267, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 114.41, 'max_sentence1_length': 376, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 152.12, 'max_sentence2_length': 541, 'unique_sentence2': 1996}, 'fuc_Latn-nde_Latn': {'num_samples': 1997, 'number_of_characters': 526219, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 114.41, 'max_sentence1_length': 376, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 149.09, 'max_sentence2_length': 590, 'unique_sentence2': 1997}, 'fuc_Latn-nya_Latn': {'num_samples': 1997, 'number_of_characters': 512762, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 114.41, 'max_sentence1_length': 376, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 142.35, 'max_sentence2_length': 464, 'unique_sentence2': 1993}, 'fuc_Latn-sna_Latn': {'num_samples': 1997, 'number_of_characters': 526810, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 114.41, 'max_sentence1_length': 376, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 149.39, 'max_sentence2_length': 511, 'unique_sentence2': 1995}, 'fuc_Latn-ven_Latn': {'num_samples': 1997, 'number_of_characters': 528236, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 114.41, 'max_sentence1_length': 376, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 150.1, 'max_sentence2_length': 535, 'unique_sentence2': 1993}, 'gle_Latn-cym_Latn': {'num_samples': 1997, 'number_of_characters': 561314, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 147.63, 'max_sentence1_length': 461, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 133.45, 'max_sentence2_length': 444, 'unique_sentence2': 1997}, 'gle_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 542529, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 147.63, 'max_sentence1_length': 461, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'glg_Latn-cat_Latn': {'num_samples': 1997, 'number_of_characters': 554946, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 136.2, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 141.69, 'max_sentence2_length': 460, 'unique_sentence2': 1997}, 'glg_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 519706, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 136.2, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'glg_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 565094, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 136.2, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'glg_Latn-ita_Latn': {'num_samples': 1997, 'number_of_characters': 561203, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 136.2, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 144.83, 'max_sentence2_length': 623, 'unique_sentence2': 1996}, 'glg_Latn-mlt_Latn': {'num_samples': 1997, 'number_of_characters': 549461, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 136.2, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 138.95, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'glg_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 549201, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 136.2, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'glg_Latn-ron_Latn': {'num_samples': 1997, 'number_of_characters': 564471, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 136.2, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 146.46, 'max_sentence2_length': 518, 'unique_sentence2': 1997}, 'glg_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 560186, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 136.2, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'guj_Gujr-ben_Beng': {'num_samples': 1997, 'number_of_characters': 488733, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'guj_Gujr-div_Thaa': {'num_samples': 1997, 'number_of_characters': 548779, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'guj_Gujr-eng_Latn': {'num_samples': 1997, 'number_of_characters': 492651, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'guj_Gujr-eus_Latn': {'num_samples': 1997, 'number_of_characters': 520134, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'guj_Gujr-hin_Deva': {'num_samples': 1997, 'number_of_characters': 506506, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'guj_Gujr-kan_Knda': {'num_samples': 1997, 'number_of_characters': 510467, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'guj_Gujr-mar_Deva': {'num_samples': 1997, 'number_of_characters': 505818, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'guj_Gujr-nep_Deva': {'num_samples': 1997, 'number_of_characters': 493154, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'guj_Gujr-pan_Guru': {'num_samples': 1997, 'number_of_characters': 495353, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'guj_Gujr-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 503672, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'guj_Gujr-snd_Arab': {'num_samples': 1997, 'number_of_characters': 465258, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'guj_Gujr-tam_Taml': {'num_samples': 1997, 'number_of_characters': 554892, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'guj_Gujr-tel_Telu': {'num_samples': 1997, 'number_of_characters': 492458, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'guj_Gujr-urd_Arab': {'num_samples': 1997, 'number_of_characters': 492929, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 122.65, 'max_sentence1_length': 378, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'hau_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 437473, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'hau_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 517686, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'hau_Latn-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 516067, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'hau_Latn-nso_Latn': {'num_samples': 1997, 'number_of_characters': 561465, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'hau_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 507397, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'hau_Latn-som_Latn': {'num_samples': 1997, 'number_of_characters': 561258, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'hau_Latn-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 558108, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'hau_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 542475, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'hau_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 435204, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'hau_Latn-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 604249, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'hau_Latn-wol_Latn': {'num_samples': 1997, 'number_of_characters': 509769, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'hau_Latn-xho_Latn': {'num_samples': 1997, 'number_of_characters': 538056, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'hau_Latn-yor_Latn': {'num_samples': 1997, 'number_of_characters': 586054, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'hau_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 527698, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 135.19, 'max_sentence1_length': 483, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'heb_Hebr-arb_Arab': {'num_samples': 1997, 'number_of_characters': 431477, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'heb_Hebr-ben_Beng': {'num_samples': 1997, 'number_of_characters': 444098, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'heb_Hebr-ckb_Arab': {'num_samples': 1997, 'number_of_characters': 452663, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 126.37, 'max_sentence2_length': 399, 'unique_sentence2': 1995}, 'heb_Hebr-deu_Latn': {'num_samples': 1997, 'number_of_characters': 495946, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'heb_Hebr-ell_Grek': {'num_samples': 1997, 'number_of_characters': 499423, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'heb_Hebr-eng_Latn': {'num_samples': 1997, 'number_of_characters': 448016, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'heb_Hebr-fas_Arab': {'num_samples': 1997, 'number_of_characters': 443635, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'heb_Hebr-fin_Latn': {'num_samples': 1997, 'number_of_characters': 470096, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'heb_Hebr-fra_Latn': {'num_samples': 1997, 'number_of_characters': 493404, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'heb_Hebr-hin_Deva': {'num_samples': 1997, 'number_of_characters': 461871, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'heb_Hebr-hun_Latn': {'num_samples': 1997, 'number_of_characters': 478672, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'heb_Hebr-ind_Latn': {'num_samples': 1997, 'number_of_characters': 487268, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'heb_Hebr-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 311922, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'heb_Hebr-kmr_Latn': {'num_samples': 1997, 'number_of_characters': 446242, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 123.16, 'max_sentence2_length': 420, 'unique_sentence2': 1996}, 'heb_Hebr-kor_Hang': {'num_samples': 1997, 'number_of_characters': 333701, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'heb_Hebr-lit_Latn': {'num_samples': 1997, 'number_of_characters': 459693, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'heb_Hebr-mey_Arab': {'num_samples': 1997, 'number_of_characters': 414131, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 107.08, 'max_sentence2_length': 392, 'unique_sentence2': 1993}, 'heb_Hebr-nld_Latn': {'num_samples': 1997, 'number_of_characters': 492211, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'heb_Hebr-pol_Latn': {'num_samples': 1997, 'number_of_characters': 478162, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'heb_Hebr-por_Latn': {'num_samples': 1997, 'number_of_characters': 477511, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'heb_Hebr-prs_Arab': {'num_samples': 1997, 'number_of_characters': 442832, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 121.45, 'max_sentence2_length': 365, 'unique_sentence2': 1997}, 'heb_Hebr-pus_Arab': {'num_samples': 1997, 'number_of_characters': 442929, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 121.5, 'max_sentence2_length': 366, 'unique_sentence2': 1996}, 'heb_Hebr-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 475189, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'heb_Hebr-shi_Arab': {'num_samples': 1997, 'number_of_characters': 415209, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 107.62, 'max_sentence2_length': 378, 'unique_sentence2': 1996}, 'heb_Hebr-spa_Latn': {'num_samples': 1997, 'number_of_characters': 488496, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'heb_Hebr-swa_Latn': {'num_samples': 1997, 'number_of_characters': 472805, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'heb_Hebr-swe_Latn': {'num_samples': 1997, 'number_of_characters': 452123, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'heb_Hebr-tam_Taml': {'num_samples': 1997, 'number_of_characters': 510257, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'heb_Hebr-tgk_Cyrl': {'num_samples': 1997, 'number_of_characters': 474443, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 137.28, 'max_sentence2_length': 451, 'unique_sentence2': 1995}, 'heb_Hebr-tur_Latn': {'num_samples': 1997, 'number_of_characters': 465909, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'heb_Hebr-vie_Latn': {'num_samples': 1997, 'number_of_characters': 471417, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'heb_Hebr-zho_Hant': {'num_samples': 1997, 'number_of_characters': 291774, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'heb_Hebr-zul_Latn': {'num_samples': 1997, 'number_of_characters': 458028, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 100.3, 'max_sentence1_length': 375, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'hin_Deva-arb_Arab': {'num_samples': 1997, 'number_of_characters': 492756, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'hin_Deva-ben_Beng': {'num_samples': 1997, 'number_of_characters': 505377, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'hin_Deva-deu_Latn': {'num_samples': 1997, 'number_of_characters': 557225, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'hin_Deva-div_Thaa': {'num_samples': 1997, 'number_of_characters': 565423, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'hin_Deva-ell_Grek': {'num_samples': 1997, 'number_of_characters': 560702, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'hin_Deva-eng_Latn': {'num_samples': 1997, 'number_of_characters': 509295, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'hin_Deva-eus_Latn': {'num_samples': 1997, 'number_of_characters': 536778, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'hin_Deva-fas_Arab': {'num_samples': 1997, 'number_of_characters': 504914, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'hin_Deva-fin_Latn': {'num_samples': 1997, 'number_of_characters': 531375, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'hin_Deva-fra_Latn': {'num_samples': 1997, 'number_of_characters': 554683, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'hin_Deva-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 506506, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'hin_Deva-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 461871, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'hin_Deva-hun_Latn': {'num_samples': 1997, 'number_of_characters': 539951, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'hin_Deva-ind_Latn': {'num_samples': 1997, 'number_of_characters': 548547, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'hin_Deva-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 373201, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'hin_Deva-kan_Knda': {'num_samples': 1997, 'number_of_characters': 527111, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'hin_Deva-kor_Hang': {'num_samples': 1997, 'number_of_characters': 394980, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'hin_Deva-lit_Latn': {'num_samples': 1997, 'number_of_characters': 520972, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'hin_Deva-mar_Deva': {'num_samples': 1997, 'number_of_characters': 522462, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'hin_Deva-nep_Deva': {'num_samples': 1997, 'number_of_characters': 509798, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'hin_Deva-nld_Latn': {'num_samples': 1997, 'number_of_characters': 553490, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'hin_Deva-pan_Guru': {'num_samples': 1997, 'number_of_characters': 511997, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'hin_Deva-pol_Latn': {'num_samples': 1997, 'number_of_characters': 539441, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'hin_Deva-por_Latn': {'num_samples': 1997, 'number_of_characters': 538790, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'hin_Deva-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 536468, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'hin_Deva-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 520316, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'hin_Deva-snd_Arab': {'num_samples': 1997, 'number_of_characters': 481902, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'hin_Deva-spa_Latn': {'num_samples': 1997, 'number_of_characters': 549775, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'hin_Deva-swa_Latn': {'num_samples': 1997, 'number_of_characters': 534084, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'hin_Deva-swe_Latn': {'num_samples': 1997, 'number_of_characters': 513402, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'hin_Deva-tam_Taml': {'num_samples': 1997, 'number_of_characters': 571536, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'hin_Deva-tel_Telu': {'num_samples': 1997, 'number_of_characters': 509102, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'hin_Deva-tur_Latn': {'num_samples': 1997, 'number_of_characters': 527188, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'hin_Deva-urd_Arab': {'num_samples': 1997, 'number_of_characters': 509573, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'hin_Deva-vie_Latn': {'num_samples': 1997, 'number_of_characters': 532696, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'hin_Deva-zho_Hant': {'num_samples': 1997, 'number_of_characters': 353053, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'hin_Deva-zul_Latn': {'num_samples': 1997, 'number_of_characters': 519307, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 130.98, 'max_sentence1_length': 394, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'hmn_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 578510, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 165.64, 'max_sentence1_length': 643, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'hrv_Latn-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 512015, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'hrv_Latn-bos_Latn': {'num_samples': 1997, 'number_of_characters': 510835, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'hrv_Latn-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 525814, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'hrv_Latn-ces_Latn': {'num_samples': 1997, 'number_of_characters': 497243, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'hrv_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 503645, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'hrv_Latn-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 523816, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'hrv_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 533791, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'hrv_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 530818, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'hrv_Latn-slk_Latn': {'num_samples': 1997, 'number_of_characters': 508894, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'hrv_Latn-slv_Latn': {'num_samples': 1997, 'number_of_characters': 508821, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'hrv_Latn-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 508228, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'hrv_Latn-srp_Latn': {'num_samples': 1997, 'number_of_characters': 512066, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'hrv_Latn-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 518708, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 128.15, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'hun_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 509557, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'hun_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 522178, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'hun_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 574026, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'hun_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 577503, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'hun_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 526096, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'hun_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 521715, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'hun_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 548176, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'hun_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 571484, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'hun_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 478672, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'hun_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 539951, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'hun_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 565348, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'hun_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 390002, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'hun_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 411781, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'hun_Latn-lav_Latn': {'num_samples': 1997, 'number_of_characters': 546564, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 134.3, 'max_sentence2_length': 503, 'unique_sentence2': 1994}, 'hun_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 537773, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'hun_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 570291, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'hun_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 556242, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'hun_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 555591, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'hun_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 553269, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'hun_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 566576, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'hun_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 550885, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'hun_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 530203, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'hun_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 588337, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'hun_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 543989, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'hun_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 549497, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'hun_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 369854, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'hun_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 536108, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 139.4, 'max_sentence1_length': 508, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'hye_Armn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 563842, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 132.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'hye_Armn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 512435, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 132.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'hye_Armn-kat_Geor': {'num_samples': 1997, 'number_of_characters': 531307, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 132.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 133.5, 'max_sentence2_length': 503, 'unique_sentence2': 1995}, 'hye_Armn-sqi_Latn': {'num_samples': 1997, 'number_of_characters': 548322, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 132.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 142.02, 'max_sentence2_length': 461, 'unique_sentence2': 1996}, 'ibo_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 413608, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'ibo_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 493821, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ibo_Latn-hau_Latn': {'num_samples': 1997, 'number_of_characters': 516067, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'ibo_Latn-nso_Latn': {'num_samples': 1997, 'number_of_characters': 537600, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'ibo_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 483532, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'ibo_Latn-som_Latn': {'num_samples': 1997, 'number_of_characters': 537393, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'ibo_Latn-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 534243, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'ibo_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 518610, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'ibo_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 411339, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'ibo_Latn-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 580384, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'ibo_Latn-wol_Latn': {'num_samples': 1997, 'number_of_characters': 485904, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'ibo_Latn-xho_Latn': {'num_samples': 1997, 'number_of_characters': 514191, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'ibo_Latn-yor_Latn': {'num_samples': 1997, 'number_of_characters': 562189, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'ibo_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 503833, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 123.24, 'max_sentence1_length': 469, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'ind_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 518153, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'ind_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 530774, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'ind_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 582622, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'ind_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 586099, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'ind_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 534692, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ind_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 530311, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'ind_Latn-fij_Latn': {'num_samples': 1997, 'number_of_characters': 587477, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 150.48, 'max_sentence2_length': 448, 'unique_sentence2': 1988}, 'ind_Latn-fil_Latn': {'num_samples': 1997, 'number_of_characters': 580392, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 146.93, 'max_sentence2_length': 554, 'unique_sentence2': 1997}, 'ind_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 556772, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'ind_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 580080, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'ind_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 487268, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'ind_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 548547, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'ind_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 565348, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'ind_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 398598, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'ind_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 420377, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'ind_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 546369, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'ind_Latn-mal_Mlym': {'num_samples': 1997, 'number_of_characters': 591124, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 152.3, 'max_sentence2_length': 540, 'unique_sentence2': 1996}, 'ind_Latn-mlg_Latn': {'num_samples': 1997, 'number_of_characters': 607280, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 160.39, 'max_sentence2_length': 559, 'unique_sentence2': 1994}, 'ind_Latn-mri_Latn': {'num_samples': 1997, 'number_of_characters': 561096, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 137.27, 'max_sentence2_length': 443, 'unique_sentence2': 1997}, 'ind_Latn-msa_Latn': {'num_samples': 1997, 'number_of_characters': 564155, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 138.8, 'max_sentence2_length': 463, 'unique_sentence2': 1997}, 'ind_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 578887, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'ind_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 564838, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'ind_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 564187, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'ind_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 561865, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'ind_Latn-smo_Latn': {'num_samples': 1997, 'number_of_characters': 564827, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 139.14, 'max_sentence2_length': 431, 'unique_sentence2': 1996}, 'ind_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 575172, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'ind_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 559481, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'ind_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 538799, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'ind_Latn-tah_Latn': {'num_samples': 1997, 'number_of_characters': 596595, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 155.04, 'max_sentence2_length': 524, 'unique_sentence2': 1997}, 'ind_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 596933, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'ind_Latn-ton_Latn': {'num_samples': 1997, 'number_of_characters': 600612, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 157.06, 'max_sentence2_length': 468, 'unique_sentence2': 1997}, 'ind_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 552585, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'ind_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 558093, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'ind_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 378450, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'ind_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 544704, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 143.7, 'max_sentence1_length': 486, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'isl_Latn-afr_Latn': {'num_samples': 1997, 'number_of_characters': 530560, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 131.3, 'max_sentence1_length': 399, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 134.38, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'isl_Latn-dan_Latn': {'num_samples': 1997, 'number_of_characters': 514346, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 131.3, 'max_sentence1_length': 399, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 126.26, 'max_sentence2_length': 522, 'unique_sentence2': 1995}, 'isl_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 557858, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 131.3, 'max_sentence1_length': 399, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'isl_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 509928, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 131.3, 'max_sentence1_length': 399, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'isl_Latn-fao_Latn': {'num_samples': 1997, 'number_of_characters': 520011, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 131.3, 'max_sentence1_length': 399, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.1, 'max_sentence2_length': 433, 'unique_sentence2': 1997}, 'isl_Latn-ltz_Latn': {'num_samples': 1997, 'number_of_characters': 542965, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 131.3, 'max_sentence1_length': 399, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 140.59, 'max_sentence2_length': 543, 'unique_sentence2': 1996}, 'isl_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 554123, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 131.3, 'max_sentence1_length': 399, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'isl_Latn-nno_Latn': {'num_samples': 1997, 'number_of_characters': 510565, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 131.3, 'max_sentence1_length': 399, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.37, 'max_sentence2_length': 417, 'unique_sentence2': 1996}, 'isl_Latn-nob_Latn': {'num_samples': 1997, 'number_of_characters': 513652, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 131.3, 'max_sentence1_length': 399, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.91, 'max_sentence2_length': 482, 'unique_sentence2': 1996}, 'isl_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 514035, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 131.3, 'max_sentence1_length': 399, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'ita_Latn-cat_Latn': {'num_samples': 1997, 'number_of_characters': 572177, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 144.83, 'max_sentence1_length': 623, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 141.69, 'max_sentence2_length': 460, 'unique_sentence2': 1997}, 'ita_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 536937, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 144.83, 'max_sentence1_length': 623, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ita_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 582325, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 144.83, 'max_sentence1_length': 623, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'ita_Latn-glg_Latn': {'num_samples': 1997, 'number_of_characters': 561203, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 144.83, 'max_sentence1_length': 623, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 136.2, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'ita_Latn-mlt_Latn': {'num_samples': 1997, 'number_of_characters': 566692, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 144.83, 'max_sentence1_length': 623, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 138.95, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'ita_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 566432, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 144.83, 'max_sentence1_length': 623, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'ita_Latn-ron_Latn': {'num_samples': 1997, 'number_of_characters': 581702, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 144.83, 'max_sentence1_length': 623, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 146.46, 'max_sentence2_length': 518, 'unique_sentence2': 1997}, 'ita_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 577417, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 144.83, 'max_sentence1_length': 623, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'jpn_Jpan-arb_Arab': {'num_samples': 1997, 'number_of_characters': 342807, 'unique_pairs': 1995, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'jpn_Jpan-ben_Beng': {'num_samples': 1997, 'number_of_characters': 355428, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'jpn_Jpan-deu_Latn': {'num_samples': 1997, 'number_of_characters': 407276, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'jpn_Jpan-ell_Grek': {'num_samples': 1997, 'number_of_characters': 410753, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'jpn_Jpan-eng_Latn': {'num_samples': 1997, 'number_of_characters': 359346, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'jpn_Jpan-fas_Arab': {'num_samples': 1997, 'number_of_characters': 354965, 'unique_pairs': 1995, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'jpn_Jpan-fin_Latn': {'num_samples': 1997, 'number_of_characters': 381426, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'jpn_Jpan-fra_Latn': {'num_samples': 1997, 'number_of_characters': 404734, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'jpn_Jpan-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 311922, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'jpn_Jpan-hin_Deva': {'num_samples': 1997, 'number_of_characters': 373201, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'jpn_Jpan-hun_Latn': {'num_samples': 1997, 'number_of_characters': 390002, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'jpn_Jpan-ind_Latn': {'num_samples': 1997, 'number_of_characters': 398598, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'jpn_Jpan-kor_Hang': {'num_samples': 1997, 'number_of_characters': 245031, 'unique_pairs': 1995, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'jpn_Jpan-lit_Latn': {'num_samples': 1997, 'number_of_characters': 371023, 'unique_pairs': 1995, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'jpn_Jpan-nld_Latn': {'num_samples': 1997, 'number_of_characters': 403541, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'jpn_Jpan-pol_Latn': {'num_samples': 1997, 'number_of_characters': 389492, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'jpn_Jpan-por_Latn': {'num_samples': 1997, 'number_of_characters': 388841, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'jpn_Jpan-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 386519, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'jpn_Jpan-spa_Latn': {'num_samples': 1997, 'number_of_characters': 399826, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'jpn_Jpan-swa_Latn': {'num_samples': 1997, 'number_of_characters': 384135, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'jpn_Jpan-swe_Latn': {'num_samples': 1997, 'number_of_characters': 363453, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'jpn_Jpan-tam_Taml': {'num_samples': 1997, 'number_of_characters': 421587, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'jpn_Jpan-tur_Latn': {'num_samples': 1997, 'number_of_characters': 377239, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'jpn_Jpan-vie_Latn': {'num_samples': 1997, 'number_of_characters': 382747, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'jpn_Jpan-yue_Hant': {'num_samples': 1997, 'number_of_characters': 190513, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 4, 'average_sentence2_length': 39.5, 'max_sentence2_length': 133, 'unique_sentence2': 1996}, 'jpn_Jpan-zho_Hans': {'num_samples': 1997, 'number_of_characters': 196587, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 4, 'average_sentence2_length': 42.54, 'max_sentence2_length': 263, 'unique_sentence2': 1997}, 'jpn_Jpan-zho_Hant': {'num_samples': 1997, 'number_of_characters': 203104, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'jpn_Jpan-zul_Latn': {'num_samples': 1997, 'number_of_characters': 369358, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 55.9, 'max_sentence1_length': 189, 'unique_sentence1': 1994, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'kan_Knda-ben_Beng': {'num_samples': 1997, 'number_of_characters': 509338, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'kan_Knda-div_Thaa': {'num_samples': 1997, 'number_of_characters': 569384, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'kan_Knda-eng_Latn': {'num_samples': 1997, 'number_of_characters': 513256, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'kan_Knda-eus_Latn': {'num_samples': 1997, 'number_of_characters': 540739, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'kan_Knda-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 510467, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'kan_Knda-hin_Deva': {'num_samples': 1997, 'number_of_characters': 527111, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'kan_Knda-mar_Deva': {'num_samples': 1997, 'number_of_characters': 526423, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'kan_Knda-nep_Deva': {'num_samples': 1997, 'number_of_characters': 513759, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'kan_Knda-pan_Guru': {'num_samples': 1997, 'number_of_characters': 515958, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'kan_Knda-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 524277, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'kan_Knda-snd_Arab': {'num_samples': 1997, 'number_of_characters': 485863, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'kan_Knda-tam_Taml': {'num_samples': 1997, 'number_of_characters': 575497, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'kan_Knda-tel_Telu': {'num_samples': 1997, 'number_of_characters': 513063, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'kan_Knda-urd_Arab': {'num_samples': 1997, 'number_of_characters': 513534, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 132.97, 'max_sentence1_length': 449, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'kat_Geor-ell_Grek': {'num_samples': 1997, 'number_of_characters': 565719, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 133.5, 'max_sentence1_length': 503, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'kat_Geor-eng_Latn': {'num_samples': 1997, 'number_of_characters': 514312, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 133.5, 'max_sentence1_length': 503, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'kat_Geor-hye_Armn': {'num_samples': 1997, 'number_of_characters': 531307, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 133.5, 'max_sentence1_length': 503, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 132.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'kat_Geor-sqi_Latn': {'num_samples': 1997, 'number_of_characters': 550199, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 133.5, 'max_sentence1_length': 503, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 142.02, 'max_sentence2_length': 461, 'unique_sentence2': 1996}, 'kaz_Cyrl-aze_Latn': {'num_samples': 1997, 'number_of_characters': 529910, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 130.33, 'max_sentence1_length': 473, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 135.02, 'max_sentence2_length': 398, 'unique_sentence2': 1997}, 'kaz_Cyrl-bak_Cyrl': {'num_samples': 1997, 'number_of_characters': 506602, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 130.33, 'max_sentence1_length': 473, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 123.35, 'max_sentence2_length': 437, 'unique_sentence2': 1995}, 'kaz_Cyrl-eng_Latn': {'num_samples': 1997, 'number_of_characters': 507996, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 130.33, 'max_sentence1_length': 473, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'kaz_Cyrl-kir_Cyrl': {'num_samples': 1997, 'number_of_characters': 511140, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 130.33, 'max_sentence1_length': 473, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.62, 'max_sentence2_length': 395, 'unique_sentence2': 1996}, 'kaz_Cyrl-tat_Cyrl': {'num_samples': 1997, 'number_of_characters': 506202, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 130.33, 'max_sentence1_length': 473, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 123.15, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'kaz_Cyrl-tuk_Latn': {'num_samples': 1997, 'number_of_characters': 545550, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 130.33, 'max_sentence1_length': 473, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 142.85, 'max_sentence2_length': 576, 'unique_sentence2': 1996}, 'kaz_Cyrl-tur_Latn': {'num_samples': 1997, 'number_of_characters': 525889, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 130.33, 'max_sentence1_length': 473, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'kaz_Cyrl-uig_Arab': {'num_samples': 1997, 'number_of_characters': 571298, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 130.33, 'max_sentence1_length': 473, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 155.74, 'max_sentence2_length': 592, 'unique_sentence2': 1996}, 'kaz_Cyrl-uzb_Latn': {'num_samples': 1997, 'number_of_characters': 553971, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 130.33, 'max_sentence1_length': 473, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 147.07, 'max_sentence2_length': 470, 'unique_sentence2': 1996}, 'khm_Khmr-bod_Tibt': {'num_samples': 1997, 'number_of_characters': 589120, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 144.46, 'max_sentence1_length': 517, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 150.54, 'max_sentence2_length': 478, 'unique_sentence2': 1993}, 'khm_Khmr-dzo_Tibt': {'num_samples': 1997, 'number_of_characters': 531712, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 144.46, 'max_sentence1_length': 517, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 121.79, 'max_sentence2_length': 411, 'unique_sentence2': 1992}, 'khm_Khmr-eng_Latn': {'num_samples': 1997, 'number_of_characters': 536211, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 144.46, 'max_sentence1_length': 517, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'khm_Khmr-lao_Laoo': {'num_samples': 1997, 'number_of_characters': 555471, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 144.46, 'max_sentence1_length': 517, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 133.69, 'max_sentence2_length': 507, 'unique_sentence2': 1997}, 'khm_Khmr-mon_Mong': {'num_samples': 1997, 'number_of_characters': 547539, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 144.46, 'max_sentence1_length': 517, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 129.72, 'max_sentence2_length': 414, 'unique_sentence2': 1997}, 'khm_Khmr-mya_Mymr': {'num_samples': 1997, 'number_of_characters': 600345, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 144.46, 'max_sentence1_length': 517, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 156.16, 'max_sentence2_length': 773, 'unique_sentence2': 1997}, 'khm_Khmr-tha_Thai': {'num_samples': 1997, 'number_of_characters': 525959, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 144.46, 'max_sentence1_length': 517, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 118.91, 'max_sentence2_length': 439, 'unique_sentence2': 1996}, 'kin_Latn-bem_Latn': {'num_samples': 1997, 'number_of_characters': 602279, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 152.12, 'max_sentence1_length': 541, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 149.47, 'max_sentence2_length': 465, 'unique_sentence2': 1997}, 'kin_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 551507, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 152.12, 'max_sentence1_length': 541, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'kin_Latn-ewe_Latn': {'num_samples': 1997, 'number_of_characters': 542765, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 152.12, 'max_sentence1_length': 541, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 119.67, 'max_sentence2_length': 493, 'unique_sentence2': 1994}, 'kin_Latn-fuc_Latn': {'num_samples': 1997, 'number_of_characters': 532267, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 152.12, 'max_sentence1_length': 541, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 114.41, 'max_sentence2_length': 376, 'unique_sentence2': 1996}, 'kin_Latn-nde_Latn': {'num_samples': 1997, 'number_of_characters': 601526, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 152.12, 'max_sentence1_length': 541, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 149.09, 'max_sentence2_length': 590, 'unique_sentence2': 1997}, 'kin_Latn-nya_Latn': {'num_samples': 1997, 'number_of_characters': 588069, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 152.12, 'max_sentence1_length': 541, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 142.35, 'max_sentence2_length': 464, 'unique_sentence2': 1993}, 'kin_Latn-sna_Latn': {'num_samples': 1997, 'number_of_characters': 602117, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 152.12, 'max_sentence1_length': 541, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 149.39, 'max_sentence2_length': 511, 'unique_sentence2': 1995}, 'kin_Latn-ven_Latn': {'num_samples': 1997, 'number_of_characters': 603543, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 152.12, 'max_sentence1_length': 541, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 150.1, 'max_sentence2_length': 535, 'unique_sentence2': 1993}, 'kir_Cyrl-aze_Latn': {'num_samples': 1997, 'number_of_characters': 520498, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 125.62, 'max_sentence1_length': 395, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 135.02, 'max_sentence2_length': 398, 'unique_sentence2': 1997}, 'kir_Cyrl-bak_Cyrl': {'num_samples': 1997, 'number_of_characters': 497190, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 125.62, 'max_sentence1_length': 395, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 123.35, 'max_sentence2_length': 437, 'unique_sentence2': 1995}, 'kir_Cyrl-eng_Latn': {'num_samples': 1997, 'number_of_characters': 498584, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 125.62, 'max_sentence1_length': 395, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'kir_Cyrl-kaz_Cyrl': {'num_samples': 1997, 'number_of_characters': 511140, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.62, 'max_sentence1_length': 395, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 130.33, 'max_sentence2_length': 473, 'unique_sentence2': 1996}, 'kir_Cyrl-tat_Cyrl': {'num_samples': 1997, 'number_of_characters': 496790, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.62, 'max_sentence1_length': 395, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 123.15, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'kir_Cyrl-tuk_Latn': {'num_samples': 1997, 'number_of_characters': 536138, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.62, 'max_sentence1_length': 395, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 142.85, 'max_sentence2_length': 576, 'unique_sentence2': 1996}, 'kir_Cyrl-tur_Latn': {'num_samples': 1997, 'number_of_characters': 516477, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 125.62, 'max_sentence1_length': 395, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'kir_Cyrl-uig_Arab': {'num_samples': 1997, 'number_of_characters': 561886, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 125.62, 'max_sentence1_length': 395, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 155.74, 'max_sentence2_length': 592, 'unique_sentence2': 1996}, 'kir_Cyrl-uzb_Latn': {'num_samples': 1997, 'number_of_characters': 544559, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.62, 'max_sentence1_length': 395, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 147.07, 'max_sentence2_length': 470, 'unique_sentence2': 1996}, 'kmr_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 477127, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 123.16, 'max_sentence1_length': 420, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'kmr_Latn-ckb_Arab': {'num_samples': 1997, 'number_of_characters': 498313, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 123.16, 'max_sentence1_length': 420, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 126.37, 'max_sentence2_length': 399, 'unique_sentence2': 1995}, 'kmr_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 493666, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 123.16, 'max_sentence1_length': 420, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'kmr_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 489285, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 123.16, 'max_sentence1_length': 420, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'kmr_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 446242, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 123.16, 'max_sentence1_length': 420, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'kmr_Latn-mey_Arab': {'num_samples': 1997, 'number_of_characters': 459781, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 123.16, 'max_sentence1_length': 420, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 107.08, 'max_sentence2_length': 392, 'unique_sentence2': 1993}, 'kmr_Latn-prs_Arab': {'num_samples': 1997, 'number_of_characters': 488482, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 123.16, 'max_sentence1_length': 420, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 121.45, 'max_sentence2_length': 365, 'unique_sentence2': 1997}, 'kmr_Latn-pus_Arab': {'num_samples': 1997, 'number_of_characters': 488579, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 123.16, 'max_sentence1_length': 420, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 121.5, 'max_sentence2_length': 366, 'unique_sentence2': 1996}, 'kmr_Latn-shi_Arab': {'num_samples': 1997, 'number_of_characters': 460859, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 123.16, 'max_sentence1_length': 420, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 107.62, 'max_sentence2_length': 378, 'unique_sentence2': 1996}, 'kmr_Latn-tgk_Cyrl': {'num_samples': 1997, 'number_of_characters': 520093, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 123.16, 'max_sentence1_length': 420, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 137.28, 'max_sentence2_length': 451, 'unique_sentence2': 1995}, 'kor_Hang-arb_Arab': {'num_samples': 1997, 'number_of_characters': 364586, 'unique_pairs': 1995, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'kor_Hang-ben_Beng': {'num_samples': 1997, 'number_of_characters': 377207, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'kor_Hang-deu_Latn': {'num_samples': 1997, 'number_of_characters': 429055, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'kor_Hang-ell_Grek': {'num_samples': 1997, 'number_of_characters': 432532, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'kor_Hang-eng_Latn': {'num_samples': 1997, 'number_of_characters': 381125, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'kor_Hang-fas_Arab': {'num_samples': 1997, 'number_of_characters': 376744, 'unique_pairs': 1995, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'kor_Hang-fin_Latn': {'num_samples': 1997, 'number_of_characters': 403205, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'kor_Hang-fra_Latn': {'num_samples': 1997, 'number_of_characters': 426513, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'kor_Hang-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 333701, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'kor_Hang-hin_Deva': {'num_samples': 1997, 'number_of_characters': 394980, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'kor_Hang-hun_Latn': {'num_samples': 1997, 'number_of_characters': 411781, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'kor_Hang-ind_Latn': {'num_samples': 1997, 'number_of_characters': 420377, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'kor_Hang-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 245031, 'unique_pairs': 1995, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'kor_Hang-lit_Latn': {'num_samples': 1997, 'number_of_characters': 392802, 'unique_pairs': 1995, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'kor_Hang-nld_Latn': {'num_samples': 1997, 'number_of_characters': 425320, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'kor_Hang-pol_Latn': {'num_samples': 1997, 'number_of_characters': 411271, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'kor_Hang-por_Latn': {'num_samples': 1997, 'number_of_characters': 410620, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'kor_Hang-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 408298, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'kor_Hang-spa_Latn': {'num_samples': 1997, 'number_of_characters': 421605, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'kor_Hang-swa_Latn': {'num_samples': 1997, 'number_of_characters': 405914, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'kor_Hang-swe_Latn': {'num_samples': 1997, 'number_of_characters': 385232, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'kor_Hang-tam_Taml': {'num_samples': 1997, 'number_of_characters': 443366, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'kor_Hang-tur_Latn': {'num_samples': 1997, 'number_of_characters': 399018, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'kor_Hang-vie_Latn': {'num_samples': 1997, 'number_of_characters': 404526, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'kor_Hang-yue_Hant': {'num_samples': 1997, 'number_of_characters': 212292, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 4, 'average_sentence2_length': 39.5, 'max_sentence2_length': 133, 'unique_sentence2': 1996}, 'kor_Hang-zho_Hans': {'num_samples': 1997, 'number_of_characters': 218366, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 4, 'average_sentence2_length': 42.54, 'max_sentence2_length': 263, 'unique_sentence2': 1997}, 'kor_Hang-zho_Hant': {'num_samples': 1997, 'number_of_characters': 224883, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'kor_Hang-zul_Latn': {'num_samples': 1997, 'number_of_characters': 391137, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 66.8, 'max_sentence1_length': 217, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'lao_Laoo-bod_Tibt': {'num_samples': 1997, 'number_of_characters': 567609, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 133.69, 'max_sentence1_length': 507, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 150.54, 'max_sentence2_length': 478, 'unique_sentence2': 1993}, 'lao_Laoo-dzo_Tibt': {'num_samples': 1997, 'number_of_characters': 510201, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 133.69, 'max_sentence1_length': 507, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 121.79, 'max_sentence2_length': 411, 'unique_sentence2': 1992}, 'lao_Laoo-eng_Latn': {'num_samples': 1997, 'number_of_characters': 514700, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 133.69, 'max_sentence1_length': 507, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'lao_Laoo-khm_Khmr': {'num_samples': 1997, 'number_of_characters': 555471, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 133.69, 'max_sentence1_length': 507, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 144.46, 'max_sentence2_length': 517, 'unique_sentence2': 1996}, 'lao_Laoo-mon_Mong': {'num_samples': 1997, 'number_of_characters': 526028, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 133.69, 'max_sentence1_length': 507, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 129.72, 'max_sentence2_length': 414, 'unique_sentence2': 1997}, 'lao_Laoo-mya_Mymr': {'num_samples': 1997, 'number_of_characters': 578834, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 133.69, 'max_sentence1_length': 507, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 156.16, 'max_sentence2_length': 773, 'unique_sentence2': 1997}, 'lao_Laoo-tha_Thai': {'num_samples': 1997, 'number_of_characters': 504448, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 133.69, 'max_sentence1_length': 507, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 118.91, 'max_sentence2_length': 439, 'unique_sentence2': 1996}, 'lav_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 515908, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 134.3, 'max_sentence1_length': 503, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'lav_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 537988, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 134.3, 'max_sentence1_length': 503, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'lav_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 546564, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 134.3, 'max_sentence1_length': 503, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'lav_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 527585, 'unique_pairs': 1995, 'min_sentence1_length': 7, 'average_sentence1_length': 134.3, 'max_sentence1_length': 503, 'unique_sentence1': 1994, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'lit_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 490578, 'unique_pairs': 1995, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'lit_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 503199, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'lit_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 555047, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'lit_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 558524, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'lit_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 507117, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'lit_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 502736, 'unique_pairs': 1995, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'lit_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 529197, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'lit_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 552505, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'lit_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 459693, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'lit_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 520972, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'lit_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 537773, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'lit_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 546369, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'lit_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 371023, 'unique_pairs': 1995, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'lit_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 392802, 'unique_pairs': 1995, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'lit_Latn-lav_Latn': {'num_samples': 1997, 'number_of_characters': 527585, 'unique_pairs': 1995, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 134.3, 'max_sentence2_length': 503, 'unique_sentence2': 1994}, 'lit_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 551312, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'lit_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 537263, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'lit_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 536612, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'lit_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 534290, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'lit_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 547597, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'lit_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 531906, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'lit_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 511224, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'lit_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 569358, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'lit_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 525010, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'lit_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 530518, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'lit_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 350875, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'lit_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 517129, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 129.89, 'max_sentence1_length': 446, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'ltz_Latn-afr_Latn': {'num_samples': 1997, 'number_of_characters': 549109, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 140.59, 'max_sentence1_length': 543, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 134.38, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'ltz_Latn-dan_Latn': {'num_samples': 1997, 'number_of_characters': 532895, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 140.59, 'max_sentence1_length': 543, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 126.26, 'max_sentence2_length': 522, 'unique_sentence2': 1995}, 'ltz_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 576407, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 140.59, 'max_sentence1_length': 543, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'ltz_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 528477, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 140.59, 'max_sentence1_length': 543, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ltz_Latn-fao_Latn': {'num_samples': 1997, 'number_of_characters': 538560, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 140.59, 'max_sentence1_length': 543, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.1, 'max_sentence2_length': 433, 'unique_sentence2': 1997}, 'ltz_Latn-isl_Latn': {'num_samples': 1997, 'number_of_characters': 542965, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 140.59, 'max_sentence1_length': 543, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 131.3, 'max_sentence2_length': 399, 'unique_sentence2': 1996}, 'ltz_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 572672, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 140.59, 'max_sentence1_length': 543, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'ltz_Latn-nno_Latn': {'num_samples': 1997, 'number_of_characters': 529114, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 140.59, 'max_sentence1_length': 543, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.37, 'max_sentence2_length': 417, 'unique_sentence2': 1996}, 'ltz_Latn-nob_Latn': {'num_samples': 1997, 'number_of_characters': 532201, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 140.59, 'max_sentence1_length': 543, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.91, 'max_sentence2_length': 482, 'unique_sentence2': 1996}, 'ltz_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 532584, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 140.59, 'max_sentence1_length': 543, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'mal_Mlym-eng_Latn': {'num_samples': 1997, 'number_of_characters': 551872, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 152.3, 'max_sentence1_length': 540, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'mal_Mlym-fij_Latn': {'num_samples': 1997, 'number_of_characters': 604657, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 152.3, 'max_sentence1_length': 540, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 150.48, 'max_sentence2_length': 448, 'unique_sentence2': 1988}, 'mal_Mlym-fil_Latn': {'num_samples': 1997, 'number_of_characters': 597572, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 152.3, 'max_sentence1_length': 540, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 146.93, 'max_sentence2_length': 554, 'unique_sentence2': 1997}, 'mal_Mlym-ind_Latn': {'num_samples': 1997, 'number_of_characters': 591124, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 152.3, 'max_sentence1_length': 540, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'mal_Mlym-mlg_Latn': {'num_samples': 1997, 'number_of_characters': 624460, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 152.3, 'max_sentence1_length': 540, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 160.39, 'max_sentence2_length': 559, 'unique_sentence2': 1994}, 'mal_Mlym-mri_Latn': {'num_samples': 1997, 'number_of_characters': 578276, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 152.3, 'max_sentence1_length': 540, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 137.27, 'max_sentence2_length': 443, 'unique_sentence2': 1997}, 'mal_Mlym-msa_Latn': {'num_samples': 1997, 'number_of_characters': 581335, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 152.3, 'max_sentence1_length': 540, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 138.8, 'max_sentence2_length': 463, 'unique_sentence2': 1997}, 'mal_Mlym-smo_Latn': {'num_samples': 1997, 'number_of_characters': 582007, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 152.3, 'max_sentence1_length': 540, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 139.14, 'max_sentence2_length': 431, 'unique_sentence2': 1996}, 'mal_Mlym-tah_Latn': {'num_samples': 1997, 'number_of_characters': 613775, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 152.3, 'max_sentence1_length': 540, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 155.04, 'max_sentence2_length': 524, 'unique_sentence2': 1997}, 'mal_Mlym-ton_Latn': {'num_samples': 1997, 'number_of_characters': 617792, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 152.3, 'max_sentence1_length': 540, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 157.06, 'max_sentence2_length': 468, 'unique_sentence2': 1997}, 'mar_Deva-ben_Beng': {'num_samples': 1997, 'number_of_characters': 504689, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'mar_Deva-div_Thaa': {'num_samples': 1997, 'number_of_characters': 564735, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'mar_Deva-eng_Latn': {'num_samples': 1997, 'number_of_characters': 508607, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'mar_Deva-eus_Latn': {'num_samples': 1997, 'number_of_characters': 536090, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'mar_Deva-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 505818, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'mar_Deva-hin_Deva': {'num_samples': 1997, 'number_of_characters': 522462, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'mar_Deva-kan_Knda': {'num_samples': 1997, 'number_of_characters': 526423, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'mar_Deva-nep_Deva': {'num_samples': 1997, 'number_of_characters': 509110, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'mar_Deva-pan_Guru': {'num_samples': 1997, 'number_of_characters': 511309, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'mar_Deva-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 519628, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'mar_Deva-snd_Arab': {'num_samples': 1997, 'number_of_characters': 481214, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'mar_Deva-tam_Taml': {'num_samples': 1997, 'number_of_characters': 570848, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'mar_Deva-tel_Telu': {'num_samples': 1997, 'number_of_characters': 508414, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'mar_Deva-urd_Arab': {'num_samples': 1997, 'number_of_characters': 508885, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 130.64, 'max_sentence1_length': 443, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'mey_Arab-arb_Arab': {'num_samples': 1997, 'number_of_characters': 445016, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 107.08, 'max_sentence1_length': 392, 'unique_sentence1': 1993, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'mey_Arab-ckb_Arab': {'num_samples': 1997, 'number_of_characters': 466202, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 107.08, 'max_sentence1_length': 392, 'unique_sentence1': 1993, 'min_sentence2_length': 5, 'average_sentence2_length': 126.37, 'max_sentence2_length': 399, 'unique_sentence2': 1995}, 'mey_Arab-eng_Latn': {'num_samples': 1997, 'number_of_characters': 461555, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 107.08, 'max_sentence1_length': 392, 'unique_sentence1': 1993, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'mey_Arab-fas_Arab': {'num_samples': 1997, 'number_of_characters': 457174, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 107.08, 'max_sentence1_length': 392, 'unique_sentence1': 1993, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'mey_Arab-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 414131, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 107.08, 'max_sentence1_length': 392, 'unique_sentence1': 1993, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'mey_Arab-kmr_Latn': {'num_samples': 1997, 'number_of_characters': 459781, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 107.08, 'max_sentence1_length': 392, 'unique_sentence1': 1993, 'min_sentence2_length': 7, 'average_sentence2_length': 123.16, 'max_sentence2_length': 420, 'unique_sentence2': 1996}, 'mey_Arab-prs_Arab': {'num_samples': 1997, 'number_of_characters': 456371, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 107.08, 'max_sentence1_length': 392, 'unique_sentence1': 1993, 'min_sentence2_length': 8, 'average_sentence2_length': 121.45, 'max_sentence2_length': 365, 'unique_sentence2': 1997}, 'mey_Arab-pus_Arab': {'num_samples': 1997, 'number_of_characters': 456468, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 107.08, 'max_sentence1_length': 392, 'unique_sentence1': 1993, 'min_sentence2_length': 8, 'average_sentence2_length': 121.5, 'max_sentence2_length': 366, 'unique_sentence2': 1996}, 'mey_Arab-shi_Arab': {'num_samples': 1997, 'number_of_characters': 428748, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 107.08, 'max_sentence1_length': 392, 'unique_sentence1': 1993, 'min_sentence2_length': 3, 'average_sentence2_length': 107.62, 'max_sentence2_length': 378, 'unique_sentence2': 1996}, 'mey_Arab-tgk_Cyrl': {'num_samples': 1997, 'number_of_characters': 487982, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 107.08, 'max_sentence1_length': 392, 'unique_sentence1': 1993, 'min_sentence2_length': 3, 'average_sentence2_length': 137.28, 'max_sentence2_length': 451, 'unique_sentence2': 1995}, 'mkd_Cyrl-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 523981, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'mkd_Cyrl-bos_Latn': {'num_samples': 1997, 'number_of_characters': 522801, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'mkd_Cyrl-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 537780, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'mkd_Cyrl-ces_Latn': {'num_samples': 1997, 'number_of_characters': 509209, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'mkd_Cyrl-eng_Latn': {'num_samples': 1997, 'number_of_characters': 515611, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'mkd_Cyrl-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 523816, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'mkd_Cyrl-pol_Latn': {'num_samples': 1997, 'number_of_characters': 545757, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'mkd_Cyrl-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 542784, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'mkd_Cyrl-slk_Latn': {'num_samples': 1997, 'number_of_characters': 520860, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'mkd_Cyrl-slv_Latn': {'num_samples': 1997, 'number_of_characters': 520787, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'mkd_Cyrl-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 520194, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'mkd_Cyrl-srp_Latn': {'num_samples': 1997, 'number_of_characters': 524032, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'mkd_Cyrl-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 530674, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.15, 'max_sentence1_length': 451, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'mlg_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 568028, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 160.39, 'max_sentence1_length': 559, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'mlg_Latn-fij_Latn': {'num_samples': 1997, 'number_of_characters': 620813, 'unique_pairs': 1995, 'min_sentence1_length': 6, 'average_sentence1_length': 160.39, 'max_sentence1_length': 559, 'unique_sentence1': 1994, 'min_sentence2_length': 10, 'average_sentence2_length': 150.48, 'max_sentence2_length': 448, 'unique_sentence2': 1988}, 'mlg_Latn-fil_Latn': {'num_samples': 1997, 'number_of_characters': 613728, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 160.39, 'max_sentence1_length': 559, 'unique_sentence1': 1994, 'min_sentence2_length': 10, 'average_sentence2_length': 146.93, 'max_sentence2_length': 554, 'unique_sentence2': 1997}, 'mlg_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 607280, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 160.39, 'max_sentence1_length': 559, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'mlg_Latn-mal_Mlym': {'num_samples': 1997, 'number_of_characters': 624460, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 160.39, 'max_sentence1_length': 559, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 152.3, 'max_sentence2_length': 540, 'unique_sentence2': 1996}, 'mlg_Latn-mri_Latn': {'num_samples': 1997, 'number_of_characters': 594432, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 160.39, 'max_sentence1_length': 559, 'unique_sentence1': 1994, 'min_sentence2_length': 9, 'average_sentence2_length': 137.27, 'max_sentence2_length': 443, 'unique_sentence2': 1997}, 'mlg_Latn-msa_Latn': {'num_samples': 1997, 'number_of_characters': 597491, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 160.39, 'max_sentence1_length': 559, 'unique_sentence1': 1994, 'min_sentence2_length': 6, 'average_sentence2_length': 138.8, 'max_sentence2_length': 463, 'unique_sentence2': 1997}, 'mlg_Latn-smo_Latn': {'num_samples': 1997, 'number_of_characters': 598163, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 160.39, 'max_sentence1_length': 559, 'unique_sentence1': 1994, 'min_sentence2_length': 8, 'average_sentence2_length': 139.14, 'max_sentence2_length': 431, 'unique_sentence2': 1996}, 'mlg_Latn-tah_Latn': {'num_samples': 1997, 'number_of_characters': 629931, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 160.39, 'max_sentence1_length': 559, 'unique_sentence1': 1994, 'min_sentence2_length': 9, 'average_sentence2_length': 155.04, 'max_sentence2_length': 524, 'unique_sentence2': 1997}, 'mlg_Latn-ton_Latn': {'num_samples': 1997, 'number_of_characters': 633948, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 160.39, 'max_sentence1_length': 559, 'unique_sentence1': 1994, 'min_sentence2_length': 7, 'average_sentence2_length': 157.06, 'max_sentence2_length': 468, 'unique_sentence2': 1997}, 'mlt_Latn-cat_Latn': {'num_samples': 1997, 'number_of_characters': 560435, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 138.95, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 141.69, 'max_sentence2_length': 460, 'unique_sentence2': 1997}, 'mlt_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 525195, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 138.95, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'mlt_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 570583, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 138.95, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'mlt_Latn-glg_Latn': {'num_samples': 1997, 'number_of_characters': 549461, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 138.95, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 136.2, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'mlt_Latn-ita_Latn': {'num_samples': 1997, 'number_of_characters': 566692, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 138.95, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 144.83, 'max_sentence2_length': 623, 'unique_sentence2': 1996}, 'mlt_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 554690, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 138.95, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'mlt_Latn-ron_Latn': {'num_samples': 1997, 'number_of_characters': 569960, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 138.95, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 146.46, 'max_sentence2_length': 518, 'unique_sentence2': 1997}, 'mlt_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 565675, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 138.95, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'mon_Mong-bod_Tibt': {'num_samples': 1997, 'number_of_characters': 559677, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 129.72, 'max_sentence1_length': 414, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 150.54, 'max_sentence2_length': 478, 'unique_sentence2': 1993}, 'mon_Mong-dzo_Tibt': {'num_samples': 1997, 'number_of_characters': 502269, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 129.72, 'max_sentence1_length': 414, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 121.79, 'max_sentence2_length': 411, 'unique_sentence2': 1992}, 'mon_Mong-eng_Latn': {'num_samples': 1997, 'number_of_characters': 506768, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 129.72, 'max_sentence1_length': 414, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'mon_Mong-khm_Khmr': {'num_samples': 1997, 'number_of_characters': 547539, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 129.72, 'max_sentence1_length': 414, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 144.46, 'max_sentence2_length': 517, 'unique_sentence2': 1996}, 'mon_Mong-lao_Laoo': {'num_samples': 1997, 'number_of_characters': 526028, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 129.72, 'max_sentence1_length': 414, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 133.69, 'max_sentence2_length': 507, 'unique_sentence2': 1997}, 'mon_Mong-mya_Mymr': {'num_samples': 1997, 'number_of_characters': 570902, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 129.72, 'max_sentence1_length': 414, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 156.16, 'max_sentence2_length': 773, 'unique_sentence2': 1997}, 'mon_Mong-tha_Thai': {'num_samples': 1997, 'number_of_characters': 496516, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 129.72, 'max_sentence1_length': 414, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 118.91, 'max_sentence2_length': 439, 'unique_sentence2': 1996}, 'mri_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 521844, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 137.27, 'max_sentence1_length': 443, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'mri_Latn-fij_Latn': {'num_samples': 1997, 'number_of_characters': 574629, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 137.27, 'max_sentence1_length': 443, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 150.48, 'max_sentence2_length': 448, 'unique_sentence2': 1988}, 'mri_Latn-fil_Latn': {'num_samples': 1997, 'number_of_characters': 567544, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 137.27, 'max_sentence1_length': 443, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 146.93, 'max_sentence2_length': 554, 'unique_sentence2': 1997}, 'mri_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 561096, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 137.27, 'max_sentence1_length': 443, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'mri_Latn-mal_Mlym': {'num_samples': 1997, 'number_of_characters': 578276, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 137.27, 'max_sentence1_length': 443, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 152.3, 'max_sentence2_length': 540, 'unique_sentence2': 1996}, 'mri_Latn-mlg_Latn': {'num_samples': 1997, 'number_of_characters': 594432, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 137.27, 'max_sentence1_length': 443, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 160.39, 'max_sentence2_length': 559, 'unique_sentence2': 1994}, 'mri_Latn-msa_Latn': {'num_samples': 1997, 'number_of_characters': 551307, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 137.27, 'max_sentence1_length': 443, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 138.8, 'max_sentence2_length': 463, 'unique_sentence2': 1997}, 'mri_Latn-smo_Latn': {'num_samples': 1997, 'number_of_characters': 551979, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 137.27, 'max_sentence1_length': 443, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 139.14, 'max_sentence2_length': 431, 'unique_sentence2': 1996}, 'mri_Latn-tah_Latn': {'num_samples': 1997, 'number_of_characters': 583747, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 137.27, 'max_sentence1_length': 443, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 155.04, 'max_sentence2_length': 524, 'unique_sentence2': 1997}, 'mri_Latn-ton_Latn': {'num_samples': 1997, 'number_of_characters': 587764, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 137.27, 'max_sentence1_length': 443, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 157.06, 'max_sentence2_length': 468, 'unique_sentence2': 1997}, 'msa_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 524903, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 138.8, 'max_sentence1_length': 463, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'msa_Latn-fij_Latn': {'num_samples': 1997, 'number_of_characters': 577688, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 138.8, 'max_sentence1_length': 463, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 150.48, 'max_sentence2_length': 448, 'unique_sentence2': 1988}, 'msa_Latn-fil_Latn': {'num_samples': 1997, 'number_of_characters': 570603, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 138.8, 'max_sentence1_length': 463, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 146.93, 'max_sentence2_length': 554, 'unique_sentence2': 1997}, 'msa_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 564155, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 138.8, 'max_sentence1_length': 463, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'msa_Latn-mal_Mlym': {'num_samples': 1997, 'number_of_characters': 581335, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 138.8, 'max_sentence1_length': 463, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 152.3, 'max_sentence2_length': 540, 'unique_sentence2': 1996}, 'msa_Latn-mlg_Latn': {'num_samples': 1997, 'number_of_characters': 597491, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 138.8, 'max_sentence1_length': 463, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 160.39, 'max_sentence2_length': 559, 'unique_sentence2': 1994}, 'msa_Latn-mri_Latn': {'num_samples': 1997, 'number_of_characters': 551307, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 138.8, 'max_sentence1_length': 463, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 137.27, 'max_sentence2_length': 443, 'unique_sentence2': 1997}, 'msa_Latn-smo_Latn': {'num_samples': 1997, 'number_of_characters': 555038, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 138.8, 'max_sentence1_length': 463, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 139.14, 'max_sentence2_length': 431, 'unique_sentence2': 1996}, 'msa_Latn-tah_Latn': {'num_samples': 1997, 'number_of_characters': 586806, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 138.8, 'max_sentence1_length': 463, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 155.04, 'max_sentence2_length': 524, 'unique_sentence2': 1997}, 'msa_Latn-ton_Latn': {'num_samples': 1997, 'number_of_characters': 590823, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 138.8, 'max_sentence1_length': 463, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 157.06, 'max_sentence2_length': 468, 'unique_sentence2': 1997}, 'mya_Mymr-bod_Tibt': {'num_samples': 1997, 'number_of_characters': 612483, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 156.16, 'max_sentence1_length': 773, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 150.54, 'max_sentence2_length': 478, 'unique_sentence2': 1993}, 'mya_Mymr-dzo_Tibt': {'num_samples': 1997, 'number_of_characters': 555075, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 156.16, 'max_sentence1_length': 773, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 121.79, 'max_sentence2_length': 411, 'unique_sentence2': 1992}, 'mya_Mymr-eng_Latn': {'num_samples': 1997, 'number_of_characters': 559574, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 156.16, 'max_sentence1_length': 773, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'mya_Mymr-khm_Khmr': {'num_samples': 1997, 'number_of_characters': 600345, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 156.16, 'max_sentence1_length': 773, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 144.46, 'max_sentence2_length': 517, 'unique_sentence2': 1996}, 'mya_Mymr-lao_Laoo': {'num_samples': 1997, 'number_of_characters': 578834, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 156.16, 'max_sentence1_length': 773, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 133.69, 'max_sentence2_length': 507, 'unique_sentence2': 1997}, 'mya_Mymr-mon_Mong': {'num_samples': 1997, 'number_of_characters': 570902, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 156.16, 'max_sentence1_length': 773, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 129.72, 'max_sentence2_length': 414, 'unique_sentence2': 1997}, 'mya_Mymr-tha_Thai': {'num_samples': 1997, 'number_of_characters': 549322, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 156.16, 'max_sentence1_length': 773, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 118.91, 'max_sentence2_length': 439, 'unique_sentence2': 1996}, 'nde_Latn-bem_Latn': {'num_samples': 1997, 'number_of_characters': 596231, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.09, 'max_sentence1_length': 590, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 149.47, 'max_sentence2_length': 465, 'unique_sentence2': 1997}, 'nde_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 545459, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.09, 'max_sentence1_length': 590, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'nde_Latn-ewe_Latn': {'num_samples': 1997, 'number_of_characters': 536717, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.09, 'max_sentence1_length': 590, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 119.67, 'max_sentence2_length': 493, 'unique_sentence2': 1994}, 'nde_Latn-fuc_Latn': {'num_samples': 1997, 'number_of_characters': 526219, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.09, 'max_sentence1_length': 590, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 114.41, 'max_sentence2_length': 376, 'unique_sentence2': 1996}, 'nde_Latn-kin_Latn': {'num_samples': 1997, 'number_of_characters': 601526, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.09, 'max_sentence1_length': 590, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 152.12, 'max_sentence2_length': 541, 'unique_sentence2': 1996}, 'nde_Latn-nya_Latn': {'num_samples': 1997, 'number_of_characters': 582021, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.09, 'max_sentence1_length': 590, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 142.35, 'max_sentence2_length': 464, 'unique_sentence2': 1993}, 'nde_Latn-sna_Latn': {'num_samples': 1997, 'number_of_characters': 596069, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.09, 'max_sentence1_length': 590, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 149.39, 'max_sentence2_length': 511, 'unique_sentence2': 1995}, 'nde_Latn-ven_Latn': {'num_samples': 1997, 'number_of_characters': 597495, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.09, 'max_sentence1_length': 590, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 150.1, 'max_sentence2_length': 535, 'unique_sentence2': 1993}, 'nep_Deva-ben_Beng': {'num_samples': 1997, 'number_of_characters': 492025, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'nep_Deva-div_Thaa': {'num_samples': 1997, 'number_of_characters': 552071, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'nep_Deva-eng_Latn': {'num_samples': 1997, 'number_of_characters': 495943, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'nep_Deva-eus_Latn': {'num_samples': 1997, 'number_of_characters': 523426, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'nep_Deva-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 493154, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'nep_Deva-hin_Deva': {'num_samples': 1997, 'number_of_characters': 509798, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'nep_Deva-kan_Knda': {'num_samples': 1997, 'number_of_characters': 513759, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'nep_Deva-mar_Deva': {'num_samples': 1997, 'number_of_characters': 509110, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'nep_Deva-pan_Guru': {'num_samples': 1997, 'number_of_characters': 498645, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'nep_Deva-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 506964, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'nep_Deva-snd_Arab': {'num_samples': 1997, 'number_of_characters': 468550, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'nep_Deva-tam_Taml': {'num_samples': 1997, 'number_of_characters': 558184, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'nep_Deva-tel_Telu': {'num_samples': 1997, 'number_of_characters': 495750, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'nep_Deva-urd_Arab': {'num_samples': 1997, 'number_of_characters': 496221, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 124.3, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'nld_Latn-afr_Latn': {'num_samples': 1997, 'number_of_characters': 560267, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 134.38, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'nld_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 523096, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'nld_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 535717, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'nld_Latn-dan_Latn': {'num_samples': 1997, 'number_of_characters': 544053, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 126.26, 'max_sentence2_length': 522, 'unique_sentence2': 1995}, 'nld_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 587565, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'nld_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 591042, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'nld_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 539635, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'nld_Latn-fao_Latn': {'num_samples': 1997, 'number_of_characters': 549718, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.1, 'max_sentence2_length': 433, 'unique_sentence2': 1997}, 'nld_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 535254, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'nld_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 561715, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'nld_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 585023, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'nld_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 492211, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'nld_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 553490, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'nld_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 570291, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'nld_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 578887, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'nld_Latn-isl_Latn': {'num_samples': 1997, 'number_of_characters': 554123, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 131.3, 'max_sentence2_length': 399, 'unique_sentence2': 1996}, 'nld_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 403541, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'nld_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 425320, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'nld_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 551312, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'nld_Latn-ltz_Latn': {'num_samples': 1997, 'number_of_characters': 572672, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 140.59, 'max_sentence2_length': 543, 'unique_sentence2': 1996}, 'nld_Latn-nno_Latn': {'num_samples': 1997, 'number_of_characters': 540272, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.37, 'max_sentence2_length': 417, 'unique_sentence2': 1996}, 'nld_Latn-nob_Latn': {'num_samples': 1997, 'number_of_characters': 543359, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.91, 'max_sentence2_length': 482, 'unique_sentence2': 1996}, 'nld_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 569781, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'nld_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 569130, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'nld_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 566808, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'nld_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 580115, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'nld_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 564424, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'nld_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 543742, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'nld_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 601876, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'nld_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 557528, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'nld_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 563036, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'nld_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 383393, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'nld_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 549647, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 146.18, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'nno_Latn-afr_Latn': {'num_samples': 1997, 'number_of_characters': 516709, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.37, 'max_sentence1_length': 417, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 134.38, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'nno_Latn-dan_Latn': {'num_samples': 1997, 'number_of_characters': 500495, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.37, 'max_sentence1_length': 417, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 126.26, 'max_sentence2_length': 522, 'unique_sentence2': 1995}, 'nno_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 544007, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.37, 'max_sentence1_length': 417, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'nno_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 496077, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.37, 'max_sentence1_length': 417, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'nno_Latn-fao_Latn': {'num_samples': 1997, 'number_of_characters': 506160, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.37, 'max_sentence1_length': 417, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.1, 'max_sentence2_length': 433, 'unique_sentence2': 1997}, 'nno_Latn-isl_Latn': {'num_samples': 1997, 'number_of_characters': 510565, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.37, 'max_sentence1_length': 417, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 131.3, 'max_sentence2_length': 399, 'unique_sentence2': 1996}, 'nno_Latn-ltz_Latn': {'num_samples': 1997, 'number_of_characters': 529114, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.37, 'max_sentence1_length': 417, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 140.59, 'max_sentence2_length': 543, 'unique_sentence2': 1996}, 'nno_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 540272, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.37, 'max_sentence1_length': 417, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'nno_Latn-nob_Latn': {'num_samples': 1997, 'number_of_characters': 499801, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.37, 'max_sentence1_length': 417, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.91, 'max_sentence2_length': 482, 'unique_sentence2': 1996}, 'nno_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 500184, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.37, 'max_sentence1_length': 417, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'nob_Latn-afr_Latn': {'num_samples': 1997, 'number_of_characters': 519796, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.91, 'max_sentence1_length': 482, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 134.38, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'nob_Latn-dan_Latn': {'num_samples': 1997, 'number_of_characters': 503582, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.91, 'max_sentence1_length': 482, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 126.26, 'max_sentence2_length': 522, 'unique_sentence2': 1995}, 'nob_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 547094, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.91, 'max_sentence1_length': 482, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'nob_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 499164, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 125.91, 'max_sentence1_length': 482, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'nob_Latn-fao_Latn': {'num_samples': 1997, 'number_of_characters': 509247, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 125.91, 'max_sentence1_length': 482, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.1, 'max_sentence2_length': 433, 'unique_sentence2': 1997}, 'nob_Latn-isl_Latn': {'num_samples': 1997, 'number_of_characters': 513652, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.91, 'max_sentence1_length': 482, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 131.3, 'max_sentence2_length': 399, 'unique_sentence2': 1996}, 'nob_Latn-ltz_Latn': {'num_samples': 1997, 'number_of_characters': 532201, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.91, 'max_sentence1_length': 482, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 140.59, 'max_sentence2_length': 543, 'unique_sentence2': 1996}, 'nob_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 543359, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.91, 'max_sentence1_length': 482, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'nob_Latn-nno_Latn': {'num_samples': 1997, 'number_of_characters': 499801, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.91, 'max_sentence1_length': 482, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.37, 'max_sentence2_length': 417, 'unique_sentence2': 1996}, 'nob_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 503271, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 125.91, 'max_sentence1_length': 482, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'nso_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 459006, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'nso_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 539219, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'nso_Latn-hau_Latn': {'num_samples': 1997, 'number_of_characters': 561465, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'nso_Latn-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 537600, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'nso_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 528930, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'nso_Latn-som_Latn': {'num_samples': 1997, 'number_of_characters': 582791, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'nso_Latn-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 579641, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'nso_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 564008, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'nso_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 456737, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'nso_Latn-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 625782, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'nso_Latn-wol_Latn': {'num_samples': 1997, 'number_of_characters': 531302, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'nso_Latn-xho_Latn': {'num_samples': 1997, 'number_of_characters': 559589, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'nso_Latn-yor_Latn': {'num_samples': 1997, 'number_of_characters': 607587, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'nso_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 549231, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 145.97, 'max_sentence1_length': 487, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'nya_Latn-bem_Latn': {'num_samples': 1997, 'number_of_characters': 582774, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 142.35, 'max_sentence1_length': 464, 'unique_sentence1': 1993, 'min_sentence2_length': 8, 'average_sentence2_length': 149.47, 'max_sentence2_length': 465, 'unique_sentence2': 1997}, 'nya_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 532002, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 142.35, 'max_sentence1_length': 464, 'unique_sentence1': 1993, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'nya_Latn-ewe_Latn': {'num_samples': 1997, 'number_of_characters': 523260, 'unique_pairs': 1995, 'min_sentence1_length': 10, 'average_sentence1_length': 142.35, 'max_sentence1_length': 464, 'unique_sentence1': 1993, 'min_sentence2_length': 6, 'average_sentence2_length': 119.67, 'max_sentence2_length': 493, 'unique_sentence2': 1994}, 'nya_Latn-fuc_Latn': {'num_samples': 1997, 'number_of_characters': 512762, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 142.35, 'max_sentence1_length': 464, 'unique_sentence1': 1993, 'min_sentence2_length': 6, 'average_sentence2_length': 114.41, 'max_sentence2_length': 376, 'unique_sentence2': 1996}, 'nya_Latn-kin_Latn': {'num_samples': 1997, 'number_of_characters': 588069, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 142.35, 'max_sentence1_length': 464, 'unique_sentence1': 1993, 'min_sentence2_length': 10, 'average_sentence2_length': 152.12, 'max_sentence2_length': 541, 'unique_sentence2': 1996}, 'nya_Latn-nde_Latn': {'num_samples': 1997, 'number_of_characters': 582021, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 142.35, 'max_sentence1_length': 464, 'unique_sentence1': 1993, 'min_sentence2_length': 6, 'average_sentence2_length': 149.09, 'max_sentence2_length': 590, 'unique_sentence2': 1997}, 'nya_Latn-sna_Latn': {'num_samples': 1997, 'number_of_characters': 582612, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 142.35, 'max_sentence1_length': 464, 'unique_sentence1': 1993, 'min_sentence2_length': 6, 'average_sentence2_length': 149.39, 'max_sentence2_length': 511, 'unique_sentence2': 1995}, 'nya_Latn-ven_Latn': {'num_samples': 1997, 'number_of_characters': 584038, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 142.35, 'max_sentence1_length': 464, 'unique_sentence1': 1993, 'min_sentence2_length': 10, 'average_sentence2_length': 150.1, 'max_sentence2_length': 535, 'unique_sentence2': 1993}, 'orm_Ethi-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 404938, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'orm_Ethi-eng_Latn': {'num_samples': 1997, 'number_of_characters': 485151, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'orm_Ethi-hau_Latn': {'num_samples': 1997, 'number_of_characters': 507397, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'orm_Ethi-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 483532, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'orm_Ethi-nso_Latn': {'num_samples': 1997, 'number_of_characters': 528930, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'orm_Ethi-som_Latn': {'num_samples': 1997, 'number_of_characters': 528723, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'orm_Ethi-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 525573, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'orm_Ethi-swa_Latn': {'num_samples': 1997, 'number_of_characters': 509940, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'orm_Ethi-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 402669, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'orm_Ethi-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 571714, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'orm_Ethi-wol_Latn': {'num_samples': 1997, 'number_of_characters': 477234, 'unique_pairs': 1992, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'orm_Ethi-xho_Latn': {'num_samples': 1997, 'number_of_characters': 505521, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'orm_Ethi-yor_Latn': {'num_samples': 1997, 'number_of_characters': 553519, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'orm_Ethi-zul_Latn': {'num_samples': 1997, 'number_of_characters': 495163, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 118.89, 'max_sentence1_length': 466, 'unique_sentence1': 1984, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'pan_Guru-ben_Beng': {'num_samples': 1997, 'number_of_characters': 494224, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'pan_Guru-div_Thaa': {'num_samples': 1997, 'number_of_characters': 554270, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'pan_Guru-eng_Latn': {'num_samples': 1997, 'number_of_characters': 498142, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'pan_Guru-eus_Latn': {'num_samples': 1997, 'number_of_characters': 525625, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'pan_Guru-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 495353, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'pan_Guru-hin_Deva': {'num_samples': 1997, 'number_of_characters': 511997, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'pan_Guru-kan_Knda': {'num_samples': 1997, 'number_of_characters': 515958, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'pan_Guru-mar_Deva': {'num_samples': 1997, 'number_of_characters': 511309, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'pan_Guru-nep_Deva': {'num_samples': 1997, 'number_of_characters': 498645, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'pan_Guru-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 509163, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'pan_Guru-snd_Arab': {'num_samples': 1997, 'number_of_characters': 470749, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'pan_Guru-tam_Taml': {'num_samples': 1997, 'number_of_characters': 560383, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'pan_Guru-tel_Telu': {'num_samples': 1997, 'number_of_characters': 497949, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'pan_Guru-urd_Arab': {'num_samples': 1997, 'number_of_characters': 498420, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 125.4, 'max_sentence1_length': 383, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'pol_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 509047, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'pol_Latn-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 533956, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'pol_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 521668, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'pol_Latn-bos_Latn': {'num_samples': 1997, 'number_of_characters': 532776, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'pol_Latn-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 547755, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'pol_Latn-ces_Latn': {'num_samples': 1997, 'number_of_characters': 519184, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'pol_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 573516, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'pol_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 576993, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'pol_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 525586, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'pol_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 521205, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'pol_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 547666, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'pol_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 570974, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'pol_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 478162, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'pol_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 539441, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'pol_Latn-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 533791, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'pol_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 556242, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'pol_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 564838, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'pol_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 389492, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'pol_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 411271, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'pol_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 537263, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'pol_Latn-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 545757, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'pol_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 569781, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'pol_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 555081, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'pol_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 552759, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'pol_Latn-slk_Latn': {'num_samples': 1997, 'number_of_characters': 530835, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'pol_Latn-slv_Latn': {'num_samples': 1997, 'number_of_characters': 530762, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'pol_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 566066, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'pol_Latn-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 530169, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'pol_Latn-srp_Latn': {'num_samples': 1997, 'number_of_characters': 534007, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'pol_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 550375, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'pol_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 529693, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'pol_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 587827, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'pol_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 543479, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'pol_Latn-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 540649, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'pol_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 548987, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'pol_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 369344, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'pol_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 535598, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 139.14, 'max_sentence1_length': 468, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'por_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 508396, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'por_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 521017, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'por_Latn-cat_Latn': {'num_samples': 1997, 'number_of_characters': 560175, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 141.69, 'max_sentence2_length': 460, 'unique_sentence2': 1997}, 'por_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 572865, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'por_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 576342, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'por_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 524935, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'por_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 520554, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'por_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 547015, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'por_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 570323, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'por_Latn-glg_Latn': {'num_samples': 1997, 'number_of_characters': 549201, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 136.2, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'por_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 477511, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'por_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 538790, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'por_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 555591, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'por_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 564187, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'por_Latn-ita_Latn': {'num_samples': 1997, 'number_of_characters': 566432, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 144.83, 'max_sentence2_length': 623, 'unique_sentence2': 1996}, 'por_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 388841, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'por_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 410620, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'por_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 536612, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'por_Latn-mlt_Latn': {'num_samples': 1997, 'number_of_characters': 554690, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 138.95, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'por_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 569130, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'por_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 555081, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'por_Latn-ron_Latn': {'num_samples': 1997, 'number_of_characters': 569700, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 146.46, 'max_sentence2_length': 518, 'unique_sentence2': 1997}, 'por_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 552108, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'por_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 565415, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'por_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 549724, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'por_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 529042, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'por_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 587176, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'por_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 542828, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'por_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 548336, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'por_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 368693, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'por_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 534947, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 138.82, 'max_sentence1_length': 497, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'prs_Arab-arb_Arab': {'num_samples': 1997, 'number_of_characters': 473717, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.45, 'max_sentence1_length': 365, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'prs_Arab-ckb_Arab': {'num_samples': 1997, 'number_of_characters': 494903, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.45, 'max_sentence1_length': 365, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 126.37, 'max_sentence2_length': 399, 'unique_sentence2': 1995}, 'prs_Arab-eng_Latn': {'num_samples': 1997, 'number_of_characters': 490256, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.45, 'max_sentence1_length': 365, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'prs_Arab-fas_Arab': {'num_samples': 1997, 'number_of_characters': 485875, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.45, 'max_sentence1_length': 365, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'prs_Arab-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 442832, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.45, 'max_sentence1_length': 365, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'prs_Arab-kmr_Latn': {'num_samples': 1997, 'number_of_characters': 488482, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.45, 'max_sentence1_length': 365, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 123.16, 'max_sentence2_length': 420, 'unique_sentence2': 1996}, 'prs_Arab-mey_Arab': {'num_samples': 1997, 'number_of_characters': 456371, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.45, 'max_sentence1_length': 365, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 107.08, 'max_sentence2_length': 392, 'unique_sentence2': 1993}, 'prs_Arab-pus_Arab': {'num_samples': 1997, 'number_of_characters': 485169, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.45, 'max_sentence1_length': 365, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 121.5, 'max_sentence2_length': 366, 'unique_sentence2': 1996}, 'prs_Arab-shi_Arab': {'num_samples': 1997, 'number_of_characters': 457449, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.45, 'max_sentence1_length': 365, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 107.62, 'max_sentence2_length': 378, 'unique_sentence2': 1996}, 'prs_Arab-tgk_Cyrl': {'num_samples': 1997, 'number_of_characters': 516683, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.45, 'max_sentence1_length': 365, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 137.28, 'max_sentence2_length': 451, 'unique_sentence2': 1995}, 'pus_Arab-arb_Arab': {'num_samples': 1997, 'number_of_characters': 473814, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 121.5, 'max_sentence1_length': 366, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'pus_Arab-ckb_Arab': {'num_samples': 1997, 'number_of_characters': 495000, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 121.5, 'max_sentence1_length': 366, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 126.37, 'max_sentence2_length': 399, 'unique_sentence2': 1995}, 'pus_Arab-eng_Latn': {'num_samples': 1997, 'number_of_characters': 490353, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.5, 'max_sentence1_length': 366, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'pus_Arab-fas_Arab': {'num_samples': 1997, 'number_of_characters': 485972, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 121.5, 'max_sentence1_length': 366, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'pus_Arab-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 442929, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 121.5, 'max_sentence1_length': 366, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'pus_Arab-kmr_Latn': {'num_samples': 1997, 'number_of_characters': 488579, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 121.5, 'max_sentence1_length': 366, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 123.16, 'max_sentence2_length': 420, 'unique_sentence2': 1996}, 'pus_Arab-mey_Arab': {'num_samples': 1997, 'number_of_characters': 456468, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.5, 'max_sentence1_length': 366, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 107.08, 'max_sentence2_length': 392, 'unique_sentence2': 1993}, 'pus_Arab-prs_Arab': {'num_samples': 1997, 'number_of_characters': 485169, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.5, 'max_sentence1_length': 366, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 121.45, 'max_sentence2_length': 365, 'unique_sentence2': 1997}, 'pus_Arab-shi_Arab': {'num_samples': 1997, 'number_of_characters': 457546, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 121.5, 'max_sentence1_length': 366, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 107.62, 'max_sentence2_length': 378, 'unique_sentence2': 1996}, 'pus_Arab-tgk_Cyrl': {'num_samples': 1997, 'number_of_characters': 516780, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 121.5, 'max_sentence1_length': 366, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 137.28, 'max_sentence2_length': 451, 'unique_sentence2': 1995}, 'ron_Latn-cat_Latn': {'num_samples': 1997, 'number_of_characters': 575445, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 146.46, 'max_sentence1_length': 518, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 141.69, 'max_sentence2_length': 460, 'unique_sentence2': 1997}, 'ron_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 540205, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 146.46, 'max_sentence1_length': 518, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ron_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 585593, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 146.46, 'max_sentence1_length': 518, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'ron_Latn-glg_Latn': {'num_samples': 1997, 'number_of_characters': 564471, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 146.46, 'max_sentence1_length': 518, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 136.2, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'ron_Latn-ita_Latn': {'num_samples': 1997, 'number_of_characters': 581702, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 146.46, 'max_sentence1_length': 518, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 144.83, 'max_sentence2_length': 623, 'unique_sentence2': 1996}, 'ron_Latn-mlt_Latn': {'num_samples': 1997, 'number_of_characters': 569960, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 146.46, 'max_sentence1_length': 518, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 138.95, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'ron_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 569700, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 146.46, 'max_sentence1_length': 518, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'ron_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 580685, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 146.46, 'max_sentence1_length': 518, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'rus_Cyrl-arb_Arab': {'num_samples': 1997, 'number_of_characters': 506074, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'rus_Cyrl-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 530983, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'rus_Cyrl-ben_Beng': {'num_samples': 1997, 'number_of_characters': 518695, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'rus_Cyrl-bos_Latn': {'num_samples': 1997, 'number_of_characters': 529803, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'rus_Cyrl-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 544782, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'rus_Cyrl-ces_Latn': {'num_samples': 1997, 'number_of_characters': 516211, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'rus_Cyrl-deu_Latn': {'num_samples': 1997, 'number_of_characters': 570543, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'rus_Cyrl-ell_Grek': {'num_samples': 1997, 'number_of_characters': 574020, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'rus_Cyrl-eng_Latn': {'num_samples': 1997, 'number_of_characters': 522613, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'rus_Cyrl-fas_Arab': {'num_samples': 1997, 'number_of_characters': 518232, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'rus_Cyrl-fin_Latn': {'num_samples': 1997, 'number_of_characters': 544693, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'rus_Cyrl-fra_Latn': {'num_samples': 1997, 'number_of_characters': 568001, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'rus_Cyrl-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 475189, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'rus_Cyrl-hin_Deva': {'num_samples': 1997, 'number_of_characters': 536468, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'rus_Cyrl-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 530818, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'rus_Cyrl-hun_Latn': {'num_samples': 1997, 'number_of_characters': 553269, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'rus_Cyrl-ind_Latn': {'num_samples': 1997, 'number_of_characters': 561865, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'rus_Cyrl-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 386519, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'rus_Cyrl-kor_Hang': {'num_samples': 1997, 'number_of_characters': 408298, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'rus_Cyrl-lit_Latn': {'num_samples': 1997, 'number_of_characters': 534290, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'rus_Cyrl-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 542784, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'rus_Cyrl-nld_Latn': {'num_samples': 1997, 'number_of_characters': 566808, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'rus_Cyrl-pol_Latn': {'num_samples': 1997, 'number_of_characters': 552759, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'rus_Cyrl-por_Latn': {'num_samples': 1997, 'number_of_characters': 552108, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'rus_Cyrl-slk_Latn': {'num_samples': 1997, 'number_of_characters': 527862, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'rus_Cyrl-slv_Latn': {'num_samples': 1997, 'number_of_characters': 527789, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'rus_Cyrl-spa_Latn': {'num_samples': 1997, 'number_of_characters': 563093, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'rus_Cyrl-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 527196, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'rus_Cyrl-srp_Latn': {'num_samples': 1997, 'number_of_characters': 531034, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'rus_Cyrl-swa_Latn': {'num_samples': 1997, 'number_of_characters': 547402, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'rus_Cyrl-swe_Latn': {'num_samples': 1997, 'number_of_characters': 526720, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'rus_Cyrl-tam_Taml': {'num_samples': 1997, 'number_of_characters': 584854, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'rus_Cyrl-tur_Latn': {'num_samples': 1997, 'number_of_characters': 540506, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'rus_Cyrl-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 537676, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'rus_Cyrl-vie_Latn': {'num_samples': 1997, 'number_of_characters': 546014, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'rus_Cyrl-zho_Hant': {'num_samples': 1997, 'number_of_characters': 366371, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'rus_Cyrl-zul_Latn': {'num_samples': 1997, 'number_of_characters': 532625, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 137.65, 'max_sentence1_length': 419, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'shi_Arab-arb_Arab': {'num_samples': 1997, 'number_of_characters': 446094, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 107.62, 'max_sentence1_length': 378, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'shi_Arab-ckb_Arab': {'num_samples': 1997, 'number_of_characters': 467280, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 107.62, 'max_sentence1_length': 378, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 126.37, 'max_sentence2_length': 399, 'unique_sentence2': 1995}, 'shi_Arab-eng_Latn': {'num_samples': 1997, 'number_of_characters': 462633, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 107.62, 'max_sentence1_length': 378, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'shi_Arab-fas_Arab': {'num_samples': 1997, 'number_of_characters': 458252, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 107.62, 'max_sentence1_length': 378, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'shi_Arab-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 415209, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 107.62, 'max_sentence1_length': 378, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'shi_Arab-kmr_Latn': {'num_samples': 1997, 'number_of_characters': 460859, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 107.62, 'max_sentence1_length': 378, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 123.16, 'max_sentence2_length': 420, 'unique_sentence2': 1996}, 'shi_Arab-mey_Arab': {'num_samples': 1997, 'number_of_characters': 428748, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 107.62, 'max_sentence1_length': 378, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 107.08, 'max_sentence2_length': 392, 'unique_sentence2': 1993}, 'shi_Arab-prs_Arab': {'num_samples': 1997, 'number_of_characters': 457449, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 107.62, 'max_sentence1_length': 378, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 121.45, 'max_sentence2_length': 365, 'unique_sentence2': 1997}, 'shi_Arab-pus_Arab': {'num_samples': 1997, 'number_of_characters': 457546, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 107.62, 'max_sentence1_length': 378, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 121.5, 'max_sentence2_length': 366, 'unique_sentence2': 1996}, 'shi_Arab-tgk_Cyrl': {'num_samples': 1997, 'number_of_characters': 489060, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 107.62, 'max_sentence1_length': 378, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 137.28, 'max_sentence2_length': 451, 'unique_sentence2': 1995}, 'sin_Sinh-ben_Beng': {'num_samples': 1997, 'number_of_characters': 502543, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'sin_Sinh-div_Thaa': {'num_samples': 1997, 'number_of_characters': 562589, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'sin_Sinh-eng_Latn': {'num_samples': 1997, 'number_of_characters': 506461, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'sin_Sinh-eus_Latn': {'num_samples': 1997, 'number_of_characters': 533944, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'sin_Sinh-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 503672, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'sin_Sinh-hin_Deva': {'num_samples': 1997, 'number_of_characters': 520316, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'sin_Sinh-kan_Knda': {'num_samples': 1997, 'number_of_characters': 524277, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'sin_Sinh-mar_Deva': {'num_samples': 1997, 'number_of_characters': 519628, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'sin_Sinh-nep_Deva': {'num_samples': 1997, 'number_of_characters': 506964, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'sin_Sinh-pan_Guru': {'num_samples': 1997, 'number_of_characters': 509163, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'sin_Sinh-snd_Arab': {'num_samples': 1997, 'number_of_characters': 479068, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'sin_Sinh-tam_Taml': {'num_samples': 1997, 'number_of_characters': 568702, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'sin_Sinh-tel_Telu': {'num_samples': 1997, 'number_of_characters': 506268, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'sin_Sinh-urd_Arab': {'num_samples': 1997, 'number_of_characters': 506739, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 129.56, 'max_sentence1_length': 441, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'slk_Latn-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 509059, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'slk_Latn-bos_Latn': {'num_samples': 1997, 'number_of_characters': 507879, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'slk_Latn-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 522858, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'slk_Latn-ces_Latn': {'num_samples': 1997, 'number_of_characters': 494287, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'slk_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 500689, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'slk_Latn-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 508894, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'slk_Latn-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 520860, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'slk_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 530835, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'slk_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 527862, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'slk_Latn-slv_Latn': {'num_samples': 1997, 'number_of_characters': 505865, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'slk_Latn-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 505272, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'slk_Latn-srp_Latn': {'num_samples': 1997, 'number_of_characters': 509110, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'slk_Latn-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 515752, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 126.67, 'max_sentence1_length': 403, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'slv_Latn-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 508986, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'slv_Latn-bos_Latn': {'num_samples': 1997, 'number_of_characters': 507806, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'slv_Latn-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 522785, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'slv_Latn-ces_Latn': {'num_samples': 1997, 'number_of_characters': 494214, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'slv_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 500616, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'slv_Latn-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 508821, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'slv_Latn-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 520787, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'slv_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 530762, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'slv_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 527789, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'slv_Latn-slk_Latn': {'num_samples': 1997, 'number_of_characters': 505865, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'slv_Latn-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 505199, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'slv_Latn-srp_Latn': {'num_samples': 1997, 'number_of_characters': 509037, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'slv_Latn-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 515679, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.64, 'max_sentence1_length': 463, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'smo_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 525575, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 139.14, 'max_sentence1_length': 431, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'smo_Latn-fij_Latn': {'num_samples': 1997, 'number_of_characters': 578360, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 139.14, 'max_sentence1_length': 431, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 150.48, 'max_sentence2_length': 448, 'unique_sentence2': 1988}, 'smo_Latn-fil_Latn': {'num_samples': 1997, 'number_of_characters': 571275, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 139.14, 'max_sentence1_length': 431, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 146.93, 'max_sentence2_length': 554, 'unique_sentence2': 1997}, 'smo_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 564827, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 139.14, 'max_sentence1_length': 431, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'smo_Latn-mal_Mlym': {'num_samples': 1997, 'number_of_characters': 582007, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 139.14, 'max_sentence1_length': 431, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 152.3, 'max_sentence2_length': 540, 'unique_sentence2': 1996}, 'smo_Latn-mlg_Latn': {'num_samples': 1997, 'number_of_characters': 598163, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 139.14, 'max_sentence1_length': 431, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 160.39, 'max_sentence2_length': 559, 'unique_sentence2': 1994}, 'smo_Latn-mri_Latn': {'num_samples': 1997, 'number_of_characters': 551979, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 139.14, 'max_sentence1_length': 431, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 137.27, 'max_sentence2_length': 443, 'unique_sentence2': 1997}, 'smo_Latn-msa_Latn': {'num_samples': 1997, 'number_of_characters': 555038, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 139.14, 'max_sentence1_length': 431, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 138.8, 'max_sentence2_length': 463, 'unique_sentence2': 1997}, 'smo_Latn-tah_Latn': {'num_samples': 1997, 'number_of_characters': 587478, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 139.14, 'max_sentence1_length': 431, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 155.04, 'max_sentence2_length': 524, 'unique_sentence2': 1997}, 'smo_Latn-ton_Latn': {'num_samples': 1997, 'number_of_characters': 591495, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 139.14, 'max_sentence1_length': 431, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 157.06, 'max_sentence2_length': 468, 'unique_sentence2': 1997}, 'sna_Latn-bem_Latn': {'num_samples': 1997, 'number_of_characters': 596822, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.39, 'max_sentence1_length': 511, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 149.47, 'max_sentence2_length': 465, 'unique_sentence2': 1997}, 'sna_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 546050, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.39, 'max_sentence1_length': 511, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'sna_Latn-ewe_Latn': {'num_samples': 1997, 'number_of_characters': 537308, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 149.39, 'max_sentence1_length': 511, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 119.67, 'max_sentence2_length': 493, 'unique_sentence2': 1994}, 'sna_Latn-fuc_Latn': {'num_samples': 1997, 'number_of_characters': 526810, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.39, 'max_sentence1_length': 511, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 114.41, 'max_sentence2_length': 376, 'unique_sentence2': 1996}, 'sna_Latn-kin_Latn': {'num_samples': 1997, 'number_of_characters': 602117, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.39, 'max_sentence1_length': 511, 'unique_sentence1': 1995, 'min_sentence2_length': 10, 'average_sentence2_length': 152.12, 'max_sentence2_length': 541, 'unique_sentence2': 1996}, 'sna_Latn-nde_Latn': {'num_samples': 1997, 'number_of_characters': 596069, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 149.39, 'max_sentence1_length': 511, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 149.09, 'max_sentence2_length': 590, 'unique_sentence2': 1997}, 'sna_Latn-nya_Latn': {'num_samples': 1997, 'number_of_characters': 582612, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 149.39, 'max_sentence1_length': 511, 'unique_sentence1': 1995, 'min_sentence2_length': 10, 'average_sentence2_length': 142.35, 'max_sentence2_length': 464, 'unique_sentence2': 1993}, 'sna_Latn-ven_Latn': {'num_samples': 1997, 'number_of_characters': 598086, 'unique_pairs': 1995, 'min_sentence1_length': 6, 'average_sentence1_length': 149.39, 'max_sentence1_length': 511, 'unique_sentence1': 1995, 'min_sentence2_length': 10, 'average_sentence2_length': 150.1, 'max_sentence2_length': 535, 'unique_sentence2': 1993}, 'snd_Arab-ben_Beng': {'num_samples': 1997, 'number_of_characters': 464129, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'snd_Arab-div_Thaa': {'num_samples': 1997, 'number_of_characters': 524175, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'snd_Arab-eng_Latn': {'num_samples': 1997, 'number_of_characters': 468047, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'snd_Arab-eus_Latn': {'num_samples': 1997, 'number_of_characters': 495530, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'snd_Arab-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 465258, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'snd_Arab-hin_Deva': {'num_samples': 1997, 'number_of_characters': 481902, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'snd_Arab-kan_Knda': {'num_samples': 1997, 'number_of_characters': 485863, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'snd_Arab-mar_Deva': {'num_samples': 1997, 'number_of_characters': 481214, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'snd_Arab-nep_Deva': {'num_samples': 1997, 'number_of_characters': 468550, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'snd_Arab-pan_Guru': {'num_samples': 1997, 'number_of_characters': 470749, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'snd_Arab-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 479068, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'snd_Arab-tam_Taml': {'num_samples': 1997, 'number_of_characters': 530288, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'snd_Arab-tel_Telu': {'num_samples': 1997, 'number_of_characters': 467854, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'snd_Arab-urd_Arab': {'num_samples': 1997, 'number_of_characters': 468325, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 110.33, 'max_sentence1_length': 335, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'som_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 458799, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'som_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 539012, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'som_Latn-hau_Latn': {'num_samples': 1997, 'number_of_characters': 561258, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'som_Latn-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 537393, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'som_Latn-nso_Latn': {'num_samples': 1997, 'number_of_characters': 582791, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'som_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 528723, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'som_Latn-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 579434, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'som_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 563801, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'som_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 456530, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'som_Latn-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 625575, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'som_Latn-wol_Latn': {'num_samples': 1997, 'number_of_characters': 531095, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'som_Latn-xho_Latn': {'num_samples': 1997, 'number_of_characters': 559382, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'som_Latn-yor_Latn': {'num_samples': 1997, 'number_of_characters': 607380, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'som_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 549024, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 145.86, 'max_sentence1_length': 455, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'spa_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 519381, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'spa_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 532002, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'spa_Latn-cat_Latn': {'num_samples': 1997, 'number_of_characters': 571160, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 141.69, 'max_sentence2_length': 460, 'unique_sentence2': 1997}, 'spa_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 583850, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'spa_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 587327, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'spa_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 535920, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'spa_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 531539, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'spa_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 558000, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'spa_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 581308, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'spa_Latn-glg_Latn': {'num_samples': 1997, 'number_of_characters': 560186, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 136.2, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'spa_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 488496, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'spa_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 549775, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'spa_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 566576, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'spa_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 575172, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'spa_Latn-ita_Latn': {'num_samples': 1997, 'number_of_characters': 577417, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 144.83, 'max_sentence2_length': 623, 'unique_sentence2': 1996}, 'spa_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 399826, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'spa_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 421605, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'spa_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 547597, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'spa_Latn-mlt_Latn': {'num_samples': 1997, 'number_of_characters': 565675, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 138.95, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'spa_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 580115, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'spa_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 566066, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'spa_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 565415, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'spa_Latn-ron_Latn': {'num_samples': 1997, 'number_of_characters': 580685, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 146.46, 'max_sentence2_length': 518, 'unique_sentence2': 1997}, 'spa_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 563093, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'spa_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 560709, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'spa_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 540027, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'spa_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 598161, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'spa_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 553813, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'spa_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 559321, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'spa_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 379678, 'unique_pairs': 1996, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'spa_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 545932, 'unique_pairs': 1997, 'min_sentence1_length': 1, 'average_sentence1_length': 144.32, 'max_sentence1_length': 504, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'sqi_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 582734, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 142.02, 'max_sentence1_length': 461, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'sqi_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 531327, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 142.02, 'max_sentence1_length': 461, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'sqi_Latn-hye_Armn': {'num_samples': 1997, 'number_of_characters': 548322, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 142.02, 'max_sentence1_length': 461, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 132.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'sqi_Latn-kat_Geor': {'num_samples': 1997, 'number_of_characters': 550199, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 142.02, 'max_sentence1_length': 461, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 133.5, 'max_sentence2_length': 503, 'unique_sentence2': 1995}, 'srp_Cyrl-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 508393, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'srp_Cyrl-bos_Latn': {'num_samples': 1997, 'number_of_characters': 507213, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'srp_Cyrl-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 522192, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'srp_Cyrl-ces_Latn': {'num_samples': 1997, 'number_of_characters': 493621, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'srp_Cyrl-eng_Latn': {'num_samples': 1997, 'number_of_characters': 500023, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'srp_Cyrl-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 508228, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'srp_Cyrl-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 520194, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'srp_Cyrl-pol_Latn': {'num_samples': 1997, 'number_of_characters': 530169, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'srp_Cyrl-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 527196, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'srp_Cyrl-slk_Latn': {'num_samples': 1997, 'number_of_characters': 505272, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'srp_Cyrl-slv_Latn': {'num_samples': 1997, 'number_of_characters': 505199, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'srp_Cyrl-srp_Latn': {'num_samples': 1997, 'number_of_characters': 508444, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'srp_Cyrl-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 515086, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 126.34, 'max_sentence1_length': 439, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'srp_Latn-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 512231, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'srp_Latn-bos_Latn': {'num_samples': 1997, 'number_of_characters': 511051, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'srp_Latn-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 526030, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'srp_Latn-ces_Latn': {'num_samples': 1997, 'number_of_characters': 497459, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'srp_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 503861, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'srp_Latn-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 512066, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'srp_Latn-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 524032, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'srp_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 534007, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'srp_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 531034, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'srp_Latn-slk_Latn': {'num_samples': 1997, 'number_of_characters': 509110, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'srp_Latn-slv_Latn': {'num_samples': 1997, 'number_of_characters': 509037, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'srp_Latn-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 508444, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'srp_Latn-ukr_Cyrl': {'num_samples': 1997, 'number_of_characters': 518924, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 128.26, 'max_sentence1_length': 452, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 131.59, 'max_sentence2_length': 440, 'unique_sentence2': 1996}, 'ssw_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 455649, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'ssw_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 535862, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ssw_Latn-hau_Latn': {'num_samples': 1997, 'number_of_characters': 558108, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'ssw_Latn-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 534243, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'ssw_Latn-nso_Latn': {'num_samples': 1997, 'number_of_characters': 579641, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'ssw_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 525573, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'ssw_Latn-som_Latn': {'num_samples': 1997, 'number_of_characters': 579434, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'ssw_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 560651, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'ssw_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 453380, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'ssw_Latn-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 622425, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'ssw_Latn-wol_Latn': {'num_samples': 1997, 'number_of_characters': 527945, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'ssw_Latn-xho_Latn': {'num_samples': 1997, 'number_of_characters': 556232, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'ssw_Latn-yor_Latn': {'num_samples': 1997, 'number_of_characters': 604230, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'ssw_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 545874, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 144.29, 'max_sentence1_length': 510, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'swa_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 440016, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'swa_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 503690, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'swa_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 516311, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'swa_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 568159, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'swa_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 571636, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'swa_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 520229, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'swa_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 515848, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'swa_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 542309, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'swa_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 565617, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'swa_Latn-hau_Latn': {'num_samples': 1997, 'number_of_characters': 542475, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'swa_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 472805, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'swa_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 534084, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'swa_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 550885, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'swa_Latn-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 518610, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'swa_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 559481, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'swa_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 384135, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'swa_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 405914, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'swa_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 531906, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'swa_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 564424, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'swa_Latn-nso_Latn': {'num_samples': 1997, 'number_of_characters': 564008, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'swa_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 509940, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'swa_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 550375, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'swa_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 549724, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'swa_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 547402, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'swa_Latn-som_Latn': {'num_samples': 1997, 'number_of_characters': 563801, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'swa_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 560709, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'swa_Latn-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 560651, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'swa_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 524336, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'swa_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 582470, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'swa_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 437747, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'swa_Latn-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 606792, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'swa_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 538122, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'swa_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 543630, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'swa_Latn-wol_Latn': {'num_samples': 1997, 'number_of_characters': 512312, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'swa_Latn-xho_Latn': {'num_samples': 1997, 'number_of_characters': 540599, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'swa_Latn-yor_Latn': {'num_samples': 1997, 'number_of_characters': 588597, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'swa_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 363987, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'swa_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 530241, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 136.46, 'max_sentence1_length': 430, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'swe_Latn-afr_Latn': {'num_samples': 1997, 'number_of_characters': 520179, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 134.38, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'swe_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 483008, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'swe_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 495629, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'swe_Latn-dan_Latn': {'num_samples': 1997, 'number_of_characters': 503965, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 126.26, 'max_sentence2_length': 522, 'unique_sentence2': 1995}, 'swe_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 547477, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'swe_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 550954, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'swe_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 499547, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'swe_Latn-fao_Latn': {'num_samples': 1997, 'number_of_characters': 509630, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.1, 'max_sentence2_length': 433, 'unique_sentence2': 1997}, 'swe_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 495166, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'swe_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 521627, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'swe_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 544935, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'swe_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 452123, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'swe_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 513402, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'swe_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 530203, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'swe_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 538799, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'swe_Latn-isl_Latn': {'num_samples': 1997, 'number_of_characters': 514035, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 131.3, 'max_sentence2_length': 399, 'unique_sentence2': 1996}, 'swe_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 363453, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'swe_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 385232, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'swe_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 511224, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'swe_Latn-ltz_Latn': {'num_samples': 1997, 'number_of_characters': 532584, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 140.59, 'max_sentence2_length': 543, 'unique_sentence2': 1996}, 'swe_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 543742, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'swe_Latn-nno_Latn': {'num_samples': 1997, 'number_of_characters': 500184, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.37, 'max_sentence2_length': 417, 'unique_sentence2': 1996}, 'swe_Latn-nob_Latn': {'num_samples': 1997, 'number_of_characters': 503271, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.91, 'max_sentence2_length': 482, 'unique_sentence2': 1996}, 'swe_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 529693, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'swe_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 529042, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'swe_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 526720, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'swe_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 540027, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'swe_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 524336, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'swe_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 561788, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'swe_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 517440, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'swe_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 522948, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'swe_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 343305, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'swe_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 509559, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 126.1, 'max_sentence1_length': 430, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'tah_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 557343, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 155.04, 'max_sentence1_length': 524, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'tah_Latn-fij_Latn': {'num_samples': 1997, 'number_of_characters': 610128, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 155.04, 'max_sentence1_length': 524, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 150.48, 'max_sentence2_length': 448, 'unique_sentence2': 1988}, 'tah_Latn-fil_Latn': {'num_samples': 1997, 'number_of_characters': 603043, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 155.04, 'max_sentence1_length': 524, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 146.93, 'max_sentence2_length': 554, 'unique_sentence2': 1997}, 'tah_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 596595, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 155.04, 'max_sentence1_length': 524, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'tah_Latn-mal_Mlym': {'num_samples': 1997, 'number_of_characters': 613775, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 155.04, 'max_sentence1_length': 524, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 152.3, 'max_sentence2_length': 540, 'unique_sentence2': 1996}, 'tah_Latn-mlg_Latn': {'num_samples': 1997, 'number_of_characters': 629931, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 155.04, 'max_sentence1_length': 524, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 160.39, 'max_sentence2_length': 559, 'unique_sentence2': 1994}, 'tah_Latn-mri_Latn': {'num_samples': 1997, 'number_of_characters': 583747, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 155.04, 'max_sentence1_length': 524, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 137.27, 'max_sentence2_length': 443, 'unique_sentence2': 1997}, 'tah_Latn-msa_Latn': {'num_samples': 1997, 'number_of_characters': 586806, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 155.04, 'max_sentence1_length': 524, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 138.8, 'max_sentence2_length': 463, 'unique_sentence2': 1997}, 'tah_Latn-smo_Latn': {'num_samples': 1997, 'number_of_characters': 587478, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 155.04, 'max_sentence1_length': 524, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 139.14, 'max_sentence2_length': 431, 'unique_sentence2': 1996}, 'tah_Latn-ton_Latn': {'num_samples': 1997, 'number_of_characters': 623263, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 155.04, 'max_sentence1_length': 524, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 157.06, 'max_sentence2_length': 468, 'unique_sentence2': 1997}, 'tam_Taml-arb_Arab': {'num_samples': 1997, 'number_of_characters': 541142, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'tam_Taml-ben_Beng': {'num_samples': 1997, 'number_of_characters': 553763, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'tam_Taml-deu_Latn': {'num_samples': 1997, 'number_of_characters': 605611, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'tam_Taml-div_Thaa': {'num_samples': 1997, 'number_of_characters': 613809, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'tam_Taml-ell_Grek': {'num_samples': 1997, 'number_of_characters': 609088, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'tam_Taml-eng_Latn': {'num_samples': 1997, 'number_of_characters': 557681, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'tam_Taml-eus_Latn': {'num_samples': 1997, 'number_of_characters': 585164, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'tam_Taml-fas_Arab': {'num_samples': 1997, 'number_of_characters': 553300, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'tam_Taml-fin_Latn': {'num_samples': 1997, 'number_of_characters': 579761, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'tam_Taml-fra_Latn': {'num_samples': 1997, 'number_of_characters': 603069, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'tam_Taml-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 554892, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'tam_Taml-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 510257, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'tam_Taml-hin_Deva': {'num_samples': 1997, 'number_of_characters': 571536, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'tam_Taml-hun_Latn': {'num_samples': 1997, 'number_of_characters': 588337, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'tam_Taml-ind_Latn': {'num_samples': 1997, 'number_of_characters': 596933, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'tam_Taml-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 421587, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'tam_Taml-kan_Knda': {'num_samples': 1997, 'number_of_characters': 575497, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'tam_Taml-kor_Hang': {'num_samples': 1997, 'number_of_characters': 443366, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'tam_Taml-lit_Latn': {'num_samples': 1997, 'number_of_characters': 569358, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'tam_Taml-mar_Deva': {'num_samples': 1997, 'number_of_characters': 570848, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'tam_Taml-nep_Deva': {'num_samples': 1997, 'number_of_characters': 558184, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'tam_Taml-nld_Latn': {'num_samples': 1997, 'number_of_characters': 601876, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'tam_Taml-pan_Guru': {'num_samples': 1997, 'number_of_characters': 560383, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'tam_Taml-pol_Latn': {'num_samples': 1997, 'number_of_characters': 587827, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'tam_Taml-por_Latn': {'num_samples': 1997, 'number_of_characters': 587176, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'tam_Taml-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 584854, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'tam_Taml-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 568702, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'tam_Taml-snd_Arab': {'num_samples': 1997, 'number_of_characters': 530288, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'tam_Taml-spa_Latn': {'num_samples': 1997, 'number_of_characters': 598161, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'tam_Taml-swa_Latn': {'num_samples': 1997, 'number_of_characters': 582470, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'tam_Taml-swe_Latn': {'num_samples': 1997, 'number_of_characters': 561788, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'tam_Taml-tel_Telu': {'num_samples': 1997, 'number_of_characters': 557488, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'tam_Taml-tur_Latn': {'num_samples': 1997, 'number_of_characters': 575574, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'tam_Taml-urd_Arab': {'num_samples': 1997, 'number_of_characters': 557959, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'tam_Taml-vie_Latn': {'num_samples': 1997, 'number_of_characters': 581082, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'tam_Taml-zho_Hant': {'num_samples': 1997, 'number_of_characters': 401439, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'tam_Taml-zul_Latn': {'num_samples': 1997, 'number_of_characters': 567693, 'unique_pairs': 1997, 'min_sentence1_length': 11, 'average_sentence1_length': 155.21, 'max_sentence1_length': 581, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'tat_Cyrl-aze_Latn': {'num_samples': 1997, 'number_of_characters': 515560, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 123.15, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 135.02, 'max_sentence2_length': 398, 'unique_sentence2': 1997}, 'tat_Cyrl-bak_Cyrl': {'num_samples': 1997, 'number_of_characters': 492252, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 123.15, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 123.35, 'max_sentence2_length': 437, 'unique_sentence2': 1995}, 'tat_Cyrl-eng_Latn': {'num_samples': 1997, 'number_of_characters': 493646, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 123.15, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'tat_Cyrl-kaz_Cyrl': {'num_samples': 1997, 'number_of_characters': 506202, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 123.15, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 130.33, 'max_sentence2_length': 473, 'unique_sentence2': 1996}, 'tat_Cyrl-kir_Cyrl': {'num_samples': 1997, 'number_of_characters': 496790, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 123.15, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.62, 'max_sentence2_length': 395, 'unique_sentence2': 1996}, 'tat_Cyrl-tuk_Latn': {'num_samples': 1997, 'number_of_characters': 531200, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 123.15, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 142.85, 'max_sentence2_length': 576, 'unique_sentence2': 1996}, 'tat_Cyrl-tur_Latn': {'num_samples': 1997, 'number_of_characters': 511539, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 123.15, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'tat_Cyrl-uig_Arab': {'num_samples': 1997, 'number_of_characters': 556948, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 123.15, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 155.74, 'max_sentence2_length': 592, 'unique_sentence2': 1996}, 'tat_Cyrl-uzb_Latn': {'num_samples': 1997, 'number_of_characters': 539621, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 123.15, 'max_sentence1_length': 539, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 147.07, 'max_sentence2_length': 470, 'unique_sentence2': 1996}, 'tel_Telu-ben_Beng': {'num_samples': 1997, 'number_of_characters': 491329, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'tel_Telu-div_Thaa': {'num_samples': 1997, 'number_of_characters': 551375, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'tel_Telu-eng_Latn': {'num_samples': 1997, 'number_of_characters': 495247, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'tel_Telu-eus_Latn': {'num_samples': 1997, 'number_of_characters': 522730, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'tel_Telu-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 492458, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'tel_Telu-hin_Deva': {'num_samples': 1997, 'number_of_characters': 509102, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'tel_Telu-kan_Knda': {'num_samples': 1997, 'number_of_characters': 513063, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'tel_Telu-mar_Deva': {'num_samples': 1997, 'number_of_characters': 508414, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'tel_Telu-nep_Deva': {'num_samples': 1997, 'number_of_characters': 495750, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'tel_Telu-pan_Guru': {'num_samples': 1997, 'number_of_characters': 497949, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'tel_Telu-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 506268, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'tel_Telu-snd_Arab': {'num_samples': 1997, 'number_of_characters': 467854, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'tel_Telu-tam_Taml': {'num_samples': 1997, 'number_of_characters': 557488, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'tel_Telu-urd_Arab': {'num_samples': 1997, 'number_of_characters': 495525, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 123.95, 'max_sentence1_length': 412, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.19, 'max_sentence2_length': 390, 'unique_sentence2': 1996}, 'tgk_Cyrl-arb_Arab': {'num_samples': 1997, 'number_of_characters': 505328, 'unique_pairs': 1995, 'min_sentence1_length': 3, 'average_sentence1_length': 137.28, 'max_sentence1_length': 451, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'tgk_Cyrl-ckb_Arab': {'num_samples': 1997, 'number_of_characters': 526514, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 137.28, 'max_sentence1_length': 451, 'unique_sentence1': 1995, 'min_sentence2_length': 5, 'average_sentence2_length': 126.37, 'max_sentence2_length': 399, 'unique_sentence2': 1995}, 'tgk_Cyrl-eng_Latn': {'num_samples': 1997, 'number_of_characters': 521867, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 137.28, 'max_sentence1_length': 451, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'tgk_Cyrl-fas_Arab': {'num_samples': 1997, 'number_of_characters': 517486, 'unique_pairs': 1995, 'min_sentence1_length': 3, 'average_sentence1_length': 137.28, 'max_sentence1_length': 451, 'unique_sentence1': 1995, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'tgk_Cyrl-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 474443, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 137.28, 'max_sentence1_length': 451, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'tgk_Cyrl-kmr_Latn': {'num_samples': 1997, 'number_of_characters': 520093, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 137.28, 'max_sentence1_length': 451, 'unique_sentence1': 1995, 'min_sentence2_length': 7, 'average_sentence2_length': 123.16, 'max_sentence2_length': 420, 'unique_sentence2': 1996}, 'tgk_Cyrl-mey_Arab': {'num_samples': 1997, 'number_of_characters': 487982, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 137.28, 'max_sentence1_length': 451, 'unique_sentence1': 1995, 'min_sentence2_length': 6, 'average_sentence2_length': 107.08, 'max_sentence2_length': 392, 'unique_sentence2': 1993}, 'tgk_Cyrl-prs_Arab': {'num_samples': 1997, 'number_of_characters': 516683, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 137.28, 'max_sentence1_length': 451, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 121.45, 'max_sentence2_length': 365, 'unique_sentence2': 1997}, 'tgk_Cyrl-pus_Arab': {'num_samples': 1997, 'number_of_characters': 516780, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 137.28, 'max_sentence1_length': 451, 'unique_sentence1': 1995, 'min_sentence2_length': 8, 'average_sentence2_length': 121.5, 'max_sentence2_length': 366, 'unique_sentence2': 1996}, 'tgk_Cyrl-shi_Arab': {'num_samples': 1997, 'number_of_characters': 489060, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 137.28, 'max_sentence1_length': 451, 'unique_sentence1': 1995, 'min_sentence2_length': 3, 'average_sentence2_length': 107.62, 'max_sentence2_length': 378, 'unique_sentence2': 1996}, 'tha_Thai-bod_Tibt': {'num_samples': 1997, 'number_of_characters': 538097, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 118.91, 'max_sentence1_length': 439, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 150.54, 'max_sentence2_length': 478, 'unique_sentence2': 1993}, 'tha_Thai-dzo_Tibt': {'num_samples': 1997, 'number_of_characters': 480689, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 118.91, 'max_sentence1_length': 439, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 121.79, 'max_sentence2_length': 411, 'unique_sentence2': 1992}, 'tha_Thai-eng_Latn': {'num_samples': 1997, 'number_of_characters': 485188, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 118.91, 'max_sentence1_length': 439, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'tha_Thai-khm_Khmr': {'num_samples': 1997, 'number_of_characters': 525959, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 118.91, 'max_sentence1_length': 439, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 144.46, 'max_sentence2_length': 517, 'unique_sentence2': 1996}, 'tha_Thai-lao_Laoo': {'num_samples': 1997, 'number_of_characters': 504448, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 118.91, 'max_sentence1_length': 439, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 133.69, 'max_sentence2_length': 507, 'unique_sentence2': 1997}, 'tha_Thai-mon_Mong': {'num_samples': 1997, 'number_of_characters': 496516, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 118.91, 'max_sentence1_length': 439, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 129.72, 'max_sentence2_length': 414, 'unique_sentence2': 1997}, 'tha_Thai-mya_Mymr': {'num_samples': 1997, 'number_of_characters': 549322, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 118.91, 'max_sentence1_length': 439, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 156.16, 'max_sentence2_length': 773, 'unique_sentence2': 1997}, 'tir_Ethi-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 332745, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'tir_Ethi-eng_Latn': {'num_samples': 1997, 'number_of_characters': 412958, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'tir_Ethi-hau_Latn': {'num_samples': 1997, 'number_of_characters': 435204, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'tir_Ethi-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 411339, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'tir_Ethi-nso_Latn': {'num_samples': 1997, 'number_of_characters': 456737, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'tir_Ethi-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 402669, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'tir_Ethi-som_Latn': {'num_samples': 1997, 'number_of_characters': 456530, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'tir_Ethi-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 453380, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'tir_Ethi-swa_Latn': {'num_samples': 1997, 'number_of_characters': 437747, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'tir_Ethi-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 499521, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'tir_Ethi-wol_Latn': {'num_samples': 1997, 'number_of_characters': 405041, 'unique_pairs': 1996, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'tir_Ethi-xho_Latn': {'num_samples': 1997, 'number_of_characters': 433328, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'tir_Ethi-yor_Latn': {'num_samples': 1997, 'number_of_characters': 481326, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'tir_Ethi-zul_Latn': {'num_samples': 1997, 'number_of_characters': 422970, 'unique_pairs': 1997, 'min_sentence1_length': 5, 'average_sentence1_length': 82.74, 'max_sentence1_length': 272, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'ton_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 561360, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 157.06, 'max_sentence1_length': 468, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ton_Latn-fij_Latn': {'num_samples': 1997, 'number_of_characters': 614145, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 157.06, 'max_sentence1_length': 468, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 150.48, 'max_sentence2_length': 448, 'unique_sentence2': 1988}, 'ton_Latn-fil_Latn': {'num_samples': 1997, 'number_of_characters': 607060, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 157.06, 'max_sentence1_length': 468, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 146.93, 'max_sentence2_length': 554, 'unique_sentence2': 1997}, 'ton_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 600612, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 157.06, 'max_sentence1_length': 468, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'ton_Latn-mal_Mlym': {'num_samples': 1997, 'number_of_characters': 617792, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 157.06, 'max_sentence1_length': 468, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 152.3, 'max_sentence2_length': 540, 'unique_sentence2': 1996}, 'ton_Latn-mlg_Latn': {'num_samples': 1997, 'number_of_characters': 633948, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 157.06, 'max_sentence1_length': 468, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 160.39, 'max_sentence2_length': 559, 'unique_sentence2': 1994}, 'ton_Latn-mri_Latn': {'num_samples': 1997, 'number_of_characters': 587764, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 157.06, 'max_sentence1_length': 468, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 137.27, 'max_sentence2_length': 443, 'unique_sentence2': 1997}, 'ton_Latn-msa_Latn': {'num_samples': 1997, 'number_of_characters': 590823, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 157.06, 'max_sentence1_length': 468, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 138.8, 'max_sentence2_length': 463, 'unique_sentence2': 1997}, 'ton_Latn-smo_Latn': {'num_samples': 1997, 'number_of_characters': 591495, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 157.06, 'max_sentence1_length': 468, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 139.14, 'max_sentence2_length': 431, 'unique_sentence2': 1996}, 'ton_Latn-tah_Latn': {'num_samples': 1997, 'number_of_characters': 623263, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 157.06, 'max_sentence1_length': 468, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 155.04, 'max_sentence2_length': 524, 'unique_sentence2': 1997}, 'tsn_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 501790, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'tsn_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 582003, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'tsn_Latn-hau_Latn': {'num_samples': 1997, 'number_of_characters': 604249, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'tsn_Latn-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 580384, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'tsn_Latn-nso_Latn': {'num_samples': 1997, 'number_of_characters': 625782, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'tsn_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 571714, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'tsn_Latn-som_Latn': {'num_samples': 1997, 'number_of_characters': 625575, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'tsn_Latn-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 622425, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'tsn_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 606792, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'tsn_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 499521, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'tsn_Latn-wol_Latn': {'num_samples': 1997, 'number_of_characters': 574086, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'tsn_Latn-xho_Latn': {'num_samples': 1997, 'number_of_characters': 602373, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'tsn_Latn-yor_Latn': {'num_samples': 1997, 'number_of_characters': 650371, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'tsn_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 592015, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 167.39, 'max_sentence1_length': 556, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'tuk_Latn-aze_Latn': {'num_samples': 1997, 'number_of_characters': 554908, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 142.85, 'max_sentence1_length': 576, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 135.02, 'max_sentence2_length': 398, 'unique_sentence2': 1997}, 'tuk_Latn-bak_Cyrl': {'num_samples': 1997, 'number_of_characters': 531600, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 142.85, 'max_sentence1_length': 576, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 123.35, 'max_sentence2_length': 437, 'unique_sentence2': 1995}, 'tuk_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 532994, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 142.85, 'max_sentence1_length': 576, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'tuk_Latn-kaz_Cyrl': {'num_samples': 1997, 'number_of_characters': 545550, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 142.85, 'max_sentence1_length': 576, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 130.33, 'max_sentence2_length': 473, 'unique_sentence2': 1996}, 'tuk_Latn-kir_Cyrl': {'num_samples': 1997, 'number_of_characters': 536138, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 142.85, 'max_sentence1_length': 576, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.62, 'max_sentence2_length': 395, 'unique_sentence2': 1996}, 'tuk_Latn-tat_Cyrl': {'num_samples': 1997, 'number_of_characters': 531200, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 142.85, 'max_sentence1_length': 576, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 123.15, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'tuk_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 550887, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 142.85, 'max_sentence1_length': 576, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'tuk_Latn-uig_Arab': {'num_samples': 1997, 'number_of_characters': 596296, 'unique_pairs': 1997, 'min_sentence1_length': 9, 'average_sentence1_length': 142.85, 'max_sentence1_length': 576, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 155.74, 'max_sentence2_length': 592, 'unique_sentence2': 1996}, 'tuk_Latn-uzb_Latn': {'num_samples': 1997, 'number_of_characters': 578969, 'unique_pairs': 1996, 'min_sentence1_length': 9, 'average_sentence1_length': 142.85, 'max_sentence1_length': 576, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 147.07, 'max_sentence2_length': 470, 'unique_sentence2': 1996}, 'tur_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 496794, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'tur_Latn-aze_Latn': {'num_samples': 1997, 'number_of_characters': 535247, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 135.02, 'max_sentence2_length': 398, 'unique_sentence2': 1997}, 'tur_Latn-bak_Cyrl': {'num_samples': 1997, 'number_of_characters': 511939, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 123.35, 'max_sentence2_length': 437, 'unique_sentence2': 1995}, 'tur_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 509415, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'tur_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 561263, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'tur_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 564740, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'tur_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 513333, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'tur_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 508952, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'tur_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 535413, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'tur_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 558721, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'tur_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 465909, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'tur_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 527188, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'tur_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 543989, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'tur_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 552585, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'tur_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 377239, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'tur_Latn-kaz_Cyrl': {'num_samples': 1997, 'number_of_characters': 525889, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 130.33, 'max_sentence2_length': 473, 'unique_sentence2': 1996}, 'tur_Latn-kir_Cyrl': {'num_samples': 1997, 'number_of_characters': 516477, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 125.62, 'max_sentence2_length': 395, 'unique_sentence2': 1996}, 'tur_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 399018, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'tur_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 525010, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'tur_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 557528, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'tur_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 543479, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'tur_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 542828, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'tur_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 540506, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'tur_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 553813, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'tur_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 538122, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'tur_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 517440, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'tur_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 575574, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'tur_Latn-tat_Cyrl': {'num_samples': 1997, 'number_of_characters': 511539, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 123.15, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'tur_Latn-tuk_Latn': {'num_samples': 1997, 'number_of_characters': 550887, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 142.85, 'max_sentence2_length': 576, 'unique_sentence2': 1996}, 'tur_Latn-uig_Arab': {'num_samples': 1997, 'number_of_characters': 576635, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 155.74, 'max_sentence2_length': 592, 'unique_sentence2': 1996}, 'tur_Latn-uzb_Latn': {'num_samples': 1997, 'number_of_characters': 559308, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 147.07, 'max_sentence2_length': 470, 'unique_sentence2': 1996}, 'tur_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 536734, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'tur_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 357091, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'tur_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 523345, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 133.01, 'max_sentence1_length': 504, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'uig_Arab-aze_Latn': {'num_samples': 1997, 'number_of_characters': 580656, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 155.74, 'max_sentence1_length': 592, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 135.02, 'max_sentence2_length': 398, 'unique_sentence2': 1997}, 'uig_Arab-bak_Cyrl': {'num_samples': 1997, 'number_of_characters': 557348, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 155.74, 'max_sentence1_length': 592, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 123.35, 'max_sentence2_length': 437, 'unique_sentence2': 1995}, 'uig_Arab-eng_Latn': {'num_samples': 1997, 'number_of_characters': 558742, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 155.74, 'max_sentence1_length': 592, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'uig_Arab-kaz_Cyrl': {'num_samples': 1997, 'number_of_characters': 571298, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 155.74, 'max_sentence1_length': 592, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 130.33, 'max_sentence2_length': 473, 'unique_sentence2': 1996}, 'uig_Arab-kir_Cyrl': {'num_samples': 1997, 'number_of_characters': 561886, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 155.74, 'max_sentence1_length': 592, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.62, 'max_sentence2_length': 395, 'unique_sentence2': 1996}, 'uig_Arab-tat_Cyrl': {'num_samples': 1997, 'number_of_characters': 556948, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 155.74, 'max_sentence1_length': 592, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 123.15, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'uig_Arab-tuk_Latn': {'num_samples': 1997, 'number_of_characters': 596296, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 155.74, 'max_sentence1_length': 592, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 142.85, 'max_sentence2_length': 576, 'unique_sentence2': 1996}, 'uig_Arab-tur_Latn': {'num_samples': 1997, 'number_of_characters': 576635, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 155.74, 'max_sentence1_length': 592, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'uig_Arab-uzb_Latn': {'num_samples': 1997, 'number_of_characters': 604717, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 155.74, 'max_sentence1_length': 592, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 147.07, 'max_sentence2_length': 470, 'unique_sentence2': 1996}, 'ukr_Cyrl-bel_Cyrl': {'num_samples': 1997, 'number_of_characters': 518873, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.24, 'max_sentence2_length': 422, 'unique_sentence2': 1996}, 'ukr_Cyrl-bos_Latn': {'num_samples': 1997, 'number_of_characters': 517693, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 127.65, 'max_sentence2_length': 434, 'unique_sentence2': 1996}, 'ukr_Cyrl-bul_Cyrl': {'num_samples': 1997, 'number_of_characters': 532672, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 135.15, 'max_sentence2_length': 493, 'unique_sentence2': 1996}, 'ukr_Cyrl-ces_Latn': {'num_samples': 1997, 'number_of_characters': 504101, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 120.84, 'max_sentence2_length': 474, 'unique_sentence2': 1997}, 'ukr_Cyrl-eng_Latn': {'num_samples': 1997, 'number_of_characters': 510503, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ukr_Cyrl-hrv_Latn': {'num_samples': 1997, 'number_of_characters': 518708, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 128.15, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'ukr_Cyrl-mkd_Cyrl': {'num_samples': 1997, 'number_of_characters': 530674, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.15, 'max_sentence2_length': 451, 'unique_sentence2': 1997}, 'ukr_Cyrl-pol_Latn': {'num_samples': 1997, 'number_of_characters': 540649, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'ukr_Cyrl-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 537676, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'ukr_Cyrl-slk_Latn': {'num_samples': 1997, 'number_of_characters': 515752, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 126.67, 'max_sentence2_length': 403, 'unique_sentence2': 1996}, 'ukr_Cyrl-slv_Latn': {'num_samples': 1997, 'number_of_characters': 515679, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.64, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'ukr_Cyrl-srp_Cyrl': {'num_samples': 1997, 'number_of_characters': 515086, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 126.34, 'max_sentence2_length': 439, 'unique_sentence2': 1995}, 'ukr_Cyrl-srp_Latn': {'num_samples': 1997, 'number_of_characters': 518924, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 131.59, 'max_sentence1_length': 440, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 128.26, 'max_sentence2_length': 452, 'unique_sentence2': 1996}, 'urd_Arab-ben_Beng': {'num_samples': 1997, 'number_of_characters': 491800, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'urd_Arab-div_Thaa': {'num_samples': 1997, 'number_of_characters': 551846, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 152.15, 'max_sentence2_length': 609, 'unique_sentence2': 1996}, 'urd_Arab-eng_Latn': {'num_samples': 1997, 'number_of_characters': 495718, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'urd_Arab-eus_Latn': {'num_samples': 1997, 'number_of_characters': 523201, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 137.81, 'max_sentence2_length': 393, 'unique_sentence2': 1997}, 'urd_Arab-guj_Gujr': {'num_samples': 1997, 'number_of_characters': 492929, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 122.65, 'max_sentence2_length': 378, 'unique_sentence2': 1997}, 'urd_Arab-hin_Deva': {'num_samples': 1997, 'number_of_characters': 509573, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'urd_Arab-kan_Knda': {'num_samples': 1997, 'number_of_characters': 513534, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 132.97, 'max_sentence2_length': 449, 'unique_sentence2': 1996}, 'urd_Arab-mar_Deva': {'num_samples': 1997, 'number_of_characters': 508885, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 130.64, 'max_sentence2_length': 443, 'unique_sentence2': 1995}, 'urd_Arab-nep_Deva': {'num_samples': 1997, 'number_of_characters': 496221, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 124.3, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'urd_Arab-pan_Guru': {'num_samples': 1997, 'number_of_characters': 498420, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 125.4, 'max_sentence2_length': 383, 'unique_sentence2': 1996}, 'urd_Arab-sin_Sinh': {'num_samples': 1997, 'number_of_characters': 506739, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 129.56, 'max_sentence2_length': 441, 'unique_sentence2': 1996}, 'urd_Arab-snd_Arab': {'num_samples': 1997, 'number_of_characters': 468325, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 110.33, 'max_sentence2_length': 335, 'unique_sentence2': 1996}, 'urd_Arab-tam_Taml': {'num_samples': 1997, 'number_of_characters': 557959, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'urd_Arab-tel_Telu': {'num_samples': 1997, 'number_of_characters': 495525, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 124.19, 'max_sentence1_length': 390, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 123.95, 'max_sentence2_length': 412, 'unique_sentence2': 1996}, 'uzb_Latn-aze_Latn': {'num_samples': 1997, 'number_of_characters': 563329, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 147.07, 'max_sentence1_length': 470, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 135.02, 'max_sentence2_length': 398, 'unique_sentence2': 1997}, 'uzb_Latn-bak_Cyrl': {'num_samples': 1997, 'number_of_characters': 540021, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 147.07, 'max_sentence1_length': 470, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 123.35, 'max_sentence2_length': 437, 'unique_sentence2': 1995}, 'uzb_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 541415, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 147.07, 'max_sentence1_length': 470, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'uzb_Latn-kaz_Cyrl': {'num_samples': 1997, 'number_of_characters': 553971, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 147.07, 'max_sentence1_length': 470, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 130.33, 'max_sentence2_length': 473, 'unique_sentence2': 1996}, 'uzb_Latn-kir_Cyrl': {'num_samples': 1997, 'number_of_characters': 544559, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 147.07, 'max_sentence1_length': 470, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 125.62, 'max_sentence2_length': 395, 'unique_sentence2': 1996}, 'uzb_Latn-tat_Cyrl': {'num_samples': 1997, 'number_of_characters': 539621, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 147.07, 'max_sentence1_length': 470, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 123.15, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'uzb_Latn-tuk_Latn': {'num_samples': 1997, 'number_of_characters': 578969, 'unique_pairs': 1996, 'min_sentence1_length': 8, 'average_sentence1_length': 147.07, 'max_sentence1_length': 470, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 142.85, 'max_sentence2_length': 576, 'unique_sentence2': 1996}, 'uzb_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 559308, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 147.07, 'max_sentence1_length': 470, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'uzb_Latn-uig_Arab': {'num_samples': 1997, 'number_of_characters': 604717, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 147.07, 'max_sentence1_length': 470, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 155.74, 'max_sentence2_length': 592, 'unique_sentence2': 1996}, 'ven_Latn-bem_Latn': {'num_samples': 1997, 'number_of_characters': 598248, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.1, 'max_sentence1_length': 535, 'unique_sentence1': 1993, 'min_sentence2_length': 8, 'average_sentence2_length': 149.47, 'max_sentence2_length': 465, 'unique_sentence2': 1997}, 'ven_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 547476, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.1, 'max_sentence1_length': 535, 'unique_sentence1': 1993, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'ven_Latn-ewe_Latn': {'num_samples': 1997, 'number_of_characters': 538734, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 150.1, 'max_sentence1_length': 535, 'unique_sentence1': 1993, 'min_sentence2_length': 6, 'average_sentence2_length': 119.67, 'max_sentence2_length': 493, 'unique_sentence2': 1994}, 'ven_Latn-fuc_Latn': {'num_samples': 1997, 'number_of_characters': 528236, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.1, 'max_sentence1_length': 535, 'unique_sentence1': 1993, 'min_sentence2_length': 6, 'average_sentence2_length': 114.41, 'max_sentence2_length': 376, 'unique_sentence2': 1996}, 'ven_Latn-kin_Latn': {'num_samples': 1997, 'number_of_characters': 603543, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 150.1, 'max_sentence1_length': 535, 'unique_sentence1': 1993, 'min_sentence2_length': 10, 'average_sentence2_length': 152.12, 'max_sentence2_length': 541, 'unique_sentence2': 1996}, 'ven_Latn-nde_Latn': {'num_samples': 1997, 'number_of_characters': 597495, 'unique_pairs': 1997, 'min_sentence1_length': 10, 'average_sentence1_length': 150.1, 'max_sentence1_length': 535, 'unique_sentence1': 1993, 'min_sentence2_length': 6, 'average_sentence2_length': 149.09, 'max_sentence2_length': 590, 'unique_sentence2': 1997}, 'ven_Latn-nya_Latn': {'num_samples': 1997, 'number_of_characters': 584038, 'unique_pairs': 1996, 'min_sentence1_length': 10, 'average_sentence1_length': 150.1, 'max_sentence1_length': 535, 'unique_sentence1': 1993, 'min_sentence2_length': 10, 'average_sentence2_length': 142.35, 'max_sentence2_length': 464, 'unique_sentence2': 1993}, 'ven_Latn-sna_Latn': {'num_samples': 1997, 'number_of_characters': 598086, 'unique_pairs': 1995, 'min_sentence1_length': 10, 'average_sentence1_length': 150.1, 'max_sentence1_length': 535, 'unique_sentence1': 1993, 'min_sentence2_length': 6, 'average_sentence2_length': 149.39, 'max_sentence2_length': 511, 'unique_sentence2': 1995}, 'vie_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 502302, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'vie_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 514923, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'vie_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 566771, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'vie_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 570248, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'vie_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 518841, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'vie_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 514460, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'vie_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 540921, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'vie_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 564229, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'vie_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 471417, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'vie_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 532696, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'vie_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 549497, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'vie_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 558093, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'vie_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 382747, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'vie_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 404526, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'vie_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 530518, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'vie_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 563036, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'vie_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 548987, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'vie_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 548336, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'vie_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 546014, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'vie_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 559321, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'vie_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 543630, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'vie_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 522948, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'vie_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 581082, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'vie_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 536734, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'vie_Latn-yue_Hant': {'num_samples': 1997, 'number_of_characters': 350008, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 39.5, 'max_sentence2_length': 133, 'unique_sentence2': 1996}, 'vie_Latn-zho_Hans': {'num_samples': 1997, 'number_of_characters': 356082, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 42.54, 'max_sentence2_length': 263, 'unique_sentence2': 1997}, 'vie_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 362599, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'vie_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 528853, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 135.76, 'max_sentence1_length': 437, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'wol_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 407310, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'wol_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 487523, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'wol_Latn-hau_Latn': {'num_samples': 1997, 'number_of_characters': 509769, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'wol_Latn-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 485904, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'wol_Latn-nso_Latn': {'num_samples': 1997, 'number_of_characters': 531302, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'wol_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 477234, 'unique_pairs': 1992, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'wol_Latn-som_Latn': {'num_samples': 1997, 'number_of_characters': 531095, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'wol_Latn-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 527945, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'wol_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 512312, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'wol_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 405041, 'unique_pairs': 1996, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'wol_Latn-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 574086, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'wol_Latn-xho_Latn': {'num_samples': 1997, 'number_of_characters': 507893, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'wol_Latn-yor_Latn': {'num_samples': 1997, 'number_of_characters': 555891, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'wol_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 497535, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 120.08, 'max_sentence1_length': 405, 'unique_sentence1': 1990, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'xho_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 435597, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'xho_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 515810, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'xho_Latn-hau_Latn': {'num_samples': 1997, 'number_of_characters': 538056, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'xho_Latn-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 514191, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'xho_Latn-nso_Latn': {'num_samples': 1997, 'number_of_characters': 559589, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'xho_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 505521, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'xho_Latn-som_Latn': {'num_samples': 1997, 'number_of_characters': 559382, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'xho_Latn-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 556232, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'xho_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 540599, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'xho_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 433328, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'xho_Latn-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 602373, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'xho_Latn-wol_Latn': {'num_samples': 1997, 'number_of_characters': 507893, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'xho_Latn-yor_Latn': {'num_samples': 1997, 'number_of_characters': 584178, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'xho_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 525822, 'unique_pairs': 1997, 'min_sentence1_length': 6, 'average_sentence1_length': 134.25, 'max_sentence1_length': 492, 'unique_sentence1': 1997, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'yor_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 483595, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'yor_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 563808, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'yor_Latn-hau_Latn': {'num_samples': 1997, 'number_of_characters': 586054, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'yor_Latn-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 562189, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'yor_Latn-nso_Latn': {'num_samples': 1997, 'number_of_characters': 607587, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'yor_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 553519, 'unique_pairs': 1996, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'yor_Latn-som_Latn': {'num_samples': 1997, 'number_of_characters': 607380, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'yor_Latn-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 604230, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'yor_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 588597, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'yor_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 481326, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'yor_Latn-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 650371, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'yor_Latn-wol_Latn': {'num_samples': 1997, 'number_of_characters': 555891, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'yor_Latn-xho_Latn': {'num_samples': 1997, 'number_of_characters': 584178, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'yor_Latn-zul_Latn': {'num_samples': 1997, 'number_of_characters': 573820, 'unique_pairs': 1997, 'min_sentence1_length': 7, 'average_sentence1_length': 158.28, 'max_sentence1_length': 582, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'yue_Hant-eng_Latn': {'num_samples': 1997, 'number_of_characters': 326607, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 39.5, 'max_sentence1_length': 133, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'yue_Hant-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 190513, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 39.5, 'max_sentence1_length': 133, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'yue_Hant-kor_Hang': {'num_samples': 1997, 'number_of_characters': 212292, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 39.5, 'max_sentence1_length': 133, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'yue_Hant-vie_Latn': {'num_samples': 1997, 'number_of_characters': 350008, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 39.5, 'max_sentence1_length': 133, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'yue_Hant-zho_Hans': {'num_samples': 1997, 'number_of_characters': 163848, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 39.5, 'max_sentence1_length': 133, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 42.54, 'max_sentence2_length': 263, 'unique_sentence2': 1997}, 'yue_Hant-zho_Hant': {'num_samples': 1997, 'number_of_characters': 170365, 'unique_pairs': 1996, 'min_sentence1_length': 4, 'average_sentence1_length': 39.5, 'max_sentence1_length': 133, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'zho_Hans-eng_Latn': {'num_samples': 1997, 'number_of_characters': 332681, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 42.54, 'max_sentence1_length': 263, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'zho_Hans-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 196587, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 42.54, 'max_sentence1_length': 263, 'unique_sentence1': 1997, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'zho_Hans-kor_Hang': {'num_samples': 1997, 'number_of_characters': 218366, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 42.54, 'max_sentence1_length': 263, 'unique_sentence1': 1997, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'zho_Hans-vie_Latn': {'num_samples': 1997, 'number_of_characters': 356082, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 42.54, 'max_sentence1_length': 263, 'unique_sentence1': 1997, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'zho_Hans-yue_Hant': {'num_samples': 1997, 'number_of_characters': 163848, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 42.54, 'max_sentence1_length': 263, 'unique_sentence1': 1997, 'min_sentence2_length': 4, 'average_sentence2_length': 39.5, 'max_sentence2_length': 133, 'unique_sentence2': 1996}, 'zho_Hans-zho_Hant': {'num_samples': 1997, 'number_of_characters': 176439, 'unique_pairs': 1997, 'min_sentence1_length': 4, 'average_sentence1_length': 42.54, 'max_sentence1_length': 263, 'unique_sentence1': 1997, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}, 'zho_Hant-arb_Arab': {'num_samples': 1997, 'number_of_characters': 322659, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'zho_Hant-ben_Beng': {'num_samples': 1997, 'number_of_characters': 335280, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'zho_Hant-deu_Latn': {'num_samples': 1997, 'number_of_characters': 387128, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'zho_Hant-ell_Grek': {'num_samples': 1997, 'number_of_characters': 390605, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'zho_Hant-eng_Latn': {'num_samples': 1997, 'number_of_characters': 339198, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'zho_Hant-fas_Arab': {'num_samples': 1997, 'number_of_characters': 334817, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'zho_Hant-fin_Latn': {'num_samples': 1997, 'number_of_characters': 361278, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'zho_Hant-fra_Latn': {'num_samples': 1997, 'number_of_characters': 384586, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'zho_Hant-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 291774, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'zho_Hant-hin_Deva': {'num_samples': 1997, 'number_of_characters': 353053, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'zho_Hant-hun_Latn': {'num_samples': 1997, 'number_of_characters': 369854, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'zho_Hant-ind_Latn': {'num_samples': 1997, 'number_of_characters': 378450, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'zho_Hant-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 203104, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'zho_Hant-kor_Hang': {'num_samples': 1997, 'number_of_characters': 224883, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'zho_Hant-lit_Latn': {'num_samples': 1997, 'number_of_characters': 350875, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'zho_Hant-nld_Latn': {'num_samples': 1997, 'number_of_characters': 383393, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'zho_Hant-pol_Latn': {'num_samples': 1997, 'number_of_characters': 369344, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'zho_Hant-por_Latn': {'num_samples': 1997, 'number_of_characters': 368693, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'zho_Hant-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 366371, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'zho_Hant-spa_Latn': {'num_samples': 1997, 'number_of_characters': 379678, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'zho_Hant-swa_Latn': {'num_samples': 1997, 'number_of_characters': 363987, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'zho_Hant-swe_Latn': {'num_samples': 1997, 'number_of_characters': 343305, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'zho_Hant-tam_Taml': {'num_samples': 1997, 'number_of_characters': 401439, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'zho_Hant-tur_Latn': {'num_samples': 1997, 'number_of_characters': 357091, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'zho_Hant-vie_Latn': {'num_samples': 1997, 'number_of_characters': 362599, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'zho_Hant-yue_Hant': {'num_samples': 1997, 'number_of_characters': 170365, 'unique_pairs': 1996, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 39.5, 'max_sentence2_length': 133, 'unique_sentence2': 1996}, 'zho_Hant-zho_Hans': {'num_samples': 1997, 'number_of_characters': 176439, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 42.54, 'max_sentence2_length': 263, 'unique_sentence2': 1997}, 'zho_Hant-zul_Latn': {'num_samples': 1997, 'number_of_characters': 349210, 'unique_pairs': 1997, 'min_sentence1_length': 3, 'average_sentence1_length': 45.81, 'max_sentence1_length': 200, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 129.06, 'max_sentence2_length': 494, 'unique_sentence2': 1996}, 'zul_Latn-amh_Ethi': {'num_samples': 1997, 'number_of_characters': 425239, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 83.88, 'max_sentence2_length': 290, 'unique_sentence2': 1994}, 'zul_Latn-arb_Arab': {'num_samples': 1997, 'number_of_characters': 488913, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 115.76, 'max_sentence2_length': 362, 'unique_sentence2': 1995}, 'zul_Latn-ben_Beng': {'num_samples': 1997, 'number_of_characters': 501534, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 122.08, 'max_sentence2_length': 402, 'unique_sentence2': 1997}, 'zul_Latn-deu_Latn': {'num_samples': 1997, 'number_of_characters': 553382, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 148.05, 'max_sentence2_length': 508, 'unique_sentence2': 1996}, 'zul_Latn-ell_Grek': {'num_samples': 1997, 'number_of_characters': 556859, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 149.79, 'max_sentence2_length': 584, 'unique_sentence2': 1996}, 'zul_Latn-eng_Latn': {'num_samples': 1997, 'number_of_characters': 505452, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 124.05, 'max_sentence2_length': 437, 'unique_sentence2': 1997}, 'zul_Latn-fas_Arab': {'num_samples': 1997, 'number_of_characters': 501071, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 121.85, 'max_sentence2_length': 389, 'unique_sentence2': 1995}, 'zul_Latn-fin_Latn': {'num_samples': 1997, 'number_of_characters': 527532, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.1, 'max_sentence2_length': 463, 'unique_sentence2': 1996}, 'zul_Latn-fra_Latn': {'num_samples': 1997, 'number_of_characters': 550840, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.77, 'max_sentence2_length': 512, 'unique_sentence2': 1996}, 'zul_Latn-hau_Latn': {'num_samples': 1997, 'number_of_characters': 527698, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 135.19, 'max_sentence2_length': 483, 'unique_sentence2': 1997}, 'zul_Latn-heb_Hebr': {'num_samples': 1997, 'number_of_characters': 458028, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 100.3, 'max_sentence2_length': 375, 'unique_sentence2': 1996}, 'zul_Latn-hin_Deva': {'num_samples': 1997, 'number_of_characters': 519307, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 130.98, 'max_sentence2_length': 394, 'unique_sentence2': 1996}, 'zul_Latn-hun_Latn': {'num_samples': 1997, 'number_of_characters': 536108, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 139.4, 'max_sentence2_length': 508, 'unique_sentence2': 1997}, 'zul_Latn-ibo_Latn': {'num_samples': 1997, 'number_of_characters': 503833, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 123.24, 'max_sentence2_length': 469, 'unique_sentence2': 1997}, 'zul_Latn-ind_Latn': {'num_samples': 1997, 'number_of_characters': 544704, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 143.7, 'max_sentence2_length': 486, 'unique_sentence2': 1997}, 'zul_Latn-jpn_Jpan': {'num_samples': 1997, 'number_of_characters': 369358, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 4, 'average_sentence2_length': 55.9, 'max_sentence2_length': 189, 'unique_sentence2': 1994}, 'zul_Latn-kor_Hang': {'num_samples': 1997, 'number_of_characters': 391137, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 66.8, 'max_sentence2_length': 217, 'unique_sentence2': 1995}, 'zul_Latn-lit_Latn': {'num_samples': 1997, 'number_of_characters': 517129, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 129.89, 'max_sentence2_length': 446, 'unique_sentence2': 1995}, 'zul_Latn-nld_Latn': {'num_samples': 1997, 'number_of_characters': 549647, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 146.18, 'max_sentence2_length': 539, 'unique_sentence2': 1996}, 'zul_Latn-nso_Latn': {'num_samples': 1997, 'number_of_characters': 549231, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 145.97, 'max_sentence2_length': 487, 'unique_sentence2': 1996}, 'zul_Latn-orm_Ethi': {'num_samples': 1997, 'number_of_characters': 495163, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 118.89, 'max_sentence2_length': 466, 'unique_sentence2': 1984}, 'zul_Latn-pol_Latn': {'num_samples': 1997, 'number_of_characters': 535598, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 9, 'average_sentence2_length': 139.14, 'max_sentence2_length': 468, 'unique_sentence2': 1996}, 'zul_Latn-por_Latn': {'num_samples': 1997, 'number_of_characters': 534947, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 138.82, 'max_sentence2_length': 497, 'unique_sentence2': 1996}, 'zul_Latn-rus_Cyrl': {'num_samples': 1997, 'number_of_characters': 532625, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 137.65, 'max_sentence2_length': 419, 'unique_sentence2': 1996}, 'zul_Latn-som_Latn': {'num_samples': 1997, 'number_of_characters': 549024, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 145.86, 'max_sentence2_length': 455, 'unique_sentence2': 1997}, 'zul_Latn-spa_Latn': {'num_samples': 1997, 'number_of_characters': 545932, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 1, 'average_sentence2_length': 144.32, 'max_sentence2_length': 504, 'unique_sentence2': 1996}, 'zul_Latn-ssw_Latn': {'num_samples': 1997, 'number_of_characters': 545874, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 144.29, 'max_sentence2_length': 510, 'unique_sentence2': 1996}, 'zul_Latn-swa_Latn': {'num_samples': 1997, 'number_of_characters': 530241, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 10, 'average_sentence2_length': 136.46, 'max_sentence2_length': 430, 'unique_sentence2': 1997}, 'zul_Latn-swe_Latn': {'num_samples': 1997, 'number_of_characters': 509559, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 8, 'average_sentence2_length': 126.1, 'max_sentence2_length': 430, 'unique_sentence2': 1996}, 'zul_Latn-tam_Taml': {'num_samples': 1997, 'number_of_characters': 567693, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 11, 'average_sentence2_length': 155.21, 'max_sentence2_length': 581, 'unique_sentence2': 1997}, 'zul_Latn-tir_Ethi': {'num_samples': 1997, 'number_of_characters': 422970, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 5, 'average_sentence2_length': 82.74, 'max_sentence2_length': 272, 'unique_sentence2': 1996}, 'zul_Latn-tsn_Latn': {'num_samples': 1997, 'number_of_characters': 592015, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 167.39, 'max_sentence2_length': 556, 'unique_sentence2': 1997}, 'zul_Latn-tur_Latn': {'num_samples': 1997, 'number_of_characters': 523345, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 133.01, 'max_sentence2_length': 504, 'unique_sentence2': 1997}, 'zul_Latn-vie_Latn': {'num_samples': 1997, 'number_of_characters': 528853, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 135.76, 'max_sentence2_length': 437, 'unique_sentence2': 1996}, 'zul_Latn-wol_Latn': {'num_samples': 1997, 'number_of_characters': 497535, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 120.08, 'max_sentence2_length': 405, 'unique_sentence2': 1990}, 'zul_Latn-xho_Latn': {'num_samples': 1997, 'number_of_characters': 525822, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 6, 'average_sentence2_length': 134.25, 'max_sentence2_length': 492, 'unique_sentence2': 1997}, 'zul_Latn-yor_Latn': {'num_samples': 1997, 'number_of_characters': 573820, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 7, 'average_sentence2_length': 158.28, 'max_sentence2_length': 582, 'unique_sentence2': 1996}, 'zul_Latn-zho_Hant': {'num_samples': 1997, 'number_of_characters': 349210, 'unique_pairs': 1997, 'min_sentence1_length': 8, 'average_sentence1_length': 129.06, 'max_sentence1_length': 494, 'unique_sentence1': 1996, 'min_sentence2_length': 3, 'average_sentence2_length': 45.81, 'max_sentence2_length': 200, 'unique_sentence2': 1996}}}} | +| [NYSJudicialEthicsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [NaijaSenti](https://github.com/hausanlp/NaijaSenti) | ['hau', 'ibo', 'pcm', 'yor'] | Classification | s2s | [Social, Written] | None | None | +| [NamaaMrTydiReranking](https://huggingface.co/NAMAA-Space) (Muennighoff et al., 2022) | ['ara'] | Reranking | s2s | [Encyclopaedic, Written] | None | None | +| [NanoArguAnaRetrieval](http://argumentation.bplaced.net/arguana/data) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | [Medical, Written] | None | None | +| [NanoClimateFeverRetrieval](https://arxiv.org/abs/2012.00614) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | [Non-fiction, Academic, News] | None | None | +| [NanoDBPediaRetrieval](https://huggingface.co/datasets/zeta-alpha-ai/NanoDBPedia) (Lehmann et al., 2015) | ['eng'] | Retrieval | s2p | [Encyclopaedic] | None | None | +| [NanoFEVERRetrieval](https://fever.ai/) | ['eng'] | Retrieval | s2p | [Academic, Encyclopaedic] | None | None | +| [NanoFiQA2018Retrieval](https://sites.google.com/view/fiqa/) (Nandan Thakur, 2021) | ['eng'] | Retrieval | s2p | [Academic, Social] | None | None | +| [NanoHotpotQARetrieval](https://hotpotqa.github.io/) | ['eng'] | Retrieval | s2p | [Web, Written] | None | None | +| [NanoMSMARCORetrieval](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | [Web] | None | None | +| [NanoNFCorpusRetrieval](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | [Medical, Academic, Written] | None | None | +| [NanoNQRetrieval](https://ai.google.com/research/NaturalQuestions) (Tom Kwiatkowski, 2019) | ['eng'] | Retrieval | s2p | [Academic, Web] | None | None | +| [NanoQuoraRetrieval](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (DataCanary et al., 2017) | ['eng'] | Retrieval | s2s | [Social] | None | None | +| [NanoSCIDOCSRetrieval](https://allenai.org/data/scidocs) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | [Academic, Written, Non-fiction] | None | None | +| [NanoSciFactRetrieval](https://github.com/allenai/scifact) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | [Academic, Medical, Written] | None | None | +| [NanoTouche2020Retrieval](https://webis.de/events/touche-20/shared-task-1.html) | ['eng'] | Retrieval | s2p | [Academic] | None | None | +| [NarrativeQARetrieval](https://metatext.io/datasets/narrativeqa) (Tomáš Kočiský, 2017) | ['eng'] | Retrieval | s2p | | None | None | +| [NepaliNewsClassification](https://github.com/goru001/nlp-for-nepali) | ['nep'] | Classification | s2s | [News, Written] | None | None | +| [NeuCLIR2022Retrieval](https://neuclir.github.io/) (Lawrie et al., 2023) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | None | None | +| [NeuCLIR2022RetrievalHardNegatives](https://neuclir.github.io/) (Lawrie et al., 2023) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | None | None | +| [NeuCLIR2023Retrieval](https://neuclir.github.io/) (Dawn Lawrie, 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | None | None | +| [NeuCLIR2023RetrievalHardNegatives](https://neuclir.github.io/) (Dawn Lawrie, 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | None | None | +| [News21InstructionRetrieval](https://arxiv.org/abs/2403.15246) (Orion Weller, 2024) | ['eng'] | InstructionRetrieval | s2p | [News, Written] | None | None | +| [NewsClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [News, Written] | None | None | +| [NoRecClassification](https://aclanthology.org/L18-1661/) | ['nob'] | Classification | s2s | [Written, Reviews] | None | None | +| [NollySentiBitextMining](https://github.com/IyanuSh/NollySenti) (Shode et al., 2023) | ['eng', 'hau', 'ibo', 'pcm', 'yor'] | BitextMining | s2s | [Social, Reviews, Written] | {'train': 1640} | {'train': {'num_samples': 1640, 'number_of_characters': 445805, 'unique_pairs': 1632, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 3, 'average_sentence2_length': 135.52, 'max_sentence2_length': 1728, 'unique_sentence2': 1631, 'hf_subset_descriptive_stats': {'en-ha': {'num_samples': 410, 'number_of_characters': 115348, 'unique_pairs': 407, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 4, 'average_sentence2_length': 145.02, 'max_sentence2_length': 1728, 'unique_sentence2': 407}, 'en-ig': {'num_samples': 410, 'number_of_characters': 107173, 'unique_pairs': 409, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 5, 'average_sentence2_length': 125.08, 'max_sentence2_length': 1137, 'unique_sentence2': 408}, 'en-pcm': {'num_samples': 410, 'number_of_characters': 109955, 'unique_pairs': 408, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 3, 'average_sentence2_length': 131.87, 'max_sentence2_length': 1552, 'unique_sentence2': 408}, 'en-yo': {'num_samples': 410, 'number_of_characters': 113329, 'unique_pairs': 409, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 6, 'average_sentence2_length': 140.1, 'max_sentence2_length': 1338, 'unique_sentence2': 409}}}} | +| [NorQuadRetrieval](https://aclanthology.org/2023.nodalida-1.17/) | ['nob'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Written] | None | None | +| [NordicLangClassification](https://aclanthology.org/2021.vardial-1.8/) | ['dan', 'fao', 'isl', 'nno', 'nob', 'swe'] | Classification | s2s | [Encyclopaedic] | None | None | +| [NorwegianCourtsBitextMining](https://opus.nlpl.eu/index.php) (Tiedemann et al., 2020) | ['nno', 'nob'] | BitextMining | s2s | [Legal, Written] | {'test': 228} | {'test': {'num_samples': 228, 'number_of_characters': 37441, 'unique_pairs': 228, 'min_sentence1_length': 13, 'average_sentence1_length': 82.2, 'max_sentence1_length': 272, 'unique_sentence1': 227, 'min_sentence2_length': 10, 'average_sentence2_length': 82.02, 'max_sentence2_length': 269, 'unique_sentence2': 226}} | +| [NorwegianParliamentClassification](https://huggingface.co/datasets/NbAiLab/norwegian_parliament) | ['nob'] | Classification | s2s | [Government, Spoken] | None | None | +| [NusaParagraphEmotionClassification](https://github.com/IndoNLP/nusa-writes) | ['bbc', 'bew', 'bug', 'jav', 'mad', 'mak', 'min', 'mui', 'rej', 'sun'] | Classification | s2s | [Non-fiction, Fiction, Written] | None | None | +| [NusaParagraphTopicClassification](https://github.com/IndoNLP/nusa-writes) | ['bbc', 'bew', 'bug', 'jav', 'mad', 'mak', 'min', 'mui', 'rej', 'sun'] | Classification | s2s | [Non-fiction, Fiction, Written] | None | None | +| [NusaTranslationBitextMining](https://huggingface.co/datasets/indonlp/nusatranslation_mt) (Cahyawijaya et al., 2023) | ['abs', 'bbc', 'bew', 'bhp', 'ind', 'jav', 'mad', 'mak', 'min', 'mui', 'rej', 'sun'] | BitextMining | s2s | [Social, Written] | {'train': 50200} | {'train': {'num_samples': 50200, 'number_of_characters': 14759870, 'unique_pairs': 50140, 'min_sentence1_length': 5, 'average_sentence1_length': 145.46, 'max_sentence1_length': 873, 'unique_sentence1': 8258, 'min_sentence2_length': 5, 'average_sentence2_length': 148.57, 'max_sentence2_length': 980, 'unique_sentence2': 50102, 'hf_subset_descriptive_stats': {'ind-abs': {'num_samples': 1000, 'number_of_characters': 295680, 'unique_pairs': 999, 'min_sentence1_length': 5, 'average_sentence1_length': 148.37, 'max_sentence1_length': 727, 'unique_sentence1': 998, 'min_sentence2_length': 6, 'average_sentence2_length': 147.31, 'max_sentence2_length': 629, 'unique_sentence2': 998}, 'ind-btk': {'num_samples': 6600, 'number_of_characters': 1927907, 'unique_pairs': 6597, 'min_sentence1_length': 5, 'average_sentence1_length': 145.37, 'max_sentence1_length': 873, 'unique_sentence1': 6521, 'min_sentence2_length': 5, 'average_sentence2_length': 146.74, 'max_sentence2_length': 980, 'unique_sentence2': 6596}, 'ind-bew': {'num_samples': 6600, 'number_of_characters': 1939300, 'unique_pairs': 6595, 'min_sentence1_length': 5, 'average_sentence1_length': 145.43, 'max_sentence1_length': 873, 'unique_sentence1': 6512, 'min_sentence2_length': 6, 'average_sentence2_length': 148.41, 'max_sentence2_length': 840, 'unique_sentence2': 6590}, 'ind-bhp': {'num_samples': 1000, 'number_of_characters': 261666, 'unique_pairs': 1000, 'min_sentence1_length': 11, 'average_sentence1_length': 133.53, 'max_sentence1_length': 468, 'unique_sentence1': 999, 'min_sentence2_length': 10, 'average_sentence2_length': 128.14, 'max_sentence2_length': 459, 'unique_sentence2': 999}, 'ind-jav': {'num_samples': 6600, 'number_of_characters': 1922162, 'unique_pairs': 6594, 'min_sentence1_length': 5, 'average_sentence1_length': 145.43, 'max_sentence1_length': 873, 'unique_sentence1': 6512, 'min_sentence2_length': 5, 'average_sentence2_length': 145.81, 'max_sentence2_length': 854, 'unique_sentence2': 6585}, 'ind-mad': {'num_samples': 6600, 'number_of_characters': 1973257, 'unique_pairs': 6598, 'min_sentence1_length': 5, 'average_sentence1_length': 145.36, 'max_sentence1_length': 873, 'unique_sentence1': 6521, 'min_sentence2_length': 5, 'average_sentence2_length': 153.62, 'max_sentence2_length': 827, 'unique_sentence2': 6592}, 'ind-mak': {'num_samples': 6600, 'number_of_characters': 1953868, 'unique_pairs': 6594, 'min_sentence1_length': 5, 'average_sentence1_length': 145.43, 'max_sentence1_length': 873, 'unique_sentence1': 6512, 'min_sentence2_length': 6, 'average_sentence2_length': 150.61, 'max_sentence2_length': 888, 'unique_sentence2': 6586}, 'ind-min': {'num_samples': 6600, 'number_of_characters': 1937033, 'unique_pairs': 6595, 'min_sentence1_length': 5, 'average_sentence1_length': 145.43, 'max_sentence1_length': 873, 'unique_sentence1': 6512, 'min_sentence2_length': 6, 'average_sentence2_length': 148.06, 'max_sentence2_length': 837, 'unique_sentence2': 6591}, 'ind-mui': {'num_samples': 1000, 'number_of_characters': 301448, 'unique_pairs': 1000, 'min_sentence1_length': 11, 'average_sentence1_length': 150.45, 'max_sentence1_length': 451, 'unique_sentence1': 997, 'min_sentence2_length': 11, 'average_sentence2_length': 150.99, 'max_sentence2_length': 450, 'unique_sentence2': 1000}, 'ind-rej': {'num_samples': 1000, 'number_of_characters': 291205, 'unique_pairs': 1000, 'min_sentence1_length': 9, 'average_sentence1_length': 151.62, 'max_sentence1_length': 873, 'unique_sentence1': 998, 'min_sentence2_length': 8, 'average_sentence2_length': 139.58, 'max_sentence2_length': 784, 'unique_sentence2': 1000}, 'ind-sun': {'num_samples': 6600, 'number_of_characters': 1956344, 'unique_pairs': 6591, 'min_sentence1_length': 5, 'average_sentence1_length': 145.43, 'max_sentence1_length': 873, 'unique_sentence1': 6512, 'min_sentence2_length': 5, 'average_sentence2_length': 150.99, 'max_sentence2_length': 881, 'unique_sentence2': 6588}}}} | +| [NusaX-senti](https://arxiv.org/abs/2205.15960) (Winata et al., 2022) | ['ace', 'ban', 'bbc', 'bjn', 'bug', 'eng', 'ind', 'jav', 'mad', 'min', 'nij', 'sun'] | Classification | s2s | [Reviews, Web, Social, Constructed, Written] | None | None | +| [NusaXBitextMining](https://huggingface.co/datasets/indonlp/NusaX-senti/) (Winata et al., 2023) | ['ace', 'ban', 'bbc', 'bjn', 'bug', 'eng', 'ind', 'jav', 'mad', 'min', 'nij', 'sun'] | BitextMining | s2s | [Reviews, Written] | None | None | +| [OPP115DataRetentionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [OPP115DataSecurityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [OPP115DoNotTrackLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [OPP115FirstPartyCollectionUseLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [OPP115InternationalAndSpecificAudiencesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [OPP115PolicyChangeLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [OPP115ThirdPartySharingCollectionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [OPP115UserAccessEditAndDeletionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [OPP115UserChoiceControlLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +>>>>>>> main | [Ocnli](https://arxiv.org/abs/2010.05444) (Hai Hu, 2020) | ['cmn'] | PairClassification | s2s | | None | None | -| [OdiaNewsClassification](https://github.com/goru001/nlp-for-odia) (Anoop Kunchukuttan, 2020) | ['ory'] | Classification | s2s | [News, Written] | {'test': 2048} | {'test': 49.24} | +| [OdiaNewsClassification](https://github.com/goru001/nlp-for-odia) (Anoop Kunchukuttan, 2020) | ['ory'] | Classification | s2s | [News, Written] | None | None | | [OnlineShopping](https://aclanthology.org/2023.nodalida-1.20/) (Xiao et al., 2023) | ['cmn'] | Classification | s2s | | None | None | -| [OnlineStoreReviewSentimentClassification](https://huggingface.co/datasets/Ruqiya/Arabic_Reviews_of_SHEIN) | ['ara'] | Classification | s2s | [Reviews, Written] | {'train': 2048} | {'train': 137.2} | -| [OpusparcusPC](https://gem-benchmark.com/data_cards/opusparcus) (Mathias Creutz, 2018) | ['deu', 'eng', 'fin', 'fra', 'rus', 'swe'] | PairClassification | s2s | [Spoken, Spoken] | {'validation': 10168, 'test': 10210} | {'validation': 24.4, 'test': 23.8} | -| [OralArgumentQuestionPurposeLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 312} | {'test': 269.71} | -| [OverrulingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 2048} | {'test': 167.2} | -| [PAC](https://arxiv.org/pdf/2211.13112.pdf) (Łukasz Augustyniak, 2022) | ['pol'] | Classification | p2p | [Legal, Written] | {'test': 3453} | {'test': 185.3} | +| [OnlineStoreReviewSentimentClassification](https://huggingface.co/datasets/Ruqiya/Arabic_Reviews_of_SHEIN) | ['ara'] | Classification | s2s | [Reviews, Written] | None | None | +| [OpusparcusPC](https://gem-benchmark.com/data_cards/opusparcus) (Mathias Creutz, 2018) | ['deu', 'eng', 'fin', 'fra', 'rus', 'swe'] | PairClassification | s2s | [Spoken, Spoken] | None | None | +| [OralArgumentQuestionPurposeLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [OverrulingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [PAC](https://arxiv.org/pdf/2211.13112.pdf) (Łukasz Augustyniak, 2022) | ['pol'] | Classification | p2p | [Legal, Written] | None | None | | [PAWSX](https://aclanthology.org/2021.emnlp-main.357) (Shitao Xiao, 2024) | ['cmn'] | STS | s2s | | None | None | -| [PIQA](https://arxiv.org/abs/1911.11641) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 1838} | {'test': {'average_document_length': 99.89012998705756, 'average_query_length': 36.08052230685528, 'num_documents': 35542, 'num_queries': 1838, 'average_relevant_docs_per_query': 1.0}} | -| [PROALegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 95} | {'test': 251.73} | +| [PIQA](https://arxiv.org/abs/1911.11641) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [PROALegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [PSC](http://www.lrec-conf.org/proceedings/lrec2014/pdf/1211_Paper.pdf) | ['pol'] | PairClassification | s2s | [News, Written] | None | None | -| [PatentClassification](https://aclanthology.org/P19-1212.pdf) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 5000} | {'test': 18620.44} | -| [PawsXPairClassification](https://arxiv.org/abs/1908.11828) (Yinfei Yang, 2019) | ['cmn', 'deu', 'eng', 'fra', 'jpn', 'kor', 'spa'] | PairClassification | s2s | [Web, Encyclopaedic, Written] | {'validation': 14000, 'test': 14000} | {'test': {'num_samples': 14000, 'avg_sentence1_len': 91.17892857142857, 'avg_sentence2_len': 91.10121428571429, 'unique_labels': 2, 'labels': {'1': {'count': 6285}, '0': {'count': 7715}}, 'hf_subset_descriptive_stats': {'de': {'num_samples': 2000, 'avg_sentence1_len': 119.7815, 'avg_sentence2_len': 119.2355, 'unique_labels': 2, 'labels': {'1': {'count': 895}, '0': {'count': 1105}}}, 'en': {'num_samples': 2000, 'avg_sentence1_len': 113.7575, 'avg_sentence2_len': 113.4235, 'unique_labels': 2, 'labels': {'1': {'count': 907}, '0': {'count': 1093}}}, 'es': {'num_samples': 2000, 'avg_sentence1_len': 117.815, 'avg_sentence2_len': 117.798, 'unique_labels': 2, 'labels': {'1': {'count': 907}, '0': {'count': 1093}}}, 'fr': {'num_samples': 2000, 'avg_sentence1_len': 120.028, 'avg_sentence2_len': 119.9885, 'unique_labels': 2, 'labels': {'1': {'count': 903}, '0': {'count': 1097}}}, 'ja': {'num_samples': 2000, 'avg_sentence1_len': 58.678, 'avg_sentence2_len': 58.875, 'unique_labels': 2, 'labels': {'1': {'count': 883}, '0': {'count': 1117}}}, 'ko': {'num_samples': 2000, 'avg_sentence1_len': 64.9605, 'avg_sentence2_len': 65.114, 'unique_labels': 2, 'labels': {'1': {'count': 896}, '0': {'count': 1104}}}, 'zh': {'num_samples': 2000, 'avg_sentence1_len': 43.232, 'avg_sentence2_len': 43.274, 'unique_labels': 2, 'labels': {'1': {'count': 894}, '0': {'count': 1106}}}}}, 'validation': {'num_samples': 14000, 'avg_sentence1_len': 90.12585714285714, 'avg_sentence2_len': 90.2045, 'unique_labels': 2, 'labels': {'1': {'count': 5948}, '0': {'count': 8052}}, 'hf_subset_descriptive_stats': {'de': {'num_samples': 2000, 'avg_sentence1_len': 116.82, 'avg_sentence2_len': 117.0015, 'unique_labels': 2, 'labels': {'1': {'count': 831}, '0': {'count': 1169}}}, 'en': {'num_samples': 2000, 'avg_sentence1_len': 113.1075, 'avg_sentence2_len': 112.858, 'unique_labels': 2, 'labels': {'1': {'count': 863}, '0': {'count': 1137}}}, 'es': {'num_samples': 2000, 'avg_sentence1_len': 116.3285, 'avg_sentence2_len': 116.7275, 'unique_labels': 2, 'labels': {'1': {'count': 847}, '0': {'count': 1153}}}, 'fr': {'num_samples': 2000, 'avg_sentence1_len': 119.5045, 'avg_sentence2_len': 119.7505, 'unique_labels': 2, 'labels': {'1': {'count': 860}, '0': {'count': 1140}}}, 'ja': {'num_samples': 2000, 'avg_sentence1_len': 57.5105, 'avg_sentence2_len': 57.317, 'unique_labels': 2, 'labels': {'1': {'count': 854}, '0': {'count': 1146}}}, 'ko': {'num_samples': 2000, 'avg_sentence1_len': 65.162, 'avg_sentence2_len': 65.5155, 'unique_labels': 2, 'labels': {'1': {'count': 840}, '0': {'count': 1160}}}, 'zh': {'num_samples': 2000, 'avg_sentence1_len': 42.448, 'avg_sentence2_len': 42.2615, 'unique_labels': 2, 'labels': {'1': {'count': 853}, '0': {'count': 1147}}}}}} | -| [PersianFoodSentimentClassification](https://hooshvare.github.io/docs/datasets/sa) (Mehrdad Farahani et al., 2020) | ['fas'] | Classification | s2s | [Reviews, Written] | {'validation': 2048, 'test': 2048} | {'validation': 90.37, 'test': 90.58} | -| [PersonalJurisdictionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 50} | {'test': 381.14} | -| [PhincBitextMining](https://huggingface.co/datasets/veezbo/phinc) (Srivastava et al., 2020) | ['eng', 'hin'] | BitextMining | s2s | [Social, Written] | {'train': 13738} | {'train': 75.32} | -| [PlscClusteringP2P.v2](https://huggingface.co/datasets/rafalposwiata/plsc) | ['pol'] | Clustering | s2s | [Academic, Written] | {'test': 2048} | {'test': 1023.21} | -| [PlscClusteringS2S.v2](https://huggingface.co/datasets/rafalposwiata/plsc) | ['pol'] | Clustering | s2s | [Academic, Written] | {'test': 2048} | {'test': 84.34} | -| [PoemSentimentClassification](https://arxiv.org/abs/2011.02686) (Emily Sheng, 2020) | ['eng'] | Classification | s2s | [Reviews, Written] | {'validation': 105, 'test': 104} | {'validation': 45.3, 'test': 42.4} | +| [PatentClassification](https://aclanthology.org/P19-1212.pdf) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [PawsXPairClassification](https://arxiv.org/abs/1908.11828) (Yinfei Yang, 2019) | ['cmn', 'deu', 'eng', 'fra', 'jpn', 'kor', 'spa'] | PairClassification | s2s | [Web, Encyclopaedic, Written] | {'test': 14000, 'validation': 14000} | {'test': {'num_samples': 14000, 'number_of_characters': 2551922, 'min_sentence1_length': 2, 'avg_sentence1_length': 91.18, 'max_sentence1_length': 268, 'unique_sentence1': 13404, 'min_sentence2_length': 2, 'avg_sentence2_length': 91.1, 'max_sentence2_length': 247, 'unique_sentence2': 13462, 'unique_labels': 2, 'labels': {'1': {'count': 6285}, '0': {'count': 7715}}, 'hf_subset_descriptive_stats': {'de': {'num_samples': 2000, 'number_of_characters': 478034, 'min_sentence1_length': 2, 'avg_sentence1_length': 119.78, 'max_sentence1_length': 268, 'unique_sentence1': 1934, 'min_sentence2_length': 2, 'avg_sentence2_length': 119.24, 'max_sentence2_length': 235, 'unique_sentence2': 1938, 'unique_labels': 2, 'labels': {'1': {'count': 895}, '0': {'count': 1105}}}, 'en': {'num_samples': 2000, 'number_of_characters': 454362, 'min_sentence1_length': 25, 'avg_sentence1_length': 113.76, 'max_sentence1_length': 209, 'unique_sentence1': 1761, 'min_sentence2_length': 25, 'avg_sentence2_length': 113.42, 'max_sentence2_length': 209, 'unique_sentence2': 1800, 'unique_labels': 2, 'labels': {'1': {'count': 907}, '0': {'count': 1093}}}, 'es': {'num_samples': 2000, 'number_of_characters': 471226, 'min_sentence1_length': 2, 'avg_sentence1_length': 117.81, 'max_sentence1_length': 226, 'unique_sentence1': 1955, 'min_sentence2_length': 22, 'avg_sentence2_length': 117.8, 'max_sentence2_length': 233, 'unique_sentence2': 1959, 'unique_labels': 2, 'labels': {'1': {'count': 907}, '0': {'count': 1093}}}, 'fr': {'num_samples': 2000, 'number_of_characters': 480033, 'min_sentence1_length': 2, 'avg_sentence1_length': 120.03, 'max_sentence1_length': 238, 'unique_sentence1': 1954, 'min_sentence2_length': 2, 'avg_sentence2_length': 119.99, 'max_sentence2_length': 247, 'unique_sentence2': 1953, 'unique_labels': 2, 'labels': {'1': {'count': 903}, '0': {'count': 1097}}}, 'ja': {'num_samples': 2000, 'number_of_characters': 235106, 'min_sentence1_length': 2, 'avg_sentence1_length': 58.68, 'max_sentence1_length': 192, 'unique_sentence1': 1944, 'min_sentence2_length': 2, 'avg_sentence2_length': 58.88, 'max_sentence2_length': 198, 'unique_sentence2': 1941, 'unique_labels': 2, 'labels': {'1': {'count': 883}, '0': {'count': 1117}}}, 'ko': {'num_samples': 2000, 'number_of_characters': 260149, 'min_sentence1_length': 2, 'avg_sentence1_length': 64.96, 'max_sentence1_length': 153, 'unique_sentence1': 1954, 'min_sentence2_length': 2, 'avg_sentence2_length': 65.11, 'max_sentence2_length': 159, 'unique_sentence2': 1969, 'unique_labels': 2, 'labels': {'1': {'count': 896}, '0': {'count': 1104}}}, 'zh': {'num_samples': 2000, 'number_of_characters': 173012, 'min_sentence1_length': 2, 'avg_sentence1_length': 43.23, 'max_sentence1_length': 120, 'unique_sentence1': 1909, 'min_sentence2_length': 2, 'avg_sentence2_length': 43.27, 'max_sentence2_length': 113, 'unique_sentence2': 1909, 'unique_labels': 2, 'labels': {'1': {'count': 894}, '0': {'count': 1106}}}}}, 'validation': {'num_samples': 14000, 'number_of_characters': 2524625, 'min_sentence1_length': 2, 'avg_sentence1_length': 90.13, 'max_sentence1_length': 248, 'unique_sentence1': 13357, 'min_sentence2_length': 2, 'avg_sentence2_length': 90.2, 'max_sentence2_length': 275, 'unique_sentence2': 13397, 'unique_labels': 2, 'labels': {'1': {'count': 5948}, '0': {'count': 8052}}, 'hf_subset_descriptive_stats': {'de': {'num_samples': 2000, 'number_of_characters': 467643, 'min_sentence1_length': 2, 'avg_sentence1_length': 116.82, 'max_sentence1_length': 248, 'unique_sentence1': 1914, 'min_sentence2_length': 2, 'avg_sentence2_length': 117.0, 'max_sentence2_length': 275, 'unique_sentence2': 1920, 'unique_labels': 2, 'labels': {'1': {'count': 831}, '0': {'count': 1169}}}, 'en': {'num_samples': 2000, 'number_of_characters': 451931, 'min_sentence1_length': 25, 'avg_sentence1_length': 113.11, 'max_sentence1_length': 213, 'unique_sentence1': 1758, 'min_sentence2_length': 25, 'avg_sentence2_length': 112.86, 'max_sentence2_length': 213, 'unique_sentence2': 1771, 'unique_labels': 2, 'labels': {'1': {'count': 863}, '0': {'count': 1137}}}, 'es': {'num_samples': 2000, 'number_of_characters': 466112, 'min_sentence1_length': 2, 'avg_sentence1_length': 116.33, 'max_sentence1_length': 240, 'unique_sentence1': 1938, 'min_sentence2_length': 2, 'avg_sentence2_length': 116.73, 'max_sentence2_length': 241, 'unique_sentence2': 1941, 'unique_labels': 2, 'labels': {'1': {'count': 847}, '0': {'count': 1153}}}, 'fr': {'num_samples': 2000, 'number_of_characters': 478510, 'min_sentence1_length': 2, 'avg_sentence1_length': 119.5, 'max_sentence1_length': 233, 'unique_sentence1': 1933, 'min_sentence2_length': 2, 'avg_sentence2_length': 119.75, 'max_sentence2_length': 246, 'unique_sentence2': 1939, 'unique_labels': 2, 'labels': {'1': {'count': 860}, '0': {'count': 1140}}}, 'ja': {'num_samples': 2000, 'number_of_characters': 229655, 'min_sentence1_length': 2, 'avg_sentence1_length': 57.51, 'max_sentence1_length': 126, 'unique_sentence1': 1957, 'min_sentence2_length': 2, 'avg_sentence2_length': 57.32, 'max_sentence2_length': 121, 'unique_sentence2': 1969, 'unique_labels': 2, 'labels': {'1': {'count': 854}, '0': {'count': 1146}}}, 'ko': {'num_samples': 2000, 'number_of_characters': 261355, 'min_sentence1_length': 2, 'avg_sentence1_length': 65.16, 'max_sentence1_length': 178, 'unique_sentence1': 1963, 'min_sentence2_length': 2, 'avg_sentence2_length': 65.52, 'max_sentence2_length': 174, 'unique_sentence2': 1968, 'unique_labels': 2, 'labels': {'1': {'count': 840}, '0': {'count': 1160}}}, 'zh': {'num_samples': 2000, 'number_of_characters': 169419, 'min_sentence1_length': 2, 'avg_sentence1_length': 42.45, 'max_sentence1_length': 101, 'unique_sentence1': 1899, 'min_sentence2_length': 2, 'avg_sentence2_length': 42.26, 'max_sentence2_length': 120, 'unique_sentence2': 1895, 'unique_labels': 2, 'labels': {'1': {'count': 853}, '0': {'count': 1147}}}}}} | +| [PersianFoodSentimentClassification](https://hooshvare.github.io/docs/datasets/sa) (Mehrdad Farahani et al., 2020) | ['fas'] | Classification | s2s | [Reviews, Written] | None | None | +| [PersonalJurisdictionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [PhincBitextMining](https://huggingface.co/datasets/veezbo/phinc) (Srivastava et al., 2020) | ['eng', 'hin'] | BitextMining | s2s | [Social, Written] | {'train': 13738} | {'train': {'num_samples': 13738, 'number_of_characters': 2069457, 'unique_pairs': 13737, 'min_sentence1_length': 1, 'average_sentence1_length': 74.02, 'max_sentence1_length': 278, 'unique_sentence1': 13515, 'min_sentence2_length': 3, 'average_sentence2_length': 76.61, 'max_sentence2_length': 274, 'unique_sentence2': 13736, 'hf_subset_descriptive_stats': {'eng-eng_hin': {'num_samples': 13738, 'number_of_characters': 2069457, 'unique_pairs': 13737, 'min_sentence1_length': 1, 'average_sentence1_length': 74.02, 'max_sentence1_length': 278, 'unique_sentence1': 13515, 'min_sentence2_length': 3, 'average_sentence2_length': 76.61, 'max_sentence2_length': 274, 'unique_sentence2': 13736}}}} | +| [PlscClusteringP2P.v2](https://huggingface.co/datasets/rafalposwiata/plsc) | ['pol'] | Clustering | s2s | [Academic, Written] | None | None | +| [PlscClusteringS2S.v2](https://huggingface.co/datasets/rafalposwiata/plsc) | ['pol'] | Clustering | s2s | [Academic, Written] | None | None | +| [PoemSentimentClassification](https://arxiv.org/abs/2011.02686) (Emily Sheng, 2020) | ['eng'] | Classification | s2s | [Reviews, Written] | None | None | | [PolEmo2.0-IN](https://aclanthology.org/K19-1092.pdf) | ['pol'] | Classification | s2s | [Written, Social] | None | None | -| [PolEmo2.0-OUT](https://aclanthology.org/K19-1092.pdf) | ['pol'] | Classification | s2s | [Written, Social] | {'test': 722} | {'test': 756.2} | +| [PolEmo2.0-OUT](https://aclanthology.org/K19-1092.pdf) | ['pol'] | Classification | s2s | [Written, Social] | None | None | | [PpcPC](https://arxiv.org/pdf/2207.12759.pdf) (Sławomir Dadas, 2022) | ['pol'] | PairClassification | s2s | [Fiction, Non-fiction, Web, Written, Spoken, Social, News] | None | None | -| [PublicHealthQA](https://huggingface.co/datasets/xhluca/publichealth-qa) | ['ara', 'eng', 'fra', 'kor', 'rus', 'spa', 'vie', 'zho'] | Retrieval | s2p | [Medical, Government, Web, Written] | {'test': 888} | {'test': {'arabic': {'average_document_length': 836.8850574712644, 'average_query_length': 79.84883720930233, 'num_documents': 87, 'num_queries': 87, 'average_relevant_docs_per_query': 1.0}, 'chinese': {'average_document_length': 239.58282208588957, 'average_query_length': 24.828220858895705, 'num_documents': 163, 'num_queries': 163, 'average_relevant_docs_per_query': 1.0}, 'english': {'average_document_length': 799.3430232558139, 'average_query_length': 71.78488372093024, 'num_documents': 172, 'num_queries': 172, 'average_relevant_docs_per_query': 1.0}, 'french': {'average_document_length': 1021.6823529411764, 'average_query_length': 101.88235294117646, 'num_documents': 85, 'num_queries': 85, 'average_relevant_docs_per_query': 1.0}, 'korean': {'average_document_length': 339.0, 'average_query_length': 36.90909090909091, 'num_documents': 77, 'num_queries': 77, 'average_relevant_docs_per_query': 1.0}, 'russian': {'average_document_length': 985.1076923076923, 'average_query_length': 85.2, 'num_documents': 65, 'num_queries': 65, 'average_relevant_docs_per_query': 1.0}, 'spanish': {'average_document_length': 941.1666666666666, 'average_query_length': 84.67901234567901, 'num_documents': 162, 'num_queries': 162, 'average_relevant_docs_per_query': 1.0}, 'vietnamese': {'average_document_length': 704.5454545454545, 'average_query_length': 71.83116883116882, 'num_documents': 77, 'num_queries': 77, 'average_relevant_docs_per_query': 1.0}}} | -| [PunjabiNewsClassification](https://github.com/goru001/nlp-for-punjabi/) (Anoop Kunchukuttan, 2020) | ['pan'] | Classification | s2s | [News, Written] | {'train': 627, 'test': 157} | {'train': 4222.22, 'test': 4115.14} | +| [PublicHealthQA](https://huggingface.co/datasets/xhluca/publichealth-qa) | ['ara', 'eng', 'fra', 'kor', 'rus', 'spa', 'vie', 'zho'] | Retrieval | s2p | [Medical, Government, Web, Written] | None | None | +| [PunjabiNewsClassification](https://github.com/goru001/nlp-for-punjabi/) (Anoop Kunchukuttan, 2020) | ['pan'] | Classification | s2s | [News, Written] | None | None | | [QBQTC](https://github.com/CLUEbenchmark/QBQTC/tree/main/dataset) | ['cmn'] | STS | s2s | | None | None | +<<<<<<< HEAD | [Quail](https://text-machine.cs.uml.edu/lab2/projects/quail/) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 2720} | {'test': {'average_document_length': 27.50788422240522, 'average_query_length': 1957.3632352941177, 'num_documents': 32787, 'num_queries': 2720, 'average_relevant_docs_per_query': 1.0}} | | [Quora-PL](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2s | | None | {'validation': {'average_document_length': 65.82473022253414, 'average_query_length': 54.6006, 'num_documents': 522931, 'num_queries': 5000, 'average_relevant_docs_per_query': 1.5252}, 'test': {'average_document_length': 65.82473022253414, 'average_query_length': 54.5354, 'num_documents': 522931, 'num_queries': 10000, 'average_relevant_docs_per_query': 1.5675}} | | [Quora-PLHardNegatives](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2s | | {'test': 1000} | {'test': {'average_document_length': 67.77529631287385, 'average_query_length': 53.846, 'num_documents': 172031, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.641}} | @@ -456,10 +805,57 @@ The following tables give you an overview of the tasks in MTEB. | [SIB200ClusteringS2S](https://arxiv.org/abs/2309.07445) (Adelani et al., 2023) | ['ace', 'acm', 'acq', 'aeb', 'afr', 'ajp', 'aka', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'ast', 'awa', 'ayr', 'azb', 'azj', 'bak', 'bam', 'ban', 'bel', 'bem', 'ben', 'bho', 'bjn', 'bod', 'bos', 'bug', 'bul', 'cat', 'ceb', 'ces', 'cjk', 'ckb', 'crh', 'cym', 'dan', 'deu', 'dik', 'dyu', 'dzo', 'ell', 'eng', 'epo', 'est', 'eus', 'ewe', 'fao', 'fij', 'fin', 'fon', 'fra', 'fur', 'fuv', 'gaz', 'gla', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hne', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kac', 'kam', 'kan', 'kas', 'kat', 'kaz', 'kbp', 'kea', 'khk', 'khm', 'kik', 'kin', 'kir', 'kmb', 'kmr', 'knc', 'kon', 'kor', 'lao', 'lij', 'lim', 'lin', 'lit', 'lmo', 'ltg', 'ltz', 'lua', 'lug', 'luo', 'lus', 'lvs', 'mag', 'mai', 'mal', 'mar', 'min', 'mkd', 'mlt', 'mni', 'mos', 'mri', 'mya', 'nld', 'nno', 'nob', 'npi', 'nqo', 'nso', 'nus', 'nya', 'oci', 'ory', 'pag', 'pan', 'pap', 'pbt', 'pes', 'plt', 'pol', 'por', 'prs', 'quy', 'ron', 'run', 'rus', 'sag', 'san', 'sat', 'scn', 'shn', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'ssw', 'sun', 'swe', 'swh', 'szl', 'tam', 'taq', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tpi', 'tsn', 'tso', 'tuk', 'tum', 'tur', 'twi', 'tzm', 'uig', 'ukr', 'umb', 'urd', 'uzn', 'vec', 'vie', 'war', 'wol', 'xho', 'ydd', 'yor', 'yue', 'zho', 'zsm', 'zul'] | Clustering | s2s | [News, Written] | {'test': 1004} | {'test': 114.78} | | [SICK-BR-PC](https://linux.ime.usp.br/~thalen/SICK_PT.pdf) | ['por'] | PairClassification | s2s | [Web, Written] | {'test': 1000} | {'test': 54.89} | | [SICK-BR-STS](https://linux.ime.usp.br/~thalen/SICK_PT.pdf) | ['por'] | STS | s2s | [Web, Written] | {'test': 1000} | {'test': 54.89} | +======= +| [Quail](https://text-machine.cs.uml.edu/lab2/projects/quail/) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [Quora-PL](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2s | | None | None | +| [Quora-PLHardNegatives](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2s | | None | None | +| [QuoraRetrieval](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (DataCanary et al., 2017) | ['eng'] | Retrieval | s2s | | None | None | +| [QuoraRetrievalHardNegatives](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (DataCanary et al., 2017) | ['eng'] | Retrieval | s2s | | None | None | +| [RARbCode](https://arxiv.org/abs/2404.06347) (Xiao et al., 2024) | ['eng'] | Retrieval | s2p | [Programming, Written] | None | None | +| [RARbMath](https://arxiv.org/abs/2404.06347) (Xiao et al., 2024) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [RTE3](https://aclanthology.org/W07-1401/) | ['deu', 'eng', 'fra', 'ita'] | PairClassification | s2s | [News, Web, Encyclopaedic, Written] | None | None | +| [RUParaPhraserSTS](https://aclanthology.org/2020.ngt-1.6) (Pivovarova et al., 2017) | ['rus'] | STS | s2s | [News, Written] | None | None | +| [RedditClustering.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | s2s | [Web, Social, Written] | None | None | +| [RedditClusteringP2P.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | p2p | [Web, Social, Written] | {'test': 459389} | {'test': {'num_samples': 459389, 'number_of_characters': 334286895, 'min_text_length': 79, 'average_text_length': 727.68, 'max_text_length': 4359, 'min_labels_per_text': 2, 'average_labels_per_text': 1.0, 'max_labels_per_text': 77908, 'unique_labels': 440, 'labels': {'FortNiteBR': {'count': 436}, 'buildapc': {'count': 8484}, 'offmychest': {'count': 570}, 'nus': {'count': 45}, 'relationship_advice': {'count': 16651}, 'premed': {'count': 201}, 'dogecoin': {'count': 8108}, 'GamingLaptops': {'count': 183}, 'asktransgender': {'count': 326}, 'MachineLearning': {'count': 61}, 'puppy101': {'count': 1597}, 'GunAccessoriesForSale': {'count': 2619}, 'Random_Acts_Of_Amazon': {'count': 1115}, 'Catholicism': {'count': 183}, 'MonsterHunter': {'count': 218}, 'tipofmypenis': {'count': 87}, 'samsung': {'count': 69}, 'PersonalFinanceCanada': {'count': 341}, 'Dyson_Sphere_Program': {'count': 55}, 'bleach': {'count': 41}, 'AmItheAsshole': {'count': 3730}, 'WallStreetbetsELITE': {'count': 328}, 'GlobalPowers': {'count': 35}, 'ABraThatFits': {'count': 159}, 'PokemonGoFriends': {'count': 1165}, 'NoMansSkyTheGame': {'count': 259}, 'masseffect': {'count': 233}, 'dating_advice': {'count': 559}, 'yoga': {'count': 50}, 'depression': {'count': 515}, 'COVID19positive': {'count': 180}, 'generationology': {'count': 37}, 'feedthebeast': {'count': 192}, 'EliteDangerous': {'count': 270}, 'alcoholicsanonymous': {'count': 93}, 'GoRVing': {'count': 35}, 'thedivision': {'count': 111}, 'breakingmom': {'count': 105}, 'AskAnAmerican': {'count': 80}, 'HypnoFair': {'count': 5}, 'JustUnsubbed': {'count': 13}, 'socialanxiety': {'count': 123}, 'dirtykikpals': {'count': 202}, 'askTO': {'count': 126}, 'AskCulinary': {'count': 108}, 'Bogleheads': {'count': 71}, 'dragonquest': {'count': 45}, 'NoContract': {'count': 30}, 'gorillaz': {'count': 14}, 'MondoGore': {'count': 8}, 'comicswap': {'count': 56}, 'VirtualYoutubers': {'count': 92}, 'Gta5Modding': {'count': 28}, 'obs': {'count': 61}, 'vcu': {'count': 9}, 'KingkillerChronicle': {'count': 17}, 'AmongUs': {'count': 41}, 'wireshark': {'count': 3}, 'Dodocodes': {'count': 46}, 'Aliexpress': {'count': 40}, 'LearnerDriverUK': {'count': 12}, 'PanicAttack': {'count': 23}, 'KassadinMains': {'count': 10}, 'islam': {'count': 93}, 'chronotrigger': {'count': 4}, 'skincareexchange': {'count': 13}, 'PokemonHome': {'count': 21}, 'survivinginfidelity': {'count': 71}, 'igcse': {'count': 21}, 'C25K': {'count': 21}, 'aorus': {'count': 2}, 'idleon': {'count': 19}, 'photography': {'count': 22}, 'cryptocoins': {'count': 7}, 'CanaryWharfBets': {'count': 7}, 'KillingEve': {'count': 7}, 'GameBuilderGarage': {'count': 16}, 'SauceSharingCommunity': {'count': 7}, 'turo': {'count': 9}, 'foodscience': {'count': 14}, 'HIMYM': {'count': 20}, 'HauntingOfHillHouse': {'count': 4}, 'GoodNotes': {'count': 8}, 'RedditWritesSeinfeld': {'count': 6}, 'AirReps': {'count': 2}, 'ADHD': {'count': 3811}, 'BuddyCrossing': {'count': 446}, 'libraryofruina': {'count': 98}, 'SluttyConfessions': {'count': 2787}, 'tipofmytongue': {'count': 7145}, 'fleshlight': {'count': 128}, 'amcstock': {'count': 13910}, 'teenagers': {'count': 77908}, 'suggestmeabook': {'count': 1540}, 'dirtypenpals': {'count': 5587}, 'MinecraftServer': {'count': 177}, 'CreditCards': {'count': 669}, 'Guitar': {'count': 10952}, 'rpg': {'count': 529}, 'NoFap': {'count': 14853}, 'lfg': {'count': 1093}, 'MarsWallStreet': {'count': 935}, 'SummonSign': {'count': 931}, 'AssassinsCreedValhala': {'count': 295}, 'hoi4': {'count': 432}, 'Coins4Sale': {'count': 260}, 'xbox': {'count': 459}, 'TooAfraidToAsk': {'count': 7404}, 'NBA2k': {'count': 553}, 'KGBTR': {'count': 943}, 'roblox': {'count': 220}, 'salesforce': {'count': 214}, 'TwoXChromosomes': {'count': 1736}, 'mechmarket': {'count': 4863}, 'Gaming_Headsets': {'count': 103}, 'pittsburgh': {'count': 189}, 'CryptoMars': {'count': 1606}, 'FridayNightFunkin': {'count': 378}, 'vaginismus': {'count': 122}, 'transpositive': {'count': 10}, 'comicbooks': {'count': 274}, 'BDSMcommunity': {'count': 185}, 'aliens': {'count': 201}, 'Scotch': {'count': 64}, 'KikRoleplay': {'count': 141}, 'Kayaking': {'count': 91}, '196': {'count': 47}, 'digimon': {'count': 140}, 'Evernote': {'count': 42}, 'logh': {'count': 22}, 'arlington': {'count': 15}, 'Adopted': {'count': 8}, 'DissonautUniverse': {'count': 4}, 'Midsommar': {'count': 12}, 'SofiawithanF': {'count': 83}, 'xmpp': {'count': 6}, 'ZombsRoyale': {'count': 16}, 'accesscontrol': {'count': 8}, 'WetlanderHumor': {'count': 2}, 'PoonamPandeyFanatics': {'count': 2}, 'screenplaychallenge': {'count': 2}, 'scatstories': {'count': 2}, 'techsupport': {'count': 290}, 'whatcarshouldIbuy': {'count': 79}, 'Stormlight_Archive': {'count': 15}, 'deadbydaylight': {'count': 126}, 'bicycling': {'count': 27}, 'oculus': {'count': 64}, 'Cartalk': {'count': 33}, 'Sims4': {'count': 43}, 'NoFeeAC': {'count': 95}, 'Crypto_com': {'count': 37}, 'ITCareerQuestions': {'count': 259}, 'aromantic': {'count': 18}, 'Revu': {'count': 3}, 'exalted': {'count': 2}, 'HilariaBaldwin': {'count': 20}, 'Testosterone': {'count': 35}, 'Screenwriting': {'count': 170}, 'LifeProTips': {'count': 49}, 'steinsgate': {'count': 13}, 'Baystreetbets': {'count': 10}, 'AskGirls': {'count': 7}, 'idlechampions': {'count': 7}, 'facebook': {'count': 17}, 'tf2trade': {'count': 4}, 'mfdoom': {'count': 3}, 'FiddlesticksMains': {'count': 2}, 'HFY': {'count': 10}, 'FiestaST': {'count': 2}, 'whatsthatbook': {'count': 994}, 'GearsOfWar': {'count': 879}, 'KazuhaMains': {'count': 175}, 'RepTime': {'count': 211}, 'AstroGaming': {'count': 141}, 'metalgearsolid': {'count': 152}, 'qBittorrent': {'count': 39}, 'ELLIPAL_Official': {'count': 24}, 'raisedbynarcissists': {'count': 4895}, 'unpopularopinion': {'count': 14901}, 'ACTrade': {'count': 5679}, 'askcarsales': {'count': 1339}, 'AskVet': {'count': 1357}, 'whowouldwin': {'count': 4493}, 'playstation': {'count': 1362}, 'anime': {'count': 6531}, 'GME': {'count': 12577}, 'DotA2': {'count': 2004}, 'cryptostreetbets': {'count': 2241}, 'MonsterHunterWorld': {'count': 698}, 'Market76': {'count': 14274}, 'DnD': {'count': 5092}, 'leagueoflegends': {'count': 3683}, 'doordash_drivers': {'count': 1626}, 'theta_network': {'count': 489}, 'exmuslim': {'count': 1369}, 'gonewildaudio': {'count': 2998}, 'conspiracy': {'count': 3587}, 'heroesofthestorm': {'count': 535}, 'FanFiction': {'count': 2782}, 'Doom': {'count': 1251}, 'texas': {'count': 269}, 'Vent': {'count': 1738}, 'selfimprovement': {'count': 1284}, 'youtubers': {'count': 706}, 'askseddit': {'count': 237}, 'boardgames': {'count': 1237}, 'bravelydefault': {'count': 347}, 'ConquerorsBlade': {'count': 238}, 'ChronicPain': {'count': 527}, 'teenagersnew': {'count': 256}, 'brasil': {'count': 1092}, 'MatthiasSubmissions': {'count': 921}, 'MarylandUnemployment': {'count': 314}, 'SaltLakeCity': {'count': 411}, 'BokunoheroFanfiction': {'count': 155}, 'BenignExistence': {'count': 125}, 'GayYoungOldDating': {'count': 156}, 'Bible': {'count': 202}, 'haskell': {'count': 154}, 'seduction': {'count': 400}, 'fantasywriters': {'count': 262}, 'HiveOS': {'count': 100}, 'PerkByDaylight': {'count': 15}, 'Hedgehog': {'count': 73}, 'xmen': {'count': 263}, 'HyperRP': {'count': 122}, 'emotestories': {'count': 3}, 'tutanota': {'count': 135}, 'CultoftheFranklin': {'count': 46}, 'langrisser': {'count': 62}, 'CozyGrove': {'count': 61}, 'Sverigesforsvarsmakt': {'count': 12}, 'silverbugbets': {'count': 21}, 'WreckingBallMains': {'count': 5}, 'capitalism_in_decay': {'count': 8}, 'paintdotnet': {'count': 11}, 'u_mawadom118': {'count': 4}, 'xboxfindfriends': {'count': 2}, 'CPTSD': {'count': 540}, 'destiny2': {'count': 318}, 'Wallstreetsilver': {'count': 1013}, 'DestinyTheGame': {'count': 1107}, 'blackopscoldwar': {'count': 400}, 'InstacartShoppers': {'count': 202}, 'RocketLeagueExchange': {'count': 832}, 'apexlegends': {'count': 3265}, 'kansascity': {'count': 53}, 'namenerds': {'count': 235}, 'help': {'count': 152}, 'Kengan_Ashura': {'count': 132}, 'thetagang': {'count': 165}, 'GameSale': {'count': 262}, 'Reduction': {'count': 109}, 'sex': {'count': 906}, 'bostonr4r': {'count': 75}, 'LegendsOfRuneterra': {'count': 231}, 'overlord': {'count': 48}, 'madisonwi': {'count': 53}, 'steelseries': {'count': 79}, 'ClashOfClansRecruit': {'count': 214}, 'CharacterRant': {'count': 55}, 'AirForce': {'count': 94}, 'sexstories': {'count': 92}, 'NameThatSong': {'count': 162}, 'depressed': {'count': 74}, 'ibs': {'count': 150}, '40kLore': {'count': 269}, 'podcasts': {'count': 88}, 'miraculousladybug': {'count': 150}, 'ask': {'count': 224}, 'EverMerge': {'count': 31}, 'TMJ': {'count': 54}, 'BitLifeApp': {'count': 39}, 'FireEmblemHeroes': {'count': 100}, 'software': {'count': 62}, 'ShieldAndroidTV': {'count': 70}, 'GriefSupport': {'count': 125}, 'onewheel': {'count': 37}, 'MensRights': {'count': 80}, 'nhl': {'count': 22}, 'ClashOfClans': {'count': 107}, 'ps3homebrew': {'count': 33}, 'LightNovels': {'count': 77}, 'redsox': {'count': 34}, 'CryptoMarkets': {'count': 44}, 'ugly': {'count': 47}, 'GCXRep': {'count': 12}, 'cscareerquestionsEU': {'count': 65}, 'MindHunter': {'count': 6}, 'starcraft2coop': {'count': 15}, 'nanocurrency': {'count': 1421}, 'ModelCars': {'count': 8}, 'UKJobs': {'count': 30}, 'Netherlands': {'count': 44}, 'clonewars': {'count': 8}, 'Julia': {'count': 11}, 'Prolactinoma': {'count': 9}, 'sofi': {'count': 11}, 'royalfamily': {'count': 6}, 'ConnecticutR4R': {'count': 8}, 'weather': {'count': 5}, 'oneui': {'count': 7}, 'KTM': {'count': 5}, 'Aerials': {'count': 3}, 'seoul': {'count': 2}, 'exjw': {'count': 3281}, 'ModernMagic': {'count': 699}, 'Paladins': {'count': 1242}, 'kdramarecommends': {'count': 1611}, 'hitbtc': {'count': 330}, 'endocrinology': {'count': 75}, 'Bath': {'count': 43}, 'NassauCountyHookups': {'count': 5}, 'feminineboys': {'count': 1248}, 'dreamsmp': {'count': 2018}, 'SquaredCircle': {'count': 2255}, 'Minecraft': {'count': 8753}, 'spirituality': {'count': 1809}, 'Eldenring': {'count': 1471}, 'Sat': {'count': 1172}, 'bonnaroo': {'count': 194}, 'gardening': {'count': 1892}, 'Unemployment': {'count': 6185}, 'mac': {'count': 1847}, 'Bestbuy': {'count': 437}, 'quittingkratom': {'count': 1081}, 'lawschooladmissions': {'count': 3436}, 'NiceHash': {'count': 2135}, 'McMaster': {'count': 815}, 'covidlonghaulers': {'count': 1299}, 'stalker': {'count': 758}, 'MLBTheShow': {'count': 2721}, 'FortniteCompetitive': {'count': 998}, 'dpdr': {'count': 514}, 'appliancerepair': {'count': 720}, 'thomasthetankengine': {'count': 207}, 'delhi': {'count': 217}, 'Huel': {'count': 300}, 'leafs': {'count': 203}, 'HotWheels': {'count': 170}, '90dayfianceuncensored': {'count': 550}, 'Throwers': {'count': 142}, 'Wavyhair': {'count': 270}, 'CryptoHorde': {'count': 128}, 'ShuumatsuNoValkyrie': {'count': 453}, 'TeensMeetTeens': {'count': 432}, 'dbrand': {'count': 108}, 'SLFmeetups': {'count': 18}, '1200isplentyketo': {'count': 48}, 'passive_income': {'count': 211}, 'BroadCity': {'count': 16}, 'RevenantMain': {'count': 71}, 'extrarfl': {'count': 25}, 'AgonGame': {'count': 5}, 'FitnessDE': {'count': 3}, 'gaming': {'count': 1277}, 'livesound': {'count': 91}, 'IBO': {'count': 1896}, 'EscapefromTarkov': {'count': 1300}, 'amex': {'count': 145}, 'DMAcademy': {'count': 1411}, 'VinylCollectors': {'count': 556}, 'cardano': {'count': 716}, 'brave_browser': {'count': 159}, 'dating': {'count': 952}, 'OculusQuest': {'count': 942}, 'Superstonk': {'count': 3089}, 'MtF': {'count': 957}, 'findaleague': {'count': 207}, 'Nioh': {'count': 398}, 'IRS': {'count': 715}, 'transgendercirclejerk': {'count': 353}, 'learnmath': {'count': 489}, 'piano': {'count': 263}, 'LeagueConnect': {'count': 216}, 'eu4': {'count': 561}, 'Wordpress': {'count': 345}, 'RoleplayingForReddit': {'count': 31}, 'LOONA': {'count': 89}, 'newtothenavy': {'count': 167}, 'HaircareScience': {'count': 118}, 'appletv': {'count': 167}, 'sissypersonals': {'count': 102}, 'raleigh': {'count': 168}, 'realonlyfansreviews': {'count': 21}, 'AskGames': {'count': 49}, 'PokemonTCG': {'count': 325}, 'controlgame': {'count': 109}, 'GoogleDataStudio': {'count': 16}, 'WhiteWolfRPG': {'count': 139}, 'MECoOp': {'count': 31}, 'snuffrp': {'count': 46}, 'lockpicking': {'count': 103}, 'wicked_edge': {'count': 105}, 'BMW': {'count': 99}, 'choiceofgames': {'count': 24}, 'hisdarkmaterials': {'count': 12}, 'SakuraGakuin': {'count': 24}, 'detrans': {'count': 55}, 'Smallville': {'count': 37}, 'kingofqueens': {'count': 7}, 'JamesHoffmann': {'count': 22}, 'stashinvest': {'count': 16}, 'ABA': {'count': 79}, 'ladybusiness': {'count': 10}, 'gamegrumps': {'count': 32}, 'GodEater': {'count': 21}, 'tomorrow': {'count': 39}, 'Tomorrowland': {'count': 9}, 'BlackCountryNewRoad': {'count': 5}, 'STAYC': {'count': 3}, 'SatoshiStreetBets': {'count': 3828}, 'AskLosAngeles': {'count': 1036}, 'buildapcforme': {'count': 1689}, 'ApplyingToCollege': {'count': 10675}, 'watercooling': {'count': 1209}, 'BreakUps': {'count': 4914}, 'FIFA': {'count': 3811}, 'emacs': {'count': 712}, 'trakstocks': {'count': 691}, 'Shittyaskflying': {'count': 147}, 'AmazonFC': {'count': 1178}, 'stocks': {'count': 4610}, 'BangaloreMains': {'count': 26}, 'pokemon': {'count': 3953}, 'religion': {'count': 684}, 'cuboulder': {'count': 269}, 'self': {'count': 1688}, 'tarot': {'count': 912}, 'turtles': {'count': 49}, 'TheMagnusArchives': {'count': 300}, 'Superhero_Ideas': {'count': 34}, 'NTU': {'count': 308}, 'touhou': {'count': 623}, 'JoJolion': {'count': 50}, 'lasers': {'count': 27}, 'popperpigs': {'count': 67}, 'aggretsuko': {'count': 20}, 'Library': {'count': 5}}}} | +| [RestaurantReviewSentimentClassification](https://link.springer.com/chapter/10.1007/978-3-319-18117-2_2) (ElSahar et al., 2015) | ['ara'] | Classification | s2s | [Reviews, Written] | None | None | +| [RiaNewsRetrieval](https://arxiv.org/abs/1901.07786) (Gavrilov et al., 2019) | ['rus'] | Retrieval | s2p | [News, Written] | None | None | +| [RiaNewsRetrievalHardNegatives](https://arxiv.org/abs/1901.07786) (Gavrilov et al., 2019) | ['rus'] | Retrieval | s2p | [News, Written] | None | None | +| [Robust04InstructionRetrieval](https://arxiv.org/abs/2403.15246) (Orion Weller, 2024) | ['eng'] | InstructionRetrieval | s2p | [News, Written] | None | None | +| [RomaTalesBitextMining](https://idoc.pub/documents/idocpub-zpnxm9g35ylv) | ['hun', 'rom'] | BitextMining | s2s | [Fiction, Written] | None | None | +| [RomaniBibleClustering](https://romani.global.bible/info) | ['rom'] | Clustering | p2p | [Religious, Written] | None | None | +| [RomanianReviewsSentiment](https://arxiv.org/abs/2101.04197) (Anca Maria Tache, 2021) | ['ron'] | Classification | s2s | [Reviews, Written] | None | None | +| [RomanianSentimentClassification](https://arxiv.org/abs/2009.08712) (Dumitrescu et al., 2020) | ['ron'] | Classification | s2s | [Reviews, Written] | None | None | +| [RonSTS](https://openreview.net/forum?id=JH61CD7afTv) (Dumitrescu et al., 2021) | ['ron'] | STS | s2s | [News, Social, Web, Written] | None | None | +| [RuBQReranking](https://openreview.net/pdf?id=P5UQFFoQ4PJ) (Ivan Rybin, 2021) | ['rus'] | Reranking | s2p | [Encyclopaedic, Written] | None | None | +| [RuBQRetrieval](https://openreview.net/pdf?id=P5UQFFoQ4PJ) (Ivan Rybin, 2021) | ['rus'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [RuReviewsClassification](https://github.com/sismetanin/rureviews) (Sergey Smetanin, 2019) | ['rus'] | Classification | p2p | [Reviews, Written] | None | None | +| [RuSTSBenchmarkSTS](https://github.com/PhilipMay/stsb-multi-mt/) (Philip May, 2021) | ['rus'] | STS | s2s | [News, Social, Web, Written] | None | None | +| [RuSciBenchGRNTIClassification](https://github.com/mlsa-iai-msu-lab/ru_sci_bench/) | ['rus'] | Classification | p2p | [Academic, Written] | None | None | +| [RuSciBenchGRNTIClusteringP2P](https://github.com/mlsa-iai-msu-lab/ru_sci_bench/) | ['rus'] | Clustering | p2p | [Academic, Written] | {'test': 2048} | {'test': {'num_samples': 2048, 'number_of_characters': 1822339, 'min_text_length': 84, 'average_text_length': 889.81, 'max_text_length': 3143, 'min_labels_per_text': 73, 'average_labels_per_text': 1.0, 'max_labels_per_text': 74, 'unique_labels': 28, 'labels': {'3': {'count': 73}, '4': {'count': 73}, '20': {'count': 73}, '9': {'count': 73}, '21': {'count': 73}, '15': {'count': 73}, '16': {'count': 74}, '2': {'count': 73}, '8': {'count': 73}, '23': {'count': 73}, '6': {'count': 73}, '24': {'count': 73}, '10': {'count': 73}, '1': {'count': 73}, '17': {'count': 74}, '14': {'count': 74}, '18': {'count': 73}, '27': {'count': 73}, '19': {'count': 73}, '22': {'count': 73}, '12': {'count': 73}, '25': {'count': 73}, '5': {'count': 74}, '0': {'count': 73}, '26': {'count': 73}, '11': {'count': 73}, '13': {'count': 73}, '7': {'count': 73}}}} | +| [RuSciBenchOECDClassification](https://github.com/mlsa-iai-msu-lab/ru_sci_bench/) | ['rus'] | Classification | p2p | [Academic, Written] | None | None | +| [RuSciBenchOECDClusteringP2P](https://github.com/mlsa-iai-msu-lab/ru_sci_bench/) | ['rus'] | Clustering | p2p | [Academic, Written] | None | None | +| [SCDBPAccountabilityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [SCDBPAuditsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [SCDBPCertificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [SCDBPTrainingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [SCDBPVerificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [SCDDAccountabilityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [SCDDAuditsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [SCDDCertificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [SCDDTrainingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [SCDDVerificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [SCIDOCS](https://allenai.org/data/scidocs) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | [Academic, Written, Non-fiction] | None | None | +| [SCIDOCS-PL](https://allenai.org/data/scidocs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | None | +| [SIB200Classification](https://arxiv.org/abs/2309.07445) (Adelani et al., 2023) | ['ace', 'acm', 'acq', 'aeb', 'afr', 'ajp', 'aka', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'ast', 'awa', 'ayr', 'azb', 'azj', 'bak', 'bam', 'ban', 'bel', 'bem', 'ben', 'bho', 'bjn', 'bod', 'bos', 'bug', 'bul', 'cat', 'ceb', 'ces', 'cjk', 'ckb', 'crh', 'cym', 'dan', 'deu', 'dik', 'dyu', 'dzo', 'ell', 'eng', 'epo', 'est', 'eus', 'ewe', 'fao', 'fij', 'fin', 'fon', 'fra', 'fur', 'fuv', 'gaz', 'gla', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hne', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kac', 'kam', 'kan', 'kas', 'kat', 'kaz', 'kbp', 'kea', 'khk', 'khm', 'kik', 'kin', 'kir', 'kmb', 'kmr', 'knc', 'kon', 'kor', 'lao', 'lij', 'lim', 'lin', 'lit', 'lmo', 'ltg', 'ltz', 'lua', 'lug', 'luo', 'lus', 'lvs', 'mag', 'mai', 'mal', 'mar', 'min', 'mkd', 'mlt', 'mni', 'mos', 'mri', 'mya', 'nld', 'nno', 'nob', 'npi', 'nqo', 'nso', 'nus', 'nya', 'oci', 'ory', 'pag', 'pan', 'pap', 'pbt', 'pes', 'plt', 'pol', 'por', 'prs', 'quy', 'ron', 'run', 'rus', 'sag', 'san', 'sat', 'scn', 'shn', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'ssw', 'sun', 'swe', 'swh', 'szl', 'tam', 'taq', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tpi', 'tsn', 'tso', 'tuk', 'tum', 'tur', 'twi', 'tzm', 'uig', 'ukr', 'umb', 'urd', 'uzn', 'vec', 'vie', 'war', 'wol', 'xho', 'ydd', 'yor', 'yue', 'zho', 'zsm', 'zul'] | Classification | s2s | [News, Written] | None | None | +| [SIB200ClusteringS2S](https://arxiv.org/abs/2309.07445) (Adelani et al., 2023) | ['ace', 'acm', 'acq', 'aeb', 'afr', 'ajp', 'aka', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'ast', 'awa', 'ayr', 'azb', 'azj', 'bak', 'bam', 'ban', 'bel', 'bem', 'ben', 'bho', 'bjn', 'bod', 'bos', 'bug', 'bul', 'cat', 'ceb', 'ces', 'cjk', 'ckb', 'crh', 'cym', 'dan', 'deu', 'dik', 'dyu', 'dzo', 'ell', 'eng', 'epo', 'est', 'eus', 'ewe', 'fao', 'fij', 'fin', 'fon', 'fra', 'fur', 'fuv', 'gaz', 'gla', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hne', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kac', 'kam', 'kan', 'kas', 'kat', 'kaz', 'kbp', 'kea', 'khk', 'khm', 'kik', 'kin', 'kir', 'kmb', 'kmr', 'knc', 'kon', 'kor', 'lao', 'lij', 'lim', 'lin', 'lit', 'lmo', 'ltg', 'ltz', 'lua', 'lug', 'luo', 'lus', 'lvs', 'mag', 'mai', 'mal', 'mar', 'min', 'mkd', 'mlt', 'mni', 'mos', 'mri', 'mya', 'nld', 'nno', 'nob', 'npi', 'nqo', 'nso', 'nus', 'nya', 'oci', 'ory', 'pag', 'pan', 'pap', 'pbt', 'pes', 'plt', 'pol', 'por', 'prs', 'quy', 'ron', 'run', 'rus', 'sag', 'san', 'sat', 'scn', 'shn', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'ssw', 'sun', 'swe', 'swh', 'szl', 'tam', 'taq', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tpi', 'tsn', 'tso', 'tuk', 'tum', 'tur', 'twi', 'tzm', 'uig', 'ukr', 'umb', 'urd', 'uzn', 'vec', 'vie', 'war', 'wol', 'xho', 'ydd', 'yor', 'yue', 'zho', 'zsm', 'zul'] | Clustering | s2s | [News, Written] | None | None | +| [SICK-BR-PC](https://linux.ime.usp.br/~thalen/SICK_PT.pdf) | ['por'] | PairClassification | s2s | [Web, Written] | None | None | +| [SICK-BR-STS](https://linux.ime.usp.br/~thalen/SICK_PT.pdf) | ['por'] | STS | s2s | [Web, Written] | None | None | +>>>>>>> main | [SICK-E-PL](https://aclanthology.org/2020.lrec-1.207) | ['pol'] | PairClassification | s2s | | None | None | -| [SICK-R](https://aclanthology.org/2020.lrec-1.207) | ['eng'] | STS | s2s | | None | None | -| [SICK-R-PL](https://aclanthology.org/2020.lrec-1.207) | ['pol'] | STS | s2s | [Web, Written] | {'test': 9812} | {'test': 42.8} | +| [SICK-R](https://aclanthology.org/L14-1314/) | ['eng'] | STS | s2s | [Web, Written] | None | None | +| [SICK-R-PL](https://aclanthology.org/2020.lrec-1.207) | ['pol'] | STS | s2s | [Web, Written] | None | None | | [SICKFr](https://huggingface.co/datasets/Lajavaness/SICK-fr) | ['fra'] | STS | s2s | | None | None | +<<<<<<< HEAD | [SIQA](https://leaderboard.allenai.org/socialiqa/submissions/get-started) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 0} | {'test': {'average_document_length': 22.967085695044617, 'average_query_length': 127.75383828045035, 'num_documents': 71276, 'num_queries': 1954, 'average_relevant_docs_per_query': 1.0}} | | [SKQuadRetrieval](https://huggingface.co/datasets/TUKE-KEMT/retrieval-skquad) | ['slk'] | Retrieval | s2s | [Encyclopaedic] | {'test': 1134} | {'test': {'average_document_length': 1180.5071792496526, 'average_query_length': 53.63403880070547, 'num_documents': 6477, 'num_queries': 1134, 'average_relevant_docs_per_query': 11}} | | [SNLHierarchicalClusteringP2P](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | p2p | [Encyclopaedic, Non-fiction, Written] | {'test': 1300} | {'test': 1986.9453846153847} | @@ -473,10 +869,26 @@ The following tables give you an overview of the tasks in MTEB. | [STS16](https://www.aclweb.org/anthology/S16-1001) | ['eng'] | STS | s2s | [Blog, Web, Spoken] | {'test': 2372} | {'test': 65.3} | | [STS17](https://alt.qcri.org/semeval2017/task1/) | ['ara', 'deu', 'eng', 'fra', 'ita', 'kor', 'nld', 'spa', 'tur'] | STS | s2s | [News, Web, Written] | {'test': 500} | {'test': {'num_samples': 5346, 'average_sentence1_len': 38.14665170220726, 'average_sentence2_len': 36.72502805836139, 'avg_score': 2.3554804214989464, 'hf_subset_descriptive_stats': {'ko-ko': {'num_samples': 2846, 'average_sentence1_len': 31.991918482080113, 'average_sentence2_len': 32.44483485593816, 'avg_score': 2.469359920356055}, 'ar-ar': {'num_samples': 250, 'average_sentence1_len': 32.208, 'average_sentence2_len': 32.78, 'avg_score': 2.216800000000001}, 'en-ar': {'num_samples': 250, 'average_sentence1_len': 42.36, 'average_sentence2_len': 32.696, 'avg_score': 2.1423999999999994}, 'en-de': {'num_samples': 250, 'average_sentence1_len': 43.952, 'average_sentence2_len': 44.756, 'avg_score': 2.2776000000000014}, 'en-en': {'num_samples': 250, 'average_sentence1_len': 43.952, 'average_sentence2_len': 42.724, 'avg_score': 2.2776000000000014}, 'en-tr': {'num_samples': 250, 'average_sentence1_len': 41.916, 'average_sentence2_len': 41.6, 'avg_score': 2.1335999999999986}, 'es-en': {'num_samples': 250, 'average_sentence1_len': 50.84, 'average_sentence2_len': 42.024, 'avg_score': 2.1464000000000003}, 'es-es': {'num_samples': 250, 'average_sentence1_len': 49.836, 'average_sentence2_len': 51.224, 'avg_score': 2.2312000000000007}, 'fr-en': {'num_samples': 250, 'average_sentence1_len': 49.624, 'average_sentence2_len': 42.724, 'avg_score': 2.2776000000000014}, 'it-en': {'num_samples': 250, 'average_sentence1_len': 50.028, 'average_sentence2_len': 42.724, 'avg_score': 2.2776000000000014}, 'nl-en': {'num_samples': 250, 'average_sentence1_len': 46.816, 'average_sentence2_len': 42.724, 'avg_score': 2.2776000000000014}}}} | | [STS22.v2](https://competitions.codalab.org/competitions/33835) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'ita', 'pol', 'rus', 'spa', 'tur'] | STS | p2p | [News, Written] | {'test': 3958} | {'test': 1993.6} | +======= +| [SIQA](https://leaderboard.allenai.org/socialiqa/submissions/get-started) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [SKQuadRetrieval](https://huggingface.co/datasets/TUKE-KEMT/retrieval-skquad) | ['slk'] | Retrieval | s2s | [Encyclopaedic] | None | None | +| [SNLHierarchicalClusteringP2P](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | p2p | [Encyclopaedic, Non-fiction, Written] | None | None | +| [SNLHierarchicalClusteringS2S](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | s2s | [Encyclopaedic, Non-fiction, Written] | None | None | +| [SNLRetrieval](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Written] | None | None | +| [SRNCorpusBitextMining](https://arxiv.org/abs/2212.06383) (Zwennicker et al., 2022) | ['nld', 'srn'] | BitextMining | s2s | [Social, Web, Written] | None | None | +| [STS12](https://www.aclweb.org/anthology/S12-1051.pdf) (Agirre et al., 2012) | ['eng'] | STS | s2s | [Encyclopaedic, News, Written] | {'test': 3108} | {'test': {'num_samples': 3108, 'number_of_characters': 402118, 'min_sentence1_length': 3, 'average_sentence1_len': 63.79, 'max_sentence1_length': 220, 'unique_sentence1': 2236, 'min_sentence2_length': 7, 'average_sentence2_len': 65.59, 'max_sentence2_length': 204, 'unique_sentence2': 2797, 'min_score': 0.0, 'avg_score': 3.51, 'max_score': 5.0}} | +| [STS13](https://www.aclweb.org/anthology/S13-1004/) (Eneko Agirre, 2013) | ['eng'] | STS | s2s | [Web, News, Non-fiction, Written] | None | None | +| [STS14](https://www.aclweb.org/anthology/S14-1002) | ['eng'] | STS | s2s | [Blog, Web, Spoken] | None | None | +| [STS15](https://www.aclweb.org/anthology/S15-2010) | ['eng'] | STS | s2s | [Blog, News, Web, Written, Spoken] | None | None | +| [STS16](https://www.aclweb.org/anthology/S16-1001) | ['eng'] | STS | s2s | [Blog, Web, Spoken] | None | None | +| [STS17](https://alt.qcri.org/semeval2017/task1/) | ['ara', 'deu', 'eng', 'fra', 'ita', 'kor', 'nld', 'spa', 'tur'] | STS | s2s | [News, Web, Written] | {'test': 5346} | {'test': {'num_samples': 5346, 'number_of_characters': 400264, 'min_sentence1_length': 6, 'average_sentence1_len': 38.15, 'max_sentence1_length': 976, 'unique_sentence1': 4900, 'min_sentence2_length': 6, 'average_sentence2_len': 36.73, 'max_sentence2_length': 1007, 'unique_sentence2': 4470, 'min_score': 0.0, 'avg_score': 2.36, 'max_score': 5.0, 'hf_subset_descriptive_stats': {'ko-ko': {'num_samples': 2846, 'number_of_characters': 183387, 'min_sentence1_length': 6, 'average_sentence1_len': 31.99, 'max_sentence1_length': 976, 'unique_sentence1': 2650, 'min_sentence2_length': 6, 'average_sentence2_len': 32.44, 'max_sentence2_length': 1007, 'unique_sentence2': 2720, 'min_score': 0.0, 'avg_score': 2.47, 'max_score': 5.0}, 'ar-ar': {'num_samples': 250, 'number_of_characters': 16247, 'min_sentence1_length': 11, 'average_sentence1_len': 32.21, 'max_sentence1_length': 99, 'unique_sentence1': 250, 'min_sentence2_length': 9, 'average_sentence2_len': 32.78, 'max_sentence2_length': 83, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.22, 'max_score': 5.0}, 'en-ar': {'num_samples': 250, 'number_of_characters': 18764, 'min_sentence1_length': 13, 'average_sentence1_len': 42.36, 'max_sentence1_length': 105, 'unique_sentence1': 250, 'min_sentence2_length': 10, 'average_sentence2_len': 32.7, 'max_sentence2_length': 104, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.14, 'max_score': 5.0}, 'en-de': {'num_samples': 250, 'number_of_characters': 22177, 'min_sentence1_length': 12, 'average_sentence1_len': 43.95, 'max_sentence1_length': 94, 'unique_sentence1': 250, 'min_sentence2_length': 15, 'average_sentence2_len': 44.76, 'max_sentence2_length': 104, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.28, 'max_score': 5.0}, 'en-en': {'num_samples': 250, 'number_of_characters': 21669, 'min_sentence1_length': 12, 'average_sentence1_len': 43.95, 'max_sentence1_length': 94, 'unique_sentence1': 250, 'min_sentence2_length': 15, 'average_sentence2_len': 42.72, 'max_sentence2_length': 101, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.28, 'max_score': 5.0}, 'en-tr': {'num_samples': 250, 'number_of_characters': 20879, 'min_sentence1_length': 15, 'average_sentence1_len': 41.92, 'max_sentence1_length': 101, 'unique_sentence1': 250, 'min_sentence2_length': 10, 'average_sentence2_len': 41.6, 'max_sentence2_length': 107, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.13, 'max_score': 5.0}, 'es-en': {'num_samples': 250, 'number_of_characters': 23216, 'min_sentence1_length': 12, 'average_sentence1_len': 50.84, 'max_sentence1_length': 160, 'unique_sentence1': 250, 'min_sentence2_length': 14, 'average_sentence2_len': 42.02, 'max_sentence2_length': 117, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.15, 'max_score': 5.0}, 'es-es': {'num_samples': 250, 'number_of_characters': 25265, 'min_sentence1_length': 18, 'average_sentence1_len': 49.84, 'max_sentence1_length': 136, 'unique_sentence1': 250, 'min_sentence2_length': 13, 'average_sentence2_len': 51.22, 'max_sentence2_length': 129, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.23, 'max_score': 5.0}, 'fr-en': {'num_samples': 250, 'number_of_characters': 23087, 'min_sentence1_length': 19, 'average_sentence1_len': 49.62, 'max_sentence1_length': 115, 'unique_sentence1': 250, 'min_sentence2_length': 15, 'average_sentence2_len': 42.72, 'max_sentence2_length': 101, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.28, 'max_score': 5.0}, 'it-en': {'num_samples': 250, 'number_of_characters': 23188, 'min_sentence1_length': 15, 'average_sentence1_len': 50.03, 'max_sentence1_length': 113, 'unique_sentence1': 250, 'min_sentence2_length': 15, 'average_sentence2_len': 42.72, 'max_sentence2_length': 101, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.28, 'max_score': 5.0}, 'nl-en': {'num_samples': 250, 'number_of_characters': 22385, 'min_sentence1_length': 14, 'average_sentence1_len': 46.82, 'max_sentence1_length': 123, 'unique_sentence1': 250, 'min_sentence2_length': 15, 'average_sentence2_len': 42.72, 'max_sentence2_length': 101, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.28, 'max_score': 5.0}}}} | +| [STS22.v2](https://competitions.codalab.org/competitions/33835) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'ita', 'pol', 'rus', 'spa', 'tur'] | STS | p2p | [News, Written] | None | None | +>>>>>>> main | [STSB](https://aclanthology.org/2021.emnlp-main.357) (Shitao Xiao, 2024) | ['cmn'] | STS | s2s | | None | None | | [STSBenchmark](https://github.com/PhilipMay/stsb-multi-mt/) (Philip May, 2021) | ['eng'] | STS | s2s | | None | None | -| [STSBenchmarkMultilingualSTS](https://github.com/PhilipMay/stsb-multi-mt/) (Philip May, 2021) | ['cmn', 'deu', 'eng', 'fra', 'ita', 'nld', 'pol', 'por', 'rus', 'spa'] | STS | s2s | [News, Social, Web, Spoken, Written] | {'dev': 30000, 'test': 27580} | {'dev': 66.5, 'test': 56.1} | +| [STSBenchmarkMultilingualSTS](https://github.com/PhilipMay/stsb-multi-mt/) (Philip May, 2021) | ['cmn', 'deu', 'eng', 'fra', 'ita', 'nld', 'pol', 'por', 'rus', 'spa'] | STS | s2s | [News, Social, Web, Spoken, Written] | None | None | | [STSES](https://huggingface.co/datasets/PlanTL-GOB-ES/sts-es) (Agirre et al., 2015) | ['spa'] | STS | s2s | [Written] | None | None | +<<<<<<< HEAD | [SadeemQuestionRetrieval](https://huggingface.co/datasets/sadeem-ai/sadeem-ar-eval-retrieval-questions) | ['ara'] | Retrieval | s2p | [Written, Written] | {'test': 22979} | {'test': 500.0} | | [SanskritShlokasClassification](https://github.com/goru001/nlp-for-sanskrit) | ['san'] | Classification | s2s | [Religious, Written] | {'train': 383, 'validation': 96} | {'train': 98.415, 'validation': 96.635} | | [ScalaClassification](https://aclanthology.org/2023.nodalida-1.20/) | ['dan', 'nno', 'nob', 'swe'] | Classification | s2s | [Fiction, News, Non-fiction, Blog, Spoken, Web, Written] | {'test': 4096} | {'test': 102.72} | @@ -494,34 +906,54 @@ The following tables give you an overview of the tasks in MTEB. | [SlovakSumRetrieval](https://huggingface.co/datasets/NaiveNeuron/slovaksum) | ['slk'] | Retrieval | s2s | [News, Social, Web, Written] | {'test': 600} | {'test': {'average_document_length': 2156.445, 'average_query_length': 143.59833333333333, 'num_documents': 600, 'num_queries': 600, 'average_relevant_docs_per_query': 1.0}} | | [SouthAfricanLangClassification](https://www.kaggle.com/competitions/south-african-language-identification/) (ExploreAI Academy et al., 2022) | ['afr', 'eng', 'nbl', 'nso', 'sot', 'ssw', 'tsn', 'tso', 'ven', 'xho', 'zul'] | Classification | s2s | [Web, Non-fiction, Written] | {'test': 2048} | {'test': 247.49} | | [SpanishNewsClassification](https://huggingface.co/datasets/MarcOrfilaCarreras/spanish-news) | ['spa'] | Classification | s2s | [News, Written] | {'train': 2048} | {'train': 4218.2} | +======= +| [SadeemQuestionRetrieval](https://huggingface.co/datasets/sadeem-ai/sadeem-ar-eval-retrieval-questions) | ['ara'] | Retrieval | s2p | [Written, Written] | None | None | +| [SanskritShlokasClassification](https://github.com/goru001/nlp-for-sanskrit) | ['san'] | Classification | s2s | [Religious, Written] | None | None | +| [ScalaClassification](https://aclanthology.org/2023.nodalida-1.20/) | ['dan', 'nno', 'nob', 'swe'] | Classification | s2s | [Fiction, News, Non-fiction, Blog, Spoken, Web, Written] | None | None | +| [SciDocsRR](https://allenai.org/data/scidocs) | ['eng'] | Reranking | s2s | [Academic, Non-fiction, Written] | None | None | +| [SciFact](https://github.com/allenai/scifact) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | [Academic, Medical, Written] | None | None | +| [SciFact-PL](https://github.com/allenai/scifact) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Academic, Medical, Written] | None | None | +| [SemRel24STS](https://huggingface.co/datasets/SemRel/SemRel2024) (Nedjma Ousidhoum, 2024) | ['afr', 'amh', 'arb', 'arq', 'ary', 'eng', 'hau', 'hin', 'ind', 'kin', 'mar', 'tel'] | STS | s2s | [Spoken, Written] | None | None | +| [SensitiveTopicsClassification](https://aclanthology.org/2021.bsnlp-1.4) | ['rus'] | MultilabelClassification | s2s | [Web, Social, Written] | None | None | +| [SentimentAnalysisHindi](https://huggingface.co/datasets/OdiaGenAI/sentiment_analysis_hindi) (Shantipriya Parida, 2023) | ['hin'] | Classification | s2s | [Reviews, Written] | None | None | +| [SinhalaNewsClassification](https://huggingface.co/datasets/NLPC-UOM/Sinhala-News-Category-classification) (Nisansa de Silva, 2015) | ['sin'] | Classification | s2s | [News, Written] | None | None | +| [SinhalaNewsSourceClassification](https://huggingface.co/datasets/NLPC-UOM/Sinhala-News-Source-classification) (Dhananjaya et al., 2022) | ['sin'] | Classification | s2s | [News, Written] | None | None | +| [SiswatiNewsClassification](https://huggingface.co/datasets/dsfsi/za-isizulu-siswati-news) (Madodonga et al., 2023) | ['ssw'] | Classification | s2s | [News, Written] | None | None | +| [SlovakHateSpeechClassification](https://huggingface.co/datasets/TUKE-KEMT/hate_speech_slovak) | ['slk'] | Classification | s2s | [Social, Written] | {'test': 1319, 'train': 11870} | {'test': {'num_samples': 1319, 'number_of_characters': 122279, 'num_texts_in_train': 46, 'min_text_length': 8, 'average_text_length': 92.71, 'max_text_length': 1584, 'unique_text': 1315, 'unique_labels': 2, 'labels': {'1': {'count': 360}, '0': {'count': 959}}}, 'train': {'num_samples': 11870, 'number_of_characters': 1130860, 'num_texts_in_train': None, 'min_text_length': 7, 'average_text_length': 95.27, 'max_text_length': 2112, 'unique_text': 11655, 'unique_labels': 2, 'labels': {'1': {'count': 3245}, '0': {'count': 8625}}}} | +| [SlovakMovieReviewSentimentClassification](https://arxiv.org/pdf/2304.01922) ({ {S, 2023) | ['svk'] | Classification | s2s | [Reviews, Written] | None | None | +| [SlovakSumRetrieval](https://huggingface.co/datasets/NaiveNeuron/slovaksum) | ['slk'] | Retrieval | s2s | [News, Social, Web, Written] | None | None | +| [SouthAfricanLangClassification](https://www.kaggle.com/competitions/south-african-language-identification/) (ExploreAI Academy et al., 2022) | ['afr', 'eng', 'nbl', 'nso', 'sot', 'ssw', 'tsn', 'tso', 'ven', 'xho', 'zul'] | Classification | s2s | [Web, Non-fiction, Written] | None | None | +| [SpanishNewsClassification](https://huggingface.co/datasets/MarcOrfilaCarreras/spanish-news) | ['spa'] | Classification | s2s | [News, Written] | None | None | +>>>>>>> main | [SpanishNewsClusteringP2P](https://www.kaggle.com/datasets/kevinmorgado/spanish-news-classification) | ['spa'] | Clustering | p2p | | None | None | -| [SpanishPassageRetrievalS2P](https://mklab.iti.gr/results/spanish-passage-retrieval-dataset/) | ['spa'] | Retrieval | s2p | | None | {'test': {'average_document_length': 2635.217893792966, 'average_query_length': 67.55688622754491, 'num_documents': 10037, 'num_queries': 167, 'average_relevant_docs_per_query': 6.053892215568863}} | -| [SpanishPassageRetrievalS2S](https://mklab.iti.gr/results/spanish-passage-retrieval-dataset/) | ['spa'] | Retrieval | s2s | | None | {'test': {'average_document_length': 434.5924528301887, 'average_query_length': 67.55688622754491, 'num_documents': 265, 'num_queries': 167, 'average_relevant_docs_per_query': 7.718562874251497}} | -| [SpanishSentimentClassification](https://huggingface.co/datasets/sepidmnorozy/Spanish_sentiment) | ['spa'] | Classification | s2s | [Reviews, Written] | {'validation': 147, 'test': 296} | {'validation': 85.02, 'test': 87.91} | -| [SpartQA](https://github.com/HLR/SpartQA_generation) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 0} | {'test': {'average_document_length': 50.40829145728643, 'average_query_length': 656.2328881469115, 'num_documents': 1592, 'num_queries': 3594, 'average_relevant_docs_per_query': 1.8786867000556482}} | -| [SprintDuplicateQuestions](https://www.aclweb.org/anthology/D18-1131/) | ['eng'] | PairClassification | s2s | [Programming, Written] | {'validation': 101000, 'test': 101000} | {'validation': 65.2, 'test': 67.9} | -| [StackExchangeClustering.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | s2s | [Web, Written] | {'test': 32768} | {'test': 57.0} | -| [StackExchangeClusteringP2P.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | p2p | [Web, Written] | {'test': 2996} | {'test': 1090.7} | -| [StackOverflowDupQuestions](https://www.microsoft.com/en-us/research/uploads/prod/2019/03/nl4se18LinkSO.pdf) (Xueqing Liu, 2018) | ['eng'] | Reranking | s2s | | {'test': 3467} | {'test': 49.8} | -| [StackOverflowQA](https://arxiv.org/abs/2407.02883) (Xiangyang Li, 2024) | ['eng'] | Retrieval | p2p | [Programming, Written] | {'test': 1000} | {'test': {'average_document_length': 1202.4815613867845, 'average_query_length': 1302.6263791374122, 'num_documents': 19931, 'num_queries': 1994, 'average_relevant_docs_per_query': 1.0}} | -| [StatcanDialogueDatasetRetrieval](https://mcgill-nlp.github.io/statcan-dialogue-dataset/) | ['eng', 'fra'] | Retrieval | s2p | [Government, Web, Written] | {'dev': 1000, 'test': 1011, 'corpus': 5907} | {'dev': {'english': {'average_document_length': 6535.865413915693, 'average_query_length': 6.869244935543278, 'num_documents': 5907, 'num_queries': 543, 'average_relevant_docs_per_query': 1.4714548802946592}, 'french': {'average_document_length': 7078.072794988996, 'average_query_length': 6.860655737704918, 'num_documents': 5907, 'num_queries': 122, 'average_relevant_docs_per_query': 1.6475409836065573}}, 'test': {'english': {'average_document_length': 6535.865413915693, 'average_query_length': 7.650994575045208, 'num_documents': 5907, 'num_queries': 553, 'average_relevant_docs_per_query': 1.573236889692586}, 'french': {'average_document_length': 7078.072794988996, 'average_query_length': 5.907407407407407, 'num_documents': 5907, 'num_queries': 108, 'average_relevant_docs_per_query': 1.3055555555555556}}} | -| [SummEvalFrSummarization.v2](https://github.com/Yale-LILY/SummEval) (Fabbri et al., 2020) | ['fra'] | Summarization | p2p | [News, Written] | {'test': 2800} | {'test': 407.1} | -| [SummEvalSummarization.v2](https://github.com/Yale-LILY/SummEval) (Fabbri et al., 2020) | ['eng'] | Summarization | p2p | [News, Written] | {'test': 2800} | {'test': 359.8} | -| [SwahiliNewsClassification](https://huggingface.co/datasets/Mollel/SwahiliNewsClassification) | ['swa'] | Classification | s2s | [News, Written] | {'train': 2048} | {'train': 2438.2308135942326} | -| [SweFaqRetrieval](https://spraakbanken.gu.se/en/resources/superlim) (Berdi{ {c, 2023) | ['swe'] | Retrieval | s2s | [Government, Non-fiction, Written] | {'test': 1024} | {'test': {'average_document_length': 319.8473581213307, 'average_query_length': 70.51461988304094, 'num_documents': 511, 'num_queries': 513, 'average_relevant_docs_per_query': 1.0}} | -| [SweRecClassification](https://aclanthology.org/2023.nodalida-1.20/) | ['swe'] | Classification | s2s | [Reviews, Written] | {'test': 1024} | {'test': 318.8} | -| [SwedishSentimentClassification](https://huggingface.co/datasets/swedish_reviews) | ['swe'] | Classification | s2s | [Reviews, Written] | {'validation': 1024, 'test': 1024} | {'validation': 499.3, 'test': 498.1} | -| [SwednClusteringP2P](https://spraakbanken.gu.se/en/resources/swedn) (Monsen et al., 2021) | ['swe'] | Clustering | p2p | [News, Non-fiction, Written] | {'all': 2048} | {'all': 1619.71} | -| [SwednClusteringS2S](https://spraakbanken.gu.se/en/resources/swedn) (Monsen et al., 2021) | ['swe'] | Clustering | s2s | [News, Non-fiction, Written] | {'all': 2048} | {'all': 1619.71} | -| [SwednRetrieval](https://spraakbanken.gu.se/en/resources/swedn) (Monsen et al., 2021) | ['swe'] | Retrieval | p2p | [News, Non-fiction, Written] | {'test': 2048} | {'test': {'average_document_length': 2896.519550342131, 'average_query_length': 45.876953125, 'num_documents': 2046, 'num_queries': 1024, 'average_relevant_docs_per_query': 2.0}} | -| [SwissJudgementClassification](https://aclanthology.org/2021.nllp-1.3/) (Joel Niklaus, 2022) | ['deu', 'fra', 'ita'] | Classification | s2s | [Legal, Written] | {'test': 2048} | {'test': 3411.72} | +| [SpanishPassageRetrievalS2P](https://mklab.iti.gr/results/spanish-passage-retrieval-dataset/) | ['spa'] | Retrieval | s2p | | None | None | +| [SpanishPassageRetrievalS2S](https://mklab.iti.gr/results/spanish-passage-retrieval-dataset/) | ['spa'] | Retrieval | s2s | | None | None | +| [SpanishSentimentClassification](https://huggingface.co/datasets/sepidmnorozy/Spanish_sentiment) | ['spa'] | Classification | s2s | [Reviews, Written] | None | None | +| [SpartQA](https://github.com/HLR/SpartQA_generation) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [SprintDuplicateQuestions](https://www.aclweb.org/anthology/D18-1131/) | ['eng'] | PairClassification | s2s | [Programming, Written] | None | None | +| [StackExchangeClustering.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | s2s | [Web, Written] | None | None | +| [StackExchangeClusteringP2P.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | p2p | [Web, Written] | None | None | +| [StackOverflowDupQuestions](https://www.microsoft.com/en-us/research/uploads/prod/2019/03/nl4se18LinkSO.pdf) (Xueqing Liu, 2018) | ['eng'] | Reranking | s2s | | None | None | +| [StackOverflowQA](https://arxiv.org/abs/2407.02883) (Xiangyang Li, 2024) | ['eng'] | Retrieval | p2p | [Programming, Written] | {'test': 21925} | {'test': {'number_of_characters': 26584028, 'num_samples': 21925, 'num_queries': 1994, 'num_documents': 19931, 'min_document_length': 61, 'average_document_length': 130.32, 'max_document_length': 22234, 'unique_documents': 19931, 'min_query_length': 5, 'average_query_length': 12029.38, 'max_query_length': 46028, 'unique_queries': 1994, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1994}} | +| [StatcanDialogueDatasetRetrieval](https://mcgill-nlp.github.io/statcan-dialogue-dataset/) | ['eng', 'fra'] | Retrieval | s2p | [Government, Web, Written] | None | None | +| [SummEvalFrSummarization.v2](https://github.com/Yale-LILY/SummEval) (Fabbri et al., 2020) | ['fra'] | Summarization | p2p | [News, Written] | None | None | +| [SummEvalSummarization.v2](https://github.com/Yale-LILY/SummEval) (Fabbri et al., 2020) | ['eng'] | Summarization | p2p | [News, Written] | None | None | +| [SwahiliNewsClassification](https://huggingface.co/datasets/Mollel/SwahiliNewsClassification) | ['swa'] | Classification | s2s | [News, Written] | None | None | +| [SweFaqRetrieval](https://spraakbanken.gu.se/en/resources/superlim) (Berdi{ {c, 2023) | ['swe'] | Retrieval | s2s | [Government, Non-fiction, Written] | None | None | +| [SweRecClassification](https://aclanthology.org/2023.nodalida-1.20/) | ['swe'] | Classification | s2s | [Reviews, Written] | None | None | +| [SwedishSentimentClassification](https://huggingface.co/datasets/swedish_reviews) | ['swe'] | Classification | s2s | [Reviews, Written] | None | None | +| [SwednClusteringP2P](https://spraakbanken.gu.se/en/resources/swedn) (Monsen et al., 2021) | ['swe'] | Clustering | p2p | [News, Non-fiction, Written] | None | None | +| [SwednClusteringS2S](https://spraakbanken.gu.se/en/resources/swedn) (Monsen et al., 2021) | ['swe'] | Clustering | s2s | [News, Non-fiction, Written] | None | None | +| [SwednRetrieval](https://spraakbanken.gu.se/en/resources/swedn) (Monsen et al., 2021) | ['swe'] | Retrieval | p2p | [News, Non-fiction, Written] | None | None | +| [SwissJudgementClassification](https://aclanthology.org/2021.nllp-1.3/) (Joel Niklaus, 2022) | ['deu', 'fra', 'ita'] | Classification | s2s | [Legal, Written] | None | None | | [SyntecReranking](https://huggingface.co/datasets/lyon-nlp/mteb-fr-reranking-syntec-s2p) (Mathieu Ciancone, 2024) | ['fra'] | Reranking | s2p | [Legal, Written] | None | None | -| [SyntecRetrieval](https://huggingface.co/datasets/lyon-nlp/mteb-fr-retrieval-syntec-s2p) (Mathieu Ciancone, 2024) | ['fra'] | Retrieval | s2p | [Legal, Written] | {'test': 90} | {'test': {'average_document_length': 1224.2666666666667, 'average_query_length': 72.82, 'num_documents': 90, 'num_queries': 100, 'average_relevant_docs_per_query': 1.0}} | -| [SyntheticText2SQL](https://huggingface.co/datasets/gretelai/synthetic_text_to_sql) (Meyer et al., 2024) | ['eng', 'sql'] | Retrieval | p2p | [Programming, Written] | {'test': 1000} | {'test': {'average_document_length': 127.07126054548375, 'average_query_length': 82.90582806357888, 'num_documents': 105851, 'num_queries': 5851, 'average_relevant_docs_per_query': 1.0}} | +| [SyntecRetrieval](https://huggingface.co/datasets/lyon-nlp/mteb-fr-retrieval-syntec-s2p) (Mathieu Ciancone, 2024) | ['fra'] | Retrieval | s2p | [Legal, Written] | None | None | +| [SyntheticText2SQL](https://huggingface.co/datasets/gretelai/synthetic_text_to_sql) (Meyer et al., 2024) | ['eng', 'sql'] | Retrieval | p2p | [Programming, Written] | {'test': 111702} | {'test': {'number_of_characters': 14041553, 'num_samples': 111702, 'num_queries': 5851, 'num_documents': 105851, 'min_document_length': 13, 'average_document_length': 4.58, 'max_document_length': 281, 'unique_documents': 105851, 'min_query_length': 17, 'average_query_length': 2316.95, 'max_query_length': 762, 'unique_queries': 5851, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 5851}} | | [T2Reranking](https://arxiv.org/abs/2304.03679) (Xiaohui Xie, 2023) | ['cmn'] | Reranking | s2s | | None | None | -| [T2Retrieval](https://arxiv.org/abs/2304.03679) (Xiaohui Xie, 2023) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 874.1184182791619, 'average_query_length': 10.938847974750132, 'num_documents': 118605, 'num_queries': 22812, 'average_relevant_docs_per_query': 5.213571804313519}} | -| [TERRa](https://arxiv.org/pdf/2010.15925) (Shavrina et al., 2020) | ['rus'] | PairClassification | s2s | [News, Web, Written] | {'dev': 307} | {'dev': 138.2} | +| [T2Retrieval](https://arxiv.org/abs/2304.03679) (Xiaohui Xie, 2023) | ['cmn'] | Retrieval | s2p | | None | None | +| [TERRa](https://arxiv.org/pdf/2010.15925) (Shavrina et al., 2020) | ['rus'] | PairClassification | s2s | [News, Web, Written] | None | None | | [TNews](https://www.cluebenchmarks.com/introduce.html) | ['cmn'] | Classification | s2s | | None | None | +<<<<<<< HEAD | [TRECCOVID](https://ir.nist.gov/covidSubmit/index.html) (Kirk Roberts, 2021) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1116.7434221277986, 'average_query_length': 69.24, 'num_documents': 171332, 'num_queries': 50, 'average_relevant_docs_per_query': 493.5}} | | [TRECCOVID-PL](https://ir.nist.gov/covidSubmit/index.html) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Academic, Non-fiction, Written] | None | {'test': {'average_document_length': 1159.8020276422385, 'average_query_length': 69.42, 'num_documents': 171332, 'num_queries': 50, 'average_relevant_docs_per_query': 493.5}} | | [TV2Nordretrieval](https://huggingface.co/datasets/alexandrainst/nordjylland-news-summarization) | ['dan'] | Retrieval | p2p | [News, Non-fiction, Written] | {'test': 4096} | {'test': {'average_document_length': 1440.66552734375, 'average_query_length': 126.552734375, 'num_documents': 2048, 'num_queries': 2048, 'average_relevant_docs_per_query': 1.0}} | @@ -575,9 +1007,65 @@ The following tables give you an overview of the tasks in MTEB. | [VieStudentFeedbackClassification](https://ieeexplore.ieee.org/document/8573337) (Nguyen et al., 2018) | ['vie'] | Classification | s2s | [Reviews, Written] | {'test': 2048} | {'test': 14.22} | | [VoyageMMarcoReranking](https://arxiv.org/abs/2312.16144) (Benjamin Clavié, 2023) | ['jpn'] | Reranking | s2s | [Academic, Non-fiction, Written] | {'test': 2048} | {'test': 162} | | [WRIMEClassification](https://aclanthology.org/2021.naacl-main.169/) | ['jpn'] | Classification | s2s | [Social, Written] | {'test': 2048} | {'test': 47.78} | +======= +| [TRECCOVID](https://ir.nist.gov/covidSubmit/index.html) (Kirk Roberts, 2021) | ['eng'] | Retrieval | s2p | [Medical, Academic, Written] | None | None | +| [TRECCOVID-PL](https://ir.nist.gov/covidSubmit/index.html) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Academic, Medical, Non-fiction, Written] | None | None | +| [TV2Nordretrieval](https://huggingface.co/datasets/alexandrainst/nordjylland-news-summarization) | ['dan'] | Retrieval | p2p | [News, Non-fiction, Written] | None | None | +| [TamilNewsClassification](https://github.com/vanangamudi/tamil-news-classification) (Anoop Kunchukuttan, 2020) | ['tam'] | Classification | s2s | [News, Written] | None | None | +| [Tatoeba](https://github.com/facebookresearch/LASER/tree/main/data/tatoeba/v1) (Tatoeba community, 2021) | ['afr', 'amh', 'ang', 'ara', 'arq', 'arz', 'ast', 'awa', 'aze', 'bel', 'ben', 'ber', 'bos', 'bre', 'bul', 'cat', 'cbk', 'ceb', 'ces', 'cha', 'cmn', 'cor', 'csb', 'cym', 'dan', 'deu', 'dsb', 'dtp', 'ell', 'eng', 'epo', 'est', 'eus', 'fao', 'fin', 'fra', 'fry', 'gla', 'gle', 'glg', 'gsw', 'heb', 'hin', 'hrv', 'hsb', 'hun', 'hye', 'ido', 'ile', 'ina', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kat', 'kaz', 'khm', 'kor', 'kur', 'kzj', 'lat', 'lfn', 'lit', 'lvs', 'mal', 'mar', 'max', 'mhr', 'mkd', 'mon', 'nds', 'nld', 'nno', 'nob', 'nov', 'oci', 'orv', 'pam', 'pes', 'pms', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'spa', 'sqi', 'srp', 'swe', 'swg', 'swh', 'tam', 'tat', 'tel', 'tgl', 'tha', 'tuk', 'tur', 'tzl', 'uig', 'ukr', 'urd', 'uzb', 'vie', 'war', 'wuu', 'xho', 'yid', 'yue', 'zsm'] | BitextMining | s2s | [Written] | None | None | +| [TbilisiCityHallBitextMining](https://huggingface.co/datasets/jupyterjazz/tbilisi-city-hall-titles) | ['eng', 'kat'] | BitextMining | s2s | [News, Written] | {'test': 3640} | {'test': {'num_samples': 3640, 'number_of_characters': 572146, 'unique_pairs': 3640, 'min_sentence1_length': 13, 'average_sentence1_length': 78.59, 'max_sentence1_length': 203, 'unique_sentence1': 3636, 'min_sentence2_length': 13, 'average_sentence2_length': 78.59, 'max_sentence2_length': 203, 'unique_sentence2': 3636, 'hf_subset_descriptive_stats': {'kat_Geor-eng_Latn': {'num_samples': 1820, 'number_of_characters': 286073, 'unique_pairs': 1820, 'min_sentence1_length': 30, 'average_sentence1_length': 76.07, 'max_sentence1_length': 189, 'unique_sentence1': 1820, 'min_sentence2_length': 13, 'average_sentence2_length': 81.12, 'max_sentence2_length': 203, 'unique_sentence2': 1816}, 'eng_Latn-kat_Geor': {'num_samples': 1820, 'number_of_characters': 286073, 'unique_pairs': 1820, 'min_sentence1_length': 13, 'average_sentence1_length': 81.12, 'max_sentence1_length': 203, 'unique_sentence1': 1816, 'min_sentence2_length': 30, 'average_sentence2_length': 76.07, 'max_sentence2_length': 189, 'unique_sentence2': 1820}}}} | +| [TelemarketingSalesRuleLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [TeluguAndhraJyotiNewsClassification](https://github.com/AnushaMotamarri/Telugu-Newspaper-Article-Dataset) | ['tel'] | Classification | s2s | [News, Written] | None | None | +| [TempReasonL1](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [TempReasonL2Context](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [TempReasonL2Fact](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [TempReasonL2Pure](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [TempReasonL3Context](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [TempReasonL3Fact](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [TempReasonL3Pure](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [TenKGnadClassification](https://tblock.github.io/10kGNAD/) | ['deu'] | Classification | p2p | [News, Written] | None | None | +| [TenKGnadClusteringP2P.v2](https://tblock.github.io/10kGNAD/) | ['deu'] | Clustering | p2p | [News, Non-fiction, Written] | None | None | +| [TenKGnadClusteringS2S.v2](https://tblock.github.io/10kGNAD/) | ['deu'] | Clustering | s2s | [News, Non-fiction, Written] | None | None | +| [TextualismToolDictionariesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [TextualismToolPlainLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [ThuNewsClusteringP2P.v2](http://thuctc.thunlp.org/) (Sun et al., 2016) | ['cmn'] | Clustering | p2p | [News, Written] | None | None | +| [ThuNewsClusteringS2S.v2](http://thuctc.thunlp.org/) (Sun et al., 2016) | ['cmn'] | Clustering | s2s | [News, Written] | None | None | +| [TopiOCQA](https://mcgill-nlp.github.io/topiocqa) (Vaibhav Adlakha, 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [TopiOCQAHardNegatives](https://mcgill-nlp.github.io/topiocqa) (Vaibhav Adlakha, 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [Touche2020Retrieval.v3](https://github.com/castorini/touche-error-analysis) | ['eng'] | Retrieval | s2p | [Academic] | {'test': 303781} | {'test': {'number_of_characters': 637047138, 'num_samples': 303781, 'num_queries': 49, 'num_documents': 303732, 'min_document_length': 16, 'average_document_length': 0.01, 'max_document_length': 83, 'unique_documents': 303732, 'min_query_length': 41, 'average_query_length': 13000918.57, 'max_query_length': 105983, 'unique_queries': 49, 'min_relevant_docs_per_query': 40, 'average_relevant_docs_per_query': 58.14, 'max_relevant_docs_per_query': 87, 'unique_relevant_docs': 2732}} | +| [ToxicChatClassification](https://aclanthology.org/2023.findings-emnlp.311/) (Zi Lin, 2023) | ['eng'] | Classification | s2s | [Constructed, Written] | None | None | +| [ToxicConversationsClassification](https://www.kaggle.com/competitions/jigsaw-unintended-bias-in-toxicity-classification/overview) (cjadams, 2019) | ['eng'] | Classification | s2s | [Social, Written] | None | None | +| [TswanaNewsClassification](https://link.springer.com/chapter/10.1007/978-3-031-49002-6_17) (Vukosi Marivate, 2023) | ['tsn'] | Classification | s2s | [News, Written] | None | None | +| [TurHistQuadRetrieval](https://github.com/okanvk/Turkish-Reading-Comprehension-Question-Answering-Dataset) (Soygazi et al., 2021) | ['tur'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Academic, Written] | None | None | +| [TurkicClassification](https://huggingface.co/datasets/Electrotubbie/classification_Turkic_languages/) | ['bak', 'kaz', 'kir'] | Classification | s2s | [News, Written] | None | None | +| [TurkishMovieSentimentClassification](https://www.win.tue.nl/~mpechen/publications/pubs/MT_WISDOM2013.pdf) (Erkin Demirtas, 2013) | ['tur'] | Classification | s2s | [Reviews, Written] | None | None | +| [TurkishProductSentimentClassification](https://www.win.tue.nl/~mpechen/publications/pubs/MT_WISDOM2013.pdf) (Erkin Demirtas, 2013) | ['tur'] | Classification | s2s | [Reviews, Written] | None | None | +| [TweetEmotionClassification](https://link.springer.com/chapter/10.1007/978-3-319-77116-8_8) (Al-Khatib et al., 2018) | ['ara'] | Classification | s2s | [Social, Written] | None | None | +| [TweetSarcasmClassification](https://aclanthology.org/2020.osact-1.5/) | ['ara'] | Classification | s2s | [Social, Written] | None | None | +| [TweetSentimentClassification](https://aclanthology.org/2022.lrec-1.27) | ['ara', 'deu', 'eng', 'fra', 'hin', 'ita', 'por', 'spa'] | Classification | s2s | [Social, Written] | None | None | +| [TweetSentimentExtractionClassification](https://www.kaggle.com/competitions/tweet-sentiment-extraction/overview) (Maggie et al., 2020) | ['eng'] | Classification | s2s | [Social, Written] | None | None | +| [TweetTopicSingleClassification](https://arxiv.org/abs/2209.09824) | ['eng'] | Classification | s2s | [Social, News, Written] | None | None | +| [TwentyNewsgroupsClustering.v2](https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html) (Ken Lang, 1995) | ['eng'] | Clustering | s2s | [News, Written] | {'test': 59545} | {'test': {'num_samples': 59545, 'number_of_characters': 1907719, 'min_text_length': 11, 'average_text_length': 32.04, 'max_text_length': 120, 'min_labels_per_text': 2082, 'average_labels_per_text': 1.0, 'max_labels_per_text': 3236, 'unique_labels': 20, 'labels': {'12': {'count': 3137}, '6': {'count': 3070}, '0': {'count': 2613}, '2': {'count': 3155}, '10': {'count': 3220}, '17': {'count': 2986}, '14': {'count': 3106}, '13': {'count': 3055}, '1': {'count': 3056}, '16': {'count': 2911}, '9': {'count': 2984}, '3': {'count': 3070}, '15': {'count': 3090}, '7': {'count': 3036}, '5': {'count': 3124}, '11': {'count': 3236}, '18': {'count': 2483}, '8': {'count': 3090}, '19': {'count': 2082}, '4': {'count': 3041}}}} | +| [TwitterHjerneRetrieval](https://huggingface.co/datasets/sorenmulli/da-hashtag-twitterhjerne) (Holm et al., 2024) | ['dan'] | Retrieval | p2p | [Social, Written] | None | None | +| [TwitterSemEval2015](https://alt.qcri.org/semeval2015/task1/) | ['eng'] | PairClassification | s2s | | None | None | +| [TwitterURLCorpus](https://languagenet.github.io/) | ['eng'] | PairClassification | s2s | | {'test': 51534} | {'test': {'num_samples': 51534, 'number_of_characters': 8659940, 'min_sentence1_length': 24, 'avg_sentence1_length': 79.49, 'max_sentence1_length': 126, 'unique_sentence1': 4329, 'min_sentence2_length': 6, 'avg_sentence2_length': 88.55, 'max_sentence2_length': 608, 'unique_sentence2': 41304, 'unique_labels': 2, 'labels': {'0': {'count': 38546}, '1': {'count': 12988}}}} | +| [UCCVCommonLawLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [UkrFormalityClassification](https://huggingface.co/datasets/ukr-detect/ukr-formality-dataset-translated-gyafc) | ['ukr'] | Classification | s2s | [News, Written] | None | None | +| [UnfairTOSLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [UrduRomanSentimentClassification](https://archive.ics.uci.edu/dataset/458/roman+urdu+data+set) (Sharf,Zareen, 2018) | ['urd'] | Classification | s2s | [Social, Written] | None | None | +| [VGHierarchicalClusteringP2P](https://huggingface.co/datasets/navjordj/VG_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | p2p | [News, Non-fiction, Written] | None | None | +| [VGHierarchicalClusteringS2S](https://huggingface.co/datasets/navjordj/VG_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | p2p | [News, Non-fiction, Written] | None | None | +| [VideoRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | None | +| [VieMedEVBitextMining](https://aclanthology.org/2015.iwslt-evaluation.11/) (Nhu Vo, 2024) | ['eng', 'vie'] | BitextMining | s2s | [Medical, Written] | {'test': 2048} | {'test': {'num_samples': 2048, 'number_of_characters': 575910, 'unique_pairs': 2048, 'min_sentence1_length': 11, 'average_sentence1_length': 139.23, 'max_sentence1_length': 1291, 'unique_sentence1': 2048, 'min_sentence2_length': 11, 'average_sentence2_length': 141.98, 'max_sentence2_length': 1217, 'unique_sentence2': 2047}} | +| [VieQuADRetrieval](https://aclanthology.org/2020.coling-main.233.pdf) | ['vie'] | Retrieval | s2p | [Encyclopaedic, Non-fiction, Written] | None | None | +| [VieStudentFeedbackClassification](https://ieeexplore.ieee.org/document/8573337) (Nguyen et al., 2018) | ['vie'] | Classification | s2s | [Reviews, Written] | None | None | +| [VoyageMMarcoReranking](https://arxiv.org/abs/2312.16144) (Benjamin Clavié, 2023) | ['jpn'] | Reranking | s2s | [Academic, Non-fiction, Written] | None | None | +| [WRIMEClassification](https://aclanthology.org/2021.naacl-main.169/) | ['jpn'] | Classification | s2s | [Social, Written] | None | None | +>>>>>>> main | [Waimai](https://aclanthology.org/2023.nodalida-1.20/) (Xiao et al., 2023) | ['cmn'] | Classification | s2s | | None | None | -| [WebLINXCandidatesReranking](https://mcgill-nlp.github.io/weblinx) (Xing Han Lù, 2024) | ['eng'] | Reranking | p2p | [Academic, Web, Written] | {'validation': 1301, 'test_iid': 1438, 'test_cat': 3560, 'test_web': 3144, 'test_vis': 5298, 'test_geo': 4916} | {'validation': 1647.52, 'test_iid': 1722.63, 'test_cat': 2149.66, 'test_web': 1831.46, 'test_vis': 1737.26, 'test_geo': 1742.66} | +| [WebLINXCandidatesReranking](https://mcgill-nlp.github.io/weblinx) (Xing Han Lù, 2024) | ['eng'] | Reranking | p2p | [Academic, Web, Written] | None | None | | [WikiCitiesClustering](https://huggingface.co/datasets/wikipedia) | ['eng'] | Clustering | p2p | [Encyclopaedic, Written] | None | None | +<<<<<<< HEAD | [WikiClusteringP2P.v2](https://github.com/Rysias/wiki-clustering) | ['bos', 'cat', 'ces', 'dan', 'eus', 'glv', 'ilo', 'kur', 'lav', 'min', 'mlt', 'sco', 'sqi', 'wln'] | Clustering | p2p | [Encyclopaedic, Written] | {'test': 2048} | {'test': {'num_samples': 28672, 'average_text_length': 629.7426409040179, 'average_labels_per_text': 1.0, 'unique_labels': 39, 'labels': {'16': {'count': 541}, '3': {'count': 1607}, '12': {'count': 846}, '0': {'count': 2410}, '15': {'count': 878}, '11': {'count': 864}, '6': {'count': 787}, '9': {'count': 654}, '14': {'count': 966}, '8': {'count': 1389}, '2': {'count': 2428}, '10': {'count': 839}, '1': {'count': 1370}, '4': {'count': 2942}, '7': {'count': 2514}, '5': {'count': 1490}, '13': {'count': 918}, '19': {'count': 315}, '17': {'count': 711}, '20': {'count': 345}, '18': {'count': 800}, '24': {'count': 467}, '25': {'count': 928}, '21': {'count': 62}, '26': {'count': 270}, '22': {'count': 186}, '23': {'count': 36}, '27': {'count': 465}, '28': {'count': 62}, '36': {'count': 139}, '32': {'count': 57}, '38': {'count': 43}, '30': {'count': 52}, '34': {'count': 80}, '33': {'count': 75}, '35': {'count': 62}, '31': {'count': 63}, '37': {'count': 8}, '29': {'count': 3}}, 'hf_subset_descriptive_stats': {'bs': {'num_samples': 2048, 'average_text_length': 1046.25732421875, 'average_labels_per_text': 1.0, 'unique_labels': 17, 'labels': {'16': {'count': 268}, '3': {'count': 89}, '12': {'count': 597}, '0': {'count': 202}, '15': {'count': 113}, '11': {'count': 11}, '6': {'count': 142}, '9': {'count': 181}, '14': {'count': 179}, '8': {'count': 33}, '2': {'count': 172}, '10': {'count': 12}, '1': {'count': 7}, '4': {'count': 25}, '7': {'count': 6}, '5': {'count': 9}, '13': {'count': 2}}}, 'ca': {'num_samples': 2048, 'average_text_length': 600.73291015625, 'average_labels_per_text': 1.0, 'unique_labels': 8, 'labels': {'6': {'count': 257}, '1': {'count': 737}, '2': {'count': 284}, '4': {'count': 394}, '0': {'count': 162}, '7': {'count': 151}, '5': {'count': 55}, '3': {'count': 8}}}, 'cs': {'num_samples': 2048, 'average_text_length': 659.2294921875, 'average_labels_per_text': 1.0, 'unique_labels': 21, 'labels': {'19': {'count': 35}, '5': {'count': 624}, '17': {'count': 126}, '10': {'count': 155}, '1': {'count': 231}, '7': {'count': 215}, '11': {'count': 128}, '0': {'count': 57}, '13': {'count': 75}, '2': {'count': 83}, '3': {'count': 38}, '9': {'count': 8}, '6': {'count': 14}, '12': {'count': 9}, '16': {'count': 16}, '20': {'count': 73}, '18': {'count': 38}, '4': {'count': 60}, '15': {'count': 14}, '14': {'count': 38}, '8': {'count': 11}}}, 'da': {'num_samples': 2048, 'average_text_length': 767.58935546875, 'average_labels_per_text': 1.0, 'unique_labels': 20, 'labels': {'14': {'count': 212}, '4': {'count': 74}, '15': {'count': 16}, '8': {'count': 165}, '13': {'count': 115}, '0': {'count': 79}, '1': {'count': 34}, '9': {'count': 114}, '7': {'count': 364}, '10': {'count': 32}, '17': {'count': 66}, '18': {'count': 32}, '12': {'count': 129}, '11': {'count': 159}, '2': {'count': 66}, '3': {'count': 185}, '19': {'count': 103}, '16': {'count': 33}, '5': {'count': 56}, '6': {'count': 14}}}, 'eu': {'num_samples': 2048, 'average_text_length': 405.16015625, 'average_labels_per_text': 1.0, 'unique_labels': 5, 'labels': {'4': {'count': 383}, '0': {'count': 995}, '3': {'count': 282}, '2': {'count': 344}, '1': {'count': 44}}}, 'gv': {'num_samples': 2048, 'average_text_length': 368.01123046875, 'average_labels_per_text': 1.0, 'unique_labels': 28, 'labels': {'6': {'count': 32}, '1': {'count': 83}, '24': {'count': 13}, '17': {'count': 152}, '2': {'count': 534}, '25': {'count': 76}, '5': {'count': 198}, '15': {'count': 100}, '21': {'count': 22}, '26': {'count': 188}, '13': {'count': 230}, '20': {'count': 11}, '3': {'count': 107}, '19': {'count': 88}, '16': {'count': 55}, '22': {'count': 29}, '14': {'count': 12}, '8': {'count': 61}, '0': {'count': 5}, '10': {'count': 4}, '4': {'count': 9}, '23': {'count': 6}, '7': {'count': 3}, '9': {'count': 20}, '18': {'count': 4}, '12': {'count': 3}, '27': {'count': 1}, '11': {'count': 2}}}, 'ilo': {'num_samples': 2048, 'average_text_length': 617.90771484375, 'average_labels_per_text': 1.0, 'unique_labels': 29, 'labels': {'3': {'count': 562}, '0': {'count': 373}, '18': {'count': 521}, '8': {'count': 129}, '13': {'count': 123}, '11': {'count': 54}, '25': {'count': 8}, '27': {'count': 5}, '17': {'count': 13}, '15': {'count': 4}, '4': {'count': 28}, '7': {'count': 83}, '10': {'count': 15}, '1': {'count': 11}, '24': {'count': 15}, '14': {'count': 8}, '16': {'count': 4}, '19': {'count': 9}, '23': {'count': 10}, '26': {'count': 4}, '28': {'count': 8}, '12': {'count': 29}, '21': {'count': 12}, '6': {'count': 5}, '20': {'count': 6}, '5': {'count': 4}, '22': {'count': 2}, '9': {'count': 2}, '2': {'count': 1}}}, 'ku': {'num_samples': 2048, 'average_text_length': 421.17333984375, 'average_labels_per_text': 1.0, 'unique_labels': 39, 'labels': {'14': {'count': 14}, '36': {'count': 139}, '20': {'count': 108}, '22': {'count': 27}, '15': {'count': 102}, '32': {'count': 55}, '8': {'count': 431}, '17': {'count': 210}, '38': {'count': 43}, '30': {'count': 51}, '4': {'count': 60}, '2': {'count': 111}, '6': {'count': 95}, '34': {'count': 70}, '27': {'count': 15}, '5': {'count': 174}, '26': {'count': 37}, '0': {'count': 11}, '25': {'count': 50}, '16': {'count': 2}, '12': {'count': 16}, '24': {'count': 2}, '11': {'count': 17}, '21': {'count': 9}, '13': {'count': 20}, '1': {'count': 7}, '33': {'count': 33}, '35': {'count': 28}, '10': {'count': 11}, '31': {'count': 51}, '18': {'count': 4}, '3': {'count': 4}, '28': {'count': 8}, '37': {'count': 8}, '23': {'count': 2}, '19': {'count': 7}, '7': {'count': 6}, '9': {'count': 8}, '29': {'count': 2}}}, 'lv': {'num_samples': 2048, 'average_text_length': 770.67138671875, 'average_labels_per_text': 1.0, 'unique_labels': 16, 'labels': {'15': {'count': 288}, '2': {'count': 110}, '6': {'count': 74}, '12': {'count': 50}, '0': {'count': 171}, '14': {'count': 188}, '10': {'count': 351}, '5': {'count': 142}, '4': {'count': 300}, '13': {'count': 60}, '11': {'count': 48}, '1': {'count': 165}, '8': {'count': 53}, '7': {'count': 5}, '3': {'count': 9}, '9': {'count': 34}}}, 'min': {'num_samples': 2048, 'average_text_length': 631.74072265625, 'average_labels_per_text': 1.0, 'unique_labels': 15, 'labels': {'7': {'count': 1595}, '9': {'count': 9}, '4': {'count': 48}, '3': {'count': 83}, '2': {'count': 160}, '0': {'count': 19}, '5': {'count': 74}, '6': {'count': 12}, '10': {'count': 12}, '13': {'count': 10}, '8': {'count': 5}, '11': {'count': 13}, '12': {'count': 2}, '1': {'count': 5}, '14': {'count': 1}}}, 'mt': {'num_samples': 2048, 'average_text_length': 821.22265625, 'average_labels_per_text': 1.0, 'unique_labels': 27, 'labels': {'12': {'count': 8}, '10': {'count': 147}, '14': {'count': 180}, '17': {'count': 117}, '25': {'count': 654}, '19': {'count': 35}, '0': {'count': 77}, '3': {'count': 12}, '16': {'count': 44}, '15': {'count': 108}, '24': {'count': 267}, '6': {'count': 43}, '26': {'count': 32}, '4': {'count': 79}, '22': {'count': 67}, '9': {'count': 16}, '8': {'count': 16}, '2': {'count': 55}, '5': {'count': 6}, '11': {'count': 30}, '18': {'count': 12}, '21': {'count': 12}, '20': {'count': 15}, '23': {'count': 7}, '13': {'count': 6}, '7': {'count': 1}, '1': {'count': 2}}}, 'sco': {'num_samples': 2048, 'average_text_length': 1065.21044921875, 'average_labels_per_text': 1.0, 'unique_labels': 23, 'labels': {'18': {'count': 178}, '6': {'count': 92}, '9': {'count': 28}, '15': {'count': 106}, '8': {'count': 432}, '2': {'count': 95}, '11': {'count': 104}, '1': {'count': 42}, '13': {'count': 248}, '16': {'count': 118}, '20': {'count': 130}, '3': {'count': 171}, '22': {'count': 57}, '7': {'count': 83}, '10': {'count': 74}, '5': {'count': 6}, '4': {'count': 17}, '17': {'count': 24}, '14': {'count': 14}, '0': {'count': 7}, '19': {'count': 18}, '21': {'count': 3}, '12': {'count': 1}}}, 'sq': {'num_samples': 2048, 'average_text_length': 425.486328125, 'average_labels_per_text': 1.0, 'unique_labels': 36, 'labels': {'27': {'count': 444}, '9': {'count': 234}, '14': {'count': 120}, '0': {'count': 128}, '15': {'count': 27}, '11': {'count': 298}, '24': {'count': 170}, '28': {'count': 46}, '19': {'count': 20}, '25': {'count': 140}, '3': {'count': 47}, '2': {'count': 87}, '35': {'count': 34}, '8': {'count': 53}, '31': {'count': 12}, '17': {'count': 3}, '23': {'count': 11}, '20': {'count': 2}, '33': {'count': 42}, '10': {'count': 26}, '34': {'count': 10}, '7': {'count': 2}, '13': {'count': 29}, '4': {'count': 4}, '6': {'count': 7}, '26': {'count': 9}, '5': {'count': 16}, '30': {'count': 1}, '21': {'count': 4}, '22': {'count': 4}, '18': {'count': 11}, '32': {'count': 2}, '12': {'count': 2}, '16': {'count': 1}, '1': {'count': 1}, '29': {'count': 1}}}, 'wa': {'num_samples': 2048, 'average_text_length': 216.00390625, 'average_labels_per_text': 1.0, 'unique_labels': 6, 'labels': {'5': {'count': 126}, '4': {'count': 1461}, '0': {'count': 124}, '2': {'count': 326}, '3': {'count': 10}, '1': {'count': 1}}}}}} | | [WikipediaRerankingMultilingual](https://huggingface.co/datasets/ellamind/wikipedia-2023-11-reranking-multilingual) | ['ben', 'bul', 'ces', 'dan', 'deu', 'eng', 'fas', 'fin', 'hin', 'ita', 'nld', 'nor', 'por', 'ron', 'srp', 'swe'] | Reranking | s2p | [Encyclopaedic, Written] | {'en': 1500, 'de': 1500, 'it': 1500, 'pt': 1500, 'nl': 1500, 'cs': 1500, 'ro': 1500, 'bg': 1500, 'sr': 1500, 'fi': 1500, 'da': 1500, 'fa': 1500, 'hi': 1500, 'bn': 1500, 'no': 1500, 'sv': 1500} | {'test': {'num_samples': 24000, 'num_positive': 24000, 'num_negative': 24000, 'avg_query_len': 59.091208333333334, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0, 'hf_subset_descriptive_stats': {'bg': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 60.82666666666667, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'bn': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 47.266666666666666, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'cs': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 56.272, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'da': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 56.75066666666667, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'de': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 70.004, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'en': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 68.372, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'fa': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 48.66733333333333, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'fi': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 55.343333333333334, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'hi': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 50.77733333333333, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'it': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 70.05466666666666, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'nl': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 65.34466666666667, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'pt': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 65.11933333333333, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'ro': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 61.973333333333336, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'sr': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 55.669333333333334, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'no': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 55.288, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'sv': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 57.73, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}}}} | | [WikipediaRetrievalMultilingual](https://huggingface.co/datasets/ellamind/wikipedia-2023-11-retrieval-multilingual-queries) | ['ben', 'bul', 'ces', 'dan', 'deu', 'eng', 'fas', 'fin', 'hin', 'ita', 'nld', 'nor', 'por', 'ron', 'srp', 'swe'] | Retrieval | s2p | [Encyclopaedic, Written] | {'en': 1500, 'de': 1500, 'it': 1500, 'pt': 1500, 'nl': 1500, 'cs': 1500, 'ro': 1500, 'bg': 1500, 'sr': 1500, 'fi': 1500, 'da': 1500, 'fa': 1500, 'hi': 1500, 'bn': 1500, 'no': 1500, 'sv': 1500} | {'test': {'bg': {'average_document_length': 374.376, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'bn': {'average_document_length': 394.05044444444445, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'cs': {'average_document_length': 369.9831111111111, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'da': {'average_document_length': 345.2597037037037, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'de': {'average_document_length': 398.4137777777778, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'en': {'average_document_length': 452.9871111111111, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'fa': {'average_document_length': 345.1568888888889, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'fi': {'average_document_length': 379.71237037037037, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'hi': {'average_document_length': 410.72540740740743, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'it': {'average_document_length': 393.73437037037036, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'nl': {'average_document_length': 375.6695555555556, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'pt': {'average_document_length': 398.27237037037037, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'ro': {'average_document_length': 348.3817037037037, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'sr': {'average_document_length': 384.3131851851852, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'no': {'average_document_length': 366.93733333333336, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'sv': {'average_document_length': 369.340962962963, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}}} | @@ -595,6 +1083,25 @@ The following tables give you an overview of the tasks in MTEB. | [indonli](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) | ['ind'] | PairClassification | s2s | [Encyclopaedic, Web, News, Written] | {'test_expert': 2040} | {'test_expert': 145.88} | | [mFollowIRCrossLingualInstructionRetrieval](https://neuclir.github.io/) (Weller et al., 2024) | ['eng', 'fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'eng-fas': 80, 'eng-rus': 80, 'eng-zho': 86} | {'test': {'num_docs': 121635, 'num_queries': 123, 'average_document_length': 2331.0777818884367, 'average_query_length': 81.8780487804878, 'average_instruction_length': 389.9512195121951, 'average_changed_instruction_length': 450.5528455284553, 'average_relevant_docs_per_query': 10.30952380952381, 'average_top_ranked_per_query': 1024.3902439024391, 'hf_subset_descriptive_stats': {'eng-fas': {'num_docs': 41189, 'num_queries': 40, 'average_document_length': 3145.4990895627475, 'average_query_length': 80.075, 'average_instruction_length': 396.875, 'average_changed_instruction_length': 463.175, 'average_relevant_docs_per_query': 10.465116279069768, 'average_top_ranked_per_query': 1075}, 'eng-rus': {'num_docs': 39326, 'num_queries': 40, 'average_document_length': 2784.0813456746173, 'average_query_length': 81.875, 'average_instruction_length': 371.125, 'average_changed_instruction_length': 431.8, 'average_relevant_docs_per_query': 9.775, 'average_top_ranked_per_query': 1000}, 'eng-zho': {'num_docs': 41120, 'num_queries': 43, 'average_document_length': 1082.0501215953307, 'average_query_length': 83.55813953488372, 'average_instruction_length': 401.0232558139535, 'average_changed_instruction_length': 456.25581395348837, 'average_relevant_docs_per_query': 10.651162790697674, 'average_top_ranked_per_query': 1000}}}} | | [mFollowIRInstructionRetrieval](https://neuclir.github.io/) (Weller et al., 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'fas': 80, 'rus': 80, 'zho': 86} | {'test': {'num_docs': 121635, 'num_queries': 123, 'average_document_length': 2331.0777818884367, 'average_query_length': 57.113821138211385, 'average_instruction_length': 281.0650406504065, 'average_changed_instruction_length': 326.9430894308943, 'average_relevant_docs_per_query': 10.30952380952381, 'average_top_ranked_per_query': 1024.3902439024391, 'hf_subset_descriptive_stats': {'fas': {'num_docs': 41189, 'num_queries': 40, 'average_document_length': 3145.4990895627475, 'average_query_length': 72.65, 'average_instruction_length': 358.925, 'average_changed_instruction_length': 415.325, 'average_relevant_docs_per_query': 10.465116279069768, 'average_top_ranked_per_query': 1075}, 'rus': {'num_docs': 39326, 'num_queries': 40, 'average_document_length': 2784.0813456746173, 'average_query_length': 77.5, 'average_instruction_length': 387, 'average_changed_instruction_length': 458, 'average_relevant_docs_per_query': 9.775, 'average_top_ranked_per_query': 1000}, 'zho': {'num_docs': 41120, 'num_queries': 43, 'average_document_length': 1082.0501215953307, 'average_query_length': 23.697674418604652, 'average_instruction_length': 110.09302325581395, 'average_changed_instruction_length': 122.81395348837209, 'average_relevant_docs_per_query': 10.651162790697674, 'average_top_ranked_per_query': 1000}}}} | +======= +| [WikiClusteringP2P.v2](https://github.com/Rysias/wiki-clustering) | ['bos', 'cat', 'ces', 'dan', 'eus', 'glv', 'ilo', 'kur', 'lav', 'min', 'mlt', 'sco', 'sqi', 'wln'] | Clustering | p2p | [Encyclopaedic, Written] | None | None | +| [WikipediaRerankingMultilingual](https://huggingface.co/datasets/ellamind/wikipedia-2023-11-reranking-multilingual) | ['ben', 'bul', 'ces', 'dan', 'deu', 'eng', 'fas', 'fin', 'hin', 'ita', 'nld', 'nor', 'por', 'ron', 'srp', 'swe'] | Reranking | s2p | [Encyclopaedic, Written] | {'test': 24000} | {'test': {'num_samples': 24000, 'number_of_characters': 83866932, 'num_positive': 24000, 'num_negative': 192000, 'min_query_length': 7, 'avg_query_length': 59.09, 'max_query_length': 180, 'unique_query': 23997, 'min_positive_length': 100, 'avg_positive_length': 385.45, 'max_positive_length': 3515, 'unique_positive': 23993, 'min_negative_length': 100, 'avg_negative_length': 381.24, 'max_negative_length': 9461, 'unique_negative': 191783, 'hf_subset_descriptive_stats': {'bg': {'num_samples': 1500, 'number_of_characters': 5145316, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 18, 'avg_query_length': 60.83, 'max_query_length': 166, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 375.89, 'max_positive_length': 2241, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 374.19, 'max_negative_length': 4869, 'unique_negative': 11996}, 'bn': {'num_samples': 1500, 'number_of_characters': 5390581, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 7, 'avg_query_length': 47.27, 'max_query_length': 123, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 394.59, 'max_positive_length': 2338, 'unique_positive': 1499, 'min_negative_length': 100, 'avg_negative_length': 393.98, 'max_negative_length': 5104, 'unique_negative': 11996}, 'cs': {'num_samples': 1500, 'number_of_characters': 5079180, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 17, 'avg_query_length': 56.27, 'max_query_length': 137, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 383.84, 'max_positive_length': 2300, 'unique_positive': 1499, 'min_negative_length': 100, 'avg_negative_length': 368.25, 'max_negative_length': 3487, 'unique_negative': 11982}, 'da': {'num_samples': 1500, 'number_of_characters': 4746132, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 17, 'avg_query_length': 56.75, 'max_query_length': 137, 'unique_query': 1499, 'min_positive_length': 100, 'avg_positive_length': 351.68, 'max_positive_length': 2159, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 344.46, 'max_negative_length': 2563, 'unique_negative': 11972}, 'de': {'num_samples': 1500, 'number_of_characters': 5483592, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 20, 'avg_query_length': 70.0, 'max_query_length': 180, 'unique_query': 1499, 'min_positive_length': 100, 'avg_positive_length': 391.54, 'max_positive_length': 2674, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 399.27, 'max_negative_length': 3083, 'unique_negative': 12000}, 'en': {'num_samples': 1500, 'number_of_characters': 6217884, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 18, 'avg_query_length': 68.37, 'max_query_length': 162, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 451.73, 'max_positive_length': 3515, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 453.14, 'max_negative_length': 3662, 'unique_negative': 12000}, 'fa': {'num_samples': 1500, 'number_of_characters': 4732619, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 12, 'avg_query_length': 48.67, 'max_query_length': 119, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 347.7, 'max_positive_length': 2571, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 344.84, 'max_negative_length': 4707, 'unique_negative': 11978}, 'fi': {'num_samples': 1500, 'number_of_characters': 5209132, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 14, 'avg_query_length': 55.34, 'max_query_length': 132, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 394.71, 'max_positive_length': 2129, 'unique_positive': 1498, 'min_negative_length': 100, 'avg_negative_length': 377.84, 'max_negative_length': 2574, 'unique_negative': 11972}, 'hi': {'num_samples': 1500, 'number_of_characters': 5620959, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 13, 'avg_query_length': 50.78, 'max_query_length': 125, 'unique_query': 1499, 'min_positive_length': 100, 'avg_positive_length': 420.38, 'max_positive_length': 2361, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 409.52, 'max_negative_length': 5912, 'unique_negative': 11996}, 'it': {'num_samples': 1500, 'number_of_characters': 5420496, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 23, 'avg_query_length': 70.05, 'max_query_length': 156, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 396.97, 'max_positive_length': 2082, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 393.33, 'max_negative_length': 9461, 'unique_negative': 11993}, 'nl': {'num_samples': 1500, 'number_of_characters': 5169556, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 18, 'avg_query_length': 65.34, 'max_query_length': 136, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 380.79, 'max_positive_length': 1864, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 375.03, 'max_negative_length': 3641, 'unique_negative': 11985}, 'pt': {'num_samples': 1500, 'number_of_characters': 5474356, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 18, 'avg_query_length': 65.12, 'max_query_length': 176, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 404.02, 'max_positive_length': 3057, 'unique_positive': 1499, 'min_negative_length': 100, 'avg_negative_length': 397.55, 'max_negative_length': 2877, 'unique_negative': 11991}, 'ro': {'num_samples': 1500, 'number_of_characters': 4796113, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 14, 'avg_query_length': 61.97, 'max_query_length': 169, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 346.71, 'max_positive_length': 1917, 'unique_positive': 1499, 'min_negative_length': 100, 'avg_negative_length': 348.59, 'max_negative_length': 4213, 'unique_negative': 11971}, 'sr': {'num_samples': 1500, 'number_of_characters': 5271732, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 15, 'avg_query_length': 55.67, 'max_query_length': 146, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 386.35, 'max_positive_length': 2421, 'unique_positive': 1499, 'min_negative_length': 100, 'avg_negative_length': 384.06, 'max_negative_length': 3668, 'unique_negative': 11974}, 'no': {'num_samples': 1500, 'number_of_characters': 5036586, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 14, 'avg_query_length': 55.29, 'max_query_length': 129, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 367.72, 'max_positive_length': 1450, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 366.84, 'max_negative_length': 2841, 'unique_negative': 11996}, 'sv': {'num_samples': 1500, 'number_of_characters': 5072698, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 17, 'avg_query_length': 57.73, 'max_query_length': 133, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 372.59, 'max_positive_length': 2493, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 368.94, 'max_negative_length': 3680, 'unique_negative': 11999}}}} | +| [WikipediaRetrievalMultilingual](https://huggingface.co/datasets/ellamind/wikipedia-2023-11-retrieval-multilingual-queries) | ['ben', 'bul', 'ces', 'dan', 'deu', 'eng', 'fas', 'fin', 'hin', 'ita', 'nld', 'nor', 'por', 'ron', 'srp', 'swe'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [WinoGrande](https://winogrande.allenai.org/) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [WisesightSentimentClassification](https://github.com/PyThaiNLP/wisesight-sentiment) | ['tha'] | Classification | s2s | [Social, News, Written] | None | None | +| XMarket (Bonab et al., 2021) | ['deu', 'eng', 'spa'] | Retrieval | s2p | | None | None | +| [XNLI](https://aclanthology.org/D18-1269/) (Conneau et al., 2018) | ['ara', 'bul', 'deu', 'ell', 'eng', 'fra', 'hin', 'rus', 'spa', 'swa', 'tha', 'tur', 'vie', 'zho'] | PairClassification | s2s | [Non-fiction, Fiction, Government, Written] | {'test': 19110, 'validation': 19110} | {'test': {'num_samples': 19110, 'number_of_characters': 2907145, 'min_sentence1_length': 3, 'avg_sentence1_length': 103.24, 'max_sentence1_length': 401, 'unique_sentence1': 15328, 'min_sentence2_length': 2, 'avg_sentence2_length': 48.89, 'max_sentence2_length': 187, 'unique_sentence2': 19104, 'unique_labels': 2, 'labels': {'0': {'count': 9562}, '1': {'count': 9548}}, 'hf_subset_descriptive_stats': {'ar': {'num_samples': 1365, 'number_of_characters': 179591, 'min_sentence1_length': 11, 'avg_sentence1_length': 89.57, 'max_sentence1_length': 242, 'unique_sentence1': 1095, 'min_sentence2_length': 8, 'avg_sentence2_length': 41.99, 'max_sentence2_length': 115, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'bg': {'num_samples': 1365, 'number_of_characters': 220646, 'min_sentence1_length': 14, 'avg_sentence1_length': 110.02, 'max_sentence1_length': 303, 'unique_sentence1': 1095, 'min_sentence2_length': 8, 'avg_sentence2_length': 51.63, 'max_sentence2_length': 150, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'de': {'num_samples': 1365, 'number_of_characters': 241224, 'min_sentence1_length': 3, 'avg_sentence1_length': 119.93, 'max_sentence1_length': 301, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 56.79, 'max_sentence2_length': 187, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'el': {'num_samples': 1365, 'number_of_characters': 240222, 'min_sentence1_length': 13, 'avg_sentence1_length': 119.05, 'max_sentence1_length': 344, 'unique_sentence1': 1095, 'min_sentence2_length': 13, 'avg_sentence2_length': 56.93, 'max_sentence2_length': 172, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'en': {'num_samples': 1365, 'number_of_characters': 212223, 'min_sentence1_length': 19, 'avg_sentence1_length': 105.67, 'max_sentence1_length': 268, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 49.8, 'max_sentence2_length': 137, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'es': {'num_samples': 1365, 'number_of_characters': 232207, 'min_sentence1_length': 11, 'avg_sentence1_length': 115.43, 'max_sentence1_length': 385, 'unique_sentence1': 1094, 'min_sentence2_length': 8, 'avg_sentence2_length': 54.68, 'max_sentence2_length': 163, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'fr': {'num_samples': 1365, 'number_of_characters': 245259, 'min_sentence1_length': 9, 'avg_sentence1_length': 121.1, 'max_sentence1_length': 327, 'unique_sentence1': 1095, 'min_sentence2_length': 10, 'avg_sentence2_length': 58.58, 'max_sentence2_length': 169, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'hi': {'num_samples': 1365, 'number_of_characters': 211312, 'min_sentence1_length': 16, 'avg_sentence1_length': 104.63, 'max_sentence1_length': 401, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 50.17, 'max_sentence2_length': 162, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'ru': {'num_samples': 1365, 'number_of_characters': 222797, 'min_sentence1_length': 11, 'avg_sentence1_length': 110.77, 'max_sentence1_length': 306, 'unique_sentence1': 1095, 'min_sentence2_length': 8, 'avg_sentence2_length': 52.45, 'max_sentence2_length': 167, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'sw': {'num_samples': 1365, 'number_of_characters': 210103, 'min_sentence1_length': 10, 'avg_sentence1_length': 104.44, 'max_sentence1_length': 266, 'unique_sentence1': 1094, 'min_sentence2_length': 2, 'avg_sentence2_length': 49.48, 'max_sentence2_length': 146, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'th': {'num_samples': 1365, 'number_of_characters': 192788, 'min_sentence1_length': 12, 'avg_sentence1_length': 96.69, 'max_sentence1_length': 262, 'unique_sentence1': 1095, 'min_sentence2_length': 6, 'avg_sentence2_length': 44.54, 'max_sentence2_length': 129, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'tr': {'num_samples': 1365, 'number_of_characters': 208658, 'min_sentence1_length': 15, 'avg_sentence1_length': 103.68, 'max_sentence1_length': 255, 'unique_sentence1': 1095, 'min_sentence2_length': 6, 'avg_sentence2_length': 49.19, 'max_sentence2_length': 140, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'vi': {'num_samples': 1365, 'number_of_characters': 223549, 'min_sentence1_length': 14, 'avg_sentence1_length': 111.31, 'max_sentence1_length': 265, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 52.46, 'max_sentence2_length': 143, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'zh': {'num_samples': 1365, 'number_of_characters': 66566, 'min_sentence1_length': 4, 'avg_sentence1_length': 33.04, 'max_sentence1_length': 112, 'unique_sentence1': 1095, 'min_sentence2_length': 3, 'avg_sentence2_length': 15.73, 'max_sentence2_length': 59, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}}}, 'validation': {'num_samples': 19110, 'number_of_characters': 2909058, 'min_sentence1_length': 5, 'avg_sentence1_length': 103.21, 'max_sentence1_length': 323, 'unique_sentence1': 11171, 'min_sentence2_length': 3, 'avg_sentence2_length': 49.02, 'max_sentence2_length': 172, 'unique_sentence2': 19101, 'unique_labels': 2, 'labels': {'0': {'count': 9562}, '1': {'count': 9548}}, 'hf_subset_descriptive_stats': {'ar': {'num_samples': 1365, 'number_of_characters': 177355, 'min_sentence1_length': 13, 'avg_sentence1_length': 88.32, 'max_sentence1_length': 214, 'unique_sentence1': 798, 'min_sentence2_length': 6, 'avg_sentence2_length': 41.61, 'max_sentence2_length': 137, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'bg': {'num_samples': 1365, 'number_of_characters': 219988, 'min_sentence1_length': 16, 'avg_sentence1_length': 109.2, 'max_sentence1_length': 316, 'unique_sentence1': 798, 'min_sentence2_length': 10, 'avg_sentence2_length': 51.97, 'max_sentence2_length': 151, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'de': {'num_samples': 1365, 'number_of_characters': 241852, 'min_sentence1_length': 20, 'avg_sentence1_length': 119.81, 'max_sentence1_length': 298, 'unique_sentence1': 798, 'min_sentence2_length': 12, 'avg_sentence2_length': 57.37, 'max_sentence2_length': 162, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'el': {'num_samples': 1365, 'number_of_characters': 241275, 'min_sentence1_length': 16, 'avg_sentence1_length': 119.88, 'max_sentence1_length': 302, 'unique_sentence1': 798, 'min_sentence2_length': 6, 'avg_sentence2_length': 56.88, 'max_sentence2_length': 171, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'en': {'num_samples': 1365, 'number_of_characters': 212384, 'min_sentence1_length': 20, 'avg_sentence1_length': 105.72, 'max_sentence1_length': 271, 'unique_sentence1': 798, 'min_sentence2_length': 8, 'avg_sentence2_length': 49.88, 'max_sentence2_length': 139, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'es': {'num_samples': 1365, 'number_of_characters': 232451, 'min_sentence1_length': 14, 'avg_sentence1_length': 115.17, 'max_sentence1_length': 265, 'unique_sentence1': 798, 'min_sentence2_length': 7, 'avg_sentence2_length': 55.12, 'max_sentence2_length': 148, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'fr': {'num_samples': 1365, 'number_of_characters': 246857, 'min_sentence1_length': 19, 'avg_sentence1_length': 121.76, 'max_sentence1_length': 323, 'unique_sentence1': 798, 'min_sentence2_length': 11, 'avg_sentence2_length': 59.09, 'max_sentence2_length': 172, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'hi': {'num_samples': 1365, 'number_of_characters': 212269, 'min_sentence1_length': 18, 'avg_sentence1_length': 105.06, 'max_sentence1_length': 277, 'unique_sentence1': 798, 'min_sentence2_length': 7, 'avg_sentence2_length': 50.44, 'max_sentence2_length': 152, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'ru': {'num_samples': 1365, 'number_of_characters': 221152, 'min_sentence1_length': 15, 'avg_sentence1_length': 109.75, 'max_sentence1_length': 310, 'unique_sentence1': 798, 'min_sentence2_length': 8, 'avg_sentence2_length': 52.27, 'max_sentence2_length': 140, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'sw': {'num_samples': 1365, 'number_of_characters': 210482, 'min_sentence1_length': 13, 'avg_sentence1_length': 104.32, 'max_sentence1_length': 264, 'unique_sentence1': 798, 'min_sentence2_length': 8, 'avg_sentence2_length': 49.88, 'max_sentence2_length': 153, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'th': {'num_samples': 1365, 'number_of_characters': 192640, 'min_sentence1_length': 7, 'avg_sentence1_length': 97.28, 'max_sentence1_length': 255, 'unique_sentence1': 798, 'min_sentence2_length': 3, 'avg_sentence2_length': 43.84, 'max_sentence2_length': 140, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'tr': {'num_samples': 1365, 'number_of_characters': 208305, 'min_sentence1_length': 15, 'avg_sentence1_length': 102.97, 'max_sentence1_length': 269, 'unique_sentence1': 798, 'min_sentence2_length': 10, 'avg_sentence2_length': 49.64, 'max_sentence2_length': 139, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'vi': {'num_samples': 1365, 'number_of_characters': 224811, 'min_sentence1_length': 18, 'avg_sentence1_length': 112.26, 'max_sentence1_length': 323, 'unique_sentence1': 798, 'min_sentence2_length': 9, 'avg_sentence2_length': 52.43, 'max_sentence2_length': 159, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'zh': {'num_samples': 1365, 'number_of_characters': 67237, 'min_sentence1_length': 5, 'avg_sentence1_length': 33.41, 'max_sentence1_length': 135, 'unique_sentence1': 798, 'min_sentence2_length': 3, 'avg_sentence2_length': 15.85, 'max_sentence2_length': 66, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}}}} | +| [XNLIV2](https://arxiv.org/pdf/2301.06527) (Upadhyay et al., 2023) | ['asm', 'ben', 'bho', 'ell', 'guj', 'kan', 'mar', 'ory', 'pan', 'rus', 'san', 'tam', 'tur'] | PairClassification | s2s | [Non-fiction, Fiction, Government, Written] | None | None | +| [XPQARetrieval](https://arxiv.org/abs/2305.09249) (Shen et al., 2023) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'jpn', 'kor', 'pol', 'por', 'spa', 'tam'] | Retrieval | s2p | [Reviews, Written] | None | None | +| [XQuADRetrieval](https://huggingface.co/datasets/xquad) (Mikel Artetxe, 2019) | ['arb', 'deu', 'ell', 'eng', 'hin', 'ron', 'rus', 'spa', 'tha', 'tur', 'vie', 'zho'] | Retrieval | s2p | [Web, Written] | None | None | +| [XStance](https://github.com/ZurichNLP/xstance) | ['deu', 'fra', 'ita'] | PairClassification | s2s | [Social, Written] | None | None | +| [YahooAnswersTopicsClassification](https://huggingface.co/datasets/yahoo_answers_topics) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [Web, Written] | None | None | +| [YelpReviewFullClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [Reviews, Written] | None | None | +| [YueOpenriceReviewClassification](https://github.com/Christainx/Dataset_Cantonese_Openrice) (Xiang et al., 2019) | ['yue'] | Classification | s2s | [Reviews, Spoken] | None | None | +| [indonli](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) | ['ind'] | PairClassification | s2s | [Encyclopaedic, Web, News, Written] | None | None | +| [mFollowIRCrossLingualInstructionRetrieval](https://neuclir.github.io/) (Weller et al., 2024) | ['eng', 'fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'test': 121758} | {'test': {'num_samples': 121758, 'num_docs': 121635, 'num_queries': 123, 'number_of_characters': 283654099, 'min_document_length': 74, 'average_document_length': 2331.08, 'max_document_length': 24179, 'unique_docs': 121635, 'min_query_length': 32, 'average_query_length': 81.88, 'max_query_length': 173, 'unique_queries': 75, 'min_instruction_length': 93, 'average_instruction_length': 389.95, 'max_instruction_length': 887, 'unique_instructions': 75, 'min_changed_instruction_length': 180, 'average_changed_instruction_length': 450.55, 'max_changed_instruction_length': 974, 'unique_changed_instructions': 123, 'min_average_relevant_docs_per_query': 0, 'average_relevant_docs_per_query': 10.43, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000, 'hf_subset_descriptive_stats': {'eng-fas': {'num_samples': 41229, 'num_docs': 41189, 'num_queries': 40, 'number_of_characters': 129597567, 'min_document_length': 99, 'average_document_length': 3145.5, 'max_document_length': 24179, 'unique_docs': 41189, 'min_query_length': 34, 'average_query_length': 80.08, 'max_query_length': 124, 'unique_queries': 40, 'min_instruction_length': 150, 'average_instruction_length': 396.88, 'max_instruction_length': 887, 'unique_instructions': 40, 'min_changed_instruction_length': 205, 'average_changed_instruction_length': 463.18, 'max_changed_instruction_length': 974, 'unique_changed_instructions': 40, 'min_average_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 10.85, 'max_average_relevant_docs_per_query': 22, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}, 'eng-rus': {'num_samples': 39366, 'num_docs': 39326, 'num_queries': 40, 'number_of_characters': 109522175, 'min_document_length': 75, 'average_document_length': 2784.08, 'max_document_length': 24061, 'unique_docs': 39326, 'min_query_length': 32, 'average_query_length': 81.88, 'max_query_length': 173, 'unique_queries': 40, 'min_instruction_length': 93, 'average_instruction_length': 371.12, 'max_instruction_length': 887, 'unique_instructions': 40, 'min_changed_instruction_length': 180, 'average_changed_instruction_length': 431.8, 'max_changed_instruction_length': 957, 'unique_changed_instructions': 40, 'min_average_relevant_docs_per_query': 0, 'average_relevant_docs_per_query': 9.78, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}, 'eng-zho': {'num_samples': 41163, 'num_docs': 41120, 'num_queries': 43, 'number_of_characters': 44534357, 'min_document_length': 74, 'average_document_length': 1082.05, 'max_document_length': 23840, 'unique_docs': 41120, 'min_query_length': 32, 'average_query_length': 83.56, 'max_query_length': 159, 'unique_queries': 43, 'min_instruction_length': 157, 'average_instruction_length': 401.02, 'max_instruction_length': 731, 'unique_instructions': 43, 'min_changed_instruction_length': 209, 'average_changed_instruction_length': 456.26, 'max_changed_instruction_length': 822, 'unique_changed_instructions': 43, 'min_average_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 10.65, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}}}} | +| [mFollowIRInstructionRetrieval](https://neuclir.github.io/) (Weller et al., 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'test': 121758} | {'test': {'num_samples': 121758, 'num_docs': 121635, 'num_queries': 123, 'number_of_characters': 283622456, 'min_document_length': 74, 'average_document_length': 2331.08, 'max_document_length': 24179, 'unique_docs': 121635, 'min_query_length': 10, 'average_query_length': 57.11, 'max_query_length': 136, 'unique_queries': 123, 'min_instruction_length': 37, 'average_instruction_length': 281.07, 'max_instruction_length': 1009, 'unique_instructions': 123, 'min_changed_instruction_length': 44, 'average_changed_instruction_length': 326.94, 'max_changed_instruction_length': 1083, 'unique_changed_instructions': 123, 'min_average_relevant_docs_per_query': 0, 'average_relevant_docs_per_query': 10.43, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000, 'hf_subset_descriptive_stats': {'fas': {'num_samples': 41229, 'num_docs': 41189, 'num_queries': 40, 'number_of_characters': 129593838, 'min_document_length': 99, 'average_document_length': 3145.5, 'max_document_length': 24179, 'unique_docs': 41189, 'min_query_length': 34, 'average_query_length': 72.65, 'max_query_length': 124, 'unique_queries': 40, 'min_instruction_length': 121, 'average_instruction_length': 358.93, 'max_instruction_length': 759, 'unique_instructions': 40, 'min_changed_instruction_length': 163, 'average_changed_instruction_length': 415.32, 'max_changed_instruction_length': 842, 'unique_changed_instructions': 40, 'min_average_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 10.85, 'max_average_relevant_docs_per_query': 22, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}, 'rus': {'num_samples': 39366, 'num_docs': 39326, 'num_queries': 40, 'number_of_characters': 109523683, 'min_document_length': 75, 'average_document_length': 2784.08, 'max_document_length': 24061, 'unique_docs': 39326, 'min_query_length': 26, 'average_query_length': 77.5, 'max_query_length': 136, 'unique_queries': 40, 'min_instruction_length': 78, 'average_instruction_length': 387.0, 'max_instruction_length': 1009, 'unique_instructions': 40, 'min_changed_instruction_length': 187, 'average_changed_instruction_length': 458.0, 'max_changed_instruction_length': 1083, 'unique_changed_instructions': 40, 'min_average_relevant_docs_per_query': 0, 'average_relevant_docs_per_query': 9.78, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}, 'zho': {'num_samples': 41163, 'num_docs': 41120, 'num_queries': 43, 'number_of_characters': 44504935, 'min_document_length': 74, 'average_document_length': 1082.05, 'max_document_length': 23840, 'unique_docs': 41120, 'min_query_length': 10, 'average_query_length': 23.7, 'max_query_length': 44, 'unique_queries': 43, 'min_instruction_length': 37, 'average_instruction_length': 110.09, 'max_instruction_length': 209, 'unique_instructions': 43, 'min_changed_instruction_length': 44, 'average_changed_instruction_length': 122.81, 'max_changed_instruction_length': 229, 'unique_changed_instructions': 43, 'min_average_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 10.65, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}}}} | +>>>>>>> main
@@ -606,6 +1113,7 @@ The following tables give you an overview of the tasks in MTEB.
+<<<<<<< HEAD | Language | BitextMining | Classification | Clustering | InstructionRetrieval | MultilabelClassification | PairClassification | Reranking | Retrieval | STS | Speed | Summarization | |---|------|------|------|------|------|------|------|------|------|------|---| | aai | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | @@ -1660,6 +2168,1062 @@ The following tables give you an overview of the tasks in MTEB. | zul | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | | zyp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | Total | 1394 | 795 | 304 | 3 | 28 | 67 | 50 | 456 | 85 | 2 | 2 | +======= +| ISO Code | Language | Family | BitextMining | Classification | Clustering | InstructionRetrieval | MultilabelClassification | PairClassification | Reranking | Retrieval | STS | Speed | Summarization | Sum | +|---|------|------|------|------|------|------|------|------|------|------|------|---| +| aai | Arifama-Miniafia | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aak | Ankave | Angan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aau | Abau | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aaz | Amarasi | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| abs | Ambonese Malay | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| abt | Ambulas | Ndu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| abx | Inabaknon | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aby | Aneme Wake | Yareban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ace | Achinese | Austronesian | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| acf | Saint Lucian Creole French | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| acm | Mesopotamian Arabic | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| acq | Ta'izzi-Adeni Arabic | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| acr | Achi | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| acu | Achuar-Shiwiar | Chicham | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| adz | Adzera | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aeb | Tunisian Arabic | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| aer | Eastern Arrernte | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aey | Amele | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| afr | Afrikaans | Indo-European | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 10 | +| agd | Agarabi | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agg | Angor | Senagi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agm | Angaataha | Angan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agn | Agutaynen | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agr | Aguaruna | Chicham | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agt | Central Cagayan Agta | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agu | Aguacateco | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aia | Arosi | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aii | Assyrian Neo-Aramaic | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ajp | South Levantine Arabic | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| aka | Akan | Atlantic-Congo | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| ake | Akawaio | Cariban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| alp | Alune | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| alq | Algonquin | Algic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| als | Tosk Albanian | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | +| aly | Alyawarr | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ame | Yanesha' | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amf | Hamer-Banna | South Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amh | Amharic | Afro-Asiatic | 3 | 6 | 3 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 14 | +| amk | Ambai | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amm | Ama (Papua New Guinea) | Left May | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amn | Amanab | Border | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amo | Amo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amp | Alamblak | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amr | Amarakaeri | Harakmbut | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amu | Guerrero Amuzgo | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amx | Anmatyerre | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ang | Old English (ca. 450-1100) | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| anh | Nend | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| anp | Angika | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| anv | Denya | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aoi | Anindilyakwa | Gunwinyguan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aoj | Mufian | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aom | Ömie | Koiarian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aon | Bumbita Arapesh | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apb | Sa'a | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apc | Levantine Arabic | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| ape | Bukiyip | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apn | Apinayé | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apr | Arop-Lokep | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apu | Apurinã | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apw | Western Apache | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apz | Safeyoka | Angan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ara | Arabic | Unclassified | 2 | 12 | 0 | 0 | 0 | 2 | 2 | 9 | 2 | 0 | 0 | 29 | +| arb | Standard Arabic | Afro-Asiatic | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 2 | 1 | 0 | 0 | 8 | +| are | Western Arrarnta | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| arl | Arabela | Zaparoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| arn | Mapudungun | Araucanian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| arp | Arapaho | Algic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| arq | Algerian Arabic | Afro-Asiatic | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 4 | +| ars | Najdi Arabic | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| ary | Moroccan Arabic | Afro-Asiatic | 1 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 7 | +| arz | Egyptian Arabic | Afro-Asiatic | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | +| asm | Assamese | Indo-European | 5 | 3 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 14 | +| aso | Dano | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ast | Asturian | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| ata | Pele-Ata | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| atb | Zaiwa | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| atd | Ata Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| atg | Ivbie North-Okpela-Arhe | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| att | Pamplona Atta | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| auc | Waorani | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aui | Anuki | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| auy | Awiyaana | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| avt | Au | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| awa | Awadhi | Indo-European | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| awb | Awa (Papua New Guinea) | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| awk | Awabakal | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| awx | Awara | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ayr | Central Aymara | Aymaran | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| azb | South Azerbaijani | Turkic | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| aze | Azerbaijani | Unclassified | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| azg | San Pedro Amuzgos Amuzgo | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| azj | North Azerbaijani | Turkic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| azz | Highland Puebla Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bak | Bashkir | Turkic | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| bam | Bambara | Mande | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | +| ban | Balinese | Austronesian | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| bao | Waimaha | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bba | Baatonum | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bbb | Barai | Koiarian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bbc | Batak Toba | Austronesian | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| bbr | Girawa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bch | Bariai | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bco | Kaluli | Bosavi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bdd | Bunama | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bea | Beaver | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bef | Benabena | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bel | Belarusian | Indo-European | 4 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| bem | Bemba (Zambia) | Atlantic-Congo | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| ben | Bengali | Indo-European | 7 | 9 | 2 | 0 | 0 | 1 | 2 | 6 | 1 | 0 | 0 | 28 | +| beo | Beami | Bosavi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ber | Berber (Other) | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| beu | Blagar | Timor-Alor-Pantar | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bew | Betawi | Austronesian | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| bgc | Haryanvi | Indo-European | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| bgs | Tagabawa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bgt | Bughotu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bhb | Bhili | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bhd | Bhadrawahi | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bhg | Binandere | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bhl | Bimin | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bho | Bhojpuri | Indo-European | 2 | 2 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| bhp | Bima | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| big | Biangai | Kunimaipan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjj | Kanauji | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjk | Barok | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjn | Banjar | Austronesian | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| bjp | Fanamaket | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjr | Binumarien | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjv | Bedjond | Central Sudanic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjz | Baruga | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bkd | Binukid | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bki | Baki | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bkq | Bakairí | Cariban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bkx | Baikeno | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| blw | Balangao | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| blz | Balantak | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bmh | Kein | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bmk | Ghayavi | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bmr | Muinane | Boran | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bmu | Somba-Siawari | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bnp | Bola | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bns | Bundeli | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| boa | Bora | Boran | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bod | Tibetan | Sino-Tibetan | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | +| boj | Anjam | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bon | Bine | Eastern Trans-Fly | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bos | Bosnian | Indo-European | 3 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| box | Buamu | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| boy | Bodo (Central African Republic) | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bpr | Koronadal Blaan | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bps | Sarangani Blaan | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bqc | Boko (Benin) | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bqp | Busa | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bra | Braj | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bre | Breton | Indo-European | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| brx | Bodo (India) | Sino-Tibetan | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| bsj | Bangwinji | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bsn | Barasana-Eduria | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bsp | Baga Sitemu | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bss | Akoose | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bug | Buginese | Austronesian | 2 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| buk | Bugawac | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bul | Bulgarian | Indo-European | 3 | 4 | 1 | 0 | 1 | 1 | 1 | 2 | 0 | 0 | 0 | 13 | +| bus | Bokobaru | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bvd | Baeggu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bvr | Burarra | Maningrida | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bxh | Buhutu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| byr | Baruya | Angan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| byx | Qaqet | Baining | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bzd | Bribri | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bzh | Mapos Buang | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bzj | Belize Kriol English | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| caa | Chortí | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cab | Garifuna | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cac | Chuj | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| caf | Southern Carrier | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cak | Kaqchikel | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cao | Chácobo | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cap | Chipaya | Uru-Chipaya | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| car | Galibi Carib | Cariban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cat | Catalan | Indo-European | 3 | 2 | 2 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| cav | Cavineña | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cax | Chiquitano | Chiquitano | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbc | Carapana | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbi | Chachi | Barbacoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbk | Chavacano | Indo-European | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| cbr | Cashibo-Cacataibo | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbs | Cashinahua | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbt | Chayahuita | Cahuapanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbu | Candoshi-Shapra | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbv | Cacua | Kakua-Nukak | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cco | Comaltepec Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ceb | Cebuano | Austronesian | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | +| cek | Eastern Khumi Chin | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ces | Czech | Indo-European | 4 | 5 | 2 | 0 | 1 | 1 | 1 | 2 | 0 | 0 | 0 | 16 | +| cgc | Kagayanen | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cha | Chamorro | Austronesian | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| chd | Highland Oaxaca Chontal | Tequistlatecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| chf | Tabasco Chontal | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| chk | Chuukese | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| chq | Quiotepec Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| chv | Chuvash | Turkic | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| chz | Ozumacín Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cjk | Chokwe | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| cjo | Ashéninka Pajonal | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cjv | Chuave | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ckb | Central Kurdish | Indo-European | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | +| cle | Lealao Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| clu | Caluyanun | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cme | Cerma | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cmn | Mandarin Chinese | Sino-Tibetan | 4 | 10 | 4 | 0 | 0 | 3 | 4 | 10 | 9 | 0 | 0 | 44 | +| cmo | Central Mnong | Austroasiatic | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| cni | Asháninka | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cnl | Lalana Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cnt | Tepetotutla Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| code | unknown | Unclassified | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 37 | 0 | 0 | 0 | 37 | +| cof | Colorado | Barbacoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| con | Cofán | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cop | Coptic | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cor | Cornish | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cot | Caquinte | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cpa | Palantla Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cpb | Ucayali-Yurúa Ashéninka | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cpc | Ajyíninka Apurucayali | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cpu | Pichis Ashéninka | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cpy | South Ucayali Ashéninka | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| crh | Crimean Tatar | Turkic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| crn | El Nayar Cora | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| crx | Carrier | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| csb | Kashubian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cso | Sochiapam Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| csy | Siyin Chin | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cta | Tataltepec Chatino | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cth | Thaiphum Chin | Bookkeeping | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ctp | Western Highland Chatino | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ctu | Chol | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cub | Cubeo | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cuc | Usila Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cui | Cuiba | Guahiboan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cuk | San Blas Kuna | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cut | Teutila Cuicatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cux | Tepeuxila Cuicatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cwe | Kwere | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cya | Nopala Chatino | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cym | Welsh | Indo-European | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 8 | +| daa | Dangaléat | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dad | Marik | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dah | Gwahatike | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dan | Danish | Indo-European | 5 | 9 | 2 | 0 | 1 | 0 | 1 | 5 | 0 | 0 | 0 | 23 | +| ded | Dedua | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| deu | German | Indo-European | 6 | 14 | 7 | 0 | 1 | 6 | 2 | 18 | 4 | 0 | 0 | 58 | +| dgc | Casiguran Dumagat Agta | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dgr | Dogrib | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dgz | Daga | Dagan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dhg | Dhangu-Djangu | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dif | Dieri | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dik | Southwestern Dinka | Nilotic | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| div | Dhivehi | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dji | Djinang | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| djk | Eastern Maroon Creole | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| djr | Djambarrpuyngu | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dob | Dobu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| doi | Dogri (macrolanguage) | Unclassified | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| dop | Lukpa | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dov | Dombe | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dsb | Lower Sorbian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dtp | Kadazan Dusun | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dwr | Dawro | Ta-Ne-Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dww | Dawawa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dwy | Dhuwaya | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dyu | Dyula | Mande | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| dza | Tunzu | Atlantic-Congo | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dzo | Dzongkha | Sino-Tibetan | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| ebk | Eastern Bontok | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| eko | Koti | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ell | Modern Greek (1453-) | Indo-European | 3 | 6 | 1 | 0 | 1 | 2 | 0 | 3 | 0 | 0 | 0 | 16 | +| emi | Mussau-Emira | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| emp | Northern Emberá | Chocoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| eng | English | Indo-European | 16 | 143 | 16 | 3 | 1 | 8 | 8 | 105 | 13 | 2 | 1 | 316 | +| enq | Enga | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| epo | Esperanto | Artificial Language | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| eri | Ogea | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ese | Ese Ejja | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| esk | Northwest Alaska Inupiatun | Eskimo-Aleut | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| est | Estonian | Uralic | 2 | 2 | 1 | 0 | 1 | 0 | 0 | 2 | 0 | 0 | 0 | 8 | +| etr | Edolo | Bosavi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| eus | Basque | Unclassified | 3 | 2 | 2 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| ewe | Ewe | Atlantic-Congo | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| faa | Fasu | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fai | Faiwol | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fao | Faroese | Indo-European | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 7 | +| far | Fataleka | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fas | Persian | Indo-European | 1 | 4 | 0 | 0 | 0 | 1 | 2 | 9 | 0 | 0 | 0 | 17 | +| ffm | Maasina Fulfulde | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fij | Fijian | Austronesian | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| fil | Filipino | Austronesian | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| fin | Finnish | Uralic | 3 | 5 | 1 | 0 | 1 | 1 | 2 | 5 | 1 | 0 | 0 | 19 | +| fon | Fon | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| for | Fore | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fra | French | Indo-European | 7 | 13 | 8 | 0 | 1 | 5 | 3 | 15 | 4 | 0 | 1 | 57 | +| fry | Western Frisian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fuc | Pulaar | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fue | Borgu Fulfulde | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fuf | Pular | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fuh | Western Niger Fulfulde | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fur | Friulian | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| fuv | Nigerian Fulfulde | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| gah | Alekano | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gai | Borei | Ramu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gam | Kandawo | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gaw | Nobonob | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gaz | West Central Oromo | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| gbm | Garhwali | Indo-European | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| gdn | Umanakaina | Dagan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gdr | Wipi | Eastern Trans-Fly | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| geb | Kire | Ramu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gfk | Patpatar | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ghs | Guhu-Samane | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gla | Scottish Gaelic | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| gle | Irish | Indo-European | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| glg | Galician | Indo-European | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| glk | Gilaki | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| glv | Manx | Indo-European | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gmv | Gamo | Ta-Ne-Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gng | Ngangam | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gnn | Gumatj | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gnw | Western Bolivian Guaraní | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gof | Gofa | Ta-Ne-Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gom | Goan Konkani | Indo-European | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| grc | Ancient Greek (to 1453) | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| grn | Guarani | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| gsw | Swiss German | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gub | Guajajára | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| guh | Guahibo | Guahiboan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gui | Eastern Bolivian Guaraní | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| guj | Gujarati | Indo-European | 6 | 6 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 18 | +| gul | Sea Island Creole English | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gum | Guambiano | Barbacoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gun | Mbyá Guaraní | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| guo | Guayabero | Guahiboan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gup | Gunwinggu | Gunwinyguan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gux | Gourmanchéma | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gvc | Guanano | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gvf | Golin | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gvn | Kuku-Yalanji | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gvs | Gumawana | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gwi | Gwichʼin | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gym | Ngäbere | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gyr | Guarayu | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hat | Haitian | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | +| hau | Hausa | Afro-Asiatic | 4 | 5 | 3 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 14 | +| haw | Hawaiian | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hbo | Ancient Hebrew | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hch | Huichol | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| heb | Hebrew | Afro-Asiatic | 4 | 5 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 11 | +| heg | Helong | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hin | Hindi | Indo-European | 9 | 12 | 2 | 0 | 0 | 1 | 2 | 10 | 2 | 0 | 0 | 38 | +| hix | Hixkaryána | Cariban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hla | Halia | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hlt | Matu Chin | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hmn | Hmong | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hmo | Hiri Motu | Pidgin | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hne | Chhattisgarhi | Indo-European | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| hns | Caribbean Hindustani | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hop | Hopi | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hot | Hote | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hrv | Croatian | Indo-European | 4 | 3 | 1 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 10 | +| hsb | Upper Sorbian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hto | Minica Huitoto | Huitotoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hub | Huambisa | Chicham | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hui | Huli | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hun | Hungarian | Uralic | 5 | 3 | 1 | 0 | 1 | 0 | 0 | 2 | 0 | 0 | 0 | 12 | +| hus | Huastec | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| huu | Murui Huitoto | Huitotoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| huv | San Mateo Del Mar Huave | Huavean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hvn | Sabu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hye | Armenian | Indo-European | 3 | 3 | 1 | 0 | 0 | 1 | 0 | 1 | 0 | 0 | 0 | 9 | +| ian | Iatmul | Ndu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ibo | Igbo | Atlantic-Congo | 3 | 5 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 12 | +| ido | Ido | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ign | Ignaciano | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ikk | Ika | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ikw | Ikwere | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ile | Interlingue | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ilo | Iloko | Austronesian | 2 | 1 | 2 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | +| imo | Imbongu | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ina | Interlingua (International Auxiliary Language Association) | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| inb | Inga | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ind | Indonesian | Austronesian | 6 | 7 | 1 | 0 | 0 | 1 | 1 | 4 | 1 | 0 | 0 | 21 | +| ino | Inoke-Yate | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| iou | Tuma-Irumu | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ipi | Ipili | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| isl | Icelandic | Indo-European | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 9 | +| isn | Isanzu | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ita | Italian | Indo-European | 5 | 9 | 1 | 0 | 1 | 2 | 1 | 5 | 3 | 0 | 0 | 27 | +| iws | Sepik Iwam | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ixl | Ixil | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jac | Popti' | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jae | Yabem | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jao | Yanyuwa | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jav | Javanese | Austronesian | 4 | 7 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 13 | +| jic | Tol | Jicaquean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jid | Bu (Kaduna State) | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jiv | Shuar | Chicham | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jni | Janji | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jpn | Japanese | Japonic | 5 | 8 | 3 | 0 | 0 | 1 | 3 | 13 | 2 | 0 | 0 | 35 | +| jvn | Caribbean Javanese | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kab | Kabyle | Afro-Asiatic | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| kac | Kachin | Sino-Tibetan | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| kam | Kamba (Kenya) | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kan | Kannada | Dravidian | 6 | 7 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 19 | +| kaq | Capanahua | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kas | Kashmiri | Indo-European | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| kat | Georgian | Kartvelian | 4 | 3 | 1 | 0 | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 10 | +| kaz | Kazakh | Turkic | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| kbc | Kadiwéu | Guaicuruan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kbh | Camsá | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kbm | Iwal | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kbp | Kabiyè | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kbq | Kamano | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kdc | Kutu | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kde | Makonde | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kdl | Tsikimba | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kea | Kabuverdianu | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| kek | Kekchí | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ken | Kenyang | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kew | West Kewa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kfg | Kudiya | Dravidian | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kfy | Kumaoni | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kgf | Kube | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kgk | Kaiwá | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kgp | Kaingang | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| khk | Halh Mongolian | Mongolic-Khitan | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| khm | Khmer | Austroasiatic | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| khs | Kasua | Bosavi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| khz | Keapara | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kik | Kikuyu | Atlantic-Congo | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| kin | Kinyarwanda | Atlantic-Congo | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 8 | +| kir | Kirghiz | Turkic | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | +| kiw | Northeast Kiwai | Kiwaian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kiz | Kisi | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kje | Kisar | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kjs | East Kewa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kkc | Odoodee | East Strickland | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kkl | Kosarek Yale | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| klt | Nukna | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| klv | Maskelynes | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmb | Kimbundu | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kmg | Kâte | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmh | Kalam | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmk | Limos Kalinga | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmo | Kwoma | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmr | Northern Kurdish | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| kms | Kamasau | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmu | Kanite | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| knc | Central Kanuri | Saharan | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kne | Kankanaey | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| knf | Mankanya | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| knj | Western Kanjobal | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| knv | Tabo | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kon | Kongo | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kor | Korean | Koreanic | 4 | 8 | 1 | 0 | 1 | 2 | 1 | 9 | 3 | 0 | 0 | 29 | +| kos | Kosraean | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpf | Komba | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpg | Kapingamarangi | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpj | Karajá | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpr | Korafe-Yegha | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpw | Kobon | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpx | Mountain Koiali | Koiarian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kqa | Mum | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kqc | Doromu-Koki | Manubaran | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kqf | Kakabai | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kql | Kyenele | Yuat | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kqw | Kandas | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| krc | Karachay-Balkar | Turkic | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ksd | Kuanua | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ksj | Uare | Kwalean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ksr | Borong | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ktm | Kurti | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kto | Kuot | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kud | 'Auhelawa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kue | Kuman (Papua New Guinea) | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kup | Kunimaipa | Kunimaipan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kur | Kurdish | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kvg | Kuni-Boazi | Anim | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kvn | Border Kuna | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kwd | Kwaio | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kwf | Kwara'ae | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kwi | Awa-Cuaiquer | Barbacoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kwj | Kwanga | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kyc | Kyaka | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kyf | Kouya | Kru | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kyg | Keyagana | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kyq | Kenga | Central Sudanic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kyz | Kayabí | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kze | Kosena | Bookkeeping | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kzj | Coastal Kadazan | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lac | Lacandon | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lao | Lao | Tai-Kadai | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | +| lat | Latin | Indo-European | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| lav | Latvian | Indo-European | 1 | 2 | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| lbb | Label | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lbk | Central Bontok | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lcm | Tungag | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| leu | Kara (Papua New Guinea) | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lex | Luang | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lfn | Lingua Franca Nova | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lgl | Wala | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lid | Nyindrou | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lif | Limbu | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lij | Ligurian | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| lim | Limburgan | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| lin | Lingala | Atlantic-Congo | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| lit | Lithuanian | Indo-European | 4 | 1 | 1 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| llg | Lole | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lmo | Lombard | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| ltg | Latgalian | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| ltz | Luxembourgish | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| lua | Luba-Lulua | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| lug | Ganda | Atlantic-Congo | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| luo | Luo (Kenya and Tanzania) | Nilotic | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | +| lus | Lushai | Sino-Tibetan | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| lvs | Standard Latvian | Unclassified | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | +| lww | Lewo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| maa | San Jerónimo Tecóatl Mazatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mad | Madurese | Austronesian | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| mag | Magahi | Indo-European | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| mai | Maithili | Indo-European | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| maj | Jalapa De Díaz Mazatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mak | Makasar | Austronesian | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| mal | Malayalam | Dravidian | 7 | 7 | 2 | 0 | 0 | 0 | 0 | 2 | 1 | 0 | 0 | 19 | +| mam | Mam | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| maq | Chiquihuitlán Mazatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mar | Marathi | Indo-European | 7 | 6 | 2 | 0 | 0 | 1 | 0 | 2 | 2 | 0 | 0 | 20 | +| mau | Huautla Mazatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mav | Sateré-Mawé | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| max | North Moluccan Malay | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| maz | Central Mazahua | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbb | Western Bukidnon Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbc | Macushi | Cariban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbh | Mangseng | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbj | Nadëb | Naduhup | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbl | Maxakalí | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbs | Sarangani Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbt | Matigsalug Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mca | Maca | Mataguayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcb | Machiguenga | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcd | Sharanahua | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcf | Matsés | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mco | Coatlán Mixe | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcp | Makaa | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcq | Ese | Koiarian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcr | Menya | Angan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mdy | Male (Ethiopia) | Ta-Ne-Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| med | Melpa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mee | Mengen | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mek | Mekeo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| meq | Merey | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| met | Mato | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| meu | Motu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mey | Hassaniyya | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mgc | Morokodo | Central Sudanic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mgh | Makhuwa-Meetto | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mgw | Matumbi | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mhl | Mauwake | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mhr | Eastern Mari | Uralic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mib | Atatláhuca Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mic | Mi'kmaq | Algic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mie | Ocotepec Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mig | San Miguel El Grande Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mih | Chayuco Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mil | Peñoles Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| min | Minangkabau | Austronesian | 3 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 9 | +| mio | Pinotepa Nacional Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mir | Isthmus Mixe | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mit | Southern Puebla Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| miz | Coatzospan Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mjc | San Juan Colorado Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mkd | Macedonian | Indo-European | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | +| mkj | Mokilese | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mkl | Mokole | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mkn | Kupang Malay | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mks | Silacayoapan Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mle | Manambu | Ndu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mlg | Malagasy | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mlh | Mape | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mlp | Bargam | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mlt | Maltese | Afro-Asiatic | 2 | 2 | 2 | 0 | 2 | 0 | 0 | 1 | 0 | 0 | 0 | 9 | +| mmo | Mangga Buang | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mmx | Madak | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mna | Mbula | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mni | Manipuri | Sino-Tibetan | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| mon | Mongolian | Unclassified | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| mop | Mopán Maya | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mos | Mossi | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| mox | Molima | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mph | Maung | Iwaidjan Proper | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mpj | Martu Wangka | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mpm | Yosondúa Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mpp | Migabac | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mps | Dadibi | Teberan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mpt | Mian | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mpx | Misima-Panaeati | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mqb | Mbuko | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mqj | Mamasa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mri | Maori | Austronesian | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | +| msa | Malay (macrolanguage) | Unclassified | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| msb | Masbatenyo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| msc | Sankaran Maninka | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| msk | Mansaka | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| msm | Agusan Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| msy | Aruamu | Ramu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mti | Maiwa (Papua New Guinea) | Dagan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mto | Totontepec Mixe | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mui | Musi | Austronesian | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| mup | Malvi | Indo-European | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| mux | Bo-Ung | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| muy | Muyang | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mva | Manam | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mvn | Minaveha | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mwc | Are | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mwe | Mwera (Chimwera) | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mwf | Murrinh-Patha | Southern Daly | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mwp | Kala Lagaw Ya | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mwr | Marwari | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mxb | Tezoatlán Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mxp | Tlahuitoltepec Mixe | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mxq | Juquila Mixe | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mxt | Jamiltepec Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mya | Burmese | Sino-Tibetan | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 9 | +| myk | Mamara Senoufo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| myu | Mundurukú | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| myw | Muyuw | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| myy | Macuna | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mzz | Maiadomu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nab | Southern Nambikuára | Nambiquaran | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| naf | Nabak | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nak | Nakanai | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nas | Naasioi | South Bougainville | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nbl | South Ndebele | Unclassified | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nbq | Nggem | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nca | Iyo | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nch | Central Huasteca Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ncj | Northern Puebla Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ncl | Michoacán Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ncu | Chumburung | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nde | North Ndebele | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ndg | Ndengereko | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ndj | Ndamba | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nds | Low German | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nep | Nepali (macrolanguage) | Unclassified | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| nfa | Dhao | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ngp | Ngulu | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ngu | Guerrero Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhe | Eastern Huasteca Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhg | Tetelcingo Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhi | Zacatlán-Ahuacatlán-Tepetzintla Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nho | Takuu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhr | Naro | Khoe-Kwadi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhu | Noone | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhw | Western Huasteca Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhy | Northern Oaxaca Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nif | Nek | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nii | Nii | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nij | Ngaju | Austronesian | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| nin | Ninzo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nko | Nkonya | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nld | Dutch | Indo-European | 6 | 6 | 1 | 0 | 1 | 0 | 1 | 2 | 2 | 0 | 0 | 19 | +| nlg | Gela | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nna | Nyangumarta | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nno | Norwegian Nynorsk | Unclassified | 4 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 8 | +| nnq | Ngindo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| noa | Woun Meu | Chocoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nob | Norwegian Bokmål | Unclassified | 4 | 7 | 5 | 0 | 0 | 0 | 0 | 3 | 0 | 0 | 0 | 19 | +| noe | Nimadi | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nop | Numanggang | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nor | Norwegian | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 3 | +| not | Nomatsiguenga | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nou | Ewage-Notu | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nov | Novial | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| npi | Nepali (individual language) | Indo-European | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| npl | Southeastern Puebla Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nqo | N'Ko | Artificial Language | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| nsn | Nehan | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nso | Pedi | Atlantic-Congo | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | +| nss | Nali | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ntj | Ngaanyatjarra | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ntp | Northern Tepehuan | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ntu | Natügu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nus | Nuer | Nilotic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| nuy | Nunggubuyu | Gunwinyguan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nvm | Namiae | Koiarian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nwi | Southwest Tanna | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nya | Nyanja | Atlantic-Congo | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | +| nys | Nyungar | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nyu | Nyungwe | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| obo | Obo Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| oci | Occitan (post 1500) | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| okv | Orokaiva | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| omw | South Tairora | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ong | Olo | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ons | Ono | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ood | Tohono O'odham | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| opm | Oksapmin | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ori | Oriya (macrolanguage) | Unclassified | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| orm | Oromo | Unclassified | 1 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| orv | Old Russian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ory | Odia | Indo-European | 5 | 4 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 15 | +| ote | Mezquital Otomi | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| otm | Eastern Highland Otomi | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| otn | Tenango Otomi | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| otq | Querétaro Otomi | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ots | Estado de México Otomi | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pab | Parecís | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pad | Paumarí | Arawan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pag | Pangasinan | Austronesian | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| pah | Tenharim | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pam | Pampanga | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pan | Panjabi | Indo-European | 6 | 6 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 18 | +| pao | Northern Paiute | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pap | Papiamento | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| pbt | Southern Pashto | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| pcm | Nigerian Pidgin | Indo-European | 1 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| pes | Iranian Persian | Indo-European | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | +| pib | Yine | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pio | Piapoco | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pir | Piratapuyo | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| piu | Pintupi-Luritja | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pjt | Pitjantjatjara | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pls | San Marcos Tlacoyalco Popoloca | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| plt | Plateau Malagasy | Austronesian | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| plu | Palikúr | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pma | Paama | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pms | Piemontese | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| poe | San Juan Atzingo Popoloca | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| poh | Poqomchi' | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| poi | Highland Popoluca | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pol | Polish | Indo-European | 4 | 11 | 4 | 0 | 1 | 4 | 0 | 18 | 4 | 0 | 0 | 46 | +| pon | Pohnpeian | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| por | Portuguese | Indo-European | 4 | 9 | 1 | 0 | 2 | 2 | 1 | 5 | 3 | 0 | 0 | 27 | +| poy | Pogolo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ppo | Folopa | Teberan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| prf | Paranan | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pri | Paicî | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| prs | Dari | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| ptp | Patep | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ptu | Bambam | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pus | Pushto | Unclassified | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| pwg | Gapapaiwa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qub | Huallaga Huánuco Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| quc | K'iche' | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| quf | Lambayeque Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| quh | South Bolivian Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qul | North Bolivian Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qup | Southern Pastaza Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| quy | Ayacucho Quechua | Quechuan | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| qvc | Cajamarca Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qve | Eastern Apurímac Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvh | Huamalíes-Dos de Mayo Huánuco Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvm | Margos-Yarowilca-Lauricocha Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvn | North Junín Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvs | San Martín Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvw | Huaylla Wanca Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvz | Northern Pastaza Quichua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qwh | Huaylas Ancash Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qxh | Panao Huánuco Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qxn | Northern Conchucos Ancash Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qxo | Southern Conchucos Ancash Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rai | Ramoaaina | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| raj | Rajasthani | Unclassified | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| reg | Kara (Tanzania) | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rej | Rejang | Austronesian | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| rgu | Ringgou | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rkb | Rikbaktsa | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rmc | Carpathian Romani | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rmy | Vlax Romani | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rom | Romany | Unclassified | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| ron | Romanian | Indo-European | 5 | 6 | 1 | 0 | 1 | 0 | 1 | 3 | 1 | 0 | 0 | 18 | +| roo | Rotokas | North Bougainville | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rop | Kriol | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| row | Dela-Oenale | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rro | Waima | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ruf | Luguru | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rug | Roviana | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| run | Rundi | Atlantic-Congo | 1 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| rus | Russian | Indo-European | 5 | 13 | 6 | 0 | 2 | 4 | 2 | 16 | 4 | 0 | 0 | 52 | +| rwo | Rawa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sab | Buglere | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sag | Sango | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| sah | Yakut | Turkic | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| san | Sanskrit | Indo-European | 5 | 3 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 10 | +| sat | Santali | Austroasiatic | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| sbe | Saliba | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sbk | Safwa | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sbs | Subiya | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| scn | Sicilian | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| sco | Scots | Indo-European | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| seh | Sena | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sey | Secoya | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sgb | Mag-antsi Ayta | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sgz | Sursurunga | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| shi | Tachelhit | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| shj | Shatt | Dajuic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| shn | Shan | Tai-Kadai | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| shp | Shipibo-Conibo | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sim | Mende (Papua New Guinea) | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sin | Sinhala | Indo-European | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | +| sja | Epena | Chocoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| slk | Slovak | Indo-European | 3 | 4 | 1 | 0 | 1 | 0 | 0 | 3 | 0 | 0 | 0 | 12 | +| sll | Salt-Yui | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| slv | Slovenian | Indo-European | 3 | 4 | 1 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 10 | +| smk | Bolinao | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| smo | Samoan | Austronesian | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| sna | Shona | Atlantic-Congo | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| snc | Sinaugoro | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| snd | Sindhi | Indo-European | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| snn | Siona | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| snp | Siane | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| snx | Sam | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sny | Saniyo-Hiyewe | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| som | Somali | Afro-Asiatic | 3 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 9 | +| soq | Kanasi | Dagan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sot | Southern Sotho | Atlantic-Congo | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | +| soy | Miyobe | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| spa | Spanish | Indo-European | 4 | 13 | 4 | 0 | 1 | 2 | 2 | 13 | 4 | 0 | 0 | 43 | +| spl | Selepet | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| spm | Akukem | Ramu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| spp | Supyire Senoufo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sps | Saposa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| spy | Sabaot | Nilotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sqi | Albanian | Unclassified | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| srd | Sardinian | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| sri | Siriano | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| srm | Saramaccan | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| srn | Sranan Tongo | Indo-European | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| srp | Serbian | Indo-European | 4 | 1 | 1 | 0 | 0 | 0 | 1 | 2 | 0 | 0 | 0 | 9 | +| srq | Sirionó | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ssd | Siroi | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ssg | Seimat | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ssw | Swati | Atlantic-Congo | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | +| ssx | Samberigi | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| stp | Southeastern Tepehuan | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sua | Sulka | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sue | Suena | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sun | Sundanese | Austronesian | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 9 | +| sus | Susu | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| suz | Sunwar | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| svk | Slovakian Sign Language | Sign Language | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| swa | Swahili (macrolanguage) | Atlantic-Congo | 1 | 7 | 2 | 0 | 0 | 1 | 1 | 3 | 0 | 0 | 0 | 15 | +| swe | Swedish | Indo-European | 4 | 8 | 3 | 0 | 1 | 1 | 1 | 4 | 0 | 0 | 0 | 22 | +| swg | Swabian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| swh | Swahili (individual language) | Atlantic-Congo | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | +| swp | Suau | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sxb | Suba | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| szl | Silesian | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| tac | Lowland Tarahumara | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tah | Tahitian | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| taj | Eastern Tamang | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tam | Tamil | Dravidian | 7 | 7 | 2 | 0 | 0 | 1 | 0 | 3 | 1 | 0 | 0 | 21 | +| taq | Tamasheq | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| tat | Tatar | Turkic | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| tav | Tatuyo | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| taw | Tai | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tbc | Takia | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tbf | Mandara | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tbg | North Tairora | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tbo | Tawala | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tbz | Ditammari | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tca | Ticuna | Ticuna-Yuri | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tcs | Torres Strait Creole | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tcz | Thado Chin | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tdt | Tetun Dili | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tee | Huehuetla Tepehua | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tel | Telugu | Dravidian | 7 | 7 | 2 | 0 | 0 | 0 | 1 | 5 | 2 | 0 | 0 | 24 | +| ter | Tereno | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tet | Tetum | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tew | Tewa (USA) | Kiowa-Tanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tfr | Teribe | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tgk | Tajik | Indo-European | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | +| tgl | Tagalog | Austronesian | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| tgo | Sudest | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tgp | Tangoa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tha | Thai | Tai-Kadai | 4 | 8 | 1 | 0 | 0 | 1 | 1 | 6 | 0 | 0 | 0 | 21 | +| tif | Tifal | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tim | Timbe | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tir | Tigrinya | Afro-Asiatic | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| tiw | Tiwi | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tiy | Tiruray | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tke | Takwane | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tku | Upper Necaxa Totonac | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tlf | Telefol | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tmd | Haruai | Piawi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tna | Tacana | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tnc | Tanimuca-Retuarã | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tnk | Kwamera | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tnn | North Tanna | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tnp | Whitesands | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| toc | Coyutla Totonac | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tod | Toma | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tof | Gizrra | Eastern Trans-Fly | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| toj | Tojolabal | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ton | Tonga (Tonga Islands) | Austronesian | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| too | Xicotepec De Juárez Totonac | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| top | Papantla Totonac | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tos | Highland Totonac | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tpa | Taupota | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tpi | Tok Pisin | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| tpt | Tlachichilco Tepehua | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tpz | Tinputz | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| trc | Copala Triqui | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tsn | Tswana | Atlantic-Congo | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | +| tso | Tsonga | Atlantic-Congo | 1 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | +| tsw | Tsishingini | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ttc | Tektiteko | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tte | Bwanabwana | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tuc | Mutu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tue | Tuyuca | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tuf | Central Tunebo | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tuk | Turkmen | Turkic | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| tum | Tumbuka | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| tuo | Tucano | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tur | Turkish | Turkic | 4 | 7 | 1 | 0 | 0 | 2 | 0 | 3 | 2 | 0 | 0 | 19 | +| tvk | Southeast Ambrym | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| twi | Twi | Unclassified | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| txq | Tii | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| txu | Kayapó | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tyv | Tuvinian | Turkic | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tzj | Tz'utujil | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tzl | Talossan | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tzm | Central Atlas Tamazight | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| tzo | Tzotzil | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ubr | Ubir | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ubu | Umbu-Ungu | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| udu | Uduk | Koman | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| uig | Uighur | Turkic | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| ukr | Ukrainian | Indo-European | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | +| uli | Ulithian | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ulk | Meriam Mir | Eastern Trans-Fly | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| umb | Umbundu | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| upv | Uripiv-Wala-Rano-Atchin | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ura | Urarina | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| urb | Urubú-Kaapor | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| urd | Urdu | Indo-European | 7 | 8 | 2 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 19 | +| uri | Urim | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| urt | Urat | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| urw | Sop | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| usa | Usarufa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| usp | Uspanteco | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| uvh | Uri | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| uvl | Lote | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| uzb | Uzbek | Unclassified | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| uzn | Northern Uzbek | Turkic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | +| vec | Venetian | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| ven | Venda | Atlantic-Congo | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| vid | Vidunda | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| vie | Vietnamese | Austroasiatic | 5 | 6 | 1 | 0 | 0 | 1 | 0 | 5 | 0 | 0 | 0 | 18 | +| viv | Iduna | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| vmy | Ayautla Mazatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| waj | Waffa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wal | Wolaytta | Ta-Ne-Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wap | Wapishana | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| war | Waray (Philippines) | Austronesian | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | +| wat | Kaninuwa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wbi | Vwanji | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wbp | Warlpiri | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wed | Wedau | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wer | Weri | Kunimaipan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wim | Wik-Mungkan | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wiu | Wiru | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wiv | Vitu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wln | Walloon | Indo-European | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wmt | Walmajarri | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wmw | Mwani | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wnc | Wantoat | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wnu | Usan | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wol | Wolof | Atlantic-Congo | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | +| wos | Hanga Hundi | Ndu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wrk | Garrwa | Garrwan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wro | Worrorra | Worrorran | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wrs | Waris | Border | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wsk | Waskia | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wuu | Wu Chinese | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wuv | Wuvulu-Aua | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xav | Xavánte | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xbi | Kombio | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xed | Hdi | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xho | Xhosa | Atlantic-Congo | 3 | 3 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 10 | +| xla | Kamula | Kamula-Elevala | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xnn | Northern Kankanay | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xon | Konkomba | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xsi | Sio | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xtd | Diuxi-Tilantongo Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xtm | Magdalena Peñasco Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yaa | Yaminahua | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yad | Yagua | Peba-Yagua | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yal | Yalunka | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yap | Yapese | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yaq | Yaqui | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yby | Yaweyuha | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ycn | Yucuna | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ydd | Eastern Yiddish | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| yid | Yiddish | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yka | Yakan | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yle | Yele | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yml | Iamalele | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yon | Yongkom | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yor | Yoruba | Atlantic-Congo | 4 | 5 | 3 | 0 | 0 | 0 | 1 | 3 | 0 | 0 | 0 | 16 | +| yrb | Yareba | Yareban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yre | Yaouré | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yss | Yessan-Mayo | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yue | Yue Chinese | Sino-Tibetan | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| yuj | Karkar-Yuri | Pauwasi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yut | Yopno | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yuw | Yau (Morobe Province) | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yva | Yawa | Yawa-Saweru | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zaa | Sierra de Juárez Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zab | Western Tlacolula Valley Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zac | Ocotlán Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zad | Cajonos Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zai | Isthmus Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zaj | Zaramo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zam | Miahuatlán Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zao | Ozolotepec Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zap | Zapotec | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zar | Rincón Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zas | Santo Domingo Albarradas Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zat | Tabaa Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zav | Yatzachi Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zaw | Mitla Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zca | Coatecas Altas Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zga | Kinga | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zho | Chinese | Unclassified | 2 | 2 | 1 | 0 | 0 | 1 | 1 | 13 | 0 | 0 | 0 | 20 | +| zia | Zia | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ziw | Zigula | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zlm | Malay (individual language) | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zos | Francisco León Zoque | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpc | Choapan Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpl | Lachixío Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpm | Mixtepec Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpo | Amatlán Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpq | Zoogocho Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpu | Yalálag Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpv | Chichicapan Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpz | Texmelucan Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zsm | Standard Malay | Austronesian | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | +| zsr | Southern Rincon Zapotec | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ztq | Quioquitani-Quierí Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zty | Yatee Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zul | Zulu | Atlantic-Congo | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | +| zyp | Zyphe Chin | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| Total | None | None | None | 1394 | 795 | 304 | 3 | 28 | 67 | 51 | 473 | 85 | 2 | 2 | +>>>>>>> main
diff --git a/mteb/__init__.py b/mteb/__init__.py index bef5f7408d..6de017b1f1 100644 --- a/mteb/__init__.py +++ b/mteb/__init__.py @@ -3,9 +3,10 @@ from importlib.metadata import version from mteb.benchmarks.benchmarks import ( - MTEB_MAIN_EN, + MTEB_ENG_CLASSIC, MTEB_MAIN_RU, MTEB_RETRIEVAL_LAW, + MTEB_RETRIEVAL_MEDICAL, MTEB_RETRIEVAL_WITH_INSTRUCTIONS, CoIR, ) @@ -15,15 +16,16 @@ from mteb.overview import TASKS_REGISTRY, get_task, get_tasks from .benchmarks.benchmarks import Benchmark -from .benchmarks.get_benchmark import get_benchmark, get_benchmarks +from .benchmarks.get_benchmark import BENCHMARK_REGISTRY, get_benchmark, get_benchmarks __version__ = version("mteb") # fetch version from install metadata __all__ = [ - "MTEB_MAIN_EN", + "MTEB_ENG_CLASSIC", "MTEB_MAIN_RU", "MTEB_RETRIEVAL_LAW", + "MTEB_RETRIEVAL_MEDICAL", "MTEB_RETRIEVAL_WITH_INSTRUCTIONS", "CoIR", "TASKS_REGISTRY", @@ -37,4 +39,5 @@ "get_benchmark", "get_benchmarks", "BenchmarkResults", + "BENCHMARK_REGISTRY", ] diff --git a/mteb/abstasks/AbsTask.py b/mteb/abstasks/AbsTask.py index 5fa983bbe5..1ec1ebc4fc 100644 --- a/mteb/abstasks/AbsTask.py +++ b/mteb/abstasks/AbsTask.py @@ -1,10 +1,12 @@ from __future__ import annotations +import json import logging import random from abc import ABC, abstractmethod from collections.abc import Sequence -from typing import Any, TypedDict +from copy import copy +from typing import Any import datasets import numpy as np @@ -14,7 +16,7 @@ from sklearn.preprocessing import MultiLabelBinarizer from mteb.abstasks.stratification import _iterative_train_test_split -from mteb.abstasks.TaskMetadata import HFSubset, TaskMetadata +from mteb.abstasks.TaskMetadata import DescriptiveStatistics, HFSubset, TaskMetadata from mteb.encoder_interface import Encoder from mteb.languages import LanguageScripts @@ -53,19 +55,15 @@ def _multilabel_subsampling( return dataset_dict -class DescriptiveStatistics(TypedDict): - """Class for descriptive statistics.""" - - pass - - class AbsTask(ABC): metadata: TaskMetadata + abstask_prompt: str | None = None _eval_splits: list[str] | None = None superseded_by: None | str = None dataset: dict[HFSubset, DatasetDict] | None = None # type: ignore data_loaded: bool = False is_multilingual: bool = False + hf_subsets: list[HFSubset] | None = None def __init__(self, seed: int = 42, **kwargs: Any): self.save_suffix = kwargs.get("save_suffix", "") @@ -76,11 +74,11 @@ def __init__(self, seed: int = 42, **kwargs: Any): torch.manual_seed(self.seed) torch.cuda.manual_seed_all(self.seed) - def check_if_dataset_is_superseeded(self): - """Check if the dataset is superseeded by a newer version""" + def check_if_dataset_is_superseded(self): + """Check if the dataset is superseded by a newer version""" if self.superseded_by: logger.warning( - f"Dataset '{self.metadata.name}' is superseeded by '{self.superseded_by}', you might consider using the newer version of the dataset." + f"Dataset '{self.metadata.name}' is superseded by '{self.superseded_by}', you might consider using the newer version of the dataset." ) def dataset_transform(self): @@ -93,17 +91,18 @@ def evaluate( self, model: Encoder, split: str = "test", + subsets_to_run: list[HFSubset] | None = None, *, encode_kwargs: dict[str, Any] = {}, **kwargs: Any, ) -> dict[HFSubset, ScoresDict]: """Evaluates a Sentence Embedding Model on the task. - Returns a dict (that can be serialized to json). Args: model: Sentence embedding method. Implements a encode(sentences) method, that encodes sentences and returns a numpy matrix with the sentence embeddings split: Which datasplit to be used. + subsets_to_run: List of HFSubsets to evaluate. If None, all subsets are evaluated. encode_kwargs: Additional keyword arguments that are passed to the model's `encode` method. kwargs: Additional keyword arguments that are passed to the _evaluate_subset method. """ @@ -113,7 +112,13 @@ def evaluate( self.dataset: dict[HFSubset, DatasetDict] scores = {} - hf_subsets = list(self.dataset.keys()) if self.is_multilingual else ["default"] + if self.hf_subsets is None: + hf_subsets = list(self.dataset.keys()) + else: + hf_subsets = copy(self.hf_subsets) + + if subsets_to_run is not None: # allow overwrites of pre-filtering + hf_subsets = [s for s in hf_subsets if s in subsets_to_run] for hf_subset in hf_subsets: logger.info( @@ -194,38 +199,47 @@ def load_data(self, **kwargs): self.data_loaded = True def calculate_metadata_metrics( - self, + self, overwrite_results: bool = False ) -> dict[str, DescriptiveStatistics | dict[str, DescriptiveStatistics]]: + if self.metadata.descriptive_stat_path.exists() and not overwrite_results: + logger.info("Loading metadata descriptive statistics from cache.") + return self.metadata.descriptive_stats + self.load_data() - all_details = {} - pbar_split = tqdm.tqdm( - self.metadata_dict["eval_splits"], desc="Processing Splits..." - ) + descriptive_stats = {} + hf_subset_stat = "hf_subset_descriptive_stats" + eval_splits = self.metadata.eval_splits + if self.metadata.type in ["Classification", "MultilabelClassification"]: + eval_splits += ["train"] + + pbar_split = tqdm.tqdm(eval_splits, desc="Processing Splits...") for split in pbar_split: pbar_split.set_postfix_str(f"Split: {split}") - print(f"Processing metadata for split {split}") + logger.info(f"Processing metadata for split {split}") if self.is_multilingual: - all_details[split] = self._calculate_metrics_from_split( + descriptive_stats[split] = self._calculate_metrics_from_split( split, compute_overall=True ) - all_details[split]["hf_subset_descriptive_stats"] = {} + descriptive_stats[split][hf_subset_stat] = {} pbar_subsets = tqdm.tqdm( - self.metadata.eval_langs, desc="Processing Languages..." + self.metadata.hf_subsets_to_langscripts, + desc="Processing Languages...", ) for hf_subset in pbar_subsets: - pbar_subsets.set_postfix_str(f"Language: {hf_subset}") - print(f"Processing metadata for language {hf_subset}") + pbar_subsets.set_postfix_str(f"Huggingface subset: {hf_subset}") + logger.info(f"Processing metadata for subset {hf_subset}") split_details = self._calculate_metrics_from_split(split, hf_subset) - all_details[split]["hf_subset_descriptive_stats"][hf_subset] = ( - split_details - ) + descriptive_stats[split][hf_subset_stat][hf_subset] = split_details else: split_details = self._calculate_metrics_from_split(split) - all_details[split] = split_details + descriptive_stats[split] = split_details + + with self.metadata.descriptive_stat_path.open("w") as f: + json.dump(descriptive_stats, f, indent=4) - return all_details + return descriptive_stats @abstractmethod def _calculate_metrics_from_split( @@ -240,12 +254,8 @@ def metadata_dict(self) -> dict[str, Any]: @property def languages(self) -> list[str]: """Returns the languages of the task""" - # check if self.hf_subsets is set - if self.is_multilingual and hasattr(self, "hf_subsets"): - assert isinstance( - self.metadata.eval_langs, dict - ), "eval_langs must be dict for multilingual tasks" - eval_langs = self.metadata.eval_langs + if self.hf_subsets: + eval_langs = self.metadata.hf_subsets_to_langscripts languages = [] for lang in self.hf_subsets: @@ -263,31 +273,43 @@ def filter_eval_splits(self, eval_splits: list[str] | None) -> AbsTask: return self def filter_languages( - self, languages: list[str] | None, script: list[str] | None = None + self, + languages: list[str] | None, + script: list[str] | None = None, + hf_subsets: list[HFSubset] | None = None, + exclusive_language_filter: bool = False, ) -> AbsTask: """Filter the languages of the task. Args: languages: list of languages to filter the task by can be either a 3-letter langauge code (e.g. "eng") or also include the script (e.g. "eng-Latn") - script: list of scripts to filter the task by. Will be ignored if language code specified the script. If None, all scripts are included. + script: A list of scripts to filter the task by. Will be ignored if language code specified the script. If None, all scripts are included. If the language code does not specify the script the intersection of the language and script will be used. + hf_subsets: A list of huggingface subsets to filter on. This is useful if a dataset have multiple subsets containing the desired language, + but you only want to test on one. An example is STS22 which e.g. have both "en" and "de-en" which both contains English. + exclusive_language_filter: Some datasets contains more than one language e.g. for STS22 the subset "de-en" contain eng and deu. If + exclusive_language_filter is set to False both of these will be kept, but if set to True only those that contains all the languages + specified will be kept. """ lang_scripts = LanguageScripts.from_languages_and_scripts(languages, script) subsets_to_keep = [] - if not isinstance(self.metadata.eval_langs, dict): - self.hf_subsets = self.metadata.eval_langs - return self - - for hf_subset, langs in self.metadata.eval_langs.items(): - for langscript in langs: - if lang_scripts.contains_language( - langscript - ) or lang_scripts.contains_script(langscript): + for hf_subset, langs in self.metadata.hf_subsets_to_langscripts.items(): + if (hf_subsets is not None) and (hf_subset not in hf_subsets): + continue + if exclusive_language_filter is False: + for langscript in langs: + if lang_scripts.contains_language( + langscript + ) or lang_scripts.contains_script(langscript): + subsets_to_keep.append(hf_subset) + break + + if exclusive_language_filter is True and languages: + if lang_scripts.contains_languages(langs): subsets_to_keep.append(hf_subset) - break self.hf_subsets = subsets_to_keep return self diff --git a/mteb/abstasks/AbsTaskBitextMining.py b/mteb/abstasks/AbsTaskBitextMining.py index ea4667d9de..1c373cc2f7 100644 --- a/mteb/abstasks/AbsTaskBitextMining.py +++ b/mteb/abstasks/AbsTaskBitextMining.py @@ -9,7 +9,8 @@ from ..evaluation.evaluators import BitextMiningEvaluator from ..load_results.task_results import HFSubset, ScoresDict -from .AbsTask import AbsTask, DescriptiveStatistics +from .AbsTask import AbsTask +from .TaskMetadata import DescriptiveStatistics logger = logging.getLogger(__name__) @@ -19,13 +20,32 @@ class BitextDescriptiveStatistics(DescriptiveStatistics): Attributes: num_samples: number of samples in the dataset. + number_of_characters: Total number of symbols in the dataset. + unique_pairs: Number of duplicate pairs + + min_sentence1_length: Minimum length of sentence1 average_sentence1_length: Average length of sentence1 + max_sentence1_length: Maximum length of sentence1 + unique_sentence1: Number of duplicates in sentence1 + + min_sentence2_length: Minimum length of sentence2 average_sentence2_length: Average length of sentence2 + max_sentence2_length: Maximum length of sentence2 """ num_samples: int + number_of_characters: int + unique_pairs: int + + min_sentence1_length: int average_sentence1_length: float + max_sentence1_length: int + unique_sentence1: int + + min_sentence2_length: int average_sentence2_length: float + max_sentence2_length: int + unique_sentence2: int class AbsTaskBitextMining(AbsTask): @@ -39,6 +59,7 @@ class AbsTaskBitextMining(AbsTask): """ parallel_subsets = False + abstask_prompt = "Retrieve parallel sentences." def __init__(self, **kwargs): super().__init__(**kwargs) @@ -46,7 +67,8 @@ def __init__(self, **kwargs): def evaluate( self, model: Encoder, - split: str, + split: str = "test", + subsets_to_run: list[HFSubset] | None = None, *, encode_kwargs: dict[str, Any] = {}, **kwargs, @@ -56,6 +78,10 @@ def evaluate( hf_subsets = list(self.dataset) if self.is_multilingual else ["default"] + # If subsets_to_run is specified, filter the hf_subsets accordingly + if subsets_to_run is not None: + hf_subsets = [s for s in hf_subsets if s in subsets_to_run] + scores = {} if self.parallel_subsets: scores = self._evaluate_subset( @@ -149,11 +175,24 @@ def _calculate_metrics_from_split( sent_1, sent_2 = pairs_cols[0] sentence1 = self.dataset[split][sent_1] sentence2 = self.dataset[split][sent_2] - total_s1_len = sum([len(s1) for s1 in sentence1]) - total_s2_len = sum([len(s2) for s2 in sentence2]) - + s1_len = [len(s1) for s1 in sentence1] + s2_len = [len(s2) for s2 in sentence2] + total_s1_len = sum(s1_len) + total_s2_len = sum(s2_len) + + unique_pairs = len(set(zip(sentence1, sentence2))) + unique_sentence1 = len(set(sentence1)) + unique_sentence2 = len(set(sentence2)) return BitextDescriptiveStatistics( - average_sentence1_length=total_s1_len / len(sentence1), - average_sentence2_length=total_s2_len / len(sentence2), num_samples=len(sentence1), + number_of_characters=total_s1_len + total_s2_len, + unique_pairs=unique_pairs, + min_sentence1_length=min(s1_len), + average_sentence1_length=sum(s1_len) / len(sentence1), + max_sentence1_length=max(s1_len), + unique_sentence1=unique_sentence1, + min_sentence2_length=min(s2_len), + average_sentence2_length=total_s2_len / len(sentence2), + max_sentence2_length=max(s2_len), + unique_sentence2=unique_sentence2, ) diff --git a/mteb/abstasks/AbsTaskClassification.py b/mteb/abstasks/AbsTaskClassification.py index eacd25af42..eac41856e8 100644 --- a/mteb/abstasks/AbsTaskClassification.py +++ b/mteb/abstasks/AbsTaskClassification.py @@ -5,7 +5,6 @@ from typing import Any import numpy as np -import tqdm from mteb.encoder_interface import Encoder @@ -15,7 +14,8 @@ logRegClassificationEvaluator, ) from ..load_results.task_results import HFSubset, ScoresDict -from .AbsTask import AbsTask, DescriptiveStatistics +from .AbsTask import AbsTask +from .TaskMetadata import DescriptiveStatistics logger = logging.getLogger(__name__) @@ -25,13 +25,27 @@ class ClassificationDescriptiveStatistics(DescriptiveStatistics): Attributes: num_samples: number of samples in the dataset. + number_of_characters: Total number of symbols in the dataset. + num_texts_in_train: Number of texts in the train split + + min_text_length: Minimum length of text average_text_length: Average length of text + max_text_length: Maximum length of text + unique_text: Number of unique texts + unique_labels: Number of unique labels labels: dict of label frequencies """ num_samples: int + number_of_characters: int + num_texts_in_train: int | None + + min_text_length: int average_text_length: float + max_text_length: int + unique_text: int + unique_labels: int labels: dict[str, dict[str, int]] @@ -44,13 +58,19 @@ class AbsTaskClassification(AbsTask): must contain the following columns: text: str label: int + + Attributes: + samples_per_label: Number of samples to use pr. label. These samples are embedded and a classifier is fit using the labels and samples. + """ + abstask_prompt = "Classify user passages." + samples_per_label: int = 8 + def __init__( self, method: str = "logReg", n_experiments: int | None = None, - samples_per_label: int | None = None, k: int = 3, **kwargs, ): @@ -63,11 +83,6 @@ def __init__( if n_experiments is not None else self.metadata_dict.get("n_experiments", 10) ) - self.samples_per_label: int = ( # type: ignore - samples_per_label - if samples_per_label is not None - else self.metadata_dict.get("samples_per_label", 8) - ) # kNN parameters self.k = k @@ -80,6 +95,7 @@ def evaluate( model, eval_split: str = "test", train_split: str = "train", + subsets_to_run: list[HFSubset] | None = None, *, encode_kwargs: dict[str, Any] = {}, **kwargs, @@ -89,6 +105,8 @@ def evaluate( scores = {} hf_subsets = list(self.dataset) if self.is_multilingual else ["default"] + if subsets_to_run is not None: + hf_subsets = [s for s in hf_subsets if s in subsets_to_run] for hf_subset in hf_subsets: logger.info( @@ -199,65 +217,43 @@ def _undersample_data(self, X, y, samples_per_label: int, idxs=None): label_counter[y[i]] += 1 return X_sampled, y_sampled, idxs - def calculate_metadata_metrics( - self, - ) -> dict[ - str, - ClassificationDescriptiveStatistics - | dict[str, ClassificationDescriptiveStatistics], - ]: - self.load_data() - - # same function from parent class, but added explicitly train to splits - - all_details = {} - pbar_split = tqdm.tqdm( - self.metadata.eval_splits + ["train"], desc="Processing Splits..." - ) - for split in pbar_split: - pbar_split.set_postfix_str(f"Split: {split}") - logger.info(f"Processing metadata for split {split}") - if self.is_multilingual: - all_details[split] = self._calculate_metrics_from_split( - split, compute_overall=True - ) - all_details[split]["hf_subset_descriptive_stats"] = {} - - pbar_subset = tqdm.tqdm( - self.metadata.eval_langs, desc="Processing Languages..." - ) - for hf_subset in pbar_subset: - pbar_subset.set_postfix_str(f"Language: {hf_subset}") - logger.info(f"Processing metadata for language {hf_subset}") - split_details = self._calculate_metrics_from_split(split, hf_subset) - all_details[split][hf_subset] = split_details - else: - split_details = self._calculate_metrics_from_split(split) - all_details[split] = split_details - - return all_details - def _calculate_metrics_from_split( self, split: str, hf_subset: str | None = None, compute_overall: bool = False ) -> ClassificationDescriptiveStatistics: + train_text = [] if hf_subset: text = self.dataset[hf_subset][split]["text"] label = self.dataset[hf_subset][split]["label"] + if split != "train": + train_text = self.dataset[hf_subset]["train"]["text"] elif compute_overall: text = [] label = [] for hf_subset in self.metadata.eval_langs: text.extend(self.dataset[hf_subset][split]["text"]) label.extend(self.dataset[hf_subset][split]["label"]) + if split != "train": + train_text.extend(self.dataset[hf_subset]["train"]["text"]) else: text = self.dataset[split]["text"] label = self.dataset[split]["label"] + if split != "train": + train_text = self.dataset["train"]["text"] - total_text_len = sum([len(t) for t in text]) + text_len = [len(t) for t in text] + total_text_len = sum(text_len) label_count = Counter(label) + num_texts_in_train = ( + len(set(text) & set(train_text)) if split != "train" else None + ) return ClassificationDescriptiveStatistics( num_samples=len(text), + number_of_characters=total_text_len, + num_texts_in_train=num_texts_in_train, + min_text_length=min(text_len), average_text_length=total_text_len / len(text), + max_text_length=max(text_len), + unique_text=len(set(text)), unique_labels=len(label_count), labels={ str(label): {"count": count} for label, count in label_count.items() diff --git a/mteb/abstasks/AbsTaskClustering.py b/mteb/abstasks/AbsTaskClustering.py index 8aaff2a484..3b5d0f492d 100644 --- a/mteb/abstasks/AbsTaskClustering.py +++ b/mteb/abstasks/AbsTaskClustering.py @@ -12,7 +12,8 @@ from mteb.load_results.task_results import ScoresDict from ..evaluation.evaluators import ClusteringEvaluator -from .AbsTask import AbsTask, DescriptiveStatistics +from .AbsTask import AbsTask +from .TaskMetadata import DescriptiveStatistics logger = logging.getLogger(__name__) @@ -22,15 +23,32 @@ class ClusteringDescriptiveStatistics(DescriptiveStatistics): Attributes: num_samples: number of samples in the dataset. + number_of_characters: Total number of symbols in the dataset. + + min_text_length: Minimum length of text average_text_length: Average length of text + max_text_length: Maximum length of text + unique_texts: Number of unique texts + + min_labels_per_text: Minimum number of labels per text average_labels_per_text: Average number of labels per text + max_labels_per_text: Maximum number of labels per text unique_labels: Number of unique labels labels: dict of label frequencies """ num_samples: int + number_of_characters: int + + min_text_length: int average_text_length: float + max_text_length: int + unique_texts: int + + min_labels_per_text: int average_labels_per_text: float + max_labels_per_text: int + unique_labels: int labels: dict[str, dict[str, int]] @@ -44,6 +62,8 @@ class AbsTaskClustering(AbsTask): labels: list of str """ + abstask_prompt = "Identify categories in user passages." + def __init__(self, **kwargs): super().__init__(**kwargs) @@ -91,7 +111,11 @@ def _calculate_metrics_from_split( sentences = self.dataset[split]["sentences"] labels = self.dataset[split]["labels"] - total_text_len = sum([len(t) for t in sentences]) + text_len = [len(t) for t in sentences] + all_sentences = [] + for s in sentences: + all_sentences.extend(s) + total_text_len = sum(text_len) total_labels = [] for label in labels: if isinstance(label, list): @@ -101,8 +125,14 @@ def _calculate_metrics_from_split( label_counter = Counter(total_labels) return ClusteringDescriptiveStatistics( num_samples=len(sentences), + number_of_characters=total_text_len, + min_text_length=min(text_len), average_text_length=total_text_len / len(sentences), + max_text_length=max(text_len), + unique_texts=len(set(all_sentences)), + min_labels_per_text=min(label_counter.values()), average_labels_per_text=len(total_labels) / len(sentences), + max_labels_per_text=max(label_counter.values()), unique_labels=len(label_counter), labels={ str(label): { diff --git a/mteb/abstasks/AbsTaskClusteringFast.py b/mteb/abstasks/AbsTaskClusteringFast.py index 22ed1bb693..40e36d29e2 100644 --- a/mteb/abstasks/AbsTaskClusteringFast.py +++ b/mteb/abstasks/AbsTaskClusteringFast.py @@ -13,10 +13,10 @@ from sklearn.metrics.cluster import v_measure_score from mteb.encoder_interface import Encoder -from mteb.normalize_embeddings import normalize_embeddings_to_numpy from ..load_results.task_results import HFSubset -from .AbsTask import AbsTask, DescriptiveStatistics +from .AbsTask import AbsTask +from .TaskMetadata import DescriptiveStatistics logger = logging.getLogger(__name__) @@ -84,15 +84,31 @@ class ClusteringFastDescriptiveStatistics(DescriptiveStatistics): Attributes: num_samples: number of samples in the dataset. + number_of_characters: Total number of symbols in the dataset. + + min_text_length: Minimum length of text average_text_length: Average length of text + max_text_length: Maximum length of text + unique_texts: Number of unique texts + + min_labels_per_text: Minimum number of labels per text average_labels_per_text: Average number of labels per text + max_labels_per_text: Maximum number of labels per text unique_labels: Number of unique labels labels: dict of label frequencies """ num_samples: int + number_of_characters: int + + min_text_length: int average_text_length: float + max_text_length: int + unique_texts: int + + min_labels_per_text: int average_labels_per_text: float + max_labels_per_text: int unique_labels: int labels: dict[str, dict[str, int]] @@ -125,6 +141,7 @@ class AbsTaskClusteringFast(AbsTask): n_clusters: int = 10 k_mean_batch_size: int = 512 max_depth = None + abstask_prompt = "Identify categories in user passages." def __init__(self, **kwargs): super().__init__(**kwargs) @@ -174,12 +191,10 @@ def _evaluate_subset( ) downsampled_dataset = dataset.select(example_indices) # type: ignore - embeddings = normalize_embeddings_to_numpy( - model.encode( - downsampled_dataset["sentences"], # type: ignore - task_name=self.metadata.name, - **encode_kwargs, - ) + embeddings = model.encode( + downsampled_dataset["sentences"], # type: ignore + task_name=self.metadata.name, + **encode_kwargs, ) labels = [] @@ -225,7 +240,8 @@ def _calculate_metrics_from_split( sentences = self.dataset[split]["sentences"] labels = self.dataset[split]["labels"] - total_text_len = sum([len(t) for t in sentences]) + text_len = [len(t) for t in sentences] + total_text_len = sum(text_len) total_labels = [] for label in labels: if isinstance(label, list): @@ -235,8 +251,13 @@ def _calculate_metrics_from_split( label_counter = Counter(total_labels) return ClusteringFastDescriptiveStatistics( num_samples=len(sentences), + number_of_characters=total_text_len, + min_text_length=min(text_len), average_text_length=total_text_len / len(sentences), + max_text_length=max(text_len), + min_labels_per_text=min(label_counter.values()), average_labels_per_text=len(total_labels) / len(sentences), + max_labels_per_text=max(label_counter.values()), unique_labels=len(label_counter), labels={ str(label): { diff --git a/mteb/abstasks/AbsTaskInstructionRetrieval.py b/mteb/abstasks/AbsTaskInstructionRetrieval.py index a0107abc75..ed24a7cc87 100644 --- a/mteb/abstasks/AbsTaskInstructionRetrieval.py +++ b/mteb/abstasks/AbsTaskInstructionRetrieval.py @@ -16,8 +16,9 @@ from ..evaluation.evaluators.InstructionRetrievalEvaluator import ( InstructionRetrievalEvaluator, ) -from .AbsTask import AbsTask, DescriptiveStatistics +from .AbsTask import AbsTask from .AbsTaskRetrieval import HFDataLoader +from .TaskMetadata import DescriptiveStatistics logger = logging.getLogger(__name__) @@ -36,6 +37,7 @@ def __init__( qrels_file: str = "", streaming: bool = False, keep_in_memory: bool = False, + trust_remote_code: bool = False, ): self.corpus = {} self.queries = {} @@ -68,6 +70,7 @@ def __init__( self.qrels_file = qrels_file self.streaming = streaming self.keep_in_memory = keep_in_memory + self.trust_remote_code = trust_remote_code def load( self, split="test" @@ -222,24 +225,72 @@ class InstructionRetrievalDescriptiveStatistics(DescriptiveStatistics): """Descriptive statistics for Instruction Retrieval tasks Attributes: + num_samples: Number of samples num_queries: Number of queries num_docs: Number of documents + number_of_characters: Total number of symbols in the dataset + + min_document_length: Minimum length of documents average_document_length: Average length of documents + max_document_length: Maximum length of documents + unique_docs: Number of unique documents + + min_query_length: Minimum length of queries average_query_length: Average length of queries + max_query_length: Maximum length of queries + unique_queries: Number of unique queries + + min_instruction_length: Minimum length of instructions average_instruction_length: Average length of instructions + max_instruction_length: Maximum length of instructions + unique_instructions: Number of unique instructions + + min_changed_instruction_length: Minimum length of changed instructions average_changed_instruction_length: Average length of changed instructions + max_changed_instruction_length: Maximum length of changed instructions + unique_changed_instructions: Number of unique changed instructions + + min_average_relevant_docs_per_query: Minimum number of relevant docs per query average_relevant_docs_per_query: Average number of relevant docs per query + max_average_relevant_docs_per_query: Maximum number of relevant docs per query + + min_average_top_ranked_per_query: Minimum number of top ranked docs per query average_top_ranked_per_query: Average number of top ranked docs per query + max_average_top_ranked_per_query: Maximum number of top ranked docs per query """ + num_samples: int num_queries: int num_docs: int + number_of_characters: int + + min_document_length: int average_document_length: float + max_document_length: int + unique_docs: int + + min_query_length: int average_query_length: float + max_query_length: int + unique_queries: int + + min_instruction_length: int average_instruction_length: float + max_instruction_length: int + unique_instructions: int + + min_changed_instruction_length: int average_changed_instruction_length: float + max_changed_instruction_length: int + unique_changed_instructions: int + + min_average_relevant_docs_per_query: float average_relevant_docs_per_query: float + max_average_relevant_docs_per_query: float + + min_average_top_ranked_per_query: float average_top_ranked_per_query: float + max_average_top_ranked_per_query: float class AbsTaskInstructionRetrieval(AbsTask): @@ -258,6 +309,8 @@ class AbsTaskInstructionRetrieval(AbsTask): See https://arxiv.org/abs/2403.15246 for more details """ + abstask_prompt = "Retrieve text based on user query." + def __init__( self, **kwargs, @@ -463,6 +516,7 @@ def evaluate( self, model: Encoder, split: str = "test", + subsets_to_run: list[str] | None = None, *, encode_kwargs: dict[str, Any] = {}, **kwargs, @@ -475,7 +529,11 @@ def evaluate( ) scores = {} if self.is_multilingual: - for lang in self.hf_subsets: + hf_subsets = self.hf_subsets + if subsets_to_run is not None: + hf_subsets = [s for s in hf_subsets if s in subsets_to_run] + + for lang in hf_subsets: logger.info(f"Language: {lang}") scores[lang] = self._evaluate_subset_lang( retriever, @@ -658,43 +716,70 @@ def _calculate_metrics_from_split( changed_instructions = self.changed_instructions[split] top_ranked = self.top_ranked[split] - total_corpus_len = sum( - [len(doc.get("title", "")) + len(doc["text"]) for doc in corpus.values()] - ) - total_queries_len = sum([len(query) for query in queries.values()]) - total_instructions_len = sum( - [len(instruction) for instruction in og_instructions.values()] - ) - total_changed_instructions_len = sum( - [len(instruction) for instruction in changed_instructions.values()] - ) - num_qrels_non_zero = sum( + corpus_combined = [ + doc.get("title", "") + doc["text"] for doc in corpus.values() + ] + corpus_len = [len(doc) for doc in corpus_combined] + total_corpus_len = sum(corpus_len) + + queries_len = [len(query) for query in queries.values()] + total_queries_len = sum(queries_len) + instructions_len = [ + len(instruction) for instruction in og_instructions.values() + ] + total_instructions_len = sum(instructions_len) + changed_instructions_len = [ + len(instruction) for instruction in changed_instructions.values() + ] + total_changed_instructions_len = sum(changed_instructions_len) + qrels_non_zero = [ sum(1 for doc_id in docs if docs[doc_id] != 0) for docs in relevant_docs.values() - ) + ] + num_qrels_non_zero = sum(qrels_non_zero) qrels_per_doc = num_qrels_non_zero / len(relevant_docs) if len(queries) else 0 + ranked_per_query = [len(docs) for docs in top_ranked.values()] top_ranked_per_query = ( - sum(len(docs) for docs in top_ranked.values()) / len(queries) - if len(queries) - else 0 + sum(ranked_per_query) / len(queries) if len(queries) else 0 ) return InstructionRetrievalDescriptiveStatistics( + num_samples=len(queries) + len(corpus), num_docs=len(corpus), num_queries=len(queries), + number_of_characters=total_corpus_len + + total_queries_len + + total_instructions_len + + total_changed_instructions_len, + min_document_length=min(corpus_len), average_document_length=( total_corpus_len / len(corpus) if len(corpus) else 0 ), + max_document_length=max(corpus_len), + unique_docs=len(set(corpus_combined)), + min_query_length=min(queries_len), average_query_length=( total_queries_len / len(queries) if len(queries) else 0 ), + max_query_length=max(queries_len), + unique_queries=len(set(queries.values())), + min_instruction_length=min(instructions_len), average_instruction_length=( total_instructions_len / len(queries) if len(queries) else 0 ), + max_instruction_length=max(instructions_len), + unique_instructions=len(set(og_instructions.values())), + min_changed_instruction_length=min(changed_instructions_len), average_changed_instruction_length=( total_changed_instructions_len / len(queries) if len(queries) else 0 ), + max_changed_instruction_length=max(changed_instructions_len), + unique_changed_instructions=len(set(changed_instructions.values())), + min_average_relevant_docs_per_query=min(qrels_non_zero), average_relevant_docs_per_query=qrels_per_doc, + max_average_relevant_docs_per_query=max(qrels_non_zero), + min_average_top_ranked_per_query=min(ranked_per_query), average_top_ranked_per_query=top_ranked_per_query, + max_average_top_ranked_per_query=max(ranked_per_query), ) diff --git a/mteb/abstasks/AbsTaskMultilabelClassification.py b/mteb/abstasks/AbsTaskMultilabelClassification.py index f31cd90bb3..cbcb125021 100644 --- a/mteb/abstasks/AbsTaskMultilabelClassification.py +++ b/mteb/abstasks/AbsTaskMultilabelClassification.py @@ -13,10 +13,10 @@ from sklearn.preprocessing import MultiLabelBinarizer from mteb.encoder_interface import Encoder -from mteb.normalize_embeddings import normalize_embeddings_to_numpy from ..load_results.task_results import HFSubset, ScoresDict -from .AbsTask import AbsTask, DescriptiveStatistics +from .AbsTask import AbsTask +from .TaskMetadata import DescriptiveStatistics logger = logging.getLogger(__name__) @@ -46,15 +46,33 @@ class MultilabelClassificationDescriptiveStatistics(DescriptiveStatistics): Attributes: num_samples: number of samples in the dataset. + number_of_characters: Total number of symbols in the dataset. + number_texts_in_train: Number of texts in the train split + + min_text_length: Minimum length of text average_text_length: Average length of text + max_text_length: Maximum length of text + unique_texts: Number of unique texts + + min_labels_per_text: Minimum number of labels per text average_label_per_text: Average number of labels per text + max_labels_per_text: Maximum number of labels per text unique_labels: Number of unique labels labels: dict of label frequencies """ num_samples: int + number_of_characters: int + number_texts_in_train: int | None + + min_text_length: int average_text_length: float + max_text_length: int + unique_texts: int + + min_labels_per_text: int average_label_per_text: float + max_labels_per_text: int unique_labels: int labels: dict[str, dict[str, int]] @@ -66,14 +84,19 @@ class AbsTaskMultilabelClassification(AbsTask): self.load_data() must generate a huggingface dataset with a split matching self.metadata_dict["eval_splits"], and assign it to self.dataset. It must contain the following columns: text: str label: list[list[int]] + + Attributes: + samples_per_label: Number of samples to use pr. label. These samples are embedded and a classifier is fit using the labels and samples. + """ classifier = KNeighborsClassifier(n_neighbors=5) + abstask_prompt = "Classify user passages." + samples_per_label: int = 8 def __init__( self, n_experiments=None, - samples_per_label=None, batch_size=32, **kwargs, ): @@ -82,9 +105,7 @@ def __init__( # Bootstrap parameters self.n_experiments = n_experiments or getattr(self, "n_experiments", 10) - self.samples_per_label = samples_per_label or getattr( - self, "samples_per_label", 8 - ) + # Run metadata validation by instantiating addressing the attribute # This is quite hacky. Ideally, this would be done in the constructor of # each concrete task, but then we have to duplicate the __init__ method's @@ -100,6 +121,7 @@ def evaluate( model: Encoder, eval_split: str = "test", train_split: str = "train", + subsets_to_run: list[HFSubset] | None = None, *, encode_kwargs: dict[str, Any] = {}, **kwargs: Any, @@ -109,6 +131,9 @@ def evaluate( scores = {} hf_subsets = list(self.dataset) if self.is_multilingual else ["default"] + # If subsets_to_run is specified, filter the hf_subsets accordingly + if subsets_to_run is not None: + hf_subsets = [s for s in hf_subsets if s in subsets_to_run] for hf_subset in hf_subsets: logger.info( @@ -162,12 +187,10 @@ def _evaluate_subset( unique_train_indices = list(set(itertools.chain.from_iterable(train_samples))) unique_train_sentences = train_split.select(unique_train_indices)["text"] - _unique_train_embeddings = normalize_embeddings_to_numpy( - model.encode( - unique_train_sentences, - task_name=self.metadata.name, - **encode_kwargs, - ) + _unique_train_embeddings = model.encode( + unique_train_sentences, + task_name=self.metadata.name, + **encode_kwargs, ) unique_train_embeddings = dict( zip(unique_train_indices, _unique_train_embeddings) @@ -184,12 +207,10 @@ def _evaluate_subset( except ValueError: logger.warning("Couldn't subsample, continuing with the entire test set.") - X_test = normalize_embeddings_to_numpy( - model.encode( - test_text, - task_name=self.metadata.name, - **encode_kwargs, - ) + X_test = model.encode( + test_text, + task_name=self.metadata.name, + **encode_kwargs, ) for i_experiment, sample_indices in enumerate(train_samples): logger.info( @@ -229,29 +250,48 @@ def _undersample_data_indices(self, y, samples_per_label, idxs=None): def _calculate_metrics_from_split( self, split: str, hf_subset: str | None = None, compute_overall: bool = False ) -> MultilabelClassificationDescriptiveStatistics: + train_text = [] if hf_subset: text = self.dataset[hf_subset][split]["text"] label = self.dataset[hf_subset][split]["label"] + if split != "train": + train_text = self.dataset[hf_subset]["train"]["text"] elif compute_overall: text = [] label = [] for hf_subset in self.metadata.eval_langs: text.extend(self.dataset[hf_subset][split]["text"]) label.extend(self.dataset[hf_subset][split]["label"]) + if split != "train": + train_text.extend(self.dataset[hf_subset]["train"]["text"]) else: text = self.dataset[split]["text"] label = self.dataset[split]["label"] + if split != "train": + train_text = self.dataset["train"]["text"] - total_text_len = sum(len(t) for t in text) - total_label_len = sum(len(l) for l in label) + text_len = [len(t) for t in text] + total_text_len = sum(text_len) + label_len = [len(l) for l in label] + total_label_len = sum(label_len) total_labels = [] for l in label: total_labels.extend(l if len(l) > 0 else [None]) label_count = Counter(total_labels) + num_texts_in_train = ( + len(set(text) & set(train_text)) if split != "train" else None + ) return MultilabelClassificationDescriptiveStatistics( + num_samples=len(text), + number_of_characters=total_text_len, + number_texts_in_train=num_texts_in_train, + min_text_length=min(text_len), average_text_length=total_text_len / len(text), + max_text_length=max(text_len), + unique_texts=len(set(text)), + min_labels_per_text=min(label_len), average_label_per_text=total_label_len / len(label), - num_samples=len(text), + max_labels_per_text=max(label_len), unique_labels=len(label_count), labels={ str(label): { diff --git a/mteb/abstasks/AbsTaskPairClassification.py b/mteb/abstasks/AbsTaskPairClassification.py index 8ce32a8fcc..82ba128c28 100644 --- a/mteb/abstasks/AbsTaskPairClassification.py +++ b/mteb/abstasks/AbsTaskPairClassification.py @@ -8,7 +8,8 @@ from ..encoder_interface import Encoder from ..evaluation.evaluators import PairClassificationEvaluator from ..load_results.task_results import ScoresDict -from .AbsTask import AbsTask, DescriptiveStatistics +from .AbsTask import AbsTask +from .TaskMetadata import DescriptiveStatistics logger = logging.getLogger(__name__) @@ -18,15 +19,35 @@ class PairClassificationDescriptiveStatistics(DescriptiveStatistics): Attributes: num_samples: number of samples in the dataset. - avg_sentence1_len: Average length of sentence1 - avg_sentence2_len: Average length of sentence2 + number_of_characters: Total number of symbols in the dataset. + + min_sentence1_length: Minimum length of sentence1 + avg_sentence1_length: Average length of sentence1 + max_sentence1_length: Maximum length of sentence1 + unique_sentence1: Number of unique sentence + + min_sentence2_length: Minimum length of sentence2 + avg_sentence2_length: Average length of sentence2 + max_sentence2_length: Maximum length of sentence2 + unique_sentence2: Number of unique sentence + unique_labels: Number of unique labels labels: dict of label frequencies """ num_samples: int - avg_sentence1_len: float - avg_sentence2_len: float + number_of_characters: int + + min_sentence1_length: int + avg_sentence1_length: float + max_sentence1_length: int + unique_sentence1: int + + min_sentence2_length: int + avg_sentence2_length: float + max_sentence2_length: int + unique_sentence2: int + unique_labels: int labels: dict[str, dict[str, int]] @@ -42,6 +63,8 @@ class AbsTaskPairClassification(AbsTask): labels: list[int] """ + abstask_prompt = "Retrieve text that are semantically similar to the given text." + def __init__(self, **kwargs): super().__init__(**kwargs) @@ -104,13 +127,22 @@ def _calculate_metrics_from_split( dataset["labels"][0] if len(dataset["labels"]) == 1 else dataset["labels"] ) - total_sentence1_len = sum([len(sentence) for sentence in sentence1]) - total_sentence2_len = sum([len(sentence) for sentence in sentence2]) + sentence1_len = [len(sentence) for sentence in sentence1] + total_sentence1_len = sum(sentence1_len) + sentence2_len = [len(sentence) for sentence in sentence2] + total_sentence2_len = sum(sentence2_len) label_count = Counter(labels) return PairClassificationDescriptiveStatistics( num_samples=len(sentence1), - avg_sentence1_len=total_sentence1_len / len(sentence1), - avg_sentence2_len=total_sentence2_len / len(sentence2), + number_of_characters=total_sentence1_len + total_sentence2_len, + min_sentence1_length=min(sentence1_len), + avg_sentence1_length=total_sentence1_len / len(sentence1), + max_sentence1_length=max(sentence1_len), + unique_sentence1=len(set(sentence1)), + min_sentence2_length=min(sentence2_len), + avg_sentence2_length=total_sentence2_len / len(sentence2), + max_sentence2_length=max(sentence2_len), + unique_sentence2=len(set(sentence2)), unique_labels=len(set(labels)), labels={ str(label): {"count": count} for label, count in label_count.items() diff --git a/mteb/abstasks/AbsTaskReranking.py b/mteb/abstasks/AbsTaskReranking.py index bcbc4571d5..ab00a53a39 100644 --- a/mteb/abstasks/AbsTaskReranking.py +++ b/mteb/abstasks/AbsTaskReranking.py @@ -8,7 +8,8 @@ from mteb.load_results.task_results import ScoresDict from ..evaluation.evaluators import RerankingEvaluator -from .AbsTask import AbsTask, DescriptiveStatistics +from .AbsTask import AbsTask +from .TaskMetadata import DescriptiveStatistics class RerankingDescriptiveStatistics(DescriptiveStatistics): @@ -16,19 +17,45 @@ class RerankingDescriptiveStatistics(DescriptiveStatistics): Attributes: num_samples: number of samples in the dataset. + number_of_characters: Total number of symbols in the dataset. num_positive: Number of positive examples num_negative: Number of negative examples - avg_query_len: Average length of queries - avg_positive_len: Average length of positive examples - avg_negative_len: Average length of negative examples + + min_query_length: Minimum length of queries + avg_query_length: Average length of queries + max_query_length: Maximum length of queries + unique_query: Number of unique queries + + min_positive_length: Minimum length of positive examples + avg_positive_length: Average length of positive examples + max_positive_length: Maximum length of positive examples + unique_positive: Number of unique positive examples + + min_negative_length: Minimum length of negative examples + avg_negative_length: Average length of negative examples + max_negative_length: Maximum length of negative examples + unique_negative: Number of unique negative examples """ num_samples: int + number_of_characters: int num_positive: int num_negative: int - avg_query_len: float - avg_positive_len: float - avg_negative_len: float + + min_query_length: int + avg_query_length: float + max_query_length: int + unique_query: int + + min_positive_length: int + avg_positive_length: float + max_positive_length: int + unique_positive: int + + min_negative_length: int + avg_negative_length: float + max_negative_length: int + unique_negative: int class AbsTaskReranking(AbsTask): @@ -40,6 +67,8 @@ class AbsTaskReranking(AbsTask): negative: list[str] """ + abstask_prompt = "Retrieve text based on user query." + def __init__(self, **kwargs): super().__init__(**kwargs) @@ -70,29 +99,59 @@ def _calculate_metrics_from_split( ) -> RerankingDescriptiveStatistics: if hf_subset: query = self.dataset[hf_subset][split]["query"] - positive = self.dataset[hf_subset][split]["positive"] - negative = self.dataset[hf_subset][split]["negative"] + positive = transform_reranking_data( + self.dataset[hf_subset][split]["positive"] + ) + negative = transform_reranking_data( + self.dataset[hf_subset][split]["negative"] + ) elif compute_overall: query = [] positive = [] negative = [] for hf_subset in self.metadata.eval_langs: query.extend(self.dataset[hf_subset][split]["query"]) - positive.extend(self.dataset[hf_subset][split]["positive"]) - negative.extend(self.dataset[hf_subset][split]["negative"]) + positive.extend( + transform_reranking_data(self.dataset[hf_subset][split]["positive"]) + ) + negative.extend( + transform_reranking_data(self.dataset[hf_subset][split]["negative"]) + ) else: query = self.dataset[split]["query"] - positive = self.dataset[split]["positive"] - negative = self.dataset[split]["negative"] - - total_len_query = sum([len(q) for q in query]) - total_len_positive = sum([len(p) for p in positive]) - total_len_negative = sum([len(n) for n in negative]) + positive = transform_reranking_data(self.dataset[split]["positive"]) + negative = transform_reranking_data(self.dataset[split]["negative"]) + + len_query = [len(q) for q in query] + total_len_query = sum(len_query) + len_positive = [len(p) for p in positive] + total_len_positive = sum(len_positive) + len_negative = [len(n) for n in negative] + total_len_negative = sum(len_negative) return RerankingDescriptiveStatistics( num_samples=len(query), + number_of_characters=total_len_query + + total_len_positive + + total_len_negative, num_positive=len(positive), num_negative=len(negative), - avg_query_len=total_len_query / len(query), - avg_positive_len=total_len_positive / len(positive), - avg_negative_len=total_len_negative / len(negative), + min_query_length=min(len_query), + avg_query_length=total_len_query / len(query), + max_query_length=max(len_query), + unique_query=len(set(query)), + min_positive_length=min(len_positive), + avg_positive_length=total_len_positive / len(positive), + max_positive_length=max(len_positive), + unique_positive=len(set(positive)), + min_negative_length=min(len_negative), + avg_negative_length=total_len_negative / len(negative), + max_negative_length=max(len_negative), + unique_negative=len(set(negative)), ) + + +def transform_reranking_data(data: list[list[str]] | list[str]) -> list[str]: + """Transforms a list of lists of strings into a list of strings""" + if isinstance(data[0], str): + return data + return [item for sublist in data for item in sublist] diff --git a/mteb/abstasks/AbsTaskRetrieval.py b/mteb/abstasks/AbsTaskRetrieval.py index 6fa901c791..8a780658f7 100644 --- a/mteb/abstasks/AbsTaskRetrieval.py +++ b/mteb/abstasks/AbsTaskRetrieval.py @@ -14,7 +14,8 @@ from ..evaluation.evaluators import RetrievalEvaluator from ..load_results.task_results import ScoresDict -from .AbsTask import AbsTask, DescriptiveStatistics +from .AbsTask import AbsTask +from .TaskMetadata import DescriptiveStatistics logger = logging.getLogger(__name__) @@ -33,6 +34,7 @@ def __init__( qrels_file: str = "", streaming: bool = False, keep_in_memory: bool = False, + trust_remote_code: bool = False, ): self.corpus = {} self.queries = {} @@ -62,6 +64,7 @@ def __init__( self.qrels_file = qrels_file self.streaming = streaming self.keep_in_memory = keep_in_memory + self.trust_remote_code = trust_remote_code @staticmethod def check(fIn: str, ext: str): @@ -124,6 +127,7 @@ def _load_corpus(self): "corpus", keep_in_memory=self.keep_in_memory, streaming=self.streaming, + trust_remote_code=self.trust_remote_code, ) else: corpus_ds = load_dataset( @@ -151,6 +155,7 @@ def _load_queries(self): "queries", keep_in_memory=self.keep_in_memory, streaming=self.streaming, + trust_remote_code=self.trust_remote_code, ) else: queries_ds = load_dataset( @@ -173,6 +178,7 @@ def _load_qrels(self, split): self.hf_repo_qrels, keep_in_memory=self.keep_in_memory, streaming=self.streaming, + trust_remote_code=self.trust_remote_code, )[split] else: qrels_ds = load_dataset( @@ -196,18 +202,46 @@ class RetrievalDescriptiveStatistics(DescriptiveStatistics): """Descriptive statistics for Retrieval Attributes: - num_queries: number of samples in the dataset + num_samples: Number of queries and documents + num_queries: number of queries in the dataset + num_documents: Number of documents + number_of_characters: Total number of symbols in the dataset + + min_document_length: Minimum length of documents average_document_length: Average length of documents + max_document_length: Maximum length of documents + unique_documents: Number of unique documents + + min_query_length: Minimum length of queries average_query_length: Average length of queries - num_documents: Number of documents + max_query_length: Maximum length of queries + unique_queries: Number of unique queries + + min_relevant_docs_per_query: Minimum number of relevant documents per query average_relevant_docs_per_query: Average number of relevant documents per query + max_relevant_docs_per_query: Maximum number of relevant documents per query + unique_relevant_docs: Number of unique relevant documents """ + num_samples: int num_queries: int + num_documents: int + number_of_characters: int + + min_document_length: int average_document_length: float + max_document_length: int + unique_documents: int + + min_query_length: int average_query_length: float - num_documents: int + max_query_length: int + unique_queries: int + + min_relevant_docs_per_query: int average_relevant_docs_per_query: float + max_relevant_docs_per_query: int + unique_relevant_docs: int class AbsTaskRetrieval(AbsTask): @@ -230,6 +264,7 @@ class AbsTaskRetrieval(AbsTask): """ ignore_identical_ids: bool = False + abstask_prompt = "Retrieve text based on user query." def __init__(self, **kwargs): super().__init__(**kwargs) @@ -248,6 +283,9 @@ def load_data(self, **kwargs): hf_repo_qrels=hf_repo_qrels, streaming=False, keep_in_memory=False, + trust_remote_code=self.metadata_dict["dataset"].get( + "trust_remote_code", False + ), ).load(split=split) # Conversion from DataSet queries = {query["id"]: query["text"] for query in queries} @@ -266,6 +304,7 @@ def evaluate( self, model, split: str = "test", + subsets_to_run: list[HFSubset] | None = None, *, encode_kwargs: dict[str, Any] = {}, **kwargs, @@ -279,6 +318,8 @@ def evaluate( scores = {} hf_subsets = list(self.hf_subsets) if self.is_multilingual else ["default"] + if subsets_to_run is not None: + hf_subsets = [s for s in hf_subsets if s in subsets_to_run] for hf_subset in hf_subsets: logger.info(f"Subset: {hf_subset}") @@ -422,35 +463,48 @@ def _calculate_metrics_from_split( num_documents = len(corpus) num_queries = len(queries) - # number of qrels that are not 0 - num_qrels_non_zero = sum( - sum(1 for doc_id in docs if docs[doc_id] != 0) - for docs in relevant_docs.values() - ) - qrels_per_doc = num_qrels_non_zero / len(relevant_docs) if num_queries else 0 + # create a list of number of relevant docs per query + qrels_lengths = [ + len(relevant_docs[qid]) for qid in relevant_docs if qid in queries + ] + num_qrels = sum(qrels_lengths) + qrels_per_doc = num_qrels / len(relevant_docs) if num_queries else 0 + unique_qrels = len({doc for qid in relevant_docs for doc in relevant_docs[qid]}) return RetrievalDescriptiveStatistics( - average_document_length=doc_len, - average_query_length=query_len, - num_documents=num_documents, + number_of_characters=sum(query_len) + sum(doc_len), + num_samples=num_documents + num_queries, num_queries=num_queries, + num_documents=num_documents, + min_document_length=min(doc_len), + average_document_length=sum(doc_len) / num_documents, + max_document_length=max(doc_len), + unique_documents=len(set(corpus)), + min_query_length=min(query_len), + average_query_length=sum(query_len) / num_queries, + max_query_length=max(query_len), + unique_queries=len(set(queries)), + min_relevant_docs_per_query=min(qrels_lengths), average_relevant_docs_per_query=qrels_per_doc, + max_relevant_docs_per_query=max(qrels_lengths), + unique_relevant_docs=unique_qrels, ) def calculate_length( queries: dict[str, str], corpus: dict[str, str] -) -> tuple[float, float]: +) -> tuple[list[int], list[int]]: queries_lens = [] doc_lens = [] for query in queries.values(): - queries_lens.append(len(query)) + if isinstance(query[0], str): + queries_lens.append(len(query)) + else: + queries_lens.extend([len(turn) for turn in query]) for doc in corpus.values(): doc_lens.append(len(doc)) - doc_len = sum(doc_lens) / len(doc_lens) if doc_lens else 0 - query_len = sum(queries_lens) / len(queries_lens) if queries_lens else 0 - return query_len, doc_len + return doc_lens, queries_lens def process_docs( diff --git a/mteb/abstasks/AbsTaskSTS.py b/mteb/abstasks/AbsTaskSTS.py index 157f285951..d12b88545d 100644 --- a/mteb/abstasks/AbsTaskSTS.py +++ b/mteb/abstasks/AbsTaskSTS.py @@ -5,7 +5,8 @@ from ..evaluation.evaluators import STSEvaluator from ..load_results.task_results import ScoresDict -from .AbsTask import AbsTask, DescriptiveStatistics +from .AbsTask import AbsTask +from .TaskMetadata import DescriptiveStatistics logger = logging.getLogger(__name__) @@ -15,15 +16,37 @@ class STSDescriptiveStatistics(DescriptiveStatistics): Attributes: num_samples: number of samples in the dataset. + number_of_characters: Total number of symbols in the dataset. + + min_sentence1_length: Minimum length of sentence1 average_sentence1_len: Average length of sentence1 + max_sentence1_length: Maximum length of sentence1 + + min_sentence2_length: Minimum length of sentence2 average_sentence2_len: Average length of sentence2 + max_sentence2_length: Maximum length of sentence2 + + min_score: Minimum score avg_score: Average score + max_score: Maximum score """ num_samples: int + number_of_characters: int + + min_sentence1_length: int average_sentence1_len: float + max_sentence1_length: int + unique_sentence1: int + + min_sentence2_length: int average_sentence2_len: float + max_sentence2_length: int + unique_sentence2: int + + min_score: float avg_score: float + max_score: float class AbsTaskSTS(AbsTask): @@ -35,6 +58,8 @@ class AbsTaskSTS(AbsTask): score: float """ + abstask_prompt = "Retrieve semantically similar text." + def __init__(self, **kwargs): super().__init__(**kwargs) @@ -88,12 +113,23 @@ def _calculate_metrics_from_split( sentence2 = self.dataset[split]["sentence2"] score = self.dataset[split]["score"] - total_sentence1_len = sum([len(s) for s in sentence1]) - total_sentence2_len = sum([len(s) for s in sentence2]) + sentence1_len = [len(s) for s in sentence1] + sentence2_len = [len(s) for s in sentence2] + total_sentence1_len = sum(sentence1_len) + total_sentence2_len = sum(sentence2_len) avg_score = sum(score) / len(score) return STSDescriptiveStatistics( num_samples=len(sentence1), + number_of_characters=total_sentence1_len + total_sentence2_len, + min_sentence1_length=min(sentence1_len), average_sentence1_len=total_sentence1_len / len(sentence1), + max_sentence1_length=max(sentence1_len), + unique_sentence1=len(set(sentence1)), + min_sentence2_length=min(sentence2_len), average_sentence2_len=total_sentence2_len / len(sentence2), + max_sentence2_length=max(sentence2_len), + unique_sentence2=len(set(sentence2)), + min_score=min(score), avg_score=avg_score, + max_score=max(score), ) diff --git a/mteb/abstasks/AbsTaskSummarization.py b/mteb/abstasks/AbsTaskSummarization.py index ff03fbaab3..07fd420571 100644 --- a/mteb/abstasks/AbsTaskSummarization.py +++ b/mteb/abstasks/AbsTaskSummarization.py @@ -9,7 +9,8 @@ from mteb.load_results.task_results import ScoresDict from ..evaluation.evaluators import SummarizationEvaluator -from .AbsTask import AbsTask, DescriptiveStatistics +from .AbsTask import AbsTask +from .TaskMetadata import DescriptiveStatistics logger = logging.getLogger(__name__) @@ -19,17 +20,49 @@ class SummarizationDescriptiveStatistics(DescriptiveStatistics): Attributes: num_samples: number of samples in the dataset. - avg_text_len: Average length of text - avg_human_summaries_len: Average length of human summaries - avg_machine_summaries_len: Average length of machine summaries + number_of_characters: Total number of symbols in the dataset. + + min_text_length: Minimum length of text + avg_text_length: Average length of text + max_text_length: Maximum length of text + unique_texts: Number of unique texts + + min_human_summaries_length: Minimum length of human summaries + avg_human_summaries_length: Average length of human summaries + max_human_summaries_length: Maximum length of human summaries + unique_human_summaries: Number of unique human summaries + + min_machine_summaries_length: Minimum length of machine summaries + avg_machine_summaries_length: Average length of machine summaries + max_machine_summaries_length: Maximum length of machine summaries + unique_machine_summaries: Number of unique machine summaries + + min_relevance: Minimum relevance score avg_relevance: Average relevance score + max_relevance: Maximum relevance score """ num_samples: int - avg_text_len: float - avg_human_summaries_len: float - avg_machine_summaries_len: float + number_of_characters: int + + min_text_length: int + avg_text_length: float + max_text_length: int + unique_texts: int + + min_human_summaries_length: int + avg_human_summaries_length: float + max_human_summaries_length: int + unique_human_summaries: int + + min_machine_summaries_length: int + avg_machine_summaries_length: float + max_machine_summaries_length: int + unique_machine_summaries: int + + min_relevance: float avg_relevance: float + max_relevance: float class AbsTaskSummarization(AbsTask): @@ -43,6 +76,9 @@ class AbsTaskSummarization(AbsTask): """ evalutor = SummarizationEvaluator + abstask_prompt = ( + "Given a news summary, retrieve other semantically similar summaries." + ) def __init__(self, **kwargs): super().__init__(**kwargs) @@ -106,14 +142,39 @@ def _calculate_metrics_from_split( machine_summaries = self.dataset[split]["machine_summaries"] relevance = self.dataset[split]["relevance"] - total_text_len = sum(len(x) for x in text) - total_human_summaries_len = sum(len(x) for x in human_summaries) - total_machine_summaries_len = sum(len(x) for x in machine_summaries) + all_human_summaries = [] + for s in human_summaries: + all_human_summaries.extend(s) + + all_machine_summaries = [] + for s in machine_summaries: + all_machine_summaries.extend(s) + + text_len = [len(t) for t in text] + total_text_len = sum(text_len) + human_summaries_len = [len(s) for s in human_summaries] + total_human_summaries_len = sum(human_summaries_len) + machine_summaries_len = [len(s) for s in machine_summaries] + total_machine_summaries_len = sum(machine_summaries_len) total_relevance = sum(sum(x) / len(x) for x in relevance) return SummarizationDescriptiveStatistics( num_samples=len(text), - avg_text_len=total_text_len / len(text), - avg_human_summaries_len=total_human_summaries_len / len(text), - avg_machine_summaries_len=total_machine_summaries_len / len(text), + number_of_characters=total_text_len + + total_human_summaries_len + + total_machine_summaries_len, + min_text_length=min(text_len), + avg_text_length=total_text_len / len(text), + max_text_length=max(text_len), + unique_texts=len(set(text)), + min_human_summaries_length=min(human_summaries_len), + avg_human_summaries_length=total_human_summaries_len / len(text), + max_human_summaries_length=max(human_summaries_len), + unique_human_summaries=len(set(all_human_summaries)), + min_machine_summaries_length=min(machine_summaries_len), + avg_machine_summaries_length=total_machine_summaries_len / len(text), + max_machine_summaries_length=max(machine_summaries_len), + unique_machine_summaries=len(set(all_machine_summaries)), + min_relevance=min(relevance), avg_relevance=total_relevance / len(relevance), + max_relevance=max(relevance), ) diff --git a/mteb/abstasks/MultilingualTask.py b/mteb/abstasks/MultilingualTask.py index 3fd007df6d..6516e74bd0 100644 --- a/mteb/abstasks/MultilingualTask.py +++ b/mteb/abstasks/MultilingualTask.py @@ -12,9 +12,7 @@ def __init__(self, hf_subsets: list[str] | None = None, **kwargs): lang for lang in hf_subsets if lang in self.metadata.eval_langs ] if hf_subsets is not None and len(hf_subsets) > 0: - self.hf_subsets = ( - hf_subsets # TODO: case where user provides langs not in the dataset - ) + self.hf_subsets = hf_subsets else: self.hf_subsets = self.metadata.eval_langs self.is_multilingual = True diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 2bd7f18bdd..67aa9fbccd 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -1,13 +1,22 @@ from __future__ import annotations +import json import logging from collections.abc import Mapping from datetime import date +from pathlib import Path from typing import Annotated, Any, Union -from pydantic import AnyUrl, BaseModel, BeforeValidator, TypeAdapter, field_validator -from typing_extensions import Literal +from pydantic import ( + AnyUrl, + BaseModel, + BeforeValidator, + TypeAdapter, + field_validator, +) +from typing_extensions import Literal, TypedDict +from ..encoder_interface import PromptType from ..languages import ( ISO_LANGUAGE_SCRIPT, ISO_TO_LANGUAGE, @@ -196,6 +205,25 @@ METRIC_NAME = str METRIC_VALUE = Union[int, float, dict[str, Any]] + +class PromptDict(TypedDict, total=False): + """A dictionary containing the prompt used for the task. + + Args: + query: The prompt used for the queries in the task. + passage: The prompt used for the passages in the task. + """ + + query: str + passage: str + + +class DescriptiveStatistics(TypedDict): + """Class for descriptive statistics.""" + + pass + + logger = logging.getLogger(__name__) @@ -226,17 +254,15 @@ class TaskMetadata(BaseModel): dialect: The dialect of the data, if applicable. Ideally specified as a BCP-47 language tag. Empty list if no dialects are present. sample_creation: The method of text creation. Includes "found", "created", "machine-translated", "machine-translated and verified", and "machine-translated and localized". + prompt: The prompt used for the task. Can be a string or a dictionary containing the query and passage prompts. bibtex_citation: The BibTeX citation for the dataset. Should be an empty string if no citation is available. - n_samples: The number of samples in the dataset. This should only be for the splits evaluated on. For retrieval tasks, this should be the - number of query-document pairs. - avg_character_length: The average character length of the samples in the dataset. This should only be for the splits evaluated on. For - retrieval tasks, this will be a dict containing the character length of the queries and documents separately, as well as the total number of queries, documents, and relevance judgements per query. """ dataset: dict name: str description: str + prompt: str | PromptDict | None = None type: TASK_TYPE modalities: list[MODALITIES] = ["text"] category: TASK_CATEGORY | None = None @@ -257,8 +283,6 @@ class TaskMetadata(BaseModel): sample_creation: SAMPLE_CREATION_METHOD | None = None bibtex_citation: str | None = None - descriptive_stats: dict[METRIC_NAME, dict[SPLIT_NAME, METRIC_VALUE] | None] = {} - def validate_metadata(self) -> None: self.dataset_path_is_specified(self.dataset) self.dataset_revision_is_specified(self.dataset) @@ -278,6 +302,18 @@ def _check_dataset_revision_is_specified( cls.dataset_revision_is_specified(dataset) return dataset + @field_validator("prompt") + def _check_prompt_is_valid( + cls, prompt: str | PromptDict | None + ) -> str | PromptDict | None: + if isinstance(prompt, dict): + for key in prompt: + if key not in [e.value for e in PromptType]: + raise ValueError( + "The prompt dictionary should only contain the keys 'query' and 'passage'." + ) + return prompt + @staticmethod def dataset_path_is_specified(dataset: dict[str, Any]) -> None: """This method checks that the dataset path is specified.""" @@ -358,7 +394,9 @@ def get_script(lang: str) -> str: def is_filled(self) -> bool: """Check if all the metadata fields are filled.""" return all( - getattr(self, field_name) is not None for field_name in self.model_fields + getattr(self, field_name) is not None + for field_name in self.model_fields + if field_name != "prompt" ) @property @@ -383,5 +421,38 @@ def intext_citation(self, include_cite: bool = True) -> str: return f"\\cite{{{cite}}}" return cite + @property + def descriptive_stats(self) -> dict[str, DescriptiveStatistics] | None: + """Return the descriptive statistics for the dataset.""" + if self.descriptive_stat_path.exists(): + with self.descriptive_stat_path.open("r") as f: + return json.load(f) + return None + + @property + def descriptive_stat_path(self) -> Path: + """Return the path to the descriptive statistics file.""" + descriptive_stat_base_dir = Path(__file__).parent.parent / "descriptive_stats" + if not descriptive_stat_base_dir.exists(): + descriptive_stat_base_dir.mkdir() + task_type_dir = descriptive_stat_base_dir / self.type + if not task_type_dir.exists(): + task_type_dir.mkdir() + return task_type_dir / f"{self.name}.json" + + @property + def n_samples(self) -> dict[str, int] | None: + """Returns the number of samples in the dataset""" + stats = self.descriptive_stats + if not stats: + return None + + n_samples = {} + for subset, subset_value in stats.items(): + if subset == "hf_subset_descriptive_stats": + continue + n_samples[subset] = subset_value["num_samples"] + return n_samples + def __hash__(self) -> int: return hash(self.model_dump_json()) diff --git a/mteb/benchmarks/benchmarks.py b/mteb/benchmarks/benchmarks.py index 9c24c525ac..233c7a79b3 100644 --- a/mteb/benchmarks/benchmarks.py +++ b/mteb/benchmarks/benchmarks.py @@ -9,7 +9,7 @@ from mteb.abstasks.AbsTask import AbsTask from mteb.load_results.benchmark_results import BenchmarkResults from mteb.load_results.load_results import load_results -from mteb.overview import get_tasks +from mteb.overview import MTEBTasks, get_task, get_tasks http_url_adapter = TypeAdapter(AnyUrl) UrlString = Annotated[ @@ -27,6 +27,7 @@ class Benchmark: description: A description of the benchmark, should include its intended goal and potentially a description of its construction reference: A link reference, to a source containing additional information typically to a paper, leaderboard or github. citation: A bibtex citation + contacts: The people to contact in case of a problem in the benchmark, preferably a GitHub handle. Example: >>> Benchmark( @@ -44,6 +45,7 @@ class Benchmark: description: str | None = None reference: UrlString | None = None citation: str | None = None + contacts: list[str] | None = None def __iter__(self): return iter(self.tasks) @@ -57,87 +59,175 @@ def __getitem__(self, index): def load_results( self, base_results: None | BenchmarkResults = None ) -> BenchmarkResults: + if not hasattr(self, "results_cache"): + self.results_cache = {} + if base_results in self.results_cache: + return self.results_cache[base_results] if base_results is None: base_results = load_results() - return base_results.select_tasks(self.tasks) + results = base_results.select_tasks(self.tasks) + self.results_cache[base_results] = results + return results -MTEB_MAIN_EN = Benchmark( +MTEB_EN = Benchmark( name="MTEB(eng)", - tasks=get_tasks( - tasks=[ - "AmazonCounterfactualClassification", - "AmazonPolarityClassification", - "AmazonReviewsClassification", - "ArguAna", - "ArxivClusteringP2P", - "ArxivClusteringS2S", - "AskUbuntuDupQuestions", - "BIOSSES", - "Banking77Classification", - "BiorxivClusteringP2P", - "BiorxivClusteringS2S", - "CQADupstackAndroidRetrieval", - "CQADupstackEnglishRetrieval", - "CQADupstackGamingRetrieval", - "CQADupstackGisRetrieval", - "CQADupstackMathematicaRetrieval", - "CQADupstackPhysicsRetrieval", - "CQADupstackProgrammersRetrieval", - "CQADupstackStatsRetrieval", - "CQADupstackTexRetrieval", - "CQADupstackUnixRetrieval", - "CQADupstackWebmastersRetrieval", - "CQADupstackWordpressRetrieval", - "ClimateFEVER", - "DBPedia", - "EmotionClassification", - "FEVER", - "FiQA2018", - "HotpotQA", - "ImdbClassification", - "MSMARCO", - "MTOPDomainClassification", - "MTOPIntentClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "MedrxivClusteringP2P", - "MedrxivClusteringS2S", - "MindSmallReranking", - "NFCorpus", - "NQ", - "QuoraRetrieval", - "RedditClustering", - "RedditClusteringP2P", - "SCIDOCS", - "SICK-R", - "STS12", - "STS13", - "STS14", - "STS15", - "STS16", - "STS17", - "STS22", - "STSBenchmark", - "SciDocsRR", - "SciFact", - "SprintDuplicateQuestions", - "StackExchangeClustering", - "StackExchangeClusteringP2P", - "StackOverflowDupQuestions", - "SummEval", - "TRECCOVID", - "Touche2020Retrieval.v3", - "ToxicConversationsClassification", - "TweetSentimentExtractionClassification", - "TwentyNewsgroupsClustering", - "TwitterSemEval2015", - "TwitterURLCorpus", - ], - languages=["eng"], - eval_splits=["test"], + tasks=MTEBTasks( + get_tasks( + tasks=[ + "ArguAna", + "ArXivHierarchicalClusteringP2P", + "ArXivHierarchicalClusteringS2S", + "AskUbuntuDupQuestions", + "BIOSSES", + "Banking77Classification", + "BiorxivClusteringP2P.v2", + "CQADupstackGamingRetrieval", + "CQADupstackUnixRetrieval", + "ClimateFEVERHardNegatives", + "FEVERHardNegatives", + "FiQA2018", + "HotpotQAHardNegatives", + "ImdbClassification", + "MTOPDomainClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MedrxivClusteringP2P.v2", + "MedrxivClusteringS2S.v2", + "MindSmallReranking", + "SCIDOCS", + "SICK-R", + "STS12", + "STS13", + "STS14", + "STS15", + "STSBenchmark", + "SprintDuplicateQuestions", + "StackExchangeClustering.v2", + "StackExchangeClusteringP2P.v2", + "TRECCOVID", + "Touche2020Retrieval.v3", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "TwentyNewsgroupsClustering.v2", + "TwitterSemEval2015", + "TwitterURLCorpus", + "SummEvalSummarization.v2", + ], + languages=["eng"], + eval_splits=["test"], + exclusive_language_filter=True, + ) + + ( + get_task( + "AmazonCounterfactualClassification", + eval_splits=["test"], + hf_subsets=["en"], + ), + get_task("STS17", eval_splits=["test"], hf_subsets=["en-en"]), + get_task("STS22.v2", eval_splits=["test"], hf_subsets=["en"]), + ), ), - description="Main English benchmarks from MTEB", + description="""The new English Massive Text Embedding Benchmark. +This benchmark was created to account for the fact that many models have now been finetuned +to tasks in the original MTEB, and contains tasks that are not as frequently used for model training. +This way the new benchmark and leaderboard can give our users a more realistic expectation of models' generalization performance. + +The original MTEB leaderboard is available under the [MTEB(eng, classic)](http://mteb-leaderboard-2-demo.hf.space/?benchmark_name=MTEB%28eng%2C+classic%29) tab. + """, + citation="", + contacts=["KennethEnevoldsen", "Muennighoff"], +) + +MTEB_ENG_CLASSIC = Benchmark( + name="MTEB(eng, classic)", + tasks=MTEBTasks( + get_tasks( + tasks=[ + "AmazonPolarityClassification", + "AmazonReviewsClassification", + "ArguAna", + "ArxivClusteringP2P", + "ArxivClusteringS2S", + "AskUbuntuDupQuestions", + "BIOSSES", + "Banking77Classification", + "BiorxivClusteringP2P", + "BiorxivClusteringS2S", + "CQADupstackAndroidRetrieval", + "CQADupstackEnglishRetrieval", + "CQADupstackGamingRetrieval", + "CQADupstackGisRetrieval", + "CQADupstackMathematicaRetrieval", + "CQADupstackPhysicsRetrieval", + "CQADupstackProgrammersRetrieval", + "CQADupstackStatsRetrieval", + "CQADupstackTexRetrieval", + "CQADupstackUnixRetrieval", + "CQADupstackWebmastersRetrieval", + "CQADupstackWordpressRetrieval", + "ClimateFEVER", + "DBPedia", + "EmotionClassification", + "FEVER", + "FiQA2018", + "HotpotQA", + "ImdbClassification", + "MTOPDomainClassification", + "MTOPIntentClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MedrxivClusteringP2P", + "MedrxivClusteringS2S", + "MindSmallReranking", + "NFCorpus", + "NQ", + "QuoraRetrieval", + "RedditClustering", + "RedditClusteringP2P", + "SCIDOCS", + "SICK-R", + "STS12", + "STS13", + "STS14", + "STS15", + "STS16", + "STSBenchmark", + "SciDocsRR", + "SciFact", + "SprintDuplicateQuestions", + "StackExchangeClustering", + "StackExchangeClusteringP2P", + "StackOverflowDupQuestions", + "SummEval", + "TRECCOVID", + "Touche2020", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "TwentyNewsgroupsClustering", + "TwitterSemEval2015", + "TwitterURLCorpus", + ], + languages=["eng"], + eval_splits=["test"], + ) + + get_tasks(tasks=["MSMARCO"], languages=["eng"], eval_splits=["dev"]) + + ( + get_task( + "AmazonCounterfactualClassification", + eval_splits=["test"], + hf_subsets=["en"], + ), + get_task("STS17", eval_splits=["test"], hf_subsets=["en-en"]), + get_task("STS22", eval_splits=["test"], hf_subsets=["en"]), + ) + ), + description="""The original English benchmark by Muennighoff et al., (2023). +This page is an adaptation of the [old MTEB leaderboard](https://huggingface.co/spaces/mteb/leaderboard). + +> We recommend that you use [MTEB(eng)](http://mteb-leaderboard-2-demo.hf.space/?benchmark_name=MTEB%28eng%29) instead, +as many models have been tuned on MTEB(eng, classic) datasets, and MTEB(eng) might give a more accurate representation of models' generalization performance. + """, citation="""@inproceedings{muennighoff-etal-2023-mteb, title = "{MTEB}: Massive Text Embedding Benchmark", author = "Muennighoff, Niklas and @@ -156,6 +246,7 @@ def load_results( pages = "2014--2037", } """, + contacts=["Muennighoff"], ) MTEB_MAIN_RU = Benchmark( @@ -195,7 +286,7 @@ def load_results( "STS22", ], ), - description="Main Russian benchmarks from MTEB", + description="A Russian version of the Massive Text Embedding Benchmark with a number of novel Russian tasks in all task categories of the original MTEB.", reference="https://aclanthology.org/2023.eacl-main.148/", citation="""@misc{snegirev2024russianfocusedembeddersexplorationrumteb, title={The Russian-focused embedders' exploration: ruMTEB benchmark and Russian embedding model design}, @@ -244,8 +335,31 @@ def load_results( "LegalQuAD", ] ), - description="Legal benchmarks from MTEB.", - reference="https://aclanthology.org/2023.eacl-main.148/", + description="A benchmark of retrieval tasks in the legal domain.", + reference=None, + citation=None, +) + +MTEB_RETRIEVAL_MEDICAL = Benchmark( + name="MTEB(Medical)", + tasks=get_tasks( + tasks=[ + "CUREv1", + "NFCorpus", + "TRECCOVID", + "TRECCOVID-PL", + "SciFact", + "SciFact-PL", + "MedicalQARetrieval", + "PublicHealthQA", + "MedrxivClusteringP2P.v2", + "MedrxivClusteringS2S.v2", + "CmedqaRetrieval", + "CMedQAv2-reranking", + ], + ), + description="A curated set of MTEB tasks designed to evaluate systems in the context of medical information retrieval.", + reference="", citation=None, ) @@ -262,7 +376,10 @@ def load_results( "Tatoeba", ] ), - description="BitextMining benchmark from MINERS", + description="""Bitext Mining texts from the MINERS benchmark, a benchmark designed to evaluate the + ability of multilingual LMs in semantic retrieval tasks, + including bitext mining and classification via retrieval-augmented contexts. + """, reference="https://arxiv.org/pdf/2406.07424", citation=""" @article{winata2024miners, @@ -323,6 +440,7 @@ def load_results( archivePrefix={arXiv}, primaryClass={cs.CL} }""", + contacts=["KennethEnevoldsen", "x-tabdeveloping", "Samoed"], ) CoIR = Benchmark( @@ -354,46 +472,82 @@ def load_results( }""", ) -MTEB_FRA = Benchmark( - name="MTEB(fra)", +RAR_b = Benchmark( + name="RAR-b", tasks=get_tasks( - languages=["fra"], tasks=[ - # Classification - "AmazonReviewsClassification", - "MasakhaNEWSClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "MTOPDomainClassification", - "MTOPIntentClassification", - # Clustering - "AlloProfClusteringP2P", - "AlloProfClusteringS2S", - "HALClusteringS2S", - "MasakhaNEWSClusteringP2P", - "MasakhaNEWSClusteringS2S", - "MLSUMClusteringP2P", - "MLSUMClusteringS2S", - # Pair Classification - "OpusparcusPC", - "PawsXPairClassification", - # Reranking - "AlloprofReranking", - "SyntecReranking", - # Retrieval - "AlloprofRetrieval", - "BSARDRetrieval", - "MintakaRetrieval", - "SyntecRetrieval", - "XPQARetrieval", - # STS - "SICKFr", - "STS22", - "STSBenchmarkMultilingualSTS", - "SummEvalFr", - ], + "ARCChallenge", + "AlphaNLI", + "HellaSwag", + "WinoGrande", + "PIQA", + "SIQA", + "Quail", + "SpartQA", + "TempReasonL1", + "TempReasonL2Pure", + "TempReasonL2Fact", + "TempReasonL2Context", + "TempReasonL3Pure", + "TempReasonL3Fact", + "TempReasonL3Context", + "RARbCode", + "RARbMath", + ] + ), + description="A benchmark to evaluate reasoning capabilities of retrievers.", + reference="https://arxiv.org/abs/2404.06347", + citation="""@article{xiao2024rar, + title={RAR-b: Reasoning as Retrieval Benchmark}, + author={Xiao, Chenghao and Hudson, G Thomas and Al Moubayed, Noura}, + journal={arXiv preprint arXiv:2404.06347}, + year={2024} + }""", + contacts=["gowitheflow-1998"], +) + +MTEB_FRA = Benchmark( + name="MTEB(fra)", + tasks=MTEBTasks( + get_tasks( + languages=["fra"], + tasks=[ + # Classification + "AmazonReviewsClassification", + "MasakhaNEWSClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MTOPDomainClassification", + "MTOPIntentClassification", + # Clustering + "AlloProfClusteringP2P", + "AlloProfClusteringS2S", + "HALClusteringS2S", + "MasakhaNEWSClusteringP2P", + "MasakhaNEWSClusteringS2S", + "MLSUMClusteringP2P", + "MLSUMClusteringS2S", + # Pair Classification + "OpusparcusPC", + "PawsXPairClassification", + # Reranking + "AlloprofReranking", + "SyntecReranking", + # Retrieval + "AlloprofRetrieval", + "BSARDRetrieval", + "MintakaRetrieval", + "SyntecRetrieval", + "XPQARetrieval", + # STS + "SICKFr", + "STSBenchmarkMultilingualSTS", + "SummEvalFr", + ], + ) + + (get_task("STS22", eval_splits=["test"], hf_subsets=["fr"]),) ), - description="Main French benchmarks from MTEB", + description="MTEB-French, a French expansion of the original benchmark with high-quality native French datasets.", reference="https://arxiv.org/abs/2405.20468", citation="""@misc{ciancone2024mtebfrenchresourcesfrenchsentence, title={MTEB-French: Resources for French Sentence Embedding Evaluation and Analysis}, @@ -404,6 +558,7 @@ def load_results( primaryClass={cs.CL}, url={https://arxiv.org/abs/2405.20468}, }""", + contacts=["imenelydiaker"], ) @@ -411,6 +566,7 @@ def load_results( name="MTEB(deu)", tasks=get_tasks( languages=["deu"], + exclusive_language_filter=True, tasks=[ # Classification "AmazonCounterfactualClassification", @@ -439,7 +595,7 @@ def load_results( "STS22", ], ), - description="Main German benchmarks from MTEB", + description="A benchmark for text-embedding performance in German.", reference="https://arxiv.org/html/2401.02709v1", citation="""@misc{wehrli2024germantextembeddingclustering, title={German Text Embedding Clustering Benchmark}, @@ -450,6 +606,7 @@ def load_results( primaryClass={cs.CL}, url={https://arxiv.org/abs/2401.02709}, }""", + contacts=["slvnwhrl"], ) @@ -470,7 +627,7 @@ def load_results( "KorSTS", ], ), - description="Main Korean benchmarks from MTEB", + description="A benchmark and leaderboard for evaluation of text embedding in Korean.", reference=None, citation=None, ) @@ -478,34 +635,40 @@ def load_results( MTEB_POL = Benchmark( name="MTEB(pol)", - tasks=get_tasks( - languages=["pol"], - tasks=[ - # Classification - "AllegroReviews", - "CBD", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "PolEmo2.0-IN", - "PolEmo2.0-OUT", - "PAC", - # Clustering - "EightTagsClustering", - "PlscClusteringS2S", - "PlscClusteringP2P", - # Pair Classification - "CDSC-E", - "PpcPC", - "PSC", - "SICK-E-PL", - # STS - "CDSC-R", - "STS22", - "STSBenchmarkMultilingualSTS", - "SICK-R-PL", - ], + tasks=MTEBTasks( + get_tasks( + languages=["pol"], + tasks=[ + # Classification + "AllegroReviews", + "CBD", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "PolEmo2.0-IN", + "PolEmo2.0-OUT", + "PAC", + # Clustering + "EightTagsClustering", + "PlscClusteringS2S", + "PlscClusteringP2P", + # Pair Classification + "CDSC-E", + "PpcPC", + "PSC", + "SICK-E-PL", + # STS + "CDSC-R", + "STSBenchmarkMultilingualSTS", + "SICK-R-PL", + ], + ) + + (get_task("STS22", eval_splits=["test"], hf_subsets=["pl"]),), ), - description="Main Polish benchmarks from MTEB", + description="""Polish Massive Text Embedding Benchmark (PL-MTEB), a comprehensive benchmark for text embeddings in Polish. The PL-MTEB consists of 28 diverse NLP +tasks from 5 task types. With tasks adapted based on previously used datasets by the Polish +NLP community. In addition, a new PLSC (Polish Library of Science Corpus) dataset was created +consisting of titles and abstracts of scientific publications in Polish, which was used as the basis for +two novel clustering tasks.""", # Rephrased from the abstract reference="https://arxiv.org/abs/2405.10138", citation="""@article{poswiata2024plmteb, title={PL-MTEB: Polish Massive Text Embedding Benchmark}, @@ -513,6 +676,7 @@ def load_results( journal={arXiv preprint arXiv:2405.10138}, year={2024} }""", + contacts=["rafalposwiata"], ) MTEB_code = Benchmark( @@ -549,7 +713,7 @@ def load_results( "typescript", ], ), - description="Main code benchmarks from MTEB", + description="A massive code embedding benchmark covering retrieval tasks in a miriad of popular programming languages.", reference=None, citation=None, ) @@ -693,9 +857,10 @@ def load_results( "MIRACLRetrievalHardNegatives", ], ), - description="The Multilingual benchmarks from MMTEB. Currently under development.", + description="A large-scale multilingual expansion of MTEB, driven mainly by highly-curated community contributions covering 250+ languages.", reference=None, citation=None, + contacts=["KennethEnevoldsen", "isaac-chung"], ) MTEB_JPN = Benchmark( @@ -727,14 +892,47 @@ def load_results( "ESCIReranking", ], ), - description="Main Japanese benchmarks from MTEB", + description="JMTEB is a benchmark for evaluating Japanese text embedding models.", reference="https://github.com/sbintuitions/JMTEB", citation=None, ) +indic_languages = [ + "asm", + "awa", + "ben", + "bgc", + "bho", + "doi", + "gbm", + "gom", + "guj", + "hin", + "hne", + "kan", + "kas", + "mai", + "mal", + "mar", + "mni", + "mup", + "mwr", + "nep", + "npi", + "ori", + "ory", + "pan", + "raj", + "san", + "snd", + "tam", + "tel", + "urd", +] + MTEB_INDIC = Benchmark( - name="MTEB(indic)", + name="MTEB(Indic)", tasks=get_tasks( tasks=[ # Bitext @@ -768,13 +966,59 @@ def load_results( # reranking "WikipediaRerankingMultilingual", ], + languages=indic_languages, + exclusive_language_filter=True, ), - description="Main Indic benchmark from MMTEB", + description="A regional geopolitical text embedding benchmark targetting embedding performance on Indic languages.", reference=None, citation=None, + contacts=["KennethEnevoldsen", "isaac-chung"], ) +eu_languages = [ + # official EU languages (56) - we could include the whole economic area e.g. Norway - additioanlly we could include minority languages (probably a good idea?) + # germanic + "dan", + "eng", + "deu", + "nld", + "swe", + # romance + "fra", + "ita", + "por", + "spa", + "ron", + # slavic + "bul", + "hrv", + "ces", + "pol", + "slk", + "slv", + # baltic + "lav", + "lit", + "est", + # finno-ugric + "fin", + "hun", + # other indo european + "ell", + # non-indo european + "mlt", + "gle", + # Schengen Area + "nno", + "nob", + "isl", + "ron", + "eus", # Basque - recognized minority language + "ron", # Romanian - recognized minority language + "rom", # Romani - recognized minority language +] + MTEB_EU = Benchmark( name="MTEB(Europe)", tasks=get_tasks( @@ -853,9 +1097,138 @@ def load_results( "STS17", "SICK-R-PL", "STSES", - ] + ], + languages=eu_languages, + exclusive_language_filter=True, ), - description="Main European benchmark from MMTEB", + description="A regional geopolitical text embedding benchmark targetting embedding performance on European languages.", reference=None, citation=None, + contacts=["KennethEnevoldsen", "isaac-chung"], +) + +LONG_EMBED = Benchmark( + name="LongEmbed", + tasks=get_tasks( + tasks=[ + "LEMBNarrativeQARetrieval", + "LEMBNeedleRetrieval", + "LEMBPasskeyRetrieval", + "LEMBQMSumRetrieval", + "LEMBSummScreenFDRetrieval", + "LEMBWikimQARetrieval", + ], + ), + description="""LongEmbed is a benchmark oriented at exploring models' performance on long-context retrieval. + The benchmark comprises two synthetic tasks and four carefully chosen real-world tasks, + featuring documents of varying length and dispersed target information. + """, # Pieced together from paper abstract. + reference="https://arxiv.org/abs/2404.12096v2", + citation="""@article{zhu2024longembed, + title={LongEmbed: Extending Embedding Models for Long Context Retrieval}, + author={Zhu, Dawei and Wang, Liang and Yang, Nan and Song, Yifan and Wu, Wenhao and Wei, Furu and Li, Sujian}, + journal={arXiv preprint arXiv:2404.12096}, + year={2024} +}""", +) + +BRIGHT = Benchmark( + name="BRIGHT", + tasks=get_tasks( + tasks=["BrightRetrieval"], + ), + description="""BRIGHT: A Realistic and Challenging Benchmark for Reasoning-Intensive Retrieval. + BRIGHT is the first text retrieval + benchmark that requires intensive reasoning to retrieve relevant documents with + a dataset consisting of 1,384 real-world queries spanning diverse domains, such as + economics, psychology, mathematics, and coding. These queries are drawn from + naturally occurring and carefully curated human data. + """, + reference="https://brightbenchmark.github.io/", + citation="""@article{su2024bright, + title={Bright: A realistic and challenging benchmark for reasoning-intensive retrieval}, + author={Su, Hongjin and Yen, Howard and Xia, Mengzhou and Shi, Weijia and Muennighoff, Niklas and Wang, Han-yu and Liu, Haisu and Shi, Quan and Siegel, Zachary S and Tang, Michael and others}, + journal={arXiv preprint arXiv:2407.12883}, + year={2024} +}""", +) + +NANOBEIR = Benchmark( + name="NanoBEIR", + tasks=get_tasks( + tasks=[ + "NanoArguAnaRetrieval", + "NanoClimateFeverRetrieval", + "NanoDBPediaRetrieval", + "NanoFEVERRetrieval", + "NanoFiQA2018Retrieval", + "NanoHotpotQARetrieval", + "NanoMSMARCORetrieval", + "NanoNFCorpusRetrieval", + "NanoNQRetrieval", + "NanoQuoraRetrieval", + "NanoSCIDOCSRetrieval", + "NanoSciFactRetrieval", + "NanoTouche2020Retrieval", + ], + ), + description="A benchmark to evaluate with subsets of BEIR datasets to use less computational power", + reference="https://huggingface.co/collections/zeta-alpha-ai/nanobeir-66e1a0af21dfd93e620cd9f6", + citation=None, +) + +C_MTEB = Benchmark( + name="MTEB(Chinese)", + tasks=MTEBTasks( + get_tasks( + tasks=[ + "T2Retrieval", + "MMarcoRetrieval", + "DuRetrieval", + "CovidRetrieval", + "CmedqaRetrieval", + "EcomRetrieval", + "MedicalRetrieval", + "VideoRetrieval", + "T2Reranking", + "MMarcoReranking", + "CMedQAv1-reranking", + "CMedQAv2-reranking", + "Ocnli", + "Cmnli", + "CLSClusteringS2S", + "CLSClusteringP2P", + "ThuNewsClusteringS2S", + "ThuNewsClusteringP2P", + "LCQMC", + "PAWSX", + "AFQMC", + "QBQTC", + "TNews", + "IFlyTek", + "Waimai", + "OnlineShopping", + "JDReview", + ], + ) + + get_tasks(tasks=["MultilingualSentiment"], eval_splits=["test"]) + + get_tasks( + tasks=[ + "ATEC", + "BQ", + "STSB", + ], + eval_splits=["validation"], + ) + ), + description="The Chinese Massive Text Embedding Benchmark (C-MTEB) is a comprehensive benchmark for Chinese text embeddings covering 6 tasks and 35 datasets.", + reference="https://github.com/FlagOpen/FlagEmbedding/tree/master/research/C_MTEB", + citation="""@misc{c-pack, + title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, + author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff}, + year={2023}, + eprint={2309.07597}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +}""", ) diff --git a/mteb/caching.py b/mteb/caching.py new file mode 100644 index 0000000000..daa56c1887 --- /dev/null +++ b/mteb/caching.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import json +from typing import Callable + + +def json_cache(function: Callable): + """Caching decorator that can deal with anything json serializable""" + cached_results = {} + + def wrapper(*args, **kwargs): + key = json.dumps({"__args": args, **kwargs}) + if key in cached_results: + return cached_results[key] + result = function(*args, **kwargs) + cached_results[key] = result + return result + + return wrapper diff --git a/mteb/descriptive_stats/BitextMining/BUCC.v2.json b/mteb/descriptive_stats/BitextMining/BUCC.v2.json new file mode 100644 index 0000000000..75ef75ced5 --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/BUCC.v2.json @@ -0,0 +1,69 @@ +{ + "test": { + "num_samples": 35000, + "number_of_characters": 6640032, + "unique_pairs": 34978, + "min_sentence1_length": 16, + "average_sentence1_length": 99.10931428571429, + "max_sentence1_length": 204, + "unique_sentence1": 34978, + "min_sentence2_length": 42, + "average_sentence2_length": 90.60588571428572, + "max_sentence2_length": 159, + "unique_sentence2": 25306, + "hf_subset_descriptive_stats": { + "de-en": { + "num_samples": 9580, + "number_of_characters": 1919197, + "unique_pairs": 9573, + "min_sentence1_length": 50, + "average_sentence1_length": 109.07974947807934, + "max_sentence1_length": 204, + "unique_sentence1": 9573, + "min_sentence2_length": 46, + "average_sentence2_length": 91.25396659707724, + "max_sentence2_length": 155, + "unique_sentence2": 9570 + }, + "fr-en": { + "num_samples": 9086, + "number_of_characters": 1677545, + "unique_pairs": 9081, + "min_sentence1_length": 43, + "average_sentence1_length": 99.31785163988553, + "max_sentence1_length": 174, + "unique_sentence1": 9081, + "min_sentence2_length": 42, + "average_sentence2_length": 85.3117983711204, + "max_sentence2_length": 159, + "unique_sentence2": 9076 + }, + "ru-en": { + "num_samples": 14435, + "number_of_characters": 2808206, + "unique_pairs": 14425, + "min_sentence1_length": 40, + "average_sentence1_length": 101.6593003117423, + "max_sentence1_length": 186, + "unique_sentence1": 14425, + "min_sentence2_length": 45, + "average_sentence2_length": 92.88216141323173, + "max_sentence2_length": 159, + "unique_sentence2": 14424 + }, + "zh-en": { + "num_samples": 1899, + "number_of_characters": 235084, + "unique_pairs": 1899, + "min_sentence1_length": 16, + "average_sentence1_length": 28.429699842022117, + "max_sentence1_length": 40, + "unique_sentence1": 1899, + "min_sentence2_length": 48, + "average_sentence2_length": 95.3638757240653, + "max_sentence2_length": 159, + "unique_sentence2": 1899 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/BornholmBitextMining.json b/mteb/descriptive_stats/BitextMining/BornholmBitextMining.json new file mode 100644 index 0000000000..0675e5e0ef --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/BornholmBitextMining.json @@ -0,0 +1,15 @@ +{ + "test": { + "num_samples": 500, + "number_of_characters": 44361, + "unique_pairs": 500, + "min_sentence1_length": 1, + "average_sentence1_length": 49.834, + "max_sentence1_length": 555, + "unique_sentence1": 497, + "min_sentence2_length": 5, + "average_sentence2_length": 38.888, + "max_sentence2_length": 453, + "unique_sentence2": 491 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/IN22ConvBitextMining.json b/mteb/descriptive_stats/BitextMining/IN22ConvBitextMining.json new file mode 100644 index 0000000000..effafd237b --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/IN22ConvBitextMining.json @@ -0,0 +1,6595 @@ +{ + "test": { + "num_samples": 760518, + "number_of_characters": 82637104, + "unique_pairs": 759283, + "min_sentence1_length": 3, + "average_sentence1_length": 54.32948595562498, + "max_sentence1_length": 239, + "unique_sentence1": 34430, + "min_sentence2_length": 3, + "average_sentence2_length": 54.32948595562498, + "max_sentence2_length": 239, + "unique_sentence2": 34430, + "hf_subset_descriptive_stats": { + "asm_Beng-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 155988, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "asm_Beng-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 162044, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "asm_Beng-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 167032, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "asm_Beng-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 160716, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "asm_Beng-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 156282, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "asm_Beng-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 158269, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "asm_Beng-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 159964, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "asm_Beng-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 165177, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "asm_Beng-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 164681, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "asm_Beng-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 162408, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "asm_Beng-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 172838, + "unique_pairs": 1498, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "asm_Beng-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 162747, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "asm_Beng-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 157316, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "asm_Beng-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 160906, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "asm_Beng-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 164223, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "asm_Beng-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 160201, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "asm_Beng-san_Deva": { + "num_samples": 1503, + "number_of_characters": 158093, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "asm_Beng-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 169379, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "asm_Beng-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 162623, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "asm_Beng-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 174866, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "asm_Beng-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 157690, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "asm_Beng-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 161305, + "unique_pairs": 1498, + "min_sentence1_length": 4, + "average_sentence1_length": 53.753825681969396, + "max_sentence1_length": 208, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "ben_Beng-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 155988, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "ben_Beng-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 156448, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "ben_Beng-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 161436, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "ben_Beng-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 155120, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "ben_Beng-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 150686, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "ben_Beng-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 152673, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "ben_Beng-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 154368, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "ben_Beng-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 159581, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "ben_Beng-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 159085, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "ben_Beng-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 156812, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "ben_Beng-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 167242, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "ben_Beng-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 157151, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "ben_Beng-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 151720, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "ben_Beng-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 155310, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "ben_Beng-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 158627, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "ben_Beng-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 154605, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "ben_Beng-san_Deva": { + "num_samples": 1503, + "number_of_characters": 152497, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "ben_Beng-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 163783, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "ben_Beng-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 157027, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "ben_Beng-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 169270, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "ben_Beng-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 152094, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "ben_Beng-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 155709, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.03060545575516, + "max_sentence1_length": 178, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "brx_Deva-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 162044, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "brx_Deva-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 156448, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "brx_Deva-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 167492, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "brx_Deva-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 161176, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "brx_Deva-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 156742, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "brx_Deva-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 158729, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "brx_Deva-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 160424, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "brx_Deva-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 165637, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "brx_Deva-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 165141, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "brx_Deva-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 162868, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "brx_Deva-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 173298, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "brx_Deva-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 163207, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "brx_Deva-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 157776, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "brx_Deva-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 161366, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "brx_Deva-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 164683, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "brx_Deva-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 160661, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "brx_Deva-san_Deva": { + "num_samples": 1503, + "number_of_characters": 158553, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "brx_Deva-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 169839, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "brx_Deva-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 163083, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "brx_Deva-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 175326, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "brx_Deva-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 158150, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "brx_Deva-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 161765, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.05988023952096, + "max_sentence1_length": 210, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "doi_Deva-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 167032, + "unique_pairs": 1500, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "doi_Deva-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 161436, + "unique_pairs": 1501, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "doi_Deva-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 167492, + "unique_pairs": 1501, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "doi_Deva-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 166164, + "unique_pairs": 1500, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "doi_Deva-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 161730, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "doi_Deva-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 163717, + "unique_pairs": 1503, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "doi_Deva-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 165412, + "unique_pairs": 1503, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "doi_Deva-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 170625, + "unique_pairs": 1503, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "doi_Deva-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 170129, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "doi_Deva-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 167856, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "doi_Deva-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 178286, + "unique_pairs": 1500, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "doi_Deva-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 168195, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "doi_Deva-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 162764, + "unique_pairs": 1501, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "doi_Deva-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 166354, + "unique_pairs": 1501, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "doi_Deva-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 169671, + "unique_pairs": 1503, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "doi_Deva-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 165649, + "unique_pairs": 1500, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "doi_Deva-san_Deva": { + "num_samples": 1503, + "number_of_characters": 163541, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "doi_Deva-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 174827, + "unique_pairs": 1503, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "doi_Deva-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 168071, + "unique_pairs": 1500, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "doi_Deva-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 180314, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "doi_Deva-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 163138, + "unique_pairs": 1501, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "doi_Deva-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 166753, + "unique_pairs": 1500, + "min_sentence1_length": 6, + "average_sentence1_length": 57.37857618097139, + "max_sentence1_length": 209, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "eng_Latn-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 160716, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "eng_Latn-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 155120, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "eng_Latn-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 161176, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "eng_Latn-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 166164, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "eng_Latn-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 155414, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "eng_Latn-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 157401, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "eng_Latn-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 159096, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "eng_Latn-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 164309, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "eng_Latn-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 163813, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "eng_Latn-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 161540, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "eng_Latn-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 171970, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "eng_Latn-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 161879, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "eng_Latn-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 156448, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "eng_Latn-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 160038, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "eng_Latn-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 163355, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "eng_Latn-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 159333, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "eng_Latn-san_Deva": { + "num_samples": 1503, + "number_of_characters": 157225, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "eng_Latn-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 168511, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "eng_Latn-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 161755, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "eng_Latn-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 173998, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "eng_Latn-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 156822, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "eng_Latn-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 160437, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.17631403858949, + "max_sentence1_length": 201, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "gom_Deva-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 156282, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "gom_Deva-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 150686, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "gom_Deva-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 156742, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "gom_Deva-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 161730, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "gom_Deva-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 155414, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "gom_Deva-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 152967, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "gom_Deva-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 154662, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "gom_Deva-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 159875, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "gom_Deva-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 159379, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "gom_Deva-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 157106, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "gom_Deva-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 167536, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "gom_Deva-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 157445, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "gom_Deva-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 152014, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "gom_Deva-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 155604, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "gom_Deva-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 158921, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "gom_Deva-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 154899, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "gom_Deva-san_Deva": { + "num_samples": 1503, + "number_of_characters": 152791, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "gom_Deva-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 164077, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "gom_Deva-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 157321, + "unique_pairs": 1500, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "gom_Deva-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 169564, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "gom_Deva-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 152388, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "gom_Deva-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 156003, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 50.22621423819029, + "max_sentence1_length": 203, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "guj_Gujr-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 158269, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "guj_Gujr-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 152673, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "guj_Gujr-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 158729, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "guj_Gujr-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 163717, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "guj_Gujr-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 157401, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "guj_Gujr-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 152967, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "guj_Gujr-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 156649, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "guj_Gujr-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 161862, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "guj_Gujr-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 161366, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "guj_Gujr-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 159093, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "guj_Gujr-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 169523, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "guj_Gujr-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 159432, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "guj_Gujr-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 154001, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "guj_Gujr-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 157591, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "guj_Gujr-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 160908, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "guj_Gujr-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 156886, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "guj_Gujr-san_Deva": { + "num_samples": 1503, + "number_of_characters": 154778, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "guj_Gujr-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 166064, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "guj_Gujr-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 159308, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "guj_Gujr-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 171551, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "guj_Gujr-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 154375, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "guj_Gujr-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 157990, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 51.54823685961411, + "max_sentence1_length": 205, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "hin_Deva-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 159964, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "hin_Deva-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 154368, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "hin_Deva-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 160424, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "hin_Deva-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 165412, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "hin_Deva-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 159096, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "hin_Deva-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 154662, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "hin_Deva-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 156649, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "hin_Deva-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 163557, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "hin_Deva-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 163061, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "hin_Deva-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 160788, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "hin_Deva-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 171218, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "hin_Deva-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 161127, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "hin_Deva-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 155696, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "hin_Deva-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 159286, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "hin_Deva-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 162603, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "hin_Deva-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 158581, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "hin_Deva-san_Deva": { + "num_samples": 1503, + "number_of_characters": 156473, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "hin_Deva-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 167759, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "hin_Deva-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 161003, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "hin_Deva-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 173246, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "hin_Deva-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 156070, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "hin_Deva-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 159685, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.67598137059215, + "max_sentence1_length": 192, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "kan_Knda-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 165177, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "kan_Knda-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 159581, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "kan_Knda-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 165637, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "kan_Knda-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 170625, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "kan_Knda-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 164309, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "kan_Knda-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 159875, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "kan_Knda-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 161862, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "kan_Knda-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 163557, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "kan_Knda-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 168274, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "kan_Knda-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 166001, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "kan_Knda-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 176431, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "kan_Knda-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 166340, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "kan_Knda-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 160909, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "kan_Knda-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 164499, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "kan_Knda-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 167816, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "kan_Knda-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 163794, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "kan_Knda-san_Deva": { + "num_samples": 1503, + "number_of_characters": 161686, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "kan_Knda-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 172972, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "kan_Knda-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 166216, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "kan_Knda-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 178459, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "kan_Knda-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 161283, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "kan_Knda-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 164898, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 56.14437791084497, + "max_sentence1_length": 201, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "kas_Arab-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 164681, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "kas_Arab-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 159085, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "kas_Arab-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 165141, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "kas_Arab-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 170129, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "kas_Arab-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 163813, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "kas_Arab-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 159379, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "kas_Arab-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 161366, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "kas_Arab-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 163061, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "kas_Arab-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 168274, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "kas_Arab-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 165505, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "kas_Arab-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 175935, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "kas_Arab-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 165844, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "kas_Arab-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 160413, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "kas_Arab-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 164003, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "kas_Arab-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 167320, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "kas_Arab-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 163298, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "kas_Arab-san_Deva": { + "num_samples": 1503, + "number_of_characters": 161190, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "kas_Arab-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 172476, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "kas_Arab-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 165720, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "kas_Arab-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 177963, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "kas_Arab-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 160787, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "kas_Arab-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 164402, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 55.81437125748503, + "max_sentence1_length": 203, + "unique_sentence1": 1502, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "mai_Deva-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 162408, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "mai_Deva-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 156812, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "mai_Deva-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 162868, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "mai_Deva-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 167856, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "mai_Deva-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 161540, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "mai_Deva-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 157106, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "mai_Deva-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 159093, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "mai_Deva-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 160788, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "mai_Deva-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 166001, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "mai_Deva-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 165505, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "mai_Deva-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 173662, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "mai_Deva-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 163571, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "mai_Deva-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 158140, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "mai_Deva-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 161730, + "unique_pairs": 1500, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "mai_Deva-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 165047, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "mai_Deva-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 161025, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "mai_Deva-san_Deva": { + "num_samples": 1503, + "number_of_characters": 158917, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "mai_Deva-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 170203, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "mai_Deva-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 163447, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "mai_Deva-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 175690, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "mai_Deva-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 158514, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "mai_Deva-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 162129, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 54.3020625415835, + "max_sentence1_length": 230, + "unique_sentence1": 1499, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "mal_Mlym-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 172838, + "unique_pairs": 1498, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "mal_Mlym-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 167242, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "mal_Mlym-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 173298, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "mal_Mlym-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 178286, + "unique_pairs": 1500, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "mal_Mlym-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 171970, + "unique_pairs": 1499, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "mal_Mlym-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 167536, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "mal_Mlym-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 169523, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "mal_Mlym-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 171218, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "mal_Mlym-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 176431, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "mal_Mlym-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 175935, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "mal_Mlym-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 173662, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "mal_Mlym-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 174001, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "mal_Mlym-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 168570, + "unique_pairs": 1500, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "mal_Mlym-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 172160, + "unique_pairs": 1500, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "mal_Mlym-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 175477, + "unique_pairs": 1503, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "mal_Mlym-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 171455, + "unique_pairs": 1498, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "mal_Mlym-san_Deva": { + "num_samples": 1503, + "number_of_characters": 169347, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "mal_Mlym-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 180633, + "unique_pairs": 1501, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "mal_Mlym-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 173877, + "unique_pairs": 1499, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "mal_Mlym-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 186120, + "unique_pairs": 1502, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "mal_Mlym-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 168944, + "unique_pairs": 1500, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "mal_Mlym-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 172559, + "unique_pairs": 1499, + "min_sentence1_length": 5, + "average_sentence1_length": 61.24151696606786, + "max_sentence1_length": 219, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "mar_Deva-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 162747, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "mar_Deva-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 157151, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "mar_Deva-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 163207, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "mar_Deva-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 168195, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "mar_Deva-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 161879, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "mar_Deva-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 157445, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "mar_Deva-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 159432, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "mar_Deva-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 161127, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "mar_Deva-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 166340, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "mar_Deva-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 165844, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "mar_Deva-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 163571, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "mar_Deva-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 174001, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "mar_Deva-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 158479, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "mar_Deva-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 162069, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "mar_Deva-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 165386, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "mar_Deva-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 161364, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "mar_Deva-san_Deva": { + "num_samples": 1503, + "number_of_characters": 159256, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "mar_Deva-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 170542, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "mar_Deva-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 163786, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "mar_Deva-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 176029, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "mar_Deva-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 158853, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "mar_Deva-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 162468, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.52761144377911, + "max_sentence1_length": 221, + "unique_sentence1": 1501, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "mni_Mtei-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 157316, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "mni_Mtei-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 151720, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "mni_Mtei-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 157776, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "mni_Mtei-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 162764, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "mni_Mtei-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 156448, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "mni_Mtei-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 152014, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "mni_Mtei-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 154001, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "mni_Mtei-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 155696, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "mni_Mtei-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 160909, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "mni_Mtei-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 160413, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "mni_Mtei-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 158140, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "mni_Mtei-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 168570, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "mni_Mtei-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 158479, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "mni_Mtei-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 156638, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "mni_Mtei-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 159955, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "mni_Mtei-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 155933, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "mni_Mtei-san_Deva": { + "num_samples": 1503, + "number_of_characters": 153825, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "mni_Mtei-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 165111, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "mni_Mtei-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 158355, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "mni_Mtei-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 170598, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "mni_Mtei-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 153422, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "mni_Mtei-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 157037, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 50.91417165668663, + "max_sentence1_length": 239, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "npi_Deva-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 160906, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "npi_Deva-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 155310, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "npi_Deva-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 161366, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "npi_Deva-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 166354, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "npi_Deva-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 160038, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "npi_Deva-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 155604, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "npi_Deva-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 157591, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "npi_Deva-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 159286, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "npi_Deva-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 164499, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "npi_Deva-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 164003, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "npi_Deva-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 161730, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "npi_Deva-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 172160, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "npi_Deva-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 162069, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "npi_Deva-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 156638, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "npi_Deva-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 163545, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "npi_Deva-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 159523, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "npi_Deva-san_Deva": { + "num_samples": 1503, + "number_of_characters": 157415, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "npi_Deva-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 168701, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "npi_Deva-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 161945, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "npi_Deva-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 174188, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "npi_Deva-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 157012, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "npi_Deva-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 160627, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.30272787757818, + "max_sentence1_length": 223, + "unique_sentence1": 1497, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "ory_Orya-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 164223, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "ory_Orya-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 158627, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "ory_Orya-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 164683, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "ory_Orya-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 169671, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "ory_Orya-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 163355, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "ory_Orya-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 158921, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "ory_Orya-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 160908, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "ory_Orya-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 162603, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "ory_Orya-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 167816, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "ory_Orya-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 167320, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "ory_Orya-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 165047, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "ory_Orya-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 175477, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "ory_Orya-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 165386, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "ory_Orya-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 159955, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "ory_Orya-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 163545, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "ory_Orya-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 162840, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "ory_Orya-san_Deva": { + "num_samples": 1503, + "number_of_characters": 160732, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "ory_Orya-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 172018, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "ory_Orya-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 165262, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "ory_Orya-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 177505, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "ory_Orya-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 160329, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "ory_Orya-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 163944, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 55.509647371922824, + "max_sentence1_length": 195, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "pan_Guru-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 160201, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "pan_Guru-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 154605, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "pan_Guru-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 160661, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "pan_Guru-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 165649, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "pan_Guru-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 159333, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "pan_Guru-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 154899, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "pan_Guru-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 156886, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "pan_Guru-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 158581, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "pan_Guru-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 163794, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "pan_Guru-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 163298, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "pan_Guru-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 161025, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "pan_Guru-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 171455, + "unique_pairs": 1498, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "pan_Guru-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 161364, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "pan_Guru-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 155933, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "pan_Guru-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 159523, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "pan_Guru-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 162840, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "pan_Guru-san_Deva": { + "num_samples": 1503, + "number_of_characters": 156710, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "pan_Guru-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 167996, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "pan_Guru-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 161240, + "unique_pairs": 1498, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "pan_Guru-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 173483, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "pan_Guru-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 156307, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "pan_Guru-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 159922, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 52.83366600133067, + "max_sentence1_length": 221, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "san_Deva-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 158093, + "unique_pairs": 1501, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "san_Deva-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 152497, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "san_Deva-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 158553, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "san_Deva-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 163541, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "san_Deva-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 157225, + "unique_pairs": 1501, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "san_Deva-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 152791, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "san_Deva-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 154778, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "san_Deva-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 156473, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "san_Deva-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 161686, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "san_Deva-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 161190, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "san_Deva-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 158917, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "san_Deva-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 169347, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "san_Deva-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 159256, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "san_Deva-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 153825, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "san_Deva-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 157415, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "san_Deva-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 160732, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "san_Deva-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 156710, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "san_Deva-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 165888, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "san_Deva-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 159132, + "unique_pairs": 1501, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "san_Deva-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 171375, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "san_Deva-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 154199, + "unique_pairs": 1501, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "san_Deva-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 157814, + "unique_pairs": 1501, + "min_sentence1_length": 3, + "average_sentence1_length": 51.4311377245509, + "max_sentence1_length": 181, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "sat_Olck-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 169379, + "unique_pairs": 1502, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "sat_Olck-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 163783, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "sat_Olck-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 169839, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "sat_Olck-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 174827, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "sat_Olck-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 168511, + "unique_pairs": 1502, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "sat_Olck-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 164077, + "unique_pairs": 1502, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "sat_Olck-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 166064, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "sat_Olck-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 167759, + "unique_pairs": 1502, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "sat_Olck-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 172972, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "sat_Olck-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 172476, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "sat_Olck-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 170203, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "sat_Olck-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 180633, + "unique_pairs": 1501, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "sat_Olck-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 170542, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "sat_Olck-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 165111, + "unique_pairs": 1502, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "sat_Olck-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 168701, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "sat_Olck-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 172018, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "sat_Olck-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 167996, + "unique_pairs": 1501, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "sat_Olck-san_Deva": { + "num_samples": 1503, + "number_of_characters": 165888, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "sat_Olck-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 170418, + "unique_pairs": 1501, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "sat_Olck-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 182661, + "unique_pairs": 1503, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "sat_Olck-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 165485, + "unique_pairs": 1502, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "sat_Olck-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 169100, + "unique_pairs": 1502, + "min_sentence1_length": 7, + "average_sentence1_length": 58.94011976047904, + "max_sentence1_length": 225, + "unique_sentence1": 1500, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "snd_Deva-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 162623, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "snd_Deva-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 157027, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "snd_Deva-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 163083, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "snd_Deva-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 168071, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "snd_Deva-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 161755, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "snd_Deva-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 157321, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "snd_Deva-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 159308, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "snd_Deva-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 161003, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "snd_Deva-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 166216, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "snd_Deva-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 165720, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "snd_Deva-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 163447, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "snd_Deva-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 173877, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "snd_Deva-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 163786, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "snd_Deva-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 158355, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "snd_Deva-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 161945, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "snd_Deva-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 165262, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "snd_Deva-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 161240, + "unique_pairs": 1498, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "snd_Deva-san_Deva": { + "num_samples": 1503, + "number_of_characters": 159132, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "snd_Deva-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 170418, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "snd_Deva-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 175905, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "snd_Deva-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 158729, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "snd_Deva-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 162344, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 54.445109780439125, + "max_sentence1_length": 195, + "unique_sentence1": 1490, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "tam_Taml-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 174866, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "tam_Taml-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 169270, + "unique_pairs": 1501, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "tam_Taml-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 175326, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "tam_Taml-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 180314, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "tam_Taml-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 173998, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "tam_Taml-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 169564, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "tam_Taml-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 171551, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "tam_Taml-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 173246, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "tam_Taml-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 178459, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "tam_Taml-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 177963, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "tam_Taml-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 175690, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "tam_Taml-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 186120, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "tam_Taml-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 176029, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "tam_Taml-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 170598, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "tam_Taml-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 174188, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "tam_Taml-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 177505, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "tam_Taml-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 173483, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "tam_Taml-san_Deva": { + "num_samples": 1503, + "number_of_characters": 171375, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "tam_Taml-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 182661, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "tam_Taml-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 175905, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "tam_Taml-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 170972, + "unique_pairs": 1502, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + }, + "tam_Taml-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 174587, + "unique_pairs": 1503, + "min_sentence1_length": 3, + "average_sentence1_length": 62.590818363273456, + "max_sentence1_length": 224, + "unique_sentence1": 1492, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "tel_Telu-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 157690, + "unique_pairs": 1499, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "tel_Telu-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 152094, + "unique_pairs": 1501, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "tel_Telu-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 158150, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "tel_Telu-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 163138, + "unique_pairs": 1501, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "tel_Telu-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 156822, + "unique_pairs": 1500, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "tel_Telu-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 152388, + "unique_pairs": 1501, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "tel_Telu-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 154375, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "tel_Telu-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 156070, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "tel_Telu-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 161283, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "tel_Telu-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 160787, + "unique_pairs": 1503, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "tel_Telu-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 158514, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "tel_Telu-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 168944, + "unique_pairs": 1500, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "tel_Telu-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 158853, + "unique_pairs": 1503, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "tel_Telu-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 153422, + "unique_pairs": 1501, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "tel_Telu-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 157012, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "tel_Telu-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 160329, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "tel_Telu-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 156307, + "unique_pairs": 1499, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "tel_Telu-san_Deva": { + "num_samples": 1503, + "number_of_characters": 154199, + "unique_pairs": 1501, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "tel_Telu-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 165485, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "tel_Telu-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 158729, + "unique_pairs": 1499, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "tel_Telu-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 170972, + "unique_pairs": 1502, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "tel_Telu-urd_Arab": { + "num_samples": 1503, + "number_of_characters": 157411, + "unique_pairs": 1499, + "min_sentence1_length": 6, + "average_sentence1_length": 51.16300731869594, + "max_sentence1_length": 182, + "unique_sentence1": 1495, + "min_sentence2_length": 4, + "average_sentence2_length": 53.568196939454424, + "max_sentence2_length": 206, + "unique_sentence2": 1498 + }, + "urd_Arab-asm_Beng": { + "num_samples": 1503, + "number_of_characters": 161305, + "unique_pairs": 1498, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 53.753825681969396, + "max_sentence2_length": 208, + "unique_sentence2": 1497 + }, + "urd_Arab-ben_Beng": { + "num_samples": 1503, + "number_of_characters": 155709, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 50.03060545575516, + "max_sentence2_length": 178, + "unique_sentence2": 1497 + }, + "urd_Arab-brx_Deva": { + "num_samples": 1503, + "number_of_characters": 161765, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 54.05988023952096, + "max_sentence2_length": 210, + "unique_sentence2": 1498 + }, + "urd_Arab-doi_Deva": { + "num_samples": 1503, + "number_of_characters": 166753, + "unique_pairs": 1500, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 6, + "average_sentence2_length": 57.37857618097139, + "max_sentence2_length": 209, + "unique_sentence2": 1499 + }, + "urd_Arab-eng_Latn": { + "num_samples": 1503, + "number_of_characters": 160437, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 53.17631403858949, + "max_sentence2_length": 201, + "unique_sentence2": 1497 + }, + "urd_Arab-gom_Deva": { + "num_samples": 1503, + "number_of_characters": 156003, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 50.22621423819029, + "max_sentence2_length": 203, + "unique_sentence2": 1500 + }, + "urd_Arab-guj_Gujr": { + "num_samples": 1503, + "number_of_characters": 157990, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 51.54823685961411, + "max_sentence2_length": 205, + "unique_sentence2": 1500 + }, + "urd_Arab-hin_Deva": { + "num_samples": 1503, + "number_of_characters": 159685, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 52.67598137059215, + "max_sentence2_length": 192, + "unique_sentence2": 1497 + }, + "urd_Arab-kan_Knda": { + "num_samples": 1503, + "number_of_characters": 164898, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 56.14437791084497, + "max_sentence2_length": 201, + "unique_sentence2": 1499 + }, + "urd_Arab-kas_Arab": { + "num_samples": 1503, + "number_of_characters": 164402, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 55.81437125748503, + "max_sentence2_length": 203, + "unique_sentence2": 1502 + }, + "urd_Arab-mai_Deva": { + "num_samples": 1503, + "number_of_characters": 162129, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 54.3020625415835, + "max_sentence2_length": 230, + "unique_sentence2": 1499 + }, + "urd_Arab-mal_Mlym": { + "num_samples": 1503, + "number_of_characters": 172559, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 5, + "average_sentence2_length": 61.24151696606786, + "max_sentence2_length": 219, + "unique_sentence2": 1495 + }, + "urd_Arab-mar_Deva": { + "num_samples": 1503, + "number_of_characters": 162468, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 54.52761144377911, + "max_sentence2_length": 221, + "unique_sentence2": 1501 + }, + "urd_Arab-mni_Mtei": { + "num_samples": 1503, + "number_of_characters": 157037, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 50.91417165668663, + "max_sentence2_length": 239, + "unique_sentence2": 1498 + }, + "urd_Arab-npi_Deva": { + "num_samples": 1503, + "number_of_characters": 160627, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 53.30272787757818, + "max_sentence2_length": 223, + "unique_sentence2": 1497 + }, + "urd_Arab-ory_Orya": { + "num_samples": 1503, + "number_of_characters": 163944, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 55.509647371922824, + "max_sentence2_length": 195, + "unique_sentence2": 1500 + }, + "urd_Arab-pan_Guru": { + "num_samples": 1503, + "number_of_characters": 159922, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 52.83366600133067, + "max_sentence2_length": 221, + "unique_sentence2": 1495 + }, + "urd_Arab-san_Deva": { + "num_samples": 1503, + "number_of_characters": 157814, + "unique_pairs": 1501, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 3, + "average_sentence2_length": 51.4311377245509, + "max_sentence2_length": 181, + "unique_sentence2": 1500 + }, + "urd_Arab-sat_Olck": { + "num_samples": 1503, + "number_of_characters": 169100, + "unique_pairs": 1502, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 7, + "average_sentence2_length": 58.94011976047904, + "max_sentence2_length": 225, + "unique_sentence2": 1500 + }, + "urd_Arab-snd_Deva": { + "num_samples": 1503, + "number_of_characters": 162344, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 4, + "average_sentence2_length": 54.445109780439125, + "max_sentence2_length": 195, + "unique_sentence2": 1490 + }, + "urd_Arab-tam_Taml": { + "num_samples": 1503, + "number_of_characters": 174587, + "unique_pairs": 1503, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 3, + "average_sentence2_length": 62.590818363273456, + "max_sentence2_length": 224, + "unique_sentence2": 1492 + }, + "urd_Arab-tel_Telu": { + "num_samples": 1503, + "number_of_characters": 157411, + "unique_pairs": 1499, + "min_sentence1_length": 4, + "average_sentence1_length": 53.568196939454424, + "max_sentence1_length": 206, + "unique_sentence1": 1498, + "min_sentence2_length": 6, + "average_sentence2_length": 51.16300731869594, + "max_sentence2_length": 182, + "unique_sentence2": 1495 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/IN22GenBitextMining.json b/mteb/descriptive_stats/BitextMining/IN22GenBitextMining.json new file mode 100644 index 0000000000..c53818c9ca --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/IN22GenBitextMining.json @@ -0,0 +1,6595 @@ +{ + "test": { + "num_samples": 518144, + "number_of_characters": 162367876, + "unique_pairs": 518101, + "min_sentence1_length": 9, + "average_sentence1_length": 156.6821925951087, + "max_sentence1_length": 692, + "unique_sentence1": 23550, + "min_sentence2_length": 9, + "average_sentence2_length": 156.6821925951087, + "max_sentence2_length": 692, + "unique_sentence2": 23550, + "hf_subset_descriptive_stats": { + "asm_Beng-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 310622, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "asm_Beng-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 323609, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "asm_Beng-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 319020, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "asm_Beng-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 320098, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "asm_Beng-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 312594, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "asm_Beng-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 309440, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "asm_Beng-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 320106, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "asm_Beng-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 332064, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "asm_Beng-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 322764, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "asm_Beng-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 308682, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "asm_Beng-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 343636, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "asm_Beng-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 321784, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "asm_Beng-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 313134, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "asm_Beng-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 313419, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "asm_Beng-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 334226, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "asm_Beng-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 306863, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "asm_Beng-san_Deva": { + "num_samples": 1024, + "number_of_characters": 318079, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "asm_Beng-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 326732, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "asm_Beng-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 320421, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "asm_Beng-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 348346, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "asm_Beng-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 319045, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "asm_Beng-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 315134, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 156.6982421875, + "max_sentence1_length": 582, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "ben_Beng-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 310622, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "ben_Beng-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 313313, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "ben_Beng-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 308724, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "ben_Beng-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 309802, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "ben_Beng-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 302298, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "ben_Beng-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 299144, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "ben_Beng-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 309810, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "ben_Beng-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 321768, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "ben_Beng-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 312468, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "ben_Beng-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 298386, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "ben_Beng-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 333340, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "ben_Beng-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 311488, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "ben_Beng-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 302838, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "ben_Beng-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 303123, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "ben_Beng-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 323930, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "ben_Beng-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 296567, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "ben_Beng-san_Deva": { + "num_samples": 1024, + "number_of_characters": 307783, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "ben_Beng-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 316436, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "ben_Beng-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 310125, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "ben_Beng-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 338050, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "ben_Beng-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 308749, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "ben_Beng-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 304838, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 146.6435546875, + "max_sentence1_length": 538, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "brx_Deva-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 323609, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "brx_Deva-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 313313, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "brx_Deva-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 321711, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "brx_Deva-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 322789, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "brx_Deva-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 315285, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "brx_Deva-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 312131, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "brx_Deva-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 322797, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "brx_Deva-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 334755, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "brx_Deva-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 325455, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "brx_Deva-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 311373, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "brx_Deva-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 346327, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "brx_Deva-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 324475, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "brx_Deva-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 315825, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "brx_Deva-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 316110, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "brx_Deva-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 336917, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "brx_Deva-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 309554, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "brx_Deva-san_Deva": { + "num_samples": 1024, + "number_of_characters": 320770, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "brx_Deva-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 329423, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "brx_Deva-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 323112, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "brx_Deva-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 351037, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "brx_Deva-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 321736, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "brx_Deva-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 317825, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 159.326171875, + "max_sentence1_length": 631, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "doi_Deva-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 319020, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "doi_Deva-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 308724, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "doi_Deva-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 321711, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "doi_Deva-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 318200, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "doi_Deva-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 310696, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "doi_Deva-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 307542, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "doi_Deva-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 318208, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "doi_Deva-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 330166, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "doi_Deva-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 320866, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "doi_Deva-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 306784, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "doi_Deva-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 341738, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "doi_Deva-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 319886, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "doi_Deva-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 311236, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "doi_Deva-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 311521, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "doi_Deva-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 332328, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "doi_Deva-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 304965, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "doi_Deva-san_Deva": { + "num_samples": 1024, + "number_of_characters": 316181, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "doi_Deva-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 324834, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "doi_Deva-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 318523, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "doi_Deva-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 346448, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "doi_Deva-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 317147, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "doi_Deva-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 313236, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.8447265625, + "max_sentence1_length": 500, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "eng_Latn-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 320098, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "eng_Latn-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 309802, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "eng_Latn-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 322789, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "eng_Latn-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 318200, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "eng_Latn-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 311774, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "eng_Latn-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 308620, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "eng_Latn-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 319286, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "eng_Latn-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 331244, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "eng_Latn-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 321944, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "eng_Latn-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 307862, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "eng_Latn-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 342816, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "eng_Latn-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 320964, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "eng_Latn-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 312314, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "eng_Latn-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 312599, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "eng_Latn-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 333406, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "eng_Latn-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 306043, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "eng_Latn-san_Deva": { + "num_samples": 1024, + "number_of_characters": 317259, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "eng_Latn-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 325912, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "eng_Latn-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 319601, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "eng_Latn-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 347526, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "eng_Latn-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 318225, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "eng_Latn-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 314314, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 155.8974609375, + "max_sentence1_length": 532, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "gom_Deva-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 312594, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "gom_Deva-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 302298, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "gom_Deva-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 315285, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "gom_Deva-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 310696, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "gom_Deva-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 311774, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "gom_Deva-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 301116, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "gom_Deva-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 311782, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "gom_Deva-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 323740, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "gom_Deva-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 314440, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "gom_Deva-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 300358, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "gom_Deva-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 335312, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "gom_Deva-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 313460, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "gom_Deva-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 304810, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "gom_Deva-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 305095, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "gom_Deva-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 325902, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "gom_Deva-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 298539, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "gom_Deva-san_Deva": { + "num_samples": 1024, + "number_of_characters": 309755, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "gom_Deva-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 318408, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "gom_Deva-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 312097, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "gom_Deva-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 340022, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "gom_Deva-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 310721, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "gom_Deva-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 306810, + "unique_pairs": 1024, + "min_sentence1_length": 17, + "average_sentence1_length": 148.5693359375, + "max_sentence1_length": 537, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "guj_Gujr-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 309440, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "guj_Gujr-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 299144, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "guj_Gujr-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 312131, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "guj_Gujr-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 307542, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "guj_Gujr-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 308620, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "guj_Gujr-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 301116, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "guj_Gujr-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 308628, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "guj_Gujr-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 320586, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "guj_Gujr-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 311286, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "guj_Gujr-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 297204, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "guj_Gujr-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 332158, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "guj_Gujr-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 310306, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "guj_Gujr-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 301656, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "guj_Gujr-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 301941, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "guj_Gujr-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 322748, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "guj_Gujr-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 295385, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "guj_Gujr-san_Deva": { + "num_samples": 1024, + "number_of_characters": 306601, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "guj_Gujr-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 315254, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "guj_Gujr-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 308943, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "guj_Gujr-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 336868, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "guj_Gujr-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 307567, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "guj_Gujr-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 303656, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 145.4892578125, + "max_sentence1_length": 488, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "hin_Deva-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 320106, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "hin_Deva-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 309810, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "hin_Deva-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 322797, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "hin_Deva-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 318208, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "hin_Deva-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 319286, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "hin_Deva-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 311782, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "hin_Deva-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 308628, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "hin_Deva-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 331252, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "hin_Deva-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 321952, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "hin_Deva-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 307870, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "hin_Deva-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 342824, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "hin_Deva-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 320972, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "hin_Deva-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 312322, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "hin_Deva-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 312607, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "hin_Deva-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 333414, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "hin_Deva-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 306051, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "hin_Deva-san_Deva": { + "num_samples": 1024, + "number_of_characters": 317267, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "hin_Deva-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 325920, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "hin_Deva-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 319609, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "hin_Deva-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 347534, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "hin_Deva-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 318233, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "hin_Deva-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 314322, + "unique_pairs": 1024, + "min_sentence1_length": 21, + "average_sentence1_length": 155.9052734375, + "max_sentence1_length": 531, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "kan_Knda-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 332064, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "kan_Knda-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 321768, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "kan_Knda-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 334755, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "kan_Knda-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 330166, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "kan_Knda-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 331244, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "kan_Knda-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 323740, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "kan_Knda-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 320586, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "kan_Knda-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 331252, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "kan_Knda-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 333910, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "kan_Knda-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 319828, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "kan_Knda-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 354782, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "kan_Knda-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 332930, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "kan_Knda-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 324280, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "kan_Knda-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 324565, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "kan_Knda-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 345372, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "kan_Knda-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 318009, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "kan_Knda-san_Deva": { + "num_samples": 1024, + "number_of_characters": 329225, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "kan_Knda-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 337878, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "kan_Knda-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 331567, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "kan_Knda-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 359492, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "kan_Knda-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 330191, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "kan_Knda-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 326280, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 167.5830078125, + "max_sentence1_length": 668, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "kas_Arab-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 322764, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "kas_Arab-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 312468, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "kas_Arab-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 325455, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "kas_Arab-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 320866, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "kas_Arab-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 321944, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "kas_Arab-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 314440, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "kas_Arab-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 311286, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "kas_Arab-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 321952, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "kas_Arab-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 333910, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "kas_Arab-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 310528, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "kas_Arab-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 345482, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "kas_Arab-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 323630, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "kas_Arab-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 314980, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "kas_Arab-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 315265, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "kas_Arab-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 336072, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "kas_Arab-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 308709, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "kas_Arab-san_Deva": { + "num_samples": 1024, + "number_of_characters": 319925, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "kas_Arab-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 328578, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "kas_Arab-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 322267, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "kas_Arab-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 350192, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "kas_Arab-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 320891, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "kas_Arab-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 316980, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 158.5009765625, + "max_sentence1_length": 520, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "mai_Deva-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 308682, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "mai_Deva-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 298386, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "mai_Deva-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 311373, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "mai_Deva-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 306784, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "mai_Deva-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 307862, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "mai_Deva-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 300358, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "mai_Deva-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 297204, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "mai_Deva-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 307870, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "mai_Deva-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 319828, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "mai_Deva-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 310528, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "mai_Deva-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 331400, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "mai_Deva-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 309548, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "mai_Deva-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 300898, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "mai_Deva-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 301183, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "mai_Deva-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 321990, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "mai_Deva-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 294627, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "mai_Deva-san_Deva": { + "num_samples": 1024, + "number_of_characters": 305843, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "mai_Deva-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 314496, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "mai_Deva-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 308185, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "mai_Deva-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 336110, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "mai_Deva-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 306809, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "mai_Deva-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 302898, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 144.7490234375, + "max_sentence1_length": 562, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "mal_Mlym-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 343636, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "mal_Mlym-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 333340, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "mal_Mlym-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 346327, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "mal_Mlym-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 341738, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "mal_Mlym-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 342816, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "mal_Mlym-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 335312, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "mal_Mlym-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 332158, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "mal_Mlym-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 342824, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "mal_Mlym-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 354782, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "mal_Mlym-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 345482, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "mal_Mlym-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 331400, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "mal_Mlym-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 344502, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "mal_Mlym-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 335852, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "mal_Mlym-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 336137, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "mal_Mlym-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 356944, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "mal_Mlym-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 329581, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "mal_Mlym-san_Deva": { + "num_samples": 1024, + "number_of_characters": 340797, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "mal_Mlym-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 349450, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "mal_Mlym-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 343139, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "mal_Mlym-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 371064, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "mal_Mlym-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 341763, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "mal_Mlym-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 337852, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 178.8837890625, + "max_sentence1_length": 692, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "mar_Deva-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 321784, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "mar_Deva-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 311488, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "mar_Deva-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 324475, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "mar_Deva-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 319886, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "mar_Deva-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 320964, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "mar_Deva-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 313460, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "mar_Deva-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 310306, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "mar_Deva-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 320972, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "mar_Deva-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 332930, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "mar_Deva-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 323630, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "mar_Deva-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 309548, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "mar_Deva-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 344502, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "mar_Deva-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 314000, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "mar_Deva-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 314285, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "mar_Deva-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 335092, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "mar_Deva-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 307729, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "mar_Deva-san_Deva": { + "num_samples": 1024, + "number_of_characters": 318945, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "mar_Deva-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 327598, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "mar_Deva-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 321287, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "mar_Deva-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 349212, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "mar_Deva-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 319911, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "mar_Deva-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 316000, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 157.5439453125, + "max_sentence1_length": 555, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "mni_Mtei-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 313134, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "mni_Mtei-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 302838, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "mni_Mtei-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 315825, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "mni_Mtei-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 311236, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "mni_Mtei-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 312314, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "mni_Mtei-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 304810, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "mni_Mtei-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 301656, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "mni_Mtei-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 312322, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "mni_Mtei-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 324280, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "mni_Mtei-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 314980, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "mni_Mtei-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 300898, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "mni_Mtei-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 335852, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "mni_Mtei-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 314000, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "mni_Mtei-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 305635, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "mni_Mtei-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 326442, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "mni_Mtei-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 299079, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "mni_Mtei-san_Deva": { + "num_samples": 1024, + "number_of_characters": 310295, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "mni_Mtei-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 318948, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "mni_Mtei-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 312637, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "mni_Mtei-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 340562, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "mni_Mtei-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 311261, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "mni_Mtei-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 307350, + "unique_pairs": 1024, + "min_sentence1_length": 16, + "average_sentence1_length": 149.0966796875, + "max_sentence1_length": 597, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "npi_Deva-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 313419, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "npi_Deva-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 303123, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "npi_Deva-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 316110, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "npi_Deva-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 311521, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "npi_Deva-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 312599, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "npi_Deva-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 305095, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "npi_Deva-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 301941, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "npi_Deva-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 312607, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "npi_Deva-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 324565, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "npi_Deva-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 315265, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "npi_Deva-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 301183, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "npi_Deva-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 336137, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "npi_Deva-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 314285, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "npi_Deva-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 305635, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "npi_Deva-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 326727, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "npi_Deva-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 299364, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "npi_Deva-san_Deva": { + "num_samples": 1024, + "number_of_characters": 310580, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "npi_Deva-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 319233, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "npi_Deva-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 312922, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "npi_Deva-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 340847, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "npi_Deva-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 311546, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "npi_Deva-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 307635, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 149.375, + "max_sentence1_length": 525, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "ory_Orya-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 334226, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "ory_Orya-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 323930, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "ory_Orya-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 336917, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "ory_Orya-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 332328, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "ory_Orya-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 333406, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "ory_Orya-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 325902, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "ory_Orya-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 322748, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "ory_Orya-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 333414, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "ory_Orya-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 345372, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "ory_Orya-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 336072, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "ory_Orya-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 321990, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "ory_Orya-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 356944, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "ory_Orya-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 335092, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "ory_Orya-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 326442, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "ory_Orya-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 326727, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "ory_Orya-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 320171, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "ory_Orya-san_Deva": { + "num_samples": 1024, + "number_of_characters": 331387, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "ory_Orya-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 340040, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "ory_Orya-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 333729, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "ory_Orya-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 361654, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "ory_Orya-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 332353, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "ory_Orya-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 328442, + "unique_pairs": 1024, + "min_sentence1_length": 10, + "average_sentence1_length": 169.6943359375, + "max_sentence1_length": 578, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "pan_Guru-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 306863, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "pan_Guru-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 296567, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "pan_Guru-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 309554, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "pan_Guru-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 304965, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "pan_Guru-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 306043, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "pan_Guru-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 298539, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "pan_Guru-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 295385, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "pan_Guru-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 306051, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "pan_Guru-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 318009, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "pan_Guru-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 308709, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "pan_Guru-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 294627, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "pan_Guru-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 329581, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "pan_Guru-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 307729, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "pan_Guru-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 299079, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "pan_Guru-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 299364, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "pan_Guru-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 320171, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "pan_Guru-san_Deva": { + "num_samples": 1024, + "number_of_characters": 304024, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "pan_Guru-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 312677, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "pan_Guru-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 306366, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "pan_Guru-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 334291, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "pan_Guru-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 304990, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "pan_Guru-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 301079, + "unique_pairs": 1024, + "min_sentence1_length": 19, + "average_sentence1_length": 142.97265625, + "max_sentence1_length": 476, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "san_Deva-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 318079, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "san_Deva-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 307783, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "san_Deva-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 320770, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "san_Deva-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 316181, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "san_Deva-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 317259, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "san_Deva-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 309755, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "san_Deva-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 306601, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "san_Deva-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 317267, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "san_Deva-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 329225, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "san_Deva-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 319925, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "san_Deva-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 305843, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "san_Deva-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 340797, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "san_Deva-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 318945, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "san_Deva-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 310295, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "san_Deva-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 310580, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "san_Deva-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 331387, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "san_Deva-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 304024, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "san_Deva-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 323893, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "san_Deva-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 317582, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "san_Deva-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 345507, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "san_Deva-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 316206, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "san_Deva-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 312295, + "unique_pairs": 1024, + "min_sentence1_length": 9, + "average_sentence1_length": 153.92578125, + "max_sentence1_length": 601, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "sat_Olck-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 326732, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "sat_Olck-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 316436, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "sat_Olck-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 329423, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "sat_Olck-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 324834, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "sat_Olck-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 325912, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "sat_Olck-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 318408, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "sat_Olck-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 315254, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "sat_Olck-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 325920, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "sat_Olck-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 337878, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "sat_Olck-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 328578, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "sat_Olck-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 314496, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "sat_Olck-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 349450, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "sat_Olck-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 327598, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "sat_Olck-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 318948, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "sat_Olck-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 319233, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "sat_Olck-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 340040, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "sat_Olck-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 312677, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "sat_Olck-san_Deva": { + "num_samples": 1024, + "number_of_characters": 323893, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "sat_Olck-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 326235, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "sat_Olck-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 354160, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "sat_Olck-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 324859, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "sat_Olck-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 320948, + "unique_pairs": 1024, + "min_sentence1_length": 11, + "average_sentence1_length": 162.3759765625, + "max_sentence1_length": 536, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "snd_Deva-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 320421, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "snd_Deva-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 310125, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "snd_Deva-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 323112, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "snd_Deva-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 318523, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "snd_Deva-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 319601, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "snd_Deva-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 312097, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "snd_Deva-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 308943, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "snd_Deva-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 319609, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "snd_Deva-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 331567, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "snd_Deva-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 322267, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "snd_Deva-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 308185, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "snd_Deva-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 343139, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "snd_Deva-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 321287, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "snd_Deva-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 312637, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "snd_Deva-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 312922, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "snd_Deva-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 333729, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "snd_Deva-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 306366, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "snd_Deva-san_Deva": { + "num_samples": 1024, + "number_of_characters": 317582, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "snd_Deva-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 326235, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "snd_Deva-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 347849, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "snd_Deva-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 318548, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "snd_Deva-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 314637, + "unique_pairs": 1024, + "min_sentence1_length": 18, + "average_sentence1_length": 156.212890625, + "max_sentence1_length": 545, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "tam_Taml-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 348346, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "tam_Taml-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 338050, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "tam_Taml-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 351037, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "tam_Taml-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 346448, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "tam_Taml-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 347526, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "tam_Taml-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 340022, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "tam_Taml-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 336868, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "tam_Taml-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 347534, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "tam_Taml-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 359492, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "tam_Taml-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 350192, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "tam_Taml-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 336110, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "tam_Taml-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 371064, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "tam_Taml-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 349212, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "tam_Taml-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 340562, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "tam_Taml-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 340847, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "tam_Taml-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 361654, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "tam_Taml-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 334291, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "tam_Taml-san_Deva": { + "num_samples": 1024, + "number_of_characters": 345507, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "tam_Taml-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 354160, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "tam_Taml-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 347849, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "tam_Taml-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 346473, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + }, + "tam_Taml-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 342562, + "unique_pairs": 1024, + "min_sentence1_length": 32, + "average_sentence1_length": 183.4833984375, + "max_sentence1_length": 614, + "unique_sentence1": 1023, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "tel_Telu-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 319045, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "tel_Telu-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 308749, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "tel_Telu-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 321736, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "tel_Telu-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 317147, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "tel_Telu-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 318225, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "tel_Telu-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 310721, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "tel_Telu-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 307567, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "tel_Telu-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 318233, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "tel_Telu-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 330191, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "tel_Telu-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 320891, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "tel_Telu-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 306809, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "tel_Telu-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 341763, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "tel_Telu-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 319911, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "tel_Telu-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 311261, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "tel_Telu-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 311546, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "tel_Telu-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 332353, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "tel_Telu-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 304990, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "tel_Telu-san_Deva": { + "num_samples": 1024, + "number_of_characters": 316206, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "tel_Telu-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 324859, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "tel_Telu-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 318548, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "tel_Telu-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 346473, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "tel_Telu-urd_Arab": { + "num_samples": 1024, + "number_of_characters": 313261, + "unique_pairs": 1024, + "min_sentence1_length": 14, + "average_sentence1_length": 154.869140625, + "max_sentence1_length": 658, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 151.0498046875, + "max_sentence2_length": 574, + "unique_sentence2": 1024 + }, + "urd_Arab-asm_Beng": { + "num_samples": 1024, + "number_of_characters": 315134, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 156.6982421875, + "max_sentence2_length": 582, + "unique_sentence2": 1024 + }, + "urd_Arab-ben_Beng": { + "num_samples": 1024, + "number_of_characters": 304838, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 146.6435546875, + "max_sentence2_length": 538, + "unique_sentence2": 1024 + }, + "urd_Arab-brx_Deva": { + "num_samples": 1024, + "number_of_characters": 317825, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 159.326171875, + "max_sentence2_length": 631, + "unique_sentence2": 1024 + }, + "urd_Arab-doi_Deva": { + "num_samples": 1024, + "number_of_characters": 313236, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.8447265625, + "max_sentence2_length": 500, + "unique_sentence2": 1024 + }, + "urd_Arab-eng_Latn": { + "num_samples": 1024, + "number_of_characters": 314314, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 155.8974609375, + "max_sentence2_length": 532, + "unique_sentence2": 1024 + }, + "urd_Arab-gom_Deva": { + "num_samples": 1024, + "number_of_characters": 306810, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 17, + "average_sentence2_length": 148.5693359375, + "max_sentence2_length": 537, + "unique_sentence2": 1024 + }, + "urd_Arab-guj_Gujr": { + "num_samples": 1024, + "number_of_characters": 303656, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 145.4892578125, + "max_sentence2_length": 488, + "unique_sentence2": 1024 + }, + "urd_Arab-hin_Deva": { + "num_samples": 1024, + "number_of_characters": 314322, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 21, + "average_sentence2_length": 155.9052734375, + "max_sentence2_length": 531, + "unique_sentence2": 1024 + }, + "urd_Arab-kan_Knda": { + "num_samples": 1024, + "number_of_characters": 326280, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 167.5830078125, + "max_sentence2_length": 668, + "unique_sentence2": 1024 + }, + "urd_Arab-kas_Arab": { + "num_samples": 1024, + "number_of_characters": 316980, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 158.5009765625, + "max_sentence2_length": 520, + "unique_sentence2": 1024 + }, + "urd_Arab-mai_Deva": { + "num_samples": 1024, + "number_of_characters": 302898, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 144.7490234375, + "max_sentence2_length": 562, + "unique_sentence2": 1024 + }, + "urd_Arab-mal_Mlym": { + "num_samples": 1024, + "number_of_characters": 337852, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 13, + "average_sentence2_length": 178.8837890625, + "max_sentence2_length": 692, + "unique_sentence2": 1024 + }, + "urd_Arab-mar_Deva": { + "num_samples": 1024, + "number_of_characters": 316000, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 157.5439453125, + "max_sentence2_length": 555, + "unique_sentence2": 1024 + }, + "urd_Arab-mni_Mtei": { + "num_samples": 1024, + "number_of_characters": 307350, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 16, + "average_sentence2_length": 149.0966796875, + "max_sentence2_length": 597, + "unique_sentence2": 1024 + }, + "urd_Arab-npi_Deva": { + "num_samples": 1024, + "number_of_characters": 307635, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 149.375, + "max_sentence2_length": 525, + "unique_sentence2": 1024 + }, + "urd_Arab-ory_Orya": { + "num_samples": 1024, + "number_of_characters": 328442, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 10, + "average_sentence2_length": 169.6943359375, + "max_sentence2_length": 578, + "unique_sentence2": 1024 + }, + "urd_Arab-pan_Guru": { + "num_samples": 1024, + "number_of_characters": 301079, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 19, + "average_sentence2_length": 142.97265625, + "max_sentence2_length": 476, + "unique_sentence2": 1024 + }, + "urd_Arab-san_Deva": { + "num_samples": 1024, + "number_of_characters": 312295, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 9, + "average_sentence2_length": 153.92578125, + "max_sentence2_length": 601, + "unique_sentence2": 1024 + }, + "urd_Arab-sat_Olck": { + "num_samples": 1024, + "number_of_characters": 320948, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 11, + "average_sentence2_length": 162.3759765625, + "max_sentence2_length": 536, + "unique_sentence2": 1024 + }, + "urd_Arab-snd_Deva": { + "num_samples": 1024, + "number_of_characters": 314637, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 18, + "average_sentence2_length": 156.212890625, + "max_sentence2_length": 545, + "unique_sentence2": 1024 + }, + "urd_Arab-tam_Taml": { + "num_samples": 1024, + "number_of_characters": 342562, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 32, + "average_sentence2_length": 183.4833984375, + "max_sentence2_length": 614, + "unique_sentence2": 1023 + }, + "urd_Arab-tel_Telu": { + "num_samples": 1024, + "number_of_characters": 313261, + "unique_pairs": 1024, + "min_sentence1_length": 13, + "average_sentence1_length": 151.0498046875, + "max_sentence1_length": 574, + "unique_sentence1": 1024, + "min_sentence2_length": 14, + "average_sentence2_length": 154.869140625, + "max_sentence2_length": 658, + "unique_sentence2": 1024 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/IWSLT2017BitextMining.json b/mteb/descriptive_stats/BitextMining/IWSLT2017BitextMining.json new file mode 100644 index 0000000000..504c3f1905 --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/IWSLT2017BitextMining.json @@ -0,0 +1,329 @@ +{ + "validation": { + "num_samples": 21938, + "number_of_characters": 4256244, + "unique_pairs": 21840, + "min_sentence1_length": 2, + "average_sentence1_length": 97.0061992889051, + "max_sentence1_length": 521, + "unique_sentence1": 11563, + "min_sentence2_length": 2, + "average_sentence2_length": 97.0061992889051, + "max_sentence2_length": 521, + "unique_sentence2": 11563, + "hf_subset_descriptive_stats": { + "ar-en": { + "num_samples": 888, + "number_of_characters": 172499, + "unique_pairs": 887, + "min_sentence1_length": 4, + "average_sentence1_length": 85.48873873873873, + "max_sentence1_length": 369, + "unique_sentence1": 887, + "min_sentence2_length": 10, + "average_sentence2_length": 108.76689189189189, + "max_sentence2_length": 462, + "unique_sentence2": 881 + }, + "de-en": { + "num_samples": 888, + "number_of_characters": 202336, + "unique_pairs": 883, + "min_sentence1_length": 6, + "average_sentence1_length": 119.02702702702703, + "max_sentence1_length": 521, + "unique_sentence1": 881, + "min_sentence2_length": 10, + "average_sentence2_length": 108.82882882882883, + "max_sentence2_length": 462, + "unique_sentence2": 881 + }, + "en-ar": { + "num_samples": 888, + "number_of_characters": 172499, + "unique_pairs": 887, + "min_sentence1_length": 10, + "average_sentence1_length": 108.76689189189189, + "max_sentence1_length": 462, + "unique_sentence1": 881, + "min_sentence2_length": 4, + "average_sentence2_length": 85.48873873873873, + "max_sentence2_length": 369, + "unique_sentence2": 887 + }, + "en-de": { + "num_samples": 888, + "number_of_characters": 202336, + "unique_pairs": 883, + "min_sentence1_length": 10, + "average_sentence1_length": 108.82882882882883, + "max_sentence1_length": 462, + "unique_sentence1": 881, + "min_sentence2_length": 6, + "average_sentence2_length": 119.02702702702703, + "max_sentence2_length": 521, + "unique_sentence2": 881 + }, + "en-fr": { + "num_samples": 890, + "number_of_characters": 197619, + "unique_pairs": 883, + "min_sentence1_length": 10, + "average_sentence1_length": 108.4123595505618, + "max_sentence1_length": 462, + "unique_sentence1": 883, + "min_sentence2_length": 6, + "average_sentence2_length": 113.63146067415731, + "max_sentence2_length": 493, + "unique_sentence2": 881 + }, + "en-it": { + "num_samples": 929, + "number_of_characters": 191803, + "unique_pairs": 924, + "min_sentence1_length": 10, + "average_sentence1_length": 103.0010764262648, + "max_sentence1_length": 433, + "unique_sentence1": 922, + "min_sentence2_length": 7, + "average_sentence2_length": 103.46071044133477, + "max_sentence2_length": 444, + "unique_sentence2": 918 + }, + "en-ja": { + "num_samples": 871, + "number_of_characters": 132742, + "unique_pairs": 867, + "min_sentence1_length": 10, + "average_sentence1_length": 109.80826636050517, + "max_sentence1_length": 462, + "unique_sentence1": 864, + "min_sentence2_length": 5, + "average_sentence2_length": 42.59357060849598, + "max_sentence2_length": 225, + "unique_sentence2": 866 + }, + "en-ko": { + "num_samples": 879, + "number_of_characters": 142659, + "unique_pairs": 874, + "min_sentence1_length": 10, + "average_sentence1_length": 107.74175199089875, + "max_sentence1_length": 462, + "unique_sentence1": 872, + "min_sentence2_length": 3, + "average_sentence2_length": 54.5551763367463, + "max_sentence2_length": 250, + "unique_sentence2": 872 + }, + "en-nl": { + "num_samples": 1003, + "number_of_characters": 189637, + "unique_pairs": 1000, + "min_sentence1_length": 10, + "average_sentence1_length": 95.26819541375872, + "max_sentence1_length": 433, + "unique_sentence1": 996, + "min_sentence2_length": 4, + "average_sentence2_length": 93.80159521435692, + "max_sentence2_length": 477, + "unique_sentence2": 1000 + }, + "en-ro": { + "num_samples": 914, + "number_of_characters": 194128, + "unique_pairs": 910, + "min_sentence1_length": 10, + "average_sentence1_length": 104.72100656455142, + "max_sentence1_length": 433, + "unique_sentence1": 907, + "min_sentence2_length": 9, + "average_sentence2_length": 107.67286652078775, + "max_sentence2_length": 448, + "unique_sentence2": 910 + }, + "en-zh": { + "num_samples": 879, + "number_of_characters": 131126, + "unique_pairs": 877, + "min_sentence1_length": 10, + "average_sentence1_length": 109.36518771331058, + "max_sentence1_length": 462, + "unique_sentence1": 872, + "min_sentence2_length": 2, + "average_sentence2_length": 39.811149032992034, + "max_sentence2_length": 230, + "unique_sentence2": 867 + }, + "fr-en": { + "num_samples": 890, + "number_of_characters": 197619, + "unique_pairs": 883, + "min_sentence1_length": 6, + "average_sentence1_length": 113.63146067415731, + "max_sentence1_length": 493, + "unique_sentence1": 881, + "min_sentence2_length": 10, + "average_sentence2_length": 108.4123595505618, + "max_sentence2_length": 462, + "unique_sentence2": 883 + }, + "it-en": { + "num_samples": 929, + "number_of_characters": 191803, + "unique_pairs": 924, + "min_sentence1_length": 7, + "average_sentence1_length": 103.46071044133477, + "max_sentence1_length": 444, + "unique_sentence1": 918, + "min_sentence2_length": 10, + "average_sentence2_length": 103.0010764262648, + "max_sentence2_length": 433, + "unique_sentence2": 922 + }, + "it-nl": { + "num_samples": 1001, + "number_of_characters": 188858, + "unique_pairs": 998, + "min_sentence1_length": 7, + "average_sentence1_length": 94.64235764235764, + "max_sentence1_length": 459, + "unique_sentence1": 994, + "min_sentence2_length": 7, + "average_sentence2_length": 94.02697302697302, + "max_sentence2_length": 505, + "unique_sentence2": 998 + }, + "it-ro": { + "num_samples": 914, + "number_of_characters": 193339, + "unique_pairs": 911, + "min_sentence1_length": 7, + "average_sentence1_length": 103.90809628008753, + "max_sentence1_length": 435, + "unique_sentence1": 907, + "min_sentence2_length": 9, + "average_sentence2_length": 107.62253829321664, + "max_sentence2_length": 448, + "unique_sentence2": 910 + }, + "ja-en": { + "num_samples": 871, + "number_of_characters": 132742, + "unique_pairs": 867, + "min_sentence1_length": 5, + "average_sentence1_length": 42.59357060849598, + "max_sentence1_length": 225, + "unique_sentence1": 866, + "min_sentence2_length": 10, + "average_sentence2_length": 109.80826636050517, + "max_sentence2_length": 462, + "unique_sentence2": 864 + }, + "ko-en": { + "num_samples": 879, + "number_of_characters": 142659, + "unique_pairs": 874, + "min_sentence1_length": 3, + "average_sentence1_length": 54.5551763367463, + "max_sentence1_length": 250, + "unique_sentence1": 872, + "min_sentence2_length": 10, + "average_sentence2_length": 107.74175199089875, + "max_sentence2_length": 462, + "unique_sentence2": 872 + }, + "nl-en": { + "num_samples": 1003, + "number_of_characters": 189637, + "unique_pairs": 1000, + "min_sentence1_length": 4, + "average_sentence1_length": 93.80159521435692, + "max_sentence1_length": 477, + "unique_sentence1": 1000, + "min_sentence2_length": 10, + "average_sentence2_length": 95.26819541375872, + "max_sentence2_length": 433, + "unique_sentence2": 996 + }, + "nl-it": { + "num_samples": 1001, + "number_of_characters": 188858, + "unique_pairs": 998, + "min_sentence1_length": 7, + "average_sentence1_length": 94.02697302697302, + "max_sentence1_length": 505, + "unique_sentence1": 998, + "min_sentence2_length": 7, + "average_sentence2_length": 94.64235764235764, + "max_sentence2_length": 459, + "unique_sentence2": 994 + }, + "nl-ro": { + "num_samples": 913, + "number_of_characters": 191376, + "unique_pairs": 911, + "min_sentence1_length": 7, + "average_sentence1_length": 102.01971522453451, + "max_sentence1_length": 478, + "unique_sentence1": 909, + "min_sentence2_length": 9, + "average_sentence2_length": 107.59255202628697, + "max_sentence2_length": 515, + "unique_sentence2": 909 + }, + "ro-en": { + "num_samples": 914, + "number_of_characters": 194128, + "unique_pairs": 910, + "min_sentence1_length": 9, + "average_sentence1_length": 107.67286652078775, + "max_sentence1_length": 448, + "unique_sentence1": 910, + "min_sentence2_length": 10, + "average_sentence2_length": 104.72100656455142, + "max_sentence2_length": 433, + "unique_sentence2": 907 + }, + "ro-it": { + "num_samples": 914, + "number_of_characters": 193339, + "unique_pairs": 911, + "min_sentence1_length": 9, + "average_sentence1_length": 107.62253829321664, + "max_sentence1_length": 448, + "unique_sentence1": 910, + "min_sentence2_length": 7, + "average_sentence2_length": 103.90809628008753, + "max_sentence2_length": 435, + "unique_sentence2": 907 + }, + "ro-nl": { + "num_samples": 913, + "number_of_characters": 191376, + "unique_pairs": 911, + "min_sentence1_length": 9, + "average_sentence1_length": 107.59255202628697, + "max_sentence1_length": 515, + "unique_sentence1": 909, + "min_sentence2_length": 7, + "average_sentence2_length": 102.01971522453451, + "max_sentence2_length": 478, + "unique_sentence2": 909 + }, + "zh-en": { + "num_samples": 879, + "number_of_characters": 131126, + "unique_pairs": 877, + "min_sentence1_length": 2, + "average_sentence1_length": 39.811149032992034, + "max_sentence1_length": 230, + "unique_sentence1": 867, + "min_sentence2_length": 10, + "average_sentence2_length": 109.36518771331058, + "max_sentence2_length": 462, + "unique_sentence2": 872 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/IndicGenBenchFloresBitextMining.json b/mteb/descriptive_stats/BitextMining/IndicGenBenchFloresBitextMining.json new file mode 100644 index 0000000000..1aaed39454 --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/IndicGenBenchFloresBitextMining.json @@ -0,0 +1,1540 @@ +{ + "validation": { + "num_samples": 57826, + "number_of_characters": 14600950, + "unique_pairs": 57826, + "min_sentence1_length": 24, + "average_sentence1_length": 126.2541071490333, + "max_sentence1_length": 368, + "unique_sentence1": 29903, + "min_sentence2_length": 24, + "average_sentence2_length": 126.24390412617161, + "max_sentence2_length": 368, + "unique_sentence2": 29903, + "hf_subset_descriptive_stats": { + "ben-eng": { + "num_samples": 997, + "number_of_characters": 248469, + "unique_pairs": 997, + "min_sentence1_length": 30, + "average_sentence1_length": 123.64593781344033, + "max_sentence1_length": 320, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-ben": { + "num_samples": 997, + "number_of_characters": 248469, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 30, + "average_sentence2_length": 123.64593781344033, + "max_sentence2_length": 320, + "unique_sentence2": 997 + }, + "guj-eng": { + "num_samples": 997, + "number_of_characters": 245477, + "unique_pairs": 997, + "min_sentence1_length": 30, + "average_sentence1_length": 120.64493480441324, + "max_sentence1_length": 368, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-guj": { + "num_samples": 997, + "number_of_characters": 245477, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 30, + "average_sentence2_length": 120.64493480441324, + "max_sentence2_length": 368, + "unique_sentence2": 997 + }, + "hin-eng": { + "num_samples": 997, + "number_of_characters": 250573, + "unique_pairs": 997, + "min_sentence1_length": 31, + "average_sentence1_length": 125.75626880641926, + "max_sentence1_length": 355, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-hin": { + "num_samples": 997, + "number_of_characters": 250564, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 31, + "average_sentence2_length": 125.74724172517553, + "max_sentence2_length": 355, + "unique_sentence2": 997 + }, + "kan-eng": { + "num_samples": 997, + "number_of_characters": 257131, + "unique_pairs": 997, + "min_sentence1_length": 34, + "average_sentence1_length": 132.33400200601807, + "max_sentence1_length": 331, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-kan": { + "num_samples": 997, + "number_of_characters": 256986, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 34, + "average_sentence2_length": 132.18856569709126, + "max_sentence2_length": 331, + "unique_sentence2": 997 + }, + "mal-eng": { + "num_samples": 997, + "number_of_characters": 267295, + "unique_pairs": 997, + "min_sentence1_length": 31, + "average_sentence1_length": 142.52858575727183, + "max_sentence1_length": 360, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-mal": { + "num_samples": 997, + "number_of_characters": 267296, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 31, + "average_sentence2_length": 142.5295887662989, + "max_sentence2_length": 360, + "unique_sentence2": 997 + }, + "mar-eng": { + "num_samples": 997, + "number_of_characters": 251107, + "unique_pairs": 997, + "min_sentence1_length": 29, + "average_sentence1_length": 126.29187562688064, + "max_sentence1_length": 321, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-mar": { + "num_samples": 997, + "number_of_characters": 250897, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 29, + "average_sentence2_length": 126.08124373119358, + "max_sentence2_length": 321, + "unique_sentence2": 997 + }, + "tam-eng": { + "num_samples": 997, + "number_of_characters": 271322, + "unique_pairs": 997, + "min_sentence1_length": 30, + "average_sentence1_length": 146.567703109328, + "max_sentence1_length": 358, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-tam": { + "num_samples": 997, + "number_of_characters": 271322, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 30, + "average_sentence2_length": 146.567703109328, + "max_sentence2_length": 358, + "unique_sentence2": 997 + }, + "tel-eng": { + "num_samples": 997, + "number_of_characters": 252385, + "unique_pairs": 997, + "min_sentence1_length": 29, + "average_sentence1_length": 127.57372116349048, + "max_sentence1_length": 317, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-tel": { + "num_samples": 997, + "number_of_characters": 252380, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 29, + "average_sentence2_length": 127.56870611835507, + "max_sentence2_length": 317, + "unique_sentence2": 997 + }, + "urd-eng": { + "num_samples": 997, + "number_of_characters": 249824, + "unique_pairs": 997, + "min_sentence1_length": 37, + "average_sentence1_length": 125.00501504513541, + "max_sentence1_length": 295, + "unique_sentence1": 996, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-urd": { + "num_samples": 997, + "number_of_characters": 249824, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 37, + "average_sentence2_length": 125.00501504513541, + "max_sentence2_length": 295, + "unique_sentence2": 996 + }, + "asm-eng": { + "num_samples": 997, + "number_of_characters": 246220, + "unique_pairs": 997, + "min_sentence1_length": 30, + "average_sentence1_length": 121.3901705115346, + "max_sentence1_length": 314, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-asm": { + "num_samples": 997, + "number_of_characters": 246224, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 30, + "average_sentence2_length": 121.39418254764293, + "max_sentence2_length": 314, + "unique_sentence2": 997 + }, + "bho-eng": { + "num_samples": 997, + "number_of_characters": 246895, + "unique_pairs": 997, + "min_sentence1_length": 25, + "average_sentence1_length": 122.06720160481444, + "max_sentence1_length": 326, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-bho": { + "num_samples": 997, + "number_of_characters": 246919, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 25, + "average_sentence2_length": 122.0912738214644, + "max_sentence2_length": 326, + "unique_sentence2": 997 + }, + "nep-eng": { + "num_samples": 997, + "number_of_characters": 245984, + "unique_pairs": 997, + "min_sentence1_length": 24, + "average_sentence1_length": 121.15346038114343, + "max_sentence1_length": 307, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-nep": { + "num_samples": 997, + "number_of_characters": 245984, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 24, + "average_sentence2_length": 121.15346038114343, + "max_sentence2_length": 307, + "unique_sentence2": 997 + }, + "ory-eng": { + "num_samples": 997, + "number_of_characters": 254206, + "unique_pairs": 997, + "min_sentence1_length": 34, + "average_sentence1_length": 129.4002006018054, + "max_sentence1_length": 308, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-ory": { + "num_samples": 997, + "number_of_characters": 254206, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 34, + "average_sentence2_length": 129.4002006018054, + "max_sentence2_length": 308, + "unique_sentence2": 997 + }, + "pan-eng": { + "num_samples": 997, + "number_of_characters": 251598, + "unique_pairs": 997, + "min_sentence1_length": 29, + "average_sentence1_length": 126.78435305917753, + "max_sentence1_length": 309, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-pan": { + "num_samples": 997, + "number_of_characters": 251597, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 29, + "average_sentence2_length": 126.78335005015045, + "max_sentence2_length": 309, + "unique_sentence2": 997 + }, + "pus-eng": { + "num_samples": 997, + "number_of_characters": 247450, + "unique_pairs": 997, + "min_sentence1_length": 32, + "average_sentence1_length": 122.62387161484453, + "max_sentence1_length": 300, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-pus": { + "num_samples": 997, + "number_of_characters": 247450, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 32, + "average_sentence2_length": 122.62387161484453, + "max_sentence2_length": 300, + "unique_sentence2": 997 + }, + "san-eng": { + "num_samples": 997, + "number_of_characters": 249042, + "unique_pairs": 997, + "min_sentence1_length": 31, + "average_sentence1_length": 124.22066198595788, + "max_sentence1_length": 311, + "unique_sentence1": 994, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-san": { + "num_samples": 997, + "number_of_characters": 248877, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 31, + "average_sentence2_length": 124.05516549648947, + "max_sentence2_length": 311, + "unique_sentence2": 994 + }, + "awa-eng": { + "num_samples": 997, + "number_of_characters": 247944, + "unique_pairs": 997, + "min_sentence1_length": 34, + "average_sentence1_length": 123.11935807422267, + "max_sentence1_length": 329, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-awa": { + "num_samples": 997, + "number_of_characters": 247884, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 34, + "average_sentence2_length": 123.05917753259779, + "max_sentence2_length": 329, + "unique_sentence2": 997 + }, + "bgc-eng": { + "num_samples": 997, + "number_of_characters": 245935, + "unique_pairs": 997, + "min_sentence1_length": 27, + "average_sentence1_length": 121.10431293881645, + "max_sentence1_length": 303, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-bgc": { + "num_samples": 997, + "number_of_characters": 245935, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 27, + "average_sentence2_length": 121.10431293881645, + "max_sentence2_length": 303, + "unique_sentence2": 997 + }, + "bod-eng": { + "num_samples": 997, + "number_of_characters": 266515, + "unique_pairs": 997, + "min_sentence1_length": 26, + "average_sentence1_length": 141.74623871614844, + "max_sentence1_length": 355, + "unique_sentence1": 996, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-bod": { + "num_samples": 997, + "number_of_characters": 266495, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 26, + "average_sentence2_length": 141.72617853560683, + "max_sentence2_length": 355, + "unique_sentence2": 996 + }, + "boy-eng": { + "num_samples": 997, + "number_of_characters": 260174, + "unique_pairs": 997, + "min_sentence1_length": 31, + "average_sentence1_length": 135.38615847542627, + "max_sentence1_length": 312, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-boy": { + "num_samples": 997, + "number_of_characters": 260174, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 31, + "average_sentence2_length": 135.38615847542627, + "max_sentence2_length": 312, + "unique_sentence2": 997 + }, + "gbm-eng": { + "num_samples": 997, + "number_of_characters": 247009, + "unique_pairs": 997, + "min_sentence1_length": 30, + "average_sentence1_length": 122.18154463390171, + "max_sentence1_length": 344, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-gbm": { + "num_samples": 997, + "number_of_characters": 247009, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 30, + "average_sentence2_length": 122.18154463390171, + "max_sentence2_length": 344, + "unique_sentence2": 997 + }, + "gom-eng": { + "num_samples": 997, + "number_of_characters": 244553, + "unique_pairs": 997, + "min_sentence1_length": 31, + "average_sentence1_length": 119.71815446339016, + "max_sentence1_length": 306, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-gom": { + "num_samples": 997, + "number_of_characters": 244553, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 31, + "average_sentence2_length": 119.71815446339016, + "max_sentence2_length": 306, + "unique_sentence2": 997 + }, + "hne-eng": { + "num_samples": 997, + "number_of_characters": 246416, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 121.58676028084253, + "max_sentence1_length": 321, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-hne": { + "num_samples": 997, + "number_of_characters": 246405, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 121.57572718154464, + "max_sentence2_length": 321, + "unique_sentence2": 997 + }, + "raj-eng": { + "num_samples": 997, + "number_of_characters": 249541, + "unique_pairs": 997, + "min_sentence1_length": 32, + "average_sentence1_length": 124.72116349047141, + "max_sentence1_length": 313, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-raj": { + "num_samples": 997, + "number_of_characters": 249541, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 32, + "average_sentence2_length": 124.72116349047141, + "max_sentence2_length": 313, + "unique_sentence2": 997 + }, + "mai-eng": { + "num_samples": 997, + "number_of_characters": 247991, + "unique_pairs": 997, + "min_sentence1_length": 29, + "average_sentence1_length": 123.16649949849548, + "max_sentence1_length": 312, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-mai": { + "num_samples": 997, + "number_of_characters": 247994, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 29, + "average_sentence2_length": 123.16950852557673, + "max_sentence2_length": 312, + "unique_sentence2": 997 + }, + "mni-eng": { + "num_samples": 997, + "number_of_characters": 254308, + "unique_pairs": 997, + "min_sentence1_length": 39, + "average_sentence1_length": 129.5025075225677, + "max_sentence1_length": 310, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-mni": { + "num_samples": 997, + "number_of_characters": 254312, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 39, + "average_sentence2_length": 129.50651955867602, + "max_sentence2_length": 310, + "unique_sentence2": 997 + }, + "mup-eng": { + "num_samples": 997, + "number_of_characters": 248486, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 123.6629889669007, + "max_sentence1_length": 312, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-mup": { + "num_samples": 997, + "number_of_characters": 248486, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 123.6629889669007, + "max_sentence2_length": 312, + "unique_sentence2": 997 + }, + "mwr-eng": { + "num_samples": 997, + "number_of_characters": 248641, + "unique_pairs": 997, + "min_sentence1_length": 31, + "average_sentence1_length": 123.81845536609829, + "max_sentence1_length": 324, + "unique_sentence1": 997, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-mwr": { + "num_samples": 997, + "number_of_characters": 248641, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 31, + "average_sentence2_length": 123.81845536609829, + "max_sentence2_length": 324, + "unique_sentence2": 997 + }, + "sat-eng": { + "num_samples": 997, + "number_of_characters": 258279, + "unique_pairs": 997, + "min_sentence1_length": 37, + "average_sentence1_length": 133.4854563691073, + "max_sentence1_length": 333, + "unique_sentence1": 995, + "min_sentence2_length": 28, + "average_sentence2_length": 125.57071213640923, + "max_sentence2_length": 297, + "unique_sentence2": 997 + }, + "eng-sat": { + "num_samples": 997, + "number_of_characters": 258279, + "unique_pairs": 997, + "min_sentence1_length": 28, + "average_sentence1_length": 125.57071213640923, + "max_sentence1_length": 297, + "unique_sentence1": 997, + "min_sentence2_length": 37, + "average_sentence2_length": 133.4854563691073, + "max_sentence2_length": 333, + "unique_sentence2": 995 + } + } + }, + "test": { + "num_samples": 58696, + "number_of_characters": 15359416, + "unique_pairs": 58690, + "min_sentence1_length": 33, + "average_sentence1_length": 130.84266389532507, + "max_sentence1_length": 431, + "unique_sentence1": 30351, + "min_sentence2_length": 33, + "average_sentence2_length": 130.834724683113, + "max_sentence2_length": 431, + "unique_sentence2": 30351, + "hf_subset_descriptive_stats": { + "ben-eng": { + "num_samples": 1012, + "number_of_characters": 261008, + "unique_pairs": 1012, + "min_sentence1_length": 38, + "average_sentence1_length": 127.51185770750988, + "max_sentence1_length": 333, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-ben": { + "num_samples": 1012, + "number_of_characters": 261008, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 38, + "average_sentence2_length": 127.51185770750988, + "max_sentence2_length": 333, + "unique_sentence2": 1012 + }, + "guj-eng": { + "num_samples": 1012, + "number_of_characters": 258394, + "unique_pairs": 1012, + "min_sentence1_length": 38, + "average_sentence1_length": 124.92885375494072, + "max_sentence1_length": 349, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-guj": { + "num_samples": 1012, + "number_of_characters": 258394, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 38, + "average_sentence2_length": 124.92885375494072, + "max_sentence2_length": 349, + "unique_sentence2": 1012 + }, + "hin-eng": { + "num_samples": 1012, + "number_of_characters": 263040, + "unique_pairs": 1012, + "min_sentence1_length": 41, + "average_sentence1_length": 129.5197628458498, + "max_sentence1_length": 381, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-hin": { + "num_samples": 1012, + "number_of_characters": 263029, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 41, + "average_sentence2_length": 129.5088932806324, + "max_sentence2_length": 381, + "unique_sentence2": 1012 + }, + "kan-eng": { + "num_samples": 1012, + "number_of_characters": 270091, + "unique_pairs": 1012, + "min_sentence1_length": 43, + "average_sentence1_length": 136.48715415019763, + "max_sentence1_length": 388, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-kan": { + "num_samples": 1012, + "number_of_characters": 270021, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 43, + "average_sentence2_length": 136.4179841897233, + "max_sentence2_length": 388, + "unique_sentence2": 1012 + }, + "mal-eng": { + "num_samples": 1012, + "number_of_characters": 281302, + "unique_pairs": 1012, + "min_sentence1_length": 48, + "average_sentence1_length": 147.56521739130434, + "max_sentence1_length": 376, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-mal": { + "num_samples": 1012, + "number_of_characters": 281302, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 48, + "average_sentence2_length": 147.56521739130434, + "max_sentence2_length": 376, + "unique_sentence2": 1012 + }, + "mar-eng": { + "num_samples": 1012, + "number_of_characters": 265212, + "unique_pairs": 1012, + "min_sentence1_length": 34, + "average_sentence1_length": 131.66600790513834, + "max_sentence1_length": 356, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-mar": { + "num_samples": 1012, + "number_of_characters": 265023, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 34, + "average_sentence2_length": 131.47924901185772, + "max_sentence2_length": 355, + "unique_sentence2": 1012 + }, + "tam-eng": { + "num_samples": 1012, + "number_of_characters": 286099, + "unique_pairs": 1012, + "min_sentence1_length": 48, + "average_sentence1_length": 152.30533596837944, + "max_sentence1_length": 404, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-tam": { + "num_samples": 1012, + "number_of_characters": 286099, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 48, + "average_sentence2_length": 152.30533596837944, + "max_sentence2_length": 404, + "unique_sentence2": 1012 + }, + "tel-eng": { + "num_samples": 1012, + "number_of_characters": 264460, + "unique_pairs": 1012, + "min_sentence1_length": 39, + "average_sentence1_length": 130.92292490118578, + "max_sentence1_length": 359, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-tel": { + "num_samples": 1012, + "number_of_characters": 264447, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 39, + "average_sentence2_length": 130.9100790513834, + "max_sentence2_length": 359, + "unique_sentence2": 1012 + }, + "urd-eng": { + "num_samples": 1012, + "number_of_characters": 261886, + "unique_pairs": 1012, + "min_sentence1_length": 34, + "average_sentence1_length": 128.37944664031622, + "max_sentence1_length": 348, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-urd": { + "num_samples": 1012, + "number_of_characters": 261885, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 34, + "average_sentence2_length": 128.37845849802372, + "max_sentence2_length": 348, + "unique_sentence2": 1012 + }, + "asm-eng": { + "num_samples": 1012, + "number_of_characters": 257902, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 124.44268774703558, + "max_sentence1_length": 329, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-asm": { + "num_samples": 1012, + "number_of_characters": 257909, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 124.449604743083, + "max_sentence2_length": 329, + "unique_sentence2": 1012 + }, + "bho-eng": { + "num_samples": 1012, + "number_of_characters": 260578, + "unique_pairs": 1012, + "min_sentence1_length": 36, + "average_sentence1_length": 127.08695652173913, + "max_sentence1_length": 367, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-bho": { + "num_samples": 1012, + "number_of_characters": 260601, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 36, + "average_sentence2_length": 127.1096837944664, + "max_sentence2_length": 367, + "unique_sentence2": 1012 + }, + "nep-eng": { + "num_samples": 1012, + "number_of_characters": 258869, + "unique_pairs": 1012, + "min_sentence1_length": 34, + "average_sentence1_length": 125.39822134387352, + "max_sentence1_length": 362, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-nep": { + "num_samples": 1012, + "number_of_characters": 258869, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 34, + "average_sentence2_length": 125.39822134387352, + "max_sentence2_length": 362, + "unique_sentence2": 1012 + }, + "ory-eng": { + "num_samples": 1012, + "number_of_characters": 266805, + "unique_pairs": 1012, + "min_sentence1_length": 38, + "average_sentence1_length": 133.2401185770751, + "max_sentence1_length": 354, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-ory": { + "num_samples": 1012, + "number_of_characters": 266805, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 38, + "average_sentence2_length": 133.2401185770751, + "max_sentence2_length": 354, + "unique_sentence2": 1012 + }, + "pan-eng": { + "num_samples": 1012, + "number_of_characters": 265391, + "unique_pairs": 1012, + "min_sentence1_length": 37, + "average_sentence1_length": 131.84288537549406, + "max_sentence1_length": 380, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-pan": { + "num_samples": 1012, + "number_of_characters": 265391, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 37, + "average_sentence2_length": 131.84288537549406, + "max_sentence2_length": 380, + "unique_sentence2": 1012 + }, + "pus-eng": { + "num_samples": 1012, + "number_of_characters": 254422, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 121.00395256916995, + "max_sentence1_length": 325, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-pus": { + "num_samples": 1012, + "number_of_characters": 254421, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 121.00296442687747, + "max_sentence2_length": 325, + "unique_sentence2": 1012 + }, + "san-eng": { + "num_samples": 1012, + "number_of_characters": 260339, + "unique_pairs": 1012, + "min_sentence1_length": 33, + "average_sentence1_length": 126.85079051383399, + "max_sentence1_length": 358, + "unique_sentence1": 1011, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-san": { + "num_samples": 1012, + "number_of_characters": 260224, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 33, + "average_sentence2_length": 126.73715415019763, + "max_sentence2_length": 358, + "unique_sentence2": 1011 + }, + "awa-eng": { + "num_samples": 1012, + "number_of_characters": 260179, + "unique_pairs": 1012, + "min_sentence1_length": 34, + "average_sentence1_length": 126.69268774703558, + "max_sentence1_length": 378, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-awa": { + "num_samples": 1012, + "number_of_characters": 260137, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 34, + "average_sentence2_length": 126.65118577075098, + "max_sentence2_length": 378, + "unique_sentence2": 1012 + }, + "bgc-eng": { + "num_samples": 1012, + "number_of_characters": 257450, + "unique_pairs": 1012, + "min_sentence1_length": 38, + "average_sentence1_length": 123.99604743083005, + "max_sentence1_length": 332, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-bgc": { + "num_samples": 1012, + "number_of_characters": 257450, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 38, + "average_sentence2_length": 123.99604743083005, + "max_sentence2_length": 332, + "unique_sentence2": 1012 + }, + "bod-eng": { + "num_samples": 1012, + "number_of_characters": 280188, + "unique_pairs": 1012, + "min_sentence1_length": 42, + "average_sentence1_length": 146.46442687747034, + "max_sentence1_length": 431, + "unique_sentence1": 1009, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-bod": { + "num_samples": 1012, + "number_of_characters": 280126, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 42, + "average_sentence2_length": 146.40316205533597, + "max_sentence2_length": 431, + "unique_sentence2": 1009 + }, + "boy-eng": { + "num_samples": 1012, + "number_of_characters": 277538, + "unique_pairs": 1012, + "min_sentence1_length": 36, + "average_sentence1_length": 143.84584980237153, + "max_sentence1_length": 396, + "unique_sentence1": 1011, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-boy": { + "num_samples": 1012, + "number_of_characters": 277538, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 36, + "average_sentence2_length": 143.84584980237153, + "max_sentence2_length": 396, + "unique_sentence2": 1011 + }, + "gbm-eng": { + "num_samples": 1012, + "number_of_characters": 261027, + "unique_pairs": 1012, + "min_sentence1_length": 38, + "average_sentence1_length": 127.53063241106719, + "max_sentence1_length": 333, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-gbm": { + "num_samples": 1012, + "number_of_characters": 261027, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 38, + "average_sentence2_length": 127.53063241106719, + "max_sentence2_length": 333, + "unique_sentence2": 1012 + }, + "gom-eng": { + "num_samples": 1012, + "number_of_characters": 259182, + "unique_pairs": 1012, + "min_sentence1_length": 37, + "average_sentence1_length": 125.70750988142292, + "max_sentence1_length": 335, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-gom": { + "num_samples": 1012, + "number_of_characters": 259182, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 37, + "average_sentence2_length": 125.70750988142292, + "max_sentence2_length": 335, + "unique_sentence2": 1012 + }, + "hne-eng": { + "num_samples": 1012, + "number_of_characters": 258911, + "unique_pairs": 1012, + "min_sentence1_length": 42, + "average_sentence1_length": 125.43972332015811, + "max_sentence1_length": 327, + "unique_sentence1": 1011, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-hne": { + "num_samples": 1012, + "number_of_characters": 258915, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 42, + "average_sentence2_length": 125.44367588932806, + "max_sentence2_length": 326, + "unique_sentence2": 1011 + }, + "raj-eng": { + "num_samples": 1012, + "number_of_characters": 261987, + "unique_pairs": 1012, + "min_sentence1_length": 38, + "average_sentence1_length": 128.47924901185772, + "max_sentence1_length": 338, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-raj": { + "num_samples": 1012, + "number_of_characters": 261987, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 38, + "average_sentence2_length": 128.47924901185772, + "max_sentence2_length": 338, + "unique_sentence2": 1012 + }, + "mai-eng": { + "num_samples": 1012, + "number_of_characters": 261374, + "unique_pairs": 1012, + "min_sentence1_length": 36, + "average_sentence1_length": 127.87351778656127, + "max_sentence1_length": 350, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-mai": { + "num_samples": 1012, + "number_of_characters": 261377, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 36, + "average_sentence2_length": 127.87648221343873, + "max_sentence2_length": 350, + "unique_sentence2": 1012 + }, + "mni-eng": { + "num_samples": 1012, + "number_of_characters": 268767, + "unique_pairs": 1012, + "min_sentence1_length": 38, + "average_sentence1_length": 135.17885375494072, + "max_sentence1_length": 353, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-mni": { + "num_samples": 1012, + "number_of_characters": 268768, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 38, + "average_sentence2_length": 135.1798418972332, + "max_sentence2_length": 354, + "unique_sentence2": 1012 + }, + "mup-eng": { + "num_samples": 1012, + "number_of_characters": 262034, + "unique_pairs": 1012, + "min_sentence1_length": 40, + "average_sentence1_length": 128.52569169960475, + "max_sentence1_length": 340, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-mup": { + "num_samples": 1012, + "number_of_characters": 262034, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 40, + "average_sentence2_length": 128.52569169960475, + "max_sentence2_length": 340, + "unique_sentence2": 1012 + }, + "mwr-eng": { + "num_samples": 1012, + "number_of_characters": 263749, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.22035573122528, + "max_sentence1_length": 345, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-mwr": { + "num_samples": 1012, + "number_of_characters": 263749, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.22035573122528, + "max_sentence2_length": 345, + "unique_sentence2": 1012 + }, + "sat-eng": { + "num_samples": 1012, + "number_of_characters": 271757, + "unique_pairs": 1012, + "min_sentence1_length": 43, + "average_sentence1_length": 138.13339920948616, + "max_sentence1_length": 366, + "unique_sentence1": 1012, + "min_sentence2_length": 35, + "average_sentence2_length": 130.401185770751, + "max_sentence2_length": 368, + "unique_sentence2": 1012 + }, + "eng-sat": { + "num_samples": 1012, + "number_of_characters": 271757, + "unique_pairs": 1012, + "min_sentence1_length": 35, + "average_sentence1_length": 130.401185770751, + "max_sentence1_length": 368, + "unique_sentence1": 1012, + "min_sentence2_length": 43, + "average_sentence2_length": 138.13339920948616, + "max_sentence2_length": 366, + "unique_sentence2": 1012 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/NTREXBitextMining.json b/mteb/descriptive_stats/BitextMining/NTREXBitextMining.json new file mode 100644 index 0000000000..3adf27b3df --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/NTREXBitextMining.json @@ -0,0 +1,24925 @@ +{ + "test": { + "num_samples": 3826252, + "number_of_characters": 988355274, + "unique_pairs": 3820263, + "min_sentence1_length": 1, + "average_sentence1_length": 129.15449296073547, + "max_sentence1_length": 773, + "unique_sentence1": 241259, + "min_sentence2_length": 1, + "average_sentence2_length": 129.15449296073547, + "max_sentence2_length": 773, + "unique_sentence2": 241259, + "hf_subset_descriptive_stats": { + "afr_Latn-dan_Latn": { + "num_samples": 1997, + "number_of_characters": 520490, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 134.37756634952427, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 126.25838758137206, + "max_sentence2_length": 522, + "unique_sentence2": 1995 + }, + "afr_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 564002, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 134.37756634952427, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "afr_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 516072, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 134.37756634952427, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "afr_Latn-fao_Latn": { + "num_samples": 1997, + "number_of_characters": 526155, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 134.37756634952427, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0951427140711, + "max_sentence2_length": 433, + "unique_sentence2": 1997 + }, + "afr_Latn-isl_Latn": { + "num_samples": 1997, + "number_of_characters": 530560, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 134.37756634952427, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 131.30095142714072, + "max_sentence2_length": 399, + "unique_sentence2": 1996 + }, + "afr_Latn-ltz_Latn": { + "num_samples": 1997, + "number_of_characters": 549109, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 134.37756634952427, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 140.58938407611416, + "max_sentence2_length": 543, + "unique_sentence2": 1996 + }, + "afr_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 560267, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 134.37756634952427, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "afr_Latn-nno_Latn": { + "num_samples": 1997, + "number_of_characters": 516709, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 134.37756634952427, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.36504757135704, + "max_sentence2_length": 417, + "unique_sentence2": 1996 + }, + "afr_Latn-nob_Latn": { + "num_samples": 1997, + "number_of_characters": 519796, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 134.37756634952427, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.91086629944917, + "max_sentence2_length": 482, + "unique_sentence2": 1996 + }, + "afr_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 520179, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 134.37756634952427, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "amh_Ethi-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 415227, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "amh_Ethi-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 437473, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "amh_Ethi-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 413608, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "amh_Ethi-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 459006, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "amh_Ethi-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 404938, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "amh_Ethi-som_Latn": { + "num_samples": 1997, + "number_of_characters": 458799, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "amh_Ethi-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 455649, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "amh_Ethi-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 440016, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "amh_Ethi-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 332745, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "amh_Ethi-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 501790, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "amh_Ethi-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 407310, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "amh_Ethi-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 435597, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "amh_Ethi-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 483595, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "amh_Ethi-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 425239, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 83.87931897846771, + "max_sentence1_length": 290, + "unique_sentence1": 1994, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "arb_Arab-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 474983, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "arb_Arab-ckb_Arab": { + "num_samples": 1997, + "number_of_characters": 483548, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 126.37305958938407, + "max_sentence2_length": 399, + "unique_sentence2": 1995 + }, + "arb_Arab-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 526831, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "arb_Arab-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 530308, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "arb_Arab-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 478901, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "arb_Arab-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 474520, + "unique_pairs": 1995, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "arb_Arab-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 500981, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "arb_Arab-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 524289, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "arb_Arab-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 431477, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "arb_Arab-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 492756, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "arb_Arab-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 509557, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "arb_Arab-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 518153, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "arb_Arab-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 342807, + "unique_pairs": 1995, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "arb_Arab-kmr_Latn": { + "num_samples": 1997, + "number_of_characters": 477127, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 123.15773660490736, + "max_sentence2_length": 420, + "unique_sentence2": 1996 + }, + "arb_Arab-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 364586, + "unique_pairs": 1995, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "arb_Arab-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 490578, + "unique_pairs": 1995, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "arb_Arab-mey_Arab": { + "num_samples": 1997, + "number_of_characters": 445016, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 107.07811717576365, + "max_sentence2_length": 392, + "unique_sentence2": 1993 + }, + "arb_Arab-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 523096, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "arb_Arab-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 509047, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "arb_Arab-por_Latn": { + "num_samples": 1997, + "number_of_characters": 508396, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "arb_Arab-prs_Arab": { + "num_samples": 1997, + "number_of_characters": 473717, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 121.45017526289435, + "max_sentence2_length": 365, + "unique_sentence2": 1997 + }, + "arb_Arab-pus_Arab": { + "num_samples": 1997, + "number_of_characters": 473814, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 121.49874812218327, + "max_sentence2_length": 366, + "unique_sentence2": 1996 + }, + "arb_Arab-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 506074, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "arb_Arab-shi_Arab": { + "num_samples": 1997, + "number_of_characters": 446094, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 3, + "average_sentence2_length": 107.6179268903355, + "max_sentence2_length": 378, + "unique_sentence2": 1996 + }, + "arb_Arab-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 519381, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "arb_Arab-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 503690, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "arb_Arab-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 483008, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "arb_Arab-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 541142, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "arb_Arab-tgk_Cyrl": { + "num_samples": 1997, + "number_of_characters": 505328, + "unique_pairs": 1995, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 3, + "average_sentence2_length": 137.27941912869304, + "max_sentence2_length": 451, + "unique_sentence2": 1995 + }, + "arb_Arab-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 496794, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "arb_Arab-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 502302, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "arb_Arab-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 322659, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "arb_Arab-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 488913, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 115.76414621932899, + "max_sentence1_length": 362, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "aze_Latn-bak_Cyrl": { + "num_samples": 1997, + "number_of_characters": 515960, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 135.0195292939409, + "max_sentence1_length": 398, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 123.34802203304957, + "max_sentence2_length": 437, + "unique_sentence2": 1995 + }, + "aze_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 517354, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 135.0195292939409, + "max_sentence1_length": 398, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "aze_Latn-kaz_Cyrl": { + "num_samples": 1997, + "number_of_characters": 529910, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 135.0195292939409, + "max_sentence1_length": 398, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 130.33350025037555, + "max_sentence2_length": 473, + "unique_sentence2": 1996 + }, + "aze_Latn-kir_Cyrl": { + "num_samples": 1997, + "number_of_characters": 520498, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 135.0195292939409, + "max_sentence1_length": 398, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 125.62043064596895, + "max_sentence2_length": 395, + "unique_sentence2": 1996 + }, + "aze_Latn-tat_Cyrl": { + "num_samples": 1997, + "number_of_characters": 515560, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 135.0195292939409, + "max_sentence1_length": 398, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 123.14772158237356, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "aze_Latn-tuk_Latn": { + "num_samples": 1997, + "number_of_characters": 554908, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 135.0195292939409, + "max_sentence1_length": 398, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 142.85127691537306, + "max_sentence2_length": 576, + "unique_sentence2": 1996 + }, + "aze_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 535247, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 135.0195292939409, + "max_sentence1_length": 398, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "aze_Latn-uig_Arab": { + "num_samples": 1997, + "number_of_characters": 580656, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 135.0195292939409, + "max_sentence1_length": 398, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 155.74461692538807, + "max_sentence2_length": 592, + "unique_sentence2": 1996 + }, + "aze_Latn-uzb_Latn": { + "num_samples": 1997, + "number_of_characters": 563329, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 135.0195292939409, + "max_sentence1_length": 398, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 147.06810215322986, + "max_sentence2_length": 470, + "unique_sentence2": 1996 + }, + "bak_Cyrl-aze_Latn": { + "num_samples": 1997, + "number_of_characters": 515960, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 123.34802203304957, + "max_sentence1_length": 437, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 135.0195292939409, + "max_sentence2_length": 398, + "unique_sentence2": 1997 + }, + "bak_Cyrl-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 494046, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 123.34802203304957, + "max_sentence1_length": 437, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "bak_Cyrl-kaz_Cyrl": { + "num_samples": 1997, + "number_of_characters": 506602, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 123.34802203304957, + "max_sentence1_length": 437, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 130.33350025037555, + "max_sentence2_length": 473, + "unique_sentence2": 1996 + }, + "bak_Cyrl-kir_Cyrl": { + "num_samples": 1997, + "number_of_characters": 497190, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 123.34802203304957, + "max_sentence1_length": 437, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 125.62043064596895, + "max_sentence2_length": 395, + "unique_sentence2": 1996 + }, + "bak_Cyrl-tat_Cyrl": { + "num_samples": 1997, + "number_of_characters": 492252, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 123.34802203304957, + "max_sentence1_length": 437, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 123.14772158237356, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "bak_Cyrl-tuk_Latn": { + "num_samples": 1997, + "number_of_characters": 531600, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 123.34802203304957, + "max_sentence1_length": 437, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 142.85127691537306, + "max_sentence2_length": 576, + "unique_sentence2": 1996 + }, + "bak_Cyrl-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 511939, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 123.34802203304957, + "max_sentence1_length": 437, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "bak_Cyrl-uig_Arab": { + "num_samples": 1997, + "number_of_characters": 557348, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 123.34802203304957, + "max_sentence1_length": 437, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 155.74461692538807, + "max_sentence2_length": 592, + "unique_sentence2": 1996 + }, + "bak_Cyrl-uzb_Latn": { + "num_samples": 1997, + "number_of_characters": 540021, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 123.34802203304957, + "max_sentence1_length": 437, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 147.06810215322986, + "max_sentence2_length": 470, + "unique_sentence2": 1996 + }, + "bel_Cyrl-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 511000, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "bel_Cyrl-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 525979, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "bel_Cyrl-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 497408, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "bel_Cyrl-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 503810, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "bel_Cyrl-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 512015, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "bel_Cyrl-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 523981, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "bel_Cyrl-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 533956, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "bel_Cyrl-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 530983, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "bel_Cyrl-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 509059, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "bel_Cyrl-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 508986, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "bel_Cyrl-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 508393, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "bel_Cyrl-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 512231, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "bel_Cyrl-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 518873, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 128.2373560340511, + "max_sentence1_length": 422, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "bem_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 546212, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 149.47020530796195, + "max_sentence1_length": 465, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "bem_Latn-ewe_Latn": { + "num_samples": 1997, + "number_of_characters": 537470, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 149.47020530796195, + "max_sentence1_length": 465, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 119.6685027541312, + "max_sentence2_length": 493, + "unique_sentence2": 1994 + }, + "bem_Latn-fuc_Latn": { + "num_samples": 1997, + "number_of_characters": 526972, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 149.47020530796195, + "max_sentence1_length": 465, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 114.4116174261392, + "max_sentence2_length": 376, + "unique_sentence2": 1996 + }, + "bem_Latn-kin_Latn": { + "num_samples": 1997, + "number_of_characters": 602279, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 149.47020530796195, + "max_sentence1_length": 465, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 152.12168252378567, + "max_sentence2_length": 541, + "unique_sentence2": 1996 + }, + "bem_Latn-nde_Latn": { + "num_samples": 1997, + "number_of_characters": 596231, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 149.47020530796195, + "max_sentence1_length": 465, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 149.09313970956435, + "max_sentence2_length": 590, + "unique_sentence2": 1997 + }, + "bem_Latn-nya_Latn": { + "num_samples": 1997, + "number_of_characters": 582774, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 149.47020530796195, + "max_sentence1_length": 465, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 142.35453179769655, + "max_sentence2_length": 464, + "unique_sentence2": 1993 + }, + "bem_Latn-sna_Latn": { + "num_samples": 1997, + "number_of_characters": 596822, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 149.47020530796195, + "max_sentence1_length": 465, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 149.38908362543816, + "max_sentence2_length": 511, + "unique_sentence2": 1995 + }, + "bem_Latn-ven_Latn": { + "num_samples": 1997, + "number_of_characters": 598248, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 149.47020530796195, + "max_sentence1_length": 465, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 150.10315473209815, + "max_sentence2_length": 535, + "unique_sentence2": 1993 + }, + "ben_Beng-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 474983, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "ben_Beng-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 539452, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "ben_Beng-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 547650, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "ben_Beng-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 542929, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "ben_Beng-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 491522, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ben_Beng-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 519005, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "ben_Beng-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 487141, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "ben_Beng-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 513602, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "ben_Beng-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 536910, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "ben_Beng-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 488733, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "ben_Beng-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 444098, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "ben_Beng-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 505377, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "ben_Beng-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 522178, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "ben_Beng-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 530774, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "ben_Beng-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 355428, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "ben_Beng-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 509338, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "ben_Beng-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 377207, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "ben_Beng-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 503199, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "ben_Beng-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 504689, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "ben_Beng-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 492025, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "ben_Beng-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 535717, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "ben_Beng-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 494224, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "ben_Beng-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 521668, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "ben_Beng-por_Latn": { + "num_samples": 1997, + "number_of_characters": 521017, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "ben_Beng-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 518695, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "ben_Beng-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 502543, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "ben_Beng-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 464129, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "ben_Beng-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 532002, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "ben_Beng-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 516311, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "ben_Beng-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 495629, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "ben_Beng-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 553763, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "ben_Beng-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 491329, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "ben_Beng-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 509415, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "ben_Beng-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 491800, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "ben_Beng-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 514923, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "ben_Beng-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 335280, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "ben_Beng-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 501534, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 122.08412618928392, + "max_sentence1_length": 402, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "bod_Tibt-dzo_Tibt": { + "num_samples": 1997, + "number_of_characters": 543850, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 150.54031046569855, + "max_sentence1_length": 478, + "unique_sentence1": 1993, + "min_sentence2_length": 5, + "average_sentence2_length": 121.79318978467701, + "max_sentence2_length": 411, + "unique_sentence2": 1992 + }, + "bod_Tibt-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 548349, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 150.54031046569855, + "max_sentence1_length": 478, + "unique_sentence1": 1993, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "bod_Tibt-khm_Khmr": { + "num_samples": 1997, + "number_of_characters": 589120, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 150.54031046569855, + "max_sentence1_length": 478, + "unique_sentence1": 1993, + "min_sentence2_length": 6, + "average_sentence2_length": 144.4621932899349, + "max_sentence2_length": 517, + "unique_sentence2": 1996 + }, + "bod_Tibt-lao_Laoo": { + "num_samples": 1997, + "number_of_characters": 567609, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 150.54031046569855, + "max_sentence1_length": 478, + "unique_sentence1": 1993, + "min_sentence2_length": 7, + "average_sentence2_length": 133.69053580370556, + "max_sentence2_length": 507, + "unique_sentence2": 1997 + }, + "bod_Tibt-mon_Mong": { + "num_samples": 1997, + "number_of_characters": 559677, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 150.54031046569855, + "max_sentence1_length": 478, + "unique_sentence1": 1993, + "min_sentence2_length": 11, + "average_sentence2_length": 129.7185778668002, + "max_sentence2_length": 414, + "unique_sentence2": 1997 + }, + "bod_Tibt-mya_Mymr": { + "num_samples": 1997, + "number_of_characters": 612483, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 150.54031046569855, + "max_sentence1_length": 478, + "unique_sentence1": 1993, + "min_sentence2_length": 9, + "average_sentence2_length": 156.16124186279418, + "max_sentence2_length": 773, + "unique_sentence2": 1997 + }, + "bod_Tibt-tha_Thai": { + "num_samples": 1997, + "number_of_characters": 538097, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 150.54031046569855, + "max_sentence1_length": 478, + "unique_sentence1": 1993, + "min_sentence2_length": 7, + "average_sentence2_length": 118.91236855282925, + "max_sentence2_length": 439, + "unique_sentence2": 1996 + }, + "bos_Latn-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 511000, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "bos_Latn-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 524799, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "bos_Latn-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 496228, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "bos_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 502630, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "bos_Latn-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 510835, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "bos_Latn-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 522801, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "bos_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 532776, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "bos_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 529803, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "bos_Latn-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 507879, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "bos_Latn-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 507806, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "bos_Latn-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 507213, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "bos_Latn-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 511051, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "bos_Latn-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 517693, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 127.64646970455684, + "max_sentence1_length": 434, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "bul_Cyrl-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 525979, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "bul_Cyrl-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 524799, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "bul_Cyrl-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 511207, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "bul_Cyrl-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 517609, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "bul_Cyrl-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 525814, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "bul_Cyrl-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 537780, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "bul_Cyrl-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 547755, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "bul_Cyrl-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 544782, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "bul_Cyrl-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 522858, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "bul_Cyrl-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 522785, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "bul_Cyrl-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 522192, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "bul_Cyrl-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 526030, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "bul_Cyrl-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 532672, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 135.14722083124687, + "max_sentence1_length": 493, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "cat_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 530680, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 141.6925388082123, + "max_sentence1_length": 460, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "cat_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 576068, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 141.6925388082123, + "max_sentence1_length": 460, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "cat_Latn-glg_Latn": { + "num_samples": 1997, + "number_of_characters": 554946, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 141.6925388082123, + "max_sentence1_length": 460, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 136.19729594391586, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "cat_Latn-ita_Latn": { + "num_samples": 1997, + "number_of_characters": 572177, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 141.6925388082123, + "max_sentence1_length": 460, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 144.82573860791186, + "max_sentence2_length": 623, + "unique_sentence2": 1996 + }, + "cat_Latn-mlt_Latn": { + "num_samples": 1997, + "number_of_characters": 560435, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 141.6925388082123, + "max_sentence1_length": 460, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 138.94591887831749, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "cat_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 560175, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 141.6925388082123, + "max_sentence1_length": 460, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "cat_Latn-ron_Latn": { + "num_samples": 1997, + "number_of_characters": 575445, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 141.6925388082123, + "max_sentence1_length": 460, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 146.4621932899349, + "max_sentence2_length": 518, + "unique_sentence2": 1997 + }, + "cat_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 571160, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 141.6925388082123, + "max_sentence1_length": 460, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "ces_Latn-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 497408, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "ces_Latn-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 496228, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "ces_Latn-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 511207, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "ces_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 489038, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ces_Latn-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 497243, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "ces_Latn-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 509209, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "ces_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 519184, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "ces_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 516211, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "ces_Latn-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 494287, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "ces_Latn-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 494214, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "ces_Latn-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 493621, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "ces_Latn-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 497459, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "ces_Latn-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 504101, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 120.84026039058588, + "max_sentence1_length": 474, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "ckb_Arab-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 483548, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 126.37305958938407, + "max_sentence1_length": 399, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "ckb_Arab-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 500087, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 126.37305958938407, + "max_sentence1_length": 399, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ckb_Arab-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 495706, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 126.37305958938407, + "max_sentence1_length": 399, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "ckb_Arab-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 452663, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 126.37305958938407, + "max_sentence1_length": 399, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "ckb_Arab-kmr_Latn": { + "num_samples": 1997, + "number_of_characters": 498313, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 126.37305958938407, + "max_sentence1_length": 399, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 123.15773660490736, + "max_sentence2_length": 420, + "unique_sentence2": 1996 + }, + "ckb_Arab-mey_Arab": { + "num_samples": 1997, + "number_of_characters": 466202, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 126.37305958938407, + "max_sentence1_length": 399, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 107.07811717576365, + "max_sentence2_length": 392, + "unique_sentence2": 1993 + }, + "ckb_Arab-prs_Arab": { + "num_samples": 1997, + "number_of_characters": 494903, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 126.37305958938407, + "max_sentence1_length": 399, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 121.45017526289435, + "max_sentence2_length": 365, + "unique_sentence2": 1997 + }, + "ckb_Arab-pus_Arab": { + "num_samples": 1997, + "number_of_characters": 495000, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 126.37305958938407, + "max_sentence1_length": 399, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 121.49874812218327, + "max_sentence2_length": 366, + "unique_sentence2": 1996 + }, + "ckb_Arab-shi_Arab": { + "num_samples": 1997, + "number_of_characters": 467280, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 126.37305958938407, + "max_sentence1_length": 399, + "unique_sentence1": 1995, + "min_sentence2_length": 3, + "average_sentence2_length": 107.6179268903355, + "max_sentence2_length": 378, + "unique_sentence2": 1996 + }, + "ckb_Arab-tgk_Cyrl": { + "num_samples": 1997, + "number_of_characters": 526514, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 126.37305958938407, + "max_sentence1_length": 399, + "unique_sentence1": 1995, + "min_sentence2_length": 3, + "average_sentence2_length": 137.27941912869304, + "max_sentence2_length": 451, + "unique_sentence2": 1995 + }, + "cym_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 514225, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.4526790185278, + "max_sentence1_length": 444, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "cym_Latn-gle_Latn": { + "num_samples": 1997, + "number_of_characters": 561314, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.4526790185278, + "max_sentence1_length": 444, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 147.62593890836254, + "max_sentence2_length": 461, + "unique_sentence2": 1997 + }, + "dan_Latn-afr_Latn": { + "num_samples": 1997, + "number_of_characters": 520490, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 126.25838758137206, + "max_sentence1_length": 522, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 134.37756634952427, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "dan_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 547788, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 126.25838758137206, + "max_sentence1_length": 522, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "dan_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 499858, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 126.25838758137206, + "max_sentence1_length": 522, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "dan_Latn-fao_Latn": { + "num_samples": 1997, + "number_of_characters": 509941, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 126.25838758137206, + "max_sentence1_length": 522, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0951427140711, + "max_sentence2_length": 433, + "unique_sentence2": 1997 + }, + "dan_Latn-isl_Latn": { + "num_samples": 1997, + "number_of_characters": 514346, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 126.25838758137206, + "max_sentence1_length": 522, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 131.30095142714072, + "max_sentence2_length": 399, + "unique_sentence2": 1996 + }, + "dan_Latn-ltz_Latn": { + "num_samples": 1997, + "number_of_characters": 532895, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 126.25838758137206, + "max_sentence1_length": 522, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 140.58938407611416, + "max_sentence2_length": 543, + "unique_sentence2": 1996 + }, + "dan_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 544053, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 126.25838758137206, + "max_sentence1_length": 522, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "dan_Latn-nno_Latn": { + "num_samples": 1997, + "number_of_characters": 500495, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 126.25838758137206, + "max_sentence1_length": 522, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.36504757135704, + "max_sentence2_length": 417, + "unique_sentence2": 1996 + }, + "dan_Latn-nob_Latn": { + "num_samples": 1997, + "number_of_characters": 503582, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 126.25838758137206, + "max_sentence1_length": 522, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 125.91086629944917, + "max_sentence2_length": 482, + "unique_sentence2": 1996 + }, + "dan_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 503965, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 126.25838758137206, + "max_sentence1_length": 522, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "deu_Latn-afr_Latn": { + "num_samples": 1997, + "number_of_characters": 564002, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 134.37756634952427, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "deu_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 526831, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "deu_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 539452, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "deu_Latn-dan_Latn": { + "num_samples": 1997, + "number_of_characters": 547788, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 126.25838758137206, + "max_sentence2_length": 522, + "unique_sentence2": 1995 + }, + "deu_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 594777, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "deu_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 543370, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "deu_Latn-fao_Latn": { + "num_samples": 1997, + "number_of_characters": 553453, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0951427140711, + "max_sentence2_length": 433, + "unique_sentence2": 1997 + }, + "deu_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 538989, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "deu_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 565450, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "deu_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 588758, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "deu_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 495946, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "deu_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 557225, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "deu_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 574026, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "deu_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 582622, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "deu_Latn-isl_Latn": { + "num_samples": 1997, + "number_of_characters": 557858, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 131.30095142714072, + "max_sentence2_length": 399, + "unique_sentence2": 1996 + }, + "deu_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 407276, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "deu_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 429055, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "deu_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 555047, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "deu_Latn-ltz_Latn": { + "num_samples": 1997, + "number_of_characters": 576407, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 140.58938407611416, + "max_sentence2_length": 543, + "unique_sentence2": 1996 + }, + "deu_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 587565, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "deu_Latn-nno_Latn": { + "num_samples": 1997, + "number_of_characters": 544007, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.36504757135704, + "max_sentence2_length": 417, + "unique_sentence2": 1996 + }, + "deu_Latn-nob_Latn": { + "num_samples": 1997, + "number_of_characters": 547094, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.91086629944917, + "max_sentence2_length": 482, + "unique_sentence2": 1996 + }, + "deu_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 573516, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "deu_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 572865, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "deu_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 570543, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "deu_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 583850, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "deu_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 568159, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "deu_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 547477, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "deu_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 605611, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "deu_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 561263, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "deu_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 566771, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "deu_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 387128, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "deu_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 553382, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 148.04707060590886, + "max_sentence1_length": 508, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "div_Thaa-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 547650, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "div_Thaa-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 551568, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "div_Thaa-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 579051, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "div_Thaa-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 548779, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "div_Thaa-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 565423, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "div_Thaa-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 569384, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "div_Thaa-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 564735, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "div_Thaa-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 552071, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "div_Thaa-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 554270, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "div_Thaa-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 562589, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "div_Thaa-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 524175, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "div_Thaa-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 613809, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "div_Thaa-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 551375, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "div_Thaa-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 551846, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 152.15222834251378, + "max_sentence1_length": 609, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "dzo_Tibt-bod_Tibt": { + "num_samples": 1997, + "number_of_characters": 543850, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 121.79318978467701, + "max_sentence1_length": 411, + "unique_sentence1": 1992, + "min_sentence2_length": 8, + "average_sentence2_length": 150.54031046569855, + "max_sentence2_length": 478, + "unique_sentence2": 1993 + }, + "dzo_Tibt-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 490941, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 121.79318978467701, + "max_sentence1_length": 411, + "unique_sentence1": 1992, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "dzo_Tibt-khm_Khmr": { + "num_samples": 1997, + "number_of_characters": 531712, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 121.79318978467701, + "max_sentence1_length": 411, + "unique_sentence1": 1992, + "min_sentence2_length": 6, + "average_sentence2_length": 144.4621932899349, + "max_sentence2_length": 517, + "unique_sentence2": 1996 + }, + "dzo_Tibt-lao_Laoo": { + "num_samples": 1997, + "number_of_characters": 510201, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 121.79318978467701, + "max_sentence1_length": 411, + "unique_sentence1": 1992, + "min_sentence2_length": 7, + "average_sentence2_length": 133.69053580370556, + "max_sentence2_length": 507, + "unique_sentence2": 1997 + }, + "dzo_Tibt-mon_Mong": { + "num_samples": 1997, + "number_of_characters": 502269, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 121.79318978467701, + "max_sentence1_length": 411, + "unique_sentence1": 1992, + "min_sentence2_length": 11, + "average_sentence2_length": 129.7185778668002, + "max_sentence2_length": 414, + "unique_sentence2": 1997 + }, + "dzo_Tibt-mya_Mymr": { + "num_samples": 1997, + "number_of_characters": 555075, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 121.79318978467701, + "max_sentence1_length": 411, + "unique_sentence1": 1992, + "min_sentence2_length": 9, + "average_sentence2_length": 156.16124186279418, + "max_sentence2_length": 773, + "unique_sentence2": 1997 + }, + "dzo_Tibt-tha_Thai": { + "num_samples": 1997, + "number_of_characters": 480689, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 121.79318978467701, + "max_sentence1_length": 411, + "unique_sentence1": 1992, + "min_sentence2_length": 7, + "average_sentence2_length": 118.91236855282925, + "max_sentence2_length": 439, + "unique_sentence2": 1996 + }, + "ell_Grek-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 530308, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "ell_Grek-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 542929, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "ell_Grek-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 594777, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "ell_Grek-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 546847, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ell_Grek-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 542466, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "ell_Grek-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 568927, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "ell_Grek-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 592235, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "ell_Grek-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 499423, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "ell_Grek-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 560702, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "ell_Grek-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 577503, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "ell_Grek-hye_Armn": { + "num_samples": 1997, + "number_of_characters": 563842, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 132.55633450175262, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "ell_Grek-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 586099, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "ell_Grek-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 410753, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "ell_Grek-kat_Geor": { + "num_samples": 1997, + "number_of_characters": 565719, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 133.49624436654983, + "max_sentence2_length": 503, + "unique_sentence2": 1995 + }, + "ell_Grek-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 432532, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "ell_Grek-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 558524, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "ell_Grek-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 591042, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "ell_Grek-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 576993, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "ell_Grek-por_Latn": { + "num_samples": 1997, + "number_of_characters": 576342, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "ell_Grek-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 574020, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "ell_Grek-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 587327, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "ell_Grek-sqi_Latn": { + "num_samples": 1997, + "number_of_characters": 582734, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 142.01652478718077, + "max_sentence2_length": 461, + "unique_sentence2": 1996 + }, + "ell_Grek-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 571636, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "ell_Grek-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 550954, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "ell_Grek-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 609088, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "ell_Grek-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 564740, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "ell_Grek-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 570248, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "ell_Grek-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 390605, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "ell_Grek-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 556859, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 149.78818227341011, + "max_sentence1_length": 584, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "eng_Latn-afr_Latn": { + "num_samples": 1997, + "number_of_characters": 516072, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 134.37756634952427, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "eng_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 415227, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "eng_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 478901, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "eng_Latn-aze_Latn": { + "num_samples": 1997, + "number_of_characters": 517354, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 135.0195292939409, + "max_sentence2_length": 398, + "unique_sentence2": 1997 + }, + "eng_Latn-bak_Cyrl": { + "num_samples": 1997, + "number_of_characters": 494046, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 123.34802203304957, + "max_sentence2_length": 437, + "unique_sentence2": 1995 + }, + "eng_Latn-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 503810, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "eng_Latn-bem_Latn": { + "num_samples": 1997, + "number_of_characters": 546212, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 149.47020530796195, + "max_sentence2_length": 465, + "unique_sentence2": 1997 + }, + "eng_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 491522, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "eng_Latn-bod_Tibt": { + "num_samples": 1997, + "number_of_characters": 548349, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 150.54031046569855, + "max_sentence2_length": 478, + "unique_sentence2": 1993 + }, + "eng_Latn-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 502630, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "eng_Latn-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 517609, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "eng_Latn-cat_Latn": { + "num_samples": 1997, + "number_of_characters": 530680, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 141.6925388082123, + "max_sentence2_length": 460, + "unique_sentence2": 1997 + }, + "eng_Latn-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 489038, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "eng_Latn-ckb_Arab": { + "num_samples": 1997, + "number_of_characters": 500087, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 126.37305958938407, + "max_sentence2_length": 399, + "unique_sentence2": 1995 + }, + "eng_Latn-cym_Latn": { + "num_samples": 1997, + "number_of_characters": 514225, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 133.4526790185278, + "max_sentence2_length": 444, + "unique_sentence2": 1997 + }, + "eng_Latn-dan_Latn": { + "num_samples": 1997, + "number_of_characters": 499858, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 126.25838758137206, + "max_sentence2_length": 522, + "unique_sentence2": 1995 + }, + "eng_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 543370, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "eng_Latn-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 551568, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "eng_Latn-dzo_Tibt": { + "num_samples": 1997, + "number_of_characters": 490941, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 121.79318978467701, + "max_sentence2_length": 411, + "unique_sentence2": 1992 + }, + "eng_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 546847, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "eng_Latn-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 522923, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "eng_Latn-ewe_Latn": { + "num_samples": 1997, + "number_of_characters": 486698, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 119.6685027541312, + "max_sentence2_length": 493, + "unique_sentence2": 1994 + }, + "eng_Latn-fao_Latn": { + "num_samples": 1997, + "number_of_characters": 505523, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0951427140711, + "max_sentence2_length": 433, + "unique_sentence2": 1997 + }, + "eng_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 491059, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "eng_Latn-fij_Latn": { + "num_samples": 1997, + "number_of_characters": 548225, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 150.478217325989, + "max_sentence2_length": 448, + "unique_sentence2": 1988 + }, + "eng_Latn-fil_Latn": { + "num_samples": 1997, + "number_of_characters": 541140, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 146.93039559339007, + "max_sentence2_length": 554, + "unique_sentence2": 1997 + }, + "eng_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 517520, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "eng_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 540828, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "eng_Latn-fuc_Latn": { + "num_samples": 1997, + "number_of_characters": 476200, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 114.4116174261392, + "max_sentence2_length": 376, + "unique_sentence2": 1996 + }, + "eng_Latn-gle_Latn": { + "num_samples": 1997, + "number_of_characters": 542529, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 147.62593890836254, + "max_sentence2_length": 461, + "unique_sentence2": 1997 + }, + "eng_Latn-glg_Latn": { + "num_samples": 1997, + "number_of_characters": 519706, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 136.19729594391586, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "eng_Latn-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 492651, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "eng_Latn-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 517686, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "eng_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 448016, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "eng_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 509295, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "eng_Latn-hmn_Latn": { + "num_samples": 1997, + "number_of_characters": 578510, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 165.6434651977967, + "max_sentence2_length": 643, + "unique_sentence2": 1997 + }, + "eng_Latn-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 503645, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "eng_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 526096, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "eng_Latn-hye_Armn": { + "num_samples": 1997, + "number_of_characters": 512435, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 132.55633450175262, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "eng_Latn-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 493821, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "eng_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 534692, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "eng_Latn-isl_Latn": { + "num_samples": 1997, + "number_of_characters": 509928, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 131.30095142714072, + "max_sentence2_length": 399, + "unique_sentence2": 1996 + }, + "eng_Latn-ita_Latn": { + "num_samples": 1997, + "number_of_characters": 536937, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 144.82573860791186, + "max_sentence2_length": 623, + "unique_sentence2": 1996 + }, + "eng_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 359346, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "eng_Latn-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 513256, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "eng_Latn-kat_Geor": { + "num_samples": 1997, + "number_of_characters": 514312, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 133.49624436654983, + "max_sentence2_length": 503, + "unique_sentence2": 1995 + }, + "eng_Latn-kaz_Cyrl": { + "num_samples": 1997, + "number_of_characters": 507996, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 130.33350025037555, + "max_sentence2_length": 473, + "unique_sentence2": 1996 + }, + "eng_Latn-khm_Khmr": { + "num_samples": 1997, + "number_of_characters": 536211, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 144.4621932899349, + "max_sentence2_length": 517, + "unique_sentence2": 1996 + }, + "eng_Latn-kin_Latn": { + "num_samples": 1997, + "number_of_characters": 551507, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 152.12168252378567, + "max_sentence2_length": 541, + "unique_sentence2": 1996 + }, + "eng_Latn-kir_Cyrl": { + "num_samples": 1997, + "number_of_characters": 498584, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 125.62043064596895, + "max_sentence2_length": 395, + "unique_sentence2": 1996 + }, + "eng_Latn-kmr_Latn": { + "num_samples": 1997, + "number_of_characters": 493666, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 123.15773660490736, + "max_sentence2_length": 420, + "unique_sentence2": 1996 + }, + "eng_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 381125, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "eng_Latn-lao_Laoo": { + "num_samples": 1997, + "number_of_characters": 514700, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 133.69053580370556, + "max_sentence2_length": 507, + "unique_sentence2": 1997 + }, + "eng_Latn-lav_Latn": { + "num_samples": 1997, + "number_of_characters": 515908, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 134.29544316474713, + "max_sentence2_length": 503, + "unique_sentence2": 1994 + }, + "eng_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 507117, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "eng_Latn-ltz_Latn": { + "num_samples": 1997, + "number_of_characters": 528477, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 140.58938407611416, + "max_sentence2_length": 543, + "unique_sentence2": 1996 + }, + "eng_Latn-mal_Mlym": { + "num_samples": 1997, + "number_of_characters": 551872, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 152.30445668502753, + "max_sentence2_length": 540, + "unique_sentence2": 1996 + }, + "eng_Latn-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 508607, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "eng_Latn-mey_Arab": { + "num_samples": 1997, + "number_of_characters": 461555, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 107.07811717576365, + "max_sentence2_length": 392, + "unique_sentence2": 1993 + }, + "eng_Latn-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 515611, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "eng_Latn-mlg_Latn": { + "num_samples": 1997, + "number_of_characters": 568028, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 160.39459188783175, + "max_sentence2_length": 559, + "unique_sentence2": 1994 + }, + "eng_Latn-mlt_Latn": { + "num_samples": 1997, + "number_of_characters": 525195, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 138.94591887831749, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "eng_Latn-mon_Mong": { + "num_samples": 1997, + "number_of_characters": 506768, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 129.7185778668002, + "max_sentence2_length": 414, + "unique_sentence2": 1997 + }, + "eng_Latn-mri_Latn": { + "num_samples": 1997, + "number_of_characters": 521844, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 137.26790185277918, + "max_sentence2_length": 443, + "unique_sentence2": 1997 + }, + "eng_Latn-msa_Latn": { + "num_samples": 1997, + "number_of_characters": 524903, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 138.79969954932398, + "max_sentence2_length": 463, + "unique_sentence2": 1997 + }, + "eng_Latn-mya_Mymr": { + "num_samples": 1997, + "number_of_characters": 559574, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 156.16124186279418, + "max_sentence2_length": 773, + "unique_sentence2": 1997 + }, + "eng_Latn-nde_Latn": { + "num_samples": 1997, + "number_of_characters": 545459, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 149.09313970956435, + "max_sentence2_length": 590, + "unique_sentence2": 1997 + }, + "eng_Latn-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 495943, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "eng_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 539635, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "eng_Latn-nno_Latn": { + "num_samples": 1997, + "number_of_characters": 496077, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.36504757135704, + "max_sentence2_length": 417, + "unique_sentence2": 1996 + }, + "eng_Latn-nob_Latn": { + "num_samples": 1997, + "number_of_characters": 499164, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 125.91086629944917, + "max_sentence2_length": 482, + "unique_sentence2": 1996 + }, + "eng_Latn-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 539219, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "eng_Latn-nya_Latn": { + "num_samples": 1997, + "number_of_characters": 532002, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 142.35453179769655, + "max_sentence2_length": 464, + "unique_sentence2": 1993 + }, + "eng_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 485151, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "eng_Latn-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 498142, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "eng_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 525586, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "eng_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 524935, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "eng_Latn-prs_Arab": { + "num_samples": 1997, + "number_of_characters": 490256, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 121.45017526289435, + "max_sentence2_length": 365, + "unique_sentence2": 1997 + }, + "eng_Latn-pus_Arab": { + "num_samples": 1997, + "number_of_characters": 490353, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 121.49874812218327, + "max_sentence2_length": 366, + "unique_sentence2": 1996 + }, + "eng_Latn-ron_Latn": { + "num_samples": 1997, + "number_of_characters": 540205, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 146.4621932899349, + "max_sentence2_length": 518, + "unique_sentence2": 1997 + }, + "eng_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 522613, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "eng_Latn-shi_Arab": { + "num_samples": 1997, + "number_of_characters": 462633, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 107.6179268903355, + "max_sentence2_length": 378, + "unique_sentence2": 1996 + }, + "eng_Latn-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 506461, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "eng_Latn-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 500689, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "eng_Latn-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 500616, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "eng_Latn-smo_Latn": { + "num_samples": 1997, + "number_of_characters": 525575, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 139.1362043064597, + "max_sentence2_length": 431, + "unique_sentence2": 1996 + }, + "eng_Latn-sna_Latn": { + "num_samples": 1997, + "number_of_characters": 546050, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 149.38908362543816, + "max_sentence2_length": 511, + "unique_sentence2": 1995 + }, + "eng_Latn-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 468047, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "eng_Latn-som_Latn": { + "num_samples": 1997, + "number_of_characters": 539012, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "eng_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 535920, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "eng_Latn-sqi_Latn": { + "num_samples": 1997, + "number_of_characters": 531327, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 142.01652478718077, + "max_sentence2_length": 461, + "unique_sentence2": 1996 + }, + "eng_Latn-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 500023, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "eng_Latn-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 503861, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "eng_Latn-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 535862, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "eng_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 520229, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "eng_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 499547, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "eng_Latn-tah_Latn": { + "num_samples": 1997, + "number_of_characters": 557343, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 155.04406609914872, + "max_sentence2_length": 524, + "unique_sentence2": 1997 + }, + "eng_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 557681, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "eng_Latn-tat_Cyrl": { + "num_samples": 1997, + "number_of_characters": 493646, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 123.14772158237356, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "eng_Latn-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 495247, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "eng_Latn-tgk_Cyrl": { + "num_samples": 1997, + "number_of_characters": 521867, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 137.27941912869304, + "max_sentence2_length": 451, + "unique_sentence2": 1995 + }, + "eng_Latn-tha_Thai": { + "num_samples": 1997, + "number_of_characters": 485188, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 118.91236855282925, + "max_sentence2_length": 439, + "unique_sentence2": 1996 + }, + "eng_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 412958, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "eng_Latn-ton_Latn": { + "num_samples": 1997, + "number_of_characters": 561360, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 157.05558337506258, + "max_sentence2_length": 468, + "unique_sentence2": 1997 + }, + "eng_Latn-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 582003, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "eng_Latn-tuk_Latn": { + "num_samples": 1997, + "number_of_characters": 532994, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 142.85127691537306, + "max_sentence2_length": 576, + "unique_sentence2": 1996 + }, + "eng_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 513333, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "eng_Latn-uig_Arab": { + "num_samples": 1997, + "number_of_characters": 558742, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 155.74461692538807, + "max_sentence2_length": 592, + "unique_sentence2": 1996 + }, + "eng_Latn-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 510503, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "eng_Latn-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 495718, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "eng_Latn-uzb_Latn": { + "num_samples": 1997, + "number_of_characters": 541415, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 147.06810215322986, + "max_sentence2_length": 470, + "unique_sentence2": 1996 + }, + "eng_Latn-ven_Latn": { + "num_samples": 1997, + "number_of_characters": 547476, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 150.10315473209815, + "max_sentence2_length": 535, + "unique_sentence2": 1993 + }, + "eng_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 518841, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "eng_Latn-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 487523, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "eng_Latn-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 515810, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "eng_Latn-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 563808, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "eng_Latn-yue_Hant": { + "num_samples": 1997, + "number_of_characters": 326607, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 4, + "average_sentence2_length": 39.502754131196795, + "max_sentence2_length": 133, + "unique_sentence2": 1996 + }, + "eng_Latn-zho_Hans": { + "num_samples": 1997, + "number_of_characters": 332681, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 4, + "average_sentence2_length": 42.54431647471207, + "max_sentence2_length": 263, + "unique_sentence2": 1997 + }, + "eng_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 339198, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "eng_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 505452, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.04606910365548, + "max_sentence1_length": 437, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "eus_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 519005, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "eus_Latn-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 579051, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "eus_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 522923, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "eus_Latn-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 520134, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "eus_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 536778, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "eus_Latn-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 540739, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "eus_Latn-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 536090, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "eus_Latn-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 523426, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "eus_Latn-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 525625, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "eus_Latn-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 533944, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "eus_Latn-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 495530, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "eus_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 585164, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "eus_Latn-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 522730, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "eus_Latn-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 523201, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 137.80821231847773, + "max_sentence1_length": 393, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "ewe_Latn-bem_Latn": { + "num_samples": 1997, + "number_of_characters": 537470, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 119.6685027541312, + "max_sentence1_length": 493, + "unique_sentence1": 1994, + "min_sentence2_length": 8, + "average_sentence2_length": 149.47020530796195, + "max_sentence2_length": 465, + "unique_sentence2": 1997 + }, + "ewe_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 486698, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 119.6685027541312, + "max_sentence1_length": 493, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ewe_Latn-fuc_Latn": { + "num_samples": 1997, + "number_of_characters": 467458, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 119.6685027541312, + "max_sentence1_length": 493, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 114.4116174261392, + "max_sentence2_length": 376, + "unique_sentence2": 1996 + }, + "ewe_Latn-kin_Latn": { + "num_samples": 1997, + "number_of_characters": 542765, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 119.6685027541312, + "max_sentence1_length": 493, + "unique_sentence1": 1994, + "min_sentence2_length": 10, + "average_sentence2_length": 152.12168252378567, + "max_sentence2_length": 541, + "unique_sentence2": 1996 + }, + "ewe_Latn-nde_Latn": { + "num_samples": 1997, + "number_of_characters": 536717, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 119.6685027541312, + "max_sentence1_length": 493, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 149.09313970956435, + "max_sentence2_length": 590, + "unique_sentence2": 1997 + }, + "ewe_Latn-nya_Latn": { + "num_samples": 1997, + "number_of_characters": 523260, + "unique_pairs": 1995, + "min_sentence1_length": 6, + "average_sentence1_length": 119.6685027541312, + "max_sentence1_length": 493, + "unique_sentence1": 1994, + "min_sentence2_length": 10, + "average_sentence2_length": 142.35453179769655, + "max_sentence2_length": 464, + "unique_sentence2": 1993 + }, + "ewe_Latn-sna_Latn": { + "num_samples": 1997, + "number_of_characters": 537308, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 119.6685027541312, + "max_sentence1_length": 493, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 149.38908362543816, + "max_sentence2_length": 511, + "unique_sentence2": 1995 + }, + "ewe_Latn-ven_Latn": { + "num_samples": 1997, + "number_of_characters": 538734, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 119.6685027541312, + "max_sentence1_length": 493, + "unique_sentence1": 1994, + "min_sentence2_length": 10, + "average_sentence2_length": 150.10315473209815, + "max_sentence2_length": 535, + "unique_sentence2": 1993 + }, + "fao_Latn-afr_Latn": { + "num_samples": 1997, + "number_of_characters": 526155, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0951427140711, + "max_sentence1_length": 433, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 134.37756634952427, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "fao_Latn-dan_Latn": { + "num_samples": 1997, + "number_of_characters": 509941, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0951427140711, + "max_sentence1_length": 433, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 126.25838758137206, + "max_sentence2_length": 522, + "unique_sentence2": 1995 + }, + "fao_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 553453, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0951427140711, + "max_sentence1_length": 433, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "fao_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 505523, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0951427140711, + "max_sentence1_length": 433, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "fao_Latn-isl_Latn": { + "num_samples": 1997, + "number_of_characters": 520011, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0951427140711, + "max_sentence1_length": 433, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 131.30095142714072, + "max_sentence2_length": 399, + "unique_sentence2": 1996 + }, + "fao_Latn-ltz_Latn": { + "num_samples": 1997, + "number_of_characters": 538560, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0951427140711, + "max_sentence1_length": 433, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 140.58938407611416, + "max_sentence2_length": 543, + "unique_sentence2": 1996 + }, + "fao_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 549718, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0951427140711, + "max_sentence1_length": 433, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "fao_Latn-nno_Latn": { + "num_samples": 1997, + "number_of_characters": 506160, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0951427140711, + "max_sentence1_length": 433, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.36504757135704, + "max_sentence2_length": 417, + "unique_sentence2": 1996 + }, + "fao_Latn-nob_Latn": { + "num_samples": 1997, + "number_of_characters": 509247, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0951427140711, + "max_sentence1_length": 433, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 125.91086629944917, + "max_sentence2_length": 482, + "unique_sentence2": 1996 + }, + "fao_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 509630, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0951427140711, + "max_sentence1_length": 433, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "fas_Arab-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 474520, + "unique_pairs": 1995, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "fas_Arab-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 487141, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "fas_Arab-ckb_Arab": { + "num_samples": 1997, + "number_of_characters": 495706, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 126.37305958938407, + "max_sentence2_length": 399, + "unique_sentence2": 1995 + }, + "fas_Arab-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 538989, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "fas_Arab-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 542466, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "fas_Arab-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 491059, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "fas_Arab-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 513139, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "fas_Arab-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 536447, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "fas_Arab-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 443635, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "fas_Arab-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 504914, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "fas_Arab-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 521715, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "fas_Arab-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 530311, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "fas_Arab-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 354965, + "unique_pairs": 1995, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "fas_Arab-kmr_Latn": { + "num_samples": 1997, + "number_of_characters": 489285, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 123.15773660490736, + "max_sentence2_length": 420, + "unique_sentence2": 1996 + }, + "fas_Arab-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 376744, + "unique_pairs": 1995, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "fas_Arab-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 502736, + "unique_pairs": 1995, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "fas_Arab-mey_Arab": { + "num_samples": 1997, + "number_of_characters": 457174, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 107.07811717576365, + "max_sentence2_length": 392, + "unique_sentence2": 1993 + }, + "fas_Arab-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 535254, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "fas_Arab-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 521205, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "fas_Arab-por_Latn": { + "num_samples": 1997, + "number_of_characters": 520554, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "fas_Arab-prs_Arab": { + "num_samples": 1997, + "number_of_characters": 485875, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 121.45017526289435, + "max_sentence2_length": 365, + "unique_sentence2": 1997 + }, + "fas_Arab-pus_Arab": { + "num_samples": 1997, + "number_of_characters": 485972, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 121.49874812218327, + "max_sentence2_length": 366, + "unique_sentence2": 1996 + }, + "fas_Arab-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 518232, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "fas_Arab-shi_Arab": { + "num_samples": 1997, + "number_of_characters": 458252, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 3, + "average_sentence2_length": 107.6179268903355, + "max_sentence2_length": 378, + "unique_sentence2": 1996 + }, + "fas_Arab-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 531539, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "fas_Arab-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 515848, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "fas_Arab-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 495166, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "fas_Arab-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 553300, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "fas_Arab-tgk_Cyrl": { + "num_samples": 1997, + "number_of_characters": 517486, + "unique_pairs": 1995, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 3, + "average_sentence2_length": 137.27941912869304, + "max_sentence2_length": 451, + "unique_sentence2": 1995 + }, + "fas_Arab-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 508952, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "fas_Arab-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 514460, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "fas_Arab-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 334817, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "fas_Arab-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 501071, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 121.85227841762644, + "max_sentence1_length": 389, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "fij_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 548225, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.478217325989, + "max_sentence1_length": 448, + "unique_sentence1": 1988, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "fij_Latn-fil_Latn": { + "num_samples": 1997, + "number_of_characters": 593925, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.478217325989, + "max_sentence1_length": 448, + "unique_sentence1": 1988, + "min_sentence2_length": 10, + "average_sentence2_length": 146.93039559339007, + "max_sentence2_length": 554, + "unique_sentence2": 1997 + }, + "fij_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 587477, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.478217325989, + "max_sentence1_length": 448, + "unique_sentence1": 1988, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "fij_Latn-mal_Mlym": { + "num_samples": 1997, + "number_of_characters": 604657, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.478217325989, + "max_sentence1_length": 448, + "unique_sentence1": 1988, + "min_sentence2_length": 7, + "average_sentence2_length": 152.30445668502753, + "max_sentence2_length": 540, + "unique_sentence2": 1996 + }, + "fij_Latn-mlg_Latn": { + "num_samples": 1997, + "number_of_characters": 620813, + "unique_pairs": 1995, + "min_sentence1_length": 10, + "average_sentence1_length": 150.478217325989, + "max_sentence1_length": 448, + "unique_sentence1": 1988, + "min_sentence2_length": 6, + "average_sentence2_length": 160.39459188783175, + "max_sentence2_length": 559, + "unique_sentence2": 1994 + }, + "fij_Latn-mri_Latn": { + "num_samples": 1997, + "number_of_characters": 574629, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.478217325989, + "max_sentence1_length": 448, + "unique_sentence1": 1988, + "min_sentence2_length": 9, + "average_sentence2_length": 137.26790185277918, + "max_sentence2_length": 443, + "unique_sentence2": 1997 + }, + "fij_Latn-msa_Latn": { + "num_samples": 1997, + "number_of_characters": 577688, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.478217325989, + "max_sentence1_length": 448, + "unique_sentence1": 1988, + "min_sentence2_length": 6, + "average_sentence2_length": 138.79969954932398, + "max_sentence2_length": 463, + "unique_sentence2": 1997 + }, + "fij_Latn-smo_Latn": { + "num_samples": 1997, + "number_of_characters": 578360, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.478217325989, + "max_sentence1_length": 448, + "unique_sentence1": 1988, + "min_sentence2_length": 8, + "average_sentence2_length": 139.1362043064597, + "max_sentence2_length": 431, + "unique_sentence2": 1996 + }, + "fij_Latn-tah_Latn": { + "num_samples": 1997, + "number_of_characters": 610128, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.478217325989, + "max_sentence1_length": 448, + "unique_sentence1": 1988, + "min_sentence2_length": 9, + "average_sentence2_length": 155.04406609914872, + "max_sentence2_length": 524, + "unique_sentence2": 1997 + }, + "fij_Latn-ton_Latn": { + "num_samples": 1997, + "number_of_characters": 614145, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.478217325989, + "max_sentence1_length": 448, + "unique_sentence1": 1988, + "min_sentence2_length": 7, + "average_sentence2_length": 157.05558337506258, + "max_sentence2_length": 468, + "unique_sentence2": 1997 + }, + "fil_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 541140, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 146.93039559339007, + "max_sentence1_length": 554, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "fil_Latn-fij_Latn": { + "num_samples": 1997, + "number_of_characters": 593925, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 146.93039559339007, + "max_sentence1_length": 554, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 150.478217325989, + "max_sentence2_length": 448, + "unique_sentence2": 1988 + }, + "fil_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 580392, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 146.93039559339007, + "max_sentence1_length": 554, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "fil_Latn-mal_Mlym": { + "num_samples": 1997, + "number_of_characters": 597572, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 146.93039559339007, + "max_sentence1_length": 554, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 152.30445668502753, + "max_sentence2_length": 540, + "unique_sentence2": 1996 + }, + "fil_Latn-mlg_Latn": { + "num_samples": 1997, + "number_of_characters": 613728, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 146.93039559339007, + "max_sentence1_length": 554, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 160.39459188783175, + "max_sentence2_length": 559, + "unique_sentence2": 1994 + }, + "fil_Latn-mri_Latn": { + "num_samples": 1997, + "number_of_characters": 567544, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 146.93039559339007, + "max_sentence1_length": 554, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 137.26790185277918, + "max_sentence2_length": 443, + "unique_sentence2": 1997 + }, + "fil_Latn-msa_Latn": { + "num_samples": 1997, + "number_of_characters": 570603, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 146.93039559339007, + "max_sentence1_length": 554, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 138.79969954932398, + "max_sentence2_length": 463, + "unique_sentence2": 1997 + }, + "fil_Latn-smo_Latn": { + "num_samples": 1997, + "number_of_characters": 571275, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 146.93039559339007, + "max_sentence1_length": 554, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 139.1362043064597, + "max_sentence2_length": 431, + "unique_sentence2": 1996 + }, + "fil_Latn-tah_Latn": { + "num_samples": 1997, + "number_of_characters": 603043, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 146.93039559339007, + "max_sentence1_length": 554, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 155.04406609914872, + "max_sentence2_length": 524, + "unique_sentence2": 1997 + }, + "fil_Latn-ton_Latn": { + "num_samples": 1997, + "number_of_characters": 607060, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 146.93039559339007, + "max_sentence1_length": 554, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 157.05558337506258, + "max_sentence2_length": 468, + "unique_sentence2": 1997 + }, + "fin_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 500981, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "fin_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 513602, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "fin_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 565450, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "fin_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 568927, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "fin_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 517520, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "fin_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 513139, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "fin_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 562908, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "fin_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 470096, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "fin_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 531375, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "fin_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 548176, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "fin_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 556772, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "fin_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 381426, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "fin_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 403205, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "fin_Latn-lav_Latn": { + "num_samples": 1997, + "number_of_characters": 537988, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 134.29544316474713, + "max_sentence2_length": 503, + "unique_sentence2": 1994 + }, + "fin_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 529197, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "fin_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 561715, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "fin_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 547666, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "fin_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 547015, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "fin_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 544693, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "fin_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 558000, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "fin_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 542309, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "fin_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 521627, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "fin_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 579761, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "fin_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 535413, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "fin_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 540921, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "fin_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 361278, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "fin_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 527532, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.10265398097147, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "fra_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 524289, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "fra_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 536910, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "fra_Latn-cat_Latn": { + "num_samples": 1997, + "number_of_characters": 576068, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 141.6925388082123, + "max_sentence2_length": 460, + "unique_sentence2": 1997 + }, + "fra_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 588758, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "fra_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 592235, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "fra_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 540828, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "fra_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 536447, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "fra_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 562908, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "fra_Latn-glg_Latn": { + "num_samples": 1997, + "number_of_characters": 565094, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 136.19729594391586, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "fra_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 493404, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "fra_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 554683, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "fra_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 571484, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "fra_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 580080, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "fra_Latn-ita_Latn": { + "num_samples": 1997, + "number_of_characters": 582325, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 144.82573860791186, + "max_sentence2_length": 623, + "unique_sentence2": 1996 + }, + "fra_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 404734, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "fra_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 426513, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "fra_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 552505, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "fra_Latn-mlt_Latn": { + "num_samples": 1997, + "number_of_characters": 570583, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 138.94591887831749, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "fra_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 585023, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "fra_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 570974, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "fra_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 570323, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "fra_Latn-ron_Latn": { + "num_samples": 1997, + "number_of_characters": 585593, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 146.4621932899349, + "max_sentence2_length": 518, + "unique_sentence2": 1997 + }, + "fra_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 568001, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "fra_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 581308, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "fra_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 565617, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "fra_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 544935, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "fra_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 603069, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "fra_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 558721, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "fra_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 564229, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "fra_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 384586, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "fra_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 550840, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.7741612418628, + "max_sentence1_length": 512, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "fuc_Latn-bem_Latn": { + "num_samples": 1997, + "number_of_characters": 526972, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 114.4116174261392, + "max_sentence1_length": 376, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 149.47020530796195, + "max_sentence2_length": 465, + "unique_sentence2": 1997 + }, + "fuc_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 476200, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 114.4116174261392, + "max_sentence1_length": 376, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "fuc_Latn-ewe_Latn": { + "num_samples": 1997, + "number_of_characters": 467458, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 114.4116174261392, + "max_sentence1_length": 376, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 119.6685027541312, + "max_sentence2_length": 493, + "unique_sentence2": 1994 + }, + "fuc_Latn-kin_Latn": { + "num_samples": 1997, + "number_of_characters": 532267, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 114.4116174261392, + "max_sentence1_length": 376, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 152.12168252378567, + "max_sentence2_length": 541, + "unique_sentence2": 1996 + }, + "fuc_Latn-nde_Latn": { + "num_samples": 1997, + "number_of_characters": 526219, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 114.4116174261392, + "max_sentence1_length": 376, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 149.09313970956435, + "max_sentence2_length": 590, + "unique_sentence2": 1997 + }, + "fuc_Latn-nya_Latn": { + "num_samples": 1997, + "number_of_characters": 512762, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 114.4116174261392, + "max_sentence1_length": 376, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 142.35453179769655, + "max_sentence2_length": 464, + "unique_sentence2": 1993 + }, + "fuc_Latn-sna_Latn": { + "num_samples": 1997, + "number_of_characters": 526810, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 114.4116174261392, + "max_sentence1_length": 376, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 149.38908362543816, + "max_sentence2_length": 511, + "unique_sentence2": 1995 + }, + "fuc_Latn-ven_Latn": { + "num_samples": 1997, + "number_of_characters": 528236, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 114.4116174261392, + "max_sentence1_length": 376, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 150.10315473209815, + "max_sentence2_length": 535, + "unique_sentence2": 1993 + }, + "gle_Latn-cym_Latn": { + "num_samples": 1997, + "number_of_characters": 561314, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 147.62593890836254, + "max_sentence1_length": 461, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 133.4526790185278, + "max_sentence2_length": 444, + "unique_sentence2": 1997 + }, + "gle_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 542529, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 147.62593890836254, + "max_sentence1_length": 461, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "glg_Latn-cat_Latn": { + "num_samples": 1997, + "number_of_characters": 554946, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 136.19729594391586, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 141.6925388082123, + "max_sentence2_length": 460, + "unique_sentence2": 1997 + }, + "glg_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 519706, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 136.19729594391586, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "glg_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 565094, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 136.19729594391586, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "glg_Latn-ita_Latn": { + "num_samples": 1997, + "number_of_characters": 561203, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 136.19729594391586, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 144.82573860791186, + "max_sentence2_length": 623, + "unique_sentence2": 1996 + }, + "glg_Latn-mlt_Latn": { + "num_samples": 1997, + "number_of_characters": 549461, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 136.19729594391586, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 138.94591887831749, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "glg_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 549201, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 136.19729594391586, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "glg_Latn-ron_Latn": { + "num_samples": 1997, + "number_of_characters": 564471, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 136.19729594391586, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 146.4621932899349, + "max_sentence2_length": 518, + "unique_sentence2": 1997 + }, + "glg_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 560186, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 136.19729594391586, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "guj_Gujr-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 488733, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "guj_Gujr-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 548779, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "guj_Gujr-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 492651, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "guj_Gujr-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 520134, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "guj_Gujr-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 506506, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "guj_Gujr-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 510467, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "guj_Gujr-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 505818, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "guj_Gujr-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 493154, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "guj_Gujr-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 495353, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "guj_Gujr-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 503672, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "guj_Gujr-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 465258, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "guj_Gujr-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 554892, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "guj_Gujr-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 492458, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "guj_Gujr-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 492929, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 122.64947421131697, + "max_sentence1_length": 378, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "hau_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 437473, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "hau_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 517686, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "hau_Latn-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 516067, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "hau_Latn-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 561465, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "hau_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 507397, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "hau_Latn-som_Latn": { + "num_samples": 1997, + "number_of_characters": 561258, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "hau_Latn-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 558108, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "hau_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 542475, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "hau_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 435204, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "hau_Latn-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 604249, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "hau_Latn-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 509769, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "hau_Latn-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 538056, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "hau_Latn-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 586054, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "hau_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 527698, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 135.185778668002, + "max_sentence1_length": 483, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "heb_Hebr-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 431477, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "heb_Hebr-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 444098, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "heb_Hebr-ckb_Arab": { + "num_samples": 1997, + "number_of_characters": 452663, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 126.37305958938407, + "max_sentence2_length": 399, + "unique_sentence2": 1995 + }, + "heb_Hebr-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 495946, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "heb_Hebr-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 499423, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "heb_Hebr-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 448016, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "heb_Hebr-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 443635, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "heb_Hebr-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 470096, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "heb_Hebr-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 493404, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "heb_Hebr-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 461871, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "heb_Hebr-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 478672, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "heb_Hebr-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 487268, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "heb_Hebr-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 311922, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "heb_Hebr-kmr_Latn": { + "num_samples": 1997, + "number_of_characters": 446242, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 123.15773660490736, + "max_sentence2_length": 420, + "unique_sentence2": 1996 + }, + "heb_Hebr-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 333701, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "heb_Hebr-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 459693, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "heb_Hebr-mey_Arab": { + "num_samples": 1997, + "number_of_characters": 414131, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 107.07811717576365, + "max_sentence2_length": 392, + "unique_sentence2": 1993 + }, + "heb_Hebr-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 492211, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "heb_Hebr-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 478162, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "heb_Hebr-por_Latn": { + "num_samples": 1997, + "number_of_characters": 477511, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "heb_Hebr-prs_Arab": { + "num_samples": 1997, + "number_of_characters": 442832, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 121.45017526289435, + "max_sentence2_length": 365, + "unique_sentence2": 1997 + }, + "heb_Hebr-pus_Arab": { + "num_samples": 1997, + "number_of_characters": 442929, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 121.49874812218327, + "max_sentence2_length": 366, + "unique_sentence2": 1996 + }, + "heb_Hebr-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 475189, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "heb_Hebr-shi_Arab": { + "num_samples": 1997, + "number_of_characters": 415209, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 107.6179268903355, + "max_sentence2_length": 378, + "unique_sentence2": 1996 + }, + "heb_Hebr-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 488496, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "heb_Hebr-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 472805, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "heb_Hebr-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 452123, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "heb_Hebr-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 510257, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "heb_Hebr-tgk_Cyrl": { + "num_samples": 1997, + "number_of_characters": 474443, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 137.27941912869304, + "max_sentence2_length": 451, + "unique_sentence2": 1995 + }, + "heb_Hebr-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 465909, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "heb_Hebr-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 471417, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "heb_Hebr-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 291774, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "heb_Hebr-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 458028, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 100.29844767150726, + "max_sentence1_length": 375, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "hin_Deva-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 492756, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "hin_Deva-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 505377, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "hin_Deva-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 557225, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "hin_Deva-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 565423, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "hin_Deva-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 560702, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "hin_Deva-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 509295, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "hin_Deva-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 536778, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "hin_Deva-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 504914, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "hin_Deva-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 531375, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "hin_Deva-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 554683, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "hin_Deva-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 506506, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "hin_Deva-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 461871, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "hin_Deva-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 539951, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "hin_Deva-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 548547, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "hin_Deva-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 373201, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "hin_Deva-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 527111, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "hin_Deva-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 394980, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "hin_Deva-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 520972, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "hin_Deva-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 522462, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "hin_Deva-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 509798, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "hin_Deva-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 553490, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "hin_Deva-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 511997, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "hin_Deva-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 539441, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "hin_Deva-por_Latn": { + "num_samples": 1997, + "number_of_characters": 538790, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "hin_Deva-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 536468, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "hin_Deva-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 520316, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "hin_Deva-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 481902, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "hin_Deva-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 549775, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "hin_Deva-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 534084, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "hin_Deva-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 513402, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "hin_Deva-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 571536, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "hin_Deva-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 509102, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "hin_Deva-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 527188, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "hin_Deva-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 509573, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "hin_Deva-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 532696, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "hin_Deva-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 353053, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "hin_Deva-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 519307, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 130.9839759639459, + "max_sentence1_length": 394, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "hmn_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 578510, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 165.6434651977967, + "max_sentence1_length": 643, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "hrv_Latn-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 512015, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "hrv_Latn-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 510835, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "hrv_Latn-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 525814, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "hrv_Latn-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 497243, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "hrv_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 503645, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "hrv_Latn-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 523816, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "hrv_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 533791, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "hrv_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 530818, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "hrv_Latn-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 508894, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "hrv_Latn-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 508821, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "hrv_Latn-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 508228, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "hrv_Latn-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 512066, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "hrv_Latn-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 518708, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 128.1547320981472, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "hun_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 509557, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "hun_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 522178, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "hun_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 574026, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "hun_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 577503, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "hun_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 526096, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "hun_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 521715, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "hun_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 548176, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "hun_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 571484, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "hun_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 478672, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "hun_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 539951, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "hun_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 565348, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "hun_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 390002, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "hun_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 411781, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "hun_Latn-lav_Latn": { + "num_samples": 1997, + "number_of_characters": 546564, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 134.29544316474713, + "max_sentence2_length": 503, + "unique_sentence2": 1994 + }, + "hun_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 537773, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "hun_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 570291, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "hun_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 556242, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "hun_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 555591, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "hun_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 553269, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "hun_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 566576, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "hun_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 550885, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "hun_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 530203, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "hun_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 588337, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "hun_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 543989, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "hun_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 549497, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "hun_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 369854, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "hun_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 536108, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 139.3970956434652, + "max_sentence1_length": 508, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "hye_Armn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 563842, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 132.55633450175262, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "hye_Armn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 512435, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 132.55633450175262, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "hye_Armn-kat_Geor": { + "num_samples": 1997, + "number_of_characters": 531307, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 132.55633450175262, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 133.49624436654983, + "max_sentence2_length": 503, + "unique_sentence2": 1995 + }, + "hye_Armn-sqi_Latn": { + "num_samples": 1997, + "number_of_characters": 548322, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 132.55633450175262, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 142.01652478718077, + "max_sentence2_length": 461, + "unique_sentence2": 1996 + }, + "ibo_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 413608, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "ibo_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 493821, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ibo_Latn-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 516067, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "ibo_Latn-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 537600, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "ibo_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 483532, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "ibo_Latn-som_Latn": { + "num_samples": 1997, + "number_of_characters": 537393, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "ibo_Latn-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 534243, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "ibo_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 518610, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "ibo_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 411339, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "ibo_Latn-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 580384, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "ibo_Latn-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 485904, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "ibo_Latn-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 514191, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "ibo_Latn-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 562189, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "ibo_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 503833, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 123.23535302954431, + "max_sentence1_length": 469, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "ind_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 518153, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "ind_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 530774, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "ind_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 582622, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "ind_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 586099, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "ind_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 534692, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ind_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 530311, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "ind_Latn-fij_Latn": { + "num_samples": 1997, + "number_of_characters": 587477, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 150.478217325989, + "max_sentence2_length": 448, + "unique_sentence2": 1988 + }, + "ind_Latn-fil_Latn": { + "num_samples": 1997, + "number_of_characters": 580392, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 146.93039559339007, + "max_sentence2_length": 554, + "unique_sentence2": 1997 + }, + "ind_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 556772, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "ind_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 580080, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "ind_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 487268, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "ind_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 548547, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "ind_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 565348, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "ind_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 398598, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "ind_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 420377, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "ind_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 546369, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "ind_Latn-mal_Mlym": { + "num_samples": 1997, + "number_of_characters": 591124, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 152.30445668502753, + "max_sentence2_length": 540, + "unique_sentence2": 1996 + }, + "ind_Latn-mlg_Latn": { + "num_samples": 1997, + "number_of_characters": 607280, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 160.39459188783175, + "max_sentence2_length": 559, + "unique_sentence2": 1994 + }, + "ind_Latn-mri_Latn": { + "num_samples": 1997, + "number_of_characters": 561096, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 137.26790185277918, + "max_sentence2_length": 443, + "unique_sentence2": 1997 + }, + "ind_Latn-msa_Latn": { + "num_samples": 1997, + "number_of_characters": 564155, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 138.79969954932398, + "max_sentence2_length": 463, + "unique_sentence2": 1997 + }, + "ind_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 578887, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "ind_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 564838, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "ind_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 564187, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "ind_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 561865, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "ind_Latn-smo_Latn": { + "num_samples": 1997, + "number_of_characters": 564827, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 139.1362043064597, + "max_sentence2_length": 431, + "unique_sentence2": 1996 + }, + "ind_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 575172, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "ind_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 559481, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "ind_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 538799, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "ind_Latn-tah_Latn": { + "num_samples": 1997, + "number_of_characters": 596595, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 155.04406609914872, + "max_sentence2_length": 524, + "unique_sentence2": 1997 + }, + "ind_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 596933, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "ind_Latn-ton_Latn": { + "num_samples": 1997, + "number_of_characters": 600612, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 157.05558337506258, + "max_sentence2_length": 468, + "unique_sentence2": 1997 + }, + "ind_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 552585, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "ind_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 558093, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "ind_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 378450, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "ind_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 544704, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 143.70155232849274, + "max_sentence1_length": 486, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "isl_Latn-afr_Latn": { + "num_samples": 1997, + "number_of_characters": 530560, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 131.30095142714072, + "max_sentence1_length": 399, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 134.37756634952427, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "isl_Latn-dan_Latn": { + "num_samples": 1997, + "number_of_characters": 514346, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 131.30095142714072, + "max_sentence1_length": 399, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 126.25838758137206, + "max_sentence2_length": 522, + "unique_sentence2": 1995 + }, + "isl_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 557858, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 131.30095142714072, + "max_sentence1_length": 399, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "isl_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 509928, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 131.30095142714072, + "max_sentence1_length": 399, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "isl_Latn-fao_Latn": { + "num_samples": 1997, + "number_of_characters": 520011, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 131.30095142714072, + "max_sentence1_length": 399, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0951427140711, + "max_sentence2_length": 433, + "unique_sentence2": 1997 + }, + "isl_Latn-ltz_Latn": { + "num_samples": 1997, + "number_of_characters": 542965, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 131.30095142714072, + "max_sentence1_length": 399, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 140.58938407611416, + "max_sentence2_length": 543, + "unique_sentence2": 1996 + }, + "isl_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 554123, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 131.30095142714072, + "max_sentence1_length": 399, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "isl_Latn-nno_Latn": { + "num_samples": 1997, + "number_of_characters": 510565, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 131.30095142714072, + "max_sentence1_length": 399, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.36504757135704, + "max_sentence2_length": 417, + "unique_sentence2": 1996 + }, + "isl_Latn-nob_Latn": { + "num_samples": 1997, + "number_of_characters": 513652, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 131.30095142714072, + "max_sentence1_length": 399, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.91086629944917, + "max_sentence2_length": 482, + "unique_sentence2": 1996 + }, + "isl_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 514035, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 131.30095142714072, + "max_sentence1_length": 399, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "ita_Latn-cat_Latn": { + "num_samples": 1997, + "number_of_characters": 572177, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 144.82573860791186, + "max_sentence1_length": 623, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 141.6925388082123, + "max_sentence2_length": 460, + "unique_sentence2": 1997 + }, + "ita_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 536937, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 144.82573860791186, + "max_sentence1_length": 623, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ita_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 582325, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 144.82573860791186, + "max_sentence1_length": 623, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "ita_Latn-glg_Latn": { + "num_samples": 1997, + "number_of_characters": 561203, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 144.82573860791186, + "max_sentence1_length": 623, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 136.19729594391586, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "ita_Latn-mlt_Latn": { + "num_samples": 1997, + "number_of_characters": 566692, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 144.82573860791186, + "max_sentence1_length": 623, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 138.94591887831749, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "ita_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 566432, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 144.82573860791186, + "max_sentence1_length": 623, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "ita_Latn-ron_Latn": { + "num_samples": 1997, + "number_of_characters": 581702, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 144.82573860791186, + "max_sentence1_length": 623, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 146.4621932899349, + "max_sentence2_length": 518, + "unique_sentence2": 1997 + }, + "ita_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 577417, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 144.82573860791186, + "max_sentence1_length": 623, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "jpn_Jpan-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 342807, + "unique_pairs": 1995, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "jpn_Jpan-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 355428, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "jpn_Jpan-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 407276, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "jpn_Jpan-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 410753, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "jpn_Jpan-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 359346, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "jpn_Jpan-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 354965, + "unique_pairs": 1995, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "jpn_Jpan-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 381426, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "jpn_Jpan-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 404734, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "jpn_Jpan-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 311922, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "jpn_Jpan-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 373201, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "jpn_Jpan-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 390002, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "jpn_Jpan-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 398598, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "jpn_Jpan-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 245031, + "unique_pairs": 1995, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "jpn_Jpan-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 371023, + "unique_pairs": 1995, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "jpn_Jpan-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 403541, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "jpn_Jpan-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 389492, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "jpn_Jpan-por_Latn": { + "num_samples": 1997, + "number_of_characters": 388841, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "jpn_Jpan-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 386519, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "jpn_Jpan-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 399826, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "jpn_Jpan-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 384135, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "jpn_Jpan-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 363453, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "jpn_Jpan-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 421587, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "jpn_Jpan-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 377239, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "jpn_Jpan-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 382747, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "jpn_Jpan-yue_Hant": { + "num_samples": 1997, + "number_of_characters": 190513, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 4, + "average_sentence2_length": 39.502754131196795, + "max_sentence2_length": 133, + "unique_sentence2": 1996 + }, + "jpn_Jpan-zho_Hans": { + "num_samples": 1997, + "number_of_characters": 196587, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 4, + "average_sentence2_length": 42.54431647471207, + "max_sentence2_length": 263, + "unique_sentence2": 1997 + }, + "jpn_Jpan-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 203104, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "jpn_Jpan-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 369358, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 55.89684526790185, + "max_sentence1_length": 189, + "unique_sentence1": 1994, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "kan_Knda-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 509338, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "kan_Knda-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 569384, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "kan_Knda-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 513256, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "kan_Knda-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 540739, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "kan_Knda-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 510467, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "kan_Knda-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 527111, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "kan_Knda-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 526423, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "kan_Knda-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 513759, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "kan_Knda-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 515958, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "kan_Knda-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 524277, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "kan_Knda-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 485863, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "kan_Knda-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 575497, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "kan_Knda-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 513063, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "kan_Knda-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 513534, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 132.96745117676514, + "max_sentence1_length": 449, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "kat_Geor-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 565719, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 133.49624436654983, + "max_sentence1_length": 503, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "kat_Geor-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 514312, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 133.49624436654983, + "max_sentence1_length": 503, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "kat_Geor-hye_Armn": { + "num_samples": 1997, + "number_of_characters": 531307, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 133.49624436654983, + "max_sentence1_length": 503, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 132.55633450175262, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "kat_Geor-sqi_Latn": { + "num_samples": 1997, + "number_of_characters": 550199, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 133.49624436654983, + "max_sentence1_length": 503, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 142.01652478718077, + "max_sentence2_length": 461, + "unique_sentence2": 1996 + }, + "kaz_Cyrl-aze_Latn": { + "num_samples": 1997, + "number_of_characters": 529910, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 130.33350025037555, + "max_sentence1_length": 473, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 135.0195292939409, + "max_sentence2_length": 398, + "unique_sentence2": 1997 + }, + "kaz_Cyrl-bak_Cyrl": { + "num_samples": 1997, + "number_of_characters": 506602, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 130.33350025037555, + "max_sentence1_length": 473, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 123.34802203304957, + "max_sentence2_length": 437, + "unique_sentence2": 1995 + }, + "kaz_Cyrl-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 507996, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 130.33350025037555, + "max_sentence1_length": 473, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "kaz_Cyrl-kir_Cyrl": { + "num_samples": 1997, + "number_of_characters": 511140, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 130.33350025037555, + "max_sentence1_length": 473, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.62043064596895, + "max_sentence2_length": 395, + "unique_sentence2": 1996 + }, + "kaz_Cyrl-tat_Cyrl": { + "num_samples": 1997, + "number_of_characters": 506202, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 130.33350025037555, + "max_sentence1_length": 473, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 123.14772158237356, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "kaz_Cyrl-tuk_Latn": { + "num_samples": 1997, + "number_of_characters": 545550, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 130.33350025037555, + "max_sentence1_length": 473, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 142.85127691537306, + "max_sentence2_length": 576, + "unique_sentence2": 1996 + }, + "kaz_Cyrl-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 525889, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 130.33350025037555, + "max_sentence1_length": 473, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "kaz_Cyrl-uig_Arab": { + "num_samples": 1997, + "number_of_characters": 571298, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 130.33350025037555, + "max_sentence1_length": 473, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 155.74461692538807, + "max_sentence2_length": 592, + "unique_sentence2": 1996 + }, + "kaz_Cyrl-uzb_Latn": { + "num_samples": 1997, + "number_of_characters": 553971, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 130.33350025037555, + "max_sentence1_length": 473, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 147.06810215322986, + "max_sentence2_length": 470, + "unique_sentence2": 1996 + }, + "khm_Khmr-bod_Tibt": { + "num_samples": 1997, + "number_of_characters": 589120, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 144.4621932899349, + "max_sentence1_length": 517, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 150.54031046569855, + "max_sentence2_length": 478, + "unique_sentence2": 1993 + }, + "khm_Khmr-dzo_Tibt": { + "num_samples": 1997, + "number_of_characters": 531712, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 144.4621932899349, + "max_sentence1_length": 517, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 121.79318978467701, + "max_sentence2_length": 411, + "unique_sentence2": 1992 + }, + "khm_Khmr-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 536211, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 144.4621932899349, + "max_sentence1_length": 517, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "khm_Khmr-lao_Laoo": { + "num_samples": 1997, + "number_of_characters": 555471, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 144.4621932899349, + "max_sentence1_length": 517, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 133.69053580370556, + "max_sentence2_length": 507, + "unique_sentence2": 1997 + }, + "khm_Khmr-mon_Mong": { + "num_samples": 1997, + "number_of_characters": 547539, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 144.4621932899349, + "max_sentence1_length": 517, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 129.7185778668002, + "max_sentence2_length": 414, + "unique_sentence2": 1997 + }, + "khm_Khmr-mya_Mymr": { + "num_samples": 1997, + "number_of_characters": 600345, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 144.4621932899349, + "max_sentence1_length": 517, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 156.16124186279418, + "max_sentence2_length": 773, + "unique_sentence2": 1997 + }, + "khm_Khmr-tha_Thai": { + "num_samples": 1997, + "number_of_characters": 525959, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 144.4621932899349, + "max_sentence1_length": 517, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 118.91236855282925, + "max_sentence2_length": 439, + "unique_sentence2": 1996 + }, + "kin_Latn-bem_Latn": { + "num_samples": 1997, + "number_of_characters": 602279, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 152.12168252378567, + "max_sentence1_length": 541, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 149.47020530796195, + "max_sentence2_length": 465, + "unique_sentence2": 1997 + }, + "kin_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 551507, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 152.12168252378567, + "max_sentence1_length": 541, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "kin_Latn-ewe_Latn": { + "num_samples": 1997, + "number_of_characters": 542765, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 152.12168252378567, + "max_sentence1_length": 541, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 119.6685027541312, + "max_sentence2_length": 493, + "unique_sentence2": 1994 + }, + "kin_Latn-fuc_Latn": { + "num_samples": 1997, + "number_of_characters": 532267, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 152.12168252378567, + "max_sentence1_length": 541, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 114.4116174261392, + "max_sentence2_length": 376, + "unique_sentence2": 1996 + }, + "kin_Latn-nde_Latn": { + "num_samples": 1997, + "number_of_characters": 601526, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 152.12168252378567, + "max_sentence1_length": 541, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 149.09313970956435, + "max_sentence2_length": 590, + "unique_sentence2": 1997 + }, + "kin_Latn-nya_Latn": { + "num_samples": 1997, + "number_of_characters": 588069, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 152.12168252378567, + "max_sentence1_length": 541, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 142.35453179769655, + "max_sentence2_length": 464, + "unique_sentence2": 1993 + }, + "kin_Latn-sna_Latn": { + "num_samples": 1997, + "number_of_characters": 602117, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 152.12168252378567, + "max_sentence1_length": 541, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 149.38908362543816, + "max_sentence2_length": 511, + "unique_sentence2": 1995 + }, + "kin_Latn-ven_Latn": { + "num_samples": 1997, + "number_of_characters": 603543, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 152.12168252378567, + "max_sentence1_length": 541, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 150.10315473209815, + "max_sentence2_length": 535, + "unique_sentence2": 1993 + }, + "kir_Cyrl-aze_Latn": { + "num_samples": 1997, + "number_of_characters": 520498, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 125.62043064596895, + "max_sentence1_length": 395, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 135.0195292939409, + "max_sentence2_length": 398, + "unique_sentence2": 1997 + }, + "kir_Cyrl-bak_Cyrl": { + "num_samples": 1997, + "number_of_characters": 497190, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 125.62043064596895, + "max_sentence1_length": 395, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 123.34802203304957, + "max_sentence2_length": 437, + "unique_sentence2": 1995 + }, + "kir_Cyrl-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 498584, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 125.62043064596895, + "max_sentence1_length": 395, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "kir_Cyrl-kaz_Cyrl": { + "num_samples": 1997, + "number_of_characters": 511140, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.62043064596895, + "max_sentence1_length": 395, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 130.33350025037555, + "max_sentence2_length": 473, + "unique_sentence2": 1996 + }, + "kir_Cyrl-tat_Cyrl": { + "num_samples": 1997, + "number_of_characters": 496790, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.62043064596895, + "max_sentence1_length": 395, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 123.14772158237356, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "kir_Cyrl-tuk_Latn": { + "num_samples": 1997, + "number_of_characters": 536138, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.62043064596895, + "max_sentence1_length": 395, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 142.85127691537306, + "max_sentence2_length": 576, + "unique_sentence2": 1996 + }, + "kir_Cyrl-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 516477, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 125.62043064596895, + "max_sentence1_length": 395, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "kir_Cyrl-uig_Arab": { + "num_samples": 1997, + "number_of_characters": 561886, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 125.62043064596895, + "max_sentence1_length": 395, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 155.74461692538807, + "max_sentence2_length": 592, + "unique_sentence2": 1996 + }, + "kir_Cyrl-uzb_Latn": { + "num_samples": 1997, + "number_of_characters": 544559, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.62043064596895, + "max_sentence1_length": 395, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 147.06810215322986, + "max_sentence2_length": 470, + "unique_sentence2": 1996 + }, + "kmr_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 477127, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 123.15773660490736, + "max_sentence1_length": 420, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "kmr_Latn-ckb_Arab": { + "num_samples": 1997, + "number_of_characters": 498313, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 123.15773660490736, + "max_sentence1_length": 420, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 126.37305958938407, + "max_sentence2_length": 399, + "unique_sentence2": 1995 + }, + "kmr_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 493666, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 123.15773660490736, + "max_sentence1_length": 420, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "kmr_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 489285, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 123.15773660490736, + "max_sentence1_length": 420, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "kmr_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 446242, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 123.15773660490736, + "max_sentence1_length": 420, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "kmr_Latn-mey_Arab": { + "num_samples": 1997, + "number_of_characters": 459781, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 123.15773660490736, + "max_sentence1_length": 420, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 107.07811717576365, + "max_sentence2_length": 392, + "unique_sentence2": 1993 + }, + "kmr_Latn-prs_Arab": { + "num_samples": 1997, + "number_of_characters": 488482, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 123.15773660490736, + "max_sentence1_length": 420, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 121.45017526289435, + "max_sentence2_length": 365, + "unique_sentence2": 1997 + }, + "kmr_Latn-pus_Arab": { + "num_samples": 1997, + "number_of_characters": 488579, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 123.15773660490736, + "max_sentence1_length": 420, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 121.49874812218327, + "max_sentence2_length": 366, + "unique_sentence2": 1996 + }, + "kmr_Latn-shi_Arab": { + "num_samples": 1997, + "number_of_characters": 460859, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 123.15773660490736, + "max_sentence1_length": 420, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 107.6179268903355, + "max_sentence2_length": 378, + "unique_sentence2": 1996 + }, + "kmr_Latn-tgk_Cyrl": { + "num_samples": 1997, + "number_of_characters": 520093, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 123.15773660490736, + "max_sentence1_length": 420, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 137.27941912869304, + "max_sentence2_length": 451, + "unique_sentence2": 1995 + }, + "kor_Hang-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 364586, + "unique_pairs": 1995, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "kor_Hang-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 377207, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "kor_Hang-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 429055, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "kor_Hang-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 432532, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "kor_Hang-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 381125, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "kor_Hang-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 376744, + "unique_pairs": 1995, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "kor_Hang-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 403205, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "kor_Hang-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 426513, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "kor_Hang-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 333701, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "kor_Hang-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 394980, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "kor_Hang-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 411781, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "kor_Hang-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 420377, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "kor_Hang-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 245031, + "unique_pairs": 1995, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "kor_Hang-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 392802, + "unique_pairs": 1995, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "kor_Hang-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 425320, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "kor_Hang-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 411271, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "kor_Hang-por_Latn": { + "num_samples": 1997, + "number_of_characters": 410620, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "kor_Hang-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 408298, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "kor_Hang-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 421605, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "kor_Hang-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 405914, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "kor_Hang-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 385232, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "kor_Hang-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 443366, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "kor_Hang-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 399018, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "kor_Hang-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 404526, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "kor_Hang-yue_Hant": { + "num_samples": 1997, + "number_of_characters": 212292, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 4, + "average_sentence2_length": 39.502754131196795, + "max_sentence2_length": 133, + "unique_sentence2": 1996 + }, + "kor_Hang-zho_Hans": { + "num_samples": 1997, + "number_of_characters": 218366, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 4, + "average_sentence2_length": 42.54431647471207, + "max_sentence2_length": 263, + "unique_sentence2": 1997 + }, + "kor_Hang-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 224883, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "kor_Hang-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 391137, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 66.80270405608412, + "max_sentence1_length": 217, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "lao_Laoo-bod_Tibt": { + "num_samples": 1997, + "number_of_characters": 567609, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 133.69053580370556, + "max_sentence1_length": 507, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 150.54031046569855, + "max_sentence2_length": 478, + "unique_sentence2": 1993 + }, + "lao_Laoo-dzo_Tibt": { + "num_samples": 1997, + "number_of_characters": 510201, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 133.69053580370556, + "max_sentence1_length": 507, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 121.79318978467701, + "max_sentence2_length": 411, + "unique_sentence2": 1992 + }, + "lao_Laoo-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 514700, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 133.69053580370556, + "max_sentence1_length": 507, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "lao_Laoo-khm_Khmr": { + "num_samples": 1997, + "number_of_characters": 555471, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 133.69053580370556, + "max_sentence1_length": 507, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 144.4621932899349, + "max_sentence2_length": 517, + "unique_sentence2": 1996 + }, + "lao_Laoo-mon_Mong": { + "num_samples": 1997, + "number_of_characters": 526028, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 133.69053580370556, + "max_sentence1_length": 507, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 129.7185778668002, + "max_sentence2_length": 414, + "unique_sentence2": 1997 + }, + "lao_Laoo-mya_Mymr": { + "num_samples": 1997, + "number_of_characters": 578834, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 133.69053580370556, + "max_sentence1_length": 507, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 156.16124186279418, + "max_sentence2_length": 773, + "unique_sentence2": 1997 + }, + "lao_Laoo-tha_Thai": { + "num_samples": 1997, + "number_of_characters": 504448, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 133.69053580370556, + "max_sentence1_length": 507, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 118.91236855282925, + "max_sentence2_length": 439, + "unique_sentence2": 1996 + }, + "lav_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 515908, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 134.29544316474713, + "max_sentence1_length": 503, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "lav_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 537988, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 134.29544316474713, + "max_sentence1_length": 503, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "lav_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 546564, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 134.29544316474713, + "max_sentence1_length": 503, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "lav_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 527585, + "unique_pairs": 1995, + "min_sentence1_length": 7, + "average_sentence1_length": 134.29544316474713, + "max_sentence1_length": 503, + "unique_sentence1": 1994, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "lit_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 490578, + "unique_pairs": 1995, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "lit_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 503199, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "lit_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 555047, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "lit_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 558524, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "lit_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 507117, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "lit_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 502736, + "unique_pairs": 1995, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "lit_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 529197, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "lit_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 552505, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "lit_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 459693, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "lit_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 520972, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "lit_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 537773, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "lit_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 546369, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "lit_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 371023, + "unique_pairs": 1995, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "lit_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 392802, + "unique_pairs": 1995, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "lit_Latn-lav_Latn": { + "num_samples": 1997, + "number_of_characters": 527585, + "unique_pairs": 1995, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 134.29544316474713, + "max_sentence2_length": 503, + "unique_sentence2": 1994 + }, + "lit_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 551312, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "lit_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 537263, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "lit_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 536612, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "lit_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 534290, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "lit_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 547597, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "lit_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 531906, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "lit_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 511224, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "lit_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 569358, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "lit_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 525010, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "lit_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 530518, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "lit_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 350875, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "lit_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 517129, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 129.893340010015, + "max_sentence1_length": 446, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "ltz_Latn-afr_Latn": { + "num_samples": 1997, + "number_of_characters": 549109, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 140.58938407611416, + "max_sentence1_length": 543, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 134.37756634952427, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "ltz_Latn-dan_Latn": { + "num_samples": 1997, + "number_of_characters": 532895, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 140.58938407611416, + "max_sentence1_length": 543, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 126.25838758137206, + "max_sentence2_length": 522, + "unique_sentence2": 1995 + }, + "ltz_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 576407, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 140.58938407611416, + "max_sentence1_length": 543, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "ltz_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 528477, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 140.58938407611416, + "max_sentence1_length": 543, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ltz_Latn-fao_Latn": { + "num_samples": 1997, + "number_of_characters": 538560, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 140.58938407611416, + "max_sentence1_length": 543, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0951427140711, + "max_sentence2_length": 433, + "unique_sentence2": 1997 + }, + "ltz_Latn-isl_Latn": { + "num_samples": 1997, + "number_of_characters": 542965, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 140.58938407611416, + "max_sentence1_length": 543, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 131.30095142714072, + "max_sentence2_length": 399, + "unique_sentence2": 1996 + }, + "ltz_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 572672, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 140.58938407611416, + "max_sentence1_length": 543, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "ltz_Latn-nno_Latn": { + "num_samples": 1997, + "number_of_characters": 529114, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 140.58938407611416, + "max_sentence1_length": 543, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.36504757135704, + "max_sentence2_length": 417, + "unique_sentence2": 1996 + }, + "ltz_Latn-nob_Latn": { + "num_samples": 1997, + "number_of_characters": 532201, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 140.58938407611416, + "max_sentence1_length": 543, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.91086629944917, + "max_sentence2_length": 482, + "unique_sentence2": 1996 + }, + "ltz_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 532584, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 140.58938407611416, + "max_sentence1_length": 543, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "mal_Mlym-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 551872, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 152.30445668502753, + "max_sentence1_length": 540, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "mal_Mlym-fij_Latn": { + "num_samples": 1997, + "number_of_characters": 604657, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 152.30445668502753, + "max_sentence1_length": 540, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 150.478217325989, + "max_sentence2_length": 448, + "unique_sentence2": 1988 + }, + "mal_Mlym-fil_Latn": { + "num_samples": 1997, + "number_of_characters": 597572, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 152.30445668502753, + "max_sentence1_length": 540, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 146.93039559339007, + "max_sentence2_length": 554, + "unique_sentence2": 1997 + }, + "mal_Mlym-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 591124, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 152.30445668502753, + "max_sentence1_length": 540, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "mal_Mlym-mlg_Latn": { + "num_samples": 1997, + "number_of_characters": 624460, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 152.30445668502753, + "max_sentence1_length": 540, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 160.39459188783175, + "max_sentence2_length": 559, + "unique_sentence2": 1994 + }, + "mal_Mlym-mri_Latn": { + "num_samples": 1997, + "number_of_characters": 578276, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 152.30445668502753, + "max_sentence1_length": 540, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 137.26790185277918, + "max_sentence2_length": 443, + "unique_sentence2": 1997 + }, + "mal_Mlym-msa_Latn": { + "num_samples": 1997, + "number_of_characters": 581335, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 152.30445668502753, + "max_sentence1_length": 540, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 138.79969954932398, + "max_sentence2_length": 463, + "unique_sentence2": 1997 + }, + "mal_Mlym-smo_Latn": { + "num_samples": 1997, + "number_of_characters": 582007, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 152.30445668502753, + "max_sentence1_length": 540, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 139.1362043064597, + "max_sentence2_length": 431, + "unique_sentence2": 1996 + }, + "mal_Mlym-tah_Latn": { + "num_samples": 1997, + "number_of_characters": 613775, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 152.30445668502753, + "max_sentence1_length": 540, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 155.04406609914872, + "max_sentence2_length": 524, + "unique_sentence2": 1997 + }, + "mal_Mlym-ton_Latn": { + "num_samples": 1997, + "number_of_characters": 617792, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 152.30445668502753, + "max_sentence1_length": 540, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 157.05558337506258, + "max_sentence2_length": 468, + "unique_sentence2": 1997 + }, + "mar_Deva-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 504689, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "mar_Deva-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 564735, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "mar_Deva-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 508607, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "mar_Deva-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 536090, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "mar_Deva-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 505818, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "mar_Deva-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 522462, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "mar_Deva-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 526423, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "mar_Deva-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 509110, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "mar_Deva-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 511309, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "mar_Deva-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 519628, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "mar_Deva-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 481214, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "mar_Deva-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 570848, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "mar_Deva-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 508414, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "mar_Deva-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 508885, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 130.63945918878318, + "max_sentence1_length": 443, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "mey_Arab-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 445016, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 107.07811717576365, + "max_sentence1_length": 392, + "unique_sentence1": 1993, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "mey_Arab-ckb_Arab": { + "num_samples": 1997, + "number_of_characters": 466202, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 107.07811717576365, + "max_sentence1_length": 392, + "unique_sentence1": 1993, + "min_sentence2_length": 5, + "average_sentence2_length": 126.37305958938407, + "max_sentence2_length": 399, + "unique_sentence2": 1995 + }, + "mey_Arab-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 461555, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 107.07811717576365, + "max_sentence1_length": 392, + "unique_sentence1": 1993, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "mey_Arab-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 457174, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 107.07811717576365, + "max_sentence1_length": 392, + "unique_sentence1": 1993, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "mey_Arab-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 414131, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 107.07811717576365, + "max_sentence1_length": 392, + "unique_sentence1": 1993, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "mey_Arab-kmr_Latn": { + "num_samples": 1997, + "number_of_characters": 459781, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 107.07811717576365, + "max_sentence1_length": 392, + "unique_sentence1": 1993, + "min_sentence2_length": 7, + "average_sentence2_length": 123.15773660490736, + "max_sentence2_length": 420, + "unique_sentence2": 1996 + }, + "mey_Arab-prs_Arab": { + "num_samples": 1997, + "number_of_characters": 456371, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 107.07811717576365, + "max_sentence1_length": 392, + "unique_sentence1": 1993, + "min_sentence2_length": 8, + "average_sentence2_length": 121.45017526289435, + "max_sentence2_length": 365, + "unique_sentence2": 1997 + }, + "mey_Arab-pus_Arab": { + "num_samples": 1997, + "number_of_characters": 456468, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 107.07811717576365, + "max_sentence1_length": 392, + "unique_sentence1": 1993, + "min_sentence2_length": 8, + "average_sentence2_length": 121.49874812218327, + "max_sentence2_length": 366, + "unique_sentence2": 1996 + }, + "mey_Arab-shi_Arab": { + "num_samples": 1997, + "number_of_characters": 428748, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 107.07811717576365, + "max_sentence1_length": 392, + "unique_sentence1": 1993, + "min_sentence2_length": 3, + "average_sentence2_length": 107.6179268903355, + "max_sentence2_length": 378, + "unique_sentence2": 1996 + }, + "mey_Arab-tgk_Cyrl": { + "num_samples": 1997, + "number_of_characters": 487982, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 107.07811717576365, + "max_sentence1_length": 392, + "unique_sentence1": 1993, + "min_sentence2_length": 3, + "average_sentence2_length": 137.27941912869304, + "max_sentence2_length": 451, + "unique_sentence2": 1995 + }, + "mkd_Cyrl-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 523981, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "mkd_Cyrl-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 522801, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "mkd_Cyrl-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 537780, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "mkd_Cyrl-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 509209, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "mkd_Cyrl-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 515611, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "mkd_Cyrl-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 523816, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "mkd_Cyrl-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 545757, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "mkd_Cyrl-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 542784, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "mkd_Cyrl-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 520860, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "mkd_Cyrl-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 520787, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "mkd_Cyrl-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 520194, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "mkd_Cyrl-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 524032, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "mkd_Cyrl-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 530674, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.1467200801202, + "max_sentence1_length": 451, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "mlg_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 568028, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 160.39459188783175, + "max_sentence1_length": 559, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "mlg_Latn-fij_Latn": { + "num_samples": 1997, + "number_of_characters": 620813, + "unique_pairs": 1995, + "min_sentence1_length": 6, + "average_sentence1_length": 160.39459188783175, + "max_sentence1_length": 559, + "unique_sentence1": 1994, + "min_sentence2_length": 10, + "average_sentence2_length": 150.478217325989, + "max_sentence2_length": 448, + "unique_sentence2": 1988 + }, + "mlg_Latn-fil_Latn": { + "num_samples": 1997, + "number_of_characters": 613728, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 160.39459188783175, + "max_sentence1_length": 559, + "unique_sentence1": 1994, + "min_sentence2_length": 10, + "average_sentence2_length": 146.93039559339007, + "max_sentence2_length": 554, + "unique_sentence2": 1997 + }, + "mlg_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 607280, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 160.39459188783175, + "max_sentence1_length": 559, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "mlg_Latn-mal_Mlym": { + "num_samples": 1997, + "number_of_characters": 624460, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 160.39459188783175, + "max_sentence1_length": 559, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 152.30445668502753, + "max_sentence2_length": 540, + "unique_sentence2": 1996 + }, + "mlg_Latn-mri_Latn": { + "num_samples": 1997, + "number_of_characters": 594432, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 160.39459188783175, + "max_sentence1_length": 559, + "unique_sentence1": 1994, + "min_sentence2_length": 9, + "average_sentence2_length": 137.26790185277918, + "max_sentence2_length": 443, + "unique_sentence2": 1997 + }, + "mlg_Latn-msa_Latn": { + "num_samples": 1997, + "number_of_characters": 597491, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 160.39459188783175, + "max_sentence1_length": 559, + "unique_sentence1": 1994, + "min_sentence2_length": 6, + "average_sentence2_length": 138.79969954932398, + "max_sentence2_length": 463, + "unique_sentence2": 1997 + }, + "mlg_Latn-smo_Latn": { + "num_samples": 1997, + "number_of_characters": 598163, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 160.39459188783175, + "max_sentence1_length": 559, + "unique_sentence1": 1994, + "min_sentence2_length": 8, + "average_sentence2_length": 139.1362043064597, + "max_sentence2_length": 431, + "unique_sentence2": 1996 + }, + "mlg_Latn-tah_Latn": { + "num_samples": 1997, + "number_of_characters": 629931, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 160.39459188783175, + "max_sentence1_length": 559, + "unique_sentence1": 1994, + "min_sentence2_length": 9, + "average_sentence2_length": 155.04406609914872, + "max_sentence2_length": 524, + "unique_sentence2": 1997 + }, + "mlg_Latn-ton_Latn": { + "num_samples": 1997, + "number_of_characters": 633948, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 160.39459188783175, + "max_sentence1_length": 559, + "unique_sentence1": 1994, + "min_sentence2_length": 7, + "average_sentence2_length": 157.05558337506258, + "max_sentence2_length": 468, + "unique_sentence2": 1997 + }, + "mlt_Latn-cat_Latn": { + "num_samples": 1997, + "number_of_characters": 560435, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 138.94591887831749, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 141.6925388082123, + "max_sentence2_length": 460, + "unique_sentence2": 1997 + }, + "mlt_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 525195, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 138.94591887831749, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "mlt_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 570583, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 138.94591887831749, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "mlt_Latn-glg_Latn": { + "num_samples": 1997, + "number_of_characters": 549461, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 138.94591887831749, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 136.19729594391586, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "mlt_Latn-ita_Latn": { + "num_samples": 1997, + "number_of_characters": 566692, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 138.94591887831749, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 144.82573860791186, + "max_sentence2_length": 623, + "unique_sentence2": 1996 + }, + "mlt_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 554690, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 138.94591887831749, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "mlt_Latn-ron_Latn": { + "num_samples": 1997, + "number_of_characters": 569960, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 138.94591887831749, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 146.4621932899349, + "max_sentence2_length": 518, + "unique_sentence2": 1997 + }, + "mlt_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 565675, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 138.94591887831749, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "mon_Mong-bod_Tibt": { + "num_samples": 1997, + "number_of_characters": 559677, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 129.7185778668002, + "max_sentence1_length": 414, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 150.54031046569855, + "max_sentence2_length": 478, + "unique_sentence2": 1993 + }, + "mon_Mong-dzo_Tibt": { + "num_samples": 1997, + "number_of_characters": 502269, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 129.7185778668002, + "max_sentence1_length": 414, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 121.79318978467701, + "max_sentence2_length": 411, + "unique_sentence2": 1992 + }, + "mon_Mong-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 506768, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 129.7185778668002, + "max_sentence1_length": 414, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "mon_Mong-khm_Khmr": { + "num_samples": 1997, + "number_of_characters": 547539, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 129.7185778668002, + "max_sentence1_length": 414, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 144.4621932899349, + "max_sentence2_length": 517, + "unique_sentence2": 1996 + }, + "mon_Mong-lao_Laoo": { + "num_samples": 1997, + "number_of_characters": 526028, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 129.7185778668002, + "max_sentence1_length": 414, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 133.69053580370556, + "max_sentence2_length": 507, + "unique_sentence2": 1997 + }, + "mon_Mong-mya_Mymr": { + "num_samples": 1997, + "number_of_characters": 570902, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 129.7185778668002, + "max_sentence1_length": 414, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 156.16124186279418, + "max_sentence2_length": 773, + "unique_sentence2": 1997 + }, + "mon_Mong-tha_Thai": { + "num_samples": 1997, + "number_of_characters": 496516, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 129.7185778668002, + "max_sentence1_length": 414, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 118.91236855282925, + "max_sentence2_length": 439, + "unique_sentence2": 1996 + }, + "mri_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 521844, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 137.26790185277918, + "max_sentence1_length": 443, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "mri_Latn-fij_Latn": { + "num_samples": 1997, + "number_of_characters": 574629, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 137.26790185277918, + "max_sentence1_length": 443, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 150.478217325989, + "max_sentence2_length": 448, + "unique_sentence2": 1988 + }, + "mri_Latn-fil_Latn": { + "num_samples": 1997, + "number_of_characters": 567544, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 137.26790185277918, + "max_sentence1_length": 443, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 146.93039559339007, + "max_sentence2_length": 554, + "unique_sentence2": 1997 + }, + "mri_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 561096, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 137.26790185277918, + "max_sentence1_length": 443, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "mri_Latn-mal_Mlym": { + "num_samples": 1997, + "number_of_characters": 578276, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 137.26790185277918, + "max_sentence1_length": 443, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 152.30445668502753, + "max_sentence2_length": 540, + "unique_sentence2": 1996 + }, + "mri_Latn-mlg_Latn": { + "num_samples": 1997, + "number_of_characters": 594432, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 137.26790185277918, + "max_sentence1_length": 443, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 160.39459188783175, + "max_sentence2_length": 559, + "unique_sentence2": 1994 + }, + "mri_Latn-msa_Latn": { + "num_samples": 1997, + "number_of_characters": 551307, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 137.26790185277918, + "max_sentence1_length": 443, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 138.79969954932398, + "max_sentence2_length": 463, + "unique_sentence2": 1997 + }, + "mri_Latn-smo_Latn": { + "num_samples": 1997, + "number_of_characters": 551979, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 137.26790185277918, + "max_sentence1_length": 443, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 139.1362043064597, + "max_sentence2_length": 431, + "unique_sentence2": 1996 + }, + "mri_Latn-tah_Latn": { + "num_samples": 1997, + "number_of_characters": 583747, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 137.26790185277918, + "max_sentence1_length": 443, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 155.04406609914872, + "max_sentence2_length": 524, + "unique_sentence2": 1997 + }, + "mri_Latn-ton_Latn": { + "num_samples": 1997, + "number_of_characters": 587764, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 137.26790185277918, + "max_sentence1_length": 443, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 157.05558337506258, + "max_sentence2_length": 468, + "unique_sentence2": 1997 + }, + "msa_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 524903, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 138.79969954932398, + "max_sentence1_length": 463, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "msa_Latn-fij_Latn": { + "num_samples": 1997, + "number_of_characters": 577688, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 138.79969954932398, + "max_sentence1_length": 463, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 150.478217325989, + "max_sentence2_length": 448, + "unique_sentence2": 1988 + }, + "msa_Latn-fil_Latn": { + "num_samples": 1997, + "number_of_characters": 570603, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 138.79969954932398, + "max_sentence1_length": 463, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 146.93039559339007, + "max_sentence2_length": 554, + "unique_sentence2": 1997 + }, + "msa_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 564155, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 138.79969954932398, + "max_sentence1_length": 463, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "msa_Latn-mal_Mlym": { + "num_samples": 1997, + "number_of_characters": 581335, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 138.79969954932398, + "max_sentence1_length": 463, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 152.30445668502753, + "max_sentence2_length": 540, + "unique_sentence2": 1996 + }, + "msa_Latn-mlg_Latn": { + "num_samples": 1997, + "number_of_characters": 597491, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 138.79969954932398, + "max_sentence1_length": 463, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 160.39459188783175, + "max_sentence2_length": 559, + "unique_sentence2": 1994 + }, + "msa_Latn-mri_Latn": { + "num_samples": 1997, + "number_of_characters": 551307, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 138.79969954932398, + "max_sentence1_length": 463, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 137.26790185277918, + "max_sentence2_length": 443, + "unique_sentence2": 1997 + }, + "msa_Latn-smo_Latn": { + "num_samples": 1997, + "number_of_characters": 555038, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 138.79969954932398, + "max_sentence1_length": 463, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 139.1362043064597, + "max_sentence2_length": 431, + "unique_sentence2": 1996 + }, + "msa_Latn-tah_Latn": { + "num_samples": 1997, + "number_of_characters": 586806, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 138.79969954932398, + "max_sentence1_length": 463, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 155.04406609914872, + "max_sentence2_length": 524, + "unique_sentence2": 1997 + }, + "msa_Latn-ton_Latn": { + "num_samples": 1997, + "number_of_characters": 590823, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 138.79969954932398, + "max_sentence1_length": 463, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 157.05558337506258, + "max_sentence2_length": 468, + "unique_sentence2": 1997 + }, + "mya_Mymr-bod_Tibt": { + "num_samples": 1997, + "number_of_characters": 612483, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 156.16124186279418, + "max_sentence1_length": 773, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 150.54031046569855, + "max_sentence2_length": 478, + "unique_sentence2": 1993 + }, + "mya_Mymr-dzo_Tibt": { + "num_samples": 1997, + "number_of_characters": 555075, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 156.16124186279418, + "max_sentence1_length": 773, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 121.79318978467701, + "max_sentence2_length": 411, + "unique_sentence2": 1992 + }, + "mya_Mymr-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 559574, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 156.16124186279418, + "max_sentence1_length": 773, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "mya_Mymr-khm_Khmr": { + "num_samples": 1997, + "number_of_characters": 600345, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 156.16124186279418, + "max_sentence1_length": 773, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 144.4621932899349, + "max_sentence2_length": 517, + "unique_sentence2": 1996 + }, + "mya_Mymr-lao_Laoo": { + "num_samples": 1997, + "number_of_characters": 578834, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 156.16124186279418, + "max_sentence1_length": 773, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 133.69053580370556, + "max_sentence2_length": 507, + "unique_sentence2": 1997 + }, + "mya_Mymr-mon_Mong": { + "num_samples": 1997, + "number_of_characters": 570902, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 156.16124186279418, + "max_sentence1_length": 773, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 129.7185778668002, + "max_sentence2_length": 414, + "unique_sentence2": 1997 + }, + "mya_Mymr-tha_Thai": { + "num_samples": 1997, + "number_of_characters": 549322, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 156.16124186279418, + "max_sentence1_length": 773, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 118.91236855282925, + "max_sentence2_length": 439, + "unique_sentence2": 1996 + }, + "nde_Latn-bem_Latn": { + "num_samples": 1997, + "number_of_characters": 596231, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.09313970956435, + "max_sentence1_length": 590, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 149.47020530796195, + "max_sentence2_length": 465, + "unique_sentence2": 1997 + }, + "nde_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 545459, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.09313970956435, + "max_sentence1_length": 590, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "nde_Latn-ewe_Latn": { + "num_samples": 1997, + "number_of_characters": 536717, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.09313970956435, + "max_sentence1_length": 590, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 119.6685027541312, + "max_sentence2_length": 493, + "unique_sentence2": 1994 + }, + "nde_Latn-fuc_Latn": { + "num_samples": 1997, + "number_of_characters": 526219, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.09313970956435, + "max_sentence1_length": 590, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 114.4116174261392, + "max_sentence2_length": 376, + "unique_sentence2": 1996 + }, + "nde_Latn-kin_Latn": { + "num_samples": 1997, + "number_of_characters": 601526, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.09313970956435, + "max_sentence1_length": 590, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 152.12168252378567, + "max_sentence2_length": 541, + "unique_sentence2": 1996 + }, + "nde_Latn-nya_Latn": { + "num_samples": 1997, + "number_of_characters": 582021, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.09313970956435, + "max_sentence1_length": 590, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 142.35453179769655, + "max_sentence2_length": 464, + "unique_sentence2": 1993 + }, + "nde_Latn-sna_Latn": { + "num_samples": 1997, + "number_of_characters": 596069, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.09313970956435, + "max_sentence1_length": 590, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 149.38908362543816, + "max_sentence2_length": 511, + "unique_sentence2": 1995 + }, + "nde_Latn-ven_Latn": { + "num_samples": 1997, + "number_of_characters": 597495, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.09313970956435, + "max_sentence1_length": 590, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 150.10315473209815, + "max_sentence2_length": 535, + "unique_sentence2": 1993 + }, + "nep_Deva-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 492025, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "nep_Deva-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 552071, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "nep_Deva-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 495943, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "nep_Deva-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 523426, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "nep_Deva-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 493154, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "nep_Deva-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 509798, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "nep_Deva-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 513759, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "nep_Deva-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 509110, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "nep_Deva-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 498645, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "nep_Deva-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 506964, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "nep_Deva-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 468550, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "nep_Deva-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 558184, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "nep_Deva-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 495750, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "nep_Deva-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 496221, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 124.29794692038057, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "nld_Latn-afr_Latn": { + "num_samples": 1997, + "number_of_characters": 560267, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 134.37756634952427, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "nld_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 523096, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "nld_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 535717, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "nld_Latn-dan_Latn": { + "num_samples": 1997, + "number_of_characters": 544053, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 126.25838758137206, + "max_sentence2_length": 522, + "unique_sentence2": 1995 + }, + "nld_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 587565, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "nld_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 591042, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "nld_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 539635, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "nld_Latn-fao_Latn": { + "num_samples": 1997, + "number_of_characters": 549718, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0951427140711, + "max_sentence2_length": 433, + "unique_sentence2": 1997 + }, + "nld_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 535254, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "nld_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 561715, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "nld_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 585023, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "nld_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 492211, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "nld_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 553490, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "nld_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 570291, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "nld_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 578887, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "nld_Latn-isl_Latn": { + "num_samples": 1997, + "number_of_characters": 554123, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 131.30095142714072, + "max_sentence2_length": 399, + "unique_sentence2": 1996 + }, + "nld_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 403541, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "nld_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 425320, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "nld_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 551312, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "nld_Latn-ltz_Latn": { + "num_samples": 1997, + "number_of_characters": 572672, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 140.58938407611416, + "max_sentence2_length": 543, + "unique_sentence2": 1996 + }, + "nld_Latn-nno_Latn": { + "num_samples": 1997, + "number_of_characters": 540272, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.36504757135704, + "max_sentence2_length": 417, + "unique_sentence2": 1996 + }, + "nld_Latn-nob_Latn": { + "num_samples": 1997, + "number_of_characters": 543359, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.91086629944917, + "max_sentence2_length": 482, + "unique_sentence2": 1996 + }, + "nld_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 569781, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "nld_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 569130, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "nld_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 566808, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "nld_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 580115, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "nld_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 564424, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "nld_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 543742, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "nld_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 601876, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "nld_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 557528, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "nld_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 563036, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "nld_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 383393, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "nld_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 549647, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 146.1767651477216, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "nno_Latn-afr_Latn": { + "num_samples": 1997, + "number_of_characters": 516709, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.36504757135704, + "max_sentence1_length": 417, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 134.37756634952427, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "nno_Latn-dan_Latn": { + "num_samples": 1997, + "number_of_characters": 500495, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.36504757135704, + "max_sentence1_length": 417, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 126.25838758137206, + "max_sentence2_length": 522, + "unique_sentence2": 1995 + }, + "nno_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 544007, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.36504757135704, + "max_sentence1_length": 417, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "nno_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 496077, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.36504757135704, + "max_sentence1_length": 417, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "nno_Latn-fao_Latn": { + "num_samples": 1997, + "number_of_characters": 506160, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.36504757135704, + "max_sentence1_length": 417, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0951427140711, + "max_sentence2_length": 433, + "unique_sentence2": 1997 + }, + "nno_Latn-isl_Latn": { + "num_samples": 1997, + "number_of_characters": 510565, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.36504757135704, + "max_sentence1_length": 417, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 131.30095142714072, + "max_sentence2_length": 399, + "unique_sentence2": 1996 + }, + "nno_Latn-ltz_Latn": { + "num_samples": 1997, + "number_of_characters": 529114, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.36504757135704, + "max_sentence1_length": 417, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 140.58938407611416, + "max_sentence2_length": 543, + "unique_sentence2": 1996 + }, + "nno_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 540272, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.36504757135704, + "max_sentence1_length": 417, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "nno_Latn-nob_Latn": { + "num_samples": 1997, + "number_of_characters": 499801, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.36504757135704, + "max_sentence1_length": 417, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.91086629944917, + "max_sentence2_length": 482, + "unique_sentence2": 1996 + }, + "nno_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 500184, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.36504757135704, + "max_sentence1_length": 417, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "nob_Latn-afr_Latn": { + "num_samples": 1997, + "number_of_characters": 519796, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.91086629944917, + "max_sentence1_length": 482, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 134.37756634952427, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "nob_Latn-dan_Latn": { + "num_samples": 1997, + "number_of_characters": 503582, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.91086629944917, + "max_sentence1_length": 482, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 126.25838758137206, + "max_sentence2_length": 522, + "unique_sentence2": 1995 + }, + "nob_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 547094, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.91086629944917, + "max_sentence1_length": 482, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "nob_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 499164, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 125.91086629944917, + "max_sentence1_length": 482, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "nob_Latn-fao_Latn": { + "num_samples": 1997, + "number_of_characters": 509247, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 125.91086629944917, + "max_sentence1_length": 482, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0951427140711, + "max_sentence2_length": 433, + "unique_sentence2": 1997 + }, + "nob_Latn-isl_Latn": { + "num_samples": 1997, + "number_of_characters": 513652, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.91086629944917, + "max_sentence1_length": 482, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 131.30095142714072, + "max_sentence2_length": 399, + "unique_sentence2": 1996 + }, + "nob_Latn-ltz_Latn": { + "num_samples": 1997, + "number_of_characters": 532201, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.91086629944917, + "max_sentence1_length": 482, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 140.58938407611416, + "max_sentence2_length": 543, + "unique_sentence2": 1996 + }, + "nob_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 543359, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.91086629944917, + "max_sentence1_length": 482, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "nob_Latn-nno_Latn": { + "num_samples": 1997, + "number_of_characters": 499801, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.91086629944917, + "max_sentence1_length": 482, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.36504757135704, + "max_sentence2_length": 417, + "unique_sentence2": 1996 + }, + "nob_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 503271, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 125.91086629944917, + "max_sentence1_length": 482, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "nso_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 459006, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "nso_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 539219, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "nso_Latn-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 561465, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "nso_Latn-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 537600, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "nso_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 528930, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "nso_Latn-som_Latn": { + "num_samples": 1997, + "number_of_characters": 582791, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "nso_Latn-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 579641, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "nso_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 564008, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "nso_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 456737, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "nso_Latn-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 625782, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "nso_Latn-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 531302, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "nso_Latn-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 559589, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "nso_Latn-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 607587, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "nso_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 549231, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 145.96845267901853, + "max_sentence1_length": 487, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "nya_Latn-bem_Latn": { + "num_samples": 1997, + "number_of_characters": 582774, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 142.35453179769655, + "max_sentence1_length": 464, + "unique_sentence1": 1993, + "min_sentence2_length": 8, + "average_sentence2_length": 149.47020530796195, + "max_sentence2_length": 465, + "unique_sentence2": 1997 + }, + "nya_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 532002, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 142.35453179769655, + "max_sentence1_length": 464, + "unique_sentence1": 1993, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "nya_Latn-ewe_Latn": { + "num_samples": 1997, + "number_of_characters": 523260, + "unique_pairs": 1995, + "min_sentence1_length": 10, + "average_sentence1_length": 142.35453179769655, + "max_sentence1_length": 464, + "unique_sentence1": 1993, + "min_sentence2_length": 6, + "average_sentence2_length": 119.6685027541312, + "max_sentence2_length": 493, + "unique_sentence2": 1994 + }, + "nya_Latn-fuc_Latn": { + "num_samples": 1997, + "number_of_characters": 512762, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 142.35453179769655, + "max_sentence1_length": 464, + "unique_sentence1": 1993, + "min_sentence2_length": 6, + "average_sentence2_length": 114.4116174261392, + "max_sentence2_length": 376, + "unique_sentence2": 1996 + }, + "nya_Latn-kin_Latn": { + "num_samples": 1997, + "number_of_characters": 588069, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 142.35453179769655, + "max_sentence1_length": 464, + "unique_sentence1": 1993, + "min_sentence2_length": 10, + "average_sentence2_length": 152.12168252378567, + "max_sentence2_length": 541, + "unique_sentence2": 1996 + }, + "nya_Latn-nde_Latn": { + "num_samples": 1997, + "number_of_characters": 582021, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 142.35453179769655, + "max_sentence1_length": 464, + "unique_sentence1": 1993, + "min_sentence2_length": 6, + "average_sentence2_length": 149.09313970956435, + "max_sentence2_length": 590, + "unique_sentence2": 1997 + }, + "nya_Latn-sna_Latn": { + "num_samples": 1997, + "number_of_characters": 582612, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 142.35453179769655, + "max_sentence1_length": 464, + "unique_sentence1": 1993, + "min_sentence2_length": 6, + "average_sentence2_length": 149.38908362543816, + "max_sentence2_length": 511, + "unique_sentence2": 1995 + }, + "nya_Latn-ven_Latn": { + "num_samples": 1997, + "number_of_characters": 584038, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 142.35453179769655, + "max_sentence1_length": 464, + "unique_sentence1": 1993, + "min_sentence2_length": 10, + "average_sentence2_length": 150.10315473209815, + "max_sentence2_length": 535, + "unique_sentence2": 1993 + }, + "orm_Ethi-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 404938, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "orm_Ethi-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 485151, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "orm_Ethi-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 507397, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "orm_Ethi-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 483532, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "orm_Ethi-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 528930, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "orm_Ethi-som_Latn": { + "num_samples": 1997, + "number_of_characters": 528723, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "orm_Ethi-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 525573, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "orm_Ethi-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 509940, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "orm_Ethi-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 402669, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "orm_Ethi-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 571714, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "orm_Ethi-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 477234, + "unique_pairs": 1992, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "orm_Ethi-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 505521, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "orm_Ethi-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 553519, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "orm_Ethi-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 495163, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 118.89384076114172, + "max_sentence1_length": 466, + "unique_sentence1": 1984, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "pan_Guru-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 494224, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "pan_Guru-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 554270, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "pan_Guru-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 498142, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "pan_Guru-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 525625, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "pan_Guru-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 495353, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "pan_Guru-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 511997, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "pan_Guru-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 515958, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "pan_Guru-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 511309, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "pan_Guru-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 498645, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "pan_Guru-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 509163, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "pan_Guru-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 470749, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "pan_Guru-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 560383, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "pan_Guru-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 497949, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "pan_Guru-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 498420, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 125.39909864797195, + "max_sentence1_length": 383, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "pol_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 509047, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "pol_Latn-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 533956, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "pol_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 521668, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "pol_Latn-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 532776, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "pol_Latn-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 547755, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "pol_Latn-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 519184, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "pol_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 573516, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "pol_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 576993, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "pol_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 525586, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "pol_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 521205, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "pol_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 547666, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "pol_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 570974, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "pol_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 478162, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "pol_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 539441, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "pol_Latn-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 533791, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "pol_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 556242, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "pol_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 564838, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "pol_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 389492, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "pol_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 411271, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "pol_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 537263, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "pol_Latn-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 545757, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "pol_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 569781, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "pol_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 555081, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "pol_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 552759, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "pol_Latn-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 530835, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "pol_Latn-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 530762, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "pol_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 566066, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "pol_Latn-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 530169, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "pol_Latn-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 534007, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "pol_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 550375, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "pol_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 529693, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "pol_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 587827, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "pol_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 543479, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "pol_Latn-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 540649, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "pol_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 548987, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "pol_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 369344, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "pol_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 535598, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 139.14171256885328, + "max_sentence1_length": 468, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "por_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 508396, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "por_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 521017, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "por_Latn-cat_Latn": { + "num_samples": 1997, + "number_of_characters": 560175, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 141.6925388082123, + "max_sentence2_length": 460, + "unique_sentence2": 1997 + }, + "por_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 572865, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "por_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 576342, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "por_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 524935, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "por_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 520554, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "por_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 547015, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "por_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 570323, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "por_Latn-glg_Latn": { + "num_samples": 1997, + "number_of_characters": 549201, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 136.19729594391586, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "por_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 477511, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "por_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 538790, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "por_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 555591, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "por_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 564187, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "por_Latn-ita_Latn": { + "num_samples": 1997, + "number_of_characters": 566432, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 144.82573860791186, + "max_sentence2_length": 623, + "unique_sentence2": 1996 + }, + "por_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 388841, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "por_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 410620, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "por_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 536612, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "por_Latn-mlt_Latn": { + "num_samples": 1997, + "number_of_characters": 554690, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 138.94591887831749, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "por_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 569130, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "por_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 555081, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "por_Latn-ron_Latn": { + "num_samples": 1997, + "number_of_characters": 569700, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 146.4621932899349, + "max_sentence2_length": 518, + "unique_sentence2": 1997 + }, + "por_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 552108, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "por_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 565415, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "por_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 549724, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "por_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 529042, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "por_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 587176, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "por_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 542828, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "por_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 548336, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "por_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 368693, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "por_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 534947, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 138.81572358537807, + "max_sentence1_length": 497, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "prs_Arab-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 473717, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.45017526289435, + "max_sentence1_length": 365, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "prs_Arab-ckb_Arab": { + "num_samples": 1997, + "number_of_characters": 494903, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.45017526289435, + "max_sentence1_length": 365, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 126.37305958938407, + "max_sentence2_length": 399, + "unique_sentence2": 1995 + }, + "prs_Arab-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 490256, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.45017526289435, + "max_sentence1_length": 365, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "prs_Arab-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 485875, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.45017526289435, + "max_sentence1_length": 365, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "prs_Arab-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 442832, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.45017526289435, + "max_sentence1_length": 365, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "prs_Arab-kmr_Latn": { + "num_samples": 1997, + "number_of_characters": 488482, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.45017526289435, + "max_sentence1_length": 365, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 123.15773660490736, + "max_sentence2_length": 420, + "unique_sentence2": 1996 + }, + "prs_Arab-mey_Arab": { + "num_samples": 1997, + "number_of_characters": 456371, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.45017526289435, + "max_sentence1_length": 365, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 107.07811717576365, + "max_sentence2_length": 392, + "unique_sentence2": 1993 + }, + "prs_Arab-pus_Arab": { + "num_samples": 1997, + "number_of_characters": 485169, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.45017526289435, + "max_sentence1_length": 365, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 121.49874812218327, + "max_sentence2_length": 366, + "unique_sentence2": 1996 + }, + "prs_Arab-shi_Arab": { + "num_samples": 1997, + "number_of_characters": 457449, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.45017526289435, + "max_sentence1_length": 365, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 107.6179268903355, + "max_sentence2_length": 378, + "unique_sentence2": 1996 + }, + "prs_Arab-tgk_Cyrl": { + "num_samples": 1997, + "number_of_characters": 516683, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.45017526289435, + "max_sentence1_length": 365, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 137.27941912869304, + "max_sentence2_length": 451, + "unique_sentence2": 1995 + }, + "pus_Arab-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 473814, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 121.49874812218327, + "max_sentence1_length": 366, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "pus_Arab-ckb_Arab": { + "num_samples": 1997, + "number_of_characters": 495000, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 121.49874812218327, + "max_sentence1_length": 366, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 126.37305958938407, + "max_sentence2_length": 399, + "unique_sentence2": 1995 + }, + "pus_Arab-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 490353, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.49874812218327, + "max_sentence1_length": 366, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "pus_Arab-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 485972, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 121.49874812218327, + "max_sentence1_length": 366, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "pus_Arab-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 442929, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 121.49874812218327, + "max_sentence1_length": 366, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "pus_Arab-kmr_Latn": { + "num_samples": 1997, + "number_of_characters": 488579, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 121.49874812218327, + "max_sentence1_length": 366, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 123.15773660490736, + "max_sentence2_length": 420, + "unique_sentence2": 1996 + }, + "pus_Arab-mey_Arab": { + "num_samples": 1997, + "number_of_characters": 456468, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.49874812218327, + "max_sentence1_length": 366, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 107.07811717576365, + "max_sentence2_length": 392, + "unique_sentence2": 1993 + }, + "pus_Arab-prs_Arab": { + "num_samples": 1997, + "number_of_characters": 485169, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.49874812218327, + "max_sentence1_length": 366, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 121.45017526289435, + "max_sentence2_length": 365, + "unique_sentence2": 1997 + }, + "pus_Arab-shi_Arab": { + "num_samples": 1997, + "number_of_characters": 457546, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 121.49874812218327, + "max_sentence1_length": 366, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 107.6179268903355, + "max_sentence2_length": 378, + "unique_sentence2": 1996 + }, + "pus_Arab-tgk_Cyrl": { + "num_samples": 1997, + "number_of_characters": 516780, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 121.49874812218327, + "max_sentence1_length": 366, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 137.27941912869304, + "max_sentence2_length": 451, + "unique_sentence2": 1995 + }, + "ron_Latn-cat_Latn": { + "num_samples": 1997, + "number_of_characters": 575445, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 146.4621932899349, + "max_sentence1_length": 518, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 141.6925388082123, + "max_sentence2_length": 460, + "unique_sentence2": 1997 + }, + "ron_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 540205, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 146.4621932899349, + "max_sentence1_length": 518, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ron_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 585593, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 146.4621932899349, + "max_sentence1_length": 518, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "ron_Latn-glg_Latn": { + "num_samples": 1997, + "number_of_characters": 564471, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 146.4621932899349, + "max_sentence1_length": 518, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 136.19729594391586, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "ron_Latn-ita_Latn": { + "num_samples": 1997, + "number_of_characters": 581702, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 146.4621932899349, + "max_sentence1_length": 518, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 144.82573860791186, + "max_sentence2_length": 623, + "unique_sentence2": 1996 + }, + "ron_Latn-mlt_Latn": { + "num_samples": 1997, + "number_of_characters": 569960, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 146.4621932899349, + "max_sentence1_length": 518, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 138.94591887831749, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "ron_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 569700, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 146.4621932899349, + "max_sentence1_length": 518, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "ron_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 580685, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 146.4621932899349, + "max_sentence1_length": 518, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "rus_Cyrl-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 506074, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "rus_Cyrl-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 530983, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "rus_Cyrl-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 518695, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "rus_Cyrl-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 529803, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "rus_Cyrl-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 544782, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "rus_Cyrl-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 516211, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "rus_Cyrl-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 570543, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "rus_Cyrl-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 574020, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "rus_Cyrl-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 522613, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "rus_Cyrl-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 518232, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "rus_Cyrl-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 544693, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "rus_Cyrl-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 568001, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "rus_Cyrl-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 475189, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "rus_Cyrl-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 536468, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "rus_Cyrl-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 530818, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "rus_Cyrl-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 553269, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "rus_Cyrl-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 561865, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "rus_Cyrl-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 386519, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "rus_Cyrl-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 408298, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "rus_Cyrl-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 534290, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "rus_Cyrl-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 542784, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "rus_Cyrl-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 566808, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "rus_Cyrl-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 552759, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "rus_Cyrl-por_Latn": { + "num_samples": 1997, + "number_of_characters": 552108, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "rus_Cyrl-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 527862, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "rus_Cyrl-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 527789, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "rus_Cyrl-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 563093, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "rus_Cyrl-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 527196, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "rus_Cyrl-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 531034, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "rus_Cyrl-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 547402, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "rus_Cyrl-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 526720, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "rus_Cyrl-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 584854, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "rus_Cyrl-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 540506, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "rus_Cyrl-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 537676, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "rus_Cyrl-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 546014, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "rus_Cyrl-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 366371, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "rus_Cyrl-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 532625, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 137.6529794692038, + "max_sentence1_length": 419, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "shi_Arab-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 446094, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 107.6179268903355, + "max_sentence1_length": 378, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "shi_Arab-ckb_Arab": { + "num_samples": 1997, + "number_of_characters": 467280, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 107.6179268903355, + "max_sentence1_length": 378, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 126.37305958938407, + "max_sentence2_length": 399, + "unique_sentence2": 1995 + }, + "shi_Arab-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 462633, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 107.6179268903355, + "max_sentence1_length": 378, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "shi_Arab-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 458252, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 107.6179268903355, + "max_sentence1_length": 378, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "shi_Arab-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 415209, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 107.6179268903355, + "max_sentence1_length": 378, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "shi_Arab-kmr_Latn": { + "num_samples": 1997, + "number_of_characters": 460859, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 107.6179268903355, + "max_sentence1_length": 378, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 123.15773660490736, + "max_sentence2_length": 420, + "unique_sentence2": 1996 + }, + "shi_Arab-mey_Arab": { + "num_samples": 1997, + "number_of_characters": 428748, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 107.6179268903355, + "max_sentence1_length": 378, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 107.07811717576365, + "max_sentence2_length": 392, + "unique_sentence2": 1993 + }, + "shi_Arab-prs_Arab": { + "num_samples": 1997, + "number_of_characters": 457449, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 107.6179268903355, + "max_sentence1_length": 378, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 121.45017526289435, + "max_sentence2_length": 365, + "unique_sentence2": 1997 + }, + "shi_Arab-pus_Arab": { + "num_samples": 1997, + "number_of_characters": 457546, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 107.6179268903355, + "max_sentence1_length": 378, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 121.49874812218327, + "max_sentence2_length": 366, + "unique_sentence2": 1996 + }, + "shi_Arab-tgk_Cyrl": { + "num_samples": 1997, + "number_of_characters": 489060, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 107.6179268903355, + "max_sentence1_length": 378, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 137.27941912869304, + "max_sentence2_length": 451, + "unique_sentence2": 1995 + }, + "sin_Sinh-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 502543, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "sin_Sinh-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 562589, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "sin_Sinh-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 506461, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "sin_Sinh-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 533944, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "sin_Sinh-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 503672, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "sin_Sinh-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 520316, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "sin_Sinh-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 524277, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "sin_Sinh-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 519628, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "sin_Sinh-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 506964, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "sin_Sinh-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 509163, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "sin_Sinh-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 479068, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "sin_Sinh-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 568702, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "sin_Sinh-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 506268, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "sin_Sinh-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 506739, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 129.56484727090637, + "max_sentence1_length": 441, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "slk_Latn-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 509059, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "slk_Latn-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 507879, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "slk_Latn-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 522858, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "slk_Latn-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 494287, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "slk_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 500689, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "slk_Latn-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 508894, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "slk_Latn-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 520860, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "slk_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 530835, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "slk_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 527862, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "slk_Latn-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 505865, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "slk_Latn-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 505272, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "slk_Latn-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 509110, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "slk_Latn-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 515752, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 126.67451176765148, + "max_sentence1_length": 403, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "slv_Latn-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 508986, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "slv_Latn-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 507806, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "slv_Latn-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 522785, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "slv_Latn-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 494214, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "slv_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 500616, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "slv_Latn-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 508821, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "slv_Latn-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 520787, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "slv_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 530762, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "slv_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 527789, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "slv_Latn-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 505865, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "slv_Latn-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 505199, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "slv_Latn-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 509037, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "slv_Latn-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 515679, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.63795693540311, + "max_sentence1_length": 463, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "smo_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 525575, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 139.1362043064597, + "max_sentence1_length": 431, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "smo_Latn-fij_Latn": { + "num_samples": 1997, + "number_of_characters": 578360, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 139.1362043064597, + "max_sentence1_length": 431, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 150.478217325989, + "max_sentence2_length": 448, + "unique_sentence2": 1988 + }, + "smo_Latn-fil_Latn": { + "num_samples": 1997, + "number_of_characters": 571275, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 139.1362043064597, + "max_sentence1_length": 431, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 146.93039559339007, + "max_sentence2_length": 554, + "unique_sentence2": 1997 + }, + "smo_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 564827, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 139.1362043064597, + "max_sentence1_length": 431, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "smo_Latn-mal_Mlym": { + "num_samples": 1997, + "number_of_characters": 582007, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 139.1362043064597, + "max_sentence1_length": 431, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 152.30445668502753, + "max_sentence2_length": 540, + "unique_sentence2": 1996 + }, + "smo_Latn-mlg_Latn": { + "num_samples": 1997, + "number_of_characters": 598163, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 139.1362043064597, + "max_sentence1_length": 431, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 160.39459188783175, + "max_sentence2_length": 559, + "unique_sentence2": 1994 + }, + "smo_Latn-mri_Latn": { + "num_samples": 1997, + "number_of_characters": 551979, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 139.1362043064597, + "max_sentence1_length": 431, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 137.26790185277918, + "max_sentence2_length": 443, + "unique_sentence2": 1997 + }, + "smo_Latn-msa_Latn": { + "num_samples": 1997, + "number_of_characters": 555038, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 139.1362043064597, + "max_sentence1_length": 431, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 138.79969954932398, + "max_sentence2_length": 463, + "unique_sentence2": 1997 + }, + "smo_Latn-tah_Latn": { + "num_samples": 1997, + "number_of_characters": 587478, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 139.1362043064597, + "max_sentence1_length": 431, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 155.04406609914872, + "max_sentence2_length": 524, + "unique_sentence2": 1997 + }, + "smo_Latn-ton_Latn": { + "num_samples": 1997, + "number_of_characters": 591495, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 139.1362043064597, + "max_sentence1_length": 431, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 157.05558337506258, + "max_sentence2_length": 468, + "unique_sentence2": 1997 + }, + "sna_Latn-bem_Latn": { + "num_samples": 1997, + "number_of_characters": 596822, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.38908362543816, + "max_sentence1_length": 511, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 149.47020530796195, + "max_sentence2_length": 465, + "unique_sentence2": 1997 + }, + "sna_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 546050, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.38908362543816, + "max_sentence1_length": 511, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "sna_Latn-ewe_Latn": { + "num_samples": 1997, + "number_of_characters": 537308, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 149.38908362543816, + "max_sentence1_length": 511, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 119.6685027541312, + "max_sentence2_length": 493, + "unique_sentence2": 1994 + }, + "sna_Latn-fuc_Latn": { + "num_samples": 1997, + "number_of_characters": 526810, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.38908362543816, + "max_sentence1_length": 511, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 114.4116174261392, + "max_sentence2_length": 376, + "unique_sentence2": 1996 + }, + "sna_Latn-kin_Latn": { + "num_samples": 1997, + "number_of_characters": 602117, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.38908362543816, + "max_sentence1_length": 511, + "unique_sentence1": 1995, + "min_sentence2_length": 10, + "average_sentence2_length": 152.12168252378567, + "max_sentence2_length": 541, + "unique_sentence2": 1996 + }, + "sna_Latn-nde_Latn": { + "num_samples": 1997, + "number_of_characters": 596069, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 149.38908362543816, + "max_sentence1_length": 511, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 149.09313970956435, + "max_sentence2_length": 590, + "unique_sentence2": 1997 + }, + "sna_Latn-nya_Latn": { + "num_samples": 1997, + "number_of_characters": 582612, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 149.38908362543816, + "max_sentence1_length": 511, + "unique_sentence1": 1995, + "min_sentence2_length": 10, + "average_sentence2_length": 142.35453179769655, + "max_sentence2_length": 464, + "unique_sentence2": 1993 + }, + "sna_Latn-ven_Latn": { + "num_samples": 1997, + "number_of_characters": 598086, + "unique_pairs": 1995, + "min_sentence1_length": 6, + "average_sentence1_length": 149.38908362543816, + "max_sentence1_length": 511, + "unique_sentence1": 1995, + "min_sentence2_length": 10, + "average_sentence2_length": 150.10315473209815, + "max_sentence2_length": 535, + "unique_sentence2": 1993 + }, + "snd_Arab-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 464129, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "snd_Arab-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 524175, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "snd_Arab-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 468047, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "snd_Arab-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 495530, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "snd_Arab-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 465258, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "snd_Arab-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 481902, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "snd_Arab-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 485863, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "snd_Arab-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 481214, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "snd_Arab-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 468550, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "snd_Arab-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 470749, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "snd_Arab-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 479068, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "snd_Arab-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 530288, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "snd_Arab-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 467854, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "snd_Arab-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 468325, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 110.32899349023535, + "max_sentence1_length": 335, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "som_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 458799, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "som_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 539012, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "som_Latn-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 561258, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "som_Latn-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 537393, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "som_Latn-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 582791, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "som_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 528723, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "som_Latn-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 579434, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "som_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 563801, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "som_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 456530, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "som_Latn-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 625575, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "som_Latn-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 531095, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "som_Latn-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 559382, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "som_Latn-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 607380, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "som_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 549024, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 145.8647971957937, + "max_sentence1_length": 455, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "spa_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 519381, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "spa_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 532002, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "spa_Latn-cat_Latn": { + "num_samples": 1997, + "number_of_characters": 571160, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 141.6925388082123, + "max_sentence2_length": 460, + "unique_sentence2": 1997 + }, + "spa_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 583850, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "spa_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 587327, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "spa_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 535920, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "spa_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 531539, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "spa_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 558000, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "spa_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 581308, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "spa_Latn-glg_Latn": { + "num_samples": 1997, + "number_of_characters": 560186, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 136.19729594391586, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "spa_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 488496, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "spa_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 549775, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "spa_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 566576, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "spa_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 575172, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "spa_Latn-ita_Latn": { + "num_samples": 1997, + "number_of_characters": 577417, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 144.82573860791186, + "max_sentence2_length": 623, + "unique_sentence2": 1996 + }, + "spa_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 399826, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "spa_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 421605, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "spa_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 547597, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "spa_Latn-mlt_Latn": { + "num_samples": 1997, + "number_of_characters": 565675, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 138.94591887831749, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "spa_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 580115, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "spa_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 566066, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "spa_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 565415, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "spa_Latn-ron_Latn": { + "num_samples": 1997, + "number_of_characters": 580685, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 146.4621932899349, + "max_sentence2_length": 518, + "unique_sentence2": 1997 + }, + "spa_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 563093, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "spa_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 560709, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "spa_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 540027, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "spa_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 598161, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "spa_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 553813, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "spa_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 559321, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "spa_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 379678, + "unique_pairs": 1996, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "spa_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 545932, + "unique_pairs": 1997, + "min_sentence1_length": 1, + "average_sentence1_length": 144.3164747120681, + "max_sentence1_length": 504, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "sqi_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 582734, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 142.01652478718077, + "max_sentence1_length": 461, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "sqi_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 531327, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 142.01652478718077, + "max_sentence1_length": 461, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "sqi_Latn-hye_Armn": { + "num_samples": 1997, + "number_of_characters": 548322, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 142.01652478718077, + "max_sentence1_length": 461, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 132.55633450175262, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "sqi_Latn-kat_Geor": { + "num_samples": 1997, + "number_of_characters": 550199, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 142.01652478718077, + "max_sentence1_length": 461, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 133.49624436654983, + "max_sentence2_length": 503, + "unique_sentence2": 1995 + }, + "srp_Cyrl-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 508393, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "srp_Cyrl-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 507213, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "srp_Cyrl-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 522192, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "srp_Cyrl-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 493621, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "srp_Cyrl-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 500023, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "srp_Cyrl-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 508228, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "srp_Cyrl-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 520194, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "srp_Cyrl-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 530169, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "srp_Cyrl-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 527196, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "srp_Cyrl-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 505272, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "srp_Cyrl-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 505199, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "srp_Cyrl-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 508444, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "srp_Cyrl-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 515086, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 126.34101151727592, + "max_sentence1_length": 439, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "srp_Latn-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 512231, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "srp_Latn-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 511051, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "srp_Latn-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 526030, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "srp_Latn-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 497459, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "srp_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 503861, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "srp_Latn-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 512066, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "srp_Latn-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 524032, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "srp_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 534007, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "srp_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 531034, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "srp_Latn-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 509110, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "srp_Latn-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 509037, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "srp_Latn-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 508444, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "srp_Latn-ukr_Cyrl": { + "num_samples": 1997, + "number_of_characters": 518924, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 128.26289434151226, + "max_sentence1_length": 452, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 131.58888332498748, + "max_sentence2_length": 440, + "unique_sentence2": 1996 + }, + "ssw_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 455649, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "ssw_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 535862, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ssw_Latn-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 558108, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "ssw_Latn-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 534243, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "ssw_Latn-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 579641, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "ssw_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 525573, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "ssw_Latn-som_Latn": { + "num_samples": 1997, + "number_of_characters": 579434, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "ssw_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 560651, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "ssw_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 453380, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "ssw_Latn-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 622425, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "ssw_Latn-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 527945, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "ssw_Latn-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 556232, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "ssw_Latn-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 604230, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "ssw_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 545874, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 144.28743114672008, + "max_sentence1_length": 510, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "swa_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 440016, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "swa_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 503690, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "swa_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 516311, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "swa_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 568159, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "swa_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 571636, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "swa_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 520229, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "swa_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 515848, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "swa_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 542309, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "swa_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 565617, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "swa_Latn-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 542475, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "swa_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 472805, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "swa_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 534084, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "swa_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 550885, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "swa_Latn-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 518610, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "swa_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 559481, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "swa_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 384135, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "swa_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 405914, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "swa_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 531906, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "swa_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 564424, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "swa_Latn-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 564008, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "swa_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 509940, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "swa_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 550375, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "swa_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 549724, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "swa_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 547402, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "swa_Latn-som_Latn": { + "num_samples": 1997, + "number_of_characters": 563801, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "swa_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 560709, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "swa_Latn-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 560651, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "swa_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 524336, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "swa_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 582470, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "swa_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 437747, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "swa_Latn-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 606792, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "swa_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 538122, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "swa_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 543630, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "swa_Latn-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 512312, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "swa_Latn-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 540599, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "swa_Latn-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 588597, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "swa_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 363987, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "swa_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 530241, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 136.45918878317477, + "max_sentence1_length": 430, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "swe_Latn-afr_Latn": { + "num_samples": 1997, + "number_of_characters": 520179, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 134.37756634952427, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "swe_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 483008, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "swe_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 495629, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "swe_Latn-dan_Latn": { + "num_samples": 1997, + "number_of_characters": 503965, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 126.25838758137206, + "max_sentence2_length": 522, + "unique_sentence2": 1995 + }, + "swe_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 547477, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "swe_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 550954, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "swe_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 499547, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "swe_Latn-fao_Latn": { + "num_samples": 1997, + "number_of_characters": 509630, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0951427140711, + "max_sentence2_length": 433, + "unique_sentence2": 1997 + }, + "swe_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 495166, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "swe_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 521627, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "swe_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 544935, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "swe_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 452123, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "swe_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 513402, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "swe_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 530203, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "swe_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 538799, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "swe_Latn-isl_Latn": { + "num_samples": 1997, + "number_of_characters": 514035, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 131.30095142714072, + "max_sentence2_length": 399, + "unique_sentence2": 1996 + }, + "swe_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 363453, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "swe_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 385232, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "swe_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 511224, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "swe_Latn-ltz_Latn": { + "num_samples": 1997, + "number_of_characters": 532584, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 140.58938407611416, + "max_sentence2_length": 543, + "unique_sentence2": 1996 + }, + "swe_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 543742, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "swe_Latn-nno_Latn": { + "num_samples": 1997, + "number_of_characters": 500184, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.36504757135704, + "max_sentence2_length": 417, + "unique_sentence2": 1996 + }, + "swe_Latn-nob_Latn": { + "num_samples": 1997, + "number_of_characters": 503271, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.91086629944917, + "max_sentence2_length": 482, + "unique_sentence2": 1996 + }, + "swe_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 529693, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "swe_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 529042, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "swe_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 526720, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "swe_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 540027, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "swe_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 524336, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "swe_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 561788, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "swe_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 517440, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "swe_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 522948, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "swe_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 343305, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "swe_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 509559, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 126.10265398097145, + "max_sentence1_length": 430, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "tah_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 557343, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 155.04406609914872, + "max_sentence1_length": 524, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "tah_Latn-fij_Latn": { + "num_samples": 1997, + "number_of_characters": 610128, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 155.04406609914872, + "max_sentence1_length": 524, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 150.478217325989, + "max_sentence2_length": 448, + "unique_sentence2": 1988 + }, + "tah_Latn-fil_Latn": { + "num_samples": 1997, + "number_of_characters": 603043, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 155.04406609914872, + "max_sentence1_length": 524, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 146.93039559339007, + "max_sentence2_length": 554, + "unique_sentence2": 1997 + }, + "tah_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 596595, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 155.04406609914872, + "max_sentence1_length": 524, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "tah_Latn-mal_Mlym": { + "num_samples": 1997, + "number_of_characters": 613775, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 155.04406609914872, + "max_sentence1_length": 524, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 152.30445668502753, + "max_sentence2_length": 540, + "unique_sentence2": 1996 + }, + "tah_Latn-mlg_Latn": { + "num_samples": 1997, + "number_of_characters": 629931, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 155.04406609914872, + "max_sentence1_length": 524, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 160.39459188783175, + "max_sentence2_length": 559, + "unique_sentence2": 1994 + }, + "tah_Latn-mri_Latn": { + "num_samples": 1997, + "number_of_characters": 583747, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 155.04406609914872, + "max_sentence1_length": 524, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 137.26790185277918, + "max_sentence2_length": 443, + "unique_sentence2": 1997 + }, + "tah_Latn-msa_Latn": { + "num_samples": 1997, + "number_of_characters": 586806, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 155.04406609914872, + "max_sentence1_length": 524, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 138.79969954932398, + "max_sentence2_length": 463, + "unique_sentence2": 1997 + }, + "tah_Latn-smo_Latn": { + "num_samples": 1997, + "number_of_characters": 587478, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 155.04406609914872, + "max_sentence1_length": 524, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 139.1362043064597, + "max_sentence2_length": 431, + "unique_sentence2": 1996 + }, + "tah_Latn-ton_Latn": { + "num_samples": 1997, + "number_of_characters": 623263, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 155.04406609914872, + "max_sentence1_length": 524, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 157.05558337506258, + "max_sentence2_length": 468, + "unique_sentence2": 1997 + }, + "tam_Taml-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 541142, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "tam_Taml-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 553763, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "tam_Taml-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 605611, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "tam_Taml-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 613809, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "tam_Taml-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 609088, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "tam_Taml-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 557681, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "tam_Taml-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 585164, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "tam_Taml-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 553300, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "tam_Taml-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 579761, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "tam_Taml-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 603069, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "tam_Taml-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 554892, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "tam_Taml-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 510257, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "tam_Taml-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 571536, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "tam_Taml-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 588337, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "tam_Taml-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 596933, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "tam_Taml-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 421587, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "tam_Taml-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 575497, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "tam_Taml-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 443366, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "tam_Taml-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 569358, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "tam_Taml-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 570848, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "tam_Taml-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 558184, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "tam_Taml-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 601876, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "tam_Taml-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 560383, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "tam_Taml-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 587827, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "tam_Taml-por_Latn": { + "num_samples": 1997, + "number_of_characters": 587176, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "tam_Taml-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 584854, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "tam_Taml-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 568702, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "tam_Taml-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 530288, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "tam_Taml-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 598161, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "tam_Taml-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 582470, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "tam_Taml-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 561788, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "tam_Taml-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 557488, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "tam_Taml-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 575574, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "tam_Taml-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 557959, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "tam_Taml-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 581082, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "tam_Taml-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 401439, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "tam_Taml-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 567693, + "unique_pairs": 1997, + "min_sentence1_length": 11, + "average_sentence1_length": 155.21331997996995, + "max_sentence1_length": 581, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "tat_Cyrl-aze_Latn": { + "num_samples": 1997, + "number_of_characters": 515560, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 123.14772158237356, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 135.0195292939409, + "max_sentence2_length": 398, + "unique_sentence2": 1997 + }, + "tat_Cyrl-bak_Cyrl": { + "num_samples": 1997, + "number_of_characters": 492252, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 123.14772158237356, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 123.34802203304957, + "max_sentence2_length": 437, + "unique_sentence2": 1995 + }, + "tat_Cyrl-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 493646, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 123.14772158237356, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "tat_Cyrl-kaz_Cyrl": { + "num_samples": 1997, + "number_of_characters": 506202, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 123.14772158237356, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 130.33350025037555, + "max_sentence2_length": 473, + "unique_sentence2": 1996 + }, + "tat_Cyrl-kir_Cyrl": { + "num_samples": 1997, + "number_of_characters": 496790, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 123.14772158237356, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.62043064596895, + "max_sentence2_length": 395, + "unique_sentence2": 1996 + }, + "tat_Cyrl-tuk_Latn": { + "num_samples": 1997, + "number_of_characters": 531200, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 123.14772158237356, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 142.85127691537306, + "max_sentence2_length": 576, + "unique_sentence2": 1996 + }, + "tat_Cyrl-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 511539, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 123.14772158237356, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "tat_Cyrl-uig_Arab": { + "num_samples": 1997, + "number_of_characters": 556948, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 123.14772158237356, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 155.74461692538807, + "max_sentence2_length": 592, + "unique_sentence2": 1996 + }, + "tat_Cyrl-uzb_Latn": { + "num_samples": 1997, + "number_of_characters": 539621, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 123.14772158237356, + "max_sentence1_length": 539, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 147.06810215322986, + "max_sentence2_length": 470, + "unique_sentence2": 1996 + }, + "tel_Telu-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 491329, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "tel_Telu-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 551375, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "tel_Telu-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 495247, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "tel_Telu-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 522730, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "tel_Telu-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 492458, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "tel_Telu-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 509102, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "tel_Telu-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 513063, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "tel_Telu-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 508414, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "tel_Telu-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 495750, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "tel_Telu-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 497949, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "tel_Telu-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 506268, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "tel_Telu-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 467854, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "tel_Telu-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 557488, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "tel_Telu-urd_Arab": { + "num_samples": 1997, + "number_of_characters": 495525, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 123.9494241362043, + "max_sentence1_length": 412, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.18527791687531, + "max_sentence2_length": 390, + "unique_sentence2": 1996 + }, + "tgk_Cyrl-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 505328, + "unique_pairs": 1995, + "min_sentence1_length": 3, + "average_sentence1_length": 137.27941912869304, + "max_sentence1_length": 451, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "tgk_Cyrl-ckb_Arab": { + "num_samples": 1997, + "number_of_characters": 526514, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 137.27941912869304, + "max_sentence1_length": 451, + "unique_sentence1": 1995, + "min_sentence2_length": 5, + "average_sentence2_length": 126.37305958938407, + "max_sentence2_length": 399, + "unique_sentence2": 1995 + }, + "tgk_Cyrl-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 521867, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 137.27941912869304, + "max_sentence1_length": 451, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "tgk_Cyrl-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 517486, + "unique_pairs": 1995, + "min_sentence1_length": 3, + "average_sentence1_length": 137.27941912869304, + "max_sentence1_length": 451, + "unique_sentence1": 1995, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "tgk_Cyrl-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 474443, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 137.27941912869304, + "max_sentence1_length": 451, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "tgk_Cyrl-kmr_Latn": { + "num_samples": 1997, + "number_of_characters": 520093, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 137.27941912869304, + "max_sentence1_length": 451, + "unique_sentence1": 1995, + "min_sentence2_length": 7, + "average_sentence2_length": 123.15773660490736, + "max_sentence2_length": 420, + "unique_sentence2": 1996 + }, + "tgk_Cyrl-mey_Arab": { + "num_samples": 1997, + "number_of_characters": 487982, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 137.27941912869304, + "max_sentence1_length": 451, + "unique_sentence1": 1995, + "min_sentence2_length": 6, + "average_sentence2_length": 107.07811717576365, + "max_sentence2_length": 392, + "unique_sentence2": 1993 + }, + "tgk_Cyrl-prs_Arab": { + "num_samples": 1997, + "number_of_characters": 516683, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 137.27941912869304, + "max_sentence1_length": 451, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 121.45017526289435, + "max_sentence2_length": 365, + "unique_sentence2": 1997 + }, + "tgk_Cyrl-pus_Arab": { + "num_samples": 1997, + "number_of_characters": 516780, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 137.27941912869304, + "max_sentence1_length": 451, + "unique_sentence1": 1995, + "min_sentence2_length": 8, + "average_sentence2_length": 121.49874812218327, + "max_sentence2_length": 366, + "unique_sentence2": 1996 + }, + "tgk_Cyrl-shi_Arab": { + "num_samples": 1997, + "number_of_characters": 489060, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 137.27941912869304, + "max_sentence1_length": 451, + "unique_sentence1": 1995, + "min_sentence2_length": 3, + "average_sentence2_length": 107.6179268903355, + "max_sentence2_length": 378, + "unique_sentence2": 1996 + }, + "tha_Thai-bod_Tibt": { + "num_samples": 1997, + "number_of_characters": 538097, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 118.91236855282925, + "max_sentence1_length": 439, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 150.54031046569855, + "max_sentence2_length": 478, + "unique_sentence2": 1993 + }, + "tha_Thai-dzo_Tibt": { + "num_samples": 1997, + "number_of_characters": 480689, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 118.91236855282925, + "max_sentence1_length": 439, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 121.79318978467701, + "max_sentence2_length": 411, + "unique_sentence2": 1992 + }, + "tha_Thai-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 485188, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 118.91236855282925, + "max_sentence1_length": 439, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "tha_Thai-khm_Khmr": { + "num_samples": 1997, + "number_of_characters": 525959, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 118.91236855282925, + "max_sentence1_length": 439, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 144.4621932899349, + "max_sentence2_length": 517, + "unique_sentence2": 1996 + }, + "tha_Thai-lao_Laoo": { + "num_samples": 1997, + "number_of_characters": 504448, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 118.91236855282925, + "max_sentence1_length": 439, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 133.69053580370556, + "max_sentence2_length": 507, + "unique_sentence2": 1997 + }, + "tha_Thai-mon_Mong": { + "num_samples": 1997, + "number_of_characters": 496516, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 118.91236855282925, + "max_sentence1_length": 439, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 129.7185778668002, + "max_sentence2_length": 414, + "unique_sentence2": 1997 + }, + "tha_Thai-mya_Mymr": { + "num_samples": 1997, + "number_of_characters": 549322, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 118.91236855282925, + "max_sentence1_length": 439, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 156.16124186279418, + "max_sentence2_length": 773, + "unique_sentence2": 1997 + }, + "tir_Ethi-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 332745, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "tir_Ethi-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 412958, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "tir_Ethi-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 435204, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "tir_Ethi-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 411339, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "tir_Ethi-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 456737, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "tir_Ethi-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 402669, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "tir_Ethi-som_Latn": { + "num_samples": 1997, + "number_of_characters": 456530, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "tir_Ethi-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 453380, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "tir_Ethi-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 437747, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "tir_Ethi-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 499521, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "tir_Ethi-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 405041, + "unique_pairs": 1996, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "tir_Ethi-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 433328, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "tir_Ethi-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 481326, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "tir_Ethi-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 422970, + "unique_pairs": 1997, + "min_sentence1_length": 5, + "average_sentence1_length": 82.743114672008, + "max_sentence1_length": 272, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "ton_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 561360, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 157.05558337506258, + "max_sentence1_length": 468, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ton_Latn-fij_Latn": { + "num_samples": 1997, + "number_of_characters": 614145, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 157.05558337506258, + "max_sentence1_length": 468, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 150.478217325989, + "max_sentence2_length": 448, + "unique_sentence2": 1988 + }, + "ton_Latn-fil_Latn": { + "num_samples": 1997, + "number_of_characters": 607060, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 157.05558337506258, + "max_sentence1_length": 468, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 146.93039559339007, + "max_sentence2_length": 554, + "unique_sentence2": 1997 + }, + "ton_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 600612, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 157.05558337506258, + "max_sentence1_length": 468, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "ton_Latn-mal_Mlym": { + "num_samples": 1997, + "number_of_characters": 617792, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 157.05558337506258, + "max_sentence1_length": 468, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 152.30445668502753, + "max_sentence2_length": 540, + "unique_sentence2": 1996 + }, + "ton_Latn-mlg_Latn": { + "num_samples": 1997, + "number_of_characters": 633948, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 157.05558337506258, + "max_sentence1_length": 468, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 160.39459188783175, + "max_sentence2_length": 559, + "unique_sentence2": 1994 + }, + "ton_Latn-mri_Latn": { + "num_samples": 1997, + "number_of_characters": 587764, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 157.05558337506258, + "max_sentence1_length": 468, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 137.26790185277918, + "max_sentence2_length": 443, + "unique_sentence2": 1997 + }, + "ton_Latn-msa_Latn": { + "num_samples": 1997, + "number_of_characters": 590823, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 157.05558337506258, + "max_sentence1_length": 468, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 138.79969954932398, + "max_sentence2_length": 463, + "unique_sentence2": 1997 + }, + "ton_Latn-smo_Latn": { + "num_samples": 1997, + "number_of_characters": 591495, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 157.05558337506258, + "max_sentence1_length": 468, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 139.1362043064597, + "max_sentence2_length": 431, + "unique_sentence2": 1996 + }, + "ton_Latn-tah_Latn": { + "num_samples": 1997, + "number_of_characters": 623263, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 157.05558337506258, + "max_sentence1_length": 468, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 155.04406609914872, + "max_sentence2_length": 524, + "unique_sentence2": 1997 + }, + "tsn_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 501790, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "tsn_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 582003, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "tsn_Latn-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 604249, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "tsn_Latn-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 580384, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "tsn_Latn-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 625782, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "tsn_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 571714, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "tsn_Latn-som_Latn": { + "num_samples": 1997, + "number_of_characters": 625575, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "tsn_Latn-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 622425, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "tsn_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 606792, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "tsn_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 499521, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "tsn_Latn-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 574086, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "tsn_Latn-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 602373, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "tsn_Latn-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 650371, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "tsn_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 592015, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 167.39258888332498, + "max_sentence1_length": 556, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "tuk_Latn-aze_Latn": { + "num_samples": 1997, + "number_of_characters": 554908, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 142.85127691537306, + "max_sentence1_length": 576, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 135.0195292939409, + "max_sentence2_length": 398, + "unique_sentence2": 1997 + }, + "tuk_Latn-bak_Cyrl": { + "num_samples": 1997, + "number_of_characters": 531600, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 142.85127691537306, + "max_sentence1_length": 576, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 123.34802203304957, + "max_sentence2_length": 437, + "unique_sentence2": 1995 + }, + "tuk_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 532994, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 142.85127691537306, + "max_sentence1_length": 576, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "tuk_Latn-kaz_Cyrl": { + "num_samples": 1997, + "number_of_characters": 545550, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 142.85127691537306, + "max_sentence1_length": 576, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 130.33350025037555, + "max_sentence2_length": 473, + "unique_sentence2": 1996 + }, + "tuk_Latn-kir_Cyrl": { + "num_samples": 1997, + "number_of_characters": 536138, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 142.85127691537306, + "max_sentence1_length": 576, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.62043064596895, + "max_sentence2_length": 395, + "unique_sentence2": 1996 + }, + "tuk_Latn-tat_Cyrl": { + "num_samples": 1997, + "number_of_characters": 531200, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 142.85127691537306, + "max_sentence1_length": 576, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 123.14772158237356, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "tuk_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 550887, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 142.85127691537306, + "max_sentence1_length": 576, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "tuk_Latn-uig_Arab": { + "num_samples": 1997, + "number_of_characters": 596296, + "unique_pairs": 1997, + "min_sentence1_length": 9, + "average_sentence1_length": 142.85127691537306, + "max_sentence1_length": 576, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 155.74461692538807, + "max_sentence2_length": 592, + "unique_sentence2": 1996 + }, + "tuk_Latn-uzb_Latn": { + "num_samples": 1997, + "number_of_characters": 578969, + "unique_pairs": 1996, + "min_sentence1_length": 9, + "average_sentence1_length": 142.85127691537306, + "max_sentence1_length": 576, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 147.06810215322986, + "max_sentence2_length": 470, + "unique_sentence2": 1996 + }, + "tur_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 496794, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "tur_Latn-aze_Latn": { + "num_samples": 1997, + "number_of_characters": 535247, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 135.0195292939409, + "max_sentence2_length": 398, + "unique_sentence2": 1997 + }, + "tur_Latn-bak_Cyrl": { + "num_samples": 1997, + "number_of_characters": 511939, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 123.34802203304957, + "max_sentence2_length": 437, + "unique_sentence2": 1995 + }, + "tur_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 509415, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "tur_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 561263, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "tur_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 564740, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "tur_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 513333, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "tur_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 508952, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "tur_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 535413, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "tur_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 558721, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "tur_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 465909, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "tur_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 527188, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "tur_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 543989, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "tur_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 552585, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "tur_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 377239, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "tur_Latn-kaz_Cyrl": { + "num_samples": 1997, + "number_of_characters": 525889, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 130.33350025037555, + "max_sentence2_length": 473, + "unique_sentence2": 1996 + }, + "tur_Latn-kir_Cyrl": { + "num_samples": 1997, + "number_of_characters": 516477, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 125.62043064596895, + "max_sentence2_length": 395, + "unique_sentence2": 1996 + }, + "tur_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 399018, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "tur_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 525010, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "tur_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 557528, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "tur_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 543479, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "tur_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 542828, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "tur_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 540506, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "tur_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 553813, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "tur_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 538122, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "tur_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 517440, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "tur_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 575574, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "tur_Latn-tat_Cyrl": { + "num_samples": 1997, + "number_of_characters": 511539, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 123.14772158237356, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "tur_Latn-tuk_Latn": { + "num_samples": 1997, + "number_of_characters": 550887, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 142.85127691537306, + "max_sentence2_length": 576, + "unique_sentence2": 1996 + }, + "tur_Latn-uig_Arab": { + "num_samples": 1997, + "number_of_characters": 576635, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 155.74461692538807, + "max_sentence2_length": 592, + "unique_sentence2": 1996 + }, + "tur_Latn-uzb_Latn": { + "num_samples": 1997, + "number_of_characters": 559308, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 147.06810215322986, + "max_sentence2_length": 470, + "unique_sentence2": 1996 + }, + "tur_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 536734, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "tur_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 357091, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "tur_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 523345, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 133.00600901352027, + "max_sentence1_length": 504, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "uig_Arab-aze_Latn": { + "num_samples": 1997, + "number_of_characters": 580656, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 155.74461692538807, + "max_sentence1_length": 592, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 135.0195292939409, + "max_sentence2_length": 398, + "unique_sentence2": 1997 + }, + "uig_Arab-bak_Cyrl": { + "num_samples": 1997, + "number_of_characters": 557348, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 155.74461692538807, + "max_sentence1_length": 592, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 123.34802203304957, + "max_sentence2_length": 437, + "unique_sentence2": 1995 + }, + "uig_Arab-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 558742, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 155.74461692538807, + "max_sentence1_length": 592, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "uig_Arab-kaz_Cyrl": { + "num_samples": 1997, + "number_of_characters": 571298, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 155.74461692538807, + "max_sentence1_length": 592, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 130.33350025037555, + "max_sentence2_length": 473, + "unique_sentence2": 1996 + }, + "uig_Arab-kir_Cyrl": { + "num_samples": 1997, + "number_of_characters": 561886, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 155.74461692538807, + "max_sentence1_length": 592, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.62043064596895, + "max_sentence2_length": 395, + "unique_sentence2": 1996 + }, + "uig_Arab-tat_Cyrl": { + "num_samples": 1997, + "number_of_characters": 556948, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 155.74461692538807, + "max_sentence1_length": 592, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 123.14772158237356, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "uig_Arab-tuk_Latn": { + "num_samples": 1997, + "number_of_characters": 596296, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 155.74461692538807, + "max_sentence1_length": 592, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 142.85127691537306, + "max_sentence2_length": 576, + "unique_sentence2": 1996 + }, + "uig_Arab-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 576635, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 155.74461692538807, + "max_sentence1_length": 592, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "uig_Arab-uzb_Latn": { + "num_samples": 1997, + "number_of_characters": 604717, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 155.74461692538807, + "max_sentence1_length": 592, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 147.06810215322986, + "max_sentence2_length": 470, + "unique_sentence2": 1996 + }, + "ukr_Cyrl-bel_Cyrl": { + "num_samples": 1997, + "number_of_characters": 518873, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.2373560340511, + "max_sentence2_length": 422, + "unique_sentence2": 1996 + }, + "ukr_Cyrl-bos_Latn": { + "num_samples": 1997, + "number_of_characters": 517693, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 127.64646970455684, + "max_sentence2_length": 434, + "unique_sentence2": 1996 + }, + "ukr_Cyrl-bul_Cyrl": { + "num_samples": 1997, + "number_of_characters": 532672, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 135.14722083124687, + "max_sentence2_length": 493, + "unique_sentence2": 1996 + }, + "ukr_Cyrl-ces_Latn": { + "num_samples": 1997, + "number_of_characters": 504101, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 120.84026039058588, + "max_sentence2_length": 474, + "unique_sentence2": 1997 + }, + "ukr_Cyrl-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 510503, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ukr_Cyrl-hrv_Latn": { + "num_samples": 1997, + "number_of_characters": 518708, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 128.1547320981472, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "ukr_Cyrl-mkd_Cyrl": { + "num_samples": 1997, + "number_of_characters": 530674, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.1467200801202, + "max_sentence2_length": 451, + "unique_sentence2": 1997 + }, + "ukr_Cyrl-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 540649, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "ukr_Cyrl-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 537676, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "ukr_Cyrl-slk_Latn": { + "num_samples": 1997, + "number_of_characters": 515752, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 126.67451176765148, + "max_sentence2_length": 403, + "unique_sentence2": 1996 + }, + "ukr_Cyrl-slv_Latn": { + "num_samples": 1997, + "number_of_characters": 515679, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.63795693540311, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "ukr_Cyrl-srp_Cyrl": { + "num_samples": 1997, + "number_of_characters": 515086, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 126.34101151727592, + "max_sentence2_length": 439, + "unique_sentence2": 1995 + }, + "ukr_Cyrl-srp_Latn": { + "num_samples": 1997, + "number_of_characters": 518924, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 131.58888332498748, + "max_sentence1_length": 440, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 128.26289434151226, + "max_sentence2_length": 452, + "unique_sentence2": 1996 + }, + "urd_Arab-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 491800, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "urd_Arab-div_Thaa": { + "num_samples": 1997, + "number_of_characters": 551846, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 152.15222834251378, + "max_sentence2_length": 609, + "unique_sentence2": 1996 + }, + "urd_Arab-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 495718, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "urd_Arab-eus_Latn": { + "num_samples": 1997, + "number_of_characters": 523201, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 137.80821231847773, + "max_sentence2_length": 393, + "unique_sentence2": 1997 + }, + "urd_Arab-guj_Gujr": { + "num_samples": 1997, + "number_of_characters": 492929, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 122.64947421131697, + "max_sentence2_length": 378, + "unique_sentence2": 1997 + }, + "urd_Arab-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 509573, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "urd_Arab-kan_Knda": { + "num_samples": 1997, + "number_of_characters": 513534, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 132.96745117676514, + "max_sentence2_length": 449, + "unique_sentence2": 1996 + }, + "urd_Arab-mar_Deva": { + "num_samples": 1997, + "number_of_characters": 508885, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 130.63945918878318, + "max_sentence2_length": 443, + "unique_sentence2": 1995 + }, + "urd_Arab-nep_Deva": { + "num_samples": 1997, + "number_of_characters": 496221, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 124.29794692038057, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "urd_Arab-pan_Guru": { + "num_samples": 1997, + "number_of_characters": 498420, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 125.39909864797195, + "max_sentence2_length": 383, + "unique_sentence2": 1996 + }, + "urd_Arab-sin_Sinh": { + "num_samples": 1997, + "number_of_characters": 506739, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 129.56484727090637, + "max_sentence2_length": 441, + "unique_sentence2": 1996 + }, + "urd_Arab-snd_Arab": { + "num_samples": 1997, + "number_of_characters": 468325, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 110.32899349023535, + "max_sentence2_length": 335, + "unique_sentence2": 1996 + }, + "urd_Arab-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 557959, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "urd_Arab-tel_Telu": { + "num_samples": 1997, + "number_of_characters": 495525, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 124.18527791687531, + "max_sentence1_length": 390, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 123.9494241362043, + "max_sentence2_length": 412, + "unique_sentence2": 1996 + }, + "uzb_Latn-aze_Latn": { + "num_samples": 1997, + "number_of_characters": 563329, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 147.06810215322986, + "max_sentence1_length": 470, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 135.0195292939409, + "max_sentence2_length": 398, + "unique_sentence2": 1997 + }, + "uzb_Latn-bak_Cyrl": { + "num_samples": 1997, + "number_of_characters": 540021, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 147.06810215322986, + "max_sentence1_length": 470, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 123.34802203304957, + "max_sentence2_length": 437, + "unique_sentence2": 1995 + }, + "uzb_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 541415, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 147.06810215322986, + "max_sentence1_length": 470, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "uzb_Latn-kaz_Cyrl": { + "num_samples": 1997, + "number_of_characters": 553971, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 147.06810215322986, + "max_sentence1_length": 470, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 130.33350025037555, + "max_sentence2_length": 473, + "unique_sentence2": 1996 + }, + "uzb_Latn-kir_Cyrl": { + "num_samples": 1997, + "number_of_characters": 544559, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 147.06810215322986, + "max_sentence1_length": 470, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 125.62043064596895, + "max_sentence2_length": 395, + "unique_sentence2": 1996 + }, + "uzb_Latn-tat_Cyrl": { + "num_samples": 1997, + "number_of_characters": 539621, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 147.06810215322986, + "max_sentence1_length": 470, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 123.14772158237356, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "uzb_Latn-tuk_Latn": { + "num_samples": 1997, + "number_of_characters": 578969, + "unique_pairs": 1996, + "min_sentence1_length": 8, + "average_sentence1_length": 147.06810215322986, + "max_sentence1_length": 470, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 142.85127691537306, + "max_sentence2_length": 576, + "unique_sentence2": 1996 + }, + "uzb_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 559308, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 147.06810215322986, + "max_sentence1_length": 470, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "uzb_Latn-uig_Arab": { + "num_samples": 1997, + "number_of_characters": 604717, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 147.06810215322986, + "max_sentence1_length": 470, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 155.74461692538807, + "max_sentence2_length": 592, + "unique_sentence2": 1996 + }, + "ven_Latn-bem_Latn": { + "num_samples": 1997, + "number_of_characters": 598248, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.10315473209815, + "max_sentence1_length": 535, + "unique_sentence1": 1993, + "min_sentence2_length": 8, + "average_sentence2_length": 149.47020530796195, + "max_sentence2_length": 465, + "unique_sentence2": 1997 + }, + "ven_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 547476, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.10315473209815, + "max_sentence1_length": 535, + "unique_sentence1": 1993, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "ven_Latn-ewe_Latn": { + "num_samples": 1997, + "number_of_characters": 538734, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 150.10315473209815, + "max_sentence1_length": 535, + "unique_sentence1": 1993, + "min_sentence2_length": 6, + "average_sentence2_length": 119.6685027541312, + "max_sentence2_length": 493, + "unique_sentence2": 1994 + }, + "ven_Latn-fuc_Latn": { + "num_samples": 1997, + "number_of_characters": 528236, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.10315473209815, + "max_sentence1_length": 535, + "unique_sentence1": 1993, + "min_sentence2_length": 6, + "average_sentence2_length": 114.4116174261392, + "max_sentence2_length": 376, + "unique_sentence2": 1996 + }, + "ven_Latn-kin_Latn": { + "num_samples": 1997, + "number_of_characters": 603543, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 150.10315473209815, + "max_sentence1_length": 535, + "unique_sentence1": 1993, + "min_sentence2_length": 10, + "average_sentence2_length": 152.12168252378567, + "max_sentence2_length": 541, + "unique_sentence2": 1996 + }, + "ven_Latn-nde_Latn": { + "num_samples": 1997, + "number_of_characters": 597495, + "unique_pairs": 1997, + "min_sentence1_length": 10, + "average_sentence1_length": 150.10315473209815, + "max_sentence1_length": 535, + "unique_sentence1": 1993, + "min_sentence2_length": 6, + "average_sentence2_length": 149.09313970956435, + "max_sentence2_length": 590, + "unique_sentence2": 1997 + }, + "ven_Latn-nya_Latn": { + "num_samples": 1997, + "number_of_characters": 584038, + "unique_pairs": 1996, + "min_sentence1_length": 10, + "average_sentence1_length": 150.10315473209815, + "max_sentence1_length": 535, + "unique_sentence1": 1993, + "min_sentence2_length": 10, + "average_sentence2_length": 142.35453179769655, + "max_sentence2_length": 464, + "unique_sentence2": 1993 + }, + "ven_Latn-sna_Latn": { + "num_samples": 1997, + "number_of_characters": 598086, + "unique_pairs": 1995, + "min_sentence1_length": 10, + "average_sentence1_length": 150.10315473209815, + "max_sentence1_length": 535, + "unique_sentence1": 1993, + "min_sentence2_length": 6, + "average_sentence2_length": 149.38908362543816, + "max_sentence2_length": 511, + "unique_sentence2": 1995 + }, + "vie_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 502302, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "vie_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 514923, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "vie_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 566771, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "vie_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 570248, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "vie_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 518841, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "vie_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 514460, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "vie_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 540921, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "vie_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 564229, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "vie_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 471417, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "vie_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 532696, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "vie_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 549497, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "vie_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 558093, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "vie_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 382747, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "vie_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 404526, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "vie_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 530518, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "vie_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 563036, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "vie_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 548987, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "vie_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 548336, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "vie_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 546014, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "vie_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 559321, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "vie_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 543630, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "vie_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 522948, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "vie_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 581082, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "vie_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 536734, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "vie_Latn-yue_Hant": { + "num_samples": 1997, + "number_of_characters": 350008, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 39.502754131196795, + "max_sentence2_length": 133, + "unique_sentence2": 1996 + }, + "vie_Latn-zho_Hans": { + "num_samples": 1997, + "number_of_characters": 356082, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 42.54431647471207, + "max_sentence2_length": 263, + "unique_sentence2": 1997 + }, + "vie_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 362599, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "vie_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 528853, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 135.764146219329, + "max_sentence1_length": 437, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "wol_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 407310, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "wol_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 487523, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "wol_Latn-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 509769, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "wol_Latn-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 485904, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "wol_Latn-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 531302, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "wol_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 477234, + "unique_pairs": 1992, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "wol_Latn-som_Latn": { + "num_samples": 1997, + "number_of_characters": 531095, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "wol_Latn-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 527945, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "wol_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 512312, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "wol_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 405041, + "unique_pairs": 1996, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "wol_Latn-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 574086, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "wol_Latn-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 507893, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "wol_Latn-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 555891, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "wol_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 497535, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 120.08162243365048, + "max_sentence1_length": 405, + "unique_sentence1": 1990, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "xho_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 435597, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "xho_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 515810, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "xho_Latn-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 538056, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "xho_Latn-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 514191, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "xho_Latn-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 559589, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "xho_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 505521, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "xho_Latn-som_Latn": { + "num_samples": 1997, + "number_of_characters": 559382, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "xho_Latn-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 556232, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "xho_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 540599, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "xho_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 433328, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "xho_Latn-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 602373, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "xho_Latn-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 507893, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "xho_Latn-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 584178, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "xho_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 525822, + "unique_pairs": 1997, + "min_sentence1_length": 6, + "average_sentence1_length": 134.2463695543315, + "max_sentence1_length": 492, + "unique_sentence1": 1997, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "yor_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 483595, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "yor_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 563808, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "yor_Latn-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 586054, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "yor_Latn-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 562189, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "yor_Latn-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 607587, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "yor_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 553519, + "unique_pairs": 1996, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "yor_Latn-som_Latn": { + "num_samples": 1997, + "number_of_characters": 607380, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "yor_Latn-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 604230, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "yor_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 588597, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "yor_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 481326, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "yor_Latn-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 650371, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "yor_Latn-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 555891, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "yor_Latn-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 584178, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "yor_Latn-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 573820, + "unique_pairs": 1997, + "min_sentence1_length": 7, + "average_sentence1_length": 158.2814221331998, + "max_sentence1_length": 582, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "yue_Hant-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 326607, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 39.502754131196795, + "max_sentence1_length": 133, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "yue_Hant-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 190513, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 39.502754131196795, + "max_sentence1_length": 133, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "yue_Hant-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 212292, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 39.502754131196795, + "max_sentence1_length": 133, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "yue_Hant-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 350008, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 39.502754131196795, + "max_sentence1_length": 133, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "yue_Hant-zho_Hans": { + "num_samples": 1997, + "number_of_characters": 163848, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 39.502754131196795, + "max_sentence1_length": 133, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 42.54431647471207, + "max_sentence2_length": 263, + "unique_sentence2": 1997 + }, + "yue_Hant-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 170365, + "unique_pairs": 1996, + "min_sentence1_length": 4, + "average_sentence1_length": 39.502754131196795, + "max_sentence1_length": 133, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "zho_Hans-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 332681, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 42.54431647471207, + "max_sentence1_length": 263, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "zho_Hans-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 196587, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 42.54431647471207, + "max_sentence1_length": 263, + "unique_sentence1": 1997, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "zho_Hans-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 218366, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 42.54431647471207, + "max_sentence1_length": 263, + "unique_sentence1": 1997, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "zho_Hans-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 356082, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 42.54431647471207, + "max_sentence1_length": 263, + "unique_sentence1": 1997, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "zho_Hans-yue_Hant": { + "num_samples": 1997, + "number_of_characters": 163848, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 42.54431647471207, + "max_sentence1_length": 263, + "unique_sentence1": 1997, + "min_sentence2_length": 4, + "average_sentence2_length": 39.502754131196795, + "max_sentence2_length": 133, + "unique_sentence2": 1996 + }, + "zho_Hans-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 176439, + "unique_pairs": 1997, + "min_sentence1_length": 4, + "average_sentence1_length": 42.54431647471207, + "max_sentence1_length": 263, + "unique_sentence1": 1997, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + }, + "zho_Hant-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 322659, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "zho_Hant-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 335280, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "zho_Hant-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 387128, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "zho_Hant-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 390605, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "zho_Hant-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 339198, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "zho_Hant-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 334817, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "zho_Hant-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 361278, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "zho_Hant-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 384586, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "zho_Hant-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 291774, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "zho_Hant-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 353053, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "zho_Hant-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 369854, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "zho_Hant-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 378450, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "zho_Hant-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 203104, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "zho_Hant-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 224883, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "zho_Hant-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 350875, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "zho_Hant-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 383393, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "zho_Hant-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 369344, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "zho_Hant-por_Latn": { + "num_samples": 1997, + "number_of_characters": 368693, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "zho_Hant-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 366371, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "zho_Hant-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 379678, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "zho_Hant-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 363987, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "zho_Hant-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 343305, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "zho_Hant-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 401439, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "zho_Hant-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 357091, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "zho_Hant-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 362599, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "zho_Hant-yue_Hant": { + "num_samples": 1997, + "number_of_characters": 170365, + "unique_pairs": 1996, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 39.502754131196795, + "max_sentence2_length": 133, + "unique_sentence2": 1996 + }, + "zho_Hant-zho_Hans": { + "num_samples": 1997, + "number_of_characters": 176439, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 42.54431647471207, + "max_sentence2_length": 263, + "unique_sentence2": 1997 + }, + "zho_Hant-zul_Latn": { + "num_samples": 1997, + "number_of_characters": 349210, + "unique_pairs": 1997, + "min_sentence1_length": 3, + "average_sentence1_length": 45.80771156735103, + "max_sentence1_length": 200, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 129.0595893840761, + "max_sentence2_length": 494, + "unique_sentence2": 1996 + }, + "zul_Latn-amh_Ethi": { + "num_samples": 1997, + "number_of_characters": 425239, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 83.87931897846771, + "max_sentence2_length": 290, + "unique_sentence2": 1994 + }, + "zul_Latn-arb_Arab": { + "num_samples": 1997, + "number_of_characters": 488913, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 115.76414621932899, + "max_sentence2_length": 362, + "unique_sentence2": 1995 + }, + "zul_Latn-ben_Beng": { + "num_samples": 1997, + "number_of_characters": 501534, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 122.08412618928392, + "max_sentence2_length": 402, + "unique_sentence2": 1997 + }, + "zul_Latn-deu_Latn": { + "num_samples": 1997, + "number_of_characters": 553382, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 148.04707060590886, + "max_sentence2_length": 508, + "unique_sentence2": 1996 + }, + "zul_Latn-ell_Grek": { + "num_samples": 1997, + "number_of_characters": 556859, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 149.78818227341011, + "max_sentence2_length": 584, + "unique_sentence2": 1996 + }, + "zul_Latn-eng_Latn": { + "num_samples": 1997, + "number_of_characters": 505452, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 124.04606910365548, + "max_sentence2_length": 437, + "unique_sentence2": 1997 + }, + "zul_Latn-fas_Arab": { + "num_samples": 1997, + "number_of_characters": 501071, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 121.85227841762644, + "max_sentence2_length": 389, + "unique_sentence2": 1995 + }, + "zul_Latn-fin_Latn": { + "num_samples": 1997, + "number_of_characters": 527532, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.10265398097147, + "max_sentence2_length": 463, + "unique_sentence2": 1996 + }, + "zul_Latn-fra_Latn": { + "num_samples": 1997, + "number_of_characters": 550840, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.7741612418628, + "max_sentence2_length": 512, + "unique_sentence2": 1996 + }, + "zul_Latn-hau_Latn": { + "num_samples": 1997, + "number_of_characters": 527698, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 135.185778668002, + "max_sentence2_length": 483, + "unique_sentence2": 1997 + }, + "zul_Latn-heb_Hebr": { + "num_samples": 1997, + "number_of_characters": 458028, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 100.29844767150726, + "max_sentence2_length": 375, + "unique_sentence2": 1996 + }, + "zul_Latn-hin_Deva": { + "num_samples": 1997, + "number_of_characters": 519307, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 130.9839759639459, + "max_sentence2_length": 394, + "unique_sentence2": 1996 + }, + "zul_Latn-hun_Latn": { + "num_samples": 1997, + "number_of_characters": 536108, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 139.3970956434652, + "max_sentence2_length": 508, + "unique_sentence2": 1997 + }, + "zul_Latn-ibo_Latn": { + "num_samples": 1997, + "number_of_characters": 503833, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 123.23535302954431, + "max_sentence2_length": 469, + "unique_sentence2": 1997 + }, + "zul_Latn-ind_Latn": { + "num_samples": 1997, + "number_of_characters": 544704, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 143.70155232849274, + "max_sentence2_length": 486, + "unique_sentence2": 1997 + }, + "zul_Latn-jpn_Jpan": { + "num_samples": 1997, + "number_of_characters": 369358, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 4, + "average_sentence2_length": 55.89684526790185, + "max_sentence2_length": 189, + "unique_sentence2": 1994 + }, + "zul_Latn-kor_Hang": { + "num_samples": 1997, + "number_of_characters": 391137, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 66.80270405608412, + "max_sentence2_length": 217, + "unique_sentence2": 1995 + }, + "zul_Latn-lit_Latn": { + "num_samples": 1997, + "number_of_characters": 517129, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 129.893340010015, + "max_sentence2_length": 446, + "unique_sentence2": 1995 + }, + "zul_Latn-nld_Latn": { + "num_samples": 1997, + "number_of_characters": 549647, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 146.1767651477216, + "max_sentence2_length": 539, + "unique_sentence2": 1996 + }, + "zul_Latn-nso_Latn": { + "num_samples": 1997, + "number_of_characters": 549231, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 145.96845267901853, + "max_sentence2_length": 487, + "unique_sentence2": 1996 + }, + "zul_Latn-orm_Ethi": { + "num_samples": 1997, + "number_of_characters": 495163, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 118.89384076114172, + "max_sentence2_length": 466, + "unique_sentence2": 1984 + }, + "zul_Latn-pol_Latn": { + "num_samples": 1997, + "number_of_characters": 535598, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 9, + "average_sentence2_length": 139.14171256885328, + "max_sentence2_length": 468, + "unique_sentence2": 1996 + }, + "zul_Latn-por_Latn": { + "num_samples": 1997, + "number_of_characters": 534947, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 138.81572358537807, + "max_sentence2_length": 497, + "unique_sentence2": 1996 + }, + "zul_Latn-rus_Cyrl": { + "num_samples": 1997, + "number_of_characters": 532625, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 137.6529794692038, + "max_sentence2_length": 419, + "unique_sentence2": 1996 + }, + "zul_Latn-som_Latn": { + "num_samples": 1997, + "number_of_characters": 549024, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 145.8647971957937, + "max_sentence2_length": 455, + "unique_sentence2": 1997 + }, + "zul_Latn-spa_Latn": { + "num_samples": 1997, + "number_of_characters": 545932, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 1, + "average_sentence2_length": 144.3164747120681, + "max_sentence2_length": 504, + "unique_sentence2": 1996 + }, + "zul_Latn-ssw_Latn": { + "num_samples": 1997, + "number_of_characters": 545874, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 144.28743114672008, + "max_sentence2_length": 510, + "unique_sentence2": 1996 + }, + "zul_Latn-swa_Latn": { + "num_samples": 1997, + "number_of_characters": 530241, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 10, + "average_sentence2_length": 136.45918878317477, + "max_sentence2_length": 430, + "unique_sentence2": 1997 + }, + "zul_Latn-swe_Latn": { + "num_samples": 1997, + "number_of_characters": 509559, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 8, + "average_sentence2_length": 126.10265398097145, + "max_sentence2_length": 430, + "unique_sentence2": 1996 + }, + "zul_Latn-tam_Taml": { + "num_samples": 1997, + "number_of_characters": 567693, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 11, + "average_sentence2_length": 155.21331997996995, + "max_sentence2_length": 581, + "unique_sentence2": 1997 + }, + "zul_Latn-tir_Ethi": { + "num_samples": 1997, + "number_of_characters": 422970, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 5, + "average_sentence2_length": 82.743114672008, + "max_sentence2_length": 272, + "unique_sentence2": 1996 + }, + "zul_Latn-tsn_Latn": { + "num_samples": 1997, + "number_of_characters": 592015, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 167.39258888332498, + "max_sentence2_length": 556, + "unique_sentence2": 1997 + }, + "zul_Latn-tur_Latn": { + "num_samples": 1997, + "number_of_characters": 523345, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 133.00600901352027, + "max_sentence2_length": 504, + "unique_sentence2": 1997 + }, + "zul_Latn-vie_Latn": { + "num_samples": 1997, + "number_of_characters": 528853, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 135.764146219329, + "max_sentence2_length": 437, + "unique_sentence2": 1996 + }, + "zul_Latn-wol_Latn": { + "num_samples": 1997, + "number_of_characters": 497535, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 120.08162243365048, + "max_sentence2_length": 405, + "unique_sentence2": 1990 + }, + "zul_Latn-xho_Latn": { + "num_samples": 1997, + "number_of_characters": 525822, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 6, + "average_sentence2_length": 134.2463695543315, + "max_sentence2_length": 492, + "unique_sentence2": 1997 + }, + "zul_Latn-yor_Latn": { + "num_samples": 1997, + "number_of_characters": 573820, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 7, + "average_sentence2_length": 158.2814221331998, + "max_sentence2_length": 582, + "unique_sentence2": 1996 + }, + "zul_Latn-zho_Hant": { + "num_samples": 1997, + "number_of_characters": 349210, + "unique_pairs": 1997, + "min_sentence1_length": 8, + "average_sentence1_length": 129.0595893840761, + "max_sentence1_length": 494, + "unique_sentence1": 1996, + "min_sentence2_length": 3, + "average_sentence2_length": 45.80771156735103, + "max_sentence2_length": 200, + "unique_sentence2": 1996 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/NollySentiBitextMining.json b/mteb/descriptive_stats/BitextMining/NollySentiBitextMining.json new file mode 100644 index 0000000000..754f13c767 --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/NollySentiBitextMining.json @@ -0,0 +1,69 @@ +{ + "train": { + "num_samples": 1640, + "number_of_characters": 445805, + "unique_pairs": 1632, + "min_sentence1_length": 3, + "average_sentence1_length": 136.3170731707317, + "max_sentence1_length": 1698, + "unique_sentence1": 405, + "min_sentence2_length": 3, + "average_sentence2_length": 135.515243902439, + "max_sentence2_length": 1728, + "unique_sentence2": 1631, + "hf_subset_descriptive_stats": { + "en-ha": { + "num_samples": 410, + "number_of_characters": 115348, + "unique_pairs": 407, + "min_sentence1_length": 3, + "average_sentence1_length": 136.3170731707317, + "max_sentence1_length": 1698, + "unique_sentence1": 405, + "min_sentence2_length": 4, + "average_sentence2_length": 145.01951219512196, + "max_sentence2_length": 1728, + "unique_sentence2": 407 + }, + "en-ig": { + "num_samples": 410, + "number_of_characters": 107173, + "unique_pairs": 409, + "min_sentence1_length": 3, + "average_sentence1_length": 136.3170731707317, + "max_sentence1_length": 1698, + "unique_sentence1": 405, + "min_sentence2_length": 5, + "average_sentence2_length": 125.08048780487805, + "max_sentence2_length": 1137, + "unique_sentence2": 408 + }, + "en-pcm": { + "num_samples": 410, + "number_of_characters": 109955, + "unique_pairs": 408, + "min_sentence1_length": 3, + "average_sentence1_length": 136.3170731707317, + "max_sentence1_length": 1698, + "unique_sentence1": 405, + "min_sentence2_length": 3, + "average_sentence2_length": 131.8658536585366, + "max_sentence2_length": 1552, + "unique_sentence2": 408 + }, + "en-yo": { + "num_samples": 410, + "number_of_characters": 113329, + "unique_pairs": 409, + "min_sentence1_length": 3, + "average_sentence1_length": 136.3170731707317, + "max_sentence1_length": 1698, + "unique_sentence1": 405, + "min_sentence2_length": 6, + "average_sentence2_length": 140.0951219512195, + "max_sentence2_length": 1338, + "unique_sentence2": 409 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/NorwegianCourtsBitextMining.json b/mteb/descriptive_stats/BitextMining/NorwegianCourtsBitextMining.json new file mode 100644 index 0000000000..96403e4c83 --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/NorwegianCourtsBitextMining.json @@ -0,0 +1,15 @@ +{ + "test": { + "num_samples": 228, + "number_of_characters": 37441, + "unique_pairs": 228, + "min_sentence1_length": 13, + "average_sentence1_length": 82.19736842105263, + "max_sentence1_length": 272, + "unique_sentence1": 227, + "min_sentence2_length": 10, + "average_sentence2_length": 82.01754385964912, + "max_sentence2_length": 269, + "unique_sentence2": 226 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/NusaTranslationBitextMining.json b/mteb/descriptive_stats/BitextMining/NusaTranslationBitextMining.json new file mode 100644 index 0000000000..9efdf2f8d7 --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/NusaTranslationBitextMining.json @@ -0,0 +1,160 @@ +{ + "train": { + "num_samples": 50200, + "number_of_characters": 14759870, + "unique_pairs": 50140, + "min_sentence1_length": 5, + "average_sentence1_length": 145.4552390438247, + "max_sentence1_length": 873, + "unique_sentence1": 8258, + "min_sentence2_length": 5, + "average_sentence2_length": 148.56607569721115, + "max_sentence2_length": 980, + "unique_sentence2": 50102, + "hf_subset_descriptive_stats": { + "ind-abs": { + "num_samples": 1000, + "number_of_characters": 295680, + "unique_pairs": 999, + "min_sentence1_length": 5, + "average_sentence1_length": 148.366, + "max_sentence1_length": 727, + "unique_sentence1": 998, + "min_sentence2_length": 6, + "average_sentence2_length": 147.314, + "max_sentence2_length": 629, + "unique_sentence2": 998 + }, + "ind-btk": { + "num_samples": 6600, + "number_of_characters": 1927907, + "unique_pairs": 6597, + "min_sentence1_length": 5, + "average_sentence1_length": 145.36666666666667, + "max_sentence1_length": 873, + "unique_sentence1": 6521, + "min_sentence2_length": 5, + "average_sentence2_length": 146.74045454545455, + "max_sentence2_length": 980, + "unique_sentence2": 6596 + }, + "ind-bew": { + "num_samples": 6600, + "number_of_characters": 1939300, + "unique_pairs": 6595, + "min_sentence1_length": 5, + "average_sentence1_length": 145.4280303030303, + "max_sentence1_length": 873, + "unique_sentence1": 6512, + "min_sentence2_length": 6, + "average_sentence2_length": 148.40530303030303, + "max_sentence2_length": 840, + "unique_sentence2": 6590 + }, + "ind-bhp": { + "num_samples": 1000, + "number_of_characters": 261666, + "unique_pairs": 1000, + "min_sentence1_length": 11, + "average_sentence1_length": 133.528, + "max_sentence1_length": 468, + "unique_sentence1": 999, + "min_sentence2_length": 10, + "average_sentence2_length": 128.138, + "max_sentence2_length": 459, + "unique_sentence2": 999 + }, + "ind-jav": { + "num_samples": 6600, + "number_of_characters": 1922162, + "unique_pairs": 6594, + "min_sentence1_length": 5, + "average_sentence1_length": 145.42772727272728, + "max_sentence1_length": 873, + "unique_sentence1": 6512, + "min_sentence2_length": 5, + "average_sentence2_length": 145.8089393939394, + "max_sentence2_length": 854, + "unique_sentence2": 6585 + }, + "ind-mad": { + "num_samples": 6600, + "number_of_characters": 1973257, + "unique_pairs": 6598, + "min_sentence1_length": 5, + "average_sentence1_length": 145.35545454545453, + "max_sentence1_length": 873, + "unique_sentence1": 6521, + "min_sentence2_length": 5, + "average_sentence2_length": 153.6228787878788, + "max_sentence2_length": 827, + "unique_sentence2": 6592 + }, + "ind-mak": { + "num_samples": 6600, + "number_of_characters": 1953868, + "unique_pairs": 6594, + "min_sentence1_length": 5, + "average_sentence1_length": 145.42772727272728, + "max_sentence1_length": 873, + "unique_sentence1": 6512, + "min_sentence2_length": 6, + "average_sentence2_length": 150.6128787878788, + "max_sentence2_length": 888, + "unique_sentence2": 6586 + }, + "ind-min": { + "num_samples": 6600, + "number_of_characters": 1937033, + "unique_pairs": 6595, + "min_sentence1_length": 5, + "average_sentence1_length": 145.42772727272728, + "max_sentence1_length": 873, + "unique_sentence1": 6512, + "min_sentence2_length": 6, + "average_sentence2_length": 148.0621212121212, + "max_sentence2_length": 837, + "unique_sentence2": 6591 + }, + "ind-mui": { + "num_samples": 1000, + "number_of_characters": 301448, + "unique_pairs": 1000, + "min_sentence1_length": 11, + "average_sentence1_length": 150.454, + "max_sentence1_length": 451, + "unique_sentence1": 997, + "min_sentence2_length": 11, + "average_sentence2_length": 150.994, + "max_sentence2_length": 450, + "unique_sentence2": 1000 + }, + "ind-rej": { + "num_samples": 1000, + "number_of_characters": 291205, + "unique_pairs": 1000, + "min_sentence1_length": 9, + "average_sentence1_length": 151.622, + "max_sentence1_length": 873, + "unique_sentence1": 998, + "min_sentence2_length": 8, + "average_sentence2_length": 139.583, + "max_sentence2_length": 784, + "unique_sentence2": 1000 + }, + "ind-sun": { + "num_samples": 6600, + "number_of_characters": 1956344, + "unique_pairs": 6591, + "min_sentence1_length": 5, + "average_sentence1_length": 145.42772727272728, + "max_sentence1_length": 873, + "unique_sentence1": 6512, + "min_sentence2_length": 5, + "average_sentence2_length": 150.9880303030303, + "max_sentence2_length": 881, + "unique_sentence2": 6588 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/PhincBitextMining.json b/mteb/descriptive_stats/BitextMining/PhincBitextMining.json new file mode 100644 index 0000000000..f4b237d87d --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/PhincBitextMining.json @@ -0,0 +1,30 @@ +{ + "train": { + "num_samples": 13738, + "number_of_characters": 2069457, + "unique_pairs": 13737, + "min_sentence1_length": 1, + "average_sentence1_length": 74.02300189256079, + "max_sentence1_length": 278, + "unique_sentence1": 13515, + "min_sentence2_length": 3, + "average_sentence2_length": 76.61442713640996, + "max_sentence2_length": 274, + "unique_sentence2": 13736, + "hf_subset_descriptive_stats": { + "eng-eng_hin": { + "num_samples": 13738, + "number_of_characters": 2069457, + "unique_pairs": 13737, + "min_sentence1_length": 1, + "average_sentence1_length": 74.02300189256079, + "max_sentence1_length": 278, + "unique_sentence1": 13515, + "min_sentence2_length": 3, + "average_sentence2_length": 76.61442713640996, + "max_sentence2_length": 274, + "unique_sentence2": 13736 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/TbilisiCityHallBitextMining.json b/mteb/descriptive_stats/BitextMining/TbilisiCityHallBitextMining.json new file mode 100644 index 0000000000..12f4003727 --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/TbilisiCityHallBitextMining.json @@ -0,0 +1,43 @@ +{ + "test": { + "num_samples": 3640, + "number_of_characters": 572146, + "unique_pairs": 3640, + "min_sentence1_length": 13, + "average_sentence1_length": 78.59148351648352, + "max_sentence1_length": 203, + "unique_sentence1": 3636, + "min_sentence2_length": 13, + "average_sentence2_length": 78.59148351648352, + "max_sentence2_length": 203, + "unique_sentence2": 3636, + "hf_subset_descriptive_stats": { + "kat_Geor-eng_Latn": { + "num_samples": 1820, + "number_of_characters": 286073, + "unique_pairs": 1820, + "min_sentence1_length": 30, + "average_sentence1_length": 76.06593406593407, + "max_sentence1_length": 189, + "unique_sentence1": 1820, + "min_sentence2_length": 13, + "average_sentence2_length": 81.11703296703297, + "max_sentence2_length": 203, + "unique_sentence2": 1816 + }, + "eng_Latn-kat_Geor": { + "num_samples": 1820, + "number_of_characters": 286073, + "unique_pairs": 1820, + "min_sentence1_length": 13, + "average_sentence1_length": 81.11703296703297, + "max_sentence1_length": 203, + "unique_sentence1": 1816, + "min_sentence2_length": 30, + "average_sentence2_length": 76.06593406593407, + "max_sentence2_length": 189, + "unique_sentence2": 1820 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/BitextMining/VieMedEVBitextMining.json b/mteb/descriptive_stats/BitextMining/VieMedEVBitextMining.json new file mode 100644 index 0000000000..2d97df573e --- /dev/null +++ b/mteb/descriptive_stats/BitextMining/VieMedEVBitextMining.json @@ -0,0 +1,15 @@ +{ + "test": { + "num_samples": 2048, + "number_of_characters": 575910, + "unique_pairs": 2048, + "min_sentence1_length": 11, + "average_sentence1_length": 139.22802734375, + "max_sentence1_length": 1291, + "unique_sentence1": 2048, + "min_sentence2_length": 11, + "average_sentence2_length": 141.97802734375, + "max_sentence2_length": 1217, + "unique_sentence2": 2047 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Classification/LanguageClassification.json b/mteb/descriptive_stats/Classification/LanguageClassification.json new file mode 100644 index 0000000000..6622d23be1 --- /dev/null +++ b/mteb/descriptive_stats/Classification/LanguageClassification.json @@ -0,0 +1,146 @@ +{ + "test": { + "num_samples": 2048, + "number_of_characters": 224352, + "num_texts_in_train": 31, + "min_text_length": 14, + "average_text_length": 109.546875, + "max_text_length": 1270, + "unique_text": 2025, + "unique_labels": 20, + "labels": { + "17": { + "count": 102 + }, + "0": { + "count": 102 + }, + "11": { + "count": 102 + }, + "4": { + "count": 103 + }, + "3": { + "count": 102 + }, + "1": { + "count": 102 + }, + "10": { + "count": 102 + }, + "2": { + "count": 103 + }, + "16": { + "count": 103 + }, + "9": { + "count": 103 + }, + "5": { + "count": 102 + }, + "7": { + "count": 102 + }, + "13": { + "count": 102 + }, + "14": { + "count": 103 + }, + "12": { + "count": 102 + }, + "15": { + "count": 103 + }, + "19": { + "count": 102 + }, + "18": { + "count": 102 + }, + "6": { + "count": 103 + }, + "8": { + "count": 103 + } + } + }, + "train": { + "num_samples": 70000, + "number_of_characters": 7760299, + "num_texts_in_train": null, + "min_text_length": 2, + "average_text_length": 110.86141428571429, + "max_text_length": 2422, + "unique_text": 68978, + "unique_labels": 20, + "labels": { + "12": { + "count": 3500 + }, + "1": { + "count": 3500 + }, + "19": { + "count": 3500 + }, + "15": { + "count": 3500 + }, + "13": { + "count": 3500 + }, + "11": { + "count": 3500 + }, + "17": { + "count": 3500 + }, + "14": { + "count": 3500 + }, + "16": { + "count": 3500 + }, + "5": { + "count": 3500 + }, + "0": { + "count": 3500 + }, + "8": { + "count": 3500 + }, + "7": { + "count": 3500 + }, + "2": { + "count": 3500 + }, + "3": { + "count": 3500 + }, + "10": { + "count": 3500 + }, + "6": { + "count": 3500 + }, + "18": { + "count": 3500 + }, + "4": { + "count": 3500 + }, + "9": { + "count": 3500 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Classification/SlovakHateSpeechClassification.json b/mteb/descriptive_stats/Classification/SlovakHateSpeechClassification.json new file mode 100644 index 0000000000..63fcfd3e51 --- /dev/null +++ b/mteb/descriptive_stats/Classification/SlovakHateSpeechClassification.json @@ -0,0 +1,38 @@ +{ + "test": { + "num_samples": 1319, + "number_of_characters": 122279, + "num_texts_in_train": 46, + "min_text_length": 8, + "average_text_length": 92.70583775587566, + "max_text_length": 1584, + "unique_text": 1315, + "unique_labels": 2, + "labels": { + "1": { + "count": 360 + }, + "0": { + "count": 959 + } + } + }, + "train": { + "num_samples": 11870, + "number_of_characters": 1130860, + "num_texts_in_train": null, + "min_text_length": 7, + "average_text_length": 95.27042965459141, + "max_text_length": 2112, + "unique_text": 11655, + "unique_labels": 2, + "labels": { + "1": { + "count": 3245 + }, + "0": { + "count": 8625 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Clustering/ArXivHierarchicalClusteringP2P.json b/mteb/descriptive_stats/Clustering/ArXivHierarchicalClusteringP2P.json new file mode 100644 index 0000000000..e6066a83c2 --- /dev/null +++ b/mteb/descriptive_stats/Clustering/ArXivHierarchicalClusteringP2P.json @@ -0,0 +1,402 @@ +{ + "test": { + "num_samples": 2048, + "number_of_characters": 2065284, + "min_text_length": 103, + "average_text_length": 1008.439453125, + "max_text_length": 2103, + "min_labels_per_text": 1, + "average_labels_per_text": 1.46337890625, + "max_labels_per_text": 381, + "unique_labels": 129, + "labels": { + "cs": { + "count": 356 + }, + "math": { + "count": 381 + }, + "OC": { + "count": 11 + }, + "hep-lat": { + "count": 13 + }, + "hep": { + "count": 98 + }, + "astro-ph": { + "count": 213 + }, + "eess": { + "count": 76 + }, + "quant-ph": { + "count": 135 + }, + "DC": { + "count": 5 + }, + "cond-mat": { + "count": 274 + }, + "hep-th": { + "count": 66 + }, + "SP": { + "count": 33 + }, + "hep-ph": { + "count": 69 + }, + "FA": { + "count": 6 + }, + "nucl-th": { + "count": 17 + }, + "q-bio": { + "count": 80 + }, + "HE": { + "count": 22 + }, + "HC": { + "count": 2 + }, + "stat": { + "count": 60 + }, + "ML": { + "count": 16 + }, + "IV": { + "count": 13 + }, + "stat-mech": { + "count": 47 + }, + "DS": { + "count": 14 + }, + "ME": { + "count": 12 + }, + "CC": { + "count": 2 + }, + "mtrl-sci": { + "count": 22 + }, + "PE": { + "count": 16 + }, + "NT": { + "count": 11 + }, + "SC": { + "count": 6 + }, + "AG": { + "count": 13 + }, + "physics": { + "count": 81 + }, + "ins-det": { + "count": 9 + }, + "GA": { + "count": 18 + }, + "BM": { + "count": 6 + }, + "GN": { + "count": 17 + }, + "NA": { + "count": 15 + }, + "app-ph": { + "count": 7 + }, + "RT": { + "count": 6 + }, + "other": { + "count": 37 + }, + "soft": { + "count": 15 + }, + "CO": { + "count": 33 + }, + "supr-con": { + "count": 21 + }, + "chem-ph": { + "count": 3 + }, + "DM": { + "count": 2 + }, + "MN": { + "count": 12 + }, + "q-fin": { + "count": 27 + }, + "PM": { + "count": 2 + }, + "AP": { + "count": 27 + }, + "gr-qc": { + "count": 15 + }, + "quant-gas": { + "count": 8 + }, + "mes-hall": { + "count": 33 + }, + "IT": { + "count": 19 + }, + "SI": { + "count": 6 + }, + "SG": { + "count": 3 + }, + "bio-ph": { + "count": 2 + }, + "SR": { + "count": 16 + }, + "soc-ph": { + "count": 5 + }, + "hep-ex": { + "count": 15 + }, + "DG": { + "count": 11 + }, + "NE": { + "count": 5 + }, + "CR": { + "count": 6 + }, + "CL": { + "count": 12 + }, + "RM": { + "count": 3 + }, + "econ": { + "count": 17 + }, + "nlin": { + "count": 5 + }, + "PS": { + "count": 1 + }, + "LG": { + "count": 26 + }, + "QA": { + "count": 9 + }, + "str-el": { + "count": 26 + }, + "CV": { + "count": 34 + }, + "MF": { + "count": 6 + }, + "IM": { + "count": 7 + }, + "EM": { + "count": 6 + }, + "TH": { + "count": 5 + }, + "PR": { + "count": 20 + }, + "AT": { + "count": 4 + }, + "OA": { + "count": 4 + }, + "CP": { + "count": 6 + }, + "LO": { + "count": 14 + }, + "flu-dyn": { + "count": 6 + }, + "atom-ph": { + "count": 8 + }, + "class-ph": { + "count": 1 + }, + "SY": { + "count": 20 + }, + "IR": { + "count": 1 + }, + "plasm-ph": { + "count": 8 + }, + "CE": { + "count": 2 + }, + "AO": { + "count": 1 + }, + "comp-ph": { + "count": 3 + }, + "optics": { + "count": 12 + }, + "MG": { + "count": 4 + }, + "ST": { + "count": 6 + }, + "nucl-ex": { + "count": 6 + }, + "CY": { + "count": 9 + }, + "ao-ph": { + "count": 2 + }, + "DB": { + "count": 1 + }, + "math-ph": { + "count": 10 + }, + "NC": { + "count": 13 + }, + "GT": { + "count": 11 + }, + "TO": { + "count": 2 + }, + "AI": { + "count": 9 + }, + "NI": { + "count": 2 + }, + "gen-ph": { + "count": 4 + }, + "OT": { + "count": 4 + }, + "SD": { + "count": 2 + }, + "dis-nn": { + "count": 4 + }, + "RO": { + "count": 7 + }, + "CA": { + "count": 6 + }, + "FL": { + "count": 1 + }, + "SE": { + "count": 5 + }, + "EP": { + "count": 9 + }, + "hist-ph": { + "count": 1 + }, + "QM": { + "count": 9 + }, + "ed-ph": { + "count": 2 + }, + "GR": { + "count": 4 + }, + "MS": { + "count": 1 + }, + "CD": { + "count": 1 + }, + "ET": { + "count": 1 + }, + "acc-ph": { + "count": 5 + }, + "AC": { + "count": 2 + }, + "OH": { + "count": 1 + }, + "EC": { + "count": 2 + }, + "DL": { + "count": 1 + }, + "AS": { + "count": 3 + }, + "geo-ph": { + "count": 2 + }, + "CG": { + "count": 3 + }, + "CB": { + "count": 1 + }, + "AR": { + "count": 1 + }, + "TR": { + "count": 1 + }, + "atm-clus": { + "count": 1 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Clustering/BiorxivClusteringS2S.json b/mteb/descriptive_stats/Clustering/BiorxivClusteringS2S.json new file mode 100644 index 0000000000..2d9a0a01bb --- /dev/null +++ b/mteb/descriptive_stats/Clustering/BiorxivClusteringS2S.json @@ -0,0 +1,94 @@ +{ + "test": { + "num_samples": 10, + "number_of_characters": 75000, + "min_text_length": 5000, + "average_text_length": 7500.0, + "max_text_length": 10000, + "unique_texts": 41555, + "min_labels_per_text": 1, + "average_labels_per_text": 7500.0, + "max_labels_per_text": 14251, + "unique_labels": 26, + "labels": { + "neuroscience": { + "count": 14251 + }, + "genetics": { + "count": 2282 + }, + "biophysics": { + "count": 3864 + }, + "animal behavior and cognition": { + "count": 1148 + }, + "genomics": { + "count": 3422 + }, + "systems biology": { + "count": 1544 + }, + "ecology": { + "count": 3469 + }, + "immunology": { + "count": 3517 + }, + "evolutionary biology": { + "count": 3756 + }, + "molecular biology": { + "count": 2772 + }, + "bioengineering": { + "count": 2169 + }, + "cancer biology": { + "count": 2922 + }, + "plant biology": { + "count": 2640 + }, + "microbiology": { + "count": 7176 + }, + "physiology": { + "count": 1251 + }, + "synthetic biology": { + "count": 686 + }, + "pharmacology and toxicology": { + "count": 864 + }, + "zoology": { + "count": 433 + }, + "bioinformatics": { + "count": 6294 + }, + "cell biology": { + "count": 4433 + }, + "developmental biology": { + "count": 2352 + }, + "biochemistry": { + "count": 2790 + }, + "scientific communication and education": { + "count": 349 + }, + "paleontology": { + "count": 120 + }, + "pathology": { + "count": 495 + }, + "epidemiology": { + "count": 1 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Clustering/MedrxivClusteringP2P.v2.json b/mteb/descriptive_stats/Clustering/MedrxivClusteringP2P.v2.json new file mode 100644 index 0000000000..0370d5147e --- /dev/null +++ b/mteb/descriptive_stats/Clustering/MedrxivClusteringP2P.v2.json @@ -0,0 +1,168 @@ +{ + "test": { + "num_samples": 37500, + "number_of_characters": 74294927, + "min_text_length": 148, + "average_text_length": 1981.1980533333333, + "max_text_length": 38759, + "min_labels_per_text": 6, + "average_labels_per_text": 1.0, + "max_labels_per_text": 8830, + "unique_labels": 51, + "labels": { + "epidemiology": { + "count": 6656 + }, + "public and global health": { + "count": 3595 + }, + "oncology": { + "count": 845 + }, + "allergy and immunology": { + "count": 464 + }, + "orthopedics": { + "count": 104 + }, + "health informatics": { + "count": 1107 + }, + "occupational and environmental health": { + "count": 415 + }, + "infectious diseases": { + "count": 8830 + }, + "genetic and genomic medicine": { + "count": 1918 + }, + "health policy": { + "count": 527 + }, + "gastroenterology": { + "count": 343 + }, + "radiology and imaging": { + "count": 541 + }, + "pain medicine": { + "count": 121 + }, + "neurology": { + "count": 1773 + }, + "primary care research": { + "count": 232 + }, + "rheumatology": { + "count": 189 + }, + "endocrinology": { + "count": 419 + }, + "hematology": { + "count": 202 + }, + "addiction medicine": { + "count": 178 + }, + "pediatrics": { + "count": 589 + }, + "cardiovascular medicine": { + "count": 855 + }, + "obstetrics and gynecology": { + "count": 373 + }, + "health systems and quality improvement": { + "count": 491 + }, + "nephrology": { + "count": 241 + }, + "respiratory medicine": { + "count": 482 + }, + "geriatric medicine": { + "count": 169 + }, + "dentistry and oral medicine": { + "count": 159 + }, + "psychiatry and clinical psychology": { + "count": 1781 + }, + "nutrition": { + "count": 240 + }, + "intensive care and critical care medicine": { + "count": 368 + }, + "rehabilitation medicine and physical therapy": { + "count": 322 + }, + "otolaryngology": { + "count": 166 + }, + "nursing": { + "count": 93 + }, + "transplantation": { + "count": 118 + }, + "health economics": { + "count": 327 + }, + "sports medicine": { + "count": 180 + }, + "hiv aids": { + "count": 363 + }, + "dermatology": { + "count": 98 + }, + "pathology": { + "count": 223 + }, + "emergency medicine": { + "count": 191 + }, + "pharmacology and therapeutics": { + "count": 221 + }, + "ophthalmology": { + "count": 220 + }, + "medical ethics": { + "count": 46 + }, + "palliative medicine": { + "count": 45 + }, + "sexual and reproductive health": { + "count": 156 + }, + "medical education": { + "count": 203 + }, + "surgery": { + "count": 162 + }, + "urology": { + "count": 65 + }, + "anesthesia": { + "count": 72 + }, + "toxicology": { + "count": 16 + }, + "forensic medicine": { + "count": 6 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Clustering/MedrxivClusteringS2S.v2.json b/mteb/descriptive_stats/Clustering/MedrxivClusteringS2S.v2.json new file mode 100644 index 0000000000..7b55ddd4dc --- /dev/null +++ b/mteb/descriptive_stats/Clustering/MedrxivClusteringS2S.v2.json @@ -0,0 +1,168 @@ +{ + "test": { + "num_samples": 37500, + "number_of_characters": 4301276, + "min_text_length": 18, + "average_text_length": 114.70069333333333, + "max_text_length": 339, + "min_labels_per_text": 6, + "average_labels_per_text": 1.0, + "max_labels_per_text": 8830, + "unique_labels": 51, + "labels": { + "epidemiology": { + "count": 6656 + }, + "public and global health": { + "count": 3595 + }, + "oncology": { + "count": 845 + }, + "allergy and immunology": { + "count": 464 + }, + "orthopedics": { + "count": 104 + }, + "health informatics": { + "count": 1107 + }, + "occupational and environmental health": { + "count": 415 + }, + "infectious diseases": { + "count": 8830 + }, + "genetic and genomic medicine": { + "count": 1918 + }, + "health policy": { + "count": 527 + }, + "gastroenterology": { + "count": 343 + }, + "radiology and imaging": { + "count": 541 + }, + "pain medicine": { + "count": 121 + }, + "neurology": { + "count": 1773 + }, + "primary care research": { + "count": 232 + }, + "rheumatology": { + "count": 189 + }, + "endocrinology": { + "count": 419 + }, + "hematology": { + "count": 202 + }, + "addiction medicine": { + "count": 178 + }, + "pediatrics": { + "count": 589 + }, + "cardiovascular medicine": { + "count": 855 + }, + "obstetrics and gynecology": { + "count": 373 + }, + "health systems and quality improvement": { + "count": 491 + }, + "nephrology": { + "count": 241 + }, + "respiratory medicine": { + "count": 482 + }, + "geriatric medicine": { + "count": 169 + }, + "dentistry and oral medicine": { + "count": 159 + }, + "psychiatry and clinical psychology": { + "count": 1781 + }, + "nutrition": { + "count": 240 + }, + "intensive care and critical care medicine": { + "count": 368 + }, + "rehabilitation medicine and physical therapy": { + "count": 322 + }, + "otolaryngology": { + "count": 166 + }, + "nursing": { + "count": 93 + }, + "transplantation": { + "count": 118 + }, + "health economics": { + "count": 327 + }, + "sports medicine": { + "count": 180 + }, + "hiv aids": { + "count": 363 + }, + "dermatology": { + "count": 98 + }, + "pathology": { + "count": 223 + }, + "emergency medicine": { + "count": 191 + }, + "pharmacology and therapeutics": { + "count": 221 + }, + "ophthalmology": { + "count": 220 + }, + "medical ethics": { + "count": 46 + }, + "palliative medicine": { + "count": 45 + }, + "sexual and reproductive health": { + "count": 156 + }, + "medical education": { + "count": 203 + }, + "surgery": { + "count": 162 + }, + "urology": { + "count": 65 + }, + "anesthesia": { + "count": 72 + }, + "toxicology": { + "count": 16 + }, + "forensic medicine": { + "count": 6 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Clustering/RedditClusteringP2P.v2.json b/mteb/descriptive_stats/Clustering/RedditClusteringP2P.v2.json new file mode 100644 index 0000000000..ba997dbefc --- /dev/null +++ b/mteb/descriptive_stats/Clustering/RedditClusteringP2P.v2.json @@ -0,0 +1,1335 @@ +{ + "test": { + "num_samples": 459389, + "number_of_characters": 334286895, + "min_text_length": 79, + "average_text_length": 727.6771864367671, + "max_text_length": 4359, + "min_labels_per_text": 2, + "average_labels_per_text": 1.0, + "max_labels_per_text": 77908, + "unique_labels": 440, + "labels": { + "FortNiteBR": { + "count": 436 + }, + "buildapc": { + "count": 8484 + }, + "offmychest": { + "count": 570 + }, + "nus": { + "count": 45 + }, + "relationship_advice": { + "count": 16651 + }, + "premed": { + "count": 201 + }, + "dogecoin": { + "count": 8108 + }, + "GamingLaptops": { + "count": 183 + }, + "asktransgender": { + "count": 326 + }, + "MachineLearning": { + "count": 61 + }, + "puppy101": { + "count": 1597 + }, + "GunAccessoriesForSale": { + "count": 2619 + }, + "Random_Acts_Of_Amazon": { + "count": 1115 + }, + "Catholicism": { + "count": 183 + }, + "MonsterHunter": { + "count": 218 + }, + "tipofmypenis": { + "count": 87 + }, + "samsung": { + "count": 69 + }, + "PersonalFinanceCanada": { + "count": 341 + }, + "Dyson_Sphere_Program": { + "count": 55 + }, + "bleach": { + "count": 41 + }, + "AmItheAsshole": { + "count": 3730 + }, + "WallStreetbetsELITE": { + "count": 328 + }, + "GlobalPowers": { + "count": 35 + }, + "ABraThatFits": { + "count": 159 + }, + "PokemonGoFriends": { + "count": 1165 + }, + "NoMansSkyTheGame": { + "count": 259 + }, + "masseffect": { + "count": 233 + }, + "dating_advice": { + "count": 559 + }, + "yoga": { + "count": 50 + }, + "depression": { + "count": 515 + }, + "COVID19positive": { + "count": 180 + }, + "generationology": { + "count": 37 + }, + "feedthebeast": { + "count": 192 + }, + "EliteDangerous": { + "count": 270 + }, + "alcoholicsanonymous": { + "count": 93 + }, + "GoRVing": { + "count": 35 + }, + "thedivision": { + "count": 111 + }, + "breakingmom": { + "count": 105 + }, + "AskAnAmerican": { + "count": 80 + }, + "HypnoFair": { + "count": 5 + }, + "JustUnsubbed": { + "count": 13 + }, + "socialanxiety": { + "count": 123 + }, + "dirtykikpals": { + "count": 202 + }, + "askTO": { + "count": 126 + }, + "AskCulinary": { + "count": 108 + }, + "Bogleheads": { + "count": 71 + }, + "dragonquest": { + "count": 45 + }, + "NoContract": { + "count": 30 + }, + "gorillaz": { + "count": 14 + }, + "MondoGore": { + "count": 8 + }, + "comicswap": { + "count": 56 + }, + "VirtualYoutubers": { + "count": 92 + }, + "Gta5Modding": { + "count": 28 + }, + "obs": { + "count": 61 + }, + "vcu": { + "count": 9 + }, + "KingkillerChronicle": { + "count": 17 + }, + "AmongUs": { + "count": 41 + }, + "wireshark": { + "count": 3 + }, + "Dodocodes": { + "count": 46 + }, + "Aliexpress": { + "count": 40 + }, + "LearnerDriverUK": { + "count": 12 + }, + "PanicAttack": { + "count": 23 + }, + "KassadinMains": { + "count": 10 + }, + "islam": { + "count": 93 + }, + "chronotrigger": { + "count": 4 + }, + "skincareexchange": { + "count": 13 + }, + "PokemonHome": { + "count": 21 + }, + "survivinginfidelity": { + "count": 71 + }, + "igcse": { + "count": 21 + }, + "C25K": { + "count": 21 + }, + "aorus": { + "count": 2 + }, + "idleon": { + "count": 19 + }, + "photography": { + "count": 22 + }, + "cryptocoins": { + "count": 7 + }, + "CanaryWharfBets": { + "count": 7 + }, + "KillingEve": { + "count": 7 + }, + "GameBuilderGarage": { + "count": 16 + }, + "SauceSharingCommunity": { + "count": 7 + }, + "turo": { + "count": 9 + }, + "foodscience": { + "count": 14 + }, + "HIMYM": { + "count": 20 + }, + "HauntingOfHillHouse": { + "count": 4 + }, + "GoodNotes": { + "count": 8 + }, + "RedditWritesSeinfeld": { + "count": 6 + }, + "AirReps": { + "count": 2 + }, + "ADHD": { + "count": 3811 + }, + "BuddyCrossing": { + "count": 446 + }, + "libraryofruina": { + "count": 98 + }, + "SluttyConfessions": { + "count": 2787 + }, + "tipofmytongue": { + "count": 7145 + }, + "fleshlight": { + "count": 128 + }, + "amcstock": { + "count": 13910 + }, + "teenagers": { + "count": 77908 + }, + "suggestmeabook": { + "count": 1540 + }, + "dirtypenpals": { + "count": 5587 + }, + "MinecraftServer": { + "count": 177 + }, + "CreditCards": { + "count": 669 + }, + "Guitar": { + "count": 10952 + }, + "rpg": { + "count": 529 + }, + "NoFap": { + "count": 14853 + }, + "lfg": { + "count": 1093 + }, + "MarsWallStreet": { + "count": 935 + }, + "SummonSign": { + "count": 931 + }, + "AssassinsCreedValhala": { + "count": 295 + }, + "hoi4": { + "count": 432 + }, + "Coins4Sale": { + "count": 260 + }, + "xbox": { + "count": 459 + }, + "TooAfraidToAsk": { + "count": 7404 + }, + "NBA2k": { + "count": 553 + }, + "KGBTR": { + "count": 943 + }, + "roblox": { + "count": 220 + }, + "salesforce": { + "count": 214 + }, + "TwoXChromosomes": { + "count": 1736 + }, + "mechmarket": { + "count": 4863 + }, + "Gaming_Headsets": { + "count": 103 + }, + "pittsburgh": { + "count": 189 + }, + "CryptoMars": { + "count": 1606 + }, + "FridayNightFunkin": { + "count": 378 + }, + "vaginismus": { + "count": 122 + }, + "transpositive": { + "count": 10 + }, + "comicbooks": { + "count": 274 + }, + "BDSMcommunity": { + "count": 185 + }, + "aliens": { + "count": 201 + }, + "Scotch": { + "count": 64 + }, + "KikRoleplay": { + "count": 141 + }, + "Kayaking": { + "count": 91 + }, + "196": { + "count": 47 + }, + "digimon": { + "count": 140 + }, + "Evernote": { + "count": 42 + }, + "logh": { + "count": 22 + }, + "arlington": { + "count": 15 + }, + "Adopted": { + "count": 8 + }, + "DissonautUniverse": { + "count": 4 + }, + "Midsommar": { + "count": 12 + }, + "SofiawithanF": { + "count": 83 + }, + "xmpp": { + "count": 6 + }, + "ZombsRoyale": { + "count": 16 + }, + "accesscontrol": { + "count": 8 + }, + "WetlanderHumor": { + "count": 2 + }, + "PoonamPandeyFanatics": { + "count": 2 + }, + "screenplaychallenge": { + "count": 2 + }, + "scatstories": { + "count": 2 + }, + "techsupport": { + "count": 290 + }, + "whatcarshouldIbuy": { + "count": 79 + }, + "Stormlight_Archive": { + "count": 15 + }, + "deadbydaylight": { + "count": 126 + }, + "bicycling": { + "count": 27 + }, + "oculus": { + "count": 64 + }, + "Cartalk": { + "count": 33 + }, + "Sims4": { + "count": 43 + }, + "NoFeeAC": { + "count": 95 + }, + "Crypto_com": { + "count": 37 + }, + "ITCareerQuestions": { + "count": 259 + }, + "aromantic": { + "count": 18 + }, + "Revu": { + "count": 3 + }, + "exalted": { + "count": 2 + }, + "HilariaBaldwin": { + "count": 20 + }, + "Testosterone": { + "count": 35 + }, + "Screenwriting": { + "count": 170 + }, + "LifeProTips": { + "count": 49 + }, + "steinsgate": { + "count": 13 + }, + "Baystreetbets": { + "count": 10 + }, + "AskGirls": { + "count": 7 + }, + "idlechampions": { + "count": 7 + }, + "facebook": { + "count": 17 + }, + "tf2trade": { + "count": 4 + }, + "mfdoom": { + "count": 3 + }, + "FiddlesticksMains": { + "count": 2 + }, + "HFY": { + "count": 10 + }, + "FiestaST": { + "count": 2 + }, + "whatsthatbook": { + "count": 994 + }, + "GearsOfWar": { + "count": 879 + }, + "KazuhaMains": { + "count": 175 + }, + "RepTime": { + "count": 211 + }, + "AstroGaming": { + "count": 141 + }, + "metalgearsolid": { + "count": 152 + }, + "qBittorrent": { + "count": 39 + }, + "ELLIPAL_Official": { + "count": 24 + }, + "raisedbynarcissists": { + "count": 4895 + }, + "unpopularopinion": { + "count": 14901 + }, + "ACTrade": { + "count": 5679 + }, + "askcarsales": { + "count": 1339 + }, + "AskVet": { + "count": 1357 + }, + "whowouldwin": { + "count": 4493 + }, + "playstation": { + "count": 1362 + }, + "anime": { + "count": 6531 + }, + "GME": { + "count": 12577 + }, + "DotA2": { + "count": 2004 + }, + "cryptostreetbets": { + "count": 2241 + }, + "MonsterHunterWorld": { + "count": 698 + }, + "Market76": { + "count": 14274 + }, + "DnD": { + "count": 5092 + }, + "leagueoflegends": { + "count": 3683 + }, + "doordash_drivers": { + "count": 1626 + }, + "theta_network": { + "count": 489 + }, + "exmuslim": { + "count": 1369 + }, + "gonewildaudio": { + "count": 2998 + }, + "conspiracy": { + "count": 3587 + }, + "heroesofthestorm": { + "count": 535 + }, + "FanFiction": { + "count": 2782 + }, + "Doom": { + "count": 1251 + }, + "texas": { + "count": 269 + }, + "Vent": { + "count": 1738 + }, + "selfimprovement": { + "count": 1284 + }, + "youtubers": { + "count": 706 + }, + "askseddit": { + "count": 237 + }, + "boardgames": { + "count": 1237 + }, + "bravelydefault": { + "count": 347 + }, + "ConquerorsBlade": { + "count": 238 + }, + "ChronicPain": { + "count": 527 + }, + "teenagersnew": { + "count": 256 + }, + "brasil": { + "count": 1092 + }, + "MatthiasSubmissions": { + "count": 921 + }, + "MarylandUnemployment": { + "count": 314 + }, + "SaltLakeCity": { + "count": 411 + }, + "BokunoheroFanfiction": { + "count": 155 + }, + "BenignExistence": { + "count": 125 + }, + "GayYoungOldDating": { + "count": 156 + }, + "Bible": { + "count": 202 + }, + "haskell": { + "count": 154 + }, + "seduction": { + "count": 400 + }, + "fantasywriters": { + "count": 262 + }, + "HiveOS": { + "count": 100 + }, + "PerkByDaylight": { + "count": 15 + }, + "Hedgehog": { + "count": 73 + }, + "xmen": { + "count": 263 + }, + "HyperRP": { + "count": 122 + }, + "emotestories": { + "count": 3 + }, + "tutanota": { + "count": 135 + }, + "CultoftheFranklin": { + "count": 46 + }, + "langrisser": { + "count": 62 + }, + "CozyGrove": { + "count": 61 + }, + "Sverigesforsvarsmakt": { + "count": 12 + }, + "silverbugbets": { + "count": 21 + }, + "WreckingBallMains": { + "count": 5 + }, + "capitalism_in_decay": { + "count": 8 + }, + "paintdotnet": { + "count": 11 + }, + "u_mawadom118": { + "count": 4 + }, + "xboxfindfriends": { + "count": 2 + }, + "CPTSD": { + "count": 540 + }, + "destiny2": { + "count": 318 + }, + "Wallstreetsilver": { + "count": 1013 + }, + "DestinyTheGame": { + "count": 1107 + }, + "blackopscoldwar": { + "count": 400 + }, + "InstacartShoppers": { + "count": 202 + }, + "RocketLeagueExchange": { + "count": 832 + }, + "apexlegends": { + "count": 3265 + }, + "kansascity": { + "count": 53 + }, + "namenerds": { + "count": 235 + }, + "help": { + "count": 152 + }, + "Kengan_Ashura": { + "count": 132 + }, + "thetagang": { + "count": 165 + }, + "GameSale": { + "count": 262 + }, + "Reduction": { + "count": 109 + }, + "sex": { + "count": 906 + }, + "bostonr4r": { + "count": 75 + }, + "LegendsOfRuneterra": { + "count": 231 + }, + "overlord": { + "count": 48 + }, + "madisonwi": { + "count": 53 + }, + "steelseries": { + "count": 79 + }, + "ClashOfClansRecruit": { + "count": 214 + }, + "CharacterRant": { + "count": 55 + }, + "AirForce": { + "count": 94 + }, + "sexstories": { + "count": 92 + }, + "NameThatSong": { + "count": 162 + }, + "depressed": { + "count": 74 + }, + "ibs": { + "count": 150 + }, + "40kLore": { + "count": 269 + }, + "podcasts": { + "count": 88 + }, + "miraculousladybug": { + "count": 150 + }, + "ask": { + "count": 224 + }, + "EverMerge": { + "count": 31 + }, + "TMJ": { + "count": 54 + }, + "BitLifeApp": { + "count": 39 + }, + "FireEmblemHeroes": { + "count": 100 + }, + "software": { + "count": 62 + }, + "ShieldAndroidTV": { + "count": 70 + }, + "GriefSupport": { + "count": 125 + }, + "onewheel": { + "count": 37 + }, + "MensRights": { + "count": 80 + }, + "nhl": { + "count": 22 + }, + "ClashOfClans": { + "count": 107 + }, + "ps3homebrew": { + "count": 33 + }, + "LightNovels": { + "count": 77 + }, + "redsox": { + "count": 34 + }, + "CryptoMarkets": { + "count": 44 + }, + "ugly": { + "count": 47 + }, + "GCXRep": { + "count": 12 + }, + "cscareerquestionsEU": { + "count": 65 + }, + "MindHunter": { + "count": 6 + }, + "starcraft2coop": { + "count": 15 + }, + "nanocurrency": { + "count": 1421 + }, + "ModelCars": { + "count": 8 + }, + "UKJobs": { + "count": 30 + }, + "Netherlands": { + "count": 44 + }, + "clonewars": { + "count": 8 + }, + "Julia": { + "count": 11 + }, + "Prolactinoma": { + "count": 9 + }, + "sofi": { + "count": 11 + }, + "royalfamily": { + "count": 6 + }, + "ConnecticutR4R": { + "count": 8 + }, + "weather": { + "count": 5 + }, + "oneui": { + "count": 7 + }, + "KTM": { + "count": 5 + }, + "Aerials": { + "count": 3 + }, + "seoul": { + "count": 2 + }, + "exjw": { + "count": 3281 + }, + "ModernMagic": { + "count": 699 + }, + "Paladins": { + "count": 1242 + }, + "kdramarecommends": { + "count": 1611 + }, + "hitbtc": { + "count": 330 + }, + "endocrinology": { + "count": 75 + }, + "Bath": { + "count": 43 + }, + "NassauCountyHookups": { + "count": 5 + }, + "feminineboys": { + "count": 1248 + }, + "dreamsmp": { + "count": 2018 + }, + "SquaredCircle": { + "count": 2255 + }, + "Minecraft": { + "count": 8753 + }, + "spirituality": { + "count": 1809 + }, + "Eldenring": { + "count": 1471 + }, + "Sat": { + "count": 1172 + }, + "bonnaroo": { + "count": 194 + }, + "gardening": { + "count": 1892 + }, + "Unemployment": { + "count": 6185 + }, + "mac": { + "count": 1847 + }, + "Bestbuy": { + "count": 437 + }, + "quittingkratom": { + "count": 1081 + }, + "lawschooladmissions": { + "count": 3436 + }, + "NiceHash": { + "count": 2135 + }, + "McMaster": { + "count": 815 + }, + "covidlonghaulers": { + "count": 1299 + }, + "stalker": { + "count": 758 + }, + "MLBTheShow": { + "count": 2721 + }, + "FortniteCompetitive": { + "count": 998 + }, + "dpdr": { + "count": 514 + }, + "appliancerepair": { + "count": 720 + }, + "thomasthetankengine": { + "count": 207 + }, + "delhi": { + "count": 217 + }, + "Huel": { + "count": 300 + }, + "leafs": { + "count": 203 + }, + "HotWheels": { + "count": 170 + }, + "90dayfianceuncensored": { + "count": 550 + }, + "Throwers": { + "count": 142 + }, + "Wavyhair": { + "count": 270 + }, + "CryptoHorde": { + "count": 128 + }, + "ShuumatsuNoValkyrie": { + "count": 453 + }, + "TeensMeetTeens": { + "count": 432 + }, + "dbrand": { + "count": 108 + }, + "SLFmeetups": { + "count": 18 + }, + "1200isplentyketo": { + "count": 48 + }, + "passive_income": { + "count": 211 + }, + "BroadCity": { + "count": 16 + }, + "RevenantMain": { + "count": 71 + }, + "extrarfl": { + "count": 25 + }, + "AgonGame": { + "count": 5 + }, + "FitnessDE": { + "count": 3 + }, + "gaming": { + "count": 1277 + }, + "livesound": { + "count": 91 + }, + "IBO": { + "count": 1896 + }, + "EscapefromTarkov": { + "count": 1300 + }, + "amex": { + "count": 145 + }, + "DMAcademy": { + "count": 1411 + }, + "VinylCollectors": { + "count": 556 + }, + "cardano": { + "count": 716 + }, + "brave_browser": { + "count": 159 + }, + "dating": { + "count": 952 + }, + "OculusQuest": { + "count": 942 + }, + "Superstonk": { + "count": 3089 + }, + "MtF": { + "count": 957 + }, + "findaleague": { + "count": 207 + }, + "Nioh": { + "count": 398 + }, + "IRS": { + "count": 715 + }, + "transgendercirclejerk": { + "count": 353 + }, + "learnmath": { + "count": 489 + }, + "piano": { + "count": 263 + }, + "LeagueConnect": { + "count": 216 + }, + "eu4": { + "count": 561 + }, + "Wordpress": { + "count": 345 + }, + "RoleplayingForReddit": { + "count": 31 + }, + "LOONA": { + "count": 89 + }, + "newtothenavy": { + "count": 167 + }, + "HaircareScience": { + "count": 118 + }, + "appletv": { + "count": 167 + }, + "sissypersonals": { + "count": 102 + }, + "raleigh": { + "count": 168 + }, + "realonlyfansreviews": { + "count": 21 + }, + "AskGames": { + "count": 49 + }, + "PokemonTCG": { + "count": 325 + }, + "controlgame": { + "count": 109 + }, + "GoogleDataStudio": { + "count": 16 + }, + "WhiteWolfRPG": { + "count": 139 + }, + "MECoOp": { + "count": 31 + }, + "snuffrp": { + "count": 46 + }, + "lockpicking": { + "count": 103 + }, + "wicked_edge": { + "count": 105 + }, + "BMW": { + "count": 99 + }, + "choiceofgames": { + "count": 24 + }, + "hisdarkmaterials": { + "count": 12 + }, + "SakuraGakuin": { + "count": 24 + }, + "detrans": { + "count": 55 + }, + "Smallville": { + "count": 37 + }, + "kingofqueens": { + "count": 7 + }, + "JamesHoffmann": { + "count": 22 + }, + "stashinvest": { + "count": 16 + }, + "ABA": { + "count": 79 + }, + "ladybusiness": { + "count": 10 + }, + "gamegrumps": { + "count": 32 + }, + "GodEater": { + "count": 21 + }, + "tomorrow": { + "count": 39 + }, + "Tomorrowland": { + "count": 9 + }, + "BlackCountryNewRoad": { + "count": 5 + }, + "STAYC": { + "count": 3 + }, + "SatoshiStreetBets": { + "count": 3828 + }, + "AskLosAngeles": { + "count": 1036 + }, + "buildapcforme": { + "count": 1689 + }, + "ApplyingToCollege": { + "count": 10675 + }, + "watercooling": { + "count": 1209 + }, + "BreakUps": { + "count": 4914 + }, + "FIFA": { + "count": 3811 + }, + "emacs": { + "count": 712 + }, + "trakstocks": { + "count": 691 + }, + "Shittyaskflying": { + "count": 147 + }, + "AmazonFC": { + "count": 1178 + }, + "stocks": { + "count": 4610 + }, + "BangaloreMains": { + "count": 26 + }, + "pokemon": { + "count": 3953 + }, + "religion": { + "count": 684 + }, + "cuboulder": { + "count": 269 + }, + "self": { + "count": 1688 + }, + "tarot": { + "count": 912 + }, + "turtles": { + "count": 49 + }, + "TheMagnusArchives": { + "count": 300 + }, + "Superhero_Ideas": { + "count": 34 + }, + "NTU": { + "count": 308 + }, + "touhou": { + "count": 623 + }, + "JoJolion": { + "count": 50 + }, + "lasers": { + "count": 27 + }, + "popperpigs": { + "count": 67 + }, + "aggretsuko": { + "count": 20 + }, + "Library": { + "count": 5 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Clustering/RuSciBenchGRNTIClusteringP2P.json b/mteb/descriptive_stats/Clustering/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 0000000000..126cd893bc --- /dev/null +++ b/mteb/descriptive_stats/Clustering/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,99 @@ +{ + "test": { + "num_samples": 2048, + "number_of_characters": 1822339, + "min_text_length": 84, + "average_text_length": 889.81396484375, + "max_text_length": 3143, + "min_labels_per_text": 73, + "average_labels_per_text": 1.0, + "max_labels_per_text": 74, + "unique_labels": 28, + "labels": { + "3": { + "count": 73 + }, + "4": { + "count": 73 + }, + "20": { + "count": 73 + }, + "9": { + "count": 73 + }, + "21": { + "count": 73 + }, + "15": { + "count": 73 + }, + "16": { + "count": 74 + }, + "2": { + "count": 73 + }, + "8": { + "count": 73 + }, + "23": { + "count": 73 + }, + "6": { + "count": 73 + }, + "24": { + "count": 73 + }, + "10": { + "count": 73 + }, + "1": { + "count": 73 + }, + "17": { + "count": 74 + }, + "14": { + "count": 74 + }, + "18": { + "count": 73 + }, + "27": { + "count": 73 + }, + "19": { + "count": 73 + }, + "22": { + "count": 73 + }, + "12": { + "count": 73 + }, + "25": { + "count": 73 + }, + "5": { + "count": 74 + }, + "0": { + "count": 73 + }, + "26": { + "count": 73 + }, + "11": { + "count": 73 + }, + "13": { + "count": 73 + }, + "7": { + "count": 73 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Clustering/TwentyNewsgroupsClustering.v2.json b/mteb/descriptive_stats/Clustering/TwentyNewsgroupsClustering.v2.json new file mode 100644 index 0000000000..77be5a3b77 --- /dev/null +++ b/mteb/descriptive_stats/Clustering/TwentyNewsgroupsClustering.v2.json @@ -0,0 +1,75 @@ +{ + "test": { + "num_samples": 59545, + "number_of_characters": 1907719, + "min_text_length": 11, + "average_text_length": 32.03827357460744, + "max_text_length": 120, + "min_labels_per_text": 2082, + "average_labels_per_text": 1.0, + "max_labels_per_text": 3236, + "unique_labels": 20, + "labels": { + "12": { + "count": 3137 + }, + "6": { + "count": 3070 + }, + "0": { + "count": 2613 + }, + "2": { + "count": 3155 + }, + "10": { + "count": 3220 + }, + "17": { + "count": 2986 + }, + "14": { + "count": 3106 + }, + "13": { + "count": 3055 + }, + "1": { + "count": 3056 + }, + "16": { + "count": 2911 + }, + "9": { + "count": 2984 + }, + "3": { + "count": 3070 + }, + "15": { + "count": 3090 + }, + "7": { + "count": 3036 + }, + "5": { + "count": 3124 + }, + "11": { + "count": 3236 + }, + "18": { + "count": 2483 + }, + "8": { + "count": 3090 + }, + "19": { + "count": 2082 + }, + "4": { + "count": 3041 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Clustering/WikiClusteringP2P.json b/mteb/descriptive_stats/Clustering/WikiClusteringP2P.json new file mode 100644 index 0000000000..4c1f303098 --- /dev/null +++ b/mteb/descriptive_stats/Clustering/WikiClusteringP2P.json @@ -0,0 +1,1948 @@ +{ + "test": { + "num_samples": 140, + "number_of_characters": 71680, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 49704, + "min_labels_per_text": 1, + "average_labels_per_text": 512.0, + "max_labels_per_text": 3986, + "unique_labels": 282, + "labels": { + "Nauke": { + "count": 1492 + }, + "Dru\u00c5\u00a1tvo": { + "count": 504 + }, + "Priroda": { + "count": 448 + }, + "Kultura": { + "count": 1042 + }, + "Tehnologija": { + "count": 671 + }, + "Tehnika": { + "count": 281 + }, + "Geografija": { + "count": 431 + }, + "Informatika": { + "count": 355 + }, + "Koncepti": { + "count": 83 + }, + "Humanisti\u00c4\u008dke_nauke": { + "count": 62 + }, + "Informacija": { + "count": 21 + }, + "Historija": { + "count": 223 + }, + "Matematika": { + "count": 74 + }, + "Okoli\u00c5\u00a1": { + "count": 6 + }, + "Jezik": { + "count": 15 + }, + "Misao": { + "count": 28 + }, + "Energija": { + "count": 16 + }, + "Llocs": { + "count": 642 + }, + "Ci\u00c3\u00a8ncia": { + "count": 1844 + }, + "Humanitats": { + "count": 984 + }, + "Tecnologia": { + "count": 377 + }, + "Biografies": { + "count": 406 + }, + "Cultura": { + "count": 710 + }, + "Informaci\u00c3\u00b3": { + "count": 137 + }, + "Esdeveniments": { + "count": 20 + }, + "Lid\u00c3\u00a9": { + "count": 1559 + }, + "Geografie": { + "count": 1659 + }, + "\u00c4\u008cas": { + "count": 88 + }, + "Politika": { + "count": 818 + }, + "V\u00c4\u009bda": { + "count": 314 + }, + "Technika": { + "count": 189 + }, + "Informace": { + "count": 96 + }, + "\u00c5\u00bdivot": { + "count": 184 + }, + "Vojenstv\u00c3\u00ad": { + "count": 34 + }, + "Um\u00c4\u009bn\u00c3\u00ad": { + "count": 95 + }, + "P\u00c5\u0099\u00c3\u00adroda": { + "count": 387 + }, + "Spole\u00c4\u008dnost": { + "count": 319 + }, + "Historie": { + "count": 391 + }, + "Sport": { + "count": 57 + }, + "Dorozum\u00c3\u00adv\u00c3\u00a1n\u00c3\u00ad": { + "count": 142 + }, + "Zdravotnictv\u00c3\u00ad": { + "count": 94 + }, + "Vzd\u00c4\u009bl\u00c3\u00a1v\u00c3\u00a1n\u00c3\u00ad": { + "count": 41 + }, + "P\u00c5\u0099edm\u00c4\u009bty": { + "count": 21 + }, + "Pr\u00c3\u00a1vo": { + "count": 28 + }, + "Natur": { + "count": 398 + }, + "Kultur": { + "count": 911 + }, + "Samfund": { + "count": 531 + }, + "Politik": { + "count": 287 + }, + "Computersystemer": { + "count": 84 + }, + "Erhvervsliv": { + "count": 164 + }, + "Teknik": { + "count": 166 + }, + "Sproget": { + "count": 41 + }, + "Humaniora": { + "count": 139 + }, + "Uddannelse": { + "count": 79 + }, + "Anvendt_videnskab": { + "count": 199 + }, + "Kunst": { + "count": 412 + }, + "Sundhed": { + "count": 83 + }, + "Videnskab": { + "count": 257 + }, + "Matematik": { + "count": 79 + }, + "Geografi": { + "count": 671 + }, + "Liv": { + "count": 286 + }, + "Personer": { + "count": 322 + }, + "Jura": { + "count": 35 + }, + "Entitateak": { + "count": 2486 + }, + "Humanitateak": { + "count": 861 + }, + "Mundua": { + "count": 706 + }, + "Zientzia_eta_teknologia": { + "count": 957 + }, + "Entretenimendua": { + "count": 110 + }, + "Chron-oaylleeaght": { + "count": 1334 + }, + "\u00c3\u0087heeraghyn": { + "count": 470 + }, + "Oaylleeaght": { + "count": 575 + }, + "\u00c3\u0087heer-oaylleeaght": { + "count": 190 + }, + "Persoonyn": { + "count": 249 + }, + "Bea": { + "count": 207 + }, + "Cultoor": { + "count": 269 + }, + "Glare": { + "count": 152 + }, + "Dooghys": { + "count": 494 + }, + "Smooinaght": { + "count": 28 + }, + "Sheshaght": { + "count": 221 + }, + "\u00c3\u0087haghnoaylleeaght": { + "count": 33 + }, + "Strughtoor": { + "count": 54 + }, + "Politickaght": { + "count": 138 + }, + "Ronnaghyn_rere_bun-chooish": { + "count": 381 + }, + "Eirinys": { + "count": 80 + }, + "Ynsagh": { + "count": 16 + }, + "Am": { + "count": 12 + }, + "Leigh": { + "count": 9 + }, + "Studeyrys-sheelnaue": { + "count": 72 + }, + "Keirdyn_scoillaragh": { + "count": 50 + }, + "Shennaghys": { + "count": 11 + }, + "Dellal": { + "count": 23 + }, + "Maddaght": { + "count": 6 + }, + "Meanyn_mooarey": { + "count": 7 + }, + "Oaylleeaghtyn_feamagh": { + "count": 30 + }, + "Fysseree": { + "count": 7 + }, + "\u00c3\u0087hymmyltaght": { + "count": 2 + }, + "Katutubo": { + "count": 135 + }, + "Biag": { + "count": 933 + }, + "Pakasaritaan": { + "count": 1302 + }, + "Relihion": { + "count": 20 + }, + "Dagiti_kinatao": { + "count": 1405 + }, + "Lubong": { + "count": 308 + }, + "Siensia_ken_teknolohia": { + "count": 12 + }, + "Gobierno": { + "count": 208 + }, + "Heograpia": { + "count": 323 + }, + "Etika": { + "count": 12 + }, + "Pilosopia": { + "count": 26 + }, + "Tattao": { + "count": 20 + }, + "Dagiti_konsepto": { + "count": 69 + }, + "Panagkukua_ti_tao": { + "count": 16 + }, + "Pannakaammo": { + "count": 29 + }, + "Palpaliwa": { + "count": 22 + }, + "Pagsasao": { + "count": 33 + }, + "Kagimongan": { + "count": 37 + }, + "Salun-at": { + "count": 9 + }, + "Industria": { + "count": 5 + }, + "Linteg": { + "count": 1 + }, + "Makan_ken_mainum": { + "count": 19 + }, + "Dagiti_akademiko_a_disiplina": { + "count": 27 + }, + "Edukasion": { + "count": 9 + }, + "Musika": { + "count": 1 + }, + "Negosio": { + "count": 10 + }, + "Inhenieria": { + "count": 1 + }, + "Paspasamak": { + "count": 6 + }, + "Dagiti_banag": { + "count": 2 + }, + "Annuroten": { + "count": 1 + }, + "Enerhia": { + "count": 1 + }, + "Kes": { + "count": 524 + }, + "Medya": { + "count": 67 + }, + "Xwarin_\u00c3\u00bb_vexwarin": { + "count": 127 + }, + "Civak": { + "count": 277 + }, + "L\u00c3\u00aakol\u00c3\u00aen\u00c3\u00aan_mirovahiy\u00c3\u00aa": { + "count": 269 + }, + "Zeman": { + "count": 174 + }, + "Teknoloj\u00c3\u00ae": { + "count": 92 + }, + "D\u00c3\u00aes\u00c3\u00aepl\u00c3\u00aen\u00c3\u00aan_akadem\u00c3\u00aek": { + "count": 237 + }, + "Erdn\u00c3\u00aegar\u00c3\u00ae": { + "count": 1078 + }, + "D\u00c3\u00aerok": { + "count": 436 + }, + "Gerd\u00c3\u00bbn": { + "count": 42 + }, + "Jiyan": { + "count": 255 + }, + "\u00c5\u009eah\u00c3\u00ae": { + "count": 107 + }, + "Siyaset": { + "count": 126 + }, + "\u00c3\u0087and": { + "count": 349 + }, + "Zan\u00c3\u00aen": { + "count": 82 + }, + "D\u00c3\u00aen": { + "count": 150 + }, + "Helwesta_mirovan": { + "count": 41 + }, + "Zanist": { + "count": 138 + }, + "Hey\u00c3\u00aen": { + "count": 49 + }, + "Ziman": { + "count": 69 + }, + "Xweza": { + "count": 129 + }, + "Matemat\u00c3\u00aek": { + "count": 23 + }, + "Agah\u00c3\u00ae": { + "count": 17 + }, + "Dad": { + "count": 9 + }, + "\u00c3\u008enternet": { + "count": 19 + }, + "Hik\u00c3\u00bbmet": { + "count": 35 + }, + "Konsept": { + "count": 10 + }, + "Perwerde": { + "count": 6 + }, + "Werzi\u00c5\u009f": { + "count": 20 + }, + "Felsefe": { + "count": 28 + }, + "Et\u00c3\u00aek": { + "count": 19 + }, + "Wize": { + "count": 6 + }, + "Abor\u00c3\u00ae": { + "count": 27 + }, + "Tendurust\u00c3\u00ae": { + "count": 38 + }, + "Ragihandin": { + "count": 5 + }, + "Karwer\u00c3\u00ae": { + "count": 6 + }, + "Endezyar\u00c3\u00ae": { + "count": 16 + }, + "Le\u00c5\u009fker\u00c3\u00ae": { + "count": 18 + }, + "Kult\u00c5\u00abra": { + "count": 749 + }, + "Latvija": { + "count": 356 + }, + "\u00c4\u00a2eogr\u00c4\u0081fija": { + "count": 721 + }, + "Zin\u00c4\u0081tne_un_tehnolo\u00c4\u00a3ijas": { + "count": 469 + }, + "V\u00c4\u0093sture": { + "count": 149 + }, + "Sabiedr\u00c4\u00abba": { + "count": 878 + }, + "J\u00c4\u0093dzieni": { + "count": 23 + }, + "Dz\u00c4\u00abv\u00c4\u00abba": { + "count": 274 + }, + "Cilv\u00c4\u0093ki": { + "count": 428 + }, + "Sports": { + "count": 166 + }, + "M\u00c4\u0081ksla": { + "count": 185 + }, + "Daba": { + "count": 412 + }, + "Vesel\u00c4\u00abba": { + "count": 124 + }, + "Reli\u00c4\u00a3ija": { + "count": 86 + }, + "Notikumi": { + "count": 13 + }, + "Makaluak_iduik": { + "count": 3986 + }, + "Indonesia": { + "count": 184 + }, + "Budayo": { + "count": 399 + }, + "Ilimu_pangatauan": { + "count": 121 + }, + "Makanan": { + "count": 12 + }, + "Seni": { + "count": 32 + }, + "Organisasi": { + "count": 23 + }, + "Karajo": { + "count": 30 + }, + "Astronomi": { + "count": 13 + }, + "Sarugo": { + "count": 30 + }, + "Tokoh": { + "count": 26 + }, + "Agamo": { + "count": 48 + }, + "Sijarah": { + "count": 5 + }, + "Teknologi": { + "count": 1 + }, + "Ulahrago": { + "count": 2 + }, + "\u00c4\u00a0eografija": { + "count": 1634 + }, + "Arti": { + "count": 194 + }, + "Gvern": { + "count": 107 + }, + "Reli\u00c4\u00a1jon": { + "count": 293 + }, + "Dixxiplini_akkademi\u00c4\u008bi": { + "count": 139 + }, + "Nies": { + "count": 270 + }, + "So\u00c4\u008bjet\u00c3\u00a0": { + "count": 88 + }, + "Natura": { + "count": 449 + }, + "Sa\u00c4\u00a7\u00c4\u00a7a": { + "count": 31 + }, + "Xjenza": { + "count": 667 + }, + "Storja": { + "count": 167 + }, + "Ekonomija": { + "count": 199 + }, + "Lingwa": { + "count": 76 + }, + "Filosofija": { + "count": 14 + }, + "\u00c4\u00a6ajja_ta\\_Kuljum": { + "count": 81 + }, + "Edukazzjoni": { + "count": 30 + }, + "Mu\u00c5\u00bcika": { + "count": 14 + }, + "Komunikazzjoni_umana": { + "count": 39 + }, + "Spettaklu": { + "count": 38 + }, + "Kronolo\u00c4\u00a1ija": { + "count": 39 + }, + "Avvenimenti": { + "count": 6 + }, + "Li\u00c4\u00a1i": { + "count": 19 + }, + "Teknolo\u00c4\u00a1ija": { + "count": 17 + }, + "In\u00c4\u00a1inerija": { + "count": 2 + }, + "Life": { + "count": 621 + }, + "Naitur": { + "count": 265 + }, + "Society": { + "count": 446 + }, + "Humanities": { + "count": 259 + }, + "History": { + "count": 184 + }, + "Airts": { + "count": 106 + }, + "Technology": { + "count": 324 + }, + "Fowk": { + "count": 208 + }, + "Concepts": { + "count": 237 + }, + "Cultur": { + "count": 427 + }, + "Environs": { + "count": 231 + }, + "Warld": { + "count": 141 + }, + "Politics": { + "count": 294 + }, + "Eddication": { + "count": 42 + }, + "Airt": { + "count": 18 + }, + "Heal": { + "count": 70 + }, + "Science_an_technology": { + "count": 60 + }, + "Mathematics": { + "count": 36 + }, + "Law": { + "count": 3 + }, + "Tuils": { + "count": 7 + }, + "Employment": { + "count": 14 + }, + "Gjeografi": { + "count": 586 + }, + "Politik\u00c3\u00ab": { + "count": 351 + }, + "Let\u00c3\u00abrsi": { + "count": 67 + }, + "Administrat\u00c3\u00ab_publike": { + "count": 320 + }, + "Shoq\u00c3\u00abri": { + "count": 116 + }, + "Sporte": { + "count": 105 + }, + "Shkenc\u00c3\u00ab": { + "count": 1109 + }, + "Kultur\u00c3\u00ab": { + "count": 299 + }, + "Arte": { + "count": 217 + }, + "Persona": { + "count": 425 + }, + "Histori": { + "count": 744 + }, + "Mitologji": { + "count": 5 + }, + "Gjuh\u00c3\u00absi": { + "count": 64 + }, + "Teknologji": { + "count": 84 + }, + "Kinematografi": { + "count": 72 + }, + "Media": { + "count": 51 + }, + "Sigurime": { + "count": 31 + }, + "Loj\u00c3\u00abra": { + "count": 3 + }, + "Fe": { + "count": 131 + }, + "Bujq\u00c3\u00absi": { + "count": 41 + }, + "Ngjarje": { + "count": 11 + }, + "Biografi": { + "count": 116 + }, + "Matematik\u00c3\u00ab": { + "count": 27 + }, + "Teknik\u00c3\u00ab": { + "count": 26 + }, + "Drejt\u00c3\u00absi": { + "count": 18 + }, + "Organizata": { + "count": 27 + }, + "Jeta": { + "count": 4 + }, + "Agronomi": { + "count": 3 + }, + "Natyr\u00c3\u00ab": { + "count": 11 + }, + "Sh\u00c3\u00abndeti": { + "count": 3 + }, + "Shkencat_humane": { + "count": 22 + }, + "Shp\u00c3\u00abrblime": { + "count": 2 + }, + "Blegtori": { + "count": 10 + }, + "L\u00c3\u00abnd\u00c3\u00ab": { + "count": 8 + }, + "Enciklopedistika": { + "count": 6 + }, + "Economeye": { + "count": 816 + }, + "Syinces": { + "count": 3653 + }, + "Vicaedje_des_djins": { + "count": 314 + }, + "Creyance": { + "count": 310 + }, + "Rilom\u00c3\u00aay\u00c3\u00a8s_djins": { + "count": 25 + }, + "Date": { + "count": 2 + } + }, + "hf_subset_descriptive_stats": { + "bs": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 3860, + "min_labels_per_text": 6, + "average_labels_per_text": 512.0, + "max_labels_per_text": 1492, + "unique_labels": 17, + "labels": { + "Nauke": { + "count": 1492 + }, + "Dru\u00c5\u00a1tvo": { + "count": 504 + }, + "Priroda": { + "count": 448 + }, + "Kultura": { + "count": 453 + }, + "Tehnologija": { + "count": 671 + }, + "Tehnika": { + "count": 281 + }, + "Geografija": { + "count": 431 + }, + "Informatika": { + "count": 355 + }, + "Koncepti": { + "count": 83 + }, + "Humanisti\u00c4\u008dke_nauke": { + "count": 62 + }, + "Informacija": { + "count": 21 + }, + "Historija": { + "count": 223 + }, + "Matematika": { + "count": 31 + }, + "Okoli\u00c5\u00a1": { + "count": 6 + }, + "Jezik": { + "count": 15 + }, + "Misao": { + "count": 28 + }, + "Energija": { + "count": 16 + } + } + }, + "ca": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 4596, + "min_labels_per_text": 20, + "average_labels_per_text": 512.0, + "max_labels_per_text": 1844, + "unique_labels": 8, + "labels": { + "Llocs": { + "count": 642 + }, + "Ci\u00c3\u00a8ncia": { + "count": 1844 + }, + "Humanitats": { + "count": 984 + }, + "Tecnologia": { + "count": 377 + }, + "Biografies": { + "count": 406 + }, + "Cultura": { + "count": 710 + }, + "Informaci\u00c3\u00b3": { + "count": 137 + }, + "Esdeveniments": { + "count": 20 + } + } + }, + "cs": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 4782, + "min_labels_per_text": 21, + "average_labels_per_text": 512.0, + "max_labels_per_text": 1559, + "unique_labels": 21, + "labels": { + "Lid\u00c3\u00a9": { + "count": 1559 + }, + "Geografie": { + "count": 578 + }, + "\u00c4\u008cas": { + "count": 88 + }, + "Politika": { + "count": 538 + }, + "V\u00c4\u009bda": { + "count": 314 + }, + "Technika": { + "count": 189 + }, + "Informace": { + "count": 96 + }, + "\u00c5\u00bdivot": { + "count": 184 + }, + "Vojenstv\u00c3\u00ad": { + "count": 34 + }, + "Um\u00c4\u009bn\u00c3\u00ad": { + "count": 95 + }, + "P\u00c5\u0099\u00c3\u00adroda": { + "count": 387 + }, + "Spole\u00c4\u008dnost": { + "count": 319 + }, + "Historie": { + "count": 207 + }, + "Sport": { + "count": 22 + }, + "Kultura": { + "count": 150 + }, + "Matematika": { + "count": 34 + }, + "Dorozum\u00c3\u00adv\u00c3\u00a1n\u00c3\u00ad": { + "count": 142 + }, + "Zdravotnictv\u00c3\u00ad": { + "count": 94 + }, + "Vzd\u00c4\u009bl\u00c3\u00a1v\u00c3\u00a1n\u00c3\u00ad": { + "count": 41 + }, + "P\u00c5\u0099edm\u00c4\u009bty": { + "count": 21 + }, + "Pr\u00c3\u00a1vo": { + "count": 28 + } + } + }, + "da": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 4725, + "min_labels_per_text": 35, + "average_labels_per_text": 512.0, + "max_labels_per_text": 911, + "unique_labels": 20, + "labels": { + "Natur": { + "count": 398 + }, + "Kultur": { + "count": 911 + }, + "Samfund": { + "count": 531 + }, + "Politik": { + "count": 287 + }, + "Computersystemer": { + "count": 84 + }, + "Erhvervsliv": { + "count": 164 + }, + "Teknik": { + "count": 166 + }, + "Sproget": { + "count": 41 + }, + "Humaniora": { + "count": 139 + }, + "Uddannelse": { + "count": 79 + }, + "Anvendt_videnskab": { + "count": 199 + }, + "Kunst": { + "count": 412 + }, + "Sundhed": { + "count": 83 + }, + "Historie": { + "count": 184 + }, + "Videnskab": { + "count": 257 + }, + "Matematik": { + "count": 79 + }, + "Geografi": { + "count": 463 + }, + "Liv": { + "count": 286 + }, + "Personer": { + "count": 322 + }, + "Jura": { + "count": 35 + } + } + }, + "eu": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 4474, + "min_labels_per_text": 110, + "average_labels_per_text": 512.0, + "max_labels_per_text": 2486, + "unique_labels": 5, + "labels": { + "Entitateak": { + "count": 2486 + }, + "Humanitateak": { + "count": 861 + }, + "Mundua": { + "count": 706 + }, + "Zientzia_eta_teknologia": { + "count": 957 + }, + "Entretenimendua": { + "count": 110 + } + } + }, + "gv": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 2717, + "min_labels_per_text": 2, + "average_labels_per_text": 512.0, + "max_labels_per_text": 1334, + "unique_labels": 28, + "labels": { + "Chron-oaylleeaght": { + "count": 1334 + }, + "\u00c3\u0087heeraghyn": { + "count": 470 + }, + "Oaylleeaght": { + "count": 575 + }, + "\u00c3\u0087heer-oaylleeaght": { + "count": 190 + }, + "Persoonyn": { + "count": 249 + }, + "Bea": { + "count": 207 + }, + "Cultoor": { + "count": 269 + }, + "Glare": { + "count": 152 + }, + "Dooghys": { + "count": 494 + }, + "Smooinaght": { + "count": 28 + }, + "Sheshaght": { + "count": 221 + }, + "\u00c3\u0087haghnoaylleeaght": { + "count": 33 + }, + "Strughtoor": { + "count": 54 + }, + "Politickaght": { + "count": 138 + }, + "Ronnaghyn_rere_bun-chooish": { + "count": 381 + }, + "Eirinys": { + "count": 80 + }, + "Ynsagh": { + "count": 16 + }, + "Am": { + "count": 12 + }, + "Leigh": { + "count": 9 + }, + "Studeyrys-sheelnaue": { + "count": 72 + }, + "Keirdyn_scoillaragh": { + "count": 50 + }, + "Shennaghys": { + "count": 11 + }, + "Dellal": { + "count": 23 + }, + "Maddaght": { + "count": 6 + }, + "Meanyn_mooarey": { + "count": 7 + }, + "Oaylleeaghtyn_feamagh": { + "count": 30 + }, + "Fysseree": { + "count": 7 + }, + "\u00c3\u0087hymmyltaght": { + "count": 2 + } + } + }, + "ilo": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 2258, + "min_labels_per_text": 1, + "average_labels_per_text": 512.0, + "max_labels_per_text": 1405, + "unique_labels": 34, + "labels": { + "Katutubo": { + "count": 135 + }, + "Biag": { + "count": 933 + }, + "Pakasaritaan": { + "count": 1302 + }, + "Kultura": { + "count": 72 + }, + "Relihion": { + "count": 20 + }, + "Dagiti_kinatao": { + "count": 1405 + }, + "Lubong": { + "count": 308 + }, + "Siensia_ken_teknolohia": { + "count": 12 + }, + "Gobierno": { + "count": 208 + }, + "Heograpia": { + "count": 323 + }, + "Matematika": { + "count": 9 + }, + "Etika": { + "count": 12 + }, + "Pilosopia": { + "count": 26 + }, + "Tattao": { + "count": 20 + }, + "Dagiti_konsepto": { + "count": 69 + }, + "Panagkukua_ti_tao": { + "count": 16 + }, + "Pannakaammo": { + "count": 29 + }, + "Palpaliwa": { + "count": 22 + }, + "Pagsasao": { + "count": 33 + }, + "Kagimongan": { + "count": 37 + }, + "Salun-at": { + "count": 9 + }, + "Industria": { + "count": 5 + }, + "Linteg": { + "count": 1 + }, + "Makan_ken_mainum": { + "count": 19 + }, + "Dagiti_akademiko_a_disiplina": { + "count": 27 + }, + "Edukasion": { + "count": 9 + }, + "Musika": { + "count": 1 + }, + "Politika": { + "count": 37 + }, + "Negosio": { + "count": 10 + }, + "Inhenieria": { + "count": 1 + }, + "Paspasamak": { + "count": 6 + }, + "Dagiti_banag": { + "count": 2 + }, + "Annuroten": { + "count": 1 + }, + "Enerhia": { + "count": 1 + } + } + }, + "ku": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 3365, + "min_labels_per_text": 5, + "average_labels_per_text": 512.0, + "max_labels_per_text": 1078, + "unique_labels": 39, + "labels": { + "Kes": { + "count": 524 + }, + "Medya": { + "count": 67 + }, + "Xwarin_\u00c3\u00bb_vexwarin": { + "count": 127 + }, + "Civak": { + "count": 277 + }, + "L\u00c3\u00aakol\u00c3\u00aen\u00c3\u00aan_mirovahiy\u00c3\u00aa": { + "count": 269 + }, + "Zeman": { + "count": 174 + }, + "Teknoloj\u00c3\u00ae": { + "count": 92 + }, + "D\u00c3\u00aes\u00c3\u00aepl\u00c3\u00aen\u00c3\u00aan_akadem\u00c3\u00aek": { + "count": 237 + }, + "Erdn\u00c3\u00aegar\u00c3\u00ae": { + "count": 1078 + }, + "D\u00c3\u00aerok": { + "count": 436 + }, + "Gerd\u00c3\u00bbn": { + "count": 42 + }, + "Jiyan": { + "count": 255 + }, + "\u00c5\u009eah\u00c3\u00ae": { + "count": 107 + }, + "Siyaset": { + "count": 126 + }, + "\u00c3\u0087and": { + "count": 349 + }, + "Zan\u00c3\u00aen": { + "count": 82 + }, + "D\u00c3\u00aen": { + "count": 150 + }, + "Helwesta_mirovan": { + "count": 41 + }, + "Zanist": { + "count": 138 + }, + "Hey\u00c3\u00aen": { + "count": 49 + }, + "Ziman": { + "count": 69 + }, + "Xweza": { + "count": 129 + }, + "Matemat\u00c3\u00aek": { + "count": 23 + }, + "Agah\u00c3\u00ae": { + "count": 17 + }, + "Dad": { + "count": 9 + }, + "\u00c3\u008enternet": { + "count": 19 + }, + "Hik\u00c3\u00bbmet": { + "count": 35 + }, + "Konsept": { + "count": 10 + }, + "Perwerde": { + "count": 6 + }, + "Werzi\u00c5\u009f": { + "count": 20 + }, + "Felsefe": { + "count": 28 + }, + "Et\u00c3\u00aek": { + "count": 19 + }, + "Wize": { + "count": 6 + }, + "Abor\u00c3\u00ae": { + "count": 27 + }, + "Tendurust\u00c3\u00ae": { + "count": 38 + }, + "Ragihandin": { + "count": 5 + }, + "Karwer\u00c3\u00ae": { + "count": 6 + }, + "Endezyar\u00c3\u00ae": { + "count": 16 + }, + "Le\u00c5\u009fker\u00c3\u00ae": { + "count": 18 + } + } + }, + "lv": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 4540, + "min_labels_per_text": 13, + "average_labels_per_text": 512.0, + "max_labels_per_text": 878, + "unique_labels": 16, + "labels": { + "Kult\u00c5\u00abra": { + "count": 749 + }, + "Latvija": { + "count": 356 + }, + "\u00c4\u00a2eogr\u00c4\u0081fija": { + "count": 721 + }, + "Zin\u00c4\u0081tne_un_tehnolo\u00c4\u00a3ijas": { + "count": 469 + }, + "V\u00c4\u0093sture": { + "count": 149 + }, + "Sabiedr\u00c4\u00abba": { + "count": 878 + }, + "J\u00c4\u0093dzieni": { + "count": 23 + }, + "Dz\u00c4\u00abv\u00c4\u00abba": { + "count": 274 + }, + "Cilv\u00c4\u0093ki": { + "count": 428 + }, + "Sports": { + "count": 120 + }, + "M\u00c4\u0081ksla": { + "count": 185 + }, + "Politika": { + "count": 133 + }, + "Daba": { + "count": 412 + }, + "Vesel\u00c4\u00abba": { + "count": 124 + }, + "Reli\u00c4\u00a3ija": { + "count": 86 + }, + "Notikumi": { + "count": 13 + } + } + }, + "min": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 3881, + "min_labels_per_text": 1, + "average_labels_per_text": 512.0, + "max_labels_per_text": 3986, + "unique_labels": 16, + "labels": { + "Makaluak_iduik": { + "count": 3986 + }, + "Indonesia": { + "count": 184 + }, + "Budayo": { + "count": 399 + }, + "Ilimu_pangatauan": { + "count": 121 + }, + "Geografi": { + "count": 208 + }, + "Makanan": { + "count": 12 + }, + "Seni": { + "count": 32 + }, + "Organisasi": { + "count": 23 + }, + "Karajo": { + "count": 30 + }, + "Astronomi": { + "count": 13 + }, + "Sarugo": { + "count": 30 + }, + "Tokoh": { + "count": 26 + }, + "Agamo": { + "count": 48 + }, + "Sijarah": { + "count": 5 + }, + "Teknologi": { + "count": 1 + }, + "Ulahrago": { + "count": 2 + } + } + }, + "mt": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 1887, + "min_labels_per_text": 2, + "average_labels_per_text": 512.0, + "max_labels_per_text": 1634, + "unique_labels": 27, + "labels": { + "\u00c4\u00a0eografija": { + "count": 1634 + }, + "Arti": { + "count": 194 + }, + "Gvern": { + "count": 107 + }, + "Reli\u00c4\u00a1jon": { + "count": 293 + }, + "Dixxiplini_akkademi\u00c4\u008bi": { + "count": 139 + }, + "Nies": { + "count": 270 + }, + "So\u00c4\u008bjet\u00c3\u00a0": { + "count": 88 + }, + "Natura": { + "count": 449 + }, + "Sa\u00c4\u00a7\u00c4\u00a7a": { + "count": 31 + }, + "Xjenza": { + "count": 667 + }, + "Storja": { + "count": 167 + }, + "Ekonomija": { + "count": 199 + }, + "Kultura": { + "count": 367 + }, + "Lingwa": { + "count": 76 + }, + "Filosofija": { + "count": 14 + }, + "\u00c4\u00a6ajja_ta\\_Kuljum": { + "count": 81 + }, + "Edukazzjoni": { + "count": 30 + }, + "Politika": { + "count": 110 + }, + "Mu\u00c5\u00bcika": { + "count": 14 + }, + "Komunikazzjoni_umana": { + "count": 39 + }, + "Spettaklu": { + "count": 38 + }, + "Kronolo\u00c4\u00a1ija": { + "count": 39 + }, + "Avvenimenti": { + "count": 6 + }, + "Li\u00c4\u00a1i": { + "count": 19 + }, + "Teknolo\u00c4\u00a1ija": { + "count": 17 + }, + "Sport": { + "count": 30 + }, + "In\u00c4\u00a1inerija": { + "count": 2 + } + } + }, + "sco": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 2605, + "min_labels_per_text": 3, + "average_labels_per_text": 512.0, + "max_labels_per_text": 1081, + "unique_labels": 23, + "labels": { + "Life": { + "count": 621 + }, + "Naitur": { + "count": 265 + }, + "Geografie": { + "count": 1081 + }, + "Society": { + "count": 446 + }, + "Humanities": { + "count": 259 + }, + "History": { + "count": 184 + }, + "Airts": { + "count": 106 + }, + "Technology": { + "count": 324 + }, + "Fowk": { + "count": 208 + }, + "Concepts": { + "count": 237 + }, + "Cultur": { + "count": 427 + }, + "Environs": { + "count": 231 + }, + "Warld": { + "count": 141 + }, + "Politics": { + "count": 294 + }, + "Eddication": { + "count": 42 + }, + "Airt": { + "count": 18 + }, + "Heal": { + "count": 70 + }, + "Science_an_technology": { + "count": 60 + }, + "Sports": { + "count": 46 + }, + "Mathematics": { + "count": 36 + }, + "Law": { + "count": 3 + }, + "Tuils": { + "count": 7 + }, + "Employment": { + "count": 14 + } + } + }, + "sq": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 3741, + "min_labels_per_text": 2, + "average_labels_per_text": 512.0, + "max_labels_per_text": 1109, + "unique_labels": 36, + "labels": { + "Gjeografi": { + "count": 586 + }, + "Politik\u00c3\u00ab": { + "count": 351 + }, + "Let\u00c3\u00abrsi": { + "count": 67 + }, + "Administrat\u00c3\u00ab_publike": { + "count": 320 + }, + "Shoq\u00c3\u00abri": { + "count": 116 + }, + "Sporte": { + "count": 105 + }, + "Shkenc\u00c3\u00ab": { + "count": 1109 + }, + "Kultur\u00c3\u00ab": { + "count": 299 + }, + "Arte": { + "count": 217 + }, + "Persona": { + "count": 425 + }, + "Histori": { + "count": 744 + }, + "Mitologji": { + "count": 5 + }, + "Gjuh\u00c3\u00absi": { + "count": 64 + }, + "Teknologji": { + "count": 84 + }, + "Kinematografi": { + "count": 72 + }, + "Media": { + "count": 51 + }, + "Sigurime": { + "count": 31 + }, + "Loj\u00c3\u00abra": { + "count": 3 + }, + "Fe": { + "count": 131 + }, + "Bujq\u00c3\u00absi": { + "count": 41 + }, + "Ngjarje": { + "count": 11 + }, + "Biografi": { + "count": 116 + }, + "Matematik\u00c3\u00ab": { + "count": 27 + }, + "Teknik\u00c3\u00ab": { + "count": 26 + }, + "Drejt\u00c3\u00absi": { + "count": 18 + }, + "Organizata": { + "count": 27 + }, + "Jeta": { + "count": 4 + }, + "Sport": { + "count": 5 + }, + "Agronomi": { + "count": 3 + }, + "Natyr\u00c3\u00ab": { + "count": 11 + }, + "Sh\u00c3\u00abndeti": { + "count": 3 + }, + "Shkencat_humane": { + "count": 22 + }, + "Shp\u00c3\u00abrblime": { + "count": 2 + }, + "Blegtori": { + "count": 10 + }, + "L\u00c3\u00abnd\u00c3\u00ab": { + "count": 8 + }, + "Enciklopedistika": { + "count": 6 + } + } + }, + "wa": { + "num_samples": 10, + "number_of_characters": 5120, + "min_text_length": 512, + "average_text_length": 512.0, + "max_text_length": 512, + "unique_texts": 2317, + "min_labels_per_text": 2, + "average_labels_per_text": 512.0, + "max_labels_per_text": 3653, + "unique_labels": 6, + "labels": { + "Economeye": { + "count": 816 + }, + "Syinces": { + "count": 3653 + }, + "Vicaedje_des_djins": { + "count": 314 + }, + "Creyance": { + "count": 310 + }, + "Rilom\u00c3\u00aay\u00c3\u00a8s_djins": { + "count": 25 + }, + "Date": { + "count": 2 + } + } + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/InstructionRetrieval/Core17InstructionRetrieval.json b/mteb/descriptive_stats/InstructionRetrieval/Core17InstructionRetrieval.json new file mode 100644 index 0000000000..897b23d7c7 --- /dev/null +++ b/mteb/descriptive_stats/InstructionRetrieval/Core17InstructionRetrieval.json @@ -0,0 +1,30 @@ +{ + "test": { + "num_samples": 19919, + "num_docs": 19899, + "num_queries": 20, + "number_of_characters": 44450333, + "min_document_length": 7, + "average_document_length": 2233.0329664807277, + "max_document_length": 2959, + "unique_docs": 19143, + "min_query_length": 55, + "average_query_length": 109.75, + "max_query_length": 278, + "unique_queries": 20, + "min_instruction_length": 102, + "average_instruction_length": 295.55, + "max_instruction_length": 811, + "unique_instructions": 20, + "min_changed_instruction_length": 151, + "average_changed_instruction_length": 355.2, + "max_changed_instruction_length": 837, + "unique_changed_instructions": 20, + "min_average_relevant_docs_per_query": 4, + "average_relevant_docs_per_query": 32.7, + "max_average_relevant_docs_per_query": 55, + "min_average_top_ranked_per_query": 1000, + "average_top_ranked_per_query": 1000.0, + "max_average_top_ranked_per_query": 1000 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/MultilabelClassification/CEDRClassification.json b/mteb/descriptive_stats/MultilabelClassification/CEDRClassification.json new file mode 100644 index 0000000000..d5d91adf50 --- /dev/null +++ b/mteb/descriptive_stats/MultilabelClassification/CEDRClassification.json @@ -0,0 +1,68 @@ +{ + "test": { + "num_samples": 1882, + "number_of_characters": 171649, + "number_texts_in_train": 7, + "min_text_length": 6, + "average_text_length": 91.20563230605738, + "max_text_length": 220, + "unique_texts": 1875, + "min_labels_per_text": 0, + "average_label_per_text": 0.620616365568544, + "max_labels_per_text": 2, + "unique_labels": 6, + "labels": { + "None": { + "count": 734 + }, + "3": { + "count": 141 + }, + "2": { + "count": 170 + }, + "1": { + "count": 379 + }, + "0": { + "count": 353 + }, + "4": { + "count": 125 + } + } + }, + "train": { + "num_samples": 7528, + "number_of_characters": 697322, + "number_texts_in_train": null, + "min_text_length": 5, + "average_text_length": 92.63044633368757, + "max_text_length": 280, + "unique_texts": 7500, + "min_labels_per_text": 0, + "average_label_per_text": 0.6101222104144527, + "max_labels_per_text": 3, + "unique_labels": 6, + "labels": { + "None": { + "count": 3043 + }, + "2": { + "count": 607 + }, + "0": { + "count": 1569 + }, + "3": { + "count": 589 + }, + "1": { + "count": 1417 + }, + "4": { + "count": 411 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/PairClassification/PawsXPairClassification.json b/mteb/descriptive_stats/PairClassification/PawsXPairClassification.json new file mode 100644 index 0000000000..849724bdba --- /dev/null +++ b/mteb/descriptive_stats/PairClassification/PawsXPairClassification.json @@ -0,0 +1,342 @@ +{ + "test": { + "num_samples": 14000, + "number_of_characters": 2551922, + "min_sentence1_length": 2, + "avg_sentence1_length": 91.17892857142857, + "max_sentence1_length": 268, + "unique_sentence1": 13404, + "min_sentence2_length": 2, + "avg_sentence2_length": 91.10121428571429, + "max_sentence2_length": 247, + "unique_sentence2": 13462, + "unique_labels": 2, + "labels": { + "1": { + "count": 6285 + }, + "0": { + "count": 7715 + } + }, + "hf_subset_descriptive_stats": { + "de": { + "num_samples": 2000, + "number_of_characters": 478034, + "min_sentence1_length": 2, + "avg_sentence1_length": 119.7815, + "max_sentence1_length": 268, + "unique_sentence1": 1934, + "min_sentence2_length": 2, + "avg_sentence2_length": 119.2355, + "max_sentence2_length": 235, + "unique_sentence2": 1938, + "unique_labels": 2, + "labels": { + "1": { + "count": 895 + }, + "0": { + "count": 1105 + } + } + }, + "en": { + "num_samples": 2000, + "number_of_characters": 454362, + "min_sentence1_length": 25, + "avg_sentence1_length": 113.7575, + "max_sentence1_length": 209, + "unique_sentence1": 1761, + "min_sentence2_length": 25, + "avg_sentence2_length": 113.4235, + "max_sentence2_length": 209, + "unique_sentence2": 1800, + "unique_labels": 2, + "labels": { + "1": { + "count": 907 + }, + "0": { + "count": 1093 + } + } + }, + "es": { + "num_samples": 2000, + "number_of_characters": 471226, + "min_sentence1_length": 2, + "avg_sentence1_length": 117.815, + "max_sentence1_length": 226, + "unique_sentence1": 1955, + "min_sentence2_length": 22, + "avg_sentence2_length": 117.798, + "max_sentence2_length": 233, + "unique_sentence2": 1959, + "unique_labels": 2, + "labels": { + "1": { + "count": 907 + }, + "0": { + "count": 1093 + } + } + }, + "fr": { + "num_samples": 2000, + "number_of_characters": 480033, + "min_sentence1_length": 2, + "avg_sentence1_length": 120.028, + "max_sentence1_length": 238, + "unique_sentence1": 1954, + "min_sentence2_length": 2, + "avg_sentence2_length": 119.9885, + "max_sentence2_length": 247, + "unique_sentence2": 1953, + "unique_labels": 2, + "labels": { + "1": { + "count": 903 + }, + "0": { + "count": 1097 + } + } + }, + "ja": { + "num_samples": 2000, + "number_of_characters": 235106, + "min_sentence1_length": 2, + "avg_sentence1_length": 58.678, + "max_sentence1_length": 192, + "unique_sentence1": 1944, + "min_sentence2_length": 2, + "avg_sentence2_length": 58.875, + "max_sentence2_length": 198, + "unique_sentence2": 1941, + "unique_labels": 2, + "labels": { + "1": { + "count": 883 + }, + "0": { + "count": 1117 + } + } + }, + "ko": { + "num_samples": 2000, + "number_of_characters": 260149, + "min_sentence1_length": 2, + "avg_sentence1_length": 64.9605, + "max_sentence1_length": 153, + "unique_sentence1": 1954, + "min_sentence2_length": 2, + "avg_sentence2_length": 65.114, + "max_sentence2_length": 159, + "unique_sentence2": 1969, + "unique_labels": 2, + "labels": { + "1": { + "count": 896 + }, + "0": { + "count": 1104 + } + } + }, + "zh": { + "num_samples": 2000, + "number_of_characters": 173012, + "min_sentence1_length": 2, + "avg_sentence1_length": 43.232, + "max_sentence1_length": 120, + "unique_sentence1": 1909, + "min_sentence2_length": 2, + "avg_sentence2_length": 43.274, + "max_sentence2_length": 113, + "unique_sentence2": 1909, + "unique_labels": 2, + "labels": { + "1": { + "count": 894 + }, + "0": { + "count": 1106 + } + } + } + } + }, + "validation": { + "num_samples": 14000, + "number_of_characters": 2524625, + "min_sentence1_length": 2, + "avg_sentence1_length": 90.12585714285714, + "max_sentence1_length": 248, + "unique_sentence1": 13357, + "min_sentence2_length": 2, + "avg_sentence2_length": 90.2045, + "max_sentence2_length": 275, + "unique_sentence2": 13397, + "unique_labels": 2, + "labels": { + "1": { + "count": 5948 + }, + "0": { + "count": 8052 + } + }, + "hf_subset_descriptive_stats": { + "de": { + "num_samples": 2000, + "number_of_characters": 467643, + "min_sentence1_length": 2, + "avg_sentence1_length": 116.82, + "max_sentence1_length": 248, + "unique_sentence1": 1914, + "min_sentence2_length": 2, + "avg_sentence2_length": 117.0015, + "max_sentence2_length": 275, + "unique_sentence2": 1920, + "unique_labels": 2, + "labels": { + "1": { + "count": 831 + }, + "0": { + "count": 1169 + } + } + }, + "en": { + "num_samples": 2000, + "number_of_characters": 451931, + "min_sentence1_length": 25, + "avg_sentence1_length": 113.1075, + "max_sentence1_length": 213, + "unique_sentence1": 1758, + "min_sentence2_length": 25, + "avg_sentence2_length": 112.858, + "max_sentence2_length": 213, + "unique_sentence2": 1771, + "unique_labels": 2, + "labels": { + "1": { + "count": 863 + }, + "0": { + "count": 1137 + } + } + }, + "es": { + "num_samples": 2000, + "number_of_characters": 466112, + "min_sentence1_length": 2, + "avg_sentence1_length": 116.3285, + "max_sentence1_length": 240, + "unique_sentence1": 1938, + "min_sentence2_length": 2, + "avg_sentence2_length": 116.7275, + "max_sentence2_length": 241, + "unique_sentence2": 1941, + "unique_labels": 2, + "labels": { + "1": { + "count": 847 + }, + "0": { + "count": 1153 + } + } + }, + "fr": { + "num_samples": 2000, + "number_of_characters": 478510, + "min_sentence1_length": 2, + "avg_sentence1_length": 119.5045, + "max_sentence1_length": 233, + "unique_sentence1": 1933, + "min_sentence2_length": 2, + "avg_sentence2_length": 119.7505, + "max_sentence2_length": 246, + "unique_sentence2": 1939, + "unique_labels": 2, + "labels": { + "1": { + "count": 860 + }, + "0": { + "count": 1140 + } + } + }, + "ja": { + "num_samples": 2000, + "number_of_characters": 229655, + "min_sentence1_length": 2, + "avg_sentence1_length": 57.5105, + "max_sentence1_length": 126, + "unique_sentence1": 1957, + "min_sentence2_length": 2, + "avg_sentence2_length": 57.317, + "max_sentence2_length": 121, + "unique_sentence2": 1969, + "unique_labels": 2, + "labels": { + "1": { + "count": 854 + }, + "0": { + "count": 1146 + } + } + }, + "ko": { + "num_samples": 2000, + "number_of_characters": 261355, + "min_sentence1_length": 2, + "avg_sentence1_length": 65.162, + "max_sentence1_length": 178, + "unique_sentence1": 1963, + "min_sentence2_length": 2, + "avg_sentence2_length": 65.5155, + "max_sentence2_length": 174, + "unique_sentence2": 1968, + "unique_labels": 2, + "labels": { + "1": { + "count": 840 + }, + "0": { + "count": 1160 + } + } + }, + "zh": { + "num_samples": 2000, + "number_of_characters": 169419, + "min_sentence1_length": 2, + "avg_sentence1_length": 42.448, + "max_sentence1_length": 101, + "unique_sentence1": 1899, + "min_sentence2_length": 2, + "avg_sentence2_length": 42.2615, + "max_sentence2_length": 120, + "unique_sentence2": 1895, + "unique_labels": 2, + "labels": { + "1": { + "count": 853 + }, + "0": { + "count": 1147 + } + } + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/PairClassification/TwitterURLCorpus.json b/mteb/descriptive_stats/PairClassification/TwitterURLCorpus.json new file mode 100644 index 0000000000..473a765dd9 --- /dev/null +++ b/mteb/descriptive_stats/PairClassification/TwitterURLCorpus.json @@ -0,0 +1,23 @@ +{ + "test": { + "num_samples": 51534, + "number_of_characters": 8659940, + "min_sentence1_length": 24, + "avg_sentence1_length": 79.48919160166103, + "max_sentence1_length": 126, + "unique_sentence1": 4329, + "min_sentence2_length": 6, + "avg_sentence2_length": 88.5540419916948, + "max_sentence2_length": 608, + "unique_sentence2": 41304, + "unique_labels": 2, + "labels": { + "0": { + "count": 38546 + }, + "1": { + "count": 12988 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/PairClassification/XNLI.json b/mteb/descriptive_stats/PairClassification/XNLI.json new file mode 100644 index 0000000000..867fafdc85 --- /dev/null +++ b/mteb/descriptive_stats/PairClassification/XNLI.json @@ -0,0 +1,636 @@ +{ + "test": { + "num_samples": 19110, + "number_of_characters": 2907145, + "min_sentence1_length": 3, + "avg_sentence1_length": 103.23793825222397, + "max_sentence1_length": 401, + "unique_sentence1": 15328, + "min_sentence2_length": 2, + "avg_sentence2_length": 48.88895866038723, + "max_sentence2_length": 187, + "unique_sentence2": 19104, + "unique_labels": 2, + "labels": { + "0": { + "count": 9562 + }, + "1": { + "count": 9548 + } + }, + "hf_subset_descriptive_stats": { + "ar": { + "num_samples": 1365, + "number_of_characters": 179591, + "min_sentence1_length": 11, + "avg_sentence1_length": 89.57362637362637, + "max_sentence1_length": 242, + "unique_sentence1": 1095, + "min_sentence2_length": 8, + "avg_sentence2_length": 41.99487179487179, + "max_sentence2_length": 115, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "bg": { + "num_samples": 1365, + "number_of_characters": 220646, + "min_sentence1_length": 14, + "avg_sentence1_length": 110.01611721611722, + "max_sentence1_length": 303, + "unique_sentence1": 1095, + "min_sentence2_length": 8, + "avg_sentence2_length": 51.62930402930403, + "max_sentence2_length": 150, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "de": { + "num_samples": 1365, + "number_of_characters": 241224, + "min_sentence1_length": 3, + "avg_sentence1_length": 119.92600732600732, + "max_sentence1_length": 301, + "unique_sentence1": 1095, + "min_sentence2_length": 9, + "avg_sentence2_length": 56.794871794871796, + "max_sentence2_length": 187, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "el": { + "num_samples": 1365, + "number_of_characters": 240222, + "min_sentence1_length": 13, + "avg_sentence1_length": 119.05421245421246, + "max_sentence1_length": 344, + "unique_sentence1": 1095, + "min_sentence2_length": 13, + "avg_sentence2_length": 56.93260073260073, + "max_sentence2_length": 172, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "en": { + "num_samples": 1365, + "number_of_characters": 212223, + "min_sentence1_length": 19, + "avg_sentence1_length": 105.67032967032966, + "max_sentence1_length": 268, + "unique_sentence1": 1095, + "min_sentence2_length": 9, + "avg_sentence2_length": 49.8043956043956, + "max_sentence2_length": 137, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "es": { + "num_samples": 1365, + "number_of_characters": 232207, + "min_sentence1_length": 11, + "avg_sentence1_length": 115.43296703296703, + "max_sentence1_length": 385, + "unique_sentence1": 1094, + "min_sentence2_length": 8, + "avg_sentence2_length": 54.68205128205128, + "max_sentence2_length": 163, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "fr": { + "num_samples": 1365, + "number_of_characters": 245259, + "min_sentence1_length": 9, + "avg_sentence1_length": 121.0967032967033, + "max_sentence1_length": 327, + "unique_sentence1": 1095, + "min_sentence2_length": 10, + "avg_sentence2_length": 58.58021978021978, + "max_sentence2_length": 169, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "hi": { + "num_samples": 1365, + "number_of_characters": 211312, + "min_sentence1_length": 16, + "avg_sentence1_length": 104.63443223443224, + "max_sentence1_length": 401, + "unique_sentence1": 1095, + "min_sentence2_length": 9, + "avg_sentence2_length": 50.17289377289377, + "max_sentence2_length": 162, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "ru": { + "num_samples": 1365, + "number_of_characters": 222797, + "min_sentence1_length": 11, + "avg_sentence1_length": 110.76923076923077, + "max_sentence1_length": 306, + "unique_sentence1": 1095, + "min_sentence2_length": 8, + "avg_sentence2_length": 52.452014652014654, + "max_sentence2_length": 167, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "sw": { + "num_samples": 1365, + "number_of_characters": 210103, + "min_sentence1_length": 10, + "avg_sentence1_length": 104.43956043956044, + "max_sentence1_length": 266, + "unique_sentence1": 1094, + "min_sentence2_length": 2, + "avg_sentence2_length": 49.48205128205128, + "max_sentence2_length": 146, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "th": { + "num_samples": 1365, + "number_of_characters": 192788, + "min_sentence1_length": 12, + "avg_sentence1_length": 96.6923076923077, + "max_sentence1_length": 262, + "unique_sentence1": 1095, + "min_sentence2_length": 6, + "avg_sentence2_length": 44.544322344322346, + "max_sentence2_length": 129, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "tr": { + "num_samples": 1365, + "number_of_characters": 208658, + "min_sentence1_length": 15, + "avg_sentence1_length": 103.67765567765568, + "max_sentence1_length": 255, + "unique_sentence1": 1095, + "min_sentence2_length": 6, + "avg_sentence2_length": 49.18534798534799, + "max_sentence2_length": 140, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "vi": { + "num_samples": 1365, + "number_of_characters": 223549, + "min_sentence1_length": 14, + "avg_sentence1_length": 111.31208791208792, + "max_sentence1_length": 265, + "unique_sentence1": 1095, + "min_sentence2_length": 9, + "avg_sentence2_length": 52.46007326007326, + "max_sentence2_length": 143, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "zh": { + "num_samples": 1365, + "number_of_characters": 66566, + "min_sentence1_length": 4, + "avg_sentence1_length": 33.03589743589744, + "max_sentence1_length": 112, + "unique_sentence1": 1095, + "min_sentence2_length": 3, + "avg_sentence2_length": 15.73040293040293, + "max_sentence2_length": 59, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + } + } + }, + "validation": { + "num_samples": 19110, + "number_of_characters": 2909058, + "min_sentence1_length": 5, + "avg_sentence1_length": 103.20790162218734, + "max_sentence1_length": 323, + "unique_sentence1": 11171, + "min_sentence2_length": 3, + "avg_sentence2_length": 49.01909994767138, + "max_sentence2_length": 172, + "unique_sentence2": 19101, + "unique_labels": 2, + "labels": { + "0": { + "count": 9562 + }, + "1": { + "count": 9548 + } + }, + "hf_subset_descriptive_stats": { + "ar": { + "num_samples": 1365, + "number_of_characters": 177355, + "min_sentence1_length": 13, + "avg_sentence1_length": 88.31868131868131, + "max_sentence1_length": 214, + "unique_sentence1": 798, + "min_sentence2_length": 6, + "avg_sentence2_length": 41.61172161172161, + "max_sentence2_length": 137, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "bg": { + "num_samples": 1365, + "number_of_characters": 219988, + "min_sentence1_length": 16, + "avg_sentence1_length": 109.196336996337, + "max_sentence1_length": 316, + "unique_sentence1": 798, + "min_sentence2_length": 10, + "avg_sentence2_length": 51.967032967032964, + "max_sentence2_length": 151, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "de": { + "num_samples": 1365, + "number_of_characters": 241852, + "min_sentence1_length": 20, + "avg_sentence1_length": 119.81172161172161, + "max_sentence1_length": 298, + "unique_sentence1": 798, + "min_sentence2_length": 12, + "avg_sentence2_length": 57.36923076923077, + "max_sentence2_length": 162, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "el": { + "num_samples": 1365, + "number_of_characters": 241275, + "min_sentence1_length": 16, + "avg_sentence1_length": 119.87545787545787, + "max_sentence1_length": 302, + "unique_sentence1": 798, + "min_sentence2_length": 6, + "avg_sentence2_length": 56.88278388278388, + "max_sentence2_length": 171, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "en": { + "num_samples": 1365, + "number_of_characters": 212384, + "min_sentence1_length": 20, + "avg_sentence1_length": 105.71648351648352, + "max_sentence1_length": 271, + "unique_sentence1": 798, + "min_sentence2_length": 8, + "avg_sentence2_length": 49.87619047619047, + "max_sentence2_length": 139, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "es": { + "num_samples": 1365, + "number_of_characters": 232451, + "min_sentence1_length": 14, + "avg_sentence1_length": 115.17289377289377, + "max_sentence1_length": 265, + "unique_sentence1": 798, + "min_sentence2_length": 7, + "avg_sentence2_length": 55.120879120879124, + "max_sentence2_length": 148, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "fr": { + "num_samples": 1365, + "number_of_characters": 246857, + "min_sentence1_length": 19, + "avg_sentence1_length": 121.75897435897436, + "max_sentence1_length": 323, + "unique_sentence1": 798, + "min_sentence2_length": 11, + "avg_sentence2_length": 59.08864468864469, + "max_sentence2_length": 172, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "hi": { + "num_samples": 1365, + "number_of_characters": 212269, + "min_sentence1_length": 18, + "avg_sentence1_length": 105.06446886446886, + "max_sentence1_length": 277, + "unique_sentence1": 798, + "min_sentence2_length": 7, + "avg_sentence2_length": 50.44395604395604, + "max_sentence2_length": 152, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "ru": { + "num_samples": 1365, + "number_of_characters": 221152, + "min_sentence1_length": 15, + "avg_sentence1_length": 109.74725274725274, + "max_sentence1_length": 310, + "unique_sentence1": 798, + "min_sentence2_length": 8, + "avg_sentence2_length": 52.26886446886447, + "max_sentence2_length": 140, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "sw": { + "num_samples": 1365, + "number_of_characters": 210482, + "min_sentence1_length": 13, + "avg_sentence1_length": 104.32234432234432, + "max_sentence1_length": 264, + "unique_sentence1": 798, + "min_sentence2_length": 8, + "avg_sentence2_length": 49.87692307692308, + "max_sentence2_length": 153, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "th": { + "num_samples": 1365, + "number_of_characters": 192640, + "min_sentence1_length": 7, + "avg_sentence1_length": 97.28498168498169, + "max_sentence1_length": 255, + "unique_sentence1": 798, + "min_sentence2_length": 3, + "avg_sentence2_length": 43.843223443223444, + "max_sentence2_length": 140, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "tr": { + "num_samples": 1365, + "number_of_characters": 208305, + "min_sentence1_length": 15, + "avg_sentence1_length": 102.96630036630036, + "max_sentence1_length": 269, + "unique_sentence1": 798, + "min_sentence2_length": 10, + "avg_sentence2_length": 49.63809523809524, + "max_sentence2_length": 139, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "vi": { + "num_samples": 1365, + "number_of_characters": 224811, + "min_sentence1_length": 18, + "avg_sentence1_length": 112.26373626373626, + "max_sentence1_length": 323, + "unique_sentence1": 798, + "min_sentence2_length": 9, + "avg_sentence2_length": 52.432967032967035, + "max_sentence2_length": 159, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + }, + "zh": { + "num_samples": 1365, + "number_of_characters": 67237, + "min_sentence1_length": 5, + "avg_sentence1_length": 33.41098901098901, + "max_sentence1_length": 135, + "unique_sentence1": 798, + "min_sentence2_length": 3, + "avg_sentence2_length": 15.846886446886447, + "max_sentence2_length": 66, + "unique_sentence2": 1365, + "unique_labels": 2, + "labels": { + "0": { + "count": 683 + }, + "1": { + "count": 682 + } + } + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Reranking/AskUbuntuDupQuestions.json b/mteb/descriptive_stats/Reranking/AskUbuntuDupQuestions.json new file mode 100644 index 0000000000..a0ced7def7 --- /dev/null +++ b/mteb/descriptive_stats/Reranking/AskUbuntuDupQuestions.json @@ -0,0 +1,20 @@ +{ + "test": { + "num_samples": 375, + "number_of_characters": 413674, + "num_positive": 2255, + "num_negative": 5245, + "min_query_length": 17, + "avg_query_length": 50.205333333333336, + "max_query_length": 148, + "unique_query": 374, + "min_positive_length": 15, + "avg_positive_length": 52.54013303769401, + "max_positive_length": 152, + "unique_positive": 2165, + "min_negative_length": 15, + "avg_negative_length": 52.69189704480458, + "max_negative_length": 148, + "unique_negative": 5002 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Reranking/ESCIReranking.json b/mteb/descriptive_stats/Reranking/ESCIReranking.json new file mode 100644 index 0000000000..9c9556be9d --- /dev/null +++ b/mteb/descriptive_stats/Reranking/ESCIReranking.json @@ -0,0 +1,76 @@ +{ + "test": { + "num_samples": 29285, + "number_of_characters": 254538331, + "num_positive": 271416, + "num_negative": 44235, + "min_query_length": 1, + "avg_query_length": 19.691890046098685, + "max_query_length": 151, + "unique_query": 29269, + "min_positive_length": 1, + "avg_positive_length": 803.9230995961918, + "max_positive_length": 8640, + "unique_positive": 217712, + "min_negative_length": 1, + "avg_negative_length": 808.501458121397, + "max_negative_length": 4441, + "unique_negative": 39551, + "hf_subset_descriptive_stats": { + "us": { + "num_samples": 21296, + "number_of_characters": 186915609, + "num_positive": 189375, + "num_negative": 25463, + "min_query_length": 1, + "avg_query_length": 21.440833959429, + "max_query_length": 151, + "unique_query": 21296, + "min_positive_length": 1, + "avg_positive_length": 868.3698006600661, + "max_positive_length": 5545, + "unique_positive": 150734, + "min_negative_length": 1, + "avg_negative_length": 864.4493578918431, + "max_negative_length": 3779, + "unique_negative": 23073 + }, + "es": { + "num_samples": 3703, + "number_of_characters": 48861389, + "num_positive": 39110, + "num_negative": 10183, + "min_query_length": 3, + "avg_query_length": 20.681609505806104, + "max_query_length": 59, + "unique_query": 3703, + "min_positive_length": 1, + "avg_positive_length": 980.9613142418818, + "max_positive_length": 8640, + "unique_positive": 32921, + "min_negative_length": 1, + "avg_negative_length": 1023.2159481488756, + "max_negative_length": 4441, + "unique_negative": 9285 + }, + "jp": { + "num_samples": 4286, + "number_of_characters": 18761333, + "num_positive": 42931, + "num_negative": 8589, + "min_query_length": 1, + "avg_query_length": 10.146756882874476, + "max_query_length": 60, + "unique_query": 4286, + "min_positive_length": 1, + "avg_positive_length": 358.35792317905475, + "max_positive_length": 3488, + "unique_positive": 35165, + "min_negative_length": 1, + "avg_negative_length": 388.075445337059, + "max_negative_length": 3940, + "unique_negative": 7289 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Reranking/WikipediaRerankingMultilingual.json b/mteb/descriptive_stats/Reranking/WikipediaRerankingMultilingual.json new file mode 100644 index 0000000000..0506ff39e5 --- /dev/null +++ b/mteb/descriptive_stats/Reranking/WikipediaRerankingMultilingual.json @@ -0,0 +1,310 @@ +{ + "test": { + "num_samples": 24000, + "number_of_characters": 83866932, + "num_positive": 24000, + "num_negative": 192000, + "min_query_length": 7, + "avg_query_length": 59.091208333333334, + "max_query_length": 180, + "unique_query": 23997, + "min_positive_length": 100, + "avg_positive_length": 385.45120833333334, + "max_positive_length": 3515, + "unique_positive": 23993, + "min_negative_length": 100, + "avg_negative_length": 381.23913541666667, + "max_negative_length": 9461, + "unique_negative": 191783, + "hf_subset_descriptive_stats": { + "bg": { + "num_samples": 1500, + "number_of_characters": 5145316, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 18, + "avg_query_length": 60.82666666666667, + "max_query_length": 166, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 375.88866666666667, + "max_positive_length": 2241, + "unique_positive": 1500, + "min_negative_length": 100, + "avg_negative_length": 374.18691666666666, + "max_negative_length": 4869, + "unique_negative": 11996 + }, + "bn": { + "num_samples": 1500, + "number_of_characters": 5390581, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 7, + "avg_query_length": 47.266666666666666, + "max_query_length": 123, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 394.5946666666667, + "max_positive_length": 2338, + "unique_positive": 1499, + "min_negative_length": 100, + "avg_negative_length": 393.98241666666667, + "max_negative_length": 5104, + "unique_negative": 11996 + }, + "cs": { + "num_samples": 1500, + "number_of_characters": 5079180, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 17, + "avg_query_length": 56.272, + "max_query_length": 137, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 383.8446666666667, + "max_positive_length": 2300, + "unique_positive": 1499, + "min_negative_length": 100, + "avg_negative_length": 368.2504166666667, + "max_negative_length": 3487, + "unique_negative": 11982 + }, + "da": { + "num_samples": 1500, + "number_of_characters": 4746132, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 17, + "avg_query_length": 56.75066666666667, + "max_query_length": 137, + "unique_query": 1499, + "min_positive_length": 100, + "avg_positive_length": 351.6813333333333, + "max_positive_length": 2159, + "unique_positive": 1500, + "min_negative_length": 100, + "avg_negative_length": 344.457, + "max_negative_length": 2563, + "unique_negative": 11972 + }, + "de": { + "num_samples": 1500, + "number_of_characters": 5483592, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 20, + "avg_query_length": 70.004, + "max_query_length": 180, + "unique_query": 1499, + "min_positive_length": 100, + "avg_positive_length": 391.5366666666667, + "max_positive_length": 2674, + "unique_positive": 1500, + "min_negative_length": 100, + "avg_negative_length": 399.27341666666666, + "max_negative_length": 3083, + "unique_negative": 12000 + }, + "en": { + "num_samples": 1500, + "number_of_characters": 6217884, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 18, + "avg_query_length": 68.372, + "max_query_length": 162, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 451.72866666666664, + "max_positive_length": 3515, + "unique_positive": 1500, + "min_negative_length": 100, + "avg_negative_length": 453.14441666666664, + "max_negative_length": 3662, + "unique_negative": 12000 + }, + "fa": { + "num_samples": 1500, + "number_of_characters": 4732619, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 12, + "avg_query_length": 48.66733333333333, + "max_query_length": 119, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 347.704, + "max_positive_length": 2571, + "unique_positive": 1500, + "min_negative_length": 100, + "avg_negative_length": 344.8385, + "max_negative_length": 4707, + "unique_negative": 11978 + }, + "fi": { + "num_samples": 1500, + "number_of_characters": 5209132, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 14, + "avg_query_length": 55.343333333333334, + "max_query_length": 132, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 394.7126666666667, + "max_positive_length": 2129, + "unique_positive": 1498, + "min_negative_length": 100, + "avg_negative_length": 377.83733333333333, + "max_negative_length": 2574, + "unique_negative": 11972 + }, + "hi": { + "num_samples": 1500, + "number_of_characters": 5620959, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 13, + "avg_query_length": 50.77733333333333, + "max_query_length": 125, + "unique_query": 1499, + "min_positive_length": 100, + "avg_positive_length": 420.3786666666667, + "max_positive_length": 2361, + "unique_positive": 1500, + "min_negative_length": 100, + "avg_negative_length": 409.51875, + "max_negative_length": 5912, + "unique_negative": 11996 + }, + "it": { + "num_samples": 1500, + "number_of_characters": 5420496, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 23, + "avg_query_length": 70.05466666666666, + "max_query_length": 156, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 396.97333333333336, + "max_positive_length": 2082, + "unique_positive": 1500, + "min_negative_length": 100, + "avg_negative_length": 393.3295, + "max_negative_length": 9461, + "unique_negative": 11993 + }, + "nl": { + "num_samples": 1500, + "number_of_characters": 5169556, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 18, + "avg_query_length": 65.34466666666667, + "max_query_length": 136, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 380.79133333333334, + "max_positive_length": 1864, + "unique_positive": 1500, + "min_negative_length": 100, + "avg_negative_length": 375.02933333333334, + "max_negative_length": 3641, + "unique_negative": 11985 + }, + "pt": { + "num_samples": 1500, + "number_of_characters": 5474356, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 18, + "avg_query_length": 65.11933333333333, + "max_query_length": 176, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 404.01933333333335, + "max_positive_length": 3057, + "unique_positive": 1499, + "min_negative_length": 100, + "avg_negative_length": 397.554, + "max_negative_length": 2877, + "unique_negative": 11991 + }, + "ro": { + "num_samples": 1500, + "number_of_characters": 4796113, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 14, + "avg_query_length": 61.973333333333336, + "max_query_length": 169, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 346.70866666666666, + "max_positive_length": 1917, + "unique_positive": 1499, + "min_negative_length": 100, + "avg_negative_length": 348.5908333333333, + "max_negative_length": 4213, + "unique_negative": 11971 + }, + "sr": { + "num_samples": 1500, + "number_of_characters": 5271732, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 15, + "avg_query_length": 55.669333333333334, + "max_query_length": 146, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 386.34933333333333, + "max_positive_length": 2421, + "unique_positive": 1499, + "min_negative_length": 100, + "avg_negative_length": 384.0586666666667, + "max_negative_length": 3668, + "unique_negative": 11974 + }, + "no": { + "num_samples": 1500, + "number_of_characters": 5036586, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 14, + "avg_query_length": 55.288, + "max_query_length": 129, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 367.72, + "max_positive_length": 1450, + "unique_positive": 1500, + "min_negative_length": 100, + "avg_negative_length": 366.8395, + "max_negative_length": 2841, + "unique_negative": 11996 + }, + "sv": { + "num_samples": 1500, + "number_of_characters": 5072698, + "num_positive": 1500, + "num_negative": 12000, + "min_query_length": 17, + "avg_query_length": 57.73, + "max_query_length": 133, + "unique_query": 1500, + "min_positive_length": 100, + "avg_positive_length": 372.58733333333333, + "max_positive_length": 2493, + "unique_positive": 1500, + "min_negative_length": 100, + "avg_negative_length": 368.93516666666665, + "max_negative_length": 3680, + "unique_negative": 11999 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/AppsRetrieval.json b/mteb/descriptive_stats/Retrieval/AppsRetrieval.json new file mode 100644 index 0000000000..caaab2453b --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/AppsRetrieval.json @@ -0,0 +1,20 @@ +{ + "test": { + "number_of_characters": 11335620, + "num_samples": 12530, + "num_queries": 3765, + "num_documents": 8765, + "min_document_length": 152, + "average_document_length": 717.2737022247576, + "max_document_length": 5742, + "unique_documents": 8765, + "min_query_length": 6, + "average_query_length": 1340.9604249667996, + "max_query_length": 289049, + "unique_queries": 3765, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 3765 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/AutoRAGRetrieval.json b/mteb/descriptive_stats/Retrieval/AutoRAGRetrieval.json new file mode 100644 index 0000000000..62c9124a48 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/AutoRAGRetrieval.json @@ -0,0 +1,11 @@ +{ + "test": { + "number_of_characters": 894.2168128654971, + "num_samples": 834, + "num_queries": 114, + "num_documents": 720, + "average_document_length": 1.1452816358024691, + "average_query_length": 0.610649430594029, + "average_relevant_docs_per_query": 1.0 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/BelebeleRetrieval.json b/mteb/descriptive_stats/Retrieval/BelebeleRetrieval.json new file mode 100644 index 0000000000..78c8a7e121 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/BelebeleRetrieval.json @@ -0,0 +1,6790 @@ +{ + "test": { + "number_of_characters": 25574620, + "num_samples": 521866, + "num_queries": 338378, + "num_documents": 183488, + "min_document_length": 4, + "average_document_length": 137.38034094872688, + "max_document_length": 237, + "unique_documents": 183488, + "min_query_length": 2, + "average_query_length": 1.0845149507355678, + "max_query_length": 2, + "unique_queries": 338378, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0000413738481817, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 183488, + "hf_subset_descriptive_stats": { + "acm_Arab-acm_Arab": { + "number_of_characters": 51232, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 102.98360655737704, + "max_document_length": 129, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "acm_Arab-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-acm_Arab": { + "number_of_characters": 51232, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 102.98360655737704, + "max_document_length": 129, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "afr_Latn-afr_Latn": { + "number_of_characters": 71217, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 143.93647540983608, + "max_document_length": 159, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "afr_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-afr_Latn": { + "number_of_characters": 71217, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 143.93647540983608, + "max_document_length": 159, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "als_Latn-als_Latn": { + "number_of_characters": 69498, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 140.4139344262295, + "max_document_length": 175, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "als_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-als_Latn": { + "number_of_characters": 69498, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 140.4139344262295, + "max_document_length": 175, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "amh_Ethi-amh_Ethi": { + "number_of_characters": 45221, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 90.66598360655738, + "max_document_length": 100, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "amh_Ethi-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-amh_Ethi": { + "number_of_characters": 45221, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 90.66598360655738, + "max_document_length": 100, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "apc_Arab-apc_Arab": { + "number_of_characters": 51248, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 103.01639344262296, + "max_document_length": 134, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "apc_Arab-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-apc_Arab": { + "number_of_characters": 51248, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 103.01639344262296, + "max_document_length": 134, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "arb_Arab-arb_Arab": { + "number_of_characters": 53671, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 107.98155737704919, + "max_document_length": 134, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "arb_Arab-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-arb_Arab": { + "number_of_characters": 53671, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 107.98155737704919, + "max_document_length": 134, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "arb_Latn-arb_Latn": { + "number_of_characters": 61298, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 123.61065573770492, + "max_document_length": 160, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "arb_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-arb_Latn": { + "number_of_characters": 61298, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 123.61065573770492, + "max_document_length": 160, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ars_Arab-ars_Arab": { + "number_of_characters": 51765, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 104.07581967213115, + "max_document_length": 119, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ars_Arab-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ars_Arab": { + "number_of_characters": 51765, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 104.07581967213115, + "max_document_length": 119, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ary_Arab-ary_Arab": { + "number_of_characters": 60261, + "num_samples": 1386, + "num_queries": 898, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 121.48565573770492, + "max_document_length": 138, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.086859688195991, + "max_query_length": 2, + "unique_queries": 898, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ary_Arab-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ary_Arab": { + "number_of_characters": 60261, + "num_samples": 1386, + "num_queries": 898, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 121.48565573770492, + "max_document_length": 138, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.086859688195991, + "max_query_length": 2, + "unique_queries": 898, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "arz_Arab-arz_Arab": { + "number_of_characters": 52403, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 105.38319672131148, + "max_document_length": 115, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "arz_Arab-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-arz_Arab": { + "number_of_characters": 52403, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 105.38319672131148, + "max_document_length": 115, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "asm_Beng-asm_Beng": { + "number_of_characters": 62410, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 4, + "average_document_length": 125.88934426229508, + "max_document_length": 158, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "asm_Beng-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-asm_Beng": { + "number_of_characters": 62410, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 4, + "average_document_length": 125.88934426229508, + "max_document_length": 158, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "azj_Latn-azj_Latn": { + "number_of_characters": 67137, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 135.57581967213116, + "max_document_length": 156, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "azj_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-azj_Latn": { + "number_of_characters": 67137, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 135.57581967213116, + "max_document_length": 156, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "bam_Latn-bam_Latn": { + "number_of_characters": 66084, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 133.41803278688525, + "max_document_length": 166, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "bam_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-bam_Latn": { + "number_of_characters": 66084, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 133.41803278688525, + "max_document_length": 166, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ben_Beng-ben_Beng": { + "number_of_characters": 63512, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 9, + "average_document_length": 128.14754098360655, + "max_document_length": 175, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ben_Beng-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ben_Beng": { + "number_of_characters": 63512, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 9, + "average_document_length": 128.14754098360655, + "max_document_length": 175, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ben_Latn-ben_Latn": { + "number_of_characters": 68285, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 9, + "average_document_length": 137.92827868852459, + "max_document_length": 185, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ben_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ben_Latn": { + "number_of_characters": 68285, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 9, + "average_document_length": 137.92827868852459, + "max_document_length": 185, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "bod_Tibt-bod_Tibt": { + "number_of_characters": 79188, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 160.2704918032787, + "max_document_length": 213, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "bod_Tibt-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-bod_Tibt": { + "number_of_characters": 79188, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 160.2704918032787, + "max_document_length": 213, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "bul_Cyrl-bul_Cyrl": { + "number_of_characters": 66577, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 134.42827868852459, + "max_document_length": 177, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "bul_Cyrl-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-bul_Cyrl": { + "number_of_characters": 66577, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 134.42827868852459, + "max_document_length": 177, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "cat_Latn-cat_Latn": { + "number_of_characters": 68842, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 139.06967213114754, + "max_document_length": 163, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "cat_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-cat_Latn": { + "number_of_characters": 68842, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 139.06967213114754, + "max_document_length": 163, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ceb_Latn-ceb_Latn": { + "number_of_characters": 74053, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 149.74795081967213, + "max_document_length": 184, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ceb_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ceb_Latn": { + "number_of_characters": 74053, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 149.74795081967213, + "max_document_length": 184, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ces_Latn-ces_Latn": { + "number_of_characters": 61936, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 124.91803278688525, + "max_document_length": 139, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ces_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ces_Latn": { + "number_of_characters": 61936, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 124.91803278688525, + "max_document_length": 139, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ckb_Arab-ckb_Arab": { + "number_of_characters": 64917, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 131.0266393442623, + "max_document_length": 178, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ckb_Arab-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ckb_Arab": { + "number_of_characters": 64917, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 131.0266393442623, + "max_document_length": 178, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "dan_Latn-dan_Latn": { + "number_of_characters": 66648, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 134.5737704918033, + "max_document_length": 159, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "dan_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-dan_Latn": { + "number_of_characters": 66648, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 134.5737704918033, + "max_document_length": 159, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "deu_Latn-deu_Latn": { + "number_of_characters": 68768, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 138.91803278688525, + "max_document_length": 182, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "deu_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-deu_Latn": { + "number_of_characters": 68768, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 138.91803278688525, + "max_document_length": 182, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ell_Grek-ell_Grek": { + "number_of_characters": 79210, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 160.3155737704918, + "max_document_length": 212, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ell_Grek-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ell_Grek": { + "number_of_characters": 79210, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 160.3155737704918, + "max_document_length": 212, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "est_Latn-est_Latn": { + "number_of_characters": 61779, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 124.59631147540983, + "max_document_length": 164, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "est_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-est_Latn": { + "number_of_characters": 61779, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 124.59631147540983, + "max_document_length": 164, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eus_Latn-eus_Latn": { + "number_of_characters": 67979, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 137.3012295081967, + "max_document_length": 169, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eus_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-eus_Latn": { + "number_of_characters": 67979, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 137.3012295081967, + "max_document_length": 169, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "fin_Latn-fin_Latn": { + "number_of_characters": 66234, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 133.72540983606558, + "max_document_length": 161, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "fin_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-fin_Latn": { + "number_of_characters": 66234, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 133.72540983606558, + "max_document_length": 161, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "fra_Latn-fra_Latn": { + "number_of_characters": 82464, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 19, + "average_document_length": 166.98360655737704, + "max_document_length": 204, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "fra_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-fra_Latn": { + "number_of_characters": 82464, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 19, + "average_document_length": 166.98360655737704, + "max_document_length": 204, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "fuv_Latn-fuv_Latn": { + "number_of_characters": 53555, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 107.7438524590164, + "max_document_length": 122, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "fuv_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-fuv_Latn": { + "number_of_characters": 53555, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 107.7438524590164, + "max_document_length": 122, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "gaz_Latn-gaz_Latn": { + "number_of_characters": 78315, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 158.48155737704917, + "max_document_length": 191, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "gaz_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-gaz_Latn": { + "number_of_characters": 78315, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 158.48155737704917, + "max_document_length": 191, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "grn_Latn-grn_Latn": { + "number_of_characters": 68572, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 138.51639344262296, + "max_document_length": 161, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "grn_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-grn_Latn": { + "number_of_characters": 68572, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 138.51639344262296, + "max_document_length": 161, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "guj_Gujr-guj_Gujr": { + "number_of_characters": 57007, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 114.81762295081967, + "max_document_length": 138, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "guj_Gujr-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-guj_Gujr": { + "number_of_characters": 57007, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 114.81762295081967, + "max_document_length": 138, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hat_Latn-hat_Latn": { + "number_of_characters": 64558, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 130.29098360655738, + "max_document_length": 179, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hat_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-hat_Latn": { + "number_of_characters": 64558, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 130.29098360655738, + "max_document_length": 179, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hau_Latn-hau_Latn": { + "number_of_characters": 78240, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 158.327868852459, + "max_document_length": 183, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hau_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-hau_Latn": { + "number_of_characters": 78240, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 158.327868852459, + "max_document_length": 183, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "heb_Hebr-heb_Hebr": { + "number_of_characters": 50598, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 101.68442622950819, + "max_document_length": 134, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "heb_Hebr-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-heb_Hebr": { + "number_of_characters": 50598, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 101.68442622950819, + "max_document_length": 134, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hin_Deva-hin_Deva": { + "number_of_characters": 66332, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 133.9262295081967, + "max_document_length": 165, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hin_Deva-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-hin_Deva": { + "number_of_characters": 66332, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 133.9262295081967, + "max_document_length": 165, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hin_Latn-hin_Latn": { + "number_of_characters": 68307, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 137.9733606557377, + "max_document_length": 170, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hin_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-hin_Latn": { + "number_of_characters": 68307, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 137.9733606557377, + "max_document_length": 170, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hrv_Latn-hrv_Latn": { + "number_of_characters": 62928, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 126.95081967213115, + "max_document_length": 175, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hrv_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-hrv_Latn": { + "number_of_characters": 62928, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 126.95081967213115, + "max_document_length": 175, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hun_Latn-hun_Latn": { + "number_of_characters": 67941, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 137.2233606557377, + "max_document_length": 176, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hun_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-hun_Latn": { + "number_of_characters": 67941, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 137.2233606557377, + "max_document_length": 176, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hye_Armn-hye_Armn": { + "number_of_characters": 68859, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 139.1045081967213, + "max_document_length": 193, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hye_Armn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-hye_Armn": { + "number_of_characters": 68859, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 139.1045081967213, + "max_document_length": 193, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ibo_Latn-ibo_Latn": { + "number_of_characters": 66167, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 19, + "average_document_length": 133.58811475409837, + "max_document_length": 156, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0011123470522802, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "ibo_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ibo_Latn": { + "number_of_characters": 66167, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 19, + "average_document_length": 133.58811475409837, + "max_document_length": 156, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0011123470522802, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "ilo_Latn-ilo_Latn": { + "number_of_characters": 78161, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 158.16598360655738, + "max_document_length": 187, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ilo_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ilo_Latn": { + "number_of_characters": 78161, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 158.16598360655738, + "max_document_length": 187, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ind_Latn-ind_Latn": { + "number_of_characters": 74871, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 19, + "average_document_length": 151.42418032786884, + "max_document_length": 207, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ind_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ind_Latn": { + "number_of_characters": 74871, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 19, + "average_document_length": 151.42418032786884, + "max_document_length": 207, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "isl_Latn-isl_Latn": { + "number_of_characters": 70522, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 142.5122950819672, + "max_document_length": 170, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "isl_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-isl_Latn": { + "number_of_characters": 70522, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 142.5122950819672, + "max_document_length": 170, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ita_Latn-ita_Latn": { + "number_of_characters": 76124, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 153.99180327868854, + "max_document_length": 185, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ita_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ita_Latn": { + "number_of_characters": 76124, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 153.99180327868854, + "max_document_length": 185, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "jav_Latn-jav_Latn": { + "number_of_characters": 71722, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 144.97131147540983, + "max_document_length": 174, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "jav_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-jav_Latn": { + "number_of_characters": 71722, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 144.97131147540983, + "max_document_length": 174, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "jpn_Jpan-jpn_Jpan": { + "number_of_characters": 33187, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 66.0061475409836, + "max_document_length": 76, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "jpn_Jpan-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-jpn_Jpan": { + "number_of_characters": 33187, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 66.0061475409836, + "max_document_length": 76, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kac_Latn-kac_Latn": { + "number_of_characters": 89655, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 181.71926229508196, + "max_document_length": 195, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kac_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-kac_Latn": { + "number_of_characters": 89655, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 181.71926229508196, + "max_document_length": 195, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kan_Knda-kan_Knda": { + "number_of_characters": 65899, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 133.0389344262295, + "max_document_length": 165, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kan_Knda-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-kan_Knda": { + "number_of_characters": 65899, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 133.0389344262295, + "max_document_length": 165, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kat_Geor-kat_Geor": { + "number_of_characters": 68309, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 137.97745901639345, + "max_document_length": 175, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kat_Geor-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-kat_Geor": { + "number_of_characters": 68309, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 137.97745901639345, + "max_document_length": 175, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kaz_Cyrl-kaz_Cyrl": { + "number_of_characters": 64657, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 130.49385245901638, + "max_document_length": 158, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kaz_Cyrl-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-kaz_Cyrl": { + "number_of_characters": 64657, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 130.49385245901638, + "max_document_length": 158, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kea_Latn-kea_Latn": { + "number_of_characters": 69323, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 140.05532786885246, + "max_document_length": 183, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kea_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-kea_Latn": { + "number_of_characters": 69323, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 140.05532786885246, + "max_document_length": 183, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "khk_Cyrl-khk_Cyrl": { + "number_of_characters": 66977, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 135.24795081967213, + "max_document_length": 162, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "khk_Cyrl-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-khk_Cyrl": { + "number_of_characters": 66977, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 135.24795081967213, + "max_document_length": 162, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "khm_Khmr-khm_Khmr": { + "number_of_characters": 69150, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 139.70081967213116, + "max_document_length": 169, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "khm_Khmr-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-khm_Khmr": { + "number_of_characters": 69150, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 139.70081967213116, + "max_document_length": 169, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kin_Latn-kin_Latn": { + "number_of_characters": 72803, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 147.18647540983608, + "max_document_length": 194, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0011123470522802, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "kin_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-kin_Latn": { + "number_of_characters": 72803, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 147.18647540983608, + "max_document_length": 194, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0011123470522802, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "kir_Cyrl-kir_Cyrl": { + "number_of_characters": 67957, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 137.25614754098362, + "max_document_length": 182, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kir_Cyrl-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-kir_Cyrl": { + "number_of_characters": 67957, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 137.25614754098362, + "max_document_length": 182, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kor_Hang-kor_Hang": { + "number_of_characters": 32708, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 65.02459016393442, + "max_document_length": 88, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "kor_Hang-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-kor_Hang": { + "number_of_characters": 32708, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 65.02459016393442, + "max_document_length": 88, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "lao_Laoo-lao_Laoo": { + "number_of_characters": 57958, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 116.76639344262296, + "max_document_length": 142, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "lao_Laoo-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-lao_Laoo": { + "number_of_characters": 57958, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 116.76639344262296, + "max_document_length": 142, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "lin_Latn-lin_Latn": { + "number_of_characters": 74223, + "num_samples": 1386, + "num_queries": 898, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 150.09631147540983, + "max_document_length": 183, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.086859688195991, + "max_query_length": 2, + "unique_queries": 898, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0022271714922049, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "lin_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-lin_Latn": { + "number_of_characters": 74223, + "num_samples": 1386, + "num_queries": 898, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 150.09631147540983, + "max_document_length": 183, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.086859688195991, + "max_query_length": 2, + "unique_queries": 898, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0022271714922049, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "lit_Latn-lit_Latn": { + "number_of_characters": 62805, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 126.69877049180327, + "max_document_length": 167, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "lit_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-lit_Latn": { + "number_of_characters": 62805, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 126.69877049180327, + "max_document_length": 167, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "lug_Latn-lug_Latn": { + "number_of_characters": 71566, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 144.6516393442623, + "max_document_length": 237, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0011123470522802, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "lug_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-lug_Latn": { + "number_of_characters": 71566, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 144.6516393442623, + "max_document_length": 237, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0011123470522802, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "luo_Latn-luo_Latn": { + "number_of_characters": 66805, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 134.8954918032787, + "max_document_length": 178, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "luo_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-luo_Latn": { + "number_of_characters": 66805, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 134.8954918032787, + "max_document_length": 178, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "lvs_Latn-lvs_Latn": { + "number_of_characters": 63957, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 129.0594262295082, + "max_document_length": 172, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "lvs_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-lvs_Latn": { + "number_of_characters": 63957, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 129.0594262295082, + "max_document_length": 172, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "mal_Mlym-mal_Mlym": { + "number_of_characters": 73599, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 148.81762295081967, + "max_document_length": 191, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "mal_Mlym-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-mal_Mlym": { + "number_of_characters": 73599, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 148.81762295081967, + "max_document_length": 191, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "mar_Deva-mar_Deva": { + "number_of_characters": 62671, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 126.42418032786885, + "max_document_length": 160, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0011123470522802, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "mar_Deva-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-mar_Deva": { + "number_of_characters": 62671, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 126.42418032786885, + "max_document_length": 160, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0011123470522802, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "mkd_Cyrl-mkd_Cyrl": { + "number_of_characters": 67588, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 136.5, + "max_document_length": 180, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "mkd_Cyrl-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-mkd_Cyrl": { + "number_of_characters": 67588, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 136.5, + "max_document_length": 180, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "mlt_Latn-mlt_Latn": { + "number_of_characters": 68480, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 138.327868852459, + "max_document_length": 185, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "mlt_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-mlt_Latn": { + "number_of_characters": 68480, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 138.327868852459, + "max_document_length": 185, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "mri_Latn-mri_Latn": { + "number_of_characters": 74519, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 150.702868852459, + "max_document_length": 185, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "mri_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-mri_Latn": { + "number_of_characters": 74519, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 150.702868852459, + "max_document_length": 185, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "mya_Mymr-mya_Mymr": { + "number_of_characters": 81331, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 164.66188524590163, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "mya_Mymr-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-mya_Mymr": { + "number_of_characters": 81331, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 164.66188524590163, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "nld_Latn-nld_Latn": { + "number_of_characters": 68789, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 138.9610655737705, + "max_document_length": 183, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "nld_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-nld_Latn": { + "number_of_characters": 68789, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 138.9610655737705, + "max_document_length": 183, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "nob_Latn-nob_Latn": { + "number_of_characters": 64917, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 131.0266393442623, + "max_document_length": 168, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "nob_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-nob_Latn": { + "number_of_characters": 64917, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 131.0266393442623, + "max_document_length": 168, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "npi_Deva-npi_Deva": { + "number_of_characters": 61183, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 123.375, + "max_document_length": 154, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "npi_Deva-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-npi_Deva": { + "number_of_characters": 61183, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 123.375, + "max_document_length": 154, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "npi_Latn-npi_Latn": { + "number_of_characters": 65683, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 20, + "average_document_length": 132.59631147540983, + "max_document_length": 154, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "npi_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-npi_Latn": { + "number_of_characters": 65683, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 20, + "average_document_length": 132.59631147540983, + "max_document_length": 154, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "nso_Latn-nso_Latn": { + "number_of_characters": 79073, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 160.03483606557376, + "max_document_length": 235, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "nso_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-nso_Latn": { + "number_of_characters": 79073, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 160.03483606557376, + "max_document_length": 235, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "nya_Latn-nya_Latn": { + "number_of_characters": 82685, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 167.43647540983608, + "max_document_length": 215, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "nya_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-nya_Latn": { + "number_of_characters": 82685, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 167.43647540983608, + "max_document_length": 215, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ory_Orya-ory_Orya": { + "number_of_characters": 66638, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 10, + "average_document_length": 134.55327868852459, + "max_document_length": 168, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ory_Orya-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ory_Orya": { + "number_of_characters": 66638, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 10, + "average_document_length": 134.55327868852459, + "max_document_length": 168, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "pan_Guru-pan_Guru": { + "number_of_characters": 66944, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 135.18032786885246, + "max_document_length": 157, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "pan_Guru-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-pan_Guru": { + "number_of_characters": 66944, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 135.18032786885246, + "max_document_length": 157, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "pbt_Arab-pbt_Arab": { + "number_of_characters": 61880, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 124.80327868852459, + "max_document_length": 155, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "pbt_Arab-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-pbt_Arab": { + "number_of_characters": 61880, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 124.80327868852459, + "max_document_length": 155, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "pes_Arab-pes_Arab": { + "number_of_characters": 59252, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 119.41803278688525, + "max_document_length": 152, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "pes_Arab-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-pes_Arab": { + "number_of_characters": 59252, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 119.41803278688525, + "max_document_length": 152, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "plt_Latn-plt_Latn": { + "number_of_characters": 86472, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 175.19672131147541, + "max_document_length": 222, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "plt_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-plt_Latn": { + "number_of_characters": 86472, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 175.19672131147541, + "max_document_length": 222, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "pol_Latn-pol_Latn": { + "number_of_characters": 67664, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 136.65573770491804, + "max_document_length": 196, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "pol_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-pol_Latn": { + "number_of_characters": 67664, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 136.65573770491804, + "max_document_length": 196, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "por_Latn-por_Latn": { + "number_of_characters": 71281, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 144.06762295081967, + "max_document_length": 179, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "por_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-por_Latn": { + "number_of_characters": 71281, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 144.06762295081967, + "max_document_length": 179, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ron_Latn-ron_Latn": { + "number_of_characters": 71844, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 145.22131147540983, + "max_document_length": 181, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ron_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ron_Latn": { + "number_of_characters": 71844, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 145.22131147540983, + "max_document_length": 181, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "rus_Cyrl-rus_Cyrl": { + "number_of_characters": 75823, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 153.375, + "max_document_length": 196, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "rus_Cyrl-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-rus_Cyrl": { + "number_of_characters": 75823, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 153.375, + "max_document_length": 196, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "shn_Mymr-shn_Mymr": { + "number_of_characters": 69288, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 139.98360655737704, + "max_document_length": 159, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "shn_Mymr-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-shn_Mymr": { + "number_of_characters": 69288, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 139.98360655737704, + "max_document_length": 159, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sin_Latn-sin_Latn": { + "number_of_characters": 85996, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 19, + "average_document_length": 174.22131147540983, + "max_document_length": 224, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sin_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-sin_Latn": { + "number_of_characters": 85996, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 19, + "average_document_length": 174.22131147540983, + "max_document_length": 224, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sin_Sinh-sin_Sinh": { + "number_of_characters": 63902, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 128.94672131147541, + "max_document_length": 159, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sin_Sinh-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-sin_Sinh": { + "number_of_characters": 63902, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 128.94672131147541, + "max_document_length": 159, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "slk_Latn-slk_Latn": { + "number_of_characters": 62663, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 126.4077868852459, + "max_document_length": 146, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "slk_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-slk_Latn": { + "number_of_characters": 62663, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 126.4077868852459, + "max_document_length": 146, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "slv_Latn-slv_Latn": { + "number_of_characters": 62895, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 126.88319672131148, + "max_document_length": 176, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "slv_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-slv_Latn": { + "number_of_characters": 62895, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 126.88319672131148, + "max_document_length": 176, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sna_Latn-sna_Latn": { + "number_of_characters": 74071, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 20, + "average_document_length": 149.78483606557376, + "max_document_length": 191, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sna_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-sna_Latn": { + "number_of_characters": 74071, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 20, + "average_document_length": 149.78483606557376, + "max_document_length": 191, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "snd_Arab-snd_Arab": { + "number_of_characters": 58057, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 116.96926229508196, + "max_document_length": 164, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "snd_Arab-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-snd_Arab": { + "number_of_characters": 58057, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 116.96926229508196, + "max_document_length": 164, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "som_Latn-som_Latn": { + "number_of_characters": 82838, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 167.75, + "max_document_length": 201, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "som_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-som_Latn": { + "number_of_characters": 82838, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 167.75, + "max_document_length": 201, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sot_Latn-sot_Latn": { + "number_of_characters": 75794, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 153.3155737704918, + "max_document_length": 186, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sot_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-sot_Latn": { + "number_of_characters": 75794, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 153.3155737704918, + "max_document_length": 186, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "spa_Latn-spa_Latn": { + "number_of_characters": 74920, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 151.52459016393442, + "max_document_length": 180, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "spa_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-spa_Latn": { + "number_of_characters": 74920, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 151.52459016393442, + "max_document_length": 180, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "srp_Cyrl-srp_Cyrl": { + "number_of_characters": 61657, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 124.34631147540983, + "max_document_length": 160, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0011123470522802, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "srp_Cyrl-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-srp_Cyrl": { + "number_of_characters": 61657, + "num_samples": 1387, + "num_queries": 899, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 124.34631147540983, + "max_document_length": 160, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.085650723025584, + "max_query_length": 2, + "unique_queries": 899, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0011123470522802, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 488 + }, + "ssw_Latn-ssw_Latn": { + "number_of_characters": 73964, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 149.5655737704918, + "max_document_length": 182, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ssw_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ssw_Latn": { + "number_of_characters": 73964, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 149.5655737704918, + "max_document_length": 182, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sun_Latn-sun_Latn": { + "number_of_characters": 71320, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 144.14754098360655, + "max_document_length": 173, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sun_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-sun_Latn": { + "number_of_characters": 71320, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 144.14754098360655, + "max_document_length": 173, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "swe_Latn-swe_Latn": { + "number_of_characters": 62785, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 126.6577868852459, + "max_document_length": 154, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "swe_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-swe_Latn": { + "number_of_characters": 62785, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 126.6577868852459, + "max_document_length": 154, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "swh_Latn-swh_Latn": { + "number_of_characters": 73480, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 148.5737704918033, + "max_document_length": 194, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "swh_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-swh_Latn": { + "number_of_characters": 73480, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 148.5737704918033, + "max_document_length": 194, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tam_Taml-tam_Taml": { + "number_of_characters": 73991, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 20, + "average_document_length": 149.62090163934425, + "max_document_length": 181, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tam_Taml-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-tam_Taml": { + "number_of_characters": 73991, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 20, + "average_document_length": 149.62090163934425, + "max_document_length": 181, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tel_Telu-tel_Telu": { + "number_of_characters": 65945, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 133.13319672131146, + "max_document_length": 149, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tel_Telu-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-tel_Telu": { + "number_of_characters": 65945, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 133.13319672131146, + "max_document_length": 149, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tgk_Cyrl-tgk_Cyrl": { + "number_of_characters": 67829, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 136.99385245901638, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tgk_Cyrl-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-tgk_Cyrl": { + "number_of_characters": 67829, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 136.99385245901638, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tgl_Latn-tgl_Latn": { + "number_of_characters": 75087, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 151.86680327868854, + "max_document_length": 184, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tgl_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-tgl_Latn": { + "number_of_characters": 75087, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 151.86680327868854, + "max_document_length": 184, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tha_Thai-tha_Thai": { + "number_of_characters": 54496, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 109.67213114754098, + "max_document_length": 123, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tha_Thai-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-tha_Thai": { + "number_of_characters": 54496, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 109.67213114754098, + "max_document_length": 123, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tir_Ethi-tir_Ethi": { + "number_of_characters": 47775, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 95.89959016393442, + "max_document_length": 110, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tir_Ethi-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-tir_Ethi": { + "number_of_characters": 47775, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 95.89959016393442, + "max_document_length": 110, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tsn_Latn-tsn_Latn": { + "number_of_characters": 79391, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 160.68647540983608, + "max_document_length": 204, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tsn_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-tsn_Latn": { + "number_of_characters": 79391, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 160.68647540983608, + "max_document_length": 204, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tso_Latn-tso_Latn": { + "number_of_characters": 83501, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 169.10860655737704, + "max_document_length": 215, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tso_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-tso_Latn": { + "number_of_characters": 83501, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 169.10860655737704, + "max_document_length": 215, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tur_Latn-tur_Latn": { + "number_of_characters": 65382, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 131.9795081967213, + "max_document_length": 158, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "tur_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-tur_Latn": { + "number_of_characters": 65382, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 131.9795081967213, + "max_document_length": 158, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ukr_Cyrl-ukr_Cyrl": { + "number_of_characters": 65850, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 132.93852459016392, + "max_document_length": 159, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ukr_Cyrl-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-ukr_Cyrl": { + "number_of_characters": 65850, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 132.93852459016392, + "max_document_length": 159, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "urd_Arab-urd_Arab": { + "number_of_characters": 64450, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 130.06967213114754, + "max_document_length": 187, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "urd_Arab-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-urd_Arab": { + "number_of_characters": 64450, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 130.06967213114754, + "max_document_length": 187, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "urd_Latn-urd_Latn": { + "number_of_characters": 82039, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 166.1127049180328, + "max_document_length": 230, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "urd_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-urd_Latn": { + "number_of_characters": 82039, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 166.1127049180328, + "max_document_length": 230, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "uzn_Latn-uzn_Latn": { + "number_of_characters": 70828, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 143.13934426229508, + "max_document_length": 175, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "uzn_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-uzn_Latn": { + "number_of_characters": 70828, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 143.13934426229508, + "max_document_length": 175, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "vie_Latn-vie_Latn": { + "number_of_characters": 66724, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 134.7295081967213, + "max_document_length": 161, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "vie_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-vie_Latn": { + "number_of_characters": 66724, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 134.7295081967213, + "max_document_length": 161, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "war_Latn-war_Latn": { + "number_of_characters": 78444, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 158.74590163934425, + "max_document_length": 207, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "war_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-war_Latn": { + "number_of_characters": 78444, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 158.74590163934425, + "max_document_length": 207, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "wol_Latn-wol_Latn": { + "number_of_characters": 64521, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 130.21516393442624, + "max_document_length": 139, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "wol_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-wol_Latn": { + "number_of_characters": 64521, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 130.21516393442624, + "max_document_length": 139, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "xho_Latn-xho_Latn": { + "number_of_characters": 71629, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 144.78073770491804, + "max_document_length": 179, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "xho_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-xho_Latn": { + "number_of_characters": 71629, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 16, + "average_document_length": 144.78073770491804, + "max_document_length": 179, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "yor_Latn-yor_Latn": { + "number_of_characters": 62752, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 126.59016393442623, + "max_document_length": 143, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "yor_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-yor_Latn": { + "number_of_characters": 62752, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 126.59016393442623, + "max_document_length": 143, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "zho_Hans-zho_Hans": { + "number_of_characters": 20549, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 7, + "average_document_length": 40.10860655737705, + "max_document_length": 64, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "zho_Hans-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-zho_Hans": { + "number_of_characters": 20549, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 7, + "average_document_length": 40.10860655737705, + "max_document_length": 64, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "zho_Hant-zho_Hant": { + "number_of_characters": 19947, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 7, + "average_document_length": 38.875, + "max_document_length": 45, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "zho_Hant-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-zho_Hant": { + "number_of_characters": 19947, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 7, + "average_document_length": 38.875, + "max_document_length": 45, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "zsm_Latn-zsm_Latn": { + "number_of_characters": 72008, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 145.55737704918033, + "max_document_length": 210, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "zsm_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-zsm_Latn": { + "number_of_characters": 72008, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 13, + "average_document_length": 145.55737704918033, + "max_document_length": 210, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "zul_Latn-zul_Latn": { + "number_of_characters": 69413, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 140.23975409836066, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "zul_Latn-eng_Latn": { + "number_of_characters": 70589, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 142.64959016393442, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "eng_Latn-zul_Latn": { + "number_of_characters": 69413, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 140.23975409836066, + "max_document_length": 171, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "arb_Arab-arb_Latn": { + "number_of_characters": 61298, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 12, + "average_document_length": 123.61065573770492, + "max_document_length": 160, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "arb_Latn-arb_Arab": { + "number_of_characters": 53671, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 107.98155737704919, + "max_document_length": 134, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ben_Beng-ben_Latn": { + "number_of_characters": 68285, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 9, + "average_document_length": 137.92827868852459, + "max_document_length": 185, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "ben_Latn-ben_Beng": { + "number_of_characters": 63512, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 9, + "average_document_length": 128.14754098360655, + "max_document_length": 175, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hin_Deva-hin_Latn": { + "number_of_characters": 68307, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 137.9733606557377, + "max_document_length": 170, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "hin_Latn-hin_Deva": { + "number_of_characters": 66332, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 14, + "average_document_length": 133.9262295081967, + "max_document_length": 165, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "npi_Deva-npi_Latn": { + "number_of_characters": 65683, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 20, + "average_document_length": 132.59631147540983, + "max_document_length": 154, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "npi_Latn-npi_Deva": { + "number_of_characters": 61183, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 18, + "average_document_length": 123.375, + "max_document_length": 154, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sin_Sinh-sin_Latn": { + "number_of_characters": 85996, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 19, + "average_document_length": 174.22131147540983, + "max_document_length": 224, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "sin_Latn-sin_Sinh": { + "number_of_characters": 63902, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 17, + "average_document_length": 128.94672131147541, + "max_document_length": 159, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "urd_Arab-urd_Latn": { + "number_of_characters": 82039, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 15, + "average_document_length": 166.1127049180328, + "max_document_length": 230, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + }, + "urd_Latn-urd_Arab": { + "number_of_characters": 64450, + "num_samples": 1388, + "num_queries": 900, + "num_documents": 488, + "min_document_length": 11, + "average_document_length": 130.06967213114754, + "max_document_length": 187, + "unique_documents": 488, + "min_query_length": 2, + "average_query_length": 1.0844444444444445, + "max_query_length": 2, + "unique_queries": 900, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 488 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/COIRCodeSearchNetRetrieval.json b/mteb/descriptive_stats/Retrieval/COIRCodeSearchNetRetrieval.json new file mode 100644 index 0000000000..3d27f624b9 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/COIRCodeSearchNetRetrieval.json @@ -0,0 +1,130 @@ +{ + "test": { + "number_of_characters": 36843313, + "num_samples": 1056326, + "num_queries": 52561, + "num_documents": 1003765, + "min_document_length": 54, + "average_document_length": 34.70511822986456, + "max_document_length": 334374, + "unique_documents": 1003765, + "min_query_length": 2, + "average_query_length": 38.19428854093339, + "max_query_length": 2, + "unique_queries": 52561, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 52561, + "hf_subset_descriptive_stats": { + "python": { + "number_of_characters": 14574651, + "num_samples": 295228, + "num_queries": 14918, + "num_documents": 280310, + "min_document_length": 95, + "average_document_length": 49.994759373550714, + "max_document_length": 14008, + "unique_documents": 280310, + "min_query_length": 2, + "average_query_length": 37.5801045716584, + "max_query_length": 2, + "unique_queries": 14918, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 14918 + }, + "javascript": { + "number_of_characters": 2587540, + "num_samples": 68145, + "num_queries": 3291, + "num_documents": 64854, + "min_document_length": 87, + "average_document_length": 37.89792456903198, + "max_document_length": 334374, + "unique_documents": 64854, + "min_query_length": 2, + "average_query_length": 39.412944393801276, + "max_query_length": 2, + "unique_queries": 3291, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 3291 + }, + "go": { + "number_of_characters": 3641108, + "num_samples": 190562, + "num_queries": 8122, + "num_documents": 182440, + "min_document_length": 54, + "average_document_length": 17.957838193378645, + "max_document_length": 5280, + "unique_documents": 182440, + "min_query_length": 2, + "average_query_length": 44.9248953459739, + "max_query_length": 2, + "unique_queries": 8122, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 8122 + }, + "ruby": { + "number_of_characters": 629446, + "num_samples": 28831, + "num_queries": 1261, + "num_documents": 27570, + "min_document_length": 83, + "average_document_length": 20.830830612985128, + "max_document_length": 3992, + "unique_documents": 27570, + "min_query_length": 2, + "average_query_length": 43.72720063441713, + "max_query_length": 2, + "unique_queries": 1261, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1261 + }, + "java": { + "number_of_characters": 6791137, + "num_samples": 191821, + "num_queries": 10955, + "num_documents": 180866, + "min_document_length": 77, + "average_document_length": 35.54789180940586, + "max_document_length": 7615, + "unique_documents": 180866, + "min_query_length": 2, + "average_query_length": 33.019808306709265, + "max_query_length": 2, + "unique_queries": 10955, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10955 + }, + "php": { + "number_of_characters": 8619431, + "num_samples": 281739, + "num_queries": 14014, + "num_documents": 267725, + "min_document_length": 94, + "average_document_length": 30.195091978709495, + "max_document_length": 4904, + "unique_documents": 267725, + "min_query_length": 2, + "average_query_length": 38.20822035107749, + "max_query_length": 2, + "unique_queries": 14014, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 14014 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/CodeEditSearchRetrieval.json b/mteb/descriptive_stats/Retrieval/CodeEditSearchRetrieval.json new file mode 100644 index 0000000000..6d73096d42 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/CodeEditSearchRetrieval.json @@ -0,0 +1,256 @@ +{ + "train": { + "number_of_characters": 935841, + "num_samples": 26000, + "num_queries": 13000, + "num_documents": 13000, + "min_document_length": 18, + "average_document_length": 70.98776923076923, + "max_document_length": 2532, + "unique_documents": 13000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 13000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 13000, + "hf_subset_descriptive_stats": { + "python": { + "number_of_characters": 70519, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 21, + "average_document_length": 69.519, + "max_document_length": 1811, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "javascript": { + "number_of_characters": 57880, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 18, + "average_document_length": 56.88, + "max_document_length": 601, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "typescript": { + "number_of_characters": 61092, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 19, + "average_document_length": 60.092, + "max_document_length": 659, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "go": { + "number_of_characters": 71797, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 19, + "average_document_length": 70.797, + "max_document_length": 1529, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "ruby": { + "number_of_characters": 67900, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 20, + "average_document_length": 66.9, + "max_document_length": 751, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "java": { + "number_of_characters": 63984, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 23, + "average_document_length": 62.984, + "max_document_length": 807, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "php": { + "number_of_characters": 62927, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 21, + "average_document_length": 61.927, + "max_document_length": 766, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "c": { + "number_of_characters": 98588, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 20, + "average_document_length": 97.588, + "max_document_length": 1672, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "c++": { + "number_of_characters": 115480, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 22, + "average_document_length": 114.48, + "max_document_length": 1856, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "rust": { + "number_of_characters": 68503, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 19, + "average_document_length": 67.503, + "max_document_length": 2532, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "swift": { + "number_of_characters": 58279, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 19, + "average_document_length": 57.279, + "max_document_length": 727, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "scala": { + "number_of_characters": 65833, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 22, + "average_document_length": 64.833, + "max_document_length": 685, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "shell": { + "number_of_characters": 73059, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 18, + "average_document_length": 72.059, + "max_document_length": 813, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/CodeFeedbackMT.json b/mteb/descriptive_stats/Retrieval/CodeFeedbackMT.json new file mode 100644 index 0000000000..1be18319cd --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/CodeFeedbackMT.json @@ -0,0 +1,20 @@ +{ + "test": { + "number_of_characters": 156266302, + "num_samples": 79660, + "num_queries": 13277, + "num_documents": 66383, + "min_document_length": 127, + "average_document_length": 885.131117906693, + "max_document_length": 32432, + "unique_documents": 66383, + "min_query_length": 2, + "average_query_length": 7344.177374406869, + "max_query_length": 9403, + "unique_queries": 13277, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 13277 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/CodeFeedbackST.json b/mteb/descriptive_stats/Retrieval/CodeFeedbackST.json new file mode 100644 index 0000000000..4511605dd5 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/CodeFeedbackST.json @@ -0,0 +1,20 @@ +{ + "test": { + "number_of_characters": 260957682, + "num_samples": 187832, + "num_queries": 31306, + "num_documents": 156526, + "min_document_length": 26, + "average_document_length": 144.85253568097312, + "max_document_length": 13851, + "unique_documents": 156526, + "min_query_length": 1, + "average_query_length": 7611.464064396601, + "max_query_length": 11354, + "unique_queries": 31306, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 31306 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/CodeSearchNetCCRetrieval.json b/mteb/descriptive_stats/Retrieval/CodeSearchNetCCRetrieval.json new file mode 100644 index 0000000000..a817119b43 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/CodeSearchNetCCRetrieval.json @@ -0,0 +1,130 @@ +{ + "test": { + "number_of_characters": 22407915, + "num_samples": 1058035, + "num_queries": 52561, + "num_documents": 1005474, + "min_document_length": 23, + "average_document_length": 20.28592186371801, + "max_document_length": 214210, + "unique_documents": 1005474, + "min_query_length": 2, + "average_query_length": 38.259317745096176, + "max_query_length": 2, + "unique_queries": 52561, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 52561, + "hf_subset_descriptive_stats": { + "python": { + "number_of_characters": 8792958, + "num_samples": 295570, + "num_queries": 14918, + "num_documents": 280652, + "min_document_length": 38, + "average_document_length": 29.330466200133973, + "max_document_length": 8326, + "unique_documents": 280652, + "min_query_length": 2, + "average_query_length": 37.62595522187961, + "max_query_length": 2, + "unique_queries": 14918, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 14918 + }, + "javascript": { + "number_of_characters": 1590642, + "num_samples": 68492, + "num_queries": 3291, + "num_documents": 65201, + "min_document_length": 40, + "average_document_length": 22.395975521847824, + "max_document_length": 214210, + "unique_documents": 65201, + "min_query_length": 2, + "average_query_length": 39.6238225463385, + "max_query_length": 2, + "unique_queries": 3291, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 3291 + }, + "go": { + "number_of_characters": 2264134, + "num_samples": 190857, + "num_queries": 8122, + "num_documents": 182735, + "min_document_length": 23, + "average_document_length": 10.390259118395491, + "max_document_length": 3589, + "unique_documents": 182735, + "min_query_length": 2, + "average_query_length": 44.99753755232701, + "max_query_length": 2, + "unique_queries": 8122, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 8122 + }, + "ruby": { + "number_of_characters": 391703, + "num_samples": 28849, + "num_queries": 1261, + "num_documents": 27588, + "min_document_length": 36, + "average_document_length": 12.198310859794113, + "max_document_length": 2244, + "unique_documents": 27588, + "min_query_length": 2, + "average_query_length": 43.75574940523394, + "max_query_length": 2, + "unique_queries": 1261, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1261 + }, + "java": { + "number_of_characters": 4114584, + "num_samples": 192016, + "num_queries": 10955, + "num_documents": 181061, + "min_document_length": 38, + "average_document_length": 20.724849636310413, + "max_document_length": 5066, + "unique_documents": 181061, + "min_query_length": 2, + "average_query_length": 33.055408489274306, + "max_query_length": 2, + "unique_queries": 10955, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10955 + }, + "php": { + "number_of_characters": 5253894, + "num_samples": 282251, + "num_queries": 14014, + "num_documents": 268237, + "min_document_length": 40, + "average_document_length": 17.586760961388624, + "max_document_length": 2995, + "unique_documents": 268237, + "min_query_length": 2, + "average_query_length": 38.28129013843299, + "max_query_length": 2, + "unique_queries": 14014, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 14014 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/CodeSearchNetRetrieval.json b/mteb/descriptive_stats/Retrieval/CodeSearchNetRetrieval.json new file mode 100644 index 0000000000..853c4c79c6 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/CodeSearchNetRetrieval.json @@ -0,0 +1,130 @@ +{ + "test": { + "number_of_characters": 1950074, + "num_samples": 12000, + "num_queries": 6000, + "num_documents": 6000, + "min_document_length": 2, + "average_document_length": 324.01233333333334, + "max_document_length": 17533, + "unique_documents": 6000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 6000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 6000, + "hf_subset_descriptive_stats": { + "python": { + "number_of_characters": 467546, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 8, + "average_document_length": 466.546, + "max_document_length": 8636, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "javascript": { + "number_of_characters": 187018, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 2, + "average_document_length": 186.018, + "max_document_length": 7657, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "go": { + "number_of_characters": 126213, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 14, + "average_document_length": 125.213, + "max_document_length": 1501, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "ruby": { + "number_of_characters": 314818, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 5, + "average_document_length": 313.818, + "max_document_length": 17533, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "java": { + "number_of_characters": 691360, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 2, + "average_document_length": 690.36, + "max_document_length": 6473, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + }, + "php": { + "number_of_characters": 163119, + "num_samples": 2000, + "num_queries": 1000, + "num_documents": 1000, + "min_document_length": 5, + "average_document_length": 162.119, + "max_document_length": 1240, + "unique_documents": 1000, + "min_query_length": 1, + "average_query_length": 1.0, + "max_query_length": 1, + "unique_queries": 1000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1000 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/CodeTransOceanContest.json b/mteb/descriptive_stats/Retrieval/CodeTransOceanContest.json new file mode 100644 index 0000000000..07081e69c3 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/CodeTransOceanContest.json @@ -0,0 +1,20 @@ +{ + "test": { + "number_of_characters": 1744286, + "num_samples": 1229, + "num_queries": 221, + "num_documents": 1008, + "min_document_length": 8, + "average_document_length": 221.90178571428572, + "max_document_length": 4147, + "unique_documents": 1008, + "min_query_length": 8, + "average_query_length": 6880.58371040724, + "max_query_length": 10852, + "unique_queries": 221, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 221 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/CodeTransOceanDL.json b/mteb/descriptive_stats/Retrieval/CodeTransOceanDL.json new file mode 100644 index 0000000000..042658caad --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/CodeTransOceanDL.json @@ -0,0 +1,20 @@ +{ + "test": { + "number_of_characters": 1543912, + "num_samples": 996, + "num_queries": 180, + "num_documents": 816, + "min_document_length": 376, + "average_document_length": 411.97549019607845, + "max_document_length": 8285, + "unique_documents": 816, + "min_query_length": 58, + "average_query_length": 6709.666666666667, + "max_query_length": 8469, + "unique_queries": 180, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 180 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/CosQA.json b/mteb/descriptive_stats/Retrieval/CosQA.json new file mode 100644 index 0000000000..d8f17d4b21 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/CosQA.json @@ -0,0 +1,20 @@ +{ + "test": { + "number_of_characters": 5728450, + "num_samples": 21104, + "num_queries": 500, + "num_documents": 20604, + "min_document_length": 18, + "average_document_length": 0.8933702193748787, + "max_document_length": 83, + "unique_documents": 20604, + "min_query_length": 88, + "average_query_length": 11420.086, + "max_query_length": 6396, + "unique_queries": 500, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 500 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/JaqketRetrieval.json b/mteb/descriptive_stats/Retrieval/JaqketRetrieval.json new file mode 100644 index 0000000000..4598b2af77 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/JaqketRetrieval.json @@ -0,0 +1,20 @@ +{ + "test": { + "number_of_characters": 428294530, + "num_samples": 115226, + "num_queries": 997, + "num_documents": 114229, + "min_document_length": 16, + "average_document_length": 0.4425671239352529, + "max_document_length": 98, + "unique_documents": 114229, + "min_query_length": 8, + "average_query_length": 429532.5737211635, + "max_query_length": 188424, + "unique_queries": 997, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 989 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/NFCorpus.json b/mteb/descriptive_stats/Retrieval/NFCorpus.json new file mode 100644 index 0000000000..94df0b0cfb --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/NFCorpus.json @@ -0,0 +1,11 @@ +{ + "test": { + "number_of_characters": 1612.5486310130989, + "num_samples": 3956, + "num_queries": 323, + "num_documents": 3633, + "average_document_length": 0.43787060972495073, + "average_query_length": 0.06738299034784193, + "average_relevant_docs_per_query": 38.18575851393189 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/StackOverflowQA.json b/mteb/descriptive_stats/Retrieval/StackOverflowQA.json new file mode 100644 index 0000000000..51972461e6 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/StackOverflowQA.json @@ -0,0 +1,20 @@ +{ + "test": { + "number_of_characters": 26584028, + "num_samples": 21925, + "num_queries": 1994, + "num_documents": 19931, + "min_document_length": 61, + "average_document_length": 130.32145903366614, + "max_document_length": 22234, + "unique_documents": 19931, + "min_query_length": 5, + "average_query_length": 12029.38365095286, + "max_query_length": 46028, + "unique_queries": 1994, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1994 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/SyntheticText2SQL.json b/mteb/descriptive_stats/Retrieval/SyntheticText2SQL.json new file mode 100644 index 0000000000..56c3964a58 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/SyntheticText2SQL.json @@ -0,0 +1,20 @@ +{ + "test": { + "number_of_characters": 14041553, + "num_samples": 111702, + "num_queries": 5851, + "num_documents": 105851, + "min_document_length": 13, + "average_document_length": 4.582686984534865, + "max_document_length": 281, + "unique_documents": 105851, + "min_query_length": 17, + "average_query_length": 2316.9494103572038, + "max_query_length": 762, + "unique_queries": 5851, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 5851 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/Touche2020.json b/mteb/descriptive_stats/Retrieval/Touche2020.json new file mode 100644 index 0000000000..a3c37a54ee --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/Touche2020.json @@ -0,0 +1,20 @@ +{ + "test": { + "number_of_characters": 658107591, + "num_samples": 382594, + "num_queries": 49, + "num_documents": 382545, + "min_document_length": 16, + "average_document_length": 0.0055627442523101854, + "max_document_length": 83, + "unique_documents": 382545, + "min_query_length": 3, + "average_query_length": 13430723.734693877, + "max_query_length": 106072, + "unique_queries": 49, + "min_relevant_docs_per_query": 40, + "average_relevant_docs_per_query": 45.183673469387756, + "max_relevant_docs_per_query": 52, + "unique_relevant_docs": 2099 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/Touche2020Retrieval.v3.json b/mteb/descriptive_stats/Retrieval/Touche2020Retrieval.v3.json new file mode 100644 index 0000000000..1b436abd75 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/Touche2020Retrieval.v3.json @@ -0,0 +1,20 @@ +{ + "test": { + "number_of_characters": 637047138, + "num_samples": 303781, + "num_queries": 49, + "num_documents": 303732, + "min_document_length": 16, + "average_document_length": 0.007006176497701922, + "max_document_length": 83, + "unique_documents": 303732, + "min_query_length": 41, + "average_query_length": 13000918.57142857, + "max_query_length": 105983, + "unique_queries": 49, + "min_relevant_docs_per_query": 40, + "average_relevant_docs_per_query": 58.142857142857146, + "max_relevant_docs_per_query": 87, + "unique_relevant_docs": 2732 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/mFollowIRCrossLingualInstructionRetrieval.json b/mteb/descriptive_stats/Retrieval/mFollowIRCrossLingualInstructionRetrieval.json new file mode 100644 index 0000000000..f23a5ea1be --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/mFollowIRCrossLingualInstructionRetrieval.json @@ -0,0 +1,116 @@ +{ + "test": { + "num_samples": 121758, + "num_docs": 121635, + "num_queries": 123, + "number_of_characters": 283654099, + "min_document_length": 74, + "average_document_length": 2331.0777818884367, + "max_document_length": 24179, + "unique_docs": 121635, + "min_query_length": 32, + "average_query_length": 81.8780487804878, + "max_query_length": 173, + "unique_queries": 75, + "min_instruction_length": 93, + "average_instruction_length": 389.9512195121951, + "max_instruction_length": 887, + "unique_instructions": 75, + "min_changed_instruction_length": 180, + "average_changed_instruction_length": 450.5528455284553, + "max_changed_instruction_length": 974, + "unique_changed_instructions": 123, + "min_average_relevant_docs_per_query": 0, + "average_relevant_docs_per_query": 10.43089430894309, + "max_average_relevant_docs_per_query": 24, + "min_average_top_ranked_per_query": 1000, + "average_top_ranked_per_query": 1000.0, + "max_average_top_ranked_per_query": 1000, + "hf_subset_descriptive_stats": { + "eng-fas": { + "num_samples": 41229, + "num_docs": 41189, + "num_queries": 40, + "number_of_characters": 129597567, + "min_document_length": 99, + "average_document_length": 3145.4990895627475, + "max_document_length": 24179, + "unique_docs": 41189, + "min_query_length": 34, + "average_query_length": 80.075, + "max_query_length": 124, + "unique_queries": 40, + "min_instruction_length": 150, + "average_instruction_length": 396.875, + "max_instruction_length": 887, + "unique_instructions": 40, + "min_changed_instruction_length": 205, + "average_changed_instruction_length": 463.175, + "max_changed_instruction_length": 974, + "unique_changed_instructions": 40, + "min_average_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 10.85, + "max_average_relevant_docs_per_query": 22, + "min_average_top_ranked_per_query": 1000, + "average_top_ranked_per_query": 1000.0, + "max_average_top_ranked_per_query": 1000 + }, + "eng-rus": { + "num_samples": 39366, + "num_docs": 39326, + "num_queries": 40, + "number_of_characters": 109522175, + "min_document_length": 75, + "average_document_length": 2784.0813456746173, + "max_document_length": 24061, + "unique_docs": 39326, + "min_query_length": 32, + "average_query_length": 81.875, + "max_query_length": 173, + "unique_queries": 40, + "min_instruction_length": 93, + "average_instruction_length": 371.125, + "max_instruction_length": 887, + "unique_instructions": 40, + "min_changed_instruction_length": 180, + "average_changed_instruction_length": 431.8, + "max_changed_instruction_length": 957, + "unique_changed_instructions": 40, + "min_average_relevant_docs_per_query": 0, + "average_relevant_docs_per_query": 9.775, + "max_average_relevant_docs_per_query": 24, + "min_average_top_ranked_per_query": 1000, + "average_top_ranked_per_query": 1000.0, + "max_average_top_ranked_per_query": 1000 + }, + "eng-zho": { + "num_samples": 41163, + "num_docs": 41120, + "num_queries": 43, + "number_of_characters": 44534357, + "min_document_length": 74, + "average_document_length": 1082.0501215953307, + "max_document_length": 23840, + "unique_docs": 41120, + "min_query_length": 32, + "average_query_length": 83.55813953488372, + "max_query_length": 159, + "unique_queries": 43, + "min_instruction_length": 157, + "average_instruction_length": 401.0232558139535, + "max_instruction_length": 731, + "unique_instructions": 43, + "min_changed_instruction_length": 209, + "average_changed_instruction_length": 456.25581395348837, + "max_changed_instruction_length": 822, + "unique_changed_instructions": 43, + "min_average_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 10.651162790697674, + "max_average_relevant_docs_per_query": 24, + "min_average_top_ranked_per_query": 1000, + "average_top_ranked_per_query": 1000.0, + "max_average_top_ranked_per_query": 1000 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Retrieval/mFollowIRInstructionRetrieval.json b/mteb/descriptive_stats/Retrieval/mFollowIRInstructionRetrieval.json new file mode 100644 index 0000000000..54ae5d1ec2 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/mFollowIRInstructionRetrieval.json @@ -0,0 +1,116 @@ +{ + "test": { + "num_samples": 121758, + "num_docs": 121635, + "num_queries": 123, + "number_of_characters": 283622456, + "min_document_length": 74, + "average_document_length": 2331.0777818884367, + "max_document_length": 24179, + "unique_docs": 121635, + "min_query_length": 10, + "average_query_length": 57.113821138211385, + "max_query_length": 136, + "unique_queries": 123, + "min_instruction_length": 37, + "average_instruction_length": 281.0650406504065, + "max_instruction_length": 1009, + "unique_instructions": 123, + "min_changed_instruction_length": 44, + "average_changed_instruction_length": 326.9430894308943, + "max_changed_instruction_length": 1083, + "unique_changed_instructions": 123, + "min_average_relevant_docs_per_query": 0, + "average_relevant_docs_per_query": 10.43089430894309, + "max_average_relevant_docs_per_query": 24, + "min_average_top_ranked_per_query": 1000, + "average_top_ranked_per_query": 1000.0, + "max_average_top_ranked_per_query": 1000, + "hf_subset_descriptive_stats": { + "fas": { + "num_samples": 41229, + "num_docs": 41189, + "num_queries": 40, + "number_of_characters": 129593838, + "min_document_length": 99, + "average_document_length": 3145.4990895627475, + "max_document_length": 24179, + "unique_docs": 41189, + "min_query_length": 34, + "average_query_length": 72.65, + "max_query_length": 124, + "unique_queries": 40, + "min_instruction_length": 121, + "average_instruction_length": 358.925, + "max_instruction_length": 759, + "unique_instructions": 40, + "min_changed_instruction_length": 163, + "average_changed_instruction_length": 415.325, + "max_changed_instruction_length": 842, + "unique_changed_instructions": 40, + "min_average_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 10.85, + "max_average_relevant_docs_per_query": 22, + "min_average_top_ranked_per_query": 1000, + "average_top_ranked_per_query": 1000.0, + "max_average_top_ranked_per_query": 1000 + }, + "rus": { + "num_samples": 39366, + "num_docs": 39326, + "num_queries": 40, + "number_of_characters": 109523683, + "min_document_length": 75, + "average_document_length": 2784.0813456746173, + "max_document_length": 24061, + "unique_docs": 39326, + "min_query_length": 26, + "average_query_length": 77.5, + "max_query_length": 136, + "unique_queries": 40, + "min_instruction_length": 78, + "average_instruction_length": 387.0, + "max_instruction_length": 1009, + "unique_instructions": 40, + "min_changed_instruction_length": 187, + "average_changed_instruction_length": 458.0, + "max_changed_instruction_length": 1083, + "unique_changed_instructions": 40, + "min_average_relevant_docs_per_query": 0, + "average_relevant_docs_per_query": 9.775, + "max_average_relevant_docs_per_query": 24, + "min_average_top_ranked_per_query": 1000, + "average_top_ranked_per_query": 1000.0, + "max_average_top_ranked_per_query": 1000 + }, + "zho": { + "num_samples": 41163, + "num_docs": 41120, + "num_queries": 43, + "number_of_characters": 44504935, + "min_document_length": 74, + "average_document_length": 1082.0501215953307, + "max_document_length": 23840, + "unique_docs": 41120, + "min_query_length": 10, + "average_query_length": 23.697674418604652, + "max_query_length": 44, + "unique_queries": 43, + "min_instruction_length": 37, + "average_instruction_length": 110.09302325581395, + "max_instruction_length": 209, + "unique_instructions": 43, + "min_changed_instruction_length": 44, + "average_changed_instruction_length": 122.81395348837209, + "max_changed_instruction_length": 229, + "unique_changed_instructions": 43, + "min_average_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 10.651162790697674, + "max_average_relevant_docs_per_query": 24, + "min_average_top_ranked_per_query": 1000, + "average_top_ranked_per_query": 1000.0, + "max_average_top_ranked_per_query": 1000 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/STS/STS12.json b/mteb/descriptive_stats/STS/STS12.json new file mode 100644 index 0000000000..a7e11197ac --- /dev/null +++ b/mteb/descriptive_stats/STS/STS12.json @@ -0,0 +1,17 @@ +{ + "test": { + "num_samples": 3108, + "number_of_characters": 402118, + "min_sentence1_length": 3, + "average_sentence1_len": 63.78893178893179, + "max_sentence1_length": 220, + "unique_sentence1": 2236, + "min_sentence2_length": 7, + "average_sentence2_len": 65.5926640926641, + "max_sentence2_length": 204, + "unique_sentence2": 2797, + "min_score": 0.0, + "avg_score": 3.5060643500643507, + "max_score": 5.0 + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/STS/STS17.json b/mteb/descriptive_stats/STS/STS17.json new file mode 100644 index 0000000000..912738035b --- /dev/null +++ b/mteb/descriptive_stats/STS/STS17.json @@ -0,0 +1,184 @@ +{ + "test": { + "num_samples": 5346, + "number_of_characters": 400264, + "min_sentence1_length": 6, + "average_sentence1_len": 38.14665170220726, + "max_sentence1_length": 976, + "unique_sentence1": 4900, + "min_sentence2_length": 6, + "average_sentence2_len": 36.72502805836139, + "max_sentence2_length": 1007, + "unique_sentence2": 4470, + "min_score": 0.0, + "avg_score": 2.3554804214989464, + "max_score": 5.0, + "hf_subset_descriptive_stats": { + "ko-ko": { + "num_samples": 2846, + "number_of_characters": 183387, + "min_sentence1_length": 6, + "average_sentence1_len": 31.991918482080113, + "max_sentence1_length": 976, + "unique_sentence1": 2650, + "min_sentence2_length": 6, + "average_sentence2_len": 32.44483485593816, + "max_sentence2_length": 1007, + "unique_sentence2": 2720, + "min_score": 0.0, + "avg_score": 2.469359920356055, + "max_score": 5.0 + }, + "ar-ar": { + "num_samples": 250, + "number_of_characters": 16247, + "min_sentence1_length": 11, + "average_sentence1_len": 32.208, + "max_sentence1_length": 99, + "unique_sentence1": 250, + "min_sentence2_length": 9, + "average_sentence2_len": 32.78, + "max_sentence2_length": 83, + "unique_sentence2": 250, + "min_score": 0.0, + "avg_score": 2.216800000000001, + "max_score": 5.0 + }, + "en-ar": { + "num_samples": 250, + "number_of_characters": 18764, + "min_sentence1_length": 13, + "average_sentence1_len": 42.36, + "max_sentence1_length": 105, + "unique_sentence1": 250, + "min_sentence2_length": 10, + "average_sentence2_len": 32.696, + "max_sentence2_length": 104, + "unique_sentence2": 250, + "min_score": 0.0, + "avg_score": 2.1423999999999994, + "max_score": 5.0 + }, + "en-de": { + "num_samples": 250, + "number_of_characters": 22177, + "min_sentence1_length": 12, + "average_sentence1_len": 43.952, + "max_sentence1_length": 94, + "unique_sentence1": 250, + "min_sentence2_length": 15, + "average_sentence2_len": 44.756, + "max_sentence2_length": 104, + "unique_sentence2": 250, + "min_score": 0.0, + "avg_score": 2.2776000000000014, + "max_score": 5.0 + }, + "en-en": { + "num_samples": 250, + "number_of_characters": 21669, + "min_sentence1_length": 12, + "average_sentence1_len": 43.952, + "max_sentence1_length": 94, + "unique_sentence1": 250, + "min_sentence2_length": 15, + "average_sentence2_len": 42.724, + "max_sentence2_length": 101, + "unique_sentence2": 250, + "min_score": 0.0, + "avg_score": 2.2776000000000014, + "max_score": 5.0 + }, + "en-tr": { + "num_samples": 250, + "number_of_characters": 20879, + "min_sentence1_length": 15, + "average_sentence1_len": 41.916, + "max_sentence1_length": 101, + "unique_sentence1": 250, + "min_sentence2_length": 10, + "average_sentence2_len": 41.6, + "max_sentence2_length": 107, + "unique_sentence2": 250, + "min_score": 0.0, + "avg_score": 2.1335999999999986, + "max_score": 5.0 + }, + "es-en": { + "num_samples": 250, + "number_of_characters": 23216, + "min_sentence1_length": 12, + "average_sentence1_len": 50.84, + "max_sentence1_length": 160, + "unique_sentence1": 250, + "min_sentence2_length": 14, + "average_sentence2_len": 42.024, + "max_sentence2_length": 117, + "unique_sentence2": 250, + "min_score": 0.0, + "avg_score": 2.1464000000000003, + "max_score": 5.0 + }, + "es-es": { + "num_samples": 250, + "number_of_characters": 25265, + "min_sentence1_length": 18, + "average_sentence1_len": 49.836, + "max_sentence1_length": 136, + "unique_sentence1": 250, + "min_sentence2_length": 13, + "average_sentence2_len": 51.224, + "max_sentence2_length": 129, + "unique_sentence2": 250, + "min_score": 0.0, + "avg_score": 2.2312000000000007, + "max_score": 5.0 + }, + "fr-en": { + "num_samples": 250, + "number_of_characters": 23087, + "min_sentence1_length": 19, + "average_sentence1_len": 49.624, + "max_sentence1_length": 115, + "unique_sentence1": 250, + "min_sentence2_length": 15, + "average_sentence2_len": 42.724, + "max_sentence2_length": 101, + "unique_sentence2": 250, + "min_score": 0.0, + "avg_score": 2.2776000000000014, + "max_score": 5.0 + }, + "it-en": { + "num_samples": 250, + "number_of_characters": 23188, + "min_sentence1_length": 15, + "average_sentence1_len": 50.028, + "max_sentence1_length": 113, + "unique_sentence1": 250, + "min_sentence2_length": 15, + "average_sentence2_len": 42.724, + "max_sentence2_length": 101, + "unique_sentence2": 250, + "min_score": 0.0, + "avg_score": 2.2776000000000014, + "max_score": 5.0 + }, + "nl-en": { + "num_samples": 250, + "number_of_characters": 22385, + "min_sentence1_length": 14, + "average_sentence1_len": 46.816, + "max_sentence1_length": 123, + "unique_sentence1": 250, + "min_sentence2_length": 15, + "average_sentence2_len": 42.724, + "max_sentence2_length": 101, + "unique_sentence2": 250, + "min_score": 0.0, + "avg_score": 2.2776000000000014, + "max_score": 5.0 + } + } + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Summarization/SummEval.json b/mteb/descriptive_stats/Summarization/SummEval.json new file mode 100644 index 0000000000..4c2f133abb --- /dev/null +++ b/mteb/descriptive_stats/Summarization/SummEval.json @@ -0,0 +1,55 @@ +{ + "test": { + "num_samples": 100, + "number_of_characters": 212735, + "min_text_length": 626, + "avg_text_length": 2100.35, + "max_text_length": 3153, + "unique_texts": 100, + "min_human_summaries_length": 11, + "avg_human_summaries_length": 11.0, + "max_human_summaries_length": 11, + "unique_human_summaries": 1100, + "min_machine_summaries_length": 16, + "avg_machine_summaries_length": 16.0, + "max_machine_summaries_length": 16, + "unique_machine_summaries": 1548, + "min_relevance": [ + 1.0, + 1.3333333333333333, + 3.6666666666666665, + 2.3333333333333335, + 3.6666666666666665, + 3.0, + 4.333333333333333, + 4.0, + 2.6666666666666665, + 4.0, + 2.0, + 4.666666666666667, + 4.333333333333333, + 1.0, + 2.0, + 1.0 + ], + "avg_relevance": 3.7770833333333336, + "max_relevance": [ + 5.0, + 4.666666666666667, + 4.333333333333333, + 2.6666666666666665, + 4.666666666666667, + 4.666666666666667, + 4.666666666666667, + 4.333333333333333, + 4.0, + 4.333333333333333, + 4.666666666666667, + 4.666666666666667, + 4.333333333333333, + 2.3333333333333335, + 4.666666666666667, + 4.666666666666667 + ] + } +} \ No newline at end of file diff --git a/mteb/encoder_interface.py b/mteb/encoder_interface.py index e2725a3ac9..5a66330cdc 100644 --- a/mteb/encoder_interface.py +++ b/mteb/encoder_interface.py @@ -25,15 +25,13 @@ class Encoder(Protocol): In general the interface is kept aligned with sentence-transformers interface. In cases where exceptions occurs these are handled within MTEB. """ - device: str | None - def __init__(self, device: str | None = None) -> None: """The initialization function for the encoder. Used when calling it from the mteb run CLI. Args: device: The device to use for encoding. Can be ignored if the encoder is not using a device (e.g. for API) """ - pass + self.device = device def encode( self, diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py index c58d3d38de..70378931c2 100644 --- a/mteb/evaluation/MTEB.py +++ b/mteb/evaluation/MTEB.py @@ -5,7 +5,7 @@ import os import traceback from collections.abc import Iterable -from copy import copy +from copy import copy, deepcopy from datetime import datetime from itertools import chain from pathlib import Path @@ -15,6 +15,7 @@ import datasets from sentence_transformers import CrossEncoder, SentenceTransformer +from mteb.abstasks.AbsTask import ScoresDict from mteb.encoder_interface import Encoder from mteb.model_meta import ModelMeta from mteb.models import model_meta_from_sentence_transformers @@ -84,6 +85,8 @@ def __init__( self._version = version self.err_logs_path = err_logs_path + self.last_evaluated_splits = {} + self.select_tasks(**kwargs) def deprecation_warning( @@ -290,8 +293,9 @@ def load_tasks_data(self): def _run_eval( task: AbsTask, model: Encoder, - split, - output_folder, + split: str, + output_folder: str | None, + subsets_to_run: list[str] | None = None, *, encode_kwargs: dict[str, Any], **kwargs: Any, @@ -300,6 +304,7 @@ def _run_eval( results = task.evaluate( model, split, + subsets_to_run=subsets_to_run, output_folder=output_folder, encode_kwargs=encode_kwargs, **kwargs, @@ -307,12 +312,77 @@ def _run_eval( tock = time() return results, tick, tock + @staticmethod + def _get_missing_splits( + existing_results: TaskResult | None, task_eval_splits: list[str] + ) -> list[str]: + if existing_results is None: + return task_eval_splits + + missing_splits = [] + for split in task_eval_splits: + if split not in existing_results.scores: + missing_splits.append(split) + elif not existing_results.scores[ + split + ]: # Check if the split has any scores + missing_splits.append(split) + + return missing_splits + + @staticmethod + def _merge_results( + existing_results: TaskResult, new_results: TaskResult + ) -> TaskResult: + merged_scores = existing_results.scores.copy() + + for split, scores in new_results.scores.items(): + if split in merged_scores: + merged_scores[split] = MTEB._merge_split_scores( + merged_scores[split], scores + ) + else: + merged_scores[split] = scores + + existing_kg_co2_emissions = ( + existing_results.kg_co2_emissions + if existing_results.kg_co2_emissions + else 0 + ) + new_kg_co2_emissions = ( + new_results.kg_co2_emissions if new_results.kg_co2_emissions else 0 + ) + merged_kg_co2_emissions = None + if existing_kg_co2_emissions and new_kg_co2_emissions: + merged_kg_co2_emissions = existing_kg_co2_emissions + new_kg_co2_emissions + merged_results = TaskResult( + dataset_revision=new_results.dataset_revision, + task_name=new_results.task_name, + mteb_version=new_results.mteb_version, + scores=merged_scores, + evaluation_time=existing_results.evaluation_time + + new_results.evaluation_time, + kg_co2_emissions=merged_kg_co2_emissions, + ) + + return merged_results + + @staticmethod + def _merge_split_scores( + existing_scores: list[ScoresDict], new_scores: list[ScoresDict] + ) -> list[ScoresDict]: + merged = {score["hf_subset"]: score for score in existing_scores} + for score in new_scores: + merged[score["hf_subset"]] = score + return list(merged.values()) + def run( self, model: SentenceTransformer | Encoder, verbosity: int = 1, output_folder: str | None = "results", - eval_splits=None, + eval_splits: list[str] | None = None, + eval_subsets: list[str] | None = None, overwrite_results: bool = False, raise_error: bool = True, co2_tracker: bool = False, @@ -324,12 +394,14 @@ def run( Args: model: Model to be used for evaluation verbosity: Verbosity level. Default is 1. - 0: print tasks tqdm progress bar - 1: print tasks tqdm progress bar and scores - 2: print everything (including datasets loading) + 0: Only shows a progress bar for tasks being processed. + 1: Shows a progress bar and prints task scores. + 2: Prints detailed output, including messages about loading datasets and task scores. + 3: Prints comprehensive logs for debugging, including all data loading and evaluation details. output_folder: Folder where the results will be saved. Default to 'results'. Where it will save the results in the format: `{output_folder}/{model_name}/{model_revision}/{task_name}.json`. eval_splits: List of splits to evaluate on. If None, the splits are taken from the task metadata. + eval_subsets: List of subsets to evaluate on. If None, the subsets are taken from the task metadata. overwrite_results: Whether to overwrite existing results. raise_error: Whether to raise an error if an exception occurs during evaluation. co2_tracker: Whether to enable or disable CO2 emissions tracker using codecarbon. @@ -346,69 +418,144 @@ def run( ) encode_kwargs["batch_size"] = kwargs["batch_size"] - # Set logging - if verbosity < 2: - datasets.logging.set_verbosity(40) - datasets.logging.disable_progress_bar() + # update logging to account for different levels of Verbosity (similar to the command line) + + if verbosity == 0: + datasets.logging.set_verbosity(logging.CRITICAL) # 40 + datasets.logging.disable_progress_bar() # Disable progress bar + elif verbosity == 1: + datasets.logging.set_verbosity(logging.WARNING) + datasets.logging.disable_progress_bar() # Disable progress bar + elif verbosity == 2: + datasets.logging.set_verbosity(logging.INFO) + elif verbosity == 3: + datasets.logging.set_verbosity(logging.DEBUG) meta = self.create_model_meta(model) output_path = self.create_output_folder(meta, output_folder) if isinstance(model, (SentenceTransformer, CrossEncoder)): model = SentenceTransformerWrapper(model) + ## Disable co2_tracker for API models + if "API" in meta.framework: + co2_tracker = False + if output_path: self._save_model_metadata(meta, output_path) # Run selected tasks logger.info(f"\n\n## Evaluating {len(self.tasks)} tasks:") - self.print_selected_tasks() + + if verbosity > 0: + self.print_selected_tasks() + evaluation_results = [] original_tasks = ( self.tasks.copy() ) # save them in case we re-use the object (e.g. for reranking) + + # To evaluate missing splits, we keep track of the task name and the corresponding splits. + self.last_evaluated_splits = {} + while len(self.tasks) > 0: task = self.tasks[0] logger.info( f"\n\n********************** Evaluating {task.metadata.name} **********************" ) - # skip evaluation if the model does not support the task modalities. - task_modalities = "".join(sorted(task.metadata.modalities)) - if ("".join(sorted(meta.modalities)) != task_modalities) and ( - not set(task.metadata.modalities).issubset(set(meta.modalities)) - ): + if "bm25s" in meta.name and task.metadata.type != "Retrieval": logger.warning( - f"{meta.name} only supports {meta.modalities}, but the task modalities are {task.metadata.modalities}. Skipping task." + f"bm25s only supports Retrieval tasks, but the task type is {task.metadata.type}. Skipping task." ) del self.tasks[0] # empty memory continue - # skip evaluation if results folder exists and overwrite_results is False + task_eval_splits = ( + eval_splits if eval_splits is not None else task.eval_splits + ) + task_subsets = list(task.metadata.hf_subsets_to_langscripts.keys()) + + existing_results = None + save_path = None + final_splits_to_run = task_eval_splits + missing_evaluations = self._get_missing_evaluations( + existing_results, + task_eval_splits, + task_subsets, + eval_subsets, + ) + if output_path: save_path = output_path / f"{task.metadata.name}{task.save_suffix}.json" - if save_path.exists() and not overwrite_results: + if save_path.exists(): + existing_results = TaskResult.from_disk(save_path) + + # Unified call to get missing splits and subsets + missing_evaluations = self._get_missing_evaluations( + existing_results, + task_eval_splits, + task_subsets, + eval_subsets, + ) + + if overwrite_results: + final_splits_to_run = task_eval_splits + else: + # Determine final splits to run + final_splits_to_run = [] + # We need to run any split that is fully missing or has missing subsets + for sp, info in missing_evaluations.items(): + if info["whole_split_missing"] or info["missing_subsets"]: + final_splits_to_run.append(sp) + + if not overwrite_results and len(final_splits_to_run) == 0: + logger.info( + f"{task.metadata.name} results already exists. Loading results from disk." + f" Set overwrite_results=True to overwrite or `--overwrite`." + ) + evaluation_results.append(existing_results) + del self.tasks[0] # empty memory + continue + + # If no splits need to be run and results exist, skip + if not final_splits_to_run: + if existing_results is not None: + evaluation_results.append(existing_results) + else: logger.info( - f"{task.metadata.name} results already exists. Loading results from disk. Set overwrite_results=True to overwrite." + f"No splits to evaluate for {task.metadata.name}. Skipping evaluation." ) - mteb_results = TaskResult.from_disk(save_path) - evaluation_results.append(mteb_results) - del self.tasks[0] # empty memory - continue - try: - task_eval_splits = ( - eval_splits if eval_splits is not None else task.eval_splits - ) + self.last_evaluated_splits[task.metadata.name] = [] + del self.tasks[0] + continue - # load data - logger.info(f"Loading dataset for {task.metadata_dict['name']}") - task.check_if_dataset_is_superseeded() - task.load_data(eval_splits=task_eval_splits, **kwargs) + try: + task.check_if_dataset_is_superseded() + task.load_data(**kwargs) - # run evaluation task_results = {} evaluation_time = 0 kg_co2_emissions: int | None = 0 if co2_tracker else None - for split in task_eval_splits: + + self.last_evaluated_splits[task.metadata.name] = [] + + for split in final_splits_to_run: + info = missing_evaluations[split] + + # Determine subsets to run for this split + # If the whole split is missing, run all required subsets + # If only some subsets are missing, run only those + subsets_to_run = ( + info["missing_subsets"] + if not overwrite_results + else (eval_subsets or task_subsets) + ) + + if ( + info["whole_split_missing"] or overwrite_results + ) and task_subsets is None: + subsets_to_run = ["default"] + if co2_tracker: try: from codecarbon import EmissionsTracker @@ -416,7 +563,6 @@ def run( raise ImportError( "To use the CO2 emissions tracker, please install codecarbon using 'pip install codecarbon'" ) - with EmissionsTracker( save_to_file=False, save_to_api=False, logging_logger=logger ) as tracker: @@ -426,6 +572,7 @@ def run( split, output_folder, encode_kwargs=encode_kwargs, + subsets_to_run=subsets_to_run, **kwargs, ) @@ -438,6 +585,7 @@ def run( model, split, output_folder, + subsets_to_run=subsets_to_run, encode_kwargs=encode_kwargs, **kwargs, ) @@ -449,23 +597,30 @@ def run( task_results[split] = results if verbosity >= 1: - logger.info(f"Scores: {results}") + logger.info(f"Scores: {task_results[split]}") - mteb_task_result = TaskResult.from_task_results( + self.last_evaluated_splits[task.metadata.name].append(split) + + # Create new TaskResult + new_results = TaskResult.from_task_results( task, task_results, evaluation_time=evaluation_time, kg_co2_emissions=kg_co2_emissions, ) - # save results + # Merge with existing if needed + if output_path and save_path.exists(): + existing_results = TaskResult.from_disk(save_path) + if existing_results: + merged_results = self._merge_results(existing_results, new_results) + else: + merged_results = new_results + if output_path: - with open(save_path, "w") as f_out: - json.dump( - mteb_task_result.to_dict(), f_out, indent=2, sort_keys=True - ) + merged_results.to_disk(save_path) - evaluation_results.append(mteb_task_result) + evaluation_results.append(merged_results) except Exception as e: logger.error( @@ -484,7 +639,6 @@ def run( # empty memory del self.tasks[0] - # restore original tasks self.tasks = original_tasks return evaluation_results @@ -535,3 +689,64 @@ def _save_model_metadata(model_meta: ModelMeta, output_folder: Path) -> None: with save_path.open("w") as f: json.dump(model_meta.to_dict(), f) + + def get_last_evaluated_splits(self): + """Returns a dictionary of tasks and their evaluated splits from the most recent run. + Tasks with empty lists indicate that results already existed and no splits were evaluated. + """ + return deepcopy( + {task: list(splits) for task, splits in self.last_evaluated_splits.items()} + ) + + @staticmethod + def _get_missing_evaluations( + existing_results: TaskResult | None, + task_eval_splits: list[str], + task_eval_langs: list[str], + eval_subsets: list[str] | None, + ) -> dict[str, dict[str, Any]]: + """Return a dictionary for each split, indicating if the whole split is missing and which subsets are missing.""" + missing_evaluations = { + split: {"whole_split_missing": False, "missing_subsets": []} + for split in task_eval_splits + } + + # Determine subsets to consider if multilingual + if eval_subsets is None: + # If no eval_langs specified, consider all subsets + subsets_to_consider = task_eval_langs + else: + subsets_to_consider = [ + subset for subset in task_eval_langs if subset in eval_subsets + ] + + # If no existing results, all splits and subsets are missing + if existing_results is None: + for split in task_eval_splits: + missing_evaluations[split]["whole_split_missing"] = True + missing_evaluations[split]["missing_subsets"] = list( + subsets_to_consider + ) + return missing_evaluations + + # If we have existing results, check which splits and subsets are missing + for split in task_eval_splits: + if split not in existing_results.scores: + # Whole split missing + missing_evaluations[split]["whole_split_missing"] = True + missing_evaluations[split]["missing_subsets"] = list( + subsets_to_consider + ) + else: + # Some subsets may be missing + existing_subsets = { + score_dict["hf_subset"] + for score_dict in existing_results.scores[split] + } + missing_subsets = [ + s for s in subsets_to_consider if s not in existing_subsets + ] + if missing_subsets: + missing_evaluations[split]["missing_subsets"] = missing_subsets + + return missing_evaluations diff --git a/mteb/evaluation/evaluators/BitextMiningEvaluator.py b/mteb/evaluation/evaluators/BitextMiningEvaluator.py index 122abdf799..1c03d3bb57 100644 --- a/mteb/evaluation/evaluators/BitextMiningEvaluator.py +++ b/mteb/evaluation/evaluators/BitextMiningEvaluator.py @@ -9,7 +9,6 @@ from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score from mteb.encoder_interface import Encoder -from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator from .utils import cos_sim @@ -45,26 +44,29 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): def compute_metrics(self, model: Encoder, encode_kwargs: dict[str, Any] = {}): pair_elements = {p for pair in self.pairs for p in pair} - subsets = [ - col for col in self.sentences.features.keys() if col in pair_elements - ] + if isinstance(self.sentences, Dataset): + subsets = [ + col for col in self.sentences.features.keys() if col in pair_elements + ] + else: + # BUCC outputs a dict instead of a Dataset + subsets = list(pair_elements) n_subsets = len(subsets) embeddings = {} for sub in tqdm.tqdm(subsets, desc=f"Encoding {n_subsets}x{self.n} sentences"): - emb = model.encode( + embeddings[sub] = model.encode( self.sentences[sub], task_name=self.task_name, **encode_kwargs, ) - embeddings[sub] = normalize_embeddings_to_numpy(emb) scores = {} for i, (key1, key2) in enumerate( tqdm.tqdm(self.pairs, desc="Matching sentences") ): scores[f"{key1}-{key2}"] = self._compute_metrics( - embeddings[key1], embeddings[key2] + embeddings[key1], embeddings[key2], model ) # in case of default pair unnest the dict @@ -78,10 +80,13 @@ def _compute_metrics( self, embeddings1, embeddings2, + model: Encoder, ): # Find nearest neighbors logger.info("Finding nearest neighbors...") - nearest_neighbors = self._similarity_search(embeddings1, embeddings2, top_k=1) + nearest_neighbors = self._similarity_search( + embeddings1, embeddings2, model, top_k=1 + ) # Compute errors logger.info("Computing metrics...") @@ -108,10 +113,10 @@ def _similarity_search( self, query_embeddings, corpus_embeddings, + model: Encoder, query_chunk_size: int = 100, corpus_chunk_size: int = 500000, top_k: int = 10, - score_function=cos_sim, ): """This function performs a cosine similarity search between a list of query embeddings and a list of corpus embeddings. It can be used for Information Retrieval / Semantic Search for corpora up to about 1 Million entries. @@ -119,10 +124,10 @@ def _similarity_search( Args: query_embeddings: A 2 dimensional tensor with the query embeddings. corpus_embeddings: A 2 dimensional tensor with the corpus embeddings. + model: The model used to encode the queries and corpus. This is used to check if the embeddings are on the same device and to encode the queries and corpus if they are not already tensors. query_chunk_size: Process 100 queries simultaneously. Increasing that value increases the speed, but requires more memory. corpus_chunk_size: Scans the corpus 100k entries at a time. Increasing that value increases the speed, but requires more memory. top_k: Retrieve top k matching entries. - score_function: Function for computing scores. By default, cosine similarity. Returns: Returns a list with one entry for each query. Each entry is a list of dictionaries with the keys 'corpus_id' and 'score', sorted by decreasing cosine similarity scores. @@ -144,7 +149,7 @@ def _similarity_search( # Iterate over chunks of the corpus for corpus_start_idx in range(0, len(corpus_embeddings), corpus_chunk_size): # Compute cosine similarities - cos_scores = score_function( + similarity_scores = cos_sim( query_embeddings[ query_start_idx : query_start_idx + query_chunk_size ], @@ -153,10 +158,20 @@ def _similarity_search( ], ) + if hasattr(model, "similarity"): + similarity_scores = model.similarity( + query_embeddings[ + query_start_idx : query_start_idx + query_chunk_size + ], + corpus_embeddings[ + corpus_start_idx : corpus_start_idx + corpus_chunk_size + ], + ) + # Get top-k scores cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( - cos_scores, - min(top_k, len(cos_scores[0])), + similarity_scores, + min(top_k, len(similarity_scores[0])), dim=1, largest=True, sorted=False, @@ -164,7 +179,7 @@ def _similarity_search( cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() - for query_itr in range(len(cos_scores)): + for query_itr in range(len(similarity_scores)): for sub_corpus_id, score in zip( cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr], diff --git a/mteb/evaluation/evaluators/ClassificationEvaluator.py b/mteb/evaluation/evaluators/ClassificationEvaluator.py index a5bb725315..955f269de7 100644 --- a/mteb/evaluation/evaluators/ClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/ClassificationEvaluator.py @@ -15,7 +15,6 @@ from torch import Tensor from mteb.encoder_interface import Encoder -from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator @@ -63,19 +62,17 @@ def __call__(self, model, test_cache=None): max_accuracy = 0 max_f1 = 0 max_ap = 0 - emb = model.encode( + X_train = model.encode( self.sentences_train, task_name=self.task_name, **self.encode_kwargs, ) - X_train = normalize_embeddings_to_numpy(emb) if test_cache is None: - emb = model.encode( + X_test = model.encode( self.sentences_test, task_name=self.task_name, **self.encode_kwargs, ) - X_test = normalize_embeddings_to_numpy(emb) test_cache = X_test else: X_test = test_cache @@ -139,21 +136,17 @@ def __call__(self, model: Encoder, test_cache=None): max_accuracy = 0 max_f1 = 0 max_ap = 0 - X_train = normalize_embeddings_to_numpy( - model.encode( - self.sentences_train, - task_name=self.task_name, - **self.encode_kwargs, - ) + X_train = model.encode( + self.sentences_train, + task_name=self.task_name, + **self.encode_kwargs, ) if test_cache is None: - X_test = normalize_embeddings_to_numpy( - model.encode( - self.sentences_test, - task_name=self.task_name, - **self.encode_kwargs, - ) + X_test = model.encode( + self.sentences_test, + task_name=self.task_name, + **self.encode_kwargs, ) test_cache = X_test else: @@ -295,20 +288,16 @@ def __call__(self, model, test_cache=None): max_iter=self.max_iter, verbose=1 if logger.isEnabledFor(logging.DEBUG) else 0, ) - X_train = normalize_embeddings_to_numpy( - model.encode( - self.sentences_train, - task_name=self.task_name, - **self.encode_kwargs, - ) + X_train = model.encode( + self.sentences_train, + task_name=self.task_name, + **self.encode_kwargs, ) if test_cache is None: - X_test = normalize_embeddings_to_numpy( - model.encode( - self.sentences_test, - task_name=self.task_name, - **self.encode_kwargs, - ) + X_test = model.encode( + self.sentences_test, + task_name=self.task_name, + **self.encode_kwargs, ) test_cache = X_test else: diff --git a/mteb/evaluation/evaluators/ClusteringEvaluator.py b/mteb/evaluation/evaluators/ClusteringEvaluator.py index 138ac565d6..b0a21e4469 100644 --- a/mteb/evaluation/evaluators/ClusteringEvaluator.py +++ b/mteb/evaluation/evaluators/ClusteringEvaluator.py @@ -8,7 +8,6 @@ from sklearn import metrics from mteb.encoder_interface import Encoder -from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator @@ -38,12 +37,10 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): if "batch_size" not in encode_kwargs: encode_kwargs["batch_size"] = 32 - corpus_embeddings = normalize_embeddings_to_numpy( - model.encode( - self.sentences, - task_name=self.task_name, - **encode_kwargs, - ) + corpus_embeddings = model.encode( + self.sentences, + task_name=self.task_name, + **encode_kwargs, ) logger.info("Fitting Mini-Batch K-Means model...") diff --git a/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py b/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py index f17dad9872..154717d9f1 100644 --- a/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/InstructionRetrievalEvaluator.py @@ -34,7 +34,6 @@ def __call__( corpus, queries, self.top_k, - self.score_function, task_name=self.task_name, # type: ignore instructions=instructions, **kwargs, @@ -44,7 +43,6 @@ def __call__( corpus, queries, self.top_k, - self.score_function, instructions=instructions, request_qid=qid, task_name=self.task_name, diff --git a/mteb/evaluation/evaluators/PairClassificationEvaluator.py b/mteb/evaluation/evaluators/PairClassificationEvaluator.py index c402383fcd..7a53b7bdf5 100644 --- a/mteb/evaluation/evaluators/PairClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/PairClassificationEvaluator.py @@ -13,7 +13,6 @@ ) from mteb.encoder_interface import Encoder, EncoderWithSimilarity -from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator @@ -90,12 +89,10 @@ def compute_metrics( logger.warning( f"Found {n_duplicates}/{total_sents} duplicates in the input data. Only encoding unique sentences." ) - embeddings = normalize_embeddings_to_numpy( - model.encode( - sentences, - task_name=self.task_name, - **encode_kwargs, - ) + embeddings = model.encode( + sentences, + task_name=self.task_name, + **encode_kwargs, ) emb_dict = dict(zip(sentences, embeddings)) embeddings1 = [emb_dict[sent] for sent in self.sentences1] diff --git a/mteb/evaluation/evaluators/RerankingEvaluator.py b/mteb/evaluation/evaluators/RerankingEvaluator.py index 44994681be..3c45126bdf 100644 --- a/mteb/evaluation/evaluators/RerankingEvaluator.py +++ b/mteb/evaluation/evaluators/RerankingEvaluator.py @@ -9,7 +9,6 @@ from sklearn.metrics import average_precision_score from mteb.evaluation.evaluators.RetrievalEvaluator import RetrievalEvaluator -from mteb.normalize_embeddings import normalize_embeddings_to_numpy from ...encoder_interface import Encoder, PromptType from .Evaluator import Evaluator @@ -35,7 +34,6 @@ def __init__( task_name: str | None = None, mrr_at_k: int = 10, name: str = "", - similarity_fct=cos_sim, encode_kwargs: dict[str, Any] = {}, use_batched_encoding: bool = True, limit: int | None = None, @@ -49,7 +47,6 @@ def __init__( self.samples = samples self.name = name self.mrr_at_k = mrr_at_k - self.similarity_fct = similarity_fct self.use_batched_encoding = use_batched_encoding self.task_name = task_name self.k_values = k_values @@ -86,7 +83,7 @@ def compute_metrics_batched(self, model: Encoder): """ logger.info("Encoding queries...") if isinstance(self.samples[0]["query"], str): - all_query_embs = normalize_embeddings_to_numpy( + all_query_embs = np.asarray( model.encode( [sample["query"] for sample in self.samples], task_name=self.task_name, @@ -212,6 +209,7 @@ def _encode_candidates_batched( all_mrr_scores, all_ap_scores, all_conf_scores, + model, ) def _encode_candidates_individual( @@ -258,6 +256,7 @@ def _encode_candidates_individual( all_mrr_scores, all_ap_scores, all_conf_scores, + model, ) def _collect_results(self, all_mrr_scores, all_ap_scores, all_conf_scores): @@ -317,7 +316,7 @@ def _encode_candidates_miracl_batched(self, all_query_embs, model: Encoder): docs_idx += num_doc fake_qid = str(query_idx) - results[fake_qid] = self.rerank(query_emb, docs_emb) + results[fake_qid] = self.rerank(query_emb, docs_emb, model) qrels[fake_qid] = { str(i): 1 if doc in positive else 0 for i, doc in enumerate(docs) } @@ -352,7 +351,7 @@ def _encode_candidates_miracl_individual(self, model: Encoder): ) fake_qid = str(i) - results[fake_qid] = self.rerank(query_emb, docs_emb) + results[fake_qid] = self.rerank(query_emb, docs_emb, model) qrels[fake_qid] = { str(i): 1 if doc in positive else 0 for i, doc in enumerate(docs) } @@ -372,7 +371,7 @@ def _collect_miracl_results(self, results, qrels): return scores_miracl def rerank( - self, query_emb: torch.Tensor, docs_emb: torch.Tensor + self, query_emb: np.ndarray, docs_emb: np.ndarray, model: Encoder ) -> dict[str, float]: """Rerank documents (docs_emb) given the query (query_emb) @@ -380,6 +379,7 @@ def rerank( query_emb: Query embedding of shape `(num_queries, hidden_size)`) if `num_queries` > 0: we take the closest document to any of the queries docs_emb: Candidates documents embeddings of shape `(num_pos+num_neg, hidden_size)`) + model: Model to use for computing similarity scores if model.similarity is available Returns: similarity_scores: @@ -390,7 +390,10 @@ def rerank( if not docs_emb.shape[0]: return {"empty-docid": 0} - pred_scores = self.similarity_fct(query_emb, docs_emb) + if hasattr(model, "similarity"): + pred_scores = model.similarity(query_emb, docs_emb) + else: + pred_scores = cos_sim(query_emb, docs_emb) if len(pred_scores.shape) > 1: pred_scores = torch.amax(pred_scores, dim=0) @@ -406,8 +409,9 @@ def _apply_sim_scores( all_mrr_scores, all_ap_scores, all_conf_scores, + model: Encoder, ): - sim_scores = self._compute_sim_scores_instance(query_emb, docs_emb) + sim_scores = self._compute_sim_scores_instance(query_emb, docs_emb, model) scores = self._compute_metrics_instance(sim_scores, is_relevant) conf_scores = self.conf_scores(sim_scores.tolist()) @@ -431,9 +435,9 @@ def _encode_unique_texts( all_unique_texts.append(text) all_texts_indexes.append(index_map[text_hash]) logger.warning( - f"A total on {len(all_texts) - len(all_unique_texts)}/{len(all_texts)} duplicate texts were found during encoding. Only encoding unique text and duplicating embeddings across." + f"A total of {len(all_texts) - len(all_unique_texts)}/{len(all_texts)} duplicate texts were found during encoding. Only encoding unique text and duplicating embeddings across." ) - all_unique_texts_embs = normalize_embeddings_to_numpy( + all_unique_texts_embs = np.asarray( model.encode( all_unique_texts, task_name=task_name, @@ -444,7 +448,7 @@ def _encode_unique_texts( return all_unique_texts_embs[all_texts_indexes] def _compute_sim_scores_instance( - self, query_emb: torch.Tensor, docs_emb: torch.Tensor + self, query_emb: np.ndarray, docs_emb: np.ndarray, model: Encoder ) -> torch.Tensor: """Computes similarity scores for a single instance = (query, positives, negatives) @@ -452,11 +456,15 @@ def _compute_sim_scores_instance( query_emb: Query embedding, with shape `(num_queries, hidden_size)` if `num_queries` > 0: we take the closest document to any of the queries docs_emb: Candidates documents embeddings, with shape `(num_pos+num_neg, hidden_size)` + model: Model to use for computing similarity scores if model.similarity is available Returns: sim_scores: Query-documents similarity scores, with shape `(num_pos+num_neg,)` """ - sim_scores = self.similarity_fct(query_emb, docs_emb) + if hasattr(model, "similarity"): + sim_scores = model.similarity(query_emb, docs_emb) + else: + sim_scores = cos_sim(query_emb, docs_emb) if len(sim_scores.shape) > 1: sim_scores = torch.amax(sim_scores, dim=0) diff --git a/mteb/evaluation/evaluators/RetrievalEvaluator.py b/mteb/evaluation/evaluators/RetrievalEvaluator.py index 23bbc280e9..cdf497e5a6 100644 --- a/mteb/evaluation/evaluators/RetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/RetrievalEvaluator.py @@ -16,14 +16,12 @@ from mteb.encoder_interface import Encoder, PromptType from mteb.model_meta import ModelMeta -from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator from .utils import ( confidence_scores, convert_conv_history_to_query, cos_sim, - dot_score, download, hole, mrr, @@ -78,11 +76,6 @@ def __init__( if "convert_to_tensor" not in encode_kwargs: encode_kwargs["convert_to_tensor"] = True - self.score_functions = {"cos_sim": cos_sim, "dot": dot_score} - self.score_function_desc = { - "cos_sim": "Cosine Similarity", - "dot": "Dot Product", - } self.corpus_chunk_size = corpus_chunk_size if isinstance(previous_results, Path): self.previous_results = str(previous_results) @@ -107,21 +100,12 @@ def search( corpus: dict[str, dict[str, str]], queries: dict[str, str | list[str]], top_k: int, - score_function: str, task_name: str, instructions: dict[str, str] | None = None, request_qid: str | None = None, return_sorted: bool = False, **kwargs, ) -> dict[str, dict[str, float]]: - # Create embeddings for all queries using model.encode - # Runs semantic search against the corpus embeddings - # Returns a ranked list with the corpus ids - if score_function not in self.score_functions: - raise ValueError( - f"score function: {score_function} must be either (cos_sim) for cosine similarity or (dot) for dot product" - ) - logger.info("Encoding Queries.") query_ids = list(queries.keys()) self.results = {qid: {} for qid in query_ids} @@ -136,13 +120,11 @@ def search( **self.encode_kwargs, ) else: - query_embeddings = normalize_embeddings_to_numpy( - self.model.encode( - queries, # type: ignore - task_name=task_name, - prompt_type=PromptType.query, - **self.encode_kwargs, - ) + query_embeddings = self.model.encode( + queries, # type: ignore + task_name=task_name, + prompt_type=PromptType.query, + **self.encode_kwargs, ) logger.info("Sorting Corpus by document length (Longest first)...") @@ -153,9 +135,6 @@ def search( corpus = [corpus[cid] for cid in corpus_ids] # type: ignore logger.info("Encoding Corpus in batches... Warning: This might take a while!") - logger.info( - f"Scoring Function: {self.score_function_desc[score_function]} ({score_function})" - ) itr = range(0, len(corpus), self.corpus_chunk_size) @@ -177,42 +156,53 @@ def search( ) else: # Encode chunk of corpus - sub_corpus_embeddings = normalize_embeddings_to_numpy( - self.model.encode( - corpus[corpus_start_idx:corpus_end_idx], # type: ignore - task_name=task_name, - prompt_type=PromptType.passage, - request_qid=request_qid, - **self.encode_kwargs, - ) + sub_corpus_embeddings = self.model.encode( + corpus[corpus_start_idx:corpus_end_idx], # type: ignore + task_name=task_name, + prompt_type=PromptType.passage, + request_qid=request_qid, + **self.encode_kwargs, ) if self.save_corpus_embeddings and request_qid: self.corpus_embeddings[request_qid].append(sub_corpus_embeddings) - # Compute similarites using either cosine-similarity or dot product - cos_scores = self.score_functions[score_function]( - query_embeddings, sub_corpus_embeddings - ) - cos_scores[torch.isnan(cos_scores)] = -1 + # Compute similarites using self defined similarity otherwise default to cosine-similarity + if hasattr(self.model, "similarity"): + similarity_scores = self.model.similarity( + query_embeddings, sub_corpus_embeddings + ) + else: + similarity_scores = cos_sim(query_embeddings, sub_corpus_embeddings) + is_nan = torch.isnan(similarity_scores) + if is_nan.sum() > 0: + logger.warning( + f"Found {is_nan.sum()} NaN values in the similarity scores. Replacing NaN values with -1." + ) + similarity_scores[is_nan] = -1 # Get top-k values - cos_scores_top_k_values, cos_scores_top_k_idx = torch.topk( - cos_scores, + similarity_scores_top_k_values, similarity_scores_top_k_idx = torch.topk( + similarity_scores, min( top_k + 1, - len(cos_scores[1]) if len(cos_scores) > 1 else len(cos_scores[-1]), + len(similarity_scores[1]) + if len(similarity_scores) > 1 + else len(similarity_scores[-1]), ), dim=1, largest=True, sorted=return_sorted, ) - cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist() - cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist() + similarity_scores_top_k_values = ( + similarity_scores_top_k_values.cpu().tolist() + ) + similarity_scores_top_k_idx = similarity_scores_top_k_idx.cpu().tolist() for query_itr in range(len(query_embeddings)): query_id = query_ids[query_itr] for sub_corpus_id, score in zip( - cos_scores_top_k_idx[query_itr], cos_scores_top_k_values[query_itr] + similarity_scores_top_k_idx[query_itr], + similarity_scores_top_k_values[query_itr], ): corpus_id = corpus_ids[corpus_start_idx + sub_corpus_id] if len(result_heaps[query_id]) < top_k: @@ -317,13 +307,17 @@ def search_cross_encoder( assert ( len(queries_in_pair) == len(corpus_in_pair) == len(instructions_in_pair) ) + corpus_in_pair = corpus_to_str(list(corpus_in_pair)) - if isinstance(self.model.model, CrossEncoder): + if hasattr(self.model, "model") and isinstance( + self.model.model, CrossEncoder + ): # can't take instructions, so add them here - queries_in_pair = [ - f"{q} {i}".strip() - for i, q in zip(instructions_in_pair, queries_in_pair) - ] + if instructions_in_pair[0] is not None: + queries_in_pair = [ + f"{q} {i}".strip() + for i, q in zip(instructions_in_pair, queries_in_pair) + ] scores = self.model.predict(list(zip(queries_in_pair, corpus_in_pair))) # type: ignore else: # may use the instructions in a unique way, so give them also @@ -356,10 +350,8 @@ def encode_conversations( "Model doesn't have encode_conversations fallback to default implementation" ) queries = self.convert_conv_history_to_query(model, conversations) # type: ignore - return normalize_embeddings_to_numpy( - model.encode( - queries, task_name=task_name, prompt_type=PromptType.query, **kwargs - ) + return model.encode( + queries, task_name=task_name, prompt_type=PromptType.query, **kwargs ) # type: ignore @staticmethod @@ -385,6 +377,9 @@ def __init__(self, model, **kwargs): self.save_corpus_embeddings = kwargs.get("save_corpus_embeddings", False) self.corpus_embeddings = {} + if hasattr(self.model, "similarity") and callable(self.model.similarity): + self.similarity = self.model.similarity + def encode_corpus( self, corpus: list[dict[str, str]], @@ -402,14 +397,12 @@ def encode_corpus( return self.corpus_embeddings[request_qid] sentences = corpus_to_str(corpus) - corpus_embeddings = normalize_embeddings_to_numpy( - self.model.encode( - sentences, - task_name=task_name, - prompt_type=prompt_type, - batch_size=batch_size, - **kwargs, - ) + corpus_embeddings = self.model.encode( + sentences, + task_name=task_name, + prompt_type=prompt_type, + batch_size=batch_size, + **kwargs, ) if self.save_corpus_embeddings and request_qid: @@ -419,19 +412,16 @@ def encode_corpus( def encode( self, sentences: list[str], - *, task_name: str, prompt_type: PromptType | None = None, - **kwargs: Any, + **kwargs, ): if prompt_type and prompt_type == PromptType.passage: return self.encode_corpus( sentences, task_name, prompt_type=prompt_type, **kwargs ) - return normalize_embeddings_to_numpy( - self.model.encode( - sentences, task_name=task_name, prompt_type=prompt_type, **kwargs - ) + return self.model.encode( + sentences, task_name=task_name, prompt_type=prompt_type, **kwargs ) @@ -447,7 +437,6 @@ def __init__( retriever, task_name: str | None = None, k_values: list[int] = [1, 3, 5, 10, 20, 100, 1000], - score_function: str = "cos_sim", encode_kwargs: dict[str, Any] = {}, **kwargs, ): @@ -469,7 +458,6 @@ def __init__( self.top_k = ( max(k_values) if "top_k" not in kwargs else kwargs["top_k"] ) # can lower it if reranking - self.score_function = score_function self.task_name = task_name def __call__( @@ -483,14 +471,14 @@ def __call__( if self.is_cross_encoder: return self.retriever.search_cross_encoder(corpus, queries, self.top_k) elif ( - hasattr(self.retriever.model, "mteb_model_meta") - and self.retriever.model.mteb_model_meta.name == "bm25s" + hasattr(self.retriever.model.model, "mteb_model_meta") + and self.retriever.model.model.mteb_model_meta.name == "bm25s" ): - return self.retriever.model.search( + return self.retriever.model.model.search( corpus, queries, self.top_k, - self.score_function, + score_function="bm25", task_name=self.task_name, # type: ignore ) else: @@ -498,7 +486,6 @@ def __call__( corpus, queries, self.top_k, - self.score_function, task_name=self.task_name, # type: ignore ) diff --git a/mteb/evaluation/evaluators/STSEvaluator.py b/mteb/evaluation/evaluators/STSEvaluator.py index 3d769fa957..2f6e8e4a46 100644 --- a/mteb/evaluation/evaluators/STSEvaluator.py +++ b/mteb/evaluation/evaluators/STSEvaluator.py @@ -12,7 +12,6 @@ ) from mteb.encoder_interface import Encoder, EncoderWithSimilarity -from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator @@ -45,19 +44,15 @@ def __call__( *, encode_kwargs: dict[str, Any] = {}, ): - embeddings1 = normalize_embeddings_to_numpy( - model.encode( - self.sentences1, - task_name=self.task_name, - **encode_kwargs, - ) + embeddings1 = model.encode( + self.sentences1, + task_name=self.task_name, + **encode_kwargs, ) - embeddings2 = normalize_embeddings_to_numpy( - model.encode( - self.sentences2, - task_name=self.task_name, - **encode_kwargs, - ) + embeddings2 = model.encode( + self.sentences2, + task_name=self.task_name, + **encode_kwargs, ) logger.info("Evaluating...") diff --git a/mteb/evaluation/evaluators/SummarizationEvaluator.py b/mteb/evaluation/evaluators/SummarizationEvaluator.py index 153ef8c42b..df077fd44a 100644 --- a/mteb/evaluation/evaluators/SummarizationEvaluator.py +++ b/mteb/evaluation/evaluators/SummarizationEvaluator.py @@ -10,7 +10,6 @@ from scipy.stats import pearsonr, spearmanr from mteb.encoder_interface import Encoder, EncoderWithSimilarity -from mteb.normalize_embeddings import normalize_embeddings_to_numpy from .Evaluator import Evaluator from .utils import cos_sim, dot_score @@ -75,29 +74,25 @@ def __call__( ] logger.info("Encoding human summaries...") - embs_human_summaries_all = normalize_embeddings_to_numpy( - model.encode( - [ - summary - for human_summaries in self.human_summaries - for summary in human_summaries - ], - task_name=self.task_name, - **encode_kwargs, - ) + embs_human_summaries_all = model.encode( + [ + summary + for human_summaries in self.human_summaries + for summary in human_summaries + ], + task_name=self.task_name, + **encode_kwargs, ) logger.info("Encoding machine summaries...") - embs_machine_summaries_all = normalize_embeddings_to_numpy( - model.encode( - [ - summary - for machine_summaries in self.machine_summaries - for summary in machine_summaries - ], - task_name=self.task_name, - **encode_kwargs, - ) + embs_machine_summaries_all = model.encode( + [ + summary + for machine_summaries in self.machine_summaries + for summary in machine_summaries + ], + task_name=self.task_name, + **encode_kwargs, ) # Split the embeddings into the original human & machine summaries diff --git a/mteb/language_family.json b/mteb/language_family.json new file mode 100644 index 0000000000..5770aa6712 --- /dev/null +++ b/mteb/language_family.json @@ -0,0 +1,62611 @@ +{ + "aaa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "North-Central Edoid", + "level6": "Afenmai-Bendel" + }, + "aab": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Alumic", + "level5": "Alumu-Akpondu" + }, + "aac": { + "level0": "Suki-Gogodala", + "level1": "Gogodalic", + "level2": "Ari-Waruna" + }, + "aad": { + "level0": "Sepik" + }, + "aae": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Albanian", + "level3": "Albanian-Tosk", + "level4": "Southern Tosk" + }, + "aaf": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "aag": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Nuclear Palai", + "level4": "Yangum-Ambrak" + }, + "aah": { + "level0": "Nuclear Torricelli", + "level1": "Kombio-Arapesh-Urat", + "level2": "Arapesh", + "level3": "Mufian-Bukiyip-Abu", + "level4": "Bukiyip-Abu" + }, + "aai": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Are linkage" + }, + "aak": { + "level0": "Angan", + "level1": "Nuclear Angan", + "level2": "Ankave-Tainae-Akoye" + }, + "aal": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Kotoko-Buduma", + "level5": "Kotoko Septentrional", + "level6": "Kotoko Septentrional 1" + }, + "aam": { + "level0": "Bookkeeping" + }, + "aan": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup V", + "level6": "Arawetic" + }, + "aao": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "North African Arabic" + }, + "aap": { + "level0": "Cariban", + "level1": "Pekodian", + "level2": "Xinguan" + }, + "aaq": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Maritimes-Southern New England Algonquian", + "level5": "Northern Eastern Algonquian", + "level6": "Abenaki" + }, + "aar": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Saho-Afar" + }, + "aas": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "South Cushitic" + }, + "aat": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Albanian", + "level3": "Albanian-Tosk", + "level4": "Southern Tosk" + }, + "aau": { + "level0": "Sepik" + }, + "aaw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Arawe", + "level11": "West Arawe" + }, + "aax": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Dumut", + "level6": "Mandobo" + }, + "aay": { + "level0": "Bookkeeping" + }, + "aaz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Rote-Meto", + "level4": "West Rote", + "level5": "Dengka-Meto", + "level6": "Meto" + }, + "aba": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Agneby" + }, + "abb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Basaa-Yaunde (A40-70)", + "level8": "Basaa (A.40)", + "level9": "Abo-Barombi" + }, + "abc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Luzon", + "level3": "Sambalic" + }, + "abd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Alabat-Manide Agta" + }, + "abe": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Maritimes-Southern New England Algonquian", + "level5": "Northern Eastern Algonquian", + "level6": "Abenaki" + }, + "abf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Paitanic" + }, + "abg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Siane-Yagaria", + "level5": "Kamano-Yagaria", + "level6": "Unclassified Kamano-Yagaria" + }, + "abh": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Eastern Arabic", + "level7": "Central Asian Arabic" + }, + "abi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Agneby" + }, + "abj": { + "level0": "Great Andamanese", + "level1": "South Great Andamanese" + }, + "abk": { + "level0": "Abkhaz-Adyge", + "level1": "Abkhaz-Abaza" + }, + "abl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Lampungic" + }, + "abm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Ekoid-Mbe", + "level6": "Ekoid", + "level7": "Bakor-Ejagham", + "level8": "Bakor", + "level9": "Northern Bakor", + "level10": "Abanyom-Nkem-Nkum" + }, + "abn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Central Delta", + "level5": "Abua-Odual" + }, + "abo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "North Tivoid" + }, + "abp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Luzon", + "level3": "Sambalic", + "level4": "Abellen-Botolan" + }, + "abq": { + "level0": "Abkhaz-Adyge", + "level1": "Abkhaz-Abaza" + }, + "abr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Central Tano", + "level7": "Akanic" + }, + "abs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay", + "level6": "Eastern Indonesia Trade Malay", + "level7": "Ambonic Malay" + }, + "abt": { + "level0": "Ndu", + "level1": "Nuclear Ndu", + "level2": "Ambulas-Hanga-Hundi" + }, + "abu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Western Tano" + }, + "abv": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Arabian Peninsula Arabic", + "level7": "North Arabian Beduin Arabic" + }, + "abw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Omosan" + }, + "abx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Sama-Bajaw" + }, + "aby": { + "level0": "Yareban", + "level1": "Doriri-Abia" + }, + "abz": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "Central Alor", + "level4": "Abuic" + }, + "aca": { + "level0": "Arawakan", + "level1": "Japura-Colombia", + "level2": "Nuclear Japura-Colombia", + "level3": "Northeast Japura-Colombia", + "level4": "Piapoco-Achagua" + }, + "acb": { + "level0": "Bookkeeping" + }, + "acc": { + "level0": "Bookkeeping" + }, + "acd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Oti North Guang", + "level9": "Mountain Oti North Guang", + "level10": "Gikyode-Ginyanga", + "level11": "Gikyode-Foodo" + }, + "ace": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic" + }, + "acf": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil", + "level14": "Macro-French", + "level15": "Circum-Caribbean French", + "level16": "Lesser Antillean French Creole" + }, + "ach": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Southern Lwoo" + }, + "aci": { + "level0": "Great Andamanese", + "level1": "North Andamanese-Akakede", + "level2": "Northern Great Andamanese", + "level3": "Bo-Cari" + }, + "ack": { + "level0": "Great Andamanese", + "level1": "North Andamanese-Akakede", + "level2": "Northern Great Andamanese", + "level3": "Jeru-Kora" + }, + "acl": { + "level0": "Great Andamanese", + "level1": "South Great Andamanese" + }, + "acm": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Eastern Arabic" + }, + "acn": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Southern Burmish", + "level5": "Achangic" + }, + "acp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Shiroro-Kamuku", + "level6": "Kamuku-Hungwarya", + "level7": "Kamuku", + "level8": "Rogo-Sagamuk-Sama-Sambuga", + "level9": "Sagamuk-Sama-Sambuga" + }, + "acq": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Arabian Peninsula Arabic" + }, + "acr": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Quichean", + "level4": "Core Quichean", + "level5": "Quiche-Achi" + }, + "acs": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Cerrado", + "level3": "Central Je" + }, + "acu": { + "level0": "Chicham", + "level1": "Shuaric" + }, + "acv": { + "level0": "Palaihnihan" + }, + "acw": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Arabian Peninsula Arabic" + }, + "acx": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Arabian Peninsula Arabic", + "level7": "North Arabian Beduin Arabic" + }, + "acy": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Levantine-Cypriot Arabic" + }, + "acz": { + "level0": "Narrow Talodi", + "level1": "Buram-Saraf", + "level2": "Acheron-Tocho" + }, + "ada": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Ga-Dangme" + }, + "adb": { + "level0": "Bookkeeping" + }, + "add": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Nkambe" + }, + "ade": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Na-Togo", + "level4": "Basila-Adele" + }, + "adf": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Arabian Peninsula Arabic", + "level7": "North Arabian Beduin Arabic", + "level8": "Dhofaric" + }, + "adg": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Arandic", + "level3": "South Arandic", + "level4": "Upper Arrernte", + "level5": "Central-Eastern Arrernte" + }, + "adh": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Southern Lwoo", + "level4": "Adhola-Alur-Luo", + "level5": "Adhola-Luo" + }, + "adi": { + "level0": "Sino-Tibetan", + "level1": "Macro-Tani", + "level2": "Tani", + "level3": "Eastern Tani" + }, + "adj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Agneby" + }, + "adl": { + "level0": "Sino-Tibetan", + "level1": "Macro-Tani", + "level2": "Tani", + "level3": "Pre-Western Tani", + "level4": "Western Tani", + "level5": "Subansiri" + }, + "adn": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "West Alor-Straits-Pantar", + "level4": "West Alor" + }, + "ado": { + "level0": "Ramu", + "level1": "Agoan" + }, + "adp": { + "level0": "Bookkeeping" + }, + "adq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Western Gbe", + "level5": "Eweic" + }, + "adr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata", + "level4": "Lamaholot Barat", + "level5": "North Lembata-Adonara" + }, + "ads": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "adt": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Thura-Yura", + "level3": "Core Thura Yura", + "level4": "Northern Thura-Yura" + }, + "adu": { + "level0": "Bookkeeping" + }, + "adw": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VI", + "level6": "Kawahiva", + "level7": "Nuclear Kawahiva", + "level8": "Central Kawahiva", + "level9": "Amondava-Uru-Eu-Wau-Wau" + }, + "adx": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "North-Eastern Tibetic" + }, + "ady": { + "level0": "Abkhaz-Adyge", + "level1": "Circassian" + }, + "adz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Upper Markham" + }, + "aea": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Norman Pama", + "level3": "Kuthant-Gurdjar", + "level4": "Rib-Gurdjar" + }, + "aeb": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "North African Arabic", + "level7": "Malta-Tunisian Arabic" + }, + "aec": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Egyptic Arabic", + "level7": "Egypto-Sudanic Arabic" + }, + "aed": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "aee": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Pashayi", + "level5": "Eastern Pashayi" + }, + "aek": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Voh-Kone-Cem-Pac", + "level10": "Voh-Kone", + "level11": "Bwatooic", + "level12": "Haeke-Bwatoo" + }, + "ael": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields" + }, + "aem": { + "level0": "Austroasiatic", + "level1": "Vietic", + "level2": "Chutic", + "level3": "East Chutic" + }, + "aen": { + "level0": "Sign Language", + "level1": "Auxiliary Sign Systems" + }, + "aeq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Gujaratic", + "level10": "Western Gujaratic" + }, + "aer": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Arandic", + "level3": "South Arandic", + "level4": "Upper Arrernte", + "level5": "Central-Eastern Arrernte" + }, + "aeu": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Hanic", + "level7": "Ha-Ya", + "level8": "Akhaic", + "level9": "Akeuic" + }, + "aew": { + "level0": "Keram", + "level1": "East Keram" + }, + "aex": { + "level0": "Bookkeeping" + }, + "aey": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Gum" + }, + "aez": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean", + "level3": "Nuclear Binanderean", + "level4": "South Binanderean", + "level5": "Orokaivic" + }, + "afb": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Arabian Peninsula Arabic", + "level7": "North Arabian Beduin Arabic" + }, + "afd": { + "level0": "Arafundi" + }, + "afe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Bendic", + "level6": "Nuclear Bendic" + }, + "afg": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "afh": { + "level0": "Artificial Language" + }, + "afi": { + "level0": "Ramu", + "level1": "Goam", + "level2": "Tamolan" + }, + "afk": { + "level0": "Arafundi" + }, + "afn": { + "level0": "Ijoid" + }, + "afo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau" + }, + "afp": { + "level0": "Arafundi" + }, + "afr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch", + "level6": "Middle-Modern Dutch", + "level7": "Modern Dutch", + "level8": "Global Dutch", + "level9": "Afrikaansic" + }, + "afs": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Gullah-Nevis-Antigua", + "level15": "Gullah" + }, + "aft": { + "level0": "Nyimang" + }, + "afu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "South Guang" + }, + "afz": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "East Tariku", + "level3": "Eritai-Obokuitai-Biritai" + }, + "aga": { + "level0": "Unattested", + "level1": "Arawakan (Unattested)" + }, + "agb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "East-West Central Delta Cross", + "level7": "Mbembe-Legbo", + "level8": "Legboic" + }, + "agc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Idomoid", + "level4": "Akweya", + "level5": "Etulo-Idoma", + "level6": "Nuclear Idoma", + "level7": "Idoma-Agatu-Okpogu" + }, + "agd": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Kainantu", + "level3": "Gauwa", + "level4": "Gadsup-Agarabi" + }, + "age": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Kewa-Huli", + "level3": "Sau-Angal-Kewa", + "level4": "Angal-Kewa", + "level5": "Angal Mendi" + }, + "agf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tanimbar-Bomberai", + "level3": "Nuta" + }, + "agg": { + "level0": "Senagi" + }, + "agh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Middle Bomokandian", + "level15": "Late Bomokandian" + }, + "agi": { + "level0": "Unattested", + "level1": "Dravidian (Unattested)" + }, + "agj": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Amharic-Argobba" + }, + "agk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bikol", + "level5": "Inagta Bikol" + }, + "agl": { + "level0": "East Strickland" + }, + "agm": { + "level0": "Angan" + }, + "agn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Kalamian" + }, + "ago": { + "level0": "Angan", + "level1": "Nuclear Angan", + "level2": "Ankave-Tainae-Akoye", + "level3": "Tainae-Akoye" + }, + "agq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "West Ring", + "level10": "Aghemic", + "level11": "Aghem-Weh" + }, + "agr": { + "level0": "Chicham" + }, + "ags": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid" + }, + "agt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Cagayan Valley", + "level4": "Ibanagic", + "level5": "Gaddangic" + }, + "agu": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Mamean", + "level4": "Ixilan" + }, + "agv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Luzon" + }, + "agw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Makira" + }, + "agx": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Lezgic", + "level3": "Samur", + "level4": "Eastern Samur", + "level5": "Tabasaran-Aghul-Lezgi", + "level6": "Aghul-Lezgi", + "level7": "Aghulic" + }, + "agy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran" + }, + "agz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bikol", + "level5": "Inagta Bikol" + }, + "aha": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Central Tano", + "level7": "Bia", + "level8": "Southern Bia" + }, + "ahb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Eastern Malakula linkage", + "level8": "Central-Southeast Malakula", + "level9": "Southeastern Malakula linkage", + "level10": "Port Sandwich-Axamb-Avok" + }, + "ahe": { + "level0": "Bookkeeping" + }, + "ahg": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "Agaw", + "level3": "Northern-Eastern-Western Agaw" + }, + "ahh": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Awyu", + "level6": "Mappi-Digul Awyu" + }, + "ahi": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Grebo-Aizi", + "level4": "Aizi" + }, + "ahk": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Hanic", + "level7": "Ha-Ya", + "level8": "Akhaic" + }, + "ahl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Ka-Togo", + "level4": "Kposo-Ahlo-Bowili" + }, + "ahm": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Grebo-Aizi", + "level4": "Aizi" + }, + "ahn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Ayere-Ahan" + }, + "aho": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Sukaphic" + }, + "ahp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Unclassified Volta-Congo" + }, + "ahr": { + "level0": "Bookkeeping" + }, + "ahs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Northwestern Benue-Congo Plateau", + "level6": "Koroic", + "level7": "Tinoric" + }, + "aht": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Southern Alaskan Athabaskan" + }, + "aia": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Makira" + }, + "aib": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Turkestan", + "level4": "Modern Turkestan", + "level5": "Uyghuric" + }, + "aic": { + "level0": "Border", + "level1": "Bewani", + "level2": "Pagi-Kilmeri" + }, + "aid": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Northern Pama", + "level3": "Linngithigh-Alngith" + }, + "aie": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage" + }, + "aif": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "West Palai", + "level3": "Agi-Yeri" + }, + "aig": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Gullah-Nevis-Antigua" + }, + "aih": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Kam-Sui", + "level3": "Then-MMS", + "level4": "Maonan-Mak-Sui", + "level5": "Mak-Ai-Cham" + }, + "aii": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "North-Eastern Neo-Aramaic" + }, + "aij": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "North-Eastern Neo-Aramaic", + "level11": "Trans-Zab", + "level12": "Western Trans-Zab" + }, + "aik": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Jilic-Eggonic", + "level5": "Eggon-Ake" + }, + "ail": { + "level0": "Bosavi", + "level1": "Bosavi Watershed" + }, + "aim": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Central Old Kuki" + }, + "ain": { + "level0": "Ainu", + "level1": "Hokkaido-Kuril Ainu" + }, + "aio": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Sukaphic", + "level11": "Mogaung", + "level12": "Assam Tai A" + }, + "aip": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Kwer-Kopkaka-Burumakok", + "level6": "Kwer-Burumakok" + }, + "aiq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Farsic", + "level9": "Eastern Farsic" + }, + "air": { + "level0": "Greater Kwerba", + "level1": "Kwerba-Samarokena", + "level2": "Samarokena-Airoran" + }, + "ait": { + "level0": "Tupian", + "level1": "Arikem-Tupari", + "level2": "Arikemic" + }, + "aiw": { + "level0": "South Omotic", + "level1": "AHK", + "level2": "Aari-Gayil" + }, + "aix": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Pasismanua" + }, + "aiy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Oriental", + "level5": "Gbanu-Manza-Ngbaka", + "level6": "Manza-Ngbaka", + "level7": "Manzaic", + "level8": "Ngbaka-Manza-Ali" + }, + "aja": { + "level0": "Kresh-Aja" + }, + "ajg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe" + }, + "aji": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Southern New Caledonian", + "level8": "Mid-Southern New Caledonian", + "level9": "Houailou" + }, + "ajs": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "aju": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "North African Arabic" + }, + "ajw": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.2", + "level5": "Nuclear West Chadic B.2", + "level6": "Western West Chadic B.2" + }, + "ajz": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Karbic" + }, + "aka": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Central Tano", + "level7": "Akanic" + }, + "akb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran", + "level3": "Batakic", + "level4": "Central-Southern Batak", + "level5": "Southern Batak", + "level6": "Angkola-Mandailing" + }, + "akd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross" + }, + "ake": { + "level0": "Cariban", + "level1": "Venezuelan Cariban", + "level2": "Pemong-Panare", + "level3": "Pemongan", + "level4": "Kapong" + }, + "akf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Idomoid", + "level4": "Yatye-Akpa" + }, + "akg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Sumba-Hawu", + "level5": "Sumba", + "level6": "Central-East Sumbanese", + "level7": "Central Sumbanese" + }, + "akh": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Kewa-Huli", + "level3": "Sau-Angal-Kewa", + "level4": "Angal-Kewa", + "level5": "Angal Mendi" + }, + "aki": { + "level0": "Ramu", + "level1": "Aian" + }, + "akj": { + "level0": "Great Andamanese", + "level1": "North Andamanese-Akakede", + "level2": "Northern Great Andamanese", + "level3": "Jeru-Kora" + }, + "akk": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "East Semitic" + }, + "akl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "West Bisayan" + }, + "akm": { + "level0": "Great Andamanese", + "level1": "North Andamanese-Akakede", + "level2": "Northern Great Andamanese", + "level3": "Bo-Cari" + }, + "akn": { + "level0": "Bookkeeping" + }, + "ako": { + "level0": "Cariban", + "level1": "Guianan", + "level2": "Taranoan", + "level3": "Tiriyoan" + }, + "akp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Na-Togo", + "level4": "Lelemic", + "level5": "Lelemi-Akpafu" + }, + "akq": { + "level0": "Sepik", + "level1": "Yellow River" + }, + "akr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "South Santo", + "level9": "Araki-Tangoa" + }, + "aks": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Gurma-Yom-Naudem", + "level11": "Gurma", + "level12": "Gurma A" + }, + "akt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Arawe", + "level11": "East Arawe" + }, + "aku": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Yukubenic", + "level5": "Akum-Beezen" + }, + "akv": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Andic", + "level4": "Akhvakhic" + }, + "akw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Mboshi (C.20)" + }, + "akx": { + "level0": "Great Andamanese", + "level1": "North Andamanese-Akakede" + }, + "aky": { + "level0": "Great Andamanese", + "level1": "Middle Great Andamanese", + "level2": "Okol-Opucikwar" + }, + "akz": { + "level0": "Muskogean", + "level1": "Alabaman-Koasati" + }, + "ala": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Idomoid", + "level4": "Akweya", + "level5": "Etulo-Idoma", + "level6": "Nuclear Idoma" + }, + "alc": { + "level0": "Kawesqar", + "level1": "North Central Alacufan" + }, + "ald": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Avikam-Alladian" + }, + "ale": { + "level0": "Eskimo-Aleut", + "level1": "Aleutic" + }, + "alf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Bendic", + "level6": "Nuclear Bendic", + "level7": "Bukpic" + }, + "alh": { + "level0": "Mangarrayi-Maran", + "level1": "Maran" + }, + "ali": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles" + }, + "alj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Mangyan" + }, + "alk": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric" + }, + "all": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "alm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "South-Central Santo" + }, + "aln": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Albanian" + }, + "alo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "West Piru Bay", + "level5": "Hoamoal", + "level6": "East Hoamoal" + }, + "alp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Three Rivers", + "level4": "Amalumute", + "level5": "Northwest Seram", + "level6": "Ulat Inai" + }, + "alq": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian", + "level4": "Ojibwa-Potawatomi", + "level5": "Ojibwa", + "level6": "Severn-Algonquin" + }, + "alr": { + "level0": "Chukotko-Kamchatkan", + "level1": "Chukotian" + }, + "als": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Albanian", + "level3": "Albanian-Tosk" + }, + "alt": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Southeast Kipchak", + "level5": "East Kipchak" + }, + "alu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Southern Malaita" + }, + "alw": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Highland East Cushitic", + "level4": "Sidaama-Hadiyya-Kambaata", + "level5": "Hadiyya-Kambaata", + "level6": "Kambaataic" + }, + "alx": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Nuclear Palai", + "level4": "Bragat-Aruop-Amol" + }, + "aly": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Arandic", + "level3": "South Arandic", + "level4": "Upper Arrernte" + }, + "alz": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Southern Lwoo", + "level4": "Adhola-Alur-Luo" + }, + "ama": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup V", + "level6": "Arawetic" + }, + "amb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "North Tivoid" + }, + "amc": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Headwaters Pano" + }, + "amd": { + "level0": "Bookkeeping" + }, + "ame": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha" + }, + "amf": { + "level0": "South Omotic", + "level1": "AHK", + "level2": "Hamer-Karo" + }, + "amg": { + "level0": "Iwaidjan Proper" + }, + "amh": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Amharic-Argobba" + }, + "ami": { + "level0": "Austronesian", + "level1": "East Formosan", + "level2": "Central East Formosan" + }, + "amj": { + "level0": "Furan" + }, + "amk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Central-Western Yapen", + "level7": "Ansus-Ambai" + }, + "aml": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Khasian" + }, + "amm": { + "level0": "Left May", + "level1": "Western Left May" + }, + "amn": { + "level0": "Border", + "level1": "Warisic" + }, + "amo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos" + }, + "amp": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Eastern Sepik Hill" + }, + "amq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua", + "level6": "Seram Straits", + "level7": "Uliase", + "level8": "Hatuhaha", + "level9": "Saparuan", + "level10": "Elpaputi" + }, + "amr": { + "level0": "Harakmbut" + }, + "ams": { + "level0": "Japonic", + "level1": "Ryukyuan", + "level2": "Northern Ryukyuan", + "level3": "Amami", + "level4": "Nuclear Amami", + "level5": "Oshima" + }, + "amt": { + "level0": "Amto-Musan" + }, + "amu": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Amuzgoan" + }, + "amv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "West Central Maluku" + }, + "amw": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Western Aramaic" + }, + "amx": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Arandic", + "level3": "South Arandic", + "level4": "Upper Arrernte" + }, + "amy": { + "level0": "Western Daly", + "level1": "Maranunggu-Ame-Manda", + "level2": "Ame-Manda" + }, + "amz": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Northern Pama", + "level3": "Gudang-Northeast Paman", + "level4": "Northeast Paman", + "level5": "Uradhic" + }, + "anb": { + "level0": "Zaparoan", + "level1": "Iquito-Arabela", + "level2": "Arabela-Andoa" + }, + "anc": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3" + }, + "and": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Central-Western Yapen", + "level7": "Ansus-Ambai" + }, + "ane": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Southern New Caledonian", + "level8": "Mid-Southern New Caledonian" + }, + "anf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Ka-Togo", + "level4": "Kebu-Animere" + }, + "ang": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic" + }, + "anh": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Sogeram", + "level5": "Apalic", + "level6": "Greater West Sogeram", + "level7": "West Sogeram" + }, + "ani": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Andic" + }, + "anj": { + "level0": "Ramu", + "level1": "Aian" + }, + "ank": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3", + "level6": "Goemaic" + }, + "anl": { + "level0": "Sino-Tibetan", + "level1": "Mruic" + }, + "anm": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Anal-Lamgang" + }, + "ann": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross" + }, + "anp": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Magadhan" + }, + "anq": { + "level0": "Jarawa-Onge" + }, + "ans": { + "level0": "Chocoan", + "level1": "Unclassified Chocoan" + }, + "ant": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Wati", + "level3": "Unclassified Wati" + }, + "anu": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Northern Lwoo" + }, + "anv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Mamfe", + "level6": "Kendem-Denya" + }, + "anw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Efikic", + "level8": "Okop Usem" + }, + "anx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "East Manus" + }, + "any": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Central Tano", + "level7": "Bia", + "level8": "Northern Bia", + "level9": "Anyinic" + }, + "aoa": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Lower Guinea Portuguese", + "level15": "Bantu Layer Lower Guinea Portuguese", + "level16": "Saotomic" + }, + "aob": { + "level0": "Anim", + "level1": "Tirio" + }, + "aoc": { + "level0": "Cariban", + "level1": "Venezuelan Cariban", + "level2": "Pemong-Panare", + "level3": "Pemongan" + }, + "aod": { + "level0": "Ramu", + "level1": "Goam", + "level2": "Ataitan" + }, + "aoe": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Kewa-Huli", + "level3": "Sau-Angal-Kewa", + "level4": "Angal-Kewa", + "level5": "Angal Mendi" + }, + "aof": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Nuclear Palai", + "level4": "Bragat-Aruop-Amol" + }, + "aog": { + "level0": "Lower Sepik" + }, + "aoh": { + "level0": "Unattested", + "level1": "Chocoan (Unattested)" + }, + "aoi": { + "level0": "Gunwinyguan", + "level1": "Eastern Gunwinyguan", + "level2": "Wubuy-Anindilyakwa" + }, + "aoj": { + "level0": "Nuclear Torricelli", + "level1": "Kombio-Arapesh-Urat", + "level2": "Arapesh", + "level3": "Mufian-Bukiyip-Abu" + }, + "aok": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Southern New Caledonian", + "level8": "Mid-Southern New Caledonian", + "level9": "Houailou" + }, + "aol": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata", + "level4": "Lamaholot Barat" + }, + "aom": { + "level0": "Koiarian", + "level1": "Baraic" + }, + "aon": { + "level0": "Nuclear Torricelli", + "level1": "Kombio-Arapesh-Urat", + "level2": "Arapesh" + }, + "aor": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "South Santo" + }, + "aos": { + "level0": "Border", + "level1": "Taikat-Awyi" + }, + "aot": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Kochic" + }, + "aou": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Southwestern Kra", + "level3": "Western Kra", + "level4": "Gauic", + "level5": "Gelaoic", + "level6": "Northern Gelao", + "level7": "Ahouic" + }, + "aox": { + "level0": "Arawakan", + "level1": "Negro-Roraima", + "level2": "Pidjanan", + "level3": "Wapishanan", + "level4": "Wapishana-Atorai" + }, + "aoz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Rote-Meto", + "level4": "West Rote", + "level5": "Dengka-Meto", + "level6": "Meto", + "level7": "Central Meto" + }, + "apb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira" + }, + "apc": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Levantine-Cypriot Arabic" + }, + "apd": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Egyptic Arabic", + "level7": "Egypto-Sudanic Arabic", + "level8": "Sudanese-Chadian Arabic", + "level9": "East Sudanic Arabic" + }, + "ape": { + "level0": "Nuclear Torricelli", + "level1": "Kombio-Arapesh-Urat", + "level2": "Arapesh", + "level3": "Mufian-Bukiyip-Abu", + "level4": "Bukiyip-Abu" + }, + "apf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Northeastern Luzon", + "level4": "Nuclear Northeastern Luzon", + "level5": "Paranan-Pahanan" + }, + "apg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Barito-Mahakam" + }, + "aph": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Tamar", + "level6": "Yakkha-Athpariyic", + "level7": "Athpariyic" + }, + "api": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VI", + "level6": "Kawahiva" + }, + "apj": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Apachean", + "level4": "Southwestern Apachean", + "level5": "Eastern Southwestern Apachean" + }, + "apk": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Apachean" + }, + "apl": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Apachean", + "level4": "Southwestern Apachean", + "level5": "Eastern Southwestern Apachean" + }, + "apm": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Apachean", + "level4": "Southwestern Apachean", + "level5": "Western Southwestern Apachean" + }, + "apn": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Cerrado", + "level3": "Goyaz", + "level4": "Northern Je" + }, + "apo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Arawe", + "level11": "West Arawe" + }, + "app": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "South Pentecost" + }, + "apq": { + "level0": "Great Andamanese", + "level1": "Middle Great Andamanese", + "level2": "Okol-Opucikwar" + }, + "apr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Korap linkage" + }, + "aps": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Siau", + "level8": "Sissano-Tumleo", + "level9": "Sera-Sissano", + "level10": "Sissanoic" + }, + "apt": { + "level0": "Sino-Tibetan", + "level1": "Macro-Tani", + "level2": "Tani", + "level3": "Pre-Western Tani", + "level4": "Western Tani" + }, + "apu": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Purus-Chamicuro", + "level3": "Purus" + }, + "apv": { + "level0": "Unattested", + "level1": "Nambiquaran (Unattested)" + }, + "apw": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Apachean", + "level4": "Southwestern Apachean", + "level5": "Western Southwestern Apachean" + }, + "apx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Wetar-Atauro", + "level4": "Wetar", + "level5": "Perai-Tugun-Aputai", + "level6": "Perai-Aputai" + }, + "apy": { + "level0": "Cariban" + }, + "apz": { + "level0": "Angan", + "level1": "Nuclear Angan", + "level2": "Wojokesic" + }, + "aqc": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Lezgic" + }, + "aqd": { + "level0": "Dogon", + "level1": "West Dogon", + "level2": "Penangic" + }, + "aqg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid" + }, + "aqk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Ninzic" + }, + "aqm": { + "level0": "Kayagaric" + }, + "aqn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran" + }, + "aqr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Southern New Caledonian", + "level8": "Mid-Southern New Caledonian", + "level9": "Houailou" + }, + "aqt": { + "level0": "Lengua-Mascoy", + "level1": "Eastern Enlhet-Enenlhet" + }, + "aqz": { + "level0": "Tupian", + "level1": "Arikem-Tupari", + "level2": "Tuparic", + "level3": "Nuclear Tuparic", + "level4": "Corumbiara" + }, + "arb": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic" + }, + "arc": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic" + }, + "ard": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Palku", + "level3": "Arabana-Wangganguru" + }, + "are": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Arandic", + "level3": "South Arandic", + "level4": "Upper Arrernte" + }, + "arg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Unshifted Western Romance" + }, + "arh": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Magdalenic", + "level3": "Northern Magdalenic", + "level4": "Arhuacic", + "level5": "Eastern-Southern Arhuacic" + }, + "ari": { + "level0": "Caddoan", + "level1": "Northern Caddoan", + "level2": "Pawnee-Kitsai", + "level3": "Pawnee-Arikara" + }, + "arj": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Eastern Eastern Tucanoan", + "level3": "Eastern Eastern Tucanoan II", + "level4": "Kotiria-Piratapuyo", + "level5": "Piratapuyic", + "level6": "Arapaso-Miriti" + }, + "ark": { + "level0": "Nuclear-Macro-Je", + "level1": "Jabuti" + }, + "arl": { + "level0": "Zaparoan", + "level1": "Iquito-Arabela", + "level2": "Arabela-Andoa" + }, + "arn": { + "level0": "Araucanian" + }, + "aro": { + "level0": "Pano-Tacanan", + "level1": "Tacanan", + "level2": "Takanik-Chamik", + "level3": "Takanik", + "level4": "Araona-Toromono" + }, + "arp": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Arapahoic", + "level4": "Arapaho-Gros Ventre-Besawunena" + }, + "arq": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "North African Arabic" + }, + "arr": { + "level0": "Tupian", + "level1": "Purubora-Ramarama", + "level2": "Ramarama" + }, + "ars": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Arabian Peninsula Arabic", + "level7": "North Arabian Beduin Arabic" + }, + "aru": { + "level0": "Arawan" + }, + "arv": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Western Omo-Tana" + }, + "arw": { + "level0": "Arawakan", + "level1": "Caribbean Arawakan", + "level2": "Antillean Arawakan", + "level3": "Ineric" + }, + "arx": { + "level0": "Tupian", + "level1": "Monde", + "level2": "Gavianic", + "level3": "Nuclear Gavianic" + }, + "ary": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "North African Arabic", + "level7": "Moroccan-Andalusian Arabic" + }, + "arz": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Egyptic Arabic", + "level7": "Egypto-Sudanic Arabic" + }, + "asa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Pare-Taveta", + "level10": "Pareic" + }, + "asb": { + "level0": "Siouan", + "level1": "Mississippi Valley", + "level2": "Dakotan", + "level3": "Nakoda" + }, + "asc": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Asmat-Kamoro", + "level3": "Asmat" + }, + "ase": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "American Sign" + }, + "asf": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "BSLic", + "level3": "BANZL", + "level4": "Auslanic" + }, + "asg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Kambari-Cicipu", + "level6": "Kambaric", + "level7": "West Kambaric" + }, + "asi": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Asmat-Kamoro", + "level3": "Sabakor" + }, + "asj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Beboid", + "level6": "Eastern Beboid", + "level7": "Nsari-Nooni-Ncane" + }, + "ask": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Nuristani" + }, + "asl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "West Piru Bay" + }, + "asm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Kamrupa", + "level10": "Eastern Kamrupa" + }, + "asn": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup V" + }, + "aso": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Gahuku" + }, + "asp": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic" + }, + "asq": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Central European Sign", + "level4": "Nuclear Central European Sign" + }, + "asr": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Mundaric", + "level5": "Asuric" + }, + "ass": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "Central Tivoid", + "level7": "Central Tivoid B" + }, + "ast": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Asturo-Leonese" + }, + "asu": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup IV", + "level6": "Tupi-Guarani Subgroup IV.A" + }, + "asv": { + "level0": "Central Sudanic", + "level1": "Mangbetu-Asua", + "level2": "Mangbetuic" + }, + "asw": { + "level0": "Sign Language", + "level1": "Auxiliary Sign Systems" + }, + "asx": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Warup", + "level4": "Nuclear Warup" + }, + "asy": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Asmat-Kamoro", + "level3": "Asmat", + "level4": "Central-Yaosakor Asmat" + }, + "asz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera" + }, + "atb": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Northern Burmish", + "level5": "Maruic" + }, + "atc": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Madre de Dios Pano" + }, + "atd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "Central and Southern Manobo", + "level5": "East-West-Central Manobo", + "level6": "East and Central Manobo", + "level7": "Central Manobo" + }, + "ate": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Sogeram", + "level5": "Apalic", + "level6": "Greater West Sogeram", + "level7": "West Sogeram" + }, + "atf": { + "level0": "Bookkeeping" + }, + "atg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Igwic", + "level7": "Ikpeshic" + }, + "ati": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo" + }, + "atj": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Cree-Montagnais-Naskapi" + }, + "atk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian" + }, + "atl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bikol", + "level5": "Inagta Bikol" + }, + "atm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Unclassified Bisayan" + }, + "atn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Southern Tatic", + "level10": "Vafsic" + }, + "ato": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Southwest Grassfields", + "level8": "Menka-Atong" + }, + "atp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Cagayan Valley", + "level4": "Ibanagic", + "level5": "Atta" + }, + "atq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Pitu Ulunna Salu", + "level6": "Matangnga-Aralle-Tabulahan" + }, + "atr": { + "level0": "Cariban", + "level1": "Yawaperi" + }, + "ats": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Arapahoic", + "level4": "Arapaho-Gros Ventre-Besawunena" + }, + "att": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Cagayan Valley", + "level4": "Ibanagic", + "level5": "Atta" + }, + "atu": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Dinka-Nuer", + "level3": "Nuer-Reel" + }, + "atv": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "South Siberian Turkic", + "level3": "Northern Altai-Lower Chulym" + }, + "atw": { + "level0": "Palaihnihan" + }, + "aty": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "Southern Vanuatu" + }, + "atz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon" + }, + "aua": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Temotu", + "level5": "Utupua-Vanikoro", + "level6": "Utupua" + }, + "aub": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Riverine Phula", + "level7": "Downriver Riverine Phula", + "level8": "Phupha-Alugu" + }, + "aud": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian" + }, + "aug": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Western Gbe", + "level5": "Eweic" + }, + "auh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Sabi", + "level8": "Malungu-Central Sabi", + "level9": "Central Sabi", + "level10": "Bemba (M.40)" + }, + "aui": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage" + }, + "auj": { + "level0": "Afro-Asiatic", + "level1": "Berber" + }, + "auk": { + "level0": "Nuclear Torricelli", + "level1": "Nuclear Maimai", + "level2": "Heyo-Yahang" + }, + "aul": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Eastern Malakula linkage", + "level8": "Central-Southeast Malakula" + }, + "aum": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Ebira-Nupoid", + "level4": "Macro-Nupoid", + "level5": "Nupoid" + }, + "aun": { + "level0": "Nuclear Torricelli", + "level1": "West Wapei", + "level2": "One", + "level3": "Central-Northern One" + }, + "auo": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.1", + "level5": "Ngizim-Southwestern Bade", + "level6": "Shira-Southwestern Bade", + "level7": "Shira" + }, + "aup": { + "level0": "Anim", + "level1": "Tirio", + "level2": "Nuclear Tirio" + }, + "auq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Sarmi", + "level8": "Anus-Podena" + }, + "aur": { + "level0": "Nuclear Torricelli", + "level1": "Kombio-Arapesh-Urat", + "level2": "Kombio-Yambes", + "level3": "Unclassified Kombio-Yambes" + }, + "aut": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Proximal", + "level13": "Southern East Polynesian Proximal", + "level14": "Tahitian-Austral" + }, + "auu": { + "level0": "Nuclear Trans New Guinea", + "level1": "Paniai Lakes", + "level2": "Auye-Dao" + }, + "auv": { + "level0": "Bookkeeping" + }, + "auw": { + "level0": "Border", + "level1": "Taikat-Awyi" + }, + "aux": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VIII", + "level6": "Guaja-Kaapor-Ava", + "level7": "Guaja-Aure-Aura" + }, + "auy": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Kainantu", + "level3": "Gauwa", + "level4": "Auyana" + }, + "auz": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Eastern Arabic", + "level7": "Central Asian Arabic", + "level8": "Xorasan-Qashqa-Darya Arabic" + }, + "ava": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic" + }, + "avb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Arawe", + "level11": "East Arawe" + }, + "avd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Southern Tatic" + }, + "ave": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian" + }, + "avi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Avikam-Alladian" + }, + "avk": { + "level0": "Artificial Language" + }, + "avl": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Egyptic Arabic" + }, + "avm": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Northern Pama", + "level3": "Gudang-Northeast Paman", + "level4": "Northeast Paman", + "level5": "Uradhic", + "level6": "Yadhaykenu-Angkamuthi" + }, + "avn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Ka-Togo", + "level4": "Avatime-Nyangbo" + }, + "avo": { + "level0": "Unattested", + "level1": "Arawakan (Unattested)" + }, + "avs": { + "level0": "Zaparoan", + "level1": "Zaparo-Abishira" + }, + "avt": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Wapeic", + "level4": "Au-Olo-Elkei" + }, + "avu": { + "level0": "Central Sudanic", + "level1": "Moru-Madi", + "level2": "Central Moru-Madi" + }, + "avv": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VIII", + "level6": "Guaja-Kaapor-Ava" + }, + "awa": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Eastern Hindi", + "level9": "Awadhic" + }, + "awb": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Kainantu", + "level3": "Gauwa", + "level4": "Awa-Oweina" + }, + "awc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Kambari-Cicipu" + }, + "awe": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani" + }, + "awg": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Northern Pama", + "level3": "Albatross Bay", + "level4": "Anguthimri-Yangathimri-Yuputhimri", + "level5": "Anguthimri-Yangathimri" + }, + "awh": { + "level0": "Bayono-Awbono" + }, + "awi": { + "level0": "Kamula-Elevala", + "level1": "Elevala" + }, + "awk": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "New South Wales Pama-Nyungan", + "level3": "Yuin-Kuri", + "level4": "Kuri", + "level5": "Hunter-Hastings" + }, + "awm": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Kabenau" + }, + "awn": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "Agaw" + }, + "awo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Tula-Longuda", + "level6": "Tula-Waja", + "level7": "Tulaic", + "level8": "Tula-Ma-Yebu", + "level9": "Awak-Kamo" + }, + "awr": { + "level0": "Lakes Plain", + "level1": "Far West Lakes Plain" + }, + "aws": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Awyu" + }, + "awt": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup V", + "level6": "Arawetic" + }, + "awu": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Awyu", + "level6": "Mappi-Digul Awyu" + }, + "awv": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Awyu" + }, + "aww": { + "level0": "Sepik", + "level1": "Yellow River" + }, + "awx": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Wantoatic", + "level4": "Wantoat-Awara" + }, + "awy": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Awyu" + }, + "axb": { + "level0": "Guaicuruan", + "level1": "Guaicuru del Sur" + }, + "axk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Likouala-Sangha", + "level10": "Bwamba-Ngondi-Pande-Mbati-Aka" + }, + "axl": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Arandic", + "level3": "South Arandic" + }, + "axx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Southern New Caledonian", + "level8": "Mid-Southern New Caledonian" + }, + "aya": { + "level0": "Ramu", + "level1": "Lower Ramu", + "level2": "Ottilien", + "level3": "Bosngun-Awar" + }, + "ayb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Western Phla-Phera" + }, + "ayc": { + "level0": "Aymaran", + "level1": "Central-Southern Aymara" + }, + "ayd": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Northeastern Pama", + "level4": "Umbindhamuic" + }, + "aye": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Ayere-Ahan" + }, + "ayg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Oti North Guang", + "level9": "Mountain Oti North Guang", + "level10": "Gikyode-Ginyanga" + }, + "ayh": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Arabian Peninsula Arabic" + }, + "ayi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "East-West Central Delta Cross", + "level7": "Mbembe-Legbo", + "level8": "Legboic", + "level9": "Lenyima-Leyigha" + }, + "ayk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Southern Northwestern Edoid", + "level7": "Okpe-Akuku-Idesa" + }, + "ayl": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "North African Arabic" + }, + "ayn": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Arabian Peninsula Arabic", + "level7": "Judeo-Muslim Sanaani Arabic" + }, + "ayo": { + "level0": "Zamucoan", + "level1": "Zamuco-Ayoreo" + }, + "ayp": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Eastern Arabic", + "level7": "Qeltu" + }, + "ayq": { + "level0": "Sepik", + "level1": "Sepik Tama", + "level2": "Mayo-Pasi", + "level3": "Yimin-Bel" + }, + "ayr": { + "level0": "Aymaran", + "level1": "Central-Southern Aymara" + }, + "ays": { + "level0": "Unattested", + "level1": "Austronesian (Unattested)" + }, + "ayt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Luzon", + "level3": "Sambalic" + }, + "ayu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Ninzic" + }, + "ayx": { + "level0": "Bookkeeping" + }, + "ayy": { + "level0": "Unattested", + "level1": "Austronesian (Unattested)" + }, + "ayz": { + "level0": "Maybratic" + }, + "aza": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Axioid" + }, + "azb": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Oghuz", + "level3": "Nuclear Oghuz", + "level4": "Central Oghuz" + }, + "azd": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Western Periphery-North Guerrero Nahuatl", + "level6": "Western Periphery Nahuatl", + "level7": "Durango Nahuatl" + }, + "azg": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Amuzgoan" + }, + "azj": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Oghuz", + "level3": "Nuclear Oghuz", + "level4": "Central Oghuz" + }, + "azm": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Amuzgoan" + }, + "azn": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Western Periphery-North Guerrero Nahuatl", + "level6": "Western Periphery Nahuatl", + "level7": "Durango Nahuatl" + }, + "azo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Ngembaic" + }, + "azr": { + "level0": "Bookkeeping" + }, + "azt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Cagayan Valley", + "level4": "Ibanagic", + "level5": "Atta" + }, + "azz": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl", + "level5": "Sierra de Puebla Nahuatl" + }, + "baa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Choiseul", + "level10": "East Choiseul", + "level11": "Southeast Choiseul" + }, + "bab": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Wolof-BKK", + "level3": "Nyun", + "level4": "Bainounk" + }, + "bae": { + "level0": "Arawakan", + "level1": "Medio Rio Negro", + "level2": "Bareic" + }, + "baf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam" + }, + "bag": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Sanaga-West Mbam (A.40)", + "level10": "Sanaga (A.60)" + }, + "bah": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Gullah-Nevis-Antigua", + "level15": "Gullah", + "level16": "Bahamian Gullah" + }, + "baj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Karey-Barakai" + }, + "bak": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Northwest Kipchak", + "level5": "North Kipchak", + "level6": "Bashkiric" + }, + "bam": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding" + }, + "ban": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bali-Sasak-Sumbawa" + }, + "bao": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Eastern Eastern Tucanoan", + "level3": "Eastern Eastern Tucanoan I", + "level4": "Bara-Tatuyo" + }, + "bap": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Southern Kiranti", + "level6": "Bantawic" + }, + "bar": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Bairisch" + }, + "bas": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Basaa-Yaunde (A40-70)", + "level8": "Basaa (A.40)", + "level9": "Basaa-Bakoko", + "level10": "Basaa-Hijuk" + }, + "bau": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan", + "level7": "Jarawaic" + }, + "bav": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "South Ring", + "level9": "Babungoic" + }, + "baw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Ngembaic", + "level10": "Mankonic" + }, + "bax": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Mbam-Nkam Nun" + }, + "bay": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Batuley-Mariri" + }, + "baz": { + "level0": "Bookkeeping" + }, + "bba": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur" + }, + "bbb": { + "level0": "Koiarian", + "level1": "Baraic", + "level2": "Barai-Namiae" + }, + "bbc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran", + "level3": "Batakic", + "level4": "Central-Southern Batak", + "level5": "Southern Batak" + }, + "bbd": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Gum", + "level5": "Panim-Isebe-Bau" + }, + "bbe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Eastern Mundu-Baka", + "level7": "Mayogo-Bangba" + }, + "bbf": { + "level0": "Baibai-Fas" + }, + "bbg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "West Kikongo", + "level20": "Vilic", + "level21": "Lumbuic", + "level22": "Ngubi-Sangu-Sira-Punu", + "level23": "Sangu-Sira-Punu", + "level24": "Sangu-Sira", + "level25": "Sira-Barama" + }, + "bbh": { + "level0": "Austroasiatic", + "level1": "Mangic", + "level2": "Pakanic" + }, + "bbi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Basaa-Yaunde (A40-70)", + "level8": "Basaa (A.40)", + "level9": "Abo-Barombi" + }, + "bbj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Bamileke", + "level10": "East Bamileke" + }, + "bbk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "Center Ring" + }, + "bbl": { + "level0": "Nakh-Daghestanian", + "level1": "Nakh" + }, + "bbm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Mongala", + "level11": "Motemboic", + "level12": "Bujaic", + "level13": "Budja (C.37)" + }, + "bbn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "Bali-Vitu" + }, + "bbo": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Duun-Bobo", + "level4": "Bobo" + }, + "bbp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic" + }, + "bbq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Mbam-Nkam Nun", + "level10": "Nun MCNB" + }, + "bbr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Kokon" + }, + "bbs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Agoi-Doko-Iyoniyong" + }, + "bbt": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.2", + "level5": "Nuclear West Chadic B.2" + }, + "bbu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan" + }, + "bbv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Korap linkage" + }, + "bbw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Mbam-Nkam Nun" + }, + "bbx": { + "level0": "Bookkeeping" + }, + "bby": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields" + }, + "bbz": { + "level0": "Bookkeeping" + }, + "bca": { + "level0": "Sino-Tibetan", + "level1": "Macro-Bai", + "level2": "Baic", + "level3": "South-Central Bai" + }, + "bcb": { + "level0": "Bookkeeping" + }, + "bcc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Balochic", + "level8": "Southern-Western Balochi", + "level9": "Southern Balochi-Koroshi" + }, + "bcd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar", + "level5": "North Babaric" + }, + "bce": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Mbam-Nkam Nun", + "level10": "Nun MCNB" + }, + "bcf": { + "level0": "Kiwaian" + }, + "bcg": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Naluic" + }, + "bch": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Ngero", + "level8": "Eastern Ngero" + }, + "bci": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Central Tano", + "level7": "Bia", + "level8": "Northern Bia" + }, + "bcj": { + "level0": "Nyulnyulan", + "level1": "Western Nyulnyulan", + "level2": "Bardic" + }, + "bck": { + "level0": "Bunaban" + }, + "bcl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bikol", + "level5": "Coastal Bikol" + }, + "bcm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Banoni-Piva" + }, + "bcn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Mumuye-Yandang", + "level6": "Yandangic", + "level7": "Bali-Kpasam" + }, + "bco": { + "level0": "Bosavi", + "level1": "Bosavi Watershed", + "level2": "Kaluli-Sunia" + }, + "bcp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Bali-Beeke" + }, + "bcq": { + "level0": "Ta-Ne-Omotic" + }, + "bcr": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central British Columbia Athabaskan", + "level4": "Carrieric" + }, + "bcs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "North-South Central Delta Cross", + "level7": "Ubaghara-Kohumono", + "level8": "Kohumonoic" + }, + "bct": { + "level0": "Central Sudanic", + "level1": "Membi-Mangbutu-Efe", + "level2": "Mangbutu-Efe", + "level3": "Leseic" + }, + "bcu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Bel-Roinji-Nenaya", + "level9": "Bel", + "level10": "Eastern Bel" + }, + "bcv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Jukun-Mbembe-Wurbo", + "level6": "Wurbo-Wannu", + "level7": "Wurbo" + }, + "bcw": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Higic", + "level5": "Nkafa-Kirya-Bana" + }, + "bcx": { + "level0": "Bookkeeping" + }, + "bcy": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic", + "level5": "Bata-Bwatiye" + }, + "bcz": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Wolof-BKK", + "level3": "Nyun", + "level4": "Bainounk" + }, + "bda": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Jola", + "level6": "Bayot" + }, + "bdb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito" + }, + "bdc": { + "level0": "Chocoan", + "level1": "Embera", + "level2": "Atrato", + "level3": "Panama-Baudo-Atrato" + }, + "bdd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Dobu-Duau linkage", + "level9": "Bunama-Mwatebu" + }, + "bde": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.1", + "level5": "Ngizim-Southwestern Bade", + "level6": "Shira-Southwestern Bade" + }, + "bdf": { + "level0": "Koiarian", + "level1": "Koiaric", + "level2": "Biage-Mountain Koiali" + }, + "bdg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Northeast Sabahan" + }, + "bdh": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "Baka-Beli" + }, + "bdi": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Burun" + }, + "bdj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Sereic", + "level6": "Sere-Indri", + "level7": "Sere-Bviri", + "level8": "Bai-Viri" + }, + "bdk": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Lezgic", + "level3": "Samur", + "level4": "Southern Samur" + }, + "bdl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Sama-Bajaw", + "level5": "Sulu-Borneo", + "level6": "Borneo Coast Bajaw" + }, + "bdm": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Kotoko-Buduma" + }, + "bdn": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Maroua" + }, + "bdo": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Bagirmic", + "level6": "Morom-Jaya-Naba", + "level7": "Bayo-Morom" + }, + "bdp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Unclassified Northeast Savanna Bantu", + "level9": "Bende-Tongwe" + }, + "bdq": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "Tampuon-Bahnar" + }, + "bdr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Sama-Bajaw", + "level5": "Sulu-Borneo", + "level6": "Borneo Coast Bajaw" + }, + "bds": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "South Cushitic", + "level3": "Greater West Rift South Cushitic", + "level4": "West Rift South Cushitic" + }, + "bdt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Meridional-Occidental", + "level5": "Bokoto-Gbeya" + }, + "bdu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Lundu-Balong (A.10)", + "level8": "Greater Manenguba", + "level9": "Bafaw-Balong-Manenguba", + "level10": "Bafawic-Bakweric", + "level11": "Bafawic" + }, + "bdv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Macro-Oriya" + }, + "bdw": { + "level0": "West Bomberai", + "level1": "Nuclear West Bomberai" + }, + "bdx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "Rampi-Seko-Badaic", + "level4": "Seko", + "level5": "Panasuanic" + }, + "bdy": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "New South Wales Pama-Nyungan", + "level3": "Greater Bandjalangic", + "level4": "Bandjalangic", + "level5": "Inland Bandjalang" + }, + "bdz": { + "level0": "Unattested", + "level1": "Indo-European (Unattested)" + }, + "bea": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Northwestern Canada Athabaskan", + "level4": "Cordillera Athabaskan", + "level5": "Beaver-Sekani" + }, + "beb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Basaa-Yaunde (A40-70)", + "level8": "Yaunde-Fang (A.70)", + "level9": "Ewondo-Bebele" + }, + "bec": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "Central Tivoid", + "level7": "Central Tivoid A" + }, + "bed": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tanimbar-Bomberai", + "level3": "Nuta" + }, + "bee": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Eastern West Himalayish", + "level4": "Pithauragarh", + "level5": "Darma-Byangsi-Chaudangsi", + "level6": "Darma-Byangsi" + }, + "bef": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Siane-Yagaria" + }, + "beg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Berawan-Lower Baram", + "level5": "Lower Baram", + "level6": "Central Lower Baram A" + }, + "beh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Oti-Volta Oriental" + }, + "bei": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Benyadu-Bekati", + "level4": "Bakati'" + }, + "bej": { + "level0": "Afro-Asiatic", + "level1": "Cushitic" + }, + "bek": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Bebeli-Mangseng" + }, + "bel": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "East Slavic" + }, + "bem": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Sabi", + "level8": "Malungu-Central Sabi", + "level9": "Central Sabi", + "level10": "Bemba (M.40)" + }, + "ben": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga" + }, + "beo": { + "level0": "Bosavi", + "level1": "Etoro-Bedamini" + }, + "bep": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "Rampi-Seko-Badaic", + "level4": "Badaic-Limola", + "level5": "Badaic" + }, + "beq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic" + }, + "bes": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Kim-Besme-Goundo" + }, + "bet": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Beteic", + "level3": "Western Bete" + }, + "beu": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "West Alor-Straits-Pantar", + "level4": "Kaera-Straits", + "level5": "Blagaric" + }, + "bev": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Beteic", + "level3": "Western Bete" + }, + "bew": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Betawic" + }, + "bex": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "Baka-Beli", + "level3": "Morokodo-Beli", + "level4": "Lori" + }, + "bey": { + "level0": "Nuclear Torricelli" + }, + "bez": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Southern Tanzania Highlands Bantu", + "level9": "Bena-Hehe" + }, + "bfa": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Barian", + "level3": "Nuclear Barian" + }, + "bfb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil", + "level8": "Pauri-Nahali" + }, + "bfc": { + "level0": "Sino-Tibetan", + "level1": "Macro-Bai", + "level2": "Baic" + }, + "bfd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Ngembaic", + "level10": "Mankonic" + }, + "bfe": { + "level0": "Tor-Orya", + "level1": "Tor", + "level2": "Coastal Tor", + "level3": "Betaf-Vitou" + }, + "bff": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Oriental", + "level5": "Mbodomo-Bofi" + }, + "bfg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Kayan-Murik-Modang", + "level4": "Kayan-Murik", + "level5": "Kayanic", + "level6": "Rejang-Makaham Kayan" + }, + "bfh": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Tonda" + }, + "bfi": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "BSLic", + "level3": "BANZL" + }, + "bfj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Mbam-Nkam Nun" + }, + "bfk": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "bfl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Central Bandaic", + "level8": "Central Core Bandaic" + }, + "bfm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "Center Ring", + "level10": "Mmen-Bum" + }, + "bfn": { + "level0": "Timor-Alor-Pantar" + }, + "bfo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Northwest Oti-Volta", + "level13": "Safaliba-Dagaare", + "level14": "Dagaaric", + "level15": "North-West Dagaric", + "level16": "Birifor" + }, + "bfp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Ngembaic" + }, + "bfq": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Badaga-Kannada" + }, + "bfr": { + "level0": "Unclassifiable" + }, + "bfs": { + "level0": "Sino-Tibetan", + "level1": "Macro-Bai", + "level2": "Baic", + "level3": "South-Central Bai" + }, + "bft": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Western Archaic Tibetan", + "level5": "Shamskatic" + }, + "bfu": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Eastern West Himalayish", + "level4": "Central-Eastern West Himalayish" + }, + "bfw": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "Gutob-Remo" + }, + "bfx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Central Bisayan", + "level6": "Peripheral Central Bisayan" + }, + "bfy": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Eastern Hindi", + "level9": "Awadhic" + }, + "bfz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Nuclear Himachali" + }, + "bga": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Northwestern Kainji", + "level6": "Dukaic", + "level7": "Main-Gwamhi" + }, + "bgb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Saluan-Banggai", + "level6": "Western Saluan-Banggai", + "level7": "Saluanic" + }, + "bgc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Western Hindi" + }, + "bgd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil", + "level8": "Pauri-Nahali" + }, + "bge": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "bgf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Meridional-Occidental", + "level5": "Gbaya Meridional" + }, + "bgg": { + "level0": "Sino-Tibetan", + "level1": "Kho-Bwa" + }, + "bgh": { + "level0": "Bookkeeping" + }, + "bgi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bilic" + }, + "bgj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Mbam-Nkam Nun" + }, + "bgk": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Khao-Bit" + }, + "bgn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Balochic", + "level8": "Southern-Western Balochi" + }, + "bgo": { + "level0": "Atlantic-Congo", + "level1": "Mel", + "level2": "Northern Mel" + }, + "bgp": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Balochic" + }, + "bgq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Bagri-Jandavra" + }, + "bgr": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin", + "level5": "Lai Chin", + "level6": "Falamic" + }, + "bgs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "Central and Southern Manobo", + "level5": "South Manobo" + }, + "bgt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Guadalcanal-Nggelic", + "level6": "Nuclear Guadalcanal-Nggelic", + "level7": "Nggelic" + }, + "bgu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Mambila-Mbongno", + "level10": "Mbongno-Mvano" + }, + "bgv": { + "level0": "Anim", + "level1": "Marind-Boazi-Yaqai", + "level2": "Yaqayic" + }, + "bgw": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Halbic" + }, + "bgx": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Oghuz", + "level3": "Nuclear Oghuz", + "level4": "West Oghuz" + }, + "bgy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "East Seram", + "level4": "Setic" + }, + "bgz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Saluan-Banggai", + "level6": "Eastern Saluan-Banggai" + }, + "bha": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Western Hindi", + "level10": "Bundeli-Bharia" + }, + "bhb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "bhc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera" + }, + "bhd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Kangric-Chamealic-Bhattiyali", + "level8": "Chamealic", + "level9": "Bhadrawahi-Bhalesi-Curahi", + "level10": "Bhadarwahic" + }, + "bhe": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Western Rajasthani" + }, + "bhg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean", + "level3": "Nuclear Binanderean" + }, + "bhh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Farsic", + "level9": "Eastern Farsic", + "level10": "Tajikic" + }, + "bhi": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil", + "level8": "Pauri-Nahali" + }, + "bhj": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Northwestern Kiranti", + "level5": "Bahing-Sunwar" + }, + "bhk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bikol", + "level5": "Inland Bikol" + }, + "bhl": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Mountain Ok", + "level6": "Division A Mountain Ok" + }, + "bhm": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Modern South Arabian", + "level4": "Hobyot-Western MSA", + "level5": "Western MSA" + }, + "bhn": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "Bohtan" + }, + "bho": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Magadhan", + "level10": "Bhojpuric" + }, + "bhp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata" + }, + "bhq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Tukangbesi-Bonerate", + "level8": "Tukang Besi" + }, + "bhr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "Southwestern Malagasic", + "level7": "South West-Central Malagasic", + "level8": "Nuclear South West-Central Malagasic", + "level9": "Inland-Western Malagasic", + "level10": "Bara-Tanosy" + }, + "bhs": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Dabaic", + "level5": "Buwal-Gavar" + }, + "bht": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Kangric-Chamealic-Bhattiyali", + "level8": "Chamealic" + }, + "bhu": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Halbic" + }, + "bhv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Kayan-Murik-Modang", + "level4": "Kayan-Murik", + "level5": "Kayanic" + }, + "bhw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Biakic", + "level6": "Biak-Roon" + }, + "bhx": { + "level0": "Bookkeeping" + }, + "bhy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Komoic", + "level15": "Bilaic" + }, + "bhz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "Rampi-Seko-Badaic", + "level4": "Badaic-Limola", + "level5": "Badaic" + }, + "bia": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Kartu-Nhanda", + "level3": "Kartu" + }, + "bib": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Bisa-Busa" + }, + "bic": { + "level0": "Bookkeeping" + }, + "bid": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Dangla-Mabire-Birgit", + "level6": "Dangla" + }, + "bie": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kumil-Tibor", + "level6": "Kumil" + }, + "bif": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Jaad" + }, + "big": { + "level0": "Kunimaipan" + }, + "bii": { + "level0": "Bookkeeping" + }, + "bil": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan", + "level7": "Numan" + }, + "bim": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Gurma-Yom-Naudem", + "level11": "Gurma", + "level12": "Gurma B", + "level13": "Gourmantche-Moba", + "level14": "Moba-Bimoba" + }, + "bin": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "North-Central Edoid", + "level6": "Central Plains Edoid" + }, + "bio": { + "level0": "Kwomtari-Nai" + }, + "bip": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Komoic", + "level15": "Bilaic", + "level16": "Bila-Kaiku" + }, + "biq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "West Manus", + "level8": "West Manus I" + }, + "bir": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Engan", + "level3": "Outer Enga" + }, + "bis": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Pacific Creole English", + "level12": "Early Melanesian Pidgin" + }, + "bit": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Central Sepik Hill", + "level3": "Bahinemic" + }, + "biu": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin", + "level5": "Mizoic", + "level6": "Hmaric" + }, + "biv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Northwest Oti-Volta", + "level13": "Safaliba-Dagaare", + "level14": "Dagaaric", + "level15": "North-West Dagaric", + "level16": "Birifor" + }, + "biw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Western A80", + "level10": "Makaaic", + "level11": "Southern Makaaic" + }, + "bix": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Mundaric", + "level5": "Asuric" + }, + "biy": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Mundaric" + }, + "biz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Bobangic", + "level13": "Bobangic Riverain" + }, + "bja": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Mongala", + "level11": "Motemboic", + "level12": "Bujaic", + "level13": "Budja (C.37)" + }, + "bjb": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Thura-Yura", + "level3": "Core Thura Yura", + "level4": "Northern Thura-Yura" + }, + "bjc": { + "level0": "Yareban", + "level1": "Yareba-Bariji-Nawaru" + }, + "bjd": { + "level0": "Bookkeeping" + }, + "bje": { + "level0": "Hmong-Mien", + "level1": "Mienic" + }, + "bjf": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "North-Eastern Neo-Aramaic", + "level11": "Northwestern Jewish Neo-Aramaic" + }, + "bjg": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bijogo" + }, + "bjh": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Central Sepik Hill", + "level3": "Bahinemic" + }, + "bji": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Highland East Cushitic" + }, + "bjj": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Western Hindi" + }, + "bjk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Madak linkage" + }, + "bjl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "Willaumez linkage", + "level7": "Bola-Bulu" + }, + "bjm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Gorani", + "level9": "Shabak-Bajalani" + }, + "bjn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "East Borneo Malay", + "level6": "Banjar-Berau-Brunei Malay", + "level7": "Banjar-Bukit Malay" + }, + "bjo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Central Bandaic", + "level8": "Central Core Bandaic", + "level9": "Mid-Southern Central Core Bandaic" + }, + "bjq": { + "level0": "Bookkeeping" + }, + "bjr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Kainantu", + "level3": "Tairora" + }, + "bjs": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Barbados-Eustatius", + "level15": "Barbados-Trinidad" + }, + "bjt": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Balanta" + }, + "bju": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid" + }, + "bjv": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Logone-Chari", + "level7": "Bediondo" + }, + "bjw": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Bakwe-Wane" + }, + "bjx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Kalinga", + "level8": "Northern Kalinga", + "level9": "Northwest Kalinga" + }, + "bjy": { + "level0": "Pama-Nyungan", + "level1": "Rockhampton-Gladstone" + }, + "bjz": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean", + "level3": "Nuclear Binanderean", + "level4": "South Binanderean", + "level5": "Coastal Binanderean", + "level6": "Baruga-Doghoro" + }, + "bka": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bikwin-Jen", + "level5": "Southern Bikwin-Jen", + "level6": "Bambuka-Gomu-Leelau" + }, + "bkb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Nuclear Cordilleran", + "level7": "Bontok-Kankanay", + "level8": "Bontok" + }, + "bkc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Western Mundu-Baka", + "level7": "Baka-Gundi", + "level8": "Baka complex" + }, + "bkd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "North Manobo", + "level5": "Kinamiguin-Bukidnon", + "level6": "Bukidnon" + }, + "bkf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Bali-Beeke" + }, + "bkg": { + "level0": "Bookkeeping" + }, + "bkh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Basaa-Yaunde (A40-70)", + "level8": "Basaa (A.40)", + "level9": "Basaa-Bakoko" + }, + "bki": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Epi-Efate", + "level7": "Epi", + "level8": "Baki-Bierebo" + }, + "bkj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Likouala-Sangha", + "level10": "Bwamba-Ngondi-Pande-Mbati-Aka" + }, + "bkk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Shinaic" + }, + "bkl": { + "level0": "Tor-Orya", + "level1": "Tor" + }, + "bkm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "Center Ring", + "level10": "Komic" + }, + "bkn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Punan", + "level6": "Bukat-Ukit-Beketan-Lugat-Lisum" + }, + "bko": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Bamileke", + "level10": "East Bamileke" + }, + "bkp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Interieur", + "level12": "Lobalic" + }, + "bkq": { + "level0": "Cariban", + "level1": "Pekodian" + }, + "bkr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "South West Greater Barito" + }, + "bks": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Central Bisayan", + "level6": "Peripheral Central Bisayan", + "level7": "Masbate-Sorsogon" + }, + "bkt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Bobangic", + "level13": "Bobangic Riverain" + }, + "bku": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "South Mangyan", + "level4": "Buhid-Taubuid" + }, + "bkv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Bendic" + }, + "bkw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Mpoic" + }, + "bkx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Rote-Meto", + "level4": "West Rote", + "level5": "Dengka-Meto", + "level6": "Meto", + "level7": "Central Meto" + }, + "bky": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Bendic", + "level6": "Nuclear Bendic" + }, + "bkz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Eastern Bungku-Tolaki", + "level8": "East Coast Bungku-Tolaki" + }, + "bla": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot" + }, + "blc": { + "level0": "Salishan" + }, + "bld": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Gorontalo-Mongondow", + "level4": "Gorontalic" + }, + "ble": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Balanta" + }, + "blf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Gorontalo-Mongondow", + "level4": "Gorontalic" + }, + "blg": { + "level0": "Bookkeeping" + }, + "blh": { + "level0": "Kru", + "level1": "Greater Western Kru" + }, + "bli": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Mongoic", + "level11": "Bolia-Ntomba" + }, + "blj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan" + }, + "blk": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Peripheral Karen" + }, + "bll": { + "level0": "Siouan", + "level1": "Ohio Valley Siouan", + "level2": "Southeastern Siouan" + }, + "blm": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "Baka-Beli", + "level3": "Morokodo-Beli" + }, + "bln": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bikol", + "level5": "Coastal Bikol" + }, + "blo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Na-Togo", + "level4": "Basila-Adele" + }, + "blp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Santa Isabel", + "level10": "Central Santa Isabel", + "level11": "Zazao-Blanga" + }, + "blq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "South-East Admiralty", + "level7": "Lou-Paluai" + }, + "blr": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Waic", + "level5": "Bulangic" + }, + "bls": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Tominic", + "level5": "Southern Tomini" + }, + "blt": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Chiang Saeng", + "level10": "Black Tai" + }, + "blu": { + "level0": "Bookkeeping" + }, + "blv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Northern Njila", + "level9": "Mbundu (H.20)" + }, + "blw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Nuclear Cordilleran" + }, + "blx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Luzon", + "level3": "Sambalic", + "level4": "Mag-Ayta" + }, + "bly": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental" + }, + "blz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Saluan-Banggai", + "level6": "Eastern Saluan-Banggai" + }, + "bma": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan" + }, + "bmb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega", + "level8": "Bembe-Buyu" + }, + "bmc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Kairiru-Manam", + "level8": "Manamic linkage", + "level9": "Bam-Manam" + }, + "bmd": { + "level0": "Atlantic-Congo", + "level1": "Mel", + "level2": "Northern Mel" + }, + "bme": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Western Mundu-Baka", + "level7": "Baka-Gundi" + }, + "bmf": { + "level0": "Atlantic-Congo", + "level1": "Mel", + "level2": "Southern Mel", + "level3": "Bullom", + "level4": "Northern Bullom" + }, + "bmg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Bamweic" + }, + "bmh": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Kokon" + }, + "bmi": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Bagirmic" + }, + "bmj": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Unclassified Bihari", + "level10": "Kuswaric" + }, + "bmk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Are linkage", + "level10": "Boanaki-Paiwa" + }, + "bml": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Bombomic" + }, + "bmm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "North-Central Malagasic", + "level7": "Northern Malagasic", + "level8": "Tsimihety-Betsimisaraka" + }, + "bmn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "Oumic", + "level9": "Magoric" + }, + "bmo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Mbam-Nkam Nun" + }, + "bmp": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Warup", + "level4": "Nuclear Warup", + "level5": "Unclassified Nuclear Warup" + }, + "bmq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Bwamu" + }, + "bmr": { + "level0": "Boran" + }, + "bms": { + "level0": "Saharan", + "level1": "Western Saharan", + "level2": "Kanuri-Kanembu", + "level3": "Kanuric", + "level4": "East Kanuri" + }, + "bmt": { + "level0": "Hmong-Mien", + "level1": "Mienic", + "level2": "Mien-Mun" + }, + "bmu": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Rawlinson", + "level5": "Pindiu", + "level6": "Kosorong-Burum-Mindik" + }, + "bmv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "Center Ring", + "level10": "Mmen-Bum" + }, + "bmw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)" + }, + "bmx": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Garuh-Foran" + }, + "bmy": { + "level0": "Bookkeeping" + }, + "bmz": { + "level0": "Anim", + "level1": "Tirio", + "level2": "Nuclear Tirio" + }, + "bna": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Tukangbesi-Bonerate" + }, + "bnb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Northern Murutic", + "level8": "Lowland Murut" + }, + "bnd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "Banda-Geser" + }, + "bne": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Gorontalo-Mongondow", + "level4": "Gorontalic" + }, + "bnf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "East Seram", + "level4": "East Rivers Seram" + }, + "bng": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Sawabantu", + "level8": "Bengaic" + }, + "bnh": { + "level0": "Bookkeeping" + }, + "bni": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Bobangic", + "level13": "Bobangic Riverain", + "level14": "Bobangi-Bangala-Lingala" + }, + "bnj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "South Mangyan", + "level4": "Buhid-Taubuid", + "level5": "Batangan" + }, + "bnk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Epi-Efate", + "level7": "Epi", + "level8": "Baki-Bierebo" + }, + "bnl": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Unclassified East Cushitic" + }, + "bnm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Sawabantu", + "level8": "Bengaic" + }, + "bnn": { + "level0": "Austronesian" + }, + "bno": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan" + }, + "bnp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "Willaumez linkage", + "level7": "Bola-Bulu" + }, + "bnq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sangiric", + "level3": "Southern Sangiric" + }, + "bnr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "East Santo", + "level9": "Southeast Santo" + }, + "bns": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Western Hindi", + "level10": "Bundeli-Bharia" + }, + "bnu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Makassaric", + "level5": "Nuclear Makassaric" + }, + "bnv": { + "level0": "Tor-Orya", + "level1": "Tor", + "level2": "Coastal Tor" + }, + "bnw": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Central Sepik Hill", + "level3": "Nuclear Central Sepik Hill" + }, + "bnx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Luba-Kaonde", + "level9": "Lubaic", + "level10": "Bangubangu-Kasai" + }, + "bny": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan" + }, + "bnz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Yukubenic", + "level5": "Akum-Beezen", + "level6": "Beezen-Baazem" + }, + "boa": { + "level0": "Boran" + }, + "bob": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Eastern Omo-Tana", + "level8": "Karre-Boni" + }, + "boc": { + "level0": "Bookkeeping" + }, + "bod": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan" + }, + "boe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Yemne-Kimbi", + "level6": "Ji" + }, + "bof": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding" + }, + "bog": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "boh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Kwa-Kasai North" + }, + "boi": { + "level0": "Chumashan", + "level1": "Southern Chumashan", + "level2": "Central Chumashan" + }, + "boj": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Mindjim", + "level4": "Lower Minjim", + "level5": "Coastal Minjim" + }, + "bok": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Likouala-Sangha", + "level10": "Impfondoic" + }, + "bol": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Nuclear Boleic", + "level8": "Galambu-Bele", + "level9": "Kirfi-Bele", + "level10": "Ngamo-Bele", + "level11": "Bolanci-Bele" + }, + "bom": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Beromic", + "level5": "Iten-Cara-Berom", + "level6": "Cara-Berom" + }, + "bon": { + "level0": "Eastern Trans-Fly" + }, + "boo": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Soninke-Bozo", + "level4": "Bozo", + "level5": "Nuclear Bozo", + "level6": "Ti-Bozo" + }, + "bop": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Yupna", + "level4": "Kewieng-Bonkiman-Nokopo" + }, + "bor": { + "level0": "Bororoan", + "level1": "Bororo-Otuke" + }, + "bos": { + "level0": "Indo-European", + "level1": "Balto-Slavic", + "level2": "Slavic" + }, + "bot": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi" + }, + "bou": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "West Ruvu", + "level11": "Seuta", + "level12": "Bondei-Shambala" + }, + "bov": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Ka-Togo", + "level4": "Kposo-Ahlo-Bowili" + }, + "bow": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Tonda" + }, + "box": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Bwamu", + "level8": "Nuclear Bwamu" + }, + "boy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Ngbele-Ngenda", + "level15": "Ngendan", + "level16": "Unclassified Ngendan" + }, + "boz": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Soninke-Bozo", + "level4": "Bozo", + "level5": "Nuclear Bozo", + "level6": "Ti-Bozo" + }, + "bpa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Ambrym", + "level7": "Orkon-West Ambrym", + "level8": "West Ambrym", + "level9": "Southwest Ambrym" + }, + "bpb": { + "level0": "Unattested", + "level1": "Barbacoan (Unattested)" + }, + "bpc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Beboid", + "level6": "Eastern Beboid", + "level7": "Nsari-Nooni-Ncane", + "level8": "Nooni-Ncane", + "level9": "Ncane-Cung", + "level10": "Cung-Mbuk" + }, + "bpd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Central Bandaic", + "level8": "Central Core Bandaic" + }, + "bpe": { + "level0": "Sko", + "level1": "Skou-Serra-Piore", + "level2": "Barupu Lagoon" + }, + "bpg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Sarmi" + }, + "bph": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Andic", + "level4": "Botlikh-Godoberi" + }, + "bpi": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Silopic" + }, + "bpj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Mbagani-Lwalwa" + }, + "bpk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Southern New Caledonian", + "level8": "Mid-Southern New Caledonian" + }, + "bpl": { + "level0": "Pidgin", + "level1": "Malay-based pidgin" + }, + "bpm": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Unclassified Rai Coast" + }, + "bpn": { + "level0": "Hmong-Mien", + "level1": "Mienic", + "level2": "Zaominic" + }, + "bpo": { + "level0": "Bookkeeping" + }, + "bpp": { + "level0": "Kaure-Kosare" + }, + "bpq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay", + "level6": "Eastern Indonesia Trade Malay", + "level7": "Ambonic Malay" + }, + "bpr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bilic", + "level3": "Tboli-Blaan", + "level4": "Blaan" + }, + "bps": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bilic", + "level3": "Tboli-Blaan", + "level4": "Blaan" + }, + "bpt": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Flinders-Barrow" + }, + "bpu": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Mindjim", + "level4": "Lower Minjim", + "level5": "Coastal Minjim" + }, + "bpv": { + "level0": "Anim", + "level1": "Marind-Boazi-Yaqai", + "level2": "Marindic" + }, + "bpw": { + "level0": "Left May", + "level1": "Western Left May", + "level2": "Iteri-Bo" + }, + "bpx": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil", + "level8": "Rathawi-Palya" + }, + "bpy": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga" + }, + "bpz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Rote-Meto", + "level4": "Nuclear Rote", + "level5": "Central East Rote", + "level6": "Southeast Rote" + }, + "bqa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Oti North Guang", + "level9": "River Oti North Guang", + "level10": "Chumbuli" + }, + "bqb": { + "level0": "Greater Kwerba", + "level1": "Kwerba-Samarokena", + "level2": "Kwerbaic" + }, + "bqc": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Bisa-Busa", + "level3": "Samo-Busa", + "level4": "Busan", + "level5": "Boko-Busa" + }, + "bqd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Konja" + }, + "bqf": { + "level0": "Bookkeeping" + }, + "bqg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Eastern Grusi", + "level9": "Tem-Chala", + "level10": "Bago-Delo-Cala" + }, + "bqh": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Eastern Tibetic" + }, + "bqi": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Luric-Dezfulic", + "level8": "Luric", + "level9": "Bakhtiari-Southern Lori" + }, + "bqj": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Jola", + "level6": "Gusilay-Bandial" + }, + "bqk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Central Bandaic", + "level8": "Central Core Bandaic" + }, + "bql": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Numugenan", + "level6": "Karian-Usan-Yaben" + }, + "bqm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Lundu-Balong (A.10)", + "level8": "Greater Manenguba", + "level9": "Bafaw-Balong-Manenguba", + "level10": "Bafawic-Bakweric", + "level11": "Bakweric" + }, + "bqn": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Central European Sign" + }, + "bqo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Southwest Grassfields" + }, + "bqp": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Bisa-Busa", + "level3": "Samo-Busa", + "level4": "Busan", + "level5": "Boko-Busa" + }, + "bqq": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "East Tariku", + "level3": "Eritai-Obokuitai-Biritai" + }, + "bqr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Northern Murutic", + "level8": "Sumambu-Tagal", + "level9": "Tidung-Bulusu" + }, + "bqs": { + "level0": "Ramu", + "level1": "Lower Ramu", + "level2": "Ottilien", + "level3": "Bosngun-Awar" + }, + "bqt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Mbam-Nkam Nun", + "level10": "Nun MCNB" + }, + "bqu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Ngbele-Ngenda", + "level15": "Ngendan", + "level16": "Unclassified Ngendan" + }, + "bqv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Northwestern Benue-Congo Plateau", + "level6": "Koroic", + "level7": "Tinoric" + }, + "bqw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid" + }, + "bqx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Kambari-Cicipu", + "level6": "Kambaric", + "level7": "East Kambaric" + }, + "bqy": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "bqz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Lundu-Balong (A.10)", + "level8": "Greater Manenguba", + "level9": "Bafaw-Balong-Manenguba", + "level10": "Manenguba", + "level11": "Central Manenguba" + }, + "bra": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Western Hindi" + }, + "brb": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "Nuclear West Bahnaric" + }, + "brc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch", + "level6": "Middle-Modern Dutch", + "level7": "Modern Dutch", + "level8": "Southwestern Dutch", + "level9": "Zeeuwic" + }, + "brd": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Newaric", + "level4": "Thangmi-Baram" + }, + "bre": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Insular Celtic", + "level6": "Brythonic", + "level7": "Southwestern Brythonic", + "level8": "Middle-Modern Southwestern Brythonic", + "level9": "Bretonic" + }, + "brf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Komoic" + }, + "brg": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Bolivian Arawakan" + }, + "brh": { + "level0": "Dravidian", + "level1": "North Dravidian" + }, + "bri": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Lundu-Balong (A.10)", + "level8": "Greater Manenguba", + "level9": "Bafaw-Balong-Manenguba", + "level10": "Bafawic-Bakweric", + "level11": "Bakweric" + }, + "brj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Epi-Efate", + "level7": "Epi", + "level8": "Bieria-Maii" + }, + "brk": { + "level0": "Nubian", + "level1": "Central Nubian" + }, + "brl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Sotho-Tswana (S.30)", + "level11": "Northern Sotho" + }, + "brm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Zandic", + "level6": "Barambo-Pambia" + }, + "brn": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Isthmic Chibchan", + "level3": "Western Isthmic Chibchan" + }, + "bro": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Southern Tibetic" + }, + "brp": { + "level0": "Geelvink Bay", + "level1": "Barapasi-Sauri-Kofei" + }, + "brq": { + "level0": "Ramu", + "level1": "Goam", + "level2": "Tamolan", + "level3": "Breri-Romkun" + }, + "brr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Guadalcanal-Nggelic", + "level6": "Southeast Guadalcanal" + }, + "brs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio", + "level5": "Greater Kaili", + "level6": "Common Kaili" + }, + "brt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "North Tivoid" + }, + "bru": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "West Katuic", + "level3": "Brou-So", + "level4": "Eastern Bru-Katang" + }, + "brv": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "West Katuic", + "level3": "Brou-So", + "level4": "Western Bru-So" + }, + "brw": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "South-Western Dravidian" + }, + "brx": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Boroic", + "level4": "Tiwa-Boro", + "level5": "Bodo-Mech-Kachari" + }, + "bry": { + "level0": "Ndu", + "level1": "Nuclear Ndu", + "level2": "Sawosic", + "level3": "Burui-Gaikundi" + }, + "brz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Bel-Roinji-Nenaya", + "level9": "Bel", + "level10": "Western Bel" + }, + "bsb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Bisaya-Lotud" + }, + "bsc": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Tenda", + "level3": "Bassari-Bedik-Bapen" + }, + "bsd": { + "level0": "Bookkeeping" + }, + "bse": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "South Ring" + }, + "bsf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Shiroro-Kamuku", + "level6": "Shiroro" + }, + "bsg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian" + }, + "bsh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Nuristani" + }, + "bsi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Lundu-Balong (A.10)", + "level8": "Greater Manenguba", + "level9": "Bafaw-Balong-Manenguba", + "level10": "Manenguba" + }, + "bsj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Tula-Longuda", + "level6": "Tula-Waja", + "level7": "Tulaic", + "level8": "Tula-Ma-Yebu", + "level9": "Nuclear Tulaic" + }, + "bsl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Basa", + "level7": "Kontagora-Gumna-Koromba", + "level8": "Gumna-Kontagora" + }, + "bsm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Central-Western Yapen", + "level7": "Central Yapen", + "level8": "Serui-Busami" + }, + "bsn": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Western Eastern Tucanoan", + "level3": "Barasano-Eduria-Macuna" + }, + "bso": { + "level0": "Bookkeeping" + }, + "bsp": { + "level0": "Atlantic-Congo", + "level1": "Mel", + "level2": "Northern Mel" + }, + "bsq": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Bassa-Klao", + "level5": "Bassaic" + }, + "bsr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Basa", + "level7": "Kontagora-Gumna-Koromba", + "level8": "Gumna-Kontagora" + }, + "bss": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Lundu-Balong (A.10)", + "level8": "Greater Manenguba", + "level9": "Bafaw-Balong-Manenguba", + "level10": "Manenguba", + "level11": "Central Manenguba" + }, + "bst": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "North-West Ometo" + }, + "bsu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Eastern Bungku-Tolaki", + "level8": "East Coast Bungku-Tolaki" + }, + "bsv": { + "level0": "Bookkeeping" + }, + "bsw": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Eastern Omo-Tana", + "level8": "Baiso-Jiiddu" + }, + "bsx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Tarokoid", + "level5": "Yangkam-Tarok-Pe" + }, + "bsy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Bisaya-Lotud" + }, + "bsz": { + "level0": "Bookkeeping" + }, + "bta": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic", + "level5": "Bata-Bwatiye" + }, + "btb": { + "level0": "Bookkeeping" + }, + "btc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Bati-Mbure-Yambassa" + }, + "btd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran", + "level3": "Batakic", + "level4": "Northern Batak" + }, + "bte": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "Ningic" + }, + "btf": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Dangla-Mabire-Birgit", + "level6": "Birgit-Mogum-Toram" + }, + "btg": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Beteic", + "level3": "Eastern Bete" + }, + "bth": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Bidayuh-Southern Land Dayak", + "level4": "Bidayuh", + "level5": "Central-Western Bidayuh" + }, + "bti": { + "level0": "Geelvink Bay", + "level1": "Burate-Wate" + }, + "btj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "East Borneo Malay", + "level6": "Banjar-Berau-Brunei Malay", + "level7": "Berau-Brunei Malay", + "level8": "Bruneic Malay", + "level9": "Brunei-Bacan Malay" + }, + "btl": { + "level0": "Bookkeeping" + }, + "btm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran", + "level3": "Batakic", + "level4": "Central-Southern Batak", + "level5": "Southern Batak", + "level6": "Angkola-Mandailing" + }, + "btn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "West Bisayan", + "level6": "Kuyan", + "level7": "Datagnon-Santa Teresa-Semirara" + }, + "bto": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bikol", + "level5": "Inland Bikol" + }, + "btp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Kilivila-Misima", + "level8": "Kilivilic" + }, + "btq": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "North Aslian", + "level4": "Maniq-Menraq-Batek", + "level5": "Menraq-Batek", + "level6": "Batekic" + }, + "btr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Maewo" + }, + "bts": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran", + "level3": "Batakic", + "level4": "Central-Southern Batak" + }, + "btt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Bendic", + "level6": "Nuclear Bendic", + "level7": "Bete-Obanliku" + }, + "btu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "North Tivoid" + }, + "btv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Kohistani", + "level8": "Indus Kohistanic", + "level9": "Outer Indus Kohistani", + "level10": "Bateri-Mankiyali" + }, + "btw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "South Bisayan", + "level6": "Butuan-Tausug" + }, + "btx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran", + "level3": "Batakic", + "level4": "Northern Batak" + }, + "bty": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "East Seram", + "level4": "East Rivers Seram" + }, + "btz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran", + "level3": "Batakic", + "level4": "Northern Batak" + }, + "bub": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Adamawa Bua", + "level5": "Riverine Bua", + "level6": "Bua-Lua", + "level7": "Ba-Korom" + }, + "buc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "North-Central Malagasic" + }, + "bud": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Gurma-Yom-Naudem", + "level11": "Gurma", + "level12": "Gurma A" + }, + "buf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Nkutsuic", + "level11": "Bushoong-Wongo-Lele" + }, + "bug": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Tamanic-Bugis", + "level5": "Bugis" + }, + "buh": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Paheng-Younuo" + }, + "bui": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Likouala-Sangha", + "level10": "Mokiba-Ngando" + }, + "buj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Basa", + "level7": "Kontagora-Gumna-Koromba" + }, + "buk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "North Huon Gulf linkage" + }, + "bul": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "South Slavic", + "level5": "Eastern South Slavic", + "level6": "Macedo-Bulgarian" + }, + "bum": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Basaa-Yaunde (A40-70)", + "level8": "Yaunde-Fang (A.70)" + }, + "bun": { + "level0": "Atlantic-Congo", + "level1": "Mel", + "level2": "Southern Mel", + "level3": "Bullom" + }, + "buo": { + "level0": "South Bougainville", + "level1": "Buinic", + "level2": "Buin" + }, + "bup": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Nuclear Muna-Buton", + "level8": "Munan" + }, + "buq": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Manep-Barem" + }, + "bus": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Bisa-Busa", + "level3": "Samo-Busa", + "level4": "Busan", + "level5": "Boko-Busa" + }, + "but": { + "level0": "Nuclear Torricelli", + "level1": "Marienberg" + }, + "buu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke", + "level11": "So-Poke", + "level12": "So-Lebonya", + "level13": "Lebonya", + "level14": "Bantu D33", + "level15": "Budu-Ndaka-Mbo" + }, + "buv": { + "level0": "Yuat", + "level1": "Miyak-Bun-Biwat", + "level2": "Bun-Mundukumo" + }, + "buw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "B10-B30", + "level8": "Okani (B.30)", + "level9": "Southern Okani" + }, + "bux": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi East", + "level6": "Boghomic" + }, + "buy": { + "level0": "Atlantic-Congo", + "level1": "Mel", + "level2": "Southern Mel", + "level3": "Bullom", + "level4": "Northern Bullom" + }, + "buz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Beboid", + "level6": "Eastern Beboid", + "level7": "Bebe-Kemezung", + "level8": "Naki-Kemezung", + "level9": "Nakic" + }, + "bva": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "Barainic" + }, + "bvb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi" + }, + "bvc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Central-Northern Malaita", + "level9": "North Malaitan" + }, + "bvd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Central-Northern Malaita", + "level9": "North Malaitan" + }, + "bve": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "East Borneo Malay", + "level6": "Banjar-Berau-Brunei Malay", + "level7": "Berau-Brunei Malay" + }, + "bvf": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.3", + "level5": "Sokoroic", + "level6": "Miltuic" + }, + "bvg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Lundu-Balong (A.10)" + }, + "bvh": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Unclassified Boleic" + }, + "bvi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Sereic", + "level6": "Sere-Indri", + "level7": "Sere-Bviri", + "level8": "Bai-Viri" + }, + "bvj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Ogonoid", + "level5": "West Ogonoid" + }, + "bvk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Punan", + "level6": "Bukat-Ukit-Beketan-Lugat-Lisum" + }, + "bvl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "West-Central South American Sign" + }, + "bvm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "South Ring" + }, + "bvn": { + "level0": "Nuclear Torricelli", + "level1": "Marienberg" + }, + "bvo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Adamawa Bua", + "level5": "Inland Bua", + "level6": "Bolgo-Koke" + }, + "bvp": { + "level0": "Bookkeeping" + }, + "bvq": { + "level0": "Central Sudanic", + "level1": "Membi-Mangbutu-Efe", + "level2": "Unclassified Membi-Mangbutu-Efe" + }, + "bvr": { + "level0": "Maningrida", + "level1": "Bureran" + }, + "bvs": { + "level0": "Bookkeeping" + }, + "bvt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "Banda-Geser", + "level4": "Seran Laut", + "level5": "Geser-Gorom-Bati" + }, + "bvu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "East Borneo Malay", + "level6": "Banjar-Berau-Brunei Malay", + "level7": "Banjar-Bukit Malay" + }, + "bvw": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Teraic", + "level5": "Eastern Tera" + }, + "bvx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Likouala-Sangha" + }, + "bvy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Central Bisayan", + "level6": "Warayan", + "level7": "Samar-Waray" + }, + "bvz": { + "level0": "Geelvink Bay" + }, + "bwa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Voh-Kone-Cem-Pac", + "level10": "Voh-Kone", + "level11": "Bwatooic", + "level12": "Haeke-Bwatoo" + }, + "bwb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Western Fijian" + }, + "bwc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Sabi", + "level8": "Malungu-Central Sabi" + }, + "bwd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Bwaidoga linkage", + "level9": "Bwaidoka-Iduna" + }, + "bwe": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Central Karen", + "level3": "Geba-Bwe" + }, + "bwf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Dobu-Duau linkage", + "level9": "Boselewa-Galeya" + }, + "bwg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Tumbuka-Sena-Nyanja", + "level8": "Sena-Nyanja", + "level9": "Senaic" + }, + "bwh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid" + }, + "bwi": { + "level0": "Arawakan", + "level1": "Japura-Colombia", + "level2": "Nuclear Japura-Colombia", + "level3": "Northeast Japura-Colombia", + "level4": "Baniwa-Curripaco-Tariano", + "level5": "Baniwa-Curripaco" + }, + "bwj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Bwamu", + "level8": "Nuclear Bwamu" + }, + "bwk": { + "level0": "Mailuan", + "level1": "Bauwakic" + }, + "bwl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Mongala" + }, + "bwm": { + "level0": "Yuat", + "level1": "Miyak-Bun-Biwat", + "level2": "Bun-Mundukumo" + }, + "bwn": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Paheng-Younuo", + "level3": "Paheng" + }, + "bwo": { + "level0": "Ta-Ne-Omotic", + "level1": "Kefoid" + }, + "bwp": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Dumut", + "level6": "Mandobo" + }, + "bwq": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Duun-Bobo", + "level4": "Bobo" + }, + "bwr": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Bura-Marghi", + "level6": "Buraic" + }, + "bws": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Bombomic" + }, + "bwt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Lundu-Balong (A.10)", + "level8": "Greater Manenguba", + "level9": "Bafaw-Balong-Manenguba", + "level10": "Bafawic-Bakweric", + "level11": "Bafawic" + }, + "bwu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Buli-Koma" + }, + "bwv": { + "level0": "Bookkeeping" + }, + "bww": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Middle Bomokandian", + "level15": "Late Bomokandian", + "level16": "Pagabeteic" + }, + "bwx": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Bu-Nao-Bunu" + }, + "bwy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Bwamu", + "level8": "Nuclear Bwamu" + }, + "bwz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "West Kikongo", + "level20": "Vilic", + "level21": "Lumbuic", + "level22": "Lumbu-Bwisi" + }, + "bxa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Makira" + }, + "bxb": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Northern Lwoo" + }, + "bxc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Sawabantu", + "level8": "Bengaic", + "level9": "Unclassified Bengaic" + }, + "bxd": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Northern Burmish", + "level5": "Maruic" + }, + "bxf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Label-Bilur" + }, + "bxg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Bobangic", + "level13": "Bobangic Riverain", + "level14": "Bobangi-Bangala-Lingala", + "level15": "Lingala-Bangala" + }, + "bxh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "Suauic" + }, + "bxi": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Central Karnic", + "level3": "Western Central Karnic", + "level4": "Pirlatapa-Dieric" + }, + "bxj": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Kanyara" + }, + "bxk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Northern Luyia" + }, + "bxl": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Jogo-Jeri", + "level6": "Jeri" + }, + "bxm": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Eastern Mongolic", + "level3": "Khalkha-Buriat", + "level4": "Buriat" + }, + "bxn": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Kanyara", + "level4": "Thalanyji-Burduna" + }, + "bxo": { + "level0": "Pidgin", + "level1": "Hausa-based pidgin" + }, + "bxp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Basaa-Yaunde (A40-70)", + "level8": "Yaunde-Fang (A.70)", + "level9": "Ewondo-Bebele" + }, + "bxq": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Nuclear Boleic", + "level8": "Galambu-Bele", + "level9": "Kirfi-Bele", + "level10": "Ngamo-Bele", + "level11": "Bolanci-Bele" + }, + "bxr": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Eastern Mongolic", + "level3": "Khalkha-Buriat", + "level4": "Buriat" + }, + "bxs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Southwest Grassfields" + }, + "bxt": { + "level0": "Bookkeeping" + }, + "bxu": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Eastern Mongolic", + "level3": "Khalkha-Buriat", + "level4": "Buriat" + }, + "bxv": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Bagirmic", + "level6": "Morom-Jaya-Naba", + "level7": "Naba-Berakou" + }, + "bxw": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Duun-Bobo", + "level4": "Duun-Jo", + "level5": "Duun-Seenku", + "level6": "Duun" + }, + "bxx": { + "level0": "Bookkeeping" + }, + "bxz": { + "level0": "Mailuan", + "level1": "Binaharic" + }, + "bya": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Palawanic", + "level4": "Northern Palawanic", + "level5": "Batak-Central Tagbanwa" + }, + "byb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Beboid" + }, + "byc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "North-South Central Delta Cross", + "level7": "Ubaghara-Kohumono" + }, + "byd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Benyadu-Bekati" + }, + "bye": { + "level0": "Sepik", + "level1": "Ram", + "level2": "Pouye-Karawa" + }, + "byf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Yukubenic", + "level5": "Bete-Lufu" + }, + "byg": { + "level0": "Dajuic", + "level1": "Western Dajuic", + "level2": "Nyala Dajuic" + }, + "byh": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kham-Magar-Chepang", + "level4": "Chepangic" + }, + "byi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega", + "level8": "Bembe-Buyu" + }, + "byj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Kauru" + }, + "byk": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Lakkia-Biao", + "level3": "Biaoic" + }, + "byl": { + "level0": "Bayono-Awbono" + }, + "bym": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Southern Maric", + "level5": "Bidyaric" + }, + "byn": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "Agaw", + "level3": "Northern-Eastern-Western Agaw", + "level4": "Northeastern Agaw" + }, + "byo": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Hanic", + "level7": "Bi-Ka" + }, + "byp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Bendic", + "level6": "Nuclear Bendic" + }, + "byq": { + "level0": "Austronesian", + "level1": "East Formosan", + "level2": "Northern East Formosan" + }, + "byr": { + "level0": "Angan", + "level1": "Nuclear Angan", + "level2": "Baruya-Simbari" + }, + "bys": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bikwin-Jen", + "level5": "Northern Bikwin-Jen", + "level6": "Burak-Loo" + }, + "byt": { + "level0": "Saharan", + "level1": "Eastern Saharan" + }, + "byv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Bamileke", + "level10": "East Bamileke" + }, + "byw": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Tamar", + "level6": "Yakkha-Athpariyic", + "level7": "Athpariyic" + }, + "byx": { + "level0": "Baining" + }, + "byy": { + "level0": "Bookkeeping" + }, + "byz": { + "level0": "Ramu" + }, + "bza": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Southwest Mande", + "level4": "Mende-Loma", + "level5": "Mende-Bandi", + "level6": "Bandi-Zialo" + }, + "bzb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Saluan-Banggai", + "level6": "Western Saluan-Banggai" + }, + "bzc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "North-Central Malagasic", + "level7": "Central-Eastern Malagasic" + }, + "bzd": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Isthmic Chibchan", + "level3": "Western Isthmic Chibchan", + "level4": "Viceitic" + }, + "bze": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Soninke-Bozo", + "level4": "Bozo", + "level5": "Nuclear Bozo" + }, + "bzf": { + "level0": "Ndu" + }, + "bzg": { + "level0": "Austronesian", + "level1": "Western Plains Austronesian", + "level2": "Central Western Plains" + }, + "bzh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Buang linkage", + "level9": "Mapos-Mangga-Wagau" + }, + "bzi": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Bisoid", + "level7": "Bisu-Pyen-Laomian" + }, + "bzj": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Western Caribbean Creole", + "level14": "Miskitoic Creole English", + "level15": "Belize-Miskito Creole English" + }, + "bzk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Western Caribbean Creole", + "level14": "Miskitoic Creole English", + "level15": "Belize-Miskito Creole English" + }, + "bzl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tolitoli" + }, + "bzm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Mongala", + "level11": "Motemboic" + }, + "bzn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "West Piru Bay", + "level5": "Hoamoal", + "level6": "East Hoamoal" + }, + "bzp": { + "level0": "South Bird's Head Family", + "level1": "East South Bird's Head", + "level2": "Kemberanic" + }, + "bzq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "South Halmahera", + "level6": "Central-Eastern South Halmahera" + }, + "bzr": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Eastern Maric" + }, + "bzs": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic" + }, + "bzt": { + "level0": "Artificial Language" + }, + "bzv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Beboid", + "level6": "Eastern Beboid", + "level7": "Bebe-Kemezung" + }, + "bzw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Basa" + }, + "bzx": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Soninke-Bozo", + "level4": "Bozo" + }, + "bzy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Bendic", + "level6": "Nuclear Bendic", + "level7": "Bete-Obanliku" + }, + "bzz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "Central Tivoid", + "level7": "Central Tivoid A", + "level8": "Tiv-Evand" + }, + "caa": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Cholan-Tzeltalan", + "level4": "Cholan", + "level5": "Chorti-Cholti" + }, + "cab": { + "level0": "Arawakan", + "level1": "Caribbean Arawakan", + "level2": "Antillean Arawakan", + "level3": "Ineric", + "level4": "Island Carib-Garifuna" + }, + "cac": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Kanjobalan-Chujean", + "level4": "Chujean" + }, + "cad": { + "level0": "Caddoan" + }, + "cae": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Cangin", + "level3": "Saafi-Noon-Lehar", + "level4": "Noon-Lehar" + }, + "caf": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central British Columbia Athabaskan", + "level4": "Carrieric", + "level5": "Dakelh" + }, + "cag": { + "level0": "Mataguayan", + "level1": "Mataguayo I" + }, + "cah": { + "level0": "Zaparoan", + "level1": "Iquito-Arabela", + "level2": "Cahuarano-Iquito" + }, + "cak": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Quichean", + "level4": "Core Quichean", + "level5": "Cakchiquel-Tzutujil", + "level6": "Kaqchikelic" + }, + "cal": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Nuclear Trukic", + "level10": "Central Trukic", + "level11": "Satawalese-Carolinian", + "level12": "Macro-Carolinian" + }, + "cam": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Voh-Kone-Cem-Pac", + "level10": "Cem-Pac" + }, + "can": { + "level0": "Lower Sepik" + }, + "cao": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Bolivian Nawa" + }, + "cap": { + "level0": "Uru-Chipaya" + }, + "caq": { + "level0": "Austroasiatic", + "level1": "Nicobaric" + }, + "car": { + "level0": "Cariban", + "level1": "Guianan" + }, + "cat": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance" + }, + "cav": { + "level0": "Pano-Tacanan", + "level1": "Tacanan" + }, + "caw": { + "level0": "Speech Register", + "level1": "Quechua-Puquina" + }, + "cax": { + "level0": "Chiquitano" + }, + "cay": { + "level0": "Iroquoian", + "level1": "Northern Iroquoian" + }, + "cbb": { + "level0": "Arawakan", + "level1": "Japura-Colombia", + "level2": "Nuclear Japura-Colombia", + "level3": "Northeast Japura-Colombia" + }, + "cbc": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Eastern Eastern Tucanoan", + "level3": "Eastern Eastern Tucanoan II", + "level4": "Pisamira-Yuruti", + "level5": "Pisamira-Carapana" + }, + "cbd": { + "level0": "Cariban", + "level1": "Guianan", + "level2": "Taranoan" + }, + "cbe": { + "level0": "Bookkeeping" + }, + "cbg": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Magdalenic", + "level3": "Northern Magdalenic" + }, + "cbh": { + "level0": "Bookkeeping" + }, + "cbi": { + "level0": "Barbacoan", + "level1": "Awa-Southern Barbacoan", + "level2": "Cayapa-Colorado" + }, + "cbj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede", + "level7": "Eastern Ede" + }, + "cbk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Castilic", + "level13": "South Castilic", + "level14": "Ternate-Zamboanga-Cavite" + }, + "cbl": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin", + "level5": "Lai Chin", + "level6": "Falamic" + }, + "cbm": { + "level0": "Bookkeeping" + }, + "cbn": { + "level0": "Austroasiatic", + "level1": "Monic" + }, + "cbo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "North-Central Jos", + "level10": "Chokobo-Lemoro-Sanga" + }, + "cbq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Kainji Lake", + "level5": "Upper Niger Kainji", + "level6": "Oleran" + }, + "cbr": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano" + }, + "cbs": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Headwaters Pano" + }, + "cbt": { + "level0": "Cahuapanan" + }, + "cbv": { + "level0": "Kakua-Nukak" + }, + "cbw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Central Bisayan", + "level6": "Warayan", + "level7": "Samar-Waray" + }, + "cby": { + "level0": "Unclassifiable" + }, + "cca": { + "level0": "Bookkeeping" + }, + "ccc": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Purus-Chamicuro", + "level3": "Chamicuro-Morike" + }, + "ccd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Brazil-Portugal Portuguese" + }, + "cce": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Nguni-Tsonga-Copi", + "level11": "Tsonga-Copi", + "level12": "Chopi (S.60)" + }, + "ccg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Dakoid", + "level6": "Taram-Dirim-Nnakenyare" + }, + "cch": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Piti-Atsam" + }, + "ccj": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Wolof-BKK", + "level3": "Nyun", + "level4": "Buy" + }, + "ccl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Sabaki-Swahili", + "level11": "Swahili (G.40)", + "level12": "Mombasa-Lamu-Inland Swahili" + }, + "ccm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay" + }, + "cco": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Chinantec Group V" + }, + "ccp": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga", + "level10": "Southeastern Bengali" + }, + "ccq": { + "level0": "Bookkeeping" + }, + "ccr": { + "level0": "Misumalpan", + "level1": "Sumalpan", + "level2": "Matagalpan" + }, + "ccx": { + "level0": "Bookkeeping" + }, + "ccy": { + "level0": "Bookkeeping" + }, + "cda": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Eastern Tibetic" + }, + "cde": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Teluguic" + }, + "cdf": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Central Old Kuki" + }, + "cdg": { + "level0": "Bookkeeping" + }, + "cdh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Kangric-Chamealic-Bhattiyali", + "level8": "Chamealic" + }, + "cdi": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "cdj": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Kangric-Chamealic-Bhattiyali", + "level8": "Chamealic", + "level9": "Bhadrawahi-Bhalesi-Curahi" + }, + "cdm": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kham-Magar-Chepang", + "level4": "Chepangic" + }, + "cdn": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Eastern West Himalayish", + "level4": "Pithauragarh", + "level5": "Darma-Byangsi-Chaudangsi" + }, + "cdo": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Min", + "level3": "Coastal Min" + }, + "cdr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Shiroro-Kamuku", + "level6": "Kamuku-Hungwarya", + "level7": "Kamuku" + }, + "cds": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "American Sign" + }, + "cdy": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Kam-Sui", + "level3": "Then-MMS", + "level4": "Maonan-Mak-Sui", + "level5": "Maonan-Chadong" + }, + "cdz": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Mundaric" + }, + "cea": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Tsamosan", + "level3": "Coastal Tsamosan" + }, + "ceb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan" + }, + "ceg": { + "level0": "Zamucoan" + }, + "cek": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Khomic" + }, + "cen": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Zaric", + "level6": "Nuclear Zaric", + "level7": "Izeric" + }, + "ces": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "West Slavic", + "level5": "Czech-Slovak" + }, + "cey": { + "level0": "Bookkeeping", + "level1": "Pending Report Release" + }, + "cfa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Tula-Longuda", + "level6": "Tula-Waja", + "level7": "Tulaic" + }, + "cfd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Beromic", + "level5": "Iten-Cara-Berom", + "level6": "Cara-Berom" + }, + "cfg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Jukun-Mbembe-Wurbo", + "level6": "Wurbo-Wannu", + "level7": "Wurbo" + }, + "cfm": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin", + "level5": "Lai Chin", + "level6": "Falamic" + }, + "cga": { + "level0": "Yuat" + }, + "cgc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "North Manobo" + }, + "cgg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "Rutara", + "level11": "North Rutara", + "level12": "Nkore-Kiga-Nyoro-Tooro", + "level13": "Nkore-Kiga" + }, + "cgk": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Southern Tibetic", + "level7": "Dzongkhic" + }, + "cha": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian" + }, + "chb": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Magdalenic", + "level3": "Southern Magdalenic", + "level4": "Chibcha-Duit" + }, + "chc": { + "level0": "Siouan", + "level1": "Catawban" + }, + "chd": { + "level0": "Tequistlatecan" + }, + "che": { + "level0": "Nakh-Daghestanian", + "level1": "Nakh", + "level2": "Chechen-Ingush" + }, + "chf": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Cholan-Tzeltalan", + "level4": "Cholan", + "level5": "Chol-Chontal" + }, + "chg": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Turkestan" + }, + "chh": { + "level0": "Chinookan", + "level1": "Lower Chinookan" + }, + "chj": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Northwestern Chinantec", + "level6": "Chinantec Group I" + }, + "chk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Nuclear Trukic", + "level10": "Central Trukic", + "level11": "Eastern Trukic", + "level12": "Mortlockese-Trukese" + }, + "chl": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Californian Uto-Aztecan", + "level3": "Cupan", + "level4": "Cahuilla-Cupeno" + }, + "chn": { + "level0": "Chinookan", + "level1": "Lower Chinookan" + }, + "cho": { + "level0": "Muskogean", + "level1": "Western Muskogean" + }, + "chp": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Northwestern Canada Athabaskan" + }, + "chq": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Chinantec Group V" + }, + "chr": { + "level0": "Iroquoian" + }, + "chs": { + "level0": "Bookkeeping" + }, + "cht": { + "level0": "Hibito-Cholon" + }, + "chu": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "South Slavic", + "level5": "Eastern South Slavic" + }, + "chv": { + "level0": "Turkic", + "level1": "Bolgar" + }, + "chw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Chuwaboic", + "level9": "Chuwabo-Maindo" + }, + "chx": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Tamangic", + "level5": "Gurungic", + "level6": "Thakali-Chantyal" + }, + "chy": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian" + }, + "chz": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Southeastern Chinantec", + "level6": "Chinantec Group III" + }, + "cia": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Nuclear Muna-Buton", + "level8": "Butonic" + }, + "cib": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Fongbeic" + }, + "cic": { + "level0": "Muskogean", + "level1": "Western Muskogean" + }, + "cie": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mandaraic", + "level6": "Dghwedeic", + "level7": "Gudufic" + }, + "cih": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Kangric-Chamealic-Bhattiyali", + "level8": "Chamealic", + "level9": "Bhadrawahi-Bhalesi-Curahi", + "level10": "Bhadarwahic", + "level11": "Chinali-Lahul Lohar" + }, + "cik": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Western West Himalayish", + "level4": "Kinnauric" + }, + "cim": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Bairisch", + "level10": "Global South Bavarian" + }, + "cin": { + "level0": "Tupian", + "level1": "Monde", + "level2": "Gavianic", + "level3": "Nuclear Gavianic" + }, + "cip": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Tlapanec-Manguean", + "level3": "Manguean" + }, + "cir": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Southern New Caledonian", + "level8": "Mid-Southern New Caledonian" + }, + "cit": { + "level0": "Bookkeeping" + }, + "ciw": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian", + "level4": "Ojibwa-Potawatomi", + "level5": "Ojibwa", + "level6": "Nuclear Ojibwe", + "level7": "Central-Eastern-Southwestern Ojibwa" + }, + "ciy": { + "level0": "Cariban", + "level1": "Venezuelan Cariban", + "level2": "Mapoyo-Tamanaku", + "level3": "Cumana" + }, + "cja": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic", + "level4": "Chamic", + "level5": "Cham" + }, + "cje": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic", + "level4": "Chamic", + "level5": "Chru-Northern Cham", + "level6": "Chruic" + }, + "cjh": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Tsamosan", + "level3": "Inland Tsamosan" + }, + "cji": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Andic" + }, + "cjk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Chokwe-Ngangela-Nyemba (K.20)", + "level11": "Chokwe-Lwena" + }, + "cjm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic", + "level4": "Chamic", + "level5": "Cham" + }, + "cjn": { + "level0": "Sepik", + "level1": "Iwam-Wogamus", + "level2": "Wogamusin-Chenapian" + }, + "cjo": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha", + "level3": "Pre-Andine Maipuran", + "level4": "Asha-Ashe-Kak-Matsi-Nan", + "level5": "Asha-Ashe-Kak", + "level6": "Ashe-Asha", + "level7": "Asheninka" + }, + "cjp": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Isthmic Chibchan", + "level3": "Western Isthmic Chibchan", + "level4": "Viceitic" + }, + "cjr": { + "level0": "Bookkeeping" + }, + "cjs": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "South Siberian Turkic", + "level3": "Sayan-Yenisei Turkic", + "level4": "Yenisey Turkic" + }, + "cjv": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Simbu", + "level3": "Chuave-Nomane" + }, + "cjy": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic", + "level4": "Northern Chinese" + }, + "cka": { + "level0": "Bookkeeping" + }, + "ckb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Laki-Kurdish", + "level8": "Kurdish" + }, + "ckc": { + "level0": "Bookkeeping" + }, + "ckd": { + "level0": "Bookkeeping" + }, + "cke": { + "level0": "Bookkeeping" + }, + "ckf": { + "level0": "Bookkeeping" + }, + "ckh": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Jingpho-Luish", + "level3": "Luish" + }, + "cki": { + "level0": "Bookkeeping" + }, + "ckj": { + "level0": "Bookkeeping" + }, + "ckk": { + "level0": "Bookkeeping" + }, + "ckl": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Bura-Marghi", + "level6": "Buraic" + }, + "ckn": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "South Peripheral Kuki-Chin", + "level5": "Choic", + "level6": "Daai-Nghmoye-Muun-Kaang" + }, + "cko": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Central Tano", + "level7": "Bia", + "level8": "Northern Bia" + }, + "ckq": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Mubic" + }, + "ckr": { + "level0": "Baining" + }, + "cks": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil", + "level14": "Macro-French" + }, + "ckt": { + "level0": "Chukotko-Kamchatkan", + "level1": "Chukotian", + "level2": "R-Koryakic" + }, + "cku": { + "level0": "Muskogean", + "level1": "Alabaman-Koasati" + }, + "ckv": { + "level0": "Austronesian", + "level1": "East Formosan", + "level2": "Northern East Formosan" + }, + "ckw": { + "level0": "Bookkeeping" + }, + "ckx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "Central Tivoid", + "level7": "Central Tivoid B" + }, + "cky": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3", + "level6": "Kofyar-Mushere-Chip" + }, + "ckz": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Quichean", + "level4": "Core Quichean", + "level5": "Cakchiquel-Tzutujil", + "level6": "Kaqchikelic" + }, + "cla": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.4", + "level5": "Ronic" + }, + "clc": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central British Columbia Athabaskan" + }, + "cld": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "North-Eastern Neo-Aramaic" + }, + "cle": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Southeastern Chinantec", + "level6": "Chinantec Group IV" + }, + "clh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Kohistani", + "level8": "Indus Kohistanic", + "level9": "Outer Indus Kohistani" + }, + "cli": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Western Grusi", + "level9": "Sisaala-Chakali", + "level10": "Chakalic", + "level11": "Chakali-Tamprusi-Vagala", + "level12": "Chakali-Tamprusi" + }, + "clj": { + "level0": "Bookkeeping", + "level1": "Pending Report Release" + }, + "clk": { + "level0": "Sino-Tibetan", + "level1": "Digarish" + }, + "cll": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Eastern Grusi", + "level9": "Tem-Chala", + "level10": "Bago-Delo-Cala", + "level11": "Delo-Cala" + }, + "clm": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Central Salish", + "level3": "Straits Salish" + }, + "clo": { + "level0": "Tequistlatecan", + "level1": "Eastern Tequistlatecan" + }, + "clt": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Maraic", + "level5": "Nuclear Maraic" + }, + "clu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "West Bisayan", + "level6": "Kuyan", + "level7": "Datagnon-Santa Teresa-Semirara" + }, + "clw": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "South Siberian Turkic", + "level3": "Northern Altai-Lower Chulym" + }, + "cly": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Chatino", + "level5": "Core Chatino", + "level6": "Coastal Chatino", + "level7": "Eastern Chatino" + }, + "cma": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "South Bahnaric", + "level4": "Koho-Maa" + }, + "cme": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Kirma-Tyurama" + }, + "cmi": { + "level0": "Chocoan", + "level1": "Embera", + "level2": "San Juan", + "level3": "Upper San Juan" + }, + "cml": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Tamanic-Bugis", + "level5": "Bugis" + }, + "cmn": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic", + "level4": "Northern Chinese", + "level5": "Mandarinic" + }, + "cmo": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "South Bahnaric", + "level4": "Mnong-Stieng-Chrau", + "level5": "Mnong", + "level6": "Southern-Central Mnong" + }, + "cmr": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Khomic" + }, + "cms": { + "level0": "Indo-European", + "level1": "Unclassified Indo-European" + }, + "cmt": { + "level0": "Speech Register", + "level1": "Zulu-Sotho" + }, + "cna": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Western Archaic Tibetan", + "level5": "Kenhatic" + }, + "cnb": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "South Peripheral Kuki-Chin", + "level5": "Choic" + }, + "cnc": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Bisoid", + "level7": "Phunoi-Coong" + }, + "cng": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic", + "level4": "Qiang", + "level5": "Upstream-Nu Qiang" + }, + "cnh": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin", + "level5": "Lai Chin" + }, + "cni": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha", + "level3": "Pre-Andine Maipuran", + "level4": "Asha-Ashe-Kak-Matsi-Nan", + "level5": "Asha-Ashe-Kak", + "level6": "Ashe-Asha" + }, + "cnk": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Khomic" + }, + "cnl": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Southeastern Chinantec", + "level6": "Chinantec Group IV" + }, + "cnm": { + "level0": "Bookkeeping" + }, + "cno": { + "level0": "Bookkeeping" + }, + "cnp": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic", + "level4": "Yue-Pinghua", + "level5": "Pinghua" + }, + "cnq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Beboid", + "level6": "Eastern Beboid", + "level7": "Nsari-Nooni-Ncane", + "level8": "Nooni-Ncane", + "level9": "Ncane-Cung", + "level10": "Cung-Mbuk" + }, + "cns": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Asmat-Kamoro", + "level3": "Asmat", + "level4": "Central-Yaosakor Asmat" + }, + "cnt": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Northwestern Chinantec", + "level6": "Chinantec Group II" + }, + "cnu": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Greater Zenatic", + "level3": "Zenatic" + }, + "cnw": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Northeastern Kuki-Chin", + "level5": "Sizangic" + }, + "coa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Betawic" + }, + "cob": { + "level0": "Mayan", + "level1": "Huastecan Mayan" + }, + "coc": { + "level0": "Cochimi-Yuman", + "level1": "Yuman", + "level2": "General Yuman", + "level3": "Delta-Californian Yuman" + }, + "cod": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup III", + "level7": "Omagua-Kokama" + }, + "coe": { + "level0": "Tucanoan", + "level1": "Western Tucanoan", + "level2": "Koreguaje-Tama" + }, + "cof": { + "level0": "Barbacoan", + "level1": "Awa-Southern Barbacoan", + "level2": "Cayapa-Colorado" + }, + "cog": { + "level0": "Austroasiatic", + "level1": "Pearic", + "level2": "Western Pearic" + }, + "coh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Mijikenda-Pokomo-Comorian", + "level11": "Mijikenda", + "level12": "Northern Mijikenda" + }, + "coj": { + "level0": "Cochimi-Yuman", + "level1": "Cochimic" + }, + "cok": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Corachol", + "level4": "Coran" + }, + "col": { + "level0": "Salishan", + "level1": "Interior Salish", + "level2": "Southern Interior Salish" + }, + "com": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Numic", + "level3": "Central Numic" + }, + "coo": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Central Salish", + "level3": "North Georgia Central Salish" + }, + "cop": { + "level0": "Afro-Asiatic", + "level1": "Egyptian" + }, + "coq": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Pacific Coast Athabaskan", + "level4": "Oregon Athabaskan", + "level5": "Rogue River" + }, + "cor": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Insular Celtic", + "level6": "Brythonic", + "level7": "Southwestern Brythonic", + "level8": "Middle-Modern Southwestern Brythonic" + }, + "cos": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Southern Romance", + "level8": "Sardo-Corsican", + "level9": "Corsic" + }, + "cot": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha", + "level3": "Pre-Andine Maipuran", + "level4": "Asha-Ashe-Kak-Matsi-Nan", + "level5": "Asha-Ashe-Kak" + }, + "cou": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Tenda" + }, + "cov": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Kam-Sui", + "level3": "Mulam-Kam", + "level4": "Kamic", + "level5": "Northern Kam" + }, + "cow": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Tsamosan", + "level3": "Inland Tsamosan" + }, + "cox": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha", + "level3": "Pre-Andine Maipuran", + "level4": "Asha-Ashe-Kak-Matsi-Nan", + "level5": "Matsi-Nan" + }, + "coy": { + "level0": "Bookkeeping" + }, + "coz": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Ixcatec-Chocho-Popolocan", + "level5": "Chocho-Popolocan" + }, + "cpa": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Northwestern Chinantec", + "level6": "Chinantec Group II" + }, + "cpb": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha", + "level3": "Pre-Andine Maipuran", + "level4": "Asha-Ashe-Kak-Matsi-Nan", + "level5": "Asha-Ashe-Kak", + "level6": "Ashe-Asha", + "level7": "Ashe-Asha Norte" + }, + "cpc": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha", + "level3": "Pre-Andine Maipuran", + "level4": "Asha-Ashe-Kak-Matsi-Nan", + "level5": "Asha-Ashe-Kak", + "level6": "Ashe-Asha", + "level7": "Ashe-Asha Norte" + }, + "cpg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Graeco-Phrygian", + "level3": "Greek", + "level4": "South Greek", + "level5": "Central Greek", + "level6": "Koineic Greek", + "level7": "Modern Koineic Greek", + "level8": "Pontic-Cappadocian Greek" + }, + "cpi": { + "level0": "Pidgin", + "level1": "English-based pidgin" + }, + "cpn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "South Guang", + "level8": "Hill South Guang", + "level9": "Gua-Cherepon" + }, + "cpo": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Duun-Bobo", + "level4": "Duun-Jo", + "level5": "Duun-Seenku", + "level6": "Duun" + }, + "cps": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Central Bisayan", + "level6": "Peripheral Central Bisayan", + "level7": "Capiznon-Ilonggo-Kawayan" + }, + "cpu": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha", + "level3": "Pre-Andine Maipuran", + "level4": "Asha-Ashe-Kak-Matsi-Nan", + "level5": "Asha-Ashe-Kak", + "level6": "Ashe-Asha", + "level7": "Ashe-Asha Norte" + }, + "cpx": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Min", + "level3": "Coastal Min" + }, + "cpy": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha", + "level3": "Pre-Andine Maipuran", + "level4": "Asha-Ashe-Kak-Matsi-Nan", + "level5": "Asha-Ashe-Kak", + "level6": "Ashe-Asha", + "level7": "Asheninka" + }, + "cqd": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Chuanqiandian", + "level7": "First Vernacular Hmong", + "level8": "Far Western Miao" + }, + "cqu": { + "level0": "Bookkeeping" + }, + "cra": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo" + }, + "crb": { + "level0": "Arawakan", + "level1": "Caribbean Arawakan", + "level2": "Antillean Arawakan", + "level3": "Ineric", + "level4": "Island Carib-Garifuna" + }, + "crc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Ambrym" + }, + "crd": { + "level0": "Salishan", + "level1": "Interior Salish", + "level2": "Southern Interior Salish" + }, + "crf": { + "level0": "Chocoan", + "level1": "Unclassified Chocoan" + }, + "crg": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Cree-Montagnais-Naskapi", + "level4": "Cree", + "level5": "Plains Creeic" + }, + "crh": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Northwest Kipchak", + "level5": "West Kipchak", + "level6": "Crimean Tatar-Urum", + "level7": "Crimeaic" + }, + "cri": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Lower Guinea Portuguese", + "level15": "Bantu Layer Lower Guinea Portuguese", + "level16": "Saotomic" + }, + "crj": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Cree-Montagnais-Naskapi", + "level4": "Cree" + }, + "crk": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Cree-Montagnais-Naskapi", + "level4": "Cree", + "level5": "Plains Creeic" + }, + "crl": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Cree-Montagnais-Naskapi", + "level4": "Cree" + }, + "crm": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Cree-Montagnais-Naskapi", + "level4": "Cree" + }, + "crn": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Corachol", + "level4": "Coran" + }, + "cro": { + "level0": "Siouan", + "level1": "Missouri River Siouan" + }, + "crq": { + "level0": "Mataguayan", + "level1": "Mataguayo II", + "level2": "Chorote" + }, + "crr": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian" + }, + "crs": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil", + "level14": "Macro-French", + "level15": "Isle-de-France Creole" + }, + "crt": { + "level0": "Mataguayan", + "level1": "Mataguayo II", + "level2": "Chorote" + }, + "cru": { + "level0": "Bookkeeping" + }, + "crv": { + "level0": "Austroasiatic", + "level1": "Nicobaric", + "level2": "Nuclear Nicobaric", + "level3": "Chowra-Teressa" + }, + "crw": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "South Bahnaric", + "level4": "Mnong-Stieng-Chrau" + }, + "crx": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central British Columbia Athabaskan", + "level4": "Carrieric", + "level5": "Dakelh" + }, + "cry": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Northwestern Benue-Congo Plateau", + "level6": "Hyamic" + }, + "crz": { + "level0": "Chumashan", + "level1": "Southern Chumashan" + }, + "csa": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Northwestern Chinantec", + "level6": "Chinantec Group I", + "level7": "Tlacoatzintepec-Chiltepec" + }, + "csb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "West Slavic", + "level5": "Lechitic" + }, + "csc": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Spanish Sign", + "level3": "Nuclear Spanish Sign" + }, + "csd": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Old Chiangmai-Bangkok Sign" + }, + "cse": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Central European Sign", + "level4": "Nuclear Central European Sign" + }, + "csf": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "csg": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "csh": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "South Peripheral Kuki-Chin", + "level5": "Ashoic" + }, + "csi": { + "level0": "Miwok-Costanoan", + "level1": "Miwokan", + "level2": "Western Miwokan" + }, + "csj": { + "level0": "Bookkeeping", + "level1": "Pending Report Release" + }, + "csk": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Jola", + "level6": "FH-Jola" + }, + "csl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "CSLic" + }, + "csm": { + "level0": "Miwok-Costanoan", + "level1": "Miwokan", + "level2": "Eastern Miwokan", + "level3": "Sierra Miwokan" + }, + "csn": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "West-Central South American Sign" + }, + "cso": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Northwestern Chinantec", + "level6": "Chinantec Group I" + }, + "csp": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic", + "level4": "Yue-Pinghua", + "level5": "Pinghua" + }, + "csq": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Yugoslav Sign" + }, + "csr": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic" + }, + "css": { + "level0": "Miwok-Costanoan", + "level1": "Costanoan", + "level2": "Southern Costanoan" + }, + "cst": { + "level0": "Miwok-Costanoan", + "level1": "Costanoan" + }, + "csv": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "South Peripheral Kuki-Chin", + "level5": "Ashoic" + }, + "csw": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Cree-Montagnais-Naskapi", + "level4": "Cree" + }, + "csx": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "csy": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Northeastern Kuki-Chin", + "level5": "Sizangic" + }, + "csz": { + "level0": "Coosan" + }, + "cta": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Chatino", + "level5": "Core Chatino", + "level6": "Coastal Chatino" + }, + "ctc": { + "level0": "Bookkeeping" + }, + "ctd": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Northeastern Kuki-Chin", + "level5": "Thadoic" + }, + "cte": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Southeastern Chinantec", + "level6": "Chinantec Group IV" + }, + "ctg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga", + "level10": "Southeastern Bengali" + }, + "cth": { + "level0": "Bookkeeping", + "level1": "Pending Report Release" + }, + "cti": { + "level0": "Bookkeeping" + }, + "ctl": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Northwestern Chinantec", + "level6": "Chinantec Group I", + "level7": "Tlacoatzintepec-Chiltepec" + }, + "ctn": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Tamar" + }, + "cto": { + "level0": "Chocoan", + "level1": "Embera", + "level2": "Atrato" + }, + "ctp": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Chatino", + "level5": "Core Chatino", + "level6": "Coastal Chatino", + "level7": "Eastern Chatino" + }, + "cts": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bikol" + }, + "ctt": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "ctu": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Cholan-Tzeltalan", + "level4": "Cholan", + "level5": "Chol-Chontal" + }, + "cty": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Badaga-Kannada", + "level5": "Kannadoid" + }, + "ctz": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Chatino", + "level5": "Core Chatino", + "level6": "Coastal Chatino", + "level7": "Eastern Chatino" + }, + "cua": { + "level0": "Austroasiatic", + "level1": "Bahnaric" + }, + "cub": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Western Eastern Tucanoan", + "level3": "Cubeo-Desano" + }, + "cuc": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Northwestern Chinantec", + "level6": "Chinantec Group I" + }, + "cuh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Central Kenya Bantu", + "level9": "Eastern Kirinyaga" + }, + "cui": { + "level0": "Guahiboan", + "level1": "Nuclear Guahiboan", + "level2": "Central Guahibo" + }, + "cuj": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Purus-Chamicuro", + "level3": "Purus", + "level4": "Yineic", + "level5": "Western Yineic" + }, + "cuk": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Isthmic Chibchan", + "level3": "Eastern Isthmic Chibchan", + "level4": "Kuna" + }, + "cul": { + "level0": "Arawan", + "level1": "Madi-Madiha", + "level2": "Madiha" + }, + "cum": { + "level0": "Bookkeeping" + }, + "cun": { + "level0": "Bookkeeping" + }, + "cuo": { + "level0": "Cariban", + "level1": "Venezuelan Cariban", + "level2": "Mapoyo-Tamanaku", + "level3": "Cumana" + }, + "cup": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Californian Uto-Aztecan", + "level3": "Cupan", + "level4": "Cahuilla-Cupeno" + }, + "cuq": { + "level0": "Tai-Kadai", + "level1": "Hlaic", + "level2": "Nuclear Hlaic" + }, + "cur": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Tamar" + }, + "cut": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Cuicatec" + }, + "cuu": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Southern Shanic", + "level11": "Wuding-Yuanyang Tai" + }, + "cuv": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Matakam", + "level5": "Mefele-Cuvok" + }, + "cux": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Cuicatec" + }, + "cvg": { + "level0": "Sino-Tibetan", + "level1": "Kho-Bwa", + "level2": "Western Kho-Bwa", + "level3": "Chug-Lish" + }, + "cvn": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Chinantecan", + "level4": "Central-Eastern Chinantec", + "level5": "Southeastern Chinantec", + "level6": "Chinantec Group III" + }, + "cwa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Nyanza Mara", + "level11": "North Mara", + "level12": "Unclassified North Mara" + }, + "cwb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Chuwaboic", + "level9": "Chuwabo-Maindo" + }, + "cwd": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Cree-Montagnais-Naskapi", + "level4": "Cree" + }, + "cwe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "East Ruvu", + "level11": "Central East Ruvu" + }, + "cwg": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "North Aslian" + }, + "cwt": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Jola", + "level6": "FH-Jola", + "level7": "PF-Jola", + "level8": "Kwatay-Karon-Mlomp" + }, + "cxh": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Southwest South Bauchi", + "level7": "Zeemic", + "level8": "Nuclear Zeemic" + }, + "cya": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Chatino", + "level5": "Core Chatino", + "level6": "Coastal Chatino", + "level7": "Eastern Chatino" + }, + "cym": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Insular Celtic", + "level6": "Brythonic", + "level7": "Old-Modern Welsh" + }, + "cyo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "West Bisayan", + "level6": "Kuyan" + }, + "czh": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic", + "level4": "Wu-Hui Chinese" + }, + "czn": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Chatino", + "level5": "Core Chatino" + }, + "czo": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Min", + "level3": "Inland Min" + }, + "czt": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Maraic" + }, + "daa": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Dangla-Mabire-Birgit", + "level6": "Dangla" + }, + "dac": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Buang linkage", + "level9": "Mumeng", + "level10": "Dambi-Kumaru" + }, + "dad": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Bel-Roinji-Nenaya", + "level9": "Bel", + "level10": "Western Bel" + }, + "dae": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Southern Samba-Duru", + "level7": "Diic" + }, + "daf": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Mano-Dan", + "level4": "Guro-Dan", + "level5": "Dan-Toura" + }, + "dag": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Southeast Western Oti-Volta", + "level13": "Mampruli-Dagbani" + }, + "dah": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Warup" + }, + "dai": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day" + }, + "daj": { + "level0": "Dajuic", + "level1": "Western Dajuic", + "level2": "Nyala Dajuic" + }, + "dak": { + "level0": "Siouan", + "level1": "Mississippi Valley", + "level2": "Dakotan", + "level3": "Sioux" + }, + "dal": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic" + }, + "dam": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Northwestern Kainji", + "level6": "Clela-Damakawa" + }, + "dan": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "North Germanic", + "level5": "South Scandinavian" + }, + "dao": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "South Peripheral Kuki-Chin", + "level5": "Choic", + "level6": "Daai-Nghmoye-Muun-Kaang" + }, + "dap": { + "level0": "Bookkeeping" + }, + "daq": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Gondi", + "level4": "Southeast Gondi", + "level5": "South Bastar Gondi-Koya" + }, + "dar": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Dargwic" + }, + "das": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Wee", + "level5": "Guere-Krahn" + }, + "dat": { + "level0": "Bookkeeping" + }, + "dau": { + "level0": "Dajuic", + "level1": "Western Dajuic" + }, + "dav": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Kilimanjaro-Taita", + "level9": "Taita-Sagalla" + }, + "daw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Mansakan" + }, + "dax": { + "level0": "Pama-Nyungan", + "level1": "Yuulngu", + "level2": "Southern Yolngu", + "level3": "Southern-Eastern Yolngu" + }, + "daz": { + "level0": "Nuclear Trans New Guinea", + "level1": "Paniai Lakes", + "level2": "Auye-Dao" + }, + "dbb": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Nuclear Boleic", + "level8": "Gera-Geruma-Kubi-Deno", + "level9": "Kubi-Deno" + }, + "dbd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Tula-Longuda", + "level6": "Tula-Waja", + "level7": "Tulaic", + "level8": "Tula-Ma-Yebu", + "level9": "Nuclear Tulaic" + }, + "dbe": { + "level0": "Tor-Orya", + "level1": "Tor", + "level2": "Coastal Tor" + }, + "dbf": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "Central Tariku" + }, + "dbg": { + "level0": "Dogon", + "level1": "North Plateau Dogon" + }, + "dbi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Northern Benue-Congo Plateau", + "level5": "Nuclear Northern Benue-Congo Plateau" + }, + "dbj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Northeast Sabahan" + }, + "dbl": { + "level0": "Pama-Nyungan" + }, + "dbm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan", + "level7": "Jarawaic" + }, + "dbn": { + "level0": "Inanwatan" + }, + "dbo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan", + "level7": "Jaku-Gubi" + }, + "dbp": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.1" + }, + "dbq": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Dabaic", + "level5": "Daba-Mazagway-Kola" + }, + "dbr": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Eastern Omo-Tana", + "level8": "Dabarre-Tunni" + }, + "dbt": { + "level0": "Dogon", + "level1": "Nangan Dogon" + }, + "dbu": { + "level0": "Dogon", + "level1": "North Plateau Dogon", + "level2": "Yanda-Bondum-Tebul" + }, + "dbv": { + "level0": "Unattested", + "level1": "Atlantic-Congo (Unattested)" + }, + "dbw": { + "level0": "Dogon", + "level1": "Nangan Dogon" + }, + "dcr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch", + "level6": "Middle-Modern Dutch", + "level7": "Modern Dutch", + "level8": "Southwestern Dutch", + "level9": "Zeeuwic" + }, + "ddd": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Lotuxo-Maa", + "level4": "Lotuxo", + "level5": "Lopit-Dongotono", + "level6": "Dongotonic" + }, + "dde": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Nuclear Northern Kikongo" + }, + "ddg": { + "level0": "Timor-Alor-Pantar", + "level1": "East Timor", + "level2": "Fataluku-Oirata" + }, + "ddi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Bwaidoga linkage" + }, + "ddj": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Ngumpin-Yapa", + "level3": "Ngumpin", + "level4": "Western Ngumpin" + }, + "ddn": { + "level0": "Songhay", + "level1": "Eastern Songhay", + "level2": "Zarma-Kaado-Dendi" + }, + "ddo": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Tsezic", + "level4": "West Tsezic" + }, + "ddr": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "Victorian Pama-Nyungan", + "level3": "Eastern Victoria", + "level4": "Dhudhuroa-Pallanganmiddang" + }, + "dds": { + "level0": "Dogon", + "level1": "Escarpment Dogon" + }, + "ddw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar", + "level5": "North Babaric" + }, + "dec": { + "level0": "Narrow Talodi", + "level1": "Buram-Saraf", + "level2": "Buram Hill Chain" + }, + "ded": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Rawlinson", + "level5": "Pindiu" + }, + "dee": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Bassa-Klao", + "level5": "Bassaic" + }, + "def": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Luric-Dezfulic" + }, + "deg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Delta Edoid", + "level6": "Degema-Engenni" + }, + "deh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Farsic", + "level9": "Eastern Farsic" + }, + "dei": { + "level0": "Geelvink Bay" + }, + "dek": { + "level0": "Unattested", + "level1": "Atlantic-Congo (Unattested)" + }, + "dep": { + "level0": "Pidgin", + "level1": "Delaware-based pidgin" + }, + "deq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Ngbandi-Mongoba-Kazibati", + "level6": "Ngbandic", + "level7": "Nuclear Ngbandic" + }, + "der": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo" + }, + "des": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Western Eastern Tucanoan", + "level3": "Cubeo-Desano", + "level4": "Yupua-Siriano-Desano", + "level5": "Siriano-Desano" + }, + "deu": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Upper Franconian", + "level10": "Global German" + }, + "dev": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Yupna", + "level4": "Bwana-Moam-Tapen" + }, + "dez": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Nkutsuic", + "level11": "Nkutsu-Lokenye" + }, + "dga": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Northwest Oti-Volta", + "level13": "Safaliba-Dagaare", + "level14": "Dagaaric", + "level15": "Central-South Dagaric" + }, + "dgb": { + "level0": "Dogon", + "level1": "West Dogon", + "level2": "Penangic" + }, + "dgc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Northeastern Luzon", + "level4": "Nuclear Northeastern Luzon" + }, + "dgd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Northwest Oti-Volta", + "level13": "Safaliba-Dagaare", + "level14": "Dagaaric", + "level15": "Central-South Dagaric", + "level16": "South Dagaric" + }, + "dge": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Warup", + "level4": "Nuclear Warup", + "level5": "Degenanic" + }, + "dgg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Are linkage", + "level10": "Are-Doga" + }, + "dgh": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mandaraic", + "level6": "Dghwedeic" + }, + "dgi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Northwest Oti-Volta", + "level13": "Safaliba-Dagaare", + "level14": "Dagaaric", + "level15": "North-West Dagaric" + }, + "dgk": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Logone-Chari", + "level7": "Sido" + }, + "dgn": { + "level0": "Yangmanic" + }, + "dgo": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Kangric-Chamealic-Bhattiyali", + "level8": "Kangri-Dogri" + }, + "dgr": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Northwestern Canada Athabaskan", + "level4": "Slaveyic" + }, + "dgs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Dogoso-Khe" + }, + "dgu": { + "level0": "Bookkeeping" + }, + "dgx": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean", + "level3": "Nuclear Binanderean", + "level4": "South Binanderean", + "level5": "Coastal Binanderean", + "level6": "Baruga-Doghoro" + }, + "dgz": { + "level0": "Dagan", + "level1": "Central Dagan", + "level2": "Southwest Dagan" + }, + "dha": { + "level0": "Bookkeeping" + }, + "dhd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Eastern Rajasthani" + }, + "dhg": { + "level0": "Pama-Nyungan", + "level1": "Yuulngu", + "level2": "Southern Yolngu", + "level3": "Southern-Eastern Yolngu" + }, + "dhi": { + "level0": "Sino-Tibetan", + "level1": "Dhimal-Lhokpu-Toto" + }, + "dhl": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Kanyara", + "level4": "Thalanyji-Burduna" + }, + "dhm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia", + "level11": "Herero (R.30)" + }, + "dhn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Khandesic" + }, + "dho": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "dhr": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Mantharta", + "level4": "Wariyangga-Dhargari" + }, + "dhs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Central Kenya Bantu", + "level9": "Kamba-Dhaisu" + }, + "dhu": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "New South Wales Pama-Nyungan", + "level3": "Yuin-Kuri", + "level4": "Yuin", + "level5": "Northern Costal Yuin" + }, + "dhv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Loyalty Islands" + }, + "dhw": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Unclassified Bihari", + "level10": "Kuswaric" + }, + "dia": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Wapeic", + "level4": "Galu-Alu" + }, + "dib": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Dinka-Nuer", + "level3": "Dinka" + }, + "dic": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Neyo-Dida", + "level3": "Dida", + "level4": "Guebie-Lakota Dida" + }, + "did": { + "level0": "Surmic", + "level1": "South Surmic", + "level2": "Southwest Surmic", + "level3": "Didinga-Murle", + "level4": "Didinga-Longarim" + }, + "dif": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Central Karnic", + "level3": "Western Central Karnic", + "level4": "Pirlatapa-Dieric", + "level5": "Dieric" + }, + "dig": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Mijikenda-Pokomo-Comorian", + "level11": "Mijikenda", + "level12": "Southern Mijikenda" + }, + "dih": { + "level0": "Cochimi-Yuman", + "level1": "Yuman", + "level2": "General Yuman", + "level3": "Delta-Californian Yuman", + "level4": "Diegueno" + }, + "dii": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Bafia (A.50)", + "level8": "Nuclear Bafia (A.50)" + }, + "dij": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar", + "level5": "North Babaric" + }, + "dik": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Dinka-Nuer", + "level3": "Dinka" + }, + "dil": { + "level0": "Nubian", + "level1": "Central Nubian", + "level2": "Kordofan Nubian", + "level3": "Western Kordofan Nubian" + }, + "dim": { + "level0": "South Omotic" + }, + "dio": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Ebira-Nupoid", + "level4": "Macro-Nupoid", + "level5": "Nupoid", + "level6": "Dibo-Kupa", + "level7": "Abawa" + }, + "dip": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Dinka-Nuer", + "level3": "Dinka" + }, + "diq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Zaza" + }, + "dir": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Dakoid", + "level6": "Taram-Dirim-Nnakenyare", + "level7": "Dirim-Nnakenyare" + }, + "dis": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Boroic", + "level4": "Dimasa-Kokborok" + }, + "diu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Kwangali-Diriku" + }, + "div": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Dhivehi-Sinhala" + }, + "diw": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Dinka-Nuer", + "level3": "Dinka" + }, + "dix": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Peripheral Western Malakula", + "level9": "Southwestern Malakula" + }, + "diy": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Asmat-Kamoro", + "level3": "Asmat", + "level4": "Citak Asmat" + }, + "diz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic" + }, + "djb": { + "level0": "Pama-Nyungan", + "level1": "Yuulngu", + "level2": "Western-Inland Yolngu" + }, + "djc": { + "level0": "Dajuic", + "level1": "Western Dajuic" + }, + "djd": { + "level0": "Mirndi", + "level1": "Yirram" + }, + "dje": { + "level0": "Songhay", + "level1": "Eastern Songhay", + "level2": "Zarma-Kaado-Dendi" + }, + "djf": { + "level0": "Pama-Nyungan", + "level1": "Yimidhirr-Yalanji-Yidinic", + "level2": "Yalandyic" + }, + "dji": { + "level0": "Pama-Nyungan", + "level1": "Yuulngu", + "level2": "Western-Inland Yolngu" + }, + "djj": { + "level0": "Maningrida", + "level1": "Nakkara-Ndjebbana" + }, + "djk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Surinamese Creole English", + "level13": "Eastern Maroons", + "level14": "Ndyuka" + }, + "djm": { + "level0": "Dogon", + "level1": "Plains Dogon" + }, + "djn": { + "level0": "Gunwinyguan", + "level1": "Western Gunwinyguan" + }, + "djo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Bidayuh-Southern Land Dayak", + "level4": "Southern Land Dayak" + }, + "djr": { + "level0": "Pama-Nyungan", + "level1": "Yuulngu", + "level2": "Southern Yolngu", + "level3": "Southern-Eastern Yolngu", + "level4": "Dhuwal-Dhuwala", + "level5": "Eastern Dhuwal-Dhuwala" + }, + "dju": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Central Sepik Hill", + "level3": "Nuclear Central Sepik Hill", + "level4": "Kapriman-Watakataui" + }, + "djw": { + "level0": "Nyulnyulan", + "level1": "Western Nyulnyulan", + "level2": "Bardic" + }, + "dka": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Dakpa-Dzala" + }, + "dkg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Tarokoid", + "level5": "Bijimic-Sur-Shall", + "level6": "Kwangic", + "level7": "Vaghat" + }, + "dkk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Pitu Ulunna Salu" + }, + "dkl": { + "level0": "Bookkeeping" + }, + "dkr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Dusunic" + }, + "dks": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Dinka-Nuer", + "level3": "Dinka" + }, + "dkx": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Dabaic", + "level5": "Daba-Mazagway-Kola" + }, + "dlg": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Sakha-Dolgan" + }, + "dlk": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "Tigre-Dahalik" + }, + "dlm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Italo-Dalmatian", + "level9": "Dalmatian Romance" + }, + "dln": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin", + "level5": "Mizoic" + }, + "dma": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie", + "level17": "Northwest Kasai-Ngounie", + "level18": "Nzebi-Laali-Yaa", + "level19": "Njebi (B.50)", + "level20": "Ndjavi B" + }, + "dmb": { + "level0": "Dogon", + "level1": "West Dogon", + "level2": "Penangic" + }, + "dmc": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert" + }, + "dme": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mofuic", + "level6": "Meri", + "level7": "Dugwor-Merey" + }, + "dmf": { + "level0": "Speech Register", + "level1": "Atlantic-Congo Speech Register" + }, + "dmg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Paitanic", + "level7": "Upper Kinabatangan-Lobu" + }, + "dmk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone" + }, + "dml": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan" + }, + "dmm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Northern Mbum", + "level6": "Dama-Galke" + }, + "dmo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Beboid", + "level6": "Eastern Beboid", + "level7": "Bebe-Kemezung", + "level8": "Naki-Kemezung" + }, + "dmr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku" + }, + "dms": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Tominic", + "level5": "Southern Tomini" + }, + "dmu": { + "level0": "Pauwasi", + "level1": "Western Pauwasi" + }, + "dmv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Dusunic", + "level7": "Rungus-Mangkaak-Labuk", + "level8": "Dumpas-Sukang" + }, + "dmw": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Ngumpin-Yapa", + "level3": "Ngumpin", + "level4": "Eastern Ngumpin" + }, + "dmx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Shona (S.10)", + "level9": "Unclassified Shona (S. 10)" + }, + "dmy": { + "level0": "Sentanic" + }, + "dna": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani", + "level2": "Central Dani", + "level3": "Grand Valley Dani", + "level4": "Walakic" + }, + "dnd": { + "level0": "Border", + "level1": "Warisic", + "level2": "Nuclear Warisic", + "level3": "Simog-Daonda" + }, + "dne": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Rufijic", + "level9": "Matengic", + "level10": "Ndendeule-Ngindo" + }, + "dng": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic", + "level4": "Northern Chinese", + "level5": "Mandarinic", + "level6": "Zhongyuan" + }, + "dni": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani", + "level2": "Central Dani", + "level3": "Grand Valley Dani", + "level4": "Southeast Grand Valley Dani" + }, + "dnk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Rote-Meto", + "level4": "West Rote", + "level5": "Dengka-Meto" + }, + "dnn": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Duun-Bobo", + "level4": "Duun-Jo", + "level5": "Duun-Seenku", + "level6": "Duun" + }, + "dno": { + "level0": "Central Sudanic", + "level1": "Lenduic", + "level2": "Bale" + }, + "dnr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Peka", + "level4": "Urigina-Danaru" + }, + "dnt": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani", + "level2": "Central Dani", + "level3": "Grand Valley Dani" + }, + "dnu": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic" + }, + "dnw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani", + "level2": "Central Dani" + }, + "dny": { + "level0": "Arawan", + "level1": "Madi-Madiha", + "level2": "Madiha" + }, + "doa": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Simbu", + "level3": "Nuclear Simbu", + "level4": "Kuman-Dom-Gunaa" + }, + "dob": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Dobu-Duau linkage" + }, + "doc": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Kam-Sui", + "level3": "Mulam-Kam", + "level4": "Kamic", + "level5": "Northern Kam" + }, + "doe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "East Ruvu" + }, + "dof": { + "level0": "Mailuan" + }, + "doh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Dakoid", + "level6": "Tiba-Dong" + }, + "dok": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Tominic", + "level5": "Northern Tomini" + }, + "dol": { + "level0": "Doso-Turumsa" + }, + "don": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "West Central Papuan linkage", + "level9": "Nuclear West Central Papuan linkage" + }, + "doo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mbaic", + "level6": "Ndunga-Mba-Dongo" + }, + "dop": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Eastern Grusi", + "level9": "Kabiyeic" + }, + "doq": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "American Sign" + }, + "dor": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Southern Malaita" + }, + "dos": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Kaansa-Dogose", + "level7": "Dogose-Khisa" + }, + "dot": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Southwest South Bauchi" + }, + "dov": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Botatwe", + "level8": "Greater Eastern Botatwe", + "level9": "Central Eastern Botatwe" + }, + "dow": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Northern Samba-Duru" + }, + "dox": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Oromoid", + "level7": "Konsoid", + "level8": "Gidole-Bussa" + }, + "doy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Gonja-Dompo" + }, + "doz": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "North-West Ometo", + "level3": "Central Ometo" + }, + "dpp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic" + }, + "drb": { + "level0": "Nubian", + "level1": "Central Nubian", + "level2": "Kordofan Nubian", + "level3": "Eastern Kordofan Nubian" + }, + "drc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance" + }, + "drd": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Eastern West Himalayish", + "level4": "Pithauragarh", + "level5": "Darma-Byangsi-Chaudangsi", + "level6": "Darma-Byangsi", + "level7": "Zhangzhungic" + }, + "dre": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Dolpo-Tichurong" + }, + "drg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Dusunic", + "level7": "Rungus-Mangkaak-Labuk" + }, + "dri": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Northwestern Kainji", + "level6": "Clela-Damakawa" + }, + "drl": { + "level0": "Pama-Nyungan", + "level1": "Yarli-Baagandji" + }, + "drn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar" + }, + "dro": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Melanau-Kajang", + "level5": "Melanau" + }, + "drq": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kham-Magar-Chepang" + }, + "drr": { + "level0": "Bookkeeping" + }, + "drs": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Highland East Cushitic", + "level4": "Sidaama-Hadiyya-Kambaata", + "level5": "Sidaama-Gedeo" + }, + "dru": { + "level0": "Austronesian" + }, + "dry": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Unclassified Bihari", + "level10": "Kuswaric" + }, + "dsb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "West Slavic", + "level5": "Sorbian" + }, + "dse": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Dutch-Belgian Sign" + }, + "dsh": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Western Omo-Tana" + }, + "dsi": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Peripherique", + "level6": "Koulfaic" + }, + "dsk": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Southwest South Bauchi", + "level7": "Zeemic" + }, + "dsl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "West Scandinavian Sign", + "level4": "Danish Sign" + }, + "dsn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Biakic" + }, + "dso": { + "level0": "Bookkeeping" + }, + "dsq": { + "level0": "Songhay", + "level1": "Northwest Songhay", + "level2": "Northern Songhay" + }, + "dsz": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "dta": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic" + }, + "dtb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Dusunic", + "level7": "Rungus-Mangkaak-Labuk", + "level8": "Dumpas-Sukang" + }, + "dtd": { + "level0": "Wakashan", + "level1": "Southern Wakashan", + "level2": "Makah-Nitinat" + }, + "dti": { + "level0": "Dogon", + "level1": "North Plateau Dogon", + "level2": "Yanda-Bondum-Tebul", + "level3": "Yanda-Ana" + }, + "dtk": { + "level0": "Dogon", + "level1": "Plains Dogon", + "level2": "Western Plains Dogon" + }, + "dtm": { + "level0": "Dogon", + "level1": "Plains Dogon", + "level2": "Western Plains Dogon" + }, + "dtn": { + "level0": "Gumuz" + }, + "dto": { + "level0": "Dogon", + "level1": "Escarpment Dogon" + }, + "dtp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Dusunic", + "level7": "Kadazan-Sugut-Minokok" + }, + "dtr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Bisaya-Lotud" + }, + "dts": { + "level0": "Dogon", + "level1": "Escarpment Dogon" + }, + "dtt": { + "level0": "Dogon", + "level1": "Plains Dogon" + }, + "dtu": { + "level0": "Dogon", + "level1": "North Plateau Dogon", + "level2": "Yanda-Bondum-Tebul" + }, + "dty": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Indo-Aryan Northern zone", + "level8": "Eastern Pahari" + }, + "dua": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Sawabantu", + "level8": "Dualaic", + "level9": "Duala-Malimba" + }, + "dub": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Gujaratic" + }, + "dud": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Northwestern Kainji", + "level6": "Dukaic" + }, + "due": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine" + }, + "duf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Southern New Caledonian", + "level8": "Extreme Southern New Caledonian" + }, + "dug": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Mijikenda-Pokomo-Comorian", + "level11": "Mijikenda", + "level12": "Northern Mijikenda" + }, + "duh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "dui": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Greater Yaganon", + "level4": "Yaganon" + }, + "duj": { + "level0": "Pama-Nyungan", + "level1": "Yuulngu", + "level2": "Southern Yolngu", + "level3": "Southern-Eastern Yolngu", + "level4": "Dhuwal-Dhuwala", + "level5": "Western Dhuwal-Dhuwala" + }, + "duk": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Nuru" + }, + "dul": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Alabat-Manide Agta" + }, + "dum": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch", + "level6": "Middle-Modern Dutch" + }, + "dun": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage" + }, + "duo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Northeastern Luzon" + }, + "dup": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic" + }, + "duq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage" + }, + "dur": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Southern Samba-Duru", + "level7": "Diic" + }, + "dus": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Upper Dudhkosi" + }, + "duu": { + "level0": "Sino-Tibetan", + "level1": "Nungish", + "level2": "Gunong" + }, + "duv": { + "level0": "Lakes Plain", + "level1": "Tariku" + }, + "duw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito" + }, + "dux": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Duun-Bobo", + "level4": "Duun-Jo", + "level5": "Duun-Seenku", + "level6": "Duun" + }, + "duy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Unclassified Northern Luzon" + }, + "duz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Unclassified Central Adamawa" + }, + "dva": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Dobu-Duau linkage" + }, + "dwa": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.2" + }, + "dwk": { + "level0": "Bookkeeping", + "level1": "Pending Report Release" + }, + "dwl": { + "level0": "Bookkeeping" + }, + "dwr": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "North-West Ometo", + "level3": "Central Ometo", + "level4": "Dawro-Gofa-Gamo" + }, + "dws": { + "level0": "Artificial Language" + }, + "dww": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Kakabai linkage" + }, + "dwz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Unclassified Bihari", + "level10": "Kuswaric" + }, + "dya": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Lobiri-Jaane" + }, + "dyb": { + "level0": "Nyulnyulan", + "level1": "Western Nyulnyulan", + "level2": "Nyulnyulic" + }, + "dyd": { + "level0": "Nyulnyulan", + "level1": "Eastern Nyulnyulan", + "level2": "Yawuric" + }, + "dyg": { + "level0": "Unattested", + "level1": "Austronesian (Unattested)" + }, + "dyi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "South Senufo", + "level5": "Tagbana-Jimini" + }, + "dyk": { + "level0": "Bookkeeping" + }, + "dym": { + "level0": "Dogon", + "level1": "North Plateau Dogon", + "level2": "Yanda-Bondum-Tebul", + "level3": "Yanda-Ana" + }, + "dyn": { + "level0": "Pama-Nyungan", + "level1": "Macleay-New England" + }, + "dyo": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Jola" + }, + "dyr": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Southwest South Bauchi", + "level7": "Zakse-Saya" + }, + "dyu": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding" + }, + "dyy": { + "level0": "Pama-Nyungan", + "level1": "Yimidhirr-Yalanji-Yidinic", + "level2": "Yidinic" + }, + "dza": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "North-Central Jos" + }, + "dzd": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Nuclear Boleic", + "level8": "Galambu-Bele", + "level9": "Kirfi-Bele", + "level10": "Giiwo-Daza" + }, + "dze": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Mantharta", + "level4": "Djiwarli-Thiin" + }, + "dzg": { + "level0": "Saharan", + "level1": "Western Saharan", + "level2": "Tebu" + }, + "dzl": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Dakpa-Dzala" + }, + "dzn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Bamweic" + }, + "dzo": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Southern Tibetic", + "level7": "Dzongkhic", + "level8": "Nuclear Dzongkhic" + }, + "ebg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "West Lower Cross", + "level7": "Oroic", + "level8": "Ebughu-Oro" + }, + "ebo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Ngungwel-Eboo" + }, + "ebr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Potou" + }, + "ebu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Central Kenya Bantu" + }, + "ecr": { + "level0": "Unclassifiable" + }, + "ecs": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "West-Central South American Sign" + }, + "ecy": { + "level0": "Unclassifiable" + }, + "eee": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Northern Tai", + "level7": "Unclassified Northern Tai" + }, + "efa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross" + }, + "efe": { + "level0": "Central Sudanic", + "level1": "Membi-Mangbutu-Efe", + "level2": "Mangbutu-Efe", + "level3": "Leseic" + }, + "efi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Efikic", + "level8": "Okop Usem", + "level9": "Efik-Ibibio" + }, + "ega": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo" + }, + "egl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Italian", + "level12": "Emiliano-Romagnolo" + }, + "ego": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Jilic-Eggonic", + "level5": "Eggon-Ake" + }, + "egy": { + "level0": "Afro-Asiatic", + "level1": "Egyptian" + }, + "ehs": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "ehu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Osse", + "level7": "Ukue-Ehueun" + }, + "eip": { + "level0": "Nuclear Trans New Guinea", + "level1": "Mek", + "level2": "Eastern Mek" + }, + "eit": { + "level0": "Nuclear Torricelli", + "level1": "Kombio-Arapesh-Urat", + "level2": "Kombio-Yambes", + "level3": "Kombioic" + }, + "eiv": { + "level0": "North Bougainville", + "level1": "Rotokas-Askopan" + }, + "eja": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Jola", + "level6": "FH-Jola", + "level7": "PF-Jola", + "level8": "Her-Ejamat" + }, + "eka": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Ekoid-Mbe", + "level6": "Ekoid", + "level7": "Bakor-Ejagham", + "level8": "Bakor", + "level9": "Northern Bakor", + "level10": "Nnam-Ekajuk" + }, + "eke": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Ekit-Etebi" + }, + "ekg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Paniai Lakes", + "level2": "Mee-Wodani" + }, + "eki": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Efikic", + "level8": "Unclassified Efikic" + }, + "ekk": { + "level0": "Uralic", + "level1": "Finnic", + "level2": "Coastal Finnic", + "level3": "Neva", + "level4": "Central Finnic" + }, + "ekl": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Santalic" + }, + "ekm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Bati-Mbure-Yambassa", + "level10": "Mbure-Yambassa", + "level11": "Yambassa (A.60)", + "level12": "Mmala-Elip-Gunu", + "level13": "Elip-Gunu" + }, + "eko": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Koti-Nathembo" + }, + "ekp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Igboid" + }, + "ekr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Idomoid", + "level4": "Yatye-Akpa" + }, + "eky": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Central Karen", + "level3": "Kayah-Yintale", + "level4": "Kayah" + }, + "ele": { + "level0": "Nuclear Torricelli", + "level1": "Marienberg", + "level2": "Elepi-Kamasau-Marienberg" + }, + "elh": { + "level0": "Nubian", + "level1": "Central Nubian", + "level2": "Kordofan Nubian", + "level3": "Western Kordofan Nubian" + }, + "eli": { + "level0": "Narrow Talodi", + "level1": "Buram-Saraf", + "level2": "Nding-Tasomi" + }, + "elk": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Wapeic", + "level4": "Au-Olo-Elkei", + "level5": "Olo-Elkei" + }, + "ell": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Graeco-Phrygian", + "level3": "Greek", + "level4": "South Greek", + "level5": "Central Greek", + "level6": "Koineic Greek", + "level7": "Modern Koineic Greek", + "level8": "Nuclear Modern Greek" + }, + "elm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Ogonoid", + "level5": "West Ogonoid" + }, + "elo": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Western Omo-Tana" + }, + "elp": { + "level0": "Bookkeeping" + }, + "elu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "East Manus", + "level8": "Kurti-Kele-Ere", + "level9": "Kurti-Elu" + }, + "ema": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "North-Central Edoid", + "level6": "Central Plains Edoid", + "level7": "Emaic" + }, + "emb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Tamanic-Bugis", + "level5": "Tamanic" + }, + "eme": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VIII", + "level6": "Wayampi-Zoe-Emerillon", + "level7": "Zoe-Emerillon" + }, + "emg": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Upper Arun", + "level6": "Mewahang" + }, + "emi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "St. Matthias" + }, + "emk": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding", + "level9": "Manenkan" + }, + "emn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "Central Tivoid", + "level7": "Central Tivoid B" + }, + "emo": { + "level0": "Bookkeeping" + }, + "emp": { + "level0": "Chocoan", + "level1": "Embera", + "level2": "Atrato", + "level3": "Panama-Baudo-Atrato" + }, + "emq": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic", + "level4": "Muya" + }, + "ems": { + "level0": "Eskimo-Aleut", + "level1": "Eskimo", + "level2": "Yupik" + }, + "emu": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Gondi", + "level4": "Northwest Gondi", + "level5": "Southwest Gondi", + "level6": "Muria" + }, + "emw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar", + "level5": "South Babar", + "level6": "Southwest Babar" + }, + "emx": { + "level0": "Speech Register", + "level1": "Basque-Romani" + }, + "emy": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Cholan-Tzeltalan", + "level4": "Cholan" + }, + "emz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "Center Ring", + "level10": "Komic" + }, + "ena": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Sogeram", + "level5": "Apalic" + }, + "enb": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Kalenjin", + "level3": "Northern Kalenjin" + }, + "enc": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Eastern Kra", + "level3": "Buyang", + "level4": "Northern Buyang" + }, + "end": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Central Flores-Paluqe", + "level6": "Central Flores", + "level7": "Eastern Central Flores", + "level8": "Ende-Lio" + }, + "enf": { + "level0": "Uralic", + "level1": "Samoyedic", + "level2": "Enets-Nenets", + "level3": "Enets" + }, + "eng": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English" + }, + "enh": { + "level0": "Uralic", + "level1": "Samoyedic", + "level2": "Enets-Nenets", + "level3": "Enets" + }, + "enl": { + "level0": "Lengua-Mascoy", + "level1": "Lengua" + }, + "enm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English" + }, + "enn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Delta Edoid", + "level6": "Degema-Engenni" + }, + "eno": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran" + }, + "enq": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Engan" + }, + "enr": { + "level0": "Pauwasi", + "level1": "Eastern Pauwasi" + }, + "enu": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Hanic", + "level7": "Bi-Ka" + }, + "env": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Igwic", + "level7": "Ikpeshic" + }, + "enw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "West Lower Cross", + "level7": "Oroic", + "level8": "Enwang-Uda" + }, + "enx": { + "level0": "Lengua-Mascoy", + "level1": "Lengua" + }, + "eot": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Western Tano" + }, + "epi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Delta Edoid" + }, + "epo": { + "level0": "Artificial Language", + "level1": "Esperantic" + }, + "era": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid", + "level10": "Malasa-Eravallan" + }, + "erg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "Southern Vanuatu", + "level6": "Erromanga" + }, + "erh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Southwestern Edoid" + }, + "eri": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Nuru", + "level4": "Erimaic" + }, + "erk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Epi-Efate", + "level7": "Efate", + "level8": "South Efatic" + }, + "ero": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic", + "level4": "Gyalrongic", + "level5": "West Gyalrongic", + "level6": "Horpa" + }, + "err": { + "level0": "Giimbiyu", + "level1": "Urninganggic" + }, + "ers": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Ersuic" + }, + "ert": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "East Tariku", + "level3": "Eritai-Obokuitai-Biritai" + }, + "erw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tanimbar-Bomberai", + "level3": "Nuta" + }, + "ese": { + "level0": "Pano-Tacanan", + "level1": "Tacanan", + "level2": "Takanik-Chamik" + }, + "esg": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Gondi", + "level4": "Northwest Gondi", + "level5": "Southwest Gondi", + "level6": "Southern Gondi", + "level7": "Eastern Gondi" + }, + "esh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Southern Tatic", + "level10": "Ramand-Karaj" + }, + "esi": { + "level0": "Eskimo-Aleut", + "level1": "Eskimo", + "level2": "Inuit", + "level3": "Alaskan Inupiaq" + }, + "esk": { + "level0": "Eskimo-Aleut", + "level1": "Eskimo", + "level2": "Inuit", + "level3": "Alaskan Inupiaq" + }, + "esl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Arab Sign" + }, + "esm": { + "level0": "Unattested", + "level1": "Atlantic-Congo (Unattested)" + }, + "esn": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "eso": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "RSLic" + }, + "ess": { + "level0": "Eskimo-Aleut", + "level1": "Eskimo", + "level2": "Yupik" + }, + "est": { + "level0": "Uralic", + "level1": "Finnic" + }, + "esu": { + "level0": "Eskimo-Aleut", + "level1": "Eskimo", + "level2": "Yupik" + }, + "esy": { + "level0": "Artificial Language" + }, + "etb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Ekit-Etebi" + }, + "eth": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic" + }, + "etn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Epi-Efate", + "level7": "Efate", + "level8": "South Efatic" + }, + "eto": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Basaa-Yaunde (A40-70)", + "level8": "Yaunde-Fang (A.70)" + }, + "etr": { + "level0": "Bosavi", + "level1": "Etoro-Bedamini" + }, + "ets": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "North-Central Edoid", + "level6": "Afenmai-Bendel", + "level7": "Uneme-Yekhee" + }, + "etu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Ekoid-Mbe", + "level6": "Ekoid", + "level7": "Bakor-Ejagham" + }, + "etx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Beromic", + "level5": "Iten-Cara-Berom" + }, + "etz": { + "level0": "Mairasic" + }, + "eud": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Opata-Eudeve" + }, + "eur": { + "level0": "Bookkeeping" + }, + "eve": { + "level0": "Tungusic", + "level1": "Northeastern Tungusic", + "level2": "Northern Tungusic", + "level3": "Ewenic" + }, + "evh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Southwestern Edoid" + }, + "evn": { + "level0": "Tungusic", + "level1": "Northeastern Tungusic", + "level2": "Northern Tungusic" + }, + "ewe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Western Gbe", + "level5": "Eweic" + }, + "ewo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Basaa-Yaunde (A40-70)", + "level8": "Yaunde-Fang (A.70)", + "level9": "Ewondo-Bebele" + }, + "ext": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Castilic" + }, + "eya": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak" + }, + "eyo": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Kalenjin", + "level3": "Central Kalenjin", + "level4": "Plateau Central Kalenjin", + "level5": "Western Plateau Central Kalenjin" + }, + "eze": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "North-South Central Delta Cross", + "level7": "Koring-Kukele", + "level8": "Kukele-Uzekwe" + }, + "fab": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Lower Guinea Portuguese", + "level15": "Bantu Layer Lower Guinea Portuguese" + }, + "fad": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Garuh-Foran" + }, + "faf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Makira" + }, + "fag": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Erap", + "level4": "Finungwan-Mamaa-Gusan" + }, + "fah": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Unclassified Benue-Congo" + }, + "fai": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Mountain Ok", + "level6": "Division A Mountain Ok", + "level7": "Tifal-Telefol", + "level8": "Tifalic", + "level9": "Faiwol-Seltaman" + }, + "faj": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Sogeram", + "level5": "East Sogeram" + }, + "fak": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Yemne-Kimbi" + }, + "fal": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Unclassified Volta-Congo", + "level3": "Adamawa Fali" + }, + "fam": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Unclassified Bantoid" + }, + "fan": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Basaa-Yaunde (A40-70)", + "level8": "Yaunde-Fang (A.70)" + }, + "fao": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "North Germanic", + "level5": "West Scandinavian", + "level6": "Icelandic-Faroese" + }, + "fap": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Cangin", + "level3": "Palor-Ndut" + }, + "far": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Central-Northern Malaita", + "level9": "North Malaitan" + }, + "fas": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Farsic" + }, + "fau": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "West Tariku", + "level3": "Fayu-Kirikiri" + }, + "fax": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance" + }, + "fay": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian" + }, + "faz": { + "level0": "Bookkeeping" + }, + "fcs": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic" + }, + "fer": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Sereic", + "level6": "Feroge-Mangaya" + }, + "ffi": { + "level0": "Bookkeeping" + }, + "ffm": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Fula-Sereer", + "level3": "Fula" + }, + "fia": { + "level0": "Nubian", + "level1": "Nile Nubian", + "level2": "Nobiin Nubian" + }, + "fie": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.4", + "level5": "Fyer-Tambas" + }, + "fif": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Sayhadic", + "level5": "Modern Sayhadic" + }, + "fij": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Eastern Fijian", + "level7": "Nuclear Eastern Fijian", + "level8": "Viwa-Lomaiviti-East Viti Levu" + }, + "fil": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Tagalogic", + "level5": "Tagalog-Filipino" + }, + "fin": { + "level0": "Uralic", + "level1": "Finnic", + "level2": "Coastal Finnic", + "level3": "Neva", + "level4": "North Finnic", + "level5": "Nuclear Finnish" + }, + "fip": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Mwika", + "level10": "Fipaic" + }, + "fir": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Zaric", + "level6": "Nuclear Zaric", + "level7": "Izeric" + }, + "fit": { + "level0": "Uralic", + "level1": "Finnic", + "level2": "Coastal Finnic", + "level3": "Neva", + "level4": "North Finnic", + "level5": "Nuclear Finnish" + }, + "fiw": { + "level0": "East Kutubu" + }, + "fiz": { + "level0": "Bookkeeping" + }, + "fkk": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Higic", + "level5": "Nkafa-Kirya-Bana", + "level6": "Nkafa-Kirya" + }, + "fkv": { + "level0": "Uralic", + "level1": "Finnic", + "level2": "Coastal Finnic", + "level3": "Neva", + "level4": "North Finnic", + "level5": "Nuclear Finnish" + }, + "fla": { + "level0": "Salishan", + "level1": "Interior Salish", + "level2": "Southern Interior Salish", + "level3": "Okanaganic", + "level4": "Kalispel-Spokane" + }, + "flh": { + "level0": "Lakes Plain", + "level1": "East Lakes Plain" + }, + "fli": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic", + "level5": "Gudeic", + "level6": "Gude-Jimi-Zizilivakan", + "level7": "Fali-Gude" + }, + "fll": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Unclassified Volta-Congo", + "level3": "Adamawa Fali" + }, + "flm": { + "level0": "Bookkeeping" + }, + "fln": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Flinders-Barrow" + }, + "flr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "Forest Kivu", + "level12": "Fuliiric", + "level13": "Fuliiru-Vira" + }, + "fly": { + "level0": "Speech Register", + "level1": "Indo-European Speech Register" + }, + "fmp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Bamileke", + "level10": "East Bamileke" + }, + "fmu": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Gondi", + "level4": "Northwest Gondi", + "level5": "Southwest Gondi", + "level6": "Southern Gondi", + "level7": "Eastern Gondi" + }, + "fnb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Ambrym", + "level7": "Orkon-West Ambrym" + }, + "fng": { + "level0": "Pidgin", + "level1": "Zulu-based pidgin" + }, + "fni": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Adamawa Bua", + "level5": "Inland Bua", + "level6": "Goulaic", + "level7": "Zan-Kulaalic", + "level8": "Kulaalic" + }, + "fod": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Oti North Guang", + "level9": "Mountain Oti North Guang", + "level10": "Gikyode-Ginyanga", + "level11": "Gikyode-Foodo" + }, + "foi": { + "level0": "East Kutubu" + }, + "fom": { + "level0": "Bookkeeping" + }, + "fon": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Fongbeic" + }, + "for": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Fore-Gimi" + }, + "fos": { + "level0": "Austronesian", + "level1": "East Formosan" + }, + "fpe": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "West African Creole English" + }, + "fqs": { + "level0": "Baibai-Fas" + }, + "fra": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil", + "level14": "Macro-French", + "level15": "Global French" + }, + "frc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil" + }, + "frd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tanimbar-Bomberai", + "level3": "Nuclear Tanimbar-Bomberai", + "level4": "Kei-Fordata" + }, + "fri": { + "level0": "Bookkeeping" + }, + "fro": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil" + }, + "frp": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Francoprovencalic" + }, + "frq": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Warup", + "level4": "Nuclear Warup" + }, + "frr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Frisian" + }, + "frs": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Alts\u00e4chsisch", + "level7": "Middle-Modern Low German", + "level8": "Low German", + "level9": "West Low German", + "level10": "North Low Saxon" + }, + "frt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo" + }, + "fry": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Frisian", + "level8": "Modern West Frisian", + "level9": "Westlauwers-Terschelling Frisian" + }, + "fse": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Swedish Sign", + "level3": "Finnish Sign" + }, + "fsl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic" + }, + "fss": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Swedish Sign", + "level3": "Finnish Sign" + }, + "fub": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Fula-Sereer", + "level3": "Fula", + "level4": "Eastern Fula", + "level5": "Adamawa-Bagirmi Fulfulde" + }, + "fuc": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Fula-Sereer", + "level3": "Fula" + }, + "fud": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian" + }, + "fue": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Fula-Sereer", + "level3": "Fula" + }, + "fuf": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Fula-Sereer", + "level3": "Fula" + }, + "fuh": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Fula-Sereer", + "level3": "Fula" + }, + "fui": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Fula-Sereer", + "level3": "Fula", + "level4": "Eastern Fula", + "level5": "Adamawa-Bagirmi Fulfulde" + }, + "fuj": { + "level0": "Heibanic", + "level1": "Eastern Heibanic" + }, + "fum": { + "level0": "Bookkeeping" + }, + "fuq": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Fula-Sereer", + "level3": "Fula", + "level4": "Eastern Fula" + }, + "fur": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian" + }, + "fut": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Vanuatu-Loyalty Outliers", + "level9": "Mele-Futuna" + }, + "fuu": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Peripherique", + "level6": "Barh Keita" + }, + "fuv": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Fula-Sereer", + "level3": "Fula", + "level4": "Eastern Fula" + }, + "fvr": { + "level0": "Furan" + }, + "fwa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Nmi-Pij-Fwa-Pam-Pap", + "level10": "Nmi-Fij-Fwa", + "level11": "Hyenghene" + }, + "fwe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Botatwe", + "level8": "Western Botatwe", + "level9": "Zambezi Hook" + }, + "gaa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Ga-Dangme" + }, + "gab": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.2", + "level5": "East Chadic A.2 2", + "level6": "Gabri-Kimre" + }, + "gad": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Cagayan Valley", + "level4": "Ibanagic", + "level5": "Gaddangic", + "level6": "Cagayan-Baliwon Gaddang" + }, + "gae": { + "level0": "Arawakan", + "level1": "Alto Orinoco", + "level2": "Parenic" + }, + "gaf": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka" + }, + "gag": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Oghuz", + "level3": "Nuclear Oghuz", + "level4": "West Oghuz" + }, + "gah": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Gahuku" + }, + "gai": { + "level0": "Ramu", + "level1": "Lower Ramu", + "level2": "Ottilien" + }, + "gaj": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Kainantu", + "level3": "Gauwa", + "level4": "Gadsup-Agarabi" + }, + "gak": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Sahuan", + "level3": "Nuclear Sahuan" + }, + "gal": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Wetar-Atauro" + }, + "gam": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Jimi", + "level3": "Kandawo-Narak" + }, + "gan": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic" + }, + "gao": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Sogeram", + "level5": "East Sogeram" + }, + "gap": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Garuh-Foran" + }, + "gaq": { + "level0": "Austroasiatic", + "level1": "Mundaic" + }, + "gar": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Dobu-Duau linkage", + "level9": "Boselewa-Galeya" + }, + "gas": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil", + "level8": "Garasia Bhil" + }, + "gat": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Unclassified Kainantu-Goroka" + }, + "gau": { + "level0": "Dravidian", + "level1": "Central Dravidian", + "level2": "Parji-Ollari-Gadaba", + "level3": "Ollari-Gadaba" + }, + "gav": { + "level0": "Bookkeeping" + }, + "gaw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Garuh-Foran" + }, + "gax": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Oromoid", + "level7": "Nuclear Oromo", + "level8": "Central-Eastern Oromo", + "level9": "Central Oromo" + }, + "gay": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran" + }, + "gaz": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Oromoid", + "level7": "Nuclear Oromo" + }, + "gbb": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Arandic" + }, + "gbd": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Marrngu" + }, + "gbe": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Western Sepik Hill", + "level3": "Hewa-April River" + }, + "gbf": { + "level0": "Ndu", + "level1": "Nuclear Ndu", + "level2": "Sawosic", + "level3": "Burui-Gaikundi" + }, + "gbg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Western Mundu-Baka", + "level7": "River Western Mundu-Baka" + }, + "gbh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Fongbeic" + }, + "gbi": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Mainland North Halmaheran", + "level3": "Galela-Loloda" + }, + "gbj": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "Gutob-Remo" + }, + "gbk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Kangric-Chamealic-Bhattiyali", + "level8": "Chamealic" + }, + "gbl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "gbm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Indo-Aryan Northern zone", + "level8": "Central Pahari" + }, + "gbn": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "Baka-Beli", + "level3": "Morokodo-Beli", + "level4": "Gberi-Morokodo-Mittu" + }, + "gbo": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Grebo-Aizi", + "level4": "Grebo", + "level5": "Liberian Grebo", + "level6": "North-Central Liberian Grebo" + }, + "gbp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Meridional-Occidental", + "level5": "Bokoto-Gbeya", + "level6": "Gbeya", + "level7": "Gbeya-Suma" + }, + "gbq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Meridional-Occidental", + "level5": "Bokoto-Gbeya", + "level6": "Gbeya" + }, + "gbr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Ebira-Nupoid", + "level4": "Macro-Nupoid", + "level5": "Gbagyi-Gbari" + }, + "gbs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Western Phla-Phera" + }, + "gbv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Oriental", + "level5": "Gbanu-Manza-Ngbaka" + }, + "gbw": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "North Coast Pama-Nyungan", + "level3": "Waka-Kabic", + "level4": "Eastern Waka-Kabic" + }, + "gbx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Eastern Phla-Phera" + }, + "gby": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Ebira-Nupoid", + "level4": "Macro-Nupoid", + "level5": "Gbagyi-Gbari" + }, + "gbz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Central Iran Kermanic", + "level8": "Nuclear Central Iran Kermanic", + "level9": "Yazdi-Kermani-Nayini" + }, + "gcc": { + "level0": "Baining" + }, + "gcd": { + "level0": "Tangkic", + "level1": "Southern Tangkic" + }, + "gce": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Pacific Coast Athabaskan", + "level4": "Oregon Athabaskan", + "level5": "Rogue River" + }, + "gcf": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil", + "level14": "Macro-French", + "level15": "Circum-Caribbean French", + "level16": "Lesser Antillean French Creole" + }, + "gcl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Vincent-Grenadian Creole", + "level15": "Grenada-Tobago Creole" + }, + "gcn": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean", + "level3": "Nuclear Binanderean", + "level4": "South Binanderean", + "level5": "Coastal Binanderean", + "level6": "Gaena-Korafe" + }, + "gcr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil", + "level14": "Macro-French", + "level15": "Circum-Caribbean French", + "level16": "Guyanic Creole French" + }, + "gct": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Alemannic", + "level10": "North Alemannic" + }, + "gda": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Mewaric" + }, + "gdb": { + "level0": "Dravidian", + "level1": "Central Dravidian", + "level2": "Parji-Ollari-Gadaba", + "level3": "Ollari-Gadaba" + }, + "gdc": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Northern Maric", + "level5": "Warungu-Gugu Badhun" + }, + "gdd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Bel-Roinji-Nenaya", + "level9": "Bel", + "level10": "Western Bel" + }, + "gde": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic", + "level5": "Gudeic", + "level6": "Gude-Jimi-Zizilivakan", + "level7": "Fali-Gude" + }, + "gdf": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mandaraic", + "level6": "Dghwedeic", + "level7": "Gudufic" + }, + "gdg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Cagayan Valley", + "level4": "Ibanagic", + "level5": "Gaddangic", + "level6": "Cagayan-Baliwon Gaddang" + }, + "gdh": { + "level0": "Jarrakan", + "level1": "Miriwunic" + }, + "gdi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Western Mundu-Baka", + "level7": "Baka-Gundi" + }, + "gdj": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Norman Pama", + "level3": "Kuthant-Gurdjar", + "level4": "Rib-Gurdjar" + }, + "gdk": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.1" + }, + "gdl": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Oromoid", + "level7": "Konsoid", + "level8": "Gidole-Bussa" + }, + "gdn": { + "level0": "Dagan" + }, + "gdo": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Andic", + "level4": "Botlikh-Godoberi" + }, + "gdq": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Modern South Arabian", + "level4": "Hobyot-Western MSA", + "level5": "Western MSA" + }, + "gdr": { + "level0": "Eastern Trans-Fly" + }, + "gds": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "gdu": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic" + }, + "gdx": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Mewati-Gojri" + }, + "gea": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Nuclear Boleic", + "level8": "Gera-Geruma-Kubi-Deno", + "level9": "Gera-Geruma" + }, + "geb": { + "level0": "Ramu", + "level1": "Lower Ramu", + "level2": "Ruboni" + }, + "gec": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Grebo-Aizi", + "level4": "Grebo", + "level5": "Liberian Grebo", + "level6": "North-Central Liberian Grebo", + "level7": "Barclayville-Gboloo-Central Liberian Grebo", + "level8": "Gboloo-Central Grebo" + }, + "ged": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo" + }, + "geg": { + "level0": "Bookkeeping" + }, + "geh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Bairisch", + "level10": "Global South Bavarian" + }, + "gei": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "South Halmahera" + }, + "gej": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Western Gbe" + }, + "gek": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3", + "level6": "Goemaic" + }, + "gel": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Northwestern Kainji", + "level6": "Dukaic", + "level7": "Main-Gwamhi" + }, + "gen": { + "level0": "Bookkeeping" + }, + "geq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Zandic", + "level6": "Zande-Nzakara" + }, + "ges": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "Banda-Geser", + "level4": "Seran Laut", + "level5": "Geser-Gorom-Bati" + }, + "gev": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "B10-B30", + "level8": "Okani (B.30)", + "level9": "Southern Okani" + }, + "gew": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Nuclear Boleic", + "level8": "Gera-Geruma-Kubi-Deno", + "level9": "Gera-Geruma" + }, + "gex": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Eastern Omo-Tana", + "level8": "Karre-Boni" + }, + "gey": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega", + "level8": "Mituku-Lega", + "level9": "Mitukuic" + }, + "gez": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic" + }, + "gfk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Patpatar-Minigir-Tolai" + }, + "gft": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Outer South Ethiopic", + "level6": "N-Group" + }, + "gga": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Santa Isabel", + "level10": "East Santa Isabel" + }, + "ggb": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Bassa-Klao", + "level5": "Bassaic" + }, + "ggd": { + "level0": "Pama-Nyungan", + "level1": "Paman" + }, + "gge": { + "level0": "Maningrida", + "level1": "Bureran" + }, + "ggg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Western Hindi", + "level10": "Unclassified Western Hindi", + "level11": "Ghera-Gurgula" + }, + "ggh": { + "level0": "Bookkeeping" + }, + "ggl": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Greater Yaganon", + "level4": "Yaganon", + "level5": "Ganglau-Saep" + }, + "ggm": { + "level0": "Bookkeeping" + }, + "ggr": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Alaya-Athima", + "level3": "Thaypanic" + }, + "ggt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Ngero", + "level8": "Western Ngero", + "level9": "Tuam" + }, + "ggu": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Nwa-Ben", + "level4": "Ben-Gban" + }, + "ggw": { + "level0": "Suki-Gogodala", + "level1": "Gogodalic" + }, + "gha": { + "level0": "Afro-Asiatic", + "level1": "Berber" + }, + "ghe": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Ghale", + "level5": "Nuclear Ghale" + }, + "ghh": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Ghale", + "level5": "Nuclear Ghale" + }, + "ghk": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Northern Karen" + }, + "ghl": { + "level0": "Nubian", + "level1": "Central Nubian", + "level2": "Kordofan Nubian", + "level3": "Eastern Kordofan Nubian" + }, + "ghn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "New Georgia", + "level10": "West New Georgia", + "level11": "Simboic", + "level12": "Ghanongga-Lungga" + }, + "gho": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Kabyle-Atlas Berber", + "level3": "Atlas Berber", + "level4": "Northwestern Moroccan Berber" + }, + "ghr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Western Hindi", + "level10": "Unclassified Western Hindi", + "level11": "Ghera-Gurgula" + }, + "ghs": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean" + }, + "ght": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Ghale" + }, + "gia": { + "level0": "Jarrakan" + }, + "gib": { + "level0": "Pidgin", + "level1": "Hausa-based pidgin" + }, + "gic": { + "level0": "Unclassifiable" + }, + "gid": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara" + }, + "gie": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Neyo-Dida", + "level3": "Dida", + "level4": "Guebie-Lakota Dida" + }, + "gig": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Western Rajasthani" + }, + "gih": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "New South Wales Pama-Nyungan", + "level3": "Greater Bandjalangic", + "level4": "Bandjalangic", + "level5": "Inland Bandjalang" + }, + "gii": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Eastern Omo-Tana" + }, + "gil": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian" + }, + "gim": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Fore-Gimi" + }, + "gin": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Tsezic", + "level4": "West Tsezic" + }, + "gio": { + "level0": "Bookkeeping" + }, + "gip": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Arawe", + "level11": "West Arawe" + }, + "giq": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Southwestern Kra", + "level3": "Western Kra", + "level4": "Gauic", + "level5": "Gelaoic", + "level6": "Southwestern Gelao" + }, + "gir": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Southwestern Kra", + "level3": "Western Kra", + "level4": "Gauic", + "level5": "Gelaoic", + "level6": "Northern Gelao", + "level7": "Ahouic" + }, + "gis": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Maroua", + "level5": "Giziga" + }, + "git": { + "level0": "Tsimshian", + "level1": "Nishga-Gitxsan" + }, + "giu": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Southwestern Kra", + "level3": "Western Kra", + "level4": "Gauic", + "level5": "Gelaoic", + "level6": "Northern Gelao" + }, + "giw": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Southwestern Kra", + "level3": "Western Kra", + "level4": "Gauic", + "level5": "Gelaoic", + "level6": "Southwestern Gelao" + }, + "gix": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Western Mundu-Baka", + "level7": "River Western Mundu-Baka", + "level8": "Bwaka" + }, + "giy": { + "level0": "Unattested" + }, + "giz": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Maroua", + "level5": "Giziga" + }, + "gjk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Gujaratic", + "level10": "Western Gujaratic" + }, + "gjm": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "Victorian Pama-Nyungan", + "level3": "Kulin-Bunganditj", + "level4": "Warrnambool-Bunganditj" + }, + "gjn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Gonja-Dompo" + }, + "gjr": { + "level0": "Mixed Language", + "level1": "Gurindji-Kriol" + }, + "gju": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Mewati-Gojri" + }, + "gka": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Warup", + "level4": "Nuclear Warup", + "level5": "Unclassified Nuclear Warup" + }, + "gkd": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Sogeram", + "level5": "East Sogeram", + "level6": "Aisian" + }, + "gke": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Northern Mbum", + "level6": "Dama-Galke" + }, + "gkn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Ogonoid", + "level5": "East Ogonoid" + }, + "gko": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Norman Pama" + }, + "gkp": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Southwest Mande", + "level4": "Kpelle" + }, + "gku": { + "level0": "Tuu", + "level1": "!Ui", + "level2": "Ghaap-Kalahari" + }, + "gla": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Insular Celtic", + "level6": "Goidelic", + "level7": "Modern Goidelic", + "level8": "Eastern Goidelic" + }, + "glb": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3" + }, + "glc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Adamawa Bua", + "level5": "Inland Bua", + "level6": "Goulaic" + }, + "gld": { + "level0": "Tungusic", + "level1": "Central-Western Tungusic" + }, + "gle": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Insular Celtic", + "level6": "Goidelic", + "level7": "Modern Goidelic" + }, + "glg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance" + }, + "glh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Pashayi", + "level5": "Western Pashayi" + }, + "gli": { + "level0": "Bookkeeping" + }, + "glj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Adamawa Bua", + "level5": "Inland Bua", + "level6": "Goulaic", + "level7": "Zan-Kulaalic", + "level8": "Kulaalic" + }, + "glk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Caspian", + "level8": "Gilaki-Rudbari" + }, + "gll": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Badjiri-Eastern Karnic", + "level3": "Eastern Karnic" + }, + "glo": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Nuclear Boleic", + "level8": "Galambu-Bele" + }, + "glr": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Wee", + "level5": "Guere-Krahn" + }, + "glu": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Bagirmic", + "level6": "Morom-Jaya-Naba", + "level7": "Bayo-Morom" + }, + "glv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Insular Celtic", + "level6": "Goidelic", + "level7": "Modern Goidelic", + "level8": "Eastern Goidelic" + }, + "glw": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mandaraic", + "level6": "Wandala-Malgwa-Glavda" + }, + "gma": { + "level0": "Worrorran", + "level1": "Northern Worrorran", + "level2": "Forrest River" + }, + "gmb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Central-Northern Malaita" + }, + "gmd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bikwin-Jen", + "level5": "Northern Bikwin-Jen", + "level6": "Mak-Tal" + }, + "gmg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Sogeram", + "level5": "Unclassified Sogeram" + }, + "gmh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German" + }, + "gml": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Alts\u00e4chsisch", + "level7": "Middle-Modern Low German" + }, + "gmm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Oriental", + "level5": "Mbodomo-Bofi" + }, + "gmn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Northern Samba-Duru", + "level7": "Vere-Gimme", + "level8": "Koma Alantika" + }, + "gmo": { + "level0": "Bookkeeping" + }, + "gmu": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Gum" + }, + "gmv": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "North-West Ometo", + "level3": "Central Ometo", + "level4": "Dawro-Gofa-Gamo" + }, + "gmx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Southern Tanzania Highlands Bantu", + "level9": "Kinga-Magoma" + }, + "gmy": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Graeco-Phrygian", + "level3": "Greek", + "level4": "South Greek" + }, + "gna": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Kaansa-Dogose" + }, + "gnb": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Northeastern Kuki-Chin", + "level5": "Sizangic", + "level6": "Gangte-Vaiphei" + }, + "gnc": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Unclassified Berber" + }, + "gnd": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mofuic", + "level6": "Meri" + }, + "gne": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Zaric", + "level6": "Nuclear Zaric", + "level7": "Izeric" + }, + "gng": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Gurma-Yom-Naudem", + "level11": "Gurma", + "level12": "Gurma B", + "level13": "Konkomba-Gangam" + }, + "gnh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos" + }, + "gni": { + "level0": "Bunaban" + }, + "gnj": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Nwa-Ben", + "level4": "Ben-Gban", + "level5": "Bengic" + }, + "gnk": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Non-Khoekhoe", + "level3": "West-Kxoe", + "level4": "Naro-Ana", + "level5": "Ana" + }, + "gnl": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Eastern Maric" + }, + "gnm": { + "level0": "Dagan", + "level1": "Southeast Dagan" + }, + "gnn": { + "level0": "Pama-Nyungan", + "level1": "Yuulngu", + "level2": "Southern Yolngu", + "level3": "Southern-Eastern Yolngu", + "level4": "Dhuwal-Dhuwala", + "level5": "Western Dhuwal-Dhuwala" + }, + "gno": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Gondi", + "level4": "Northwest Gondi" + }, + "gnq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Northern Murutic" + }, + "gnr": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "North Coast Pama-Nyungan", + "level3": "Waka-Kabic", + "level4": "Eastern Waka-Kabic" + }, + "gnt": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Tonda" + }, + "gnu": { + "level0": "Nuclear Torricelli", + "level1": "Unclassified Nuclear Torricelli" + }, + "gnw": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup I", + "level7": "Tupi-Guarani Subgroup I.B" + }, + "gnz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Western Mundu-Baka", + "level7": "Baka-Gundi", + "level8": "Baka complex" + }, + "goa": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Mano-Dan", + "level4": "Guro-Dan", + "level5": "Guro-Yaoure" + }, + "gob": { + "level0": "Guahiboan", + "level1": "Nuclear Guahiboan", + "level2": "Central Guahibo", + "level3": "Guahibo-Playero" + }, + "goc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Buang linkage", + "level9": "Mumeng" + }, + "god": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Beteic", + "level3": "Western Bete" + }, + "goe": { + "level0": "Sino-Tibetan" + }, + "gof": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "North-West Ometo", + "level3": "Central Ometo", + "level4": "Dawro-Gofa-Gamo" + }, + "gog": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "West Ruvu" + }, + "goh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German" + }, + "goi": { + "level0": "East Strickland", + "level1": "Kubo-Samo-Bibo" + }, + "goj": { + "level0": "Bookkeeping" + }, + "gok": { + "level0": "Bookkeeping" + }, + "gol": { + "level0": "Atlantic-Congo" + }, + "gom": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Southern zone", + "level7": "Marathic", + "level8": "Marathi-Konkani" + }, + "goo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Eastern Fijian", + "level7": "Nuclear Eastern Fijian" + }, + "gop": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Southwest Cenderawasih Bay" + }, + "goq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay", + "level6": "Eastern Indonesia Trade Malay", + "level7": "Manadoic Malay" + }, + "gor": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Gorontalo-Mongondow", + "level4": "Gorontalic" + }, + "gos": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Alts\u00e4chsisch", + "level7": "Middle-Modern Low German", + "level8": "Low German", + "level9": "West Low German", + "level10": "North Low Saxon" + }, + "got": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "East Germanic" + }, + "gou": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Dabaic", + "level5": "Buwal-Gavar" + }, + "gov": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Mano-Dan", + "level4": "Guro-Dan", + "level5": "Dan-Toura", + "level6": "Toura-Goo" + }, + "gow": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "South Cushitic", + "level3": "Greater West Rift South Cushitic", + "level4": "West Rift South Cushitic", + "level5": "Northern West Rift South Cushitic", + "level6": "Iraqwoid" + }, + "gox": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Central Bandaic", + "level8": "Central Core Bandaic", + "level9": "Mid-Southern Central Core Bandaic" + }, + "goy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Kim-Besme-Goundo" + }, + "goz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic" + }, + "gpa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Ebira-Nupoid", + "level4": "Macro-Nupoid", + "level5": "Nupoid", + "level6": "Dibo-Kupa", + "level7": "Abawa", + "level8": "Kami-Gupa" + }, + "gpe": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "West African Creole English", + "level13": "Coastal Nigerian Krio" + }, + "gqa": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Teraic", + "level5": "Eastern Tera" + }, + "gqi": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic" + }, + "gqr": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Logone-Chari", + "level7": "Bediondo" + }, + "gqu": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Southwestern Kra", + "level3": "Western Kra", + "level4": "Gauic", + "level5": "Gelaoic" + }, + "gra": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil", + "level8": "Garasia Bhil" + }, + "grc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Graeco-Phrygian", + "level3": "Greek", + "level4": "South Greek", + "level5": "Central Greek" + }, + "grd": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi East", + "level6": "Guruntumic" + }, + "grg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Gusap-Mot", + "level4": "Gira-Neko-Nekgini" + }, + "grh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Kauru" + }, + "gri": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Guadalcanal-Nggelic", + "level6": "Nuclear Guadalcanal-Nggelic", + "level7": "North and West Guadalcanal" + }, + "grj": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Grebo-Aizi", + "level4": "Grebo", + "level5": "Liberian Grebo" + }, + "grm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Dusunic" + }, + "gro": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Southern Tibetic", + "level7": "Dzongkhic" + }, + "grq": { + "level0": "Ramu", + "level1": "Agoan" + }, + "grr": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Greater Zenatic", + "level3": "Zenatic", + "level4": "Northern Saharan Oasis Berber" + }, + "grs": { + "level0": "Nimboranic", + "level1": "Outer Nimboranic", + "level2": "Mlap-Gresi-Kemtuik", + "level3": "Gresi-Kemtuik" + }, + "grt": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo" + }, + "gru": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Outer South Ethiopic", + "level6": "N-Group" + }, + "grv": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Grebo-Aizi", + "level4": "Grebo", + "level5": "Liberian Grebo", + "level6": "North-Central Liberian Grebo", + "level7": "Barclayville-Gboloo-Central Liberian Grebo", + "level8": "Gboloo-Central Grebo" + }, + "grw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Taupota linkage", + "level10": "Nuclear Taupota linkage", + "level11": "Eastern Taupota" + }, + "gry": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Grebo-Aizi", + "level4": "Grebo", + "level5": "Liberian Grebo", + "level6": "North-Central Liberian Grebo", + "level7": "Barclayville-Gboloo-Central Liberian Grebo" + }, + "grz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Patpatar-Minigir-Tolai", + "level9": "Minigir-Tolai" + }, + "gsc": { + "level0": "Bookkeeping" + }, + "gse": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "American Sign" + }, + "gsg": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "DGSic" + }, + "gsl": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Jola", + "level6": "Gusilay-Bandial" + }, + "gsm": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "gsn": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Erap", + "level4": "Finungwan-Mamaa-Gusan" + }, + "gso": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Meridional-Occidental", + "level5": "Gbaya Meridional" + }, + "gsp": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Greater Yaganon" + }, + "gss": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic" + }, + "gsw": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Alemannic", + "level10": "South Alemannic" + }, + "gti": { + "level0": "Bookkeeping" + }, + "gtu": { + "level0": "Bookkeeping" + }, + "gua": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan", + "level7": "Jaku-Gubi" + }, + "guc": { + "level0": "Arawakan", + "level1": "Caribbean Arawakan", + "level2": "Guajiro-Paraujano" + }, + "gud": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Neyo-Dida", + "level3": "Dida" + }, + "gue": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Ngumpin-Yapa", + "level3": "Ngumpin", + "level4": "Eastern Ngumpin", + "level5": "Ngumpit" + }, + "guf": { + "level0": "Pama-Nyungan", + "level1": "Yuulngu", + "level2": "Southern Yolngu", + "level3": "Southern-Eastern Yolngu", + "level4": "Dhuwal-Dhuwala", + "level5": "Eastern Dhuwal-Dhuwala" + }, + "gug": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup I", + "level7": "Tupi-Guarani Subgroup I.A", + "level8": "Paraguay-Brazil Guarani" + }, + "guh": { + "level0": "Guahiboan", + "level1": "Nuclear Guahiboan", + "level2": "Central Guahibo", + "level3": "Guahibo-Playero" + }, + "gui": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup I", + "level7": "Tupi-Guarani Subgroup I.B", + "level8": "Chiriguanic" + }, + "guj": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Gujaratic" + }, + "guk": { + "level0": "Gumuz", + "level1": "Nuclear Gumuz" + }, + "gul": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Gullah-Nevis-Antigua", + "level15": "Gullah" + }, + "gum": { + "level0": "Barbacoan", + "level1": "Coconucan" + }, + "gun": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup I", + "level7": "Tupi-Guarani Subgroup I.A" + }, + "guo": { + "level0": "Guahiboan" + }, + "gup": { + "level0": "Gunwinyguan" + }, + "guq": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup I" + }, + "gur": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Northwest Oti-Volta", + "level13": "Mossi-Farefare", + "level14": "Farefareic" + }, + "gus": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic" + }, + "gut": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Votic Chibchan" + }, + "guu": { + "level0": "Yanomamic", + "level1": "Ninam-Yanomam-Yaroame", + "level2": "Yanomam-Yaroame", + "level3": "Yanomam-Yanimamo" + }, + "guv": { + "level0": "Bookkeeping" + }, + "guw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Fongbeic" + }, + "gux": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Gurma-Yom-Naudem", + "level11": "Gurma", + "level12": "Gurma B", + "level13": "Gourmantche-Moba" + }, + "guz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Nyanza Mara", + "level11": "North Mara" + }, + "gva": { + "level0": "Lengua-Mascoy", + "level1": "Eastern Enlhet-Enenlhet" + }, + "gvc": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Eastern Eastern Tucanoan", + "level3": "Eastern Eastern Tucanoan II", + "level4": "Kotiria-Piratapuyo" + }, + "gve": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Lower Markham", + "level9": "Busu" + }, + "gvf": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Simbu", + "level3": "Nuclear Simbu", + "level4": "Golinic" + }, + "gvj": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VIII", + "level6": "Guaja-Kaapor-Ava", + "level7": "Guaja-Aure-Aura" + }, + "gvl": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Chari" + }, + "gvm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Shiroro-Kamuku", + "level6": "Shiroro" + }, + "gvn": { + "level0": "Pama-Nyungan", + "level1": "Yimidhirr-Yalanji-Yidinic", + "level2": "Yalandyic" + }, + "gvo": { + "level0": "Tupian", + "level1": "Monde", + "level2": "Gavianic", + "level3": "Nuclear Gavianic" + }, + "gvp": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Cerrado", + "level3": "Goyaz", + "level4": "Northern Je", + "level5": "Eastern Timbira", + "level6": "Southeastern Timbira" + }, + "gvr": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Tamangic", + "level5": "Gurungic" + }, + "gvs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage" + }, + "gvy": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Thura-Yura", + "level3": "Core Thura Yura", + "level4": "Northern Thura-Yura" + }, + "gwa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Potou" + }, + "gwb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan", + "level7": "Unclassified Nigerian Jarawan" + }, + "gwc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Kohistani", + "level8": "Dir-Swat Kohistani" + }, + "gwd": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Transversal Lowland East Cushitic", + "level6": "Dullay" + }, + "gwe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Kilimanjaro-Taita", + "level9": "Kilimanjaro Bantu" + }, + "gwf": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Kohistani", + "level8": "Indus Kohistanic", + "level9": "Outer Indus Kohistani" + }, + "gwg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bikwin-Jen", + "level5": "Southern Bikwin-Jen", + "level6": "Bambuka-Gomu-Leelau" + }, + "gwi": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central Alaska-Yukon Athabaskan", + "level4": "Gwichin-Han" + }, + "gwj": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Non-Khoekhoe", + "level3": "West-Kxoe", + "level4": "Naro-Ana", + "level5": "Ana" + }, + "gwn": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.1" + }, + "gwr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "North Nyanza" + }, + "gwt": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Gawarbatic" + }, + "gwu": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Southern Maric" + }, + "gww": { + "level0": "Worrorran", + "level1": "Northern Worrorran", + "level2": "Forrest River" + }, + "gwx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "South Guang", + "level8": "Hill South Guang", + "level9": "Gua-Cherepon" + }, + "gxx": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Wee", + "level5": "Guere-Krahn", + "level6": "Guere" + }, + "gya": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Meridional-Occidental" + }, + "gyb": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Rempic" + }, + "gyd": { + "level0": "Tangkic", + "level1": "Southern Tangkic", + "level2": "Kayardild-Yangkaal" + }, + "gye": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "Lameic" + }, + "gyf": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Southern Maric" + }, + "gyg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Ngbandi-Mongoba-Kazibati", + "level6": "Ngbandic" + }, + "gyi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Western A80", + "level10": "Mvumboic", + "level11": "Kwasio-Gyele" + }, + "gyl": { + "level0": "South Omotic", + "level1": "AHK", + "level2": "Aari-Gayil" + }, + "gym": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Isthmic Chibchan", + "level3": "Eastern Isthmic Chibchan", + "level4": "Guaymiic" + }, + "gyn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Barbados-Eustatius", + "level15": "Barbados-Trinidad" + }, + "gyr": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup II" + }, + "gyy": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Southern Maric", + "level5": "Margany-Gunya" + }, + "gyz": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Northwest South Bauchi", + "level7": "Gejic" + }, + "gza": { + "level0": "Blue Nile Mao", + "level1": "West Mao" + }, + "gzi": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Central Iran Kermanic", + "level8": "Nuclear Central Iran Kermanic" + }, + "gzn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "South Halmahera", + "level6": "East Makian-Gane" + }, + "haa": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central Alaska-Yukon Athabaskan", + "level4": "Gwichin-Han" + }, + "hab": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Vietnamese Sign" + }, + "hac": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Gorani" + }, + "had": { + "level0": "Hatam-Mansim" + }, + "hae": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Oromoid", + "level7": "Nuclear Oromo", + "level8": "Central-Eastern Oromo", + "level9": "South-East-North Oromo" + }, + "haf": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Vietnamese Sign" + }, + "hag": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Southeast Western Oti-Volta", + "level13": "Kamara-Hanga" + }, + "hah": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "North Bougainville Oceanic", + "level10": "Nuclear North Bougainville Oceanic", + "level11": "Buka", + "level12": "Saposa-Tinputz", + "level13": "Tinputzic" + }, + "haj": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga", + "level10": "Eastern Bengali" + }, + "hak": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic" + }, + "hal": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric", + "level3": "Kayong-Jeh-Halang", + "level4": "Jeh-Halang" + }, + "ham": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Western Sepik Hill", + "level3": "Hewa-April River" + }, + "han": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "West Highlands Kivu", + "level12": "Rundic", + "level13": "Hangaza-Shubi" + }, + "hao": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "North Bougainville Oceanic", + "level10": "Nuclear North Bougainville Oceanic", + "level11": "Buka", + "level12": "Haliaic" + }, + "hap": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani", + "level2": "Central Dani", + "level3": "Grand Valley Dani", + "level4": "Southeast Grand Valley Dani" + }, + "haq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "West Highlands Kivu", + "level12": "Rundic" + }, + "har": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Harari-East Gurage" + }, + "has": { + "level0": "Wakashan", + "level1": "Northern Wakashan" + }, + "hat": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil", + "level14": "Macro-French", + "level15": "Circum-Caribbean French" + }, + "hau": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.1" + }, + "hav": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "Forest Kivu", + "level12": "Hunde-Havu" + }, + "haw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Proximal" + }, + "hax": { + "level0": "Haida" + }, + "hay": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "Rutara", + "level11": "South Rutara" + }, + "haz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Farsic", + "level9": "Eastern Farsic" + }, + "hba": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Nkutsuic", + "level11": "Tetelaic" + }, + "hbb": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Bura-Marghi", + "level6": "Marghic", + "level7": "Kilba-South Margi" + }, + "hbn": { + "level0": "Heibanic", + "level1": "West-Central Heibanic", + "level2": "Central Heibanic", + "level3": "Ebang-Logol", + "level4": "Ebang-Laru" + }, + "hbo": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Canaanite", + "level6": "Hebrewic" + }, + "hbs": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "South Slavic", + "level5": "Western South Slavic" + }, + "hbu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Eastern Timor", + "level4": "Central Timoric A" + }, + "hca": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Western Hindi", + "level10": "Hindustani" + }, + "hch": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Corachol" + }, + "hdn": { + "level0": "Haida" + }, + "hds": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "Honduras-Panama Sign" + }, + "hdy": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Highland East Cushitic", + "level4": "Sidaama-Hadiyya-Kambaata", + "level5": "Hadiyya-Kambaata", + "level6": "Hadiyyaic" + }, + "hea": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "East Hmongic", + "level5": "Northeastern Qiandongic Miao" + }, + "heb": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Canaanite", + "level6": "Hebrewic" + }, + "hed": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Masa", + "level3": "South Masa", + "level4": "Peveic", + "level5": "Hede-Ngide" + }, + "heg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar" + }, + "heh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Southern Tanzania Highlands Bantu", + "level9": "Bena-Hehe" + }, + "hei": { + "level0": "Wakashan", + "level1": "Northern Wakashan", + "level2": "Kwakiutlan" + }, + "hem": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Luba-Kaonde", + "level9": "Lubaic", + "level10": "Bangubangu-Kasai" + }, + "her": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia", + "level11": "Herero (R.30)" + }, + "hgm": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Khoekhoe", + "level3": "North Khoekhoe" + }, + "hgw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Taupota linkage", + "level10": "Nuclear Taupota linkage", + "level11": "Eastern Taupota" + }, + "hhi": { + "level0": "Anim", + "level1": "Inland Gulf of Papua", + "level2": "West Inland Gulf of Papua", + "level3": "Hoyaic" + }, + "hhr": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Jola", + "level6": "FH-Jola", + "level7": "PF-Jola", + "level8": "Her-Ejamat" + }, + "hhy": { + "level0": "Anim", + "level1": "Inland Gulf of Papua", + "level2": "West Inland Gulf of Papua", + "level3": "Hoyaic" + }, + "hia": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Lamang-Hdi" + }, + "hib": { + "level0": "Hibito-Cholon" + }, + "hid": { + "level0": "Siouan", + "level1": "Missouri River Siouan" + }, + "hif": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Western Hindi", + "level10": "Hindustani" + }, + "hig": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Higic", + "level5": "Nkafa-Kirya-Bana", + "level6": "Nkafa-Kirya" + }, + "hih": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kumil-Tibor", + "level6": "Tibor", + "level7": "Nuclear Tibor" + }, + "hii": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Nuclear Himachali" + }, + "hij": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Basaa-Yaunde (A40-70)", + "level8": "Basaa (A.40)", + "level9": "Basaa-Bakoko", + "level10": "Basaa-Hijuk" + }, + "hik": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua", + "level6": "Seram Straits", + "level7": "Ambonic", + "level8": "Central Ambon" + }, + "hil": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Central Bisayan", + "level6": "Peripheral Central Bisayan", + "level7": "Capiznon-Ilonggo-Kawayan" + }, + "hin": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Western Hindi", + "level10": "Hindustani" + }, + "hio": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Non-Khoekhoe", + "level3": "Ost-Kxoe", + "level4": "Tshwa Khoe" + }, + "hir": { + "level0": "Unattested", + "level1": "Arawan (Unattested)" + }, + "hit": { + "level0": "Indo-European", + "level1": "Anatolian" + }, + "hiw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage", + "level7": "Hiw-Lo-Toga" + }, + "hix": { + "level0": "Cariban", + "level1": "Parukotoan", + "level2": "Waiwaian" + }, + "hji": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic" + }, + "hka": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Kilimanjaro-Taita", + "level9": "Kilimanjaro Bantu", + "level10": "Chaga", + "level11": "Central Kilimanjaro" + }, + "hke": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "Forest Kivu", + "level12": "Hunde-Havu" + }, + "hkh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Kashmiric" + }, + "hkk": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean", + "level3": "Nuclear Binanderean", + "level4": "South Binanderean", + "level5": "Orokaivic" + }, + "hkn": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "South Bahnaric", + "level4": "Mnong-Stieng-Chrau", + "level5": "Stieng" + }, + "hks": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "CSLic" + }, + "hla": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "North Bougainville Oceanic", + "level10": "Nuclear North Bougainville Oceanic", + "level11": "Buka", + "level12": "Haliaic" + }, + "hlb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Halbic" + }, + "hld": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric", + "level3": "Kayong-Jeh-Halang", + "level4": "Unclassified Kayong-Jeh-Halang" + }, + "hle": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Lipo-Lolopo", + "level7": "Unclassified Lipo-Lolopo" + }, + "hlt": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "South Peripheral Kuki-Chin", + "level5": "Choic" + }, + "hlu": { + "level0": "Indo-European", + "level1": "Anatolian", + "level2": "Luvo-Lydian", + "level3": "Luvo-Palaic", + "level4": "Luvic", + "level5": "Luvian" + }, + "hma": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Mashan" + }, + "hmb": { + "level0": "Songhay", + "level1": "Eastern Songhay" + }, + "hmc": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Huishui" + }, + "hmd": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian" + }, + "hme": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Huishui" + }, + "hmf": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Chuanqiandian", + "level7": "First Vernacular Hmong", + "level8": "Far Western Miao", + "level9": "Unclassified First Vernacular Hmong" + }, + "hmg": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Guiyang" + }, + "hmh": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Huishui" + }, + "hmi": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Huishui" + }, + "hmj": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian" + }, + "hml": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian" + }, + "hmm": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Mashan" + }, + "hmo": { + "level0": "Pidgin", + "level1": "Motu-based pidgin" + }, + "hmp": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Mashan" + }, + "hmq": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "East Hmongic", + "level5": "Northeastern Qiandongic Miao", + "level6": "Eastern Qiandongic Miao" + }, + "hmr": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin", + "level5": "Mizoic", + "level6": "Hmaric", + "level7": "Hmar-Saihriem" + }, + "hms": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "East Hmongic", + "level5": "South Qiandongic Miao" + }, + "hmt": { + "level0": "Angan", + "level1": "Nuclear Angan", + "level2": "Kapau-Menya" + }, + "hmu": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "West Alor-Straits-Pantar", + "level4": "West Alor" + }, + "hmv": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Chuanqiandian", + "level7": "First Vernacular Hmong", + "level8": "Far Western Miao", + "level9": "Unclassified First Vernacular Hmong" + }, + "hmw": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Mashan" + }, + "hmy": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Guiyang" + }, + "hmz": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Chuanqiandian", + "level7": "First Vernacular Hmong" + }, + "hna": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Dabaic" + }, + "hnd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Greater Panjabic", + "level9": "Hindko-Siraiki", + "level10": "Hindko" + }, + "hne": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Eastern Hindi" + }, + "hng": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "Southeastern Kikongo", + "level20": "Southern Kikongo" + }, + "hnh": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Non-Khoekhoe", + "level3": "West-Kxoe", + "level4": "Kxoe-Ani" + }, + "hni": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Hanic", + "level7": "Ha-Ya" + }, + "hnj": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Chuanqiandian", + "level7": "First Vernacular Hmong", + "level8": "Far Western Miao" + }, + "hnn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "South Mangyan" + }, + "hno": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Greater Panjabic", + "level9": "Hindko-Siraiki", + "level10": "Hindko" + }, + "hns": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Magadhan", + "level10": "Bhojpuric" + }, + "hnu": { + "level0": "Austroasiatic", + "level1": "Vietic", + "level2": "Cuoi" + }, + "hoa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "New Georgia", + "level10": "East New Georgia", + "level11": "Rovianic", + "level12": "Hoava-Kusaghe" + }, + "hob": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Upper Markham", + "level9": "Mountain Upper Markham" + }, + "hoc": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Mundaric", + "level5": "Ho-Mundari" + }, + "hod": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic", + "level5": "Gudeic", + "level6": "Nzanyic" + }, + "hoe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Southeastern Benue-Congo Plateau", + "level5": "Horom-Fyem" + }, + "hoh": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Modern South Arabian", + "level4": "Hobyot-Western MSA" + }, + "hoi": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central Alaska-Yukon Athabaskan", + "level4": "Koyukonic" + }, + "hoj": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Eastern Rajasthani" + }, + "hol": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Northern Njila", + "level9": "Mbala-Holu-Sondi (K.10)", + "level10": "Holu (K.10)" + }, + "hom": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Ngbele-Ngenda", + "level15": "Ngendan" + }, + "hoo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega" + }, + "hop": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan" + }, + "hor": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Chari" + }, + "hos": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Vietnamese Sign" + }, + "hot": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Misim-Yamap" + }, + "hov": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Muller-Schwaner", + "level6": "Hovongan-Kereho" + }, + "how": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Hanic" + }, + "hoy": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Badaga-Kannada", + "level5": "Kannadoid" + }, + "hoz": { + "level0": "Blue Nile Mao", + "level1": "West Mao", + "level2": "Hozo-Seze" + }, + "hpo": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Northern Burmish" + }, + "hps": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "hra": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin", + "level5": "Mizoic", + "level6": "Hmaric" + }, + "hre": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric", + "level3": "Hre-Sedang-Todrah-Monam", + "level4": "Hre-Sedang" + }, + "hrk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua", + "level6": "Seram Straits", + "level7": "Uliase", + "level8": "Hatuhaha" + }, + "hrm": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Chuanqiandian", + "level7": "First Vernacular Hmong" + }, + "hro": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic", + "level4": "Chamic" + }, + "hrr": { + "level0": "Bookkeeping" + }, + "hrt": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "Bohtan" + }, + "hrv": { + "level0": "Indo-European", + "level1": "Balto-Slavic" + }, + "hrx": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Middle German", + "level7": "West Middle German", + "level8": "Rhenish Franconian" + }, + "hrz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Northern Tatic" + }, + "hsb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "West Slavic", + "level5": "Sorbian" + }, + "hsf": { + "level0": "Bookkeeping" + }, + "hsh": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Central European Sign", + "level4": "Nuclear Central European Sign" + }, + "hsl": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "hsn": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic" + }, + "hss": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Modern South Arabian", + "level4": "Hobyot-Western MSA", + "level5": "Western MSA" + }, + "hti": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "East Seram", + "level4": "Setic" + }, + "hto": { + "level0": "Huitotoan", + "level1": "Nuclear Witotoan", + "level2": "Minica-Murui" + }, + "htu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua", + "level6": "Seram Straits", + "level7": "Ambonic", + "level8": "Northeast Ambon" + }, + "hub": { + "level0": "Chicham", + "level1": "Shuaric", + "level2": "Huambisa-Shuar" + }, + "huc": { + "level0": "Kxa" + }, + "hud": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Patakai-Manusela", + "level4": "Manusela-Huaulu" + }, + "hue": { + "level0": "Huavean", + "level1": "San Francisco-Santa Mar\u00eda Huave" + }, + "huf": { + "level0": "Kwalean", + "level1": "Humene-Kwale" + }, + "hug": { + "level0": "Harakmbut" + }, + "huh": { + "level0": "Araucanian" + }, + "hui": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Kewa-Huli" + }, + "huj": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Guiyang" + }, + "huk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Three Rivers", + "level4": "Amalumute", + "level5": "Northwest Seram" + }, + "hul": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "Sinagoro-Keapara", + "level9": "Hula-Keapara" + }, + "hum": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Hungan-Samba" + }, + "hun": { + "level0": "Uralic", + "level1": "Hungaric" + }, + "huo": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Angkuic", + "level5": "Southern Angkuic" + }, + "hup": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Pacific Coast Athabaskan", + "level4": "California Athabaskan" + }, + "huq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic", + "level4": "Chamic", + "level5": "Chru-Northern Cham", + "level6": "Northern Cham" + }, + "hur": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Central Salish", + "level3": "South Georgia Central Salish" + }, + "hus": { + "level0": "Mayan", + "level1": "Huastecan Mayan" + }, + "hut": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic" + }, + "huu": { + "level0": "Huitotoan", + "level1": "Nuclear Witotoan", + "level2": "Minica-Murui" + }, + "huv": { + "level0": "Huavean", + "level1": "San Dionisio-San Mateo Huave" + }, + "huw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "West Central Maluku", + "level3": "Sula-Buru", + "level4": "Buruic" + }, + "hux": { + "level0": "Huitotoan", + "level1": "Nuclear Witotoan" + }, + "huy": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "North-Eastern Neo-Aramaic", + "level11": "Trans-Zab" + }, + "huz": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Tsezic", + "level4": "East Tsezic" + }, + "hva": { + "level0": "Bookkeeping" + }, + "hvc": { + "level0": "Unclassifiable" + }, + "hve": { + "level0": "Huavean", + "level1": "San Dionisio-San Mateo Huave" + }, + "hvk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Voh-Kone-Cem-Pac", + "level10": "Voh-Kone", + "level11": "Bwatooic" + }, + "hvn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Sumba-Hawu", + "level5": "Hawu-Dhao" + }, + "hvv": { + "level0": "Huavean", + "level1": "San Francisco-Santa Mar\u00eda Huave" + }, + "hwa": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Bakwe-Wane" + }, + "hwc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Pacific Creole English" + }, + "hwo": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Teraic", + "level5": "Eastern Tera" + }, + "hya": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Higic" + }, + "hye": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Armenic", + "level3": "Eastern-Western Armenian" + }, + "hyw": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Armenic", + "level3": "Eastern-Western Armenian" + }, + "iai": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Loyalty Islands" + }, + "ian": { + "level0": "Ndu", + "level1": "Nuclear Ndu", + "level2": "Sawosic", + "level3": "Iatmulic" + }, + "iap": { + "level0": "Bookkeeping" + }, + "iba": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Ibanic", + "level5": "Iban-Mualang-Seberuang", + "level6": "Iban-Seberuang", + "level7": "Northern Iban" + }, + "ibb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Efikic", + "level8": "Okop Usem", + "level9": "Efik-Ibibio" + }, + "ibd": { + "level0": "Iwaidjan Proper", + "level1": "Central Iwaidjic" + }, + "ibe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid" + }, + "ibg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Cagayan Valley", + "level4": "Ibanagic" + }, + "ibh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic", + "level4": "Chamic", + "level5": "Rade-Jarai" + }, + "ibi": { + "level0": "Bookkeeping" + }, + "ibl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Southern Cordilleran", + "level6": "West Southern Cordilleran", + "level7": "Nuclear Southern Cordilleran" + }, + "ibm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Agoi-Doko-Iyoniyong" + }, + "ibn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Ibino-Iko" + }, + "ibo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Igboid", + "level4": "Nuclear Igboid", + "level5": "Central-Northern Igbo" + }, + "ibr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Ibuoroic", + "level8": "Ibuoro-ItuMbuso-Nkari", + "level9": "Ibuoro-ItuMbuso" + }, + "ibu": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Sahuan" + }, + "iby": { + "level0": "Ijoid", + "level1": "Ijo", + "level2": "Eastern Ijo", + "level3": "Nikio", + "level4": "Kio Ijo" + }, + "ica": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede", + "level7": "Western Ede", + "level8": "Southwestern Ede" + }, + "ich": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Kpan-Icen" + }, + "icl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "West Scandinavian Sign", + "level4": "Danish Sign" + }, + "icr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Western Caribbean Creole", + "level14": "Miskitoic Creole English" + }, + "ida": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia" + }, + "idb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Indo-Portuguesic", + "level15": "Northern Indo-Portuguesic" + }, + "idc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Northern Benue-Congo Plateau", + "level5": "Nuclear Northern Benue-Congo Plateau", + "level6": "Kuturmi-Ajiya" + }, + "idd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede", + "level7": "Western Ede", + "level8": "Southwestern Ede" + }, + "ide": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Efikic", + "level8": "Unclassified Efikic" + }, + "idi": { + "level0": "Pahoturi" + }, + "ido": { + "level0": "Artificial Language", + "level1": "Esperantic" + }, + "idr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Sereic", + "level6": "Sere-Indri", + "level7": "Indri-Togoyo" + }, + "ids": { + "level0": "Bookkeeping" + }, + "idt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Lakalei-Idate" + }, + "idu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Idomoid", + "level4": "Akweya", + "level5": "Etulo-Idoma", + "level6": "Nuclear Idoma", + "level7": "Idoma-Agatu-Okpogu" + }, + "ifa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Nuclear Cordilleran", + "level7": "Ifugaw" + }, + "ifb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Nuclear Cordilleran", + "level7": "Ifugaw", + "level8": "Batad-Mayoyao" + }, + "ife": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede", + "level7": "Western Ede", + "level8": "Southwestern Ede" + }, + "iff": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "Southern Vanuatu", + "level6": "Erromanga" + }, + "ifk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Nuclear Cordilleran", + "level7": "Ifugaw" + }, + "ifm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie" + }, + "ifu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Nuclear Cordilleran", + "level7": "Ifugaw", + "level8": "Batad-Mayoyao" + }, + "ify": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Southern Cordilleran", + "level6": "West Southern Cordilleran", + "level7": "Nuclear Southern Cordilleran", + "level8": "Kalanguya" + }, + "igb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Ebira-Nupoid" + }, + "ige": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Idomoid", + "level4": "Akweya" + }, + "igg": { + "level0": "Ramu", + "level1": "Goam", + "level2": "Tamolan", + "level3": "Unclassified Tamolan" + }, + "igl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid" + }, + "igm": { + "level0": "Ramu", + "level1": "Goam", + "level2": "Ataitan", + "level3": "Tangu-Igom" + }, + "ign": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Bolivian Arawakan", + "level3": "Mojeno-Paunaca", + "level4": "Moje\u00f1o" + }, + "igo": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Gum", + "level5": "Panim-Isebe-Bau" + }, + "igs": { + "level0": "Artificial Language" + }, + "igw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Igwic", + "level7": "Sasaru-Igwe" + }, + "ihb": { + "level0": "Pidgin", + "level1": "Iha-based pidgin" + }, + "ihi": { + "level0": "Bookkeeping" + }, + "ihp": { + "level0": "West Bomberai", + "level1": "Nuclear West Bomberai" + }, + "ihw": { + "level0": "Pama-Nyungan", + "level1": "Ganaic" + }, + "iii": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nuclear Nisoid", + "level8": "Nasu-Nosu" + }, + "iin": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Mantharta", + "level4": "Djiwarli-Thiin" + }, + "ijc": { + "level0": "Ijoid", + "level1": "Ijo", + "level2": "Western Ijo" + }, + "ije": { + "level0": "Ijoid", + "level1": "Ijo", + "level2": "Western Ijo", + "level3": "Inland Ijo", + "level4": "Biseni-Okordia" + }, + "ijj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede", + "level7": "Eastern Ede", + "level8": "Southeastern Ede" + }, + "ijn": { + "level0": "Ijoid", + "level1": "Ijo", + "level2": "Eastern Ijo", + "level3": "Nikio", + "level4": "Kio Ijo" + }, + "ijs": { + "level0": "Ijoid", + "level1": "Ijo", + "level2": "Eastern Ijo" + }, + "ike": { + "level0": "Eskimo-Aleut", + "level1": "Eskimo", + "level2": "Inuit" + }, + "ikh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "North-Central Edoid", + "level6": "Central Plains Edoid", + "level7": "Emaic" + }, + "iki": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Ibino-Iko" + }, + "ikk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Igboid", + "level4": "Nuclear Igboid", + "level5": "Central-Northern Igbo" + }, + "ikl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Northern Benue-Congo Plateau", + "level5": "Nuclear Northern Benue-Congo Plateau" + }, + "iko": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "East-West Central Delta Cross" + }, + "ikp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Igwic", + "level7": "Ikpeshic" + }, + "ikr": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Alaya-Athima", + "level3": "Central Alaya-Athima" + }, + "iks": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "ikt": { + "level0": "Eskimo-Aleut", + "level1": "Eskimo", + "level2": "Inuit" + }, + "ikv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Northern Benue-Congo Plateau" + }, + "ikw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Igboid", + "level4": "Nuclear Igboid" + }, + "ikx": { + "level0": "Kuliak" + }, + "ikz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Nyanza Mara", + "level11": "South Mara", + "level12": "Southwest Mara" + }, + "ila": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata", + "level4": "Lamaholot Barat", + "level5": "North Lembata-Adonara" + }, + "ilb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Botatwe", + "level8": "Greater Eastern Botatwe", + "level9": "Central Eastern Botatwe", + "level10": "Kafue" + }, + "ile": { + "level0": "Artificial Language" + }, + "ilg": { + "level0": "Iwaidjan Proper", + "level1": "Central Iwaidjic" + }, + "ili": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Turkestan", + "level4": "Modern Turkestan", + "level5": "Uyghuric" + }, + "ilk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Southern Cordilleran" + }, + "ill": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Danaw" + }, + "ilo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon" + }, + "ils": { + "level0": "Sign Language", + "level1": "Pidgin Sign Language" + }, + "ilu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Wetar-Atauro", + "level4": "Wetar" + }, + "ilv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "West Lower Cross" + }, + "ima": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid" + }, + "ime": { + "level0": "Bookkeeping" + }, + "imi": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Osum-Wadaginam-Pomoikan", + "level5": "Pomoikan", + "level6": "Anamuxric" + }, + "iml": { + "level0": "Coosan" + }, + "imn": { + "level0": "Border", + "level1": "Warisic", + "level2": "Nuclear Warisic" + }, + "imo": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Hagen", + "level3": "Aua-Gawil" + }, + "imr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar", + "level5": "South Babar", + "level6": "Southwest Babar" + }, + "imy": { + "level0": "Indo-European", + "level1": "Anatolian", + "level2": "Luvo-Lydian", + "level3": "Luvo-Palaic", + "level4": "Luvic", + "level5": "Lyco-Carian", + "level6": "Milyan-Carian" + }, + "ina": { + "level0": "Artificial Language" + }, + "inb": { + "level0": "Quechuan", + "level1": "Colombia-Ecuador Quechua", + "level2": "Ecuadorian Quechua B", + "level3": "Imbabura-Colombia-Oriente Quechua", + "level4": "Colombia-Oriente Quechua" + }, + "ind": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Standard Malay-Indonesian" + }, + "ing": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central Alaska-Yukon Athabaskan", + "level4": "Koyukonic" + }, + "inh": { + "level0": "Nakh-Daghestanian", + "level1": "Nakh", + "level2": "Chechen-Ingush" + }, + "inl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "Malaysian Sign", + "level5": "Indonesian Sign" + }, + "inm": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Sayhadic" + }, + "inn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Nuclear Cordilleran" + }, + "ino": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Siane-Yagaria", + "level5": "Kamano-Yagaria" + }, + "inp": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Purus-Chamicuro", + "level3": "Purus" + }, + "ins": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Indo-Pakistani-Nepalese Sign", + "level3": "Indo-Pakistani Sign" + }, + "int": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Southern Burmish", + "level5": "Mranmaic" + }, + "inz": { + "level0": "Chumashan", + "level1": "Southern Chumashan", + "level2": "Central Chumashan" + }, + "ior": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Outer South Ethiopic", + "level6": "TT-Group", + "level7": "Peripheral Western Gurage" + }, + "iou": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Wantoatic" + }, + "iow": { + "level0": "Siouan", + "level1": "Mississippi Valley", + "level2": "Winnebago-Chiwere" + }, + "ipi": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Engan" + }, + "ipo": { + "level0": "Anim", + "level1": "Inland Gulf of Papua" + }, + "iqu": { + "level0": "Zaparoan", + "level1": "Iquito-Arabela", + "level2": "Cahuarano-Iquito" + }, + "ire": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Southwest Cenderawasih Bay", + "level6": "Yaur-Yerisiam" + }, + "irh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "Banda-Geser", + "level4": "Seran Laut", + "level5": "Koiwai-Irarutu", + "level6": "Irarutic" + }, + "iri": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Zaric" + }, + "irk": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "South Cushitic", + "level3": "Greater West Rift South Cushitic", + "level4": "West Rift South Cushitic", + "level5": "Northern West Rift South Cushitic", + "level6": "Iraqwoid" + }, + "irr": { + "level0": "Bookkeeping" + }, + "iru": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Irula-Muduga" + }, + "irx": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Asmat-Kamoro", + "level3": "Sabakor" + }, + "iry": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Mangyan" + }, + "isa": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Unclassified Goroka" + }, + "isc": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Poyanawa Subgroup" + }, + "isd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Cagayan Valley" + }, + "ise": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Italian Sign" + }, + "isg": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic" + }, + "ish": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "North-Central Edoid", + "level6": "Central Plains Edoid" + }, + "isi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Ekoid-Mbe", + "level6": "Ekoid", + "level7": "Bakor-Ejagham", + "level8": "Bakor", + "level9": "Northern Bakor", + "level10": "Abanyom-Nkem-Nkum" + }, + "isk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Sanglechi-Ishkashimi" + }, + "isl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "North Germanic", + "level5": "West Scandinavian", + "level6": "Icelandic-Faroese" + }, + "ism": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Sarmi", + "level8": "Sobeic" + }, + "isn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Nyaturu-Nilamba" + }, + "iso": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Southwestern Edoid" + }, + "isr": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "DGSic" + }, + "ist": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Italo-Dalmatian", + "level9": "Dalmatian Romance" + }, + "isu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "West Ring", + "level10": "Aghemic" + }, + "ita": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Italo-Dalmatian", + "level9": "Italian Romance" + }, + "itb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Itneg" + }, + "itd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Northern Murutic", + "level8": "Sumambu-Tagal", + "level9": "Tidung-Bulusu", + "level10": "Tidung" + }, + "ite": { + "level0": "Chapacuran", + "level1": "Moreic-Waric", + "level2": "Moreic", + "level3": "Kujubim-More" + }, + "iti": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Itneg" + }, + "itk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Italian" + }, + "itl": { + "level0": "Chukotko-Kamchatkan", + "level1": "Kamchatkan" + }, + "itm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Ibuoroic", + "level8": "Ibuoro-ItuMbuso-Nkari", + "level9": "Ibuoro-ItuMbuso" + }, + "itr": { + "level0": "Left May", + "level1": "Western Left May", + "level2": "Iteri-Bo" + }, + "its": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri" + }, + "itt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Nuclear Cordilleran", + "level7": "Bontok-Kankanay", + "level8": "Kankanay", + "level9": "Maeng-Northern Kankanay" + }, + "itu": { + "level0": "Bookkeeping" + }, + "itv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Cagayan Valley", + "level4": "Ibanagic", + "level5": "Gaddangic" + }, + "itw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Ibuoroic" + }, + "itx": { + "level0": "Tor-Orya", + "level1": "Tor" + }, + "ity": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Kalinga", + "level8": "Masadiit" + }, + "itz": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Yucatecan", + "level3": "Nuclear Yucatecan" + }, + "ium": { + "level0": "Hmong-Mien", + "level1": "Mienic", + "level2": "Mien-Mun" + }, + "ivb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Batanic" + }, + "ivv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Batanic", + "level3": "Yami-Itbayat" + }, + "iwk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Southern Cordilleran", + "level6": "West Southern Cordilleran", + "level7": "Nuclear Southern Cordilleran" + }, + "iwm": { + "level0": "Sepik", + "level1": "Iwam-Wogamus", + "level2": "Iwamic" + }, + "iwo": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok" + }, + "iws": { + "level0": "Sepik", + "level1": "Iwam-Wogamus", + "level2": "Iwamic" + }, + "ixc": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Ixcatec-Chocho-Popolocan" + }, + "ixl": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Mamean", + "level4": "Ixilan" + }, + "iya": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Osse" + }, + "iyo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid" + }, + "iyx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie", + "level17": "Northwest Kasai-Ngounie", + "level18": "Nzebi-Laali-Yaa", + "level19": "Laali-Yaa" + }, + "izh": { + "level0": "Uralic", + "level1": "Finnic", + "level2": "Coastal Finnic", + "level3": "Neva", + "level4": "North Finnic", + "level5": "Ladogan" + }, + "izi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Igboid", + "level4": "Nuclear Igboid", + "level5": "Central-Northern Igbo" + }, + "izm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Kauru" + }, + "izr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Zaric", + "level6": "Nuclear Zaric", + "level7": "Izeric" + }, + "jaa": { + "level0": "Arawan", + "level1": "Madi-Madiha" + }, + "jab": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Northwestern Benue-Congo Plateau", + "level6": "Hyamic" + }, + "jac": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Kanjobalan-Chujean", + "level4": "Kanjobalan", + "level5": "Kanjobal-Jacaltec" + }, + "jad": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "West Manding", + "level9": "Xasonka" + }, + "jae": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "North Huon Gulf linkage" + }, + "jaf": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Teraic", + "level5": "Western Tera" + }, + "jah": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian" + }, + "jai": { + "level0": "Bookkeeping" + }, + "jaj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Santa Isabel", + "level10": "Central Santa Isabel", + "level11": "Zazao-Blanga" + }, + "jak": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Greater Riau-Johoric" + }, + "jal": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Three Rivers", + "level4": "Amalumute" + }, + "jam": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Western Caribbean Creole", + "level14": "Jamaicanic" + }, + "jao": { + "level0": "Pama-Nyungan", + "level1": "Ngarna" + }, + "jap": { + "level0": "Bookkeeping" + }, + "jaq": { + "level0": "Anim", + "level1": "Marind-Boazi-Yaqai", + "level2": "Yaqayic" + }, + "jar": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan", + "level7": "Jarawaic" + }, + "jas": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Javanesic", + "level3": "Modern Javanese", + "level4": "Global Javanese" + }, + "jat": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Greater Panjabic", + "level9": "Hindko-Siraiki", + "level10": "Siraikic" + }, + "jau": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Southwest Cenderawasih Bay", + "level6": "Yaur-Yerisiam" + }, + "jav": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Javanesic", + "level3": "Modern Javanese", + "level4": "Global Javanese" + }, + "jax": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Northern Sumatra Malay" + }, + "jay": { + "level0": "Pama-Nyungan", + "level1": "Yuulngu" + }, + "jaz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian" + }, + "jbe": { + "level0": "Bookkeeping" + }, + "jbi": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Badjiri-Eastern Karnic" + }, + "jbj": { + "level0": "South Bird's Head Family", + "level1": "East South Bird's Head", + "level2": "Kemberanic" + }, + "jbk": { + "level0": "Turama-Kikori", + "level1": "Turama-Omatian" + }, + "jbm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Tarokoid", + "level5": "Bijimic-Sur-Shall", + "level6": "Kwangic", + "level7": "Vaghat" + }, + "jbn": { + "level0": "Afro-Asiatic", + "level1": "Berber" + }, + "jbo": { + "level0": "Artificial Language" + }, + "jbr": { + "level0": "Tor-Orya", + "level1": "Tor", + "level2": "Coastal Tor" + }, + "jbt": { + "level0": "Nuclear-Macro-Je", + "level1": "Jabuti" + }, + "jbu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Jukun-Mbembe-Wurbo", + "level6": "Jukun", + "level7": "Jibu-Wase", + "level8": "Jibuic" + }, + "jcs": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "jct": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Northwest Kipchak", + "level5": "West Kipchak", + "level6": "Crimean Tatar-Urum", + "level7": "Crimeaic" + }, + "jda": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Lahauli-Spiti", + "level7": "Spiti-Jad" + }, + "jdg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Sindhic", + "level9": "Lasi-Jadgali" + }, + "jdt": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Caucasian Tat" + }, + "jeb": { + "level0": "Cahuapanan" + }, + "jee": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Chaurasiya" + }, + "jeh": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric", + "level3": "Kayong-Jeh-Halang", + "level4": "Jeh-Halang" + }, + "jei": { + "level0": "Yam", + "level1": "Morehead-Maro" + }, + "jek": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Jogo-Jeri", + "level6": "Jeri" + }, + "jel": { + "level0": "Bulaka River", + "level1": "Jelmek" + }, + "jen": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bikwin-Jen", + "level5": "Southern Bikwin-Jen", + "level6": "Jen", + "level7": "Doso-Dza" + }, + "jer": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "North-Central Jos", + "level10": "Boze-Loro" + }, + "jet": { + "level0": "Border", + "level1": "Warisic", + "level2": "Nuclear Warisic" + }, + "jeu": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Dangla-Mabire-Birgit", + "level6": "Dangla", + "level7": "Unclassified Dangla" + }, + "jgb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Ngbele-Ngenda", + "level15": "Extreme North Vestigial Suffixes Bantu" + }, + "jgo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Bamileke", + "level10": "West Bamileke" + }, + "jhi": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "North Aslian", + "level4": "Maniq-Menraq-Batek", + "level5": "Menraq-Batek" + }, + "jhs": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "jia": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Kotoko-Buduma", + "level5": "Kotoko Meridional" + }, + "jib": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Jukun-Mbembe-Wurbo", + "level6": "Jukun", + "level7": "Jibu-Wase", + "level8": "Jibuic" + }, + "jic": { + "level0": "Jicaquean" + }, + "jid": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Ninzic" + }, + "jie": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "Unclassified Biu-Mandara" + }, + "jig": { + "level0": "Mirndi" + }, + "jih": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic", + "level4": "Gyalrongic", + "level5": "West Gyalrongic", + "level6": "Horpa" + }, + "jii": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Eastern Omo-Tana", + "level8": "Baiso-Jiiddu" + }, + "jil": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Mindjim", + "level4": "Upper Minjim" + }, + "jim": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic", + "level5": "Gudeic", + "level6": "Gude-Jimi-Zizilivakan" + }, + "jio": { + "level0": "Tai-Kadai", + "level1": "Hlaic" + }, + "jiq": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic", + "level4": "Gyalrongic", + "level5": "West Gyalrongic" + }, + "jit": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Suguti" + }, + "jiu": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Jino" + }, + "jiv": { + "level0": "Chicham", + "level1": "Shuaric", + "level2": "Huambisa-Shuar" + }, + "jiy": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Jino" + }, + "jje": { + "level0": "Koreanic" + }, + "jka": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "West Alor-Straits-Pantar", + "level4": "Kaera-Straits" + }, + "jkm": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Central Karen" + }, + "jko": { + "level0": "East Strickland", + "level1": "Kubo-Samo-Bibo" + }, + "jkp": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Southern Karen" + }, + "jkr": { + "level0": "Sino-Tibetan", + "level1": "Macro-Tani", + "level2": "Koro-Holon" + }, + "jks": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "jku": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan", + "level7": "Jaku-Gubi" + }, + "jle": { + "level0": "Narrow Talodi", + "level1": "Buram-Saraf", + "level2": "Buram Hill Chain" + }, + "jls": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "American Sign" + }, + "jma": { + "level0": "Dagan", + "level1": "Central Dagan" + }, + "jmb": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.2", + "level5": "Nuclear West Chadic B.2" + }, + "jmc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Kilimanjaro-Taita", + "level9": "Kilimanjaro Bantu", + "level10": "Chaga", + "level11": "West Kilimanjaro" + }, + "jmd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tanimbar-Bomberai", + "level3": "Nuclear Tanimbar-Bomberai", + "level4": "Yamdena-Onin" + }, + "jmi": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi East", + "level6": "Guruntumic" + }, + "jml": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Indo-Aryan Northern zone", + "level8": "Eastern Pahari" + }, + "jmn": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Central Naga", + "level4": "Yimchingric", + "level5": "Makuric" + }, + "jmr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Southeast Western Oti-Volta", + "level13": "Kamara-Hanga" + }, + "jms": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Beboid", + "level6": "Eastern Beboid", + "level7": "Bebe-Kemezung", + "level8": "Naki-Kemezung", + "level9": "Nakic" + }, + "jmw": { + "level0": "Turama-Kikori", + "level1": "Turama-Omatian" + }, + "jmx": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Guerrero Mixtec", + "level7": "Coicoyan-Metlatonoc" + }, + "jna": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Western West Himalayish", + "level4": "Kinnauric", + "level5": "Thebor" + }, + "jnd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Bagri-Jandavra" + }, + "jng": { + "level0": "Yangmanic" + }, + "jni": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "North-Central Jos" + }, + "jnj": { + "level0": "Ta-Ne-Omotic" + }, + "jnl": { + "level0": "Sino-Tibetan", + "level1": "Raji-Raute", + "level2": "Raute-Rawat" + }, + "jns": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali" + }, + "job": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "Forest Kivu", + "level12": "Fuliiric", + "level13": "Fuliiru-Vira" + }, + "jod": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding", + "level9": "Maninka-Mori" + }, + "jog": { + "level0": "Bookkeeping", + "level1": "Pending Report Release" + }, + "jor": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup II", + "level7": "Warazu-Sirionoid", + "level8": "Sirionoid" + }, + "jos": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Arab Sign", + "level3": "Levantine-Iraqi Sign" + }, + "jow": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Duun-Bobo", + "level4": "Duun-Jo" + }, + "jpn": { + "level0": "Japonic", + "level1": "Japanesic", + "level2": "Japan-Taiwan Japanese" + }, + "jpr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Farsic" + }, + "jqr": { + "level0": "Aymaran", + "level1": "Tupe" + }, + "jra": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic", + "level4": "Chamic", + "level5": "Rade-Jarai" + }, + "jrr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Jukun-Mbembe-Wurbo", + "level6": "Wurbo-Wannu", + "level7": "Wurbo" + }, + "jrt": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3", + "level6": "Kofyar-Mushere-Chip", + "level7": "Kofyaric" + }, + "jru": { + "level0": "Cariban", + "level1": "Opon-Yukpan", + "level2": "Yukpan" + }, + "jsl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "JSLic" + }, + "jua": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VI", + "level6": "Kawahiva", + "level7": "Nuclear Kawahiva" + }, + "jub": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Jukun-Mbembe-Wurbo", + "level6": "Wurbo-Wannu" + }, + "juc": { + "level0": "Tungusic", + "level1": "Manchu-Jurchen" + }, + "jud": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding", + "level9": "Maninka-Mori" + }, + "juh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Jukun-Mbembe-Wurbo", + "level6": "Jukun", + "level7": "Kororofa", + "level8": "Kona" + }, + "jui": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Thura-Yura", + "level3": "Core Thura Yura", + "level4": "Unclassified Core Thura-Yura" + }, + "juk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Jukun-Mbembe-Wurbo", + "level6": "Jukun", + "level7": "Kororofa" + }, + "jul": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Sherpa-Jirel" + }, + "jum": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Burun", + "level3": "Southern Burun" + }, + "jun": { + "level0": "Austroasiatic", + "level1": "Mundaic" + }, + "juo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Jukun-Mbembe-Wurbo", + "level6": "Jukun", + "level7": "Kororofa", + "level8": "Kona" + }, + "jup": { + "level0": "Naduhup", + "level1": "Eastern Naduhup", + "level2": "Hup-Yuhup" + }, + "jur": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Jurunic" + }, + "jus": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "jut": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "North Germanic", + "level5": "South Scandinavian" + }, + "juu": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi East", + "level6": "Guruntumic", + "level7": "Tala-Sho-Zangwal" + }, + "juw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Jukun-Mbembe-Wurbo", + "level6": "Jukun", + "level7": "Jibu-Wase" + }, + "juy": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "Sora-Juray-Gorum", + "level3": "Sora-Juray" + }, + "jvd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch", + "level6": "Middle-Modern Dutch", + "level7": "Modern Dutch", + "level8": "Global Dutch" + }, + "jvn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Javanesic", + "level3": "Modern Javanese", + "level4": "Global Javanese" + }, + "jwi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Central Tano", + "level7": "Bia", + "level8": "Southern Bia", + "level9": "Jwira-Nzima" + }, + "jye": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Arabian Peninsula Arabic", + "level7": "Judeo-Muslim Sanaani Arabic" + }, + "jyy": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Bagirmic", + "level6": "Morom-Jaya-Naba" + }, + "kaa": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Southeast Kipchak", + "level5": "South Kipchak" + }, + "kab": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Kabyle-Atlas Berber" + }, + "kac": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Jingpho-Luish", + "level3": "Jingpho" + }, + "kad": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Northern Benue-Congo Plateau", + "level5": "Nuclear Northern Benue-Congo Plateau" + }, + "kae": { + "level0": "Austronesian", + "level1": "East Formosan", + "level2": "Northern East Formosan" + }, + "kaf": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Kazhouish" + }, + "kag": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Melanau-Kajang", + "level5": "Kajang", + "level6": "Kajaman-Lahanan" + }, + "kah": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Fer-Gula" + }, + "kai": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic" + }, + "kaj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Zaric", + "level6": "Nuclear Zaric", + "level7": "Katabic" + }, + "kak": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Southern Cordilleran", + "level6": "West Southern Cordilleran", + "level7": "Nuclear Southern Cordilleran", + "level8": "Kalanguya" + }, + "kal": { + "level0": "Eskimo-Aleut", + "level1": "Eskimo", + "level2": "Inuit", + "level3": "Greenlandic Inuit" + }, + "kam": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Central Kenya Bantu", + "level9": "Kamba-Dhaisu" + }, + "kan": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Badaga-Kannada", + "level5": "Kannadoid", + "level6": "Nuclear Kannaoid" + }, + "kao": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "West Manding", + "level9": "Xasonka" + }, + "kap": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Tsezic", + "level4": "East Tsezic" + }, + "kaq": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Chama subgroup", + "level5": "Shipibo-Konibo-Kapanawa" + }, + "kas": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Kashmiric" + }, + "kat": { + "level0": "Kartvelian", + "level1": "Georgian-Zan", + "level2": "Georgic" + }, + "kav": { + "level0": "Bookkeeping" + }, + "kaw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Javanesic" + }, + "kax": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Mainland North Halmaheran", + "level3": "Kao River", + "level4": "Paguic" + }, + "kay": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani" + }, + "kaz": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Southeast Kipchak", + "level5": "South Kipchak" + }, + "kba": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Mirning" + }, + "kbb": { + "level0": "Cariban", + "level1": "Parukotoan" + }, + "kbc": { + "level0": "Guaicuruan" + }, + "kbd": { + "level0": "Abkhaz-Adyge", + "level1": "Circassian" + }, + "kbe": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Northeastern Pama", + "level4": "Umpilaic" + }, + "kbf": { + "level0": "Bookkeeping" + }, + "kbg": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Southern Tibetic", + "level7": "Unclassified Southern Tibetic" + }, + "kbi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Sarmi", + "level8": "Kaptiau-Tarpia" + }, + "kbj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Ngbele-Ngenda", + "level15": "Extreme North Vestigial Suffixes Bantu" + }, + "kbk": { + "level0": "Koiarian", + "level1": "Koiaric", + "level2": "Koita-Koiari" + }, + "kbl": { + "level0": "Saharan", + "level1": "Western Saharan", + "level2": "Kanuri-Kanembu", + "level3": "Kanembuic" + }, + "kbm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage" + }, + "kbn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Central Mbum", + "level6": "Karangic", + "level7": "Kare-Pana" + }, + "kbo": { + "level0": "Central Sudanic", + "level1": "Moru-Madi", + "level2": "Central Moru-Madi", + "level3": "Kalikoic" + }, + "kbp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Eastern Grusi", + "level9": "Kabiyeic" + }, + "kbq": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Siane-Yagaria", + "level5": "Kamano-Yagaria" + }, + "kbr": { + "level0": "Ta-Ne-Omotic", + "level1": "Kefoid", + "level2": "South Gonga" + }, + "kbs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "B10-B30", + "level8": "Okani (B.30)", + "level9": "Northern Okani" + }, + "kbt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "West Central Papuan linkage" + }, + "kbu": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Unclassified Rajasthani" + }, + "kbv": { + "level0": "Senagi" + }, + "kbw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Kairiru-Manam", + "level8": "Kairiruic linkage", + "level9": "Kaiep-Terebu" + }, + "kbx": { + "level0": "Keram", + "level1": "East Keram" + }, + "kby": { + "level0": "Saharan", + "level1": "Western Saharan", + "level2": "Kanuri-Kanembu", + "level3": "Kanuric" + }, + "kbz": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.4", + "level5": "Ronic", + "level6": "Mundat-Karfa" + }, + "kca": { + "level0": "Uralic", + "level1": "Khantyic", + "level2": "Northern Khanty" + }, + "kcb": { + "level0": "Angan", + "level1": "Nuclear Angan", + "level2": "Wojokesic" + }, + "kcc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "East-West Central Delta Cross", + "level7": "Lokoic", + "level8": "Lubila-Lokaa" + }, + "kcd": { + "level0": "Yam", + "level1": "Kanum", + "level2": "Ngkrn-Ngkantr", + "level3": "Ngkantr" + }, + "kce": { + "level0": "Unattested", + "level1": "Atlantic-Congo (Unattested)" + }, + "kcf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo" + }, + "kcg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Zaric", + "level6": "Nuclear Zaric", + "level7": "Katabic" + }, + "kch": { + "level0": "Unattested", + "level1": "Atlantic-Congo (Unattested)" + }, + "kci": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Northwestern Benue-Congo Plateau", + "level6": "Gyong-Kamantan" + }, + "kcj": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Wolof-BKK", + "level3": "Nyun", + "level4": "Buy" + }, + "kck": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Shona (S.10)", + "level9": "Kalanga-Nambya" + }, + "kcl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "North Huon Gulf linkage" + }, + "kcm": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Fer-Gula" + }, + "kcn": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Egyptic Arabic", + "level7": "Egypto-Sudanic Arabic", + "level8": "Sudanese-Chadian Arabic", + "level9": "East Sudanic Arabic" + }, + "kco": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Cromwell", + "level5": "Dallman", + "level6": "Kinalakna-Kumukio" + }, + "kcp": { + "level0": "Kadugli-Krongo", + "level1": "Central-Western Kadugli-Krongo", + "level2": "Katcha-Kadugli-Miri-Kanga" + }, + "kcq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Tula-Longuda", + "level6": "Tula-Waja", + "level7": "Tulaic", + "level8": "Tula-Ma-Yebu", + "level9": "Awak-Kamo" + }, + "kcr": { + "level0": "Katla-Tima", + "level1": "Katla-Julud" + }, + "kcs": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3", + "level6": "Goemaic", + "level7": "Talic", + "level8": "Piapung-Koenoem" + }, + "kct": { + "level0": "Ramu", + "level1": "Lower Ramu", + "level2": "Ottilien", + "level3": "Watam-Kaian" + }, + "kcu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "East Ruvu", + "level11": "Central East Ruvu" + }, + "kcv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Ruund-Salampasu", + "level11": "Lunda-Ruund-Kete", + "level12": "Ruund-Kete" + }, + "kcw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu" + }, + "kcx": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "East Ometo" + }, + "kcy": { + "level0": "Songhay", + "level1": "Northwest Songhay", + "level2": "Northern Songhay" + }, + "kcz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Sukuma-Nyamwezi (F.20)", + "level9": "Nyamwezic" + }, + "kda": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "New South Wales Pama-Nyungan", + "level3": "Yuin-Kuri", + "level4": "Kuri", + "level5": "Hunter-Hastings" + }, + "kdc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "East Ruvu", + "level11": "Central East Ruvu", + "level12": "Kutu-Zaramo" + }, + "kdd": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Wati", + "level3": "Pintupic", + "level4": "Nuclear Pintupic", + "level5": "Wangkatja-Tjarra", + "level6": "Tjarra" + }, + "kde": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Ruvuma", + "level9": "Makonde-Makwe" + }, + "kdf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Mengenic", + "level9": "Mamusa-Mengen" + }, + "kdg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Sabi", + "level8": "Malungu-Central Sabi", + "level9": "Central Sabi", + "level10": "Bisa-Lamba (M.50)" + }, + "kdh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Eastern Grusi", + "level9": "Tem-Chala" + }, + "kdi": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Southern Lwoo", + "level4": "Lango-Kumam" + }, + "kdj": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Teso-Turkana", + "level4": "Turkanic" + }, + "kdk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Southern New Caledonian", + "level8": "Extreme Southern New Caledonian" + }, + "kdl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Kambari-Cicipu", + "level6": "Kambaric", + "level7": "West Kambaric" + }, + "kdm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Northwestern Benue-Congo Plateau", + "level6": "Gyong-Kamantan" + }, + "kdn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Tumbuka-Sena-Nyanja", + "level8": "Sena-Nyanja" + }, + "kdp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Ninzic", + "level5": "Kanufi-Ninkyob-Angan" + }, + "kdq": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Kochic" + }, + "kdr": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Northwest Kipchak", + "level5": "West Kipchak" + }, + "kds": { + "level0": "Bookkeeping" + }, + "kdt": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "West Katuic", + "level3": "Kuy-Souei" + }, + "kdu": { + "level0": "Nubian", + "level1": "Central Nubian", + "level2": "Kordofan Nubian", + "level3": "Eastern Kordofan Nubian" + }, + "kdv": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Jingpho-Luish", + "level3": "Luish", + "level4": "Chakpa-Kadu-Ganan" + }, + "kdw": { + "level0": "Mombum-Koneraw" + }, + "kdx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo" + }, + "kdy": { + "level0": "Tor-Orya", + "level1": "Tor", + "level2": "Coastal Tor" + }, + "kdz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Nkambe", + "level9": "Mfumteic" + }, + "kea": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Upper Guinea Portuguese" + }, + "keb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Ngomic", + "level8": "Nuclear Ngomic", + "level9": "Akeleic" + }, + "kec": { + "level0": "Kadugli-Krongo" + }, + "ked": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza" + }, + "kee": { + "level0": "Keresan" + }, + "kef": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Western Gbe", + "level5": "Kpesi-Waci" + }, + "keg": { + "level0": "Temeinic" + }, + "keh": { + "level0": "Ndu", + "level1": "Nuclear Ndu", + "level2": "Sawosic", + "level3": "Iatmulic" + }, + "kei": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tanimbar-Bomberai", + "level3": "Nuclear Tanimbar-Bomberai", + "level4": "Kei-Fordata" + }, + "kej": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "kek": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Quichean" + }, + "kem": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Timor", + "level3": "Kemak-Tukudede" + }, + "ken": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Mamfe" + }, + "keo": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Barian" + }, + "kep": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid", + "level10": "Yerukula-Korava-Kaikadi" + }, + "keq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Halbic" + }, + "ker": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.3" + }, + "kes": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Central Delta", + "level5": "Kugboic" + }, + "ket": { + "level0": "Yeniseian", + "level1": "Northern Yeniseian" + }, + "keu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Ka-Togo", + "level4": "Kebu-Animere" + }, + "kev": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "kew": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Kewa-Huli", + "level3": "Sau-Angal-Kewa", + "level4": "Angal-Kewa", + "level5": "Kewa" + }, + "kex": { + "level0": "Bookkeeping" + }, + "key": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Macro-Oriya" + }, + "kez": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "North-South Central Delta Cross", + "level7": "Koring-Kukele", + "level8": "Kukele-Uzekwe" + }, + "kfa": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu" + }, + "kfb": { + "level0": "Dravidian", + "level1": "Central Dravidian", + "level2": "Kolami-Naiki" + }, + "kfc": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Konda-Kui" + }, + "kfd": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "South-Western Dravidian", + "level4": "Koraga" + }, + "kfe": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota" + }, + "kff": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Gondi", + "level4": "Southeast Gondi", + "level5": "South Bastar Gondi-Koya" + }, + "kfg": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "South-Western Dravidian", + "level4": "Tuluic" + }, + "kfh": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "kfi": { + "level0": "Bookkeeping" + }, + "kfj": { + "level0": "Bookkeeping" + }, + "kfk": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Western West Himalayish", + "level4": "Kinnauric" + }, + "kfl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "Center Ring" + }, + "kfm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Central Iran Kermanic", + "level8": "Nuclear Central Iran Kermanic" + }, + "kfn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "West Ring" + }, + "kfo": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding", + "level9": "Maninka-Mori", + "level10": "Koro-Koyaga" + }, + "kfp": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Mundaric", + "level5": "Kodaku-Korwa" + }, + "kfq": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda" + }, + "kfr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Sindhic", + "level9": "Sindhi-Kachchi" + }, + "kfs": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Kangric-Chamealic-Bhattiyali", + "level8": "Chamealic" + }, + "kft": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Mewaric" + }, + "kfu": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Southern zone", + "level7": "Marathic" + }, + "kfv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga", + "level10": "Unclassified Gauda-Banga" + }, + "kfw": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Kolhrengic" + }, + "kfx": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Nuclear Himachali" + }, + "kfy": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Indo-Aryan Northern zone", + "level8": "Central Pahari" + }, + "kfz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur" + }, + "kga": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding", + "level9": "Maninka-Mori", + "level10": "Koro-Koyaga" + }, + "kgb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "Maya-Matbat", + "level6": "Raja Ampat Maya" + }, + "kge": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Lampungic", + "level3": "Pesisir" + }, + "kgf": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Rawlinson", + "level5": "Pindiu" + }, + "kgh": { + "level0": "Bookkeeping" + }, + "kgi": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "Malaysian Sign" + }, + "kgj": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kham-Magar-Chepang", + "level4": "Kham", + "level5": "Gamale-Parbate" + }, + "kgk": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup I", + "level7": "Tupi-Guarani Subgroup I.A", + "level8": "Paraguay-Brazil Guarani", + "level9": "Kaiowa" + }, + "kgl": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Southern Maric" + }, + "kgm": { + "level0": "Bookkeeping" + }, + "kgn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Northern Tatic" + }, + "kgo": { + "level0": "Kadugli-Krongo", + "level1": "Central-Western Kadugli-Krongo", + "level2": "Krongo-Tumtum" + }, + "kgp": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Southern Je", + "level3": "Kaingang-Xokleng", + "level4": "Kaingangic" + }, + "kgq": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Asmat-Kamoro" + }, + "kgs": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "North Coast Pama-Nyungan", + "level3": "Gumbaynggiric" + }, + "kgt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Vutic" + }, + "kgu": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Omosan" + }, + "kgv": { + "level0": "West Bomberai" + }, + "kgw": { + "level0": "Maybratic" + }, + "kgx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Southern Kaili-Wolio", + "level5": "Island Kaili-Wolio", + "level6": "Wolio-Kamaru" + }, + "kgy": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Kyirong-Kagate", + "level9": "Gyalsumdo-Nubri-Kyirong" + }, + "kha": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Khasian", + "level3": "Khasi-Pnar-Lyngngam", + "level4": "Khasi-Pnar" + }, + "khb": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Sukaphic", + "level11": "Northern Shanic", + "level12": "Sipsongpannic" + }, + "khc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Tukangbesi-Bonerate", + "level8": "Tukang Besi" + }, + "khd": { + "level0": "Yam", + "level1": "Kanum", + "level2": "Ngkrn-Ngkantr", + "level3": "Ngkantr" + }, + "khe": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Becking-Dawi" + }, + "khf": { + "level0": "Austroasiatic", + "level1": "Khmuic", + "level2": "Khmu'" + }, + "khg": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Kham-Hor" + }, + "khj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Northern Benue-Congo Plateau", + "level5": "Nuclear Northern Benue-Congo Plateau", + "level6": "Kuturmi-Ajiya" + }, + "khk": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Eastern Mongolic", + "level3": "Khalkha-Buriat", + "level4": "Mongolian" + }, + "khl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Ngero", + "level8": "Eastern Ngero", + "level9": "Kaliai-Kove" + }, + "khm": { + "level0": "Austroasiatic", + "level1": "Khmeric" + }, + "khn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Khandesic" + }, + "kho": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Saka-Wakhi", + "level5": "Saka" + }, + "khq": { + "level0": "Songhay", + "level1": "Northwest Songhay" + }, + "khr": { + "level0": "Austroasiatic", + "level1": "Mundaic" + }, + "khs": { + "level0": "Bosavi", + "level1": "Bosavi Watershed" + }, + "kht": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Sukaphic", + "level11": "Assam Tai B" + }, + "khu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia", + "level11": "Nyaneka-Nkhumbi" + }, + "khv": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Tsezic", + "level4": "West Tsezic" + }, + "khw": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan" + }, + "khx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega", + "level8": "Mituku-Lega", + "level9": "Lega", + "level10": "Western Lega" + }, + "khy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke", + "level11": "Kele-Lombo" + }, + "khz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "Sinagoro-Keapara", + "level9": "Hula-Keapara" + }, + "kia": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Kim-Besme-Goundo" + }, + "kib": { + "level0": "Heibanic", + "level1": "West-Central Heibanic", + "level2": "Central Heibanic" + }, + "kic": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian", + "level4": "Fox" + }, + "kid": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Yemne-Kimbi" + }, + "kie": { + "level0": "Maban", + "level1": "Mabang", + "level2": "Runga-Kibet" + }, + "kif": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kham-Magar-Chepang", + "level4": "Kham", + "level5": "Gamale-Parbate", + "level6": "Parbate Kham" + }, + "kig": { + "level0": "Kolopom", + "level1": "Kimaama-Riantana" + }, + "kih": { + "level0": "Border", + "level1": "Bewani", + "level2": "Pagi-Kilmeri" + }, + "kii": { + "level0": "Caddoan", + "level1": "Northern Caddoan", + "level2": "Pawnee-Kitsai" + }, + "kij": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Kilivila-Misima", + "level8": "Kilivilic", + "level9": "Kilivila-Muyuw" + }, + "kik": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Central Kenya Bantu", + "level9": "Gikuyu-Temi" + }, + "kil": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.2", + "level5": "Nuclear West Chadic B.2", + "level6": "Central West Chadic B.2", + "level7": "Warji-Gala-Kariya" + }, + "kim": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "South Siberian Turkic", + "level3": "Sayan-Yenisei Turkic", + "level4": "Sayan" + }, + "kin": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "West Highlands Kivu" + }, + "kio": { + "level0": "Kiowa-Tanoan" + }, + "kip": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kham-Magar-Chepang", + "level4": "Kham" + }, + "kiq": { + "level0": "Kaure-Kosare" + }, + "kir": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Southeast Kipchak", + "level5": "East Kipchak" + }, + "kis": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Kairiru-Manam", + "level8": "Manamic linkage", + "level9": "Kis-Wogeo" + }, + "kit": { + "level0": "Pahoturi" + }, + "kiu": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Zaza" + }, + "kiv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Sukuma-Nyamwezi (F.20)", + "level9": "Nyamwezic" + }, + "kiw": { + "level0": "Kiwaian" + }, + "kix": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "South Patkaian", + "level4": "Southeastern Patkaian", + "level5": "Lainongic", + "level6": "Khiamniungic" + }, + "kiy": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "West Tariku", + "level3": "Fayu-Kirikiri" + }, + "kiz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Southern Tanzania Highlands Bantu", + "level9": "Kisi-Pangwa" + }, + "kja": { + "level0": "Nimboranic", + "level1": "Outer Nimboranic", + "level2": "Mlap-Gresi-Kemtuik" + }, + "kjb": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Kanjobalan-Chujean", + "level4": "Kanjobalan", + "level5": "Kanjobal-Jacaltec" + }, + "kjc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Makassaric", + "level5": "Konjo" + }, + "kjd": { + "level0": "Kiwaian" + }, + "kje": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Luangic-Kisaric", + "level5": "Kisaric" + }, + "kjf": { + "level0": "Bookkeeping" + }, + "kjg": { + "level0": "Austroasiatic", + "level1": "Khmuic", + "level2": "Khmu'" + }, + "kjh": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "South Siberian Turkic", + "level3": "Sayan-Yenisei Turkic", + "level4": "Yenisey Turkic" + }, + "kji": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "West Santa Isabel" + }, + "kjj": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian" + }, + "kjk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Makassaric", + "level5": "Konjo" + }, + "kjl": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kham-Magar-Chepang", + "level4": "Kham", + "level5": "Gamale-Parbate", + "level6": "Parbate Kham" + }, + "kjm": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Khao-Bit" + }, + "kjn": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Alaya-Athima", + "level3": "Southwestern Alaya-Athima" + }, + "kjo": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Nuclear Himachali" + }, + "kjp": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Peripheral Karen", + "level3": "Pwo", + "level4": "Eastern-Western Pwo Karen" + }, + "kjq": { + "level0": "Keresan" + }, + "kjr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Eastern Yapen" + }, + "kjs": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Kewa-Huli", + "level3": "Sau-Angal-Kewa", + "level4": "Angal-Kewa", + "level5": "Kewa", + "level6": "Southeast Kewa" + }, + "kjt": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Peripheral Karen", + "level3": "Pwo", + "level4": "Northern Pwo Karen" + }, + "kju": { + "level0": "Pomoan", + "level1": "Russian River and Eastern", + "level2": "Russian River", + "level3": "Southern Pomoan-Kashaya" + }, + "kjv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "South Slavic", + "level5": "Western South Slavic" + }, + "kjx": { + "level0": "North Bougainville" + }, + "kjy": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Kewa-Huli", + "level3": "Sau-Angal-Kewa", + "level4": "Angal-Kewa", + "level5": "Kewa", + "level6": "Southeast Kewa" + }, + "kjz": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Phobjib-Chali-Bumthangic", + "level4": "Chali-Bumthangic", + "level5": "Bumthangic" + }, + "kka": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Ebira-Nupoid", + "level4": "Macro-Nupoid", + "level5": "Nupoid", + "level6": "Dibo-Kupa" + }, + "kkb": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "East Tariku", + "level3": "Doutai-Kai-Waritai" + }, + "kkc": { + "level0": "East Strickland" + }, + "kkd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Kauru" + }, + "kke": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Nuclear Mokole", + "level8": "Mixiforic" + }, + "kkf": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Tshanglic" + }, + "kkg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Kalinga", + "level8": "Northern Kalinga", + "level9": "Northwest Kalinga" + }, + "kkh": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Southern Shanic", + "level11": "Yuanic" + }, + "kki": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu" + }, + "kkj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)" + }, + "kkk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Santa Isabel", + "level10": "Central Santa Isabel" + }, + "kkl": { + "level0": "Nuclear Trans New Guinea", + "level1": "Mek", + "level2": "Western Mek" + }, + "kkm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Kiong-Korop" + }, + "kkn": { + "level0": "Bookkeeping" + }, + "kko": { + "level0": "Nubian", + "level1": "Central Nubian", + "level2": "Kordofan Nubian", + "level3": "Western Kordofan Nubian" + }, + "kkp": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Southwest Pama", + "level3": "Coastal Southwest Paman", + "level4": "Dhawa-Kaber" + }, + "kkq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Komoic", + "level15": "Bilaic", + "level16": "Bila-Kaiku" + }, + "kkr": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi East", + "level6": "Boghomic", + "level7": "Kir-Mangas" + }, + "kks": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Nuclear Boleic", + "level8": "Galambu-Bele", + "level9": "Kirfi-Bele", + "level10": "Giiwo-Daza" + }, + "kkt": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Thulung-Tilung-Koyi" + }, + "kku": { + "level0": "Unattested", + "level1": "Atlantic-Congo (Unattested)" + }, + "kkv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Maduresic" + }, + "kkw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie" + }, + "kkx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "North West Greater Barito" + }, + "kky": { + "level0": "Pama-Nyungan", + "level1": "Yimidhirr-Yalanji-Yidinic" + }, + "kkz": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Northwestern Canada Athabaskan", + "level4": "Cordillera Athabaskan", + "level5": "Nahanni" + }, + "klb": { + "level0": "Cochimi-Yuman", + "level1": "Yuman" + }, + "klc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Southern Samba-Duru", + "level7": "Sambaic" + }, + "kld": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "Wiradhuric" + }, + "kle": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Khambu", + "level6": "Kulungic" + }, + "klf": { + "level0": "Maban", + "level1": "Mabang", + "level2": "Maba-Masalit", + "level3": "Macro-Maba" + }, + "klg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Mansakan", + "level5": "Western Mansakan" + }, + "klh": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Uruwa", + "level4": "Unclassified Uruwa" + }, + "kli": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Torajic" + }, + "klj": { + "level0": "Turkic", + "level1": "Common Turkic" + }, + "klk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Kauru" + }, + "kll": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Mansakan", + "level5": "Western Mansakan", + "level6": "Kagan-Kalagan" + }, + "klm": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Kabenau" + }, + "klo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Yukubenic" + }, + "klp": { + "level0": "Angan", + "level1": "Nuclear Angan", + "level2": "Wojokesic", + "level3": "Kamasa-Susuami" + }, + "klq": { + "level0": "Turama-Kikori" + }, + "klr": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Upper Dudhkosi" + }, + "kls": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan" + }, + "klt": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Uruwa" + }, + "klu": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Bassa-Klao", + "level5": "Klao-Tajuasohn" + }, + "klv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Eastern Malakula linkage", + "level8": "Central-Southeast Malakula", + "level9": "Southeastern Malakula linkage" + }, + "klw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio", + "level5": "Greater Kaili", + "level6": "Kulawi" + }, + "klx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Bwaidoga linkage" + }, + "kly": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Southern Kaili-Wolio", + "level5": "Island Kaili-Wolio", + "level6": "Kalao-Laiyolo" + }, + "klz": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "West Alor-Straits-Pantar", + "level4": "West Alor" + }, + "kma": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Buli-Koma" + }, + "kmb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Northern Njila", + "level9": "Mbundu (H.20)" + }, + "kmc": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Kam-Sui", + "level3": "Mulam-Kam", + "level4": "Kamic" + }, + "kmd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Kalinga", + "level8": "Central and South Kalinga", + "level9": "South Kalinga", + "level10": "Southeastern Kalinga" + }, + "kme": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Sawabantu", + "level8": "Dualaic", + "level9": "Kole-Isubu" + }, + "kmf": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso" + }, + "kmg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Eastern Huon", + "level4": "Trans Vitiaz", + "level5": "Huon Tip", + "level6": "Kate-Mape" + }, + "kmh": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "Kalam-Kobon", + "level4": "Etp-Ti Kalam" + }, + "kmi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Ebira-Nupoid", + "level4": "Macro-Nupoid", + "level5": "Nupoid", + "level6": "Dibo-Kupa", + "level7": "Abawa", + "level8": "Kami-Gupa" + }, + "kmj": { + "level0": "Dravidian", + "level1": "North Dravidian", + "level2": "Kurux-Malto", + "level3": "Malto" + }, + "kmk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Kalinga", + "level8": "Northern Kalinga" + }, + "kml": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Kalinga", + "level8": "Central and South Kalinga", + "level9": "South Kalinga", + "level10": "Southeastern Kalinga" + }, + "kmm": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Kolhrengic" + }, + "kmn": { + "level0": "Sepik", + "level1": "Ram" + }, + "kmo": { + "level0": "Sepik", + "level1": "Nukuma" + }, + "kmp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Northern Samba-Duru", + "level7": "Vere-Gimme", + "level8": "Koma Alantika" + }, + "kmq": { + "level0": "Koman" + }, + "kmr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Laki-Kurdish", + "level8": "Kurdish" + }, + "kms": { + "level0": "Nuclear Torricelli", + "level1": "Marienberg", + "level2": "Elepi-Kamasau-Marienberg" + }, + "kmt": { + "level0": "Nimboranic", + "level1": "Outer Nimboranic", + "level2": "Mlap-Gresi-Kemtuik", + "level3": "Gresi-Kemtuik" + }, + "kmu": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Siane-Yagaria", + "level5": "Kamano-Yagaria" + }, + "kmv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil", + "level14": "Macro-French", + "level15": "Circum-Caribbean French", + "level16": "Guyanic Creole French" + }, + "kmw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Komoic", + "level15": "Bilaic" + }, + "kmx": { + "level0": "Kiwaian" + }, + "kmy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Northern Samba-Duru", + "level7": "Vere-Gimme", + "level8": "Vere" + }, + "kmz": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Oghuz", + "level3": "Nuclear Oghuz", + "level4": "East Oghuz" + }, + "kna": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Tangalic" + }, + "knb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Kalinga", + "level8": "Central and South Kalinga" + }, + "knc": { + "level0": "Saharan", + "level1": "Western Saharan", + "level2": "Kanuri-Kanembu", + "level3": "Kanuric", + "level4": "East Kanuri" + }, + "knd": { + "level0": "Konda-Yahadian" + }, + "kne": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Nuclear Cordilleran", + "level7": "Bontok-Kankanay", + "level8": "Kankanay" + }, + "knf": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Manjaku-Mankanya-Pepel" + }, + "kng": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "Southeastern Kikongo", + "level20": "Southern Kikongo", + "level21": "Koongo-Kituba" + }, + "knh": { + "level0": "Bookkeeping" + }, + "kni": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Ninzic", + "level5": "Kanufi-Ninkyob-Angan" + }, + "knj": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Kanjobalan-Chujean", + "level4": "Kanjobalan", + "level5": "Kanjobal-Jacaltec" + }, + "knk": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Nuclear Mokole" + }, + "knl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Ibanic" + }, + "knm": { + "level0": "Katukinan" + }, + "knn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Southern zone", + "level7": "Marathic", + "level8": "Marathi-Konkani", + "level9": "Old-Modern Marathi", + "level10": "Modern Marathi", + "level11": "Western Marathi" + }, + "kno": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Vai-Kono" + }, + "knp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Konja" + }, + "knq": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "North Aslian", + "level4": "Maniq-Menraq-Batek", + "level5": "Maniqic" + }, + "knr": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Eastern Sepik Hill" + }, + "kns": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "North Aslian", + "level4": "Maniq-Menraq-Batek", + "level5": "Maniqic" + }, + "knt": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Marubo Subgroup" + }, + "knu": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Southwest Mande", + "level4": "Kpelle" + }, + "knw": { + "level0": "Kxa", + "level1": "Ju-Kung" + }, + "knx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Western Malayic Dayak" + }, + "kny": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Luba-Kaonde", + "level9": "Lubaic" + }, + "knz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "Northern Grusi" + }, + "koa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage" + }, + "kob": { + "level0": "Bookkeeping" + }, + "koc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Ngembaic", + "level10": "Unclassified Ngembaic" + }, + "kod": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Sumba-Hawu", + "level5": "Sumba", + "level6": "Kodi-Gaura" + }, + "koe": { + "level0": "Surmic", + "level1": "South Surmic", + "level2": "Southwest Surmic", + "level3": "Baale-Olam" + }, + "kof": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Nuclear Boleic", + "level8": "Gera-Geruma-Kubi-Deno", + "level9": "Kubi-Deno" + }, + "kog": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Magdalenic", + "level3": "Northern Magdalenic", + "level4": "Arhuacic" + }, + "koh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Mboshi (C.20)", + "level10": "Koyo-Mboshi" + }, + "koi": { + "level0": "Uralic", + "level1": "Permian", + "level2": "Komi" + }, + "koj": { + "level0": "Bookkeeping" + }, + "koo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Rwenzori" + }, + "kop": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Nuru" + }, + "koq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Ndasaic", + "level8": "Kota-Mahongwe" + }, + "kor": { + "level0": "Koreanic" + }, + "kos": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Kosraean-Nauruan" + }, + "kot": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Kotoko-Buduma", + "level5": "Kotoko Central" + }, + "kou": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Adamawa Bua", + "level5": "Inland Bua", + "level6": "Bolgo-Koke" + }, + "kov": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "Ningic" + }, + "kow": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Mumuye-Yandang" + }, + "kox": { + "level0": "Bookkeeping" + }, + "koy": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central Alaska-Yukon Athabaskan", + "level4": "Koyukonic" + }, + "koz": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Kowan" + }, + "kpa": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Tangalic", + "level7": "Nuclear Tangalic", + "level8": "Tangale-Kwami-Kupto", + "level9": "Kwami-Kupto" + }, + "kpb": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "kpc": { + "level0": "Arawakan", + "level1": "Japura-Colombia", + "level2": "Nuclear Japura-Colombia", + "level3": "Northeast Japura-Colombia", + "level4": "Baniwa-Curripaco-Tariano", + "level5": "Baniwa-Curripaco" + }, + "kpd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Central Aru", + "level4": "Dobel-Koba" + }, + "kpf": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Cromwell", + "level5": "Kabwum", + "level6": "Selepet-Komba" + }, + "kpg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Carolinean Outlier Polynesian" + }, + "kph": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Oti North Guang", + "level9": "River Oti North Guang" + }, + "kpi": { + "level0": "Geelvink Bay", + "level1": "Barapasi-Sauri-Kofei", + "level2": "Sauri-Kofei" + }, + "kpj": { + "level0": "Nuclear-Macro-Je" + }, + "kpk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Kpan-Icen" + }, + "kpl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Western Mundu-Baka", + "level7": "River Western Mundu-Baka", + "level8": "Monzomboic", + "level9": "Kpala-Bakpa" + }, + "kpm": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "South Bahnaric", + "level4": "Koho-Maa" + }, + "kpn": { + "level0": "Tupian" + }, + "kpo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Ka-Togo", + "level4": "Kposo-Ahlo-Bowili" + }, + "kpq": { + "level0": "Nuclear Trans New Guinea", + "level1": "Mek", + "level2": "Western Mek" + }, + "kpr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean", + "level3": "Nuclear Binanderean", + "level4": "South Binanderean", + "level5": "Coastal Binanderean", + "level6": "Gaena-Korafe" + }, + "kps": { + "level0": "West Bird's Head", + "level1": "South West Bird's Head" + }, + "kpt": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Andic" + }, + "kpu": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar" + }, + "kpv": { + "level0": "Uralic", + "level1": "Permian", + "level2": "Komi" + }, + "kpw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "Kalam-Kobon" + }, + "kpx": { + "level0": "Koiarian", + "level1": "Koiaric", + "level2": "Biage-Mountain Koiali" + }, + "kpy": { + "level0": "Chukotko-Kamchatkan", + "level1": "Chukotian", + "level2": "R-Koryakic", + "level3": "J-Koryakic" + }, + "kpz": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Kalenjin", + "level3": "Elgon-Mau Kalenjin" + }, + "kqa": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Sogeram", + "level5": "North Sogeram" + }, + "kqb": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Eastern Huon", + "level4": "Trans Vitiaz" + }, + "kqc": { + "level0": "Manubaran" + }, + "kqd": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "North-Eastern Neo-Aramaic", + "level11": "Trans-Zab", + "level12": "Western Trans-Zab" + }, + "kqe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Mansakan", + "level5": "Western Mansakan", + "level6": "Kagan-Kalagan" + }, + "kqf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Kakabai linkage" + }, + "kqg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Dogoso-Khe" + }, + "kqh": { + "level0": "Bookkeeping" + }, + "kqi": { + "level0": "Koiarian", + "level1": "Koiaric", + "level2": "Koita-Koiari" + }, + "kqj": { + "level0": "South Bougainville", + "level1": "Nasioiic", + "level2": "Nasioi", + "level3": "South-Central Nasioi", + "level4": "South Nasioi" + }, + "kqk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Western Phla-Phera" + }, + "kql": { + "level0": "Yuat", + "level1": "Miyak-Bun-Biwat" + }, + "kqm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Kaansa-Dogose", + "level7": "Dogose-Khisa" + }, + "kqn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Luba-Kaonde", + "level9": "Kaonde-Shaba-Sanga" + }, + "kqo": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Wee" + }, + "kqp": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.2", + "level5": "East Chadic A.2 2", + "level6": "Gabri-Kimre" + }, + "kqq": { + "level0": "Nuclear-Macro-Je", + "level1": "Maxakali-Borum" + }, + "kqr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Dusunic" + }, + "kqs": { + "level0": "Atlantic-Congo", + "level1": "Mel", + "level2": "Southern Mel", + "level3": "Kissi" + }, + "kqt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Dusunic" + }, + "kqu": { + "level0": "Tuu", + "level1": "!Ui", + "level2": "Eastern !Ui" + }, + "kqv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Eastern Murutic" + }, + "kqw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Kandas-Duke of York" + }, + "kqx": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Kotoko-Buduma", + "level5": "Kotoko Central" + }, + "kqy": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "East Ometo" + }, + "kqz": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Khoekhoe", + "level3": "South Khoekhoe" + }, + "kra": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Unclassified Bihari" + }, + "krb": { + "level0": "Miwok-Costanoan", + "level1": "Costanoan" + }, + "krc": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Northwest Kipchak", + "level5": "West Kipchak", + "level6": "Kaukasus Kipchak" + }, + "krd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Eastern Timor", + "level4": "Kawaimina" + }, + "kre": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Cerrado", + "level3": "Goyaz" + }, + "krf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage", + "level7": "Koto-Olrat-Lakon" + }, + "krg": { + "level0": "Bookkeeping" + }, + "krh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Kauru" + }, + "kri": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "West African Creole English" + }, + "krj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "West Bisayan", + "level6": "Kinarayan" + }, + "krk": { + "level0": "Chukotko-Kamchatkan", + "level1": "Chukotian", + "level2": "R-Koryakic", + "level3": "J-Koryakic" + }, + "krl": { + "level0": "Uralic", + "level1": "Finnic", + "level2": "Coastal Finnic", + "level3": "Neva", + "level4": "North Finnic", + "level5": "Ladogan" + }, + "krn": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Wee", + "level5": "Guere-Krahn" + }, + "krp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Kiong-Korop" + }, + "krs": { + "level0": "Kresh-Aja", + "level1": "Kreshic" + }, + "krt": { + "level0": "Saharan", + "level1": "Western Saharan", + "level2": "Kanuri-Kanembu", + "level3": "Kanuric", + "level4": "East Kanuri" + }, + "kru": { + "level0": "Dravidian", + "level1": "North Dravidian", + "level2": "Kurux-Malto" + }, + "krw": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Wee", + "level5": "Guere-Krahn" + }, + "krx": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Jola", + "level6": "FH-Jola", + "level7": "PF-Jola", + "level8": "Kwatay-Karon-Mlomp", + "level9": "Karon-Mlomp" + }, + "kry": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Lezgic", + "level3": "Samur", + "level4": "Southern Samur" + }, + "krz": { + "level0": "Yam", + "level1": "Kanum", + "level2": "Ngkrn-Ngkantr" + }, + "ksa": { + "level0": "Unattested", + "level1": "Atlantic-Congo (Unattested)" + }, + "ksb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "West Ruvu", + "level11": "Seuta", + "level12": "Bondei-Shambala" + }, + "ksc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Kalinga", + "level8": "Central and South Kalinga", + "level9": "South Kalinga" + }, + "ksd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Patpatar-Minigir-Tolai", + "level9": "Minigir-Tolai" + }, + "kse": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "West Central Papuan linkage", + "level9": "Nuclear West Central Papuan linkage" + }, + "ksf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Bafia (A.50)", + "level8": "Nuclear Bafia (A.50)", + "level9": "Lefa-Bafia" + }, + "ksg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "New Georgia", + "level10": "East New Georgia", + "level11": "Rovianic", + "level12": "Hoava-Kusaghe" + }, + "ksh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Middle German", + "level7": "West Middle German", + "level8": "Middle Franconian", + "level9": "Ripuarian" + }, + "ksi": { + "level0": "Sko" + }, + "ksj": { + "level0": "Kwalean", + "level1": "Humene-Kwale" + }, + "ksk": { + "level0": "Siouan", + "level1": "Mississippi Valley", + "level2": "Dhegiha", + "level3": "Osage-Kansa" + }, + "ksl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Buang linkage", + "level9": "Mumeng", + "level10": "Dambi-Kumaru" + }, + "ksm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Mumuye-Yandang" + }, + "ksn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Tagalogic" + }, + "kso": { + "level0": "Bookkeeping" + }, + "ksp": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Logone", + "level7": "Gore" + }, + "ksq": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Tangalic", + "level7": "Nuclear Tangalic", + "level8": "Tangale-Kwami-Kupto", + "level9": "Kwami-Kupto" + }, + "ksr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Rawlinson", + "level5": "Pindiu", + "level6": "Kosorong-Burum-Mindik" + }, + "kss": { + "level0": "Atlantic-Congo", + "level1": "Mel", + "level2": "Southern Mel", + "level3": "Kissi" + }, + "kst": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Western Grusi" + }, + "ksu": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Sukaphic", + "level11": "Assam Tai B" + }, + "ksv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Nkutsuic", + "level11": "Tetelaic" + }, + "ksw": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Southern Karen", + "level3": "Sgaw" + }, + "ksx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata" + }, + "ksy": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga" + }, + "ksz": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Mundaric", + "level5": "Kodaku-Korwa" + }, + "kta": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric" + }, + "ktb": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Highland East Cushitic", + "level4": "Sidaama-Hadiyya-Kambaata", + "level5": "Hadiyya-Kambaata", + "level6": "Kambaataic" + }, + "ktc": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Unclassified Boleic" + }, + "ktd": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Wati", + "level3": "Unclassified Wati" + }, + "kte": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Kyirong-Kagate", + "level9": "Gyalsumdo-Nubri-Kyirong" + }, + "ktf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega", + "level8": "Unclassified Greater Lega" + }, + "ktg": { + "level0": "Pama-Nyungan", + "level1": "Kalkatungic" + }, + "kth": { + "level0": "Maban", + "level1": "Mabang", + "level2": "Maba-Masalit", + "level3": "Macro-Maba" + }, + "kti": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Lowland Ok", + "level6": "Division A Lowland Ok" + }, + "ktj": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Grebo-Aizi", + "level4": "Grebo", + "level5": "Ivorian Grebo", + "level6": "Tepo-Plapo" + }, + "ktk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Western Admiralty Islands", + "level6": "Anchorite" + }, + "ktl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Balochic", + "level8": "Southern-Western Balochi", + "level9": "Southern Balochi-Koroshi" + }, + "ktm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "East Manus", + "level8": "Kurti-Kele-Ere", + "level9": "Kurti-Elu" + }, + "ktn": { + "level0": "Tupian", + "level1": "Arikem-Tupari", + "level2": "Arikemic" + }, + "ktp": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Hanic", + "level7": "Bi-Ka" + }, + "ktq": { + "level0": "Unclassifiable" + }, + "kts": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Lowland Ok", + "level6": "Division A Lowland Ok" + }, + "ktt": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Dumut", + "level6": "Ketum-Wambon" + }, + "ktu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "Southeastern Kikongo", + "level20": "Southern Kikongo", + "level21": "Koongo-Kituba" + }, + "ktv": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "Katu", + "level3": "Nuclear Katu" + }, + "ktw": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Pacific Coast Athabaskan", + "level4": "California Athabaskan" + }, + "ktx": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano" + }, + "kty": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Middle Bomokandian", + "level15": "Late Bomokandian" + }, + "ktz": { + "level0": "Kxa", + "level1": "Ju-Kung" + }, + "kua": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia", + "level11": "Ndonga (R.20)" + }, + "kub": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid" + }, + "kuc": { + "level0": "Tor-Orya", + "level1": "Tor", + "level2": "Coastal Tor" + }, + "kud": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "Suauic" + }, + "kue": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Simbu", + "level3": "Nuclear Simbu", + "level4": "Kuman-Dom-Gunaa" + }, + "kuf": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "Katu", + "level3": "Nuclear Katu" + }, + "kug": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Ebira-Nupoid", + "level4": "Macro-Nupoid", + "level5": "Nupoid", + "level6": "Dibo-Kupa" + }, + "kuh": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Tangalic", + "level7": "Nuclear Tangalic" + }, + "kui": { + "level0": "Cariban", + "level1": "Kuikuroan", + "level2": "Nuclear Kuikuroan" + }, + "kuj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Nyanza Mara", + "level11": "North Mara", + "level12": "Kuriaic" + }, + "kuk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Manggaraiic" + }, + "kul": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.4", + "level5": "Ronic" + }, + "kum": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Northwest Kipchak", + "level5": "West Kipchak", + "level6": "Kaukasus Kipchak" + }, + "kuo": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Cromwell", + "level5": "Dallman", + "level6": "Kinalakna-Kumukio" + }, + "kup": { + "level0": "Kunimaipan" + }, + "kuq": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VI", + "level6": "Kawahiva", + "level7": "Nuclear Kawahiva", + "level8": "Central Kawahiva" + }, + "kus": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Southeast Western Oti-Volta" + }, + "kuu": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central Alaska-Yukon Athabaskan", + "level4": "Tanana-Tutchone", + "level5": "Tananaic" + }, + "kuv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tanimbar-Bomberai", + "level3": "Teor-Kur" + }, + "kuw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Central Bandaic", + "level8": "Central Core Bandaic", + "level9": "Mid-Southern Central Core Bandaic" + }, + "kux": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Wati", + "level3": "Pintupic" + }, + "kuy": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Northeastern Pama", + "level4": "Umpilaic" + }, + "kva": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Andic", + "level4": "Bagvalal-Tindi" + }, + "kvb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Central Sumatran Malay" + }, + "kvc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Ngero", + "level8": "Eastern Ngero", + "level9": "Kaliai-Kove" + }, + "kvd": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "Central Alor" + }, + "kve": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Eastern Murutic" + }, + "kvf": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.2", + "level5": "East Chadic A.2 2" + }, + "kvg": { + "level0": "Anim", + "level1": "Marind-Boazi-Yaqai", + "level2": "Boazi" + }, + "kvh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Manggaraiic" + }, + "kvi": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.3" + }, + "kvj": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Higic" + }, + "kvk": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "JSLic" + }, + "kvl": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Central Karen", + "level3": "Kayaw-Manu" + }, + "kvm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Mamfe", + "level6": "Kendem-Denya" + }, + "kvn": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Isthmic Chibchan", + "level3": "Eastern Isthmic Chibchan", + "level4": "Kuna" + }, + "kvo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Central Aru", + "level4": "Dobel-Koba" + }, + "kvp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Ujir-Kola-Kompane", + "level4": "Kola-Kompane" + }, + "kvq": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Central Karen", + "level3": "Geba-Bwe" + }, + "kvr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Northern Sumatra Malay", + "level6": "Kerinci-Minangkabau" + }, + "kvs": { + "level0": "Bookkeeping" + }, + "kvt": { + "level0": "Bookkeeping" + }, + "kvu": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Northern Karen" + }, + "kvv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Ujir-Kola-Kompane", + "level4": "Kola-Kompane" + }, + "kvw": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "East Alor", + "level3": "Sawila-Wersing" + }, + "kvx": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Western Rajasthani", + "level11": "Indus Rajasthani" + }, + "kvy": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Central Karen", + "level3": "Kayah-Yintale" + }, + "kvz": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Becking-Dawi", + "level5": "Tsakwambo-Komyandaret" + }, + "kwa": { + "level0": "Naduhup", + "level1": "Eastern Naduhup" + }, + "kwb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian" + }, + "kwc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Bobangic", + "level13": "Bobangic Riverain", + "level14": "Likwala-Likuba" + }, + "kwd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Central-Northern Malaita" + }, + "kwe": { + "level0": "Greater Kwerba", + "level1": "Kwerba-Samarokena", + "level2": "Kwerbaic" + }, + "kwf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Central-Northern Malaita" + }, + "kwg": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Peripherique", + "level6": "Barh Keita", + "level7": "Sara-Kaba" + }, + "kwh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "Banda-Geser", + "level4": "Seran Laut", + "level5": "Koiwai-Irarutu" + }, + "kwi": { + "level0": "Barbacoan", + "level1": "Awa-Southern Barbacoan" + }, + "kwj": { + "level0": "Sepik", + "level1": "Nukuma", + "level2": "Kwanga-Mende" + }, + "kwk": { + "level0": "Wakashan", + "level1": "Northern Wakashan", + "level2": "Kwakiutlan" + }, + "kwl": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3", + "level6": "Kofyar-Mushere-Chip", + "level7": "Kofyaric" + }, + "kwm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia", + "level11": "Ndonga (R.20)", + "level12": "Kwambi-Ndonga" + }, + "kwn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Kwangali-Diriku" + }, + "kwo": { + "level0": "Kwomtari-Nai" + }, + "kwp": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Unclassified Eastern Kru" + }, + "kwq": { + "level0": "Bookkeeping" + }, + "kwr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Kwer-Kopkaka-Burumakok", + "level6": "Kwer-Burumakok" + }, + "kws": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Northern Njila", + "level9": "Mbala-Holu-Sondi (K.10)", + "level10": "Holu (K.10)", + "level11": "Pheende-Kwezo" + }, + "kwt": { + "level0": "Tor-Orya", + "level1": "Tor" + }, + "kwu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)" + }, + "kwv": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Peripherique", + "level6": "Barh Keita", + "level7": "Sara-Kaba" + }, + "kww": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Surinamese Creole English", + "level13": "Eastern Maroons", + "level14": "Ndyuka" + }, + "kwx": { + "level0": "Dravidian", + "level1": "Unclassified Dravidian" + }, + "kwy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "West Kikongo" + }, + "kwz": { + "level0": "Khoe-Kwadi" + }, + "kxa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Kairiru-Manam", + "level8": "Kairiruic linkage" + }, + "kxb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano" + }, + "kxc": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Oromoid", + "level7": "Konsoid" + }, + "kxd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "East Borneo Malay", + "level6": "Banjar-Berau-Brunei Malay", + "level7": "Berau-Brunei Malay", + "level8": "Bruneic Malay", + "level9": "Brunei-Bacan Malay" + }, + "kxe": { + "level0": "Bookkeeping" + }, + "kxf": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Central Karen", + "level3": "Kayaw-Manu" + }, + "kxg": { + "level0": "Bookkeeping" + }, + "kxh": { + "level0": "South Omotic", + "level1": "AHK", + "level2": "Hamer-Karo" + }, + "kxi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Northern Murutic", + "level8": "Lowland Murut" + }, + "kxj": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Peripherique", + "level6": "Koulfaic" + }, + "kxk": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Northern Karen" + }, + "kxm": { + "level0": "Austroasiatic", + "level1": "Khmeric" + }, + "kxn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Melanau-Kajang", + "level5": "Melanau", + "level6": "Sibu-Kanowit-Tanjong" + }, + "kxp": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Gujaratic", + "level10": "Western Gujaratic" + }, + "kxq": { + "level0": "Yam", + "level1": "Kanum" + }, + "kxr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "East Manus", + "level8": "Koro-Lele-Nali-Titan" + }, + "kxs": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Southern Periphery Mongolic", + "level3": "Shirongol", + "level4": "Baoanic" + }, + "kxt": { + "level0": "Ndu", + "level1": "Nuclear Ndu", + "level2": "Sawosic" + }, + "kxu": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Konda-Kui", + "level4": "Manda-Kui", + "level5": "Kui-Kuvi" + }, + "kxv": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Konda-Kui", + "level4": "Manda-Kui", + "level5": "Kui-Kuvi" + }, + "kxw": { + "level0": "East Strickland" + }, + "kxx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Bobangic", + "level13": "Bobangic Riverain", + "level14": "Likwala-Likuba" + }, + "kxy": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric", + "level3": "Kayong-Jeh-Halang" + }, + "kxz": { + "level0": "Kiwaian", + "level1": "Turama-Kerewo" + }, + "kya": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Suguti" + }, + "kyb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Kalinga", + "level8": "Central and South Kalinga", + "level9": "South Kalinga" + }, + "kyc": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Engan" + }, + "kyd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Karey-Barakai" + }, + "kye": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Oti North Guang", + "level9": "River Oti North Guang" + }, + "kyf": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Beteic", + "level3": "Eastern Bete" + }, + "kyg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Siane-Yagaria", + "level5": "Kamano-Yagaria" + }, + "kyi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Berawan-Lower Baram", + "level5": "Lower Baram", + "level6": "Central Lower Baram A" + }, + "kyj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Southern Cordilleran", + "level6": "West Southern Cordilleran", + "level7": "Nuclear Southern Cordilleran" + }, + "kyk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Mansakan" + }, + "kyl": { + "level0": "Kalapuyan" + }, + "kym": { + "level0": "Bookkeeping" + }, + "kyn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Negrosanon" + }, + "kyo": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "West Alor-Straits-Pantar" + }, + "kyp": { + "level0": "Bookkeeping" + }, + "kyq": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Bagirmic" + }, + "kyr": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Mundurukuic" + }, + "kys": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Kayan-Murik-Modang", + "level4": "Kayan-Murik", + "level5": "Kayanic" + }, + "kyt": { + "level0": "Kayagaric", + "level1": "Kaygir-Tamagario" + }, + "kyu": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Central Karen", + "level3": "Kayah-Yintale", + "level4": "Kayah" + }, + "kyv": { + "level0": "Bookkeeping" + }, + "kyw": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Magadhan", + "level10": "Sadanic" + }, + "kyx": { + "level0": "North Bougainville" + }, + "kyy": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Kainantu", + "level3": "Unclassified Kainantu" + }, + "kyz": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VI" + }, + "kza": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "Karaboro" + }, + "kzb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua", + "level6": "Seram Straits" + }, + "kzc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Kulango-Lorom", + "level5": "Kulango" + }, + "kzd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Saluan-Banggai", + "level6": "Taliaboic" + }, + "kze": { + "level0": "Bookkeeping" + }, + "kzf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio", + "level5": "Greater Kaili", + "level6": "Common Kaili" + }, + "kzg": { + "level0": "Japonic", + "level1": "Ryukyuan", + "level2": "Northern Ryukyuan", + "level3": "Amami" + }, + "kzh": { + "level0": "Nubian", + "level1": "Nile Nubian" + }, + "kzi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Dayic" + }, + "kzk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "New Georgia" + }, + "kzl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku" + }, + "kzm": { + "level0": "South Bird's Head Family" + }, + "kzn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Chuwaboic", + "level9": "Lolo-Kokola" + }, + "kzo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie", + "level17": "Northwest Kasai-Ngounie", + "level18": "Mbere (B.60)", + "level19": "Tsitsekeic", + "level20": "Lekaningic" + }, + "kzp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Gorontalo-Mongondow", + "level4": "Gorontalic" + }, + "kzq": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic" + }, + "kzr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Central Mbum", + "level6": "Karangic" + }, + "kzs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Dusunic", + "level7": "Kadazan-Sugut-Minokok", + "level8": "Sugut-Minokok Kadazan" + }, + "kzu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Jayapura Bay", + "level8": "Eastern Jayapura Bay" + }, + "kzv": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Becking-Dawi", + "level5": "Tsakwambo-Komyandaret" + }, + "kzw": { + "level0": "Unclassifiable" + }, + "kzx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua", + "level6": "Seram Straits", + "level7": "Uliase" + }, + "kzy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Komoic", + "level15": "Bilaic" + }, + "kzz": { + "level0": "West Bird's Head", + "level1": "South West Bird's Head" + }, + "laa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Subanen", + "level4": "Nuclear Subanen" + }, + "lac": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Yucatecan", + "level3": "Nuclear Yucatecan", + "level4": "Yucatec-Lacandon" + }, + "lad": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Castilic", + "level13": "South Castilic" + }, + "lae": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Western West Himalayish", + "level4": "Lahaulic" + }, + "lag": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Mbugwe-Langi" + }, + "lai": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Nyika-Lambya" + }, + "laj": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Southern Lwoo", + "level4": "Lango-Kumam" + }, + "lam": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Sabi", + "level8": "Malungu-Central Sabi", + "level9": "Central Sabi", + "level10": "Bisa-Lamba (M.50)" + }, + "lan": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Kainji Lake", + "level5": "Upper Niger Kainji" + }, + "lao": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Thai PH", + "level9": "Lao-Thai" + }, + "lap": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Logone", + "level7": "Gore" + }, + "laq": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Eastern Kra" + }, + "lar": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "South Guang", + "level8": "Hill South Guang" + }, + "las": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Eastern Grusi", + "level9": "Kabiyeic" + }, + "lat": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin" + }, + "lau": { + "level0": "Bookkeeping" + }, + "lav": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Eastern Baltic" + }, + "law": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Tominic", + "level5": "Northern Tomini" + }, + "lax": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Boroic", + "level4": "Tiwa-Boro" + }, + "lay": { + "level0": "Bookkeeping" + }, + "laz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Lower Markham", + "level9": "Busu" + }, + "lba": { + "level0": "Bookkeeping" + }, + "lbb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Label-Bilur" + }, + "lbc": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Lakkia-Biao" + }, + "lbe": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian" + }, + "lbf": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Western West Himalayish", + "level4": "Lahaulic" + }, + "lbg": { + "level0": "Bookkeeping" + }, + "lbi": { + "level0": "Speech Register", + "level1": "Atlantic-Congo Speech Register" + }, + "lbj": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Western Archaic Tibetan", + "level5": "Kenhatic" + }, + "lbk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Nuclear Cordilleran", + "level7": "Bontok-Kankanay", + "level8": "Bontok" + }, + "lbm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga" + }, + "lbn": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic" + }, + "lbo": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "Nuclear West Bahnaric", + "level4": "Loven-Suq" + }, + "lbq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Lower Markham" + }, + "lbr": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Upper Arun", + "level6": "Lohorung-Yamphu" + }, + "lbs": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Arab Sign" + }, + "lbt": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Southwestern Kra", + "level3": "Western Kra", + "level4": "Lachic" + }, + "lbu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Lower Markham" + }, + "lbv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Madak linkage" + }, + "lbw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Western Bungku-Tolaki", + "level8": "West Coast Bungku-Tolaki" + }, + "lbx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "North East Greater Barito" + }, + "lby": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Lamalamic", + "level3": "Coastal Lamalamic" + }, + "lbz": { + "level0": "Tangkic" + }, + "lcc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "Maya-Matbat", + "level6": "Raja Ampat Maya" + }, + "lcd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Central Aru" + }, + "lce": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Northern Sumatra Malay", + "level6": "Bangka-Belitung Malay" + }, + "lcf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Northern Sumatra Malay", + "level6": "Kerinci-Minangkabau", + "level7": "Minangkabauic" + }, + "lch": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Chokwe-Ngangela-Nyemba (K.20)", + "level11": "Ngangela-Nyemba" + }, + "lcl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "West Central Maluku", + "level3": "Sula-Buru", + "level4": "Buruic" + }, + "lcm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Tungak-Nalik" + }, + "lcp": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Waic", + "level5": "Wa-Lawa", + "level6": "Lawa" + }, + "lcq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "West Piru Bay", + "level5": "Hoamoal", + "level6": "West Hoamoal" + }, + "lcs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Three Rivers", + "level4": "Amalumute", + "level5": "Northwest Seram" + }, + "ldb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Northwestern Benue-Congo Plateau", + "level6": "Koroic", + "level7": "Duyaic" + }, + "ldd": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Northwest South Bauchi", + "level7": "Polci-Luri" + }, + "ldg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "East-West Central Delta Cross", + "level7": "Mbembe-Legbo", + "level8": "Legboic", + "level9": "Lenyima-Leyigha" + }, + "ldh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Dakoid", + "level6": "Taram-Dirim-Nnakenyare", + "level7": "Dirim-Nnakenyare" + }, + "ldi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Nuclear Northern Kikongo" + }, + "ldj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "North-Central Jos", + "level10": "Chokobo-Lemoro-Sanga", + "level11": "Lemoro-Sanga" + }, + "ldk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bikwin-Jen", + "level5": "Southern Bikwin-Jen", + "level6": "Bambuka-Gomu-Leelau" + }, + "ldl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bena-Mboi" + }, + "ldm": { + "level0": "Atlantic-Congo", + "level1": "Mel", + "level2": "Northern Mel" + }, + "ldn": { + "level0": "Artificial Language" + }, + "ldo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bikwin-Jen", + "level5": "Northern Bikwin-Jen", + "level6": "Burak-Loo" + }, + "ldp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Tula-Longuda", + "level6": "Tula-Waja", + "level7": "Tulaic" + }, + "ldq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Yukubenic", + "level5": "Bete-Lufu" + }, + "lea": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega", + "level8": "Mituku-Lega", + "level9": "Lega", + "level10": "Western Lega" + }, + "leb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Sabi", + "level8": "Malungu-Central Sabi", + "level9": "Central Sabi", + "level10": "Bisa-Lamba (M.50)" + }, + "led": { + "level0": "Central Sudanic", + "level1": "Lenduic", + "level2": "Bale" + }, + "lee": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "Northern Grusi" + }, + "lef": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Na-Togo", + "level4": "Lelemic", + "level5": "Lelemi-Akpafu" + }, + "leh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Botatwe", + "level8": "Greater Eastern Botatwe", + "level9": "Central Eastern Botatwe", + "level10": "Kafue" + }, + "lei": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Kabenau" + }, + "lej": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke", + "level11": "So-Poke", + "level12": "So-Lebonya", + "level13": "Lebonya" + }, + "lek": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "East Manus" + }, + "lel": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Nkutsuic", + "level11": "Bushoong-Wongo-Lele" + }, + "lem": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Sanaga-West Mbam (A.40)", + "level10": "West Mbam (A.40)", + "level11": "Mandi-Nyokon" + }, + "leo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Sanaga-West Mbam (A.40)", + "level10": "Sanaga (A.60)" + }, + "lep": { + "level0": "Sino-Tibetan", + "level1": "Himalayish" + }, + "leq": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Engan" + }, + "ler": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "South-East Admiralty" + }, + "les": { + "level0": "Central Sudanic", + "level1": "Membi-Mangbutu-Efe", + "level2": "Mangbutu-Efe", + "level3": "Leseic" + }, + "let": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Arawe", + "level11": "East Arawe" + }, + "leu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Tungak-Nalik" + }, + "lev": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "West Alor-Straits-Pantar" + }, + "lew": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio", + "level5": "Greater Kaili", + "level6": "Common Kaili" + }, + "lex": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Luangic-Kisaric", + "level5": "Luangic" + }, + "ley": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "Rampi-Seko-Badaic", + "level4": "Badaic-Limola" + }, + "lez": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Lezgic", + "level3": "Samur", + "level4": "Eastern Samur", + "level5": "Tabasaran-Aghul-Lezgi", + "level6": "Aghul-Lezgi" + }, + "lfa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Bafia (A.50)", + "level8": "Nuclear Bafia (A.50)", + "level9": "Lefa-Bafia" + }, + "lfn": { + "level0": "Artificial Language" + }, + "lga": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "New Georgia", + "level10": "West New Georgia", + "level11": "Simboic", + "level12": "Ghanongga-Lungga" + }, + "lgb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "West Santa Isabel" + }, + "lgg": { + "level0": "Central Sudanic", + "level1": "Moru-Madi", + "level2": "Central Moru-Madi" + }, + "lgh": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Muji", + "level8": "Laghuu-Core Muji" + }, + "lgi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Dayic" + }, + "lgk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Central-Western Malakula" + }, + "lgl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Central-Northern Malaita" + }, + "lgm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega", + "level8": "Mituku-Lega", + "level9": "Lega" + }, + "lgn": { + "level0": "Koman", + "level1": "Central Koman", + "level2": "Dana-Opo" + }, + "lgq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Na-Togo" + }, + "lgr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Guadalcanal-Nggelic", + "level6": "Nuclear Guadalcanal-Nggelic", + "level7": "North and West Guadalcanal" + }, + "lgs": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "lgt": { + "level0": "Sepik", + "level1": "Sepik Tama", + "level2": "Mehek-Pahi" + }, + "lgu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira" + }, + "lgz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ngombe-Genja" + }, + "lha": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Southwestern Kra", + "level3": "Southern Kra" + }, + "lhh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua", + "level6": "Seram Straits", + "level7": "Ambonic", + "level8": "Central Ambon" + }, + "lhi": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Lahoid" + }, + "lhl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Kangric-Chamealic-Bhattiyali", + "level8": "Chamealic", + "level9": "Bhadrawahi-Bhalesi-Curahi", + "level10": "Bhadarwahic", + "level11": "Chinali-Lahul Lohar" + }, + "lhm": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic" + }, + "lhn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Melanau-Kajang", + "level5": "Kajang", + "level6": "Kajaman-Lahanan" + }, + "lhp": { + "level0": "Sino-Tibetan", + "level1": "Dhimal-Lhokpu-Toto" + }, + "lhs": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "Turoyo-Mlahso" + }, + "lht": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage", + "level7": "Hiw-Lo-Toga" + }, + "lhu": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Lahoid" + }, + "lia": { + "level0": "Atlantic-Congo", + "level1": "Limba" + }, + "lib": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "West Manus", + "level8": "West Manus II", + "level9": "Likum-Levei" + }, + "lic": { + "level0": "Tai-Kadai", + "level1": "Hlaic", + "level2": "Nuclear Hlaic" + }, + "lid": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "West Manus", + "level8": "West Manus I" + }, + "lie": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Libinzic", + "level13": "Libinza Ngiri" + }, + "lif": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Tamar" + }, + "lig": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Jogo-Jeri", + "level6": "Jogo" + }, + "lih": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Tabar linkage" + }, + "lij": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Italian" + }, + "lik": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Middle Bomokandian" + }, + "lil": { + "level0": "Salishan", + "level1": "Interior Salish", + "level2": "Northern Interior Salish" + }, + "lim": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Middle German", + "level7": "West Middle German", + "level8": "Middle Franconian", + "level9": "Ripuarian" + }, + "lin": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Bobangic", + "level13": "Bobangic Riverain", + "level14": "Bobangi-Bangala-Lingala", + "level15": "Lingala-Bangala" + }, + "lio": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Sarmi", + "level8": "Sobeic", + "level9": "Sobei-Liki" + }, + "lip": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Na-Togo", + "level4": "Lelemic", + "level5": "Likpe-Santrokofi" + }, + "liq": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Highland East Cushitic", + "level4": "Sidaama-Hadiyya-Kambaata", + "level5": "Hadiyya-Kambaata", + "level6": "Hadiyyaic" + }, + "lir": { + "level0": "Pidgin", + "level1": "English-based pidgin" + }, + "lis": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Nuclear Lisoid", + "level7": "Lisu-Laluba-Lavu" + }, + "lit": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Eastern Baltic" + }, + "liu": { + "level0": "Dajuic", + "level1": "Eastern Dajuic" + }, + "liv": { + "level0": "Uralic", + "level1": "Finnic", + "level2": "Coastal Finnic" + }, + "liw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Central Sumatran Malay", + "level6": "Music" + }, + "lix": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Nuclear Muna-Buton", + "level8": "Munan", + "level9": "Munic", + "level10": "Western Munic" + }, + "liy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Central Bandaic", + "level8": "Central Core Bandaic" + }, + "liz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Libinzic", + "level13": "Libinza Ngiri" + }, + "lje": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "Rampi-Seko-Badaic" + }, + "lji": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Southern Kaili-Wolio", + "level5": "Island Kaili-Wolio", + "level6": "Kalao-Laiyolo" + }, + "ljl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Central Flores-Paluqe", + "level6": "Central Flores", + "level7": "Eastern Central Flores", + "level8": "Ende-Lio" + }, + "ljp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Lampungic", + "level3": "Pesisir" + }, + "ljw": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric" + }, + "ljx": { + "level0": "Pama-Nyungan", + "level1": "Nyawaygic" + }, + "lka": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Lakalei-Idate" + }, + "lkb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Central-Eastern Luyia", + "level14": "Kabarasi-Tachoni-Nyala East" + }, + "lkc": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Lahoid" + }, + "lkd": { + "level0": "Nambiquaran", + "level1": "Nambikwara Complex", + "level2": "Northern Nambiquaran", + "level3": "Roosevelt" + }, + "lke": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "North Nyanza", + "level11": "Soga-Kenyi" + }, + "lkh": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Southern Tibetic" + }, + "lki": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Laki-Kurdish" + }, + "lkj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Ibanic", + "level5": "Iban-Mualang-Seberuang", + "level6": "Iban-Seberuang", + "level7": "Northern Iban" + }, + "lkl": { + "level0": "Nuclear Torricelli" + }, + "lkm": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Mirning" + }, + "lkn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage", + "level7": "Koto-Olrat-Lakon" + }, + "lko": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Western Luyia", + "level14": "Marachi-Khayo" + }, + "lkr": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Northern Lwoo" + }, + "lks": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Central-Eastern Luyia", + "level14": "Kisa-Marama-Tsotso" + }, + "lkt": { + "level0": "Siouan", + "level1": "Mississippi Valley", + "level2": "Dakotan", + "level3": "Sioux" + }, + "lku": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Pirriya-Kungkari" + }, + "lky": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Lotuxo-Maa", + "level4": "Lotuxo", + "level5": "Lotuko-Lokoya" + }, + "lla": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bena-Mboi", + "level5": "Bena" + }, + "llb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Chuwaboic", + "level9": "Lolo-Kokola" + }, + "llc": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Nuclear Mokole" + }, + "lld": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian" + }, + "lle": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "East Manus", + "level8": "Koro-Lele-Nali-Titan" + }, + "llf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "West Manus", + "level8": "West Manus I" + }, + "llg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Rote-Meto", + "level4": "Nuclear Rote" + }, + "llh": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Unclassified Lisoid" + }, + "lli": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie", + "level17": "Northwest Kasai-Ngounie", + "level18": "Nzebi-Laali-Yaa", + "level19": "Laali-Yaa" + }, + "llk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Berawan-Lower Baram", + "level5": "Lower Baram", + "level6": "Central Lower Baram B" + }, + "lll": { + "level0": "Bogia" + }, + "llm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Nuclear Muna-Buton", + "level8": "Butonic", + "level9": "East Buton" + }, + "lln": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.2", + "level5": "East Chadic A.2 1" + }, + "llo": { + "level0": "Bookkeeping" + }, + "llp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Epi-Efate", + "level7": "Efate", + "level8": "North Efatic" + }, + "llq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Gorontalo-Mongondow", + "level4": "Gorontalic" + }, + "lls": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "RSLic", + "level3": "Nuclear RSLic" + }, + "llu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Central-Northern Malaita", + "level9": "North Malaitan" + }, + "llx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Eastern Fijian" + }, + "lma": { + "level0": "Atlantic-Congo", + "level1": "Limba" + }, + "lmb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "Central Santo" + }, + "lmc": { + "level0": "Limilngan-Wulna" + }, + "lmd": { + "level0": "Narrow Talodi", + "level1": "Lumun-Torona" + }, + "lme": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Masa", + "level3": "South Masa", + "level4": "Peveic" + }, + "lmf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata", + "level4": "Lamaholot Tengah", + "level5": "Southeast Lembata" + }, + "lmg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Bibling" + }, + "lmh": { + "level0": "Bookkeeping" + }, + "lmi": { + "level0": "Central Sudanic", + "level1": "Mangbetu-Asua" + }, + "lmj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata", + "level4": "Lamaholot Tengah" + }, + "lmk": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Anal-Lamgang" + }, + "lml": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu" + }, + "lmm": { + "level0": "Bookkeeping" + }, + "lmn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Western Rajasthani" + }, + "lmo": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Italian", + "level12": "Piemontese-Lombard" + }, + "lmp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Nkambe" + }, + "lmq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata", + "level4": "Lamaholot Timur" + }, + "lmr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata", + "level4": "Lamaholot Barat" + }, + "lms": { + "level0": "Bookkeeping" + }, + "lmt": { + "level0": "Bookkeeping" + }, + "lmu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Epi-Efate", + "level7": "Epi", + "level8": "Lamenu-Lewo" + }, + "lmv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Eastern Fijian", + "level7": "Nuclear Eastern Fijian", + "level8": "Viwa-Lomaiviti-East Viti Levu" + }, + "lmw": { + "level0": "Miwok-Costanoan", + "level1": "Miwokan", + "level2": "Western Miwokan" + }, + "lmx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "West Ring" + }, + "lmy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Sumba-Hawu", + "level5": "Sumba", + "level6": "Wewewa-Laboya" + }, + "lmz": { + "level0": "Unattested" + }, + "lna": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Ngbugu-Langbasi" + }, + "lnb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia", + "level11": "Ndonga (R.20)" + }, + "lnc": { + "level0": "Bookkeeping" + }, + "lnd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Dayic" + }, + "lnh": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "Senoic", + "level4": "Lanoh-Semnam-Temiar", + "level5": "Lanoh-Semnam", + "level6": "Lanohic" + }, + "lni": { + "level0": "South Bougainville", + "level1": "Nasioiic", + "level2": "Nasioi", + "level3": "South-Central Nasioi", + "level4": "South Nasioi" + }, + "lnj": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Northern Pama", + "level3": "Linngithigh-Alngith" + }, + "lnl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Ngbugu-Langbasi" + }, + "lnm": { + "level0": "Keram", + "level1": "Ulmapo", + "level2": "Mwakai-Pondi" + }, + "lnn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "Shark Bayic" + }, + "lno": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Lotuxo-Maa", + "level4": "Lotuxo" + }, + "lns": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring" + }, + "lnu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Tula-Longuda" + }, + "loa": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Mainland North Halmaheran", + "level3": "Galela-Loloda" + }, + "lob": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Lobiri-Jaane" + }, + "loc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "West Bisayan" + }, + "loe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Saluan-Banggai", + "level6": "Western Saluan-Banggai", + "level7": "Saluanic", + "level8": "Batui-Saluan" + }, + "lof": { + "level0": "Heibanic", + "level1": "West-Central Heibanic", + "level2": "Central Heibanic", + "level3": "Ebang-Logol" + }, + "log": { + "level0": "Central Sudanic", + "level1": "Moru-Madi", + "level2": "Central Moru-Madi" + }, + "loh": { + "level0": "Surmic", + "level1": "South Surmic", + "level2": "Southwest Surmic", + "level3": "Didinga-Murle", + "level4": "Didinga-Longarim" + }, + "loi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Kulango-Lorom", + "level5": "Teenic" + }, + "loj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "South-East Admiralty", + "level7": "Lou-Paluai" + }, + "lok": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Southwest Mande", + "level4": "Mende-Loma", + "level5": "Mende-Bandi", + "level6": "Mende-Loko" + }, + "lol": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Mongoic", + "level11": "Lomongo" + }, + "lom": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Southwest Mande", + "level4": "Mende-Loma", + "level5": "Loma" + }, + "lon": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Makua-Lomwe", + "level9": "Lomweic" + }, + "loo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke", + "level11": "Kele-Lombo" + }, + "lop": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Kainji Lake", + "level5": "Upper Niger Kainji", + "level6": "Oleran" + }, + "loq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Interieur", + "level12": "Lobalic" + }, + "lor": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Kulango-Lorom", + "level5": "Teenic" + }, + "los": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "Mokoreng-Loniu" + }, + "lot": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Lotuxo-Maa", + "level4": "Lotuxo", + "level5": "Lotuko-Lokoya" + }, + "lou": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil", + "level14": "Macro-French", + "level15": "Circum-Caribbean French" + }, + "lov": { + "level0": "Bookkeeping" + }, + "low": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Paitanic", + "level7": "Upper Kinabatangan-Lobu" + }, + "lox": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Three Rivers", + "level4": "Amalumute", + "level5": "Northwest Seram" + }, + "loy": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Mustangic" + }, + "loz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Sotho-Tswana (S.30)", + "level11": "Western Sotho-Tswana", + "level12": "Central Sotho-Tswana", + "level13": "Sesotho-Lozi" + }, + "lpa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Epi-Efate", + "level7": "Efate", + "level8": "North Efatic" + }, + "lpe": { + "level0": "Lepki-Murkim-Kembra" + }, + "lpn": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Central Naga", + "level4": "Yimchingric", + "level5": "Makuric" + }, + "lpo": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Lipo-Lolopo", + "level7": "Lipo-Micha" + }, + "lpx": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Lotuxo-Maa", + "level4": "Lotuxo", + "level5": "Lopit-Dongotono" + }, + "lra": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Benyadu-Bekati", + "level4": "Bakati'", + "level5": "Rara-Sara Bakati'" + }, + "lrc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Luric-Dezfulic", + "level8": "Luric" + }, + "lre": { + "level0": "Iroquoian", + "level1": "Northern Iroquoian" + }, + "lri": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Western Luyia", + "level14": "Marachi-Khayo" + }, + "lrk": { + "level0": "Bookkeeping" + }, + "lrl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian" + }, + "lrm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Central-Eastern Luyia", + "level14": "Kisa-Marama-Tsotso" + }, + "lrn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Central Aru" + }, + "lro": { + "level0": "Heibanic", + "level1": "West-Central Heibanic", + "level2": "Central Heibanic", + "level3": "Ebang-Logol", + "level4": "Ebang-Laru" + }, + "lrr": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Upper Arun", + "level6": "Lohorung-Yamphu", + "level7": "Yamphuic" + }, + "lrt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay", + "level6": "Eastern Indonesia Trade Malay" + }, + "lrv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Central-Western Malakula" + }, + "lrz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage", + "level7": "Lemerig-Veraa" + }, + "lsa": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Komisenian" + }, + "lsc": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "lsd": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "North-Eastern Neo-Aramaic", + "level11": "Northwestern Jewish Neo-Aramaic" + }, + "lse": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri" + }, + "lsg": { + "level0": "Bookkeeping" + }, + "lsh": { + "level0": "Sino-Tibetan", + "level1": "Kho-Bwa", + "level2": "Western Kho-Bwa", + "level3": "Chug-Lish" + }, + "lsi": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Northern Burmish", + "level5": "Maruic", + "level6": "Leqic" + }, + "lsl": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "lsm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Western Luyia", + "level14": "Saamiaic" + }, + "lsn": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "lso": { + "level0": "Bookkeeping" + }, + "lsp": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "Honduras-Panama Sign" + }, + "lsr": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Nuclear Palai", + "level4": "Bragat-Aruop-Amol" + }, + "lss": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Sindhic", + "level9": "Lasi-Jadgali" + }, + "lst": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "American Sign" + }, + "lsv": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "lsw": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "lsy": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "ltc": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic" + }, + "lti": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Luangic-Kisaric", + "level5": "Luangic" + }, + "ltn": { + "level0": "Nambiquaran", + "level1": "Nambikwara Complex", + "level2": "Northern Nambiquaran", + "level3": "Roosevelt" + }, + "lto": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Central-Eastern Luyia", + "level14": "Kisa-Marama-Tsotso" + }, + "lts": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Central-Eastern Luyia", + "level14": "Kabarasi-Tachoni-Nyala East" + }, + "ltu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua", + "level6": "Seram Straits", + "level7": "Uliase", + "level8": "Hatuhaha", + "level9": "Saparuan", + "level10": "Saparua-Latu" + }, + "ltz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Middle German", + "level7": "West Middle German", + "level8": "Middle Franconian" + }, + "lua": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Luba-Kaonde", + "level9": "Lubaic", + "level10": "Bangubangu-Kasai" + }, + "lub": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Luba-Kaonde", + "level9": "Kaonde-Shaba-Sanga" + }, + "luc": { + "level0": "Central Sudanic", + "level1": "Moru-Madi", + "level2": "Central Moru-Madi" + }, + "lud": { + "level0": "Uralic", + "level1": "Finnic", + "level2": "Coastal Finnic", + "level3": "Neva", + "level4": "North Finnic", + "level5": "Ladogan", + "level6": "East Ladoga" + }, + "lue": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Chokwe-Ngangela-Nyemba (K.20)", + "level11": "Chokwe-Lwena" + }, + "luf": { + "level0": "Mailuan" + }, + "lug": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "North Nyanza" + }, + "lui": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Californian Uto-Aztecan", + "level3": "Cupan" + }, + "luj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Mbagani-Lwalwa" + }, + "luk": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Southern Tibetic", + "level7": "Dzongkhic", + "level8": "Nuclear Dzongkhic" + }, + "lul": { + "level0": "Central Sudanic", + "level1": "Moru-Madi", + "level2": "Southern Moru-Madi" + }, + "lum": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Chokwe-Ngangela-Nyemba (K.20)", + "level11": "Ngangela-Nyemba", + "level12": "Mbwela-Mbunda" + }, + "lun": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Ruund-Salampasu", + "level11": "Lunda-Ruund-Kete" + }, + "luo": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Southern Lwoo", + "level4": "Adhola-Alur-Luo", + "level5": "Adhola-Luo" + }, + "lup": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "West Kikongo", + "level20": "Vilic", + "level21": "Lumbuic", + "level22": "Lumbu-Bwisi" + }, + "luq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede", + "level7": "Eastern Ede", + "level8": "Southeastern Ede", + "level9": "Nuclear Yoruba", + "level10": "Lucumi-Yoruba" + }, + "lus": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin", + "level5": "Mizoic" + }, + "lut": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Central Salish", + "level3": "Lushootseed-Puget" + }, + "luu": { + "level0": "Bookkeeping" + }, + "luv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Sindhic", + "level9": "Unclassified Sindhic" + }, + "luw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Mambila-Mbongno", + "level10": "Mambila", + "level11": "Njerup" + }, + "luz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Luric-Dezfulic", + "level8": "Luric", + "level9": "Bakhtiari-Southern Lori" + }, + "lva": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku" + }, + "lvi": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric" + }, + "lvl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric" + }, + "lvu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata", + "level4": "Lamaholot Tengah", + "level5": "Southeast Lembata" + }, + "lwa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Mbagani-Lwalwa" + }, + "lwe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata", + "level4": "Lamaholot Timur" + }, + "lwg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Central-Eastern Luyia" + }, + "lwh": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Southwestern Kra", + "level3": "Western Kra", + "level4": "Lachic" + }, + "lwl": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Waic", + "level5": "Wa-Lawa", + "level6": "Lawa" + }, + "lwm": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Bisoid", + "level7": "Bisu-Pyen-Laomian" + }, + "lwo": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Northern Lwoo", + "level4": "Luwo-Thuri" + }, + "lws": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "lwt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata", + "level4": "Lamaholot Barat", + "level5": "Flores Lamaholot" + }, + "lwu": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lawoish" + }, + "lww": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Epi-Efate", + "level7": "Epi", + "level8": "Lamenu-Lewo" + }, + "lxm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Tungak-Nalik" + }, + "lya": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Southern Tibetic", + "level7": "Dzongkhic", + "level8": "Nuclear Dzongkhic" + }, + "lyg": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Khasian", + "level3": "Khasi-Pnar-Lyngngam", + "level4": "Lyngngamic" + }, + "lyn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Greater Luyana", + "level8": "Eastern Greater Luyana" + }, + "lzh": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic" + }, + "lzl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Central-Western Malakula" + }, + "lzn": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "South Patkaian", + "level4": "Southeastern Patkaian", + "level5": "Lainongic" + }, + "lzz": { + "level0": "Kartvelian", + "level1": "Georgian-Zan", + "level2": "Zan" + }, + "maa": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Mazatecan", + "level5": "Northwest Alta Mazatec" + }, + "mab": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Eastern Alta Mixtec", + "level7": "Southeastern Alta Mixtec", + "level8": "Teozacoalco Mixtec" + }, + "mad": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Maduresic" + }, + "mae": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Southeastern Benue-Congo Plateau" + }, + "maf": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Matakam" + }, + "mag": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Magadhan" + }, + "mah": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian" + }, + "mai": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Magadhan" + }, + "maj": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Mazatecan", + "level5": "Valley Mazatec" + }, + "mak": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Makassaric", + "level5": "Nuclear Makassaric" + }, + "mal": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "mam": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Mamean", + "level4": "Mamean" + }, + "maq": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Mazatecan" + }, + "mar": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Southern zone", + "level7": "Marathic", + "level8": "Marathi-Konkani", + "level9": "Old-Modern Marathi", + "level10": "Modern Marathi" + }, + "mas": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Lotuxo-Maa", + "level4": "Ongamo-Maa", + "level5": "Nuclear Maa" + }, + "mat": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Matlatzincan" + }, + "mau": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Mazatecan", + "level5": "Central Mazatec" + }, + "mav": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani" + }, + "maw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Southeast Western Oti-Volta", + "level13": "Mampruli-Dagbani" + }, + "max": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay", + "level6": "Eastern Indonesia Trade Malay", + "level7": "Manadoic Malay" + }, + "maz": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Otomian", + "level5": "Mazahua" + }, + "mba": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "North Manobo", + "level5": "Kinamiguin-Bukidnon", + "level6": "Bukidnon" + }, + "mbb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "Central and Southern Manobo", + "level5": "East-West-Central Manobo", + "level6": "West Manobo", + "level7": "WBM-Livunganen-Ilianen" + }, + "mbc": { + "level0": "Cariban", + "level1": "Venezuelan Cariban", + "level2": "Pemong-Panare", + "level3": "Pemongan" + }, + "mbd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "Central and Southern Manobo", + "level5": "East-West-Central Manobo", + "level6": "East and Central Manobo", + "level7": "East Manobo" + }, + "mbf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay" + }, + "mbg": { + "level0": "Bookkeeping" + }, + "mbh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Bebeli-Mangseng" + }, + "mbi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "Central and Southern Manobo", + "level5": "East-West-Central Manobo", + "level6": "West Manobo", + "level7": "WBM-Livunganen-Ilianen" + }, + "mbj": { + "level0": "Naduhup" + }, + "mbk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Siau", + "level8": "Sissano-Tumleo", + "level9": "Sera-Sissano", + "level10": "Sissanoic" + }, + "mbl": { + "level0": "Nuclear-Macro-Je", + "level1": "Maxakali-Borum", + "level2": "Maxakalian", + "level3": "Nuclear Maxakalian" + }, + "mbn": { + "level0": "Guahiboan", + "level1": "Nuclear Guahiboan" + }, + "mbo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Lundu-Balong (A.10)", + "level8": "Greater Manenguba", + "level9": "Bafaw-Balong-Manenguba", + "level10": "Manenguba" + }, + "mbp": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Magdalenic", + "level3": "Northern Magdalenic", + "level4": "Arhuacic", + "level5": "Eastern-Southern Arhuacic", + "level6": "Eastern Arhuacic" + }, + "mbq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage" + }, + "mbr": { + "level0": "Kakua-Nukak" + }, + "mbs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "Central and Southern Manobo", + "level5": "South Manobo", + "level6": "Sarangani-Tasaday-Cotabato" + }, + "mbt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "Central and Southern Manobo", + "level5": "East-West-Central Manobo", + "level6": "East and Central Manobo", + "level7": "Central Manobo" + }, + "mbu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan", + "level7": "Numan" + }, + "mbv": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Naluic" + }, + "mbw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Jimi" + }, + "mbx": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Central Sepik Hill", + "level3": "Nuclear Central Sepik Hill" + }, + "mby": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Sindhic", + "level9": "Unclassified Sindhic" + }, + "mbz": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec" + }, + "mca": { + "level0": "Mataguayan", + "level1": "Mataguayo I" + }, + "mcb": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha", + "level3": "Pre-Andine Maipuran", + "level4": "Asha-Ashe-Kak-Matsi-Nan", + "level5": "Matsi-Nan" + }, + "mcc": { + "level0": "Anim", + "level1": "Tirio", + "level2": "Nuclear Tirio" + }, + "mcd": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Headwaters Pano", + "level5": "Yaminawa Complex" + }, + "mce": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Southwestern Alta Mixtec", + "level8": "Chalcatongic" + }, + "mcf": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mayoruna Branch", + "level3": "Mayo Group", + "level4": "Matses subgroup" + }, + "mcg": { + "level0": "Cariban", + "level1": "Venezuelan Cariban", + "level2": "Mapoyo-Tamanaku", + "level3": "Mapoyo-Yawarana" + }, + "mch": { + "level0": "Cariban", + "level1": "Guianan", + "level2": "Maquiritari-Wayumara" + }, + "mci": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Rawlinson", + "level5": "Sankwep" + }, + "mcj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Mambila-Mbongno", + "level10": "Mbongno-Mvano", + "level11": "Mvano-Ndunda" + }, + "mck": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Chokwe-Ngangela-Nyemba (K.20)", + "level11": "Ngangela-Nyemba", + "level12": "Mbwela-Mbunda" + }, + "mcl": { + "level0": "Tucanoan", + "level1": "Western Tucanoan", + "level2": "Napo Tucanoan", + "level3": "Siona-Secoya", + "level4": "Sionan" + }, + "mcm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Luso-Asian Creole" + }, + "mcn": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Masa", + "level3": "North Masa", + "level4": "Masa-Gizey-Ham" + }, + "mco": { + "level0": "Mixe-Zoque", + "level1": "Mixe", + "level2": "Oaxaca Mixe", + "level3": "Lowland-Midland-South Highland Mixe", + "level4": "Lowland-Midland Mixe", + "level5": "Lowland Mixe" + }, + "mcp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Western A80", + "level10": "Makaaic", + "level11": "North-Central Makaaic" + }, + "mcq": { + "level0": "Koiarian", + "level1": "Baraic" + }, + "mcr": { + "level0": "Angan", + "level1": "Nuclear Angan", + "level2": "Kapau-Menya" + }, + "mcs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Northern Mbum", + "level6": "Tupuri-Mundang-Mambai" + }, + "mcu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Mambila-Mbongno", + "level10": "Mambila", + "level11": "Eastern Mambila" + }, + "mcv": { + "level0": "Anim", + "level1": "Inland Gulf of Papua", + "level2": "West Inland Gulf of Papua" + }, + "mcw": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.3" + }, + "mcx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Mpoic", + "level10": "Mpiemo-Ukhwejo" + }, + "mcy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Watut" + }, + "mcz": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Silopic" + }, + "mda": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Ninzic", + "level5": "Rukubic", + "level6": "Mada-Ninzam" + }, + "mdb": { + "level0": "Kiwaian", + "level1": "Turama-Kerewo" + }, + "mdc": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Mindjim", + "level4": "Lower Minjim", + "level5": "Inland Minjim" + }, + "mdd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic" + }, + "mde": { + "level0": "Maban", + "level1": "Mabang", + "level2": "Maba-Masalit", + "level3": "Macro-Maba" + }, + "mdf": { + "level0": "Uralic", + "level1": "Mordvin" + }, + "mdg": { + "level0": "Maban", + "level1": "Mabang", + "level2": "Maba-Masalit", + "level3": "Macro-Masalit" + }, + "mdh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Danaw" + }, + "mdi": { + "level0": "Central Sudanic", + "level1": "Membi-Mangbutu-Efe", + "level2": "Mangbutu-Efe" + }, + "mdj": { + "level0": "Central Sudanic", + "level1": "Mangbetu-Asua", + "level2": "Mangbetuic" + }, + "mdk": { + "level0": "Central Sudanic", + "level1": "Membi-Mangbutu-Efe", + "level2": "Mangbutu-Efe" + }, + "mdl": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "mdm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Eastern Mundu-Baka", + "level7": "Mayogo-Bangba" + }, + "mdn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Likouala-Sangha", + "level10": "Bwamba-Ngondi-Pande-Mbati-Aka" + }, + "mdo": { + "level0": "Bookkeeping" + }, + "mdp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Northern Njila", + "level9": "Mbala-Holu-Sondi (K.10)", + "level10": "Mbala-Sondi" + }, + "mdq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke" + }, + "mdr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi" + }, + "mds": { + "level0": "Manubaran" + }, + "mdt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie", + "level17": "Northwest Kasai-Ngounie", + "level18": "Mbere (B.60)", + "level19": "Tsitsekeic", + "level20": "Lekaningic" + }, + "mdu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Mboshi (C.20)" + }, + "mdv": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Southwestern Alta Mixtec", + "level8": "Chalcatongic", + "level9": "Atatlahuca-Monteverde" + }, + "mdw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Mboshi (C.20)", + "level10": "Koyo-Mboshi" + }, + "mdx": { + "level0": "Dizoid" + }, + "mdy": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "North-West Ometo" + }, + "mdz": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup IV", + "level6": "Tupi-Guarani Subgroup IV.A" + }, + "mea": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Southwest Grassfields", + "level8": "Menka-Atong" + }, + "meb": { + "level0": "Turama-Kikori", + "level1": "Turama-Omatian" + }, + "mec": { + "level0": "Mangarrayi-Maran", + "level1": "Maran" + }, + "med": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Hagen", + "level3": "Melpa-Tembagla" + }, + "mee": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Mengenic", + "level9": "Mamusa-Mengen" + }, + "mef": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Khasian", + "level3": "Khasi-Pnar-Lyngngam", + "level4": "Lyngngamic" + }, + "meh": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Southwestern Alta Mixtec" + }, + "mei": { + "level0": "Nubian" + }, + "mej": { + "level0": "East Bird's Head", + "level1": "Meax" + }, + "mek": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "West Central Papuan linkage", + "level9": "Nuclear West Central Papuan linkage" + }, + "mel": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Melanau-Kajang", + "level5": "Melanau" + }, + "mem": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Marrngu" + }, + "men": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Southwest Mande", + "level4": "Mende-Loma", + "level5": "Mende-Bandi", + "level6": "Mende-Loko" + }, + "meo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Greater Riau-Johoric", + "level6": "Northeastern Peninsular Malay" + }, + "mep": { + "level0": "Jarrakan", + "level1": "Miriwunic" + }, + "meq": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mofuic", + "level6": "Meri", + "level7": "Dugwor-Merey" + }, + "mer": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Central Kenya Bantu", + "level9": "Eastern Kirinyaga", + "level10": "Northern Kirinyaga", + "level11": "Nithi-Meru" + }, + "mes": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Mubic" + }, + "met": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Bel-Roinji-Nenaya" + }, + "meu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "West Central Papuan linkage" + }, + "mev": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Mano-Dan" + }, + "mew": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Unclassified Boleic" + }, + "mey": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "North African Arabic" + }, + "mez": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian" + }, + "mfa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Greater Riau-Johoric", + "level6": "Northeastern Peninsular Malay" + }, + "mfb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Northern Sumatra Malay", + "level6": "Bangka-Belitung Malay" + }, + "mfc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mbaic", + "level6": "Ndunga-Mba-Dongo", + "level7": "Ndunga-Mba" + }, + "mfd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Ngembaic", + "level10": "Mankonic" + }, + "mfe": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil", + "level14": "Macro-French", + "level15": "Isle-de-France Creole" + }, + "mff": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Beboid", + "level6": "Eastern Beboid", + "level7": "Bebe-Kemezung", + "level8": "Naki-Kemezung", + "level9": "Nakic" + }, + "mfg": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Nuclear Mokole", + "level8": "Mixiforic" + }, + "mfh": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mandaraic", + "level6": "Podoko" + }, + "mfi": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mandaraic", + "level6": "Wandala-Malgwa-Glavda", + "level7": "Wandala-Malgwa" + }, + "mfj": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Matakam", + "level5": "Mefele-Cuvok" + }, + "mfk": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mofuic", + "level6": "Mofu" + }, + "mfl": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Bura-Marghi", + "level6": "Buraic" + }, + "mfm": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Bura-Marghi", + "level6": "Marghic", + "level7": "Kilba-South Margi" + }, + "mfn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "East-West Central Delta Cross", + "level7": "Mbembe-Legbo" + }, + "mfo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Ekoid-Mbe" + }, + "mfp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay" + }, + "mfq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Gurma-Yom-Naudem", + "level11": "Gurma", + "level12": "Gurma B", + "level13": "Gourmantche-Moba", + "level14": "Moba-Bimoba" + }, + "mfr": { + "level0": "Western Daly", + "level1": "Bringen", + "level2": "Marithielic" + }, + "mfs": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic" + }, + "mft": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "Mokoreng-Loniu" + }, + "mfu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Chokwe-Ngangela-Nyemba (K.20)", + "level11": "Ngangela-Nyemba", + "level12": "Mbwela-Mbunda" + }, + "mfv": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Manjaku-Mankanya-Pepel", + "level6": "Cur-Bok-Cotier" + }, + "mfw": { + "level0": "Kwalean" + }, + "mfx": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "North-West Ometo", + "level3": "Central Ometo" + }, + "mfy": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Cahitan" + }, + "mfz": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Burun", + "level3": "Southern Burun" + }, + "mgb": { + "level0": "Tamaic" + }, + "mgc": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "Baka-Beli", + "level3": "Morokodo-Beli", + "level4": "Gberi-Morokodo-Mittu" + }, + "mgd": { + "level0": "Central Sudanic", + "level1": "Moru-Madi" + }, + "mge": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Logone-Chari", + "level7": "Bediondo" + }, + "mgf": { + "level0": "Bulaka River" + }, + "mgg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Mpoic" + }, + "mgh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Makua-Lomwe" + }, + "mgi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Jilic-Eggonic", + "level5": "Jilic" + }, + "mgj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Central Delta", + "level5": "Unclassified Central Delta" + }, + "mgl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage" + }, + "mgm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Timor" + }, + "mgn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Ngbandi-Mongoba-Kazibati", + "level6": "Ngbandic", + "level7": "Nuclear Ngbandic" + }, + "mgo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Momo", + "level8": "Widikum-Tadkon" + }, + "mgp": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kham-Magar-Chepang", + "level4": "Magar" + }, + "mgq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Mbeya" + }, + "mgr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Mwika", + "level10": "Fipaic", + "level11": "Maluwawaru" + }, + "mgs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Southern Tanzania Highlands Bantu", + "level9": "Manda-Ngoni" + }, + "mgt": { + "level0": "Keram", + "level1": "Ulmapo", + "level2": "Mwakai-Pondi" + }, + "mgu": { + "level0": "Mailuan" + }, + "mgv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Rufijic", + "level9": "Matengic" + }, + "mgw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Rufijic", + "level9": "Matumbic" + }, + "mgx": { + "level0": "Bookkeeping" + }, + "mgy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Rufijic" + }, + "mgz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Mbugwe-Langi" + }, + "mha": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Konda-Kui", + "level4": "Manda-Kui", + "level5": "Manda-Pengo" + }, + "mhb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Ndasaic", + "level8": "Kota-Mahongwe" + }, + "mhc": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Kanjobalan-Chujean", + "level4": "Kanjobalan" + }, + "mhd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Pare-Taveta", + "level10": "Pareic" + }, + "mhe": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "South Aslian" + }, + "mhf": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Erap", + "level4": "Finungwan-Mamaa-Gusan" + }, + "mhg": { + "level0": "Marrku-Wurrugu" + }, + "mhh": { + "level0": "Bookkeeping" + }, + "mhi": { + "level0": "Central Sudanic", + "level1": "Moru-Madi", + "level2": "Southern Moru-Madi" + }, + "mhj": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic" + }, + "mhk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Mbam-Nkam Nun" + }, + "mhl": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kumil-Tibor", + "level6": "Kumil" + }, + "mhm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Makua-Lomwe" + }, + "mhn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Bairisch", + "level10": "Global South Bavarian" + }, + "mho": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Greater Luyana", + "level8": "Western Greater Luyana", + "level9": "Mashi-Mbukushi" + }, + "mhp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Greater Riau-Johoric" + }, + "mhq": { + "level0": "Siouan" + }, + "mhr": { + "level0": "Uralic", + "level1": "Mari" + }, + "mhs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "West Central Maluku", + "level3": "Sula-Buru", + "level4": "Buruic" + }, + "mht": { + "level0": "Arawakan", + "level1": "Japura-Colombia", + "level2": "Nuclear Japura-Colombia", + "level3": "Northeast Japura-Colombia", + "level4": "Cassiquiare" + }, + "mhu": { + "level0": "Sino-Tibetan", + "level1": "Digarish" + }, + "mhv": { + "level0": "Bookkeeping" + }, + "mhw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Greater Luyana", + "level8": "Western Greater Luyana", + "level9": "Mashi-Mbukushi" + }, + "mhx": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Northern Burmish", + "level5": "Maruic" + }, + "mhy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Maanyan-Paku" + }, + "mhz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea" + }, + "mia": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian" + }, + "mib": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Southwestern Alta Mixtec", + "level8": "Chalcatongic", + "level9": "Atatlahuca-Monteverde" + }, + "mic": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Maritimes-Southern New England Algonquian", + "level5": "Northern Eastern Algonquian", + "level6": "Micmacic" + }, + "mid": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Macro-Mandaic" + }, + "mie": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec" + }, + "mif": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mofuic", + "level6": "Mofu" + }, + "mig": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Southwestern Alta Mixtec", + "level8": "Chalcatongic" + }, + "mih": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Coast Mixtec", + "level7": "East Coast Mixtec" + }, + "mii": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Northern Baja Mixtec" + }, + "mij": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Yemne-Kimbi" + }, + "mik": { + "level0": "Muskogean" + }, + "mil": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Eastern Alta Mixtec", + "level7": "Southeastern Alta Mixtec", + "level8": "Tlazoyal-Penoles" + }, + "mim": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Guerrero Mixtec", + "level7": "Nuclear Guerrero Mixtec" + }, + "min": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Northern Sumatra Malay", + "level6": "Kerinci-Minangkabau", + "level7": "Minangkabauic" + }, + "mio": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Coast Mixtec", + "level7": "West Coast Mixtec" + }, + "mip": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Northeastern Alta Mixtec" + }, + "miq": { + "level0": "Misumalpan" + }, + "mir": { + "level0": "Mixe-Zoque", + "level1": "Mixe", + "level2": "Oaxaca Mixe", + "level3": "Lowland-Midland-South Highland Mixe", + "level4": "Lowland-Midland Mixe", + "level5": "Lowland Mixe" + }, + "mit": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Northern Baja Mixtec" + }, + "miu": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Tezoatlanic" + }, + "miv": { + "level0": "Bookkeeping" + }, + "miw": { + "level0": "Angan", + "level1": "Nuclear Angan", + "level2": "Ankave-Tainae-Akoye", + "level3": "Tainae-Akoye" + }, + "mix": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Mixtepec-Yucunicoco Mixtec" + }, + "miy": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Guerrero Mixtec", + "level7": "Nuclear Guerrero Mixtec", + "level8": "Southwestern Guerrero Mixtec" + }, + "miz": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Northern Alta Mixtec" + }, + "mja": { + "level0": "Bookkeeping" + }, + "mjc": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Coast Mixtec", + "level7": "West Coast Mixtec" + }, + "mjd": { + "level0": "Maiduan" + }, + "mje": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Musguic" + }, + "mjg": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Southern Periphery Mongolic", + "level3": "Shirongol", + "level4": "Monguoric" + }, + "mjh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Tumbuka-Sena-Nyanja", + "level8": "Sena-Nyanja", + "level9": "Nyanjaic" + }, + "mji": { + "level0": "Hmong-Mien", + "level1": "Mienic", + "level2": "Mien-Mun" + }, + "mjj": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kumil-Tibor", + "level6": "Tibor", + "level7": "Nuclear Tibor" + }, + "mjk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Bel-Roinji-Nenaya", + "level9": "Bel", + "level10": "Western Bel" + }, + "mjl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali" + }, + "mjm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Kairiru-Manam", + "level8": "Manamic linkage" + }, + "mjn": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Yupna", + "level4": "Bwana-Moam-Tapen" + }, + "mjo": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "mjp": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid" + }, + "mjq": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "mjr": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "mjs": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3", + "level6": "Kofyar-Mushere-Chip" + }, + "mjt": { + "level0": "Dravidian", + "level1": "North Dravidian", + "level2": "Kurux-Malto", + "level3": "Malto" + }, + "mjv": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid", + "level10": "Muthuvan-Mannan" + }, + "mjw": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Karbic" + }, + "mjx": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Santalic" + }, + "mjy": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Delawaran", + "level5": "Mahican-Woronoco-Pojassick" + }, + "mjz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Tharuic", + "level10": "Unclassified Tharu" + }, + "mka": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Unclassified Volta-Congo" + }, + "mkb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga" + }, + "mkc": { + "level0": "Nuclear Torricelli", + "level1": "Nuclear Maimai" + }, + "mkd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "South Slavic", + "level5": "Eastern South Slavic", + "level6": "Macedo-Bulgarian" + }, + "mke": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "mkf": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.2", + "level5": "Nuclear West Chadic B.2", + "level6": "Central West Chadic B.2" + }, + "mkg": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Kam-Sui", + "level3": "Then-MMS", + "level4": "Maonan-Mak-Sui", + "level5": "Mak-Ai-Cham" + }, + "mki": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Western Rajasthani", + "level11": "Indus Rajasthani" + }, + "mkj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Ponapeic" + }, + "mkk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Western A80", + "level10": "Makaaic", + "level11": "North-Central Makaaic" + }, + "mkl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede" + }, + "mkm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Moken-Moklen" + }, + "mkn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay", + "level6": "Eastern Indonesia Trade Malay" + }, + "mko": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bikwin-Jen", + "level5": "Southern Bikwin-Jen", + "level6": "Jen", + "level7": "Doso-Dza" + }, + "mkp": { + "level0": "Yareban", + "level1": "Doriri-Abia" + }, + "mkq": { + "level0": "Miwok-Costanoan", + "level1": "Miwokan", + "level2": "Eastern Miwokan" + }, + "mkr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Manep-Barem" + }, + "mks": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Central-Western Baja Mixtec" + }, + "mkt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Voh-Kone-Cem-Pac", + "level10": "Voh-Kone" + }, + "mku": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding", + "level9": "Manenkan", + "level10": "Konya-Manya" + }, + "mkv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "East Santo", + "level9": "Mafea-Tutuba" + }, + "mkw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "Southeastern Kikongo", + "level20": "Southern Kikongo", + "level21": "Koongo-Kituba" + }, + "mkx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "North Manobo", + "level5": "Kinamiguin-Bukidnon" + }, + "mky": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "South Halmahera", + "level6": "East Makian-Gane" + }, + "mkz": { + "level0": "Timor-Alor-Pantar", + "level1": "East Timor" + }, + "mla": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "South Santo" + }, + "mlb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Bati-Mbure-Yambassa", + "level10": "Mbure-Yambassa" + }, + "mlc": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai" + }, + "mld": { + "level0": "Bookkeeping" + }, + "mle": { + "level0": "Ndu", + "level1": "Nuclear Ndu", + "level2": "Manambu-Yalaku" + }, + "mlf": { + "level0": "Austroasiatic", + "level1": "Khmuic", + "level2": "Phay-Pram", + "level3": "Tinic", + "level4": "Tin" + }, + "mlh": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Eastern Huon", + "level4": "Trans Vitiaz", + "level5": "Huon Tip", + "level6": "Kate-Mape" + }, + "mli": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Masenrempulu" + }, + "mlj": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.3", + "level5": "Sokoroic", + "level6": "Miltuic" + }, + "mlk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Sabaki-Swahili" + }, + "mll": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Northern Malakula" + }, + "mlm": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Kam-Sui", + "level3": "Mulam-Kam" + }, + "mln": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Guadalcanal-Nggelic", + "level6": "Nuclear Guadalcanal-Nggelic", + "level7": "North and West Guadalcanal" + }, + "mlo": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Jola", + "level6": "FH-Jola", + "level7": "PF-Jola", + "level8": "Kwatay-Karon-Mlomp", + "level9": "Karon-Mlomp" + }, + "mlp": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles" + }, + "mlq": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "West Manding", + "level9": "Xasonka" + }, + "mlr": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "Hurza" + }, + "mls": { + "level0": "Maban", + "level1": "Mabang", + "level2": "Maba-Masalit", + "level3": "Macro-Masalit" + }, + "mlt": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "North African Arabic", + "level7": "Malta-Tunisian Arabic" + }, + "mlu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Central-Northern Malaita", + "level9": "North Malaitan" + }, + "mlv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage" + }, + "mlw": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mofuic", + "level6": "Tokombere" + }, + "mlx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Peripheral Western Malakula", + "level9": "Southwestern Malakula", + "level10": "Southwest Coastal Malekula" + }, + "mly": { + "level0": "Bookkeeping" + }, + "mma": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan", + "level6": "Nigerian Jarawan" + }, + "mmb": { + "level0": "Somahai" + }, + "mmc": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Otomian", + "level5": "Mazahua" + }, + "mmd": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Kam-Sui", + "level3": "Then-MMS", + "level4": "Maonan-Mak-Sui", + "level5": "Maonan-Chadong" + }, + "mme": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Peripheral Western Malakula", + "level9": "Northwestern Malakula" + }, + "mmf": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.4", + "level5": "Ronic", + "level6": "Mundat-Karfa" + }, + "mmg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Ambrym" + }, + "mmh": { + "level0": "Arawakan", + "level1": "Central-Eastern Maipuran", + "level2": "Central Maipuran", + "level3": "Xinguan Arawak", + "level4": "Waura-Mehinaku-Kustenau", + "level5": "Waura-Mehinaku" + }, + "mmi": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kumil-Tibor", + "level6": "Tibor", + "level7": "Nuclear Tibor" + }, + "mmj": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Mundaric" + }, + "mmk": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Teluguic" + }, + "mml": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Angkuic", + "level5": "Southern Angkuic" + }, + "mmm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Epi-Efate", + "level7": "Epi", + "level8": "Bieria-Maii" + }, + "mmn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine" + }, + "mmo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Buang linkage", + "level9": "Mapos-Mangga-Wagau" + }, + "mmp": { + "level0": "Amto-Musan" + }, + "mmq": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Sogeram", + "level5": "East Sogeram", + "level6": "Aisian" + }, + "mmr": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "North Hmongic" + }, + "mms": { + "level0": "Bookkeeping" + }, + "mmt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Ngero", + "level8": "Western Ngero" + }, + "mmu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Bati-Mbure-Yambassa", + "level10": "Mbure-Yambassa", + "level11": "Yambassa (A.60)", + "level12": "Mmala-Elip-Gunu" + }, + "mmv": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Eastern Eastern Tucanoan", + "level3": "Eastern Eastern Tucanoan II", + "level4": "Kotiria-Piratapuyo", + "level5": "Piratapuyic", + "level6": "Arapaso-Miriti" + }, + "mmw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Vanuatu-Loyalty Outliers" + }, + "mmx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Madak linkage" + }, + "mmy": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Dangla-Mabire-Birgit", + "level6": "Dangla" + }, + "mmz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Libinzic", + "level13": "Libinza Fleuve" + }, + "mna": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage" + }, + "mnb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Nuclear Muna-Buton", + "level8": "Munan", + "level9": "Munic", + "level10": "Western Munic" + }, + "mnc": { + "level0": "Tungusic", + "level1": "Manchu-Jurchen", + "level2": "Manchu-Xibe" + }, + "mnd": { + "level0": "Tupian", + "level1": "Monde", + "level2": "Gavianic" + }, + "mne": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Bagirmic", + "level6": "Morom-Jaya-Naba", + "level7": "Naba-Berakou" + }, + "mnf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Momo", + "level8": "Mundani-Njen" + }, + "mng": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "South Bahnaric", + "level4": "Mnong-Stieng-Chrau", + "level5": "Mnong" + }, + "mnh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Central Bandaic", + "level8": "Central Core Bandaic", + "level9": "Mid-Southern Central Core Bandaic" + }, + "mni": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga" + }, + "mnj": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Eastern Iranian", + "level5": "Yidgha-Munji" + }, + "mnk": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "West Manding" + }, + "mnl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "Central Santo" + }, + "mnm": { + "level0": "Dagan", + "level1": "Central Dagan", + "level2": "Southwest Dagan" + }, + "mnn": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "South Bahnaric", + "level4": "Mnong-Stieng-Chrau", + "level5": "Mnong", + "level6": "Southern-Central Mnong" + }, + "mnp": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Min", + "level3": "Inland Min" + }, + "mnq": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "North Aslian", + "level4": "Maniq-Menraq-Batek", + "level5": "Menraq-Batek" + }, + "mnr": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Numic", + "level3": "Western Numic" + }, + "mns": { + "level0": "Uralic", + "level1": "Mansic", + "level2": "North-Central Mansi" + }, + "mnt": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Mayabic", + "level3": "Nuclear Mayabic" + }, + "mnu": { + "level0": "Mairasic" + }, + "mnv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian" + }, + "mnw": { + "level0": "Austroasiatic", + "level1": "Monic" + }, + "mnx": { + "level0": "East Bird's Head" + }, + "mny": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu" + }, + "mnz": { + "level0": "Nuclear Trans New Guinea", + "level1": "Paniai Lakes" + }, + "moa": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Nwa-Ben", + "level4": "Wan-Mwan" + }, + "mob": { + "level0": "Bookkeeping" + }, + "moc": { + "level0": "Guaicuruan", + "level1": "Guaicuru del Sur", + "level2": "Qom" + }, + "mod": { + "level0": "Pidgin", + "level1": "Choctaw-based pidgin" + }, + "moe": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Cree-Montagnais-Naskapi" + }, + "mof": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Maritimes-Southern New England Algonquian", + "level5": "Southern New England Algonquian", + "level6": "Western Southern New England Algonquian" + }, + "mog": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Gorontalo-Mongondow", + "level4": "Mongondowic" + }, + "moh": { + "level0": "Iroquoian", + "level1": "Northern Iroquoian", + "level2": "Mohawk-Oneida" + }, + "moi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bena-Mboi" + }, + "moj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Western Mundu-Baka", + "level7": "River Western Mundu-Baka", + "level8": "Monzomboic" + }, + "mom": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Tlapanec-Manguean", + "level3": "Manguean" + }, + "moo": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric", + "level3": "Hre-Sedang-Todrah-Monam" + }, + "mop": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Yucatecan" + }, + "mor": { + "level0": "Heibanic", + "level1": "West-Central Heibanic", + "level2": "Western Heibanic" + }, + "mos": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Northwest Oti-Volta", + "level13": "Mossi-Farefare", + "level14": "Mossic" + }, + "mot": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Magdalenic", + "level3": "Southern Magdalenic" + }, + "mou": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Dangla-Mabire-Birgit", + "level6": "Birgit-Mogum-Toram" + }, + "mov": { + "level0": "Cochimi-Yuman", + "level1": "Yuman", + "level2": "General Yuman", + "level3": "River Yuman" + }, + "mow": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Bobangic", + "level13": "Bobangic Riverain" + }, + "mox": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Bwaidoga linkage" + }, + "moy": { + "level0": "Ta-Ne-Omotic", + "level1": "Kefoid", + "level2": "South Gonga" + }, + "moz": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B" + }, + "mpa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Rufijic", + "level9": "Matengic" + }, + "mpb": { + "level0": "Northern Daly" + }, + "mpc": { + "level0": "Mangarrayi-Maran" + }, + "mpd": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Purus-Chamicuro", + "level3": "Purus", + "level4": "Yineic" + }, + "mpe": { + "level0": "Surmic" + }, + "mpf": { + "level0": "Bookkeeping" + }, + "mpg": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Masa", + "level3": "North Masa", + "level4": "Marba-Musey" + }, + "mph": { + "level0": "Iwaidjan Proper" + }, + "mpi": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Kotoko-Buduma", + "level5": "Kotoko Septentrional", + "level6": "Kotoko Septentrional 2" + }, + "mpj": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Wati", + "level3": "Martuwangkic", + "level4": "Warnman-Wangka" + }, + "mpk": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Musguic", + "level5": "Musgu-Mbara" + }, + "mpl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Watut" + }, + "mpm": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Southwestern Alta Mixtec", + "level8": "Chalcatongic" + }, + "mpn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Bel-Roinji-Nenaya", + "level9": "Bel", + "level10": "Eastern Bel" + }, + "mpo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Pasismanua" + }, + "mpp": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Eastern Huon", + "level4": "Trans Vitiaz", + "level5": "Huon Tip", + "level6": "Sopac" + }, + "mpq": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mayoruna Branch", + "level3": "Mayo Group", + "level4": "Matis subgroup" + }, + "mpr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "New Georgia", + "level10": "East New Georgia", + "level11": "Marovo-Vangunu" + }, + "mps": { + "level0": "Teberan" + }, + "mpt": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Mountain Ok", + "level6": "Mianic" + }, + "mpu": { + "level0": "Tupian", + "level1": "Arikem-Tupari", + "level2": "Tuparic" + }, + "mpv": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Erap", + "level4": "Boana" + }, + "mpw": { + "level0": "Arawakan", + "level1": "Negro-Roraima", + "level2": "Pidjanan" + }, + "mpx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Kilivila-Misima" + }, + "mpy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Nuclear Trukic" + }, + "mpz": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Hanic", + "level7": "Bi-Ka" + }, + "mqa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "South Halmahera", + "level6": "Central-Eastern South Halmahera" + }, + "mqb": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "Hurza" + }, + "mqd": { + "level0": "Bookkeeping" + }, + "mqe": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Unclassified Hanseman" + }, + "mqf": { + "level0": "Somahai" + }, + "mqg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "East Borneo Malay" + }, + "mqh": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Eastern Alta Mixtec", + "level7": "Southeastern Alta Mixtec", + "level8": "Tlazoyal-Penoles" + }, + "mqi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Batuley-Mariri" + }, + "mqj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Torajic" + }, + "mqk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "Central and Southern Manobo", + "level5": "East-West-Central Manobo", + "level6": "East and Central Manobo", + "level7": "East Manobo" + }, + "mql": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Oti-Volta Oriental", + "level10": "Waama-Tayari-Ditammari", + "level11": "Tayari-Ditammari", + "level12": "Ditammaric" + }, + "mqm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Distal", + "level13": "Marquesan" + }, + "mqn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Eastern Bungku-Tolaki" + }, + "mqo": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Mainland North Halmaheran", + "level3": "Kao River" + }, + "mqp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "West Piru Bay", + "level5": "Hoamoal", + "level6": "West Hoamoal" + }, + "mqq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Dusunic", + "level7": "Kadazan-Sugut-Minokok", + "level8": "Sugut-Minokok Kadazan" + }, + "mqr": { + "level0": "Tor-Orya", + "level1": "Tor" + }, + "mqs": { + "level0": "North Halmahera" + }, + "mqt": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Angkuic", + "level5": "Southern Angkuic" + }, + "mqu": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Barian" + }, + "mqv": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Wamas-Samosa-Murupi-Mosimo" + }, + "mqw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Wamas-Samosa-Murupi-Mosimo" + }, + "mqx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi" + }, + "mqy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Manggaraiic", + "level6": "Manggarai Khusus" + }, + "mqz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Korap linkage" + }, + "mra": { + "level0": "Austroasiatic", + "level1": "Khmuic", + "level2": "Phay-Pram", + "level3": "Tinic" + }, + "mrb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Maewo" + }, + "mrc": { + "level0": "Cochimi-Yuman", + "level1": "Yuman", + "level2": "General Yuman", + "level3": "River Yuman" + }, + "mrd": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kham-Magar-Chepang", + "level4": "Magar" + }, + "mre": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "OKSLic" + }, + "mrg": { + "level0": "Sino-Tibetan", + "level1": "Macro-Tani", + "level2": "Tani", + "level3": "Eastern Tani" + }, + "mrh": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Maraic", + "level5": "Nuclear Maraic" + }, + "mri": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Proximal", + "level13": "Southern East Polynesian Proximal", + "level14": "Maoric" + }, + "mrj": { + "level0": "Uralic", + "level1": "Mari" + }, + "mrk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Voh-Kone-Cem-Pac", + "level10": "Voh-Kone" + }, + "mrl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Nuclear Trukic", + "level10": "Central Trukic", + "level11": "Eastern Trukic", + "level12": "Mortlockese-Trukese" + }, + "mrm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage" + }, + "mrn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Santa Isabel", + "level10": "East Santa Isabel" + }, + "mro": { + "level0": "Sino-Tibetan", + "level1": "Mruic" + }, + "mrp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "South-Central Santo" + }, + "mrq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Distal", + "level13": "Marquesan" + }, + "mrr": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Gondi", + "level4": "Southeast Gondi" + }, + "mrs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Peripheral Western Malakula", + "level9": "Northwestern Malakula" + }, + "mrt": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Bura-Marghi", + "level6": "Marghic" + }, + "mru": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Northern Mbum", + "level6": "Tupuri-Mundang-Mambai", + "level7": "Mundangic" + }, + "mrv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Distal", + "level13": "Far East Polynesian" + }, + "mrw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Danaw" + }, + "mrx": { + "level0": "Tor-Orya", + "level1": "Tor" + }, + "mry": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Mansakan", + "level5": "Eastern Mansakan" + }, + "mrz": { + "level0": "Anim", + "level1": "Marind-Boazi-Yaqai", + "level2": "Marindic" + }, + "msb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Central Bisayan", + "level6": "Peripheral Central Bisayan", + "level7": "Masbate-Sorsogon" + }, + "msc": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding", + "level9": "Manenkan" + }, + "msd": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Meemul-Tziij" + }, + "mse": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Masa", + "level3": "North Masa", + "level4": "Marba-Musey" + }, + "msf": { + "level0": "Nimboranic", + "level1": "Outer Nimboranic" + }, + "msg": { + "level0": "West Bird's Head", + "level1": "South West Bird's Head" + }, + "msh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "Southwestern Malagasic", + "level7": "South West-Central Malagasic", + "level8": "Nuclear South West-Central Malagasic", + "level9": "Inland-Western Malagasic", + "level10": "Western Malagasic" + }, + "msi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "East Borneo Malay", + "level6": "Banjar-Berau-Brunei Malay", + "level7": "Berau-Brunei Malay", + "level8": "Bruneic Malay" + }, + "msj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mbaic" + }, + "msk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Mansakan", + "level5": "Eastern Mansakan" + }, + "msm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "Central and Southern Manobo", + "level5": "East-West-Central Manobo", + "level6": "East and Central Manobo", + "level7": "East Manobo" + }, + "msn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage" + }, + "mso": { + "level0": "Mombum-Koneraw" + }, + "msp": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Jurunic", + "level3": "Unclassified Jurunic" + }, + "msq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Extreme Northern New Caledonian", + "level9": "Kum-Nel-Yua-Cac" + }, + "msr": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "RSLic", + "level3": "Nuclear RSLic" + }, + "mss": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar", + "level5": "South Babar", + "level6": "Masela-South Babar" + }, + "mst": { + "level0": "Bookkeeping" + }, + "msu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Lower Markham", + "level9": "Busu", + "level10": "Musom-Sirak" + }, + "msv": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Kotoko-Buduma", + "level5": "Kotoko Septentrional", + "level6": "Kotoko Septentrional 1" + }, + "msw": { + "level0": "Atlantic-Congo" + }, + "msx": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Osum-Wadaginam-Pomoikan", + "level5": "Pomoikan", + "level6": "Anamuxric" + }, + "msy": { + "level0": "Ramu", + "level1": "Lower Ramu", + "level2": "Ruboni", + "level3": "Mikarewan" + }, + "msz": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Eastern Huon", + "level4": "Trans Vitiaz", + "level5": "Huon Tip", + "level6": "Sopac" + }, + "mta": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "Central and Southern Manobo", + "level5": "South Manobo", + "level6": "Sarangani-Tasaday-Cotabato" + }, + "mtb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Central Tano", + "level7": "Bia", + "level8": "Northern Bia", + "level9": "Anyinic" + }, + "mtc": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Kokon" + }, + "mtd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Ibanic", + "level5": "Iban-Mualang-Seberuang" + }, + "mte": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Mono-Uruavan" + }, + "mtf": { + "level0": "Lower Sepik", + "level1": "Nor" + }, + "mtg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Mek", + "level2": "Eastern Mek" + }, + "mth": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Central-Western Yapen", + "level7": "Central Yapen" + }, + "mti": { + "level0": "Dagan", + "level1": "Central Dagan" + }, + "mtj": { + "level0": "East Bird's Head", + "level1": "Meax" + }, + "mtk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Nkambe" + }, + "mtl": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3", + "level6": "Goemaic" + }, + "mtm": { + "level0": "Uralic", + "level1": "Samoyedic" + }, + "mtn": { + "level0": "Misumalpan", + "level1": "Sumalpan", + "level2": "Matagalpan" + }, + "mto": { + "level0": "Mixe-Zoque", + "level1": "Mixe", + "level2": "Oaxaca Mixe" + }, + "mtp": { + "level0": "Mataguayan", + "level1": "Mataguayo II", + "level2": "Wichi" + }, + "mtq": { + "level0": "Austroasiatic", + "level1": "Vietic", + "level2": "Viet-Muong", + "level3": "Muongic" + }, + "mtr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Mewaric" + }, + "mts": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Headwaters Pano", + "level5": "Yaminawa Complex" + }, + "mtt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage" + }, + "mtu": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Coast Mixtec", + "level7": "East Coast Mixtec" + }, + "mtv": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Warup", + "level4": "Nuclear Warup", + "level5": "Molet-Asaroo" + }, + "mtw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Negrosanon" + }, + "mtx": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Eastern Alta Mixtec" + }, + "mty": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Wapeic" + }, + "mtz": { + "level0": "Bookkeeping" + }, + "mua": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Northern Mbum", + "level6": "Tupuri-Mundang-Mambai", + "level7": "Mundangic" + }, + "mub": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Mubic" + }, + "muc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Yemne-Kimbi" + }, + "mud": { + "level0": "Eskimo-Aleut", + "level1": "Aleutic" + }, + "mue": { + "level0": "Mixed Language", + "level1": "Spanish-Quechua" + }, + "mug": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Musguic", + "level5": "Musgu-Mbara" + }, + "muh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Eastern Mundu-Baka" + }, + "mui": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Central Sumatran Malay", + "level6": "Music" + }, + "muj": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Dangla-Mabire-Birgit" + }, + "muk": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Mustangic" + }, + "mum": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Taupota linkage", + "level10": "Nuclear Taupota linkage", + "level11": "Eastern Taupota" + }, + "muo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Southern Samba-Duru", + "level7": "Sambaic", + "level8": "Samba-Leko-Perema-Mumbake", + "level9": "Perema-Mumbake" + }, + "mup": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "muq": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "North Hmongic" + }, + "mur": { + "level0": "Surmic", + "level1": "South Surmic", + "level2": "Southwest Surmic", + "level3": "Didinga-Murle" + }, + "mus": { + "level0": "Muskogean" + }, + "mut": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Gondi", + "level4": "Northwest Gondi", + "level5": "Southwest Gondi", + "level6": "Muria" + }, + "muu": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Transversal Lowland East Cushitic" + }, + "muv": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid", + "level10": "Muthuvan-Mannan" + }, + "muw": { + "level0": "Bookkeeping" + }, + "mux": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Hagen", + "level3": "Melpa-Tembagla" + }, + "muy": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mofuic", + "level6": "Tokombere", + "level7": "Madaic" + }, + "muz": { + "level0": "Surmic", + "level1": "South Surmic", + "level2": "Southeast Surmic", + "level3": "Pastoral Surmic", + "level4": "Tirma-Chai-Mursi" + }, + "mva": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Kairiru-Manam", + "level8": "Manamic linkage", + "level9": "Bam-Manam", + "level10": "Manam-Sepa" + }, + "mvb": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Pacific Coast Athabaskan", + "level4": "California Athabaskan" + }, + "mvc": { + "level0": "Bookkeeping" + }, + "mvd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Sumba-Hawu", + "level5": "Sumba", + "level6": "Central-East Sumbanese" + }, + "mve": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Western Rajasthani", + "level11": "Indus Rajasthani" + }, + "mvf": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Eastern Mongolic", + "level3": "Khalkha-Buriat", + "level4": "Mongolian" + }, + "mvg": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Tlaxiacic", + "level8": "Yucuane-Teita" + }, + "mvh": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.1", + "level5": "Sumrayic", + "level6": "Ndam-Tumak" + }, + "mvi": { + "level0": "Japonic", + "level1": "Ryukyuan", + "level2": "Southern Ryukyu" + }, + "mvj": { + "level0": "Bookkeeping" + }, + "mvk": { + "level0": "Yuat" + }, + "mvl": { + "level0": "Pama-Nyungan", + "level1": "Paman" + }, + "mvn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Taupota linkage" + }, + "mvo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "New Georgia", + "level10": "East New Georgia", + "level11": "Marovo-Vangunu" + }, + "mvp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Masenrempulu" + }, + "mvq": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kumil-Tibor", + "level6": "Kumil" + }, + "mvr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Central-Western Yapen", + "level7": "Ansus-Ambai" + }, + "mvt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Northern Malakula", + "level8": "North Coast Malakula", + "level9": "Botovro-Vovo-Vao" + }, + "mvu": { + "level0": "Maban", + "level1": "Mabang", + "level2": "Maba-Masalit", + "level3": "Macro-Maba" + }, + "mvv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Northern Murutic", + "level8": "Sumambu-Tagal" + }, + "mvw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Ruvuma", + "level9": "Unclassified Ruvuma" + }, + "mvx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Biakic" + }, + "mvy": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Kohistani", + "level8": "Indus Kohistanic" + }, + "mvz": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Outer South Ethiopic", + "level6": "TT-Group" + }, + "mwa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Dobu-Duau linkage", + "level9": "Bunama-Mwatebu" + }, + "mwb": { + "level0": "Nuclear Torricelli", + "level1": "Marienberg", + "level2": "Mandi-Muniwara" + }, + "mwc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Are linkage", + "level10": "Are-Doga" + }, + "mwe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Ruvuma", + "level9": "Yaoic" + }, + "mwf": { + "level0": "Southern Daly" + }, + "mwg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Arawe", + "level11": "West Arawe" + }, + "mwh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Bibling" + }, + "mwi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Peripheral Western Malakula", + "level9": "Ninde-Nati" + }, + "mwj": { + "level0": "Bookkeeping" + }, + "mwk": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "West Manding", + "level9": "Kita-Kagoro" + }, + "mwl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Asturo-Leonese" + }, + "mwm": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Chari" + }, + "mwn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Mwika", + "level10": "Fipaic", + "level11": "Maluwawaru" + }, + "mwo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Maewo" + }, + "mwp": { + "level0": "Pama-Nyungan" + }, + "mwq": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "South Peripheral Kuki-Chin", + "level5": "Choic", + "level6": "Daai-Nghmoye-Muun-Kaang" + }, + "mws": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Central Kenya Bantu", + "level9": "Eastern Kirinyaga", + "level10": "Northern Kirinyaga", + "level11": "Nithi-Meru" + }, + "mwt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Moken-Moklen" + }, + "mwu": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "Baka-Beli", + "level3": "Morokodo-Beli", + "level4": "Gberi-Morokodo-Mittu" + }, + "mwv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran" + }, + "mww": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Chuanqiandian", + "level7": "First Vernacular Hmong", + "level8": "Far Western Miao" + }, + "mwx": { + "level0": "Bookkeeping" + }, + "mwy": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Kalenjin", + "level3": "Okiek-Akie" + }, + "mwz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke", + "level11": "So-Poke", + "level12": "So-Lebonya", + "level13": "Basoo" + }, + "mxa": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Central-Western Baja Mixtec" + }, + "mxb": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Tezoatlanic" + }, + "mxc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Shona (S.10)", + "level9": "Core Shona", + "level10": "Plateau Shona" + }, + "mxd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Kayan-Murik-Modang", + "level4": "Modang-Segai" + }, + "mxe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Vanuatu-Loyalty Outliers", + "level9": "Mele-Futuna" + }, + "mxf": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Kotoko-Buduma", + "level5": "Kotoko Septentrional", + "level6": "Kotoko Septentrional 2" + }, + "mxg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Northern Njila", + "level9": "Mbala-Holu-Sondi (K.10)", + "level10": "Holu (K.10)", + "level11": "Pheende-Kwezo" + }, + "mxh": { + "level0": "Central Sudanic", + "level1": "Membi-Mangbutu-Efe", + "level2": "Mangbutu-Efe", + "level3": "Leseic" + }, + "mxi": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Unshifted Western Romance" + }, + "mxj": { + "level0": "Sino-Tibetan", + "level1": "Kman-Meyor" + }, + "mxk": { + "level0": "Bogia" + }, + "mxl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Fongbeic" + }, + "mxm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "Willaumez linkage", + "level7": "Nakanai-Meramera" + }, + "mxn": { + "level0": "West Bird's Head", + "level1": "Seget-Moi" + }, + "mxo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Greater Luyana", + "level8": "Eastern Greater Luyana" + }, + "mxp": { + "level0": "Mixe-Zoque", + "level1": "Mixe", + "level2": "Oaxaca Mixe", + "level3": "Lowland-Midland-South Highland Mixe" + }, + "mxq": { + "level0": "Mixe-Zoque", + "level1": "Mixe", + "level2": "Oaxaca Mixe", + "level3": "Lowland-Midland-South Highland Mixe", + "level4": "Lowland-Midland Mixe", + "level5": "Midland Mixe" + }, + "mxr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Kayan-Murik-Modang", + "level4": "Kayan-Murik" + }, + "mxs": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Eastern Alta Mixtec", + "level7": "Southeastern Alta Mixtec", + "level8": "Teozacoalco Mixtec" + }, + "mxt": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Coast Mixtec", + "level7": "East Coast Mixtec" + }, + "mxu": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mofuic", + "level6": "Tokombere", + "level7": "Madaic" + }, + "mxv": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Guerrero Mixtec", + "level7": "Coicoyan-Metlatonoc" + }, + "mxw": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Nambu", + "level3": "Namo-Len" + }, + "mxx": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding", + "level9": "Maninka-Mori" + }, + "mxy": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Eastern Alta Mixtec", + "level7": "Southeastern Alta Mixtec" + }, + "mxz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar", + "level5": "South Babar", + "level6": "Masela-South Babar" + }, + "mya": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Southern Burmish", + "level5": "Mranmaic", + "level6": "Nuclear Mranmaic" + }, + "myb": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Logone-Chari", + "level7": "Sido" + }, + "myd": { + "level0": "Bookkeeping" + }, + "mye": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "B10-B30" + }, + "myf": { + "level0": "Blue Nile Mao" + }, + "myg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Southwest Grassfields" + }, + "myh": { + "level0": "Wakashan", + "level1": "Southern Wakashan", + "level2": "Makah-Nitinat" + }, + "myi": { + "level0": "Bookkeeping" + }, + "myj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Sereic", + "level6": "Feroge-Mangaya" + }, + "myk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "North Senufo" + }, + "myl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio", + "level5": "Greater Kaili", + "level6": "Kulawi" + }, + "mym": { + "level0": "Surmic", + "level1": "South Surmic", + "level2": "Southeast Surmic", + "level3": "Pastoral Surmic" + }, + "myo": { + "level0": "Ta-Ne-Omotic", + "level1": "Kefoid" + }, + "myq": { + "level0": "Bookkeeping" + }, + "mys": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Outer South Ethiopic", + "level6": "TT-Group", + "level7": "Peripheral Western Gurage" + }, + "myt": { + "level0": "Bookkeeping" + }, + "myu": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Mundurukuic" + }, + "myv": { + "level0": "Uralic", + "level1": "Mordvin" + }, + "myw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Kilivila-Misima", + "level8": "Kilivilic", + "level9": "Kilivila-Muyuw" + }, + "myx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Northern Luyia" + }, + "myy": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Western Eastern Tucanoan", + "level3": "Barasano-Eduria-Macuna" + }, + "myz": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Macro-Mandaic" + }, + "mza": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Coast Mixtec", + "level7": "West Coast Mixtec" + }, + "mzb": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Greater Zenatic", + "level3": "Zenatic", + "level4": "Northern Saharan Oasis Berber" + }, + "mzc": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "West Scandinavian Sign", + "level4": "Norwegian Sign" + }, + "mzd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Sawabantu", + "level8": "Dualaic", + "level9": "Duala-Malimba" + }, + "mze": { + "level0": "Mailuan" + }, + "mzf": { + "level0": "Bookkeeping" + }, + "mzg": { + "level0": "Sign Language", + "level1": "Auxiliary Sign Systems" + }, + "mzh": { + "level0": "Mataguayan", + "level1": "Mataguayo II", + "level2": "Wichi" + }, + "mzi": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Mazatecan", + "level5": "Valley Mazatec", + "level6": "Ayautlic", + "level7": "Northern Baja Mazatec" + }, + "mzj": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding", + "level9": "Manenkan", + "level10": "Konya-Manya" + }, + "mzk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Mambila-Mbongno", + "level10": "Mambila" + }, + "mzl": { + "level0": "Mixe-Zoque", + "level1": "Mixe", + "level2": "Oaxaca Mixe", + "level3": "Lowland-Midland-South Highland Mixe", + "level4": "Lowland-Midland Mixe", + "level5": "Lowland Mixe" + }, + "mzm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Mumuye-Yandang", + "level6": "Mumuyic" + }, + "mzn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Caspian", + "level8": "Mazanderani-Shahmirzadi" + }, + "mzo": { + "level0": "Cariban", + "level1": "Kuikuroan", + "level2": "Nuclear Kuikuroan" + }, + "mzq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Western Bungku-Tolaki", + "level8": "Interior Bungku-Tolaki" + }, + "mzr": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Marubo Subgroup" + }, + "mzs": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Luso-Asian Creole" + }, + "mzt": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "North Aslian", + "level4": "Maniq-Menraq-Batek", + "level5": "Menraq-Batek", + "level6": "Batekic" + }, + "mzu": { + "level0": "Ramu", + "level1": "Goam", + "level2": "Tamolan" + }, + "mzv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Oriental", + "level5": "Gbanu-Manza-Ngbaka", + "level6": "Manza-Ngbaka", + "level7": "Manzaic" + }, + "mzw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Western Grusi", + "level9": "Sisaala-Chakali", + "level10": "Chakalic" + }, + "mzx": { + "level0": "Bookkeeping" + }, + "mzy": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "mzz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Bwaidoga linkage", + "level9": "Iamalelic" + }, + "naa": { + "level0": "Namla-Tofanma" + }, + "nab": { + "level0": "Nambiquaran", + "level1": "Nambikwara Complex" + }, + "nac": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Jimi", + "level3": "Kandawo-Narak" + }, + "nad": { + "level0": "Bookkeeping" + }, + "nae": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Three Rivers", + "level4": "Amalumute", + "level5": "Northwest Seram", + "level6": "Ulat Inai" + }, + "naf": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Rawlinson", + "level5": "Sankwep" + }, + "nag": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Kamrupa", + "level10": "Eastern Kamrupa" + }, + "naj": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Naluic" + }, + "nak": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "Willaumez linkage", + "level7": "Nakanai-Meramera" + }, + "nal": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Tungak-Nalik" + }, + "nam": { + "level0": "Southern Daly" + }, + "nan": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Min", + "level3": "Coastal Min" + }, + "nao": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Sherpa-Jirel", + "level9": "Sherpaic" + }, + "nap": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Italo-Dalmatian", + "level9": "Italian Romance" + }, + "naq": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Khoekhoe", + "level3": "North Khoekhoe" + }, + "nar": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "North-Central Jos" + }, + "nas": { + "level0": "South Bougainville", + "level1": "Nasioiic", + "level2": "Nasioi", + "level3": "South-Central Nasioi", + "level4": "Central Nasioi" + }, + "nat": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Shiroro-Kamuku", + "level6": "Kamuku-Hungwarya" + }, + "nau": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Kosraean-Nauruan" + }, + "nav": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Apachean", + "level4": "Southwestern Apachean", + "level5": "Western Southwestern Apachean" + }, + "naw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Oti North Guang", + "level9": "Mountain Oti North Guang" + }, + "nax": { + "level0": "Left May", + "level1": "Western Left May", + "level2": "Nimo-Nakwi" + }, + "nay": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "Victorian Pama-Nyungan", + "level3": "Lower Murray", + "level4": "Yaraldi-Keramin-Yitha" + }, + "naz": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Western Periphery-North Guerrero Nahuatl", + "level6": "North Guerrero Nahuatl" + }, + "nba": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Chokwe-Ngangela-Nyemba (K.20)", + "level11": "Ngangela-Nyemba" + }, + "nbb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Ekoid-Mbe", + "level6": "Ekoid" + }, + "nbc": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "South Patkaian", + "level4": "Southwestern Patkaian", + "level5": "Chang-Phom-Konyak" + }, + "nbd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Ngbele-Ngenda", + "level15": "Ngendan" + }, + "nbe": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "South Patkaian", + "level4": "Southwestern Patkaian", + "level5": "Chang-Phom-Konyak", + "level6": "Konyak-Phom" + }, + "nbf": { + "level0": "Bookkeeping" + }, + "nbg": { + "level0": "Unattested", + "level1": "Dravidian (Unattested)" + }, + "nbh": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Nuclear Boleic", + "level8": "Galambu-Bele", + "level9": "Kirfi-Bele", + "level10": "Ngamo-Bele" + }, + "nbi": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Angami-Pochuri", + "level4": "Angami-Mao", + "level5": "Naga Maoic" + }, + "nbj": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Ngumpin-Yapa", + "level3": "Ngumpin", + "level4": "Eastern Ngumpin", + "level5": "Ngumpit" + }, + "nbk": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Unclassified Hanseman" + }, + "nbm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Western Mundu-Baka", + "level7": "River Western Mundu-Baka", + "level8": "Bwaka" + }, + "nbn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "Banda-Geser", + "level4": "Seran Laut", + "level5": "Koiwai-Irarutu", + "level6": "Irarutic" + }, + "nbo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "East-West Central Delta Cross", + "level7": "Lokoic", + "level8": "Lubila-Lokaa" + }, + "nbp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Ekoid-Mbe", + "level6": "Ekoid", + "level7": "Bakor-Ejagham", + "level8": "Bakor", + "level9": "Northern Bakor", + "level10": "Nnam-Ekajuk" + }, + "nbq": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani" + }, + "nbr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Ninzic" + }, + "nbs": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "BSLic", + "level3": "South African Sign" + }, + "nbt": { + "level0": "Sino-Tibetan", + "level1": "Macro-Tani", + "level2": "Tani", + "level3": "Pre-Western Tani", + "level4": "Western Tani", + "level5": "Subansiri", + "level6": "Bangni-Tagin" + }, + "nbu": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Zemeic" + }, + "nbv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Momo", + "level8": "Widikum-Tadkon" + }, + "nbw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Ngbandi-Mongoba-Kazibati", + "level6": "Ngbandic", + "level7": "Nuclear Ngbandic" + }, + "nbx": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Badjiri-Eastern Karnic", + "level3": "Eastern Karnic" + }, + "nby": { + "level0": "Border", + "level1": "Bewani" + }, + "nca": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Gusap-Mot", + "level4": "Ufim-Rawa-Nahu" + }, + "ncb": { + "level0": "Austroasiatic", + "level1": "Nicobaric", + "level2": "Nuclear Nicobaric", + "level3": "Central Nicobar" + }, + "ncc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "East Manus" + }, + "ncd": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Khambu", + "level6": "Kulungic" + }, + "ncf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Tabar linkage" + }, + "ncg": { + "level0": "Tsimshian", + "level1": "Nishga-Gitxsan" + }, + "nch": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl", + "level5": "Huasteca Nahuatl" + }, + "nci": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Central Nahuatl" + }, + "ncj": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Central Nahuatl" + }, + "nck": { + "level0": "Maningrida", + "level1": "Nakkara-Ndjebbana" + }, + "ncl": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Western Periphery-North Guerrero Nahuatl", + "level6": "Western Periphery Nahuatl" + }, + "ncm": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Nambu" + }, + "ncn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "South-East Admiralty" + }, + "nco": { + "level0": "South Bougainville", + "level1": "Nasioiic" + }, + "ncq": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "West Katuic", + "level3": "Brou-So", + "level4": "Eastern Bru-Katang", + "level5": "Katang" + }, + "ncr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Beboid", + "level6": "Eastern Beboid", + "level7": "Nsari-Nooni-Ncane", + "level8": "Nooni-Ncane", + "level9": "Ncane-Cung" + }, + "ncs": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "nct": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Kolhrengic", + "level5": "Tarao-Chothe" + }, + "ncu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Oti North Guang", + "level9": "River Oti North Guang", + "level10": "Chumbuli" + }, + "ncx": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl" + }, + "nda": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Ndasaic", + "level8": "Samayic", + "level9": "Ndasa-Wumbvu" + }, + "ndb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "South Ring", + "level9": "Babungoic" + }, + "ndc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Shona (S.10)", + "level9": "Core Shona" + }, + "ndd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Ekoid-Mbe", + "level6": "Ekoid", + "level7": "Bakor-Ejagham", + "level8": "Bakor", + "level9": "Nde-Efutop" + }, + "nde": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Nguni-Tsonga-Copi", + "level11": "Nguni (S.40)", + "level12": "Nuclear Nguni", + "level13": "Southern Ndebele-Lowland", + "level14": "Swatic" + }, + "ndg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Rufijic", + "level9": "Matumbic" + }, + "ndh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Nyakyusa-Ndali" + }, + "ndi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Southern Samba-Duru", + "level7": "Sambaic", + "level8": "Samba-Leko-Perema-Mumbake" + }, + "ndj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Kilombero" + }, + "ndk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke", + "level11": "So-Poke", + "level12": "So-Lebonya", + "level13": "Lebonya", + "level14": "Bantu D33", + "level15": "Budu-Ndaka-Mbo", + "level16": "Ndaka-Mbo" + }, + "ndl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Bamweic" + }, + "ndm": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.1", + "level5": "Sumrayic", + "level6": "Ndam-Tumak" + }, + "ndn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Likouala-Sangha", + "level10": "Bwamba-Ngondi-Pande-Mbati-Aka" + }, + "ndo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia", + "level11": "Ndonga (R.20)", + "level12": "Kwambi-Ndonga" + }, + "ndp": { + "level0": "Central Sudanic", + "level1": "Membi-Mangbutu-Efe" + }, + "ndq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Unclassified Kunene" + }, + "ndr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid" + }, + "nds": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Alts\u00e4chsisch", + "level7": "Middle-Modern Low German", + "level8": "Low German", + "level9": "Greater East Low German" + }, + "ndt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mbaic", + "level6": "Ndunga-Mba-Dongo", + "level7": "Ndunga-Mba" + }, + "ndu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Southern Samba-Duru", + "level7": "Diic" + }, + "ndv": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Cangin", + "level3": "Palor-Ndut" + }, + "ndw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Ubangi", + "level11": "Ngiri Riverain Ubangi-Ripuaire", + "level12": "Libinzic", + "level13": "Libinza Fleuve" + }, + "ndx": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani", + "level2": "Ngalik-Nduga" + }, + "ndy": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Nduga-Luto" + }, + "ndz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Sereic", + "level6": "Sere-Indri", + "level7": "Sere-Bviri", + "level8": "Ndogo-Sere" + }, + "neb": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Mano-Dan", + "level4": "Guro-Dan", + "level5": "Dan-Toura", + "level6": "Toura-Goo" + }, + "nec": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "West Alor-Straits-Pantar", + "level4": "Pantar" + }, + "ned": { + "level0": "Bookkeeping" + }, + "nee": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Extreme Northern New Caledonian", + "level9": "Kum-Nel-Yua-Cac", + "level10": "Kum-Nel-Yua" + }, + "nef": { + "level0": "Pidgin", + "level1": "Assamese-based pidgin" + }, + "neg": { + "level0": "Tungusic", + "level1": "Northeastern Tungusic", + "level2": "Northern Tungusic", + "level3": "Negidalic" + }, + "neh": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Phobjib-Chali-Bumthangic" + }, + "nej": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Gusap-Mot", + "level4": "Gira-Neko-Nekgini" + }, + "nek": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Southern New Caledonian", + "level8": "Mid-Southern New Caledonian" + }, + "nem": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Nmi-Pij-Fwa-Pam-Pap", + "level10": "Nmi-Fij-Fwa", + "level11": "Hyenghene" + }, + "nen": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Loyalty Islands" + }, + "neo": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "East Hmongic", + "level5": "South Qiandongic Miao" + }, + "neq": { + "level0": "Mixe-Zoque", + "level1": "Mixe", + "level2": "Oaxaca Mixe", + "level3": "Lowland-Midland-South Highland Mixe", + "level4": "Lowland-Midland Mixe", + "level5": "Midland Mixe" + }, + "ner": { + "level0": "Konda-Yahadian" + }, + "nes": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Lahauli-Spiti" + }, + "net": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Engan", + "level3": "Outer Enga" + }, + "neu": { + "level0": "Artificial Language" + }, + "nev": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "Nuclear West Bahnaric" + }, + "new": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Newaric", + "level4": "Newar" + }, + "nex": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Nambu" + }, + "ney": { + "level0": "Kru", + "level1": "Eastern Kru", + "level2": "Neyo-Dida" + }, + "nez": { + "level0": "Sahaptian" + }, + "nfa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Sumba-Hawu", + "level5": "Hawu-Dhao" + }, + "nfd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau" + }, + "nfg": { + "level0": "Bookkeeping" + }, + "nfk": { + "level0": "Bookkeeping" + }, + "nfl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Temotu", + "level5": "Reefs-Santa Cruz" + }, + "nfr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "South Senufo" + }, + "nfu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Nkambe", + "level9": "Mfumteic", + "level10": "Central-Southern Mfumte" + }, + "nga": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Oriental", + "level5": "Gbanu-Manza-Ngbaka", + "level6": "Manza-Ngbaka" + }, + "ngb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Ngbandi-Mongoba-Kazibati", + "level6": "Ngbandic", + "level7": "Nuclear Ngbandic" + }, + "ngc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ngombe-Genja" + }, + "ngd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Likouala-Sangha", + "level10": "Mokiba-Ngando" + }, + "nge": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Ngembaic", + "level10": "Mankonic" + }, + "ngg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Oriental", + "level5": "Gbanu-Manza-Ngbaka", + "level6": "Manza-Ngbaka", + "level7": "Manzaic", + "level8": "Ngbaka-Manza-Ali" + }, + "ngh": { + "level0": "Tuu", + "level1": "!Ui", + "level2": "Ghaap-Kalahari" + }, + "ngi": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.1", + "level5": "Ngizim-Southwestern Bade" + }, + "ngj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Momo", + "level8": "Ngie-Oshie" + }, + "ngk": { + "level0": "Gunwinyguan" + }, + "ngl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Makua-Lomwe", + "level9": "Lomweic" + }, + "ngm": { + "level0": "Speech Register", + "level1": "Indo-European Speech Register" + }, + "ngn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Momo" + }, + "ngp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "West Ruvu", + "level11": "Seuta", + "level12": "Zigua-Nguu" + }, + "ngq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Nyanza Mara", + "level11": "South Mara", + "level12": "Western Serengeti" + }, + "ngr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Temotu", + "level5": "Reefs-Santa Cruz" + }, + "ngs": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mandaraic", + "level6": "Dghwedeic" + }, + "ngt": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "Ta'oihic" + }, + "ngu": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Central Nahuatl" + }, + "ngv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan" + }, + "ngw": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic" + }, + "ngx": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Bura-Marghi", + "level6": "Buraic" + }, + "ngy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Bafia (A.50)" + }, + "ngz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Ngungwel-Eboo" + }, + "nha": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Kartu-Nhanda" + }, + "nhb": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Nwa-Ben", + "level4": "Ben-Gban", + "level5": "Bengic" + }, + "nhc": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl", + "level5": "Isthmus-Pipil Nahuatl", + "level6": "Isthmus Nahuatl" + }, + "nhd": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup I", + "level7": "Tupi-Guarani Subgroup I.A", + "level8": "Paraguay-Brazil Guarani" + }, + "nhe": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl", + "level5": "Huasteca Nahuatl" + }, + "nhf": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda" + }, + "nhg": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Central Nahuatl" + }, + "nhh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Halbic" + }, + "nhi": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Central Nahuatl", + "level6": "Tlaxcala-Southeastern Puebla Nahuatl" + }, + "nhj": { + "level0": "Bookkeeping" + }, + "nhk": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl", + "level5": "Isthmus-Pipil Nahuatl", + "level6": "Isthmus Nahuatl" + }, + "nhm": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Central Nahuatl" + }, + "nhn": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Central Nahuatl", + "level6": "Tlaxcala-Southeastern Puebla Nahuatl", + "level7": "Tlaxcala-Puebla-Pastoral Nahuatl" + }, + "nho": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "Central Northern Outlier Polynesian", + "level12": "Takuuic" + }, + "nhp": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl", + "level5": "Isthmus-Pipil Nahuatl", + "level6": "Isthmus Nahuatl" + }, + "nhq": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl", + "level5": "Sierra de Puebla Nahuatl" + }, + "nhr": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Non-Khoekhoe", + "level3": "West-Kxoe", + "level4": "Naro-Ana" + }, + "nhs": { + "level0": "Bookkeeping" + }, + "nht": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl" + }, + "nhu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Beboid", + "level6": "Eastern Beboid", + "level7": "Nsari-Nooni-Ncane", + "level8": "Nooni-Ncane" + }, + "nhv": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Western Periphery-North Guerrero Nahuatl" + }, + "nhw": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl", + "level5": "Huasteca Nahuatl" + }, + "nhx": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl", + "level5": "Isthmus-Pipil Nahuatl", + "level6": "Isthmus Nahuatl" + }, + "nhy": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Central Nahuatl", + "level6": "Tlaxcala-Southeastern Puebla Nahuatl", + "level7": "Southeast Puebla-Northern Oaxaca Nahuatl" + }, + "nhz": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Central Nahuatl", + "level6": "Tlaxcala-Southeastern Puebla Nahuatl", + "level7": "Tlaxcala-Puebla-Pastoral Nahuatl" + }, + "nia": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran", + "level3": "Central Barrier Islands" + }, + "nib": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Erap", + "level4": "Boana" + }, + "nid": { + "level0": "Gunwinyguan", + "level1": "Eastern Gunwinyguan" + }, + "nie": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Adamawa Bua", + "level5": "Riverine Bua", + "level6": "Bua-Lua" + }, + "nif": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Erap", + "level4": "Boana", + "level5": "Nek-Nuk" + }, + "nig": { + "level0": "Gunwinyguan", + "level1": "Jala" + }, + "nih": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Nyika-Lambya", + "level10": "Nyika", + "level11": "Central and Southern Nyika", + "level12": "Mbozi-Malawi Nyika" + }, + "nii": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Wahgic" + }, + "nij": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "South West Greater Barito" + }, + "nik": { + "level0": "Austroasiatic", + "level1": "Nicobaric", + "level2": "Nuclear Nicobaric", + "level3": "Central Nicobar" + }, + "nil": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Teun-Nila-Serua", + "level5": "Nila-Serua" + }, + "nim": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Nyaturu-Nilamba" + }, + "nin": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Ninzic", + "level5": "Rukubic", + "level6": "Mada-Ninzam" + }, + "nio": { + "level0": "Uralic", + "level1": "Samoyedic" + }, + "niq": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Kalenjin", + "level3": "Central Kalenjin", + "level4": "Plateau Central Kalenjin", + "level5": "Western Plateau Central Kalenjin" + }, + "nir": { + "level0": "Nimboranic" + }, + "nis": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Erap", + "level4": "Sauk-Nimi" + }, + "nit": { + "level0": "Dravidian", + "level1": "Central Dravidian", + "level2": "Kolami-Naiki" + }, + "niu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Tongic" + }, + "niv": { + "level0": "Nivkh" + }, + "niw": { + "level0": "Left May", + "level1": "Western Left May", + "level2": "Nimo-Nakwi" + }, + "nix": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "Rutara", + "level11": "North Rutara" + }, + "niy": { + "level0": "Central Sudanic", + "level1": "Lenduic" + }, + "niz": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Wapeic", + "level4": "Ningil-Yil" + }, + "nja": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic", + "level5": "Gudeic", + "level6": "Nzanyic" + }, + "njb": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "North Patkaian", + "level4": "Noctean" + }, + "njd": { + "level0": "Bookkeeping" + }, + "njh": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Central Naga" + }, + "nji": { + "level0": "Mirndi", + "level1": "Ngurlun" + }, + "njj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Momo", + "level8": "Mundani-Njen" + }, + "njl": { + "level0": "Dajuic", + "level1": "Western Dajuic" + }, + "njm": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Angami-Pochuri", + "level4": "Angami-Mao", + "level5": "Angami-Chokri" + }, + "njn": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Zemeic", + "level3": "Nuclear Zemeic" + }, + "njo": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Central Naga" + }, + "njr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Mambila-Mbongno", + "level10": "Mambila", + "level11": "Njerup" + }, + "njs": { + "level0": "Geelvink Bay" + }, + "njt": { + "level0": "Pidgin", + "level1": "Trio-based pidgin" + }, + "nju": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Mirning" + }, + "njx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kamba-Kunyi" + }, + "njy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Mpoic", + "level10": "Njemic" + }, + "njz": { + "level0": "Sino-Tibetan", + "level1": "Macro-Tani", + "level2": "Tani", + "level3": "Pre-Western Tani", + "level4": "Western Tani", + "level5": "Subansiri" + }, + "nka": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban" + }, + "nkb": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Tangkhul-Maring", + "level3": "Maringic" + }, + "nkc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Lundu-Balong (A.10)", + "level8": "Greater Manenguba" + }, + "nkd": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Kolhrengic" + }, + "nke": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "New Georgia", + "level10": "West New Georgia" + }, + "nkf": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Zemeic" + }, + "nkg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Gusap-Mot", + "level4": "Gira-Neko-Nekgini" + }, + "nkh": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Angami-Pochuri", + "level4": "Angami-Mao" + }, + "nki": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Zemeic" + }, + "nkj": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Tangko-Nakai" + }, + "nkk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "Cape Cumberland" + }, + "nkm": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Nambu" + }, + "nkn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Chokwe-Ngangela-Nyemba (K.20)", + "level11": "Ngangela-Nyemba" + }, + "nko": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Nkonya-Nkami" + }, + "nkp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "East Uvean-Niuafo'ou" + }, + "nkq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Nkonya-Nkami" + }, + "nkr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Carolinean Outlier Polynesian" + }, + "nks": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Asmat-Kamoro", + "level3": "Asmat" + }, + "nkt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Nyika-Lambya", + "level10": "Nyika" + }, + "nku": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Kulango-Lorom", + "level5": "Kulango" + }, + "nkv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Nyika-Lambya", + "level10": "Nyika", + "level11": "Central and Southern Nyika" + }, + "nkw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Nkutsuic", + "level11": "Nkutsu-Lokenye", + "level12": "Songomenic" + }, + "nkx": { + "level0": "Ijoid", + "level1": "Ijo", + "level2": "Eastern Ijo", + "level3": "Nikio" + }, + "nky": { + "level0": "Bookkeeping" + }, + "nkz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Ibuoroic", + "level8": "Ibuoro-ItuMbuso-Nkari" + }, + "nla": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Bamileke", + "level10": "West Bamileke" + }, + "nlc": { + "level0": "Nuclear Trans New Guinea", + "level1": "Mek", + "level2": "Western Mek" + }, + "nld": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch", + "level6": "Middle-Modern Dutch", + "level7": "Modern Dutch", + "level8": "Global Dutch" + }, + "nle": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Central-Eastern Luyia", + "level14": "Kabarasi-Tachoni-Nyala East" + }, + "nlg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Guadalcanal-Nggelic", + "level6": "Nuclear Guadalcanal-Nggelic", + "level7": "Nggelic" + }, + "nli": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Gawarbatic", + "level5": "Shumashtic" + }, + "nlj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke", + "level11": "So-Poke", + "level12": "So-Lebonya", + "level13": "Lebonya", + "level14": "Bantu D33", + "level15": "Vanuma-Nyali" + }, + "nlk": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani", + "level2": "Ngalik-Nduga", + "level3": "Yalic" + }, + "nlm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Kohistani", + "level8": "Indus Kohistanic", + "level9": "Outer Indus Kohistani", + "level10": "Bateri-Mankiyali" + }, + "nlo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu" + }, + "nlq": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "South Patkaian", + "level4": "Southwestern Patkaian" + }, + "nlr": { + "level0": "Bookkeeping" + }, + "nlu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Oti North Guang", + "level9": "River Oti North Guang", + "level10": "Nchumbulu-Dwang" + }, + "nlv": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl" + }, + "nlw": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Norman Pama" + }, + "nlx": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil", + "level8": "Pauri-Nahali" + }, + "nly": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda", + "level4": "Northern Ngayarda" + }, + "nlz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Temotu", + "level5": "Reefs-Santa Cruz", + "level6": "Natugu-Nalogo" + }, + "nma": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Zemeic" + }, + "nmb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Peripheral Western Malakula", + "level9": "Northwestern Malakula" + }, + "nmc": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Logone-Chari", + "level7": "Sido" + }, + "nmd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie", + "level17": "Northwest Kasai-Ngounie", + "level18": "Mbere (B.60)", + "level19": "Tsitsekeic", + "level20": "Lekaningic" + }, + "nme": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Zemeic", + "level3": "Nuclear Zemeic", + "level4": "Mzieme-Zeme" + }, + "nmf": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Tangkhul-Maring", + "level3": "Tangkhulic", + "level4": "Nuclear Tangkhulic" + }, + "nmg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Western A80", + "level10": "Mvumboic", + "level11": "Kwasio-Gyele" + }, + "nmh": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Moyon-Monsang Naga" + }, + "nmi": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Boleic", + "level7": "Unclassified Boleic" + }, + "nmj": { + "level0": "Bookkeeping" + }, + "nmk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu" + }, + "nml": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields" + }, + "nmm": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Tamangic", + "level5": "Gurungic", + "level6": "Manangba-Nar-Phu" + }, + "nmn": { + "level0": "Tuu", + "level1": "Hua", + "level2": "Taa" + }, + "nmo": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Moyon-Monsang Naga" + }, + "nmp": { + "level0": "Nyulnyulan", + "level1": "Western Nyulnyulan" + }, + "nmq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Shona (S.10)", + "level9": "Kalanga-Nambya" + }, + "nmr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Unclassified Samba-Duru" + }, + "nms": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Peripheral Western Malakula", + "level9": "Southwestern Malakula" + }, + "nmt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Nuclear Trukic", + "level10": "Central Trukic", + "level11": "Satawalese-Carolinian", + "level12": "Macro-Carolinian" + }, + "nmu": { + "level0": "Maiduan" + }, + "nmv": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Central Karnic", + "level3": "Western Central Karnic", + "level4": "Pirlatapa-Dieric", + "level5": "Dieric" + }, + "nmw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Nimoa-Sudest" + }, + "nmx": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Nambu", + "level3": "Nama-Dre" + }, + "nmy": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Naic" + }, + "nmz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Gurma-Yom-Naudem", + "level11": "Yom-Nawdm" + }, + "nna": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Marrngu" + }, + "nnb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Rwenzori" + }, + "nnc": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.2", + "level5": "East Chadic A.2 1" + }, + "nnd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Ambae" + }, + "nne": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia", + "level11": "Ndonga (R.20)", + "level12": "Unclassified Ndonga (R.20)" + }, + "nnf": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Gusap-Mot", + "level4": "Unclassified Gusap-Mot" + }, + "nng": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Tangkhul-Maring", + "level3": "Maringic" + }, + "nnh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Bamileke", + "level10": "West Bamileke", + "level11": "Bamboutos" + }, + "nni": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Patakai-Manusela", + "level4": "Patakai" + }, + "nnj": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Teso-Turkana", + "level4": "Turkanic" + }, + "nnk": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Yupna" + }, + "nnl": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Angami-Pochuri", + "level4": "Pochuri-Northern Rengma" + }, + "nnm": { + "level0": "Sepik", + "level1": "Yellow River" + }, + "nnn": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Masa", + "level3": "South Masa", + "level4": "Peveic", + "level5": "Hede-Ngide" + }, + "nnp": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "South Patkaian", + "level4": "Southwestern Patkaian", + "level5": "Wanchoic" + }, + "nnq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Rufijic", + "level9": "Matengic", + "level10": "Ndendeule-Ngindo" + }, + "nnr": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Thura-Yura", + "level3": "Core Thura Yura", + "level4": "Southern Thura-Yura" + }, + "nnt": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Nanticoke-Conoy" + }, + "nnu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Guang", + "level7": "North Guang", + "level8": "Oti North Guang", + "level9": "River Oti North Guang", + "level10": "Nchumbulu-Dwang" + }, + "nnv": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Thura-Yura", + "level3": "Core Thura Yura", + "level4": "Unclassified Core Thura-Yura" + }, + "nnw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "Northern Grusi", + "level8": "Nuna-Kasem", + "level9": "Nuni" + }, + "nnx": { + "level0": "Bookkeeping" + }, + "nny": { + "level0": "Tangkic", + "level1": "Southern Tangkic", + "level2": "Kayardild-Yangkaal" + }, + "nnz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Bamileke", + "level10": "East Bamileke" + }, + "noa": { + "level0": "Chocoan" + }, + "noc": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Erap", + "level4": "Boana", + "level5": "Nek-Nuk" + }, + "nod": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Southern Shanic", + "level11": "Yuanic" + }, + "noe": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "nof": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Simbu", + "level3": "Chuave-Nomane" + }, + "nog": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Southeast Kipchak", + "level5": "South Kipchak" + }, + "noh": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Cromwell", + "level5": "Dallman" + }, + "noi": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil", + "level8": "Vasave-Noiri" + }, + "noj": { + "level0": "Huitotoan", + "level1": "Nonuya-Ocaina" + }, + "nok": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Central Salish", + "level3": "South Georgia Central Salish" + }, + "nom": { + "level0": "Bookkeeping" + }, + "non": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "North Germanic", + "level5": "West Scandinavian" + }, + "noo": { + "level0": "Bookkeeping" + }, + "nop": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Erap", + "level4": "Boana" + }, + "noq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Nsong-Mpiin-Ngong" + }, + "nor": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "North Germanic", + "level5": "West Scandinavian" + }, + "nos": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nisu-Nyisu", + "level8": "Nisu", + "level9": "Nuclear Nisu", + "level10": "Northern Nisu" + }, + "not": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha", + "level3": "Pre-Andine Maipuran" + }, + "nou": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean", + "level3": "Nuclear Binanderean", + "level4": "South Binanderean", + "level5": "Coastal Binanderean" + }, + "nov": { + "level0": "Artificial Language" + }, + "now": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "Rutara", + "level11": "South Rutara" + }, + "noy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Adamawa Bua", + "level5": "Riverine Bua", + "level6": "Unclassified Riverine Bua" + }, + "noz": { + "level0": "Dizoid" + }, + "npa": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Tamangic", + "level5": "Gurungic", + "level6": "Manangba-Nar-Phu" + }, + "npb": { + "level0": "Bookkeeping" + }, + "npg": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "South Patkaian", + "level4": "Southeastern Patkaian", + "level5": "Lainongic", + "level6": "Khiamniungic" + }, + "nph": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "South Patkaian", + "level4": "Southwestern Patkaian", + "level5": "Chang-Phom-Konyak", + "level6": "Konyak-Phom" + }, + "npi": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Indo-Aryan Northern zone", + "level8": "Eastern Pahari" + }, + "npl": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Central Nahuatl", + "level6": "Tlaxcala-Southeastern Puebla Nahuatl", + "level7": "Southeast Puebla-Northern Oaxaca Nahuatl" + }, + "npn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "West Manus", + "level8": "West Manus II" + }, + "npo": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Angami-Pochuri", + "level4": "Pochuri-Northern Rengma" + }, + "nps": { + "level0": "Nuclear Trans New Guinea", + "level1": "Mek", + "level2": "Western Mek" + }, + "npu": { + "level0": "Bookkeeping" + }, + "npy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "Rampi-Seko-Badaic", + "level4": "Badaic-Limola", + "level5": "Badaic" + }, + "nqg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede", + "level7": "Eastern Ede", + "level8": "Southeastern Ede" + }, + "nqk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede", + "level7": "Western Ede" + }, + "nql": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia" + }, + "nqm": { + "level0": "Kolopom" + }, + "nqn": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Nambu" + }, + "nqo": { + "level0": "Artificial Language" + }, + "nqq": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "South Patkaian", + "level4": "Southwestern Patkaian", + "level5": "Wanchoic" + }, + "nqt": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3", + "level6": "Kofyar-Mushere-Chip", + "level7": "Kofyaric" + }, + "nqy": { + "level0": "Bookkeeping", + "level1": "Pending Report Release" + }, + "nra": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Ngomic", + "level8": "Nuclear Ngomic", + "level9": "Akeleic" + }, + "nrc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Continental Transalpine Celtic", + "level6": "Unclassified Continental Transalpine Celtic" + }, + "nre": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Angami-Pochuri", + "level4": "Rengma-Simi" + }, + "nrg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "South-Central Santo" + }, + "nri": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Angami-Pochuri", + "level4": "Angami-Mao", + "level5": "Angami-Chokri" + }, + "nrk": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda", + "level4": "Northern Ngayarda" + }, + "nrl": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda", + "level4": "Central Ngayarda", + "level5": "Ngarluma-Kariyarra" + }, + "nrm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Berawan-Lower Baram", + "level5": "Lower Baram", + "level6": "Central Lower Baram B" + }, + "nrp": { + "level0": "Unclassifiable" + }, + "nrr": { + "level0": "Bookkeeping" + }, + "nrt": { + "level0": "Kalapuyan" + }, + "nru": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Naic", + "level4": "Naish" + }, + "nrx": { + "level0": "Unattested", + "level1": "Umbugarla (Unattested)" + }, + "nrz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "West Central Papuan linkage", + "level9": "Nuclear West Central Papuan linkage" + }, + "nsa": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Central Naga" + }, + "nsb": { + "level0": "Tuu", + "level1": "Hua" + }, + "nsc": { + "level0": "Unattested", + "level1": "Atlantic-Congo (Unattested)" + }, + "nsd": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nisu-Nyisu", + "level8": "Nisu", + "level9": "Nuclear Nisu" + }, + "nse": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Sabi" + }, + "nsf": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nisu-Nyisu", + "level8": "Nisu" + }, + "nsg": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Lotuxo-Maa", + "level4": "Ongamo-Maa" + }, + "nsh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Momo", + "level8": "Ngie-Oshie" + }, + "nsi": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "American Sign" + }, + "nsk": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Cree-Montagnais-Naskapi" + }, + "nsl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "West Scandinavian Sign", + "level4": "Norwegian Sign" + }, + "nsm": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Angami-Pochuri", + "level4": "Rengma-Simi" + }, + "nsn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "North Bougainville Oceanic", + "level10": "Nuclear North Bougainville Oceanic" + }, + "nso": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Sotho-Tswana (S.30)", + "level11": "Northern Sotho", + "level12": "Sepedic" + }, + "nsp": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Indo-Pakistani-Nepalese Sign" + }, + "nsq": { + "level0": "Miwok-Costanoan", + "level1": "Miwokan", + "level2": "Eastern Miwokan", + "level3": "Sierra Miwokan" + }, + "nsr": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "BSLic" + }, + "nss": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "East Manus", + "level8": "Koro-Lele-Nali-Titan" + }, + "nst": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "North Patkaian", + "level4": "Tangsa" + }, + "nsu": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl", + "level5": "Sierra de Puebla Nahuatl" + }, + "nsv": { + "level0": "Bookkeeping" + }, + "nsw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "Central Santo" + }, + "nsx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Northern Njila", + "level9": "Mbundu (H.20)" + }, + "nsy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran" + }, + "nsz": { + "level0": "Maiduan" + }, + "ntd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Northern Murutic", + "level8": "Sumambu-Tagal", + "level9": "Tidung-Bulusu", + "level10": "Tidung" + }, + "nte": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Koti-Nathembo" + }, + "nti": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Samu" + }, + "ntj": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Wati", + "level3": "Pintupic", + "level4": "Nuclear Pintupic", + "level5": "Wangkatja-Tjarra", + "level6": "Tjarra" + }, + "ntk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Nyanza Mara", + "level11": "South Mara", + "level12": "Western Serengeti", + "level13": "Southeast Mara" + }, + "ntm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Oti-Volta Oriental", + "level10": "Waama-Tayari-Ditammari", + "level11": "Tayari-Ditammari" + }, + "nto": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Mongoic", + "level11": "Bolia-Ntomba" + }, + "ntp": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tepiman", + "level3": "Tepehuan" + }, + "ntr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Eastern Grusi", + "level9": "Tem-Chala", + "level10": "Bago-Delo-Cala", + "level11": "Delo-Cala" + }, + "nts": { + "level0": "Bookkeeping" + }, + "ntu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Temotu", + "level5": "Reefs-Santa Cruz", + "level6": "Natugu-Nalogo" + }, + "ntw": { + "level0": "Iroquoian", + "level1": "Northern Iroquoian", + "level2": "Tuscarora-Nottoway" + }, + "ntx": { + "level0": "Bookkeeping", + "level1": "Pending Report Release" + }, + "nty": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Mondzish", + "level4": "Nuclear Mondzish", + "level5": "Munji-Mantsi" + }, + "ntz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Central Iran Kermanic", + "level8": "Nuclear Central Iran Kermanic", + "level9": "Kashanic" + }, + "nua": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Extreme Northern New Caledonian", + "level9": "Kum-Nel-Yua-Cac", + "level10": "Kum-Nel-Yua" + }, + "nuc": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Poyanawa Subgroup" + }, + "nud": { + "level0": "Ndu" + }, + "nue": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Central Bandaic", + "level8": "Central Core Bandaic", + "level9": "Mid-Southern Central Core Bandaic" + }, + "nuf": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Nusoish" + }, + "nug": { + "level0": "Mirndi", + "level1": "Yirram" + }, + "nuh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Mambila-Mbongno", + "level10": "Mbongno-Mvano", + "level11": "Mvano-Ndunda" + }, + "nui": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Sawabantu", + "level8": "Bengaic", + "level9": "Yasa-Kombe" + }, + "nuj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Western Luyia", + "level14": "Saamiaic" + }, + "nuk": { + "level0": "Wakashan", + "level1": "Southern Wakashan" + }, + "nul": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua", + "level6": "Seram Straits", + "level7": "Uliase", + "level8": "Hatuhaha", + "level9": "Saparuan", + "level10": "Elpaputi" + }, + "num": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "East Uvean-Niuafo'ou" + }, + "nun": { + "level0": "Sino-Tibetan", + "level1": "Nungish", + "level2": "Gunong" + }, + "nuo": { + "level0": "Austroasiatic", + "level1": "Vietic", + "level2": "Viet-Muong", + "level3": "Muongic" + }, + "nup": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Ebira-Nupoid", + "level4": "Macro-Nupoid", + "level5": "Nupoid" + }, + "nuq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "Central Northern Outlier Polynesian", + "level12": "Takuuic" + }, + "nur": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "Central Northern Outlier Polynesian", + "level12": "Takuuic" + }, + "nus": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Dinka-Nuer", + "level3": "Nuer-Reel" + }, + "nut": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Debao-Jingxi-Nung" + }, + "nuu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic" + }, + "nuv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "Northern Grusi", + "level8": "Nuna-Kasem", + "level9": "Nuni" + }, + "nuw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Yapesic" + }, + "nux": { + "level0": "Sepik", + "level1": "Sepik Tama", + "level2": "Mehek-Pahi" + }, + "nuy": { + "level0": "Gunwinyguan", + "level1": "Eastern Gunwinyguan", + "level2": "Wubuy-Anindilyakwa" + }, + "nuz": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Western Nahuatl", + "level5": "Western Periphery-North Guerrero Nahuatl", + "level6": "North Guerrero Nahuatl" + }, + "nvh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Peripheral Western Malakula", + "level9": "Southwestern Malakula" + }, + "nvm": { + "level0": "Koiarian", + "level1": "Baraic", + "level2": "Barai-Namiae" + }, + "nvo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Sanaga-West Mbam (A.40)", + "level10": "West Mbam (A.40)", + "level11": "Mandi-Nyokon" + }, + "nwa": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Arapahoic" + }, + "nwb": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Wee" + }, + "nwe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Bamileke", + "level10": "West Bamileke", + "level11": "Bamboutos" + }, + "nwi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "Southern Vanuatu", + "level6": "Tanna", + "level7": "Southern Tanna" + }, + "nwm": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "Baka-Beli", + "level3": "Morokodo-Beli", + "level4": "Lori" + }, + "nwo": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Thura-Yura", + "level3": "Core Thura Yura", + "level4": "Unclassified Core Thura-Yura" + }, + "nwr": { + "level0": "Yareban", + "level1": "Yareba-Bariji-Nawaru" + }, + "nww": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Rufijic" + }, + "nxa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Eastern Timor", + "level4": "Kawaimina" + }, + "nxd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Mongoic", + "level11": "Lomongo" + }, + "nxe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Central Flores-Paluqe", + "level6": "Central Flores", + "level7": "Eastern Central Flores", + "level8": "Nage-Keo" + }, + "nxg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Central Flores-Paluqe", + "level6": "Central Flores", + "level7": "Ngada" + }, + "nxi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Rufijic", + "level9": "Unclassified Rufijic" + }, + "nxj": { + "level0": "Bookkeeping" + }, + "nxk": { + "level0": "Bookkeeping", + "level1": "Pending Report Release" + }, + "nxl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Patakai-Manusela", + "level4": "Patakai" + }, + "nxm": { + "level0": "Unclassifiable" + }, + "nxn": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Mayabic", + "level3": "Nuclear Mayabic" + }, + "nxo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Ngomic", + "level8": "Nuclear Ngomic", + "level9": "Sake-Ndambomo" + }, + "nxq": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Naic", + "level4": "Naish" + }, + "nxr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Lowland Ok" + }, + "nxu": { + "level0": "Bookkeeping" + }, + "nxx": { + "level0": "Sentanic", + "level1": "Nuclear Sentanic", + "level2": "Sentani-Nafri" + }, + "nya": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Tumbuka-Sena-Nyanja", + "level8": "Sena-Nyanja", + "level9": "Nyanjaic" + }, + "nyb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Ka-Togo", + "level4": "Avatime-Nyangbo", + "level5": "Nyangbo-Tafi" + }, + "nyc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Ngbele-Ngenda", + "level15": "Extreme North Vestigial Suffixes Bantu" + }, + "nyd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia", + "level10": "Luyia", + "level11": "Saamia-Wanga-Bukusu", + "level12": "Saamia-Wanga", + "level13": "Central-Eastern Luyia" + }, + "nye": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Greater Luyana", + "level8": "Western Greater Luyana", + "level9": "Simaaic" + }, + "nyf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Mijikenda-Pokomo-Comorian", + "level11": "Mijikenda", + "level12": "Northern Mijikenda" + }, + "nyg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "Forest Kivu", + "level12": "Fuliiric" + }, + "nyh": { + "level0": "Nyulnyulan", + "level1": "Eastern Nyulnyulan", + "level2": "Nyikinic" + }, + "nyi": { + "level0": "Nyimang" + }, + "nyj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega" + }, + "nyk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia", + "level11": "Nyaneka-Nkhumbi" + }, + "nyl": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "West Katuic", + "level3": "Kuy-Souei" + }, + "nym": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Sukuma-Nyamwezi (F.20)", + "level9": "Nyamwezic" + }, + "nyn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "Rutara", + "level11": "North Rutara", + "level12": "Nkore-Kiga-Nyoro-Tooro", + "level13": "Nkore-Kiga" + }, + "nyo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "Rutara", + "level11": "North Rutara", + "level12": "Nkore-Kiga-Nyoro-Tooro", + "level13": "Nyoro-Tooro" + }, + "nyp": { + "level0": "Kuliak", + "level1": "Ngangea-So" + }, + "nyq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Central Iran Kermanic", + "level8": "Nuclear Central Iran Kermanic", + "level9": "Yazdi-Kermani-Nayini" + }, + "nyr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Nyika-Lambya", + "level10": "Nyika", + "level11": "Central and Southern Nyika", + "level12": "Mbozi-Malawi Nyika" + }, + "nys": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan" + }, + "nyt": { + "level0": "Pama-Nyungan", + "level1": "Nyawaygic" + }, + "nyu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Tumbuka-Sena-Nyanja", + "level8": "Sena-Nyanja", + "level9": "Senaic" + }, + "nyv": { + "level0": "Nyulnyulan", + "level1": "Western Nyulnyulan", + "level2": "Nyulnyulic" + }, + "nyx": { + "level0": "Pama-Nyungan", + "level1": "Macleay-New England" + }, + "nyy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Nyakyusa-Ndali" + }, + "nza": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Jukunoid", + "level4": "Central Jukunoid", + "level5": "Jukun-Mbembe-Wurbo" + }, + "nzb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie", + "level17": "Northwest Kasai-Ngounie", + "level18": "Nzebi-Laali-Yaa", + "level19": "Njebi (B.50)", + "level20": "Ndjavi A" + }, + "nzd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic" + }, + "nzi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Central Tano", + "level7": "Bia", + "level8": "Southern Bia", + "level9": "Jwira-Nzima" + }, + "nzk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Zandic", + "level6": "Zande-Nzakara" + }, + "nzm": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Zemeic", + "level3": "Nuclear Zemeic", + "level4": "Mzieme-Zeme" + }, + "nzr": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Northwest South Bauchi", + "level7": "Polci-Luri", + "level8": "Polcic", + "level9": "Zulic" + }, + "nzs": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "BSLic", + "level3": "BANZL" + }, + "nzy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Central Mbum", + "level6": "Karangic" + }, + "nzz": { + "level0": "Dogon", + "level1": "Nangan Dogon" + }, + "oaa": { + "level0": "Tungusic", + "level1": "Central-Western Tungusic", + "level2": "Ulchaic" + }, + "oac": { + "level0": "Tungusic", + "level1": "Northeastern Tungusic", + "level2": "Central-Eastern Tungusic", + "level3": "Oroch-Udihe" + }, + "oar": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic" + }, + "obi": { + "level0": "Chumashan" + }, + "obl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Unclassified Cameroun-Ubangian" + }, + "obo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Manobo", + "level4": "Central and Southern Manobo", + "level5": "East-West-Central Manobo", + "level6": "West Manobo" + }, + "obr": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Southern Burmish", + "level5": "Mranmaic", + "level6": "Nuclear Mranmaic" + }, + "obu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Central Delta", + "level5": "Abua-Odual" + }, + "oca": { + "level0": "Huitotoan", + "level1": "Nonuya-Ocaina" + }, + "occ": { + "level0": "Bookkeeping" + }, + "och": { + "level0": "Sino-Tibetan", + "level1": "Sinitic" + }, + "oci": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "Occitanic" + }, + "oco": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Insular Celtic", + "level6": "Brythonic", + "level7": "Southwestern Brythonic" + }, + "ocu": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Matlatzincan" + }, + "oda": { + "level0": "Bookkeeping" + }, + "odk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Western Rajasthani", + "level11": "Indus Rajasthani" + }, + "odt": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch" + }, + "odu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Central Delta", + "level5": "Abua-Odual" + }, + "ofo": { + "level0": "Siouan", + "level1": "Ohio Valley Siouan", + "level2": "Southeastern Siouan" + }, + "ofs": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Frisian" + }, + "ofu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Ekoid-Mbe", + "level6": "Ekoid", + "level7": "Bakor-Ejagham", + "level8": "Bakor", + "level9": "Nde-Efutop" + }, + "ogb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Central Delta", + "level5": "Kugboic" + }, + "ogc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Igboid", + "level4": "Nuclear Igboid" + }, + "oge": { + "level0": "Kartvelian", + "level1": "Georgian-Zan", + "level2": "Georgic" + }, + "ogg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Central Delta", + "level5": "Unclassified Central Delta" + }, + "ogo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Ogonoid", + "level5": "East Ogonoid", + "level6": "Tai-Kana" + }, + "ogu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Central Delta" + }, + "ohu": { + "level0": "Uralic", + "level1": "Hungaric" + }, + "oia": { + "level0": "Timor-Alor-Pantar", + "level1": "East Timor", + "level2": "Fataluku-Oirata" + }, + "oie": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Lotuxo-Maa", + "level4": "Lotuxo", + "level5": "Lopit-Dongotono", + "level6": "Dongotonic" + }, + "oin": { + "level0": "Nuclear Torricelli", + "level1": "West Wapei", + "level2": "One", + "level3": "Central-Northern One" + }, + "ojb": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian", + "level4": "Ojibwa-Potawatomi", + "level5": "Ojibwa", + "level6": "Nuclear Ojibwe", + "level7": "Northwestern-Saulteaux Ojibwa" + }, + "ojc": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian", + "level4": "Ojibwa-Potawatomi", + "level5": "Ojibwa", + "level6": "Nuclear Ojibwe", + "level7": "Central-Eastern-Southwestern Ojibwa" + }, + "ojg": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian", + "level4": "Ojibwa-Potawatomi", + "level5": "Ojibwa", + "level6": "Nuclear Ojibwe", + "level7": "Central-Eastern-Southwestern Ojibwa" + }, + "ojp": { + "level0": "Japonic", + "level1": "Japanesic" + }, + "ojs": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian", + "level4": "Ojibwa-Potawatomi", + "level5": "Ojibwa", + "level6": "Severn-Algonquin" + }, + "ojv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "Central Northern Outlier Polynesian" + }, + "ojw": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian", + "level4": "Ojibwa-Potawatomi", + "level5": "Ojibwa", + "level6": "Nuclear Ojibwe", + "level7": "Northwestern-Saulteaux Ojibwa" + }, + "oka": { + "level0": "Salishan", + "level1": "Interior Salish", + "level2": "Southern Interior Salish", + "level3": "Okanaganic" + }, + "okb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "West Lower Cross" + }, + "okd": { + "level0": "Ijoid", + "level1": "Ijo", + "level2": "Western Ijo", + "level3": "Inland Ijo", + "level4": "Biseni-Okordia" + }, + "oke": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Southwestern Edoid" + }, + "okg": { + "level0": "Bookkeeping" + }, + "okh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Central Tatic", + "level10": "Khalkhalic" + }, + "oki": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Kalenjin", + "level3": "Okiek-Akie" + }, + "okj": { + "level0": "Great Andamanese", + "level1": "Middle Great Andamanese" + }, + "okk": { + "level0": "Nuclear Torricelli", + "level1": "West Wapei", + "level2": "One" + }, + "okl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "OKSLic" + }, + "okn": { + "level0": "Japonic", + "level1": "Ryukyuan", + "level2": "Northern Ryukyuan", + "level3": "Amami", + "level4": "Nuclear Amami", + "level5": "Okinoerabu-Tokunoshima" + }, + "okr": { + "level0": "Ijoid", + "level1": "Ijo", + "level2": "Eastern Ijo", + "level3": "Nikio", + "level4": "Kio Ijo" + }, + "oks": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo" + }, + "oku": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "Center Ring", + "level10": "Komic" + }, + "okv": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean", + "level3": "Nuclear Binanderean", + "level4": "South Binanderean", + "level5": "Orokaivic" + }, + "okx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Southern Northwestern Edoid", + "level7": "Okpe-Akuku-Idesa" + }, + "ola": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic" + }, + "old": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Kilimanjaro-Taita", + "level9": "Kilimanjaro Bantu", + "level10": "Chaga", + "level11": "Central Kilimanjaro" + }, + "ole": { + "level0": "Sino-Tibetan" + }, + "olk": { + "level0": "Bookkeeping" + }, + "olm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Southern Northwestern Edoid" + }, + "olo": { + "level0": "Uralic", + "level1": "Finnic", + "level2": "Coastal Finnic", + "level3": "Neva", + "level4": "North Finnic", + "level5": "Ladogan", + "level6": "East Ladoga" + }, + "olu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia" + }, + "oma": { + "level0": "Siouan", + "level1": "Mississippi Valley", + "level2": "Dhegiha" + }, + "omb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Ambae" + }, + "ome": { + "level0": "Bookkeeping" + }, + "omg": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup III", + "level7": "Omagua-Kokama" + }, + "omi": { + "level0": "Central Sudanic", + "level1": "Moru-Madi", + "level2": "Central Moru-Madi", + "level3": "Kalikoic" + }, + "omk": { + "level0": "Yukaghir" + }, + "oml": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Nkutsuic" + }, + "omn": { + "level0": "Unclassifiable" + }, + "omo": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Osum-Wadaginam-Pomoikan" + }, + "omr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Southern zone", + "level7": "Marathic", + "level8": "Marathi-Konkani", + "level9": "Old-Modern Marathi" + }, + "omt": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Tatoga-Omotik" + }, + "omw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Kainantu", + "level3": "Tairora" + }, + "omx": { + "level0": "Austroasiatic", + "level1": "Monic" + }, + "ona": { + "level0": "Chonan", + "level1": "Insular Chonan" + }, + "onb": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Beic", + "level4": "Lingao" + }, + "one": { + "level0": "Iroquoian", + "level1": "Northern Iroquoian", + "level2": "Mohawk-Oneida" + }, + "ong": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Wapeic", + "level4": "Au-Olo-Elkei", + "level5": "Olo-Elkei" + }, + "oni": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tanimbar-Bomberai", + "level3": "Nuclear Tanimbar-Bomberai", + "level4": "Yamdena-Onin", + "level5": "Oninic" + }, + "onj": { + "level0": "Dagan" + }, + "onk": { + "level0": "Nuclear Torricelli", + "level1": "West Wapei", + "level2": "One", + "level3": "Central-Northern One" + }, + "onn": { + "level0": "Bosavi", + "level1": "Bosavi Watershed" + }, + "ono": { + "level0": "Iroquoian", + "level1": "Northern Iroquoian" + }, + "onp": { + "level0": "Sino-Tibetan", + "level1": "Kho-Bwa", + "level2": "Western Kho-Bwa", + "level3": "Sartang-Sherdukpen" + }, + "onr": { + "level0": "Nuclear Torricelli", + "level1": "West Wapei", + "level2": "One", + "level3": "Central-Northern One" + }, + "ons": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Eastern Huon", + "level4": "Kalasa" + }, + "ont": { + "level0": "Bookkeeping" + }, + "onu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Eastern Malakula linkage", + "level8": "Central-Southeast Malakula", + "level9": "Unua-Pangkumu" + }, + "onw": { + "level0": "Nubian", + "level1": "Nile Nubian", + "level2": "Nobiin Nubian" + }, + "onx": { + "level0": "Pidgin", + "level1": "Onin-based pidgin" + }, + "ood": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tepiman", + "level3": "Piman" + }, + "oog": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "Ta'oihic", + "level3": "Ong-Ta'oih" + }, + "oon": { + "level0": "Jarawa-Onge" + }, + "oor": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch", + "level6": "Middle-Modern Dutch", + "level7": "Modern Dutch", + "level8": "Global Dutch", + "level9": "Afrikaansic" + }, + "oos": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Sogdic-Ossetic", + "level6": "Ossetic" + }, + "opa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Southern Northwestern Edoid" + }, + "ope": { + "level0": "Bookkeeping" + }, + "opk": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Kwer-Kopkaka-Burumakok" + }, + "opm": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin" + }, + "opo": { + "level0": "Eleman", + "level1": "Western Eleman" + }, + "opt": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Opata-Eudeve" + }, + "opy": { + "level0": "Nuclear-Macro-Je" + }, + "ora": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Malaita", + "level8": "Southern Malaita" + }, + "orc": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Oromoid", + "level7": "Nuclear Oromo", + "level8": "Central-Eastern Oromo", + "level9": "Central Oromo" + }, + "ore": { + "level0": "Tucanoan", + "level1": "Western Tucanoan", + "level2": "Napo Tucanoan" + }, + "org": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "North-South Central Delta Cross", + "level7": "Koring-Kukele" + }, + "orh": { + "level0": "Tungusic", + "level1": "Northeastern Tungusic", + "level2": "Northern Tungusic" + }, + "ork": { + "level0": "Bookkeeping" + }, + "orn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Greater Riau-Johoric" + }, + "oro": { + "level0": "Eleman", + "level1": "Western Eleman" + }, + "orr": { + "level0": "Ijoid", + "level1": "Ijo", + "level2": "Western Ijo", + "level3": "Inland Ijo" + }, + "ors": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Greater Riau-Johoric" + }, + "ort": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Macro-Oriya" + }, + "oru": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Ormuri-Parachi" + }, + "orv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "East Slavic" + }, + "orw": { + "level0": "Chapacuran", + "level1": "Moreic-Waric", + "level2": "Waric", + "level3": "Wanham-Wari-Oro Win", + "level4": "Wari-Oro Win" + }, + "orx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "West Lower Cross", + "level7": "Oroic", + "level8": "Ebughu-Oro" + }, + "ory": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Macro-Oriya" + }, + "orz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Jayapura Bay" + }, + "osa": { + "level0": "Siouan", + "level1": "Mississippi Valley", + "level2": "Dhegiha", + "level3": "Osage-Kansa" + }, + "osc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Sabellic" + }, + "osi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Javanesic", + "level3": "Modern Javanese" + }, + "oso": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Igwic" + }, + "osp": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Castilic" + }, + "oss": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Sogdic-Ossetic", + "level6": "Ossetic", + "level7": "Modern Ossetic" + }, + "ost": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Southwest Grassfields" + }, + "osu": { + "level0": "Nuclear Torricelli", + "level1": "West Wapei", + "level2": "One" + }, + "osx": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Alts\u00e4chsisch" + }, + "otd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "North West Greater Barito" + }, + "ote": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Otomian", + "level5": "Otomi", + "level6": "Northwestern Otomi" + }, + "otk": { + "level0": "Bookkeeping" + }, + "otl": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Otomian", + "level5": "Otomi", + "level6": "Southern Otomi" + }, + "otm": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Otomian", + "level5": "Otomi", + "level6": "Eastern Otomi" + }, + "otn": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Otomian", + "level5": "Otomi", + "level6": "Eastern Otomi" + }, + "otq": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Otomian", + "level5": "Otomi", + "level6": "Northwestern Otomi" + }, + "otr": { + "level0": "Heibanic", + "level1": "West-Central Heibanic", + "level2": "Central Heibanic", + "level3": "Ebang-Logol" + }, + "ots": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Otomian", + "level5": "Southwestern Otomi" + }, + "ott": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Otomian", + "level5": "Southwestern Otomi" + }, + "otu": { + "level0": "Bororoan", + "level1": "Bororo-Otuke" + }, + "otw": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian", + "level4": "Ojibwa-Potawatomi", + "level5": "Ojibwa" + }, + "otx": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Otomian", + "level5": "Otomi", + "level6": "Eastern Otomi" + }, + "oty": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid" + }, + "otz": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Otomian", + "level5": "Otomi", + "level6": "Southern Otomi" + }, + "oua": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Greater Zenatic", + "level3": "Zenatic", + "level4": "Northern Saharan Oasis Berber", + "level5": "Ouargli-Oued Righ" + }, + "oub": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Wee", + "level5": "Guere-Krahn" + }, + "oue": { + "level0": "South Bougainville", + "level1": "Nasioiic", + "level2": "Nasioi", + "level3": "South-Central Nasioi", + "level4": "Central Nasioi" + }, + "oui": { + "level0": "Turkic", + "level1": "Common Turkic" + }, + "oum": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "Oumic" + }, + "oun": { + "level0": "Bookkeeping" + }, + "owi": { + "level0": "Left May" + }, + "owl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Insular Celtic", + "level6": "Brythonic", + "level7": "Old-Modern Welsh" + }, + "oyb": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "Nuclear West Bahnaric" + }, + "oyd": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "North-West Ometo", + "level3": "Central Ometo" + }, + "oym": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VIII", + "level6": "Wayampi-Zoe-Emerillon" + }, + "oyy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "Suauic" + }, + "ozm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Mpoic", + "level10": "Njemic" + }, + "pab": { + "level0": "Arawakan", + "level1": "Central-Eastern Maipuran", + "level2": "Central Maipuran", + "level3": "Xaray", + "level4": "Parecis-Nawe" + }, + "pac": { + "level0": "Austroasiatic", + "level1": "Katuic" + }, + "pad": { + "level0": "Arawan" + }, + "pae": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Middle Bomokandian", + "level15": "Late Bomokandian", + "level16": "Pagabeteic" + }, + "paf": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VI", + "level6": "Kawahiva", + "level7": "Unclassified Kawahiva" + }, + "pag": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Southern Cordilleran", + "level6": "West Southern Cordilleran" + }, + "pah": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VI", + "level6": "Kawahiva", + "level7": "Nuclear Kawahiva", + "level8": "Central Kawahiva" + }, + "pai": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Tarokoid", + "level5": "Yangkam-Tarok-Pe", + "level6": "Tarok-Pe" + }, + "paj": { + "level0": "Bookkeeping" + }, + "pak": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup IV", + "level6": "Tupi-Guarani Subgroup IV.A" + }, + "pal": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian" + }, + "pam": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Luzon" + }, + "pan": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Greater Panjabic", + "level9": "Eastern Panjabic" + }, + "pao": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Numic", + "level3": "Western Numic" + }, + "pap": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Upper Guinea Portuguese" + }, + "paq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Mewati-Gojri" + }, + "par": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Numic", + "level3": "Central Numic" + }, + "pas": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "East Tariku", + "level3": "Doutai-Kai-Waritai" + }, + "pau": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian" + }, + "pav": { + "level0": "Chapacuran", + "level1": "Moreic-Waric", + "level2": "Waric", + "level3": "Wanham-Wari-Oro Win", + "level4": "Wari-Oro Win" + }, + "paw": { + "level0": "Caddoan", + "level1": "Northern Caddoan", + "level2": "Pawnee-Kitsai", + "level3": "Pawnee-Arikara" + }, + "pax": { + "level0": "Unattested" + }, + "pay": { + "level0": "Chibchan" + }, + "pbc": { + "level0": "Cariban", + "level1": "Venezuelan Cariban", + "level2": "Pemong-Panare", + "level3": "Pemongan", + "level4": "Kapong" + }, + "pbe": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Ixcatec-Chocho-Popolocan", + "level5": "Chocho-Popolocan", + "level6": "Popolocan", + "level7": "Southwestern Popolocan", + "level8": "Tepexi-Zapotitlan" + }, + "pbf": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Ixcatec-Chocho-Popolocan", + "level5": "Chocho-Popolocan", + "level6": "Popolocan", + "level7": "Southwestern Popolocan" + }, + "pbg": { + "level0": "Arawakan", + "level1": "Caribbean Arawakan", + "level2": "Guajiro-Paraujano" + }, + "pbh": { + "level0": "Cariban", + "level1": "Venezuelan Cariban", + "level2": "Pemong-Panare" + }, + "pbi": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mandaraic", + "level6": "Podoko" + }, + "pbl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bikwin-Jen", + "level5": "Northern Bikwin-Jen", + "level6": "Mak-Tal" + }, + "pbm": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Mazatecan", + "level5": "Northwest Alta Mazatec" + }, + "pbn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Mumuye-Yandang", + "level6": "Yandangic", + "level7": "Bali-Kpasam" + }, + "pbo": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Central Atlantic", + "level3": "Bak", + "level4": "Joola-Manjaku", + "level5": "Manjaku-Mankanya-Pepel", + "level6": "Cur-Bok-Cotier" + }, + "pbp": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Jaad" + }, + "pbr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Southern Tanzania Highlands Bantu", + "level9": "Kisi-Pangwa" + }, + "pbs": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Pamean" + }, + "pbt": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Pashto", + "level5": "Nuclear Pashto" + }, + "pbu": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Pashto", + "level5": "Nuclear Pashto" + }, + "pbv": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Khasian", + "level3": "Khasi-Pnar-Lyngngam", + "level4": "Khasi-Pnar" + }, + "pbz": { + "level0": "Bookkeeping" + }, + "pca": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Ixcatec-Chocho-Popolocan", + "level5": "Chocho-Popolocan", + "level6": "Popolocan", + "level7": "Southwestern Popolocan", + "level8": "Tepexi-Zapotitlan" + }, + "pcb": { + "level0": "Austroasiatic", + "level1": "Pearic" + }, + "pcc": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Northern Tai" + }, + "pcd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil" + }, + "pce": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "West Palaungic", + "level4": "Palaung" + }, + "pcf": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid", + "level10": "Tamil-Paliyan" + }, + "pcg": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid", + "level10": "Ravulic" + }, + "pch": { + "level0": "Unattested", + "level1": "Dravidian (Unattested)" + }, + "pci": { + "level0": "Dravidian", + "level1": "Central Dravidian", + "level2": "Parji-Ollari-Gadaba" + }, + "pcj": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "Sora-Juray-Gorum" + }, + "pck": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Northeastern Kuki-Chin", + "level5": "Thadoic" + }, + "pcl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "pcm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "West African Creole English", + "level13": "Coastal Nigerian Krio", + "level14": "Nigeria-Cameroon Creole English" + }, + "pcn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Piti-Atsam" + }, + "pcp": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Bolivian Nawa" + }, + "pcr": { + "level0": "Bookkeeping" + }, + "pcw": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3", + "level6": "Goemaic", + "level7": "Talic", + "level8": "Piapung-Koenoem" + }, + "pda": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Osum-Wadaginam-Pomoikan", + "level5": "Pomoikan" + }, + "pdc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Middle German", + "level7": "West Middle German", + "level8": "Rhenish Franconian", + "level9": "Palatinate" + }, + "pdi": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Chiang Saeng", + "level10": "White Tai" + }, + "pdn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Sarmi", + "level8": "Anus-Podena" + }, + "pdo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Western Bungku-Tolaki", + "level8": "Interior Bungku-Tolaki" + }, + "pdt": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Alts\u00e4chsisch", + "level7": "Middle-Modern Low German", + "level8": "Low German", + "level9": "Greater East Low German" + }, + "pdu": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Northern Karen" + }, + "pea": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Betawic" + }, + "peb": { + "level0": "Pomoan", + "level1": "Russian River and Eastern" + }, + "pec": { + "level0": "Bookkeeping" + }, + "ped": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kaukombaran" + }, + "pee": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Tominic", + "level5": "Southern Tomini" + }, + "pef": { + "level0": "Pomoan" + }, + "peg": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Konda-Kui", + "level4": "Manda-Kui", + "level5": "Manda-Pengo" + }, + "peh": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Southern Periphery Mongolic", + "level3": "Shirongol", + "level4": "Baoanic" + }, + "pei": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean" + }, + "pej": { + "level0": "Pomoan", + "level1": "Russian River and Eastern", + "level2": "Russian River", + "level3": "Northern-Central Pomoan" + }, + "pek": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "South-East Admiralty" + }, + "pel": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Northern Sumatra Malay", + "level6": "Kerinci-Minangkabau", + "level7": "Minangkabauic" + }, + "pem": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Northern Njila", + "level9": "Mbala-Holu-Sondi (K.10)", + "level10": "Holu (K.10)", + "level11": "Pheende-Kwezo" + }, + "peo": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian" + }, + "pep": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Tonda", + "level3": "Eastern Tonda" + }, + "peq": { + "level0": "Pomoan", + "level1": "Russian River and Eastern", + "level2": "Russian River", + "level3": "Southern Pomoan-Kashaya" + }, + "pes": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Farsic" + }, + "pev": { + "level0": "Cariban", + "level1": "Venezuelan Cariban", + "level2": "Mapoyo-Tamanaku", + "level3": "Mapoyo-Yawarana" + }, + "pex": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "North Bougainville Oceanic", + "level10": "Nuclear North Bougainville Oceanic", + "level11": "Buka" + }, + "pey": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch", + "level6": "Middle-Modern Dutch", + "level7": "Modern Dutch", + "level8": "Global Dutch" + }, + "pez": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Kenyahic", + "level5": "Lowland Kenyah", + "level6": "Western Lowland Kenyah-Penan", + "level7": "Penan" + }, + "pfa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Nuclear Trukic", + "level10": "Central Trukic", + "level11": "Satawalese-Carolinian", + "level12": "Macro-Carolinian", + "level13": "Murilo-Fanapanges" + }, + "pfe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Southern Samba-Duru" + }, + "pfl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Middle German", + "level7": "West Middle German", + "level8": "Rhenish Franconian", + "level9": "Palatinate" + }, + "pga": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Egyptic Arabic", + "level7": "Egypto-Sudanic Arabic", + "level8": "Sudanese-Chadian Arabic", + "level9": "East Sudanic Arabic" + }, + "pgd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic" + }, + "pgg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Kangric-Chamealic-Bhattiyali", + "level8": "Chamealic" + }, + "pgi": { + "level0": "Border", + "level1": "Bewani", + "level2": "Pagi-Kilmeri" + }, + "pgk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Eastern Malakula linkage", + "level8": "Central-Southeast Malakula", + "level9": "Unua-Pangkumu" + }, + "pgs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Mumuye-Yandang", + "level6": "Mumuyic" + }, + "pgu": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Mainland North Halmaheran", + "level3": "Kao River", + "level4": "Paguic" + }, + "pgy": { + "level0": "Bookkeeping" + }, + "pgz": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "BSLic", + "level3": "BANZL", + "level4": "Auslanic" + }, + "pha": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Paheng-Younuo", + "level3": "Paheng" + }, + "phd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Southern zone", + "level7": "Marathic", + "level8": "Marathi-Konkani", + "level9": "Old-Modern Marathi", + "level10": "Modern Marathi", + "level11": "Western Marathi" + }, + "phg": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "Katu" + }, + "phh": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Phowa", + "level8": "Hlepho-Phukha" + }, + "phj": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Newaric", + "level4": "Newar" + }, + "phk": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Sukaphic", + "level11": "Mogaung", + "level12": "Assam Tai A" + }, + "phl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Shinaic", + "level8": "Western Shinaic", + "level9": "Dangari" + }, + "phm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Tumbuka-Sena-Nyanja", + "level8": "Sena-Nyanja", + "level9": "Senaic" + }, + "phn": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Canaanite", + "level6": "Ugarito-Phoenician", + "level7": "Phoenician-Punic" + }, + "pho": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Bisoid", + "level7": "Phunoi-Coong" + }, + "phq": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic" + }, + "phr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Greater Panjabic", + "level9": "Paharic" + }, + "pht": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Thai PH", + "level9": "Siamese" + }, + "phu": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Thai PH", + "level9": "Siamese" + }, + "phv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Farsic", + "level9": "Eastern Farsic" + }, + "phw": { + "level0": "Bookkeeping" + }, + "pia": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tepiman", + "level3": "Piman" + }, + "pib": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Purus-Chamicuro", + "level3": "Purus", + "level4": "Yineic", + "level5": "Western Yineic" + }, + "pic": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "B10-B30", + "level8": "Okani (B.30)", + "level9": "Northern Okani", + "level10": "Himba-Pinji" + }, + "pid": { + "level0": "Saliban", + "level1": "Maco-Piaroa" + }, + "pie": { + "level0": "Kiowa-Tanoan", + "level1": "Tiwa-Piro" + }, + "pif": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Ponapeic" + }, + "pig": { + "level0": "Unattested", + "level1": "Pano-Tacanan (Unattested)" + }, + "pih": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English" + }, + "pij": { + "level0": "Unclassifiable" + }, + "pil": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Gurma-Yom-Naudem", + "level11": "Yom-Nawdm" + }, + "pim": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian" + }, + "pin": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Western Sepik Hill", + "level3": "Hewa-April River" + }, + "pio": { + "level0": "Arawakan", + "level1": "Japura-Colombia", + "level2": "Nuclear Japura-Colombia", + "level3": "Northeast Japura-Colombia", + "level4": "Piapoco-Achagua" + }, + "pip": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Tangalic", + "level7": "Nuclear Tangalic", + "level8": "Peroic" + }, + "pir": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Eastern Eastern Tucanoan", + "level3": "Eastern Eastern Tucanoan II", + "level4": "Kotiria-Piratapuyo", + "level5": "Piratapuyic" + }, + "pis": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Pacific Creole English", + "level12": "Early Melanesian Pidgin" + }, + "pit": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Palku", + "level3": "Pitta-Pitta" + }, + "piu": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Wati", + "level3": "Pintupic", + "level4": "Nuclear Pintupic" + }, + "piv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian" + }, + "piw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Mwika" + }, + "pix": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Buang linkage" + }, + "piy": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Tangalic", + "level7": "Nuclear Tangalic", + "level8": "Peroic" + }, + "piz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Nmi-Pij-Fwa-Pam-Pap", + "level10": "Nmi-Fij-Fwa" + }, + "pjt": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Wati", + "level3": "Pintupic", + "level4": "Nuclear Pintupic", + "level5": "Wangkatja-Tjarra", + "level6": "Tjarra" + }, + "pkb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Mijikenda-Pokomo-Comorian" + }, + "pkc": { + "level0": "Unclassifiable" + }, + "pkg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus" + }, + "pkh": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin" + }, + "pkn": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Kuku-Wik-Ep", + "level5": "Kuku-Wik", + "level6": "Mungkanic" + }, + "pko": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Kalenjin", + "level3": "Northern Kalenjin" + }, + "pkp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Ellicean", + "level9": "Pukapukic" + }, + "pkr": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Irula-Muduga", + "level8": "Muduga-Palu" + }, + "pks": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Indo-Pakistani-Nepalese Sign", + "level3": "Indo-Pakistani Sign" + }, + "pkt": { + "level0": "Austroasiatic", + "level1": "Vietic", + "level2": "Chutic", + "level3": "East Chutic" + }, + "pku": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Maanyan-Paku" + }, + "pla": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kaukombaran" + }, + "plb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "East Santo", + "level9": "Southeast Santo" + }, + "plc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Palawanic", + "level4": "Southern Palawanic", + "level5": "Molbog-Palawan", + "level6": "Nuclear Palawan" + }, + "pld": { + "level0": "Unclassifiable" + }, + "ple": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Central Flores-Paluqe" + }, + "plg": { + "level0": "Guaicuruan", + "level1": "Guaicuru del Sur", + "level2": "Qom", + "level3": "Pilaga-Toba" + }, + "plh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua" + }, + "pli": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari" + }, + "plk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Shinaic", + "level8": "Kohistanic Shina" + }, + "pll": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "West Palaungic", + "level4": "Palaung" + }, + "pln": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Castilic", + "level13": "South Castilic" + }, + "plo": { + "level0": "Mixe-Zoque", + "level1": "Mixe" + }, + "plp": { + "level0": "Bookkeeping" + }, + "plq": { + "level0": "Indo-European", + "level1": "Anatolian", + "level2": "Luvo-Lydian", + "level3": "Luvo-Palaic" + }, + "plr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "South Senufo" + }, + "pls": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Ixcatec-Chocho-Popolocan", + "level5": "Chocho-Popolocan", + "level6": "Popolocan" + }, + "plt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "North-Central Malagasic", + "level7": "Central-Eastern Malagasic" + }, + "plu": { + "level0": "Arawakan", + "level1": "Central-Eastern Maipuran" + }, + "plv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Palawanic", + "level4": "Southern Palawanic", + "level5": "Molbog-Palawan", + "level6": "Nuclear Palawan", + "level7": "Brooke-Canipaan Palawan" + }, + "plw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Palawanic", + "level4": "Southern Palawanic", + "level5": "Molbog-Palawan", + "level6": "Nuclear Palawan", + "level7": "Brooke-Canipaan Palawan" + }, + "ply": { + "level0": "Austroasiatic", + "level1": "Mangic", + "level2": "Pakanic" + }, + "plz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Northern Murutic" + }, + "pma": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu" + }, + "pmb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Zandic", + "level6": "Barambo-Pambia" + }, + "pmc": { + "level0": "Unattested" + }, + "pmd": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "Victorian Pama-Nyungan", + "level3": "Eastern Victoria", + "level4": "Dhudhuroa-Pallanganmiddang" + }, + "pme": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Nmi-Pij-Fwa-Pam-Pap" + }, + "pmf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio", + "level5": "Pamona-Tombelala" + }, + "pmh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Southern zone" + }, + "pmi": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic", + "level4": "Pumi" + }, + "pmj": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic", + "level4": "Pumi" + }, + "pml": { + "level0": "Pidgin", + "level1": "Romance-based pidgin" + }, + "pmm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)" + }, + "pmn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Northern Mbum", + "level6": "Tupuri-Mundang-Mambai", + "level7": "Mundangic" + }, + "pmo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Central-Western Yapen", + "level7": "Central Yapen" + }, + "pmq": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Pamean" + }, + "pmr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Sogeram", + "level5": "Apalic", + "level6": "Greater West Sogeram" + }, + "pms": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Italian", + "level12": "Piemontese-Lombard" + }, + "pmt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Proximal", + "level13": "Southern East Polynesian Proximal" + }, + "pmw": { + "level0": "Miwok-Costanoan", + "level1": "Miwokan", + "level2": "Eastern Miwokan" + }, + "pmx": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Angami-Pochuri", + "level4": "Angami-Mao", + "level5": "Naga Maoic", + "level6": "Poumaic" + }, + "pmy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay", + "level6": "Eastern Indonesia Trade Malay" + }, + "pmz": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Otopame-Chinantecan", + "level3": "Otopamean", + "level4": "Pamean" + }, + "pna": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Punan", + "level6": "Punan Tubu-Bah" + }, + "pnb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Greater Panjabic" + }, + "pnc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Pitu Ulunna Salu" + }, + "pnd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Northern Njila", + "level9": "Mbundu (H.20)" + }, + "pne": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Kenyahic", + "level5": "Lowland Kenyah", + "level6": "Western Lowland Kenyah-Penan", + "level7": "Penan", + "level8": "Western Penan-Sebop" + }, + "png": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Shiroro-Kamuku", + "level6": "Shiroro" + }, + "pnh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Proximal", + "level13": "Southern East Polynesian Proximal" + }, + "pni": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Muller-Schwaner" + }, + "pnk": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Bolivian Arawakan", + "level3": "Mojeno-Paunaca" + }, + "pnl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Samu" + }, + "pnm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Punan", + "level6": "Aput-Busang-Merah-Kohi" + }, + "pnn": { + "level0": "Piawi" + }, + "pno": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Chama subgroup" + }, + "pnp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Nuclear Muna-Buton", + "level8": "Munan", + "level9": "Munic", + "level10": "Western Munic" + }, + "pnq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "Northern Grusi" + }, + "pnr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Gum", + "level5": "Panim-Isebe-Bau" + }, + "pns": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Gorontalo-Mongondow", + "level4": "Mongondowic" + }, + "pnt": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Graeco-Phrygian", + "level3": "Greek", + "level4": "South Greek", + "level5": "Central Greek", + "level6": "Koineic Greek", + "level7": "Modern Koineic Greek", + "level8": "Pontic-Cappadocian Greek" + }, + "pnu": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Jiongnai-Ho Ne" + }, + "pnv": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Kanyara" + }, + "pnw": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda", + "level4": "Central Ngayarda", + "level5": "Panytyima-Yinhawangka" + }, + "pnx": { + "level0": "Austroasiatic", + "level1": "Khmuic", + "level2": "Phay-Pram", + "level3": "Pramic" + }, + "pny": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Ngembaic" + }, + "pnz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Central Mbum", + "level6": "Karangic", + "level7": "Kare-Pana" + }, + "poa": { + "level0": "Bookkeeping" + }, + "pob": { + "level0": "Bookkeeping" + }, + "poc": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Quichean", + "level4": "Poqom" + }, + "pod": { + "level0": "Bookkeeping" + }, + "poe": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Ixcatec-Chocho-Popolocan", + "level5": "Chocho-Popolocan", + "level6": "Popolocan", + "level7": "Southwestern Popolocan" + }, + "pof": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke", + "level11": "So-Poke" + }, + "poh": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Quichean", + "level4": "Poqom" + }, + "poi": { + "level0": "Mixe-Zoque", + "level1": "Zoque", + "level2": "Gulf Zoque" + }, + "poj": { + "level0": "Bookkeeping" + }, + "pol": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "West Slavic", + "level5": "Lechitic", + "level6": "Polish-Silesian" + }, + "pom": { + "level0": "Pomoan" + }, + "pon": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Ponapeic" + }, + "poo": { + "level0": "Pomoan", + "level1": "Russian River and Eastern", + "level2": "Russian River", + "level3": "Northern-Central Pomoan" + }, + "pop": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Nmi-Pij-Fwa-Pam-Pap" + }, + "poq": { + "level0": "Mixe-Zoque", + "level1": "Zoque", + "level2": "Gulf Zoque", + "level3": "Texistepec-Ayapa Zoque" + }, + "por": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Brazil-Portugal Portuguese" + }, + "pos": { + "level0": "Mixe-Zoque", + "level1": "Mixe" + }, + "pot": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian", + "level4": "Ojibwa-Potawatomi" + }, + "pou": { + "level0": "Bookkeeping" + }, + "pov": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Upper Guinea Portuguese" + }, + "pow": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Ixcatec-Chocho-Popolocan", + "level5": "Chocho-Popolocan", + "level6": "Popolocan", + "level7": "Southwestern Popolocan", + "level8": "Tepexi-Zapotitlan" + }, + "pox": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "West Slavic", + "level5": "Lechitic" + }, + "poy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Kilombero" + }, + "ppa": { + "level0": "Bookkeeping" + }, + "ppi": { + "level0": "Cochimi-Yuman", + "level1": "Yuman", + "level2": "General Yuman", + "level3": "Pai" + }, + "ppk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio", + "level5": "Uma-Sarudu" + }, + "ppl": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec", + "level4": "Eastern Nahuatl", + "level5": "Isthmus-Pipil Nahuatl" + }, + "ppm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Central-Western Yapen", + "level7": "Central Yapen" + }, + "ppn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "North Bougainville Oceanic", + "level10": "Nuclear North Bougainville Oceanic" + }, + "ppo": { + "level0": "Teberan" + }, + "ppq": { + "level0": "Walioic", + "level1": "Pai-Sinen-Walio" + }, + "ppr": { + "level0": "Bookkeeping" + }, + "pps": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Ixcatec-Chocho-Popolocan", + "level5": "Chocho-Popolocan", + "level6": "Popolocan" + }, + "ppt": { + "level0": "Kamula-Elevala", + "level1": "Elevala" + }, + "ppu": { + "level0": "Austronesian", + "level1": "Western Plains Austronesian", + "level2": "Central Western Plains" + }, + "ppv": { + "level0": "Unattested", + "level1": "Pano-Tacanan (Unattested)" + }, + "pqa": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.2", + "level5": "Nuclear West Chadic B.2", + "level6": "Western West Chadic B.2" + }, + "pqm": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Maritimes-Southern New England Algonquian", + "level5": "Northern Eastern Algonquian", + "level6": "Micmacic" + }, + "prb": { + "level0": "Bookkeeping" + }, + "prc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Ormuri-Parachi" + }, + "prd": { + "level0": "Bookkeeping" + }, + "pre": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Lower Guinea Portuguese" + }, + "prf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Northeastern Luzon", + "level4": "Nuclear Northeastern Luzon", + "level5": "Paranan-Pahanan" + }, + "prg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic" + }, + "prh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Central Bisayan", + "level6": "Peripheral Central Bisayan" + }, + "pri": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Voh-Kone-Cem-Pac", + "level10": "Cem-Pac" + }, + "prk": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Waic", + "level5": "Wa-Lawa", + "level6": "Nuclear Waic" + }, + "prl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "West-Central South American Sign", + "level5": "Peruvian-Inmaculada Sign" + }, + "prn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Nuristani" + }, + "pro": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "Occitanic" + }, + "prp": { + "level0": "Bookkeeping" + }, + "prq": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Kampa-Amuesha", + "level3": "Pre-Andine Maipuran", + "level4": "Asha-Ashe-Kak-Matsi-Nan", + "level5": "Asha-Ashe-Kak", + "level6": "Ashe-Asha", + "level7": "Ashe-Asha Norte" + }, + "prr": { + "level0": "Puri-Coroado" + }, + "prs": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Farsic", + "level9": "Eastern Farsic" + }, + "prt": { + "level0": "Austroasiatic", + "level1": "Khmuic", + "level2": "Phay-Pram", + "level3": "Tinic", + "level4": "Tin" + }, + "pru": { + "level0": "South Bird's Head Family" + }, + "prv": { + "level0": "Bookkeeping" + }, + "prw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Numugenan", + "level6": "Yarawata-Parawen-Ukuriguma" + }, + "prx": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Western Archaic Tibetan", + "level5": "Shamskatic" + }, + "pry": { + "level0": "Bookkeeping" + }, + "prz": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Providencia-Cayman Sign" + }, + "psa": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Awyu" + }, + "psc": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "psd": { + "level0": "Sign Language", + "level1": "Auxiliary Sign Systems" + }, + "pse": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "South Sumatra Malay" + }, + "psg": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "psh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Pashayi", + "level5": "Western Pashayi" + }, + "psi": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Pashayi", + "level5": "Eastern Pashayi" + }, + "psl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic" + }, + "psm": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup II", + "level7": "Warazu-Sirionoid" + }, + "psn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "Rampi-Seko-Badaic", + "level4": "Seko", + "level5": "Panasuanic" + }, + "pso": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Central European Sign" + }, + "psp": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic" + }, + "psq": { + "level0": "Sepik", + "level1": "Sepik Tama", + "level2": "Mayo-Pasi", + "level3": "Yimin-Bel" + }, + "psr": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Swedish Sign" + }, + "pss": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Pasismanua" + }, + "pst": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Pashto", + "level5": "Nuclear Pashto" + }, + "psu": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic" + }, + "psw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Eastern Malakula linkage", + "level8": "Central-Southeast Malakula", + "level9": "Southeastern Malakula linkage", + "level10": "Port Sandwich-Axamb-Avok" + }, + "psy": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Nanticoke-Conoy" + }, + "pta": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup I", + "level7": "Tupi-Guarani Subgroup I.A", + "level8": "Paraguay-Brazil Guarani", + "level9": "Kaiowa" + }, + "pth": { + "level0": "Nuclear-Macro-Je", + "level1": "Maxakali-Borum", + "level2": "Maxakalian", + "level3": "Nuclear Maxakalian" + }, + "pti": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Wati", + "level3": "Pintupic", + "level4": "Nuclear Pintupic", + "level5": "Wangkatja-Tjarra" + }, + "ptn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "South Halmahera", + "level6": "Central-Eastern South Halmahera" + }, + "pto": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VIII", + "level6": "Wayampi-Zoe-Emerillon", + "level7": "Zoe-Emerillon" + }, + "ptp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Buang linkage", + "level9": "Mumeng", + "level10": "Zenag-Patep" + }, + "ptq": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid", + "level10": "Yerukula-Korava-Kaikadi" + }, + "ptr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "Cape Cumberland" + }, + "ptt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Masenrempulu" + }, + "ptu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Pitu Ulunna Salu", + "level6": "Matangnga-Aralle-Tabulahan" + }, + "ptv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Ambrym", + "level7": "Orkon-West Ambrym", + "level8": "West Ambrym", + "level9": "Southwest Ambrym" + }, + "ptw": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Central Salish", + "level3": "North Georgia Central Salish" + }, + "pty": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid", + "level10": "Kalanadic" + }, + "pua": { + "level0": "Tarascan" + }, + "pub": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Central Old Kuki" + }, + "puc": { + "level0": "Bookkeeping" + }, + "pud": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Punan", + "level6": "Aput-Busang-Merah-Kohi" + }, + "puf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Punan", + "level6": "Aput-Busang-Merah-Kohi" + }, + "pug": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Western Grusi" + }, + "puj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Punan", + "level6": "Punan Tubu-Bah" + }, + "puk": { + "level0": "Bookkeeping" + }, + "pum": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Southern Kiranti" + }, + "puo": { + "level0": "Austroasiatic", + "level1": "Khmuic", + "level2": "Phay-Pram" + }, + "pup": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Kabenau" + }, + "pur": { + "level0": "Tupian", + "level1": "Purubora-Ramarama" + }, + "put": { + "level0": "Bookkeeping" + }, + "puu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "West Kikongo", + "level20": "Vilic", + "level21": "Lumbuic", + "level22": "Ngubi-Sangu-Sira-Punu", + "level23": "Sangu-Sira-Punu", + "level24": "Punu-Vungu" + }, + "puw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Nuclear Trukic", + "level10": "Central Trukic", + "level11": "Eastern Trukic", + "level12": "Puluwatese-Pollapese" + }, + "pux": { + "level0": "Sko", + "level1": "Skou-Serra-Piore", + "level2": "Serra Hills" + }, + "puy": { + "level0": "Chumashan", + "level1": "Southern Chumashan", + "level2": "Central Chumashan" + }, + "puz": { + "level0": "Bookkeeping" + }, + "pwb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "North-Central Jos", + "level10": "Boze-Loro" + }, + "pwg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Are linkage", + "level10": "Boanaki-Paiwa" + }, + "pwi": { + "level0": "Wintuan" + }, + "pwm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Palawanic", + "level4": "Southern Palawanic", + "level5": "Molbog-Palawan" + }, + "pwn": { + "level0": "Austronesian" + }, + "pwo": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Peripheral Karen", + "level3": "Pwo", + "level4": "Eastern-Western Pwo Karen" + }, + "pwr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Eastern Hindi" + }, + "pww": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Peripheral Karen", + "level3": "Pwo", + "level4": "Northern Pwo Karen" + }, + "pxm": { + "level0": "Bookkeeping" + }, + "pye": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Grebo-Aizi", + "level4": "Grebo", + "level5": "Ivorian Grebo" + }, + "pym": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Southeastern Benue-Congo Plateau", + "level5": "Horom-Fyem" + }, + "pyn": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Poyanawa Subgroup" + }, + "pys": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "pyu": { + "level0": "Austronesian" + }, + "pyx": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Jingpho-Luish", + "level3": "Luish", + "level4": "Unclassified Luish" + }, + "pyy": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Bisoid", + "level7": "Bisu-Pyen-Laomian" + }, + "pze": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Northwest South Bauchi", + "level7": "Polci-Luri", + "level8": "Polcic" + }, + "pzh": { + "level0": "Austronesian", + "level1": "Northwest Formosan" + }, + "pzn": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Central Naga", + "level4": "Yimchingric" + }, + "qbb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic" + }, + "qcs": { + "level0": "Mixe-Zoque", + "level1": "Mixe" + }, + "qer": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "North Germanic", + "level5": "North Scandinavian", + "level6": "East-Central Swedic" + }, + "qgu": { + "level0": "Pama-Nyungan", + "level1": "Nyawaygic" + }, + "qhr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Sabellic" + }, + "qkn": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Badaga-Kannada", + "level5": "Kannadoid", + "level6": "Nuclear Kannaoid" + }, + "qlm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Western Caribbean Creole", + "level14": "Jamaicanic" + }, + "qmx": { + "level0": "Bookkeeping" + }, + "qok": { + "level0": "Austroasiatic", + "level1": "Khmeric" + }, + "qpp": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone" + }, + "qua": { + "level0": "Siouan", + "level1": "Mississippi Valley", + "level2": "Dhegiha" + }, + "qub": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "AP-AM-AH" + }, + "quc": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Quichean", + "level4": "Core Quichean", + "level5": "Quiche-Achi" + }, + "qud": { + "level0": "Quechuan", + "level1": "Colombia-Ecuador Quechua", + "level2": "Ecuadorian Quechua A" + }, + "quf": { + "level0": "Quechuan", + "level1": "Cajamarca-Lambayeque Quechua" + }, + "qug": { + "level0": "Quechuan", + "level1": "Colombia-Ecuador Quechua", + "level2": "Ecuadorian Quechua A" + }, + "quh": { + "level0": "Quechuan", + "level1": "Southern Quechua", + "level2": "Bolivian-Argentinian Quechua", + "level3": "South Bolivian-Argentinian Quechua" + }, + "qui": { + "level0": "Chimakuan" + }, + "quj": { + "level0": "Bookkeeping" + }, + "quk": { + "level0": "Quechuan", + "level1": "San Martin-Amazonas Quechua" + }, + "qul": { + "level0": "Quechuan", + "level1": "Southern Quechua", + "level2": "Bolivian-Argentinian Quechua" + }, + "qum": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Quichean", + "level4": "Core Quichean" + }, + "qun": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Tsamosan", + "level3": "Coastal Tsamosan" + }, + "qup": { + "level0": "Quechuan", + "level1": "Colombia-Ecuador Quechua", + "level2": "Ecuadorian Quechua B", + "level3": "Imbabura-Colombia-Oriente Quechua", + "level4": "Colombia-Oriente Quechua", + "level5": "Oriente Quechua", + "level6": "Pastaza Quechua" + }, + "quq": { + "level0": "Unclassifiable" + }, + "qur": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "Yaru Quechua" + }, + "qus": { + "level0": "Quechuan", + "level1": "Southern Quechua", + "level2": "Bolivian-Argentinian Quechua", + "level3": "South Bolivian-Argentinian Quechua" + }, + "qut": { + "level0": "Bookkeeping" + }, + "quu": { + "level0": "Bookkeeping" + }, + "quv": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Quichean", + "level4": "Core Quichean" + }, + "quw": { + "level0": "Quechuan", + "level1": "Colombia-Ecuador Quechua", + "level2": "Ecuadorian Quechua B", + "level3": "Imbabura-Colombia-Oriente Quechua", + "level4": "Colombia-Oriente Quechua", + "level5": "Oriente Quechua" + }, + "qux": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Yauyosic" + }, + "quy": { + "level0": "Quechuan", + "level1": "Southern Quechua", + "level2": "Ayacuchan Quechua" + }, + "quz": { + "level0": "Quechuan", + "level1": "Southern Quechua", + "level2": "Cuscan Quechua" + }, + "qva": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "Yaru Quechua" + }, + "qvc": { + "level0": "Quechuan", + "level1": "Cajamarca-Lambayeque Quechua" + }, + "qve": { + "level0": "Quechuan", + "level1": "Southern Quechua", + "level2": "Cuscan Quechua" + }, + "qvh": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "Huaylay" + }, + "qvi": { + "level0": "Quechuan", + "level1": "Colombia-Ecuador Quechua", + "level2": "Ecuadorian Quechua B", + "level3": "Imbabura-Colombia-Oriente Quechua" + }, + "qvj": { + "level0": "Quechuan", + "level1": "Colombia-Ecuador Quechua", + "level2": "Ecuadorian Quechua B" + }, + "qvl": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "AP-AM-AH" + }, + "qvm": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "AP-AM-AH", + "level4": "Panao-Union" + }, + "qvn": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "Yaru Quechua" + }, + "qvo": { + "level0": "Quechuan", + "level1": "Colombia-Ecuador Quechua", + "level2": "Ecuadorian Quechua B", + "level3": "Imbabura-Colombia-Oriente Quechua", + "level4": "Colombia-Oriente Quechua", + "level5": "Oriente Quechua" + }, + "qvp": { + "level0": "Quechuan", + "level1": "Quechua I" + }, + "qvs": { + "level0": "Quechuan", + "level1": "San Martin-Amazonas Quechua" + }, + "qvw": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "Jauja-Huanca" + }, + "qvy": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic" + }, + "qvz": { + "level0": "Quechuan", + "level1": "Colombia-Ecuador Quechua", + "level2": "Ecuadorian Quechua B", + "level3": "Imbabura-Colombia-Oriente Quechua", + "level4": "Colombia-Oriente Quechua", + "level5": "Oriente Quechua", + "level6": "Pastaza Quechua" + }, + "qwa": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "Huaylay", + "level4": "Corongo-Sihuas" + }, + "qwc": { + "level0": "Quechuan" + }, + "qwh": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "Huaylay" + }, + "qws": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "Huaylay", + "level4": "Corongo-Sihuas" + }, + "qwt": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Pacific Coast Athabaskan" + }, + "qxa": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "AP-AM-AH" + }, + "qxc": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Yauyosic" + }, + "qxh": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "AP-AM-AH", + "level4": "Panao-Union" + }, + "qxi": { + "level0": "Bookkeeping" + }, + "qxl": { + "level0": "Quechuan", + "level1": "Colombia-Ecuador Quechua", + "level2": "Ecuadorian Quechua A" + }, + "qxn": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "Huaylay", + "level4": "Conchucos" + }, + "qxo": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "Huaylay", + "level4": "Conchucos" + }, + "qxp": { + "level0": "Quechuan", + "level1": "Southern Quechua", + "level2": "Cuscan Quechua" + }, + "qxq": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Oghuz", + "level3": "Nuclear Oghuz" + }, + "qxr": { + "level0": "Quechuan", + "level1": "Colombia-Ecuador Quechua", + "level2": "Ecuadorian Quechua B" + }, + "qxs": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic", + "level4": "Qiang" + }, + "qxu": { + "level0": "Quechuan", + "level1": "Southern Quechua", + "level2": "Ayacuchan Quechua" + }, + "qxw": { + "level0": "Quechuan", + "level1": "Quechua I", + "level2": "Central Quechua I", + "level3": "Jauja-Huanca" + }, + "qya": { + "level0": "Artificial Language" + }, + "qyp": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Maritimes-Southern New England Algonquian", + "level5": "Southern New England Algonquian", + "level6": "Western Southern New England Algonquian" + }, + "raa": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Southern Kiranti" + }, + "rab": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Southern Kiranti" + }, + "rac": { + "level0": "Lakes Plain", + "level1": "Far West Lakes Plain", + "level2": "Rasawa-Saponi" + }, + "rad": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic", + "level4": "Chamic", + "level5": "Rade-Jarai" + }, + "rae": { + "level0": "Bookkeeping" + }, + "raf": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Upper Arun", + "level6": "Mewahang" + }, + "rag": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Greater Luyia" + }, + "rah": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Kochic" + }, + "rai": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Kandas-Duke of York" + }, + "rak": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "West Manus", + "level8": "West Manus II" + }, + "ral": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Northeastern Kuki-Chin", + "level5": "Thadoic" + }, + "ram": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Cerrado", + "level3": "Goyaz", + "level4": "Northern Je", + "level5": "Eastern Timbira", + "level6": "Southeastern Timbira" + }, + "ran": { + "level0": "Kolopom", + "level1": "Kimaama-Riantana" + }, + "rao": { + "level0": "Ramu" + }, + "rap": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Distal", + "level13": "Far East Polynesian" + }, + "rar": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Proximal", + "level13": "Southern East Polynesian Proximal" + }, + "ras": { + "level0": "Rashad" + }, + "rat": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Southern Tatic", + "level10": "Ramand-Karaj" + }, + "rau": { + "level0": "Sino-Tibetan", + "level1": "Raji-Raute", + "level2": "Raute-Rawat" + }, + "rav": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Khambu" + }, + "raw": { + "level0": "Sino-Tibetan", + "level1": "Nungish" + }, + "rax": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Mumuye-Yandang", + "level6": "Mumuyic" + }, + "ray": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Proximal", + "level13": "Southern East Polynesian Proximal" + }, + "raz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Western Bungku-Tolaki", + "level8": "West Coast Bungku-Tolaki" + }, + "rbb": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "West Palaungic" + }, + "rcf": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil", + "level13": "Central Oil", + "level14": "Macro-French" + }, + "rdb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Caspian", + "level8": "Gilaki-Rudbari" + }, + "rea": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Mindjim", + "level4": "Upper Minjim" + }, + "reb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Manggaraiic" + }, + "ree": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Kayan-Murik-Modang", + "level4": "Kayan-Murik", + "level5": "Kayanic", + "level6": "Rejang-Makaham Kayan" + }, + "reg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Suguti" + }, + "rei": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Macro-Oriya" + }, + "rej": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian" + }, + "rel": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Eastern Omo-Tana" + }, + "rem": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Headwaters Pano" + }, + "ren": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric", + "level3": "Hre-Sedang-Todrah-Monam", + "level4": "Hre-Sedang" + }, + "rer": { + "level0": "Unattested" + }, + "res": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Kainji Lake" + }, + "ret": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "West Alor-Straits-Pantar", + "level4": "Kaera-Straits", + "level5": "Blagaric" + }, + "rey": { + "level0": "Pano-Tacanan", + "level1": "Tacanan", + "level2": "Takanik-Chamik", + "level3": "Takanik" + }, + "rga": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "South Santo" + }, + "rge": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Graeco-Phrygian", + "level3": "Greek", + "level4": "South Greek", + "level5": "Central Greek", + "level6": "Koineic Greek", + "level7": "Modern Koineic Greek", + "level8": "Nuclear Modern Greek" + }, + "rgk": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Eastern West Himalayish", + "level4": "Pithauragarh" + }, + "rgn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Italian", + "level12": "Emiliano-Romagnolo" + }, + "rgr": { + "level0": "Arawakan", + "level1": "Japura-Colombia", + "level2": "Nuclear Japura-Colombia", + "level3": "Caqueta" + }, + "rgs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic", + "level4": "Chamic", + "level5": "Chru-Northern Cham", + "level6": "Chruic" + }, + "rgu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Rote-Meto", + "level4": "Nuclear Rote", + "level5": "Central East Rote", + "level6": "Southeast Rote" + }, + "rhg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga" + }, + "rhp": { + "level0": "Nuclear Torricelli", + "level1": "Nuclear Maimai", + "level2": "Heyo-Yahang" + }, + "ria": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Boroic", + "level4": "Dimasa-Kokborok", + "level5": "Tipperic" + }, + "rib": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "rie": { + "level0": "Bookkeeping" + }, + "rif": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Greater Zenatic", + "level3": "Zenatic" + }, + "ril": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "West Palaungic", + "level4": "Riang" + }, + "rim": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Nyaturu-Nilamba" + }, + "rin": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Ninzic" + }, + "rir": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Bidayuh-Southern Land Dayak", + "level4": "Southern Land Dayak" + }, + "rit": { + "level0": "Pama-Nyungan", + "level1": "Yuulngu", + "level2": "Southern Yolngu" + }, + "riu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Manggaraiic", + "level6": "Manggarai Khusus" + }, + "rjb": { + "level0": "Bookkeeping" + }, + "rjg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Manggaraiic", + "level6": "Waerana-Razong" + }, + "rji": { + "level0": "Sino-Tibetan", + "level1": "Raji-Raute" + }, + "rjs": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Kamrupa", + "level10": "Kamta", + "level11": "Western Kamta" + }, + "rka": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "South Bahnaric", + "level4": "Mnong-Stieng-Chrau", + "level5": "Mnong", + "level6": "Southern-Central Mnong" + }, + "rkb": { + "level0": "Nuclear-Macro-Je" + }, + "rkh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Proximal", + "level13": "Southern East Polynesian Proximal" + }, + "rki": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Southern Burmish", + "level5": "Mranmaic", + "level6": "Nuclear Mranmaic", + "level7": "Arakanese-Marma" + }, + "rkm": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "East Manding" + }, + "rkt": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Kamrupa", + "level10": "Kamta" + }, + "rkw": { + "level0": "Bookkeeping" + }, + "rma": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Votic Chibchan" + }, + "rmb": { + "level0": "Gunwinyguan", + "level1": "Jala" + }, + "rmc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Romani" + }, + "rmd": { + "level0": "Speech Register", + "level1": "Indo-European Speech Register" + }, + "rme": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Romani", + "level10": "Anglo-Northwestern Romani", + "level11": "British Romani" + }, + "rmf": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Romani", + "level10": "Anglo-Northwestern Romani", + "level11": "Northwestern Romani" + }, + "rmg": { + "level0": "Speech Register", + "level1": "Indo-European Speech Register", + "level2": "Scandinavian Romani" + }, + "rmh": { + "level0": "Lepki-Murkim-Kembra" + }, + "rmi": { + "level0": "Speech Register", + "level1": "Indo-European Speech Register" + }, + "rmk": { + "level0": "Ramu", + "level1": "Goam", + "level2": "Tamolan", + "level3": "Breri-Romkun" + }, + "rml": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Romani" + }, + "rmm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Luangic-Kisaric", + "level5": "Kisaric" + }, + "rmn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Romani" + }, + "rmo": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Romani", + "level10": "Anglo-Northwestern Romani", + "level11": "Northwestern Romani" + }, + "rmp": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Rempic" + }, + "rmq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Romani" + }, + "rmr": { + "level0": "Bookkeeping" + }, + "rms": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Central European Sign" + }, + "rmt": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone" + }, + "rmu": { + "level0": "Speech Register", + "level1": "Indo-European Speech Register", + "level2": "Scandinavian Romani" + }, + "rmv": { + "level0": "Artificial Language" + }, + "rmw": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Romani", + "level10": "Anglo-Northwestern Romani", + "level11": "British Romani" + }, + "rmx": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric", + "level3": "Lamamic" + }, + "rmy": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Romani" + }, + "rmz": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Southern Burmish", + "level5": "Mranmaic", + "level6": "Nuclear Mranmaic", + "level7": "Arakanese-Marma" + }, + "rna": { + "level0": "Unattested", + "level1": "Chocoan (Unattested)" + }, + "rnb": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "rnd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Ruund-Salampasu", + "level11": "Lunda-Ruund-Kete", + "level12": "Ruund-Kete" + }, + "rng": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Nguni-Tsonga-Copi", + "level11": "Tsonga-Copi", + "level12": "Tswa-Ronga (S.50)", + "level13": "Tsongan" + }, + "rnl": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin" + }, + "rnn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Biakic", + "level6": "Biak-Roon" + }, + "rnp": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Eastern West Himalayish", + "level4": "Central-Eastern West Himalayish" + }, + "rnw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Mwika", + "level10": "Fipaic", + "level11": "Maluwawaru" + }, + "rob": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Torajic" + }, + "roc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic", + "level4": "Chamic", + "level5": "Chru-Northern Cham", + "level6": "Chruic" + }, + "rod": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Shiroro-Kamuku", + "level6": "Kamuku-Hungwarya", + "level7": "Kamuku", + "level8": "Rogo-Sagamuk-Sama-Sambuga" + }, + "roe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Bel-Roinji-Nenaya" + }, + "rof": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Kilimanjaro-Taita", + "level9": "Kilimanjaro Bantu", + "level10": "Chaga" + }, + "rog": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Aceh-Chamic", + "level4": "Chamic", + "level5": "Chru-Northern Cham", + "level6": "Northern Cham" + }, + "roh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian" + }, + "rol": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Central Bisayan" + }, + "ron": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Eastern Romance", + "level8": "Northern Romanian", + "level9": "Eastern Romanian" + }, + "roo": { + "level0": "North Bougainville", + "level1": "Rotokas-Askopan" + }, + "rop": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Pacific Creole English" + }, + "ror": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Central Flores-Paluqe", + "level6": "Central Flores", + "level7": "Ngada" + }, + "rou": { + "level0": "Maban", + "level1": "Mabang", + "level2": "Runga-Kibet" + }, + "row": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Rote-Meto", + "level4": "West Rote" + }, + "rpt": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Unclassified Hanseman" + }, + "rri": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Choiseul", + "level10": "East Choiseul" + }, + "rro": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "West Central Papuan linkage", + "level9": "Nuclear West Central Papuan linkage" + }, + "rrt": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Northern Pama" + }, + "rsi": { + "level0": "Artificial Language" + }, + "rsl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "RSLic", + "level3": "Nuclear RSLic", + "level4": "Central RSLic" + }, + "rsm": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "rsn": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Unclassified L1 Sign Language" + }, + "rsw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Kauru", + "level9": "Voric" + }, + "rtc": { + "level0": "Bookkeeping", + "level1": "Pending Report Release" + }, + "rth": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sangiric", + "level3": "Southern Sangiric" + }, + "rtm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage" + }, + "rtw": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil", + "level8": "Rathawi-Palya" + }, + "rub": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu" + }, + "ruc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "Rutara", + "level11": "North Rutara" + }, + "rue": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "East Slavic", + "level5": "Ukrainian-Rusyn" + }, + "ruf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "East Ruvu" + }, + "rug": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "New Georgia", + "level10": "East New Georgia", + "level11": "Rovianic" + }, + "ruh": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Kochic" + }, + "rui": { + "level0": "Bookkeeping" + }, + "ruk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Ninzic", + "level5": "Rukubic" + }, + "run": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "West Highlands Kivu", + "level12": "Rundic" + }, + "ruo": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Eastern Romance", + "level8": "Northern Romanian" + }, + "rup": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Eastern Romance" + }, + "ruq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Eastern Romance", + "level8": "Northern Romanian", + "level9": "Eastern Romanian" + }, + "rus": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "East Slavic" + }, + "rut": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Lezgic", + "level3": "Samur", + "level4": "Western Samur" + }, + "ruu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Paitanic", + "level7": "Upper Kinabatangan-Lobu" + }, + "ruy": { + "level0": "Unattested", + "level1": "Atlantic-Congo (Unattested)" + }, + "ruz": { + "level0": "Unattested", + "level1": "Atlantic-Congo (Unattested)" + }, + "rwa": { + "level0": "Sko", + "level1": "Skou-Serra-Piore", + "level2": "Serra Hills", + "level3": "Rawo-Main Serra" + }, + "rwk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Kilimanjaro-Taita", + "level9": "Kilimanjaro Bantu", + "level10": "Chaga", + "level11": "West Kilimanjaro" + }, + "rwm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Terrien", + "level11": "Ngombe-Ababuan", + "level12": "Ababuan", + "level13": "Old Bomokandian", + "level14": "Komoic" + }, + "rwo": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Gusap-Mot", + "level4": "Ufim-Rawa-Nahu" + }, + "rwr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Mewati-Gojri" + }, + "rws": { + "level0": "Bookkeeping" + }, + "rxd": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Ngumpin-Yapa", + "level3": "Ngumpin", + "level4": "Western Ngumpin" + }, + "rxw": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Central Karnic", + "level3": "Mithaka-Karuwali" + }, + "ryn": { + "level0": "Japonic", + "level1": "Ryukyuan", + "level2": "Northern Ryukyuan", + "level3": "Amami", + "level4": "Nuclear Amami", + "level5": "Oshima" + }, + "rys": { + "level0": "Japonic", + "level1": "Ryukyuan", + "level2": "Southern Ryukyu", + "level3": "Macro-Yaeyama" + }, + "ryu": { + "level0": "Japonic", + "level1": "Ryukyuan", + "level2": "Northern Ryukyuan", + "level3": "Okinawa" + }, + "rzh": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Sayhadic", + "level5": "Modern Sayhadic" + }, + "saa": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.3", + "level5": "Sokoroic", + "level6": "Saba-Sokoro-Tamki" + }, + "sab": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Isthmic Chibchan", + "level3": "Eastern Isthmic Chibchan", + "level4": "Guaymiic" + }, + "sac": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian", + "level4": "Fox" + }, + "sae": { + "level0": "Nambiquaran" + }, + "saf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Northwest Oti-Volta", + "level13": "Safaliba-Dagaare" + }, + "sag": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Ngbandi-Mongoba-Kazibati", + "level6": "Ngbandic", + "level7": "Nuclear Ngbandic", + "level8": "Sangoic" + }, + "sah": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Sakha-Dolgan" + }, + "saj": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Sahuan", + "level3": "Nuclear Sahuan", + "level4": "Sahu-Waioli" + }, + "sak": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Ngomic", + "level8": "Nuclear Ngomic", + "level9": "Sake-Ndambomo" + }, + "san": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan" + }, + "sap": { + "level0": "Bookkeeping" + }, + "saq": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Lotuxo-Maa", + "level4": "Ongamo-Maa", + "level5": "Nuclear Maa" + }, + "sar": { + "level0": "Arawakan", + "level1": "Central-Eastern Maipuran", + "level2": "Central Maipuran", + "level3": "Xaray" + }, + "sas": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bali-Sasak-Sumbawa", + "level3": "Sasak-Sumbawa" + }, + "sat": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Santalic" + }, + "sau": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Three Rivers", + "level4": "Amalumute", + "level5": "Northwest Seram" + }, + "sav": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Cangin", + "level3": "Saafi-Noon-Lehar" + }, + "saw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Unclassified Awyu-Dumut" + }, + "sax": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "South Pentecost" + }, + "say": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Southwest South Bauchi", + "level7": "Zakse-Saya" + }, + "saz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Gujaratic" + }, + "sba": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Central", + "level6": "Sara Central Logone" + }, + "sbb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "New Georgia", + "level10": "West New Georgia", + "level11": "Simboic" + }, + "sbc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "East Manus", + "level8": "Kurti-Kele-Ere" + }, + "sbd": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Bisa-Busa", + "level3": "Samo-Busa", + "level4": "Mande Samo" + }, + "sbe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "Suauic", + "level8": "Suau chain" + }, + "sbg": { + "level0": "West Bird's Head", + "level1": "Seget-Moi" + }, + "sbh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "West Manus", + "level8": "West Manus I" + }, + "sbi": { + "level0": "Nuclear Torricelli", + "level1": "West Wapei" + }, + "sbj": { + "level0": "Maban", + "level1": "Mabang", + "level2": "Maba-Masalit", + "level3": "Macro-Masalit" + }, + "sbk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Mbeya" + }, + "sbl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Luzon", + "level3": "Sambalic", + "level4": "Abellen-Botolan" + }, + "sbm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "West Ruvu", + "level11": "Vidunda-Sagala" + }, + "sbn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Sindhic", + "level9": "Unclassified Sindhic" + }, + "sbo": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "Senoic", + "level4": "Lanoh-Semnam-Temiar", + "level5": "Lanoh-Semnam", + "level6": "Lanohic" + }, + "sbp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Southern Tanzania Highlands Bantu", + "level9": "Wanji-Sangu" + }, + "sbq": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Sogeram", + "level5": "North Sogeram" + }, + "sbr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Eastern Murutic", + "level8": "Selungai-Sembakung Murut" + }, + "sbs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Botatwe", + "level8": "Western Botatwe", + "level9": "Machili" + }, + "sbu": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Lahauli-Spiti" + }, + "sbw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "B10-B30", + "level8": "Okani (B.30)", + "level9": "Northern Okani", + "level10": "Himba-Pinji" + }, + "sbx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Ibanic", + "level5": "Iban-Mualang-Seberuang", + "level6": "Iban-Seberuang" + }, + "sby": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Botatwe", + "level8": "Greater Eastern Botatwe" + }, + "sbz": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Saraic", + "level5": "Sara Peripherique", + "level6": "Barh Keita", + "level7": "Sara-Kaba" + }, + "sca": { + "level0": "Bookkeeping" + }, + "scb": { + "level0": "Austroasiatic", + "level1": "Vietic", + "level2": "Chutic", + "level3": "East Chutic" + }, + "sce": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Southern Periphery Mongolic", + "level3": "Shirongol", + "level4": "Baoanic" + }, + "scg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Bidayuh-Southern Land Dayak", + "level4": "Southern Land Dayak" + }, + "sch": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin", + "level5": "Mizoic", + "level6": "Hmaric" + }, + "sci": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay" + }, + "sck": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Magadhan", + "level10": "Sadanic", + "level11": "Sadri-Panchpargania", + "level12": "India-Nepal-Bangladesh Sadri" + }, + "scl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Shinaic" + }, + "scn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Italo-Dalmatian", + "level9": "Italian Romance" + }, + "sco": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic" + }, + "scp": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Kyirong-Kagate", + "level9": "Yolmo-Kagate" + }, + "scq": { + "level0": "Austroasiatic", + "level1": "Pearic", + "level2": "Western Pearic", + "level3": "Southern Chong" + }, + "scs": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Northwestern Canada Athabaskan", + "level4": "Slaveyic", + "level5": "Slave" + }, + "sct": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "West Katuic", + "level3": "Brou-So", + "level4": "Eastern Bru-Katang", + "level5": "Katang" + }, + "scu": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Western West Himalayish", + "level4": "Kinnauric", + "level5": "Thebor" + }, + "scv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos" + }, + "scw": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.4", + "level5": "Ronic" + }, + "scx": { + "level0": "Unclassifiable" + }, + "sda": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Torajic" + }, + "sdb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Gorani", + "level9": "Shabak-Bajalani" + }, + "sdc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Italian" + }, + "sde": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Kauru", + "level9": "Voric" + }, + "sdg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Shinaic", + "level8": "Western Shinaic" + }, + "sdh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Laki-Kurdish", + "level8": "Kurdish" + }, + "sdi": { + "level0": "Bookkeeping" + }, + "sdj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Nuclear Northern Kikongo" + }, + "sdk": { + "level0": "Ndu", + "level1": "Nuclear Ndu", + "level2": "Sawosic", + "level3": "Iatmulic" + }, + "sdl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Arab Sign" + }, + "sdm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Bidayuh-Southern Land Dayak", + "level4": "Southern Land Dayak" + }, + "sdn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Southern Romance", + "level8": "Sardo-Corsican", + "level9": "Corsic" + }, + "sdo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Bidayuh-Southern Land Dayak", + "level4": "Bidayuh" + }, + "sdp": { + "level0": "Sino-Tibetan", + "level1": "Kho-Bwa", + "level2": "Western Kho-Bwa", + "level3": "Sartang-Sherdukpen" + }, + "sdr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Magadhan", + "level10": "Sadanic", + "level11": "Sadri-Panchpargania", + "level12": "India-Nepal-Bangladesh Sadri" + }, + "sds": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Greater Zenatic", + "level3": "Zenatic", + "level4": "Zuara-Sened" + }, + "sdt": { + "level0": "Bookkeeping" + }, + "sdu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio", + "level5": "Uma-Sarudu" + }, + "sdx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Melanau-Kajang", + "level5": "Melanau", + "level6": "Sibu-Kanowit-Tanjong" + }, + "sea": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "Senoic" + }, + "sec": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Central Salish", + "level3": "North Georgia Central Salish" + }, + "sed": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric", + "level3": "Hre-Sedang-Todrah-Monam", + "level4": "Hre-Sedang" + }, + "see": { + "level0": "Iroquoian", + "level1": "Northern Iroquoian" + }, + "sef": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo" + }, + "seg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Mijikenda-Pokomo-Comorian", + "level11": "Mijikenda", + "level12": "Southern Mijikenda" + }, + "seh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Tumbuka-Sena-Nyanja", + "level8": "Sena-Nyanja", + "level9": "Senaic" + }, + "sej": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Eastern Huon", + "level4": "Trans Vitiaz", + "level5": "Huon Tip" + }, + "sek": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Northwestern Canada Athabaskan", + "level4": "Cordillera Athabaskan", + "level5": "Beaver-Sekani" + }, + "sel": { + "level0": "Uralic", + "level1": "Samoyedic", + "level2": "Kamas-Selkup" + }, + "sen": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "North Senufo" + }, + "sep": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "North Senufo", + "level5": "Supyiric" + }, + "seq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "North Senufo" + }, + "ser": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Californian Uto-Aztecan", + "level3": "Serran" + }, + "ses": { + "level0": "Songhay", + "level1": "Eastern Songhay" + }, + "set": { + "level0": "Sentanic", + "level1": "Nuclear Sentanic", + "level2": "Sentani-Nafri" + }, + "seu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Central-Western Yapen", + "level7": "Central Yapen", + "level8": "Serui-Busami" + }, + "sev": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "South Senufo" + }, + "sew": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Dobu-Duau linkage" + }, + "sey": { + "level0": "Tucanoan", + "level1": "Western Tucanoan", + "level2": "Napo Tucanoan", + "level3": "Siona-Secoya" + }, + "sez": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Maraic" + }, + "sfb": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Dutch-Belgian Sign", + "level4": "Belgian Sign" + }, + "sfe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Subanen", + "level4": "Nuclear Subanen", + "level5": "East Nuclear Subanen" + }, + "sfm": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Nuclear Hmongic", + "level4": "West Hmongic", + "level5": "Greater Chuanqiandian", + "level6": "Chuanqiandian" + }, + "sfs": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "BSLic", + "level3": "South African Sign" + }, + "sfw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Central Tano", + "level7": "Bia", + "level8": "Northern Bia" + }, + "sga": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Insular Celtic", + "level6": "Goidelic" + }, + "sgb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Luzon", + "level3": "Sambalic", + "level4": "Mag-Ayta" + }, + "sgc": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Kalenjin", + "level3": "Central Kalenjin" + }, + "sgd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "South Bisayan", + "level6": "Surigao" + }, + "sge": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Kayan-Murik-Modang", + "level4": "Modang-Segai" + }, + "sgg": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "DGSic" + }, + "sgh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Eastern Iranian", + "level5": "Shughni-Yazgulami", + "level6": "Shughnic" + }, + "sgi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute" + }, + "sgk": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Bisoid" + }, + "sgm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "North Nyanza" + }, + "sgo": { + "level0": "Bookkeeping" + }, + "sgp": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Jingpho-Luish", + "level3": "Jingpho" + }, + "sgr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Komisenian" + }, + "sgt": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Southern Tibetic" + }, + "sgu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "East Seram", + "level4": "Setic" + }, + "sgw": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Outer South Ethiopic", + "level6": "TT-Group" + }, + "sgx": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic" + }, + "sgy": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Sanglechi-Ishkashimi" + }, + "sgz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage" + }, + "sha": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Beromic" + }, + "shb": { + "level0": "Yanomamic", + "level1": "Ninam-Yanomam-Yaroame" + }, + "shc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Northern Njila", + "level9": "Mbala-Holu-Sondi (K.10)", + "level10": "Mbala-Sondi" + }, + "shd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Shinaic" + }, + "she": { + "level0": "Dizoid" + }, + "shg": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Non-Khoekhoe", + "level3": "Ost-Kxoe" + }, + "shh": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Numic", + "level3": "Central Numic" + }, + "shi": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Kabyle-Atlas Berber", + "level3": "Atlas Berber" + }, + "shj": { + "level0": "Dajuic", + "level1": "Eastern Dajuic" + }, + "shk": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Northern Lwoo" + }, + "shl": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Maraic", + "level5": "Nuclear Maraic" + }, + "shm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Central Tatic", + "level10": "Khalkhalic" + }, + "shn": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Southern Shanic" + }, + "sho": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Bisa-Busa", + "level3": "Samo-Busa", + "level4": "Busan", + "level5": "Kyenga-Shanga" + }, + "shp": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Chama subgroup", + "level5": "Shipibo-Konibo-Kapanawa" + }, + "shq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Botatwe", + "level8": "Greater Eastern Botatwe", + "level9": "Central Eastern Botatwe", + "level10": "Kafue" + }, + "shr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "Forest Kivu" + }, + "shs": { + "level0": "Salishan", + "level1": "Interior Salish", + "level2": "Northern Interior Salish", + "level3": "Thompsonic" + }, + "sht": { + "level0": "Shastan", + "level1": "Nuclear Shastan" + }, + "shu": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Egyptic Arabic", + "level7": "Egypto-Sudanic Arabic", + "level8": "Sudanese-Chadian Arabic" + }, + "shv": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Modern South Arabian", + "level4": "Eastern MSA" + }, + "shw": { + "level0": "Heibanic", + "level1": "West-Central Heibanic" + }, + "shx": { + "level0": "Hmong-Mien", + "level1": "Hmongic", + "level2": "Nuclear Hmongic-Ho Ne", + "level3": "Jiongnai-Ho Ne", + "level4": "Ho Neic" + }, + "shy": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Greater Zenatic", + "level3": "Zenatic" + }, + "sia": { + "level0": "Uralic", + "level1": "Saami", + "level2": "Eastern Saami", + "level3": "Mainland Eastern Saami" + }, + "sib": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Kenyahic", + "level5": "Lowland Kenyah", + "level6": "Western Lowland Kenyah-Penan", + "level7": "Penan", + "level8": "Western Penan-Sebop" + }, + "sic": { + "level0": "Bookkeeping" + }, + "sid": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Highland East Cushitic", + "level4": "Sidaama-Hadiyya-Kambaata", + "level5": "Sidaama-Gedeo" + }, + "sie": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Greater Luyana", + "level8": "Western Greater Luyana", + "level9": "Simaaic" + }, + "sig": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Western Grusi", + "level9": "Sisaala-Chakali", + "level10": "Sisaala" + }, + "sih": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Southern New Caledonian", + "level8": "Mid-Southern New Caledonian" + }, + "sij": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf" + }, + "sik": { + "level0": "Bookkeeping" + }, + "sil": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Western Grusi", + "level9": "Sisaala-Chakali", + "level10": "Sisaala" + }, + "sim": { + "level0": "Sepik", + "level1": "Nukuma", + "level2": "Kwanga-Mende" + }, + "sin": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Dhivehi-Sinhala", + "level6": "Sinhalaic" + }, + "sip": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Southern Tibetic", + "level7": "Dzongkhic" + }, + "siq": { + "level0": "Bosavi", + "level1": "Bosavi Watershed", + "level2": "Kaluli-Sunia" + }, + "sir": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.2", + "level5": "Nuclear West Chadic B.2", + "level6": "Western West Chadic B.2" + }, + "siu": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Wapeic", + "level4": "Galu-Alu" + }, + "siv": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Central Sepik Hill", + "level3": "Nuclear Central Sepik Hill" + }, + "siw": { + "level0": "South Bougainville", + "level1": "Buinic" + }, + "six": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Peka" + }, + "siy": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Central Iran Kermanic" + }, + "siz": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Greater Zenatic", + "level3": "Libyan-Egyptian Oases Berber" + }, + "sja": { + "level0": "Chocoan", + "level1": "Embera", + "level2": "San Juan" + }, + "sjb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Punan", + "level6": "Punan Tubu-Bah" + }, + "sjd": { + "level0": "Uralic", + "level1": "Saami", + "level2": "Eastern Saami", + "level3": "Peninsular Eastern Saami" + }, + "sje": { + "level0": "Uralic", + "level1": "Saami", + "level2": "Western Saami", + "level3": "Central Western Saami", + "level4": "Lule-Pite Saami" + }, + "sjg": { + "level0": "Tamaic", + "level1": "Tama-Sungor-Miisiirii", + "level2": "Tama-Sungor" + }, + "sjk": { + "level0": "Uralic", + "level1": "Saami", + "level2": "Eastern Saami", + "level3": "Mainland Eastern Saami" + }, + "sjl": { + "level0": "Sino-Tibetan", + "level1": "Miji" + }, + "sjm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Sama-Bajaw", + "level5": "Sulu-Borneo", + "level6": "Borneo Coast Bajaw" + }, + "sjn": { + "level0": "Artificial Language" + }, + "sjo": { + "level0": "Tungusic", + "level1": "Manchu-Jurchen", + "level2": "Manchu-Xibe" + }, + "sjp": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Kamrupa", + "level10": "Kamta", + "level11": "Western Kamta" + }, + "sjr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage" + }, + "sjs": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Kabyle-Atlas Berber", + "level3": "Atlas Berber", + "level4": "Northwestern Moroccan Berber" + }, + "sjt": { + "level0": "Uralic", + "level1": "Saami", + "level2": "Eastern Saami", + "level3": "Peninsular Eastern Saami" + }, + "sju": { + "level0": "Uralic", + "level1": "Saami", + "level2": "Western Saami", + "level3": "Southwestern Saami" + }, + "sjw": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Great Lakes Algonquian" + }, + "skb": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek" + }, + "skc": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Erap", + "level4": "Sauk-Nimi" + }, + "skd": { + "level0": "Miwok-Costanoan", + "level1": "Miwokan", + "level2": "Eastern Miwokan", + "level3": "Sierra Miwokan" + }, + "ske": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "South Pentecost", + "level7": "Seke-Sowa" + }, + "skf": { + "level0": "Tupian", + "level1": "Arikem-Tupari", + "level2": "Tuparic", + "level3": "Nuclear Tuparic", + "level4": "Corumbiara" + }, + "skg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "Southwestern Malagasic", + "level7": "South West-Central Malagasic", + "level8": "Nuclear South West-Central Malagasic", + "level9": "Inland-Western Malagasic", + "level10": "Western Malagasic" + }, + "skh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran", + "level3": "Central Barrier Islands" + }, + "ski": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata" + }, + "skj": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Tamangic" + }, + "skl": { + "level0": "Bookkeeping" + }, + "skm": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Uruwa", + "level4": "Sakam-Som" + }, + "skn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Subanen", + "level4": "Western Subanen" + }, + "sko": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "Rampi-Seko-Badaic", + "level4": "Seko" + }, + "skp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Melanau-Kajang", + "level5": "Kajang" + }, + "skq": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Soninke-Bozo", + "level4": "Soninkean" + }, + "skr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Greater Panjabic", + "level9": "Hindko-Siraiki", + "level10": "Siraikic" + }, + "sks": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kaukombaran" + }, + "skt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Kwa-Kasai North", + "level15": "Sakata-Tiinic" + }, + "sku": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "South Santo" + }, + "skv": { + "level0": "Sko", + "level1": "Skou-Serra-Piore", + "level2": "Skouic" + }, + "skw": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch", + "level6": "Middle-Modern Dutch", + "level7": "Modern Dutch", + "level8": "Southwestern Dutch", + "level9": "Zeeuwic" + }, + "skx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "Rampi-Seko-Badaic", + "level4": "Seko" + }, + "sky": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian" + }, + "skz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tanimbar-Bomberai", + "level3": "Nuclear Tanimbar-Bomberai", + "level4": "Yamdena-Onin", + "level5": "Oninic" + }, + "slb": { + "level0": "Bookkeeping" + }, + "slc": { + "level0": "Saliban" + }, + "sld": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Western Grusi", + "level9": "Sisaala-Chakali", + "level10": "Sisaala", + "level11": "Northwestern Sisaala" + }, + "sle": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Badaga-Kannada", + "level5": "Kannadoid" + }, + "slf": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Italian Sign" + }, + "slg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Eastern Murutic", + "level8": "Selungai-Sembakung Murut" + }, + "slh": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Central Salish", + "level3": "Lushootseed-Puget" + }, + "sli": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Middle German", + "level7": "East Middle German", + "level8": "Schlesisch-Wilmesau" + }, + "slj": { + "level0": "Bookkeeping" + }, + "slk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "West Slavic", + "level5": "Czech-Slovak" + }, + "sll": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Simbu", + "level3": "Nuclear Simbu", + "level4": "Golinic" + }, + "slm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Sama-Bajaw", + "level5": "Sulu-Borneo" + }, + "slp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Lembata", + "level4": "Lamaholot Barat", + "level5": "Flores Lamaholot" + }, + "slq": { + "level0": "Bookkeeping" + }, + "slr": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Oghuz" + }, + "sls": { + "level0": "Bookkeeping" + }, + "slt": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Sila-Wanya-Cosao" + }, + "slu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "South Tanimbar" + }, + "slv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "South Slavic", + "level5": "Western South Slavic" + }, + "slw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Eastern Huon", + "level4": "Kalasa" + }, + "slx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Chokwe-Lunda", + "level10": "Ruund-Salampasu" + }, + "sly": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Makassaric" + }, + "slz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "Maya-Matbat", + "level6": "Raja Ampat Maya" + }, + "sma": { + "level0": "Uralic", + "level1": "Saami", + "level2": "Western Saami", + "level3": "Southwestern Saami" + }, + "smb": { + "level0": "Angan", + "level1": "Nuclear Angan", + "level2": "Baruya-Simbari" + }, + "smc": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Uruwa", + "level4": "Sakam-Som" + }, + "sme": { + "level0": "Uralic", + "level1": "Saami", + "level2": "Western Saami", + "level3": "Central Western Saami" + }, + "smf": { + "level0": "Border", + "level1": "Warisic", + "level2": "Nuclear Warisic", + "level3": "Simog-Daonda" + }, + "smg": { + "level0": "Baining" + }, + "smh": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nuclear Nisoid", + "level8": "Nasu-Nosu", + "level9": "Nesu-Nasu", + "level10": "Nasu-Gepu" + }, + "smj": { + "level0": "Uralic", + "level1": "Saami", + "level2": "Western Saami", + "level3": "Central Western Saami", + "level4": "Lule-Pite Saami" + }, + "smk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Luzon", + "level3": "Sambalic", + "level4": "Tina-Bolinao" + }, + "sml": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Sama-Bajaw", + "level5": "Sulu-Borneo", + "level6": "Inner Sulu Sama" + }, + "smm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Tharuic", + "level10": "Unclassified Tharu" + }, + "smn": { + "level0": "Uralic", + "level1": "Saami", + "level2": "Eastern Saami", + "level3": "Mainland Eastern Saami" + }, + "smo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Ellicean", + "level9": "Pukapukic", + "level10": "Samoan-Tokelauan" + }, + "smp": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Canaanite", + "level6": "Hebrewic" + }, + "smq": { + "level0": "East Strickland", + "level1": "Kubo-Samo-Bibo" + }, + "smr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sumatran" + }, + "sms": { + "level0": "Uralic", + "level1": "Saami", + "level2": "Eastern Saami", + "level3": "Mainland Eastern Saami" + }, + "smt": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Northeastern Kuki-Chin", + "level5": "Sizangic" + }, + "smu": { + "level0": "Austroasiatic", + "level1": "Pearic", + "level2": "Western Pearic" + }, + "smv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Southern zone", + "level7": "Marathic", + "level8": "Marathi-Konkani", + "level9": "Old-Modern Marathi", + "level10": "Modern Marathi" + }, + "smw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bali-Sasak-Sumbawa", + "level3": "Sasak-Sumbawa" + }, + "smx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Hungan-Samba" + }, + "smy": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian" + }, + "smz": { + "level0": "South Bougainville", + "level1": "Nasioiic", + "level2": "Nasioi", + "level3": "Simekuic" + }, + "sna": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Shona (S.10)", + "level9": "Core Shona", + "level10": "Plateau Shona", + "level11": "Central Shona" + }, + "snc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "Sinagoro-Keapara" + }, + "snd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Sindhic", + "level9": "Sindhi-Kachchi" + }, + "sne": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Bidayuh-Southern Land Dayak", + "level4": "Bidayuh", + "level5": "Central-Western Bidayuh" + }, + "snf": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Cangin", + "level3": "Saafi-Noon-Lehar", + "level4": "Noon-Lehar" + }, + "sng": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Luba-Kaonde", + "level9": "Kaonde-Shaba-Sanga" + }, + "snh": { + "level0": "Unattested", + "level1": "Pano-Tacanan (Unattested)" + }, + "sni": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Chama subgroup" + }, + "snj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Ngbandi-Mongoba-Kazibati", + "level6": "Ngbandic", + "level7": "Nuclear Ngbandic", + "level8": "Sangoic" + }, + "snk": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Soninke-Bozo", + "level4": "Soninkean" + }, + "snl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sangiric", + "level3": "Northern Sangiric", + "level4": "Sangil-Sangir" + }, + "snm": { + "level0": "Central Sudanic", + "level1": "Moru-Madi", + "level2": "Southern Moru-Madi" + }, + "snn": { + "level0": "Tucanoan", + "level1": "Western Tucanoan", + "level2": "Napo Tucanoan", + "level3": "Siona-Secoya", + "level4": "Sionan" + }, + "snp": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Siane-Yagaria" + }, + "snq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "West Kikongo", + "level20": "Vilic", + "level21": "Lumbuic", + "level22": "Ngubi-Sangu-Sira-Punu", + "level23": "Sangu-Sira-Punu", + "level24": "Sangu-Sira" + }, + "snr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Gum" + }, + "sns": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Peripheral Western Malakula", + "level9": "Southwestern Malakula", + "level10": "Southwest Coastal Malekula" + }, + "snu": { + "level0": "Border", + "level1": "Warisic", + "level2": "Nuclear Warisic" + }, + "snv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Dayic" + }, + "snw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Na-Togo", + "level4": "Lelemic", + "level5": "Likpe-Santrokofi" + }, + "snx": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Mindjim", + "level4": "Lower Minjim", + "level5": "Inland Minjim" + }, + "sny": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Western Sepik Hill" + }, + "snz": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Evapia", + "level4": "Nuclear Evapia" + }, + "soa": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Chiang Saeng", + "level10": "Black Tai" + }, + "sob": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Sarmi", + "level8": "Sobeic", + "level9": "Sobei-Liki" + }, + "soc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke", + "level11": "So-Poke", + "level12": "So-Lebonya", + "level13": "Basoo" + }, + "sod": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega", + "level8": "Mituku-Lega", + "level9": "Songola-Binja" + }, + "soe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Nkutsuic", + "level11": "Nkutsu-Lokenye", + "level12": "Songomenic" + }, + "sog": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Sogdic-Ossetic", + "level6": "Sogdic", + "level7": "Sogdian-Yagnobi" + }, + "soh": { + "level0": "Eastern Jebel", + "level1": "Aka-Kelo-Molo" + }, + "soi": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Tharuic", + "level10": "Eastern Tharu", + "level11": "Dangaura-Khuna-Sonaha" + }, + "soj": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Central Iran Kermanic", + "level8": "Nuclear Central Iran Kermanic", + "level9": "Kashanic" + }, + "sok": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.3", + "level5": "Sokoroic", + "level6": "Saba-Sokoro-Tamki" + }, + "sol": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "North Bougainville Oceanic" + }, + "som": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Eastern Omo-Tana" + }, + "soo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Nsong-Mpiin-Ngong" + }, + "sop": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Luba-Kaonde" + }, + "soq": { + "level0": "Dagan", + "level1": "Southeast Dagan" + }, + "sor": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.1", + "level5": "Sumrayic", + "level6": "Sarwa-Sumray" + }, + "sos": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Northwestern Mande", + "level3": "Duun-Bobo", + "level4": "Duun-Jo", + "level5": "Duun-Seenku" + }, + "sot": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Sotho-Tswana (S.30)", + "level11": "Western Sotho-Tswana", + "level12": "Central Sotho-Tswana", + "level13": "Sesotho-Lozi" + }, + "sou": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Thai PH", + "level9": "Lao-Thai" + }, + "sov": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Western Trukic", + "level10": "Sonsorol-Tobi" + }, + "sow": { + "level0": "Border", + "level1": "Warisic", + "level2": "Nuclear Warisic", + "level3": "Waina-Punda" + }, + "sox": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Western A80", + "level10": "Makaaic", + "level11": "Southern Makaaic" + }, + "soy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Unclassified North Volta-Congo" + }, + "soz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Central Kenya Bantu", + "level9": "Gikuyu-Temi" + }, + "spa": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Castilic" + }, + "spb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Eastern Littoral Piru Bay" + }, + "spd": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Greater Yaganon", + "level4": "Yaganon", + "level5": "Ganglau-Saep" + }, + "spe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Kairiru-Manam", + "level8": "Manamic linkage", + "level9": "Bam-Manam", + "level10": "Manam-Sepa" + }, + "spg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Punan", + "level6": "Aput-Busang-Merah-Kohi" + }, + "spi": { + "level0": "Lakes Plain", + "level1": "Far West Lakes Plain", + "level2": "Rasawa-Saponi" + }, + "spk": { + "level0": "Ndu", + "level1": "Nuclear Ndu", + "level2": "Sawosic" + }, + "spl": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Cromwell", + "level5": "Kabwum", + "level6": "Selepet-Komba" + }, + "spm": { + "level0": "Ramu", + "level1": "Lower Ramu", + "level2": "Ruboni", + "level3": "Mikarewan" + }, + "spn": { + "level0": "Lengua-Mascoy", + "level1": "Eastern Enlhet-Enenlhet" + }, + "spo": { + "level0": "Salishan", + "level1": "Interior Salish", + "level2": "Southern Interior Salish", + "level3": "Okanaganic", + "level4": "Kalispel-Spokane" + }, + "spp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "North Senufo", + "level5": "Supyiric" + }, + "spq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Castilic", + "level13": "South Castilic" + }, + "spr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua", + "level6": "Seram Straits", + "level7": "Uliase", + "level8": "Hatuhaha", + "level9": "Saparuan", + "level10": "Saparua-Latu" + }, + "sps": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "North Bougainville Oceanic", + "level10": "Nuclear North Bougainville Oceanic", + "level11": "Buka", + "level12": "Saposa-Tinputz" + }, + "spt": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Lahauli-Spiti", + "level7": "Spiti-Jad" + }, + "spu": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "Nuclear West Bahnaric" + }, + "spv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Macro-Oriya" + }, + "spy": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Kalenjin", + "level3": "Elgon-Mau Kalenjin" + }, + "sqa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Shiroro-Kamuku", + "level6": "Kamuku-Hungwarya", + "level7": "Kamuku", + "level8": "Rogo-Sagamuk-Sama-Sambuga", + "level9": "Sagamuk-Sama-Sambuga" + }, + "sqh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "Lameic" + }, + "sqk": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "sqm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gbaya-Manza-Ngbaka", + "level4": "Gbaya Meridional-Occidental", + "level5": "Bokoto-Gbeya", + "level6": "Gbeya", + "level7": "Gbeya-Suma" + }, + "sqn": { + "level0": "Iroquoian", + "level1": "Northern Iroquoian" + }, + "sqo": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Komisenian" + }, + "sqq": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "Nuclear West Bahnaric", + "level4": "Loven-Suq" + }, + "sqs": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "BSLic" + }, + "sqt": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Modern South Arabian", + "level4": "Eastern MSA" + }, + "squ": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Central Salish", + "level3": "South Georgia Central Salish" + }, + "sqx": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "sra": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Silopic" + }, + "srb": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "Sora-Juray-Gorum", + "level3": "Sora-Juray" + }, + "src": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Southern Romance", + "level8": "Sardo-Corsican", + "level9": "Sardinian" + }, + "sre": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Benyadu-Bekati", + "level4": "Bakati'", + "level5": "Rara-Sara Bakati'" + }, + "srf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Lower Markham", + "level9": "Busu", + "level10": "Musom-Sirak" + }, + "srg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "West Bisayan", + "level6": "Kinarayan" + }, + "srh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Eastern Iranian", + "level5": "Shughni-Yazgulami", + "level6": "Shughnic" + }, + "sri": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Western Eastern Tucanoan", + "level3": "Cubeo-Desano", + "level4": "Yupua-Siriano-Desano", + "level5": "Siriano-Desano" + }, + "srk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Paitanic" + }, + "srl": { + "level0": "Greater Kwerba" + }, + "srm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Surinamese Creole English" + }, + "srn": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Surinamese Creole English", + "level13": "Eastern Maroons" + }, + "sro": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Southern Romance", + "level8": "Sardo-Corsican", + "level9": "Sardinian" + }, + "srp": { + "level0": "Indo-European", + "level1": "Balto-Slavic" + }, + "srq": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup II", + "level7": "Warazu-Sirionoid", + "level8": "Sirionoid" + }, + "srr": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Fula-Sereer" + }, + "srs": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan" + }, + "srt": { + "level0": "Geelvink Bay", + "level1": "Barapasi-Sauri-Kofei", + "level2": "Sauri-Kofei" + }, + "sru": { + "level0": "Tupian", + "level1": "Monde" + }, + "srv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Central Bisayan", + "level6": "Warayan" + }, + "srw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Teun-Nila-Serua", + "level5": "Nila-Serua" + }, + "srx": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Nuclear Himachali" + }, + "sry": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Siau", + "level8": "Sissano-Tumleo", + "level9": "Sera-Sissano" + }, + "srz": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Caspian", + "level8": "Mazanderani-Shahmirzadi" + }, + "ssb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Sama-Bajaw", + "level5": "Sulu-Borneo", + "level6": "Inner Sulu Sama" + }, + "ssc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Nyanza Mara", + "level11": "North Mara", + "level12": "Kuriaic" + }, + "ssd": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Kabenau" + }, + "sse": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Sama-Bajaw", + "level5": "Sulu-Borneo", + "level6": "Inner Sulu Sama" + }, + "ssf": { + "level0": "Austronesian", + "level1": "Western Plains Austronesian" + }, + "ssg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Western Admiralty Islands" + }, + "ssh": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Arabian Peninsula Arabic", + "level7": "North Arabian Beduin Arabic", + "level8": "Dhofaric" + }, + "ssi": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Greater Panjabic", + "level9": "Eastern Panjabic" + }, + "ssj": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Evapia", + "level4": "Nuclear Evapia", + "level5": "Kesawai-Wia" + }, + "ssk": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Eastern West Himalayish", + "level4": "Central-Eastern West Himalayish" + }, + "ssl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Western Grusi", + "level9": "Sisaala-Chakali", + "level10": "Sisaala", + "level11": "Northwestern Sisaala" + }, + "ssm": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "Senoic", + "level4": "Lanoh-Semnam-Temiar", + "level5": "Lanoh-Semnam" + }, + "ssn": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Oromoid", + "level7": "Nuclear Oromo", + "level8": "Central-Eastern Oromo", + "level9": "South-East-North Oromo" + }, + "sso": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Siau", + "level8": "Sissano-Tumleo", + "level9": "Sera-Sissano", + "level10": "Sissanoic" + }, + "ssp": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Spanish Sign", + "level3": "Nuclear Spanish Sign" + }, + "ssr": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic" + }, + "sss": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "West Katuic", + "level3": "Brou-So", + "level4": "Western Bru-So" + }, + "sst": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Simbu", + "level3": "Nuclear Simbu", + "level4": "Golinic" + }, + "ssu": { + "level0": "Angan", + "level1": "Nuclear Angan", + "level2": "Wojokesic", + "level3": "Kamasa-Susuami" + }, + "ssv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "Shark Bayic" + }, + "ssw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Nguni-Tsonga-Copi", + "level11": "Nguni (S.40)", + "level12": "Nuclear Nguni", + "level13": "Southern Ndebele-Lowland", + "level14": "Swatic" + }, + "ssx": { + "level0": "Nuclear Trans New Guinea", + "level1": "Enga-Kewa-Huli", + "level2": "Kewa-Huli", + "level3": "Sau-Angal-Kewa" + }, + "ssy": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Saho-Afar" + }, + "ssz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Pasismanua" + }, + "sta": { + "level0": "Pidgin", + "level1": "Swahili-based pidgin", + "level2": "Upcountry Swahili" + }, + "stb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Subanen", + "level4": "Nuclear Subanen", + "level5": "East Nuclear Subanen" + }, + "stc": { + "level0": "Bookkeeping" + }, + "std": { + "level0": "Unattested" + }, + "ste": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "East Seram", + "level4": "Setic" + }, + "stf": { + "level0": "Nuclear Torricelli", + "level1": "West Wapei" + }, + "stg": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "Triengic" + }, + "sth": { + "level0": "Speech Register", + "level1": "Irish-English" + }, + "sti": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "South Bahnaric", + "level4": "Mnong-Stieng-Chrau", + "level5": "Stieng" + }, + "stj": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Bisa-Busa", + "level3": "Samo-Busa", + "level4": "Mande Samo" + }, + "stk": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Tonda" + }, + "stm": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Mountain Ok", + "level6": "Division A Mountain Ok", + "level7": "Tifal-Telefol", + "level8": "Tifalic", + "level9": "Faiwol-Seltaman" + }, + "stn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Longgu-Malaita-Makira", + "level6": "Malaita-Makira", + "level7": "Makira" + }, + "sto": { + "level0": "Siouan", + "level1": "Mississippi Valley", + "level2": "Dakotan", + "level3": "Nakoda" + }, + "stp": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tepiman", + "level3": "Tepehuan", + "level4": "Southern Tepehuan" + }, + "stq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Frisian" + }, + "str": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Central Salish", + "level3": "Straits Salish" + }, + "sts": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Gawarbatic", + "level5": "Shumashtic" + }, + "stt": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "South Bahnaric", + "level4": "Mnong-Stieng-Chrau", + "level5": "Stieng" + }, + "stu": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Waic", + "level5": "Bulangic" + }, + "stv": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Harari-East Gurage", + "level6": "Silte-Wolane" + }, + "stw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Nuclear Trukic", + "level10": "Central Trukic", + "level11": "Satawalese-Carolinian" + }, + "sty": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Northwest Kipchak", + "level5": "North Kipchak" + }, + "sub": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Yaka-Suku" + }, + "suc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Subanen", + "level4": "Western Subanen" + }, + "sue": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean", + "level3": "North Binanderean" + }, + "suf": { + "level0": "Bookkeeping" + }, + "sug": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Mountain Ok", + "level6": "Mianic" + }, + "suh": { + "level0": "Bookkeeping" + }, + "sui": { + "level0": "Suki-Gogodala" + }, + "suj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "West Highlands Kivu", + "level12": "Rundic", + "level13": "Hangaza-Shubi" + }, + "suk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Sukuma-Nyamwezi (F.20)", + "level9": "Nyamwezic" + }, + "sun": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian" + }, + "suo": { + "level0": "Sko", + "level1": "Skou-Serra-Piore", + "level2": "Barupu Lagoon" + }, + "suq": { + "level0": "Surmic", + "level1": "South Surmic", + "level2": "Southeast Surmic", + "level3": "Pastoral Surmic", + "level4": "Tirma-Chai-Mursi" + }, + "sur": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3" + }, + "sus": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Susu-Yalunka" + }, + "sut": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Tlapanec-Manguean", + "level3": "Subtiaba-Tlapanec" + }, + "suu": { + "level0": "Bookkeeping" + }, + "suv": { + "level0": "Sino-Tibetan", + "level1": "Kho-Bwa", + "level2": "Puroikic" + }, + "suw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Sukuma-Nyamwezi (F.20)" + }, + "suy": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Cerrado", + "level3": "Goyaz", + "level4": "Northern Je" + }, + "suz": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Northwestern Kiranti", + "level5": "Bahing-Sunwar" + }, + "sva": { + "level0": "Kartvelian" + }, + "svb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Siau" + }, + "svc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Vincent-Grenadian Creole" + }, + "sve": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar", + "level5": "South Babar", + "level6": "Masela-South Babar" + }, + "svk": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Central European Sign", + "level4": "Nuclear Central European Sign" + }, + "svm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "South Slavic", + "level5": "Western South Slavic" + }, + "svr": { + "level0": "Bookkeeping" + }, + "swa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid" + }, + "swb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Mijikenda-Pokomo-Comorian", + "level11": "Comorian Bantu", + "level12": "Shindzwani-Shimaore" + }, + "swc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Sabaki-Swahili", + "level11": "Swahili (G.40)", + "level12": "Mombasa-Lamu-Inland Swahili" + }, + "swe": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "North Germanic", + "level5": "North Scandinavian", + "level6": "East-Central Swedic", + "level7": "East Swedic" + }, + "swf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Sereic", + "level6": "Sere-Indri", + "level7": "Sere-Bviri", + "level8": "Ndogo-Sere", + "level9": "Tagbu-Sere" + }, + "swg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Alemannic", + "level10": "North Alemannic" + }, + "swh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Sabaki-Swahili", + "level11": "Swahili (G.40)", + "level12": "Mombasa-Lamu-Inland Swahili" + }, + "swi": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Kam-Sui", + "level3": "Then-MMS", + "level4": "Maonan-Mak-Sui" + }, + "swj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "West Kikongo", + "level20": "Vilic", + "level21": "Lumbuic", + "level22": "Ngubi-Sangu-Sira-Punu", + "level23": "Sangu-Sira-Punu", + "level24": "Sangu-Sira", + "level25": "Sira-Barama" + }, + "swk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Tumbuka-Sena-Nyanja", + "level8": "Sena-Nyanja", + "level9": "Senaic" + }, + "swl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Swedish Sign" + }, + "swm": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Wamas-Samosa-Murupi-Mosimo" + }, + "swn": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Greater Zenatic", + "level3": "Libyan-Egyptian Oases Berber" + }, + "swo": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Headwaters Pano", + "level5": "Yaminawa Complex" + }, + "swp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "Suauic", + "level8": "Suau chain" + }, + "swq": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic", + "level5": "Gudeic", + "level6": "Sharwa-Tsuvan" + }, + "swr": { + "level0": "Yawa-Saweru" + }, + "sws": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "South Tanimbar" + }, + "swt": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "East Alor", + "level3": "Sawila-Wersing" + }, + "swu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Gorontalo-Mongondow", + "level4": "Gorontalic" + }, + "swv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani" + }, + "sww": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "South Pentecost", + "level7": "Seke-Sowa" + }, + "swx": { + "level0": "Arawan" + }, + "swy": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.1", + "level5": "Sumrayic", + "level6": "Sarwa-Sumray" + }, + "sxb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Nyanza Mara", + "level11": "North Mara" + }, + "sxc": { + "level0": "Unclassifiable" + }, + "sxe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Ndasaic", + "level8": "Samayic" + }, + "sxg": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Naic" + }, + "sxk": { + "level0": "Kalapuyan" + }, + "sxm": { + "level0": "Bookkeeping" + }, + "sxn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sangiric", + "level3": "Northern Sangiric", + "level4": "Sangil-Sangir" + }, + "sxr": { + "level0": "Austronesian", + "level1": "Tsouic", + "level2": "Kanakanavu-Saaroa" + }, + "sxs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Igwic", + "level7": "Sasaru-Igwe" + }, + "sxu": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Middle German", + "level7": "East Middle German" + }, + "sxw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Western Phla-Phera" + }, + "sya": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "North West Greater Barito" + }, + "syb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Subanen", + "level4": "Nuclear Subanen", + "level5": "East Nuclear Subanen" + }, + "syc": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic" + }, + "syi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30" + }, + "syk": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara" + }, + "syl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga", + "level10": "Eastern Bengali" + }, + "sym": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Bisa-Busa", + "level3": "Samo-Busa", + "level4": "Mande Samo" + }, + "syo": { + "level0": "Austroasiatic", + "level1": "Pearic", + "level2": "Western Pearic", + "level3": "Southern Chong" + }, + "sys": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi" + }, + "syw": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Kyirong-Kagate", + "level9": "Yolmo-Kagate" + }, + "syx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Ndasaic", + "level8": "Samayic" + }, + "syy": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "sza": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "South Aslian", + "level3": "Semelai-Semaq" + }, + "szb": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok" + }, + "szc": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "South Aslian", + "level3": "Semelai-Semaq" + }, + "szd": { + "level0": "Bookkeeping" + }, + "sze": { + "level0": "Blue Nile Mao", + "level1": "West Mao", + "level2": "Hozo-Seze" + }, + "szg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Mongoic" + }, + "szl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "West Slavic", + "level5": "Lechitic", + "level6": "Polish-Silesian" + }, + "szn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "West Central Maluku", + "level3": "Sula-Buru" + }, + "szp": { + "level0": "Inanwatan" + }, + "szs": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "BSLic", + "level3": "BANZL", + "level4": "Auslanic" + }, + "szv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Sawabantu", + "level8": "Dualaic", + "level9": "Kole-Isubu" + }, + "szw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "South Halmahera", + "level6": "Central-Eastern South Halmahera" + }, + "szy": { + "level0": "Austronesian", + "level1": "East Formosan", + "level2": "Central East Formosan" + }, + "taa": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central Alaska-Yukon Athabaskan", + "level4": "Tanana-Tutchone", + "level5": "Tananaic" + }, + "tab": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Lezgic", + "level3": "Samur", + "level4": "Eastern Samur", + "level5": "Tabasaran-Aghul-Lezgi" + }, + "tac": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tarahumara-Guarijio", + "level3": "Tarahumaran" + }, + "tad": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "West Tariku" + }, + "tae": { + "level0": "Arawakan", + "level1": "Japura-Colombia", + "level2": "Nuclear Japura-Colombia", + "level3": "Northeast Japura-Colombia", + "level4": "Baniwa-Curripaco-Tariano" + }, + "taf": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup IV" + }, + "tag": { + "level0": "Rashad" + }, + "tah": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Northern Outlier Polynesian-East Polynesian", + "level9": "Solomons Northern Outlier Polynesian-East Polynesian", + "level10": "Central Northern Outlier Polynesian-East Polynesian", + "level11": "East Polynesian", + "level12": "East Polynesian Proximal", + "level13": "Southern East Polynesian Proximal", + "level14": "Tahitian-Austral" + }, + "taj": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Tamangic", + "level5": "Nuclear Tamang" + }, + "tak": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi East", + "level6": "Guruntumic", + "level7": "Tala-Sho-Zangwal", + "level8": "Tala-Zamwar" + }, + "tal": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.3", + "level6": "Goemaic", + "level7": "Talic" + }, + "tam": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid", + "level10": "Tamil-Paliyan" + }, + "tan": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.2-3", + "level5": "West Chadic A.2", + "level6": "Tangalic", + "level7": "Nuclear Tangalic", + "level8": "Tangale-Kwami-Kupto" + }, + "tao": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Batanic", + "level3": "Yami-Itbayat" + }, + "tap": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Sabi", + "level8": "Malungu-Central Sabi" + }, + "taq": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Tuareg", + "level3": "Southern Tuareg" + }, + "tar": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tarahumara-Guarijio", + "level3": "Tarahumaran" + }, + "tas": { + "level0": "Pidgin", + "level1": "French-based pidgin" + }, + "tat": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Northwest Kipchak", + "level5": "North Kipchak", + "level6": "Bashkiric" + }, + "tau": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central Alaska-Yukon Athabaskan", + "level4": "Tanana-Tutchone", + "level5": "Tananaic", + "level6": "Upper Tananaic" + }, + "tav": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Eastern Eastern Tucanoan", + "level3": "Eastern Eastern Tucanoan I", + "level4": "Bara-Tatuyo" + }, + "taw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "Kalam-Kobon", + "level4": "Etp-Ti Kalam" + }, + "tax": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.3", + "level5": "Sokoroic", + "level6": "Saba-Sokoro-Tamki" + }, + "tay": { + "level0": "Austronesian", + "level1": "Atayalic" + }, + "taz": { + "level0": "Narrow Talodi", + "level1": "Buram-Saraf", + "level2": "Acheron-Tocho" + }, + "tbb": { + "level0": "Bookkeeping" + }, + "tbc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Bel-Roinji-Nenaya", + "level9": "Bel", + "level10": "Western Bel" + }, + "tbe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Temotu", + "level5": "Utupua-Vanikoro", + "level6": "Utupua" + }, + "tbf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Tabar linkage" + }, + "tbg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Kainantu", + "level3": "Tairora" + }, + "tbh": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "New South Wales Pama-Nyungan", + "level3": "Yuin-Kuri", + "level4": "Yuin", + "level5": "Northern Costal Yuin" + }, + "tbi": { + "level0": "Eastern Jebel" + }, + "tbj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Tungak-Nalik" + }, + "tbk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Kalamian" + }, + "tbl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bilic", + "level3": "Tboli-Blaan" + }, + "tbm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Sereic", + "level6": "Sere-Indri", + "level7": "Sere-Bviri", + "level8": "Ndogo-Sere", + "level9": "Tagbu-Sere" + }, + "tbn": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Magdalenic", + "level3": "Southern Magdalenic", + "level4": "Tunebo" + }, + "tbo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Taupota linkage", + "level10": "Nuclear Taupota linkage" + }, + "tbp": { + "level0": "Lakes Plain", + "level1": "East Lakes Plain" + }, + "tbr": { + "level0": "Kadugli-Krongo", + "level1": "Central-Western Kadugli-Krongo", + "level2": "Krongo-Tumtum" + }, + "tbs": { + "level0": "Ramu", + "level1": "Goam", + "level2": "Ataitan" + }, + "tbt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "Forest Kivu" + }, + "tbu": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan" + }, + "tbw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Palawanic", + "level4": "Northern Palawanic" + }, + "tbx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Buang linkage" + }, + "tby": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Mainland North Halmaheran" + }, + "tbz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Oti-Volta Oriental", + "level10": "Waama-Tayari-Ditammari", + "level11": "Tayari-Ditammari", + "level12": "Ditammaric" + }, + "tca": { + "level0": "Ticuna-Yuri" + }, + "tcb": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central Alaska-Yukon Athabaskan", + "level4": "Tanana-Tutchone", + "level5": "Tananaic", + "level6": "Upper Tananaic" + }, + "tcc": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Tatoga-Omotik", + "level3": "Gemein Datooga", + "level4": "North-Central Datooga" + }, + "tcd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Ka-Togo", + "level4": "Avatime-Nyangbo", + "level5": "Nyangbo-Tafi" + }, + "tce": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central Alaska-Yukon Athabaskan", + "level4": "Tanana-Tutchone", + "level5": "Tutchone" + }, + "tcf": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Tlapanec-Manguean", + "level3": "Subtiaba-Tlapanec", + "level4": "Mephaa", + "level5": "North-Central Mephaa" + }, + "tcg": { + "level0": "Kayagaric", + "level1": "Kaygir-Tamagario" + }, + "tch": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Gullah-Nevis-Antigua", + "level15": "Gullah", + "level16": "Bahamian Gullah" + }, + "tci": { + "level0": "Yam", + "level1": "Morehead-Maro", + "level2": "Tonda", + "level3": "Eastern Tonda" + }, + "tck": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie", + "level17": "Northwest Kasai-Ngounie", + "level18": "Mbere (B.60)", + "level19": "Tsitsekeic" + }, + "tcl": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Jingpho-Luish", + "level3": "Luish", + "level4": "Unclassified Luish" + }, + "tcn": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Dolpo-Tichurong" + }, + "tco": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Southern Burmish", + "level5": "Mranmaic" + }, + "tcp": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Core Central Kuki-Chin" + }, + "tcq": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "East Tariku", + "level3": "Doutai-Kai-Waritai" + }, + "tcs": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Pacific Creole English", + "level12": "Early Melanesian Pidgin" + }, + "tct": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Kam-Sui", + "level3": "Then-MMS" + }, + "tcu": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tarahumara-Guarijio", + "level3": "Tarahumaran" + }, + "tcw": { + "level0": "Totonacan", + "level1": "Totonac", + "level2": "Central Totonacan", + "level3": "Northern Totonacan", + "level4": "Necaxan" + }, + "tcx": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda" + }, + "tcy": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "South-Western Dravidian", + "level4": "Tuluic" + }, + "tcz": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Northeastern Kuki-Chin", + "level5": "Thadoic" + }, + "tda": { + "level0": "Songhay", + "level1": "Northwest Songhay", + "level2": "Northern Songhay" + }, + "tdb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Magadhan", + "level10": "Sadanic", + "level11": "Sadri-Panchpargania" + }, + "tdc": { + "level0": "Chocoan", + "level1": "Embera", + "level2": "San Juan", + "level3": "Upper San Juan" + }, + "tdd": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Sukaphic", + "level11": "Northern Shanic" + }, + "tde": { + "level0": "Dogon", + "level1": "West Dogon" + }, + "tdf": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "Triengic" + }, + "tdg": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Tamangic", + "level5": "Nuclear Tamang" + }, + "tdh": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Thulung-Tilung-Koyi" + }, + "tdi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Western Bungku-Tolaki", + "level8": "Interior Bungku-Tolaki" + }, + "tdj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Tominic", + "level5": "Southern Tomini" + }, + "tdk": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic A", + "level4": "West Chadic A.4", + "level5": "Fyer-Tambas" + }, + "tdl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Tarokoid", + "level5": "Bijimic-Sur-Shall" + }, + "tdn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Minahasan", + "level3": "North Minahasan", + "level4": "Northeast Minahasan" + }, + "tdo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Mumuye-Yandang", + "level6": "Yandangic", + "level7": "Waka-Yendang-Teme" + }, + "tdq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Unclassified Benue-Congo" + }, + "tdr": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric", + "level3": "Hre-Sedang-Todrah-Monam" + }, + "tds": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "East Tariku", + "level3": "Doutai-Kai-Waritai" + }, + "tdt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Eastern Timor", + "level4": "Central Timoric A", + "level5": "Tetunic" + }, + "tdv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Alumic" + }, + "tdx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "Southwestern Malagasic" + }, + "tdy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Mangyan" + }, + "tea": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "Senoic", + "level4": "Lanoh-Semnam-Temiar" + }, + "teb": { + "level0": "Bookkeeping" + }, + "tec": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Kalenjin", + "level3": "Central Kalenjin", + "level4": "Plateau Central Kalenjin", + "level5": "Western Plateau Central Kalenjin" + }, + "ted": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Grebo-Aizi", + "level4": "Grebo", + "level5": "Ivorian Grebo", + "level6": "Tepo-Plapo" + }, + "tee": { + "level0": "Totonacan", + "level1": "Tepehua" + }, + "tef": { + "level0": "Austroasiatic", + "level1": "Nicobaric", + "level2": "Nuclear Nicobaric", + "level3": "Chowra-Teressa" + }, + "teg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie", + "level17": "Northwest Kasai-Ngounie", + "level18": "Mbere (B.60)" + }, + "teh": { + "level0": "Chonan", + "level1": "Continental Chonan" + }, + "tei": { + "level0": "Nuclear Torricelli", + "level1": "Kombio-Arapesh-Urat", + "level2": "Kombio-Yambes", + "level3": "Kombioic" + }, + "tek": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie" + }, + "tel": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Teluguic" + }, + "tem": { + "level0": "Atlantic-Congo", + "level1": "Mel", + "level2": "Northern Mel" + }, + "ten": { + "level0": "Tucanoan", + "level1": "Western Tucanoan", + "level2": "Koreguaje-Tama" + }, + "teo": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Teso-Turkana" + }, + "tep": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tepiman", + "level3": "Tepehuan", + "level4": "Southern Tepehuan" + }, + "teq": { + "level0": "Temeinic" + }, + "ter": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Bolivian Arawakan" + }, + "tes": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Javanesic", + "level3": "Modern Javanese" + }, + "tet": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Eastern Timor", + "level4": "Central Timoric A", + "level5": "Tetunic" + }, + "teu": { + "level0": "Kuliak", + "level1": "Ngangea-So" + }, + "tev": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tanimbar-Bomberai", + "level3": "Teor-Kur" + }, + "tew": { + "level0": "Kiowa-Tanoan", + "level1": "Tewa" + }, + "tex": { + "level0": "Surmic", + "level1": "South Surmic", + "level2": "Southwest Surmic", + "level3": "Didinga-Murle" + }, + "tey": { + "level0": "Kadugli-Krongo", + "level1": "Central-Western Kadugli-Krongo" + }, + "tez": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Western Berber" + }, + "tfi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Eastern Phla-Phera" + }, + "tfn": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Southern Alaskan Athabaskan" + }, + "tfo": { + "level0": "Geelvink Bay" + }, + "tfr": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Isthmic Chibchan", + "level3": "Western Isthmic Chibchan" + }, + "tft": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Ternatean" + }, + "tga": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Kilimanjaro-Taita", + "level9": "Taita-Sagalla" + }, + "tgb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Dusunic" + }, + "tgc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "Tungak-Nalik" + }, + "tgd": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.2", + "level5": "Nuclear West Chadic B.2" + }, + "tge": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Tamangic" + }, + "tgf": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Phobjib-Chali-Bumthangic", + "level4": "Chali-Bumthangic" + }, + "tgg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage" + }, + "tgh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Vincent-Grenadian Creole", + "level15": "Grenada-Tobago Creole" + }, + "tgi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Banoni-Piva" + }, + "tgj": { + "level0": "Sino-Tibetan", + "level1": "Macro-Tani", + "level2": "Tani", + "level3": "Pre-Western Tani", + "level4": "Western Tani", + "level5": "Subansiri", + "level6": "Bangni-Tagin" + }, + "tgk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Farsic", + "level9": "Eastern Farsic", + "level10": "Tajikic" + }, + "tgl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Tagalogic", + "level5": "Tagalog-Filipino" + }, + "tgn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "South Bisayan", + "level6": "Surigao" + }, + "tgo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Nimoa-Sudest" + }, + "tgp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "South Santo", + "level9": "Araki-Tangoa" + }, + "tgq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Dayic" + }, + "tgr": { + "level0": "Bookkeeping" + }, + "tgs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage" + }, + "tgt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Palawanic", + "level4": "Northern Palawanic", + "level5": "Batak-Central Tagbanwa" + }, + "tgu": { + "level0": "Ramu", + "level1": "Goam", + "level2": "Ataitan", + "level3": "Tangu-Igom" + }, + "tgv": { + "level0": "Bookkeeping" + }, + "tgw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "South Senufo", + "level5": "Tagbana-Jimini" + }, + "tgx": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Northwestern Canada Athabaskan", + "level4": "Cordillera Athabaskan", + "level5": "Nahanni" + }, + "tgy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Sereic", + "level6": "Sere-Indri", + "level7": "Indri-Togoyo" + }, + "tgz": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Alaya-Athima", + "level3": "Central Alaya-Athima" + }, + "tha": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Thai PH", + "level9": "Lao-Thai" + }, + "thc": { + "level0": "Bookkeeping" + }, + "thd": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Southwest Pama", + "level3": "Upper Southwest Paman" + }, + "the": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Tharuic", + "level10": "Eastern Tharu" + }, + "thf": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Newaric", + "level4": "Thangmi-Baram" + }, + "thh": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tarahumara-Guarijio", + "level3": "Tarahumaran", + "level4": "Unclassified Tarahumaran" + }, + "thk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Central Kenya Bantu", + "level9": "Eastern Kirinyaga", + "level10": "Northern Kirinyaga" + }, + "thl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Tharuic", + "level10": "Eastern Tharu", + "level11": "Dangaura-Khuna-Sonaha" + }, + "thm": { + "level0": "Austroasiatic", + "level1": "Vietic", + "level2": "Chutic" + }, + "thn": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "thp": { + "level0": "Salishan", + "level1": "Interior Salish", + "level2": "Northern Interior Salish", + "level3": "Thompsonic" + }, + "thq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Tharuic", + "level10": "Eastern Tharu" + }, + "thr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Tharuic" + }, + "ths": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Kaike-Ghale-Tamangic", + "level3": "Ghale-Tamangic", + "level4": "Tamangic", + "level5": "Gurungic", + "level6": "Thakali-Chantyal" + }, + "tht": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Northwestern Canada Athabaskan", + "level4": "Cordillera Athabaskan", + "level5": "Nahanni" + }, + "thu": { + "level0": "Nilotic", + "level1": "Western Nilotic", + "level2": "Lwoo", + "level3": "Northern Lwoo", + "level4": "Luwo-Thuri" + }, + "thv": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Tuareg" + }, + "thw": { + "level0": "Bookkeeping" + }, + "thx": { + "level0": "Bookkeeping" + }, + "thy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bikwin-Jen", + "level5": "Southern Bikwin-Jen", + "level6": "Jen" + }, + "thz": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Tuareg", + "level3": "Southern Tuareg" + }, + "tia": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Greater Zenatic", + "level3": "Zenatic", + "level4": "Northern Saharan Oasis Berber" + }, + "tic": { + "level0": "Heibanic", + "level1": "West-Central Heibanic", + "level2": "Western Heibanic" + }, + "tie": { + "level0": "Bookkeeping" + }, + "tif": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Mountain Ok", + "level6": "Division A Mountain Ok", + "level7": "Tifal-Telefol", + "level8": "Tifalic" + }, + "tig": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "Tigre-Dahalik" + }, + "tih": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic", + "level6": "Murutic", + "level7": "Northern Murutic", + "level8": "Lowland Murut" + }, + "tii": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Kwa-Kasai North", + "level15": "Sakata-Tiinic", + "level16": "Tiinic" + }, + "tij": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Thulung-Tilung-Koyi" + }, + "tik": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid" + }, + "til": { + "level0": "Salishan", + "level1": "Coast Salish" + }, + "tim": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Huon", + "level3": "Western Huon", + "level4": "Cromwell", + "level5": "Kabwum" + }, + "tin": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Avar-Andic-Tsezic", + "level3": "Andic", + "level4": "Bagvalal-Tindi" + }, + "tio": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "North Bougainville Oceanic", + "level10": "Nuclear North Bougainville Oceanic", + "level11": "Buka", + "level12": "Saposa-Tinputz", + "level13": "Tinputzic" + }, + "tip": { + "level0": "Greater Kwerba", + "level1": "Kwerba-Samarokena", + "level2": "Kwerbaic" + }, + "tiq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Tiefoic" + }, + "tir": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic" + }, + "tis": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Kalinga-Itneg", + "level7": "Kalinga", + "level8": "Masadiit" + }, + "tiu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Cagayan Valley", + "level4": "Ibanagic" + }, + "tiv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "Central Tivoid", + "level7": "Central Tivoid A", + "level8": "Tiv-Evand", + "level9": "Tiv-Iyive-Otanga" + }, + "tix": { + "level0": "Kiowa-Tanoan", + "level1": "Tiwa-Piro", + "level2": "Tiwa" + }, + "tiy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bilic" + }, + "tiz": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Southern Shanic", + "level11": "Wuding-Yuanyang Tai" + }, + "tja": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Bassa-Klao", + "level5": "Klao-Tajuasohn" + }, + "tjg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Barito-Mahakam" + }, + "tji": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Tujia" + }, + "tjj": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Northern Pama", + "level3": "Albatross Bay", + "level4": "Anguthimri-Yangathimri-Yuputhimri", + "level5": "Anguthimri-Yangathimri" + }, + "tjl": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Sukaphic", + "level11": "Mogaung" + }, + "tjn": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Jogo-Jeri", + "level6": "Jogo" + }, + "tjo": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Greater Zenatic", + "level3": "Zenatic", + "level4": "Northern Saharan Oasis Berber", + "level5": "Ouargli-Oued Righ" + }, + "tjp": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Wati", + "level3": "Unclassified Wati" + }, + "tjs": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Tujia" + }, + "tju": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda", + "level4": "Central Ngayarda" + }, + "tka": { + "level0": "Unattested" + }, + "tkb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Tharuic", + "level10": "Unclassified Tharu" + }, + "tkd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Timor", + "level3": "Kemak-Tukudede" + }, + "tke": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Chuwaboic" + }, + "tkf": { + "level0": "Unattested", + "level1": "Tupian (Unattested)" + }, + "tkg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "Southwestern Malagasic", + "level7": "South West-Central Malagasic" + }, + "tkk": { + "level0": "Bookkeeping" + }, + "tkl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Ellicean", + "level9": "Pukapukic", + "level10": "Samoan-Tokelauan" + }, + "tkn": { + "level0": "Japonic", + "level1": "Ryukyuan", + "level2": "Northern Ryukyuan", + "level3": "Amami", + "level4": "Nuclear Amami", + "level5": "Okinoerabu-Tokunoshima" + }, + "tkp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian" + }, + "tkq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Ogonoid", + "level5": "East Ogonoid", + "level6": "Tai-Kana" + }, + "tkr": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Lezgic", + "level3": "Samur", + "level4": "Western Samur" + }, + "tks": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Southern Tatic", + "level10": "Ramand-Karaj" + }, + "tkt": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Bihari", + "level9": "Tharuic", + "level10": "Eastern Tharu" + }, + "tku": { + "level0": "Totonacan", + "level1": "Totonac", + "level2": "Central Totonacan", + "level3": "Northern Totonacan", + "level4": "Necaxan" + }, + "tkv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Korap linkage" + }, + "tkw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Temotu", + "level5": "Utupua-Vanikoro" + }, + "tkx": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Tangko-Nakai" + }, + "tkz": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric" + }, + "tla": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tepiman", + "level3": "Tepehuan", + "level4": "Southern Tepehuan" + }, + "tlb": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Mainland North Halmaheran", + "level3": "Tobelo-Tugutil" + }, + "tlc": { + "level0": "Totonacan", + "level1": "Totonac" + }, + "tld": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Sangiric", + "level3": "Northern Sangiric" + }, + "tle": { + "level0": "Bookkeeping" + }, + "tlf": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Mountain Ok", + "level6": "Division A Mountain Ok", + "level7": "Tifal-Telefol" + }, + "tlg": { + "level0": "Namla-Tofanma" + }, + "tlh": { + "level0": "Artificial Language" + }, + "tli": { + "level0": "Athabaskan-Eyak-Tlingit" + }, + "tlj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "Rutara", + "level11": "North Rutara" + }, + "tlk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Eastern Bungku-Tolaki", + "level8": "East Coast Bungku-Tolaki" + }, + "tll": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Nkutsuic", + "level11": "Tetelaic" + }, + "tlm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo" + }, + "tln": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Torajic" + }, + "tlo": { + "level0": "Narrow Talodi", + "level1": "Buram-Saraf", + "level2": "Nding-Tasomi" + }, + "tlp": { + "level0": "Totonacan", + "level1": "Totonac", + "level2": "Central Totonacan" + }, + "tlq": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Angkuic", + "level5": "Southern Angkuic" + }, + "tlr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southeast Solomonic", + "level5": "Guadalcanal-Nggelic", + "level6": "Southeast Guadalcanal" + }, + "tls": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "East Santo", + "level9": "Southeast Santo" + }, + "tlt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Eastern Littoral Piru Bay" + }, + "tlu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Piru Bay", + "level4": "East Piru Bay", + "level5": "Solehua", + "level6": "Seram Straits", + "level7": "Ambonic", + "level8": "Northeast Ambon" + }, + "tlv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Saluan-Banggai", + "level6": "Taliaboic" + }, + "tlw": { + "level0": "Bookkeeping" + }, + "tlx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "West Manus", + "level8": "West Manus II", + "level9": "Likum-Levei" + }, + "tly": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Central Tatic" + }, + "tlz": { + "level0": "Bookkeeping" + }, + "tma": { + "level0": "Tamaic", + "level1": "Tama-Sungor-Miisiirii", + "level2": "Tama-Sungor" + }, + "tmb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Peripheral Western Malakula", + "level9": "Southwestern Malakula" + }, + "tmc": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.1", + "level5": "Sumrayic", + "level6": "Ndam-Tumak" + }, + "tmd": { + "level0": "Piawi" + }, + "tme": { + "level0": "Unattested" + }, + "tmf": { + "level0": "Lengua-Mascoy", + "level1": "Eastern Enlhet-Enenlhet" + }, + "tmg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Castilic", + "level13": "South Castilic", + "level14": "Ternate-Zamboanga-Cavite" + }, + "tmi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "Nuclear Santo", + "level8": "East Santo", + "level9": "Mafea-Tutuba" + }, + "tmj": { + "level0": "Greater Kwerba", + "level1": "Kwerba-Samarokena", + "level2": "Samarokena-Airoran" + }, + "tmk": { + "level0": "Bookkeeping" + }, + "tml": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Asmat-Kamoro", + "level3": "Asmat", + "level4": "Citak Asmat" + }, + "tmm": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Chiang Saeng", + "level10": "White Tai" + }, + "tmn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Tamanic-Bugis", + "level5": "Tamanic" + }, + "tmo": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "South Aslian", + "level3": "Semelai-Semaq" + }, + "tmq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Siau", + "level8": "Sissano-Tumleo", + "level9": "Ali-Tumleo" + }, + "tmr": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "North-Eastern Neo-Aramaic" + }, + "tms": { + "level0": "Katla-Tima" + }, + "tmt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "Cape Cumberland" + }, + "tmu": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "Central Tariku" + }, + "tmv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Ngiri", + "level10": "Ngiri Riverain Mongala", + "level11": "Motemboic" + }, + "tmw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Greater Riau-Johoric" + }, + "tmx": { + "level0": "Bookkeeping" + }, + "tmy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage" + }, + "tmz": { + "level0": "Cariban", + "level1": "Venezuelan Cariban", + "level2": "Mapoyo-Tamanaku" + }, + "tna": { + "level0": "Pano-Tacanan", + "level1": "Tacanan", + "level2": "Takanik-Chamik", + "level3": "Takanik" + }, + "tnb": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Magdalenic", + "level3": "Southern Magdalenic", + "level4": "Tunebo" + }, + "tnc": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "South Eastern Tucanoan" + }, + "tnd": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Magdalenic", + "level3": "Southern Magdalenic", + "level4": "Tunebo" + }, + "tnf": { + "level0": "Bookkeeping" + }, + "tng": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic A", + "level4": "East Chadic A.2", + "level5": "East Chadic A.2 2" + }, + "tnh": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kaukombaran" + }, + "tni": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea" + }, + "tnj": { + "level0": "Bookkeeping" + }, + "tnk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "Southern Vanuatu", + "level6": "Tanna", + "level7": "Southern Tanna" + }, + "tnl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "Southern Vanuatu", + "level6": "Tanna", + "level7": "Northern Tanna linkage" + }, + "tnm": { + "level0": "Sentanic", + "level1": "Nuclear Sentanic" + }, + "tnn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "Southern Vanuatu", + "level6": "Tanna", + "level7": "Northern Tanna linkage", + "level8": "Whitesands-North Tanna linkage" + }, + "tno": { + "level0": "Pano-Tacanan", + "level1": "Tacanan", + "level2": "Takanik-Chamik", + "level3": "Takanik", + "level4": "Araona-Toromono" + }, + "tnp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "Southern Vanuatu", + "level6": "Tanna", + "level7": "Northern Tanna linkage", + "level8": "Whitesands-North Tanna linkage" + }, + "tnq": { + "level0": "Arawakan", + "level1": "Caribbean Arawakan", + "level2": "Antillean Arawakan" + }, + "tnr": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Tenda", + "level3": "Bassari-Bedik-Bapen", + "level4": "Bedik-Bapen" + }, + "tns": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "St. Matthias" + }, + "tnt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Minahasan", + "level3": "North Minahasan" + }, + "tnu": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P" + }, + "tnv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Eastern zone", + "level7": "Oriya-Gauda-Kamrupa", + "level8": "Gauda-Kamrupa", + "level9": "Gauda-Banga", + "level10": "Southeastern Bengali" + }, + "tnw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Minahasan" + }, + "tnx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Temotu", + "level5": "Utupua-Vanikoro" + }, + "tny": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Unclassified Northeast Savanna Bantu", + "level9": "Bende-Tongwe" + }, + "tnz": { + "level0": "Austroasiatic", + "level1": "Aslian", + "level2": "Central-Northern Aslian", + "level3": "North Aslian", + "level4": "Maniq-Menraq-Batek", + "level5": "Maniqic" + }, + "tob": { + "level0": "Guaicuruan", + "level1": "Guaicuru del Sur", + "level2": "Qom", + "level3": "Pilaga-Toba" + }, + "toc": { + "level0": "Totonacan", + "level1": "Totonac", + "level2": "Central Totonacan", + "level3": "Lowland-Sierra Totonacan", + "level4": "Sierra Totonacan" + }, + "tod": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Southwest Mande", + "level4": "Mende-Loma", + "level5": "Loma" + }, + "toe": { + "level0": "Bookkeeping" + }, + "tof": { + "level0": "Eastern Trans-Fly" + }, + "tog": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Tumbuka-Sena-Nyanja", + "level8": "Tumbukic" + }, + "toh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Nguni-Tsonga-Copi", + "level11": "Tsonga-Copi", + "level12": "Chopi (S.60)" + }, + "toi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Botatwe", + "level8": "Greater Eastern Botatwe", + "level9": "Central Eastern Botatwe", + "level10": "Kafue" + }, + "toj": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Kanjobalan-Chujean", + "level4": "Chujean" + }, + "tok": { + "level0": "Artificial Language" + }, + "tol": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Pacific Coast Athabaskan", + "level4": "Oregon Athabaskan" + }, + "tom": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Minahasan", + "level3": "North Minahasan", + "level4": "Northeast Minahasan" + }, + "ton": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Tongic" + }, + "too": { + "level0": "Totonacan", + "level1": "Totonac", + "level2": "Central Totonacan", + "level3": "Northern Totonacan" + }, + "top": { + "level0": "Totonacan", + "level1": "Totonac", + "level2": "Central Totonacan", + "level3": "Lowland-Sierra Totonacan" + }, + "toq": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Teso-Turkana", + "level4": "Turkanic" + }, + "tor": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Central Bandaic", + "level8": "Central Core Bandaic" + }, + "tos": { + "level0": "Totonacan", + "level1": "Totonac", + "level2": "Central Totonacan", + "level3": "Lowland-Sierra Totonacan", + "level4": "Sierra Totonacan" + }, + "tot": { + "level0": "Bookkeeping" + }, + "tou": { + "level0": "Austroasiatic", + "level1": "Vietic", + "level2": "Cuoi" + }, + "tov": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Central Tatic", + "level10": "Taromic" + }, + "tow": { + "level0": "Kiowa-Tanoan" + }, + "tox": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Western Trukic", + "level10": "Sonsorol-Tobi" + }, + "toy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio" + }, + "toz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Unclassified Mbum" + }, + "tpa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Taupota linkage", + "level10": "Nuclear Taupota linkage", + "level11": "Eastern Taupota", + "level12": "Taupota-Waiema" + }, + "tpc": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Tlapanec-Manguean", + "level3": "Subtiaba-Tlapanec", + "level4": "Mephaa" + }, + "tpe": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Boroic", + "level4": "Dimasa-Kokborok", + "level5": "Tipperic" + }, + "tpf": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Sarmi", + "level8": "Kaptiau-Tarpia" + }, + "tpg": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "East Alor" + }, + "tpi": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Pacific Creole English", + "level12": "Early Melanesian Pidgin" + }, + "tpj": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup I", + "level7": "Tupi-Guarani Subgroup I.B", + "level8": "Chiriguanic" + }, + "tpl": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Tlapanec-Manguean", + "level3": "Subtiaba-Tlapanec", + "level4": "Mephaa", + "level5": "North-Central Mephaa", + "level6": "West-Central Mephaa" + }, + "tpm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Western Grusi", + "level9": "Sisaala-Chakali", + "level10": "Chakalic", + "level11": "Chakali-Tamprusi-Vagala", + "level12": "Chakali-Tamprusi" + }, + "tpn": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup III" + }, + "tpo": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Chiang Saeng", + "level10": "Red Tai", + "level11": "Tai Muong" + }, + "tpp": { + "level0": "Totonacan", + "level1": "Tepehua" + }, + "tpr": { + "level0": "Tupian", + "level1": "Arikem-Tupari", + "level2": "Tuparic", + "level3": "Nuclear Tuparic", + "level4": "Wayoro-Tupari" + }, + "tpt": { + "level0": "Totonacan", + "level1": "Tepehua" + }, + "tpu": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "West Bahnaric", + "level3": "Tampuon-Bahnar" + }, + "tpv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Nuclear Trukic", + "level10": "Central Trukic", + "level11": "Satawalese-Carolinian", + "level12": "Macro-Carolinian", + "level13": "Murilo-Fanapanges" + }, + "tpx": { + "level0": "Otomanguean", + "level1": "Western Otomanguean", + "level2": "Tlapanec-Manguean", + "level3": "Subtiaba-Tlapanec", + "level4": "Mephaa", + "level5": "North-Central Mephaa", + "level6": "West-Central Mephaa" + }, + "tpz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "North Bougainville Oceanic", + "level10": "Nuclear North Bougainville Oceanic", + "level11": "Buka", + "level12": "Saposa-Tinputz", + "level13": "Tinputzic" + }, + "tqb": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup IV", + "level6": "Tupi-Guarani Subgroup IV.B" + }, + "tql": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage" + }, + "tqm": { + "level0": "Doso-Turumsa" + }, + "tqn": { + "level0": "Sahaptian", + "level1": "Sahaptin", + "level2": "Southern Sahaptin" + }, + "tqo": { + "level0": "Eleman", + "level1": "Eastern Eleman" + }, + "tqp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage" + }, + "tqq": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Eastern Omo-Tana", + "level8": "Dabarre-Tunni" + }, + "tqr": { + "level0": "Narrow Talodi", + "level1": "Lumun-Torona" + }, + "tqt": { + "level0": "Totonacan", + "level1": "Totonac", + "level2": "Central Totonacan" + }, + "tra": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Kohistani", + "level8": "Unclassified Kohistani" + }, + "trb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Kairiru-Manam", + "level8": "Kairiruic linkage", + "level9": "Kaiep-Terebu" + }, + "trc": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Trique" + }, + "trd": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Mundaric" + }, + "tre": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru" + }, + "trf": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Barbados-Eustatius", + "level15": "Barbados-Trinidad" + }, + "trg": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "North-Eastern Neo-Aramaic", + "level11": "Trans-Zab" + }, + "trh": { + "level0": "Dagan" + }, + "tri": { + "level0": "Cariban", + "level1": "Guianan", + "level2": "Taranoan", + "level3": "Tiriyoan" + }, + "trj": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Dangla-Mabire-Birgit", + "level6": "Birgit-Mogum-Toram" + }, + "trl": { + "level0": "Unclassifiable" + }, + "trm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Nuristani", + "level4": "Nuristani Kalasha-Tregami" + }, + "trn": { + "level0": "Arawakan", + "level1": "Southern Maipuran", + "level2": "Bolivian Arawakan", + "level3": "Mojeno-Paunaca", + "level4": "Moje\u00f1o" + }, + "tro": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Northwestern Kuki-Chin", + "level4": "Kolhrengic", + "level5": "Tarao-Chothe" + }, + "trp": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Boroic", + "level4": "Dimasa-Kokborok", + "level5": "Tipperic" + }, + "trq": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Trique" + }, + "trs": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Trique" + }, + "trt": { + "level0": "Geelvink Bay", + "level1": "Burate-Wate" + }, + "tru": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Aramaic", + "level6": "Imperial-Middle-Modern Aramaic", + "level7": "Middle-Modern Aramaic", + "level8": "Eastern Aramaic", + "level9": "Central Eastern Aramaic", + "level10": "Turoyo-Mlahso" + }, + "trv": { + "level0": "Austronesian", + "level1": "Atayalic" + }, + "trw": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Kohistani", + "level8": "Dir-Swat Kohistani" + }, + "trx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Bidayuh-Southern Land Dayak", + "level4": "Bidayuh" + }, + "try": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Sukaphic", + "level11": "Unclassified Sukaphic" + }, + "trz": { + "level0": "Chapacuran", + "level1": "Moreic-Waric", + "level2": "Moreic" + }, + "tsa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie", + "level17": "Northwest Kasai-Ngounie", + "level18": "Nzebi-Laali-Yaa", + "level19": "Njebi (B.50)", + "level20": "Ndjavi A" + }, + "tsb": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Transversal Lowland East Cushitic", + "level6": "Dullay" + }, + "tsc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Nguni-Tsonga-Copi", + "level11": "Tsonga-Copi", + "level12": "Tswa-Ronga (S.50)" + }, + "tsd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Graeco-Phrygian", + "level3": "Greek", + "level4": "North Greek" + }, + "tse": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Italian Sign" + }, + "tsf": { + "level0": "Bookkeeping" + }, + "tsg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "South Bisayan", + "level6": "Butuan-Tausug" + }, + "tsh": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic", + "level5": "Gudeic", + "level6": "Sharwa-Tsuvan" + }, + "tsi": { + "level0": "Tsimshian" + }, + "tsj": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Tshanglic" + }, + "tsk": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Kham-Hor" + }, + "tsl": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai" + }, + "tsm": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "tsn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Sotho-Tswana (S.30)", + "level11": "Western Sotho-Tswana", + "level12": "Central Sotho-Tswana" + }, + "tso": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Nguni-Tsonga-Copi", + "level11": "Tsonga-Copi", + "level12": "Tswa-Ronga (S.50)", + "level13": "Tsongan" + }, + "tsp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Tusia" + }, + "tsq": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic" + }, + "tsr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "Southwest Santo" + }, + "tss": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "JSLic" + }, + "tst": { + "level0": "Songhay", + "level1": "Eastern Songhay" + }, + "tsu": { + "level0": "Austronesian", + "level1": "Tsouic" + }, + "tsv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "B10-B30", + "level8": "Okani (B.30)", + "level9": "Southern Okani" + }, + "tsw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Kambari-Cicipu", + "level6": "Kambaric", + "level7": "East Kambaric" + }, + "tsx": { + "level0": "Anim", + "level1": "Inland Gulf of Papua", + "level2": "West Inland Gulf of Papua" + }, + "tsy": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "tsz": { + "level0": "Tarascan" + }, + "tta": { + "level0": "Siouan", + "level1": "Ohio Valley Siouan" + }, + "ttb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Dakoid", + "level6": "Tiba-Dong" + }, + "ttc": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Mamean", + "level4": "Mamean" + }, + "tte": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "Suauic" + }, + "ttf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Sanaga-West Mbam (A.40)", + "level10": "West Mbam (A.40)" + }, + "ttg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Berawan-Lower Baram", + "level5": "Lower Baram", + "level6": "Central Lower Baram A" + }, + "tth": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "Ta'oihic", + "level3": "Ong-Ta'oih" + }, + "tti": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Jayapura Bay", + "level8": "Eastern Jayapura Bay" + }, + "ttj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "Rutara", + "level11": "North Rutara", + "level12": "Nkore-Kiga-Nyoro-Tooro", + "level13": "Nyoro-Tooro" + }, + "ttk": { + "level0": "Barbacoan", + "level1": "Coconucan" + }, + "ttl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Botatwe", + "level8": "Western Botatwe", + "level9": "Machili" + }, + "ttm": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Central Alaska-Yukon Athabaskan", + "level4": "Tanana-Tutchone", + "level5": "Tutchone" + }, + "ttn": { + "level0": "Pauwasi", + "level1": "Western Pauwasi" + }, + "tto": { + "level0": "Austroasiatic", + "level1": "Katuic", + "level2": "Ta'oihic", + "level3": "Ong-Ta'oih" + }, + "ttp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio", + "level5": "Pamona-Tombelala" + }, + "ttq": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Tuareg", + "level3": "Southern Tuareg" + }, + "ttr": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Teraic", + "level5": "Western Tera" + }, + "tts": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Thai PH", + "level9": "Lao-Thai", + "level10": "Sakon Nakhon" + }, + "ttt": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian", + "level7": "Farsic-Caucasian Tat", + "level8": "Caucasian Tat" + }, + "ttu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Mono-Uruavan" + }, + "ttv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "East Manus", + "level8": "Koro-Lele-Nali-Titan" + }, + "ttw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Kenyahic", + "level5": "Lowland Kenyah", + "level6": "Western Lowland Kenyah-Penan" + }, + "ttx": { + "level0": "Bookkeeping" + }, + "tty": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "East Tariku" + }, + "ttz": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Kyirong-Kagate" + }, + "tua": { + "level0": "Nuclear Torricelli", + "level1": "Marienberg", + "level2": "Mandi-Muniwara" + }, + "tub": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Californian Uto-Aztecan" + }, + "tuc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Ngero", + "level8": "Western Ngero", + "level9": "Tuam" + }, + "tue": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Eastern Eastern Tucanoan", + "level3": "Eastern Eastern Tucanoan II", + "level4": "Pisamira-Yuruti", + "level5": "Tuyuca-Yuruti" + }, + "tuf": { + "level0": "Chibchan", + "level1": "Core Chibchan", + "level2": "Magdalenic", + "level3": "Southern Magdalenic", + "level4": "Tunebo" + }, + "tug": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Adamawa Bua", + "level5": "Riverine Bua" + }, + "tuh": { + "level0": "Taulil-Butam" + }, + "tui": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Northern Mbum", + "level6": "Tupuri-Mundang-Mambai" + }, + "tuj": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Mainland North Halmaheran", + "level3": "Tobelo-Tugutil" + }, + "tuk": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Oghuz", + "level3": "Nuclear Oghuz", + "level4": "East Oghuz" + }, + "tul": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Tula-Longuda", + "level6": "Tula-Waja", + "level7": "Tulaic", + "level8": "Tula-Ma-Yebu", + "level9": "Nuclear Tulaic" + }, + "tum": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Tumbuka-Sena-Nyanja", + "level8": "Tumbukic" + }, + "tuo": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Eastern Eastern Tucanoan", + "level3": "Eastern Eastern Tucanoan I" + }, + "tuq": { + "level0": "Saharan", + "level1": "Western Saharan", + "level2": "Tebu" + }, + "tur": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Oghuz", + "level3": "Nuclear Oghuz", + "level4": "West Oghuz" + }, + "tus": { + "level0": "Iroquoian", + "level1": "Northern Iroquoian", + "level2": "Tuscarora-Nottoway" + }, + "tuu": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Pacific Coast Athabaskan", + "level4": "Oregon Athabaskan", + "level5": "Rogue River" + }, + "tuv": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Teso-Lotuxo-Maa", + "level3": "Teso-Turkana", + "level4": "Turkanic" + }, + "tux": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa" + }, + "tuy": { + "level0": "Nilotic", + "level1": "Southern Nilotic", + "level2": "Kalenjin", + "level3": "Central Kalenjin", + "level4": "Plateau Central Kalenjin" + }, + "tuz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Kirma-Tyurama" + }, + "tva": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Choiseul", + "level10": "West Choiseul" + }, + "tvd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Kambari-Cicipu", + "level6": "Kambaric", + "level7": "East Kambaric" + }, + "tve": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Teun-Nila-Serua" + }, + "tvi": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Southwest South Bauchi", + "level7": "Zeemic", + "level8": "Nuclear Zeemic" + }, + "tvk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Ambrym" + }, + "tvl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Ellicean" + }, + "tvm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar", + "level5": "South Babar", + "level6": "Southwest Babar" + }, + "tvn": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Burmish", + "level4": "Southern Burmish", + "level5": "Mranmaic", + "level6": "Nuclear Mranmaic" + }, + "tvo": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Ternatean" + }, + "tvs": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Pare-Taveta" + }, + "tvt": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "North Patkaian", + "level4": "Noctean", + "level5": "Tutsic" + }, + "tvu": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Sanaga-West Mbam (A.40)", + "level10": "West Mbam (A.40)", + "level11": "Mandi-Nyokon" + }, + "tvw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio", + "level5": "Greater Kaili" + }, + "tvy": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Luso-Asian Creole" + }, + "twa": { + "level0": "Salishan", + "level1": "Coast Salish", + "level2": "Central Salish" + }, + "twb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "South Mangyan", + "level4": "Buhid-Taubuid", + "level5": "Batangan" + }, + "twc": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.1", + "level5": "Ngizim-Southwestern Bade", + "level6": "Shira-Southwestern Bade", + "level7": "Shira" + }, + "twe": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "West Alor-Straits-Pantar", + "level4": "Pantar", + "level5": "Teiwa-Sar" + }, + "twf": { + "level0": "Kiowa-Tanoan", + "level1": "Tiwa-Piro", + "level2": "Tiwa", + "level3": "Taos-Picuris" + }, + "twg": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "West Alor-Straits-Pantar", + "level4": "Kaera-Straits", + "level5": "Blagaric" + }, + "twh": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Chiang Saeng", + "level10": "White Tai" + }, + "twl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Shona (S.10)", + "level9": "Core Shona", + "level10": "Plateau Shona", + "level11": "Central Shona" + }, + "twm": { + "level0": "Bookkeeping" + }, + "twn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Mambila-Mbongno", + "level10": "Mambila", + "level11": "Eastern Mambila" + }, + "two": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Sotho-Tswana (S.30)", + "level11": "Northern Sotho", + "level12": "Sepedic" + }, + "twp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Eastern Admiralty Islands", + "level6": "Manus", + "level7": "East Manus", + "level8": "Kurti-Kele-Ere" + }, + "twq": { + "level0": "Songhay", + "level1": "Northwest Songhay", + "level2": "Northern Songhay" + }, + "twr": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tarahumara-Guarijio", + "level3": "Tarahumaran", + "level4": "Unclassified Tarahumaran" + }, + "twt": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup IV", + "level6": "Tupi-Guarani Subgroup IV.B" + }, + "twu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Rote-Meto", + "level4": "Nuclear Rote", + "level5": "Central East Rote" + }, + "tww": { + "level0": "Walioic" + }, + "twx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Shona (S.10)", + "level9": "Core Shona", + "level10": "Plateau Shona" + }, + "twy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "North East Greater Barito" + }, + "txa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Dusunic", + "level6": "Paitanic" + }, + "txb": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Tokharian" + }, + "txc": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan" + }, + "txe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tolitoli" + }, + "txg": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic", + "level4": "Gyalrongic", + "level5": "West Gyalrongic", + "level6": "Horpa" + }, + "txh": { + "level0": "Indo-European", + "level1": "Unclassified Indo-European" + }, + "txi": { + "level0": "Cariban", + "level1": "Pekodian", + "level2": "Xinguan" + }, + "txj": { + "level0": "Saharan", + "level1": "Western Saharan", + "level2": "Kanuri-Kanembu", + "level3": "Kanembuic" + }, + "txm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Tominic", + "level5": "Northern Tomini" + }, + "txn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru" + }, + "txo": { + "level0": "Sino-Tibetan", + "level1": "Dhimal-Lhokpu-Toto" + }, + "txq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Rote-Meto", + "level4": "Nuclear Rote" + }, + "txr": { + "level0": "Unclassifiable" + }, + "txs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Minahasan", + "level3": "North Minahasan", + "level4": "Northeast Minahasan" + }, + "txt": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Asmat-Kamoro", + "level3": "Asmat", + "level4": "Citak Asmat" + }, + "txu": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Cerrado", + "level3": "Goyaz", + "level4": "Northern Je" + }, + "txx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Sabahan", + "level4": "Southwest Sabahan", + "level5": "Greater Murutic" + }, + "txy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "Southwestern Malagasic", + "level7": "South West-Central Malagasic", + "level8": "Nuclear South West-Central Malagasic", + "level9": "Inland-Western Malagasic", + "level10": "Bara-Tanosy" + }, + "tya": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Unclassified Rai Coast" + }, + "tye": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Bisa-Busa", + "level3": "Samo-Busa", + "level4": "Busan", + "level5": "Kyenga-Shanga" + }, + "tyh": { + "level0": "Austroasiatic", + "level1": "Khmuic", + "level2": "Phay-Pram", + "level3": "Pramic" + }, + "tyi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie" + }, + "tyj": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Chiang Saeng", + "level10": "Red Tai", + "level11": "Tai Muong" + }, + "tyl": { + "level0": "Bookkeeping" + }, + "tyn": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Ndeiram" + }, + "typ": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Alaya-Athima", + "level3": "Thaypanic" + }, + "tyr": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Chiang Saeng", + "level10": "Red Tai" + }, + "tys": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai" + }, + "tyt": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Chiang Saeng", + "level10": "Red Tai" + }, + "tyu": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Non-Khoekhoe", + "level3": "Ost-Kxoe", + "level4": "Tshwa Khoe" + }, + "tyv": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "South Siberian Turkic", + "level3": "Sayan-Yenisei Turkic", + "level4": "Sayan" + }, + "tyx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie" + }, + "tyy": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Tarokoid", + "level5": "Bijimic-Sur-Shall", + "level6": "Kwangic" + }, + "tyz": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai" + }, + "tza": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "tzb": { + "level0": "Bookkeeping" + }, + "tzc": { + "level0": "Bookkeeping" + }, + "tze": { + "level0": "Bookkeeping" + }, + "tzh": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Cholan-Tzeltalan", + "level4": "Tzeltalan" + }, + "tzj": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Quichean", + "level4": "Core Quichean", + "level5": "Cakchiquel-Tzutujil" + }, + "tzl": { + "level0": "Artificial Language" + }, + "tzm": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Kabyle-Atlas Berber", + "level3": "Atlas Berber" + }, + "tzn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Wetar-Atauro", + "level4": "Wetar", + "level5": "Perai-Tugun-Aputai" + }, + "tzo": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Western Mayan", + "level3": "Cholan-Tzeltalan", + "level4": "Tzeltalan" + }, + "tzs": { + "level0": "Bookkeeping" + }, + "tzt": { + "level0": "Bookkeeping" + }, + "tzu": { + "level0": "Bookkeeping" + }, + "tzx": { + "level0": "Lower Sepik", + "level1": "Karawarian" + }, + "tzz": { + "level0": "Bookkeeping" + }, + "uam": { + "level0": "Unclassifiable" + }, + "uan": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P" + }, + "uar": { + "level0": "Eleman", + "level1": "Eastern Eleman" + }, + "uba": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Bendic", + "level6": "Nuclear Bendic", + "level7": "Bukpic" + }, + "ubi": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.3" + }, + "ubm": { + "level0": "Bookkeeping" + }, + "ubr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Are linkage" + }, + "ubu": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Hagen", + "level3": "Aua-Gawil" + }, + "uby": { + "level0": "Abkhaz-Adyge" + }, + "uda": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "West Lower Cross", + "level7": "Oroic", + "level8": "Enwang-Uda" + }, + "ude": { + "level0": "Tungusic", + "level1": "Northeastern Tungusic", + "level2": "Central-Eastern Tungusic", + "level3": "Oroch-Udihe" + }, + "udg": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Irula-Muduga", + "level8": "Muduga-Palu" + }, + "udi": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Lezgic", + "level3": "Samur", + "level4": "Eastern Samur", + "level5": "Udi-Aghwan" + }, + "udj": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Ujir-Kola-Kompane" + }, + "udl": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Margi-Mandara-Mofu", + "level5": "Mofuic", + "level6": "Tokombere", + "level7": "Madaic" + }, + "udm": { + "level0": "Uralic", + "level1": "Permian" + }, + "udu": { + "level0": "Koman", + "level1": "Central Koman", + "level2": "Komo-Uduk" + }, + "ues": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Nuclear Muna-Buton", + "level8": "Munan", + "level9": "Munic", + "level10": "Western Munic" + }, + "ufi": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Gusap-Mot", + "level4": "Ufim-Rawa-Nahu" + }, + "uga": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Canaanite", + "level6": "Ugarito-Phoenician" + }, + "ugb": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Kuku-Wik-Ep", + "level5": "Kuku-Wik", + "level6": "Paman Kuku" + }, + "uge": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "New Georgia", + "level10": "East New Georgia" + }, + "ugh": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Dargwic" + }, + "ugn": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic" + }, + "ugo": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese" + }, + "ugy": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "uha": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Osse" + }, + "uig": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Turkestan", + "level4": "Modern Turkestan", + "level5": "Uyghuric" + }, + "uis": { + "level0": "South Bougainville", + "level1": "Buinic", + "level2": "Buin" + }, + "uiv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "Central Tivoid", + "level7": "Central Tivoid A", + "level8": "Tiv-Evand", + "level9": "Tiv-Iyive-Otanga" + }, + "uji": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Jilic-Eggonic", + "level5": "Jilic" + }, + "uka": { + "level0": "South Bird's Head Family" + }, + "ukg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Numugenan", + "level6": "Yarawata-Parawen-Ukuriguma" + }, + "ukh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Makaa-Kako (A.80-90)", + "level8": "Makaa-Njem (A.80)", + "level9": "Mpoic", + "level10": "Mpiemo-Ukhwejo" + }, + "ukl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "RSLic", + "level3": "Nuclear RSLic", + "level4": "Central RSLic" + }, + "ukp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Bendic", + "level6": "Nuclear Bendic", + "level7": "Bukpic" + }, + "ukq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross", + "level6": "Central Lower Cross", + "level7": "Efikic" + }, + "ukr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Balto-Slavic", + "level3": "Slavic", + "level4": "East Slavic", + "level5": "Ukrainian-Rusyn" + }, + "uks": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "uku": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Northwestern Edoid", + "level6": "Osse", + "level7": "Ukue-Ehueun" + }, + "ukv": { + "level0": "Nilotic", + "level1": "Eastern Nilotic", + "level2": "Barian", + "level3": "Nuclear Barian" + }, + "ukw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Igboid", + "level4": "Nuclear Igboid", + "level5": "Central-Northern Igbo" + }, + "uky": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Southwest Pama", + "level3": "Upper Southwest Paman" + }, + "ula": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Shiroro-Kamuku", + "level6": "Shiroro" + }, + "ulb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede", + "level7": "Eastern Ede", + "level8": "Southeastern Ede", + "level9": "Nuclear Yoruba" + }, + "ulc": { + "level0": "Tungusic", + "level1": "Central-Western Tungusic", + "level2": "Ulchaic" + }, + "uli": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Western Trukic" + }, + "ulk": { + "level0": "Eastern Trans-Fly" + }, + "ull": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "ulm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Pitu Ulunna Salu", + "level6": "Matangnga-Aralle-Tabulahan" + }, + "uln": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Upper Franconian", + "level10": "Global German" + }, + "ulu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Kenyahic", + "level5": "Highland Kenyah", + "level6": "Upper Pujungan" + }, + "ulw": { + "level0": "Misumalpan", + "level1": "Sumalpan", + "level2": "Sumuic" + }, + "uly": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Northwest South Bauchi", + "level7": "Polci-Luri", + "level8": "Polcic" + }, + "uma": { + "level0": "Sahaptian", + "level1": "Sahaptin", + "level2": "Southern Sahaptin" + }, + "umb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene" + }, + "umd": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Northeastern Pama", + "level4": "Umbindhamuic" + }, + "umg": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Lamalamic", + "level3": "Coastal Lamalamic" + }, + "umi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Punan", + "level6": "Bukat-Ukit-Beketan-Lugat-Lisum" + }, + "umm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "North-South Central Delta Cross", + "level7": "Ubaghara-Kohumono", + "level8": "Kohumonoic" + }, + "umn": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Patkaian", + "level3": "South Patkaian", + "level4": "Southeastern Patkaian" + }, + "umo": { + "level0": "Bororoan" + }, + "ump": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Northeastern Pama", + "level4": "Umpilaic" + }, + "ums": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Tominic", + "level5": "Southern Tomini" + }, + "umu": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Delawaran", + "level5": "Common Delaware" + }, + "una": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Watut" + }, + "une": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "North-Central Edoid", + "level6": "Afenmai-Bendel", + "level7": "Uneme-Yekhee" + }, + "ung": { + "level0": "Worrorran" + }, + "uni": { + "level0": "Sko", + "level1": "Skou-Serra-Piore", + "level2": "Barupu Lagoon" + }, + "unk": { + "level0": "Arawakan", + "level1": "Central-Eastern Maipuran", + "level2": "Central Maipuran", + "level3": "Xaray", + "level4": "Parecis-Nawe" + }, + "unm": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Delawaran", + "level5": "Common Delaware" + }, + "unn": { + "level0": "Pama-Nyungan", + "level1": "Ganaic" + }, + "unr": { + "level0": "Austroasiatic", + "level1": "Mundaic", + "level2": "North Munda", + "level3": "Kherwarian", + "level4": "Mundaric", + "level5": "Ho-Mundari" + }, + "unu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "Suauic" + }, + "unx": { + "level0": "Bookkeeping" + }, + "unz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Northern Kaili-Wolio", + "level5": "Greater Kaili", + "level6": "Common Kaili" + }, + "uok": { + "level0": "Bookkeeping" + }, + "uon": { + "level0": "Austronesian", + "level1": "Northwest Formosan" + }, + "upi": { + "level0": "Border", + "level1": "Warisic", + "level2": "Nuclear Warisic", + "level3": "Waina-Punda" + }, + "upv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Eastern Malakula linkage" + }, + "urb": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VIII", + "level6": "Guaja-Kaapor-Ava" + }, + "urc": { + "level0": "Giimbiyu", + "level1": "Urninganggic" + }, + "urd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Shaurasenic", + "level8": "Indo-Aryan Central zone", + "level9": "Western Hindi", + "level10": "Hindustani" + }, + "ure": { + "level0": "Uru-Chipaya" + }, + "urf": { + "level0": "Bookkeeping" + }, + "urg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Peka", + "level4": "Urigina-Danaru" + }, + "urh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Akpes-Edoid", + "level4": "Edoid", + "level5": "Southwestern Edoid" + }, + "uri": { + "level0": "Nuclear Torricelli" + }, + "urk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Greater Riau-Johoric", + "level6": "Northeastern Peninsular Malay" + }, + "url": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu" + }, + "urm": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Mountain Ok", + "level6": "Division A Mountain Ok", + "level7": "Tifal-Telefol", + "level8": "Tifalic" + }, + "urn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Tanimbar-Bomberai", + "level3": "Nuclear Tanimbar-Bomberai", + "level4": "Yamdena-Onin", + "level5": "Oninic" + }, + "uro": { + "level0": "Baining" + }, + "urp": { + "level0": "Unclassifiable" + }, + "urr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage" + }, + "urt": { + "level0": "Nuclear Torricelli", + "level1": "Kombio-Arapesh-Urat" + }, + "uru": { + "level0": "Tupian", + "level1": "Purubora-Ramarama", + "level2": "Ramarama" + }, + "urv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Mono-Uruavan" + }, + "urw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Peka" + }, + "urx": { + "level0": "Nuclear Torricelli", + "level1": "Marienberg", + "level2": "Elepi-Kamasau-Marienberg" + }, + "ury": { + "level0": "Tor-Orya" + }, + "urz": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VI", + "level6": "Kawahiva", + "level7": "Nuclear Kawahiva", + "level8": "Central Kawahiva", + "level9": "Amondava-Uru-Eu-Wau-Wau" + }, + "usa": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Kainantu", + "level3": "Gauwa", + "level4": "Auyana" + }, + "ush": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Shinaic", + "level8": "Kohistanic Shina" + }, + "usi": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Boroic", + "level4": "Dimasa-Kokborok", + "level5": "Tipperic" + }, + "usk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Lower Cross", + "level5": "Nuclear Lower Cross" + }, + "usp": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Quichean-Mamean", + "level3": "Greater Quichean" + }, + "usu": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Nuru", + "level4": "Erimaic" + }, + "uta": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Tivoid", + "level6": "Central Tivoid", + "level7": "Central Tivoid A", + "level8": "Tiv-Evand", + "level9": "Tiv-Iyive-Otanga" + }, + "ute": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Numic", + "level3": "Southern Numic" + }, + "utp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Temotu", + "level5": "Utupua-Vanikoro", + "level6": "Utupua" + }, + "utr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Idomoid", + "level4": "Akweya", + "level5": "Etulo-Idoma" + }, + "utu": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Silopic", + "level6": "Silopi-Utu" + }, + "uum": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Northwest Kipchak", + "level5": "West Kipchak", + "level6": "Crimean Tatar-Urum" + }, + "uur": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "Southern Vanuatu", + "level6": "Erromanga" + }, + "uuu": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Angkuic" + }, + "uve": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "Vanuatu-Loyalty Outliers" + }, + "uvh": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Erap" + }, + "uvl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Mengenic" + }, + "uwa": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Kuku-Wik-Ep", + "level5": "Kuku-Wik", + "level6": "Paman Kuku" + }, + "uya": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Agoi-Doko-Iyoniyong" + }, + "uzn": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Turkestan", + "level4": "Modern Turkestan", + "level5": "Uzbek" + }, + "uzs": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Turkestan", + "level4": "Modern Turkestan", + "level5": "Uzbek" + }, + "vaa": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "vae": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental", + "level3": "Nuclear SBB Occidental", + "level4": "Nduga-Luto" + }, + "vaf": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Southern Tatic", + "level10": "Vafsic" + }, + "vag": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "East-West Grusi", + "level8": "Western Grusi", + "level9": "Sisaala-Chakali", + "level10": "Chakalic", + "level11": "Chakali-Tamprusi-Vagala" + }, + "vah": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Southern zone", + "level7": "Marathic", + "level8": "Marathi-Konkani", + "level9": "Old-Modern Marathi", + "level10": "Modern Marathi" + }, + "vai": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Vai-Kono" + }, + "vaj": { + "level0": "Kxa", + "level1": "Ju-Kung" + }, + "val": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Buang linkage" + }, + "vam": { + "level0": "Sko", + "level1": "Skou-Serra-Piore", + "level2": "Skouic", + "level3": "Eastern Skouic", + "level4": "West Coast Skouic" + }, + "van": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "West Palai" + }, + "vao": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Northern Malakula", + "level8": "North Coast Malakula", + "level9": "Botovro-Vovo-Vao" + }, + "vap": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Northeastern Kuki-Chin", + "level5": "Sizangic", + "level6": "Gangte-Vaiphei" + }, + "var": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Tarahumara-Guarijio" + }, + "vas": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil", + "level8": "Vasave-Noiri" + }, + "vau": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke", + "level11": "So-Poke", + "level12": "So-Lebonya", + "level13": "Lebonya", + "level14": "Bantu D33", + "level15": "Vanuma-Nyali" + }, + "vav": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Southern zone", + "level7": "Marathic", + "level8": "Marathi-Konkani", + "level9": "Old-Modern Marathi", + "level10": "Modern Marathi" + }, + "vay": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Northwestern Kiranti" + }, + "vbb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar", + "level5": "South Babar", + "level6": "Masela-South Babar" + }, + "vec": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Italian" + }, + "ved": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Dhivehi-Sinhala", + "level6": "Sinhalaic" + }, + "vem": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Lamang-Hdi" + }, + "ven": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu" + }, + "veo": { + "level0": "Chumashan", + "level1": "Southern Chumashan", + "level2": "Central Chumashan" + }, + "vep": { + "level0": "Uralic", + "level1": "Finnic", + "level2": "Coastal Finnic", + "level3": "Neva", + "level4": "North Finnic", + "level5": "Ladogan", + "level6": "East Ladoga" + }, + "ver": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Northern Samba-Duru", + "level7": "Vere-Gimme", + "level8": "Vere" + }, + "vgr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Gujaratic", + "level10": "Western Gujaratic" + }, + "vgt": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Dutch-Belgian Sign", + "level4": "Belgian Sign" + }, + "vic": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "Caribbean English Creole", + "level13": "Eastern Caribbean Creole", + "level14": "Barbados-Eustatius" + }, + "vid": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "West Ruvu", + "level11": "Vidunda-Sagala" + }, + "vie": { + "level0": "Austroasiatic", + "level1": "Vietic", + "level2": "Viet-Muong" + }, + "vif": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "West Kikongo", + "level20": "Vilic" + }, + "vig": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur" + }, + "vin": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "Western Lakes Bantu", + "level10": "Kivu", + "level11": "West Highlands Kivu" + }, + "vis": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid" + }, + "vit": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Unclassified Narrow Grassfields" + }, + "viv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Bwaidoga linkage", + "level9": "Bwaidoka-Iduna" + }, + "vka": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda", + "level4": "Central Ngayarda", + "level5": "Ngarluma-Kariyarra" + }, + "vki": { + "level0": "Bookkeeping" + }, + "vkk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "South Sumatra Malay" + }, + "vkl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Eastern Bungku-Tolaki", + "level8": "East Coast Bungku-Tolaki" + }, + "vkm": { + "level0": "Kamakanan", + "level1": "Nuclear Kamakanan" + }, + "vkn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Unclassified Western Benue-Congo Plateau" + }, + "vko": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Western Bungku-Tolaki", + "level8": "West Coast Bungku-Tolaki" + }, + "vkp": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Southwestern Shifted Romance", + "level11": "West Ibero-Romance", + "level12": "Galician Romance", + "level13": "Macro-Portuguese", + "level14": "Indo-Portuguesic", + "level15": "Northern Indo-Portuguesic" + }, + "vkt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Greater Riau-Johoric" + }, + "vku": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda", + "level4": "Central Ngayarda", + "level5": "Yindjibarndi-Kurrama" + }, + "vky": { + "level0": "Bookkeeping" + }, + "vkz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Ebira-Nupoid", + "level4": "Macro-Nupoid", + "level5": "Nupoid" + }, + "vlp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "Cape Cumberland" + }, + "vlr": { + "level0": "Bookkeeping" + }, + "vls": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch", + "level6": "Middle-Modern Dutch", + "level7": "Modern Dutch", + "level8": "Southwestern Dutch" + }, + "vma": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda", + "level4": "Central Ngayarda" + }, + "vmb": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Finasleigh Pama" + }, + "vmc": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec" + }, + "vmd": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "South-Western Dravidian", + "level4": "Koraga" + }, + "vme": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Southwest Maluku", + "level4": "Babar", + "level5": "South Babar", + "level6": "Masela-South Babar" + }, + "vmf": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Upper Franconian", + "level10": "Greater East Franconian" + }, + "vmg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Patpatar-Minigir-Tolai", + "level9": "Minigir-Tolai" + }, + "vmh": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Central Tatic" + }, + "vmi": { + "level0": "Worrorran", + "level1": "Northern Worrorran", + "level2": "Forrest River" + }, + "vmj": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Coast Mixtec" + }, + "vmk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Makua-Lomwe" + }, + "vml": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Kartu-Nhanda", + "level3": "Kartu" + }, + "vmm": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Eastern Alta Mixtec", + "level7": "Southeastern Alta Mixtec", + "level8": "Teozacoalco Mixtec" + }, + "vmp": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Mazatecan", + "level5": "Valley Mazatec", + "level6": "Ayautlic", + "level7": "Northern Baja Mazatec" + }, + "vmq": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Northeastern Alta Mixtec" + }, + "vmr": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Chuwaboic" + }, + "vms": { + "level0": "Unattested" + }, + "vmu": { + "level0": "Pama-Nyungan", + "level1": "Yimidhirr-Yalanji-Yidinic", + "level2": "Yalandyic" + }, + "vmv": { + "level0": "Maiduan" + }, + "vmw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Makua-Lomwe" + }, + "vmx": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Eastern Alta Mixtec", + "level7": "Southeastern Alta Mixtec", + "level8": "Teozacoalco Mixtec" + }, + "vmy": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Mazatecan", + "level5": "Valley Mazatec", + "level6": "Ayautlic" + }, + "vmz": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Popolocan-Mazatecan", + "level4": "Mazatecan", + "level5": "Central Mazatec" + }, + "vnk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Temotu", + "level5": "Utupua-Vanikoro" + }, + "vnm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Western Malakula linkage", + "level8": "Central-Western Malakula" + }, + "vnp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "Cape Cumberland" + }, + "vol": { + "level0": "Artificial Language" + }, + "vor": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bena-Mboi", + "level5": "Bena", + "level6": "Yungur-Voro" + }, + "vot": { + "level0": "Uralic", + "level1": "Finnic", + "level2": "Coastal Finnic", + "level3": "Neva", + "level4": "Central Finnic" + }, + "vra": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage", + "level7": "Lemerig-Veraa" + }, + "vro": { + "level0": "Uralic", + "level1": "Finnic" + }, + "vrs": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "New Ireland-Northwest Solomonic linkage", + "level7": "St George linkage", + "level8": "Northwest Solomonic", + "level9": "Choiseul", + "level10": "West Choiseul" + }, + "vrt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Central Vanuatu", + "level6": "Malakula", + "level7": "Eastern Malakula linkage", + "level8": "Central-Southeast Malakula" + }, + "vsi": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "RSLic", + "level3": "Nuclear RSLic", + "level4": "Central RSLic" + }, + "vsl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Spanish Sign" + }, + "vsv": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Spanish Sign", + "level3": "Nuclear Spanish Sign" + }, + "vto": { + "level0": "Tor-Orya", + "level1": "Tor", + "level2": "Coastal Tor", + "level3": "Betaf-Vitou" + }, + "vum": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "West Kikongo", + "level20": "Vilic", + "level21": "Lumbuic", + "level22": "Ngubi-Sangu-Sira-Punu", + "level23": "Sangu-Sira-Punu", + "level24": "Punu-Vungu" + }, + "vun": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Kilimanjaro-Taita", + "level9": "Kilimanjaro Bantu", + "level10": "Chaga", + "level11": "Central Kilimanjaro" + }, + "vut": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Vutic", + "level10": "Vute-Wawa" + }, + "vwa": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Waic", + "level5": "Wa-Lawa", + "level6": "Nuclear Waic" + }, + "waa": { + "level0": "Sahaptian", + "level1": "Sahaptin", + "level2": "Northern Sahaptin" + }, + "wab": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Bel-Roinji-Nenaya", + "level9": "Bel", + "level10": "Eastern Bel" + }, + "wac": { + "level0": "Chinookan" + }, + "wad": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Central-Western Yapen", + "level7": "Ansus-Ambai" + }, + "wae": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Alemannic", + "level10": "South Alemannic" + }, + "waf": { + "level0": "Unattested" + }, + "wag": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Taupota linkage", + "level10": "Nuclear Taupota linkage", + "level11": "Eastern Taupota", + "level12": "Taupota-Waiema" + }, + "wah": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Seram Laut", + "level3": "Banda-Geser", + "level4": "Seran Laut" + }, + "wai": { + "level0": "Unattested", + "level1": "Tor-Orya (Unattested)" + }, + "waj": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Kainantu", + "level3": "Tairora" + }, + "wal": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "North-West Ometo", + "level3": "Central Ometo" + }, + "wam": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Maritimes-Southern New England Algonquian", + "level5": "Southern New England Algonquian" + }, + "wan": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Nwa-Ben", + "level4": "Wan-Mwan" + }, + "wao": { + "level0": "Yuki-Wappo" + }, + "wap": { + "level0": "Arawakan", + "level1": "Negro-Roraima", + "level2": "Pidjanan", + "level3": "Wapishanan", + "level4": "Wapishana-Atorai" + }, + "war": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater Central Philippine", + "level3": "Central Philippine", + "level4": "Bisayan", + "level5": "Central Bisayan", + "level6": "Warayan", + "level7": "Samar-Waray" + }, + "wat": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Are linkage" + }, + "wau": { + "level0": "Arawakan", + "level1": "Central-Eastern Maipuran", + "level2": "Central Maipuran", + "level3": "Xinguan Arawak", + "level4": "Waura-Mehinaku-Kustenau", + "level5": "Waura-Mehinaku" + }, + "wav": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Mumuye-Yandang", + "level6": "Yandangic", + "level7": "Waka-Yendang-Teme", + "level8": "Waka-Yandang" + }, + "waw": { + "level0": "Cariban", + "level1": "Parukotoan", + "level2": "Waiwaian" + }, + "wax": { + "level0": "Ramu", + "level1": "Lower Ramu", + "level2": "Ottilien", + "level3": "Watam-Kaian" + }, + "way": { + "level0": "Cariban", + "level1": "Guianan", + "level2": "Wayanaic" + }, + "waz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Upper Markham", + "level9": "Mountain Upper Markham" + }, + "wbb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Eastern Yapen" + }, + "wbe": { + "level0": "Lakes Plain", + "level1": "Tariku", + "level2": "East Tariku", + "level3": "Doutai-Kai-Waritai" + }, + "wbf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Samu" + }, + "wbh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Mwika", + "level10": "Fipaic", + "level11": "Maluwawaru" + }, + "wbi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Southern Tanzania Highlands Bantu", + "level9": "Wanji-Sangu" + }, + "wbj": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "South Cushitic", + "level3": "Greater West Rift South Cushitic", + "level4": "West Rift South Cushitic", + "level5": "Northern West Rift South Cushitic" + }, + "wbk": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Nuristani", + "level4": "Nuristani Kalasha-Tregami" + }, + "wbl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Saka-Wakhi" + }, + "wbm": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "East Palaungic", + "level4": "Waic", + "level5": "Wa-Lawa", + "level6": "Nuclear Waic" + }, + "wbp": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Ngumpin-Yapa", + "level3": "Yapa" + }, + "wbq": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Teluguic" + }, + "wbr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Bhil" + }, + "wbt": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Wati", + "level3": "Martuwangkic", + "level4": "Warnman-Wangka" + }, + "wbv": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Kartu-Nhanda", + "level3": "Kartu" + }, + "wbw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Cenderawasih Bay", + "level5": "Yapen", + "level6": "Central-Western Yapen", + "level7": "Ansus-Ambai" + }, + "wca": { + "level0": "Yanomamic", + "level1": "Ninam-Yanomam-Yaroame", + "level2": "Yanomam-Yaroame", + "level3": "Yanomam-Yanimamo" + }, + "wci": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Western Gbe", + "level5": "Kpesi-Waci" + }, + "wdd": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie", + "level14": "Moyen Kasai-Ngounie", + "level15": "Interior Kasai-Ngounie", + "level16": "West Kasai-Ngounie", + "level17": "Northwest Kasai-Ngounie", + "level18": "Nzebi-Laali-Yaa", + "level19": "Njebi (B.50)", + "level20": "Ndjavi B" + }, + "wdg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Kalamic-South Adelbert", + "level3": "South Adelbert", + "level4": "Osum-Wadaginam-Pomoikan" + }, + "wdu": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Southern Maric" + }, + "wea": { + "level0": "Sino-Tibetan", + "level1": "Karenic", + "level2": "Southern Karen", + "level3": "Sgaw" + }, + "wec": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Wee", + "level5": "Guere-Krahn", + "level6": "Guere" + }, + "wed": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Taupota linkage", + "level10": "Nuclear Taupota linkage", + "level11": "Wedauic" + }, + "weh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "West Ring", + "level10": "Aghemic", + "level11": "Aghem-Weh" + }, + "wei": { + "level0": "Anim", + "level1": "Tirio", + "level2": "Nuclear Tirio" + }, + "wem": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Fongbeic" + }, + "weo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Three Rivers" + }, + "wep": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Alts\u00e4chsisch", + "level7": "Middle-Modern Low German", + "level8": "Low German", + "level9": "West Low German" + }, + "wer": { + "level0": "Kunimaipan", + "level1": "Weric" + }, + "wes": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English", + "level10": "Macro-English", + "level11": "Guinea Coast Creole English", + "level12": "West African Creole English", + "level13": "Coastal Nigerian Krio", + "level14": "Nigeria-Cameroon Creole English" + }, + "wet": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Wetar-Atauro", + "level4": "Wetar", + "level5": "Perai-Tugun-Aputai", + "level6": "Perai-Aputai" + }, + "weu": { + "level0": "Bookkeeping" + }, + "wew": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Sumba-Hawu", + "level5": "Sumba", + "level6": "Wewewa-Laboya" + }, + "wfg": { + "level0": "Pauwasi", + "level1": "Eastern Pauwasi" + }, + "wga": { + "level0": "Pama-Nyungan", + "level1": "Ngarna", + "level2": "Southern Ngarna", + "level3": "Ngarru" + }, + "wgb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "Suauic" + }, + "wgg": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Palku", + "level3": "Arabana-Wangganguru" + }, + "wgi": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Wahgic" + }, + "wgo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera" + }, + "wgu": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Thura-Yura" + }, + "wgw": { + "level0": "Bookkeeping" + }, + "wgy": { + "level0": "Pama-Nyungan" + }, + "wha": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Nunusaku", + "level3": "Patakai-Manusela", + "level4": "Manusela-Huaulu" + }, + "whg": { + "level0": "Nuclear Trans New Guinea", + "level1": "Chimbu-Wahgi", + "level2": "Wahgic" + }, + "whk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Kenyahic", + "level5": "Lowland Kenyah" + }, + "whu": { + "level0": "Bookkeeping" + }, + "wib": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Tusia" + }, + "wic": { + "level0": "Caddoan", + "level1": "Northern Caddoan" + }, + "wie": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Kuku-Wik-Ep", + "level5": "Ep-Meanha-Keyenganh" + }, + "wif": { + "level0": "Unattested", + "level1": "Pama-Nyungan (Unattested)" + }, + "wig": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Ngatharra-Ngathana-Iinychanya" + }, + "wih": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Kuku-Wik-Ep", + "level5": "Ep-Meanha-Keyenganh" + }, + "wii": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Wapeic" + }, + "wij": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Kuku-Wik-Ep", + "level5": "Kuku-Wik", + "level6": "Mungkanic", + "level7": "Mungkan-Mungkanhu" + }, + "wik": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Ngatharra-Ngathana-Iinychanya" + }, + "wil": { + "level0": "Worrorran", + "level1": "Northern Worrorran" + }, + "wim": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Kuku-Wik-Ep", + "level5": "Kuku-Wik", + "level6": "Mungkanic", + "level7": "Mungkan-Mungkanhu" + }, + "win": { + "level0": "Siouan", + "level1": "Mississippi Valley", + "level2": "Winnebago-Chiwere" + }, + "wir": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Tupi-Guarani Subgroup VI", + "level6": "Kawahiva", + "level7": "Unclassified Kawahiva" + }, + "wit": { + "level0": "Wintuan" + }, + "wiv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Meso Melanesian linkage", + "level6": "Bali-Vitu" + }, + "wiw": { + "level0": "Bookkeeping" + }, + "wiy": { + "level0": "Algic" + }, + "wja": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Tula-Longuda", + "level6": "Tula-Waja" + }, + "wji": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.2", + "level5": "Nuclear West Chadic B.2", + "level6": "Central West Chadic B.2", + "level7": "Warji-Gala-Kariya" + }, + "wka": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "South Cushitic", + "level3": "Greater West Rift South Cushitic" + }, + "wkb": { + "level0": "Bookkeeping" + }, + "wkd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Sarmi", + "level8": "Sobeic" + }, + "wkl": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid", + "level10": "Kalanadic" + }, + "wku": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid", + "level10": "Kalanadic" + }, + "wkw": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "North Coast Pama-Nyungan", + "level3": "Waka-Kabic", + "level4": "Western Waka-Kabic" + }, + "wla": { + "level0": "Walioic", + "level1": "Pai-Sinen-Walio" + }, + "wlc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Mijikenda-Pokomo-Comorian", + "level11": "Comorian Bantu" + }, + "wle": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Harari-East Gurage", + "level6": "Silte-Wolane" + }, + "wlg": { + "level0": "Gunwinyguan" + }, + "wlh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Timor" + }, + "wli": { + "level0": "North Halmahera", + "level1": "Northern North Halmahera", + "level2": "Sahuan", + "level3": "Nuclear Sahuan", + "level4": "Sahu-Waioli" + }, + "wlk": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Pacific Coast Athabaskan", + "level4": "California Athabaskan" + }, + "wll": { + "level0": "Nubian", + "level1": "Central Nubian", + "level2": "Kordofan Nubian", + "level3": "Western Kordofan Nubian" + }, + "wln": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan", + "level4": "Latinic", + "level5": "Imperial Latin", + "level6": "Romance", + "level7": "Italo-Western Romance", + "level8": "Western Romance", + "level9": "Shifted Western Romance", + "level10": "Northwestern Shifted Romance", + "level11": "Gallo-Rhaetian", + "level12": "Oil" + }, + "wlo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Southern Kaili-Wolio", + "level5": "Island Kaili-Wolio", + "level6": "Wolio-Kamaru" + }, + "wlr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo", + "level8": "Southwest Santo" + }, + "wls": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Tokalau Fijian", + "level6": "Polynesian", + "level7": "Nuclear Polynesian", + "level8": "East Uvean-Niuafo'ou" + }, + "wlu": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "North Coast Pama-Nyungan", + "level3": "Waka-Kabic", + "level4": "Western Waka-Kabic" + }, + "wlv": { + "level0": "Mataguayan", + "level1": "Mataguayo II", + "level2": "Wichi" + }, + "wlw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani", + "level2": "Central Dani", + "level3": "Grand Valley Dani", + "level4": "Walakic" + }, + "wlx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Northwest Oti-Volta", + "level13": "Safaliba-Dagaare", + "level14": "Dagaaric", + "level15": "Central-South Dagaric", + "level16": "South Dagaric" + }, + "wly": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Southern Kiranti", + "level6": "Bantawic" + }, + "wma": { + "level0": "Unattested" + }, + "wmb": { + "level0": "Mirndi", + "level1": "Ngurlun" + }, + "wmc": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Wamas-Samosa-Murupi-Mosimo" + }, + "wmd": { + "level0": "Nambiquaran", + "level1": "Nambikwara Complex", + "level2": "Northern Nambiquaran" + }, + "wme": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Chaurasiya" + }, + "wmg": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic", + "level4": "Muya" + }, + "wmh": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Timor-Babar", + "level3": "Eastern Timor", + "level4": "Kawaimina" + }, + "wmi": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Finasleigh Pama" + }, + "wmm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Greater South Sulawesi", + "level3": "South Sulawesi", + "level4": "Northern South Sulawesi", + "level5": "Masenrempulu" + }, + "wmn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Mid-Northern New Caledonian", + "level9": "Voh-Kone-Cem-Pac", + "level10": "Voh-Kone" + }, + "wmo": { + "level0": "Nuclear Torricelli" + }, + "wms": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Dumut", + "level6": "Ketum-Wambon" + }, + "wmt": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Ngumpin-Yapa", + "level3": "Ngumpin", + "level4": "Western Ngumpin" + }, + "wmw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Sabaki-Swahili" + }, + "wmx": { + "level0": "Sko", + "level1": "Skou-Serra-Piore", + "level2": "Serra Hills", + "level3": "Rawo-Main Serra" + }, + "wnb": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kumil-Tibor", + "level6": "Tibor" + }, + "wnc": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Wantoatic", + "level4": "Wantoat-Awara" + }, + "wnd": { + "level0": "Mangarrayi-Maran", + "level1": "Maran" + }, + "wne": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Pashto" + }, + "wng": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Ndeiram" + }, + "wni": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Mijikenda-Pokomo-Comorian", + "level11": "Comorian Bantu", + "level12": "Shindzwani-Shimaore" + }, + "wnk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Sumba-Hawu", + "level5": "Sumba", + "level6": "Central-East Sumbanese", + "level7": "Central Sumbanese" + }, + "wnm": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Palku", + "level3": "Pitta-Pitta" + }, + "wno": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani" + }, + "wnp": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei" + }, + "wnu": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Numugenan", + "level6": "Karian-Usan-Yaben" + }, + "wny": { + "level0": "Garrwan" + }, + "woa": { + "level0": "Northern Daly" + }, + "wob": { + "level0": "Kru", + "level1": "Greater Western Kru", + "level2": "Western Kru", + "level3": "Wee-Bassa-Klao", + "level4": "Wee" + }, + "woc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Kairiru-Manam", + "level8": "Manamic linkage", + "level9": "Kis-Wogeo" + }, + "wod": { + "level0": "Nuclear Trans New Guinea", + "level1": "Paniai Lakes", + "level2": "Mee-Wodani" + }, + "woe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Micronesian", + "level5": "Central Micronesian", + "level6": "Western Micronesian", + "level7": "Chuukic-Ponapeic", + "level8": "Trukic", + "level9": "Nuclear Trukic" + }, + "wof": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Wolof-BKK", + "level3": "Wolofic" + }, + "wog": { + "level0": "Sepik", + "level1": "Iwam-Wogamus", + "level2": "Wogamusin-Chenapian" + }, + "woi": { + "level0": "Timor-Alor-Pantar", + "level1": "Alor-Pantar", + "level2": "Nuclear Alor-Pantar", + "level3": "Central Alor", + "level4": "Abuic" + }, + "wok": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Southern Samba-Duru" + }, + "wol": { + "level0": "Atlantic-Congo", + "level1": "North-Central Atlantic", + "level2": "Wolof-BKK", + "level3": "Wolofic" + }, + "wom": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Samba-Duru", + "level6": "Southern Samba-Duru", + "level7": "Sambaic", + "level8": "Samba-Leko-Perema-Mumbake", + "level9": "Perema-Mumbake" + }, + "won": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Nkutsuic", + "level11": "Bushoong-Wongo-Lele" + }, + "woo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Aru", + "level3": "Central Aru" + }, + "wor": { + "level0": "Geelvink Bay" + }, + "wos": { + "level0": "Ndu", + "level1": "Nuclear Ndu", + "level2": "Ambulas-Hanga-Hundi" + }, + "wow": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Eastern Bungku-Tolaki", + "level8": "East Coast Bungku-Tolaki" + }, + "woy": { + "level0": "Unattested" + }, + "wpc": { + "level0": "Saliban", + "level1": "Maco-Piaroa" + }, + "wrb": { + "level0": "Pama-Nyungan", + "level1": "Ngarna", + "level2": "Southern Ngarna", + "level3": "Thawa" + }, + "wrd": { + "level0": "Bookkeeping" + }, + "wre": { + "level0": "Unattested", + "level1": "Atlantic-Congo (Unattested)" + }, + "wrg": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Northern Maric", + "level5": "Warungu-Gugu Badhun" + }, + "wrh": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "Wiradhuric" + }, + "wri": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Mantharta", + "level4": "Wariyangga-Dhargari" + }, + "wrk": { + "level0": "Garrwan" + }, + "wrl": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic", + "level2": "Ngumpin-Yapa", + "level3": "Yapa" + }, + "wrm": { + "level0": "Pama-Nyungan", + "level1": "Desert Nyungic" + }, + "wrn": { + "level0": "Heibanic", + "level1": "Eastern Heibanic" + }, + "wro": { + "level0": "Worrorran", + "level1": "Western Worrorran" + }, + "wrp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea" + }, + "wrr": { + "level0": "Yangmanic" + }, + "wrs": { + "level0": "Border", + "level1": "Warisic", + "level2": "Nuclear Warisic" + }, + "wru": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Western Bungku-Tolaki", + "level8": "West Coast Bungku-Tolaki" + }, + "wrv": { + "level0": "Suki-Gogodala", + "level1": "Gogodalic", + "level2": "Ari-Waruna" + }, + "wrw": { + "level0": "Pama-Nyungan", + "level1": "Unclassified Pama-Nyungan" + }, + "wrx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Manggaraiic", + "level6": "Waerana-Razong" + }, + "wry": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani" + }, + "wrz": { + "level0": "Gunwinyguan", + "level1": "Western Gunwinyguan", + "level2": "Warrayic" + }, + "wsa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Lower Mamberamo" + }, + "wsg": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian II", + "level3": "Gondi", + "level4": "Northwest Gondi", + "level5": "Southwest Gondi", + "level6": "Southern Gondi" + }, + "wsi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Espiritu Santo", + "level7": "West Santo" + }, + "wsk": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Kowan" + }, + "wsr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Kainantu", + "level3": "Gauwa", + "level4": "Awa-Oweina" + }, + "wss": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Nyo", + "level4": "Potou-Tano", + "level5": "Tano", + "level6": "Central Tano", + "level7": "Akanic" + }, + "wsu": { + "level0": "Unattested" + }, + "wsv": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Kohistani" + }, + "wtf": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Evapia" + }, + "wth": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "Victorian Pama-Nyungan", + "level3": "Kulin-Bunganditj", + "level4": "Kulin", + "level5": "Nuclear Kulin" + }, + "wtk": { + "level0": "Sepik", + "level1": "Sepik Hill", + "level2": "Central Sepik Hill", + "level3": "Nuclear Central Sepik Hill", + "level4": "Kapriman-Watakataui" + }, + "wtm": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Midlands Indo-Aryan", + "level7": "Apabhramsic", + "level8": "Gujarati-Rajasthani", + "level9": "Rajasthani", + "level10": "Mewati-Gojri" + }, + "wtw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Kaili-Wolio", + "level4": "Southern Kaili-Wolio" + }, + "wua": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama" + }, + "wub": { + "level0": "Worrorran", + "level1": "Northern Worrorran" + }, + "wud": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Fongbeic" + }, + "wuh": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic", + "level4": "Northern Chinese", + "level5": "Mandarinic" + }, + "wul": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani", + "level2": "Ngalik-Nduga" + }, + "wum": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Ndasaic", + "level8": "Samayic", + "level9": "Ndasa-Wumbvu" + }, + "wun": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Corridor Bantu", + "level8": "Mbozi", + "level9": "Mwika" + }, + "wur": { + "level0": "Marrku-Wurrugu" + }, + "wut": { + "level0": "Sko", + "level1": "Skou-Serra-Piore", + "level2": "Skouic", + "level3": "Eastern Skouic", + "level4": "West Coast Skouic" + }, + "wuu": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic", + "level4": "Wu-Hui Chinese" + }, + "wuv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Admiralty Islands", + "level5": "Western Admiralty Islands" + }, + "wux": { + "level0": "Limilngan-Wulna" + }, + "wuy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "Maya-Matbat", + "level6": "Raja Ampat Maya" + }, + "wwa": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Oti-Volta Oriental", + "level10": "Waama-Tayari-Ditammari" + }, + "wwb": { + "level0": "Unclassifiable" + }, + "wwo": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "North and Central Vanuatu", + "level5": "Northern Vanuatu", + "level6": "Torres-Banks linkage" + }, + "wwr": { + "level0": "Nyulnyulan", + "level1": "Eastern Nyulnyulan", + "level2": "Nyikinic" + }, + "www": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Vutic", + "level10": "Vute-Wawa" + }, + "wxa": { + "level0": "Sino-Tibetan", + "level1": "Sinitic" + }, + "wya": { + "level0": "Iroquoian", + "level1": "Northern Iroquoian" + }, + "wyb": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "Wiradhuric" + }, + "wyi": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "Victorian Pama-Nyungan", + "level3": "Kulin-Bunganditj", + "level4": "Kulin", + "level5": "Nuclear Kulin" + }, + "wym": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Middle German", + "level7": "East Middle German", + "level8": "Schlesisch-Wilmesau" + }, + "wyr": { + "level0": "Tupian", + "level1": "Arikem-Tupari", + "level2": "Tuparic", + "level3": "Nuclear Tuparic", + "level4": "Wayoro-Tupari" + }, + "wyy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Central Pacific linkage", + "level5": "Western Fijian" + }, + "xaa": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "North African Arabic", + "level7": "Moroccan-Andalusian Arabic" + }, + "xab": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Alumic", + "level5": "Hasha-Sambe" + }, + "xac": { + "level0": "Sino-Tibetan", + "level1": "Brahmaputran", + "level2": "Bodo-Garo", + "level3": "Boroic", + "level4": "Tiwa-Boro", + "level5": "Bodo-Mech-Kachari" + }, + "xag": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Lezgic", + "level3": "Samur", + "level4": "Eastern Samur", + "level5": "Udi-Aghwan" + }, + "xah": { + "level0": "Bookkeeping" + }, + "xai": { + "level0": "Unclassifiable" + }, + "xal": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Eastern Mongolic", + "level3": "Khalkha-Buriat", + "level4": "Mongolian" + }, + "xam": { + "level0": "Tuu", + "level1": "!Ui" + }, + "xan": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "Agaw", + "level3": "Northern-Eastern-Western Agaw", + "level4": "Northeastern Agaw" + }, + "xao": { + "level0": "Bookkeeping" + }, + "xap": { + "level0": "Muskogean", + "level1": "Alabaman-Koasati" + }, + "xas": { + "level0": "Uralic", + "level1": "Samoyedic", + "level2": "Kamas-Selkup" + }, + "xat": { + "level0": "Katukinan" + }, + "xau": { + "level0": "Greater Kwerba", + "level1": "Kwerba-Samarokena", + "level2": "Kwerbaic" + }, + "xav": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Cerrado", + "level3": "Central Je" + }, + "xaw": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Numic", + "level3": "Southern Numic" + }, + "xay": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Kayan-Murik-Modang", + "level4": "Kayan-Murik", + "level5": "Kayanic", + "level6": "Rejang-Makaham Kayan" + }, + "xba": { + "level0": "Bookkeeping" + }, + "xbc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB" + }, + "xbe": { + "level0": "Pama-Nyungan", + "level1": "East Queensland Border Pama Nyungan", + "level2": "Yugambalic", + "level3": "Yugambal-Bigambal" + }, + "xbg": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "Victorian Pama-Nyungan", + "level3": "Kulin-Bunganditj", + "level4": "Warrnambool-Bunganditj" + }, + "xbi": { + "level0": "Nuclear Torricelli", + "level1": "Kombio-Arapesh-Urat", + "level2": "Kombio-Yambes", + "level3": "Kombioic" + }, + "xbo": { + "level0": "Turkic", + "level1": "Bolgar" + }, + "xbr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Sumba-Hawu", + "level5": "Sumba", + "level6": "Central-East Sumbanese" + }, + "xbw": { + "level0": "Unclassifiable" + }, + "xbx": { + "level0": "Bookkeeping" + }, + "xcc": { + "level0": "Unclassifiable" + }, + "xce": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic" + }, + "xcg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Cisalpine Celtic" + }, + "xch": { + "level0": "Chimakuan" + }, + "xcl": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Armenic" + }, + "xco": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Sogdic-Ossetic", + "level6": "Sogdic" + }, + "xcr": { + "level0": "Indo-European", + "level1": "Anatolian", + "level2": "Luvo-Lydian", + "level3": "Luvo-Palaic", + "level4": "Luvic", + "level5": "Lyco-Carian", + "level6": "Milyan-Carian" + }, + "xct": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan" + }, + "xcv": { + "level0": "Yukaghir", + "level1": "Kolymic" + }, + "xda": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "New South Wales Pama-Nyungan", + "level3": "Yuin-Kuri", + "level4": "Kuri", + "level5": "Sydney-Hawkesbury" + }, + "xdc": { + "level0": "Indo-European", + "level1": "Unclassified Indo-European" + }, + "xdk": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "New South Wales Pama-Nyungan", + "level3": "Yuin-Kuri", + "level4": "Kuri", + "level5": "Sydney-Hawkesbury" + }, + "xdo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Njila", + "level8": "Southern Njila", + "level9": "Kunene", + "level10": "Cimbebasia" + }, + "xdq": { + "level0": "Nakh-Daghestanian", + "level1": "Daghestanian", + "level2": "Dargwic", + "level3": "South Dargwa" + }, + "xdy": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic" + }, + "xeb": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "East Semitic" + }, + "xed": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Lamang-Hdi" + }, + "xeg": { + "level0": "Tuu", + "level1": "!Ui", + "level2": "Eastern !Ui" + }, + "xel": { + "level0": "Eastern Jebel", + "level1": "Aka-Kelo-Molo" + }, + "xem": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Land Dayak", + "level3": "Bidayuh-Southern Land Dayak", + "level4": "Southern Land Dayak" + }, + "xer": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Cerrado", + "level3": "Central Je" + }, + "xes": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Evapia", + "level4": "Nuclear Evapia", + "level5": "Kesawai-Wia" + }, + "xet": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup I" + }, + "xeu": { + "level0": "Eleman", + "level1": "Western Eleman" + }, + "xfa": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Latino-Faliscan" + }, + "xga": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Continental Transalpine Celtic", + "level6": "Unclassified Continental Transalpine Celtic" + }, + "xgb": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Nwa-Ben", + "level4": "Unclassified Nwa-Ben" + }, + "xgd": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Northern Pama", + "level3": "Gudang-Northeast Paman" + }, + "xgf": { + "level0": "Uto-Aztecan", + "level1": "Northern Uto-Aztecan", + "level2": "Californian Uto-Aztecan", + "level3": "Serran" + }, + "xgm": { + "level0": "Pama-Nyungan", + "level1": "Rockhampton-Gladstone" + }, + "xgu": { + "level0": "Worrorran", + "level1": "Western Worrorran" + }, + "xgw": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Guwa-Yanda" + }, + "xhd": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Sayhadic" + }, + "xhe": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Continental Indo-Aryan", + "level6": "Indo-Aryan Northwestern zone", + "level7": "Sindhi-Lahnda", + "level8": "Sindhic" + }, + "xho": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Nguni-Tsonga-Copi", + "level11": "Nguni (S.40)", + "level12": "Nuclear Nguni", + "level13": "Southern Ndebele-Lowland" + }, + "xhu": { + "level0": "Hurro-Urartian" + }, + "xhv": { + "level0": "Bookkeeping" + }, + "xii": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Khoekhoe", + "level3": "South Khoekhoe" + }, + "xil": { + "level0": "Unclassifiable" + }, + "xip": { + "level0": "Unattested", + "level1": "Pano-Tacanan (Unattested)" + }, + "xir": { + "level0": "Arawakan", + "level1": "Negro-Roraima", + "level2": "Bahuanaic" + }, + "xiv": { + "level0": "Unattested" + }, + "xiy": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Jurunic" + }, + "xjb": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "New South Wales Pama-Nyungan", + "level3": "Greater Bandjalangic", + "level4": "Bandjalangic", + "level5": "Coastal Bandjalang" + }, + "xka": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Nuclear Eastern Dardic", + "level7": "Shinaic", + "level8": "Western Shinaic", + "level9": "Dangari" + }, + "xkb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede", + "level7": "Western Ede", + "level8": "Southwestern Ede" + }, + "xkc": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Central Tatic" + }, + "xkd": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Kayan-Murik-Modang", + "level4": "Kayan-Murik", + "level5": "Kayanic" + }, + "xke": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Central Sarawak", + "level4": "Punan-Muller-Schwaner", + "level5": "Muller-Schwaner", + "level6": "Hovongan-Kereho" + }, + "xkf": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Phobjib-Chali-Bumthangic", + "level4": "Chali-Bumthangic", + "level5": "Bumthangic" + }, + "xkg": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Manding-Jogo", + "level5": "Manding-Vai", + "level6": "Manding-Mokole", + "level7": "Manding", + "level8": "West Manding", + "level9": "Kita-Kagoro" + }, + "xkh": { + "level0": "Unattested", + "level1": "Cariban (Unattested)" + }, + "xki": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "xkj": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Central Tatic", + "level10": "Khalkhalic" + }, + "xkk": { + "level0": "Austroasiatic", + "level1": "Bahnaric", + "level2": "North Bahnaric", + "level3": "Lamamic" + }, + "xkl": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Kenyahic", + "level5": "Highland Kenyah" + }, + "xkm": { + "level0": "Bookkeeping" + }, + "xkn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "Kayan-Murik-Modang", + "level4": "Kayan-Murik", + "level5": "Kayanic" + }, + "xko": { + "level0": "Bookkeeping" + }, + "xkp": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian", + "level7": "Adharic", + "level8": "Tatic", + "level9": "Central Tatic", + "level10": "Taromic" + }, + "xkq": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Eastern Bungku-Tolaki", + "level8": "East Coast Bungku-Tolaki" + }, + "xkr": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Cerrado", + "level3": "Central Je" + }, + "xks": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Nuclear Muna-Buton", + "level8": "Butonic", + "level9": "East Buton" + }, + "xkt": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Oti-Volta Occidental", + "level11": "Nuclear Oti-Volta Occidental", + "level12": "Northwest Oti-Volta", + "level13": "Mossi-Farefare", + "level14": "Mossic" + }, + "xku": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kamba-Kunyi" + }, + "xkv": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Sotho-Tswana (S.30)", + "level11": "Western Sotho-Tswana" + }, + "xkw": { + "level0": "Lepki-Murkim-Kembra" + }, + "xkx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage", + "level8": "Southwest New Britain linkage", + "level9": "Arawe-Pasismanua", + "level10": "Pasismanua" + }, + "xky": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Kenyahic", + "level5": "Highland Kenyah", + "level6": "Upper Pujungan" + }, + "xkz": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Phobjib-Chali-Bumthangic", + "level4": "Chali-Bumthangic", + "level5": "Bumthangic" + }, + "xla": { + "level0": "Kamula-Elevala" + }, + "xlb": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Delawaran", + "level5": "Mahican-Woronoco-Pojassick" + }, + "xlc": { + "level0": "Indo-European", + "level1": "Anatolian", + "level2": "Luvo-Lydian", + "level3": "Luvo-Palaic", + "level4": "Luvic", + "level5": "Lyco-Carian", + "level6": "Lyco-Sidetic" + }, + "xld": { + "level0": "Indo-European", + "level1": "Anatolian", + "level2": "Luvo-Lydian" + }, + "xle": { + "level0": "Unclassifiable" + }, + "xlg": { + "level0": "Unclassifiable" + }, + "xlo": { + "level0": "Algic", + "level1": "Algonquian-Blackfoot", + "level2": "Algonquian", + "level3": "Eastern Algonquian", + "level4": "Maritimes-Southern New England Algonquian", + "level5": "Southern New England Algonquian" + }, + "xlp": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Cisalpine Celtic" + }, + "xls": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Unclassified Italic" + }, + "xlu": { + "level0": "Indo-European", + "level1": "Anatolian", + "level2": "Luvo-Lydian", + "level3": "Luvo-Palaic", + "level4": "Luvic", + "level5": "Luvian" + }, + "xly": { + "level0": "Unclassifiable" + }, + "xmb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Jarawan" + }, + "xmc": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Makua-Lomwe" + }, + "xmd": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Dabaic" + }, + "xmf": { + "level0": "Kartvelian", + "level1": "Georgian-Zan", + "level2": "Zan" + }, + "xmg": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Bamileke", + "level10": "West Bamileke" + }, + "xmh": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Kuku-Wik-Ep", + "level5": "Kuku-Wik", + "level6": "Paman Kuku" + }, + "xmi": { + "level0": "Unattested" + }, + "xmj": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "North Biu-Mandara", + "level4": "Kotoko-Buduma", + "level5": "Kotoko Meridional" + }, + "xml": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic", + "level4": "Malaysian Sign" + }, + "xmm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Vehicular Malay", + "level6": "Eastern Indonesia Trade Malay", + "level7": "Manadoic Malay" + }, + "xmo": { + "level0": "Unattested", + "level1": "Tupian (Unattested)" + }, + "xmp": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Compromise Middle Pama", + "level3": "Wik", + "level4": "Kuku-Wik-Ep", + "level5": "Kuku-Wik", + "level6": "Paman Kuku" + }, + "xmq": { + "level0": "Bookkeeping" + }, + "xms": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "ASLic" + }, + "xmt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "Maya-Matbat" + }, + "xmu": { + "level0": "Eastern Daly" + }, + "xmv": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "North-Central Malagasic", + "level7": "Northern Malagasic" + }, + "xmw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Southeast Barito", + "level5": "Malagasic", + "level6": "North-Central Malagasic", + "level7": "Northern Malagasic", + "level8": "Tsimihety-Betsimisaraka" + }, + "xmx": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Raja Ampat-South Halmahera", + "level5": "Salawati-Batta" + }, + "xmy": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Mayabic" + }, + "xmz": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Bungku-Tolaki", + "level7": "Eastern Bungku-Tolaki", + "level8": "East Coast Bungku-Tolaki" + }, + "xna": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian" + }, + "xnb": { + "level0": "Austronesian", + "level1": "Tsouic", + "level2": "Kanakanavu-Saaroa" + }, + "xng": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic" + }, + "xnh": { + "level0": "Bookkeeping" + }, + "xnj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Southern Tanzania Highlands Bantu", + "level9": "Manda-Ngoni", + "level10": "Tanzania-Mozambique Ngoni" + }, + "xnm": { + "level0": "Nyulnyulan", + "level1": "Eastern Nyulnyulan", + "level2": "Unclassified Eastern Nyulnyulan" + }, + "xnn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Meso-Cordilleran", + "level4": "South-Central Cordilleran", + "level5": "Central Cordilleran", + "level6": "Nuclear Cordilleran", + "level7": "Bontok-Kankanay", + "level8": "Kankanay", + "level9": "Maeng-Northern Kankanay" + }, + "xnq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Southern Tanzania Highlands Bantu", + "level9": "Manda-Ngoni", + "level10": "Tanzania-Mozambique Ngoni" + }, + "xnr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Indo-Aryan", + "level4": "Middle-Modern Indo-Aryan", + "level5": "Eastern Dardic", + "level6": "Himachali", + "level7": "Kangric-Chamealic-Bhattiyali", + "level8": "Kangri-Dogri" + }, + "xns": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Western West Himalayish", + "level4": "Kinnauric" + }, + "xny": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda", + "level4": "Central Ngayarda" + }, + "xod": { + "level0": "South Bird's Head Family", + "level1": "East South Bird's Head" + }, + "xog": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza", + "level10": "North Nyanza", + "level11": "Soga-Kenyi" + }, + "xoi": { + "level0": "Ramu", + "level1": "Goam", + "level2": "Tamolan", + "level3": "Unclassified Tamolan" + }, + "xok": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Southern Je", + "level3": "Kaingang-Xokleng" + }, + "xom": { + "level0": "Koman", + "level1": "Central Koman", + "level2": "Komo-Uduk" + }, + "xon": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Northern Central Gur", + "level6": "Bwamu-Oti-Volta", + "level7": "Oti-Volta", + "level8": "Nuclear Oti-Volta", + "level9": "Gurma-Yom-Oti-Volta Occidental", + "level10": "Gurma-Yom-Naudem", + "level11": "Gurma", + "level12": "Gurma B", + "level13": "Konkomba-Gangam" + }, + "xop": { + "level0": "Lower Sepik", + "level1": "Nor" + }, + "xor": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mayoruna Branch", + "level3": "Mayo Group", + "level4": "Matses subgroup" + }, + "xow": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Kumil-Tibor", + "level6": "Tibor", + "level7": "Nuclear Tibor" + }, + "xpa": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Pirriya-Kungkari" + }, + "xpb": { + "level0": "North-Eastern Tasmanian" + }, + "xpc": { + "level0": "Turkic", + "level1": "Common Turkic", + "level2": "Kipchak-Turkestan", + "level3": "Kipchak", + "level4": "Unclassified Kipchak" + }, + "xpe": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Southwest Mande", + "level4": "Kpelle" + }, + "xpf": { + "level0": "South-Eastern Tasmanian" + }, + "xpg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Graeco-Phrygian" + }, + "xph": { + "level0": "North-Eastern Tasmanian" + }, + "xpi": { + "level0": "Unclassifiable" + }, + "xpk": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mayoruna Branch", + "level3": "Mayo Group", + "level4": "Matses subgroup" + }, + "xpl": { + "level0": "Western Tasmanian" + }, + "xpm": { + "level0": "Yeniseian" + }, + "xpn": { + "level0": "Unclassifiable" + }, + "xpo": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Corachol-Aztecan", + "level3": "Aztec" + }, + "xpr": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Central Iranian PB", + "level6": "Northwestern Iranian" + }, + "xps": { + "level0": "Indo-European", + "level1": "Anatolian", + "level2": "Luvo-Lydian", + "level3": "Luvo-Palaic", + "level4": "Luvic", + "level5": "Unclassified Luvic" + }, + "xpu": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Northwest Semitic", + "level5": "Canaanite", + "level6": "Ugarito-Phoenician", + "level7": "Phoenician-Punic" + }, + "xpw": { + "level0": "Western Tasmanian", + "level1": "Western Coastal Tasmanian" + }, + "xpx": { + "level0": "Western Tasmanian", + "level1": "Western Coastal Tasmanian" + }, + "xpz": { + "level0": "South-Eastern Tasmanian" + }, + "xqt": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Sayhadic" + }, + "xrb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Senufo", + "level4": "Karaboro" + }, + "xre": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Cerrado", + "level3": "Goyaz", + "level4": "Northern Je", + "level5": "Eastern Timbira" + }, + "xrn": { + "level0": "Yeniseian" + }, + "xrr": { + "level0": "Unclassifiable" + }, + "xrt": { + "level0": "Unclassifiable" + }, + "xru": { + "level0": "Western Daly", + "level1": "Bringen", + "level2": "Marithielic" + }, + "xrw": { + "level0": "Sepik", + "level1": "Ram", + "level2": "Pouye-Karawa" + }, + "xsa": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Sayhadic" + }, + "xsb": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Central Luzon", + "level3": "Sambalic", + "level4": "Tina-Bolinao" + }, + "xsd": { + "level0": "Indo-European", + "level1": "Anatolian", + "level2": "Luvo-Lydian", + "level3": "Luvo-Palaic", + "level4": "Luvic", + "level5": "Lyco-Carian", + "level6": "Lyco-Sidetic" + }, + "xse": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Asmat-Kamoro" + }, + "xsh": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Northwestern Benue-Congo Plateau", + "level6": "Hyamic" + }, + "xsi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Ngero-Vitiaz linkage", + "level7": "Vitiaz linkage" + }, + "xsk": { + "level0": "Bookkeeping" + }, + "xsl": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Northwestern Canada Athabaskan", + "level4": "Slaveyic", + "level5": "Slave" + }, + "xsm": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Gur", + "level4": "Central Gur", + "level5": "Southern Central Gur", + "level6": "Grusi", + "level7": "Northern Grusi", + "level8": "Nuna-Kasem" + }, + "xsn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Kainji", + "level4": "Central Kainji", + "level5": "Basa-Eastern Kainji", + "level6": "Eastern Kainji", + "level7": "Jos", + "level8": "Northern Jos", + "level9": "North-Central Jos", + "level10": "Chokobo-Lemoro-Sanga", + "level11": "Lemoro-Sanga" + }, + "xso": { + "level0": "Unclassifiable" + }, + "xsp": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Silopic", + "level6": "Silopi-Utu" + }, + "xsq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "North Mozambique Bantu", + "level8": "Makua-Lomwe" + }, + "xsr": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Middle Old Tibetan", + "level5": "Late Old Tibetan", + "level6": "Central Tibetan", + "level7": "South-Western Tibetic", + "level8": "Sherpa-Jirel", + "level9": "Sherpaic" + }, + "xss": { + "level0": "Bookkeeping" + }, + "xst": { + "level0": "Bookkeeping" + }, + "xsu": { + "level0": "Yanomamic" + }, + "xsy": { + "level0": "Austronesian", + "level1": "Northwest Formosan" + }, + "xta": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Guerrero Mixtec", + "level7": "Nuclear Guerrero Mixtec" + }, + "xtb": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Northern Baja Mixtec" + }, + "xtc": { + "level0": "Kadugli-Krongo", + "level1": "Central-Western Kadugli-Krongo", + "level2": "Katcha-Kadugli-Miri-Kanga" + }, + "xtd": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Eastern Alta Mixtec" + }, + "xte": { + "level0": "Nuclear Trans New Guinea", + "level1": "Mek", + "level2": "Eastern Mek" + }, + "xtg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Celtic", + "level3": "Nuclear Celtic", + "level4": "Core Celtic", + "level5": "Continental Transalpine Celtic" + }, + "xti": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Southwestern Alta Mixtec", + "level8": "Chalcatongic", + "level9": "Sinicahua-Tijaltepec" + }, + "xtj": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Tlaxiacic", + "level8": "Yucuane-Teita" + }, + "xtl": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Southwestern Alta Mixtec", + "level8": "Chalcatongic", + "level9": "Sinicahua-Tijaltepec" + }, + "xtm": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Tlaxiacic" + }, + "xtn": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec" + }, + "xto": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Tokharian" + }, + "xtp": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Eastern Alta Mixtec", + "level7": "Southeastern Alta Mixtec", + "level8": "Teozacoalco Mixtec", + "level9": "Sindihuic" + }, + "xtq": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Saka-Wakhi", + "level5": "Saka" + }, + "xts": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Eastern Alta Mixtec", + "level7": "Southeastern Alta Mixtec", + "level8": "Teozacoalco Mixtec", + "level9": "Sindihuic" + }, + "xtt": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Western Alta Mixtec", + "level7": "Tlaxiacic" + }, + "xtu": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Northern Alta Mixtec" + }, + "xtv": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "New South Wales Pama-Nyungan", + "level3": "Yuin-Kuri", + "level4": "Yuin" + }, + "xtw": { + "level0": "Nambiquaran", + "level1": "Nambikwara Complex", + "level2": "Northern Nambiquaran", + "level3": "Roosevelt" + }, + "xty": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Amuzgo-Mixtecan", + "level3": "Mixtecan", + "level4": "Mixtec-Cuicatec", + "level5": "Mixtec", + "level6": "Guerrero Mixtec", + "level7": "Nuclear Guerrero Mixtec", + "level8": "Southwestern Guerrero Mixtec" + }, + "xtz": { + "level0": "Bookkeeping" + }, + "xua": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Badaga-Kannada", + "level5": "Kannadoid" + }, + "xub": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Badaga-Kannada", + "level5": "Kannadoid" + }, + "xuf": { + "level0": "Bookkeeping" + }, + "xug": { + "level0": "Japonic", + "level1": "Ryukyuan", + "level2": "Northern Ryukyuan", + "level3": "Okinawa" + }, + "xuj": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Badaga-Kannada", + "level5": "Kannadoid" + }, + "xum": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic", + "level3": "Sabellic" + }, + "xuo": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Mbumic", + "level5": "Central Mbum" + }, + "xup": { + "level0": "Athabaskan-Eyak-Tlingit", + "level1": "Athabaskan-Eyak", + "level2": "Athabaskan", + "level3": "Pacific Coast Athabaskan", + "level4": "Oregon Athabaskan" + }, + "xur": { + "level0": "Hurro-Urartian" + }, + "xut": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Norman Pama", + "level3": "Kuthant-Gurdjar" + }, + "xuu": { + "level0": "Khoe-Kwadi", + "level1": "Khoe", + "level2": "Non-Khoekhoe", + "level3": "West-Kxoe", + "level4": "Kxoe-Ani" + }, + "xve": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Italic" + }, + "xwc": { + "level0": "Siouan", + "level1": "Catawban" + }, + "xwe": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Western Phla-Phera" + }, + "xwg": { + "level0": "Surmic", + "level1": "South Surmic", + "level2": "Southeast Surmic" + }, + "xwl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Gbe", + "level4": "Eastern Gbe", + "level5": "Western Phla-Phera" + }, + "xwr": { + "level0": "Greater Kwerba", + "level1": "Kwerba-Samarokena", + "level2": "Kwerbaic" + }, + "xxb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Kwa Volta-Congo", + "level3": "Na-Togo" + }, + "xxk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Bima-Lembata", + "level3": "Flores-Sumba-Hawu", + "level4": "Flores Barat", + "level5": "Central Flores-Paluqe", + "level6": "Central Flores", + "level7": "Eastern Central Flores", + "level8": "Nage-Keo" + }, + "xxr": { + "level0": "Nuclear-Macro-Je", + "level1": "Maxakali-Borum", + "level2": "Maxakalian", + "level3": "Nuclear Maxakalian", + "level4": "Unclassified Nuclear Maxakalian" + }, + "xya": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "North Coast Pama-Nyungan", + "level3": "Gumbaynggiric" + }, + "xyb": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Southern Maric", + "level5": "Bidyaric" + }, + "xyl": { + "level0": "Unattested", + "level1": "Nambiquaran (Unattested)" + }, + "xyy": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "Victorian Pama-Nyungan", + "level3": "Eastern Victoria" + }, + "xzh": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "West Himalayish", + "level3": "Eastern West Himalayish", + "level4": "Pithauragarh", + "level5": "Darma-Byangsi-Chaudangsi", + "level6": "Darma-Byangsi", + "level7": "Zhangzhungic" + }, + "yaa": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Headwaters Pano", + "level5": "Yaminawa Complex" + }, + "yab": { + "level0": "Naduhup", + "level1": "Eastern Naduhup", + "level2": "Hup-Yuhup" + }, + "yac": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani", + "level2": "Ngalik-Nduga", + "level3": "Yalic" + }, + "yad": { + "level0": "Peba-Yagua" + }, + "yaf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Yaka-Suku" + }, + "yah": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Eastern Iranian", + "level5": "Shughni-Yazgulami" + }, + "yai": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Iranian PBS", + "level5": "Sogdic-Ossetic", + "level6": "Sogdic", + "level7": "Sogdian-Yagnobi" + }, + "yaj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic", + "level7": "Central Bandaic" + }, + "yak": { + "level0": "Sahaptian", + "level1": "Sahaptin", + "level2": "Northern Sahaptin" + }, + "yal": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Central Mande", + "level4": "Susu-Yalunka" + }, + "yam": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Nkambe" + }, + "yan": { + "level0": "Misumalpan", + "level1": "Sumalpan", + "level2": "Sumuic" + }, + "yao": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Ruvuma", + "level9": "Yaoic" + }, + "yap": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Yapesic" + }, + "yaq": { + "level0": "Uto-Aztecan", + "level1": "Southern Uto-Aztecan", + "level2": "Cahitan" + }, + "yar": { + "level0": "Cariban", + "level1": "Venezuelan Cariban", + "level2": "Mapoyo-Tamanaku", + "level3": "Mapoyo-Yawarana" + }, + "yas": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Bati-Mbure-Yambassa", + "level10": "Mbure-Yambassa", + "level11": "Yambassa (A.60)", + "level12": "Mmala-Elip-Gunu", + "level13": "Elip-Gunu" + }, + "yat": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Sanaga-West Mbam (A.40)", + "level10": "West Mbam (A.40)" + }, + "yav": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Mbam-Bubi", + "level7": "Mbam", + "level8": "Nuclear Mbam", + "level9": "Bati-Mbure-Yambassa", + "level10": "Mbure-Yambassa", + "level11": "Yambassa (A.60)" + }, + "yaw": { + "level0": "Arawakan", + "level1": "Central-Eastern Maipuran", + "level2": "Central Maipuran", + "level3": "Xinguan Arawak" + }, + "yax": { + "level0": "Bookkeeping" + }, + "yay": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "North-South Central Delta Cross", + "level7": "Ubaghara-Kohumono", + "level8": "Kohumonoic" + }, + "yaz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Delta Cross", + "level4": "Upper Cross", + "level5": "Central Upper Cross", + "level6": "East-West Central Delta Cross", + "level7": "Lokoic" + }, + "yba": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Idomoid", + "level4": "Akweya", + "level5": "Etulo-Idoma", + "level6": "Nuclear Idoma" + }, + "ybb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Mbam-Nkam", + "level8": "Southern Mbam-Nkam", + "level9": "Bamileke", + "level10": "West Bamileke", + "level11": "Bamboutos" + }, + "ybd": { + "level0": "Bookkeeping" + }, + "ybe": { + "level0": "Turkic", + "level1": "Common Turkic" + }, + "ybh": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Tamar", + "level6": "Yakkha-Athpariyic" + }, + "ybi": { + "level0": "Sino-Tibetan", + "level1": "Himalayish", + "level2": "Mahakiranti", + "level3": "Kiranti", + "level4": "Eastern Kiranti", + "level5": "Upper Arun", + "level6": "Lohorung-Yamphu", + "level7": "Yamphuic" + }, + "ybj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Alumic", + "level5": "Hasha-Sambe" + }, + "ybk": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Muji", + "level8": "Laghuu-Core Muji", + "level9": "Thopho-Core Muji", + "level10": "Core Muji", + "level11": "Nuclear Core Muji", + "level12": "Bokha-Phuma" + }, + "ybl": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Yukubenic" + }, + "ybm": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Numugenan", + "level6": "Karian-Usan-Yaben" + }, + "ybn": { + "level0": "Arawakan", + "level1": "Medio Rio Negro", + "level2": "Marauia-Castana" + }, + "ybo": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Greater Yaganon", + "level4": "Yaganon" + }, + "ybx": { + "level0": "Walioic" + }, + "yby": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Siane-Yagaria" + }, + "ych": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nuclear Nisoid", + "level8": "Nasu-Nosu", + "level9": "Nesu-Nasu", + "level10": "Nasu-Gepu", + "level11": "Unclassified Nasu-Gepu" + }, + "ycl": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Lipo-Lolopo" + }, + "ycn": { + "level0": "Arawakan", + "level1": "Japura-Colombia", + "level2": "Nuclear Japura-Colombia", + "level3": "Caqueta" + }, + "ycp": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Hanic", + "level7": "Ha-Ya", + "level8": "Akhaic" + }, + "ycr": { + "level0": "Japonic", + "level1": "Japanesic", + "level2": "Japan-Taiwan Japanese" + }, + "yda": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Guwa-Yanda" + }, + "ydd": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Middle German", + "level7": "East Middle German", + "level8": "Schlesisch-Wilmesau" + }, + "yde": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Nuclear Palai", + "level4": "Yangum-Ambrak", + "level5": "Yangum" + }, + "ydg": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Central Eastern Iranian", + "level5": "Yidgha-Munji" + }, + "ydk": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Mabuso", + "level4": "Hanseman", + "level5": "Rempic" + }, + "yds": { + "level0": "Bookkeeping" + }, + "yea": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Malayalamoid", + "level10": "Ravulic" + }, + "yec": { + "level0": "Mixed Language", + "level1": "German-Yiddish-Romani-Rotwelsch" + }, + "yee": { + "level0": "Lower Sepik", + "level1": "Karawarian" + }, + "yei": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Northern Bantoid", + "level5": "Mambiloid", + "level6": "Nizaa-Mambila-Vute", + "level7": "Konja-Mambila-Vute", + "level8": "Mambila-Vute", + "level9": "Mambila-Mbongno", + "level10": "Mambila", + "level11": "Njerup" + }, + "yej": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Graeco-Phrygian", + "level3": "Greek", + "level4": "South Greek", + "level5": "Central Greek", + "level6": "Koineic Greek", + "level7": "Modern Koineic Greek" + }, + "yel": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Cuvette", + "level10": "Nkutsuic" + }, + "yen": { + "level0": "Bookkeeping" + }, + "yer": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "Tarokoid", + "level5": "Yangkam-Tarok-Pe", + "level6": "Tarok-Pe" + }, + "yes": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Northwestern Benue-Congo Plateau", + "level6": "Koroic" + }, + "yeu": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid", + "level10": "Yerukula-Korava-Kaikadi" + }, + "yev": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "West Palai", + "level3": "Agi-Yeri" + }, + "yey": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu" + }, + "ygl": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Nuclear Palai", + "level4": "Yangum-Ambrak", + "level5": "Yangum" + }, + "ygm": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Warup", + "level4": "Nuclear Warup", + "level5": "Degenanic" + }, + "ygp": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nuclear Nisoid", + "level8": "Nasu-Nosu", + "level9": "Nesu-Nasu", + "level10": "Nasu-Gepu" + }, + "ygr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Siane-Yagaria", + "level5": "Kamano-Yagaria" + }, + "ygs": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "ygu": { + "level0": "Unattested", + "level1": "Mangarrayi-Maran (Unattested)" + }, + "ygw": { + "level0": "Angan", + "level1": "Nuclear Angan" + }, + "yha": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Southwestern Kra", + "level3": "Southern Kra" + }, + "yhd": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "Eastern Arabic", + "level7": "Qeltu" + }, + "yhl": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Phowa", + "level8": "Hlepho-Phukha" + }, + "yia": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan" + }, + "yib": { + "level0": "Bookkeeping" + }, + "yif": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Unclassified Nisoid" + }, + "yig": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nuclear Nisoid", + "level8": "Nasu-Nosu", + "level9": "Nesu-Nasu", + "level10": "Nesu" + }, + "yih": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "High German", + "level6": "Upper German", + "level7": "Middle-Modern High German", + "level8": "Modern High German", + "level9": "Upper Franconian", + "level10": "Greater East Franconian" + }, + "yii": { + "level0": "Pama-Nyungan", + "level1": "Yimidhirr-Yalanji-Yidinic", + "level2": "Yidinic" + }, + "yij": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda", + "level4": "Central Ngayarda", + "level5": "Yindjibarndi-Kurrama" + }, + "yik": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Nuclear Lisoid", + "level7": "Lisu-Laluba-Lavu", + "level8": "Laluba-Lavu", + "level9": "Lalo", + "level10": "Greater Lalo", + "level11": "Core Lalo" + }, + "yil": { + "level0": "Pama-Nyungan", + "level1": "Ngarna", + "level2": "Southern Ngarna", + "level3": "Ngarru" + }, + "yim": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Angami-Ao", + "level3": "Central Naga", + "level4": "Yimchingric" + }, + "yin": { + "level0": "Austroasiatic", + "level1": "Khasi-Palaung", + "level2": "Palaungic", + "level3": "West Palaungic", + "level4": "Riang" + }, + "yio": { + "level0": "Bookkeeping" + }, + "yip": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish" + }, + "yiq": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Lipo-Lolopo", + "level7": "Lipo-Micha" + }, + "yir": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Greater Awyu", + "level4": "Awyu-Dumut", + "level5": "Awyu" + }, + "yis": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Wapeic", + "level4": "Yau-Yis" + }, + "yit": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Nuclear Lisoid", + "level7": "Lisu-Laluba-Lavu", + "level8": "Laluba-Lavu", + "level9": "Lalo", + "level10": "Greater Lalo", + "level11": "Core Lalo", + "level12": "Unclassified Core Lalu" + }, + "yiu": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nuclear Nisoid" + }, + "yiv": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nisu-Nyisu", + "level8": "Nisu", + "level9": "Nuclear Nisu", + "level10": "Northern Nisu" + }, + "yix": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Axioid", + "level7": "Sani-Axi-Azhe", + "level8": "Sani-Axi" + }, + "yiy": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Southwest Pama", + "level3": "Coastal Southwest Paman" + }, + "yiz": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Axioid", + "level7": "Sani-Axi-Azhe", + "level8": "Sani-Axi" + }, + "yka": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Basap-Greater Barito", + "level3": "Greater Barito linkage", + "level4": "Sama-Bajaw" + }, + "ykg": { + "level0": "Yukaghir" + }, + "ykh": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Eastern Mongolic" + }, + "yki": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "South Halmahera-West New Guinea", + "level4": "Lower Mamberamo", + "level5": "Yoke-Pauwi" + }, + "ykk": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Are-Taupota linkage", + "level9": "Taupota linkage", + "level10": "Nuclear Taupota linkage", + "level11": "Wedauic" + }, + "ykl": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Phowa", + "level8": "Hlepho-Phukha", + "level9": "Khlula-Zokhuo" + }, + "ykm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Schouten linkage", + "level7": "Siau", + "level8": "Sissano-Tumleo", + "level9": "Ali-Tumleo" + }, + "ykn": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Nuclear Lisoid", + "level7": "Lisu-Laluba-Lavu", + "level8": "Laluba-Lavu", + "level9": "Kuansi-Kuamasi-Sonaga", + "level10": "Kuansi-Kuamasi" + }, + "yko": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Sawabantu", + "level8": "Bengaic", + "level9": "Yasa-Kombe" + }, + "ykr": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean" + }, + "ykt": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Mondzish" + }, + "yku": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Nuclear Lisoid", + "level7": "Lisu-Laluba-Lavu", + "level8": "Laluba-Lavu", + "level9": "Kuansi-Kuamasi-Sonaga", + "level10": "Kuansi-Kuamasi" + }, + "yky": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Ngbandi-Mongoba-Kazibati", + "level6": "Ngbandic", + "level7": "Nuclear Ngbandic" + }, + "yla": { + "level0": "Keram", + "level1": "Ulmapo" + }, + "ylg": { + "level0": "Ndu", + "level1": "Nuclear Ndu", + "level2": "Manambu-Yalaku" + }, + "yli": { + "level0": "Nuclear Trans New Guinea", + "level1": "Dani", + "level2": "Ngalik-Nduga", + "level3": "Yalic" + }, + "yll": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Wapeic", + "level4": "Ningil-Yil" + }, + "ylm": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Lipo-Lolopo", + "level7": "Unclassified Lipo-Lolopo" + }, + "yln": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Eastern Kra", + "level3": "Buyang", + "level4": "Northern Buyang" + }, + "ylo": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Unclassified Lisoid" + }, + "ylr": { + "level0": "Pama-Nyungan", + "level1": "Kalkatungic" + }, + "ylu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Lower Markham", + "level9": "Busu" + }, + "yly": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Southern Melanesian", + "level5": "New Caledonian", + "level6": "Mainland New Caledonian", + "level7": "Northern New Caledonian", + "level8": "Extreme Northern New Caledonian", + "level9": "Nyalayu" + }, + "yma": { + "level0": "Bookkeeping" + }, + "ymb": { + "level0": "Nuclear Torricelli", + "level1": "Kombio-Arapesh-Urat", + "level2": "Kombio-Yambes" + }, + "ymc": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Muji", + "level8": "Laghuu-Core Muji", + "level9": "Thopho-Core Muji", + "level10": "Core Muji", + "level11": "Nuclear Core Muji", + "level12": "Northern-Southern Muji" + }, + "ymd": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Hani-Jino", + "level5": "Bisoid-Hanic", + "level6": "Hanic", + "level7": "Ha-Ya", + "level8": "Akhaic" + }, + "yme": { + "level0": "Peba-Yagua", + "level1": "Peba-Yameo" + }, + "ymg": { + "level0": "Bookkeeping" + }, + "ymh": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Lipo-Lolopo", + "level7": "Unclassified Lipo-Lolopo", + "level8": "Southwestern Lolo" + }, + "ymi": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Muji" + }, + "ymj": { + "level0": "Bookkeeping" + }, + "ymk": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Rufiji-Ruvuma", + "level8": "Ruvuma", + "level9": "Makonde-Makwe" + }, + "yml": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Nuclear Papuan Tip linkage", + "level7": "North Papuan Mainland-D'Entrecasteaux linkage", + "level8": "Bwaidoga linkage", + "level9": "Iamalelic" + }, + "ymm": { + "level0": "Afro-Asiatic", + "level1": "Cushitic", + "level2": "East Cushitic", + "level3": "Lowland East Cushitic", + "level4": "Southern Lowland East Cushitic", + "level5": "Mainstream Lowland East Cushitic", + "level6": "Omo-Tana", + "level7": "Eastern Omo-Tana" + }, + "ymn": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Sarmi-Jayapura Bay", + "level7": "Sarmi" + }, + "ymo": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Nuclear Palai", + "level4": "Yangum-Ambrak", + "level5": "Yangum" + }, + "ymp": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Misim-Yamap" + }, + "ymq": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Muji", + "level8": "Laghuu-Core Muji", + "level9": "Thopho-Core Muji", + "level10": "Core Muji" + }, + "ymr": { + "level0": "Dravidian", + "level1": "South Dravidian", + "level2": "South Dravidian I", + "level3": "Tamil-Kannada", + "level4": "Tamil-Kota", + "level5": "Tamil-Toda", + "level6": "Tamil-Irula", + "level7": "Tamil-Kodagu", + "level8": "Tamil-Malayalam", + "level9": "Tamiloid", + "level10": "Malasa-Eravallan" + }, + "ymx": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Muji", + "level8": "Laghuu-Core Muji", + "level9": "Thopho-Core Muji", + "level10": "Core Muji", + "level11": "Nuclear Core Muji", + "level12": "Northern-Southern Muji" + }, + "ymz": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Muji", + "level8": "Laghuu-Core Muji", + "level9": "Thopho-Core Muji", + "level10": "Core Muji", + "level11": "Nuclear Core Muji" + }, + "yna": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nuclear Nisoid", + "level8": "Unclassified Nuclear Nisoid" + }, + "ynd": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Central Karnic", + "level3": "Western Central Karnic", + "level4": "Yandruwandhic" + }, + "yne": { + "level0": "Bookkeeping" + }, + "yng": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Mundu-Baka", + "level6": "Western Mundu-Baka", + "level7": "River Western Mundu-Baka", + "level8": "Monzomboic", + "level9": "Kpala-Bakpa" + }, + "ynh": { + "level0": "Bookkeeping" + }, + "ynk": { + "level0": "Eskimo-Aleut", + "level1": "Eskimo", + "level2": "Yupik" + }, + "ynl": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Rai Coast", + "level3": "Nuru" + }, + "yno": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Tai P", + "level9": "Shanic", + "level10": "Sukaphic", + "level11": "Northern Shanic", + "level12": "Sipsongpannic" + }, + "ynq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Mumuye-Yandang", + "level6": "Yandangic", + "level7": "Waka-Yendang-Teme", + "level8": "Waka-Yandang" + }, + "yns": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie" + }, + "ynu": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "South Eastern Tucanoan" + }, + "yob": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "Oumic", + "level9": "Magoric" + }, + "yog": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Northern Luzon", + "level3": "Cagayan Valley", + "level4": "Ibanagic", + "level5": "Gaddangic" + }, + "yoi": { + "level0": "Japonic", + "level1": "Ryukyuan", + "level2": "Southern Ryukyu", + "level3": "Macro-Yaeyama" + }, + "yok": { + "level0": "Yokutsan", + "level1": "General Yokuts", + "level2": "Nim Yokuts" + }, + "yol": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "North Sea Germanic", + "level6": "Anglo-Frisian", + "level7": "Anglic", + "level8": "Later Anglic", + "level9": "Middle-Modern English" + }, + "yom": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended", + "level13": "Kikongo Language Cluster", + "level14": "Nuclear KLC", + "level15": "Kikongoic", + "level16": "Kambakunyic Kikongo", + "level17": "Kilaadic Kikongo", + "level18": "Central-Southern Kikongo", + "level19": "West Kikongo" + }, + "yon": { + "level0": "Nuclear Trans New Guinea", + "level1": "Asmat-Awyu-Ok", + "level2": "Awyu-Ok", + "level3": "Ok-Oksapmin", + "level4": "Ok", + "level5": "Lowland Ok", + "level6": "Division A Lowland Ok" + }, + "yor": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Defoid", + "level4": "Yoruboid", + "level5": "Edekiri", + "level6": "Ede", + "level7": "Eastern Ede", + "level8": "Southeastern Ede", + "level9": "Nuclear Yoruba", + "level10": "Lucumi-Yoruba" + }, + "yos": { + "level0": "Bookkeeping" + }, + "yot": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Central Adamawa", + "level5": "Mumuye-Yandang", + "level6": "Yandangic", + "level7": "Bali-Kpasam" + }, + "yox": { + "level0": "Japonic", + "level1": "Ryukyuan", + "level2": "Northern Ryukyuan", + "level3": "Amami" + }, + "yoy": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai", + "level6": "Sapa-Southwestern Tai", + "level7": "Southwestern Tai", + "level8": "Southwestern Thai PH", + "level9": "Lao-Thai", + "level10": "Sakon Nakhon" + }, + "ypa": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Riverine Phula", + "level7": "Upriver Riverine Phula" + }, + "ypb": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Phowa", + "level8": "Ani-Labo" + }, + "ypg": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Riverine Phula", + "level7": "Upriver Riverine Phula", + "level8": "Pholic" + }, + "yph": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Riverine Phula", + "level7": "Downriver Riverine Phula", + "level8": "Phupha-Alugu" + }, + "ypl": { + "level0": "Bookkeeping" + }, + "ypm": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Muji", + "level8": "Laghuu-Core Muji", + "level9": "Thopho-Core Muji", + "level10": "Core Muji", + "level11": "Nuclear Core Muji", + "level12": "Bokha-Phuma" + }, + "ypn": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Phowa", + "level8": "Ani-Labo" + }, + "ypo": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Riverine Phula", + "level7": "Upriver Riverine Phula", + "level8": "Pholic" + }, + "ypp": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Riverine Phula", + "level7": "Downriver Riverine Phula", + "level8": "Phupa-Phuza" + }, + "ypw": { + "level0": "Bookkeeping" + }, + "ypz": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Riverine Phula", + "level7": "Downriver Riverine Phula", + "level8": "Phupa-Phuza" + }, + "yrb": { + "level0": "Yareban", + "level1": "Yareba-Bariji-Nawaru" + }, + "yre": { + "level0": "Mande", + "level1": "Eastern Mande", + "level2": "Southeastern Mande", + "level3": "Mano-Dan", + "level4": "Guro-Dan", + "level5": "Guro-Yaoure" + }, + "yri": { + "level0": "Bookkeeping" + }, + "yrk": { + "level0": "Uralic", + "level1": "Samoyedic", + "level2": "Enets-Nenets", + "level3": "Nenets" + }, + "yrl": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup III" + }, + "yrn": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Eastern Kra", + "level3": "Buyang" + }, + "yro": { + "level0": "Yanomamic", + "level1": "Ninam-Yanomam-Yaroame", + "level2": "Yanomam-Yaroame" + }, + "yrs": { + "level0": "Bookkeeping" + }, + "yrw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Madang", + "level2": "Croisilles", + "level3": "Greater Northern Adelbert", + "level4": "Northern Adelbert", + "level5": "Numugenan", + "level6": "Yarawata-Parawen-Ukuriguma" + }, + "ysd": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Kazhouish" + }, + "ysg": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Nuclear Lisoid", + "level7": "Lisu-Laluba-Lavu", + "level8": "Laluba-Lavu", + "level9": "Kuansi-Kuamasi-Sonaga" + }, + "ysl": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "LSFic", + "level3": "Yugoslav Sign" + }, + "ysm": { + "level0": "Sign Language", + "level1": "L1 Sign Language", + "level2": "Myanmar Sign" + }, + "ysn": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Axioid", + "level7": "Sani-Axi-Azhe" + }, + "yso": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Unclassified Southeastern Ngwi" + }, + "ysp": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Lipo-Lolopo", + "level7": "Unclassified Lipo-Lolopo", + "level8": "Southwestern Lolo" + }, + "ysr": { + "level0": "Eskimo-Aleut", + "level1": "Eskimo" + }, + "yss": { + "level0": "Sepik", + "level1": "Sepik Tama", + "level2": "Mayo-Pasi" + }, + "ysy": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nuclear Nisoid", + "level8": "Nasu-Nosu", + "level9": "Nesu-Nasu", + "level10": "Nasu-Gepu" + }, + "yta": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Nuclear Lisoid", + "level7": "Lisu-Laluba-Lavu", + "level8": "Laluba-Lavu" + }, + "ytl": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Nuclear Lisoid" + }, + "ytp": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Muji", + "level8": "Laghuu-Core Muji", + "level9": "Thopho-Core Muji" + }, + "ytw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Yupna", + "level4": "Unclassified Yupna" + }, + "yua": { + "level0": "Mayan", + "level1": "Core Mayan", + "level2": "Yucatecan", + "level3": "Nuclear Yucatecan", + "level4": "Yucatec-Lacandon" + }, + "yub": { + "level0": "Pama-Nyungan", + "level1": "East Queensland Border Pama Nyungan", + "level2": "Yugambalic", + "level3": "Yugambal-Bigambal" + }, + "yud": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Central Semitic", + "level4": "Arabian", + "level5": "Arabic", + "level6": "North African Arabic" + }, + "yue": { + "level0": "Sino-Tibetan", + "level1": "Sinitic", + "level2": "Classical-Middle-Modern Sinitic", + "level3": "Middle-Modern Sinitic", + "level4": "Yue-Pinghua" + }, + "yuf": { + "level0": "Cochimi-Yuman", + "level1": "Yuman", + "level2": "General Yuman", + "level3": "Pai" + }, + "yug": { + "level0": "Yeniseian", + "level1": "Northern Yeniseian" + }, + "yui": { + "level0": "Tucanoan", + "level1": "Eastern Tucanoan", + "level2": "Eastern Eastern Tucanoan", + "level3": "Eastern Eastern Tucanoan II", + "level4": "Pisamira-Yuruti", + "level5": "Tuyuca-Yuruti" + }, + "yuj": { + "level0": "Pauwasi", + "level1": "Eastern Pauwasi" + }, + "yuk": { + "level0": "Yuki-Wappo" + }, + "yul": { + "level0": "Central Sudanic", + "level1": "Sara-Bongo-Bagirmi", + "level2": "SBB Occidental" + }, + "yum": { + "level0": "Cochimi-Yuman", + "level1": "Yuman", + "level2": "General Yuman", + "level3": "River Yuman" + }, + "yun": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Bena-Mboi", + "level5": "Bena", + "level6": "Yungur-Voro" + }, + "yup": { + "level0": "Cariban", + "level1": "Opon-Yukpan", + "level2": "Yukpan" + }, + "yuq": { + "level0": "Tupian", + "level1": "Eastern Tupian", + "level2": "Maweti-Guarani", + "level3": "Aweti-Guarani", + "level4": "Tupi-Guarani", + "level5": "Southern Tupi-Guarani", + "level6": "Tupi-Guarani Subgroup II", + "level7": "Warazu-Sirionoid", + "level8": "Sirionoid" + }, + "yur": { + "level0": "Algic" + }, + "yus": { + "level0": "Bookkeeping" + }, + "yut": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Yupna", + "level4": "Kewieng-Bonkiman-Nokopo" + }, + "yuu": { + "level0": "Bookkeeping" + }, + "yuw": { + "level0": "Nuclear Trans New Guinea", + "level1": "Finisterre-Huon", + "level2": "Finisterre-Saruwaged", + "level3": "Uruwa" + }, + "yux": { + "level0": "Yukaghir", + "level1": "Kolymic" + }, + "yuy": { + "level0": "Mongolic-Khitan", + "level1": "Mongolic", + "level2": "Southern Periphery Mongolic" + }, + "yva": { + "level0": "Yawa-Saweru" + }, + "yvt": { + "level0": "Arawakan", + "level1": "Alto Orinoco", + "level2": "Parenic" + }, + "ywa": { + "level0": "Sepik", + "level1": "Sepik Tama", + "level2": "Mayo-Pasi", + "level3": "Yimin-Bel" + }, + "ywg": { + "level0": "Pama-Nyungan", + "level1": "South-West Pama-Nyungan", + "level2": "Pilbara", + "level3": "Ngayarda", + "level4": "Central Ngayarda", + "level5": "Panytyima-Yinhawangka" + }, + "ywl": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Nuclear Lisoid", + "level7": "Lisu-Laluba-Lavu", + "level8": "Laluba-Lavu", + "level9": "Lalo", + "level10": "Greater Lalo", + "level11": "Core Lalo", + "level12": "Central-Western Lalo" + }, + "ywm": { + "level0": "Bookkeeping" + }, + "ywn": { + "level0": "Pano-Tacanan", + "level1": "Panoan", + "level2": "Mainline Pano", + "level3": "Pano Nawa", + "level4": "Headwaters Pano", + "level5": "Yaminawa Complex" + }, + "ywq": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nuclear Nisoid", + "level8": "Nasu-Nosu", + "level9": "Nesu-Nasu", + "level10": "Nasu-Gepu" + }, + "ywr": { + "level0": "Nyulnyulan", + "level1": "Eastern Nyulnyulan", + "level2": "Yawuric" + }, + "ywt": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Lisoid", + "level6": "Nuclear Lisoid", + "level7": "Lisu-Laluba-Lavu", + "level8": "Laluba-Lavu", + "level9": "Lalo", + "level10": "Greater Lalo", + "level11": "Core Lalo", + "level12": "Central-Western Lalo" + }, + "ywu": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nuclear Nisoid", + "level8": "Nasu-Nosu", + "level9": "Nesu-Nasu", + "level10": "Nesu" + }, + "yww": { + "level0": "Pama-Nyungan", + "level1": "Karnic", + "level2": "Central Karnic", + "level3": "Western Central Karnic", + "level4": "Yandruwandhic" + }, + "yxm": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Northern Pama" + }, + "yym": { + "level0": "Bookkeeping" + }, + "yyu": { + "level0": "Nuclear Torricelli", + "level1": "Wapei-Palei", + "level2": "Central Torricelli", + "level3": "Wapeic", + "level4": "Yau-Yis" + }, + "yyz": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Nisoid", + "level7": "Nuclear Nisoid", + "level8": "Nasu-Nosu", + "level9": "Nesu-Nasu", + "level10": "Nasu-Gepu", + "level11": "Unclassified Nasu-Gepu" + }, + "yzg": { + "level0": "Tai-Kadai", + "level1": "Kadaic", + "level2": "Eastern Kra", + "level3": "Buyang", + "level4": "Northern Buyang" + }, + "yzk": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Ni-Li-Kazhouish", + "level5": "Southeastern Ngwi", + "level6": "Highland Phula", + "level7": "Phowa", + "level8": "Hlepho-Phukha", + "level9": "Khlula-Zokhuo" + }, + "zaa": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec", + "level8": "Sierra Juarezic" + }, + "zab": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Western Valley Zapotec" + }, + "zac": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Western Valley Zapotec", + "level9": "Extended Ocotepec Zapotec" + }, + "zad": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec", + "level8": "Cajonosic" + }, + "zae": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec", + "level8": "Sierra Juarezic" + }, + "zaf": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec" + }, + "zag": { + "level0": "Saharan", + "level1": "Eastern Saharan" + }, + "zah": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi East", + "level6": "Guruntumic", + "level7": "Tala-Sho-Zangwal", + "level8": "Tala-Zamwar" + }, + "zai": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Western Valley Zapotec", + "level9": "Extended Ocotepec Zapotec" + }, + "zaj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "East Ruvu", + "level11": "Central East Ruvu", + "level12": "Kutu-Zaramo" + }, + "zak": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "East Nyanza", + "level10": "Nyanza Mara", + "level11": "South Mara", + "level12": "Southwest Mara" + }, + "zal": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Lolo-Burmese", + "level3": "Loloish", + "level4": "Nusoish" + }, + "zam": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Miahuatecano", + "level8": "Miahuateco" + }, + "zao": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Miahuatecano", + "level8": "Miahuateco" + }, + "zaq": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec", + "level8": "Sierra Juarezic" + }, + "zar": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec", + "level8": "Rinconic" + }, + "zas": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec" + }, + "zat": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec", + "level8": "Cajonosic" + }, + "zau": { + "level0": "Sino-Tibetan", + "level1": "Bodic", + "level2": "Bodish", + "level3": "Early Old Tibetan", + "level4": "Western Archaic Tibetan", + "level5": "Kenhatic" + }, + "zav": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec", + "level8": "Cajonosic" + }, + "zaw": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec" + }, + "zax": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Cisyautepeque\u00f1o" + }, + "zay": { + "level0": "Ta-Ne-Omotic", + "level1": "Ometo", + "level2": "East Ometo" + }, + "zaz": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Southwest South Bauchi", + "level7": "Zakse-Saya" + }, + "zbc": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Berawan-Lower Baram", + "level5": "Berawan", + "level6": "Central-East Berawan" + }, + "zbe": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Berawan-Lower Baram", + "level5": "Berawan", + "level6": "Central-East Berawan" + }, + "zbl": { + "level0": "Artificial Language" + }, + "zbt": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Saluan-Banggai", + "level6": "Western Saluan-Banggai", + "level7": "Saluanic", + "level8": "Batui-Saluan" + }, + "zbu": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Northwest South Bauchi", + "level7": "Gejic" + }, + "zbw": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "North Borneo Malayo-Polynesian", + "level3": "North Sarawakan", + "level4": "Berawan-Lower Baram", + "level5": "Berawan" + }, + "zca": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Macrocoatecano" + }, + "zch": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Northern Tai", + "level7": "Hongshui He", + "level8": "Western Hongshui He" + }, + "zdj": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Coastal NEC Bantu", + "level10": "Mijikenda-Pokomo-Comorian", + "level11": "Comorian Bantu" + }, + "zea": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Germanic", + "level3": "Northwest Germanic", + "level4": "West Germanic", + "level5": "Macro-Dutch", + "level6": "Middle-Modern Dutch", + "level7": "Modern Dutch", + "level8": "Southwestern Dutch", + "level9": "Zeeuwic" + }, + "zeg": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "South Huon Gulf linkage", + "level8": "Buang linkage", + "level9": "Mumeng", + "level10": "Zenag-Patep" + }, + "zeh": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Northern Tai", + "level7": "Hongshui He", + "level8": "Western Hongshui He" + }, + "zem": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Southwest South Bauchi", + "level7": "Zeemic", + "level8": "Nuclear Zeemic" + }, + "zen": { + "level0": "Afro-Asiatic", + "level1": "Berber", + "level2": "Western Berber" + }, + "zga": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Southern Tanzania Highlands Bantu", + "level9": "Kinga-Magoma" + }, + "zgb": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Northern Tai" + }, + "zgm": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Debao-Jingxi-Nung" + }, + "zgn": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Northern Tai", + "level7": "Yei Zhuang" + }, + "zgr": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "Papuan Tip linkage", + "level6": "Peripheral Papuan Tip linkage", + "level7": "Central Papuan Oceanic", + "level8": "Oumic", + "level9": "Magoric" + }, + "zhb": { + "level0": "Sino-Tibetan", + "level1": "Burmo-Qiangic", + "level2": "Na-Qiangic", + "level3": "Qiangic" + }, + "zhd": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Wenma-Southwestern Tai" + }, + "zhi": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Benue-Congo Plateau", + "level4": "West-Central Benue-Congo Plateau", + "level5": "Northwestern Benue-Congo Plateau", + "level6": "Hyamic", + "level7": "Zhiric" + }, + "zhn": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Debao-Jingxi-Nung" + }, + "zhw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Wide Grassfields", + "level6": "Narrow Grassfields", + "level7": "Ring", + "level8": "Center-West Ring", + "level9": "West Ring", + "level10": "Aghemic" + }, + "zia": { + "level0": "Nuclear Trans New Guinea", + "level1": "Greater Binanderean", + "level2": "Binanderean", + "level3": "North Binanderean" + }, + "zib": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "zik": { + "level0": "Anim", + "level1": "Marind-Boazi-Yaqai", + "level2": "Boazi" + }, + "zil": { + "level0": "Mande", + "level1": "Western Mande", + "level2": "Manding-Kpelle", + "level3": "Southwest Mande", + "level4": "Mende-Loma", + "level5": "Mende-Bandi", + "level6": "Bandi-Zialo" + }, + "zim": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Masa", + "level3": "South Masa" + }, + "zin": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Great Lakes Bantu", + "level9": "West Nyanza" + }, + "ziw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Northeast Savanna Bantu", + "level8": "Northeast Coastal Bantu", + "level9": "Ruvu", + "level10": "West Ruvu", + "level11": "Seuta", + "level12": "Zigua-Nguu" + }, + "ziz": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Biu-Mandara", + "level3": "South Biu-Mandara", + "level4": "Bataic", + "level5": "Gudeic", + "level6": "Gude-Jimi-Zizilivakan" + }, + "zka": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Celebic", + "level3": "Greater Eastern Celebic", + "level4": "Eastern Celebic", + "level5": "Southeastern Celebic", + "level6": "Muna-Buton", + "level7": "Nuclear Muna-Buton", + "level8": "Munan", + "level9": "Munic" + }, + "zkg": { + "level0": "Unclassifiable" + }, + "zko": { + "level0": "Yeniseian" + }, + "zkp": { + "level0": "Nuclear-Macro-Je", + "level1": "Je", + "level2": "Southern Je", + "level3": "Kaingang-Xokleng", + "level4": "Kaingangic" + }, + "zkr": { + "level0": "Sino-Tibetan", + "level1": "Kman-Meyor" + }, + "zkt": { + "level0": "Mongolic-Khitan" + }, + "zku": { + "level0": "Pama-Nyungan", + "level1": "Arandic-Thura-Yura", + "level2": "Thura-Yura", + "level3": "Core Thura Yura", + "level4": "Southern Thura-Yura" + }, + "zla": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "Luban", + "level8": "Luba-Kaonde" + }, + "zlj": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Northern Tai", + "level7": "Lianshan-Liujiang" + }, + "zlm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Greater Riau-Johoric" + }, + "zln": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Northern Tai", + "level7": "Lianshan-Liujiang" + }, + "zlq": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Northern Tai", + "level7": "Hongshui He" + }, + "zlu": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi West", + "level6": "Northwest South Bauchi", + "level7": "Polci-Luri", + "level8": "Polcic", + "level9": "Zulic" + }, + "zma": { + "level0": "Western Daly", + "level1": "Maranunggu-Ame-Manda", + "level2": "Ame-Manda" + }, + "zmb": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega", + "level8": "Mituku-Lega", + "level9": "Songola-Binja" + }, + "zmc": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Southern Maric", + "level5": "Margany-Gunya" + }, + "zmd": { + "level0": "Western Daly", + "level1": "Bringen", + "level2": "Marithielic" + }, + "zme": { + "level0": "Giimbiyu" + }, + "zmf": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "Kwilu-Ngounie", + "level13": "Kasai-Ngounie" + }, + "zmg": { + "level0": "Western Daly", + "level1": "Bringen", + "level2": "Maringarr-Matige" + }, + "zmh": { + "level0": "Baining", + "level1": "Unclassified Baining" + }, + "zmi": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Northern Sumatra Malay", + "level6": "Kerinci-Minangkabau", + "level7": "Minangkabauic" + }, + "zmj": { + "level0": "Western Daly", + "level1": "Bringen", + "level2": "Marithielic" + }, + "zmk": { + "level0": "Pama-Nyungan", + "level1": "Greater Maric", + "level2": "Guwa-Maric", + "level3": "Maric", + "level4": "Southern Maric" + }, + "zml": { + "level0": "Eastern Daly" + }, + "zmm": { + "level0": "Western Daly", + "level1": "Bringen" + }, + "zmn": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Bantu A-B10-B20-B30", + "level7": "Ngomic", + "level8": "Nuclear Ngomic" + }, + "zmo": { + "level0": "Eastern Jebel", + "level1": "Aka-Kelo-Molo" + }, + "zmp": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "West-Coastal Bantu", + "level8": "Nzadic", + "level9": "Lweric", + "level10": "Dingic", + "level11": "Loange-Atlantic", + "level12": "KLC Extended" + }, + "zmq": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Greater Lega", + "level8": "Mituku-Lega", + "level9": "Mitukuic" + }, + "zmr": { + "level0": "Western Daly", + "level1": "Maranunggu-Ame-Manda" + }, + "zms": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic" + }, + "zmt": { + "level0": "Western Daly", + "level1": "Bringen", + "level2": "Maringarr-Matige" + }, + "zmu": { + "level0": "Pama-Nyungan", + "level1": "Southeastern Pama-Nyungan", + "level2": "New South Wales Pama-Nyungan", + "level3": "Muruwaric" + }, + "zmv": { + "level0": "Pama-Nyungan", + "level1": "Paman", + "level2": "Lamalamic" + }, + "zmw": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Inner Basin Bantu", + "level9": "Keleic", + "level10": "Kele-Poke", + "level11": "So-Poke", + "level12": "So-Lebonya", + "level13": "Lebonya", + "level14": "Bantu D33", + "level15": "Budu-Ndaka-Mbo", + "level16": "Ndaka-Mbo" + }, + "zmx": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "Central-Western Bantu", + "level7": "North Zaire River", + "level8": "Rivers Bantu", + "level9": "Likouala-Sangha", + "level10": "Impfondoic" + }, + "zmy": { + "level0": "Western Daly", + "level1": "Bringen", + "level2": "Marithielic" + }, + "zmz": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Bandaic", + "level6": "Nuclear Bandaic" + }, + "zna": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Bua-Kim-Day", + "level4": "Adamawa Bua", + "level5": "Inland Bua", + "level6": "Goulaic", + "level7": "Zan-Kulaalic" + }, + "zne": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "North Volta-Congo", + "level3": "Cameroun-Ubangian", + "level4": "Ubangi", + "level5": "Zandic", + "level6": "Zande-Nzakara" + }, + "zng": { + "level0": "Austroasiatic", + "level1": "Mangic" + }, + "znk": { + "level0": "Unattested", + "level1": "Iwaidjan Proper (Unattested)" + }, + "zns": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "West Chadic", + "level3": "West Chadic B", + "level4": "West Chadic B.3", + "level5": "South Bauchi East", + "level6": "Boghomic", + "level7": "Kir-Mangas" + }, + "zoc": { + "level0": "Mixe-Zoque", + "level1": "Zoque", + "level2": "Chiapas-Jitotolteco Zoque", + "level3": "Chiapas Zoque" + }, + "zoh": { + "level0": "Mixe-Zoque", + "level1": "Zoque" + }, + "zom": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Peripheral Kuki-Chin", + "level4": "Northeastern Kuki-Chin", + "level5": "Sizangic" + }, + "zoo": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec" + }, + "zoq": { + "level0": "Mixe-Zoque", + "level1": "Zoque", + "level2": "Gulf Zoque", + "level3": "Texistepec-Ayapa Zoque" + }, + "zor": { + "level0": "Mixe-Zoque", + "level1": "Zoque", + "level2": "Chiapas-Jitotolteco Zoque", + "level3": "Chiapas Zoque" + }, + "zos": { + "level0": "Mixe-Zoque", + "level1": "Zoque", + "level2": "Chiapas-Jitotolteco Zoque", + "level3": "Chiapas Zoque" + }, + "zpa": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Transyautepecan", + "level9": "Northeast Yautepec" + }, + "zpb": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Miahuatecano" + }, + "zpc": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec" + }, + "zpd": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec", + "level8": "Sierra Juarezic" + }, + "zpe": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Transyautepecan", + "level9": "Northeast Tehuantepec" + }, + "zpf": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec" + }, + "zpg": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Transyautepecan", + "level9": "Northeast Tehuantepec" + }, + "zph": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "West Zapotec" + }, + "zpi": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Cisyautepeque\u00f1o" + }, + "zpj": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Transyautepecan", + "level9": "Northeast Yautepec" + }, + "zpk": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Macrocoatecano", + "level7": "Amatecano" + }, + "zpl": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "West Zapotec", + "level6": "West-Central West Zapotec" + }, + "zpm": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Cisyautepeque\u00f1o", + "level9": "Mixtepec-Quioquitani-Quieri Zapotec" + }, + "zpn": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Western Valley Zapotec", + "level9": "Extended Ocotepec Zapotec", + "level10": "Tilquiapanic" + }, + "zpo": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Macrocoatecano", + "level7": "Amatecano" + }, + "zpp": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "West Zapotec", + "level6": "West-Central West Zapotec" + }, + "zpq": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec", + "level8": "Cajonosic" + }, + "zpr": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Cisyautepeque\u00f1o" + }, + "zps": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Macrocoatecano", + "level7": "Coatecano", + "level8": "Coatlan-Loxicha Zapotec" + }, + "zpt": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Macrocoatecano", + "level7": "Coatecano" + }, + "zpu": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec", + "level8": "Cajonosic" + }, + "zpv": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Western Valley Zapotec" + }, + "zpw": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Papabuco" + }, + "zpx": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Macrocoatecano", + "level7": "Coatecano", + "level8": "Coatlan-Loxicha Zapotec" + }, + "zpy": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec" + }, + "zpz": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Papabuco" + }, + "zqe": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Northern Tai", + "level7": "Yei Zhuang" + }, + "zrg": { + "level0": "Bookkeeping" + }, + "zrn": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "East Chadic", + "level3": "East Chadic B", + "level4": "East Chadic B.1", + "level5": "Mubic" + }, + "zro": { + "level0": "Zaparoan", + "level1": "Zaparo-Abishira" + }, + "zrp": { + "level0": "Bookkeeping" + }, + "zrs": { + "level0": "Mairasic" + }, + "zsa": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Upper Markham", + "level9": "Mountain Upper Markham" + }, + "zsl": { + "level0": "Sign Language", + "level1": "L1 Sign Language" + }, + "zsm": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Malayo-Chamic", + "level3": "Malayic", + "level4": "Nuclear Malayic", + "level5": "Standard Malay-Indonesian" + }, + "zsu": { + "level0": "Austronesian", + "level1": "Malayo-Polynesian", + "level2": "Eastern Malayo-Polynesian", + "level3": "Oceanic", + "level4": "Western Oceanic linkage", + "level5": "North New Guinea linkage", + "level6": "Huon Gulf", + "level7": "Markham", + "level8": "Upper Markham", + "level9": "Mountain Upper Markham" + }, + "ztc": { + "level0": "Bookkeeping" + }, + "zte": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Papabuco" + }, + "ztg": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Cisyautepeque\u00f1o" + }, + "ztl": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Cisyautepeque\u00f1o" + }, + "ztm": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Miahuatecano", + "level8": "Miahuateco" + }, + "ztn": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec" + }, + "ztp": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Miahuatecano", + "level8": "Miahuateco" + }, + "ztq": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Cisyautepeque\u00f1o", + "level9": "Mixtepec-Quioquitani-Quieri Zapotec" + }, + "zts": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Western Valley Zapotec", + "level9": "Extended Ocotepec Zapotec", + "level10": "Tilquiapanic" + }, + "ztt": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec" + }, + "ztu": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Western Valley Zapotec" + }, + "ztx": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Central Core Zapotec", + "level8": "Western Valley Zapotec", + "level9": "Extended Ocotepec Zapotec" + }, + "zty": { + "level0": "Otomanguean", + "level1": "Eastern Otomanguean", + "level2": "Popoloca-Zapotecan", + "level3": "Zapotecan", + "level4": "Zapotec", + "level5": "Core Zapotec", + "level6": "Narrow Core Zapotec", + "level7": "Northern Core Zapotec", + "level8": "Rinconic" + }, + "zuh": { + "level0": "Nuclear Trans New Guinea", + "level1": "Kainantu-Goroka", + "level2": "Goroka", + "level3": "Nuclear Goroka", + "level4": "Gahuku" + }, + "zul": { + "level0": "Atlantic-Congo", + "level1": "Volta-Congo", + "level2": "Benue-Congo", + "level3": "Bantoid", + "level4": "Southern Bantoid", + "level5": "Narrow Bantu", + "level6": "East Bantu", + "level7": "Southern Bantu", + "level8": "Nuclear Southern Bantu", + "level9": "Dimsuffix Southern Bantu", + "level10": "Nguni-Tsonga-Copi", + "level11": "Nguni (S.40)", + "level12": "Nuclear Nguni", + "level13": "Southern Ndebele-Lowland" + }, + "zum": { + "level0": "Indo-European", + "level1": "Classical Indo-European", + "level2": "Indo-Iranian", + "level3": "Iranian", + "level4": "Southwestern Iranian", + "level5": "Middle-Modern Persian", + "level6": "Modern Southwestern Iranian" + }, + "zuy": { + "level0": "Afro-Asiatic", + "level1": "Chadic", + "level2": "Masa", + "level3": "North Masa", + "level4": "Unclassified North Masa" + }, + "zwa": { + "level0": "Afro-Asiatic", + "level1": "Semitic", + "level2": "West Semitic", + "level3": "Ethiosemitic", + "level4": "South Ethiopic", + "level5": "Harari-East Gurage" + }, + "zyb": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Yongnan-Yongbei" + }, + "zyg": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Central-Southwestern Tai", + "level5": "Debao-Jingxi-Nung" + }, + "zyj": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Northern Tai" + }, + "zyn": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic", + "level6": "Yongnan-Yongbei" + }, + "zyp": { + "level0": "Sino-Tibetan", + "level1": "Kuki-Chin-Naga", + "level2": "Kuki-Chin", + "level3": "Central Kuki-Chin", + "level4": "Maraic", + "level5": "Nuclear Maraic" + }, + "zzj": { + "level0": "Tai-Kadai", + "level1": "Kam-Tai", + "level2": "Daic-Beic", + "level3": "Daic", + "level4": "Northern Daic-Sek", + "level5": "Northern Daic" + } +} \ No newline at end of file diff --git a/mteb/languages.py b/mteb/languages.py index f7f6477503..e83dd308cd 100644 --- a/mteb/languages.py +++ b/mteb/languages.py @@ -5,6 +5,7 @@ from __future__ import annotations import json +from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path @@ -17,7 +18,7 @@ # Language mappings path_to_lang_codes = Path(__file__).parent / "iso_639_3_to_language.json" path_to_lang_scripts = Path(__file__).parent / "iso_15924_to_script.json" - +path_to_lang_fam = Path(__file__).parent / "language_family.json" with path_to_lang_codes.open("r") as f: ISO_TO_LANGUAGE = json.load(f) @@ -25,6 +26,11 @@ with path_to_lang_scripts.open("r") as f: ISO_TO_SCRIPT = json.load(f) +with path_to_lang_fam.open("r") as f: + ISO_TO_FAM = json.load(f) + +ISO_TO_FAM_LEVEL0 = {k: v["level0"] for k, v in ISO_TO_FAM.items()} + @dataclass class LanguageScripts: @@ -76,5 +82,19 @@ def contains_language(self, language: str) -> bool: return True return False + def contains_languages(self, languages: Iterable[str]) -> bool: + """Whether is contains all of the languages""" + for l in languages: + if not self.contains_language(l): + return False + return True + def contains_script(self, script: str) -> bool: return script in self.scripts + + def contains_scripts(self, scripts: Iterable[str]) -> bool: + """Whether is contains all of the scripts""" + for s in scripts: + if not self.contains_script(s): + return False + return True diff --git a/mteb/leaderboard/app.py b/mteb/leaderboard/app.py index 7d49d009d1..5ee5a6b9da 100644 --- a/mteb/leaderboard/app.py +++ b/mteb/leaderboard/app.py @@ -1,34 +1,195 @@ from __future__ import annotations -from collections import defaultdict +import itertools +import json +import logging +import tempfile +import time from pathlib import Path +from typing import Literal +from urllib.parse import urlencode import gradio as gr +import pandas as pd from gradio_rangeslider import RangeSlider import mteb +from mteb.caching import json_cache +from mteb.leaderboard.figures import performance_size_plot, radar_chart from mteb.leaderboard.table import scores_to_tables +logger = logging.getLogger(__name__) + def load_results(): results_cache_path = Path(__file__).parent.joinpath("__cached_results.json") if not results_cache_path.exists(): - all_results = mteb.load_results() + all_results = ( + mteb.load_results(only_main_score=True, require_model_meta=False) + .join_revisions() + .filter_models() + ) all_results.to_disk(results_cache_path) return all_results else: - return mteb.BenchmarkResults.from_disk(results_cache_path) + with results_cache_path.open() as cache_file: + return mteb.BenchmarkResults.from_validated(**json.load(cache_file)) + + +def produce_benchmark_link(benchmark_name: str, request: gr.Request) -> str: + """Produces a URL for the selected benchmark.""" + params = urlencode( + { + "benchmark_name": benchmark_name, + } + ) + base_url = request.request.base_url + url = f"{base_url}?{params}" + md = f"```\n{url}\n```" + return md + + +DEFAULT_BENCHMARK_NAME = "MTEB(Multilingual)" + + +def set_benchmark_on_load(request: gr.Request): + query_params = request.query_params + return query_params.get("benchmark_name", DEFAULT_BENCHMARK_NAME) + + +def download_table(table: pd.DataFrame) -> Path: + file = tempfile.NamedTemporaryFile(delete=False, suffix=".csv") + table.to_csv(file) + return file.name + + +def update_citation(benchmark_name: str) -> str: + benchmark = mteb.get_benchmark(benchmark_name) + if str(benchmark.citation) != "None": + citation = f"```bibtex\n{benchmark.citation}\n```" + else: + citation = "" + return citation + +def update_description( + benchmark_name: str, languages: list[str], task_types: list[str], domains: list[str] +) -> str: + benchmark = mteb.get_benchmark(benchmark_name) + description = f"## {benchmark.name}\n{benchmark.description}\n" + n_languages = len(languages) + n_task_types = len(task_types) + n_tasks = len(benchmark.tasks) + n_domains = len(domains) + description += f" - **Number of languages**: {n_languages}\n" + description += f" - **Number of datasets**: {n_tasks}\n" + description += f" - **Number of task types**: {n_task_types}\n" + description += f" - **Number of domains**: {n_domains}\n" + if str(benchmark.reference) != "None": + description += f"\n[Click for More Info]({benchmark.reference})" + + return description + + +def format_list(props: list[str]): + if props is None: + return "" + if len(props) > 3: + return ", ".join(props[:3]) + "..." + return ", ".join(props) + + +def update_task_info(task_names: str) -> gr.DataFrame: + tasks = mteb.get_tasks(tasks=task_names) + df = tasks.to_dataframe( + properties=["name", "type", "languages", "domains", "reference", "main_score"] + ) + df["languages"] = df["languages"].map(format_list) + df = df.sort_values("name") + df["domains"] = df["domains"].map(format_list) + df["name"] = "[" + df["name"] + "](" + df["reference"] + ")" + df = df.rename( + columns={ + "name": "Task Name", + "type": "Task Type", + "languages": "Languages", + "domains": "Domains", + "main_score": "Metric", + } + ) + df = df.drop(columns="reference") + return gr.DataFrame(df, datatype=["markdown"] + ["str"] * (len(df.columns) - 1)) -all_results = load_results().filter_models() # Model sizes in million parameters -min_model_size, max_model_size = 8, 46703 +MIN_MODEL_SIZE, MAX_MODEL_SIZE = 0, 10_000 + + +def filter_models( + model_names, + task_select, + availability, + compatibility, + instructions, + model_size, + zero_shot_setting, +): + lower, upper = model_size + # Setting to None, when the user doesn't specify anything + if (lower == MIN_MODEL_SIZE) and (upper == MAX_MODEL_SIZE): + lower, upper = None, None + else: + # Multiplying by millions + lower = lower * 1e6 + upper = upper * 1e6 + model_metas = mteb.get_model_metas( + model_names=model_names, + open_weights=availability, + use_instructions=instructions, + frameworks=compatibility, + n_parameters_range=(lower, upper), + ) + tasks = mteb.get_tasks(tasks=task_select) + models_to_keep = set() + for model_meta in model_metas: + is_model_zero_shot = model_meta.is_zero_shot_on(tasks) + if is_model_zero_shot is None: + if zero_shot_setting == "hard": + continue + elif not is_model_zero_shot: + if zero_shot_setting != "off": + continue + models_to_keep.add(model_meta.name) + return list(models_to_keep) + + +logger.info("Loading all benchmark results") +all_results = load_results() benchmarks = mteb.get_benchmarks() +all_benchmark_results = { + benchmark.name: benchmark.load_results(base_results=all_results) + for benchmark in benchmarks +} +default_benchmark = mteb.get_benchmark(DEFAULT_BENCHMARK_NAME) +default_results = all_benchmark_results[default_benchmark.name] +logger.info("Benchmark results loaded") -default_benchmark = mteb.get_benchmark("MTEB(Multilingual)") -default_results = default_benchmark.load_results(base_results=all_results) +default_scores = default_results.get_scores(format="long") +all_models = list({entry["model_name"] for entry in default_scores}) +filtered_models = filter_models( + all_models, + default_results.task_names, + availability=None, + compatibility=[], + instructions=None, + model_size=(MIN_MODEL_SIZE, MAX_MODEL_SIZE), + zero_shot_setting="soft", +) + +summary_table, per_task_table = scores_to_tables( + [entry for entry in default_scores if entry["model_name"] in filtered_models] +) benchmark_select = gr.Dropdown( [bench.name for bench in benchmarks], @@ -37,29 +198,29 @@ def load_results(): info="Select one of our expert-selected benchmarks from MTEB publications.", ) lang_select = gr.Dropdown( - default_results.languages, - value=default_results.languages, + all_results.languages, + value=sorted(default_results.languages), multiselect=True, label="Language", info="Select languages to include.", ) type_select = gr.Dropdown( - default_results.task_types, - value=default_results.task_types, + all_results.task_types, + value=sorted(default_results.task_types), multiselect=True, label="Task Type", info="Select task types to include.", ) domain_select = gr.Dropdown( - default_results.domains, - value=default_results.domains, + all_results.domains, + value=sorted(default_results.domains), multiselect=True, label="Domain", info="Select domains to include.", ) task_select = gr.Dropdown( - default_results.task_names, - value=default_results.task_names, + all_results.task_names, + value=sorted(default_results.task_names), allow_custom_value=True, multiselect=True, label="Task", @@ -72,7 +233,26 @@ def load_results(): with gr.Blocks(fill_width=True, theme=gr.themes.Base(), head=head) as demo: with gr.Row(): - with gr.Column(scale=1): + with gr.Column(scale=5): + gr.Markdown( + """ + ### Benchmarks + Select one of the hand-curated benchmarks from our publications and modify them using one of the following filters to fit your needs. + """ + ) + with gr.Group(): + with gr.Row(elem_classes="overflow-y-scroll max-h-80"): + with gr.Column(): + benchmark_select.render() + with gr.Accordion("Select Languages", open=False): + lang_select.render() + with gr.Accordion("Select Task Types", open=False): + type_select.render() + with gr.Accordion("Select Domains", open=False): + domain_select.render() + with gr.Accordion("Add and remove tasks:", open=False): + task_select.render() + with gr.Column(scale=8): gr.Markdown( """ ### Model Selection @@ -80,7 +260,24 @@ def load_results(): """, ) with gr.Group(): - with gr.Row(elem_classes="overflow-y-scroll max-h-80"): + with gr.Row(): + searchbar = gr.Textbox( + label="Search Models", + info="Press Enter to search.\nSearch models by name (RegEx sensitive. Separate queries with `|`)", + interactive=True, + ) + compatibility = gr.CheckboxGroup( + [ + ( + "Should be sentence-transformers compatible", + "Sentence Transformers", + ) + ], + value=[], + label="Compatibility", + interactive=True, + ) + with gr.Row(elem_classes=""): with gr.Column(): availability = gr.Radio( [ @@ -92,17 +289,6 @@ def load_results(): label="Availability", interactive=True, ) - compatibility = gr.CheckboxGroup( - [ - ( - "Should be sentence-transformers compatible", - "sbert_compatible", - ) - ], - value=[], - label="Compatibility", - interactive=True, - ) instructions = gr.Radio( [ ("Only Instruction-tuned", True), @@ -113,110 +299,338 @@ def load_results(): label="Instructions", interactive=True, ) + with gr.Column(): + zero_shot = gr.Radio( + [ + ( + "Only Zero-shot", + "hard", + ), + ("Allow Unknown", "soft"), + ("Allow all", "off"), + ], + value="soft", + label="Zero-shot", + interactive=True, + ) model_size = RangeSlider( - minimum=0, - maximum=8000, - value=(0, 8000), + minimum=MIN_MODEL_SIZE, + maximum=MAX_MODEL_SIZE, + value=(MIN_MODEL_SIZE, MAX_MODEL_SIZE), label="Model Size (#M Parameters)", interactive=True, ) - with gr.Column(scale=2): + scores = gr.State(default_scores) + models = gr.State(filtered_models) + with gr.Row(): + with gr.Column(): + description = gr.Markdown( + update_description, + inputs=[benchmark_select, lang_select, type_select, domain_select], + ) + citation = gr.Markdown(update_citation, inputs=[benchmark_select]) + with gr.Accordion("Share this benchmark:", open=False): + gr.Markdown(produce_benchmark_link, inputs=[benchmark_select]) + with gr.Column(): + with gr.Tab("Performance per Model Size"): + plot = gr.Plot(performance_size_plot, inputs=[summary_table]) + gr.Markdown( + "*We only display models that have been run on all tasks in the benchmark*" + ) + with gr.Tab("Performance per Task Type (Radar Chart)"): + radar_plot = gr.Plot(radar_chart, inputs=[summary_table]) + gr.Markdown( + "*We only display models that have been run on all task types in the benchmark*" + ) + with gr.Tab("Summary"): + gr.Markdown( + """ + ✅ - Model is zero-shot on the benchmark
+ ⚠️ - Training data unknown
+ ❌ - Model is **NOT** zero-shot on the benchmark + """ + ) + summary_table.render() + download_summary = gr.DownloadButton("Download Table") + download_summary.click( + download_table, inputs=[summary_table], outputs=[download_summary] + ) + with gr.Accordion( + "What do aggregate measures (Rank(Borda), Mean(Task), etc.) mean?", + open=False, + ): gr.Markdown( """ - ### Benchmarks - Select one of the hand-curated benchmarks from our publication. - Or create one from scratch based on your use case. + **Rank(borda)** is computed based on the [borda count](https://en.wikipedia.org/wiki/Borda_count), where each task is treated as a preference voter, which gives votes on the models in accordance with their relative performance on the task. The best model obtains the highest number of votes. The model with the highest number of votes across tasks obtains the highest rank. The Borda rank tends to prefer models that perform well broadly across tasks. However, given that it is a rank it can be unclear if the two models perform similarly. + + **Mean(Task)**: This is a naïve average computed across all the tasks within the benchmark. This score is simple to understand and is continuous as opposed to the Borda rank. However, the mean can overvalue tasks with higher variance in its scores. + + **Mean(TaskType)**: This is a weighted average across different task categories, such as classification or retrieval. It is computed by first computing the average by task category and then computing the average on each category. Similar to the Mean(Task) this measure is continuous and tends to overvalue tasks with higher variance. This score also prefers models that perform well across all task categories. + """ + ) + with gr.Accordion( + "What does zero-shot mean?", + open=False, + ): + gr.Markdown( + """ +A model is considered zero-shot if it is not trained on any splits of the datasets used to derive the tasks. +E.g., if a model is trained on Natural Questions, it cannot be considered zero-shot on benchmarks containing the task “NQ” which is derived from Natural Questions. +This definition creates a few edge cases. For instance, multiple models are typically trained on Wikipedia title and body pairs, but we do not define this as leakage on, e.g., “WikipediaRetrievalMultilingual” and “WikiClusteringP2P” as these datasets are not based on title-body pairs. +Distilled, further fine-tunes or in other ways, derivative models inherit the datasets of their parent models. +Based on community feedback and research findings, This definition could change in the future. """ ) - with gr.Group(): - with gr.Row(elem_classes="overflow-y-scroll max-h-80"): - with gr.Column(): - benchmark_select.render() - with gr.Accordion("Select Languages", open=False): - lang_select.render() - with gr.Accordion("Select Task Types", open=False): - type_select.render() - with gr.Accordion("Select Domains", open=False): - domain_select.render() - with gr.Accordion("Add and remove tasks:", open=False): - task_select.render() - default_scores = default_results.get_scores(format="long") - scores = gr.State(default_scores) - summary, per_task = scores_to_tables(default_scores) - with gr.Tab("Summary"): - summary_table = gr.DataFrame(summary) with gr.Tab("Performance per task"): - per_task_table = gr.DataFrame(per_task) + per_task_table.render() + download_per_task = gr.DownloadButton("Download Table") + download_per_task.click( + download_table, inputs=[per_task_table], outputs=[download_per_task] + ) + with gr.Tab("Task information"): + task_info_table = gr.DataFrame(update_task_info, inputs=[task_select]) - @gr.on(inputs=[scores], outputs=[summary_table, per_task_table]) - def update_tables(scores): - summary, per_task = scores_to_tables(scores) - return summary, per_task + # This sets the benchmark from the URL query parameters + demo.load(set_benchmark_on_load, inputs=[], outputs=[benchmark_select]) - @gr.on( - inputs=[benchmark_select], - outputs=[ - lang_select, - type_select, - domain_select, - ], - ) - def on_select_benchmark(benchmark_name): + @json_cache + def on_benchmark_select(benchmark_name): + start_time = time.time() benchmark = mteb.get_benchmark(benchmark_name) - benchmark_results = benchmark.load_results(base_results=all_results) + languages = [task.languages for task in benchmark.tasks if task.languages] + languages = set(itertools.chain.from_iterable(languages)) + languages = sorted(languages) + domains = [ + task.metadata.domains for task in benchmark.tasks if task.metadata.domains + ] + domains = set(itertools.chain.from_iterable(domains)) + types = {task.metadata.type for task in benchmark.tasks if task.metadata.type} + languages, domains, types = ( + sorted(languages), + sorted(domains), + sorted(types), + ) + elapsed = time.time() - start_time + benchmark_results = all_benchmark_results[benchmark_name] + scores = benchmark_results.get_scores(format="long") + logger.info(f"on_benchmark_select callback: {elapsed}s") return ( - benchmark_results.languages, - benchmark_results.task_types, - benchmark_results.domains, + languages, + domains, + types, + [task.metadata.name for task in benchmark.tasks], + scores, ) - @gr.on( - inputs=[benchmark_select, lang_select, type_select, domain_select], - outputs=[task_select], + benchmark_select.change( + on_benchmark_select, + inputs=[benchmark_select], + outputs=[lang_select, domain_select, type_select, task_select, scores], ) - def update_task_list(benchmark_name, languages, task_types, domains): - benchmark = mteb.get_benchmark(benchmark_name) - benchmark_results = benchmark.load_results(base_results=all_results) - task_to_lang_set = defaultdict(set) - task_to_type = {} - task_to_domains = defaultdict(set) - for model_res in benchmark_results: - for task_res in model_res: - task_to_lang_set[task_res.task_name] |= set(task_res.languages) - task_to_domains[task_res.task_name] |= set(task_res.domains) - task_to_type[task_res.task_name] = task_res.task_type - res = [] - for task_name in benchmark_results.task_names: - if not (task_to_domains[task_name] & set(domains)): + + @json_cache + def update_scores_on_lang_change(benchmark_name, languages): + start_time = time.time() + benchmark_results = all_benchmark_results[benchmark_name] + scores = benchmark_results.get_scores(languages=languages, format="long") + elapsed = time.time() - start_time + logger.info(f"update_scores callback: {elapsed}s") + return scores + + lang_select.input( + update_scores_on_lang_change, + inputs=[benchmark_select, lang_select], + outputs=[scores], + ) + + def update_task_list(benchmark_name, type_select, domain_select, lang_select): + start_time = time.time() + tasks_to_keep = [] + for task in mteb.get_benchmark(benchmark_name).tasks: + if task.metadata.type not in type_select: continue - if not (task_to_lang_set[task_name] & set(languages)): + if not (set(task.metadata.domains or []) & set(domain_select)): continue - if task_to_type[task_name] not in task_types: + if not (set(task.languages or []) & set(lang_select)): continue - res.append(task_name) - return res + tasks_to_keep.append(task.metadata.name) + elapsed = time.time() - start_time + logger.info(f"update_task_list callback: {elapsed}s") + return tasks_to_keep + + type_select.input( + update_task_list, + inputs=[benchmark_select, type_select, domain_select, lang_select], + outputs=[task_select], + ) + domain_select.input( + update_task_list, + inputs=[benchmark_select, type_select, domain_select, lang_select], + outputs=[task_select], + ) + lang_select.input( + update_task_list, + inputs=[benchmark_select, type_select, domain_select, lang_select], + outputs=[task_select], + ) + + def update_models( + scores: list[dict], + tasks: list[str], + availability: bool | None, + compatibility: list[str], + instructions: bool | None, + model_size: tuple[int, int], + zero_shot: Literal["hard", "soft", "off"], + ): + start_time = time.time() + model_names = list({entry["model_name"] for entry in scores}) + filtered_models = filter_models( + model_names, + tasks, + availability, + compatibility, + instructions, + model_size, + zero_shot_setting=zero_shot, + ) + elapsed = time.time() - start_time + logger.info(f"update_models callback: {elapsed}s") + return filtered_models - @gr.on( + scores.change( + update_models, inputs=[ - benchmark_select, + scores, task_select, - lang_select, - type_select, - domain_select, + availability, + compatibility, + instructions, + model_size, + zero_shot, ], - outputs=[scores], + outputs=[models], + ) + task_select.change( + update_models, + inputs=[ + scores, + task_select, + availability, + compatibility, + instructions, + model_size, + zero_shot, + ], + outputs=[models], + ) + availability.input( + update_models, + inputs=[ + scores, + task_select, + availability, + compatibility, + instructions, + model_size, + zero_shot, + ], + outputs=[models], + ) + compatibility.input( + update_models, + inputs=[ + scores, + task_select, + availability, + compatibility, + instructions, + model_size, + zero_shot, + ], + outputs=[models], + ) + instructions.input( + update_models, + inputs=[ + scores, + task_select, + availability, + compatibility, + instructions, + model_size, + zero_shot, + ], + outputs=[models], + ) + model_size.input( + update_models, + inputs=[ + scores, + task_select, + availability, + compatibility, + instructions, + model_size, + zero_shot, + ], + outputs=[models], + ) + zero_shot.change( + update_models, + inputs=[ + scores, + task_select, + availability, + compatibility, + instructions, + model_size, + zero_shot, + ], + outputs=[models], + ) + + def update_tables( + scores, + search_query: str, + tasks, + models_to_keep, + ): + start_time = time.time() + tasks = set(tasks) + models_to_keep = set(models_to_keep) + filtered_scores = [] + for entry in scores: + if entry["task_name"] not in tasks: + continue + if entry["model_name"] not in models_to_keep: + continue + filtered_scores.append(entry) + summary, per_task = scores_to_tables(filtered_scores, search_query) + elapsed = time.time() - start_time + logger.info(f"update_tables callback: {elapsed}s") + return summary, per_task + + task_select.change( + update_tables, + inputs=[scores, searchbar, task_select, models], + outputs=[summary_table, per_task_table], + ) + scores.change( + update_tables, + inputs=[scores, searchbar, task_select, models], + outputs=[summary_table, per_task_table], + ) + models.change( + update_tables, + inputs=[scores, searchbar, task_select, models], + outputs=[summary_table, per_task_table], + ) + searchbar.submit( + update_tables, + inputs=[scores, searchbar, task_select, models], + outputs=[summary_table, per_task_table], ) - def update_scores(benchmark_name, task_names, languages, task_types, domains): - benchmark = mteb.get_benchmark(benchmark_name) - benchmark_results = benchmark.load_results(base_results=all_results) - benchmark_results = benchmark_results.filter_tasks( - languages=languages, - task_names=task_names, - task_types=task_types, - domains=domains, - ) - scores = benchmark_results.get_scores(languages=languages, format="long") - return scores if __name__ == "__main__": diff --git a/mteb/leaderboard/figures.py b/mteb/leaderboard/figures.py new file mode 100644 index 0000000000..8ae0b6f8b5 --- /dev/null +++ b/mteb/leaderboard/figures.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import plotly.express as px +import plotly.graph_objects as go + + +def text_plot(text: str): + """Returns empty scatter plot with text added, this can be great for error messages.""" + return px.scatter(template="plotly_white").add_annotation( + text=text, showarrow=False, font=dict(size=20) + ) + + +def failsafe_plot(fun): + """Decorator that turns the function producing a figure failsafe. + This is necessary, because once a Callback encounters an exception it + becomes useless in Gradio. + """ + + def wrapper(*args, **kwargs): + try: + return fun(*args, **kwargs) + except Exception as e: + return text_plot(f"Couldn't produce plot. Reason: {e}") + + return wrapper + + +def parse_n_params(text: str) -> int: + if text.endswith("M"): + return float(text[:-1]) * 1e6 + if text.endswith("B"): + return float(text[:-1]) * 1e9 + + +def parse_model_name(name: str) -> str: + if name is None: + return "" + if "]" not in name: + return name + name, _ = name.split("]") + return name[1:] + + +def parse_float(value) -> float: + try: + return float(value) + except ValueError: + return np.nan + + +models_to_annotate = [ + "all-MiniLM-L6-v2", + "GritLM-7B", + "LaBSE", + "multilingual-e5-large-instruct", +] + + +def add_size_guide(fig: go.Figure): + xpos = [5 * 1e9] * 4 + ypos = [7.8, 8.5, 9, 10] + sizes = [256, 1024, 2048, 4096] + fig.add_trace( + go.Scatter( + showlegend=False, + opacity=0.3, + mode="markers", + marker=dict( + size=np.sqrt(sizes), + color="rgba(0,0,0,0)", + line=dict(color="black", width=2), + ), + x=xpos, + y=ypos, + ) + ) + fig.add_annotation( + text="Embedding Size:", + font=dict(size=16), + x=np.log10(1.5e9), + y=10, + showarrow=False, + opacity=0.3, + ) + for x, y, size in zip(xpos, np.linspace(7.5, 14, 4), sizes): + fig.add_annotation( + text=f"{size}", + font=dict(size=12), + x=np.log10(x), + y=y, + showarrow=True, + ay=0, + ax=50, + opacity=0.3, + arrowwidth=2, + ) + return fig + + +@failsafe_plot +def performance_size_plot(df: pd.DataFrame) -> go.Figure: + df = df.copy() + df["Number of Parameters"] = df["Number of Parameters"].map(parse_n_params) + df["Model"] = df["Model"].map(parse_model_name) + df["model_text"] = df["Model"].where(df["Model"].isin(models_to_annotate), "") + df["Embedding Dimensions"] = df["Embedding Dimensions"].map(parse_float) + df["Max Tokens"] = df["Max Tokens"].map(parse_float) + df["Log(Tokens)"] = np.log10(df["Max Tokens"]) + df["Mean (Task)"] = df["Mean (Task)"].map(parse_float) + df = df.dropna( + subset=["Mean (Task)", "Number of Parameters", "Embedding Dimensions"] + ) + if not len(df.index): + return go.Figure() + min_score, max_score = df["Mean (Task)"].min(), df["Mean (Task)"].max() + df["sqrt(dim)"] = np.sqrt(df["Embedding Dimensions"]) + fig = px.scatter( + df, + x="Number of Parameters", + y="Mean (Task)", + log_x=True, + template="plotly_white", + text="model_text", + size="sqrt(dim)", + color="Log(Tokens)", + range_color=[2, 5], + range_x=[8 * 1e6, 11 * 1e9], + range_y=[min(0, min_score * 1.25), max_score * 1.25], + hover_data={ + "Max Tokens": True, + "Embedding Dimensions": True, + "Number of Parameters": True, + "Mean (Task)": True, + "Rank (Borda)": True, + "Log(Tokens)": False, + "sqrt(dim)": False, + "model_text": False, + }, + hover_name="Model", + ) + # Note: it's important that this comes before setting the size mode + fig = add_size_guide(fig) + fig.update_traces( + marker=dict( + sizemode="diameter", + sizeref=1.5, + sizemin=0, + ) + ) + fig.add_annotation(x=1e9, y=10, text="Model size:") + fig.update_layout( + coloraxis_colorbar=dict( # noqa + title="Max Tokens", + tickvals=[2, 3, 4, 5], + ticktext=[ + "100", + "1K", + "10K", + "100K", + ], + ), + hoverlabel=dict( # noqa + bgcolor="white", + font_size=16, + ), + ) + fig.update_traces( + textposition="top center", + ) + fig.update_layout( + font=dict(size=16, color="black"), # noqa + margin=dict(b=20, t=10, l=20, r=10), # noqa + ) + return fig + + +TOP_N = 5 +task_types = [ + "BitextMining", + "Classification", + "MultilabelClassification", + "Clustering", + "PairClassification", + "Reranking", + "Retrieval", + "STS", + "Summarization", + # "InstructionRetrieval", + # Not displayed, because the scores are negative, + # doesn't work well with the radar chart. + "Speed", +] + +line_colors = [ + "#EE4266", + "#00a6ed", + "#ECA72C", + "#B42318", + "#3CBBB1", +] +fill_colors = [ + "rgba(238,66,102,0.05)", + "rgba(0,166,237,0.05)", + "rgba(236,167,44,0.05)", + "rgba(180,35,24,0.05)", + "rgba(60,187,177,0.05)", +] + + +@failsafe_plot +def radar_chart(df: pd.DataFrame) -> go.Figure: + df = df.copy() + df["Model"] = df["Model"].map(parse_model_name) + # Remove whitespace + task_type_columns = [ + column for column in df.columns if "".join(column.split()) in task_types + ] + if len(task_type_columns) <= 1: + raise ValueError( + "Couldn't produce radar chart, the benchmark only contains one task category." + ) + df = df[["Model", *task_type_columns]].set_index("Model") + df = df.replace("", np.nan) + df = df.dropna() + df = df.head(TOP_N) + df = df.iloc[::-1] + fig = go.Figure() + for i, (model_name, row) in enumerate(df.iterrows()): + fig.add_trace( + go.Scatterpolar( + name=model_name, + r=[row[task_type] for task_type in task_type_columns] + + [row[task_type_columns[0]]], + theta=task_type_columns + [task_type_columns[0]], + showlegend=True, + mode="lines", + line=dict(width=2, color=line_colors[i]), + fill="toself", + fillcolor="rgba(0,0,0,0)", + ) + ) + fig.update_layout( + font=dict(size=16, color="black"), # noqa + template="plotly_white", + polar=dict( + radialaxis=dict( + visible=True, + gridcolor="black", + linecolor="rgba(0,0,0,0)", + gridwidth=1, + showticklabels=False, + ticks="", + ), + angularaxis=dict( + gridcolor="black", gridwidth=1.5, linecolor="rgba(0,0,0,0)" + ), + ), + legend=dict( + orientation="h", + yanchor="bottom", + y=-0.6, + xanchor="left", + x=-0.05, + entrywidthmode="fraction", + entrywidth=1 / 5, + ), + ) + return fig diff --git a/mteb/leaderboard/table.py b/mteb/leaderboard/table.py index 570d5bc6dd..ef28392cf7 100644 --- a/mteb/leaderboard/table.py +++ b/mteb/leaderboard/table.py @@ -1,89 +1,251 @@ from __future__ import annotations +import math +import re +from collections import defaultdict + import gradio as gr import numpy as np import pandas as pd +from pandas.api.types import is_numeric_dtype + +from mteb.models.overview import get_model_meta +from mteb.overview import get_task, get_tasks + + +def borda_count(scores: pd.Series) -> pd.Series: + n = len(scores) + ranks = scores.rank(method="average", ascending=False) + counts = n - ranks + return counts + -from mteb.overview import get_task +def get_borda_rank(score_table: pd.DataFrame) -> pd.Series: + borda_counts = score_table.apply(borda_count, axis="index") + mean_borda = borda_counts.sum(axis=1) + return mean_borda.rank(method="min", ascending=False).astype(int) def format_scores(score: float) -> float: - return score * 100 + return round(score * 100, 2) + + +def format_n_parameters(n_parameters) -> str: + if (n_parameters is None) or (not int(n_parameters)): + return "Unknown" + n_thousand = int(n_parameters // 1e3) + if n_thousand < 1: + return str(int(n_parameters)) + n_zeros = math.log10(n_thousand) + if n_zeros >= 6: + return str(n_thousand // (10**6)) + "B" + if n_zeros >= 3: + return str(n_thousand // (10**3)) + "M" + return str(n_thousand) + "K" + + +def split_on_capital(s: str) -> str: + """Splits on capital letters and joins with spaces""" + return " ".join(re.findall(r"[A-Z]?[a-z]+|[A-Z]+(?=[A-Z]|$)", s)) + + +def get_column_widths(df: pd.DataFrame) -> list[str]: + widths = [] + for column_name in df.columns: + column_word_lengths = [len(word) for word in column_name.split()] + if is_numeric_dtype(df[column_name]): + value_lengths = [len(f"{value:.2f}") for value in df[column_name]] + else: + value_lengths = [len(str(value)) for value in df[column_name]] + try: + max_length = max(max(column_word_lengths), max(value_lengths)) + n_pixels = 35 + (max_length * 12.5) + widths.append(f"{n_pixels}px") + except Exception: + widths.append("50px") + return widths + + +def get_column_types(df: pd.DataFrame) -> list[str]: + types = [] + for column_name in df.columns: + if is_numeric_dtype(df[column_name]): + types.append("number") + else: + types.append("str") + return types + + +def get_means_per_types(per_task: pd.DataFrame): + task_names_per_type = defaultdict(list) + for task_name in per_task.columns: + task_type = get_task(task_name).metadata.type + task_names_per_type[task_type].append(task_name) + records = [] + for task_type, tasks in task_names_per_type.items(): + for model_name, scores in per_task.iterrows(): + records.append( + dict( + model_name=model_name, + task_type=task_type, + score=scores[tasks].mean(skipna=False), + ) + ) + return pd.DataFrame.from_records(records) + + +def failsafe_get_model_meta(model_name): + try: + return get_model_meta(model_name) + except Exception: + return None + +def format_max_tokens(max_tokens: float | None) -> str: + if max_tokens is None: + return "Unknown" + if max_tokens == np.inf: + return "Infinite" + return str(int(max_tokens)) -def scores_to_tables(scores_long: list[dict]): + +def get_zero_shot_emoji(model_meta, tasks): + if model_meta is None: + return "⚠️" + is_zero_shot = model_meta.is_zero_shot_on(tasks) + if is_zero_shot is None: + return "⚠️" + if is_zero_shot: + return "✅" + return "❌" + + +def scores_to_tables( + scores_long: list[dict], search_query: str | None = None +) -> tuple[gr.DataFrame, gr.DataFrame]: if not scores_long: - return gr.DataFrame(), gr.DataFrame() + no_results_frame = pd.DataFrame( + {"No results": ["You can try relaxing your criteria"]} + ) + return gr.DataFrame(no_results_frame), gr.DataFrame(no_results_frame) data = pd.DataFrame.from_records(scores_long) - data["task_type"] = data["task_name"].map( - lambda task_name: get_task(task_name).metadata.type - ) - mean_per_type = ( - data.groupby(["model_name", "model_revision", "task_type"])[["score"]] - .agg(np.nanmean) - .reset_index() - ) - typed_mean = ( - mean_per_type.groupby(["model_name", "model_revision"])[["score"]] - .agg(np.nanmean) - .rename(columns={"score": "mean_by_task_type"}) - ) + per_task = data.pivot(index="model_name", columns="task_name", values="score") + mean_per_type = get_means_per_types(per_task) mean_per_type = mean_per_type.pivot( - index=["model_name", "model_revision"], columns="task_type", values="score" + index="model_name", columns="task_type", values="score" + ) + mean_per_type.columns = [ + split_on_capital(column) for column in mean_per_type.columns + ] + to_remove = per_task.isna().all(axis="columns") + if search_query: + names = per_task.index.get_level_values("model_name") + names = pd.Series(names, index=per_task.index) + to_remove |= ~names.str.contains(search_query, regex=True) + if to_remove.all(): + no_results_frame = pd.DataFrame( + {"No results": ["You can try relaxing your criteria"]} + ) + return gr.DataFrame(no_results_frame), gr.DataFrame(no_results_frame) + models_to_remove = list(per_task[to_remove].index) + typed_mean = mean_per_type.mean(skipna=False, axis=1) + overall_mean = per_task.mean(skipna=False, axis=1) + joint_table = mean_per_type.copy() + per_task = per_task.drop(models_to_remove, axis=0) + joint_table = joint_table.drop(models_to_remove, axis=0) + joint_table.insert(0, "mean", overall_mean) + joint_table.insert(1, "mean_by_task_type", typed_mean) + joint_table["borda_rank"] = get_borda_rank(per_task) + joint_table = joint_table.sort_values("borda_rank", ascending=True) + per_task["borda_rank"] = joint_table["borda_rank"] + per_task = per_task.sort_values("borda_rank", ascending=True) + per_task = per_task.drop(columns=["borda_rank"]) + joint_table = joint_table.reset_index() + model_metas = joint_table["model_name"].map(failsafe_get_model_meta) + joint_table = joint_table[model_metas.notna()] + joint_table["model_link"] = model_metas.map(lambda m: m.reference) + joint_table.insert( + 1, + "Max Tokens", + model_metas.map(lambda m: format_max_tokens(m.max_tokens)), ) - per_task = data.pivot( - index=["model_name", "model_revision"], columns="task_name", values="score" + joint_table.insert( + 1, + "Embedding Dimensions", + model_metas.map(lambda m: str(int(m.embed_dim)) if m.embed_dim else "Unknown"), ) - to_remove = per_task.isna().any(axis="columns") - overall_mean = ( - data.groupby(["model_name", "model_revision"])[["score"]] - .agg(np.nanmean) - .rename(columns={"score": "mean"}) + joint_table.insert( + 1, + "Number of Parameters", + model_metas.map(lambda m: format_n_parameters(m.n_parameters)), ) - per_task = per_task[~to_remove] - mean_per_type = mean_per_type[~to_remove] - overall_mean = overall_mean[~to_remove] - mean_rank = per_task.rank(ascending=False, numeric_only=True).mean( - axis=1, skipna=True + tasks = get_tasks(tasks=list(data["task_name"].unique())) + joint_table.insert( + 1, "Zero-shot", model_metas.map(lambda m: get_zero_shot_emoji(m, tasks)) ) - joint_table = overall_mean.join([typed_mean, mean_per_type]) - joint_table.insert(0, "mean_rank", mean_rank) - joint_table = joint_table.reset_index() - joint_table = joint_table.sort_values("mean", ascending=False) + # Removing HF organization from model joint_table["model_name"] = joint_table["model_name"].map( lambda name: name.split("/")[-1] ) + # Adding markdown link to model names + name_w_link = ( + "[" + joint_table["model_name"] + "](" + joint_table["model_link"] + ")" + ) + joint_table["model_name"] = joint_table["model_name"].mask( + joint_table["model_link"].notna(), name_w_link + ) + joint_table = joint_table.drop(columns=["model_link"]) joint_table = joint_table.rename( columns={ "model_name": "Model", - "mean_by_task_type": "Mean by Task Type", - "mean": "Mean", - "mean_rank": "Mean Rank", + "mean_by_task_type": "Mean (TaskType)", + "mean": "Mean (Task)", } ) - joint_table = joint_table.drop(columns=["model_revision"]) - joint_table.insert( - 0, "Rank", joint_table["Mean"].rank(ascending=False).map(int).map(str) + per_task = per_task.reset_index() + per_task["model_name"] = per_task["model_name"].map( + lambda name: name.split("/")[-1] ) per_task = per_task.rename( columns={ "model_name": "Model", } ) - per_task = per_task.reset_index().drop(columns=["model_revision"]) - numerics = joint_table.select_dtypes("number").columns - to_format = ["Mean", "Mean by Task Type", *mean_per_type.columns] - joint_table[to_format] = joint_table[to_format].map(format_scores) - joint_table = joint_table.style.highlight_max( - subset=to_format, - props="font-weight: bold", - ).format("{:.2f}", subset=numerics) - joint_table = joint_table.highlight_min( - subset=["Mean Rank"], props="font-weight: bold" + joint_table.insert(0, "Rank (Borda)", joint_table.pop("borda_rank")) + column_widths = get_column_widths(joint_table) + task_column_widths = get_column_widths(per_task) + # overriding for model name + column_widths[1] = "250px" + column_types = get_column_types(joint_table) + # setting model name column to markdown + column_types[1] = "markdown" + score_columns = ["Mean (Task)", "Mean (TaskType)", *mean_per_type.columns] + joint_table[score_columns] = joint_table[score_columns].map(format_scores) + joint_table_style = ( + joint_table.style.format( + { + **{column: "{:.2f}" for column in score_columns}, + "Rank (Borda)": "{:.0f}", + }, + na_rep="", + ) + .highlight_min("Rank (Borda)", props="font-weight: bold") + .highlight_max(subset=score_columns, props="font-weight: bold") + ) + task_score_columns = per_task.select_dtypes("number").columns + per_task[task_score_columns] *= 100 + per_task_style = per_task.style.format( + "{:.2f}", subset=task_score_columns, na_rep="" + ).highlight_max(subset=task_score_columns, props="font-weight: bold") + return ( + gr.DataFrame( + joint_table_style, + column_widths=column_widths, + datatype=column_types, + interactive=False, + wrap=True, + ), + gr.DataFrame( + per_task_style, column_widths=task_column_widths, interactive=False + ), ) - numerics = per_task.select_dtypes("number").columns - per_task[numerics] = per_task[numerics].map(format_scores) - per_task = per_task.style.highlight_max( - subset=numerics, props="font-weight: bold" - ).format("{:.2f}", subset=numerics) - return joint_table, per_task diff --git a/mteb/load_results/benchmark_results.py b/mteb/load_results/benchmark_results.py index e3060dfbbb..e1632a3dec 100644 --- a/mteb/load_results/benchmark_results.py +++ b/mteb/load_results/benchmark_results.py @@ -1,20 +1,19 @@ from __future__ import annotations import json +import warnings from collections import defaultdict from collections.abc import Iterable from pathlib import Path from typing import Any, Callable, Literal import numpy as np +import pandas as pd +from packaging.version import InvalidVersion, Version from pydantic import BaseModel, ConfigDict from mteb.abstasks.AbsTask import AbsTask, ScoresDict -from mteb.abstasks.TaskMetadata import ( - ISO_LANGUAGE_SCRIPT, - TASK_DOMAIN, - TASK_TYPE, -) +from mteb.abstasks.TaskMetadata import ISO_LANGUAGE_SCRIPT, TASK_DOMAIN, TASK_TYPE from mteb.languages import ISO_LANGUAGE from mteb.load_results.task_results import TaskResult from mteb.models.overview import get_model_metas @@ -35,6 +34,13 @@ def __repr__(self) -> str: n_entries = len(self.task_results) return f"ModelResult(model_name={self.model_name}, model_revision={self.model_revision}, task_results=[...](#{n_entries}))" + @classmethod + def from_validated(cls, **data) -> ModelResult: + data["task_results"] = [ + TaskResult.from_validated(**res) for res in data["task_results"] + ] + return cls.model_construct(**data) + def filter_tasks( self, task_names: list[str] | None = None, @@ -57,7 +63,7 @@ def filter_tasks( if (task_types is not None) and (task_result.task_type not in task_types): continue new_task_results.append(task_result) - return type(self)( + return type(self).model_construct( model_name=self.model_name, model_revision=self.model_revision, task_results=new_task_results, @@ -70,7 +76,7 @@ def select_tasks(self, tasks: list[AbsTask]) -> ModelResult: for task_res in self.task_results if task_res.task_name in task_name_to_task ] - return type(self)( + return type(self).model_construct( model_name=self.model_name, model_revision=self.model_revision, task_results=new_task_results, @@ -81,41 +87,71 @@ def get_scores( splits: list[Split] | None = None, languages: list[ISO_LANGUAGE | ISO_LANGUAGE_SCRIPT] | None = None, scripts: list[ISO_LANGUAGE_SCRIPT] | None = None, - getter: Callable[[ScoresDict], Score] = lambda scores: scores["main_score"], - aggregation: Callable[[list[Score]], Any] = np.mean, + getter: Callable[[ScoresDict], Score] | None = None, + aggregation: Callable[[list[Score]], Any] | None = None, format: Literal["wide", "long"] = "wide", ) -> dict | list: + if (getter is not None) or (aggregation is not None) or (scripts is not None): + use_fast = False + getter = ( + getter if getter is not None else lambda scores: scores["main_score"] + ) + aggregation = aggregation if aggregation is not None else np.mean + else: + use_fast = True if format == "wide": - scores = { - res.task_name: res.get_score( - splits=splits, - languages=languages, - scripts=scripts, - getter=getter, - aggregation=aggregation, - ) - for res in self.task_results - } + scores = {} + for res in self.task_results: + try: + if use_fast: + scores[res.task_name] = res.get_score_fast( + splits=splits, + languages=languages, + ) + else: + scores[res.task_name] = res.get_score( + splits=splits, + languages=languages, + aggregation=aggregation, + getter=getter, + scripts=scripts, + ) + except Exception as e: + warnings.warn( + f"Couldn't get scores for {res.task_name} due to {e}." + ) return scores if format == "long": entries = [] for task_res in self.task_results: - entry = dict( # noqa - model_name=self.model_name, - model_revision=self.model_revision, - task_name=task_res.task_name, - score=task_res.get_score( - splits=splits, - languages=languages, - getter=getter, - aggregation=aggregation, - ), - mteb_version=task_res.mteb_version, - dataset_revision=task_res.dataset_revision, - evaluation_time=task_res.evaluation_time, - kg_co2_emissions=task_res.kg_co2_emissions, - ) - entries.append(entry) + try: + if use_fast: + score = task_res.get_score_fast( + splits=splits, languages=languages + ) + else: + score = task_res.get_score( + splits=splits, + languages=languages, + aggregation=aggregation, + getter=getter, + scripts=scripts, + ) + entry = dict( # noqa + model_name=self.model_name, + model_revision=self.model_revision, + task_name=task_res.task_name, + score=score, + mteb_version=task_res.mteb_version, + dataset_revision=task_res.dataset_revision, + evaluation_time=task_res.evaluation_time, + kg_co2_emissions=task_res.kg_co2_emissions, + ) + entries.append(entry) + except Exception as e: + warnings.warn( + f"Couldn't get scores for {task_res.task_name} due to {e}." + ) return entries def __iter__(self): @@ -157,6 +193,9 @@ def __repr__(self) -> str: n_models = len(self.model_results) return f"BenchmarkResults(model_results=[...](#{n_models}))" + def __hash__(self) -> int: + return id(self) + def filter_tasks( self, task_names: list[str] | None = None, @@ -173,7 +212,7 @@ def filter_tasks( ) for res in self.model_results ] - return type(self)( + return type(self).model_construct( model_results=[res for res in model_results if res.task_results] ) @@ -181,65 +220,144 @@ def select_tasks(self, tasks: list[AbsTask]) -> BenchmarkResults: new_model_results = [ model_res.select_tasks(tasks) for model_res in self.model_results ] - return type(self)(model_results=new_model_results) + return type(self).model_construct(model_results=new_model_results) def filter_models( self, model_names: Iterable[str] | None = None, languages: Iterable[str] | None = None, - open_source: bool | None = None, + open_weights: bool | None = None, frameworks: Iterable[str] | None = None, n_parameters_range: tuple[int | None, int | None] = (None, None), + use_instructions: bool | None = None, + zero_shot_on: list[AbsTask] | None = None, ) -> BenchmarkResults: + # if model_names is None: + # model_names = [model_res.model_name for model_res in self] model_metas = get_model_metas( - model_names, languages, open_source, frameworks, n_parameters_range + model_names=model_names, + languages=languages, + open_weights=open_weights, + frameworks=frameworks, + n_parameters_range=n_parameters_range, + use_instructions=use_instructions, + zero_shot_on=zero_shot_on, ) - model_revision_pairs = {(meta.name, meta.revision) for meta in model_metas} + models = {meta.name for meta in model_metas} + # model_revision_pairs = {(meta.name, meta.revision) for meta in model_metas} new_model_results = [] for model_res in self: - if (model_res.model_name, model_res.model_revision) in model_revision_pairs: + if model_res.model_name in models: new_model_results.append(model_res) - return type(self)(model_results=new_model_results) + return type(self).model_construct(model_results=new_model_results) + + def join_revisions(self): + def parse_version(version_str: str) -> Version | None: + try: + return Version(version_str) + except (InvalidVersion, TypeError): + return None + + def keep_best(group: pd.DataFrame) -> pd.DataFrame: + is_main_revision = group["revision"] == group["main_revision"] + # If the main revision is present we select that + if is_main_revision.sum() > 0: + return group[is_main_revision].head(n=1) + unique_revisions = group["revision"].unique() + # Filtering out no_revision_available if other revisions are present + if (len(unique_revisions) > 1) and ( + "no_revision_available" in unique_revisions + ): + group = group[group["revision"] != "no_revision_available"] + # If there are any not-NA mteb versions, we select the latest one + if group["mteb_version"].notna().any(): + group = group.dropna(subset=["mteb_version"]) + group = group.sort_values("mteb_version", ascending=False) + return group.head(n=1) + return group.head(n=1) + + records = [] + for model_result in self: + for task_result in model_result: + records.append( + dict( + model=model_result.model_name, + revision=model_result.model_revision, + task_name=task_result.task_name, + mteb_version=task_result.mteb_version, + task_result=task_result, + ) + ) + task_df = pd.DataFrame.from_records(records) + model_to_main_revision = { + meta.name: meta.revision for meta in get_model_metas() + } + task_df["main_revision"] = task_df["model"].map(model_to_main_revision) + task_df["mteb_version"] = task_df["mteb_version"].map(parse_version) + task_df = ( + task_df.groupby(["model", "task_name"]) + .apply(keep_best) + .reset_index(drop=True) + ) + model_results = [] + for (model, model_revision), group in task_df.groupby(["model", "revision"]): + model_result = ModelResult.model_construct( + model_name=model, + model_revision=model_revision, + task_results=list(group["task_result"]), + ) + model_results.append(model_result) + return BenchmarkResults.model_construct(model_results=model_results) def get_scores( self, splits: list[Split] | None = None, languages: list[ISO_LANGUAGE | ISO_LANGUAGE_SCRIPT] | None = None, scripts: list[ISO_LANGUAGE_SCRIPT] | None = None, - getter: Callable[[ScoresDict], Score] = lambda scores: scores["main_score"], - aggregation: Callable[[list[Score]], Any] = np.mean, + getter: Callable[[ScoresDict], Score] = None, + aggregation: Callable[[list[Score]], Any] = None, format: Literal["wide", "long"] = "wide", ) -> list[dict]: entries = [] if format == "wide": for model_res in self: - model_scores = model_res.get_scores( - splits=splits, - languages=languages, - scripts=scripts, - getter=getter, - aggregation=aggregation, - format="wide", - ) - entries.append( - { - "model": model_res.model_name, - "revision": model_res.model_revision, - **model_scores, - } - ) - if format == "long": - for model_res in self: - entries.extend( - model_res.get_scores( + try: + model_scores = model_res.get_scores( splits=splits, languages=languages, scripts=scripts, getter=getter, aggregation=aggregation, - format="long", + format="wide", + ) + entries.append( + { + "model": model_res.model_name, + "revision": model_res.model_revision, + **model_scores, + } + ) + except Exception as e: + warnings.warn( + f"Couldn't get scores for {model_res.model_name}({model_res.model_revision}), due to: {e}" + ) + if format == "long": + for model_res in self: + try: + entries.extend( + model_res.get_scores( + splits=splits, + languages=languages, + scripts=scripts, + getter=getter, + aggregation=aggregation, + format="long", + ) + ) + except Exception as e: + warnings.warn( + f"Couldn't get scores for {model_res.model_name}({model_res.model_revision}), due to: {e}" ) - ) return entries def __iter__(self): @@ -280,6 +398,13 @@ def to_disk(self, path: Path | str) -> None: with path.open("w") as out_file: out_file.write(self.model_dump_json(indent=2)) + @classmethod + def from_validated(cls, **data) -> BenchmarkResults: + model_results = [] + for model_res in data["model_results"]: + model_results.append(ModelResult.from_validated(**model_res)) + return cls.model_construct(model_results=model_results) + @classmethod def from_disk(cls, path: Path | str) -> BenchmarkResults: path = Path(path) diff --git a/mteb/load_results/load_results.py b/mteb/load_results/load_results.py index 8601420427..ef851a1dc2 100644 --- a/mteb/load_results/load_results.py +++ b/mteb/load_results/load_results.py @@ -90,6 +90,7 @@ def load_results( tasks: Sequence[AbsTask] | Sequence[str] | None = None, validate_and_filter: bool = True, require_model_meta: bool = True, + only_main_score: bool = False, ) -> BenchmarkResults: """Loads the results from the latest version of the results repository. The results are cached locally in the MTEB_CACHE directory. This directory can be set using the MTEB_CACHE environment variable or defaults to "~/.cache/mteb". @@ -103,6 +104,7 @@ def load_results( extract the model name and revision from the path. validate_and_filter: If True it will validate that the results object for the task contains the correct splits and filter out splits from the results object that are not default in the task metadata. Defaults to True. + only_main_score: If True, only the main score will be loaded. """ repo_directory = download_of_results(results_repo, download_latest=download_latest) model_paths = [p for p in (repo_directory / "results").glob("*") if p.is_dir()] @@ -137,6 +139,7 @@ def load_results( continue model_name, revision = model_name_and_revision + model_name = model_name.replace("__", "/") if models_to_keep is not None and model_name not in models_to_keep: continue elif models_to_keep is not None and models_to_keep[model_name] is not None: @@ -146,7 +149,12 @@ def load_results( task_json_files = [ f for f in revision_path.glob("*.json") if "model_meta.json" != f.name ] - _results = [TaskResult.from_disk(f) for f in task_json_files] + _results = [] + for f in task_json_files: + task_res = TaskResult.from_disk(f) + if only_main_score: + task_res = task_res.only_main_score() + _results.append(task_res) # filter out tasks that are not in the tasks list if tasks is not None: @@ -163,7 +171,7 @@ def load_results( r = r.validate_and_filter_scores(task=task) filtered_results.append(r) except Exception as e: - logger.warning( + logger.info( f"Validation failed for {r.task_name} in {model_name} {revision}: {e}" ) _results = filtered_results diff --git a/mteb/load_results/task_results.py b/mteb/load_results/task_results.py index b6da0ba304..72cae5a93d 100644 --- a/mteb/load_results/task_results.py +++ b/mteb/load_results/task_results.py @@ -156,9 +156,9 @@ class TaskResult(BaseModel): dataset_revision: str task_name: str - mteb_version: str + mteb_version: str | None scores: dict[Split, list[ScoresDict]] - evaluation_time: float + evaluation_time: float | None kg_co2_emissions: float | None = None @classmethod @@ -293,10 +293,12 @@ def from_disk(cls, path: Path, load_historic_data: bool = True) -> TaskResult: pre_1_11_load = ( ( "mteb_version" in data + and data["mteb_version"] is not None and Version(data["mteb_version"]) < Version("1.11.0") ) or "mteb_version" not in data ) # assume it is before 1.11.0 if the version is not present + try: obj = cls.model_validate(data) except Exception as e: @@ -307,9 +309,11 @@ def from_disk(cls, path: Path, load_historic_data: bool = True) -> TaskResult: ) obj = cls._convert_from_before_v1_11_0(data) - pre_v_12_48 = "mteb_version" in data and Version( - data["mteb_version"] - ) < Version("1.12.48") + pre_v_12_48 = ( + "mteb_version" in data + and data["mteb_version"] is not None + and Version(data["mteb_version"]) < Version("1.12.48") + ) if pre_v_12_48: cls._fix_pair_classification_scores(obj) @@ -383,15 +387,16 @@ def _convert_from_before_v1_11_0(cls, data: dict) -> TaskResult: main_score = task.metadata.main_score for split, split_score in scores.items(): for hf_subset, hf_subset_scores in split_score.items(): - if task.metadata.type == "STS": - for name, prev_name in [ - ("cosine", "cos_sim"), - ("manhattan", "manhattan"), - ("euclidean", "euclidean"), - ]: - prev_name_scores = hf_subset_scores.pop( - prev_name, {"spearman": "NaN"} - ) + for name, prev_name in [ + ("cosine", "cos_sim"), + ("manhattan", "manhattan"), + ("euclidean", "euclidean"), + ("dot", "dot"), + ("max", "max"), + ("similarity", "similarity"), + ]: + prev_name_scores = hf_subset_scores.pop(prev_name, None) + if prev_name_scores is not None: for k, v in prev_name_scores.items(): hf_subset_scores[f"{name}_{k}"] = v @@ -464,12 +469,61 @@ def get_score( values.append(getter(scores)) break - return aggregation(values) + return aggregation(values) + + def get_score_fast(self, splits: str | None, languages: str | None) -> float: + """Sped up version of get_score that will be used if no aggregation, script or getter needs to be specified.""" + if splits is None: + splits = self.scores + val_sum = 0 + n_val = 0 + for split in splits: + if split not in self.scores: + raise ValueError(f"Split missing from scores: {split}") + + for scores in self.scores[split]: + langs = scores["languages"] + hf_subset = scores["hf_subset"] + main_score = scores.get("main_score", None) + if main_score is None: + raise ValueError(f"Missing main score for subset: {hf_subset}") + if languages is None: + val_sum += main_score + n_val += 1 + continue + for lang in langs: + if lang.split("-")[0] in languages: + val_sum += main_score + n_val += 1 + break + if n_val == 0: + raise ValueError("No splits had scores for the specified languages.") + return val_sum / n_val + + @classmethod + def from_validated(cls, **data) -> TaskResult: + return cls.model_construct(**data) def __repr__(self) -> str: return f"TaskResult(task_name={self.task_name}, scores=...)" - def validate_and_filter_scores(self, task: AbsTask | None = None) -> AbsTask: + def only_main_score(self) -> TaskResult: + new_scores = {} + for split in self.scores: + new_scores[split] = [] + for subset_scores in self.scores[split]: + new_scores[split].append( + { + "hf_subset": subset_scores.get("hf_subset", "default"), + "main_score": subset_scores.get("main_score", np.nan), + "languages": subset_scores.get("languages", []), + } + ) + new_res = {**self.to_dict(), "scores": new_scores} + new_res = TaskResult.from_validated(**new_res) + return new_res + + def validate_and_filter_scores(self, task: AbsTask | None = None) -> TaskResult: """This ensures that the scores are correct for the given task, by removing any splits besides those specified in the task metadata. Additionally it also ensure that all of the splits required as well as the languages are present in the scores. Returns new TaskResult object. @@ -503,12 +557,21 @@ def validate_and_filter_scores(self, task: AbsTask | None = None) -> AbsTask: new_scores[split].append(_scores) seen_subsets.add(_scores["hf_subset"]) if seen_subsets != hf_subsets: - raise ValueError( - f"Missing subsets {hf_subsets - seen_subsets} for split {split}" + missing_subsets = hf_subsets - seen_subsets + if len(missing_subsets) > 2: + subset1, subset2 = list(missing_subsets)[:2] + missing_subsets_str = f"{{'{subset1}', '{subset2}', ...}}" + else: + missing_subsets_str = str(missing_subsets) + + logger.warning( + f"{task.metadata.name}: Missing subsets {missing_subsets_str} for split {split}" ) seen_splits.add(split) if seen_splits != set(splits): - raise ValueError(f"Missing splits {set(splits) - seen_splits}") + logger.warning( + f"{task.metadata.name}: Missing splits {set(splits) - seen_splits}" + ) new_res = {**self.to_dict(), "scores": new_scores} - new_res = TaskResult.from_dict(new_res) + new_res = TaskResult.from_validated(**new_res) return new_res diff --git a/mteb/model_meta.py b/mteb/model_meta.py index b81dd1e1c5..6486c849ea 100644 --- a/mteb/model_meta.py +++ b/mteb/model_meta.py @@ -4,8 +4,9 @@ from functools import partial from typing import TYPE_CHECKING, Any, Callable, Literal -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict +from mteb.abstasks.AbsTask import AbsTask from mteb.abstasks.TaskMetadata import STR_DATE, STR_URL from mteb.encoder_interface import Encoder @@ -26,8 +27,11 @@ "TensorFlow", "API", "Tevatron", + "NumPy", + "PyLate", + "ColBERT", ] -DISTANCE_METRICS = Literal["cosine"] +DISTANCE_METRICS = Literal["cosine", "max_sim", "dot"] def sentence_transformers_loader( @@ -56,7 +60,6 @@ class ModelMeta(BaseModel): name: The name of the model, ideally the name on huggingface. n_parameters: The number of parameters in the model, e.g. 7_000_000 for a 7M parameter model. Can be None if the the number of parameters is not known (e.g. for proprietary models) or if the loader returns a SentenceTransformer model from which it can be derived. - memory_usage: The amount of memory the model uses in GB. Can be None if the memory usage is not known (e.g. for proprietary models). max_tokens: The maximum number of tokens the model can handle. Can be None if the maximum number of tokens is not known (e.g. for proprietary models). embed_dim: The dimension of the embeddings produced by the model. Currently all models are assumed to produce fixed-size embeddings. @@ -64,38 +67,44 @@ class ModelMeta(BaseModel): release_date: The date the model's revision was released. license: The license under which the model is released. Required if open_weights is True. open_weights: Whether the model is open source or proprietary. - public_training_data: Whether the training data used to train the model is publicly available. - public_training_code: Whether the code used to train the model is publicly available. + public_training_code: A link to the publicly available training code. If none it is assumed that the training code is not publicly available. + public_training_data: A link to the publicly available training data. If none it is assumed that the training data is not publicly available. similarity_fn_name: The distance metric used by the model. framework: The framework the model is implemented in, can be a list of frameworks e.g. `["Sentence Transformers", "PyTorch"]`. reference: A URL to the model's page on huggingface or another source. languages: The languages the model is intended for specified as a 3 letter language code followed by a script code e.g. "eng-Latn" for English in the Latin script. - use_instuctions: Whether the model uses instructions E.g. for prompt-based models. This also include models that require a specific format for + use_instructions: Whether the model uses instructions E.g. for prompt-based models. This also include models that require a specific format for input such as "query: {document}" or "passage: {document}". - zero_shot_benchmarks: A list of benchmarks on which the model has been evaluated in a zero-shot setting. By default we assume that all models - are evaluated non-zero-shot unless specified otherwise. + training_datasets: A dictionary of datasets that the model was trained on. Names should be names as their appear in `mteb` for example + {"ArguAna": ["test"]} if the model is trained on the ArguAna test set. This field is used to determine if a model generalizes zero-shot to + a benchmark as well as mark dataset contaminations. + adapted_from: Name of the model from which this model is adapted from. For quantizations, fine-tunes, long doc extensions, etc. + superseded_by: Name of the model that supersedes this model, e.g. nvidia/NV-Embed-v2 supersedes v1. modalities: A list of strings representing the modalities the model supports. Default is ["text]. """ + model_config = ConfigDict(extra="forbid") + name: str | None revision: str | None release_date: STR_DATE | None languages: list[ISO_LANGUAGE_SCRIPT] | None loader: Callable[..., Encoder] | None = None - n_parameters: int | None = None - memory_usage: float | None = None - max_tokens: int | None = None - embed_dim: int | None = None - license: str | None = None - open_weights: bool | None = None - public_training_data: bool | None = None - public_training_code: bool | None = None - framework: list[FRAMEWORKS] = [] + n_parameters: int | None + max_tokens: float | None + embed_dim: int | None + license: str | None + open_weights: bool | None + public_training_code: str | None + public_training_data: str | bool | None + framework: list[FRAMEWORKS] reference: STR_URL | None = None - similarity_fn_name: DISTANCE_METRICS | None = None - use_instuctions: bool | None = None - zero_shot_benchmarks: list[str] | None = None + similarity_fn_name: DISTANCE_METRICS | None + use_instructions: bool | None + training_datasets: dict[str, list[str]] | None + adapted_from: str | None = None + superseded_by: str | None = None modalities: list[MODALITIES] = ["text"] def to_dict(self): @@ -119,9 +128,24 @@ def load_model(self, **kwargs: Any) -> Encoder: loader = self.loader model: Encoder = loader(**kwargs) # type: ignore + model.mteb_model_meta = self return model def model_name_as_path(self) -> str: if self.name is None: raise ValueError("Model name is not set") return self.name.replace("/", "__").replace(" ", "_") + + def is_zero_shot_on(self, tasks: list[AbsTask]) -> bool | None: + """Indicates whether the given model can be considered + zero-shot or not on the given tasks. + Returns None if no training data is specified on the model. + """ + if self.training_datasets is None: + return None + benchmark_datasets = set() + for task in tasks: + benchmark_datasets.add(task.metadata.name) + model_datasets = {ds_name for ds_name, splits in self.training_datasets.items()} + intersection = model_datasets & benchmark_datasets + return len(intersection) == 0 diff --git a/mteb/models/align_models.py b/mteb/models/align_models.py index 911e601750..9e0827dece 100644 --- a/mteb/models/align_models.py +++ b/mteb/models/align_models.py @@ -140,10 +140,21 @@ def get_fused_embeddings( ), name="kakaobrain/align-base", languages=["eng_Latn"], - open_source=True, revision="e96a37facc7b1f59090ece82293226b817afd6ba", release_date="2023-02-24", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) if __name__ == "__main__": diff --git a/mteb/models/arctic_models.py b/mteb/models/arctic_models.py index 3c350a0ad7..f765b01bff 100644 --- a/mteb/models/arctic_models.py +++ b/mteb/models/arctic_models.py @@ -1,8 +1,344 @@ from __future__ import annotations -from mteb.model_meta import ModelMeta +from functools import partial -arctic_m_v1_5 = ModelMeta( +from mteb.model_meta import ModelMeta, sentence_transformers_loader + +LANGUAGES_V2_0 = [ + "afr_Latn", + "ara_Arab", + "aze_Latn", + "bel_Cyrl", + "bul_Cyrl", + "ben_Beng", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "ell_Grek", + "eng_Latn", + "spa_Latn", + "est_Latn", + "eus_Latn", + "fas_Arab", + "fin_Latn", + "fra_Latn", + "glg_Latn", + "guj_Gujr", + "heb_Hebr", + "hin_Deva", + "hrv_Latn", + "hat_Latn", + "hun_Latn", + "hye_Armn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jpn_Jpan", + "jav_Latn", + "kat_Geor", + "kaz_Cyrl", + "khm_Khmr", + "kan_Knda", + "kor_Hang", + "kir_Cyrl", + "lao_Laoo", + "lit_Latn", + "lav_Latn", + "mkd_Cyrl", + "mal_Mlym", + "mon_Cyrl", + "mar_Deva", + "msa_Latn", + "mya_Mymr", + "nep_Deva", + "nld_Latn", + "pan_Guru", + "pol_Latn", + "por_Latn", + "que_Latn", + "ron_Latn", + "rus_Cyrl", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "som_Latn", + "sqi_Latn", + "srp_Cyrl", + "swe_Latn", + "swa_Latn", + "tam_Taml", + "tel_Telu", + "tha_Thai", + "tgl_Latn", + "tur_Latn", + "ukr_Cyrl", + "urd_Arab", + "vie_Latn", + "yor_Latn", + "zho_Hans", +] + +arctic_embed_xs = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="Snowflake/snowflake-arctic-embed-xs", + revision="742da4f66e1823b5b4dbe6c320a1375a1fd85f9e", + ), + name="Snowflake/snowflake-arctic-embed-xs", + revision="742da4f66e1823b5b4dbe6c320a1375a1fd85f9e", + release_date="2024-07-08", # initial commit of hf model. + languages=["eng_Latn"], + open_weights=True, + framework=["Sentence Transformers", "PyTorch"], + n_parameters=22_600_000, + max_tokens=512, + embed_dim=384, + license="apache-2.0", + reference="https://huggingface.co/Snowflake/snowflake-arctic-embed-xs", + similarity_fn_name="cosine", + use_instructions=True, + adapted_from="sentence-transformers/all-MiniLM-L6-v2", + superseded_by=None, + public_training_code=None, + public_training_data=None, + training_datasets={ + # source: https://arxiv.org/pdf/2405.05374 + # splits not specified to assuming everything + # in MTEB + "NQ": ["test"], + "NQHardNegatives": ["test"], + "HotPotQA": ["test"], + "HotPotQAHardNegatives": ["test"], + "HotPotQA-PL": ["test"], # translated from hotpotQA (not trained on) + "FEVER": ["test"], + "FEVERHardNegatives": ["test"], + # not in MTEB + # trained on stack exchange (title-body) + # "stackexchange": [], + # potentially means that: + # "StackExchangeClusteringP2P": ["test"], + # "StackExchangeClusteringP2P.v2": ["test"], + # "StackExchangeClustering": ["test"], + # "StackExchangeClustering.v2": ["test"], + # not in MTEB + # "paq": [], + # "s2orc": [], + # "other": [], # undisclosed including webdata + }, # also use synthetic +) + + +arctic_embed_s = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="Snowflake/snowflake-arctic-embed-s", + revision="d3c1d2d433dd0fdc8e9ca01331a5f225639e798f", + ), + name="Snowflake/snowflake-arctic-embed-s", + revision="d3c1d2d433dd0fdc8e9ca01331a5f225639e798f", + release_date="2024-04-12", # initial commit of hf model. + languages=["eng_Latn"], + open_weights=True, + framework=["Sentence Transformers", "PyTorch"], + n_parameters=32_200_000, + max_tokens=512, + embed_dim=384, + license="apache-2.0", + reference="https://huggingface.co/Snowflake/snowflake-arctic-embed-s", + similarity_fn_name="cosine", + use_instructions=True, + adapted_from="intfloat/e5-small-unsupervised", + superseded_by=None, + public_training_code=None, + public_training_data=None, # couldn't find + training_datasets={ + # source: https://arxiv.org/pdf/2405.05374 + # splits not specified to assuming everything + # in MTEB + "NQ": ["test"], + "NQHardNegatives": ["test"], + "HotPotQA": ["test"], + "HotPotQAHardNegatives": ["test"], + "HotPotQA-PL": ["test"], # translated from hotpotQA (not trained on) + "FEVER": ["test"], + "FEVERHardNegatives": ["test"], + # not in MTEB + # trained on stack exchange (title-body) + # "stackexchange": [], + # potentially means that: + # "StackExchangeClusteringP2P": ["test"], + # "StackExchangeClusteringP2P.v2": ["test"], + # "StackExchangeClustering": ["test"], + # "StackExchangeClustering.v2": ["test"], + # not in MTEB + # "paq": [], + # "s2orc": [], + # "other": [], # undisclosed including webdata + }, # also use synthetic +) + + +arctic_embed_m = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="Snowflake/snowflake-arctic-embed-m", + revision="cc17beacbac32366782584c8752220405a0f3f40", + ), + name="Snowflake/snowflake-arctic-embed-m", + revision="cc17beacbac32366782584c8752220405a0f3f40", + release_date="2024-04-12", # initial commit of hf model. + languages=["eng_Latn"], + open_weights=True, + framework=["Sentence Transformers", "PyTorch"], + n_parameters=109_000_000, + max_tokens=512, + embed_dim=768, + license="apache-2.0", + reference="https://huggingface.co/Snowflake/snowflake-arctic-embed-m", + similarity_fn_name="cosine", + use_instructions=True, + adapted_from="intfloat/e5-base-unsupervised", + superseded_by="Snowflake/snowflake-arctic-embed-m-v1.5", + public_training_code=None, + public_training_data=None, # couldn't find + training_datasets={ + # source: https://arxiv.org/pdf/2405.05374 + # splits not specified to assuming everything + # in MTEB + "NQ": ["test"], + "NQHardNegatives": ["test"], + "HotPotQA": ["test"], + "HotPotQAHardNegatives": ["test"], + "HotPotQA-PL": ["test"], # translated from hotpotQA (not trained on) + "FEVER": ["test"], + "FEVERHardNegatives": ["test"], + # not in MTEB + # trained on stack exchange (title-body) + # "stackexchange": [], + # potentially means that: + # "StackExchangeClusteringP2P": ["test"], + # "StackExchangeClusteringP2P.v2": ["test"], + # "StackExchangeClustering": ["test"], + # "StackExchangeClustering.v2": ["test"], + # not in MTEB + # "paq": [], + # "s2orc": [], + # "other": [], # undisclosed including webdata + }, # also use synthetic +) + +arctic_embed_m_long = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="Snowflake/snowflake-arctic-embed-m-long", + revision="89d0f6ab196eead40b90cb6f9fefec01a908d2d1", + trust_remote_code=True, + ), + name="Snowflake/snowflake-arctic-embed-m-long", + revision="89d0f6ab196eead40b90cb6f9fefec01a908d2d1", + release_date="2024-04-12", # initial commit of hf model. + languages=["eng_Latn"], + open_weights=True, + framework=["Sentence Transformers", "PyTorch"], + n_parameters=137_000_000, + max_tokens=2048, + embed_dim=768, + license="apache-2.0", + reference="https://huggingface.co/Snowflake/snowflake-arctic-embed-m-long", + similarity_fn_name="cosine", + use_instructions=True, + adapted_from="nomic-ai/nomic-embed-text-v1-unsupervised", + superseded_by="Snowflake/snowflake-arctic-embed-m-v2.0", + public_training_code=None, + public_training_data=None, # couldn't find + training_datasets={ + # source: https://arxiv.org/pdf/2405.05374 + # splits not specified to assuming everything + # in MTEB + "NQ": ["test"], + "NQHardNegatives": ["test"], + "HotPotQA": ["test"], + "HotPotQAHardNegatives": ["test"], + "HotPotQA-PL": ["test"], # translated from hotpotQA (not trained on) + "FEVER": ["test"], + "FEVERHardNegatives": ["test"], + # trained on stack exchange, unsure if sources match + # not in MTEB + # trained on stack exchange (title-body) + # "stackexchange": [], + # potentially means that: + # "StackExchangeClusteringP2P": ["test"], + # "StackExchangeClusteringP2P.v2": ["test"], + # "StackExchangeClustering": ["test"], + # "StackExchangeClustering.v2": ["test"], + # not in MTEB + # "paq": [], + # "s2orc": [], + # "other": [], # undisclosed including webdata + }, # also use synthetic +) + +arctic_embed_l = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="Snowflake/snowflake-arctic-embed-l", + revision="9a9e5834d2e89cdd8bb72b64111dde496e4fe78c", + ), + name="Snowflake/snowflake-arctic-embed-l", + revision="9a9e5834d2e89cdd8bb72b64111dde496e4fe78c", + release_date="2024-04-12", # initial commit of hf model. + languages=["eng_Latn"], + open_weights=True, + framework=["Sentence Transformers", "PyTorch"], + n_parameters=335_000_000, + max_tokens=512, + embed_dim=1024, + license="apache-2.0", + reference="https://huggingface.co/Snowflake/snowflake-arctic-embed-l", + similarity_fn_name="cosine", + use_instructions=True, + adapted_from="intfloat/e5-base-unsupervised", + superseded_by="Snowflake/snowflake-arctic-embed-l-v2.0", + public_training_code=None, + public_training_data=None, # couldn't find + training_datasets={ + # source: https://arxiv.org/pdf/2405.05374 + # splits not specified to assuming everything + # in MTEB + "NQ": ["test"], + "NQHardNegatives": ["test"], + "HotPotQA": ["test"], + "HotPotQAHardNegatives": ["test"], + "HotPotQA-PL": ["test"], # translated from hotpotQA (not trained on) + "FEVER": ["test"], + "FEVERHardNegatives": ["test"], + # not in MTEB + # trained on stack exchange (title-body) + # "stackexchange": [], + # potentially means that: + # "StackExchangeClusteringP2P": ["test"], + # "StackExchangeClusteringP2P.v2": ["test"], + # "StackExchangeClustering": ["test"], + # "StackExchangeClustering.v2": ["test"], + # not in MTEB + # "paq": [], + # "s2orc": [], + # "other": [], # undisclosed including webdata + }, # also use synthetic +) + +arctic_embed_m_v1_5 = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="Snowflake/snowflake-arctic-embed-m-v1.5", + revision="97eab2e17fcb7ccb8bb94d6e547898fa1a6a0f47", + model_prompts={ + "query": "Represent this sentence for searching relevant passages: " + }, + ), name="Snowflake/snowflake-arctic-embed-m-v1.5", revision="97eab2e17fcb7ccb8bb94d6e547898fa1a6a0f47", release_date="2024-07-08", # initial commit of hf model. @@ -10,11 +346,114 @@ open_weights=True, framework=["Sentence Transformers", "PyTorch"], n_parameters=109_000_000, - memory_usage=None, max_tokens=512, - embed_dim=256, + embed_dim=768, license="apache-2.0", reference="https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5", - similarity_fn_name="cosine_similarity", - use_instuctions=False, + similarity_fn_name="cosine", + use_instructions=True, + adapted_from=None, + superseded_by="Snowflake/snowflake-arctic-embed-m-v2.0", + public_training_code=None, + public_training_data=None, + training_datasets=None, +) + +arctic_embed_m_v2_0 = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="Snowflake/snowflake-arctic-embed-m-v2.0", + revision="f2a7d59d80dfda5b1d14f096f3ce88bb6bf9ebdc", + trust_remote_code=True, + ), + name="Snowflake/snowflake-arctic-embed-m-v2.0", + revision="f2a7d59d80dfda5b1d14f096f3ce88bb6bf9ebdc", + release_date="2024-12-04", # initial commit of hf model. + languages=LANGUAGES_V2_0, + open_weights=True, + framework=["Sentence Transformers", "PyTorch"], + n_parameters=305_000_000, + max_tokens=8192, + embed_dim=768, + license="apache-2.0", + reference="https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v2.0", + similarity_fn_name="cosine", + use_instructions=True, + adapted_from="Alibaba-NLP/gte-multilingual-base", + superseded_by=None, + public_training_code=None, + public_training_data=None, # couldn't find + training_datasets={ + # source: https://arxiv.org/pdf/2405.05374 + # splits not specified to assuming everything + # in MTEB + "NQ": ["test"], + "NQHardNegatives": ["test"], + "HotPotQA": ["test"], + "HotPotQAHardNegatives": ["test"], + "HotPotQA-PL": ["test"], # translated from hotpotQA (not trained on) + "FEVER": ["test"], + "FEVERHardNegatives": ["test"], + # not in MTEB + # trained on stack exchange (title-body) + # "stackexchange": [], + # potentially means that: + # "StackExchangeClusteringP2P": ["test"], + # "StackExchangeClusteringP2P.v2": ["test"], + # "StackExchangeClustering": ["test"], + # "StackExchangeClustering.v2": ["test"], + # not in MTEB + # "paq": [], + # "s2orc": [], + # "other": [], # undisclosed including webdata + }, # also use synthetic +) + +arctic_embed_l_v2_0 = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="Snowflake/snowflake-arctic-embed-l-v2.0", + revision="edc2df7b6c25794b340229ca082e7c78782e6374", + ), + name="Snowflake/snowflake-arctic-embed-l-v2.0", + revision="edc2df7b6c25794b340229ca082e7c78782e6374", + release_date="2024-12-04", # initial commit of hf model. + languages=LANGUAGES_V2_0, + open_weights=True, + framework=["Sentence Transformers", "PyTorch"], + n_parameters=568_000_000, + max_tokens=8192, + embed_dim=1024, + license="apache-2.0", + reference="https://huggingface.co/Snowflake/snowflake-arctic-embed-l-v2.0", + similarity_fn_name="cosine", + use_instructions=True, + adapted_from="BAAI/bge-m3-retromae", + superseded_by=None, + public_training_code=None, + public_training_data=None, # couldn't find + training_datasets={ + # source: https://arxiv.org/pdf/2405.05374 + # splits not specified to assuming everything + # in MTEB + "NQ": ["test"], + "NQHardNegatives": ["test"], + "HotPotQA": ["test"], + "HotPotQAHardNegatives": ["test"], + "HotPotQA-PL": ["test"], # translated from hotpotQA (not trained on) + "FEVER": ["test"], + "FEVERHardNegatives": ["test"], + # not in MTEB + # trained on stack exchange (title-body) + # "stackexchange": [], + # potentially means that: + # "StackExchangeClusteringP2P": ["test"], + # "StackExchangeClusteringP2P.v2": ["test"], + # "StackExchangeClustering": ["test"], + # "StackExchangeClustering.v2": ["test"], + # not in MTEB + # "paq": [], + # "s2orc": [], + # "other": [], # undisclosed including webdata + }, # also use synthetic ) diff --git a/mteb/models/bge_models.py b/mteb/models/bge_models.py index 61200d72e0..6529b0804a 100644 --- a/mteb/models/bge_models.py +++ b/mteb/models/bge_models.py @@ -4,10 +4,295 @@ from mteb.model_meta import ModelMeta, sentence_transformers_loader +from .e5_instruct import E5_MISTRAL_TRAINING_DATA + model_prompts = {"query": "Represent this sentence for searching relevant passages: "} +model_prompts_zh = {"query": "为这个句子生成表示以用于检索相关文章："} + +bge_m3_training_data = { + # source: https://arxiv.org/abs/2402.03216 + "MIRACLRetrieval": ["train"], + "MIRACLRetrievalHardNegatives": ["train"], + "MIRACLReranking": ["train"], + "LeCaRDv2": ["train"], + "CMedQAv1-reranking": ["train"], + "CMedQAv2-reranking": ["train"], + "MrTidyRetrieval": ["train"], + "T2Reranking": ["train"], + "MSMARCO": ["train"], + "MSMARCOHardNegatives": ["train"], + "NanoMSMARCORetrieval": ["train"], + "MSMARCO-PL": ["train"], # translation not trained on + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on + "HotpotQA": ["train"], + "HotpotQA-PL": ["train"], # translation not trained on + "HotpotQAHardNegatives": ["train"], + "T2Retrieval": ["train"], + "DuReader": ["train"], + "MMarcoReranking": ["train"], + "CodeSearchNet": ["train"], + # not in mteb + # "s2orc" + # Wikipedia + # "xP3" + # "mC4" + # "CC-News" + # "MTP" + # "NLLB" + # "CCMatrix" + # TriviaQA + # COL-IEE + # PubMedQA + # SQuAD + # SimCSE + # mMARCO-ZH + # LawGPT + # NLI-zh2, LeCaRDv2, + # NLI, MultiLongDoc (their syntetic) + # + synthetic data +} + +bge_training_data = { + # source: https://data.baai.ac.cn/details/BAAI-MTP + "NQ": ["test"], + "NQHardNegatives": ["test"], + "AmazonReviewsClassification": [ + "validation", + "test", + ], # assumed from: amazon_reviews_multi + "MLQARetrieval": [ + "validation", + "test", + ], # assumed from mlqa (question, context) + # not in mteb + # Dataset Pairs + # wudao (title, passage) + # cmrc2018 (query, context) + # dureader (query, context) + # simclue (sentence_a, sentence_b) + # csl (title, abstract) + # amazon_reviews_multi (title, body) + # wiki_atomic_edits (base_sentence, edited_sentence) + # mlqa (question, context) + # xlsum (title, summary) (title, text) + # "sentence-transformers data": [], # https://huggingface.co/datasets/sentence-transformers/embedding-training-data # TODO check this further + # "wikipedia": [], # title + section title, passage + # "reddit": [], # title, body + # "stackexchange": [], # (title, upvoted answer) (title+body, upvoted answer) + # "s2orc": [], # (title, abstract) (title, citation title) (abstract, citation abstract) +} + +bge_chinese_training_data = { + # source: https://arxiv.org/pdf/2309.07597 + "T2Retrieval": ["train"], + "DuReader": ["train"], + "MMarcoReranking": ["train"], + "CMedQAv2-reranking": ["train"], + "Cmnli": ["train"], + "Ocnli": ["train"], + # not in mteb + # - multi-cpr + # - NLI-zh + # Dataset Pairs + # wudao (title, passage) + # cmrc2018 (query, context) + # dureader (query, context) + # simclue (sentence_a, sentence_b) + # csl (title, abstract) + # amazon_reviews_multi (title, body) + # wiki_atomic_edits (base_sentence, edited_sentence) + # mlqa (question, context) + # xlsum (title, summary) (title, text) + # "sentence-transformers data": [], # https://huggingface.co/datasets/sentence-transformers/embedding-training-data # TODO check this further + # "wikipedia": [], # title + section title, passage + # "reddit": [], # title, body + # "stackexchange": [], # (title, upvoted answer) (title+body, upvoted answer) + # "s2orc": [], # (title, abstract) (title, citation title) (abstract, citation abstract) +} + +# https://huggingface.co/BAAI/bge-m3/discussions/29 +bgem3_languages = [ + "afr_Latn", # af + # als + "amh_Ethi", # am + # an + # ar + "azj_Latn", # arz + # as + "ast_Latn", # ast + # av + # az + "azj_Latn", # azb + # ba + # bar + # bcl + "ben_Beng", # be + "bul_Cyrl", # bg + # bh + # bn + # bo + "bel_Cyrl", # bpy + # br + # bs + # bxr + "cat_Latn", # ca + # cbk + # ce + "ceb_Latn", # ceb + "ckb_Arab", # ckb + # co + # cs + # cv + # cy + "dan_Latn", # da + "deu_Latn", # de + # diq + # dsb + # dty + # dv + "ell_Grek", # el + # eml + "eng_Latn", # en + # eo + "est_Latn", # es + # et + # eu + # fa + "fin_Latn", # fi + "fra_Latn", # fr + # fy + # ga + # gd + "glg_Latn", # gl + # gn + # gom + "guj_Gujr", # gu + # gv + "heb_Hebr", # he + "hin_Deva", # hi + # hif + # hr + # hsb + # ht + # hu + # hy + # ia + # id + # ie + # ilo + # io + # is + "ita_Latn", # it + "jpn_Jpan", # ja + # jbo + # jv + # ka + # kk + # km + # kn + "kor_Hang", # ko + # krc + # ku + # kv + # kw + # ky + # la + # lb + # lez + # li + # lmo + # lo + # lt + # lv + # mai + # mg + # mhr + # min + # mk + # ml + # mn + # mr + # mrj + # ms + # mt + # mwl + # my + # myv + # mzn + # nah + # nap + # nds + # ne + # new + # nl + # nn + # no + # oc + # or + # os + # pa + # pam + # pfl + # pl + # pms + # pnb + # ps + # pt + # qu + # rm + # ro + "rus_Cyrl", # ru + # sa + # sah + # sc + # scn + # sco + # sd + # sh + # si + # sk + # sl + # so + # sq + # sr + # su + # sv + # sw + # ta + # te + # tg + "tha_Thai", # th + # tk + # tl + # tr + # tt + # tyv + # ug + "ukr_Cyrl", # uk + # ur + # uz + # vec + # vep + # vi + # vls + # vo + # wa + # war + # wuu + # xal + # xmf + # yi + # yo + # yue + "zho_Hans", # zh +] + bge_small_en_v1_5 = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="BAAI/bge-small-en-v1.5", revision="5c38ec7c405ec4b44b94cc5a9bb96e735b38267a", @@ -19,18 +304,20 @@ revision="5c38ec7c405ec4b44b94cc5a9bb96e735b38267a", release_date="2023-09-12", # initial commit of hf model. n_parameters=24_000_000, - memory_usage=None, embed_dim=512, license="mit", max_tokens=512, reference="https://huggingface.co/BAAI/bge-small-en-v1.5", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=True, + public_training_code=None, + public_training_data="https://data.baai.ac.cn/details/BAAI-MTP", + training_datasets=bge_training_data, ) bge_base_en_v1_5 = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="BAAI/bge-base-en-v1.5", revision="a5beb1e3e68b9ab74eb54cfd186867f64f240e1a", @@ -42,18 +329,20 @@ revision="a5beb1e3e68b9ab74eb54cfd186867f64f240e1a", release_date="2023-09-11", # initial commit of hf model. n_parameters=438_000_000, - memory_usage=None, embed_dim=768, license="mit", max_tokens=512, reference="https://huggingface.co/BAAI/bge-base-en-v1.5", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=True, + public_training_code=None, # seemingly released (at least for some models, but the link is broken + public_training_data="https://data.baai.ac.cn/details/BAAI-MTP", + training_datasets=bge_training_data, ) bge_large_en_v1_5 = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="BAAI/bge-large-en-v1.5", revision="d4aa6901d3a41ba39fb536a557fa166f842b0e09", @@ -65,12 +354,227 @@ revision="d4aa6901d3a41ba39fb536a557fa166f842b0e09", release_date="2023-09-12", # initial commit of hf model. n_parameters=1_340_000_000, - memory_usage=None, embed_dim=1024, license="mit", max_tokens=512, reference="https://huggingface.co/BAAI/bge-large-en-v1.5", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=True, + public_training_code=None, # seemingly released (at least for some models, but the link is broken + public_training_data="https://data.baai.ac.cn/details/BAAI-MTP", + training_datasets=bge_training_data, +) + +bge_small_zh_v1_5 = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="BAAI/bge-small-zh-v1.5", + revision="7999e1d3359715c523056ef9478215996d62a620", + model_prompts=model_prompts_zh, + ), + name="BAAI/bge-small-zh-v1.5", + languages=["zho_Hans"], + open_weights=True, + revision="7999e1d3359715c523056ef9478215996d62a620", + release_date="2023-09-12", # initial commit of hf model. + n_parameters=24_000_000, + embed_dim=512, + license="mit", + max_tokens=512, + reference="https://huggingface.co/BAAI/bge-small-zh-v1.5", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=bge_chinese_training_data, +) + +bge_base_zh_v1_5 = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="BAAI/bge-base-zh-v1.5", + revision="f03589ceff5aac7111bd60cfc7d497ca17ecac65", + model_prompts=model_prompts_zh, + ), + name="BAAI/bge-base-zh-v1.5", + languages=["zho_Hans"], + open_weights=True, + revision="f03589ceff5aac7111bd60cfc7d497ca17ecac65", + release_date="2023-09-11", # initial commit of hf model. + n_parameters=438_000_000, + embed_dim=768, + license="mit", + max_tokens=512, + reference="https://huggingface.co/BAAI/bge-base-zh-v1.5", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=bge_chinese_training_data, +) + +bge_large_zh_v1_5 = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="BAAI/bge-large-zh-v1.5", + revision="79e7739b6ab944e86d6171e44d24c997fc1e0116", + model_prompts=model_prompts_zh, + ), + name="BAAI/bge-large-zh-v1.5", + languages=["zho_Hans"], + open_weights=True, + revision="79e7739b6ab944e86d6171e44d24c997fc1e0116", + release_date="2023-09-12", # initial commit of hf model. + n_parameters=1_340_000_000, + embed_dim=1024, + license="mit", + max_tokens=512, + reference="https://huggingface.co/BAAI/bge-large-zh-v1.5", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=bge_chinese_training_data, +) + +bge_m3 = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="BAAI/bge-m3", + revision="5617a9f61b028005a4858fdac845db406aefb181", + ), + name="BAAI/bge-m3", + languages=bgem3_languages, + open_weights=True, + revision="5617a9f61b028005a4858fdac845db406aefb181", + release_date="2024-06-28", + n_parameters=568_000_000, + embed_dim=4096, + license="mit", + max_tokens=8194, + reference="https://huggingface.co/BAAI/bge-m3", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + public_training_code=None, + public_training_data="https://huggingface.co/datasets/cfli/bge-full-data", + training_datasets=bge_m3_training_data, +) + + +bge_multilingual_gemma2 = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="BAAI/bge-multilingual-gemma2", + revision="992e13d8984fde2c31ef8a3cb2c038aeec513b8a", + ), + name="BAAI/bge-multilingual-gemma2", + languages=[ + "eng_Latn", + "zho_Hans", + "kor_Hang", + "kor_Latn", + "fra_Latn", + "jpn_Jpan", + "jpn_Latn", + ], # This list is incomlete. Their description says "and more". + # I'm also unsure about the scripts. + open_weights=True, + revision="992e13d8984fde2c31ef8a3cb2c038aeec513b8a", + release_date="2024-07-25", # initial commit of hf model. + n_parameters=9.24 * 1e9, + embed_dim=3584, # from old C-MTEB leaderboard + license="gemma", + max_tokens=8192, # from old C-MTEB leaderboard + reference="https://huggingface.co/BAAI/bge-multilingual-gemma2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + public_training_code=None, + public_training_data=None, + training_datasets=None, # not disclosed +) + +# Contents of cfli/bge-full-data +bge_full_data = { + # source: https://arxiv.org/pdf/2409.15700 + # Charles Goodhart is turning back and forth + # in his grave as I'm annotating this + # |Retrieval| + # ELI5 + # SQuaD + # TriviaQA + # QuoraDuplicateQuestions + "HotpotQA": ["train"], + "FEVER": ["train"], + "MSMARCO": ["train"], + "NQ": ["train"], + "ArguAna": ["train"], + "FiQA2018": ["train"], + # |Reranking| + "SciDocsReranking": ["train"], + "StackOverflowDupQuestions": ["train"], + # |Classification| + "AmazonReviewsClassification": ["train"], + "AmazonCounterfactualClassification": ["train"], + "Banking77Classification": ["train"], + "EmotionClassification": ["train"], + "TweetSentimentExtractionClassification": ["train"], + "MTOPIntentClassification": ["train"], + "ImdbClassification": ["train"], + "ToxicConversationsClassification": ["train"], + # |Clustering| + "ArxivClusteringS2S": ["train"], + "ArxivClusteringP2P": ["train"], + "BiorxivClusteringS2S": ["train"], + "BiorxivClusteringP2P": ["train"], + "MedrxivClusteringS2S": ["train"], + "MedrxivClusteringP2P": ["train"], + "BiorxivClusteringS2S.v2": ["train"], + "BiorxivClusteringP2P.v2": ["train"], + "MedrxivClusteringS2S.v2": ["train"], + "MedrxivClusteringP2P.v2": ["train"], + "RedditClusteringP2P": ["train"], + "RedditClustering": ["train"], + "RedditClustering.v2": ["train"], + "TwentyNewsgroupsClustering": ["train"], + "TwentyNewsgroupsClustering.v2": ["train"], + # |STS| + "STS22": ["train"], + "STS22.v2": ["train"], + "STSBenchmark": ["train"], +} + +bge_en_icl = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="BAAI/bge-en-icl", + revision="971c7e1445cc86656ca0bd85ed770b8675a40bb5", + ), + name="BAAI/bge-en-icl", + languages=[ + "eng_Latn", + ], + open_weights=True, + revision="971c7e1445cc86656ca0bd85ed770b8675a40bb5", + release_date="2024-07-25", # initial commit of hf model. + n_parameters=7.11 * 1e9, + embed_dim=4096, + license="apache-2", + max_tokens=32768, + reference="https://huggingface.co/BAAI/bge-en-icl", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + public_training_code="https://github.com/FlagOpen/FlagEmbedding", + public_training_data="https://huggingface.co/datasets/cfli/bge-full-data", + training_datasets={ + **E5_MISTRAL_TRAINING_DATA, + **bge_full_data, + }, + adapted_from="intfloat/e5-mistral-7b-instruct", ) diff --git a/mteb/models/blip2_models.py b/mteb/models/blip2_models.py index d013e49c28..af15e205c7 100644 --- a/mteb/models/blip2_models.py +++ b/mteb/models/blip2_models.py @@ -222,10 +222,21 @@ def get_fused_embeddings( ), name="Salesforce/blip2-opt-2.7b", languages=["eng_Latn"], - open_source=True, revision="51572668da0eb669e01a189dc22abe6088589a24", release_date="2024-03-22", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) blip2_opt_6_7b_coco = ModelMeta( @@ -235,10 +246,21 @@ def get_fused_embeddings( ), name="Salesforce/blip2-opt-6.7b-coco", languages=["eng_Latn"], - open_source=True, revision="0d580de59320a25a4d2c386387bcef310d5f286e", release_date="2024-03-31", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) diff --git a/mteb/models/blip_models.py b/mteb/models/blip_models.py index 1be907e9b8..d1670af4dc 100644 --- a/mteb/models/blip_models.py +++ b/mteb/models/blip_models.py @@ -161,10 +161,21 @@ def get_fused_embeddings( ), name="Salesforce/blip-image-captioning-large", languages=["eng_Latn"], - open_source=True, revision="2227ac38c9f16105cb0412e7cab4759978a8fd90", release_date="2023-12-07", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) blip_image_captioning_base = ModelMeta( @@ -174,10 +185,21 @@ def get_fused_embeddings( ), name="Salesforce/blip-image-captioning-base", languages=["eng_Latn"], - open_source=True, revision="89b09ea1789f7addf2f6d6f0dfc4ce10ab58ef84", release_date="2023-08-01", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) @@ -188,10 +210,21 @@ def get_fused_embeddings( ), name="Salesforce/blip-vqa-base", languages=["eng_Latn"], - open_source=True, revision="c7df8e7cd7aa2ee9af18f56e2b29e59a92651b64", release_date="2023-12-07", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) blip_vqa_capfilt_large = ModelMeta( @@ -201,10 +234,21 @@ def get_fused_embeddings( ), name="Salesforce/blip-vqa-capfilt-large", languages=["eng_Latn"], - open_source=True, revision="e53f95265aeab69013fabb5380500ab984adbbb4", release_date="2023-01-22", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) blip_itm_base_coco = ModelMeta( @@ -214,10 +258,21 @@ def get_fused_embeddings( ), name="Salesforce/blip-itm-base-coco", languages=["eng_Latn"], - open_source=True, revision="7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f", release_date="2023-08-01", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) blip_itm_large_coco = ModelMeta( @@ -227,10 +282,21 @@ def get_fused_embeddings( ), name="Salesforce/blip-itm-large-coco", languages=["eng_Latn"], - open_source=True, revision="fef05cafc05298067cbbca00b125749394a77a6f", release_date="2023-08-01", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) blip_itm_base_flickr = ModelMeta( @@ -240,10 +306,21 @@ def get_fused_embeddings( ), name="Salesforce/blip-itm-base-flickr", languages=["eng_Latn"], - open_source=True, revision="1de29e660d91ae1786c1876212ea805a22eab251", release_date="2023-08-01", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) blip_itm_large_flickr = ModelMeta( @@ -253,10 +330,21 @@ def get_fused_embeddings( ), name="Salesforce/blip-itm-large-flickr", languages=["eng_Latn"], - open_source=True, revision="bda12e6506758f54261b5ab174b2c55a3ba143fb", release_date="2023-08-01", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) diff --git a/mteb/models/bm25.py b/mteb/models/bm25.py index 180929329b..6e3d3747d9 100644 --- a/mteb/models/bm25.py +++ b/mteb/models/bm25.py @@ -17,7 +17,7 @@ def bm25_loader(**kwargs): import Stemmer except ImportError: raise ImportError( - "bm25s or Stemmer is not installed. Please install it with `pip install bm25s Stemmer`." + "bm25s or PyStemmer is not installed. Please install it with `pip install mteb[bm25s]`." ) class BM25Search(DRESModel, Wrapper): @@ -58,7 +58,17 @@ def search( ) -> dict[str, dict[str, float]]: logger.info("Encoding Corpus...") corpus_ids = list(corpus.keys()) - corpus_with_ids = [{"doc_id": cid, **corpus[cid]} for cid in corpus_ids] + corpus_with_ids = [ + { + "doc_id": cid, + **( + {"text": corpus[cid]} + if isinstance(corpus[cid], str) + else corpus[cid] + ), + } + for cid in corpus_ids + ] corpus_texts = [ "\n".join([doc.get("title", ""), doc["text"]]) @@ -121,12 +131,14 @@ def encode(self, texts: list[str], **kwargs): revision="0_1_10", release_date="2024-07-10", ## release of version 0.1.10 n_parameters=None, - memory_usage=None, embed_dim=None, license=None, max_tokens=None, - reference=None, + reference="https://github.com/xhluca/bm25s", similarity_fn_name=None, framework=[], - use_instuctions=False, + use_instructions=False, + public_training_code="https://github.com/xhluca/bm25s", + public_training_data=None, + training_datasets=None, ) diff --git a/mteb/models/cde_models.py b/mteb/models/cde_models.py new file mode 100644 index 0000000000..78870ef129 --- /dev/null +++ b/mteb/models/cde_models.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import logging + +from mteb.model_meta import ModelMeta + +from .bge_models import bge_full_data + +logger = logging.getLogger(__name__) + + +cde_small_v1 = ModelMeta( + loader=None, # I will leave this at None for now, + name="jxm/cde-small-v1", + languages=["eng_Latn"], + open_weights=True, + revision="8d5736163718a8b65cd787b75ed61020d18bad3c", + release_date="2024-09-24", + n_parameters=int(281 * 1e6), # Though the second-stage model is only 140M + max_tokens=512, + embed_dim=768, + license="mit", + similarity_fn_name="cosine", + framework=["Sentence Transformers"], + reference="https://huggingface.co/jxm/cde-small-v1", + use_instructions=True, + adapted_from="nomic-ai/nomic-bert-2048", + superseded_by="jxm/cde-small-v2", + training_datasets=bge_full_data, + public_training_code="https://github.com/jxmorris12/cde", + public_training_data="https://huggingface.co/datasets/cfli/bge-full-data", +) + +cde_small_v2 = ModelMeta( + loader=None, # I will leave this at None for now, + name="jxm/cde-small-v2", + languages=["eng_Latn"], + open_weights=True, + revision="a7e5882ad52c27ea2831fc8258f24379c25cb459", + release_date="2025-01-13", + n_parameters=int(306 * 1e6), # Though the second-stage model is only 140M + max_tokens=512, + embed_dim=768, + license="mit", + similarity_fn_name="cosine", + framework=["Sentence Transformers"], + reference="https://huggingface.co/jxm/cde-small-v1", + use_instructions=True, + adapted_from="answerdotai/ModernBERT-base", + superseded_by="jxm/cde-small-v2", + training_datasets=bge_full_data, + public_training_code="https://github.com/jxmorris12/cde", + public_training_data="https://huggingface.co/datasets/cfli/bge-full-data", +) diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index 141a684da8..089732cffe 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -144,10 +144,21 @@ def get_fused_embeddings( ), name="openai/clip-vit-large-patch14", languages=["eng_Latn"], - open_source=True, revision="32bd64288804d66eefd0ccbe215aa642df71cc41", release_date="2021-02-26", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) clip_vit_base_patch32 = ModelMeta( @@ -157,10 +168,21 @@ def get_fused_embeddings( ), name="openai/clip-vit-base-patch32", languages=["eng_Latn"], - open_source=True, revision="3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", release_date="2021-02-26", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) clip_vit_base_patch16 = ModelMeta( @@ -170,10 +192,21 @@ def get_fused_embeddings( ), name="openai/clip-vit-base-patch16", languages=["eng_Latn"], - open_source=True, revision="57c216476eefef5ab752ec549e440a49ae4ae5f3", release_date="2021-02-26", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) if __name__ == "__main__": diff --git a/mteb/models/cohere_models.py b/mteb/models/cohere_models.py index 276bc6587c..60ff63ee81 100644 --- a/mteb/models/cohere_models.py +++ b/mteb/models/cohere_models.py @@ -5,16 +5,125 @@ import numpy as np import torch +import tqdm from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta -from mteb.models.sentence_transformer_wrapper import ( - get_prompt_name, - validate_task_to_prompt_name, -) from .wrapper import Wrapper +supported_languages = [ + "afr-Latn", + "amh-Ethi", + "ara-Arab", + "asm-Beng", + "aze-Latn", + "bel-Cyrl", + "bul-Cyrl", + "ben-Beng", + "bod-Tibt", + "bos-Latn", + "cat-Latn", + "ceb-Latn", + "cos-Latn", + "ces-Latn", + "cym-Latn", + "dan-Latn", + "deu-Latn", + "ell-Grek", + "eng-Latn", + "epo-Latn", + "spa-Latn", + "est-Latn", + "eus-Latn", + "fas-Arab", + "fin-Latn", + "fra-Latn", + "fry-Latn", + "gle-Latn", + "gla-Latn", + "glg-Latn", + "guj-Gujr", + "hau-Latn", + "haw-Latn", + "heb-Hebr", + "hin-Deva", + "hmn-Latn", + "hrv-Latn", + "hat-Latn", + "hun-Latn", + "hye-Armn", + "ind-Latn", + "ibo-Latn", + "isl-Latn", + "ita-Latn", + "jpn-Jpan", + "jav-Latn", + "kat-Geor", + "kaz-Cyrl", + "khm-Khmr", + "kan-Knda", + "kor-Kore", + "kur-Arab", + "kir-Cyrl", + "lat-Latn", + "ltz-Latn", + "lao-Laoo", + "lit-Latn", + "lav-Latn", + "mlg-Latn", + "mri-Latn", + "mkd-Cyrl", + "mal-Mlym", + "mon-Cyrl", + "mar-Deva", + "msa-Latn", + "mlt-Latn", + "mya-Mymr", + "nep-Deva", + "nld-Latn", + "nor-Latn", + "nya-Latn", + "ori-Orya", + "pan-Guru", + "pol-Latn", + "por-Latn", + "ron-Latn", + "rus-Cyrl", + "kin-Latn", + "sin-Sinh", + "slk-Latn", + "slv-Latn", + "smo-Latn", + "sna-Latn", + "som-Latn", + "sqi-Latn", + "srp-Cyrl", + "sot-Latn", + "sun-Latn", + "swe-Latn", + "swa-Latn", + "tam-Taml", + "tel-Telu", + "tgk-Cyrl", + "tha-Thai", + "tuk-Latn", + "tgl-Latn", + "tur-Latn", + "tat-Cyrl", + "uig-Arab", + "ukr-Cyrl", + "urd-Arab", + "uzb-Latn", + "vie-Latn", + "wol-Latn", + "xho-Latn", + "yid-Hebr", + "yor-Latn", + "zho-Hans", + "zul-Latn", +] + # Implementation follows https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/blob/main/src/seb/registered_models/cohere_models.py class CohereTextEmbeddingModel(Wrapper): @@ -28,29 +137,47 @@ def __init__( self.model_name = model_name self.sep = sep self.model_prompts = ( - validate_task_to_prompt_name(model_prompts) if model_prompts else None + self.validate_task_to_prompt_name(model_prompts) if model_prompts else None ) def _embed( - self, sentences: list[str], cohere_task_type: str, retries: int = 5 + self, + sentences: list[str], + cohere_task_type: str, + show_progress_bar: bool = False, + retries: int = 5, ) -> torch.Tensor: import cohere # type: ignore + max_batch_size = 256 + + batches = [ + sentences[i : i + max_batch_size] + for i in range(0, len(sentences), max_batch_size) + ] + client = cohere.Client() - while retries > 0: # Cohere's API is not always reliable - try: - response = client.embed( - texts=list(sentences), - model=self.model_name, - input_type=cohere_task_type, - ) - break - except Exception as e: - print(f"Retrying... {retries} retries left.") - retries -= 1 - if retries == 0: - raise e - return torch.tensor(response.embeddings) + + all_embeddings = [] + + for batch in tqdm.tqdm(batches, leave=False, disable=not show_progress_bar): + while retries > 0: # Cohere's API is not always reliable + try: + response = client.embed( + texts=batch, + model=self.model_name, + input_type=cohere_task_type, + ) + break + except Exception as e: + print(f"Retrying... {retries} retries left.") + retries -= 1 + if retries == 0: + raise e + + all_embeddings.extend(torch.tensor(response.embeddings).numpy()) + + return np.array(all_embeddings) def encode( self, @@ -60,11 +187,24 @@ def encode( prompt_type: PromptType | None = None, **kwargs: Any, ) -> np.ndarray: - cohere_task_type = get_prompt_name(self.model_prompts, task_name, prompt_type) + prompt_name = self.get_prompt_name(self.model_prompts, task_name, prompt_type) + cohere_task_type = self.model_prompts.get(prompt_name) + if cohere_task_type is None: # search_document is recommended if unknown (https://cohere.com/blog/introducing-embed-v3) cohere_task_type = "search_document" - return self._embed(sentences, cohere_task_type=cohere_task_type).numpy() + + show_progress_bar = ( + False + if "show_progress_bar" not in kwargs + else kwargs.pop("show_progress_bar") + ) + + return self._embed( + sentences, + cohere_task_type=cohere_task_type, + show_progress_bar=show_progress_bar, + ) model_prompts = { @@ -76,43 +216,97 @@ def encode( } cohere_mult_3 = ModelMeta( - loader=partial( + loader=partial( # type: ignore CohereTextEmbeddingModel, model_name="embed-multilingual-v3.0", model_prompts=model_prompts, ), - name="embed-multilingual-v3.0", - languages=[], # Unknown, but support >100 languages + name="Cohere/Cohere-embed-multilingual-v3.0", + languages=supported_languages, open_weights=False, revision="1", release_date="2023-11-02", n_parameters=None, - memory_usage=None, max_tokens=None, - embed_dim=1024, + embed_dim=512, + reference="https://cohere.com/blog/introducing-embed-v3", license=None, similarity_fn_name="cosine", framework=["API"], - use_instuctions=False, + use_instructions=True, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, ) cohere_eng_3 = ModelMeta( - loader=partial( + loader=partial( # type: ignore CohereTextEmbeddingModel, - model_name="embed-multilingual-v3.0", + model_name="embed-english-v3.0", model_prompts=model_prompts, ), - name="embed-english-v3.0", + name="Cohere/Cohere-embed-english-v3.0", languages=["eng-Latn"], open_weights=False, + reference="https://cohere.com/blog/introducing-embed-v3", revision="1", release_date="2023-11-02", n_parameters=None, - memory_usage=None, - max_tokens=None, + max_tokens=512, embed_dim=1024, license=None, similarity_fn_name="cosine", framework=["API"], - use_instuctions=False, + use_instructions=True, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, +) + +cohere_mult_light_3 = ModelMeta( + loader=partial( + CohereTextEmbeddingModel, + model_name="embed-multilingual-light-v3.0", + model_prompts=model_prompts, + ), + name="Cohere/Cohere-embed-multilingual-light-v3.0", + languages=supported_languages, + open_weights=False, + revision="1", + reference="https://cohere.com/blog/introducing-embed-v3", + release_date="2023-11-02", + n_parameters=None, + max_tokens=512, + embed_dim=384, + license=None, + similarity_fn_name="cosine", + framework=["API"], + use_instructions=True, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, +) + +cohere_eng_light_3 = ModelMeta( + loader=partial( + CohereTextEmbeddingModel, + model_name="embed-english-light-v3.0", + model_prompts=model_prompts, + ), + name="Cohere/Cohere-embed-english-light-v3.0", + languages=["eng-Latn"], + open_weights=False, + reference="https://cohere.com/blog/introducing-embed-v3", + revision="1", + release_date="2023-11-02", + n_parameters=None, + max_tokens=512, + embed_dim=384, + license=None, + similarity_fn_name="cosine", + framework=["API"], + use_instructions=True, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, ) diff --git a/mteb/models/cohere_v.py b/mteb/models/cohere_v.py index 749fc42016..d9dcc481b4 100644 --- a/mteb/models/cohere_v.py +++ b/mteb/models/cohere_v.py @@ -185,34 +185,42 @@ def get_fused_embeddings( loader=partial(cohere_v_loader, model_name="embed-multilingual-v3.0"), name="embed-multilingual-v3.0-v", languages=[], # Unknown, but support >100 languages - open_source=False, revision="1", release_date="2024-10-24", n_parameters=None, - memory_usage=None, max_tokens=None, embed_dim=1024, license=None, similarity_fn_name="cosine", framework=[], modalities=["image", "text"], + open_weights=None, + public_training_code=None, + public_training_data=None, + reference=None, + use_instructions=None, + training_datasets=None, ) cohere_eng_3 = ModelMeta( loader=partial(cohere_v_loader, model_name="embed-english-v3.0"), name="embed-english-v3.0-v", languages=["eng-Latn"], - open_source=False, revision="1", release_date="2024-10-24", n_parameters=None, - memory_usage=None, max_tokens=None, embed_dim=1024, license=None, similarity_fn_name="cosine", framework=[], modalities=["image", "text"], + open_weights=None, + public_training_code=None, + public_training_data=None, + reference=None, + use_instructions=None, + training_datasets=None, ) if __name__ == "__main__": diff --git a/mteb/models/colbert_models.py b/mteb/models/colbert_models.py new file mode 100644 index 0000000000..8bb66948ad --- /dev/null +++ b/mteb/models/colbert_models.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import logging +from collections.abc import Sequence +from functools import partial +from typing import Any + +import numpy as np +import torch + +from mteb.encoder_interface import PromptType +from mteb.model_meta import ModelMeta + +from .wrapper import Wrapper + +logger = logging.getLogger(__name__) + + +class ColBERTWrapper(Wrapper): + def __init__( + self, + model_name: str, + revision: str | None = None, + model_prompts: dict[str, str] | None = None, + **kwargs, + ) -> None: + """Wrapper for ColBERT models. + + Args: + model_name: The ColBERT model to load from HuggingFace Hub. + revision: The revision of the model to use. + model_prompts: A dictionary mapping task names to prompt names. + First priority is given to the composed prompt of task name + prompt type (query or passage), then to the specific task prompt, + then to the composed prompt of task type + prompt type, then to the specific task type prompt, + and finally to the specific prompt type. + **kwargs: Additional arguments to pass to the model. + """ + try: + from pylate import models as colbert_model + except ModuleNotFoundError as e: + raise ModuleNotFoundError( + "To use the ColBERT models `pylate` is required. Please install it with `pip install mteb[pylate]`." + ) from e + + self.model_name = model_name + self.model = colbert_model.ColBERT(self.model_name, revision=revision, **kwargs) + if ( + model_prompts is None + and hasattr(self.model, "prompts") + and len(self.model.prompts) > 0 + ): + try: + model_prompts = self.validate_task_to_prompt_name(self.model.prompts) + except ValueError: + model_prompts = None + elif model_prompts is not None and hasattr(self.model, "prompts"): + logger.info(f"Model prompts will be overwritten with {model_prompts}") + self.model.prompts = model_prompts + self.model_prompts = self.validate_task_to_prompt_name(model_prompts) + + def encode( + self, + sentences: Sequence[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + """Encodes the given sentences using the encoder. + + Args: + sentences: The sentences to encode. + task_name: The name of the task. Pylate uses this to + determine which prompt to use from a specified dictionary. + prompt_type: The name type of prompt. (query or passage) + **kwargs: Additional arguments to pass to the encoder. + + The order of priorities for prompt selection are: + 1. Composed prompt of task name + prompt type (query or passage) + 2. Specific task prompt + 3. Composed prompt of task type + prompt type (query or passage) + 4. Specific task type prompt + 5. Specific prompt type (query or passage) + + Returns: + The encoded sentences as a numpy array. + """ + prompt_name = None + if self.model_prompts is not None: + prompt_name = self.get_prompt_name( + self.model_prompts, task_name, prompt_type + ) + if prompt_name: + logger.info( + f"Using prompt_name={prompt_name} for task={task_name} prompt_type={prompt_type}" + ) + else: + logger.info( + f"No model prompts found for task={task_name} prompt_type={prompt_type}" + ) + logger.info(f"Encoding {len(sentences)} sentences.") + + pred = self.model.encode( + sentences, + prompt_name=prompt_name, + is_query=True if prompt_type == PromptType.query else False, + **kwargs, + ) + + # encode returns a list of tensors shaped (x, token_dim) where x is the number of tokens in the sentence + # we need to pad these tensors to the same length + # Tensors have varying lengths; therefore, they need to be padded with zeros to ensure uniformity before being combined + # output shape will be (batch_size, len(max(tokens)), embedding_token_dim) + pred = torch.nn.utils.rnn.pad_sequence(pred, batch_first=True, padding_value=0) + + return pred.cpu().numpy() + + def similarity(self, a: np.ndarray, b: np.ndarray) -> np.ndarray: + """Computes the max-similarity max_sim(a[i], b[j]) for all i and j. + Works with a Tensor of the shape (batch_size, num_tokens, token_dim) + + Return: + Matrix with res[i][j] = max_sim(a[i], b[j]) + """ # noqa: D402 + if not isinstance(a, torch.Tensor): + a = torch.tensor(a, dtype=torch.float32) + + if not isinstance(b, torch.Tensor): + b = torch.tensor(b, dtype=torch.float32) + + if len(a.shape) == 2: + a = a.unsqueeze(0) + + if len(b.shape) == 2: + b = b.unsqueeze(0) + + scores = torch.einsum( + "ash,bth->abst", + a, + b, + ) + + return scores.max(axis=-1).values.sum(axis=-1) + + +colbert_v2 = ModelMeta( + loader=partial( + ColBERTWrapper, + model_name="colbert-ir/colbertv2.0", + ), + name="colbert-ir/colbertv2.0", + languages=["eng_Latn"], + open_weights=True, + revision="c1e84128e85ef755c096a95bdb06b47793b13acf", + public_training_code=None, + public_training_data=None, + release_date="2024-09-21", + n_parameters=110 * 1e6, + max_tokens=180, # Reduced for Benchmarking - see ColBERT paper + embed_dim=None, # Bag of Embeddings (128) for each token + license="mit", + similarity_fn_name="max_sim", + framework=["PyLate", "ColBERT"], + reference="https://huggingface.co/colbert-ir/colbertv2.0", + use_instructions=False, + adapted_from=None, + superseded_by=None, + training_datasets={ + "MSMARCO": ["train"], # dev? + }, +) + + +jina_colbert_v2 = ModelMeta( + loader=partial( + ColBERTWrapper, + model_name="jinaai/jina-colbert-v2", + query_prefix="[QueryMarker]", + document_prefix="[DocumentMarker]", + attend_to_expansion_tokens=True, + trust_remote_code=True, + ), + name="jinaai/jina-colbert-v2", + languages=[ # list of languages the model has been evaluated on + "ara-Arab", # Arabic + "ben-Beng", # Bengali + "deu-Latn", # German + "spa-Latn", # Spanish + "eng-Latn", # English + "fas-Arab", # Persian + "fin-Latn", # Finnish + "fra-Latn", # French + "hin-Deva", # Hindi + "ind-Latn", # Indonesian + "jpn-Jpan", # Japanese + "kor-Kore", # Korean + "rus-Cyrl", # Russian + "swa-Latn", # Swahili + "tel-Telu", # Telugu + "tha-Thai", # Thai + "yor-Latn", # Yoruba + "zho-Hans", # Chinese (Simplified) + "nld-Latn", # Dutch + "ita-Latn", # Italian + "por-Latn", # Portuguese + "vie-Latn", # Vietnamese + ], + open_weights=True, + revision="4cf816e5e2b03167b132a3c847a9ecd48ba708e1", + public_training_code=None, + public_training_data=None, + release_date="2024-08-16", + n_parameters=559 * 1e6, + max_tokens=8192, + embed_dim=None, # Bag of Embeddings (128) for each token + license="cc-by-nc-4.0", + similarity_fn_name="max_sim", + framework=["PyLate", "ColBERT"], + reference="https://huggingface.co/jinaai/jina-colbert-v2", + use_instructions=False, + adapted_from=None, + superseded_by=None, + training_datasets={ + "MSMARCO": ["train"], + "DuRetrieval": [], + "MIRACL": ["train"], + }, +) diff --git a/mteb/models/dino_models.py b/mteb/models/dino_models.py index 76512b30f9..51aad5a5b8 100644 --- a/mteb/models/dino_models.py +++ b/mteb/models/dino_models.py @@ -127,10 +127,21 @@ def get_fused_embeddings( ), name="facebook/dinov2-small", languages=["eng_Latn"], - open_source=True, revision="ed25f3a31f01632728cabb09d1542f84ab7b0056", release_date="2023-07-18", modalities=["image"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) dinov2_base = ModelMeta( @@ -140,10 +151,21 @@ def get_fused_embeddings( ), name="facebook/dinov2-base", languages=["eng_Latn"], - open_source=True, revision="f9e44c814b77203eaa57a6bdbbd535f21ede1415", release_date="2023-07-18", modalities=["image"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) dinov2_large = ModelMeta( @@ -153,10 +175,21 @@ def get_fused_embeddings( ), name="facebook/dinov2-large", languages=["eng_Latn"], - open_source=True, revision="47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", release_date="2023-07-18", modalities=["image"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) dinov2_giant = ModelMeta( @@ -166,10 +199,21 @@ def get_fused_embeddings( ), name="facebook/dinov2-giant", languages=["eng_Latn"], - open_source=True, revision="611a9d42f2335e0f921f1e313ad3c1b7178d206d", release_date="2023-07-18", modalities=["image"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) if __name__ == "__main__": diff --git a/mteb/models/e5_instruct.py b/mteb/models/e5_instruct.py index 0be991347e..3eed189d33 100644 --- a/mteb/models/e5_instruct.py +++ b/mteb/models/e5_instruct.py @@ -1,60 +1,35 @@ from __future__ import annotations -from collections.abc import Sequence from functools import partial -from typing import Any -import numpy as np import torch from mteb.model_meta import ModelMeta -from ..encoder_interface import PromptType -from .e5_models import E5_PAPER_RELEASE_DATE, XLMR_LANGUAGES -from .instructions import task_to_instruction -from .wrapper import Wrapper +from .e5_models import E5_PAPER_RELEASE_DATE, E5_TRAINING_DATA, XLMR_LANGUAGES +from .instruct_wrapper import instruct_wrapper MISTRAL_LANGUAGES = ["eng_Latn", "fra_Latn", "deu_Latn", "ita_Latn", "spa_Latn"] -def e5_instruction(instruction: str) -> str: - return f"Instruct: {instruction}\nQuery: " +E5_INSTRUCTION = "Instruct: {instruction}\nQuery: " -def e5_loader(**kwargs): - try: - from gritlm import GritLM - except ImportError: - raise ImportError( - "Please install `pip install gritlm` to use E5 Instruct models." - ) - - class E5InstructWrapper(GritLM, Wrapper): - def encode( - self, - sentences: Sequence[str], - *args, - task_name: str, - prompt_type: PromptType | None = None, - **kwargs: Any, - ) -> np.ndarray: - if "instruction" in kwargs: - instruction = kwargs.pop("instruction", "") - else: - instruction = task_to_instruction( - task_name, prompt_type == PromptType.query - ) - if instruction: - kwargs["instruction"] = e5_instruction(instruction) - return super().encode(sentences, *args, **kwargs) - - return E5InstructWrapper(**kwargs) - +E5_MISTRAL_TRAINING_DATA = { + **E5_TRAINING_DATA, + "FEVER": ["train"], + "FEVERHardNegatives": ["train"], + "FEVER-PL": ["train"], # translation not trained on + "HotpotQA": ["train"], + "HotpotQAHardNegatives": ["train"], + "HotpotQA-PL": ["train"], # translation not trained on +} e5_instruct = ModelMeta( - loader=partial( - e5_loader, + loader=partial( # type: ignore + instruct_wrapper, model_name_or_path="intfloat/multilingual-e5-large-instruct", + instruction_template=E5_INSTRUCTION, attn="cccc", pooling_method="mean", mode="embedding", @@ -68,19 +43,22 @@ def encode( release_date=E5_PAPER_RELEASE_DATE, framework=["GritLM", "PyTorch"], similarity_fn_name="cosine", - use_instuctions=True, + use_instructions=True, reference="https://huggingface.co/intfloat/multilingual-e5-large-instruct", n_parameters=560_000_000, - memory_usage=None, embed_dim=1024, license="mit", max_tokens=514, + public_training_code=None, + public_training_data=None, + training_datasets=E5_TRAINING_DATA, ) e5_mistral = ModelMeta( - loader=partial( - e5_loader, + loader=partial( # type: ignore + instruct_wrapper, model_name_or_path="intfloat/e5-mistral-7b-instruct", + instruction_template=E5_INSTRUCTION, attn="cccc", pooling_method="lasttoken", mode="embedding", @@ -96,11 +74,88 @@ def encode( release_date=E5_PAPER_RELEASE_DATE, framework=["GritLM", "PyTorch"], similarity_fn_name="cosine", - use_instuctions=True, + use_instructions=True, reference="https://huggingface.co/intfloat/e5-mistral-7b-instruct", n_parameters=7_111_000_000, - memory_usage=None, embed_dim=4096, license="mit", max_tokens=32768, + public_training_code=None, + public_training_data=None, + training_datasets=E5_TRAINING_DATA, +) + +zeta_alpha_ai__Zeta_Alpha_E5_Mistral = ModelMeta( + loader=partial( # type: ignore + instruct_wrapper, + model_name_or_path="zeta-alpha-ai/Zeta-Alpha-E5-Mistral", + instruction_template=E5_INSTRUCTION, + attn="cccc", + pooling_method="lasttoken", + mode="embedding", + torch_dtype=torch.bfloat16, + # The ST script does not normalize while the HF one does so unclear what to do + # https://huggingface.co/intfloat/e5-mistral-7b-instruct#transformers + normalized=True, + ), + name="zeta-alpha-ai/Zeta-Alpha-E5-Mistral", + revision="c791d37474fa6a5c72eb3a2522be346bc21fbfc3", + release_date="2024-08-30", + languages=["eng_Latn"], + n_parameters=7110660096, + max_tokens=32768.0, + embed_dim=4096, + license="mit", + open_weights=True, + public_training_data=None, + public_training_code=None, + framework=["PyTorch"], + reference="https://huggingface.co/zeta-alpha-ai/Zeta-Alpha-E5-Mistral", + similarity_fn_name="cosine", + use_instructions=True, + training_datasets={ + # copied from e5 + # source: https://arxiv.org/pdf/2212.03533 + "NQ": ["test"], + "NQHardNegatives": ["test"], + "MSMARCO": ["train"], # dev? + # source: https://www.zeta-alpha.com/post/fine-tuning-an-llm-for-state-of-the-art-retrieval-zeta-alpha-s-top-10-submission-to-the-the-mteb-be + # "Arguana", + # "FEVER", + # "FIQA", + # "HotPotQA", + # "MsMarco (passage)", + # "NFCorpus", + # "SciFact", + # "NLI", + # "SQuad", + # "StackExchange", + # "TriviaQA", + # "SciRep", + # "SciRepEval" + # mteb + # https://huggingface.co/datasets/mteb/raw_arxiv + # "ArxivClusteringS2S": ["train"], + # "ArxivClusteringP2P": ["train"], + # https://huggingface.co/datasets/mteb/raw_biorxiv + # "BiorxivClusteringS2S": ["train"], + # "BiorxivClusteringP2P": ["train"], + # https://huggingface.co/datasets/mteb/raw_medrxiv + # "MedrxivClusteringS2S": ["train"], + # "MedrxivClusteringP2P": ["train"], + # as their train datasets + "AmazonCounterfactualClassification": ["train"], + "AmazonReviewsClassification": ["train"], + "Banking77Classification": ["train"], + "EmotionClassification": ["train"], + "MTOPIntentClassification": ["train"], + "ToxicConversationsClassification": ["train"], + "TweetSentimentExtractionClassification": ["train"], + "ImdbClassification": ["train"], + "STS12": ["train"], + "STS22": ["train"], + "STSBenchmark": ["train"], + }, + adapted_from="intfloat/e5-mistral-7b-instruct", + superseded_by=None, ) diff --git a/mteb/models/e5_models.py b/mteb/models/e5_models.py index 5549c7dd86..0ad15e7320 100644 --- a/mteb/models/e5_models.py +++ b/mteb/models/e5_models.py @@ -113,8 +113,31 @@ PromptType.passage.value: "passage: ", } +E5_TRAINING_DATA = { + # from 4.2 in https://arxiv.org/pdf/2212.03533 + # also pre-training data from a variety of sources (stackexchange, semantic scholar, reddit, CC, ...) + "MSMARCO": ["train"], + "MSMARCOHardNegatives": ["train"], + "NanoMSMARCORetrieval": ["train"], + "MSMARCO-PL": ["train"], # translation not trained on + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on +} + +ME5_TRAINING_DATA = { + **E5_TRAINING_DATA, + "FEVER": ["train"], + "FEVERHardNegatives": ["train"], + "FEVER-PL": ["train"], # translation not trained on + "HotpotQA": ["train"], + "HotpotQAHardNegatives": ["train"], + "HotpotQA-PL": ["train"], # translation not trained on +} + e5_mult_small = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="intfloat/multilingual-e5-small", revision="fd1525a9fd15316a2d503bf26ab031a61d056e98", @@ -126,18 +149,20 @@ revision="fd1525a9fd15316a2d503bf26ab031a61d056e98", release_date=E5_PAPER_RELEASE_DATE, n_parameters=118_000_000, - memory_usage=None, embed_dim=384, license="mit", max_tokens=512, reference="https://huggingface.co/intfloat/multilingual-e5-small", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=ME5_TRAINING_DATA, ) e5_mult_base = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="intfloat/multilingual-e5-base", model_prompts=model_prompts, @@ -148,18 +173,20 @@ revision="d13f1b27baf31030b7fd040960d60d909913633f", release_date=E5_PAPER_RELEASE_DATE, n_parameters=278_000_000, - memory_usage=None, embed_dim=768, license="mit", max_tokens=514, reference="https://huggingface.co/intfloat/multilingual-e5-base", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=ME5_TRAINING_DATA, ) e5_mult_large = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="intfloat/multilingual-e5-large", revision="ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb", @@ -171,18 +198,20 @@ revision="ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb", release_date=E5_PAPER_RELEASE_DATE, n_parameters=560_000_000, - memory_usage=None, embed_dim=1024, license="mit", max_tokens=514, reference="https://huggingface.co/intfloat/multilingual-e5-large", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=ME5_TRAINING_DATA, ) e5_eng_small_v2 = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="intfloat/e5-small-v2", model_prompts=model_prompts, @@ -193,18 +222,20 @@ revision="dca8b1a9dae0d4575df2bf423a5edb485a431236", release_date=E5_PAPER_RELEASE_DATE, n_parameters=33_000_000, - memory_usage=None, embed_dim=384, license="mit", max_tokens=512, reference="https://huggingface.co/intfloat/e5-small-v2", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=E5_TRAINING_DATA, ) e5_eng_small = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="intfloat/e5-small", revision="e272f3049e853b47cb5ca3952268c6662abda68f", @@ -216,18 +247,20 @@ revision="e272f3049e853b47cb5ca3952268c6662abda68f", release_date=E5_PAPER_RELEASE_DATE, n_parameters=33_000_000, - memory_usage=None, embed_dim=384, license="mit", max_tokens=512, reference="https://huggingface.co/intfloat/e5-small", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=E5_TRAINING_DATA, ) e5_eng_base_v2 = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="intfloat/e5-base-v2", revision="1c644c92ad3ba1efdad3f1451a637716616a20e8", @@ -238,19 +271,23 @@ open_weights=True, revision="1c644c92ad3ba1efdad3f1451a637716616a20e8", release_date=E5_PAPER_RELEASE_DATE, - n_parameters=278_000_000, - memory_usage=None, + n_parameters=109_000_000, embed_dim=768, license="mit", - max_tokens=514, + max_tokens=512, reference="https://huggingface.co/intfloat/e5-base-v2", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=True, + use_instructions=True, + superseded_by=None, + adapted_from=None, + public_training_code=None, + public_training_data=None, + training_datasets=E5_TRAINING_DATA, ) e5_eng_large_v2 = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="intfloat/e5-large-v2", revision="b322e09026e4ea05f42beadf4d661fb4e101d311", @@ -261,13 +298,71 @@ open_weights=True, revision="b322e09026e4ea05f42beadf4d661fb4e101d311", release_date=E5_PAPER_RELEASE_DATE, - n_parameters=560_000_000, - memory_usage=None, + n_parameters=335_000_000, embed_dim=1024, license="mit", max_tokens=514, reference="https://huggingface.co/intfloat/e5-large-v2", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=True, + use_instructions=True, + superseded_by=None, + adapted_from=None, + public_training_code=None, + public_training_data=None, + training_datasets=E5_TRAINING_DATA, +) + +e5_large = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="intfloat/e5-large", + revision="4dc6d853a804b9c8886ede6dda8a073b7dc08a81", + model_prompts=model_prompts, + ), + name="intfloat/e5-large", + languages=["eng-Latn"], + open_weights=True, + revision="4dc6d853a804b9c8886ede6dda8a073b7dc08a81", + release_date="2022-12-26", + n_parameters=335_000_000, + embed_dim=1024, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/intfloat/e5-large", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + superseded_by="intfloat/e5-large-v2", + adapted_from=None, + public_training_code=None, + public_training_data=None, + training_datasets=E5_TRAINING_DATA, +) + +e5_base = ModelMeta( + loader=partial( + sentence_transformers_loader, + model_name="intfloat/e5-base", + revision="b533fe4636f4a2507c08ddab40644d20b0006d6a", + model_prompts=model_prompts, + ), + name="intfloat/e5-base", + languages=["eng-Latn"], + open_weights=True, + revision="b533fe4636f4a2507c08ddab40644d20b0006d6a", + release_date="2022-12-26", + n_parameters=109_000_000, + embed_dim=768, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/intfloat/e5-base", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + superseded_by="intfloat/e5-base-v2", + adapted_from=None, + public_training_code=None, + public_training_data=None, + training_datasets=E5_TRAINING_DATA, ) diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index 8b96427f61..ba72c42c0f 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -195,10 +195,21 @@ def get_fused_embeddings( ), name="royokong/e5-v", languages=["eng_Latn"], - open_source=True, revision="0c1f22679417b3ae925d779442221c40cd1861ab", release_date="2024-07-17", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) if __name__ == "__main__": diff --git a/mteb/models/evaclip_models.py b/mteb/models/evaclip_models.py index 9be55cfc85..c5395bfa25 100644 --- a/mteb/models/evaclip_models.py +++ b/mteb/models/evaclip_models.py @@ -172,10 +172,21 @@ def get_fused_embeddings( ), name="EVA02-CLIP-B-16", languages=["eng_Latn"], - open_source=True, revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) EVA02_CLIP_L_14 = ModelMeta( @@ -185,10 +196,21 @@ def get_fused_embeddings( ), name="EVA02-CLIP-L-14", languages=["eng_Latn"], - open_source=True, revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) EVA02_CLIP_bigE_14 = ModelMeta( @@ -198,10 +220,21 @@ def get_fused_embeddings( ), name="EVA02-CLIP-bigE-14", languages=["eng_Latn"], - open_source=True, revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) @@ -212,8 +245,19 @@ def get_fused_embeddings( ), name="EVA02-CLIP-bigE-14-plus", languages=["eng_Latn"], - open_source=True, revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) diff --git a/mteb/models/gme_models.py b/mteb/models/gme_models.py index b04dbf68d0..804dfbc84d 100644 --- a/mteb/models/gme_models.py +++ b/mteb/models/gme_models.py @@ -1,449 +1,62 @@ from __future__ import annotations import logging -import math -import os -from functools import partial -from typing import Any -import torch -from PIL import Image -from torch.utils.data import DataLoader -from tqdm.autonotebook import tqdm -from transformers import AutoModelForVision2Seq, AutoProcessor - -import mteb -from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta -from .instructions import DEFAULT_PROMPTS, TASKNAME2INSTRUCTIONS - -logging.basicConfig(level=logging.WARNING) logger = logging.getLogger(__name__) -HF_GME_QWEN2VL_2B = "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct" -HF_GME_QWEN2VL_7B = "Alibaba-NLP/gme-Qwen2-VL-7B-Instruct" - - -def get_gme_instruction(task_name: str, is_query: bool = True) -> str: - # TODO Prompts for other multimodal tasks. - if task_name in TASKNAME2INSTRUCTIONS: - prompt = TASKNAME2INSTRUCTIONS[task_name] - if isinstance(prompt, tuple): - prompt = prompt[0] if is_query else prompt[1] - else: - meta = mteb.get_task(task_name).metadata - prompt = DEFAULT_PROMPTS.get(meta.type, None) - - if isinstance(prompt, str) and prompt[-1] != ".": - prompt += "." - return prompt - - -class Encoder(torch.nn.Module): - def __init__( - self, - base, - processor, - max_length=1800, - normalize=True, - ) -> None: - super().__init__() - self.base = base - self.processor = processor - self.max_length = max_length - self.normalize = normalize - self.processor.tokenizer.padding_side = "right" - self.defualt_instruction = "You are a helpful assistant." - - def forward( - self, - input_ids: torch.LongTensor | None = None, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: list[torch.FloatTensor] | None = None, - inputs_embeds: torch.FloatTensor | None = None, - pixel_values: torch.Tensor | None = None, - # pixel_values_videos: torch.FloatTensor | None = None, - image_grid_thw: torch.LongTensor | None = None, - # video_grid_thw: torch.LongTensor | None = None, - pooling_mask: torch.LongTensor | None = None, - **kwargs, - ) -> torch.Tensor: - if inputs_embeds is None: - inputs_embeds = self.base.model.embed_tokens(input_ids) - if pixel_values is not None: - pixel_values = pixel_values.type(self.base.visual.get_dtype()) - image_embeds = self.base.visual( - pixel_values, grid_thw=image_grid_thw - ).to(inputs_embeds.device) - image_mask = input_ids == self.base.config.image_token_id - inputs_embeds[image_mask] = image_embeds - # if pixel_values_videos is not None: - # pixel_values_videos = pixel_values_videos.type(self.base.visual.get_dtype()) - # video_embeds = self.base.visual(pixel_values_videos, grid_thw=video_grid_thw).to(inputs_embeds.device) - # video_mask = input_ids == self.base.config.video_token_id - # inputs_embeds[video_mask] = video_embeds - if attention_mask is not None: - attention_mask = attention_mask.to(inputs_embeds.device) - - outputs = self.base.model( - input_ids=None, - position_ids=position_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - ) - - pooling_mask = attention_mask if pooling_mask is None else pooling_mask - left_padding = pooling_mask[:, -1].sum() == pooling_mask.shape[0] # TODO - if left_padding: - embeddings = outputs.last_hidden_state[:, -1] - else: - sequence_lengths = pooling_mask.sum(dim=1) - 1 - batch_size = outputs.last_hidden_state.shape[0] - embeddings = outputs.last_hidden_state[ - torch.arange(batch_size, device=outputs.last_hidden_state.device), - sequence_lengths, - ] - if self.normalize: - embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1) - return embeddings.contiguous() - - def embed( - self, - texts: list[str], - images: list[Image.Image], - device, - instruction=None, - **kwargs, - ): - instruction = instruction or self.defualt_instruction - # Inputs must be batched - input_texts, input_images = [], [] - for t, i in zip(texts, images): - input_str = "" - if i is None: - input_images = None # All examples in the same batch are consistent - else: - input_str += "<|vision_start|><|image_pad|><|vision_end|>" - i = fetch_image(i) - input_images.append(i) - if t is not None: - input_str += t - msg = f"<|im_start|>system\n{instruction}<|im_end|>\n<|im_start|>user\n{input_str}<|im_end|>\n<|im_start|>assistant\n<|endoftext|>" - input_texts.append(msg) - - inputs = self.processor( - text=input_texts, - images=input_images, - padding=True, - truncation=True, - max_length=self.max_length, - return_tensors="pt", - ) - inputs = {k: v.to(device) for k, v in inputs.items()} # TODO - embeddings = self.forward(**inputs) - return embeddings - - -class GmeQwen2VL: - def __init__( - self, - model_name: str = HF_GME_QWEN2VL_2B, - model_path: str | None = None, - device: str = "cuda" if torch.cuda.is_available() else "cpu", - min_image_tokens=4, - max_image_tokens=1280, - max_length=1800, - **kwargs, - ) -> None: - model_name = model_path or model_name - base = AutoModelForVision2Seq.from_pretrained( - model_name, torch_dtype=torch.float16, **kwargs - ) - min_pixels = min_image_tokens * 28 * 28 - max_pixels = max_image_tokens * 28 * 28 - processor = AutoProcessor.from_pretrained( - model_name, min_pixels=min_pixels, max_pixels=max_pixels, **kwargs - ) - self.model = Encoder(base, processor, max_length=max_length) - self.model.eval() - self.device = device - self.sep = " " - - def encode( - self, - sentences: list[str], - *, - task_name: str | None = None, - prompt_type: PromptType | None = None, - **kwargs: Any, - ): - return self.get_fused_embeddings( - texts=sentences, task_name=task_name, prompt_type=prompt_type, **kwargs - ) - - def encode_queries(self, queries: list[str], **kwargs): - kwargs.update(prompt_type=PromptType.query) - embeddings = self.encode(queries, **kwargs) - return embeddings - - def encode_corpus(self, corpus: list[dict[str, str]], **kwargs): - if type(corpus) is dict: - sentences = [ - (corpus["title"][i] + self.sep + corpus["text"][i]).strip() - if "title" in corpus - else corpus["text"][i].strip() - for i in range(len(corpus["text"])) - ] - else: - sentences = [ - (doc["title"] + self.sep + doc["text"]).strip() - if "title" in doc - else doc["text"].strip() - for doc in corpus - ] - kwargs.update(prompt_type=PromptType.passage) - embeddings = self.encode(sentences, is_query=False, **kwargs) - return embeddings - - def get_image_embeddings(self, images: list[Image.Image] | DataLoader, **kwargs): - return self.get_fused_embeddings(images=images, **kwargs) - - def get_text_embeddings(self, texts: list[str], **kwargs): - return self.get_fused_embeddings(texts=texts, **kwargs) - - def calculate_probs(self, text_embeddings, image_embeddings): - text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) - image_embeddings = image_embeddings / image_embeddings.norm( - dim=-1, keepdim=True - ) - logits = torch.matmul(image_embeddings, text_embeddings.T) - probs = (logits * 100).softmax(dim=-1) - return probs - - def get_fused_embeddings( - self, - texts: list[str] | None = None, - images: list[Image.Image] | DataLoader | None = None, - task_name: str | None = None, - prompt_type: PromptType | None = None, - tqdm_mininterval: int = 15, - instruction=None, - **kwargs: Any, - ): - if prompt_type == PromptType.passage: - instruction = None - elif instruction is None: - instruction = get_gme_instruction(task_name) - self.model = self.model.to(self.device) - - if isinstance(images, DataLoader): - image_loader = images - batch_size = image_loader.batch_size - image_loader.dataset.transform = None - else: - batch_size = kwargs.pop("batch_size", 32) - if images is None: - image_loader = None - else: - image_loader = DataLoader( - images, - batch_size=batch_size, - shuffle=False, - collate_fn=custom_collate_fn, - num_workers=min(math.floor(os.cpu_count() / 2), 8), - ) - - if texts is None: - assert image_loader is not None - n_batch = len(image_loader) - else: - n_batch = len(texts) // batch_size + int(len(texts) % batch_size > 0) - image_loader = image_loader or [None] * n_batch - - all_embeddings = [] - none_batch = [None] * batch_size - show_progress_bar = kwargs.pop("show_progress_bar", True) - pbar = tqdm( - total=n_batch, - disable=not show_progress_bar, - mininterval=tqdm_mininterval, - miniters=n_batch // 10, - desc="encode", - ) - for n, (i, img_batch) in enumerate( - zip(range(0, n_batch * batch_size, batch_size), image_loader) - ): - text_batch = none_batch if texts is None else texts[i : i + batch_size] - img_batch = none_batch if img_batch is None else img_batch - inputs = dict( - texts=text_batch, images=img_batch, instruction=instruction, **kwargs - ) - with torch.inference_mode(): - embeddings = self.model.embed(**inputs, device=self.device) - all_embeddings.append(embeddings.cpu()) - pbar.update(1) - pbar.close() - all_embeddings = torch.cat(all_embeddings, dim=0) - return all_embeddings - - -def custom_collate_fn(batch): - return batch - -### Copied from qwen_vl_utils.vision_process.py -IMAGE_FACTOR = 28 -MIN_PIXELS = 4 * 28 * 28 -MAX_PIXELS = 16384 * 28 * 28 -MAX_RATIO = 200 - - -def round_by_factor(number: int, factor: int) -> int: - """Returns the closest integer to 'number' that is divisible by 'factor'.""" - return round(number / factor) * factor - - -def ceil_by_factor(number: int, factor: int) -> int: - """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'.""" - return math.ceil(number / factor) * factor - - -def floor_by_factor(number: int, factor: int) -> int: - """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'.""" - return math.floor(number / factor) * factor - - -def smart_resize( - height: int, - width: int, - factor: int = IMAGE_FACTOR, - min_pixels: int = MIN_PIXELS, - max_pixels: int = MAX_PIXELS, -) -> tuple[int, int]: - """Rescales the image so that the following conditions are met: - - 1. Both dimensions (height and width) are divisible by 'factor'. - - 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. - - 3. The aspect ratio of the image is maintained as closely as possible. - """ - h_bar = max(factor, round_by_factor(height, factor)) - w_bar = max(factor, round_by_factor(width, factor)) - if h_bar * w_bar > max_pixels: - beta = math.sqrt((height * width) / max_pixels) - h_bar = floor_by_factor(height / beta, factor) - w_bar = floor_by_factor(width / beta, factor) - elif h_bar * w_bar < min_pixels: - beta = math.sqrt(min_pixels / (height * width)) - h_bar = ceil_by_factor(height * beta, factor) - w_bar = ceil_by_factor(width * beta, factor) - - if max(h_bar, w_bar) / min(h_bar, w_bar) > MAX_RATIO: - logger.warning( - f"Absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(h_bar, w_bar) / min(h_bar, w_bar)}" - ) - if h_bar > w_bar: - h_bar = w_bar * MAX_RATIO - else: - w_bar = h_bar * MAX_RATIO - return h_bar, w_bar - - -def fetch_image( - image: str | Image.Image, size_factor: int = IMAGE_FACTOR -) -> Image.Image: - image_obj = None - if isinstance(image, Image.Image): - image_obj = image - elif image.startswith("http://") or image.startswith("https://"): - import requests - - image_obj = Image.open(requests.get(image, stream=True).raw) - elif image.startswith("file://"): - image_obj = Image.open(image[7:]) - elif image.startswith("data:image"): - import base64 - from io import BytesIO - - if "base64," in image: - _, base64_data = image.split("base64,", 1) - data = base64.b64decode(base64_data) - image_obj = Image.open(BytesIO(data)) - else: - image_obj = Image.open(image) - if image_obj is None: - raise ValueError( - f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}" - ) - image = image_obj.convert("RGB") - ## resize - # if "resized_height" in ele and "resized_width" in ele: - # resized_height, resized_width = smart_resize( - # ele["resized_height"], - # ele["resized_width"], - # factor=size_factor, - # ) - # else: - width, height = image.size - # min_pixels = ele.get("min_pixels", MIN_PIXELS) - # max_pixels = ele.get("max_pixels", MAX_PIXELS) - resized_height, resized_width = smart_resize( - height, - width, - factor=size_factor, - min_pixels=MIN_PIXELS, - max_pixels=MAX_PIXELS, - ) - image = image.resize((resized_width, resized_height)) - - return image - - -### - - -gme_qwen2vl_2b = ModelMeta( - loader=partial( - GmeQwen2VL, - model_name=HF_GME_QWEN2VL_2B, - ), - name=HF_GME_QWEN2VL_2B, - languages=["eng_Latn", "cmn-Hans"], +gme_qwen2_vl_2b_instruct = ModelMeta( + loader=None, + name="Alibaba-NLP/gme-Qwen2-VL-2B-Instruct", + languages=["eng_Latn"], open_weights=True, - revision="ce765ae71b8cdb208203cd8fb64a170b1b84293a", - release_date="2024-12-24", - n_parameters=2_210_000_000, - memory_usage=None, - embed_dim=1536, - license="apache-2.0", + revision="cfeb66885b598de483cc04eb08c7d9da534d7afe", + release_date="2024-12-21", + n_parameters=int(2.21 * 1e9), max_tokens=32768, - reference="https://huggingface.co/" + HF_GME_QWEN2VL_2B, + embed_dim=1536, + license="mit", similarity_fn_name="cosine", framework=["PyTorch"], - use_instuctions=True, + reference="https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct", + use_instructions=True, + adapted_from=None, + superseded_by=None, + training_datasets={ + # Only annotating text data for now + # source: https://arxiv.org/pdf/2412.16855 + "MSMARCO": ["train"], + "MSMARCO.v2": ["train"], + }, + public_training_code=None, + public_training_data=None, ) -gme_qwen2vl_7b = ModelMeta( - loader=partial( - GmeQwen2VL, - model_name=HF_GME_QWEN2VL_7B, - ), - name=HF_GME_QWEN2VL_7B, - languages=["eng_Latn", "cmn-Hans"], +gme_qwen2_vl_7b_instruct = ModelMeta( + loader=None, + name="Alibaba-NLP/gme-Qwen2-VL-2B-Instruct", + languages=["eng_Latn"], open_weights=True, - revision="477027a6480f8630363be77751f169cc3434b673", - release_date="2024-12-24", - n_parameters=8_290_000_000, - memory_usage=None, - embed_dim=3584, - license="apache-2.0", + revision="d42eca5a540526cfa982a349724b24b25c12a95e", + release_date="2024-12-21", + n_parameters=int(8.29 * 1e9), max_tokens=32768, - reference="https://huggingface.co/" + HF_GME_QWEN2VL_2B, + embed_dim=3584, + license="mit", similarity_fn_name="cosine", framework=["PyTorch"], - use_instuctions=True, + reference="https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct", + use_instructions=True, + adapted_from=None, + superseded_by=None, + training_datasets={ + # Only annotating text data for now + # source: https://arxiv.org/pdf/2412.16855 + "MSMARCO": ["train"], + "MSMARCO.v2": ["train"], + }, + public_training_code=None, + public_training_data=None, ) diff --git a/mteb/models/gme_v_models.py b/mteb/models/gme_v_models.py new file mode 100644 index 0000000000..be4f7b207a --- /dev/null +++ b/mteb/models/gme_v_models.py @@ -0,0 +1,453 @@ +from __future__ import annotations + +import logging +import math +import os +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm.autonotebook import tqdm +from transformers import AutoModelForVision2Seq, AutoProcessor + +import mteb +from mteb.encoder_interface import PromptType +from mteb.model_meta import ModelMeta + +from .instructions import DEFAULT_PROMPTS, TASKNAME2INSTRUCTIONS + +logging.basicConfig(level=logging.WARNING) +logger = logging.getLogger(__name__) + +HF_GME_QWEN2VL_2B = "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct" +HF_GME_QWEN2VL_7B = "Alibaba-NLP/gme-Qwen2-VL-7B-Instruct" + + +def get_gme_instruction(task_name: str, is_query: bool = True) -> str: + # TODO Prompts for other multimodal tasks. + if task_name in TASKNAME2INSTRUCTIONS: + prompt = TASKNAME2INSTRUCTIONS[task_name] + if isinstance(prompt, tuple): + prompt = prompt[0] if is_query else prompt[1] + else: + meta = mteb.get_task(task_name).metadata + prompt = DEFAULT_PROMPTS.get(meta.type, None) + + if isinstance(prompt, str) and prompt[-1] != ".": + prompt += "." + return prompt + + +class Encoder(torch.nn.Module): + def __init__( + self, + base, + processor, + max_length=1800, + normalize=True, + ) -> None: + super().__init__() + self.base = base + self.processor = processor + self.max_length = max_length + self.normalize = normalize + self.processor.tokenizer.padding_side = "right" + self.defualt_instruction = "You are a helpful assistant." + + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: list[torch.FloatTensor] | None = None, + inputs_embeds: torch.FloatTensor | None = None, + pixel_values: torch.Tensor | None = None, + # pixel_values_videos: torch.FloatTensor | None = None, + image_grid_thw: torch.LongTensor | None = None, + # video_grid_thw: torch.LongTensor | None = None, + pooling_mask: torch.LongTensor | None = None, + **kwargs, + ) -> torch.Tensor: + if inputs_embeds is None: + inputs_embeds = self.base.model.embed_tokens(input_ids) + if pixel_values is not None: + pixel_values = pixel_values.type(self.base.visual.get_dtype()) + image_embeds = self.base.visual( + pixel_values, grid_thw=image_grid_thw + ).to(inputs_embeds.device) + image_mask = input_ids == self.base.config.image_token_id + inputs_embeds[image_mask] = image_embeds + # if pixel_values_videos is not None: + # pixel_values_videos = pixel_values_videos.type(self.base.visual.get_dtype()) + # video_embeds = self.base.visual(pixel_values_videos, grid_thw=video_grid_thw).to(inputs_embeds.device) + # video_mask = input_ids == self.base.config.video_token_id + # inputs_embeds[video_mask] = video_embeds + if attention_mask is not None: + attention_mask = attention_mask.to(inputs_embeds.device) + + outputs = self.base.model( + input_ids=None, + position_ids=position_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + ) + + pooling_mask = attention_mask if pooling_mask is None else pooling_mask + left_padding = pooling_mask[:, -1].sum() == pooling_mask.shape[0] # TODO + if left_padding: + embeddings = outputs.last_hidden_state[:, -1] + else: + sequence_lengths = pooling_mask.sum(dim=1) - 1 + batch_size = outputs.last_hidden_state.shape[0] + embeddings = outputs.last_hidden_state[ + torch.arange(batch_size, device=outputs.last_hidden_state.device), + sequence_lengths, + ] + if self.normalize: + embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1) + return embeddings.contiguous() + + def embed( + self, + texts: list[str], + images: list[Image.Image], + device, + instruction=None, + **kwargs, + ): + instruction = instruction or self.defualt_instruction + # Inputs must be batched + input_texts, input_images = [], [] + for t, i in zip(texts, images): + input_str = "" + if i is None: + input_images = None # All examples in the same batch are consistent + else: + input_str += "<|vision_start|><|image_pad|><|vision_end|>" + i = fetch_image(i) + input_images.append(i) + if t is not None: + input_str += t + msg = f"<|im_start|>system\n{instruction}<|im_end|>\n<|im_start|>user\n{input_str}<|im_end|>\n<|im_start|>assistant\n<|endoftext|>" + input_texts.append(msg) + + inputs = self.processor( + text=input_texts, + images=input_images, + padding=True, + truncation=True, + max_length=self.max_length, + return_tensors="pt", + ) + inputs = {k: v.to(device) for k, v in inputs.items()} # TODO + embeddings = self.forward(**inputs) + return embeddings + + +class GmeQwen2VL: + def __init__( + self, + model_name: str = HF_GME_QWEN2VL_2B, + model_path: str | None = None, + device: str = "cuda" if torch.cuda.is_available() else "cpu", + min_image_tokens=4, + max_image_tokens=1280, + max_length=1800, + **kwargs, + ) -> None: + model_name = model_path or model_name + base = AutoModelForVision2Seq.from_pretrained( + model_name, torch_dtype=torch.float16, **kwargs + ) + min_pixels = min_image_tokens * 28 * 28 + max_pixels = max_image_tokens * 28 * 28 + processor = AutoProcessor.from_pretrained( + model_name, min_pixels=min_pixels, max_pixels=max_pixels, **kwargs + ) + self.model = Encoder(base, processor, max_length=max_length) + self.model.eval() + self.device = device + self.sep = " " + + def encode( + self, + sentences: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + **kwargs: Any, + ): + return self.get_fused_embeddings( + texts=sentences, task_name=task_name, prompt_type=prompt_type, **kwargs + ) + + def encode_queries(self, queries: list[str], **kwargs): + kwargs.update(prompt_type=PromptType.query) + embeddings = self.encode(queries, **kwargs) + return embeddings + + def encode_corpus(self, corpus: list[dict[str, str]], **kwargs): + if type(corpus) is dict: + sentences = [ + (corpus["title"][i] + self.sep + corpus["text"][i]).strip() + if "title" in corpus + else corpus["text"][i].strip() + for i in range(len(corpus["text"])) + ] + else: + sentences = [ + (doc["title"] + self.sep + doc["text"]).strip() + if "title" in doc + else doc["text"].strip() + for doc in corpus + ] + kwargs.update(prompt_type=PromptType.passage) + embeddings = self.encode(sentences, is_query=False, **kwargs) + return embeddings + + def get_image_embeddings(self, images: list[Image.Image] | DataLoader, **kwargs): + return self.get_fused_embeddings(images=images, **kwargs) + + def get_text_embeddings(self, texts: list[str], **kwargs): + return self.get_fused_embeddings(texts=texts, **kwargs) + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] | None = None, + images: list[Image.Image] | DataLoader | None = None, + task_name: str | None = None, + prompt_type: PromptType | None = None, + tqdm_mininterval: int = 15, + instruction=None, + **kwargs: Any, + ): + if prompt_type == PromptType.passage: + instruction = None + elif instruction is None: + instruction = get_gme_instruction(task_name) + self.model = self.model.to(self.device) + + if isinstance(images, DataLoader): + image_loader = images + batch_size = image_loader.batch_size + image_loader.dataset.transform = None + else: + batch_size = kwargs.pop("batch_size", 32) + if images is None: + image_loader = None + else: + image_loader = DataLoader( + images, + batch_size=batch_size, + shuffle=False, + collate_fn=custom_collate_fn, + num_workers=min(math.floor(os.cpu_count() / 2), 8), + ) + + if texts is None: + assert image_loader is not None + n_batch = len(image_loader) + else: + n_batch = len(texts) // batch_size + int(len(texts) % batch_size > 0) + image_loader = image_loader or [None] * n_batch + + all_embeddings = [] + none_batch = [None] * batch_size + show_progress_bar = kwargs.pop("show_progress_bar", True) + pbar = tqdm( + total=n_batch, + disable=not show_progress_bar, + mininterval=tqdm_mininterval, + miniters=n_batch // 10, + desc="encode", + ) + for n, (i, img_batch) in enumerate( + zip(range(0, n_batch * batch_size, batch_size), image_loader) + ): + text_batch = none_batch if texts is None else texts[i : i + batch_size] + img_batch = none_batch if img_batch is None else img_batch + inputs = dict( + texts=text_batch, images=img_batch, instruction=instruction, **kwargs + ) + with torch.inference_mode(): + embeddings = self.model.embed(**inputs, device=self.device) + all_embeddings.append(embeddings.cpu()) + pbar.update(1) + pbar.close() + all_embeddings = torch.cat(all_embeddings, dim=0) + return all_embeddings + + +def custom_collate_fn(batch): + return batch + + +### Copied from qwen_vl_utils.vision_process.py +IMAGE_FACTOR = 28 +MIN_PIXELS = 4 * 28 * 28 +MAX_PIXELS = 16384 * 28 * 28 +MAX_RATIO = 200 + + +def round_by_factor(number: int, factor: int) -> int: + """Returns the closest integer to 'number' that is divisible by 'factor'.""" + return round(number / factor) * factor + + +def ceil_by_factor(number: int, factor: int) -> int: + """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'.""" + return math.ceil(number / factor) * factor + + +def floor_by_factor(number: int, factor: int) -> int: + """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'.""" + return math.floor(number / factor) * factor + + +def smart_resize( + height: int, + width: int, + factor: int = IMAGE_FACTOR, + min_pixels: int = MIN_PIXELS, + max_pixels: int = MAX_PIXELS, +) -> tuple[int, int]: + """Rescales the image so that the following conditions are met: + + 1. Both dimensions (height and width) are divisible by 'factor'. + + 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. + + 3. The aspect ratio of the image is maintained as closely as possible. + """ + h_bar = max(factor, round_by_factor(height, factor)) + w_bar = max(factor, round_by_factor(width, factor)) + if h_bar * w_bar > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + h_bar = floor_by_factor(height / beta, factor) + w_bar = floor_by_factor(width / beta, factor) + elif h_bar * w_bar < min_pixels: + beta = math.sqrt(min_pixels / (height * width)) + h_bar = ceil_by_factor(height * beta, factor) + w_bar = ceil_by_factor(width * beta, factor) + + if max(h_bar, w_bar) / min(h_bar, w_bar) > MAX_RATIO: + logger.warning( + f"Absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(h_bar, w_bar) / min(h_bar, w_bar)}" + ) + if h_bar > w_bar: + h_bar = w_bar * MAX_RATIO + else: + w_bar = h_bar * MAX_RATIO + return h_bar, w_bar + + +def fetch_image( + image: str | Image.Image, size_factor: int = IMAGE_FACTOR +) -> Image.Image: + image_obj = None + if isinstance(image, Image.Image): + image_obj = image + elif image.startswith("http://") or image.startswith("https://"): + import requests + + image_obj = Image.open(requests.get(image, stream=True).raw) + elif image.startswith("file://"): + image_obj = Image.open(image[7:]) + elif image.startswith("data:image"): + import base64 + from io import BytesIO + + if "base64," in image: + _, base64_data = image.split("base64,", 1) + data = base64.b64decode(base64_data) + image_obj = Image.open(BytesIO(data)) + else: + image_obj = Image.open(image) + if image_obj is None: + raise ValueError( + f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}" + ) + image = image_obj.convert("RGB") + ## resize + # if "resized_height" in ele and "resized_width" in ele: + # resized_height, resized_width = smart_resize( + # ele["resized_height"], + # ele["resized_width"], + # factor=size_factor, + # ) + # else: + width, height = image.size + # min_pixels = ele.get("min_pixels", MIN_PIXELS) + # max_pixels = ele.get("max_pixels", MAX_PIXELS) + resized_height, resized_width = smart_resize( + height, + width, + factor=size_factor, + min_pixels=MIN_PIXELS, + max_pixels=MAX_PIXELS, + ) + image = image.resize((resized_width, resized_height)) + + return image + + +### + + +gme_qwen2vl_2b = ModelMeta( + loader=partial( + GmeQwen2VL, + model_name=HF_GME_QWEN2VL_2B, + ), + name=HF_GME_QWEN2VL_2B, + languages=["eng_Latn", "cmn-Hans"], + open_weights=True, + revision="ce765ae71b8cdb208203cd8fb64a170b1b84293a", + release_date="2024-12-24", + n_parameters=2_210_000_000, + embed_dim=1536, + license="apache-2.0", + max_tokens=32768, + reference="https://huggingface.co/" + HF_GME_QWEN2VL_2B, + similarity_fn_name="cosine", + framework=["PyTorch"], + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=None, +) + +gme_qwen2vl_7b = ModelMeta( + loader=partial( + GmeQwen2VL, + model_name=HF_GME_QWEN2VL_7B, + ), + name=HF_GME_QWEN2VL_7B, + languages=["eng_Latn", "cmn-Hans"], + open_weights=True, + revision="477027a6480f8630363be77751f169cc3434b673", + release_date="2024-12-24", + n_parameters=8_290_000_000, + embed_dim=3584, + license="apache-2.0", + max_tokens=32768, + reference="https://huggingface.co/" + HF_GME_QWEN2VL_2B, + similarity_fn_name="cosine", + framework=["PyTorch"], + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=None, +) diff --git a/mteb/models/google_models.py b/mteb/models/google_models.py index 3ec6f384c0..40d316fee7 100644 --- a/mteb/models/google_models.py +++ b/mteb/models/google_models.py @@ -4,16 +4,44 @@ from typing import Any import numpy as np +import tqdm from mteb.encoder_interface import Encoder, PromptType from mteb.model_meta import ModelMeta -from mteb.models.sentence_transformer_wrapper import ( - get_prompt_name, - validate_task_to_prompt_name, -) from .wrapper import Wrapper +MULTILINGUAL_EVALUATED_LANGUAGES = [ + "arb_Arab", + "ben_Beng", + "eng_Latn", + "spa_Latn", + "deu_Latn", + "pes_Arab", + "fin_Latn", + "fra_Latn", + "hin_Deva", + "ind_Latn", + "jpn_Jpan", + "kor_Hang", + "rus_Cyrl", + "swh_Latn", + "tel_Telu", + "tha_Thai", + "yor_Latn", + "zho_Hant", + "zho_Hans", +] + +MODEL_PROMPTS = { + "Classification": "CLASSIFICATION", + "MultilabelClassification": "CLASSIFICATION", + "Clustering": "CLUSTERING", + "STS": "SIMILARITY", + PromptType.query.value: "RETRIEVAL_QUERY", + PromptType.passage.value: "RETRIEVAL_DOCUMENT", +} + class GoogleTextEmbeddingModel(Encoder, Wrapper): def __init__( @@ -25,13 +53,14 @@ def __init__( ) -> None: self.model_name = model_name self.model_prompts = ( - validate_task_to_prompt_name(model_prompts) if model_prompts else None + self.validate_task_to_prompt_name(model_prompts) if model_prompts else None ) def _embed( self, texts: list[str], google_task_type: str | None = None, + show_progress_bar: bool = False, titles: list[str] | None = None, dimensionality: int | None = 768, ) -> list[list[float]]: @@ -58,14 +87,28 @@ def _embed( inputs = [ TextEmbeddingInput(text, task_type=google_task_type) for text in texts ] + kwargs = {"output_dimensionality": dimensionality} if dimensionality else {} - try: - embeddings = model.get_embeddings(inputs, **kwargs) - # Except the very rare google.api_core.exceptions.InternalServerError - except Exception as e: - print("Retrying once after error:", e) - embeddings = model.get_embeddings(inputs, **kwargs) - return np.asarray([embedding.values for embedding in embeddings]) + + max_batch_size = 16 ## Vertex API limits the number of instances per call to 250, but there is also a limit of tokens involved. Let's be conservative and set it to 16 by default. TODO: in a future PR, leverage the CountTokens API to get the optimum batch size for each request. + batches = [ + inputs[i : i + max_batch_size] + for i in range(0, len(inputs), max_batch_size) + ] + + all_embeddings = [] + + for batch in tqdm.tqdm(batches, leave=False, disable=not show_progress_bar): + try: + embeddings_batch = model.get_embeddings(batch, **kwargs) + # Except the very rare google.api_core.exceptions.InternalServerError + except Exception as e: + print("Retrying once after error:", e) + embeddings_batch = model.get_embeddings(batch, **kwargs) + + all_embeddings.extend([embedding.values for embedding in embeddings_batch]) + + return np.asarray(all_embeddings) def encode( self, @@ -74,35 +117,87 @@ def encode( prompt_type: PromptType | None = None, **kwargs: Any, ) -> np.ndarray: - google_task_type = get_prompt_name(self.model_prompts, task_name, prompt_type) - return self._embed(sentences, google_task_type=google_task_type) + prompt_name = self.get_prompt_name(self.model_prompts, task_name, prompt_type) + google_task_type = self.model_prompts.get(prompt_name) + show_progress_bar = ( + False + if "show_progress_bar" not in kwargs + else kwargs.pop("show_progress_bar") + ) -name = "text-embedding-004" -google_emb_004 = ModelMeta( + return self._embed( + sentences, + google_task_type=google_task_type, + show_progress_bar=show_progress_bar, + ) + + +google_text_emb_004 = ModelMeta( loader=partial( GoogleTextEmbeddingModel, - model_name=name, - model_prompts={ - "Classification": "CLASSIFICATION", - "MultilabelClassification": "CLASSIFICATION", - "Clustering": "CLUSTERING", - "STS": "SIMILARITY", - PromptType.query.value: "RETRIEVAL_QUERY", - PromptType.passage.value: "RETRIEVAL_DOCUMENT", - }, + model_name="text-embedding-004", + model_prompts=MODEL_PROMPTS, ), - name=name, + name="google/text-embedding-004", languages=["eng-Latn"], open_weights=False, revision="1", # revision is intended for implementation - release_date=None, # couldnt figure this out + release_date="2024-05-14", + n_parameters=None, + max_tokens=2048, + embed_dim=768, + license=None, + similarity_fn_name="cosine", # assumed + framework=["API"], + use_instructions=True, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, +) + +google_text_emb_005 = ModelMeta( + loader=partial( + GoogleTextEmbeddingModel, + model_name="text-embedding-005", + model_prompts=MODEL_PROMPTS, + ), + name="google/text-embedding-005", + languages=["eng-Latn"], + open_weights=False, + revision="1", # revision is intended for implementation + release_date="2024-11-18", + n_parameters=None, + max_tokens=2048, + embed_dim=768, + license=None, + similarity_fn_name="cosine", # assumed + framework=["API"], + use_instructions=True, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, +) + +google_text_multilingual_emb_002 = ModelMeta( + loader=partial( + GoogleTextEmbeddingModel, + model_name="text-multilingual-embedding-002", + model_prompts=MODEL_PROMPTS, + ), + name="google/text-multilingual-embedding-002", + languages=MULTILINGUAL_EVALUATED_LANGUAGES, # From the list of evaluated languages in https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#supported_text_languages + open_weights=False, + revision="1", # revision is intended for implementation + release_date="2024-05-14", n_parameters=None, - memory_usage=None, max_tokens=2048, embed_dim=768, license=None, similarity_fn_name="cosine", # assumed framework=["API"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, ) diff --git a/mteb/models/gritlm_models.py b/mteb/models/gritlm_models.py index 3f5dc173a1..35d0543811 100644 --- a/mteb/models/gritlm_models.py +++ b/mteb/models/gritlm_models.py @@ -1,59 +1,39 @@ from __future__ import annotations import logging -from collections.abc import Sequence from functools import partial -from typing import Any - -import numpy as np from mteb.model_meta import ModelMeta -from ..encoder_interface import PromptType -from .instructions import task_to_instruction -from .wrapper import Wrapper +from .e5_models import E5_TRAINING_DATA +from .instruct_wrapper import instruct_wrapper logger = logging.getLogger(__name__) -def gritlm_instruction(instruction: str = "") -> str: +GRIT_LM_TRAINING_DATA = { + **E5_TRAINING_DATA, # source https://arxiv.org/pdf/2402.09906 + # also uses medi2 which contains fever and hotpotqa: + "FEVER": ["train"], + "FEVERHardNegatives": ["train"], + "FEVER-PL": ["train"], # translation not trained on + "HotpotQA": ["train"], + "HotpotQAHardNegatives": ["train"], + "HotpotQA-PL": ["train"], # translation not trained on +} + + +def gritlm_instruction(instruction: str = "", prompt_type=None) -> str: return ( "<|user|>\n" + instruction + "\n<|embed|>\n" if instruction else "<|embed|>\n" ) -def gritlm_loader(**kwargs): - try: - from gritlm import GritLM - except ImportError: - raise ImportError("Please install `pip install gritlm` to use GritLM models.") - - class GritLMWrapper(GritLM, Wrapper): - def encode( - self, - sentences: Sequence[str], - *args, - task_name: str, - prompt_type: PromptType | None = None, - **kwargs: Any, - ) -> np.ndarray: - if "instruction" in kwargs: - instruction = kwargs.pop("instruction", "") - else: - instruction = task_to_instruction( - task_name, prompt_type == PromptType.query - ) - if instruction: - kwargs["instruction"] = gritlm_instruction(instruction) - return super().encode(sentences, *args, **kwargs) - - return GritLMWrapper(**kwargs) - - gritlm7b = ModelMeta( - loader=partial( - gritlm_loader, + loader=partial( # type: ignore + instruct_wrapper, model_name_or_path="GritLM/GritLM-7B", + instruction_template=gritlm_instruction, mode="embedding", torch_dtype="auto", ), @@ -63,19 +43,24 @@ def encode( revision="13f00a0e36500c80ce12870ea513846a066004af", release_date="2024-02-15", n_parameters=7_240_000_000, - memory_usage=None, embed_dim=4096, license="apache-2.0", max_tokens=4096, reference="https://huggingface.co/GritLM/GritLM-7B", similarity_fn_name="cosine", framework=["GritLM", "PyTorch"], - use_instuctions=True, + use_instructions=True, + training_datasets=GRIT_LM_TRAINING_DATA, + # section 3.1 "We finetune our final models from Mistral 7B [68] and Mixtral 8x7B [69] using adaptations of E5 [160] and the Tülu 2 data + public_training_code="https://github.com/ContextualAI/gritlm", + public_training_data=None, ) + gritlm8x7b = ModelMeta( - loader=partial( - gritlm_loader, + loader=partial( # type: ignore + instruct_wrapper, model_name_or_path="GritLM/GritLM-8x7B", + instruction_template=gritlm_instruction, mode="embedding", torch_dtype="auto", ), @@ -85,12 +70,15 @@ def encode( revision="7f089b13e3345510281733ca1e6ff871b5b4bc76", release_date="2024-02-15", n_parameters=57_920_000_000, - memory_usage=None, embed_dim=4096, license="apache-2.0", max_tokens=4096, reference="https://huggingface.co/GritLM/GritLM-8x7B", similarity_fn_name="cosine", framework=["GritLM", "PyTorch"], - use_instuctions=True, + use_instructions=True, + training_datasets=GRIT_LM_TRAINING_DATA, + # section 3.1 "We finetune our final models from Mistral 7B [68] and Mixtral 8x7B [69] using adaptations of E5 [160] and the Tülu 2 data + public_training_code="https://github.com/ContextualAI/gritlm", + public_training_data=None, ) diff --git a/mteb/models/gte_models.py b/mteb/models/gte_models.py index b6cc9bfb2e..4de4b610f2 100644 --- a/mteb/models/gte_models.py +++ b/mteb/models/gte_models.py @@ -1,63 +1,37 @@ from __future__ import annotations -from collections.abc import Sequence from functools import partial -from typing import Any -import numpy as np +import torch from mteb.encoder_interface import PromptType -from mteb.model_meta import ModelMeta - -from .instructions import task_to_instruction -from .wrapper import Wrapper - - -def gte_instruction(instruction: str) -> str: - return f"Instruct: {instruction}\nQuery: " - - -def gte_loader(**kwargs): - try: - from gritlm import GritLM - except ImportError: - raise ImportError( - "Please install `pip install gritlm` to use gte-Qwen2-7B-instruct." - ) - - class GTEWrapper(GritLM, Wrapper): - def encode( - self, - sentences: Sequence[str], - *args, - task_name: str, - prompt_type: PromptType | None = None, - **kwargs: Any, - ) -> np.ndarray: - if "instruction" in kwargs: - instruction = kwargs.pop("instruction", "") - else: - instruction = task_to_instruction( - task_name, prompt_type == PromptType.query - ) - if instruction: - kwargs["instruction"] = gte_instruction(instruction) - return super().encode(sentences, *args, **kwargs) - - return GTEWrapper(**kwargs) +from mteb.model_meta import ModelMeta, sentence_transformers_loader +from mteb.models.instruct_wrapper import instruct_wrapper + + +def instruction_template( + instruction: str, prompt_type: PromptType | None = None +) -> str: + return ( + f"Instruct: {instruction}\nQuery: " + if (prompt_type is None or prompt_type == PromptType.query) and instruction + else "" + ) gte_Qwen2_7B_instruct = ModelMeta( - loader=partial( - gte_loader, + loader=partial( # type: ignore + instruct_wrapper, model_name_or_path="Alibaba-NLP/gte-Qwen2-7B-instruct", - attn="cccc", + instruction_template=instruction_template, + attn="bbcc", pooling_method="lasttoken", mode="embedding", - torch_dtype="auto", + torch_dtype=torch.float16, # The ST script does not normalize while the HF one does so unclear what to do # https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct#sentence-transformers normalized=True, + embed_eos="<|endoftext|>", ), name="Alibaba-NLP/gte-Qwen2-7B-instruct", languages=None, @@ -65,11 +39,265 @@ def encode( revision="e26182b2122f4435e8b3ebecbf363990f409b45b", release_date="2024-06-15", # initial commit of hf model. n_parameters=7_613_000_000, - memory_usage=None, embed_dim=3584, license="apache-2.0", reference="https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=None, + max_tokens=131072, +) + + +gte_Qwen1_5_7B_instruct = ModelMeta( + loader=partial( # type: ignore + instruct_wrapper, + model_name_or_path="Alibaba-NLP/gte-Qwen1.5-7B-instruct", + instruction_template=instruction_template, + attn="bbcc", + pooling_method="lasttoken", + mode="embedding", + torch_dtype=torch.float16, + normalized=True, + embed_eos="<|endoftext|>", + ), + name="Alibaba-NLP/gte-Qwen1.5-7B-instruct", + languages=["eng_Latn"], + open_weights=True, + revision="07d27e5226328010336563bc1b564a5e3436a298", + release_date="2024-04-20", # initial commit of hf model. + n_parameters=7_720_000_000, + embed_dim=4096, + license="apache-2.0", + max_tokens=32768, + reference="https://huggingface.co/Alibaba-NLP/gte-Qwen1.5-7B-instruct", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=None, +) + + +gte_Qwen2_1_5B_instruct = ModelMeta( + loader=partial( # type: ignore + instruct_wrapper, + model_name_or_path="Alibaba-NLP/gte-Qwen2-1.5B-instruct", + instruction_template=instruction_template, + attn="bbcc", + pooling_method="lasttoken", + mode="embedding", + torch_dtype=torch.float16, + normalized=True, + embed_eos="<|endoftext|>", + ), + name="Alibaba-NLP/gte-Qwen2-1.5B-instruct", + languages=["eng_Latn"], + open_weights=True, + revision="c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd", + release_date="2024-07-29", # initial commit of hf model. + n_parameters=1_780_000_000, + embed_dim=8960, + license="apache-2.0", + max_tokens=131072, + reference="https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=None, +) + +gte_small_zh = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="thenlper/gte-small-zh", + revision="af7bd46fbb00b3a6963c8dd7f1786ddfbfbe973a", + ), + name="thenlper/gte-small-zh", + languages=["zho_Hans"], + open_weights=True, + revision="af7bd46fbb00b3a6963c8dd7f1786ddfbfbe973a", + release_date="2023-11-08", # initial commit of hf model. + n_parameters=30.3 * 1e6, + embed_dim=1024, + license="mit", + max_tokens=512, + reference="https://huggingface.co/thenlper/gte-small-zh", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + public_training_code=None, + public_training_data=None, + training_datasets=None, # Not disclosed +) + +gte_base_zh = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="thenlper/gte-base-zh", + revision="71ab7947d6fac5b64aa299e6e40e6c2b2e85976c", + ), + name="thenlper/gte-base-zh", + languages=["zho_Hans"], + open_weights=True, + revision="71ab7947d6fac5b64aa299e6e40e6c2b2e85976c", + release_date="2023-11-08", # initial commit of hf model. + n_parameters=102 * 1e6, + embed_dim=1024, + license="mit", + max_tokens=512, + reference="https://huggingface.co/thenlper/gte-base-zh", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + public_training_code=None, + public_training_data=None, + training_datasets=None, # Not disclosed +) + +gte_large_zh = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="thenlper/gte-large-zh", + revision="64c364e579de308104a9b2c170ca009502f4f545", + ), + name="thenlper/gte-large-zh", + languages=["zho_Hans"], + open_weights=True, + revision="64c364e579de308104a9b2c170ca009502f4f545", + release_date="2023-11-08", # initial commit of hf model. + n_parameters=326 * 1e6, + embed_dim=1024, + license="mit", + max_tokens=512, + reference="https://huggingface.co/thenlper/gte-large-zh", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + public_training_code=None, + public_training_data=None, + training_datasets=None, # Not disclosed +) + +gte_multilingual_langs = [ + "afr_Latn", + "ara_Arab", + "aze_Latn", + "bel_Cyrl", + "bul_Cyrl", + "ben_Beng", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "ell_Grek", + "eng_Latn", + "spa_Latn", + "est_Latn", + "eus_Latn", + "fas_Arab", + "fin_Latn", + "fra_Latn", + "glg_Latn", + "guj_Gujr", + "heb_Hebr", + "hin_Deva", + "hrv_Latn", + "hat_Latn", + "hun_Latn", + "hye_Armn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jpn_Jpan", + "jav_Latn", + "kat_Geor", + "kaz_Cyrl", + "khm_Khmr", + "kan_Knda", + "kor_Hang", + "kir_Cyrl", + "lao_Laoo", + "lit_Latn", + "lav_Latn", + "mkd_Cyrl", + "mal_Mlym", + "mon_Cyrl", + "mar_Deva", + "msa_Latn", + "mya_Mymr", + "nep_Deva", + "nld_Latn", + "nor_Latn", + "pan_Guru", + "pol_Latn", + "por_Latn", + "que_Latn", + "ron_Latn", + "rus_Cyrl", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "swa_Latn", + "tam_Taml", + "tel_Telu", + "tha_Thai", + "tgl_Latn", + "tur_Latn", + "ukr_Cyrl", + "urd_Arab", + "vie_Latn", + "yor_Latn", + "zho_Hans", +] +# Source: https://arxiv.org/pdf/2407.19669 +gte_multi_training_data = { + "T2Retrieval": ["train"], + "DuReader": ["train"], + "MMarcoReranking": ["train"], + "CMedQAv2-reranking": ["train"], + "NQ": ["train"], + "MSMARCO": ["train"], + "HotpotQA": ["train"], + "FEVER": ["train"], + "MIRACLReranking": ["train"], + "MrTidyRetrieval": ["train"], + "MultiLongDocRetrieval": ["train"], + # not in MTEB: + # - TriviaQA + # - SQuAD + # - AllNLI + # - Multi-CPR +} + +gte_multilingual_base = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="Alibaba-NLP/gte-multilingual-base", + revision="ca1791e0bcc104f6db161f27de1340241b13c5a4", + ), + name="Alibaba-NLP/gte-multilingual-base", + languages=gte_multilingual_langs, + open_weights=True, + revision="ca1791e0bcc104f6db161f27de1340241b13c5a4", + release_date="2024-07-20", # initial commit of hf model. + n_parameters=305 * 1e6, + embed_dim=1024, + license="apache-2", + max_tokens=8192, + reference="https://huggingface.co/Alibaba-NLP/gte-multilingual-base", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + public_training_code=None, + public_training_data=None, # couldn't find + training_datasets=gte_multi_training_data, ) diff --git a/mteb/models/ibm_granite_models.py b/mteb/models/ibm_granite_models.py new file mode 100644 index 0000000000..e7c3b8b022 --- /dev/null +++ b/mteb/models/ibm_granite_models.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +from functools import partial + +from mteb.model_meta import ModelMeta, sentence_transformers_loader + +GRANITE_LANGUAGES = [ + "ara_Latn", + "ces_Latn", + "deu_Latn", + "eng_Latn", + "spa_Latn", + "fra_Latn", + "ita_Latn", + "jpn_Latn", + "kor_Latn", + "nld_Latn", + "por_Latn", + "zho_Hant", + "zho_Hans", +] + +granite_training_data = { + # Multilingual MC4 + # Multilingual Webhose + # English Wikipedia + # Multilingual Wikimedia + "WikipediaRetrievalMultilingual": [], + "WikipediaRerankingMultilingual": [], + # Miracl Corpus (Title-Body) + # Stack Exchange Duplicate questions (titles) + # Stack Exchange Duplicate questions (titles) + # Stack Exchange Duplicate questions (bodies) + "StackOverflowDupQuestions": [], + "AskUbuntuDupQuestions": [], + # Stack Exchange (Title, Answer) pairs + # Stack Exchange (Title, Body) pairs + # Stack Exchange (Title, Body) pairs + # Machine Translations of Stack Exchange Duplicate questions (titles) + # Machine Translations of Stack Exchange (Title+Body, Answer) pairs + "StackExchangeClusteringP2P": [], + "StackExchangeClusteringP2P.v2": [], + "StackExchangeClustering": [], + "StackExchangeClustering.v2": [], + # SearchQA + # S2ORC (Title, Abstract) + # WikiAnswers Duplicate question pairs + # CCNews + # XSum + # SimpleWiki + # Machine Translated Cross Lingual Parallel Corpora + # SPECTER citation triplets + # Machine Translations of SPECTER citation triplets + # Natural Questions (NQ) + "NQ": ["test"], + "NQHardNegatives": ["test"], + # SQuAD2.0 + # HotpotQA + "HotPotQA": ["test"], + "HotPotQAHardNegatives": ["test"], + "HotPotQA-PL": ["test"], # translated from hotpotQA (not trained on) + # Fever + "FEVER": ["test"], + "FEVERHardNegatives": ["test"], + # PubMed + # Multilingual Miracl Triples + "MIRACLRetrieval": ["train"], + "MIRACLRetrievalHardNegatives": ["train"], + "MIRACLReranking": ["train"], + # Multilingual MrTydi Triples + "MrTidyRetrieval": ["train"], + # Sadeeem Question Asnwering + # DBPedia Title-Body Pairs + "DBPedia": ["train"], + # Synthetic: English Query-Wikipedia Passage + # Synthetic: English Fact Verification + # Synthetic: Multilingual Query-Wikipedia Passage + # Synthetic: Multilingual News Summaries + # IBM Internal Triples + # IBM Internal Title-Body Pairs +} + +granite_107m_multilingual = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="ibm-granite/granite-embedding-107m-multilingual", + revision="47db56afe692f731540413c67dd818ff492277e7", + ), + name="ibm-granite/granite-embedding-107m-multilingual", + languages=GRANITE_LANGUAGES, + open_weights=True, + revision="47db56afe692f731540413c67dd818ff492277e7", + release_date="2024-12-18", + n_parameters=107_000_000, + embed_dim=384, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/ibm-granite/granite-embedding-107m-multilingual", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + adapted_from=None, + superseded_by=None, + public_training_code=None, + public_training_data=None, + use_instructions=False, + training_datasets=granite_training_data, +) + +granite_278m_multilingual = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="ibm-granite/granite-embedding-278m-multilingual", + revision="84e3546b88b0cb69f8078608a1df558020bcbf1f", + ), + name="ibm-granite/granite-embedding-278m-multilingual", + languages=GRANITE_LANGUAGES, + open_weights=True, + revision="84e3546b88b0cb69f8078608a1df558020bcbf1f", + release_date="2024-12-18", + n_parameters=278_000_000, + embed_dim=768, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/ibm-granite/granite-embedding-278m-multilingual", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + adapted_from=None, + superseded_by=None, + public_training_code=None, + public_training_data=None, + use_instructions=False, + training_datasets=granite_training_data, +) + +granite_30m_english = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="ibm-granite/granite-embedding-30m-english", + revision="eddbb57470f896b5f8e2bfcb823d8f0e2d2024a5", + ), + name="ibm-granite/granite-embedding-30m-english", + languages=["eng_Latn"], + open_weights=True, + revision="eddbb57470f896b5f8e2bfcb823d8f0e2d2024a5", + release_date="2024-12-18", + n_parameters=30_000_000, + embed_dim=384, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/ibm-granite/granite-embedding-30m-english", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + adapted_from=None, + superseded_by=None, + public_training_code=None, + public_training_data=None, + use_instructions=False, + training_datasets=granite_training_data, +) + +granite_125m_english = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="ibm-granite/granite-embedding-125m-english", + revision="e48d3a5b47eaa18e3fe07d4676e187fd80f32730", + ), + name="ibm-granite/granite-embedding-125m-english", + languages=["eng_Latn"], + open_weights=True, + revision="e48d3a5b47eaa18e3fe07d4676e187fd80f32730", + release_date="2024-12-18", + n_parameters=125_000_000, + embed_dim=768, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/ibm-granite/granite-embedding-125m-english", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + adapted_from=None, + superseded_by=None, + public_training_code=None, + public_training_data=None, + use_instructions=False, + training_datasets=granite_training_data, +) diff --git a/mteb/models/inf_models.py b/mteb/models/inf_models.py new file mode 100644 index 0000000000..0d40ff3ef2 --- /dev/null +++ b/mteb/models/inf_models.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from functools import partial + +from mteb.model_meta import ModelMeta, sentence_transformers_loader + +inf_retriever_v1 = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="infly/inf-retriever-v1", + revision="d2d074546028c0012b5cc6af78c4fac24896e67f", + trust_remote_code=True, + ), + name="infly/inf-retriever-v1", + languages=["eng_Latn", "zho_Hans"], + open_weights=True, + revision="d2d074546028c0012b5cc6af78c4fac24896e67f", + release_date="2024-12-24", # initial commit of hf model. + n_parameters=7_069_121_024, + embed_dim=3584, + license="apache-2.0", + max_tokens=131_072, + reference="https://huggingface.co/infly/inf-retriever-v1", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + adapted_from="Alibaba-NLP/gte-Qwen2-7B-instruct", + public_training_code=None, + public_training_data=None, + training_datasets=None, +) diff --git a/mteb/models/instruct_wrapper.py b/mteb/models/instruct_wrapper.py new file mode 100644 index 0000000000..2ee3a09b56 --- /dev/null +++ b/mteb/models/instruct_wrapper.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import logging +from collections.abc import Sequence +from typing import Any, Callable + +import numpy as np +import torch + +from mteb.encoder_interface import PromptType + +from .wrapper import Wrapper + +logger = logging.getLogger(__name__) + + +def instruct_wrapper( + model_name_or_path: str, + mode: str, + instruction_template: str | Callable[[str], str] | None = None, + **kwargs, +): + try: + from gritlm import GritLM + except ImportError: + raise ImportError( + f"Please install `pip install mteb[gritlm]` to use {model_name_or_path}." + ) + + class InstructWrapper(GritLM, Wrapper): + def __init__( + self, + model_name_or_path: str, + mode: str, + instruction_template: str | Callable[[str], str] | None = None, + **kwargs, + ): + if ( + isinstance(instruction_template, str) + and "{instruction}" not in instruction_template + ): + raise ValueError( + "Instruction template must contain the string '{instruction}'." + ) + if instruction_template is None: + logger.warning( + "No instruction template provided. Instructions will be used as-is." + ) + + if "gte-Qwen" in model_name_or_path: + logger.warning( + "Instructions are used in both query and docs, which may cause performance discrepancies from the original implementation." + ) + + self.instruction_template = instruction_template + super().__init__(model_name_or_path=model_name_or_path, mode=mode, **kwargs) + + def encode( + self, + sentences: Sequence[str], + *args, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + instruction = self.get_instruction(task_name, prompt_type) + + if self.instruction_template: + instruction = self.format_instruction(instruction, prompt_type) + + logger.info(f"Using instruction: '{instruction}' for task: '{task_name}'") + embeddings = super().encode( + sentences, instruction=instruction, *args, **kwargs + ) + if isinstance(embeddings, torch.Tensor): + # sometimes in kwargs can be return_tensors=True + embeddings = embeddings.cpu().detach().float().numpy() + return embeddings + + return InstructWrapper(model_name_or_path, mode, instruction_template, **kwargs) diff --git a/mteb/models/jasper_models.py b/mteb/models/jasper_models.py new file mode 100644 index 0000000000..dbd1615ad8 --- /dev/null +++ b/mteb/models/jasper_models.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import logging +from collections.abc import Sequence +from functools import partial +from typing import Any, Callable + +import numpy as np +import torch +from sentence_transformers import SentenceTransformer + +import mteb +from mteb.encoder_interface import PromptType +from mteb.model_meta import ModelMeta + +from .nvidia_models import nvidia_training_datasets +from .wrapper import Wrapper + +logger = logging.getLogger(__name__) + + +class JasperWrapper(Wrapper): + def __init__( + self, + model_name: str, + revision: str, + instruction_template: str | Callable[[str], str] | None = None, + max_seq_length: int = 2048, + **kwargs: Any, + ): + self.model_name = model_name + self.model = SentenceTransformer(model_name, revision=revision, **kwargs) + self.instruction_template = instruction_template + self.model.max_seq_length = max_seq_length + + def encode( + self, + sentences: Sequence[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + task = mteb.get_task(task_name=task_name) + instruction = self.get_task_instruction(task_name, prompt_type) + + # to passage prompts won't be applied to passages + if prompt_type == PromptType.passage and task.metadata.type == "s2p": + instruction = None + + embeddings = self.model.encode( + sentences, + normalize_embeddings=True, + prompt=instruction, + **kwargs, + ) + + if isinstance(embeddings, torch.Tensor): + # sometimes in kwargs can be return_tensors=True + embeddings = embeddings.cpu().detach().float().numpy() + return embeddings + + +jasper_en_v1 = ModelMeta( + loader=partial( # type: ignore + JasperWrapper, + model_name="infgrad/jasper_en_vision_language_v1", + revision="d6330ce98f8a0d741e781df845904c9484f00efa", + config_kwargs={"is_text_encoder": True, "vector_dim": 12288}, + model_kwargs={ + "attn_implementation": "sdpa", + "torch_dtype": torch.float16, + }, + trust_remote_code=True, + max_seq_length=2048, + instruction_template="Instruct: {instruction}\nQuery: ", + ), + name="infgrad/jasper_en_vision_language_v1", + languages=["eng-Latn"], + open_weights=True, + revision="d6330ce98f8a0d741e781df845904c9484f00efa", + release_date="2024-12-11", # first commit + n_parameters=1_999_000_000, + max_tokens=131072, + embed_dim=8960, + license="apache-2.0", + reference="https://huggingface.co/infgrad/jasper_en_vision_language_v1/tree/main", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + adapted_from=None, + superseded_by=None, + training_datasets=nvidia_training_datasets, # "In jasper model the teacher model is nvidia/NV-Embed-v2", source https://huggingface.co/infgrad/jasper_en_vision_language_v1 + # "non_mteb": ["BAAI/Infinity-MM", "HuggingFaceFW/fineweb-edu"], + public_training_code=None, + public_training_data=None, +) diff --git a/mteb/models/jina_clip.py b/mteb/models/jina_clip.py index d24dcf1827..bff02a76c3 100644 --- a/mteb/models/jina_clip.py +++ b/mteb/models/jina_clip.py @@ -155,10 +155,21 @@ def encode( # type: ignore ), name="jinaai/jina-clip-v1", languages=["eng_Latn"], - open_source=True, revision="06150c7c382d7a4faedc7d5a0d8cdb59308968f4", release_date="2024-05-30", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) diff --git a/mteb/models/jina_models.py b/mteb/models/jina_models.py new file mode 100644 index 0000000000..e855ad3c7a --- /dev/null +++ b/mteb/models/jina_models.py @@ -0,0 +1,329 @@ +from __future__ import annotations + +import logging +from collections.abc import Sequence +from functools import partial +from typing import Any + +import numpy as np +import torch +from sentence_transformers import __version__ as st_version + +from mteb.model_meta import ModelMeta + +from ..encoder_interface import PromptType +from .sentence_transformer_wrapper import SentenceTransformerWrapper + +logger = logging.getLogger(__name__) + +MIN_SENTENCE_TRANSFORMERS_VERSION = (3, 1, 0) +CURRENT_SENTENCE_TRANSFORMERS_VERSION = tuple(map(int, st_version.split("."))) + +XLMR_LANGUAGES = [ + "afr_Latn", + "amh_Latn", + "ara_Latn", + "asm_Latn", + "aze_Latn", + "bel_Latn", + "bul_Latn", + "ben_Latn", + "ben_Beng", + "bre_Latn", + "bos_Latn", + "cat_Latn", + "ces_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "ell_Latn", + "eng_Latn", + "epo_Latn", + "spa_Latn", + "est_Latn", + "eus_Latn", + "fas_Latn", + "fin_Latn", + "fra_Latn", + "fry_Latn", + "gle_Latn", + "gla_Latn", + "glg_Latn", + "guj_Latn", + "hau_Latn", + "heb_Latn", + "hin_Latn", + "hin_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jpn_Latn", + "jav_Latn", + "kat_Latn", + "kaz_Latn", + "khm_Latn", + "kan_Latn", + "kor_Latn", + "kur_Latn", + "kir_Latn", + "lat_Latn", + "lao_Latn", + "lit_Latn", + "lav_Latn", + "mlg_Latn", + "mkd_Latn", + "mal_Latn", + "mon_Latn", + "mar_Latn", + "msa_Latn", + "mya_Latn", + "nep_Latn", + "nld_Latn", + "nob_Latn", + "orm_Latn", + "ori_Latn", + "pan_Latn", + "pol_Latn", + "pus_Latn", + "por_Latn", + "ron_Latn", + "rus_Latn", + "san_Latn", + "snd_Latn", + "sin_Latn", + "slk_Latn", + "slv_Latn", + "som_Latn", + "sqi_Latn", + "srp_Latn", + "sun_Latn", + "swe_Latn", + "swa_Latn", + "tam_Latn", + "tam_Taml", + "tel_Latn", + "tel_Telu", + "tha_Latn", + "tgl_Latn", + "tur_Latn", + "uig_Latn", + "ukr_Latn", + "urd_Latn", + "urd_Arab", + "uzb_Latn", + "vie_Latn", + "xho_Latn", + "yid_Latn", + "zho_Hant", + "zho_Hans", +] + + +class JinaWrapper(SentenceTransformerWrapper): + """following the hf model card documentation.""" + + jina_task_to_prompt = { + "retrieval.query": "Represent the query for retrieving evidence documents: ", + "retrieval.passage": "Represent the document for retrieval: ", + } + + def __init__( + self, + model: str, + revision: str | None = None, + model_prompts: dict[str, str] | None = None, + **kwargs, + ) -> None: + if CURRENT_SENTENCE_TRANSFORMERS_VERSION < MIN_SENTENCE_TRANSFORMERS_VERSION: + raise RuntimeError( + f"sentence_transformers version {st_version} is lower than the required version 3.1.0" + ) + try: + import einops # noqa: F401 + except ImportError: + raise ImportError( + "To use the jina-embeddings-v3 models `einops` is required. Please install it with `pip install mteb[jina]`." + ) + try: + import flash_attn # noqa: F401 + except ImportError: + logger.warning( + "Using flash_attn for jina-embeddings-v3 models is recommended. Please install it with `pip install mteb[flash_attention]`." + "Fallback to native implementation." + ) + super().__init__(model, revision, model_prompts, **kwargs) + + def encode( + self, + sentences: Sequence[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + prompt_name = self.get_prompt_name(self.model_prompts, task_name, prompt_type) + if prompt_name: + logger.info( + f"Using prompt_name={prompt_name} for task={task_name} prompt_type={prompt_type}" + ) + else: + logger.info( + f"No model prompts found for task={task_name} prompt_type={prompt_type}" + ) + logger.info(f"Encoding {len(sentences)} sentences.") + + jina_task_name = self.model_prompts.get(prompt_name, None) + + embeddings = self.model.encode( + sentences, + task=jina_task_name, + prompt=self.jina_task_to_prompt.get(jina_task_name, None), + **kwargs, + ) + + if isinstance(embeddings, torch.Tensor): + # sometimes in kwargs can be return_tensors=True + embeddings = embeddings.cpu().detach().float().numpy() + return embeddings + + +jina_embeddings_v3 = ModelMeta( + loader=partial( # type: ignore + JinaWrapper, + model="jinaai/jina-embeddings-v3", + revision="215a6e121fa0183376388ac6b1ae230326bfeaed", + trust_remote_code=True, + model_prompts={ + "Retrieval-query": "retrieval.query", + "Retrieval-passage": "retrieval.passage", + "Clustering": "separation", + "Classification": "classification", + "STS": "text-matching", + "PairClassification": "classification", + "BitextMining": "text-matching", + "MultilabelClassification": "classification", + "Reranking": "separation", + "Summarization": "text-matching", + }, + ), + name="jinaai/jina-embeddings-v3", + languages=XLMR_LANGUAGES, + open_weights=True, + revision="215a6e121fa0183376388ac6b1ae230326bfeaed", + release_date="2024-09-18", # official release date + n_parameters=int(572 * 1e6), + max_tokens=8194, + embed_dim=4096, + license="cc-by-nc-4.0", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + reference="https://huggingface.co/jinaai/jina-embeddings-v3", + public_training_code=None, + public_training_data=None, + training_datasets={ + # CulturaX + "STS12": [], + # "SICK": [], + # "WMT19": [], + # "MADLAD-3B": [], + # NLI + "MSMARCO": ["train"], + "MSMARCOHardNegatives": ["train"], + "NanoMSMARCORetrieval": ["train"], + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on + # oasst1, oasst2 + }, + adapted_from="XLM-RoBERTa", +) + + +jina_embeddings_v2_base_en = ModelMeta( + name="jinaai/jina-embeddings-v2-base-en", + languages=["eng-Latn"], + open_weights=True, + revision="6e85f575bc273f1fd840a658067d0157933c83f0", + release_date="2023-09-27", + n_parameters=137_000_000, + embed_dim=768, + license="apache-2.0", + max_tokens=8192, + reference="https://huggingface.co/jinaai/jina-embeddings-v2-base-en", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + training_datasets=None, + public_training_code=None, + public_training_data=None, +) + +jina_embeddings_v2_small_en = ModelMeta( + name="jinaai/jina-embeddings-v2-small-en", + languages=["eng-Latn"], + open_weights=True, + revision="796cff318cdd4e5fbe8b7303a1ef8cbec36996ef", + release_date="2023-09-27", + n_parameters=32_700_000, + embed_dim=512, + license="apache-2.0", + max_tokens=8192, + reference="https://huggingface.co/jinaai/jina-embeddings-v2-small-en", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + training_datasets=None, + public_training_code=None, + public_training_data=None, +) + +jina_embedding_b_en_v1 = ModelMeta( + name="jinaai/jina-embedding-b-en-v1", + languages=["eng-Latn"], + open_weights=True, + revision="aa0645035294a8c0607ce5bb700aba982cdff32c", + release_date="2023-07-07", + n_parameters=110_000_000, + embed_dim=768, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/jinaai/jina-embedding-b-en-v1", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by="jinaai/jina-embeddings-v2-base-en", + adapted_from=None, + training_datasets=None, + public_training_code=None, + public_training_data=None, +) + +jina_embedding_s_en_v1 = ModelMeta( + name="jinaai/jina-embedding-s-en-v1", + languages=["eng-Latn"], + open_weights=True, + revision="c1fed70aa4823a640f1a7150a276e4d3b08dce08", + release_date="2023-07-07", + n_parameters=35_000_000, + embed_dim=512, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/jinaai/jina-embedding-s-en-v1", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by="jinaai/jina-embeddings-v2-small-en", + adapted_from=None, + training_datasets=None, + public_training_code=None, + public_training_data=None, +) diff --git a/mteb/models/lens_models.py b/mteb/models/lens_models.py new file mode 100644 index 0000000000..2cf055abd4 --- /dev/null +++ b/mteb/models/lens_models.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from mteb.model_meta import ModelMeta + +lens_d4000 = ModelMeta( + loader=None, # TODO: implement this in the future + name="yibinlei/LENS-d4000", + languages=None, + open_weights=True, + revision="e473b33364e6c48a324796fd1411d3b93670c6fe", + release_date="2025-01-17", + n_parameters=int(7.11 * 1e9), + embed_dim=4000, + license="apache-2.0", + reference="https://huggingface.co/yibinlei/LENS-d4000", + similarity_fn_name="cosine", + framework=["PyTorch"], + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=None, + max_tokens=32768, +) + +lens_d8000 = ModelMeta( + loader=None, # TODO: implement this in the future + name="yibinlei/LENS-d8000", + languages=None, + open_weights=True, + revision="a0b87bd91cb27b6f2f0b0fe22c28026da1d464ef", + release_date="2025-01-17", + n_parameters=int(7.11 * 1e9), + embed_dim=8000, + license="apache-2.0", + reference="https://huggingface.co/yibinlei/LENS-d8000", + similarity_fn_name="cosine", + framework=["PyTorch"], + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=None, + max_tokens=32768, +) diff --git a/mteb/models/linq_models.py b/mteb/models/linq_models.py new file mode 100644 index 0000000000..ead10ebf71 --- /dev/null +++ b/mteb/models/linq_models.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from functools import partial + +import torch + +from mteb.encoder_interface import PromptType +from mteb.model_meta import ModelMeta +from mteb.models.instruct_wrapper import instruct_wrapper + + +def instruction_template( + instruction: str, prompt_type: PromptType | None = None +) -> str: + return f"Instruct: {instruction}\nQuery: " if instruction else "" + + +Linq_Embed_Mistral = ModelMeta( + loader=partial( # type: ignore + instruct_wrapper, + model_name_or_path="Linq-AI-Research/Linq-Embed-Mistral", + instruction_template=instruction_template, + attn="cccc", + pooling_method="lasttoken", + mode="embedding", + torch_dtype=torch.bfloat16, + normalized=True, + ), + name="Linq-AI-Research/Linq-Embed-Mistral", + languages=["eng_Latn"], + open_weights=True, + revision="0c1a0b0589177079acc552433cad51d7c9132379", + release_date="2024-05-29", # initial commit of hf model. + n_parameters=7_110_000_000, + embed_dim=4096, + license="cc-by-nc-4.0", + max_tokens=32768, + reference="https://huggingface.co/Linq-AI-Research/Linq-Embed-Mistral", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=None, +) diff --git a/mteb/models/llm2vec_models.py b/mteb/models/llm2vec_models.py index 79f8d7950c..28197e5c84 100644 --- a/mteb/models/llm2vec_models.py +++ b/mteb/models/llm2vec_models.py @@ -9,8 +9,6 @@ from mteb.encoder_interface import Encoder, PromptType from mteb.model_meta import ModelMeta -from .instructions import task_to_instruction -from .sentence_transformer_wrapper import validate_task_to_prompt_name from .wrapper import Wrapper logger = logging.getLogger(__name__) @@ -22,6 +20,31 @@ def llm2vec_instruction(instruction): return instruction +llm2vec_supervised_training_data = { + # source, section g1: https://arxiv.org/pdf/2404.05961 + # splits assumed but unkown + "HotpotQA": ["train"], + "HotpotQA-PL": ["train"], # translation not trained on + "HotpotQAHardNegatives": ["train"], + "MSMARCO": ["train"], + "MSMARCOHardNegatives": ["train"], + "NanoMSMARCORetrieval": ["train"], + "MSMARCO-PL": ["train"], # translation not trained on + "MIRACLRetrieval": ["train"], + "MIRACLRetrievalHardNegatives": ["train"], + "MIRACLReranking": ["train"], + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on + "FEVER": ["train"], + "FEVERHardNegatives": ["train"], + "NanoFEVERRetrieval": ["train"], + "MrTidyRetrieval": ["train"], + "T2Reranking": ["train"], +} + + class LLM2VecWrapper(Wrapper): def __init__( self, @@ -46,7 +69,7 @@ def __init__( "LLM2Vec models were trained with flash attention enabled. For optimal performance, please install the `flash_attn` package with `pip install flash-attn --no-build-isolation`." ) self.model_prompts = ( - validate_task_to_prompt_name(model_prompts) if model_prompts else None + self.validate_task_to_prompt_name(model_prompts) if model_prompts else None ) if device: @@ -65,9 +88,7 @@ def encode( prompt_type: PromptType | None = None, **kwargs: Any, # noqa ) -> np.ndarray: - instruction = llm2vec_instruction( - task_to_instruction(task_name, prompt_type == PromptType.query) - ) + instruction = llm2vec_instruction(self.get_instruction(task_name, prompt_type)) sentences = [[instruction, sentence] for sentence in sentences] return self.model.encode(sentences, **kwargs) @@ -96,14 +117,16 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="baa8ebf04a1c2500e61288e7dad65e8ae42601a7", # TODO: Not sure what to put here as a model is made of two peft repos, each with a different revision release_date="2024-04-09", n_parameters=7_505_000_000, - memory_usage=None, max_tokens=8192, embed_dim=4096, license="mit", reference="https://huggingface.co/McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised", similarity_fn_name="cosine", framework=["LLM2Vec", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs", + training_datasets=llm2vec_supervised_training_data, + public_training_data=None, ) llm2vec_llama3_8b_unsupervised = ModelMeta( @@ -120,14 +143,16 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="1cb7b735326d13a8541db8f57f35da5373f5e9c6", release_date="2024-04-09", n_parameters=7_505_000_000, - memory_usage=None, max_tokens=8192, embed_dim=4096, license="mit", reference="https://huggingface.co/McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-unsup-simcse", similarity_fn_name="cosine", framework=["LLM2Vec", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs", + training_datasets={}, + public_training_data=None, ) @@ -145,14 +170,16 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="0ae69bdd5816105778b971c3138e8f8a18eaa3ae", release_date="2024-04-09", n_parameters=7_111_000_000, - memory_usage=None, max_tokens=32768, embed_dim=4096, license="mit", reference="https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised", similarity_fn_name="cosine", framework=["LLM2Vec", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs", + training_datasets=llm2vec_supervised_training_data, + public_training_data=None, ) llm2vec_mistral7b_unsupervised = ModelMeta( @@ -169,14 +196,16 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="2c055a5d77126c0d3dc6cd8ffa30e2908f4f45f8", release_date="2024-04-09", n_parameters=7_111_000_000, - memory_usage=None, max_tokens=32768, embed_dim=4096, license="mit", reference="https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-unsup-simcse", similarity_fn_name="cosine", framework=["LLM2Vec", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs", + training_datasets={}, + public_training_data=None, ) llm2vec_llama2_7b_supervised = ModelMeta( @@ -193,14 +222,16 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="2c055a5d77126c0d3dc6cd8ffa30e2908f4f45f8", release_date="2024-04-09", n_parameters=7_111_000_000, - memory_usage=None, max_tokens=32768, embed_dim=4096, license="mit", reference="https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised", similarity_fn_name="cosine", framework=["LLM2Vec", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs", + training_datasets=llm2vec_supervised_training_data, + public_training_data=None, ) llm2vec_llama2_7b_unsupervised = ModelMeta( @@ -217,14 +248,16 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="a76944871d169ebe7c97eb921764cd063afed785", release_date="2024-04-09", n_parameters=7_111_000_000, - memory_usage=None, max_tokens=32768, embed_dim=4096, license="mit", reference="https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-unsup-simcse", similarity_fn_name="cosine", framework=["LLM2Vec", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs", + training_datasets={}, + public_training_data=None, ) llm2vec_sheared_llama_supervised = ModelMeta( @@ -241,14 +274,16 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="a5943d406c6b016fef3f07906aac183cf1a0b47d", release_date="2024-04-09", n_parameters=7_111_000_000, - memory_usage=None, max_tokens=32768, embed_dim=4096, license="mit", reference="https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-supervised", similarity_fn_name="cosine", framework=["LLM2Vec", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs", + training_datasets=llm2vec_supervised_training_data, + public_training_data=None, ) llm2vec_sheared_llama_unsupervised = ModelMeta( @@ -265,12 +300,14 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="a5943d406c6b016fef3f07906aac183cf1a0b47d", release_date="2024-04-09", n_parameters=7_111_000_000, - memory_usage=None, max_tokens=32768, embed_dim=4096, license="mit", reference="https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-unsup-simcse", similarity_fn_name="cosine", framework=["LLM2Vec", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs", + training_datasets={}, + public_training_data=None, ) diff --git a/mteb/models/misc_models.py b/mteb/models/misc_models.py new file mode 100644 index 0000000000..140d8bac74 --- /dev/null +++ b/mteb/models/misc_models.py @@ -0,0 +1,1777 @@ +from __future__ import annotations + +from functools import partial + +import torch + +from mteb.model_meta import ModelMeta, sentence_transformers_loader +from mteb.models.e5_models import E5_TRAINING_DATA + +from .bge_models import bge_m3_training_data, bge_training_data +from .sentence_transformers_models import sent_trf_training_dataset + +Haon_Chen__speed_embedding_7b_instruct = ModelMeta( + name="Haon-Chen/speed-embedding-7b-instruct", + revision="c167e9a8144b397622ce47b85d9edcdeecef3d3f", + release_date="2024-10-31", + languages=["eng_Latn"], + loader=None, + n_parameters=7110660096, + max_tokens=32768.0, + embed_dim=None, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/Haon-Chen/speed-embedding-7b-instruct", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="mistralai/Mistral-7B-v0.1", + superseded_by=None, +) +Gameselo__STS_multilingual_mpnet_base_v2 = ModelMeta( + name="Gameselo/STS-multilingual-mpnet-base-v2", + revision="449f917af30f590fc31f9ffb226c94f21a2f47b8", + release_date="2024-06-07", + languages=[], + loader=None, + n_parameters=278043648, + max_tokens=514.0, + embed_dim=768, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Gameselo/STS-multilingual-mpnet-base-v2", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + superseded_by=None, +) +HIT_TMG__KaLM_embedding_multilingual_mini_instruct_v1 = ModelMeta( + name="HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1", + revision="45e42c89990c40aca042659133fc8b13c28634b5", + release_date="2024-10-23", + languages=None, + loader=None, + n_parameters=494032768, + max_tokens=131072.0, + embed_dim=896, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="/mnt/shgeminicephfs/wx-dc-plt-hpc/xinshuohu/Output/Embedding/Qwen2-0.5B-eos_mean_pretrain_0806_1e-4_uen_sft_1022_filtered_v2_inst_3node_g8_1e-5_sin-0.1_mrl", + superseded_by=None, +) +HIT_TMG__KaLM_embedding_multilingual_mini_v1 = ModelMeta( + name="HIT-TMG/KaLM-embedding-multilingual-mini-v1", + revision="8a82a0cd2b322b91723e252486f7cce6fd8ac9d3", + release_date="2024-08-27", + languages=None, + loader=None, + n_parameters=494032768, + max_tokens=131072.0, + embed_dim=896, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/HIT-TMG/KaLM-embedding-multilingual-mini-v1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="/mnt/shgeminicephfs/wx-dc-plt-hpc/xinshuohu/Output/Embedding/Qwen2-0.5B-eos_mean_pretrain_0806_1e-4_uen_sft_0902_filtered_v2_3node_g8_1e-5_sin-0.1", + superseded_by=None, +) +Hum_Works__lodestone_base_4096_v1 = ModelMeta( + name="Hum-Works/lodestone-base-4096-v1", + revision="9bbc2d0b57dd2198aea029404b0f976712a7d966", + release_date="2023-08-25", + languages=["eng_Latn"], + loader=None, + n_parameters=None, + max_tokens=None, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/Hum-Works/lodestone-base-4096-v1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets={ + "MSMARCO": ["train"], + "MSMARCOHardNegatives": ["train"], + "NanoMSMARCORetrieval": ["train"], + "MSMARCO-PL": ["train"], # translation not trained on + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on + # not in MTEB + # "s2orc": ["train"], + # "flax-sentence-embeddings/stackexchange_title_body_jsonl": ["train"], + # "flax-sentence-embeddings/stackexchange_titlebody_best_voted_answer_jsonl": [ + # "train" + # ], + # "flax-sentence-embeddings/stackexchange_title_best_voted_answer_jsonl": [ + # "train" + # ], + # "flax-sentence-embeddings/stackexchange_titlebody_best_and_down_voted_answer_jsonl": [ + # "train" + # ], + # "sentence-transformers/reddit-title-body": ["train"], + # "msmarco": ["train"], + # "gooaq": ["train"], + # "yahoo_answers_topics": ["train"], + # "code_search_net": ["train"], + # "search_qa": ["train"], + # "eli5": ["train"], + # "snli": ["train"], + # "multi_nli": ["train"], + # "wikihow": ["train"], + # "natural_questions": ["train"], + # "trivia_qa": ["train"], + # "embedding-data/sentence-compression": ["train"], + # "embedding-data/flickr30k-captions": ["train"], + # "embedding-data/altlex": ["train"], + # "embedding-data/simple-wiki": ["train"], + # "embedding-data/QQP": ["train"], + # "embedding-data/SPECTER": ["train"], + # "embedding-data/PAQ_pairs": ["train"], + # "embedding-data/WikiAnswers": ["train"], + # "sentence-transformers/embedding-training-data": ["train"], + }, + adapted_from="hum-lodestone-v1", + superseded_by=None, +) +Jaume__gemma_2b_embeddings = ModelMeta( + name="Jaume/gemma-2b-embeddings", + revision="86431f65d7c3f66b2af096c61e614a2958f191f1", + release_date="2024-06-29", + languages=[], + loader=None, + n_parameters=2506172416, + max_tokens=8192.0, + embed_dim=2048, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Jaume/gemma-2b-embeddings", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets={}, + adapted_from="google/gemma-2b", + superseded_by=None, +) +BeastyZ__e5_R_mistral_7b = ModelMeta( + name="BeastyZ/e5-R-mistral-7b", + revision="3f810a6a7fd220369ad248e3705cf13d71803602", + release_date="2024-06-28", + languages=["eng_Latn"], + loader=None, + n_parameters=7241732096, + max_tokens=32768.0, + embed_dim=None, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/BeastyZ/e5-R-mistral-7b", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=E5_TRAINING_DATA, + # not MTEB: {"BeastyZ/E5-R": ["train"]}, + adapted_from="/ConRetriever/public_weight_mistral", + superseded_by=None, +) +Lajavaness__bilingual_embedding_base = ModelMeta( + name="Lajavaness/bilingual-embedding-base", + revision="0bfc54bb2aa2666dd84715289c7ef58a95eb4d8d", + release_date="2024-06-26", + languages=None, + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="Lajavaness/bilingual-embedding-base", + revision="0bfc54bb2aa2666dd84715289c7ef58a95eb4d8d", + trust_remote_code=True, + ), + n_parameters=278043648, + max_tokens=514.0, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Lajavaness/bilingual-embedding-base", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="dangvantuan/bilingual_impl", + superseded_by=None, +) +Lajavaness__bilingual_embedding_large = ModelMeta( + name="Lajavaness/bilingual-embedding-large", + revision="e83179d7a66e8aed1b3015e98bb5ae234ed89598", + release_date="2024-06-24", + languages=["fra_Latn", "eng_Latn"], + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="Lajavaness/bilingual-embedding-large", + revision="e83179d7a66e8aed1b3015e98bb5ae234ed89598", + trust_remote_code=True, + ), + n_parameters=559890432, + max_tokens=514.0, + embed_dim=1024, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Lajavaness/bilingual-embedding-large", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="dangvantuan/bilingual_impl", + superseded_by=None, +) +Lajavaness__bilingual_embedding_small = ModelMeta( + name="Lajavaness/bilingual-embedding-small", + revision="ed4a1dd814de0db81d4a4e287c296a03194463e3", + release_date="2024-07-17", + languages=["fra_Latn", "eng_Latn"], + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="Lajavaness/bilingual-embedding-small", + revision="ed4a1dd814de0db81d4a4e287c296a03194463e3", + trust_remote_code=True, + ), + n_parameters=117653760, + max_tokens=512.0, + embed_dim=384, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Lajavaness/bilingual-embedding-small", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="dangvantuan/bilingual_impl", + superseded_by=None, +) +Mihaiii__Bulbasaur = ModelMeta( + name="Mihaiii/Bulbasaur", + revision="6876f839e18ae36224049a41194a431953f08747", + release_date="2024-04-27", + languages=None, + loader=None, + n_parameters=17389824, + max_tokens=512.0, + embed_dim=384, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Mihaiii/Bulbasaur", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, # source model is GTE-tiny where training data is unknown + # {"Mihaiii/qa-assistant": ["train"]}, + adapted_from="Mihaiii/dwsdwass", + superseded_by=None, +) +Mihaiii__Ivysaur = ModelMeta( + name="Mihaiii/Ivysaur", + revision="65914d976f45beb4bda7485c39d88865b4ce6554", + release_date="2024-04-27", + languages=None, + loader=None, + n_parameters=22713216, + max_tokens=512.0, + embed_dim=384, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Mihaiii/Ivysaur", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, # source model is GTE-tiny where training data is unknown + # not MTEB: {"Mihaiii/qa-assistant": ["train"]}, + adapted_from="Mihaiii/jhjghjgh", + superseded_by=None, +) +Mihaiii__Squirtle = ModelMeta( + name="Mihaiii/Squirtle", + revision="5b991da48a9286637a256d4a35aab87a1a57b78a", + release_date="2024-04-30", + languages=None, + loader=None, + n_parameters=15615360, + max_tokens=512.0, + embed_dim=384, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Mihaiii/Squirtle", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=bge_training_data, # source model is bge-base-en-v1.5 + # not MTEB: {"Mihaiii/qa-assistant": ["train"]}, + adapted_from="Mihaiii/test21", + superseded_by=None, +) +Mihaiii__Venusaur = ModelMeta( + name="Mihaiii/Venusaur", + revision="0dc817f0addbb7bab8feeeeaded538f9ffeb3419", + release_date="2024-04-29", + languages=None, + loader=None, + n_parameters=15615360, + max_tokens=512.0, + embed_dim=384, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Mihaiii/Venusaur", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, # source model is unkown + # {"Mihaiii/qa-assistant": ["train"]}, + adapted_from="Mihaiii/test14", + superseded_by=None, +) +Mihaiii__Wartortle = ModelMeta( + name="Mihaiii/Wartortle", + revision="14caca5253414d38a7d28b62d1b7c30ef3293a87", + release_date="2024-04-30", + languages=None, + loader=None, + n_parameters=17389824, + max_tokens=512.0, + embed_dim=384, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Mihaiii/Wartortle", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=bge_training_data, # distill from bge-base-en-v1.5 + # {"Mihaiii/qa-assistant": ["train"]}, + adapted_from="Mihaiii/test22", + superseded_by=None, +) +Mihaiii__gte_micro = ModelMeta( + name="Mihaiii/gte-micro", + revision="6fd2397cb9dfa7c901aedf9a2a44d3c888ccafdd", + release_date="2024-04-21", + languages=None, + loader=None, + n_parameters=17389824, + max_tokens=512.0, + embed_dim=384, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Mihaiii/gte-micro", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +Mihaiii__gte_micro_v4 = ModelMeta( + name="Mihaiii/gte-micro-v4", + revision="78e1a4b348f8524c3ab2e3e3475788f5adb8c98f", + release_date="2024-04-22", + languages=None, + loader=None, + n_parameters=19164288, + max_tokens=512.0, + embed_dim=384, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Mihaiii/gte-micro-v4", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +OrdalieTech__Solon_embeddings_large_0_1 = ModelMeta( + name="OrdalieTech/Solon-embeddings-large-0.1", + revision="9f6465f6ea2f6d10c6294bc15d84edf87d47cdef", + release_date="2023-12-09", + languages=["fra_Latn"], + loader=None, + n_parameters=559890432, + max_tokens=514.0, + embed_dim=1024, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/OrdalieTech/Solon-embeddings-large-0.1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="solon-large-06-BIG", + superseded_by=None, +) +Omartificial_Intelligence_Space__Arabert_all_nli_triplet_Matryoshka = ModelMeta( + name="Omartificial-Intelligence-Space/Arabert-all-nli-triplet-Matryoshka", + revision="d0361a36f6fe69febfc8550d0918abab174f6f30", + release_date="2024-06-16", + languages=["ara_Arab"], + loader=None, + n_parameters=135193344, + max_tokens=512.0, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Omartificial-Intelligence-Space/Arabert-all-nli-triplet-Matryoshka", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets={}, # not in MTEB: {"Omartificial-Intelligence-Space/Arabic-NLi-Triplet": ["train"]}, + adapted_from="aubmindlab/bert-base-arabertv02", + superseded_by=None, +) +Omartificial_Intelligence_Space__Arabic_MiniLM_L12_v2_all_nli_triplet = ModelMeta( + name="Omartificial-Intelligence-Space/Arabic-MiniLM-L12-v2-all-nli-triplet", + revision="6916465c43b984e955aa6dc72851474f0128f428", + release_date="2024-06-25", + languages=["ara_Arab"], + loader=None, + n_parameters=117653760, + max_tokens=512.0, + embed_dim=384, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Omartificial-Intelligence-Space/Arabic-MiniLM-L12-v2-all-nli-triplet", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=sent_trf_training_dataset, + # not in MTEB + # {"Omartificial-Intelligence-Space/Arabic-NLi-Triplet": ["train"]}, + adapted_from="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + superseded_by=None, +) +Omartificial_Intelligence_Space__Arabic_all_nli_triplet_Matryoshka = ModelMeta( + name="Omartificial-Intelligence-Space/Arabic-all-nli-triplet-Matryoshka", + revision="1ca467cc576bd76666a4d21b24ee43afa914dd10", + release_date="2024-06-14", + languages=["ara_Arab"], + loader=None, + n_parameters=278043648, + max_tokens=514.0, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Omartificial-Intelligence-Space/Arabic-all-nli-triplet-Matryoshka", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=sent_trf_training_dataset, # derived from + # not in MTEB: + # {"Omartificial-Intelligence-Space/Arabic-NLi-Triplet": ["train"]}, + adapted_from="sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + superseded_by=None, +) +Omartificial_Intelligence_Space__Arabic_labse_Matryoshka = ModelMeta( + name="Omartificial-Intelligence-Space/Arabic-labse-Matryoshka", + revision="ee6d5e33c78ed582ade47fd452a74ea52aa5bfe2", + release_date="2024-06-16", + languages=["ara_Arab"], + loader=None, + n_parameters=470926848, + max_tokens=512.0, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Omartificial-Intelligence-Space/Arabic-labse-Matryoshka", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, # derived from labSE + # as well as: + # {"Omartificial-Intelligence-Space/Arabic-NLi-Triplet": ["train"]}, + adapted_from="sentence-transformers/LaBSE", + superseded_by=None, +) +Omartificial_Intelligence_Space__Arabic_mpnet_base_all_nli_triplet = ModelMeta( + name="Omartificial-Intelligence-Space/Arabic-mpnet-base-all-nli-triplet", + revision="2628cb641e040f44328195fadcdfb58e6d5cffa7", + release_date="2024-06-15", + languages=["ara_Arab"], + loader=None, + n_parameters=109486464, + max_tokens=514.0, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Omartificial-Intelligence-Space/Arabic-mpnet-base-all-nli-triplet", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=sent_trf_training_dataset, + # not in MTEB: + # {"Omartificial-Intelligence-Space/Arabic-NLi-Triplet": ["train"]}, + adapted_from="tomaarsen/mpnet-base-all-nli-triplet", + superseded_by=None, +) +Omartificial_Intelligence_Space__Marbert_all_nli_triplet_Matryoshka = ModelMeta( + name="Omartificial-Intelligence-Space/Marbert-all-nli-triplet-Matryoshka", + revision="ecf3274e164f057c4a3dd70691cae0265d87a9d0", + release_date="2024-06-17", + languages=["ara_Arab"], + loader=None, + n_parameters=162841344, + max_tokens=512.0, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Omartificial-Intelligence-Space/Marbert-all-nli-triplet-Matryoshka", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets={}, # not in MTEB: "Omartificial-Intelligence-Space/Arabic-NLi-Triplet": ["train"]}, + adapted_from="UBC-NLP/MARBERTv2", + superseded_by=None, +) +consciousAI__cai_lunaris_text_embeddings = ModelMeta( + name="consciousAI/cai-lunaris-text-embeddings", + revision="8332c464d13505968ff7a6e2213f36fd8730b4c7", + release_date="2023-06-22", + languages=None, + loader=None, + n_parameters=None, + max_tokens=512.0, + embed_dim=1024, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/consciousAI/cai-lunaris-text-embeddings", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="/root/.cache/torch/sentence_transformers/intfloat_e5-large-v2", + superseded_by=None, +) +consciousAI__cai_stellaris_text_embeddings = ModelMeta( + name="consciousAI/cai-stellaris-text-embeddings", + revision="c000ec4b29588daf0f4a0b2ad4e72ee807d8efc0", + release_date="2023-06-23", + languages=None, + loader=None, + n_parameters=None, + max_tokens=514.0, + embed_dim=768, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/consciousAI/cai-stellaris-text-embeddings", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="/root/.cache/torch/sentence_transformers/sentence-transformers_all-mpnet-base-v1/", + superseded_by=None, +) +manu__bge_m3_custom_fr = ModelMeta( + name="manu/bge-m3-custom-fr", + revision="ed3ef88678ba83ddf4c0fab71a93cb90d89a9078", + release_date="2024-04-11", + languages=None, + loader=None, + n_parameters=567754752, + max_tokens=8194.0, + embed_dim=1024, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/manu/bge-m3-custom-fr", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="data/bge-m3-custom", + superseded_by=None, +) +manu__sentence_croissant_alpha_v0_2 = ModelMeta( + name="manu/sentence_croissant_alpha_v0.2", + revision="4610b8cea65d7dd59e0b04af50753933fe5b29b2", + release_date="2024-03-15", + languages=None, + loader=None, + n_parameters=1279887360, + max_tokens=2048.0, + embed_dim=2048, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/manu/sentence_croissant_alpha_v0.2", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="croissantllm/CroissantCool", + superseded_by="manu/sentence_croissant_alpha_v0.3", +) +manu__sentence_croissant_alpha_v0_3 = ModelMeta( + name="manu/sentence_croissant_alpha_v0.3", + revision="4ac16754f3d81aba76cc32955dc9ee4122df96eb", + release_date="2024-04-26", + languages=None, + loader=None, + n_parameters=1279887360, + max_tokens=2048.0, + embed_dim=2048, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/manu/sentence_croissant_alpha_v0.3", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="croissantllm/CroissantCool-v0.2", + superseded_by="manu/sentence_croissant_alpha_v0.4", +) +manu__sentence_croissant_alpha_v0_4 = ModelMeta( + name="manu/sentence_croissant_alpha_v0.4", + revision="0ce6372e6a3c21134dcf26dcde13cca869c767fc", + release_date="2024-04-27", + languages=["fra_Latn", "eng_Latn"], + loader=None, + n_parameters=1279887360, + max_tokens=2048.0, + embed_dim=2048, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/manu/sentence_croissant_alpha_v0.4", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + # Not in MTEB: {"manu/embedding_data_v2_100k": ["train"]}, + adapted_from="croissantllm/CroissantCool-v0.2", + superseded_by=None, +) +thenlper__gte_base = ModelMeta( + name="thenlper/gte-base", + revision="c078288308d8dee004ab72c6191778064285ec0c", + release_date="2023-07-27", + languages=["eng_Latn"], + loader=None, + n_parameters=109482752, + max_tokens=512.0, + embed_dim=768, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/thenlper/gte-base", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +thenlper__gte_large = ModelMeta( + name="thenlper/gte-large", + revision="4bef63f39fcc5e2d6b0aae83089f307af4970164", + release_date="2023-07-27", + languages=["eng_Latn"], + loader=None, + n_parameters=335142400, + max_tokens=512.0, + embed_dim=1024, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/thenlper/gte-large", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +thenlper__gte_small = ModelMeta( + name="thenlper/gte-small", + revision="17e1f347d17fe144873b1201da91788898c639cd", + release_date="2023-07-27", + languages=["eng_Latn"], + loader=None, + n_parameters=33360512, + max_tokens=512.0, + embed_dim=384, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/thenlper/gte-small", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +OrlikB__KartonBERT_USE_base_v1 = ModelMeta( + name="OrlikB/KartonBERT-USE-base-v1", + revision="1f59dd58fe57995c0e867d5e29f03763eae99645", + release_date="2024-09-30", + languages=["pol_Latn"], + loader=None, + n_parameters=103705344, + max_tokens=512.0, + embed_dim=768, + license="gpl-3.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/OrlikB/KartonBERT-USE-base-v1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="KartonBERT-USE-base-v1", + superseded_by=None, +) +OrlikB__st_polish_kartonberta_base_alpha_v1 = ModelMeta( + name="OrlikB/st-polish-kartonberta-base-alpha-v1", + revision="5590a0e2d7bb43674e44d7076b3ff157f7d4a1cb", + release_date="2023-11-12", + languages=["pol_Latn"], + loader=None, + n_parameters=None, + max_tokens=514.0, + embed_dim=768, + license="lgpl", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/OrlikB/st-polish-kartonberta-base-alpha-v1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="st-polish-kartonberta-base-alpha-v1", + superseded_by=None, +) +sdadas__mmlw_e5_base = ModelMeta( + name="sdadas/mmlw-e5-base", + revision="f10628ed55b5ec400502aff439bd714a6da0af30", + release_date="2023-11-17", + languages=["pol_Latn"], + loader=None, + n_parameters=278043648, + max_tokens=514.0, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/sdadas/mmlw-e5-base", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="intfloat/multilingual-e5-base", + superseded_by=None, +) +dwzhu__e5_base_4k = ModelMeta( + name="dwzhu/e5-base-4k", + revision="1b5664b8cb2bccd8c309429b7bfe5864402e8fbc", + release_date="2024-03-28", + languages=["eng_Latn"], + loader=None, + n_parameters=None, + max_tokens=4096.0, + embed_dim=None, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/dwzhu/e5-base-4k", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="/mnt/default/longembed/models/intfloat/e5-base-v2", + superseded_by=None, +) +sdadas__mmlw_e5_large = ModelMeta( + name="sdadas/mmlw-e5-large", + revision="5c143fb045ebed664fd85b43fc45155999eb110f", + release_date="2023-11-17", + languages=["pol_Latn"], + loader=None, + n_parameters=559890432, + max_tokens=514.0, + embed_dim=1024, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/sdadas/mmlw-e5-large", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="intfloat/multilingual-e5-large", + superseded_by=None, +) +sdadas__mmlw_e5_small = ModelMeta( + name="sdadas/mmlw-e5-small", + revision="ff1298cb6d997f18b794d2f3d73cad2ba2ad739a", + release_date="2023-11-17", + languages=["pol_Latn"], + loader=None, + n_parameters=117653760, + max_tokens=512.0, + embed_dim=384, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/sdadas/mmlw-e5-small", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="intfloat/multilingual-e5-small", + superseded_by=None, +) +sdadas__mmlw_roberta_base = ModelMeta( + name="sdadas/mmlw-roberta-base", + revision="0ac7f23f6c96af601fa6a17852bd08d5136d6365", + release_date="2023-11-17", + languages=["pol_Latn"], + loader=None, + n_parameters=124442880, + max_tokens=514.0, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/sdadas/mmlw-roberta-base", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="sdadas/polish-roberta-base-v2", + superseded_by=None, +) +sdadas__mmlw_roberta_large = ModelMeta( + name="sdadas/mmlw-roberta-large", + revision="b8058066a8de32d0737b3cd82d8b4f4108745af9", + release_date="2023-11-17", + languages=["pol_Latn"], + loader=None, + n_parameters=434961408, + max_tokens=514.0, + embed_dim=1024, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/sdadas/mmlw-roberta-large", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="sdadas/polish-roberta-large-v2", + superseded_by=None, +) +izhx__udever_bloom_1b1 = ModelMeta( + name="izhx/udever-bloom-1b1", + revision="7bf1ee29878cb040b2708a691aa4b61f27eaa252", + release_date="2023-10-24", + languages=[ + "aka_Latn", + "ara_Arab", + "asm_Beng", + "bam_Latn", + "ben_Beng", + "cat_Latn", + "eng_Latn", + "spa_Latn", + "eus_Latn", + "fon_Latn", + "fra_Latn", + "guj_Gujr", + "hin_Deva", + "ind_Latn", + "ibo_Latn", + "kik_Latn", + "kan_Knda", + "lug_Latn", + "lin_Latn", + "mal_Mlym", + "mar_Deva", + "nep_Deva", + "nso_Latn", + "nya_Latn", + "ori_Orya", + "pan_Guru", + "por_Latn", + "run_Latn", + "kin_Latn", + "sna_Latn", + "sot_Latn", + "swa_Latn", + "tam_Taml", + "tel_Telu", + "tsn_Latn", + "tso_Latn", + "tum_Latn", + "twi_Latn", + "urd_Arab", + "vie_Latn", + "wol_Latn", + "xho_Latn", + "yor_Latn", + "zho_Hans", + "zul_Latn", + ], + loader=None, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license="bigscience-bloom-rail-1.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/izhx/udever-bloom-1b1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="bigscience/bloom-1b1", + superseded_by=None, +) +izhx__udever_bloom_3b = ModelMeta( + name="izhx/udever-bloom-3b", + revision="4edd8affe80ca89ba0f6b6ba4103fc7f25fc57b2", + release_date="2023-10-24", + languages=[ + "aka_Latn", + "ara_Arab", + "asm_Beng", + "bam_Latn", + "ben_Beng", + "cat_Latn", + "eng_Latn", + "spa_Latn", + "eus_Latn", + "fon_Latn", + "fra_Latn", + "guj_Gujr", + "hin_Deva", + "ind_Latn", + "ibo_Latn", + "kik_Latn", + "kan_Knda", + "lug_Latn", + "lin_Latn", + "mal_Mlym", + "mar_Deva", + "nep_Deva", + "nso_Latn", + "nya_Latn", + "ori_Orya", + "pan_Guru", + "por_Latn", + "run_Latn", + "kin_Latn", + "sna_Latn", + "sot_Latn", + "swa_Latn", + "tam_Taml", + "tel_Telu", + "tsn_Latn", + "tso_Latn", + "tum_Latn", + "twi_Latn", + "urd_Arab", + "vie_Latn", + "wol_Latn", + "xho_Latn", + "yor_Latn", + "zho_Hans", + "zul_Latn", + ], + loader=None, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license="bigscience-bloom-rail-1.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/izhx/udever-bloom-3b", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="bigscience/bloom-3b", + superseded_by=None, +) +izhx__udever_bloom_560m = ModelMeta( + name="izhx/udever-bloom-560m", + revision="b2a723e355946ec5a5c5fbed3459766627ded2bb", + release_date="2023-10-24", + languages=[ + "aka_Latn", + "ara_Arab", + "asm_Beng", + "bam_Latn", + "ben_Beng", + "cat_Latn", + "eng_Latn", + "spa_Latn", + "eus_Latn", + "fon_Latn", + "fra_Latn", + "guj_Gujr", + "hin_Deva", + "ind_Latn", + "ibo_Latn", + "kik_Latn", + "kan_Knda", + "lug_Latn", + "lin_Latn", + "mal_Mlym", + "mar_Deva", + "nep_Deva", + "nso_Latn", + "nya_Latn", + "ori_Orya", + "pan_Guru", + "por_Latn", + "run_Latn", + "kin_Latn", + "sna_Latn", + "sot_Latn", + "swa_Latn", + "tam_Taml", + "tel_Telu", + "tsn_Latn", + "tso_Latn", + "tum_Latn", + "twi_Latn", + "urd_Arab", + "vie_Latn", + "wol_Latn", + "xho_Latn", + "yor_Latn", + "zho_Hans", + "zul_Latn", + ], + loader=None, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license="bigscience-bloom-rail-1.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/izhx/udever-bloom-560m", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="bigscience/bloom-560m", + superseded_by=None, +) +izhx__udever_bloom_7b1 = ModelMeta( + name="izhx/udever-bloom-7b1", + revision="18e8d3e6dbd94868584877f2e72a105a17df22ef", + release_date="2023-10-24", + languages=[ + "aka_Latn", + "ara_Arab", + "asm_Beng", + "bam_Latn", + "ben_Beng", + "cat_Latn", + "eng_Latn", + "spa_Latn", + "eus_Latn", + "fon_Latn", + "fra_Latn", + "guj_Gujr", + "hin_Deva", + "ind_Latn", + "ibo_Latn", + "kik_Latn", + "kan_Knda", + "lug_Latn", + "lin_Latn", + "mal_Mlym", + "mar_Deva", + "nep_Deva", + "nso_Latn", + "nya_Latn", + "ori_Orya", + "pan_Guru", + "por_Latn", + "run_Latn", + "kin_Latn", + "sna_Latn", + "sot_Latn", + "swa_Latn", + "tam_Taml", + "tel_Telu", + "tsn_Latn", + "tso_Latn", + "tum_Latn", + "twi_Latn", + "urd_Arab", + "vie_Latn", + "wol_Latn", + "xho_Latn", + "yor_Latn", + "zho_Hans", + "zul_Latn", + ], + loader=None, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license="bigscience-bloom-rail-1.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/izhx/udever-bloom-7b1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="bigscience/bloom-7b1", + superseded_by=None, +) +avsolatorio__GIST_Embedding_v0 = ModelMeta( + name="avsolatorio/GIST-Embedding-v0", + revision="bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c", + release_date="2024-01-31", + languages=["eng_Latn"], + loader=None, + n_parameters=109482240, + max_tokens=512.0, + embed_dim=768, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/avsolatorio/GIST-Embedding-v0", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +avsolatorio__GIST_all_MiniLM_L6_v2 = ModelMeta( + name="avsolatorio/GIST-all-MiniLM-L6-v2", + revision="ea89dfad053bba14677bb784a4269898abbdce44", + release_date="2024-02-03", + languages=["eng_Latn"], + loader=None, + n_parameters=22713216, + max_tokens=512.0, + embed_dim=384, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/avsolatorio/GIST-all-MiniLM-L6-v2", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +avsolatorio__GIST_large_Embedding_v0 = ModelMeta( + name="avsolatorio/GIST-large-Embedding-v0", + revision="7831200e2f7819b994490c091cf3258a2b821f0c", + release_date="2024-02-14", + languages=["eng_Latn"], + loader=None, + n_parameters=335141888, + max_tokens=512.0, + embed_dim=1024, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/avsolatorio/GIST-large-Embedding-v0", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +avsolatorio__GIST_small_Embedding_v0 = ModelMeta( + name="avsolatorio/GIST-small-Embedding-v0", + revision="d6c4190f9e01b9994dc7cac99cf2f2b85cfb57bc", + release_date="2024-02-03", + languages=["eng_Latn"], + loader=None, + n_parameters=33360000, + max_tokens=512.0, + embed_dim=384, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/avsolatorio/GIST-small-Embedding-v0", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +bigscience__sgpt_bloom_7b1_msmarco = ModelMeta( + name="bigscience/sgpt-bloom-7b1-msmarco", + revision="dc579f3d2d5a0795eba2049e16c3e36c74007ad3", + release_date="2022-08-26", + languages=None, + loader=None, + n_parameters=None, + max_tokens=None, + embed_dim=4096, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/bigscience/sgpt-bloom-7b1-msmarco", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/6b3/bloom-7b1", + superseded_by=None, +) +aari1995__German_Semantic_STS_V2 = ModelMeta( + name="aari1995/German_Semantic_STS_V2", + revision="22912542b0ec7a7ef369837e28ffe6352a27afc9", + release_date="2022-11-17", + languages=["deu_Latn"], + loader=None, + n_parameters=335736320, + max_tokens=512.0, + embed_dim=1024, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/aari1995/German_Semantic_STS_V2", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, # couldn't figure out the source model + # {"stsb_multi_mt": ["train"]}, + adapted_from="/content/drive/MyDrive/Stanford_NLU/Project/false_friends/gbert_large_sts_only", + superseded_by=None, +) +abhinand__MedEmbed_small_v0_1 = ModelMeta( + name="abhinand/MedEmbed-small-v0.1", + revision="40a5850d046cfdb56154e332b4d7099b63e8d50e", + release_date="2024-10-20", + languages=["eng_Latn"], + loader=None, + n_parameters=33360000, + max_tokens=512.0, + embed_dim=384, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/abhinand/MedEmbed-small-v0.1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets={ + "MedicalQARetrieval": ["train"], + "NFCorpus": ["train"], + "PublicHealthQA": ["train"], + "TRECCOVID": ["train"], + "ArguAna": ["train"], + }, + adapted_from="./medical-bge-small-v1-mix1", + superseded_by=None, +) +avsolatorio__NoInstruct_small_Embedding_v0 = ModelMeta( + name="avsolatorio/NoInstruct-small-Embedding-v0", + revision="b38747000553d8268915c95a55fc87e707c9aadd", + release_date="2024-05-01", + languages=["eng_Latn"], + loader=None, + n_parameters=33360000, + max_tokens=512.0, + embed_dim=384, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/avsolatorio/NoInstruct-small-Embedding-v0", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +brahmairesearch__slx_v0_1 = ModelMeta( + name="brahmairesearch/slx-v0.1", + revision="688c83fd1a7f34b25575a2bc26cfd87c11b4ce71", + release_date="2024-08-13", + languages=["eng_Latn"], + loader=None, + n_parameters=22713216, + max_tokens=512.0, + embed_dim=384, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/brahmairesearch/slx-v0.1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +deepfile__embedder_100p = ModelMeta( + name="deepfile/embedder-100p", + revision="aa02f08f11517977fbcdc94dc9dbf9a1ca152d9b", + release_date="2023-07-24", + languages=None, + loader=None, + n_parameters=None, + max_tokens=514.0, + embed_dim=768, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/deepfile/embedder-100p", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + superseded_by=None, +) +deepvk__USER_bge_m3 = ModelMeta( + name="deepvk/USER-bge-m3", + revision="0cc6cfe48e260fb0474c753087a69369e88709ae", + release_date="2024-07-05", + languages=["rus_Cyrl"], + loader=None, + n_parameters=359026688, + max_tokens=8194.0, + embed_dim=1024, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/deepvk/USER-bge-m3", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=bge_m3_training_data, # derived from. + # not in MTEB: + # "deepvk/ru-HNP": ["train"], + # "deepvk/ru-WANLI": ["train"], + # "Shitao/bge-m3-data": ["train"], + # "RussianNLP/russian_super_glue": ["train"], + # "reciTAL/mlsum": ["train"], + # "Milana/russian_keywords": ["train"], + # "IlyaGusev/gazeta": ["train"], + # "d0rj/gsm8k-ru": ["train"], + # "bragovo/dsum_ru": ["train"], + # "CarlBrendt/Summ_Dialog_News": ["train"], + adapted_from="USER-bge-m3", + superseded_by=None, +) +infgrad__stella_base_en_v2 = ModelMeta( + name="infgrad/stella-base-en-v2", + revision="c9e80ff9892d80b39dc54e30a7873f91ea161034", + release_date="2023-10-19", + languages=["eng_Latn"], + loader=None, + n_parameters=None, + max_tokens=512.0, + embed_dim=None, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/infgrad/stella-base-en-v2", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +malenia1__ternary_weight_embedding = ModelMeta( + name="malenia1/ternary-weight-embedding", + revision="a1208fb7f646647bb62639fd2e1eb6cc2ef3738e", + release_date="2024-10-23", + languages=None, + loader=None, + n_parameters=98688000, + max_tokens=512.0, + embed_dim=1024, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference="https://huggingface.co/malenia1/ternary-weight-embedding", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="ternary-weight-embedding", + superseded_by=None, +) +omarelshehy__arabic_english_sts_matryoshka = ModelMeta( + name="omarelshehy/arabic-english-sts-matryoshka", + revision="763d116fbe8bf7883c64635c862feeaa3768bb64", + release_date="2024-10-13", + languages=["ara_Arab", "eng_Latn"], + loader=None, + n_parameters=559890432, + max_tokens=514.0, + embed_dim=1024, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/omarelshehy/arabic-english-sts-matryoshka", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="FacebookAI/xlm-roberta-large", + superseded_by=None, +) +openbmb__MiniCPM_Embedding = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="openbmb/MiniCPM-Embedding", + revision="c0cb2de33fb366e17c30f9d53142ff11bc18e049", + # https://huggingface.co/openbmb/MiniCPM-Embedding/blob/c0cb2de33fb366e17c30f9d53142ff11bc18e049/README.md?code=true#L405 + model_kwargs={ + # "attn_implementation": "flash_attention_2", + "torch_dtype": torch.float16, + }, + trust_remote_code=True, + ), + name="openbmb/MiniCPM-Embedding", + revision="c0cb2de33fb366e17c30f9d53142ff11bc18e049", + release_date="2024-09-04", + languages=["zho_Hans", "eng_Latn"], + n_parameters=2724880896, + max_tokens=512.0, + embed_dim=2304, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/openbmb/MiniCPM-Embedding", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from=None, + superseded_by=None, +) +shibing624__text2vec_base_multilingual = ModelMeta( + name="shibing624/text2vec-base-multilingual", + revision="6633dc49e554de7105458f8f2e96445c6598e9d1", + release_date="2023-06-22", + languages=[ + "zho_Hans", + "eng_Latn", + "deu_Latn", + "fra_Latn", + "ita_Latn", + "nld_Latn", + "por_Latn", + "pol_Latn", + "rus_Cyrl", + ], + loader=None, + n_parameters=117654272, + max_tokens=512.0, + embed_dim=384, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/shibing624/text2vec-base-multilingual", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=sent_trf_training_dataset, + # not MTEB: {"shibing624/nli-zh-all": ["train"]}, + adapted_from="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + superseded_by=None, +) +silma_ai__silma_embeddding_matryoshka_v0_1 = ModelMeta( + name="silma-ai/silma-embeddding-matryoshka-v0.1", + revision="a520977a9542ebdb8a7206df6b7ff6977f1886ea", + release_date="2024-10-12", + languages=["ara_Arab", "eng_Latn"], + loader=None, + n_parameters=135193344, + max_tokens=512.0, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/silma-ai/silma-embeddding-matryoshka-v0.1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + adapted_from="/workspace/v3-matryoshka_aubmindlab-bert-base-arabertv02-2024-10-12_13-55-06/checkpoint-26250", + superseded_by=None, +) + +sbert_chinese_general_v1 = ModelMeta( + name="DMetaSoul/sbert-chinese-general-v1", + revision="bd27765956bcc2fcf682de0097819947ac10037e", + release_date="2022-03-25", + languages=["zho_Hans"], + loader=None, + n_parameters=None, # Not visible on repo + max_tokens=512, + embed_dim=128, + license="apache-2", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/DMetaSoul/sbert-chinese-general-v1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets={ + "PAWSX": ["train"], + "PawsXPairClassification": ["train"], # they do not specify which one + # They might have trained on other datasets too, they don't say: + # "trained on semantically similar datasets such as NLI, PAWS-X, PKU-Paraphrase-Bank, and STS." + }, + superseded_by=None, +) +dmeta_embedding_zh_small = ModelMeta( + name="DMetaSoul/Dmeta-embedding-zh-small", + revision="2050d3439a2f68999dd648c1697471acaac37a29", + release_date="2024-03-25", + languages=["zho_Hans"], + loader=None, + n_parameters=74.2 * 1e6, + max_tokens=1024, + embed_dim=768, + license="apache-2", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/DMetaSoul/Dmeta-embedding-zh-small/", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, # They don't specify + superseded_by=None, +) +xiaobu_embedding = ModelMeta( + name="lier007/xiaobu-embedding", + revision="59c79d82eb5223cd9895f6eb8e825c7fa10e4e92", + release_date="2024-01-09", + languages=["zho_Hans"], + loader=None, + n_parameters=326 * 1e6, + max_tokens=512, + embed_dim=1024, + license="not specified", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/lier007/xiaobu-embedding", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, # Finetuned from GTE, none of them disclose training data + superseded_by=None, + adapted_from="thenlper/gte-large-zh", +) +xiaobu_embedding_v2 = ModelMeta( + name="lier007/xiaobu-embedding-v2", + revision="1912f2e59a5c2ef802a471d735a38702a5c9485e", + release_date="2024-06-30", + languages=["zho_Hans"], + loader=None, + n_parameters=326 * 1e6, + max_tokens=512, + embed_dim=768, + license="not specified", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/lier007/xiaobu-embedding-v2", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, # Finetuned from piccolo-embedding, none of them say + superseded_by=None, + adapted_from="sensenova/piccolo-base-zh", +) +yinka_embedding = ModelMeta( + name="Classical/Yinka", + revision="59c79d82eb5223cd9895f6eb8e825c7fa10e4e92", + release_date="2024-01-09", + languages=["zho_Hans"], + loader=None, + n_parameters=326 * 1e6, + max_tokens=512, + embed_dim=1024, + license="not specified", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Classical/Yinka", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, # Not disclosed + superseded_by=None, + adapted_from="dunzhang/stella-mrl-large-zh-v3.5-1792d", +) +conan_embedding = ModelMeta( + name="TencentBAC/Conan-embedding-v1", + revision="bb9749a57d4f02fd71722386f8d0f5a9398d7eeb", + release_date="2024-08-22", + languages=["zho_Hans"], + loader=None, + n_parameters=326 * 1e6, + max_tokens=512, + embed_dim=768, + license="cc-by-nc-4.0", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/Classical/Yinka", + similarity_fn_name="cosine", + use_instructions=None, + # source: https://arxiv.org/pdf/2408.15710 + training_datasets=None, # They "scraped" things from the internet, we don't know, could be leakage + superseded_by=None, +) +ember_v1 = ModelMeta( + name="llmrails/ember-v1", + revision="5e5ce5904901f6ce1c353a95020f17f09e5d021d", + release_date="2023-10-10", + languages=["eng_Latn"], + n_parameters=335 * 1e6, + max_tokens=512, + embed_dim=1024, + license="mit", + open_weights=True, + public_training_code=None, + public_training_data=None, + framework=["PyTorch", "Sentence Transformers"], + reference="https://huggingface.co/llmrails/ember-v1", + similarity_fn_name="cosine", + use_instructions=None, + training_datasets=None, + superseded_by=None, +) +amazon_titan_text_embeddings_v2 = ModelMeta( + name="amazon/Titan-text-embeddings-v2", + revision="1", + release_date="2024-04-30", + languages=["eng_Latn"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license="proprietary", + open_weights=False, + public_training_code=None, + public_training_data=None, + framework=[], + reference="https://huggingface.co/amazon/Titan-text-embeddings-v2", + similarity_fn_name="cosine", + use_instructions=False, + training_datasets=None, + superseded_by=None, +) diff --git a/mteb/models/moco_models.py b/mteb/models/moco_models.py index a64cef499e..9493b8e5a6 100644 --- a/mteb/models/moco_models.py +++ b/mteb/models/moco_models.py @@ -145,10 +145,21 @@ def get_fused_embeddings( ), name="nyu-visionx/moco-v3-vit-b", languages=["eng_Latn"], - open_source=True, revision="7d091cd70772c5c0ecf7f00b5f12ca609a99d69d", release_date="2024-06-03", modalities=["image"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) mocov3_vit_large = ModelMeta( @@ -158,8 +169,19 @@ def get_fused_embeddings( ), name="nyu-visionx/moco-v3-vit-l", languages=["eng_Latn"], - open_source=True, revision="7bf75358d616f39b9716148bf4e3425f3bd35b47", release_date="2024-06-03", modalities=["image"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) diff --git a/mteb/models/model2vec_models.py b/mteb/models/model2vec_models.py new file mode 100644 index 0000000000..33da211c7a --- /dev/null +++ b/mteb/models/model2vec_models.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import logging +from collections.abc import Sequence +from functools import partial +from typing import Any + +import numpy as np + +from mteb.model_meta import ModelMeta + +from .bge_models import bge_training_data +from .wrapper import Wrapper + +logger = logging.getLogger(__name__) + + +class Model2VecWrapper(Wrapper): + def __init__( + self, + model_name: str, + **kwargs, + ) -> None: + """Wrapper for Model2Vec models. + + Args: + model_name: The Model2Vec model to load from HuggingFace Hub. + **kwargs: Additional arguments to pass to the wrapper. + """ + try: + from model2vec import StaticModel # type: ignore + except ModuleNotFoundError as e: + raise ModuleNotFoundError( + "To use the Model2Vec models `model2vec` is required. Please install it with `pip install mteb[model2vec]`." + ) from e + + self.model_name = model_name + self.static_model = StaticModel.from_pretrained(self.model_name) + + def encode( + self, + sentences: Sequence[str], + **kwargs: Any, + ) -> np.ndarray: + """Encodes the given sentences using the encoder. + + Args: + sentences: The sentences to encode. + **kwargs: Additional arguments to pass to the encoder. + + Returns: + The encoded sentences. + """ + return self.static_model.encode(sentences).astype(np.float32) + + +m2v_base_glove_subword = ModelMeta( + loader=partial( + Model2VecWrapper, + model_name="minishlab/M2V_base_glove_subword", + ), + name="minishlab/M2V_base_glove_subword", + languages=["eng_Latn"], + open_weights=True, + revision="5f4f5ca159b7321a8b39739bba0794fa0debddf4", + release_date="2024-09-21", + n_parameters=int(103 * 1e6), + max_tokens=np.inf, # Theoretically infinite + embed_dim=256, + license="mit", + similarity_fn_name="cosine", + framework=["NumPy"], + reference="https://huggingface.co/minishlab/M2V_base_glove_subword", + use_instructions=False, + adapted_from="BAAI/bge-base-en-v1.5", + superseded_by=None, + training_datasets=bge_training_data, # distilled + public_training_code="https://github.com/MinishLab/model2vec", + public_training_data=None, +) + + +m2v_base_glove = ModelMeta( + loader=partial( + Model2VecWrapper, + model_name="minishlab/M2V_base_glove", + ), + name="minishlab/M2V_base_glove", + languages=["eng_Latn"], + open_weights=True, + revision="38ebd7f10f71e67fa8db898290f92b82e9cfff2b", + release_date="2024-09-21", + n_parameters=int(102 * 1e6), + max_tokens=np.inf, + embed_dim=256, + license="mit", + similarity_fn_name="cosine", + framework=["NumPy"], + reference="https://huggingface.co/minishlab/M2V_base_glove", + use_instructions=False, + adapted_from="BAAI/bge-base-en-v1.5", + superseded_by=None, + training_datasets=bge_training_data, # distilled + public_training_code="https://github.com/MinishLab/model2vec", + public_training_data=None, +) + +m2v_base_output = ModelMeta( + loader=partial( + Model2VecWrapper, + model_name="minishlab/M2V_base_output", + ), + name="minishlab/M2V_base_output", + languages=["eng_Latn"], + open_weights=True, + revision="02460ae401a22b09d2c6652e23371398329551e2", + release_date="2024-09-21", + n_parameters=int(7.56 * 1e6), + max_tokens=np.inf, + embed_dim=256, + license="mit", + similarity_fn_name="cosine", + framework=["NumPy"], + reference="https://huggingface.co/minishlab/M2V_base_output", + use_instructions=False, + adapted_from="BAAI/bge-base-en-v1.5", + superseded_by=None, + training_datasets=bge_training_data, # distilled + public_training_code="https://github.com/MinishLab/model2vec", + public_training_data=None, +) + +m2v_multilingual_output = ModelMeta( + loader=partial( + Model2VecWrapper, + model_name="minishlab/M2V_multilingual_output", + ), + name="minishlab/M2V_multilingual_output", + languages=["eng_Latn"], + open_weights=True, + revision="2cf4ec4e1f51aeca6c55cf9b93097d00711a6305", + release_date="2024-09-21", + n_parameters=int(128 * 1e6), + max_tokens=np.inf, + embed_dim=256, + license="mit", + similarity_fn_name="cosine", + framework=["NumPy"], + reference="https://huggingface.co/minishlab/M2V_multilingual_output", + use_instructions=False, + adapted_from="sentence-transformers/LaBSE", + superseded_by=None, + training_datasets=None, + public_training_code="https://github.com/MinishLab/model2vec", + public_training_data=None, +) + +potion_base_2m = ModelMeta( + loader=partial( + Model2VecWrapper, + model_name="minishlab/potion-base-2M", + ), + name="minishlab/potion-base-2M", + languages=["eng_Latn"], + open_weights=True, + revision="86db093558fbced2072b929eb1690bce5272bd4b", + release_date="2024-10-29", + n_parameters=2 * 1e6, + max_tokens=np.inf, + embed_dim=64, + license="mit", + similarity_fn_name="cosine", + framework=["NumPy"], + reference="https://huggingface.co/minishlab/potion-base-2M", + use_instructions=False, + adapted_from="BAAI/bge-base-en-v1.5", + superseded_by=None, + training_datasets=bge_training_data, # distilled + public_training_code="https://github.com/MinishLab/model2vec", + public_training_data=None, +) + +potion_base_4m = ModelMeta( + loader=partial( + Model2VecWrapper, + model_name="minishlab/potion-base-4M", + ), + name="minishlab/potion-base-4M", + languages=["eng_Latn"], + open_weights=True, + revision="81b1802ada41afcd0987a37dc15e569c9fa76f04", + release_date="2024-10-29", + n_parameters=3.78 * 1e6, + max_tokens=np.inf, + embed_dim=128, + license="mit", + similarity_fn_name="cosine", + framework=["NumPy"], + reference="https://huggingface.co/minishlab/potion-base-4M", + use_instructions=False, + adapted_from="BAAI/bge-base-en-v1.5", + superseded_by=None, + training_datasets=bge_training_data, # distilled + public_training_code="https://github.com/MinishLab/model2vec", + public_training_data=None, +) + +potion_base_8m = ModelMeta( + loader=partial( + Model2VecWrapper, + model_name="minishlab/potion-base-8M", + ), + name="minishlab/potion-base-8M", + languages=["eng_Latn"], + open_weights=True, + revision="dcbec7aa2d52fc76754ac6291803feedd8c619ce", + release_date="2024-10-29", + n_parameters=7.56 * 1e6, + max_tokens=np.inf, + embed_dim=256, + license="mit", + similarity_fn_name="cosine", + framework=["NumPy"], + reference="https://huggingface.co/minishlab/potion-base-8M", + use_instructions=False, + adapted_from="BAAI/bge-base-en-v1.5", + superseded_by=None, + training_datasets=bge_training_data, # distilled + public_training_code="https://github.com/MinishLab/model2vec", + public_training_data=None, +) diff --git a/mteb/models/moka_models.py b/mteb/models/moka_models.py new file mode 100644 index 0000000000..1504b40789 --- /dev/null +++ b/mteb/models/moka_models.py @@ -0,0 +1,147 @@ +"""Moka AI's Chinese embedding models""" + +from __future__ import annotations + +from mteb.model_meta import ModelMeta + +sent_trf_training_dataset = { + # derived from datasheets + "MSMARCO": ["train"], + "MSMARCOHardNegatives": ["train"], + "NanoMSMARCORetrieval": ["train"], + "MSMARCO-PL": ["train"], # translation not trained on + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on + # not in MTEB + # "s2orc": ["train"], + # "flax-sentence-embeddings/stackexchange_xml": ["train"], + # "ms_marco": ["train"], + # "gooaq": ["train"], + # "yahoo_answers_topics": ["train"], + # "code_search_net": ["train"], + # "search_qa": ["train"], + # "eli5": ["train"], + # "snli": ["train"], + # "multi_nli": ["train"], + # "wikihow": ["train"], + # "natural_questions": ["train"], + # "trivia_qa": ["train"], + # "embedding-data/sentence-compression": ["train"], + # "embedding-data/flickr30k-captions": ["train"], + # "embedding-data/altlex": ["train"], + # "embedding-data/simple-wiki": ["train"], + # "embedding-data/QQP": ["train"], + # "embedding-data/SPECTER": ["train"], + # "embedding-data/PAQ_pairs": ["train"], + # "embedding-data/WikiAnswers": ["train"], +} +medi_dataset = { + **sent_trf_training_dataset, + # not in MTEB: + # - Super-NI + # - KILT (https://arxiv.org/abs/2009.02252) + # - MedMCQA (https://proceedings.mlr.press/v174/pal22a/pal22a.pdf) +} +m3e_dataset = { + **medi_dataset, + "AmazonReviewsClassification": ["train"], # Possibly also test, hard to know + "Ocnli": ["train"], + "BQ": ["train"], + "LCQMC": ["train"], + "MIRACLReranking": ["train"], + "PAWSX": ["train"], + # not in MTEB: + # - cmrc2018 + # - belle_2m + # - firefily + # - alpaca_gpt4 + # - zhihu_kol + # - hc3_chinese + # - amazon_reviews_multi (intersects with AmazonReviewsClassification) + # - qa: Encyclopedia QA dataset + # - xlsum + # - wiki_atomic_edit + # - chatmed_consult + # - webqa + # - dureader_robust + # - csl + # - lawzhidao + # - CINLID + # - DuSQL + # - Zhuiyi-NL2SQL + # - Cspider + # - news2016zh + # - baike2018qa + # - webtext2019zh + # - SimCLUE + # - SQuAD +} + +m3e_base = ModelMeta( + name="moka-ai/m3e-base", + languages=["zho_Hans", "eng-Latn"], + open_weights=True, + revision="764b537a0e50e5c7d64db883f2d2e051cbe3c64c", + release_date="2023-06-06", # first commit + n_parameters=102 * 1e6, + embed_dim=768, + # They don't give a specific license but commercial use is not allowed + license="unspecified-noncommercial", + max_tokens=512, + reference="https://huggingface.co/moka-ai/m3e-base", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + public_training_code=None, + public_training_data=None, # Not published + training_datasets=m3e_dataset, +) + +m3e_small = ModelMeta( + name="moka-ai/m3e-small", + languages=["zho_Hans", "eng-Latn"], + open_weights=True, + revision="44c696631b2a8c200220aaaad5f987f096e986df", + release_date="2023-06-02", # first commit + n_parameters=None, # Can't be seen on HF page + embed_dim=512, + # They don't give a specific license but commercial use is not allowed + license="unspecified-noncommercial", + max_tokens=512, + reference="https://huggingface.co/moka-ai/m3e-small", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + public_training_code=None, + public_training_data=None, # Not published + training_datasets=m3e_dataset, +) + + +m3e_large = ModelMeta( + name="moka-ai/m3e-large", + languages=["zho_Hans", "eng-Latn"], + open_weights=True, + revision="12900375086c37ba5d83d1e417b21dc7d1d1f388", + release_date="2023-06-21", # first commit + n_parameters=None, # Can't be seen on HF page + embed_dim=768, + # They don't give a specific license but commercial use is not allowed + license="unspecified-noncommercial", + max_tokens=512, + reference="https://huggingface.co/moka-ai/m3e-large", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + public_training_code=None, + public_training_data=None, # Not published + training_datasets=m3e_dataset, +) diff --git a/mteb/models/mxbai_models.py b/mteb/models/mxbai_models.py index b5451e30ec..921db17871 100644 --- a/mteb/models/mxbai_models.py +++ b/mteb/models/mxbai_models.py @@ -5,7 +5,7 @@ from mteb.model_meta import ModelMeta, sentence_transformers_loader mxbai_embed_large_v1 = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="mixedbread-ai/mxbai-embed-large-v1", revision="990580e27d329c7408b3741ecff85876e128e203", @@ -19,12 +19,14 @@ revision="990580e27d329c7408b3741ecff85876e128e203", release_date="2024-03-07", # initial commit of hf model. n_parameters=335_000_000, - memory_usage=None, max_tokens=512, embed_dim=1024, license="apache-2.0", reference="https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=None, ) diff --git a/mteb/models/no_instruct_sentence_models.py b/mteb/models/no_instruct_sentence_models.py new file mode 100644 index 0000000000..9ff5cf901f --- /dev/null +++ b/mteb/models/no_instruct_sentence_models.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import numpy as np +import torch +from transformers import AutoModel, AutoTokenizer + +from mteb.encoder_interface import PromptType +from mteb.model_meta import ModelMeta +from mteb.models.wrapper import Wrapper + +from .utils import batched + + +class NoInstructWrapper(Wrapper): + def __init__( + self, + model_name: str, + revision: str, + model_prompts: dict[str, str] | None = None, + **kwargs: Any, + ): + self.model_name = model_name + device = kwargs.pop("device", None) + self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") + self.model = AutoModel.from_pretrained( + model_name, revision=revision, **kwargs + ).to(self.device) + self.model.eval() + self.tokenizer = AutoTokenizer.from_pretrained( + model_name, revision=revision, **kwargs + ) + + def encode( # type: ignore + self, + sentences: list[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + batch_size: int = 32, + **kwargs: Any, + ): + embeddings = [] + for batch in batched(sentences, batch_size): + # Tokenize the batch + encoding = self.tokenizer( + batch, + padding=True, + truncation=True, + return_tensors="pt", + return_attention_mask=True, + ).to(self.device) + + input_ids = encoding["input_ids"] + attention_mask = encoding["attention_mask"] + + # Forward pass + with torch.no_grad(): + outputs = self.model(input_ids=input_ids, attention_mask=attention_mask) + + # The model is optimized to use the mean pooling for queries, + # while the sentence / document embedding uses the [CLS] representation. + if prompt_type == PromptType.query: + # Mean pooling + vectors = outputs.last_hidden_state * attention_mask.unsqueeze(2) + pooled_vectors = vectors.sum(dim=1) / attention_mask.sum( + dim=-1, keepdim=True + ) + else: + # [CLS] token representation + pooled_vectors = outputs.last_hidden_state[:, 0, :] + + # Append pooled vectors to result + embeddings.append(pooled_vectors.cpu().detach().numpy()) + + return np.concatenate(embeddings, axis=0) + + +no_instruct_small_v0 = ModelMeta( + loader=partial( + NoInstructWrapper, + model_name="avsolatorio/NoInstruct-small-Embedding-v0", + revision="b38747000553d8268915c95a55fc87e707c9aadd", + ), + name="avsolatorio/NoInstruct-small-Embedding-v0", + languages=["eng-Latn"], + open_weights=True, + revision="b38747000553d8268915c95a55fc87e707c9aadd", + release_date="2024-05-01", # first commit + n_parameters=33_400_000, + max_tokens=512, + embed_dim=384, + license="mit", + reference="https://huggingface.co/avsolatorio/NoInstruct-small-Embedding-v0", + similarity_fn_name="cosine", + framework=["PyTorch"], + use_instructions=False, + adapted_from=None, + superseded_by=None, + public_training_code=None, + public_training_data=None, + training_datasets=None, +) diff --git a/mteb/models/nomic_models.py b/mteb/models/nomic_models.py index 00c9341b3b..15c7df1230 100644 --- a/mteb/models/nomic_models.py +++ b/mteb/models/nomic_models.py @@ -4,23 +4,24 @@ from functools import partial from typing import Any +import numpy as np import torch import torch.nn.functional as F -from sentence_transformers import SentenceTransformer +import transformers +from packaging.version import Version +import mteb from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta -from mteb.models.sentence_transformer_wrapper import ( - get_prompt_name, - validate_task_to_prompt_name, -) -from .wrapper import Wrapper +from .sentence_transformer_wrapper import SentenceTransformerWrapper logger = logging.getLogger(__name__) +MODERN_BERT_TRANSFORMERS_MIN_VERSION = "4.48.0" + -class NomicWrapper(Wrapper): +class NomicWrapper(SentenceTransformerWrapper): """following the hf model card documentation.""" def __init__( @@ -31,10 +32,15 @@ def __init__( **kwargs: Any, ): self.model_name = model_name - self.model = SentenceTransformer(model_name, revision=revision, **kwargs) - self.model_prompts = ( - validate_task_to_prompt_name(model_prompts) if model_prompts else None - ) + if model_name == "nomic-ai/modernbert-embed-base" and ( + Version(transformers.__version__).release + < Version(MODERN_BERT_TRANSFORMERS_MIN_VERSION).release + ): + raise RuntimeError( + f"Current transformers version is {transformers.__version__} is lower than the required version" + f" {MODERN_BERT_TRANSFORMERS_MIN_VERSION}" + ) + super().__init__(model_name, revision, model_prompts, **kwargs) def to(self, device: torch.device) -> None: self.model.to(device) @@ -47,39 +53,131 @@ def encode( # type: ignore prompt_type: PromptType | None = None, batch_size: int = 32, **kwargs: Any, - ): - input_type = get_prompt_name(self.model_prompts, task_name, prompt_type) - + ) -> np.ndarray: # default to search_document if input_type and prompt_name are not provided - if input_type is None: - input_type = "search_document" - - sentences = [f"{input_type}: {sentence}" for sentence in sentences] - - emb = self.model.encode(sentences, batch_size=batch_size, **kwargs) + prompt_name = ( + self.get_prompt_name(self.model_prompts, task_name, prompt_type) + or PromptType.passage.value + ) + task = mteb.get_task(task_name) + # normalization not applied to classification + # https://github.com/nomic-ai/contrastors/blob/5f7b461e5a13b5636692d1c9f1141b27232fe966/src/contrastors/eval/mteb_eval/eval_mteb.py#L172 + normalize = task.metadata.type not in ( + "Classification", + "MultilabelClassification", + "PairClassification", + "Reranking", + "STS", + "Summarization", + ) + emb = self.model.encode( + sentences, + prompt_name=prompt_name, + batch_size=batch_size, + **kwargs, + ) # v1.5 has a non-trainable layer norm to unit normalize the embeddings for binary quantization # the outputs are similar to if we just normalized but keeping the same for consistency if self.model_name == "nomic-ai/nomic-embed-text-v1.5": if not isinstance(emb, torch.Tensor): emb = torch.tensor(emb) emb = F.layer_norm(emb, normalized_shape=(emb.shape[1],)) - emb = F.normalize(emb, p=2, dim=1) - if kwargs.get("convert_to_tensor", False): - emb = emb.cpu().detach().numpy() + if normalize: + emb = F.normalize(emb, p=2, dim=1) + if isinstance(emb, torch.Tensor): + emb = emb.cpu().detach().float().numpy() return emb +nomic_training_data = { + # https://github.com/nomic-ai/contrastors/blob/5f7b461e5a13b5636692d1c9f1141b27232fe966/src/contrastors/configs/data/contrastive_pretrain.yaml + # reddit_title_body + "RedditClustering": [], + "RedditClusteringP2P": [], + "RedditClustering.v2": [], + "RedditClusteringP2P.v2": [], + # amazon_reviews + # amazonqa + "AmazonPolarityClassification": [], + "AmazonReviewsClassification": [], + "AmazonCounterfactualClassification": [], + # paq + # s2orc_citation_titles + # s2orc_title_abstract + # s2orc_abstract_citation + # s2orc_abstract_body + # wikianswers + # wikipedia + "WikipediaRetrievalMultilingual": [], + "WikipediaRerankingMultilingual": [], + # gooaq + # codesearch + "CodeSearchNetCCRetrieval": [], + "COIRCodeSearchNetRetrieval": [], + # yahoo_title_answer + # yahoo_qa + # yahoo_title_question + "YahooAnswersTopicsClassification": [], + # agnews + # ccnews + # npr + # eli5 + # cnn + # stackexchange_duplicate_questions + # stackexchange_title_body + # stackexchange_body_body + "StackExchangeClustering.v2": [], + "StackExchangeClusteringP2P.v2": [], + # sentence_compression + # wikihow + # altlex + # quora + "QuoraRetrieval": [], + "NanoQuoraRetrieval": [], + # simplewiki + # squad + "FQuADRetrieval": [], + # https://github.com/nomic-ai/contrastors/blob/5f7b461e5a13b5636692d1c9f1141b27232fe966/src/contrastors/configs/data/finetune_triplets.yaml + # msmaro + "MSMARCO": ["train"], + "MSMARCOHardNegatives": ["train"], + "NanoMSMARCORetrieval": ["train"], + # nq_triples + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on + # nli_triplets + # reddit + # medi_wiki + # medi_stackexchange + # medi_flickr + # medi_supernli + # hotpot + "HotPotQA": ["test"], + "HotPotQAHardNegatives": ["test"], + "HotPotQA-PL": ["test"], # translated from hotpotQA (not trained on) + # fever + "FEVER": ["test"], + "FEVERHardNegatives": ["test"], +} + +# https://github.com/nomic-ai/contrastors/blob/5f7b461e5a13b5636692d1c9f1141b27232fe966/src/contrastors/eval/mteb_eval/eval_mteb.py#L142-L159 model_prompts = { "Classification": "classification: ", "MultilabelClassification": "classification: ", "Clustering": "clustering: ", + "PairClassification": "classification: ", + "Reranking": "classification: ", + "STS": "classification: ", + "Summarization": "classification: ", PromptType.query.value: "search_query: ", PromptType.passage.value: "search_document: ", } nomic_embed_v1_5 = ModelMeta( - loader=partial( # type: ignore + loader=partial( NomicWrapper, trust_remote_code=True, model_name="nomic-ai/nomic-embed-text-v1.5", @@ -91,10 +189,23 @@ def encode( # type: ignore open_weights=True, revision="b0753ae76394dd36bcfb912a46018088bca48be0", release_date="2024-02-10", # first commit + n_parameters=137_000_000, + max_tokens=8192, + embed_dim=768, + license="apache-2.0", + reference="https://huggingface.co/nomic-ai/nomic-embed-text-v1.5", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + adapted_from=None, + superseded_by=None, + public_training_data=None, + public_training_code="https://github.com/nomic-ai/contrastors/blob/5f7b461e5a13b5636692d1c9f1141b27232fe966/src/contrastors/configs/train/contrastive_finetune.yaml", + training_datasets=nomic_training_data, ) nomic_embed_v1 = ModelMeta( - loader=partial( # type: ignore + loader=partial( NomicWrapper, trust_remote_code=True, model_name="nomic-ai/nomic-embed-text-v1", @@ -107,12 +218,104 @@ def encode( # type: ignore revision="0759316f275aa0cb93a5b830973843ca66babcf5", release_date="2024-01-31", # first commit n_parameters=None, - memory_usage=None, max_tokens=8192, embed_dim=768, license="apache-2.0", reference="https://huggingface.co/nomic-ai/nomic-embed-text-v1", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=True, + use_instructions=True, + adapted_from=None, + superseded_by="nomic-ai/nomic-embed-text-v1.5", + public_training_code="https://github.com/nomic-ai/contrastors/blob/5f7b461e5a13b5636692d1c9f1141b27232fe966/src/contrastors/configs/train/contrastive_finetune.yaml", + training_datasets=nomic_training_data, + public_training_data=None, +) + +nomic_embed_v1_ablated = ModelMeta( + loader=partial( + NomicWrapper, + trust_remote_code=True, + model_name="nomic-ai/nomic-embed-text-v1-ablated", + revision="7d948905c5d5d3874fa55a925d68e49dbf411e5f", + model_prompts=model_prompts, + ), + name="nomic-ai/nomic-embed-text-v1-ablated", + languages=["eng-Latn"], + open_weights=True, + revision="7d948905c5d5d3874fa55a925d68e49dbf411e5f", + release_date="2024-01-15", # first commit + n_parameters=None, + max_tokens=8192, + embed_dim=768, + license="apache-2.0", + reference="https://huggingface.co/nomic-ai/nomic-embed-text-v1-ablated", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + adapted_from=None, + superseded_by=None, + public_training_code="https://github.com/nomic-ai/contrastors/blob/5f7b461e5a13b5636692d1c9f1141b27232fe966/src/contrastors/configs/train/contrastive_finetune.yaml", + training_datasets=nomic_training_data, + public_training_data=None, +) + + +nomic_embed_v1_unsupervised = ModelMeta( + loader=partial( + NomicWrapper, + trust_remote_code=True, + model_name="nomic-ai/nomic-embed-text-v1-unsupervised", + revision="b53d557b15ae63852847c222d336c1609eced93c", + model_prompts=model_prompts, + ), + name="nomic-ai/nomic-embed-text-v1-unsupervised", + languages=["eng-Latn"], + open_weights=True, + revision="b53d557b15ae63852847c222d336c1609eced93c", + release_date="2024-01-15", # first commit + n_parameters=None, + max_tokens=8192, + embed_dim=768, + license="apache-2.0", + reference="https://huggingface.co/nomic-ai/nomic-embed-text-v1-unsupervised", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + adapted_from=None, + superseded_by=None, + public_training_code="https://github.com/nomic-ai/contrastors/blob/5f7b461e5a13b5636692d1c9f1141b27232fe966/src/contrastors/configs/train/contrastive_finetune.yaml", + training_datasets=nomic_training_data, + public_training_data=None, +) + +nomic_modern_bert_embed = ModelMeta( + loader=partial( + NomicWrapper, + model_name="nomic-ai/modernbert-embed-base", + revision="5960f1566fb7cb1adf1eb6e816639cf4646d9b12", + model_prompts=model_prompts, + model_kwargs={ + "torch_dtype": torch.float16, + }, + ), + name="nomic-ai/modernbert-embed-base", + languages=["eng-Latn"], + open_weights=True, + revision="5960f1566fb7cb1adf1eb6e816639cf4646d9b12", + release_date="2024-12-29", + n_parameters=149_000_000, + max_tokens=8192, + embed_dim=768, + license="apache-2.0", + reference="https://huggingface.co/nomic-ai/modernbert-embed-base", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + adapted_from="answerdotai/ModernBERT-base", + public_training_code="https://github.com/nomic-ai/contrastors/blob/5f7b461e5a13b5636692d1c9f1141b27232fe966/src/contrastors/configs/train/contrastive_pretrain_modernbert.yaml", + # https://github.com/nomic-ai/contrastors/blob/5f7b461e5a13b5636692d1c9f1141b27232fe966/src/contrastors/configs/train/contrastive_finetune_modernnomic.yaml + superseded_by=None, + training_datasets=nomic_training_data, + public_training_data=None, ) diff --git a/mteb/models/nomic_models_vision.py b/mteb/models/nomic_models_vision.py index 098e0d8502..126743a97d 100644 --- a/mteb/models/nomic_models_vision.py +++ b/mteb/models/nomic_models_vision.py @@ -168,10 +168,21 @@ def get_fused_embeddings( ), name="nomic-ai/nomic-embed-vision-v1.5", languages=["eng_Latn"], - open_source=True, revision="af2246fffdab78d8458418480e4886a8e48b70a7", release_date="2024-06-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) if __name__ == "__main__": diff --git a/mteb/models/nvidia_models.py b/mteb/models/nvidia_models.py new file mode 100644 index 0000000000..1997a85274 --- /dev/null +++ b/mteb/models/nvidia_models.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import logging +from collections.abc import Sequence +from functools import partial +from typing import Any + +import numpy as np +import torch +from sentence_transformers import CrossEncoder, SentenceTransformer + +from mteb.encoder_interface import PromptType +from mteb.model_meta import ModelMeta +from mteb.models.sentence_transformer_wrapper import SentenceTransformerWrapper + +logger = logging.getLogger(__name__) + + +def instruction_template( + instruction: str, prompt_type: PromptType | None = None +) -> str: + return f"Instruct: {instruction}\nQuery: " if instruction else "" + + +class NvEmbedWrapper(SentenceTransformerWrapper): + def __init__( + self, + model: str | SentenceTransformer | CrossEncoder, + revision: str | None = None, + model_prompts: dict[str, str] | None = None, + **kwargs, + ) -> None: + super().__init__(model, revision, model_prompts, **kwargs) + self.model.max_seq_length = 32768 + self.model.tokenizer.padding_side = "right" + logger.warning( + "Instructions are used in both query and docs, which may cause performance discrepancies from the original implementation." + ) + + def encode( + self, + sentences: Sequence[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + # Add eos token to each input example + sentences = [example + self.model.tokenizer.eos_token for example in sentences] + + instruction = "" + if prompt_type == PromptType.query: + instruction = self.get_instruction(task_name, prompt_type) + + prompt = instruction_template(instruction) + + if prompt: + logger.info(f"Using {prompt=} for task={task_name} {prompt_type=}") + else: + logger.info(f"No model prompts found for task={task_name} {prompt_type=}") + + logger.info(f"Encoding {len(sentences)} sentences.") + + embeddings = self.model.encode( + sentences, + prompt=prompt, + normalize_embeddings=True, + **kwargs, + ) + if isinstance(embeddings, torch.Tensor): + embeddings = embeddings.cpu().detach().float().numpy() + return embeddings + + +nvidia_training_datasets = { + # source: https://arxiv.org/pdf/2405.17428 + "ArguAna": ["train"], + "ArguAna-PL": ["train"], + "NanoArguAnaRetrieval": ["train"], + "HotpotQA": ["train"], + "HotpotQA-PL": ["train"], # translation not trained on + "HotpotQAHardNegatives": ["train"], + "MSMARCO": ["train"], + "MSMARCOHardNegatives": ["train"], + "NanoMSMARCORetrieval": ["train"], + "MSMARCO-PL": ["train"], # translation not trained on + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on + "FEVER": ["train"], + "FEVERHardNegatives": ["train"], + "NanoFEVERRetrieval": ["train"], + "FiQA2018": ["train"], + "FiQA2018-PL": ["train"], # translation not trained on + "STS12": ["train"], + "STS22": ["train"], + "AmazonReviewsClassification": ["train"], + "AmazonCounterfactualClassification": ["train"], + "Banking77Classification": ["train"], + "EmotionClassification": ["train"], + "ImdbClassification": ["train"], + "MTOPIntentClassification": ["train"], + "ToxicConversationsClassification": ["train"], + "TweetSentimentExtractionClassification": ["train"], + "ArxivClusteringP2P": ["train"], + "ArxivClusteringP2P.v2": ["train"], + "ArxivClusteringS2S": ["train"], + "ArxivClusteringS2S.v2": ["train"], + "BiorxivClusteringP2P": ["train"], + "BiorxivClusteringP2P.v2": ["train"], + "BiorxivClusteringS2S": ["train"], + "BiorxivClusteringS2S.v2": ["train"], + "MedrxivClusteringP2P": ["train"], + "MedrxivClusteringP2P.v2": ["train"], + "MedrxivClusteringS2S": ["train"], + "MedrxivClusteringS2S.v2": ["train"], + "TwentyNewsgroupsClustering": ["train"], + "TwentyNewsgroupsClustering.v2": ["train"], + "STSBenchmark": ["train"], + "STSBenchmarkMultilingualSTS": ["train"], # translated, not trained on +} +NV_embed_v2 = ModelMeta( + loader=partial( # type: ignore + NvEmbedWrapper, + model="nvidia/NV-Embed-v2", + trust_remote_code=True, + ), + name="nvidia/NV-Embed-v2", + languages=["eng_Latn"], + open_weights=True, + revision="7604d305b621f14095a1aa23d351674c2859553a", + release_date="2024-09-09", # initial commit of hf model. + n_parameters=7_850_000_000, + embed_dim=4096, + license="cc-by-nc-4.0", + max_tokens=32768, + reference="https://huggingface.co/nvidia/NV-Embed-v2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + training_datasets=nvidia_training_datasets, + public_training_code=None, + public_training_data=None, +) + +NV_embed_v1 = ModelMeta( + loader=partial( # type: ignore + NvEmbedWrapper, + model="nvidia/NV-Embed-v1", + trust_remote_code=True, + ), + name="nvidia/NV-Embed-v1", + languages=["eng_Latn"], + open_weights=True, + revision="570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c", + release_date="2024-09-13", # initial commit of hf model. + n_parameters=7_850_000_000, + embed_dim=4096, + license="cc-by-nc-4.0", + max_tokens=32768, + reference="https://huggingface.co/nvidia/NV-Embed-v1", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + training_datasets=nvidia_training_datasets, + public_training_code=None, + public_training_data=None, +) diff --git a/mteb/models/openai_models.py b/mteb/models/openai_models.py index d1eaf61644..079e7c9361 100644 --- a/mteb/models/openai_models.py +++ b/mteb/models/openai_models.py @@ -5,6 +5,7 @@ from typing import Any import numpy as np +import tqdm from mteb.model_meta import ModelMeta from mteb.requires_package import requires_package @@ -15,16 +16,37 @@ class OpenAIWrapper(Wrapper): - def __init__(self, model_name: str, embed_dim: int | None = None, **kwargs) -> None: + def __init__( + self, + model_name: str, + max_tokens: int, + tokenizer_name: str = "cl100k_base", # since all models use this tokenizer now + embed_dim: int | None = None, + **kwargs, + ) -> None: + """Wrapper for OpenAIs embedding API. + To handle documents larger than 8191 tokens, we truncate the document to the specified sequence length. + """ requires_package(self, "openai", "Openai text embedding") from openai import OpenAI + requires_package(self, "tiktoken", "Tiktoken package") + import tiktoken + self._client = OpenAI() self._model_name = model_name self._embed_dim = embed_dim + self._max_tokens = max_tokens + self._encoding = tiktoken.get_encoding(tokenizer_name) + + def truncate_text_tokens(self, text): + """Truncate a string to have `max_tokens` according to the given encoding.""" + truncated_sentence = self._encoding.encode(text)[: self._max_tokens] + return self._encoding.decode(truncated_sentence) def encode(self, sentences: list[str], **kwargs: Any) -> np.ndarray: requires_package(self, "openai", "Openai text embedding") + from openai import NotGiven if self._model_name == "text-embedding-ada-002" and self._embed_dim is not None: @@ -32,21 +54,59 @@ def encode(self, sentences: list[str], **kwargs: Any) -> np.ndarray: "Reducing embedding size available only for text-embedding-3-* models" ) - max_batch_size = 2048 + trimmed_sentences = [] + for sentence in sentences: + encoded_sentence = self._encoding.encode(sentence) + if len(encoded_sentence) > self._max_tokens: + truncated_sentence = self.truncate_text_tokens(sentence) + trimmed_sentences.append(truncated_sentence) + else: + trimmed_sentences.append(sentence) + + max_batch_size = kwargs.get("batch_size", 2048) sublists = [ - sentences[i : i + max_batch_size] - for i in range(0, len(sentences), max_batch_size) + trimmed_sentences[i : i + max_batch_size] + for i in range(0, len(trimmed_sentences), max_batch_size) ] + show_progress_bar = ( + False + if "show_progress_bar" not in kwargs + else kwargs.pop("show_progress_bar") + ) + all_embeddings = [] - for sublist in sublists: - response = self._client.embeddings.create( - input=sublist, - model=self._model_name, - encoding_format="float", - dimensions=self._embed_dim or NotGiven(), - ) + for sublist in tqdm.tqdm(sublists, leave=False, disable=not show_progress_bar): + try: + response = self._client.embeddings.create( + input=sublist, + model=self._model_name, + encoding_format="float", + dimensions=self._embed_dim or NotGiven(), + ) + except Exception as e: + # Sleep due to too many requests + logger.info("Sleeping for 10 seconds due to error", e) + import time + + time.sleep(10) + try: + response = self._client.embeddings.create( + input=sublist, + model=self._model_name, + encoding_format="float", + dimensions=self._embed_dim or NotGiven(), + ) + except Exception as e: + logger.info("Sleeping for 60 seconds due to error", e) + time.sleep(60) + response = self._client.embeddings.create( + input=sublist, + model=self._model_name, + encoding_format="float", + dimensions=self._embed_dim or NotGiven(), + ) all_embeddings.extend(self._to_numpy(response)) return np.array(all_embeddings) @@ -56,47 +116,74 @@ def _to_numpy(self, embedding_response) -> np.ndarray: text_embedding_3_small = ModelMeta( - name="text-embedding-3-small", - revision="1", + name="openai/text-embedding-3-small", + revision="2", release_date="2024-01-25", languages=None, # supported languages not specified - loader=partial(OpenAIWrapper, model_name="text-embedding-3-small"), + loader=partial( + OpenAIWrapper, + model_name="text-embedding-3-small", + tokenizer_name="cl100k_base", + max_tokens=8191, + ), max_tokens=8191, embed_dim=1536, open_weights=False, n_parameters=None, - memory_usage=None, license=None, reference="https://openai.com/index/new-embedding-models-and-api-updates/", similarity_fn_name="cosine", framework=["API"], - use_instuctions=False, + use_instructions=False, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, ) text_embedding_3_large = ModelMeta( - name="text-embedding-3-large", - revision="1", + name="openai/text-embedding-3-large", + revision="2", release_date="2024-01-25", languages=None, # supported languages not specified - loader=partial(OpenAIWrapper, model_name="text-embedding-3-large"), + loader=partial( + OpenAIWrapper, + model_name="text-embedding-3-large", + tokenizer_name="cl100k_base", + max_tokens=8191, + ), max_tokens=8191, embed_dim=3072, open_weights=False, + reference="https://openai.com/index/new-embedding-models-and-api-updates/", framework=["API"], - use_instuctions=False, + use_instructions=False, n_parameters=None, - memory_usage=None, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, + license=None, + similarity_fn_name=None, ) text_embedding_ada_002 = ModelMeta( - name="text-embedding-ada-002", - revision="1", + name="openai/text-embedding-ada-002", + revision="2", release_date="2022-12-15", languages=None, # supported languages not specified - loader=partial(OpenAIWrapper, model_name="text-embedding-ada-002"), + loader=partial( + OpenAIWrapper, + model_name="text-embedding-ada-002", + tokenizer_name="cl100k_base", + max_tokens=8191, + ), + reference="https://openai.com/index/new-and-improved-embedding-model/", max_tokens=8191, embed_dim=1536, open_weights=False, framework=["API"], - use_instuctions=False, + use_instructions=False, n_parameters=None, - memory_usage=None, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, + license=None, + similarity_fn_name=None, ) diff --git a/mteb/models/openclip_models.py b/mteb/models/openclip_models.py index 39469a7385..b39b2ca02e 100644 --- a/mteb/models/openclip_models.py +++ b/mteb/models/openclip_models.py @@ -157,10 +157,21 @@ def get_fused_embeddings( ), name="laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", languages=["eng_Latn"], - open_source=True, revision="84c9828e63dc9a9351d1fe637c346d4c1c4db341", release_date="2023-04-26", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) CLIP_ViT_B_32_DataComp_XL_s13B_b90K = ModelMeta( @@ -170,10 +181,21 @@ def get_fused_embeddings( ), name="laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", languages=["eng_Latn"], - open_source=True, revision="f0e2ffa09cbadab3db6a261ec1ec56407ce42912", release_date="2023-04-26", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) CLIP_ViT_B_16_DataComp_XL_s13B_b90K = ModelMeta( @@ -183,10 +205,21 @@ def get_fused_embeddings( ), name="laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", languages=["eng_Latn"], - open_source=True, revision="d110532e8d4ff91c574ee60a342323f28468b287", release_date="2023-04-26", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) CLIP_ViT_bigG_14_laion2B_39B_b160k = ModelMeta( @@ -196,10 +229,21 @@ def get_fused_embeddings( ), name="laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", languages=["eng_Latn"], - open_source=True, revision="bc7788f151930d91b58474715fdce5524ad9a189", release_date="2023-01-23", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) CLIP_ViT_g_14_laion2B_s34B_b88K = ModelMeta( @@ -209,10 +253,21 @@ def get_fused_embeddings( ), name="laion/CLIP-ViT-g-14-laion2B-s34B-b88K", languages=["eng_Latn"], - open_source=True, revision="15efd0f6ac0c40c0f9da7becca03c974d7012604", release_date="2023-03-06", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) CLIP_ViT_H_14_laion2B_s32B_b79K = ModelMeta( @@ -222,10 +277,21 @@ def get_fused_embeddings( ), name="laion/CLIP-ViT-H-14-laion2B-s32B-b79K", languages=["eng_Latn"], - open_source=True, revision="de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", release_date="2022-09-15", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) CLIP_ViT_L_14_laion2B_s32B_b82K = ModelMeta( @@ -235,10 +301,21 @@ def get_fused_embeddings( ), name="laion/CLIP-ViT-L-14-laion2B-s32B-b82K", languages=["eng_Latn"], - open_source=True, revision="1627032197142fbe2a7cfec626f4ced3ae60d07a", release_date="2022-09-15", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) CLIP_ViT_B_32_laion2B_s34B_b79K = ModelMeta( @@ -248,8 +325,19 @@ def get_fused_embeddings( ), name="laion/CLIP-ViT-B-32-laion2B-s34B-b79K", languages=["eng_Latn"], - open_source=True, revision="08f73555f1b2fb7c82058aebbd492887a94968ef", release_date="2022-09-15", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) diff --git a/mteb/models/overview.py b/mteb/models/overview.py index 8b94aafcd0..7535c8939a 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -2,21 +2,27 @@ import logging from collections.abc import Iterable +from functools import lru_cache from typing import Any +from huggingface_hub import ModelCard from sentence_transformers import SentenceTransformer +from mteb.abstasks.AbsTask import AbsTask from mteb.encoder_interface import Encoder from mteb.model_meta import ModelMeta from mteb.models import ( align_models, + arctic_models, bge_models, blip2_models, blip_models, bm25, + cde_models, clip_models, cohere_models, cohere_v, + colbert_models, dino_models, e5_instruct, e5_models, @@ -26,14 +32,26 @@ google_models, gritlm_models, gte_models, + ibm_granite_models, + inf_models, + jasper_models, jina_clip, + jina_models, + lens_models, + linq_models, llm2vec_models, + misc_models, moco_models, + model2vec_models, + moka_models, mxbai_models, + no_instruct_sentence_models, nomic_models, nomic_models_vision, + nvidia_models, openai_models, openclip_models, + piccolo_models, promptriever_models, repllama_models, rerankers_custom, @@ -42,6 +60,9 @@ salesforce_models, sentence_transformers_models, siglip_models, + stella_models, + text2vec_models, + uae_models, vista_models, vlm2vec_models, voyage_models, @@ -52,13 +73,16 @@ model_modules = [ align_models, + arctic_models, bge_models, - blip_models, blip2_models, + blip_models, bm25, clip_models, + cde_models, cohere_models, cohere_v, + colbert_models, dino_models, e5_instruct, e5_models, @@ -68,28 +92,42 @@ google_models, gritlm_models, gte_models, + gme_models, + ibm_granite_models, + inf_models, + jasper_models, + jina_models, jina_clip, + lens_models, + linq_models, llm2vec_models, + misc_models, + model2vec_models, + moka_models, moco_models, mxbai_models, + no_instruct_sentence_models, nomic_models, nomic_models_vision, - cohere_models, - clip_models, + nvidia_models, openai_models, openclip_models, + piccolo_models, + promptriever_models, + repllama_models, + rerankers_custom, + rerankers_monot5_based, ru_sentence_models, salesforce_models, sentence_transformers_models, siglip_models, vista_models, - voyage_models, - voyage_v, vlm2vec_models, - repllama_models, - promptriever_models, - rerankers_monot5_based, - rerankers_custom, + voyage_v, + stella_models, + text2vec_models, + uae_models, + voyage_models, ] MODEL_REGISTRY = {} @@ -102,9 +140,11 @@ def get_model_metas( model_names: Iterable[str] | None = None, languages: Iterable[str] | None = None, - open_source: bool | None = None, + open_weights: bool | None = None, frameworks: Iterable[str] | None = None, n_parameters_range: tuple[int | None, int | None] = (None, None), + use_instructions: bool | None = None, + zero_shot_on: list[AbsTask] | None = None, ) -> list[ModelMeta]: """Load all models' metadata that fit the specified criteria.""" res = [] @@ -119,11 +159,15 @@ def get_model_metas( languages <= set(model_meta.languages) ): continue - if (open_source is not None) and (model_meta.open_source != open_source): + if (open_weights is not None) and (model_meta.open_weights != open_weights): continue if (frameworks is not None) and not (frameworks <= set(model_meta.framework)): continue - upper, lower = n_parameters_range + if (use_instructions is not None) and ( + model_meta.use_instructions != use_instructions + ): + continue + lower, upper = n_parameters_range n_parameters = model_meta.n_parameters if upper is not None: if (n_parameters is None) or (n_parameters > upper): @@ -131,6 +175,9 @@ def get_model_metas( if lower is not None: if (n_parameters is None) or (n_parameters < lower): continue + if zero_shot_on is not None: + if not model_meta.is_zero_shot_on(zero_shot_on): + continue res.append(model_meta) return res @@ -176,21 +223,65 @@ def get_model_meta(model_name: str, revision: str | None = None) -> ModelMeta: return MODEL_REGISTRY[model_name] else: # assume it is a sentence-transformers model logger.info( - "Model not found in model registry, assuming it is a sentence-transformers model." + "Model not found in model registry, assuming it is on HF Hub model." ) logger.info( - f"Attempting to extract metadata by loading the model ({model_name}) using sentence-transformers." - ) - model = SentenceTransformer( - model_name, revision=revision, trust_remote_code=True + f"Attempting to extract metadata by loading the model ({model_name}) using HuggingFace." ) - meta = model_meta_from_sentence_transformers(model) - + meta = model_meta_from_hf_hub(model_name) meta.revision = revision meta.name = model_name return meta +@lru_cache +def model_meta_from_hf_hub(model_name: str) -> ModelMeta: + try: + card = ModelCard.load(model_name) + card_data = card.data.to_dict() + frameworks = ["PyTorch"] + if card_data.get("library_name", None) == "sentence-transformers": + frameworks.append("Sentence Transformers") + return ModelMeta( + name=model_name, + revision=card_data.get("base_model_revision", None), + # TODO + release_date=None, + # TODO: We need a mapping between conflicting language codes + languages=None, + license=card_data.get("license", None), + framework=frameworks, + training_datasets=card_data.get("datasets", None), + similarity_fn_name=None, + n_parameters=None, + max_tokens=None, + embed_dim=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + use_instructions=None, + ) + except Exception as e: + logger.warning(f"Failed to extract metadata from model: {e}.") + return ModelMeta( + name=model_name, + revision=None, + languages=None, + release_date=None, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=[], + ) + + def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMeta: try: name = ( @@ -210,6 +301,15 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe languages=languages, framework=["Sentence Transformers"], similarity_fn_name=model.similarity_fn_name, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + use_instructions=None, + training_datasets=None, ) except AttributeError as e: logger.warning( @@ -220,5 +320,16 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe revision=None, languages=None, release_date=None, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=[], ) return meta diff --git a/mteb/models/piccolo_models.py b/mteb/models/piccolo_models.py new file mode 100644 index 0000000000..d51487b8ba --- /dev/null +++ b/mteb/models/piccolo_models.py @@ -0,0 +1,48 @@ +"""Piccolo Chinese embedding models by SenseNova""" + +from __future__ import annotations + +from mteb.model_meta import ModelMeta + +piccolo_base_zh = ModelMeta( + name="sensenova/piccolo-base-zh", + languages=["zho_Hans"], + open_weights=True, + revision="47c0a63b8f667c3482e05b2fd45577bb19252196", + release_date="2023-09-04", # first commit + n_parameters=None, # can't see on model card + embed_dim=768, + license="mit", + max_tokens=512, + reference="https://huggingface.co/sensenova/piccolo-base-zh", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + public_training_code=None, + public_training_data=None, + training_datasets=None, # They don't specify +) + +piccolo_large_zh_v2 = ModelMeta( + name="sensenova/piccolo-large-zh-v2", + languages=["zho_Hans"], + open_weights=False, # They "temporarily" removed it in may last year + # "Due to certain internal company considerations" + revision="05948c1d889355936bdf9db7d30df57dd78d25a3", + release_date="2024-04-22", # first commit + n_parameters=None, # we don't know because they removed the model + embed_dim=1024, + license="not specified", + max_tokens=512, + reference="https://huggingface.co/sensenova/piccolo-large-zh-v2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + public_training_code=None, + public_training_data=None, + training_datasets=None, # They don't say +) diff --git a/mteb/models/promptriever_models.py b/mteb/models/promptriever_models.py index b3ed5ca876..287fd3ef91 100644 --- a/mteb/models/promptriever_models.py +++ b/mteb/models/promptriever_models.py @@ -56,14 +56,16 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="01c7f73d771dfac7d292323805ebc428287df4f9-30b14e3813c0fa45facfd01a594580c3fe5ecf23", # base-peft revision release_date="2024-09-15", n_parameters=7_000_000, - memory_usage=None, max_tokens=4096, embed_dim=4096, license="apache-2.0", + training_datasets={"samaya-ai/msmarco-w-instructions": ["train"]}, reference="https://huggingface.co/samaya-ai/promptriever-llama2-7b-v1", similarity_fn_name="cosine", framework=["PyTorch", "Tevatron"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, ) promptriever_llama3 = ModelMeta( @@ -78,16 +80,18 @@ def loader_inner(**kwargs: Any) -> Encoder: languages=["eng_Latn"], open_weights=True, revision="48d6d0fc4e02fb1269b36940650a1b7233035cbb-2ead22cfb1b0e0c519c371c63c2ab90ffc511b8a", # base-peft revision + training_datasets={"samaya-ai/msmarco-w-instructions": ["train"]}, release_date="2024-09-15", n_parameters=8_000_000, - memory_usage=None, max_tokens=8192, embed_dim=4096, license="apache-2.0", reference="https://huggingface.co/samaya-ai/promptriever-llama3.1-8b-v1", similarity_fn_name="cosine", framework=["PyTorch", "Tevatron"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, ) @@ -105,14 +109,16 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="5206a32e0bd3067aef1ce90f5528ade7d866253f-8b677258615625122c2eb7329292b8c402612c21", # base-peft revision release_date="2024-09-15", n_parameters=8_000_000, - memory_usage=None, max_tokens=8192, embed_dim=4096, + training_datasets={"samaya-ai/msmarco-w-instructions": ["train"]}, license="apache-2.0", reference="https://huggingface.co/samaya-ai/promptriever-llama3.1-8b-instruct-v1", similarity_fn_name="cosine", framework=["PyTorch", "Tevatron"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, ) promptriever_mistral_v1 = ModelMeta( @@ -129,12 +135,14 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="7231864981174d9bee8c7687c24c8344414eae6b-876d63e49b6115ecb6839893a56298fadee7e8f5", # base-peft revision release_date="2024-09-15", n_parameters=7_000_000, - memory_usage=None, + training_datasets={"samaya-ai/msmarco-w-instructions": ["train"]}, max_tokens=4096, embed_dim=4096, license="apache-2.0", reference="https://huggingface.co/samaya-ai/promptriever-mistral-v0.1-7b-v1", similarity_fn_name="cosine", framework=["PyTorch", "Tevatron"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, ) diff --git a/mteb/models/repllama_models.py b/mteb/models/repllama_models.py index 6f2f93f169..2c5ef6e446 100644 --- a/mteb/models/repllama_models.py +++ b/mteb/models/repllama_models.py @@ -11,10 +11,6 @@ from mteb.encoder_interface import Encoder, PromptType from mteb.model_meta import ModelMeta -from mteb.models.sentence_transformer_wrapper import ( - get_prompt_name, - validate_task_to_prompt_name, -) from .wrapper import Wrapper @@ -54,7 +50,7 @@ def __init__( self.model.config.max_length = 512 self.tokenizer.model_max_length = 512 self.model_prompts = ( - validate_task_to_prompt_name(model_prompts) if model_prompts else None + self.validate_task_to_prompt_name(model_prompts) if model_prompts else None ) def create_batch_dict(self, tokenizer, input_texts): @@ -89,7 +85,7 @@ def encode( ) -> np.ndarray: batch_size = 16 if "batch_size" not in kwargs else kwargs.pop("batch_size") all_embeddings = [] - prompt = get_prompt_name(self.model_prompts, task_name, prompt_type) + prompt = self.get_prompt_name(self.model_prompts, task_name, prompt_type) if prompt: sentences = [f"{prompt}{sentence}".strip() for sentence in sentences] for i in tqdm.tqdm(range(0, len(sentences), batch_size)): @@ -144,15 +140,17 @@ def loader_inner(**kwargs: Any) -> Encoder: open_weights=True, revision="01c7f73d771dfac7d292323805ebc428287df4f9-6097554dfe6e7d93e92f55010b678bcca1e233a8", # base-peft revision release_date="2023-10-11", + training_datasets={"Tevatron/msmarco-passage-aug": ["train"]}, n_parameters=7_000_000, - memory_usage=None, max_tokens=4096, embed_dim=4096, license="apache-2.0", reference="https://huggingface.co/samaya-ai/castorini/repllama-v1-7b-lora-passage", similarity_fn_name="cosine", framework=["PyTorch", "Tevatron"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, ) @@ -171,12 +169,14 @@ def loader_inner(**kwargs: Any) -> Encoder: revision="01c7f73d771dfac7d292323805ebc428287df4f9-ad5c1d0938a1e02954bcafb4d811ba2f34052e71", # base-peft revision release_date="2024-09-15", n_parameters=7_000_000, - memory_usage=None, max_tokens=4096, embed_dim=4096, license="apache-2.0", reference="https://huggingface.co/samaya-ai/RepLLaMA-reproduced", similarity_fn_name="cosine", framework=["PyTorch", "Tevatron"], - use_instuctions=True, + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=None, ) diff --git a/mteb/models/rerankers_custom.py b/mteb/models/rerankers_custom.py index ded6b0be5a..0e2c8d8f73 100644 --- a/mteb/models/rerankers_custom.py +++ b/mteb/models/rerankers_custom.py @@ -11,6 +11,7 @@ from mteb.encoder_interface import Encoder from mteb.evaluation.evaluators.RetrievalEvaluator import DenseRetrievalExactSearch from mteb.model_meta import ModelMeta +from mteb.models.bge_models import bge_m3_training_data logger = logging.getLogger(__name__) @@ -22,6 +23,7 @@ def __init__( batch_size: int = 4, fp_options: bool = None, silent: bool = False, + **kwargs, ): self.model_name_or_path = model_name_or_path self.batch_size = batch_size @@ -34,7 +36,7 @@ def __init__( self.fp_options = torch.float32 elif self.fp_options == "bfloat16": self.fp_options = torch.bfloat16 - print(f"Using fp_options of {self.fp_options}") + logger.info(f"Using fp_options of {self.fp_options}") self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.silent = silent self.first_print = True # for debugging @@ -70,7 +72,12 @@ def __init__( @torch.inference_mode() def predict(self, input_to_rerank, **kwargs): - queries, passages, instructions = list(zip(*input_to_rerank)) + inputs = list(zip(*input_to_rerank)) + if len(input_to_rerank[0]) == 2: + queries, passages = inputs + instructions = None + else: + queries, passages, instructions = inputs if instructions is not None and instructions[0] is not None: assert len(instructions) == len(queries) queries = [f"{q} {i}".strip() for i, q in zip(instructions, queries)] @@ -112,7 +119,13 @@ def __init__( @torch.inference_mode() def predict(self, input_to_rerank, **kwargs): - queries, passages, instructions = list(zip(*input_to_rerank)) + inputs = list(zip(*input_to_rerank)) + if len(input_to_rerank[0]) == 2: + queries, passages = inputs + instructions = None + else: + queries, passages, instructions = inputs + if instructions is not None and instructions[0] is not None: queries = [f"{q} {i}".strip() for i, q in zip(instructions, queries)] @@ -152,7 +165,13 @@ def __init__( ) def predict(self, input_to_rerank, **kwargs): - queries, passages, instructions = list(zip(*input_to_rerank)) + inputs = list(zip(*input_to_rerank)) + if len(input_to_rerank[0]) == 2: + queries, passages = inputs + instructions = None + else: + queries, passages, instructions = inputs + if instructions is not None and instructions[0] is not None: queries = [f"{q} {i}".strip() for i, q in zip(instructions, queries)] @@ -175,40 +194,60 @@ def loader_inner(**kwargs: Any) -> Encoder: monobert_large = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=MonoBERTReranker, model_name_or_path="castorini/monobert-large-msmarco", - fp_options="float1616", + fp_options="float16", ), name="castorini/monobert-large-msmarco", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="0a97706f3827389da43b83348d5d18c9d53876fa", release_date="2020-05-28", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=["Sentence Transformers", "PyTorch"], ) # languages unclear: https://huggingface.co/jinaai/jina-reranker-v2-base-multilingual/discussions/28 jina_reranker_multilingual = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=JinaReranker, model_name_or_path="jinaai/jina-reranker-v2-base-multilingual", - fp_options="float1616", + fp_options="float16", ), name="jinaai/jina-reranker-v2-base-multilingual", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="126747772a932960028d9f4dc93bd5d9c4869be4", release_date="2024-09-26", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=["Sentence Transformers", "PyTorch"], ) bge_reranker_v2_m3 = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=BGEReranker, model_name_or_path="BAAI/bge-reranker-v2-m3", - fp_options="float1616", + fp_options="float16", ), name="BAAI/bge-reranker-v2-m3", languages=[ @@ -245,7 +284,17 @@ def loader_inner(**kwargs: Any) -> Encoder: "vie_Latn", "zho_Hant", ], - open_source=True, + open_weights=True, revision="953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e", release_date="2024-06-24", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=bge_m3_training_data, + framework=["Sentence Transformers", "PyTorch"], ) diff --git a/mteb/models/rerankers_monot5_based.py b/mteb/models/rerankers_monot5_based.py index cd54bbd5cf..c53b364000 100644 --- a/mteb/models/rerankers_monot5_based.py +++ b/mteb/models/rerankers_monot5_based.py @@ -4,11 +4,7 @@ from functools import partial import torch -from transformers import ( - AutoModelForCausalLM, - AutoModelForSeq2SeqLM, - AutoTokenizer, -) +from transformers import AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer from mteb.model_meta import ModelMeta from mteb.models.rerankers_custom import RerankerWrapper, _loader @@ -94,8 +90,10 @@ def get_prediction_tokens( token_true_id = tokenizer.get_vocab()[token_true] return token_false_id, token_true_id else: - raise Exception(f"We don't know the indexes for the non-relevant/relevant tokens for\ - the checkpoint {model_name_or_path} and you did not provide any.") + raise Exception( + f"We don't know the indexes for the non-relevant/relevant tokens for\ + the checkpoint {model_name_or_path} and you did not provide any." + ) else: token_false_id = tokenizer.get_vocab()[token_false] token_true_id = tokenizer.get_vocab()[token_true] @@ -103,7 +101,12 @@ def get_prediction_tokens( @torch.inference_mode() def predict(self, input_to_rerank, **kwargs): - queries, passages, instructions = list(zip(*input_to_rerank)) + inputs = list(zip(*input_to_rerank)) + if len(input_to_rerank[0]) == 2: + queries, passages = inputs + instructions = None + else: + queries, passages, instructions = inputs if instructions is not None and instructions[0] is not None: queries = [f"{q} {i}".strip() for i, q in zip(instructions, queries)] @@ -192,7 +195,13 @@ def __init__( @torch.inference_mode() def predict(self, input_to_rerank, **kwargs): - queries, passages, instructions = list(zip(*input_to_rerank)) + inputs = list(zip(*input_to_rerank)) + if len(input_to_rerank[0]) == 2: + queries, passages = inputs + instructions = None + else: + queries, passages, instructions = inputs + if instructions is not None and instructions[0] is not None: # logger.info(f"Adding instructions to LLAMA queries") queries = [ @@ -276,7 +285,7 @@ def get_prediction_tokens(self, *args, **kwargs): monot5_small = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=MonoT5Reranker, model_name_or_path="castorini/monot5-small-msmarco-10k", @@ -284,13 +293,23 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="castorini/monot5-small-msmarco-10k", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="77f8e3f7b1eb1afe353aa21a7c3a2fc8feca702e", release_date="2022-03-28", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=["PyTorch"], ) monot5_base = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=MonoT5Reranker, model_name_or_path="castorini/monot5-base-msmarco-10k", @@ -298,13 +317,23 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="castorini/monot5-base-msmarco-10k", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="f15657ab3d2a5dd0b9a30c8c0b6a0a73c9cb5884", release_date="2022-03-28", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=["PyTorch"], ) monot5_large = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=MonoT5Reranker, model_name_or_path="castorini/monot5-large-msmarco-10k", @@ -312,13 +341,23 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="castorini/monot5-large-msmarco-10k", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="48cfad1d8dd587670393f27ee8ec41fde63e3d98", release_date="2022-03-28", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=["PyTorch"], ) monot5_3b = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=MonoT5Reranker, model_name_or_path="castorini/monot5-3b-msmarco-10k", @@ -326,13 +365,23 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="castorini/monot5-3b-msmarco-10k", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="bc0c419a438c81f592f878ce32430a1823f5db6c", release_date="2022-03-28", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=["PyTorch"], ) flant5_base = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=FLANT5Reranker, model_name_or_path="google/flan-t5-base", @@ -340,13 +389,34 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="google/flan-t5-base", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="7bcac572ce56db69c1ea7c8af255c5d7c9672fc2", release_date="2022-10-21", + training_datasets={ + "svakulenk0/qrecc": ["train"], + "taskmaster2": ["train"], + "djaym7/wiki_dialog": ["train"], + "deepmind/code_contests": ["train"], + "lambada": ["train"], + "gsm8k": ["train"], + "aqua_rat": ["train"], + "esnli": ["train"], + "quasc": ["train"], + "qed": ["train"], + }, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + framework=["PyTorch"], ) flant5_large = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=FLANT5Reranker, model_name_or_path="google/flan-t5-large", @@ -354,13 +424,34 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="google/flan-t5-large", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="0613663d0d48ea86ba8cb3d7a44f0f65dc596a2a", release_date="2022-10-21", + training_datasets={ + "svakulenk0/qrecc": ["train"], + "taskmaster2": ["train"], + "djaym7/wiki_dialog": ["train"], + "deepmind/code_contests": ["train"], + "lambada": ["train"], + "gsm8k": ["train"], + "aqua_rat": ["train"], + "esnli": ["train"], + "quasc": ["train"], + "qed": ["train"], + }, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + framework=["PyTorch"], ) flant5_xl = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=FLANT5Reranker, model_name_or_path="google/flan-t5-xl", @@ -368,13 +459,34 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="google/flan-t5-xl", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="7d6315df2c2fb742f0f5b556879d730926ca9001", release_date="2022-10-21", + training_datasets={ + "svakulenk0/qrecc": ["train"], + "taskmaster2": ["train"], + "djaym7/wiki_dialog": ["train"], + "deepmind/code_contests": ["train"], + "lambada": ["train"], + "gsm8k": ["train"], + "aqua_rat": ["train"], + "esnli": ["train"], + "quasc": ["train"], + "qed": ["train"], + }, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + framework=["PyTorch"], ) flant5_xxl = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=FLANT5Reranker, model_name_or_path="google/flan-t5-xxl", @@ -382,14 +494,35 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="google/flan-t5-xxl", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="ae7c9136adc7555eeccc78cdd960dfd60fb346ce", release_date="2022-10-21", + training_datasets={ + "svakulenk0/qrecc": ["train"], + "taskmaster2": ["train"], + "djaym7/wiki_dialog": ["train"], + "deepmind/code_contests": ["train"], + "lambada": ["train"], + "gsm8k": ["train"], + "aqua_rat": ["train"], + "esnli": ["train"], + "quasc": ["train"], + "qed": ["train"], + }, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + framework=["PyTorch"], ) llama2_7b = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=LlamaReranker, model_name_or_path="meta-llama/Llama-2-7b-hf", @@ -397,13 +530,23 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="meta-llama/Llama-2-7b-hf", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="01c7f73d771dfac7d292323805ebc428287df4f9", release_date="2023-07-18", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=["PyTorch"], ) llama2_7b_chat = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=LlamaReranker, model_name_or_path="meta-llama/Llama-2-7b-chat-hf", @@ -411,13 +554,23 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="meta-llama/Llama-2-7b-chat-hf", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="f5db02db724555f92da89c216ac04704f23d4590", release_date="2023-07-18", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=["PyTorch"], ) mistral_7b = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=MistralReranker, model_name_or_path="mistralai/Mistral-7B-Instruct-v0.2", @@ -425,13 +578,23 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="mistralai/Mistral-7B-Instruct-v0.2", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="3ad372fc79158a2148299e3318516c786aeded6c", release_date="2023-12-11", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=["PyTorch"], ) followir_7b = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=FollowIRReranker, model_name_or_path="jhu-clsp/FollowIR-7B", @@ -439,9 +602,19 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="jhu-clsp/FollowIR-7B", languages=["eng_Latn"], - open_source=True, + open_weights=True, revision="4d25d437e38b510c01852070c0731e8f6e1875d1", release_date="2024-04-29", + training_datasets={"jhu-clsp/FollowIR-train": ["train"]}, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + framework=["PyTorch"], ) @@ -550,7 +723,7 @@ def get_prediction_tokens(self, *args, **kwargs): ] mt5_base_mmarco_v2 = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=MonoT5Reranker, model_name_or_path="unicamp-dl/mt5-base-mmarco-v2", @@ -558,13 +731,23 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="unicamp-dl/mt5-base-mmarco-v2", languages=mt5_languages, - open_source=True, + open_weights=True, revision="cc0a949b9f21efcaba45c8cabb998ad02ce8d4e7", release_date="2022-01-05", + training_datasets={"msmarco": ["train"]}, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + framework=["PyTorch"], ) mt5_13b_mmarco_100k = ModelMeta( - loader=partial( + loader=partial( # type: ignore _loader, wrapper=MonoT5Reranker, model_name_or_path="unicamp-dl/mt5-13b-mmarco-100k", @@ -572,7 +755,17 @@ def get_prediction_tokens(self, *args, **kwargs): ), name="unicamp-dl/mt5-13b-mmarco-100k", languages=mt5_languages, - open_source=True, + open_weights=True, revision="e1a4317e102a525ea9e16745ad21394a4f1bffbc", release_date="2022-11-04", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=["PyTorch"], ) diff --git a/mteb/models/ru_sentence_models.py b/mteb/models/ru_sentence_models.py index 1328005a33..a91b6e7286 100644 --- a/mteb/models/ru_sentence_models.py +++ b/mteb/models/ru_sentence_models.py @@ -6,38 +6,53 @@ from mteb.model_meta import ModelMeta, sentence_transformers_loader -rubert_tiny2 = ModelMeta( - name="cointegrated/rubert-tiny2", +from .bge_models import bge_m3_training_data + +rubert_tiny = ModelMeta( + name="cointegrated/rubert-tiny", languages=["rus_Cyrl"], open_weights=True, - revision="dad72b8f77c5eef6995dd3e4691b758ba56b90c3", - release_date="2021-10-28", + revision="5441c5ea8026d4f6d7505ec004845409f1259fb1", + release_date="2021-05-24", n_parameters=29_400_000, - memory_usage=None, embed_dim=312, license="mit", max_tokens=2048, - reference="https://huggingface.co/cointegrated/rubert-tiny2", + reference="https://huggingface.co/cointegrated/rubert-tiny", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + public_training_code="https://gist.github.com/avidale/7bc6350f26196918bf339c01261f5c60", + training_datasets={ + # [Yandex Translate corpus](https://translate.yandex.ru/corpus), [OPUS-100](https://huggingface.co/datasets/opus100) + "Tatoeba": ["train"], + }, + adapted_from="google-bert/bert-base-multilingual-cased", + public_training_data=None, ) -rubert_tiny = ModelMeta( - name="cointegrated/rubert-tiny", +rubert_tiny2 = ModelMeta( + name="cointegrated/rubert-tiny2", languages=["rus_Cyrl"], open_weights=True, - revision="5441c5ea8026d4f6d7505ec004845409f1259fb1", - release_date="2021-05-24", + revision="dad72b8f77c5eef6995dd3e4691b758ba56b90c3", + release_date="2021-10-28", n_parameters=29_400_000, - memory_usage=None, embed_dim=312, license="mit", max_tokens=2048, - reference="https://huggingface.co/cointegrated/rubert-tiny", + reference="https://huggingface.co/cointegrated/rubert-tiny2", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + public_training_code="https://colab.research.google.com/drive/1mSWfIQ6PIlteLVZ9DKKpcorycgLIKZLf?usp=sharing", + training_datasets={ + # https://huggingface.co/datasets/cointegrated/ru-paraphrase-NMT-Leipzig + # Wikipedia https://huggingface.co/datasets/Madjogger/JamSpell_dataset + # https://huggingface.co/datasets/imvladikon/leipzig_corpora_collection + }, + adapted_from="cointegrated/rubert-tiny", + public_training_data=None, ) sbert_large_nlu_ru = ModelMeta( @@ -47,14 +62,16 @@ revision="af977d5dfa46a3635e29bf0ef383f2df2a08d47a", release_date="2020-11-20", n_parameters=427_000_000, - memory_usage=None, embed_dim=1024, license="mit", max_tokens=512, # best guess reference="https://huggingface.co/ai-forever/sbert_large_nlu_ru", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + public_training_code=None, + public_training_data=None, + training_datasets=None, ) sbert_large_mt_nlu_ru = ModelMeta( @@ -64,22 +81,27 @@ revision="05300876c2b83f46d3ddd422a7f17e45cf633bb0", release_date="2021-05-18", n_parameters=427_000_000, - memory_usage=None, embed_dim=1024, license="Not specified", max_tokens=512, # best guess reference="https://huggingface.co/ai-forever/sbert_large_mt_nlu_ru", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + public_training_code=None, + public_training_data=None, + training_datasets={ + # SNLI, MNLI + # https://github.com/brmson/dataset-sts + }, ) user_base_ru = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="deepvk/USER-base", revision="436a489a2087d61aa670b3496a9915f84e46c861", - prompts={"query": "query: ", "passage": "passage: "}, + model_prompts={"query": "query: ", "passage": "passage: "}, ), name="deepvk/USER-base", languages=["rus_Cyrl"], @@ -87,14 +109,94 @@ revision="436a489a2087d61aa670b3496a9915f84e46c861", release_date="2024-06-10", n_parameters=427_000_000, - memory_usage=None, + embed_dim=768, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/deepvk/USER-base", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + adapted_from="https://huggingface.co/deepvk/deberta-v1-base", + use_instructions=True, + training_datasets={ + "BibleNLPBitextMining": ["train"], + # https://github.com/unicamp-dl/mMARCO + # deepvk/ru-HNP + # deepvk/ru-WANLI + # MedNLI + # RCB + "TERRa": ["train"], + # Tapaco + # Opus100 + # BiblePar + # RudetoxifierDataDetox + # RuParadetox + "MIRACL": ["train"], + # MLDR + # Lenta + "MLSUMClusteringP2P": ["train"], + "MLSUMClusteringP2P.v2": ["train"], + "MLSUMClusteringS2S": ["train"], + "MLSUMClusteringS2S.v2": ["train"], + "MrTidyRetrieval": ["train"], + # "Panorama" + # PravoIsrael + # xlsum + # Fialka-v1 + # RussianKeywords + # Gazeta + # Gsm8k-ru + # DSumRu + # SummDialogNews + }, + public_training_code=None, + public_training_data=None, +) + +user_bge_m3 = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="deepvk/USER-bge-m3", + revision="0cc6cfe48e260fb0474c753087a69369e88709ae", + ), + name="deepvk/USER-bge-m3", + languages=["rus_Cyrl"], + open_weights=True, + revision="0cc6cfe48e260fb0474c753087a69369e88709ae", + release_date="2024-07-05", + n_parameters=359_026_688, embed_dim=1024, - license="Not specified", - max_tokens=512, # best guess - reference="https://huggingface.co/ai-forever/sbert_large_mt_nlu_ru", + license="apache-2.0", + max_tokens=8194, + reference="https://huggingface.co/deepvk/USER-base", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + adapted_from="https://huggingface.co/BAAI/bge-m3", + use_instructions=False, + training_datasets={ + "BibleNLPBitextMining": ["train"], + "MLSUMClusteringP2P": ["train"], + "MLSUMClusteringP2P.v2": ["train"], + "MLSUMClusteringS2S": ["train"], + "MLSUMClusteringS2S.v2": ["train"], + **bge_m3_training_data, + # not MTEB: + # "deepvk/ru-HNP": ["train"], + # "deepvk/ru-WANLI": ["train"], + # "Shitao/bge-m3-data": ["train"], + # "RussianNLP/russian_super_glue": ["train"], + # "reciTAL/mlsum": ["train"], + # "Helsinki-NLP/opus-100": ["train"], + # "Helsinki-NLP/bible_para": ["train"], + # "d0rj/rudetoxifier_data_detox": ["train"], + # "s-nlp/ru_paradetox": ["train"], + # "Milana/russian_keywords": ["train"], + # "IlyaGusev/gazeta": ["train"], + # "d0rj/gsm8k-ru": ["train"], + # "bragovo/dsum_ru": ["train"], + # "CarlBrendt/Summ_Dialog_News": ["train"], + }, + public_training_code=None, + public_training_data=None, ) deberta_v1_ru = ModelMeta( @@ -104,14 +206,17 @@ revision="bdd30b0e19757e6940c92c7aff19e8fc0a60dff4", release_date="2023-02-07", n_parameters=124_000_000, - memory_usage=None, embed_dim=768, license="apache-2.0", max_tokens=512, reference="https://huggingface.co/deepvk/deberta-v1-base", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + # Wikipedia, Books, Twitter comments, Pikabu, Proza.ru, Film subtitles, News websites, and Social corpus + public_training_code=None, + public_training_data=None, + training_datasets=None, ) rubert_base_cased = ModelMeta( @@ -121,14 +226,16 @@ revision="4036cab694767a299f2b9e6492909664d9414229", release_date="2020-03-04", n_parameters=1280_000_000, - memory_usage=None, embed_dim=768, license="Not specified", - max_tokens=512, # best guess + max_tokens=512, reference="https://huggingface.co/DeepPavlov/rubert-base-cased", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + public_training_code=None, + public_training_data=None, + training_datasets=None, ) distilrubert_small_cased_conversational = ModelMeta( @@ -138,14 +245,16 @@ revision="e348066b4a7279b97138038299bddc6580a9169a", release_date="2022-06-28", n_parameters=107_000_000, - memory_usage=None, embed_dim=768, license="Not specified", max_tokens=512, reference="https://huggingface.co/DeepPavlov/distilrubert-small-cased-conversational", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + public_training_code=None, + public_training_data=None, + training_datasets=None, ) rubert_base_cased_sentence = ModelMeta( @@ -155,14 +264,19 @@ revision="78b5122d6365337dd4114281b0d08cd1edbb3bc8", release_date="2020-03-04", n_parameters=107_000_000, - memory_usage=None, embed_dim=768, license="Not specified", max_tokens=512, reference="https://huggingface.co/DeepPavlov/rubert-base-cased-sentence", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + public_training_code=None, + public_training_data=None, + training_datasets={ + # "SNLI": [], + "XNLI": ["dev"] + }, ) labse_en_ru = ModelMeta( @@ -172,14 +286,17 @@ revision="cf0714e606d4af551e14ad69a7929cd6b0da7f7e", release_date="2021-06-10", n_parameters=129_000_000, - memory_usage=None, embed_dim=768, license="Not specified", max_tokens=512, reference="https://huggingface.co/cointegrated/LaBSE-en-ru", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + public_training_code="https://colab.research.google.com/drive/1dnPRn0-ugj3vZgSpyCC9sgslM2SuSfHy?usp=sharing", + public_training_data=None, + training_datasets=None, + adapted_from="sentence-transformers/LaBSE", ) rubert_tiny_turbo = ModelMeta( @@ -189,14 +306,18 @@ revision="8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054", release_date="2024-06-21", n_parameters=129_000_000, - memory_usage=None, embed_dim=312, license="mit", max_tokens=512, reference="https://huggingface.co/sergeyzh/rubert-tiny-turbo", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + public_training_code=None, + public_training_data=None, + training_datasets=None, # source model in unknown + # Not MTEB: {"IlyaGusev/gazeta": ["train"], "zloelias/lenta-ru": ["train"]}, + adapted_from="cointegrated/rubert-tiny2", ) labse_ru_turbo = ModelMeta( @@ -206,23 +327,27 @@ revision="1940b046c6b5e125df11722b899130329d0a46da", release_date="2024-06-27", n_parameters=129_000_000, - memory_usage=None, embed_dim=312, license="mit", max_tokens=512, reference="https://huggingface.co/sergeyzh/LaBSE-ru-turbo", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + training_datasets=None, + # not MTEB: {"IlyaGusev/gazeta": ["train"], "zloelias/lenta-ru": ["train"]}, + public_training_code=None, + adapted_from="cointegrated/LaBSE-en-ru", + public_training_data=None, ) rosberta_ru_en = ModelMeta( - loader=partial( + loader=partial( # type: ignore sentence_transformers_loader, model_name="ai-forever/ru-en-RoSBERTa", revision="89fb1651989adbb1cfcfdedafd7d102951ad0555", - prompts={ + model_prompts={ "Classification": "classification: ", "Clustering": "clustering: ", "query": "search_query: ", @@ -234,4 +359,29 @@ open_weights=True, revision="89fb1651989adbb1cfcfdedafd7d102951ad0555", release_date="2024-07-29", + use_instructions=True, + n_parameters=404_000_000, + max_tokens=514, + embed_dim=1024, + license="mit", + similarity_fn_name="cosine", + adapted_from="ai-forever/ruRoberta-large", + training_datasets={ + # https://huggingface.co/ai-forever/ruRoberta-large + # https://huggingface.co/datasets/IlyaGusev/yandex_q_full + # https://huggingface.co/datasets/IlyaGusev/pikabu + # https://huggingface.co/datasets/IlyaGusev/ru_stackoverflow + # https://huggingface.co/datasets/IlyaGusev/habr + # https://huggingface.co/datasets/its5Q/habr_qna + # NewsCommentary + # MultiParaCrawl + "XNLI": [], + "XNLIV2": [], + "LanguageClassification": [], # XNLI + "MIRACLReranking": ["train"], + "MIRACLRetrieval": ["train"], + }, + public_training_data=None, + public_training_code=None, + framework=["Sentence Transformers", "PyTorch"], ) diff --git a/mteb/models/salesforce_models.py b/mteb/models/salesforce_models.py index fe70f597ae..235057a6f8 100644 --- a/mteb/models/salesforce_models.py +++ b/mteb/models/salesforce_models.py @@ -1,61 +1,42 @@ from __future__ import annotations -from collections.abc import Sequence from functools import partial -from typing import Any - -import numpy as np -import torch +from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta +from mteb.models.instruct_wrapper import instruct_wrapper -from ..encoder_interface import PromptType -from .instructions import task_to_instruction -from .wrapper import Wrapper - - -def sfr_instruction(instruction: str) -> str: - return f"Instruct: {instruction}\nQuery: " +from .e5_instruct import E5_MISTRAL_TRAINING_DATA -def sfr_loader(**kwargs): - try: - from gritlm import GritLM - except ImportError: - raise ImportError( - "Please install `pip install gritlm` to use SFR_Embedding_2_R." - ) +def instruction_template( + instruction: str, prompt_type: PromptType | None = None +) -> str: + return f"Instruct: {instruction}\nQuery: " if instruction else "" - class SFRWrapper(GritLM, Wrapper): - def encode( - self, - sentences: Sequence[str], - *args, - task_name: str, - prompt_type: PromptType | None = None, - **kwargs: Any, - ) -> np.ndarray: - if "instruction" in kwargs: - instruction = kwargs.pop("instruction", "") - else: - instruction = task_to_instruction( - task_name, prompt_type == PromptType.query - ) - if instruction: - kwargs["instruction"] = sfr_instruction(instruction) - return super().encode(*args, **kwargs) - - return SFRWrapper(**kwargs) +SFR_TRAINING_DATA = { # inherits from e5 + **E5_MISTRAL_TRAINING_DATA, + # From previously released blogpost which now have been taken down: + "FiQA2018": ["train"], + "FiQA2018-PL": ["train"], + "FEVER": ["train"], + "FEVERHardNegatives": ["train"], + "FEVER-PL": ["train"], # translation not trained on + "HotpotQA": ["train"], + "HotpotQAHardNegatives": ["train"], + "HotpotQA-PL": ["train"], # translation not trained on +} SFR_Embedding_2_R = ModelMeta( - loader=partial( - sfr_loader, + loader=partial( # type: ignore + instruct_wrapper, model_name_or_path="Salesforce/SFR-Embedding-2_R", + instruction_template=instruction_template, attn="cccc", pooling_method="lasttoken", mode="embedding", - torch_dtype=torch.bfloat16, + torch_dtype="auto", # The ST script does not normalize while the HF one does so unclear what to do # https://huggingface.co/Salesforce/SFR-Embedding-2_R normalized=True, @@ -66,12 +47,45 @@ def encode( revision="91762139d94ed4371a9fa31db5551272e0b83818", release_date="2024-06-14", # initial commit of hf model. n_parameters=7_110_000_000, - memory_usage=None, embed_dim=4096, license="cc-by-nc-4.0", max_tokens=32768, reference="https://huggingface.co/Salesforce/SFR-Embedding-2_R", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=True, + use_instructions=True, + adapted_from="intfloat/e5-mistral-7b-instruct", + public_training_code=None, + public_training_data=None, + training_datasets=SFR_TRAINING_DATA, +) + + +SFR_Embedding_Mistral = ModelMeta( + loader=partial( # type: ignore + instruct_wrapper, + model_name_or_path="Salesforce/SFR-Embedding-Mistral", + instruction_template=instruction_template, + attn="cccc", + pooling_method="lasttoken", + mode="embedding", + torch_dtype="auto", + normalized=True, + ), + name="Salesforce/SFR-Embedding-Mistral", + languages=["eng_Latn"], + open_weights=True, + revision="938c560d1c236aa563b2dbdf084f28ab28bccb11", + release_date="2024-01-24", # initial commit of hf model. + n_parameters=7_110_000_000, + embed_dim=4096, + license="cc-by-nc-4.0", + max_tokens=32768, + reference="https://huggingface.co/Salesforce/SFR-Embedding-Mistral", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code=None, + public_training_data=None, + training_datasets=SFR_TRAINING_DATA, ) diff --git a/mteb/models/sentence_transformer_wrapper.py b/mteb/models/sentence_transformer_wrapper.py index 809b7d50b2..4a7dbd8ffa 100644 --- a/mteb/models/sentence_transformer_wrapper.py +++ b/mteb/models/sentence_transformer_wrapper.py @@ -2,13 +2,12 @@ import logging from collections.abc import Sequence -from typing import Any, get_args +from typing import Any import numpy as np +import torch from sentence_transformers import CrossEncoder, SentenceTransformer -import mteb -from mteb.abstasks.TaskMetadata import TASK_TYPE from mteb.encoder_interface import PromptType from .wrapper import Wrapper @@ -36,13 +35,9 @@ def __init__( **kwargs: Additional arguments to pass to the SentenceTransformer model. """ if isinstance(model, str): - self.model = SentenceTransformer( - model, revision=revision, trust_remote_code=True, **kwargs - ) - self.device = self.model.device + self.model = SentenceTransformer(model, revision=revision, **kwargs) else: self.model = model - self.device = None if ( model_prompts is None @@ -50,17 +45,23 @@ def __init__( and len(self.model.prompts) > 0 ): try: - model_prompts = validate_task_to_prompt_name(self.model.prompts) - except ValueError: + model_prompts = self.validate_task_to_prompt_name(self.model.prompts) + except KeyError: model_prompts = None + logger.warning( + "Model prompts are not in the expected format. Ignoring them." + ) elif model_prompts is not None and hasattr(self.model, "prompts"): logger.info(f"Model prompts will be overwritten with {model_prompts}") self.model.prompts = model_prompts - self.model_prompts = validate_task_to_prompt_name(model_prompts) + self.model_prompts = self.validate_task_to_prompt_name(model_prompts) if isinstance(self.model, CrossEncoder): self.predict = self._predict + if hasattr(self.model, "similarity") and callable(self.model.similarity): + self.similarity = self.model.similarity + def encode( self, sentences: Sequence[str], @@ -91,10 +92,12 @@ def encode( """ prompt_name = None if self.model_prompts is not None: - prompt_name = get_prompt_name(self.model_prompts, task_name, prompt_type) + prompt_name = self.get_prompt_name( + self.model_prompts, task_name, prompt_type + ) if prompt_name: logger.info( - f"Using prompt_nane={prompt_name} for task={task_name} prompt_type={prompt_type}" + f"Using prompt_name={prompt_name} for task={task_name} prompt_type={prompt_type}" ) else: logger.info( @@ -107,6 +110,9 @@ def encode( prompt_name=prompt_name, **kwargs, ) + if isinstance(embeddings, torch.Tensor): + # sometimes in kwargs can be return_tensors=True + embeddings = embeddings.cpu().detach().float().numpy() return embeddings def _predict( @@ -119,75 +125,3 @@ def _predict( convert_to_numpy=True, **kwargs, ) - - -def get_prompt_name( - task_to_prompt: dict[str, str] | None, - task_name: str, - prompt_type: PromptType | None, -) -> str | None: - """A wrapper function around the model.encode method that handles the prompt_name argument and standardizes the output to a numpy array. - The order of priorities for prompt selection are: - 1. Composed prompt of task name + prompt type (query or passage) - 2. Specific task prompt - 3. Composed prompt of task type + prompt type (query or passage) - 4. Specific task type prompt - 5. Specific prompt type (query or passage) - - - Args: - task_to_prompt: The tasks names and their corresponding prompt_names - task_name: The task name to use for building the encoding prompt - prompt_type: The prompt type (e.g. "query" | "passage") to use for building the encoding prompt - """ - import mteb - - task = mteb.get_task(task_name=task_name) - task_type = task.metadata.type - prompt_type_value = prompt_type.value if prompt_type else None - - if ( - task_name - and prompt_type - and f"{task_name}-{prompt_type_value}" in task_to_prompt - ): - return f"{task_name}-{prompt_type_value}" - if task_name and task_name in task_to_prompt: - return task_name - if ( - task_type - and prompt_type - and f"{task_type}-{prompt_type_value}" in task_to_prompt - ): - return f"{task_type}-{prompt_type_value}" - if task_type and task_type in task_to_prompt: - return task_type - if prompt_type and prompt_type_value in task_to_prompt: - return prompt_type_value - logger.info( - "No combination of task name and prompt type was found in model prompts." - ) - return None - - -def validate_task_to_prompt_name( - task_to_prompt_name: dict[str, str] | None, -) -> dict[str, str] | None: - if task_to_prompt_name is None: - return task_to_prompt_name - task_types = get_args(TASK_TYPE) - prompt_types = [e.value for e in PromptType] - for task_name in task_to_prompt_name: - if "-" in task_name: - task_name, prompt_type = task_name.split("-") - if prompt_type not in prompt_types: - raise ValueError( - f"Prompt type {prompt_type} is not valid. Valid prompt types are {prompt_types}" - ) - if task_name not in task_types and task_name not in prompt_types: - task = mteb.get_task(task_name=task_name) - if not task: - raise ValueError( - f"Task name {task_name} is not valid. Valid task names are task types [{task_types}], prompt types [{prompt_types}] and task names" - ) - return task_to_prompt_name diff --git a/mteb/models/sentence_transformers_models.py b/mteb/models/sentence_transformers_models.py index 9a33e0f64f..eec65049d5 100644 --- a/mteb/models/sentence_transformers_models.py +++ b/mteb/models/sentence_transformers_models.py @@ -60,70 +60,226 @@ "zho_Hant", ] +sent_trf_training_dataset = { + # derived from datasheets + "MSMARCO": ["train"], + "MSMARCOHardNegatives": ["train"], + "NanoMSMARCORetrieval": ["train"], + "MSMARCO-PL": ["train"], # translation not trained on + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on + # not in MTEB + # "s2orc": ["train"], + # "flax-sentence-embeddings/stackexchange_xml": ["train"], + # "ms_marco": ["train"], + # "gooaq": ["train"], + # "yahoo_answers_topics": ["train"], + # "code_search_net": ["train"], + # "search_qa": ["train"], + # "eli5": ["train"], + # "snli": ["train"], + # "multi_nli": ["train"], + # "wikihow": ["train"], + # "natural_questions": ["train"], + # "trivia_qa": ["train"], + # "embedding-data/sentence-compression": ["train"], + # "embedding-data/flickr30k-captions": ["train"], + # "embedding-data/altlex": ["train"], + # "embedding-data/simple-wiki": ["train"], + # "embedding-data/QQP": ["train"], + # "embedding-data/SPECTER": ["train"], + # "embedding-data/PAQ_pairs": ["train"], + # "embedding-data/WikiAnswers": ["train"], +} + all_MiniLM_L6_v2 = ModelMeta( name="sentence-transformers/all-MiniLM-L6-v2", languages=["eng-Latn"], open_weights=True, - revision="8b3219a92973c328a8e22fadcfa821b5dc75636a", # can be any + revision="8b3219a92973c328a8e22fadcfa821b5dc75636a", release_date="2021-08-30", n_parameters=22_700_000, - memory_usage=None, embed_dim=384, license="apache-2.0", - max_tokens=512, - reference="https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + max_tokens=256, + reference="https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + training_datasets=sent_trf_training_dataset, + public_training_code=None, + public_training_data=None, +) + +all_MiniLM_L12_v2 = ModelMeta( + name="sentence-transformers/all-MiniLM-L12-v2", + languages=["eng-Latn"], + open_weights=True, + revision="364dd28d28dcd3359b537f3cf1f5348ba679da62", + release_date="2021-08-30", + n_parameters=33_400_000, + embed_dim=384, + license="apache-2.0", + max_tokens=256, + reference="https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + superseded_by=None, + adapted_from=None, + training_datasets=sent_trf_training_dataset, + public_training_code=None, + public_training_data=None, ) paraphrase_multilingual_MiniLM_L12_v2 = ModelMeta( name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", languages=paraphrase_langs, open_weights=True, - revision="bf3bf13ab40c3157080a7ab344c831b9ad18b5eb", # can be any + revision="bf3bf13ab40c3157080a7ab344c831b9ad18b5eb", release_date="2019-11-01", # release date of paper n_parameters=118_000_000, - memory_usage=None, embed_dim=768, license="apache-2.0", max_tokens=512, reference="https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + superseded_by=None, + adapted_from=None, + training_datasets=sent_trf_training_dataset, # assumed (probably some parallel as well) + public_training_code=None, + public_training_data=None, ) paraphrase_multilingual_mpnet_base_v2 = ModelMeta( name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2", languages=paraphrase_langs, open_weights=True, - revision="79f2382ceacceacdf38563d7c5d16b9ff8d725d6", # can be any + revision="79f2382ceacceacdf38563d7c5d16b9ff8d725d6", release_date="2019-11-01", # release date of paper n_parameters=278_000_000, - memory_usage=None, embed_dim=768, license="apache-2.0", max_tokens=512, reference="https://huggingface.co/sentence-transformers/paraphrase-multilingual-mpnet-base-v2", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + superseded_by=None, + adapted_from=None, + training_datasets=sent_trf_training_dataset, + # + https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/paraphrases/training.py + # which include (not in MTEB): + # "all-nli": all_nli_train_dataset, + # "sentence-compression": sentence_compression_train_dataset, + # "simple-wiki": simple_wiki_train_dataset, + # "altlex": altlex_train_dataset, + # "quora-duplicates": quora_train_dataset, + # "coco-captions": coco_train_dataset, + # "flickr30k-captions": flickr_train_dataset, + # "yahoo-answers": yahoo_answers_train_dataset, + # "stack-exchange": stack_exchange_train_dataset, + public_training_code=None, + public_training_data=None, ) labse = ModelMeta( name="sentence-transformers/LaBSE", languages=paraphrase_langs, open_weights=True, - revision="e34fab64a3011d2176c99545a93d5cbddc9a91b7", # can be any + revision="e34fab64a3011d2176c99545a93d5cbddc9a91b7", release_date="2019-11-01", # release date of paper n_parameters=471_000_000, - memory_usage=None, embed_dim=768, license="apache-2.0", max_tokens=512, reference="https://huggingface.co/sentence-transformers/LaBSE", similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch"], - use_instuctions=False, + use_instructions=False, + superseded_by=None, + adapted_from=None, + training_datasets=None, # scraped and mined webdata including CC, wiki, see section 3.1 https://aclanthology.org/2022.acl-long.62.pdf + public_training_code="https://www.kaggle.com/models/google/labse/tensorFlow2/labse/2?tfhub-redirect=true", + public_training_data=None, +) + +multi_qa_MiniLM_L6_cos_v1 = ModelMeta( + name="sentence-transformer/multi-qa-MiniLM-L6-cos-v1", + languages=["eng-Latn"], + open_weights=True, + revision="b207367332321f8e44f96e224ef15bc607f4dbf0", + release_date="2021-08-30", + n_parameters=22_700_000, + embed_dim=384, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/sentence-transformers/multi-qa-MiniLM-L6-cos-v1", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from="nreimers/MiniLM-L6-H384-uncased", + training_datasets=sent_trf_training_dataset, # assumed + public_training_code=None, + public_training_data=None, +) + +all_mpnet_base_v2 = ModelMeta( + name="sentence-transformers/all-mpnet-base-v2", + languages=["eng-Latn"], + open_weights=True, + revision="9a3225965996d404b775526de6dbfe85d3368642", + release_date="2021-08-30", + n_parameters=109_000_000, + embed_dim=768, + license="apache-2.0", + max_tokens=384, + reference="https://huggingface.co/sentence-transformers/all-mpnet-base-v2", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + training_datasets=sent_trf_training_dataset, + public_training_code=None, + public_training_data=None, +) + + +microllama_text_embedding = ModelMeta( + name="keeeeenw/MicroLlama-text-embedding", + languages=["eng-Latn"], + open_weights=True, + revision="98f70f14cdf12d7ea217ed2fd4e808b0195f1e7e", + release_date="2024-11-10", + n_parameters=272_000_000, + embed_dim=1024, + license="apache-2.0", + max_tokens=2048, + reference="https://huggingface.co/keeeeenw/MicroLlama-text-embedding", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + training_datasets={ + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on + # not in MTEB + # "sentence-transformers/all-nli": ["train"], + # "sentence-transformers/stsb": ["train"], + # "sentence-transformers/quora-duplicates": ["train"], + # "sentence-transformers/natural-questions": ["train"], + }, + public_training_code=None, + public_training_data=None, ) diff --git a/mteb/models/siglip_models.py b/mteb/models/siglip_models.py index 14ddb81002..739b5aa59a 100644 --- a/mteb/models/siglip_models.py +++ b/mteb/models/siglip_models.py @@ -162,10 +162,21 @@ def get_fused_embeddings( ), name="google/siglip-so400m-patch14-224", languages=["eng_Latn"], - open_source=True, revision="d04cf29fca7b6374f74d8bea1969314492266b5e", release_date="2024-01-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) siglip_so400m_patch14_384 = ModelMeta( @@ -175,10 +186,21 @@ def get_fused_embeddings( ), name="google/siglip-so400m-patch14-384", languages=["eng_Latn"], - open_source=True, revision="9fdffc58afc957d1a03a25b10dba0329ab15c2a3", release_date="2024-01-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) siglip_so400m_patch16_256_i18n = ModelMeta( @@ -188,10 +210,21 @@ def get_fused_embeddings( ), name="google/siglip-so400m-patch16-256-i18n", languages=["eng_Latn"], - open_source=True, revision="365d321c0cfdea96bc28e3a29787a11a062681a1", release_date="2024-01-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) siglip_base_patch16_256_multilingual = ModelMeta( @@ -201,10 +234,21 @@ def get_fused_embeddings( ), name="google/siglip-base-patch16-256-multilingual", languages=["eng_Latn"], - open_source=True, revision="8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6", release_date="2024-01-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) siglip_base_patch16_256 = ModelMeta( @@ -214,10 +258,21 @@ def get_fused_embeddings( ), name="google/siglip-base-patch16-256", languages=["eng_Latn"], - open_source=True, revision="b078df89e446d623010d890864d4207fe6399f61", release_date="2024-01-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) siglip_base_patch16_512 = ModelMeta( @@ -227,10 +282,21 @@ def get_fused_embeddings( ), name="google/siglip-base-patch16-512", languages=["eng_Latn"], - open_source=True, revision="753a949581523b60257d93e18391e8c27f72eb22", release_date="2024-01-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) siglip_base_patch16_384 = ModelMeta( @@ -240,10 +306,21 @@ def get_fused_embeddings( ), name="google/siglip-base-patch16-384", languages=["eng_Latn"], - open_source=True, revision="41aec1c83b32e0a6fca20ad88ba058aa5b5ea394", release_date="2024-01-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) siglip_base_patch16_224 = ModelMeta( @@ -253,10 +330,21 @@ def get_fused_embeddings( ), name="google/siglip-base-patch16-224", languages=["eng_Latn"], - open_source=True, revision="7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed", release_date="2024-01-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) siglip_large_patch16_256 = ModelMeta( @@ -266,10 +354,21 @@ def get_fused_embeddings( ), name="google/siglip-large-patch16-256", languages=["eng_Latn"], - open_source=True, revision="d0da9f876e7d66b4e250cd2450c3ba2ce735e447", release_date="2024-01-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) siglip_large_patch16_384 = ModelMeta( @@ -279,10 +378,21 @@ def get_fused_embeddings( ), name="google/siglip-large-patch16-384", languages=["eng_Latn"], - open_source=True, revision="ce005573a40965dfd21fd937fbdeeebf2439fc35", release_date="2024-01-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) if __name__ == "__main__": diff --git a/mteb/models/stella_models.py b/mteb/models/stella_models.py new file mode 100644 index 0000000000..92d5db7c8a --- /dev/null +++ b/mteb/models/stella_models.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +from functools import partial + +from mteb.model_meta import ModelMeta +from mteb.models.instruct_wrapper import instruct_wrapper + +stella_en_400M = ModelMeta( + # https://huggingface.co/dunzhang/stella_en_400M_v5/discussions/21#671a6205ac1e2416090f2bf4 + loader=partial( # type: ignore + instruct_wrapper, + model_name_or_path="dunzhang/stella_en_400M_v5", + attn="cccc", + pooling_method="lasttoken", + mode="embedding", + torch_dtype="auto", + ), + name="dunzhang/stella_en_400M_v5", + languages=["eng_Latn"], + open_weights=True, + use_instructions=True, + revision="1bb50bc7bb726810eac2140e62155b88b0df198f", + release_date="2024-07-12", + n_parameters=435_000_000, + max_tokens=8192, + embed_dim=4096, + license="mit", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch", "GritLM"], + reference="https://huggingface.co/dunzhang/stella_en_400M_v5", + training_datasets=None, + # will be at https://github.com/NLPJCL/RAG-Retrieval + public_training_code=None, + public_training_data=None, +) + +stella_en_1_5b = ModelMeta( + loader=partial( # type: ignore + instruct_wrapper, + model_name_or_path="dunzhang/stella_en_1.5B_v5", + attn="cccc", + pooling_method="lasttoken", + mode="embedding", + torch_dtype="auto", + ), + name="dunzhang/stella_en_1.5B_v5", + languages=["eng_Latn"], + open_weights=True, + use_instructions=True, + revision="d03be74b361d4eb24f42a2fe5bd2e29917df4604", + release_date="2024-07-12", + n_parameters=1_540_000_000, + max_tokens=131072, + embed_dim=8960, + license="mit", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch", "GritLM"], + reference="https://huggingface.co/dunzhang/stella_en_1.5B_v5", + # will be at https://github.com/NLPJCL/RAG-Retrieval + training_datasets=None, + public_training_code=None, + public_training_data=None, +) + +stella_large_zh_v3_1792d = ModelMeta( + name="dunzhang/stella-large-zh-v3-1792d", + languages=["zho_Hans"], + open_weights=True, + revision="d5d39eb8cd11c80a63df53314e59997074469f09", + release_date="2024-02-17", + n_parameters=None, # can't see on model card + embed_dim=1792, + license="not specified", + max_tokens=512, + reference="https://huggingface.co/dunzhang/stella-large-zh-v3-1792d", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by="dunzhang/stella-mrl-large-zh-v3.5-1792d", + adapted_from=None, + public_training_code=None, + public_training_data=None, + training_datasets={ + # Not in MTEB: + # - infgrad/dialogue_rewrite_llm + # - infgrad/retrieval_data_llm + }, +) + +stella_base_zh_v3_1792d = ModelMeta( + name="infgrad/stella-base-zh-v3-1792d", + languages=["zho_Hans"], + open_weights=True, + revision="82254892a0fba125aa2abf3a4800d2dd12821343", + release_date="2024-02-17", + n_parameters=None, # can't see on model card + embed_dim=1792, + license="mit", + max_tokens=512, + reference="https://huggingface.co/infgrad/stella-base-zh-v3-1792d", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + public_training_code=None, + public_training_data=None, + training_datasets={ + # Not in MTEB: + # - infgrad/dialogue_rewrite_llm + # - infgrad/retrieval_data_llm + }, +) + + +stella_mrl_large_zh_v3_5_1792d = ModelMeta( + name="dunzhang/stella-mrl-large-zh-v3.5-1792d", + languages=["zho_Hans"], + open_weights=True, + revision="17bb1c32a93a8fc5f6fc9e91d5ea86da99983cfe", + release_date="2024-02-27", + n_parameters=326 * 1e6, + embed_dim=1792, + license="mit", + max_tokens=512, + reference="https://huggingface.co/dunzhang/stella-large-zh-v3-1792d", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from="dunzhang/stella-large-zh-v3-1792d", + public_training_code=None, + public_training_data=None, + training_datasets=None, # Not specified +) + +zpoint_large_embedding_zh = ModelMeta( + name="iampanda/zpoint_large_embedding_zh", + languages=["zho_Hans"], + open_weights=True, + revision="b1075144f440ab4409c05622c1179130ebd57d03", + release_date="2024-06-04", + n_parameters=326 * 1e6, + embed_dim=1792, + license="mit", + max_tokens=512, + reference="https://huggingface.co/iampanda/zpoint_large_embedding_zh", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from="dunzhang/stella-mrl-large-zh-v3.5-1792d", + public_training_code=None, + public_training_data=None, + training_datasets={ + # It's a bit unclear what they have trained on to be honest, because they don't list all + # And they also have some rather cryptic description of their training procedure, but at + # Least they disclose that they have trained on these: + "MIRACLRetrieval": ["train"], + "MIRACLReranking": ["train"], + "DuRetrieval": ["train"], + "T2Retrieval": ["train"], + "MultiLongDocRetrieval": ["train"], + # Not in MTEB: + # - Shitao/bge-reranker-data + # - FreedomIntelligence/Huatuo26M-Lite + }, +) diff --git a/mteb/models/text2vec_models.py b/mteb/models/text2vec_models.py new file mode 100644 index 0000000000..86a9bcca4f --- /dev/null +++ b/mteb/models/text2vec_models.py @@ -0,0 +1,100 @@ +"""Implementation of Text2Vec models""" + +from __future__ import annotations + +from mteb.model_meta import ModelMeta + +# I couldn't find the large model on HF for some reason +text2vec_base_chinese = ModelMeta( + name="shibing624/text2vec-base-chinese", + languages=["zho-Hans"], + open_weights=True, + revision="183bb99aa7af74355fb58d16edf8c13ae7c5433e", + release_date="2022-01-23", + n_parameters=102 * 1e6, + embed_dim=768, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/shibing624/text2vec-base-chinese", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + public_training_code=None, + public_training_data=None, # Couldn't find it + training_datasets={ + # source: https://huggingface.co/shibing624/text2vec-base-chinese + # Not in MTEB + # - shibing624/nli-zh-all/text2vec-base-chinese-sentence-dataset + # (Could have overlaps I'm not aware of) + }, +) + +text2vec_base_chinese_paraphrase = ModelMeta( + name="shibing624/text2vec-base-chinese-paraphrase", + languages=["zho-Hans"], + open_weights=True, + revision="e90c150a9c7fb55a67712a766d6820c55fb83cdd", + release_date="2023-06-19", + n_parameters=118 * 1e6, + embed_dim=768, + license="apache-2.0", + max_tokens=512, + reference="https://huggingface.co/shibing624/text2vec-base-chinese-paraphrase", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from=None, + public_training_code=None, + public_training_data=None, # Couldn't find it + training_datasets={ + # source: https://huggingface.co/shibing624/text2vec-base-chinese + # Not in MTEB + # - shibing624/nli-zh-all/text2vec-base-chinese-paraphrase + # (Could have overlaps I'm not aware of) + }, +) + + +text2vec_multi_langs = [ + "deu-Latn", # German (de) + "eng-Latn", # English (en) + "spa-Latn", # Spanish (es) + "fra-Latn", # French (fr) + "ita-Latn", # Italian (it) + "nld-Latn", # Dutch (nl) + "pol-Latn", # Polish (pl) + "por-Latn", # Portuguese (pt) + "rus-Cyrl", # Russian (ru) + "zho-Hans", # Chinese (Simplified, zh) +] +text2vec_base_multilingual = ModelMeta( + name="shibing624/text2vec-base-multilingual", + languages=text2vec_multi_langs, + open_weights=True, + revision="6633dc49e554de7105458f8f2e96445c6598e9d1", + release_date="2023-06-22", + # While it can be loaded with SBERT, it has one suspicious file according to huggingface + # So probably best not to. + loader=None, + n_parameters=118 * 1e6, + embed_dim=384, + license="apache-2.0", + max_tokens=256, + reference="https://huggingface.co/shibing624/text2vec-base-chinese-paraphrase", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + superseded_by=None, + adapted_from="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + public_training_code=None, + public_training_data=None, # Couldn't find it + training_datasets={ + # source: https://huggingface.co/shibing624/text2vec-base-chinese + # Not in MTEB + # - shibing624/nli-zh-all/tree/main/text2vec-base-multilingual-dataset + # # (Could have overlaps I'm not aware of) + }, +) diff --git a/mteb/models/uae_models.py b/mteb/models/uae_models.py new file mode 100644 index 0000000000..8d97703ef6 --- /dev/null +++ b/mteb/models/uae_models.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import logging +from collections.abc import Sequence +from functools import partial +from typing import Any + +import numpy as np +import torch + +from mteb.encoder_interface import PromptType +from mteb.model_meta import ModelMeta + +from .sentence_transformer_wrapper import SentenceTransformerWrapper + +logger = logging.getLogger(__name__) + + +class UAEWrapper(SentenceTransformerWrapper): + """following the hf model card documentation.""" + + def encode( + self, + sentences: Sequence[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + prompt_name = self.get_prompt_name(self.model_prompts, task_name, prompt_type) + if prompt_name: + logger.info( + f"Using prompt_name={prompt_name} for task={task_name} prompt_type={prompt_type}" + ) + else: + logger.info( + f"No model prompts found for task={task_name} prompt_type={prompt_type}" + ) + logger.info(f"Encoding {len(sentences)} sentences.") + if prompt_name and prompt_name in self.model.prompts: + prompt = self.model.prompts[prompt_name] + sentences = [prompt.format(text=sentence) for sentence in sentences] + + embeddings = self.model.encode( + sentences, + **kwargs, + ) + if isinstance(embeddings, torch.Tensor): + # sometimes in kwargs can be return_tensors=True + embeddings = embeddings.cpu().detach().float().numpy() + return embeddings + + +uae_large_v1 = ModelMeta( + loader=partial( # type: ignore + UAEWrapper, + model="WhereIsAI/UAE-Large-V1", + revision="369c368f70f16a613f19f5598d4f12d9f44235d4", + # https://github.com/SeanLee97/AnglE/blob/b04eae166d8596b47293c75b4664d3ad820d7331/angle_emb/angle.py#L291-L314 + model_prompts={ + "query": "Represent this sentence for searching relevant passages: {text}", + "Summarization": 'Summarize sentence "{text}" in one word:"', + }, + ), + name="WhereIsAI/UAE-Large-V1", + languages=["eng_Latn"], + open_weights=True, + revision="369c368f70f16a613f19f5598d4f12d9f44235d4", + release_date="2023-12-04", # initial commit of hf model. + n_parameters=335_000, + max_tokens=512, + embed_dim=1024, + license="mit", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + reference="https://huggingface.co/WhereIsAI/UAE-Large-V1", + use_instructions=True, + training_datasets={ + # source: https://arxiv.org/pdf/2309.12871 + # not in MTEB + "MNLI": [], + "NLI": [], + "SNLI": [], + }, + public_training_code=None, + public_training_data=None, +) diff --git a/mteb/models/utils.py b/mteb/models/utils.py new file mode 100644 index 0000000000..78d778eb7f --- /dev/null +++ b/mteb/models/utils.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from itertools import islice + + +# https://docs.python.org/3/library/itertools.html#itertools.batched +# Added in version 3.12. +def batched(iterable, n: int, *, strict: bool = False) -> tuple: + # batched('ABCDEFG', 3) → ABC DEF G + if n < 1: + raise ValueError("n must be at least one") + iterator = iter(iterable) + while batch := tuple(islice(iterator, n)): + if strict and len(batch) != n: + raise ValueError("batched(): incomplete batch") + yield batch diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py index ee4a5ca0ad..3ea5e13fe6 100644 --- a/mteb/models/vista_models.py +++ b/mteb/models/vista_models.py @@ -241,10 +241,21 @@ def calculate_probs(self, text_embeddings, image_embeddings): ), name="BAAI/bge-visualized-base", languages=["eng_Latn"], - open_source=True, revision="98db10b10d22620010d06f11733346e1c98c34aa", release_date="2024-06-06", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) visualized_bge_m3 = ModelMeta( @@ -256,10 +267,21 @@ def calculate_probs(self, text_embeddings, image_embeddings): ), name="BAAI/bge-visualized-m3", languages=["eng_Latn"], - open_source=True, revision="98db10b10d22620010d06f11733346e1c98c34aa", release_date="2024-06-06", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) if __name__ == "__main__": diff --git a/mteb/models/vlm2vec_models.py b/mteb/models/vlm2vec_models.py index e5a160753c..efdcf27756 100644 --- a/mteb/models/vlm2vec_models.py +++ b/mteb/models/vlm2vec_models.py @@ -369,10 +369,21 @@ def get_fused_embeddings( ), name="TIGER-Lab/VLM2Vec-LoRA", languages=["eng_Latn"], - open_source=True, revision="7403b6327958071c1e33c822c7453adadccc7298", release_date="2024-10-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) vlm2vec_full = ModelMeta( @@ -382,8 +393,19 @@ def get_fused_embeddings( ), name="TIGER-Lab/VLM2Vec-Full", languages=["eng_Latn"], - open_source=True, revision="e9afa98002097ac2471827ba23ea1f2ddd229480", release_date="2024-10-08", modalities=["image", "text"], + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=None, + public_training_code=None, + public_training_data=None, + framework=["PyTorch"], + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, ) diff --git a/mteb/models/voyage_models.py b/mteb/models/voyage_models.py index 4e79f189ee..a637dee36a 100644 --- a/mteb/models/voyage_models.py +++ b/mteb/models/voyage_models.py @@ -8,14 +8,15 @@ from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta -from mteb.models.sentence_transformer_wrapper import ( - get_prompt_name, - validate_task_to_prompt_name, -) from mteb.requires_package import requires_package from .wrapper import Wrapper +VOYAGE_TRAINING_DATA = { + # Self-reported (message from VoyageAI member) + # synthetic data +} + def token_limit(max_tpm: int, interval: int = 60): limit_interval_start_ts = time.time() @@ -86,7 +87,7 @@ def __init__( self._model_name = model_name self._max_tpm = max_tpm self.model_prompts = ( - validate_task_to_prompt_name(model_prompts) if model_prompts else None + self.validate_task_to_prompt_name(model_prompts) if model_prompts else None ) def encode( @@ -98,9 +99,7 @@ def encode( prompt_type: PromptType | None = None, **kwargs: Any, ) -> np.ndarray: - input_type = ( - get_prompt_name(self.model_prompts, task_name, prompt_type) or "document" - ) + input_type = self.model_prompts.get(prompt_type.value, "document") return self._batched_encode(sentences, batch_size, input_type) def _batched_encode( @@ -144,11 +143,11 @@ def _batched_encode( } voyage_large_2_instruct = ModelMeta( - name="voyage-large-2-instruct", + name="voyageai/voyage-large-2-instruct", revision="1", release_date="2024-05-05", languages=None, # supported languages not specified - loader=partial( + loader=partial( # type: ignore VoyageWrapper, model_name="voyage-large-2-instruct", model_prompts=model_prompts, @@ -157,20 +156,22 @@ def _batched_encode( embed_dim=1024, open_weights=False, n_parameters=None, - memory_usage=None, license=None, reference="https://blog.voyageai.com/2024/05/05/voyage-large-2-instruct-instruction-tuned-and-rank-1-on-mteb/", similarity_fn_name="cosine", framework=["API"], - use_instuctions=True, + use_instructions=True, + training_datasets=VOYAGE_TRAINING_DATA, + public_training_code=None, + public_training_data=None, ) voyage_finance_2 = ModelMeta( - name="voyage-finance-2", + name="voyageai/voyage-finance-2", revision="1", release_date="2024-05-30", languages=None, # supported languages not specified - loader=partial( + loader=partial( # type: ignore VoyageWrapper, model_name="voyage-finance-2", model_prompts=model_prompts, @@ -179,20 +180,22 @@ def _batched_encode( embed_dim=1024, open_weights=False, n_parameters=None, - memory_usage=None, license=None, reference="https://blog.voyageai.com/2024/06/03/domain-specific-embeddings-finance-edition-voyage-finance-2/", similarity_fn_name="cosine", framework=["API"], - use_instuctions=False, + use_instructions=True, + training_datasets=VOYAGE_TRAINING_DATA, + public_training_code=None, + public_training_data=None, ) voyage_law_2 = ModelMeta( - name="voyage-law-2", + name="voyageai/voyage-law-2", revision="1", release_date="2024-04-15", languages=None, # supported languages not specified - loader=partial( + loader=partial( # type: ignore VoyageWrapper, model_name="voyage-law-2", model_prompts=model_prompts, @@ -201,20 +204,22 @@ def _batched_encode( embed_dim=1024, open_weights=False, n_parameters=None, - memory_usage=None, license=None, reference="https://blog.voyageai.com/2024/04/15/domain-specific-embeddings-and-retrieval-legal-edition-voyage-law-2/", similarity_fn_name="cosine", framework=["API"], - use_instuctions=False, + use_instructions=True, + training_datasets=VOYAGE_TRAINING_DATA, + public_training_code=None, + public_training_data=None, ) voyage_code_2 = ModelMeta( - name="voyage-code-2", + name="voyageai/voyage-code-2", revision="1", release_date="2024-01-23", languages=None, # supported languages not specified - loader=partial( + loader=partial( # type: ignore VoyageWrapper, model_name="voyage-code-2", model_prompts=model_prompts, @@ -223,12 +228,14 @@ def _batched_encode( embed_dim=1536, open_weights=False, n_parameters=None, - memory_usage=None, license=None, reference="https://blog.voyageai.com/2024/01/23/voyage-code-2-elevate-your-code-retrieval/", similarity_fn_name="cosine", framework=["API"], - use_instuctions=False, + use_instructions=True, + training_datasets=VOYAGE_TRAINING_DATA, + public_training_code=None, + public_training_data=None, ) voyage_large_2 = ModelMeta( @@ -236,7 +243,7 @@ def _batched_encode( revision="1", release_date="2023-10-29", languages=None, # supported languages not specified - loader=partial( + loader=partial( # type: ignore VoyageWrapper, model_name="voyage-large-2", model_prompts=model_prompts, @@ -245,20 +252,22 @@ def _batched_encode( embed_dim=1536, open_weights=False, n_parameters=None, - memory_usage=None, license=None, reference="https://blog.voyageai.com/2023/10/29/voyage-embeddings/", similarity_fn_name="cosine", framework=["API"], - use_instuctions=False, + use_instructions=True, + training_datasets=VOYAGE_TRAINING_DATA, + public_training_code=None, + public_training_data=None, ) voyage_2 = ModelMeta( - name="voyage-2", + name="voyageai/voyage-2", revision="1", release_date="2023-10-29", languages=None, # supported languages not specified - loader=partial( + loader=partial( # type: ignore VoyageWrapper, model_name="voyage-2", model_prompts=model_prompts, @@ -267,19 +276,21 @@ def _batched_encode( embed_dim=1024, open_weights=False, n_parameters=None, - memory_usage=None, license=None, reference="https://blog.voyageai.com/2023/10/29/voyage-embeddings/", similarity_fn_name="cosine", framework=["API"], - use_instuctions=False, + use_instructions=True, + training_datasets=VOYAGE_TRAINING_DATA, + public_training_code=None, + public_training_data=None, ) voyage_multilingual_2 = ModelMeta( - name="voyage-multilingual-2", + name="voyageai/voyage-multilingual-2", revision="1", release_date="2024-06-10", languages=None, # supported languages not specified - loader=partial( + loader=partial( # type: ignore VoyageWrapper, model_name="voyage-multilingual-2", model_prompts=model_prompts, @@ -288,10 +299,132 @@ def _batched_encode( embed_dim=1024, open_weights=False, n_parameters=None, - memory_usage=None, license=None, reference="https://blog.voyageai.com/2024/06/10/voyage-multilingual-2-multilingual-embedding-model/", similarity_fn_name="cosine", framework=["API"], - use_instuctions=False, + use_instructions=True, + training_datasets=VOYAGE_TRAINING_DATA, + public_training_code=None, + public_training_data=None, +) + +voyage_3 = ModelMeta( + name="voyageai/voyage-3", + revision="1", + release_date="2024-09-18", + languages=None, # supported languages not specified + loader=partial( + VoyageWrapper, + model_name="voyage-3", + model_prompts=model_prompts, + ), + max_tokens=32000, + embed_dim=1024, + open_weights=False, + n_parameters=None, + license=None, + reference="https://blog.voyageai.com/2024/09/18/voyage-3/", + similarity_fn_name="cosine", + framework=["API"], + use_instructions=True, + training_datasets=VOYAGE_TRAINING_DATA, + public_training_code=None, + public_training_data=None, +) + +voyage_3_lite = ModelMeta( + name="voyageai/voyage-3-lite", + revision="1", + release_date="2024-09-18", + languages=None, # supported languages not specified + loader=partial( + VoyageWrapper, + model_name="voyage-3-lite", + model_prompts=model_prompts, + ), + max_tokens=32000, + embed_dim=512, + open_weights=False, + n_parameters=None, + license=None, + reference="https://blog.voyageai.com/2024/09/18/voyage-3/", + similarity_fn_name="cosine", + framework=["API"], + use_instructions=True, + training_datasets=VOYAGE_TRAINING_DATA, + public_training_code=None, + public_training_data=None, +) + + +voyage_3_exp = ModelMeta( + name="voyageai/voyage-3-m-exp", + revision="1", + release_date=None, # not released + languages=None, # supported languages not specified + loader=partial( + VoyageWrapper, + model_name="voyage-3-m-exp", + model_prompts=model_prompts, + ), + max_tokens=32000, + embed_dim=512, + open_weights=False, + n_parameters=None, + license=None, + reference="https://huggingface.co/voyageai/voyage-3-m-exp", + similarity_fn_name="cosine", + framework=["API"], + use_instructions=True, + training_datasets={ + # MTEB(eng, classic) training data: + "ArguAna": ["train"], + "ArguAna-PL": ["train"], + "NanoArguAnaRetrieval": ["train"], + "HotpotQA": ["train"], + "HotpotQA-PL": ["train"], # translation not trained on + "HotpotQAHardNegatives": ["train"], + "MSMARCO": ["train"], + "MSMARCOHardNegatives": ["train"], + "NanoMSMARCORetrieval": ["train"], + "MSMARCO-PL": ["train"], # translation not trained on + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on + "FEVER": ["train"], + "FEVERHardNegatives": ["train"], + "NanoFEVERRetrieval": ["train"], + "FiQA2018": ["train"], + "FiQA2018-PL": ["train"], # translation not trained on + "STS12": ["train"], + "STS22": ["train"], + "AmazonReviewsClassification": ["train"], + "AmazonCounterfactualClassification": ["train"], + "Banking77Classification": ["train"], + "EmotionClassification": ["train"], + "ImdbClassification": ["train"], + "MTOPIntentClassification": ["train"], + "ToxicConversationsClassification": ["train"], + "TweetSentimentExtractionClassification": ["train"], + "ArxivClusteringP2P": ["train"], + "ArxivClusteringP2P.v2": ["train"], + "ArxivClusteringS2S": ["train"], + "ArxivClusteringS2S.v2": ["train"], + "BiorxivClusteringP2P": ["train"], + "BiorxivClusteringP2P.v2": ["train"], + "BiorxivClusteringS2S": ["train"], + "BiorxivClusteringS2S.v2": ["train"], + "MedrxivClusteringP2P": ["train"], + "MedrxivClusteringP2P.v2": ["train"], + "MedrxivClusteringS2S": ["train"], + "MedrxivClusteringS2S.v2": ["train"], + "TwentyNewsgroupsClustering": ["train"], + "TwentyNewsgroupsClustering.v2": ["train"], + "STSBenchmark": ["train"], + "STSBenchmarkMultilingualSTS": ["train"], # translated, not trained on + }, + public_training_code=None, + public_training_data=None, ) diff --git a/mteb/models/voyage_v.py b/mteb/models/voyage_v.py index 7fbf20e350..3b08a7b564 100644 --- a/mteb/models/voyage_v.py +++ b/mteb/models/voyage_v.py @@ -245,17 +245,21 @@ def get_fused_embeddings( loader=partial(voyage_v_loader, model_name="voyage-multimodal-3"), name="voyage-multimodal-3", languages=[], # Unknown - open_source=False, revision="1", release_date="2024-11-10", n_parameters=None, - memory_usage=None, max_tokens=None, embed_dim=1024, license=None, similarity_fn_name="cosine", framework=[], modalities=["image", "text"], + open_weights=None, + public_training_code=None, + public_training_data=None, + reference=None, + use_instructions=None, + training_datasets=None, ) if __name__ == "__main__": diff --git a/mteb/models/wrapper.py b/mteb/models/wrapper.py index c310ab38d9..956071d3dc 100644 --- a/mteb/models/wrapper.py +++ b/mteb/models/wrapper.py @@ -1,7 +1,123 @@ from __future__ import annotations +import logging +from typing import Callable, get_args + +import mteb +from mteb.abstasks.TaskMetadata import TASK_TYPE +from mteb.encoder_interface import PromptType + +logger = logging.getLogger(__name__) + class Wrapper: - """Class to indicate that this is a wrapper for a model.""" + """Base class to indicate that this is a wrapper for a model. + Also contains some utility functions for wrappers for working with prompts and instructions. + """ + + instruction_template: str | Callable[[str, str], str] | None = None + + @staticmethod + def get_prompt_name( + task_to_prompt: dict[str, str] | None, + task_name: str, + prompt_type: PromptType | None, + ) -> str | None: + """A wrapper function around the model.encode method that handles the prompt_name argument and standardizes the output to a numpy array. + The order of priorities for prompt selection are: + 1. Composed prompt of task name + prompt type (query or passage) + 2. Specific task prompt + 3. Composed prompt of task type + prompt type (query or passage) + 4. Specific task type prompt + 5. Specific prompt type (query or passage) + + + Args: + task_to_prompt: The tasks names and their corresponding prompt_names + task_name: The task name to use for building the encoding prompt + prompt_type: The prompt type (e.g. "query" | "passage") to use for building the encoding prompt + """ + task = mteb.get_task(task_name=task_name) + task_type = task.metadata.type + prompt_type_value = prompt_type.value if prompt_type else None + + if ( + task_name + and prompt_type + and f"{task_name}-{prompt_type_value}" in task_to_prompt + ): + return f"{task_name}-{prompt_type_value}" + if task_name and task_name in task_to_prompt: + return task_name + if ( + task_type + and prompt_type + and f"{task_type}-{prompt_type_value}" in task_to_prompt + ): + return f"{task_type}-{prompt_type_value}" + if task_type and task_type in task_to_prompt: + return task_type + if prompt_type and prompt_type_value in task_to_prompt: + return prompt_type_value + logger.info( + "No combination of task name and prompt type was found in model prompts." + ) + return None + + @staticmethod + def validate_task_to_prompt_name( + task_to_prompt_name: dict[str, str] | None, + ) -> dict[str, str] | None: + if task_to_prompt_name is None: + return task_to_prompt_name + task_types = get_args(TASK_TYPE) + prompt_types = [e.value for e in PromptType] + for task_name in task_to_prompt_name: + if "-" in task_name: + task_name, prompt_type = task_name.split("-") + if prompt_type not in prompt_types: + msg = f"Prompt type {prompt_type} is not valid. Valid prompt types are {prompt_types}" + logger.warning(msg) + raise KeyError(msg) + if task_name not in task_types and task_name not in prompt_types: + task = mteb.get_task(task_name=task_name) + if not task: + msg = f"Task name {task_name} is not valid. Valid task names are task types [{task_types}], prompt types [{prompt_types}] and task names" + logger.warning(msg) + raise KeyError(msg) + return task_to_prompt_name + + @staticmethod + def get_instruction(task_name: str, prompt_type: PromptType | None) -> str: + """Get the instruction/prompt to be used for encoding sentences.""" + task = mteb.get_task(task_name=task_name) + task_metadata = task.metadata + if isinstance(task_metadata.prompt, dict) and prompt_type: + if task_metadata.prompt.get(prompt_type.value): + return task_metadata.prompt[prompt_type.value] + logger.warning( + f"Prompt type '{prompt_type}' not found in task metadata for task '{task_name}'." + ) + return "" + if task_metadata.prompt: + return task_metadata.prompt + return task.abstask_prompt + + def format_instruction( + self, instruction: str, prompt_type: PromptType | None = None + ) -> str: + if isinstance(self.instruction_template, str): + if "{instruction}" not in self.instruction_template: + raise ValueError( + "Instruction template must contain the string '{instruction}'." + ) + return self.instruction_template.format(instruction=instruction) + return self.instruction_template(instruction, prompt_type) - pass + def get_task_instruction( + self, task_name: str, prompt_type: PromptType | None + ) -> str: + instruction = self.get_instruction(task_name, prompt_type) + if self.instruction_template: + return self.format_instruction(instruction) + return instruction diff --git a/mteb/overview.py b/mteb/overview.py index 7b1bfbb426..ced0e7729f 100644 --- a/mteb/overview.py +++ b/mteb/overview.py @@ -57,7 +57,7 @@ def check_is_valid_language(lang: str) -> None: ) -def filter_superseeded_datasets(tasks: list[AbsTask]) -> list[AbsTask]: +def filter_superseded_datasets(tasks: list[AbsTask]) -> list[AbsTask]: return [t for t in tasks if t.superseded_by is None] @@ -230,8 +230,9 @@ def get_tasks( task_types: list[TASK_TYPE] | None = None, categories: list[TASK_CATEGORY] | None = None, tasks: list[str] | None = None, - exclude_superseeded: bool = True, + exclude_superseded: bool = True, eval_splits: list[str] | None = None, + exclusive_language_filter: bool = False, ) -> MTEBTasks: """Get a list of tasks based on the specified filters. @@ -245,8 +246,11 @@ def get_tasks( categories: A list of task categories these include "s2s" (sentence to sentence), "s2p" (sentence to paragraph) and "p2p" (paragraph to paragraph). tasks: A list of task names to include. If None, all tasks which pass the filters are included. - exclude_superseeded: A boolean flag to exclude datasets which are superseeded by another. + exclude_superseded: A boolean flag to exclude datasets which are superseded by another. eval_splits: A list of evaluation splits to include. If None, all splits are included. + exclusive_language_filter: Some datasets contains more than one language e.g. for STS22 the subset "de-en" contain eng and deu. If + exclusive_language_filter is set to False both of these will be kept, but if set to True only those that contains all the languages + specified will be kept. Returns: A list of all initialized tasks objects which pass all of the filters (AND operation). @@ -254,12 +258,20 @@ def get_tasks( Examples: >>> get_tasks(languages=["eng", "deu"], script=["Latn"], domains=["Legal"]) >>> get_tasks(languages=["eng"], script=["Latn"], task_types=["Classification"]) - >>> get_tasks(languages=["eng"], script=["Latn"], task_types=["Clustering"], exclude_superseeded=False) + >>> get_tasks(languages=["eng"], script=["Latn"], task_types=["Clustering"], exclude_superseded=False) >>> get_tasks(languages=["eng"], tasks=["WikipediaRetrievalMultilingual"], eval_splits=["test"]) + >>> get_tasks(tasks=["STS22"], languages=["eng"], exclusive_language_filter=True) # don't include multilingual subsets containing English """ if tasks: _tasks = [ - get_task(task, languages, script, eval_splits=eval_splits) for task in tasks + get_task( + task, + languages, + script, + eval_splits=eval_splits, + exclusive_language_filter=exclusive_language_filter, + ) + for task in tasks ] return MTEBTasks(_tasks) @@ -278,8 +290,8 @@ def get_tasks( _tasks = filter_tasks_by_task_types(_tasks, task_types) if categories: _tasks = filter_task_by_categories(_tasks, categories) - if exclude_superseeded: - _tasks = filter_superseeded_datasets(_tasks) + if exclude_superseded: + _tasks = filter_superseded_datasets(_tasks) return MTEBTasks(_tasks) @@ -289,6 +301,8 @@ def get_task( languages: list[str] | None = None, script: list[str] | None = None, eval_splits: list[str] | None = None, + hf_subsets: list[str] | None = None, + exclusive_language_filter: bool = False, ) -> AbsTask: """Get a task by name. @@ -298,6 +312,10 @@ def get_task( "eng-Latn". For multilingual tasks this will also remove languages that are not in the specified list. script: A list of script codes (ISO 15924 codes). If None, all scripts are included. For multilingual tasks this will also remove scripts eval_splits: A list of evaluation splits to include. If None, all splits are included. + hf_subsets: A list of Huggingface subsets to evaluate on. + exclusive_language_filter: Some datasets contains more than one language e.g. for STS22 the subset "de-en" contain eng and deu. If + exclusive_language_filter is set to False both of these will be kept, but if set to True only those that contains all the languages + specified will be kept. Returns: An initialized task object. @@ -319,4 +337,9 @@ def get_task( task = TASKS_REGISTRY[task_name]() if eval_splits: task.filter_eval_splits(eval_splits=eval_splits) - return task.filter_languages(languages, script) + return task.filter_languages( + languages, + script, + hf_subsets=hf_subsets, + exclusive_language_filter=exclusive_language_filter, + ) diff --git a/mteb/task_selection.py b/mteb/task_selection.py index 20d91a97b7..4f00e542c6 100644 --- a/mteb/task_selection.py +++ b/mteb/task_selection.py @@ -10,7 +10,7 @@ from sklearn.preprocessing import StandardScaler from tqdm import tqdm -import mteb +from mteb.load_results.benchmark_results import BenchmarkResults MODEL_NAME = str REVISION = str @@ -35,7 +35,7 @@ def mse_with_zscore(x: list[float], y: list[float]) -> float: def results_to_dataframe( - mteb_results: dict[MODEL_NAME, dict[REVISION, list[mteb.MTEBResults]]], + mteb_results: BenchmarkResults, drop_na: bool = True, **kwargs: Any, ) -> pd.DataFrame: @@ -47,17 +47,16 @@ def results_to_dataframe( **kwargs: Additional keyword arguments to be passed to the `get_score` method of the `MTEBResults` class. """ data = [] - for model_name, revisions in mteb_results.items(): - for rev, tasks_results in revisions.items(): - for task_result in tasks_results: - data.append( - { - "Model": model_name, - "Revision": rev, - "task": task_result.task_name, - "main_score": task_result.get_score(**kwargs), - } - ) + for model_res in mteb_results: + for task_result in model_res.task_results: + data.append( + { + "Model": model_res.model_name, + "Revision": model_res.model_revision, + "task": task_result.task_name, + "main_score": task_result.get_score(**kwargs), + } + ) df = pd.DataFrame(data) if drop_na: diff --git a/mteb/tasks/BitextMining/dan/BornholmskBitextMining.py b/mteb/tasks/BitextMining/dan/BornholmskBitextMining.py index 53a9ccaf2b..242f51ac37 100644 --- a/mteb/tasks/BitextMining/dan/BornholmskBitextMining.py +++ b/mteb/tasks/BitextMining/dan/BornholmskBitextMining.py @@ -29,26 +29,19 @@ class BornholmBitextMining(AbsTaskBitextMining): sample_creation="created", bibtex_citation=""" @inproceedings{derczynskiBornholmskNaturalLanguage2019, - title = {Bornholmsk natural language processing: Resources and tools}, - url = {https://pure.itu.dk/ws/files/84551091/W19_6138.pdf}, - shorttitle = {Bornholmsk natural language processing}, - pages = {338--344}, - booktitle = {Proceedings of the Nordic Conference of Computational Linguistics (2019)}, - publisher = {Linköping University Electronic Press}, - author = {Derczynski, Leon and Kjeldsen, Alex Speed}, - urldate = {2024-04-24}, - date = {2019}, - file = {Available Version (via Google Scholar):/Users/au554730/Zotero/storage/FBQ73ZYN/Derczynski and Kjeldsen - 2019 - Bornholmsk natural language processing Resources .pdf:application/pdf}, + title = {Bornholmsk natural language processing: Resources and tools}, + url = {https://pure.itu.dk/ws/files/84551091/W19_6138.pdf}, + shorttitle = {Bornholmsk natural language processing}, + pages = {338--344}, + booktitle = {Proceedings of the Nordic Conference of Computational Linguistics (2019)}, + publisher = {Linköping University Electronic Press}, + author = {Derczynski, Leon and Kjeldsen, Alex Speed}, + urldate = {2024-04-24}, + date = {2019}, + file = {Available Version (via Google Scholar):/Users/au554730/Zotero/storage/FBQ73ZYN/Derczynski and Kjeldsen - 2019 - Bornholmsk natural language processing Resources .pdf:application/pdf}, } """, - descriptive_stats={ - "n_samples": {"test": 500}, - "test": { - "average_sentence1_length": 49.834, - "average_sentence2_length": 38.888, - "num_samples": 500, - }, - }, + prompt="Retrieve parallel sentences.", ) def dataset_transform(self): diff --git a/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py b/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py index 9d5bfbaa8d..6c6816fb5d 100644 --- a/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py +++ b/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py @@ -41,10 +41,6 @@ class TbilisiCityHallBitextMining(AbsTaskBitextMining, MultilingualTask): annotations_creators="derived", dialect=[], bibtex_citation="", - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 1820}, - "avg_character_length": {_EVAL_SPLIT: 78}, - }, ) def load_data(self, **kwargs) -> None: diff --git a/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py b/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py index d36b8f074d..8c2563bbf3 100644 --- a/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py @@ -1,5 +1,7 @@ from __future__ import annotations +import logging + from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata @@ -14,6 +16,8 @@ _SPLITS = ["test"] +logger = logging.getLogger(__name__) + class BUCCBitextMining(AbsTaskBitextMining, MultilingualTask): superseded_by = "BUCC.v2" @@ -22,6 +26,7 @@ class BUCCBitextMining(AbsTaskBitextMining, MultilingualTask): dataset={ "path": "mteb/bucc-bitext-mining", "revision": "d51519689f32196a32af33b075a01d0e7c51e252", + "trust_remote_code": True, }, description="BUCC bitext mining dataset", reference="https://comparable.limsi.fr/bucc2018/bucc2018-task.html", @@ -56,10 +61,6 @@ class BUCCBitextMining(AbsTaskBitextMining, MultilingualTask): pages = "60--67", abstract = "This paper presents the BUCC 2017 shared task on parallel sentence extraction from comparable corpora. It recalls the design of the datasets, presents their final construction and statistics and the methods used to evaluate system results. 13 runs were submitted to the shared task by 4 teams, covering three of the four proposed language pairs: French-English (7 runs), German-English (3 runs), and Chinese-English (3 runs). The best F-scores as measured against the gold standard were 0.84 (German-English), 0.80 (French-English), and 0.43 (Chinese-English). Because of the design of the dataset, in which not all gold parallel sentence pairs are known, these are only minimum values. We examined manually a small sample of the false negative sentence pairs for the most precise French-English runs and estimated the number of parallel sentence pairs not yet in the provided gold standard. Adding them to the gold standard leads to revised estimates for the French-English F-scores of at most +1.5pt. This suggests that the BUCC 2017 datasets provide a reasonable approximate evaluation of the parallel sentence spotting task.", }""", - descriptive_stats={ - "n_samples": {"test": 641684}, - "avg_character_length": {"test": 101.3}, - }, ) def dataset_transform(self): @@ -74,8 +75,9 @@ def dataset_transform(self): sentence1 = data["sentence1"][0] sentence2 = data["sentence2"][0] sentence1 = [sentence1[i] for (i, j) in gold] - print(lang, len(gold)) - print(len(sentence1), len(sentence2)) + logger.info(f"Lang {lang} num gold {len(gold)}") + logger.info(f"Lang {lang} num sentence1 {len(sentence1)}") + logger.info(f"Lang {lang} num sentence2 {len(sentence2)}") dataset[lang][split] = { "sentence1": sentence1, "sentence2": sentence2, diff --git a/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py b/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py index 25722426c8..f6ab4a20d5 100644 --- a/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py +++ b/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py @@ -56,8 +56,4 @@ class BUCCBitextMiningFast(AbsTaskBitextMining, MultilingualTask): pages = "60--67", abstract = "This paper presents the BUCC 2017 shared task on parallel sentence extraction from comparable corpora. It recalls the design of the datasets, presents their final construction and statistics and the methods used to evaluate system results. 13 runs were submitted to the shared task by 4 teams, covering three of the four proposed language pairs: French-English (7 runs), German-English (3 runs), and Chinese-English (3 runs). The best F-scores as measured against the gold standard were 0.84 (German-English), 0.80 (French-English), and 0.43 (Chinese-English). Because of the design of the dataset, in which not all gold parallel sentence pairs are known, these are only minimum values. We examined manually a small sample of the false negative sentence pairs for the most precise French-English runs and estimated the number of parallel sentence pairs not yet in the provided gold standard. Adding them to the gold standard leads to revised estimates for the French-English F-scores of at most +1.5pt. This suggests that the BUCC 2017 datasets provide a reasonable approximate evaluation of the parallel sentence spotting task.", }""", - descriptive_stats={ - "n_samples": {"test": 641684}, - "avg_character_length": {"test": 101.3}, - }, ) diff --git a/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py b/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py index 58c127963b..07724153c9 100644 --- a/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py @@ -884,10 +884,6 @@ class BibleNLPBitextMining(AbsTaskBitextMining, MultilingualTask): annotations_creators="expert-annotated", dialect=[], sample_creation="created", - descriptive_stats={ - "n_samples": {"train": _N}, - "avg_character_length": {"train": 120}, - }, bibtex_citation="""@article{akerman2023ebible, title={The eBible Corpus: Data and Model Benchmarks for Bible Translation for Low-Resource Languages}, author={Akerman, Vesa and Baines, David and Daspit, Damien and Hermjakob, Ulf and Jang, Taeho and Leong, Colin and Martin, Michael and Mathew, Joel and Robie, Jonathan and Schwarting, Marcus}, diff --git a/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py b/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py index b600162a60..b7806d60ac 100644 --- a/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py @@ -42,7 +42,6 @@ class DiaBLaBitextMining(AbsTaskBitextMining, MultilingualTask): year={2019} } """, - descriptive_stats={"n_samples": {}, "avg_character_length": {}}, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py b/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py index 59abd3bf0f..786b5f0fd9 100644 --- a/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py @@ -268,10 +268,6 @@ class FloresBitextMining(AbsTaskBitextMining, MultilingualTask): year={2022} } """, - descriptive_stats={ - "n_samples": {"dev": 997, "devtest": 1012}, - "avg_character_length": {}, - }, ) def load_data(self, **kwargs: Any) -> None: diff --git a/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py b/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py index 4676aa1906..61a8717507 100644 --- a/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py @@ -100,2545 +100,6 @@ class IN22ConvBitextMining(AbsTaskBitextMining, MultilingualTask): url={https://openreview.net/forum?id=vfT4YuzAYA}, note={} }""", - descriptive_stats={ - "test": { - "average_sentence1_length": 54.32948595562498, - "average_sentence2_length": 54.32948595562498, - "num_samples": 760518, - "hf_subset_descriptive_stats": { - "asm_Beng-ben_Beng": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "asm_Beng-brx_Deva": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "asm_Beng-doi_Deva": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "asm_Beng-eng_Latn": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "asm_Beng-gom_Deva": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "asm_Beng-guj_Gujr": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "asm_Beng-hin_Deva": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "asm_Beng-kan_Knda": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "asm_Beng-kas_Arab": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "asm_Beng-mai_Deva": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "asm_Beng-mal_Mlym": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "asm_Beng-mar_Deva": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "asm_Beng-mni_Mtei": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "asm_Beng-npi_Deva": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "asm_Beng-ory_Orya": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "asm_Beng-pan_Guru": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "asm_Beng-san_Deva": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "asm_Beng-sat_Olck": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "asm_Beng-snd_Deva": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "asm_Beng-tam_Taml": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "asm_Beng-tel_Telu": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "asm_Beng-urd_Arab": { - "average_sentence1_length": 53.753825681969396, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "ben_Beng-asm_Beng": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "ben_Beng-brx_Deva": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "ben_Beng-doi_Deva": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "ben_Beng-eng_Latn": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "ben_Beng-gom_Deva": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "ben_Beng-guj_Gujr": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "ben_Beng-hin_Deva": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "ben_Beng-kan_Knda": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "ben_Beng-kas_Arab": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "ben_Beng-mai_Deva": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "ben_Beng-mal_Mlym": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "ben_Beng-mar_Deva": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "ben_Beng-mni_Mtei": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "ben_Beng-npi_Deva": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "ben_Beng-ory_Orya": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "ben_Beng-pan_Guru": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "ben_Beng-san_Deva": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "ben_Beng-sat_Olck": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "ben_Beng-snd_Deva": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "ben_Beng-tam_Taml": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "ben_Beng-tel_Telu": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "ben_Beng-urd_Arab": { - "average_sentence1_length": 50.03060545575516, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "brx_Deva-asm_Beng": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "brx_Deva-ben_Beng": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "brx_Deva-doi_Deva": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "brx_Deva-eng_Latn": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "brx_Deva-gom_Deva": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "brx_Deva-guj_Gujr": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "brx_Deva-hin_Deva": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "brx_Deva-kan_Knda": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "brx_Deva-kas_Arab": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "brx_Deva-mai_Deva": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "brx_Deva-mal_Mlym": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "brx_Deva-mar_Deva": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "brx_Deva-mni_Mtei": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "brx_Deva-npi_Deva": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "brx_Deva-ory_Orya": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "brx_Deva-pan_Guru": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "brx_Deva-san_Deva": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "brx_Deva-sat_Olck": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "brx_Deva-snd_Deva": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "brx_Deva-tam_Taml": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "brx_Deva-tel_Telu": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "brx_Deva-urd_Arab": { - "average_sentence1_length": 54.05988023952096, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "doi_Deva-asm_Beng": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "doi_Deva-ben_Beng": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "doi_Deva-brx_Deva": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "doi_Deva-eng_Latn": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "doi_Deva-gom_Deva": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "doi_Deva-guj_Gujr": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "doi_Deva-hin_Deva": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "doi_Deva-kan_Knda": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "doi_Deva-kas_Arab": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "doi_Deva-mai_Deva": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "doi_Deva-mal_Mlym": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "doi_Deva-mar_Deva": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "doi_Deva-mni_Mtei": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "doi_Deva-npi_Deva": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "doi_Deva-ory_Orya": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "doi_Deva-pan_Guru": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "doi_Deva-san_Deva": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "doi_Deva-sat_Olck": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "doi_Deva-snd_Deva": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "doi_Deva-tam_Taml": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "doi_Deva-tel_Telu": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "doi_Deva-urd_Arab": { - "average_sentence1_length": 57.37857618097139, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "eng_Latn-asm_Beng": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "eng_Latn-ben_Beng": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "eng_Latn-brx_Deva": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "eng_Latn-doi_Deva": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "eng_Latn-gom_Deva": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "eng_Latn-guj_Gujr": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "eng_Latn-hin_Deva": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "eng_Latn-kan_Knda": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "eng_Latn-kas_Arab": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "eng_Latn-mai_Deva": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "eng_Latn-mal_Mlym": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "eng_Latn-mar_Deva": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "eng_Latn-mni_Mtei": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "eng_Latn-npi_Deva": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "eng_Latn-ory_Orya": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "eng_Latn-pan_Guru": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "eng_Latn-san_Deva": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "eng_Latn-sat_Olck": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "eng_Latn-snd_Deva": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "eng_Latn-tam_Taml": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "eng_Latn-tel_Telu": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "eng_Latn-urd_Arab": { - "average_sentence1_length": 53.17631403858949, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "gom_Deva-asm_Beng": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "gom_Deva-ben_Beng": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "gom_Deva-brx_Deva": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "gom_Deva-doi_Deva": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "gom_Deva-eng_Latn": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "gom_Deva-guj_Gujr": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "gom_Deva-hin_Deva": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "gom_Deva-kan_Knda": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "gom_Deva-kas_Arab": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "gom_Deva-mai_Deva": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "gom_Deva-mal_Mlym": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "gom_Deva-mar_Deva": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "gom_Deva-mni_Mtei": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "gom_Deva-npi_Deva": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "gom_Deva-ory_Orya": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "gom_Deva-pan_Guru": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "gom_Deva-san_Deva": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "gom_Deva-sat_Olck": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "gom_Deva-snd_Deva": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "gom_Deva-tam_Taml": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "gom_Deva-tel_Telu": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "gom_Deva-urd_Arab": { - "average_sentence1_length": 50.22621423819029, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "guj_Gujr-asm_Beng": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "guj_Gujr-ben_Beng": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "guj_Gujr-brx_Deva": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "guj_Gujr-doi_Deva": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "guj_Gujr-eng_Latn": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "guj_Gujr-gom_Deva": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "guj_Gujr-hin_Deva": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "guj_Gujr-kan_Knda": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "guj_Gujr-kas_Arab": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "guj_Gujr-mai_Deva": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "guj_Gujr-mal_Mlym": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "guj_Gujr-mar_Deva": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "guj_Gujr-mni_Mtei": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "guj_Gujr-npi_Deva": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "guj_Gujr-ory_Orya": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "guj_Gujr-pan_Guru": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "guj_Gujr-san_Deva": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "guj_Gujr-sat_Olck": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "guj_Gujr-snd_Deva": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "guj_Gujr-tam_Taml": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "guj_Gujr-tel_Telu": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "guj_Gujr-urd_Arab": { - "average_sentence1_length": 51.54823685961411, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "hin_Deva-asm_Beng": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "hin_Deva-ben_Beng": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "hin_Deva-brx_Deva": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "hin_Deva-doi_Deva": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "hin_Deva-eng_Latn": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "hin_Deva-gom_Deva": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "hin_Deva-guj_Gujr": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "hin_Deva-kan_Knda": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "hin_Deva-kas_Arab": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "hin_Deva-mai_Deva": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "hin_Deva-mal_Mlym": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "hin_Deva-mar_Deva": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "hin_Deva-mni_Mtei": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "hin_Deva-npi_Deva": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "hin_Deva-ory_Orya": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "hin_Deva-pan_Guru": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "hin_Deva-san_Deva": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "hin_Deva-sat_Olck": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "hin_Deva-snd_Deva": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "hin_Deva-tam_Taml": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "hin_Deva-tel_Telu": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "hin_Deva-urd_Arab": { - "average_sentence1_length": 52.67598137059215, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "kan_Knda-asm_Beng": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "kan_Knda-ben_Beng": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "kan_Knda-brx_Deva": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "kan_Knda-doi_Deva": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "kan_Knda-eng_Latn": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "kan_Knda-gom_Deva": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "kan_Knda-guj_Gujr": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "kan_Knda-hin_Deva": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "kan_Knda-kas_Arab": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "kan_Knda-mai_Deva": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "kan_Knda-mal_Mlym": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "kan_Knda-mar_Deva": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "kan_Knda-mni_Mtei": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "kan_Knda-npi_Deva": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "kan_Knda-ory_Orya": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "kan_Knda-pan_Guru": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "kan_Knda-san_Deva": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "kan_Knda-sat_Olck": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "kan_Knda-snd_Deva": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "kan_Knda-tam_Taml": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "kan_Knda-tel_Telu": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "kan_Knda-urd_Arab": { - "average_sentence1_length": 56.14437791084497, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "kas_Arab-asm_Beng": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "kas_Arab-ben_Beng": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "kas_Arab-brx_Deva": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "kas_Arab-doi_Deva": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "kas_Arab-eng_Latn": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "kas_Arab-gom_Deva": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "kas_Arab-guj_Gujr": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "kas_Arab-hin_Deva": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "kas_Arab-kan_Knda": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "kas_Arab-mai_Deva": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "kas_Arab-mal_Mlym": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "kas_Arab-mar_Deva": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "kas_Arab-mni_Mtei": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "kas_Arab-npi_Deva": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "kas_Arab-ory_Orya": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "kas_Arab-pan_Guru": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "kas_Arab-san_Deva": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "kas_Arab-sat_Olck": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "kas_Arab-snd_Deva": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "kas_Arab-tam_Taml": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "kas_Arab-tel_Telu": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "kas_Arab-urd_Arab": { - "average_sentence1_length": 55.81437125748503, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "mai_Deva-asm_Beng": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "mai_Deva-ben_Beng": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "mai_Deva-brx_Deva": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "mai_Deva-doi_Deva": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "mai_Deva-eng_Latn": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "mai_Deva-gom_Deva": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "mai_Deva-guj_Gujr": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "mai_Deva-hin_Deva": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "mai_Deva-kan_Knda": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "mai_Deva-kas_Arab": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "mai_Deva-mal_Mlym": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "mai_Deva-mar_Deva": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "mai_Deva-mni_Mtei": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "mai_Deva-npi_Deva": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "mai_Deva-ory_Orya": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "mai_Deva-pan_Guru": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "mai_Deva-san_Deva": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "mai_Deva-sat_Olck": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "mai_Deva-snd_Deva": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "mai_Deva-tam_Taml": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "mai_Deva-tel_Telu": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "mai_Deva-urd_Arab": { - "average_sentence1_length": 54.3020625415835, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "mal_Mlym-asm_Beng": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "mal_Mlym-ben_Beng": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "mal_Mlym-brx_Deva": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "mal_Mlym-doi_Deva": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "mal_Mlym-eng_Latn": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "mal_Mlym-gom_Deva": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "mal_Mlym-guj_Gujr": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "mal_Mlym-hin_Deva": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "mal_Mlym-kan_Knda": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "mal_Mlym-kas_Arab": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "mal_Mlym-mai_Deva": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "mal_Mlym-mar_Deva": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "mal_Mlym-mni_Mtei": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "mal_Mlym-npi_Deva": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "mal_Mlym-ory_Orya": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "mal_Mlym-pan_Guru": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "mal_Mlym-san_Deva": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "mal_Mlym-sat_Olck": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "mal_Mlym-snd_Deva": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "mal_Mlym-tam_Taml": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "mal_Mlym-tel_Telu": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "mal_Mlym-urd_Arab": { - "average_sentence1_length": 61.24151696606786, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "mar_Deva-asm_Beng": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "mar_Deva-ben_Beng": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "mar_Deva-brx_Deva": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "mar_Deva-doi_Deva": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "mar_Deva-eng_Latn": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "mar_Deva-gom_Deva": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "mar_Deva-guj_Gujr": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "mar_Deva-hin_Deva": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "mar_Deva-kan_Knda": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "mar_Deva-kas_Arab": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "mar_Deva-mai_Deva": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "mar_Deva-mal_Mlym": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "mar_Deva-mni_Mtei": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "mar_Deva-npi_Deva": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "mar_Deva-ory_Orya": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "mar_Deva-pan_Guru": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "mar_Deva-san_Deva": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "mar_Deva-sat_Olck": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "mar_Deva-snd_Deva": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "mar_Deva-tam_Taml": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "mar_Deva-tel_Telu": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "mar_Deva-urd_Arab": { - "average_sentence1_length": 54.52761144377911, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "mni_Mtei-asm_Beng": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "mni_Mtei-ben_Beng": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "mni_Mtei-brx_Deva": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "mni_Mtei-doi_Deva": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "mni_Mtei-eng_Latn": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "mni_Mtei-gom_Deva": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "mni_Mtei-guj_Gujr": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "mni_Mtei-hin_Deva": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "mni_Mtei-kan_Knda": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "mni_Mtei-kas_Arab": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "mni_Mtei-mai_Deva": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "mni_Mtei-mal_Mlym": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "mni_Mtei-mar_Deva": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "mni_Mtei-npi_Deva": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "mni_Mtei-ory_Orya": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "mni_Mtei-pan_Guru": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "mni_Mtei-san_Deva": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "mni_Mtei-sat_Olck": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "mni_Mtei-snd_Deva": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "mni_Mtei-tam_Taml": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "mni_Mtei-tel_Telu": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "mni_Mtei-urd_Arab": { - "average_sentence1_length": 50.91417165668663, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "npi_Deva-asm_Beng": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "npi_Deva-ben_Beng": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "npi_Deva-brx_Deva": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "npi_Deva-doi_Deva": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "npi_Deva-eng_Latn": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "npi_Deva-gom_Deva": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "npi_Deva-guj_Gujr": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "npi_Deva-hin_Deva": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "npi_Deva-kan_Knda": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "npi_Deva-kas_Arab": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "npi_Deva-mai_Deva": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "npi_Deva-mal_Mlym": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "npi_Deva-mar_Deva": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "npi_Deva-mni_Mtei": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "npi_Deva-ory_Orya": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "npi_Deva-pan_Guru": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "npi_Deva-san_Deva": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "npi_Deva-sat_Olck": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "npi_Deva-snd_Deva": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "npi_Deva-tam_Taml": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "npi_Deva-tel_Telu": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "npi_Deva-urd_Arab": { - "average_sentence1_length": 53.30272787757818, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "ory_Orya-asm_Beng": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "ory_Orya-ben_Beng": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "ory_Orya-brx_Deva": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "ory_Orya-doi_Deva": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "ory_Orya-eng_Latn": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "ory_Orya-gom_Deva": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "ory_Orya-guj_Gujr": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "ory_Orya-hin_Deva": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "ory_Orya-kan_Knda": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "ory_Orya-kas_Arab": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "ory_Orya-mai_Deva": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "ory_Orya-mal_Mlym": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "ory_Orya-mar_Deva": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "ory_Orya-mni_Mtei": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "ory_Orya-npi_Deva": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "ory_Orya-pan_Guru": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "ory_Orya-san_Deva": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "ory_Orya-sat_Olck": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "ory_Orya-snd_Deva": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "ory_Orya-tam_Taml": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "ory_Orya-tel_Telu": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "ory_Orya-urd_Arab": { - "average_sentence1_length": 55.509647371922824, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "pan_Guru-asm_Beng": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "pan_Guru-ben_Beng": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "pan_Guru-brx_Deva": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "pan_Guru-doi_Deva": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "pan_Guru-eng_Latn": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "pan_Guru-gom_Deva": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "pan_Guru-guj_Gujr": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "pan_Guru-hin_Deva": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "pan_Guru-kan_Knda": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "pan_Guru-kas_Arab": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "pan_Guru-mai_Deva": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "pan_Guru-mal_Mlym": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "pan_Guru-mar_Deva": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "pan_Guru-mni_Mtei": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "pan_Guru-npi_Deva": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "pan_Guru-ory_Orya": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "pan_Guru-san_Deva": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "pan_Guru-sat_Olck": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "pan_Guru-snd_Deva": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "pan_Guru-tam_Taml": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "pan_Guru-tel_Telu": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "pan_Guru-urd_Arab": { - "average_sentence1_length": 52.83366600133067, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "san_Deva-asm_Beng": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "san_Deva-ben_Beng": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "san_Deva-brx_Deva": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "san_Deva-doi_Deva": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "san_Deva-eng_Latn": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "san_Deva-gom_Deva": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "san_Deva-guj_Gujr": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "san_Deva-hin_Deva": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "san_Deva-kan_Knda": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "san_Deva-kas_Arab": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "san_Deva-mai_Deva": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "san_Deva-mal_Mlym": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "san_Deva-mar_Deva": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "san_Deva-mni_Mtei": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "san_Deva-npi_Deva": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "san_Deva-ory_Orya": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "san_Deva-pan_Guru": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "san_Deva-sat_Olck": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "san_Deva-snd_Deva": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "san_Deva-tam_Taml": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "san_Deva-tel_Telu": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "san_Deva-urd_Arab": { - "average_sentence1_length": 51.4311377245509, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "sat_Olck-asm_Beng": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "sat_Olck-ben_Beng": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "sat_Olck-brx_Deva": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "sat_Olck-doi_Deva": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "sat_Olck-eng_Latn": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "sat_Olck-gom_Deva": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "sat_Olck-guj_Gujr": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "sat_Olck-hin_Deva": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "sat_Olck-kan_Knda": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "sat_Olck-kas_Arab": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "sat_Olck-mai_Deva": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "sat_Olck-mal_Mlym": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "sat_Olck-mar_Deva": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "sat_Olck-mni_Mtei": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "sat_Olck-npi_Deva": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "sat_Olck-ory_Orya": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "sat_Olck-pan_Guru": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "sat_Olck-san_Deva": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "sat_Olck-snd_Deva": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "sat_Olck-tam_Taml": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "sat_Olck-tel_Telu": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "sat_Olck-urd_Arab": { - "average_sentence1_length": 58.94011976047904, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "snd_Deva-asm_Beng": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "snd_Deva-ben_Beng": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "snd_Deva-brx_Deva": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "snd_Deva-doi_Deva": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "snd_Deva-eng_Latn": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "snd_Deva-gom_Deva": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "snd_Deva-guj_Gujr": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "snd_Deva-hin_Deva": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "snd_Deva-kan_Knda": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "snd_Deva-kas_Arab": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "snd_Deva-mai_Deva": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "snd_Deva-mal_Mlym": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "snd_Deva-mar_Deva": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "snd_Deva-mni_Mtei": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "snd_Deva-npi_Deva": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "snd_Deva-ory_Orya": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "snd_Deva-pan_Guru": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "snd_Deva-san_Deva": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "snd_Deva-sat_Olck": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "snd_Deva-tam_Taml": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "snd_Deva-tel_Telu": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "snd_Deva-urd_Arab": { - "average_sentence1_length": 54.445109780439125, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "tam_Taml-asm_Beng": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "tam_Taml-ben_Beng": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "tam_Taml-brx_Deva": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "tam_Taml-doi_Deva": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "tam_Taml-eng_Latn": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "tam_Taml-gom_Deva": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "tam_Taml-guj_Gujr": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "tam_Taml-hin_Deva": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "tam_Taml-kan_Knda": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "tam_Taml-kas_Arab": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "tam_Taml-mai_Deva": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "tam_Taml-mal_Mlym": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "tam_Taml-mar_Deva": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "tam_Taml-mni_Mtei": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "tam_Taml-npi_Deva": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "tam_Taml-ory_Orya": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "tam_Taml-pan_Guru": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "tam_Taml-san_Deva": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "tam_Taml-sat_Olck": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "tam_Taml-snd_Deva": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "tam_Taml-tel_Telu": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - "tam_Taml-urd_Arab": { - "average_sentence1_length": 62.590818363273456, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "tel_Telu-asm_Beng": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "tel_Telu-ben_Beng": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "tel_Telu-brx_Deva": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "tel_Telu-doi_Deva": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "tel_Telu-eng_Latn": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "tel_Telu-gom_Deva": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "tel_Telu-guj_Gujr": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "tel_Telu-hin_Deva": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "tel_Telu-kan_Knda": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "tel_Telu-kas_Arab": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "tel_Telu-mai_Deva": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "tel_Telu-mal_Mlym": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "tel_Telu-mar_Deva": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "tel_Telu-mni_Mtei": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "tel_Telu-npi_Deva": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "tel_Telu-ory_Orya": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "tel_Telu-pan_Guru": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "tel_Telu-san_Deva": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "tel_Telu-sat_Olck": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "tel_Telu-snd_Deva": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "tel_Telu-tam_Taml": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "tel_Telu-urd_Arab": { - "average_sentence1_length": 51.16300731869594, - "average_sentence2_length": 53.568196939454424, - "num_samples": 1503, - }, - "urd_Arab-asm_Beng": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 53.753825681969396, - "num_samples": 1503, - }, - "urd_Arab-ben_Beng": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 50.03060545575516, - "num_samples": 1503, - }, - "urd_Arab-brx_Deva": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 54.05988023952096, - "num_samples": 1503, - }, - "urd_Arab-doi_Deva": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 57.37857618097139, - "num_samples": 1503, - }, - "urd_Arab-eng_Latn": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 53.17631403858949, - "num_samples": 1503, - }, - "urd_Arab-gom_Deva": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 50.22621423819029, - "num_samples": 1503, - }, - "urd_Arab-guj_Gujr": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 51.54823685961411, - "num_samples": 1503, - }, - "urd_Arab-hin_Deva": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 52.67598137059215, - "num_samples": 1503, - }, - "urd_Arab-kan_Knda": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 56.14437791084497, - "num_samples": 1503, - }, - "urd_Arab-kas_Arab": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 55.81437125748503, - "num_samples": 1503, - }, - "urd_Arab-mai_Deva": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 54.3020625415835, - "num_samples": 1503, - }, - "urd_Arab-mal_Mlym": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 61.24151696606786, - "num_samples": 1503, - }, - "urd_Arab-mar_Deva": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 54.52761144377911, - "num_samples": 1503, - }, - "urd_Arab-mni_Mtei": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 50.91417165668663, - "num_samples": 1503, - }, - "urd_Arab-npi_Deva": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 53.30272787757818, - "num_samples": 1503, - }, - "urd_Arab-ory_Orya": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 55.509647371922824, - "num_samples": 1503, - }, - "urd_Arab-pan_Guru": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 52.83366600133067, - "num_samples": 1503, - }, - "urd_Arab-san_Deva": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 51.4311377245509, - "num_samples": 1503, - }, - "urd_Arab-sat_Olck": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 58.94011976047904, - "num_samples": 1503, - }, - "urd_Arab-snd_Deva": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 54.445109780439125, - "num_samples": 1503, - }, - "urd_Arab-tam_Taml": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 62.590818363273456, - "num_samples": 1503, - }, - "urd_Arab-tel_Telu": { - "average_sentence1_length": 53.568196939454424, - "average_sentence2_length": 51.16300731869594, - "num_samples": 1503, - }, - }, - } - }, ) def load_data(self, **kwargs: Any) -> None: diff --git a/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py b/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py index 174a27bc8f..503c64e5f0 100644 --- a/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py @@ -94,10 +94,6 @@ class IN22GenBitextMining(AbsTaskBitextMining, MultilingualTask): url={https://openreview.net/forum?id=vfT4YuzAYA}, note={} }""", - descriptive_stats={ - "n_samples": {"test": 1024}, - "avg_character_length": {"test": 156.7}, - }, ) def load_data(self, **kwargs: Any) -> None: diff --git a/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py b/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py index 3ceffef598..ee83b6f5ca 100644 --- a/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py @@ -81,10 +81,6 @@ class IWSLT2017BitextMining(AbsTaskBitextMining, MultilingualTask): pages = "2--14", } """, - descriptive_stats={ - "n_samples": {"validation": 21928}, - "avg_character_length": {"validation": 95.4}, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py b/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py index acb6c72779..38efd482a0 100644 --- a/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py @@ -128,10 +128,6 @@ class IndicGenBenchFloresBitextMining(AbsTaskBitextMining, MultilingualTask): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"validation": 997, "test": 1012}, - "avg_character_length": {"validation": 126.25, "test": 130.84}, - }, ) def load_data(self, **kwargs: Any) -> None: diff --git a/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py b/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py index 9350f36583..8abb8ce1ff 100644 --- a/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py @@ -40,8 +40,4 @@ class LinceMTBitextMining(AbsTaskBitextMining, MultilingualTask): year={2020} } """, - descriptive_stats={ - "n_samples": {"train": 8060}, - "avg_character_length": {"train": 58.67}, - }, ) diff --git a/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py b/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py index 1144d7d285..0137d9330d 100644 --- a/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py @@ -270,10 +270,6 @@ class NTREXBitextMining(AbsTaskBitextMining, MultilingualTask): annotations_creators="expert-annotated", dialect=[], sample_creation="human-translated and localized", - descriptive_stats={ - "n_samples": {"test": _N * len(_EVAL_LANGS)}, - "avg_character_length": {"test": 120}, - }, bibtex_citation=""" @inproceedings{federmann-etal-2022-ntrex, title = "{NTREX}-128 {--} News Test References for {MT} Evaluation of 128 Languages", diff --git a/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py b/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py index 019a3b5e71..4662833008 100644 --- a/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py @@ -43,8 +43,4 @@ class NollySentiBitextMining(AbsTaskBitextMining, MultilingualTask): year={2023} } """, - descriptive_stats={ - "n_samples": {"train": 1640}, - "avg_character_length": {"train": 135.91}, - }, ) diff --git a/mteb/tasks/BitextMining/multilingual/NorwegianCourtsBitextMining.py b/mteb/tasks/BitextMining/multilingual/NorwegianCourtsBitextMining.py index e013d17a1d..81a880974c 100644 --- a/mteb/tasks/BitextMining/multilingual/NorwegianCourtsBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NorwegianCourtsBitextMining.py @@ -34,10 +34,7 @@ class NorwegianCourtsBitextMining(AbsTaskBitextMining): year={2020} } """, - descriptive_stats={ - "n_samples": {"test": 2050}, - "avg_character_length": {"test": 1884.0}, - }, + prompt="Retrieve parallel sentences in Norwegian Bokmål and Nynorsk", ) def dataset_transform(self): diff --git a/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py b/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py index e4d53dee43..c328461746 100644 --- a/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py @@ -51,69 +51,4 @@ class NusaTranslationBitextMining(AbsTaskBitextMining, MultilingualTask): } """, - descriptive_stats={ - "n_samples": {"train": 50200}, - "train": { - "average_sentence1_length": 145.4552390438247, - "average_sentence2_length": 148.56607569721115, - "num_samples": 50200, - "hf_subset_descriptive_stats": { - "ind-abs": { - "average_sentence1_length": 148.366, - "average_sentence2_length": 147.314, - "num_samples": 1000, - }, - "ind-btk": { - "average_sentence1_length": 145.36666666666667, - "average_sentence2_length": 146.74045454545455, - "num_samples": 6600, - }, - "ind-bew": { - "average_sentence1_length": 145.4280303030303, - "average_sentence2_length": 148.40530303030303, - "num_samples": 6600, - }, - "ind-bhp": { - "average_sentence1_length": 133.528, - "average_sentence2_length": 128.138, - "num_samples": 1000, - }, - "ind-jav": { - "average_sentence1_length": 145.42772727272728, - "average_sentence2_length": 145.8089393939394, - "num_samples": 6600, - }, - "ind-mad": { - "average_sentence1_length": 145.35545454545453, - "average_sentence2_length": 153.6228787878788, - "num_samples": 6600, - }, - "ind-mak": { - "average_sentence1_length": 145.42772727272728, - "average_sentence2_length": 150.6128787878788, - "num_samples": 6600, - }, - "ind-min": { - "average_sentence1_length": 145.42772727272728, - "average_sentence2_length": 148.0621212121212, - "num_samples": 6600, - }, - "ind-mui": { - "average_sentence1_length": 150.454, - "average_sentence2_length": 150.994, - "num_samples": 1000, - }, - "ind-rej": { - "average_sentence1_length": 151.622, - "average_sentence2_length": 139.583, - "num_samples": 1000, - }, - "ind-sun": { - "average_sentence1_length": 145.42772727272728, - "average_sentence2_length": 150.9880303030303, - "num_samples": 6600, - }, - }, - }, - }, ) diff --git a/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py b/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py index 3ea0bc4b1b..2f49c3acc4 100644 --- a/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py @@ -58,8 +58,4 @@ class NusaXBitextMining(AbsTaskBitextMining, MultilingualTask): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"train": 5500}, - "avg_character_length": {"train": 157.15}, - }, ) diff --git a/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py b/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py index 4f22ce44b0..c7fec75637 100644 --- a/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py @@ -40,8 +40,4 @@ class PhincBitextMining(AbsTaskBitextMining, MultilingualTask): year={2020} } """, - descriptive_stats={ - "n_samples": {"train": 13738}, - "avg_character_length": {"train": 75.32}, - }, ) diff --git a/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py b/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py index 2b67e06db1..28f11bfcbf 100644 --- a/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py @@ -33,10 +33,6 @@ class RomaTalesBitextMining(AbsTaskBitextMining, MultilingualTask): dialect=["Lovari"], sample_creation="created", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 215}, - "avg_character_length": {"test": 316.8046511627907}, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py b/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py index d1f539f8c2..4312332022 100644 --- a/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py @@ -152,8 +152,4 @@ class TatoebaBitextMining(AbsTaskBitextMining, MultilingualTask): year = {2021}, } """, - descriptive_stats={ - "n_samples": {"test": 2000}, - "avg_character_length": {"test": 39.4}, - }, ) diff --git a/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py b/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py index 22e7c3d7f7..b4072553b6 100644 --- a/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py +++ b/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py @@ -46,10 +46,6 @@ class SRNCorpusBitextMining(AbsTaskBitextMining, MultilingualTask): annotations_creators="human-annotated", dialect=[], sample_creation="found", - descriptive_stats={ - "n_samples": {"test": _N}, - "avg_character_length": {"test": 55}, - }, bibtex_citation=""" @article{zwennicker2022towards, title={Towards a general purpose machine translation system for Sranantongo}, diff --git a/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py b/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py index c8328040d2..8dea762f86 100644 --- a/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py +++ b/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py @@ -39,10 +39,6 @@ class VieMedEVBitextMining(AbsTaskBitextMining): booktitle = {Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING)}, year = {2024} }""", - descriptive_stats={ - "n_samples": {"test": TEST_SAMPLES}, - "avg_character_length": {"test": 139.23}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ara/AJGT.py b/mteb/tasks/Classification/ara/AJGT.py index b39b0f5031..2baa389794 100644 --- a/mteb/tasks/Classification/ara/AJGT.py +++ b/mteb/tasks/Classification/ara/AJGT.py @@ -36,8 +36,4 @@ class AJGT(AbsTaskClassification): organization={Springer} } """, - descriptive_stats={ - "n_samples": {"train": 1800}, - "avg_character_length": {"train": 46.81}, - }, ) diff --git a/mteb/tasks/Classification/ara/HotelReviewSentimentClassification.py b/mteb/tasks/Classification/ara/HotelReviewSentimentClassification.py index 416cf44bb7..24b7bc33fc 100644 --- a/mteb/tasks/Classification/ara/HotelReviewSentimentClassification.py +++ b/mteb/tasks/Classification/ara/HotelReviewSentimentClassification.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 2048 - class HotelReviewSentimentClassification(AbsTaskClassification): metadata = TaskMetadata( @@ -39,10 +37,6 @@ class HotelReviewSentimentClassification(AbsTaskClassification): publisher={Springer} } """, - descriptive_stats={ - "n_samples": {"train": N_SAMPLES}, - "avg_character_length": {"train": 137.2}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ara/OnlineStoreReviewSentimentClassification.py b/mteb/tasks/Classification/ara/OnlineStoreReviewSentimentClassification.py index 81ec501a5e..500cc68218 100644 --- a/mteb/tasks/Classification/ara/OnlineStoreReviewSentimentClassification.py +++ b/mteb/tasks/Classification/ara/OnlineStoreReviewSentimentClassification.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 2048 - class OnlineStoreReviewSentimentClassification(AbsTaskClassification): metadata = TaskMetadata( @@ -29,10 +27,6 @@ class OnlineStoreReviewSentimentClassification(AbsTaskClassification): dialect=["ara-Arab-SA"], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"train": N_SAMPLES}, - "avg_character_length": {"train": 137.2}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ara/RestaurantReviewSentimentClassification.py b/mteb/tasks/Classification/ara/RestaurantReviewSentimentClassification.py index 22bd5574e0..363d0526d7 100644 --- a/mteb/tasks/Classification/ara/RestaurantReviewSentimentClassification.py +++ b/mteb/tasks/Classification/ara/RestaurantReviewSentimentClassification.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 2048 - class RestaurantReviewSentimentClassification(AbsTaskClassification): metadata = TaskMetadata( @@ -38,10 +36,6 @@ class RestaurantReviewSentimentClassification(AbsTaskClassification): organization={Springer} } """, - descriptive_stats={ - "n_samples": {"train": N_SAMPLES}, - "avg_character_length": {"train": 231.4}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ara/TweetEmotionClassification.py b/mteb/tasks/Classification/ara/TweetEmotionClassification.py index 22e3af698c..e7fb8687ac 100644 --- a/mteb/tasks/Classification/ara/TweetEmotionClassification.py +++ b/mteb/tasks/Classification/ara/TweetEmotionClassification.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 2048 - class TweetEmotionClassification(AbsTaskClassification): metadata = TaskMetadata( @@ -39,10 +37,6 @@ class TweetEmotionClassification(AbsTaskClassification): organization={Springer} } """, - descriptive_stats={ - "n_samples": {"train": N_SAMPLES}, - "avg_character_length": {"train": 78.8}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ara/TweetSarcasmClassification.py b/mteb/tasks/Classification/ara/TweetSarcasmClassification.py index 3c804780f5..9c5f141d0b 100644 --- a/mteb/tasks/Classification/ara/TweetSarcasmClassification.py +++ b/mteb/tasks/Classification/ara/TweetSarcasmClassification.py @@ -48,10 +48,6 @@ class TweetSarcasmClassification(AbsTaskClassification): ISBN = "979-10-95546-51-1", } """, - descriptive_stats={ - "n_samples": {"test": 2110}, - "avg_character_length": {"test": 102.1}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ben/BengaliDocumentClassification.py b/mteb/tasks/Classification/ben/BengaliDocumentClassification.py index c5ed6aa451..145eba57ab 100644 --- a/mteb/tasks/Classification/ben/BengaliDocumentClassification.py +++ b/mteb/tasks/Classification/ben/BengaliDocumentClassification.py @@ -42,10 +42,6 @@ class BengaliDocumentClassification(AbsTaskClassification): pages = "52--67" } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 1658.1}, - }, ) def dataset_transform(self) -> None: diff --git a/mteb/tasks/Classification/ben/BengaliHateSpeechClassification.py b/mteb/tasks/Classification/ben/BengaliHateSpeechClassification.py index 59fa5721b1..86763f0e50 100644 --- a/mteb/tasks/Classification/ben/BengaliHateSpeechClassification.py +++ b/mteb/tasks/Classification/ben/BengaliHateSpeechClassification.py @@ -34,10 +34,6 @@ class BengaliHateSpeechClassification(AbsTaskClassification): year={2020} } """, - descriptive_stats={ - "n_samples": {"train": 3418}, - "avg_character_length": {"train": 103.42}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ben/BengaliSentimentAnalysis.py b/mteb/tasks/Classification/ben/BengaliSentimentAnalysis.py index c2fb31f72c..87af91c8a8 100644 --- a/mteb/tasks/Classification/ben/BengaliSentimentAnalysis.py +++ b/mteb/tasks/Classification/ben/BengaliSentimentAnalysis.py @@ -33,10 +33,6 @@ class BengaliSentimentAnalysis(AbsTaskClassification): pages={50--60}, year={2020} }""", - descriptive_stats={ - "n_samples": {"train": 11807}, - "avg_character_length": {"train": 69.66}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/bul/BulgarianStoreReviewSentimentClassfication.py b/mteb/tasks/Classification/bul/BulgarianStoreReviewSentimentClassfication.py index bdd77e53d8..7878fa89e2 100644 --- a/mteb/tasks/Classification/bul/BulgarianStoreReviewSentimentClassfication.py +++ b/mteb/tasks/Classification/bul/BulgarianStoreReviewSentimentClassfication.py @@ -36,10 +36,6 @@ class BulgarianStoreReviewSentimentClassfication(AbsTaskClassification): url = {https://doi.org/10.7910/DVN/TXIK9P} } """, - descriptive_stats={ - "n_samples": {"test": 182}, - "avg_character_length": {"test": 316.7}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ces/CSFDCZMovieReviewSentimentClassification.py b/mteb/tasks/Classification/ces/CSFDCZMovieReviewSentimentClassification.py index 69d4e358e8..dcd87417da 100644 --- a/mteb/tasks/Classification/ces/CSFDCZMovieReviewSentimentClassification.py +++ b/mteb/tasks/Classification/ces/CSFDCZMovieReviewSentimentClassification.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 2048 - class CSFDCZMovieReviewSentimentClassification(AbsTaskClassification): metadata = TaskMetadata( @@ -38,20 +36,13 @@ class CSFDCZMovieReviewSentimentClassification(AbsTaskClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "avg_character_length": {"test": 386.5}, - }, ) - - @property - def metadata_dict(self): - md = super().metadata_dict - # Increase the samples_per_label in order to improve baseline performance - md["samples_per_label"] = 20 - return md + # Increase the samples_per_label in order to improve baseline performance + samples_per_label = 20 def dataset_transform(self): + N_SAMPLES = 2048 + self.dataset = self.dataset.rename_columns( {"comment": "text", "rating_int": "label"} ) diff --git a/mteb/tasks/Classification/ces/CzechProductReviewSentimentClassification.py b/mteb/tasks/Classification/ces/CzechProductReviewSentimentClassification.py index 8405140936..8705a73c39 100644 --- a/mteb/tasks/Classification/ces/CzechProductReviewSentimentClassification.py +++ b/mteb/tasks/Classification/ces/CzechProductReviewSentimentClassification.py @@ -44,18 +44,8 @@ class CzechProductReviewSentimentClassification(AbsTaskClassification): pages = "65--74", } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 153.26}, - }, ) - - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 16 - return metadata_dict + samples_per_label = 16 def dataset_transform(self) -> None: self.dataset = self.dataset.rename_columns( diff --git a/mteb/tasks/Classification/ces/CzechSoMeSentimentClassification.py b/mteb/tasks/Classification/ces/CzechSoMeSentimentClassification.py index add6a19e9e..0e61196b19 100644 --- a/mteb/tasks/Classification/ces/CzechSoMeSentimentClassification.py +++ b/mteb/tasks/Classification/ces/CzechSoMeSentimentClassification.py @@ -44,18 +44,8 @@ class CzechSoMeSentimentClassification(AbsTaskClassification): pages = "65--74", } """, - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": {"test": 59.89}, - }, ) - - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 16 - return metadata_dict + samples_per_label = 16 def dataset_transform(self) -> None: self.dataset = self.dataset.rename_columns( diff --git a/mteb/tasks/Classification/ces/CzechSubjectivityClassification.py b/mteb/tasks/Classification/ces/CzechSubjectivityClassification.py index 5603b606ab..18bcc7e10e 100644 --- a/mteb/tasks/Classification/ces/CzechSubjectivityClassification.py +++ b/mteb/tasks/Classification/ces/CzechSubjectivityClassification.py @@ -39,8 +39,4 @@ class CzechSubjectivityClassification(AbsTaskClassification): pages = "1381--1391", } """, - descriptive_stats={ - "n_samples": {"validation": 500, "test": 2000}, - "avg_character_length": {"validation": 108.2, "test": 108.3}, - }, ) diff --git a/mteb/tasks/Classification/dan/AngryTweetsClassification.py b/mteb/tasks/Classification/dan/AngryTweetsClassification.py index fc1f177e4f..b22efde7a5 100644 --- a/mteb/tasks/Classification/dan/AngryTweetsClassification.py +++ b/mteb/tasks/Classification/dan/AngryTweetsClassification.py @@ -33,15 +33,7 @@ class AngryTweetsClassification(AbsTaskClassification): pages={460--466}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 1050}, - "avg_character_length": {"test": 156.1}, - }, + prompt="Classify Danish tweets by sentiment. (positive, negative, neutral).", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 16 - return metadata_dict + samples_per_label = 16 diff --git a/mteb/tasks/Classification/dan/DKHateClassification.py b/mteb/tasks/Classification/dan/DKHateClassification.py index 65a407a56e..fb6c04cc40 100644 --- a/mteb/tasks/Classification/dan/DKHateClassification.py +++ b/mteb/tasks/Classification/dan/DKHateClassification.py @@ -55,18 +55,10 @@ class DKHateClassification(AbsTaskClassification): language = "English", ISBN = "979-10-95546-34-4", }""", - descriptive_stats={ - "n_samples": {"test": 329}, - "avg_character_length": {"test": 104.0}, - }, + prompt="Classify Danish tweets based on offensiveness (offensive, not offensive)", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = dict(self.metadata) - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 16 - return metadata_dict + samples_per_label = 16 def dataset_transform(self): # convert label to a 0/1 label diff --git a/mteb/tasks/Classification/dan/DanishPoliticalCommentsClassification.py b/mteb/tasks/Classification/dan/DanishPoliticalCommentsClassification.py index bb2b9bff15..8f82e91ecc 100644 --- a/mteb/tasks/Classification/dan/DanishPoliticalCommentsClassification.py +++ b/mteb/tasks/Classification/dan/DanishPoliticalCommentsClassification.py @@ -36,18 +36,10 @@ class DanishPoliticalCommentsClassification(AbsTaskClassification): year={2019}, institution={IT University of Copenhagen}, }""", - descriptive_stats={ - "n_samples": {"train": 9010}, - "avg_character_length": {"train": 69.9}, - }, + prompt="Classify Danish political comments for sentiment", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = dict(self.metadata) - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 16 - return metadata_dict + samples_per_label = 16 def dataset_transform(self): self.dataset = self.dataset.rename_column("sentence", "text") diff --git a/mteb/tasks/Classification/dan/DdiscoCohesionClassification.py b/mteb/tasks/Classification/dan/DdiscoCohesionClassification.py index 6a7410178e..b28396869e 100644 --- a/mteb/tasks/Classification/dan/DdiscoCohesionClassification.py +++ b/mteb/tasks/Classification/dan/DdiscoCohesionClassification.py @@ -56,7 +56,6 @@ class DdiscoCohesionClassification(AbsTaskClassification): abstract = "To date, there has been no resource for studying discourse coherence on real-world Danish texts. Discourse coherence has mostly been approached with the assumption that incoherent texts can be represented by coherent texts in which sentences have been shuffled. However, incoherent real-world texts rarely resemble that. We thus present DDisCo, a dataset including text from the Danish Wikipedia and Reddit annotated for discourse coherence. We choose to annotate real-world texts instead of relying on artificially incoherent text for training and testing models. Then, we evaluate the performance of several methods, including neural networks, on the dataset.", } """, - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/dan/LccSentimentClassification.py b/mteb/tasks/Classification/dan/LccSentimentClassification.py index dbaf564a5b..39b974dcd3 100644 --- a/mteb/tasks/Classification/dan/LccSentimentClassification.py +++ b/mteb/tasks/Classification/dan/LccSentimentClassification.py @@ -46,15 +46,7 @@ class LccSentimentClassification(AbsTaskClassification): url = "http://www.lrec-conf.org/proceedings/lrec2006/pdf/641_pdf.pdf", abstract = "A simple and flexible schema for storing and presenting monolingual language resources is proposed. In this format, data for 18 different languages is already available in various sizes. The data is provided free of charge for online use and download. The main target is to ease the application of algorithms for monolingual and interlingual studies.", }""", - descriptive_stats={ - "n_samples": {"test": 150}, - "avg_character_length": {"test": 118.7}, - }, + prompt="Classify texts based on sentiment", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 16 - return metadata_dict + samples_per_label = 16 diff --git a/mteb/tasks/Classification/deu/GermanPoliticiansTwitterSentimentClassification.py b/mteb/tasks/Classification/deu/GermanPoliticiansTwitterSentimentClassification.py index ec2021eaac..02cbe51f5f 100644 --- a/mteb/tasks/Classification/deu/GermanPoliticiansTwitterSentimentClassification.py +++ b/mteb/tasks/Classification/deu/GermanPoliticiansTwitterSentimentClassification.py @@ -48,10 +48,6 @@ class GermanPoliticiansTwitterSentimentClassification(AbsTaskClassification): pages = "74--87", } """, - descriptive_stats={ - "n_samples": {"test": 357}, - "avg_character_length": {"test": 302.48}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/deu/TenKGnadClassification.py b/mteb/tasks/Classification/deu/TenKGnadClassification.py index 220fa4ea16..592d66c983 100644 --- a/mteb/tasks/Classification/deu/TenKGnadClassification.py +++ b/mteb/tasks/Classification/deu/TenKGnadClassification.py @@ -39,8 +39,4 @@ class TenKGnadClassification(AbsTaskClassification): Month = aug } """, - descriptive_stats={ - "n_samples": {"test": 1028}, - "avg_character_length": {"test": 2627.31}, - }, ) diff --git a/mteb/tasks/Classification/ell/GreekLegalCodeClassification.py b/mteb/tasks/Classification/ell/GreekLegalCodeClassification.py index d7549ffb33..29fb9bbb90 100644 --- a/mteb/tasks/Classification/ell/GreekLegalCodeClassification.py +++ b/mteb/tasks/Classification/ell/GreekLegalCodeClassification.py @@ -41,10 +41,6 @@ class GreekLegalCodeClassification(AbsTaskClassification): pages = "63--75" } """, - descriptive_stats={ - "n_samples": {"validation": TEST_SAMPLES, "test": TEST_SAMPLES}, - "avg_character_length": {"validation": 4046.8, "test": 4200.8}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/eng/AmazonPolarityClassification.py b/mteb/tasks/Classification/eng/AmazonPolarityClassification.py index 38bf1616aa..ab0ea29e97 100644 --- a/mteb/tasks/Classification/eng/AmazonPolarityClassification.py +++ b/mteb/tasks/Classification/eng/AmazonPolarityClassification.py @@ -36,8 +36,5 @@ class AmazonPolarityClassification(AbsTaskClassification): year={2013}, url={https://api.semanticscholar.org/CorpusID:6440341} }""", - descriptive_stats={ - "n_samples": {"test": 400000}, - "avg_character_length": {"test": 431.4}, - }, + prompt="Classify Amazon reviews into positive or negative sentiment", ) diff --git a/mteb/tasks/Classification/eng/ArxivClassification.py b/mteb/tasks/Classification/eng/ArxivClassification.py index ca4a6657cd..92bd473a74 100644 --- a/mteb/tasks/Classification/eng/ArxivClassification.py +++ b/mteb/tasks/Classification/eng/ArxivClassification.py @@ -37,5 +37,4 @@ class ArxivClassification(AbsTaskClassification): pages={40707-40718}, doi={10.1109/ACCESS.2019.2907992} }""", - descriptive_stats={"n_samples": {"test": 2048}, "avg_character_length": {}}, ) diff --git a/mteb/tasks/Classification/eng/Banking77Classification.py b/mteb/tasks/Classification/eng/Banking77Classification.py index ca8d759196..5b6db45c64 100644 --- a/mteb/tasks/Classification/eng/Banking77Classification.py +++ b/mteb/tasks/Classification/eng/Banking77Classification.py @@ -53,8 +53,5 @@ class Banking77Classification(AbsTaskClassification): doi = "10.18653/v1/2020.nlp4convai-1.5", pages = "38--45", }""", - descriptive_stats={ - "n_samples": {"test": 3080}, - "avg_character_length": {"test": 54.2}, - }, + prompt="Given a online banking query, find the corresponding intents", ) diff --git a/mteb/tasks/Classification/eng/DBpediaClassification.py b/mteb/tasks/Classification/eng/DBpediaClassification.py index 8d76a9e5bd..ac7ee41ae8 100644 --- a/mteb/tasks/Classification/eng/DBpediaClassification.py +++ b/mteb/tasks/Classification/eng/DBpediaClassification.py @@ -39,10 +39,6 @@ class DBpediaClassification(AbsTaskClassification): year = {2015} } """, - descriptive_stats={ - "n_samples": {"test": 70000}, - "avg_character_length": {"test": 281.40}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/eng/EmotionClassification.py b/mteb/tasks/Classification/eng/EmotionClassification.py index 24879e89bb..05133cb17f 100644 --- a/mteb/tasks/Classification/eng/EmotionClassification.py +++ b/mteb/tasks/Classification/eng/EmotionClassification.py @@ -16,7 +16,7 @@ class EmotionClassification(AbsTaskClassification): type="Classification", category="s2s", modalities=["text"], - eval_splits=["validation", "test"], + eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", date=( @@ -50,15 +50,7 @@ class EmotionClassification(AbsTaskClassification): pages = "3687--3697", abstract = "Emotions are expressed in nuanced ways, which varies by collective or individual experiences, knowledge, and beliefs. Therefore, to understand emotion, as conveyed through text, a robust mechanism capable of capturing and modeling different linguistic nuances and phenomena is needed. We propose a semi-supervised, graph-based algorithm to produce rich structural descriptors which serve as the building blocks for constructing contextualized affect representations from text. The pattern-based representations are further enriched with word embeddings and evaluated through several emotion recognition tasks. Our experimental results demonstrate that the proposed method outperforms state-of-the-art techniques on emotion recognition tasks.", }""", - descriptive_stats={ - "n_samples": {"validation": 2000, "test": 2000}, - "avg_character_length": {"validation": 95.3, "test": 95.6}, - }, + prompt="Classify the emotion expressed in the given Twitter message into one of the six emotions: anger, fear, joy, love, sadness, and surprise", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 16 - return metadata_dict + samples_per_label = 16 diff --git a/mteb/tasks/Classification/eng/FinancialPhrasebankClassification.py b/mteb/tasks/Classification/eng/FinancialPhrasebankClassification.py index 4bc6ce91a2..6ddb37c42a 100644 --- a/mteb/tasks/Classification/eng/FinancialPhrasebankClassification.py +++ b/mteb/tasks/Classification/eng/FinancialPhrasebankClassification.py @@ -37,10 +37,6 @@ class FinancialPhrasebankClassification(AbsTaskClassification): volume={65} } """, - descriptive_stats={ - "n_samples": {"train": 4840}, - "avg_character_length": {"train": 121.96}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/eng/FrenkEnClassification.py b/mteb/tasks/Classification/eng/FrenkEnClassification.py index 2ed1fce68c..1d597f19a2 100644 --- a/mteb/tasks/Classification/eng/FrenkEnClassification.py +++ b/mteb/tasks/Classification/eng/FrenkEnClassification.py @@ -36,8 +36,4 @@ class FrenkEnClassification(AbsTaskClassification): primaryClass={cs.CL}, url={https://arxiv.org/abs/1906.02045} }""", - descriptive_stats={ - "n_samples": {"test": 2300}, - "avg_character_length": {"test": 188.75}, - }, ) diff --git a/mteb/tasks/Classification/eng/ImdbClassification.py b/mteb/tasks/Classification/eng/ImdbClassification.py index ce48c718d1..75b540bf47 100644 --- a/mteb/tasks/Classification/eng/ImdbClassification.py +++ b/mteb/tasks/Classification/eng/ImdbClassification.py @@ -48,8 +48,5 @@ class ImdbClassification(AbsTaskClassification): url = "https://aclanthology.org/P11-1015", pages = "142--150", }""", - descriptive_stats={ - "n_samples": {"test": 25000}, - "avg_character_length": {"test": 1293.8}, - }, + prompt="Classify the sentiment expressed in the given movie review text from the IMDB dataset", ) diff --git a/mteb/tasks/Classification/eng/LegalBenchClassification.py b/mteb/tasks/Classification/eng/LegalBenchClassification.py index 28981aa4e2..4e3f25554f 100644 --- a/mteb/tasks/Classification/eng/LegalBenchClassification.py +++ b/mteb/tasks/Classification/eng/LegalBenchClassification.py @@ -41,10 +41,6 @@ class CanadaTaxCourtOutcomesLegalBenchClassification(AbsTaskClassification): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"test": 244}, - "avg_character_length": {"test": 622.60}, - }, ) def dataset_transform(self): @@ -91,10 +87,6 @@ class ContractNLIConfidentialityOfAgreementLegalBenchClassification( journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 82}, - "avg_character_length": {"test": 473.17}, - }, ) def dataset_transform(self): @@ -145,10 +137,6 @@ class ContractNLIExplicitIdentificationLegalBenchClassification(AbsTaskClassific journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 109}, - "avg_character_length": {"test": 506.12}, - }, ) def dataset_transform(self): @@ -201,10 +189,6 @@ class ContractNLIInclusionOfVerballyConveyedInformationLegalBenchClassification( journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 139}, - "avg_character_length": {"test": 525.75}, - }, ) def dataset_transform(self): @@ -255,10 +239,6 @@ class ContractNLILimitedUseLegalBenchClassification(AbsTaskClassification): journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 208}, - "avg_character_length": {"test": 407.51}, - }, ) def dataset_transform(self): @@ -309,10 +289,6 @@ class ContractNLINoLicensingLegalBenchClassification(AbsTaskClassification): journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 162}, - "avg_character_length": {"test": 419.42}, - }, ) def dataset_transform(self): @@ -365,10 +341,6 @@ class ContractNLINoticeOnCompelledDisclosureLegalBenchClassification( journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 142}, - "avg_character_length": {"test": 503.45}, - }, ) def dataset_transform(self): @@ -421,10 +393,6 @@ class ContractNLIPermissibleAcquirementOfSimilarInformationLegalBenchClassificat journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 178}, - "avg_character_length": {"test": 427.40}, - }, ) def dataset_transform(self): @@ -475,10 +443,6 @@ class ContractNLIPermissibleCopyLegalBenchClassification(AbsTaskClassification): journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 87}, - "avg_character_length": {"test": 386.84}, - }, ) def dataset_transform(self): @@ -531,10 +495,6 @@ class ContractNLIPermissibleDevelopmentOfSimilarInformationLegalBenchClassificat journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 136}, - "avg_character_length": {"test": 396.40}, - }, ) def dataset_transform(self): @@ -587,10 +547,6 @@ class ContractNLIPermissiblePostAgreementPossessionLegalBenchClassification( journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 111}, - "avg_character_length": {"test": 529.09}, - }, ) def dataset_transform(self): @@ -643,10 +599,6 @@ class ContractNLIReturnOfConfidentialInformationLegalBenchClassification( journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 66}, - "avg_character_length": {"test": 478.29}, - }, ) def dataset_transform(self): @@ -697,10 +649,6 @@ class ContractNLISharingWithEmployeesLegalBenchClassification(AbsTaskClassificat journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 170}, - "avg_character_length": {"test": 548.63}, - }, ) def dataset_transform(self): @@ -751,10 +699,6 @@ class ContractNLISharingWithThirdPartiesLegalBenchClassification(AbsTaskClassifi journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 180}, - "avg_character_length": {"test": 517.29}, - }, ) def dataset_transform(self): @@ -805,10 +749,6 @@ class ContractNLISurvivalOfObligationsLegalBenchClassification(AbsTaskClassifica journal={arXiv preprint arXiv:2110.01799}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 157}, - "avg_character_length": {"test": 417.64}, - }, ) def dataset_transform(self): @@ -854,10 +794,6 @@ class CorporateLobbyingLegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": 490}, - "avg_character_length": {"test": 6039.85}, - }, ) def dataset_transform(self): @@ -924,10 +860,6 @@ class CUADAffiliateLicenseLicenseeLegalBenchClassification(AbsTaskClassification year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 198}, - "avg_character_length": {"test": 484.11}, - }, ) def dataset_transform(self): @@ -979,10 +911,6 @@ class CUADAffiliateLicenseLicensorLegalBenchClassification(AbsTaskClassification year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 88}, - "avg_character_length": {"test": 633.40}, - }, ) def dataset_transform(self): @@ -1034,10 +962,6 @@ class CUADAntiAssignmentLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 1172}, - "avg_character_length": {"test": 340.81}, - }, ) def dataset_transform(self): @@ -1089,10 +1013,6 @@ class CUADAuditRightsLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 1216}, - "avg_character_length": {"test": 337.14}, - }, ) def dataset_transform(self): @@ -1144,10 +1064,6 @@ class CUADCapOnLiabilityLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 1246}, - "avg_character_length": {"test": 375.74}, - }, ) def dataset_transform(self): @@ -1199,10 +1115,6 @@ class CUADChangeOfControlLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 416}, - "avg_character_length": {"test": 391.96}, - }, ) def dataset_transform(self): @@ -1256,10 +1168,6 @@ class CUADCompetitiveRestrictionExceptionLegalBenchClassification( year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 220}, - "avg_character_length": {"test": 433.04}, - }, ) def dataset_transform(self): @@ -1311,10 +1219,6 @@ class CUADCovenantNotToSueLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 308}, - "avg_character_length": {"test": 402.97}, - }, ) def dataset_transform(self): @@ -1366,10 +1270,6 @@ class CUADEffectiveDateLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 236}, - "avg_character_length": {"test": 277.62}, - }, ) def dataset_transform(self): @@ -1421,10 +1321,6 @@ class CUADExclusivityLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 762}, - "avg_character_length": {"test": 369.17}, - }, ) def dataset_transform(self): @@ -1476,10 +1372,6 @@ class CUADExpirationDateLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 876}, - "avg_character_length": {"test": 309.27}, - }, ) def dataset_transform(self): @@ -1531,10 +1423,6 @@ class CUADGoverningLawLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 876}, - "avg_character_length": {"test": 289.87}, - }, ) def dataset_transform(self): @@ -1586,10 +1474,6 @@ class CUADInsuranceLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 1030}, - "avg_character_length": {"test": 365.54}, - }, ) def dataset_transform(self): @@ -1641,10 +1525,6 @@ class CUADIPOwnershipAssignmentLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 576}, - "avg_character_length": {"test": 414.00}, - }, ) def dataset_transform(self): @@ -1696,10 +1576,6 @@ class CUADIrrevocableOrPerpetualLicenseLegalBenchClassification(AbsTaskClassific year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 280}, - "avg_character_length": {"test": 473.40}, - }, ) def dataset_transform(self): @@ -1751,10 +1627,6 @@ class CUADJointIPOwnershipLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 192}, - "avg_character_length": {"test": 374.17}, - }, ) def dataset_transform(self): @@ -1806,10 +1678,6 @@ class CUADLicenseGrantLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 1396}, - "avg_character_length": {"test": 409.89}, - }, ) def dataset_transform(self): @@ -1861,10 +1729,6 @@ class CUADLiquidatedDamagesLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 220}, - "avg_character_length": {"test": 351.76}, - }, ) def dataset_transform(self): @@ -1916,10 +1780,6 @@ class CUADMinimumCommitmentLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 772}, - "avg_character_length": {"test": 364.16}, - }, ) def dataset_transform(self): @@ -1971,10 +1831,6 @@ class CUADMostFavoredNationLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 64}, - "avg_character_length": {"test": 418.75}, - }, ) def dataset_transform(self): @@ -2026,10 +1882,6 @@ class CUADNoSolicitOfCustomersLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 84}, - "avg_character_length": {"test": 392.89}, - }, ) def dataset_transform(self): @@ -2081,10 +1933,6 @@ class CUADNoSolicitOfEmployeesLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 142}, - "avg_character_length": {"test": 417.94}, - }, ) def dataset_transform(self): @@ -2136,10 +1984,6 @@ class CUADNonCompeteLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 442}, - "avg_character_length": {"test": 383.20}, - }, ) def dataset_transform(self): @@ -2191,10 +2035,6 @@ class CUADNonDisparagementLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 100}, - "avg_character_length": {"test": 403.08}, - }, ) def dataset_transform(self): @@ -2246,10 +2086,6 @@ class CUADNonTransferableLicenseLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 542}, - "avg_character_length": {"test": 399.16}, - }, ) def dataset_transform(self): @@ -2301,10 +2137,6 @@ class CUADNoticePeriodToTerminateRenewalLegalBenchClassification(AbsTaskClassifi year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 222}, - "avg_character_length": {"test": 354.85}, - }, ) def dataset_transform(self): @@ -2356,10 +2188,6 @@ class CUADPostTerminationServicesLegalBenchClassification(AbsTaskClassification) year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 808}, - "avg_character_length": {"test": 422.53}, - }, ) def dataset_transform(self): @@ -2411,10 +2239,6 @@ class CUADPriceRestrictionsLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 46}, - "avg_character_length": {"test": 324.71}, - }, ) def dataset_transform(self): @@ -2466,10 +2290,6 @@ class CUADRenewalTermLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 386}, - "avg_character_length": {"test": 340.87}, - }, ) def dataset_transform(self): @@ -2521,10 +2341,6 @@ class CUADRevenueProfitSharingLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 774}, - "avg_character_length": {"test": 371.55}, - }, ) def dataset_transform(self): @@ -2576,10 +2392,6 @@ class CUADRofrRofoRofnLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 690}, - "avg_character_length": {"test": 395.46}, - }, ) def dataset_transform(self): @@ -2631,10 +2443,6 @@ class CUADSourceCodeEscrowLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 118}, - "avg_character_length": {"test": 399.18}, - }, ) def dataset_transform(self): @@ -2686,10 +2494,6 @@ class CUADTerminationForConvenienceLegalBenchClassification(AbsTaskClassificatio year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 430}, - "avg_character_length": {"test": 326.30}, - }, ) def dataset_transform(self): @@ -2741,10 +2545,6 @@ class CUADThirdPartyBeneficiaryLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 68}, - "avg_character_length": {"test": 261.04}, - }, ) def dataset_transform(self): @@ -2796,10 +2596,6 @@ class CUADUncappedLiabilityLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 294}, - "avg_character_length": {"test": 441.04}, - }, ) def dataset_transform(self): @@ -2851,10 +2647,6 @@ class CUADUnlimitedAllYouCanEatLicenseLegalBenchClassification(AbsTaskClassifica year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 48}, - "avg_character_length": {"test": 368.08}, - }, ) def dataset_transform(self): @@ -2906,10 +2698,6 @@ class CUADVolumeRestrictionLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 322}, - "avg_character_length": {"test": 306.27}, - }, ) def dataset_transform(self): @@ -2961,10 +2749,6 @@ class CUADWarrantyDurationLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 320}, - "avg_character_length": {"test": 352.27}, - }, ) def dataset_transform(self): @@ -3010,10 +2794,6 @@ class DefinitionClassificationLegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 1337}, - "avg_character_length": {"test": 253.72}, - }, ) def dataset_transform(self): @@ -3059,10 +2839,6 @@ class Diversity1LegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 300}, - "avg_character_length": {"test": 103.21}, - }, ) def dataset_transform(self): @@ -3132,10 +2908,6 @@ class Diversity2LegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 300}, - "avg_character_length": {"test": 0}, - }, ) def dataset_transform(self): @@ -3205,10 +2977,6 @@ class Diversity3LegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 300}, - "avg_character_length": {"test": 135.46}, - }, ) def dataset_transform(self): @@ -3278,10 +3046,6 @@ class Diversity4LegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 300}, - "avg_character_length": {"test": 144.52}, - }, ) def dataset_transform(self): @@ -3351,10 +3115,6 @@ class Diversity5LegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 300}, - "avg_character_length": {"test": 174.77}, - }, ) def dataset_transform(self): @@ -3424,10 +3184,6 @@ class Diversity6LegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 300}, - "avg_character_length": {"test": 301.01}, - }, ) def dataset_transform(self): @@ -3505,10 +3261,6 @@ class FunctionOfDecisionSectionLegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 367}, - "avg_character_length": {"test": 551.07}, - }, ) def dataset_transform(self): @@ -3557,10 +3309,6 @@ class InsurancePolicyInterpretationLegalBenchClassification(AbsTaskClassificatio primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 133}, - "avg_character_length": {"test": 521.88}, - }, ) def dataset_transform(self): @@ -3613,10 +3361,6 @@ class InternationalCitizenshipQuestionsLegalBenchClassification(AbsTaskClassific publisher = {Global Citizenship Observatory} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 206.18}, - }, ) def dataset_transform(self): @@ -3667,10 +3411,6 @@ class JCrewBlockerLegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 54}, - "avg_character_length": {"test": 1092.22}, - }, ) def dataset_transform(self): @@ -3724,10 +3464,6 @@ class LearnedHandsBenefitsLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 66}, - "avg_character_length": {"test": 1308.44}, - }, ) def dataset_transform(self): @@ -3781,10 +3517,6 @@ class LearnedHandsBusinessLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 174}, - "avg_character_length": {"test": 1144.51}, - }, ) def dataset_transform(self): @@ -3838,10 +3570,6 @@ class LearnedHandsConsumerLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 614}, - "avg_character_length": {"test": 1277.45}, - }, ) def dataset_transform(self): @@ -3895,10 +3623,6 @@ class LearnedHandsCourtsLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 192}, - "avg_character_length": {"test": 1171.02}, - }, ) def dataset_transform(self): @@ -3952,10 +3676,6 @@ class LearnedHandsCrimeLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 688}, - "avg_character_length": {"test": 1212.90}, - }, ) def dataset_transform(self): @@ -4009,10 +3729,6 @@ class LearnedHandsDivorceLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 150}, - "avg_character_length": {"test": 1242.43}, - }, ) def dataset_transform(self): @@ -4066,10 +3782,6 @@ class LearnedHandsDomesticViolenceLegalBenchClassification(AbsTaskClassification urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 174}, - "avg_character_length": {"test": 1360.83}, - }, ) def dataset_transform(self): @@ -4123,10 +3835,6 @@ class LearnedHandsEducationLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 56}, - "avg_character_length": {"test": 1397.44}, - }, ) def dataset_transform(self): @@ -4180,10 +3888,6 @@ class LearnedHandsEmploymentLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 710}, - "avg_character_length": {"test": 1262.74}, - }, ) def dataset_transform(self): @@ -4237,10 +3941,6 @@ class LearnedHandsEstatesLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 178}, - "avg_character_length": {"test": 1200.70}, - }, ) def dataset_transform(self): @@ -4294,10 +3994,6 @@ class LearnedHandsFamilyLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 1338.27}, - }, ) def dataset_transform(self): @@ -4354,10 +4050,6 @@ class LearnedHandsHealthLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 226}, - "avg_character_length": {"test": 1472.59}, - }, ) def dataset_transform(self): @@ -4411,10 +4103,6 @@ class LearnedHandsHousingLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 1322.54}, - }, ) def dataset_transform(self): @@ -4471,10 +4159,6 @@ class LearnedHandsImmigrationLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 134}, - "avg_character_length": {"test": 1216.31}, - }, ) def dataset_transform(self): @@ -4528,10 +4212,6 @@ class LearnedHandsTortsLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 432}, - "avg_character_length": {"test": 1406.97}, - }, ) def dataset_transform(self): @@ -4585,10 +4265,6 @@ class LearnedHandsTrafficLegalBenchClassification(AbsTaskClassification): urldate = {2022-05-21} } """, - descriptive_stats={ - "n_samples": {"test": 556}, - "avg_character_length": {"test": 1182.91}, - }, ) def dataset_transform(self): @@ -4634,10 +4310,6 @@ class LegalReasoningCausalityLegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": 55}, - "avg_character_length": {"test": 1563.76}, - }, ) def dataset_transform(self): @@ -4867,10 +4539,6 @@ class MAUDLegalBenchClassification(AbsTaskClassification): year={2023} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 1802.93}, - }, ) def load_data(self, **kwargs: Any) -> None: @@ -4972,10 +4640,6 @@ class NYSJudicialEthicsLegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": 292}, - "avg_character_length": {"test": 159.45}, - }, ) def dataset_transform(self): @@ -5030,10 +4694,6 @@ class OPP115DataRetentionLegalBenchClassification(AbsTaskClassification): year={2016} } """, - descriptive_stats={ - "n_samples": {"test": 88}, - "avg_character_length": {"test": 195.20}, - }, ) def dataset_transform(self): @@ -5086,10 +4746,6 @@ class OPP115DataSecurityLegalBenchClassification(AbsTaskClassification): year={2016} } """, - descriptive_stats={ - "n_samples": {"test": 1334}, - "avg_character_length": {"test": 246.69}, - }, ) def dataset_transform(self): @@ -5142,10 +4798,6 @@ class OPP115DoNotTrackLegalBenchClassification(AbsTaskClassification): year={2016} } """, - descriptive_stats={ - "n_samples": {"test": 110}, - "avg_character_length": {"test": 223.16}, - }, ) def dataset_transform(self): @@ -5198,10 +4850,6 @@ class OPP115FirstPartyCollectionUseLegalBenchClassification(AbsTaskClassificatio year={2016} } """, - descriptive_stats={ - "n_samples": {"test": 2086}, - "avg_character_length": {"test": 204.25}, - }, ) def dataset_transform(self): @@ -5256,10 +4904,6 @@ class OPP115InternationalAndSpecificAudiencesLegalBenchClassification( year={2016} } """, - descriptive_stats={ - "n_samples": {"test": 980}, - "avg_character_length": {"test": 327.71}, - }, ) def dataset_transform(self): @@ -5312,10 +4956,6 @@ class OPP115PolicyChangeLegalBenchClassification(AbsTaskClassification): year={2016} } """, - descriptive_stats={ - "n_samples": {"test": 431}, - "avg_character_length": {"test": 200.99}, - }, ) def dataset_transform(self): @@ -5368,10 +5008,6 @@ class OPP115ThirdPartySharingCollectionLegalBenchClassification(AbsTaskClassific year={2016} } """, - descriptive_stats={ - "n_samples": {"test": 1590}, - "avg_character_length": {"test": 223.64}, - }, ) def dataset_transform(self): @@ -5424,10 +5060,6 @@ class OPP115UserAccessEditAndDeletionLegalBenchClassification(AbsTaskClassificat year={2016} } """, - descriptive_stats={ - "n_samples": {"test": 462}, - "avg_character_length": {"test": 218.59}, - }, ) def dataset_transform(self): @@ -5480,10 +5112,6 @@ class OPP115UserChoiceControlLegalBenchClassification(AbsTaskClassification): year={2016} } """, - descriptive_stats={ - "n_samples": {"test": 1546}, - "avg_character_length": {"test": 210.62}, - }, ) def dataset_transform(self): @@ -5537,10 +5165,6 @@ class OralArgumentQuestionPurposeLegalBenchClassification(AbsTaskClassification) primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 312}, - "avg_character_length": {"test": 269.71}, - }, ) def dataset_transform(self): @@ -5589,10 +5213,6 @@ class OverrulingLegalBenchClassification(AbsTaskClassification): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 167.20}, - }, ) def dataset_transform(self): @@ -5641,10 +5261,6 @@ class PersonalJurisdictionLegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 50}, - "avg_character_length": {"test": 381.14}, - }, ) def dataset_transform(self): @@ -5690,10 +5306,6 @@ class PROALegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} }, """, - descriptive_stats={ - "n_samples": {"test": 95}, - "avg_character_length": {"test": 251.73}, - }, ) def dataset_transform(self): @@ -5748,10 +5360,6 @@ class SCDBPAccountabilityLegalBenchClassification(AbsTaskClassification): publisher={HeinOnline} } """, - descriptive_stats={ - "n_samples": {"test": 379}, - "avg_character_length": {"test": 3520}, - }, ) def dataset_transform(self): @@ -5806,10 +5414,6 @@ class SCDBPAuditsLegalBenchClassification(AbsTaskClassification): publisher={HeinOnline} } """, - descriptive_stats={ - "n_samples": {"test": 379}, - "avg_character_length": {"test": 3507}, - }, ) def dataset_transform(self): @@ -5864,10 +5468,6 @@ class SCDBPCertificationLegalBenchClassification(AbsTaskClassification): publisher={HeinOnline} } """, - descriptive_stats={ - "n_samples": {"test": 378}, - "avg_character_length": {"test": 3507}, - }, ) def dataset_transform(self): @@ -5922,10 +5522,6 @@ class SCDBPTrainingLegalBenchClassification(AbsTaskClassification): publisher={HeinOnline} } """, - descriptive_stats={ - "n_samples": {"test": 379}, - "avg_character_length": {"test": 3506}, - }, ) def dataset_transform(self): @@ -5980,10 +5576,6 @@ class SCDBPVerificationLegalBenchClassification(AbsTaskClassification): publisher={HeinOnline} } """, - descriptive_stats={ - "n_samples": {"test": 379}, - "avg_character_length": {"test": 3498}, - }, ) def dataset_transform(self): @@ -6038,10 +5630,6 @@ class SCDDAccountabilityLegalBenchClassification(AbsTaskClassification): publisher={HeinOnline} } """, - descriptive_stats={ - "n_samples": {"test": 378}, - "avg_character_length": {"test": 3522}, - }, ) def dataset_transform(self): @@ -6096,10 +5684,6 @@ class SCDDAuditsLegalBenchClassification(AbsTaskClassification): publisher={HeinOnline} } """, - descriptive_stats={ - "n_samples": {"test": 379}, - "avg_character_length": {"test": 3506}, - }, ) def dataset_transform(self): @@ -6154,10 +5738,6 @@ class SCDDCertificationLegalBenchClassification(AbsTaskClassification): publisher={HeinOnline} } """, - descriptive_stats={ - "n_samples": {"test": 378}, - "avg_character_length": {"test": 3518}, - }, ) def dataset_transform(self): @@ -6212,10 +5792,6 @@ class SCDDTrainingLegalBenchClassification(AbsTaskClassification): publisher={HeinOnline} } """, - descriptive_stats={ - "n_samples": {"test": 379}, - "avg_character_length": {"test": 3499}, - }, ) def dataset_transform(self): @@ -6270,10 +5846,6 @@ class SCDDVerificationLegalBenchClassification(AbsTaskClassification): publisher={HeinOnline} } """, - descriptive_stats={ - "n_samples": {"test": 379}, - "avg_character_length": {"test": 3503}, - }, ) def dataset_transform(self): @@ -6319,10 +5891,6 @@ class TelemarketingSalesRuleLegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": 47}, - "avg_character_length": {"test": 348.29}, - }, ) def dataset_transform(self): @@ -6368,10 +5936,6 @@ class TextualismToolDictionariesLegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": 107}, - "avg_character_length": {"test": 943.23}, - }, ) def dataset_transform(self): @@ -6417,10 +5981,6 @@ class TextualismToolPlainLegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": 165}, - "avg_character_length": {"test": 997.97}, - }, ) def dataset_transform(self): @@ -6466,10 +6026,6 @@ class UCCVCommonLawLegalBenchClassification(AbsTaskClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": 94}, - "avg_character_length": {"test": 114.127}, - }, ) def dataset_transform(self): @@ -6526,10 +6082,6 @@ class UnfairTOSLegalBenchClassification(AbsTaskClassification): publisher={Springer} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 184.69}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/eng/NewsClassification.py b/mteb/tasks/Classification/eng/NewsClassification.py index 1ba06bb94c..e09aa04255 100644 --- a/mteb/tasks/Classification/eng/NewsClassification.py +++ b/mteb/tasks/Classification/eng/NewsClassification.py @@ -41,8 +41,4 @@ class NewsClassification(AbsTaskClassification): volume = {28}, year = {2015} }""", - descriptive_stats={ - "n_samples": {"test": 7600}, - "avg_character_length": {"test": 235.29}, - }, ) diff --git a/mteb/tasks/Classification/eng/PatentClassification.py b/mteb/tasks/Classification/eng/PatentClassification.py index 6ae0eabd58..9f10a8a794 100644 --- a/mteb/tasks/Classification/eng/PatentClassification.py +++ b/mteb/tasks/Classification/eng/PatentClassification.py @@ -45,10 +45,6 @@ class PatentClassification(AbsTaskClassification): pages = "2204--2213", abstract = "Most existing text summarization datasets are compiled from the news domain, where summaries have a flattened discourse structure. In such datasets, summary-worthy content often appears in the beginning of input articles. Moreover, large segments from input articles are present verbatim in their respective summaries. These issues impede the learning and evaluation of systems that can understand an article{'}s global content structure as well as produce abstractive summaries with high compression ratio. In this work, we present a novel dataset, BIGPATENT, consisting of 1.3 million records of U.S. patent documents along with human written abstractive summaries. Compared to existing summarization datasets, BIGPATENT has the following properties: i) summaries contain a richer discourse structure with more recurring entities, ii) salient content is evenly distributed in the input, and iii) lesser and shorter extractive fragments are present in the summaries. Finally, we train and evaluate baselines and popular learning models on BIGPATENT to shed light on new challenges and motivate future directions for summarization research.", }""", - descriptive_stats={ - "n_samples": {"test": 5000}, - "avg_character_length": {"test": 18620.44}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/eng/PoemSentimentClassification.py b/mteb/tasks/Classification/eng/PoemSentimentClassification.py index 54a28138ac..f0110308ee 100644 --- a/mteb/tasks/Classification/eng/PoemSentimentClassification.py +++ b/mteb/tasks/Classification/eng/PoemSentimentClassification.py @@ -37,10 +37,6 @@ class PoemSentimentClassification(AbsTaskClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"validation": 105, "test": 104}, - "avg_character_length": {"validation": 45.3, "test": 42.4}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/eng/ToxicChatClassification.py b/mteb/tasks/Classification/eng/ToxicChatClassification.py index 3ccbc715bf..51dd5066d3 100644 --- a/mteb/tasks/Classification/eng/ToxicChatClassification.py +++ b/mteb/tasks/Classification/eng/ToxicChatClassification.py @@ -45,10 +45,6 @@ class ToxicChatClassification(AbsTaskClassification): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"test": 1427}, - "avg_character_length": {"test": 189.4}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/eng/ToxicConversationsClassification.py b/mteb/tasks/Classification/eng/ToxicConversationsClassification.py index d346f8743d..f99d44534d 100644 --- a/mteb/tasks/Classification/eng/ToxicConversationsClassification.py +++ b/mteb/tasks/Classification/eng/ToxicConversationsClassification.py @@ -36,18 +36,10 @@ class ToxicConversationsClassification(AbsTaskClassification): year = {2019}, url = {https://kaggle.com/competitions/jigsaw-unintended-bias-in-toxicity-classification} }""", - descriptive_stats={ - "n_samples": {"test": 50000}, - "avg_character_length": {"test": 296.6}, - }, + prompt="Classify the given comments as either toxic or not toxic", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 16 - return metadata_dict + samples_per_label = 16 def dataset_transform(self): self.dataset = self.stratified_subsampling( diff --git a/mteb/tasks/Classification/eng/TweetSentimentExtractionClassification.py b/mteb/tasks/Classification/eng/TweetSentimentExtractionClassification.py index c865339e30..d77c44936e 100644 --- a/mteb/tasks/Classification/eng/TweetSentimentExtractionClassification.py +++ b/mteb/tasks/Classification/eng/TweetSentimentExtractionClassification.py @@ -36,15 +36,7 @@ class TweetSentimentExtractionClassification(AbsTaskClassification): year = {2020}, url = {https://kaggle.com/competitions/tweet-sentiment-extraction} }""", - descriptive_stats={ - "n_samples": {"test": 3534}, - "avg_character_length": {"test": 67.8}, - }, + prompt="Classify the sentiment of a given tweet as either positive, negative, or neutral", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = dict(self.metadata) - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 32 - return metadata_dict + samples_per_label = 32 diff --git a/mteb/tasks/Classification/eng/TweetTopicSingleClassification.py b/mteb/tasks/Classification/eng/TweetTopicSingleClassification.py index 946644e38d..6c7d4e2bbb 100644 --- a/mteb/tasks/Classification/eng/TweetTopicSingleClassification.py +++ b/mteb/tasks/Classification/eng/TweetTopicSingleClassification.py @@ -48,10 +48,6 @@ class TweetTopicSingleClassification(AbsTaskClassification): publisher = "International Committee on Computational Linguistics" } """, - descriptive_stats={ - "n_samples": {"test_2021": 1693}, - "avg_character_length": {"test_2021": 167.66}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/eng/YahooAnswersTopicsClassification.py b/mteb/tasks/Classification/eng/YahooAnswersTopicsClassification.py index be3b475548..9369b0f6b1 100644 --- a/mteb/tasks/Classification/eng/YahooAnswersTopicsClassification.py +++ b/mteb/tasks/Classification/eng/YahooAnswersTopicsClassification.py @@ -39,18 +39,9 @@ class YahooAnswersTopicsClassification(AbsTaskClassification): volume = {28}, year = {2015} }""", - descriptive_stats={ - "n_samples": {"test": 60000}, - "avg_character_length": {"test": 346.35}, - }, ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = dict(self.metadata) - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 32 - return metadata_dict + samples_per_label = 32 def dataset_transform(self): self.dataset = self.dataset.remove_columns( diff --git a/mteb/tasks/Classification/eng/YelpReviewFullClassification.py b/mteb/tasks/Classification/eng/YelpReviewFullClassification.py index e945a17408..584d5b5266 100644 --- a/mteb/tasks/Classification/eng/YelpReviewFullClassification.py +++ b/mteb/tasks/Classification/eng/YelpReviewFullClassification.py @@ -39,15 +39,9 @@ class YelpReviewFullClassification(AbsTaskClassification): year = {2015} } """, - descriptive_stats={"n_samples": {"test": 50000}, "avg_character_length": {}}, ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = dict(self.metadata) - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 128 - return metadata_dict + samples_per_label = 128 def dataset_transform(self): self.dataset = self.stratified_subsampling( diff --git a/mteb/tasks/Classification/est/estonian_valence.py b/mteb/tasks/Classification/est/estonian_valence.py index 48b8c58764..3f15ee7925 100644 --- a/mteb/tasks/Classification/est/estonian_valence.py +++ b/mteb/tasks/Classification/est/estonian_valence.py @@ -38,13 +38,6 @@ class EstonianValenceClassification(AbsTaskClassification): url = "https://figshare.com/articles/dataset/Estonian_Valence_Corpus_Eesti_valentsikorpus/24517054", doi = "10.6084/m9.figshare.24517054.v1" }""", - descriptive_stats={ - "n_samples": {"train": 3270, "test": 818}, - "avg_character_length": { - "train": 226.70642201834863, - "test": 231.5085574572127, - }, - }, ) def dataset_transform(self): @@ -58,9 +51,4 @@ def dataset_transform(self): lambda x: {"label": lab2idx[x["label"]]}, remove_columns=["label"] ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = dict(self.metadata) - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 16 - return metadata_dict + samples_per_label = 16 diff --git a/mteb/tasks/Classification/fas/PersianFoodSentimentClassification.py b/mteb/tasks/Classification/fas/PersianFoodSentimentClassification.py index 7ff7aef939..f7389e57bc 100644 --- a/mteb/tasks/Classification/fas/PersianFoodSentimentClassification.py +++ b/mteb/tasks/Classification/fas/PersianFoodSentimentClassification.py @@ -37,10 +37,6 @@ class PersianFoodSentimentClassification(AbsTaskClassification): volume={abs/2005.12515} } """, - descriptive_stats={ - "n_samples": {"validation": TEST_SAMPLES, "test": TEST_SAMPLES}, - "avg_character_length": {"validation": 90.37, "test": 90.58}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py b/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py index 385e71fb2e..3715103ca2 100644 --- a/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py +++ b/mteb/tasks/Classification/fil/FilipinoHateSpeechClassification.py @@ -40,10 +40,6 @@ class FilipinoHateSpeechClassification(AbsTaskClassification): year={2019} } """, - descriptive_stats={ - "n_samples": {"validation": TEST_SAMPLES, "test": TEST_SAMPLES}, - "avg_character_length": {"validation": 88.1, "test": 87.4}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py b/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py index 0c05524caa..d91af36567 100644 --- a/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py +++ b/mteb/tasks/Classification/fil/FilipinoShopeeReviewsClassification.py @@ -35,10 +35,6 @@ class FilipinoShopeeReviewsClassification(AbsTaskClassification): issue={08}, pages={72--82} }""", - descriptive_stats={ - "n_samples": {"validation": 2250, "test": 2250}, - "avg_character_length": {"validation": 143.8, "test": 145.1}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/fin/FinToxicityClassification.py b/mteb/tasks/Classification/fin/FinToxicityClassification.py index b6daff471b..2b582c0143 100644 --- a/mteb/tasks/Classification/fin/FinToxicityClassification.py +++ b/mteb/tasks/Classification/fin/FinToxicityClassification.py @@ -42,10 +42,6 @@ class FinToxicityClassification(AbsTaskClassification): month = may, year = "2023", }""", - descriptive_stats={ - "n_samples": {"train": 2048, "test": 2048}, - "avg_character_length": {"train": 432.63, "test": 401.03}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/fra/FrenchBookReviews.py b/mteb/tasks/Classification/fra/FrenchBookReviews.py index 855e6df775..cb9c7b37c9 100644 --- a/mteb/tasks/Classification/fra/FrenchBookReviews.py +++ b/mteb/tasks/Classification/fra/FrenchBookReviews.py @@ -28,10 +28,6 @@ class FrenchBookReviews(AbsTaskClassification): sample_creation="found", bibtex_citation=""" """, - descriptive_stats={ - "n_samples": {"train": 2048}, - "avg_character_length": {"train": 311.5}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/fra/MovieReviewSentimentClassification.py b/mteb/tasks/Classification/fra/MovieReviewSentimentClassification.py index b88dd76d57..ea1971a715 100644 --- a/mteb/tasks/Classification/fra/MovieReviewSentimentClassification.py +++ b/mteb/tasks/Classification/fra/MovieReviewSentimentClassification.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 1024 - class MovieReviewSentimentClassification(AbsTaskClassification): metadata = TaskMetadata( @@ -36,10 +34,6 @@ class MovieReviewSentimentClassification(AbsTaskClassification): year = {2020}, } """, - descriptive_stats={ - "n_samples": {"validation": N_SAMPLES, "test": N_SAMPLES}, - "avg_character_length": {"validation": 550.3, "test": 558.1}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/guj/GujaratiNewsClassification.py b/mteb/tasks/Classification/guj/GujaratiNewsClassification.py index a11c26b4d5..0c93c0dd21 100644 --- a/mteb/tasks/Classification/guj/GujaratiNewsClassification.py +++ b/mteb/tasks/Classification/guj/GujaratiNewsClassification.py @@ -27,10 +27,6 @@ class GujaratiNewsClassification(AbsTaskClassification): dialect=[], sample_creation="found", bibtex_citation="", # none found - descriptive_stats={ - "n_samples": {"train": 5269, "test": 1318}, - "avg_character_length": {"train": 61.95, "test": 61.91}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py b/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py index e1ada66712..a4162801b3 100644 --- a/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py +++ b/mteb/tasks/Classification/heb/HebrewSentimentAnalysis.py @@ -43,10 +43,6 @@ class HebrewSentimentAnalysis(AbsTaskClassification): pages = "2242--2252" } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 113.57}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/hin/HindiDiscourseClassification.py b/mteb/tasks/Classification/hin/HindiDiscourseClassification.py index e68ee15b2b..936fabe2cd 100644 --- a/mteb/tasks/Classification/hin/HindiDiscourseClassification.py +++ b/mteb/tasks/Classification/hin/HindiDiscourseClassification.py @@ -50,10 +50,6 @@ class HindiDiscourseClassification(AbsTaskClassification): language = "English", ISBN = "979-10-95546-34-4", }""", - descriptive_stats={ - "n_samples": {"train": 2048}, - "avg_character_length": {"train": 79.23828125}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/hin/SentimentAnalysisHindi.py b/mteb/tasks/Classification/hin/SentimentAnalysisHindi.py index 8465b7d142..3699f0c35d 100644 --- a/mteb/tasks/Classification/hin/SentimentAnalysisHindi.py +++ b/mteb/tasks/Classification/hin/SentimentAnalysisHindi.py @@ -33,10 +33,6 @@ class SentimentAnalysisHindi(AbsTaskClassification): publisher = {Hugging Face}, journal = {Hugging Face repository}, howpublished = {{https://huggingface.co/OdiaGenAI}}, } """, - descriptive_stats={ - "n_samples": {"train": 2497}, - "avg_character_length": {"train": 81.29}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/hrv/FrenkHrClassification.py b/mteb/tasks/Classification/hrv/FrenkHrClassification.py index c05aab9579..c646f863ae 100644 --- a/mteb/tasks/Classification/hrv/FrenkHrClassification.py +++ b/mteb/tasks/Classification/hrv/FrenkHrClassification.py @@ -36,8 +36,4 @@ class FrenkHrClassification(AbsTaskClassification): primaryClass={cs.CL}, url={https://arxiv.org/abs/1906.02045} }""", - descriptive_stats={ - "n_samples": {"test": 2120}, - "avg_character_length": {"test": 89.86}, - }, ) diff --git a/mteb/tasks/Classification/ind/IndonesianIdClickbaitClassification.py b/mteb/tasks/Classification/ind/IndonesianIdClickbaitClassification.py index 7b318c406a..9fece9e214 100644 --- a/mteb/tasks/Classification/ind/IndonesianIdClickbaitClassification.py +++ b/mteb/tasks/Classification/ind/IndonesianIdClickbaitClassification.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 2048 - class IndonesianIdClickbaitClassification(AbsTaskClassification): metadata = TaskMetadata( @@ -43,10 +41,6 @@ class IndonesianIdClickbaitClassification(AbsTaskClassification): abstract = "News analysis is a popular task in Natural Language Processing (NLP). In particular, the problem of clickbait in news analysis has gained attention in recent years [1, 2]. However, the majority of the tasks has been focused on English news, in which there is already a rich representative resource. For other languages, such as Indonesian, there is still a lack of resource for clickbait tasks. Therefore, we introduce the CLICK-ID dataset of Indonesian news headlines extracted from 12 Indonesian online news publishers. It is comprised of 15,000 annotated headlines with clickbait and non-clickbait labels. Using the CLICK-ID dataset, we then developed an Indonesian clickbait classification model achieving favourable performance. We believe that this corpus will be useful for replicable experiments in clickbait detection or other experiments in NLP areas." } """, - descriptive_stats={ - "n_samples": {"train": N_SAMPLES}, - "avg_character_length": {"train": 64.28}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ind/IndonesianMongabayConservationClassification.py b/mteb/tasks/Classification/ind/IndonesianMongabayConservationClassification.py index 331f864e62..91e54bc137 100644 --- a/mteb/tasks/Classification/ind/IndonesianMongabayConservationClassification.py +++ b/mteb/tasks/Classification/ind/IndonesianMongabayConservationClassification.py @@ -54,10 +54,6 @@ class IndonesianMongabayConservationClassification(AbsTaskClassification): pages = "30--54", } """, - descriptive_stats={ - "n_samples": {"validation": 984, "test": 970}, - "avg_character_length": {"validation": 1675.8, "test": 1675.5}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ita/ItaCaseholdClassification.py b/mteb/tasks/Classification/ita/ItaCaseholdClassification.py index 866802eede..837383ff69 100644 --- a/mteb/tasks/Classification/ita/ItaCaseholdClassification.py +++ b/mteb/tasks/Classification/ita/ItaCaseholdClassification.py @@ -45,10 +45,6 @@ class ItaCaseholdClassification(AbsTaskClassification): series = {ICAIL '23} } """, - descriptive_stats={ - "n_samples": {"test": 221}, - "avg_character_length": {"test": 4207.9}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ita/ItalianLinguistAcceptabilityClassification.py b/mteb/tasks/Classification/ita/ItalianLinguistAcceptabilityClassification.py index 73c317d391..9509f4d9ed 100644 --- a/mteb/tasks/Classification/ita/ItalianLinguistAcceptabilityClassification.py +++ b/mteb/tasks/Classification/ita/ItalianLinguistAcceptabilityClassification.py @@ -44,10 +44,6 @@ class ItalianLinguisticAcceptabilityClassification(AbsTaskClassification): pages = "2929--2940" } """, - descriptive_stats={ - "n_samples": {"train": 7801, "test": 975}, - "avg_character_length": {"train": 35.95, "test": 36.67}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/jav/JavaneseIMDBClassification.py b/mteb/tasks/Classification/jav/JavaneseIMDBClassification.py index 8cff5c0b85..bc79f0b851 100644 --- a/mteb/tasks/Classification/jav/JavaneseIMDBClassification.py +++ b/mteb/tasks/Classification/jav/JavaneseIMDBClassification.py @@ -37,10 +37,6 @@ class JavaneseIMDBClassification(AbsTaskClassification): organization={IEEE} } """, - descriptive_stats={ - "n_samples": {"test": 25_000}, - "avg_character_length": {"test": 481.83}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/jpn/WRIMEClassification.py b/mteb/tasks/Classification/jpn/WRIMEClassification.py index a7fd229dde..623a266177 100644 --- a/mteb/tasks/Classification/jpn/WRIMEClassification.py +++ b/mteb/tasks/Classification/jpn/WRIMEClassification.py @@ -54,10 +54,6 @@ class WRIMEClassification(AbsTaskClassification): pages = "2095--2104", abstract = "We annotate 17,000 SNS posts with both the writer{'}s subjective emotional intensity and the reader{'}s objective one to construct a Japanese emotion analysis dataset. In this study, we explore the difference between the emotional intensity of the writer and that of the readers with this dataset. We found that the reader cannot fully detect the emotions of the writer, especially anger and trust. In addition, experimental results in estimating the emotional intensity show that it is more difficult to estimate the writer{'}s subjective labels than the readers{'}. The large gap between the subjective and objective emotions imply the complexity of the mapping from a post to the subjective emotion intensities, which also leads to a lower performance with machine learning models.", }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 47.78}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/kan/KannadaNewsClassification.py b/mteb/tasks/Classification/kan/KannadaNewsClassification.py index 0e6bf8ea80..f005e56518 100644 --- a/mteb/tasks/Classification/kan/KannadaNewsClassification.py +++ b/mteb/tasks/Classification/kan/KannadaNewsClassification.py @@ -33,10 +33,6 @@ class KannadaNewsClassification(AbsTaskClassification): year={2020}, journal={arXiv preprint arXiv:2005.00085}, }""", - descriptive_stats={ - "n_samples": {"train": 6460}, - "avg_character_length": {"train": 65.88}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/kat/GeorgianSentimentClassification.py b/mteb/tasks/Classification/kat/GeorgianSentimentClassification.py index 00481fb835..e34d148a36 100644 --- a/mteb/tasks/Classification/kat/GeorgianSentimentClassification.py +++ b/mteb/tasks/Classification/kat/GeorgianSentimentClassification.py @@ -55,8 +55,4 @@ class GeorgianSentimentClassification(AbsTaskClassification): abstract = "This paper presents, to the best of our knowledge, the first ever publicly available annotated dataset for sentiment classification and semantic polarity dictionary for Georgian. The characteristics of these resources and the process of their creation are described in detail. The results of various experiments on the performance of both lexicon- and machine learning-based models for Georgian sentiment classification are also reported. Both 3-label (positive, neutral, negative) and 4-label settings (same labels + mixed) are considered. The machine learning models explored include, i.a., logistic regression, SVMs, and transformed-based models. We also explore transfer learning- and translation-based (to a well-supported language) approaches. The obtained results for Georgian are on par with the state-of-the-art results in sentiment classification for well studied languages when using training data of comparable size.", } """, - descriptive_stats={ - "n_samples": {"train": 330, "test": 1200}, - "avg_character_length": {"train": 114.26, "test": 118.06}, - }, ) diff --git a/mteb/tasks/Classification/kor/KlueTC.py b/mteb/tasks/Classification/kor/KlueTC.py index a9e31046aa..8536927d49 100644 --- a/mteb/tasks/Classification/kor/KlueTC.py +++ b/mteb/tasks/Classification/kor/KlueTC.py @@ -35,10 +35,6 @@ class KlueTC(AbsTaskClassification): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"validation": 2048}, - "avg_character_length": {"validation": 27.079609091907326}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/kor/KorFin.py b/mteb/tasks/Classification/kor/KorFin.py index 9c439e51b4..a22b7d5cfe 100644 --- a/mteb/tasks/Classification/kor/KorFin.py +++ b/mteb/tasks/Classification/kor/KorFin.py @@ -39,10 +39,6 @@ class KorFin(AbsTaskClassification): year={2023} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 75.28}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/kor/KorHateClassification.py b/mteb/tasks/Classification/kor/KorHateClassification.py index 816a2bb1b1..38b1c23b94 100644 --- a/mteb/tasks/Classification/kor/KorHateClassification.py +++ b/mteb/tasks/Classification/kor/KorHateClassification.py @@ -43,10 +43,6 @@ class KorHateClassification(AbsTaskClassification): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"train": 2048, "test": 471}, - "avg_character_length": {"train": 38.57, "test": 38.86}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/kor/KorSarcasmClassification.py b/mteb/tasks/Classification/kor/KorSarcasmClassification.py index a09eaf9786..f5a51aeb5b 100644 --- a/mteb/tasks/Classification/kor/KorSarcasmClassification.py +++ b/mteb/tasks/Classification/kor/KorSarcasmClassification.py @@ -18,8 +18,7 @@ class KorSarcasmClassification(AbsTaskClassification): """, dataset={ "path": "SpellOnYou/kor_sarcasm", - "revision": "8079d24b9f1278c6fbc992921c1271457a1064ff", - "trust_remote_code": True, + "revision": "3d96e36e10a88d5b7a3f617cf8362d997504494b", }, reference="https://github.com/SpellOnYou/korean-sarcasm", type="Classification", @@ -45,10 +44,6 @@ class KorSarcasmClassification(AbsTaskClassification): howpublished = {https://github.com/SpellOnYou/korean-sarcasm} } """, - descriptive_stats={ - "n_samples": {"train": 2048, "test": 301}, - "avg_character_length": {"train": 48.45, "test": 46.77}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/kur/KurdishSentimentClassification.py b/mteb/tasks/Classification/kur/KurdishSentimentClassification.py index 4db5a52616..2f9564caff 100644 --- a/mteb/tasks/Classification/kur/KurdishSentimentClassification.py +++ b/mteb/tasks/Classification/kur/KurdishSentimentClassification.py @@ -37,8 +37,4 @@ class KurdishSentimentClassification(AbsTaskClassification): doi = {10.1007/s10579-023-09716-6} } """, - descriptive_stats={ - "n_samples": {"train": 6000, "test": 1987}, - "avg_character_length": {"train": 59.38, "test": 56.11}, - }, ) diff --git a/mteb/tasks/Classification/mal/MalayalamNewsClassification.py b/mteb/tasks/Classification/mal/MalayalamNewsClassification.py index 8a5c3b3772..e454700717 100644 --- a/mteb/tasks/Classification/mal/MalayalamNewsClassification.py +++ b/mteb/tasks/Classification/mal/MalayalamNewsClassification.py @@ -32,10 +32,6 @@ class MalayalamNewsClassification(AbsTaskClassification): year={2020}, journal={arXiv preprint arXiv:2005.00085}, }""", - descriptive_stats={ - "n_samples": {"train": 5036, "test": 1260}, - "avg_character_length": {"train": 79.48, "test": 80.44}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/mar/MarathiNewsClassification.py b/mteb/tasks/Classification/mar/MarathiNewsClassification.py index 18aff925cb..7fa104c444 100644 --- a/mteb/tasks/Classification/mar/MarathiNewsClassification.py +++ b/mteb/tasks/Classification/mar/MarathiNewsClassification.py @@ -32,10 +32,6 @@ class MarathiNewsClassification(AbsTaskClassification): year={2020}, journal={arXiv preprint arXiv:2005.00085}, }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 52.37}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/mkd/MacedonianTweetSentimentClassification.py b/mteb/tasks/Classification/mkd/MacedonianTweetSentimentClassification.py index f66105763c..58a555c6b1 100644 --- a/mteb/tasks/Classification/mkd/MacedonianTweetSentimentClassification.py +++ b/mteb/tasks/Classification/mkd/MacedonianTweetSentimentClassification.py @@ -42,8 +42,4 @@ class MacedonianTweetSentimentClassification(AbsTaskClassification): url = "https://aclanthology.org/R15-1034", pages = "249--257", }""", - descriptive_stats={ - "n_samples": {"test": 1139}, - "avg_character_length": {"test": 67.6}, - }, ) diff --git a/mteb/tasks/Classification/multilingual/AfriSentiClassification.py b/mteb/tasks/Classification/multilingual/AfriSentiClassification.py index c21f8c5e50..8a4a79d68b 100644 --- a/mteb/tasks/Classification/multilingual/AfriSentiClassification.py +++ b/mteb/tasks/Classification/multilingual/AfriSentiClassification.py @@ -57,10 +57,6 @@ class AfriSentiClassification(MultilingualTask, AbsTaskClassification): author=Shamsuddeen Hassan Muhammad and Idris Abdulmumin and Abinew Ali Ayele and Nedjma Ousidhoum and David Ifeoluwa Adelani and Seid Muhie Yimam and Ibrahim Sa'id Ahmad and Meriem Beloucif and Saif Mohammad and Sebastian Ruder and Oumaima Hourrane and Pavel Brazdil and Felermino D'ario M'ario Ant'onio Ali and Davis Davis and Salomey Osei and Bello Shehu Bello and Falalu Ibrahim and Tajuddeen Gwadabe and Samuel Rutunda and Tadesse Belay and Wendimu Baye Messelle and Hailu Beshada Balcha and Sisay Adugna Chala and Hagos Tesfahun Gebremichael and Bernard Opoku and Steven Arthur, year=2023 }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 74.77}, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Classification/multilingual/AfriSentiLangClassification.py b/mteb/tasks/Classification/multilingual/AfriSentiLangClassification.py index d434440bf9..f11c1b32c2 100644 --- a/mteb/tasks/Classification/multilingual/AfriSentiLangClassification.py +++ b/mteb/tasks/Classification/multilingual/AfriSentiLangClassification.py @@ -41,18 +41,9 @@ class AfriSentiLangClassification(AbsTaskClassification): sample_creation="found", bibtex_citation=""" """, - descriptive_stats={ - "n_samples": {"test": 5754}, - "avg_character_length": {"test": 77.84}, - }, ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 32 - return metadata_dict + samples_per_label = 32 def dataset_transform(self): self.dataset = self.dataset.rename_column("tweet", "text") diff --git a/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py b/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py index 21adc105eb..112d4e0b27 100644 --- a/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py +++ b/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py @@ -59,15 +59,7 @@ class AmazonCounterfactualClassification(MultilingualTask, AbsTaskClassification pages = "7092--7108", abstract = "Counterfactual statements describe events that did not or cannot take place. We consider the problem of counterfactual detection (CFD) in product reviews. For this purpose, we annotate a multilingual CFD dataset from Amazon product reviews covering counterfactual statements written in English, German, and Japanese languages. The dataset is unique as it contains counterfactuals in multiple languages, covers a new application area of e-commerce reviews, and provides high quality professional annotations. We train CFD models using different text representation methods and classifiers. We find that these models are robust against the selectional biases introduced due to cue phrase-based sentence selection. Moreover, our CFD dataset is compatible with prior datasets and can be merged to learn accurate CFD models. Applying machine translation on English counterfactual examples to create multilingual data performs poorly, demonstrating the language-specificity of this problem, which has been ignored so far.", }""", - descriptive_stats={ - "n_samples": {"validation": 335, "test": 670}, - "avg_character_length": {"validation": 109.2, "test": 106.1}, - }, + prompt="Classify a given Amazon customer review text as either counterfactual or not-counterfactual", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 32 - return metadata_dict + samples_per_label = 32 diff --git a/mteb/tasks/Classification/multilingual/AmazonReviewsClassification.py b/mteb/tasks/Classification/multilingual/AmazonReviewsClassification.py index 97422af11a..774ad9f01d 100644 --- a/mteb/tasks/Classification/multilingual/AmazonReviewsClassification.py +++ b/mteb/tasks/Classification/multilingual/AmazonReviewsClassification.py @@ -43,8 +43,5 @@ class AmazonReviewsClassification(MultilingualTask, AbsTaskClassification): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"validation": 30000, "test": 30000}, - "avg_character_length": {"validation": 159.2, "test": 160.4}, - }, + prompt="Classify the given Amazon review into its appropriate rating category", ) diff --git a/mteb/tasks/Classification/multilingual/CataloniaTweetClassification.py b/mteb/tasks/Classification/multilingual/CataloniaTweetClassification.py index bc78dbcb20..c21fee9cfa 100644 --- a/mteb/tasks/Classification/multilingual/CataloniaTweetClassification.py +++ b/mteb/tasks/Classification/multilingual/CataloniaTweetClassification.py @@ -64,10 +64,6 @@ class CataloniaTweetClassification(MultilingualTask, AbsTaskClassification): pages = "1368--1375", ISBN = "979-10-95546-34-4", }""", - descriptive_stats={ - "n_samples": {"validation": 2000, "test": 2000}, - "avg_character_length": {"validation": 202.61, "test": 200.49}, - }, ) def dataset_transform(self): @@ -79,6 +75,5 @@ def dataset_transform(self): self.dataset[lang], seed=self.seed, splits=["validation", "test"], - n_samples=2000, ) self.dataset[lang] = self.dataset[lang].remove_columns(["id_str"]) diff --git a/mteb/tasks/Classification/multilingual/CyrillicTurkicLangClassification.py b/mteb/tasks/Classification/multilingual/CyrillicTurkicLangClassification.py index 4c234bee96..3c0d2ca2a2 100644 --- a/mteb/tasks/Classification/multilingual/CyrillicTurkicLangClassification.py +++ b/mteb/tasks/Classification/multilingual/CyrillicTurkicLangClassification.py @@ -44,10 +44,6 @@ class CyrillicTurkicLangClassification(AbsTaskClassification): year={2012} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 92.22}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/multilingual/HinDialectClassification.py b/mteb/tasks/Classification/multilingual/HinDialectClassification.py index af19ef5d14..c9d6b36669 100644 --- a/mteb/tasks/Classification/multilingual/HinDialectClassification.py +++ b/mteb/tasks/Classification/multilingual/HinDialectClassification.py @@ -3,29 +3,29 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -_LANGUAGES = { - "pan": ["pan-Guru"], - "bgc": ["bgc-Deva"], - "mag": ["mag-Deva"], - "bns": ["bns-Deva"], - "kfq": ["kfg-Deva"], - "noe": ["noe-Deva"], - "bhb": ["bhb-Deva"], - "bho": ["bho-Deva"], - "gbm": ["gbm-Deva"], - "mup": ["mup-Deva"], - "anp": ["anp-Deva"], - "hne": ["hne-Deva"], - "bra": ["bra-Deva"], - "raj": ["raj-Deva"], - "awa": ["awa-Deva"], - "guj": ["guj-Gujr"], - "ben": ["ben-Beng"], - "bhd": ["bhd-Deva"], - "kfy": ["kfy-Deva"], - "mar": ["mar-Deva"], - "bjj": ["bjj-Deva"], -} +_LANGUAGES = [ + "pan-Guru", + "bgc-Deva", + "mag-Deva", + "bns-Deva", + "kfg-Deva", + "noe-Deva", + "bhb-Deva", + "bho-Deva", + "gbm-Deva", + "mup-Deva", + "anp-Deva", + "hne-Deva", + "bra-Deva", + "raj-Deva", + "awa-Deva", + "guj-Gujr", + "ben-Beng", + "bhd-Deva", + "kfy-Deva", + "mar-Deva", + "bjj-Deva", +] class HinDialectClassification(AbsTaskClassification): @@ -59,10 +59,6 @@ class HinDialectClassification(AbsTaskClassification): copyright = {Creative Commons - Attribution-{NonCommercial}-{ShareAlike} 4.0 International ({CC} {BY}-{NC}-{SA} 4.0)}, year = {2022} } """, - descriptive_stats={ - "n_samples": {"test": 1152}, - "avg_character_length": {"test": 583.82}, - }, ) def dataset_transform(self) -> None: diff --git a/mteb/tasks/Classification/multilingual/IndicLangClassification.py b/mteb/tasks/Classification/multilingual/IndicLangClassification.py index 407d472253..47564cf501 100644 --- a/mteb/tasks/Classification/multilingual/IndicLangClassification.py +++ b/mteb/tasks/Classification/multilingual/IndicLangClassification.py @@ -101,10 +101,6 @@ class IndicLangClassification(AbsTaskClassification): doi = "10.18653/v1/2023.acl-short.71", pages = "816--826" }""", - descriptive_stats={ - "n_samples": {"test": 30418}, - "avg_character_length": {"test": 106.5}, - }, ) def load_data(self, **kwargs: Any) -> None: diff --git a/mteb/tasks/Classification/multilingual/IndicNLPNewsClassification.py b/mteb/tasks/Classification/multilingual/IndicNLPNewsClassification.py index 4f67133241..3995917696 100644 --- a/mteb/tasks/Classification/multilingual/IndicNLPNewsClassification.py +++ b/mteb/tasks/Classification/multilingual/IndicNLPNewsClassification.py @@ -45,10 +45,6 @@ class IndicNLPNewsClassification(MultilingualTask, AbsTaskClassification): year={2020}, journal={arXiv preprint arXiv:2005.00085} }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 1169.053974484789}, - }, ) def dataset_transform(self): @@ -61,7 +57,6 @@ def dataset_transform(self): if self.dataset[lang]["test"].num_rows > 2048: self.dataset[lang] = self.stratified_subsampling( self.dataset[lang], - n_samples=2048, seed=self.seed, splits=["test"], ) diff --git a/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py b/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py index bd9058918b..2687422935 100644 --- a/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py +++ b/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py @@ -51,10 +51,6 @@ class IndicSentimentClassification(MultilingualTask, AbsTaskClassification): year = {2022}, doi = {10.18653/v1/2023.acl-long.693} }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": {"test": 137.6}, - }, ) def dataset_transform(self) -> None: diff --git a/mteb/tasks/Classification/multilingual/LanguageClassification.py b/mteb/tasks/Classification/multilingual/LanguageClassification.py index 2d0514578c..9ebcfc7406 100644 --- a/mteb/tasks/Classification/multilingual/LanguageClassification.py +++ b/mteb/tasks/Classification/multilingual/LanguageClassification.py @@ -64,63 +64,6 @@ class LanguageClassification(AbsTaskClassification): publisher = {Association for Computational Linguistics}, location = {Brussels, Belgium}, }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "test": { - "num_samples": 2048, - "average_text_length": 109.546875, - "unique_labels": 20, - "labels": { - "17": {"count": 102}, - "0": {"count": 102}, - "11": {"count": 102}, - "4": {"count": 103}, - "3": {"count": 102}, - "1": {"count": 102}, - "10": {"count": 102}, - "2": {"count": 103}, - "16": {"count": 103}, - "9": {"count": 103}, - "5": {"count": 102}, - "7": {"count": 102}, - "13": {"count": 102}, - "14": {"count": 103}, - "12": {"count": 102}, - "15": {"count": 103}, - "19": {"count": 102}, - "18": {"count": 102}, - "6": {"count": 103}, - "8": {"count": 103}, - }, - }, - "train": { - "num_samples": 70000, - "average_text_length": 110.86141428571429, - "unique_labels": 20, - "labels": { - "12": {"count": 3500}, - "1": {"count": 3500}, - "19": {"count": 3500}, - "15": {"count": 3500}, - "13": {"count": 3500}, - "11": {"count": 3500}, - "17": {"count": 3500}, - "14": {"count": 3500}, - "16": {"count": 3500}, - "5": {"count": 3500}, - "0": {"count": 3500}, - "8": {"count": 3500}, - "7": {"count": 3500}, - "2": {"count": 3500}, - "3": {"count": 3500}, - "10": {"count": 3500}, - "6": {"count": 3500}, - "18": {"count": 3500}, - "4": {"count": 3500}, - "9": {"count": 3500}, - }, - }, - }, ) def dataset_transform(self) -> None: diff --git a/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py b/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py index 4f5c793aeb..eb8713fd6d 100644 --- a/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py +++ b/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py @@ -59,388 +59,5 @@ class MTOPDomainClassification(MultilingualTask, AbsTaskClassification): abstract = "Scaling semantic parsing models for task-oriented dialog systems to new languages is often expensive and time-consuming due to the lack of available datasets. Available datasets suffer from several shortcomings: a) they contain few languages b) they contain small amounts of labeled examples per language c) they are based on the simple intent and slot detection paradigm for non-compositional queries. In this paper, we present a new multilingual dataset, called MTOP, comprising of 100k annotated utterances in 6 languages across 11 domains. We use this dataset and other publicly available datasets to conduct a comprehensive benchmarking study on using various state-of-the-art multilingual pre-trained models for task-oriented semantic parsing. We achieve an average improvement of +6.3 points on Slot F1 for the two existing multilingual datasets, over best results reported in their experiments. Furthermore, we demonstrate strong zero-shot performance using pre-trained models combined with automatic translation and alignment, and a proposed distant supervision method to reduce the noise in slot label projection.", } """, - descriptive_stats={ - "n_samples": {"validation": 2235, "test": 4386}, - "validation": { - "num_samples": 10837, - "average_text_length": 39.85374181046415, - "unique_labels": 11, - "labels": { - "1": {"count": 1688}, - "10": {"count": 754}, - "7": {"count": 849}, - "3": {"count": 681}, - "6": {"count": 985}, - "2": {"count": 647}, - "9": {"count": 872}, - "0": {"count": 833}, - "5": {"count": 1182}, - "4": {"count": 982}, - "8": {"count": 1364}, - }, - "hf_subset_descriptive_stats": {}, - "en": { - "num_samples": 2235, - "average_text_length": 36.53825503355705, - "unique_labels": 11, - "labels": { - "1": {"count": 329}, - "10": {"count": 185}, - "7": {"count": 183}, - "3": {"count": 134}, - "6": {"count": 186}, - "2": {"count": 123}, - "9": {"count": 196}, - "0": {"count": 176}, - "5": {"count": 228}, - "4": {"count": 207}, - "8": {"count": 288}, - }, - }, - "de": { - "num_samples": 1815, - "average_text_length": 42.824793388429754, - "unique_labels": 11, - "labels": { - "0": {"count": 99}, - "1": {"count": 303}, - "2": {"count": 104}, - "3": {"count": 122}, - "6": {"count": 165}, - "4": {"count": 157}, - "7": {"count": 141}, - "5": {"count": 203}, - "8": {"count": 220}, - "10": {"count": 133}, - "9": {"count": 168}, - }, - }, - "es": { - "num_samples": 1527, - "average_text_length": 44.34839554682384, - "unique_labels": 11, - "labels": { - "1": {"count": 197}, - "6": {"count": 166}, - "4": {"count": 138}, - "10": {"count": 103}, - "3": {"count": 104}, - "5": {"count": 190}, - "2": {"count": 115}, - "8": {"count": 212}, - "7": {"count": 82}, - "9": {"count": 76}, - "0": {"count": 144}, - }, - }, - "fr": { - "num_samples": 1577, - "average_text_length": 43.12492073557387, - "unique_labels": 11, - "labels": { - "0": {"count": 125}, - "1": {"count": 278}, - "2": {"count": 92}, - "3": {"count": 89}, - "4": {"count": 137}, - "7": {"count": 145}, - "6": {"count": 138}, - "5": {"count": 168}, - "8": {"count": 203}, - "9": {"count": 124}, - "10": {"count": 78}, - }, - }, - "hi": { - "num_samples": 2012, - "average_text_length": 39.139662027833005, - "unique_labels": 11, - "labels": { - "0": {"count": 161}, - "1": {"count": 304}, - "3": {"count": 126}, - "4": {"count": 193}, - "2": {"count": 109}, - "10": {"count": 154}, - "5": {"count": 208}, - "6": {"count": 167}, - "7": {"count": 172}, - "8": {"count": 235}, - "9": {"count": 183}, - }, - }, - "th": { - "num_samples": 1671, - "average_text_length": 34.726511071214844, - "unique_labels": 11, - "labels": { - "0": {"count": 128}, - "1": {"count": 277}, - "2": {"count": 104}, - "3": {"count": 106}, - "4": {"count": 150}, - "5": {"count": 185}, - "6": {"count": 163}, - "7": {"count": 126}, - "8": {"count": 206}, - "9": {"count": 125}, - "10": {"count": 101}, - }, - }, - }, - "test": { - "num_samples": 19680, - "average_text_length": 39.71443089430894, - "unique_labels": 11, - "labels": { - "2": {"count": 977}, - "5": {"count": 2372}, - "6": {"count": 2014}, - "8": {"count": 2572}, - "9": {"count": 1317}, - "1": {"count": 3065}, - "10": {"count": 1330}, - "3": {"count": 1351}, - "0": {"count": 1459}, - "7": {"count": 1535}, - "4": {"count": 1688}, - }, - "hf_subset_descriptive_stats": {}, - "en": { - "num_samples": 4386, - "average_text_length": 36.79343365253078, - "unique_labels": 11, - "labels": { - "2": {"count": 197}, - "5": {"count": 487}, - "6": {"count": 418}, - "8": {"count": 613}, - "9": {"count": 346}, - "1": {"count": 613}, - "10": {"count": 358}, - "3": {"count": 290}, - "0": {"count": 341}, - "7": {"count": 354}, - "4": {"count": 369}, - }, - }, - "de": { - "num_samples": 3549, - "average_text_length": 42.67258382642998, - "unique_labels": 11, - "labels": { - "0": {"count": 193}, - "10": {"count": 264}, - "1": {"count": 553}, - "2": {"count": 163}, - "3": {"count": 256}, - "5": {"count": 439}, - "4": {"count": 306}, - "6": {"count": 353}, - "7": {"count": 279}, - "8": {"count": 452}, - "9": {"count": 291}, - }, - }, - "es": { - "num_samples": 2998, - "average_text_length": 43.552034689793196, - "unique_labels": 11, - "labels": { - "1": {"count": 401}, - "6": {"count": 352}, - "4": {"count": 246}, - "10": {"count": 206}, - "3": {"count": 231}, - "5": {"count": 404}, - "2": {"count": 177}, - "8": {"count": 435}, - "7": {"count": 156}, - "9": {"count": 126}, - "0": {"count": 264}, - }, - }, - "fr": { - "num_samples": 3193, - "average_text_length": 43.854995302223614, - "unique_labels": 11, - "labels": { - "0": {"count": 253}, - "1": {"count": 551}, - "2": {"count": 159}, - "3": {"count": 190}, - "4": {"count": 280}, - "6": {"count": 330}, - "5": {"count": 356}, - "7": {"count": 272}, - "8": {"count": 462}, - "10": {"count": 159}, - "9": {"count": 181}, - }, - }, - "hi": { - "num_samples": 2789, - "average_text_length": 37.395123700250984, - "unique_labels": 11, - "labels": { - "0": {"count": 208}, - "1": {"count": 470}, - "5": {"count": 335}, - "3": {"count": 195}, - "4": {"count": 242}, - "2": {"count": 132}, - "6": {"count": 267}, - "7": {"count": 262}, - "8": {"count": 265}, - "10": {"count": 186}, - "9": {"count": 227}, - }, - }, - "th": { - "num_samples": 2765, - "average_text_length": 33.94792043399638, - "unique_labels": 11, - "labels": { - "0": {"count": 200}, - "1": {"count": 477}, - "2": {"count": 149}, - "3": {"count": 189}, - "4": {"count": 245}, - "6": {"count": 294}, - "5": {"count": 351}, - "7": {"count": 212}, - "8": {"count": 345}, - "9": {"count": 146}, - "10": {"count": 157}, - }, - }, - }, - "train": { - "num_samples": 73928, - "average_text_length": 39.73095444215994, - "unique_labels": 11, - "labels": { - "0": {"count": 5262}, - "5": {"count": 8334}, - "6": {"count": 6961}, - "9": {"count": 5313}, - "1": {"count": 11107}, - "8": {"count": 9698}, - "10": {"count": 5084}, - "2": {"count": 4770}, - "4": {"count": 6644}, - "3": {"count": 5191}, - "7": {"count": 5564}, - }, - "hf_subset_descriptive_stats": {}, - "en": { - "num_samples": 15667, - "average_text_length": 36.57222186761984, - "unique_labels": 11, - "labels": { - "0": {"count": 1165}, - "5": {"count": 1657}, - "6": {"count": 1402}, - "9": {"count": 1303}, - "1": {"count": 2187}, - "8": {"count": 2157}, - "10": {"count": 1219}, - "2": {"count": 929}, - "4": {"count": 1353}, - "3": {"count": 1064}, - "7": {"count": 1231}, - }, - }, - "de": { - "num_samples": 13424, - "average_text_length": 43.226013110846246, - "unique_labels": 11, - "labels": { - "0": {"count": 761}, - "10": {"count": 996}, - "4": {"count": 1185}, - "1": {"count": 2016}, - "7": {"count": 1029}, - "5": {"count": 1484}, - "2": {"count": 814}, - "3": {"count": 980}, - "6": {"count": 1265}, - "8": {"count": 1767}, - "9": {"count": 1127}, - }, - }, - "es": { - "num_samples": 10934, - "average_text_length": 43.60691421254801, - "unique_labels": 11, - "labels": { - "1": {"count": 1459}, - "6": {"count": 1188}, - "4": {"count": 928}, - "10": {"count": 743}, - "3": {"count": 830}, - "5": {"count": 1396}, - "2": {"count": 823}, - "8": {"count": 1555}, - "7": {"count": 525}, - "9": {"count": 560}, - "0": {"count": 927}, - }, - }, - "fr": { - "num_samples": 11814, - "average_text_length": 43.594802776367025, - "unique_labels": 11, - "labels": { - "0": {"count": 861}, - "10": {"count": 668}, - "1": {"count": 1968}, - "7": {"count": 975}, - "5": {"count": 1261}, - "2": {"count": 799}, - "3": {"count": 734}, - "4": {"count": 1082}, - "6": {"count": 1113}, - "8": {"count": 1656}, - "9": {"count": 697}, - }, - }, - "hi": { - "num_samples": 11330, - "average_text_length": 37.592144748455425, - "unique_labels": 11, - "labels": { - "0": {"count": 794}, - "1": {"count": 1741}, - "7": {"count": 974}, - "2": {"count": 670}, - "3": {"count": 831}, - "5": {"count": 1272}, - "6": {"count": 940}, - "4": {"count": 1073}, - "10": {"count": 786}, - "8": {"count": 1281}, - "9": {"count": 968}, - }, - }, - "th": { - "num_samples": 10759, - "average_text_length": 34.04043126684636, - "unique_labels": 11, - "labels": { - "0": {"count": 754}, - "10": {"count": 672}, - "1": {"count": 1736}, - "7": {"count": 830}, - "2": {"count": 735}, - "3": {"count": 752}, - "5": {"count": 1264}, - "6": {"count": 1053}, - "4": {"count": 1023}, - "8": {"count": 1282}, - "9": {"count": 658}, - }, - }, - }, - }, + prompt="Classify the intent domain of the given utterance in task-oriented conversation", ) diff --git a/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py b/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py index be9bb79131..52863107b6 100644 --- a/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py +++ b/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py @@ -59,8 +59,5 @@ class MTOPIntentClassification(MultilingualTask, AbsTaskClassification): abstract = "Scaling semantic parsing models for task-oriented dialog systems to new languages is often expensive and time-consuming due to the lack of available datasets. Available datasets suffer from several shortcomings: a) they contain few languages b) they contain small amounts of labeled examples per language c) they are based on the simple intent and slot detection paradigm for non-compositional queries. In this paper, we present a new multilingual dataset, called MTOP, comprising of 100k annotated utterances in 6 languages across 11 domains. We use this dataset and other publicly available datasets to conduct a comprehensive benchmarking study on using various state-of-the-art multilingual pre-trained models for task-oriented semantic parsing. We achieve an average improvement of +6.3 points on Slot F1 for the two existing multilingual datasets, over best results reported in their experiments. Furthermore, we demonstrate strong zero-shot performance using pre-trained models combined with automatic translation and alignment, and a proposed distant supervision method to reduce the noise in slot label projection.", } """, - descriptive_stats={ - "n_samples": {"validation": 2235, "test": 4386}, - "avg_character_length": {"validation": 36.5, "test": 36.8}, - }, + prompt="Classify the intent of the given utterance in task-oriented conversation", ) diff --git a/mteb/tasks/Classification/multilingual/MasakhaNEWSClassification.py b/mteb/tasks/Classification/multilingual/MasakhaNEWSClassification.py index b81183f887..e5152a84d9 100644 --- a/mteb/tasks/Classification/multilingual/MasakhaNEWSClassification.py +++ b/mteb/tasks/Classification/multilingual/MasakhaNEWSClassification.py @@ -54,10 +54,6 @@ class MasakhaNEWSClassification(AbsTaskClassification, MultilingualTask): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"test": 422}, - "avg_character_length": {"test": 5116.6}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py b/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py index 9af5992499..e790b27663 100644 --- a/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py +++ b/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py @@ -90,8 +90,5 @@ class MassiveIntentClassification(MultilingualTask, AbsTaskClassification): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"validation": 2033, "test": 2974}, - "avg_character_length": {"validation": 34.8, "test": 34.6}, - }, + prompt="Given a user utterance as query, find the user intents", ) diff --git a/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py b/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py index d59ae1e41f..80e8583ecc 100644 --- a/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py +++ b/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py @@ -90,8 +90,5 @@ class MassiveScenarioClassification(MultilingualTask, AbsTaskClassification): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"validation": 2033, "test": 2974}, - "avg_character_length": {"validation": 34.8, "test": 34.6}, - }, + prompt="Given a user utterance as query, find the user scenarios", ) diff --git a/mteb/tasks/Classification/multilingual/MultiHateClassification.py b/mteb/tasks/Classification/multilingual/MultiHateClassification.py index c0c0e997c1..f20ba592c1 100644 --- a/mteb/tasks/Classification/multilingual/MultiHateClassification.py +++ b/mteb/tasks/Classification/multilingual/MultiHateClassification.py @@ -92,10 +92,6 @@ class MultiHateClassification(MultilingualTask, AbsTaskClassification): abstract = "Hate speech detection models are typically evaluated on held-out test sets. However, this risks painting an incomplete and potentially misleading picture of model performance because of increasingly well-documented systematic gaps and biases in hate speech datasets. To enable more targeted diagnostic insights, recent research has thus introduced functional tests for hate speech detection models. However, these tests currently only exist for English-language content, which means that they cannot support the development of more effective models in other languages spoken by billions across the world. To help address this issue, we introduce Multilingual HateCheck (MHC), a suite of functional tests for multilingual hate speech detection models. MHC covers 34 functionalities across ten languages, which is more languages than any other hate speech dataset. To illustrate MHC{'}s utility, we train and test a high-performing multilingual hate speech detection model, and reveal critical model weaknesses for monolingual and cross-lingual applications.", } """, - descriptive_stats={ - "n_samples": {"test": 10000}, - "avg_character_length": {"test": 45.9}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py b/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py index 0f762f747f..1108dd7cf8 100644 --- a/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py +++ b/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py @@ -89,10 +89,6 @@ class MultilingualSentimentClassification(AbsTaskClassification, MultilingualTas pages = "89--95", } """, - descriptive_stats={ - "n_samples": {"test": 7000}, - "avg_character_length": {"test": 56}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/multilingual/NaijaSenti.py b/mteb/tasks/Classification/multilingual/NaijaSenti.py index 20d1498345..b31333236e 100644 --- a/mteb/tasks/Classification/multilingual/NaijaSenti.py +++ b/mteb/tasks/Classification/multilingual/NaijaSenti.py @@ -60,10 +60,6 @@ class NaijaSenti(AbsTaskClassification, MultilingualTask): url = "https://aclanthology.org/2022.lrec-1.63", pages = "590--602", }""", - descriptive_stats={ - "n_samples": {"test": 4800}, - "avg_character_length": {"test": 72.81}, - }, ) def load_data(self, **kwargs: Any) -> None: diff --git a/mteb/tasks/Classification/multilingual/NordicLangClassification.py b/mteb/tasks/Classification/multilingual/NordicLangClassification.py index d800d105b1..2a89e44a23 100644 --- a/mteb/tasks/Classification/multilingual/NordicLangClassification.py +++ b/mteb/tasks/Classification/multilingual/NordicLangClassification.py @@ -55,18 +55,10 @@ class NordicLangClassification(AbsTaskClassification): abstract = "Automatic language identification is a challenging problem. Discriminating between closely related languages is especially difficult. This paper presents a machine learning approach for automatic language identification for the Nordic languages, which often suffer miscategorisation by existing state-of-the-art tools. Concretely we will focus on discrimination between six Nordic languages: Danish, Swedish, Norwegian (Nynorsk), Norwegian (Bokm{\aa}l), Faroese and Icelandic.", } """, - descriptive_stats={ - "n_samples": {"test": 3000}, - "avg_character_length": {"test": 78.2}, - }, + prompt="Classify texts based on language", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 32 - return metadata_dict + samples_per_label = 32 def dataset_transform(self): self.dataset = self.dataset.rename_columns( diff --git a/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py b/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py index 2a866c9792..fca11b365c 100644 --- a/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py +++ b/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py @@ -54,12 +54,4 @@ class NusaParagraphEmotionClassification(MultilingualTask, AbsTaskClassification pages = "921--945", } """, - descriptive_stats={ - "n_samples": {"train": 15516, "validation": 2948, "test": 6250}, - "avg_character_length": { - "train": 740.24, - "validation": 740.66, - "test": 740.71, - }, - }, ) diff --git a/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py b/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py index 6d7d745a43..effd257709 100644 --- a/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py +++ b/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py @@ -54,12 +54,4 @@ class NusaParagraphTopicClassification(MultilingualTask, AbsTaskClassification): pages = "921--945", } """, - descriptive_stats={ - "n_samples": {"train": 15516, "validation": 2948, "test": 6250}, - "avg_character_length": { - "train": 740.24, - "validation": 740.66, - "test": 740.71, - }, - }, ) diff --git a/mteb/tasks/Classification/multilingual/NusaXSenti.py b/mteb/tasks/Classification/multilingual/NusaXSenti.py index a701bf02a7..1b9fa2460a 100644 --- a/mteb/tasks/Classification/multilingual/NusaXSenti.py +++ b/mteb/tasks/Classification/multilingual/NusaXSenti.py @@ -55,8 +55,4 @@ class NusaXSentiClassification(AbsTaskClassification, MultilingualTask): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": 4800}, - "avg_character_length": {"test": 52.4}, - }, ) diff --git a/mteb/tasks/Classification/multilingual/SIB200Classification.py b/mteb/tasks/Classification/multilingual/SIB200Classification.py index 936d6d7774..88e5d4b9c8 100644 --- a/mteb/tasks/Classification/multilingual/SIB200Classification.py +++ b/mteb/tasks/Classification/multilingual/SIB200Classification.py @@ -238,14 +238,6 @@ class SIB200Classification(MultilingualTask, AbsTaskClassification): journal={arXiv preprint arXiv:2309.07445}, year={2023} }""", - descriptive_stats={ - "n_samples": {"train": 701, "validation": 99, "test": 204}, - "avg_character_length": { - "train": 111.24, - "validation": 97.11, - "test": 135.53, - }, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/multilingual/ScalaClassification.py b/mteb/tasks/Classification/multilingual/ScalaClassification.py index 2354e851f9..4d055d3578 100644 --- a/mteb/tasks/Classification/multilingual/ScalaClassification.py +++ b/mteb/tasks/Classification/multilingual/ScalaClassification.py @@ -51,18 +51,10 @@ class ScalaClassification(AbsTaskClassification, MultilingualTask): url = "https://aclanthology.org/2023.nodalida-1.20", pages = "185--201", }""", - descriptive_stats={ - "n_samples": {"test": len(_LANGS) * 1024}, - "avg_character_length": {"test": 102.72}, - }, + prompt="Classify passages in Scandinavian Languages based on linguistic acceptability", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 32 - return metadata_dict + samples_per_label = 32 def dataset_transform(self): for lang in self.dataset.keys(): diff --git a/mteb/tasks/Classification/multilingual/SouthAfricanLangClassification.py b/mteb/tasks/Classification/multilingual/SouthAfricanLangClassification.py index b23756ac97..217d300ec0 100644 --- a/mteb/tasks/Classification/multilingual/SouthAfricanLangClassification.py +++ b/mteb/tasks/Classification/multilingual/SouthAfricanLangClassification.py @@ -3,19 +3,19 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -_LANGUAGES = { - "afr": ["afr-Latn"], - "eng": ["eng-Latn"], - "nbl": ["nbl-Latn"], - "nso": ["nso-Latn"], - "sot": ["sot-Latn"], - "ssw": ["ssw-Latn"], - "tsn": ["tsn-Latn"], - "tso": ["tso-Latn"], - "ven": ["ven-Latn"], - "xho": ["xho-Latn"], - "zul": ["zul-Latn"], -} +_LANGUAGES = [ + "afr-Latn", + "eng-Latn", + "nbl-Latn", + "nso-Latn", + "sot-Latn", + "ssw-Latn", + "tsn-Latn", + "tso-Latn", + "ven-Latn", + "xho-Latn", + "zul-Latn", +] class SouthAfricanLangClassification(AbsTaskClassification): @@ -47,10 +47,6 @@ class SouthAfricanLangClassification(AbsTaskClassification): year = {2022}, url = {https://kaggle.com/competitions/south-african-language-identification} }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 247.49}, - }, ) def dataset_transform(self) -> None: diff --git a/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py b/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py index ca8ecb30bd..92aa43268c 100644 --- a/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py +++ b/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py @@ -43,10 +43,6 @@ class SwissJudgementClassification(MultilingualTask, AbsTaskClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 3411.72}, - }, ) def dataset_transform(self): @@ -59,7 +55,6 @@ def dataset_transform(self): seed=42, splits=["test"], label="label", - n_samples=min(2048, len(dataset["text"])) - 2, ) self.dataset[lang]["test"] = subsampled_dataset_dict["test"] diff --git a/mteb/tasks/Classification/multilingual/TurkicClassification.py b/mteb/tasks/Classification/multilingual/TurkicClassification.py index 327765c092..ec947fce4d 100644 --- a/mteb/tasks/Classification/multilingual/TurkicClassification.py +++ b/mteb/tasks/Classification/multilingual/TurkicClassification.py @@ -38,10 +38,6 @@ class TurkicClassification(MultilingualTask, AbsTaskClassification): sample_creation="found", bibtex_citation=""" """, - descriptive_stats={ - "n_samples": {"train": 193056}, - "avg_character_length": {"train": 1103.13}, - }, ) def transform_data(self, dataset, lang): diff --git a/mteb/tasks/Classification/multilingual/TweetSentimentClassification.py b/mteb/tasks/Classification/multilingual/TweetSentimentClassification.py index bf7a8469cd..4105f975a9 100644 --- a/mteb/tasks/Classification/multilingual/TweetSentimentClassification.py +++ b/mteb/tasks/Classification/multilingual/TweetSentimentClassification.py @@ -55,10 +55,6 @@ class TweetSentimentClassification(MultilingualTask, AbsTaskClassification): abstract = "Language models are ubiquitous in current NLP, and their multilingual capacity has recently attracted considerable attention. However, current analyses have almost exclusively focused on (multilingual variants of) standard benchmarks, and have relied on clean pre-training and task-specific corpora as multilingual signals. In this paper, we introduce XLM-T, a model to train and evaluate multilingual language models in Twitter. In this paper we provide: (1) a new strong multilingual baseline consisting of an XLM-R (Conneau et al. 2020) model pre-trained on millions of tweets in over thirty languages, alongside starter code to subsequently fine-tune on a target task; and (2) a set of unified sentiment analysis Twitter datasets in eight different languages and a XLM-T model trained on this dataset.", } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 83.51}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/mya/MyanmarNews.py b/mteb/tasks/Classification/mya/MyanmarNews.py index 70a603e8b2..8418e20533 100644 --- a/mteb/tasks/Classification/mya/MyanmarNews.py +++ b/mteb/tasks/Classification/mya/MyanmarNews.py @@ -36,10 +36,6 @@ class MyanmarNews(AbsTaskClassification): month = {February}, pages = {401--408} }""", - descriptive_stats={ - "n_samples": {"train": 2048}, - "avg_character_length": {"train": 174.2}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/nep/NepaliNewsClassification.py b/mteb/tasks/Classification/nep/NepaliNewsClassification.py index 5985cf232c..85cc8d9661 100644 --- a/mteb/tasks/Classification/nep/NepaliNewsClassification.py +++ b/mteb/tasks/Classification/nep/NepaliNewsClassification.py @@ -47,10 +47,6 @@ class NepaliNewsClassification(AbsTaskClassification): abstract = "We present iNLTK, an open-source NLP library consisting of pre-trained language models and out-of-the-box support for Data Augmentation, Textual Similarity, Sentence Embeddings, Word Embeddings, Tokenization and Text Generation in 13 Indic Languages. By using pre-trained models from iNLTK for text classification on publicly available datasets, we significantly outperform previously reported results. On these datasets, we also show that by using pre-trained models and data augmentation from iNLTK, we can achieve more than 95{\%} of the previous best performance by using less than 10{\%} of the training data. iNLTK is already being widely used by the community and has 40,000+ downloads, 600+ stars and 100+ forks on GitHub.", } """, - descriptive_stats={ - "n_samples": {"train": 5975, "test": 1495}, - "avg_character_length": {"train": 196.61, "test": 196.017}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/nld/DutchBookReviewSentimentClassification.py b/mteb/tasks/Classification/nld/DutchBookReviewSentimentClassification.py index efd7076d59..f0ee1b07dc 100644 --- a/mteb/tasks/Classification/nld/DutchBookReviewSentimentClassification.py +++ b/mteb/tasks/Classification/nld/DutchBookReviewSentimentClassification.py @@ -43,8 +43,4 @@ class DutchBookReviewSentimentClassification(AbsTaskClassification): bibsource = {dblp computer science bibliography, https://dblp.org} } """, - descriptive_stats={ - "n_samples": {"test": 2224}, - "avg_character_length": {"test": 1443.0}, - }, ) diff --git a/mteb/tasks/Classification/nob/NoRecClassification.py b/mteb/tasks/Classification/nob/NoRecClassification.py index 6ab978cfa2..920389d84c 100644 --- a/mteb/tasks/Classification/nob/NoRecClassification.py +++ b/mteb/tasks/Classification/nob/NoRecClassification.py @@ -57,8 +57,5 @@ class NoRecClassification(AbsTaskClassification): url = "https://aclanthology.org/L18-1661", } """, - descriptive_stats={ - "n_samples": {"test": 2050}, - "avg_character_length": {"test": 82}, - }, + prompt="Classify Norwegian reviews by sentiment", ) diff --git a/mteb/tasks/Classification/nob/NorwegianParliamentClassification.py b/mteb/tasks/Classification/nob/NorwegianParliamentClassification.py index 5397df4f58..e46ae6612a 100644 --- a/mteb/tasks/Classification/nob/NorwegianParliamentClassification.py +++ b/mteb/tasks/Classification/nob/NorwegianParliamentClassification.py @@ -45,8 +45,5 @@ class NorwegianParliamentClassification(AbsTaskClassification): pages = "20--29", abstract = "In this work, we show the process of building a large-scale training set from digital and digitized collections at a national library. The resulting Bidirectional Encoder Representations from Transformers (BERT)-based language model for Norwegian outperforms multilingual BERT (mBERT) models in several token and sequence classification tasks for both Norwegian Bokm{\aa}l and Norwegian Nynorsk. Our model also improves the mBERT performance for other languages present in the corpus such as English, Swedish, and Danish. For languages not included in the corpus, the weights degrade moderately while keeping strong multilingual properties. Therefore, we show that building high-quality models within a memory institution using somewhat noisy optical character recognition (OCR) content is feasible, and we hope to pave the way for other memory institutions to follow.", }""", - descriptive_stats={ - "n_samples": {"test": 1200, "validation": 1200}, - "avg_character_length": {"test": 1884.0, "validation": 1911.0}, - }, + prompt="Classify parliament speeches in Norwegian based on political affiliation", ) diff --git a/mteb/tasks/Classification/ory/OdiaNewsClassification.py b/mteb/tasks/Classification/ory/OdiaNewsClassification.py index 4459cb325c..6e89c50ab1 100644 --- a/mteb/tasks/Classification/ory/OdiaNewsClassification.py +++ b/mteb/tasks/Classification/ory/OdiaNewsClassification.py @@ -32,10 +32,6 @@ class OdiaNewsClassification(AbsTaskClassification): year={2020}, journal={arXiv preprint arXiv:2005.00085}, }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 49.24}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/pan/PunjabiNewsClassification.py b/mteb/tasks/Classification/pan/PunjabiNewsClassification.py index 68e4492e22..fb948d7746 100644 --- a/mteb/tasks/Classification/pan/PunjabiNewsClassification.py +++ b/mteb/tasks/Classification/pan/PunjabiNewsClassification.py @@ -32,10 +32,6 @@ class PunjabiNewsClassification(AbsTaskClassification): year={2020}, journal={arXiv preprint arXiv:2005.00085}, }""", - descriptive_stats={ - "n_samples": {"train": 627, "test": 157}, - "avg_character_length": {"train": 4222.22, "test": 4115.14}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/pol/PolishClassification.py b/mteb/tasks/Classification/pol/PolishClassification.py index 1d7cf699f9..6db91ecb25 100644 --- a/mteb/tasks/Classification/pol/PolishClassification.py +++ b/mteb/tasks/Classification/pol/PolishClassification.py @@ -35,10 +35,6 @@ class CbdClassification(AbsTaskClassification): url = {http://2019.poleval.pl/files/poleval2019.pdf}, isbn = "978-83-63159-28-3"} }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": {"test": 93.2}, - }, ) @@ -80,7 +76,6 @@ class PolEmo2InClassification(AbsTaskClassification): pages = "980--991", abstract = "In this article we present an extended version of PolEmo {--} a corpus of consumer reviews from 4 domains: medicine, hotels, products and school. Current version (PolEmo 2.0) contains 8,216 reviews having 57,466 sentences. Each text and sentence was manually annotated with sentiment in 2+1 scheme, which gives a total of 197,046 annotations. We obtained a high value of Positive Specific Agreement, which is 0.91 for texts and 0.88 for sentences. PolEmo 2.0 is publicly available under a Creative Commons copyright license. We explored recent deep learning approaches for the recognition of sentiment, such as Bi-directional Long Short-Term Memory (BiLSTM) and Bidirectional Encoder Representations from Transformers (BERT).", }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @@ -109,10 +104,6 @@ class PolEmo2OutClassification(AbsTaskClassification): dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={ - "n_samples": {"test": 722}, - "avg_character_length": {"test": 756.2}, - }, ) @@ -139,10 +130,6 @@ class AllegroReviewsClassification(AbsTaskClassification): dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={ - "n_samples": {"test": 1006}, - "avg_character_length": {"test": 477.2}, - }, ) @@ -178,8 +165,4 @@ class PacClassification(AbsTaskClassification): primaryClass={cs.CL}, url={https://arxiv.org/abs/2211.13112}, }""", - descriptive_stats={ - "n_samples": {"test": 3453}, - "avg_character_length": {"test": 185.3}, - }, ) diff --git a/mteb/tasks/Classification/por/HateSpeechPortugueseClassification.py b/mteb/tasks/Classification/por/HateSpeechPortugueseClassification.py index 43a6b086dd..a7abf6b0f9 100644 --- a/mteb/tasks/Classification/por/HateSpeechPortugueseClassification.py +++ b/mteb/tasks/Classification/por/HateSpeechPortugueseClassification.py @@ -49,10 +49,6 @@ class HateSpeechPortugueseClassification(AbsTaskClassification): pages = "94--104", } """, - descriptive_stats={ - "n_samples": {"train": 2048}, - "avg_character_length": {"train": 101.02}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ron/Moroco.py b/mteb/tasks/Classification/ron/Moroco.py index d761823d56..2324f16ef8 100644 --- a/mteb/tasks/Classification/ron/Moroco.py +++ b/mteb/tasks/Classification/ron/Moroco.py @@ -41,10 +41,6 @@ class Moroco(AbsTaskClassification): pages={688--698}, } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 1710.94}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ron/RomanianReviewsSentiment.py b/mteb/tasks/Classification/ron/RomanianReviewsSentiment.py index bc4aabcbd0..6666d615a3 100644 --- a/mteb/tasks/Classification/ron/RomanianReviewsSentiment.py +++ b/mteb/tasks/Classification/ron/RomanianReviewsSentiment.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 2048 - class RomanianReviewsSentiment(AbsTaskClassification): metadata = TaskMetadata( @@ -38,10 +36,6 @@ class RomanianReviewsSentiment(AbsTaskClassification): year = {2021} } """, - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "avg_character_length": {"test": 588.6}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/ron/RomanianSentimentClassification.py b/mteb/tasks/Classification/ron/RomanianSentimentClassification.py index 880ab33774..1bcfd0052c 100644 --- a/mteb/tasks/Classification/ron/RomanianSentimentClassification.py +++ b/mteb/tasks/Classification/ron/RomanianSentimentClassification.py @@ -36,10 +36,6 @@ class RomanianSentimentClassification(AbsTaskClassification): year={2020} } """, - descriptive_stats={ - "n_samples": {"test": TEST_SAMPLES}, - "avg_character_length": {"test": 67.6}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/rus/GeoreviewClassification.py b/mteb/tasks/Classification/rus/GeoreviewClassification.py index 89b5bef408..3a9298ead2 100644 --- a/mteb/tasks/Classification/rus/GeoreviewClassification.py +++ b/mteb/tasks/Classification/rus/GeoreviewClassification.py @@ -28,10 +28,7 @@ class GeoreviewClassification(AbsTaskClassification): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 409.0}, - }, + prompt="Classify the organization rating based on the reviews", ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/rus/HeadlineClassification.py b/mteb/tasks/Classification/rus/HeadlineClassification.py index 7e272b1747..ca16fd6a85 100644 --- a/mteb/tasks/Classification/rus/HeadlineClassification.py +++ b/mteb/tasks/Classification/rus/HeadlineClassification.py @@ -50,10 +50,7 @@ class HeadlineClassification(AbsTaskClassification): pages = "54--59", abstract = "The article is focused on automatic development and ranking of a large corpus for Russian paraphrase generation which proves to be the first corpus of such type in Russian computational linguistics. Existing manually annotated paraphrase datasets for Russian are limited to small-sized ParaPhraser corpus and ParaPlag which are suitable for a set of NLP tasks, such as paraphrase and plagiarism detection, sentence similarity and relatedness estimation, etc. Due to size restrictions, these datasets can hardly be applied in end-to-end text generation solutions. Meanwhile, paraphrase generation requires a large amount of training data. In our study we propose a solution to the problem: we collect, rank and evaluate a new publicly available headline paraphrase corpus (ParaPhraser Plus), and then perform text generation experiments with manual evaluation on automatically ranked corpora using the Universal Transformer architecture.", }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 61.6}, - }, + prompt="Classify the topic or theme of the given news headline", ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/rus/InappropriatenessClassification.py b/mteb/tasks/Classification/rus/InappropriatenessClassification.py index b14e439680..306266d3fa 100644 --- a/mteb/tasks/Classification/rus/InappropriatenessClassification.py +++ b/mteb/tasks/Classification/rus/InappropriatenessClassification.py @@ -54,10 +54,7 @@ class InappropriatenessClassification(AbsTaskClassification): pages = "26--36", abstract = "Not all topics are equally {``}flammable{''} in terms of toxicity: a calm discussion of turtles or fishing less often fuels inappropriate toxic dialogues than a discussion of politics or sexual minorities. We define a set of sensitive topics that can yield inappropriate and toxic messages and describe the methodology of collecting and labelling a dataset for appropriateness. While toxicity in user-generated data is well-studied, we aim at defining a more fine-grained notion of inappropriateness. The core of inappropriateness is that it can harm the reputation of a speaker. This is different from toxicity in two respects: (i) inappropriateness is topic-related, and (ii) inappropriate message is not toxic but still unacceptable. We collect and release two datasets for Russian: a topic-labelled dataset and an appropriateness-labelled dataset. We also release pre-trained classification models trained on this data.", }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 97.7}, - }, + prompt="Classify the given message as either sensitive topic or not", ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/rus/KinopoiskClassification.py b/mteb/tasks/Classification/rus/KinopoiskClassification.py index 619d807afb..2fa32a7fdf 100644 --- a/mteb/tasks/Classification/rus/KinopoiskClassification.py +++ b/mteb/tasks/Classification/rus/KinopoiskClassification.py @@ -35,8 +35,5 @@ class KinopoiskClassification(AbsTaskClassification): pages={48--58}, year={2013} }""", - descriptive_stats={ - "n_samples": {"test": 1500}, - "avg_character_length": {"test": 1897.3}, - }, + prompt="Classify the sentiment expressed in the given movie review text", ) diff --git a/mteb/tasks/Classification/rus/RuReviewsClassification.py b/mteb/tasks/Classification/rus/RuReviewsClassification.py index ffa16000fd..7303f3f85d 100644 --- a/mteb/tasks/Classification/rus/RuReviewsClassification.py +++ b/mteb/tasks/Classification/rus/RuReviewsClassification.py @@ -38,10 +38,7 @@ class RuReviewsClassification(AbsTaskClassification): ISSN={2378-1963}, month={July} }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 133.2}, - }, + prompt="Classify product reviews into positive, negative or neutral sentiment", ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/rus/RuSciBenchGRNTIClassification.py b/mteb/tasks/Classification/rus/RuSciBenchGRNTIClassification.py index e6e276664a..c2c737eea5 100644 --- a/mteb/tasks/Classification/rus/RuSciBenchGRNTIClassification.py +++ b/mteb/tasks/Classification/rus/RuSciBenchGRNTIClassification.py @@ -27,10 +27,7 @@ class RuSciBenchGRNTIClassification(AbsTaskClassification): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 890.1}, - }, + prompt="Classify the category of scientific papers based on the titles and abstracts", ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/rus/RuSciBenchOECDClassification.py b/mteb/tasks/Classification/rus/RuSciBenchOECDClassification.py index b08367e4af..b32f7c2b6e 100644 --- a/mteb/tasks/Classification/rus/RuSciBenchOECDClassification.py +++ b/mteb/tasks/Classification/rus/RuSciBenchOECDClassification.py @@ -27,10 +27,7 @@ class RuSciBenchOECDClassification(AbsTaskClassification): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 838.9}, - }, + prompt="Classify the category of scientific papers based on the titles and abstracts", ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/san/SanskritShlokasClassification.py b/mteb/tasks/Classification/san/SanskritShlokasClassification.py index 4e22db6f07..806e468f00 100644 --- a/mteb/tasks/Classification/san/SanskritShlokasClassification.py +++ b/mteb/tasks/Classification/san/SanskritShlokasClassification.py @@ -47,10 +47,6 @@ class SanskritShlokasClassification(AbsTaskClassification): abstract = "We present iNLTK, an open-source NLP library consisting of pre-trained language models and out-of-the-box support for Data Augmentation, Textual Similarity, Sentence Embeddings, Word Embeddings, Tokenization and Text Generation in 13 Indic Languages. By using pre-trained models from iNLTK for text classification on publicly available datasets, we significantly outperform previously reported results. On these datasets, we also show that by using pre-trained models and data augmentation from iNLTK, we can achieve more than 95{\%} of the previous best performance by using less than 10{\%} of the training data. iNLTK is already being widely used by the community and has 40,000+ downloads, 600+ stars and 100+ forks on GitHub.", } """, - descriptive_stats={ - "n_samples": {"train": 383, "validation": 96}, - "avg_character_length": {"train": 98.415, "validation": 96.635}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/sin/SinhalaNewsClassification.py b/mteb/tasks/Classification/sin/SinhalaNewsClassification.py index 6c993af44b..98d414b3c0 100644 --- a/mteb/tasks/Classification/sin/SinhalaNewsClassification.py +++ b/mteb/tasks/Classification/sin/SinhalaNewsClassification.py @@ -38,10 +38,6 @@ class SinhalaNewsClassification(AbsTaskClassification): journal = {Year of Publication}, year = {2022}, }""", - descriptive_stats={ - "n_samples": {"train": 3327}, - "avg_character_length": {"train": 148.04}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/sin/SinhalaNewsSourceClassification.py b/mteb/tasks/Classification/sin/SinhalaNewsSourceClassification.py index 21347094c3..a7bd9763a7 100644 --- a/mteb/tasks/Classification/sin/SinhalaNewsSourceClassification.py +++ b/mteb/tasks/Classification/sin/SinhalaNewsSourceClassification.py @@ -33,10 +33,6 @@ class SinhalaNewsSourceClassification(AbsTaskClassification): journal = {Year of Publication}, year = {2022}, }""", - descriptive_stats={ - "n_samples": {"train": 24094}, - "avg_character_length": {"train": 56.08}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/slk/CSFDSKMovieReviewSentimentClassification.py b/mteb/tasks/Classification/slk/CSFDSKMovieReviewSentimentClassification.py index 587fa1ff7b..1f99fff1c8 100644 --- a/mteb/tasks/Classification/slk/CSFDSKMovieReviewSentimentClassification.py +++ b/mteb/tasks/Classification/slk/CSFDSKMovieReviewSentimentClassification.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 2048 - class CSFDSKMovieReviewSentimentClassification(AbsTaskClassification): metadata = TaskMetadata( @@ -38,20 +36,14 @@ class CSFDSKMovieReviewSentimentClassification(AbsTaskClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "avg_character_length": {"test": 366.2}, - }, ) - @property - def metadata_dict(self): - md = super().metadata_dict - # Increase the samples_per_label in order to improve baseline performance - md["samples_per_label"] = 20 - return md + # Increase the samples_per_label in order to improve baseline performance + samples_per_label = 20 def dataset_transform(self): + N_SAMPLES = 2048 + self.dataset = self.dataset.rename_columns( {"comment": "text", "rating_int": "label"} ) diff --git a/mteb/tasks/Classification/slk/SlovakHateSpeechClassification.py b/mteb/tasks/Classification/slk/SlovakHateSpeechClassification.py index ad1d29dcf7..bd131ece85 100644 --- a/mteb/tasks/Classification/slk/SlovakHateSpeechClassification.py +++ b/mteb/tasks/Classification/slk/SlovakHateSpeechClassification.py @@ -27,8 +27,4 @@ class SlovakHateSpeechClassification(AbsTaskClassification): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 1319}, - "avg_character_length": {"test": 92.71}, - }, ) diff --git a/mteb/tasks/Classification/slv/FrenkSlClassification.py b/mteb/tasks/Classification/slv/FrenkSlClassification.py index 63a5af2ec0..120555da58 100644 --- a/mteb/tasks/Classification/slv/FrenkSlClassification.py +++ b/mteb/tasks/Classification/slv/FrenkSlClassification.py @@ -36,10 +36,6 @@ class FrenkSlClassification(AbsTaskClassification): primaryClass={cs.CL}, url={https://arxiv.org/abs/1906.02045} }""", - descriptive_stats={ - "n_samples": {"test": 2177}, - "avg_character_length": {"test": 136.61}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/spa/SpanishNewsClassification.py b/mteb/tasks/Classification/spa/SpanishNewsClassification.py index 6804034ef2..59ac97ba20 100644 --- a/mteb/tasks/Classification/spa/SpanishNewsClassification.py +++ b/mteb/tasks/Classification/spa/SpanishNewsClassification.py @@ -28,10 +28,6 @@ class SpanishNewsClassification(AbsTaskClassification): sample_creation="found", bibtex_citation=""" """, - descriptive_stats={ - "n_samples": {"train": 2048}, - "avg_character_length": {"train": 4218.2}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/spa/SpanishSentimentClassification.py b/mteb/tasks/Classification/spa/SpanishSentimentClassification.py index 0b0e147273..28e56f87c9 100644 --- a/mteb/tasks/Classification/spa/SpanishSentimentClassification.py +++ b/mteb/tasks/Classification/spa/SpanishSentimentClassification.py @@ -51,8 +51,4 @@ class SpanishSentimentClassification(AbsTaskClassification): pages = "89--95", } """, - descriptive_stats={ - "n_samples": {"validation": 147, "test": 296}, - "avg_character_length": {"validation": 85.02, "test": 87.91}, - }, ) diff --git a/mteb/tasks/Classification/ssw/SiswatiNewsClassification.py b/mteb/tasks/Classification/ssw/SiswatiNewsClassification.py index 516e9b3a20..d51b42f88d 100644 --- a/mteb/tasks/Classification/ssw/SiswatiNewsClassification.py +++ b/mteb/tasks/Classification/ssw/SiswatiNewsClassification.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 2800 - class SiswatiNewsClassification(AbsTaskClassification): metadata = TaskMetadata( @@ -30,10 +28,6 @@ class SiswatiNewsClassification(AbsTaskClassification): sample_creation="found", bibtex_citation="""@article{Madodonga_Marivate_Adendorff_2023, title={Izindaba-Tindzaba: Machine learning news categorisation for Long and Short Text for isiZulu and Siswati}, volume={4}, url={https://upjournals.up.ac.za/index.php/dhasa/article/view/4449}, DOI={10.55492/dhasa.v4i01.4449}, author={Madodonga, Andani and Marivate, Vukosi and Adendorff, Matthew}, year={2023}, month={Jan.} } """, - descriptive_stats={ - "n_samples": {"train": 80}, - "avg_character_length": {"train": 354.2}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/svk/SlovakMovieReviewSentimentClassification.py b/mteb/tasks/Classification/svk/SlovakMovieReviewSentimentClassification.py index 10df4d28e9..8918c4a1a4 100644 --- a/mteb/tasks/Classification/svk/SlovakMovieReviewSentimentClassification.py +++ b/mteb/tasks/Classification/svk/SlovakMovieReviewSentimentClassification.py @@ -34,10 +34,6 @@ class SlovakMovieReviewSentimentClassification(AbsTaskClassification): year={2023} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 366.17}, - }, ) def dataset_transform(self) -> None: diff --git a/mteb/tasks/Classification/swa/SwahiliNewsClassification.py b/mteb/tasks/Classification/swa/SwahiliNewsClassification.py index c297a4528d..6a4cb6bdc8 100644 --- a/mteb/tasks/Classification/swa/SwahiliNewsClassification.py +++ b/mteb/tasks/Classification/swa/SwahiliNewsClassification.py @@ -36,10 +36,6 @@ class SwahiliNewsClassification(AbsTaskClassification): url = "https://doi.org/10.5281/zenodo.5514203" } """, - descriptive_stats={ - "n_samples": {"train": 2048}, - "avg_character_length": {"train": 2438.2308135942326}, - }, ) def dataset_transform(self) -> None: diff --git a/mteb/tasks/Classification/swe/DalajClassification.py b/mteb/tasks/Classification/swe/DalajClassification.py index b43dd8a168..780fe65dbf 100644 --- a/mteb/tasks/Classification/swe/DalajClassification.py +++ b/mteb/tasks/Classification/swe/DalajClassification.py @@ -35,18 +35,10 @@ class DalajClassification(AbsTaskClassification): Year = {2021}, Eprint = {arXiv:2105.06681}, }""", - descriptive_stats={ - "n_samples": {"test": 444}, - "avg_character_length": {"test": 243.8}, - }, + prompt="Classify texts based on linguistic acceptability in Swedish", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 16 - return metadata_dict + samples_per_label = 16 def dataset_transform(self): """This dataset consist of two columns of relevance, "original_sentence" and "corrected_sentence". diff --git a/mteb/tasks/Classification/swe/SweRecClassification.py b/mteb/tasks/Classification/swe/SweRecClassification.py index 7980fe294d..7083ade1fb 100644 --- a/mteb/tasks/Classification/swe/SweRecClassification.py +++ b/mteb/tasks/Classification/swe/SweRecClassification.py @@ -40,8 +40,5 @@ class SweRecClassification(AbsTaskClassification): pages = "185--201", } """, - descriptive_stats={ - "n_samples": {"test": 1024}, - "avg_character_length": {"test": 318.8}, - }, + prompt="Classify Swedish reviews by sentiment", ) diff --git a/mteb/tasks/Classification/swe/SwedishSentimentClassification.py b/mteb/tasks/Classification/swe/SwedishSentimentClassification.py index 0b62ba0668..4c0fdc16cb 100644 --- a/mteb/tasks/Classification/swe/SwedishSentimentClassification.py +++ b/mteb/tasks/Classification/swe/SwedishSentimentClassification.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 1024 - class SwedishSentimentClassification(AbsTaskClassification): metadata = TaskMetadata( @@ -30,10 +28,6 @@ class SwedishSentimentClassification(AbsTaskClassification): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"validation": N_SAMPLES, "test": N_SAMPLES}, - "avg_character_length": {"validation": 499.3, "test": 498.1}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/tam/TamilNewsClassification.py b/mteb/tasks/Classification/tam/TamilNewsClassification.py index 73cf10adba..af9698d0b1 100644 --- a/mteb/tasks/Classification/tam/TamilNewsClassification.py +++ b/mteb/tasks/Classification/tam/TamilNewsClassification.py @@ -32,10 +32,6 @@ class TamilNewsClassification(AbsTaskClassification): year={2020}, journal={arXiv preprint arXiv:2005.00085}, }""", - descriptive_stats={ - "n_samples": {"train": 14521, "test": 3631}, - "avg_character_length": {"train": 56.50, "test": 56.52}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/tel/TeluguAndhraJyotiNewsClassification.py b/mteb/tasks/Classification/tel/TeluguAndhraJyotiNewsClassification.py index 0458fd0e66..3d07293c64 100644 --- a/mteb/tasks/Classification/tel/TeluguAndhraJyotiNewsClassification.py +++ b/mteb/tasks/Classification/tel/TeluguAndhraJyotiNewsClassification.py @@ -27,10 +27,6 @@ class TeluguAndhraJyotiNewsClassification(AbsTaskClassification): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 4329}, - "avg_character_length": {"test": 1428.28}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/tha/WisesightSentimentClassification.py b/mteb/tasks/Classification/tha/WisesightSentimentClassification.py index 2799b58b37..3a76003d5b 100644 --- a/mteb/tasks/Classification/tha/WisesightSentimentClassification.py +++ b/mteb/tasks/Classification/tha/WisesightSentimentClassification.py @@ -42,10 +42,6 @@ class WisesightSentimentClassification(AbsTaskClassification): } """, - descriptive_stats={ - "n_samples": {"train": 2048}, - "avg_character_length": {"train": 103.42}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/tha/WongnaiReviewsClassification .py b/mteb/tasks/Classification/tha/WongnaiReviewsClassification .py index bc164bcd85..1a0bfb0834 100644 --- a/mteb/tasks/Classification/tha/WongnaiReviewsClassification .py +++ b/mteb/tasks/Classification/tha/WongnaiReviewsClassification .py @@ -37,10 +37,6 @@ class WongnaiReviewsClassification(AbsTaskClassification): doi = {10.5281/zenodo.3852912}, url = {https://doi.org/10.5281/zenodo.3852912} }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 540.3717}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/tsn/TswanaNewsClassification.py b/mteb/tasks/Classification/tsn/TswanaNewsClassification.py index d49e3c5cd1..c1eee27779 100644 --- a/mteb/tasks/Classification/tsn/TswanaNewsClassification.py +++ b/mteb/tasks/Classification/tsn/TswanaNewsClassification.py @@ -38,8 +38,4 @@ class TswanaNewsClassification(AbsTaskClassification): software_url = {https://huggingface.co/dsfsi/PuoBERTa} } """, - descriptive_stats={ - "n_samples": {"validation": 487, "test": 487}, - "avg_character_length": {"validation": 2417.72, "test": 2369.52}, - }, ) diff --git a/mteb/tasks/Classification/tur/TurkishMovieSentimentClassification.py b/mteb/tasks/Classification/tur/TurkishMovieSentimentClassification.py index 70dab008d8..64981c6ec2 100644 --- a/mteb/tasks/Classification/tur/TurkishMovieSentimentClassification.py +++ b/mteb/tasks/Classification/tur/TurkishMovieSentimentClassification.py @@ -35,10 +35,6 @@ class TurkishMovieSentimentClassification(AbsTaskClassification): url={https://api.semanticscholar.org/CorpusID:3912960} } """, - descriptive_stats={ - "n_samples": {"test": 2644}, - "avg_character_length": {"test": 141.50}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/tur/TurkishProductSentimentClassification.py b/mteb/tasks/Classification/tur/TurkishProductSentimentClassification.py index d755d51ba7..c33c537c69 100644 --- a/mteb/tasks/Classification/tur/TurkishProductSentimentClassification.py +++ b/mteb/tasks/Classification/tur/TurkishProductSentimentClassification.py @@ -35,8 +35,4 @@ class TurkishProductSentimentClassification(AbsTaskClassification): url={https://api.semanticscholar.org/CorpusID:3912960} } """, - descriptive_stats={ - "n_samples": {"test": 800}, - "avg_character_length": {"test": 246.85}, - }, ) diff --git a/mteb/tasks/Classification/ukr/UkrFormalityClassification.py b/mteb/tasks/Classification/ukr/UkrFormalityClassification.py index 8b446a3fe3..0a7f08b8e0 100644 --- a/mteb/tasks/Classification/ukr/UkrFormalityClassification.py +++ b/mteb/tasks/Classification/ukr/UkrFormalityClassification.py @@ -42,10 +42,6 @@ class UkrFormalityClassification(AbsTaskClassification): publisher = "Association for Computational Linguistics", url = "https://aclanthology.org/N18-1012", }""", - descriptive_stats={ - "n_samples": {"train": 2048, "test": 2048}, - "avg_character_length": {"train": 52.10, "test": 53.07}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/urd/UrduRomanSentimentClassification.py b/mteb/tasks/Classification/urd/UrduRomanSentimentClassification.py index 6b1eddda97..62440ef9c2 100644 --- a/mteb/tasks/Classification/urd/UrduRomanSentimentClassification.py +++ b/mteb/tasks/Classification/urd/UrduRomanSentimentClassification.py @@ -36,10 +36,6 @@ class UrduRomanSentimentClassification(AbsTaskClassification): note = {{DOI}: https://doi.org/10.24432/C58325} } """, - descriptive_stats={ - "n_samples": {"train": 2048}, - "avg_character_length": {"train": 68.248}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/vie/VieStudentFeedbackClassification.py b/mteb/tasks/Classification/vie/VieStudentFeedbackClassification.py index dc3dac9dea..901d2861f9 100644 --- a/mteb/tasks/Classification/vie/VieStudentFeedbackClassification.py +++ b/mteb/tasks/Classification/vie/VieStudentFeedbackClassification.py @@ -39,10 +39,6 @@ class VieStudentFeedbackClassification(AbsTaskClassification): pages={19-24}, doi={10.1109/KSE.2018.8573337} }""", - descriptive_stats={ - "n_samples": {"test": TEST_SAMPLES}, - "avg_character_length": {"test": 14.22}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Classification/zho/CMTEBClassification.py b/mteb/tasks/Classification/zho/CMTEBClassification.py index 9b49fa7003..7e790ecf9a 100644 --- a/mteb/tasks/Classification/zho/CMTEBClassification.py +++ b/mteb/tasks/Classification/zho/CMTEBClassification.py @@ -69,14 +69,10 @@ class TNews(AbsTaskClassification): doi = "10.18653/v1/2020.coling-main.419", pages = "4762--4772", }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, + prompt="Classify the fine-grained category of the given news title", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["samples_per_label"] = 32 - return metadata_dict + samples_per_label = 32 class IFlyTek(AbsTaskClassification): @@ -145,13 +141,14 @@ class IFlyTek(AbsTaskClassification): pages = "4762--4772", abstract = "The advent of natural language understanding (NLU) benchmarks for English, such as GLUE and SuperGLUE allows new NLU models to be evaluated across a diverse set of tasks. These comprehensive benchmarks have facilitated a broad range of research and applications in natural language processing (NLP). The problem, however, is that most such benchmarks are limited to English, which has made it difficult to replicate many of the successes in English NLU for other languages. To help remedy this issue, we introduce the first large-scale Chinese Language Understanding Evaluation (CLUE) benchmark. CLUE is an open-ended, community-driven project that brings together 9 tasks spanning several well-established single-sentence/sentence-pair classification tasks, as well as machine reading comprehension, all on original Chinese text. To establish results on these tasks, we report scores using an exhaustive set of current state-of-the-art pre-trained Chinese models (9 in total). We also introduce a number of supplementary datasets and additional tools to help facilitate further progress on Chinese NLU. Our benchmark is released at https://www.cluebenchmarks.com", }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, + prompt="Given an App description text, find the appropriate fine-grained category", ) + samples_per_label = 32 + @property def metadata_dict(self) -> dict[str, str]: metadata_dict = super().metadata_dict - metadata_dict["samples_per_label"] = 32 metadata_dict["n_experiments"] = 5 return metadata_dict @@ -179,14 +176,10 @@ class MultilingualSentiment(AbsTaskClassification): dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={"n_samples": None, "avg_character_length": None}, + prompt="Classify sentiment of the customer review into positive, neutral, or negative", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["samples_per_label"] = 32 - return metadata_dict + samples_per_label = 32 class JDReview(AbsTaskClassification): @@ -217,14 +210,10 @@ class JDReview(AbsTaskClassification): journal={arXiv preprint arXiv:2309.07597}, year={2023} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, + prompt="Classify the customer review for iPhone on e-commerce platform into positive or negative", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["samples_per_label"] = 32 - return metadata_dict + samples_per_label = 32 class OnlineShopping(AbsTaskClassification): @@ -255,14 +244,10 @@ class OnlineShopping(AbsTaskClassification): journal={arXiv preprint arXiv:2309.07597}, year={2023} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, + prompt="Classify the customer review for online shopping into positive or negative", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["samples_per_label"] = 32 - return metadata_dict + samples_per_label = 32 class Waimai(AbsTaskClassification): @@ -293,12 +278,7 @@ class Waimai(AbsTaskClassification): journal={arXiv preprint arXiv:2309.07597}, year={2023} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, + prompt="Classify the customer review from a food takeaway platform into positive or negative", ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["samples_per_label"] = 32 - - return metadata_dict + samples_per_label = 32 diff --git a/mteb/tasks/Classification/zho/YueOpenriceReviewClassification.py b/mteb/tasks/Classification/zho/YueOpenriceReviewClassification.py index 926199dc83..2189708719 100644 --- a/mteb/tasks/Classification/zho/YueOpenriceReviewClassification.py +++ b/mteb/tasks/Classification/zho/YueOpenriceReviewClassification.py @@ -34,18 +34,9 @@ class YueOpenriceReviewClassification(AbsTaskClassification): year={2019}, organization={KDD WISDOM} }""", - descriptive_stats={ - "n_samples": {"test": 6161}, - "avg_character_length": {"test": 173.0}, - }, ) - @property - def metadata_dict(self) -> dict[str, str]: - metadata_dict = super().metadata_dict - metadata_dict["n_experiments"] = 10 - metadata_dict["samples_per_label"] = 32 - return metadata_dict + samples_per_label = 32 def dataset_transform(self): self.dataset = self.stratified_subsampling( diff --git a/mteb/tasks/Classification/zul/IsiZuluNewsClassification.py b/mteb/tasks/Classification/zul/IsiZuluNewsClassification.py index 18f8a21e5c..26e3d16553 100644 --- a/mteb/tasks/Classification/zul/IsiZuluNewsClassification.py +++ b/mteb/tasks/Classification/zul/IsiZuluNewsClassification.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 2800 - class IsiZuluNewsClassification(AbsTaskClassification): metadata = TaskMetadata( @@ -30,10 +28,6 @@ class IsiZuluNewsClassification(AbsTaskClassification): sample_creation="found", bibtex_citation="""@article{Madodonga_Marivate_Adendorff_2023, title={Izindaba-Tindzaba: Machine learning news categorisation for Long and Short Text for isiZulu and Siswati}, volume={4}, url={https://upjournals.up.ac.za/index.php/dhasa/article/view/4449}, DOI={10.55492/dhasa.v4i01.4449}, author={Madodonga, Andani and Marivate, Vukosi and Adendorff, Matthew}, year={2023}, month={Jan.} } """, - descriptive_stats={ - "n_samples": {"train": 752}, - "avg_character_length": {"train": 43.1}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/deu/BlurbsClusteringP2P.py b/mteb/tasks/Clustering/deu/BlurbsClusteringP2P.py index 1a88c48750..17ac058740 100644 --- a/mteb/tasks/Clustering/deu/BlurbsClusteringP2P.py +++ b/mteb/tasks/Clustering/deu/BlurbsClusteringP2P.py @@ -39,10 +39,6 @@ class BlurbsClusteringP2P(AbsTaskClustering): year={2019}, url={https://api.semanticscholar.org/CorpusID:208334484} }""", - descriptive_stats={ - "n_samples": {"test": 174637}, - "avg_character_length": {"test": 664.09}, - }, ) @@ -83,10 +79,6 @@ class BlurbsClusteringP2PFast(AbsTaskClusteringFast): year={2019}, url={https://api.semanticscholar.org/CorpusID:208334484} }""", - descriptive_stats={ - "n_samples": {"test": NUM_SAMPLES}, - "avg_character_length": {"test": 664.09}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/deu/BlurbsClusteringS2S.py b/mteb/tasks/Clustering/deu/BlurbsClusteringS2S.py index 7b9dc43e5d..67366ed13d 100644 --- a/mteb/tasks/Clustering/deu/BlurbsClusteringS2S.py +++ b/mteb/tasks/Clustering/deu/BlurbsClusteringS2S.py @@ -47,10 +47,6 @@ class BlurbsClusteringS2S(AbsTaskClustering): year={2019}, url={https://api.semanticscholar.org/CorpusID:208334484} }""", - descriptive_stats={ - "n_samples": {"test": 174637}, - "avg_character_length": {"test": 23.02}, - }, ) @@ -92,10 +88,6 @@ class BlurbsClusteringS2SFast(AbsTaskClusteringFast): year={2019}, url={https://api.semanticscholar.org/CorpusID:208334484} }""", - descriptive_stats={ - "n_samples": {"test": NUM_SAMPLES}, - "avg_character_length": {"test": 23.02}, - }, ) def dataset_transform(self): @@ -124,5 +116,4 @@ def dataset_transform(self): self.seed, self.metadata.eval_splits, label="labels", - n_samples=NUM_SAMPLES, ) diff --git a/mteb/tasks/Clustering/deu/TenKGnadClusteringP2P.py b/mteb/tasks/Clustering/deu/TenKGnadClusteringP2P.py index d43b94546c..f3a2d6599e 100644 --- a/mteb/tasks/Clustering/deu/TenKGnadClusteringP2P.py +++ b/mteb/tasks/Clustering/deu/TenKGnadClusteringP2P.py @@ -30,10 +30,6 @@ class TenKGnadClusteringP2P(AbsTaskClustering): dialect=[], sample_creation="found", bibtex_citation=None, - descriptive_stats={ - "n_samples": {"test": 45914}, - "avg_character_length": {"test": 2641.03}, - }, ) @@ -67,10 +63,6 @@ class TenKGnadClusteringP2PFast(AbsTaskClusteringFast): sample_creation="found", bibtex_citation=None, # none found # due to duplicates - descriptive_stats={ - "n_samples": {"test": 10275}, - "avg_character_length": {"test": 2641.03}, - }, ) def dataset_transform(self) -> None: diff --git a/mteb/tasks/Clustering/deu/TenKGnadClusteringS2S.py b/mteb/tasks/Clustering/deu/TenKGnadClusteringS2S.py index 726d73bac6..66b8bc0f1d 100644 --- a/mteb/tasks/Clustering/deu/TenKGnadClusteringS2S.py +++ b/mteb/tasks/Clustering/deu/TenKGnadClusteringS2S.py @@ -31,10 +31,6 @@ class TenKGnadClusteringS2S(AbsTaskClustering): dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={ - "n_samples": {"test": 45914}, - "avg_character_length": {"test": 50.96}, - }, ) @@ -68,10 +64,6 @@ class TenKGnadClusteringS2SFast(AbsTaskClusteringFast): sample_creation="found", bibtex_citation=None, # none found # due to duplicates - descriptive_stats={ - "n_samples": {"test": 10267}, - "avg_character_length": {"test": 50.96}, - }, ) def dataset_transform(self) -> None: diff --git a/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py b/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py index 7a7748fd7c..8bf13a2131 100644 --- a/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py +++ b/mteb/tasks/Clustering/eng/ArXivHierarchicalClustering.py @@ -38,146 +38,6 @@ class ArXivHierarchicalClusteringP2P(AbsTaskClusteringFast): dialect=["Thematic clustering"], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "test": { - "num_samples": 2048, - "average_text_length": 1008.439453125, - "average_labels_per_text": 1.46337890625, - "unique_labels": 129, - "labels": { - "cs": {"count": 356}, - "math": {"count": 381}, - "OC": {"count": 11}, - "hep-lat": {"count": 13}, - "hep": {"count": 98}, - "astro-ph": {"count": 213}, - "eess": {"count": 76}, - "quant-ph": {"count": 135}, - "DC": {"count": 5}, - "cond-mat": {"count": 274}, - "hep-th": {"count": 66}, - "SP": {"count": 33}, - "hep-ph": {"count": 69}, - "FA": {"count": 6}, - "nucl-th": {"count": 17}, - "q-bio": {"count": 80}, - "HE": {"count": 22}, - "HC": {"count": 2}, - "stat": {"count": 60}, - "ML": {"count": 16}, - "IV": {"count": 13}, - "stat-mech": {"count": 47}, - "DS": {"count": 14}, - "ME": {"count": 12}, - "CC": {"count": 2}, - "mtrl-sci": {"count": 22}, - "PE": {"count": 16}, - "NT": {"count": 11}, - "SC": {"count": 6}, - "AG": {"count": 13}, - "physics": {"count": 81}, - "ins-det": {"count": 9}, - "GA": {"count": 18}, - "BM": {"count": 6}, - "GN": {"count": 17}, - "NA": {"count": 15}, - "app-ph": {"count": 7}, - "RT": {"count": 6}, - "other": {"count": 37}, - "soft": {"count": 15}, - "CO": {"count": 33}, - "supr-con": {"count": 21}, - "chem-ph": {"count": 3}, - "DM": {"count": 2}, - "MN": {"count": 12}, - "q-fin": {"count": 27}, - "PM": {"count": 2}, - "AP": {"count": 27}, - "gr-qc": {"count": 15}, - "quant-gas": {"count": 8}, - "mes-hall": {"count": 33}, - "IT": {"count": 19}, - "SI": {"count": 6}, - "SG": {"count": 3}, - "bio-ph": {"count": 2}, - "SR": {"count": 16}, - "soc-ph": {"count": 5}, - "hep-ex": {"count": 15}, - "DG": {"count": 11}, - "NE": {"count": 5}, - "CR": {"count": 6}, - "CL": {"count": 12}, - "RM": {"count": 3}, - "econ": {"count": 17}, - "nlin": {"count": 5}, - "PS": {"count": 1}, - "LG": {"count": 26}, - "QA": {"count": 9}, - "str-el": {"count": 26}, - "CV": {"count": 34}, - "MF": {"count": 6}, - "IM": {"count": 7}, - "EM": {"count": 6}, - "TH": {"count": 5}, - "PR": {"count": 20}, - "AT": {"count": 4}, - "OA": {"count": 4}, - "CP": {"count": 6}, - "LO": {"count": 14}, - "flu-dyn": {"count": 6}, - "atom-ph": {"count": 8}, - "class-ph": {"count": 1}, - "SY": {"count": 20}, - "IR": {"count": 1}, - "plasm-ph": {"count": 8}, - "CE": {"count": 2}, - "AO": {"count": 1}, - "comp-ph": {"count": 3}, - "optics": {"count": 12}, - "MG": {"count": 4}, - "ST": {"count": 6}, - "nucl-ex": {"count": 6}, - "CY": {"count": 9}, - "ao-ph": {"count": 2}, - "DB": {"count": 1}, - "math-ph": {"count": 10}, - "NC": {"count": 13}, - "GT": {"count": 11}, - "TO": {"count": 2}, - "AI": {"count": 9}, - "NI": {"count": 2}, - "gen-ph": {"count": 4}, - "OT": {"count": 4}, - "SD": {"count": 2}, - "dis-nn": {"count": 4}, - "RO": {"count": 7}, - "CA": {"count": 6}, - "FL": {"count": 1}, - "SE": {"count": 5}, - "EP": {"count": 9}, - "hist-ph": {"count": 1}, - "QM": {"count": 9}, - "ed-ph": {"count": 2}, - "GR": {"count": 4}, - "MS": {"count": 1}, - "CD": {"count": 1}, - "ET": {"count": 1}, - "acc-ph": {"count": 5}, - "AC": {"count": 2}, - "OH": {"count": 1}, - "EC": {"count": 2}, - "DL": {"count": 1}, - "AS": {"count": 3}, - "geo-ph": {"count": 2}, - "CG": {"count": 3}, - "CB": {"count": 1}, - "AR": {"count": 1}, - "TR": {"count": 1}, - "atm-clus": {"count": 1}, - }, - }, - }, ) def dataset_transform(self): @@ -218,10 +78,6 @@ class ArXivHierarchicalClusteringS2S(AbsTaskClusteringFast): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "avg_character_length": {"test": 1009.98}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/eng/ArxivClusteringP2P.py b/mteb/tasks/Clustering/eng/ArxivClusteringP2P.py index e74170eea1..72f831599c 100644 --- a/mteb/tasks/Clustering/eng/ArxivClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/ArxivClusteringP2P.py @@ -37,10 +37,7 @@ class ArxivClusteringP2P(AbsTaskClustering): author={arXiv.org submitters}, year={2024} }""", - descriptive_stats={ - "n_samples": {"test": 732723}, - "avg_character_length": {"test": 1009.98}, - }, + prompt="Identify the main and secondary category of Arxiv papers based on the titles and abstracts", ) @@ -78,10 +75,7 @@ class ArxivClusteringP2PFast(AbsTaskClustering): author={arXiv.org submitters}, year={2024} }""", # None found - descriptive_stats={ - "n_samples": {"test": 250_000}, - "avg_character_length": {"test": 1009.98}, - }, + prompt="Identify the main and secondary category of Arxiv papers based on the titles and abstracts", ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/eng/ArxivClusteringS2S.py b/mteb/tasks/Clustering/eng/ArxivClusteringS2S.py index ce65275015..c74766061d 100644 --- a/mteb/tasks/Clustering/eng/ArxivClusteringS2S.py +++ b/mteb/tasks/Clustering/eng/ArxivClusteringS2S.py @@ -36,8 +36,5 @@ class ArxivClusteringS2S(AbsTaskClustering): author={arXiv.org submitters}, year={2024} }""", - descriptive_stats={ - "n_samples": {"test": 732723}, - "avg_character_length": {"test": 74}, - }, + prompt="Identify the main and secondary category of Arxiv papers based on the titles", ) diff --git a/mteb/tasks/Clustering/eng/BigPatentClustering.py b/mteb/tasks/Clustering/eng/BigPatentClustering.py index 756ec4db32..7df254ab51 100644 --- a/mteb/tasks/Clustering/eng/BigPatentClustering.py +++ b/mteb/tasks/Clustering/eng/BigPatentClustering.py @@ -52,7 +52,6 @@ class BigPatentClustering(AbsTaskClustering): biburl = {https://dblp.org/rec/journals/corr/abs-1906-03741.bib}, bibsource = {dblp computer science bibliography, https://dblp.org} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @@ -99,10 +98,6 @@ class BigPatentClusteringFast(AbsTaskClusteringFast): biburl = {https://dblp.org/rec/journals/corr/abs-1906-03741.bib}, bibsource = {dblp computer science bibliography, https://dblp.org} }""", - descriptive_stats={ - "n_samples": {"test": NUM_SAMPLES}, - "avg_character_length": {"test": 30995.5}, - }, ) def dataset_transform(self): @@ -113,5 +108,4 @@ def dataset_transform(self): self.seed, self.metadata.eval_splits, label="labels", - n_samples=NUM_SAMPLES, ) diff --git a/mteb/tasks/Clustering/eng/BiorxivClusteringP2P.py b/mteb/tasks/Clustering/eng/BiorxivClusteringP2P.py index bf7186df7a..998dcec0e7 100644 --- a/mteb/tasks/Clustering/eng/BiorxivClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/BiorxivClusteringP2P.py @@ -31,10 +31,7 @@ class BiorxivClusteringP2PFast(AbsTaskClusteringFast): dialect=[], sample_creation="created", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 2151}, - "avg_character_length": {"test": 1664.0}, - }, + prompt="Identify the main category of Biorxiv papers based on the titles and abstracts", ) def dataset_transform(self): @@ -66,8 +63,5 @@ class BiorxivClusteringP2P(AbsTaskClustering): dialect=[], sample_creation="created", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 75000}, - "avg_character_length": {"test": 1666.2}, - }, + prompt="Identify the main category of Biorxiv papers based on the titles and abstracts", ) diff --git a/mteb/tasks/Clustering/eng/BiorxivClusteringS2S.py b/mteb/tasks/Clustering/eng/BiorxivClusteringS2S.py index 2038148ce1..3cf5b69ba4 100644 --- a/mteb/tasks/Clustering/eng/BiorxivClusteringS2S.py +++ b/mteb/tasks/Clustering/eng/BiorxivClusteringS2S.py @@ -31,10 +31,7 @@ class BiorxivClusteringS2SFast(AbsTaskClusteringFast): dialect=[], sample_creation="created", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 2151}, - "avg_character_length": {"test": 101.7}, - }, + prompt="Identify the main category of Biorxiv papers based on the titles", ) def dataset_transform(self): @@ -66,41 +63,5 @@ class BiorxivClusteringS2S(AbsTaskClustering): dialect=[], sample_creation="created", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 75000}, - "test": { - "num_samples": 10, - "average_text_length": 7500.0, - "average_labels_per_text": 7500.0, - "unique_labels": 26, - "labels": { - "neuroscience": {"count": 14251}, - "genetics": {"count": 2282}, - "biophysics": {"count": 3864}, - "animal behavior and cognition": {"count": 1148}, - "genomics": {"count": 3422}, - "systems biology": {"count": 1544}, - "ecology": {"count": 3469}, - "immunology": {"count": 3517}, - "evolutionary biology": {"count": 3756}, - "molecular biology": {"count": 2772}, - "bioengineering": {"count": 2169}, - "cancer biology": {"count": 2922}, - "plant biology": {"count": 2640}, - "microbiology": {"count": 7176}, - "physiology": {"count": 1251}, - "synthetic biology": {"count": 686}, - "pharmacology and toxicology": {"count": 864}, - "zoology": {"count": 433}, - "bioinformatics": {"count": 6294}, - "cell biology": {"count": 4433}, - "developmental biology": {"count": 2352}, - "biochemistry": {"count": 2790}, - "scientific communication and education": {"count": 349}, - "paleontology": {"count": 120}, - "pathology": {"count": 495}, - "epidemiology": {"count": 1}, - }, - }, - }, + prompt="Identify the main category of Biorxiv papers based on the titles", ) diff --git a/mteb/tasks/Clustering/eng/MedrxivClusteringP2P.py b/mteb/tasks/Clustering/eng/MedrxivClusteringP2P.py index ac338f152f..c897145069 100644 --- a/mteb/tasks/Clustering/eng/MedrxivClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/MedrxivClusteringP2P.py @@ -35,10 +35,7 @@ class MedrxivClusteringP2PFast(AbsTaskClusteringFast): dialect=[], sample_creation="created", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 1500}, - "avg_character_length": {"test": 1984.7}, - }, + prompt="Identify the main category of Medrxiv papers based on the titles and abstracts", ) def dataset_transform(self): @@ -77,8 +74,5 @@ class MedrxivClusteringP2P(AbsTaskClustering): dialect=[], sample_creation="created", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 37500}, - "avg_character_length": {"test": 1981.2}, - }, + prompt="Identify the main category of Medrxiv papers based on the titles and abstracts", ) diff --git a/mteb/tasks/Clustering/eng/MedrxivClusteringS2S.py b/mteb/tasks/Clustering/eng/MedrxivClusteringS2S.py index e0062db193..0e913de1cc 100644 --- a/mteb/tasks/Clustering/eng/MedrxivClusteringS2S.py +++ b/mteb/tasks/Clustering/eng/MedrxivClusteringS2S.py @@ -35,10 +35,7 @@ class MedrxivClusteringS2SFast(AbsTaskClusteringFast): dialect=[], sample_creation="created", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 1500}, - "avg_character_length": {"test": 114.9}, - }, + prompt="Identify the main category of Medrxiv papers based on the titles", ) def dataset_transform(self): @@ -77,8 +74,5 @@ class MedrxivClusteringS2S(AbsTaskClustering): dialect=[], sample_creation="created", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 37500}, - "avg_character_length": {"test": 114.7}, - }, + prompt="Identify the main category of Medrxiv papers based on the titles", ) diff --git a/mteb/tasks/Clustering/eng/RedditClustering.py b/mteb/tasks/Clustering/eng/RedditClustering.py index 58d40fbd43..c9efbe954a 100644 --- a/mteb/tasks/Clustering/eng/RedditClustering.py +++ b/mteb/tasks/Clustering/eng/RedditClustering.py @@ -47,10 +47,7 @@ class RedditFastClusteringS2S(AbsTaskClusteringFast): archivePrefix = {arXiv}, eprint = {2104.07081} }""", - descriptive_stats={ - "n_samples": {"test": 32768}, - "avg_character_length": {"test": 64.7}, - }, + prompt="Identify the topic or theme of Reddit posts based on the titles", ) def dataset_transform(self): @@ -68,7 +65,6 @@ def dataset_transform(self): self.seed, self.metadata.eval_splits, label="labels", - n_samples=32768, ) self.max_fraction_of_documents_to_embed = None @@ -110,8 +106,5 @@ class RedditClustering(AbsTaskClustering): archivePrefix = {arXiv}, eprint = {2104.07081} }""", - descriptive_stats={ - "n_samples": {"test": 420464}, - "avg_character_length": {"test": 64.7}, - }, + prompt="Identify the topic or theme of Reddit posts based on the titles", ) diff --git a/mteb/tasks/Clustering/eng/RedditClusteringP2P.py b/mteb/tasks/Clustering/eng/RedditClusteringP2P.py index d81559e246..1e8d51cdfa 100644 --- a/mteb/tasks/Clustering/eng/RedditClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/RedditClusteringP2P.py @@ -50,10 +50,7 @@ class RedditClusteringP2P(AbsTaskClustering): archivePrefix = {arXiv}, eprint = {2104.07081} }""", - descriptive_stats={ - "n_samples": {"test": 459399}, - "avg_character_length": {"test": 727.7}, - }, + prompt="Identify the topic or theme of Reddit posts based on the titles and posts", ) @@ -92,10 +89,7 @@ class RedditFastClusteringP2P(AbsTaskClusteringFast): archivePrefix = {arXiv}, eprint = {2104.07081} }""", - descriptive_stats={ - "n_samples": {"test": 18375}, - "avg_character_length": {"test": 727.7}, - }, + prompt="Identify the topic or theme of Reddit posts based on the titles and posts", ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/eng/StackExchangeClustering.py b/mteb/tasks/Clustering/eng/StackExchangeClustering.py index 881d77e20e..b123ab5bd1 100644 --- a/mteb/tasks/Clustering/eng/StackExchangeClustering.py +++ b/mteb/tasks/Clustering/eng/StackExchangeClustering.py @@ -47,10 +47,7 @@ class StackExchangeClusteringFast(AbsTaskClusteringFast): archivePrefix = {arXiv}, eprint = {2104.07081} }""", - descriptive_stats={ - "n_samples": {"test": 32768}, - "avg_character_length": {"test": 57.0}, - }, + prompt="Identify the topic or theme of StackExchange posts based on the titles", ) def dataset_transform(self): @@ -70,7 +67,6 @@ def dataset_transform(self): self.seed, self.metadata.eval_splits, label="labels", - n_samples=32768, ) self.max_fraction_of_documents_to_embed = None @@ -112,8 +108,5 @@ class StackExchangeClustering(AbsTaskClustering): archivePrefix = {arXiv}, eprint = {2104.07081} }""", - descriptive_stats={ - "n_samples": {"test": 373850}, - "avg_character_length": {"test": 57.0}, - }, + prompt="Identify the topic or theme of StackExchange posts based on the titles", ) diff --git a/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py b/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py index 993b1e0db8..c411138e9f 100644 --- a/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py @@ -49,10 +49,7 @@ class StackExchangeClusteringP2PFast(AbsTaskClusteringFast): archivePrefix = {arXiv}, eprint = {2104.07081} }""", - descriptive_stats={ - "n_samples": {"test": 2996}, - "avg_character_length": {"test": 1090.7}, - }, + prompt="Identify the topic or theme of StackExchange posts based on the given paragraphs", ) def dataset_transform(self): @@ -95,7 +92,6 @@ class StackExchangeClusteringP2P(AbsTaskClustering): eval_langs=["eng-Latn"], main_score="v_measure", date=None, - form=None, domains=None, task_subtypes=None, license=None, @@ -115,8 +111,5 @@ class StackExchangeClusteringP2P(AbsTaskClustering): archivePrefix = {arXiv}, eprint = {2104.07081} }""", - descriptive_stats={ - "n_samples": {"test": 75000}, - "avg_character_length": {"test": 1090.7}, - }, + prompt="Identify the topic or theme of StackExchange posts based on the given paragraphs", ) diff --git a/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py b/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py index 6e9aef97df..8747cfe7b3 100644 --- a/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py +++ b/mteb/tasks/Clustering/eng/TwentyNewsgroupsClustering.py @@ -49,10 +49,7 @@ class TwentyNewsgroupsClustering(AbsTaskClustering): author = {Ken Lang}, } """, - descriptive_stats={ - "n_samples": {"test": 59545}, - "avg_character_length": {"test": 32.0}, - }, + prompt="Identify the topic or theme of the given news articles", ) @@ -92,10 +89,7 @@ class TwentyNewsgroupsClusteringFast(AbsTaskClusteringFast): author = {Ken Lang}, } """, - descriptive_stats={ - "n_samples": {"test": 2381}, - "avg_character_length": {"test": 32.0}, - }, + prompt="Identify the topic or theme of the given news articles", ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/eng/WikiCitiesClustering.py b/mteb/tasks/Clustering/eng/WikiCitiesClustering.py index 135fad580f..be897938a8 100644 --- a/mteb/tasks/Clustering/eng/WikiCitiesClustering.py +++ b/mteb/tasks/Clustering/eng/WikiCitiesClustering.py @@ -32,5 +32,4 @@ class WikiCitiesClustering(AbsTaskClustering): title = "Wikimedia Downloads", url = "https://dumps.wikimedia.org" }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) diff --git a/mteb/tasks/Clustering/fra/AlloProfClusteringP2P.py b/mteb/tasks/Clustering/fra/AlloProfClusteringP2P.py index a900043551..d48175172c 100644 --- a/mteb/tasks/Clustering/fra/AlloProfClusteringP2P.py +++ b/mteb/tasks/Clustering/fra/AlloProfClusteringP2P.py @@ -48,7 +48,6 @@ class AlloProfClusteringP2P(AbsTaskClustering): year = {2023}, copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) def create_description(self, example): @@ -108,10 +107,6 @@ class AlloProfClusteringP2PFast(AbsTaskClusteringFast): copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International} } """, - descriptive_stats={ - "n_samples": {"test": 2556}, - "avg_character_length": {"test": 3539.5}, - }, ) def create_description(self, example): diff --git a/mteb/tasks/Clustering/fra/AlloProfClusteringS2S.py b/mteb/tasks/Clustering/fra/AlloProfClusteringS2S.py index c46e239689..74f5bddcaa 100644 --- a/mteb/tasks/Clustering/fra/AlloProfClusteringS2S.py +++ b/mteb/tasks/Clustering/fra/AlloProfClusteringS2S.py @@ -48,7 +48,6 @@ class AlloProfClusteringS2S(AbsTaskClustering): year = {2023}, copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) def dataset_transform(self): @@ -105,10 +104,6 @@ class AlloProfClusteringS2SFast(AbsTaskClusteringFast): copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International} } """, - descriptive_stats={ - "n_samples": {"test": 2556}, - "avg_character_length": {"test": 32.8}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/fra/HALClusteringS2S.py b/mteb/tasks/Clustering/fra/HALClusteringS2S.py index 442176d640..7b1f40e3e6 100644 --- a/mteb/tasks/Clustering/fra/HALClusteringS2S.py +++ b/mteb/tasks/Clustering/fra/HALClusteringS2S.py @@ -48,7 +48,6 @@ class HALClusteringS2S(AbsTaskClustering): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) def dataset_transform(self): @@ -96,10 +95,6 @@ class HALClusteringS2SFast(AbsTaskClusteringFast): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"test": NUM_SAMPLES}, - "avg_character_length": {"test": 86.6}, - }, ) def dataset_transform(self): @@ -133,5 +128,4 @@ def dataset_transform(self): self.seed, self.metadata.eval_splits, label="labels", - n_samples=NUM_SAMPLES, ) diff --git a/mteb/tasks/Clustering/jpn/LivedoorNewsClustering.py b/mteb/tasks/Clustering/jpn/LivedoorNewsClustering.py index e9ca78b325..abad8b676c 100644 --- a/mteb/tasks/Clustering/jpn/LivedoorNewsClustering.py +++ b/mteb/tasks/Clustering/jpn/LivedoorNewsClustering.py @@ -32,10 +32,6 @@ class LivedoorNewsClusteringv2(AbsTaskClusteringFast): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 1106}, - "avg_character_length": {"test": 1082.61}, - }, ) def dataset_transform(self): @@ -79,10 +75,6 @@ class LivedoorNewsClustering(AbsTaskClusteringFast): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 1107}, - "avg_character_length": {"test": 1082.61}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/jpn/MewsC16JaClustering.py b/mteb/tasks/Clustering/jpn/MewsC16JaClustering.py index 6dd93313d2..5c8bfe01fa 100644 --- a/mteb/tasks/Clustering/jpn/MewsC16JaClustering.py +++ b/mteb/tasks/Clustering/jpn/MewsC16JaClustering.py @@ -52,10 +52,6 @@ class MewsC16JaClustering(AbsTaskClusteringFast): abstract = "We present EASE, a novel method for learning sentence embeddings via contrastive learning between sentences and their related entities.The advantage of using entity supervision is twofold: (1) entities have been shown to be a strong indicator of text semantics and thus should provide rich training signals for sentence embeddings; (2) entities are defined independently of languages and thus offer useful cross-lingual alignment supervision.We evaluate EASE against other unsupervised models both in monolingual and multilingual settings.We show that EASE exhibits competitive or better performance in English semantic textual similarity (STS) and short text clustering (STC) tasks and it significantly outperforms baseline methods in multilingual settings on a variety of tasks.Our source code, pre-trained models, and newly constructed multi-lingual STC dataset are available at https://github.com/studio-ousia/ease.", } """, - descriptive_stats={ - "n_samples": {"test": 992}, - "avg_character_length": {"test": 95}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py b/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py index 0c65e82772..8f649a745b 100644 --- a/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py @@ -56,10 +56,6 @@ class IndicReviewsClusteringP2P(AbsTaskClustering, MultilingualTask): year = {2022}, doi = {10.18653/v1/2023.acl-long.693} }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": {"test": 137.6}, - }, ) def load_data(self, **kwargs: Any) -> None: diff --git a/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py b/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py index 9f823e774f..0a832bb228 100644 --- a/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py @@ -50,10 +50,6 @@ class MLSUMClusteringP2P(AbsTaskClustering, MultilingualTask): journal={arXiv preprint arXiv:2004.14900}, year={2020} }""", - descriptive_stats={ - "n_samples": {"validation": 38561, "test": 41206}, - "avg_character_length": {"validation": 4613, "test": 4810}, - }, ) def load_data(self, **kwargs): @@ -124,10 +120,6 @@ class MLSUMClusteringP2PFast(AbsTaskClusteringFast, MultilingualTask): journal={arXiv preprint arXiv:2004.14900}, year={2020} }""", - descriptive_stats={ - "n_samples": {"validation": N_SAMPLES, "test": N_SAMPLES}, - "avg_character_length": {"validation": 4613, "test": 4810}, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py b/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py index 9e3978ff30..f5e19874a4 100644 --- a/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py +++ b/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py @@ -50,10 +50,6 @@ class MLSUMClusteringS2S(AbsTaskClustering, MultilingualTask): journal={arXiv preprint arXiv:2004.14900}, year={2020} }""", - descriptive_stats={ - "n_samples": {"validation": 38561, "test": 41206}, - "avg_character_length": {"validation": 4613, "test": 4810}, - }, ) def load_data(self, **kwargs): @@ -119,10 +115,6 @@ class MLSUMClusteringS2SFast(AbsTaskClusteringFast, MultilingualTask): journal={arXiv preprint arXiv:2004.14900}, year={2020} }""", - descriptive_stats={ - "n_samples": {"validation": 750, "test": 756}, - "avg_character_length": {"validation": 4613, "test": 4810}, - }, ) def load_data(self, **kwargs): @@ -167,5 +159,4 @@ def dataset_transform(self, lang): self.seed, self.metadata.eval_splits, label="labels", - n_samples=N_SAMPLES, ) diff --git a/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py b/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py index c74b3ac52d..480cceff8f 100644 --- a/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py @@ -60,7 +60,6 @@ class MasakhaNEWSClusteringP2P(AbsTaskClustering, MultilingualTask): year={2023}, volume={} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringS2S.py b/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringS2S.py index 4f79807e46..7e8b22b9af 100644 --- a/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringS2S.py +++ b/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringS2S.py @@ -59,7 +59,6 @@ class MasakhaNEWSClusteringS2S(AbsTaskClustering, MultilingualTask): year={2023}, volume={} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py b/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py index 54bd52a771..8569b55cd5 100644 --- a/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py +++ b/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py @@ -243,10 +243,6 @@ class SIB200ClusteringFast(MultilingualTask, AbsTaskClusteringFast): journal={arXiv preprint arXiv:2309.07445}, year={2023} }""", # combined train, validation, and test into test. - descriptive_stats={ - "n_samples": {"test": 1004}, - "avg_character_length": {"test": 114.78}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py b/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py index 7fdc6f286d..77e86a100e 100644 --- a/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py @@ -52,725 +52,6 @@ class WikiClusteringP2P(AbsTaskClustering, MultilingualTask): dialect=[], sample_creation="created", bibtex_citation=None, # None exists - descriptive_stats={ - "n_samples": {"test": 71680}, - "test": { - "num_samples": 140, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 282, - "labels": { - "Nauke": {"count": 1492}, - "Dru\u00c5\u00a1tvo": {"count": 504}, - "Priroda": {"count": 448}, - "Kultura": {"count": 1042}, - "Tehnologija": {"count": 671}, - "Tehnika": {"count": 281}, - "Geografija": {"count": 431}, - "Informatika": {"count": 355}, - "Koncepti": {"count": 83}, - "Humanisti\u00c4\u008dke_nauke": {"count": 62}, - "Informacija": {"count": 21}, - "Historija": {"count": 223}, - "Matematika": {"count": 74}, - "Okoli\u00c5\u00a1": {"count": 6}, - "Jezik": {"count": 15}, - "Misao": {"count": 28}, - "Energija": {"count": 16}, - "Llocs": {"count": 642}, - "Ci\u00c3\u00a8ncia": {"count": 1844}, - "Humanitats": {"count": 984}, - "Tecnologia": {"count": 377}, - "Biografies": {"count": 406}, - "Cultura": {"count": 710}, - "Informaci\u00c3\u00b3": {"count": 137}, - "Esdeveniments": {"count": 20}, - "Lid\u00c3\u00a9": {"count": 1559}, - "Geografie": {"count": 1659}, - "\u00c4\u008cas": {"count": 88}, - "Politika": {"count": 818}, - "V\u00c4\u009bda": {"count": 314}, - "Technika": {"count": 189}, - "Informace": {"count": 96}, - "\u00c5\u00bdivot": {"count": 184}, - "Vojenstv\u00c3\u00ad": {"count": 34}, - "Um\u00c4\u009bn\u00c3\u00ad": {"count": 95}, - "P\u00c5\u0099\u00c3\u00adroda": {"count": 387}, - "Spole\u00c4\u008dnost": {"count": 319}, - "Historie": {"count": 391}, - "Sport": {"count": 57}, - "Dorozum\u00c3\u00adv\u00c3\u00a1n\u00c3\u00ad": {"count": 142}, - "Zdravotnictv\u00c3\u00ad": {"count": 94}, - "Vzd\u00c4\u009bl\u00c3\u00a1v\u00c3\u00a1n\u00c3\u00ad": { - "count": 41 - }, - "P\u00c5\u0099edm\u00c4\u009bty": {"count": 21}, - "Pr\u00c3\u00a1vo": {"count": 28}, - "Natur": {"count": 398}, - "Kultur": {"count": 911}, - "Samfund": {"count": 531}, - "Politik": {"count": 287}, - "Computersystemer": {"count": 84}, - "Erhvervsliv": {"count": 164}, - "Teknik": {"count": 166}, - "Sproget": {"count": 41}, - "Humaniora": {"count": 139}, - "Uddannelse": {"count": 79}, - "Anvendt_videnskab": {"count": 199}, - "Kunst": {"count": 412}, - "Sundhed": {"count": 83}, - "Videnskab": {"count": 257}, - "Matematik": {"count": 79}, - "Geografi": {"count": 671}, - "Liv": {"count": 286}, - "Personer": {"count": 322}, - "Jura": {"count": 35}, - "Entitateak": {"count": 2486}, - "Humanitateak": {"count": 861}, - "Mundua": {"count": 706}, - "Zientzia_eta_teknologia": {"count": 957}, - "Entretenimendua": {"count": 110}, - "Chron-oaylleeaght": {"count": 1334}, - "\u00c3\u0087heeraghyn": {"count": 470}, - "Oaylleeaght": {"count": 575}, - "\u00c3\u0087heer-oaylleeaght": {"count": 190}, - "Persoonyn": {"count": 249}, - "Bea": {"count": 207}, - "Cultoor": {"count": 269}, - "Glare": {"count": 152}, - "Dooghys": {"count": 494}, - "Smooinaght": {"count": 28}, - "Sheshaght": {"count": 221}, - "\u00c3\u0087haghnoaylleeaght": {"count": 33}, - "Strughtoor": {"count": 54}, - "Politickaght": {"count": 138}, - "Ronnaghyn_rere_bun-chooish": {"count": 381}, - "Eirinys": {"count": 80}, - "Ynsagh": {"count": 16}, - "Am": {"count": 12}, - "Leigh": {"count": 9}, - "Studeyrys-sheelnaue": {"count": 72}, - "Keirdyn_scoillaragh": {"count": 50}, - "Shennaghys": {"count": 11}, - "Dellal": {"count": 23}, - "Maddaght": {"count": 6}, - "Meanyn_mooarey": {"count": 7}, - "Oaylleeaghtyn_feamagh": {"count": 30}, - "Fysseree": {"count": 7}, - "\u00c3\u0087hymmyltaght": {"count": 2}, - "Katutubo": {"count": 135}, - "Biag": {"count": 933}, - "Pakasaritaan": {"count": 1302}, - "Relihion": {"count": 20}, - "Dagiti_kinatao": {"count": 1405}, - "Lubong": {"count": 308}, - "Siensia_ken_teknolohia": {"count": 12}, - "Gobierno": {"count": 208}, - "Heograpia": {"count": 323}, - "Etika": {"count": 12}, - "Pilosopia": {"count": 26}, - "Tattao": {"count": 20}, - "Dagiti_konsepto": {"count": 69}, - "Panagkukua_ti_tao": {"count": 16}, - "Pannakaammo": {"count": 29}, - "Palpaliwa": {"count": 22}, - "Pagsasao": {"count": 33}, - "Kagimongan": {"count": 37}, - "Salun-at": {"count": 9}, - "Industria": {"count": 5}, - "Linteg": {"count": 1}, - "Makan_ken_mainum": {"count": 19}, - "Dagiti_akademiko_a_disiplina": {"count": 27}, - "Edukasion": {"count": 9}, - "Musika": {"count": 1}, - "Negosio": {"count": 10}, - "Inhenieria": {"count": 1}, - "Paspasamak": {"count": 6}, - "Dagiti_banag": {"count": 2}, - "Annuroten": {"count": 1}, - "Enerhia": {"count": 1}, - "Kes": {"count": 524}, - "Medya": {"count": 67}, - "Xwarin_\u00c3\u00bb_vexwarin": {"count": 127}, - "Civak": {"count": 277}, - "L\u00c3\u00aakol\u00c3\u00aen\u00c3\u00aan_mirovahiy\u00c3\u00aa": { - "count": 269 - }, - "Zeman": {"count": 174}, - "Teknoloj\u00c3\u00ae": {"count": 92}, - "D\u00c3\u00aes\u00c3\u00aepl\u00c3\u00aen\u00c3\u00aan_akadem\u00c3\u00aek": { - "count": 237 - }, - "Erdn\u00c3\u00aegar\u00c3\u00ae": {"count": 1078}, - "D\u00c3\u00aerok": {"count": 436}, - "Gerd\u00c3\u00bbn": {"count": 42}, - "Jiyan": {"count": 255}, - "\u00c5\u009eah\u00c3\u00ae": {"count": 107}, - "Siyaset": {"count": 126}, - "\u00c3\u0087and": {"count": 349}, - "Zan\u00c3\u00aen": {"count": 82}, - "D\u00c3\u00aen": {"count": 150}, - "Helwesta_mirovan": {"count": 41}, - "Zanist": {"count": 138}, - "Hey\u00c3\u00aen": {"count": 49}, - "Ziman": {"count": 69}, - "Xweza": {"count": 129}, - "Matemat\u00c3\u00aek": {"count": 23}, - "Agah\u00c3\u00ae": {"count": 17}, - "Dad": {"count": 9}, - "\u00c3\u008enternet": {"count": 19}, - "Hik\u00c3\u00bbmet": {"count": 35}, - "Konsept": {"count": 10}, - "Perwerde": {"count": 6}, - "Werzi\u00c5\u009f": {"count": 20}, - "Felsefe": {"count": 28}, - "Et\u00c3\u00aek": {"count": 19}, - "Wize": {"count": 6}, - "Abor\u00c3\u00ae": {"count": 27}, - "Tendurust\u00c3\u00ae": {"count": 38}, - "Ragihandin": {"count": 5}, - "Karwer\u00c3\u00ae": {"count": 6}, - "Endezyar\u00c3\u00ae": {"count": 16}, - "Le\u00c5\u009fker\u00c3\u00ae": {"count": 18}, - "Kult\u00c5\u00abra": {"count": 749}, - "Latvija": {"count": 356}, - "\u00c4\u00a2eogr\u00c4\u0081fija": {"count": 721}, - "Zin\u00c4\u0081tne_un_tehnolo\u00c4\u00a3ijas": {"count": 469}, - "V\u00c4\u0093sture": {"count": 149}, - "Sabiedr\u00c4\u00abba": {"count": 878}, - "J\u00c4\u0093dzieni": {"count": 23}, - "Dz\u00c4\u00abv\u00c4\u00abba": {"count": 274}, - "Cilv\u00c4\u0093ki": {"count": 428}, - "Sports": {"count": 166}, - "M\u00c4\u0081ksla": {"count": 185}, - "Daba": {"count": 412}, - "Vesel\u00c4\u00abba": {"count": 124}, - "Reli\u00c4\u00a3ija": {"count": 86}, - "Notikumi": {"count": 13}, - "Makaluak_iduik": {"count": 3986}, - "Indonesia": {"count": 184}, - "Budayo": {"count": 399}, - "Ilimu_pangatauan": {"count": 121}, - "Makanan": {"count": 12}, - "Seni": {"count": 32}, - "Organisasi": {"count": 23}, - "Karajo": {"count": 30}, - "Astronomi": {"count": 13}, - "Sarugo": {"count": 30}, - "Tokoh": {"count": 26}, - "Agamo": {"count": 48}, - "Sijarah": {"count": 5}, - "Teknologi": {"count": 1}, - "Ulahrago": {"count": 2}, - "\u00c4\u00a0eografija": {"count": 1634}, - "Arti": {"count": 194}, - "Gvern": {"count": 107}, - "Reli\u00c4\u00a1jon": {"count": 293}, - "Dixxiplini_akkademi\u00c4\u008bi": {"count": 139}, - "Nies": {"count": 270}, - "So\u00c4\u008bjet\u00c3\u00a0": {"count": 88}, - "Natura": {"count": 449}, - "Sa\u00c4\u00a7\u00c4\u00a7a": {"count": 31}, - "Xjenza": {"count": 667}, - "Storja": {"count": 167}, - "Ekonomija": {"count": 199}, - "Lingwa": {"count": 76}, - "Filosofija": {"count": 14}, - "\u00c4\u00a6ajja_ta\\_Kuljum": {"count": 81}, - "Edukazzjoni": {"count": 30}, - "Mu\u00c5\u00bcika": {"count": 14}, - "Komunikazzjoni_umana": {"count": 39}, - "Spettaklu": {"count": 38}, - "Kronolo\u00c4\u00a1ija": {"count": 39}, - "Avvenimenti": {"count": 6}, - "Li\u00c4\u00a1i": {"count": 19}, - "Teknolo\u00c4\u00a1ija": {"count": 17}, - "In\u00c4\u00a1inerija": {"count": 2}, - "Life": {"count": 621}, - "Naitur": {"count": 265}, - "Society": {"count": 446}, - "Humanities": {"count": 259}, - "History": {"count": 184}, - "Airts": {"count": 106}, - "Technology": {"count": 324}, - "Fowk": {"count": 208}, - "Concepts": {"count": 237}, - "Cultur": {"count": 427}, - "Environs": {"count": 231}, - "Warld": {"count": 141}, - "Politics": {"count": 294}, - "Eddication": {"count": 42}, - "Airt": {"count": 18}, - "Heal": {"count": 70}, - "Science_an_technology": {"count": 60}, - "Mathematics": {"count": 36}, - "Law": {"count": 3}, - "Tuils": {"count": 7}, - "Employment": {"count": 14}, - "Gjeografi": {"count": 586}, - "Politik\u00c3\u00ab": {"count": 351}, - "Let\u00c3\u00abrsi": {"count": 67}, - "Administrat\u00c3\u00ab_publike": {"count": 320}, - "Shoq\u00c3\u00abri": {"count": 116}, - "Sporte": {"count": 105}, - "Shkenc\u00c3\u00ab": {"count": 1109}, - "Kultur\u00c3\u00ab": {"count": 299}, - "Arte": {"count": 217}, - "Persona": {"count": 425}, - "Histori": {"count": 744}, - "Mitologji": {"count": 5}, - "Gjuh\u00c3\u00absi": {"count": 64}, - "Teknologji": {"count": 84}, - "Kinematografi": {"count": 72}, - "Media": {"count": 51}, - "Sigurime": {"count": 31}, - "Loj\u00c3\u00abra": {"count": 3}, - "Fe": {"count": 131}, - "Bujq\u00c3\u00absi": {"count": 41}, - "Ngjarje": {"count": 11}, - "Biografi": {"count": 116}, - "Matematik\u00c3\u00ab": {"count": 27}, - "Teknik\u00c3\u00ab": {"count": 26}, - "Drejt\u00c3\u00absi": {"count": 18}, - "Organizata": {"count": 27}, - "Jeta": {"count": 4}, - "Agronomi": {"count": 3}, - "Natyr\u00c3\u00ab": {"count": 11}, - "Sh\u00c3\u00abndeti": {"count": 3}, - "Shkencat_humane": {"count": 22}, - "Shp\u00c3\u00abrblime": {"count": 2}, - "Blegtori": {"count": 10}, - "L\u00c3\u00abnd\u00c3\u00ab": {"count": 8}, - "Enciklopedistika": {"count": 6}, - "Economeye": {"count": 816}, - "Syinces": {"count": 3653}, - "Vicaedje_des_djins": {"count": 314}, - "Creyance": {"count": 310}, - "Rilom\u00c3\u00aay\u00c3\u00a8s_djins": {"count": 25}, - "Date": {"count": 2}, - }, - "hf_subset_descriptive_stats": { - "bs": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 17, - "labels": { - "Nauke": {"count": 1492}, - "Dru\u00c5\u00a1tvo": {"count": 504}, - "Priroda": {"count": 448}, - "Kultura": {"count": 453}, - "Tehnologija": {"count": 671}, - "Tehnika": {"count": 281}, - "Geografija": {"count": 431}, - "Informatika": {"count": 355}, - "Koncepti": {"count": 83}, - "Humanisti\u00c4\u008dke_nauke": {"count": 62}, - "Informacija": {"count": 21}, - "Historija": {"count": 223}, - "Matematika": {"count": 31}, - "Okoli\u00c5\u00a1": {"count": 6}, - "Jezik": {"count": 15}, - "Misao": {"count": 28}, - "Energija": {"count": 16}, - }, - }, - "ca": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 8, - "labels": { - "Llocs": {"count": 642}, - "Ci\u00c3\u00a8ncia": {"count": 1844}, - "Humanitats": {"count": 984}, - "Tecnologia": {"count": 377}, - "Biografies": {"count": 406}, - "Cultura": {"count": 710}, - "Informaci\u00c3\u00b3": {"count": 137}, - "Esdeveniments": {"count": 20}, - }, - }, - "cs": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 21, - "labels": { - "Lid\u00c3\u00a9": {"count": 1559}, - "Geografie": {"count": 578}, - "\u00c4\u008cas": {"count": 88}, - "Politika": {"count": 538}, - "V\u00c4\u009bda": {"count": 314}, - "Technika": {"count": 189}, - "Informace": {"count": 96}, - "\u00c5\u00bdivot": {"count": 184}, - "Vojenstv\u00c3\u00ad": {"count": 34}, - "Um\u00c4\u009bn\u00c3\u00ad": {"count": 95}, - "P\u00c5\u0099\u00c3\u00adroda": {"count": 387}, - "Spole\u00c4\u008dnost": {"count": 319}, - "Historie": {"count": 207}, - "Sport": {"count": 22}, - "Kultura": {"count": 150}, - "Matematika": {"count": 34}, - "Dorozum\u00c3\u00adv\u00c3\u00a1n\u00c3\u00ad": { - "count": 142 - }, - "Zdravotnictv\u00c3\u00ad": {"count": 94}, - "Vzd\u00c4\u009bl\u00c3\u00a1v\u00c3\u00a1n\u00c3\u00ad": { - "count": 41 - }, - "P\u00c5\u0099edm\u00c4\u009bty": {"count": 21}, - "Pr\u00c3\u00a1vo": {"count": 28}, - }, - }, - "da": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 20, - "labels": { - "Natur": {"count": 398}, - "Kultur": {"count": 911}, - "Samfund": {"count": 531}, - "Politik": {"count": 287}, - "Computersystemer": {"count": 84}, - "Erhvervsliv": {"count": 164}, - "Teknik": {"count": 166}, - "Sproget": {"count": 41}, - "Humaniora": {"count": 139}, - "Uddannelse": {"count": 79}, - "Anvendt_videnskab": {"count": 199}, - "Kunst": {"count": 412}, - "Sundhed": {"count": 83}, - "Historie": {"count": 184}, - "Videnskab": {"count": 257}, - "Matematik": {"count": 79}, - "Geografi": {"count": 463}, - "Liv": {"count": 286}, - "Personer": {"count": 322}, - "Jura": {"count": 35}, - }, - }, - "eu": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 5, - "labels": { - "Entitateak": {"count": 2486}, - "Humanitateak": {"count": 861}, - "Mundua": {"count": 706}, - "Zientzia_eta_teknologia": {"count": 957}, - "Entretenimendua": {"count": 110}, - }, - }, - "gv": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 28, - "labels": { - "Chron-oaylleeaght": {"count": 1334}, - "\u00c3\u0087heeraghyn": {"count": 470}, - "Oaylleeaght": {"count": 575}, - "\u00c3\u0087heer-oaylleeaght": {"count": 190}, - "Persoonyn": {"count": 249}, - "Bea": {"count": 207}, - "Cultoor": {"count": 269}, - "Glare": {"count": 152}, - "Dooghys": {"count": 494}, - "Smooinaght": {"count": 28}, - "Sheshaght": {"count": 221}, - "\u00c3\u0087haghnoaylleeaght": {"count": 33}, - "Strughtoor": {"count": 54}, - "Politickaght": {"count": 138}, - "Ronnaghyn_rere_bun-chooish": {"count": 381}, - "Eirinys": {"count": 80}, - "Ynsagh": {"count": 16}, - "Am": {"count": 12}, - "Leigh": {"count": 9}, - "Studeyrys-sheelnaue": {"count": 72}, - "Keirdyn_scoillaragh": {"count": 50}, - "Shennaghys": {"count": 11}, - "Dellal": {"count": 23}, - "Maddaght": {"count": 6}, - "Meanyn_mooarey": {"count": 7}, - "Oaylleeaghtyn_feamagh": {"count": 30}, - "Fysseree": {"count": 7}, - "\u00c3\u0087hymmyltaght": {"count": 2}, - }, - }, - "ilo": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 34, - "labels": { - "Katutubo": {"count": 135}, - "Biag": {"count": 933}, - "Pakasaritaan": {"count": 1302}, - "Kultura": {"count": 72}, - "Relihion": {"count": 20}, - "Dagiti_kinatao": {"count": 1405}, - "Lubong": {"count": 308}, - "Siensia_ken_teknolohia": {"count": 12}, - "Gobierno": {"count": 208}, - "Heograpia": {"count": 323}, - "Matematika": {"count": 9}, - "Etika": {"count": 12}, - "Pilosopia": {"count": 26}, - "Tattao": {"count": 20}, - "Dagiti_konsepto": {"count": 69}, - "Panagkukua_ti_tao": {"count": 16}, - "Pannakaammo": {"count": 29}, - "Palpaliwa": {"count": 22}, - "Pagsasao": {"count": 33}, - "Kagimongan": {"count": 37}, - "Salun-at": {"count": 9}, - "Industria": {"count": 5}, - "Linteg": {"count": 1}, - "Makan_ken_mainum": {"count": 19}, - "Dagiti_akademiko_a_disiplina": {"count": 27}, - "Edukasion": {"count": 9}, - "Musika": {"count": 1}, - "Politika": {"count": 37}, - "Negosio": {"count": 10}, - "Inhenieria": {"count": 1}, - "Paspasamak": {"count": 6}, - "Dagiti_banag": {"count": 2}, - "Annuroten": {"count": 1}, - "Enerhia": {"count": 1}, - }, - }, - "ku": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 39, - "labels": { - "Kes": {"count": 524}, - "Medya": {"count": 67}, - "Xwarin_\u00c3\u00bb_vexwarin": {"count": 127}, - "Civak": {"count": 277}, - "L\u00c3\u00aakol\u00c3\u00aen\u00c3\u00aan_mirovahiy\u00c3\u00aa": { - "count": 269 - }, - "Zeman": {"count": 174}, - "Teknoloj\u00c3\u00ae": {"count": 92}, - "D\u00c3\u00aes\u00c3\u00aepl\u00c3\u00aen\u00c3\u00aan_akadem\u00c3\u00aek": { - "count": 237 - }, - "Erdn\u00c3\u00aegar\u00c3\u00ae": {"count": 1078}, - "D\u00c3\u00aerok": {"count": 436}, - "Gerd\u00c3\u00bbn": {"count": 42}, - "Jiyan": {"count": 255}, - "\u00c5\u009eah\u00c3\u00ae": {"count": 107}, - "Siyaset": {"count": 126}, - "\u00c3\u0087and": {"count": 349}, - "Zan\u00c3\u00aen": {"count": 82}, - "D\u00c3\u00aen": {"count": 150}, - "Helwesta_mirovan": {"count": 41}, - "Zanist": {"count": 138}, - "Hey\u00c3\u00aen": {"count": 49}, - "Ziman": {"count": 69}, - "Xweza": {"count": 129}, - "Matemat\u00c3\u00aek": {"count": 23}, - "Agah\u00c3\u00ae": {"count": 17}, - "Dad": {"count": 9}, - "\u00c3\u008enternet": {"count": 19}, - "Hik\u00c3\u00bbmet": {"count": 35}, - "Konsept": {"count": 10}, - "Perwerde": {"count": 6}, - "Werzi\u00c5\u009f": {"count": 20}, - "Felsefe": {"count": 28}, - "Et\u00c3\u00aek": {"count": 19}, - "Wize": {"count": 6}, - "Abor\u00c3\u00ae": {"count": 27}, - "Tendurust\u00c3\u00ae": {"count": 38}, - "Ragihandin": {"count": 5}, - "Karwer\u00c3\u00ae": {"count": 6}, - "Endezyar\u00c3\u00ae": {"count": 16}, - "Le\u00c5\u009fker\u00c3\u00ae": {"count": 18}, - }, - }, - "lv": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 16, - "labels": { - "Kult\u00c5\u00abra": {"count": 749}, - "Latvija": {"count": 356}, - "\u00c4\u00a2eogr\u00c4\u0081fija": {"count": 721}, - "Zin\u00c4\u0081tne_un_tehnolo\u00c4\u00a3ijas": { - "count": 469 - }, - "V\u00c4\u0093sture": {"count": 149}, - "Sabiedr\u00c4\u00abba": {"count": 878}, - "J\u00c4\u0093dzieni": {"count": 23}, - "Dz\u00c4\u00abv\u00c4\u00abba": {"count": 274}, - "Cilv\u00c4\u0093ki": {"count": 428}, - "Sports": {"count": 120}, - "M\u00c4\u0081ksla": {"count": 185}, - "Politika": {"count": 133}, - "Daba": {"count": 412}, - "Vesel\u00c4\u00abba": {"count": 124}, - "Reli\u00c4\u00a3ija": {"count": 86}, - "Notikumi": {"count": 13}, - }, - }, - "min": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 16, - "labels": { - "Makaluak_iduik": {"count": 3986}, - "Indonesia": {"count": 184}, - "Budayo": {"count": 399}, - "Ilimu_pangatauan": {"count": 121}, - "Geografi": {"count": 208}, - "Makanan": {"count": 12}, - "Seni": {"count": 32}, - "Organisasi": {"count": 23}, - "Karajo": {"count": 30}, - "Astronomi": {"count": 13}, - "Sarugo": {"count": 30}, - "Tokoh": {"count": 26}, - "Agamo": {"count": 48}, - "Sijarah": {"count": 5}, - "Teknologi": {"count": 1}, - "Ulahrago": {"count": 2}, - }, - }, - "mt": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 27, - "labels": { - "\u00c4\u00a0eografija": {"count": 1634}, - "Arti": {"count": 194}, - "Gvern": {"count": 107}, - "Reli\u00c4\u00a1jon": {"count": 293}, - "Dixxiplini_akkademi\u00c4\u008bi": {"count": 139}, - "Nies": {"count": 270}, - "So\u00c4\u008bjet\u00c3\u00a0": {"count": 88}, - "Natura": {"count": 449}, - "Sa\u00c4\u00a7\u00c4\u00a7a": {"count": 31}, - "Xjenza": {"count": 667}, - "Storja": {"count": 167}, - "Ekonomija": {"count": 199}, - "Kultura": {"count": 367}, - "Lingwa": {"count": 76}, - "Filosofija": {"count": 14}, - "\u00c4\u00a6ajja_ta\\_Kuljum": {"count": 81}, - "Edukazzjoni": {"count": 30}, - "Politika": {"count": 110}, - "Mu\u00c5\u00bcika": {"count": 14}, - "Komunikazzjoni_umana": {"count": 39}, - "Spettaklu": {"count": 38}, - "Kronolo\u00c4\u00a1ija": {"count": 39}, - "Avvenimenti": {"count": 6}, - "Li\u00c4\u00a1i": {"count": 19}, - "Teknolo\u00c4\u00a1ija": {"count": 17}, - "Sport": {"count": 30}, - "In\u00c4\u00a1inerija": {"count": 2}, - }, - }, - "sco": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 23, - "labels": { - "Life": {"count": 621}, - "Naitur": {"count": 265}, - "Geografie": {"count": 1081}, - "Society": {"count": 446}, - "Humanities": {"count": 259}, - "History": {"count": 184}, - "Airts": {"count": 106}, - "Technology": {"count": 324}, - "Fowk": {"count": 208}, - "Concepts": {"count": 237}, - "Cultur": {"count": 427}, - "Environs": {"count": 231}, - "Warld": {"count": 141}, - "Politics": {"count": 294}, - "Eddication": {"count": 42}, - "Airt": {"count": 18}, - "Heal": {"count": 70}, - "Science_an_technology": {"count": 60}, - "Sports": {"count": 46}, - "Mathematics": {"count": 36}, - "Law": {"count": 3}, - "Tuils": {"count": 7}, - "Employment": {"count": 14}, - }, - }, - "sq": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 36, - "labels": { - "Gjeografi": {"count": 586}, - "Politik\u00c3\u00ab": {"count": 351}, - "Let\u00c3\u00abrsi": {"count": 67}, - "Administrat\u00c3\u00ab_publike": {"count": 320}, - "Shoq\u00c3\u00abri": {"count": 116}, - "Sporte": {"count": 105}, - "Shkenc\u00c3\u00ab": {"count": 1109}, - "Kultur\u00c3\u00ab": {"count": 299}, - "Arte": {"count": 217}, - "Persona": {"count": 425}, - "Histori": {"count": 744}, - "Mitologji": {"count": 5}, - "Gjuh\u00c3\u00absi": {"count": 64}, - "Teknologji": {"count": 84}, - "Kinematografi": {"count": 72}, - "Media": {"count": 51}, - "Sigurime": {"count": 31}, - "Loj\u00c3\u00abra": {"count": 3}, - "Fe": {"count": 131}, - "Bujq\u00c3\u00absi": {"count": 41}, - "Ngjarje": {"count": 11}, - "Biografi": {"count": 116}, - "Matematik\u00c3\u00ab": {"count": 27}, - "Teknik\u00c3\u00ab": {"count": 26}, - "Drejt\u00c3\u00absi": {"count": 18}, - "Organizata": {"count": 27}, - "Jeta": {"count": 4}, - "Sport": {"count": 5}, - "Agronomi": {"count": 3}, - "Natyr\u00c3\u00ab": {"count": 11}, - "Sh\u00c3\u00abndeti": {"count": 3}, - "Shkencat_humane": {"count": 22}, - "Shp\u00c3\u00abrblime": {"count": 2}, - "Blegtori": {"count": 10}, - "L\u00c3\u00abnd\u00c3\u00ab": {"count": 8}, - "Enciklopedistika": {"count": 6}, - }, - }, - "wa": { - "num_samples": 10, - "average_text_length": 512.0, - "average_labels_per_text": 512.0, - "unique_labels": 6, - "labels": { - "Economeye": {"count": 816}, - "Syinces": {"count": 3653}, - "Vicaedje_des_djins": {"count": 314}, - "Creyance": {"count": 310}, - "Rilom\u00c3\u00aay\u00c3\u00a8s_djins": {"count": 25}, - "Date": {"count": 2}, - }, - }, - }, - }, - }, ) @@ -800,460 +81,6 @@ class WikiClusteringFastP2P(AbsTaskClusteringFast, MultilingualTask): dialect=[], sample_creation="created", bibtex_citation="", # None exists - descriptive_stats={ - "n_samples": {"test": 2048}, - "test": { - "num_samples": 28672, - "average_text_length": 629.7426409040179, - "average_labels_per_text": 1.0, - "unique_labels": 39, - "labels": { - "16": {"count": 541}, - "3": {"count": 1607}, - "12": {"count": 846}, - "0": {"count": 2410}, - "15": {"count": 878}, - "11": {"count": 864}, - "6": {"count": 787}, - "9": {"count": 654}, - "14": {"count": 966}, - "8": {"count": 1389}, - "2": {"count": 2428}, - "10": {"count": 839}, - "1": {"count": 1370}, - "4": {"count": 2942}, - "7": {"count": 2514}, - "5": {"count": 1490}, - "13": {"count": 918}, - "19": {"count": 315}, - "17": {"count": 711}, - "20": {"count": 345}, - "18": {"count": 800}, - "24": {"count": 467}, - "25": {"count": 928}, - "21": {"count": 62}, - "26": {"count": 270}, - "22": {"count": 186}, - "23": {"count": 36}, - "27": {"count": 465}, - "28": {"count": 62}, - "36": {"count": 139}, - "32": {"count": 57}, - "38": {"count": 43}, - "30": {"count": 52}, - "34": {"count": 80}, - "33": {"count": 75}, - "35": {"count": 62}, - "31": {"count": 63}, - "37": {"count": 8}, - "29": {"count": 3}, - }, - "hf_subset_descriptive_stats": { - "bs": { - "num_samples": 2048, - "average_text_length": 1046.25732421875, - "average_labels_per_text": 1.0, - "unique_labels": 17, - "labels": { - "16": {"count": 268}, - "3": {"count": 89}, - "12": {"count": 597}, - "0": {"count": 202}, - "15": {"count": 113}, - "11": {"count": 11}, - "6": {"count": 142}, - "9": {"count": 181}, - "14": {"count": 179}, - "8": {"count": 33}, - "2": {"count": 172}, - "10": {"count": 12}, - "1": {"count": 7}, - "4": {"count": 25}, - "7": {"count": 6}, - "5": {"count": 9}, - "13": {"count": 2}, - }, - }, - "ca": { - "num_samples": 2048, - "average_text_length": 600.73291015625, - "average_labels_per_text": 1.0, - "unique_labels": 8, - "labels": { - "6": {"count": 257}, - "1": {"count": 737}, - "2": {"count": 284}, - "4": {"count": 394}, - "0": {"count": 162}, - "7": {"count": 151}, - "5": {"count": 55}, - "3": {"count": 8}, - }, - }, - "cs": { - "num_samples": 2048, - "average_text_length": 659.2294921875, - "average_labels_per_text": 1.0, - "unique_labels": 21, - "labels": { - "19": {"count": 35}, - "5": {"count": 624}, - "17": {"count": 126}, - "10": {"count": 155}, - "1": {"count": 231}, - "7": {"count": 215}, - "11": {"count": 128}, - "0": {"count": 57}, - "13": {"count": 75}, - "2": {"count": 83}, - "3": {"count": 38}, - "9": {"count": 8}, - "6": {"count": 14}, - "12": {"count": 9}, - "16": {"count": 16}, - "20": {"count": 73}, - "18": {"count": 38}, - "4": {"count": 60}, - "15": {"count": 14}, - "14": {"count": 38}, - "8": {"count": 11}, - }, - }, - "da": { - "num_samples": 2048, - "average_text_length": 767.58935546875, - "average_labels_per_text": 1.0, - "unique_labels": 20, - "labels": { - "14": {"count": 212}, - "4": {"count": 74}, - "15": {"count": 16}, - "8": {"count": 165}, - "13": {"count": 115}, - "0": {"count": 79}, - "1": {"count": 34}, - "9": {"count": 114}, - "7": {"count": 364}, - "10": {"count": 32}, - "17": {"count": 66}, - "18": {"count": 32}, - "12": {"count": 129}, - "11": {"count": 159}, - "2": {"count": 66}, - "3": {"count": 185}, - "19": {"count": 103}, - "16": {"count": 33}, - "5": {"count": 56}, - "6": {"count": 14}, - }, - }, - "eu": { - "num_samples": 2048, - "average_text_length": 405.16015625, - "average_labels_per_text": 1.0, - "unique_labels": 5, - "labels": { - "4": {"count": 383}, - "0": {"count": 995}, - "3": {"count": 282}, - "2": {"count": 344}, - "1": {"count": 44}, - }, - }, - "gv": { - "num_samples": 2048, - "average_text_length": 368.01123046875, - "average_labels_per_text": 1.0, - "unique_labels": 28, - "labels": { - "6": {"count": 32}, - "1": {"count": 83}, - "24": {"count": 13}, - "17": {"count": 152}, - "2": {"count": 534}, - "25": {"count": 76}, - "5": {"count": 198}, - "15": {"count": 100}, - "21": {"count": 22}, - "26": {"count": 188}, - "13": {"count": 230}, - "20": {"count": 11}, - "3": {"count": 107}, - "19": {"count": 88}, - "16": {"count": 55}, - "22": {"count": 29}, - "14": {"count": 12}, - "8": {"count": 61}, - "0": {"count": 5}, - "10": {"count": 4}, - "4": {"count": 9}, - "23": {"count": 6}, - "7": {"count": 3}, - "9": {"count": 20}, - "18": {"count": 4}, - "12": {"count": 3}, - "27": {"count": 1}, - "11": {"count": 2}, - }, - }, - "ilo": { - "num_samples": 2048, - "average_text_length": 617.90771484375, - "average_labels_per_text": 1.0, - "unique_labels": 29, - "labels": { - "3": {"count": 562}, - "0": {"count": 373}, - "18": {"count": 521}, - "8": {"count": 129}, - "13": {"count": 123}, - "11": {"count": 54}, - "25": {"count": 8}, - "27": {"count": 5}, - "17": {"count": 13}, - "15": {"count": 4}, - "4": {"count": 28}, - "7": {"count": 83}, - "10": {"count": 15}, - "1": {"count": 11}, - "24": {"count": 15}, - "14": {"count": 8}, - "16": {"count": 4}, - "19": {"count": 9}, - "23": {"count": 10}, - "26": {"count": 4}, - "28": {"count": 8}, - "12": {"count": 29}, - "21": {"count": 12}, - "6": {"count": 5}, - "20": {"count": 6}, - "5": {"count": 4}, - "22": {"count": 2}, - "9": {"count": 2}, - "2": {"count": 1}, - }, - }, - "ku": { - "num_samples": 2048, - "average_text_length": 421.17333984375, - "average_labels_per_text": 1.0, - "unique_labels": 39, - "labels": { - "14": {"count": 14}, - "36": {"count": 139}, - "20": {"count": 108}, - "22": {"count": 27}, - "15": {"count": 102}, - "32": {"count": 55}, - "8": {"count": 431}, - "17": {"count": 210}, - "38": {"count": 43}, - "30": {"count": 51}, - "4": {"count": 60}, - "2": {"count": 111}, - "6": {"count": 95}, - "34": {"count": 70}, - "27": {"count": 15}, - "5": {"count": 174}, - "26": {"count": 37}, - "0": {"count": 11}, - "25": {"count": 50}, - "16": {"count": 2}, - "12": {"count": 16}, - "24": {"count": 2}, - "11": {"count": 17}, - "21": {"count": 9}, - "13": {"count": 20}, - "1": {"count": 7}, - "33": {"count": 33}, - "35": {"count": 28}, - "10": {"count": 11}, - "31": {"count": 51}, - "18": {"count": 4}, - "3": {"count": 4}, - "28": {"count": 8}, - "37": {"count": 8}, - "23": {"count": 2}, - "19": {"count": 7}, - "7": {"count": 6}, - "9": {"count": 8}, - "29": {"count": 2}, - }, - }, - "lv": { - "num_samples": 2048, - "average_text_length": 770.67138671875, - "average_labels_per_text": 1.0, - "unique_labels": 16, - "labels": { - "15": {"count": 288}, - "2": {"count": 110}, - "6": {"count": 74}, - "12": {"count": 50}, - "0": {"count": 171}, - "14": {"count": 188}, - "10": {"count": 351}, - "5": {"count": 142}, - "4": {"count": 300}, - "13": {"count": 60}, - "11": {"count": 48}, - "1": {"count": 165}, - "8": {"count": 53}, - "7": {"count": 5}, - "3": {"count": 9}, - "9": {"count": 34}, - }, - }, - "min": { - "num_samples": 2048, - "average_text_length": 631.74072265625, - "average_labels_per_text": 1.0, - "unique_labels": 15, - "labels": { - "7": {"count": 1595}, - "9": {"count": 9}, - "4": {"count": 48}, - "3": {"count": 83}, - "2": {"count": 160}, - "0": {"count": 19}, - "5": {"count": 74}, - "6": {"count": 12}, - "10": {"count": 12}, - "13": {"count": 10}, - "8": {"count": 5}, - "11": {"count": 13}, - "12": {"count": 2}, - "1": {"count": 5}, - "14": {"count": 1}, - }, - }, - "mt": { - "num_samples": 2048, - "average_text_length": 821.22265625, - "average_labels_per_text": 1.0, - "unique_labels": 27, - "labels": { - "12": {"count": 8}, - "10": {"count": 147}, - "14": {"count": 180}, - "17": {"count": 117}, - "25": {"count": 654}, - "19": {"count": 35}, - "0": {"count": 77}, - "3": {"count": 12}, - "16": {"count": 44}, - "15": {"count": 108}, - "24": {"count": 267}, - "6": {"count": 43}, - "26": {"count": 32}, - "4": {"count": 79}, - "22": {"count": 67}, - "9": {"count": 16}, - "8": {"count": 16}, - "2": {"count": 55}, - "5": {"count": 6}, - "11": {"count": 30}, - "18": {"count": 12}, - "21": {"count": 12}, - "20": {"count": 15}, - "23": {"count": 7}, - "13": {"count": 6}, - "7": {"count": 1}, - "1": {"count": 2}, - }, - }, - "sco": { - "num_samples": 2048, - "average_text_length": 1065.21044921875, - "average_labels_per_text": 1.0, - "unique_labels": 23, - "labels": { - "18": {"count": 178}, - "6": {"count": 92}, - "9": {"count": 28}, - "15": {"count": 106}, - "8": {"count": 432}, - "2": {"count": 95}, - "11": {"count": 104}, - "1": {"count": 42}, - "13": {"count": 248}, - "16": {"count": 118}, - "20": {"count": 130}, - "3": {"count": 171}, - "22": {"count": 57}, - "7": {"count": 83}, - "10": {"count": 74}, - "5": {"count": 6}, - "4": {"count": 17}, - "17": {"count": 24}, - "14": {"count": 14}, - "0": {"count": 7}, - "19": {"count": 18}, - "21": {"count": 3}, - "12": {"count": 1}, - }, - }, - "sq": { - "num_samples": 2048, - "average_text_length": 425.486328125, - "average_labels_per_text": 1.0, - "unique_labels": 36, - "labels": { - "27": {"count": 444}, - "9": {"count": 234}, - "14": {"count": 120}, - "0": {"count": 128}, - "15": {"count": 27}, - "11": {"count": 298}, - "24": {"count": 170}, - "28": {"count": 46}, - "19": {"count": 20}, - "25": {"count": 140}, - "3": {"count": 47}, - "2": {"count": 87}, - "35": {"count": 34}, - "8": {"count": 53}, - "31": {"count": 12}, - "17": {"count": 3}, - "23": {"count": 11}, - "20": {"count": 2}, - "33": {"count": 42}, - "10": {"count": 26}, - "34": {"count": 10}, - "7": {"count": 2}, - "13": {"count": 29}, - "4": {"count": 4}, - "6": {"count": 7}, - "26": {"count": 9}, - "5": {"count": 16}, - "30": {"count": 1}, - "21": {"count": 4}, - "22": {"count": 4}, - "18": {"count": 11}, - "32": {"count": 2}, - "12": {"count": 2}, - "16": {"count": 1}, - "1": {"count": 1}, - "29": {"count": 1}, - }, - }, - "wa": { - "num_samples": 2048, - "average_text_length": 216.00390625, - "average_labels_per_text": 1.0, - "unique_labels": 6, - "labels": { - "5": {"count": 126}, - "4": {"count": 1461}, - "0": {"count": 124}, - "2": {"count": 326}, - "3": {"count": 10}, - "1": {"count": 1}, - }, - }, - }, - }, - }, ) def dataset_transform(self): @@ -1290,5 +117,4 @@ def dataset_transform(self): self.seed, self.metadata.eval_splits, label="labels", - n_samples=2048, ) diff --git a/mteb/tasks/Clustering/nob/SNLHierarchicalClustering.py b/mteb/tasks/Clustering/nob/SNLHierarchicalClustering.py index 3ec8e8e3ef..fd1ded3cfa 100644 --- a/mteb/tasks/Clustering/nob/SNLHierarchicalClustering.py +++ b/mteb/tasks/Clustering/nob/SNLHierarchicalClustering.py @@ -42,10 +42,7 @@ class SNLHierarchicalClusteringP2P(AbsTaskClusteringFast): year={2023}, school={Norwegian University of Life Sciences, {\AA}s} }""", - descriptive_stats={ - "n_samples": {"test": 1300}, - "avg_character_length": {"test": 1986.9453846153847}, - }, + prompt="Identify categories in a Norwegian lexicon", ) max_depth = 5 @@ -87,10 +84,7 @@ class SNLHierarchicalClusteringS2S(AbsTaskClusteringFast): year={2023}, school={Norwegian University of Life Sciences, {\AA}s} }""", - descriptive_stats={ - "n_samples": {"test": 1300}, - "avg_character_length": {"test": 242.22384615384615}, - }, + prompt="Identify categories in a Norwegian lexicon", ) max_depth = 5 diff --git a/mteb/tasks/Clustering/nob/VGHierarchicalClustering.py b/mteb/tasks/Clustering/nob/VGHierarchicalClustering.py index ff708d00a3..059fbd5447 100644 --- a/mteb/tasks/Clustering/nob/VGHierarchicalClustering.py +++ b/mteb/tasks/Clustering/nob/VGHierarchicalClustering.py @@ -42,10 +42,7 @@ class VGHierarchicalClusteringP2P(AbsTaskClusteringFast): year={2023}, school={Norwegian University of Life Sciences, {\AA}s} }""", - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "avg_character_length": {"test": 2670.3243084794544}, - }, + prompt="Identify the categories (e.g. sports) of given articles in Norwegian", ) def dataset_transform(self) -> None: @@ -90,10 +87,7 @@ class VGHierarchicalClusteringS2S(AbsTaskClusteringFast): year={2023}, school={Norwegian University of Life Sciences, {\AA}s} }""", - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "avg_character_length": {"test": 139.31247668283325}, - }, + prompt="Identify the categories (e.g. sports) of given articles in Norwegian", ) def dataset_transform(self) -> None: diff --git a/mteb/tasks/Clustering/nob/snl_clustering.py b/mteb/tasks/Clustering/nob/snl_clustering.py index 0acb6d52b1..9256fc66c0 100644 --- a/mteb/tasks/Clustering/nob/snl_clustering.py +++ b/mteb/tasks/Clustering/nob/snl_clustering.py @@ -51,10 +51,6 @@ class SNLClustering(AbsTaskClustering): year={2023}, school={Norwegian University of Life Sciences, {\AA}s} }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 1101.30}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/nob/vg_clustering.py b/mteb/tasks/Clustering/nob/vg_clustering.py index 6c6c692fb7..f1050e796b 100644 --- a/mteb/tasks/Clustering/nob/vg_clustering.py +++ b/mteb/tasks/Clustering/nob/vg_clustering.py @@ -51,10 +51,6 @@ class VGClustering(AbsTaskClustering): year={2023}, school={Norwegian University of Life Sciences, {\AA}s} }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 1009.65}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/pol/PolishClustering.py b/mteb/tasks/Clustering/pol/PolishClustering.py index cf87fe9fa8..4c858fadf6 100644 --- a/mteb/tasks/Clustering/pol/PolishClustering.py +++ b/mteb/tasks/Clustering/pol/PolishClustering.py @@ -69,10 +69,6 @@ class EightTagsClustering(AbsTaskClustering): language = "English", ISBN = "979-10-95546-34-4", }""", - descriptive_stats={ - "n_samples": {"test": 49373}, - "avg_character_length": {"test": 78.23}, - }, ) @@ -132,10 +128,6 @@ class EightTagsClusteringFast(AbsTaskClusteringFast): language = "English", ISBN = "979-10-95546-34-4", }""", - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "avg_character_length": {"test": 78.73}, - }, ) def dataset_transform(self): @@ -181,10 +173,6 @@ class PlscClusteringS2S(AbsTaskClusteringFast): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 17534}, - "avg_character_length": {"test": 84.34}, - }, ) @@ -212,10 +200,6 @@ class PlscClusteringS2SFast(AbsTaskClusteringFast): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "avg_character_length": {"test": 84.34}, - }, ) def dataset_transform(self): @@ -270,10 +254,6 @@ class PlscClusteringP2P(AbsTaskClusteringFast): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 17537}, - "avg_character_length": {"test": 1023.21}, - }, ) @@ -301,10 +281,6 @@ class PlscClusteringP2PFast(AbsTaskClusteringFast): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "avg_character_length": {"test": 1023.21}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/rom/RomaniBibleClustering.py b/mteb/tasks/Clustering/rom/RomaniBibleClustering.py index 8801261ea8..7afb9adab7 100644 --- a/mteb/tasks/Clustering/rom/RomaniBibleClustering.py +++ b/mteb/tasks/Clustering/rom/RomaniBibleClustering.py @@ -27,8 +27,4 @@ class RomaniBibleClustering(AbsTaskClustering): dialect=["Kalderash"], sample_creation="human-translated and localized", bibtex_citation=None, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 132.2}, - }, ) diff --git a/mteb/tasks/Clustering/rus/GeoreviewClusteringP2P.py b/mteb/tasks/Clustering/rus/GeoreviewClusteringP2P.py index 38f5625934..ad1882b438 100644 --- a/mteb/tasks/Clustering/rus/GeoreviewClusteringP2P.py +++ b/mteb/tasks/Clustering/rus/GeoreviewClusteringP2P.py @@ -31,8 +31,5 @@ class GeoreviewClusteringP2P(AbsTaskClusteringFast): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 2000}, - "avg_character_length": {"test": 384.5}, - }, + prompt="Identify the organization category based on the reviews", ) diff --git a/mteb/tasks/Clustering/rus/RuSciBenchGRNTIClusteringP2P.py b/mteb/tasks/Clustering/rus/RuSciBenchGRNTIClusteringP2P.py index dab6be4db9..e61b1b1e39 100644 --- a/mteb/tasks/Clustering/rus/RuSciBenchGRNTIClusteringP2P.py +++ b/mteb/tasks/Clustering/rus/RuSciBenchGRNTIClusteringP2P.py @@ -32,45 +32,7 @@ class RuSciBenchGRNTIClusteringP2P(AbsTaskClusteringFast): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 2048}, - "test": { - "num_samples": 2048, - "average_text_length": 889.81396484375, - "average_labels_per_text": 1.0, - "unique_labels": 28, - "labels": { - "3": {"count": 73}, - "4": {"count": 73}, - "20": {"count": 73}, - "9": {"count": 73}, - "21": {"count": 73}, - "15": {"count": 73}, - "16": {"count": 74}, - "2": {"count": 73}, - "8": {"count": 73}, - "23": {"count": 73}, - "6": {"count": 73}, - "24": {"count": 73}, - "10": {"count": 73}, - "1": {"count": 73}, - "17": {"count": 74}, - "14": {"count": 74}, - "18": {"count": 73}, - "27": {"count": 73}, - "19": {"count": 73}, - "22": {"count": 73}, - "12": {"count": 73}, - "25": {"count": 73}, - "5": {"count": 74}, - "0": {"count": 73}, - "26": {"count": 73}, - "11": {"count": 73}, - "13": {"count": 73}, - "7": {"count": 73}, - }, - }, - }, + prompt="Identify the category of scientific papers based on the titles and abstracts", ) def dataset_transform(self): @@ -82,6 +44,5 @@ def dataset_transform(self): self.dataset, seed=self.seed, splits=["test"], - n_samples=2048, label="labels", ) diff --git a/mteb/tasks/Clustering/rus/RuSciBenchOECDClusteringP2P.py b/mteb/tasks/Clustering/rus/RuSciBenchOECDClusteringP2P.py index 25a27ea264..6bd79de5f8 100644 --- a/mteb/tasks/Clustering/rus/RuSciBenchOECDClusteringP2P.py +++ b/mteb/tasks/Clustering/rus/RuSciBenchOECDClusteringP2P.py @@ -32,10 +32,7 @@ class RuSciBenchOECDClusteringP2P(AbsTaskClusteringFast): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 838.9}, - }, + prompt="Identify the category of scientific papers based on the titles and abstracts", ) def dataset_transform(self): @@ -47,6 +44,5 @@ def dataset_transform(self): self.dataset, seed=self.seed, splits=["test"], - n_samples=2048, label="labels", ) diff --git a/mteb/tasks/Clustering/spa/SpanishNewsClusteringP2P.py b/mteb/tasks/Clustering/spa/SpanishNewsClusteringP2P.py index 1c72d8c550..39f11560d5 100644 --- a/mteb/tasks/Clustering/spa/SpanishNewsClusteringP2P.py +++ b/mteb/tasks/Clustering/spa/SpanishNewsClusteringP2P.py @@ -28,5 +28,4 @@ class SpanishNewsClusteringP2P(AbsTaskClustering): dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) diff --git a/mteb/tasks/Clustering/swe/SwednClustering.py b/mteb/tasks/Clustering/swe/SwednClustering.py index 552e8451a4..bef817ab6f 100644 --- a/mteb/tasks/Clustering/swe/SwednClustering.py +++ b/mteb/tasks/Clustering/swe/SwednClustering.py @@ -87,10 +87,7 @@ class SwednClusteringP2P(AbsTaskClusteringFast): booktitle={Proceedings of CLARIN Annual Conference}, year={2021} }""", - descriptive_stats={ - "n_samples": {"all": 2048}, - "avg_character_length": {"all": 1619.71}, - }, + prompt="Identify news categories in Swedish passages", ) def dataset_transform(self): @@ -130,10 +127,7 @@ class SwednClusteringFastS2S(AbsTaskClusteringFast): booktitle={Proceedings of CLARIN Annual Conference}, year={2021} }""", - descriptive_stats={ - "n_samples": {"all": 2048}, - "avg_character_length": {"all": 1619.71}, - }, + prompt="Identify news categories in Swedish passages", ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/swe/swedn_clustering.py b/mteb/tasks/Clustering/swe/swedn_clustering.py index 90b74d8734..ab13883172 100644 --- a/mteb/tasks/Clustering/swe/swedn_clustering.py +++ b/mteb/tasks/Clustering/swe/swedn_clustering.py @@ -54,10 +54,6 @@ class SwednClustering(AbsTaskClustering): booktitle={Proceedings of CLARIN Annual Conference}, year={2021} }""", - descriptive_stats={ - "n_samples": {"all": 2048}, - "avg_character_length": {"all": 1619.71}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Clustering/zho/CMTEBClustering.py b/mteb/tasks/Clustering/zho/CMTEBClustering.py index 4ce12d9bbd..fa0704b098 100644 --- a/mteb/tasks/Clustering/zho/CMTEBClustering.py +++ b/mteb/tasks/Clustering/zho/CMTEBClustering.py @@ -47,10 +47,7 @@ class CLSClusteringFastS2S(AbsTaskClusteringFast): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"test": NUM_SAMPLES}, - "avg_character_length": {}, - }, + prompt="Identify the main category of scholar papers based on the titles", ) def dataset_transform(self): @@ -70,7 +67,6 @@ def dataset_transform(self): self.seed, self.metadata.eval_splits, label="labels", - n_samples=NUM_SAMPLES, ) @@ -107,10 +103,7 @@ class CLSClusteringFastP2P(AbsTaskClusteringFast): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"test": NUM_SAMPLES}, - "avg_character_length": {}, - }, + prompt="Identify the main category of scholar papers based on the titles and abstracts", ) def dataset_transform(self): @@ -130,7 +123,6 @@ def dataset_transform(self): self.seed, self.metadata.eval_splits, label="labels", - n_samples=NUM_SAMPLES, ) @@ -166,7 +158,7 @@ class CLSClusteringS2S(AbsTaskClustering): year={2022} } """, - descriptive_stats={"n_samples": {"test": 100000}, "avg_character_length": None}, + prompt="Identify the main category of scholar papers based on the titles", ) @@ -200,7 +192,7 @@ class CLSClusteringP2P(AbsTaskClustering): journal={arXiv preprint arXiv:2209.05034}, year={2022} }""", - descriptive_stats={"n_samples": {"test": 100000}, "avg_character_length": None}, + prompt="Identify the main category of scholar papers based on the titles and abstracts", ) @@ -237,10 +229,7 @@ class ThuNewsClusteringFastS2S(AbsTaskClusteringFast): publisher = {THU Natural Language Processing Lab}, url = {https://github.com/thunlp/THUCTC} }""", - descriptive_stats={ - "n_samples": {"test": NUM_SAMPLES}, - "avg_character_length": {}, - }, + prompt="Identify the topic or theme of the given news articles based on the titles", ) def dataset_transform(self): @@ -260,7 +249,6 @@ def dataset_transform(self): self.seed, self.metadata.eval_splits, label="labels", - n_samples=NUM_SAMPLES, ) @@ -297,10 +285,7 @@ class ThuNewsClusteringFastP2P(AbsTaskClusteringFast): publisher = {THU Natural Language Processing Lab}, url = {https://github.com/thunlp/THUCTC} }""", - descriptive_stats={ - "n_samples": {"test": NUM_SAMPLES}, - "avg_character_length": {}, - }, + prompt="Identify the topic or theme of the given news articles based on the titles and contents", ) def dataset_transform(self): @@ -320,7 +305,6 @@ def dataset_transform(self): self.seed, self.metadata.eval_splits, label="labels", - n_samples=NUM_SAMPLES, ) @@ -363,7 +347,7 @@ class ThuNewsClusteringS2S(AbsTaskClustering): year={2006} } """, - descriptive_stats={"n_samples": {"test": 100000}, "avg_character_length": None}, + prompt="Identify the topic or theme of the given news articles based on the titles", ) @@ -406,5 +390,5 @@ class ThuNewsClusteringP2P(AbsTaskClustering): year={2006} } """, - descriptive_stats={"n_samples": {"test": 100000}, "avg_character_length": None}, + prompt="Identify the topic or theme of the given news articles based on the titles and contents", ) diff --git a/mteb/tasks/InstructionRetrieval/eng/Core17InstructionRetrieval.py b/mteb/tasks/InstructionRetrieval/eng/Core17InstructionRetrieval.py index 9b52f282b2..14fa5b45b9 100644 --- a/mteb/tasks/InstructionRetrieval/eng/Core17InstructionRetrieval.py +++ b/mteb/tasks/InstructionRetrieval/eng/Core17InstructionRetrieval.py @@ -35,17 +35,4 @@ class Core17InstructionRetrieval(AbsTaskInstructionRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": {"eng": 19919 * 2}, - "test": { - "num_docs": 19899, - "num_queries": 20, - "average_document_length": 2233.0329664807277, - "average_query_length": 109.75, - "average_instruction_length": 295.55, - "average_changed_instruction_length": 355.2, - "average_relevant_docs_per_query": 32.7, - "average_top_ranked_per_query": 1000.0, - }, - }, ) diff --git a/mteb/tasks/InstructionRetrieval/eng/News21InstructionRetrieval.py b/mteb/tasks/InstructionRetrieval/eng/News21InstructionRetrieval.py index d693091279..3973ca0b75 100644 --- a/mteb/tasks/InstructionRetrieval/eng/News21InstructionRetrieval.py +++ b/mteb/tasks/InstructionRetrieval/eng/News21InstructionRetrieval.py @@ -35,8 +35,4 @@ class News21InstructionRetrieval(AbsTaskInstructionRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": {"eng": 30953 * 2}, - "avg_character_length": {"eng": 2983.724665391969}, - }, ) diff --git a/mteb/tasks/InstructionRetrieval/eng/Robust04InstructionRetrieval.py b/mteb/tasks/InstructionRetrieval/eng/Robust04InstructionRetrieval.py index c68dfabc18..1d3cb5c923 100644 --- a/mteb/tasks/InstructionRetrieval/eng/Robust04InstructionRetrieval.py +++ b/mteb/tasks/InstructionRetrieval/eng/Robust04InstructionRetrieval.py @@ -35,8 +35,4 @@ class Robust04InstructionRetrieval(AbsTaskInstructionRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": {"eng": 47544 * 2}, - "avg_character_length": {"eng": 2471.0398058252426}, - }, ) diff --git a/mteb/tasks/InstructionRetrieval/multilingual/mFollowIR.py b/mteb/tasks/InstructionRetrieval/multilingual/mFollowIR.py index 04c1a56e19..9452beb8de 100644 --- a/mteb/tasks/InstructionRetrieval/multilingual/mFollowIR.py +++ b/mteb/tasks/InstructionRetrieval/multilingual/mFollowIR.py @@ -200,51 +200,6 @@ class mFollowIRCrossLingual(MultilingualTask, AbsTaskInstructionRetrieval): journal={arXiv preprint TODO}, year={2024} }""", - descriptive_stats={ - "n_samples": {"eng-fas": 40 * 2, "eng-rus": 40 * 2, "eng-zho": 43 * 2}, - "test": { - "num_docs": 121635, - "num_queries": 123, - "average_document_length": 2331.0777818884367, - "average_query_length": 81.8780487804878, - "average_instruction_length": 389.9512195121951, - "average_changed_instruction_length": 450.5528455284553, - "average_relevant_docs_per_query": 10.30952380952381, - "average_top_ranked_per_query": 1024.3902439024391, - "hf_subset_descriptive_stats": { - "eng-fas": { - "num_docs": 41189, - "num_queries": 40, - "average_document_length": 3145.4990895627475, - "average_query_length": 80.075, - "average_instruction_length": 396.875, - "average_changed_instruction_length": 463.175, - "average_relevant_docs_per_query": 10.465116279069768, - "average_top_ranked_per_query": 1075, - }, - "eng-rus": { - "num_docs": 39326, - "num_queries": 40, - "average_document_length": 2784.0813456746173, - "average_query_length": 81.875, - "average_instruction_length": 371.125, - "average_changed_instruction_length": 431.8, - "average_relevant_docs_per_query": 9.775, - "average_top_ranked_per_query": 1000, - }, - "eng-zho": { - "num_docs": 41120, - "num_queries": 43, - "average_document_length": 1082.0501215953307, - "average_query_length": 83.55813953488372, - "average_instruction_length": 401.0232558139535, - "average_changed_instruction_length": 456.25581395348837, - "average_relevant_docs_per_query": 10.651162790697674, - "average_top_ranked_per_query": 1000, - }, - }, - }, - }, ) def load_data(self, **kwargs): @@ -298,51 +253,6 @@ class mFollowIR(MultilingualTask, AbsTaskInstructionRetrieval): journal={arXiv preprint TODO}, year={2024} }""", - descriptive_stats={ - "n_samples": {"fas": 40 * 2, "rus": 40 * 2, "zho": 43 * 2}, - "test": { - "num_docs": 121635, - "num_queries": 123, - "average_document_length": 2331.0777818884367, - "average_query_length": 57.113821138211385, - "average_instruction_length": 281.0650406504065, - "average_changed_instruction_length": 326.9430894308943, - "average_relevant_docs_per_query": 10.30952380952381, - "average_top_ranked_per_query": 1024.3902439024391, - "hf_subset_descriptive_stats": { - "fas": { - "num_docs": 41189, - "num_queries": 40, - "average_document_length": 3145.4990895627475, - "average_query_length": 72.65, - "average_instruction_length": 358.925, - "average_changed_instruction_length": 415.325, - "average_relevant_docs_per_query": 10.465116279069768, - "average_top_ranked_per_query": 1075, - }, - "rus": { - "num_docs": 39326, - "num_queries": 40, - "average_document_length": 2784.0813456746173, - "average_query_length": 77.5, - "average_instruction_length": 387, - "average_changed_instruction_length": 458, - "average_relevant_docs_per_query": 9.775, - "average_top_ranked_per_query": 1000, - }, - "zho": { - "num_docs": 41120, - "num_queries": 43, - "average_document_length": 1082.0501215953307, - "average_query_length": 23.697674418604652, - "average_instruction_length": 110.09302325581395, - "average_changed_instruction_length": 122.81395348837209, - "average_relevant_docs_per_query": 10.651162790697674, - "average_top_ranked_per_query": 1000, - }, - }, - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/MultiLabelClassification/kor/KorHateSpeechMLClassification.py b/mteb/tasks/MultiLabelClassification/kor/KorHateSpeechMLClassification.py index 8c069dd351..bf969c4b2e 100644 --- a/mteb/tasks/MultiLabelClassification/kor/KorHateSpeechMLClassification.py +++ b/mteb/tasks/MultiLabelClassification/kor/KorHateSpeechMLClassification.py @@ -55,10 +55,6 @@ class KorHateSpeechMLClassification(AbsTaskMultilabelClassification): url = "https://aclanthology.org/2022.coling-1.311", pages = "3530--3538", }""", - descriptive_stats={ - "n_samples": {"train": 8192, "test": 2048}, - "avg_character_length": {"train": 33.67, "test": 34.67}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/MultiLabelClassification/mlt/MalteseNewsClassification.py b/mteb/tasks/MultiLabelClassification/mlt/MalteseNewsClassification.py index ea30843a86..d6cfc33482 100644 --- a/mteb/tasks/MultiLabelClassification/mlt/MalteseNewsClassification.py +++ b/mteb/tasks/MultiLabelClassification/mlt/MalteseNewsClassification.py @@ -43,10 +43,6 @@ class MalteseNewsClassification(AbsTaskMultilabelClassification): year = "2024", publisher = "Association for Computational Linguistics", }""", - descriptive_stats={ - "n_samples": {"train": 10784, "test": 2297}, - "avg_character_length": {"train": 1595.63, "test": 1752.1}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py b/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py index b5b8d4ec26..0aeff946aa 100644 --- a/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py +++ b/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py @@ -70,705 +70,4 @@ class MultiEURLEXMultilabelClassification( url = {https://arxiv.org/abs/2109.00904} } """, - descriptive_stats={ - "n_samples": {"test": 5000}, - "test": { - "average_text_length": 12014.408930434782, - "average_label_per_text": 3.5938, - "num_samples": 115000, - "unique_labels": 21, - "labels": { - "18": {"count": 50784}, - "15": {"count": 30981}, - "5": {"count": 24978}, - "6": {"count": 45080}, - "3": {"count": 63687}, - "17": {"count": 37743}, - "1": {"count": 15019}, - "20": {"count": 14030}, - "0": {"count": 17802}, - "2": {"count": 22402}, - "19": {"count": 10212}, - "9": {"count": 3772}, - "4": {"count": 9062}, - "10": {"count": 7705}, - "11": {"count": 12213}, - "7": {"count": 14306}, - "12": {"count": 11799}, - "8": {"count": 13800}, - "13": {"count": 2346}, - "14": {"count": 4255}, - "16": {"count": 1311}, - }, - "hf_subset_descriptive_stats": { - "en": { - "average_text_length": 11720.2926, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "de": { - "average_text_length": 12865.4162, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "fr": { - "average_text_length": 13081.1098, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "it": { - "average_text_length": 12763.4786, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "es": { - "average_text_length": 13080.29, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "pl": { - "average_text_length": 12282.5926, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "ro": { - "average_text_length": 12836.9322, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "nl": { - "average_text_length": 12857.9742, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "el": { - "average_text_length": 12998.143, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "hu": { - "average_text_length": 12424.641, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "pt": { - "average_text_length": 12482.4616, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "cs": { - "average_text_length": 10783.4676, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "sv": { - "average_text_length": 11612.4774, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "bg": { - "average_text_length": 12235.4268, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "da": { - "average_text_length": 11773.958, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "fi": { - "average_text_length": 12087.6862, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "sk": { - "average_text_length": 11130.814, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "lt": { - "average_text_length": 11245.3566, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "hr": { - "average_text_length": 11022.142, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "sl": { - "average_text_length": 10620.0594, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "et": { - "average_text_length": 10898.4312, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "lv": { - "average_text_length": 10938.5102, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - "mt": { - "average_text_length": 12589.7442, - "average_label_per_text": 3.5938, - "num_samples": 5000, - "unique_labels": 21, - "labels": { - "18": {"count": 2208}, - "15": {"count": 1347}, - "5": {"count": 1086}, - "6": {"count": 1960}, - "3": {"count": 2769}, - "17": {"count": 1641}, - "1": {"count": 653}, - "20": {"count": 610}, - "0": {"count": 774}, - "2": {"count": 974}, - "19": {"count": 444}, - "9": {"count": 164}, - "4": {"count": 394}, - "10": {"count": 335}, - "11": {"count": 531}, - "7": {"count": 622}, - "12": {"count": 513}, - "8": {"count": 600}, - "13": {"count": 102}, - "14": {"count": 185}, - "16": {"count": 57}, - }, - }, - }, - }, - }, ) diff --git a/mteb/tasks/MultiLabelClassification/por/BrazilianToxicTweetsClassification.py b/mteb/tasks/MultiLabelClassification/por/BrazilianToxicTweetsClassification.py index bbc81a0cb8..f56fa78d06 100644 --- a/mteb/tasks/MultiLabelClassification/por/BrazilianToxicTweetsClassification.py +++ b/mteb/tasks/MultiLabelClassification/por/BrazilianToxicTweetsClassification.py @@ -17,10 +17,8 @@ class BrazilianToxicTweetsClassification(AbsTaskMultilabelClassification): """, reference="https://paperswithcode.com/dataset/told-br", dataset={ - "path": "JAugusto97/told-br", - "revision": "fb4f11a5bc68b99891852d20f1ec074be6289768", - "name": "multilabel", - "trust_remote_code": True, + "path": "mteb/told-br", + "revision": "f333c1fcfa3ab43f008a327c8bd0140441354d34", }, type="MultilabelClassification", category="s2s", @@ -50,10 +48,6 @@ class BrazilianToxicTweetsClassification(AbsTaskMultilabelClassification): eprint = {2010.04543}, timestamp = {Tue, 15 Dec 2020 16:10:16 +0100}, }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 85.05}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/MultiLabelClassification/rus/CEDRClassification.py b/mteb/tasks/MultiLabelClassification/rus/CEDRClassification.py index 77ebe99caa..87795138d4 100644 --- a/mteb/tasks/MultiLabelClassification/rus/CEDRClassification.py +++ b/mteb/tasks/MultiLabelClassification/rus/CEDRClassification.py @@ -38,21 +38,5 @@ class CEDRClassification(AbsTaskMultilabelClassification): publisher={Elsevier} } """, - descriptive_stats={ - "n_samples": {"test": 1882}, - "test": { - "average_text_length": 91.20563230605738, - "average_label_per_text": 0.620616365568544, - "num_samples": 1882, - "unique_labels": 6, - "labels": { - "null": {"count": 734}, - "3": {"count": 141}, - "2": {"count": 170}, - "1": {"count": 379}, - "0": {"count": 353}, - "4": {"count": 125}, - }, - }, - }, + prompt="Given a comment as query, find expressed emotions (joy, sadness, surprise, fear, and anger)", ) diff --git a/mteb/tasks/MultiLabelClassification/rus/SensitiveTopicsClassification.py b/mteb/tasks/MultiLabelClassification/rus/SensitiveTopicsClassification.py index c53a2a6f43..fc199313d6 100644 --- a/mteb/tasks/MultiLabelClassification/rus/SensitiveTopicsClassification.py +++ b/mteb/tasks/MultiLabelClassification/rus/SensitiveTopicsClassification.py @@ -56,8 +56,5 @@ class SensitiveTopicsClassification(AbsTaskMultilabelClassification): pages = "26--36", abstract = "Not all topics are equally {``}flammable{''} in terms of toxicity: a calm discussion of turtles or fishing less often fuels inappropriate toxic dialogues than a discussion of politics or sexual minorities. We define a set of sensitive topics that can yield inappropriate and toxic messages and describe the methodology of collecting and labelling a dataset for appropriateness. While toxicity in user-generated data is well-studied, we aim at defining a more fine-grained notion of inappropriateness. The core of inappropriateness is that it can harm the reputation of a speaker. This is different from toxicity in two respects: (i) inappropriateness is topic-related, and (ii) inappropriate message is not toxic but still unacceptable. We collect and release two datasets for Russian: a topic-labelled dataset and an appropriateness-labelled dataset. We also release pre-trained classification models trained on this data.", }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 95.3}, - }, + prompt="Given a sentence as query, find sensitive topics", ) diff --git a/mteb/tasks/PairClassification/ara/ArEntail.py b/mteb/tasks/PairClassification/ara/ArEntail.py index 588027cb0e..9afce29d71 100644 --- a/mteb/tasks/PairClassification/ara/ArEntail.py +++ b/mteb/tasks/PairClassification/ara/ArEntail.py @@ -37,10 +37,6 @@ class ArEntail(AbsTaskPairClassification): year={2024}, publisher={Springer} }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": {"test": 65.77}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/ces/CTKFactsNLI.py b/mteb/tasks/PairClassification/ces/CTKFactsNLI.py index d2e3296df8..48b2738dd5 100644 --- a/mteb/tasks/PairClassification/ces/CTKFactsNLI.py +++ b/mteb/tasks/PairClassification/ces/CTKFactsNLI.py @@ -37,13 +37,6 @@ class CTKFactsNLI(AbsTaskPairClassification): year={2023}, publisher={Springer} }""", # after removing label 1=NOT ENOUGH INFO - descriptive_stats={ - "n_samples": { - "test": 375, - "validation": 305, - }, - "avg_character_length": {"test": 225.62, "validation": 219.32}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/deu/FalseFriendsDeEnPC.py b/mteb/tasks/PairClassification/deu/FalseFriendsDeEnPC.py index 81a7725c26..c1ee188fd2 100644 --- a/mteb/tasks/PairClassification/deu/FalseFriendsDeEnPC.py +++ b/mteb/tasks/PairClassification/deu/FalseFriendsDeEnPC.py @@ -36,10 +36,6 @@ class FalseFriendsDeEnPC(AbsTaskPairClassification): abstract="This paper explores the robustness of multilingual language models against false friends. False friends are words that sound or are written the same in two different languages but have different meaning. Generally, it is argued that multilingual models, such as XLM-RoBERTA, can outperform monolingual models in most tasks on conventional datasets. However, false friends are not considered in these tests. In this paper, experiments with a false friends dataset show that multilingual models are not robust against false friends; they have problems creating monolingual representations and differentiating between meanings of similarly written words in different languages. An attempt of word-based finetuning multilingual models on false friends pairs is promising, however the results do not generally solve the presented problem and still, monolingual models are more robust against false friends." } """, - descriptive_stats={ - "n_samples": {"test": 1524}, - "avg_character_length": {"test": 40.3}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/eng/LegalBenchPC.py b/mteb/tasks/PairClassification/eng/LegalBenchPC.py index 57f5631fa3..534d086264 100644 --- a/mteb/tasks/PairClassification/eng/LegalBenchPC.py +++ b/mteb/tasks/PairClassification/eng/LegalBenchPC.py @@ -117,10 +117,6 @@ class LegalBenchPC(AbsTaskPairClassification): year={2019} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 287.18}, - }, ) def load_data(self, **kwargs: Any) -> None: diff --git a/mteb/tasks/PairClassification/eng/SprintDuplicateQuestionsPC.py b/mteb/tasks/PairClassification/eng/SprintDuplicateQuestionsPC.py index 44aa50e286..4c1ea598e2 100644 --- a/mteb/tasks/PairClassification/eng/SprintDuplicateQuestionsPC.py +++ b/mteb/tasks/PairClassification/eng/SprintDuplicateQuestionsPC.py @@ -17,7 +17,7 @@ class SprintDuplicateQuestionsPC(AbsTaskPairClassification): type="PairClassification", category="s2s", modalities=["text"], - eval_splits=["validation", "test"], + eval_splits=["test"], eval_langs=["eng-Latn"], main_score="max_ap", date=( @@ -30,6 +30,7 @@ class SprintDuplicateQuestionsPC(AbsTaskPairClassification): annotations_creators="derived", dialect=[], sample_creation="found", + prompt="Retrieve duplicate questions from Sprint forum", bibtex_citation="""@inproceedings{shah-etal-2018-adversarial, title = "Adversarial Domain Adaptation for Duplicate Question Detection", author = "Shah, Darsh and @@ -51,10 +52,6 @@ class SprintDuplicateQuestionsPC(AbsTaskPairClassification): pages = "1056--1063", abstract = "We address the problem of detecting duplicate questions in forums, which is an important step towards automating the process of answering new questions. As finding and annotating such potential duplicates manually is very tedious and costly, automatic methods based on machine learning are a viable alternative. However, many forums do not have annotated data, i.e., questions labeled by experts as duplicates, and thus a promising solution is to use domain adaptation from another forum that has such annotations. Here we focus on adversarial domain adaptation, deriving important findings about when it performs well and what properties of the domains are important in this regard. Our experiments with StackExchange data show an average improvement of 5.6{\%} over the best baseline across multiple pairs of domains.", }""", - descriptive_stats={ - "n_samples": {"validation": 101000, "test": 101000}, - "avg_character_length": {"validation": 65.2, "test": 67.9}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/eng/TwitterSemEval2015PC.py b/mteb/tasks/PairClassification/eng/TwitterSemEval2015PC.py index a030142c57..b8bc686d87 100644 --- a/mteb/tasks/PairClassification/eng/TwitterSemEval2015PC.py +++ b/mteb/tasks/PairClassification/eng/TwitterSemEval2015PC.py @@ -45,10 +45,7 @@ class TwitterSemEval2015PC(AbsTaskPairClassification): doi = "10.18653/v1/S15-2001", pages = "1--11", }""", - descriptive_stats={ - "n_samples": {"test": 16777}, - "avg_character_length": {"test": 38.3}, - }, + prompt="Retrieve tweets that are semantically similar to the given tweet", ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/eng/TwitterURLCorpusPC.py b/mteb/tasks/PairClassification/eng/TwitterURLCorpusPC.py index 0c059aaa8d..24839e5938 100644 --- a/mteb/tasks/PairClassification/eng/TwitterURLCorpusPC.py +++ b/mteb/tasks/PairClassification/eng/TwitterURLCorpusPC.py @@ -46,16 +46,7 @@ class TwitterURLCorpusPC(AbsTaskPairClassification): pages = "1224--1234", abstract = "A major challenge in paraphrase research is the lack of parallel corpora. In this paper, we present a new method to collect large-scale sentential paraphrases from Twitter by linking tweets through shared URLs. The main advantage of our method is its simplicity, as it gets rid of the classifier or human in the loop needed to select data before annotation and subsequent application of paraphrase identification algorithms in the previous work. We present the largest human-labeled paraphrase corpus to date of 51,524 sentence pairs and the first cross-domain benchmarking for automatic paraphrase identification. In addition, we show that more than 30,000 new sentential paraphrases can be easily and continuously captured every month at {\textasciitilde}70{\%} precision, and demonstrate their utility for downstream NLP tasks through phrasal paraphrase extraction. We make our code and data freely available.", }""", - descriptive_stats={ - "n_samples": {"test": 51534}, - "test": { - "num_samples": 51534, - "avg_sentence1_len": 79.48919160166103, - "avg_sentence2_len": 88.5540419916948, - "unique_labels": 2, - "labels": {"0": {"count": 38546}, "1": {"count": 12988}}, - }, - }, + prompt="Retrieve tweets that are semantically similar to the given tweet", ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/fas/FarsTail.py b/mteb/tasks/PairClassification/fas/FarsTail.py index aa74bc5d39..552e953f77 100644 --- a/mteb/tasks/PairClassification/fas/FarsTail.py +++ b/mteb/tasks/PairClassification/fas/FarsTail.py @@ -36,10 +36,6 @@ class FarsTail(AbsTaskPairClassification): publisher={Springer}, doi={10.1007/s00500-023-08959-3} }""", # after removing neutral - descriptive_stats={ - "n_samples": {"test": 1029}, - "avg_character_length": {"test": 125.84}, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/PairClassification/hye/ArmenianParaphrasePC.py b/mteb/tasks/PairClassification/hye/ArmenianParaphrasePC.py index d430680e19..6a0d5a86d9 100644 --- a/mteb/tasks/PairClassification/hye/ArmenianParaphrasePC.py +++ b/mteb/tasks/PairClassification/hye/ArmenianParaphrasePC.py @@ -36,10 +36,6 @@ class ArmenianParaphrasePC(AbsTaskPairClassification): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"train": 4023, "test": 1470}, - "avg_character_length": {"train": 243.81, "test": 241.37}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/ind/IndoNLI.py b/mteb/tasks/PairClassification/ind/IndoNLI.py index ed62d5ab23..ac0976e475 100644 --- a/mteb/tasks/PairClassification/ind/IndoNLI.py +++ b/mteb/tasks/PairClassification/ind/IndoNLI.py @@ -39,10 +39,6 @@ class IndoNLI(AbsTaskPairClassification): pages = "10511--10527", }""", # after removing neutral - descriptive_stats={ - "n_samples": {"test_expert": 2040}, - "avg_character_length": {"test_expert": 145.88}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/kor/KlueNLI.py b/mteb/tasks/PairClassification/kor/KlueNLI.py index b21891cb9e..381b5ed113 100644 --- a/mteb/tasks/PairClassification/kor/KlueNLI.py +++ b/mteb/tasks/PairClassification/kor/KlueNLI.py @@ -35,10 +35,6 @@ class KlueNLI(AbsTaskPairClassification): archivePrefix={arXiv}, primaryClass={cs.CL} }""", # 3000 - neutral samples - descriptive_stats={ - "n_samples": {"validation": 2000}, - "avg_character_length": {"validation": 35.01}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/multilingual/IndicXnliPairClassification.py b/mteb/tasks/PairClassification/multilingual/IndicXnliPairClassification.py index f13829dcbf..33fa179737 100644 --- a/mteb/tasks/PairClassification/multilingual/IndicXnliPairClassification.py +++ b/mteb/tasks/PairClassification/multilingual/IndicXnliPairClassification.py @@ -60,10 +60,7 @@ class IndicXnliPairClassification(AbsTaskPairClassification, MultilingualTask): copyright = {Creative Commons Attribution 4.0 International} } """, - descriptive_stats={ - "n_samples": {"test": 5010}, - "avg_character_length": {"test": 77.24}, - }, # average of premise and hypothesis + # average of premise and hypothesis ) def dataset_transform(self) -> None: diff --git a/mteb/tasks/PairClassification/multilingual/OpusparcusPC.py b/mteb/tasks/PairClassification/multilingual/OpusparcusPC.py index fa6ac82c84..56e599364b 100644 --- a/mteb/tasks/PairClassification/multilingual/OpusparcusPC.py +++ b/mteb/tasks/PairClassification/multilingual/OpusparcusPC.py @@ -47,10 +47,6 @@ class OpusparcusPC(AbsTaskPairClassification, MultilingualTask): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"validation": 10168, "test": 10210}, - "avg_character_length": {"validation": 24.4, "test": 23.8}, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py b/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py index f2e5f805f5..8864e8394c 100644 --- a/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py +++ b/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py @@ -45,125 +45,6 @@ class PawsXPairClassification(MultilingualTask, AbsTaskPairClassification): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"validation": 14000, "test": 14000}, - "test": { - "num_samples": 14000, - "avg_sentence1_len": 91.17892857142857, - "avg_sentence2_len": 91.10121428571429, - "unique_labels": 2, - "labels": {"1": {"count": 6285}, "0": {"count": 7715}}, - "hf_subset_descriptive_stats": { - "de": { - "num_samples": 2000, - "avg_sentence1_len": 119.7815, - "avg_sentence2_len": 119.2355, - "unique_labels": 2, - "labels": {"1": {"count": 895}, "0": {"count": 1105}}, - }, - "en": { - "num_samples": 2000, - "avg_sentence1_len": 113.7575, - "avg_sentence2_len": 113.4235, - "unique_labels": 2, - "labels": {"1": {"count": 907}, "0": {"count": 1093}}, - }, - "es": { - "num_samples": 2000, - "avg_sentence1_len": 117.815, - "avg_sentence2_len": 117.798, - "unique_labels": 2, - "labels": {"1": {"count": 907}, "0": {"count": 1093}}, - }, - "fr": { - "num_samples": 2000, - "avg_sentence1_len": 120.028, - "avg_sentence2_len": 119.9885, - "unique_labels": 2, - "labels": {"1": {"count": 903}, "0": {"count": 1097}}, - }, - "ja": { - "num_samples": 2000, - "avg_sentence1_len": 58.678, - "avg_sentence2_len": 58.875, - "unique_labels": 2, - "labels": {"1": {"count": 883}, "0": {"count": 1117}}, - }, - "ko": { - "num_samples": 2000, - "avg_sentence1_len": 64.9605, - "avg_sentence2_len": 65.114, - "unique_labels": 2, - "labels": {"1": {"count": 896}, "0": {"count": 1104}}, - }, - "zh": { - "num_samples": 2000, - "avg_sentence1_len": 43.232, - "avg_sentence2_len": 43.274, - "unique_labels": 2, - "labels": {"1": {"count": 894}, "0": {"count": 1106}}, - }, - }, - }, - "validation": { - "num_samples": 14000, - "avg_sentence1_len": 90.12585714285714, - "avg_sentence2_len": 90.2045, - "unique_labels": 2, - "labels": {"1": {"count": 5948}, "0": {"count": 8052}}, - "hf_subset_descriptive_stats": { - "de": { - "num_samples": 2000, - "avg_sentence1_len": 116.82, - "avg_sentence2_len": 117.0015, - "unique_labels": 2, - "labels": {"1": {"count": 831}, "0": {"count": 1169}}, - }, - "en": { - "num_samples": 2000, - "avg_sentence1_len": 113.1075, - "avg_sentence2_len": 112.858, - "unique_labels": 2, - "labels": {"1": {"count": 863}, "0": {"count": 1137}}, - }, - "es": { - "num_samples": 2000, - "avg_sentence1_len": 116.3285, - "avg_sentence2_len": 116.7275, - "unique_labels": 2, - "labels": {"1": {"count": 847}, "0": {"count": 1153}}, - }, - "fr": { - "num_samples": 2000, - "avg_sentence1_len": 119.5045, - "avg_sentence2_len": 119.7505, - "unique_labels": 2, - "labels": {"1": {"count": 860}, "0": {"count": 1140}}, - }, - "ja": { - "num_samples": 2000, - "avg_sentence1_len": 57.5105, - "avg_sentence2_len": 57.317, - "unique_labels": 2, - "labels": {"1": {"count": 854}, "0": {"count": 1146}}, - }, - "ko": { - "num_samples": 2000, - "avg_sentence1_len": 65.162, - "avg_sentence2_len": 65.5155, - "unique_labels": 2, - "labels": {"1": {"count": 840}, "0": {"count": 1160}}, - }, - "zh": { - "num_samples": 2000, - "avg_sentence1_len": 42.448, - "avg_sentence2_len": 42.2615, - "unique_labels": 2, - "labels": {"1": {"count": 853}, "0": {"count": 1147}}, - }, - }, - }, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/multilingual/RTE3.py b/mteb/tasks/PairClassification/multilingual/RTE3.py index a79dc03910..9a03fedb4f 100644 --- a/mteb/tasks/PairClassification/multilingual/RTE3.py +++ b/mteb/tasks/PairClassification/multilingual/RTE3.py @@ -52,10 +52,6 @@ class RTE3(MultilingualTask, AbsTaskPairClassification): } """, # sum of 4 languages after neutral filtering - descriptive_stats={ - "n_samples": {"test": 1923}, - "avg_character_length": {"test": 124.79}, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/PairClassification/multilingual/XNLI.py b/mteb/tasks/PairClassification/multilingual/XNLI.py index 826fabda67..8f3f795bad 100644 --- a/mteb/tasks/PairClassification/multilingual/XNLI.py +++ b/mteb/tasks/PairClassification/multilingual/XNLI.py @@ -60,223 +60,6 @@ class XNLI(MultilingualTask, AbsTaskPairClassification): location = {Brussels, Belgium}, } """, - descriptive_stats={ - "n_samples": {"validation": 2163, "test": 2460}, - "test": { - "num_samples": 19110, - "avg_sentence1_len": 103.23793825222397, - "avg_sentence2_len": 48.88895866038723, - "unique_labels": 2, - "labels": {"0": {"count": 9562}, "1": {"count": 9548}}, - "hf_subset_descriptive_stats": { - "ar": { - "num_samples": 1365, - "avg_sentence1_len": 89.57362637362637, - "avg_sentence2_len": 41.99487179487179, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "bg": { - "num_samples": 1365, - "avg_sentence1_len": 110.01611721611722, - "avg_sentence2_len": 51.62930402930403, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "de": { - "num_samples": 1365, - "avg_sentence1_len": 119.92600732600732, - "avg_sentence2_len": 56.794871794871796, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "el": { - "num_samples": 1365, - "avg_sentence1_len": 119.05421245421246, - "avg_sentence2_len": 56.93260073260073, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "en": { - "num_samples": 1365, - "avg_sentence1_len": 105.67032967032966, - "avg_sentence2_len": 49.8043956043956, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "es": { - "num_samples": 1365, - "avg_sentence1_len": 115.43296703296703, - "avg_sentence2_len": 54.68205128205128, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "fr": { - "num_samples": 1365, - "avg_sentence1_len": 121.0967032967033, - "avg_sentence2_len": 58.58021978021978, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "hi": { - "num_samples": 1365, - "avg_sentence1_len": 104.63443223443224, - "avg_sentence2_len": 50.17289377289377, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "ru": { - "num_samples": 1365, - "avg_sentence1_len": 110.76923076923077, - "avg_sentence2_len": 52.452014652014654, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "sw": { - "num_samples": 1365, - "avg_sentence1_len": 104.43956043956044, - "avg_sentence2_len": 49.48205128205128, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "th": { - "num_samples": 1365, - "avg_sentence1_len": 96.6923076923077, - "avg_sentence2_len": 44.544322344322346, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "tr": { - "num_samples": 1365, - "avg_sentence1_len": 103.67765567765568, - "avg_sentence2_len": 49.18534798534799, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "vi": { - "num_samples": 1365, - "avg_sentence1_len": 111.31208791208792, - "avg_sentence2_len": 52.46007326007326, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "zh": { - "num_samples": 1365, - "avg_sentence1_len": 33.03589743589744, - "avg_sentence2_len": 15.73040293040293, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - }, - }, - "validation": { - "num_samples": 19110, - "avg_sentence1_len": 103.20790162218734, - "avg_sentence2_len": 49.01909994767138, - "unique_labels": 2, - "labels": {"0": {"count": 9562}, "1": {"count": 9548}}, - "hf_subset_descriptive_stats": { - "ar": { - "num_samples": 1365, - "avg_sentence1_len": 88.31868131868131, - "avg_sentence2_len": 41.61172161172161, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "bg": { - "num_samples": 1365, - "avg_sentence1_len": 109.196336996337, - "avg_sentence2_len": 51.967032967032964, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "de": { - "num_samples": 1365, - "avg_sentence1_len": 119.81172161172161, - "avg_sentence2_len": 57.36923076923077, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "el": { - "num_samples": 1365, - "avg_sentence1_len": 119.87545787545787, - "avg_sentence2_len": 56.88278388278388, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "en": { - "num_samples": 1365, - "avg_sentence1_len": 105.71648351648352, - "avg_sentence2_len": 49.87619047619047, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "es": { - "num_samples": 1365, - "avg_sentence1_len": 115.17289377289377, - "avg_sentence2_len": 55.120879120879124, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "fr": { - "num_samples": 1365, - "avg_sentence1_len": 121.75897435897436, - "avg_sentence2_len": 59.08864468864469, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "hi": { - "num_samples": 1365, - "avg_sentence1_len": 105.06446886446886, - "avg_sentence2_len": 50.44395604395604, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "ru": { - "num_samples": 1365, - "avg_sentence1_len": 109.74725274725274, - "avg_sentence2_len": 52.26886446886447, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "sw": { - "num_samples": 1365, - "avg_sentence1_len": 104.32234432234432, - "avg_sentence2_len": 49.87692307692308, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "th": { - "num_samples": 1365, - "avg_sentence1_len": 97.28498168498169, - "avg_sentence2_len": 43.843223443223444, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "tr": { - "num_samples": 1365, - "avg_sentence1_len": 102.96630036630036, - "avg_sentence2_len": 49.63809523809524, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "vi": { - "num_samples": 1365, - "avg_sentence1_len": 112.26373626373626, - "avg_sentence2_len": 52.432967032967035, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - "zh": { - "num_samples": 1365, - "avg_sentence1_len": 33.41098901098901, - "avg_sentence2_len": 15.846886446886447, - "unique_labels": 2, - "labels": {"0": {"count": 683}, "1": {"count": 682}}, - }, - }, - }, - }, ) def dataset_transform(self): @@ -357,10 +140,7 @@ class XNLIV2(MultilingualTask, AbsTaskPairClassification): organization={IEEE} } """, - descriptive_stats={ - "n_samples": {"test": 5010}, - "avg_character_length": {"test": 80.06}, - }, # average of premise and hypothesis + # average of premise and hypothesis ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/multilingual/XStance.py b/mteb/tasks/PairClassification/multilingual/XStance.py index e6b60861da..03d4f066e7 100644 --- a/mteb/tasks/PairClassification/multilingual/XStance.py +++ b/mteb/tasks/PairClassification/multilingual/XStance.py @@ -46,10 +46,7 @@ class XStance(MultilingualTask, AbsTaskPairClassification): url = "http://ceur-ws.org/Vol-2624/paper9.pdf" } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 152.41}, - }, # length of`sent1` + `sent2` + # length of`sent1` + `sent2` ) def load_data(self, **kwargs): @@ -61,7 +58,7 @@ def load_data(self, **kwargs): self.dataset = {} path = self.metadata_dict["dataset"]["path"] revision = self.metadata_dict["dataset"]["revision"] - raw_dataset = load_dataset(path, revision=revision) + raw_dataset = load_dataset(path, revision=revision, trust_remote_code=True) def convert_example(example): return { @@ -103,8 +100,8 @@ def dataset_transform(self): for split in self.metadata.eval_splits: _dataset[lang][split] = [ { - "sent1": self.dataset[lang][split]["sent1"], - "sent2": self.dataset[lang][split]["sent2"], + "sentence1": self.dataset[lang][split]["sentence1"], + "sentence2": self.dataset[lang][split]["sentence2"], "labels": self.dataset[lang][split]["labels"], } ] diff --git a/mteb/tasks/PairClassification/pol/PolishPC.py b/mteb/tasks/PairClassification/pol/PolishPC.py index 2166cebf1c..51a041b704 100644 --- a/mteb/tasks/PairClassification/pol/PolishPC.py +++ b/mteb/tasks/PairClassification/pol/PolishPC.py @@ -57,7 +57,6 @@ class SickePLPC(AbsTaskPairClassification): language = "English", ISBN = "979-10-95546-34-4", }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) def dataset_transform(self): @@ -104,7 +103,6 @@ class PpcPC(AbsTaskPairClassification): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) def dataset_transform(self): @@ -150,7 +148,6 @@ class CdscePC(AbsTaskPairClassification): pages = "784--792", abstract = "The paper presents a procedure of building an evaluation dataset. for the validation of compositional distributional semantics models estimated for languages other than English. The procedure generally builds on steps designed to assemble the SICK corpus, which contains pairs of English sentences annotated for semantic relatedness and entailment, because we aim at building a comparable dataset. However, the implementation of particular building steps significantly differs from the original SICK design assumptions, which is caused by both lack of necessary extraneous resources for an investigated language and the need for language-specific transformation rules. The designed procedure is verified on Polish, a fusional language with a relatively free word order, and contributes to building a Polish evaluation dataset. The resource consists of 10K sentence pairs which are human-annotated for semantic relatedness and entailment. The dataset may be used for the evaluation of compositional distributional semantics models of Polish.", }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) def dataset_transform(self): @@ -202,7 +199,6 @@ class PscPC(AbsTaskPairClassification): pages = "3712--3715", abstract = "This article presents the Polish Summaries Corpus, a new resource created to support the development and evaluation of the tools for automated single-document summarization of Polish. The Corpus contains a large number of manual summaries of news articles, with many independently created summaries for a single text. Such approach is supposed to overcome the annotator bias, which is often described as a problem during the evaluation of the summarization algorithms against a single gold standard. There are several summarizers developed specifically for Polish language, but their in-depth evaluation and comparison was impossible without a large, manually created corpus. We present in detail the process of text selection, annotation process and the contents of the corpus, which includes both abstract free-word summaries, as well as extraction-based summaries created by selecting text spans from the original document. Finally, we describe how that resource could be used not only for the evaluation of the existing summarization tools, but also for studies on the human summarization process in Polish language.", }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/por/Assin2RTE.py b/mteb/tasks/PairClassification/por/Assin2RTE.py index 41f9aa43e8..aa0046cb6e 100644 --- a/mteb/tasks/PairClassification/por/Assin2RTE.py +++ b/mteb/tasks/PairClassification/por/Assin2RTE.py @@ -34,10 +34,6 @@ class Assin2RTE(AbsTaskPairClassification): year={2020}, organization={Springer} }""", - descriptive_stats={ - "n_samples": {"test": 2448}, - "avg_character_length": {"test": 53.55}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/por/SickBrPC.py b/mteb/tasks/PairClassification/por/SickBrPC.py index f7bf92e69b..e8320a4775 100644 --- a/mteb/tasks/PairClassification/por/SickBrPC.py +++ b/mteb/tasks/PairClassification/por/SickBrPC.py @@ -3,8 +3,6 @@ from mteb.abstasks.AbsTaskPairClassification import AbsTaskPairClassification from mteb.abstasks.TaskMetadata import TaskMetadata -N_SAMPLES = 1000 - class SickBrPC(AbsTaskPairClassification): metadata = TaskMetadata( @@ -49,10 +47,6 @@ class SickBrPC(AbsTaskPairClassification): isbn="978-3-319-99722-3" } """, - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "avg_character_length": {"test": 54.89}, - }, ) def dataset_transform(self): @@ -67,7 +61,6 @@ def dataset_transform(self): seed=self.seed, splits=self.metadata.eval_splits, label="entailment_label", - n_samples=N_SAMPLES, ) for split in self.metadata.eval_splits: diff --git a/mteb/tasks/PairClassification/rus/TERRa.py b/mteb/tasks/PairClassification/rus/TERRa.py index fdb711c8d6..ee904f9eb3 100644 --- a/mteb/tasks/PairClassification/rus/TERRa.py +++ b/mteb/tasks/PairClassification/rus/TERRa.py @@ -43,10 +43,7 @@ class TERRa(AbsTaskPairClassification): journal={arXiv preprint arXiv:2010.15925}, year={2020} }""", - descriptive_stats={ - "n_samples": {"dev": 307}, - "avg_character_length": {"dev": 138.2}, - }, + prompt="Given a premise, retrieve a hypothesis that is entailed by the premise", ) def dataset_transform(self): diff --git a/mteb/tasks/PairClassification/zho/CMTEBPairClassification.py b/mteb/tasks/PairClassification/zho/CMTEBPairClassification.py index 51d9e22b07..d6bfbdef95 100644 --- a/mteb/tasks/PairClassification/zho/CMTEBPairClassification.py +++ b/mteb/tasks/PairClassification/zho/CMTEBPairClassification.py @@ -34,7 +34,7 @@ class Ocnli(AbsTaskPairClassification): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, + prompt="Retrieve semantically similar text.", ) def dataset_transform(self): @@ -107,7 +107,7 @@ class Cmnli(AbsTaskPairClassification): doi = "10.18653/v1/2020.coling-main.419", pages = "4762--4772", }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, + prompt="Retrieve semantically similar text.", ) def dataset_transform(self): diff --git a/mteb/tasks/Reranking/__init__.py b/mteb/tasks/Reranking/__init__.py index a4b302a17f..2c3a27919a 100644 --- a/mteb/tasks/Reranking/__init__.py +++ b/mteb/tasks/Reranking/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +from .ara.NamaaMrTydiReranking import * from .eng.AskUbuntuDupQuestions import * from .eng.MindSmallReranking import * from .eng.SciDocsReranking import * diff --git a/mteb/tasks/Reranking/ara/NamaaMrTydiReranking.py b/mteb/tasks/Reranking/ara/NamaaMrTydiReranking.py new file mode 100644 index 0000000000..4a9d755747 --- /dev/null +++ b/mteb/tasks/Reranking/ara/NamaaMrTydiReranking.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from ....abstasks.AbsTaskReranking import AbsTaskReranking + + +class NamaaMrTydiReranking(AbsTaskReranking): + metadata = TaskMetadata( + name="NamaaMrTydiReranking", + description="Mr. TyDi is a multi-lingual benchmark dataset built on TyDi, covering eleven typologically diverse languages. It is designed for monolingual retrieval, specifically to evaluate ranking with learned dense representations. This dataset adapts the arabic test split for Reranking evaluation purposes by the addition of multiple (Hard) Negatives to each query and positive", + reference="https://huggingface.co/NAMAA-Space", + dataset={ + "path": "NAMAA-Space/mteb-eval-mrtydi", + "revision": "502637220a7ad0ecc5c39ff5518d7508d2624af8", + }, + type="Reranking", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["ara-Arab"], + main_score="map", + date=("2023-11-01", "2024-05-15"), + domains=["Encyclopaedic", "Written"], + task_subtypes=[], + license="cc-by-sa-3.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{muennighoff2022mteb, + doi = {10.48550/ARXIV.2210.07316}, + url = {https://arxiv.org/abs/2210.07316}, + author = {Muennighoff, Niklas and Tazi, Nouamane and Magne, Lo{\"\i}c and Reimers, Nils}, + title = {MTEB: Massive Text Embedding Benchmark}, + publisher = {arXiv}, + journal={arXiv preprint arXiv:2210.07316}, + year = {2022} +}""", + ) diff --git a/mteb/tasks/Reranking/ara/__init__.py b/mteb/tasks/Reranking/ara/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Reranking/eng/AskUbuntuDupQuestions.py b/mteb/tasks/Reranking/eng/AskUbuntuDupQuestions.py index 07d6118111..90fe689cdd 100644 --- a/mteb/tasks/Reranking/eng/AskUbuntuDupQuestions.py +++ b/mteb/tasks/Reranking/eng/AskUbuntuDupQuestions.py @@ -27,6 +27,7 @@ class AskUbuntuDupQuestions(AbsTaskReranking): annotations_creators=None, dialect=None, sample_creation=None, + prompt="Retrieve duplicate questions from AskUbuntu forum", bibtex_citation="""@article{wang-2021-TSDAE, title = "TSDAE: Using Transformer-based Sequential Denoising Auto-Encoderfor Unsupervised Sentence Embedding Learning", author = "Wang, Kexin and Reimers, Nils and Gurevych, Iryna", @@ -35,15 +36,4 @@ class AskUbuntuDupQuestions(AbsTaskReranking): year = "2021", url = "https://arxiv.org/abs/2104.06979", }""", - descriptive_stats={ - "n_samples": {"test": 2255}, - "test": { - "num_samples": 375, - "num_positive": 375, - "num_negative": 375, - "avg_query_len": 50.205333333333336, - "avg_positive_len": 6.013333333333334, - "avg_negative_len": 13.986666666666666, - }, - }, ) diff --git a/mteb/tasks/Reranking/eng/MindSmallReranking.py b/mteb/tasks/Reranking/eng/MindSmallReranking.py index a5ef13a603..bdc01edbcb 100644 --- a/mteb/tasks/Reranking/eng/MindSmallReranking.py +++ b/mteb/tasks/Reranking/eng/MindSmallReranking.py @@ -27,6 +27,7 @@ class MindSmallReranking(AbsTaskReranking): annotations_creators="expert-annotated", dialect=[], sample_creation="found", + prompt="Retrieve relevant news articles based on user browsing history", bibtex_citation="""@inproceedings{wu-etal-2020-mind, title = "{MIND}: A Large-scale Dataset for News Recommendation", author = "Wu, Fangzhao and Qiao, Ying and Chen, Jiun-Hung and Wu, Chuhan and Qi, Tao and Lian, Jianxun and Liu, Danyang and Xie, Xing and Gao, Jianfeng and Wu, Winnie and Zhou, Ming", @@ -46,8 +47,4 @@ class MindSmallReranking(AbsTaskReranking): Many natural language processing techniques such as effective text representation methods and pre-trained language models can effectively improve the performance of news recommendation. The MIND dataset will be available at https://msnews.github.io}.", }""", - descriptive_stats={ - "n_samples": {"test": 107968}, - "avg_character_length": {"test": 70.9}, - }, ) diff --git a/mteb/tasks/Reranking/eng/SciDocsReranking.py b/mteb/tasks/Reranking/eng/SciDocsReranking.py index f85ffaed8d..183566cfe6 100644 --- a/mteb/tasks/Reranking/eng/SciDocsReranking.py +++ b/mteb/tasks/Reranking/eng/SciDocsReranking.py @@ -27,6 +27,7 @@ class SciDocsReranking(AbsTaskReranking): annotations_creators=None, dialect=None, sample_creation="found", + prompt="Given a title of a scientific paper, retrieve the titles of other relevant papers", bibtex_citation=""" @inproceedings{cohan-etal-2020-specter, title = "{SPECTER}: Document-level Representation Learning using Citation-informed Transformers", @@ -50,8 +51,4 @@ class SciDocsReranking(AbsTaskReranking): abstract = "Representation learning is a critical ingredient for natural language processing systems. Recent Transformer language models like BERT learn powerful textual representations, but these models are targeted towards token- and sentence-level training objectives and do not leverage information on inter-document relatedness, which limits their document-level representation power. For applications on scientific documents, such as classification and recommendation, accurate embeddings of documents are a necessity. We propose SPECTER, a new method to generate document-level embedding of scientific papers based on pretraining a Transformer language model on a powerful signal of document-level relatedness: the citation graph. Unlike existing pretrained language models, Specter can be easily applied to downstream applications without task-specific fine-tuning. Additionally, to encourage further research on document-level models, we introduce SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation prediction, to document classification and recommendation. We show that Specter outperforms a variety of competitive baselines on the benchmark.", } """, - descriptive_stats={ - "n_samples": {"test": 19599}, - "avg_character_length": {"test": 69.0}, - }, ) diff --git a/mteb/tasks/Reranking/eng/StackOverflowDupQuestions.py b/mteb/tasks/Reranking/eng/StackOverflowDupQuestions.py index f18f24e047..9e47461620 100644 --- a/mteb/tasks/Reranking/eng/StackOverflowDupQuestions.py +++ b/mteb/tasks/Reranking/eng/StackOverflowDupQuestions.py @@ -27,6 +27,7 @@ class StackOverflowDupQuestions(AbsTaskReranking): annotations_creators=None, dialect=None, sample_creation=None, + prompt="Retrieve duplicate questions from StackOverflow forum", bibtex_citation="""@article{Liu2018LinkSOAD, title={LinkSO: a dataset for learning to retrieve similar question answer pairs on software development forums}, author={Xueqing Liu and Chi Wang and Yue Leng and ChengXiang Zhai}, @@ -34,8 +35,4 @@ class StackOverflowDupQuestions(AbsTaskReranking): year={2018}, url={https://api.semanticscholar.org/CorpusID:53111679} }""", - descriptive_stats={ - "n_samples": {"test": 3467}, - "avg_character_length": {"test": 49.8}, - }, ) diff --git a/mteb/tasks/Reranking/eng/WebLINXCandidatesReranking.py b/mteb/tasks/Reranking/eng/WebLINXCandidatesReranking.py index 4790a9460f..981dfa4eef 100644 --- a/mteb/tasks/Reranking/eng/WebLINXCandidatesReranking.py +++ b/mteb/tasks/Reranking/eng/WebLINXCandidatesReranking.py @@ -47,24 +47,6 @@ class WebLINXCandidatesReranking(AbsTaskReranking): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": { - "validation": 1301, - "test_iid": 1438, - "test_cat": 3560, - "test_web": 3144, - "test_vis": 5298, - "test_geo": 4916, - }, - "avg_character_length": { - "validation": 1647.52, - "test_iid": 1722.63, - "test_cat": 2149.66, - "test_web": 1831.46, - "test_vis": 1737.26, - "test_geo": 1742.66, - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Reranking/fra/AlloprofReranking.py b/mteb/tasks/Reranking/fra/AlloprofReranking.py index 3e5509c936..20d24f03ec 100644 --- a/mteb/tasks/Reranking/fra/AlloprofReranking.py +++ b/mteb/tasks/Reranking/fra/AlloprofReranking.py @@ -39,10 +39,6 @@ class AlloprofReranking(AbsTaskReranking): year = {2023}, copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International} }""", - descriptive_stats={ - "n_samples": {"test": 2316, "train": 9264}, - "avg_character_length": None, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Reranking/fra/SyntecReranking.py b/mteb/tasks/Reranking/fra/SyntecReranking.py index 3b12625e69..3f9188bd33 100644 --- a/mteb/tasks/Reranking/fra/SyntecReranking.py +++ b/mteb/tasks/Reranking/fra/SyntecReranking.py @@ -37,7 +37,6 @@ class SyntecReranking(AbsTaskReranking): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Reranking/jpn/MMarcoReranking.py b/mteb/tasks/Reranking/jpn/MMarcoReranking.py index bcfa5bba05..dd37f16af7 100644 --- a/mteb/tasks/Reranking/jpn/MMarcoReranking.py +++ b/mteb/tasks/Reranking/jpn/MMarcoReranking.py @@ -26,16 +26,13 @@ class VoyageMMarcoReranking(AbsTaskReranking): annotations_creators="derived", dialect=["jpn-Jpan"], sample_creation="found", + prompt="Given a Japanese search query, retrieve web passages that answer the question", bibtex_citation="""@misc{clavié2023jacolbert, title={JaColBERT and Hard Negatives, Towards Better Japanese-First Embeddings for Retrieval: Early Technical Report}, author={Benjamin Clavié}, year={2023}, eprint={2312.16144}, archivePrefix={arXiv},}""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": {"test": 162}, - }, ) def dataset_transform(self): diff --git a/mteb/tasks/Reranking/multilingual/ESCIReranking.py b/mteb/tasks/Reranking/multilingual/ESCIReranking.py index c3597c2fdf..03c6608f27 100644 --- a/mteb/tasks/Reranking/multilingual/ESCIReranking.py +++ b/mteb/tasks/Reranking/multilingual/ESCIReranking.py @@ -47,40 +47,4 @@ class ESCIReranking(MultilingualTask, AbsTaskReranking): dialect=[], sample_creation="created", bibtex_citation=_CITATION, - descriptive_stats={ - "test": { - "num_samples": 29285, - "num_positive": 29285, - "num_negative": 29285, - "avg_query_len": 19.691890046098685, - "avg_positive_len": 9.268089465596722, - "avg_negative_len": 1.5105002561038074, - "hf_subset_descriptive_stats": { - "us": { - "num_samples": 21296, - "num_positive": 21296, - "num_negative": 21296, - "avg_query_len": 21.440833959429, - "avg_positive_len": 8.892515026296017, - "avg_negative_len": 1.1956705484598047, - }, - "es": { - "num_samples": 3703, - "num_positive": 3703, - "num_negative": 3703, - "avg_query_len": 20.681609505806104, - "avg_positive_len": 10.561706724277613, - "avg_negative_len": 2.749932487172563, - }, - "jp": { - "num_samples": 4286, - "num_positive": 4286, - "num_negative": 4286, - "avg_query_len": 10.146756882874476, - "avg_positive_len": 10.016565562295847, - "avg_negative_len": 2.003966402239851, - }, - }, - } - }, ) diff --git a/mteb/tasks/Reranking/multilingual/MIRACLReranking.py b/mteb/tasks/Reranking/multilingual/MIRACLReranking.py index 4d226842bf..4d90ce641d 100644 --- a/mteb/tasks/Reranking/multilingual/MIRACLReranking.py +++ b/mteb/tasks/Reranking/multilingual/MIRACLReranking.py @@ -74,9 +74,8 @@ class MIRACLReranking(MultilingualTask, AbsTaskReranking): dialect=[], sample_creation="created", bibtex_citation=_CITATION, - descriptive_stats={ - "n_samples": {"dev": 44608}, - "avg_character_length": {"dev": 506.30}, + prompt={ + "query": "Given a question, retrieve Wikipedia passages that answer the question" }, ) diff --git a/mteb/tasks/Reranking/multilingual/WikipediaRerankingMultilingual.py b/mteb/tasks/Reranking/multilingual/WikipediaRerankingMultilingual.py index 4d718f6f20..3bfbd04f13 100644 --- a/mteb/tasks/Reranking/multilingual/WikipediaRerankingMultilingual.py +++ b/mteb/tasks/Reranking/multilingual/WikipediaRerankingMultilingual.py @@ -52,162 +52,4 @@ class WikipediaRerankingMultilingual(MultilingualTask, AbsTaskReranking): title = "Wikimedia Downloads", url = "https://dumps.wikimedia.org" }""", - descriptive_stats={ - "n_samples": { - "en": 1500, - "de": 1500, - "it": 1500, - "pt": 1500, - "nl": 1500, - "cs": 1500, - "ro": 1500, - "bg": 1500, - "sr": 1500, - "fi": 1500, - "da": 1500, - "fa": 1500, - "hi": 1500, - "bn": 1500, - "no": 1500, - "sv": 1500, - }, - "test": { - "num_samples": 24000, - "num_positive": 24000, - "num_negative": 24000, - "avg_query_len": 59.091208333333334, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - "hf_subset_descriptive_stats": { - "bg": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 60.82666666666667, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "bn": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 47.266666666666666, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "cs": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 56.272, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "da": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 56.75066666666667, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "de": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 70.004, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "en": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 68.372, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "fa": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 48.66733333333333, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "fi": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 55.343333333333334, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "hi": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 50.77733333333333, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "it": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 70.05466666666666, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "nl": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 65.34466666666667, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "pt": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 65.11933333333333, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "ro": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 61.973333333333336, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "sr": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 55.669333333333334, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "no": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 55.288, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - "sv": { - "num_samples": 1500, - "num_positive": 1500, - "num_negative": 1500, - "avg_query_len": 57.73, - "avg_positive_len": 1.0, - "avg_negative_len": 8.0, - }, - }, - }, - }, ) diff --git a/mteb/tasks/Reranking/rus/RuBQReranking.py b/mteb/tasks/Reranking/rus/RuBQReranking.py index 5303f413eb..fb79a17588 100644 --- a/mteb/tasks/Reranking/rus/RuBQReranking.py +++ b/mteb/tasks/Reranking/rus/RuBQReranking.py @@ -34,8 +34,7 @@ class RuBQReranking(AbsTaskReranking): year={2021}, pages={532--547} }""", - descriptive_stats={ - "n_samples": {"test": 1551}, - "avg_character_length": {"test": 499.9}, + prompt={ + "query": "Given a question, retrieve Wikipedia passages that answer the question.", }, ) diff --git a/mteb/tasks/Reranking/zho/CMTEBReranking.py b/mteb/tasks/Reranking/zho/CMTEBReranking.py index 7aa26c4ce0..7a33f7ae0a 100644 --- a/mteb/tasks/Reranking/zho/CMTEBReranking.py +++ b/mteb/tasks/Reranking/zho/CMTEBReranking.py @@ -27,6 +27,7 @@ class T2Reranking(AbsTaskReranking): annotations_creators=None, dialect=None, sample_creation=None, + prompt="Given a Chinese search query, retrieve web passages that answer the question", bibtex_citation="""@misc{xie2023t2ranking, title={T2Ranking: A large-scale Chinese Benchmark for Passage Ranking}, author={Xiaohui Xie and Qian Dong and Bingning Wang and Feiyang Lv and Ting Yao and Weinan Gan and Zhijing Wu and Xiangsheng Li and Haitao Li and Yiqun Liu and Jin Ma}, @@ -35,7 +36,6 @@ class T2Reranking(AbsTaskReranking): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @@ -62,6 +62,7 @@ class MMarcoReranking(AbsTaskReranking): annotations_creators=None, dialect=None, sample_creation=None, + prompt="Given a Chinese search query, retrieve web passages that answer the question", bibtex_citation="""@misc{bonifacio2021mmarco, title={mMARCO: A Multilingual Version of MS MARCO Passage Ranking Dataset}, author={Luiz Henrique Bonifacio and Vitor Jeronymo and Hugo Queiroz Abonizio and Israel Campiotti and Marzieh Fadaee and and Roberto Lotufo and Rodrigo Nogueira}, @@ -70,7 +71,6 @@ class MMarcoReranking(AbsTaskReranking): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @@ -78,6 +78,7 @@ class CMedQAv1(AbsTaskReranking): metadata = TaskMetadata( name="CMedQAv1-reranking", description="Chinese community medical question answering", + prompt="Given a Chinese community medical question, retrieve replies that best answer the question", reference="https://github.com/zhangsheng93/cMedQA", dataset={ "path": "C-MTEB/CMedQAv1-reranking", @@ -106,10 +107,6 @@ class CMedQAv1(AbsTaskReranking): year={2017}, publisher={Multidisciplinary Digital Publishing Institute} }""", - descriptive_stats={ - "n_samples": {"test": 2000}, - "avg_character_length": {"test": 165}, - }, ) @@ -117,6 +114,7 @@ class CMedQAv2(AbsTaskReranking): metadata = TaskMetadata( name="CMedQAv2-reranking", description="Chinese community medical question answering", + prompt="Given a Chinese community medical question, retrieve replies that best answer the question", reference="https://github.com/zhangsheng93/cMedQA2", dataset={ "path": "C-MTEB/CMedQAv2-reranking", @@ -130,7 +128,7 @@ class CMedQAv2(AbsTaskReranking): main_score="map", date=None, form=None, - domains=None, + domains=["Medical", "Written"], task_subtypes=None, license=None, annotations_creators=None, @@ -148,5 +146,4 @@ class CMedQAv2(AbsTaskReranking): doi={10.1109/ACCESS.2018.2883637}, ISSN={2169-3536}, month={},}""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) diff --git a/mteb/tasks/Retrieval/__init__.py b/mteb/tasks/Retrieval/__init__.py index c29a4383c7..d83df7ec5e 100644 --- a/mteb/tasks/Retrieval/__init__.py +++ b/mteb/tasks/Retrieval/__init__.py @@ -64,6 +64,19 @@ from .eng.MLQuestions import * from .eng.MSMARCORetrieval import * from .eng.MSMARCOv2Retrieval import * +from .eng.NanoArguAnaRetrieval import * +from .eng.NanoClimateFeverRetrieval import * +from .eng.NanoDBPediaRetrieval import * +from .eng.NanoFEVERRetrieval import * +from .eng.NanoFiQA2018Retrieval import * +from .eng.NanoHotpotQARetrieval import * +from .eng.NanoMSMARCORetrieval import * +from .eng.NanoNFCorpusRetrieval import * +from .eng.NanoNQRetrieval import * +from .eng.NanoQuoraRetrieval import * +from .eng.NanoSCIDOCSRetrieval import * +from .eng.NanoSciFactRetrieval import * +from .eng.NanoTouche2020Retrieval import * from .eng.NarrativeQARetrieval import * from .eng.NFCorpusRetrieval import * from .eng.NQRetrieval import * @@ -100,10 +113,12 @@ from .jpn.NLPJournalTitleAbsRetrieval import * from .jpn.NLPJournalTitleIntroRetrieval import * from .kat.GeorgianFAQRetrieval import * +from .kor.AutoRAGRetrieval import * from .kor.KoStrategyQA import * from .multilingual.BelebeleRetrieval import * from .multilingual.CrossLingualSemanticDiscriminationWMT19 import * from .multilingual.CrossLingualSemanticDiscriminationWMT21 import * +from .multilingual.CUREv1Retrieval import * from .multilingual.IndicQARetrieval import * from .multilingual.MintakaRetrieval import * from .multilingual.MIRACLRetrieval import * diff --git a/mteb/tasks/Retrieval/ara/SadeemQuestionRetrieval.py b/mteb/tasks/Retrieval/ara/SadeemQuestionRetrieval.py index d6efb88828..2009a91c79 100644 --- a/mteb/tasks/Retrieval/ara/SadeemQuestionRetrieval.py +++ b/mteb/tasks/Retrieval/ara/SadeemQuestionRetrieval.py @@ -38,10 +38,6 @@ class SadeemQuestionRetrieval(AbsTaskRetrieval): author = "abubakr.soliman@sadeem.app" } """, - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 22979}, - "avg_character_length": {_EVAL_SPLIT: 500.0}, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/code/AppsRetrieval.py b/mteb/tasks/Retrieval/code/AppsRetrieval.py index 03b6df10af..e207f8e340 100644 --- a/mteb/tasks/Retrieval/code/AppsRetrieval.py +++ b/mteb/tasks/Retrieval/code/AppsRetrieval.py @@ -34,16 +34,4 @@ class AppsRetrieval(AbsTaskRetrieval): journal={NeurIPS}, year={2021} }""", - descriptive_stats={ - "n_samples": { - _EVAL_SPLIT: 1000, - }, - "test": { - "average_document_length": 575.0086708499715, - "average_query_length": 1669.8284196547145, - "num_documents": 8765, - "num_queries": 3765, - "average_relevant_docs_per_query": 1.0, - }, - }, ) diff --git a/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py b/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py index 170e6c8348..29858026a6 100644 --- a/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py +++ b/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py @@ -97,57 +97,6 @@ class COIRCodeSearchNetRetrieval(MultilingualTask, AbsTaskRetrieval): dialect=[], sample_creation="found", bibtex_citation="@article{husain2019codesearchnet, title={{CodeSearchNet} challenge: Evaluating the state of semantic code search}, author={Husain, Hamel and Wu, Ho-Hsiang and Gazit, Tiferet and Allamanis, Miltiadis and Brockschmidt, Marc}, journal={arXiv preprint arXiv:1909.09436}, year={2019} }", - descriptive_stats={ - "n_samples": { - _EVAL_SPLIT: 1000, - }, - "avg_character_length": { - "test": { - "python": { - "average_document_length": 466.546, - "average_query_length": 862.842, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "javascript": { - "average_document_length": 186.018, - "average_query_length": 1415.632, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "go": { - "average_document_length": 125.213, - "average_query_length": 563.729, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "ruby": { - "average_document_length": 313.818, - "average_query_length": 577.634, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "java": { - "average_document_length": 420.287, - "average_query_length": 690.36, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "php": { - "average_document_length": 162.119, - "average_query_length": 712.129, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - }, - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py b/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py index 6351cc723e..e3175fa324 100644 --- a/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py @@ -47,106 +47,6 @@ class CodeEditSearchRetrieval(MultilingualTask, AbsTaskRetrieval): dialect=[], sample_creation="found", bibtex_citation="@article{muennighoff2023octopack, title={OctoPack: Instruction Tuning Code Large Language Models}, author={Niklas Muennighoff and Qian Liu and Armel Zebaze and Qinkai Zheng and Binyuan Hui and Terry Yue Zhuo and Swayam Singh and Xiangru Tang and Leandro von Werra and Shayne Longpre}, journal={arXiv preprint arXiv:2308.07124}, year={2023} }", - descriptive_stats={ - "n_samples": { - _EVAL_SPLIT: 1000 * len(_LANGS), - }, - "avg_character_length": { - "train": { - "python": { - "average_document_length": 597.592, - "average_query_length": 69.519, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "javascript": { - "average_document_length": 582.554, - "average_query_length": 56.88, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "typescript": { - "average_document_length": 580.877, - "average_query_length": 60.092, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "go": { - "average_document_length": 548.498, - "average_query_length": 70.797, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "ruby": { - "average_document_length": 518.895, - "average_query_length": 66.9, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "java": { - "average_document_length": 620.332, - "average_query_length": 62.984, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "php": { - "average_document_length": 545.452, - "average_query_length": 61.927, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "c": { - "average_document_length": 475.868, - "average_query_length": 97.588, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "c++": { - "average_document_length": 544.446, - "average_query_length": 114.48, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "rust": { - "average_document_length": 609.548, - "average_query_length": 67.503, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "swift": { - "average_document_length": 574.62, - "average_query_length": 57.279, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "scala": { - "average_document_length": 495.485, - "average_query_length": 64.833, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "shell": { - "average_document_length": 486.519, - "average_query_length": 72.059, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/code/CodeFeedbackMTRetrieval.py b/mteb/tasks/Retrieval/code/CodeFeedbackMTRetrieval.py index 3f307f12a3..3018501c46 100644 --- a/mteb/tasks/Retrieval/code/CodeFeedbackMTRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeFeedbackMTRetrieval.py @@ -37,18 +37,4 @@ class CodeFeedbackMT(AbsTaskRetrieval): primaryClass={cs.SE}, url={https://arxiv.org/abs/2402.14658}, }""", - descriptive_stats={ - "n_samples": { - _EVAL_SPLIT: 1000, - }, - "avg_character_length": { - "test": { - "average_document_length": 1467.879728243677, - "average_query_length": 4425.522256533855, - "num_documents": 66383, - "num_queries": 13277, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/code/CodeFeedbackSTRetrieval.py b/mteb/tasks/Retrieval/code/CodeFeedbackSTRetrieval.py index caae17bada..2a99c990c4 100644 --- a/mteb/tasks/Retrieval/code/CodeFeedbackSTRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeFeedbackSTRetrieval.py @@ -37,18 +37,4 @@ class CodeFeedbackST(AbsTaskRetrieval): primaryClass={cs.IR}, url={https://arxiv.org/abs/2407.02883}, }""", - descriptive_stats={ - "n_samples": { - _EVAL_SPLIT: 1000, - }, - "avg_character_length": { - "test": { - "average_document_length": 1521.3317148588733, - "average_query_length": 724.2441704465598, - "num_documents": 156526, - "num_queries": 31306, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py b/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py index 7751f5ed2a..3f5ca2e028 100644 --- a/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py @@ -104,57 +104,6 @@ class CodeSearchNetCCRetrieval(MultilingualTask, AbsTaskRetrieval): primaryClass={cs.IR}, url={https://arxiv.org/abs/2407.02883}, }""", - descriptive_stats={ - "n_samples": { - _EVAL_SPLIT: 1000, - }, - "avg_character_length": { - "test": { - "python": { - "average_document_length": 388.31577184555965, - "average_query_length": 551.7934039415471, - "num_documents": 280652, - "num_queries": 14918, - "average_relevant_docs_per_query": 1.0, - }, - "javascript": { - "average_document_length": 276.0730050152605, - "average_query_length": 443.70707991491946, - "num_documents": 65201, - "num_queries": 3291, - "average_relevant_docs_per_query": 1.0, - }, - "go": { - "average_document_length": 185.0307932251621, - "average_query_length": 233.76803742920464, - "num_documents": 182735, - "num_queries": 8122, - "average_relevant_docs_per_query": 1.0, - }, - "ruby": { - "average_document_length": 214.86204146730464, - "average_query_length": 266.8731165741475, - "num_documents": 27588, - "num_queries": 1261, - "average_relevant_docs_per_query": 1.0, - }, - "java": { - "average_document_length": 281.96280259139183, - "average_query_length": 342.5341853035144, - "num_documents": 181061, - "num_queries": 10955, - "average_relevant_docs_per_query": 1.0, - }, - "php": { - "average_document_length": 268.9752569556027, - "average_query_length": 336.62194947909234, - "num_documents": 268237, - "num_queries": 14014, - "average_relevant_docs_per_query": 1.0, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py b/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py index a76ac3b231..ddcef675f5 100644 --- a/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py @@ -33,57 +33,6 @@ class CodeSearchNetRetrieval(MultilingualTask, AbsTaskRetrieval): dialect=[], sample_creation="found", bibtex_citation="@article{husain2019codesearchnet, title={{CodeSearchNet} challenge: Evaluating the state of semantic code search}, author={Husain, Hamel and Wu, Ho-Hsiang and Gazit, Tiferet and Allamanis, Miltiadis and Brockschmidt, Marc}, journal={arXiv preprint arXiv:1909.09436}, year={2019} }", - descriptive_stats={ - "n_samples": { - _EVAL_SPLIT: 1000, - }, - "avg_character_length": { - "test": { - "python": { - "average_document_length": 862.842, - "average_query_length": 466.546, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "javascript": { - "average_document_length": 1415.632, - "average_query_length": 186.018, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "go": { - "average_document_length": 563.729, - "average_query_length": 125.213, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "ruby": { - "average_document_length": 577.634, - "average_query_length": 313.818, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "java": { - "average_document_length": 420.287, - "average_query_length": 690.36, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - "php": { - "average_document_length": 712.129, - "average_query_length": 162.119, - "num_documents": 1000, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - }, - }, - }, - }, ) def load_data(self, **kwargs): @@ -119,7 +68,7 @@ def load_data(self, **kwargs): sub = sub[ : min( len(sub), - self.metadata.descriptive_stats["n_samples"][self._EVAL_SPLIT], + 1000, ) ] diff --git a/mteb/tasks/Retrieval/code/CodeTransOceanContestRetrieval.py b/mteb/tasks/Retrieval/code/CodeTransOceanContestRetrieval.py index b7200b38a9..160e165c2b 100644 --- a/mteb/tasks/Retrieval/code/CodeTransOceanContestRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeTransOceanContestRetrieval.py @@ -37,15 +37,4 @@ class CodeTransOceanContestRetrieval(AbsTaskRetrieval): primaryClass={cs.AI}, url={https://arxiv.org/abs/2310.04951}, }""", - descriptive_stats={ - "avg_character_length": { - "test": { - "average_document_length": 1528.9156746031747, - "average_query_length": 1012.1131221719457, - "num_documents": 1008, - "num_queries": 221, - "average_relevant_docs_per_query": 1.0, - } - } - }, ) diff --git a/mteb/tasks/Retrieval/code/CodeTransOceanDLRetrieval.py b/mteb/tasks/Retrieval/code/CodeTransOceanDLRetrieval.py index 3b61e0e9c4..06bf940b7d 100644 --- a/mteb/tasks/Retrieval/code/CodeTransOceanDLRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeTransOceanDLRetrieval.py @@ -37,15 +37,4 @@ class CodeTransOceanDLRetrieval(AbsTaskRetrieval): primaryClass={cs.AI}, url={https://arxiv.org/abs/2310.04951}, }""", - descriptive_stats={ - "avg_character_length": { - "test": { - "average_document_length": 1479.0735294117646, - "average_query_length": 1867.6222222222223, - "num_documents": 816, - "num_queries": 180, - "average_relevant_docs_per_query": 1.0, - } - } - }, ) diff --git a/mteb/tasks/Retrieval/code/CosQARetrieval.py b/mteb/tasks/Retrieval/code/CosQARetrieval.py index c51b266ea5..ddb1992be9 100644 --- a/mteb/tasks/Retrieval/code/CosQARetrieval.py +++ b/mteb/tasks/Retrieval/code/CosQARetrieval.py @@ -37,15 +37,4 @@ class CosQARetrieval(AbsTaskRetrieval): primaryClass={cs.CL}, url={https://arxiv.org/abs/2105.13239}, }""", - descriptive_stats={ - "avg_character_length": { - "test": { - "average_document_length": 276.132741215298, - "average_query_length": 36.814, - "num_documents": 20604, - "num_queries": 500, - "average_relevant_docs_per_query": 1.0, - } - } - }, ) diff --git a/mteb/tasks/Retrieval/code/StackOverflowQARetrieval.py b/mteb/tasks/Retrieval/code/StackOverflowQARetrieval.py index bd1d2da5ea..3f06da1660 100644 --- a/mteb/tasks/Retrieval/code/StackOverflowQARetrieval.py +++ b/mteb/tasks/Retrieval/code/StackOverflowQARetrieval.py @@ -37,18 +37,4 @@ class StackOverflowQARetrieval(AbsTaskRetrieval): primaryClass={cs.IR}, url={https://arxiv.org/abs/2407.02883}, }""", - descriptive_stats={ - "n_samples": { - _EVAL_SPLIT: 1000, - }, - "avg_character_length": { - "test": { - "average_document_length": 1202.4815613867845, - "average_query_length": 1302.6263791374122, - "num_documents": 19931, - "num_queries": 1994, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/code/SyntheticText2SqlRetrieval.py b/mteb/tasks/Retrieval/code/SyntheticText2SqlRetrieval.py index 267d02048a..cd4cd8835e 100644 --- a/mteb/tasks/Retrieval/code/SyntheticText2SqlRetrieval.py +++ b/mteb/tasks/Retrieval/code/SyntheticText2SqlRetrieval.py @@ -35,18 +35,4 @@ class SyntheticText2SQLRetrieval(AbsTaskRetrieval): year = {2024}, url = {https://huggingface.co/datasets/gretelai/synthetic-text-to-sql} }""", - descriptive_stats={ - "n_samples": { - _EVAL_SPLIT: 1000, - }, - "avg_character_length": { - "test": { - "average_document_length": 127.07126054548375, - "average_query_length": 82.90582806357888, - "num_documents": 105851, - "num_queries": 5851, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py b/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py index 2468463a13..6a7b239f2f 100644 --- a/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py +++ b/mteb/tasks/Retrieval/dan/DanFeverRetrieval.py @@ -44,17 +44,8 @@ class DanFeverRetrieval(AbsTaskRetrieval): abstract = "We present a dataset, DanFEVER, intended for multilingual misinformation research. The dataset is in Danish and has the same format as the well-known English FEVER dataset. It can be used for testing methods in multilingual settings, as well as for creating models in production for the Danish language.", } """, - descriptive_stats={ - "n_samples": {"train": 8897}, - "avg_character_length": { - "train": { - "average_document_length": 312.1117274167987, - "average_query_length": 50.26957476855484, - "num_documents": 2524, - "num_queries": 6373, - "average_relevant_docs_per_query": 0.48721167425074535, - } - }, + prompt={ + "query": "Given a claim in Danish, retrieve documents that support the claim" }, task_subtypes=["Claim verification"], ) @@ -156,17 +147,8 @@ class DanFever(AbsTaskRetrieval): abstract = "We present a dataset, DanFEVER, intended for multilingual misinformation research. The dataset is in Danish and has the same format as the well-known English FEVER dataset. It can be used for testing methods in multilingual settings, as well as for creating models in production for the Danish language.", } """, - descriptive_stats={ - "n_samples": {"train": 8897}, - "avg_character_length": { - "train": { - "average_document_length": 312.1117274167987, - "average_query_length": 50.26957476855484, - "num_documents": 2524, - "num_queries": 6373, - "average_relevant_docs_per_query": 0.48721167425074535, - } - }, + prompt={ + "query": "Given a claim in Danish, retrieve documents that support the claim" }, task_subtypes=["Claim verification"], ) diff --git a/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py b/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py index 0f81e28618..1abc46fcc9 100644 --- a/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py +++ b/mteb/tasks/Retrieval/dan/TV2Nordretrieval.py @@ -55,17 +55,8 @@ class TV2Nordretrieval(AbsTaskRetrieval): pages = "2440--2445", abstract = "To date, there has been no resource for studying discourse coherence on real-world Danish texts. Discourse coherence has mostly been approached with the assumption that incoherent texts can be represented by coherent texts in which sentences have been shuffled. However, incoherent real-world texts rarely resemble that. We thus present DDisCo, a dataset including text from the Danish Wikipedia and Reddit annotated for discourse coherence. We choose to annotate real-world texts instead of relying on artificially incoherent text for training and testing models. Then, we evaluate the performance of several methods, including neural networks, on the dataset.", }""", - descriptive_stats={ - "n_samples": {"test": 4096}, - "avg_character_length": { - "test": { - "average_document_length": 1440.66552734375, - "average_query_length": 126.552734375, - "num_documents": 2048, - "num_queries": 2048, - "average_relevant_docs_per_query": 1.0, - }, - }, + prompt={ + "query": "Given a summary of a Danish news article retrieve the corresponding news article" }, task_subtypes=["Article retrieval"], ) diff --git a/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py b/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py index 85e9d1b8aa..5bc91789e7 100644 --- a/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py +++ b/mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py @@ -34,18 +34,7 @@ class TwitterHjerneRetrieval(AbsTaskRetrieval): year={2024} } """, - descriptive_stats={ - "n_samples": {"train": 340}, - "avg_character_length": { - "train": { - "average_document_length": 128.85114503816794, - "average_query_length": 166.3846153846154, - "num_documents": 262, - "num_queries": 78, - "average_relevant_docs_per_query": 3.358974358974359, - }, - }, - }, + prompt={"query": "Retrieve answers to questions asked in Danish tweets"}, task_subtypes=["Question answering"], ) diff --git a/mteb/tasks/Retrieval/deu/GerDaLIRRetrieval.py b/mteb/tasks/Retrieval/deu/GerDaLIRRetrieval.py index 69cbbcafa2..111eb986ed 100644 --- a/mteb/tasks/Retrieval/deu/GerDaLIRRetrieval.py +++ b/mteb/tasks/Retrieval/deu/GerDaLIRRetrieval.py @@ -44,18 +44,6 @@ class GerDaLIR(AbsTaskRetrieval): pages = "123--128", abstract = "We present GerDaLIR, a German Dataset for Legal Information Retrieval based on case documents from the open legal information platform Open Legal Data. The dataset consists of 123K queries, each labelled with at least one relevant document in a collection of 131K case documents. We conduct several baseline experiments including BM25 and a state-of-the-art neural re-ranker. With our dataset, we aim to provide a standardized benchmark for German LIR and promote open research in this area. Beyond that, our dataset comprises sufficient training data to be used as a downstream task for German or multilingual language models.", }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 15483.237726805888, - "average_query_length": 1027.3495690356156, - "num_documents": 131445, - "num_queries": 12298, - "average_relevant_docs_per_query": 1.1704342169458448, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/deu/GerDaLIRSmallRetrieval.py b/mteb/tasks/Retrieval/deu/GerDaLIRSmallRetrieval.py index a60c676c86..d80487251e 100644 --- a/mteb/tasks/Retrieval/deu/GerDaLIRSmallRetrieval.py +++ b/mteb/tasks/Retrieval/deu/GerDaLIRSmallRetrieval.py @@ -40,16 +40,4 @@ class GerDaLIRSmall(AbsTaskRetrieval): pages = "123--128", abstract = "We present GerDaLIR, a German Dataset for Legal Information Retrieval based on case documents from the open legal information platform Open Legal Data. The dataset consists of 123K queries, each labelled with at least one relevant document in a collection of 131K case documents. We conduct several baseline experiments including BM25 and a state-of-the-art neural re-ranker. With our dataset, we aim to provide a standardized benchmark for German LIR and promote open research in this area. Beyond that, our dataset comprises sufficient training data to be used as a downstream task for German or multilingual language models.", }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 19706.823653325308, - "average_query_length": 1031.0680889324833, - "num_documents": 9969, - "num_queries": 12234, - "average_relevant_docs_per_query": 1.1705084191597188, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/deu/GermanDPRRetrieval.py b/mteb/tasks/Retrieval/deu/GermanDPRRetrieval.py index 09c0e5cc51..19c90dc52e 100644 --- a/mteb/tasks/Retrieval/deu/GermanDPRRetrieval.py +++ b/mteb/tasks/Retrieval/deu/GermanDPRRetrieval.py @@ -40,18 +40,6 @@ class GermanDPR(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1288.3410987482614, - "average_query_length": 64.38439024390244, - "num_documents": 2876, - "num_queries": 1025, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) @staticmethod @@ -95,6 +83,9 @@ def load_data(self, **kwargs): ) corpus.update(neg_docs) relevant_docs[q_id] = {k: 1 for k in pos_docs} + corpus = { + key: doc.get("title", "") + " " + doc["text"] for key, doc in corpus.items() + } self.queries = {self._EVAL_SPLIT: queries} self.corpus = {self._EVAL_SPLIT: corpus} self.relevant_docs = {self._EVAL_SPLIT: relevant_docs} diff --git a/mteb/tasks/Retrieval/deu/GermanGovServiceRetrieval.py b/mteb/tasks/Retrieval/deu/GermanGovServiceRetrieval.py index d3cf1eb13f..10604f42a8 100644 --- a/mteb/tasks/Retrieval/deu/GermanGovServiceRetrieval.py +++ b/mteb/tasks/Retrieval/deu/GermanGovServiceRetrieval.py @@ -45,18 +45,6 @@ class GermanGovServiceRetrieval(AbsTaskRetrieval): url = {https://huggingface.co/datasets/it-at-m/LHM-Dienstleistungen-QA} }""", sample_creation="found", - descriptive_stats={ - "n_samples": {"test": 357}, - "avg_character_length": { - "test": { - "average_document_length": 1246.4571428571428, - "average_query_length": 68.17977528089888, - "num_documents": 105, - "num_queries": 356, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) @staticmethod diff --git a/mteb/tasks/Retrieval/deu/GermanQuADRetrieval.py b/mteb/tasks/Retrieval/deu/GermanQuADRetrieval.py index f2702218a8..dec5b4e033 100644 --- a/mteb/tasks/Retrieval/deu/GermanQuADRetrieval.py +++ b/mteb/tasks/Retrieval/deu/GermanQuADRetrieval.py @@ -57,18 +57,6 @@ class GermanQuADRetrieval(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1941.090717299578, - "average_query_length": 56.74773139745916, - "num_documents": 474, - "num_queries": 2204, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/deu/LegalQuADRetrieval.py b/mteb/tasks/Retrieval/deu/LegalQuADRetrieval.py index d9bf38380a..a3676bb13e 100644 --- a/mteb/tasks/Retrieval/deu/LegalQuADRetrieval.py +++ b/mteb/tasks/Retrieval/deu/LegalQuADRetrieval.py @@ -38,16 +38,4 @@ class LegalQuAD(AbsTaskRetrieval): keywords={Knowledge engineering;Law;Semantic search;Conferences;Bit error rate;NLP;knowledge extraction;question-answering;semantic search;document retrieval;German language}, doi={10.1109/AIKE52691.2021.00011} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 19481.955, - "average_query_length": 71.965, - "num_documents": 200, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/ell/GreekCivicsQA.py b/mteb/tasks/Retrieval/ell/GreekCivicsQA.py index 2c8bad3621..2f9860052b 100644 --- a/mteb/tasks/Retrieval/ell/GreekCivicsQA.py +++ b/mteb/tasks/Retrieval/ell/GreekCivicsQA.py @@ -32,18 +32,6 @@ class GreekCivicsQA(AbsTaskRetrieval): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"default": 407}, - "avg_character_length": { - "default": { - "average_document_length": 1074.894348894349, - "average_query_length": 77.06142506142506, - "num_documents": 407, - "num_queries": 407, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/eng/AILACasedocsRetrieval.py b/mteb/tasks/Retrieval/eng/AILACasedocsRetrieval.py index 9af2bf3026..d1d56e7737 100644 --- a/mteb/tasks/Retrieval/eng/AILACasedocsRetrieval.py +++ b/mteb/tasks/Retrieval/eng/AILACasedocsRetrieval.py @@ -42,16 +42,4 @@ class AILACasedocs(AbsTaskRetrieval): doi = {10.5281/zenodo.4063986}, url = {https://doi.org/10.5281/zenodo.4063986} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 26948.344086021505, - "average_query_length": 3038.42, - "num_documents": 186, - "num_queries": 50, - "average_relevant_docs_per_query": 3.9, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/AILAStatutesRetrieval.py b/mteb/tasks/Retrieval/eng/AILAStatutesRetrieval.py index db84601a7a..4577c64ed6 100644 --- a/mteb/tasks/Retrieval/eng/AILAStatutesRetrieval.py +++ b/mteb/tasks/Retrieval/eng/AILAStatutesRetrieval.py @@ -42,16 +42,4 @@ class AILAStatutes(AbsTaskRetrieval): doi = {10.5281/zenodo.4063986}, url = {https://doi.org/10.5281/zenodo.4063986} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1973.6341463414635, - "average_query_length": 3038.42, - "num_documents": 82, - "num_queries": 50, - "average_relevant_docs_per_query": 4.34, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/ARCChallengeRetrieval.py b/mteb/tasks/Retrieval/eng/ARCChallengeRetrieval.py index b999b30ce7..7488e902d2 100644 --- a/mteb/tasks/Retrieval/eng/ARCChallengeRetrieval.py +++ b/mteb/tasks/Retrieval/eng/ARCChallengeRetrieval.py @@ -41,16 +41,5 @@ class ARCChallenge(AbsTaskRetrieval): year={2018} } """, - descriptive_stats={ - "n_samples": {"test": 1172}, - "avg_character_length": { - "test": { - "average_document_length": 30.94235294117647, - "average_query_length": 131.56569965870307, - "num_documents": 9350, - "num_queries": 1172, - "average_relevant_docs_per_query": 1.0, - } - }, - }, + prompt={"query": "Retrieve the answer to the question."}, ) diff --git a/mteb/tasks/Retrieval/eng/AlphaNLIRetrieval.py b/mteb/tasks/Retrieval/eng/AlphaNLIRetrieval.py index a7cca69d14..3fd53b5ab5 100644 --- a/mteb/tasks/Retrieval/eng/AlphaNLIRetrieval.py +++ b/mteb/tasks/Retrieval/eng/AlphaNLIRetrieval.py @@ -42,16 +42,7 @@ class AlphaNLI(AbsTaskRetrieval): year={2019} } """, - descriptive_stats={ - "n_samples": {"test": 1532}, - "avg_character_length": { - "test": { - "average_document_length": 43.42647308646886, - "average_query_length": 103.05483028720627, - "num_documents": 241347, - "num_queries": 1532, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given the following start and end of a story, retrieve a possible reason that leads to the end." }, ) diff --git a/mteb/tasks/Retrieval/eng/ArguAnaRetrieval.py b/mteb/tasks/Retrieval/eng/ArguAnaRetrieval.py index 81a416381c..ff608bab6e 100644 --- a/mteb/tasks/Retrieval/eng/ArguAnaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/ArguAnaRetrieval.py @@ -39,16 +39,5 @@ class ArguAna(AbsTaskRetrieval): country = {Italy}, url = {http://www.cl.uni-heidelberg.de/~riezler/publications/papers/ECIR2016.pdf} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1029.2327645838136, - "average_query_length": 1192.7204836415362, - "num_documents": 8674, - "num_queries": 1406, - "average_relevant_docs_per_query": 1.0, - } - }, - }, + prompt={"query": "Given a claim, find documents that refute the claim"}, ) diff --git a/mteb/tasks/Retrieval/eng/BrightRetrieval.py b/mteb/tasks/Retrieval/eng/BrightRetrieval.py index 351afa86bf..393b121f3f 100644 --- a/mteb/tasks/Retrieval/eng/BrightRetrieval.py +++ b/mteb/tasks/Retrieval/eng/BrightRetrieval.py @@ -50,6 +50,7 @@ class BrightRetrieval(MultilingualTask, AbsTaskRetrieval): domains=["Non-fiction", "Written"], task_subtypes=["Article retrieval"], license="cc-by-4.0", + socioeconomic_status="low", annotations_creators="derived", dialect=[], sample_creation="found", @@ -65,13 +66,6 @@ class BrightRetrieval(MultilingualTask, AbsTaskRetrieval): url={https://arxiv.org/abs/2407.12883}, } """, - descriptive_stats={ - "n_samples": {"standard": 1334914, "long": 7048}, - "avg_character_length": { - "standard": 800.3994729248476, - "long": 46527.35839954597, - }, - }, ) def load_bright_data( diff --git a/mteb/tasks/Retrieval/eng/CQADupstackAndroidRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackAndroidRetrieval.py index 1bc4267718..b95c61af47 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackAndroidRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackAndroidRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackAndroidRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 593.701974084703, - "average_query_length": 51.76680972818312, - "num_documents": 22998, - "num_queries": 699, - "average_relevant_docs_per_query": 2.4263233190271816, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/CQADupstackEnglishRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackEnglishRetrieval.py index b425095dd1..d9f1c1f344 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackEnglishRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackEnglishRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackEnglishRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 482.4710971880361, - "average_query_length": 48.32993630573248, - "num_documents": 40221, - "num_queries": 1570, - "average_relevant_docs_per_query": 2.3980891719745223, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/CQADupstackGamingRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackGamingRetrieval.py index 14c117bfc0..8c89299957 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackGamingRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackGamingRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackGamingRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 488.74152888457206, - "average_query_length": 48.772413793103446, - "num_documents": 45301, - "num_queries": 1595, - "average_relevant_docs_per_query": 1.418808777429467, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/CQADupstackGisRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackGisRetrieval.py index 04503fb72a..8ed296b003 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackGisRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackGisRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackGisRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1012.167813587693, - "average_query_length": 52.2, - "num_documents": 37637, - "num_queries": 885, - "average_relevant_docs_per_query": 1.2587570621468926, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/CQADupstackMathematicaRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackMathematicaRetrieval.py index 2dd0b71bd3..0d1804e5e7 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackMathematicaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackMathematicaRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackMathematicaRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1153.4967375037413, - "average_query_length": 48.90547263681592, - "num_documents": 16705, - "num_queries": 804, - "average_relevant_docs_per_query": 1.6890547263681592, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/CQADupstackPhysicsRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackPhysicsRetrieval.py index 4f2768945f..77402252f9 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackPhysicsRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackPhysicsRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackPhysicsRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 818.6476145735463, - "average_query_length": 53.36477382098171, - "num_documents": 38316, - "num_queries": 1039, - "average_relevant_docs_per_query": 1.8604427333974976, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/CQADupstackProgrammersRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackProgrammersRetrieval.py index e7302364de..1fa63dd20a 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackProgrammersRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackProgrammersRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackProgrammersRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1055.7033814022875, - "average_query_length": 55.1837899543379, - "num_documents": 32176, - "num_queries": 876, - "average_relevant_docs_per_query": 1.9121004566210045, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/CQADupstackStatsRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackStatsRetrieval.py index 95e1d6fb6d..8b2ee5950a 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackStatsRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackStatsRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackStatsRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1055.1668598736662, - "average_query_length": 56.31748466257669, - "num_documents": 42269, - "num_queries": 652, - "average_relevant_docs_per_query": 1.4003067484662577, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/CQADupstackTexRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackTexRetrieval.py index 633f917a81..2e87f49710 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackTexRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackTexRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackTexRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1297.09043177285, - "average_query_length": 46.935306262904334, - "num_documents": 68184, - "num_queries": 2906, - "average_relevant_docs_per_query": 1.7735719201651754, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/CQADupstackUnixRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackUnixRetrieval.py index 35ccd70f00..f86d886519 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackUnixRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackUnixRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackUnixRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1004.8120383267908, - "average_query_length": 50.32369402985075, - "num_documents": 47382, - "num_queries": 1072, - "average_relevant_docs_per_query": 1.5792910447761195, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/CQADupstackWebmastersRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackWebmastersRetrieval.py index 05df28f386..eedacec19a 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackWebmastersRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackWebmastersRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackWebmastersRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 707.3635736857225, - "average_query_length": 51.93478260869565, - "num_documents": 17405, - "num_queries": 506, - "average_relevant_docs_per_query": 2.7569169960474307, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/CQADupstackWordpressRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackWordpressRetrieval.py index 4495be7a5d..e70255c371 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackWordpressRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackWordpressRetrieval.py @@ -44,16 +44,4 @@ class CQADupstackWordpressRetrieval(AbsTaskRetrieval): publisher = {ACM}, address = {New York, NY, USA}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1122.7690155333814, - "average_query_length": 48.7264325323475, - "num_documents": 48605, - "num_queries": 541, - "average_relevant_docs_per_query": 1.3752310536044363, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py index 0e63882677..d60b7a3817 100644 --- a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py @@ -35,17 +35,8 @@ class ClimateFEVER(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 538.241873443325, - "average_query_length": 123.39934853420195, - "num_documents": 5416593, - "num_queries": 1535, - "average_relevant_docs_per_query": 3.0495114006514656, - } - }, + prompt={ + "query": "Given a claim about climate change, retrieve documents that support or refute the claim" }, ) @@ -80,16 +71,4 @@ class ClimateFEVERHardNegatives(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": { - "test": { - "average_document_length": 1245.4236333727013, - "average_query_length": 121.879, - "num_documents": 47416, - "num_queries": 1000, - "average_relevant_docs_per_query": 3.048, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/DBPediaRetrieval.py b/mteb/tasks/Retrieval/eng/DBPediaRetrieval.py index 24e1a9a499..77c0020aa0 100644 --- a/mteb/tasks/Retrieval/eng/DBPediaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/DBPediaRetrieval.py @@ -17,7 +17,7 @@ class DBPedia(AbsTaskRetrieval): type="Retrieval", category="s2p", modalities=["text"], - eval_splits=["dev", "test"], + eval_splits=["test"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2017-01-01", "2017-01-01"), # best guess: based on publication date @@ -37,17 +37,8 @@ class DBPedia(AbsTaskRetrieval): doi = {10.1145/3077136.3080751}, publisher = {ACM} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1122.7690155333814, - "average_query_length": 48.7264325323475, - "num_documents": 48605, - "num_queries": 541, - "average_relevant_docs_per_query": 1.3752310536044363, - } - }, + prompt={ + "query": "Given a query, retrieve relevant entity descriptions from DBPedia" }, ) @@ -84,16 +75,4 @@ class DBPediaHardNegatives(AbsTaskRetrieval): doi = {10.1145/3077136.3080751}, publisher = {ACM} }""", - descriptive_stats={ - "n_samples": {"test": 400}, - "avg_character_length": { - "test": { - "average_document_length": 338.58561119129564, - "average_query_length": 34.085, - "num_documents": 90070, - "num_queries": 400, - "average_relevant_docs_per_query": 38.215, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/FEVERRetrieval.py b/mteb/tasks/Retrieval/eng/FEVERRetrieval.py index 058332c94c..776fd2fbe6 100644 --- a/mteb/tasks/Retrieval/eng/FEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/FEVERRetrieval.py @@ -23,7 +23,7 @@ class FEVER(AbsTaskRetrieval): type="Retrieval", category="s2p", modalities=["text"], - eval_splits=["train", "dev", "test"], + eval_splits=["test"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, @@ -52,31 +52,8 @@ class FEVER(AbsTaskRetrieval): pages = "809--819", abstract = "In this paper we introduce a new publicly available dataset for verification against textual sources, FEVER: Fact Extraction and VERification. It consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The claims are classified as Supported, Refuted or NotEnoughInfo by annotators achieving 0.6841 in Fleiss kappa. For the first two classes, the annotators also recorded the sentence(s) forming the necessary evidence for their judgment. To characterize the challenge of the dataset presented, we develop a pipeline approach and compare it to suitably designed oracles. The best accuracy we achieve on labeling a claim accompanied by the correct evidence is 31.87{\%}, while if we ignore the evidence we achieve 50.91{\%}. Thus we believe that FEVER is a challenging testbed that will help stimulate progress on claim verification against textual sources.", }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "train": { - "average_document_length": 538.2340070317589, - "average_query_length": 47.56034058828886, - "num_documents": 5416568, - "num_queries": 109810, - "average_relevant_docs_per_query": 1.2757034878426372, - }, - "dev": { - "average_document_length": 538.2340070317589, - "average_query_length": 47.326282628262824, - "num_documents": 5416568, - "num_queries": 6666, - "average_relevant_docs_per_query": 1.211971197119712, - }, - "test": { - "average_document_length": 538.2340070317589, - "average_query_length": 49.60546054605461, - "num_documents": 5416568, - "num_queries": 6666, - "average_relevant_docs_per_query": 1.1906690669066906, - }, - }, + prompt={ + "query": "Given a claim, retrieve documents that support or refute the claim" }, ) @@ -128,16 +105,4 @@ class FEVERHardNegatives(AbsTaskRetrieval): pages = "809--819", abstract = "In this paper we introduce a new publicly available dataset for verification against textual sources, FEVER: Fact Extraction and VERification. It consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The claims are classified as Supported, Refuted or NotEnoughInfo by annotators achieving 0.6841 in Fleiss kappa. For the first two classes, the annotators also recorded the sentence(s) forming the necessary evidence for their judgment. To characterize the challenge of the dataset presented, we develop a pipeline approach and compare it to suitably designed oracles. The best accuracy we achieve on labeling a claim accompanied by the correct evidence is 31.87{\%}, while if we ignore the evidence we achieve 50.91{\%}. Thus we believe that FEVER is a challenging testbed that will help stimulate progress on claim verification against textual sources.", }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": { - "test": { - "average_document_length": 695.4370242764114, - "average_query_length": 49.62, - "num_documents": 163698, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.171, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/FaithDialRetrieval.py b/mteb/tasks/Retrieval/eng/FaithDialRetrieval.py index 3c8e3921e0..8cd87ed04b 100644 --- a/mteb/tasks/Retrieval/eng/FaithDialRetrieval.py +++ b/mteb/tasks/Retrieval/eng/FaithDialRetrieval.py @@ -49,18 +49,6 @@ class FaithDialRetrieval(AbsTaskRetrieval): doi={10.1162/tacl_a_00529} } """, - descriptive_stats={ - "n_samples": {"test": 2042}, - "avg_character_length": { - "test": { - "average_document_length": 140.61062447018932, - "average_query_length": 4.926542605288932, - "num_documents": 3539, - "num_queries": 2042, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) # TODO: Will be removed if curated and added to mteb HF diff --git a/mteb/tasks/Retrieval/eng/FeedbackQARetrieval.py b/mteb/tasks/Retrieval/eng/FeedbackQARetrieval.py index 852a87857a..44f0ac2522 100644 --- a/mteb/tasks/Retrieval/eng/FeedbackQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/FeedbackQARetrieval.py @@ -50,16 +50,4 @@ class FeedbackQARetrieval(AbsTaskRetrieval): pages = "926--937" } """, - descriptive_stats={ - "n_samples": {"test": 1992}, - "avg_character_length": { - "test": { - "average_document_length": 1174.7986463620982, - "average_query_length": 72.33182730923694, - "num_documents": 2364, - "num_queries": 1992, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/FiQA2018Retrieval.py b/mteb/tasks/Retrieval/eng/FiQA2018Retrieval.py index d95d395022..1489cd168c 100644 --- a/mteb/tasks/Retrieval/eng/FiQA2018Retrieval.py +++ b/mteb/tasks/Retrieval/eng/FiQA2018Retrieval.py @@ -19,7 +19,7 @@ class FiQA2018(AbsTaskRetrieval): type="Retrieval", category="s2p", modalities=["text"], - eval_splits=["train", "dev", "test"], + eval_splits=["test"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, @@ -37,30 +37,7 @@ class FiQA2018(AbsTaskRetrieval): year={2021}, url={https://openreview.net/forum?id=wCu6T5xFjeJ} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "train": { - "average_document_length": 767.2108157812554, - "average_query_length": 61.49763636363636, - "num_documents": 57638, - "num_queries": 5500, - "average_relevant_docs_per_query": 2.5756363636363635, - }, - "dev": { - "average_document_length": 767.2108157812554, - "average_query_length": 62.756, - "num_documents": 57638, - "num_queries": 500, - "average_relevant_docs_per_query": 2.476, - }, - "test": { - "average_document_length": 767.2108157812554, - "average_query_length": 62.7037037037037, - "num_documents": 57638, - "num_queries": 648, - "average_relevant_docs_per_query": 2.632716049382716, - }, - }, + prompt={ + "query": "Given a financial question, retrieve user replies that best answer the question" }, ) diff --git a/mteb/tasks/Retrieval/eng/HagridRetrieval.py b/mteb/tasks/Retrieval/eng/HagridRetrieval.py index 1b02000a6d..f9953caade 100644 --- a/mteb/tasks/Retrieval/eng/HagridRetrieval.py +++ b/mteb/tasks/Retrieval/eng/HagridRetrieval.py @@ -42,18 +42,6 @@ class HagridRetrieval(AbsTaskRetrieval): year={2023}, journal={arXiv:2307.16883}, }""", - descriptive_stats={ - "n_samples": {"train": 1922}, - "avg_character_length": { - "dev": { - "average_document_length": 228.36693548387098, - "average_query_length": 40.064516129032256, - "num_documents": 496, - "num_queries": 496, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/eng/HellaSwagRetrieval.py b/mteb/tasks/Retrieval/eng/HellaSwagRetrieval.py index 9bef36b76a..81b53e5c42 100644 --- a/mteb/tasks/Retrieval/eng/HellaSwagRetrieval.py +++ b/mteb/tasks/Retrieval/eng/HellaSwagRetrieval.py @@ -41,16 +41,7 @@ class HellaSwag(AbsTaskRetrieval): year={2019} } """, - descriptive_stats={ - "n_samples": {"test": 10042}, - "avg_character_length": { - "test": { - "average_document_length": 137.36519014671472, - "average_query_length": 224.53654650468033, - "num_documents": 199162, - "num_queries": 10042, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given the following unfinished context, retrieve the most plausible ending to finish it." }, ) diff --git a/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py b/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py index a11e8b0d79..2a347d9a05 100644 --- a/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/HotpotQARetrieval.py @@ -20,7 +20,7 @@ class HotpotQA(AbsTaskRetrieval): type="Retrieval", category="s2p", modalities=["text"], - eval_splits=["train", "dev", "test"], + eval_splits=["test"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=("2018-01-01", "2018-12-31"), # best guess: based on publication date @@ -53,31 +53,8 @@ class HotpotQA(AbsTaskRetrieval): pages = "2369--2380", abstract = "Existing question answering (QA) datasets fail to train QA systems to perform complex reasoning and provide explanations for answers. We introduce HotpotQA, a new dataset with 113k Wikipedia-based question-answer pairs with four key features: (1) the questions require finding and reasoning over multiple supporting documents to answer; (2) the questions are diverse and not constrained to any pre-existing knowledge bases or knowledge schemas; (3) we provide sentence-level supporting facts required for reasoning, allowing QA systems to reason with strong supervision and explain the predictions; (4) we offer a new type of factoid comparison questions to test QA systems{'} ability to extract relevant facts and perform necessary comparison. We show that HotpotQA is challenging for the latest QA systems, and the supporting facts enable models to improve performance and make explainable predictions.", }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "train": { - "average_document_length": 287.9079517072212, - "average_query_length": 105.54965882352941, - "num_documents": 5233329, - "num_queries": 85000, - "average_relevant_docs_per_query": 2.0, - }, - "dev": { - "average_document_length": 287.9079517072212, - "average_query_length": 105.35634294106848, - "num_documents": 5233329, - "num_queries": 5447, - "average_relevant_docs_per_query": 2.0, - }, - "test": { - "average_document_length": 287.9079517072212, - "average_query_length": 92.17096556380824, - "num_documents": 5233329, - "num_queries": 7405, - "average_relevant_docs_per_query": 2.0, - }, - }, + prompt={ + "query": "Given a multi-hop question, retrieve documents that can help answer the question" }, ) @@ -130,16 +107,4 @@ class HotpotQAHardNegatives(AbsTaskRetrieval): pages = "2369--2380", abstract = "Existing question answering (QA) datasets fail to train QA systems to perform complex reasoning and provide explanations for answers. We introduce HotpotQA, a new dataset with 113k Wikipedia-based question-answer pairs with four key features: (1) the questions require finding and reasoning over multiple supporting documents to answer; (2) the questions are diverse and not constrained to any pre-existing knowledge bases or knowledge schemas; (3) we provide sentence-level supporting facts required for reasoning, allowing QA systems to reason with strong supervision and explain the predictions; (4) we offer a new type of factoid comparison questions to test QA systems{'} ability to extract relevant facts and perform necessary comparison. We show that HotpotQA is challenging for the latest QA systems, and the supporting facts enable models to improve performance and make explainable predictions.", }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": { - "test": { - "average_document_length": 373.558822095461, - "average_query_length": 92.584, - "num_documents": 225621, - "num_queries": 1000, - "average_relevant_docs_per_query": 2.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py b/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py index e41c85dd3a..3d45290d71 100644 --- a/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBNarrativeQARetrieval.py @@ -57,18 +57,6 @@ class LEMBNarrativeQARetrieval(AbsTaskRetrieval): abstract = "", } """, - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 10804}, - "avg_character_length": { - "test": { - "average_document_length": 326753.5323943662, - "average_query_length": 47.89453536223562, - "num_documents": 355, - "num_queries": 10449, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py index a6ea725e0d..c467843d01 100644 --- a/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py @@ -49,76 +49,6 @@ class LEMBNeedleRetrieval(AbsTaskRetrieval): year={2024} } """, - descriptive_stats={ - "n_samples": { - "test_256": 150, - "test_512": 150, - "test_1024": 150, - "test_2048": 150, - "test_4096": 150, - "test_8192": 150, - "test_16384": 150, - "test_32768": 150, - }, - "avg_character_length": { - "test_256": { - "average_document_length": 1013.22, - "average_query_length": 60.48, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_512": { - "average_document_length": 2009.96, - "average_query_length": 57.3, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_1024": { - "average_document_length": 4069.9, - "average_query_length": 58.28, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_2048": { - "average_document_length": 8453.82, - "average_query_length": 59.92, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_4096": { - "average_document_length": 17395.8, - "average_query_length": 55.86, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_8192": { - "average_document_length": 35203.82, - "average_query_length": 59.6, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_16384": { - "average_document_length": 72054.8, - "average_query_length": 59.12, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_32768": { - "average_document_length": 141769.8, - "average_query_length": 58.34, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py index 0323d65bd7..f3c9b96485 100644 --- a/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBPasskeyRetrieval.py @@ -49,76 +49,6 @@ class LEMBPasskeyRetrieval(AbsTaskRetrieval): year={2024} } """, - descriptive_stats={ - "n_samples": { - "test_256": 150, - "test_512": 150, - "test_1024": 150, - "test_2048": 150, - "test_4096": 150, - "test_8192": 150, - "test_16384": 150, - "test_32768": 150, - }, - "avg_character_length": { - "test_256": { - "average_document_length": 876.24, - "average_query_length": 38.1, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_512": { - "average_document_length": 1785.2, - "average_query_length": 37.76, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_1024": { - "average_document_length": 3607.18, - "average_query_length": 37.68, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_2048": { - "average_document_length": 7242.2, - "average_query_length": 37.8, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_4096": { - "average_document_length": 14518.16, - "average_query_length": 37.64, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_8192": { - "average_document_length": 29071.16, - "average_query_length": 37.54, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_16384": { - "average_document_length": 58175.16, - "average_query_length": 38.12, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - "test_32768": { - "average_document_length": 116380.16, - "average_query_length": 37.74, - "num_documents": 100, - "num_queries": 50, - "average_relevant_docs_per_query": 1.0, - }, - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py index e7f21f8227..c302e4758a 100644 --- a/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBQMSumRetrieval.py @@ -66,18 +66,6 @@ class LEMBQMSumRetrieval(AbsTaskRetrieval): abstract = "", } """, - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 1724}, - "avg_character_length": { - "test": { - "average_document_length": 53335.817258883246, - "average_query_length": 433.50294695481335, - "num_documents": 197, - "num_queries": 1527, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py b/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py index d45c938663..c2c6b6db03 100644 --- a/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBSummScreenFDRetrieval.py @@ -53,18 +53,6 @@ class LEMBSummScreenFDRetrieval(AbsTaskRetrieval): abstract = "", } """, - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 672}, - "avg_character_length": { - "validation": { - "average_document_length": 30854.32738095238, - "average_query_length": 591.4910714285714, - "num_documents": 336, - "num_queries": 336, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py b/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py index c5f815e2e2..04e8b3bb86 100644 --- a/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py @@ -41,18 +41,6 @@ class LEMBWikimQARetrieval(AbsTaskRetrieval): year={2020} } """, - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 500}, - "avg_character_length": { - "test": { - "average_document_length": 37445.60333333333, - "average_query_length": 67.57, - "num_documents": 300, - "num_queries": 300, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/eng/LegalBenchConsumerContractsQARetrieval.py b/mteb/tasks/Retrieval/eng/LegalBenchConsumerContractsQARetrieval.py index 27451e30c6..39923194ec 100644 --- a/mteb/tasks/Retrieval/eng/LegalBenchConsumerContractsQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/LegalBenchConsumerContractsQARetrieval.py @@ -40,16 +40,4 @@ class LegalBenchConsumerContractsQA(AbsTaskRetrieval): journal={arXiv preprint arXiv:2103.06268}, year={2021} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 2745.8246753246754, - "average_query_length": 92.4090909090909, - "num_documents": 154, - "num_queries": 396, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/LegalBenchCorporateLobbyingRetrieval.py b/mteb/tasks/Retrieval/eng/LegalBenchCorporateLobbyingRetrieval.py index 939e4cfecc..25eeee5dc4 100644 --- a/mteb/tasks/Retrieval/eng/LegalBenchCorporateLobbyingRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LegalBenchCorporateLobbyingRetrieval.py @@ -97,16 +97,4 @@ class LegalBenchCorporateLobbying(AbsTaskRetrieval): publisher={Springer} } """, - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1157.2225705329154, - "average_query_length": 177.87941176470588, - "num_documents": 319, - "num_queries": 340, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/LegalSummarizationRetrieval.py b/mteb/tasks/Retrieval/eng/LegalSummarizationRetrieval.py index 884cc546fe..3fc4cf167d 100644 --- a/mteb/tasks/Retrieval/eng/LegalSummarizationRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LegalSummarizationRetrieval.py @@ -39,16 +39,4 @@ class LegalSummarization(AbsTaskRetrieval): url = "https://www.aclweb.org/anthology/W19-2201", pages = "1--11", }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 606.1643835616438, - "average_query_length": 103.19014084507042, - "num_documents": 438, - "num_queries": 284, - "average_relevant_docs_per_query": 1.545774647887324, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/LitSearchRetrieval.py b/mteb/tasks/Retrieval/eng/LitSearchRetrieval.py index dbaa9a6de7..2c823e85dd 100644 --- a/mteb/tasks/Retrieval/eng/LitSearchRetrieval.py +++ b/mteb/tasks/Retrieval/eng/LitSearchRetrieval.py @@ -40,18 +40,6 @@ class LitSearchRetrieval(AbsTaskRetrieval): author={Ajith, Anirudh and Xia, Mengzhou and Chevalier, Alexis and Goyal, Tanya and Chen, Danqi and Gao, Tianyu}, year={2024} }""", - descriptive_stats={ - "n_samples": {"test": 597}, - "avg_character_length": { - "test": { - "average_document_length": 841.2769, - "average_query_length": 141.20, - "num_documents": 64183, - "num_queries": 597, - "average_relevant_docs_per_query": 1.070351, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/eng/MLQuestions.py b/mteb/tasks/Retrieval/eng/MLQuestions.py index 01ebe4dde4..6b594be445 100644 --- a/mteb/tasks/Retrieval/eng/MLQuestions.py +++ b/mteb/tasks/Retrieval/eng/MLQuestions.py @@ -56,25 +56,6 @@ class MLQuestionsRetrieval(AbsTaskRetrieval): abstract = "In this work, we introduce back-training, an alternative to self-training for unsupervised domain adaptation (UDA). While self-training generates synthetic training data where natural inputs are aligned with noisy outputs, back-training results in natural outputs aligned with noisy inputs. This significantly reduces the gap between target domain and synthetic data distribution, and reduces model overfitting to source domain. We run UDA experiments on question generation and passage retrieval from the Natural Questions domain to machine learning and biomedical domains. We find that back-training vastly outperforms self-training by a mean improvement of 7.8 BLEU-4 points on generation, and 17.6{\%} top-20 retrieval accuracy across both domains. We further propose consistency filters to remove low-quality synthetic data before training. We also release a new domain-adaptation dataset - MLQuestions containing 35K unaligned questions, 50K unaligned passages, and 3K aligned question-passage pairs.", } """, - descriptive_stats={ - "n_samples": {"dev": 1500, "test": 1500}, - "avg_character_length": { - "dev": { - "average_document_length": 258.8772727272727, - "average_query_length": 45.05533333333333, - "num_documents": 11000, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "test": { - "average_document_length": 258.8772727272727, - "average_query_length": 45.75333333333333, - "num_documents": 11000, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py b/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py index dd9260c260..5ada0cf887 100644 --- a/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py +++ b/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py @@ -19,7 +19,7 @@ class MSMARCO(AbsTaskRetrieval): type="Retrieval", category="s2p", modalities=["text"], - eval_splits=["train", "dev", "test"], + eval_splits=["dev"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, @@ -49,31 +49,8 @@ class MSMARCO(AbsTaskRetrieval): bibsource = {dblp computer science bibliography, https://dblp.org} } }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "train": { - "average_document_length": 335.79716603691344, - "average_query_length": 33.21898281898998, - "num_documents": 8841823, - "num_queries": 502939, - "average_relevant_docs_per_query": 1.0592755781516248, - }, - "dev": { - "average_document_length": 335.79716603691344, - "average_query_length": 33.2621776504298, - "num_documents": 8841823, - "num_queries": 6980, - "average_relevant_docs_per_query": 1.0654727793696275, - }, - "test": { - "average_document_length": 335.79716603691344, - "average_query_length": 32.74418604651163, - "num_documents": 8841823, - "num_queries": 43, - "average_relevant_docs_per_query": 95.3953488372093, - }, - }, + prompt={ + "query": "Given a web search query, retrieve relevant passages that answer the query" }, ) @@ -122,16 +99,4 @@ class MSMARCOHardNegatives(AbsTaskRetrieval): bibsource = {dblp computer science bibliography, https://dblp.org} } }""", - descriptive_stats={ - "n_samples": {"test": 43}, - "avg_character_length": { - "test": { - "average_document_length": 355.2909668633681, - "average_query_length": 32.74418604651163, - "num_documents": 8812, - "num_queries": 43, - "average_relevant_docs_per_query": 95.3953488372093, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/MSMARCOv2Retrieval.py b/mteb/tasks/Retrieval/eng/MSMARCOv2Retrieval.py index 18a42f8eec..d3b10738cf 100644 --- a/mteb/tasks/Retrieval/eng/MSMARCOv2Retrieval.py +++ b/mteb/tasks/Retrieval/eng/MSMARCOv2Retrieval.py @@ -47,5 +47,4 @@ class MSMARCOv2(AbsTaskRetrieval): bibsource = {dblp computer science bibliography, https://dblp.org} } }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) diff --git a/mteb/tasks/Retrieval/eng/MedicalQARetrieval.py b/mteb/tasks/Retrieval/eng/MedicalQARetrieval.py index 4a8b4bb3b7..12607572bd 100644 --- a/mteb/tasks/Retrieval/eng/MedicalQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/MedicalQARetrieval.py @@ -36,16 +36,4 @@ class MedicalQARetrieval(AbsTaskRetrieval): year = {2019}, url = {https://bmcbioinformatics.biomedcentral.com/articles/10.1186/s12859-019-3119-4} } """, - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": { - "test": { - "average_document_length": 1153.482421875, - "average_query_length": 52.4794921875, - "num_documents": 2048, - "num_queries": 2048, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/NFCorpusRetrieval.py b/mteb/tasks/Retrieval/eng/NFCorpusRetrieval.py index 6b4b47f681..31f4eb60b1 100644 --- a/mteb/tasks/Retrieval/eng/NFCorpusRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NFCorpusRetrieval.py @@ -21,7 +21,7 @@ class NFCorpus(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, + domains=["Medical", "Academic", "Written"], task_subtypes=None, license=None, annotations_creators=None, @@ -37,16 +37,7 @@ class NFCorpus(AbsTaskRetrieval): country = {Italy}, url = {http://www.cl.uni-heidelberg.de/~riezler/publications/papers/ECIR2016.pdf} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1589.783925130746, - "average_query_length": 21.764705882352942, - "num_documents": 3633, - "num_queries": 323, - "average_relevant_docs_per_query": 38.18575851393189, - } - }, + prompt={ + "query": "Given a question, retrieve relevant documents that best answer the question" }, ) diff --git a/mteb/tasks/Retrieval/eng/NQRetrieval.py b/mteb/tasks/Retrieval/eng/NQRetrieval.py index 0d11c0a4dc..661bf3e0e2 100644 --- a/mteb/tasks/Retrieval/eng/NQRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NQRetrieval.py @@ -33,17 +33,8 @@ class NQ(AbsTaskRetrieval): and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational Linguistics}}""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 492.2287851281462, - "average_query_length": 48.17902665121669, - "num_documents": 2681468, - "num_queries": 3452, - "average_relevant_docs_per_query": 1.2169756662804172, - } - }, + prompt={ + "query": "Given a question, retrieve Wikipedia passages that answer the question" }, ) @@ -76,16 +67,4 @@ class NQHardNegatives(AbsTaskRetrieval): and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational Linguistics}}""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": { - "test": { - "average_document_length": 602.7903551179953, - "average_query_length": 47.878, - "num_documents": 198779, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.213, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py new file mode 100644 index 0000000000..7b5a728537 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoArguAnaRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoArguAnaRetrieval", + description="NanoArguAna is a smaller subset of ArguAna, a dataset for argument retrieval in debate contexts.", + reference="http://argumentation.bplaced.net/arguana/data", + dataset={ + "path": "zeta-alpha-ai/NanoArguAna", + "revision": "8f4a982d470a32c45817738b9d29042ca55d75ad", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2020-01-01", "2020-12-31"], + domains=["Medical", "Written"], + task_subtypes=["Discourse coherence"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{boteva2016, + author = {Boteva, Vera and Gholipour, Demian and Sokolov, Artem and Riezler, Stefan}, + title = {A Full-Text Learning to Rank Dataset for Medical Information Retrieval}, + journal = {Proceedings of the 38th European Conference on Information Retrieval}, + journal-abbrev = {ECIR}, + year = {2016}, + city = {Padova}, + country = {Italy}, + url = {http://www.cl.uni-heidelberg.de/~riezler/publications/papers/ECIR2016.pdf} +}""", + prompt={"query": "Given a claim, find documents that refute the claim"}, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoArguAna", + "corpus", + revision="8f4a982d470a32c45817738b9d29042ca55d75ad", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoArguAna", + "queries", + revision="8f4a982d470a32c45817738b9d29042ca55d75ad", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoArguAna", + "qrels", + revision="8f4a982d470a32c45817738b9d29042ca55d75ad", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py new file mode 100644 index 0000000000..b297dec5e3 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoClimateFeverRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoClimateFeverRetrieval", + description="NanoClimateFever is a small version of the BEIR dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change.", + reference="https://arxiv.org/abs/2012.00614", + dataset={ + "path": "zeta-alpha-ai/NanoClimateFEVER", + "revision": "96741bfa30b9f56db8c9eb7d08e775ed6474f206", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2020-01-01", "2020-12-31"], + domains=["Non-fiction", "Academic", "News"], + task_subtypes=["Claim verification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@misc{diggelmann2021climatefever, + title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, + author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold}, + year={2021}, + eprint={2012.00614}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +}""", + prompt={ + "query": "Given a claim about climate change, retrieve documents that support or refute the claim" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoClimateFEVER", + "corpus", + revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoClimateFEVER", + "queries", + revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoClimateFEVER", + "qrels", + revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py new file mode 100644 index 0000000000..37826697be --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoDBPediaRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoDBPediaRetrieval", + description="NanoDBPediaRetrieval is a small version of the standard test collection for entity search over the DBpedia knowledge base.", + reference="https://huggingface.co/datasets/zeta-alpha-ai/NanoDBPedia", + dataset={ + "path": "zeta-alpha-ai/NanoDBPedia", + "revision": "438f1c25129f05db6238699b5afdc9c6b58d2096", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2015-01-01", "2015-12-31"], + domains=["Encyclopaedic"], + task_subtypes=["Topic classification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{lehmann2015dbpedia, title={DBpedia: A large-scale, multilingual knowledge base extracted from Wikipedia}, author={Lehmann, Jens and et al.}, journal={Semantic Web}, year={2015}}""", + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoDBPedia", + "corpus", + revision="438f1c25129f05db6238699b5afdc9c6b58d2096", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoDBPedia", + "queries", + revision="438f1c25129f05db6238699b5afdc9c6b58d2096", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoDBPedia", + "qrels", + revision="438f1c25129f05db6238699b5afdc9c6b58d2096", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py new file mode 100644 index 0000000000..636bfd12a1 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoFEVERRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoFEVERRetrieval", + description="NanoFEVER is a smaller version of " + + "FEVER (Fact Extraction and VERification), which consists of 185,445 claims generated by altering sentences" + + " extracted from Wikipedia and subsequently verified without knowledge of the sentence they were" + + " derived from.", + reference="https://fever.ai/", + dataset={ + "path": "zeta-alpha-ai/NanoFEVER", + "revision": "a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Academic", "Encyclopaedic"], + task_subtypes=["Claim verification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{thorne-etal-2018-fever, + title = "{FEVER}: a Large-scale Dataset for Fact Extraction and {VER}ification", + author = "Thorne, James and + Vlachos, Andreas and + Christodoulopoulos, Christos and + Mittal, Arpit", + editor = "Walker, Marilyn and + Ji, Heng and + Stent, Amanda", + booktitle = "Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)", + month = jun, + year = "2018", + address = "New Orleans, Louisiana", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/N18-1074", + doi = "10.18653/v1/N18-1074", + pages = "809--819", + abstract = "In this paper we introduce a new publicly available dataset for verification against textual sources, FEVER: Fact Extraction and VERification. It consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The claims are classified as Supported, Refuted or NotEnoughInfo by annotators achieving 0.6841 in Fleiss kappa. For the first two classes, the annotators also recorded the sentence(s) forming the necessary evidence for their judgment. To characterize the challenge of the dataset presented, we develop a pipeline approach and compare it to suitably designed oracles. The best accuracy we achieve on labeling a claim accompanied by the correct evidence is 31.87{\%}, while if we ignore the evidence we achieve 50.91{\%}. Thus we believe that FEVER is a challenging testbed that will help stimulate progress on claim verification against textual sources.", +}""", + prompt={ + "query": "Given a claim, retrieve documents that support or refute the claim" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoFEVER", + "corpus", + revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoFEVER", + "queries", + revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoFEVER", + "qrels", + revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py new file mode 100644 index 0000000000..4129a18137 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoFiQA2018Retrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoFiQA2018Retrieval", + description="NanoFiQA2018 is a smaller subset of the Financial Opinion Mining and Question Answering dataset.", + reference="https://sites.google.com/view/fiqa/", + dataset={ + "path": "zeta-alpha-ai/NanoFiQA2018", + "revision": "4163ba032953d5044a7a6244261413f609c14342", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Academic", "Social"], + task_subtypes=["Sentiment/Hate speech"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{ +thakur2021beir, +title={{BEIR}: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models}, +author={Nandan Thakur and Nils Reimers and Andreas R{\"u}ckl{\'e} and Abhishek Srivastava and Iryna Gurevych}, +booktitle={Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)}, +year={2021}, +url={https://openreview.net/forum?id=wCu6T5xFjeJ} +}""", + prompt={ + "query": "Given a financial question, retrieve user replies that best answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoFiQA2018", + "corpus", + revision="4163ba032953d5044a7a6244261413f609c14342", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoFiQA2018", + "queries", + revision="4163ba032953d5044a7a6244261413f609c14342", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoFiQA2018", + "qrels", + revision="4163ba032953d5044a7a6244261413f609c14342", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py new file mode 100644 index 0000000000..6c5a0a1b1d --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoHotpotQARetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoHotpotQARetrieval", + description="NanoHotpotQARetrieval is a smaller subset of the " + + "HotpotQA dataset, which is a question answering dataset featuring natural, multi-hop questions, with strong" + + " supervision for supporting facts to enable more explainable question answering systems.", + reference="https://hotpotqa.github.io/", + dataset={ + "path": "zeta-alpha-ai/NanoHotpotQA", + "revision": "d79c0cdda980aba54842756770928035e1b61a51", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Web", "Written"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{yang-etal-2018-hotpotqa, + title = "{H}otpot{QA}: A Dataset for Diverse, Explainable Multi-hop Question Answering", + author = "Yang, Zhilin and + Qi, Peng and + Zhang, Saizheng and + Bengio, Yoshua and + Cohen, William and + Salakhutdinov, Ruslan and + Manning, Christopher D.", + editor = "Riloff, Ellen and + Chiang, David and + Hockenmaier, Julia and + Tsujii, Jun{'}ichi", + booktitle = "Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing", + month = oct # "-" # nov, + year = "2018", + address = "Brussels, Belgium", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/D18-1259", + doi = "10.18653/v1/D18-1259", + pages = "2369--2380", + abstract = "Existing question answering (QA) datasets fail to train QA systems to perform complex reasoning and provide explanations for answers. We introduce HotpotQA, a new dataset with 113k Wikipedia-based question-answer pairs with four key features: (1) the questions require finding and reasoning over multiple supporting documents to answer; (2) the questions are diverse and not constrained to any pre-existing knowledge bases or knowledge schemas; (3) we provide sentence-level supporting facts required for reasoning, allowing QA systems to reason with strong supervision and explain the predictions; (4) we offer a new type of factoid comparison questions to test QA systems{'} ability to extract relevant facts and perform necessary comparison. We show that HotpotQA is challenging for the latest QA systems, and the supporting facts enable models to improve performance and make explainable predictions.", +}""", + prompt={ + "query": "Given a multi-hop question, retrieve documents that can help answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoHotpotQA", + "corpus", + revision="d79c0cdda980aba54842756770928035e1b61a51", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoHotpotQA", + "queries", + revision="d79c0cdda980aba54842756770928035e1b61a51", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoHotpotQA", + "qrels", + revision="d79c0cdda980aba54842756770928035e1b61a51", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py new file mode 100644 index 0000000000..c603e2cc5b --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoMSMARCORetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoMSMARCORetrieval", + description="NanoMSMARCORetrieval is a smaller subset of MS MARCO, a collection of datasets focused on deep learning in search.", + reference="https://microsoft.github.io/msmarco/", + dataset={ + "path": "zeta-alpha-ai/NanoMSMARCO", + "revision": "7b8ff22f2771dc65ac5b439f222eb19a1f56abda", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2016-01-01", "2016-12-31"], + domains=["Web"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{DBLP:journals/corr/NguyenRSGTMD16, + author = {Tri Nguyen and + Mir Rosenberg and + Xia Song and + Jianfeng Gao and + Saurabh Tiwary and + Rangan Majumder and + Li Deng}, + title = {{MS} {MARCO:} {A} Human Generated MAchine Reading COmprehension Dataset}, + journal = {CoRR}, + volume = {abs/1611.09268}, + year = {2016}, + url = {http://arxiv.org/abs/1611.09268}, + archivePrefix = {arXiv}, + eprint = {1611.09268}, + timestamp = {Mon, 13 Aug 2018 16:49:03 +0200}, + biburl = {https://dblp.org/rec/journals/corr/NguyenRSGTMD16.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} +}""", + prompt={ + "query": "Given a web search query, retrieve relevant passages that answer the query" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoMSMARCO", + "corpus", + revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoMSMARCO", + "queries", + revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoMSMARCO", + "qrels", + revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py new file mode 100644 index 0000000000..725c7e889c --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoNFCorpusRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoNFCorpusRetrieval", + description="NanoNFCorpus is a smaller subset of NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval.", + reference="https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/", + dataset={ + "path": "zeta-alpha-ai/NanoNFCorpus", + "revision": "dd542a7efb9ad2136b9e00768b60fca9038f8156", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2016-01-01", "2016-12-31"], + domains=["Medical", "Academic", "Written"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{boteva2016, + author = {Boteva, Vera and Gholipour, Demian and Sokolov, Artem and Riezler, Stefan}, + title = {A Full-Text Learning to Rank Dataset for Medical Information Retrieval}, + journal = {Proceedings of the 38th European Conference on Information Retrieval}, + journal-abbrev = {ECIR}, + year = {2016}, + city = {Padova}, + country = {Italy}, + url = {http://www.cl.uni-heidelberg.de/~riezler/publications/papers/ECIR2016.pdf} +}""", + prompt={ + "query": "Given a question, retrieve relevant documents that best answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoNFCorpus", + "corpus", + revision="dd542a7efb9ad2136b9e00768b60fca9038f8156", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoNFCorpus", + "queries", + revision="dd542a7efb9ad2136b9e00768b60fca9038f8156", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoNFCorpus", + "qrels", + revision="dd542a7efb9ad2136b9e00768b60fca9038f8156", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py new file mode 100644 index 0000000000..538a0881fa --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoNQRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoNQRetrieval", + description="NanoNQ is a smaller subset of a dataset which contains questions from real users, and it requires QA systems to read and comprehend an entire Wikipedia article that may or may not contain the answer to the question.", + reference="https://ai.google.com/research/NaturalQuestions", + dataset={ + "path": "zeta-alpha-ai/NanoNQ", + "revision": "77540146379abf95df8326a3c5bb9eb21c7146c3", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2019-01-01", "2019-12-31"], + domains=["Academic", "Web"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{47761,title = {Natural Questions: a Benchmark for Question Answering Research}, + author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh + and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee + and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le + and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational + Linguistics}}""", + prompt={ + "query": "Given a question, retrieve Wikipedia passages that answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoNQ", + "corpus", + revision="77540146379abf95df8326a3c5bb9eb21c7146c3", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoNQ", + "queries", + revision="77540146379abf95df8326a3c5bb9eb21c7146c3", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoNQ", + "qrels", + revision="77540146379abf95df8326a3c5bb9eb21c7146c3", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py new file mode 100644 index 0000000000..ac527acba2 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoQuoraRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoQuoraRetrieval", + description="NanoQuoraRetrieval is a smaller subset of the " + + "QuoraRetrieval dataset, which is based on questions that are marked as duplicates on the Quora platform. Given a" + + " question, find other (duplicate) questions.", + reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs", + dataset={ + "path": "zeta-alpha-ai/NanoQuoraRetrieval", + "revision": "2ab2d73e6c862026282808b913a34f4136928545", + }, + type="Retrieval", + category="s2s", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2017-01-01", "2017-12-31"], + domains=["Social"], + task_subtypes=["Duplicate Detection"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@misc{quora-question-pairs, + author = {DataCanary, hilfialkaff, Lili Jiang, Meg Risdal, Nikhil Dandekar, tomtung}, + title = {Quora Question Pairs}, + publisher = {Kaggle}, + year = {2017}, + url = {https://kaggle.com/competitions/quora-question-pairs} +}""", + prompt={ + "query": "Given a question, retrieve questions that are semantically equivalent to the given question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoQuoraRetrieval", + "corpus", + revision="2ab2d73e6c862026282808b913a34f4136928545", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoQuoraRetrieval", + "queries", + revision="2ab2d73e6c862026282808b913a34f4136928545", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoQuoraRetrieval", + "qrels", + revision="2ab2d73e6c862026282808b913a34f4136928545", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py new file mode 100644 index 0000000000..f521d693d0 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoSCIDOCSRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoSCIDOCSRetrieval", + description="NanoFiQA2018 is a smaller subset of " + + "SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation" + + " prediction, to document classification and recommendation.", + reference="https://allenai.org/data/scidocs", + dataset={ + "path": "zeta-alpha-ai/NanoSCIDOCS", + "revision": "484eb90549fc3f0b9c42b3551e80ceb999515537", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2020-01-01", "2020-12-31"], + domains=["Academic", "Written", "Non-fiction"], + task_subtypes=[], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{specter2020cohan, + title={SPECTER: Document-level Representation Learning using Citation-informed Transformers}, + author={Arman Cohan and Sergey Feldman and Iz Beltagy and Doug Downey and Daniel S. Weld}, + booktitle={ACL}, + year={2020} +}""", + prompt={ + "query": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoSCIDOCS", + "corpus", + revision="484eb90549fc3f0b9c42b3551e80ceb999515537", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoSCIDOCS", + "queries", + revision="484eb90549fc3f0b9c42b3551e80ceb999515537", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoSCIDOCS", + "qrels", + revision="484eb90549fc3f0b9c42b3551e80ceb999515537", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py new file mode 100644 index 0000000000..a24fa4e102 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoSciFactRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoSciFactRetrieval", + description="NanoSciFact is a smaller subset of SciFact, which verifies scientific claims using evidence from the research literature containing scientific paper abstracts.", + reference="https://github.com/allenai/scifact", + dataset={ + "path": "zeta-alpha-ai/NanoSciFact", + "revision": "309f1d1ae3ae2e092444a8a0c25bed59b82318bc", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Academic", "Medical", "Written"], + task_subtypes=["Claim verification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{specter2020cohan, + title={SPECTER: Document-level Representation Learning using Citation-informed Transformers}, + author={Arman Cohan and Sergey Feldman and Iz Beltagy and Doug Downey and Daniel S. Weld}, + booktitle={ACL}, + year={2020} +}""", + prompt={ + "query": "Given a scientific claim, retrieve documents that support or refute the claim" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoSciFact", + "corpus", + revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoSciFact", + "queries", + revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoSciFact", + "qrels", + revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py new file mode 100644 index 0000000000..b5fccbedf6 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from collections import defaultdict + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoTouche2020Retrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoTouche2020Retrieval", + description="NanoTouche2020 is a smaller subset of Touché Task 1: Argument Retrieval for Controversial Questions.", + reference="https://webis.de/events/touche-20/shared-task-1.html", + dataset={ + "path": "zeta-alpha-ai/NanoTouche2020", + "revision": "0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2020-09-23", "2020-09-23"), + domains=["Academic"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@dataset{potthast_2022_6862281, + author = {Potthast, Martin and + Gienapp, Lukas and + Wachsmuth, Henning and + Hagen, Matthias and + Fröbe, Maik and + Bondarenko, Alexander and + Ajjour, Yamen and + Stein, Benno}, + title = {{Touché20-Argument-Retrieval-for-Controversial- + Questions}}, + month = jul, + year = 2022, + publisher = {Zenodo}, + doi = {10.5281/zenodo.6862281}, + url = {https://doi.org/10.5281/zenodo.6862281} +}""", + prompt={ + "query": "Given a question, retrieve detailed and persuasive arguments that answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoTouche2020", + "corpus", + revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoTouche2020", + "queries", + revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoTouche2020", + "qrels", + revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + relevant_docs = {} + + for split in self.relevant_docs: + relevant_docs[split] = defaultdict(dict) + for query_id, corpus_id in zip( + self.relevant_docs[split]["query-id"], + self.relevant_docs[split]["corpus-id"], + ): + relevant_docs[split][query_id][corpus_id] = 1 + self.relevant_docs = relevant_docs + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NarrativeQARetrieval.py b/mteb/tasks/Retrieval/eng/NarrativeQARetrieval.py index 4f9bb143b6..d973ec45ae 100644 --- a/mteb/tasks/Retrieval/eng/NarrativeQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/NarrativeQARetrieval.py @@ -42,18 +42,6 @@ class NarrativeQARetrieval(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 326753.5323943662, - "average_query_length": 47.730889457232166, - "num_documents": 355, - "num_queries": 10557, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/eng/PiqaRetrieval.py b/mteb/tasks/Retrieval/eng/PiqaRetrieval.py index df2ae359b2..335c252a7e 100644 --- a/mteb/tasks/Retrieval/eng/PiqaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/PiqaRetrieval.py @@ -44,16 +44,5 @@ class PIQA(AbsTaskRetrieval): year={2020} } """, - descriptive_stats={ - "n_samples": {"test": 1838}, - "avg_character_length": { - "test": { - "average_document_length": 99.89012998705756, - "average_query_length": 36.08052230685528, - "num_documents": 35542, - "num_queries": 1838, - "average_relevant_docs_per_query": 1.0, - } - }, - }, + prompt={"query": "Given the following goal, retrieve a possible solution."}, ) diff --git a/mteb/tasks/Retrieval/eng/QuailRetrieval.py b/mteb/tasks/Retrieval/eng/QuailRetrieval.py index 35e27da8a4..221e11cc0f 100644 --- a/mteb/tasks/Retrieval/eng/QuailRetrieval.py +++ b/mteb/tasks/Retrieval/eng/QuailRetrieval.py @@ -44,16 +44,7 @@ class Quail(AbsTaskRetrieval): year={2020} } """, - descriptive_stats={ - "n_samples": {"test": 2720}, - "avg_character_length": { - "test": { - "average_document_length": 27.50788422240522, - "average_query_length": 1957.3632352941177, - "num_documents": 32787, - "num_queries": 2720, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given the following context and question, retrieve the correct answer." }, ) diff --git a/mteb/tasks/Retrieval/eng/QuoraRetrieval.py b/mteb/tasks/Retrieval/eng/QuoraRetrieval.py index 378c1d35f6..73660fb573 100644 --- a/mteb/tasks/Retrieval/eng/QuoraRetrieval.py +++ b/mteb/tasks/Retrieval/eng/QuoraRetrieval.py @@ -22,7 +22,7 @@ class QuoraRetrieval(AbsTaskRetrieval): type="Retrieval", category="s2s", modalities=["text"], - eval_splits=["dev", "test"], + eval_splits=["test"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, @@ -39,24 +39,8 @@ class QuoraRetrieval(AbsTaskRetrieval): year = {2017}, url = {https://kaggle.com/competitions/quora-question-pairs} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "average_document_length": 62.158154708747425, - "average_query_length": 51.5342, - "num_documents": 522931, - "num_queries": 5000, - "average_relevant_docs_per_query": 1.5252, - }, - "test": { - "average_document_length": 62.158154708747425, - "average_query_length": 51.5396, - "num_documents": 522931, - "num_queries": 10000, - "average_relevant_docs_per_query": 1.5675, - }, - }, + prompt={ + "query": "Given a question, retrieve questions that are semantically equivalent to the given question" }, ) @@ -95,16 +79,4 @@ class QuoraRetrievalHardNegatives(AbsTaskRetrieval): year = {2017}, url = {https://kaggle.com/competitions/quora-question-pairs} }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": { - "test": { - "average_document_length": 58.96963812985781, - "average_query_length": 51.228, - "num_documents": 177163, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.641, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/RARbCodeRetrieval.py b/mteb/tasks/Retrieval/eng/RARbCodeRetrieval.py index bf5bb87d14..b42cd4bd71 100644 --- a/mteb/tasks/Retrieval/eng/RARbCodeRetrieval.py +++ b/mteb/tasks/Retrieval/eng/RARbCodeRetrieval.py @@ -52,16 +52,5 @@ class RARbCode(AbsTaskRetrieval): year={2019} } """, - descriptive_stats={ - "n_samples": {"test": 1484}, - "avg_character_length": { - "test": { - "average_document_length": 793.6813076734267, - "average_query_length": 375.7506738544474, - "num_documents": 301482, - "num_queries": 1484, - "average_relevant_docs_per_query": 1.0, - } - }, - }, + prompt={"query": "Retrieve the answer for the following coding problem."}, ) diff --git a/mteb/tasks/Retrieval/eng/RARbMathRetrieval.py b/mteb/tasks/Retrieval/eng/RARbMathRetrieval.py index a41e6960a6..88855a8eaf 100644 --- a/mteb/tasks/Retrieval/eng/RARbMathRetrieval.py +++ b/mteb/tasks/Retrieval/eng/RARbMathRetrieval.py @@ -53,16 +53,5 @@ class RARbMath(AbsTaskRetrieval): year={2023} } """, - descriptive_stats={ - "n_samples": {"test": 6319}, - "avg_character_length": { - "test": { - "average_document_length": 504.0197829347469, - "average_query_length": 210.30732710871973, - "num_documents": 389376, - "num_queries": 6319, - "average_relevant_docs_per_query": 1.0, - } - }, - }, + prompt={"query": "Retrieve the answer for the following math problem."}, ) diff --git a/mteb/tasks/Retrieval/eng/SCIDOCSRetrieval.py b/mteb/tasks/Retrieval/eng/SCIDOCSRetrieval.py index 1bee731a91..231c695d48 100644 --- a/mteb/tasks/Retrieval/eng/SCIDOCSRetrieval.py +++ b/mteb/tasks/Retrieval/eng/SCIDOCSRetrieval.py @@ -36,16 +36,7 @@ class SCIDOCS(AbsTaskRetrieval): booktitle={ACL}, year={2020} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1203.3659819932182, - "average_query_length": 71.632, - "num_documents": 25657, - "num_queries": 1000, - "average_relevant_docs_per_query": 4.928, - } - }, + prompt={ + "query": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper" }, ) diff --git a/mteb/tasks/Retrieval/eng/SciFactRetrieval.py b/mteb/tasks/Retrieval/eng/SciFactRetrieval.py index 5e6f37e363..8caa0c2af5 100644 --- a/mteb/tasks/Retrieval/eng/SciFactRetrieval.py +++ b/mteb/tasks/Retrieval/eng/SciFactRetrieval.py @@ -17,11 +17,11 @@ class SciFact(AbsTaskRetrieval): type="Retrieval", category="s2p", modalities=["text"], - eval_splits=["train", "test"], + eval_splits=["test"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, + domains=["Academic", "Medical", "Written"], task_subtypes=None, license=None, annotations_creators=None, @@ -33,23 +33,7 @@ class SciFact(AbsTaskRetrieval): booktitle={ACL}, year={2020} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "train": { - "average_document_length": 1498.4152035500674, - "average_query_length": 88.58838071693448, - "num_documents": 5183, - "num_queries": 809, - "average_relevant_docs_per_query": 1.1359703337453646, - }, - "test": { - "average_document_length": 1498.4152035500674, - "average_query_length": 90.34666666666666, - "num_documents": 5183, - "num_queries": 300, - "average_relevant_docs_per_query": 1.13, - }, - }, + prompt={ + "query": "Given a scientific claim, retrieve documents that support or refute the claim" }, ) diff --git a/mteb/tasks/Retrieval/eng/SiqaRetrieval.py b/mteb/tasks/Retrieval/eng/SiqaRetrieval.py index a3c30d8021..b8c42f7675 100644 --- a/mteb/tasks/Retrieval/eng/SiqaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/SiqaRetrieval.py @@ -41,16 +41,7 @@ class SIQA(AbsTaskRetrieval): year={2019} } """, - descriptive_stats={ - "n_samples": {"test": 0}, - "avg_character_length": { - "test": { - "average_document_length": 22.967085695044617, - "average_query_length": 127.75383828045035, - "num_documents": 71276, - "num_queries": 1954, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given the following context and question, retrieve the correct answer." }, ) diff --git a/mteb/tasks/Retrieval/eng/SpartQARetrieval.py b/mteb/tasks/Retrieval/eng/SpartQARetrieval.py index 5e81117121..c0262f01cd 100644 --- a/mteb/tasks/Retrieval/eng/SpartQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/SpartQARetrieval.py @@ -41,16 +41,7 @@ class SpartQA(AbsTaskRetrieval): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 0}, - "avg_character_length": { - "test": { - "average_document_length": 50.40829145728643, - "average_query_length": 656.2328881469115, - "num_documents": 1592, - "num_queries": 3594, - "average_relevant_docs_per_query": 1.8786867000556482, - } - }, + prompt={ + "query": "Given the following spatial reasoning question, retrieve the right answer." }, ) diff --git a/mteb/tasks/Retrieval/eng/TRECCOVIDRetrieval.py b/mteb/tasks/Retrieval/eng/TRECCOVIDRetrieval.py index 5ed2bf42fc..00c96c0d04 100644 --- a/mteb/tasks/Retrieval/eng/TRECCOVIDRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TRECCOVIDRetrieval.py @@ -21,7 +21,7 @@ class TRECCOVID(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, + domains=["Medical", "Academic", "Written"], task_subtypes=None, license=None, annotations_creators=None, @@ -35,16 +35,7 @@ class TRECCOVID(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1116.7434221277986, - "average_query_length": 69.24, - "num_documents": 171332, - "num_queries": 50, - "average_relevant_docs_per_query": 493.5, - } - }, + prompt={ + "query": "Given a query on COVID-19, retrieve documents that answer the query" }, ) diff --git a/mteb/tasks/Retrieval/eng/TempReasonL1Retrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL1Retrieval.py index c057f78c6c..392dd1c1b7 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL1Retrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL1Retrieval.py @@ -41,16 +41,7 @@ class TempReasonL1(AbsTaskRetrieval): year={2023} } """, - descriptive_stats={ - "n_samples": {"test": 4000}, - "avg_character_length": { - "test": { - "average_document_length": 8.989843250159948, - "average_query_length": 50.22375, - "num_documents": 12504, - "num_queries": 4000, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given the following question about time, retrieve the correct answer." }, ) diff --git a/mteb/tasks/Retrieval/eng/TempReasonL2ContextRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL2ContextRetrieval.py index 3ed662a548..924c1621f5 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL2ContextRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL2ContextRetrieval.py @@ -41,16 +41,7 @@ class TempReasonL2Context(AbsTaskRetrieval): year={2023} } """, - descriptive_stats={ - "n_samples": {"test": 0}, - "avg_character_length": { - "test": { - "average_document_length": 19.823525685690758, - "average_query_length": 11919.25792106726, - "num_documents": 15787, - "num_queries": 5397, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given the following question, facts and contexts, retrieve the correct answer." }, ) diff --git a/mteb/tasks/Retrieval/eng/TempReasonL2FactRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL2FactRetrieval.py index ce62c02f81..4e1fc53a29 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL2FactRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL2FactRetrieval.py @@ -41,16 +41,7 @@ class TempReasonL2Fact(AbsTaskRetrieval): year={2023} } """, - descriptive_stats={ - "n_samples": {"test": 5397}, - "avg_character_length": { - "test": { - "average_document_length": 19.823525685690758, - "average_query_length": 830.7268853066519, - "num_documents": 15787, - "num_queries": 5397, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given the following question and facts, retrieve the correct answer." }, ) diff --git a/mteb/tasks/Retrieval/eng/TempReasonL2PureRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL2PureRetrieval.py index 8b775752c1..b69989af03 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL2PureRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL2PureRetrieval.py @@ -41,16 +41,5 @@ class TempReasonL2Pure(AbsTaskRetrieval): year={2023} } """, - descriptive_stats={ - "n_samples": {"test": 5397}, - "avg_character_length": { - "test": { - "average_document_length": 19.823525685690758, - "average_query_length": 55.94089308875301, - "num_documents": 15787, - "num_queries": 5397, - "average_relevant_docs_per_query": 1.0, - } - }, - }, + prompt={"query": "Given the following question, retrieve the correct answer."}, ) diff --git a/mteb/tasks/Retrieval/eng/TempReasonL3ContextRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL3ContextRetrieval.py index e1a43b3d92..65f70ab13a 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL3ContextRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL3ContextRetrieval.py @@ -41,16 +41,7 @@ class TempReasonL3Context(AbsTaskRetrieval): year={2023} } """, - descriptive_stats={ - "n_samples": {"test": 4426}, - "avg_character_length": { - "test": { - "average_document_length": 19.80534984678243, - "average_query_length": 13424.633077270673, - "num_documents": 15664, - "num_queries": 4426, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given the following question, facts and contexts, retrieve the correct answer." }, ) diff --git a/mteb/tasks/Retrieval/eng/TempReasonL3FactRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL3FactRetrieval.py index bd9d017e53..65db6a70ba 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL3FactRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL3FactRetrieval.py @@ -41,16 +41,7 @@ class TempReasonL3Fact(AbsTaskRetrieval): year={2023} } """, - descriptive_stats={ - "n_samples": {"test": 4426}, - "avg_character_length": { - "test": { - "average_document_length": 19.80534984678243, - "average_query_length": 896.0754631721645, - "num_documents": 15664, - "num_queries": 4426, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given the following question and facts, retrieve the correct answer." }, ) diff --git a/mteb/tasks/Retrieval/eng/TempReasonL3PureRetrieval.py b/mteb/tasks/Retrieval/eng/TempReasonL3PureRetrieval.py index 162a9988aa..32738f7180 100644 --- a/mteb/tasks/Retrieval/eng/TempReasonL3PureRetrieval.py +++ b/mteb/tasks/Retrieval/eng/TempReasonL3PureRetrieval.py @@ -41,16 +41,5 @@ class TempReasonL3Pure(AbsTaskRetrieval): year={2023} } """, - descriptive_stats={ - "n_samples": {"test": 4426}, - "avg_character_length": { - "test": { - "average_document_length": 19.80534984678243, - "average_query_length": 74.44012652507908, - "num_documents": 15664, - "num_queries": 4426, - "average_relevant_docs_per_query": 1.0, - } - }, - }, + prompt={"query": "Given the following question, retrieve the correct answer."}, ) diff --git a/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py b/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py index 415bc3045b..23c916f393 100644 --- a/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/TopiOCQARetrieval.py @@ -50,18 +50,6 @@ class TopiOCQARetrieval(AbsTaskRetrieval): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"dev": 2514}, - "avg_character_length": { - "validation": { - "average_document_length": 478.8968086416064, - "average_query_length": 12.579952267303103, - "num_documents": 25700592, - "num_queries": 2514, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) # TODO: Will be removed if curated and added to mteb HF @@ -145,16 +133,4 @@ class TopiOCQARetrievalHardNegatives(AbsTaskRetrieval): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": { - "validation": { - "average_document_length": 538.7586536643946, - "average_query_length": 12.85, - "num_documents": 89933, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/eng/Touche2020Retrieval.py b/mteb/tasks/Retrieval/eng/Touche2020Retrieval.py index 01b955b19d..afff7de60c 100644 --- a/mteb/tasks/Retrieval/eng/Touche2020Retrieval.py +++ b/mteb/tasks/Retrieval/eng/Touche2020Retrieval.py @@ -45,17 +45,8 @@ class Touche2020(AbsTaskRetrieval): doi = {10.5281/zenodo.6862281}, url = {https://doi.org/10.5281/zenodo.6862281} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1719.3347658445412, - "average_query_length": 43.42857142857143, - "num_documents": 382545, - "num_queries": 49, - "average_relevant_docs_per_query": 19.020408163265305, - } - }, + prompt={ + "query": "Given a question, retrieve detailed and persuasive arguments that answer the question" }, ) @@ -89,13 +80,4 @@ class Touche2020v3Retrieval(AbsTaskRetrieval): year = 2024, address_ = "Washington, D.C." }""", - descriptive_stats={ - "test": { - "average_document_length": 2096.391812518931, - "average_query_length": 43.42857142857143, - "num_documents": 303732, - "num_queries": 49, - "average_relevant_docs_per_query": 34.93877551020408, - } - }, ) diff --git a/mteb/tasks/Retrieval/eng/WinoGrandeRetrieval.py b/mteb/tasks/Retrieval/eng/WinoGrandeRetrieval.py index 13cfe8f727..01b5f2d1cc 100644 --- a/mteb/tasks/Retrieval/eng/WinoGrandeRetrieval.py +++ b/mteb/tasks/Retrieval/eng/WinoGrandeRetrieval.py @@ -45,16 +45,7 @@ class WinoGrande(AbsTaskRetrieval): publisher={ACM New York, NY, USA} } """, - descriptive_stats={ - "n_samples": {"test": 0}, - "avg_character_length": { - "test": { - "average_document_length": 7.68243375858685, - "average_query_length": 111.78216258879242, - "num_documents": 5095, - "num_queries": 1267, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given the following sentence, retrieve an appropriate answer to fill in the missing underscored part." }, ) diff --git a/mteb/tasks/Retrieval/est/estqa.py b/mteb/tasks/Retrieval/est/estqa.py index 28efd3a71a..b8eebb61d9 100644 --- a/mteb/tasks/Retrieval/est/estqa.py +++ b/mteb/tasks/Retrieval/est/estqa.py @@ -40,16 +40,4 @@ class EstQA(AbsTaskRetrieval): year = 2021 } """, - descriptive_stats={ - "n_samples": {"test": 603}, - "avg_character_length": { - "test": { - "average_document_length": 785.595041322314, - "average_query_length": 55.32006633499171, - "num_documents": 121, - "num_queries": 603, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/fra/AlloprofRetrieval.py b/mteb/tasks/Retrieval/fra/AlloprofRetrieval.py index 71d7349faa..ada02b511b 100644 --- a/mteb/tasks/Retrieval/fra/AlloprofRetrieval.py +++ b/mteb/tasks/Retrieval/fra/AlloprofRetrieval.py @@ -40,18 +40,6 @@ class AlloprofRetrieval(AbsTaskRetrieval): year = {2023}, copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International} }""", - descriptive_stats={ - "n_samples": {"train": 2048}, - "avg_character_length": { - "test": { - "average_document_length": 3505.705399061033, - "average_query_length": 170.71286701208982, - "num_documents": 2556, - "num_queries": 2316, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/fra/BSARDRetrieval.py b/mteb/tasks/Retrieval/fra/BSARDRetrieval.py index f853713fde..93509c51fc 100644 --- a/mteb/tasks/Retrieval/fra/BSARDRetrieval.py +++ b/mteb/tasks/Retrieval/fra/BSARDRetrieval.py @@ -44,18 +44,6 @@ class BSARDRetrieval(AbsTaskRetrieval): doi = {10.18653/v1/2022.acl-long.468}, pages = {6789–6803}, }""", - descriptive_stats={ - "n_samples": {"test": 222}, - "avg_character_length": { - "test": { - "average_document_length": 880.2900631820793, - "average_query_length": 144.77027027027026, - "num_documents": 22633, - "num_queries": 222, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/fra/FQuADRetrieval.py b/mteb/tasks/Retrieval/fra/FQuADRetrieval.py index 8bd6ef1e55..20a54b8232 100644 --- a/mteb/tasks/Retrieval/fra/FQuADRetrieval.py +++ b/mteb/tasks/Retrieval/fra/FQuADRetrieval.py @@ -50,25 +50,6 @@ class FQuADRetrieval(AbsTaskRetrieval): doi = "10.18653/v1/2020.findings-emnlp.107", pages = "1193--1208", }""", - descriptive_stats={ - "n_samples": {"test": 400, "validation": 100}, - "avg_character_length": { - "test": { - "average_document_length": 896.3308550185874, - "average_query_length": 58.52, - "num_documents": 269, - "num_queries": 400, - "average_relevant_docs_per_query": 1.0, - }, - "validation": { - "average_document_length": 895.1340206185567, - "average_query_length": 54.13, - "num_documents": 97, - "num_queries": 100, - "average_relevant_docs_per_query": 1.0, - }, - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/fra/SyntecRetrieval.py b/mteb/tasks/Retrieval/fra/SyntecRetrieval.py index 4240de38f3..4e5dd52c51 100644 --- a/mteb/tasks/Retrieval/fra/SyntecRetrieval.py +++ b/mteb/tasks/Retrieval/fra/SyntecRetrieval.py @@ -39,18 +39,6 @@ class SyntecRetrieval(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"test": 90}, - "avg_character_length": { - "test": { - "average_document_length": 1224.2666666666667, - "average_query_length": 72.82, - "num_documents": 90, - "num_queries": 100, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/hun/HunSum2.py b/mteb/tasks/Retrieval/hun/HunSum2.py index 1ef5808507..70bd9c0359 100644 --- a/mteb/tasks/Retrieval/hun/HunSum2.py +++ b/mteb/tasks/Retrieval/hun/HunSum2.py @@ -44,20 +44,6 @@ class HunSum2AbstractiveRetrieval(AbsTaskRetrieval): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": { - "test": 1998, - }, - "avg_character_length": { - "test": { - "average_document_length": 2511.0315315315315, - "average_query_length": 201.2112112112112, - "num_documents": 1998, - "num_queries": 1998, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/jpn/JaGovFaqsRetrieval.py b/mteb/tasks/Retrieval/jpn/JaGovFaqsRetrieval.py index da4db59d07..3960ab6f19 100644 --- a/mteb/tasks/Retrieval/jpn/JaGovFaqsRetrieval.py +++ b/mteb/tasks/Retrieval/jpn/JaGovFaqsRetrieval.py @@ -35,18 +35,6 @@ class JaGovFaqsRetrieval(AbsTaskRetrieval): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: _MAX_EVAL_SIZE}, - "avg_character_length": { - "test": { - "average_document_length": 210.02601561814512, - "average_query_length": 59.48193359375, - "num_documents": 22794, - "num_queries": 2048, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/jpn/JaQuADRetrieval.py b/mteb/tasks/Retrieval/jpn/JaQuADRetrieval.py index b47cecaa98..07fb165632 100644 --- a/mteb/tasks/Retrieval/jpn/JaQuADRetrieval.py +++ b/mteb/tasks/Retrieval/jpn/JaQuADRetrieval.py @@ -37,18 +37,6 @@ class JaQuADRetrieval(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"validation": 2048}, - "avg_character_length": { - "validation": { - "average_document_length": 155.80922362309224, - "average_query_length": 30.826171875, - "num_documents": 3014, - "num_queries": 2048, - "average_relevant_docs_per_query": 2.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/jpn/JaqketRetrieval.py b/mteb/tasks/Retrieval/jpn/JaqketRetrieval.py index 0af7d06772..bff152e239 100644 --- a/mteb/tasks/Retrieval/jpn/JaqketRetrieval.py +++ b/mteb/tasks/Retrieval/jpn/JaqketRetrieval.py @@ -34,13 +34,4 @@ class JaqketRetrieval(AbsTaskRetrieval): url = "https://www.anlp.jp/proceedings/annual_meeting/2020/pdf_dir/P2-24.pdf" note= "in Japanese" }""", - descriptive_stats={ - "test": { - "average_document_length": 3747.995228882333, - "average_query_length": 50.70611835506519, - "num_documents": 114229, - "num_queries": 997, - "average_relevant_docs_per_query": 1.0, - } - }, ) diff --git a/mteb/tasks/Retrieval/jpn/NLPJournalAbsIntroRetrieval.py b/mteb/tasks/Retrieval/jpn/NLPJournalAbsIntroRetrieval.py index 47a284cb23..d7b0a60adf 100644 --- a/mteb/tasks/Retrieval/jpn/NLPJournalAbsIntroRetrieval.py +++ b/mteb/tasks/Retrieval/jpn/NLPJournalAbsIntroRetrieval.py @@ -32,18 +32,6 @@ class NLPJournalAbsIntroRetrieval(AbsTaskRetrieval): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 404}, - "avg_character_length": { - "test": { - "average_document_length": 2052.8611111111113, - "average_query_length": 439.2772277227723, - "num_documents": 504, - "num_queries": 404, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/jpn/NLPJournalTitleAbsRetrieval.py b/mteb/tasks/Retrieval/jpn/NLPJournalTitleAbsRetrieval.py index 6a9ba43cd5..0a7be8965b 100644 --- a/mteb/tasks/Retrieval/jpn/NLPJournalTitleAbsRetrieval.py +++ b/mteb/tasks/Retrieval/jpn/NLPJournalTitleAbsRetrieval.py @@ -32,18 +32,6 @@ class NLPJournalTitleAbsRetrieval(AbsTaskRetrieval): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 404}, - "avg_character_length": { - "test": { - "average_document_length": 441.6746031746032, - "average_query_length": 27.60891089108911, - "num_documents": 504, - "num_queries": 404, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/jpn/NLPJournalTitleIntroRetrieval.py b/mteb/tasks/Retrieval/jpn/NLPJournalTitleIntroRetrieval.py index 408dc013c2..dc4507adca 100644 --- a/mteb/tasks/Retrieval/jpn/NLPJournalTitleIntroRetrieval.py +++ b/mteb/tasks/Retrieval/jpn/NLPJournalTitleIntroRetrieval.py @@ -32,18 +32,6 @@ class NLPJournalTitleIntroRetrieval(AbsTaskRetrieval): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 404}, - "avg_character_length": { - "test": { - "average_document_length": 2052.8611111111113, - "average_query_length": 27.60891089108911, - "num_documents": 504, - "num_queries": 404, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/kat/GeorgianFAQRetrieval.py b/mteb/tasks/Retrieval/kat/GeorgianFAQRetrieval.py index 82b0392faa..f870e999c9 100644 --- a/mteb/tasks/Retrieval/kat/GeorgianFAQRetrieval.py +++ b/mteb/tasks/Retrieval/kat/GeorgianFAQRetrieval.py @@ -34,18 +34,6 @@ class GeorgianFAQRetrieval(AbsTaskRetrieval): annotations_creators="derived", dialect=[], bibtex_citation="", - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 2566}, - "avg_character_length": { - "test": { - "average_document_length": 511.24668745128605, - "average_query_length": 61.69551656920078, - "num_documents": 2566, - "num_queries": 2565, - "average_relevant_docs_per_query": 1.0003898635477584, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/kor/AutoRAGRetrieval.py b/mteb/tasks/Retrieval/kor/AutoRAGRetrieval.py new file mode 100644 index 0000000000..4a24e04e9c --- /dev/null +++ b/mteb/tasks/Retrieval/kor/AutoRAGRetrieval.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval + + +class AutoRAGRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="AutoRAGRetrieval", + description="This dataset enables the evaluation of Korean RAG performance across various domains—finance, public sector, healthcare, legal, and commerce—by providing publicly accessible documents, questions, and answers.", + reference="https://arxiv.org/abs/2410.20878", + dataset={ + "path": "yjoonjang/markers_bm", + "revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", + }, + type="Retrieval", + prompt="Retrieve text based on user query.", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["kor-Hang"], + main_score="ndcg_at_10", + date=("2024-08-03", "2024-08-03"), + domains=["Government", "Medical", "Legal", "Social"], + task_subtypes=["Article retrieval"], + license="mit", + annotations_creators="human-annotated", + dialect=[], + sample_creation="created", + bibtex_citation="""@misc{kim2024autoragautomatedframeworkoptimization, + title={AutoRAG: Automated Framework for optimization of Retrieval Augmented Generation Pipeline}, + author={Dongkyu Kim and Byoungwook Kim and Donggeon Han and Matouš Eibich}, + year={2024}, + eprint={2410.20878}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2410.20878}, +}""", + ) diff --git a/mteb/tasks/Retrieval/kor/KoStrategyQA.py b/mteb/tasks/Retrieval/kor/KoStrategyQA.py index 4219692689..ce64da5432 100644 --- a/mteb/tasks/Retrieval/kor/KoStrategyQA.py +++ b/mteb/tasks/Retrieval/kor/KoStrategyQA.py @@ -33,16 +33,4 @@ class KoStrategyQA(AbsTaskRetrieval): journal = {Transactions of the Association for Computational Linguistics (TACL)}, year = {2021}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "average_document_length": 319.25953950924225, - "average_query_length": 22.75337837837838, - "num_documents": 9251, - "num_queries": 592, - "average_relevant_docs_per_query": 1.9341216216216217, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py b/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py index 8066b4e2d0..2a45205cfd 100644 --- a/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py @@ -202,2650 +202,6 @@ class BelebeleRetrieval(MultilingualTask, AbsTaskRetrieval): task_subtypes=["Question answering"], annotations_creators="expert-annotated", dialect=[], - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 103500}, - "test": { - "average_document_length": 487.3975028339728, - "average_query_length": 74.49551684802204, - "num_documents": 183488, - "num_queries": 338378, - "average_relevant_docs_per_query": 1.0, - "hf_subset_descriptive_stats": { - "acm_Arab-acm_Arab": { - "average_document_length": 416.4733606557377, - "average_query_length": 55.84, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "acm_Arab-eng_Latn": { - "average_document_length": 416.4733606557377, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-acm_Arab": { - "average_document_length": 475.51024590163934, - "average_query_length": 55.84, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "afr_Latn-afr_Latn": { - "average_document_length": 503.6659836065574, - "average_query_length": 78.04555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "afr_Latn-eng_Latn": { - "average_document_length": 503.6659836065574, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-afr_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 78.04555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "als_Latn-als_Latn": { - "average_document_length": 534.016393442623, - "average_query_length": 76.13555555555556, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "als_Latn-eng_Latn": { - "average_document_length": 534.016393442623, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-als_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 76.13555555555556, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "amh_Ethi-amh_Ethi": { - "average_document_length": 319.8688524590164, - "average_query_length": 49.16111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "amh_Ethi-eng_Latn": { - "average_document_length": 319.8688524590164, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-amh_Ethi": { - "average_document_length": 475.51024590163934, - "average_query_length": 49.16111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "apc_Arab-apc_Arab": { - "average_document_length": 393.0553278688525, - "average_query_length": 55.85777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "apc_Arab-eng_Latn": { - "average_document_length": 393.0553278688525, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-apc_Arab": { - "average_document_length": 475.51024590163934, - "average_query_length": 55.85777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "arb_Arab-arb_Arab": { - "average_document_length": 421.96311475409834, - "average_query_length": 58.55, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "arb_Arab-eng_Latn": { - "average_document_length": 421.96311475409834, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-arb_Arab": { - "average_document_length": 475.51024590163934, - "average_query_length": 58.55, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "arb_Latn-arb_Latn": { - "average_document_length": 555.6188524590164, - "average_query_length": 67.02444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "arb_Latn-eng_Latn": { - "average_document_length": 555.6188524590164, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-arb_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 67.02444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ars_Arab-ars_Arab": { - "average_document_length": 422.5553278688525, - "average_query_length": 56.43222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ars_Arab-eng_Latn": { - "average_document_length": 422.5553278688525, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ars_Arab": { - "average_document_length": 475.51024590163934, - "average_query_length": 56.43222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ary_Arab-ary_Arab": { - "average_document_length": 411.1475409836066, - "average_query_length": 66.01893095768374, - "num_documents": 488, - "num_queries": 898, - "average_relevant_docs_per_query": 1.0, - }, - "ary_Arab-eng_Latn": { - "average_document_length": 411.1475409836066, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ary_Arab": { - "average_document_length": 475.51024590163934, - "average_query_length": 66.01893095768374, - "num_documents": 488, - "num_queries": 898, - "average_relevant_docs_per_query": 1.0, - }, - "arz_Arab-arz_Arab": { - "average_document_length": 412.05122950819674, - "average_query_length": 57.14111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "arz_Arab-eng_Latn": { - "average_document_length": 412.05122950819674, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-arz_Arab": { - "average_document_length": 475.51024590163934, - "average_query_length": 57.14111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "asm_Beng-asm_Beng": { - "average_document_length": 458.5983606557377, - "average_query_length": 68.26, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "asm_Beng-eng_Latn": { - "average_document_length": 458.5983606557377, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-asm_Beng": { - "average_document_length": 475.51024590163934, - "average_query_length": 68.26, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "azj_Latn-azj_Latn": { - "average_document_length": 519.6127049180328, - "average_query_length": 73.51222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "azj_Latn-eng_Latn": { - "average_document_length": 519.6127049180328, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-azj_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 73.51222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "bam_Latn-bam_Latn": { - "average_document_length": 457.3114754098361, - "average_query_length": 72.34222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "bam_Latn-eng_Latn": { - "average_document_length": 457.3114754098361, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-bam_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 72.34222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ben_Beng-ben_Beng": { - "average_document_length": 467.7745901639344, - "average_query_length": 69.48444444444445, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ben_Beng-eng_Latn": { - "average_document_length": 467.7745901639344, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ben_Beng": { - "average_document_length": 475.51024590163934, - "average_query_length": 69.48444444444445, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ben_Latn-ben_Latn": { - "average_document_length": 522.8934426229508, - "average_query_length": 74.78777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ben_Latn-eng_Latn": { - "average_document_length": 522.8934426229508, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ben_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 74.78777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "bod_Tibt-bod_Tibt": { - "average_document_length": 533.3872950819672, - "average_query_length": 86.90222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "bod_Tibt-eng_Latn": { - "average_document_length": 533.3872950819672, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-bod_Tibt": { - "average_document_length": 475.51024590163934, - "average_query_length": 86.90222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "bul_Cyrl-bul_Cyrl": { - "average_document_length": 496.97131147540983, - "average_query_length": 72.89, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "bul_Cyrl-eng_Latn": { - "average_document_length": 496.97131147540983, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-bul_Cyrl": { - "average_document_length": 475.51024590163934, - "average_query_length": 72.89, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "cat_Latn-cat_Latn": { - "average_document_length": 525.4467213114754, - "average_query_length": 75.40666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "cat_Latn-eng_Latn": { - "average_document_length": 525.4467213114754, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-cat_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 75.40666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ceb_Latn-ceb_Latn": { - "average_document_length": 570.8483606557377, - "average_query_length": 81.19666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ceb_Latn-eng_Latn": { - "average_document_length": 570.8483606557377, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ceb_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 81.19666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ces_Latn-ces_Latn": { - "average_document_length": 461.0061475409836, - "average_query_length": 67.73333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ces_Latn-eng_Latn": { - "average_document_length": 461.0061475409836, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ces_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 67.73333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ckb_Arab-ckb_Arab": { - "average_document_length": 462.98770491803276, - "average_query_length": 71.04555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ckb_Arab-eng_Latn": { - "average_document_length": 462.98770491803276, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ckb_Arab": { - "average_document_length": 475.51024590163934, - "average_query_length": 71.04555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "dan_Latn-dan_Latn": { - "average_document_length": 489.4856557377049, - "average_query_length": 72.96888888888888, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "dan_Latn-eng_Latn": { - "average_document_length": 489.4856557377049, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-dan_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 72.96888888888888, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "deu_Latn-deu_Latn": { - "average_document_length": 555.1659836065573, - "average_query_length": 75.32444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "deu_Latn-eng_Latn": { - "average_document_length": 555.1659836065573, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-deu_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 75.32444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ell_Grek-ell_Grek": { - "average_document_length": 568.3872950819672, - "average_query_length": 86.92666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ell_Grek-eng_Latn": { - "average_document_length": 568.3872950819672, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ell_Grek": { - "average_document_length": 475.51024590163934, - "average_query_length": 86.92666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-eng_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "est_Latn-est_Latn": { - "average_document_length": 467.1475409836066, - "average_query_length": 67.55888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "est_Latn-eng_Latn": { - "average_document_length": 467.1475409836066, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-est_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 67.55888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eus_Latn-eus_Latn": { - "average_document_length": 506.19262295081967, - "average_query_length": 74.44777777777777, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eus_Latn-eng_Latn": { - "average_document_length": 506.19262295081967, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-eus_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 74.44777777777777, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "fin_Latn-fin_Latn": { - "average_document_length": 507.5, - "average_query_length": 72.50888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "fin_Latn-eng_Latn": { - "average_document_length": 507.5, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-fin_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 72.50888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "fra_Latn-fra_Latn": { - "average_document_length": 564.8401639344262, - "average_query_length": 90.54222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "fra_Latn-eng_Latn": { - "average_document_length": 564.8401639344262, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-fra_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 90.54222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "fuv_Latn-fuv_Latn": { - "average_document_length": 443.4733606557377, - "average_query_length": 58.42111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "fuv_Latn-eng_Latn": { - "average_document_length": 443.4733606557377, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-fuv_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 58.42111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "gaz_Latn-gaz_Latn": { - "average_document_length": 563.5389344262295, - "average_query_length": 85.93222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "gaz_Latn-eng_Latn": { - "average_document_length": 563.5389344262295, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-gaz_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 85.93222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "grn_Latn-grn_Latn": { - "average_document_length": 480.3299180327869, - "average_query_length": 75.10666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "grn_Latn-eng_Latn": { - "average_document_length": 480.3299180327869, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-grn_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 75.10666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "guj_Gujr-guj_Gujr": { - "average_document_length": 458.1885245901639, - "average_query_length": 62.25666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "guj_Gujr-eng_Latn": { - "average_document_length": 458.1885245901639, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-guj_Gujr": { - "average_document_length": 475.51024590163934, - "average_query_length": 62.25666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hat_Latn-hat_Latn": { - "average_document_length": 438.6700819672131, - "average_query_length": 70.64666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hat_Latn-eng_Latn": { - "average_document_length": 438.6700819672131, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-hat_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 70.64666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hau_Latn-hau_Latn": { - "average_document_length": 507.24590163934425, - "average_query_length": 85.8488888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hau_Latn-eng_Latn": { - "average_document_length": 507.24590163934425, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-hau_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 85.8488888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "heb_Hebr-heb_Hebr": { - "average_document_length": 371.36270491803276, - "average_query_length": 55.135555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "heb_Hebr-eng_Latn": { - "average_document_length": 371.36270491803276, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-heb_Hebr": { - "average_document_length": 475.51024590163934, - "average_query_length": 55.135555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hin_Deva-hin_Deva": { - "average_document_length": 473.55737704918033, - "average_query_length": 72.61777777777777, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hin_Deva-eng_Latn": { - "average_document_length": 473.55737704918033, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-hin_Deva": { - "average_document_length": 475.51024590163934, - "average_query_length": 72.61777777777777, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hin_Latn-hin_Latn": { - "average_document_length": 541.7315573770492, - "average_query_length": 74.81222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hin_Latn-eng_Latn": { - "average_document_length": 541.7315573770492, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-hin_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 74.81222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hrv_Latn-hrv_Latn": { - "average_document_length": 469.202868852459, - "average_query_length": 68.83555555555556, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hrv_Latn-eng_Latn": { - "average_document_length": 469.202868852459, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-hrv_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 68.83555555555556, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hun_Latn-hun_Latn": { - "average_document_length": 501.1946721311475, - "average_query_length": 74.40555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hun_Latn-eng_Latn": { - "average_document_length": 501.1946721311475, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-hun_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 74.40555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hye_Armn-hye_Armn": { - "average_document_length": 527.5102459016393, - "average_query_length": 75.42555555555556, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hye_Armn-eng_Latn": { - "average_document_length": 527.5102459016393, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-hye_Armn": { - "average_document_length": 475.51024590163934, - "average_query_length": 75.42555555555556, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ibo_Latn-ibo_Latn": { - "average_document_length": 482.3483606557377, - "average_query_length": 72.51501668520578, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0011123470522802, - }, - "ibo_Latn-eng_Latn": { - "average_document_length": 482.3483606557377, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ibo_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 72.51501668520578, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0011123470522802, - }, - "ilo_Latn-ilo_Latn": { - "average_document_length": 574.6987704918033, - "average_query_length": 85.7611111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ilo_Latn-eng_Latn": { - "average_document_length": 574.6987704918033, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ilo_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 85.7611111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ind_Latn-ind_Latn": { - "average_document_length": 516.0573770491803, - "average_query_length": 82.10555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ind_Latn-eng_Latn": { - "average_document_length": 516.0573770491803, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ind_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 82.10555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "isl_Latn-isl_Latn": { - "average_document_length": 470.73975409836066, - "average_query_length": 77.27333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "isl_Latn-eng_Latn": { - "average_document_length": 470.73975409836066, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-isl_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 77.27333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ita_Latn-ita_Latn": { - "average_document_length": 560.9344262295082, - "average_query_length": 83.49777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ita_Latn-eng_Latn": { - "average_document_length": 560.9344262295082, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ita_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 83.49777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "jav_Latn-jav_Latn": { - "average_document_length": 494.1803278688525, - "average_query_length": 78.60666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "jav_Latn-eng_Latn": { - "average_document_length": 494.1803278688525, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-jav_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 78.60666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "jpn_Jpan-jpn_Jpan": { - "average_document_length": 207.74795081967213, - "average_query_length": 35.79, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "jpn_Jpan-eng_Latn": { - "average_document_length": 207.74795081967213, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-jpn_Jpan": { - "average_document_length": 475.51024590163934, - "average_query_length": 35.79, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kac_Latn-kac_Latn": { - "average_document_length": 605.2889344262295, - "average_query_length": 98.64182424916574, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0, - }, - "kac_Latn-eng_Latn": { - "average_document_length": 605.2889344262295, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-kac_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 98.64182424916574, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0, - }, - "kan_Knda-kan_Knda": { - "average_document_length": 498.9077868852459, - "average_query_length": 72.13666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kan_Knda-eng_Latn": { - "average_document_length": 498.9077868852459, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-kan_Knda": { - "average_document_length": 475.51024590163934, - "average_query_length": 72.13666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kat_Geor-kat_Geor": { - "average_document_length": 521.7766393442623, - "average_query_length": 74.81444444444445, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kat_Geor-eng_Latn": { - "average_document_length": 521.7766393442623, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-kat_Geor": { - "average_document_length": 475.51024590163934, - "average_query_length": 74.81444444444445, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kaz_Cyrl-kaz_Cyrl": { - "average_document_length": 488.2110655737705, - "average_query_length": 70.75666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kaz_Cyrl-eng_Latn": { - "average_document_length": 488.2110655737705, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-kaz_Cyrl": { - "average_document_length": 475.51024590163934, - "average_query_length": 70.75666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kea_Latn-kea_Latn": { - "average_document_length": 471.5594262295082, - "average_query_length": 75.94111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kea_Latn-eng_Latn": { - "average_document_length": 471.5594262295082, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-kea_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 75.94111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "khk_Cyrl-khk_Cyrl": { - "average_document_length": 496.655737704918, - "average_query_length": 73.33444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "khk_Cyrl-eng_Latn": { - "average_document_length": 496.655737704918, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-khk_Cyrl": { - "average_document_length": 475.51024590163934, - "average_query_length": 73.33444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "khm_Khmr-khm_Khmr": { - "average_document_length": 562.4139344262295, - "average_query_length": 75.74888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "khm_Khmr-eng_Latn": { - "average_document_length": 562.4139344262295, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-khm_Khmr": { - "average_document_length": 475.51024590163934, - "average_query_length": 75.74888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kin_Latn-kin_Latn": { - "average_document_length": 529.2520491803278, - "average_query_length": 79.89655172413794, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0011123470522802, - }, - "kin_Latn-eng_Latn": { - "average_document_length": 529.2520491803278, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-kin_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 79.89655172413794, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0011123470522802, - }, - "kir_Cyrl-kir_Cyrl": { - "average_document_length": 487.80737704918033, - "average_query_length": 74.42333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kir_Cyrl-eng_Latn": { - "average_document_length": 487.80737704918033, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-kir_Cyrl": { - "average_document_length": 475.51024590163934, - "average_query_length": 74.42333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kor_Hang-kor_Hang": { - "average_document_length": 241.32991803278688, - "average_query_length": 35.257777777777775, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "kor_Hang-eng_Latn": { - "average_document_length": 241.32991803278688, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-kor_Hang": { - "average_document_length": 475.51024590163934, - "average_query_length": 35.257777777777775, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "lao_Laoo-lao_Laoo": { - "average_document_length": 471.6495901639344, - "average_query_length": 63.31333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "lao_Laoo-eng_Latn": { - "average_document_length": 471.6495901639344, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-lao_Laoo": { - "average_document_length": 475.51024590163934, - "average_query_length": 63.31333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "lin_Latn-lin_Latn": { - "average_document_length": 512.9016393442623, - "average_query_length": 81.56681514476615, - "num_documents": 488, - "num_queries": 898, - "average_relevant_docs_per_query": 1.0022271714922049, - }, - "lin_Latn-eng_Latn": { - "average_document_length": 512.9016393442623, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-lin_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 81.56681514476615, - "num_documents": 488, - "num_queries": 898, - "average_relevant_docs_per_query": 1.0022271714922049, - }, - "lit_Latn-lit_Latn": { - "average_document_length": 474.0553278688525, - "average_query_length": 68.69888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "lit_Latn-eng_Latn": { - "average_document_length": 474.0553278688525, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-lit_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 68.69888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "lug_Latn-lug_Latn": { - "average_document_length": 485.73975409836066, - "average_query_length": 78.52057842046719, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0011123470522802, - }, - "lug_Latn-eng_Latn": { - "average_document_length": 485.73975409836066, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-lug_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 78.52057842046719, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0011123470522802, - }, - "luo_Latn-luo_Latn": { - "average_document_length": 497.53688524590166, - "average_query_length": 73.14333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "luo_Latn-eng_Latn": { - "average_document_length": 497.53688524590166, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-luo_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 73.14333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "lvs_Latn-lvs_Latn": { - "average_document_length": 487.21311475409834, - "average_query_length": 69.97888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "lvs_Latn-eng_Latn": { - "average_document_length": 487.21311475409834, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-lvs_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 69.97888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "mal_Mlym-mal_Mlym": { - "average_document_length": 539.2827868852459, - "average_query_length": 80.69222222222223, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "mal_Mlym-eng_Latn": { - "average_document_length": 539.2827868852459, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-mal_Mlym": { - "average_document_length": 475.51024590163934, - "average_query_length": 80.69222222222223, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "mar_Deva-mar_Deva": { - "average_document_length": 478.67418032786884, - "average_query_length": 68.62625139043382, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0011123470522802, - }, - "mar_Deva-eng_Latn": { - "average_document_length": 478.67418032786884, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-mar_Deva": { - "average_document_length": 475.51024590163934, - "average_query_length": 68.62625139043382, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0011123470522802, - }, - "mkd_Cyrl-mkd_Cyrl": { - "average_document_length": 495.77868852459017, - "average_query_length": 74.01333333333334, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "mkd_Cyrl-eng_Latn": { - "average_document_length": 495.77868852459017, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-mkd_Cyrl": { - "average_document_length": 475.51024590163934, - "average_query_length": 74.01333333333334, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "mlt_Latn-mlt_Latn": { - "average_document_length": 525.8995901639345, - "average_query_length": 75.00444444444445, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "mlt_Latn-eng_Latn": { - "average_document_length": 525.8995901639345, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-mlt_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 75.00444444444445, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "mri_Latn-mri_Latn": { - "average_document_length": 526.0860655737705, - "average_query_length": 81.71444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "mri_Latn-eng_Latn": { - "average_document_length": 526.0860655737705, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-mri_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 81.71444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "mya_Mymr-mya_Mymr": { - "average_document_length": 590.389344262295, - "average_query_length": 89.28333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "mya_Mymr-eng_Latn": { - "average_document_length": 590.389344262295, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-mya_Mymr": { - "average_document_length": 475.51024590163934, - "average_query_length": 89.28333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "nld_Latn-nld_Latn": { - "average_document_length": 529.1434426229508, - "average_query_length": 75.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "nld_Latn-eng_Latn": { - "average_document_length": 529.1434426229508, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-nld_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 75.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "nob_Latn-nob_Latn": { - "average_document_length": 479.13729508196724, - "average_query_length": 71.04555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "nob_Latn-eng_Latn": { - "average_document_length": 479.13729508196724, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-nob_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 71.04555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "npi_Deva-npi_Deva": { - "average_document_length": 456.9590163934426, - "average_query_length": 66.89666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "npi_Deva-eng_Latn": { - "average_document_length": 456.9590163934426, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-npi_Deva": { - "average_document_length": 475.51024590163934, - "average_query_length": 66.89666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "npi_Latn-npi_Latn": { - "average_document_length": 515.9815573770492, - "average_query_length": 71.89666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "npi_Latn-eng_Latn": { - "average_document_length": 515.9815573770492, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-npi_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 71.89666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "nso_Latn-nso_Latn": { - "average_document_length": 548.0225409836065, - "average_query_length": 86.77444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "nso_Latn-eng_Latn": { - "average_document_length": 548.0225409836065, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-nso_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 86.77444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "nya_Latn-nya_Latn": { - "average_document_length": 532.3934426229508, - "average_query_length": 90.78777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "nya_Latn-eng_Latn": { - "average_document_length": 532.3934426229508, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-nya_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 90.78777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ory_Orya-ory_Orya": { - "average_document_length": 487.78483606557376, - "average_query_length": 72.95777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ory_Orya-eng_Latn": { - "average_document_length": 487.78483606557376, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ory_Orya": { - "average_document_length": 475.51024590163934, - "average_query_length": 72.95777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "pan_Guru-pan_Guru": { - "average_document_length": 480.2438524590164, - "average_query_length": 73.29777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "pan_Guru-eng_Latn": { - "average_document_length": 480.2438524590164, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-pan_Guru": { - "average_document_length": 475.51024590163934, - "average_query_length": 73.29777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "pbt_Arab-pbt_Arab": { - "average_document_length": 453.3299180327869, - "average_query_length": 67.67111111111112, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "pbt_Arab-eng_Latn": { - "average_document_length": 453.3299180327869, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-pbt_Arab": { - "average_document_length": 475.51024590163934, - "average_query_length": 67.67111111111112, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "pes_Arab-pes_Arab": { - "average_document_length": 448.84631147540983, - "average_query_length": 64.75111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "pes_Arab-eng_Latn": { - "average_document_length": 448.84631147540983, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-pes_Arab": { - "average_document_length": 475.51024590163934, - "average_query_length": 64.75111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "plt_Latn-plt_Latn": { - "average_document_length": 581.2745901639345, - "average_query_length": 94.99555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "plt_Latn-eng_Latn": { - "average_document_length": 581.2745901639345, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-plt_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 94.99555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "pol_Latn-pol_Latn": { - "average_document_length": 504.0409836065574, - "average_query_length": 74.09777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "pol_Latn-eng_Latn": { - "average_document_length": 504.0409836065574, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-pol_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 74.09777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "por_Latn-por_Latn": { - "average_document_length": 517.2827868852459, - "average_query_length": 78.11666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "por_Latn-eng_Latn": { - "average_document_length": 517.2827868852459, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-por_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 78.11666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ron_Latn-ron_Latn": { - "average_document_length": 534.8668032786885, - "average_query_length": 78.74222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ron_Latn-eng_Latn": { - "average_document_length": 534.8668032786885, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ron_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 78.74222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "rus_Cyrl-rus_Cyrl": { - "average_document_length": 520.1700819672132, - "average_query_length": 83.16333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "rus_Cyrl-eng_Latn": { - "average_document_length": 520.1700819672132, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-rus_Cyrl": { - "average_document_length": 475.51024590163934, - "average_query_length": 83.16333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "shn_Mymr-shn_Mymr": { - "average_document_length": 676.172131147541, - "average_query_length": 75.90222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "shn_Mymr-eng_Latn": { - "average_document_length": 676.172131147541, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-shn_Mymr": { - "average_document_length": 475.51024590163934, - "average_query_length": 75.90222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "sin_Latn-sin_Latn": { - "average_document_length": 590.7889344262295, - "average_query_length": 94.46666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "sin_Latn-eng_Latn": { - "average_document_length": 590.7889344262295, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-sin_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 94.46666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "sin_Sinh-sin_Sinh": { - "average_document_length": 478.66803278688525, - "average_query_length": 69.91777777777777, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "sin_Sinh-eng_Latn": { - "average_document_length": 478.66803278688525, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-sin_Sinh": { - "average_document_length": 475.51024590163934, - "average_query_length": 69.91777777777777, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "slk_Latn-slk_Latn": { - "average_document_length": 476.7766393442623, - "average_query_length": 68.5411111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "slk_Latn-eng_Latn": { - "average_document_length": 476.7766393442623, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-slk_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 68.5411111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "slv_Latn-slv_Latn": { - "average_document_length": 474.84631147540983, - "average_query_length": 68.79888888888888, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "slv_Latn-eng_Latn": { - "average_document_length": 474.84631147540983, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-slv_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 68.79888888888888, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "sna_Latn-sna_Latn": { - "average_document_length": 532.5860655737705, - "average_query_length": 81.30700778642937, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0, - }, - "sna_Latn-eng_Latn": { - "average_document_length": 532.5860655737705, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-sna_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 81.30700778642937, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0, - }, - "snd_Arab-snd_Arab": { - "average_document_length": 431.48770491803276, - "average_query_length": 63.42333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "snd_Arab-eng_Latn": { - "average_document_length": 431.48770491803276, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-snd_Arab": { - "average_document_length": 475.51024590163934, - "average_query_length": 63.42333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "som_Latn-som_Latn": { - "average_document_length": 542.0737704918033, - "average_query_length": 90.95777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "som_Latn-eng_Latn": { - "average_document_length": 542.0737704918033, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-som_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 90.95777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "sot_Latn-sot_Latn": { - "average_document_length": 573.3258196721312, - "average_query_length": 83.13111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "sot_Latn-eng_Latn": { - "average_document_length": 573.3258196721312, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-sot_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 83.13111111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "spa_Latn-spa_Latn": { - "average_document_length": 564.3319672131148, - "average_query_length": 82.16, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "spa_Latn-eng_Latn": { - "average_document_length": 564.3319672131148, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-spa_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 82.16, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "srp_Cyrl-srp_Cyrl": { - "average_document_length": 471.84631147540983, - "average_query_length": 67.49833147942158, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0011123470522802, - }, - "srp_Cyrl-eng_Latn": { - "average_document_length": 471.84631147540983, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-srp_Cyrl": { - "average_document_length": 475.51024590163934, - "average_query_length": 67.49833147942158, - "num_documents": 488, - "num_queries": 899, - "average_relevant_docs_per_query": 1.0011123470522802, - }, - "ssw_Latn-ssw_Latn": { - "average_document_length": 535.0901639344262, - "average_query_length": 81.09777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ssw_Latn-eng_Latn": { - "average_document_length": 535.0901639344262, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ssw_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 81.09777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "sun_Latn-sun_Latn": { - "average_document_length": 495.3032786885246, - "average_query_length": 78.16, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "sun_Latn-eng_Latn": { - "average_document_length": 495.3032786885246, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-sun_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 78.16, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "swe_Latn-swe_Latn": { - "average_document_length": 480.6803278688525, - "average_query_length": 68.67666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "swe_Latn-eng_Latn": { - "average_document_length": 480.6803278688525, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-swe_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 68.67666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "swh_Latn-swh_Latn": { - "average_document_length": 499.0983606557377, - "average_query_length": 80.56, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "swh_Latn-eng_Latn": { - "average_document_length": 499.0983606557377, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-swh_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 80.56, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tam_Taml-tam_Taml": { - "average_document_length": 555.5286885245902, - "average_query_length": 81.12777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tam_Taml-eng_Latn": { - "average_document_length": 555.5286885245902, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-tam_Taml": { - "average_document_length": 475.51024590163934, - "average_query_length": 81.12777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tel_Telu-tel_Telu": { - "average_document_length": 481.5245901639344, - "average_query_length": 72.18777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tel_Telu-eng_Latn": { - "average_document_length": 481.5245901639344, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-tel_Telu": { - "average_document_length": 475.51024590163934, - "average_query_length": 72.18777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tgk_Cyrl-tgk_Cyrl": { - "average_document_length": 528.516393442623, - "average_query_length": 74.28111111111112, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tgk_Cyrl-eng_Latn": { - "average_document_length": 528.516393442623, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-tgk_Cyrl": { - "average_document_length": 475.51024590163934, - "average_query_length": 74.28111111111112, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tgl_Latn-tgl_Latn": { - "average_document_length": 597.6270491803278, - "average_query_length": 82.34555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tgl_Latn-eng_Latn": { - "average_document_length": 597.6270491803278, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-tgl_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 82.34555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tha_Thai-tha_Thai": { - "average_document_length": 456.1659836065574, - "average_query_length": 59.46666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tha_Thai-eng_Latn": { - "average_document_length": 456.1659836065574, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-tha_Thai": { - "average_document_length": 475.51024590163934, - "average_query_length": 59.46666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tir_Ethi-tir_Ethi": { - "average_document_length": 327.6967213114754, - "average_query_length": 51.99888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tir_Ethi-eng_Latn": { - "average_document_length": 327.6967213114754, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-tir_Ethi": { - "average_document_length": 475.51024590163934, - "average_query_length": 51.99888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tsn_Latn-tsn_Latn": { - "average_document_length": 591.7131147540983, - "average_query_length": 87.12777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tsn_Latn-eng_Latn": { - "average_document_length": 591.7131147540983, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-tsn_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 87.12777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tso_Latn-tso_Latn": { - "average_document_length": 569.6475409836065, - "average_query_length": 91.69444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tso_Latn-eng_Latn": { - "average_document_length": 569.6475409836065, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-tso_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 91.69444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tur_Latn-tur_Latn": { - "average_document_length": 489.0409836065574, - "average_query_length": 71.56222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "tur_Latn-eng_Latn": { - "average_document_length": 489.0409836065574, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-tur_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 71.56222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ukr_Cyrl-ukr_Cyrl": { - "average_document_length": 488.11475409836066, - "average_query_length": 72.08222222222223, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ukr_Cyrl-eng_Latn": { - "average_document_length": 488.11475409836066, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-ukr_Cyrl": { - "average_document_length": 475.51024590163934, - "average_query_length": 72.08222222222223, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "urd_Arab-urd_Arab": { - "average_document_length": 470.452868852459, - "average_query_length": 70.52666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "urd_Arab-eng_Latn": { - "average_document_length": 470.452868852459, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-urd_Arab": { - "average_document_length": 475.51024590163934, - "average_query_length": 70.52666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "urd_Latn-urd_Latn": { - "average_document_length": 590.5348360655738, - "average_query_length": 90.07, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "urd_Latn-eng_Latn": { - "average_document_length": 590.5348360655738, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-urd_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 90.07, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "uzn_Latn-uzn_Latn": { - "average_document_length": 539.2418032786885, - "average_query_length": 77.61333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "uzn_Latn-eng_Latn": { - "average_document_length": 539.2418032786885, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-uzn_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 77.61333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "vie_Latn-vie_Latn": { - "average_document_length": 499.8360655737705, - "average_query_length": 73.05333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "vie_Latn-eng_Latn": { - "average_document_length": 499.8360655737705, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-vie_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 73.05333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "war_Latn-war_Latn": { - "average_document_length": 592.8688524590164, - "average_query_length": 86.07555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "war_Latn-eng_Latn": { - "average_document_length": 592.8688524590164, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-war_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 86.07555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "wol_Latn-wol_Latn": { - "average_document_length": 456.9795081967213, - "average_query_length": 70.60555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "wol_Latn-eng_Latn": { - "average_document_length": 456.9795081967213, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-wol_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 70.60555555555555, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "xho_Latn-xho_Latn": { - "average_document_length": 505.0655737704918, - "average_query_length": 78.50333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "xho_Latn-eng_Latn": { - "average_document_length": 505.0655737704918, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-xho_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 78.50333333333333, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "yor_Latn-yor_Latn": { - "average_document_length": 459.5204918032787, - "average_query_length": 68.64, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "yor_Latn-eng_Latn": { - "average_document_length": 459.5204918032787, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-yor_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 68.64, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "zho_Hans-zho_Hans": { - "average_document_length": 159.76024590163934, - "average_query_length": 21.747777777777777, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "zho_Hans-eng_Latn": { - "average_document_length": 159.76024590163934, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-zho_Hans": { - "average_document_length": 475.51024590163934, - "average_query_length": 21.747777777777777, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "zho_Hant-zho_Hant": { - "average_document_length": 149.77254098360655, - "average_query_length": 21.07888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "zho_Hant-eng_Latn": { - "average_document_length": 149.77254098360655, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-zho_Hant": { - "average_document_length": 475.51024590163934, - "average_query_length": 21.07888888888889, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "zsm_Latn-zsm_Latn": { - "average_document_length": 528.9139344262295, - "average_query_length": 78.92444444444445, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "zsm_Latn-eng_Latn": { - "average_document_length": 528.9139344262295, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-zsm_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 78.92444444444445, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "zul_Latn-zul_Latn": { - "average_document_length": 532.9713114754098, - "average_query_length": 76.0411111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "zul_Latn-eng_Latn": { - "average_document_length": 532.9713114754098, - "average_query_length": 77.34777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "eng_Latn-zul_Latn": { - "average_document_length": 475.51024590163934, - "average_query_length": 76.0411111111111, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "arb_Arab-arb_Latn": { - "average_document_length": 421.96311475409834, - "average_query_length": 67.02444444444444, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "arb_Latn-arb_Arab": { - "average_document_length": 555.6188524590164, - "average_query_length": 58.55, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ben_Beng-ben_Latn": { - "average_document_length": 467.7745901639344, - "average_query_length": 74.78777777777778, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "ben_Latn-ben_Beng": { - "average_document_length": 522.8934426229508, - "average_query_length": 69.48444444444445, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hin_Deva-hin_Latn": { - "average_document_length": 473.55737704918033, - "average_query_length": 74.81222222222222, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "hin_Latn-hin_Deva": { - "average_document_length": 541.7315573770492, - "average_query_length": 72.61777777777777, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "npi_Deva-npi_Latn": { - "average_document_length": 456.9590163934426, - "average_query_length": 71.89666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "npi_Latn-npi_Deva": { - "average_document_length": 515.9815573770492, - "average_query_length": 66.89666666666666, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "sin_Sinh-sin_Latn": { - "average_document_length": 478.66803278688525, - "average_query_length": 94.46666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "sin_Latn-sin_Sinh": { - "average_document_length": 590.7889344262295, - "average_query_length": 69.91777777777777, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "urd_Arab-urd_Latn": { - "average_document_length": 470.452868852459, - "average_query_length": 90.07, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - "urd_Latn-urd_Arab": { - "average_document_length": 590.5348360655738, - "average_query_length": 70.52666666666667, - "num_documents": 488, - "num_queries": 900, - "average_relevant_docs_per_query": 1.0, - }, - }, - }, - }, bibtex_citation="""@article{bandarkar2023belebele, title={The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants}, author={Lucas Bandarkar and Davis Liang and Benjamin Muller and Mikel Artetxe and Satya Narayan Shukla and Donald Husa and Naman Goyal and Abhinandan Krishnan and Luke Zettlemoyer and Madian Khabsa}, diff --git a/mteb/tasks/Retrieval/multilingual/CUREv1Retrieval.py b/mteb/tasks/Retrieval/multilingual/CUREv1Retrieval.py new file mode 100644 index 0000000000..6e97786a77 --- /dev/null +++ b/mteb/tasks/Retrieval/multilingual/CUREv1Retrieval.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from enum import Enum + +from datasets import DatasetDict, load_dataset + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from ....abstasks.MultilingualTask import MultilingualTask + +_LANGUAGES = { + "en": ["eng-Latn", "eng-Latn"], + "es": ["spa-Latn", "eng-Latn"], + "fr": ["fra-Latn", "eng-Latn"], +} + + +class CUREv1Splits(str, Enum): + all = "All" + dentistry_and_oral_health = "Dentistry and Oral Health" + dermatology = "Dermatology" + gastroenterology = "Gastroenterology" + genetics = "Genetics" + neuroscience_and_neurology = "Neuroscience and Neurology" + orthopedic_surgery = "Orthopedic Surgery" + otorhinolaryngology = "Otorhinolaryngology" + plastic_surgery = "Plastic Surgery" + psychiatry_and_psychology = "Psychiatry and Psychology" + pulmonology = "Pulmonology" + + @classmethod + def names(cls) -> list[str]: + return sorted(cls._member_names_) + + +class CUREv1Retrieval(MultilingualTask, AbsTaskRetrieval): + metadata = TaskMetadata( + dataset={ + "path": "clinia/CUREv1", + "revision": "3bcf51c91e04d04a8a3329dfbe988b964c5cbe83", + }, + name="CUREv1", + description="Collection of query-passage pairs curated by medical professionals, across 10 disciplines and 3 cross-lingual settings.", + type="Retrieval", + modalities=["text"], + category="s2p", + reference="https://huggingface.co/datasets/clinia/CUREv1", + eval_splits=CUREv1Splits.names(), + eval_langs=_LANGUAGES, + main_score="ndcg_at_10", + date=("2024-01-01", "2024-10-31"), + domains=["Medical", "Academic", "Written"], + task_subtypes=[], + license="cc-by-nc-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="created", + bibtex_citation="", + prompt={ + "query": "Given a question by a medical professional, retrieve relevant passages that best answer the question", + }, + ) + + def _load_corpus(self, split: str, cache_dir: str | None = None): + ds = load_dataset( + path=self.metadata_dict["dataset"]["path"], + revision=self.metadata_dict["dataset"]["revision"], + name="corpus", + split=split, + cache_dir=cache_dir, + ) + + corpus = { + doc["_id"]: {"title": doc["title"], "text": doc["text"]} for doc in ds + } + + return corpus + + def _load_qrels(self, split: str, cache_dir: str | None = None): + ds = load_dataset( + path=self.metadata_dict["dataset"]["path"], + revision=self.metadata_dict["dataset"]["revision"], + name="qrels", + split=split, + cache_dir=cache_dir, + ) + + qrels = {} + + for qrel in ds: + query_id = qrel["query-id"] + doc_id = qrel["corpus-id"] + score = int(qrel["score"]) + if query_id not in qrels: + qrels[query_id] = {} + qrels[query_id][doc_id] = score + + return qrels + + def _load_queries(self, split: str, language: str, cache_dir: str | None = None): + ds = load_dataset( + path=self.metadata_dict["dataset"]["path"], + revision=self.metadata_dict["dataset"]["revision"], + name=f"queries-{language}", + split=split, + cache_dir=cache_dir, + ) + + queries = {query["_id"]: query["text"] for query in ds} + + return queries + + def load_data(self, **kwargs): + if self.data_loaded: + return + + eval_splits = kwargs.get("eval_splits", self.metadata.eval_splits) + languages = kwargs.get("eval_langs", self.metadata.eval_langs) + cache_dir = kwargs.get("cache_dir", None) + + # Iterate over splits and languages + corpus = { + language: {split: None for split in eval_splits} for language in languages + } + queries = { + language: {split: None for split in eval_splits} for language in languages + } + relevant_docs = { + language: {split: None for split in eval_splits} for language in languages + } + for split in eval_splits: + # Since this is a cross-lingual dataset, the corpus and the relevant documents do not depend on the language + split_corpus = self._load_corpus(split=split, cache_dir=cache_dir) + split_qrels = self._load_qrels(split=split, cache_dir=cache_dir) + + # Queries depend on the language + for language in languages: + corpus[language][split] = split_corpus + relevant_docs[language][split] = split_qrels + + queries[language][split] = self._load_queries( + split=split, language=language, cache_dir=cache_dir + ) + + # Convert into DatasetDict + self.corpus = DatasetDict(corpus) + self.queries = DatasetDict(queries) + self.relevant_docs = DatasetDict(relevant_docs) + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py index d46c87e7e8..4ca7c5e495 100644 --- a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py +++ b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py @@ -53,27 +53,6 @@ class CrossLingualSemanticDiscriminationWMT19(AbsTaskRetrieval, MultilingualTask dialect=[], sample_creation="LM-generated and verified", bibtex_citation="preprint_coming", - descriptive_stats={ - "n_samples": {"test": 2946}, - "avg_character_length": { - "test": { - "deu-fra": { - "average_document_length": 147.49857433808555, - "average_query_length": 152.95587236931433, - "num_documents": 7365, - "num_queries": 1473, - "average_relevant_docs_per_query": 1.0, - }, - "fra-deu": { - "average_document_length": 154.21968771215208, - "average_query_length": 145.877800407332, - "num_documents": 7365, - "num_queries": 1473, - "average_relevant_docs_per_query": 1.0, - }, - }, - }, - }, ) def __init__(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py index 871eb030be..f5c0262308 100644 --- a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py +++ b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py @@ -53,27 +53,6 @@ class CrossLingualSemanticDiscriminationWMT21(AbsTaskRetrieval, MultilingualTask dialect=[], sample_creation="LM-generated and verified", bibtex_citation="preprint_coming", - descriptive_stats={ - "n_samples": {"test": 1786}, - "avg_character_length": { - "test": { - "deu-fra": { - "average_document_length": 177.26270996640537, - "average_query_length": 171.73012318029114, - "num_documents": 4465, - "num_queries": 893, - "average_relevant_docs_per_query": 1.0, - }, - "fra-deu": { - "average_document_length": 174.45061590145576, - "average_query_length": 176.99216125419932, - "num_documents": 4465, - "num_queries": 893, - "average_relevant_docs_per_query": 1.0, - }, - }, - }, - }, ) def __init__(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py b/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py index 2c6e1c70eb..62a166f89c 100644 --- a/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py @@ -54,90 +54,6 @@ class IndicQARetrieval(MultilingualTask, AbsTaskRetrieval): year = {2022}, doi = {10.18653/v1/2023.acl-long.693} }""", - descriptive_stats={ - "n_samples": {"test": 18586}, - "avg_character_length": { - "test": { - "as": { - "average_document_length": 1401.28, - "average_query_length": 56.60504201680672, - "num_documents": 250, - "num_queries": 1785, - "average_relevant_docs_per_query": 1.0016806722689076, - }, - "bn": { - "average_document_length": 2196.012, - "average_query_length": 57.069239500567534, - "num_documents": 250, - "num_queries": 1762, - "average_relevant_docs_per_query": 1.0005675368898979, - }, - "gu": { - "average_document_length": 960.4959677419355, - "average_query_length": 60.3712158808933, - "num_documents": 248, - "num_queries": 2015, - "average_relevant_docs_per_query": 1.0009925558312656, - }, - "hi": { - "average_document_length": 2550.770114942529, - "average_query_length": 52.84909326424871, - "num_documents": 261, - "num_queries": 1544, - "average_relevant_docs_per_query": 1.0019430051813472, - }, - "kn": { - "average_document_length": 882.7354085603113, - "average_query_length": 50.58734344100198, - "num_documents": 257, - "num_queries": 1517, - "average_relevant_docs_per_query": 1.0, - }, - "ml": { - "average_document_length": 2522.6437246963565, - "average_query_length": 75.93635790800252, - "num_documents": 247, - "num_queries": 1587, - "average_relevant_docs_per_query": 1.0, - }, - "mr": { - "average_document_length": 1711.74, - "average_query_length": 58.785, - "num_documents": 250, - "num_queries": 1600, - "average_relevant_docs_per_query": 1.0, - }, - "or": { - "average_document_length": 801.9206349206349, - "average_query_length": 55.072792362768496, - "num_documents": 252, - "num_queries": 1676, - "average_relevant_docs_per_query": 1.0011933174224343, - }, - "pa": { - "average_document_length": 1423.5062240663901, - "average_query_length": 58.394925178919976, - "num_documents": 241, - "num_queries": 1537, - "average_relevant_docs_per_query": 1.0013012361743656, - }, - "ta": { - "average_document_length": 2288.2608695652175, - "average_query_length": 54.06211869107044, - "num_documents": 253, - "num_queries": 1803, - "average_relevant_docs_per_query": 1.0005546311702718, - }, - "te": { - "average_document_length": 2936.176, - "average_query_length": 67.00634371395617, - "num_documents": 250, - "num_queries": 1734, - "average_relevant_docs_per_query": 1.0, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py b/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py index f9e5239c5f..2b21177297 100644 --- a/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py @@ -133,138 +133,8 @@ class MIRACLRetrieval(MultilingualTask, AbsTaskRetrieval): url = {https://doi.org/10.1162/tacl\_a\_00595}, eprint = {https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl\_a\_00595/2157340/tacl\_a\_00595.pdf}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "ar": { - "average_document_length": 318.6539598547405, - "average_query_length": 29.480662983425415, - "num_documents": 2061414, - "num_queries": 2896, - "average_relevant_docs_per_query": 1.953729281767956, - }, - "bn": { - "average_document_length": 383.2428136511194, - "average_query_length": 46.98053527980535, - "num_documents": 297265, - "num_queries": 411, - "average_relevant_docs_per_query": 2.099756690997567, - }, - "de": { - "average_document_length": 414.28004442393404, - "average_query_length": 46.0, - "num_documents": 15866222, - "num_queries": 305, - "average_relevant_docs_per_query": 2.6590163934426227, - }, - "en": { - "average_document_length": 401.0042914921588, - "average_query_length": 40.247809762202756, - "num_documents": 32893221, - "num_queries": 799, - "average_relevant_docs_per_query": 2.911138923654568, - }, - "es": { - "average_document_length": 403.71153493754986, - "average_query_length": 47.373456790123456, - "num_documents": 10373953, - "num_queries": 648, - "average_relevant_docs_per_query": 4.609567901234568, - }, - "fa": { - "average_document_length": 262.6478385010321, - "average_query_length": 41.1503164556962, - "num_documents": 2207172, - "num_queries": 632, - "average_relevant_docs_per_query": 2.079113924050633, - }, - "fi": { - "average_document_length": 359.87767671935734, - "average_query_length": 38.63493312352478, - "num_documents": 1883509, - "num_queries": 1271, - "average_relevant_docs_per_query": 1.925255704169945, - }, - "fr": { - "average_document_length": 343.6283550271699, - "average_query_length": 43.883381924198254, - "num_documents": 14636953, - "num_queries": 343, - "average_relevant_docs_per_query": 2.131195335276968, - }, - "hi": { - "average_document_length": 370.96196845914386, - "average_query_length": 53.34, - "num_documents": 506264, - "num_queries": 350, - "average_relevant_docs_per_query": 2.1485714285714286, - }, - "id": { - "average_document_length": 350.2785651811673, - "average_query_length": 37.958333333333336, - "num_documents": 1446315, - "num_queries": 960, - "average_relevant_docs_per_query": 3.216666666666667, - }, - "ja": { - "average_document_length": 145.8538220556965, - "average_query_length": 17.71395348837209, - "num_documents": 6953614, - "num_queries": 860, - "average_relevant_docs_per_query": 2.0813953488372094, - }, - "ko": { - "average_document_length": 173.97649170809927, - "average_query_length": 21.624413145539908, - "num_documents": 1486752, - "num_queries": 213, - "average_relevant_docs_per_query": 2.568075117370892, - }, - "ru": { - "average_document_length": 332.2475377512674, - "average_query_length": 44.13258785942492, - "num_documents": 9543918, - "num_queries": 1252, - "average_relevant_docs_per_query": 2.8434504792332267, - }, - "sw": { - "average_document_length": 228.71348655286377, - "average_query_length": 38.97095435684647, - "num_documents": 131924, - "num_queries": 482, - "average_relevant_docs_per_query": 1.887966804979253, - }, - "te": { - "average_document_length": 396.2108674545774, - "average_query_length": 38.11231884057971, - "num_documents": 518079, - "num_queries": 828, - "average_relevant_docs_per_query": 1.0314009661835748, - }, - "th": { - "average_document_length": 356.8283496198581, - "average_query_length": 42.87585266030014, - "num_documents": 542166, - "num_queries": 733, - "average_relevant_docs_per_query": 1.8321964529331514, - }, - "yo": { - "average_document_length": 159.35250698366738, - "average_query_length": 37.6890756302521, - "num_documents": 49043, - "num_queries": 119, - "average_relevant_docs_per_query": 1.2100840336134453, - }, - "zh": { - "average_document_length": 119.9458931721347, - "average_query_length": 10.867684478371501, - "num_documents": 4934368, - "num_queries": 393, - "average_relevant_docs_per_query": 2.5292620865139948, - }, - } - }, + prompt={ + "query": "Given a question, retrieve Wikipedia passages that answer the question" }, ) @@ -451,146 +321,6 @@ class MIRACLRetrievalHardNegatives(MultilingualTask, AbsTaskRetrieval): url = {https://doi.org/10.1162/tacl\_a\_00595}, eprint = {https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl\_a\_00595/2157340/tacl\_a\_00595.pdf}, }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "average_document_length": 417.6655323669399, - "average_query_length": 37.46957385337667, - "num_documents": 2449382, - "num_queries": 11076, - "average_relevant_docs_per_query": 2.3643011917659806, - "hf_subset_descriptive_stats": { - "ar": { - "average_document_length": 438.1872433017704, - "average_query_length": 29.584, - "num_documents": 192103, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.982, - }, - "bn": { - "average_document_length": 383.2428136511194, - "average_query_length": 46.98053527980535, - "num_documents": 297265, - "num_queries": 411, - "average_relevant_docs_per_query": 2.099756690997567, - }, - "de": { - "average_document_length": 513.7796484139344, - "average_query_length": 46.0, - "num_documents": 71277, - "num_queries": 305, - "average_relevant_docs_per_query": 2.6590163934426227, - }, - "en": { - "average_document_length": 529.2486406963214, - "average_query_length": 40.247809762202756, - "num_documents": 178768, - "num_queries": 799, - "average_relevant_docs_per_query": 2.911138923654568, - }, - "es": { - "average_document_length": 535.8023645655877, - "average_query_length": 47.373456790123456, - "num_documents": 146750, - "num_queries": 648, - "average_relevant_docs_per_query": 4.609567901234568, - }, - "fa": { - "average_document_length": 411.2648282882721, - "average_query_length": 41.1503164556962, - "num_documents": 133596, - "num_queries": 632, - "average_relevant_docs_per_query": 2.079113924050633, - }, - "fi": { - "average_document_length": 462.9445310289844, - "average_query_length": 38.646, - "num_documents": 194415, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.918, - }, - "fr": { - "average_document_length": 460.40909271865917, - "average_query_length": 43.883381924198254, - "num_documents": 75357, - "num_queries": 343, - "average_relevant_docs_per_query": 2.131195335276968, - }, - "hi": { - "average_document_length": 498.6759426632417, - "average_query_length": 53.34, - "num_documents": 63066, - "num_queries": 350, - "average_relevant_docs_per_query": 2.1485714285714286, - }, - "id": { - "average_document_length": 494.1689807519638, - "average_query_length": 37.958333333333336, - "num_documents": 168173, - "num_queries": 960, - "average_relevant_docs_per_query": 3.216666666666667, - }, - "ja": { - "average_document_length": 206.13654293407583, - "average_query_length": 17.71395348837209, - "num_documents": 185319, - "num_queries": 860, - "average_relevant_docs_per_query": 2.0813953488372094, - }, - "ko": { - "average_document_length": 257.82646155267594, - "average_query_length": 21.624413145539908, - "num_documents": 43293, - "num_queries": 213, - "average_relevant_docs_per_query": 2.568075117370892, - }, - "ru": { - "average_document_length": 476.0820349224605, - "average_query_length": 44.055, - "num_documents": 219114, - "num_queries": 1000, - "average_relevant_docs_per_query": 2.833, - }, - "sw": { - "average_document_length": 228.71348655286377, - "average_query_length": 38.97095435684647, - "num_documents": 131924, - "num_queries": 482, - "average_relevant_docs_per_query": 1.887966804979253, - }, - "te": { - "average_document_length": 601.7099283059209, - "average_query_length": 38.11231884057971, - "num_documents": 101961, - "num_queries": 828, - "average_relevant_docs_per_query": 1.0314009661835748, - }, - "th": { - "average_document_length": 478.8818849711528, - "average_query_length": 42.87585266030014, - "num_documents": 116649, - "num_queries": 733, - "average_relevant_docs_per_query": 1.8321964529331514, - }, - "yo": { - "average_document_length": 159.35250698366738, - "average_query_length": 37.6890756302521, - "num_documents": 49043, - "num_queries": 119, - "average_relevant_docs_per_query": 1.2100840336134453, - }, - "zh": { - "average_document_length": 147.36211243527777, - "average_query_length": 10.867684478371501, - "num_documents": 81309, - "num_queries": 393, - "average_relevant_docs_per_query": 2.5292620865139948, - }, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py b/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py index 3afc98bb2c..c03f280b22 100644 --- a/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py @@ -110,701 +110,6 @@ class MLQARetrieval(AbsTaskRetrieval, MultilingualTask): year = 2019, eid = {arXiv: 1910.07475} }""", - descriptive_stats={ - "n_samples": {"test": 158083, "validation": 15747}, - "avg_character_length": { - "validation": { - "ara-ara": { - "average_document_length": 693.8883826879271, - "average_query_length": 42.321083172147, - "num_documents": 439, - "num_queries": 517, - "average_relevant_docs_per_query": 1.0, - }, - "ara-deu": { - "average_document_length": 759.3882352941176, - "average_query_length": 55.14492753623188, - "num_documents": 170, - "num_queries": 207, - "average_relevant_docs_per_query": 1.0, - }, - "ara-eng": { - "average_document_length": 693.8883826879271, - "average_query_length": 50.029013539651835, - "num_documents": 439, - "num_queries": 517, - "average_relevant_docs_per_query": 1.0, - }, - "ara-spa": { - "average_document_length": 654.3071428571428, - "average_query_length": 53.68944099378882, - "num_documents": 140, - "num_queries": 161, - "average_relevant_docs_per_query": 1.0, - }, - "ara-hin": { - "average_document_length": 626.5935483870968, - "average_query_length": 51.956989247311824, - "num_documents": 155, - "num_queries": 186, - "average_relevant_docs_per_query": 1.0, - }, - "ara-vie": { - "average_document_length": 804.6216216216217, - "average_query_length": 49.57055214723926, - "num_documents": 148, - "num_queries": 163, - "average_relevant_docs_per_query": 1.0, - }, - "ara-zho": { - "average_document_length": 787.3161290322581, - "average_query_length": 15.617021276595745, - "num_documents": 155, - "num_queries": 188, - "average_relevant_docs_per_query": 1.0, - }, - "deu-ara": { - "average_document_length": 702.1675977653631, - "average_query_length": 43.06280193236715, - "num_documents": 179, - "num_queries": 207, - "average_relevant_docs_per_query": 1.0, - }, - "deu-deu": { - "average_document_length": 721.405701754386, - "average_query_length": 52.572265625, - "num_documents": 456, - "num_queries": 512, - "average_relevant_docs_per_query": 1.0, - }, - "deu-eng": { - "average_document_length": 721.405701754386, - "average_query_length": 48.33984375, - "num_documents": 456, - "num_queries": 512, - "average_relevant_docs_per_query": 1.0, - }, - "deu-spa": { - "average_document_length": 677.2762430939226, - "average_query_length": 50.60204081632653, - "num_documents": 181, - "num_queries": 196, - "average_relevant_docs_per_query": 1.0, - }, - "deu-hin": { - "average_document_length": 685.917808219178, - "average_query_length": 47.01840490797546, - "num_documents": 146, - "num_queries": 163, - "average_relevant_docs_per_query": 1.0, - }, - "deu-vie": { - "average_document_length": 921.6196319018405, - "average_query_length": 46.81868131868132, - "num_documents": 163, - "num_queries": 182, - "average_relevant_docs_per_query": 1.0, - }, - "deu-zho": { - "average_document_length": 736.6347305389221, - "average_query_length": 14.936842105263159, - "num_documents": 167, - "num_queries": 190, - "average_relevant_docs_per_query": 1.0, - }, - "eng-ara": { - "average_document_length": 979.3447488584475, - "average_query_length": 42.321083172147, - "num_documents": 438, - "num_queries": 517, - "average_relevant_docs_per_query": 1.0, - }, - "eng-deu": { - "average_document_length": 947.3109619686801, - "average_query_length": 52.572265625, - "num_documents": 447, - "num_queries": 512, - "average_relevant_docs_per_query": 1.0, - }, - "eng-eng": { - "average_document_length": 940.2842535787321, - "average_query_length": 49.01480836236934, - "num_documents": 978, - "num_queries": 1148, - "average_relevant_docs_per_query": 1.0, - }, - "eng-spa": { - "average_document_length": 904.3166287015945, - "average_query_length": 52.146, - "num_documents": 439, - "num_queries": 500, - "average_relevant_docs_per_query": 1.0, - }, - "eng-hin": { - "average_document_length": 926.9621749408983, - "average_query_length": 49.3905325443787, - "num_documents": 423, - "num_queries": 507, - "average_relevant_docs_per_query": 1.0, - }, - "eng-vie": { - "average_document_length": 1011.8296460176991, - "average_query_length": 48.082191780821915, - "num_documents": 452, - "num_queries": 511, - "average_relevant_docs_per_query": 1.0, - }, - "eng-zho": { - "average_document_length": 1001.5046511627907, - "average_query_length": 15.39484126984127, - "num_documents": 430, - "num_queries": 504, - "average_relevant_docs_per_query": 1.0, - }, - "spa-ara": { - "average_document_length": 674.3586206896551, - "average_query_length": 41.36024844720497, - "num_documents": 145, - "num_queries": 161, - "average_relevant_docs_per_query": 1.0, - }, - "spa-deu": { - "average_document_length": 544.0489130434783, - "average_query_length": 51.86734693877551, - "num_documents": 184, - "num_queries": 196, - "average_relevant_docs_per_query": 1.0, - }, - "spa-eng": { - "average_document_length": 641.8215859030837, - "average_query_length": 49.156, - "num_documents": 454, - "num_queries": 500, - "average_relevant_docs_per_query": 1.0, - }, - "spa-spa": { - "average_document_length": 641.8215859030837, - "average_query_length": 52.146, - "num_documents": 454, - "num_queries": 500, - "average_relevant_docs_per_query": 1.0, - }, - "spa-hin": { - "average_document_length": 703.3212121212122, - "average_query_length": 48.080213903743314, - "num_documents": 165, - "num_queries": 187, - "average_relevant_docs_per_query": 1.0, - }, - "spa-vie": { - "average_document_length": 737.8579545454545, - "average_query_length": 48.82539682539682, - "num_documents": 176, - "num_queries": 189, - "average_relevant_docs_per_query": 1.0, - }, - "spa-zho": { - "average_document_length": 605.52, - "average_query_length": 15.590062111801242, - "num_documents": 150, - "num_queries": 161, - "average_relevant_docs_per_query": 1.0, - }, - "hin-ara": { - "average_document_length": 670.0394736842105, - "average_query_length": 43.623655913978496, - "num_documents": 152, - "num_queries": 186, - "average_relevant_docs_per_query": 1.0, - }, - "hin-deu": { - "average_document_length": 596.9718309859155, - "average_query_length": 51.41717791411043, - "num_documents": 142, - "num_queries": 163, - "average_relevant_docs_per_query": 1.0, - }, - "hin-eng": { - "average_document_length": 691.5482352941176, - "average_query_length": 49.75936883629191, - "num_documents": 425, - "num_queries": 507, - "average_relevant_docs_per_query": 1.0, - }, - "hin-spa": { - "average_document_length": 718.4904458598726, - "average_query_length": 52.75935828877005, - "num_documents": 157, - "num_queries": 187, - "average_relevant_docs_per_query": 1.0, - }, - "hin-hin": { - "average_document_length": 691.5482352941176, - "average_query_length": 49.3905325443787, - "num_documents": 425, - "num_queries": 507, - "average_relevant_docs_per_query": 1.0, - }, - "hin-vie": { - "average_document_length": 778.484076433121, - "average_query_length": 48.35028248587571, - "num_documents": 157, - "num_queries": 177, - "average_relevant_docs_per_query": 1.0, - }, - "hin-zho": { - "average_document_length": 685.0679012345679, - "average_query_length": 15.97883597883598, - "num_documents": 162, - "num_queries": 189, - "average_relevant_docs_per_query": 1.0, - }, - "vie-ara": { - "average_document_length": 886.6052631578947, - "average_query_length": 41.214723926380366, - "num_documents": 152, - "num_queries": 163, - "average_relevant_docs_per_query": 1.0, - }, - "vie-deu": { - "average_document_length": 981.4534161490683, - "average_query_length": 51.27472527472528, - "num_documents": 161, - "num_queries": 182, - "average_relevant_docs_per_query": 1.0, - }, - "vie-eng": { - "average_document_length": 892.7250554323725, - "average_query_length": 48.09001956947162, - "num_documents": 451, - "num_queries": 511, - "average_relevant_docs_per_query": 1.0, - }, - "vie-spa": { - "average_document_length": 936.6746987951807, - "average_query_length": 51.851851851851855, - "num_documents": 166, - "num_queries": 189, - "average_relevant_docs_per_query": 1.0, - }, - "vie-hin": { - "average_document_length": 869.0509554140127, - "average_query_length": 46.44632768361582, - "num_documents": 157, - "num_queries": 177, - "average_relevant_docs_per_query": 1.0, - }, - "vie-vie": { - "average_document_length": 892.7250554323725, - "average_query_length": 48.082191780821915, - "num_documents": 451, - "num_queries": 511, - "average_relevant_docs_per_query": 1.0, - }, - "vie-zho": { - "average_document_length": 960.7349397590361, - "average_query_length": 15.048913043478262, - "num_documents": 166, - "num_queries": 184, - "average_relevant_docs_per_query": 1.0, - }, - "zho-ara": { - "average_document_length": 238.75155279503105, - "average_query_length": 44.34574468085106, - "num_documents": 161, - "num_queries": 188, - "average_relevant_docs_per_query": 1.0, - }, - "zho-deu": { - "average_document_length": 257.109756097561, - "average_query_length": 53.84736842105263, - "num_documents": 164, - "num_queries": 190, - "average_relevant_docs_per_query": 1.0, - }, - "zho-eng": { - "average_document_length": 246.65237020316027, - "average_query_length": 50.15079365079365, - "num_documents": 443, - "num_queries": 504, - "average_relevant_docs_per_query": 1.0, - }, - "zho-spa": { - "average_document_length": 249.6081081081081, - "average_query_length": 52.857142857142854, - "num_documents": 148, - "num_queries": 161, - "average_relevant_docs_per_query": 1.0, - }, - "zho-hin": { - "average_document_length": 238.5521472392638, - "average_query_length": 52.05291005291005, - "num_documents": 163, - "num_queries": 189, - "average_relevant_docs_per_query": 1.0, - }, - "zho-vie": { - "average_document_length": 268.32142857142856, - "average_query_length": 49.33695652173913, - "num_documents": 168, - "num_queries": 184, - "average_relevant_docs_per_query": 1.0, - }, - "zho-zho": { - "average_document_length": 246.65237020316027, - "average_query_length": 15.39484126984127, - "num_documents": 443, - "num_queries": 504, - "average_relevant_docs_per_query": 1.0, - }, - }, - "test": { - "ara-ara": { - "average_document_length": 698.5714593198451, - "average_query_length": 41.26176636039752, - "num_documents": 4646, - "num_queries": 5333, - "average_relevant_docs_per_query": 1.000375023438965, - }, - "ara-deu": { - "average_document_length": 592.5728542914171, - "average_query_length": 51.27730582524272, - "num_documents": 1503, - "num_queries": 1648, - "average_relevant_docs_per_query": 1.0006067961165048, - }, - "ara-eng": { - "average_document_length": 698.5714593198451, - "average_query_length": 48.556451612903224, - "num_documents": 4646, - "num_queries": 5332, - "average_relevant_docs_per_query": 1.000562640660165, - }, - "ara-spa": { - "average_document_length": 713.4833239118146, - "average_query_length": 51.406471183013146, - "num_documents": 1769, - "num_queries": 1978, - "average_relevant_docs_per_query": 1.0, - }, - "ara-hin": { - "average_document_length": 702.1388888888889, - "average_query_length": 48.71818678317859, - "num_documents": 1512, - "num_queries": 1831, - "average_relevant_docs_per_query": 1.0, - }, - "ara-vie": { - "average_document_length": 745.4528096017458, - "average_query_length": 48.815828041035665, - "num_documents": 1833, - "num_queries": 2047, - "average_relevant_docs_per_query": 1.0, - }, - "ara-zho": { - "average_document_length": 774.4593639575971, - "average_query_length": 14.985355648535565, - "num_documents": 1698, - "num_queries": 1912, - "average_relevant_docs_per_query": 1.0, - }, - "deu-ara": { - "average_document_length": 719.6800267201069, - "average_query_length": 39.54578532443905, - "num_documents": 1497, - "num_queries": 1649, - "average_relevant_docs_per_query": 1.0, - }, - "deu-deu": { - "average_document_length": 725.5304712558599, - "average_query_length": 51.610680257035234, - "num_documents": 4053, - "num_queries": 4513, - "average_relevant_docs_per_query": 1.0008863283846665, - }, - "deu-eng": { - "average_document_length": 725.5304712558599, - "average_query_length": 47.07777531575449, - "num_documents": 4053, - "num_queries": 4513, - "average_relevant_docs_per_query": 1.0008863283846665, - }, - "deu-spa": { - "average_document_length": 740.5414052697616, - "average_query_length": 50.098591549295776, - "num_documents": 1594, - "num_queries": 1775, - "average_relevant_docs_per_query": 1.0005633802816902, - }, - "deu-hin": { - "average_document_length": 674.3714063714064, - "average_query_length": 45.146153846153844, - "num_documents": 1287, - "num_queries": 1430, - "average_relevant_docs_per_query": 1.0, - }, - "deu-vie": { - "average_document_length": 760.1198945981555, - "average_query_length": 46.64358208955224, - "num_documents": 1518, - "num_queries": 1675, - "average_relevant_docs_per_query": 1.0, - }, - "deu-zho": { - "average_document_length": 771.3367697594501, - "average_query_length": 14.942592592592593, - "num_documents": 1455, - "num_queries": 1620, - "average_relevant_docs_per_query": 1.0006172839506173, - }, - "eng-ara": { - "average_document_length": 1008.3584455058619, - "average_query_length": 41.26176636039752, - "num_documents": 4606, - "num_queries": 5333, - "average_relevant_docs_per_query": 1.000375023438965, - }, - "eng-deu": { - "average_document_length": 910.3226686507936, - "average_query_length": 51.610680257035234, - "num_documents": 4032, - "num_queries": 4513, - "average_relevant_docs_per_query": 1.0008863283846665, - }, - "eng-eng": { - "average_document_length": 983.0993344090359, - "average_query_length": 47.960714902434816, - "num_documents": 9916, - "num_queries": 11582, - "average_relevant_docs_per_query": 1.000690726990157, - }, - "eng-spa": { - "average_document_length": 967.4622376109068, - "average_query_length": 50.923252713768804, - "num_documents": 4621, - "num_queries": 5251, - "average_relevant_docs_per_query": 1.000380879832413, - }, - "eng-hin": { - "average_document_length": 986.0465631929046, - "average_query_length": 47.328315703824245, - "num_documents": 4059, - "num_queries": 4916, - "average_relevant_docs_per_query": 1.000406834825061, - }, - "eng-vie": { - "average_document_length": 1048.6062197940744, - "average_query_length": 48.094085532302095, - "num_documents": 4759, - "num_queries": 5495, - "average_relevant_docs_per_query": 1.0, - }, - "eng-zho": { - "average_document_length": 1063.8536257833482, - "average_query_length": 15.019080996884735, - "num_documents": 4468, - "num_queries": 5136, - "average_relevant_docs_per_query": 1.0001947040498442, - }, - "spa-ara": { - "average_document_length": 645.5182320441988, - "average_query_length": 40.78412537917088, - "num_documents": 1810, - "num_queries": 1978, - "average_relevant_docs_per_query": 1.0, - }, - "spa-deu": { - "average_document_length": 586.6057810578105, - "average_query_length": 51.870913190529876, - "num_documents": 1626, - "num_queries": 1774, - "average_relevant_docs_per_query": 1.0011273957158964, - }, - "spa-eng": { - "average_document_length": 630.6735979836169, - "average_query_length": 47.827907862173994, - "num_documents": 4761, - "num_queries": 5253, - "average_relevant_docs_per_query": 1.0, - }, - "spa-spa": { - "average_document_length": 630.6735979836169, - "average_query_length": 50.923252713768804, - "num_documents": 4761, - "num_queries": 5251, - "average_relevant_docs_per_query": 1.000380879832413, - }, - "spa-hin": { - "average_document_length": 613.3478260869565, - "average_query_length": 46.36680208937899, - "num_documents": 1518, - "num_queries": 1723, - "average_relevant_docs_per_query": 1.0, - }, - "spa-vie": { - "average_document_length": 659.6179295624333, - "average_query_length": 48.1595639246779, - "num_documents": 1874, - "num_queries": 2018, - "average_relevant_docs_per_query": 1.0, - }, - "spa-zho": { - "average_document_length": 668.6646171045277, - "average_query_length": 15.115562403697997, - "num_documents": 1789, - "num_queries": 1947, - "average_relevant_docs_per_query": 1.0, - }, - "hin-ara": { - "average_document_length": 765.0352862849534, - "average_query_length": 42.04642271982523, - "num_documents": 1502, - "num_queries": 1831, - "average_relevant_docs_per_query": 1.0, - }, - "hin-deu": { - "average_document_length": 719.676862745098, - "average_query_length": 51.002799160251925, - "num_documents": 1275, - "num_queries": 1429, - "average_relevant_docs_per_query": 1.000699790062981, - }, - "hin-eng": { - "average_document_length": 760.9956086850451, - "average_query_length": 47.91232709519935, - "num_documents": 4099, - "num_queries": 4916, - "average_relevant_docs_per_query": 1.000406834825061, - }, - "hin-spa": { - "average_document_length": 753.5010281014394, - "average_query_length": 50.46689895470383, - "num_documents": 1459, - "num_queries": 1722, - "average_relevant_docs_per_query": 1.0005807200929153, - }, - "hin-hin": { - "average_document_length": 760.9956086850451, - "average_query_length": 47.328315703824245, - "num_documents": 4099, - "num_queries": 4916, - "average_relevant_docs_per_query": 1.000406834825061, - }, - "hin-vie": { - "average_document_length": 789.9253822629969, - "average_query_length": 48.21160760143811, - "num_documents": 1635, - "num_queries": 1947, - "average_relevant_docs_per_query": 1.0, - }, - "hin-zho": { - "average_document_length": 834.2057448229793, - "average_query_length": 15.101301641199774, - "num_documents": 1497, - "num_queries": 1767, - "average_relevant_docs_per_query": 1.0, - }, - "vie-ara": { - "average_document_length": 992.2129527991218, - "average_query_length": 41.82462139716659, - "num_documents": 1822, - "num_queries": 2047, - "average_relevant_docs_per_query": 1.0, - }, - "vie-deu": { - "average_document_length": 861.0610079575597, - "average_query_length": 51.58721624850657, - "num_documents": 1508, - "num_queries": 1674, - "average_relevant_docs_per_query": 1.0005973715651135, - }, - "vie-eng": { - "average_document_length": 913.8633993743483, - "average_query_length": 48.11086837793555, - "num_documents": 4795, - "num_queries": 5493, - "average_relevant_docs_per_query": 1.0003640997633352, - }, - "vie-spa": { - "average_document_length": 940.0322580645161, - "average_query_length": 51.13386217154189, - "num_documents": 1829, - "num_queries": 2017, - "average_relevant_docs_per_query": 1.0004957858205255, - }, - "vie-hin": { - "average_document_length": 838.1713414634146, - "average_query_length": 47.484334874165384, - "num_documents": 1640, - "num_queries": 1947, - "average_relevant_docs_per_query": 1.0, - }, - "vie-vie": { - "average_document_length": 913.8633993743483, - "average_query_length": 48.094085532302095, - "num_documents": 4795, - "num_queries": 5495, - "average_relevant_docs_per_query": 1.0, - }, - "vie-zho": { - "average_document_length": 999.064534883721, - "average_query_length": 15.045805455481215, - "num_documents": 1720, - "num_queries": 1943, - "average_relevant_docs_per_query": 1.0, - }, - "zho-ara": { - "average_document_length": 253.71303841676368, - "average_query_length": 42.04866562009419, - "num_documents": 1718, - "num_queries": 1911, - "average_relevant_docs_per_query": 1.000523286237572, - }, - "zho-deu": { - "average_document_length": 241.84631147540983, - "average_query_length": 52.25107958050586, - "num_documents": 1464, - "num_queries": 1621, - "average_relevant_docs_per_query": 1.0, - }, - "zho-eng": { - "average_document_length": 247.55609326880776, - "average_query_length": 48.64167478091529, - "num_documents": 4546, - "num_queries": 5135, - "average_relevant_docs_per_query": 1.0003894839337877, - }, - "zho-spa": { - "average_document_length": 254.44552196235026, - "average_query_length": 51.90446841294299, - "num_documents": 1753, - "num_queries": 1947, - "average_relevant_docs_per_query": 1.0, - }, - "zho-hin": { - "average_document_length": 229.60590163934427, - "average_query_length": 49.06625141562854, - "num_documents": 1525, - "num_queries": 1766, - "average_relevant_docs_per_query": 1.0005662514156286, - }, - "zho-vie": { - "average_document_length": 266.1140401146132, - "average_query_length": 49.27328872876994, - "num_documents": 1745, - "num_queries": 1943, - "average_relevant_docs_per_query": 1.0, - }, - "zho-zho": { - "average_document_length": 247.55609326880776, - "average_query_length": 15.019080996884735, - "num_documents": 4546, - "num_queries": 5136, - "average_relevant_docs_per_query": 1.0001947040498442, - }, - }, - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py b/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py index 2850438c3c..3a44ba4e09 100644 --- a/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py @@ -100,69 +100,6 @@ class MintakaRetrieval(MultilingualTask, AbsTaskRetrieval): url = "https://aclanthology.org/2022.coling-1.138", pages = "1604--1619" }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "ar": { - "average_document_length": 12.736418511066399, - "average_query_length": 55.275533363595095, - "num_documents": 1491, - "num_queries": 2203, - "average_relevant_docs_per_query": 1.0, - }, - "de": { - "average_document_length": 14.40060422960725, - "average_query_length": 65.41322662173546, - "num_documents": 1655, - "num_queries": 2374, - "average_relevant_docs_per_query": 1.0, - }, - "es": { - "average_document_length": 14.291789722386296, - "average_query_length": 64.88325082508251, - "num_documents": 1693, - "num_queries": 2424, - "average_relevant_docs_per_query": 1.0, - }, - "fr": { - "average_document_length": 14.407234539089849, - "average_query_length": 68.88452088452088, - "num_documents": 1714, - "num_queries": 2442, - "average_relevant_docs_per_query": 1.0, - }, - "hi": { - "average_document_length": 12.71038961038961, - "average_query_length": 58.404637247569184, - "num_documents": 770, - "num_queries": 1337, - "average_relevant_docs_per_query": 1.0, - }, - "it": { - "average_document_length": 14.365985576923077, - "average_query_length": 64.39707724425887, - "num_documents": 1664, - "num_queries": 2395, - "average_relevant_docs_per_query": 1.0004175365344468, - }, - "ja": { - "average_document_length": 9.167713567839195, - "average_query_length": 29.961937716262977, - "num_documents": 1592, - "num_queries": 2312, - "average_relevant_docs_per_query": 1.0, - }, - "pt": { - "average_document_length": 14.244471744471744, - "average_query_length": 60.42225998300765, - "num_documents": 1628, - "num_queries": 2354, - "average_relevant_docs_per_query": 1.0004248088360237, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py b/mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py index f7bf5f9dc8..0b65d3b8f8 100644 --- a/mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py @@ -113,7 +113,6 @@ class MrTidyRetrieval(MultilingualTask, AbsTaskRetrieval): year={2021}, journal={arXiv:2108.08787}, }""", - descriptive_stats={}, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/MultiLongDocRetrieval.py b/mteb/tasks/Retrieval/multilingual/MultiLongDocRetrieval.py index b405cc532e..025a34ef6a 100644 --- a/mteb/tasks/Retrieval/multilingual/MultiLongDocRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MultiLongDocRetrieval.py @@ -38,7 +38,11 @@ def load_mldr_data( for lang in langs: lang_corpus = datasets.load_dataset( - path, f"corpus-{lang}", cache_dir=cache_dir, revision=revision + path, + f"corpus-{lang}", + cache_dir=cache_dir, + revision=revision, + trust_remote_code=True, )["corpus"] lang_corpus = {e["docid"]: {"text": e["text"]} for e in lang_corpus} lang_data = datasets.load_dataset(path, lang, cache_dir=cache_dir) @@ -65,7 +69,6 @@ class MultiLongDocRetrieval(MultilingualTask, AbsTaskRetrieval): dataset={ "path": "Shitao/MLDR", "revision": "d67138e705d963e346253a80e59676ddb418810a", - "trust_remote_code": True, }, type="Retrieval", category="s2p", @@ -98,197 +101,6 @@ class MultiLongDocRetrieval(MultilingualTask, AbsTaskRetrieval): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "ar": { - "average_document_length": 29234.48153016958, - "average_query_length": 69.27, - "num_documents": 7607, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "de": { - "average_document_length": 33771.2111, - "average_query_length": 153.63, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "en": { - "average_document_length": 13332.76764, - "average_query_length": 81.22, - "num_documents": 200000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "es": { - "average_document_length": 36567.1736990891, - "average_query_length": 123.11, - "num_documents": 9551, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "fr": { - "average_document_length": 36009.4934, - "average_query_length": 142.165, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "hi": { - "average_document_length": 18688.50788229112, - "average_query_length": 77.995, - "num_documents": 3806, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "it": { - "average_document_length": 36633.9969, - "average_query_length": 99.615, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "ja": { - "average_document_length": 14480.7508, - "average_query_length": 61.625, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "ko": { - "average_document_length": 13813.441224093263, - "average_query_length": 58.845, - "num_documents": 6176, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "pt": { - "average_document_length": 32127.576952351956, - "average_query_length": 122.275, - "num_documents": 6569, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "ru": { - "average_document_length": 35934.8756, - "average_query_length": 87.875, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "th": { - "average_document_length": 25993.2696, - "average_query_length": 107.81, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "zh": { - "average_document_length": 6039.059725, - "average_query_length": 26.79, - "num_documents": 200000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - }, - "test": { - "ar": { - "average_document_length": 29234.48153016958, - "average_query_length": 75.77, - "num_documents": 7607, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "de": { - "average_document_length": 33771.2111, - "average_query_length": 123.65, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "en": { - "average_document_length": 13332.76764, - "average_query_length": 81.33, - "num_documents": 200000, - "num_queries": 800, - "average_relevant_docs_per_query": 1.0, - }, - "es": { - "average_document_length": 36567.1736990891, - "average_query_length": 131.985, - "num_documents": 9551, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "fr": { - "average_document_length": 36009.4934, - "average_query_length": 149.795, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "hi": { - "average_document_length": 18688.50788229112, - "average_query_length": 103.76, - "num_documents": 3806, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "it": { - "average_document_length": 36633.9969, - "average_query_length": 114.595, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "ja": { - "average_document_length": 14480.7508, - "average_query_length": 55.73, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "ko": { - "average_document_length": 13813.441224093263, - "average_query_length": 58.72, - "num_documents": 6176, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "pt": { - "average_document_length": 32127.576952351956, - "average_query_length": 113.455, - "num_documents": 6569, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "ru": { - "average_document_length": 35934.8756, - "average_query_length": 94.87, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "th": { - "average_document_length": 25993.2696, - "average_query_length": 97.99, - "num_documents": 10000, - "num_queries": 200, - "average_relevant_docs_per_query": 1.0, - }, - "zh": { - "average_document_length": 6039.059725, - "average_query_length": 24.70875, - "num_documents": 200000, - "num_queries": 800, - "average_relevant_docs_per_query": 1.0, - }, - }, - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py b/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py index 893f3b51e0..6c48a6731d 100644 --- a/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py +++ b/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py @@ -86,34 +86,6 @@ class NeuCLIR2022Retrieval(MultilingualTask, AbsTaskRetrieval): journal={arXiv preprint arXiv:2304.12367}, year={2023} }""", - descriptive_stats={ - "n_samples": {"fas": 2232130, "zho": 3179323, "rus": 4627657}, - "avg_character_length": { - "test": { - "fas": { - "average_document_length": 2032.093148525817, - "average_query_length": 85.4298245614035, - "num_documents": 2232016, - "num_queries": 114, - "average_relevant_docs_per_query": 12.912280701754385, - }, - "rus": { - "average_document_length": 1757.9129983233004, - "average_query_length": 85.58771929824562, - "num_documents": 4627543, - "num_queries": 114, - "average_relevant_docs_per_query": 16.57017543859649, - }, - "zho": { - "average_document_length": 743.1426659901881, - "average_query_length": 24.17543859649123, - "num_documents": 3179209, - "num_queries": 114, - "average_relevant_docs_per_query": 18.710526315789473, - }, - } - }, - }, ) def load_data(self, **kwargs): @@ -227,41 +199,6 @@ class NeuCLIR2022RetrievalHardNegatives(MultilingualTask, AbsTaskRetrieval): journal={arXiv preprint arXiv:2304.12367}, year={2023} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 2066.9453653646488, - "average_query_length": 63.529411764705884, - "num_documents": 27931, - "num_queries": 136, - "average_relevant_docs_per_query": 40.39705882352941, - "hf_subset_descriptive_stats": { - "fas": { - "average_document_length": 2816.847782031074, - "average_query_length": 83.26666666666667, - "num_documents": 8882, - "num_queries": 45, - "average_relevant_docs_per_query": 32.71111111111111, - }, - "rus": { - "average_document_length": 2446.5574277854193, - "average_query_length": 85.56818181818181, - "num_documents": 8724, - "num_queries": 44, - "average_relevant_docs_per_query": 42.93181818181818, - }, - "zho": { - "average_document_length": 1101.0984987893462, - "average_query_length": 24.0, - "num_documents": 10325, - "num_queries": 47, - "average_relevant_docs_per_query": 45.38297872340426, - }, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py b/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py index 2cde1a6e28..88432333cc 100644 --- a/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py +++ b/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py @@ -87,34 +87,6 @@ class NeuCLIR2023Retrieval(MultilingualTask, AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": {"fas": 2232092, "zho": 3179285, "rus": 4627619}, - "avg_character_length": { - "test": { - "fas": { - "average_document_length": 2032.093148525817, - "average_query_length": 65.48684210526316, - "num_documents": 2232016, - "num_queries": 76, - "average_relevant_docs_per_query": 66.28947368421052, - }, - "rus": { - "average_document_length": 1757.9129983233004, - "average_query_length": 74.4342105263158, - "num_documents": 4627543, - "num_queries": 76, - "average_relevant_docs_per_query": 62.223684210526315, - }, - "zho": { - "average_document_length": 743.1426659901881, - "average_query_length": 22.210526315789473, - "num_documents": 3179209, - "num_queries": 76, - "average_relevant_docs_per_query": 53.68421052631579, - }, - } - }, - }, ) def load_data(self, **kwargs): @@ -230,41 +202,6 @@ class NeuCLIR2023RetrievalHardNegatives(MultilingualTask, AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 2236.175955333482, - "average_query_length": 54.10267857142857, - "num_documents": 49433, - "num_queries": 224, - "average_relevant_docs_per_query": 61.816964285714285, - "hf_subset_descriptive_stats": { - "fas": { - "average_document_length": 2895.869857421016, - "average_query_length": 65.89189189189189, - "num_documents": 15921, - "num_queries": 74, - "average_relevant_docs_per_query": 68.08108108108108, - }, - "rus": { - "average_document_length": 2724.294762109928, - "average_query_length": 74.41333333333333, - "num_documents": 16247, - "num_queries": 75, - "average_relevant_docs_per_query": 63.053333333333335, - }, - "zho": { - "average_document_length": 1168.4984071821605, - "average_query_length": 22.16, - "num_documents": 17265, - "num_queries": 75, - "average_relevant_docs_per_query": 54.4, - }, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py b/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py index b1526a42a7..6f7d188b7b 100644 --- a/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py @@ -43,6 +43,9 @@ def _load_publichealthqa_data( answer_ids = {answer: _id for _id, answer in enumerate(set(data["answer"]))} for row in data: + if row["question"] is None or row["answer"] is None: + # There are some questions and answers that are None in the original dataset, specifically in the Arabic subset. + continue question = row["question"] answer = row["answer"] query_id = f"Q{question_ids[question]}" @@ -92,69 +95,6 @@ class PublicHealthQARetrieval(MultilingualTask, AbsTaskRetrieval): publisher = { Hugging Face } } """, - descriptive_stats={ - "n_samples": {"test": 888}, - "avg_character_length": { - "test": { - "arabic": { - "average_document_length": 836.8850574712644, - "average_query_length": 79.84883720930233, - "num_documents": 87, - "num_queries": 87, - "average_relevant_docs_per_query": 1.0, - }, - "chinese": { - "average_document_length": 239.58282208588957, - "average_query_length": 24.828220858895705, - "num_documents": 163, - "num_queries": 163, - "average_relevant_docs_per_query": 1.0, - }, - "english": { - "average_document_length": 799.3430232558139, - "average_query_length": 71.78488372093024, - "num_documents": 172, - "num_queries": 172, - "average_relevant_docs_per_query": 1.0, - }, - "french": { - "average_document_length": 1021.6823529411764, - "average_query_length": 101.88235294117646, - "num_documents": 85, - "num_queries": 85, - "average_relevant_docs_per_query": 1.0, - }, - "korean": { - "average_document_length": 339.0, - "average_query_length": 36.90909090909091, - "num_documents": 77, - "num_queries": 77, - "average_relevant_docs_per_query": 1.0, - }, - "russian": { - "average_document_length": 985.1076923076923, - "average_query_length": 85.2, - "num_documents": 65, - "num_queries": 65, - "average_relevant_docs_per_query": 1.0, - }, - "spanish": { - "average_document_length": 941.1666666666666, - "average_query_length": 84.67901234567901, - "num_documents": 162, - "num_queries": 162, - "average_relevant_docs_per_query": 1.0, - }, - "vietnamese": { - "average_document_length": 704.5454545454545, - "average_query_length": 71.83116883116882, - "num_documents": 77, - "num_queries": 77, - "average_relevant_docs_per_query": 1.0, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/StatcanDialogueDatasetRetrieval.py b/mteb/tasks/Retrieval/multilingual/StatcanDialogueDatasetRetrieval.py index b9b1b030ea..ab7e178c82 100644 --- a/mteb/tasks/Retrieval/multilingual/StatcanDialogueDatasetRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/StatcanDialogueDatasetRetrieval.py @@ -100,43 +100,6 @@ class StatcanDialogueDatasetRetrieval(MultilingualTask, AbsTaskRetrieval): pages = "2799--2829", } """, - descriptive_stats={ - "n_samples": {"dev": 1000, "test": 1011, "corpus": 5907}, - "avg_character_length": { - "dev": { - "english": { - "average_document_length": 6535.865413915693, - "average_query_length": 6.869244935543278, - "num_documents": 5907, - "num_queries": 543, - "average_relevant_docs_per_query": 1.4714548802946592, - }, - "french": { - "average_document_length": 7078.072794988996, - "average_query_length": 6.860655737704918, - "num_documents": 5907, - "num_queries": 122, - "average_relevant_docs_per_query": 1.6475409836065573, - }, - }, - "test": { - "english": { - "average_document_length": 6535.865413915693, - "average_query_length": 7.650994575045208, - "num_documents": 5907, - "num_queries": 553, - "average_relevant_docs_per_query": 1.573236889692586, - }, - "french": { - "average_document_length": 7078.072794988996, - "average_query_length": 5.907407407407407, - "num_documents": 5907, - "num_queries": 108, - "average_relevant_docs_per_query": 1.3055555555555556, - }, - }, - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/WikipediaRetrievalMultilingual.py b/mteb/tasks/Retrieval/multilingual/WikipediaRetrievalMultilingual.py index b4356ec0cb..a78fa4110d 100644 --- a/mteb/tasks/Retrieval/multilingual/WikipediaRetrievalMultilingual.py +++ b/mteb/tasks/Retrieval/multilingual/WikipediaRetrievalMultilingual.py @@ -112,142 +112,6 @@ class WikipediaRetrievalMultilingual(MultilingualTask, AbsTaskRetrieval): dialect=[], sample_creation="LM-generated and verified", bibtex_citation="", - descriptive_stats={ - "n_samples": { - "en": 1500, - "de": 1500, - "it": 1500, - "pt": 1500, - "nl": 1500, - "cs": 1500, - "ro": 1500, - "bg": 1500, - "sr": 1500, - "fi": 1500, - "da": 1500, - "fa": 1500, - "hi": 1500, - "bn": 1500, - "no": 1500, - "sv": 1500, - }, - "avg_character_length": { - "test": { - "bg": { - "average_document_length": 374.376, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "bn": { - "average_document_length": 394.05044444444445, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "cs": { - "average_document_length": 369.9831111111111, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "da": { - "average_document_length": 345.2597037037037, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "de": { - "average_document_length": 398.4137777777778, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "en": { - "average_document_length": 452.9871111111111, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "fa": { - "average_document_length": 345.1568888888889, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "fi": { - "average_document_length": 379.71237037037037, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "hi": { - "average_document_length": 410.72540740740743, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "it": { - "average_document_length": 393.73437037037036, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "nl": { - "average_document_length": 375.6695555555556, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "pt": { - "average_document_length": 398.27237037037037, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "ro": { - "average_document_length": 348.3817037037037, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "sr": { - "average_document_length": 384.3131851851852, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "no": { - "average_document_length": 366.93733333333336, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - "sv": { - "average_document_length": 369.340962962963, - "average_query_length": 1.0, - "num_documents": 13500, - "num_queries": 1500, - "average_relevant_docs_per_query": 1.0, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/XMarketRetrieval.py b/mteb/tasks/Retrieval/multilingual/XMarketRetrieval.py index 6f9981498f..01d240eb9d 100644 --- a/mteb/tasks/Retrieval/multilingual/XMarketRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/XMarketRetrieval.py @@ -30,6 +30,7 @@ def _load_xmarket_data( languages=[lang], split=split, cache_dir=cache_dir, + trust_remote_code=True, ) query_rows = datasets.load_dataset( path, @@ -38,6 +39,7 @@ def _load_xmarket_data( revision=revision, split=split, cache_dir=cache_dir, + trust_remote_code=True, ) qrels_rows = datasets.load_dataset( path, @@ -46,6 +48,7 @@ def _load_xmarket_data( revision=revision, split=split, cache_dir=cache_dir, + trust_remote_code=True, ) corpus[lang][split] = {row["_id"]: row for row in corpus_rows} @@ -69,7 +72,6 @@ class XMarket(MultilingualTask, AbsTaskRetrieval): dataset={ "path": "jinaai/xmarket_ml", "revision": "dfe57acff5b62c23732a7b7d3e3fb84ff501708b", - "trust_remote_code": True, }, type="Retrieval", category="s2p", @@ -93,34 +95,6 @@ class XMarket(MultilingualTask, AbsTaskRetrieval): author={Bonab, Hamed and Aliannejadi, Mohammad and Vardasbi, Ali and Kanoulas, Evangelos and Allan, James}, year={2021}, month=oct, collection={CIKM ’21} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "de": { - "average_document_length": 187.4061197288943, - "average_query_length": 15.717612088184294, - "num_documents": 70526, - "num_queries": 4037, - "average_relevant_docs_per_query": 54.3522417636859, - }, - "en": { - "average_document_length": 452.792089662076, - "average_query_length": 15.881635344543357, - "num_documents": 218777, - "num_queries": 9099, - "average_relevant_docs_per_query": 85.43719090009891, - }, - "es": { - "average_document_length": 279.67909262759923, - "average_query_length": 19.97062937062937, - "num_documents": 39675, - "num_queries": 3575, - "average_relevant_docs_per_query": 36.01006993006993, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py b/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py index cb5d7d618f..72cbbd6dab 100644 --- a/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py @@ -92,265 +92,6 @@ class XPQARetrieval(AbsTaskRetrieval, MultilingualTask): pages={103--115}, year={2023} }""", - descriptive_stats={ - "n_samples": {"test": 19801}, - "avg_character_length": { - "test": { - "ara-ara": { - "average_document_length": 61.88361204013378, - "average_query_length": 29.688, - "num_documents": 1495, - "num_queries": 750, - "average_relevant_docs_per_query": 2.004, - }, - "eng-ara": { - "average_document_length": 125.26940639269407, - "average_query_length": 29.688, - "num_documents": 1533, - "num_queries": 750, - "average_relevant_docs_per_query": 2.058666666666667, - }, - "ara-eng": { - "average_document_length": 61.88361204013378, - "average_query_length": 39.5188679245283, - "num_documents": 1495, - "num_queries": 742, - "average_relevant_docs_per_query": 2.024258760107817, - }, - "deu-deu": { - "average_document_length": 69.54807692307692, - "average_query_length": 55.51827676240209, - "num_documents": 1248, - "num_queries": 766, - "average_relevant_docs_per_query": 1.6318537859007833, - }, - "eng-deu": { - "average_document_length": 115.77118078719145, - "average_query_length": 55.51827676240209, - "num_documents": 1499, - "num_queries": 766, - "average_relevant_docs_per_query": 1.9634464751958225, - }, - "deu-eng": { - "average_document_length": 69.54807692307692, - "average_query_length": 51.88903394255875, - "num_documents": 1248, - "num_queries": 766, - "average_relevant_docs_per_query": 1.6318537859007833, - }, - "spa-spa": { - "average_document_length": 68.27511591962906, - "average_query_length": 46.711223203026485, - "num_documents": 1941, - "num_queries": 793, - "average_relevant_docs_per_query": 2.4489281210592684, - }, - "eng-spa": { - "average_document_length": 123.43698347107438, - "average_query_length": 46.711223203026485, - "num_documents": 1936, - "num_queries": 793, - "average_relevant_docs_per_query": 2.472887767969735, - }, - "spa-eng": { - "average_document_length": 68.27511591962906, - "average_query_length": 47.21059268600252, - "num_documents": 1941, - "num_queries": 793, - "average_relevant_docs_per_query": 2.4489281210592684, - }, - "fra-fra": { - "average_document_length": 76.99354005167959, - "average_query_length": 56.0520694259012, - "num_documents": 1548, - "num_queries": 749, - "average_relevant_docs_per_query": 2.069425901201602, - }, - "eng-fra": { - "average_document_length": 137.31242532855435, - "average_query_length": 56.0520694259012, - "num_documents": 1674, - "num_queries": 749, - "average_relevant_docs_per_query": 2.248331108144192, - }, - "fra-eng": { - "average_document_length": 76.99354005167959, - "average_query_length": 49.58744993324433, - "num_documents": 1548, - "num_queries": 749, - "average_relevant_docs_per_query": 2.069425901201602, - }, - "hin-hin": { - "average_document_length": 47.20783373301359, - "average_query_length": 33.47783783783784, - "num_documents": 1251, - "num_queries": 925, - "average_relevant_docs_per_query": 1.3902702702702703, - }, - "eng-hin": { - "average_document_length": 106.67662682602922, - "average_query_length": 33.47783783783784, - "num_documents": 1506, - "num_queries": 925, - "average_relevant_docs_per_query": 1.8054054054054054, - }, - "hin-eng": { - "average_document_length": 47.20783373301359, - "average_query_length": 34.98574561403509, - "num_documents": 1251, - "num_queries": 912, - "average_relevant_docs_per_query": 1.4100877192982457, - }, - "ita-ita": { - "average_document_length": 59.778301886792455, - "average_query_length": 49.14932126696833, - "num_documents": 1272, - "num_queries": 663, - "average_relevant_docs_per_query": 1.9245852187028658, - }, - "eng-ita": { - "average_document_length": 123.07302075326672, - "average_query_length": 49.14932126696833, - "num_documents": 1301, - "num_queries": 663, - "average_relevant_docs_per_query": 1.9849170437405732, - }, - "ita-eng": { - "average_document_length": 59.778301886792455, - "average_query_length": 49.040723981900456, - "num_documents": 1272, - "num_queries": 663, - "average_relevant_docs_per_query": 1.9245852187028658, - }, - "jpn-jpn": { - "average_document_length": 41.030605871330415, - "average_query_length": 23.296969696969697, - "num_documents": 1601, - "num_queries": 825, - "average_relevant_docs_per_query": 1.9406060606060607, - }, - "eng-jpn": { - "average_document_length": 126.2647564469914, - "average_query_length": 23.296969696969697, - "num_documents": 1745, - "num_queries": 825, - "average_relevant_docs_per_query": 2.1187878787878787, - }, - "jpn-eng": { - "average_document_length": 41.030605871330415, - "average_query_length": 51.416058394160586, - "num_documents": 1601, - "num_queries": 822, - "average_relevant_docs_per_query": 1.9476885644768855, - }, - "kor-kor": { - "average_document_length": 31.22722159730034, - "average_query_length": 21.81804281345566, - "num_documents": 889, - "num_queries": 654, - "average_relevant_docs_per_query": 1.5642201834862386, - }, - "eng-kor": { - "average_document_length": 112.41231822070145, - "average_query_length": 21.81804281345566, - "num_documents": 1169, - "num_queries": 654, - "average_relevant_docs_per_query": 1.952599388379205, - }, - "kor-eng": { - "average_document_length": 31.22722159730034, - "average_query_length": 43.9527687296417, - "num_documents": 889, - "num_queries": 614, - "average_relevant_docs_per_query": 1.6661237785016287, - }, - "pol-pol": { - "average_document_length": 50.66814439518683, - "average_query_length": 53.72101910828025, - "num_documents": 1579, - "num_queries": 785, - "average_relevant_docs_per_query": 2.080254777070064, - }, - "eng-pol": { - "average_document_length": 112.96919566457501, - "average_query_length": 53.72101910828025, - "num_documents": 1753, - "num_queries": 785, - "average_relevant_docs_per_query": 2.385987261146497, - }, - "pol-eng": { - "average_document_length": 50.66814439518683, - "average_query_length": 54.1994851994852, - "num_documents": 1579, - "num_queries": 777, - "average_relevant_docs_per_query": 2.101673101673102, - }, - "por-por": { - "average_document_length": 75.9845869297164, - "average_query_length": 42.58875, - "num_documents": 1622, - "num_queries": 800, - "average_relevant_docs_per_query": 2.14, - }, - "eng-por": { - "average_document_length": 111.42525930445393, - "average_query_length": 42.58875, - "num_documents": 1639, - "num_queries": 800, - "average_relevant_docs_per_query": 2.21875, - }, - "por-eng": { - "average_document_length": 75.9845869297164, - "average_query_length": 46.57967377666248, - "num_documents": 1622, - "num_queries": 797, - "average_relevant_docs_per_query": 2.148055207026349, - }, - "tam-tam": { - "average_document_length": 64.89019607843137, - "average_query_length": 33.267263427109974, - "num_documents": 1275, - "num_queries": 782, - "average_relevant_docs_per_query": 1.6994884910485935, - }, - "eng-tam": { - "average_document_length": 96.96361185983828, - "average_query_length": 33.267263427109974, - "num_documents": 1484, - "num_queries": 782, - "average_relevant_docs_per_query": 2.0255754475703327, - }, - "tam-eng": { - "average_document_length": 64.89019607843137, - "average_query_length": 34.777633289986994, - "num_documents": 1275, - "num_queries": 769, - "average_relevant_docs_per_query": 1.728218465539662, - }, - "cmn-cmn": { - "average_document_length": 20.958944281524925, - "average_query_length": 12.21116504854369, - "num_documents": 1705, - "num_queries": 824, - "average_relevant_docs_per_query": 2.0716019417475726, - }, - "eng-cmn": { - "average_document_length": 108.31593874078276, - "average_query_length": 12.21116504854369, - "num_documents": 1763, - "num_queries": 824, - "average_relevant_docs_per_query": 2.2633495145631066, - }, - "cmn-eng": { - "average_document_length": 20.958944281524925, - "average_query_length": 41.24390243902439, - "num_documents": 1705, - "num_queries": 820, - "average_relevant_docs_per_query": 2.0817073170731706, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py b/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py index a4772002cf..4d952896e3 100644 --- a/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py @@ -64,97 +64,6 @@ class XQuADRetrieval(MultilingualTask, AbsTaskRetrieval): year={2021}, url={https://openreview.net/forum?id=JH61CD7afTv} }""", - descriptive_stats={ - "n_samples": {"test": 1190}, - "avg_character_length": { - "validation": { - "ar": { - "average_document_length": 683.4666666666667, - "average_query_length": 53.327993254637434, - "num_documents": 240, - "num_queries": 1186, - "average_relevant_docs_per_query": 1.0, - }, - "de": { - "average_document_length": 894.0666666666667, - "average_query_length": 69.04318374259103, - "num_documents": 240, - "num_queries": 1181, - "average_relevant_docs_per_query": 1.0, - }, - "el": { - "average_document_length": 894.3791666666667, - "average_query_length": 68.61317567567568, - "num_documents": 240, - "num_queries": 1184, - "average_relevant_docs_per_query": 1.0, - }, - "en": { - "average_document_length": 784.8333333333334, - "average_query_length": 61.25063291139241, - "num_documents": 240, - "num_queries": 1185, - "average_relevant_docs_per_query": 1.0, - }, - "es": { - "average_document_length": 883.8041666666667, - "average_query_length": 68.23817567567568, - "num_documents": 240, - "num_queries": 1184, - "average_relevant_docs_per_query": 1.0, - }, - "hi": { - "average_document_length": 764.9416666666667, - "average_query_length": 59.684699915469146, - "num_documents": 240, - "num_queries": 1183, - "average_relevant_docs_per_query": 1.0, - }, - "ro": { - "average_document_length": 878.4458333333333, - "average_query_length": 67.17229729729729, - "num_documents": 240, - "num_queries": 1184, - "average_relevant_docs_per_query": 1.0, - }, - "ru": { - "average_document_length": 850.1875, - "average_query_length": 64.94261603375527, - "num_documents": 240, - "num_queries": 1185, - "average_relevant_docs_per_query": 1.0, - }, - "th": { - "average_document_length": 736.7583333333333, - "average_query_length": 55.103389830508476, - "num_documents": 240, - "num_queries": 1180, - "average_relevant_docs_per_query": 1.0, - }, - "tr": { - "average_document_length": 788.3, - "average_query_length": 60.876689189189186, - "num_documents": 240, - "num_queries": 1184, - "average_relevant_docs_per_query": 1.0, - }, - "vi": { - "average_document_length": 803.9083333333333, - "average_query_length": 61.62859560067682, - "num_documents": 240, - "num_queries": 1182, - "average_relevant_docs_per_query": 1.0, - }, - "zh": { - "average_document_length": 252.4, - "average_query_length": 18.460626587637595, - "num_documents": 240, - "num_queries": 1181, - "average_relevant_docs_per_query": 1.0, - }, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/nob/norquad.py b/mteb/tasks/Retrieval/nob/norquad.py index ce11b4b710..f578cefec8 100644 --- a/mteb/tasks/Retrieval/nob/norquad.py +++ b/mteb/tasks/Retrieval/nob/norquad.py @@ -46,17 +46,8 @@ class NorQuadRetrieval(AbsTaskRetrieval): pages = "159--168", abstract = "In this paper we present NorQuAD: the first Norwegian question answering dataset for machine reading comprehension. The dataset consists of 4,752 manually created question-answer pairs. We here detail the data collection procedure and present statistics of the dataset. We also benchmark several multilingual and Norwegian monolingual language models on the dataset and compare them against human performance. The dataset will be made freely available.", }""", - descriptive_stats={ - "n_samples": {"test": 2602}, - "avg_character_length": { - "test": { - "average_document_length": 214.5114503816794, - "average_query_length": 47.896484375, - "num_documents": 1048, - "num_queries": 1024, - "average_relevant_docs_per_query": 2.0, - } - }, + prompt={ + "query": "Given a question in Norwegian, retrieve the answer from Wikipedia articles" }, ) diff --git a/mteb/tasks/Retrieval/nob/snl_retrieval.py b/mteb/tasks/Retrieval/nob/snl_retrieval.py index 3c8045fe6d..cf64834329 100644 --- a/mteb/tasks/Retrieval/nob/snl_retrieval.py +++ b/mteb/tasks/Retrieval/nob/snl_retrieval.py @@ -33,18 +33,7 @@ class SNLRetrieval(AbsTaskRetrieval): year={2023}, school={Norwegian University of Life Sciences, {\AA}s} }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": { - "test": { - "average_document_length": 1986.9453846153847, - "average_query_length": 14.906153846153845, - "num_documents": 1300, - "num_queries": 1300, - "average_relevant_docs_per_query": 1.0, - }, - }, - }, + prompt={"query": "Given a lexicon headline in Norwegian, retrieve its article"}, task_subtypes=["Article retrieval"], ) diff --git a/mteb/tasks/Retrieval/pol/ArguAnaPLRetrieval.py b/mteb/tasks/Retrieval/pol/ArguAnaPLRetrieval.py index c158870c09..342f727144 100644 --- a/mteb/tasks/Retrieval/pol/ArguAnaPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/ArguAnaPLRetrieval.py @@ -38,16 +38,4 @@ class ArguAnaPL(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1060.702674659903, - "average_query_length": 1224.8022759601706, - "num_documents": 8674, - "num_queries": 1406, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/pol/DBPediaPLRetrieval.py b/mteb/tasks/Retrieval/pol/DBPediaPLRetrieval.py index 8d01491463..6b96336365 100644 --- a/mteb/tasks/Retrieval/pol/DBPediaPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/DBPediaPLRetrieval.py @@ -38,18 +38,6 @@ class DBPediaPL(AbsTaskRetrieval): doi = {10.1145/3077136.3080751}, publisher = {ACM} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 311.7007956561823, - "average_query_length": 35.45, - "num_documents": 4635922, - "num_queries": 400, - "average_relevant_docs_per_query": 38.215, - } - }, - }, ) @@ -86,16 +74,4 @@ class DBPediaPLHardNegatives(AbsTaskRetrieval): doi = {10.1145/3077136.3080751}, publisher = {ACM} }""", - descriptive_stats={ - "n_samples": {"test": 400}, - "avg_character_length": { - "test": { - "average_document_length": 363.468546000768, - "average_query_length": 35.45, - "num_documents": 88542, - "num_queries": 400, - "average_relevant_docs_per_query": 38.215, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/pol/FiQAPLRetrieval.py b/mteb/tasks/Retrieval/pol/FiQAPLRetrieval.py index 882d5d9e48..0a125f5e4f 100644 --- a/mteb/tasks/Retrieval/pol/FiQAPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/FiQAPLRetrieval.py @@ -38,16 +38,4 @@ class FiQAPLRetrieval(AbsTaskRetrieval): year={2021}, url={https://openreview.net/forum?id=wCu6T5xFjeJ} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 795.2371699226205, - "average_query_length": 70.00771604938272, - "num_documents": 57638, - "num_queries": 648, - "average_relevant_docs_per_query": 2.632716049382716, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/pol/HotpotQAPLRetrieval.py b/mteb/tasks/Retrieval/pol/HotpotQAPLRetrieval.py index c9bab26a2f..de9c8c267f 100644 --- a/mteb/tasks/Retrieval/pol/HotpotQAPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/HotpotQAPLRetrieval.py @@ -36,18 +36,6 @@ class HotpotQAPL(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 292.26835882093405, - "average_query_length": 94.64064821066847, - "num_documents": 5233329, - "num_queries": 7405, - "average_relevant_docs_per_query": 2.0, - } - }, - }, ) @@ -82,16 +70,4 @@ class HotpotQAPLHardNegatives(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": { - "test": { - "average_document_length": 438.3888210025661, - "average_query_length": 95.161, - "num_documents": 212774, - "num_queries": 1000, - "average_relevant_docs_per_query": 2.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/pol/MSMARCOPLRetrieval.py b/mteb/tasks/Retrieval/pol/MSMARCOPLRetrieval.py index a3cd81f620..91db471a84 100644 --- a/mteb/tasks/Retrieval/pol/MSMARCOPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/MSMARCOPLRetrieval.py @@ -38,18 +38,6 @@ class MSMARCOPL(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 349.3574939240471, - "average_query_length": 33.02325581395349, - "num_documents": 8841823, - "num_queries": 43, - "average_relevant_docs_per_query": 95.3953488372093, - } - }, - }, ) @@ -86,16 +74,4 @@ class MSMARCOPLHardNegatives(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": {"test": 43}, - "avg_character_length": { - "test": { - "average_document_length": 382.3476426537285, - "average_query_length": 33.02325581395349, - "num_documents": 9481, - "num_queries": 43, - "average_relevant_docs_per_query": 95.3953488372093, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/pol/NFCorpusPLRetrieval.py b/mteb/tasks/Retrieval/pol/NFCorpusPLRetrieval.py index bffb4a3116..7da4941ede 100644 --- a/mteb/tasks/Retrieval/pol/NFCorpusPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/NFCorpusPLRetrieval.py @@ -36,16 +36,4 @@ class NFCorpusPL(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1652.1926782273604, - "average_query_length": 24.390092879256965, - "num_documents": 3633, - "num_queries": 323, - "average_relevant_docs_per_query": 38.18575851393189, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/pol/NQPLRetrieval.py b/mteb/tasks/Retrieval/pol/NQPLRetrieval.py index 697778fef4..ff0d4b03a4 100644 --- a/mteb/tasks/Retrieval/pol/NQPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/NQPLRetrieval.py @@ -36,18 +36,6 @@ class NQPL(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 502.14302128535564, - "average_query_length": 48.31662804171495, - "num_documents": 2681468, - "num_queries": 3452, - "average_relevant_docs_per_query": 1.2169756662804172, - } - }, - }, ) @@ -82,16 +70,4 @@ class NQPLHardNegatives(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": { - "test": { - "average_document_length": 610.7449138094336, - "average_query_length": 48.381, - "num_documents": 184765, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.213, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/pol/QuoraPLRetrieval.py b/mteb/tasks/Retrieval/pol/QuoraPLRetrieval.py index 17b32f5a0d..632a333a9d 100644 --- a/mteb/tasks/Retrieval/pol/QuoraPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/QuoraPLRetrieval.py @@ -36,25 +36,6 @@ class QuoraPLRetrieval(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "validation": { - "average_document_length": 65.82473022253414, - "average_query_length": 54.6006, - "num_documents": 522931, - "num_queries": 5000, - "average_relevant_docs_per_query": 1.5252, - }, - "test": { - "average_document_length": 65.82473022253414, - "average_query_length": 54.5354, - "num_documents": 522931, - "num_queries": 10000, - "average_relevant_docs_per_query": 1.5675, - }, - }, - }, ) @@ -89,16 +70,4 @@ class QuoraPLRetrievalHardNegatives(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": { - "test": { - "average_document_length": 67.77529631287385, - "average_query_length": 53.846, - "num_documents": 172031, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.641, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/pol/SCIDOCSPLRetrieval.py b/mteb/tasks/Retrieval/pol/SCIDOCSPLRetrieval.py index 7c9776fae0..218a4ee5b5 100644 --- a/mteb/tasks/Retrieval/pol/SCIDOCSPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/SCIDOCSPLRetrieval.py @@ -36,16 +36,4 @@ class SCIDOCSPL(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1270.0791986592353, - "average_query_length": 80.671, - "num_documents": 25657, - "num_queries": 1000, - "average_relevant_docs_per_query": 4.928, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/pol/SciFactPLRetrieval.py b/mteb/tasks/Retrieval/pol/SciFactPLRetrieval.py index fdcce6bdfc..92d61b42bd 100644 --- a/mteb/tasks/Retrieval/pol/SciFactPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/SciFactPLRetrieval.py @@ -22,7 +22,7 @@ class SciFactPL(AbsTaskRetrieval): eval_langs=["pol-Latn"], main_score="ndcg_at_10", date=None, - domains=None, + domains=["Academic", "Medical", "Written"], task_subtypes=None, license=None, annotations_creators=None, @@ -36,16 +36,4 @@ class SciFactPL(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1553.5178468068686, - "average_query_length": 95.44, - "num_documents": 5183, - "num_queries": 300, - "average_relevant_docs_per_query": 1.13, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/pol/TRECCOVIDPLRetrieval.py b/mteb/tasks/Retrieval/pol/TRECCOVIDPLRetrieval.py index 04778c0227..f9f331191a 100644 --- a/mteb/tasks/Retrieval/pol/TRECCOVIDPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/TRECCOVIDPLRetrieval.py @@ -25,7 +25,7 @@ class TRECCOVIDPL(AbsTaskRetrieval): "2019-12-01", "2022-12-31", ), # approximate date of covid pandemic start and end (best guess) - domains=["Academic", "Non-fiction", "Written"], + domains=["Academic", "Medical", "Non-fiction", "Written"], task_subtypes=["Article retrieval"], license="not specified", annotations_creators="derived", @@ -39,16 +39,4 @@ class TRECCOVIDPL(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 1159.8020276422385, - "average_query_length": 69.42, - "num_documents": 171332, - "num_queries": 50, - "average_relevant_docs_per_query": 493.5, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/rus/RiaNewsRetrieval.py b/mteb/tasks/Retrieval/rus/RiaNewsRetrieval.py index ffd0c919b2..6ccc6393ce 100644 --- a/mteb/tasks/Retrieval/rus/RiaNewsRetrieval.py +++ b/mteb/tasks/Retrieval/rus/RiaNewsRetrieval.py @@ -35,18 +35,7 @@ class RiaNewsRetrieval(AbsTaskRetrieval): booktitle={Proceedings of the 41st European Conference on Information Retrieval}, year={2019} }""", - descriptive_stats={ - "n_samples": {"test": 10000}, - "avg_character_length": { - "test": { - "average_document_length": 1165.6429557148213, - "average_query_length": 62.4029, - "num_documents": 704344, - "num_queries": 10000, - "average_relevant_docs_per_query": 1.0, - } - }, - }, + prompt={"query": "Given a news title, retrieve relevant news article"}, ) @@ -80,16 +69,4 @@ class RiaNewsRetrievalHardNegatives(AbsTaskRetrieval): booktitle={Proceedings of the 41st European Conference on Information Retrieval}, year={2019} }""", - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": { - "test": { - "average_document_length": 1225.7253146619116, - "average_query_length": 62.338, - "num_documents": 191237, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) diff --git a/mteb/tasks/Retrieval/rus/RuBQRetrieval.py b/mteb/tasks/Retrieval/rus/RuBQRetrieval.py index bb56ca10ac..3bb1bb35e9 100644 --- a/mteb/tasks/Retrieval/rus/RuBQRetrieval.py +++ b/mteb/tasks/Retrieval/rus/RuBQRetrieval.py @@ -36,16 +36,7 @@ class RuBQRetrieval(AbsTaskRetrieval): year={2021}, pages={532--547} }""", - descriptive_stats={ - "n_samples": {"test": 2845}, - "avg_character_length": { - "test": { - "average_document_length": 448.94659134903037, - "average_query_length": 45.29609929078014, - "num_documents": 56826, - "num_queries": 1692, - "average_relevant_docs_per_query": 1.6814420803782506, - } - }, + prompt={ + "query": "Given a question, retrieve Wikipedia passages that answer the question" }, ) diff --git a/mteb/tasks/Retrieval/slk/SKQuadRetrieval.py b/mteb/tasks/Retrieval/slk/SKQuadRetrieval.py index 2795ef82d3..3db21e1558 100644 --- a/mteb/tasks/Retrieval/slk/SKQuadRetrieval.py +++ b/mteb/tasks/Retrieval/slk/SKQuadRetrieval.py @@ -34,18 +34,6 @@ class SKQuadRetrieval(AbsTaskRetrieval): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 1134}, - "avg_character_length": { - "test": { - "average_document_length": 1180.5071792496526, - "average_query_length": 53.63403880070547, - "num_documents": 6477, - "num_queries": 1134, - "average_relevant_docs_per_query": 11, - } - }, - }, ) def load_data(self, eval_splits=None, **kwargs): diff --git a/mteb/tasks/Retrieval/slk/SlovakSumRetrieval.py b/mteb/tasks/Retrieval/slk/SlovakSumRetrieval.py index 4158f908a3..0b26fd1079 100644 --- a/mteb/tasks/Retrieval/slk/SlovakSumRetrieval.py +++ b/mteb/tasks/Retrieval/slk/SlovakSumRetrieval.py @@ -42,18 +42,6 @@ class SlovakSumRetrieval(AbsTaskRetrieval): date = {2024}, } """, - descriptive_stats={ - "n_samples": {"test": 600}, - "avg_character_length": { - "test": { - "average_document_length": 2156.445, - "average_query_length": 143.59833333333333, - "num_documents": 600, - "num_queries": 600, - "average_relevant_docs_per_query": 1.0, - } - }, - }, ) def load_data(self, **kwargs): @@ -61,7 +49,7 @@ def load_data(self, **kwargs): return self.corpus, self.queries, self.relevant_docs = {}, {}, {} dataset_path = self.metadata_dict["dataset"]["path"] - n_sample = self.metadata_dict["descriptive_stats"]["n_samples"]["test"] + n_sample = 600 for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): split_ds = datasets.load_dataset( diff --git a/mteb/tasks/Retrieval/spa/SpanishPassageRetrievalS2P.py b/mteb/tasks/Retrieval/spa/SpanishPassageRetrievalS2P.py index f366d60ad9..8ef0681dcd 100644 --- a/mteb/tasks/Retrieval/spa/SpanishPassageRetrievalS2P.py +++ b/mteb/tasks/Retrieval/spa/SpanishPassageRetrievalS2P.py @@ -53,18 +53,6 @@ class SpanishPassageRetrievalS2P(AbsTaskRetrieval): isbn="978-3-030-15719-7" } """, - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 2635.217893792966, - "average_query_length": 67.55688622754491, - "num_documents": 10037, - "num_queries": 167, - "average_relevant_docs_per_query": 6.053892215568863, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/spa/SpanishPassageRetrievalS2S.py b/mteb/tasks/Retrieval/spa/SpanishPassageRetrievalS2S.py index ac80ac8aa8..86b45f1f4c 100644 --- a/mteb/tasks/Retrieval/spa/SpanishPassageRetrievalS2S.py +++ b/mteb/tasks/Retrieval/spa/SpanishPassageRetrievalS2S.py @@ -53,18 +53,6 @@ class SpanishPassageRetrievalS2S(AbsTaskRetrieval): isbn="978-3-030-15719-7" } """, - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 434.5924528301887, - "average_query_length": 67.55688622754491, - "num_documents": 265, - "num_queries": 167, - "average_relevant_docs_per_query": 7.718562874251497, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py b/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py index f01cb25db8..eccc7d9ab7 100644 --- a/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py +++ b/mteb/tasks/Retrieval/swe/SweFaqRetrieval.py @@ -37,18 +37,7 @@ class SweFaqRetrieval(AbsTaskRetrieval): pages={8137--8153}, year={2023} }""", # for the benchmark in which this dataset is used - descriptive_stats={ - "n_samples": {"test": 1024}, - "avg_character_length": { - "test": { - "average_document_length": 319.8473581213307, - "average_query_length": 70.51461988304094, - "num_documents": 511, - "num_queries": 513, - "average_relevant_docs_per_query": 1.0, - } - }, - }, + prompt={"query": "Retrieve answers given questions in Swedish"}, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/swe/SwednRetrieval.py b/mteb/tasks/Retrieval/swe/SwednRetrieval.py index 381961542c..acd7b65de7 100644 --- a/mteb/tasks/Retrieval/swe/SwednRetrieval.py +++ b/mteb/tasks/Retrieval/swe/SwednRetrieval.py @@ -36,17 +36,8 @@ class SwednRetrieval(AbsTaskRetrieval): booktitle={Proceedings of CLARIN Annual Conference}, year={2021} }""", - descriptive_stats={ - "n_samples": {"test": 2048}, - "avg_character_length": { - "test": { - "average_document_length": 2896.519550342131, - "average_query_length": 45.876953125, - "num_documents": 2046, - "num_queries": 1024, - "average_relevant_docs_per_query": 2.0, - } - }, + prompt={ + "query": "Given a Swedish news headline retrieve summaries or news articles" }, ) diff --git a/mteb/tasks/Retrieval/tur/TurHistQuad.py b/mteb/tasks/Retrieval/tur/TurHistQuad.py index d896e36fa0..e7aa10ac96 100644 --- a/mteb/tasks/Retrieval/tur/TurHistQuad.py +++ b/mteb/tasks/Retrieval/tur/TurHistQuad.py @@ -41,18 +41,6 @@ class TurHistQuadRetrieval(AbsTaskRetrieval): doi={10.1109/UBMK52708.2021.9559013}} """, - descriptive_stats={ - "n_samples": {"test": 1330}, - "avg_character_length": { - "test": { - "average_document_length": 172.12118713932398, - "average_query_length": 62.5302734375, - "num_documents": 1213, - "num_queries": 1024, - "average_relevant_docs_per_query": 2.0, - } - }, - }, ) def load_data(self, **kwargs) -> None: diff --git a/mteb/tasks/Retrieval/vie/VieQuADRetrieval.py b/mteb/tasks/Retrieval/vie/VieQuADRetrieval.py index 1b7cab08dd..07ec5aba8b 100644 --- a/mteb/tasks/Retrieval/vie/VieQuADRetrieval.py +++ b/mteb/tasks/Retrieval/vie/VieQuADRetrieval.py @@ -50,18 +50,6 @@ class VieQuADRetrieval(AbsTaskRetrieval): url = "https://aclanthology.org/2020.coling-main.233", doi = "10.18653/v1/2020.coling-main.233", pages = "2595--2605"}""", - descriptive_stats={ - "n_samples": {"validation": TEST_SAMPLES}, - "avg_character_length": { - "validation": { - "average_document_length": 222.61244979919678, - "average_query_length": 65.51513671875, - "num_documents": 2490, - "num_queries": 2048, - "average_relevant_docs_per_query": 2.0, - } - }, - }, ) def load_data(self, **kwargs): diff --git a/mteb/tasks/Retrieval/zho/CMTEBRetrieval.py b/mteb/tasks/Retrieval/zho/CMTEBRetrieval.py index a3895b387d..ad26652ccd 100644 --- a/mteb/tasks/Retrieval/zho/CMTEBRetrieval.py +++ b/mteb/tasks/Retrieval/zho/CMTEBRetrieval.py @@ -59,17 +59,8 @@ class T2Retrieval(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.IR} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "average_document_length": 874.1184182791619, - "average_query_length": 10.938847974750132, - "num_documents": 118605, - "num_queries": 22812, - "average_relevant_docs_per_query": 5.213571804313519, - } - }, + prompt={ + "query": "Given a Chinese search query, retrieve web passages that answer the question" }, ) @@ -119,17 +110,8 @@ class MMarcoRetrieval(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "average_document_length": 114.41787048392986, - "average_query_length": 10.51131805157593, - "num_documents": 106813, - "num_queries": 6980, - "average_relevant_docs_per_query": 1.0654727793696275, - } - }, + prompt={ + "query": "Given a web search query, retrieve relevant passages that answer the query" }, ) @@ -177,17 +159,8 @@ class DuRetrieval(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "average_document_length": 331.3219967800322, - "average_query_length": 9.289, - "num_documents": 100001, - "num_queries": 2000, - "average_relevant_docs_per_query": 4.9195, - } - }, + prompt={ + "query": "Given a Chinese search query, retrieve web passages that answer the question" }, ) @@ -228,17 +201,8 @@ class CovidRetrieval(AbsTaskRetrieval): dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "average_document_length": 332.4152658473415, - "average_query_length": 25.9304531085353, - "num_documents": 100001, - "num_queries": 949, - "average_relevant_docs_per_query": 1.0105374077976819, - } - }, + prompt={ + "query": "Given a question on COVID-19, retrieve news articles that answer the question" }, ) @@ -272,24 +236,15 @@ class CmedqaRetrieval(AbsTaskRetrieval): eval_langs=["cmn-Hans"], main_score="ndcg_at_10", date=None, - domains=None, + domains=["Medical", "Written"], task_subtypes=None, license=None, annotations_creators=None, dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "average_document_length": 307.7710222897771, - "average_query_length": 48.470367591897976, - "num_documents": 100001, - "num_queries": 3999, - "average_relevant_docs_per_query": 1.86271567891973, - } - }, + prompt={ + "query": "Given a Chinese community medical question, retrieve replies that best answer the question" }, ) @@ -332,17 +287,8 @@ class EcomRetrieval(AbsTaskRetrieval): dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "average_document_length": 32.98041664189015, - "average_query_length": 6.798, - "num_documents": 100902, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given a user query from an e-commerce website, retrieve description sentences of relevant products" }, ) @@ -385,17 +331,8 @@ class MedicalRetrieval(AbsTaskRetrieval): dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "average_document_length": 122.04231725066585, - "average_query_length": 17.938, - "num_documents": 100999, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given a medical question, retrieve user replies that best answer the question" }, ) @@ -438,17 +375,8 @@ class VideoRetrieval(AbsTaskRetrieval): dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "dev": { - "average_document_length": 31.048855642524522, - "average_query_length": 7.365, - "num_documents": 100930, - "num_queries": 1000, - "average_relevant_docs_per_query": 1.0, - } - }, + prompt={ + "query": "Given a video search query, retrieve the titles of relevant videos" }, ) diff --git a/mteb/tasks/Retrieval/zho/LeCaRDv2Retrieval.py b/mteb/tasks/Retrieval/zho/LeCaRDv2Retrieval.py index 587f435389..6187bb5d3e 100644 --- a/mteb/tasks/Retrieval/zho/LeCaRDv2Retrieval.py +++ b/mteb/tasks/Retrieval/zho/LeCaRDv2Retrieval.py @@ -35,16 +35,4 @@ class LeCaRDv2(AbsTaskRetrieval): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": None, - "avg_character_length": { - "test": { - "average_document_length": 7232.823978919631, - "average_query_length": 4259.440251572327, - "num_documents": 3795, - "num_queries": 159, - "average_relevant_docs_per_query": 24.50314465408805, - } - }, - }, ) diff --git a/mteb/tasks/STS/deu/GermanSTSBenchmarkSTS.py b/mteb/tasks/STS/deu/GermanSTSBenchmarkSTS.py index 7b2b5b59fb..34add4378e 100644 --- a/mteb/tasks/STS/deu/GermanSTSBenchmarkSTS.py +++ b/mteb/tasks/STS/deu/GermanSTSBenchmarkSTS.py @@ -34,7 +34,6 @@ class GermanSTSBenchmarkSTS(AbsTaskSTS): year={2021}, url={https://github.com/PhilipMay/stsb-multi-mt} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property diff --git a/mteb/tasks/STS/eng/BiossesSTS.py b/mteb/tasks/STS/eng/BiossesSTS.py index d8f11c2fbd..ce54e37789 100644 --- a/mteb/tasks/STS/eng/BiossesSTS.py +++ b/mteb/tasks/STS/eng/BiossesSTS.py @@ -42,7 +42,6 @@ class BiossesSTS(AbsTaskSTS): url = {https://doi.org/10.1093/bioinformatics/btx238}, eprint = {https://academic.oup.com/bioinformatics/article-pdf/33/14/i49/50315066/bioinformatics\_33\_14\_i49.pdf}, }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property diff --git a/mteb/tasks/STS/eng/STS12STS.py b/mteb/tasks/STS/eng/STS12STS.py index 5cf0b1ccfc..b222b42c66 100644 --- a/mteb/tasks/STS/eng/STS12STS.py +++ b/mteb/tasks/STS/eng/STS12STS.py @@ -40,15 +40,6 @@ class STS12STS(AbsTaskSTS): location = {Montr\'{e}al, Canada}, series = {SemEval '12} }""", - descriptive_stats={ - "n_samples": {"test": 6216}, - "test": { - "num_samples": 3108, - "average_sentence1_len": 63.78893178893179, - "average_sentence2_len": 65.5926640926641, - "avg_score": 3.5060643500643507, - }, - }, ) @property diff --git a/mteb/tasks/STS/eng/STS13STS.py b/mteb/tasks/STS/eng/STS13STS.py index 716f42fdef..415eafbc23 100644 --- a/mteb/tasks/STS/eng/STS13STS.py +++ b/mteb/tasks/STS/eng/STS13STS.py @@ -34,10 +34,6 @@ class STS13STS(AbsTaskSTS): year={2013}, url={https://api.semanticscholar.org/CorpusID:10241043} }""", - descriptive_stats={ - "n_samples": {"test": 3000}, - "avg_character_length": {"test": 54.0}, - }, ) @property diff --git a/mteb/tasks/STS/eng/STS14STS.py b/mteb/tasks/STS/eng/STS14STS.py index 12bc9a4d18..933cc124da 100644 --- a/mteb/tasks/STS/eng/STS14STS.py +++ b/mteb/tasks/STS/eng/STS14STS.py @@ -45,10 +45,6 @@ class STS14STS(AbsTaskSTS): doi = "10.3115/v1/S14-1002", pages = "12--21", }""", - descriptive_stats={ - "n_samples": {"test": 7500}, - "avg_character_length": {"test": 54.3}, - }, ) @property diff --git a/mteb/tasks/STS/eng/STS15STS.py b/mteb/tasks/STS/eng/STS15STS.py index a2c84dd3de..99e81aa90f 100644 --- a/mteb/tasks/STS/eng/STS15STS.py +++ b/mteb/tasks/STS/eng/STS15STS.py @@ -43,10 +43,6 @@ class STS15STS(AbsTaskSTS): doi = "10.18653/v1/S15-2010", pages = "56--63", }""", - descriptive_stats={ - "n_samples": {"test": 6000}, - "avg_character_length": {"test": 57.7}, - }, ) @property diff --git a/mteb/tasks/STS/eng/STS16STS.py b/mteb/tasks/STS/eng/STS16STS.py index ca93d0867d..94c978d4fc 100644 --- a/mteb/tasks/STS/eng/STS16STS.py +++ b/mteb/tasks/STS/eng/STS16STS.py @@ -49,10 +49,6 @@ class STS16STS(AbsTaskSTS): doi = "10.18653/v1/S16-1001", pages = "1--18", }""", - descriptive_stats={ - "n_samples": {"test": 2372}, - "avg_character_length": {"test": 65.3}, - }, ) @property diff --git a/mteb/tasks/STS/eng/STSBenchmarkSTS.py b/mteb/tasks/STS/eng/STSBenchmarkSTS.py index c76d52a749..099fba6773 100644 --- a/mteb/tasks/STS/eng/STSBenchmarkSTS.py +++ b/mteb/tasks/STS/eng/STSBenchmarkSTS.py @@ -33,7 +33,6 @@ class STSBenchmarkSTS(AbsTaskSTS): year={2021}, url={https://github.com/PhilipMay/stsb-multi-mt} }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property diff --git a/mteb/tasks/STS/eng/SickrSTS.py b/mteb/tasks/STS/eng/SickrSTS.py index 6c5267e0ea..1c93fff578 100644 --- a/mteb/tasks/STS/eng/SickrSTS.py +++ b/mteb/tasks/STS/eng/SickrSTS.py @@ -12,8 +12,8 @@ class SickrSTS(AbsTaskSTS): "path": "mteb/sickr-sts", "revision": "20a6d6f312dd54037fe07a32d58e5e168867909d", }, - description="Semantic Textual Similarity SICK-R dataset as described here:", - reference="https://aclanthology.org/2020.lrec-1.207", + description="Semantic Textual Similarity SICK-R dataset", + reference="https://aclanthology.org/L14-1314/", type="STS", category="s2s", modalities=["text"], @@ -21,43 +21,38 @@ class SickrSTS(AbsTaskSTS): eval_langs=["eng-Latn"], main_score="cosine_spearman", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, + domains=["Web", "Written"], + task_subtypes=["Textual Entailment"], + license="cc-by-nc-sa-3.0", + annotations_creators="human-annotated", dialect=None, sample_creation=None, - bibtex_citation="""@inproceedings{dadas-etal-2020-evaluation, - title = "Evaluation of Sentence Representations in {P}olish", - author = "Dadas, Slawomir and - Pere{\l}kiewicz, Micha{\l} and - Po{\'s}wiata, Rafa{\l}", + bibtex_citation="""@inproceedings{marelli-etal-2014-sick, + title = "A {SICK} cure for the evaluation of compositional distributional semantic models", + author = "Marelli, Marco and + Menini, Stefano and + Baroni, Marco and + Bentivogli, Luisa and + Bernardi, Raffaella and + Zamparelli, Roberto", editor = "Calzolari, Nicoletta and - B{\'e}chet, Fr{\'e}d{\'e}ric and - Blache, Philippe and Choukri, Khalid and - Cieri, Christopher and Declerck, Thierry and - Goggi, Sara and - Isahara, Hitoshi and + Loftsson, Hrafn and Maegaard, Bente and Mariani, Joseph and - Mazo, H{\'e}l{\`e}ne and Moreno, Asuncion and Odijk, Jan and Piperidis, Stelios", - booktitle = "Proceedings of the Twelfth Language Resources and Evaluation Conference", + booktitle = "Proceedings of the Ninth International Conference on Language Resources and Evaluation ({LREC}'14)", month = may, - year = "2020", - address = "Marseille, France", - publisher = "European Language Resources Association", - url = "https://aclanthology.org/2020.lrec-1.207", - pages = "1674--1680", - abstract = "Methods for learning sentence representations have been actively developed in recent years. However, the lack of pre-trained models and datasets annotated at the sentence level has been a problem for low-resource languages such as Polish which led to less interest in applying these methods to language-specific tasks. In this study, we introduce two new Polish datasets for evaluating sentence embeddings and provide a comprehensive evaluation of eight sentence representation methods including Polish and multilingual models. We consider classic word embedding models, recently developed contextual embeddings and multilingual sentence encoders, showing strengths and weaknesses of specific approaches. We also examine different methods of aggregating word vectors into a single sentence vector.", - language = "English", - ISBN = "979-10-95546-34-4", + year = "2014", + address = "Reykjavik, Iceland", + publisher = "European Language Resources Association (ELRA)", + url = "http://www.lrec-conf.org/proceedings/lrec2014/pdf/363_Paper.pdf", + pages = "216--223", + abstract = "Shared and internationally recognized benchmarks are fundamental for the development of any computational system. We aim to help the research community working on compositional distributional semantic models (CDSMs) by providing SICK (Sentences Involving Compositional Knowldedge), a large size English benchmark tailored for them. SICK consists of about 10,000 English sentence pairs that include many examples of the lexical, syntactic and semantic phenomena that CDSMs are expected to account for, but do not require dealing with other aspects of existing sentential data sets (idiomatic multiword expressions, named entities, telegraphic language) that are not within the scope of CDSMs. By means of crowdsourcing techniques, each pair was annotated for two crucial semantic tasks: relatedness in meaning (with a 5-point rating scale as gold score) and entailment relation between the two elements (with three possible gold labels: entailment, contradiction, and neutral). The SICK data set was used in SemEval-2014 Task 1, and it freely available for research purposes.", }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property diff --git a/mteb/tasks/STS/fao/FaroeseSTS.py b/mteb/tasks/STS/fao/FaroeseSTS.py index 2d45698a36..156485321a 100644 --- a/mteb/tasks/STS/fao/FaroeseSTS.py +++ b/mteb/tasks/STS/fao/FaroeseSTS.py @@ -41,10 +41,6 @@ class FaroeseSTS(AbsTaskSTS): publisher = {Link{\"o}ping University Electronic Press, Sweden}, } """, - descriptive_stats={ - "n_samples": {"train": 729}, - "avg_character_length": {"train": 43.6}, - }, ) @property diff --git a/mteb/tasks/STS/fin/FinParaSTS.py b/mteb/tasks/STS/fin/FinParaSTS.py index f7a73b3730..6ed513ade8 100644 --- a/mteb/tasks/STS/fin/FinParaSTS.py +++ b/mteb/tasks/STS/fin/FinParaSTS.py @@ -56,10 +56,6 @@ class FinParaSTS(AbsTaskSTS): abstract = "In this paper, we introduce the first fully manually annotated paraphrase corpus for Finnish containing 53,572 paraphrase pairs harvested from alternative subtitles and news headings. Out of all paraphrase pairs in our corpus 98{\%} are manually classified to be paraphrases at least in their given context, if not in all contexts. Additionally, we establish a manual candidate selection method and demonstrate its feasibility in high quality paraphrase selection in terms of both cost and quality.", } """, - descriptive_stats={ - "n_samples": {"test": N_SAMPLES, "validation": N_SAMPLES}, - "avg_character_length": {"test": 59.0, "validation": 58.8}, - }, ) @property diff --git a/mteb/tasks/STS/fra/SickFrSTS.py b/mteb/tasks/STS/fra/SickFrSTS.py index dca188f50c..241aa60163 100644 --- a/mteb/tasks/STS/fra/SickFrSTS.py +++ b/mteb/tasks/STS/fra/SickFrSTS.py @@ -28,7 +28,6 @@ class SickFrSTS(AbsTaskSTS): dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property diff --git a/mteb/tasks/STS/jpn/JSICK.py b/mteb/tasks/STS/jpn/JSICK.py index a5e05f0b45..554a3abf1d 100644 --- a/mteb/tasks/STS/jpn/JSICK.py +++ b/mteb/tasks/STS/jpn/JSICK.py @@ -39,10 +39,6 @@ class JSICK(AbsTaskSTS): publisher={MIT Press One Broadway, 12th Floor, Cambridge, Massachusetts 02142, USA~…} } """, - descriptive_stats={ - "n_samples": {"test": 1986}, - "avg_character_length": {"test": 21.47}, - }, ) @property diff --git a/mteb/tasks/STS/jpn/JSTS.py b/mteb/tasks/STS/jpn/JSTS.py index 7838b50132..4993359190 100644 --- a/mteb/tasks/STS/jpn/JSTS.py +++ b/mteb/tasks/STS/jpn/JSTS.py @@ -57,10 +57,6 @@ class JSTS(AbsTaskSTS): pages = "2957--2966", abstract = "To develop high-performance natural language understanding (NLU) models, it is necessary to have a benchmark to evaluate and analyze NLU ability from various perspectives. While the English NLU benchmark, GLUE, has been the forerunner, benchmarks are now being released for languages other than English, such as CLUE for Chinese and FLUE for French; but there is no such benchmark for Japanese. We build a Japanese NLU benchmark, JGLUE, from scratch without translation to measure the general NLU ability in Japanese. We hope that JGLUE will facilitate NLU research in Japanese.", }""", - descriptive_stats={ - "n_samples": {"valudtion": 1457}, - "avg_character_length": {"valudtion": 46.34}, - }, ) @property diff --git a/mteb/tasks/STS/kor/KlueSTS.py b/mteb/tasks/STS/kor/KlueSTS.py index 1046e9b87c..af55fb5bc0 100644 --- a/mteb/tasks/STS/kor/KlueSTS.py +++ b/mteb/tasks/STS/kor/KlueSTS.py @@ -36,10 +36,6 @@ class KlueSTS(AbsTaskSTS): archivePrefix={arXiv}, primaryClass={cs.CL} }""", - descriptive_stats={ - "n_samples": {"validation": 519}, - "avg_character_length": {"validation": 33.178227360308284}, - }, ) @property diff --git a/mteb/tasks/STS/kor/KorSTS.py b/mteb/tasks/STS/kor/KorSTS.py index 39e3f17264..6ab1437bb1 100644 --- a/mteb/tasks/STS/kor/KorSTS.py +++ b/mteb/tasks/STS/kor/KorSTS.py @@ -33,10 +33,6 @@ class KorSTS(AbsTaskSTS): journal={arXiv preprint arXiv:2004.03289}, year={2020} }""", - descriptive_stats={ - "n_samples": {"test": 1379}, - "avg_character_length": {"test": 29.279433139534884}, - }, ) @property diff --git a/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py b/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py index 463db58232..b5a5c67b86 100644 --- a/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py +++ b/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py @@ -73,10 +73,6 @@ class IndicCrosslingualSTS(AbsTaskSTS, MultilingualTask): url = {https://doi.org/10.1162/tacl\_a\_00452}, eprint = {https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl\_a\_00452/1987010/tacl\_a\_00452.pdf}, }""", - descriptive_stats={ - "n_samples": {"test": 10020}, - "avg_character_length": {"test": 76.22}, - }, ) @property diff --git a/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py b/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py index b3b4cbdb7e..0e7928fe8b 100644 --- a/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py +++ b/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py @@ -65,83 +65,6 @@ class STS17Crosslingual(AbsTaskSTS, MultilingualTask): pages = "1--14", abstract = "Semantic Textual Similarity (STS) measures the meaning similarity of sentences. Applications include machine translation (MT), summarization, generation, question answering (QA), short answer grading, semantic search, dialog and conversational systems. The STS shared task is a venue for assessing the current state-of-the-art. The 2017 task focuses on multilingual and cross-lingual pairs with one sub-track exploring MT quality estimation (MTQE) data. The task obtained strong participation from 31 teams, with 17 participating in \textit{all language tracks}. We summarize performance and review a selection of well performing methods. Analysis highlights common errors, providing insight into the limitations of existing models. To support ongoing work on semantic representations, the \textit{STS Benchmark} is introduced as a new shared training and evaluation set carefully selected from the corpus of English STS shared task data (2012-2017).", }""", - descriptive_stats={ - "n_samples": {"test": 500}, - "test": { - "num_samples": 5346, - "average_sentence1_len": 38.14665170220726, - "average_sentence2_len": 36.72502805836139, - "avg_score": 2.3554804214989464, - "hf_subset_descriptive_stats": { - "ko-ko": { - "num_samples": 2846, - "average_sentence1_len": 31.991918482080113, - "average_sentence2_len": 32.44483485593816, - "avg_score": 2.469359920356055, - }, - "ar-ar": { - "num_samples": 250, - "average_sentence1_len": 32.208, - "average_sentence2_len": 32.78, - "avg_score": 2.216800000000001, - }, - "en-ar": { - "num_samples": 250, - "average_sentence1_len": 42.36, - "average_sentence2_len": 32.696, - "avg_score": 2.1423999999999994, - }, - "en-de": { - "num_samples": 250, - "average_sentence1_len": 43.952, - "average_sentence2_len": 44.756, - "avg_score": 2.2776000000000014, - }, - "en-en": { - "num_samples": 250, - "average_sentence1_len": 43.952, - "average_sentence2_len": 42.724, - "avg_score": 2.2776000000000014, - }, - "en-tr": { - "num_samples": 250, - "average_sentence1_len": 41.916, - "average_sentence2_len": 41.6, - "avg_score": 2.1335999999999986, - }, - "es-en": { - "num_samples": 250, - "average_sentence1_len": 50.84, - "average_sentence2_len": 42.024, - "avg_score": 2.1464000000000003, - }, - "es-es": { - "num_samples": 250, - "average_sentence1_len": 49.836, - "average_sentence2_len": 51.224, - "avg_score": 2.2312000000000007, - }, - "fr-en": { - "num_samples": 250, - "average_sentence1_len": 49.624, - "average_sentence2_len": 42.724, - "avg_score": 2.2776000000000014, - }, - "it-en": { - "num_samples": 250, - "average_sentence1_len": 50.028, - "average_sentence2_len": 42.724, - "avg_score": 2.2776000000000014, - }, - "nl-en": { - "num_samples": 250, - "average_sentence1_len": 46.816, - "average_sentence2_len": 42.724, - "avg_score": 2.2776000000000014, - }, - }, - }, - }, ) @property diff --git a/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py b/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py index ae394b5512..0e294aeb5a 100644 --- a/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py +++ b/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py @@ -77,10 +77,6 @@ class STS22CrosslingualSTSv2(AbsTaskSTS, MultilingualTask): doi = "10.18653/v1/2022.semeval-1.155", pages = "1094--1106", }""", - descriptive_stats={ - "n_samples": {"test": 3958}, - "avg_character_length": {"test": 1993.6}, - }, ) @property @@ -143,10 +139,6 @@ class STS22CrosslingualSTS(AbsTaskSTS, MultilingualTask): doi = "10.18653/v1/2022.semeval-1.155", pages = "1094--1106", }""", - descriptive_stats={ - "n_samples": {"test": 8056}, - "avg_character_length": {"test": 1993.6}, - }, ) @property diff --git a/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py b/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py index 7aacebe342..eaf5ff1afb 100644 --- a/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py +++ b/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py @@ -52,10 +52,6 @@ class STSBenchmarkMultilingualSTS(AbsTaskSTS, MultilingualTask): year={2021}, url={https://github.com/PhilipMay/stsb-multi-mt} }""", - descriptive_stats={ - "n_samples": {"dev": 30000, "test": 27580}, - "avg_character_length": {"dev": 66.5, "test": 56.1}, - }, ) @property diff --git a/mteb/tasks/STS/multilingual/SemRel24STS.py b/mteb/tasks/STS/multilingual/SemRel24STS.py index b990170215..ea503eb1b6 100644 --- a/mteb/tasks/STS/multilingual/SemRel24STS.py +++ b/mteb/tasks/STS/multilingual/SemRel24STS.py @@ -63,10 +63,6 @@ class SemRel24STS(AbsTaskSTS, MultilingualTask): primaryClass={cs.CL} } """, - descriptive_stats={ - "n_samples": {"dev": 2089, "test": 7498}, - "avg_character_length": {"dev": 163.1, "test": 145.9}, - }, ) @property diff --git a/mteb/tasks/STS/pol/PolishSTS.py b/mteb/tasks/STS/pol/PolishSTS.py index 38bc37d50e..9115f37996 100644 --- a/mteb/tasks/STS/pol/PolishSTS.py +++ b/mteb/tasks/STS/pol/PolishSTS.py @@ -57,10 +57,6 @@ class SickrPLSTS(AbsTaskSTS): ISBN = "979-10-95546-34-4", } """, - descriptive_stats={ - "n_samples": {"test": 9812}, - "avg_character_length": {"test": 42.8}, - }, ) @property @@ -111,10 +107,6 @@ class CdscrSTS(AbsTaskSTS): } """, - descriptive_stats={ - "n_samples": {"test": 1000}, - "avg_character_length": {"test": 75.24}, - }, ) @property diff --git a/mteb/tasks/STS/por/Assin2STS.py b/mteb/tasks/STS/por/Assin2STS.py index 9140849547..e96ae97c34 100644 --- a/mteb/tasks/STS/por/Assin2STS.py +++ b/mteb/tasks/STS/por/Assin2STS.py @@ -34,10 +34,6 @@ class Assin2STS(AbsTaskSTS): year={2020}, organization={Springer} }""", - descriptive_stats={ - "n_samples": {"test": 2448}, - "avg_character_length": {"test": 53.55}, - }, ) @property diff --git a/mteb/tasks/STS/por/SickBrSTS.py b/mteb/tasks/STS/por/SickBrSTS.py index 4db82be50b..5298ab5437 100644 --- a/mteb/tasks/STS/por/SickBrSTS.py +++ b/mteb/tasks/STS/por/SickBrSTS.py @@ -50,10 +50,6 @@ class SickBrSTS(AbsTaskSTS): isbn="978-3-319-99722-3" } """, - descriptive_stats={ - "n_samples": {"test": N_SAMPLES}, - "avg_character_length": {"test": 54.89}, - }, ) @property @@ -64,14 +60,12 @@ def metadata_dict(self) -> dict[str, str]: return metadata_dict def dataset_transform(self): - for split in self.dataset: - self.dataset.update( - { - split: self.dataset[split].train_test_split( - test_size=N_SAMPLES, seed=self.seed, label="entailment_label" - )["test"] - } - ) + self.dataset = self.stratified_subsampling( + self.dataset, + seed=42, + splits=self.metadata.eval_splits, + label="entailment_label", + ) self.dataset = self.dataset.rename_columns( { diff --git a/mteb/tasks/STS/ron/RonSTS.py b/mteb/tasks/STS/ron/RonSTS.py index d75eca251c..4941cba3e6 100644 --- a/mteb/tasks/STS/ron/RonSTS.py +++ b/mteb/tasks/STS/ron/RonSTS.py @@ -36,10 +36,6 @@ class RonSTS(AbsTaskSTS): year={2021} } """, - descriptive_stats={ - "n_samples": {"test": 1379}, - "avg_character_length": {"test": 60.5}, - }, # avg across sent1 and sent2 ) @property diff --git a/mteb/tasks/STS/rus/RUParaPhraserSTS.py b/mteb/tasks/STS/rus/RUParaPhraserSTS.py index b577cbf6d8..9174f2f661 100644 --- a/mteb/tasks/STS/rus/RUParaPhraserSTS.py +++ b/mteb/tasks/STS/rus/RUParaPhraserSTS.py @@ -51,10 +51,6 @@ class RUParaPhraserSTS(AbsTaskSTS): organization={Springer} } """, - descriptive_stats={ - "n_samples": {"test": 1924}, - "avg_character_length": {"test": 61.25}, - }, ) @property diff --git a/mteb/tasks/STS/rus/RuSTSBenchmarkSTS.py b/mteb/tasks/STS/rus/RuSTSBenchmarkSTS.py index 35bf75ba04..eca26691fa 100644 --- a/mteb/tasks/STS/rus/RuSTSBenchmarkSTS.py +++ b/mteb/tasks/STS/rus/RuSTSBenchmarkSTS.py @@ -34,10 +34,6 @@ class RuSTSBenchmarkSTS(AbsTaskSTS): year={2021}, url={https://github.com/PhilipMay/stsb-multi-mt} }""", - descriptive_stats={ - "n_samples": {"test": 1264}, - "avg_character_length": {"test": 54.2}, - }, ) @property diff --git a/mteb/tasks/STS/spa/STSES.py b/mteb/tasks/STS/spa/STSES.py index e56acedf4a..8bdbf227a2 100644 --- a/mteb/tasks/STS/spa/STSES.py +++ b/mteb/tasks/STS/spa/STSES.py @@ -47,7 +47,6 @@ class STSES(AbsTaskSTS): year={2014} } """, - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property diff --git a/mteb/tasks/STS/zho/CMTEBSTS.py b/mteb/tasks/STS/zho/CMTEBSTS.py index 94bf2d126a..e428e24156 100644 --- a/mteb/tasks/STS/zho/CMTEBSTS.py +++ b/mteb/tasks/STS/zho/CMTEBSTS.py @@ -47,7 +47,6 @@ class ATEC(AbsTaskSTS): pages = "4348--4366", abstract = "We propose a novel problem within end-to-end learning of task oriented dialogs (TOD), in which the dialog system mimics a troubleshooting agent who helps a user by diagnosing their problem (e.g., car not starting). Such dialogs are grounded in domain-specific flowcharts, which the agent is supposed to follow during the conversation. Our task exposes novel technical challenges for neural TOD, such as grounding an utterance to the flowchart without explicit annotation, referring to additional manual pages when user asks a clarification question, and ability to follow unseen flowcharts at test time. We release a dataset (FLODIAL) consisting of 2,738 dialogs grounded on 12 different troubleshooting flowcharts. We also design a neural model, FLONET, which uses a retrieval-augmented generation architecture to train the dialog agent. Our experiments find that FLONET can do zero-shot transfer to unseen flowcharts, and sets a strong baseline for future research.", }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property @@ -89,7 +88,6 @@ class BQ(AbsTaskSTS): primaryClass={cs.CL}, url={https://arxiv.org/abs/2309.07597}, }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property @@ -131,7 +129,6 @@ class LCQMC(AbsTaskSTS): primaryClass={cs.CL}, url={https://arxiv.org/abs/2309.07597}, }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property @@ -173,7 +170,6 @@ class PAWSX(AbsTaskSTS): primaryClass={cs.CL}, url={https://arxiv.org/abs/2309.07597}, }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property @@ -215,7 +211,6 @@ class STSB(AbsTaskSTS): primaryClass={cs.CL}, url={https://arxiv.org/abs/2309.07597}, }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property @@ -268,7 +263,6 @@ class AFQMC(AbsTaskSTS): pages = "4348--4366", abstract = "We propose a novel problem within end-to-end learning of task oriented dialogs (TOD), in which the dialog system mimics a troubleshooting agent who helps a user by diagnosing their problem (e.g., car not starting). Such dialogs are grounded in domain-specific flowcharts, which the agent is supposed to follow during the conversation. Our task exposes novel technical challenges for neural TOD, such as grounding an utterance to the flowchart without explicit annotation, referring to additional manual pages when user asks a clarification question, and ability to follow unseen flowcharts at test time. We release a dataset (FLODIAL) consisting of 2,738 dialogs grounded on 12 different troubleshooting flowcharts. We also design a neural model, FLONET, which uses a retrieval-augmented generation architecture to train the dialog agent. Our experiments find that FLONET can do zero-shot transfer to unseen flowcharts, and sets a strong baseline for future research.", }""", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property @@ -302,7 +296,6 @@ class QBQTC(AbsTaskSTS): dialect=None, sample_creation=None, bibtex_citation=None, - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @property diff --git a/mteb/tasks/SpeedTask/CPUSpeedTask.py b/mteb/tasks/SpeedTask/CPUSpeedTask.py index 75f223378f..19871b9ba5 100644 --- a/mteb/tasks/SpeedTask/CPUSpeedTask.py +++ b/mteb/tasks/SpeedTask/CPUSpeedTask.py @@ -24,8 +24,4 @@ class CPUSpeedTask(AbsTaskSpeedTask): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 1}, - "avg_character_length": {"test": 3591}, - }, ) diff --git a/mteb/tasks/SpeedTask/GPUSpeedTask.py b/mteb/tasks/SpeedTask/GPUSpeedTask.py index 4c29368e4f..cba6da0dfc 100644 --- a/mteb/tasks/SpeedTask/GPUSpeedTask.py +++ b/mteb/tasks/SpeedTask/GPUSpeedTask.py @@ -25,8 +25,4 @@ class GPUSpeedTask(AbsTaskSpeedTask): dialect=[], sample_creation="found", bibtex_citation="", - descriptive_stats={ - "n_samples": {"test": 1}, - "avg_character_length": {"test": 3591}, - }, ) diff --git a/mteb/tasks/Summarization/eng/SummEvalSummarization.py b/mteb/tasks/Summarization/eng/SummEvalSummarization.py index b88c1d6874..8f64d1bbf5 100644 --- a/mteb/tasks/Summarization/eng/SummEvalSummarization.py +++ b/mteb/tasks/Summarization/eng/SummEvalSummarization.py @@ -38,16 +38,6 @@ class SummEvalSummarization(AbsTaskSummarization): journal={arXiv preprint arXiv:2007.12626}, year={2020} }""", - descriptive_stats={ - "n_samples": {"test": 2800}, - "test": { - "num_samples": 100, - "avg_text_len": 2100.35, - "avg_human_summaries_len": 11.0, - "avg_machine_summaries_len": 16.0, - "avg_relevance": 3.7770833333333336, - }, - }, ) @property @@ -86,10 +76,6 @@ class SummEvalSummarizationv2(AbsTaskSummarization): journal={arXiv preprint arXiv:2007.12626}, year={2020} }""", - descriptive_stats={ - "n_samples": {"test": 2800}, - "avg_character_length": {"test": 359.8}, - }, ) @property diff --git a/mteb/tasks/Summarization/fra/SummEvalFrSummarization.py b/mteb/tasks/Summarization/fra/SummEvalFrSummarization.py index 822f31af3a..660f03502e 100644 --- a/mteb/tasks/Summarization/fra/SummEvalFrSummarization.py +++ b/mteb/tasks/Summarization/fra/SummEvalFrSummarization.py @@ -37,10 +37,6 @@ class SummEvalFrSummarization(AbsTaskSummarization): journal={arXiv preprint arXiv:2007.12626}, year={2020} }""", - descriptive_stats={ - "n_samples": {"test": 2800}, - "avg_character_length": {"test": 407.1}, - }, ) @property @@ -80,10 +76,6 @@ class SummEvalFrSummarizationv2(AbsTaskSummarization): journal={arXiv preprint arXiv:2007.12626}, year={2020} }""", - descriptive_stats={ - "n_samples": {"test": 2800}, - "avg_character_length": {"test": 407.1}, - }, ) @property diff --git a/pyproject.toml b/pyproject.toml index 088613b36a..2c1040047c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mteb" -version = "1.16.5" +version = "1.29.14" description = "Massive Text Embedding Benchmark" readme = "README.md" authors = [ @@ -25,7 +25,7 @@ classifiers = [ ] requires-python = ">=3.9" dependencies = [ - "datasets>=2.19.0", + "datasets>=2.19.0,<3.0.0", "numpy>=1.0.0,<3.0.0", "requests>=2.26.0", "scikit_learn>=1.0.2", @@ -58,9 +58,16 @@ dev = ["ruff==0.6.4", # locked so we don't get PRs which fail only due to a lint codecarbon = ["codecarbon"] speedtask = ["GPUtil>=1.4.0", "psutil>=5.9.8"] peft = ["peft>=0.11.0"] -leaderboard = ["gradio>=4.44.0", "gradio_rangeslider>=0.0.6"] +leaderboard = ["gradio>=5.7.1", "gradio_rangeslider>=0.0.8", "plotly>=5.24.0"] flagembedding = ["FlagEmbedding"] -e5v = ["accelerate>=0.26.0"] +jina = ["einops>=0.8.0"] +flash_attention = ["flash-attn>=2.6.3"] +openai = ["openai>=1.41.0", "tiktoken>=0.8.0"] +model2vec = ["model2vec>=0.3.0"] +pylate = ["pylate>=1.1.4"] +bm25s = ["bm25s>=0.2.6", "PyStemmer>=2.2.0.3"] +gritlm = ["gritlm>=1.0.2"] +xformers = ["xformers>=0.0.29"] [tool.coverage.report] @@ -90,6 +97,7 @@ exclude = ["tests", "results"] [tool.setuptools.package-data] "*" = ["*.json"] +"mteb.abstasks" = ["the_ugly_duckling.txt"] [tool.ruff] @@ -120,6 +128,7 @@ ignore = ["E501", # line too long "D107", # Missing docstring in __init__ "D205", # 1 blank line required between summary line and description "D415", # First line should end with a period + "C408", # don't use unecc. collection call, e.g. dict over {} ] [tool.ruff.lint.flake8-implicit-str-concat] diff --git a/scripts/compare_leaderboard_results.py b/scripts/compare_leaderboard_results.py new file mode 100644 index 0000000000..1fe9c3d766 --- /dev/null +++ b/scripts/compare_leaderboard_results.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import json +import logging +from collections import defaultdict +from pathlib import Path + +from mteb import get_benchmark, load_results + +logging.basicConfig(level=logging.INFO) + +models = [ + "intfloat/multilingual-e5-small", + # Add other models here +] +benchmark = get_benchmark("MTEB(Chinese)") + +results = [] + +# in same folder as mteb repo +# git clone https://github.com/embeddings-benchmark/leaderboard +# get path of current file +base_path = Path(__file__).parent.parent.parent / "leaderboard" / "boards_data" + + +for model_name_to_search in models: + model_results = load_results( + models=[model_name_to_search], + tasks=benchmark.tasks, + only_main_score=True, + require_model_meta=False, + ) + + cur_model = {task.metadata.name: defaultdict(dict) for task in benchmark.tasks} + for model_res in model_results: + for task_res in model_res.task_results: + task_name = task_res.task.metadata.name + + split = ( + "test" + if "test" in task_res.task.metadata.eval_splits + else task_res.task.metadata.eval_splits[0] + ) + if split in task_res.scores: + scores = [score["main_score"] for score in task_res.scores[split]] + cur_model[task_name]["new"] = round( + (sum(scores) / len(scores)) * 100, 2 + ) + + for lang_path in base_path.iterdir(): + data_tasks_path = lang_path / "data_tasks" + + for task_dir in data_tasks_path.iterdir(): + if task_dir.is_dir(): + results_file_path = task_dir / "default.jsonl" + if results_file_path.exists(): + with open(results_file_path) as file: + for line in file: + data = json.loads(line) + model_name = data.get("Model", "") + if model_name_to_search in model_name: + for key, value in data.items(): + if key in [ + "index", + "Rank", + "Model", + "Model Size (Million Parameters)", + "Memory Usage (GB, fp32)", + "Embedding Dimensions", + "Max Tokens", + "Average", + ]: + continue + for benchmark_task in benchmark.tasks: + if benchmark_task.metadata.name in key: + cur_model[benchmark_task.metadata.name][ + "old" + ] = value + + sorted_cur_model = { + task.metadata.name: cur_model[task.metadata.name] + for task in benchmark.tasks + if task.metadata.name in cur_model + } + results.append({"model": model_name_to_search, "results": sorted_cur_model}) + +# Write results to JSONL file +with open("results.jsonl", "w") as file: + for result in results: + file.write(json.dumps(result) + "\n") + +# Write results to Markdown file +with open("results.md", "w") as file: + for result in results: + file.write(f"## Model: {result['model']}\n\n") + file.write("| Task Name | Old Leaderboard | New Leaderboard |\n") + file.write("|-----------|-----------------|-----------------|\n") + for task_name, scores in result["results"].items(): + old_score = scores.get("old", "N/A") + new_score = scores.get("new", "N/A") + file.write(f"| {task_name} | {old_score} | {new_score} |\n") + file.write("\n") diff --git a/scripts/create_language_family_mapping.py b/scripts/create_language_family_mapping.py new file mode 100644 index 0000000000..50700d9654 --- /dev/null +++ b/scripts/create_language_family_mapping.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from pyglottolog.api import Glottolog, lls +from tqdm import tqdm + +glottolog = Glottolog( + "/home/ubuntu/isaac/work/glottolog" +) # Download the Glottolog repository + + +def get_languages_with_iso_by_languoid(languoid, level=0, prev_fam=None): + # Recursively gather all descendant languages with ISO codes + if prev_fam is None: + prev_fam = {} # Start with a fresh dictionary for each top-level languoid + + if not isinstance(languoid, lls.Languoid): + return + + for descendant in languoid.children: + # Create a copy of `prev_fam` to avoid overwriting + current_fam = prev_fam.copy() + current_fam[f"level{level}"] = languoid.name + + if descendant.level.name == "language": # Direct languages + if descendant.iso: + iso_key = descendant.iso + if len(ISO2FAMILY.get(iso_key, {})) > len(current_fam): + continue + ISO2FAMILY[iso_key] = current_fam + elif descendant.level.name == "family": # Subfamilies, recurse + get_languages_with_iso_by_languoid(descendant, level + 1, current_fam) + + +all_languoids = list(glottolog.languoids()) +with Path("language_family.json").open("r") as f: + ISO2FAMILY = json.load(f) + +for languoid in tqdm(all_languoids, total=len(all_languoids)): + get_languages_with_iso_by_languoid(languoid) + +ISO2FAMILY = dict(sorted(ISO2FAMILY.items())) + +with Path("language_family.json").open("w") as f: + json.dump(ISO2FAMILY, f, indent=3) diff --git a/scripts/extract_model_names.py b/scripts/extract_model_names.py new file mode 100644 index 0000000000..6cbaa2c298 --- /dev/null +++ b/scripts/extract_model_names.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import argparse +import ast +import logging +from pathlib import Path + +from git import Repo + +logging.basicConfig(level=logging.INFO) + + +def get_changed_files(base_branch="main"): + repo_path = Path(__file__).parent.parent + repo = Repo(repo_path) + repo.remotes.origin.fetch(base_branch) + + base_commit = repo.commit(f"origin/{base_branch}") + head_commit = repo.commit("HEAD") + + diff = repo.git.diff("--name-only", base_commit, head_commit) + + changed_files = diff.splitlines() + return [ + f + for f in changed_files + if f.startswith("mteb/models/") + and f.endswith(".py") + and "overview" not in f + and "init" not in f + ] + + +def extract_model_names( + files: list[str], return_one_model_name_per_file=False +) -> list[str]: + model_names = [] + first_model_found = False + for file in files: + with open(file) as f: + tree = ast.parse(f.read()) + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + if ( + isinstance(target, ast.Name) + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "ModelMeta" + ): + model_name = next( + ( + kw.value.value + for kw in node.value.keywords + if kw.arg == "name" + ), + None, + ) + if model_name: + model_names.append(model_name) + first_model_found = True + if return_one_model_name_per_file and first_model_found: + logging.info(f"Found model name {model_name} in file {file}") + break # NOTE: Only take the first model_name per file to avoid disk out of space issue. + return model_names + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "base_branch", + nargs="?", + default="main", + help="Base branch to compare changes with", + ) + parser.add_argument( + "--return_one_model_name_per_file", + action="store_true", + default=False, + help="Only return one model name per file.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + """ + Can pass in base branch as an argument. Defaults to 'main'. + e.g. python extract_model_names.py mieb + """ + + args = parse_args() + + base_branch = args.base_branch + changed_files = get_changed_files(base_branch) + model_names = extract_model_names( + changed_files, + return_one_model_name_per_file=args.return_one_model_name_per_file, + ) + output_file = Path(__file__).parent / "model_names.txt" + with output_file.open("w") as f: + f.write(" ".join(model_names)) diff --git a/scripts/generate_metadata.py b/scripts/generate_metadata.py new file mode 100644 index 0000000000..4ae87fdbca --- /dev/null +++ b/scripts/generate_metadata.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import json +import warnings +from pathlib import Path + +import iso639 +from huggingface_hub import HfApi, ModelCard, hf_hub_download +from tqdm import tqdm + +from mteb.model_meta import ModelMeta + +to_keep = [ + "Haon-Chen/speed-embedding-7b-instruct", + "Gameselo/STS-multilingual-mpnet-base-v2", + "HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1", + "HIT-TMG/KaLM-embedding-multilingual-mini-v1", + "Hum-Works/lodestone-base-4096-v1", + "Jaume/gemma-2b-embeddings", + "BeastyZ/e5-R-mistral-7b", + "Lajavaness/bilingual-embedding-base", + "Lajavaness/bilingual-embedding-large", + "Lajavaness/bilingual-embedding-small", + "Mihaiii/Bulbasaur", + "Mihaiii/Ivysaur", + "Mihaiii/Squirtle", + "Mihaiii/Venusaur", + "Mihaiii/Wartortle", + "Mihaiii/gte-micro", + "Mihaiii/gte-micro-v4", + "OrdalieTech/Solon-embeddings-large-0.1", + "Omartificial-Intelligence-Space/Arabert-all-nli-triplet-Matryoshka", + "Omartificial-Intelligence-Space/Arabic-MiniLM-L12-v2-all-nli-triplet", + "Omartificial-Intelligence-Space/Arabic-all-nli-triplet-Matryoshka", + "Omartificial-Intelligence-Space/Arabic-labse-Matryoshka", + "Omartificial-Intelligence-Space/Arabic-mpnet-base-all-nli-triplet", + "Omartificial-Intelligence-Space/Marbert-all-nli-triplet-Matryoshka", + "consciousAI/cai-lunaris-text-embeddings", + "consciousAI/cai-stellaris-text-embeddings", + "manu/bge-m3-custom-fr", + "manu/sentence_croissant_alpha_v0.2", + "manu/sentence_croissant_alpha_v0.3", + "manu/sentence_croissant_alpha_v0.4", + "thenlper/gte-base", + "thenlper/gte-large", + "thenlper/gte-small", + "OrlikB/KartonBERT-USE-base-v1", + "OrlikB/st-polish-kartonberta-base-alpha-v1", + "sdadas/mmlw-e5-base", # some models are monolingual adaptions of a another models (I would include them for now) + "dwzhu/e5-base-4k", # e.g. this is a long doc adaption of e5 + "sdadas/mmlw-e5-large", + "sdadas/mmlw-e5-small", + "sdadas/mmlw-roberta-base", + "sdadas/mmlw-roberta-large", + "izhx/udever-bloom-1b1", + "izhx/udever-bloom-3b", + "izhx/udever-bloom-560m", + "izhx/udever-bloom-7b1", + "avsolatorio/GIST-Embedding-v0", + "avsolatorio/GIST-all-MiniLM-L6-v2", + "avsolatorio/GIST-large-Embedding-v0", + "avsolatorio/GIST-small-Embedding-v0", + "bigscience/sgpt-bloom-7b1-msmarco", + "aari1995/German_Semantic_STS_V2", + "abhinand/MedEmbed-small-v0.1", + "avsolatorio/NoInstruct-small-Embedding-v0", + "brahmairesearch/slx-v0.1", + "deepfile/embedder-100p", + "deepvk/USER-bge-m3", + "infgrad/stella-base-en-v2", + "malenia1/ternary-weight-embedding", + "omarelshehy/arabic-english-sts-matryoshka", + "openbmb/MiniCPM-Embedding", + "shibing624/text2vec-base-multilingual", + "silma-ai/silma-embeddding-matryoshka-v0.1", + "zeta-alpha-ai/Zeta-Alpha-E5-Mistral", +] + +lang_to_script = { + "bam": "Latn", + "zul": "Latn", + "tsn": "Latn", + "rus": "Cyrl", + "mar": "Deva", + "ori": "Orya", + "swa": "Latn", + "vie": "Latn", + "nld": "Latn", + "kan": "Knda", + "yor": "Latn", + "urd": "Arab", + "guj": "Gujr", + "eng": "Latn", + "tso": "Latn", + "zho": "Hans", # Can also be "Hant" depending on region + "deu": "Latn", + "sna": "Latn", + "nso": "Latn", + "pol": "Latn", + "sot": "Latn", + "mal": "Mlym", + "cat": "Latn", + "lug": "Latn", + "spa": "Latn", + "wol": "Latn", + "tum": "Latn", + "xho": "Latn", + "fra": "Latn", + "tam": "Taml", + "pan": "Guru", + "twi": "Latn", + "tel": "Telu", + "ibo": "Latn", + "kik": "Latn", + "run": "Latn", + "hin": "Deva", + "ben": "Beng", + "fon": "Latn", + "ita": "Latn", + "nya": "Latn", + "aka": "Latn", + "por": "Latn", + "asm": "Beng", + "eus": "Latn", + "lin": "Latn", + "nep": "Deva", + "kin": "Latn", + "ind": "Latn", + "ara": "Arab", +} + + +def convert_code(code: str) -> str | None: + """Converts between two-letter and three-letter language codes""" + try: + lang_code = iso639.Language.match(code).part3 + script = lang_to_script[lang_code] + return f"{lang_code}_{script}" + except Exception as e: + print(f"Couldn't convert {code}, reason: {e}") + return None + + +api = HfApi() + + +def get_embedding_dimensions(model_name: str) -> int | None: + try: + file_path = hf_hub_download( + repo_id=model_name, filename="1_Pooling/config.json" + ) + with open(file_path) as in_file: + pooling_config = json.loads(in_file.read()) + return pooling_config.get("word_embedding_dimension", None) + except Exception as e: + print(f"Couldn't get embedding size for {model_name}, reason: {e}") + return None + + +def get_max_token(model_name: str) -> int | None: + try: + file_path = hf_hub_download(repo_id=model_name, filename="config.json") + with open(file_path) as in_file: + config = json.loads(in_file.read()) + return config.get("max_position_embeddings", None) + except Exception as e: + print(f"Couldn't get embedding size for {model_name}, reason: {e}") + return None + + +def get_base_model(model_name: str) -> str | None: + try: + file_path = hf_hub_download(repo_id=model_name, filename="config.json") + with open(file_path) as in_file: + config = json.loads(in_file.read()) + base_model = config.get("_name_or_path", None) + if base_model != model_name: + return base_model + else: + return None + except Exception as e: + print(f"Couldn't get base model for {model_name}, reason: {e}") + return None + + +def model_meta_from_hf_hub(model_name: str) -> ModelMeta: + try: + card = ModelCard.load(model_name) + card_data = card.data.to_dict() + frameworks = ["PyTorch"] + if card_data.get("library_name", None) == "sentence-transformers": + frameworks.append("Sentence Transformers") + languages = card_data.get("language", None) + if isinstance(languages, str): + languages = [languages] + if languages is not None: + languages = [convert_code(l) for l in languages] + languages = [l for l in languages if l is not None] + repo_info = api.repo_info(model_name) + revision = repo_info.sha + release_date = repo_info.created_at.strftime("%Y-%m-%d") + try: + n_parameters = repo_info.safetensors.total + except Exception as e: + print(f"Couldn't get model size for {model_name}, reason: {e}") + n_parameters = None + n_dimensions = get_embedding_dimensions(model_name) + datasets = card_data.get("datasets", None) + if isinstance(datasets, str): + datasets = [datasets] + if datasets is not None: + training_datasets = {ds: ["train"] for ds in datasets} + else: + training_datasets = None + return ModelMeta( + name=model_name, + revision=revision, + release_date=release_date, + languages=languages, + license=card_data.get("license", None), + framework=frameworks, + n_parameters=n_parameters, + adapted_from=get_base_model(model_name), + training_datasets=training_datasets, + open_weights=True, + superseded_by=None, + max_tokens=get_max_token(model_name), + embed_dim=n_dimensions, + similarity_fn_name="cosine", + reference=f"https://huggingface.co/{model_name}", + ) + except Exception as e: + warnings.warn(f"Failed to extract metadata from model: {e}.") + return ModelMeta( + name=model_name, + revision=None, + languages=None, + release_date=None, + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + open_weights=True, + public_training_code=None, + public_training_data=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + frameworks=[], + ) + + +def code_from_meta(meta: ModelMeta) -> str: + template = "{variable_name} ={meta}\n" + variable_name = meta.name.replace("/", "__").replace("-", "_").replace(".", "_") + return template.format(variable_name=variable_name, meta=meta.__repr__()) + + +def main(): + out_path = Path("mteb/models/misc_models.py") + with open(out_path, "w") as out_file: + out_file.write("from mteb.model_meta import ModelMeta\n\n") + for model in tqdm(to_keep, desc="Generating metadata for all models."): + meta = model_meta_from_hf_hub(model) + out_file.write(code_from_meta(meta)) + + +if __name__ == "__main__": + main() diff --git a/scripts/task_selection/task_selection_eng_lite.ipynb b/scripts/task_selection/task_selection_eng_lite.ipynb index 6b9e36a7b3..ec81fe2893 100644 --- a/scripts/task_selection/task_selection_eng_lite.ipynb +++ b/scripts/task_selection/task_selection_eng_lite.ipynb @@ -70,7 +70,10 @@ } ], "source": [ - "from mteb.benchmarks import MTEB_MAIN_EN\n", + "import mteb\n", + "\n", + "MTEB_MAIN_EN = mteb.get_benchmark(\"MTEB(eng, classic)\")\n", + "\n", "\n", "tasks = MTEB_MAIN_EN.tasks\n", "\n", diff --git a/scripts/task_selection/task_selection_eu.ipynb b/scripts/task_selection/task_selection_eu.ipynb index 5323b59b78..b685de89d0 100644 --- a/scripts/task_selection/task_selection_eu.ipynb +++ b/scripts/task_selection/task_selection_eu.ipynb @@ -5590,7 +5590,9 @@ } ], "source": [ - "from mteb.benchmarks import MTEB_MAIN_EN\n", + "import mteb\n", + "\n", + "MTEB_MAIN_EN = mteb.get_benchmark(\"MTEB(eng, classic)\")\n", "\n", "exceptions = [\n", " \"STS22.v2\",\n", diff --git a/scripts/task_selection/task_selection_mult.ipynb b/scripts/task_selection/task_selection_mult.ipynb index eb40e62b2b..e2344137f2 100644 --- a/scripts/task_selection/task_selection_mult.ipynb +++ b/scripts/task_selection/task_selection_mult.ipynb @@ -1083,7 +1083,10 @@ } ], "source": [ - "from mteb.benchmarks import MTEB_MAIN_EN\n", + "import mteb\n", + "\n", + "MTEB_MAIN_EN = mteb.get_benchmark(\"MTEB(eng, classic)\")\n", + "\n", "\n", "exceptions = [\n", " \"STS16\",\n", diff --git a/tests/test_TaskMetadata.py b/tests/test_TaskMetadata.py index ae5fa8f5b0..2b606c2c19 100644 --- a/tests/test_TaskMetadata.py +++ b/tests/test_TaskMetadata.py @@ -4,6 +4,7 @@ import pytest +from mteb import AbsTask from mteb.abstasks.TaskMetadata import TaskMetadata from mteb.overview import get_tasks @@ -203,7 +204,6 @@ def test_given_dataset_config_then_it_is_valid(): dialect=None, sample_creation=None, bibtex_citation="", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) assert my_task.dataset["path"] == "test/dataset" assert my_task.dataset["revision"] == "1.0" @@ -229,7 +229,6 @@ def test_given_missing_dataset_path_then_it_throws(): dialect=None, sample_creation=None, bibtex_citation="", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @@ -256,7 +255,6 @@ def test_given_missing_revision_path_then_it_throws(): dialect=None, sample_creation=None, bibtex_citation="", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) @@ -281,7 +279,6 @@ def test_given_none_revision_path_then_it_logs_warning(caplog): dialect=None, sample_creation=None, bibtex_citation="", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ) assert my_task.dataset["revision"] is None @@ -321,7 +318,6 @@ def test_unfilled_metadata_is_not_filled(): dialect=None, sample_creation=None, bibtex_citation="", - descriptive_stats={"n_samples": None, "avg_character_length": None}, ).is_filled() is False ) @@ -351,10 +347,6 @@ def test_filled_metadata_is_filled(): dialect=[], sample_creation="found", bibtex_citation="Someone et al", - descriptive_stats={ - "n_samples": {"train": 1}, - "avg_character_length": {"train": 1}, - }, ).is_filled() is True ) @@ -532,3 +524,587 @@ def test_disallow_trust_remote_code_in_new_datasets(): assert ( task.metadata.name not in exceptions ), f"Dataset {task.metadata.name} should not trust remote code" + + +def test_empy_descriptive_stat_in_new_datasets(): + # DON'T ADD NEW DATASETS TO THIS LIST + # THIS IS ONLY INTENDED FOR HISTORIC DATASETS + exceptions = [ + "TbilisiCityHallBitextMining", + "BibleNLPBitextMining", + "BUCC.v2", + "DiaBlaBitextMining", + "FloresBitextMining", + "IN22GenBitextMining", + "IndicGenBenchFloresBitextMining", + "IWSLT2017BitextMining", + "LinceMTBitextMining", + "NollySentiBitextMining", + "NorwegianCourtsBitextMining", + "NTREXBitextMining", + "NusaXBitextMining", + "PhincBitextMining", + "RomaTalesBitextMining", + "Tatoeba", + "SRNCorpusBitextMining", + "VieMedEVBitextMining", + "AJGT", + "HotelReviewSentimentClassification", + "OnlineStoreReviewSentimentClassification", + "RestaurantReviewSentimentClassification", + "TweetEmotionClassification", + "TweetSarcasmClassification", + "BengaliDocumentClassification", + "BengaliHateSpeechClassification", + "BengaliSentimentAnalysis", + "BulgarianStoreReviewSentimentClassfication", + "CSFDCZMovieReviewSentimentClassification", + "CzechProductReviewSentimentClassification", + "CzechSoMeSentimentClassification", + "CzechSubjectivityClassification", + "AngryTweetsClassification", + "DanishPoliticalCommentsClassification", + "DKHateClassification", + "LccSentimentClassification", + "GermanPoliticiansTwitterSentimentClassification", + "TenKGnadClassification", + "GreekLegalCodeClassification", + "AmazonPolarityClassification", + "ArxivClassification", + "Banking77Classification", + "DBpediaClassification", + "EmotionClassification", + "FinancialPhrasebankClassification", + "FrenkEnClassification", + "ImdbClassification", + "CanadaTaxCourtOutcomesLegalBenchClassification", + "ContractNLIConfidentialityOfAgreementLegalBenchClassification", + "ContractNLIExplicitIdentificationLegalBenchClassification", + "ContractNLIInclusionOfVerballyConveyedInformationLegalBenchClassification", + "ContractNLILimitedUseLegalBenchClassification", + "ContractNLINoLicensingLegalBenchClassification", + "ContractNLINoticeOnCompelledDisclosureLegalBenchClassification", + "ContractNLIPermissibleAcquirementOfSimilarInformationLegalBenchClassification", + "ContractNLIPermissibleCopyLegalBenchClassification", + "ContractNLIPermissibleDevelopmentOfSimilarInformationLegalBenchClassification", + "ContractNLIPermissiblePostAgreementPossessionLegalBenchClassification", + "ContractNLIReturnOfConfidentialInformationLegalBenchClassification", + "ContractNLISharingWithEmployeesLegalBenchClassification", + "ContractNLISharingWithThirdPartiesLegalBenchClassification", + "ContractNLISurvivalOfObligationsLegalBenchClassification", + "CorporateLobbyingLegalBenchClassification", + "CUADAffiliateLicenseLicenseeLegalBenchClassification", + "CUADAffiliateLicenseLicensorLegalBenchClassification", + "CUADAntiAssignmentLegalBenchClassification", + "CUADAuditRightsLegalBenchClassification", + "CUADCapOnLiabilityLegalBenchClassification", + "CUADChangeOfControlLegalBenchClassification", + "CUADCompetitiveRestrictionExceptionLegalBenchClassification", + "CUADCovenantNotToSueLegalBenchClassification", + "CUADEffectiveDateLegalBenchClassification", + "CUADExclusivityLegalBenchClassification", + "CUADExpirationDateLegalBenchClassification", + "CUADGoverningLawLegalBenchClassification", + "CUADInsuranceLegalBenchClassification", + "CUADIPOwnershipAssignmentLegalBenchClassification", + "CUADIrrevocableOrPerpetualLicenseLegalBenchClassification", + "CUADJointIPOwnershipLegalBenchClassification", + "CUADLicenseGrantLegalBenchClassification", + "CUADLiquidatedDamagesLegalBenchClassification", + "CUADMinimumCommitmentLegalBenchClassification", + "CUADMostFavoredNationLegalBenchClassification", + "CUADNoSolicitOfCustomersLegalBenchClassification", + "CUADNoSolicitOfEmployeesLegalBenchClassification", + "CUADNonCompeteLegalBenchClassification", + "CUADNonDisparagementLegalBenchClassification", + "CUADNonTransferableLicenseLegalBenchClassification", + "CUADNoticePeriodToTerminateRenewalLegalBenchClassification", + "CUADPostTerminationServicesLegalBenchClassification", + "CUADPriceRestrictionsLegalBenchClassification", + "CUADRenewalTermLegalBenchClassification", + "CUADRevenueProfitSharingLegalBenchClassification", + "CUADRofrRofoRofnLegalBenchClassification", + "CUADSourceCodeEscrowLegalBenchClassification", + "CUADTerminationForConvenienceLegalBenchClassification", + "CUADThirdPartyBeneficiaryLegalBenchClassification", + "CUADUncappedLiabilityLegalBenchClassification", + "CUADUnlimitedAllYouCanEatLicenseLegalBenchClassification", + "CUADVolumeRestrictionLegalBenchClassification", + "CUADWarrantyDurationLegalBenchClassification", + "DefinitionClassificationLegalBenchClassification", + "Diversity1LegalBenchClassification", + "Diversity2LegalBenchClassification", + "Diversity3LegalBenchClassification", + "Diversity4LegalBenchClassification", + "Diversity5LegalBenchClassification", + "Diversity6LegalBenchClassification", + "FunctionOfDecisionSectionLegalBenchClassification", + "InsurancePolicyInterpretationLegalBenchClassification", + "InternationalCitizenshipQuestionsLegalBenchClassification", + "JCrewBlockerLegalBenchClassification", + "LearnedHandsBenefitsLegalBenchClassification", + "LearnedHandsBusinessLegalBenchClassification", + "LearnedHandsConsumerLegalBenchClassification", + "LearnedHandsCourtsLegalBenchClassification", + "LearnedHandsCrimeLegalBenchClassification", + "LearnedHandsDivorceLegalBenchClassification", + "LearnedHandsDomesticViolenceLegalBenchClassification", + "LearnedHandsEducationLegalBenchClassification", + "LearnedHandsEmploymentLegalBenchClassification", + "LearnedHandsEstatesLegalBenchClassification", + "LearnedHandsFamilyLegalBenchClassification", + "LearnedHandsHealthLegalBenchClassification", + "LearnedHandsHousingLegalBenchClassification", + "LearnedHandsImmigrationLegalBenchClassification", + "LearnedHandsTortsLegalBenchClassification", + "LearnedHandsTrafficLegalBenchClassification", + "LegalReasoningCausalityLegalBenchClassification", + "MAUDLegalBenchClassification", + "NYSJudicialEthicsLegalBenchClassification", + "OPP115DataRetentionLegalBenchClassification", + "OPP115DataSecurityLegalBenchClassification", + "OPP115DoNotTrackLegalBenchClassification", + "OPP115FirstPartyCollectionUseLegalBenchClassification", + "OPP115InternationalAndSpecificAudiencesLegalBenchClassification", + "OPP115PolicyChangeLegalBenchClassification", + "OPP115ThirdPartySharingCollectionLegalBenchClassification", + "OPP115UserAccessEditAndDeletionLegalBenchClassification", + "OPP115UserChoiceControlLegalBenchClassification", + "OralArgumentQuestionPurposeLegalBenchClassification", + "OverrulingLegalBenchClassification", + "PersonalJurisdictionLegalBenchClassification", + "PROALegalBenchClassification", + "SCDBPAccountabilityLegalBenchClassification", + "SCDBPAuditsLegalBenchClassification", + "SCDBPCertificationLegalBenchClassification", + "SCDBPTrainingLegalBenchClassification", + "SCDBPVerificationLegalBenchClassification", + "SCDDAccountabilityLegalBenchClassification", + "SCDDAuditsLegalBenchClassification", + "SCDDCertificationLegalBenchClassification", + "SCDDTrainingLegalBenchClassification", + "SCDDVerificationLegalBenchClassification", + "TelemarketingSalesRuleLegalBenchClassification", + "TextualismToolDictionariesLegalBenchClassification", + "TextualismToolPlainLegalBenchClassification", + "UCCVCommonLawLegalBenchClassification", + "UnfairTOSLegalBenchClassification", + "NewsClassification", + "PatentClassification", + "PoemSentimentClassification", + "ToxicChatClassification", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "TweetTopicSingleClassification", + "YahooAnswersTopicsClassification", + "YelpReviewFullClassification", + "EstonianValenceClassification", + "PersianFoodSentimentClassification", + "FilipinoHateSpeechClassification", + "FilipinoShopeeReviewsClassification", + "FinToxicityClassification", + "FrenchBookReviews", + "MovieReviewSentimentClassification", + "GujaratiNewsClassification", + "HebrewSentimentAnalysis", + "HindiDiscourseClassification", + "SentimentAnalysisHindi", + "FrenkHrClassification", + "IndonesianIdClickbaitClassification", + "IndonesianMongabayConservationClassification", + "ItaCaseholdClassification", + "Itacola", + "JavaneseIMDBClassification", + "WRIMEClassification", + "KannadaNewsClassification", + "KLUE-TC", + "KorFin", + "KorHateClassification", + "KorSarcasmClassification", + "KurdishSentimentClassification", + "MalayalamNewsClassification", + "MarathiNewsClassification", + "MacedonianTweetSentimentClassification", + "AfriSentiClassification", + "AfriSentiLangClassification", + "AmazonCounterfactualClassification", + "AmazonReviewsClassification", + "CataloniaTweetClassification", + "CyrillicTurkicLangClassification", + "HinDialectClassification", + "IndicLangClassification", + "IndicNLPNewsClassification", + "IndicSentimentClassification", + "MasakhaNEWSClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MTOPDomainClassification", + "MTOPIntentClassification", + "MultiHateClassification", + "MultilingualSentimentClassification", + "NaijaSenti", + "NordicLangClassification", + "NusaParagraphEmotionClassification", + "NusaParagraphTopicClassification", + "NusaX-senti", + "ScalaClassification", + "SIB200Classification", + "SouthAfricanLangClassification", + "SwissJudgementClassification", + "TurkicClassification", + "TweetSentimentClassification", + "MyanmarNews", + "NepaliNewsClassification", + "DutchBookReviewSentimentClassification", + "NoRecClassification", + "NorwegianParliamentClassification", + "OdiaNewsClassification", + "PunjabiNewsClassification", + "CBD", + "PolEmo2.0-IN", + "PolEmo2.0-OUT", + "AllegroReviews", + "PAC", + "HateSpeechPortugueseClassification", + "Moroco", + "RomanianReviewsSentiment", + "RomanianSentimentClassification", + "GeoreviewClassification", + "HeadlineClassification", + "InappropriatenessClassification", + "KinopoiskClassification", + "RuReviewsClassification", + "RuSciBenchGRNTIClassification", + "RuSciBenchOECDClassification", + "SanskritShlokasClassification", + "SinhalaNewsClassification", + "SinhalaNewsSourceClassification", + "CSFDSKMovieReviewSentimentClassification", + "FrenkSlClassification", + "SpanishNewsClassification", + "SpanishSentimentClassification", + "SiswatiNewsClassification", + "SlovakMovieReviewSentimentClassification", + "SwahiliNewsClassification", + "DalajClassification", + "SwedishSentimentClassification", + "SweRecClassification", + "TamilNewsClassification", + "TeluguAndhraJyotiNewsClassification", + "WisesightSentimentClassification", + "TswanaNewsClassification", + "TurkishMovieSentimentClassification", + "TurkishProductSentimentClassification", + "UkrFormalityClassification", + "UrduRomanSentimentClassification", + "VieStudentFeedbackClassification", + "TNews", + "IFlyTek", + "MultilingualSentiment", + "JDReview", + "OnlineShopping", + "Waimai", + "YueOpenriceReviewClassification", + "IsiZuluNewsClassification", + "WikiCitiesClustering", + "IndicReviewsClusteringP2P", + "MasakhaNEWSClusteringP2P", + "MasakhaNEWSClusteringS2S", + "RomaniBibleClustering", + "SpanishNewsClusteringP2P", + "BlurbsClusteringP2P.v2", + "BlurbsClusteringS2S.v2", + "TenKGnadClusteringP2P.v2", + "TenKGnadClusteringS2S.v2", + "ArXivHierarchicalClusteringS2S", + "BigPatentClustering.v2", + "BiorxivClusteringP2P.v2", + "BiorxivClusteringS2S.v2", + "MedrxivClusteringP2P.v2", + "MedrxivClusteringS2S.v2", + "RedditClustering.v2", + "RedditClusteringP2P.v2", + "StackExchangeClustering.v2", + "StackExchangeClusteringP2P.v2", + "TwentyNewsgroupsClustering.v2", + "AlloProfClusteringP2P.v2", + "AlloProfClusteringS2S.v2", + "HALClusteringS2S.v2", + "LivedoorNewsClustering.v2", + "MewsC16JaClustering", + "MLSUMClusteringP2P.v2", + "MLSUMClusteringS2S.v2", + "SIB200ClusteringS2S", + "WikiClusteringP2P.v2", + "SNLHierarchicalClusteringP2P", + "SNLHierarchicalClusteringS2S", + "VGHierarchicalClusteringP2P", + "VGHierarchicalClusteringS2S", + "EightTagsClustering.v2", + "PlscClusteringS2S.v2", + "PlscClusteringP2P.v2", + "GeoreviewClusteringP2P", + "RuSciBenchOECDClusteringP2P", + "SwednClusteringP2P", + "SwednClusteringS2S", + "CLSClusteringS2S.v2", + "CLSClusteringP2P.v2", + "ThuNewsClusteringS2S.v2", + "ThuNewsClusteringP2P.v2", + "SadeemQuestionRetrieval", + "DanFeverRetrieval", + "TV2Nordretrieval", + "TwitterHjerneRetrieval", + "GerDaLIR", + "GerDaLIRSmall", + "GermanDPR", + "GermanGovServiceRetrieval", + "GermanQuAD-Retrieval", + "LegalQuAD", + "GreekCivicsQA", + "AILACasedocs", + "AILAStatutes", + "AlphaNLI", + "ARCChallenge", + "ArguAna", + "BrightRetrieval", + "ClimateFEVER", + "ClimateFEVERHardNegatives", + "CQADupstackAndroidRetrieval", + "CQADupstackEnglishRetrieval", + "CQADupstackGamingRetrieval", + "CQADupstackGisRetrieval", + "CQADupstackMathematicaRetrieval", + "CQADupstackPhysicsRetrieval", + "CQADupstackProgrammersRetrieval", + "CQADupstackStatsRetrieval", + "CQADupstackTexRetrieval", + "CQADupstackUnixRetrieval", + "CQADupstackWebmastersRetrieval", + "CQADupstackWordpressRetrieval", + "DBPedia", + "DBPediaHardNegatives", + "FaithDial", + "FeedbackQARetrieval", + "FEVER", + "FEVERHardNegatives", + "FiQA2018", + "HagridRetrieval", + "HellaSwag", + "HotpotQA", + "HotpotQAHardNegatives", + "LegalBenchConsumerContractsQA", + "LegalBenchCorporateLobbying", + "LegalSummarization", + "LEMBNarrativeQARetrieval", + "LEMBNeedleRetrieval", + "LEMBPasskeyRetrieval", + "LEMBQMSumRetrieval", + "LEMBSummScreenFDRetrieval", + "LEMBWikimQARetrieval", + "LitSearchRetrieval", + "MedicalQARetrieval", + "MLQuestions", + "MSMARCO", + "MSMARCOHardNegatives", + "MSMARCOv2", + "NarrativeQARetrieval", + "NFCorpus", + "NQ", + "NQHardNegatives", + "PIQA", + "Quail", + "QuoraRetrieval", + "QuoraRetrievalHardNegatives", + "RARbCode", + "RARbMath", + "SCIDOCS", + "SciFact", + "SIQA", + "SpartQA", + "TempReasonL1", + "TempReasonL2Context", + "TempReasonL2Fact", + "TempReasonL2Pure", + "TempReasonL3Context", + "TempReasonL3Fact", + "TempReasonL3Pure", + "TopiOCQA", + "TopiOCQAHardNegatives", + "TRECCOVID", + "WinoGrande", + "EstQA", + "AlloprofRetrieval", + "BSARDRetrieval", + "FQuADRetrieval", + "SyntecRetrieval", + "HunSum2AbstractiveRetrieval", + "JaGovFaqsRetrieval", + "JaQuADRetrieval", + "NLPJournalAbsIntroRetrieval", + "NLPJournalTitleAbsRetrieval", + "NLPJournalTitleIntroRetrieval", + "GeorgianFAQRetrieval", + "Ko-StrategyQA", + "CrossLingualSemanticDiscriminationWMT19", + "CrossLingualSemanticDiscriminationWMT21", + "IndicQARetrieval", + "MintakaRetrieval", + "MIRACLRetrieval", + "MIRACLRetrievalHardNegatives", + "MLQARetrieval", + "MrTidyRetrieval", + "MultiLongDocRetrieval", + "NeuCLIR2022Retrieval", + "NeuCLIR2022RetrievalHardNegatives", + "NeuCLIR2023Retrieval", + "NeuCLIR2023RetrievalHardNegatives", + "PublicHealthQA", + "StatcanDialogueDatasetRetrieval", + "WikipediaRetrievalMultilingual", + "XMarket", + "XPQARetrieval", + "XQuADRetrieval", + "NorQuadRetrieval", + "SNLRetrieval", + "ArguAna-PL", + "DBPedia-PL", + "DBPedia-PLHardNegatives", + "FiQA-PL", + "HotpotQA-PL", + "HotpotQA-PLHardNegatives", + "MSMARCO-PL", + "MSMARCO-PLHardNegatives", + "NFCorpus-PL", + "NQ-PL", + "NQ-PLHardNegatives", + "Quora-PL", + "Quora-PLHardNegatives", + "SCIDOCS-PL", + "SciFact-PL", + "TRECCOVID-PL", + "RiaNewsRetrieval", + "RiaNewsRetrievalHardNegatives", + "RuBQRetrieval", + "SKQuadRetrieval", + "SlovakSumRetrieval", + "SpanishPassageRetrievalS2P", + "SpanishPassageRetrievalS2S", + "SwednRetrieval", + "SweFaqRetrieval", + "TurHistQuadRetrieval", + "VieQuADRetrieval", + "T2Retrieval", + "MMarcoRetrieval", + "DuRetrieval", + "CovidRetrieval", + "CmedqaRetrieval", + "EcomRetrieval", + "MedicalRetrieval", + "VideoRetrieval", + "LeCaRDv2", + "News21InstructionRetrieval", + "Robust04InstructionRetrieval", + "KorHateSpeechMLClassification", + "MalteseNewsClassification", + "BrazilianToxicTweetsClassification", + "SensitiveTopicsClassification", + "ArEntail", + "CTKFactsNLI", + "FalseFriendsGermanEnglish", + "LegalBenchPC", + "SprintDuplicateQuestions", + "TwitterSemEval2015", + "FarsTail", + "ArmenianParaphrasePC", + "indonli", + "KLUE-NLI", + "OpusparcusPC", + "RTE3", + "XNLIV2", + "XStance", + "SICK-E-PL", + "PpcPC", + "CDSC-E", + "PSC", + "Assin2RTE", + "SICK-BR-PC", + "TERRa", + "Ocnli", + "Cmnli", + "MindSmallReranking", + "SciDocsRR", + "StackOverflowDupQuestions", + "WebLINXCandidatesReranking", + "AlloprofReranking", + "SyntecReranking", + "VoyageMMarcoReranking", + "MIRACLReranking", + "RuBQReranking", + "T2Reranking", + "MMarcoReranking", + "CMedQAv1-reranking", + "CMedQAv2-reranking", + "CPUSpeedTask", + "GPUSpeedTask", + "GermanSTSBenchmark", + "BIOSSES", + "SICK-R", + "STS13", + "STS14", + "STS15", + "STS16", + "STSBenchmark", + "FaroeseSTS", + "FinParaSTS", + "SICKFr", + "JSICK", + "JSTS", + "KLUE-STS", + "KorSTS", + "IndicCrosslingualSTS", + "SemRel24STS", + "STS22.v2", + "STSBenchmarkMultilingualSTS", + "SICK-R-PL", + "CDSC-R", + "Assin2STS", + "SICK-BR-STS", + "RonSTS", + "RUParaPhraserSTS", + "RuSTSBenchmarkSTS", + "STSES", + "ATEC", + "BQ", + "LCQMC", + "PAWSX", + "STSB", + "AFQMC", + "QBQTC", + "SummEvalSummarization.v2", + "SummEvalFrSummarization.v2", + ] + + assert ( + 553 == len(exceptions) + ), "The number of exceptions has changed. Please do not add new datasets to this list." + + exceptions = [] + + for task in get_tasks(): + if task.metadata.descriptive_stats is None: + assert ( + task.metadata.name not in exceptions + ), f"Dataset {task.metadata.name} should have descriptive stats" + + +@pytest.mark.parametrize("task", get_tasks()) +def test_eval_langs_correctly_specified(task: AbsTask): + if task.is_multilingual: + assert isinstance( + task.metadata.eval_langs, dict + ), f"{task.metadata.name} should have eval_langs as a dict" + else: + assert isinstance( + task.metadata.eval_langs, list + ), f"{task.metadata.name} should have eval_langs as a list" diff --git a/tests/test_benchmark/mock_models.py b/tests/test_benchmark/mock_models.py index ad401cf3dc..1043c791f6 100644 --- a/tests/test_benchmark/mock_models.py +++ b/tests/test_benchmark/mock_models.py @@ -34,7 +34,7 @@ def encode(self, sentences, prompt_name: str | None = None, **kwargs): return torch.randn(len(sentences), 10).numpy() -class MockTorchbf16Encoder(mteb.Encoder): +class MockTorchbf16Encoder(SentenceTransformer): def __init__(self): pass diff --git a/tests/test_benchmark/mock_tasks.py b/tests/test_benchmark/mock_tasks.py index bbd07b4019..eea73b2e69 100644 --- a/tests/test_benchmark/mock_tasks.py +++ b/tests/test_benchmark/mock_tasks.py @@ -67,42 +67,55 @@ class MockClassificationTask(AbsTaskClassification): + expected_stats = { + "test": { + "num_samples": 2, + "number_of_characters": 52, + "num_texts_in_train": 1, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 1}, "1": {"count": 1}}, + }, + "train": { + "num_samples": 2, + "number_of_characters": 53, + "num_texts_in_train": None, + "min_text_length": 23, + "average_text_length": 26.5, + "max_text_length": 30, + "unique_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 1}, "1": {"count": 1}}, + }, + } + metadata = TaskMetadata( type="Classification", name="MockClassificationTask", main_score="accuracy", **general_args, # type: ignore - descriptive_stats={ - "test": { - "num_samples": 2, - "average_text_length": 26.0, - "unique_labels": 2, - "labels": {"0": {"count": 1}, "1": {"count": 1}}, - }, - "train": { - "num_samples": 2, - "average_text_length": 26.0, - "unique_labels": 2, - "labels": {"0": {"count": 1}, "1": {"count": 1}}, - }, - }, ) def load_data(self, **kwargs): - texts = ["This is a test sentence", "This is another test sentence"] + train_texts = ["This is a test sentence", "This is another train sentence"] + test_texts = ["This is a test sentence", "This is another test sentence"] + labels = [0, 1] self.dataset = DatasetDict( { "test": Dataset.from_dict( { - "text": texts, + "text": test_texts, "label": labels, } ), "train": Dataset.from_dict( { - "text": texts, + "text": train_texts, "label": labels, } ), @@ -112,67 +125,101 @@ def load_data(self, **kwargs): class MockMultilingualClassificationTask(AbsTaskClassification, MultilingualTask): - metadata = TaskMetadata( - type="Classification", - name="MockMultilingualClassificationTask", - main_score="accuracy", - descriptive_stats={ - "test": { - "num_samples": 4, - "average_text_length": 26.0, - "unique_labels": 2, - "labels": {"0": {"count": 2}, "1": {"count": 2}}, - "hf_subset_descriptive_stats": {}, + expected_stats = { + "test": { + "num_samples": 4, + "number_of_characters": 104, + "num_texts_in_train": 1, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 2}, "1": {"count": 2}}, + "hf_subset_descriptive_stats": { "eng": { "num_samples": 2, + "number_of_characters": 52, + "num_texts_in_train": 1, + "min_text_length": 23, "average_text_length": 26.0, + "max_text_length": 29, + "unique_text": 2, "unique_labels": 2, "labels": {"0": {"count": 1}, "1": {"count": 1}}, }, "fra": { "num_samples": 2, + "number_of_characters": 52, + "num_texts_in_train": 1, + "min_text_length": 23, "average_text_length": 26.0, + "max_text_length": 29, + "unique_text": 2, "unique_labels": 2, "labels": {"0": {"count": 1}, "1": {"count": 1}}, }, }, - "train": { - "num_samples": 4, - "average_text_length": 26.0, - "unique_labels": 2, - "labels": {"0": {"count": 2}, "1": {"count": 2}}, - "hf_subset_descriptive_stats": {}, + }, + "train": { + "num_samples": 4, + "number_of_characters": 106, + "num_texts_in_train": None, + "min_text_length": 23, + "average_text_length": 26.5, + "max_text_length": 30, + "unique_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 2}, "1": {"count": 2}}, + "hf_subset_descriptive_stats": { "eng": { "num_samples": 2, - "average_text_length": 26.0, + "number_of_characters": 53, + "num_texts_in_train": None, + "min_text_length": 23, + "average_text_length": 26.5, + "max_text_length": 30, + "unique_text": 2, "unique_labels": 2, "labels": {"0": {"count": 1}, "1": {"count": 1}}, }, "fra": { "num_samples": 2, - "average_text_length": 26.0, + "number_of_characters": 53, + "num_texts_in_train": None, + "min_text_length": 23, + "average_text_length": 26.5, + "max_text_length": 30, + "unique_text": 2, "unique_labels": 2, "labels": {"0": {"count": 1}, "1": {"count": 1}}, }, }, }, + } + + metadata = TaskMetadata( + type="Classification", + name="MockMultilingualClassificationTask", + main_score="accuracy", **general_args, # type: ignore ) metadata.eval_langs = multilingual_eval_langs def load_data(self, **kwargs): - texts = ["This is a test sentence", "This is another test sentence"] + train_texts = ["This is a test sentence", "This is another train sentence"] + test_texts = ["This is a test sentence", "This is another test sentence"] labels = [0, 1] data = { "test": Dataset.from_dict( { - "text": texts, + "text": test_texts, "label": labels, } ), "train": Dataset.from_dict( { - "text": texts, + "text": train_texts, "label": labels, } ), @@ -188,17 +235,26 @@ def load_data(self, **kwargs): class MockBitextMiningTask(AbsTaskBitextMining): + expected_stats = { + "test": { + "num_samples": 2, + "number_of_characters": 113, + "unique_pairs": 2, + "min_sentence1_length": 23, + "average_sentence1_length": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "average_sentence2_length": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + } + } + metadata = TaskMetadata( type="BitextMining", name="MockBitextMiningTask", main_score="accuracy", - descriptive_stats={ - "test": { - "average_sentence1_length": 26.0, - "average_sentence2_length": 30.5, - "num_samples": 2, - } - }, **general_args, # type: ignore ) @@ -223,29 +279,54 @@ def load_data(self, **kwargs): class MockMultilingualBitextMiningTask(AbsTaskBitextMining, MultilingualTask): + expected_stats = { + "test": { + "num_samples": 4, + "number_of_characters": 226, + "unique_pairs": 2, + "min_sentence1_length": 23, + "average_sentence1_length": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "average_sentence2_length": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 2, + "number_of_characters": 113, + "unique_pairs": 2, + "min_sentence1_length": 23, + "average_sentence1_length": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "average_sentence2_length": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + }, + "fra": { + "num_samples": 2, + "number_of_characters": 113, + "unique_pairs": 2, + "min_sentence1_length": 23, + "average_sentence1_length": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "average_sentence2_length": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + }, + }, + } + } + metadata = TaskMetadata( type="BitextMining", name="MockMultilingualBitextMiningTask", main_score="accuracy", - descriptive_stats={ - "test": { - "average_sentence1_length": 26.0, - "average_sentence2_length": 30.5, - "num_samples": 4, - "hf_subset_descriptive_stats": { - "eng": { - "average_sentence1_length": 26.0, - "average_sentence2_length": 30.5, - "num_samples": 2, - }, - "fra": { - "average_sentence1_length": 26.0, - "average_sentence2_length": 30.5, - "num_samples": 2, - }, - }, - } - }, **general_args, # type: ignore ) metadata.eval_langs = multilingual_eval_langs @@ -275,30 +356,54 @@ def load_data(self, **kwargs): class MockMultilingualParallelBitextMiningTask(AbsTaskBitextMining, MultilingualTask): parallel_subsets = True + expected_stats = { + "test": { + "num_samples": 4, + "number_of_characters": 226, + "unique_pairs": 4, + "min_sentence1_length": 23, + "average_sentence1_length": 28.25, + "max_sentence1_length": 37, + "unique_sentence1": 4, + "min_sentence2_length": 23, + "average_sentence2_length": 28.25, + "max_sentence2_length": 37, + "unique_sentence2": 4, + "hf_subset_descriptive_stats": { + "eng_Latn-fra_Latn": { + "num_samples": 2, + "number_of_characters": 113, + "unique_pairs": 2, + "min_sentence1_length": 23, + "average_sentence1_length": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "average_sentence2_length": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + }, + "fra_Latn-eng_Latn": { + "num_samples": 2, + "number_of_characters": 113, + "unique_pairs": 2, + "min_sentence1_length": 24, + "average_sentence1_length": 30.5, + "max_sentence1_length": 37, + "unique_sentence1": 2, + "min_sentence2_length": 23, + "average_sentence2_length": 26.0, + "max_sentence2_length": 29, + "unique_sentence2": 2, + }, + }, + } + } metadata = TaskMetadata( type="BitextMining", name="MockMultilingualParallelBitextMiningTask", main_score="accuracy", - descriptive_stats={ - "test": { - "average_sentence1_length": 28.25, - "average_sentence2_length": 28.25, - "num_samples": 4, - "hf_subset_descriptive_stats": { - "eng_Latn-fra_Latn": { - "average_sentence1_length": 26.0, - "average_sentence2_length": 30.5, - "num_samples": 2, - }, - "fra_Latn-eng_Latn": { - "average_sentence1_length": 30.5, - "average_sentence2_length": 26.0, - "num_samples": 2, - }, - }, - } - }, **general_args, # type: ignore ) metadata.eval_langs = { @@ -327,19 +432,26 @@ def load_data(self, **kwargs): class MockClusteringTask(AbsTaskClustering): + expected_stats = { + "test": { + "num_samples": 1, + "number_of_characters": 3, + "min_text_length": 3, + "average_text_length": 3.0, + "max_text_length": 3, + "unique_texts": 3, + "min_labels_per_text": 1, + "average_labels_per_text": 3.0, + "max_labels_per_text": 1, + "unique_labels": 3, + "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, + } + } + metadata = TaskMetadata( type="Clustering", name="MockClusteringTask", main_score="v_measure", - descriptive_stats={ - "test": { - "num_samples": 1, - "average_text_length": 3.0, - "average_labels_per_text": 3.0, - "unique_labels": 3, - "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, - } - }, **general_args, # type: ignore ) @@ -367,43 +479,54 @@ def load_data(self, **kwargs): class MockMultilingualClusteringTask(AbsTaskClustering, MultilingualTask): + expected_stats = { + "test": { + "num_samples": 2, + "number_of_characters": 6, + "min_text_length": 3, + "average_text_length": 3.0, + "max_text_length": 3, + "unique_texts": 3, + "min_labels_per_text": 2, + "average_labels_per_text": 3.0, + "max_labels_per_text": 2, + "unique_labels": 3, + "labels": {"0": {"count": 2}, "1": {"count": 2}, "2": {"count": 2}}, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 1, + "number_of_characters": 3, + "min_text_length": 3, + "average_text_length": 3.0, + "max_text_length": 3, + "unique_texts": 3, + "min_labels_per_text": 1, + "average_labels_per_text": 3.0, + "max_labels_per_text": 1, + "unique_labels": 3, + "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, + }, + "fra": { + "num_samples": 1, + "number_of_characters": 3, + "min_text_length": 3, + "average_text_length": 3.0, + "max_text_length": 3, + "unique_texts": 3, + "min_labels_per_text": 1, + "average_labels_per_text": 3.0, + "max_labels_per_text": 1, + "unique_labels": 3, + "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, + }, + }, + } + } + metadata = TaskMetadata( type="Clustering", name="MockMultilingualClusteringTask", main_score="v_measure", - descriptive_stats={ - "test": { - "num_samples": 2, - "average_text_length": 3.0, - "average_labels_per_text": 3.0, - "unique_labels": 3, - "labels": {"0": {"count": 2}, "1": {"count": 2}, "2": {"count": 2}}, - "hf_subset_descriptive_stats": { - "eng": { - "num_samples": 1, - "average_text_length": 3.0, - "average_labels_per_text": 3.0, - "unique_labels": 3, - "labels": { - "0": {"count": 1}, - "1": {"count": 1}, - "2": {"count": 1}, - }, - }, - "fra": { - "num_samples": 1, - "average_text_length": 3.0, - "average_labels_per_text": 3.0, - "unique_labels": 3, - "labels": { - "0": {"count": 1}, - "1": {"count": 1}, - "2": {"count": 1}, - }, - }, - }, - } - }, **general_args, # type: ignore ) metadata.eval_langs = multilingual_eval_langs @@ -438,19 +561,25 @@ def load_data(self, **kwargs): class MockClusteringFastTask(AbsTaskClusteringFast): max_document_to_embed = 3 max_fraction_of_documents_to_embed = None + expected_stats = { + "test": { + "num_samples": 3, + "number_of_characters": 81, + "min_text_length": 23, + "average_text_length": 27.0, + "max_text_length": 29, + "min_labels_per_text": 1, + "average_labels_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 3, + "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, + } + } + metadata = TaskMetadata( type="Clustering", name="MockClusteringFastTask", main_score="v_measure", - descriptive_stats={ - "test": { - "num_samples": 3, - "average_text_length": 27.0, - "average_labels_per_text": 1.0, - "unique_labels": 3, - "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, - } - }, **general_args, # type: ignore ) @@ -478,43 +607,51 @@ def load_data(self, **kwargs): class MockMultilingualClusteringFastTask(AbsTaskClusteringFast, MultilingualTask): max_document_to_embed = 3 max_fraction_of_documents_to_embed = None + expected_stats = { + "test": { + "num_samples": 6, + "number_of_characters": 162, + "min_text_length": 23, + "average_text_length": 27.0, + "max_text_length": 29, + "min_labels_per_text": 2, + "average_labels_per_text": 1.0, + "max_labels_per_text": 2, + "unique_labels": 3, + "labels": {"0": {"count": 2}, "1": {"count": 2}, "2": {"count": 2}}, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 3, + "number_of_characters": 81, + "min_text_length": 23, + "average_text_length": 27.0, + "max_text_length": 29, + "min_labels_per_text": 1, + "average_labels_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 3, + "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, + }, + "fra": { + "num_samples": 3, + "number_of_characters": 81, + "min_text_length": 23, + "average_text_length": 27.0, + "max_text_length": 29, + "min_labels_per_text": 1, + "average_labels_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 3, + "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, + }, + }, + } + } + metadata = TaskMetadata( type="Clustering", name="MockMultilingualClusteringFastTask", main_score="v_measure", - descriptive_stats={ - "test": { - "num_samples": 6, - "average_text_length": 27.0, - "average_labels_per_text": 1.0, - "unique_labels": 3, - "labels": {"0": {"count": 2}, "1": {"count": 2}, "2": {"count": 2}}, - "hf_subset_descriptive_stats": { - "eng": { - "num_samples": 3, - "average_text_length": 27.0, - "average_labels_per_text": 1.0, - "unique_labels": 3, - "labels": { - "0": {"count": 1}, - "1": {"count": 1}, - "2": {"count": 1}, - }, - }, - "fra": { - "num_samples": 3, - "average_text_length": 27.0, - "average_labels_per_text": 1.0, - "unique_labels": 3, - "labels": { - "0": {"count": 1}, - "1": {"count": 1}, - "2": {"count": 1}, - }, - }, - }, - } - }, **general_args, # type: ignore ) metadata.eval_langs = multilingual_eval_langs @@ -545,19 +682,27 @@ def load_data(self, **kwargs): class MockPairClassificationTask(AbsTaskPairClassification): + expected_stats = { + "test": { + "num_samples": 2, + "number_of_characters": 113, + "min_sentence1_length": 23, + "avg_sentence1_length": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "avg_sentence2_length": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + } + } + metadata = TaskMetadata( type="PairClassification", name="MockPairClassificationTask", main_score="similarity_ap", - descriptive_stats={ - "test": { - "num_samples": 2, - "avg_sentence1_len": 26.0, - "avg_sentence2_len": 30.5, - "unique_labels": 2, - "labels": {"1": {"count": 1}, "0": {"count": 1}}, - } - }, **general_args, # type: ignore ) @@ -588,35 +733,57 @@ def load_data(self, **kwargs): class MockMultilingualPairClassificationTask( AbsTaskPairClassification, MultilingualTask ): + expected_stats = { + "test": { + "num_samples": 4, + "number_of_characters": 226, + "min_sentence1_length": 23, + "avg_sentence1_length": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "avg_sentence2_length": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + "unique_labels": 2, + "labels": {"1": {"count": 2}, "0": {"count": 2}}, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 2, + "number_of_characters": 113, + "min_sentence1_length": 23, + "avg_sentence1_length": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "avg_sentence2_length": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, + "fra": { + "num_samples": 2, + "number_of_characters": 113, + "min_sentence1_length": 23, + "avg_sentence1_length": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "avg_sentence2_length": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, + }, + } + } + metadata = TaskMetadata( type="PairClassification", name="MockMultilingualPairClassificationTask", main_score="similarity_ap", - descriptive_stats={ - "test": { - "num_samples": 4, - "avg_sentence1_len": 26.0, - "avg_sentence2_len": 30.5, - "unique_labels": 2, - "labels": {"1": {"count": 2}, "0": {"count": 2}}, - "hf_subset_descriptive_stats": { - "eng": { - "num_samples": 2, - "avg_sentence1_len": 26.0, - "avg_sentence2_len": 30.5, - "unique_labels": 2, - "labels": {"1": {"count": 1}, "0": {"count": 1}}, - }, - "fra": { - "num_samples": 2, - "avg_sentence1_len": 26.0, - "avg_sentence2_len": 30.5, - "unique_labels": 2, - "labels": {"1": {"count": 1}, "0": {"count": 1}}, - }, - }, - } - }, **general_args, # type: ignore ) metadata.eval_langs = multilingual_eval_langs @@ -649,18 +816,28 @@ def load_data(self, **kwargs): class MockSTSTask(AbsTaskSTS): + expected_stats = { + "test": { + "num_samples": 2, + "number_of_characters": 113, + "min_sentence1_length": 23, + "average_sentence1_len": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "average_sentence2_len": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + "min_score": 0, + "avg_score": 0.5, + "max_score": 1, + } + } + metadata = TaskMetadata( type="STS", name="MockSTSTask", main_score="cosine_spearman", - descriptive_stats={ - "test": { - "num_samples": 2, - "average_sentence1_len": 26.0, - "average_sentence2_len": 30.5, - "avg_score": 0.5, - } - }, **general_args, # type: ignore ) @@ -694,32 +871,60 @@ def metadata_dict(self) -> dict[str, str]: class MockMultilingualSTSTask(AbsTaskSTS, MultilingualTask): + expected_stats = { + "test": { + "num_samples": 4, + "number_of_characters": 226, + "min_sentence1_length": 23, + "average_sentence1_len": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "average_sentence2_len": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + "min_score": 0, + "avg_score": 0.5, + "max_score": 1, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 2, + "number_of_characters": 113, + "min_sentence1_length": 23, + "average_sentence1_len": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "average_sentence2_len": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + "min_score": 0, + "avg_score": 0.5, + "max_score": 1, + }, + "fra": { + "num_samples": 2, + "number_of_characters": 113, + "min_sentence1_length": 23, + "average_sentence1_len": 26.0, + "max_sentence1_length": 29, + "unique_sentence1": 2, + "min_sentence2_length": 24, + "average_sentence2_len": 30.5, + "max_sentence2_length": 37, + "unique_sentence2": 2, + "min_score": 0, + "avg_score": 0.5, + "max_score": 1, + }, + }, + } + } + metadata = TaskMetadata( type="STS", name="MockMultilingualSTSTask", main_score="cosine_spearman", - descriptive_stats={ - "test": { - "num_samples": 4, - "average_sentence1_len": 26.0, - "average_sentence2_len": 30.5, - "avg_score": 0.5, - "hf_subset_descriptive_stats": { - "eng": { - "num_samples": 2, - "average_sentence1_len": 26.0, - "average_sentence2_len": 30.5, - "avg_score": 0.5, - }, - "fra": { - "num_samples": 2, - "average_sentence1_len": 26.0, - "average_sentence2_len": 30.5, - "avg_score": 0.5, - }, - }, - } - }, **general_args, # type: ignore ) metadata.eval_langs = multilingual_eval_langs @@ -741,12 +946,10 @@ def load_data(self, **kwargs): ), } - self.dataset = DatasetDict( - { - "eng": data, - "fra": data, - } - ) + self.dataset = {} + for lang in self.hf_subsets: + self.dataset[lang] = data + self.data_loaded = True @property @@ -758,19 +961,32 @@ def metadata_dict(self) -> dict[str, str]: class MockSummarizationTask(AbsTaskSummarization): + expected_stats = { + "test": { + "num_samples": 2, + "number_of_characters": 60, + "min_text_length": 23, + "avg_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + "min_human_summaries_length": 2, + "avg_human_summaries_length": 2.0, + "max_human_summaries_length": 2, + "unique_human_summaries": 2, + "min_machine_summaries_length": 2, + "avg_machine_summaries_length": 2.0, + "max_machine_summaries_length": 2, + "unique_machine_summaries": 2, + "min_relevance": [0, 1], + "avg_relevance": 0.5, + "max_relevance": [1, 0], + } + } + metadata = TaskMetadata( type="Summarization", name="MockSummarizationTask", main_score="cosine_spearman", - descriptive_stats={ - "test": { - "num_samples": 2, - "avg_text_len": 26.0, - "avg_human_summaries_len": 2.0, - "avg_machine_summaries_len": 2.0, - "avg_relevance": 0.5, - } - }, **general_args, # type: ignore ) @@ -809,35 +1025,72 @@ def metadata_dict(self) -> dict[str, str]: class MockMultilingualSummarizationTask(AbsTaskSummarization, MultilingualTask): + expected_stats = { + "test": { + "num_samples": 4, + "number_of_characters": 120, + "min_text_length": 23, + "avg_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + "min_human_summaries_length": 2, + "avg_human_summaries_length": 2.0, + "max_human_summaries_length": 2, + "unique_human_summaries": 2, + "min_machine_summaries_length": 2, + "avg_machine_summaries_length": 2.0, + "max_machine_summaries_length": 2, + "unique_machine_summaries": 2, + "min_relevance": [0, 1], + "avg_relevance": 0.5, + "max_relevance": [1, 0], + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 2, + "number_of_characters": 60, + "min_text_length": 23, + "avg_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + "min_human_summaries_length": 2, + "avg_human_summaries_length": 2.0, + "max_human_summaries_length": 2, + "unique_human_summaries": 2, + "min_machine_summaries_length": 2, + "avg_machine_summaries_length": 2.0, + "max_machine_summaries_length": 2, + "unique_machine_summaries": 2, + "min_relevance": [0, 1], + "avg_relevance": 0.5, + "max_relevance": [1, 0], + }, + "fra": { + "num_samples": 2, + "number_of_characters": 60, + "min_text_length": 23, + "avg_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + "min_human_summaries_length": 2, + "avg_human_summaries_length": 2.0, + "max_human_summaries_length": 2, + "unique_human_summaries": 2, + "min_machine_summaries_length": 2, + "avg_machine_summaries_length": 2.0, + "max_machine_summaries_length": 2, + "unique_machine_summaries": 2, + "min_relevance": [0, 1], + "avg_relevance": 0.5, + "max_relevance": [1, 0], + }, + }, + } + } + metadata = TaskMetadata( type="Summarization", name="MockMultilingualSummarizationTask", main_score="cosine_spearman", - descriptive_stats={ - "test": { - "num_samples": 4, - "avg_text_len": 26.0, - "avg_human_summaries_len": 2.0, - "avg_machine_summaries_len": 2.0, - "avg_relevance": 0.5, - "hf_subset_descriptive_stats": { - "eng": { - "num_samples": 2, - "avg_text_len": 26.0, - "avg_human_summaries_len": 2.0, - "avg_machine_summaries_len": 2.0, - "avg_relevance": 0.5, - }, - "fra": { - "num_samples": 2, - "avg_text_len": 26.0, - "avg_human_summaries_len": 2.0, - "avg_machine_summaries_len": 2.0, - "avg_relevance": 0.5, - }, - }, - } - }, **general_args, # type: ignore ) metadata.eval_langs = multilingual_eval_langs @@ -880,20 +1133,31 @@ def metadata_dict(self) -> dict[str, str]: class MockRerankingTask(AbsTaskReranking): + expected_stats = { + "test": { + "num_samples": 2, + "number_of_characters": 172, + "num_positive": 2, + "num_negative": 2, + "min_query_length": 23, + "avg_query_length": 26.0, + "max_query_length": 29, + "unique_query": 2, + "min_positive_length": 27, + "avg_positive_length": 30.0, + "max_positive_length": 33, + "unique_positive": 2, + "min_negative_length": 27, + "avg_negative_length": 30.0, + "max_negative_length": 33, + "unique_negative": 2, + } + } + metadata = TaskMetadata( type="Reranking", name="MockRerankingTask", main_score="map", - descriptive_stats={ - "test": { - "num_samples": 2, - "num_positive": 2, - "num_negative": 2, - "avg_query_len": 26.0, - "avg_positive_len": 30.0, - "avg_negative_len": 30.0, - } - }, **general_args, # type: ignore ) @@ -923,38 +1187,69 @@ def load_data(self, **kwargs): class MockMultilingualRerankingTask(AbsTaskReranking, MultilingualTask): + expected_stats = { + "test": { + "num_samples": 4, + "number_of_characters": 344, + "num_positive": 4, + "num_negative": 4, + "min_query_length": 23, + "avg_query_length": 26.0, + "max_query_length": 29, + "unique_query": 2, + "min_positive_length": 27, + "avg_positive_length": 30.0, + "max_positive_length": 33, + "unique_positive": 2, + "min_negative_length": 27, + "avg_negative_length": 30.0, + "max_negative_length": 33, + "unique_negative": 2, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 2, + "number_of_characters": 172, + "num_positive": 2, + "num_negative": 2, + "min_query_length": 23, + "avg_query_length": 26.0, + "max_query_length": 29, + "unique_query": 2, + "min_positive_length": 27, + "avg_positive_length": 30.0, + "max_positive_length": 33, + "unique_positive": 2, + "min_negative_length": 27, + "avg_negative_length": 30.0, + "max_negative_length": 33, + "unique_negative": 2, + }, + "fra": { + "num_samples": 2, + "number_of_characters": 172, + "num_positive": 2, + "num_negative": 2, + "min_query_length": 23, + "avg_query_length": 26.0, + "max_query_length": 29, + "unique_query": 2, + "min_positive_length": 27, + "avg_positive_length": 30.0, + "max_positive_length": 33, + "unique_positive": 2, + "min_negative_length": 27, + "avg_negative_length": 30.0, + "max_negative_length": 33, + "unique_negative": 2, + }, + }, + } + } + metadata = TaskMetadata( type="Reranking", name="MockMultilingualRerankingTask", main_score="map", - descriptive_stats={ - "test": { - "num_samples": 4, - "num_positive": 4, - "num_negative": 4, - "avg_query_len": 26.0, - "avg_positive_len": 30.0, - "avg_negative_len": 30.0, - "hf_subset_descriptive_stats": { - "eng": { - "num_samples": 2, - "num_positive": 2, - "num_negative": 2, - "avg_query_len": 26.0, - "avg_positive_len": 30.0, - "avg_negative_len": 30.0, - }, - "fra": { - "num_samples": 2, - "num_positive": 2, - "num_negative": 2, - "avg_query_len": 26.0, - "avg_positive_len": 30.0, - "avg_negative_len": 30.0, - }, - }, - } - }, **general_args, # type: ignore ) metadata.eval_langs = multilingual_eval_langs @@ -988,20 +1283,50 @@ def load_data(self, **kwargs): class MockRetrievalTask(AbsTaskRetrieval): + expected_stats = { + "test": { + "number_of_characters": 112, + "num_samples": 4, + "num_queries": 2, + "num_documents": 2, + "min_document_length": 23, + "average_document_length": 26.0, + "max_document_length": 29, + "unique_documents": 2, + "min_query_length": 27, + "average_query_length": 30.0, + "max_query_length": 33, + "unique_queries": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 2.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "val": { + "number_of_characters": 112, + "num_samples": 4, + "num_queries": 2, + "num_documents": 2, + "min_document_length": 23, + "average_document_length": 26.0, + "max_document_length": 29, + "unique_documents": 2, + "min_query_length": 27, + "average_query_length": 30.0, + "max_query_length": 33, + "unique_queries": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 2.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + } + metadata = TaskMetadata( type="Retrieval", name="MockRetrievalTask", main_score="ndcg_at_10", - descriptive_stats={ - "test": { - "average_document_length": 30.0, - "average_query_length": 26.0, - "num_documents": 2, - "num_queries": 2, - "average_relevant_docs_per_query": 1.0, - } - }, - **general_args, # type: ignore + **dict(general_args | {"eval_splits": ["val", "test"]}), # type: ignore ) def load_data(self, **kwargs): @@ -1009,13 +1334,21 @@ def load_data(self, **kwargs): "test": { "q1": "This is a test sentence", "q2": "This is another test sentence", - } + }, + "val": { + "q1": "This is a test sentence", + "q2": "This is another test sentence", + }, } self.corpus = { "test": { "d1": "This is a positive sentence", "d2": "This is another positive sentence", - } + }, + "val": { + "d1": "This is a positive sentence", + "d2": "This is another positive sentence", + }, } self.relevant_docs = { @@ -1023,41 +1356,135 @@ def load_data(self, **kwargs): "q1": {"d1": 1, "d2": 0}, "q2": {"d1": 0, "d2": 1}, }, + "val": { + "q1": {"d1": 1, "d2": 0}, + "q2": {"d1": 0, "d2": 1}, + }, } self.data_loaded = True class MockMultilingualRetrievalTask(AbsTaskRetrieval, MultilingualTask): + expected_stats = { + "val": { + "number_of_characters": 224, + "num_samples": 8, + "num_queries": 4, + "num_documents": 4, + "min_document_length": 23, + "average_document_length": 26.0, + "max_document_length": 29, + "unique_documents": 4, + "min_query_length": 27, + "average_query_length": 30.0, + "max_query_length": 33, + "unique_queries": 4, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 2.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 4, + "hf_subset_descriptive_stats": { + "eng": { + "number_of_characters": 112, + "num_samples": 4, + "num_queries": 2, + "num_documents": 2, + "min_document_length": 23, + "average_document_length": 26.0, + "max_document_length": 29, + "unique_documents": 2, + "min_query_length": 27, + "average_query_length": 30.0, + "max_query_length": 33, + "unique_queries": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 2.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "fra": { + "number_of_characters": 112, + "num_samples": 4, + "num_queries": 2, + "num_documents": 2, + "min_document_length": 23, + "average_document_length": 26.0, + "max_document_length": 29, + "unique_documents": 2, + "min_query_length": 27, + "average_query_length": 30.0, + "max_query_length": 33, + "unique_queries": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 2.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + }, + }, + "test": { + "number_of_characters": 224, + "num_samples": 8, + "num_queries": 4, + "num_documents": 4, + "min_document_length": 23, + "average_document_length": 26.0, + "max_document_length": 29, + "unique_documents": 4, + "min_query_length": 27, + "average_query_length": 30.0, + "max_query_length": 33, + "unique_queries": 4, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 2.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 4, + "hf_subset_descriptive_stats": { + "eng": { + "number_of_characters": 112, + "num_samples": 4, + "num_queries": 2, + "num_documents": 2, + "min_document_length": 23, + "average_document_length": 26.0, + "max_document_length": 29, + "unique_documents": 2, + "min_query_length": 27, + "average_query_length": 30.0, + "max_query_length": 33, + "unique_queries": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 2.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "fra": { + "number_of_characters": 112, + "num_samples": 4, + "num_queries": 2, + "num_documents": 2, + "min_document_length": 23, + "average_document_length": 26.0, + "max_document_length": 29, + "unique_documents": 2, + "min_query_length": 27, + "average_query_length": 30.0, + "max_query_length": 33, + "unique_queries": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 2.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + }, + }, + } + metadata = TaskMetadata( type="Retrieval", name="MockMultilingualRetrievalTask", main_score="ndcg_at_10", - descriptive_stats={ - "test": { - "average_document_length": 30.0, - "average_query_length": 26.0, - "num_documents": 4, - "num_queries": 4, - "average_relevant_docs_per_query": 1.0, - "hf_subset_descriptive_stats": { - "eng": { - "average_document_length": 30.0, - "average_query_length": 26.0, - "num_documents": 2, - "num_queries": 2, - "average_relevant_docs_per_query": 1.0, - }, - "fra": { - "average_document_length": 30.0, - "average_query_length": 26.0, - "num_documents": 2, - "num_queries": 2, - "average_relevant_docs_per_query": 1.0, - }, - }, - } - }, - **general_args, # type: ignore + **dict(general_args | {"eval_splits": ["val", "test"]}), # type: ignore ) metadata.eval_langs = multilingual_eval_langs @@ -1066,14 +1493,22 @@ def load_data(self, **kwargs): "test": { "q1": "This is a test sentence", "q2": "This is another test sentence", - } + }, + "val": { + "q1": "This is a test sentence", + "q2": "This is another test sentence", + }, } self.queries = {"eng": queries, "fra": queries} corpus = { "test": { "d1": "This is a positive sentence", "d2": "This is another positive sentence", - } + }, + "val": { + "d1": "This is a positive sentence", + "d2": "This is another positive sentence", + }, } self.corpus = {"eng": corpus, "fra": corpus} @@ -1082,6 +1517,10 @@ def load_data(self, **kwargs): "q1": {"d1": 1, "d2": 0}, "q2": {"d1": 0, "d2": 1}, }, + "val": { + "q1": {"d1": 1, "d2": 0}, + "q2": {"d1": 0, "d2": 1}, + }, } self.relevant_docs = { "eng": relevant_docs, @@ -1091,37 +1530,60 @@ def load_data(self, **kwargs): class MockMultilabelClassification(AbsTaskMultilabelClassification): + expected_stats = { + "test": { + "num_samples": 6, + "number_of_characters": 156, + "number_texts_in_train": 1, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + "min_labels_per_text": 2, + "average_label_per_text": 2.0, + "max_labels_per_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 6}, "1": {"count": 6}}, + }, + "train": { + "num_samples": 6, + "number_of_characters": 159, + "number_texts_in_train": None, + "min_text_length": 23, + "average_text_length": 26.5, + "max_text_length": 30, + "unique_texts": 2, + "min_labels_per_text": 2, + "average_label_per_text": 2.0, + "max_labels_per_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 6}, "1": {"count": 6}}, + }, + } + metadata = TaskMetadata( type="MultilabelClassification", name="MockMultilabelClassification", main_score="lrap", - descriptive_stats={ - "test": { - "average_text_length": 26.0, - "average_label_per_text": 2.0, - "num_samples": 6, - "unique_labels": 2, - "labels": {"0": {"count": 6}, "1": {"count": 6}}, - } - }, **general_args, # type: ignore ) def load_data(self, **kwargs): - texts = ["This is a test sentence", "This is another test sentence"] * 3 + train_texts = ["This is a test sentence", "This is another train sentence"] * 3 + test_texts = ["This is a test sentence", "This is another test sentence"] * 3 labels = [[0, 1], [1, 0]] * 3 self.dataset = DatasetDict( { "test": Dataset.from_dict( { - "text": texts, + "text": test_texts, "label": labels, } ), "train": Dataset.from_dict( { - "text": texts, + "text": train_texts, "label": labels, } ), @@ -1133,53 +1595,120 @@ def load_data(self, **kwargs): class MockMultilingualMultilabelClassification( AbsTaskMultilabelClassification, MultilingualTask ): + expected_stats = { + "test": { + "num_samples": 12, + "number_of_characters": 312, + "number_texts_in_train": 1, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + "min_labels_per_text": 2, + "average_label_per_text": 2.0, + "max_labels_per_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 12}, "1": {"count": 12}}, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 6, + "number_of_characters": 156, + "number_texts_in_train": 1, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + "min_labels_per_text": 2, + "average_label_per_text": 2.0, + "max_labels_per_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 6}, "1": {"count": 6}}, + }, + "fra": { + "num_samples": 6, + "number_of_characters": 156, + "number_texts_in_train": 1, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + "min_labels_per_text": 2, + "average_label_per_text": 2.0, + "max_labels_per_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 6}, "1": {"count": 6}}, + }, + }, + }, + "train": { + "num_samples": 12, + "number_of_characters": 318, + "number_texts_in_train": None, + "min_text_length": 23, + "average_text_length": 26.5, + "max_text_length": 30, + "unique_texts": 2, + "min_labels_per_text": 2, + "average_label_per_text": 2.0, + "max_labels_per_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 12}, "1": {"count": 12}}, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 6, + "number_of_characters": 159, + "number_texts_in_train": None, + "min_text_length": 23, + "average_text_length": 26.5, + "max_text_length": 30, + "unique_texts": 2, + "min_labels_per_text": 2, + "average_label_per_text": 2.0, + "max_labels_per_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 6}, "1": {"count": 6}}, + }, + "fra": { + "num_samples": 6, + "number_of_characters": 159, + "number_texts_in_train": None, + "min_text_length": 23, + "average_text_length": 26.5, + "max_text_length": 30, + "unique_texts": 2, + "min_labels_per_text": 2, + "average_label_per_text": 2.0, + "max_labels_per_text": 2, + "unique_labels": 2, + "labels": {"0": {"count": 6}, "1": {"count": 6}}, + }, + }, + }, + } + metadata = TaskMetadata( type="MultilabelClassification", name="MockMultilingualMultilabelClassification", main_score="lrap", - descriptive_stats={ - "test": { - "average_text_length": 26.0, - "average_label_per_text": 2.0, - "num_samples": 12, - "unique_labels": 2, - "labels": {"0": {"count": 12}, "1": {"count": 12}}, - "hf_subset_descriptive_stats": { - "eng": { - "average_text_length": 26.0, - "average_label_per_text": 2.0, - "num_samples": 6, - "unique_labels": 2, - "labels": {"0": {"count": 6}, "1": {"count": 6}}, - }, - "fra": { - "average_text_length": 26.0, - "average_label_per_text": 2.0, - "num_samples": 6, - "unique_labels": 2, - "labels": {"0": {"count": 6}, "1": {"count": 6}}, - }, - }, - } - }, **general_args, # type: ignore ) metadata.eval_langs = multilingual_eval_langs def load_data(self, **kwargs): - texts = ["This is a test sentence", "This is another test sentence"] * 3 + train_texts = ["This is a test sentence", "This is another train sentence"] * 3 + test_texts = ["This is a test sentence", "This is another test sentence"] * 3 labels = [[0, 1], [1, 0]] * 3 data = { "test": Dataset.from_dict( { - "text": texts, + "text": test_texts, "label": labels, } ), "train": Dataset.from_dict( { - "text": texts, + "text": train_texts, "label": labels, } ), @@ -1196,22 +1725,41 @@ def load_data(self, **kwargs): class MockInstructionRetrival(AbsTaskInstructionRetrieval): do_length_ablation = True + expected_stats = { + "test": { + "num_samples": 4, + "num_docs": 2, + "num_queries": 2, + "number_of_characters": 244, + "min_document_length": 27, + "average_document_length": 30.0, + "max_document_length": 33, + "unique_docs": 2, + "min_query_length": 23, + "average_query_length": 26.0, + "max_query_length": 29, + "unique_queries": 2, + "min_instruction_length": 26, + "average_instruction_length": 29.0, + "max_instruction_length": 32, + "unique_instructions": 2, + "min_changed_instruction_length": 34, + "average_changed_instruction_length": 37.0, + "max_changed_instruction_length": 40, + "unique_changed_instructions": 2, + "min_average_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_average_relevant_docs_per_query": 1, + "min_average_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_average_top_ranked_per_query": 2, + } + } + metadata = TaskMetadata( type="InstructionRetrieval", name="MockInstructionRetrival", main_score="p-MRR", - descriptive_stats={ - "test": { - "num_docs": 2, - "num_queries": 2, - "average_document_length": 30.0, - "average_query_length": 26.0, - "average_instruction_length": 29.0, - "average_changed_instruction_length": 37.0, - "average_relevant_docs_per_query": 1.0, - "average_top_ranked_per_query": 2.0, - } - }, **general_args, # type: ignore ) @@ -1280,44 +1828,99 @@ class MockMultilingualInstructionRetrival( AbsTaskInstructionRetrieval, MultilingualTask ): do_length_ablation = True + expected_stats = { + "test": { + "num_samples": 8, + "num_docs": 4, + "num_queries": 4, + "number_of_characters": 488, + "min_document_length": 27, + "average_document_length": 30.0, + "max_document_length": 33, + "unique_docs": 2, + "min_query_length": 23, + "average_query_length": 26.0, + "max_query_length": 29, + "unique_queries": 2, + "min_instruction_length": 26, + "average_instruction_length": 29.0, + "max_instruction_length": 32, + "unique_instructions": 2, + "min_changed_instruction_length": 34, + "average_changed_instruction_length": 37.0, + "max_changed_instruction_length": 40, + "unique_changed_instructions": 2, + "min_average_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_average_relevant_docs_per_query": 1, + "min_average_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_average_top_ranked_per_query": 2, + "hf_subset_descriptive_stats": { + "eng": { + "num_samples": 4, + "num_docs": 2, + "num_queries": 2, + "number_of_characters": 244, + "min_document_length": 27, + "average_document_length": 30.0, + "max_document_length": 33, + "unique_docs": 2, + "min_query_length": 23, + "average_query_length": 26.0, + "max_query_length": 29, + "unique_queries": 2, + "min_instruction_length": 26, + "average_instruction_length": 29.0, + "max_instruction_length": 32, + "unique_instructions": 2, + "min_changed_instruction_length": 34, + "average_changed_instruction_length": 37.0, + "max_changed_instruction_length": 40, + "unique_changed_instructions": 2, + "min_average_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_average_relevant_docs_per_query": 1, + "min_average_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_average_top_ranked_per_query": 2, + }, + "fra": { + "num_samples": 4, + "num_docs": 2, + "num_queries": 2, + "number_of_characters": 244, + "min_document_length": 27, + "average_document_length": 30.0, + "max_document_length": 33, + "unique_docs": 2, + "min_query_length": 23, + "average_query_length": 26.0, + "max_query_length": 29, + "unique_queries": 2, + "min_instruction_length": 26, + "average_instruction_length": 29.0, + "max_instruction_length": 32, + "unique_instructions": 2, + "min_changed_instruction_length": 34, + "average_changed_instruction_length": 37.0, + "max_changed_instruction_length": 40, + "unique_changed_instructions": 2, + "min_average_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_average_relevant_docs_per_query": 1, + "min_average_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_average_top_ranked_per_query": 2, + }, + }, + } + } + metadata = TaskMetadata( type="InstructionRetrieval", name="MockMultilingualInstructionRetrival", main_score="p-MRR", - descriptive_stats={ - "test": { - "num_docs": 4, - "num_queries": 4, - "average_document_length": 30.0, - "average_query_length": 26.0, - "average_instruction_length": 29.0, - "average_changed_instruction_length": 37.0, - "average_relevant_docs_per_query": 1.0, - "average_top_ranked_per_query": 2.0, - "hf_subset_descriptive_stats": { - "eng": { - "num_docs": 2, - "num_queries": 2, - "average_document_length": 30.0, - "average_query_length": 26.0, - "average_instruction_length": 29.0, - "average_changed_instruction_length": 37.0, - "average_relevant_docs_per_query": 1.0, - "average_top_ranked_per_query": 2.0, - }, - "fra": { - "num_docs": 2, - "num_queries": 2, - "average_document_length": 30.0, - "average_query_length": 26.0, - "average_instruction_length": 29.0, - "average_changed_instruction_length": 37.0, - "average_relevant_docs_per_query": 1.0, - "average_top_ranked_per_query": 2.0, - }, - }, - } - }, **general_args, # type: ignore ) metadata.eval_langs = multilingual_eval_langs diff --git a/tests/test_benchmark/test_benchmark.py b/tests/test_benchmark/test_benchmark.py index d898517528..ff3e1d5c86 100644 --- a/tests/test_benchmark/test_benchmark.py +++ b/tests/test_benchmark/test_benchmark.py @@ -201,7 +201,9 @@ def test_benchmark_names_must_be_unique(): assert len(names) == len(set(names)) -@pytest.mark.parametrize("name", ["MTEB(eng)", "MTEB(rus)", "MTEB(Scandinavian)"]) +@pytest.mark.parametrize( + "name", ["MTEB(eng, classic)", "MTEB(rus)", "MTEB(Scandinavian)"] +) def test_get_benchmark(name): benchmark = mteb.get_benchmark(benchmark_name=name) assert isinstance(benchmark, mteb.Benchmark) diff --git a/tests/test_cli.py b/tests/test_cli.py index f79624c61f..4f13bebbc6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -27,8 +27,8 @@ def test_available_benchmarks(): result = subprocess.run(command, shell=True, capture_output=True, text=True) assert result.returncode == 0, "Command failed" assert ( - "MTEB(eng)" in result.stdout - ), "Sample benchmark MTEB(eng) task not found in available benchmarks" + "MTEB(eng, classic)" in result.stdout + ), "Sample benchmark MTEB(eng, classic) task not found in available benchmarks" run_task_fixures = [ @@ -189,7 +189,7 @@ def test_create_meta_from_existing(existing_readme_name: str, gold_readme_name: def test_save_predictions(): - command = f"{sys.executable} -m mteb run -m all-MiniLM-L6-v2 -t NFCorpus --output_folder tests/results --save_predictions" + command = f"{sys.executable} -m mteb run -m average_word_embeddings_komninos -t NFCorpus --output_folder tests/results --save_predictions" result = subprocess.run(command, shell=True, capture_output=True, text=True) assert result.returncode == 0, "Command failed" test_folder = Path(__file__).parent diff --git a/tests/test_evaluation/test_split_evaluation.py b/tests/test_evaluation/test_split_evaluation.py new file mode 100644 index 0000000000..7db10e09d4 --- /dev/null +++ b/tests/test_evaluation/test_split_evaluation.py @@ -0,0 +1,380 @@ +from __future__ import annotations + +import pytest + +from mteb import MTEB +from tests.test_benchmark.mock_models import ( + MockSentenceTransformer, +) +from tests.test_benchmark.mock_tasks import ( + MockMultilingualRetrievalTask, + MockMultilingualSTSTask, + MockRetrievalTask, +) + + +@pytest.fixture +def model(): + return MockSentenceTransformer() + + +@pytest.fixture +def tasks(): + return [MockRetrievalTask()] + + +@pytest.fixture +def multilingual_tasks(): + return [MockMultilingualRetrievalTask()] + + +def test_all_splits_evaluated(model, tasks, tmp_path): + evaluation = MTEB(tasks=tasks) + results = evaluation.run( + model, + eval_splits=["val", "test"], + output_folder=str(tmp_path / "all_splits_evaluated"), + verbosity=2, + ) + + assert "MockRetrievalTask" == results[0].task_name + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert set(last_evaluated_splits["MockRetrievalTask"]) == {"val", "test"} + assert len(last_evaluated_splits["MockRetrievalTask"]) == 2 + assert results[0].scores.keys() == {"val", "test"} + + +def test_one_missing_split(model, tasks, tmp_path): + evaluation = MTEB(tasks=tasks) + results = evaluation.run( + model, + eval_splits=["val"], + output_folder=str(tmp_path / "testcase2"), + verbosity=2, + ) + + assert "MockRetrievalTask" == results[0].task_name + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert set(last_evaluated_splits["MockRetrievalTask"]) == {"val"} + assert len(last_evaluated_splits["MockRetrievalTask"]) == 1 + assert results[0].scores.keys() == {"val"} + + results2 = evaluation.run( + model, + eval_splits=["val", "test"], + output_folder=str(tmp_path / "testcase2"), + verbosity=2, + ) + + assert "MockRetrievalTask" == results2[0].task_name + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert set(last_evaluated_splits["MockRetrievalTask"]) == {"test"} + assert len(last_evaluated_splits["MockRetrievalTask"]) == 1 + assert results2[0].scores.keys() == {"test", "val"} + + +def test_no_missing_splits(model, tasks, tmp_path): + evaluation = MTEB(tasks=tasks) + results = evaluation.run( + model, + eval_splits=["val", "test"], + output_folder=str(tmp_path / "testcase3"), + verbosity=2, + ) + + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert "MockRetrievalTask" in last_evaluated_splits + assert len(last_evaluated_splits["MockRetrievalTask"]) == 2 + assert results[0].scores.keys() == {"test", "val"} + + evaluation = MTEB(tasks=tasks) + results = evaluation.run( + model, + eval_splits=["val", "test"], + output_folder=str(tmp_path / "testcase3"), + verbosity=2, + ) + + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert len(last_evaluated_splits) == 0 + assert results[0].scores.keys() == {"test", "val"} + + +def test_all_languages_evaluated(model, multilingual_tasks, tmp_path): + evaluation = MTEB(tasks=multilingual_tasks) + results = evaluation.run( + model, + eval_splits=["test"], + output_folder=str(tmp_path / "all_lang_evaluated"), + verbosity=2, + eval_subsets=None, + ) + assert "MockMultilingualRetrievalTask" == results[0].task_name + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert "MockMultilingualRetrievalTask" in last_evaluated_splits + assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1 + assert last_evaluated_splits["MockMultilingualRetrievalTask"] == ["test"] + assert results[0].scores.keys() == {"test"} + assert len(results[0].scores["test"]) == 2 + + +def test_missing_language(model, multilingual_tasks, tmp_path): + evaluation = MTEB(tasks=multilingual_tasks) + results = evaluation.run( + model, + eval_splits=["test"], + output_folder=str(tmp_path / "missing_lang_test"), + verbosity=2, + eval_subsets=["eng"], + ) + + assert "MockMultilingualRetrievalTask" == results[0].task_name + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert "MockMultilingualRetrievalTask" in last_evaluated_splits + assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1 + assert last_evaluated_splits["MockMultilingualRetrievalTask"] == ["test"] + assert results[0].scores.keys() == {"test"} + assert results[0].languages == ["eng"] + + results = evaluation.run( + model, + eval_splits=["test"], + output_folder=str(tmp_path / "missing_lang_test"), + verbosity=2, + eval_subsets=["eng", "fra"], + ) + + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1 + assert last_evaluated_splits["MockMultilingualRetrievalTask"] == ["test"] + assert sorted(results[0].languages) == ["eng", "fra"] + assert results[0].scores.keys() == {"test"} + assert len(results[0].scores["test"]) == 2 + + +def test_no_missing_languages(model, multilingual_tasks, tmp_path): + evaluation = MTEB(tasks=multilingual_tasks) + results = evaluation.run( + model, + eval_splits=["test"], + output_folder=str(tmp_path / "no_missing_lang_test"), + verbosity=2, + eval_subsets=["eng", "fra"], + ) + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert "MockMultilingualRetrievalTask" in last_evaluated_splits + assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1 + assert results[0].scores.keys() == {"test"} + assert len(results[0].scores["test"]) == 2 + assert sorted(results[0].languages) == ["eng", "fra"] + + evaluation = MTEB(tasks=multilingual_tasks) + results = evaluation.run( + model, + eval_splits=["test"], + output_folder=str(tmp_path / "no_missing_lang_test"), + verbosity=2, + eval_subsets=["eng", "fra"], + ) + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert len(last_evaluated_splits) == 0 + assert results[0].scores.keys() == {"test"} + assert len(results[0].scores["test"]) == 2 + assert sorted(results[0].languages) == ["eng", "fra"] + + +def test_partial_languages(model, multilingual_tasks, tmp_path): + evaluation = MTEB(tasks=multilingual_tasks) + results = evaluation.run( + model, + eval_splits=["test"], + output_folder=str(tmp_path / "partial_lang_test"), + verbosity=2, + eval_subsets=["fra"], + ) + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1 + assert last_evaluated_splits["MockMultilingualRetrievalTask"] == ["test"] + assert results[0].scores.keys() == {"test"} + assert len(results[0].scores["test"]) == 1 + assert results[0].languages == ["fra"] + + results = evaluation.run( + model, + eval_splits=["test"], + output_folder=str(tmp_path / "partial_lang_test"), + verbosity=2, + eval_subsets=["fra", "eng"], + ) + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1 + assert last_evaluated_splits["MockMultilingualRetrievalTask"] == ["test"] + assert results[0].scores.keys() == {"test"} + assert len(results[0].scores["test"]) == 2 + assert sorted(results[0].languages) == ["eng", "fra"] + + +def test_multilingual_one_missing_split_no_missing_lang( + model, multilingual_tasks, tmp_path +): + evaluation = MTEB(tasks=multilingual_tasks) + results = evaluation.run( + model, + eval_splits=["val"], + output_folder=str(tmp_path / "partial_langs_partial_splits"), + verbosity=2, + eval_subsets=["eng", "fra"], + ) + + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1 + assert set(last_evaluated_splits["MockMultilingualRetrievalTask"]) == {"val"} + assert sorted(results[0].languages) == ["eng", "fra"] + assert results[0].scores.keys() == {"val"} + assert len(results[0].scores["val"]) == 2 + + results = evaluation.run( + model, + eval_splits=["val", "test"], + output_folder=str(tmp_path / "partial_langs_partial_splits"), + verbosity=2, + eval_subsets=["eng", "fra"], + ) + + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert set(last_evaluated_splits["MockMultilingualRetrievalTask"]) == {"test"} + assert sorted(results[0].languages) == ["eng", "fra"] + assert results[0].scores.keys() == {"test", "val"} + assert len(results[0].scores["test"]) == 2 + assert len(results[0].scores["val"]) == 2 + + +def test_multilingual_one_missing_lang_in_one_split( + model, multilingual_tasks, tmp_path +): + evaluation = MTEB(tasks=multilingual_tasks) + results = evaluation.run( + model, + eval_splits=["val"], + output_folder=str(tmp_path / "one_lang_one_split"), + verbosity=2, + eval_subsets=["eng", "fra"], + ) + + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert set(last_evaluated_splits["MockMultilingualRetrievalTask"]) == {"val"} + assert sorted(results[0].languages) == ["eng", "fra"] + assert results[0].scores.keys() == {"val"} + assert len(results[0].scores["val"]) == 2 + + results = evaluation.run( + model, + eval_splits=["val", "test"], + output_folder=str(tmp_path / "one_lang_one_split"), + verbosity=2, + eval_subsets=["eng"], + ) + + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert set(last_evaluated_splits["MockMultilingualRetrievalTask"]) == {"test"} + assert sorted(results[0].languages) == ["eng", "fra"] + assert results[0].scores.keys() == {"test", "val"} + assert len(results[0].scores["test"]) == 1 + assert len(results[0].scores["val"]) == 2 + + results = evaluation.run( + model, + eval_splits=["test"], + output_folder=str(tmp_path / "one_lang_one_split"), + verbosity=2, + eval_subsets=["eng", "fra"], + ) + + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert set(last_evaluated_splits["MockMultilingualRetrievalTask"]) == {"test"} + assert sorted(results[0].languages) == ["eng", "fra"] + # output merged result with previous results + assert results[0].scores.keys() == {"test", "val"} + assert len(results[0].scores["test"]) == 2 + + +def test_all_splits_evaluated_with_overwrite(model, tasks, tmp_path): + evaluation = MTEB(tasks=tasks) + results = evaluation.run( + model, + eval_splits=["val"], + output_folder=str(tmp_path / "all_splits_evaluated_with_overwrite"), + verbosity=2, + ) + + assert "MockRetrievalTask" == results[0].task_name + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert len(last_evaluated_splits["MockRetrievalTask"]) == 1 + assert set(last_evaluated_splits["MockRetrievalTask"]) == {"val"} + assert results[0].scores.keys() == {"val"} + + results2 = evaluation.run( + model, + eval_splits=["val", "test"], + output_folder=str(tmp_path / "all_splits_evaluated_with_overwrite"), + verbosity=2, + overwrite_results=True, + ) + assert "MockRetrievalTask" == results2[0].task_name + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert len(last_evaluated_splits["MockRetrievalTask"]) == 2 + assert set(last_evaluated_splits["MockRetrievalTask"]) == {"val", "test"} + assert results2[0].scores.keys() == {"val", "test"} + + +def test_all_splits_subsets_evaluated_with_overwrite( + model, multilingual_tasks, tmp_path +): + evaluation = MTEB(tasks=multilingual_tasks) + results = evaluation.run( + model, + eval_splits=[ + "test", + ], + output_folder=str(tmp_path / "all_splits_subsets_evaluated_with_overwrite"), + verbosity=2, + eval_subsets=["fra"], + ) + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert "MockMultilingualRetrievalTask" in last_evaluated_splits + assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1 + assert results[0].scores.keys() == {"test"} + for split in ["test"]: + assert len(results[0].scores[split]) == 1 + assert sorted(results[0].languages) == ["fra"] + + results2 = evaluation.run( + model, + eval_splits=["test"], + output_folder=str(tmp_path / "all_splits_subsets_evaluated_with_overwrite"), + verbosity=2, + eval_subsets=["fra", "eng"], + overwrite_results=True, + ) + last_evaluated_splits = evaluation.get_last_evaluated_splits() + assert "MockMultilingualRetrievalTask" in last_evaluated_splits + assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1 + assert results2[0].scores.keys() == {"test"} + for split in ["test"]: + assert len(results2[0].scores[split]) == 2 + assert sorted(results2[0].languages) == ["eng", "fra"] + + +def test_splits_evaluated_with_prefiltering(): + """Test that the evaluation only runs on the specified languages. Issue https://github.com/embeddings-benchmark/mteb/pull/1787#issuecomment-2598205049""" + task = MockMultilingualSTSTask().filter_languages(languages=["fra"]) + + evaluation = MTEB(tasks=[task]) + + results = evaluation.run(MockSentenceTransformer(), overwrite_results=True) + result_scores = results[0].scores + + assert len(result_scores) == 1 + assert "test" in result_scores + assert len(result_scores["test"]) == 1 + assert result_scores["test"][0]["hf_subset"] == "fra" diff --git a/tests/test_models/model_load_failures.json b/tests/test_models/model_load_failures.json new file mode 100644 index 0000000000..f1be1c940b --- /dev/null +++ b/tests/test_models/model_load_failures.json @@ -0,0 +1,197 @@ +{ + "Alibaba-NLP/gte-Qwen1.5-7B-instruct": "Over threshold. Not tested.", + "Alibaba-NLP/gte-Qwen2-1.5B-instruct": "None", + "Alibaba-NLP/gte-Qwen2-7B-instruct": "Over threshold. Not tested.", + "BAAI/bge-base-en-v1.5": "None", + "BAAI/bge-large-en-v1.5": "Over threshold. Not tested.", + "BAAI/bge-reranker-v2-m3": "None", + "BAAI/bge-small-en-v1.5": "None", + "BAAI/bge-small-en-v1.5 BAAI/bge-base-en-v1.5 BAAI/bge-large-en-v1.5": null, + "BeastyZ/e5-R-mistral-7b": "Over threshold. Not tested.", + "Cohere/Cohere-embed-english-light-v3.0": "None", + "Cohere/Cohere-embed-english-v3.0": "None", + "Cohere/Cohere-embed-multilingual-light-v3.0": "None", + "Cohere/Cohere-embed-multilingual-v3.0": "None", + "DeepPavlov/distilrubert-small-cased-conversational": "None", + "DeepPavlov/rubert-base-cased": "None", + "DeepPavlov/rubert-base-cased-sentence": "None", + "Gameselo/STS-multilingual-mpnet-base-v2": "None", + "GritLM/GritLM-7B": "Over threshold. Not tested.", + "GritLM/GritLM-8x7B": "Over threshold. Not tested.", + "HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1": "None", + "HIT-TMG/KaLM-embedding-multilingual-mini-v1": "None", + "Haon-Chen/speed-embedding-7b-instruct": "Over threshold. Not tested.", + "Hum-Works/lodestone-base-4096-v1": "None", + "Jaume/gemma-2b-embeddings": "Over threshold. Not tested.", + "Lajavaness/bilingual-embedding-base": "None", + "Lajavaness/bilingual-embedding-large": "None", + "Lajavaness/bilingual-embedding-small": "None", + "Linq-AI-Research/Linq-Embed-Mistral": "Over threshold. Not tested.", + "McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised": "Over threshold. Not tested.", + "McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-unsup-simcse": "Over threshold. Not tested.", + "McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised": "Over threshold. Not tested.", + "McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-unsup-simcse": "Over threshold. Not tested.", + "McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised": "Over threshold. Not tested.", + "McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-unsup-simcse": "Over threshold. Not tested.", + "McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-supervised": "Over threshold. Not tested.", + "McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-unsup-simcse": "Over threshold. Not tested.", + "Mihaiii/Bulbasaur": "None", + "Mihaiii/Ivysaur": "None", + "Mihaiii/Squirtle": "None", + "Mihaiii/Venusaur": "None", + "Mihaiii/Wartortle": "None", + "Mihaiii/gte-micro": "None", + "Mihaiii/gte-micro-v4": "None", + "Omartificial-Intelligence-Space/Arabert-all-nli-triplet-Matryoshka": "None", + "Omartificial-Intelligence-Space/Arabic-MiniLM-L12-v2-all-nli-triplet": "None", + "Omartificial-Intelligence-Space/Arabic-all-nli-triplet-Matryoshka": "None", + "Omartificial-Intelligence-Space/Arabic-labse-Matryoshka": "None", + "Omartificial-Intelligence-Space/Arabic-mpnet-base-all-nli-triplet": "None", + "Omartificial-Intelligence-Space/Marbert-all-nli-triplet-Matryoshka": "None", + "OrdalieTech/Solon-embeddings-large-0.1": "None", + "OrlikB/KartonBERT-USE-base-v1": "None", + "OrlikB/st-polish-kartonberta-base-alpha-v1": "None", + "Salesforce/SFR-Embedding-2_R": "Over threshold. Not tested.", + "Salesforce/SFR-Embedding-Mistral": "Over threshold. Not tested.", + "Snowflake/snowflake-arctic-embed-l": "None", + "Snowflake/snowflake-arctic-embed-l-v2.0": "None", + "Snowflake/snowflake-arctic-embed-m": "None", + "Snowflake/snowflake-arctic-embed-m-long": "None", + "Snowflake/snowflake-arctic-embed-m-v1.5": "None", + "Snowflake/snowflake-arctic-embed-m-v2.0": "None", + "Snowflake/snowflake-arctic-embed-s": "None", + "Snowflake/snowflake-arctic-embed-xs": "None", + "WhereIsAI/UAE-Large-V1": "None", + "aari1995/German_Semantic_STS_V2": "None", + "abhinand/MedEmbed-small-v0.1": "None", + "ai-forever/ru-en-RoSBERTa": "None", + "ai-forever/sbert_large_mt_nlu_ru": "None", + "ai-forever/sbert_large_nlu_ru": "None", + "avsolatorio/GIST-Embedding-v0": "None", + "avsolatorio/GIST-all-MiniLM-L6-v2": "None", + "avsolatorio/GIST-large-Embedding-v0": "None", + "avsolatorio/GIST-small-Embedding-v0": "None", + "avsolatorio/NoInstruct-small-Embedding-v0": "None", + "bigscience/sgpt-bloom-7b1-msmarco": "None", + "bm25s": "None", + "brahmairesearch/slx-v0.1": "None", + "castorini/monobert-large-msmarco": "None", + "castorini/monot5-3b-msmarco-10k": "None", + "castorini/monot5-base-msmarco-10k": "None", + "castorini/monot5-large-msmarco-10k": "None", + "castorini/monot5-small-msmarco-10k": "None", + "castorini/repllama-v1-7b-lora-passage": "You are trying to access a gated repo.\nMake sure to have access to it at https://huggingface.co/meta-llama/Llama-2-7b-hf.\n401 Client Error. (Request ID: Root=1-67794457-7e56cbf325381c760c430207;a79cc472-a4fc-49dc-80f0-9d4b8cb5ef42)\n\nCannot access gated repo for url https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/config.json.\nAccess to model meta-llama/Llama-2-7b-hf is restricted. You must have access to it and be authenticated to access it. Please log in.", + "cointegrated/LaBSE-en-ru": "None", + "cointegrated/rubert-tiny": "None", + "cointegrated/rubert-tiny2": "None", + "colbert-ir/colbertv2.0": "None", + "consciousAI/cai-lunaris-text-embeddings": "None", + "consciousAI/cai-stellaris-text-embeddings": "None", + "deepfile/embedder-100p": "None", + "deepvk/USER-base": "None", + "deepvk/USER-bge-m3": "None", + "deepvk/deberta-v1-base": "None", + "dunzhang/stella_en_1.5B_v5": "None", + "dunzhang/stella_en_400M_v5": "None", + "dwzhu/e5-base-4k": "None", + "google/flan-t5-base": "None", + "google/flan-t5-large": "None", + "google/flan-t5-xl": "None", + "google/flan-t5-xxl": "None", + "google/text-embedding-004": "None", + "google/text-embedding-005": "None", + "google/text-multilingual-embedding-002": "None", + "ibm-granite/granite-embedding-107m-multilingual": "None", + "ibm-granite/granite-embedding-125m-english": "None", + "ibm-granite/granite-embedding-278m-multilingual": "None", + "ibm-granite/granite-embedding-30m-english": "None", + "infgrad/jasper_en_vision_language_v1": "Over threshold. Not tested.", + "infgrad/stella-base-en-v2": "None", + "intfloat/e5-base": "None", + "intfloat/e5-base-v2": "None", + "intfloat/e5-large": "None", + "intfloat/e5-large-v2": "None", + "intfloat/e5-mistral-7b-instruct": "Over threshold. Not tested.", + "intfloat/e5-small": "None", + "intfloat/e5-small-v2": "None", + "intfloat/multilingual-e5-base": "None", + "intfloat/multilingual-e5-large": "None", + "intfloat/multilingual-e5-large-instruct": "None", + "intfloat/multilingual-e5-small": "None", + "izhx/udever-bloom-1b1": "None", + "izhx/udever-bloom-3b": "None", + "izhx/udever-bloom-560m": "None", + "izhx/udever-bloom-7b1": "None", + "jhu-clsp/FollowIR-7B": "None", + "jinaai/jina-colbert-v2": "None", + "jinaai/jina-embedding-b-en-v1": "None", + "jinaai/jina-embedding-s-en-v1": "None", + "jinaai/jina-embeddings-v2-base-en": "None", + "jinaai/jina-embeddings-v2-small-en": "None", + "jinaai/jina-embeddings-v3": "None", + "jinaai/jina-reranker-v2-base-multilingual": "None", + "keeeeenw/MicroLlama-text-embedding": "None", + "malenia1/ternary-weight-embedding": "None", + "manu/bge-m3-custom-fr": "None", + "manu/sentence_croissant_alpha_v0.2": "None", + "manu/sentence_croissant_alpha_v0.3": "Over threshold. Not tested.", + "manu/sentence_croissant_alpha_v0.4": "Over threshold. Not tested.", + "meta-llama/Llama-2-7b-chat-hf": "None", + "meta-llama/Llama-2-7b-hf": "None", + "minishlab/M2V_base_glove": "None", + "minishlab/M2V_base_glove_subword": "None", + "minishlab/M2V_base_output": "None", + "minishlab/M2V_multilingual_output": "None", + "minishlab/potion-base-2M": "None", + "minishlab/potion-base-4M": "None", + "minishlab/potion-base-8M": "None", + "mistralai/Mistral-7B-Instruct-v0.2": "None", + "mixedbread-ai/mxbai-embed-large-v1": "None", + "nomic-ai/nomic-embed-text-v1": "None", + "nomic-ai/nomic-embed-text-v1-ablated": "None", + "nomic-ai/nomic-embed-text-v1-unsupervised": "None", + "nomic-ai/nomic-embed-text-v1.5": "None", + "nvidia/NV-Embed-v1": "Over threshold. Not tested.", + "nvidia/NV-Embed-v2": "Over threshold. Not tested.", + "omarelshehy/arabic-english-sts-matryoshka": "None", + "openai/text-embedding-3-large": "None", + "openai/text-embedding-3-small": "None", + "openai/text-embedding-ada-002": "None", + "openbmb/MiniCPM-Embedding": "Over threshold. Not tested.", + "samaya-ai/RepLLaMA-reproduced": "You are trying to access a gated repo.\nMake sure to have access to it at https://huggingface.co/meta-llama/Llama-2-7b-hf.\n401 Client Error. (Request ID: Root=1-6779403c-1bd84d333e938afa4e7cf86b;b873eea6-3c10-4659-b6da-2288d83e721b)\n\nCannot access gated repo for url https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/config.json.\nAccess to model meta-llama/Llama-2-7b-hf is restricted. You must have access to it and be authenticated to access it. Please log in.", + "samaya-ai/promptriever-llama2-7b-v1": "You are trying to access a gated repo.\nMake sure to have access to it at https://huggingface.co/meta-llama/Llama-2-7b-hf.\n401 Client Error. (Request ID: Root=1-677940f7-6c2bfcaa7985abb1165185ff;efdd2ef8-60a0-45c3-a92b-b24784b30b43)\n\nCannot access gated repo for url https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/config.json.\nAccess to model meta-llama/Llama-2-7b-hf is restricted. You must have access to it and be authenticated to access it. Please log in.", + "samaya-ai/promptriever-llama3.1-8b-instruct-v1": "You are trying to access a gated repo.\nMake sure to have access to it at https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct.\n401 Client Error. (Request ID: Root=1-6779430b-3277d7961f3c88ab56ecf91f;a476a013-b28f-47c6-bd95-e3d6fe823468)\n\nCannot access gated repo for url https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/resolve/main/config.json.\nAccess to model meta-llama/Llama-3.1-8B-Instruct is restricted. You must have access to it and be authenticated to access it. Please log in.", + "samaya-ai/promptriever-llama3.1-8b-v1": "You are trying to access a gated repo.\nMake sure to have access to it at https://huggingface.co/meta-llama/Meta-Llama-3.1-8B.\n401 Client Error. (Request ID: Root=1-677bba8f-608cf825273d8d2b0670b5ad;066bb2fa-3bef-4fb9-b3cb-4c5ffee41047)\n\nCannot access gated repo for url https://huggingface.co/meta-llama/Meta-Llama-3.1-8B/resolve/main/config.json.\nAccess to model meta-llama/Llama-3.1-8B is restricted. You must have access to it and be authenticated to access it. Please log in.", + "samaya-ai/promptriever-mistral-v0.1-7b-v1": "You are trying to access a gated repo.\nMake sure to have access to it at https://huggingface.co/mistralai/Mistral-7B-v0.1.\n401 Client Error. (Request ID: Root=1-67794457-688a6d9c24a9e8f15cf70d28;da3a233f-7c7c-4919-9cee-72a1d66acdb6)\n\nCannot access gated repo for url https://huggingface.co/mistralai/Mistral-7B-v0.1/resolve/main/config.json.\nAccess to model mistralai/Mistral-7B-v0.1 is restricted. You must have access to it and be authenticated to access it. Please log in.", + "sdadas/mmlw-e5-base": "None", + "sdadas/mmlw-e5-large": "None", + "sdadas/mmlw-e5-small": "None", + "sdadas/mmlw-roberta-base": "None", + "sdadas/mmlw-roberta-large": "None", + "sentence-transformer/multi-qa-MiniLM-L6-cos-v1": "sentence-transformer/multi-qa-MiniLM-L6-cos-v1 is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a token having permission to this repo either by logging in with `huggingface-cli login` or by passing `token=`", + "sentence-transformers/LaBSE": "None", + "sentence-transformers/all-MiniLM-L12-v2": "None", + "sentence-transformers/all-MiniLM-L6-v2": "None", + "sentence-transformers/all-mpnet-base-v2": "None", + "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2": "None", + "sentence-transformers/paraphrase-multilingual-mpnet-base-v2": "None", + "sergeyzh/LaBSE-ru-turbo": "None", + "sergeyzh/rubert-tiny-turbo": "None", + "shibing624/text2vec-base-multilingual": "None", + "silma-ai/silma-embeddding-matryoshka-v0.1": "None", + "thenlper/gte-base": "None", + "thenlper/gte-large": "None", + "thenlper/gte-small": "None", + "unicamp-dl/mt5-13b-mmarco-100k": "None", + "unicamp-dl/mt5-base-mmarco-v2": "None", + "voyage-large-2": "None", + "voyageai/voyage-2": "None", + "voyageai/voyage-3": "None", + "voyageai/voyage-3-lite": "None", + "voyageai/voyage-code-2": "None", + "voyageai/voyage-finance-2": "None", + "voyageai/voyage-large-2-instruct": "None", + "voyageai/voyage-law-2": "None", + "voyageai/voyage-multilingual-2": "None", + "zeta-alpha-ai/Zeta-Alpha-E5-Mistral": "Over threshold. Not tested." +} \ No newline at end of file diff --git a/tests/test_models/model_loading.py b/tests/test_models/model_loading.py new file mode 100644 index 0000000000..3f22db733f --- /dev/null +++ b/tests/test_models/model_loading.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import argparse +import json +import logging +from pathlib import Path + +from huggingface_hub import scan_cache_dir + +from mteb import get_model, get_model_meta +from mteb.models.overview import MODEL_REGISTRY + +logging.basicConfig(level=logging.INFO) + + +def teardown_function(): + hf_cache_info = scan_cache_dir() + all_revisions = [] + for repo in list(hf_cache_info.repos): + for revision in list(repo.revisions): + all_revisions.append(revision.commit_hash) + + delete_strategy = scan_cache_dir().delete_revisions(*all_revisions) + print("Will free " + delete_strategy.expected_freed_size_str) + delete_strategy.execute() + + +def get_model_below_n_param_threshold(model_name: str) -> str: + """Test that we can get all models with a number of parameters below a threshold.""" + model_meta = get_model_meta(model_name=model_name) + assert model_meta is not None + if model_meta.n_parameters is not None: + if model_meta.n_parameters >= 2e9: + return "Over threshold. Not tested." + elif "API" in model_meta.framework: + try: + m = get_model(model_name) + if m is not None: + del m + return "None" + except Exception as e: + logging.warning(f"Failed to load model {model_name} with error {e}") + return e.__str__() + try: + m = get_model(model_name) + if m is not None: + del m + return "None" + except Exception as e: + logging.warning(f"Failed to load model {model_name} with error {e}") + return e.__str__() + finally: + teardown_function() + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--omit_previous_success", + action="store_true", + default=False, + help="Omit models that have been successfully loaded in the past", + ) + parser.add_argument( + "--run_missing", + action="store_true", + default=False, + help="Run the missing models in the registry that are missing from existing results.", + ) + parser.add_argument( + "--model_name", + type=str, + nargs="+", + default=None, + help="Run the script for specific model names, e.g. model_1, model_2", + ) + parser.add_argument( + "--model_name_file", + type=str, + default=None, + help="Filename containing space-separated model names to test.", + ) + + return parser.parse_args() + + +if __name__ == "__main__": + output_file = Path(__file__).parent / "model_load_failures.json" + + args = parse_args() + + # Load existing results if the file exists + results = {} + if output_file.exists(): + with output_file.open("r") as f: + results = json.load(f) + + if args.model_name: + all_model_names = args.model_name + elif args.model_name_file: + all_model_names = [] + if Path(args.model_name_file).exists(): + with open(args.model_name_file) as f: + all_model_names = f.read().strip().split() + else: + logging.warning( + f"Model name file {args.model_name_file} does not exist. Exiting." + ) + exit(1) + else: + omit_keys = [] + if args.run_missing: + omit_keys = list(results.keys()) + elif args.omit_previous_success: + omit_keys = [k for k, v in results.items() if v == "None"] + + all_model_names = list(set(MODEL_REGISTRY.keys()) - set(omit_keys)) + + for model_name in all_model_names: + error_msg = get_model_below_n_param_threshold(model_name) + results[model_name] = error_msg + + results = dict(sorted(results.items())) + + # Write the results to the file after each iteration + with output_file.open("w") as f: + json.dump(results, f, indent=4) diff --git a/tests/test_overview.py b/tests/test_overview.py index 73df5dc193..127e54f279 100644 --- a/tests/test_overview.py +++ b/tests/test_overview.py @@ -37,20 +37,20 @@ def test_get_task(task_name: str, eval_splits: list[str] | None): @pytest.mark.parametrize("script", [["Latn"], ["Cyrl"], None]) @pytest.mark.parametrize("domains", [["Legal"], ["Medical", "Non-fiction"], None]) @pytest.mark.parametrize("task_types", [["Classification"], ["Clustering"], None]) -@pytest.mark.parametrize("exclude_superseeded_datasets", [True, False]) +@pytest.mark.parametrize("exclude_superseded_datasets", [True, False]) def test_get_tasks( languages: list[str], script: list[str], domains: list[TASK_DOMAIN], task_types: list[TASK_TYPE] | None, - exclude_superseeded_datasets: bool, + exclude_superseded_datasets: bool, ): tasks = mteb.get_tasks( languages=languages, script=script, domains=domains, task_types=task_types, - exclude_superseeded=exclude_superseeded_datasets, + exclude_superseded=exclude_superseded_datasets, ) for task in tasks: @@ -65,7 +65,7 @@ def test_get_tasks( assert set(domains).intersection(set(task_domains)) if task_types: assert task.metadata.type in task_types - if exclude_superseeded_datasets: + if exclude_superseded_datasets: assert task.superseded_by is None diff --git a/tests/test_reproducible_workflow.py b/tests/test_reproducible_workflow.py index afe9f98c8d..ffc892c44b 100644 --- a/tests/test_reproducible_workflow.py +++ b/tests/test_reproducible_workflow.py @@ -8,7 +8,7 @@ from mteb import MTEB from mteb.encoder_interface import Encoder from mteb.model_meta import ModelMeta -from mteb.models.sentence_transformer_wrapper import validate_task_to_prompt_name +from mteb.models.wrapper import Wrapper from tests.test_benchmark.task_grid import TASK_TEST_GRID logging.basicConfig(level=logging.INFO) @@ -62,14 +62,14 @@ def test_validate_task_to_prompt_name(task_name: str | mteb.AbsTask): "query": "prompt_name", "passage": "prompt_name", } - validate_task_to_prompt_name(model_prompts) + Wrapper.validate_task_to_prompt_name(model_prompts) def test_validate_task_to_prompt_name_fail(): with pytest.raises(KeyError): - validate_task_to_prompt_name( + Wrapper.validate_task_to_prompt_name( {"task_name": "prompt_name", "task_name-query": "prompt_name"} ) - with pytest.raises(ValueError): - validate_task_to_prompt_name({"task_name-task_name": "prompt_name"}) + with pytest.raises(KeyError): + Wrapper.validate_task_to_prompt_name({"task_name-task_name": "prompt_name"}) diff --git a/tests/test_tasks/test_all_abstasks.py b/tests/test_tasks/test_all_abstasks.py index e74b43e60b..902c541390 100644 --- a/tests/test_tasks/test_all_abstasks.py +++ b/tests/test_tasks/test_all_abstasks.py @@ -104,10 +104,10 @@ def test_dataset_availability(): asyncio.run(check_datasets_are_available_on_hf(tasks)) -def test_superseeded_dataset_exists(): - tasks = mteb.get_tasks(exclude_superseeded=False) +def test_superseded_dataset_exists(): + tasks = mteb.get_tasks(exclude_superseded=False) for task in tasks: if task.superseded_by: assert ( task.superseded_by in TASKS_REGISTRY - ), f"{task} is superseeded by {task.superseded_by} but {task.superseded_by} is not in the TASKS_REGISTRY" + ), f"{task} is superseded by {task.superseded_by} but {task.superseded_by} is not in the TASKS_REGISTRY" diff --git a/tests/test_tasks/test_metadata.py b/tests/test_tasks/test_metadata.py index 437866c2c8..3d206da5c8 100644 --- a/tests/test_tasks/test_metadata.py +++ b/tests/test_tasks/test_metadata.py @@ -8,7 +8,9 @@ @pytest.mark.parametrize("task", MOCK_TASK_TEST_GRID) def test_descriptive_stats(task): result_stat = task.calculate_metadata_metrics() - task_stat = task.metadata.descriptive_stats + # remove descriptive task file + task.metadata.descriptive_stat_path.unlink() + task_stat = task.expected_stats for key, value in result_stat.items(): assert key in task_stat assert value == task_stat[key] diff --git a/tests/test_tasks/test_mteb_rerank.py b/tests/test_tasks/test_mteb_rerank.py index c540bb41ee..dc65dae905 100644 --- a/tests/test_tasks/test_mteb_rerank.py +++ b/tests/test_tasks/test_mteb_rerank.py @@ -6,8 +6,7 @@ from sentence_transformers import CrossEncoder, SentenceTransformer -from mteb import MTEB -from mteb.model_meta import ModelMeta +from mteb import MTEB, ModelMeta logging.basicConfig(level=logging.INFO) @@ -373,7 +372,19 @@ def test_reranker_same_ndcg1(): open_weights=True, revision=ce_revision, release_date="2021-04-15", + n_parameters=None, + max_tokens=None, + embed_dim=None, + license=None, + public_training_code=None, + public_training_data=None, + reference=None, + similarity_fn_name=None, + use_instructions=None, + training_datasets=None, + framework=["Sentence Transformers", "PyTorch"], ) + eval = MTEB(tasks=["SciFact"]) eval.run( de, diff --git a/tests/test_tasks/test_retrieval_abstask.py b/tests/test_tasks/test_retrieval_abstask.py index 6dca5bb43e..86998b9a78 100644 --- a/tests/test_tasks/test_retrieval_abstask.py +++ b/tests/test_tasks/test_retrieval_abstask.py @@ -1,10 +1,14 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import pytest -from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.tasks.Retrieval.eng.NFCorpusRetrieval import NFCorpus +if TYPE_CHECKING: + from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval + @pytest.mark.parametrize("task", [NFCorpus()]) def test_abstask_calculate_metadata_metrics(task: AbsTaskRetrieval): From b9fe9f04e24c270d1480ebdf28d791d6f0ae6995 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Fri, 24 Jan 2025 22:35:53 +0900 Subject: [PATCH 137/154] [mieb] Fill in align model meta (#1863) * add align model meta * trigger ci * fix meta --- mteb/models/align_models.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/mteb/models/align_models.py b/mteb/models/align_models.py index 9e0827dece..b190a5410a 100644 --- a/mteb/models/align_models.py +++ b/mteb/models/align_models.py @@ -143,18 +143,20 @@ def get_fused_embeddings( revision="e96a37facc7b1f59090ece82293226b817afd6ba", release_date="2023-02-24", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, + n_parameters=176_000_000, + max_tokens=64, + embed_dim=768, license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + open_weights=True, + public_training_code="https://github.com/kakaobrain/coyo-align", + public_training_data=True, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/kakaobrain/align-base", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # COYO-700M + }, ) if __name__ == "__main__": From 6ca11d2cd8e3e3fd59ff09e5b7a016d835ebf76e Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sun, 26 Jan 2025 23:54:26 +0900 Subject: [PATCH 138/154] [mieb] Fill in clip and open clip model meta (#1876) * add clip and open clip model meta * fix training_datasets --- mteb/models/clip_models.py | 36 +++---- mteb/models/openclip_models.py | 176 ++++++++++++++++++--------------- 2 files changed, 114 insertions(+), 98 deletions(-) diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index 089732cffe..f323756b44 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -147,17 +147,17 @@ def get_fused_embeddings( revision="32bd64288804d66eefd0ccbe215aa642df71cc41", release_date="2021-02-26", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, + n_parameters=428_000_000, + max_tokens=77, + embed_dim=768, license=None, - open_weights=None, + open_weights=True, public_training_code=None, public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/openai/clip-vit-large-patch14", similarity_fn_name=None, - use_instructions=None, + use_instructions=False, training_datasets=None, ) @@ -171,17 +171,17 @@ def get_fused_embeddings( revision="3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", release_date="2021-02-26", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, + n_parameters=151_000_000, + max_tokens=77, + embed_dim=512, license=None, - open_weights=None, + open_weights=True, public_training_code=None, public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/openai/clip-vit-base-patch32", similarity_fn_name=None, - use_instructions=None, + use_instructions=False, training_datasets=None, ) @@ -195,17 +195,17 @@ def get_fused_embeddings( revision="57c216476eefef5ab752ec549e440a49ae4ae5f3", release_date="2021-02-26", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, + n_parameters=151_000_000, + max_tokens=77, + embed_dim=512, license=None, - open_weights=None, + open_weights=True, public_training_code=None, public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/openai/clip-vit-base-patch16", similarity_fn_name=None, - use_instructions=None, + use_instructions=False, training_datasets=None, ) diff --git a/mteb/models/openclip_models.py b/mteb/models/openclip_models.py index b39b2ca02e..26e89e6cb6 100644 --- a/mteb/models/openclip_models.py +++ b/mteb/models/openclip_models.py @@ -160,18 +160,20 @@ def get_fused_embeddings( revision="84c9828e63dc9a9351d1fe637c346d4c1c4db341", release_date="2023-04-26", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=428_000_000, + max_tokens=77, + embed_dim=768, + license="mit", + open_weights=True, + public_training_code="https://github.com/mlfoundations/open_clip", + public_training_data="https://huggingface.co/datasets/mlfoundations/datacomp_1b", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # DataComp-1B + }, ) CLIP_ViT_B_32_DataComp_XL_s13B_b90K = ModelMeta( @@ -184,18 +186,20 @@ def get_fused_embeddings( revision="f0e2ffa09cbadab3db6a261ec1ec56407ce42912", release_date="2023-04-26", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=151_000_000, + max_tokens=77, + embed_dim=512, + license="mit", + open_weights=True, + public_training_code="https://github.com/mlfoundations/open_clip", + public_training_data="https://huggingface.co/datasets/mlfoundations/datacomp_1b", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # DataComp-1B + }, ) CLIP_ViT_B_16_DataComp_XL_s13B_b90K = ModelMeta( @@ -208,18 +212,20 @@ def get_fused_embeddings( revision="d110532e8d4ff91c574ee60a342323f28468b287", release_date="2023-04-26", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=150_000_000, + max_tokens=77, + embed_dim=512, + license="mit", + open_weights=True, + public_training_code="https://github.com/mlfoundations/open_clip", + public_training_data="https://huggingface.co/datasets/mlfoundations/datacomp_1b", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # DataComp-1B + }, ) CLIP_ViT_bigG_14_laion2B_39B_b160k = ModelMeta( @@ -232,18 +238,20 @@ def get_fused_embeddings( revision="bc7788f151930d91b58474715fdce5524ad9a189", release_date="2023-01-23", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=2_540_000_000, + max_tokens=77, + embed_dim=1280, + license="mit", + open_weights=True, + public_training_code="https://github.com/mlfoundations/open_clip", + public_training_data="https://laion.ai/blog/laion-5b/", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # 2 Billion sample English subset of LAION-5B + }, ) CLIP_ViT_g_14_laion2B_s34B_b88K = ModelMeta( @@ -256,18 +264,20 @@ def get_fused_embeddings( revision="15efd0f6ac0c40c0f9da7becca03c974d7012604", release_date="2023-03-06", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=1_367_000_000, + max_tokens=77, + embed_dim=1024, + license="mit", + open_weights=True, + public_training_code="https://github.com/mlfoundations/open_clip", + public_training_data="https://laion.ai/blog/laion-5b/", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/laion/CLIP-ViT-g-14-laion2B-s34B-b88K", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # 2 Billion sample English subset of LAION-5B + }, ) CLIP_ViT_H_14_laion2B_s32B_b79K = ModelMeta( @@ -280,18 +290,20 @@ def get_fused_embeddings( revision="de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", release_date="2022-09-15", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=986_000_000, + max_tokens=77, + embed_dim=1024, + license="mit", + open_weights=True, + public_training_code="https://github.com/mlfoundations/open_clip", + public_training_data="https://laion.ai/blog/laion-5b/", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # 2 Billion sample English subset of LAION-5B + }, ) CLIP_ViT_L_14_laion2B_s32B_b82K = ModelMeta( @@ -304,18 +316,20 @@ def get_fused_embeddings( revision="1627032197142fbe2a7cfec626f4ced3ae60d07a", release_date="2022-09-15", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=428_000_000, + max_tokens=77, + embed_dim=768, + license="mit", + open_weights=True, + public_training_code="https://github.com/mlfoundations/open_clip", + public_training_data="https://laion.ai/blog/laion-5b/", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/laion/CLIP-ViT-L-14-laion2B-s32B-b82K", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # 2 Billion sample English subset of LAION-5B + }, ) CLIP_ViT_B_32_laion2B_s34B_b79K = ModelMeta( @@ -328,16 +342,18 @@ def get_fused_embeddings( revision="08f73555f1b2fb7c82058aebbd492887a94968ef", release_date="2022-09-15", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=151_000_000, + max_tokens=77, + embed_dim=512, + license="mit", + open_weights=True, + public_training_code="https://github.com/mlfoundations/open_clip", + public_training_data="https://laion.ai/blog/laion-5b/", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/laion/CLIP-ViT-B-32-laion2B-s34B-b79K", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # 2 Billion sample English subset of LAION-5B + }, ) From edaf0d6f5270a160b293bef405a4637b98faabbd Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sun, 26 Jan 2025 23:54:54 +0900 Subject: [PATCH 139/154] [mieb] Fill in blip model meta (#1874) * add blip and blip 2 model meta * fix references * fix training datasets --- mteb/models/blip2_models.py | 38 +++++--- mteb/models/blip_models.py | 188 +++++++++++++++++++++--------------- 2 files changed, 130 insertions(+), 96 deletions(-) diff --git a/mteb/models/blip2_models.py b/mteb/models/blip2_models.py index af15e205c7..9cb914b15f 100644 --- a/mteb/models/blip2_models.py +++ b/mteb/models/blip2_models.py @@ -215,6 +215,12 @@ def get_fused_embeddings( return BLIP2ModelWrapper(**kwargs) +blip2_training_datasets = { + # COCO + # CC3M+CC12M+SBU + # LAION400M +} + blip2_opt_2_7b = ModelMeta( loader=partial( blip2_loader, @@ -225,18 +231,18 @@ def get_fused_embeddings( revision="51572668da0eb669e01a189dc22abe6088589a24", release_date="2024-03-22", modalities=["image", "text"], - n_parameters=None, + n_parameters=3_740_000_000, max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + embed_dim=768, + license="mit", + open_weights=True, + public_training_code="https://github.com/salesforce/LAVIS/tree/main/projects/blip2", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/Salesforce/blip2-opt-2.7b", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=blip2_training_datasets, ) blip2_opt_6_7b_coco = ModelMeta( @@ -249,18 +255,18 @@ def get_fused_embeddings( revision="0d580de59320a25a4d2c386387bcef310d5f286e", release_date="2024-03-31", modalities=["image", "text"], - n_parameters=None, + n_parameters=7_750_000_000, max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + embed_dim=768, + license="mit", + open_weights=True, + public_training_code="https://github.com/salesforce/LAVIS/tree/main/projects/blip2", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/Salesforce/blip2-opt-6.7b-coco", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=blip2_training_datasets, ) diff --git a/mteb/models/blip_models.py b/mteb/models/blip_models.py index d1670af4dc..da23a2be3d 100644 --- a/mteb/models/blip_models.py +++ b/mteb/models/blip_models.py @@ -164,18 +164,22 @@ def get_fused_embeddings( revision="2227ac38c9f16105cb0412e7cab4759978a8fd90", release_date="2023-12-07", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=470_000_000, + max_tokens=512, + embed_dim=768, + license="bsd-3-clause", + open_weights=True, + public_training_code="https://github.com/salesforce/BLIP", + public_training_data="https://github.com/salesforce/BLIP", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/Salesforce/blip-image-captioning-large", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # COCO + # CC3M+CC12M+SBU + # LAION115M + }, ) blip_image_captioning_base = ModelMeta( @@ -188,18 +192,22 @@ def get_fused_embeddings( revision="89b09ea1789f7addf2f6d6f0dfc4ce10ab58ef84", release_date="2023-08-01", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=247_000_000, + max_tokens=512, + embed_dim=768, + license="bsd-3-clause", + open_weights=True, + public_training_code="https://github.com/salesforce/BLIP", + public_training_data="https://github.com/salesforce/BLIP", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/Salesforce/blip-image-captioning-base", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # COCO + # CC3M+CC12M+SBU + # LAION115M + }, ) @@ -213,18 +221,21 @@ def get_fused_embeddings( revision="c7df8e7cd7aa2ee9af18f56e2b29e59a92651b64", release_date="2023-12-07", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=247_000_000, + max_tokens=512, + embed_dim=768, + license="bsd-3-clause", + open_weights=True, + public_training_code="https://github.com/salesforce/BLIP", + public_training_data="https://github.com/salesforce/BLIP", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/Salesforce/blip-vqa-base", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # CC3M+CC12M+SBU + # LAION115M + }, ) blip_vqa_capfilt_large = ModelMeta( @@ -237,18 +248,21 @@ def get_fused_embeddings( revision="e53f95265aeab69013fabb5380500ab984adbbb4", release_date="2023-01-22", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=247_000_000, + max_tokens=512, + embed_dim=768, + license="bsd-3-clause", + open_weights=True, + public_training_code="https://github.com/salesforce/BLIP", + public_training_data="https://github.com/salesforce/BLIP", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/Salesforce/blip-vqa-capfilt-large", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # CC3M+CC12M+SBU + # LAION115M + }, ) blip_itm_base_coco = ModelMeta( @@ -261,18 +275,21 @@ def get_fused_embeddings( revision="7eaa90c11850c0b17fc38c6a11e7d88bd6ac231f", release_date="2023-08-01", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=247_000_000, + max_tokens=512, + embed_dim=768, + license="bsd-3-clause", + open_weights=True, + public_training_code="https://github.com/salesforce/BLIP", + public_training_data="https://github.com/salesforce/BLIP", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/Salesforce/blip-itm-base-coco", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # CC3M+CC12M+SBU + # LAION115M + }, ) blip_itm_large_coco = ModelMeta( @@ -285,18 +302,22 @@ def get_fused_embeddings( revision="fef05cafc05298067cbbca00b125749394a77a6f", release_date="2023-08-01", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=470_000_000, + max_tokens=512, + embed_dim=768, + license="bsd-3-clause", + open_weights=True, + public_training_code="https://github.com/salesforce/BLIP", + public_training_data="https://github.com/salesforce/BLIP", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/Salesforce/blip-itm-large-coco", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # COCO + # CC3M+CC12M+SBU + # LAION115M + }, ) blip_itm_base_flickr = ModelMeta( @@ -309,18 +330,22 @@ def get_fused_embeddings( revision="1de29e660d91ae1786c1876212ea805a22eab251", release_date="2023-08-01", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=247_000_000, + max_tokens=512, + embed_dim=768, + license="bsd-3-clause", + open_weights=True, + public_training_code="https://github.com/salesforce/BLIP", + public_training_data="https://github.com/salesforce/BLIP", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/Salesforce/blip-itm-base-flickr", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # CC3M+CC12M+SBU + # LAION115M + # Flickr30k + }, ) blip_itm_large_flickr = ModelMeta( @@ -333,18 +358,21 @@ def get_fused_embeddings( revision="bda12e6506758f54261b5ab174b2c55a3ba143fb", release_date="2023-08-01", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=470_000_000, + max_tokens=512, + embed_dim=768, + license="bsd-3-clause", + open_weights=True, + public_training_code="https://github.com/salesforce/BLIP", + public_training_data="https://github.com/salesforce/BLIP", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/Salesforce/blip-itm-large-flickr", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets={ + # CC3M+CC12M+SBU + # LAION115M + }, ) From d1130861944c0b64c1dfe82bb79b283da18b04cc Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 27 Jan 2025 17:49:22 +0900 Subject: [PATCH 140/154] [mieb] Fill in cohere_v and dinov2 model meta (#1880) fill in cohere_v and dino model meta --- mteb/models/cohere_v.py | 12 +++---- mteb/models/dino_models.py | 70 +++++++++++++++++++++----------------- 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/mteb/models/cohere_v.py b/mteb/models/cohere_v.py index d9dcc481b4..0b22994373 100644 --- a/mteb/models/cohere_v.py +++ b/mteb/models/cohere_v.py @@ -194,11 +194,11 @@ def get_fused_embeddings( similarity_fn_name="cosine", framework=[], modalities=["image", "text"], - open_weights=None, + open_weights=False, public_training_code=None, public_training_data=None, - reference=None, - use_instructions=None, + reference="https://huggingface.co/Cohere/Cohere-embed-multilingual-v3.0", + use_instructions=False, training_datasets=None, ) @@ -215,11 +215,11 @@ def get_fused_embeddings( similarity_fn_name="cosine", framework=[], modalities=["image", "text"], - open_weights=None, + open_weights=False, public_training_code=None, public_training_data=None, - reference=None, - use_instructions=None, + reference="https://huggingface.co/Cohere/Cohere-embed-english-v3.0", + use_instructions=False, training_datasets=None, ) diff --git a/mteb/models/dino_models.py b/mteb/models/dino_models.py index 51aad5a5b8..a123fcb52f 100644 --- a/mteb/models/dino_models.py +++ b/mteb/models/dino_models.py @@ -120,6 +120,12 @@ def get_fused_embeddings( return image_embeddings +dinov2_training_datasets = { + # LVD-142M + # ImageNet-22k +} + + dinov2_small = ModelMeta( loader=partial( DINOModelWrapper, @@ -130,18 +136,18 @@ def get_fused_embeddings( revision="ed25f3a31f01632728cabb09d1542f84ab7b0056", release_date="2023-07-18", modalities=["image"], - n_parameters=None, + n_parameters=22_100_000, max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + embed_dim=384, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/facebookresearch/dinov2", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/facebook/dinov2-small", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=dinov2_training_datasets, ) dinov2_base = ModelMeta( @@ -154,18 +160,18 @@ def get_fused_embeddings( revision="f9e44c814b77203eaa57a6bdbbd535f21ede1415", release_date="2023-07-18", modalities=["image"], - n_parameters=None, + n_parameters=86_600_000, max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/facebookresearch/dinov2", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/facebook/dinov2-base", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=dinov2_training_datasets, ) dinov2_large = ModelMeta( @@ -178,18 +184,18 @@ def get_fused_embeddings( revision="47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", release_date="2023-07-18", modalities=["image"], - n_parameters=None, + n_parameters=304_000_000, max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + embed_dim=1024, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/facebookresearch/dinov2", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/facebook/dinov2-large", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=dinov2_training_datasets, ) dinov2_giant = ModelMeta( @@ -202,18 +208,18 @@ def get_fused_embeddings( revision="611a9d42f2335e0f921f1e313ad3c1b7178d206d", release_date="2023-07-18", modalities=["image"], - n_parameters=None, + n_parameters=1_140_000_000, max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + embed_dim=1536, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/facebookresearch/dinov2", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/facebook/dinov2-giant", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=dinov2_training_datasets, ) if __name__ == "__main__": From b347f2e8e26595310ca2d053ff6674f33bfdf576 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Tue, 28 Jan 2025 15:19:09 +0900 Subject: [PATCH 141/154] [mieb] Fill in e5v and eva clip model meta (#1885) * add e5v model meta * add some eva_clip2 meta * add the rest of meta --- mteb/models/e5_v.py | 20 ++++---- mteb/models/evaclip_models.py | 87 +++++++++++++++++++---------------- 2 files changed, 59 insertions(+), 48 deletions(-) diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index ba72c42c0f..dcbe0a1d71 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -198,18 +198,20 @@ def get_fused_embeddings( revision="0c1f22679417b3ae925d779442221c40cd1861ab", release_date="2024-07-17", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, + n_parameters=8_360_000_000, + max_tokens=8192, + embed_dim=4096, license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + open_weights=True, + public_training_code="https://github.com/kongds/E5-V", + public_training_data="https://huggingface.co/datasets/princeton-nlp/datasets-for-simcse", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/royokong/e5-v", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=True, + training_datasets={ + # princeton-nlp/datasets-for-simcse + }, ) if __name__ == "__main__": diff --git a/mteb/models/evaclip_models.py b/mteb/models/evaclip_models.py index c5395bfa25..fdd25771d4 100644 --- a/mteb/models/evaclip_models.py +++ b/mteb/models/evaclip_models.py @@ -24,7 +24,7 @@ def evaclip_loader(**kwargs): # https://github.com/baaivision/EVA/tree/master/EVA-CLIP#setup raise ImportError( "Please run `git clone git@github.com:baaivision/EVA.git`," - "`pip install ninja`" + "`pip install ninja timm`" "`pip install -v -U git+https://github.com/facebookresearch/xformers.git@main#egg=xformers`" "`git clone https://github.com/NVIDIA/apex && cd apex && pip install -v --disable-pip-version-check --no-build-isolation --no-cache-dir ./`" ) @@ -165,6 +165,15 @@ def get_fused_embeddings( return EvaCLIPWrapper(**kwargs) +training_code = "https://github.com/baaivision/EVA/tree/master/EVA-CLIP" +training_datasets = { + # COYO-700M, random sample 400M. https://github.com/kakaobrain/coyo-dataset + # LAION-2B, random sample 1.6B. https://laion.ai/blog/laion-5b/ +} +laion_2b = { + # LAION-2B +} + EVA02_CLIP_B_16 = ModelMeta( loader=partial( evaclip_loader, @@ -175,18 +184,18 @@ def get_fused_embeddings( revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=149_000_000, + max_tokens=77, + embed_dim=512, + license="mit", + open_weights=True, + public_training_code=training_code, public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/QuanSun/EVA-CLIP", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=training_datasets, ) EVA02_CLIP_L_14 = ModelMeta( @@ -199,18 +208,18 @@ def get_fused_embeddings( revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=428_000_000, + max_tokens=77, + embed_dim=768, + license="mit", + open_weights=True, + public_training_code=training_code, public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/QuanSun/EVA-CLIP", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=training_datasets, ) EVA02_CLIP_bigE_14 = ModelMeta( @@ -223,18 +232,18 @@ def get_fused_embeddings( revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=4_700_000_000, + max_tokens=77, + embed_dim=1024, + license="mit", + open_weights=True, + public_training_code=training_code, + public_training_data="https://laion.ai/blog/laion-5b/", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/QuanSun/EVA-CLIP", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=laion_2b, ) @@ -248,16 +257,16 @@ def get_fused_embeddings( revision="11afd202f2ae80869d6cef18b1ec775e79bd8d12", release_date="2023-04-26", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=5_000_000_000, + max_tokens=77, + embed_dim=1024, + license="mit", + open_weights=True, + public_training_code=training_code, + public_training_data="https://laion.ai/blog/laion-5b/", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/QuanSun/EVA-CLIP", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=laion_2b, ) From e8b2256692f8b6f69f5dfeb5782f6d482f88bd12 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Tue, 28 Jan 2025 19:25:55 +0900 Subject: [PATCH 142/154] [mieb] Fill out gme v and jina clip model meta (#1887) * add gme_v model meta * fill in jina clip meta * jina clip v1 training data * correct type * cosmetic fix * handle case where name is a variable and not a string * Update mteb/models/gme_v_models.py Co-authored-by: Roman Solomatin --------- Co-authored-by: Roman Solomatin --- mteb/models/gme_v_models.py | 26 ++++++++++++++++++++++++-- mteb/models/jina_clip.py | 23 +++++++++++++++-------- scripts/extract_model_names.py | 28 ++++++++++++++++++++-------- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/mteb/models/gme_v_models.py b/mteb/models/gme_v_models.py index be4f7b207a..14812c0859 100644 --- a/mteb/models/gme_v_models.py +++ b/mteb/models/gme_v_models.py @@ -404,6 +404,26 @@ def fetch_image( ### +training_data = { + "MSMARCO": ["train"], + "NQ": ["train"], + "NQHardNegatives": ["train"], + "NanoNQRetrieval": ["train"], + "NQ-PL": ["train"], # translation not trained on + "HotpotQA": ["train"], + "HotpotQA-PL": ["train"], # translation not trained on + "HotpotQAHardNegatives": ["train"], + # TriviaQA (Joshi et al., 2017), + # SQuAD (Rajpurkar et al., 2016), + "FEVER": ["train"], + # AllNLI for SimCSE (Gao et al., 2021), selecting a total of 1 million entries. + # ImageNet (Deng et al., 2009) + # LAION (Schuhmann et al., 2022), + # mscoco (Lin et al., 2014), + # Docmatix (Laurenc¸on et al., 2024) + # synthetic data + # M-BEIR (Wei et al., 2024) +} gme_qwen2vl_2b = ModelMeta( @@ -416,6 +436,7 @@ def fetch_image( open_weights=True, revision="ce765ae71b8cdb208203cd8fb64a170b1b84293a", release_date="2024-12-24", + modalities=["image", "text"], n_parameters=2_210_000_000, embed_dim=1536, license="apache-2.0", @@ -426,7 +447,7 @@ def fetch_image( use_instructions=True, public_training_code=None, public_training_data=None, - training_datasets=None, + training_datasets=training_data, ) gme_qwen2vl_7b = ModelMeta( @@ -439,6 +460,7 @@ def fetch_image( open_weights=True, revision="477027a6480f8630363be77751f169cc3434b673", release_date="2024-12-24", + modalities=["image", "text"], n_parameters=8_290_000_000, embed_dim=3584, license="apache-2.0", @@ -449,5 +471,5 @@ def fetch_image( use_instructions=True, public_training_code=None, public_training_data=None, - training_datasets=None, + training_datasets=training_data, ) diff --git a/mteb/models/jina_clip.py b/mteb/models/jina_clip.py index bff02a76c3..1f9a597803 100644 --- a/mteb/models/jina_clip.py +++ b/mteb/models/jina_clip.py @@ -158,18 +158,25 @@ def encode( # type: ignore revision="06150c7c382d7a4faedc7d5a0d8cdb59308968f4", release_date="2024-05-30", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, + n_parameters=223_000_000, + max_tokens=8192, + embed_dim=768, + license="apache-2.0", + open_weights=True, public_training_code=None, public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/jinaai/jina-clip-v1", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=True, + training_datasets={ + # LAION400M + # ShareGPT4V + "MSMARCO": ["train"], + # NQ + # HotpotQA + # Natural Language Inference (NLI) dataset (Bowman et al., 2015) + }, ) diff --git a/scripts/extract_model_names.py b/scripts/extract_model_names.py index 6cbaa2c298..36cfc572e9 100644 --- a/scripts/extract_model_names.py +++ b/scripts/extract_model_names.py @@ -48,14 +48,26 @@ def extract_model_names( and isinstance(node.value.func, ast.Name) and node.value.func.id == "ModelMeta" ): - model_name = next( - ( - kw.value.value - for kw in node.value.keywords - if kw.arg == "name" - ), - None, - ) + try: + model_name = next( + ( + kw.value.value + for kw in node.value.keywords + if kw.arg == "name" + ), + None, + ) + except AttributeError: + # For cases where name is assigned a variable and not a direct string, + # e.g. in gme_v_models.py: `name=HF_GME_QWEN2VL_2B` + model_name = next( + ( + kw.value.id + for kw in node.value.keywords + if kw.arg == "name" + ), + None, + ) if model_name: model_names.append(model_name) first_model_found = True From a783b04eda535bb48788c14dadcbfa6e0cc9198d Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Tue, 28 Jan 2025 21:33:10 +0900 Subject: [PATCH 143/154] [mieb] Fill in mocov3 and nomic vision model meta (#1890) * add moco model meta * add nomic v model meta * add training dataset for nomic --- mteb/models/moco_models.py | 36 +++++++++++++++++------------- mteb/models/nomic_models_vision.py | 21 +++++++++-------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/mteb/models/moco_models.py b/mteb/models/moco_models.py index 9493b8e5a6..1c896331bc 100644 --- a/mteb/models/moco_models.py +++ b/mteb/models/moco_models.py @@ -138,6 +138,10 @@ def get_fused_embeddings( return MOCOv3Wrapper(**kwargs) +mocov3_training_datasets = { + # imagenet +} + mocov3_vit_base = ModelMeta( loader=partial( mocov3_loader, @@ -148,18 +152,18 @@ def get_fused_embeddings( revision="7d091cd70772c5c0ecf7f00b5f12ca609a99d69d", release_date="2024-06-03", modalities=["image"], - n_parameters=None, + n_parameters=86_600_000, max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + embed_dim=768, + license="cc-by-nc-4.0", + open_weights=True, + public_training_code="https://github.com/facebookresearch/moco-v3", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://github.com/facebookresearch/moco-v3", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=mocov3_training_datasets, ) mocov3_vit_large = ModelMeta( @@ -172,16 +176,16 @@ def get_fused_embeddings( revision="7bf75358d616f39b9716148bf4e3425f3bd35b47", release_date="2024-06-03", modalities=["image"], - n_parameters=None, + n_parameters=304_000_000, max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + embed_dim=1024, + license="cc-by-nc-4.0", + open_weights=True, + public_training_code="https://github.com/facebookresearch/moco-v3", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://github.com/facebookresearch/moco-v3", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=mocov3_training_datasets, ) diff --git a/mteb/models/nomic_models_vision.py b/mteb/models/nomic_models_vision.py index 126743a97d..7ea87bd38e 100644 --- a/mteb/models/nomic_models_vision.py +++ b/mteb/models/nomic_models_vision.py @@ -171,18 +171,21 @@ def get_fused_embeddings( revision="af2246fffdab78d8458418480e4886a8e48b70a7", release_date="2024-06-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=92_900_000, + max_tokens=2048, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/nomic-ai/contrastors", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/nomic-ai/nomic-embed-vision-v1.5", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=True, + training_datasets={ + # https://arxiv.org/pdf/2406.18587 + # DFN-2B + }, ) if __name__ == "__main__": From 0c5f0fb5feed442a2a50991247b6d44436d7956d Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 29 Jan 2025 00:35:41 +0900 Subject: [PATCH 144/154] [mieb] Fill in siglip model meta (#1894) add siglip model meta --- mteb/models/siglip_models.py | 184 ++++++++++++++++++----------------- 1 file changed, 94 insertions(+), 90 deletions(-) diff --git a/mteb/models/siglip_models.py b/mteb/models/siglip_models.py index 739b5aa59a..3bca2e508e 100644 --- a/mteb/models/siglip_models.py +++ b/mteb/models/siglip_models.py @@ -155,6 +155,10 @@ def get_fused_embeddings( return image_embeddings +siglip_training_datasets = { + # WebLI https://arxiv.org/abs/2209.06794 +} + siglip_so400m_patch14_224 = ModelMeta( loader=partial( SiglipModelWrapper, @@ -165,18 +169,18 @@ def get_fused_embeddings( revision="d04cf29fca7b6374f74d8bea1969314492266b5e", release_date="2024-01-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=877_000_000, + max_tokens=16, + embed_dim=1152, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/google/siglip-so400m-patch14-224", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=siglip_training_datasets, ) siglip_so400m_patch14_384 = ModelMeta( @@ -189,18 +193,18 @@ def get_fused_embeddings( revision="9fdffc58afc957d1a03a25b10dba0329ab15c2a3", release_date="2024-01-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=878_000_000, + max_tokens=64, + embed_dim=1152, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/google/siglip-so400m-patch14-384", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=siglip_training_datasets, ) siglip_so400m_patch16_256_i18n = ModelMeta( @@ -213,18 +217,18 @@ def get_fused_embeddings( revision="365d321c0cfdea96bc28e3a29787a11a062681a1", release_date="2024-01-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=1_130_000_000, + max_tokens=64, + embed_dim=1152, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/google/siglip-so400m-patch16-256-i18n", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=siglip_training_datasets, ) siglip_base_patch16_256_multilingual = ModelMeta( @@ -237,18 +241,18 @@ def get_fused_embeddings( revision="8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6", release_date="2024-01-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=371_000_000, + max_tokens=64, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/google/siglip-base-patch16-256-multilingual", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=siglip_training_datasets, ) siglip_base_patch16_256 = ModelMeta( @@ -261,18 +265,18 @@ def get_fused_embeddings( revision="b078df89e446d623010d890864d4207fe6399f61", release_date="2024-01-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=203_000_000, + max_tokens=64, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/google/siglip-base-patch16-256", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=siglip_training_datasets, ) siglip_base_patch16_512 = ModelMeta( @@ -285,18 +289,18 @@ def get_fused_embeddings( revision="753a949581523b60257d93e18391e8c27f72eb22", release_date="2024-01-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=204_000_000, + max_tokens=64, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/google/siglip-base-patch16-512", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=siglip_training_datasets, ) siglip_base_patch16_384 = ModelMeta( @@ -309,18 +313,18 @@ def get_fused_embeddings( revision="41aec1c83b32e0a6fca20ad88ba058aa5b5ea394", release_date="2024-01-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=203_000_000, + max_tokens=64, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/google/siglip-base-patch16-384", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=siglip_training_datasets, ) siglip_base_patch16_224 = ModelMeta( @@ -333,18 +337,18 @@ def get_fused_embeddings( revision="7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed", release_date="2024-01-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=203_000_000, + max_tokens=64, + embed_dim=768, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/google/siglip-base-patch16-224", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=siglip_training_datasets, ) siglip_large_patch16_256 = ModelMeta( @@ -357,18 +361,18 @@ def get_fused_embeddings( revision="d0da9f876e7d66b4e250cd2450c3ba2ce735e447", release_date="2024-01-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=652_000_000, + max_tokens=64, + embed_dim=1024, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/google/siglip-large-patch16-256", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=siglip_training_datasets, ) siglip_large_patch16_384 = ModelMeta( @@ -381,18 +385,18 @@ def get_fused_embeddings( revision="ce005573a40965dfd21fd937fbdeeebf2439fc35", release_date="2024-01-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, + n_parameters=652_000_000, + max_tokens=64, + embed_dim=1024, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", public_training_data=None, framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/google/siglip-large-patch16-384", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=siglip_training_datasets, ) if __name__ == "__main__": From d551bf95be394818e7e94b378d949dff53e8d848 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 29 Jan 2025 23:40:08 +0900 Subject: [PATCH 145/154] [mieb] Fill in vista vlm2vec voyage v model meta (#1903) * add vista model meta * add vlm2vec model meta * add more meta --- mteb/models/vista_models.py | 34 +++++++++++++++------------- mteb/models/vlm2vec_models.py | 42 +++++++++++++++++++---------------- 2 files changed, 42 insertions(+), 34 deletions(-) diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py index 3ea5e13fe6..2a887fb48d 100644 --- a/mteb/models/vista_models.py +++ b/mteb/models/vista_models.py @@ -232,6 +232,10 @@ def calculate_probs(self, text_embeddings, image_embeddings): return VisualizedBGEWrapper(**kwargs) +vista_training_datasets = { + # VISTA_S2 +} + visualized_bge_base = ModelMeta( loader=partial( vista_loader, @@ -244,18 +248,18 @@ def calculate_probs(self, text_embeddings, image_embeddings): revision="98db10b10d22620010d06f11733346e1c98c34aa", release_date="2024-06-06", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, + n_parameters=196_000_000, + max_tokens=77, + embed_dim=768, license=None, - open_weights=None, + open_weights=True, public_training_code=None, - public_training_data=None, + public_training_data="https://huggingface.co/datasets/JUNJIE99/VISTA_S2", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/BAAI/bge-visualized", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=vista_training_datasets, ) visualized_bge_m3 = ModelMeta( @@ -271,17 +275,17 @@ def calculate_probs(self, text_embeddings, image_embeddings): release_date="2024-06-06", modalities=["image", "text"], n_parameters=None, - max_tokens=None, - embed_dim=None, + max_tokens=77, + embed_dim=1024, license=None, - open_weights=None, + open_weights=True, public_training_code=None, - public_training_data=None, + public_training_data="https://huggingface.co/datasets/JUNJIE99/VISTA_S2", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/BAAI/bge-visualized", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=False, + training_datasets=vista_training_datasets, ) if __name__ == "__main__": diff --git a/mteb/models/vlm2vec_models.py b/mteb/models/vlm2vec_models.py index efdcf27756..7ca458c6a0 100644 --- a/mteb/models/vlm2vec_models.py +++ b/mteb/models/vlm2vec_models.py @@ -362,6 +362,10 @@ def get_fused_embeddings( return fused_embeddings +vlm2vec_training_datasets = { + # MMEB-train +} + vlm2vec_lora = ModelMeta( loader=partial( VLM2VecWrapper, @@ -373,17 +377,17 @@ def get_fused_embeddings( release_date="2024-10-08", modalities=["image", "text"], n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + max_tokens=131072, + embed_dim=3072, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/TIGER-AI-Lab/VLM2Vec", + public_training_data="https://huggingface.co/datasets/TIGER-Lab/MMEB-train", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/TIGER-Lab/VLM2Vec-LoRA", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=True, + training_datasets=vlm2vec_training_datasets, ) vlm2vec_full = ModelMeta( @@ -396,16 +400,16 @@ def get_fused_embeddings( revision="e9afa98002097ac2471827ba23ea1f2ddd229480", release_date="2024-10-08", modalities=["image", "text"], - n_parameters=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, + n_parameters=4_150_000_000, + max_tokens=131072, + embed_dim=3072, + license="apache-2.0", + open_weights=True, + public_training_code="https://github.com/TIGER-AI-Lab/VLM2Vec", + public_training_data="https://huggingface.co/TIGER-Lab/VLM2Vec-Full", framework=["PyTorch"], - reference=None, + reference="https://huggingface.co/TIGER-Lab/VLM2Vec-Full", similarity_fn_name=None, - use_instructions=None, - training_datasets=None, + use_instructions=True, + training_datasets=vlm2vec_training_datasets, ) From 788f8c41d1e9aa3a03de1b888435ecb5b20df4f7 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 3 Feb 2025 21:19:57 +0900 Subject: [PATCH 146/154] [mieb] merge from main once more (#1942) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * 1.31.4 Automatically generated by python-semantic-release * Update tasks table * fix: Limited plotly version to be less than 6.0.0 (#1902) Limited plotly version to be less than 6.0.0 * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * update stella/jasper metainfo (#1896) update stella meta * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * 1.31.5 Automatically generated by python-semantic-release * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Feat: Add FaMTEB (Farsi/Persian Text Embedding Benchmark) (#1843) * Add Summary Retrieval Task * Add FaMTEBClassification * Add FaMTEBClustering * Add FaMTEBPairClassification * Add FaMTEBRetrieval and BEIRFA and FaMTEBSTS * Add FaMTEBSummaryRetrieval * Add FaMTEB to benchmarks * fix benchmark names * temporary fix metadata * Fix dataset revisions * Update SummaryRetrievalEvaluator.py * Update task files * Update task files * add data domain and subtask description * Update AbsTaskSummaryRetrieval and FaMTEBSummaryRetrieval * Update AbsTaskSummaryRetrieval * Add mock task * Update AbsTaskSummaryRetrieval * Update AbsTaskSummaryRetrieval * make lint * Refactor SummaryRetrieval to subclass BitextMining * Add aggregated datasets --------- Co-authored-by: mehran Co-authored-by: e.zeinivand Co-authored-by: Erfun76 <59398902+Erfun76@users.noreply.github.com> * Update tasks table * Docs: update docs according to current state (#1870) * update docs * Apply suggestions from code review Co-authored-by: Isaac Chung * update readme * Update README.md Co-authored-by: Isaac Chung --------- Co-authored-by: Isaac Chung * Update tasks table * Update tasks table * Update tasks table * Adding a banner to the new MMTEB leaderboard (#1908) * Adding a banner to the new MMTEB leaderboard * linting * Update mteb/leaderboard/app.py Co-authored-by: Isaac Chung * adding reference to mteb arena --------- Co-authored-by: Isaac Chung * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * fix: Filling missing metadata for leaderboard release (#1895) * Update ArxivClusteringS2S.py * fill some metadat for retrieval * fill in the reste of missing metadata * fix metadata * fix climatefever metadata * fix: Added CQADupstack annotations * removed annotation for non-exisitant task * format * Added financial to other financial dataset * Moved ArguAna annotation to derivate datasets --------- Co-authored-by: Kenneth Enevoldsen * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * 1.31.6 Automatically generated by python-semantic-release * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * fix: remove SummaryRetrieval as a type (#1915) * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * fix: revert rename and add to description (#1918) * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * Update tasks table * docs: Add sort to domains for task metadata (#1922) Tests currently go into an infinite loop. This should prevent that. * Update tasks table * 1.31.7 Automatically generated by python-semantic-release * docs: Updated citation for mteb(scandinavian) (#1914) fix: Updated citation for mteb(scandinavian) * fix: Add datasets in CodeRAG-Bench (#1595) * add three out of four datasets in CodeRAG-Bench * add verified CodeRAGStackoverflowPostsRetrieval dataset * clean up code and make some comments * fixed lint errors * addressed comments about code-rag datasets: fixed grammar and remove unnessary code and loop * roll back files which is not supposed to change * fixed the comments in split_by_first_newline() and make the methods private by adding a underscore prefix * refactor to use common args * update task descriptions * add entry in benchmarks * correct the alphanumeric order for the dataset * add in tasks.md * add in tasks.md * update task metadata * update importing path * fix lint errors * correct CodeRAG task metadata description field and id for stackoverflow-posts * fix error in test --------- Co-authored-by: Isaac Chung * Update tasks table * 1.31.8 Automatically generated by python-semantic-release * Leaderboard: Acks (#1930) Add acs * omit instructions.py --------- Co-authored-by: github-actions[bot] Co-authored-by: github-actions Co-authored-by: Márton Kardos Co-authored-by: Roman Solomatin Co-authored-by: Mehran Sarmadi <128898167+mehran-sarmadi@users.noreply.github.com> Co-authored-by: mehran Co-authored-by: e.zeinivand Co-authored-by: Erfun76 <59398902+Erfun76@users.noreply.github.com> Co-authored-by: Wissam Siblini <36303760+wissam-sib@users.noreply.github.com> Co-authored-by: Imene Kerboua <33312980+imenelydiaker@users.noreply.github.com> Co-authored-by: Kenneth Enevoldsen Co-authored-by: Pengfei He Co-authored-by: Niklas Muennighoff --- .github/pull_request_template.md | 13 +- .gitignore | 3 +- README.md | 23 +- docs/adding_a_benchmark.md | 7 + docs/adding_a_leaderboard_tab.md | 15 - docs/adding_a_model.md | 139 +- docs/benchmarks.md | 33 +- docs/create_tasks_table.py | 4 +- docs/mmteb/points_table.md | 303 +- docs/tasks.md | 4091 ++++++----------- mteb/abstasks/AbsTask.py | 3 +- mteb/abstasks/AbsTaskBitextMining.py | 2 +- mteb/abstasks/TaskMetadata.py | 23 +- mteb/abstasks/aggregate_task_metadata.py | 172 + mteb/abstasks/aggregated_task.py | 149 + mteb/benchmarks/benchmarks.py | 173 +- mteb/create_meta.py | 8 +- mteb/evaluation/MTEB.py | 42 +- mteb/leaderboard/app.py | 42 +- mteb/load_results/benchmark_results.py | 23 +- mteb/load_results/task_results.py | 43 +- mteb/models/arctic_models.py | 3 +- mteb/models/bedrock_models.py | 264 ++ mteb/models/e5_instruct.py | 1 - mteb/models/e5_models.py | 1 - mteb/models/gritlm_models.py | 1 - mteb/models/instruct_wrapper.py | 85 + mteb/models/instructions.py | 430 -- mteb/models/jasper_models.py | 17 +- mteb/models/jina_models.py | 24 + mteb/models/lens_models.py | 10 +- mteb/models/nvidia_models.py | 75 +- mteb/models/overview.py | 2 + mteb/models/salesforce_models.py | 1 - mteb/models/stella_models.py | 6 +- mteb/models/voyage_models.py | 8 +- mteb/tasks/BitextMining/__init__.py | 1 + .../eng/PubChemSMILESBitextMining.py | 68 + mteb/tasks/BitextMining/eng/__init__.py | 0 mteb/tasks/Classification/__init__.py | 18 + .../eng/FinancialPhrasebankClassification.py | 2 +- .../eng/SDSEyeProtectionClassification.py | 44 + .../eng/SDSGlovesClassification.py | 44 + .../eng/WikipediaBioMetChemClassification.py | 37 + .../WikipediaBiolumNeurochemClassification.py | 37 + ...kipediaChemEngSpecialtiesClassification.py | 37 + .../eng/WikipediaChemFieldsClassification.py | 37 + .../WikipediaChemistryTopicsClassification.py | 37 + ...pediaCompChemSpectroscopyClassification.py | 37 + ...ediaCryobiologySeparationClassification.py | 37 + ...CrystallographyAnalyticalClassification.py | 37 + ...ediaGreenhouseEnantiopureClassification.py | 37 + .../WikipediaIsotopesFissionClassification.py | 37 + .../WikipediaLuminescenceClassification.py | 37 + ...WikipediaOrganicInorganicClassification.py | 37 + ...ipediaSaltsSemiconductorsClassification.py | 37 + ...ipediaSolidStateColloidalClassification.py | 37 + ...kipediaTheoreticalAppliedClassification.py | 37 + .../fas/FaMTEBClassification.py | 635 +++ mteb/tasks/Classification/kor/KorFin.py | 2 +- mteb/tasks/Clustering/__init__.py | 3 + .../Clustering/eng/ArxivClusteringS2S.py | 14 +- mteb/tasks/Clustering/eng/RedditClustering.py | 15 +- .../Clustering/eng/RedditClusteringP2P.py | 15 +- .../Clustering/eng/StackExchangeClustering.py | 15 +- .../eng/StackExchangeClusteringP2P.py | 14 +- ...WikipediaChemistrySpecialtiesClustering.py | 37 + .../eng/WikipediaChemistryTopicsClustering.py | 37 + mteb/tasks/Clustering/fas/FaMTEBClustering.py | 211 + mteb/tasks/Clustering/fas/__init__.py | 0 .../Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py | 3 + .../Any2AnyRetrieval/eng/EDIST2ITRetrieval.py | 1 + .../eng/EncyclopediaVQAIT2ITRetrieval.py | 3 + .../eng/Fashion200kI2TRetrieval.py | 3 + .../eng/Fashion200kT2IRetrieval.py | 3 + .../eng/FashionIQIT2IRetrieval.py | 3 + .../eng/Flickr30kI2TRetrieval.py | 1 + .../eng/Flickr30kT2IRetrieval.py | 1 + .../eng/InfoSeekIT2ITRetrieval.py | 3 + .../eng/InfoSeekIT2TRetrieval.py | 3 + .../eng/LLaVAIT2TRetrieval.py | 3 + .../eng/MSCOCOI2TRetrieval.py | 3 + .../eng/MSCOCOT2IRetrieval.py | 1 + .../eng/NIGHTSI2IRetrieval.py | 3 + .../eng/OKVQAIT2TRetrieval.py | 3 + .../eng/OVENIT2ITRetrieval.py | 3 + .../Any2AnyRetrieval/eng/OVENIT2TRetrieval.py | 3 + .../eng/ReMuQIT2TRetrieval.py | 3 + .../eng/VidoreBenchRetrieval.py | 10 + .../eng/VisualNewsI2TRetrieval.py | 1 + .../eng/VisualNewsT2IRetrieval.py | 3 + .../eng/WebQAT2ITRetrieval.py | 1 + .../Any2AnyRetrieval/eng/WebQAT2TRetrieval.py | 3 + mteb/tasks/PairClassification/__init__.py | 6 + .../eng/PubChemAISentenceParaphrasePC.py | 60 + .../PairClassification/eng/PubChemSMILESPC.py | 128 + .../eng/PubChemSynonymPC.py | 61 + .../eng/PubChemWikiParagraphsPC.py | 60 + .../eng/TwitterSemEval2015PC.py | 12 +- .../eng/TwitterURLCorpusPC.py | 12 +- .../fas/FaMTEBPairClassification.py | 282 ++ .../PubChemWikiPairClassification.py | 77 + .../Reranking/eng/AskUbuntuDupQuestions.py | 8 +- .../eng/StackOverflowDupQuestions.py | 14 +- mteb/tasks/Retrieval/__init__.py | 5 + mteb/tasks/Retrieval/code/CodeRAG.py | 272 ++ .../eng/CQADupstackAndroidRetrieval.py | 12 +- .../eng/CQADupstackEnglishRetrieval.py | 12 +- .../eng/CQADupstackGamingRetrieval.py | 12 +- .../Retrieval/eng/CQADupstackGisRetrieval.py | 12 +- .../eng/CQADupstackMathematicaRetrieval.py | 12 +- .../eng/CQADupstackPhysicsRetrieval.py | 12 +- .../eng/CQADupstackProgrammersRetrieval.py | 2 +- .../eng/CQADupstackStatsRetrieval.py | 12 +- .../Retrieval/eng/CQADupstackTexRetrieval.py | 12 +- .../Retrieval/eng/CQADupstackUnixRetrieval.py | 12 +- .../eng/CQADupstackWebmastersRetrieval.py | 12 +- .../eng/CQADupstackWordpressRetrieval.py | 12 +- .../Retrieval/eng/ChemHotpotQARetrieval.py | 60 + mteb/tasks/Retrieval/eng/ChemNQRetrieval.py | 45 + .../Retrieval/eng/ClimateFEVERRetrieval.py | 24 +- mteb/tasks/Retrieval/eng/FEVERRetrieval.py | 12 +- mteb/tasks/Retrieval/eng/FiQA2018Retrieval.py | 12 +- mteb/tasks/Retrieval/eng/MSMARCORetrieval.py | 46 +- .../tasks/Retrieval/eng/MSMARCOv2Retrieval.py | 23 +- mteb/tasks/Retrieval/eng/NQRetrieval.py | 15 +- mteb/tasks/Retrieval/eng/QuoraRetrieval.py | 12 +- mteb/tasks/Retrieval/eng/SciFactRetrieval.py | 4 +- mteb/tasks/Retrieval/fas/BEIRFa.py | 662 +++ mteb/tasks/Retrieval/fas/FaMTEBRetrieval.py | 140 + mteb/tasks/Retrieval/fas/__init__.py | 0 mteb/tasks/Retrieval/kor/AutoRAGRetrieval.py | 2 +- .../tasks/Retrieval/pol/ArguAnaPLRetrieval.py | 6 +- mteb/tasks/Retrieval/pol/FiQAPLRetrieval.py | 12 +- mteb/tasks/STS/__init__.py | 1 + mteb/tasks/STS/eng/BiossesSTS.py | 12 +- mteb/tasks/STS/eng/STSBenchmarkSTS.py | 12 +- mteb/tasks/STS/fas/FaMTEBSTS.py | 104 + mteb/tasks/STS/fas/__init__.py | 0 mteb/tasks/SummaryRetrieval/__init__.py | 3 + .../fas/FaMTEBSummaryRetrieval.py | 97 + mteb/tasks/SummaryRetrieval/fas/__init__.py | 0 mteb/tasks/__init__.py | 2 + .../aggregated_tasks/CQADupStackRetrieval.py | 62 + .../CQADupStackRetrievalFa.py | 46 + .../SynPerChatbotConvSAClassification.py | 40 + mteb/tasks/aggregated_tasks/__init__.py | 11 + pyproject.toml | 4 +- scripts/extract_model_names.py | 1 + tests/test_TaskMetadata.py | 3 + tests/test_overview.py | 2 +- tests/test_tasks/test_all_abstasks.py | 3 + 152 files changed, 6849 insertions(+), 3793 deletions(-) create mode 100644 docs/adding_a_benchmark.md delete mode 100644 docs/adding_a_leaderboard_tab.md create mode 100644 mteb/abstasks/aggregate_task_metadata.py create mode 100644 mteb/abstasks/aggregated_task.py create mode 100644 mteb/models/bedrock_models.py delete mode 100644 mteb/models/instructions.py create mode 100644 mteb/tasks/BitextMining/eng/PubChemSMILESBitextMining.py create mode 100644 mteb/tasks/BitextMining/eng/__init__.py create mode 100644 mteb/tasks/Classification/eng/SDSEyeProtectionClassification.py create mode 100644 mteb/tasks/Classification/eng/SDSGlovesClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaBioMetChemClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaBiolumNeurochemClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaChemEngSpecialtiesClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaChemFieldsClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaChemistryTopicsClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaCompChemSpectroscopyClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaCryobiologySeparationClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaCrystallographyAnalyticalClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaGreenhouseEnantiopureClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaIsotopesFissionClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaLuminescenceClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaOrganicInorganicClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaSaltsSemiconductorsClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaSolidStateColloidalClassification.py create mode 100644 mteb/tasks/Classification/eng/WikipediaTheoreticalAppliedClassification.py create mode 100644 mteb/tasks/Classification/fas/FaMTEBClassification.py create mode 100644 mteb/tasks/Clustering/eng/WikipediaChemistrySpecialtiesClustering.py create mode 100644 mteb/tasks/Clustering/eng/WikipediaChemistryTopicsClustering.py create mode 100644 mteb/tasks/Clustering/fas/FaMTEBClustering.py create mode 100644 mteb/tasks/Clustering/fas/__init__.py create mode 100644 mteb/tasks/PairClassification/eng/PubChemAISentenceParaphrasePC.py create mode 100644 mteb/tasks/PairClassification/eng/PubChemSMILESPC.py create mode 100644 mteb/tasks/PairClassification/eng/PubChemSynonymPC.py create mode 100644 mteb/tasks/PairClassification/eng/PubChemWikiParagraphsPC.py create mode 100644 mteb/tasks/PairClassification/fas/FaMTEBPairClassification.py create mode 100644 mteb/tasks/PairClassification/multilingual/PubChemWikiPairClassification.py create mode 100644 mteb/tasks/Retrieval/code/CodeRAG.py create mode 100644 mteb/tasks/Retrieval/eng/ChemHotpotQARetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/ChemNQRetrieval.py create mode 100644 mteb/tasks/Retrieval/fas/BEIRFa.py create mode 100644 mteb/tasks/Retrieval/fas/FaMTEBRetrieval.py create mode 100644 mteb/tasks/Retrieval/fas/__init__.py create mode 100644 mteb/tasks/STS/fas/FaMTEBSTS.py create mode 100644 mteb/tasks/STS/fas/__init__.py create mode 100644 mteb/tasks/SummaryRetrieval/__init__.py create mode 100644 mteb/tasks/SummaryRetrieval/fas/FaMTEBSummaryRetrieval.py create mode 100644 mteb/tasks/SummaryRetrieval/fas/__init__.py create mode 100644 mteb/tasks/aggregated_tasks/CQADupStackRetrieval.py create mode 100644 mteb/tasks/aggregated_tasks/CQADupStackRetrievalFa.py create mode 100644 mteb/tasks/aggregated_tasks/SynPerChatbotConvSAClassification.py create mode 100644 mteb/tasks/aggregated_tasks/__init__.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index be1d1c7418..71ed2554f9 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,11 +4,18 @@ -## Checklist +### Code Quality +- [ ] **Code Formatted**: Format the code using `make lint` to maintain consistent style. -- [ ] Run tests locally to make sure nothing is broken using `make test`. -- [ ] Run the formatter to format the code using `make lint`. +### Documentation + +- [ ] **Updated Documentation**: Add or update documentation to reflect the changes introduced in this PR. + +### Testing + +- [ ] **New Tests Added**: Write tests to cover new functionality. Validate with `make test-with-coverage`. +- [ ] **Tests Passed**: Run tests locally using `make test` or `make test-with-coverage` to ensure no existing functionality is broken. ### Adding datasets checklist diff --git a/.gitignore b/.gitignore index 977fe8dc1a..d5cc51748b 100644 --- a/.gitignore +++ b/.gitignore @@ -147,4 +147,5 @@ results/ uv.lock # model loading tests -model_names.txt \ No newline at end of file +model_names.txt +mteb/leaderboard/__cached_results.json diff --git a/README.md b/README.md index f556cad894..59cc5da9e2 100644 --- a/README.md +++ b/README.md @@ -472,24 +472,24 @@ evaluation.run(model, ...) ## Documentation -| Documentation | | -| ------------------------------ | ---------------------- | -| 📋 [Tasks] | Overview of available tasks | -| 📐 [Benchmarks] | Overview of available benchmarks | -| 📈 [Leaderboard] | The interactive leaderboard of the benchmark | -| 🤖 [Adding a model] | Information related to how to submit a model to the leaderboard | +| Documentation | | +|--------------------------------|-------------------------------------------------------------------------------------| +| 📋 [Tasks] | Overview of available tasks | +| 📐 [Benchmarks] | Overview of available benchmarks | +| 📈 [Leaderboard] | The interactive leaderboard of the benchmark | +| 🤖 [Adding a model] | Information related to how to submit a model to MTEB and to the leaderboard | | 👩‍🔬 [Reproducible workflows] | Information related to how to reproduce and create reproducible workflows with MTEB | -| 👩‍💻 [Adding a dataset] | How to add a new task/dataset to MTEB |  -| 👩‍💻 [Adding a leaderboard tab] | How to add a new leaderboard tab to MTEB |  -| 🤝 [Contributing] | How to contribute to MTEB and set it up for development | -| 🌐 [MMTEB] | An open-source effort to extend MTEB to cover a broad set of languages |   +| 👩‍💻 [Adding a dataset] | How to add a new task/dataset to MTEB | +| 👩‍💻 [Adding a benchmark] | How to add a new benchmark to MTEB and to the leaderboard | +| 🤝 [Contributing] | How to contribute to MTEB and set it up for development | +| 🌐 [MMTEB] | An open-source effort to extend MTEB to cover a broad set of languages | [Tasks]: docs/tasks.md [Benchmarks]: docs/benchmarks.md [Contributing]: CONTRIBUTING.md [Adding a model]: docs/adding_a_model.md [Adding a dataset]: docs/adding_a_dataset.md -[Adding a leaderboard tab]: docs/adding_a_leaderboard_tab.md +[Adding a benchmark]: docs/adding_a_benchmark.md [Leaderboard]: https://huggingface.co/spaces/mteb/leaderboard [MMTEB]: docs/mmteb/readme.md [Reproducible workflows]: docs/reproducible_workflow.md @@ -517,5 +517,6 @@ You may also want to read and cite the amazing work that has extended MTEB & int - Orion Weller, Benjamin Chang, Sean MacAvaney, Kyle Lo, Arman Cohan, Benjamin Van Durme, Dawn Lawrie, Luca Soldaini. "[FollowIR: Evaluating and Teaching Information Retrieval Models to Follow Instructions](https://arxiv.org/abs/2403.15246)" arXiv 2024 - Dawei Zhu, Liang Wang, Nan Yang, Yifan Song, Wenhao Wu, Furu Wei, Sujian Li. "[LongEmbed: Extending Embedding Models for Long Context Retrieval](https://arxiv.org/abs/2404.12096)" arXiv 2024 - Kenneth Enevoldsen, Márton Kardos, Niklas Muennighoff, Kristoffer Laigaard Nielbo. "[The Scandinavian Embedding Benchmarks: Comprehensive Assessment of Multilingual and Monolingual Text Embedding](https://arxiv.org/abs/2406.02396)" arXiv 2024 +- Ali Shiraee Kasmaee, Mohammad Khodadad, Mohammad Arshi Saloot, Nick Sherck, Stephen Dokas, Hamidreza Mahyar, Soheila Samiee. "[ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance & Efficiency on a Specific Domain](https://arxiv.org/abs/2412.00532)" arXiv 2024 For works that have used MTEB for benchmarking, you can find them on the [leaderboard](https://huggingface.co/spaces/mteb/leaderboard). diff --git a/docs/adding_a_benchmark.md b/docs/adding_a_benchmark.md new file mode 100644 index 0000000000..56a042fdb9 --- /dev/null +++ b/docs/adding_a_benchmark.md @@ -0,0 +1,7 @@ +## Adding a benchmark + +The MTEB Leaderboard is available [here](https://huggingface.co/spaces/mteb/leaderboard) and we encourage additions of new benchmarks. To add a new benchmark: + +1. Add your benchmark to [benchmark.py](../mteb/benchmarks/benchmarks.py) as a `Benchmark` object, and select the MTEB tasks that will be in the benchmark. If some of the tasks do not exist in MTEB, follow the "add a dataset" instructions to add them. +2. Open a PR at https://github.com/embedding-benchmark/results with results of models on your benchmark. +3. When PRs are merged, your benchmark will be added to the leaderboard automatically after the next workflow trigger. \ No newline at end of file diff --git a/docs/adding_a_leaderboard_tab.md b/docs/adding_a_leaderboard_tab.md deleted file mode 100644 index 260293ed5c..0000000000 --- a/docs/adding_a_leaderboard_tab.md +++ /dev/null @@ -1,15 +0,0 @@ -## Adding a new Leaderboard tab - -The MTEB Leaderboard is available [here](https://huggingface.co/spaces/mteb/leaderboard) and we love new leaderboard tabs. To add a new leaderboard tab: - -1. Open a PR in https://hf.co/datasets/mteb/results with: -- All results added in existing model folders or new folders -- Updated paths.json (see snippet results.py) -- If adding any new models, their names added to results.py -- If you have access to all models you are adding, you can also [add results via the metadata](https://github.com/embeddings-benchmark/mteb/blob/main/docs/adding_a_model.md) for all of them / some of them -2. Open a PR at https://huggingface.co/spaces/mteb/leaderboard modifying app.py to add your tab: -- Add any new models & their specs to the global lists -- Add your tab, credits etc to where the other tabs are defined -- If you're adding new results to existing models, remove those models from `EXTERNAL_MODEL_RESULTS.json` such that they can be reloaded with the new results and are not cached. -- You may also have to uncomment `, download_mode='force_redownload', verification_mode="no_checks")` where the datasets are loaded to experiment locally without caching of results -- Test that it runs & works locally as you desire with python app.py, **please add screenshots to the PR** diff --git a/docs/adding_a_model.md b/docs/adding_a_model.md index f87d723934..088199e264 100644 --- a/docs/adding_a_model.md +++ b/docs/adding_a_model.md @@ -2,7 +2,63 @@ The MTEB Leaderboard is available [here](https://huggingface.co/spaces/mteb/leaderboard). To submit to it: -1. **Run the desired model on MTEB:** +1. **Add meta information about your model to [model dir](../mteb/models/)**. + ```python + from mteb.model_meta import ModelMeta + + bge_m3 = ModelMeta( + name="model_name", + languages=["model_languages"], # in format eng-Latn + open_weights=True, + revision="5617a9f61b028005a4858fdac845db406aefb181", + release_date="2024-06-28", + n_parameters=568_000_000, + embed_dim=4096, + license="mit", + max_tokens=8194, + reference="https://huggingface.co/BAAI/bge-m3", + similarity_fn_name="cosine", + framework=["Sentence Transformers", "PyTorch"], + use_instructions=False, + public_training_code=None, + public_training_data="https://huggingface.co/datasets/cfli/bge-full-data", + training_datasets={"your_dataset": ["train"]}, + ) + ``` + By default, the model will run using the [`sentence_transformers_loader`](../mteb/models/sentence_transformer_wrapper.py) loader function. If you need to use a custom implementation, you can specify the `loader` parameter in the `ModelMeta` class. For example: + ```python + from mteb.models.wrapper import Wrapper + from mteb.encoder_interface import PromptType + import numpy as np + + class CustomWrapper(Wrapper): + def __init__(self, model_name, model_revision): + super().__init__(model_name, model_revision) + # your custom implementation here + + def encode( + self, + sentences: list[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs + ) -> np.ndarray: + # your custom implementation here + return np.zeros((len(sentences), self.embed_dim)) + ``` + Then you can specify the `loader` parameter in the `ModelMeta` class: + ```python + your_model = ModelMeta( + loader=partial( + CustomWrapper, + model_name="model_name", + model_revision="5617a9f61b028005a4858fdac845db406aefb181" + ), + ... + ) + ``` +2. **Run the desired model on MTEB:** Either use the Python API: @@ -28,64 +84,51 @@ mteb run -m {model_name} -t {task_names} These will save the results in a folder called `results/{model_name}/{model_revision}`. -<<<<<<< HEAD -1. **Format the results using the CLI:** -======= 2. **Push Results to the Leaderboard** To add results to the public leaderboard you can push your results to the [results repository](https://github.com/embeddings-benchmark/results) via a PR. Once merged they will appear on the leaderboard after a day. +3. **Wait for a refresh the leaderboard** -3. (Optional) **Add results to the model card:** - -`mteb` implements a cli for adding results to the model card: ->>>>>>> main - -```bash -mteb create_meta --results_folder results/{model_name}/{model_revision} --output_path model_card.md -``` - -To add the content to the public model simply copy the content of the `model_card.md` file to the top of a `README.md` file of your model on the Hub. See [here](https://huggingface.co/Muennighoff/SGPT-5.8B-weightedmean-msmarco-specb-bitfit/blob/main/README.md) for an example. - -If the readme already exists: - -```bash -mteb create_meta --results_folder results/{model_name}/{model_revision} --output_path model_card.md --from_existing your_existing_readme.md -``` - -<<<<<<< HEAD -2. **Add the frontmatter to model repository:** +**Notes:** -Copy the content of the `model_card.md` file to the top of a `README.md` file of your model on the Hub. See [here](https://huggingface.co/Muennighoff/SGPT-5.8B-weightedmean-msmarco-specb-bitfit/blob/main/README.md) for an example. -======= -Note that running the model on many tasks may lead to a huge readme front matter. ->>>>>>> main +##### Using Prompts with Sentence Transformers -3. **Wait for a refresh the leaderboard:** +If your model uses Sentence Transformers and requires different prompts for encoding the queries and corpus, you can take advantage of the `prompts` [parameter](https://sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer). -The leaderboard [automatically refreshes daily](https://github.com/embeddings-benchmark/leaderboard/commits/main/) so once submitted you only need to wait for the automatic refresh. You can find the workflows for the leaderboard refresh [here](https://github.com/embeddings-benchmark/leaderboard/tree/main/.github/workflows). If you experience issues with the leaderboard please create an [issue](https://github.com/embeddings-benchmark/mteb/issues). +Internally, `mteb` uses `query` for encoding the queries and `passage` as the prompt names for encoding the corpus. This is aligned with the default names used by Sentence Transformers. -**Notes:** -- We remove models with scores that cannot be reproduced, so please ensure that your model is accessible and scores can be reproduced. -<<<<<<< HEAD -- An alternative way of submitting to the leaderboard is by opening a PR with your results [here](https://github.com/embeddings-benchmark/results) & checking that they are displayed correctly by [locally running the leaderboard](https://github.com/embeddings-benchmark/leaderboard?tab=readme-ov-file#developer-setup) -======= ->>>>>>> main +###### Adding the prompts in the model configuration (Preferred) -- ##### Using Prompts with Sentence Transformers - - If your model uses Sentence Transformers and requires different prompts for encoding the queries and corpus, you can take advantage of the `prompts` [parameter](https://sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer). - - Internally, `mteb` uses the prompt named `query` for encoding the queries and `passage` as the prompt name for encoding the corpus. This is aligned with the default names used by Sentence Transformers. +You can directly add the prompts when saving and uploading your model to the Hub. For an example, refer to this [configuration file](https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5/blob/3b5a16eaf17e47bd997da998988dce5877a57092/config_sentence_transformers.json). These prompts can then be specified in the ModelMeta object. - ###### Adding the prompts in the model configuration (Preferred) - You can directly add the prompts when saving and uploading your model to the Hub. For an example, refer to this [configuration file](https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5/blob/3b5a16eaf17e47bd997da998988dce5877a57092/config_sentence_transformers.json). +```python +model = ModelMeta( + loader=partial( # type: ignore + sentence_transformers_loader, + model_name="intfloat/multilingual-e5-small", + revision="fd1525a9fd15316a2d503bf26ab031a61d056e98", + model_prompts={ + "query": "query: ", + "passage": "passage: ", + }, + ), +) +``` +If you are unable to directly add the prompts in the model configuration, you can instantiate the model using the `sentence_transformers_loader` and pass `prompts` as an argument. For more details, see the `mteb/models/bge_models.py` file. - ###### Instantiating the Model with Prompts +##### Adding instruction models -<<<<<<< HEAD - If you are unable to directly add the prompts in the model configuration, you can instantiate the model using the `sentence_transformers_loader` and pass `prompts` as an argument. For more details, see the `mteb/models/bge_models.py` file. -======= - If you are unable to directly add the prompts in the model configuration, you can instantiate the model using the `sentence_transformers_loader` and pass `prompts` as an argument. For more details, see the `mteb/models/bge_models.py` file. ->>>>>>> main +Models that use instructions can use the [`InstructSentenceTransformerWrapper`](../mteb/models/instruct_wrapper.py). For example: +```python +model = ModelMeta( + loader=partial( + InstructSentenceTransformerWrapper, + model="nvidia/NV-Embed-v1", + revision="7604d305b621f14095a1aa23d351674c2859553a", + instruction_template="Instruct: {instruction}\nQuery: ", + ), + ... +) +``` diff --git a/docs/benchmarks.md b/docs/benchmarks.md index a5abe50215..7c0f07d878 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -7,16 +7,27 @@ The following table gives you an overview of the benchmarks in MTEB. | Name | # Tasks | Task Types | Domains | Languages | |------|---------|------------|---------|-----------| -| [CoIR](https://github.com/CoIR-team/coir) | 10 | {'Retrieval': 10} | [Written, Programming] | python,c++,sql,go,eng,php,javascript,ruby,java | -| [MINERSBitextMining](https://arxiv.org/pdf/2406.07424) | 7 | {'BitextMining': 7} | [Written, Social, Reviews] | sun,kaz,tzl,ido,abs,arq,yue,tam,nij,glg,slk,hsb,ber,xho,cbk,pol,uzb,ina,kab,swh,amh,fao,kzj,lfn,uig,sqi,deu,ang,ind,bug,pms,ibo,cym,eus,spa,ceb,tgl,ron,isl,ita,csb,cha,fin,est,pes,jpn,tel,tha,oci,cmn,min,fry,bbc,epo,lit,rus,bos,hrv,war,ara,bjn,mkd,srp,ast,nno,urd,pam,aze,eng,ace,bew,kor,dan,awa,mui,hye,ban,cor,ben,gle,swe,mad,bul,lat,cat,nob,fra,pcm,ell,mar,vie,tat,ukr,gsw,kat,arz,dsb,lvs,nld,tur,bel,max,nds,afr,khm,dtp,yor,ces,gla,zsm,mak,ile,nov,orv,bre,swg,rej,mhr,mon,mal,jav,heb,slv,bhp,kur,wuu,tuk,por,hun,hin,hau,yid | +| [BRIGHT](https://brightbenchmark.github.io/) | 1 | {'Retrieval': 1} | [Non-fiction] | eng | +| [ChemTEB](https://arxiv.org/abs/2412.00532) | 27 | {'BitextMining': 1, 'Classification': 17, 'Clustering': 2, 'PairClassification': 5, 'Retrieval': 2} | [Chemistry] | nld,tur,eng,ces,kor,zho,spa,hin,jpn,deu,fra,msa,por | +| [CoIR](https://github.com/CoIR-team/coir) | 10 | {'Retrieval': 10} | [Written, Programming] | javascript,ruby,sql,go,eng,java,php,python,c++ | +| [LongEmbed](https://arxiv.org/abs/2404.12096v2) | 6 | {'Retrieval': 6} | [Fiction, Academic, Written, Blog, Non-fiction, Spoken, Encyclopaedic] | eng | +| [MINERSBitextMining](https://arxiv.org/pdf/2406.07424) | 7 | {'BitextMining': 7} | [Reviews, Written, Social] | sqi,ban,srp,jpn,nds,lat,por,mon,kur,bul,slv,mak,deu,uzb,yor,kzj,max,kat,cha,yid,zsm,spa,pms,mhr,min,fao,heb,nij,mui,tuk,rus,bew,swe,pes,slk,ceb,bjn,ido,abs,ukr,ina,kab,tgl,cor,dan,kaz,fry,rej,hrv,ces,lfn,glg,dsb,hau,ace,urd,ben,yue,nld,eng,epo,ron,xho,wuu,cmn,ind,ang,hsb,mad,pam,nov,swh,bbc,pcm,ara,hye,mkd,nno,ast,jav,lvs,mal,swg,nob,tat,arz,vie,ile,tam,est,ber,bre,csb,pol,afr,cbk,bug,tzl,kor,ibo,hun,war,aze,tha,mar,uig,gla,orv,hin,amh,bel,sun,fin,cat,awa,gsw,isl,oci,ell,cym,arq,ita,fra,bos,dtp,eus,bhp,tel,tur,khm,lit,gle | +| MTEB(Europe, beta) | 74 | {'BitextMining': 7, 'Classification': 21, 'Clustering': 8, 'Retrieval': 15, 'InstructionRetrieval': 3, 'MultilabelClassification': 2, 'PairClassification': 6, 'Reranking': 3, 'STS': 9} | [Web, Fiction, Social, Academic, Religious, Written, Medical, Blog, Constructed, Non-fiction, Legal, News, Government, Reviews, Spoken, Encyclopaedic, Programming, Subtitles] | qvm,esk,nlg,toj,gup,llg,jpn,azj,for,lav,kmh,por,bsj,tna,upv,cta,smk,zty,qvz,ntj,ton,uvh,cjk,kgf,gaw,bak,seh,jiv,hui,ksr,uli,kwi,qvw,kkl,arl,msk,omw,aai,tet,yby,mva,fao,kgk,min,kac,dji,box,rus,chz,emp,ktm,bps,bon,nus,bss,cut,sue,meq,kpr,rwo,ceb,zaj,mib,aui,apc,kdl,mxb,okv,rai,big,reg,ulk,mlg,yap,tpt,hrv,nak,plu,nde,kyc,arp,hau,ary,alp,apr,caa,mbh,uvl,zat,bjp,urd,bki,lin,mek,hlt,iws,spl,xav,yml,lcm,ese,xho,are,mux,lww,ndg,ntu,tzj,ame,yss,zar,fil,aii,csy,gvs,zpm,amh,spp,ken,avt,ltz,swh,viv,kmk,zul,bqp,cav,wln,leu,tcs,tuf,mkd,clu,msy,too,ast,amx,quf,jav,yre,nhe,tat,lbk,maj,msm,rug,nor,tbc,prf,pad,zlm,kze,wnc,fai,cbs,mai,aoi,mxq,bao,kos,mlh,nep,mkl,roo,umb,poh,bod,nna,aey,afr,aly,cac,maa,aze,fon,tha,mhl,chd,tpi,tzm,acq,kyz,nbq,yle,ape,bco,att,nin,mkj,yuj,ata,djr,atb,enq,cpb,sxb,rmc,zas,guj,kbq,gfk,tgo,acm,cux,fin,npi,etr,tsn,dob,mpt,alq,byx,cak,cso,spy,oci,asm,ttc,nwi,srn,hmn,gyr,hto,ngu,cpa,tif,fuv,kue,yuw,ote,mgw,ssg,bos,mvn,dop,aso,mox,ndj,stp,mpp,nas,kon,mks,caf,mbs,mcd,wap,cco,tod,aon,aom,cnl,srp,zga,lat,sja,kpj,nhi,nko,swp,bho,blw,mih,mon,sna,bgs,als,kyf,kur,bul,uzb,knj,mam,yor,zos,gdr,aka,bam,bmh,gnw,lid,cha,msc,zpl,gun,qxn,zsm,spa,mgh,nca,cpc,quc,hvn,bvr,agu,ngp,aak,jni,mau,sab,wos,huv,swe,kea,tum,pes,som,pbt,mmo,amo,kgp,taq,sbe,mil,nhg,bmu,bvd,wrs,atg,muy,tpa,ign,vmy,uri,chf,cek,knf,pib,soy,boa,ces,xed,pma,hix,kbc,orm,sim,ace,nhw,kud,ppo,xnn,yut,snx,ilo,zaa,nld,bsp,aau,myk,grn,bkq,cme,bbb,ssd,fur,knc,knv,heg,urw,ayr,ons,sat,crx,rop,szl,suz,ncl,anh,kto,tca,chk,xla,qxh,ziw,ntp,azb,ara,tew,sot,cjv,djk,usa,ltg,cap,arz,lmo,vec,jao,wer,dhg,vie,ded,hop,khk,faa,tam,sus,mwc,ikk,kek,mie,trc,tue,ura,crh,bkd,bzj,kwj,klt,sps,jid,xsi,swa,qxo,lim,nqo,hns,tmd,mbt,mbc,ibo,hun,wrk,bnp,abt,kaq,car,kiz,nvm,nfa,gul,guo,uzn,beo,aer,nhy,otm,cjo,tgk,bel,eri,mca,wsk,rro,row,bsn,tpz,fij,tvk,msb,mpx,abx,poy,sgb,kas,tcz,top,dif,awk,cbc,bea,ell,myy,pus,bmr,ssx,pao,ebk,ajp,opm,wnu,gub,acr,tbf,ubr,cth,taj,aby,kde,mqj,zao,khm,hat,gle,azg,cbv,ian,apu,ptp,kbm,met,plt,sag,agd,pag,ydd,ckb,mzz,div,kmg,miz,tac,tuo,gvn,boj,tee,mph,mna,qwh,gng,agg,mle,rgu,haw,med,kyg,mig,nhu,tnc,waj,kat,lua,zpz,kpx,tof,ven,dzo,yaa,bqc,klv,qul,kqw,bef,gai,heb,nuy,zac,mcr,zpc,ssw,meu,tuk,gui,kmo,usp,otq,khs,ksj,xbi,nya,cya,aoj,kmr,grc,sny,snp,mir,piu,geb,tgl,dik,agn,dan,qvn,kaz,kbp,mto,tiy,xon,zav,dww,zap,kqa,lac,kne,wat,cbt,naf,inb,kwf,crn,azz,wim,ben,wro,poi,yue,awb,cgc,eng,mjc,amf,mps,mwe,ncu,cle,tdt,hne,zai,gdn,toc,bhl,kir,ron,fue,kyq,ixl,ghs,ncj,tbz,nnq,mio,kwd,mxp,beu,sbk,fuh,gym,ztq,mey,ikw,pab,kmb,cof,tso,ipi,byr,aia,wiv,agm,npl,ter,hye,iou,tku,nno,cnt,kqc,sll,lvs,gnn,nob,dah,nii,san,wuv,udu,gux,ots,zpq,cuk,mbj,nab,bjz,hbo,imo,mcf,glk,zam,twi,srd,sin,zca,qvc,agr,con,kjs,zaw,mav,gum,dov,ood,soq,tte,msa,chq,cbk,isn,kpf,ptu,mri,cao,aeb,cni,aaz,yon,pan,sgz,rom,mop,gwi,nou,uig,gla,far,atd,hin,tnp,bbr,kpg,huu,arn,jvn,cat,awa,amm,urb,run,mit,pir,gam,adz,tir,isl,pls,mlt,qve,nyu,txu,tbg,dwy,quy,ruf,kiw,shp,amr,ita,maq,dgr,fra,kin,ubu,gof,gaz,mgc,cmo,ctu,tel,eus,mcq,bpr,ino,snd,bgt,mwf,acu,jic,kkc,jac,lit,xtd,dyu,kvn,zyp,prs,cop,auc,wed,apb,sqi,ban,wal,poe,tnk,myu,otn,kje,ong,bkx,zsr,hch,agt,wiu,spm,zpu,scn,sri,myw,buk,kdc,zho,sbs,slv,deu,kqf,kvg,tgp,bhg,dwr,xtm,amu,wbp,tim,ory,tos,kan,kbh,mya,mwp,mcb,shn,bdd,cub,yrb,tbo,yal,lug,tah,txq,emi,hub,nso,slk,zpo,zpv,bmk,nss,bjn,nch,bzd,shj,ukr,mbl,tlf,kab,kew,kpw,luo,cpy,kmu,kup,zab,pri,snc,wbi,acf,gmv,glg,amp,qup,nop,srq,yka,apw,mqb,wmt,bch,ewe,sey,lbb,epo,qvh,taw,fuc,kql,ksd,smo,gvf,cmn,yad,ind,qvs,obo,wmw,nsn,anv,mic,pap,ake,fas,cbr,bjr,glv,mdy,tsw,gvc,noa,bus,bjv,cwe,pon,pio,snn,mal,nho,bba,jae,mxt,wol,nif,ycn,lao,tfr,ffm,qub,hus,bzh,mlp,mti,not,nys,tzo,arb,mos,kam,cuc,dgc,pah,pjt,est,bxh,hot,bre,kms,cot,awx,bjk,pwg,cpu,hla,mpm,fuf,pol,tnn,shi,auy,mpj,tuc,bug,kor,zad,war,ars,rkb,mni,cbu,lif,mar,dad,mee,dgz,mco,kik,apz,mkn,sco,mbb,maz,lij,khz,hmo,guh,sun,cbi,lgl,nhr,tiw,daa,amn,amk,tke,lex,mag,cym,eko,zia,mcp,gah,urt,sua,cab,quh,srm,vid,blz,mmx,apn,tur,rmy,bem,yaq,ctp,cui,lus,tav,cax,yva | +| MTEB(Indic, beta) | 23 | {'BitextMining': 4, 'Clustering': 1, 'Classification': 13, 'STS': 1, 'PairClassification': 1, 'Retrieval': 2, 'Reranking': 1} | [Web, Fiction, Social, Encyclopaedic, Religious, Written, Constructed, Non-fiction, Legal, News, Spoken, Reviews, Government] | ban,pag,ckb,ydd,srp,azj,jpn,bho,por,sna,als,scn,cjk,zho,mwr,bul,slv,deu,yor,bak,ory,aka,bam,kat,lua,kan,dzo,mya,zsm,spa,shn,min,nus,fao,heb,kac,lug,tuk,kea,rus,ssw,tum,swe,nso,pes,slk,som,mup,pbt,nya,ceb,bjn,kmr,apc,taq,ukr,kab,luo,tgl,dik,dan,kaz,kbp,hrv,ces,glg,ary,hau,ace,urd,ben,boy,ewe,ilo,yue,lin,nld,eng,hne,epo,kir,grn,ron,xho,smo,fur,knc,cmn,ind,ayr,sat,szl,pap,fas,kmb,tso,ltz,swh,brx,zul,azb,doi,ara,hye,mkd,nno,ast,jav,lvs,mal,lao,sot,wol,nob,ltg,tat,san,arz,lmo,vec,nor,vie,sag,khk,arb,mos,kam,tam,bgc,mai,gbm,srd,est,twi,crh,sin,nep,swa,umb,bod,pol,lim,nqo,afr,bug,kor,ibo,mri,hun,aeb,war,ars,mni,fon,tha,mar,tpi,tzm,acq,pan,uzn,kik,gla,uig,hin,lij,tgk,amh,bel,sun,acm,guj,fin,cat,awa,fij,npi,run,tsn,kas,tir,isl,asm,mlt,ell,oci,mag,cym,pus,gom,quy,ajp,raj,fuv,ita,kin,bos,fra,gaz,eus,tel,tur,snd,kon,khm,bem,dyu,gle,hat,lit,prs,lus,plt | +| MTEB(Medical) | 12 | {'Retrieval': 9, 'Clustering': 2, 'Reranking': 1} | [Web, Academic, Medical, Written, Non-fiction, Government] | rus,eng,kor,ara,spa,zho,vie,fra,pol,cmn | +| MTEB(Multilingual, beta) | 132 | {'BitextMining': 13, 'Classification': 43, 'Clustering': 17, 'Retrieval': 18, 'InstructionRetrieval': 3, 'MultilabelClassification': 5, 'PairClassification': 11, 'Reranking': 6, 'STS': 16} | [Web, Fiction, Social, Academic, Religious, Written, Medical, Blog, Constructed, Non-fiction, Legal, Government, News, Reviews, Spoken, Encyclopaedic, Programming, Subtitles] | qvm,esk,nlg,toj,gup,llg,jpn,azj,for,lav,kmh,por,bsj,tna,upv,cta,smk,zty,qvz,ntj,ton,uvh,cjk,kgf,gaw,bak,seh,jiv,hui,ksr,uli,kwi,qvw,kkl,arl,msk,omw,aai,tet,yby,mva,fao,kgk,min,kac,dji,mui,box,rus,chz,emp,bew,ktm,bps,bon,nus,bss,cut,sue,meq,kpr,rwo,ceb,zaj,mib,aui,apc,kdl,mxb,okv,rai,big,reg,ulk,mlg,yap,tpt,rej,hrv,nak,plu,nde,lfn,kyc,arp,hau,ary,alp,apr,caa,mbh,uvl,zat,bjp,urd,bki,lin,mek,hlt,iws,spl,xav,yml,lcm,ese,xho,are,mux,lww,ndg,ntu,tzj,ame,yss,zar,fil,aii,csy,gvs,zpm,amh,spp,ken,avt,ltz,swh,viv,kmk,zul,bqp,cav,wln,leu,tcs,tuf,mkd,clu,msy,too,ast,amx,quf,jav,yre,nhe,tat,lbk,maj,msm,rug,nor,tbc,prf,pad,zlm,kze,wnc,fai,cbs,mai,aoi,mxq,bao,kos,mlh,nep,mkl,roo,umb,poh,bod,nna,aey,afr,aly,cac,maa,aze,fon,tha,mhl,chd,tpi,tzm,acq,kyz,nbq,yle,ape,bco,att,nin,mkj,yuj,ata,djr,atb,enq,cpb,sxb,rmc,zas,guj,kbq,gfk,tgo,acm,cux,fin,npi,etr,tsn,dob,mpt,alq,byx,cak,cso,spy,oci,asm,ttc,nwi,srn,hmn,gyr,hto,arq,ngu,cpa,tif,fuv,raj,kue,yuw,ote,mgw,ssg,bos,mvn,dop,aso,mox,ndj,stp,mpp,nas,kon,mks,caf,mbs,mcd,wap,cco,tod,aon,aom,cnl,srp,zga,lat,sja,kpj,nhi,nko,swp,bho,blw,mih,mon,sna,bgs,als,kyf,kur,bul,uzb,knj,mam,yor,zos,gdr,aka,bam,bmh,gnw,lid,cha,msc,zpl,gun,qxn,zsm,spa,mgh,nca,yid,pms,mhr,cpc,quc,hvn,bvr,agu,svk,ngp,aak,jni,mau,sab,wos,huv,swe,kea,tum,pes,som,mup,pbt,mmo,amo,kgp,ido,taq,sbe,mil,nhg,bmu,bvd,wrs,atg,muy,tpa,chv,ign,vmy,cor,uri,fry,chf,cek,knf,pib,soy,boa,ces,xed,pma,hix,kbc,orm,sim,ace,nhw,kud,ppo,xnn,yut,boy,snx,ilo,zaa,nld,bsp,aau,myk,grn,bkq,cme,bbb,ssd,fur,knc,wuu,knv,heg,urw,ayr,ons,sat,crx,ang,hsb,rop,szl,suz,mad,ncl,anh,kto,tca,chk,xla,qxh,brx,ziw,ntp,azb,ara,tew,sot,cjv,djk,usa,ltg,cap,arz,lmo,vec,jao,wer,dhg,vie,ded,hop,khk,faa,tam,bgc,sus,mwc,ikk,kek,mie,trc,tue,ura,crh,ber,bkd,bzj,kwj,klt,sps,jid,xsi,swa,qxo,csb,lim,nqo,hns,tmd,mbt,mbc,ibo,hun,wrk,bnp,abt,kaq,car,kiz,nvm,nfa,gul,guo,uzn,beo,aer,nhy,otm,orv,cjo,tgk,bel,eri,mca,wsk,rro,row,bsn,tpz,fij,tvk,msb,mpx,abx,poy,sgb,kas,tcz,top,dif,awk,cbc,bea,ell,myy,pus,bmr,ssx,pao,ebk,ajp,opm,wnu,gub,acr,max,tbf,ubr,cth,taj,aby,kde,mqj,zao,tyv,khm,hat,gle,azg,cbv,ian,apu,ptp,kbm,met,plt,sag,agd,sah,pag,ydd,ckb,mzz,div,kmg,miz,tac,tuo,gvn,boj,tee,mph,mna,qwh,gng,agg,mle,mak,rgu,haw,med,kyg,mig,nhu,tnc,waj,kat,lua,zpz,kpx,tof,ven,dzo,yaa,bqc,klv,qul,kqw,bef,gai,heb,nuy,zac,mcr,zpc,ssw,meu,tuk,gui,kmo,usp,otq,khs,ksj,xbi,nya,cya,aoj,kmr,grc,sny,snp,mir,piu,geb,tgl,dik,agn,dan,qvn,kaz,kbp,mto,tiy,xon,zav,dww,zap,kqa,lac,kne,wat,cbt,naf,inb,kwf,crn,azz,wim,ben,wro,poi,yue,awb,cgc,eng,mjc,amf,mps,mwe,ncu,cle,tdt,hne,zai,gdn,toc,bhl,kir,ron,fue,kyq,ixl,ghs,ncj,tbz,nnq,mio,kwd,mxp,beu,sbk,fuh,gym,ztq,mey,ikw,pab,pam,kmb,cof,tso,ipi,byr,aia,wiv,pcm,agm,doi,npl,ter,hye,iou,tku,nno,cnt,kqc,sll,lvs,gnn,nob,dah,nii,san,wuv,udu,gux,ots,zpq,cuk,mbj,nab,bjz,hbo,imo,mcf,glk,zam,twi,srd,sin,zca,qvc,agr,con,kjs,zaw,mav,gum,dov,ood,soq,tte,msa,chq,cbk,tzl,isn,kpf,ptu,mri,cao,aeb,cni,aaz,yon,pan,sgz,rom,mop,gwi,nou,uig,gla,far,atd,hin,tnp,bbr,kpg,huu,arn,jvn,cat,awa,amm,urb,run,mit,pir,gam,adz,tir,isl,pls,mlt,gsw,qve,nyu,txu,tbg,dwy,quy,ruf,kiw,shp,amr,ita,maq,dgr,fra,kin,ubu,gof,gaz,mgc,cmo,ctu,tel,eus,mcq,bpr,ino,snd,bgt,mwf,acu,jic,kkc,jac,lit,xtd,dyu,kvn,zyp,prs,cop,auc,wed,apb,sqi,ban,wal,poe,tnk,myu,otn,kje,ong,bkx,zsr,nds,hch,agt,wiu,spm,zpu,scn,sri,myw,buk,kdc,zho,sbs,slv,mwr,deu,kqf,kvg,tgp,bhg,dwr,xtm,amu,wbp,tim,ory,kzj,tos,kan,kbh,mya,mwp,mcb,shn,bdd,cub,yrb,tbo,yal,nij,lug,tah,txq,emi,hub,nso,slk,zpo,zpv,bmk,nss,bjn,nch,abs,bzd,shj,ukr,mbl,ina,tlf,kab,kew,kpw,luo,cpy,kmu,kup,zab,pri,snc,wbi,acf,gmv,glg,dsb,amp,qup,nop,srq,yka,apw,mqb,wmt,bch,ewe,sey,lbb,epo,qvh,taw,fuc,kql,ksd,smo,gvf,cmn,yad,ind,qvs,obo,wmw,nsn,anv,mic,pap,ake,nov,fas,cbr,bjr,glv,mdy,bbc,tsw,gvc,noa,bus,bjv,cwe,pon,pio,snn,swg,mal,nho,bba,jae,mxt,wol,nif,ycn,lao,tfr,ffm,qub,hus,bzh,mlp,mti,not,nys,ile,tzo,arb,mos,kam,cuc,dgc,pah,pjt,gbm,est,bxh,hot,bre,kms,cot,awx,bjk,pwg,cpu,hla,mpm,fuf,pol,tnn,shi,auy,mpj,tuc,bug,kor,zad,war,ars,rkb,mni,cbu,lif,mar,krc,dad,mee,dgz,mco,kik,apz,mkn,sco,mbb,maz,lij,khz,hmo,guh,sun,cbi,lgl,nhr,tiw,daa,amn,amk,tke,lex,mag,cym,gom,eko,zia,mcp,gah,urt,sua,cab,quh,srm,dtp,vid,blz,bhp,mmx,apn,tur,rmy,bem,yaq,ctp,cui,lus,tav,cax,yva | | [MTEB(Retrieval w/Instructions)](https://arxiv.org/abs/2403.15246) | 3 | {'InstructionRetrieval': 3} | [Written, News] | eng | -| [MTEB(Scandinavian)](https://kennethenevoldsen.github.io/scandinavian-embedding-benchmark/) | 28 | {'BitextMining': 2, 'Classification': 13, 'Retrieval': 7, 'Clustering': 6} | [Encyclopaedic, Spoken, Non-fiction, Government, News, Fiction, Social, Blog, Reviews, Written, Web, Legal] | nob,fao,swe,isl,dan,nno | -| MTEB(code) | 12 | {'Retrieval': 12} | [Written, Programming] | python,c++,sql,c,go,eng,shell,typescript,php,scala,rust,swift,javascript,ruby,java | -| [MTEB(deu)](https://arxiv.org/html/2401.02709v1) | 19 | {'Classification': 6, 'Clustering': 4, 'PairClassification': 2, 'Reranking': 1, 'Retrieval': 4, 'STS': 2} | [Encyclopaedic, Spoken, News, Reviews, Written, Web] | eng,deu,pol,fra | -| MTEB(eng) | 67 | {'Classification': 12, 'Retrieval': 26, 'Clustering': 11, 'Reranking': 4, 'STS': 10, 'PairClassification': 3, 'Summarization': 1} | [Encyclopaedic, Spoken, Non-fiction, Blog, News, Medical, Social, Programming, Written, Reviews, Web, Academic] | tur,fra,eng,cmn,pol,ita,nld,spa,deu,ara | -| [MTEB(fra)](https://arxiv.org/abs/2405.20468) | 26 | {'Classification': 6, 'Clustering': 7, 'PairClassification': 2, 'Reranking': 2, 'Retrieval': 5, 'STS': 3, 'Summarization': 1} | [Encyclopaedic, Spoken, Non-fiction, News, Social, Reviews, Written, Web, Legal, Academic] | eng,deu,pol,fra | -| MTEB(kor) | 6 | {'Classification': 1, 'Reranking': 1, 'Retrieval': 2, 'STS': 2} | [Encyclopaedic, Spoken, News, Reviews, Written, Web] | kor | -| [MTEB(law)](https://aclanthology.org/2023.eacl-main.148/) | 8 | {'Retrieval': 8} | [Written, Legal] | eng,deu,zho | -| [MTEB(pol)](https://arxiv.org/abs/2405.10138) | 18 | {'Classification': 7, 'Clustering': 3, 'PairClassification': 4, 'STS': 4} | [Spoken, Non-fiction, News, Fiction, Social, Written, Web, Legal, Academic] | pol,deu,eng,fra | -| [MTEB(rus)](https://aclanthology.org/2023.eacl-main.148/) | 23 | {'Classification': 9, 'Clustering': 3, 'MultilabelClassification': 2, 'PairClassification': 1, 'Reranking': 2, 'Retrieval': 3, 'STS': 3} | [Encyclopaedic, Spoken, Blog, News, Social, Reviews, Written, Web, Academic] | rus | +| [MTEB(Scandinavian)](https://kennethenevoldsen.github.io/scandinavian-embedding-benchmark/) | 28 | {'BitextMining': 2, 'Classification': 13, 'Retrieval': 7, 'Clustering': 6} | [Web, Fiction, Social, Written, Blog, Non-fiction, Legal, News, Spoken, Reviews, Government, Encyclopaedic] | swe,nno,isl,dan,fao,nob | +| MTEB(code) | 12 | {'Retrieval': 12} | [Written, Programming] | javascript,ruby,sql,go,c,eng,shell,typescript,rust,java,php,python,scala,swift,c++ | +| [MTEB(deu)](https://arxiv.org/html/2401.02709v1) | 19 | {'Classification': 6, 'Clustering': 4, 'PairClassification': 2, 'Reranking': 1, 'Retrieval': 4, 'STS': 2} | [Web, Written, News, Spoken, Reviews, Encyclopaedic] | pol,deu,fra,eng | +| MTEB(eng, beta) | 41 | {'Classification': 8, 'Retrieval': 10, 'Clustering': 8, 'Reranking': 2, 'STS': 9, 'PairClassification': 3, 'Summarization': 1} | [Web, Academic, Social, Written, Medical, Blog, Non-fiction, News, Spoken, Reviews, Encyclopaedic, Programming] | nld,tur,eng,ara,spa,ita,deu,fra,pol,cmn | +| MTEB(eng, classic) | 67 | {'Classification': 12, 'Retrieval': 26, 'Clustering': 11, 'Reranking': 4, 'STS': 10, 'PairClassification': 3, 'Summarization': 1} | [Web, Academic, Social, Written, Medical, Blog, Non-fiction, News, Spoken, Reviews, Encyclopaedic, Programming] | nld,tur,eng,ara,spa,ita,deu,fra,pol,cmn | +| [MTEB(fra)](https://arxiv.org/abs/2405.20468) | 26 | {'Classification': 6, 'Clustering': 7, 'PairClassification': 2, 'Reranking': 2, 'Retrieval': 5, 'STS': 3, 'Summarization': 1} | [Web, Academic, Social, Written, Non-fiction, Legal, News, Spoken, Reviews, Encyclopaedic] | pol,deu,fra,eng | +| [MTEB(jpn)](https://github.com/sbintuitions/JMTEB) | 16 | {'Clustering': 2, 'Classification': 4, 'STS': 2, 'PairClassification': 1, 'Retrieval': 6, 'Reranking': 1} | [Web, Academic, Written, Non-fiction, News, Spoken, Reviews, Encyclopaedic] | jpn | +| MTEB(kor) | 6 | {'Classification': 1, 'Reranking': 1, 'Retrieval': 2, 'STS': 2} | [Web, Written, News, Spoken, Reviews, Encyclopaedic] | kor | +| [MTEB(law)](https://aclanthology.org/2023.eacl-main.148/) | 8 | {'Retrieval': 8} | [Written, Legal] | deu,zho,eng | +| [MTEB(pol)](https://arxiv.org/abs/2405.10138) | 18 | {'Classification': 7, 'Clustering': 3, 'PairClassification': 4, 'STS': 4} | [Web, Fiction, Academic, Social, Written, Non-fiction, Legal, News, Spoken] | pol,deu,fra,eng | +| [MTEB(rus)](https://aclanthology.org/2023.eacl-main.148/) | 23 | {'Classification': 9, 'Clustering': 3, 'MultilabelClassification': 2, 'PairClassification': 1, 'Reranking': 2, 'Retrieval': 3, 'STS': 3} | [Web, Social, Academic, Written, Blog, News, Spoken, Reviews, Encyclopaedic] | rus | +| [NanoBEIR](https://huggingface.co/collections/zeta-alpha-ai/nanobeir-66e1a0af21dfd93e620cd9f6) | 13 | {'Retrieval': 13} | [Web, Academic, Social, Medical, Written, Non-fiction, News, Encyclopaedic] | eng | +| [RAR-b](https://arxiv.org/abs/2404.06347) | 17 | {'Retrieval': 17} | [Encyclopaedic, Written, Programming] | eng | \ No newline at end of file diff --git a/docs/create_tasks_table.py b/docs/create_tasks_table.py index 4a1be0cd89..33dca958cb 100644 --- a/docs/create_tasks_table.py +++ b/docs/create_tasks_table.py @@ -50,7 +50,9 @@ def task_to_markdown_row(task: mteb.AbsTask) -> str: f"[{name}]({task.metadata.reference})" if task.metadata.reference else name ) domains = ( - "[" + ", ".join(task.metadata.domains) + "]" if task.metadata.domains else "" + "[" + ", ".join(sorted(task.metadata.domains)) + "]" + if task.metadata.domains + else "" ) n_samples = task.metadata.n_samples dataset_statistics = round_floats_in_dict(task.metadata.descriptive_stats) diff --git a/docs/mmteb/points_table.md b/docs/mmteb/points_table.md index 85978dcc00..dfb4a6b31c 100644 --- a/docs/mmteb/points_table.md +++ b/docs/mmteb/points_table.md @@ -2,206 +2,103 @@ _Note_: this table is **autogenerated** and should not be edited. It is intended to get an overview of contributions. -<<<<<<< HEAD - | GitHub | Paper writing | New dataset | Review PR | Bug fixes | Coordination | Dataset annotations | New task | Running Models | Total | -|:------------------|----------------:|--------------:|------------:|------------:|---------------:|----------------------:|-----------:|-----------------:|--------:| -| KennethEnevoldsen | 0 | 68 | 326 | 87 | 81 | 35 | 0 | 0 | 597 | -| isaac-chung | 12 | 120 | 194 | 50 | 54 | 1 | 2 | 0 | 433 | -| imenelydiaker | 0 | 120 | 144 | 24 | 70 | 0 | 0 | 0 | 358 | -| awinml | 0 | 300 | 2 | 0 | 0 | 0 | 0 | 0 | 302 | -| x-tabdeveloping | 0 | 144 | 32 | 10 | 41 | 0 | 12 | 0 | 239 | -| davidstap | 0 | 176 | 0 | 0 | 0 | 0 | 0 | 0 | 176 | -| jaygala24 | 0 | 149 | 0 | 0 | 0 | 0 | 0 | 0 | 149 | -| wissam-sib | 0 | 134 | 6 | 4 | 0 | 0 | 0 | 0 | 144 | -| Muennighoff | 0 | 0 | 48 | 0 | 70 | 0 | 0 | 24 | 142 | -| orionw | 0 | 0 | 20 | 20 | 75 | 0 | 10 | 0 | 125 | -| dokato | 0 | 94 | 6 | 12 | 0 | 0 | 0 | 0 | 112 | -| gentaiscool | 0 | 110 | 0 | 0 | 0 | 0 | 0 | 0 | 110 | -| jupyterjazz | 0 | 108 | 0 | 0 | 0 | 0 | 0 | 0 | 108 | -| SaitejaUtpala | 0 | 102 | 0 | 0 | 0 | 0 | 0 | 0 | 102 | -| vaibhavad | 0 | 6 | 4 | 8 | 75 | 0 | 0 | 0 | 93 | -| schmarion | 0 | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | -| MathieuCiancone | 0 | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | -| GabrielSequeira | 0 | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | -| digantamisra98 | 0 | 71 | 0 | 0 | 0 | 0 | 0 | 0 | 71 | -| shreeya-dhakal | 0 | 54 | 8 | 0 | 0 | 0 | 0 | 0 | 62 | -| Rysias | 0 | 58 | 0 | 0 | 0 | 0 | 0 | 0 | 58 | -| Samoed | 0 | 18 | 2 | 22 | 0 | 0 | 0 | 9 | 51 | -| sivareddyg | 0 | 0 | 0 | 0 | 50 | 0 | 0 | 0 | 50 | -| gowitheflow-1998 | 0 | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 50 | -| asparius | 0 | 34 | 14 | 0 | 0 | 0 | 0 | 0 | 48 | -| Akash190104 | 0 | 46 | 0 | 0 | 0 | 0 | 0 | 0 | 46 | -| MartinBernstorff | 0 | 2 | 8 | 13 | 20 | 0 | 0 | 0 | 43 | -| akshita-sukhlecha | 0 | 36 | 0 | 4 | 0 | 0 | 0 | 0 | 40 | -| staoxiao | 0 | 40 | 0 | 0 | 0 | 0 | 0 | 0 | 40 | -| bp-high | 0 | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | -| rafalposwiata | 0 | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | -| KranthiGV | 0 | 20 | 14 | 0 | 0 | 0 | 0 | 0 | 34 | -| loicmagne | 0 | 0 | 0 | 28 | 0 | 0 | 0 | 0 | 28 | -| ShawonAshraf | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| bjoernpl | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| jphme | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| rasdani | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| violenil | 0 | 26 | 0 | 0 | 0 | 0 | 0 | 0 | 26 | -| mariyahendriksen | 24 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 24 | -| dwzhu-pku | 0 | 24 | 0 | 0 | 0 | 0 | 0 | 0 | 24 | -| hgissbkh | 3 | 0 | 2 | 13 | 0 | 0 | 5 | 0 | 23 | -| taeminlee | 0 | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | -| kwojtasi | 0 | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | -| jankounchained | 0 | 14 | 0 | 8 | 0 | 0 | 0 | 0 | 22 | -| tomaarsen | 0 | 0 | 2 | 0 | 20 | 0 | 0 | 0 | 22 | -| crystina-z | 0 | 21 | 0 | 0 | 0 | 0 | 0 | 0 | 21 | -| mrshu | 0 | 16 | 4 | 0 | 0 | 1 | 0 | 0 | 21 | -| john-b-yang | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | -| rbroc | 0 | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | -| mmhamdy | 0 | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | -| ManuelFay | 0 | 2 | 0 | 13 | 0 | 0 | 5 | 0 | 20 | -| AlexeyVatolin | 0 | 0 | 0 | 20 | 0 | 0 | 0 | 0 | 20 | -| Andrian0s | 0 | 14 | 4 | 2 | 0 | 0 | 0 | 0 | 20 | -| thakur-nandan | 0 | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | -| manandey | 0 | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | -| PranjalChitale | 0 | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | -| dipam7 | 0 | 14 | 2 | 0 | 0 | 0 | 0 | 0 | 16 | -| sted97 | 0 | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | -| Sakshamrzt | 0 | 12 | 4 | 0 | 0 | 0 | 0 | 0 | 16 | -| taidnguyen | 0 | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 14 | -| artemsnegirev | 0 | 12 | 0 | 0 | 0 | 2 | 0 | 0 | 14 | -| slvnwhrl | 0 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | -| anpalmak2003 | 0 | 9 | 0 | 0 | 0 | 3 | 0 | 0 | 12 | -| Art3mis07 | 0 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | -| guenthermi | 0 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | -| jordiclive | 0 | 2 | 0 | 10 | 0 | 0 | 0 | 0 | 12 | -| xhluca | 0 | 6 | 2 | 4 | 0 | 0 | 0 | 0 | 12 | -| henilp105 | 0 | 0 | 0 | 2 | 0 | 9 | 0 | 0 | 11 | -| MariyaTikhonova | 0 | 7 | 0 | 0 | 0 | 4 | 0 | 0 | 11 | -| ab1992ao | 0 | 8 | 0 | 0 | 0 | 3 | 0 | 0 | 11 | -| tmp_handle | 0 | 0 | 0 | 0 | 10 | 0 | 0 | 0 | 10 | -| swj0419 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| Ruqyai | 0 | 2 | 8 | 0 | 0 | 0 | 0 | 0 | 10 | -| ZhengLiu101 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| Alenush | 0 | 6 | 0 | 0 | 0 | 4 | 0 | 0 | 10 | -| ABorghini | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| simon-clematide | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| sarahooker | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| guangyusong | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| HLasse | 0 | 0 | 0 | 5 | 0 | 5 | 0 | 0 | 10 | -| cassanof | 0 | 8 | 0 | 1 | 0 | 0 | 0 | 1 | 10 | -| hongjin-su | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| xiamengzhou | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| xu3kev | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| howard-yen | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| malteos | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| ljvmiranda921 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| marcobellagente93 | 0 | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| izhx | 0 | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| MexicanLemonade | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| antoniolanza1996 | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 2 | -| achibb | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| NouamaneTazi | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | -| PhilipMay | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | -| cslizc | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| bakrianoo | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| hanhainebula | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| monikernemo | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -======= - | GitHub | New dataset | Review PR | Bug fixes | Coordination | Paper writing | Dataset annotations | Running Models | New task | Total | -|:------------------|--------------:|------------:|------------:|---------------:|----------------:|----------------------:|-----------------:|-----------:|--------:| -| KennethEnevoldsen | 68 | 326 | 87 | 81 | 0 | 35 | 0 | 0 | 597 | -| isaac-chung | 120 | 194 | 50 | 54 | 12 | 1 | 0 | 2 | 433 | -| imenelydiaker | 120 | 144 | 24 | 70 | 0 | 0 | 0 | 0 | 358 | -| awinml | 300 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 302 | -| x-tabdeveloping | 144 | 32 | 10 | 41 | 0 | 0 | 0 | 12 | 239 | -| davidstap | 176 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 176 | -| jaygala24 | 149 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 149 | -| wissam-sib | 134 | 6 | 4 | 0 | 0 | 0 | 0 | 0 | 144 | -| Muennighoff | 0 | 48 | 0 | 70 | 0 | 0 | 24 | 0 | 142 | -| orionw | 0 | 20 | 20 | 75 | 0 | 0 | 0 | 10 | 125 | -| dokato | 94 | 6 | 12 | 0 | 0 | 0 | 0 | 0 | 112 | -| gentaiscool | 110 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 110 | -| jupyterjazz | 108 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 108 | -| SaitejaUtpala | 102 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 102 | -| vaibhavad | 6 | 4 | 8 | 75 | 0 | 0 | 0 | 0 | 93 | -| schmarion | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | -| MathieuCiancone | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | -| GabrielSequeira | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | -| digantamisra98 | 71 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 71 | -| shreeya-dhakal | 54 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 62 | -| Rysias | 58 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 58 | -| Samoed | 18 | 2 | 22 | 0 | 0 | 0 | 9 | 0 | 51 | -| sivareddyg | 0 | 0 | 0 | 50 | 0 | 0 | 0 | 0 | 50 | -| gowitheflow-1998 | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 50 | -| asparius | 34 | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 48 | -| Akash190104 | 46 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 46 | -| MartinBernstorff | 2 | 8 | 13 | 20 | 0 | 0 | 0 | 0 | 43 | -| akshita-sukhlecha | 36 | 0 | 4 | 0 | 0 | 0 | 0 | 0 | 40 | -| staoxiao | 40 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 40 | -| bp-high | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | -| rafalposwiata | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | -| KranthiGV | 20 | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 34 | -| loicmagne | 0 | 0 | 28 | 0 | 0 | 0 | 0 | 0 | 28 | -| ShawonAshraf | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| bjoernpl | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| jphme | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| rasdani | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | -| violenil | 26 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 26 | -| mariyahendriksen | 0 | 0 | 0 | 0 | 24 | 0 | 0 | 0 | 24 | -| dwzhu-pku | 24 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 24 | -| hgissbkh | 0 | 2 | 13 | 0 | 3 | 0 | 0 | 5 | 23 | -| taeminlee | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | -| kwojtasi | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | -| jankounchained | 14 | 0 | 8 | 0 | 0 | 0 | 0 | 0 | 22 | -| tomaarsen | 0 | 2 | 0 | 20 | 0 | 0 | 0 | 0 | 22 | -| crystina-z | 21 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 21 | -| mrshu | 16 | 4 | 0 | 0 | 0 | 1 | 0 | 0 | 21 | -| john-b-yang | 0 | 0 | 0 | 0 | 20 | 0 | 0 | 0 | 20 | -| rbroc | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | -| mmhamdy | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | -| ManuelFay | 2 | 0 | 13 | 0 | 0 | 0 | 0 | 5 | 20 | -| AlexeyVatolin | 0 | 0 | 20 | 0 | 0 | 0 | 0 | 0 | 20 | -| Andrian0s | 14 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 20 | -| thakur-nandan | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | -| manandey | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | -| PranjalChitale | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | -| dipam7 | 14 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | -| sted97 | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | -| Sakshamrzt | 12 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | -| taidnguyen | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 14 | -| artemsnegirev | 12 | 0 | 0 | 0 | 0 | 2 | 0 | 0 | 14 | -| slvnwhrl | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | -| anpalmak2003 | 9 | 0 | 0 | 0 | 0 | 3 | 0 | 0 | 12 | -| Art3mis07 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | -| guenthermi | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | -| jordiclive | 2 | 0 | 10 | 0 | 0 | 0 | 0 | 0 | 12 | -| xhluca | 6 | 2 | 4 | 0 | 0 | 0 | 0 | 0 | 12 | -| henilp105 | 0 | 0 | 2 | 0 | 0 | 9 | 0 | 0 | 11 | -| MariyaTikhonova | 7 | 0 | 0 | 0 | 0 | 4 | 0 | 0 | 11 | -| ab1992ao | 8 | 0 | 0 | 0 | 0 | 3 | 0 | 0 | 11 | -| tmp_handle | 0 | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 10 | -| swj0419 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| Ruqyai | 2 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| ZhengLiu101 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| Alenush | 6 | 0 | 0 | 0 | 0 | 4 | 0 | 0 | 10 | -| ABorghini | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| simon-clematide | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| sarahooker | 0 | 0 | 0 | 0 | 10 | 0 | 0 | 0 | 10 | -| guangyusong | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| HLasse | 0 | 0 | 5 | 0 | 0 | 5 | 0 | 0 | 10 | -| cassanof | 8 | 0 | 1 | 0 | 0 | 0 | 1 | 0 | 10 | -| hongjin-su | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| xiamengzhou | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| xu3kev | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| howard-yen | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| malteos | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| ljvmiranda921 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | -| marcobellagente93 | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| izhx | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| MexicanLemonade | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| antoniolanza1996 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | -| achibb | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| NouamaneTazi | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| PhilipMay | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| cslizc | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| bakrianoo | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| hanhainebula | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| monikernemo | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | ->>>>>>> main + | GitHub | New dataset | Review PR | Running Models | Bug fixes | Coordination | Dataset annotations | Paper writing | New task | Total | +|:------------------|--------------:|------------:|-----------------:|------------:|---------------:|----------------------:|----------------:|-----------:|--------:| +| KennethEnevoldsen | 68 | 326 | 0 | 87 | 81 | 35 | 0 | 0 | 597 | +| isaac-chung | 120 | 194 | 0 | 50 | 54 | 1 | 12 | 2 | 433 | +| imenelydiaker | 120 | 144 | 0 | 24 | 70 | 0 | 0 | 0 | 358 | +| awinml | 300 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 302 | +| x-tabdeveloping | 144 | 32 | 0 | 10 | 41 | 0 | 0 | 12 | 239 | +| davidstap | 176 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 176 | +| jaygala24 | 149 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 149 | +| wissam-sib | 134 | 6 | 0 | 4 | 0 | 0 | 0 | 0 | 144 | +| Muennighoff | 0 | 48 | 24 | 0 | 70 | 0 | 0 | 0 | 142 | +| orionw | 0 | 20 | 0 | 20 | 75 | 0 | 0 | 10 | 125 | +| dokato | 94 | 6 | 0 | 12 | 0 | 0 | 0 | 0 | 112 | +| gentaiscool | 110 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 110 | +| jupyterjazz | 108 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 108 | +| SaitejaUtpala | 102 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 102 | +| vaibhavad | 6 | 4 | 0 | 8 | 75 | 0 | 0 | 0 | 93 | +| schmarion | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| MathieuCiancone | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| GabrielSequeira | 88 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 88 | +| digantamisra98 | 71 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 71 | +| shreeya-dhakal | 54 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 62 | +| Rysias | 58 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 58 | +| Samoed | 18 | 2 | 9 | 22 | 0 | 0 | 0 | 0 | 51 | +| sivareddyg | 0 | 0 | 0 | 0 | 50 | 0 | 0 | 0 | 50 | +| gowitheflow-1998 | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 50 | +| asparius | 34 | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 48 | +| Akash190104 | 46 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 46 | +| MartinBernstorff | 2 | 8 | 0 | 13 | 20 | 0 | 0 | 0 | 43 | +| akshita-sukhlecha | 36 | 0 | 0 | 4 | 0 | 0 | 0 | 0 | 40 | +| staoxiao | 40 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 40 | +| bp-high | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | +| rafalposwiata | 36 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 36 | +| KranthiGV | 20 | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 34 | +| loicmagne | 0 | 0 | 0 | 28 | 0 | 0 | 0 | 0 | 28 | +| ShawonAshraf | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| bjoernpl | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| jphme | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| rasdani | 28 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 28 | +| violenil | 26 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 26 | +| mariyahendriksen | 0 | 0 | 0 | 0 | 0 | 0 | 24 | 0 | 24 | +| dwzhu-pku | 24 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 24 | +| hgissbkh | 0 | 2 | 0 | 13 | 0 | 0 | 3 | 5 | 23 | +| taeminlee | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | +| kwojtasi | 22 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 22 | +| jankounchained | 14 | 0 | 0 | 8 | 0 | 0 | 0 | 0 | 22 | +| tomaarsen | 0 | 2 | 0 | 0 | 20 | 0 | 0 | 0 | 22 | +| crystina-z | 21 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 21 | +| mrshu | 16 | 4 | 0 | 0 | 0 | 1 | 0 | 0 | 21 | +| john-b-yang | 0 | 0 | 0 | 0 | 0 | 0 | 20 | 0 | 20 | +| rbroc | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | +| mmhamdy | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 20 | +| ManuelFay | 2 | 0 | 0 | 13 | 0 | 0 | 0 | 5 | 20 | +| AlexeyVatolin | 0 | 0 | 0 | 20 | 0 | 0 | 0 | 0 | 20 | +| Andrian0s | 14 | 4 | 0 | 2 | 0 | 0 | 0 | 0 | 20 | +| thakur-nandan | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | +| manandey | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 18 | +| PranjalChitale | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| dipam7 | 14 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| sted97 | 16 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| Sakshamrzt | 12 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 16 | +| taidnguyen | 14 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 14 | +| artemsnegirev | 12 | 0 | 0 | 0 | 0 | 2 | 0 | 0 | 14 | +| slvnwhrl | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| anpalmak2003 | 9 | 0 | 0 | 0 | 0 | 3 | 0 | 0 | 12 | +| Art3mis07 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| guenthermi | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 12 | +| jordiclive | 2 | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 12 | +| xhluca | 6 | 2 | 0 | 4 | 0 | 0 | 0 | 0 | 12 | +| henilp105 | 0 | 0 | 0 | 2 | 0 | 9 | 0 | 0 | 11 | +| MariyaTikhonova | 7 | 0 | 0 | 0 | 0 | 4 | 0 | 0 | 11 | +| ab1992ao | 8 | 0 | 0 | 0 | 0 | 3 | 0 | 0 | 11 | +| tmp_handle | 0 | 0 | 0 | 0 | 10 | 0 | 0 | 0 | 10 | +| swj0419 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| Ruqyai | 2 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| ZhengLiu101 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| Alenush | 6 | 0 | 0 | 0 | 0 | 4 | 0 | 0 | 10 | +| ABorghini | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| simon-clematide | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| sarahooker | 0 | 0 | 0 | 0 | 0 | 0 | 10 | 0 | 10 | +| guangyusong | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| HLasse | 0 | 0 | 0 | 5 | 0 | 5 | 0 | 0 | 10 | +| cassanof | 8 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 10 | +| hongjin-su | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| xiamengzhou | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| xu3kev | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| howard-yen | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| malteos | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| ljvmiranda921 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| marcobellagente93 | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| izhx | 6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| MexicanLemonade | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| antoniolanza1996 | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 2 | +| achibb | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| NouamaneTazi | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| PhilipMay | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| cslizc | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| bakrianoo | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| hanhainebula | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| monikernemo | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | \ No newline at end of file diff --git a/docs/tasks.md b/docs/tasks.md index d4e6b376ad..bc9a4e99a4 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -12,38 +12,17 @@ The following tables give you an overview of the tasks in MTEB. | [AILAStatutes](https://zenodo.org/records/4063986) | ['eng'] | Retrieval | p2p | [Legal, Written] | None | None | | [AJGT](https://link.springer.com/chapter/10.1007/978-3-319-60042-0_66/) (Alomari et al., 2017) | ['ara'] | Classification | s2s | [Social, Written] | None | None | | [ARCChallenge](https://allenai.org/data/arc) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [AROCocoOrder](https://proceedings.neurips.cc/paper_files/paper/2023/hash/63461de0b4cb760fc498e85b18a7fe81-Abstract-Datasets_and_Benchmarks.html) (Hsieh et al., 2024) | ['eng'] | ImageTextPairClassification | i2t | [Encyclopaedic] | None | None | +| [AROFlickrOrder](https://proceedings.neurips.cc/paper_files/paper/2023/hash/63461de0b4cb760fc498e85b18a7fe81-Abstract-Datasets_and_Benchmarks.html) (Hsieh et al., 2024) | ['eng'] | ImageTextPairClassification | i2t | [Encyclopaedic] | None | None | +| [AROVisualAttribution](https://openreview.net/forum?id=KRLUvxh8uaX) (Yuksekgonul et al., 2023) | ['eng'] | ImageTextPairClassification | i2t | [Encyclopaedic] | None | None | +| [AROVisualRelation](https://openreview.net/forum?id=KRLUvxh8uaX) (Yuksekgonul et al., 2023) | ['eng'] | ImageTextPairClassification | i2t | [Encyclopaedic] | None | None | | [ATEC](https://aclanthology.org/2021.emnlp-main.357) | ['cmn'] | STS | s2s | | None | None | -<<<<<<< HEAD -| [AfriSentiClassification](https://arxiv.org/abs/2302.08956) | ['amh', 'arq', 'ary', 'hau', 'ibo', 'kin', 'pcm', 'por', 'swa', 'tso', 'twi', 'yor'] | Classification | s2s | [Social, Written] | {'test': 2048} | {'test': 74.77} | -| [AfriSentiLangClassification](https://huggingface.co/datasets/HausaNLP/afrisenti-lid-data/) | ['amh', 'arq', 'ary', 'hau', 'ibo', 'kin', 'pcm', 'por', 'swa', 'tso', 'twi', 'yor'] | Classification | s2s | [Social, Written] | {'test': 5754} | {'test': 77.84} | -| [AllegroReviews](https://aclanthology.org/2020.acl-main.111.pdf) | ['pol'] | Classification | s2s | | {'test': 1006} | {'test': 477.2} | -| [AlloProfClusteringP2P.v2](https://huggingface.co/datasets/lyon-nlp/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Clustering | p2p | [Encyclopaedic, Written] | {'test': 2556} | {'test': 3539.5} | -| [AlloProfClusteringS2S.v2](https://huggingface.co/datasets/lyon-nlp/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Clustering | s2s | [Encyclopaedic, Written] | {'test': 2556} | {'test': 32.8} | -| [AlloprofReranking](https://huggingface.co/datasets/antoinelb7/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Reranking | s2p | [Web, Academic, Written] | {'test': 2316, 'train': 9264} | None | -| [AlloprofRetrieval](https://huggingface.co/datasets/antoinelb7/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Retrieval | s2p | [Encyclopaedic, Written] | {'train': 2048} | {'test': {'average_document_length': 3505.705399061033, 'average_query_length': 170.71286701208982, 'num_documents': 2556, 'num_queries': 2316, 'average_relevant_docs_per_query': 1.0}} | -| [AlphaNLI](https://leaderboard.allenai.org/anli/submissions/get-started) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 1532} | {'test': {'average_document_length': 43.42647308646886, 'average_query_length': 103.05483028720627, 'num_documents': 241347, 'num_queries': 1532, 'average_relevant_docs_per_query': 1.0}} | -| [AmazonCounterfactualClassification](https://arxiv.org/abs/2104.06893) | ['deu', 'eng', 'jpn'] | Classification | s2s | [Reviews, Written] | {'validation': 335, 'test': 670} | {'validation': 109.2, 'test': 106.1} | -| [AmazonPolarityClassification](https://huggingface.co/datasets/amazon_polarity) (Julian McAuley, 2013) | ['eng'] | Classification | p2p | [Reviews, Written] | {'test': 400000} | {'test': 431.4} | -| [AmazonReviewsClassification](https://arxiv.org/abs/2010.02573) (Phillip Keung, 2020) | ['cmn', 'deu', 'eng', 'fra', 'jpn', 'spa'] | Classification | s2s | [Reviews, Written] | {'validation': 30000, 'test': 30000} | {'validation': 159.2, 'test': 160.4} | -| [AngryTweetsClassification](https://aclanthology.org/2021.nodalida-main.53/) (Pauli et al., 2021) | ['dan'] | Classification | s2s | [Social, Written] | {'test': 1050} | {'test': 156.1} | -| [AppsRetrieval](https://arxiv.org/abs/2105.09938) (Dan Hendrycks, 2021) | ['eng', 'python'] | Retrieval | p2p | [Programming, Written] | {'test': 1000} | {'test': {'average_document_length': 575.0086708499715, 'average_query_length': 1669.8284196547145, 'num_documents': 8765, 'num_queries': 3765, 'average_relevant_docs_per_query': 1.0}} | -| [ArEntail](https://link.springer.com/article/10.1007/s10579-024-09731-1) (Obeidat et al., 2024) | ['ara'] | PairClassification | s2s | [News, Written] | {'test': 1000} | {'test': 65.77} | -| [ArXivHierarchicalClusteringP2P](https://www.kaggle.com/Cornell-University/arxiv) | ['eng'] | Clustering | p2p | [Academic, Written] | {'test': 2048} | {'test': {'num_samples': 2048, 'average_text_length': 1008.439453125, 'average_labels_per_text': 1.46337890625, 'unique_labels': 129, 'labels': {'cs': {'count': 356}, 'math': {'count': 381}, 'OC': {'count': 11}, 'hep-lat': {'count': 13}, 'hep': {'count': 98}, 'astro-ph': {'count': 213}, 'eess': {'count': 76}, 'quant-ph': {'count': 135}, 'DC': {'count': 5}, 'cond-mat': {'count': 274}, 'hep-th': {'count': 66}, 'SP': {'count': 33}, 'hep-ph': {'count': 69}, 'FA': {'count': 6}, 'nucl-th': {'count': 17}, 'q-bio': {'count': 80}, 'HE': {'count': 22}, 'HC': {'count': 2}, 'stat': {'count': 60}, 'ML': {'count': 16}, 'IV': {'count': 13}, 'stat-mech': {'count': 47}, 'DS': {'count': 14}, 'ME': {'count': 12}, 'CC': {'count': 2}, 'mtrl-sci': {'count': 22}, 'PE': {'count': 16}, 'NT': {'count': 11}, 'SC': {'count': 6}, 'AG': {'count': 13}, 'physics': {'count': 81}, 'ins-det': {'count': 9}, 'GA': {'count': 18}, 'BM': {'count': 6}, 'GN': {'count': 17}, 'NA': {'count': 15}, 'app-ph': {'count': 7}, 'RT': {'count': 6}, 'other': {'count': 37}, 'soft': {'count': 15}, 'CO': {'count': 33}, 'supr-con': {'count': 21}, 'chem-ph': {'count': 3}, 'DM': {'count': 2}, 'MN': {'count': 12}, 'q-fin': {'count': 27}, 'PM': {'count': 2}, 'AP': {'count': 27}, 'gr-qc': {'count': 15}, 'quant-gas': {'count': 8}, 'mes-hall': {'count': 33}, 'IT': {'count': 19}, 'SI': {'count': 6}, 'SG': {'count': 3}, 'bio-ph': {'count': 2}, 'SR': {'count': 16}, 'soc-ph': {'count': 5}, 'hep-ex': {'count': 15}, 'DG': {'count': 11}, 'NE': {'count': 5}, 'CR': {'count': 6}, 'CL': {'count': 12}, 'RM': {'count': 3}, 'econ': {'count': 17}, 'nlin': {'count': 5}, 'PS': {'count': 1}, 'LG': {'count': 26}, 'QA': {'count': 9}, 'str-el': {'count': 26}, 'CV': {'count': 34}, 'MF': {'count': 6}, 'IM': {'count': 7}, 'EM': {'count': 6}, 'TH': {'count': 5}, 'PR': {'count': 20}, 'AT': {'count': 4}, 'OA': {'count': 4}, 'CP': {'count': 6}, 'LO': {'count': 14}, 'flu-dyn': {'count': 6}, 'atom-ph': {'count': 8}, 'class-ph': {'count': 1}, 'SY': {'count': 20}, 'IR': {'count': 1}, 'plasm-ph': {'count': 8}, 'CE': {'count': 2}, 'AO': {'count': 1}, 'comp-ph': {'count': 3}, 'optics': {'count': 12}, 'MG': {'count': 4}, 'ST': {'count': 6}, 'nucl-ex': {'count': 6}, 'CY': {'count': 9}, 'ao-ph': {'count': 2}, 'DB': {'count': 1}, 'math-ph': {'count': 10}, 'NC': {'count': 13}, 'GT': {'count': 11}, 'TO': {'count': 2}, 'AI': {'count': 9}, 'NI': {'count': 2}, 'gen-ph': {'count': 4}, 'OT': {'count': 4}, 'SD': {'count': 2}, 'dis-nn': {'count': 4}, 'RO': {'count': 7}, 'CA': {'count': 6}, 'FL': {'count': 1}, 'SE': {'count': 5}, 'EP': {'count': 9}, 'hist-ph': {'count': 1}, 'QM': {'count': 9}, 'ed-ph': {'count': 2}, 'GR': {'count': 4}, 'MS': {'count': 1}, 'CD': {'count': 1}, 'ET': {'count': 1}, 'acc-ph': {'count': 5}, 'AC': {'count': 2}, 'OH': {'count': 1}, 'EC': {'count': 2}, 'DL': {'count': 1}, 'AS': {'count': 3}, 'geo-ph': {'count': 2}, 'CG': {'count': 3}, 'CB': {'count': 1}, 'AR': {'count': 1}, 'TR': {'count': 1}, 'atm-clus': {'count': 1}}}} | -| [ArXivHierarchicalClusteringS2S](https://www.kaggle.com/Cornell-University/arxiv) | ['eng'] | Clustering | p2p | [Academic, Written] | {'test': 2048} | {'test': 1009.98} | -| [ArguAna](http://argumentation.bplaced.net/arguana/data) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | [Medical, Written] | None | {'test': {'average_document_length': 1029.2327645838136, 'average_query_length': 1192.7204836415362, 'num_documents': 8674, 'num_queries': 1406, 'average_relevant_docs_per_query': 1.0}} | -| [ArguAna-PL](https://huggingface.co/datasets/clarin-knext/arguana-pl) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1060.702674659903, 'average_query_length': 1224.8022759601706, 'num_documents': 8674, 'num_queries': 1406, 'average_relevant_docs_per_query': 1.0}} | -| [ArmenianParaphrasePC](https://github.com/ivannikov-lab/arpa-paraphrase-corpus) (Arthur Malajyan, 2020) | ['hye'] | PairClassification | s2s | [News, Written] | {'train': 4023, 'test': 1470} | {'train': 243.81, 'test': 241.37} | -| [ArxivClassification](https://ieeexplore.ieee.org/document/8675939) (He et al., 2019) | ['eng'] | Classification | s2s | [Academic, Written] | {'test': 2048} | {} | -| [AskUbuntuDupQuestions](https://github.com/taolei87/askubuntu) | ['eng'] | Reranking | s2s | | {'test': 2255} | {'test': {'num_samples': 375, 'num_positive': 375, 'num_negative': 375, 'avg_query_len': 50.205333333333336, 'avg_positive_len': 6.013333333333334, 'avg_negative_len': 13.986666666666666}} | -| [Assin2RTE](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) (Real et al., 2020) | ['por'] | PairClassification | s2s | [Written] | {'test': 2448} | {'test': 53.55} | -| [Assin2STS](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) (Real et al., 2020) | ['por'] | STS | s2s | [Written] | {'test': 2448} | {'test': 53.55} | -======= | [AfriSentiClassification](https://arxiv.org/abs/2302.08956) | ['amh', 'arq', 'ary', 'hau', 'ibo', 'kin', 'pcm', 'por', 'swa', 'tso', 'twi', 'yor'] | Classification | s2s | [Social, Written] | None | None | | [AfriSentiLangClassification](https://huggingface.co/datasets/HausaNLP/afrisenti-lid-data/) | ['amh', 'arq', 'ary', 'hau', 'ibo', 'kin', 'pcm', 'por', 'swa', 'tso', 'twi', 'yor'] | Classification | s2s | [Social, Written] | None | None | | [AllegroReviews](https://aclanthology.org/2020.acl-main.111.pdf) | ['pol'] | Classification | s2s | | None | None | | [AlloProfClusteringP2P.v2](https://huggingface.co/datasets/lyon-nlp/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Clustering | p2p | [Encyclopaedic, Written] | None | None | | [AlloProfClusteringS2S.v2](https://huggingface.co/datasets/lyon-nlp/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Clustering | s2s | [Encyclopaedic, Written] | None | None | -| [AlloprofReranking](https://huggingface.co/datasets/antoinelb7/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Reranking | s2p | [Web, Academic, Written] | None | None | +| [AlloprofReranking](https://huggingface.co/datasets/antoinelb7/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Reranking | s2p | [Academic, Web, Written] | None | None | | [AlloprofRetrieval](https://huggingface.co/datasets/antoinelb7/alloprof) (Lefebvre-Brossard et al., 2023) | ['fra'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | | [AlphaNLI](https://leaderboard.allenai.org/anli/submissions/get-started) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | | [AmazonCounterfactualClassification](https://arxiv.org/abs/2104.06893) | ['deu', 'eng', 'jpn'] | Classification | s2s | [Reviews, Written] | None | None | @@ -55,229 +34,86 @@ The following tables give you an overview of the tasks in MTEB. | [ArXivHierarchicalClusteringP2P](https://www.kaggle.com/Cornell-University/arxiv) | ['eng'] | Clustering | p2p | [Academic, Written] | {'test': 2048} | {'test': {'num_samples': 2048, 'number_of_characters': 2065284, 'min_text_length': 103, 'average_text_length': 1008.44, 'max_text_length': 2103, 'min_labels_per_text': 1, 'average_labels_per_text': 1.46, 'max_labels_per_text': 381, 'unique_labels': 129, 'labels': {'cs': {'count': 356}, 'math': {'count': 381}, 'OC': {'count': 11}, 'hep-lat': {'count': 13}, 'hep': {'count': 98}, 'astro-ph': {'count': 213}, 'eess': {'count': 76}, 'quant-ph': {'count': 135}, 'DC': {'count': 5}, 'cond-mat': {'count': 274}, 'hep-th': {'count': 66}, 'SP': {'count': 33}, 'hep-ph': {'count': 69}, 'FA': {'count': 6}, 'nucl-th': {'count': 17}, 'q-bio': {'count': 80}, 'HE': {'count': 22}, 'HC': {'count': 2}, 'stat': {'count': 60}, 'ML': {'count': 16}, 'IV': {'count': 13}, 'stat-mech': {'count': 47}, 'DS': {'count': 14}, 'ME': {'count': 12}, 'CC': {'count': 2}, 'mtrl-sci': {'count': 22}, 'PE': {'count': 16}, 'NT': {'count': 11}, 'SC': {'count': 6}, 'AG': {'count': 13}, 'physics': {'count': 81}, 'ins-det': {'count': 9}, 'GA': {'count': 18}, 'BM': {'count': 6}, 'GN': {'count': 17}, 'NA': {'count': 15}, 'app-ph': {'count': 7}, 'RT': {'count': 6}, 'other': {'count': 37}, 'soft': {'count': 15}, 'CO': {'count': 33}, 'supr-con': {'count': 21}, 'chem-ph': {'count': 3}, 'DM': {'count': 2}, 'MN': {'count': 12}, 'q-fin': {'count': 27}, 'PM': {'count': 2}, 'AP': {'count': 27}, 'gr-qc': {'count': 15}, 'quant-gas': {'count': 8}, 'mes-hall': {'count': 33}, 'IT': {'count': 19}, 'SI': {'count': 6}, 'SG': {'count': 3}, 'bio-ph': {'count': 2}, 'SR': {'count': 16}, 'soc-ph': {'count': 5}, 'hep-ex': {'count': 15}, 'DG': {'count': 11}, 'NE': {'count': 5}, 'CR': {'count': 6}, 'CL': {'count': 12}, 'RM': {'count': 3}, 'econ': {'count': 17}, 'nlin': {'count': 5}, 'PS': {'count': 1}, 'LG': {'count': 26}, 'QA': {'count': 9}, 'str-el': {'count': 26}, 'CV': {'count': 34}, 'MF': {'count': 6}, 'IM': {'count': 7}, 'EM': {'count': 6}, 'TH': {'count': 5}, 'PR': {'count': 20}, 'AT': {'count': 4}, 'OA': {'count': 4}, 'CP': {'count': 6}, 'LO': {'count': 14}, 'flu-dyn': {'count': 6}, 'atom-ph': {'count': 8}, 'class-ph': {'count': 1}, 'SY': {'count': 20}, 'IR': {'count': 1}, 'plasm-ph': {'count': 8}, 'CE': {'count': 2}, 'AO': {'count': 1}, 'comp-ph': {'count': 3}, 'optics': {'count': 12}, 'MG': {'count': 4}, 'ST': {'count': 6}, 'nucl-ex': {'count': 6}, 'CY': {'count': 9}, 'ao-ph': {'count': 2}, 'DB': {'count': 1}, 'math-ph': {'count': 10}, 'NC': {'count': 13}, 'GT': {'count': 11}, 'TO': {'count': 2}, 'AI': {'count': 9}, 'NI': {'count': 2}, 'gen-ph': {'count': 4}, 'OT': {'count': 4}, 'SD': {'count': 2}, 'dis-nn': {'count': 4}, 'RO': {'count': 7}, 'CA': {'count': 6}, 'FL': {'count': 1}, 'SE': {'count': 5}, 'EP': {'count': 9}, 'hist-ph': {'count': 1}, 'QM': {'count': 9}, 'ed-ph': {'count': 2}, 'GR': {'count': 4}, 'MS': {'count': 1}, 'CD': {'count': 1}, 'ET': {'count': 1}, 'acc-ph': {'count': 5}, 'AC': {'count': 2}, 'OH': {'count': 1}, 'EC': {'count': 2}, 'DL': {'count': 1}, 'AS': {'count': 3}, 'geo-ph': {'count': 2}, 'CG': {'count': 3}, 'CB': {'count': 1}, 'AR': {'count': 1}, 'TR': {'count': 1}, 'atm-clus': {'count': 1}}}} | | [ArXivHierarchicalClusteringS2S](https://www.kaggle.com/Cornell-University/arxiv) | ['eng'] | Clustering | p2p | [Academic, Written] | None | None | | [ArguAna](http://argumentation.bplaced.net/arguana/data) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | [Medical, Written] | None | None | -| [ArguAna-PL](https://huggingface.co/datasets/clarin-knext/arguana-pl) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | None | +| [ArguAna-Fa](https://huggingface.co/datasets/MCINext/arguana-fa) | ['fas'] | Retrieval | s2p | [Blog] | None | None | +| [ArguAna-PL](https://huggingface.co/datasets/clarin-knext/arguana-pl) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Medical, Written] | None | None | | [ArmenianParaphrasePC](https://github.com/ivannikov-lab/arpa-paraphrase-corpus) (Arthur Malajyan, 2020) | ['hye'] | PairClassification | s2s | [News, Written] | None | None | | [ArxivClassification](https://ieeexplore.ieee.org/document/8675939) (He et al., 2019) | ['eng'] | Classification | s2s | [Academic, Written] | None | None | -| [AskUbuntuDupQuestions](https://github.com/taolei87/askubuntu) | ['eng'] | Reranking | s2s | | {'test': 375} | {'test': {'num_samples': 375, 'number_of_characters': 413674, 'num_positive': 2255, 'num_negative': 5245, 'min_query_length': 17, 'avg_query_length': 50.21, 'max_query_length': 148, 'unique_query': 374, 'min_positive_length': 15, 'avg_positive_length': 52.54, 'max_positive_length': 152, 'unique_positive': 2165, 'min_negative_length': 15, 'avg_negative_length': 52.69, 'max_negative_length': 148, 'unique_negative': 5002}} | +| [AskUbuntuDupQuestions](https://github.com/taolei87/askubuntu) | ['eng'] | Reranking | s2s | [Programming, Web] | {'test': 375} | {'test': {'num_samples': 375, 'number_of_characters': 413674, 'num_positive': 2255, 'num_negative': 5245, 'min_query_length': 17, 'avg_query_length': 50.21, 'max_query_length': 148, 'unique_query': 374, 'min_positive_length': 15, 'avg_positive_length': 52.54, 'max_positive_length': 152, 'unique_positive': 2165, 'min_negative_length': 15, 'avg_negative_length': 52.69, 'max_negative_length': 148, 'unique_negative': 5002}} | | [Assin2RTE](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) (Real et al., 2020) | ['por'] | PairClassification | s2s | [Written] | None | None | | [Assin2STS](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) (Real et al., 2020) | ['por'] | STS | s2s | [Written] | None | None | -| [AutoRAGRetrieval](https://arxiv.org/abs/2410.20878) (Dongkyu Kim, 2024) | ['kor'] | Retrieval | s2p | [Government, Medical, Legal, Social] | {'test': 834} | {'test': {'number_of_characters': 894.22, 'num_samples': 834, 'num_queries': 114, 'num_documents': 720, 'average_document_length': 1.15, 'average_query_length': 0.61, 'average_relevant_docs_per_query': 1.0}} | ->>>>>>> main -| [BIOSSES](https://tabilab.cmpe.boun.edu.tr/BIOSSES/DataSet.html) (Soğancıoğlu et al., 2017) | ['eng'] | STS | s2s | | None | None | +| [AutoRAGRetrieval](https://arxiv.org/abs/2410.20878) (Dongkyu Kim, 2024) | ['kor'] | Retrieval | s2p | [Financial, Government, Legal, Medical, Social] | {'test': 834} | {'test': {'number_of_characters': 894.22, 'num_samples': 834, 'num_queries': 114, 'num_documents': 720, 'average_document_length': 1.15, 'average_query_length': 0.61, 'average_relevant_docs_per_query': 1.0}} | +| [BIOSSES](https://tabilab.cmpe.boun.edu.tr/BIOSSES/DataSet.html) (Soğancıoğlu et al., 2017) | ['eng'] | STS | s2s | [Medical] | None | None | +| [BLINKIT2IMultiChoice](https://arxiv.org/abs/2404.12390) (Fu et al., 2024) | ['eng'] | Any2AnyMultiChoice | it2i | [Encyclopaedic] | None | None | +| [BLINKIT2IRetrieval](https://arxiv.org/abs/2404.12390) (Fu et al., 2024) | ['eng'] | Any2AnyRetrieval | it2i | [Encyclopaedic] | None | None | +| [BLINKIT2TMultiChoice](https://arxiv.org/abs/2404.12390) (Fu et al., 2024) | ['eng'] | Any2AnyMultiChoice | it2t | [Encyclopaedic] | None | None | +| [BLINKIT2TRetrieval](https://arxiv.org/abs/2404.12390) (Fu et al., 2024) | ['eng'] | Any2AnyRetrieval | it2t | [Encyclopaedic] | None | None | | [BQ](https://aclanthology.org/2021.emnlp-main.357) (Shitao Xiao, 2024) | ['cmn'] | STS | s2s | | None | None | | [BSARDRetrieval](https://huggingface.co/datasets/maastrichtlawtech/bsard) (Louis et al., 2022) | ['fra'] | Retrieval | s2p | [Legal, Spoken] | None | None | | [BUCC.v2](https://comparable.limsi.fr/bucc2018/bucc2018-task.html) | ['cmn', 'deu', 'eng', 'fra', 'rus'] | BitextMining | s2s | [Written] | {'test': 35000} | {'test': {'num_samples': 35000, 'number_of_characters': 6640032, 'unique_pairs': 34978, 'min_sentence1_length': 16, 'average_sentence1_length': 99.11, 'max_sentence1_length': 204, 'unique_sentence1': 34978, 'min_sentence2_length': 42, 'average_sentence2_length': 90.61, 'max_sentence2_length': 159, 'unique_sentence2': 25306, 'hf_subset_descriptive_stats': {'de-en': {'num_samples': 9580, 'number_of_characters': 1919197, 'unique_pairs': 9573, 'min_sentence1_length': 50, 'average_sentence1_length': 109.08, 'max_sentence1_length': 204, 'unique_sentence1': 9573, 'min_sentence2_length': 46, 'average_sentence2_length': 91.25, 'max_sentence2_length': 155, 'unique_sentence2': 9570}, 'fr-en': {'num_samples': 9086, 'number_of_characters': 1677545, 'unique_pairs': 9081, 'min_sentence1_length': 43, 'average_sentence1_length': 99.32, 'max_sentence1_length': 174, 'unique_sentence1': 9081, 'min_sentence2_length': 42, 'average_sentence2_length': 85.31, 'max_sentence2_length': 159, 'unique_sentence2': 9076}, 'ru-en': {'num_samples': 14435, 'number_of_characters': 2808206, 'unique_pairs': 14425, 'min_sentence1_length': 40, 'average_sentence1_length': 101.66, 'max_sentence1_length': 186, 'unique_sentence1': 14425, 'min_sentence2_length': 45, 'average_sentence2_length': 92.88, 'max_sentence2_length': 159, 'unique_sentence2': 14424}, 'zh-en': {'num_samples': 1899, 'number_of_characters': 235084, 'unique_pairs': 1899, 'min_sentence1_length': 16, 'average_sentence1_length': 28.43, 'max_sentence1_length': 40, 'unique_sentence1': 1899, 'min_sentence2_length': 48, 'average_sentence2_length': 95.36, 'max_sentence2_length': 159, 'unique_sentence2': 1899}}}} | | [Banking77Classification](https://arxiv.org/abs/2003.04807) | ['eng'] | Classification | s2s | [Written] | None | None | -| [BelebeleRetrieval](https://arxiv.org/abs/2308.16884) (Lucas Bandarkar, 2023) | ['acm', 'afr', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'azj', 'bam', 'ben', 'bod', 'bul', 'cat', 'ceb', 'ces', 'ckb', 'dan', 'deu', 'ell', 'eng', 'est', 'eus', 'fin', 'fra', 'fuv', 'gaz', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kac', 'kan', 'kat', 'kaz', 'kea', 'khk', 'khm', 'kin', 'kir', 'kor', 'lao', 'lin', 'lit', 'lug', 'luo', 'lvs', 'mal', 'mar', 'mkd', 'mlt', 'mri', 'mya', 'nld', 'nob', 'npi', 'nso', 'nya', 'ory', 'pan', 'pbt', 'pes', 'plt', 'pol', 'por', 'ron', 'rus', 'shn', 'sin', 'slk', 'slv', 'sna', 'snd', 'som', 'sot', 'spa', 'srp', 'ssw', 'sun', 'swe', 'swh', 'tam', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tsn', 'tso', 'tur', 'ukr', 'urd', 'uzn', 'vie', 'war', 'wol', 'xho', 'yor', 'zho', 'zsm', 'zul'] | Retrieval | s2p | [Web, News, Written] | {'test': 521866} | {'test': {'number_of_characters': 25574620, 'num_samples': 521866, 'num_queries': 338378, 'num_documents': 183488, 'min_document_length': 4, 'average_document_length': 137.38, 'max_document_length': 237, 'unique_documents': 183488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 338378, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 183488, 'hf_subset_descriptive_stats': {'acm_Arab-acm_Arab': {'number_of_characters': 51232, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 102.98, 'max_document_length': 129, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'acm_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-acm_Arab': {'number_of_characters': 51232, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 102.98, 'max_document_length': 129, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'afr_Latn-afr_Latn': {'number_of_characters': 71217, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 143.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'afr_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-afr_Latn': {'number_of_characters': 71217, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 143.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'als_Latn-als_Latn': {'number_of_characters': 69498, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 140.41, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'als_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-als_Latn': {'number_of_characters': 69498, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 140.41, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'amh_Ethi-amh_Ethi': {'number_of_characters': 45221, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 90.67, 'max_document_length': 100, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'amh_Ethi-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-amh_Ethi': {'number_of_characters': 45221, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 90.67, 'max_document_length': 100, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'apc_Arab-apc_Arab': {'number_of_characters': 51248, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 103.02, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'apc_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-apc_Arab': {'number_of_characters': 51248, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 103.02, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Arab-arb_Arab': {'number_of_characters': 53671, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 107.98, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-arb_Arab': {'number_of_characters': 53671, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 107.98, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Latn-arb_Latn': {'number_of_characters': 61298, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 123.61, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-arb_Latn': {'number_of_characters': 61298, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 123.61, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ars_Arab-ars_Arab': {'number_of_characters': 51765, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 104.08, 'max_document_length': 119, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ars_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ars_Arab': {'number_of_characters': 51765, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 104.08, 'max_document_length': 119, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ary_Arab-ary_Arab': {'number_of_characters': 60261, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 121.49, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ary_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ary_Arab': {'number_of_characters': 60261, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 121.49, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arz_Arab-arz_Arab': {'number_of_characters': 52403, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 105.38, 'max_document_length': 115, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arz_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-arz_Arab': {'number_of_characters': 52403, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 105.38, 'max_document_length': 115, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'asm_Beng-asm_Beng': {'number_of_characters': 62410, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 4, 'average_document_length': 125.89, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'asm_Beng-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-asm_Beng': {'number_of_characters': 62410, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 4, 'average_document_length': 125.89, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'azj_Latn-azj_Latn': {'number_of_characters': 67137, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.58, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'azj_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-azj_Latn': {'number_of_characters': 67137, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.58, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bam_Latn-bam_Latn': {'number_of_characters': 66084, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 133.42, 'max_document_length': 166, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bam_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-bam_Latn': {'number_of_characters': 66084, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 133.42, 'max_document_length': 166, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Beng-ben_Beng': {'number_of_characters': 63512, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 128.15, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Beng-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ben_Beng': {'number_of_characters': 63512, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 128.15, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Latn-ben_Latn': {'number_of_characters': 68285, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 137.93, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ben_Latn': {'number_of_characters': 68285, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 137.93, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bod_Tibt-bod_Tibt': {'number_of_characters': 79188, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.27, 'max_document_length': 213, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bod_Tibt-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-bod_Tibt': {'number_of_characters': 79188, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.27, 'max_document_length': 213, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bul_Cyrl-bul_Cyrl': {'number_of_characters': 66577, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.43, 'max_document_length': 177, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bul_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-bul_Cyrl': {'number_of_characters': 66577, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.43, 'max_document_length': 177, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'cat_Latn-cat_Latn': {'number_of_characters': 68842, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.07, 'max_document_length': 163, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'cat_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-cat_Latn': {'number_of_characters': 68842, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.07, 'max_document_length': 163, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ceb_Latn-ceb_Latn': {'number_of_characters': 74053, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 149.75, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ceb_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ceb_Latn': {'number_of_characters': 74053, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 149.75, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ces_Latn-ces_Latn': {'number_of_characters': 61936, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 124.92, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ces_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ces_Latn': {'number_of_characters': 61936, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 124.92, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ckb_Arab-ckb_Arab': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 131.03, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ckb_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ckb_Arab': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 131.03, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'dan_Latn-dan_Latn': {'number_of_characters': 66648, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.57, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'dan_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-dan_Latn': {'number_of_characters': 66648, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.57, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'deu_Latn-deu_Latn': {'number_of_characters': 68768, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 138.92, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'deu_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-deu_Latn': {'number_of_characters': 68768, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 138.92, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ell_Grek-ell_Grek': {'number_of_characters': 79210, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.32, 'max_document_length': 212, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ell_Grek-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ell_Grek': {'number_of_characters': 79210, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.32, 'max_document_length': 212, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'est_Latn-est_Latn': {'number_of_characters': 61779, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.6, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'est_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-est_Latn': {'number_of_characters': 61779, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.6, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eus_Latn-eus_Latn': {'number_of_characters': 67979, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 137.3, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eus_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-eus_Latn': {'number_of_characters': 67979, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 137.3, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fin_Latn-fin_Latn': {'number_of_characters': 66234, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-fin_Latn': {'number_of_characters': 66234, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fra_Latn-fra_Latn': {'number_of_characters': 82464, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 166.98, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fra_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-fra_Latn': {'number_of_characters': 82464, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 166.98, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fuv_Latn-fuv_Latn': {'number_of_characters': 53555, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 107.74, 'max_document_length': 122, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fuv_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-fuv_Latn': {'number_of_characters': 53555, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 107.74, 'max_document_length': 122, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'gaz_Latn-gaz_Latn': {'number_of_characters': 78315, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 158.48, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'gaz_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-gaz_Latn': {'number_of_characters': 78315, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 158.48, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'grn_Latn-grn_Latn': {'number_of_characters': 68572, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 138.52, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'grn_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-grn_Latn': {'number_of_characters': 68572, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 138.52, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'guj_Gujr-guj_Gujr': {'number_of_characters': 57007, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 114.82, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'guj_Gujr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-guj_Gujr': {'number_of_characters': 57007, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 114.82, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hat_Latn-hat_Latn': {'number_of_characters': 64558, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.29, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hat_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hat_Latn': {'number_of_characters': 64558, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.29, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hau_Latn-hau_Latn': {'number_of_characters': 78240, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.33, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hau_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hau_Latn': {'number_of_characters': 78240, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.33, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'heb_Hebr-heb_Hebr': {'number_of_characters': 50598, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 101.68, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'heb_Hebr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-heb_Hebr': {'number_of_characters': 50598, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 101.68, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Deva-hin_Deva': {'number_of_characters': 66332, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.93, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Deva-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hin_Deva': {'number_of_characters': 66332, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.93, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Latn-hin_Latn': {'number_of_characters': 68307, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.97, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hin_Latn': {'number_of_characters': 68307, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.97, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hrv_Latn-hrv_Latn': {'number_of_characters': 62928, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.95, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hrv_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hrv_Latn': {'number_of_characters': 62928, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.95, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hun_Latn-hun_Latn': {'number_of_characters': 67941, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 137.22, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hun_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hun_Latn': {'number_of_characters': 67941, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 137.22, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hye_Armn-hye_Armn': {'number_of_characters': 68859, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.1, 'max_document_length': 193, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hye_Armn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hye_Armn': {'number_of_characters': 68859, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.1, 'max_document_length': 193, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ibo_Latn-ibo_Latn': {'number_of_characters': 66167, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 133.59, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'ibo_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ibo_Latn': {'number_of_characters': 66167, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 133.59, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'ilo_Latn-ilo_Latn': {'number_of_characters': 78161, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.17, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ilo_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ilo_Latn': {'number_of_characters': 78161, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.17, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ind_Latn-ind_Latn': {'number_of_characters': 74871, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 151.42, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ind_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ind_Latn': {'number_of_characters': 74871, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 151.42, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'isl_Latn-isl_Latn': {'number_of_characters': 70522, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 142.51, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'isl_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-isl_Latn': {'number_of_characters': 70522, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 142.51, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ita_Latn-ita_Latn': {'number_of_characters': 76124, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 153.99, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ita_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ita_Latn': {'number_of_characters': 76124, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 153.99, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jav_Latn-jav_Latn': {'number_of_characters': 71722, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 144.97, 'max_document_length': 174, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jav_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-jav_Latn': {'number_of_characters': 71722, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 144.97, 'max_document_length': 174, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jpn_Jpan-jpn_Jpan': {'number_of_characters': 33187, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 66.01, 'max_document_length': 76, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jpn_Jpan-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-jpn_Jpan': {'number_of_characters': 33187, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 66.01, 'max_document_length': 76, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kac_Latn-kac_Latn': {'number_of_characters': 89655, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 181.72, 'max_document_length': 195, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kac_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kac_Latn': {'number_of_characters': 89655, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 181.72, 'max_document_length': 195, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kan_Knda-kan_Knda': {'number_of_characters': 65899, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.04, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kan_Knda-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kan_Knda': {'number_of_characters': 65899, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.04, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kat_Geor-kat_Geor': {'number_of_characters': 68309, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.98, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kat_Geor-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kat_Geor': {'number_of_characters': 68309, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.98, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kaz_Cyrl-kaz_Cyrl': {'number_of_characters': 64657, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.49, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kaz_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kaz_Cyrl': {'number_of_characters': 64657, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.49, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kea_Latn-kea_Latn': {'number_of_characters': 69323, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.06, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kea_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kea_Latn': {'number_of_characters': 69323, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.06, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khk_Cyrl-khk_Cyrl': {'number_of_characters': 66977, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 135.25, 'max_document_length': 162, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khk_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-khk_Cyrl': {'number_of_characters': 66977, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 135.25, 'max_document_length': 162, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khm_Khmr-khm_Khmr': {'number_of_characters': 69150, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 139.7, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khm_Khmr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-khm_Khmr': {'number_of_characters': 69150, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 139.7, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kin_Latn-kin_Latn': {'number_of_characters': 72803, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 147.19, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'kin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kin_Latn': {'number_of_characters': 72803, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 147.19, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'kir_Cyrl-kir_Cyrl': {'number_of_characters': 67957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 137.26, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kir_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kir_Cyrl': {'number_of_characters': 67957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 137.26, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kor_Hang-kor_Hang': {'number_of_characters': 32708, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 65.02, 'max_document_length': 88, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kor_Hang-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kor_Hang': {'number_of_characters': 32708, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 65.02, 'max_document_length': 88, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lao_Laoo-lao_Laoo': {'number_of_characters': 57958, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 116.77, 'max_document_length': 142, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lao_Laoo-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lao_Laoo': {'number_of_characters': 57958, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 116.77, 'max_document_length': 142, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lin_Latn-lin_Latn': {'number_of_characters': 74223, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 150.1, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'lin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lin_Latn': {'number_of_characters': 74223, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 150.1, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'lit_Latn-lit_Latn': {'number_of_characters': 62805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 126.7, 'max_document_length': 167, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lit_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lit_Latn': {'number_of_characters': 62805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 126.7, 'max_document_length': 167, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lug_Latn-lug_Latn': {'number_of_characters': 71566, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 144.65, 'max_document_length': 237, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'lug_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lug_Latn': {'number_of_characters': 71566, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 144.65, 'max_document_length': 237, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'luo_Latn-luo_Latn': {'number_of_characters': 66805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 134.9, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'luo_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-luo_Latn': {'number_of_characters': 66805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 134.9, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lvs_Latn-lvs_Latn': {'number_of_characters': 63957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 129.06, 'max_document_length': 172, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lvs_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lvs_Latn': {'number_of_characters': 63957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 129.06, 'max_document_length': 172, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mal_Mlym-mal_Mlym': {'number_of_characters': 73599, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.82, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mal_Mlym-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mal_Mlym': {'number_of_characters': 73599, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.82, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mar_Deva-mar_Deva': {'number_of_characters': 62671, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 126.42, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'mar_Deva-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mar_Deva': {'number_of_characters': 62671, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 126.42, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'mkd_Cyrl-mkd_Cyrl': {'number_of_characters': 67588, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 136.5, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mkd_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mkd_Cyrl': {'number_of_characters': 67588, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 136.5, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mlt_Latn-mlt_Latn': {'number_of_characters': 68480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 138.33, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mlt_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mlt_Latn': {'number_of_characters': 68480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 138.33, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mri_Latn-mri_Latn': {'number_of_characters': 74519, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 150.7, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mri_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mri_Latn': {'number_of_characters': 74519, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 150.7, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mya_Mymr-mya_Mymr': {'number_of_characters': 81331, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 164.66, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mya_Mymr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mya_Mymr': {'number_of_characters': 81331, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 164.66, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nld_Latn-nld_Latn': {'number_of_characters': 68789, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 138.96, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nld_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nld_Latn': {'number_of_characters': 68789, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 138.96, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nob_Latn-nob_Latn': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 131.03, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nob_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nob_Latn': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 131.03, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Deva-npi_Deva': {'number_of_characters': 61183, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 123.38, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Deva-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-npi_Deva': {'number_of_characters': 61183, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 123.38, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Latn-npi_Latn': {'number_of_characters': 65683, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 132.6, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-npi_Latn': {'number_of_characters': 65683, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 132.6, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nso_Latn-nso_Latn': {'number_of_characters': 79073, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 160.03, 'max_document_length': 235, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nso_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nso_Latn': {'number_of_characters': 79073, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 160.03, 'max_document_length': 235, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nya_Latn-nya_Latn': {'number_of_characters': 82685, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.44, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nya_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nya_Latn': {'number_of_characters': 82685, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.44, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ory_Orya-ory_Orya': {'number_of_characters': 66638, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 10, 'average_document_length': 134.55, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ory_Orya-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ory_Orya': {'number_of_characters': 66638, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 10, 'average_document_length': 134.55, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pan_Guru-pan_Guru': {'number_of_characters': 66944, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.18, 'max_document_length': 157, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pan_Guru-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pan_Guru': {'number_of_characters': 66944, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.18, 'max_document_length': 157, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pbt_Arab-pbt_Arab': {'number_of_characters': 61880, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 124.8, 'max_document_length': 155, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pbt_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pbt_Arab': {'number_of_characters': 61880, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 124.8, 'max_document_length': 155, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pes_Arab-pes_Arab': {'number_of_characters': 59252, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 119.42, 'max_document_length': 152, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pes_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pes_Arab': {'number_of_characters': 59252, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 119.42, 'max_document_length': 152, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'plt_Latn-plt_Latn': {'number_of_characters': 86472, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 175.2, 'max_document_length': 222, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'plt_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-plt_Latn': {'number_of_characters': 86472, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 175.2, 'max_document_length': 222, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pol_Latn-pol_Latn': {'number_of_characters': 67664, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 136.66, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pol_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pol_Latn': {'number_of_characters': 67664, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 136.66, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'por_Latn-por_Latn': {'number_of_characters': 71281, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.07, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'por_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-por_Latn': {'number_of_characters': 71281, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.07, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ron_Latn-ron_Latn': {'number_of_characters': 71844, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 145.22, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ron_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ron_Latn': {'number_of_characters': 71844, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 145.22, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'rus_Cyrl-rus_Cyrl': {'number_of_characters': 75823, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 153.38, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'rus_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-rus_Cyrl': {'number_of_characters': 75823, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 153.38, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'shn_Mymr-shn_Mymr': {'number_of_characters': 69288, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 139.98, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'shn_Mymr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-shn_Mymr': {'number_of_characters': 69288, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 139.98, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Latn-sin_Latn': {'number_of_characters': 85996, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 174.22, 'max_document_length': 224, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sin_Latn': {'number_of_characters': 85996, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 174.22, 'max_document_length': 224, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Sinh-sin_Sinh': {'number_of_characters': 63902, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 128.95, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Sinh-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sin_Sinh': {'number_of_characters': 63902, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 128.95, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slk_Latn-slk_Latn': {'number_of_characters': 62663, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 126.41, 'max_document_length': 146, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slk_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-slk_Latn': {'number_of_characters': 62663, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 126.41, 'max_document_length': 146, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slv_Latn-slv_Latn': {'number_of_characters': 62895, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.88, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slv_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-slv_Latn': {'number_of_characters': 62895, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.88, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sna_Latn-sna_Latn': {'number_of_characters': 74071, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.78, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sna_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sna_Latn': {'number_of_characters': 74071, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.78, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'snd_Arab-snd_Arab': {'number_of_characters': 58057, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 116.97, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'snd_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-snd_Arab': {'number_of_characters': 58057, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 116.97, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'som_Latn-som_Latn': {'number_of_characters': 82838, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.75, 'max_document_length': 201, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'som_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-som_Latn': {'number_of_characters': 82838, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.75, 'max_document_length': 201, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sot_Latn-sot_Latn': {'number_of_characters': 75794, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 153.32, 'max_document_length': 186, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sot_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sot_Latn': {'number_of_characters': 75794, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 153.32, 'max_document_length': 186, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'spa_Latn-spa_Latn': {'number_of_characters': 74920, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 151.52, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'spa_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-spa_Latn': {'number_of_characters': 74920, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 151.52, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'srp_Cyrl-srp_Cyrl': {'number_of_characters': 61657, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.35, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'srp_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-srp_Cyrl': {'number_of_characters': 61657, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.35, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'ssw_Latn-ssw_Latn': {'number_of_characters': 73964, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 149.57, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ssw_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ssw_Latn': {'number_of_characters': 73964, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 149.57, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sun_Latn-sun_Latn': {'number_of_characters': 71320, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 144.15, 'max_document_length': 173, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sun_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sun_Latn': {'number_of_characters': 71320, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 144.15, 'max_document_length': 173, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swe_Latn-swe_Latn': {'number_of_characters': 62785, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 126.66, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swe_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-swe_Latn': {'number_of_characters': 62785, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 126.66, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swh_Latn-swh_Latn': {'number_of_characters': 73480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.57, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swh_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-swh_Latn': {'number_of_characters': 73480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.57, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tam_Taml-tam_Taml': {'number_of_characters': 73991, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.62, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tam_Taml-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tam_Taml': {'number_of_characters': 73991, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.62, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tel_Telu-tel_Telu': {'number_of_characters': 65945, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 133.13, 'max_document_length': 149, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tel_Telu-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tel_Telu': {'number_of_characters': 65945, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 133.13, 'max_document_length': 149, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgk_Cyrl-tgk_Cyrl': {'number_of_characters': 67829, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 136.99, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgk_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tgk_Cyrl': {'number_of_characters': 67829, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 136.99, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgl_Latn-tgl_Latn': {'number_of_characters': 75087, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 151.87, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgl_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tgl_Latn': {'number_of_characters': 75087, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 151.87, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tha_Thai-tha_Thai': {'number_of_characters': 54496, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 109.67, 'max_document_length': 123, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tha_Thai-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tha_Thai': {'number_of_characters': 54496, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 109.67, 'max_document_length': 123, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tir_Ethi-tir_Ethi': {'number_of_characters': 47775, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 95.9, 'max_document_length': 110, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tir_Ethi-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tir_Ethi': {'number_of_characters': 47775, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 95.9, 'max_document_length': 110, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tsn_Latn-tsn_Latn': {'number_of_characters': 79391, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 160.69, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tsn_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tsn_Latn': {'number_of_characters': 79391, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 160.69, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tso_Latn-tso_Latn': {'number_of_characters': 83501, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 169.11, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tso_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tso_Latn': {'number_of_characters': 83501, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 169.11, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tur_Latn-tur_Latn': {'number_of_characters': 65382, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 131.98, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tur_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tur_Latn': {'number_of_characters': 65382, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 131.98, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ukr_Cyrl-ukr_Cyrl': {'number_of_characters': 65850, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 132.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ukr_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ukr_Cyrl': {'number_of_characters': 65850, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 132.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Arab-urd_Arab': {'number_of_characters': 64450, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 130.07, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-urd_Arab': {'number_of_characters': 64450, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 130.07, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Latn-urd_Latn': {'number_of_characters': 82039, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 166.11, 'max_document_length': 230, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-urd_Latn': {'number_of_characters': 82039, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 166.11, 'max_document_length': 230, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'uzn_Latn-uzn_Latn': {'number_of_characters': 70828, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 143.14, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'uzn_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-uzn_Latn': {'number_of_characters': 70828, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 143.14, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'vie_Latn-vie_Latn': {'number_of_characters': 66724, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 134.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'vie_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-vie_Latn': {'number_of_characters': 66724, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 134.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'war_Latn-war_Latn': {'number_of_characters': 78444, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 158.75, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'war_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-war_Latn': {'number_of_characters': 78444, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 158.75, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'wol_Latn-wol_Latn': {'number_of_characters': 64521, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 130.22, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'wol_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-wol_Latn': {'number_of_characters': 64521, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 130.22, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'xho_Latn-xho_Latn': {'number_of_characters': 71629, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.78, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'xho_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-xho_Latn': {'number_of_characters': 71629, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.78, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'yor_Latn-yor_Latn': {'number_of_characters': 62752, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 126.59, 'max_document_length': 143, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'yor_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-yor_Latn': {'number_of_characters': 62752, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 126.59, 'max_document_length': 143, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hans-zho_Hans': {'number_of_characters': 20549, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 40.11, 'max_document_length': 64, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hans-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zho_Hans': {'number_of_characters': 20549, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 40.11, 'max_document_length': 64, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hant-zho_Hant': {'number_of_characters': 19947, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 38.88, 'max_document_length': 45, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hant-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zho_Hant': {'number_of_characters': 19947, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 38.88, 'max_document_length': 45, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zsm_Latn-zsm_Latn': {'number_of_characters': 72008, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 145.56, 'max_document_length': 210, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zsm_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zsm_Latn': {'number_of_characters': 72008, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 145.56, 'max_document_length': 210, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zul_Latn-zul_Latn': {'number_of_characters': 69413, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.24, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zul_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zul_Latn': {'number_of_characters': 69413, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.24, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Arab-arb_Latn': {'number_of_characters': 61298, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 123.61, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Latn-arb_Arab': {'number_of_characters': 53671, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 107.98, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Beng-ben_Latn': {'number_of_characters': 68285, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 137.93, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Latn-ben_Beng': {'number_of_characters': 63512, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 128.15, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Deva-hin_Latn': {'number_of_characters': 68307, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.97, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Latn-hin_Deva': {'number_of_characters': 66332, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.93, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Deva-npi_Latn': {'number_of_characters': 65683, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 132.6, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Latn-npi_Deva': {'number_of_characters': 61183, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 123.38, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Sinh-sin_Latn': {'number_of_characters': 85996, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 174.22, 'max_document_length': 224, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Latn-sin_Sinh': {'number_of_characters': 63902, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 128.95, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Arab-urd_Latn': {'number_of_characters': 82039, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 166.11, 'max_document_length': 230, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Latn-urd_Arab': {'number_of_characters': 64450, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 130.07, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}}}} | +| [BelebeleRetrieval](https://arxiv.org/abs/2308.16884) (Lucas Bandarkar, 2023) | ['acm', 'afr', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'azj', 'bam', 'ben', 'bod', 'bul', 'cat', 'ceb', 'ces', 'ckb', 'dan', 'deu', 'ell', 'eng', 'est', 'eus', 'fin', 'fra', 'fuv', 'gaz', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kac', 'kan', 'kat', 'kaz', 'kea', 'khk', 'khm', 'kin', 'kir', 'kor', 'lao', 'lin', 'lit', 'lug', 'luo', 'lvs', 'mal', 'mar', 'mkd', 'mlt', 'mri', 'mya', 'nld', 'nob', 'npi', 'nso', 'nya', 'ory', 'pan', 'pbt', 'pes', 'plt', 'pol', 'por', 'ron', 'rus', 'shn', 'sin', 'slk', 'slv', 'sna', 'snd', 'som', 'sot', 'spa', 'srp', 'ssw', 'sun', 'swe', 'swh', 'tam', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tsn', 'tso', 'tur', 'ukr', 'urd', 'uzn', 'vie', 'war', 'wol', 'xho', 'yor', 'zho', 'zsm', 'zul'] | Retrieval | s2p | [News, Web, Written] | {'test': 521866} | {'test': {'number_of_characters': 25574620, 'num_samples': 521866, 'num_queries': 338378, 'num_documents': 183488, 'min_document_length': 4, 'average_document_length': 137.38, 'max_document_length': 237, 'unique_documents': 183488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 338378, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 183488, 'hf_subset_descriptive_stats': {'acm_Arab-acm_Arab': {'number_of_characters': 51232, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 102.98, 'max_document_length': 129, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'acm_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-acm_Arab': {'number_of_characters': 51232, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 102.98, 'max_document_length': 129, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'afr_Latn-afr_Latn': {'number_of_characters': 71217, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 143.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'afr_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-afr_Latn': {'number_of_characters': 71217, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 143.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'als_Latn-als_Latn': {'number_of_characters': 69498, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 140.41, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'als_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-als_Latn': {'number_of_characters': 69498, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 140.41, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'amh_Ethi-amh_Ethi': {'number_of_characters': 45221, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 90.67, 'max_document_length': 100, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'amh_Ethi-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-amh_Ethi': {'number_of_characters': 45221, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 90.67, 'max_document_length': 100, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'apc_Arab-apc_Arab': {'number_of_characters': 51248, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 103.02, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'apc_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-apc_Arab': {'number_of_characters': 51248, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 103.02, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Arab-arb_Arab': {'number_of_characters': 53671, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 107.98, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-arb_Arab': {'number_of_characters': 53671, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 107.98, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Latn-arb_Latn': {'number_of_characters': 61298, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 123.61, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-arb_Latn': {'number_of_characters': 61298, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 123.61, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ars_Arab-ars_Arab': {'number_of_characters': 51765, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 104.08, 'max_document_length': 119, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ars_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ars_Arab': {'number_of_characters': 51765, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 104.08, 'max_document_length': 119, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ary_Arab-ary_Arab': {'number_of_characters': 60261, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 121.49, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ary_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ary_Arab': {'number_of_characters': 60261, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 121.49, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arz_Arab-arz_Arab': {'number_of_characters': 52403, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 105.38, 'max_document_length': 115, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arz_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-arz_Arab': {'number_of_characters': 52403, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 105.38, 'max_document_length': 115, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'asm_Beng-asm_Beng': {'number_of_characters': 62410, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 4, 'average_document_length': 125.89, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'asm_Beng-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-asm_Beng': {'number_of_characters': 62410, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 4, 'average_document_length': 125.89, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'azj_Latn-azj_Latn': {'number_of_characters': 67137, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.58, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'azj_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-azj_Latn': {'number_of_characters': 67137, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.58, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bam_Latn-bam_Latn': {'number_of_characters': 66084, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 133.42, 'max_document_length': 166, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bam_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-bam_Latn': {'number_of_characters': 66084, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 133.42, 'max_document_length': 166, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Beng-ben_Beng': {'number_of_characters': 63512, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 128.15, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Beng-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ben_Beng': {'number_of_characters': 63512, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 128.15, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Latn-ben_Latn': {'number_of_characters': 68285, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 137.93, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ben_Latn': {'number_of_characters': 68285, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 137.93, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bod_Tibt-bod_Tibt': {'number_of_characters': 79188, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.27, 'max_document_length': 213, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bod_Tibt-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-bod_Tibt': {'number_of_characters': 79188, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.27, 'max_document_length': 213, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bul_Cyrl-bul_Cyrl': {'number_of_characters': 66577, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.43, 'max_document_length': 177, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'bul_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-bul_Cyrl': {'number_of_characters': 66577, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.43, 'max_document_length': 177, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'cat_Latn-cat_Latn': {'number_of_characters': 68842, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.07, 'max_document_length': 163, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'cat_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-cat_Latn': {'number_of_characters': 68842, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.07, 'max_document_length': 163, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ceb_Latn-ceb_Latn': {'number_of_characters': 74053, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 149.75, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ceb_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ceb_Latn': {'number_of_characters': 74053, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 149.75, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ces_Latn-ces_Latn': {'number_of_characters': 61936, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 124.92, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ces_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ces_Latn': {'number_of_characters': 61936, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 124.92, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ckb_Arab-ckb_Arab': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 131.03, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ckb_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ckb_Arab': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 131.03, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'dan_Latn-dan_Latn': {'number_of_characters': 66648, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.57, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'dan_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-dan_Latn': {'number_of_characters': 66648, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 134.57, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'deu_Latn-deu_Latn': {'number_of_characters': 68768, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 138.92, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'deu_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-deu_Latn': {'number_of_characters': 68768, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 138.92, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ell_Grek-ell_Grek': {'number_of_characters': 79210, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.32, 'max_document_length': 212, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ell_Grek-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ell_Grek': {'number_of_characters': 79210, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 160.32, 'max_document_length': 212, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'est_Latn-est_Latn': {'number_of_characters': 61779, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.6, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'est_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-est_Latn': {'number_of_characters': 61779, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.6, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eus_Latn-eus_Latn': {'number_of_characters': 67979, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 137.3, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eus_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-eus_Latn': {'number_of_characters': 67979, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 137.3, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fin_Latn-fin_Latn': {'number_of_characters': 66234, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-fin_Latn': {'number_of_characters': 66234, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fra_Latn-fra_Latn': {'number_of_characters': 82464, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 166.98, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fra_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-fra_Latn': {'number_of_characters': 82464, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 166.98, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fuv_Latn-fuv_Latn': {'number_of_characters': 53555, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 107.74, 'max_document_length': 122, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'fuv_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-fuv_Latn': {'number_of_characters': 53555, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 107.74, 'max_document_length': 122, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'gaz_Latn-gaz_Latn': {'number_of_characters': 78315, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 158.48, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'gaz_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-gaz_Latn': {'number_of_characters': 78315, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 158.48, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'grn_Latn-grn_Latn': {'number_of_characters': 68572, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 138.52, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'grn_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-grn_Latn': {'number_of_characters': 68572, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 138.52, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'guj_Gujr-guj_Gujr': {'number_of_characters': 57007, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 114.82, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'guj_Gujr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-guj_Gujr': {'number_of_characters': 57007, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 114.82, 'max_document_length': 138, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hat_Latn-hat_Latn': {'number_of_characters': 64558, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.29, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hat_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hat_Latn': {'number_of_characters': 64558, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.29, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hau_Latn-hau_Latn': {'number_of_characters': 78240, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.33, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hau_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hau_Latn': {'number_of_characters': 78240, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.33, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'heb_Hebr-heb_Hebr': {'number_of_characters': 50598, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 101.68, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'heb_Hebr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-heb_Hebr': {'number_of_characters': 50598, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 101.68, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Deva-hin_Deva': {'number_of_characters': 66332, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.93, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Deva-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hin_Deva': {'number_of_characters': 66332, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.93, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Latn-hin_Latn': {'number_of_characters': 68307, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.97, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hin_Latn': {'number_of_characters': 68307, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.97, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hrv_Latn-hrv_Latn': {'number_of_characters': 62928, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.95, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hrv_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hrv_Latn': {'number_of_characters': 62928, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.95, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hun_Latn-hun_Latn': {'number_of_characters': 67941, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 137.22, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hun_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hun_Latn': {'number_of_characters': 67941, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 137.22, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hye_Armn-hye_Armn': {'number_of_characters': 68859, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.1, 'max_document_length': 193, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hye_Armn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-hye_Armn': {'number_of_characters': 68859, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 139.1, 'max_document_length': 193, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ibo_Latn-ibo_Latn': {'number_of_characters': 66167, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 133.59, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'ibo_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ibo_Latn': {'number_of_characters': 66167, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 133.59, 'max_document_length': 156, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'ilo_Latn-ilo_Latn': {'number_of_characters': 78161, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.17, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ilo_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ilo_Latn': {'number_of_characters': 78161, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 158.17, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ind_Latn-ind_Latn': {'number_of_characters': 74871, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 151.42, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ind_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ind_Latn': {'number_of_characters': 74871, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 151.42, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'isl_Latn-isl_Latn': {'number_of_characters': 70522, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 142.51, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'isl_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-isl_Latn': {'number_of_characters': 70522, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 142.51, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ita_Latn-ita_Latn': {'number_of_characters': 76124, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 153.99, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ita_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ita_Latn': {'number_of_characters': 76124, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 153.99, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jav_Latn-jav_Latn': {'number_of_characters': 71722, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 144.97, 'max_document_length': 174, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jav_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-jav_Latn': {'number_of_characters': 71722, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 144.97, 'max_document_length': 174, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jpn_Jpan-jpn_Jpan': {'number_of_characters': 33187, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 66.01, 'max_document_length': 76, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'jpn_Jpan-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-jpn_Jpan': {'number_of_characters': 33187, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 66.01, 'max_document_length': 76, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kac_Latn-kac_Latn': {'number_of_characters': 89655, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 181.72, 'max_document_length': 195, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kac_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kac_Latn': {'number_of_characters': 89655, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 181.72, 'max_document_length': 195, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kan_Knda-kan_Knda': {'number_of_characters': 65899, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.04, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kan_Knda-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kan_Knda': {'number_of_characters': 65899, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.04, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kat_Geor-kat_Geor': {'number_of_characters': 68309, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.98, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kat_Geor-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kat_Geor': {'number_of_characters': 68309, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.98, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kaz_Cyrl-kaz_Cyrl': {'number_of_characters': 64657, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.49, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kaz_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kaz_Cyrl': {'number_of_characters': 64657, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 130.49, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kea_Latn-kea_Latn': {'number_of_characters': 69323, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.06, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kea_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kea_Latn': {'number_of_characters': 69323, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.06, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khk_Cyrl-khk_Cyrl': {'number_of_characters': 66977, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 135.25, 'max_document_length': 162, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khk_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-khk_Cyrl': {'number_of_characters': 66977, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 135.25, 'max_document_length': 162, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khm_Khmr-khm_Khmr': {'number_of_characters': 69150, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 139.7, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'khm_Khmr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-khm_Khmr': {'number_of_characters': 69150, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 139.7, 'max_document_length': 169, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kin_Latn-kin_Latn': {'number_of_characters': 72803, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 147.19, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'kin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kin_Latn': {'number_of_characters': 72803, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 147.19, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'kir_Cyrl-kir_Cyrl': {'number_of_characters': 67957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 137.26, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kir_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kir_Cyrl': {'number_of_characters': 67957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 137.26, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kor_Hang-kor_Hang': {'number_of_characters': 32708, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 65.02, 'max_document_length': 88, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'kor_Hang-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-kor_Hang': {'number_of_characters': 32708, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 65.02, 'max_document_length': 88, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lao_Laoo-lao_Laoo': {'number_of_characters': 57958, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 116.77, 'max_document_length': 142, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lao_Laoo-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lao_Laoo': {'number_of_characters': 57958, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 116.77, 'max_document_length': 142, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lin_Latn-lin_Latn': {'number_of_characters': 74223, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 150.1, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'lin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lin_Latn': {'number_of_characters': 74223, 'num_samples': 1386, 'num_queries': 898, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 150.1, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 898, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'lit_Latn-lit_Latn': {'number_of_characters': 62805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 126.7, 'max_document_length': 167, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lit_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lit_Latn': {'number_of_characters': 62805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 126.7, 'max_document_length': 167, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lug_Latn-lug_Latn': {'number_of_characters': 71566, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 144.65, 'max_document_length': 237, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'lug_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lug_Latn': {'number_of_characters': 71566, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 144.65, 'max_document_length': 237, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'luo_Latn-luo_Latn': {'number_of_characters': 66805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 134.9, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'luo_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-luo_Latn': {'number_of_characters': 66805, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 134.9, 'max_document_length': 178, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lvs_Latn-lvs_Latn': {'number_of_characters': 63957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 129.06, 'max_document_length': 172, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'lvs_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-lvs_Latn': {'number_of_characters': 63957, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 129.06, 'max_document_length': 172, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mal_Mlym-mal_Mlym': {'number_of_characters': 73599, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.82, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mal_Mlym-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mal_Mlym': {'number_of_characters': 73599, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.82, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mar_Deva-mar_Deva': {'number_of_characters': 62671, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 126.42, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'mar_Deva-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mar_Deva': {'number_of_characters': 62671, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 126.42, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'mkd_Cyrl-mkd_Cyrl': {'number_of_characters': 67588, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 136.5, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mkd_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mkd_Cyrl': {'number_of_characters': 67588, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 136.5, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mlt_Latn-mlt_Latn': {'number_of_characters': 68480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 138.33, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mlt_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mlt_Latn': {'number_of_characters': 68480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 138.33, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mri_Latn-mri_Latn': {'number_of_characters': 74519, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 150.7, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mri_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mri_Latn': {'number_of_characters': 74519, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 150.7, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mya_Mymr-mya_Mymr': {'number_of_characters': 81331, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 164.66, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'mya_Mymr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-mya_Mymr': {'number_of_characters': 81331, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 164.66, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nld_Latn-nld_Latn': {'number_of_characters': 68789, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 138.96, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nld_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nld_Latn': {'number_of_characters': 68789, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 138.96, 'max_document_length': 183, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nob_Latn-nob_Latn': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 131.03, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nob_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nob_Latn': {'number_of_characters': 64917, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 131.03, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Deva-npi_Deva': {'number_of_characters': 61183, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 123.38, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Deva-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-npi_Deva': {'number_of_characters': 61183, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 123.38, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Latn-npi_Latn': {'number_of_characters': 65683, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 132.6, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-npi_Latn': {'number_of_characters': 65683, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 132.6, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nso_Latn-nso_Latn': {'number_of_characters': 79073, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 160.03, 'max_document_length': 235, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nso_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nso_Latn': {'number_of_characters': 79073, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 160.03, 'max_document_length': 235, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nya_Latn-nya_Latn': {'number_of_characters': 82685, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.44, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'nya_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-nya_Latn': {'number_of_characters': 82685, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.44, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ory_Orya-ory_Orya': {'number_of_characters': 66638, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 10, 'average_document_length': 134.55, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ory_Orya-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ory_Orya': {'number_of_characters': 66638, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 10, 'average_document_length': 134.55, 'max_document_length': 168, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pan_Guru-pan_Guru': {'number_of_characters': 66944, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.18, 'max_document_length': 157, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pan_Guru-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pan_Guru': {'number_of_characters': 66944, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 135.18, 'max_document_length': 157, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pbt_Arab-pbt_Arab': {'number_of_characters': 61880, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 124.8, 'max_document_length': 155, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pbt_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pbt_Arab': {'number_of_characters': 61880, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 124.8, 'max_document_length': 155, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pes_Arab-pes_Arab': {'number_of_characters': 59252, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 119.42, 'max_document_length': 152, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pes_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pes_Arab': {'number_of_characters': 59252, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 119.42, 'max_document_length': 152, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'plt_Latn-plt_Latn': {'number_of_characters': 86472, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 175.2, 'max_document_length': 222, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'plt_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-plt_Latn': {'number_of_characters': 86472, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 175.2, 'max_document_length': 222, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pol_Latn-pol_Latn': {'number_of_characters': 67664, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 136.66, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'pol_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-pol_Latn': {'number_of_characters': 67664, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 136.66, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'por_Latn-por_Latn': {'number_of_characters': 71281, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.07, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'por_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-por_Latn': {'number_of_characters': 71281, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.07, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ron_Latn-ron_Latn': {'number_of_characters': 71844, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 145.22, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ron_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ron_Latn': {'number_of_characters': 71844, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 145.22, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'rus_Cyrl-rus_Cyrl': {'number_of_characters': 75823, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 153.38, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'rus_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-rus_Cyrl': {'number_of_characters': 75823, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 153.38, 'max_document_length': 196, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'shn_Mymr-shn_Mymr': {'number_of_characters': 69288, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 139.98, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'shn_Mymr-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-shn_Mymr': {'number_of_characters': 69288, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 139.98, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Latn-sin_Latn': {'number_of_characters': 85996, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 174.22, 'max_document_length': 224, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sin_Latn': {'number_of_characters': 85996, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 174.22, 'max_document_length': 224, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Sinh-sin_Sinh': {'number_of_characters': 63902, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 128.95, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Sinh-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sin_Sinh': {'number_of_characters': 63902, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 128.95, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slk_Latn-slk_Latn': {'number_of_characters': 62663, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 126.41, 'max_document_length': 146, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slk_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-slk_Latn': {'number_of_characters': 62663, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 126.41, 'max_document_length': 146, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slv_Latn-slv_Latn': {'number_of_characters': 62895, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.88, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'slv_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-slv_Latn': {'number_of_characters': 62895, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 126.88, 'max_document_length': 176, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sna_Latn-sna_Latn': {'number_of_characters': 74071, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.78, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sna_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sna_Latn': {'number_of_characters': 74071, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.78, 'max_document_length': 191, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'snd_Arab-snd_Arab': {'number_of_characters': 58057, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 116.97, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'snd_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-snd_Arab': {'number_of_characters': 58057, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 116.97, 'max_document_length': 164, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'som_Latn-som_Latn': {'number_of_characters': 82838, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.75, 'max_document_length': 201, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'som_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-som_Latn': {'number_of_characters': 82838, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 167.75, 'max_document_length': 201, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sot_Latn-sot_Latn': {'number_of_characters': 75794, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 153.32, 'max_document_length': 186, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sot_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sot_Latn': {'number_of_characters': 75794, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 153.32, 'max_document_length': 186, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'spa_Latn-spa_Latn': {'number_of_characters': 74920, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 151.52, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'spa_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-spa_Latn': {'number_of_characters': 74920, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 151.52, 'max_document_length': 180, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'srp_Cyrl-srp_Cyrl': {'number_of_characters': 61657, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.35, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'srp_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-srp_Cyrl': {'number_of_characters': 61657, 'num_samples': 1387, 'num_queries': 899, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 124.35, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.09, 'max_query_length': 2, 'unique_queries': 899, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 2, 'unique_relevant_docs': 488}, 'ssw_Latn-ssw_Latn': {'number_of_characters': 73964, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 149.57, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ssw_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ssw_Latn': {'number_of_characters': 73964, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 149.57, 'max_document_length': 182, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sun_Latn-sun_Latn': {'number_of_characters': 71320, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 144.15, 'max_document_length': 173, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sun_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-sun_Latn': {'number_of_characters': 71320, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 144.15, 'max_document_length': 173, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swe_Latn-swe_Latn': {'number_of_characters': 62785, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 126.66, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swe_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-swe_Latn': {'number_of_characters': 62785, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 126.66, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swh_Latn-swh_Latn': {'number_of_characters': 73480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.57, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'swh_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-swh_Latn': {'number_of_characters': 73480, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 148.57, 'max_document_length': 194, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tam_Taml-tam_Taml': {'number_of_characters': 73991, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.62, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tam_Taml-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tam_Taml': {'number_of_characters': 73991, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 149.62, 'max_document_length': 181, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tel_Telu-tel_Telu': {'number_of_characters': 65945, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 133.13, 'max_document_length': 149, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tel_Telu-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tel_Telu': {'number_of_characters': 65945, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 133.13, 'max_document_length': 149, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgk_Cyrl-tgk_Cyrl': {'number_of_characters': 67829, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 136.99, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgk_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tgk_Cyrl': {'number_of_characters': 67829, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 136.99, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgl_Latn-tgl_Latn': {'number_of_characters': 75087, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 151.87, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tgl_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tgl_Latn': {'number_of_characters': 75087, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 151.87, 'max_document_length': 184, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tha_Thai-tha_Thai': {'number_of_characters': 54496, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 109.67, 'max_document_length': 123, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tha_Thai-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tha_Thai': {'number_of_characters': 54496, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 109.67, 'max_document_length': 123, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tir_Ethi-tir_Ethi': {'number_of_characters': 47775, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 95.9, 'max_document_length': 110, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tir_Ethi-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tir_Ethi': {'number_of_characters': 47775, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 95.9, 'max_document_length': 110, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tsn_Latn-tsn_Latn': {'number_of_characters': 79391, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 160.69, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tsn_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tsn_Latn': {'number_of_characters': 79391, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 160.69, 'max_document_length': 204, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tso_Latn-tso_Latn': {'number_of_characters': 83501, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 169.11, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tso_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tso_Latn': {'number_of_characters': 83501, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 169.11, 'max_document_length': 215, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tur_Latn-tur_Latn': {'number_of_characters': 65382, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 131.98, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'tur_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-tur_Latn': {'number_of_characters': 65382, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 131.98, 'max_document_length': 158, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ukr_Cyrl-ukr_Cyrl': {'number_of_characters': 65850, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 132.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ukr_Cyrl-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-ukr_Cyrl': {'number_of_characters': 65850, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 132.94, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Arab-urd_Arab': {'number_of_characters': 64450, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 130.07, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Arab-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-urd_Arab': {'number_of_characters': 64450, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 130.07, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Latn-urd_Latn': {'number_of_characters': 82039, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 166.11, 'max_document_length': 230, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-urd_Latn': {'number_of_characters': 82039, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 166.11, 'max_document_length': 230, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'uzn_Latn-uzn_Latn': {'number_of_characters': 70828, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 143.14, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'uzn_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-uzn_Latn': {'number_of_characters': 70828, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 143.14, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'vie_Latn-vie_Latn': {'number_of_characters': 66724, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 134.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'vie_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-vie_Latn': {'number_of_characters': 66724, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 134.73, 'max_document_length': 161, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'war_Latn-war_Latn': {'number_of_characters': 78444, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 158.75, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'war_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-war_Latn': {'number_of_characters': 78444, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 158.75, 'max_document_length': 207, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'wol_Latn-wol_Latn': {'number_of_characters': 64521, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 130.22, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'wol_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-wol_Latn': {'number_of_characters': 64521, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 130.22, 'max_document_length': 139, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'xho_Latn-xho_Latn': {'number_of_characters': 71629, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.78, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'xho_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-xho_Latn': {'number_of_characters': 71629, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 16, 'average_document_length': 144.78, 'max_document_length': 179, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'yor_Latn-yor_Latn': {'number_of_characters': 62752, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 126.59, 'max_document_length': 143, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'yor_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-yor_Latn': {'number_of_characters': 62752, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 126.59, 'max_document_length': 143, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hans-zho_Hans': {'number_of_characters': 20549, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 40.11, 'max_document_length': 64, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hans-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zho_Hans': {'number_of_characters': 20549, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 40.11, 'max_document_length': 64, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hant-zho_Hant': {'number_of_characters': 19947, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 38.88, 'max_document_length': 45, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zho_Hant-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zho_Hant': {'number_of_characters': 19947, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 7, 'average_document_length': 38.88, 'max_document_length': 45, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zsm_Latn-zsm_Latn': {'number_of_characters': 72008, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 145.56, 'max_document_length': 210, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zsm_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zsm_Latn': {'number_of_characters': 72008, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 13, 'average_document_length': 145.56, 'max_document_length': 210, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zul_Latn-zul_Latn': {'number_of_characters': 69413, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.24, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'zul_Latn-eng_Latn': {'number_of_characters': 70589, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 142.65, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'eng_Latn-zul_Latn': {'number_of_characters': 69413, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 140.24, 'max_document_length': 171, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Arab-arb_Latn': {'number_of_characters': 61298, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 12, 'average_document_length': 123.61, 'max_document_length': 160, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'arb_Latn-arb_Arab': {'number_of_characters': 53671, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 107.98, 'max_document_length': 134, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Beng-ben_Latn': {'number_of_characters': 68285, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 137.93, 'max_document_length': 185, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'ben_Latn-ben_Beng': {'number_of_characters': 63512, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 9, 'average_document_length': 128.15, 'max_document_length': 175, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Deva-hin_Latn': {'number_of_characters': 68307, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 137.97, 'max_document_length': 170, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'hin_Latn-hin_Deva': {'number_of_characters': 66332, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 14, 'average_document_length': 133.93, 'max_document_length': 165, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Deva-npi_Latn': {'number_of_characters': 65683, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 20, 'average_document_length': 132.6, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'npi_Latn-npi_Deva': {'number_of_characters': 61183, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 18, 'average_document_length': 123.38, 'max_document_length': 154, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Sinh-sin_Latn': {'number_of_characters': 85996, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 19, 'average_document_length': 174.22, 'max_document_length': 224, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'sin_Latn-sin_Sinh': {'number_of_characters': 63902, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 17, 'average_document_length': 128.95, 'max_document_length': 159, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Arab-urd_Latn': {'number_of_characters': 82039, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 15, 'average_document_length': 166.11, 'max_document_length': 230, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}, 'urd_Latn-urd_Arab': {'number_of_characters': 64450, 'num_samples': 1388, 'num_queries': 900, 'num_documents': 488, 'min_document_length': 11, 'average_document_length': 130.07, 'max_document_length': 187, 'unique_documents': 488, 'min_query_length': 2, 'average_query_length': 1.08, 'max_query_length': 2, 'unique_queries': 900, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 488}}}} | | [BengaliDocumentClassification](https://aclanthology.org/2023.eacl-main.4) | ['ben'] | Classification | s2s | [News, Written] | None | None | | [BengaliHateSpeechClassification](https://huggingface.co/datasets/bn_hate_speech) (Karim et al., 2020) | ['ben'] | Classification | s2s | [News, Written] | None | None | | [BengaliSentimentAnalysis](https://data.mendeley.com/datasets/p6zc7krs37/4) (Sazzed et al., 2020) | ['ben'] | Classification | s2s | [Reviews, Written] | None | None | +| [BeytooteClustering](https://mcinext.com/) | ['fas'] | Clustering | p2p | [News] | None | None | | [BibleNLPBitextMining](https://arxiv.org/abs/2304.09919) (Akerman et al., 2023) | ['aai', 'aak', 'aau', 'aaz', 'abt', 'abx', 'aby', 'acf', 'acr', 'acu', 'adz', 'aer', 'aey', 'agd', 'agg', 'agm', 'agn', 'agr', 'agt', 'agu', 'aia', 'aii', 'aka', 'ake', 'alp', 'alq', 'als', 'aly', 'ame', 'amf', 'amk', 'amm', 'amn', 'amo', 'amp', 'amr', 'amu', 'amx', 'anh', 'anv', 'aoi', 'aoj', 'aom', 'aon', 'apb', 'ape', 'apn', 'apr', 'apu', 'apw', 'apz', 'arb', 'are', 'arl', 'arn', 'arp', 'asm', 'aso', 'ata', 'atb', 'atd', 'atg', 'att', 'auc', 'aui', 'auy', 'avt', 'awb', 'awk', 'awx', 'azb', 'azg', 'azz', 'bao', 'bba', 'bbb', 'bbr', 'bch', 'bco', 'bdd', 'bea', 'bef', 'bel', 'ben', 'beo', 'beu', 'bgs', 'bgt', 'bhg', 'bhl', 'big', 'bjk', 'bjp', 'bjr', 'bjv', 'bjz', 'bkd', 'bki', 'bkq', 'bkx', 'blw', 'blz', 'bmh', 'bmk', 'bmr', 'bmu', 'bnp', 'boa', 'boj', 'bon', 'box', 'bpr', 'bps', 'bqc', 'bqp', 'bre', 'bsj', 'bsn', 'bsp', 'bss', 'buk', 'bus', 'bvd', 'bvr', 'bxh', 'byr', 'byx', 'bzd', 'bzh', 'bzj', 'caa', 'cab', 'cac', 'caf', 'cak', 'cao', 'cap', 'car', 'cav', 'cax', 'cbc', 'cbi', 'cbk', 'cbr', 'cbs', 'cbt', 'cbu', 'cbv', 'cco', 'ceb', 'cek', 'ces', 'cgc', 'cha', 'chd', 'chf', 'chk', 'chq', 'chz', 'cjo', 'cjv', 'ckb', 'cle', 'clu', 'cme', 'cmn', 'cni', 'cnl', 'cnt', 'cof', 'con', 'cop', 'cot', 'cpa', 'cpb', 'cpc', 'cpu', 'cpy', 'crn', 'crx', 'cso', 'csy', 'cta', 'cth', 'ctp', 'ctu', 'cub', 'cuc', 'cui', 'cuk', 'cut', 'cux', 'cwe', 'cya', 'daa', 'dad', 'dah', 'dan', 'ded', 'deu', 'dgc', 'dgr', 'dgz', 'dhg', 'dif', 'dik', 'dji', 'djk', 'djr', 'dob', 'dop', 'dov', 'dwr', 'dww', 'dwy', 'ebk', 'eko', 'emi', 'emp', 'eng', 'enq', 'epo', 'eri', 'ese', 'esk', 'etr', 'ewe', 'faa', 'fai', 'far', 'ffm', 'for', 'fra', 'fue', 'fuf', 'fuh', 'gah', 'gai', 'gam', 'gaw', 'gdn', 'gdr', 'geb', 'gfk', 'ghs', 'glk', 'gmv', 'gng', 'gnn', 'gnw', 'gof', 'grc', 'gub', 'guh', 'gui', 'guj', 'gul', 'gum', 'gun', 'guo', 'gup', 'gux', 'gvc', 'gvf', 'gvn', 'gvs', 'gwi', 'gym', 'gyr', 'hat', 'hau', 'haw', 'hbo', 'hch', 'heb', 'heg', 'hin', 'hix', 'hla', 'hlt', 'hmo', 'hns', 'hop', 'hot', 'hrv', 'hto', 'hub', 'hui', 'hun', 'hus', 'huu', 'huv', 'hvn', 'ian', 'ign', 'ikk', 'ikw', 'ilo', 'imo', 'inb', 'ind', 'ino', 'iou', 'ipi', 'isn', 'ita', 'iws', 'ixl', 'jac', 'jae', 'jao', 'jic', 'jid', 'jiv', 'jni', 'jpn', 'jvn', 'kan', 'kaq', 'kbc', 'kbh', 'kbm', 'kbq', 'kdc', 'kde', 'kdl', 'kek', 'ken', 'kew', 'kgf', 'kgk', 'kgp', 'khs', 'khz', 'kik', 'kiw', 'kiz', 'kje', 'kjs', 'kkc', 'kkl', 'klt', 'klv', 'kmg', 'kmh', 'kmk', 'kmo', 'kms', 'kmu', 'kne', 'knf', 'knj', 'knv', 'kos', 'kpf', 'kpg', 'kpj', 'kpr', 'kpw', 'kpx', 'kqa', 'kqc', 'kqf', 'kql', 'kqw', 'ksd', 'ksj', 'ksr', 'ktm', 'kto', 'kud', 'kue', 'kup', 'kvg', 'kvn', 'kwd', 'kwf', 'kwi', 'kwj', 'kyc', 'kyf', 'kyg', 'kyq', 'kyz', 'kze', 'lac', 'lat', 'lbb', 'lbk', 'lcm', 'leu', 'lex', 'lgl', 'lid', 'lif', 'lin', 'lit', 'llg', 'lug', 'luo', 'lww', 'maa', 'maj', 'mal', 'mam', 'maq', 'mar', 'mau', 'mav', 'maz', 'mbb', 'mbc', 'mbh', 'mbj', 'mbl', 'mbs', 'mbt', 'mca', 'mcb', 'mcd', 'mcf', 'mco', 'mcp', 'mcq', 'mcr', 'mdy', 'med', 'mee', 'mek', 'meq', 'met', 'meu', 'mgc', 'mgh', 'mgw', 'mhl', 'mib', 'mic', 'mie', 'mig', 'mih', 'mil', 'mio', 'mir', 'mit', 'miz', 'mjc', 'mkj', 'mkl', 'mkn', 'mks', 'mle', 'mlh', 'mlp', 'mmo', 'mmx', 'mna', 'mop', 'mox', 'mph', 'mpj', 'mpm', 'mpp', 'mps', 'mpt', 'mpx', 'mqb', 'mqj', 'msb', 'msc', 'msk', 'msm', 'msy', 'mti', 'mto', 'mux', 'muy', 'mva', 'mvn', 'mwc', 'mwe', 'mwf', 'mwp', 'mxb', 'mxp', 'mxq', 'mxt', 'mya', 'myk', 'myu', 'myw', 'myy', 'mzz', 'nab', 'naf', 'nak', 'nas', 'nbq', 'nca', 'nch', 'ncj', 'ncl', 'ncu', 'ndg', 'ndj', 'nfa', 'ngp', 'ngu', 'nhe', 'nhg', 'nhi', 'nho', 'nhr', 'nhu', 'nhw', 'nhy', 'nif', 'nii', 'nin', 'nko', 'nld', 'nlg', 'nna', 'nnq', 'noa', 'nop', 'not', 'nou', 'npi', 'npl', 'nsn', 'nss', 'ntj', 'ntp', 'ntu', 'nuy', 'nvm', 'nwi', 'nya', 'nys', 'nyu', 'obo', 'okv', 'omw', 'ong', 'ons', 'ood', 'opm', 'ory', 'ote', 'otm', 'otn', 'otq', 'ots', 'pab', 'pad', 'pah', 'pan', 'pao', 'pes', 'pib', 'pio', 'pir', 'piu', 'pjt', 'pls', 'plu', 'pma', 'poe', 'poh', 'poi', 'pol', 'pon', 'por', 'poy', 'ppo', 'prf', 'pri', 'ptp', 'ptu', 'pwg', 'qub', 'quc', 'quf', 'quh', 'qul', 'qup', 'qvc', 'qve', 'qvh', 'qvm', 'qvn', 'qvs', 'qvw', 'qvz', 'qwh', 'qxh', 'qxn', 'qxo', 'rai', 'reg', 'rgu', 'rkb', 'rmc', 'rmy', 'ron', 'roo', 'rop', 'row', 'rro', 'ruf', 'rug', 'rus', 'rwo', 'sab', 'san', 'sbe', 'sbk', 'sbs', 'seh', 'sey', 'sgb', 'sgz', 'shj', 'shp', 'sim', 'sja', 'sll', 'smk', 'snc', 'snn', 'snp', 'snx', 'sny', 'som', 'soq', 'soy', 'spa', 'spl', 'spm', 'spp', 'sps', 'spy', 'sri', 'srm', 'srn', 'srp', 'srq', 'ssd', 'ssg', 'ssx', 'stp', 'sua', 'sue', 'sus', 'suz', 'swe', 'swh', 'swp', 'sxb', 'tac', 'taj', 'tam', 'tav', 'taw', 'tbc', 'tbf', 'tbg', 'tbo', 'tbz', 'tca', 'tcs', 'tcz', 'tdt', 'tee', 'tel', 'ter', 'tet', 'tew', 'tfr', 'tgk', 'tgl', 'tgo', 'tgp', 'tha', 'tif', 'tim', 'tiw', 'tiy', 'tke', 'tku', 'tlf', 'tmd', 'tna', 'tnc', 'tnk', 'tnn', 'tnp', 'toc', 'tod', 'tof', 'toj', 'ton', 'too', 'top', 'tos', 'tpa', 'tpi', 'tpt', 'tpz', 'trc', 'tsw', 'ttc', 'tte', 'tuc', 'tue', 'tuf', 'tuo', 'tur', 'tvk', 'twi', 'txq', 'txu', 'tzj', 'tzo', 'ubr', 'ubu', 'udu', 'uig', 'ukr', 'uli', 'ulk', 'upv', 'ura', 'urb', 'urd', 'uri', 'urt', 'urw', 'usa', 'usp', 'uvh', 'uvl', 'vid', 'vie', 'viv', 'vmy', 'waj', 'wal', 'wap', 'wat', 'wbi', 'wbp', 'wed', 'wer', 'wim', 'wiu', 'wiv', 'wmt', 'wmw', 'wnc', 'wnu', 'wol', 'wos', 'wrk', 'wro', 'wrs', 'wsk', 'wuv', 'xav', 'xbi', 'xed', 'xla', 'xnn', 'xon', 'xsi', 'xtd', 'xtm', 'yaa', 'yad', 'yal', 'yap', 'yaq', 'yby', 'ycn', 'yka', 'yle', 'yml', 'yon', 'yor', 'yrb', 'yre', 'yss', 'yuj', 'yut', 'yuw', 'yva', 'zaa', 'zab', 'zac', 'zad', 'zai', 'zaj', 'zam', 'zao', 'zap', 'zar', 'zas', 'zat', 'zav', 'zaw', 'zca', 'zga', 'zia', 'ziw', 'zlm', 'zos', 'zpc', 'zpl', 'zpm', 'zpo', 'zpq', 'zpu', 'zpv', 'zpz', 'zsr', 'ztq', 'zty', 'zyp'] | BitextMining | s2s | [Religious, Written] | None | None | | [BigPatentClustering.v2](https://huggingface.co/datasets/NortheasternUniversity/big_patent) (Eva Sharma and Chen Li and Lu Wang, 2019) | ['eng'] | Clustering | p2p | [Legal, Written] | None | None | | [BiorxivClusteringP2P.v2](https://api.biorxiv.org/) | ['eng'] | Clustering | p2p | [Academic, Written] | None | None | | [BiorxivClusteringS2S.v2](https://api.biorxiv.org/) | ['eng'] | Clustering | s2s | [Academic, Written] | None | None | +| [Birdsnap](https://openaccess.thecvf.com/content_cvpr_2014/html/Berg_Birdsnap_Large-scale_Fine-grained_2014_CVPR_paper.html) (Berg et al., 2014) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [BirdsnapZeroShot](https://openaccess.thecvf.com/content_cvpr_2014/html/Berg_Birdsnap_Large-scale_Fine-grained_2014_CVPR_paper.html) (Berg et al., 2014) | ['eng'] | ZeroShotClassification | i2t | [Encyclopaedic] | None | None | | [BlurbsClusteringP2P.v2](https://www.inf.uni-hamburg.de/en/inst/ab/lt/resources/data/germeval-2019-hmc.html) (Steffen Remus, 2019) | ['deu'] | Clustering | p2p | [Fiction, Written] | None | None | | [BlurbsClusteringS2S.v2](https://www.inf.uni-hamburg.de/en/inst/ab/lt/resources/data/germeval-2019-hmc.html) (Steffen Remus, 2019) | ['deu'] | Clustering | s2s | [Fiction, Written] | None | None | -| [BornholmBitextMining](https://aclanthology.org/W19-6138/) | ['dan'] | BitextMining | s2s | [Web, Social, Fiction, Written] | {'test': 500} | {'test': {'num_samples': 500, 'number_of_characters': 44361, 'unique_pairs': 500, 'min_sentence1_length': 1, 'average_sentence1_length': 49.83, 'max_sentence1_length': 555, 'unique_sentence1': 497, 'min_sentence2_length': 5, 'average_sentence2_length': 38.89, 'max_sentence2_length': 453, 'unique_sentence2': 491}} | +| [BornholmBitextMining](https://aclanthology.org/W19-6138/) | ['dan'] | BitextMining | s2s | [Fiction, Social, Web, Written] | {'test': 500} | {'test': {'num_samples': 500, 'number_of_characters': 44361, 'unique_pairs': 500, 'min_sentence1_length': 1, 'average_sentence1_length': 49.83, 'max_sentence1_length': 555, 'unique_sentence1': 497, 'min_sentence2_length': 5, 'average_sentence2_length': 38.89, 'max_sentence2_length': 453, 'unique_sentence2': 491}} | | [BrazilianToxicTweetsClassification](https://paperswithcode.com/dataset/told-br) (Joao Augusto Leite and Diego F. Silva and Kalina Bontcheva and Carolina Scarton, 2020) | ['por'] | MultilabelClassification | s2s | [Constructed, Written] | None | None | -| [BrightRetrieval](https://huggingface.co/datasets/xlangai/BRIGHT) (Hongjin Su, 2024) | ['eng'] | Retrieval | s2p | [Non-fiction] | None | None | +| [BrightRetrieval](https://huggingface.co/datasets/xlangai/BRIGHT) (Hongjin Su, 2024) | ['eng'] | Retrieval | s2p | [Non-fiction, Written] | None | None | | [BulgarianStoreReviewSentimentClassfication](https://doi.org/10.7910/DVN/TXIK9P) (Georgieva-Trifonova et al., 2018) | ['bul'] | Classification | s2s | [Reviews, Written] | None | None | -| [CBD](http://2019.poleval.pl/files/poleval2019.pdf) | ['pol'] | Classification | s2s | [Written, Social] | None | None | +| [CBD](http://2019.poleval.pl/files/poleval2019.pdf) | ['pol'] | Classification | s2s | [Social, Written] | None | None | | [CDSC-E](https://aclanthology.org/P17-1073.pdf) | ['pol'] | PairClassification | s2s | [Written] | None | None | -<<<<<<< HEAD -| [CDSC-R](https://aclanthology.org/P17-1073.pdf) | ['pol'] | STS | s2s | [Web, Written] | {'test': 1000} | {'test': 75.24} | -| [CEDRClassification](https://www.sciencedirect.com/science/article/pii/S1877050921013247) (Sboev et al., 2021) | ['rus'] | MultilabelClassification | s2s | [Web, Social, Blog, Written] | {'test': 1882} | {'test': {'average_text_length': 91.20563230605738, 'average_label_per_text': 0.620616365568544, 'num_samples': 1882, 'unique_labels': 6, 'labels': {'null': {'count': 734}, '3': {'count': 141}, '2': {'count': 170}, '1': {'count': 379}, '0': {'count': 353}, '4': {'count': 125}}}} | -| [CLSClusteringP2P.v2](https://arxiv.org/abs/2209.05034) (Yudong Li, 2022) | ['cmn'] | Clustering | p2p | [Academic, Written] | {'test': 2048} | {} | -| [CLSClusteringS2S.v2](https://arxiv.org/abs/2209.05034) (Yudong Li, 2022) | ['cmn'] | Clustering | s2s | [Academic, Written] | {'test': 2048} | {} | -| [CMedQAv1-reranking](https://github.com/zhangsheng93/cMedQA) (Zhang et al., 2017) | ['cmn'] | Reranking | s2s | [Medical, Written] | {'test': 2000} | {'test': 165} | -| [CMedQAv2-reranking](https://github.com/zhangsheng93/cMedQA2) (S. Zhang, 2018) | ['cmn'] | Reranking | s2s | | None | None | -| [COIRCodeSearchNetRetrieval](https://huggingface.co/datasets/code_search_net/) (Husain et al., 2019) | ['go', 'java', 'javascript', 'php', 'python', 'ruby'] | Retrieval | p2p | [Programming, Written] | {'test': 1000} | {'test': {'python': {'average_document_length': 466.546, 'average_query_length': 862.842, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'javascript': {'average_document_length': 186.018, 'average_query_length': 1415.632, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'go': {'average_document_length': 125.213, 'average_query_length': 563.729, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'ruby': {'average_document_length': 313.818, 'average_query_length': 577.634, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'java': {'average_document_length': 420.287, 'average_query_length': 690.36, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'php': {'average_document_length': 162.119, 'average_query_length': 712.129, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}}} | -| [CPUSpeedTask](https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/blob/c8376f967d1294419be1d3eb41217d04cd3a65d3/src/seb/registered_tasks/speed.py#L83-L96) | ['eng'] | Speed | s2s | [Fiction, Written] | {'test': 1} | {'test': 3591} | -| [CQADupstackAndroidRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 593.701974084703, 'average_query_length': 51.76680972818312, 'num_documents': 22998, 'num_queries': 699, 'average_relevant_docs_per_query': 2.4263233190271816}} | -| [CQADupstackEnglishRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 482.4710971880361, 'average_query_length': 48.32993630573248, 'num_documents': 40221, 'num_queries': 1570, 'average_relevant_docs_per_query': 2.3980891719745223}} | -| [CQADupstackGamingRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 488.74152888457206, 'average_query_length': 48.772413793103446, 'num_documents': 45301, 'num_queries': 1595, 'average_relevant_docs_per_query': 1.418808777429467}} | -| [CQADupstackGisRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1012.167813587693, 'average_query_length': 52.2, 'num_documents': 37637, 'num_queries': 885, 'average_relevant_docs_per_query': 1.2587570621468926}} | -| [CQADupstackMathematicaRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1153.4967375037413, 'average_query_length': 48.90547263681592, 'num_documents': 16705, 'num_queries': 804, 'average_relevant_docs_per_query': 1.6890547263681592}} | -| [CQADupstackPhysicsRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 818.6476145735463, 'average_query_length': 53.36477382098171, 'num_documents': 38316, 'num_queries': 1039, 'average_relevant_docs_per_query': 1.8604427333974976}} | -| [CQADupstackProgrammersRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Programming, Written, Non-fiction] | None | {'test': {'average_document_length': 1055.7033814022875, 'average_query_length': 55.1837899543379, 'num_documents': 32176, 'num_queries': 876, 'average_relevant_docs_per_query': 1.9121004566210045}} | -| [CQADupstackStatsRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1055.1668598736662, 'average_query_length': 56.31748466257669, 'num_documents': 42269, 'num_queries': 652, 'average_relevant_docs_per_query': 1.4003067484662577}} | -| [CQADupstackTexRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1297.09043177285, 'average_query_length': 46.935306262904334, 'num_documents': 68184, 'num_queries': 2906, 'average_relevant_docs_per_query': 1.7735719201651754}} | -| [CQADupstackUnixRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1004.8120383267908, 'average_query_length': 50.32369402985075, 'num_documents': 47382, 'num_queries': 1072, 'average_relevant_docs_per_query': 1.5792910447761195}} | -| [CQADupstackWebmastersRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 707.3635736857225, 'average_query_length': 51.93478260869565, 'num_documents': 17405, 'num_queries': 506, 'average_relevant_docs_per_query': 2.7569169960474307}} | -| [CQADupstackWordpressRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1122.7690155333814, 'average_query_length': 48.7264325323475, 'num_documents': 48605, 'num_queries': 541, 'average_relevant_docs_per_query': 1.3752310536044363}} | -| [CSFDCZMovieReviewSentimentClassification](https://arxiv.org/abs/2304.01922) (Michal Štefánik, 2023) | ['ces'] | Classification | s2s | [Reviews, Written] | {'test': 2048} | {'test': 386.5} | -| [CSFDSKMovieReviewSentimentClassification](https://arxiv.org/abs/2304.01922) (Michal Štefánik, 2023) | ['slk'] | Classification | s2s | [Reviews, Written] | {'test': 2048} | {'test': 366.2} | -| [CTKFactsNLI](https://arxiv.org/abs/2201.11115) (Ullrich et al., 2023) | ['ces'] | PairClassification | s2s | [News, Written] | {'test': 375, 'validation': 305} | {'test': 225.62, 'validation': 219.32} | -| [CUADAffiliateLicenseLicenseeLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 198} | {'test': 484.11} | -| [CUADAffiliateLicenseLicensorLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 88} | {'test': 633.4} | -| [CUADAntiAssignmentLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 1172} | {'test': 340.81} | -| [CUADAuditRightsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 1216} | {'test': 337.14} | -| [CUADCapOnLiabilityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 1246} | {'test': 375.74} | -| [CUADChangeOfControlLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 416} | {'test': 391.96} | -| [CUADCompetitiveRestrictionExceptionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 220} | {'test': 433.04} | -| [CUADCovenantNotToSueLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 308} | {'test': 402.97} | -| [CUADEffectiveDateLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 236} | {'test': 277.62} | -| [CUADExclusivityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 762} | {'test': 369.17} | -| [CUADExpirationDateLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 876} | {'test': 309.27} | -| [CUADGoverningLawLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 876} | {'test': 289.87} | -| [CUADIPOwnershipAssignmentLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 576} | {'test': 414.0} | -| [CUADInsuranceLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 1030} | {'test': 365.54} | -| [CUADIrrevocableOrPerpetualLicenseLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 280} | {'test': 473.4} | -| [CUADJointIPOwnershipLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 192} | {'test': 374.17} | -| [CUADLicenseGrantLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 1396} | {'test': 409.89} | -| [CUADLiquidatedDamagesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 220} | {'test': 351.76} | -| [CUADMinimumCommitmentLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 772} | {'test': 364.16} | -| [CUADMostFavoredNationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 64} | {'test': 418.75} | -| [CUADNoSolicitOfCustomersLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 84} | {'test': 392.89} | -| [CUADNoSolicitOfEmployeesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 142} | {'test': 417.94} | -| [CUADNonCompeteLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 442} | {'test': 383.2} | -| [CUADNonDisparagementLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 100} | {'test': 403.08} | -| [CUADNonTransferableLicenseLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 542} | {'test': 399.16} | -| [CUADNoticePeriodToTerminateRenewalLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 222} | {'test': 354.85} | -| [CUADPostTerminationServicesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 808} | {'test': 422.53} | -| [CUADPriceRestrictionsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 46} | {'test': 324.71} | -| [CUADRenewalTermLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 386} | {'test': 340.87} | -| [CUADRevenueProfitSharingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 774} | {'test': 371.55} | -| [CUADRofrRofoRofnLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 690} | {'test': 395.46} | -| [CUADSourceCodeEscrowLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 118} | {'test': 399.18} | -| [CUADTerminationForConvenienceLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 430} | {'test': 326.3} | -| [CUADThirdPartyBeneficiaryLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 68} | {'test': 261.04} | -| [CUADUncappedLiabilityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 294} | {'test': 441.04} | -| [CUADUnlimitedAllYouCanEatLicenseLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 48} | {'test': 368.08} | -| [CUADVolumeRestrictionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 322} | {'test': 306.27} | -| [CUADWarrantyDurationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 320} | {'test': 352.27} | -| [CanadaTaxCourtOutcomesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 244} | {'test': 622.6} | -| [CataloniaTweetClassification](https://aclanthology.org/2020.lrec-1.171/) | ['cat', 'spa'] | Classification | s2s | [Social, Government, Written] | {'validation': 2000, 'test': 2000} | {'validation': 202.61, 'test': 200.49} | -| [ClimateFEVER](https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 538.241873443325, 'average_query_length': 123.39934853420195, 'num_documents': 5416593, 'num_queries': 1535, 'average_relevant_docs_per_query': 3.0495114006514656}} | -| [ClimateFEVERHardNegatives](https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | | {'test': 1000} | {'test': {'average_document_length': 1245.4236333727013, 'average_query_length': 121.879, 'num_documents': 47416, 'num_queries': 1000, 'average_relevant_docs_per_query': 3.048}} | -| [CmedqaRetrieval](https://aclanthology.org/2022.emnlp-main.357.pdf) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 307.7710222897771, 'average_query_length': 48.470367591897976, 'num_documents': 100001, 'num_queries': 3999, 'average_relevant_docs_per_query': 1.86271567891973}} | -| [Cmnli](https://huggingface.co/datasets/clue/viewer/cmnli) | ['cmn'] | PairClassification | s2s | | None | None | -| [CodeEditSearchRetrieval](https://huggingface.co/datasets/cassanof/CodeEditSearch/viewer) (Niklas Muennighoff, 2023) | ['c', 'c++', 'go', 'java', 'javascript', 'php', 'python', 'ruby', 'rust', 'scala', 'shell', 'swift', 'typescript'] | Retrieval | p2p | [Programming, Written] | {'train': 13000} | {'train': {'python': {'average_document_length': 597.592, 'average_query_length': 69.519, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'javascript': {'average_document_length': 582.554, 'average_query_length': 56.88, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'typescript': {'average_document_length': 580.877, 'average_query_length': 60.092, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'go': {'average_document_length': 548.498, 'average_query_length': 70.797, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'ruby': {'average_document_length': 518.895, 'average_query_length': 66.9, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'java': {'average_document_length': 620.332, 'average_query_length': 62.984, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'php': {'average_document_length': 545.452, 'average_query_length': 61.927, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'c': {'average_document_length': 475.868, 'average_query_length': 97.588, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'c++': {'average_document_length': 544.446, 'average_query_length': 114.48, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'rust': {'average_document_length': 609.548, 'average_query_length': 67.503, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'swift': {'average_document_length': 574.62, 'average_query_length': 57.279, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'scala': {'average_document_length': 495.485, 'average_query_length': 64.833, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'shell': {'average_document_length': 486.519, 'average_query_length': 72.059, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}}} | -| [CodeFeedbackMT](https://arxiv.org/abs/2402.14658) (Tianyu Zheng, 2024) | ['eng'] | Retrieval | p2p | [Programming, Written] | {'test': 1000} | {'test': {'average_document_length': 1467.879728243677, 'average_query_length': 4425.522256533855, 'num_documents': 66383, 'num_queries': 13277, 'average_relevant_docs_per_query': 1.0}} | -| [CodeFeedbackST](https://arxiv.org/abs/2407.02883) (Xiangyang Li, 2024) | ['eng'] | Retrieval | p2p | [Programming, Written] | {'test': 1000} | {'test': {'average_document_length': 1521.3317148588733, 'average_query_length': 724.2441704465598, 'num_documents': 156526, 'num_queries': 31306, 'average_relevant_docs_per_query': 1.0}} | -| [CodeSearchNetCCRetrieval](https://arxiv.org/abs/2407.02883) (Xiangyang Li, 2024) | ['go', 'java', 'javascript', 'php', 'python', 'ruby'] | Retrieval | p2p | [Programming, Written] | {'test': 1000} | {'test': {'python': {'average_document_length': 388.31577184555965, 'average_query_length': 551.7934039415471, 'num_documents': 280652, 'num_queries': 14918, 'average_relevant_docs_per_query': 1.0}, 'javascript': {'average_document_length': 276.0730050152605, 'average_query_length': 443.70707991491946, 'num_documents': 65201, 'num_queries': 3291, 'average_relevant_docs_per_query': 1.0}, 'go': {'average_document_length': 185.0307932251621, 'average_query_length': 233.76803742920464, 'num_documents': 182735, 'num_queries': 8122, 'average_relevant_docs_per_query': 1.0}, 'ruby': {'average_document_length': 214.86204146730464, 'average_query_length': 266.8731165741475, 'num_documents': 27588, 'num_queries': 1261, 'average_relevant_docs_per_query': 1.0}, 'java': {'average_document_length': 281.96280259139183, 'average_query_length': 342.5341853035144, 'num_documents': 181061, 'num_queries': 10955, 'average_relevant_docs_per_query': 1.0}, 'php': {'average_document_length': 268.9752569556027, 'average_query_length': 336.62194947909234, 'num_documents': 268237, 'num_queries': 14014, 'average_relevant_docs_per_query': 1.0}}} | -| [CodeSearchNetRetrieval](https://huggingface.co/datasets/code_search_net/) (Husain et al., 2019) | ['go', 'java', 'javascript', 'php', 'python', 'ruby'] | Retrieval | p2p | [Programming, Written] | {'test': 1000} | {'test': {'python': {'average_document_length': 862.842, 'average_query_length': 466.546, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'javascript': {'average_document_length': 1415.632, 'average_query_length': 186.018, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'go': {'average_document_length': 563.729, 'average_query_length': 125.213, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'ruby': {'average_document_length': 577.634, 'average_query_length': 313.818, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'java': {'average_document_length': 420.287, 'average_query_length': 690.36, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}, 'php': {'average_document_length': 712.129, 'average_query_length': 162.119, 'num_documents': 1000, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}}} | -| [CodeTransOceanContest](https://arxiv.org/abs/2310.04951) (Weixiang Yan, 2023) | ['c++', 'python'] | Retrieval | p2p | [Programming, Written] | | {'test': {'average_document_length': 1528.9156746031747, 'average_query_length': 1012.1131221719457, 'num_documents': 1008, 'num_queries': 221, 'average_relevant_docs_per_query': 1.0}} | -| [CodeTransOceanDL](https://arxiv.org/abs/2310.04951) (Weixiang Yan, 2023) | ['python'] | Retrieval | p2p | [Programming, Written] | | {'test': {'average_document_length': 1479.0735294117646, 'average_query_length': 1867.6222222222223, 'num_documents': 816, 'num_queries': 180, 'average_relevant_docs_per_query': 1.0}} | -| [ContractNLIConfidentialityOfAgreementLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 82} | {'test': 473.17} | -| [ContractNLIExplicitIdentificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 109} | {'test': 506.12} | -| [ContractNLIInclusionOfVerballyConveyedInformationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 139} | {'test': 525.75} | -| [ContractNLILimitedUseLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 208} | {'test': 407.51} | -| [ContractNLINoLicensingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 162} | {'test': 419.42} | -| [ContractNLINoticeOnCompelledDisclosureLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 142} | {'test': 503.45} | -| [ContractNLIPermissibleAcquirementOfSimilarInformationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 178} | {'test': 427.4} | -| [ContractNLIPermissibleCopyLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 87} | {'test': 386.84} | -| [ContractNLIPermissibleDevelopmentOfSimilarInformationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 136} | {'test': 396.4} | -| [ContractNLIPermissiblePostAgreementPossessionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 111} | {'test': 529.09} | -| [ContractNLIReturnOfConfidentialInformationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 66} | {'test': 478.29} | -| [ContractNLISharingWithEmployeesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 170} | {'test': 548.63} | -| [ContractNLISharingWithThirdPartiesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 180} | {'test': 517.29} | -| [ContractNLISurvivalOfObligationsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 157} | {'test': 417.64} | -| [Core17InstructionRetrieval](https://arxiv.org/abs/2403.15246) (Orion Weller, 2024) | ['eng'] | InstructionRetrieval | s2p | [News, Written] | {'eng': 39838} | {'test': {'num_docs': 19899, 'num_queries': 20, 'average_document_length': 2233.0329664807277, 'average_query_length': 109.75, 'average_instruction_length': 295.55, 'average_changed_instruction_length': 355.2, 'average_relevant_docs_per_query': 32.7, 'average_top_ranked_per_query': 1000.0}} | -| [CorporateLobbyingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 490} | {'test': 6039.85} | -| [CosQA](https://arxiv.org/abs/2105.13239) (Junjie Huang, 2021) | ['eng', 'python'] | Retrieval | p2p | [Programming, Written] | | {'test': {'average_document_length': 276.132741215298, 'average_query_length': 36.814, 'num_documents': 20604, 'num_queries': 500, 'average_relevant_docs_per_query': 1.0}} | -| [CovidRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 332.4152658473415, 'average_query_length': 25.9304531085353, 'num_documents': 100001, 'num_queries': 949, 'average_relevant_docs_per_query': 1.0105374077976819}} | -| [CrossLingualSemanticDiscriminationWMT19](https://huggingface.co/datasets/Andrianos/clsd_wmt19_21) | ['deu', 'fra'] | Retrieval | s2s | [News, Written] | {'test': 2946} | {'test': {'deu-fra': {'average_document_length': 147.49857433808555, 'average_query_length': 152.95587236931433, 'num_documents': 7365, 'num_queries': 1473, 'average_relevant_docs_per_query': 1.0}, 'fra-deu': {'average_document_length': 154.21968771215208, 'average_query_length': 145.877800407332, 'num_documents': 7365, 'num_queries': 1473, 'average_relevant_docs_per_query': 1.0}}} | -| [CrossLingualSemanticDiscriminationWMT21](https://huggingface.co/datasets/Andrianos/clsd_wmt19_21) | ['deu', 'fra'] | Retrieval | s2s | [News, Written] | {'test': 1786} | {'test': {'deu-fra': {'average_document_length': 177.26270996640537, 'average_query_length': 171.73012318029114, 'num_documents': 4465, 'num_queries': 893, 'average_relevant_docs_per_query': 1.0}, 'fra-deu': {'average_document_length': 174.45061590145576, 'average_query_length': 176.99216125419932, 'num_documents': 4465, 'num_queries': 893, 'average_relevant_docs_per_query': 1.0}}} | -| [CyrillicTurkicLangClassification](https://huggingface.co/datasets/tatiana-merz/cyrillic_turkic_langs) (Goldhahn et al., 2012) | ['bak', 'chv', 'kaz', 'kir', 'krc', 'rus', 'sah', 'tat', 'tyv'] | Classification | s2s | [Web, Written] | {'test': 2048} | {'test': 92.22} | -| [CzechProductReviewSentimentClassification](https://aclanthology.org/W13-1609/) | ['ces'] | Classification | s2s | [Reviews, Written] | {'test': 2048} | {'test': 153.26} | -| [CzechSoMeSentimentClassification](https://aclanthology.org/W13-1609/) | ['ces'] | Classification | s2s | [Reviews, Written] | {'test': 1000} | {'test': 59.89} | -| [CzechSubjectivityClassification](https://arxiv.org/abs/2009.08712) | ['ces'] | Classification | s2s | [Reviews, Written] | {'validation': 500, 'test': 2000} | {'validation': 108.2, 'test': 108.3} | -| [DBPedia](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['eng'] | Retrieval | s2p | [Written, Encyclopaedic] | None | {'test': {'average_document_length': 1122.7690155333814, 'average_query_length': 48.7264325323475, 'num_documents': 48605, 'num_queries': 541, 'average_relevant_docs_per_query': 1.3752310536044363}} | -| [DBPedia-PL](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['pol'] | Retrieval | s2p | [Written, Encyclopaedic] | None | {'test': {'average_document_length': 311.7007956561823, 'average_query_length': 35.45, 'num_documents': 4635922, 'num_queries': 400, 'average_relevant_docs_per_query': 38.215}} | -| [DBPedia-PLHardNegatives](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['pol'] | Retrieval | s2p | [Written, Encyclopaedic] | {'test': 400} | {'test': {'average_document_length': 363.468546000768, 'average_query_length': 35.45, 'num_documents': 88542, 'num_queries': 400, 'average_relevant_docs_per_query': 38.215}} | -| [DBPediaHardNegatives](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['eng'] | Retrieval | s2p | [Written, Encyclopaedic] | {'test': 400} | {'test': {'average_document_length': 338.58561119129564, 'average_query_length': 34.085, 'num_documents': 90070, 'num_queries': 400, 'average_relevant_docs_per_query': 38.215}} | -| [DBpediaClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [Encyclopaedic, Written] | {'test': 70000} | {'test': 281.4} | -| [DKHateClassification](https://aclanthology.org/2020.lrec-1.430/) | ['dan'] | Classification | s2s | [Social, Written] | {'test': 329} | {'test': 104.0} | -| [DalajClassification](https://spraakbanken.gu.se/en/resources/superlim) | ['swe'] | Classification | s2s | [Non-fiction, Written] | {'test': 444} | {'test': 243.8} | -| [DanFeverRetrieval](https://aclanthology.org/2021.nodalida-main.47/) | ['dan'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Spoken] | {'train': 8897} | {'train': {'average_document_length': 312.1117274167987, 'average_query_length': 50.26957476855484, 'num_documents': 2524, 'num_queries': 6373, 'average_relevant_docs_per_query': 0.48721167425074535}} | -| [DanishPoliticalCommentsClassification](https://huggingface.co/datasets/danish_political_comments) (Mads Guldborg Kjeldgaard Kongsbak, 2019) | ['dan'] | Classification | s2s | [Social, Written] | {'train': 9010} | {'train': 69.9} | -| [DefinitionClassificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 1337} | {'test': 253.72} | -| [DiaBlaBitextMining](https://inria.hal.science/hal-03021633) (González et al., 2019) | ['eng', 'fra'] | BitextMining | s2s | [Social, Written] | {} | {} | -| [Diversity1LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 300} | {'test': 103.21} | -| [Diversity2LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 300} | {'test': 0} | -| [Diversity3LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 300} | {'test': 135.46} | -| [Diversity4LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 300} | {'test': 144.52} | -| [Diversity5LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 300} | {'test': 174.77} | -| [Diversity6LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 300} | {'test': 301.01} | -| [DuRetrieval](https://aclanthology.org/2022.emnlp-main.357.pdf) (Yifu Qiu, 2022) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 331.3219967800322, 'average_query_length': 9.289, 'num_documents': 100001, 'num_queries': 2000, 'average_relevant_docs_per_query': 4.9195}} | -| [DutchBookReviewSentimentClassification](https://github.com/benjaminvdb/DBRD) (Benjamin et al., 2019) | ['nld'] | Classification | s2s | [Reviews, Written] | {'test': 2224} | {'test': 1443.0} | -| [ESCIReranking](https://github.com/amazon-science/esci-data/) (Chandan K. Reddy, 2022) | ['eng', 'jpn', 'spa'] | Reranking | s2p | [Written] | | | -| [EcomRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 32.98041664189015, 'average_query_length': 6.798, 'num_documents': 100902, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}} | -| [EightTagsClustering.v2](https://aclanthology.org/2020.lrec-1.207.pdf) | ['pol'] | Clustering | s2s | [Social, Written] | {'test': 2048} | {'test': 78.73} | -| [EmotionClassification](https://www.aclweb.org/anthology/D18-1404) | ['eng'] | Classification | s2s | [Social, Written] | {'validation': 2000, 'test': 2000} | {'validation': 95.3, 'test': 95.6} | -| [EstQA](https://www.semanticscholar.org/paper/Extractive-Question-Answering-for-Estonian-Language-182912IAPM-Alum%C3%A4e/ea4f60ab36cadca059c880678bc4c51e293a85d6?utm_source=direct_link) | ['est'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 603} | {'test': {'average_document_length': 785.595041322314, 'average_query_length': 55.32006633499171, 'num_documents': 121, 'num_queries': 603, 'average_relevant_docs_per_query': 1.0}} | -| [EstonianValenceClassification](https://figshare.com/articles/dataset/Estonian_Valence_Corpus_Eesti_valentsikorpus/24517054) | ['est'] | Classification | s2s | [News, Written] | {'train': 3270, 'test': 818} | {'train': 226.70642201834863, 'test': 231.5085574572127} | -| [FEVER](https://fever.ai/) | ['eng'] | Retrieval | s2p | | None | {'train': {'average_document_length': 538.2340070317589, 'average_query_length': 47.56034058828886, 'num_documents': 5416568, 'num_queries': 109810, 'average_relevant_docs_per_query': 1.2757034878426372}, 'dev': {'average_document_length': 538.2340070317589, 'average_query_length': 47.326282628262824, 'num_documents': 5416568, 'num_queries': 6666, 'average_relevant_docs_per_query': 1.211971197119712}, 'test': {'average_document_length': 538.2340070317589, 'average_query_length': 49.60546054605461, 'num_documents': 5416568, 'num_queries': 6666, 'average_relevant_docs_per_query': 1.1906690669066906}} | -| [FEVERHardNegatives](https://fever.ai/) | ['eng'] | Retrieval | s2p | | {'test': 1000} | {'test': {'average_document_length': 695.4370242764114, 'average_query_length': 49.62, 'num_documents': 163698, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.171}} | -| [FQuADRetrieval](https://huggingface.co/datasets/manu/fquad2_test) | ['fra'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 400, 'validation': 100} | {'test': {'average_document_length': 896.3308550185874, 'average_query_length': 58.52, 'num_documents': 269, 'num_queries': 400, 'average_relevant_docs_per_query': 1.0}, 'validation': {'average_document_length': 895.1340206185567, 'average_query_length': 54.13, 'num_documents': 97, 'num_queries': 100, 'average_relevant_docs_per_query': 1.0}} | -| [FaithDial](https://mcgill-nlp.github.io/FaithDial) (Dziri et al., 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 2042} | {'test': {'average_document_length': 140.61062447018932, 'average_query_length': 4.926542605288932, 'num_documents': 3539, 'num_queries': 2042, 'average_relevant_docs_per_query': 1.0}} | -| [FalseFriendsGermanEnglish](https://drive.google.com/file/d/1jgq0nBnV-UiYNxbKNrrr2gxDEHm-DMKH/view?usp=share_link) | ['deu'] | PairClassification | s2s | [Written] | {'test': 1524} | {'test': 40.3} | -| [FaroeseSTS](https://aclanthology.org/2023.nodalida-1.74.pdf) | ['fao'] | STS | s2s | [News, Web, Written] | {'train': 729} | {'train': 43.6} | -| [FarsTail](https://link.springer.com/article/10.1007/s00500-023-08959-3) (Amirkhani et al., 2023) | ['fas'] | PairClassification | s2s | [Academic, Written] | {'test': 1029} | {'test': 125.84} | -| [FeedbackQARetrieval](https://arxiv.org/abs/2204.03025) | ['eng'] | Retrieval | s2p | [Web, Government, Medical, Written] | {'test': 1992} | {'test': {'average_document_length': 1174.7986463620982, 'average_query_length': 72.33182730923694, 'num_documents': 2364, 'num_queries': 1992, 'average_relevant_docs_per_query': 1.0}} | -| [FiQA-PL](https://sites.google.com/view/fiqa/) (Nandan Thakur, 2021) | ['pol'] | Retrieval | s2p | | None | {'test': {'average_document_length': 795.2371699226205, 'average_query_length': 70.00771604938272, 'num_documents': 57638, 'num_queries': 648, 'average_relevant_docs_per_query': 2.632716049382716}} | -| [FiQA2018](https://sites.google.com/view/fiqa/) (Nandan Thakur, 2021) | ['eng'] | Retrieval | s2p | | None | {'train': {'average_document_length': 767.2108157812554, 'average_query_length': 61.49763636363636, 'num_documents': 57638, 'num_queries': 5500, 'average_relevant_docs_per_query': 2.5756363636363635}, 'dev': {'average_document_length': 767.2108157812554, 'average_query_length': 62.756, 'num_documents': 57638, 'num_queries': 500, 'average_relevant_docs_per_query': 2.476}, 'test': {'average_document_length': 767.2108157812554, 'average_query_length': 62.7037037037037, 'num_documents': 57638, 'num_queries': 648, 'average_relevant_docs_per_query': 2.632716049382716}} | -| [FilipinoHateSpeechClassification](https://pcj.csp.org.ph/index.php/pcj/issue/download/29/PCJ%20V14%20N1%20pp1-14%202019) (Neil Vicente Cabasag et al., 2019) | ['fil'] | Classification | s2s | [Social, Written] | {'validation': 2048, 'test': 2048} | {'validation': 88.1, 'test': 87.4} | -| [FilipinoShopeeReviewsClassification](https://uijrt.com/articles/v4/i8/UIJRTV4I80009.pdf) | ['fil'] | Classification | s2s | [Social, Written] | {'validation': 2250, 'test': 2250} | {'validation': 143.8, 'test': 145.1} | -| [FinParaSTS](https://huggingface.co/datasets/TurkuNLP/turku_paraphrase_corpus) | ['fin'] | STS | s2s | [News, Subtitles, Written] | {'test': 1000, 'validation': 1000} | {'test': 59.0, 'validation': 58.8} | -| [FinToxicityClassification](https://aclanthology.org/2023.nodalida-1.68) | ['fin'] | Classification | s2s | [News, Written] | {'train': 2048, 'test': 2048} | {'train': 432.63, 'test': 401.03} | -| [FinancialPhrasebankClassification](https://arxiv.org/abs/1307.5336) (P. Malo, 2014) | ['eng'] | Classification | s2s | [News, Written] | {'train': 4840} | {'train': 121.96} | -| [FloresBitextMining](https://huggingface.co/datasets/facebook/flores) (Goyal et al., 2022) | ['ace', 'acm', 'acq', 'aeb', 'afr', 'ajp', 'aka', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'ast', 'awa', 'ayr', 'azb', 'azj', 'bak', 'bam', 'ban', 'bel', 'bem', 'ben', 'bho', 'bjn', 'bod', 'bos', 'bug', 'bul', 'cat', 'ceb', 'ces', 'cjk', 'ckb', 'crh', 'cym', 'dan', 'deu', 'dik', 'dyu', 'dzo', 'ell', 'eng', 'epo', 'est', 'eus', 'ewe', 'fao', 'fij', 'fin', 'fon', 'fra', 'fur', 'fuv', 'gaz', 'gla', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hne', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kac', 'kam', 'kan', 'kas', 'kat', 'kaz', 'kbp', 'kea', 'khk', 'khm', 'kik', 'kin', 'kir', 'kmb', 'kmr', 'knc', 'kon', 'kor', 'lao', 'lij', 'lim', 'lin', 'lit', 'lmo', 'ltg', 'ltz', 'lua', 'lug', 'luo', 'lus', 'lvs', 'mag', 'mai', 'mal', 'mar', 'min', 'mkd', 'mlt', 'mni', 'mos', 'mri', 'mya', 'nld', 'nno', 'nob', 'npi', 'nso', 'nus', 'nya', 'oci', 'ory', 'pag', 'pan', 'pap', 'pbt', 'pes', 'plt', 'pol', 'por', 'prs', 'quy', 'ron', 'run', 'rus', 'sag', 'san', 'sat', 'scn', 'shn', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'ssw', 'sun', 'swe', 'swh', 'szl', 'tam', 'taq', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tpi', 'tsn', 'tso', 'tuk', 'tum', 'tur', 'twi', 'tzm', 'uig', 'ukr', 'umb', 'urd', 'uzn', 'vec', 'vie', 'war', 'wol', 'xho', 'ydd', 'yor', 'yue', 'zho', 'zsm', 'zul'] | BitextMining | s2s | [Non-fiction, Encyclopaedic, Written] | {'dev': 997, 'devtest': 1012} | {} | -| [FrenchBookReviews](https://huggingface.co/datasets/Abirate/french_book_reviews) | ['fra'] | Classification | s2s | [Reviews, Written] | {'train': 2048} | {'train': 311.5} | -| [FrenkEnClassification](https://arxiv.org/abs/1906.02045) (Nikola Ljubešić, 2019) | ['eng'] | Classification | s2s | [Social, Written] | {'test': 2300} | {'test': 188.75} | -| [FrenkHrClassification](https://arxiv.org/abs/1906.02045) (Nikola Ljubešić, 2019) | ['hrv'] | Classification | s2s | [Social, Written] | {'test': 2120} | {'test': 89.86} | -| [FrenkSlClassification](https://arxiv.org/pdf/1906.02045) (Nikola Ljubešić, 2019) | ['slv'] | Classification | s2s | [Social, Written] | {'test': 2177} | {'test': 136.61} | -| [FunctionOfDecisionSectionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 367} | {'test': 551.07} | -| [GPUSpeedTask](https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/blob/c8376f967d1294419be1d3eb41217d04cd3a65d3/src/seb/registered_tasks/speed.py#L83-L96) | ['eng'] | Speed | s2s | [Fiction, Written] | {'test': 1} | {'test': 3591} | -| [GeoreviewClassification](https://github.com/yandex/geo-reviews-dataset-2023) | ['rus'] | Classification | p2p | [Reviews, Written] | {'test': 2048} | {'test': 409.0} | -| [GeoreviewClusteringP2P](https://github.com/yandex/geo-reviews-dataset-2023) | ['rus'] | Clustering | p2p | [Reviews, Written] | {'test': 2000} | {'test': 384.5} | -| [GeorgianFAQRetrieval](https://huggingface.co/datasets/jupyterjazz/georgian-faq) | ['kat'] | Retrieval | s2p | [Web, Written] | {'test': 2566} | {'test': {'average_document_length': 511.24668745128605, 'average_query_length': 61.69551656920078, 'num_documents': 2566, 'num_queries': 2565, 'average_relevant_docs_per_query': 1.0003898635477584}} | -| [GerDaLIR](https://github.com/lavis-nlp/GerDaLIR) | ['deu'] | Retrieval | s2p | | None | {'test': {'average_document_length': 15483.237726805888, 'average_query_length': 1027.3495690356156, 'num_documents': 131445, 'num_queries': 12298, 'average_relevant_docs_per_query': 1.1704342169458448}} | -| [GerDaLIRSmall](https://github.com/lavis-nlp/GerDaLIR) | ['deu'] | Retrieval | p2p | [Legal, Written] | None | {'test': {'average_document_length': 19706.823653325308, 'average_query_length': 1031.0680889324833, 'num_documents': 9969, 'num_queries': 12234, 'average_relevant_docs_per_query': 1.1705084191597188}} | -| [GermanDPR](https://huggingface.co/datasets/deepset/germandpr) (Timo Möller, 2021) | ['deu'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1288.3410987482614, 'average_query_length': 64.38439024390244, 'num_documents': 2876, 'num_queries': 1025, 'average_relevant_docs_per_query': 1.0}} | -| [GermanGovServiceRetrieval](https://huggingface.co/datasets/it-at-m/LHM-Dienstleistungen-QA) | ['deu'] | Retrieval | s2p | [Government, Written] | {'test': 357} | {'test': {'average_document_length': 1246.4571428571428, 'average_query_length': 68.17977528089888, 'num_documents': 105, 'num_queries': 356, 'average_relevant_docs_per_query': 1.0}} | -| [GermanPoliticiansTwitterSentimentClassification](https://aclanthology.org/2022.konvens-1.9) | ['deu'] | Classification | s2s | [Social, Government, Written] | {'test': 357} | {'test': 302.48} | -| [GermanQuAD-Retrieval](https://www.kaggle.com/datasets/GermanQuAD) (Timo Möller, 2021) | ['deu'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1941.090717299578, 'average_query_length': 56.74773139745916, 'num_documents': 474, 'num_queries': 2204, 'average_relevant_docs_per_query': 1.0}} | -| [GermanSTSBenchmark](https://github.com/t-systems-on-site-services-gmbh/german-STSbenchmark) (Philip May, 2021) | ['deu'] | STS | s2s | | None | None | -| [GreekCivicsQA](https://huggingface.co/datasets/antoinelb7/alloprof) | ['ell'] | Retrieval | s2p | [Academic, Written] | {'default': 407} | {'default': {'average_document_length': 1074.894348894349, 'average_query_length': 77.06142506142506, 'num_documents': 407, 'num_queries': 407, 'average_relevant_docs_per_query': 1.0}} | -| [GreekLegalCodeClassification](https://arxiv.org/abs/2109.15298) | ['ell'] | Classification | s2s | [Legal, Written] | {'validation': 2048, 'test': 2048} | {'validation': 4046.8, 'test': 4200.8} | -| [GujaratiNewsClassification](https://github.com/goru001/nlp-for-gujarati) | ['guj'] | Classification | s2s | [News, Written] | {'train': 5269, 'test': 1318} | {'train': 61.95, 'test': 61.91} | -| [HALClusteringS2S.v2](https://huggingface.co/datasets/lyon-nlp/clustering-hal-s2s) (Mathieu Ciancone, 2024) | ['fra'] | Clustering | s2s | [Academic, Written] | {'test': 2048} | {'test': 86.6} | -| [HagridRetrieval](https://github.com/project-miracl/hagrid) (Ehsan Kamalloo, 2023) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | {'train': 1922} | {'dev': {'average_document_length': 228.36693548387098, 'average_query_length': 40.064516129032256, 'num_documents': 496, 'num_queries': 496, 'average_relevant_docs_per_query': 1.0}} | -| [HateSpeechPortugueseClassification](https://aclanthology.org/W19-3510) | ['por'] | Classification | s2s | [Social, Written] | {'train': 2048} | {'train': 101.02} | -| [HeadlineClassification](https://aclanthology.org/2020.ngt-1.6/) | ['rus'] | Classification | s2s | [News, Written] | {'test': 2048} | {'test': 61.6} | -| [HebrewSentimentAnalysis](https://huggingface.co/datasets/hebrew_sentiment) | ['heb'] | Classification | s2s | [Reviews, Written] | {'test': 2048} | {'test': 113.57} | -| [HellaSwag](https://rowanzellers.com/hellaswag/) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 10042} | {'test': {'average_document_length': 137.36519014671472, 'average_query_length': 224.53654650468033, 'num_documents': 199162, 'num_queries': 10042, 'average_relevant_docs_per_query': 1.0}} | -| [HinDialectClassification](https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-4839) (Bafna et al., 2022) | ['anp', 'awa', 'ben', 'bgc', 'bhb', 'bhd', 'bho', 'bjj', 'bns', 'bra', 'gbm', 'guj', 'hne', 'kfg', 'kfy', 'mag', 'mar', 'mup', 'noe', 'pan', 'raj'] | Classification | s2s | [Social, Spoken, Written] | {'test': 1152} | {'test': 583.82} | -| [HindiDiscourseClassification](https://aclanthology.org/2020.lrec-1.149/) | ['hin'] | Classification | s2s | [Fiction, Social, Written] | {'train': 2048} | {'train': 79.23828125} | -| [HotelReviewSentimentClassification](https://link.springer.com/chapter/10.1007/978-3-319-67056-0_3) (Elnagar et al., 2018) | ['ara'] | Classification | s2s | [Reviews, Written] | {'train': 2048} | {'train': 137.2} | -| [HotpotQA](https://hotpotqa.github.io/) | ['eng'] | Retrieval | s2p | [Web, Written] | None | {'train': {'average_document_length': 287.9079517072212, 'average_query_length': 105.54965882352941, 'num_documents': 5233329, 'num_queries': 85000, 'average_relevant_docs_per_query': 2.0}, 'dev': {'average_document_length': 287.9079517072212, 'average_query_length': 105.35634294106848, 'num_documents': 5233329, 'num_queries': 5447, 'average_relevant_docs_per_query': 2.0}, 'test': {'average_document_length': 287.9079517072212, 'average_query_length': 92.17096556380824, 'num_documents': 5233329, 'num_queries': 7405, 'average_relevant_docs_per_query': 2.0}} | -| [HotpotQA-PL](https://hotpotqa.github.io/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | {'test': {'average_document_length': 292.26835882093405, 'average_query_length': 94.64064821066847, 'num_documents': 5233329, 'num_queries': 7405, 'average_relevant_docs_per_query': 2.0}} | -| [HotpotQA-PLHardNegatives](https://hotpotqa.github.io/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | {'test': 1000} | {'test': {'average_document_length': 438.3888210025661, 'average_query_length': 95.161, 'num_documents': 212774, 'num_queries': 1000, 'average_relevant_docs_per_query': 2.0}} | -| [HotpotQAHardNegatives](https://hotpotqa.github.io/) | ['eng'] | Retrieval | s2p | [Web, Written] | {'test': 1000} | {'test': {'average_document_length': 373.558822095461, 'average_query_length': 92.584, 'num_documents': 225621, 'num_queries': 1000, 'average_relevant_docs_per_query': 2.0}} | -| [HunSum2AbstractiveRetrieval](https://arxiv.org/abs/2404.03555) (Botond Barta, 2024) | ['hun'] | Retrieval | s2p | [News, Written] | {'test': 1998} | {'test': {'average_document_length': 2511.0315315315315, 'average_query_length': 201.2112112112112, 'num_documents': 1998, 'num_queries': 1998, 'average_relevant_docs_per_query': 1.0}} | -======= | [CDSC-R](https://aclanthology.org/P17-1073.pdf) | ['pol'] | STS | s2s | [Web, Written] | None | None | -| [CEDRClassification](https://www.sciencedirect.com/science/article/pii/S1877050921013247) (Sboev et al., 2021) | ['rus'] | MultilabelClassification | s2s | [Web, Social, Blog, Written] | {'test': 1882, 'train': 7528} | {'test': {'num_samples': 1882, 'number_of_characters': 171649, 'number_texts_in_train': 7, 'min_text_length': 6, 'average_text_length': 91.21, 'max_text_length': 220, 'unique_texts': 1875, 'min_labels_per_text': 0, 'average_label_per_text': 0.62, 'max_labels_per_text': 2, 'unique_labels': 6, 'labels': {'None': {'count': 734}, '3': {'count': 141}, '2': {'count': 170}, '1': {'count': 379}, '0': {'count': 353}, '4': {'count': 125}}}, 'train': {'num_samples': 7528, 'number_of_characters': 697322, 'number_texts_in_train': None, 'min_text_length': 5, 'average_text_length': 92.63, 'max_text_length': 280, 'unique_texts': 7500, 'min_labels_per_text': 0, 'average_label_per_text': 0.61, 'max_labels_per_text': 3, 'unique_labels': 6, 'labels': {'None': {'count': 3043}, '2': {'count': 607}, '0': {'count': 1569}, '3': {'count': 589}, '1': {'count': 1417}, '4': {'count': 411}}}} | +| [CEDRClassification](https://www.sciencedirect.com/science/article/pii/S1877050921013247) (Sboev et al., 2021) | ['rus'] | MultilabelClassification | s2s | [Blog, Social, Web, Written] | {'test': 1882, 'train': 7528} | {'test': {'num_samples': 1882, 'number_of_characters': 171649, 'number_texts_in_train': 7, 'min_text_length': 6, 'average_text_length': 91.21, 'max_text_length': 220, 'unique_texts': 1875, 'min_labels_per_text': 0, 'average_label_per_text': 0.62, 'max_labels_per_text': 2, 'unique_labels': 6, 'labels': {'None': {'count': 734}, '3': {'count': 141}, '2': {'count': 170}, '1': {'count': 379}, '0': {'count': 353}, '4': {'count': 125}}}, 'train': {'num_samples': 7528, 'number_of_characters': 697322, 'number_texts_in_train': None, 'min_text_length': 5, 'average_text_length': 92.63, 'max_text_length': 280, 'unique_texts': 7500, 'min_labels_per_text': 0, 'average_label_per_text': 0.61, 'max_labels_per_text': 3, 'unique_labels': 6, 'labels': {'None': {'count': 3043}, '2': {'count': 607}, '0': {'count': 1569}, '3': {'count': 589}, '1': {'count': 1417}, '4': {'count': 411}}}} | +| [CExaPPC](https://github.com/exaco/exappc) | ['fas'] | PairClassification | s2s | [Social, Web] | None | None | +| [CIFAR10](https://huggingface.co/datasets/uoft-cs/cifar10) (Alex Krizhevsky, 2009) | ['eng'] | ImageClassification | i2i | [Web] | None | None | +| [CIFAR100](https://huggingface.co/datasets/uoft-cs/cifar100) (Alex Krizhevsky, 2009) | ['eng'] | ImageClassification | i2t | [Web] | None | None | +| [CIFAR100Clustering](https://huggingface.co/datasets/uoft-cs/cifar100) (Alex Krizhevsky, 2009) | ['eng'] | ImageClustering | i2t | [Web] | None | None | +| [CIFAR100ZeroShot](https://huggingface.co/datasets/uoft-cs/cifar100) (Alex Krizhevsky, 2009) | ['eng'] | ZeroShotClassification | i2t | [Web] | None | None | +| [CIFAR10Clustering](https://huggingface.co/datasets/uoft-cs/cifar10) (Alex Krizhevsky, 2009) | ['eng'] | ImageClustering | i2i | [Web] | None | None | +| [CIFAR10ZeroShot](https://huggingface.co/datasets/uoft-cs/cifar10) (Alex Krizhevsky, 2009) | ['eng'] | ZeroShotClassification | i2t | [Web] | None | None | +| [CIRRIT2IRetrieval](https://openaccess.thecvf.com/content/ICCV2021/html/Liu_Image_Retrieval_on_Real-Life_Images_With_Pre-Trained_Vision-and-Language_Models_ICCV_2021_paper.html) (Liu et al., 2021) | ['eng'] | Any2AnyRetrieval | it2i | [Encyclopaedic] | None | None | +| [CLEVRCountZeroShot](https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html) (Johnson et al., 2017) | ['eng'] | ZeroShotClassification | i2t | [Constructed] | None | None | +| [CLEVRZeroShot](https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html) (Johnson et al., 2017) | ['eng'] | ZeroShotClassification | i2t | [Constructed] | None | None | | [CLSClusteringP2P.v2](https://arxiv.org/abs/2209.05034) (Yudong Li, 2022) | ['cmn'] | Clustering | p2p | [Academic, Written] | None | None | | [CLSClusteringS2S.v2](https://arxiv.org/abs/2209.05034) (Yudong Li, 2022) | ['cmn'] | Clustering | s2s | [Academic, Written] | None | None | | [CMedQAv1-reranking](https://github.com/zhangsheng93/cMedQA) (Zhang et al., 2017) | ['cmn'] | Reranking | s2s | [Medical, Written] | None | None | | [CMedQAv2-reranking](https://github.com/zhangsheng93/cMedQA2) (S. Zhang, 2018) | ['cmn'] | Reranking | s2s | [Medical, Written] | None | None | | [COIRCodeSearchNetRetrieval](https://huggingface.co/datasets/code_search_net/) (Husain et al., 2019) | ['go', 'java', 'javascript', 'php', 'python', 'ruby'] | Retrieval | p2p | [Programming, Written] | {'test': 1056326} | {'test': {'number_of_characters': 36843313, 'num_samples': 1056326, 'num_queries': 52561, 'num_documents': 1003765, 'min_document_length': 54, 'average_document_length': 34.71, 'max_document_length': 334374, 'unique_documents': 1003765, 'min_query_length': 2, 'average_query_length': 38.19, 'max_query_length': 2, 'unique_queries': 52561, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 52561, 'hf_subset_descriptive_stats': {'python': {'number_of_characters': 14574651, 'num_samples': 295228, 'num_queries': 14918, 'num_documents': 280310, 'min_document_length': 95, 'average_document_length': 49.99, 'max_document_length': 14008, 'unique_documents': 280310, 'min_query_length': 2, 'average_query_length': 37.58, 'max_query_length': 2, 'unique_queries': 14918, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 14918}, 'javascript': {'number_of_characters': 2587540, 'num_samples': 68145, 'num_queries': 3291, 'num_documents': 64854, 'min_document_length': 87, 'average_document_length': 37.9, 'max_document_length': 334374, 'unique_documents': 64854, 'min_query_length': 2, 'average_query_length': 39.41, 'max_query_length': 2, 'unique_queries': 3291, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 3291}, 'go': {'number_of_characters': 3641108, 'num_samples': 190562, 'num_queries': 8122, 'num_documents': 182440, 'min_document_length': 54, 'average_document_length': 17.96, 'max_document_length': 5280, 'unique_documents': 182440, 'min_query_length': 2, 'average_query_length': 44.92, 'max_query_length': 2, 'unique_queries': 8122, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 8122}, 'ruby': {'number_of_characters': 629446, 'num_samples': 28831, 'num_queries': 1261, 'num_documents': 27570, 'min_document_length': 83, 'average_document_length': 20.83, 'max_document_length': 3992, 'unique_documents': 27570, 'min_query_length': 2, 'average_query_length': 43.73, 'max_query_length': 2, 'unique_queries': 1261, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1261}, 'java': {'number_of_characters': 6791137, 'num_samples': 191821, 'num_queries': 10955, 'num_documents': 180866, 'min_document_length': 77, 'average_document_length': 35.55, 'max_document_length': 7615, 'unique_documents': 180866, 'min_query_length': 2, 'average_query_length': 33.02, 'max_query_length': 2, 'unique_queries': 10955, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 10955}, 'php': {'number_of_characters': 8619431, 'num_samples': 281739, 'num_queries': 14014, 'num_documents': 267725, 'min_document_length': 94, 'average_document_length': 30.2, 'max_document_length': 4904, 'unique_documents': 267725, 'min_query_length': 2, 'average_query_length': 38.21, 'max_query_length': 2, 'unique_queries': 14014, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 14014}}}} | | [CPUSpeedTask](https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/blob/c8376f967d1294419be1d3eb41217d04cd3a65d3/src/seb/registered_tasks/speed.py#L83-L96) | ['eng'] | Speed | s2s | [Fiction, Written] | None | None | -| [CQADupstackAndroidRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | -| [CQADupstackEnglishRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | -| [CQADupstackGamingRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | -| [CQADupstackGisRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | -| [CQADupstackMathematicaRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | -| [CQADupstackPhysicsRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | -| [CQADupstackProgrammersRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Programming, Written, Non-fiction] | None | None | -| [CQADupstackStatsRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | -| [CQADupstackTexRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | -| [CQADupstackUnixRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | -| [CQADupstackWebmastersRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | -| [CQADupstackWordpressRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | | None | None | +| [CQADupstackAndroidRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Non-fiction, Programming, Web, Written] | None | None | +| [CQADupstackAndroidRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-android-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [CQADupstackEnglishRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Written] | None | None | +| [CQADupstackEnglishRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-english-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [CQADupstackGamingRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Web, Written] | None | None | +| [CQADupstackGamingRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-gaming-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [CQADupstackGisRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Non-fiction, Written] | None | None | +| [CQADupstackGisRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-gis-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [CQADupstackMathematicaRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Academic, Non-fiction, Written] | None | None | +| [CQADupstackMathematicaRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-mathematica-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [CQADupstackPhysicsRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Academic, Non-fiction, Written] | None | None | +| [CQADupstackPhysicsRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-physics-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [CQADupstackProgrammersRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Non-fiction, Programming, Written] | None | None | +| [CQADupstackProgrammersRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-programmers-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [CQADupstackRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | None | [Academic, Non-fiction, Programming, Web, Written] | None | None | +| CQADupstackRetrieval-Fa | ['fas'] | Retrieval | None | [Web] | None | None | +| [CQADupstackStatsRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Academic, Non-fiction, Written] | None | None | +| [CQADupstackStatsRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-stats-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [CQADupstackTexRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Non-fiction, Written] | None | None | +| [CQADupstackTexRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-tex-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [CQADupstackUnixRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Programming, Web, Written] | None | None | +| [CQADupstackUnixRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-unix-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [CQADupstackWebmastersRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Web, Written] | None | None | +| [CQADupstackWebmastersRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-webmasters-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [CQADupstackWordpressRetrieval](http://nlp.cis.unimelb.edu.au/resources/cqadupstack/) (Hoogeveen et al., 2015) | ['eng'] | Retrieval | s2p | [Programming, Web, Written] | None | None | +| [CQADupstackWordpressRetrieval-Fa](https://huggingface.co/datasets/MCINext/cqadupstack-wordpress-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | | [CSFDCZMovieReviewSentimentClassification](https://arxiv.org/abs/2304.01922) (Michal Štefánik, 2023) | ['ces'] | Classification | s2s | [Reviews, Written] | None | None | | [CSFDSKMovieReviewSentimentClassification](https://arxiv.org/abs/2304.01922) (Michal Štefánik, 2023) | ['slk'] | Classification | s2s | [Reviews, Written] | None | None | | [CTKFactsNLI](https://arxiv.org/abs/2201.11115) (Ullrich et al., 2023) | ['ces'] | PairClassification | s2s | [News, Written] | None | None | @@ -319,16 +155,30 @@ The following tables give you an overview of the tasks in MTEB. | [CUADUnlimitedAllYouCanEatLicenseLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [CUADVolumeRestrictionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [CUADWarrantyDurationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | -| [CUREv1](https://huggingface.co/datasets/clinia/CUREv1) | ['eng', 'fra', 'spa'] | Retrieval | s2p | [Medical, Academic, Written] | None | None | +| [CUB200I2IRetrieval](https://www.florian-schroff.de/publications/CUB-200.pdf) (Welinder et al., 2010) | ['eng'] | Any2AnyRetrieval | i2i | [Encyclopaedic] | None | None | +| [CUREv1](https://huggingface.co/datasets/clinia/CUREv1) | ['eng', 'fra', 'spa'] | Retrieval | s2p | [Academic, Medical, Written] | None | None | +| [CVBenchCount](https://arxiv.org/pdf/2406.16860) (Tong et al., 2024) | ['eng'] | Any2TextMutipleChoice | it2t | [Academic] | None | None | +| [CVBenchDepth](https://arxiv.org/pdf/2406.16860) (Tong et al., 2024) | ['eng'] | Any2TextMutipleChoice | it2t | [Academic] | None | None | +| [CVBenchDistance](https://arxiv.org/pdf/2406.16860) (Tong et al., 2024) | ['eng'] | Any2TextMutipleChoice | it2t | [Academic] | None | None | +| [CVBenchRelation](https://arxiv.org/pdf/2406.16860) (Tong et al., 2024) | ['eng'] | Any2TextMutipleChoice | it2t | [Academic] | None | None | +| [Caltech101](https://ieeexplore.ieee.org/document/1384978) (Li Fei-Fei, 2004) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [Caltech101ZeroShot](https://ieeexplore.ieee.org/document/1384978) (Li Fei-Fei, 2004) | ['eng'] | ZeroShotClassification | i2t | [Encyclopaedic] | None | None | | [CanadaTaxCourtOutcomesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | -| [CataloniaTweetClassification](https://aclanthology.org/2020.lrec-1.171/) | ['cat', 'spa'] | Classification | s2s | [Social, Government, Written] | None | None | -| [ClimateFEVER](https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | | None | None | -| [ClimateFEVERHardNegatives](https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | | None | None | +| [CataloniaTweetClassification](https://aclanthology.org/2020.lrec-1.171/) | ['cat', 'spa'] | Classification | s2s | [Government, Social, Written] | None | None | +| [ChemHotpotQARetrieval](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Retrieval | s2p | [Chemistry] | None | None | +| [ChemNQRetrieval](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Retrieval | s2p | [Chemistry] | None | None | +| [ClimateFEVER](https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [ClimateFEVER-Fa](https://huggingface.co/datasets/MCINext/climate-fever-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [ClimateFEVERHardNegatives](https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | | [CmedqaRetrieval](https://aclanthology.org/2022.emnlp-main.357.pdf) | ['cmn'] | Retrieval | s2p | [Medical, Written] | None | None | | [Cmnli](https://huggingface.co/datasets/clue/viewer/cmnli) | ['cmn'] | PairClassification | s2s | | None | None | | [CodeEditSearchRetrieval](https://huggingface.co/datasets/cassanof/CodeEditSearch/viewer) (Niklas Muennighoff, 2023) | ['c', 'c++', 'go', 'java', 'javascript', 'php', 'python', 'ruby', 'rust', 'scala', 'shell', 'swift', 'typescript'] | Retrieval | p2p | [Programming, Written] | {'train': 26000} | {'train': {'number_of_characters': 935841, 'num_samples': 26000, 'num_queries': 13000, 'num_documents': 13000, 'min_document_length': 18, 'average_document_length': 70.99, 'max_document_length': 2532, 'unique_documents': 13000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 13000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 13000, 'hf_subset_descriptive_stats': {'python': {'number_of_characters': 70519, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 21, 'average_document_length': 69.52, 'max_document_length': 1811, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'javascript': {'number_of_characters': 57880, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 18, 'average_document_length': 56.88, 'max_document_length': 601, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'typescript': {'number_of_characters': 61092, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 19, 'average_document_length': 60.09, 'max_document_length': 659, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'go': {'number_of_characters': 71797, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 19, 'average_document_length': 70.8, 'max_document_length': 1529, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'ruby': {'number_of_characters': 67900, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 20, 'average_document_length': 66.9, 'max_document_length': 751, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'java': {'number_of_characters': 63984, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 23, 'average_document_length': 62.98, 'max_document_length': 807, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'php': {'number_of_characters': 62927, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 21, 'average_document_length': 61.93, 'max_document_length': 766, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'c': {'number_of_characters': 98588, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 20, 'average_document_length': 97.59, 'max_document_length': 1672, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'c++': {'number_of_characters': 115480, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 22, 'average_document_length': 114.48, 'max_document_length': 1856, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'rust': {'number_of_characters': 68503, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 19, 'average_document_length': 67.5, 'max_document_length': 2532, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'swift': {'number_of_characters': 58279, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 19, 'average_document_length': 57.28, 'max_document_length': 727, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'scala': {'number_of_characters': 65833, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 22, 'average_document_length': 64.83, 'max_document_length': 685, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'shell': {'number_of_characters': 73059, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 18, 'average_document_length': 72.06, 'max_document_length': 813, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}}}} | | [CodeFeedbackMT](https://arxiv.org/abs/2402.14658) (Tianyu Zheng, 2024) | ['eng'] | Retrieval | p2p | [Programming, Written] | {'test': 79660} | {'test': {'number_of_characters': 156266302, 'num_samples': 79660, 'num_queries': 13277, 'num_documents': 66383, 'min_document_length': 127, 'average_document_length': 885.13, 'max_document_length': 32432, 'unique_documents': 66383, 'min_query_length': 2, 'average_query_length': 7344.18, 'max_query_length': 9403, 'unique_queries': 13277, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 13277}} | | [CodeFeedbackST](https://arxiv.org/abs/2407.02883) (Xiangyang Li, 2024) | ['eng'] | Retrieval | p2p | [Programming, Written] | {'test': 187832} | {'test': {'number_of_characters': 260957682, 'num_samples': 187832, 'num_queries': 31306, 'num_documents': 156526, 'min_document_length': 26, 'average_document_length': 144.85, 'max_document_length': 13851, 'unique_documents': 156526, 'min_query_length': 1, 'average_query_length': 7611.46, 'max_query_length': 11354, 'unique_queries': 31306, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 31306}} | +| [CodeRAGLibraryDocumentationSolutions](https://arxiv.org/pdf/2406.14497) (Zora Zhiruo Wang, 2024) | ['python'] | Reranking | s2s | [Programming] | None | None | +| [CodeRAGOnlineTutorials](https://arxiv.org/pdf/2406.14497) (Zora Zhiruo Wang, 2024) | ['python'] | Reranking | s2s | [Programming] | None | None | +| [CodeRAGProgrammingSolutions](https://arxiv.org/pdf/2406.14497) (Zora Zhiruo Wang, 2024) | ['python'] | Reranking | s2s | [Programming] | None | None | +| [CodeRAGStackoverflowPosts](https://arxiv.org/pdf/2406.14497) (Zora Zhiruo Wang, 2024) | ['python'] | Reranking | s2s | [Programming] | None | None | | [CodeSearchNetCCRetrieval](https://arxiv.org/abs/2407.02883) (Xiangyang Li, 2024) | ['go', 'java', 'javascript', 'php', 'python', 'ruby'] | Retrieval | p2p | [Programming, Written] | {'test': 1058035} | {'test': {'number_of_characters': 22407915, 'num_samples': 1058035, 'num_queries': 52561, 'num_documents': 1005474, 'min_document_length': 23, 'average_document_length': 20.29, 'max_document_length': 214210, 'unique_documents': 1005474, 'min_query_length': 2, 'average_query_length': 38.26, 'max_query_length': 2, 'unique_queries': 52561, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 52561, 'hf_subset_descriptive_stats': {'python': {'number_of_characters': 8792958, 'num_samples': 295570, 'num_queries': 14918, 'num_documents': 280652, 'min_document_length': 38, 'average_document_length': 29.33, 'max_document_length': 8326, 'unique_documents': 280652, 'min_query_length': 2, 'average_query_length': 37.63, 'max_query_length': 2, 'unique_queries': 14918, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 14918}, 'javascript': {'number_of_characters': 1590642, 'num_samples': 68492, 'num_queries': 3291, 'num_documents': 65201, 'min_document_length': 40, 'average_document_length': 22.4, 'max_document_length': 214210, 'unique_documents': 65201, 'min_query_length': 2, 'average_query_length': 39.62, 'max_query_length': 2, 'unique_queries': 3291, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 3291}, 'go': {'number_of_characters': 2264134, 'num_samples': 190857, 'num_queries': 8122, 'num_documents': 182735, 'min_document_length': 23, 'average_document_length': 10.39, 'max_document_length': 3589, 'unique_documents': 182735, 'min_query_length': 2, 'average_query_length': 45.0, 'max_query_length': 2, 'unique_queries': 8122, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 8122}, 'ruby': {'number_of_characters': 391703, 'num_samples': 28849, 'num_queries': 1261, 'num_documents': 27588, 'min_document_length': 36, 'average_document_length': 12.2, 'max_document_length': 2244, 'unique_documents': 27588, 'min_query_length': 2, 'average_query_length': 43.76, 'max_query_length': 2, 'unique_queries': 1261, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1261}, 'java': {'number_of_characters': 4114584, 'num_samples': 192016, 'num_queries': 10955, 'num_documents': 181061, 'min_document_length': 38, 'average_document_length': 20.72, 'max_document_length': 5066, 'unique_documents': 181061, 'min_query_length': 2, 'average_query_length': 33.06, 'max_query_length': 2, 'unique_queries': 10955, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 10955}, 'php': {'number_of_characters': 5253894, 'num_samples': 282251, 'num_queries': 14014, 'num_documents': 268237, 'min_document_length': 40, 'average_document_length': 17.59, 'max_document_length': 2995, 'unique_documents': 268237, 'min_query_length': 2, 'average_query_length': 38.28, 'max_query_length': 2, 'unique_queries': 14014, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 14014}}}} | | [CodeSearchNetRetrieval](https://huggingface.co/datasets/code_search_net/) (Husain et al., 2019) | ['go', 'java', 'javascript', 'php', 'python', 'ruby'] | Retrieval | p2p | [Programming, Written] | {'test': 12000} | {'test': {'number_of_characters': 1950074, 'num_samples': 12000, 'num_queries': 6000, 'num_documents': 6000, 'min_document_length': 2, 'average_document_length': 324.01, 'max_document_length': 17533, 'unique_documents': 6000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 6000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 6000, 'hf_subset_descriptive_stats': {'python': {'number_of_characters': 467546, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 8, 'average_document_length': 466.55, 'max_document_length': 8636, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'javascript': {'number_of_characters': 187018, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 2, 'average_document_length': 186.02, 'max_document_length': 7657, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'go': {'number_of_characters': 126213, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 14, 'average_document_length': 125.21, 'max_document_length': 1501, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'ruby': {'number_of_characters': 314818, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 5, 'average_document_length': 313.82, 'max_document_length': 17533, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'java': {'number_of_characters': 691360, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 2, 'average_document_length': 690.36, 'max_document_length': 6473, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}, 'php': {'number_of_characters': 163119, 'num_samples': 2000, 'num_queries': 1000, 'num_documents': 1000, 'min_document_length': 5, 'average_document_length': 162.12, 'max_document_length': 1240, 'unique_documents': 1000, 'min_query_length': 1, 'average_query_length': 1.0, 'max_query_length': 1, 'unique_queries': 1000, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1000}}}} | | [CodeTransOceanContest](https://arxiv.org/abs/2310.04951) (Weixiang Yan, 2023) | ['c++', 'python'] | Retrieval | p2p | [Programming, Written] | {'test': 1229} | {'test': {'number_of_characters': 1744286, 'num_samples': 1229, 'num_queries': 221, 'num_documents': 1008, 'min_document_length': 8, 'average_document_length': 221.9, 'max_document_length': 4147, 'unique_documents': 1008, 'min_query_length': 8, 'average_query_length': 6880.58, 'max_query_length': 10852, 'unique_queries': 221, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 221}} | @@ -350,6 +200,8 @@ The following tables give you an overview of the tasks in MTEB. | [Core17InstructionRetrieval](https://arxiv.org/abs/2403.15246) (Orion Weller, 2024) | ['eng'] | InstructionRetrieval | s2p | [News, Written] | {'test': 19919} | {'test': {'num_samples': 19919, 'num_docs': 19899, 'num_queries': 20, 'number_of_characters': 44450333, 'min_document_length': 7, 'average_document_length': 2233.03, 'max_document_length': 2959, 'unique_docs': 19143, 'min_query_length': 55, 'average_query_length': 109.75, 'max_query_length': 278, 'unique_queries': 20, 'min_instruction_length': 102, 'average_instruction_length': 295.55, 'max_instruction_length': 811, 'unique_instructions': 20, 'min_changed_instruction_length': 151, 'average_changed_instruction_length': 355.2, 'max_changed_instruction_length': 837, 'unique_changed_instructions': 20, 'min_average_relevant_docs_per_query': 4, 'average_relevant_docs_per_query': 32.7, 'max_average_relevant_docs_per_query': 55, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}} | | [CorporateLobbyingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [CosQA](https://arxiv.org/abs/2105.13239) (Junjie Huang, 2021) | ['eng', 'python'] | Retrieval | p2p | [Programming, Written] | {'test': 21104} | {'test': {'number_of_characters': 5728450, 'num_samples': 21104, 'num_queries': 500, 'num_documents': 20604, 'min_document_length': 18, 'average_document_length': 0.89, 'max_document_length': 83, 'unique_documents': 20604, 'min_query_length': 88, 'average_query_length': 11420.09, 'max_query_length': 6396, 'unique_queries': 500, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 500}} | +| [Country211](https://huggingface.co/datasets/clip-benchmark/wds_country211) (Radford et al., 2021) | ['eng'] | ImageClassification | i2i | [Scene] | None | None | +| [Country211ZeroShot](https://huggingface.co/datasets/clip-benchmark/wds_country211) (Radford et al., 2021) | ['eng'] | ZeroShotClassification | i2t | [Scene] | None | None | | [CovidRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | None | | [CrossLingualSemanticDiscriminationWMT19](https://huggingface.co/datasets/Andrianos/clsd_wmt19_21) | ['deu', 'fra'] | Retrieval | s2s | [News, Written] | None | None | | [CrossLingualSemanticDiscriminationWMT21](https://huggingface.co/datasets/Andrianos/clsd_wmt19_21) | ['deu', 'fra'] | Retrieval | s2s | [News, Written] | None | None | @@ -357,17 +209,23 @@ The following tables give you an overview of the tasks in MTEB. | [CzechProductReviewSentimentClassification](https://aclanthology.org/W13-1609/) | ['ces'] | Classification | s2s | [Reviews, Written] | None | None | | [CzechSoMeSentimentClassification](https://aclanthology.org/W13-1609/) | ['ces'] | Classification | s2s | [Reviews, Written] | None | None | | [CzechSubjectivityClassification](https://arxiv.org/abs/2009.08712) | ['ces'] | Classification | s2s | [Reviews, Written] | None | None | -| [DBPedia](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['eng'] | Retrieval | s2p | [Written, Encyclopaedic] | None | None | -| [DBPedia-PL](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['pol'] | Retrieval | s2p | [Written, Encyclopaedic] | None | None | -| [DBPedia-PLHardNegatives](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['pol'] | Retrieval | s2p | [Written, Encyclopaedic] | None | None | -| [DBPediaHardNegatives](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['eng'] | Retrieval | s2p | [Written, Encyclopaedic] | None | None | +| [DBPedia](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [DBPedia-Fa](https://huggingface.co/datasets/MCINext/dbpedia-fa) | ['fas'] | Retrieval | s2p | [Encyclopaedic] | None | None | +| [DBPedia-PL](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['pol'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [DBPedia-PLHardNegatives](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['pol'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [DBPediaHardNegatives](https://github.com/iai-group/DBpedia-Entity/) (Hasibi et al., 2017) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | | [DBpediaClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [Encyclopaedic, Written] | None | None | | [DKHateClassification](https://aclanthology.org/2020.lrec-1.430/) | ['dan'] | Classification | s2s | [Social, Written] | None | None | +| [DTD](https://www.robots.ox.ac.uk/~vgg/data/dtd/) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [DTDZeroShot](https://www.robots.ox.ac.uk/~vgg/data/dtd/) | ['eng'] | ZeroShotClassification | i2t | [Encyclopaedic] | None | None | | [DalajClassification](https://spraakbanken.gu.se/en/resources/superlim) | ['swe'] | Classification | s2s | [Non-fiction, Written] | None | None | | [DanFeverRetrieval](https://aclanthology.org/2021.nodalida-main.47/) | ['dan'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Spoken] | None | None | | [DanishPoliticalCommentsClassification](https://huggingface.co/datasets/danish_political_comments) (Mads Guldborg Kjeldgaard Kongsbak, 2019) | ['dan'] | Classification | s2s | [Social, Written] | None | None | +| [DeepSentiPers](https://github.com/JoyeBright/DeepSentiPers) | ['fas'] | Classification | s2s | [Reviews] | None | None | | [DefinitionClassificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [DiaBlaBitextMining](https://inria.hal.science/hal-03021633) (González et al., 2019) | ['eng', 'fra'] | BitextMining | s2s | [Social, Written] | None | None | +| [DigikalamagClassification](https://hooshvare.github.io/docs/datasets/tc) | ['fas'] | Classification | p2p | [Web] | None | None | +| [DigikalamagClustering](https://hooshvare.github.io/docs/datasets/tc) | ['fas'] | Clustering | p2p | [Web] | None | None | | [Diversity1LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [Diversity2LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [Diversity3LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | @@ -376,34 +234,57 @@ The following tables give you an overview of the tasks in MTEB. | [Diversity6LegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [DuRetrieval](https://aclanthology.org/2022.emnlp-main.357.pdf) (Yifu Qiu, 2022) | ['cmn'] | Retrieval | s2p | | None | None | | [DutchBookReviewSentimentClassification](https://github.com/benjaminvdb/DBRD) (Benjamin et al., 2019) | ['nld'] | Classification | s2s | [Reviews, Written] | None | None | +| [EDIST2ITRetrieval](https://aclanthology.org/2023.emnlp-main.297/) (Liu et al., 2023) | ['eng'] | Any2AnyRetrieval | t2it | [News] | None | None | | [ESCIReranking](https://github.com/amazon-science/esci-data/) (Chandan K. Reddy, 2022) | ['eng', 'jpn', 'spa'] | Reranking | s2p | [Written] | {'test': 29285} | {'test': {'num_samples': 29285, 'number_of_characters': 254538331, 'num_positive': 271416, 'num_negative': 44235, 'min_query_length': 1, 'avg_query_length': 19.69, 'max_query_length': 151, 'unique_query': 29269, 'min_positive_length': 1, 'avg_positive_length': 803.92, 'max_positive_length': 8640, 'unique_positive': 217712, 'min_negative_length': 1, 'avg_negative_length': 808.5, 'max_negative_length': 4441, 'unique_negative': 39551, 'hf_subset_descriptive_stats': {'us': {'num_samples': 21296, 'number_of_characters': 186915609, 'num_positive': 189375, 'num_negative': 25463, 'min_query_length': 1, 'avg_query_length': 21.44, 'max_query_length': 151, 'unique_query': 21296, 'min_positive_length': 1, 'avg_positive_length': 868.37, 'max_positive_length': 5545, 'unique_positive': 150734, 'min_negative_length': 1, 'avg_negative_length': 864.45, 'max_negative_length': 3779, 'unique_negative': 23073}, 'es': {'num_samples': 3703, 'number_of_characters': 48861389, 'num_positive': 39110, 'num_negative': 10183, 'min_query_length': 3, 'avg_query_length': 20.68, 'max_query_length': 59, 'unique_query': 3703, 'min_positive_length': 1, 'avg_positive_length': 980.96, 'max_positive_length': 8640, 'unique_positive': 32921, 'min_negative_length': 1, 'avg_negative_length': 1023.22, 'max_negative_length': 4441, 'unique_negative': 9285}, 'jp': {'num_samples': 4286, 'number_of_characters': 18761333, 'num_positive': 42931, 'num_negative': 8589, 'min_query_length': 1, 'avg_query_length': 10.15, 'max_query_length': 60, 'unique_query': 4286, 'min_positive_length': 1, 'avg_positive_length': 358.36, 'max_positive_length': 3488, 'unique_positive': 35165, 'min_negative_length': 1, 'avg_negative_length': 388.08, 'max_negative_length': 3940, 'unique_negative': 7289}}}} | | [EcomRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | None | | [EightTagsClustering.v2](https://aclanthology.org/2020.lrec-1.207.pdf) | ['pol'] | Clustering | s2s | [Social, Written] | None | None | | [EmotionClassification](https://www.aclweb.org/anthology/D18-1404) | ['eng'] | Classification | s2s | [Social, Written] | None | None | +| [EncyclopediaVQAIT2ITRetrieval](https://github.com/google-research/google-research/tree/master/encyclopedic_vqa) (Mensink et al., 2023) | ['eng'] | Any2AnyRetrieval | it2it | [Encyclopaedic] | None | None | | [EstQA](https://www.semanticscholar.org/paper/Extractive-Question-Answering-for-Estonian-Language-182912IAPM-Alum%C3%A4e/ea4f60ab36cadca059c880678bc4c51e293a85d6?utm_source=direct_link) | ['est'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | | [EstonianValenceClassification](https://figshare.com/articles/dataset/Estonian_Valence_Corpus_Eesti_valentsikorpus/24517054) | ['est'] | Classification | s2s | [News, Written] | None | None | -| [FEVER](https://fever.ai/) | ['eng'] | Retrieval | s2p | | None | None | +| [EuroSAT](https://ieeexplore.ieee.org/document/8736785) (Helber et al., 2019) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [EuroSATZeroShot](https://ieeexplore.ieee.org/document/8736785) (Helber et al., 2019) | ['eng'] | ZeroShotClassification | i2t | [Encyclopaedic] | None | None | +| [FER2013](https://arxiv.org/abs/1412.6572) (Ian J. Goodfellow, 2015) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [FER2013ZeroShot](https://arxiv.org/abs/1412.6572) (Ian J. Goodfellow, 2015) | ['eng'] | ZeroShotClassification | i2t | [Encyclopaedic] | None | None | +| [FEVER](https://fever.ai/) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | | [FEVERHardNegatives](https://fever.ai/) | ['eng'] | Retrieval | s2p | | None | None | +| [FGVCAircraft](https://arxiv.org/abs/1306.5151) (Subhransu Maji, 2013) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [FGVCAircraftZeroShot](https://arxiv.org/abs/1306.5151) (Subhransu Maji, 2013) | ['eng'] | ZeroShotClassification | i2t | [Encyclopaedic] | None | None | +| [FORBI2IRetrieval](https://github.com/pxiangwu/FORB) (Pengxiang Wu, 2023) | ['eng'] | Any2AnyRetrieval | i2i | [Encyclopaedic] | None | None | | [FQuADRetrieval](https://huggingface.co/datasets/manu/fquad2_test) | ['fra'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | | [FaithDial](https://mcgill-nlp.github.io/FaithDial) (Dziri et al., 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | | [FalseFriendsGermanEnglish](https://drive.google.com/file/d/1jgq0nBnV-UiYNxbKNrrr2gxDEHm-DMKH/view?usp=share_link) | ['deu'] | PairClassification | s2s | [Written] | None | None | | [FaroeseSTS](https://aclanthology.org/2023.nodalida-1.74.pdf) | ['fao'] | STS | s2s | [News, Web, Written] | None | None | | [FarsTail](https://link.springer.com/article/10.1007/s00500-023-08959-3) (Amirkhani et al., 2023) | ['fas'] | PairClassification | s2s | [Academic, Written] | None | None | -| [FeedbackQARetrieval](https://arxiv.org/abs/2204.03025) | ['eng'] | Retrieval | s2p | [Web, Government, Medical, Written] | None | None | -| [FiQA-PL](https://sites.google.com/view/fiqa/) (Nandan Thakur, 2021) | ['pol'] | Retrieval | s2p | | None | None | -| [FiQA2018](https://sites.google.com/view/fiqa/) (Nandan Thakur, 2021) | ['eng'] | Retrieval | s2p | | None | None | +| [FarsiParaphraseDetection](https://huggingface.co/datasets/alighasemi/farsi_paraphrase_detection) | ['fas'] | PairClassification | s2s | | None | None | +| [Farsick](https://github.com/ZahraGhasemi-AI/FarSick) | ['fas'] | STS | s2s | | None | None | +| [Fashion200kI2TRetrieval](https://openaccess.thecvf.com/content_iccv_2017/html/Han_Automatic_Spatially-Aware_Fashion_ICCV_2017_paper.html) (Han et al., 2017) | ['eng'] | Any2AnyRetrieval | i2t | [Encyclopaedic] | None | None | +| [Fashion200kT2IRetrieval](https://openaccess.thecvf.com/content_iccv_2017/html/Han_Automatic_Spatially-Aware_Fashion_ICCV_2017_paper.html) (Han et al., 2017) | ['eng'] | Any2AnyRetrieval | t2i | [Encyclopaedic] | None | None | +| [FashionIQIT2IRetrieval](https://openaccess.thecvf.com/content/CVPR2021/html/Wu_Fashion_IQ_A_New_Dataset_Towards_Retrieving_Images_by_Natural_CVPR_2021_paper.html) (Wu et al., 2021) | ['eng'] | Any2AnyRetrieval | it2i | [Encyclopaedic] | None | None | +| [FeedbackQARetrieval](https://arxiv.org/abs/2204.03025) | ['eng'] | Retrieval | s2p | [Government, Medical, Web, Written] | None | None | +| [FiQA-PL](https://sites.google.com/view/fiqa/) (Nandan Thakur, 2021) | ['pol'] | Retrieval | s2p | [Financial, Written] | None | None | +| [FiQA2018](https://sites.google.com/view/fiqa/) (Nandan Thakur, 2021) | ['eng'] | Retrieval | s2p | [Financial, Written] | None | None | +| [FiQA2018-Fa](https://huggingface.co/datasets/MCINext/fiqa-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | | [FilipinoHateSpeechClassification](https://pcj.csp.org.ph/index.php/pcj/issue/download/29/PCJ%20V14%20N1%20pp1-14%202019) (Neil Vicente Cabasag et al., 2019) | ['fil'] | Classification | s2s | [Social, Written] | None | None | | [FilipinoShopeeReviewsClassification](https://uijrt.com/articles/v4/i8/UIJRTV4I80009.pdf) | ['fil'] | Classification | s2s | [Social, Written] | None | None | | [FinParaSTS](https://huggingface.co/datasets/TurkuNLP/turku_paraphrase_corpus) | ['fin'] | STS | s2s | [News, Subtitles, Written] | None | None | | [FinToxicityClassification](https://aclanthology.org/2023.nodalida-1.68) | ['fin'] | Classification | s2s | [News, Written] | None | None | -| [FinancialPhrasebankClassification](https://arxiv.org/abs/1307.5336) (P. Malo, 2014) | ['eng'] | Classification | s2s | [News, Written] | None | None | -| [FloresBitextMining](https://huggingface.co/datasets/facebook/flores) (Goyal et al., 2022) | ['ace', 'acm', 'acq', 'aeb', 'afr', 'ajp', 'aka', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'ast', 'awa', 'ayr', 'azb', 'azj', 'bak', 'bam', 'ban', 'bel', 'bem', 'ben', 'bho', 'bjn', 'bod', 'bos', 'bug', 'bul', 'cat', 'ceb', 'ces', 'cjk', 'ckb', 'crh', 'cym', 'dan', 'deu', 'dik', 'dyu', 'dzo', 'ell', 'eng', 'epo', 'est', 'eus', 'ewe', 'fao', 'fij', 'fin', 'fon', 'fra', 'fur', 'fuv', 'gaz', 'gla', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hne', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kac', 'kam', 'kan', 'kas', 'kat', 'kaz', 'kbp', 'kea', 'khk', 'khm', 'kik', 'kin', 'kir', 'kmb', 'kmr', 'knc', 'kon', 'kor', 'lao', 'lij', 'lim', 'lin', 'lit', 'lmo', 'ltg', 'ltz', 'lua', 'lug', 'luo', 'lus', 'lvs', 'mag', 'mai', 'mal', 'mar', 'min', 'mkd', 'mlt', 'mni', 'mos', 'mri', 'mya', 'nld', 'nno', 'nob', 'npi', 'nso', 'nus', 'nya', 'oci', 'ory', 'pag', 'pan', 'pap', 'pbt', 'pes', 'plt', 'pol', 'por', 'prs', 'quy', 'ron', 'run', 'rus', 'sag', 'san', 'sat', 'scn', 'shn', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'ssw', 'sun', 'swe', 'swh', 'szl', 'tam', 'taq', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tpi', 'tsn', 'tso', 'tuk', 'tum', 'tur', 'twi', 'tzm', 'uig', 'ukr', 'umb', 'urd', 'uzn', 'vec', 'vie', 'war', 'wol', 'xho', 'ydd', 'yor', 'yue', 'zho', 'zsm', 'zul'] | BitextMining | s2s | [Non-fiction, Encyclopaedic, Written] | None | None | +| [FinancialPhrasebankClassification](https://arxiv.org/abs/1307.5336) (P. Malo, 2014) | ['eng'] | Classification | s2s | [Financial, News, Written] | None | None | +| [Flickr30kI2TRetrieval](https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31) (Peter Young, 2014) | ['eng'] | Any2AnyRetrieval | i2t | [Web, Written] | None | None | +| [Flickr30kT2IRetrieval](https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31) (Peter Young, 2014) | ['eng'] | Any2AnyRetrieval | t2i | [Web, Written] | None | None | +| [FloresBitextMining](https://huggingface.co/datasets/facebook/flores) (Goyal et al., 2022) | ['ace', 'acm', 'acq', 'aeb', 'afr', 'ajp', 'aka', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'ast', 'awa', 'ayr', 'azb', 'azj', 'bak', 'bam', 'ban', 'bel', 'bem', 'ben', 'bho', 'bjn', 'bod', 'bos', 'bug', 'bul', 'cat', 'ceb', 'ces', 'cjk', 'ckb', 'crh', 'cym', 'dan', 'deu', 'dik', 'dyu', 'dzo', 'ell', 'eng', 'epo', 'est', 'eus', 'ewe', 'fao', 'fij', 'fin', 'fon', 'fra', 'fur', 'fuv', 'gaz', 'gla', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hne', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kac', 'kam', 'kan', 'kas', 'kat', 'kaz', 'kbp', 'kea', 'khk', 'khm', 'kik', 'kin', 'kir', 'kmb', 'kmr', 'knc', 'kon', 'kor', 'lao', 'lij', 'lim', 'lin', 'lit', 'lmo', 'ltg', 'ltz', 'lua', 'lug', 'luo', 'lus', 'lvs', 'mag', 'mai', 'mal', 'mar', 'min', 'mkd', 'mlt', 'mni', 'mos', 'mri', 'mya', 'nld', 'nno', 'nob', 'npi', 'nso', 'nus', 'nya', 'oci', 'ory', 'pag', 'pan', 'pap', 'pbt', 'pes', 'plt', 'pol', 'por', 'prs', 'quy', 'ron', 'run', 'rus', 'sag', 'san', 'sat', 'scn', 'shn', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'ssw', 'sun', 'swe', 'swh', 'szl', 'tam', 'taq', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tpi', 'tsn', 'tso', 'tuk', 'tum', 'tur', 'twi', 'tzm', 'uig', 'ukr', 'umb', 'urd', 'uzn', 'vec', 'vie', 'war', 'wol', 'xho', 'ydd', 'yor', 'yue', 'zho', 'zsm', 'zul'] | BitextMining | s2s | [Encyclopaedic, Non-fiction, Written] | None | None | +| [Food101Classification](https://huggingface.co/datasets/ethz/food101) (Bossard et al., 2014) | ['eng'] | ImageClassification | i2i | [Web] | None | None | +| [Food101ZeroShot](https://huggingface.co/datasets/ethz/food101) (Bossard et al., 2014) | ['eng'] | ZeroShotClassification | i2t | [Web] | None | None | | [FrenchBookReviews](https://huggingface.co/datasets/Abirate/french_book_reviews) | ['fra'] | Classification | s2s | [Reviews, Written] | None | None | | [FrenkEnClassification](https://arxiv.org/abs/1906.02045) (Nikola Ljubešić, 2019) | ['eng'] | Classification | s2s | [Social, Written] | None | None | | [FrenkHrClassification](https://arxiv.org/abs/1906.02045) (Nikola Ljubešić, 2019) | ['hrv'] | Classification | s2s | [Social, Written] | None | None | | [FrenkSlClassification](https://arxiv.org/pdf/1906.02045) (Nikola Ljubešić, 2019) | ['slv'] | Classification | s2s | [Social, Written] | None | None | | [FunctionOfDecisionSectionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [GLDv2I2IRetrieval](https://openaccess.thecvf.com/content_CVPR_2020/html/Weyand_Google_Landmarks_Dataset_v2_-_A_Large-Scale_Benchmark_for_Instance-Level_CVPR_2020_paper.html) (Weyand et al., 2020) | ['eng'] | Any2AnyRetrieval | i2i | [Encyclopaedic] | None | None | +| [GLDv2I2TRetrieval](https://openaccess.thecvf.com/content_CVPR_2020/html/Weyand_Google_Landmarks_Dataset_v2_-_A_Large-Scale_Benchmark_for_Instance-Level_CVPR_2020_paper.html) (Weyand et al., 2020) | ['eng'] | Any2AnyRetrieval | i2t | [Encyclopaedic] | None | None | | [GPUSpeedTask](https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/blob/c8376f967d1294419be1d3eb41217d04cd3a65d3/src/seb/registered_tasks/speed.py#L83-L96) | ['eng'] | Speed | s2s | [Fiction, Written] | None | None | +| [GTSRB](https://benchmark.ini.rub.de/) (Stallkamp et al., 2011) | ['eng'] | ImageClassification | i2i | [Scene] | None | None | +| [GTSRBZeroShot](https://benchmark.ini.rub.de/) (Stallkamp et al., 2011) | ['eng'] | ZeroShotClassification | i2t | [Scene] | None | None | | [GeoreviewClassification](https://github.com/yandex/geo-reviews-dataset-2023) | ['rus'] | Classification | p2p | [Reviews, Written] | None | None | | [GeoreviewClusteringP2P](https://github.com/yandex/geo-reviews-dataset-2023) | ['rus'] | Clustering | p2p | [Reviews, Written] | None | None | | [GeorgianFAQRetrieval](https://huggingface.co/datasets/jupyterjazz/georgian-faq) | ['kat'] | Retrieval | s2p | [Web, Written] | None | None | @@ -411,7 +292,7 @@ The following tables give you an overview of the tasks in MTEB. | [GerDaLIRSmall](https://github.com/lavis-nlp/GerDaLIR) | ['deu'] | Retrieval | p2p | [Legal, Written] | None | None | | [GermanDPR](https://huggingface.co/datasets/deepset/germandpr) (Timo Möller, 2021) | ['deu'] | Retrieval | s2p | | None | None | | [GermanGovServiceRetrieval](https://huggingface.co/datasets/it-at-m/LHM-Dienstleistungen-QA) | ['deu'] | Retrieval | s2p | [Government, Written] | None | None | -| [GermanPoliticiansTwitterSentimentClassification](https://aclanthology.org/2022.konvens-1.9) | ['deu'] | Classification | s2s | [Social, Government, Written] | None | None | +| [GermanPoliticiansTwitterSentimentClassification](https://aclanthology.org/2022.konvens-1.9) | ['deu'] | Classification | s2s | [Government, Social, Written] | None | None | | [GermanQuAD-Retrieval](https://www.kaggle.com/datasets/GermanQuAD) (Timo Möller, 2021) | ['deu'] | Retrieval | s2p | | None | None | | [GermanSTSBenchmark](https://github.com/t-systems-on-site-services-gmbh/german-STSbenchmark) (Philip May, 2021) | ['deu'] | STS | s2s | | None | None | | [GreekCivicsQA](https://huggingface.co/datasets/antoinelb7/alloprof) | ['ell'] | Retrieval | s2p | [Academic, Written] | None | None | @@ -419,7 +300,10 @@ The following tables give you an overview of the tasks in MTEB. | [GujaratiNewsClassification](https://github.com/goru001/nlp-for-gujarati) | ['guj'] | Classification | s2s | [News, Written] | None | None | | [HALClusteringS2S.v2](https://huggingface.co/datasets/lyon-nlp/clustering-hal-s2s) (Mathieu Ciancone, 2024) | ['fra'] | Clustering | s2s | [Academic, Written] | None | None | | [HagridRetrieval](https://github.com/project-miracl/hagrid) (Ehsan Kamalloo, 2023) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [HamshahriClustring](https://github.com/mallahyari/Farsi-datasets) | ['fas'] | Clustering | p2p | [News] | None | None | | [HateSpeechPortugueseClassification](https://aclanthology.org/W19-3510) | ['por'] | Classification | s2s | [Social, Written] | None | None | +| [HatefulMemesI2TRetrieval](https://arxiv.org/pdf/2005.04790) (Kiela et al., 2020) | ['eng'] | Any2AnyRetrieval | i2t | [Encyclopaedic] | None | None | +| [HatefulMemesT2IRetrieval](https://arxiv.org/pdf/2005.04790) (Kiela et al., 2020) | ['eng'] | Any2AnyRetrieval | t2i | [Encyclopaedic] | None | None | | [HeadlineClassification](https://aclanthology.org/2020.ngt-1.6/) | ['rus'] | Classification | s2s | [News, Written] | None | None | | [HebrewSentimentAnalysis](https://huggingface.co/datasets/hebrew_sentiment) | ['heb'] | Classification | s2s | [Reviews, Written] | None | None | | [HellaSwag](https://rowanzellers.com/hellaswag/) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | @@ -427,115 +311,54 @@ The following tables give you an overview of the tasks in MTEB. | [HindiDiscourseClassification](https://aclanthology.org/2020.lrec-1.149/) | ['hin'] | Classification | s2s | [Fiction, Social, Written] | None | None | | [HotelReviewSentimentClassification](https://link.springer.com/chapter/10.1007/978-3-319-67056-0_3) (Elnagar et al., 2018) | ['ara'] | Classification | s2s | [Reviews, Written] | None | None | | [HotpotQA](https://hotpotqa.github.io/) | ['eng'] | Retrieval | s2p | [Web, Written] | None | None | +| [HotpotQA-Fa](https://huggingface.co/datasets/MCINext/hotpotqa-fa) | ['fas'] | Retrieval | s2p | [Encyclopaedic] | None | None | | [HotpotQA-PL](https://hotpotqa.github.io/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | None | | [HotpotQA-PLHardNegatives](https://hotpotqa.github.io/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | None | | [HotpotQAHardNegatives](https://hotpotqa.github.io/) | ['eng'] | Retrieval | s2p | [Web, Written] | None | None | | [HunSum2AbstractiveRetrieval](https://arxiv.org/abs/2404.03555) (Botond Barta, 2024) | ['hun'] | Retrieval | s2p | [News, Written] | None | None | ->>>>>>> main | [IFlyTek](https://www.cluebenchmarks.com/introduce.html) | ['cmn'] | Classification | s2s | | None | None | -| [IN22ConvBitextMining](https://huggingface.co/datasets/ai4bharat/IN22-Conv) (Jay Gala, 2023) | ['asm', 'ben', 'brx', 'doi', 'eng', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Social, Spoken, Fiction, Spoken] | {'test': 760518} | {'test': {'num_samples': 760518, 'number_of_characters': 82637104, 'unique_pairs': 759283, 'min_sentence1_length': 3, 'average_sentence1_length': 54.33, 'max_sentence1_length': 239, 'unique_sentence1': 34430, 'min_sentence2_length': 3, 'average_sentence2_length': 54.33, 'max_sentence2_length': 239, 'unique_sentence2': 34430, 'hf_subset_descriptive_stats': {'asm_Beng-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155988, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'asm_Beng-brx_Deva': {'num_samples': 1503, 'number_of_characters': 162044, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'asm_Beng-doi_Deva': {'num_samples': 1503, 'number_of_characters': 167032, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'asm_Beng-eng_Latn': {'num_samples': 1503, 'number_of_characters': 160716, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'asm_Beng-gom_Deva': {'num_samples': 1503, 'number_of_characters': 156282, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'asm_Beng-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 158269, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'asm_Beng-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159964, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'asm_Beng-kan_Knda': {'num_samples': 1503, 'number_of_characters': 165177, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'asm_Beng-kas_Arab': {'num_samples': 1503, 'number_of_characters': 164681, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'asm_Beng-mai_Deva': {'num_samples': 1503, 'number_of_characters': 162408, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'asm_Beng-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 172838, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'asm_Beng-mar_Deva': {'num_samples': 1503, 'number_of_characters': 162747, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'asm_Beng-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 157316, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'asm_Beng-npi_Deva': {'num_samples': 1503, 'number_of_characters': 160906, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'asm_Beng-ory_Orya': {'num_samples': 1503, 'number_of_characters': 164223, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'asm_Beng-pan_Guru': {'num_samples': 1503, 'number_of_characters': 160201, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'asm_Beng-san_Deva': {'num_samples': 1503, 'number_of_characters': 158093, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'asm_Beng-sat_Olck': {'num_samples': 1503, 'number_of_characters': 169379, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'asm_Beng-snd_Deva': {'num_samples': 1503, 'number_of_characters': 162623, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'asm_Beng-tam_Taml': {'num_samples': 1503, 'number_of_characters': 174866, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'asm_Beng-tel_Telu': {'num_samples': 1503, 'number_of_characters': 157690, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'asm_Beng-urd_Arab': {'num_samples': 1503, 'number_of_characters': 161305, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'ben_Beng-asm_Beng': {'num_samples': 1503, 'number_of_characters': 155988, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'ben_Beng-brx_Deva': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'ben_Beng-doi_Deva': {'num_samples': 1503, 'number_of_characters': 161436, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'ben_Beng-eng_Latn': {'num_samples': 1503, 'number_of_characters': 155120, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'ben_Beng-gom_Deva': {'num_samples': 1503, 'number_of_characters': 150686, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'ben_Beng-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 152673, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'ben_Beng-hin_Deva': {'num_samples': 1503, 'number_of_characters': 154368, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'ben_Beng-kan_Knda': {'num_samples': 1503, 'number_of_characters': 159581, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'ben_Beng-kas_Arab': {'num_samples': 1503, 'number_of_characters': 159085, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'ben_Beng-mai_Deva': {'num_samples': 1503, 'number_of_characters': 156812, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'ben_Beng-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 167242, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'ben_Beng-mar_Deva': {'num_samples': 1503, 'number_of_characters': 157151, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'ben_Beng-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 151720, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'ben_Beng-npi_Deva': {'num_samples': 1503, 'number_of_characters': 155310, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'ben_Beng-ory_Orya': {'num_samples': 1503, 'number_of_characters': 158627, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'ben_Beng-pan_Guru': {'num_samples': 1503, 'number_of_characters': 154605, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'ben_Beng-san_Deva': {'num_samples': 1503, 'number_of_characters': 152497, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'ben_Beng-sat_Olck': {'num_samples': 1503, 'number_of_characters': 163783, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'ben_Beng-snd_Deva': {'num_samples': 1503, 'number_of_characters': 157027, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'ben_Beng-tam_Taml': {'num_samples': 1503, 'number_of_characters': 169270, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'ben_Beng-tel_Telu': {'num_samples': 1503, 'number_of_characters': 152094, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'ben_Beng-urd_Arab': {'num_samples': 1503, 'number_of_characters': 155709, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'brx_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162044, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'brx_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'brx_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 167492, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'brx_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161176, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'brx_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 156742, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'brx_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'brx_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 160424, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'brx_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 165637, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'brx_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165141, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'brx_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 162868, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'brx_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 173298, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'brx_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 163207, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'brx_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 157776, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'brx_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'brx_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 164683, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'brx_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 160661, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'brx_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 158553, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'brx_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 169839, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'brx_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 163083, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'brx_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 175326, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'brx_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158150, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'brx_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 161765, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'doi_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 167032, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'doi_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 161436, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'doi_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 167492, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'doi_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 166164, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'doi_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'doi_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 163717, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'doi_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 165412, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'doi_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 170625, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'doi_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 170129, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'doi_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 167856, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'doi_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 178286, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'doi_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 168195, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'doi_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 162764, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'doi_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 166354, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'doi_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 169671, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'doi_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 165649, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'doi_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 163541, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'doi_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 174827, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'doi_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 168071, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'doi_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 180314, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'doi_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 163138, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'doi_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 166753, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'eng_Latn-asm_Beng': {'num_samples': 1503, 'number_of_characters': 160716, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'eng_Latn-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155120, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'eng_Latn-brx_Deva': {'num_samples': 1503, 'number_of_characters': 161176, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'eng_Latn-doi_Deva': {'num_samples': 1503, 'number_of_characters': 166164, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'eng_Latn-gom_Deva': {'num_samples': 1503, 'number_of_characters': 155414, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'eng_Latn-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 157401, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'eng_Latn-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159096, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'eng_Latn-kan_Knda': {'num_samples': 1503, 'number_of_characters': 164309, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'eng_Latn-kas_Arab': {'num_samples': 1503, 'number_of_characters': 163813, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'eng_Latn-mai_Deva': {'num_samples': 1503, 'number_of_characters': 161540, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'eng_Latn-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 171970, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'eng_Latn-mar_Deva': {'num_samples': 1503, 'number_of_characters': 161879, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'eng_Latn-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'eng_Latn-npi_Deva': {'num_samples': 1503, 'number_of_characters': 160038, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'eng_Latn-ory_Orya': {'num_samples': 1503, 'number_of_characters': 163355, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'eng_Latn-pan_Guru': {'num_samples': 1503, 'number_of_characters': 159333, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'eng_Latn-san_Deva': {'num_samples': 1503, 'number_of_characters': 157225, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'eng_Latn-sat_Olck': {'num_samples': 1503, 'number_of_characters': 168511, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'eng_Latn-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161755, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'eng_Latn-tam_Taml': {'num_samples': 1503, 'number_of_characters': 173998, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'eng_Latn-tel_Telu': {'num_samples': 1503, 'number_of_characters': 156822, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'eng_Latn-urd_Arab': {'num_samples': 1503, 'number_of_characters': 160437, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'gom_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 156282, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'gom_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 150686, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'gom_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 156742, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'gom_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'gom_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 155414, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'gom_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 152967, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'gom_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 154662, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'gom_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 159875, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'gom_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 159379, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'gom_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 157106, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'gom_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 167536, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'gom_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 157445, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'gom_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 152014, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'gom_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 155604, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'gom_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 158921, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'gom_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 154899, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'gom_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 152791, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'gom_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 164077, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'gom_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 157321, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'gom_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 169564, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'gom_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 152388, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'gom_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 156003, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'guj_Gujr-asm_Beng': {'num_samples': 1503, 'number_of_characters': 158269, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'guj_Gujr-ben_Beng': {'num_samples': 1503, 'number_of_characters': 152673, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'guj_Gujr-brx_Deva': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'guj_Gujr-doi_Deva': {'num_samples': 1503, 'number_of_characters': 163717, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'guj_Gujr-eng_Latn': {'num_samples': 1503, 'number_of_characters': 157401, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'guj_Gujr-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152967, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'guj_Gujr-hin_Deva': {'num_samples': 1503, 'number_of_characters': 156649, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'guj_Gujr-kan_Knda': {'num_samples': 1503, 'number_of_characters': 161862, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'guj_Gujr-kas_Arab': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'guj_Gujr-mai_Deva': {'num_samples': 1503, 'number_of_characters': 159093, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'guj_Gujr-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 169523, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'guj_Gujr-mar_Deva': {'num_samples': 1503, 'number_of_characters': 159432, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'guj_Gujr-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 154001, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'guj_Gujr-npi_Deva': {'num_samples': 1503, 'number_of_characters': 157591, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'guj_Gujr-ory_Orya': {'num_samples': 1503, 'number_of_characters': 160908, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'guj_Gujr-pan_Guru': {'num_samples': 1503, 'number_of_characters': 156886, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'guj_Gujr-san_Deva': {'num_samples': 1503, 'number_of_characters': 154778, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'guj_Gujr-sat_Olck': {'num_samples': 1503, 'number_of_characters': 166064, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'guj_Gujr-snd_Deva': {'num_samples': 1503, 'number_of_characters': 159308, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'guj_Gujr-tam_Taml': {'num_samples': 1503, 'number_of_characters': 171551, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'guj_Gujr-tel_Telu': {'num_samples': 1503, 'number_of_characters': 154375, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'guj_Gujr-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157990, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'hin_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 159964, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'hin_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 154368, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'hin_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 160424, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'hin_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 165412, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'hin_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 159096, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'hin_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 154662, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'hin_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 156649, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'hin_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 163557, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'hin_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 163061, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'hin_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 160788, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'hin_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 171218, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'hin_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 161127, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'hin_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 155696, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'hin_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 159286, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'hin_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 162603, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'hin_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 158581, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'hin_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 156473, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'hin_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 167759, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'hin_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'hin_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 173246, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'hin_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 156070, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'hin_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 159685, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'kan_Knda-asm_Beng': {'num_samples': 1503, 'number_of_characters': 165177, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'kan_Knda-ben_Beng': {'num_samples': 1503, 'number_of_characters': 159581, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'kan_Knda-brx_Deva': {'num_samples': 1503, 'number_of_characters': 165637, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'kan_Knda-doi_Deva': {'num_samples': 1503, 'number_of_characters': 170625, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'kan_Knda-eng_Latn': {'num_samples': 1503, 'number_of_characters': 164309, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'kan_Knda-gom_Deva': {'num_samples': 1503, 'number_of_characters': 159875, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'kan_Knda-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 161862, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'kan_Knda-hin_Deva': {'num_samples': 1503, 'number_of_characters': 163557, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'kan_Knda-kas_Arab': {'num_samples': 1503, 'number_of_characters': 168274, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'kan_Knda-mai_Deva': {'num_samples': 1503, 'number_of_characters': 166001, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'kan_Knda-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 176431, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'kan_Knda-mar_Deva': {'num_samples': 1503, 'number_of_characters': 166340, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'kan_Knda-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 160909, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'kan_Knda-npi_Deva': {'num_samples': 1503, 'number_of_characters': 164499, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'kan_Knda-ory_Orya': {'num_samples': 1503, 'number_of_characters': 167816, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'kan_Knda-pan_Guru': {'num_samples': 1503, 'number_of_characters': 163794, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'kan_Knda-san_Deva': {'num_samples': 1503, 'number_of_characters': 161686, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'kan_Knda-sat_Olck': {'num_samples': 1503, 'number_of_characters': 172972, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'kan_Knda-snd_Deva': {'num_samples': 1503, 'number_of_characters': 166216, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'kan_Knda-tam_Taml': {'num_samples': 1503, 'number_of_characters': 178459, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'kan_Knda-tel_Telu': {'num_samples': 1503, 'number_of_characters': 161283, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'kan_Knda-urd_Arab': {'num_samples': 1503, 'number_of_characters': 164898, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'kas_Arab-asm_Beng': {'num_samples': 1503, 'number_of_characters': 164681, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'kas_Arab-ben_Beng': {'num_samples': 1503, 'number_of_characters': 159085, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'kas_Arab-brx_Deva': {'num_samples': 1503, 'number_of_characters': 165141, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'kas_Arab-doi_Deva': {'num_samples': 1503, 'number_of_characters': 170129, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'kas_Arab-eng_Latn': {'num_samples': 1503, 'number_of_characters': 163813, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'kas_Arab-gom_Deva': {'num_samples': 1503, 'number_of_characters': 159379, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'kas_Arab-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'kas_Arab-hin_Deva': {'num_samples': 1503, 'number_of_characters': 163061, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'kas_Arab-kan_Knda': {'num_samples': 1503, 'number_of_characters': 168274, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'kas_Arab-mai_Deva': {'num_samples': 1503, 'number_of_characters': 165505, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'kas_Arab-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 175935, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'kas_Arab-mar_Deva': {'num_samples': 1503, 'number_of_characters': 165844, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'kas_Arab-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 160413, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'kas_Arab-npi_Deva': {'num_samples': 1503, 'number_of_characters': 164003, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'kas_Arab-ory_Orya': {'num_samples': 1503, 'number_of_characters': 167320, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'kas_Arab-pan_Guru': {'num_samples': 1503, 'number_of_characters': 163298, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'kas_Arab-san_Deva': {'num_samples': 1503, 'number_of_characters': 161190, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'kas_Arab-sat_Olck': {'num_samples': 1503, 'number_of_characters': 172476, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'kas_Arab-snd_Deva': {'num_samples': 1503, 'number_of_characters': 165720, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'kas_Arab-tam_Taml': {'num_samples': 1503, 'number_of_characters': 177963, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'kas_Arab-tel_Telu': {'num_samples': 1503, 'number_of_characters': 160787, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'kas_Arab-urd_Arab': {'num_samples': 1503, 'number_of_characters': 164402, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mai_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162408, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mai_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 156812, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mai_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 162868, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mai_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 167856, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mai_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161540, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mai_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 157106, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mai_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 159093, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mai_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 160788, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mai_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 166001, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mai_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165505, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mai_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 173662, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'mai_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 163571, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'mai_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 158140, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'mai_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mai_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 165047, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mai_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 161025, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mai_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 158917, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mai_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 170203, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mai_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 163447, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mai_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 175690, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mai_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158514, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mai_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 162129, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mal_Mlym-asm_Beng': {'num_samples': 1503, 'number_of_characters': 172838, 'unique_pairs': 1498, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mal_Mlym-ben_Beng': {'num_samples': 1503, 'number_of_characters': 167242, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mal_Mlym-brx_Deva': {'num_samples': 1503, 'number_of_characters': 173298, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mal_Mlym-doi_Deva': {'num_samples': 1503, 'number_of_characters': 178286, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mal_Mlym-eng_Latn': {'num_samples': 1503, 'number_of_characters': 171970, 'unique_pairs': 1499, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mal_Mlym-gom_Deva': {'num_samples': 1503, 'number_of_characters': 167536, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mal_Mlym-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 169523, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mal_Mlym-hin_Deva': {'num_samples': 1503, 'number_of_characters': 171218, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mal_Mlym-kan_Knda': {'num_samples': 1503, 'number_of_characters': 176431, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mal_Mlym-kas_Arab': {'num_samples': 1503, 'number_of_characters': 175935, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mal_Mlym-mai_Deva': {'num_samples': 1503, 'number_of_characters': 173662, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'mal_Mlym-mar_Deva': {'num_samples': 1503, 'number_of_characters': 174001, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'mal_Mlym-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 168570, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'mal_Mlym-npi_Deva': {'num_samples': 1503, 'number_of_characters': 172160, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mal_Mlym-ory_Orya': {'num_samples': 1503, 'number_of_characters': 175477, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mal_Mlym-pan_Guru': {'num_samples': 1503, 'number_of_characters': 171455, 'unique_pairs': 1498, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mal_Mlym-san_Deva': {'num_samples': 1503, 'number_of_characters': 169347, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mal_Mlym-sat_Olck': {'num_samples': 1503, 'number_of_characters': 180633, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mal_Mlym-snd_Deva': {'num_samples': 1503, 'number_of_characters': 173877, 'unique_pairs': 1499, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mal_Mlym-tam_Taml': {'num_samples': 1503, 'number_of_characters': 186120, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mal_Mlym-tel_Telu': {'num_samples': 1503, 'number_of_characters': 168944, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mal_Mlym-urd_Arab': {'num_samples': 1503, 'number_of_characters': 172559, 'unique_pairs': 1499, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mar_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162747, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mar_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 157151, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mar_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 163207, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mar_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 168195, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mar_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161879, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mar_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 157445, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mar_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 159432, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mar_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 161127, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mar_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 166340, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mar_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165844, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mar_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 163571, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'mar_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 174001, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'mar_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 158479, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'mar_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 162069, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mar_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 165386, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mar_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 161364, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mar_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 159256, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mar_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 170542, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mar_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 163786, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mar_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 176029, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mar_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158853, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mar_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 162468, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mni_Mtei-asm_Beng': {'num_samples': 1503, 'number_of_characters': 157316, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mni_Mtei-ben_Beng': {'num_samples': 1503, 'number_of_characters': 151720, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mni_Mtei-brx_Deva': {'num_samples': 1503, 'number_of_characters': 157776, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mni_Mtei-doi_Deva': {'num_samples': 1503, 'number_of_characters': 162764, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mni_Mtei-eng_Latn': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mni_Mtei-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152014, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mni_Mtei-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 154001, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mni_Mtei-hin_Deva': {'num_samples': 1503, 'number_of_characters': 155696, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mni_Mtei-kan_Knda': {'num_samples': 1503, 'number_of_characters': 160909, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mni_Mtei-kas_Arab': {'num_samples': 1503, 'number_of_characters': 160413, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mni_Mtei-mai_Deva': {'num_samples': 1503, 'number_of_characters': 158140, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'mni_Mtei-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 168570, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'mni_Mtei-mar_Deva': {'num_samples': 1503, 'number_of_characters': 158479, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'mni_Mtei-npi_Deva': {'num_samples': 1503, 'number_of_characters': 156638, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mni_Mtei-ory_Orya': {'num_samples': 1503, 'number_of_characters': 159955, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mni_Mtei-pan_Guru': {'num_samples': 1503, 'number_of_characters': 155933, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mni_Mtei-san_Deva': {'num_samples': 1503, 'number_of_characters': 153825, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mni_Mtei-sat_Olck': {'num_samples': 1503, 'number_of_characters': 165111, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mni_Mtei-snd_Deva': {'num_samples': 1503, 'number_of_characters': 158355, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mni_Mtei-tam_Taml': {'num_samples': 1503, 'number_of_characters': 170598, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mni_Mtei-tel_Telu': {'num_samples': 1503, 'number_of_characters': 153422, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mni_Mtei-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157037, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'npi_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 160906, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'npi_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155310, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'npi_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'npi_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 166354, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'npi_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 160038, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'npi_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 155604, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'npi_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 157591, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'npi_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159286, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'npi_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 164499, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'npi_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 164003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'npi_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'npi_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 172160, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'npi_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 162069, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'npi_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 156638, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'npi_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 163545, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'npi_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 159523, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'npi_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 157415, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'npi_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 168701, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'npi_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161945, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'npi_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 174188, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'npi_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 157012, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'npi_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 160627, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'ory_Orya-asm_Beng': {'num_samples': 1503, 'number_of_characters': 164223, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'ory_Orya-ben_Beng': {'num_samples': 1503, 'number_of_characters': 158627, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'ory_Orya-brx_Deva': {'num_samples': 1503, 'number_of_characters': 164683, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'ory_Orya-doi_Deva': {'num_samples': 1503, 'number_of_characters': 169671, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'ory_Orya-eng_Latn': {'num_samples': 1503, 'number_of_characters': 163355, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'ory_Orya-gom_Deva': {'num_samples': 1503, 'number_of_characters': 158921, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'ory_Orya-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 160908, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'ory_Orya-hin_Deva': {'num_samples': 1503, 'number_of_characters': 162603, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'ory_Orya-kan_Knda': {'num_samples': 1503, 'number_of_characters': 167816, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'ory_Orya-kas_Arab': {'num_samples': 1503, 'number_of_characters': 167320, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'ory_Orya-mai_Deva': {'num_samples': 1503, 'number_of_characters': 165047, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'ory_Orya-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 175477, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'ory_Orya-mar_Deva': {'num_samples': 1503, 'number_of_characters': 165386, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'ory_Orya-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 159955, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'ory_Orya-npi_Deva': {'num_samples': 1503, 'number_of_characters': 163545, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'ory_Orya-pan_Guru': {'num_samples': 1503, 'number_of_characters': 162840, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'ory_Orya-san_Deva': {'num_samples': 1503, 'number_of_characters': 160732, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'ory_Orya-sat_Olck': {'num_samples': 1503, 'number_of_characters': 172018, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'ory_Orya-snd_Deva': {'num_samples': 1503, 'number_of_characters': 165262, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'ory_Orya-tam_Taml': {'num_samples': 1503, 'number_of_characters': 177505, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'ory_Orya-tel_Telu': {'num_samples': 1503, 'number_of_characters': 160329, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'ory_Orya-urd_Arab': {'num_samples': 1503, 'number_of_characters': 163944, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'pan_Guru-asm_Beng': {'num_samples': 1503, 'number_of_characters': 160201, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'pan_Guru-ben_Beng': {'num_samples': 1503, 'number_of_characters': 154605, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'pan_Guru-brx_Deva': {'num_samples': 1503, 'number_of_characters': 160661, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'pan_Guru-doi_Deva': {'num_samples': 1503, 'number_of_characters': 165649, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'pan_Guru-eng_Latn': {'num_samples': 1503, 'number_of_characters': 159333, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'pan_Guru-gom_Deva': {'num_samples': 1503, 'number_of_characters': 154899, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'pan_Guru-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 156886, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'pan_Guru-hin_Deva': {'num_samples': 1503, 'number_of_characters': 158581, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'pan_Guru-kan_Knda': {'num_samples': 1503, 'number_of_characters': 163794, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'pan_Guru-kas_Arab': {'num_samples': 1503, 'number_of_characters': 163298, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'pan_Guru-mai_Deva': {'num_samples': 1503, 'number_of_characters': 161025, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'pan_Guru-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 171455, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'pan_Guru-mar_Deva': {'num_samples': 1503, 'number_of_characters': 161364, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'pan_Guru-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 155933, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'pan_Guru-npi_Deva': {'num_samples': 1503, 'number_of_characters': 159523, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'pan_Guru-ory_Orya': {'num_samples': 1503, 'number_of_characters': 162840, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'pan_Guru-san_Deva': {'num_samples': 1503, 'number_of_characters': 156710, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'pan_Guru-sat_Olck': {'num_samples': 1503, 'number_of_characters': 167996, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'pan_Guru-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161240, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'pan_Guru-tam_Taml': {'num_samples': 1503, 'number_of_characters': 173483, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'pan_Guru-tel_Telu': {'num_samples': 1503, 'number_of_characters': 156307, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'pan_Guru-urd_Arab': {'num_samples': 1503, 'number_of_characters': 159922, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'san_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 158093, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'san_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 152497, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'san_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 158553, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'san_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 163541, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'san_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 157225, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'san_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152791, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'san_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 154778, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'san_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 156473, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'san_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 161686, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'san_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 161190, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'san_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 158917, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'san_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 169347, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'san_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 159256, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'san_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 153825, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'san_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 157415, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'san_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 160732, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'san_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 156710, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'san_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 165888, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'san_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 159132, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'san_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 171375, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'san_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 154199, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'san_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157814, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'sat_Olck-asm_Beng': {'num_samples': 1503, 'number_of_characters': 169379, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'sat_Olck-ben_Beng': {'num_samples': 1503, 'number_of_characters': 163783, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'sat_Olck-brx_Deva': {'num_samples': 1503, 'number_of_characters': 169839, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'sat_Olck-doi_Deva': {'num_samples': 1503, 'number_of_characters': 174827, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'sat_Olck-eng_Latn': {'num_samples': 1503, 'number_of_characters': 168511, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'sat_Olck-gom_Deva': {'num_samples': 1503, 'number_of_characters': 164077, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'sat_Olck-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 166064, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'sat_Olck-hin_Deva': {'num_samples': 1503, 'number_of_characters': 167759, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'sat_Olck-kan_Knda': {'num_samples': 1503, 'number_of_characters': 172972, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'sat_Olck-kas_Arab': {'num_samples': 1503, 'number_of_characters': 172476, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'sat_Olck-mai_Deva': {'num_samples': 1503, 'number_of_characters': 170203, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'sat_Olck-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 180633, 'unique_pairs': 1501, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'sat_Olck-mar_Deva': {'num_samples': 1503, 'number_of_characters': 170542, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'sat_Olck-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 165111, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'sat_Olck-npi_Deva': {'num_samples': 1503, 'number_of_characters': 168701, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'sat_Olck-ory_Orya': {'num_samples': 1503, 'number_of_characters': 172018, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'sat_Olck-pan_Guru': {'num_samples': 1503, 'number_of_characters': 167996, 'unique_pairs': 1501, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'sat_Olck-san_Deva': {'num_samples': 1503, 'number_of_characters': 165888, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'sat_Olck-snd_Deva': {'num_samples': 1503, 'number_of_characters': 170418, 'unique_pairs': 1501, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'sat_Olck-tam_Taml': {'num_samples': 1503, 'number_of_characters': 182661, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'sat_Olck-tel_Telu': {'num_samples': 1503, 'number_of_characters': 165485, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'sat_Olck-urd_Arab': {'num_samples': 1503, 'number_of_characters': 169100, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'snd_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162623, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'snd_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 157027, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'snd_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 163083, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'snd_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 168071, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'snd_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161755, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'snd_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 157321, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'snd_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 159308, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'snd_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 161003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'snd_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 166216, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'snd_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165720, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'snd_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 163447, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'snd_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 173877, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'snd_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 163786, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'snd_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 158355, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'snd_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 161945, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'snd_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 165262, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'snd_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 161240, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'snd_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 159132, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'snd_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 170418, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'snd_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 175905, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'snd_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'snd_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 162344, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'tam_Taml-asm_Beng': {'num_samples': 1503, 'number_of_characters': 174866, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'tam_Taml-ben_Beng': {'num_samples': 1503, 'number_of_characters': 169270, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'tam_Taml-brx_Deva': {'num_samples': 1503, 'number_of_characters': 175326, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'tam_Taml-doi_Deva': {'num_samples': 1503, 'number_of_characters': 180314, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'tam_Taml-eng_Latn': {'num_samples': 1503, 'number_of_characters': 173998, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'tam_Taml-gom_Deva': {'num_samples': 1503, 'number_of_characters': 169564, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'tam_Taml-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 171551, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'tam_Taml-hin_Deva': {'num_samples': 1503, 'number_of_characters': 173246, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'tam_Taml-kan_Knda': {'num_samples': 1503, 'number_of_characters': 178459, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'tam_Taml-kas_Arab': {'num_samples': 1503, 'number_of_characters': 177963, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'tam_Taml-mai_Deva': {'num_samples': 1503, 'number_of_characters': 175690, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'tam_Taml-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 186120, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'tam_Taml-mar_Deva': {'num_samples': 1503, 'number_of_characters': 176029, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'tam_Taml-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 170598, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'tam_Taml-npi_Deva': {'num_samples': 1503, 'number_of_characters': 174188, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'tam_Taml-ory_Orya': {'num_samples': 1503, 'number_of_characters': 177505, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'tam_Taml-pan_Guru': {'num_samples': 1503, 'number_of_characters': 173483, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'tam_Taml-san_Deva': {'num_samples': 1503, 'number_of_characters': 171375, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'tam_Taml-sat_Olck': {'num_samples': 1503, 'number_of_characters': 182661, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'tam_Taml-snd_Deva': {'num_samples': 1503, 'number_of_characters': 175905, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'tam_Taml-tel_Telu': {'num_samples': 1503, 'number_of_characters': 170972, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'tam_Taml-urd_Arab': {'num_samples': 1503, 'number_of_characters': 174587, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'tel_Telu-asm_Beng': {'num_samples': 1503, 'number_of_characters': 157690, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'tel_Telu-ben_Beng': {'num_samples': 1503, 'number_of_characters': 152094, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'tel_Telu-brx_Deva': {'num_samples': 1503, 'number_of_characters': 158150, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'tel_Telu-doi_Deva': {'num_samples': 1503, 'number_of_characters': 163138, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'tel_Telu-eng_Latn': {'num_samples': 1503, 'number_of_characters': 156822, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'tel_Telu-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152388, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'tel_Telu-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 154375, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'tel_Telu-hin_Deva': {'num_samples': 1503, 'number_of_characters': 156070, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'tel_Telu-kan_Knda': {'num_samples': 1503, 'number_of_characters': 161283, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'tel_Telu-kas_Arab': {'num_samples': 1503, 'number_of_characters': 160787, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'tel_Telu-mai_Deva': {'num_samples': 1503, 'number_of_characters': 158514, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'tel_Telu-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 168944, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'tel_Telu-mar_Deva': {'num_samples': 1503, 'number_of_characters': 158853, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'tel_Telu-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 153422, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'tel_Telu-npi_Deva': {'num_samples': 1503, 'number_of_characters': 157012, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'tel_Telu-ory_Orya': {'num_samples': 1503, 'number_of_characters': 160329, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'tel_Telu-pan_Guru': {'num_samples': 1503, 'number_of_characters': 156307, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'tel_Telu-san_Deva': {'num_samples': 1503, 'number_of_characters': 154199, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'tel_Telu-sat_Olck': {'num_samples': 1503, 'number_of_characters': 165485, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'tel_Telu-snd_Deva': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'tel_Telu-tam_Taml': {'num_samples': 1503, 'number_of_characters': 170972, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'tel_Telu-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157411, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'urd_Arab-asm_Beng': {'num_samples': 1503, 'number_of_characters': 161305, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'urd_Arab-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155709, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'urd_Arab-brx_Deva': {'num_samples': 1503, 'number_of_characters': 161765, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'urd_Arab-doi_Deva': {'num_samples': 1503, 'number_of_characters': 166753, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'urd_Arab-eng_Latn': {'num_samples': 1503, 'number_of_characters': 160437, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'urd_Arab-gom_Deva': {'num_samples': 1503, 'number_of_characters': 156003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'urd_Arab-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 157990, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'urd_Arab-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159685, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'urd_Arab-kan_Knda': {'num_samples': 1503, 'number_of_characters': 164898, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'urd_Arab-kas_Arab': {'num_samples': 1503, 'number_of_characters': 164402, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'urd_Arab-mai_Deva': {'num_samples': 1503, 'number_of_characters': 162129, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'urd_Arab-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 172559, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'urd_Arab-mar_Deva': {'num_samples': 1503, 'number_of_characters': 162468, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'urd_Arab-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 157037, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'urd_Arab-npi_Deva': {'num_samples': 1503, 'number_of_characters': 160627, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'urd_Arab-ory_Orya': {'num_samples': 1503, 'number_of_characters': 163944, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'urd_Arab-pan_Guru': {'num_samples': 1503, 'number_of_characters': 159922, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'urd_Arab-san_Deva': {'num_samples': 1503, 'number_of_characters': 157814, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'urd_Arab-sat_Olck': {'num_samples': 1503, 'number_of_characters': 169100, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'urd_Arab-snd_Deva': {'num_samples': 1503, 'number_of_characters': 162344, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'urd_Arab-tam_Taml': {'num_samples': 1503, 'number_of_characters': 174587, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'urd_Arab-tel_Telu': {'num_samples': 1503, 'number_of_characters': 157411, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}}}} | -| [IN22GenBitextMining](https://huggingface.co/datasets/ai4bharat/IN22-Gen) (Jay Gala, 2023) | ['asm', 'ben', 'brx', 'doi', 'eng', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Web, Legal, Government, News, Religious, Non-fiction, Written] | {'test': 518144} | {'test': {'num_samples': 518144, 'number_of_characters': 162367876, 'unique_pairs': 518101, 'min_sentence1_length': 9, 'average_sentence1_length': 156.68, 'max_sentence1_length': 692, 'unique_sentence1': 23550, 'min_sentence2_length': 9, 'average_sentence2_length': 156.68, 'max_sentence2_length': 692, 'unique_sentence2': 23550, 'hf_subset_descriptive_stats': {'asm_Beng-ben_Beng': {'num_samples': 1024, 'number_of_characters': 310622, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'asm_Beng-brx_Deva': {'num_samples': 1024, 'number_of_characters': 323609, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'asm_Beng-doi_Deva': {'num_samples': 1024, 'number_of_characters': 319020, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'asm_Beng-eng_Latn': {'num_samples': 1024, 'number_of_characters': 320098, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'asm_Beng-gom_Deva': {'num_samples': 1024, 'number_of_characters': 312594, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'asm_Beng-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 309440, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'asm_Beng-hin_Deva': {'num_samples': 1024, 'number_of_characters': 320106, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'asm_Beng-kan_Knda': {'num_samples': 1024, 'number_of_characters': 332064, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'asm_Beng-kas_Arab': {'num_samples': 1024, 'number_of_characters': 322764, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'asm_Beng-mai_Deva': {'num_samples': 1024, 'number_of_characters': 308682, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'asm_Beng-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 343636, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'asm_Beng-mar_Deva': {'num_samples': 1024, 'number_of_characters': 321784, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'asm_Beng-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 313134, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'asm_Beng-npi_Deva': {'num_samples': 1024, 'number_of_characters': 313419, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'asm_Beng-ory_Orya': {'num_samples': 1024, 'number_of_characters': 334226, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'asm_Beng-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306863, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'asm_Beng-san_Deva': {'num_samples': 1024, 'number_of_characters': 318079, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'asm_Beng-sat_Olck': {'num_samples': 1024, 'number_of_characters': 326732, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'asm_Beng-snd_Deva': {'num_samples': 1024, 'number_of_characters': 320421, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'asm_Beng-tam_Taml': {'num_samples': 1024, 'number_of_characters': 348346, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'asm_Beng-tel_Telu': {'num_samples': 1024, 'number_of_characters': 319045, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'asm_Beng-urd_Arab': {'num_samples': 1024, 'number_of_characters': 315134, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'ben_Beng-asm_Beng': {'num_samples': 1024, 'number_of_characters': 310622, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'ben_Beng-brx_Deva': {'num_samples': 1024, 'number_of_characters': 313313, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'ben_Beng-doi_Deva': {'num_samples': 1024, 'number_of_characters': 308724, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'ben_Beng-eng_Latn': {'num_samples': 1024, 'number_of_characters': 309802, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'ben_Beng-gom_Deva': {'num_samples': 1024, 'number_of_characters': 302298, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'ben_Beng-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 299144, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'ben_Beng-hin_Deva': {'num_samples': 1024, 'number_of_characters': 309810, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'ben_Beng-kan_Knda': {'num_samples': 1024, 'number_of_characters': 321768, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'ben_Beng-kas_Arab': {'num_samples': 1024, 'number_of_characters': 312468, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'ben_Beng-mai_Deva': {'num_samples': 1024, 'number_of_characters': 298386, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'ben_Beng-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 333340, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'ben_Beng-mar_Deva': {'num_samples': 1024, 'number_of_characters': 311488, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'ben_Beng-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 302838, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'ben_Beng-npi_Deva': {'num_samples': 1024, 'number_of_characters': 303123, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'ben_Beng-ory_Orya': {'num_samples': 1024, 'number_of_characters': 323930, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'ben_Beng-pan_Guru': {'num_samples': 1024, 'number_of_characters': 296567, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'ben_Beng-san_Deva': {'num_samples': 1024, 'number_of_characters': 307783, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'ben_Beng-sat_Olck': {'num_samples': 1024, 'number_of_characters': 316436, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'ben_Beng-snd_Deva': {'num_samples': 1024, 'number_of_characters': 310125, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'ben_Beng-tam_Taml': {'num_samples': 1024, 'number_of_characters': 338050, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'ben_Beng-tel_Telu': {'num_samples': 1024, 'number_of_characters': 308749, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'ben_Beng-urd_Arab': {'num_samples': 1024, 'number_of_characters': 304838, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'brx_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 323609, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'brx_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 313313, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'brx_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 321711, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'brx_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 322789, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'brx_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 315285, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'brx_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 312131, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'brx_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 322797, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'brx_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 334755, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'brx_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 325455, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'brx_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 311373, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'brx_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 346327, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'brx_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 324475, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'brx_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 315825, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'brx_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 316110, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'brx_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 336917, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'brx_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 309554, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'brx_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 320770, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'brx_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 329423, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'brx_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 323112, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'brx_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 351037, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'brx_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 321736, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'brx_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 317825, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'doi_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 319020, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'doi_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 308724, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'doi_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 321711, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'doi_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 318200, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'doi_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 310696, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'doi_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 307542, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'doi_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 318208, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'doi_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 330166, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'doi_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 320866, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'doi_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 306784, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'doi_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 341738, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'doi_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 319886, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'doi_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 311236, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'doi_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 311521, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'doi_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 332328, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'doi_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 304965, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'doi_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 316181, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'doi_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 324834, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'doi_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 318523, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'doi_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 346448, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'doi_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 317147, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'doi_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 313236, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'eng_Latn-asm_Beng': {'num_samples': 1024, 'number_of_characters': 320098, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'eng_Latn-ben_Beng': {'num_samples': 1024, 'number_of_characters': 309802, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'eng_Latn-brx_Deva': {'num_samples': 1024, 'number_of_characters': 322789, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'eng_Latn-doi_Deva': {'num_samples': 1024, 'number_of_characters': 318200, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'eng_Latn-gom_Deva': {'num_samples': 1024, 'number_of_characters': 311774, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'eng_Latn-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 308620, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'eng_Latn-hin_Deva': {'num_samples': 1024, 'number_of_characters': 319286, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'eng_Latn-kan_Knda': {'num_samples': 1024, 'number_of_characters': 331244, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'eng_Latn-kas_Arab': {'num_samples': 1024, 'number_of_characters': 321944, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'eng_Latn-mai_Deva': {'num_samples': 1024, 'number_of_characters': 307862, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'eng_Latn-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 342816, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'eng_Latn-mar_Deva': {'num_samples': 1024, 'number_of_characters': 320964, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'eng_Latn-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 312314, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'eng_Latn-npi_Deva': {'num_samples': 1024, 'number_of_characters': 312599, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'eng_Latn-ory_Orya': {'num_samples': 1024, 'number_of_characters': 333406, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'eng_Latn-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306043, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'eng_Latn-san_Deva': {'num_samples': 1024, 'number_of_characters': 317259, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'eng_Latn-sat_Olck': {'num_samples': 1024, 'number_of_characters': 325912, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'eng_Latn-snd_Deva': {'num_samples': 1024, 'number_of_characters': 319601, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'eng_Latn-tam_Taml': {'num_samples': 1024, 'number_of_characters': 347526, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'eng_Latn-tel_Telu': {'num_samples': 1024, 'number_of_characters': 318225, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'eng_Latn-urd_Arab': {'num_samples': 1024, 'number_of_characters': 314314, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'gom_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 312594, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'gom_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 302298, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'gom_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 315285, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'gom_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 310696, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'gom_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 311774, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'gom_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 301116, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'gom_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 311782, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'gom_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 323740, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'gom_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 314440, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'gom_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 300358, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'gom_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 335312, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'gom_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 313460, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'gom_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 304810, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'gom_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 305095, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'gom_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 325902, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'gom_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 298539, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'gom_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 309755, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'gom_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 318408, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'gom_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 312097, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'gom_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 340022, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'gom_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 310721, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'gom_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 306810, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'guj_Gujr-asm_Beng': {'num_samples': 1024, 'number_of_characters': 309440, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'guj_Gujr-ben_Beng': {'num_samples': 1024, 'number_of_characters': 299144, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'guj_Gujr-brx_Deva': {'num_samples': 1024, 'number_of_characters': 312131, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'guj_Gujr-doi_Deva': {'num_samples': 1024, 'number_of_characters': 307542, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'guj_Gujr-eng_Latn': {'num_samples': 1024, 'number_of_characters': 308620, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'guj_Gujr-gom_Deva': {'num_samples': 1024, 'number_of_characters': 301116, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'guj_Gujr-hin_Deva': {'num_samples': 1024, 'number_of_characters': 308628, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'guj_Gujr-kan_Knda': {'num_samples': 1024, 'number_of_characters': 320586, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'guj_Gujr-kas_Arab': {'num_samples': 1024, 'number_of_characters': 311286, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'guj_Gujr-mai_Deva': {'num_samples': 1024, 'number_of_characters': 297204, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'guj_Gujr-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 332158, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'guj_Gujr-mar_Deva': {'num_samples': 1024, 'number_of_characters': 310306, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'guj_Gujr-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 301656, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'guj_Gujr-npi_Deva': {'num_samples': 1024, 'number_of_characters': 301941, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'guj_Gujr-ory_Orya': {'num_samples': 1024, 'number_of_characters': 322748, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'guj_Gujr-pan_Guru': {'num_samples': 1024, 'number_of_characters': 295385, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'guj_Gujr-san_Deva': {'num_samples': 1024, 'number_of_characters': 306601, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'guj_Gujr-sat_Olck': {'num_samples': 1024, 'number_of_characters': 315254, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'guj_Gujr-snd_Deva': {'num_samples': 1024, 'number_of_characters': 308943, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'guj_Gujr-tam_Taml': {'num_samples': 1024, 'number_of_characters': 336868, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'guj_Gujr-tel_Telu': {'num_samples': 1024, 'number_of_characters': 307567, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'guj_Gujr-urd_Arab': {'num_samples': 1024, 'number_of_characters': 303656, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'hin_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 320106, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'hin_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 309810, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'hin_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 322797, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'hin_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 318208, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'hin_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 319286, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'hin_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 311782, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'hin_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 308628, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'hin_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 331252, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'hin_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 321952, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'hin_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 307870, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'hin_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 342824, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'hin_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 320972, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'hin_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 312322, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'hin_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 312607, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'hin_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 333414, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'hin_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306051, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'hin_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 317267, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'hin_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 325920, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'hin_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 319609, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'hin_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 347534, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'hin_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 318233, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'hin_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 314322, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'kan_Knda-asm_Beng': {'num_samples': 1024, 'number_of_characters': 332064, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'kan_Knda-ben_Beng': {'num_samples': 1024, 'number_of_characters': 321768, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'kan_Knda-brx_Deva': {'num_samples': 1024, 'number_of_characters': 334755, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'kan_Knda-doi_Deva': {'num_samples': 1024, 'number_of_characters': 330166, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'kan_Knda-eng_Latn': {'num_samples': 1024, 'number_of_characters': 331244, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'kan_Knda-gom_Deva': {'num_samples': 1024, 'number_of_characters': 323740, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'kan_Knda-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 320586, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'kan_Knda-hin_Deva': {'num_samples': 1024, 'number_of_characters': 331252, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'kan_Knda-kas_Arab': {'num_samples': 1024, 'number_of_characters': 333910, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'kan_Knda-mai_Deva': {'num_samples': 1024, 'number_of_characters': 319828, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'kan_Knda-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 354782, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'kan_Knda-mar_Deva': {'num_samples': 1024, 'number_of_characters': 332930, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'kan_Knda-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 324280, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'kan_Knda-npi_Deva': {'num_samples': 1024, 'number_of_characters': 324565, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'kan_Knda-ory_Orya': {'num_samples': 1024, 'number_of_characters': 345372, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'kan_Knda-pan_Guru': {'num_samples': 1024, 'number_of_characters': 318009, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'kan_Knda-san_Deva': {'num_samples': 1024, 'number_of_characters': 329225, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'kan_Knda-sat_Olck': {'num_samples': 1024, 'number_of_characters': 337878, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'kan_Knda-snd_Deva': {'num_samples': 1024, 'number_of_characters': 331567, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'kan_Knda-tam_Taml': {'num_samples': 1024, 'number_of_characters': 359492, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'kan_Knda-tel_Telu': {'num_samples': 1024, 'number_of_characters': 330191, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'kan_Knda-urd_Arab': {'num_samples': 1024, 'number_of_characters': 326280, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'kas_Arab-asm_Beng': {'num_samples': 1024, 'number_of_characters': 322764, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'kas_Arab-ben_Beng': {'num_samples': 1024, 'number_of_characters': 312468, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'kas_Arab-brx_Deva': {'num_samples': 1024, 'number_of_characters': 325455, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'kas_Arab-doi_Deva': {'num_samples': 1024, 'number_of_characters': 320866, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'kas_Arab-eng_Latn': {'num_samples': 1024, 'number_of_characters': 321944, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'kas_Arab-gom_Deva': {'num_samples': 1024, 'number_of_characters': 314440, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'kas_Arab-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 311286, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'kas_Arab-hin_Deva': {'num_samples': 1024, 'number_of_characters': 321952, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'kas_Arab-kan_Knda': {'num_samples': 1024, 'number_of_characters': 333910, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'kas_Arab-mai_Deva': {'num_samples': 1024, 'number_of_characters': 310528, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'kas_Arab-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 345482, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'kas_Arab-mar_Deva': {'num_samples': 1024, 'number_of_characters': 323630, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'kas_Arab-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 314980, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'kas_Arab-npi_Deva': {'num_samples': 1024, 'number_of_characters': 315265, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'kas_Arab-ory_Orya': {'num_samples': 1024, 'number_of_characters': 336072, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'kas_Arab-pan_Guru': {'num_samples': 1024, 'number_of_characters': 308709, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'kas_Arab-san_Deva': {'num_samples': 1024, 'number_of_characters': 319925, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'kas_Arab-sat_Olck': {'num_samples': 1024, 'number_of_characters': 328578, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'kas_Arab-snd_Deva': {'num_samples': 1024, 'number_of_characters': 322267, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'kas_Arab-tam_Taml': {'num_samples': 1024, 'number_of_characters': 350192, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'kas_Arab-tel_Telu': {'num_samples': 1024, 'number_of_characters': 320891, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'kas_Arab-urd_Arab': {'num_samples': 1024, 'number_of_characters': 316980, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mai_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 308682, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mai_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 298386, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mai_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 311373, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mai_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 306784, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mai_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 307862, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mai_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 300358, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mai_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 297204, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mai_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 307870, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mai_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 319828, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mai_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 310528, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mai_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 331400, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'mai_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 309548, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'mai_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 300898, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'mai_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 301183, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mai_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 321990, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mai_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 294627, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mai_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 305843, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mai_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 314496, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mai_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 308185, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mai_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 336110, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mai_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 306809, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mai_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 302898, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mal_Mlym-asm_Beng': {'num_samples': 1024, 'number_of_characters': 343636, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mal_Mlym-ben_Beng': {'num_samples': 1024, 'number_of_characters': 333340, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mal_Mlym-brx_Deva': {'num_samples': 1024, 'number_of_characters': 346327, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mal_Mlym-doi_Deva': {'num_samples': 1024, 'number_of_characters': 341738, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mal_Mlym-eng_Latn': {'num_samples': 1024, 'number_of_characters': 342816, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mal_Mlym-gom_Deva': {'num_samples': 1024, 'number_of_characters': 335312, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mal_Mlym-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 332158, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mal_Mlym-hin_Deva': {'num_samples': 1024, 'number_of_characters': 342824, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mal_Mlym-kan_Knda': {'num_samples': 1024, 'number_of_characters': 354782, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mal_Mlym-kas_Arab': {'num_samples': 1024, 'number_of_characters': 345482, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mal_Mlym-mai_Deva': {'num_samples': 1024, 'number_of_characters': 331400, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'mal_Mlym-mar_Deva': {'num_samples': 1024, 'number_of_characters': 344502, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'mal_Mlym-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 335852, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'mal_Mlym-npi_Deva': {'num_samples': 1024, 'number_of_characters': 336137, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mal_Mlym-ory_Orya': {'num_samples': 1024, 'number_of_characters': 356944, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mal_Mlym-pan_Guru': {'num_samples': 1024, 'number_of_characters': 329581, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mal_Mlym-san_Deva': {'num_samples': 1024, 'number_of_characters': 340797, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mal_Mlym-sat_Olck': {'num_samples': 1024, 'number_of_characters': 349450, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mal_Mlym-snd_Deva': {'num_samples': 1024, 'number_of_characters': 343139, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mal_Mlym-tam_Taml': {'num_samples': 1024, 'number_of_characters': 371064, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mal_Mlym-tel_Telu': {'num_samples': 1024, 'number_of_characters': 341763, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mal_Mlym-urd_Arab': {'num_samples': 1024, 'number_of_characters': 337852, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mar_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 321784, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mar_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 311488, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mar_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 324475, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mar_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 319886, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mar_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 320964, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mar_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 313460, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mar_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 310306, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mar_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 320972, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mar_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 332930, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mar_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 323630, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mar_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 309548, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'mar_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 344502, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'mar_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 314000, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'mar_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 314285, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mar_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 335092, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mar_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 307729, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mar_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 318945, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mar_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 327598, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mar_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 321287, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mar_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 349212, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mar_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 319911, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mar_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 316000, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mni_Mtei-asm_Beng': {'num_samples': 1024, 'number_of_characters': 313134, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mni_Mtei-ben_Beng': {'num_samples': 1024, 'number_of_characters': 302838, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mni_Mtei-brx_Deva': {'num_samples': 1024, 'number_of_characters': 315825, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mni_Mtei-doi_Deva': {'num_samples': 1024, 'number_of_characters': 311236, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mni_Mtei-eng_Latn': {'num_samples': 1024, 'number_of_characters': 312314, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mni_Mtei-gom_Deva': {'num_samples': 1024, 'number_of_characters': 304810, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mni_Mtei-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 301656, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mni_Mtei-hin_Deva': {'num_samples': 1024, 'number_of_characters': 312322, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mni_Mtei-kan_Knda': {'num_samples': 1024, 'number_of_characters': 324280, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mni_Mtei-kas_Arab': {'num_samples': 1024, 'number_of_characters': 314980, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mni_Mtei-mai_Deva': {'num_samples': 1024, 'number_of_characters': 300898, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'mni_Mtei-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 335852, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'mni_Mtei-mar_Deva': {'num_samples': 1024, 'number_of_characters': 314000, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'mni_Mtei-npi_Deva': {'num_samples': 1024, 'number_of_characters': 305635, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mni_Mtei-ory_Orya': {'num_samples': 1024, 'number_of_characters': 326442, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mni_Mtei-pan_Guru': {'num_samples': 1024, 'number_of_characters': 299079, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mni_Mtei-san_Deva': {'num_samples': 1024, 'number_of_characters': 310295, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mni_Mtei-sat_Olck': {'num_samples': 1024, 'number_of_characters': 318948, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mni_Mtei-snd_Deva': {'num_samples': 1024, 'number_of_characters': 312637, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mni_Mtei-tam_Taml': {'num_samples': 1024, 'number_of_characters': 340562, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mni_Mtei-tel_Telu': {'num_samples': 1024, 'number_of_characters': 311261, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mni_Mtei-urd_Arab': {'num_samples': 1024, 'number_of_characters': 307350, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'npi_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 313419, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'npi_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 303123, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'npi_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 316110, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'npi_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 311521, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'npi_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 312599, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'npi_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 305095, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'npi_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 301941, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'npi_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 312607, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'npi_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 324565, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'npi_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 315265, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'npi_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 301183, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'npi_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 336137, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'npi_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 314285, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'npi_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 305635, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'npi_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 326727, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'npi_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 299364, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'npi_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 310580, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'npi_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 319233, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'npi_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 312922, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'npi_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 340847, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'npi_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 311546, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'npi_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 307635, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'ory_Orya-asm_Beng': {'num_samples': 1024, 'number_of_characters': 334226, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'ory_Orya-ben_Beng': {'num_samples': 1024, 'number_of_characters': 323930, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'ory_Orya-brx_Deva': {'num_samples': 1024, 'number_of_characters': 336917, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'ory_Orya-doi_Deva': {'num_samples': 1024, 'number_of_characters': 332328, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'ory_Orya-eng_Latn': {'num_samples': 1024, 'number_of_characters': 333406, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'ory_Orya-gom_Deva': {'num_samples': 1024, 'number_of_characters': 325902, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'ory_Orya-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 322748, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'ory_Orya-hin_Deva': {'num_samples': 1024, 'number_of_characters': 333414, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'ory_Orya-kan_Knda': {'num_samples': 1024, 'number_of_characters': 345372, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'ory_Orya-kas_Arab': {'num_samples': 1024, 'number_of_characters': 336072, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'ory_Orya-mai_Deva': {'num_samples': 1024, 'number_of_characters': 321990, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'ory_Orya-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 356944, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'ory_Orya-mar_Deva': {'num_samples': 1024, 'number_of_characters': 335092, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'ory_Orya-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 326442, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'ory_Orya-npi_Deva': {'num_samples': 1024, 'number_of_characters': 326727, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'ory_Orya-pan_Guru': {'num_samples': 1024, 'number_of_characters': 320171, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'ory_Orya-san_Deva': {'num_samples': 1024, 'number_of_characters': 331387, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'ory_Orya-sat_Olck': {'num_samples': 1024, 'number_of_characters': 340040, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'ory_Orya-snd_Deva': {'num_samples': 1024, 'number_of_characters': 333729, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'ory_Orya-tam_Taml': {'num_samples': 1024, 'number_of_characters': 361654, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'ory_Orya-tel_Telu': {'num_samples': 1024, 'number_of_characters': 332353, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'ory_Orya-urd_Arab': {'num_samples': 1024, 'number_of_characters': 328442, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'pan_Guru-asm_Beng': {'num_samples': 1024, 'number_of_characters': 306863, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'pan_Guru-ben_Beng': {'num_samples': 1024, 'number_of_characters': 296567, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'pan_Guru-brx_Deva': {'num_samples': 1024, 'number_of_characters': 309554, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'pan_Guru-doi_Deva': {'num_samples': 1024, 'number_of_characters': 304965, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'pan_Guru-eng_Latn': {'num_samples': 1024, 'number_of_characters': 306043, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'pan_Guru-gom_Deva': {'num_samples': 1024, 'number_of_characters': 298539, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'pan_Guru-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 295385, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'pan_Guru-hin_Deva': {'num_samples': 1024, 'number_of_characters': 306051, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'pan_Guru-kan_Knda': {'num_samples': 1024, 'number_of_characters': 318009, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'pan_Guru-kas_Arab': {'num_samples': 1024, 'number_of_characters': 308709, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'pan_Guru-mai_Deva': {'num_samples': 1024, 'number_of_characters': 294627, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'pan_Guru-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 329581, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'pan_Guru-mar_Deva': {'num_samples': 1024, 'number_of_characters': 307729, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'pan_Guru-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 299079, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'pan_Guru-npi_Deva': {'num_samples': 1024, 'number_of_characters': 299364, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'pan_Guru-ory_Orya': {'num_samples': 1024, 'number_of_characters': 320171, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'pan_Guru-san_Deva': {'num_samples': 1024, 'number_of_characters': 304024, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'pan_Guru-sat_Olck': {'num_samples': 1024, 'number_of_characters': 312677, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'pan_Guru-snd_Deva': {'num_samples': 1024, 'number_of_characters': 306366, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'pan_Guru-tam_Taml': {'num_samples': 1024, 'number_of_characters': 334291, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'pan_Guru-tel_Telu': {'num_samples': 1024, 'number_of_characters': 304990, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'pan_Guru-urd_Arab': {'num_samples': 1024, 'number_of_characters': 301079, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'san_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 318079, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'san_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 307783, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'san_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 320770, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'san_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 316181, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'san_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 317259, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'san_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 309755, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'san_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 306601, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'san_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 317267, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'san_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 329225, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'san_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 319925, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'san_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 305843, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'san_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 340797, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'san_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 318945, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'san_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 310295, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'san_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 310580, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'san_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 331387, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'san_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 304024, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'san_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 323893, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'san_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 317582, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'san_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 345507, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'san_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 316206, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'san_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 312295, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'sat_Olck-asm_Beng': {'num_samples': 1024, 'number_of_characters': 326732, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'sat_Olck-ben_Beng': {'num_samples': 1024, 'number_of_characters': 316436, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'sat_Olck-brx_Deva': {'num_samples': 1024, 'number_of_characters': 329423, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'sat_Olck-doi_Deva': {'num_samples': 1024, 'number_of_characters': 324834, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'sat_Olck-eng_Latn': {'num_samples': 1024, 'number_of_characters': 325912, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'sat_Olck-gom_Deva': {'num_samples': 1024, 'number_of_characters': 318408, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'sat_Olck-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 315254, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'sat_Olck-hin_Deva': {'num_samples': 1024, 'number_of_characters': 325920, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'sat_Olck-kan_Knda': {'num_samples': 1024, 'number_of_characters': 337878, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'sat_Olck-kas_Arab': {'num_samples': 1024, 'number_of_characters': 328578, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'sat_Olck-mai_Deva': {'num_samples': 1024, 'number_of_characters': 314496, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'sat_Olck-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 349450, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'sat_Olck-mar_Deva': {'num_samples': 1024, 'number_of_characters': 327598, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'sat_Olck-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 318948, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'sat_Olck-npi_Deva': {'num_samples': 1024, 'number_of_characters': 319233, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'sat_Olck-ory_Orya': {'num_samples': 1024, 'number_of_characters': 340040, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'sat_Olck-pan_Guru': {'num_samples': 1024, 'number_of_characters': 312677, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'sat_Olck-san_Deva': {'num_samples': 1024, 'number_of_characters': 323893, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'sat_Olck-snd_Deva': {'num_samples': 1024, 'number_of_characters': 326235, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'sat_Olck-tam_Taml': {'num_samples': 1024, 'number_of_characters': 354160, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'sat_Olck-tel_Telu': {'num_samples': 1024, 'number_of_characters': 324859, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'sat_Olck-urd_Arab': {'num_samples': 1024, 'number_of_characters': 320948, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'snd_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 320421, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'snd_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 310125, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'snd_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 323112, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'snd_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 318523, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'snd_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 319601, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'snd_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 312097, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'snd_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 308943, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'snd_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 319609, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'snd_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 331567, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'snd_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 322267, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'snd_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 308185, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'snd_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 343139, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'snd_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 321287, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'snd_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 312637, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'snd_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 312922, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'snd_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 333729, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'snd_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306366, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'snd_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 317582, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'snd_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 326235, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'snd_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 347849, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'snd_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 318548, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'snd_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 314637, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'tam_Taml-asm_Beng': {'num_samples': 1024, 'number_of_characters': 348346, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'tam_Taml-ben_Beng': {'num_samples': 1024, 'number_of_characters': 338050, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'tam_Taml-brx_Deva': {'num_samples': 1024, 'number_of_characters': 351037, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'tam_Taml-doi_Deva': {'num_samples': 1024, 'number_of_characters': 346448, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'tam_Taml-eng_Latn': {'num_samples': 1024, 'number_of_characters': 347526, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'tam_Taml-gom_Deva': {'num_samples': 1024, 'number_of_characters': 340022, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'tam_Taml-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 336868, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'tam_Taml-hin_Deva': {'num_samples': 1024, 'number_of_characters': 347534, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'tam_Taml-kan_Knda': {'num_samples': 1024, 'number_of_characters': 359492, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'tam_Taml-kas_Arab': {'num_samples': 1024, 'number_of_characters': 350192, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'tam_Taml-mai_Deva': {'num_samples': 1024, 'number_of_characters': 336110, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'tam_Taml-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 371064, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'tam_Taml-mar_Deva': {'num_samples': 1024, 'number_of_characters': 349212, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'tam_Taml-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 340562, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'tam_Taml-npi_Deva': {'num_samples': 1024, 'number_of_characters': 340847, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'tam_Taml-ory_Orya': {'num_samples': 1024, 'number_of_characters': 361654, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'tam_Taml-pan_Guru': {'num_samples': 1024, 'number_of_characters': 334291, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'tam_Taml-san_Deva': {'num_samples': 1024, 'number_of_characters': 345507, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'tam_Taml-sat_Olck': {'num_samples': 1024, 'number_of_characters': 354160, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'tam_Taml-snd_Deva': {'num_samples': 1024, 'number_of_characters': 347849, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'tam_Taml-tel_Telu': {'num_samples': 1024, 'number_of_characters': 346473, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'tam_Taml-urd_Arab': {'num_samples': 1024, 'number_of_characters': 342562, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'tel_Telu-asm_Beng': {'num_samples': 1024, 'number_of_characters': 319045, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'tel_Telu-ben_Beng': {'num_samples': 1024, 'number_of_characters': 308749, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'tel_Telu-brx_Deva': {'num_samples': 1024, 'number_of_characters': 321736, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'tel_Telu-doi_Deva': {'num_samples': 1024, 'number_of_characters': 317147, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'tel_Telu-eng_Latn': {'num_samples': 1024, 'number_of_characters': 318225, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'tel_Telu-gom_Deva': {'num_samples': 1024, 'number_of_characters': 310721, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'tel_Telu-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 307567, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'tel_Telu-hin_Deva': {'num_samples': 1024, 'number_of_characters': 318233, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'tel_Telu-kan_Knda': {'num_samples': 1024, 'number_of_characters': 330191, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'tel_Telu-kas_Arab': {'num_samples': 1024, 'number_of_characters': 320891, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'tel_Telu-mai_Deva': {'num_samples': 1024, 'number_of_characters': 306809, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'tel_Telu-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 341763, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'tel_Telu-mar_Deva': {'num_samples': 1024, 'number_of_characters': 319911, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'tel_Telu-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 311261, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'tel_Telu-npi_Deva': {'num_samples': 1024, 'number_of_characters': 311546, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'tel_Telu-ory_Orya': {'num_samples': 1024, 'number_of_characters': 332353, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'tel_Telu-pan_Guru': {'num_samples': 1024, 'number_of_characters': 304990, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'tel_Telu-san_Deva': {'num_samples': 1024, 'number_of_characters': 316206, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'tel_Telu-sat_Olck': {'num_samples': 1024, 'number_of_characters': 324859, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'tel_Telu-snd_Deva': {'num_samples': 1024, 'number_of_characters': 318548, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'tel_Telu-tam_Taml': {'num_samples': 1024, 'number_of_characters': 346473, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'tel_Telu-urd_Arab': {'num_samples': 1024, 'number_of_characters': 313261, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'urd_Arab-asm_Beng': {'num_samples': 1024, 'number_of_characters': 315134, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'urd_Arab-ben_Beng': {'num_samples': 1024, 'number_of_characters': 304838, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'urd_Arab-brx_Deva': {'num_samples': 1024, 'number_of_characters': 317825, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'urd_Arab-doi_Deva': {'num_samples': 1024, 'number_of_characters': 313236, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'urd_Arab-eng_Latn': {'num_samples': 1024, 'number_of_characters': 314314, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'urd_Arab-gom_Deva': {'num_samples': 1024, 'number_of_characters': 306810, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'urd_Arab-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 303656, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'urd_Arab-hin_Deva': {'num_samples': 1024, 'number_of_characters': 314322, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'urd_Arab-kan_Knda': {'num_samples': 1024, 'number_of_characters': 326280, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'urd_Arab-kas_Arab': {'num_samples': 1024, 'number_of_characters': 316980, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'urd_Arab-mai_Deva': {'num_samples': 1024, 'number_of_characters': 302898, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'urd_Arab-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 337852, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'urd_Arab-mar_Deva': {'num_samples': 1024, 'number_of_characters': 316000, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'urd_Arab-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 307350, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'urd_Arab-npi_Deva': {'num_samples': 1024, 'number_of_characters': 307635, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'urd_Arab-ory_Orya': {'num_samples': 1024, 'number_of_characters': 328442, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'urd_Arab-pan_Guru': {'num_samples': 1024, 'number_of_characters': 301079, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'urd_Arab-san_Deva': {'num_samples': 1024, 'number_of_characters': 312295, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'urd_Arab-sat_Olck': {'num_samples': 1024, 'number_of_characters': 320948, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'urd_Arab-snd_Deva': {'num_samples': 1024, 'number_of_characters': 314637, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'urd_Arab-tam_Taml': {'num_samples': 1024, 'number_of_characters': 342562, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'urd_Arab-tel_Telu': {'num_samples': 1024, 'number_of_characters': 313261, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}}}} | -| [IWSLT2017BitextMining](https://aclanthology.org/2017.iwslt-1.1/) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'ita', 'jpn', 'kor', 'nld', 'ron'] | BitextMining | s2s | [Non-fiction, Fiction, Written] | {'validation': 21938} | {'validation': {'num_samples': 21938, 'number_of_characters': 4256244, 'unique_pairs': 21840, 'min_sentence1_length': 2, 'average_sentence1_length': 97.01, 'max_sentence1_length': 521, 'unique_sentence1': 11563, 'min_sentence2_length': 2, 'average_sentence2_length': 97.01, 'max_sentence2_length': 521, 'unique_sentence2': 11563, 'hf_subset_descriptive_stats': {'ar-en': {'num_samples': 888, 'number_of_characters': 172499, 'unique_pairs': 887, 'min_sentence1_length': 4, 'average_sentence1_length': 85.49, 'max_sentence1_length': 369, 'unique_sentence1': 887, 'min_sentence2_length': 10, 'average_sentence2_length': 108.77, 'max_sentence2_length': 462, 'unique_sentence2': 881}, 'de-en': {'num_samples': 888, 'number_of_characters': 202336, 'unique_pairs': 883, 'min_sentence1_length': 6, 'average_sentence1_length': 119.03, 'max_sentence1_length': 521, 'unique_sentence1': 881, 'min_sentence2_length': 10, 'average_sentence2_length': 108.83, 'max_sentence2_length': 462, 'unique_sentence2': 881}, 'en-ar': {'num_samples': 888, 'number_of_characters': 172499, 'unique_pairs': 887, 'min_sentence1_length': 10, 'average_sentence1_length': 108.77, 'max_sentence1_length': 462, 'unique_sentence1': 881, 'min_sentence2_length': 4, 'average_sentence2_length': 85.49, 'max_sentence2_length': 369, 'unique_sentence2': 887}, 'en-de': {'num_samples': 888, 'number_of_characters': 202336, 'unique_pairs': 883, 'min_sentence1_length': 10, 'average_sentence1_length': 108.83, 'max_sentence1_length': 462, 'unique_sentence1': 881, 'min_sentence2_length': 6, 'average_sentence2_length': 119.03, 'max_sentence2_length': 521, 'unique_sentence2': 881}, 'en-fr': {'num_samples': 890, 'number_of_characters': 197619, 'unique_pairs': 883, 'min_sentence1_length': 10, 'average_sentence1_length': 108.41, 'max_sentence1_length': 462, 'unique_sentence1': 883, 'min_sentence2_length': 6, 'average_sentence2_length': 113.63, 'max_sentence2_length': 493, 'unique_sentence2': 881}, 'en-it': {'num_samples': 929, 'number_of_characters': 191803, 'unique_pairs': 924, 'min_sentence1_length': 10, 'average_sentence1_length': 103.0, 'max_sentence1_length': 433, 'unique_sentence1': 922, 'min_sentence2_length': 7, 'average_sentence2_length': 103.46, 'max_sentence2_length': 444, 'unique_sentence2': 918}, 'en-ja': {'num_samples': 871, 'number_of_characters': 132742, 'unique_pairs': 867, 'min_sentence1_length': 10, 'average_sentence1_length': 109.81, 'max_sentence1_length': 462, 'unique_sentence1': 864, 'min_sentence2_length': 5, 'average_sentence2_length': 42.59, 'max_sentence2_length': 225, 'unique_sentence2': 866}, 'en-ko': {'num_samples': 879, 'number_of_characters': 142659, 'unique_pairs': 874, 'min_sentence1_length': 10, 'average_sentence1_length': 107.74, 'max_sentence1_length': 462, 'unique_sentence1': 872, 'min_sentence2_length': 3, 'average_sentence2_length': 54.56, 'max_sentence2_length': 250, 'unique_sentence2': 872}, 'en-nl': {'num_samples': 1003, 'number_of_characters': 189637, 'unique_pairs': 1000, 'min_sentence1_length': 10, 'average_sentence1_length': 95.27, 'max_sentence1_length': 433, 'unique_sentence1': 996, 'min_sentence2_length': 4, 'average_sentence2_length': 93.8, 'max_sentence2_length': 477, 'unique_sentence2': 1000}, 'en-ro': {'num_samples': 914, 'number_of_characters': 194128, 'unique_pairs': 910, 'min_sentence1_length': 10, 'average_sentence1_length': 104.72, 'max_sentence1_length': 433, 'unique_sentence1': 907, 'min_sentence2_length': 9, 'average_sentence2_length': 107.67, 'max_sentence2_length': 448, 'unique_sentence2': 910}, 'en-zh': {'num_samples': 879, 'number_of_characters': 131126, 'unique_pairs': 877, 'min_sentence1_length': 10, 'average_sentence1_length': 109.37, 'max_sentence1_length': 462, 'unique_sentence1': 872, 'min_sentence2_length': 2, 'average_sentence2_length': 39.81, 'max_sentence2_length': 230, 'unique_sentence2': 867}, 'fr-en': {'num_samples': 890, 'number_of_characters': 197619, 'unique_pairs': 883, 'min_sentence1_length': 6, 'average_sentence1_length': 113.63, 'max_sentence1_length': 493, 'unique_sentence1': 881, 'min_sentence2_length': 10, 'average_sentence2_length': 108.41, 'max_sentence2_length': 462, 'unique_sentence2': 883}, 'it-en': {'num_samples': 929, 'number_of_characters': 191803, 'unique_pairs': 924, 'min_sentence1_length': 7, 'average_sentence1_length': 103.46, 'max_sentence1_length': 444, 'unique_sentence1': 918, 'min_sentence2_length': 10, 'average_sentence2_length': 103.0, 'max_sentence2_length': 433, 'unique_sentence2': 922}, 'it-nl': {'num_samples': 1001, 'number_of_characters': 188858, 'unique_pairs': 998, 'min_sentence1_length': 7, 'average_sentence1_length': 94.64, 'max_sentence1_length': 459, 'unique_sentence1': 994, 'min_sentence2_length': 7, 'average_sentence2_length': 94.03, 'max_sentence2_length': 505, 'unique_sentence2': 998}, 'it-ro': {'num_samples': 914, 'number_of_characters': 193339, 'unique_pairs': 911, 'min_sentence1_length': 7, 'average_sentence1_length': 103.91, 'max_sentence1_length': 435, 'unique_sentence1': 907, 'min_sentence2_length': 9, 'average_sentence2_length': 107.62, 'max_sentence2_length': 448, 'unique_sentence2': 910}, 'ja-en': {'num_samples': 871, 'number_of_characters': 132742, 'unique_pairs': 867, 'min_sentence1_length': 5, 'average_sentence1_length': 42.59, 'max_sentence1_length': 225, 'unique_sentence1': 866, 'min_sentence2_length': 10, 'average_sentence2_length': 109.81, 'max_sentence2_length': 462, 'unique_sentence2': 864}, 'ko-en': {'num_samples': 879, 'number_of_characters': 142659, 'unique_pairs': 874, 'min_sentence1_length': 3, 'average_sentence1_length': 54.56, 'max_sentence1_length': 250, 'unique_sentence1': 872, 'min_sentence2_length': 10, 'average_sentence2_length': 107.74, 'max_sentence2_length': 462, 'unique_sentence2': 872}, 'nl-en': {'num_samples': 1003, 'number_of_characters': 189637, 'unique_pairs': 1000, 'min_sentence1_length': 4, 'average_sentence1_length': 93.8, 'max_sentence1_length': 477, 'unique_sentence1': 1000, 'min_sentence2_length': 10, 'average_sentence2_length': 95.27, 'max_sentence2_length': 433, 'unique_sentence2': 996}, 'nl-it': {'num_samples': 1001, 'number_of_characters': 188858, 'unique_pairs': 998, 'min_sentence1_length': 7, 'average_sentence1_length': 94.03, 'max_sentence1_length': 505, 'unique_sentence1': 998, 'min_sentence2_length': 7, 'average_sentence2_length': 94.64, 'max_sentence2_length': 459, 'unique_sentence2': 994}, 'nl-ro': {'num_samples': 913, 'number_of_characters': 191376, 'unique_pairs': 911, 'min_sentence1_length': 7, 'average_sentence1_length': 102.02, 'max_sentence1_length': 478, 'unique_sentence1': 909, 'min_sentence2_length': 9, 'average_sentence2_length': 107.59, 'max_sentence2_length': 515, 'unique_sentence2': 909}, 'ro-en': {'num_samples': 914, 'number_of_characters': 194128, 'unique_pairs': 910, 'min_sentence1_length': 9, 'average_sentence1_length': 107.67, 'max_sentence1_length': 448, 'unique_sentence1': 910, 'min_sentence2_length': 10, 'average_sentence2_length': 104.72, 'max_sentence2_length': 433, 'unique_sentence2': 907}, 'ro-it': {'num_samples': 914, 'number_of_characters': 193339, 'unique_pairs': 911, 'min_sentence1_length': 9, 'average_sentence1_length': 107.62, 'max_sentence1_length': 448, 'unique_sentence1': 910, 'min_sentence2_length': 7, 'average_sentence2_length': 103.91, 'max_sentence2_length': 435, 'unique_sentence2': 907}, 'ro-nl': {'num_samples': 913, 'number_of_characters': 191376, 'unique_pairs': 911, 'min_sentence1_length': 9, 'average_sentence1_length': 107.59, 'max_sentence1_length': 515, 'unique_sentence1': 909, 'min_sentence2_length': 7, 'average_sentence2_length': 102.02, 'max_sentence2_length': 478, 'unique_sentence2': 909}, 'zh-en': {'num_samples': 879, 'number_of_characters': 131126, 'unique_pairs': 877, 'min_sentence1_length': 2, 'average_sentence1_length': 39.81, 'max_sentence1_length': 230, 'unique_sentence1': 867, 'min_sentence2_length': 10, 'average_sentence2_length': 109.37, 'max_sentence2_length': 462, 'unique_sentence2': 872}}}} | +| [IN22ConvBitextMining](https://huggingface.co/datasets/ai4bharat/IN22-Conv) (Jay Gala, 2023) | ['asm', 'ben', 'brx', 'doi', 'eng', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Fiction, Social, Spoken, Spoken] | {'test': 760518} | {'test': {'num_samples': 760518, 'number_of_characters': 82637104, 'unique_pairs': 759283, 'min_sentence1_length': 3, 'average_sentence1_length': 54.33, 'max_sentence1_length': 239, 'unique_sentence1': 34430, 'min_sentence2_length': 3, 'average_sentence2_length': 54.33, 'max_sentence2_length': 239, 'unique_sentence2': 34430, 'hf_subset_descriptive_stats': {'asm_Beng-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155988, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'asm_Beng-brx_Deva': {'num_samples': 1503, 'number_of_characters': 162044, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'asm_Beng-doi_Deva': {'num_samples': 1503, 'number_of_characters': 167032, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'asm_Beng-eng_Latn': {'num_samples': 1503, 'number_of_characters': 160716, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'asm_Beng-gom_Deva': {'num_samples': 1503, 'number_of_characters': 156282, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'asm_Beng-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 158269, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'asm_Beng-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159964, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'asm_Beng-kan_Knda': {'num_samples': 1503, 'number_of_characters': 165177, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'asm_Beng-kas_Arab': {'num_samples': 1503, 'number_of_characters': 164681, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'asm_Beng-mai_Deva': {'num_samples': 1503, 'number_of_characters': 162408, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'asm_Beng-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 172838, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'asm_Beng-mar_Deva': {'num_samples': 1503, 'number_of_characters': 162747, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'asm_Beng-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 157316, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'asm_Beng-npi_Deva': {'num_samples': 1503, 'number_of_characters': 160906, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'asm_Beng-ory_Orya': {'num_samples': 1503, 'number_of_characters': 164223, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'asm_Beng-pan_Guru': {'num_samples': 1503, 'number_of_characters': 160201, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'asm_Beng-san_Deva': {'num_samples': 1503, 'number_of_characters': 158093, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'asm_Beng-sat_Olck': {'num_samples': 1503, 'number_of_characters': 169379, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'asm_Beng-snd_Deva': {'num_samples': 1503, 'number_of_characters': 162623, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'asm_Beng-tam_Taml': {'num_samples': 1503, 'number_of_characters': 174866, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'asm_Beng-tel_Telu': {'num_samples': 1503, 'number_of_characters': 157690, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'asm_Beng-urd_Arab': {'num_samples': 1503, 'number_of_characters': 161305, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 53.75, 'max_sentence1_length': 208, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'ben_Beng-asm_Beng': {'num_samples': 1503, 'number_of_characters': 155988, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'ben_Beng-brx_Deva': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'ben_Beng-doi_Deva': {'num_samples': 1503, 'number_of_characters': 161436, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'ben_Beng-eng_Latn': {'num_samples': 1503, 'number_of_characters': 155120, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'ben_Beng-gom_Deva': {'num_samples': 1503, 'number_of_characters': 150686, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'ben_Beng-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 152673, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'ben_Beng-hin_Deva': {'num_samples': 1503, 'number_of_characters': 154368, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'ben_Beng-kan_Knda': {'num_samples': 1503, 'number_of_characters': 159581, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'ben_Beng-kas_Arab': {'num_samples': 1503, 'number_of_characters': 159085, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'ben_Beng-mai_Deva': {'num_samples': 1503, 'number_of_characters': 156812, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'ben_Beng-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 167242, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'ben_Beng-mar_Deva': {'num_samples': 1503, 'number_of_characters': 157151, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'ben_Beng-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 151720, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'ben_Beng-npi_Deva': {'num_samples': 1503, 'number_of_characters': 155310, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'ben_Beng-ory_Orya': {'num_samples': 1503, 'number_of_characters': 158627, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'ben_Beng-pan_Guru': {'num_samples': 1503, 'number_of_characters': 154605, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'ben_Beng-san_Deva': {'num_samples': 1503, 'number_of_characters': 152497, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'ben_Beng-sat_Olck': {'num_samples': 1503, 'number_of_characters': 163783, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'ben_Beng-snd_Deva': {'num_samples': 1503, 'number_of_characters': 157027, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'ben_Beng-tam_Taml': {'num_samples': 1503, 'number_of_characters': 169270, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'ben_Beng-tel_Telu': {'num_samples': 1503, 'number_of_characters': 152094, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'ben_Beng-urd_Arab': {'num_samples': 1503, 'number_of_characters': 155709, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.03, 'max_sentence1_length': 178, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'brx_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162044, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'brx_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'brx_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 167492, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'brx_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161176, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'brx_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 156742, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'brx_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'brx_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 160424, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'brx_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 165637, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'brx_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165141, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'brx_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 162868, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'brx_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 173298, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'brx_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 163207, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'brx_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 157776, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'brx_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'brx_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 164683, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'brx_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 160661, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'brx_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 158553, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'brx_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 169839, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'brx_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 163083, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'brx_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 175326, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'brx_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158150, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'brx_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 161765, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.06, 'max_sentence1_length': 210, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'doi_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 167032, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'doi_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 161436, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'doi_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 167492, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'doi_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 166164, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'doi_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'doi_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 163717, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'doi_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 165412, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'doi_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 170625, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'doi_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 170129, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'doi_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 167856, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'doi_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 178286, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'doi_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 168195, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'doi_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 162764, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'doi_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 166354, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'doi_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 169671, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'doi_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 165649, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'doi_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 163541, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'doi_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 174827, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'doi_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 168071, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'doi_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 180314, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'doi_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 163138, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'doi_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 166753, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 57.38, 'max_sentence1_length': 209, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'eng_Latn-asm_Beng': {'num_samples': 1503, 'number_of_characters': 160716, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'eng_Latn-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155120, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'eng_Latn-brx_Deva': {'num_samples': 1503, 'number_of_characters': 161176, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'eng_Latn-doi_Deva': {'num_samples': 1503, 'number_of_characters': 166164, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'eng_Latn-gom_Deva': {'num_samples': 1503, 'number_of_characters': 155414, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'eng_Latn-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 157401, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'eng_Latn-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159096, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'eng_Latn-kan_Knda': {'num_samples': 1503, 'number_of_characters': 164309, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'eng_Latn-kas_Arab': {'num_samples': 1503, 'number_of_characters': 163813, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'eng_Latn-mai_Deva': {'num_samples': 1503, 'number_of_characters': 161540, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'eng_Latn-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 171970, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'eng_Latn-mar_Deva': {'num_samples': 1503, 'number_of_characters': 161879, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'eng_Latn-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'eng_Latn-npi_Deva': {'num_samples': 1503, 'number_of_characters': 160038, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'eng_Latn-ory_Orya': {'num_samples': 1503, 'number_of_characters': 163355, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'eng_Latn-pan_Guru': {'num_samples': 1503, 'number_of_characters': 159333, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'eng_Latn-san_Deva': {'num_samples': 1503, 'number_of_characters': 157225, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'eng_Latn-sat_Olck': {'num_samples': 1503, 'number_of_characters': 168511, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'eng_Latn-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161755, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'eng_Latn-tam_Taml': {'num_samples': 1503, 'number_of_characters': 173998, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'eng_Latn-tel_Telu': {'num_samples': 1503, 'number_of_characters': 156822, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'eng_Latn-urd_Arab': {'num_samples': 1503, 'number_of_characters': 160437, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.18, 'max_sentence1_length': 201, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'gom_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 156282, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'gom_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 150686, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'gom_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 156742, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'gom_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'gom_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 155414, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'gom_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 152967, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'gom_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 154662, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'gom_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 159875, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'gom_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 159379, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'gom_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 157106, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'gom_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 167536, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'gom_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 157445, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'gom_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 152014, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'gom_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 155604, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'gom_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 158921, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'gom_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 154899, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'gom_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 152791, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'gom_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 164077, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'gom_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 157321, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'gom_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 169564, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'gom_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 152388, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'gom_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 156003, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 50.23, 'max_sentence1_length': 203, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'guj_Gujr-asm_Beng': {'num_samples': 1503, 'number_of_characters': 158269, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'guj_Gujr-ben_Beng': {'num_samples': 1503, 'number_of_characters': 152673, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'guj_Gujr-brx_Deva': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'guj_Gujr-doi_Deva': {'num_samples': 1503, 'number_of_characters': 163717, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'guj_Gujr-eng_Latn': {'num_samples': 1503, 'number_of_characters': 157401, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'guj_Gujr-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152967, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'guj_Gujr-hin_Deva': {'num_samples': 1503, 'number_of_characters': 156649, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'guj_Gujr-kan_Knda': {'num_samples': 1503, 'number_of_characters': 161862, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'guj_Gujr-kas_Arab': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'guj_Gujr-mai_Deva': {'num_samples': 1503, 'number_of_characters': 159093, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'guj_Gujr-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 169523, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'guj_Gujr-mar_Deva': {'num_samples': 1503, 'number_of_characters': 159432, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'guj_Gujr-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 154001, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'guj_Gujr-npi_Deva': {'num_samples': 1503, 'number_of_characters': 157591, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'guj_Gujr-ory_Orya': {'num_samples': 1503, 'number_of_characters': 160908, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'guj_Gujr-pan_Guru': {'num_samples': 1503, 'number_of_characters': 156886, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'guj_Gujr-san_Deva': {'num_samples': 1503, 'number_of_characters': 154778, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'guj_Gujr-sat_Olck': {'num_samples': 1503, 'number_of_characters': 166064, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'guj_Gujr-snd_Deva': {'num_samples': 1503, 'number_of_characters': 159308, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'guj_Gujr-tam_Taml': {'num_samples': 1503, 'number_of_characters': 171551, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'guj_Gujr-tel_Telu': {'num_samples': 1503, 'number_of_characters': 154375, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'guj_Gujr-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157990, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 51.55, 'max_sentence1_length': 205, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'hin_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 159964, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'hin_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 154368, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'hin_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 160424, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'hin_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 165412, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'hin_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 159096, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'hin_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 154662, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'hin_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 156649, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'hin_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 163557, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'hin_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 163061, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'hin_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 160788, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'hin_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 171218, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'hin_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 161127, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'hin_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 155696, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'hin_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 159286, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'hin_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 162603, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'hin_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 158581, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'hin_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 156473, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'hin_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 167759, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'hin_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'hin_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 173246, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'hin_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 156070, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'hin_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 159685, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.68, 'max_sentence1_length': 192, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'kan_Knda-asm_Beng': {'num_samples': 1503, 'number_of_characters': 165177, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'kan_Knda-ben_Beng': {'num_samples': 1503, 'number_of_characters': 159581, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'kan_Knda-brx_Deva': {'num_samples': 1503, 'number_of_characters': 165637, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'kan_Knda-doi_Deva': {'num_samples': 1503, 'number_of_characters': 170625, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'kan_Knda-eng_Latn': {'num_samples': 1503, 'number_of_characters': 164309, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'kan_Knda-gom_Deva': {'num_samples': 1503, 'number_of_characters': 159875, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'kan_Knda-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 161862, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'kan_Knda-hin_Deva': {'num_samples': 1503, 'number_of_characters': 163557, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'kan_Knda-kas_Arab': {'num_samples': 1503, 'number_of_characters': 168274, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'kan_Knda-mai_Deva': {'num_samples': 1503, 'number_of_characters': 166001, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'kan_Knda-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 176431, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'kan_Knda-mar_Deva': {'num_samples': 1503, 'number_of_characters': 166340, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'kan_Knda-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 160909, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'kan_Knda-npi_Deva': {'num_samples': 1503, 'number_of_characters': 164499, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'kan_Knda-ory_Orya': {'num_samples': 1503, 'number_of_characters': 167816, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'kan_Knda-pan_Guru': {'num_samples': 1503, 'number_of_characters': 163794, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'kan_Knda-san_Deva': {'num_samples': 1503, 'number_of_characters': 161686, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'kan_Knda-sat_Olck': {'num_samples': 1503, 'number_of_characters': 172972, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'kan_Knda-snd_Deva': {'num_samples': 1503, 'number_of_characters': 166216, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'kan_Knda-tam_Taml': {'num_samples': 1503, 'number_of_characters': 178459, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'kan_Knda-tel_Telu': {'num_samples': 1503, 'number_of_characters': 161283, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'kan_Knda-urd_Arab': {'num_samples': 1503, 'number_of_characters': 164898, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 56.14, 'max_sentence1_length': 201, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'kas_Arab-asm_Beng': {'num_samples': 1503, 'number_of_characters': 164681, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'kas_Arab-ben_Beng': {'num_samples': 1503, 'number_of_characters': 159085, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'kas_Arab-brx_Deva': {'num_samples': 1503, 'number_of_characters': 165141, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'kas_Arab-doi_Deva': {'num_samples': 1503, 'number_of_characters': 170129, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'kas_Arab-eng_Latn': {'num_samples': 1503, 'number_of_characters': 163813, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'kas_Arab-gom_Deva': {'num_samples': 1503, 'number_of_characters': 159379, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'kas_Arab-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'kas_Arab-hin_Deva': {'num_samples': 1503, 'number_of_characters': 163061, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'kas_Arab-kan_Knda': {'num_samples': 1503, 'number_of_characters': 168274, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'kas_Arab-mai_Deva': {'num_samples': 1503, 'number_of_characters': 165505, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'kas_Arab-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 175935, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'kas_Arab-mar_Deva': {'num_samples': 1503, 'number_of_characters': 165844, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'kas_Arab-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 160413, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'kas_Arab-npi_Deva': {'num_samples': 1503, 'number_of_characters': 164003, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'kas_Arab-ory_Orya': {'num_samples': 1503, 'number_of_characters': 167320, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'kas_Arab-pan_Guru': {'num_samples': 1503, 'number_of_characters': 163298, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'kas_Arab-san_Deva': {'num_samples': 1503, 'number_of_characters': 161190, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'kas_Arab-sat_Olck': {'num_samples': 1503, 'number_of_characters': 172476, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'kas_Arab-snd_Deva': {'num_samples': 1503, 'number_of_characters': 165720, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'kas_Arab-tam_Taml': {'num_samples': 1503, 'number_of_characters': 177963, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'kas_Arab-tel_Telu': {'num_samples': 1503, 'number_of_characters': 160787, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'kas_Arab-urd_Arab': {'num_samples': 1503, 'number_of_characters': 164402, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 55.81, 'max_sentence1_length': 203, 'unique_sentence1': 1502, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mai_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162408, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mai_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 156812, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mai_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 162868, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mai_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 167856, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mai_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161540, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mai_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 157106, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mai_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 159093, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mai_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 160788, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mai_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 166001, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mai_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165505, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mai_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 173662, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'mai_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 163571, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'mai_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 158140, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'mai_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mai_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 165047, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mai_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 161025, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mai_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 158917, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mai_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 170203, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mai_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 163447, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mai_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 175690, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mai_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158514, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mai_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 162129, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 54.3, 'max_sentence1_length': 230, 'unique_sentence1': 1499, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mal_Mlym-asm_Beng': {'num_samples': 1503, 'number_of_characters': 172838, 'unique_pairs': 1498, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mal_Mlym-ben_Beng': {'num_samples': 1503, 'number_of_characters': 167242, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mal_Mlym-brx_Deva': {'num_samples': 1503, 'number_of_characters': 173298, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mal_Mlym-doi_Deva': {'num_samples': 1503, 'number_of_characters': 178286, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mal_Mlym-eng_Latn': {'num_samples': 1503, 'number_of_characters': 171970, 'unique_pairs': 1499, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mal_Mlym-gom_Deva': {'num_samples': 1503, 'number_of_characters': 167536, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mal_Mlym-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 169523, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mal_Mlym-hin_Deva': {'num_samples': 1503, 'number_of_characters': 171218, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mal_Mlym-kan_Knda': {'num_samples': 1503, 'number_of_characters': 176431, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mal_Mlym-kas_Arab': {'num_samples': 1503, 'number_of_characters': 175935, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mal_Mlym-mai_Deva': {'num_samples': 1503, 'number_of_characters': 173662, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'mal_Mlym-mar_Deva': {'num_samples': 1503, 'number_of_characters': 174001, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'mal_Mlym-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 168570, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'mal_Mlym-npi_Deva': {'num_samples': 1503, 'number_of_characters': 172160, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mal_Mlym-ory_Orya': {'num_samples': 1503, 'number_of_characters': 175477, 'unique_pairs': 1503, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mal_Mlym-pan_Guru': {'num_samples': 1503, 'number_of_characters': 171455, 'unique_pairs': 1498, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mal_Mlym-san_Deva': {'num_samples': 1503, 'number_of_characters': 169347, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mal_Mlym-sat_Olck': {'num_samples': 1503, 'number_of_characters': 180633, 'unique_pairs': 1501, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mal_Mlym-snd_Deva': {'num_samples': 1503, 'number_of_characters': 173877, 'unique_pairs': 1499, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mal_Mlym-tam_Taml': {'num_samples': 1503, 'number_of_characters': 186120, 'unique_pairs': 1502, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mal_Mlym-tel_Telu': {'num_samples': 1503, 'number_of_characters': 168944, 'unique_pairs': 1500, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mal_Mlym-urd_Arab': {'num_samples': 1503, 'number_of_characters': 172559, 'unique_pairs': 1499, 'min_sentence1_length': 5, 'average_sentence1_length': 61.24, 'max_sentence1_length': 219, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mar_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162747, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mar_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 157151, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mar_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 163207, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mar_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 168195, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mar_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161879, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mar_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 157445, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mar_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 159432, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mar_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 161127, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mar_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 166340, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mar_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165844, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mar_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 163571, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'mar_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 174001, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'mar_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 158479, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'mar_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 162069, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mar_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 165386, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mar_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 161364, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mar_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 159256, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mar_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 170542, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mar_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 163786, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mar_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 176029, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mar_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158853, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mar_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 162468, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.53, 'max_sentence1_length': 221, 'unique_sentence1': 1501, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'mni_Mtei-asm_Beng': {'num_samples': 1503, 'number_of_characters': 157316, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'mni_Mtei-ben_Beng': {'num_samples': 1503, 'number_of_characters': 151720, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'mni_Mtei-brx_Deva': {'num_samples': 1503, 'number_of_characters': 157776, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'mni_Mtei-doi_Deva': {'num_samples': 1503, 'number_of_characters': 162764, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'mni_Mtei-eng_Latn': {'num_samples': 1503, 'number_of_characters': 156448, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'mni_Mtei-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152014, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'mni_Mtei-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 154001, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'mni_Mtei-hin_Deva': {'num_samples': 1503, 'number_of_characters': 155696, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'mni_Mtei-kan_Knda': {'num_samples': 1503, 'number_of_characters': 160909, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'mni_Mtei-kas_Arab': {'num_samples': 1503, 'number_of_characters': 160413, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'mni_Mtei-mai_Deva': {'num_samples': 1503, 'number_of_characters': 158140, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'mni_Mtei-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 168570, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'mni_Mtei-mar_Deva': {'num_samples': 1503, 'number_of_characters': 158479, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'mni_Mtei-npi_Deva': {'num_samples': 1503, 'number_of_characters': 156638, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'mni_Mtei-ory_Orya': {'num_samples': 1503, 'number_of_characters': 159955, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'mni_Mtei-pan_Guru': {'num_samples': 1503, 'number_of_characters': 155933, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'mni_Mtei-san_Deva': {'num_samples': 1503, 'number_of_characters': 153825, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'mni_Mtei-sat_Olck': {'num_samples': 1503, 'number_of_characters': 165111, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'mni_Mtei-snd_Deva': {'num_samples': 1503, 'number_of_characters': 158355, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'mni_Mtei-tam_Taml': {'num_samples': 1503, 'number_of_characters': 170598, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'mni_Mtei-tel_Telu': {'num_samples': 1503, 'number_of_characters': 153422, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'mni_Mtei-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157037, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 50.91, 'max_sentence1_length': 239, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'npi_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 160906, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'npi_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155310, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'npi_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 161366, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'npi_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 166354, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'npi_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 160038, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'npi_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 155604, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'npi_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 157591, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'npi_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159286, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'npi_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 164499, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'npi_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 164003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'npi_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 161730, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'npi_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 172160, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'npi_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 162069, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'npi_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 156638, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'npi_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 163545, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'npi_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 159523, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'npi_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 157415, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'npi_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 168701, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'npi_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161945, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'npi_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 174188, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'npi_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 157012, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'npi_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 160627, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.3, 'max_sentence1_length': 223, 'unique_sentence1': 1497, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'ory_Orya-asm_Beng': {'num_samples': 1503, 'number_of_characters': 164223, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'ory_Orya-ben_Beng': {'num_samples': 1503, 'number_of_characters': 158627, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'ory_Orya-brx_Deva': {'num_samples': 1503, 'number_of_characters': 164683, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'ory_Orya-doi_Deva': {'num_samples': 1503, 'number_of_characters': 169671, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'ory_Orya-eng_Latn': {'num_samples': 1503, 'number_of_characters': 163355, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'ory_Orya-gom_Deva': {'num_samples': 1503, 'number_of_characters': 158921, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'ory_Orya-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 160908, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'ory_Orya-hin_Deva': {'num_samples': 1503, 'number_of_characters': 162603, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'ory_Orya-kan_Knda': {'num_samples': 1503, 'number_of_characters': 167816, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'ory_Orya-kas_Arab': {'num_samples': 1503, 'number_of_characters': 167320, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'ory_Orya-mai_Deva': {'num_samples': 1503, 'number_of_characters': 165047, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'ory_Orya-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 175477, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'ory_Orya-mar_Deva': {'num_samples': 1503, 'number_of_characters': 165386, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'ory_Orya-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 159955, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'ory_Orya-npi_Deva': {'num_samples': 1503, 'number_of_characters': 163545, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'ory_Orya-pan_Guru': {'num_samples': 1503, 'number_of_characters': 162840, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'ory_Orya-san_Deva': {'num_samples': 1503, 'number_of_characters': 160732, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'ory_Orya-sat_Olck': {'num_samples': 1503, 'number_of_characters': 172018, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'ory_Orya-snd_Deva': {'num_samples': 1503, 'number_of_characters': 165262, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'ory_Orya-tam_Taml': {'num_samples': 1503, 'number_of_characters': 177505, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'ory_Orya-tel_Telu': {'num_samples': 1503, 'number_of_characters': 160329, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'ory_Orya-urd_Arab': {'num_samples': 1503, 'number_of_characters': 163944, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 55.51, 'max_sentence1_length': 195, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'pan_Guru-asm_Beng': {'num_samples': 1503, 'number_of_characters': 160201, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'pan_Guru-ben_Beng': {'num_samples': 1503, 'number_of_characters': 154605, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'pan_Guru-brx_Deva': {'num_samples': 1503, 'number_of_characters': 160661, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'pan_Guru-doi_Deva': {'num_samples': 1503, 'number_of_characters': 165649, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'pan_Guru-eng_Latn': {'num_samples': 1503, 'number_of_characters': 159333, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'pan_Guru-gom_Deva': {'num_samples': 1503, 'number_of_characters': 154899, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'pan_Guru-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 156886, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'pan_Guru-hin_Deva': {'num_samples': 1503, 'number_of_characters': 158581, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'pan_Guru-kan_Knda': {'num_samples': 1503, 'number_of_characters': 163794, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'pan_Guru-kas_Arab': {'num_samples': 1503, 'number_of_characters': 163298, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'pan_Guru-mai_Deva': {'num_samples': 1503, 'number_of_characters': 161025, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'pan_Guru-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 171455, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'pan_Guru-mar_Deva': {'num_samples': 1503, 'number_of_characters': 161364, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'pan_Guru-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 155933, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'pan_Guru-npi_Deva': {'num_samples': 1503, 'number_of_characters': 159523, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'pan_Guru-ory_Orya': {'num_samples': 1503, 'number_of_characters': 162840, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'pan_Guru-san_Deva': {'num_samples': 1503, 'number_of_characters': 156710, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'pan_Guru-sat_Olck': {'num_samples': 1503, 'number_of_characters': 167996, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'pan_Guru-snd_Deva': {'num_samples': 1503, 'number_of_characters': 161240, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'pan_Guru-tam_Taml': {'num_samples': 1503, 'number_of_characters': 173483, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'pan_Guru-tel_Telu': {'num_samples': 1503, 'number_of_characters': 156307, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'pan_Guru-urd_Arab': {'num_samples': 1503, 'number_of_characters': 159922, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 52.83, 'max_sentence1_length': 221, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'san_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 158093, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'san_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 152497, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'san_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 158553, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'san_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 163541, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'san_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 157225, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'san_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152791, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'san_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 154778, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'san_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 156473, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'san_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 161686, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'san_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 161190, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'san_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 158917, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'san_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 169347, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'san_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 159256, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'san_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 153825, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'san_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 157415, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'san_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 160732, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'san_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 156710, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'san_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 165888, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'san_Deva-snd_Deva': {'num_samples': 1503, 'number_of_characters': 159132, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'san_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 171375, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'san_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 154199, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'san_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157814, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 51.43, 'max_sentence1_length': 181, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'sat_Olck-asm_Beng': {'num_samples': 1503, 'number_of_characters': 169379, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'sat_Olck-ben_Beng': {'num_samples': 1503, 'number_of_characters': 163783, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'sat_Olck-brx_Deva': {'num_samples': 1503, 'number_of_characters': 169839, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'sat_Olck-doi_Deva': {'num_samples': 1503, 'number_of_characters': 174827, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'sat_Olck-eng_Latn': {'num_samples': 1503, 'number_of_characters': 168511, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'sat_Olck-gom_Deva': {'num_samples': 1503, 'number_of_characters': 164077, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'sat_Olck-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 166064, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'sat_Olck-hin_Deva': {'num_samples': 1503, 'number_of_characters': 167759, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'sat_Olck-kan_Knda': {'num_samples': 1503, 'number_of_characters': 172972, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'sat_Olck-kas_Arab': {'num_samples': 1503, 'number_of_characters': 172476, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'sat_Olck-mai_Deva': {'num_samples': 1503, 'number_of_characters': 170203, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'sat_Olck-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 180633, 'unique_pairs': 1501, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'sat_Olck-mar_Deva': {'num_samples': 1503, 'number_of_characters': 170542, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'sat_Olck-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 165111, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'sat_Olck-npi_Deva': {'num_samples': 1503, 'number_of_characters': 168701, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'sat_Olck-ory_Orya': {'num_samples': 1503, 'number_of_characters': 172018, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'sat_Olck-pan_Guru': {'num_samples': 1503, 'number_of_characters': 167996, 'unique_pairs': 1501, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'sat_Olck-san_Deva': {'num_samples': 1503, 'number_of_characters': 165888, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'sat_Olck-snd_Deva': {'num_samples': 1503, 'number_of_characters': 170418, 'unique_pairs': 1501, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'sat_Olck-tam_Taml': {'num_samples': 1503, 'number_of_characters': 182661, 'unique_pairs': 1503, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'sat_Olck-tel_Telu': {'num_samples': 1503, 'number_of_characters': 165485, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'sat_Olck-urd_Arab': {'num_samples': 1503, 'number_of_characters': 169100, 'unique_pairs': 1502, 'min_sentence1_length': 7, 'average_sentence1_length': 58.94, 'max_sentence1_length': 225, 'unique_sentence1': 1500, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'snd_Deva-asm_Beng': {'num_samples': 1503, 'number_of_characters': 162623, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'snd_Deva-ben_Beng': {'num_samples': 1503, 'number_of_characters': 157027, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'snd_Deva-brx_Deva': {'num_samples': 1503, 'number_of_characters': 163083, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'snd_Deva-doi_Deva': {'num_samples': 1503, 'number_of_characters': 168071, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'snd_Deva-eng_Latn': {'num_samples': 1503, 'number_of_characters': 161755, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'snd_Deva-gom_Deva': {'num_samples': 1503, 'number_of_characters': 157321, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'snd_Deva-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 159308, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'snd_Deva-hin_Deva': {'num_samples': 1503, 'number_of_characters': 161003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'snd_Deva-kan_Knda': {'num_samples': 1503, 'number_of_characters': 166216, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'snd_Deva-kas_Arab': {'num_samples': 1503, 'number_of_characters': 165720, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'snd_Deva-mai_Deva': {'num_samples': 1503, 'number_of_characters': 163447, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'snd_Deva-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 173877, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'snd_Deva-mar_Deva': {'num_samples': 1503, 'number_of_characters': 163786, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'snd_Deva-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 158355, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'snd_Deva-npi_Deva': {'num_samples': 1503, 'number_of_characters': 161945, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'snd_Deva-ory_Orya': {'num_samples': 1503, 'number_of_characters': 165262, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'snd_Deva-pan_Guru': {'num_samples': 1503, 'number_of_characters': 161240, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'snd_Deva-san_Deva': {'num_samples': 1503, 'number_of_characters': 159132, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'snd_Deva-sat_Olck': {'num_samples': 1503, 'number_of_characters': 170418, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'snd_Deva-tam_Taml': {'num_samples': 1503, 'number_of_characters': 175905, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'snd_Deva-tel_Telu': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'snd_Deva-urd_Arab': {'num_samples': 1503, 'number_of_characters': 162344, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 54.45, 'max_sentence1_length': 195, 'unique_sentence1': 1490, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'tam_Taml-asm_Beng': {'num_samples': 1503, 'number_of_characters': 174866, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'tam_Taml-ben_Beng': {'num_samples': 1503, 'number_of_characters': 169270, 'unique_pairs': 1501, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'tam_Taml-brx_Deva': {'num_samples': 1503, 'number_of_characters': 175326, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'tam_Taml-doi_Deva': {'num_samples': 1503, 'number_of_characters': 180314, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'tam_Taml-eng_Latn': {'num_samples': 1503, 'number_of_characters': 173998, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'tam_Taml-gom_Deva': {'num_samples': 1503, 'number_of_characters': 169564, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'tam_Taml-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 171551, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'tam_Taml-hin_Deva': {'num_samples': 1503, 'number_of_characters': 173246, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'tam_Taml-kan_Knda': {'num_samples': 1503, 'number_of_characters': 178459, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'tam_Taml-kas_Arab': {'num_samples': 1503, 'number_of_characters': 177963, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'tam_Taml-mai_Deva': {'num_samples': 1503, 'number_of_characters': 175690, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'tam_Taml-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 186120, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'tam_Taml-mar_Deva': {'num_samples': 1503, 'number_of_characters': 176029, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'tam_Taml-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 170598, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'tam_Taml-npi_Deva': {'num_samples': 1503, 'number_of_characters': 174188, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'tam_Taml-ory_Orya': {'num_samples': 1503, 'number_of_characters': 177505, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'tam_Taml-pan_Guru': {'num_samples': 1503, 'number_of_characters': 173483, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'tam_Taml-san_Deva': {'num_samples': 1503, 'number_of_characters': 171375, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'tam_Taml-sat_Olck': {'num_samples': 1503, 'number_of_characters': 182661, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'tam_Taml-snd_Deva': {'num_samples': 1503, 'number_of_characters': 175905, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'tam_Taml-tel_Telu': {'num_samples': 1503, 'number_of_characters': 170972, 'unique_pairs': 1502, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}, 'tam_Taml-urd_Arab': {'num_samples': 1503, 'number_of_characters': 174587, 'unique_pairs': 1503, 'min_sentence1_length': 3, 'average_sentence1_length': 62.59, 'max_sentence1_length': 224, 'unique_sentence1': 1492, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'tel_Telu-asm_Beng': {'num_samples': 1503, 'number_of_characters': 157690, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'tel_Telu-ben_Beng': {'num_samples': 1503, 'number_of_characters': 152094, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'tel_Telu-brx_Deva': {'num_samples': 1503, 'number_of_characters': 158150, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'tel_Telu-doi_Deva': {'num_samples': 1503, 'number_of_characters': 163138, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'tel_Telu-eng_Latn': {'num_samples': 1503, 'number_of_characters': 156822, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'tel_Telu-gom_Deva': {'num_samples': 1503, 'number_of_characters': 152388, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'tel_Telu-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 154375, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'tel_Telu-hin_Deva': {'num_samples': 1503, 'number_of_characters': 156070, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'tel_Telu-kan_Knda': {'num_samples': 1503, 'number_of_characters': 161283, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'tel_Telu-kas_Arab': {'num_samples': 1503, 'number_of_characters': 160787, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'tel_Telu-mai_Deva': {'num_samples': 1503, 'number_of_characters': 158514, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'tel_Telu-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 168944, 'unique_pairs': 1500, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'tel_Telu-mar_Deva': {'num_samples': 1503, 'number_of_characters': 158853, 'unique_pairs': 1503, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'tel_Telu-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 153422, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'tel_Telu-npi_Deva': {'num_samples': 1503, 'number_of_characters': 157012, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'tel_Telu-ory_Orya': {'num_samples': 1503, 'number_of_characters': 160329, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'tel_Telu-pan_Guru': {'num_samples': 1503, 'number_of_characters': 156307, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'tel_Telu-san_Deva': {'num_samples': 1503, 'number_of_characters': 154199, 'unique_pairs': 1501, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'tel_Telu-sat_Olck': {'num_samples': 1503, 'number_of_characters': 165485, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'tel_Telu-snd_Deva': {'num_samples': 1503, 'number_of_characters': 158729, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'tel_Telu-tam_Taml': {'num_samples': 1503, 'number_of_characters': 170972, 'unique_pairs': 1502, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'tel_Telu-urd_Arab': {'num_samples': 1503, 'number_of_characters': 157411, 'unique_pairs': 1499, 'min_sentence1_length': 6, 'average_sentence1_length': 51.16, 'max_sentence1_length': 182, 'unique_sentence1': 1495, 'min_sentence2_length': 4, 'average_sentence2_length': 53.57, 'max_sentence2_length': 206, 'unique_sentence2': 1498}, 'urd_Arab-asm_Beng': {'num_samples': 1503, 'number_of_characters': 161305, 'unique_pairs': 1498, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.75, 'max_sentence2_length': 208, 'unique_sentence2': 1497}, 'urd_Arab-ben_Beng': {'num_samples': 1503, 'number_of_characters': 155709, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.03, 'max_sentence2_length': 178, 'unique_sentence2': 1497}, 'urd_Arab-brx_Deva': {'num_samples': 1503, 'number_of_characters': 161765, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.06, 'max_sentence2_length': 210, 'unique_sentence2': 1498}, 'urd_Arab-doi_Deva': {'num_samples': 1503, 'number_of_characters': 166753, 'unique_pairs': 1500, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 57.38, 'max_sentence2_length': 209, 'unique_sentence2': 1499}, 'urd_Arab-eng_Latn': {'num_samples': 1503, 'number_of_characters': 160437, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.18, 'max_sentence2_length': 201, 'unique_sentence2': 1497}, 'urd_Arab-gom_Deva': {'num_samples': 1503, 'number_of_characters': 156003, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 50.23, 'max_sentence2_length': 203, 'unique_sentence2': 1500}, 'urd_Arab-guj_Gujr': {'num_samples': 1503, 'number_of_characters': 157990, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 51.55, 'max_sentence2_length': 205, 'unique_sentence2': 1500}, 'urd_Arab-hin_Deva': {'num_samples': 1503, 'number_of_characters': 159685, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.68, 'max_sentence2_length': 192, 'unique_sentence2': 1497}, 'urd_Arab-kan_Knda': {'num_samples': 1503, 'number_of_characters': 164898, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 56.14, 'max_sentence2_length': 201, 'unique_sentence2': 1499}, 'urd_Arab-kas_Arab': {'num_samples': 1503, 'number_of_characters': 164402, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 55.81, 'max_sentence2_length': 203, 'unique_sentence2': 1502}, 'urd_Arab-mai_Deva': {'num_samples': 1503, 'number_of_characters': 162129, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 54.3, 'max_sentence2_length': 230, 'unique_sentence2': 1499}, 'urd_Arab-mal_Mlym': {'num_samples': 1503, 'number_of_characters': 172559, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 5, 'average_sentence2_length': 61.24, 'max_sentence2_length': 219, 'unique_sentence2': 1495}, 'urd_Arab-mar_Deva': {'num_samples': 1503, 'number_of_characters': 162468, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.53, 'max_sentence2_length': 221, 'unique_sentence2': 1501}, 'urd_Arab-mni_Mtei': {'num_samples': 1503, 'number_of_characters': 157037, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 50.91, 'max_sentence2_length': 239, 'unique_sentence2': 1498}, 'urd_Arab-npi_Deva': {'num_samples': 1503, 'number_of_characters': 160627, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 53.3, 'max_sentence2_length': 223, 'unique_sentence2': 1497}, 'urd_Arab-ory_Orya': {'num_samples': 1503, 'number_of_characters': 163944, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 55.51, 'max_sentence2_length': 195, 'unique_sentence2': 1500}, 'urd_Arab-pan_Guru': {'num_samples': 1503, 'number_of_characters': 159922, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 52.83, 'max_sentence2_length': 221, 'unique_sentence2': 1495}, 'urd_Arab-san_Deva': {'num_samples': 1503, 'number_of_characters': 157814, 'unique_pairs': 1501, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 51.43, 'max_sentence2_length': 181, 'unique_sentence2': 1500}, 'urd_Arab-sat_Olck': {'num_samples': 1503, 'number_of_characters': 169100, 'unique_pairs': 1502, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 7, 'average_sentence2_length': 58.94, 'max_sentence2_length': 225, 'unique_sentence2': 1500}, 'urd_Arab-snd_Deva': {'num_samples': 1503, 'number_of_characters': 162344, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 4, 'average_sentence2_length': 54.45, 'max_sentence2_length': 195, 'unique_sentence2': 1490}, 'urd_Arab-tam_Taml': {'num_samples': 1503, 'number_of_characters': 174587, 'unique_pairs': 1503, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 3, 'average_sentence2_length': 62.59, 'max_sentence2_length': 224, 'unique_sentence2': 1492}, 'urd_Arab-tel_Telu': {'num_samples': 1503, 'number_of_characters': 157411, 'unique_pairs': 1499, 'min_sentence1_length': 4, 'average_sentence1_length': 53.57, 'max_sentence1_length': 206, 'unique_sentence1': 1498, 'min_sentence2_length': 6, 'average_sentence2_length': 51.16, 'max_sentence2_length': 182, 'unique_sentence2': 1495}}}} | +| [IN22GenBitextMining](https://huggingface.co/datasets/ai4bharat/IN22-Gen) (Jay Gala, 2023) | ['asm', 'ben', 'brx', 'doi', 'eng', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Government, Legal, News, Non-fiction, Religious, Web, Written] | {'test': 518144} | {'test': {'num_samples': 518144, 'number_of_characters': 162367876, 'unique_pairs': 518101, 'min_sentence1_length': 9, 'average_sentence1_length': 156.68, 'max_sentence1_length': 692, 'unique_sentence1': 23550, 'min_sentence2_length': 9, 'average_sentence2_length': 156.68, 'max_sentence2_length': 692, 'unique_sentence2': 23550, 'hf_subset_descriptive_stats': {'asm_Beng-ben_Beng': {'num_samples': 1024, 'number_of_characters': 310622, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'asm_Beng-brx_Deva': {'num_samples': 1024, 'number_of_characters': 323609, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'asm_Beng-doi_Deva': {'num_samples': 1024, 'number_of_characters': 319020, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'asm_Beng-eng_Latn': {'num_samples': 1024, 'number_of_characters': 320098, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'asm_Beng-gom_Deva': {'num_samples': 1024, 'number_of_characters': 312594, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'asm_Beng-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 309440, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'asm_Beng-hin_Deva': {'num_samples': 1024, 'number_of_characters': 320106, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'asm_Beng-kan_Knda': {'num_samples': 1024, 'number_of_characters': 332064, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'asm_Beng-kas_Arab': {'num_samples': 1024, 'number_of_characters': 322764, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'asm_Beng-mai_Deva': {'num_samples': 1024, 'number_of_characters': 308682, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'asm_Beng-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 343636, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'asm_Beng-mar_Deva': {'num_samples': 1024, 'number_of_characters': 321784, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'asm_Beng-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 313134, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'asm_Beng-npi_Deva': {'num_samples': 1024, 'number_of_characters': 313419, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'asm_Beng-ory_Orya': {'num_samples': 1024, 'number_of_characters': 334226, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'asm_Beng-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306863, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'asm_Beng-san_Deva': {'num_samples': 1024, 'number_of_characters': 318079, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'asm_Beng-sat_Olck': {'num_samples': 1024, 'number_of_characters': 326732, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'asm_Beng-snd_Deva': {'num_samples': 1024, 'number_of_characters': 320421, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'asm_Beng-tam_Taml': {'num_samples': 1024, 'number_of_characters': 348346, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'asm_Beng-tel_Telu': {'num_samples': 1024, 'number_of_characters': 319045, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'asm_Beng-urd_Arab': {'num_samples': 1024, 'number_of_characters': 315134, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 156.7, 'max_sentence1_length': 582, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'ben_Beng-asm_Beng': {'num_samples': 1024, 'number_of_characters': 310622, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'ben_Beng-brx_Deva': {'num_samples': 1024, 'number_of_characters': 313313, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'ben_Beng-doi_Deva': {'num_samples': 1024, 'number_of_characters': 308724, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'ben_Beng-eng_Latn': {'num_samples': 1024, 'number_of_characters': 309802, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'ben_Beng-gom_Deva': {'num_samples': 1024, 'number_of_characters': 302298, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'ben_Beng-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 299144, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'ben_Beng-hin_Deva': {'num_samples': 1024, 'number_of_characters': 309810, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'ben_Beng-kan_Knda': {'num_samples': 1024, 'number_of_characters': 321768, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'ben_Beng-kas_Arab': {'num_samples': 1024, 'number_of_characters': 312468, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'ben_Beng-mai_Deva': {'num_samples': 1024, 'number_of_characters': 298386, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'ben_Beng-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 333340, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'ben_Beng-mar_Deva': {'num_samples': 1024, 'number_of_characters': 311488, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'ben_Beng-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 302838, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'ben_Beng-npi_Deva': {'num_samples': 1024, 'number_of_characters': 303123, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'ben_Beng-ory_Orya': {'num_samples': 1024, 'number_of_characters': 323930, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'ben_Beng-pan_Guru': {'num_samples': 1024, 'number_of_characters': 296567, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'ben_Beng-san_Deva': {'num_samples': 1024, 'number_of_characters': 307783, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'ben_Beng-sat_Olck': {'num_samples': 1024, 'number_of_characters': 316436, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'ben_Beng-snd_Deva': {'num_samples': 1024, 'number_of_characters': 310125, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'ben_Beng-tam_Taml': {'num_samples': 1024, 'number_of_characters': 338050, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'ben_Beng-tel_Telu': {'num_samples': 1024, 'number_of_characters': 308749, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'ben_Beng-urd_Arab': {'num_samples': 1024, 'number_of_characters': 304838, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 146.64, 'max_sentence1_length': 538, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'brx_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 323609, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'brx_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 313313, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'brx_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 321711, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'brx_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 322789, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'brx_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 315285, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'brx_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 312131, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'brx_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 322797, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'brx_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 334755, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'brx_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 325455, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'brx_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 311373, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'brx_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 346327, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'brx_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 324475, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'brx_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 315825, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'brx_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 316110, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'brx_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 336917, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'brx_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 309554, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'brx_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 320770, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'brx_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 329423, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'brx_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 323112, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'brx_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 351037, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'brx_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 321736, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'brx_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 317825, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 159.33, 'max_sentence1_length': 631, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'doi_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 319020, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'doi_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 308724, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'doi_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 321711, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'doi_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 318200, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'doi_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 310696, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'doi_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 307542, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'doi_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 318208, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'doi_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 330166, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'doi_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 320866, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'doi_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 306784, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'doi_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 341738, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'doi_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 319886, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'doi_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 311236, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'doi_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 311521, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'doi_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 332328, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'doi_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 304965, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'doi_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 316181, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'doi_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 324834, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'doi_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 318523, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'doi_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 346448, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'doi_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 317147, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'doi_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 313236, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.84, 'max_sentence1_length': 500, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'eng_Latn-asm_Beng': {'num_samples': 1024, 'number_of_characters': 320098, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'eng_Latn-ben_Beng': {'num_samples': 1024, 'number_of_characters': 309802, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'eng_Latn-brx_Deva': {'num_samples': 1024, 'number_of_characters': 322789, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'eng_Latn-doi_Deva': {'num_samples': 1024, 'number_of_characters': 318200, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'eng_Latn-gom_Deva': {'num_samples': 1024, 'number_of_characters': 311774, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'eng_Latn-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 308620, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'eng_Latn-hin_Deva': {'num_samples': 1024, 'number_of_characters': 319286, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'eng_Latn-kan_Knda': {'num_samples': 1024, 'number_of_characters': 331244, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'eng_Latn-kas_Arab': {'num_samples': 1024, 'number_of_characters': 321944, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'eng_Latn-mai_Deva': {'num_samples': 1024, 'number_of_characters': 307862, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'eng_Latn-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 342816, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'eng_Latn-mar_Deva': {'num_samples': 1024, 'number_of_characters': 320964, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'eng_Latn-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 312314, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'eng_Latn-npi_Deva': {'num_samples': 1024, 'number_of_characters': 312599, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'eng_Latn-ory_Orya': {'num_samples': 1024, 'number_of_characters': 333406, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'eng_Latn-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306043, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'eng_Latn-san_Deva': {'num_samples': 1024, 'number_of_characters': 317259, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'eng_Latn-sat_Olck': {'num_samples': 1024, 'number_of_characters': 325912, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'eng_Latn-snd_Deva': {'num_samples': 1024, 'number_of_characters': 319601, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'eng_Latn-tam_Taml': {'num_samples': 1024, 'number_of_characters': 347526, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'eng_Latn-tel_Telu': {'num_samples': 1024, 'number_of_characters': 318225, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'eng_Latn-urd_Arab': {'num_samples': 1024, 'number_of_characters': 314314, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 155.9, 'max_sentence1_length': 532, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'gom_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 312594, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'gom_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 302298, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'gom_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 315285, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'gom_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 310696, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'gom_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 311774, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'gom_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 301116, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'gom_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 311782, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'gom_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 323740, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'gom_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 314440, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'gom_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 300358, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'gom_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 335312, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'gom_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 313460, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'gom_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 304810, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'gom_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 305095, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'gom_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 325902, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'gom_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 298539, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'gom_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 309755, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'gom_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 318408, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'gom_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 312097, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'gom_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 340022, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'gom_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 310721, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'gom_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 306810, 'unique_pairs': 1024, 'min_sentence1_length': 17, 'average_sentence1_length': 148.57, 'max_sentence1_length': 537, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'guj_Gujr-asm_Beng': {'num_samples': 1024, 'number_of_characters': 309440, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'guj_Gujr-ben_Beng': {'num_samples': 1024, 'number_of_characters': 299144, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'guj_Gujr-brx_Deva': {'num_samples': 1024, 'number_of_characters': 312131, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'guj_Gujr-doi_Deva': {'num_samples': 1024, 'number_of_characters': 307542, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'guj_Gujr-eng_Latn': {'num_samples': 1024, 'number_of_characters': 308620, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'guj_Gujr-gom_Deva': {'num_samples': 1024, 'number_of_characters': 301116, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'guj_Gujr-hin_Deva': {'num_samples': 1024, 'number_of_characters': 308628, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'guj_Gujr-kan_Knda': {'num_samples': 1024, 'number_of_characters': 320586, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'guj_Gujr-kas_Arab': {'num_samples': 1024, 'number_of_characters': 311286, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'guj_Gujr-mai_Deva': {'num_samples': 1024, 'number_of_characters': 297204, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'guj_Gujr-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 332158, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'guj_Gujr-mar_Deva': {'num_samples': 1024, 'number_of_characters': 310306, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'guj_Gujr-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 301656, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'guj_Gujr-npi_Deva': {'num_samples': 1024, 'number_of_characters': 301941, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'guj_Gujr-ory_Orya': {'num_samples': 1024, 'number_of_characters': 322748, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'guj_Gujr-pan_Guru': {'num_samples': 1024, 'number_of_characters': 295385, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'guj_Gujr-san_Deva': {'num_samples': 1024, 'number_of_characters': 306601, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'guj_Gujr-sat_Olck': {'num_samples': 1024, 'number_of_characters': 315254, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'guj_Gujr-snd_Deva': {'num_samples': 1024, 'number_of_characters': 308943, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'guj_Gujr-tam_Taml': {'num_samples': 1024, 'number_of_characters': 336868, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'guj_Gujr-tel_Telu': {'num_samples': 1024, 'number_of_characters': 307567, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'guj_Gujr-urd_Arab': {'num_samples': 1024, 'number_of_characters': 303656, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 145.49, 'max_sentence1_length': 488, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'hin_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 320106, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'hin_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 309810, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'hin_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 322797, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'hin_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 318208, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'hin_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 319286, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'hin_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 311782, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'hin_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 308628, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'hin_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 331252, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'hin_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 321952, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'hin_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 307870, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'hin_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 342824, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'hin_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 320972, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'hin_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 312322, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'hin_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 312607, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'hin_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 333414, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'hin_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306051, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'hin_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 317267, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'hin_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 325920, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'hin_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 319609, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'hin_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 347534, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'hin_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 318233, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'hin_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 314322, 'unique_pairs': 1024, 'min_sentence1_length': 21, 'average_sentence1_length': 155.91, 'max_sentence1_length': 531, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'kan_Knda-asm_Beng': {'num_samples': 1024, 'number_of_characters': 332064, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'kan_Knda-ben_Beng': {'num_samples': 1024, 'number_of_characters': 321768, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'kan_Knda-brx_Deva': {'num_samples': 1024, 'number_of_characters': 334755, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'kan_Knda-doi_Deva': {'num_samples': 1024, 'number_of_characters': 330166, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'kan_Knda-eng_Latn': {'num_samples': 1024, 'number_of_characters': 331244, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'kan_Knda-gom_Deva': {'num_samples': 1024, 'number_of_characters': 323740, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'kan_Knda-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 320586, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'kan_Knda-hin_Deva': {'num_samples': 1024, 'number_of_characters': 331252, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'kan_Knda-kas_Arab': {'num_samples': 1024, 'number_of_characters': 333910, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'kan_Knda-mai_Deva': {'num_samples': 1024, 'number_of_characters': 319828, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'kan_Knda-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 354782, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'kan_Knda-mar_Deva': {'num_samples': 1024, 'number_of_characters': 332930, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'kan_Knda-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 324280, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'kan_Knda-npi_Deva': {'num_samples': 1024, 'number_of_characters': 324565, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'kan_Knda-ory_Orya': {'num_samples': 1024, 'number_of_characters': 345372, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'kan_Knda-pan_Guru': {'num_samples': 1024, 'number_of_characters': 318009, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'kan_Knda-san_Deva': {'num_samples': 1024, 'number_of_characters': 329225, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'kan_Knda-sat_Olck': {'num_samples': 1024, 'number_of_characters': 337878, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'kan_Knda-snd_Deva': {'num_samples': 1024, 'number_of_characters': 331567, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'kan_Knda-tam_Taml': {'num_samples': 1024, 'number_of_characters': 359492, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'kan_Knda-tel_Telu': {'num_samples': 1024, 'number_of_characters': 330191, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'kan_Knda-urd_Arab': {'num_samples': 1024, 'number_of_characters': 326280, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 167.58, 'max_sentence1_length': 668, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'kas_Arab-asm_Beng': {'num_samples': 1024, 'number_of_characters': 322764, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'kas_Arab-ben_Beng': {'num_samples': 1024, 'number_of_characters': 312468, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'kas_Arab-brx_Deva': {'num_samples': 1024, 'number_of_characters': 325455, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'kas_Arab-doi_Deva': {'num_samples': 1024, 'number_of_characters': 320866, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'kas_Arab-eng_Latn': {'num_samples': 1024, 'number_of_characters': 321944, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'kas_Arab-gom_Deva': {'num_samples': 1024, 'number_of_characters': 314440, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'kas_Arab-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 311286, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'kas_Arab-hin_Deva': {'num_samples': 1024, 'number_of_characters': 321952, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'kas_Arab-kan_Knda': {'num_samples': 1024, 'number_of_characters': 333910, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'kas_Arab-mai_Deva': {'num_samples': 1024, 'number_of_characters': 310528, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'kas_Arab-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 345482, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'kas_Arab-mar_Deva': {'num_samples': 1024, 'number_of_characters': 323630, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'kas_Arab-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 314980, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'kas_Arab-npi_Deva': {'num_samples': 1024, 'number_of_characters': 315265, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'kas_Arab-ory_Orya': {'num_samples': 1024, 'number_of_characters': 336072, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'kas_Arab-pan_Guru': {'num_samples': 1024, 'number_of_characters': 308709, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'kas_Arab-san_Deva': {'num_samples': 1024, 'number_of_characters': 319925, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'kas_Arab-sat_Olck': {'num_samples': 1024, 'number_of_characters': 328578, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'kas_Arab-snd_Deva': {'num_samples': 1024, 'number_of_characters': 322267, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'kas_Arab-tam_Taml': {'num_samples': 1024, 'number_of_characters': 350192, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'kas_Arab-tel_Telu': {'num_samples': 1024, 'number_of_characters': 320891, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'kas_Arab-urd_Arab': {'num_samples': 1024, 'number_of_characters': 316980, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 158.5, 'max_sentence1_length': 520, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mai_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 308682, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mai_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 298386, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mai_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 311373, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mai_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 306784, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mai_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 307862, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mai_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 300358, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mai_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 297204, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mai_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 307870, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mai_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 319828, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mai_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 310528, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mai_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 331400, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'mai_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 309548, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'mai_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 300898, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'mai_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 301183, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mai_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 321990, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mai_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 294627, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mai_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 305843, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mai_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 314496, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mai_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 308185, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mai_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 336110, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mai_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 306809, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mai_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 302898, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 144.75, 'max_sentence1_length': 562, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mal_Mlym-asm_Beng': {'num_samples': 1024, 'number_of_characters': 343636, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mal_Mlym-ben_Beng': {'num_samples': 1024, 'number_of_characters': 333340, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mal_Mlym-brx_Deva': {'num_samples': 1024, 'number_of_characters': 346327, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mal_Mlym-doi_Deva': {'num_samples': 1024, 'number_of_characters': 341738, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mal_Mlym-eng_Latn': {'num_samples': 1024, 'number_of_characters': 342816, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mal_Mlym-gom_Deva': {'num_samples': 1024, 'number_of_characters': 335312, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mal_Mlym-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 332158, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mal_Mlym-hin_Deva': {'num_samples': 1024, 'number_of_characters': 342824, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mal_Mlym-kan_Knda': {'num_samples': 1024, 'number_of_characters': 354782, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mal_Mlym-kas_Arab': {'num_samples': 1024, 'number_of_characters': 345482, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mal_Mlym-mai_Deva': {'num_samples': 1024, 'number_of_characters': 331400, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'mal_Mlym-mar_Deva': {'num_samples': 1024, 'number_of_characters': 344502, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'mal_Mlym-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 335852, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'mal_Mlym-npi_Deva': {'num_samples': 1024, 'number_of_characters': 336137, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mal_Mlym-ory_Orya': {'num_samples': 1024, 'number_of_characters': 356944, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mal_Mlym-pan_Guru': {'num_samples': 1024, 'number_of_characters': 329581, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mal_Mlym-san_Deva': {'num_samples': 1024, 'number_of_characters': 340797, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mal_Mlym-sat_Olck': {'num_samples': 1024, 'number_of_characters': 349450, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mal_Mlym-snd_Deva': {'num_samples': 1024, 'number_of_characters': 343139, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mal_Mlym-tam_Taml': {'num_samples': 1024, 'number_of_characters': 371064, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mal_Mlym-tel_Telu': {'num_samples': 1024, 'number_of_characters': 341763, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mal_Mlym-urd_Arab': {'num_samples': 1024, 'number_of_characters': 337852, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 178.88, 'max_sentence1_length': 692, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mar_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 321784, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mar_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 311488, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mar_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 324475, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mar_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 319886, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mar_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 320964, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mar_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 313460, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mar_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 310306, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mar_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 320972, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mar_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 332930, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mar_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 323630, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mar_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 309548, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'mar_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 344502, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'mar_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 314000, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'mar_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 314285, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mar_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 335092, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mar_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 307729, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mar_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 318945, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mar_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 327598, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mar_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 321287, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mar_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 349212, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mar_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 319911, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mar_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 316000, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 157.54, 'max_sentence1_length': 555, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'mni_Mtei-asm_Beng': {'num_samples': 1024, 'number_of_characters': 313134, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'mni_Mtei-ben_Beng': {'num_samples': 1024, 'number_of_characters': 302838, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'mni_Mtei-brx_Deva': {'num_samples': 1024, 'number_of_characters': 315825, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'mni_Mtei-doi_Deva': {'num_samples': 1024, 'number_of_characters': 311236, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'mni_Mtei-eng_Latn': {'num_samples': 1024, 'number_of_characters': 312314, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'mni_Mtei-gom_Deva': {'num_samples': 1024, 'number_of_characters': 304810, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'mni_Mtei-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 301656, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'mni_Mtei-hin_Deva': {'num_samples': 1024, 'number_of_characters': 312322, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'mni_Mtei-kan_Knda': {'num_samples': 1024, 'number_of_characters': 324280, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'mni_Mtei-kas_Arab': {'num_samples': 1024, 'number_of_characters': 314980, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'mni_Mtei-mai_Deva': {'num_samples': 1024, 'number_of_characters': 300898, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'mni_Mtei-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 335852, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'mni_Mtei-mar_Deva': {'num_samples': 1024, 'number_of_characters': 314000, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'mni_Mtei-npi_Deva': {'num_samples': 1024, 'number_of_characters': 305635, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'mni_Mtei-ory_Orya': {'num_samples': 1024, 'number_of_characters': 326442, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'mni_Mtei-pan_Guru': {'num_samples': 1024, 'number_of_characters': 299079, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'mni_Mtei-san_Deva': {'num_samples': 1024, 'number_of_characters': 310295, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'mni_Mtei-sat_Olck': {'num_samples': 1024, 'number_of_characters': 318948, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'mni_Mtei-snd_Deva': {'num_samples': 1024, 'number_of_characters': 312637, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'mni_Mtei-tam_Taml': {'num_samples': 1024, 'number_of_characters': 340562, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'mni_Mtei-tel_Telu': {'num_samples': 1024, 'number_of_characters': 311261, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'mni_Mtei-urd_Arab': {'num_samples': 1024, 'number_of_characters': 307350, 'unique_pairs': 1024, 'min_sentence1_length': 16, 'average_sentence1_length': 149.1, 'max_sentence1_length': 597, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'npi_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 313419, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'npi_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 303123, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'npi_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 316110, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'npi_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 311521, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'npi_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 312599, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'npi_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 305095, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'npi_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 301941, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'npi_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 312607, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'npi_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 324565, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'npi_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 315265, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'npi_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 301183, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'npi_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 336137, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'npi_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 314285, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'npi_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 305635, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'npi_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 326727, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'npi_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 299364, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'npi_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 310580, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'npi_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 319233, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'npi_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 312922, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'npi_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 340847, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'npi_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 311546, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'npi_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 307635, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 149.38, 'max_sentence1_length': 525, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'ory_Orya-asm_Beng': {'num_samples': 1024, 'number_of_characters': 334226, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'ory_Orya-ben_Beng': {'num_samples': 1024, 'number_of_characters': 323930, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'ory_Orya-brx_Deva': {'num_samples': 1024, 'number_of_characters': 336917, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'ory_Orya-doi_Deva': {'num_samples': 1024, 'number_of_characters': 332328, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'ory_Orya-eng_Latn': {'num_samples': 1024, 'number_of_characters': 333406, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'ory_Orya-gom_Deva': {'num_samples': 1024, 'number_of_characters': 325902, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'ory_Orya-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 322748, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'ory_Orya-hin_Deva': {'num_samples': 1024, 'number_of_characters': 333414, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'ory_Orya-kan_Knda': {'num_samples': 1024, 'number_of_characters': 345372, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'ory_Orya-kas_Arab': {'num_samples': 1024, 'number_of_characters': 336072, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'ory_Orya-mai_Deva': {'num_samples': 1024, 'number_of_characters': 321990, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'ory_Orya-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 356944, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'ory_Orya-mar_Deva': {'num_samples': 1024, 'number_of_characters': 335092, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'ory_Orya-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 326442, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'ory_Orya-npi_Deva': {'num_samples': 1024, 'number_of_characters': 326727, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'ory_Orya-pan_Guru': {'num_samples': 1024, 'number_of_characters': 320171, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'ory_Orya-san_Deva': {'num_samples': 1024, 'number_of_characters': 331387, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'ory_Orya-sat_Olck': {'num_samples': 1024, 'number_of_characters': 340040, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'ory_Orya-snd_Deva': {'num_samples': 1024, 'number_of_characters': 333729, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'ory_Orya-tam_Taml': {'num_samples': 1024, 'number_of_characters': 361654, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'ory_Orya-tel_Telu': {'num_samples': 1024, 'number_of_characters': 332353, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'ory_Orya-urd_Arab': {'num_samples': 1024, 'number_of_characters': 328442, 'unique_pairs': 1024, 'min_sentence1_length': 10, 'average_sentence1_length': 169.69, 'max_sentence1_length': 578, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'pan_Guru-asm_Beng': {'num_samples': 1024, 'number_of_characters': 306863, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'pan_Guru-ben_Beng': {'num_samples': 1024, 'number_of_characters': 296567, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'pan_Guru-brx_Deva': {'num_samples': 1024, 'number_of_characters': 309554, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'pan_Guru-doi_Deva': {'num_samples': 1024, 'number_of_characters': 304965, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'pan_Guru-eng_Latn': {'num_samples': 1024, 'number_of_characters': 306043, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'pan_Guru-gom_Deva': {'num_samples': 1024, 'number_of_characters': 298539, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'pan_Guru-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 295385, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'pan_Guru-hin_Deva': {'num_samples': 1024, 'number_of_characters': 306051, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'pan_Guru-kan_Knda': {'num_samples': 1024, 'number_of_characters': 318009, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'pan_Guru-kas_Arab': {'num_samples': 1024, 'number_of_characters': 308709, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'pan_Guru-mai_Deva': {'num_samples': 1024, 'number_of_characters': 294627, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'pan_Guru-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 329581, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'pan_Guru-mar_Deva': {'num_samples': 1024, 'number_of_characters': 307729, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'pan_Guru-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 299079, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'pan_Guru-npi_Deva': {'num_samples': 1024, 'number_of_characters': 299364, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'pan_Guru-ory_Orya': {'num_samples': 1024, 'number_of_characters': 320171, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'pan_Guru-san_Deva': {'num_samples': 1024, 'number_of_characters': 304024, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'pan_Guru-sat_Olck': {'num_samples': 1024, 'number_of_characters': 312677, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'pan_Guru-snd_Deva': {'num_samples': 1024, 'number_of_characters': 306366, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'pan_Guru-tam_Taml': {'num_samples': 1024, 'number_of_characters': 334291, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'pan_Guru-tel_Telu': {'num_samples': 1024, 'number_of_characters': 304990, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'pan_Guru-urd_Arab': {'num_samples': 1024, 'number_of_characters': 301079, 'unique_pairs': 1024, 'min_sentence1_length': 19, 'average_sentence1_length': 142.97, 'max_sentence1_length': 476, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'san_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 318079, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'san_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 307783, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'san_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 320770, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'san_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 316181, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'san_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 317259, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'san_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 309755, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'san_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 306601, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'san_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 317267, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'san_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 329225, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'san_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 319925, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'san_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 305843, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'san_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 340797, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'san_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 318945, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'san_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 310295, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'san_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 310580, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'san_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 331387, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'san_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 304024, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'san_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 323893, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'san_Deva-snd_Deva': {'num_samples': 1024, 'number_of_characters': 317582, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'san_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 345507, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'san_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 316206, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'san_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 312295, 'unique_pairs': 1024, 'min_sentence1_length': 9, 'average_sentence1_length': 153.93, 'max_sentence1_length': 601, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'sat_Olck-asm_Beng': {'num_samples': 1024, 'number_of_characters': 326732, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'sat_Olck-ben_Beng': {'num_samples': 1024, 'number_of_characters': 316436, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'sat_Olck-brx_Deva': {'num_samples': 1024, 'number_of_characters': 329423, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'sat_Olck-doi_Deva': {'num_samples': 1024, 'number_of_characters': 324834, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'sat_Olck-eng_Latn': {'num_samples': 1024, 'number_of_characters': 325912, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'sat_Olck-gom_Deva': {'num_samples': 1024, 'number_of_characters': 318408, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'sat_Olck-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 315254, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'sat_Olck-hin_Deva': {'num_samples': 1024, 'number_of_characters': 325920, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'sat_Olck-kan_Knda': {'num_samples': 1024, 'number_of_characters': 337878, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'sat_Olck-kas_Arab': {'num_samples': 1024, 'number_of_characters': 328578, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'sat_Olck-mai_Deva': {'num_samples': 1024, 'number_of_characters': 314496, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'sat_Olck-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 349450, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'sat_Olck-mar_Deva': {'num_samples': 1024, 'number_of_characters': 327598, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'sat_Olck-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 318948, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'sat_Olck-npi_Deva': {'num_samples': 1024, 'number_of_characters': 319233, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'sat_Olck-ory_Orya': {'num_samples': 1024, 'number_of_characters': 340040, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'sat_Olck-pan_Guru': {'num_samples': 1024, 'number_of_characters': 312677, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'sat_Olck-san_Deva': {'num_samples': 1024, 'number_of_characters': 323893, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'sat_Olck-snd_Deva': {'num_samples': 1024, 'number_of_characters': 326235, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'sat_Olck-tam_Taml': {'num_samples': 1024, 'number_of_characters': 354160, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'sat_Olck-tel_Telu': {'num_samples': 1024, 'number_of_characters': 324859, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'sat_Olck-urd_Arab': {'num_samples': 1024, 'number_of_characters': 320948, 'unique_pairs': 1024, 'min_sentence1_length': 11, 'average_sentence1_length': 162.38, 'max_sentence1_length': 536, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'snd_Deva-asm_Beng': {'num_samples': 1024, 'number_of_characters': 320421, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'snd_Deva-ben_Beng': {'num_samples': 1024, 'number_of_characters': 310125, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'snd_Deva-brx_Deva': {'num_samples': 1024, 'number_of_characters': 323112, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'snd_Deva-doi_Deva': {'num_samples': 1024, 'number_of_characters': 318523, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'snd_Deva-eng_Latn': {'num_samples': 1024, 'number_of_characters': 319601, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'snd_Deva-gom_Deva': {'num_samples': 1024, 'number_of_characters': 312097, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'snd_Deva-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 308943, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'snd_Deva-hin_Deva': {'num_samples': 1024, 'number_of_characters': 319609, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'snd_Deva-kan_Knda': {'num_samples': 1024, 'number_of_characters': 331567, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'snd_Deva-kas_Arab': {'num_samples': 1024, 'number_of_characters': 322267, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'snd_Deva-mai_Deva': {'num_samples': 1024, 'number_of_characters': 308185, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'snd_Deva-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 343139, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'snd_Deva-mar_Deva': {'num_samples': 1024, 'number_of_characters': 321287, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'snd_Deva-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 312637, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'snd_Deva-npi_Deva': {'num_samples': 1024, 'number_of_characters': 312922, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'snd_Deva-ory_Orya': {'num_samples': 1024, 'number_of_characters': 333729, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'snd_Deva-pan_Guru': {'num_samples': 1024, 'number_of_characters': 306366, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'snd_Deva-san_Deva': {'num_samples': 1024, 'number_of_characters': 317582, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'snd_Deva-sat_Olck': {'num_samples': 1024, 'number_of_characters': 326235, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'snd_Deva-tam_Taml': {'num_samples': 1024, 'number_of_characters': 347849, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'snd_Deva-tel_Telu': {'num_samples': 1024, 'number_of_characters': 318548, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'snd_Deva-urd_Arab': {'num_samples': 1024, 'number_of_characters': 314637, 'unique_pairs': 1024, 'min_sentence1_length': 18, 'average_sentence1_length': 156.21, 'max_sentence1_length': 545, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'tam_Taml-asm_Beng': {'num_samples': 1024, 'number_of_characters': 348346, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'tam_Taml-ben_Beng': {'num_samples': 1024, 'number_of_characters': 338050, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'tam_Taml-brx_Deva': {'num_samples': 1024, 'number_of_characters': 351037, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'tam_Taml-doi_Deva': {'num_samples': 1024, 'number_of_characters': 346448, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'tam_Taml-eng_Latn': {'num_samples': 1024, 'number_of_characters': 347526, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'tam_Taml-gom_Deva': {'num_samples': 1024, 'number_of_characters': 340022, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'tam_Taml-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 336868, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'tam_Taml-hin_Deva': {'num_samples': 1024, 'number_of_characters': 347534, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'tam_Taml-kan_Knda': {'num_samples': 1024, 'number_of_characters': 359492, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'tam_Taml-kas_Arab': {'num_samples': 1024, 'number_of_characters': 350192, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'tam_Taml-mai_Deva': {'num_samples': 1024, 'number_of_characters': 336110, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'tam_Taml-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 371064, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'tam_Taml-mar_Deva': {'num_samples': 1024, 'number_of_characters': 349212, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'tam_Taml-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 340562, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'tam_Taml-npi_Deva': {'num_samples': 1024, 'number_of_characters': 340847, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'tam_Taml-ory_Orya': {'num_samples': 1024, 'number_of_characters': 361654, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'tam_Taml-pan_Guru': {'num_samples': 1024, 'number_of_characters': 334291, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'tam_Taml-san_Deva': {'num_samples': 1024, 'number_of_characters': 345507, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'tam_Taml-sat_Olck': {'num_samples': 1024, 'number_of_characters': 354160, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'tam_Taml-snd_Deva': {'num_samples': 1024, 'number_of_characters': 347849, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'tam_Taml-tel_Telu': {'num_samples': 1024, 'number_of_characters': 346473, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}, 'tam_Taml-urd_Arab': {'num_samples': 1024, 'number_of_characters': 342562, 'unique_pairs': 1024, 'min_sentence1_length': 32, 'average_sentence1_length': 183.48, 'max_sentence1_length': 614, 'unique_sentence1': 1023, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'tel_Telu-asm_Beng': {'num_samples': 1024, 'number_of_characters': 319045, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'tel_Telu-ben_Beng': {'num_samples': 1024, 'number_of_characters': 308749, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'tel_Telu-brx_Deva': {'num_samples': 1024, 'number_of_characters': 321736, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'tel_Telu-doi_Deva': {'num_samples': 1024, 'number_of_characters': 317147, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'tel_Telu-eng_Latn': {'num_samples': 1024, 'number_of_characters': 318225, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'tel_Telu-gom_Deva': {'num_samples': 1024, 'number_of_characters': 310721, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'tel_Telu-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 307567, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'tel_Telu-hin_Deva': {'num_samples': 1024, 'number_of_characters': 318233, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'tel_Telu-kan_Knda': {'num_samples': 1024, 'number_of_characters': 330191, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'tel_Telu-kas_Arab': {'num_samples': 1024, 'number_of_characters': 320891, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'tel_Telu-mai_Deva': {'num_samples': 1024, 'number_of_characters': 306809, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'tel_Telu-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 341763, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'tel_Telu-mar_Deva': {'num_samples': 1024, 'number_of_characters': 319911, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'tel_Telu-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 311261, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'tel_Telu-npi_Deva': {'num_samples': 1024, 'number_of_characters': 311546, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'tel_Telu-ory_Orya': {'num_samples': 1024, 'number_of_characters': 332353, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'tel_Telu-pan_Guru': {'num_samples': 1024, 'number_of_characters': 304990, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'tel_Telu-san_Deva': {'num_samples': 1024, 'number_of_characters': 316206, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'tel_Telu-sat_Olck': {'num_samples': 1024, 'number_of_characters': 324859, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'tel_Telu-snd_Deva': {'num_samples': 1024, 'number_of_characters': 318548, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'tel_Telu-tam_Taml': {'num_samples': 1024, 'number_of_characters': 346473, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'tel_Telu-urd_Arab': {'num_samples': 1024, 'number_of_characters': 313261, 'unique_pairs': 1024, 'min_sentence1_length': 14, 'average_sentence1_length': 154.87, 'max_sentence1_length': 658, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 151.05, 'max_sentence2_length': 574, 'unique_sentence2': 1024}, 'urd_Arab-asm_Beng': {'num_samples': 1024, 'number_of_characters': 315134, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 156.7, 'max_sentence2_length': 582, 'unique_sentence2': 1024}, 'urd_Arab-ben_Beng': {'num_samples': 1024, 'number_of_characters': 304838, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 146.64, 'max_sentence2_length': 538, 'unique_sentence2': 1024}, 'urd_Arab-brx_Deva': {'num_samples': 1024, 'number_of_characters': 317825, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 159.33, 'max_sentence2_length': 631, 'unique_sentence2': 1024}, 'urd_Arab-doi_Deva': {'num_samples': 1024, 'number_of_characters': 313236, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.84, 'max_sentence2_length': 500, 'unique_sentence2': 1024}, 'urd_Arab-eng_Latn': {'num_samples': 1024, 'number_of_characters': 314314, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 155.9, 'max_sentence2_length': 532, 'unique_sentence2': 1024}, 'urd_Arab-gom_Deva': {'num_samples': 1024, 'number_of_characters': 306810, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 17, 'average_sentence2_length': 148.57, 'max_sentence2_length': 537, 'unique_sentence2': 1024}, 'urd_Arab-guj_Gujr': {'num_samples': 1024, 'number_of_characters': 303656, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 145.49, 'max_sentence2_length': 488, 'unique_sentence2': 1024}, 'urd_Arab-hin_Deva': {'num_samples': 1024, 'number_of_characters': 314322, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 21, 'average_sentence2_length': 155.91, 'max_sentence2_length': 531, 'unique_sentence2': 1024}, 'urd_Arab-kan_Knda': {'num_samples': 1024, 'number_of_characters': 326280, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 167.58, 'max_sentence2_length': 668, 'unique_sentence2': 1024}, 'urd_Arab-kas_Arab': {'num_samples': 1024, 'number_of_characters': 316980, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 158.5, 'max_sentence2_length': 520, 'unique_sentence2': 1024}, 'urd_Arab-mai_Deva': {'num_samples': 1024, 'number_of_characters': 302898, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 144.75, 'max_sentence2_length': 562, 'unique_sentence2': 1024}, 'urd_Arab-mal_Mlym': {'num_samples': 1024, 'number_of_characters': 337852, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 13, 'average_sentence2_length': 178.88, 'max_sentence2_length': 692, 'unique_sentence2': 1024}, 'urd_Arab-mar_Deva': {'num_samples': 1024, 'number_of_characters': 316000, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 157.54, 'max_sentence2_length': 555, 'unique_sentence2': 1024}, 'urd_Arab-mni_Mtei': {'num_samples': 1024, 'number_of_characters': 307350, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 16, 'average_sentence2_length': 149.1, 'max_sentence2_length': 597, 'unique_sentence2': 1024}, 'urd_Arab-npi_Deva': {'num_samples': 1024, 'number_of_characters': 307635, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 149.38, 'max_sentence2_length': 525, 'unique_sentence2': 1024}, 'urd_Arab-ory_Orya': {'num_samples': 1024, 'number_of_characters': 328442, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 10, 'average_sentence2_length': 169.69, 'max_sentence2_length': 578, 'unique_sentence2': 1024}, 'urd_Arab-pan_Guru': {'num_samples': 1024, 'number_of_characters': 301079, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 19, 'average_sentence2_length': 142.97, 'max_sentence2_length': 476, 'unique_sentence2': 1024}, 'urd_Arab-san_Deva': {'num_samples': 1024, 'number_of_characters': 312295, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 9, 'average_sentence2_length': 153.93, 'max_sentence2_length': 601, 'unique_sentence2': 1024}, 'urd_Arab-sat_Olck': {'num_samples': 1024, 'number_of_characters': 320948, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 11, 'average_sentence2_length': 162.38, 'max_sentence2_length': 536, 'unique_sentence2': 1024}, 'urd_Arab-snd_Deva': {'num_samples': 1024, 'number_of_characters': 314637, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 18, 'average_sentence2_length': 156.21, 'max_sentence2_length': 545, 'unique_sentence2': 1024}, 'urd_Arab-tam_Taml': {'num_samples': 1024, 'number_of_characters': 342562, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 32, 'average_sentence2_length': 183.48, 'max_sentence2_length': 614, 'unique_sentence2': 1023}, 'urd_Arab-tel_Telu': {'num_samples': 1024, 'number_of_characters': 313261, 'unique_pairs': 1024, 'min_sentence1_length': 13, 'average_sentence1_length': 151.05, 'max_sentence1_length': 574, 'unique_sentence1': 1024, 'min_sentence2_length': 14, 'average_sentence2_length': 154.87, 'max_sentence2_length': 658, 'unique_sentence2': 1024}}}} | +| [IWSLT2017BitextMining](https://aclanthology.org/2017.iwslt-1.1/) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'ita', 'jpn', 'kor', 'nld', 'ron'] | BitextMining | s2s | [Fiction, Non-fiction, Written] | {'validation': 21938} | {'validation': {'num_samples': 21938, 'number_of_characters': 4256244, 'unique_pairs': 21840, 'min_sentence1_length': 2, 'average_sentence1_length': 97.01, 'max_sentence1_length': 521, 'unique_sentence1': 11563, 'min_sentence2_length': 2, 'average_sentence2_length': 97.01, 'max_sentence2_length': 521, 'unique_sentence2': 11563, 'hf_subset_descriptive_stats': {'ar-en': {'num_samples': 888, 'number_of_characters': 172499, 'unique_pairs': 887, 'min_sentence1_length': 4, 'average_sentence1_length': 85.49, 'max_sentence1_length': 369, 'unique_sentence1': 887, 'min_sentence2_length': 10, 'average_sentence2_length': 108.77, 'max_sentence2_length': 462, 'unique_sentence2': 881}, 'de-en': {'num_samples': 888, 'number_of_characters': 202336, 'unique_pairs': 883, 'min_sentence1_length': 6, 'average_sentence1_length': 119.03, 'max_sentence1_length': 521, 'unique_sentence1': 881, 'min_sentence2_length': 10, 'average_sentence2_length': 108.83, 'max_sentence2_length': 462, 'unique_sentence2': 881}, 'en-ar': {'num_samples': 888, 'number_of_characters': 172499, 'unique_pairs': 887, 'min_sentence1_length': 10, 'average_sentence1_length': 108.77, 'max_sentence1_length': 462, 'unique_sentence1': 881, 'min_sentence2_length': 4, 'average_sentence2_length': 85.49, 'max_sentence2_length': 369, 'unique_sentence2': 887}, 'en-de': {'num_samples': 888, 'number_of_characters': 202336, 'unique_pairs': 883, 'min_sentence1_length': 10, 'average_sentence1_length': 108.83, 'max_sentence1_length': 462, 'unique_sentence1': 881, 'min_sentence2_length': 6, 'average_sentence2_length': 119.03, 'max_sentence2_length': 521, 'unique_sentence2': 881}, 'en-fr': {'num_samples': 890, 'number_of_characters': 197619, 'unique_pairs': 883, 'min_sentence1_length': 10, 'average_sentence1_length': 108.41, 'max_sentence1_length': 462, 'unique_sentence1': 883, 'min_sentence2_length': 6, 'average_sentence2_length': 113.63, 'max_sentence2_length': 493, 'unique_sentence2': 881}, 'en-it': {'num_samples': 929, 'number_of_characters': 191803, 'unique_pairs': 924, 'min_sentence1_length': 10, 'average_sentence1_length': 103.0, 'max_sentence1_length': 433, 'unique_sentence1': 922, 'min_sentence2_length': 7, 'average_sentence2_length': 103.46, 'max_sentence2_length': 444, 'unique_sentence2': 918}, 'en-ja': {'num_samples': 871, 'number_of_characters': 132742, 'unique_pairs': 867, 'min_sentence1_length': 10, 'average_sentence1_length': 109.81, 'max_sentence1_length': 462, 'unique_sentence1': 864, 'min_sentence2_length': 5, 'average_sentence2_length': 42.59, 'max_sentence2_length': 225, 'unique_sentence2': 866}, 'en-ko': {'num_samples': 879, 'number_of_characters': 142659, 'unique_pairs': 874, 'min_sentence1_length': 10, 'average_sentence1_length': 107.74, 'max_sentence1_length': 462, 'unique_sentence1': 872, 'min_sentence2_length': 3, 'average_sentence2_length': 54.56, 'max_sentence2_length': 250, 'unique_sentence2': 872}, 'en-nl': {'num_samples': 1003, 'number_of_characters': 189637, 'unique_pairs': 1000, 'min_sentence1_length': 10, 'average_sentence1_length': 95.27, 'max_sentence1_length': 433, 'unique_sentence1': 996, 'min_sentence2_length': 4, 'average_sentence2_length': 93.8, 'max_sentence2_length': 477, 'unique_sentence2': 1000}, 'en-ro': {'num_samples': 914, 'number_of_characters': 194128, 'unique_pairs': 910, 'min_sentence1_length': 10, 'average_sentence1_length': 104.72, 'max_sentence1_length': 433, 'unique_sentence1': 907, 'min_sentence2_length': 9, 'average_sentence2_length': 107.67, 'max_sentence2_length': 448, 'unique_sentence2': 910}, 'en-zh': {'num_samples': 879, 'number_of_characters': 131126, 'unique_pairs': 877, 'min_sentence1_length': 10, 'average_sentence1_length': 109.37, 'max_sentence1_length': 462, 'unique_sentence1': 872, 'min_sentence2_length': 2, 'average_sentence2_length': 39.81, 'max_sentence2_length': 230, 'unique_sentence2': 867}, 'fr-en': {'num_samples': 890, 'number_of_characters': 197619, 'unique_pairs': 883, 'min_sentence1_length': 6, 'average_sentence1_length': 113.63, 'max_sentence1_length': 493, 'unique_sentence1': 881, 'min_sentence2_length': 10, 'average_sentence2_length': 108.41, 'max_sentence2_length': 462, 'unique_sentence2': 883}, 'it-en': {'num_samples': 929, 'number_of_characters': 191803, 'unique_pairs': 924, 'min_sentence1_length': 7, 'average_sentence1_length': 103.46, 'max_sentence1_length': 444, 'unique_sentence1': 918, 'min_sentence2_length': 10, 'average_sentence2_length': 103.0, 'max_sentence2_length': 433, 'unique_sentence2': 922}, 'it-nl': {'num_samples': 1001, 'number_of_characters': 188858, 'unique_pairs': 998, 'min_sentence1_length': 7, 'average_sentence1_length': 94.64, 'max_sentence1_length': 459, 'unique_sentence1': 994, 'min_sentence2_length': 7, 'average_sentence2_length': 94.03, 'max_sentence2_length': 505, 'unique_sentence2': 998}, 'it-ro': {'num_samples': 914, 'number_of_characters': 193339, 'unique_pairs': 911, 'min_sentence1_length': 7, 'average_sentence1_length': 103.91, 'max_sentence1_length': 435, 'unique_sentence1': 907, 'min_sentence2_length': 9, 'average_sentence2_length': 107.62, 'max_sentence2_length': 448, 'unique_sentence2': 910}, 'ja-en': {'num_samples': 871, 'number_of_characters': 132742, 'unique_pairs': 867, 'min_sentence1_length': 5, 'average_sentence1_length': 42.59, 'max_sentence1_length': 225, 'unique_sentence1': 866, 'min_sentence2_length': 10, 'average_sentence2_length': 109.81, 'max_sentence2_length': 462, 'unique_sentence2': 864}, 'ko-en': {'num_samples': 879, 'number_of_characters': 142659, 'unique_pairs': 874, 'min_sentence1_length': 3, 'average_sentence1_length': 54.56, 'max_sentence1_length': 250, 'unique_sentence1': 872, 'min_sentence2_length': 10, 'average_sentence2_length': 107.74, 'max_sentence2_length': 462, 'unique_sentence2': 872}, 'nl-en': {'num_samples': 1003, 'number_of_characters': 189637, 'unique_pairs': 1000, 'min_sentence1_length': 4, 'average_sentence1_length': 93.8, 'max_sentence1_length': 477, 'unique_sentence1': 1000, 'min_sentence2_length': 10, 'average_sentence2_length': 95.27, 'max_sentence2_length': 433, 'unique_sentence2': 996}, 'nl-it': {'num_samples': 1001, 'number_of_characters': 188858, 'unique_pairs': 998, 'min_sentence1_length': 7, 'average_sentence1_length': 94.03, 'max_sentence1_length': 505, 'unique_sentence1': 998, 'min_sentence2_length': 7, 'average_sentence2_length': 94.64, 'max_sentence2_length': 459, 'unique_sentence2': 994}, 'nl-ro': {'num_samples': 913, 'number_of_characters': 191376, 'unique_pairs': 911, 'min_sentence1_length': 7, 'average_sentence1_length': 102.02, 'max_sentence1_length': 478, 'unique_sentence1': 909, 'min_sentence2_length': 9, 'average_sentence2_length': 107.59, 'max_sentence2_length': 515, 'unique_sentence2': 909}, 'ro-en': {'num_samples': 914, 'number_of_characters': 194128, 'unique_pairs': 910, 'min_sentence1_length': 9, 'average_sentence1_length': 107.67, 'max_sentence1_length': 448, 'unique_sentence1': 910, 'min_sentence2_length': 10, 'average_sentence2_length': 104.72, 'max_sentence2_length': 433, 'unique_sentence2': 907}, 'ro-it': {'num_samples': 914, 'number_of_characters': 193339, 'unique_pairs': 911, 'min_sentence1_length': 9, 'average_sentence1_length': 107.62, 'max_sentence1_length': 448, 'unique_sentence1': 910, 'min_sentence2_length': 7, 'average_sentence2_length': 103.91, 'max_sentence2_length': 435, 'unique_sentence2': 907}, 'ro-nl': {'num_samples': 913, 'number_of_characters': 191376, 'unique_pairs': 911, 'min_sentence1_length': 9, 'average_sentence1_length': 107.59, 'max_sentence1_length': 515, 'unique_sentence1': 909, 'min_sentence2_length': 7, 'average_sentence2_length': 102.02, 'max_sentence2_length': 478, 'unique_sentence2': 909}, 'zh-en': {'num_samples': 879, 'number_of_characters': 131126, 'unique_pairs': 877, 'min_sentence1_length': 2, 'average_sentence1_length': 39.81, 'max_sentence1_length': 230, 'unique_sentence1': 867, 'min_sentence2_length': 10, 'average_sentence2_length': 109.37, 'max_sentence2_length': 462, 'unique_sentence2': 872}}}} | +| [ImageCoDeT2IMultiChoice](https://aclanthology.org/2022.acl-long.241.pdf) (Krojer et al., 2022) | ['eng'] | Any2AnyMultiChoice | it2i | [Web, Written] | None | None | +| [ImageCoDeT2IRetrieval](https://aclanthology.org/2022.acl-long.241.pdf) (Krojer et al., 2022) | ['eng'] | Any2AnyRetrieval | t2i | [Web, Written] | None | None | +| [ImageNet10Clustering](https://www.kaggle.com/datasets/liusha249/imagenet10) (Deng et al., 2009) | ['eng'] | ImageClustering | i2t | [Web] | None | None | +| [ImageNetDog15Clustering](http://vision.stanford.edu/aditya86/ImageNetDogs/main.html) (Deng et al., 2009) | ['eng'] | ImageClustering | i2i | [Web] | None | None | +| [Imagenet1k](https://ieeexplore.ieee.org/document/5206848) (Deng et al., 2009) | ['eng'] | ImageClassification | i2i | [Scene] | None | None | +| [Imagenet1kZeroShot](https://ieeexplore.ieee.org/document/5206848) (Deng et al., 2009) | ['eng'] | ZeroShotClassification | i2t | [Scene] | None | None | | [ImdbClassification](http://www.aclweb.org/anthology/P11-1015) | ['eng'] | Classification | p2p | [Reviews, Written] | None | None | -| [InappropriatenessClassification](https://aclanthology.org/2021.bsnlp-1.4) | ['rus'] | Classification | s2s | [Web, Social, Written] | None | None | -| [IndicCrosslingualSTS](https://huggingface.co/datasets/jaygala24/indic_sts) (Ramesh et al., 2022) | ['asm', 'ben', 'eng', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel', 'urd'] | STS | s2s | [News, Non-fiction, Web, Spoken, Government, Written, Spoken] | None | None | -| [IndicGenBenchFloresBitextMining](https://github.com/google-research-datasets/indic-gen-bench/) (Harman Singh, 2024) | ['asm', 'awa', 'ben', 'bgc', 'bho', 'bod', 'boy', 'eng', 'gbm', 'gom', 'guj', 'hin', 'hne', 'kan', 'mai', 'mal', 'mar', 'mni', 'mup', 'mwr', 'nep', 'ory', 'pan', 'pus', 'raj', 'san', 'sat', 'tam', 'tel', 'urd'] | BitextMining | s2s | [Web, News, Written] | {'validation': 57826, 'test': 58696} | {'validation': {'num_samples': 57826, 'number_of_characters': 14600950, 'unique_pairs': 57826, 'min_sentence1_length': 24, 'average_sentence1_length': 126.25, 'max_sentence1_length': 368, 'unique_sentence1': 29903, 'min_sentence2_length': 24, 'average_sentence2_length': 126.24, 'max_sentence2_length': 368, 'unique_sentence2': 29903, 'hf_subset_descriptive_stats': {'ben-eng': {'num_samples': 997, 'number_of_characters': 248469, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 123.65, 'max_sentence1_length': 320, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-ben': {'num_samples': 997, 'number_of_characters': 248469, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 123.65, 'max_sentence2_length': 320, 'unique_sentence2': 997}, 'guj-eng': {'num_samples': 997, 'number_of_characters': 245477, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 120.64, 'max_sentence1_length': 368, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-guj': {'num_samples': 997, 'number_of_characters': 245477, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 120.64, 'max_sentence2_length': 368, 'unique_sentence2': 997}, 'hin-eng': {'num_samples': 997, 'number_of_characters': 250573, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 125.76, 'max_sentence1_length': 355, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-hin': {'num_samples': 997, 'number_of_characters': 250564, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 125.75, 'max_sentence2_length': 355, 'unique_sentence2': 997}, 'kan-eng': {'num_samples': 997, 'number_of_characters': 257131, 'unique_pairs': 997, 'min_sentence1_length': 34, 'average_sentence1_length': 132.33, 'max_sentence1_length': 331, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-kan': {'num_samples': 997, 'number_of_characters': 256986, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 34, 'average_sentence2_length': 132.19, 'max_sentence2_length': 331, 'unique_sentence2': 997}, 'mal-eng': {'num_samples': 997, 'number_of_characters': 267295, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 142.53, 'max_sentence1_length': 360, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mal': {'num_samples': 997, 'number_of_characters': 267296, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 142.53, 'max_sentence2_length': 360, 'unique_sentence2': 997}, 'mar-eng': {'num_samples': 997, 'number_of_characters': 251107, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 126.29, 'max_sentence1_length': 321, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mar': {'num_samples': 997, 'number_of_characters': 250897, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 126.08, 'max_sentence2_length': 321, 'unique_sentence2': 997}, 'tam-eng': {'num_samples': 997, 'number_of_characters': 271322, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 146.57, 'max_sentence1_length': 358, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-tam': {'num_samples': 997, 'number_of_characters': 271322, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 146.57, 'max_sentence2_length': 358, 'unique_sentence2': 997}, 'tel-eng': {'num_samples': 997, 'number_of_characters': 252385, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 127.57, 'max_sentence1_length': 317, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-tel': {'num_samples': 997, 'number_of_characters': 252380, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 127.57, 'max_sentence2_length': 317, 'unique_sentence2': 997}, 'urd-eng': {'num_samples': 997, 'number_of_characters': 249824, 'unique_pairs': 997, 'min_sentence1_length': 37, 'average_sentence1_length': 125.01, 'max_sentence1_length': 295, 'unique_sentence1': 996, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-urd': {'num_samples': 997, 'number_of_characters': 249824, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 37, 'average_sentence2_length': 125.01, 'max_sentence2_length': 295, 'unique_sentence2': 996}, 'asm-eng': {'num_samples': 997, 'number_of_characters': 246220, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 121.39, 'max_sentence1_length': 314, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-asm': {'num_samples': 997, 'number_of_characters': 246224, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 121.39, 'max_sentence2_length': 314, 'unique_sentence2': 997}, 'bho-eng': {'num_samples': 997, 'number_of_characters': 246895, 'unique_pairs': 997, 'min_sentence1_length': 25, 'average_sentence1_length': 122.07, 'max_sentence1_length': 326, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-bho': {'num_samples': 997, 'number_of_characters': 246919, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 25, 'average_sentence2_length': 122.09, 'max_sentence2_length': 326, 'unique_sentence2': 997}, 'nep-eng': {'num_samples': 997, 'number_of_characters': 245984, 'unique_pairs': 997, 'min_sentence1_length': 24, 'average_sentence1_length': 121.15, 'max_sentence1_length': 307, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-nep': {'num_samples': 997, 'number_of_characters': 245984, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 24, 'average_sentence2_length': 121.15, 'max_sentence2_length': 307, 'unique_sentence2': 997}, 'ory-eng': {'num_samples': 997, 'number_of_characters': 254206, 'unique_pairs': 997, 'min_sentence1_length': 34, 'average_sentence1_length': 129.4, 'max_sentence1_length': 308, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-ory': {'num_samples': 997, 'number_of_characters': 254206, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 34, 'average_sentence2_length': 129.4, 'max_sentence2_length': 308, 'unique_sentence2': 997}, 'pan-eng': {'num_samples': 997, 'number_of_characters': 251598, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 126.78, 'max_sentence1_length': 309, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-pan': {'num_samples': 997, 'number_of_characters': 251597, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 126.78, 'max_sentence2_length': 309, 'unique_sentence2': 997}, 'pus-eng': {'num_samples': 997, 'number_of_characters': 247450, 'unique_pairs': 997, 'min_sentence1_length': 32, 'average_sentence1_length': 122.62, 'max_sentence1_length': 300, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-pus': {'num_samples': 997, 'number_of_characters': 247450, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 32, 'average_sentence2_length': 122.62, 'max_sentence2_length': 300, 'unique_sentence2': 997}, 'san-eng': {'num_samples': 997, 'number_of_characters': 249042, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 124.22, 'max_sentence1_length': 311, 'unique_sentence1': 994, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-san': {'num_samples': 997, 'number_of_characters': 248877, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 124.06, 'max_sentence2_length': 311, 'unique_sentence2': 994}, 'awa-eng': {'num_samples': 997, 'number_of_characters': 247944, 'unique_pairs': 997, 'min_sentence1_length': 34, 'average_sentence1_length': 123.12, 'max_sentence1_length': 329, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-awa': {'num_samples': 997, 'number_of_characters': 247884, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 34, 'average_sentence2_length': 123.06, 'max_sentence2_length': 329, 'unique_sentence2': 997}, 'bgc-eng': {'num_samples': 997, 'number_of_characters': 245935, 'unique_pairs': 997, 'min_sentence1_length': 27, 'average_sentence1_length': 121.1, 'max_sentence1_length': 303, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-bgc': {'num_samples': 997, 'number_of_characters': 245935, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 27, 'average_sentence2_length': 121.1, 'max_sentence2_length': 303, 'unique_sentence2': 997}, 'bod-eng': {'num_samples': 997, 'number_of_characters': 266515, 'unique_pairs': 997, 'min_sentence1_length': 26, 'average_sentence1_length': 141.75, 'max_sentence1_length': 355, 'unique_sentence1': 996, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-bod': {'num_samples': 997, 'number_of_characters': 266495, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 26, 'average_sentence2_length': 141.73, 'max_sentence2_length': 355, 'unique_sentence2': 996}, 'boy-eng': {'num_samples': 997, 'number_of_characters': 260174, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 135.39, 'max_sentence1_length': 312, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-boy': {'num_samples': 997, 'number_of_characters': 260174, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 135.39, 'max_sentence2_length': 312, 'unique_sentence2': 997}, 'gbm-eng': {'num_samples': 997, 'number_of_characters': 247009, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 122.18, 'max_sentence1_length': 344, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-gbm': {'num_samples': 997, 'number_of_characters': 247009, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 122.18, 'max_sentence2_length': 344, 'unique_sentence2': 997}, 'gom-eng': {'num_samples': 997, 'number_of_characters': 244553, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 119.72, 'max_sentence1_length': 306, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-gom': {'num_samples': 997, 'number_of_characters': 244553, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 119.72, 'max_sentence2_length': 306, 'unique_sentence2': 997}, 'hne-eng': {'num_samples': 997, 'number_of_characters': 246416, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 121.59, 'max_sentence1_length': 321, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-hne': {'num_samples': 997, 'number_of_characters': 246405, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 121.58, 'max_sentence2_length': 321, 'unique_sentence2': 997}, 'raj-eng': {'num_samples': 997, 'number_of_characters': 249541, 'unique_pairs': 997, 'min_sentence1_length': 32, 'average_sentence1_length': 124.72, 'max_sentence1_length': 313, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-raj': {'num_samples': 997, 'number_of_characters': 249541, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 32, 'average_sentence2_length': 124.72, 'max_sentence2_length': 313, 'unique_sentence2': 997}, 'mai-eng': {'num_samples': 997, 'number_of_characters': 247991, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 123.17, 'max_sentence1_length': 312, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mai': {'num_samples': 997, 'number_of_characters': 247994, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 123.17, 'max_sentence2_length': 312, 'unique_sentence2': 997}, 'mni-eng': {'num_samples': 997, 'number_of_characters': 254308, 'unique_pairs': 997, 'min_sentence1_length': 39, 'average_sentence1_length': 129.5, 'max_sentence1_length': 310, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mni': {'num_samples': 997, 'number_of_characters': 254312, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 39, 'average_sentence2_length': 129.51, 'max_sentence2_length': 310, 'unique_sentence2': 997}, 'mup-eng': {'num_samples': 997, 'number_of_characters': 248486, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 123.66, 'max_sentence1_length': 312, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mup': {'num_samples': 997, 'number_of_characters': 248486, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 123.66, 'max_sentence2_length': 312, 'unique_sentence2': 997}, 'mwr-eng': {'num_samples': 997, 'number_of_characters': 248641, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 123.82, 'max_sentence1_length': 324, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mwr': {'num_samples': 997, 'number_of_characters': 248641, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 123.82, 'max_sentence2_length': 324, 'unique_sentence2': 997}, 'sat-eng': {'num_samples': 997, 'number_of_characters': 258279, 'unique_pairs': 997, 'min_sentence1_length': 37, 'average_sentence1_length': 133.49, 'max_sentence1_length': 333, 'unique_sentence1': 995, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-sat': {'num_samples': 997, 'number_of_characters': 258279, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 37, 'average_sentence2_length': 133.49, 'max_sentence2_length': 333, 'unique_sentence2': 995}}}, 'test': {'num_samples': 58696, 'number_of_characters': 15359416, 'unique_pairs': 58690, 'min_sentence1_length': 33, 'average_sentence1_length': 130.84, 'max_sentence1_length': 431, 'unique_sentence1': 30351, 'min_sentence2_length': 33, 'average_sentence2_length': 130.83, 'max_sentence2_length': 431, 'unique_sentence2': 30351, 'hf_subset_descriptive_stats': {'ben-eng': {'num_samples': 1012, 'number_of_characters': 261008, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 127.51, 'max_sentence1_length': 333, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-ben': {'num_samples': 1012, 'number_of_characters': 261008, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 127.51, 'max_sentence2_length': 333, 'unique_sentence2': 1012}, 'guj-eng': {'num_samples': 1012, 'number_of_characters': 258394, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 124.93, 'max_sentence1_length': 349, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-guj': {'num_samples': 1012, 'number_of_characters': 258394, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 124.93, 'max_sentence2_length': 349, 'unique_sentence2': 1012}, 'hin-eng': {'num_samples': 1012, 'number_of_characters': 263040, 'unique_pairs': 1012, 'min_sentence1_length': 41, 'average_sentence1_length': 129.52, 'max_sentence1_length': 381, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-hin': {'num_samples': 1012, 'number_of_characters': 263029, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 41, 'average_sentence2_length': 129.51, 'max_sentence2_length': 381, 'unique_sentence2': 1012}, 'kan-eng': {'num_samples': 1012, 'number_of_characters': 270091, 'unique_pairs': 1012, 'min_sentence1_length': 43, 'average_sentence1_length': 136.49, 'max_sentence1_length': 388, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-kan': {'num_samples': 1012, 'number_of_characters': 270021, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 43, 'average_sentence2_length': 136.42, 'max_sentence2_length': 388, 'unique_sentence2': 1012}, 'mal-eng': {'num_samples': 1012, 'number_of_characters': 281302, 'unique_pairs': 1012, 'min_sentence1_length': 48, 'average_sentence1_length': 147.57, 'max_sentence1_length': 376, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mal': {'num_samples': 1012, 'number_of_characters': 281302, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 48, 'average_sentence2_length': 147.57, 'max_sentence2_length': 376, 'unique_sentence2': 1012}, 'mar-eng': {'num_samples': 1012, 'number_of_characters': 265212, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 131.67, 'max_sentence1_length': 356, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mar': {'num_samples': 1012, 'number_of_characters': 265023, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 131.48, 'max_sentence2_length': 355, 'unique_sentence2': 1012}, 'tam-eng': {'num_samples': 1012, 'number_of_characters': 286099, 'unique_pairs': 1012, 'min_sentence1_length': 48, 'average_sentence1_length': 152.31, 'max_sentence1_length': 404, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-tam': {'num_samples': 1012, 'number_of_characters': 286099, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 48, 'average_sentence2_length': 152.31, 'max_sentence2_length': 404, 'unique_sentence2': 1012}, 'tel-eng': {'num_samples': 1012, 'number_of_characters': 264460, 'unique_pairs': 1012, 'min_sentence1_length': 39, 'average_sentence1_length': 130.92, 'max_sentence1_length': 359, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-tel': {'num_samples': 1012, 'number_of_characters': 264447, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 39, 'average_sentence2_length': 130.91, 'max_sentence2_length': 359, 'unique_sentence2': 1012}, 'urd-eng': {'num_samples': 1012, 'number_of_characters': 261886, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 128.38, 'max_sentence1_length': 348, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-urd': {'num_samples': 1012, 'number_of_characters': 261885, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 128.38, 'max_sentence2_length': 348, 'unique_sentence2': 1012}, 'asm-eng': {'num_samples': 1012, 'number_of_characters': 257902, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 124.44, 'max_sentence1_length': 329, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-asm': {'num_samples': 1012, 'number_of_characters': 257909, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 124.45, 'max_sentence2_length': 329, 'unique_sentence2': 1012}, 'bho-eng': {'num_samples': 1012, 'number_of_characters': 260578, 'unique_pairs': 1012, 'min_sentence1_length': 36, 'average_sentence1_length': 127.09, 'max_sentence1_length': 367, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-bho': {'num_samples': 1012, 'number_of_characters': 260601, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 36, 'average_sentence2_length': 127.11, 'max_sentence2_length': 367, 'unique_sentence2': 1012}, 'nep-eng': {'num_samples': 1012, 'number_of_characters': 258869, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 125.4, 'max_sentence1_length': 362, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-nep': {'num_samples': 1012, 'number_of_characters': 258869, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 125.4, 'max_sentence2_length': 362, 'unique_sentence2': 1012}, 'ory-eng': {'num_samples': 1012, 'number_of_characters': 266805, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 133.24, 'max_sentence1_length': 354, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-ory': {'num_samples': 1012, 'number_of_characters': 266805, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 133.24, 'max_sentence2_length': 354, 'unique_sentence2': 1012}, 'pan-eng': {'num_samples': 1012, 'number_of_characters': 265391, 'unique_pairs': 1012, 'min_sentence1_length': 37, 'average_sentence1_length': 131.84, 'max_sentence1_length': 380, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-pan': {'num_samples': 1012, 'number_of_characters': 265391, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 37, 'average_sentence2_length': 131.84, 'max_sentence2_length': 380, 'unique_sentence2': 1012}, 'pus-eng': {'num_samples': 1012, 'number_of_characters': 254422, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 121.0, 'max_sentence1_length': 325, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-pus': {'num_samples': 1012, 'number_of_characters': 254421, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 121.0, 'max_sentence2_length': 325, 'unique_sentence2': 1012}, 'san-eng': {'num_samples': 1012, 'number_of_characters': 260339, 'unique_pairs': 1012, 'min_sentence1_length': 33, 'average_sentence1_length': 126.85, 'max_sentence1_length': 358, 'unique_sentence1': 1011, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-san': {'num_samples': 1012, 'number_of_characters': 260224, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 33, 'average_sentence2_length': 126.74, 'max_sentence2_length': 358, 'unique_sentence2': 1011}, 'awa-eng': {'num_samples': 1012, 'number_of_characters': 260179, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 126.69, 'max_sentence1_length': 378, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-awa': {'num_samples': 1012, 'number_of_characters': 260137, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 126.65, 'max_sentence2_length': 378, 'unique_sentence2': 1012}, 'bgc-eng': {'num_samples': 1012, 'number_of_characters': 257450, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 124.0, 'max_sentence1_length': 332, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-bgc': {'num_samples': 1012, 'number_of_characters': 257450, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 124.0, 'max_sentence2_length': 332, 'unique_sentence2': 1012}, 'bod-eng': {'num_samples': 1012, 'number_of_characters': 280188, 'unique_pairs': 1012, 'min_sentence1_length': 42, 'average_sentence1_length': 146.46, 'max_sentence1_length': 431, 'unique_sentence1': 1009, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-bod': {'num_samples': 1012, 'number_of_characters': 280126, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 42, 'average_sentence2_length': 146.4, 'max_sentence2_length': 431, 'unique_sentence2': 1009}, 'boy-eng': {'num_samples': 1012, 'number_of_characters': 277538, 'unique_pairs': 1012, 'min_sentence1_length': 36, 'average_sentence1_length': 143.85, 'max_sentence1_length': 396, 'unique_sentence1': 1011, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-boy': {'num_samples': 1012, 'number_of_characters': 277538, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 36, 'average_sentence2_length': 143.85, 'max_sentence2_length': 396, 'unique_sentence2': 1011}, 'gbm-eng': {'num_samples': 1012, 'number_of_characters': 261027, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 127.53, 'max_sentence1_length': 333, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-gbm': {'num_samples': 1012, 'number_of_characters': 261027, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 127.53, 'max_sentence2_length': 333, 'unique_sentence2': 1012}, 'gom-eng': {'num_samples': 1012, 'number_of_characters': 259182, 'unique_pairs': 1012, 'min_sentence1_length': 37, 'average_sentence1_length': 125.71, 'max_sentence1_length': 335, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-gom': {'num_samples': 1012, 'number_of_characters': 259182, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 37, 'average_sentence2_length': 125.71, 'max_sentence2_length': 335, 'unique_sentence2': 1012}, 'hne-eng': {'num_samples': 1012, 'number_of_characters': 258911, 'unique_pairs': 1012, 'min_sentence1_length': 42, 'average_sentence1_length': 125.44, 'max_sentence1_length': 327, 'unique_sentence1': 1011, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-hne': {'num_samples': 1012, 'number_of_characters': 258915, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 42, 'average_sentence2_length': 125.44, 'max_sentence2_length': 326, 'unique_sentence2': 1011}, 'raj-eng': {'num_samples': 1012, 'number_of_characters': 261987, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 128.48, 'max_sentence1_length': 338, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-raj': {'num_samples': 1012, 'number_of_characters': 261987, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 128.48, 'max_sentence2_length': 338, 'unique_sentence2': 1012}, 'mai-eng': {'num_samples': 1012, 'number_of_characters': 261374, 'unique_pairs': 1012, 'min_sentence1_length': 36, 'average_sentence1_length': 127.87, 'max_sentence1_length': 350, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mai': {'num_samples': 1012, 'number_of_characters': 261377, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 36, 'average_sentence2_length': 127.88, 'max_sentence2_length': 350, 'unique_sentence2': 1012}, 'mni-eng': {'num_samples': 1012, 'number_of_characters': 268767, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 135.18, 'max_sentence1_length': 353, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mni': {'num_samples': 1012, 'number_of_characters': 268768, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 135.18, 'max_sentence2_length': 354, 'unique_sentence2': 1012}, 'mup-eng': {'num_samples': 1012, 'number_of_characters': 262034, 'unique_pairs': 1012, 'min_sentence1_length': 40, 'average_sentence1_length': 128.53, 'max_sentence1_length': 340, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mup': {'num_samples': 1012, 'number_of_characters': 262034, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 40, 'average_sentence2_length': 128.53, 'max_sentence2_length': 340, 'unique_sentence2': 1012}, 'mwr-eng': {'num_samples': 1012, 'number_of_characters': 263749, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.22, 'max_sentence1_length': 345, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mwr': {'num_samples': 1012, 'number_of_characters': 263749, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.22, 'max_sentence2_length': 345, 'unique_sentence2': 1012}, 'sat-eng': {'num_samples': 1012, 'number_of_characters': 271757, 'unique_pairs': 1012, 'min_sentence1_length': 43, 'average_sentence1_length': 138.13, 'max_sentence1_length': 366, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-sat': {'num_samples': 1012, 'number_of_characters': 271757, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 43, 'average_sentence2_length': 138.13, 'max_sentence2_length': 366, 'unique_sentence2': 1012}}}} | -| [IndicLangClassification](https://arxiv.org/abs/2305.15814) | ['asm', 'ben', 'brx', 'doi', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | Classification | s2s | [Web, Non-fiction, Written] | None | None | +| [InappropriatenessClassification](https://aclanthology.org/2021.bsnlp-1.4) | ['rus'] | Classification | s2s | [Social, Web, Written] | None | None | +| [IndicCrosslingualSTS](https://huggingface.co/datasets/jaygala24/indic_sts) (Ramesh et al., 2022) | ['asm', 'ben', 'eng', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel', 'urd'] | STS | s2s | [Government, News, Non-fiction, Spoken, Spoken, Web, Written] | None | None | +| [IndicGenBenchFloresBitextMining](https://github.com/google-research-datasets/indic-gen-bench/) (Harman Singh, 2024) | ['asm', 'awa', 'ben', 'bgc', 'bho', 'bod', 'boy', 'eng', 'gbm', 'gom', 'guj', 'hin', 'hne', 'kan', 'mai', 'mal', 'mar', 'mni', 'mup', 'mwr', 'nep', 'ory', 'pan', 'pus', 'raj', 'san', 'sat', 'tam', 'tel', 'urd'] | BitextMining | s2s | [News, Web, Written] | {'validation': 57826, 'test': 58696} | {'validation': {'num_samples': 57826, 'number_of_characters': 14600950, 'unique_pairs': 57826, 'min_sentence1_length': 24, 'average_sentence1_length': 126.25, 'max_sentence1_length': 368, 'unique_sentence1': 29903, 'min_sentence2_length': 24, 'average_sentence2_length': 126.24, 'max_sentence2_length': 368, 'unique_sentence2': 29903, 'hf_subset_descriptive_stats': {'ben-eng': {'num_samples': 997, 'number_of_characters': 248469, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 123.65, 'max_sentence1_length': 320, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-ben': {'num_samples': 997, 'number_of_characters': 248469, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 123.65, 'max_sentence2_length': 320, 'unique_sentence2': 997}, 'guj-eng': {'num_samples': 997, 'number_of_characters': 245477, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 120.64, 'max_sentence1_length': 368, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-guj': {'num_samples': 997, 'number_of_characters': 245477, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 120.64, 'max_sentence2_length': 368, 'unique_sentence2': 997}, 'hin-eng': {'num_samples': 997, 'number_of_characters': 250573, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 125.76, 'max_sentence1_length': 355, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-hin': {'num_samples': 997, 'number_of_characters': 250564, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 125.75, 'max_sentence2_length': 355, 'unique_sentence2': 997}, 'kan-eng': {'num_samples': 997, 'number_of_characters': 257131, 'unique_pairs': 997, 'min_sentence1_length': 34, 'average_sentence1_length': 132.33, 'max_sentence1_length': 331, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-kan': {'num_samples': 997, 'number_of_characters': 256986, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 34, 'average_sentence2_length': 132.19, 'max_sentence2_length': 331, 'unique_sentence2': 997}, 'mal-eng': {'num_samples': 997, 'number_of_characters': 267295, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 142.53, 'max_sentence1_length': 360, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mal': {'num_samples': 997, 'number_of_characters': 267296, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 142.53, 'max_sentence2_length': 360, 'unique_sentence2': 997}, 'mar-eng': {'num_samples': 997, 'number_of_characters': 251107, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 126.29, 'max_sentence1_length': 321, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mar': {'num_samples': 997, 'number_of_characters': 250897, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 126.08, 'max_sentence2_length': 321, 'unique_sentence2': 997}, 'tam-eng': {'num_samples': 997, 'number_of_characters': 271322, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 146.57, 'max_sentence1_length': 358, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-tam': {'num_samples': 997, 'number_of_characters': 271322, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 146.57, 'max_sentence2_length': 358, 'unique_sentence2': 997}, 'tel-eng': {'num_samples': 997, 'number_of_characters': 252385, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 127.57, 'max_sentence1_length': 317, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-tel': {'num_samples': 997, 'number_of_characters': 252380, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 127.57, 'max_sentence2_length': 317, 'unique_sentence2': 997}, 'urd-eng': {'num_samples': 997, 'number_of_characters': 249824, 'unique_pairs': 997, 'min_sentence1_length': 37, 'average_sentence1_length': 125.01, 'max_sentence1_length': 295, 'unique_sentence1': 996, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-urd': {'num_samples': 997, 'number_of_characters': 249824, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 37, 'average_sentence2_length': 125.01, 'max_sentence2_length': 295, 'unique_sentence2': 996}, 'asm-eng': {'num_samples': 997, 'number_of_characters': 246220, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 121.39, 'max_sentence1_length': 314, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-asm': {'num_samples': 997, 'number_of_characters': 246224, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 121.39, 'max_sentence2_length': 314, 'unique_sentence2': 997}, 'bho-eng': {'num_samples': 997, 'number_of_characters': 246895, 'unique_pairs': 997, 'min_sentence1_length': 25, 'average_sentence1_length': 122.07, 'max_sentence1_length': 326, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-bho': {'num_samples': 997, 'number_of_characters': 246919, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 25, 'average_sentence2_length': 122.09, 'max_sentence2_length': 326, 'unique_sentence2': 997}, 'nep-eng': {'num_samples': 997, 'number_of_characters': 245984, 'unique_pairs': 997, 'min_sentence1_length': 24, 'average_sentence1_length': 121.15, 'max_sentence1_length': 307, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-nep': {'num_samples': 997, 'number_of_characters': 245984, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 24, 'average_sentence2_length': 121.15, 'max_sentence2_length': 307, 'unique_sentence2': 997}, 'ory-eng': {'num_samples': 997, 'number_of_characters': 254206, 'unique_pairs': 997, 'min_sentence1_length': 34, 'average_sentence1_length': 129.4, 'max_sentence1_length': 308, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-ory': {'num_samples': 997, 'number_of_characters': 254206, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 34, 'average_sentence2_length': 129.4, 'max_sentence2_length': 308, 'unique_sentence2': 997}, 'pan-eng': {'num_samples': 997, 'number_of_characters': 251598, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 126.78, 'max_sentence1_length': 309, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-pan': {'num_samples': 997, 'number_of_characters': 251597, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 126.78, 'max_sentence2_length': 309, 'unique_sentence2': 997}, 'pus-eng': {'num_samples': 997, 'number_of_characters': 247450, 'unique_pairs': 997, 'min_sentence1_length': 32, 'average_sentence1_length': 122.62, 'max_sentence1_length': 300, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-pus': {'num_samples': 997, 'number_of_characters': 247450, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 32, 'average_sentence2_length': 122.62, 'max_sentence2_length': 300, 'unique_sentence2': 997}, 'san-eng': {'num_samples': 997, 'number_of_characters': 249042, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 124.22, 'max_sentence1_length': 311, 'unique_sentence1': 994, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-san': {'num_samples': 997, 'number_of_characters': 248877, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 124.06, 'max_sentence2_length': 311, 'unique_sentence2': 994}, 'awa-eng': {'num_samples': 997, 'number_of_characters': 247944, 'unique_pairs': 997, 'min_sentence1_length': 34, 'average_sentence1_length': 123.12, 'max_sentence1_length': 329, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-awa': {'num_samples': 997, 'number_of_characters': 247884, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 34, 'average_sentence2_length': 123.06, 'max_sentence2_length': 329, 'unique_sentence2': 997}, 'bgc-eng': {'num_samples': 997, 'number_of_characters': 245935, 'unique_pairs': 997, 'min_sentence1_length': 27, 'average_sentence1_length': 121.1, 'max_sentence1_length': 303, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-bgc': {'num_samples': 997, 'number_of_characters': 245935, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 27, 'average_sentence2_length': 121.1, 'max_sentence2_length': 303, 'unique_sentence2': 997}, 'bod-eng': {'num_samples': 997, 'number_of_characters': 266515, 'unique_pairs': 997, 'min_sentence1_length': 26, 'average_sentence1_length': 141.75, 'max_sentence1_length': 355, 'unique_sentence1': 996, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-bod': {'num_samples': 997, 'number_of_characters': 266495, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 26, 'average_sentence2_length': 141.73, 'max_sentence2_length': 355, 'unique_sentence2': 996}, 'boy-eng': {'num_samples': 997, 'number_of_characters': 260174, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 135.39, 'max_sentence1_length': 312, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-boy': {'num_samples': 997, 'number_of_characters': 260174, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 135.39, 'max_sentence2_length': 312, 'unique_sentence2': 997}, 'gbm-eng': {'num_samples': 997, 'number_of_characters': 247009, 'unique_pairs': 997, 'min_sentence1_length': 30, 'average_sentence1_length': 122.18, 'max_sentence1_length': 344, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-gbm': {'num_samples': 997, 'number_of_characters': 247009, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 30, 'average_sentence2_length': 122.18, 'max_sentence2_length': 344, 'unique_sentence2': 997}, 'gom-eng': {'num_samples': 997, 'number_of_characters': 244553, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 119.72, 'max_sentence1_length': 306, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-gom': {'num_samples': 997, 'number_of_characters': 244553, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 119.72, 'max_sentence2_length': 306, 'unique_sentence2': 997}, 'hne-eng': {'num_samples': 997, 'number_of_characters': 246416, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 121.59, 'max_sentence1_length': 321, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-hne': {'num_samples': 997, 'number_of_characters': 246405, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 121.58, 'max_sentence2_length': 321, 'unique_sentence2': 997}, 'raj-eng': {'num_samples': 997, 'number_of_characters': 249541, 'unique_pairs': 997, 'min_sentence1_length': 32, 'average_sentence1_length': 124.72, 'max_sentence1_length': 313, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-raj': {'num_samples': 997, 'number_of_characters': 249541, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 32, 'average_sentence2_length': 124.72, 'max_sentence2_length': 313, 'unique_sentence2': 997}, 'mai-eng': {'num_samples': 997, 'number_of_characters': 247991, 'unique_pairs': 997, 'min_sentence1_length': 29, 'average_sentence1_length': 123.17, 'max_sentence1_length': 312, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mai': {'num_samples': 997, 'number_of_characters': 247994, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 29, 'average_sentence2_length': 123.17, 'max_sentence2_length': 312, 'unique_sentence2': 997}, 'mni-eng': {'num_samples': 997, 'number_of_characters': 254308, 'unique_pairs': 997, 'min_sentence1_length': 39, 'average_sentence1_length': 129.5, 'max_sentence1_length': 310, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mni': {'num_samples': 997, 'number_of_characters': 254312, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 39, 'average_sentence2_length': 129.51, 'max_sentence2_length': 310, 'unique_sentence2': 997}, 'mup-eng': {'num_samples': 997, 'number_of_characters': 248486, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 123.66, 'max_sentence1_length': 312, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mup': {'num_samples': 997, 'number_of_characters': 248486, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 123.66, 'max_sentence2_length': 312, 'unique_sentence2': 997}, 'mwr-eng': {'num_samples': 997, 'number_of_characters': 248641, 'unique_pairs': 997, 'min_sentence1_length': 31, 'average_sentence1_length': 123.82, 'max_sentence1_length': 324, 'unique_sentence1': 997, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-mwr': {'num_samples': 997, 'number_of_characters': 248641, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 31, 'average_sentence2_length': 123.82, 'max_sentence2_length': 324, 'unique_sentence2': 997}, 'sat-eng': {'num_samples': 997, 'number_of_characters': 258279, 'unique_pairs': 997, 'min_sentence1_length': 37, 'average_sentence1_length': 133.49, 'max_sentence1_length': 333, 'unique_sentence1': 995, 'min_sentence2_length': 28, 'average_sentence2_length': 125.57, 'max_sentence2_length': 297, 'unique_sentence2': 997}, 'eng-sat': {'num_samples': 997, 'number_of_characters': 258279, 'unique_pairs': 997, 'min_sentence1_length': 28, 'average_sentence1_length': 125.57, 'max_sentence1_length': 297, 'unique_sentence1': 997, 'min_sentence2_length': 37, 'average_sentence2_length': 133.49, 'max_sentence2_length': 333, 'unique_sentence2': 995}}}, 'test': {'num_samples': 58696, 'number_of_characters': 15359416, 'unique_pairs': 58690, 'min_sentence1_length': 33, 'average_sentence1_length': 130.84, 'max_sentence1_length': 431, 'unique_sentence1': 30351, 'min_sentence2_length': 33, 'average_sentence2_length': 130.83, 'max_sentence2_length': 431, 'unique_sentence2': 30351, 'hf_subset_descriptive_stats': {'ben-eng': {'num_samples': 1012, 'number_of_characters': 261008, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 127.51, 'max_sentence1_length': 333, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-ben': {'num_samples': 1012, 'number_of_characters': 261008, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 127.51, 'max_sentence2_length': 333, 'unique_sentence2': 1012}, 'guj-eng': {'num_samples': 1012, 'number_of_characters': 258394, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 124.93, 'max_sentence1_length': 349, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-guj': {'num_samples': 1012, 'number_of_characters': 258394, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 124.93, 'max_sentence2_length': 349, 'unique_sentence2': 1012}, 'hin-eng': {'num_samples': 1012, 'number_of_characters': 263040, 'unique_pairs': 1012, 'min_sentence1_length': 41, 'average_sentence1_length': 129.52, 'max_sentence1_length': 381, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-hin': {'num_samples': 1012, 'number_of_characters': 263029, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 41, 'average_sentence2_length': 129.51, 'max_sentence2_length': 381, 'unique_sentence2': 1012}, 'kan-eng': {'num_samples': 1012, 'number_of_characters': 270091, 'unique_pairs': 1012, 'min_sentence1_length': 43, 'average_sentence1_length': 136.49, 'max_sentence1_length': 388, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-kan': {'num_samples': 1012, 'number_of_characters': 270021, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 43, 'average_sentence2_length': 136.42, 'max_sentence2_length': 388, 'unique_sentence2': 1012}, 'mal-eng': {'num_samples': 1012, 'number_of_characters': 281302, 'unique_pairs': 1012, 'min_sentence1_length': 48, 'average_sentence1_length': 147.57, 'max_sentence1_length': 376, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mal': {'num_samples': 1012, 'number_of_characters': 281302, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 48, 'average_sentence2_length': 147.57, 'max_sentence2_length': 376, 'unique_sentence2': 1012}, 'mar-eng': {'num_samples': 1012, 'number_of_characters': 265212, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 131.67, 'max_sentence1_length': 356, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mar': {'num_samples': 1012, 'number_of_characters': 265023, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 131.48, 'max_sentence2_length': 355, 'unique_sentence2': 1012}, 'tam-eng': {'num_samples': 1012, 'number_of_characters': 286099, 'unique_pairs': 1012, 'min_sentence1_length': 48, 'average_sentence1_length': 152.31, 'max_sentence1_length': 404, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-tam': {'num_samples': 1012, 'number_of_characters': 286099, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 48, 'average_sentence2_length': 152.31, 'max_sentence2_length': 404, 'unique_sentence2': 1012}, 'tel-eng': {'num_samples': 1012, 'number_of_characters': 264460, 'unique_pairs': 1012, 'min_sentence1_length': 39, 'average_sentence1_length': 130.92, 'max_sentence1_length': 359, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-tel': {'num_samples': 1012, 'number_of_characters': 264447, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 39, 'average_sentence2_length': 130.91, 'max_sentence2_length': 359, 'unique_sentence2': 1012}, 'urd-eng': {'num_samples': 1012, 'number_of_characters': 261886, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 128.38, 'max_sentence1_length': 348, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-urd': {'num_samples': 1012, 'number_of_characters': 261885, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 128.38, 'max_sentence2_length': 348, 'unique_sentence2': 1012}, 'asm-eng': {'num_samples': 1012, 'number_of_characters': 257902, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 124.44, 'max_sentence1_length': 329, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-asm': {'num_samples': 1012, 'number_of_characters': 257909, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 124.45, 'max_sentence2_length': 329, 'unique_sentence2': 1012}, 'bho-eng': {'num_samples': 1012, 'number_of_characters': 260578, 'unique_pairs': 1012, 'min_sentence1_length': 36, 'average_sentence1_length': 127.09, 'max_sentence1_length': 367, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-bho': {'num_samples': 1012, 'number_of_characters': 260601, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 36, 'average_sentence2_length': 127.11, 'max_sentence2_length': 367, 'unique_sentence2': 1012}, 'nep-eng': {'num_samples': 1012, 'number_of_characters': 258869, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 125.4, 'max_sentence1_length': 362, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-nep': {'num_samples': 1012, 'number_of_characters': 258869, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 125.4, 'max_sentence2_length': 362, 'unique_sentence2': 1012}, 'ory-eng': {'num_samples': 1012, 'number_of_characters': 266805, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 133.24, 'max_sentence1_length': 354, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-ory': {'num_samples': 1012, 'number_of_characters': 266805, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 133.24, 'max_sentence2_length': 354, 'unique_sentence2': 1012}, 'pan-eng': {'num_samples': 1012, 'number_of_characters': 265391, 'unique_pairs': 1012, 'min_sentence1_length': 37, 'average_sentence1_length': 131.84, 'max_sentence1_length': 380, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-pan': {'num_samples': 1012, 'number_of_characters': 265391, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 37, 'average_sentence2_length': 131.84, 'max_sentence2_length': 380, 'unique_sentence2': 1012}, 'pus-eng': {'num_samples': 1012, 'number_of_characters': 254422, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 121.0, 'max_sentence1_length': 325, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-pus': {'num_samples': 1012, 'number_of_characters': 254421, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 121.0, 'max_sentence2_length': 325, 'unique_sentence2': 1012}, 'san-eng': {'num_samples': 1012, 'number_of_characters': 260339, 'unique_pairs': 1012, 'min_sentence1_length': 33, 'average_sentence1_length': 126.85, 'max_sentence1_length': 358, 'unique_sentence1': 1011, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-san': {'num_samples': 1012, 'number_of_characters': 260224, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 33, 'average_sentence2_length': 126.74, 'max_sentence2_length': 358, 'unique_sentence2': 1011}, 'awa-eng': {'num_samples': 1012, 'number_of_characters': 260179, 'unique_pairs': 1012, 'min_sentence1_length': 34, 'average_sentence1_length': 126.69, 'max_sentence1_length': 378, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-awa': {'num_samples': 1012, 'number_of_characters': 260137, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 34, 'average_sentence2_length': 126.65, 'max_sentence2_length': 378, 'unique_sentence2': 1012}, 'bgc-eng': {'num_samples': 1012, 'number_of_characters': 257450, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 124.0, 'max_sentence1_length': 332, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-bgc': {'num_samples': 1012, 'number_of_characters': 257450, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 124.0, 'max_sentence2_length': 332, 'unique_sentence2': 1012}, 'bod-eng': {'num_samples': 1012, 'number_of_characters': 280188, 'unique_pairs': 1012, 'min_sentence1_length': 42, 'average_sentence1_length': 146.46, 'max_sentence1_length': 431, 'unique_sentence1': 1009, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-bod': {'num_samples': 1012, 'number_of_characters': 280126, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 42, 'average_sentence2_length': 146.4, 'max_sentence2_length': 431, 'unique_sentence2': 1009}, 'boy-eng': {'num_samples': 1012, 'number_of_characters': 277538, 'unique_pairs': 1012, 'min_sentence1_length': 36, 'average_sentence1_length': 143.85, 'max_sentence1_length': 396, 'unique_sentence1': 1011, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-boy': {'num_samples': 1012, 'number_of_characters': 277538, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 36, 'average_sentence2_length': 143.85, 'max_sentence2_length': 396, 'unique_sentence2': 1011}, 'gbm-eng': {'num_samples': 1012, 'number_of_characters': 261027, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 127.53, 'max_sentence1_length': 333, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-gbm': {'num_samples': 1012, 'number_of_characters': 261027, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 127.53, 'max_sentence2_length': 333, 'unique_sentence2': 1012}, 'gom-eng': {'num_samples': 1012, 'number_of_characters': 259182, 'unique_pairs': 1012, 'min_sentence1_length': 37, 'average_sentence1_length': 125.71, 'max_sentence1_length': 335, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-gom': {'num_samples': 1012, 'number_of_characters': 259182, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 37, 'average_sentence2_length': 125.71, 'max_sentence2_length': 335, 'unique_sentence2': 1012}, 'hne-eng': {'num_samples': 1012, 'number_of_characters': 258911, 'unique_pairs': 1012, 'min_sentence1_length': 42, 'average_sentence1_length': 125.44, 'max_sentence1_length': 327, 'unique_sentence1': 1011, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-hne': {'num_samples': 1012, 'number_of_characters': 258915, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 42, 'average_sentence2_length': 125.44, 'max_sentence2_length': 326, 'unique_sentence2': 1011}, 'raj-eng': {'num_samples': 1012, 'number_of_characters': 261987, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 128.48, 'max_sentence1_length': 338, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-raj': {'num_samples': 1012, 'number_of_characters': 261987, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 128.48, 'max_sentence2_length': 338, 'unique_sentence2': 1012}, 'mai-eng': {'num_samples': 1012, 'number_of_characters': 261374, 'unique_pairs': 1012, 'min_sentence1_length': 36, 'average_sentence1_length': 127.87, 'max_sentence1_length': 350, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mai': {'num_samples': 1012, 'number_of_characters': 261377, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 36, 'average_sentence2_length': 127.88, 'max_sentence2_length': 350, 'unique_sentence2': 1012}, 'mni-eng': {'num_samples': 1012, 'number_of_characters': 268767, 'unique_pairs': 1012, 'min_sentence1_length': 38, 'average_sentence1_length': 135.18, 'max_sentence1_length': 353, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mni': {'num_samples': 1012, 'number_of_characters': 268768, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 38, 'average_sentence2_length': 135.18, 'max_sentence2_length': 354, 'unique_sentence2': 1012}, 'mup-eng': {'num_samples': 1012, 'number_of_characters': 262034, 'unique_pairs': 1012, 'min_sentence1_length': 40, 'average_sentence1_length': 128.53, 'max_sentence1_length': 340, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mup': {'num_samples': 1012, 'number_of_characters': 262034, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 40, 'average_sentence2_length': 128.53, 'max_sentence2_length': 340, 'unique_sentence2': 1012}, 'mwr-eng': {'num_samples': 1012, 'number_of_characters': 263749, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.22, 'max_sentence1_length': 345, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-mwr': {'num_samples': 1012, 'number_of_characters': 263749, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.22, 'max_sentence2_length': 345, 'unique_sentence2': 1012}, 'sat-eng': {'num_samples': 1012, 'number_of_characters': 271757, 'unique_pairs': 1012, 'min_sentence1_length': 43, 'average_sentence1_length': 138.13, 'max_sentence1_length': 366, 'unique_sentence1': 1012, 'min_sentence2_length': 35, 'average_sentence2_length': 130.4, 'max_sentence2_length': 368, 'unique_sentence2': 1012}, 'eng-sat': {'num_samples': 1012, 'number_of_characters': 271757, 'unique_pairs': 1012, 'min_sentence1_length': 35, 'average_sentence1_length': 130.4, 'max_sentence1_length': 368, 'unique_sentence1': 1012, 'min_sentence2_length': 43, 'average_sentence2_length': 138.13, 'max_sentence2_length': 366, 'unique_sentence2': 1012}}}} | +| [IndicLangClassification](https://arxiv.org/abs/2305.15814) | ['asm', 'ben', 'brx', 'doi', 'gom', 'guj', 'hin', 'kan', 'kas', 'mai', 'mal', 'mar', 'mni', 'npi', 'ory', 'pan', 'san', 'sat', 'snd', 'tam', 'tel', 'urd'] | Classification | s2s | [Non-fiction, Web, Written] | None | None | | [IndicNLPNewsClassification](https://github.com/AI4Bharat/indicnlp_corpus#indicnlp-news-article-classification-dataset) (Anoop Kunchukuttan, 2020) | ['guj', 'kan', 'mal', 'mar', 'ori', 'pan', 'tam', 'tel'] | Classification | s2s | [News, Written] | None | None | | [IndicQARetrieval](https://arxiv.org/abs/2212.05409) (Sumanth Doddapaneni, 2022) | ['asm', 'ben', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel'] | Retrieval | s2p | [Web, Written] | None | None | | [IndicReviewsClusteringP2P](https://arxiv.org/abs/2212.05409) (Sumanth Doddapaneni, 2022) | ['asm', 'ben', 'brx', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel', 'urd'] | Clustering | p2p | [Reviews, Written] | None | None | | [IndicSentimentClassification](https://arxiv.org/abs/2212.05409) (Sumanth Doddapaneni, 2022) | ['asm', 'ben', 'brx', 'guj', 'hin', 'kan', 'mal', 'mar', 'ory', 'pan', 'tam', 'tel', 'urd'] | Classification | s2s | [Reviews, Written] | None | None | | [IndonesianIdClickbaitClassification](http://www.sciencedirect.com/science/article/pii/S2352340920311252) | ['ind'] | Classification | s2s | [News, Written] | None | None | | [IndonesianMongabayConservationClassification](https://aclanthology.org/2023.sealp-1.4/) | ['ind'] | Classification | s2s | [Web, Written] | None | None | +| [InfoSeekIT2ITRetrieval](https://aclanthology.org/2023.emnlp-main.925) (Chen et al., 2023) | ['eng'] | Any2AnyRetrieval | it2it | [Encyclopaedic] | None | None | +| [InfoSeekIT2TRetrieval](https://aclanthology.org/2023.emnlp-main.925) (Chen et al., 2023) | ['eng'] | Any2AnyRetrieval | it2t | [Encyclopaedic] | None | None | | [InsurancePolicyInterpretationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [InternationalCitizenshipQuestionsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [IsiZuluNewsClassification](https://huggingface.co/datasets/dsfsi/za-isizulu-siswati-news) (Madodonga et al., 2023) | ['zul'] | Classification | s2s | [News, Written] | None | None | -| [ItaCaseholdClassification](https://doi.org/10.1145/3594536.3595177) (Licari et al., 2023) | ['ita'] | Classification | s2s | [Legal, Government, Written] | None | None | +| [ItaCaseholdClassification](https://doi.org/10.1145/3594536.3595177) (Licari et al., 2023) | ['ita'] | Classification | s2s | [Government, Legal, Written] | None | None | | [Itacola](https://aclanthology.org/2021.findings-emnlp.250/) | ['ita'] | Classification | s2s | [Non-fiction, Spoken, Written] | None | None | | [JCrewBlockerLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [JDReview](https://aclanthology.org/2023.nodalida-1.20/) (Xiao et al., 2023) | ['cmn'] | Classification | s2s | | None | None | -<<<<<<< HEAD -| [JSICK](https://github.com/sbintuitions/JMTEB) (Yanaka et al., 2022) | ['jpn'] | STS | s2s | [Web, Written] | {'test': 1986} | {'test': 21.47} | -| [JSTS](https://aclanthology.org/2022.lrec-1.317.pdf#page=2.00) | ['jpn'] | STS | s2s | [Web, Written] | {'valudtion': 1457} | {'valudtion': 46.34} | -| [JaGovFaqsRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Web, Written] | {'test': 2048} | {'test': {'average_document_length': 210.02601561814512, 'average_query_length': 59.48193359375, 'num_documents': 22794, 'num_queries': 2048, 'average_relevant_docs_per_query': 1.0}} | -| [JaQuADRetrieval](https://arxiv.org/abs/2202.01764) (ByungHoon So, 2022) | ['jpn'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Written] | {'validation': 2048} | {'validation': {'average_document_length': 155.80922362309224, 'average_query_length': 30.826171875, 'num_documents': 3014, 'num_queries': 2048, 'average_relevant_docs_per_query': 2.0}} | -| [JaqketRetrieval](https://github.com/kumapo/JAQKET-dataset) | ['jpn'] | Retrieval | s2p | [Encyclopaedic, Non-fiction, Written] | | | -| [JavaneseIMDBClassification](https://github.com/w11wo/nlp-datasets#javanese-imdb) (Wongso et al., 2021) | ['jav'] | Classification | s2s | [Reviews, Written] | {'test': 25000} | {'test': 481.83} | -| [KLUE-NLI](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | PairClassification | s2s | [News, Encyclopaedic, Written] | {'validation': 2000} | {'validation': 35.01} | -| [KLUE-STS](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | STS | s2s | [Reviews, News, Spoken, Written, Spoken] | {'validation': 519} | {'validation': 33.178227360308284} | -| [KLUE-TC](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | Classification | s2s | [News, Written] | {'validation': 2048} | {'validation': 27.079609091907326} | -| [KannadaNewsClassification](https://github.com/goru001/nlp-for-kannada) (Anoop Kunchukuttan, 2020) | ['kan'] | Classification | s2s | [News, Written] | {'train': 6460} | {'train': 65.88} | -| [KinopoiskClassification](https://www.dialog-21.ru/media/1226/blinovpd.pdf) (Blinov et al., 2013) | ['rus'] | Classification | p2p | [Reviews, Written] | {'test': 1500} | {'test': 1897.3} | -| Ko-StrategyQA (Geva et al., 2021) | ['kor'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 319.25953950924225, 'average_query_length': 22.75337837837838, 'num_documents': 9251, 'num_queries': 592, 'average_relevant_docs_per_query': 1.9341216216216217}} | -| [KorFin](https://huggingface.co/datasets/amphora/korfin-asc) (Son et al., 2023) | ['kor'] | Classification | s2s | [News, Written] | {'test': 2048} | {'test': 75.28} | -| [KorHateClassification](https://paperswithcode.com/dataset/korean-hatespeech-dataset) (Jihyung Moon, 2020) | ['kor'] | Classification | s2s | [Social, Written] | {'train': 2048, 'test': 471} | {'train': 38.57, 'test': 38.86} | -| [KorHateSpeechMLClassification](https://paperswithcode.com/dataset/korean-multi-label-hate-speech-dataset) | ['kor'] | MultilabelClassification | s2s | [Social, Written] | {'train': 8192, 'test': 2048} | {'train': 33.67, 'test': 34.67} | -| [KorSTS](https://arxiv.org/abs/2004.03289) (Ham et al., 2020) | ['kor'] | STS | s2s | [News, Web] | {'test': 1379} | {'test': 29.279433139534884} | -| [KorSarcasmClassification](https://github.com/SpellOnYou/korean-sarcasm) (Kim et al., 2019) | ['kor'] | Classification | s2s | [Social, Written] | {'train': 2048, 'test': 301} | {'train': 48.45, 'test': 46.77} | -| [KurdishSentimentClassification](https://link.springer.com/article/10.1007/s10579-023-09716-6) (Badawi et al., 2024) | ['kur'] | Classification | s2s | [Web, Written] | {'train': 6000, 'test': 1987} | {'train': 59.38, 'test': 56.11} | -| [LCQMC](https://aclanthology.org/2021.emnlp-main.357) (Shitao Xiao, 2024) | ['cmn'] | STS | s2s | | None | None | -| [LEMBNarrativeQARetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) | ['eng'] | Retrieval | s2p | [Fiction, Non-fiction, Written] | {'test': 10804} | {'test': {'average_document_length': 326753.5323943662, 'average_query_length': 47.89453536223562, 'num_documents': 355, 'num_queries': 10449, 'average_relevant_docs_per_query': 1.0}} | -| [LEMBNeedleRetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) (Zhu et al., 2024) | ['eng'] | Retrieval | s2p | [Academic, Blog, Written] | {'test_256': 150, 'test_512': 150, 'test_1024': 150, 'test_2048': 150, 'test_4096': 150, 'test_8192': 150, 'test_16384': 150, 'test_32768': 150} | {'test_256': {'average_document_length': 1013.22, 'average_query_length': 60.48, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_512': {'average_document_length': 2009.96, 'average_query_length': 57.3, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_1024': {'average_document_length': 4069.9, 'average_query_length': 58.28, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_2048': {'average_document_length': 8453.82, 'average_query_length': 59.92, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_4096': {'average_document_length': 17395.8, 'average_query_length': 55.86, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_8192': {'average_document_length': 35203.82, 'average_query_length': 59.6, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_16384': {'average_document_length': 72054.8, 'average_query_length': 59.12, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_32768': {'average_document_length': 141769.8, 'average_query_length': 58.34, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}} | -| [LEMBPasskeyRetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) (Zhu et al., 2024) | ['eng'] | Retrieval | s2p | [Fiction, Written] | {'test_256': 150, 'test_512': 150, 'test_1024': 150, 'test_2048': 150, 'test_4096': 150, 'test_8192': 150, 'test_16384': 150, 'test_32768': 150} | {'test_256': {'average_document_length': 876.24, 'average_query_length': 38.1, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_512': {'average_document_length': 1785.2, 'average_query_length': 37.76, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_1024': {'average_document_length': 3607.18, 'average_query_length': 37.68, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_2048': {'average_document_length': 7242.2, 'average_query_length': 37.8, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_4096': {'average_document_length': 14518.16, 'average_query_length': 37.64, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_8192': {'average_document_length': 29071.16, 'average_query_length': 37.54, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_16384': {'average_document_length': 58175.16, 'average_query_length': 38.12, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}, 'test_32768': {'average_document_length': 116380.16, 'average_query_length': 37.74, 'num_documents': 100, 'num_queries': 50, 'average_relevant_docs_per_query': 1.0}} | -| [LEMBQMSumRetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) | ['eng'] | Retrieval | s2p | [Spoken, Written] | {'test': 1724} | {'test': {'average_document_length': 53335.817258883246, 'average_query_length': 433.50294695481335, 'num_documents': 197, 'num_queries': 1527, 'average_relevant_docs_per_query': 1.0}} | -| [LEMBSummScreenFDRetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) | ['eng'] | Retrieval | s2p | [Spoken, Written] | {'validation': 672} | {'validation': {'average_document_length': 30854.32738095238, 'average_query_length': 591.4910714285714, 'num_documents': 336, 'num_queries': 336, 'average_relevant_docs_per_query': 1.0}} | -| [LEMBWikimQARetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) (Ho et al., 2020) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 500} | {'test': {'average_document_length': 37445.60333333333, 'average_query_length': 67.57, 'num_documents': 300, 'num_queries': 300, 'average_relevant_docs_per_query': 1.0}} | -| [LanguageClassification](https://huggingface.co/datasets/papluca/language-identification) (Conneau et al., 2018) | ['ara', 'bul', 'cmn', 'deu', 'ell', 'eng', 'fra', 'hin', 'ita', 'jpn', 'nld', 'pol', 'por', 'rus', 'spa', 'swa', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Reviews, Web, Non-fiction, Fiction, Government, Written] | {'test': 2048} | {'test': {'num_samples': 2048, 'average_text_length': 109.546875, 'unique_labels': 20, 'labels': {'17': {'count': 102}, '0': {'count': 102}, '11': {'count': 102}, '4': {'count': 103}, '3': {'count': 102}, '1': {'count': 102}, '10': {'count': 102}, '2': {'count': 103}, '16': {'count': 103}, '9': {'count': 103}, '5': {'count': 102}, '7': {'count': 102}, '13': {'count': 102}, '14': {'count': 103}, '12': {'count': 102}, '15': {'count': 103}, '19': {'count': 102}, '18': {'count': 102}, '6': {'count': 103}, '8': {'count': 103}}}, 'train': {'num_samples': 70000, 'average_text_length': 110.86141428571429, 'unique_labels': 20, 'labels': {'12': {'count': 3500}, '1': {'count': 3500}, '19': {'count': 3500}, '15': {'count': 3500}, '13': {'count': 3500}, '11': {'count': 3500}, '17': {'count': 3500}, '14': {'count': 3500}, '16': {'count': 3500}, '5': {'count': 3500}, '0': {'count': 3500}, '8': {'count': 3500}, '7': {'count': 3500}, '2': {'count': 3500}, '3': {'count': 3500}, '10': {'count': 3500}, '6': {'count': 3500}, '18': {'count': 3500}, '4': {'count': 3500}, '9': {'count': 3500}}}} | -| [LccSentimentClassification](https://github.com/fnielsen/lcc-sentiment) | ['dan'] | Classification | s2s | [News, Web, Written] | {'test': 150} | {'test': 118.7} | -| [LeCaRDv2](https://github.com/THUIR/LeCaRDv2) (Haitao Li, 2023) | ['zho'] | Retrieval | p2p | [Legal, Written] | None | {'test': {'average_document_length': 7232.823978919631, 'average_query_length': 4259.440251572327, 'num_documents': 3795, 'num_queries': 159, 'average_relevant_docs_per_query': 24.50314465408805}} | -| [LearnedHandsBenefitsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 66} | {'test': 1308.44} | -| [LearnedHandsBusinessLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 174} | {'test': 1144.51} | -| [LearnedHandsConsumerLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 614} | {'test': 1277.45} | -| [LearnedHandsCourtsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 192} | {'test': 1171.02} | -| [LearnedHandsCrimeLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 688} | {'test': 1212.9} | -| [LearnedHandsDivorceLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 150} | {'test': 1242.43} | -| [LearnedHandsDomesticViolenceLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 174} | {'test': 1360.83} | -| [LearnedHandsEducationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 56} | {'test': 1397.44} | -| [LearnedHandsEmploymentLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 710} | {'test': 1262.74} | -| [LearnedHandsEstatesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 178} | {'test': 1200.7} | -| [LearnedHandsFamilyLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 2048} | {'test': 1338.27} | -| [LearnedHandsHealthLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 226} | {'test': 1472.59} | -| [LearnedHandsHousingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 2048} | {'test': 1322.54} | -| [LearnedHandsImmigrationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 134} | {'test': 1216.31} | -| [LearnedHandsTortsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 432} | {'test': 1406.97} | -| [LearnedHandsTrafficLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 556} | {'test': 1182.91} | -| [LegalBenchConsumerContractsQA](https://huggingface.co/datasets/nguha/legalbench/viewer/consumer_contracts_qa) (Koreeda et al., 2021) | ['eng'] | Retrieval | s2p | [Legal, Written] | None | {'test': {'average_document_length': 2745.8246753246754, 'average_query_length': 92.4090909090909, 'num_documents': 154, 'num_queries': 396, 'average_relevant_docs_per_query': 1.0}} | -| [LegalBenchCorporateLobbying](https://huggingface.co/datasets/nguha/legalbench/viewer/corporate_lobbying) (Neel Guha, 2023) | ['eng'] | Retrieval | s2p | [Legal, Written] | None | {'test': {'average_document_length': 1157.2225705329154, 'average_query_length': 177.87941176470588, 'num_documents': 319, 'num_queries': 340, 'average_relevant_docs_per_query': 1.0}} | -| [LegalBenchPC](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | PairClassification | s2s | [Legal, Written] | {'test': 2048} | {'test': 287.18} | -| [LegalQuAD](https://github.com/Christoph911/AIKE2021_Appendix) (Hoppe et al., 2021) | ['deu'] | Retrieval | s2p | [Legal, Written] | None | {'test': {'average_document_length': 19481.955, 'average_query_length': 71.965, 'num_documents': 200, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}} | -| [LegalReasoningCausalityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 55} | {'test': 1563.76} | -| [LegalSummarization](https://github.com/lauramanor/legal_summarization) | ['eng'] | Retrieval | s2p | [Legal, Written] | None | {'test': {'average_document_length': 606.1643835616438, 'average_query_length': 103.19014084507042, 'num_documents': 438, 'num_queries': 284, 'average_relevant_docs_per_query': 1.545774647887324}} | -| [LinceMTBitextMining](https://ritual.uh.edu/lince/) (Aguilar et al., 2020) | ['eng', 'hin'] | BitextMining | s2s | [Social, Written] | {'train': 8060} | {'train': 58.67} | -| [LitSearchRetrieval](https://github.com/princeton-nlp/LitSearch) (Ajith et al., 2024) | ['eng'] | Retrieval | s2p | [Academic, Non-fiction, Written] | {'test': 597} | {'test': {'average_document_length': 841.2769, 'average_query_length': 141.2, 'num_documents': 64183, 'num_queries': 597, 'average_relevant_docs_per_query': 1.070351}} | -| [LivedoorNewsClustering.v2](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Clustering | s2s | [News, Written] | {'test': 1106} | {'test': 1082.61} | -| [MAUDLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 2048} | {'test': 1802.93} | -| [MIRACLReranking](https://project-miracl.github.io/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Reranking | s2s | [Encyclopaedic, Written] | {'dev': 44608} | {'dev': 506.3} | -| [MIRACLRetrieval](http://miracl.ai/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | None | {'dev': {'ar': {'average_document_length': 318.6539598547405, 'average_query_length': 29.480662983425415, 'num_documents': 2061414, 'num_queries': 2896, 'average_relevant_docs_per_query': 1.953729281767956}, 'bn': {'average_document_length': 383.2428136511194, 'average_query_length': 46.98053527980535, 'num_documents': 297265, 'num_queries': 411, 'average_relevant_docs_per_query': 2.099756690997567}, 'de': {'average_document_length': 414.28004442393404, 'average_query_length': 46.0, 'num_documents': 15866222, 'num_queries': 305, 'average_relevant_docs_per_query': 2.6590163934426227}, 'en': {'average_document_length': 401.0042914921588, 'average_query_length': 40.247809762202756, 'num_documents': 32893221, 'num_queries': 799, 'average_relevant_docs_per_query': 2.911138923654568}, 'es': {'average_document_length': 403.71153493754986, 'average_query_length': 47.373456790123456, 'num_documents': 10373953, 'num_queries': 648, 'average_relevant_docs_per_query': 4.609567901234568}, 'fa': {'average_document_length': 262.6478385010321, 'average_query_length': 41.1503164556962, 'num_documents': 2207172, 'num_queries': 632, 'average_relevant_docs_per_query': 2.079113924050633}, 'fi': {'average_document_length': 359.87767671935734, 'average_query_length': 38.63493312352478, 'num_documents': 1883509, 'num_queries': 1271, 'average_relevant_docs_per_query': 1.925255704169945}, 'fr': {'average_document_length': 343.6283550271699, 'average_query_length': 43.883381924198254, 'num_documents': 14636953, 'num_queries': 343, 'average_relevant_docs_per_query': 2.131195335276968}, 'hi': {'average_document_length': 370.96196845914386, 'average_query_length': 53.34, 'num_documents': 506264, 'num_queries': 350, 'average_relevant_docs_per_query': 2.1485714285714286}, 'id': {'average_document_length': 350.2785651811673, 'average_query_length': 37.958333333333336, 'num_documents': 1446315, 'num_queries': 960, 'average_relevant_docs_per_query': 3.216666666666667}, 'ja': {'average_document_length': 145.8538220556965, 'average_query_length': 17.71395348837209, 'num_documents': 6953614, 'num_queries': 860, 'average_relevant_docs_per_query': 2.0813953488372094}, 'ko': {'average_document_length': 173.97649170809927, 'average_query_length': 21.624413145539908, 'num_documents': 1486752, 'num_queries': 213, 'average_relevant_docs_per_query': 2.568075117370892}, 'ru': {'average_document_length': 332.2475377512674, 'average_query_length': 44.13258785942492, 'num_documents': 9543918, 'num_queries': 1252, 'average_relevant_docs_per_query': 2.8434504792332267}, 'sw': {'average_document_length': 228.71348655286377, 'average_query_length': 38.97095435684647, 'num_documents': 131924, 'num_queries': 482, 'average_relevant_docs_per_query': 1.887966804979253}, 'te': {'average_document_length': 396.2108674545774, 'average_query_length': 38.11231884057971, 'num_documents': 518079, 'num_queries': 828, 'average_relevant_docs_per_query': 1.0314009661835748}, 'th': {'average_document_length': 356.8283496198581, 'average_query_length': 42.87585266030014, 'num_documents': 542166, 'num_queries': 733, 'average_relevant_docs_per_query': 1.8321964529331514}, 'yo': {'average_document_length': 159.35250698366738, 'average_query_length': 37.6890756302521, 'num_documents': 49043, 'num_queries': 119, 'average_relevant_docs_per_query': 1.2100840336134453}, 'zh': {'average_document_length': 119.9458931721347, 'average_query_length': 10.867684478371501, 'num_documents': 4934368, 'num_queries': 393, 'average_relevant_docs_per_query': 2.5292620865139948}}} | -| [MIRACLRetrievalHardNegatives](http://miracl.ai/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | None | {'dev': {'average_document_length': 417.6655323669399, 'average_query_length': 37.46957385337667, 'num_documents': 2449382, 'num_queries': 11076, 'average_relevant_docs_per_query': 2.3643011917659806, 'hf_subset_descriptive_stats': {'ar': {'average_document_length': 438.1872433017704, 'average_query_length': 29.584, 'num_documents': 192103, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.982}, 'bn': {'average_document_length': 383.2428136511194, 'average_query_length': 46.98053527980535, 'num_documents': 297265, 'num_queries': 411, 'average_relevant_docs_per_query': 2.099756690997567}, 'de': {'average_document_length': 513.7796484139344, 'average_query_length': 46.0, 'num_documents': 71277, 'num_queries': 305, 'average_relevant_docs_per_query': 2.6590163934426227}, 'en': {'average_document_length': 529.2486406963214, 'average_query_length': 40.247809762202756, 'num_documents': 178768, 'num_queries': 799, 'average_relevant_docs_per_query': 2.911138923654568}, 'es': {'average_document_length': 535.8023645655877, 'average_query_length': 47.373456790123456, 'num_documents': 146750, 'num_queries': 648, 'average_relevant_docs_per_query': 4.609567901234568}, 'fa': {'average_document_length': 411.2648282882721, 'average_query_length': 41.1503164556962, 'num_documents': 133596, 'num_queries': 632, 'average_relevant_docs_per_query': 2.079113924050633}, 'fi': {'average_document_length': 462.9445310289844, 'average_query_length': 38.646, 'num_documents': 194415, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.918}, 'fr': {'average_document_length': 460.40909271865917, 'average_query_length': 43.883381924198254, 'num_documents': 75357, 'num_queries': 343, 'average_relevant_docs_per_query': 2.131195335276968}, 'hi': {'average_document_length': 498.6759426632417, 'average_query_length': 53.34, 'num_documents': 63066, 'num_queries': 350, 'average_relevant_docs_per_query': 2.1485714285714286}, 'id': {'average_document_length': 494.1689807519638, 'average_query_length': 37.958333333333336, 'num_documents': 168173, 'num_queries': 960, 'average_relevant_docs_per_query': 3.216666666666667}, 'ja': {'average_document_length': 206.13654293407583, 'average_query_length': 17.71395348837209, 'num_documents': 185319, 'num_queries': 860, 'average_relevant_docs_per_query': 2.0813953488372094}, 'ko': {'average_document_length': 257.82646155267594, 'average_query_length': 21.624413145539908, 'num_documents': 43293, 'num_queries': 213, 'average_relevant_docs_per_query': 2.568075117370892}, 'ru': {'average_document_length': 476.0820349224605, 'average_query_length': 44.055, 'num_documents': 219114, 'num_queries': 1000, 'average_relevant_docs_per_query': 2.833}, 'sw': {'average_document_length': 228.71348655286377, 'average_query_length': 38.97095435684647, 'num_documents': 131924, 'num_queries': 482, 'average_relevant_docs_per_query': 1.887966804979253}, 'te': {'average_document_length': 601.7099283059209, 'average_query_length': 38.11231884057971, 'num_documents': 101961, 'num_queries': 828, 'average_relevant_docs_per_query': 1.0314009661835748}, 'th': {'average_document_length': 478.8818849711528, 'average_query_length': 42.87585266030014, 'num_documents': 116649, 'num_queries': 733, 'average_relevant_docs_per_query': 1.8321964529331514}, 'yo': {'average_document_length': 159.35250698366738, 'average_query_length': 37.6890756302521, 'num_documents': 49043, 'num_queries': 119, 'average_relevant_docs_per_query': 1.2100840336134453}, 'zh': {'average_document_length': 147.36211243527777, 'average_query_length': 10.867684478371501, 'num_documents': 81309, 'num_queries': 393, 'average_relevant_docs_per_query': 2.5292620865139948}}}} | -| [MLQARetrieval](https://huggingface.co/datasets/mlqa) | ['ara', 'deu', 'eng', 'hin', 'spa', 'vie', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 158083, 'validation': 15747} | {'validation': {'ara-ara': {'average_document_length': 693.8883826879271, 'average_query_length': 42.321083172147, 'num_documents': 439, 'num_queries': 517, 'average_relevant_docs_per_query': 1.0}, 'ara-deu': {'average_document_length': 759.3882352941176, 'average_query_length': 55.14492753623188, 'num_documents': 170, 'num_queries': 207, 'average_relevant_docs_per_query': 1.0}, 'ara-eng': {'average_document_length': 693.8883826879271, 'average_query_length': 50.029013539651835, 'num_documents': 439, 'num_queries': 517, 'average_relevant_docs_per_query': 1.0}, 'ara-spa': {'average_document_length': 654.3071428571428, 'average_query_length': 53.68944099378882, 'num_documents': 140, 'num_queries': 161, 'average_relevant_docs_per_query': 1.0}, 'ara-hin': {'average_document_length': 626.5935483870968, 'average_query_length': 51.956989247311824, 'num_documents': 155, 'num_queries': 186, 'average_relevant_docs_per_query': 1.0}, 'ara-vie': {'average_document_length': 804.6216216216217, 'average_query_length': 49.57055214723926, 'num_documents': 148, 'num_queries': 163, 'average_relevant_docs_per_query': 1.0}, 'ara-zho': {'average_document_length': 787.3161290322581, 'average_query_length': 15.617021276595745, 'num_documents': 155, 'num_queries': 188, 'average_relevant_docs_per_query': 1.0}, 'deu-ara': {'average_document_length': 702.1675977653631, 'average_query_length': 43.06280193236715, 'num_documents': 179, 'num_queries': 207, 'average_relevant_docs_per_query': 1.0}, 'deu-deu': {'average_document_length': 721.405701754386, 'average_query_length': 52.572265625, 'num_documents': 456, 'num_queries': 512, 'average_relevant_docs_per_query': 1.0}, 'deu-eng': {'average_document_length': 721.405701754386, 'average_query_length': 48.33984375, 'num_documents': 456, 'num_queries': 512, 'average_relevant_docs_per_query': 1.0}, 'deu-spa': {'average_document_length': 677.2762430939226, 'average_query_length': 50.60204081632653, 'num_documents': 181, 'num_queries': 196, 'average_relevant_docs_per_query': 1.0}, 'deu-hin': {'average_document_length': 685.917808219178, 'average_query_length': 47.01840490797546, 'num_documents': 146, 'num_queries': 163, 'average_relevant_docs_per_query': 1.0}, 'deu-vie': {'average_document_length': 921.6196319018405, 'average_query_length': 46.81868131868132, 'num_documents': 163, 'num_queries': 182, 'average_relevant_docs_per_query': 1.0}, 'deu-zho': {'average_document_length': 736.6347305389221, 'average_query_length': 14.936842105263159, 'num_documents': 167, 'num_queries': 190, 'average_relevant_docs_per_query': 1.0}, 'eng-ara': {'average_document_length': 979.3447488584475, 'average_query_length': 42.321083172147, 'num_documents': 438, 'num_queries': 517, 'average_relevant_docs_per_query': 1.0}, 'eng-deu': {'average_document_length': 947.3109619686801, 'average_query_length': 52.572265625, 'num_documents': 447, 'num_queries': 512, 'average_relevant_docs_per_query': 1.0}, 'eng-eng': {'average_document_length': 940.2842535787321, 'average_query_length': 49.01480836236934, 'num_documents': 978, 'num_queries': 1148, 'average_relevant_docs_per_query': 1.0}, 'eng-spa': {'average_document_length': 904.3166287015945, 'average_query_length': 52.146, 'num_documents': 439, 'num_queries': 500, 'average_relevant_docs_per_query': 1.0}, 'eng-hin': {'average_document_length': 926.9621749408983, 'average_query_length': 49.3905325443787, 'num_documents': 423, 'num_queries': 507, 'average_relevant_docs_per_query': 1.0}, 'eng-vie': {'average_document_length': 1011.8296460176991, 'average_query_length': 48.082191780821915, 'num_documents': 452, 'num_queries': 511, 'average_relevant_docs_per_query': 1.0}, 'eng-zho': {'average_document_length': 1001.5046511627907, 'average_query_length': 15.39484126984127, 'num_documents': 430, 'num_queries': 504, 'average_relevant_docs_per_query': 1.0}, 'spa-ara': {'average_document_length': 674.3586206896551, 'average_query_length': 41.36024844720497, 'num_documents': 145, 'num_queries': 161, 'average_relevant_docs_per_query': 1.0}, 'spa-deu': {'average_document_length': 544.0489130434783, 'average_query_length': 51.86734693877551, 'num_documents': 184, 'num_queries': 196, 'average_relevant_docs_per_query': 1.0}, 'spa-eng': {'average_document_length': 641.8215859030837, 'average_query_length': 49.156, 'num_documents': 454, 'num_queries': 500, 'average_relevant_docs_per_query': 1.0}, 'spa-spa': {'average_document_length': 641.8215859030837, 'average_query_length': 52.146, 'num_documents': 454, 'num_queries': 500, 'average_relevant_docs_per_query': 1.0}, 'spa-hin': {'average_document_length': 703.3212121212122, 'average_query_length': 48.080213903743314, 'num_documents': 165, 'num_queries': 187, 'average_relevant_docs_per_query': 1.0}, 'spa-vie': {'average_document_length': 737.8579545454545, 'average_query_length': 48.82539682539682, 'num_documents': 176, 'num_queries': 189, 'average_relevant_docs_per_query': 1.0}, 'spa-zho': {'average_document_length': 605.52, 'average_query_length': 15.590062111801242, 'num_documents': 150, 'num_queries': 161, 'average_relevant_docs_per_query': 1.0}, 'hin-ara': {'average_document_length': 670.0394736842105, 'average_query_length': 43.623655913978496, 'num_documents': 152, 'num_queries': 186, 'average_relevant_docs_per_query': 1.0}, 'hin-deu': {'average_document_length': 596.9718309859155, 'average_query_length': 51.41717791411043, 'num_documents': 142, 'num_queries': 163, 'average_relevant_docs_per_query': 1.0}, 'hin-eng': {'average_document_length': 691.5482352941176, 'average_query_length': 49.75936883629191, 'num_documents': 425, 'num_queries': 507, 'average_relevant_docs_per_query': 1.0}, 'hin-spa': {'average_document_length': 718.4904458598726, 'average_query_length': 52.75935828877005, 'num_documents': 157, 'num_queries': 187, 'average_relevant_docs_per_query': 1.0}, 'hin-hin': {'average_document_length': 691.5482352941176, 'average_query_length': 49.3905325443787, 'num_documents': 425, 'num_queries': 507, 'average_relevant_docs_per_query': 1.0}, 'hin-vie': {'average_document_length': 778.484076433121, 'average_query_length': 48.35028248587571, 'num_documents': 157, 'num_queries': 177, 'average_relevant_docs_per_query': 1.0}, 'hin-zho': {'average_document_length': 685.0679012345679, 'average_query_length': 15.97883597883598, 'num_documents': 162, 'num_queries': 189, 'average_relevant_docs_per_query': 1.0}, 'vie-ara': {'average_document_length': 886.6052631578947, 'average_query_length': 41.214723926380366, 'num_documents': 152, 'num_queries': 163, 'average_relevant_docs_per_query': 1.0}, 'vie-deu': {'average_document_length': 981.4534161490683, 'average_query_length': 51.27472527472528, 'num_documents': 161, 'num_queries': 182, 'average_relevant_docs_per_query': 1.0}, 'vie-eng': {'average_document_length': 892.7250554323725, 'average_query_length': 48.09001956947162, 'num_documents': 451, 'num_queries': 511, 'average_relevant_docs_per_query': 1.0}, 'vie-spa': {'average_document_length': 936.6746987951807, 'average_query_length': 51.851851851851855, 'num_documents': 166, 'num_queries': 189, 'average_relevant_docs_per_query': 1.0}, 'vie-hin': {'average_document_length': 869.0509554140127, 'average_query_length': 46.44632768361582, 'num_documents': 157, 'num_queries': 177, 'average_relevant_docs_per_query': 1.0}, 'vie-vie': {'average_document_length': 892.7250554323725, 'average_query_length': 48.082191780821915, 'num_documents': 451, 'num_queries': 511, 'average_relevant_docs_per_query': 1.0}, 'vie-zho': {'average_document_length': 960.7349397590361, 'average_query_length': 15.048913043478262, 'num_documents': 166, 'num_queries': 184, 'average_relevant_docs_per_query': 1.0}, 'zho-ara': {'average_document_length': 238.75155279503105, 'average_query_length': 44.34574468085106, 'num_documents': 161, 'num_queries': 188, 'average_relevant_docs_per_query': 1.0}, 'zho-deu': {'average_document_length': 257.109756097561, 'average_query_length': 53.84736842105263, 'num_documents': 164, 'num_queries': 190, 'average_relevant_docs_per_query': 1.0}, 'zho-eng': {'average_document_length': 246.65237020316027, 'average_query_length': 50.15079365079365, 'num_documents': 443, 'num_queries': 504, 'average_relevant_docs_per_query': 1.0}, 'zho-spa': {'average_document_length': 249.6081081081081, 'average_query_length': 52.857142857142854, 'num_documents': 148, 'num_queries': 161, 'average_relevant_docs_per_query': 1.0}, 'zho-hin': {'average_document_length': 238.5521472392638, 'average_query_length': 52.05291005291005, 'num_documents': 163, 'num_queries': 189, 'average_relevant_docs_per_query': 1.0}, 'zho-vie': {'average_document_length': 268.32142857142856, 'average_query_length': 49.33695652173913, 'num_documents': 168, 'num_queries': 184, 'average_relevant_docs_per_query': 1.0}, 'zho-zho': {'average_document_length': 246.65237020316027, 'average_query_length': 15.39484126984127, 'num_documents': 443, 'num_queries': 504, 'average_relevant_docs_per_query': 1.0}}, 'test': {'ara-ara': {'average_document_length': 698.5714593198451, 'average_query_length': 41.26176636039752, 'num_documents': 4646, 'num_queries': 5333, 'average_relevant_docs_per_query': 1.000375023438965}, 'ara-deu': {'average_document_length': 592.5728542914171, 'average_query_length': 51.27730582524272, 'num_documents': 1503, 'num_queries': 1648, 'average_relevant_docs_per_query': 1.0006067961165048}, 'ara-eng': {'average_document_length': 698.5714593198451, 'average_query_length': 48.556451612903224, 'num_documents': 4646, 'num_queries': 5332, 'average_relevant_docs_per_query': 1.000562640660165}, 'ara-spa': {'average_document_length': 713.4833239118146, 'average_query_length': 51.406471183013146, 'num_documents': 1769, 'num_queries': 1978, 'average_relevant_docs_per_query': 1.0}, 'ara-hin': {'average_document_length': 702.1388888888889, 'average_query_length': 48.71818678317859, 'num_documents': 1512, 'num_queries': 1831, 'average_relevant_docs_per_query': 1.0}, 'ara-vie': {'average_document_length': 745.4528096017458, 'average_query_length': 48.815828041035665, 'num_documents': 1833, 'num_queries': 2047, 'average_relevant_docs_per_query': 1.0}, 'ara-zho': {'average_document_length': 774.4593639575971, 'average_query_length': 14.985355648535565, 'num_documents': 1698, 'num_queries': 1912, 'average_relevant_docs_per_query': 1.0}, 'deu-ara': {'average_document_length': 719.6800267201069, 'average_query_length': 39.54578532443905, 'num_documents': 1497, 'num_queries': 1649, 'average_relevant_docs_per_query': 1.0}, 'deu-deu': {'average_document_length': 725.5304712558599, 'average_query_length': 51.610680257035234, 'num_documents': 4053, 'num_queries': 4513, 'average_relevant_docs_per_query': 1.0008863283846665}, 'deu-eng': {'average_document_length': 725.5304712558599, 'average_query_length': 47.07777531575449, 'num_documents': 4053, 'num_queries': 4513, 'average_relevant_docs_per_query': 1.0008863283846665}, 'deu-spa': {'average_document_length': 740.5414052697616, 'average_query_length': 50.098591549295776, 'num_documents': 1594, 'num_queries': 1775, 'average_relevant_docs_per_query': 1.0005633802816902}, 'deu-hin': {'average_document_length': 674.3714063714064, 'average_query_length': 45.146153846153844, 'num_documents': 1287, 'num_queries': 1430, 'average_relevant_docs_per_query': 1.0}, 'deu-vie': {'average_document_length': 760.1198945981555, 'average_query_length': 46.64358208955224, 'num_documents': 1518, 'num_queries': 1675, 'average_relevant_docs_per_query': 1.0}, 'deu-zho': {'average_document_length': 771.3367697594501, 'average_query_length': 14.942592592592593, 'num_documents': 1455, 'num_queries': 1620, 'average_relevant_docs_per_query': 1.0006172839506173}, 'eng-ara': {'average_document_length': 1008.3584455058619, 'average_query_length': 41.26176636039752, 'num_documents': 4606, 'num_queries': 5333, 'average_relevant_docs_per_query': 1.000375023438965}, 'eng-deu': {'average_document_length': 910.3226686507936, 'average_query_length': 51.610680257035234, 'num_documents': 4032, 'num_queries': 4513, 'average_relevant_docs_per_query': 1.0008863283846665}, 'eng-eng': {'average_document_length': 983.0993344090359, 'average_query_length': 47.960714902434816, 'num_documents': 9916, 'num_queries': 11582, 'average_relevant_docs_per_query': 1.000690726990157}, 'eng-spa': {'average_document_length': 967.4622376109068, 'average_query_length': 50.923252713768804, 'num_documents': 4621, 'num_queries': 5251, 'average_relevant_docs_per_query': 1.000380879832413}, 'eng-hin': {'average_document_length': 986.0465631929046, 'average_query_length': 47.328315703824245, 'num_documents': 4059, 'num_queries': 4916, 'average_relevant_docs_per_query': 1.000406834825061}, 'eng-vie': {'average_document_length': 1048.6062197940744, 'average_query_length': 48.094085532302095, 'num_documents': 4759, 'num_queries': 5495, 'average_relevant_docs_per_query': 1.0}, 'eng-zho': {'average_document_length': 1063.8536257833482, 'average_query_length': 15.019080996884735, 'num_documents': 4468, 'num_queries': 5136, 'average_relevant_docs_per_query': 1.0001947040498442}, 'spa-ara': {'average_document_length': 645.5182320441988, 'average_query_length': 40.78412537917088, 'num_documents': 1810, 'num_queries': 1978, 'average_relevant_docs_per_query': 1.0}, 'spa-deu': {'average_document_length': 586.6057810578105, 'average_query_length': 51.870913190529876, 'num_documents': 1626, 'num_queries': 1774, 'average_relevant_docs_per_query': 1.0011273957158964}, 'spa-eng': {'average_document_length': 630.6735979836169, 'average_query_length': 47.827907862173994, 'num_documents': 4761, 'num_queries': 5253, 'average_relevant_docs_per_query': 1.0}, 'spa-spa': {'average_document_length': 630.6735979836169, 'average_query_length': 50.923252713768804, 'num_documents': 4761, 'num_queries': 5251, 'average_relevant_docs_per_query': 1.000380879832413}, 'spa-hin': {'average_document_length': 613.3478260869565, 'average_query_length': 46.36680208937899, 'num_documents': 1518, 'num_queries': 1723, 'average_relevant_docs_per_query': 1.0}, 'spa-vie': {'average_document_length': 659.6179295624333, 'average_query_length': 48.1595639246779, 'num_documents': 1874, 'num_queries': 2018, 'average_relevant_docs_per_query': 1.0}, 'spa-zho': {'average_document_length': 668.6646171045277, 'average_query_length': 15.115562403697997, 'num_documents': 1789, 'num_queries': 1947, 'average_relevant_docs_per_query': 1.0}, 'hin-ara': {'average_document_length': 765.0352862849534, 'average_query_length': 42.04642271982523, 'num_documents': 1502, 'num_queries': 1831, 'average_relevant_docs_per_query': 1.0}, 'hin-deu': {'average_document_length': 719.676862745098, 'average_query_length': 51.002799160251925, 'num_documents': 1275, 'num_queries': 1429, 'average_relevant_docs_per_query': 1.000699790062981}, 'hin-eng': {'average_document_length': 760.9956086850451, 'average_query_length': 47.91232709519935, 'num_documents': 4099, 'num_queries': 4916, 'average_relevant_docs_per_query': 1.000406834825061}, 'hin-spa': {'average_document_length': 753.5010281014394, 'average_query_length': 50.46689895470383, 'num_documents': 1459, 'num_queries': 1722, 'average_relevant_docs_per_query': 1.0005807200929153}, 'hin-hin': {'average_document_length': 760.9956086850451, 'average_query_length': 47.328315703824245, 'num_documents': 4099, 'num_queries': 4916, 'average_relevant_docs_per_query': 1.000406834825061}, 'hin-vie': {'average_document_length': 789.9253822629969, 'average_query_length': 48.21160760143811, 'num_documents': 1635, 'num_queries': 1947, 'average_relevant_docs_per_query': 1.0}, 'hin-zho': {'average_document_length': 834.2057448229793, 'average_query_length': 15.101301641199774, 'num_documents': 1497, 'num_queries': 1767, 'average_relevant_docs_per_query': 1.0}, 'vie-ara': {'average_document_length': 992.2129527991218, 'average_query_length': 41.82462139716659, 'num_documents': 1822, 'num_queries': 2047, 'average_relevant_docs_per_query': 1.0}, 'vie-deu': {'average_document_length': 861.0610079575597, 'average_query_length': 51.58721624850657, 'num_documents': 1508, 'num_queries': 1674, 'average_relevant_docs_per_query': 1.0005973715651135}, 'vie-eng': {'average_document_length': 913.8633993743483, 'average_query_length': 48.11086837793555, 'num_documents': 4795, 'num_queries': 5493, 'average_relevant_docs_per_query': 1.0003640997633352}, 'vie-spa': {'average_document_length': 940.0322580645161, 'average_query_length': 51.13386217154189, 'num_documents': 1829, 'num_queries': 2017, 'average_relevant_docs_per_query': 1.0004957858205255}, 'vie-hin': {'average_document_length': 838.1713414634146, 'average_query_length': 47.484334874165384, 'num_documents': 1640, 'num_queries': 1947, 'average_relevant_docs_per_query': 1.0}, 'vie-vie': {'average_document_length': 913.8633993743483, 'average_query_length': 48.094085532302095, 'num_documents': 4795, 'num_queries': 5495, 'average_relevant_docs_per_query': 1.0}, 'vie-zho': {'average_document_length': 999.064534883721, 'average_query_length': 15.045805455481215, 'num_documents': 1720, 'num_queries': 1943, 'average_relevant_docs_per_query': 1.0}, 'zho-ara': {'average_document_length': 253.71303841676368, 'average_query_length': 42.04866562009419, 'num_documents': 1718, 'num_queries': 1911, 'average_relevant_docs_per_query': 1.000523286237572}, 'zho-deu': {'average_document_length': 241.84631147540983, 'average_query_length': 52.25107958050586, 'num_documents': 1464, 'num_queries': 1621, 'average_relevant_docs_per_query': 1.0}, 'zho-eng': {'average_document_length': 247.55609326880776, 'average_query_length': 48.64167478091529, 'num_documents': 4546, 'num_queries': 5135, 'average_relevant_docs_per_query': 1.0003894839337877}, 'zho-spa': {'average_document_length': 254.44552196235026, 'average_query_length': 51.90446841294299, 'num_documents': 1753, 'num_queries': 1947, 'average_relevant_docs_per_query': 1.0}, 'zho-hin': {'average_document_length': 229.60590163934427, 'average_query_length': 49.06625141562854, 'num_documents': 1525, 'num_queries': 1766, 'average_relevant_docs_per_query': 1.0005662514156286}, 'zho-vie': {'average_document_length': 266.1140401146132, 'average_query_length': 49.27328872876994, 'num_documents': 1745, 'num_queries': 1943, 'average_relevant_docs_per_query': 1.0}, 'zho-zho': {'average_document_length': 247.55609326880776, 'average_query_length': 15.019080996884735, 'num_documents': 4546, 'num_queries': 5136, 'average_relevant_docs_per_query': 1.0001947040498442}}} | -| [MLQuestions](https://github.com/McGill-NLP/MLQuestions) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Academic, Written] | {'dev': 1500, 'test': 1500} | {'dev': {'average_document_length': 258.8772727272727, 'average_query_length': 45.05533333333333, 'num_documents': 11000, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'test': {'average_document_length': 258.8772727272727, 'average_query_length': 45.75333333333333, 'num_documents': 11000, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}} | -| [MLSUMClusteringP2P.v2](https://huggingface.co/datasets/mteb/mlsum) (Scialom et al., 2020) | ['deu', 'fra', 'rus', 'spa'] | Clustering | p2p | [News, Written] | {'validation': 2048, 'test': 2048} | {'validation': 4613, 'test': 4810} | -| [MLSUMClusteringS2S.v2](https://huggingface.co/datasets/mteb/mlsum) (Scialom et al., 2020) | ['deu', 'fra', 'rus', 'spa'] | Clustering | s2s | [News, Written] | {'validation': 750, 'test': 756} | {'validation': 4613, 'test': 4810} | -| [MMarcoReranking](https://github.com/unicamp-dl/mMARCO) (Luiz Henrique Bonifacio, 2021) | ['cmn'] | Reranking | s2s | | None | None | -| [MMarcoRetrieval](https://arxiv.org/abs/2309.07597) (Shitao Xiao, 2024) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 114.41787048392986, 'average_query_length': 10.51131805157593, 'num_documents': 106813, 'num_queries': 6980, 'average_relevant_docs_per_query': 1.0654727793696275}} | -| [MSMARCO](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | None | {'train': {'average_document_length': 335.79716603691344, 'average_query_length': 33.21898281898998, 'num_documents': 8841823, 'num_queries': 502939, 'average_relevant_docs_per_query': 1.0592755781516248}, 'dev': {'average_document_length': 335.79716603691344, 'average_query_length': 33.2621776504298, 'num_documents': 8841823, 'num_queries': 6980, 'average_relevant_docs_per_query': 1.0654727793696275}, 'test': {'average_document_length': 335.79716603691344, 'average_query_length': 32.74418604651163, 'num_documents': 8841823, 'num_queries': 43, 'average_relevant_docs_per_query': 95.3953488372093}} | -| [MSMARCO-PL](https://microsoft.github.io/msmarco/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | {'test': {'average_document_length': 349.3574939240471, 'average_query_length': 33.02325581395349, 'num_documents': 8841823, 'num_queries': 43, 'average_relevant_docs_per_query': 95.3953488372093}} | -| [MSMARCO-PLHardNegatives](https://microsoft.github.io/msmarco/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | {'test': 43} | {'test': {'average_document_length': 382.3476426537285, 'average_query_length': 33.02325581395349, 'num_documents': 9481, 'num_queries': 43, 'average_relevant_docs_per_query': 95.3953488372093}} | -| [MSMARCOHardNegatives](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | {'test': 43} | {'test': {'average_document_length': 355.2909668633681, 'average_query_length': 32.74418604651163, 'num_documents': 8812, 'num_queries': 43, 'average_relevant_docs_per_query': 95.3953488372093}} | -======= | [JSICK](https://github.com/sbintuitions/JMTEB) (Yanaka et al., 2022) | ['jpn'] | STS | s2s | [Web, Written] | None | None | | [JSTS](https://aclanthology.org/2022.lrec-1.317.pdf#page=2.00) | ['jpn'] | STS | s2s | [Web, Written] | None | None | | [JaGovFaqsRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Web, Written] | None | None | | [JaQuADRetrieval](https://arxiv.org/abs/2202.01764) (ByungHoon So, 2022) | ['jpn'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Written] | None | None | | [JaqketRetrieval](https://github.com/kumapo/JAQKET-dataset) | ['jpn'] | Retrieval | s2p | [Encyclopaedic, Non-fiction, Written] | {'test': 115226} | {'test': {'number_of_characters': 428294530, 'num_samples': 115226, 'num_queries': 997, 'num_documents': 114229, 'min_document_length': 16, 'average_document_length': 0.44, 'max_document_length': 98, 'unique_documents': 114229, 'min_query_length': 8, 'average_query_length': 429532.57, 'max_query_length': 188424, 'unique_queries': 997, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 989}} | | [JavaneseIMDBClassification](https://github.com/w11wo/nlp-datasets#javanese-imdb) (Wongso et al., 2021) | ['jav'] | Classification | s2s | [Reviews, Written] | None | None | -| [KLUE-NLI](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | PairClassification | s2s | [News, Encyclopaedic, Written] | None | None | -| [KLUE-STS](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | STS | s2s | [Reviews, News, Spoken, Written, Spoken] | None | None | +| [KLUE-NLI](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | PairClassification | s2s | [Encyclopaedic, News, Written] | None | None | +| [KLUE-STS](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | STS | s2s | [News, Reviews, Spoken, Spoken, Written] | None | None | | [KLUE-TC](https://arxiv.org/abs/2105.09680) (Sungjoon Park, 2021) | ['kor'] | Classification | s2s | [News, Written] | None | None | | [KannadaNewsClassification](https://github.com/goru001/nlp-for-kannada) (Anoop Kunchukuttan, 2020) | ['kan'] | Classification | s2s | [News, Written] | None | None | | [KinopoiskClassification](https://www.dialog-21.ru/media/1226/blinovpd.pdf) (Blinov et al., 2013) | ['rus'] | Classification | p2p | [Reviews, Written] | None | None | | Ko-StrategyQA (Geva et al., 2021) | ['kor'] | Retrieval | s2p | | None | None | -| [KorFin](https://huggingface.co/datasets/amphora/korfin-asc) (Son et al., 2023) | ['kor'] | Classification | s2s | [News, Written] | None | None | +| [KorFin](https://huggingface.co/datasets/amphora/korfin-asc) (Son et al., 2023) | ['kor'] | Classification | s2s | [Financial, News, Written] | None | None | | [KorHateClassification](https://paperswithcode.com/dataset/korean-hatespeech-dataset) (Jihyung Moon, 2020) | ['kor'] | Classification | s2s | [Social, Written] | None | None | | [KorHateSpeechMLClassification](https://paperswithcode.com/dataset/korean-multi-label-hate-speech-dataset) | ['kor'] | MultilabelClassification | s2s | [Social, Written] | None | None | | [KorSTS](https://arxiv.org/abs/2004.03289) (Ham et al., 2020) | ['kor'] | STS | s2s | [News, Web] | None | None | @@ -548,7 +371,8 @@ The following tables give you an overview of the tasks in MTEB. | [LEMBQMSumRetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) | ['eng'] | Retrieval | s2p | [Spoken, Written] | None | None | | [LEMBSummScreenFDRetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) | ['eng'] | Retrieval | s2p | [Spoken, Written] | None | None | | [LEMBWikimQARetrieval](https://huggingface.co/datasets/dwzhu/LongEmbed) (Ho et al., 2020) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | -| [LanguageClassification](https://huggingface.co/datasets/papluca/language-identification) (Conneau et al., 2018) | ['ara', 'bul', 'cmn', 'deu', 'ell', 'eng', 'fra', 'hin', 'ita', 'jpn', 'nld', 'pol', 'por', 'rus', 'spa', 'swa', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Reviews, Web, Non-fiction, Fiction, Government, Written] | {'test': 2048, 'train': 70000} | {'test': {'num_samples': 2048, 'number_of_characters': 224352, 'num_texts_in_train': 31, 'min_text_length': 14, 'average_text_length': 109.55, 'max_text_length': 1270, 'unique_text': 2025, 'unique_labels': 20, 'labels': {'17': {'count': 102}, '0': {'count': 102}, '11': {'count': 102}, '4': {'count': 103}, '3': {'count': 102}, '1': {'count': 102}, '10': {'count': 102}, '2': {'count': 103}, '16': {'count': 103}, '9': {'count': 103}, '5': {'count': 102}, '7': {'count': 102}, '13': {'count': 102}, '14': {'count': 103}, '12': {'count': 102}, '15': {'count': 103}, '19': {'count': 102}, '18': {'count': 102}, '6': {'count': 103}, '8': {'count': 103}}}, 'train': {'num_samples': 70000, 'number_of_characters': 7760299, 'num_texts_in_train': None, 'min_text_length': 2, 'average_text_length': 110.86, 'max_text_length': 2422, 'unique_text': 68978, 'unique_labels': 20, 'labels': {'12': {'count': 3500}, '1': {'count': 3500}, '19': {'count': 3500}, '15': {'count': 3500}, '13': {'count': 3500}, '11': {'count': 3500}, '17': {'count': 3500}, '14': {'count': 3500}, '16': {'count': 3500}, '5': {'count': 3500}, '0': {'count': 3500}, '8': {'count': 3500}, '7': {'count': 3500}, '2': {'count': 3500}, '3': {'count': 3500}, '10': {'count': 3500}, '6': {'count': 3500}, '18': {'count': 3500}, '4': {'count': 3500}, '9': {'count': 3500}}}} | +| [LLaVAIT2TRetrieval](https://github.com/LinWeizheDragon/FLMR/blob/main/docs/Datasets.md) | ['eng'] | Any2AnyRetrieval | it2t | [Encyclopaedic] | None | None | +| [LanguageClassification](https://huggingface.co/datasets/papluca/language-identification) (Conneau et al., 2018) | ['ara', 'bul', 'cmn', 'deu', 'ell', 'eng', 'fra', 'hin', 'ita', 'jpn', 'nld', 'pol', 'por', 'rus', 'spa', 'swa', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Fiction, Government, Non-fiction, Reviews, Web, Written] | {'test': 2048, 'train': 70000} | {'test': {'num_samples': 2048, 'number_of_characters': 224352, 'num_texts_in_train': 31, 'min_text_length': 14, 'average_text_length': 109.55, 'max_text_length': 1270, 'unique_text': 2025, 'unique_labels': 20, 'labels': {'17': {'count': 102}, '0': {'count': 102}, '11': {'count': 102}, '4': {'count': 103}, '3': {'count': 102}, '1': {'count': 102}, '10': {'count': 102}, '2': {'count': 103}, '16': {'count': 103}, '9': {'count': 103}, '5': {'count': 102}, '7': {'count': 102}, '13': {'count': 102}, '14': {'count': 103}, '12': {'count': 102}, '15': {'count': 103}, '19': {'count': 102}, '18': {'count': 102}, '6': {'count': 103}, '8': {'count': 103}}}, 'train': {'num_samples': 70000, 'number_of_characters': 7760299, 'num_texts_in_train': None, 'min_text_length': 2, 'average_text_length': 110.86, 'max_text_length': 2422, 'unique_text': 68978, 'unique_labels': 20, 'labels': {'12': {'count': 3500}, '1': {'count': 3500}, '19': {'count': 3500}, '15': {'count': 3500}, '13': {'count': 3500}, '11': {'count': 3500}, '17': {'count': 3500}, '14': {'count': 3500}, '16': {'count': 3500}, '5': {'count': 3500}, '0': {'count': 3500}, '8': {'count': 3500}, '7': {'count': 3500}, '2': {'count': 3500}, '3': {'count': 3500}, '10': {'count': 3500}, '6': {'count': 3500}, '18': {'count': 3500}, '4': {'count': 3500}, '9': {'count': 3500}}}} | | [LccSentimentClassification](https://github.com/fnielsen/lcc-sentiment) | ['dan'] | Classification | s2s | [News, Web, Written] | None | None | | [LeCaRDv2](https://github.com/THUIR/LeCaRDv2) (Haitao Li, 2023) | ['zho'] | Retrieval | p2p | [Legal, Written] | None | None | | [LearnedHandsBenefitsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | @@ -577,21 +401,26 @@ The following tables give you an overview of the tasks in MTEB. | [LitSearchRetrieval](https://github.com/princeton-nlp/LitSearch) (Ajith et al., 2024) | ['eng'] | Retrieval | s2p | [Academic, Non-fiction, Written] | None | None | | [LivedoorNewsClustering.v2](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Clustering | s2s | [News, Written] | None | None | | [MAUDLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [METI2IRetrieval](https://arxiv.org/abs/2202.01747) (Ypsilantis et al., 2021) | ['eng'] | Any2AnyRetrieval | i2i | [Encyclopaedic] | None | None | | [MIRACLReranking](https://project-miracl.github.io/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Reranking | s2s | [Encyclopaedic, Written] | None | None | | [MIRACLRetrieval](http://miracl.ai/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | | [MIRACLRetrievalHardNegatives](http://miracl.ai/) (Zhang et al., 2023) | ['ara', 'ben', 'deu', 'eng', 'fas', 'fin', 'fra', 'hin', 'ind', 'jpn', 'kor', 'rus', 'spa', 'swa', 'tel', 'tha', 'yor', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | | [MLQARetrieval](https://huggingface.co/datasets/mlqa) | ['ara', 'deu', 'eng', 'hin', 'spa', 'vie', 'zho'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | -| [MLQuestions](https://github.com/McGill-NLP/MLQuestions) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Academic, Written] | None | None | +| [MLQuestions](https://github.com/McGill-NLP/MLQuestions) | ['eng'] | Retrieval | s2p | [Academic, Encyclopaedic, Written] | None | None | | [MLSUMClusteringP2P.v2](https://huggingface.co/datasets/mteb/mlsum) (Scialom et al., 2020) | ['deu', 'fra', 'rus', 'spa'] | Clustering | p2p | [News, Written] | None | None | | [MLSUMClusteringS2S.v2](https://huggingface.co/datasets/mteb/mlsum) (Scialom et al., 2020) | ['deu', 'fra', 'rus', 'spa'] | Clustering | s2s | [News, Written] | None | None | | [MMarcoReranking](https://github.com/unicamp-dl/mMARCO) (Luiz Henrique Bonifacio, 2021) | ['cmn'] | Reranking | s2s | | None | None | | [MMarcoRetrieval](https://arxiv.org/abs/2309.07597) (Shitao Xiao, 2024) | ['cmn'] | Retrieval | s2p | | None | None | -| [MSMARCO](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | None | None | +| [MNIST](https://en.wikipedia.org/wiki/MNIST_database) (LeCun et al., 2010) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [MNISTZeroShot](https://en.wikipedia.org/wiki/MNIST_database) (LeCun et al., 2010) | ['eng'] | ZeroShotClassification | i2t | [Encyclopaedic] | None | None | +| [MSCOCOI2TRetrieval](https://link.springer.com/chapter/10.1007/978-3-319-10602-1_48) (Lin et al., 2014) | ['eng'] | Any2AnyRetrieval | i2t | [Encyclopaedic] | None | None | +| [MSCOCOT2IRetrieval](https://link.springer.com/chapter/10.1007/978-3-319-10602-1_48) (Lin et al., 2014) | ['eng'] | Any2AnyRetrieval | t2i | [Encyclopaedic] | None | None | +| [MSMARCO](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | [Academic, Blog, Encyclopaedic, Government, Medical, News, Non-fiction, Reviews, Social, Web] | None | None | +| [MSMARCO-Fa](https://huggingface.co/datasets/MCINext/msmarco-fa) | ['fas'] | Retrieval | s2p | [Web] | None | None | | [MSMARCO-PL](https://microsoft.github.io/msmarco/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | None | | [MSMARCO-PLHardNegatives](https://microsoft.github.io/msmarco/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Web, Written] | None | None | -| [MSMARCOHardNegatives](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | None | None | ->>>>>>> main -| [MSMARCOv2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | | None | None | +| [MSMARCOHardNegatives](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | [Academic, Blog, Encyclopaedic, Government, Medical, News, Non-fiction, Reviews, Social, Web] | None | None | +| [MSMARCOv2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | [Academic, Blog, Encyclopaedic, Government, Medical, News, Non-fiction, Reviews, Social, Web] | None | None | | [MTOPDomainClassification](https://arxiv.org/pdf/2008.09335.pdf) | ['deu', 'eng', 'fra', 'hin', 'spa', 'tha'] | Classification | s2s | [Spoken, Spoken] | None | None | | [MTOPIntentClassification](https://arxiv.org/pdf/2008.09335.pdf) | ['deu', 'eng', 'fra', 'hin', 'spa', 'tha'] | Classification | s2s | [Spoken, Spoken] | None | None | | [MacedonianTweetSentimentClassification](https://aclanthology.org/R15-1034/) | ['mkd'] | Classification | s2s | [Social, Written] | None | None | @@ -599,92 +428,39 @@ The following tables give you an overview of the tasks in MTEB. | [MalteseNewsClassification](https://huggingface.co/datasets/MLRS/maltese_news_categories) | ['mlt'] | MultilabelClassification | s2s | [Constructed, Written] | None | None | | [MarathiNewsClassification](https://github.com/goru001/nlp-for-marathi) (Anoop Kunchukuttan, 2020) | ['mar'] | Classification | s2s | [News, Written] | None | None | | [MasakhaNEWSClassification](https://arxiv.org/abs/2304.09972) (David Ifeoluwa Adelani, 2023) | ['amh', 'eng', 'fra', 'hau', 'ibo', 'lin', 'lug', 'orm', 'pcm', 'run', 'sna', 'som', 'swa', 'tir', 'xho', 'yor'] | Classification | s2s | [News, Written] | None | None | -| [MasakhaNEWSClusteringP2P](https://huggingface.co/datasets/masakhane/masakhanews) (David Ifeoluwa Adelani, 2023) | ['amh', 'eng', 'fra', 'hau', 'ibo', 'lin', 'lug', 'orm', 'pcm', 'run', 'sna', 'som', 'swa', 'tir', 'xho', 'yor'] | Clustering | p2p | [News, Written, Non-fiction] | None | None | +| [MasakhaNEWSClusteringP2P](https://huggingface.co/datasets/masakhane/masakhanews) (David Ifeoluwa Adelani, 2023) | ['amh', 'eng', 'fra', 'hau', 'ibo', 'lin', 'lug', 'orm', 'pcm', 'run', 'sna', 'som', 'swa', 'tir', 'xho', 'yor'] | Clustering | p2p | [News, Non-fiction, Written] | None | None | | [MasakhaNEWSClusteringS2S](https://huggingface.co/datasets/masakhane/masakhanews) (David Ifeoluwa Adelani, 2023) | ['amh', 'eng', 'fra', 'hau', 'ibo', 'lin', 'lug', 'orm', 'pcm', 'run', 'sna', 'som', 'swa', 'tir', 'xho', 'yor'] | Clustering | s2s | | None | None | -<<<<<<< HEAD -| [MassiveIntentClassification](https://arxiv.org/abs/2204.08582) (Jack FitzGerald, 2022) | ['afr', 'amh', 'ara', 'aze', 'ben', 'cmo', 'cym', 'dan', 'deu', 'ell', 'eng', 'fas', 'fin', 'fra', 'heb', 'hin', 'hun', 'hye', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kan', 'kat', 'khm', 'kor', 'lav', 'mal', 'mon', 'msa', 'mya', 'nld', 'nob', 'pol', 'por', 'ron', 'rus', 'slv', 'spa', 'sqi', 'swa', 'swe', 'tam', 'tel', 'tgl', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Spoken] | {'validation': 2033, 'test': 2974} | {'validation': 34.8, 'test': 34.6} | -| [MassiveScenarioClassification](https://arxiv.org/abs/2204.08582) (Jack FitzGerald, 2022) | ['afr', 'amh', 'ara', 'aze', 'ben', 'cmo', 'cym', 'dan', 'deu', 'ell', 'eng', 'fas', 'fin', 'fra', 'heb', 'hin', 'hun', 'hye', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kan', 'kat', 'khm', 'kor', 'lav', 'mal', 'mon', 'msa', 'mya', 'nld', 'nob', 'pol', 'por', 'ron', 'rus', 'slv', 'spa', 'sqi', 'swa', 'swe', 'tam', 'tel', 'tgl', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Spoken] | {'validation': 2033, 'test': 2974} | {'validation': 34.8, 'test': 34.6} | -| [MedicalQARetrieval](https://bmcbioinformatics.biomedcentral.com/articles/10.1186/s12859-019-3119-4) (Asma et al., 2019) | ['eng'] | Retrieval | s2s | [Medical, Written] | {'test': 2048} | {'test': {'average_document_length': 1153.482421875, 'average_query_length': 52.4794921875, 'num_documents': 2048, 'num_queries': 2048, 'average_relevant_docs_per_query': 1.0}} | -| [MedicalRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 122.04231725066585, 'average_query_length': 17.938, 'num_documents': 100999, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}} | -| [MedrxivClusteringP2P.v2](https://api.medrxiv.org/) | ['eng'] | Clustering | p2p | [Academic, Medical, Written] | {'test': 1500} | {'test': 1984.7} | -| [MedrxivClusteringS2S.v2](https://api.medrxiv.org/) | ['eng'] | Clustering | s2s | [Academic, Medical, Written] | {'test': 1500} | {'test': 114.9} | -| [MewsC16JaClustering](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Clustering | s2s | [News, Written] | {'test': 992} | {'test': 95} | -| [MindSmallReranking](https://msnews.github.io/assets/doc/ACL2020_MIND.pdf) | ['eng'] | Reranking | s2s | [News, Written] | {'test': 107968} | {'test': 70.9} | -| MintakaRetrieval | ['ara', 'deu', 'fra', 'hin', 'ita', 'jpn', 'por', 'spa'] | Retrieval | s2p | [Encyclopaedic, Written] | None | {'test': {'ar': {'average_document_length': 12.736418511066399, 'average_query_length': 55.275533363595095, 'num_documents': 1491, 'num_queries': 2203, 'average_relevant_docs_per_query': 1.0}, 'de': {'average_document_length': 14.40060422960725, 'average_query_length': 65.41322662173546, 'num_documents': 1655, 'num_queries': 2374, 'average_relevant_docs_per_query': 1.0}, 'es': {'average_document_length': 14.291789722386296, 'average_query_length': 64.88325082508251, 'num_documents': 1693, 'num_queries': 2424, 'average_relevant_docs_per_query': 1.0}, 'fr': {'average_document_length': 14.407234539089849, 'average_query_length': 68.88452088452088, 'num_documents': 1714, 'num_queries': 2442, 'average_relevant_docs_per_query': 1.0}, 'hi': {'average_document_length': 12.71038961038961, 'average_query_length': 58.404637247569184, 'num_documents': 770, 'num_queries': 1337, 'average_relevant_docs_per_query': 1.0}, 'it': {'average_document_length': 14.365985576923077, 'average_query_length': 64.39707724425887, 'num_documents': 1664, 'num_queries': 2395, 'average_relevant_docs_per_query': 1.0004175365344468}, 'ja': {'average_document_length': 9.167713567839195, 'average_query_length': 29.961937716262977, 'num_documents': 1592, 'num_queries': 2312, 'average_relevant_docs_per_query': 1.0}, 'pt': {'average_document_length': 14.244471744471744, 'average_query_length': 60.42225998300765, 'num_documents': 1628, 'num_queries': 2354, 'average_relevant_docs_per_query': 1.0004248088360237}}} | -| [Moroco](https://huggingface.co/datasets/moroco) (Andrei M. Butnaru, 2019) | ['ron'] | Classification | s2s | [News, Written] | {'test': 2048} | {'test': 1710.94} | -| [MovieReviewSentimentClassification](https://github.com/TheophileBlard/french-sentiment-analysis-with-bert) (Théophile Blard, 2020) | ['fra'] | Classification | s2s | [Reviews, Written] | {'validation': 1024, 'test': 1024} | {'validation': 550.3, 'test': 558.1} | -| [MrTidyRetrieval](https://huggingface.co/datasets/castorini/mr-tydi) (Xinyu Zhang, 2021) | ['ara', 'ben', 'eng', 'fin', 'ind', 'jpn', 'kor', 'rus', 'swa', 'tel', 'tha'] | Retrieval | s2p | [Encyclopaedic, Written] | | | -| [MultiEURLEXMultilabelClassification](https://huggingface.co/datasets/coastalcph/multi_eurlex) (Chalkidis et al., 2021) | ['bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'est', 'fin', 'fra', 'hrv', 'hun', 'ita', 'lav', 'lit', 'mlt', 'nld', 'pol', 'por', 'ron', 'slk', 'slv', 'spa', 'swe'] | MultilabelClassification | p2p | [Legal, Government, Written] | {'test': 5000} | {'test': {'average_text_length': 12014.408930434782, 'average_label_per_text': 3.5938, 'num_samples': 115000, 'unique_labels': 21, 'labels': {'18': {'count': 50784}, '15': {'count': 30981}, '5': {'count': 24978}, '6': {'count': 45080}, '3': {'count': 63687}, '17': {'count': 37743}, '1': {'count': 15019}, '20': {'count': 14030}, '0': {'count': 17802}, '2': {'count': 22402}, '19': {'count': 10212}, '9': {'count': 3772}, '4': {'count': 9062}, '10': {'count': 7705}, '11': {'count': 12213}, '7': {'count': 14306}, '12': {'count': 11799}, '8': {'count': 13800}, '13': {'count': 2346}, '14': {'count': 4255}, '16': {'count': 1311}}, 'hf_subset_descriptive_stats': {'en': {'average_text_length': 11720.2926, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'de': {'average_text_length': 12865.4162, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'fr': {'average_text_length': 13081.1098, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'it': {'average_text_length': 12763.4786, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'es': {'average_text_length': 13080.29, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'pl': {'average_text_length': 12282.5926, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'ro': {'average_text_length': 12836.9322, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'nl': {'average_text_length': 12857.9742, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'el': {'average_text_length': 12998.143, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'hu': {'average_text_length': 12424.641, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'pt': {'average_text_length': 12482.4616, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'cs': {'average_text_length': 10783.4676, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'sv': {'average_text_length': 11612.4774, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'bg': {'average_text_length': 12235.4268, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'da': {'average_text_length': 11773.958, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'fi': {'average_text_length': 12087.6862, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'sk': {'average_text_length': 11130.814, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'lt': {'average_text_length': 11245.3566, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'hr': {'average_text_length': 11022.142, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'sl': {'average_text_length': 10620.0594, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'et': {'average_text_length': 10898.4312, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'lv': {'average_text_length': 10938.5102, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}, 'mt': {'average_text_length': 12589.7442, 'average_label_per_text': 3.5938, 'num_samples': 5000, 'unique_labels': 21, 'labels': {'18': {'count': 2208}, '15': {'count': 1347}, '5': {'count': 1086}, '6': {'count': 1960}, '3': {'count': 2769}, '17': {'count': 1641}, '1': {'count': 653}, '20': {'count': 610}, '0': {'count': 774}, '2': {'count': 974}, '19': {'count': 444}, '9': {'count': 164}, '4': {'count': 394}, '10': {'count': 335}, '11': {'count': 531}, '7': {'count': 622}, '12': {'count': 513}, '8': {'count': 600}, '13': {'count': 102}, '14': {'count': 185}, '16': {'count': 57}}}}}} | -| [MultiHateClassification](https://aclanthology.org/2022.woah-1.15/) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'nld', 'pol', 'por', 'spa'] | Classification | s2s | [Constructed, Written] | {'test': 10000} | {'test': 45.9} | -| [MultiLongDocRetrieval](https://arxiv.org/abs/2402.03216) (Jianlv Chen, 2024) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'jpn', 'kor', 'por', 'rus', 'spa', 'tha'] | Retrieval | s2p | [Encyclopaedic, Written, Web, Non-fiction, Fiction] | None | {'dev': {'ar': {'average_document_length': 29234.48153016958, 'average_query_length': 69.27, 'num_documents': 7607, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'de': {'average_document_length': 33771.2111, 'average_query_length': 153.63, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'en': {'average_document_length': 13332.76764, 'average_query_length': 81.22, 'num_documents': 200000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'es': {'average_document_length': 36567.1736990891, 'average_query_length': 123.11, 'num_documents': 9551, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'fr': {'average_document_length': 36009.4934, 'average_query_length': 142.165, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'hi': {'average_document_length': 18688.50788229112, 'average_query_length': 77.995, 'num_documents': 3806, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'it': {'average_document_length': 36633.9969, 'average_query_length': 99.615, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ja': {'average_document_length': 14480.7508, 'average_query_length': 61.625, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ko': {'average_document_length': 13813.441224093263, 'average_query_length': 58.845, 'num_documents': 6176, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'pt': {'average_document_length': 32127.576952351956, 'average_query_length': 122.275, 'num_documents': 6569, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ru': {'average_document_length': 35934.8756, 'average_query_length': 87.875, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'th': {'average_document_length': 25993.2696, 'average_query_length': 107.81, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'zh': {'average_document_length': 6039.059725, 'average_query_length': 26.79, 'num_documents': 200000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}}, 'test': {'ar': {'average_document_length': 29234.48153016958, 'average_query_length': 75.77, 'num_documents': 7607, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'de': {'average_document_length': 33771.2111, 'average_query_length': 123.65, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'en': {'average_document_length': 13332.76764, 'average_query_length': 81.33, 'num_documents': 200000, 'num_queries': 800, 'average_relevant_docs_per_query': 1.0}, 'es': {'average_document_length': 36567.1736990891, 'average_query_length': 131.985, 'num_documents': 9551, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'fr': {'average_document_length': 36009.4934, 'average_query_length': 149.795, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'hi': {'average_document_length': 18688.50788229112, 'average_query_length': 103.76, 'num_documents': 3806, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'it': {'average_document_length': 36633.9969, 'average_query_length': 114.595, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ja': {'average_document_length': 14480.7508, 'average_query_length': 55.73, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ko': {'average_document_length': 13813.441224093263, 'average_query_length': 58.72, 'num_documents': 6176, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'pt': {'average_document_length': 32127.576952351956, 'average_query_length': 113.455, 'num_documents': 6569, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'ru': {'average_document_length': 35934.8756, 'average_query_length': 94.87, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'th': {'average_document_length': 25993.2696, 'average_query_length': 97.99, 'num_documents': 10000, 'num_queries': 200, 'average_relevant_docs_per_query': 1.0}, 'zh': {'average_document_length': 6039.059725, 'average_query_length': 24.70875, 'num_documents': 200000, 'num_queries': 800, 'average_relevant_docs_per_query': 1.0}}} | -| [MultilingualSentiment](https://github.com/tyqiangz/multilingual-sentiment-datasets) | ['cmn'] | Classification | s2s | | None | None | -| [MultilingualSentimentClassification](https://huggingface.co/datasets/mteb/multilingual-sentiment-classification) | ['ara', 'bam', 'bul', 'cmn', 'cym', 'deu', 'dza', 'ell', 'eng', 'eus', 'fas', 'fin', 'heb', 'hrv', 'ind', 'jpn', 'kor', 'mlt', 'nor', 'pol', 'rus', 'slk', 'spa', 'tha', 'tur', 'uig', 'urd', 'vie', 'zho'] | Classification | s2s | [Reviews, Written] | {'test': 7000} | {'test': 56} | -| [MyanmarNews](https://huggingface.co/datasets/myanmar_news) (A. H. Khine, 2017) | ['mya'] | Classification | p2p | [News, Written] | {'train': 2048} | {'train': 174.2} | -| [NFCorpus](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1589.783925130746, 'average_query_length': 21.764705882352942, 'num_documents': 3633, 'num_queries': 323, 'average_relevant_docs_per_query': 38.18575851393189}} | -| [NFCorpus-PL](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1652.1926782273604, 'average_query_length': 24.390092879256965, 'num_documents': 3633, 'num_queries': 323, 'average_relevant_docs_per_query': 38.18575851393189}} | -| [NLPJournalAbsIntroRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Academic, Written] | {'test': 404} | {'test': {'average_document_length': 2052.8611111111113, 'average_query_length': 439.2772277227723, 'num_documents': 504, 'num_queries': 404, 'average_relevant_docs_per_query': 1.0}} | -| [NLPJournalTitleAbsRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Academic, Written] | {'test': 404} | {'test': {'average_document_length': 441.6746031746032, 'average_query_length': 27.60891089108911, 'num_documents': 504, 'num_queries': 404, 'average_relevant_docs_per_query': 1.0}} | -| [NLPJournalTitleIntroRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Academic, Written] | {'test': 404} | {'test': {'average_document_length': 2052.8611111111113, 'average_query_length': 27.60891089108911, 'num_documents': 504, 'num_queries': 404, 'average_relevant_docs_per_query': 1.0}} | -| [NQ](https://ai.google.com/research/NaturalQuestions/) (Tom Kwiatkowski, 2019) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 492.2287851281462, 'average_query_length': 48.17902665121669, 'num_documents': 2681468, 'num_queries': 3452, 'average_relevant_docs_per_query': 1.2169756662804172}} | -| [NQ-PL](https://ai.google.com/research/NaturalQuestions/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | {'test': {'average_document_length': 502.14302128535564, 'average_query_length': 48.31662804171495, 'num_documents': 2681468, 'num_queries': 3452, 'average_relevant_docs_per_query': 1.2169756662804172}} | -| [NQ-PLHardNegatives](https://ai.google.com/research/NaturalQuestions/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | {'test': 1000} | {'test': {'average_document_length': 610.7449138094336, 'average_query_length': 48.381, 'num_documents': 184765, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.213}} | -| [NQHardNegatives](https://ai.google.com/research/NaturalQuestions/) (Tom Kwiatkowski, 2019) | ['eng'] | Retrieval | s2p | | {'test': 1000} | {'test': {'average_document_length': 602.7903551179953, 'average_query_length': 47.878, 'num_documents': 198779, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.213}} | -| [NTREXBitextMining](https://huggingface.co/datasets/davidstap/NTREX) | ['afr', 'amh', 'arb', 'aze', 'bak', 'bel', 'bem', 'ben', 'bod', 'bos', 'bul', 'cat', 'ces', 'ckb', 'cym', 'dan', 'deu', 'div', 'dzo', 'ell', 'eng', 'eus', 'ewe', 'fao', 'fas', 'fij', 'fil', 'fin', 'fra', 'fuc', 'gle', 'glg', 'guj', 'hau', 'heb', 'hin', 'hmn', 'hrv', 'hun', 'hye', 'ibo', 'ind', 'isl', 'ita', 'jpn', 'kan', 'kat', 'kaz', 'khm', 'kin', 'kir', 'kmr', 'kor', 'lao', 'lav', 'lit', 'ltz', 'mal', 'mar', 'mey', 'mkd', 'mlg', 'mlt', 'mon', 'mri', 'msa', 'mya', 'nde', 'nep', 'nld', 'nno', 'nob', 'nso', 'nya', 'orm', 'pan', 'pol', 'por', 'prs', 'pus', 'ron', 'rus', 'shi', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'spa', 'sqi', 'srp', 'ssw', 'swa', 'swe', 'tah', 'tam', 'tat', 'tel', 'tgk', 'tha', 'tir', 'ton', 'tsn', 'tuk', 'tur', 'uig', 'ukr', 'urd', 'uzb', 'ven', 'vie', 'wol', 'xho', 'yor', 'yue', 'zho', 'zul'] | BitextMining | s2s | [News, Written] | {'test': 3826252} | {'test': 120} | -| [NYSJudicialEthicsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 292} | {'test': 159.45} | -| [NaijaSenti](https://github.com/hausanlp/NaijaSenti) | ['hau', 'ibo', 'pcm', 'yor'] | Classification | s2s | [Social, Written] | {'test': 4800} | {'test': 72.81} | -| [NarrativeQARetrieval](https://metatext.io/datasets/narrativeqa) (Tomáš Kočiský, 2017) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 326753.5323943662, 'average_query_length': 47.730889457232166, 'num_documents': 355, 'num_queries': 10557, 'average_relevant_docs_per_query': 1.0}} | -| [NepaliNewsClassification](https://github.com/goru001/nlp-for-nepali) | ['nep'] | Classification | s2s | [News, Written] | {'train': 5975, 'test': 1495} | {'train': 196.61, 'test': 196.017} | -| [NeuCLIR2022Retrieval](https://neuclir.github.io/) (Lawrie et al., 2023) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'fas': 2232130, 'zho': 3179323, 'rus': 4627657} | {'test': {'fas': {'average_document_length': 2032.093148525817, 'average_query_length': 85.4298245614035, 'num_documents': 2232016, 'num_queries': 114, 'average_relevant_docs_per_query': 12.912280701754385}, 'rus': {'average_document_length': 1757.9129983233004, 'average_query_length': 85.58771929824562, 'num_documents': 4627543, 'num_queries': 114, 'average_relevant_docs_per_query': 16.57017543859649}, 'zho': {'average_document_length': 743.1426659901881, 'average_query_length': 24.17543859649123, 'num_documents': 3179209, 'num_queries': 114, 'average_relevant_docs_per_query': 18.710526315789473}}} | -| [NeuCLIR2022RetrievalHardNegatives](https://neuclir.github.io/) (Lawrie et al., 2023) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | None | {'test': {'average_document_length': 2066.9453653646488, 'average_query_length': 63.529411764705884, 'num_documents': 27931, 'num_queries': 136, 'average_relevant_docs_per_query': 40.39705882352941, 'hf_subset_descriptive_stats': {'fas': {'average_document_length': 2816.847782031074, 'average_query_length': 83.26666666666667, 'num_documents': 8882, 'num_queries': 45, 'average_relevant_docs_per_query': 32.71111111111111}, 'rus': {'average_document_length': 2446.5574277854193, 'average_query_length': 85.56818181818181, 'num_documents': 8724, 'num_queries': 44, 'average_relevant_docs_per_query': 42.93181818181818}, 'zho': {'average_document_length': 1101.0984987893462, 'average_query_length': 24.0, 'num_documents': 10325, 'num_queries': 47, 'average_relevant_docs_per_query': 45.38297872340426}}}} | -| [NeuCLIR2023Retrieval](https://neuclir.github.io/) (Dawn Lawrie, 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'fas': 2232092, 'zho': 3179285, 'rus': 4627619} | {'test': {'fas': {'average_document_length': 2032.093148525817, 'average_query_length': 65.48684210526316, 'num_documents': 2232016, 'num_queries': 76, 'average_relevant_docs_per_query': 66.28947368421052}, 'rus': {'average_document_length': 1757.9129983233004, 'average_query_length': 74.4342105263158, 'num_documents': 4627543, 'num_queries': 76, 'average_relevant_docs_per_query': 62.223684210526315}, 'zho': {'average_document_length': 743.1426659901881, 'average_query_length': 22.210526315789473, 'num_documents': 3179209, 'num_queries': 76, 'average_relevant_docs_per_query': 53.68421052631579}}} | -| [NeuCLIR2023RetrievalHardNegatives](https://neuclir.github.io/) (Dawn Lawrie, 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | None | {'test': {'average_document_length': 2236.175955333482, 'average_query_length': 54.10267857142857, 'num_documents': 49433, 'num_queries': 224, 'average_relevant_docs_per_query': 61.816964285714285, 'hf_subset_descriptive_stats': {'fas': {'average_document_length': 2895.869857421016, 'average_query_length': 65.89189189189189, 'num_documents': 15921, 'num_queries': 74, 'average_relevant_docs_per_query': 68.08108108108108}, 'rus': {'average_document_length': 2724.294762109928, 'average_query_length': 74.41333333333333, 'num_documents': 16247, 'num_queries': 75, 'average_relevant_docs_per_query': 63.053333333333335}, 'zho': {'average_document_length': 1168.4984071821605, 'average_query_length': 22.16, 'num_documents': 17265, 'num_queries': 75, 'average_relevant_docs_per_query': 54.4}}}} | -| [News21InstructionRetrieval](https://arxiv.org/abs/2403.15246) (Orion Weller, 2024) | ['eng'] | InstructionRetrieval | s2p | [News, Written] | {'eng': 61906} | {'eng': 2983.724665391969} | -| [NewsClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [News, Written] | {'test': 7600} | {'test': 235.29} | -| [NoRecClassification](https://aclanthology.org/L18-1661/) | ['nob'] | Classification | s2s | [Written, Reviews] | {'test': 2050} | {'test': 82} | -| [NollySentiBitextMining](https://github.com/IyanuSh/NollySenti) (Shode et al., 2023) | ['eng', 'hau', 'ibo', 'pcm', 'yor'] | BitextMining | s2s | [Social, Reviews, Written] | {'train': 1640} | {'train': 135.91} | -| [NorQuadRetrieval](https://aclanthology.org/2023.nodalida-1.17/) | ['nob'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Written] | {'test': 2602} | {'test': {'average_document_length': 214.5114503816794, 'average_query_length': 47.896484375, 'num_documents': 1048, 'num_queries': 1024, 'average_relevant_docs_per_query': 2.0}} | -| [NordicLangClassification](https://aclanthology.org/2021.vardial-1.8/) | ['dan', 'fao', 'isl', 'nno', 'nob', 'swe'] | Classification | s2s | [Encyclopaedic] | {'test': 3000} | {'test': 78.2} | -| [NorwegianCourtsBitextMining](https://opus.nlpl.eu/index.php) (Tiedemann et al., 2020) | ['nno', 'nob'] | BitextMining | s2s | [Legal, Written] | {'test': 2050} | {'test': 1884.0} | -| [NorwegianParliamentClassification](https://huggingface.co/datasets/NbAiLab/norwegian_parliament) | ['nob'] | Classification | s2s | [Government, Spoken] | {'test': 1200, 'validation': 1200} | {'test': 1884.0, 'validation': 1911.0} | -| [NusaParagraphEmotionClassification](https://github.com/IndoNLP/nusa-writes) | ['bbc', 'bew', 'bug', 'jav', 'mad', 'mak', 'min', 'mui', 'rej', 'sun'] | Classification | s2s | [Non-fiction, Fiction, Written] | {'train': 15516, 'validation': 2948, 'test': 6250} | {'train': 740.24, 'validation': 740.66, 'test': 740.71} | -| [NusaParagraphTopicClassification](https://github.com/IndoNLP/nusa-writes) | ['bbc', 'bew', 'bug', 'jav', 'mad', 'mak', 'min', 'mui', 'rej', 'sun'] | Classification | s2s | [Non-fiction, Fiction, Written] | {'train': 15516, 'validation': 2948, 'test': 6250} | {'train': 740.24, 'validation': 740.66, 'test': 740.71} | -| [NusaTranslationBitextMining](https://huggingface.co/datasets/indonlp/nusatranslation_mt) (Cahyawijaya et al., 2023) | ['abs', 'bbc', 'bew', 'bhp', 'ind', 'jav', 'mad', 'mak', 'min', 'mui', 'rej', 'sun'] | BitextMining | s2s | [Social, Written] | {'train': 50200} | {'train': {'average_sentence1_length': 145.4552390438247, 'average_sentence2_length': 148.56607569721115, 'num_samples': 50200, 'hf_subset_descriptive_stats': {'ind-abs': {'average_sentence1_length': 148.366, 'average_sentence2_length': 147.314, 'num_samples': 1000}, 'ind-btk': {'average_sentence1_length': 145.36666666666667, 'average_sentence2_length': 146.74045454545455, 'num_samples': 6600}, 'ind-bew': {'average_sentence1_length': 145.4280303030303, 'average_sentence2_length': 148.40530303030303, 'num_samples': 6600}, 'ind-bhp': {'average_sentence1_length': 133.528, 'average_sentence2_length': 128.138, 'num_samples': 1000}, 'ind-jav': {'average_sentence1_length': 145.42772727272728, 'average_sentence2_length': 145.8089393939394, 'num_samples': 6600}, 'ind-mad': {'average_sentence1_length': 145.35545454545453, 'average_sentence2_length': 153.6228787878788, 'num_samples': 6600}, 'ind-mak': {'average_sentence1_length': 145.42772727272728, 'average_sentence2_length': 150.6128787878788, 'num_samples': 6600}, 'ind-min': {'average_sentence1_length': 145.42772727272728, 'average_sentence2_length': 148.0621212121212, 'num_samples': 6600}, 'ind-mui': {'average_sentence1_length': 150.454, 'average_sentence2_length': 150.994, 'num_samples': 1000}, 'ind-rej': {'average_sentence1_length': 151.622, 'average_sentence2_length': 139.583, 'num_samples': 1000}, 'ind-sun': {'average_sentence1_length': 145.42772727272728, 'average_sentence2_length': 150.9880303030303, 'num_samples': 6600}}}} | -| [NusaX-senti](https://arxiv.org/abs/2205.15960) (Winata et al., 2022) | ['ace', 'ban', 'bbc', 'bjn', 'bug', 'eng', 'ind', 'jav', 'mad', 'min', 'nij', 'sun'] | Classification | s2s | [Reviews, Web, Social, Constructed, Written] | {'test': 4800} | {'test': 52.4} | -| [NusaXBitextMining](https://huggingface.co/datasets/indonlp/NusaX-senti/) (Winata et al., 2023) | ['ace', 'ban', 'bbc', 'bjn', 'bug', 'eng', 'ind', 'jav', 'mad', 'min', 'nij', 'sun'] | BitextMining | s2s | [Reviews, Written] | {'train': 5500} | {'train': 157.15} | -| [OPP115DataRetentionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 88} | {'test': 195.2} | -| [OPP115DataSecurityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 1334} | {'test': 246.69} | -| [OPP115DoNotTrackLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 110} | {'test': 223.16} | -| [OPP115FirstPartyCollectionUseLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 2086} | {'test': 204.25} | -| [OPP115InternationalAndSpecificAudiencesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 980} | {'test': 327.71} | -| [OPP115PolicyChangeLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 431} | {'test': 200.99} | -| [OPP115ThirdPartySharingCollectionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 1590} | {'test': 223.64} | -| [OPP115UserAccessEditAndDeletionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 462} | {'test': 218.59} | -| [OPP115UserChoiceControlLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 1546} | {'test': 210.62} | -======= | [MassiveIntentClassification](https://arxiv.org/abs/2204.08582) (Jack FitzGerald, 2022) | ['afr', 'amh', 'ara', 'aze', 'ben', 'cmo', 'cym', 'dan', 'deu', 'ell', 'eng', 'fas', 'fin', 'fra', 'heb', 'hin', 'hun', 'hye', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kan', 'kat', 'khm', 'kor', 'lav', 'mal', 'mon', 'msa', 'mya', 'nld', 'nob', 'pol', 'por', 'ron', 'rus', 'slv', 'spa', 'sqi', 'swa', 'swe', 'tam', 'tel', 'tgl', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Spoken] | None | None | | [MassiveScenarioClassification](https://arxiv.org/abs/2204.08582) (Jack FitzGerald, 2022) | ['afr', 'amh', 'ara', 'aze', 'ben', 'cmo', 'cym', 'dan', 'deu', 'ell', 'eng', 'fas', 'fin', 'fra', 'heb', 'hin', 'hun', 'hye', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kan', 'kat', 'khm', 'kor', 'lav', 'mal', 'mon', 'msa', 'mya', 'nld', 'nob', 'pol', 'por', 'ron', 'rus', 'slv', 'spa', 'sqi', 'swa', 'swe', 'tam', 'tel', 'tgl', 'tha', 'tur', 'urd', 'vie'] | Classification | s2s | [Spoken] | None | None | | [MedicalQARetrieval](https://bmcbioinformatics.biomedcentral.com/articles/10.1186/s12859-019-3119-4) (Asma et al., 2019) | ['eng'] | Retrieval | s2s | [Medical, Written] | None | None | | [MedicalRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | None | | [MedrxivClusteringP2P.v2](https://api.medrxiv.org/) | ['eng'] | Clustering | p2p | [Academic, Medical, Written] | {'test': 37500} | {'test': {'num_samples': 37500, 'number_of_characters': 74294927, 'min_text_length': 148, 'average_text_length': 1981.2, 'max_text_length': 38759, 'min_labels_per_text': 6, 'average_labels_per_text': 1.0, 'max_labels_per_text': 8830, 'unique_labels': 51, 'labels': {'epidemiology': {'count': 6656}, 'public and global health': {'count': 3595}, 'oncology': {'count': 845}, 'allergy and immunology': {'count': 464}, 'orthopedics': {'count': 104}, 'health informatics': {'count': 1107}, 'occupational and environmental health': {'count': 415}, 'infectious diseases': {'count': 8830}, 'genetic and genomic medicine': {'count': 1918}, 'health policy': {'count': 527}, 'gastroenterology': {'count': 343}, 'radiology and imaging': {'count': 541}, 'pain medicine': {'count': 121}, 'neurology': {'count': 1773}, 'primary care research': {'count': 232}, 'rheumatology': {'count': 189}, 'endocrinology': {'count': 419}, 'hematology': {'count': 202}, 'addiction medicine': {'count': 178}, 'pediatrics': {'count': 589}, 'cardiovascular medicine': {'count': 855}, 'obstetrics and gynecology': {'count': 373}, 'health systems and quality improvement': {'count': 491}, 'nephrology': {'count': 241}, 'respiratory medicine': {'count': 482}, 'geriatric medicine': {'count': 169}, 'dentistry and oral medicine': {'count': 159}, 'psychiatry and clinical psychology': {'count': 1781}, 'nutrition': {'count': 240}, 'intensive care and critical care medicine': {'count': 368}, 'rehabilitation medicine and physical therapy': {'count': 322}, 'otolaryngology': {'count': 166}, 'nursing': {'count': 93}, 'transplantation': {'count': 118}, 'health economics': {'count': 327}, 'sports medicine': {'count': 180}, 'hiv aids': {'count': 363}, 'dermatology': {'count': 98}, 'pathology': {'count': 223}, 'emergency medicine': {'count': 191}, 'pharmacology and therapeutics': {'count': 221}, 'ophthalmology': {'count': 220}, 'medical ethics': {'count': 46}, 'palliative medicine': {'count': 45}, 'sexual and reproductive health': {'count': 156}, 'medical education': {'count': 203}, 'surgery': {'count': 162}, 'urology': {'count': 65}, 'anesthesia': {'count': 72}, 'toxicology': {'count': 16}, 'forensic medicine': {'count': 6}}}} | | [MedrxivClusteringS2S.v2](https://api.medrxiv.org/) | ['eng'] | Clustering | s2s | [Academic, Medical, Written] | {'test': 37500} | {'test': {'num_samples': 37500, 'number_of_characters': 4301276, 'min_text_length': 18, 'average_text_length': 114.7, 'max_text_length': 339, 'min_labels_per_text': 6, 'average_labels_per_text': 1.0, 'max_labels_per_text': 8830, 'unique_labels': 51, 'labels': {'epidemiology': {'count': 6656}, 'public and global health': {'count': 3595}, 'oncology': {'count': 845}, 'allergy and immunology': {'count': 464}, 'orthopedics': {'count': 104}, 'health informatics': {'count': 1107}, 'occupational and environmental health': {'count': 415}, 'infectious diseases': {'count': 8830}, 'genetic and genomic medicine': {'count': 1918}, 'health policy': {'count': 527}, 'gastroenterology': {'count': 343}, 'radiology and imaging': {'count': 541}, 'pain medicine': {'count': 121}, 'neurology': {'count': 1773}, 'primary care research': {'count': 232}, 'rheumatology': {'count': 189}, 'endocrinology': {'count': 419}, 'hematology': {'count': 202}, 'addiction medicine': {'count': 178}, 'pediatrics': {'count': 589}, 'cardiovascular medicine': {'count': 855}, 'obstetrics and gynecology': {'count': 373}, 'health systems and quality improvement': {'count': 491}, 'nephrology': {'count': 241}, 'respiratory medicine': {'count': 482}, 'geriatric medicine': {'count': 169}, 'dentistry and oral medicine': {'count': 159}, 'psychiatry and clinical psychology': {'count': 1781}, 'nutrition': {'count': 240}, 'intensive care and critical care medicine': {'count': 368}, 'rehabilitation medicine and physical therapy': {'count': 322}, 'otolaryngology': {'count': 166}, 'nursing': {'count': 93}, 'transplantation': {'count': 118}, 'health economics': {'count': 327}, 'sports medicine': {'count': 180}, 'hiv aids': {'count': 363}, 'dermatology': {'count': 98}, 'pathology': {'count': 223}, 'emergency medicine': {'count': 191}, 'pharmacology and therapeutics': {'count': 221}, 'ophthalmology': {'count': 220}, 'medical ethics': {'count': 46}, 'palliative medicine': {'count': 45}, 'sexual and reproductive health': {'count': 156}, 'medical education': {'count': 203}, 'surgery': {'count': 162}, 'urology': {'count': 65}, 'anesthesia': {'count': 72}, 'toxicology': {'count': 16}, 'forensic medicine': {'count': 6}}}} | +| [MemotionI2TRetrieval](https://aclanthology.org/2020.semeval-1.99/) (Sharma et al., 2020) | ['eng'] | Any2AnyRetrieval | i2t | [Encyclopaedic] | None | None | +| [MemotionT2IRetrieval](https://aclanthology.org/2020.semeval-1.99/) (Sharma et al., 2020) | ['eng'] | Any2AnyRetrieval | t2i | [Encyclopaedic] | None | None | | [MewsC16JaClustering](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Clustering | s2s | [News, Written] | None | None | | [MindSmallReranking](https://msnews.github.io/assets/doc/ACL2020_MIND.pdf) | ['eng'] | Reranking | s2s | [News, Written] | None | None | | MintakaRetrieval | ['ara', 'deu', 'fra', 'hin', 'ita', 'jpn', 'por', 'spa'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | | [Moroco](https://huggingface.co/datasets/moroco) (Andrei M. Butnaru, 2019) | ['ron'] | Classification | s2s | [News, Written] | None | None | | [MovieReviewSentimentClassification](https://github.com/TheophileBlard/french-sentiment-analysis-with-bert) (Théophile Blard, 2020) | ['fra'] | Classification | s2s | [Reviews, Written] | None | None | | [MrTidyRetrieval](https://huggingface.co/datasets/castorini/mr-tydi) (Xinyu Zhang, 2021) | ['ara', 'ben', 'eng', 'fin', 'ind', 'jpn', 'kor', 'rus', 'swa', 'tel', 'tha'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | -| [MultiEURLEXMultilabelClassification](https://huggingface.co/datasets/coastalcph/multi_eurlex) (Chalkidis et al., 2021) | ['bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'est', 'fin', 'fra', 'hrv', 'hun', 'ita', 'lav', 'lit', 'mlt', 'nld', 'pol', 'por', 'ron', 'slk', 'slv', 'spa', 'swe'] | MultilabelClassification | p2p | [Legal, Government, Written] | None | None | +| [MultiEURLEXMultilabelClassification](https://huggingface.co/datasets/coastalcph/multi_eurlex) (Chalkidis et al., 2021) | ['bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'est', 'fin', 'fra', 'hrv', 'hun', 'ita', 'lav', 'lit', 'mlt', 'nld', 'pol', 'por', 'ron', 'slk', 'slv', 'spa', 'swe'] | MultilabelClassification | p2p | [Government, Legal, Written] | None | None | | [MultiHateClassification](https://aclanthology.org/2022.woah-1.15/) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'nld', 'pol', 'por', 'spa'] | Classification | s2s | [Constructed, Written] | None | None | -| [MultiLongDocRetrieval](https://arxiv.org/abs/2402.03216) (Jianlv Chen, 2024) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'jpn', 'kor', 'por', 'rus', 'spa', 'tha'] | Retrieval | s2p | [Encyclopaedic, Written, Web, Non-fiction, Fiction] | None | None | +| [MultiLongDocRetrieval](https://arxiv.org/abs/2402.03216) (Jianlv Chen, 2024) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'jpn', 'kor', 'por', 'rus', 'spa', 'tha'] | Retrieval | s2p | [Encyclopaedic, Fiction, Non-fiction, Web, Written] | None | None | | [MultilingualSentiment](https://github.com/tyqiangz/multilingual-sentiment-datasets) | ['cmn'] | Classification | s2s | | None | None | | [MultilingualSentimentClassification](https://huggingface.co/datasets/mteb/multilingual-sentiment-classification) | ['ara', 'bam', 'bul', 'cmn', 'cym', 'deu', 'dza', 'ell', 'eng', 'eus', 'fas', 'fin', 'heb', 'hrv', 'ind', 'jpn', 'kor', 'mlt', 'nor', 'pol', 'rus', 'slk', 'spa', 'tha', 'tur', 'uig', 'urd', 'vie', 'zho'] | Classification | s2s | [Reviews, Written] | None | None | | [MyanmarNews](https://huggingface.co/datasets/myanmar_news) (A. H. Khine, 2017) | ['mya'] | Classification | p2p | [News, Written] | None | None | -| [NFCorpus](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | [Medical, Academic, Written] | {'test': 3956} | {'test': {'number_of_characters': 1612.55, 'num_samples': 3956, 'num_queries': 323, 'num_documents': 3633, 'average_document_length': 0.44, 'average_query_length': 0.07, 'average_relevant_docs_per_query': 38.19}} | +| [NFCorpus](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | [Academic, Medical, Written] | {'test': 3956} | {'test': {'number_of_characters': 1612.55, 'num_samples': 3956, 'num_queries': 323, 'num_documents': 3633, 'average_document_length': 0.44, 'average_query_length': 0.07, 'average_relevant_docs_per_query': 38.19}} | +| [NFCorpus-Fa](https://huggingface.co/datasets/MCINext/nfcorpus-fa) | ['fas'] | Retrieval | s2p | [Medical] | None | None | | [NFCorpus-PL](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | None | +| [NIGHTSI2IRetrieval](https://proceedings.neurips.cc/paper_files/paper/2023/hash/9f09f316a3eaf59d9ced5ffaefe97e0f-Abstract-Conference.html) (Fu et al., 2024) | ['eng'] | Any2AnyRetrieval | i2i | [Encyclopaedic] | None | None | | [NLPJournalAbsIntroRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Academic, Written] | None | None | | [NLPJournalTitleAbsRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Academic, Written] | None | None | | [NLPJournalTitleIntroRetrieval](https://github.com/sbintuitions/JMTEB) | ['jpn'] | Retrieval | s2s | [Academic, Written] | None | None | -| [NQ](https://ai.google.com/research/NaturalQuestions/) (Tom Kwiatkowski, 2019) | ['eng'] | Retrieval | s2p | | None | None | +| [NLPTwitterAnalysisClassification](https://huggingface.co/datasets/hamedhf/nlp_twitter_analysis/tree/main) | ['fas'] | Classification | s2p | [Social] | None | None | +| [NLPTwitterAnalysisClustering](https://huggingface.co/datasets/hamedhf/nlp_twitter_analysis/commits/main) | ['fas'] | Clustering | s2s | [Social] | None | None | +| [NQ](https://ai.google.com/research/NaturalQuestions/) (Tom Kwiatkowski, 2019) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [NQ-Fa](https://huggingface.co/datasets/MCINext/nq-fa) | ['fas'] | Retrieval | s2p | [Encyclopaedic] | None | None | | [NQ-PL](https://ai.google.com/research/NaturalQuestions/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | None | | [NQ-PLHardNegatives](https://ai.google.com/research/NaturalQuestions/) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | None | | [NQHardNegatives](https://ai.google.com/research/NaturalQuestions/) (Tom Kwiatkowski, 2019) | ['eng'] | Retrieval | s2p | | None | None | @@ -693,16 +469,16 @@ The following tables give you an overview of the tasks in MTEB. | [NaijaSenti](https://github.com/hausanlp/NaijaSenti) | ['hau', 'ibo', 'pcm', 'yor'] | Classification | s2s | [Social, Written] | None | None | | [NamaaMrTydiReranking](https://huggingface.co/NAMAA-Space) (Muennighoff et al., 2022) | ['ara'] | Reranking | s2s | [Encyclopaedic, Written] | None | None | | [NanoArguAnaRetrieval](http://argumentation.bplaced.net/arguana/data) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | [Medical, Written] | None | None | -| [NanoClimateFeverRetrieval](https://arxiv.org/abs/2012.00614) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | [Non-fiction, Academic, News] | None | None | +| [NanoClimateFeverRetrieval](https://arxiv.org/abs/2012.00614) (Thomas Diggelmann, 2021) | ['eng'] | Retrieval | s2p | [Academic, News, Non-fiction] | None | None | | [NanoDBPediaRetrieval](https://huggingface.co/datasets/zeta-alpha-ai/NanoDBPedia) (Lehmann et al., 2015) | ['eng'] | Retrieval | s2p | [Encyclopaedic] | None | None | | [NanoFEVERRetrieval](https://fever.ai/) | ['eng'] | Retrieval | s2p | [Academic, Encyclopaedic] | None | None | | [NanoFiQA2018Retrieval](https://sites.google.com/view/fiqa/) (Nandan Thakur, 2021) | ['eng'] | Retrieval | s2p | [Academic, Social] | None | None | | [NanoHotpotQARetrieval](https://hotpotqa.github.io/) | ['eng'] | Retrieval | s2p | [Web, Written] | None | None | | [NanoMSMARCORetrieval](https://microsoft.github.io/msmarco/) (Tri Nguyen and Mir Rosenberg and Xia Song and Jianfeng Gao and Saurabh Tiwary and Rangan Majumder and Li Deng, 2016) | ['eng'] | Retrieval | s2p | [Web] | None | None | -| [NanoNFCorpusRetrieval](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | [Medical, Academic, Written] | None | None | +| [NanoNFCorpusRetrieval](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/) (Boteva et al., 2016) | ['eng'] | Retrieval | s2p | [Academic, Medical, Written] | None | None | | [NanoNQRetrieval](https://ai.google.com/research/NaturalQuestions) (Tom Kwiatkowski, 2019) | ['eng'] | Retrieval | s2p | [Academic, Web] | None | None | | [NanoQuoraRetrieval](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (DataCanary et al., 2017) | ['eng'] | Retrieval | s2s | [Social] | None | None | -| [NanoSCIDOCSRetrieval](https://allenai.org/data/scidocs) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | [Academic, Written, Non-fiction] | None | None | +| [NanoSCIDOCSRetrieval](https://allenai.org/data/scidocs) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | [Academic, Non-fiction, Written] | None | None | | [NanoSciFactRetrieval](https://github.com/allenai/scifact) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | [Academic, Medical, Written] | None | None | | [NanoTouche2020Retrieval](https://webis.de/events/touche-20/shared-task-1.html) | ['eng'] | Retrieval | s2p | [Academic] | None | None | | [NarrativeQARetrieval](https://metatext.io/datasets/narrativeqa) (Tomáš Kočiský, 2017) | ['eng'] | Retrieval | s2p | | None | None | @@ -713,17 +489,18 @@ The following tables give you an overview of the tasks in MTEB. | [NeuCLIR2023RetrievalHardNegatives](https://neuclir.github.io/) (Dawn Lawrie, 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | None | None | | [News21InstructionRetrieval](https://arxiv.org/abs/2403.15246) (Orion Weller, 2024) | ['eng'] | InstructionRetrieval | s2p | [News, Written] | None | None | | [NewsClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [News, Written] | None | None | -| [NoRecClassification](https://aclanthology.org/L18-1661/) | ['nob'] | Classification | s2s | [Written, Reviews] | None | None | -| [NollySentiBitextMining](https://github.com/IyanuSh/NollySenti) (Shode et al., 2023) | ['eng', 'hau', 'ibo', 'pcm', 'yor'] | BitextMining | s2s | [Social, Reviews, Written] | {'train': 1640} | {'train': {'num_samples': 1640, 'number_of_characters': 445805, 'unique_pairs': 1632, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 3, 'average_sentence2_length': 135.52, 'max_sentence2_length': 1728, 'unique_sentence2': 1631, 'hf_subset_descriptive_stats': {'en-ha': {'num_samples': 410, 'number_of_characters': 115348, 'unique_pairs': 407, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 4, 'average_sentence2_length': 145.02, 'max_sentence2_length': 1728, 'unique_sentence2': 407}, 'en-ig': {'num_samples': 410, 'number_of_characters': 107173, 'unique_pairs': 409, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 5, 'average_sentence2_length': 125.08, 'max_sentence2_length': 1137, 'unique_sentence2': 408}, 'en-pcm': {'num_samples': 410, 'number_of_characters': 109955, 'unique_pairs': 408, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 3, 'average_sentence2_length': 131.87, 'max_sentence2_length': 1552, 'unique_sentence2': 408}, 'en-yo': {'num_samples': 410, 'number_of_characters': 113329, 'unique_pairs': 409, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 6, 'average_sentence2_length': 140.1, 'max_sentence2_length': 1338, 'unique_sentence2': 409}}}} | +| [NoRecClassification](https://aclanthology.org/L18-1661/) | ['nob'] | Classification | s2s | [Reviews, Written] | None | None | +| [NollySentiBitextMining](https://github.com/IyanuSh/NollySenti) (Shode et al., 2023) | ['eng', 'hau', 'ibo', 'pcm', 'yor'] | BitextMining | s2s | [Reviews, Social, Written] | {'train': 1640} | {'train': {'num_samples': 1640, 'number_of_characters': 445805, 'unique_pairs': 1632, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 3, 'average_sentence2_length': 135.52, 'max_sentence2_length': 1728, 'unique_sentence2': 1631, 'hf_subset_descriptive_stats': {'en-ha': {'num_samples': 410, 'number_of_characters': 115348, 'unique_pairs': 407, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 4, 'average_sentence2_length': 145.02, 'max_sentence2_length': 1728, 'unique_sentence2': 407}, 'en-ig': {'num_samples': 410, 'number_of_characters': 107173, 'unique_pairs': 409, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 5, 'average_sentence2_length': 125.08, 'max_sentence2_length': 1137, 'unique_sentence2': 408}, 'en-pcm': {'num_samples': 410, 'number_of_characters': 109955, 'unique_pairs': 408, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 3, 'average_sentence2_length': 131.87, 'max_sentence2_length': 1552, 'unique_sentence2': 408}, 'en-yo': {'num_samples': 410, 'number_of_characters': 113329, 'unique_pairs': 409, 'min_sentence1_length': 3, 'average_sentence1_length': 136.32, 'max_sentence1_length': 1698, 'unique_sentence1': 405, 'min_sentence2_length': 6, 'average_sentence2_length': 140.1, 'max_sentence2_length': 1338, 'unique_sentence2': 409}}}} | | [NorQuadRetrieval](https://aclanthology.org/2023.nodalida-1.17/) | ['nob'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Written] | None | None | | [NordicLangClassification](https://aclanthology.org/2021.vardial-1.8/) | ['dan', 'fao', 'isl', 'nno', 'nob', 'swe'] | Classification | s2s | [Encyclopaedic] | None | None | | [NorwegianCourtsBitextMining](https://opus.nlpl.eu/index.php) (Tiedemann et al., 2020) | ['nno', 'nob'] | BitextMining | s2s | [Legal, Written] | {'test': 228} | {'test': {'num_samples': 228, 'number_of_characters': 37441, 'unique_pairs': 228, 'min_sentence1_length': 13, 'average_sentence1_length': 82.2, 'max_sentence1_length': 272, 'unique_sentence1': 227, 'min_sentence2_length': 10, 'average_sentence2_length': 82.02, 'max_sentence2_length': 269, 'unique_sentence2': 226}} | | [NorwegianParliamentClassification](https://huggingface.co/datasets/NbAiLab/norwegian_parliament) | ['nob'] | Classification | s2s | [Government, Spoken] | None | None | -| [NusaParagraphEmotionClassification](https://github.com/IndoNLP/nusa-writes) | ['bbc', 'bew', 'bug', 'jav', 'mad', 'mak', 'min', 'mui', 'rej', 'sun'] | Classification | s2s | [Non-fiction, Fiction, Written] | None | None | -| [NusaParagraphTopicClassification](https://github.com/IndoNLP/nusa-writes) | ['bbc', 'bew', 'bug', 'jav', 'mad', 'mak', 'min', 'mui', 'rej', 'sun'] | Classification | s2s | [Non-fiction, Fiction, Written] | None | None | +| [NusaParagraphEmotionClassification](https://github.com/IndoNLP/nusa-writes) | ['bbc', 'bew', 'bug', 'jav', 'mad', 'mak', 'min', 'mui', 'rej', 'sun'] | Classification | s2s | [Fiction, Non-fiction, Written] | None | None | +| [NusaParagraphTopicClassification](https://github.com/IndoNLP/nusa-writes) | ['bbc', 'bew', 'bug', 'jav', 'mad', 'mak', 'min', 'mui', 'rej', 'sun'] | Classification | s2s | [Fiction, Non-fiction, Written] | None | None | | [NusaTranslationBitextMining](https://huggingface.co/datasets/indonlp/nusatranslation_mt) (Cahyawijaya et al., 2023) | ['abs', 'bbc', 'bew', 'bhp', 'ind', 'jav', 'mad', 'mak', 'min', 'mui', 'rej', 'sun'] | BitextMining | s2s | [Social, Written] | {'train': 50200} | {'train': {'num_samples': 50200, 'number_of_characters': 14759870, 'unique_pairs': 50140, 'min_sentence1_length': 5, 'average_sentence1_length': 145.46, 'max_sentence1_length': 873, 'unique_sentence1': 8258, 'min_sentence2_length': 5, 'average_sentence2_length': 148.57, 'max_sentence2_length': 980, 'unique_sentence2': 50102, 'hf_subset_descriptive_stats': {'ind-abs': {'num_samples': 1000, 'number_of_characters': 295680, 'unique_pairs': 999, 'min_sentence1_length': 5, 'average_sentence1_length': 148.37, 'max_sentence1_length': 727, 'unique_sentence1': 998, 'min_sentence2_length': 6, 'average_sentence2_length': 147.31, 'max_sentence2_length': 629, 'unique_sentence2': 998}, 'ind-btk': {'num_samples': 6600, 'number_of_characters': 1927907, 'unique_pairs': 6597, 'min_sentence1_length': 5, 'average_sentence1_length': 145.37, 'max_sentence1_length': 873, 'unique_sentence1': 6521, 'min_sentence2_length': 5, 'average_sentence2_length': 146.74, 'max_sentence2_length': 980, 'unique_sentence2': 6596}, 'ind-bew': {'num_samples': 6600, 'number_of_characters': 1939300, 'unique_pairs': 6595, 'min_sentence1_length': 5, 'average_sentence1_length': 145.43, 'max_sentence1_length': 873, 'unique_sentence1': 6512, 'min_sentence2_length': 6, 'average_sentence2_length': 148.41, 'max_sentence2_length': 840, 'unique_sentence2': 6590}, 'ind-bhp': {'num_samples': 1000, 'number_of_characters': 261666, 'unique_pairs': 1000, 'min_sentence1_length': 11, 'average_sentence1_length': 133.53, 'max_sentence1_length': 468, 'unique_sentence1': 999, 'min_sentence2_length': 10, 'average_sentence2_length': 128.14, 'max_sentence2_length': 459, 'unique_sentence2': 999}, 'ind-jav': {'num_samples': 6600, 'number_of_characters': 1922162, 'unique_pairs': 6594, 'min_sentence1_length': 5, 'average_sentence1_length': 145.43, 'max_sentence1_length': 873, 'unique_sentence1': 6512, 'min_sentence2_length': 5, 'average_sentence2_length': 145.81, 'max_sentence2_length': 854, 'unique_sentence2': 6585}, 'ind-mad': {'num_samples': 6600, 'number_of_characters': 1973257, 'unique_pairs': 6598, 'min_sentence1_length': 5, 'average_sentence1_length': 145.36, 'max_sentence1_length': 873, 'unique_sentence1': 6521, 'min_sentence2_length': 5, 'average_sentence2_length': 153.62, 'max_sentence2_length': 827, 'unique_sentence2': 6592}, 'ind-mak': {'num_samples': 6600, 'number_of_characters': 1953868, 'unique_pairs': 6594, 'min_sentence1_length': 5, 'average_sentence1_length': 145.43, 'max_sentence1_length': 873, 'unique_sentence1': 6512, 'min_sentence2_length': 6, 'average_sentence2_length': 150.61, 'max_sentence2_length': 888, 'unique_sentence2': 6586}, 'ind-min': {'num_samples': 6600, 'number_of_characters': 1937033, 'unique_pairs': 6595, 'min_sentence1_length': 5, 'average_sentence1_length': 145.43, 'max_sentence1_length': 873, 'unique_sentence1': 6512, 'min_sentence2_length': 6, 'average_sentence2_length': 148.06, 'max_sentence2_length': 837, 'unique_sentence2': 6591}, 'ind-mui': {'num_samples': 1000, 'number_of_characters': 301448, 'unique_pairs': 1000, 'min_sentence1_length': 11, 'average_sentence1_length': 150.45, 'max_sentence1_length': 451, 'unique_sentence1': 997, 'min_sentence2_length': 11, 'average_sentence2_length': 150.99, 'max_sentence2_length': 450, 'unique_sentence2': 1000}, 'ind-rej': {'num_samples': 1000, 'number_of_characters': 291205, 'unique_pairs': 1000, 'min_sentence1_length': 9, 'average_sentence1_length': 151.62, 'max_sentence1_length': 873, 'unique_sentence1': 998, 'min_sentence2_length': 8, 'average_sentence2_length': 139.58, 'max_sentence2_length': 784, 'unique_sentence2': 1000}, 'ind-sun': {'num_samples': 6600, 'number_of_characters': 1956344, 'unique_pairs': 6591, 'min_sentence1_length': 5, 'average_sentence1_length': 145.43, 'max_sentence1_length': 873, 'unique_sentence1': 6512, 'min_sentence2_length': 5, 'average_sentence2_length': 150.99, 'max_sentence2_length': 881, 'unique_sentence2': 6588}}}} | -| [NusaX-senti](https://arxiv.org/abs/2205.15960) (Winata et al., 2022) | ['ace', 'ban', 'bbc', 'bjn', 'bug', 'eng', 'ind', 'jav', 'mad', 'min', 'nij', 'sun'] | Classification | s2s | [Reviews, Web, Social, Constructed, Written] | None | None | +| [NusaX-senti](https://arxiv.org/abs/2205.15960) (Winata et al., 2022) | ['ace', 'ban', 'bbc', 'bjn', 'bug', 'eng', 'ind', 'jav', 'mad', 'min', 'nij', 'sun'] | Classification | s2s | [Constructed, Reviews, Social, Web, Written] | None | None | | [NusaXBitextMining](https://huggingface.co/datasets/indonlp/NusaX-senti/) (Winata et al., 2023) | ['ace', 'ban', 'bbc', 'bjn', 'bug', 'eng', 'ind', 'jav', 'mad', 'min', 'nij', 'sun'] | BitextMining | s2s | [Reviews, Written] | None | None | +| [OKVQAIT2TRetrieval](https://okvqa.allenai.org/) (Marino et al., 2019) | ['eng'] | Any2AnyRetrieval | it2t | [Encyclopaedic] | None | None | | [OPP115DataRetentionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [OPP115DataSecurityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [OPP115DoNotTrackLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | @@ -733,7 +510,8 @@ The following tables give you an overview of the tasks in MTEB. | [OPP115ThirdPartySharingCollectionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [OPP115UserAccessEditAndDeletionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [OPP115UserChoiceControlLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | ->>>>>>> main +| [OVENIT2ITRetrieval](https://openaccess.thecvf.com/content/ICCV2023/html/Hu_Open-domain_Visual_Entity_Recognition_Towards_Recognizing_Millions_of_Wikipedia_Entities_ICCV_2023_paper.html) (Hu et al., 2023) | ['eng'] | Any2AnyRetrieval | it2it | [Encyclopaedic] | None | None | +| [OVENIT2TRetrieval](https://openaccess.thecvf.com/content/ICCV2023/html/Hu_Open-domain_Visual_Entity_Recognition_Towards_Recognizing_Millions_of_Wikipedia_Entities_ICCV_2023_paper.html) (Hu et al., 2023) | ['eng'] | Any2AnyRetrieval | it2i | [Encyclopaedic] | None | None | | [Ocnli](https://arxiv.org/abs/2010.05444) (Hai Hu, 2020) | ['cmn'] | PairClassification | s2s | | None | None | | [OdiaNewsClassification](https://github.com/goru001/nlp-for-odia) (Anoop Kunchukuttan, 2020) | ['ory'] | Classification | s2s | [News, Written] | None | None | | [OnlineShopping](https://aclanthology.org/2023.nodalida-1.20/) (Xiao et al., 2023) | ['cmn'] | Classification | s2s | | None | None | @@ -741,82 +519,71 @@ The following tables give you an overview of the tasks in MTEB. | [OpusparcusPC](https://gem-benchmark.com/data_cards/opusparcus) (Mathias Creutz, 2018) | ['deu', 'eng', 'fin', 'fra', 'rus', 'swe'] | PairClassification | s2s | [Spoken, Spoken] | None | None | | [OralArgumentQuestionPurposeLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [OverrulingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [OxfordFlowersClassification](https://huggingface.co/datasets/nelorth/oxford-flowers/viewer/default/train) | ['eng'] | ImageClassification | i2i | [Reviews] | None | None | +| [OxfordPets](https://arxiv.org/abs/1306.5151) (Subhransu Maji, 2013) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [OxfordPetsZeroShot](https://arxiv.org/abs/1306.5151) (Subhransu Maji, 2013) | ['eng'] | ZeroShotClassification | i2t | [Encyclopaedic] | None | None | | [PAC](https://arxiv.org/pdf/2211.13112.pdf) (Łukasz Augustyniak, 2022) | ['pol'] | Classification | p2p | [Legal, Written] | None | None | | [PAWSX](https://aclanthology.org/2021.emnlp-main.357) (Shitao Xiao, 2024) | ['cmn'] | STS | s2s | | None | None | | [PIQA](https://arxiv.org/abs/1911.11641) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | | [PROALegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [PSC](http://www.lrec-conf.org/proceedings/lrec2014/pdf/1211_Paper.pdf) | ['pol'] | PairClassification | s2s | [News, Written] | None | None | +| [ParsinluEntail](https://github.com/persiannlp/parsinlu) | ['fas'] | PairClassification | s2s | | None | None | +| [ParsinluQueryParaphPC](https://huggingface.co/datasets/persiannlp/parsinlu_query_paraphrasing) | ['fas'] | PairClassification | s2s | | None | None | +| [PatchCamelyon](https://link.springer.com/chapter/10.1007/978-3-030-00934-2_24) | ['eng'] | ImageClassification | i2i | [Medical] | None | None | +| [PatchCamelyonZeroShot](https://link.springer.com/chapter/10.1007/978-3-030-00934-2_24) | ['eng'] | ZeroShotClassification | i2t | [Medical] | None | None | | [PatentClassification](https://aclanthology.org/P19-1212.pdf) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | -| [PawsXPairClassification](https://arxiv.org/abs/1908.11828) (Yinfei Yang, 2019) | ['cmn', 'deu', 'eng', 'fra', 'jpn', 'kor', 'spa'] | PairClassification | s2s | [Web, Encyclopaedic, Written] | {'test': 14000, 'validation': 14000} | {'test': {'num_samples': 14000, 'number_of_characters': 2551922, 'min_sentence1_length': 2, 'avg_sentence1_length': 91.18, 'max_sentence1_length': 268, 'unique_sentence1': 13404, 'min_sentence2_length': 2, 'avg_sentence2_length': 91.1, 'max_sentence2_length': 247, 'unique_sentence2': 13462, 'unique_labels': 2, 'labels': {'1': {'count': 6285}, '0': {'count': 7715}}, 'hf_subset_descriptive_stats': {'de': {'num_samples': 2000, 'number_of_characters': 478034, 'min_sentence1_length': 2, 'avg_sentence1_length': 119.78, 'max_sentence1_length': 268, 'unique_sentence1': 1934, 'min_sentence2_length': 2, 'avg_sentence2_length': 119.24, 'max_sentence2_length': 235, 'unique_sentence2': 1938, 'unique_labels': 2, 'labels': {'1': {'count': 895}, '0': {'count': 1105}}}, 'en': {'num_samples': 2000, 'number_of_characters': 454362, 'min_sentence1_length': 25, 'avg_sentence1_length': 113.76, 'max_sentence1_length': 209, 'unique_sentence1': 1761, 'min_sentence2_length': 25, 'avg_sentence2_length': 113.42, 'max_sentence2_length': 209, 'unique_sentence2': 1800, 'unique_labels': 2, 'labels': {'1': {'count': 907}, '0': {'count': 1093}}}, 'es': {'num_samples': 2000, 'number_of_characters': 471226, 'min_sentence1_length': 2, 'avg_sentence1_length': 117.81, 'max_sentence1_length': 226, 'unique_sentence1': 1955, 'min_sentence2_length': 22, 'avg_sentence2_length': 117.8, 'max_sentence2_length': 233, 'unique_sentence2': 1959, 'unique_labels': 2, 'labels': {'1': {'count': 907}, '0': {'count': 1093}}}, 'fr': {'num_samples': 2000, 'number_of_characters': 480033, 'min_sentence1_length': 2, 'avg_sentence1_length': 120.03, 'max_sentence1_length': 238, 'unique_sentence1': 1954, 'min_sentence2_length': 2, 'avg_sentence2_length': 119.99, 'max_sentence2_length': 247, 'unique_sentence2': 1953, 'unique_labels': 2, 'labels': {'1': {'count': 903}, '0': {'count': 1097}}}, 'ja': {'num_samples': 2000, 'number_of_characters': 235106, 'min_sentence1_length': 2, 'avg_sentence1_length': 58.68, 'max_sentence1_length': 192, 'unique_sentence1': 1944, 'min_sentence2_length': 2, 'avg_sentence2_length': 58.88, 'max_sentence2_length': 198, 'unique_sentence2': 1941, 'unique_labels': 2, 'labels': {'1': {'count': 883}, '0': {'count': 1117}}}, 'ko': {'num_samples': 2000, 'number_of_characters': 260149, 'min_sentence1_length': 2, 'avg_sentence1_length': 64.96, 'max_sentence1_length': 153, 'unique_sentence1': 1954, 'min_sentence2_length': 2, 'avg_sentence2_length': 65.11, 'max_sentence2_length': 159, 'unique_sentence2': 1969, 'unique_labels': 2, 'labels': {'1': {'count': 896}, '0': {'count': 1104}}}, 'zh': {'num_samples': 2000, 'number_of_characters': 173012, 'min_sentence1_length': 2, 'avg_sentence1_length': 43.23, 'max_sentence1_length': 120, 'unique_sentence1': 1909, 'min_sentence2_length': 2, 'avg_sentence2_length': 43.27, 'max_sentence2_length': 113, 'unique_sentence2': 1909, 'unique_labels': 2, 'labels': {'1': {'count': 894}, '0': {'count': 1106}}}}}, 'validation': {'num_samples': 14000, 'number_of_characters': 2524625, 'min_sentence1_length': 2, 'avg_sentence1_length': 90.13, 'max_sentence1_length': 248, 'unique_sentence1': 13357, 'min_sentence2_length': 2, 'avg_sentence2_length': 90.2, 'max_sentence2_length': 275, 'unique_sentence2': 13397, 'unique_labels': 2, 'labels': {'1': {'count': 5948}, '0': {'count': 8052}}, 'hf_subset_descriptive_stats': {'de': {'num_samples': 2000, 'number_of_characters': 467643, 'min_sentence1_length': 2, 'avg_sentence1_length': 116.82, 'max_sentence1_length': 248, 'unique_sentence1': 1914, 'min_sentence2_length': 2, 'avg_sentence2_length': 117.0, 'max_sentence2_length': 275, 'unique_sentence2': 1920, 'unique_labels': 2, 'labels': {'1': {'count': 831}, '0': {'count': 1169}}}, 'en': {'num_samples': 2000, 'number_of_characters': 451931, 'min_sentence1_length': 25, 'avg_sentence1_length': 113.11, 'max_sentence1_length': 213, 'unique_sentence1': 1758, 'min_sentence2_length': 25, 'avg_sentence2_length': 112.86, 'max_sentence2_length': 213, 'unique_sentence2': 1771, 'unique_labels': 2, 'labels': {'1': {'count': 863}, '0': {'count': 1137}}}, 'es': {'num_samples': 2000, 'number_of_characters': 466112, 'min_sentence1_length': 2, 'avg_sentence1_length': 116.33, 'max_sentence1_length': 240, 'unique_sentence1': 1938, 'min_sentence2_length': 2, 'avg_sentence2_length': 116.73, 'max_sentence2_length': 241, 'unique_sentence2': 1941, 'unique_labels': 2, 'labels': {'1': {'count': 847}, '0': {'count': 1153}}}, 'fr': {'num_samples': 2000, 'number_of_characters': 478510, 'min_sentence1_length': 2, 'avg_sentence1_length': 119.5, 'max_sentence1_length': 233, 'unique_sentence1': 1933, 'min_sentence2_length': 2, 'avg_sentence2_length': 119.75, 'max_sentence2_length': 246, 'unique_sentence2': 1939, 'unique_labels': 2, 'labels': {'1': {'count': 860}, '0': {'count': 1140}}}, 'ja': {'num_samples': 2000, 'number_of_characters': 229655, 'min_sentence1_length': 2, 'avg_sentence1_length': 57.51, 'max_sentence1_length': 126, 'unique_sentence1': 1957, 'min_sentence2_length': 2, 'avg_sentence2_length': 57.32, 'max_sentence2_length': 121, 'unique_sentence2': 1969, 'unique_labels': 2, 'labels': {'1': {'count': 854}, '0': {'count': 1146}}}, 'ko': {'num_samples': 2000, 'number_of_characters': 261355, 'min_sentence1_length': 2, 'avg_sentence1_length': 65.16, 'max_sentence1_length': 178, 'unique_sentence1': 1963, 'min_sentence2_length': 2, 'avg_sentence2_length': 65.52, 'max_sentence2_length': 174, 'unique_sentence2': 1968, 'unique_labels': 2, 'labels': {'1': {'count': 840}, '0': {'count': 1160}}}, 'zh': {'num_samples': 2000, 'number_of_characters': 169419, 'min_sentence1_length': 2, 'avg_sentence1_length': 42.45, 'max_sentence1_length': 101, 'unique_sentence1': 1899, 'min_sentence2_length': 2, 'avg_sentence2_length': 42.26, 'max_sentence2_length': 120, 'unique_sentence2': 1895, 'unique_labels': 2, 'labels': {'1': {'count': 853}, '0': {'count': 1147}}}}}} | +| [PawsXPairClassification](https://arxiv.org/abs/1908.11828) (Yinfei Yang, 2019) | ['cmn', 'deu', 'eng', 'fra', 'jpn', 'kor', 'spa'] | PairClassification | s2s | [Encyclopaedic, Web, Written] | {'test': 14000, 'validation': 14000} | {'test': {'num_samples': 14000, 'number_of_characters': 2551922, 'min_sentence1_length': 2, 'avg_sentence1_length': 91.18, 'max_sentence1_length': 268, 'unique_sentence1': 13404, 'min_sentence2_length': 2, 'avg_sentence2_length': 91.1, 'max_sentence2_length': 247, 'unique_sentence2': 13462, 'unique_labels': 2, 'labels': {'1': {'count': 6285}, '0': {'count': 7715}}, 'hf_subset_descriptive_stats': {'de': {'num_samples': 2000, 'number_of_characters': 478034, 'min_sentence1_length': 2, 'avg_sentence1_length': 119.78, 'max_sentence1_length': 268, 'unique_sentence1': 1934, 'min_sentence2_length': 2, 'avg_sentence2_length': 119.24, 'max_sentence2_length': 235, 'unique_sentence2': 1938, 'unique_labels': 2, 'labels': {'1': {'count': 895}, '0': {'count': 1105}}}, 'en': {'num_samples': 2000, 'number_of_characters': 454362, 'min_sentence1_length': 25, 'avg_sentence1_length': 113.76, 'max_sentence1_length': 209, 'unique_sentence1': 1761, 'min_sentence2_length': 25, 'avg_sentence2_length': 113.42, 'max_sentence2_length': 209, 'unique_sentence2': 1800, 'unique_labels': 2, 'labels': {'1': {'count': 907}, '0': {'count': 1093}}}, 'es': {'num_samples': 2000, 'number_of_characters': 471226, 'min_sentence1_length': 2, 'avg_sentence1_length': 117.81, 'max_sentence1_length': 226, 'unique_sentence1': 1955, 'min_sentence2_length': 22, 'avg_sentence2_length': 117.8, 'max_sentence2_length': 233, 'unique_sentence2': 1959, 'unique_labels': 2, 'labels': {'1': {'count': 907}, '0': {'count': 1093}}}, 'fr': {'num_samples': 2000, 'number_of_characters': 480033, 'min_sentence1_length': 2, 'avg_sentence1_length': 120.03, 'max_sentence1_length': 238, 'unique_sentence1': 1954, 'min_sentence2_length': 2, 'avg_sentence2_length': 119.99, 'max_sentence2_length': 247, 'unique_sentence2': 1953, 'unique_labels': 2, 'labels': {'1': {'count': 903}, '0': {'count': 1097}}}, 'ja': {'num_samples': 2000, 'number_of_characters': 235106, 'min_sentence1_length': 2, 'avg_sentence1_length': 58.68, 'max_sentence1_length': 192, 'unique_sentence1': 1944, 'min_sentence2_length': 2, 'avg_sentence2_length': 58.88, 'max_sentence2_length': 198, 'unique_sentence2': 1941, 'unique_labels': 2, 'labels': {'1': {'count': 883}, '0': {'count': 1117}}}, 'ko': {'num_samples': 2000, 'number_of_characters': 260149, 'min_sentence1_length': 2, 'avg_sentence1_length': 64.96, 'max_sentence1_length': 153, 'unique_sentence1': 1954, 'min_sentence2_length': 2, 'avg_sentence2_length': 65.11, 'max_sentence2_length': 159, 'unique_sentence2': 1969, 'unique_labels': 2, 'labels': {'1': {'count': 896}, '0': {'count': 1104}}}, 'zh': {'num_samples': 2000, 'number_of_characters': 173012, 'min_sentence1_length': 2, 'avg_sentence1_length': 43.23, 'max_sentence1_length': 120, 'unique_sentence1': 1909, 'min_sentence2_length': 2, 'avg_sentence2_length': 43.27, 'max_sentence2_length': 113, 'unique_sentence2': 1909, 'unique_labels': 2, 'labels': {'1': {'count': 894}, '0': {'count': 1106}}}}}, 'validation': {'num_samples': 14000, 'number_of_characters': 2524625, 'min_sentence1_length': 2, 'avg_sentence1_length': 90.13, 'max_sentence1_length': 248, 'unique_sentence1': 13357, 'min_sentence2_length': 2, 'avg_sentence2_length': 90.2, 'max_sentence2_length': 275, 'unique_sentence2': 13397, 'unique_labels': 2, 'labels': {'1': {'count': 5948}, '0': {'count': 8052}}, 'hf_subset_descriptive_stats': {'de': {'num_samples': 2000, 'number_of_characters': 467643, 'min_sentence1_length': 2, 'avg_sentence1_length': 116.82, 'max_sentence1_length': 248, 'unique_sentence1': 1914, 'min_sentence2_length': 2, 'avg_sentence2_length': 117.0, 'max_sentence2_length': 275, 'unique_sentence2': 1920, 'unique_labels': 2, 'labels': {'1': {'count': 831}, '0': {'count': 1169}}}, 'en': {'num_samples': 2000, 'number_of_characters': 451931, 'min_sentence1_length': 25, 'avg_sentence1_length': 113.11, 'max_sentence1_length': 213, 'unique_sentence1': 1758, 'min_sentence2_length': 25, 'avg_sentence2_length': 112.86, 'max_sentence2_length': 213, 'unique_sentence2': 1771, 'unique_labels': 2, 'labels': {'1': {'count': 863}, '0': {'count': 1137}}}, 'es': {'num_samples': 2000, 'number_of_characters': 466112, 'min_sentence1_length': 2, 'avg_sentence1_length': 116.33, 'max_sentence1_length': 240, 'unique_sentence1': 1938, 'min_sentence2_length': 2, 'avg_sentence2_length': 116.73, 'max_sentence2_length': 241, 'unique_sentence2': 1941, 'unique_labels': 2, 'labels': {'1': {'count': 847}, '0': {'count': 1153}}}, 'fr': {'num_samples': 2000, 'number_of_characters': 478510, 'min_sentence1_length': 2, 'avg_sentence1_length': 119.5, 'max_sentence1_length': 233, 'unique_sentence1': 1933, 'min_sentence2_length': 2, 'avg_sentence2_length': 119.75, 'max_sentence2_length': 246, 'unique_sentence2': 1939, 'unique_labels': 2, 'labels': {'1': {'count': 860}, '0': {'count': 1140}}}, 'ja': {'num_samples': 2000, 'number_of_characters': 229655, 'min_sentence1_length': 2, 'avg_sentence1_length': 57.51, 'max_sentence1_length': 126, 'unique_sentence1': 1957, 'min_sentence2_length': 2, 'avg_sentence2_length': 57.32, 'max_sentence2_length': 121, 'unique_sentence2': 1969, 'unique_labels': 2, 'labels': {'1': {'count': 854}, '0': {'count': 1146}}}, 'ko': {'num_samples': 2000, 'number_of_characters': 261355, 'min_sentence1_length': 2, 'avg_sentence1_length': 65.16, 'max_sentence1_length': 178, 'unique_sentence1': 1963, 'min_sentence2_length': 2, 'avg_sentence2_length': 65.52, 'max_sentence2_length': 174, 'unique_sentence2': 1968, 'unique_labels': 2, 'labels': {'1': {'count': 840}, '0': {'count': 1160}}}, 'zh': {'num_samples': 2000, 'number_of_characters': 169419, 'min_sentence1_length': 2, 'avg_sentence1_length': 42.45, 'max_sentence1_length': 101, 'unique_sentence1': 1899, 'min_sentence2_length': 2, 'avg_sentence2_length': 42.26, 'max_sentence2_length': 120, 'unique_sentence2': 1895, 'unique_labels': 2, 'labels': {'1': {'count': 853}, '0': {'count': 1147}}}}}} | | [PersianFoodSentimentClassification](https://hooshvare.github.io/docs/datasets/sa) (Mehrdad Farahani et al., 2020) | ['fas'] | Classification | s2s | [Reviews, Written] | None | None | +| [PersianTextEmotion](https://huggingface.co/datasets/SeyedAli/Persian-Text-Emotion) | ['fas'] | Classification | s2s | | None | None | +| [PersianTextTone](https://mcinext.com/) | ['fas'] | Classification | s2p | | None | None | +| [PersianWebDocumentRetrieval](https://ieeexplore.ieee.org/document/10553090) | ['fas'] | Retrieval | s2p | [Web] | None | None | | [PersonalJurisdictionLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [PhincBitextMining](https://huggingface.co/datasets/veezbo/phinc) (Srivastava et al., 2020) | ['eng', 'hin'] | BitextMining | s2s | [Social, Written] | {'train': 13738} | {'train': {'num_samples': 13738, 'number_of_characters': 2069457, 'unique_pairs': 13737, 'min_sentence1_length': 1, 'average_sentence1_length': 74.02, 'max_sentence1_length': 278, 'unique_sentence1': 13515, 'min_sentence2_length': 3, 'average_sentence2_length': 76.61, 'max_sentence2_length': 274, 'unique_sentence2': 13736, 'hf_subset_descriptive_stats': {'eng-eng_hin': {'num_samples': 13738, 'number_of_characters': 2069457, 'unique_pairs': 13737, 'min_sentence1_length': 1, 'average_sentence1_length': 74.02, 'max_sentence1_length': 278, 'unique_sentence1': 13515, 'min_sentence2_length': 3, 'average_sentence2_length': 76.61, 'max_sentence2_length': 274, 'unique_sentence2': 13736}}}} | | [PlscClusteringP2P.v2](https://huggingface.co/datasets/rafalposwiata/plsc) | ['pol'] | Clustering | s2s | [Academic, Written] | None | None | | [PlscClusteringS2S.v2](https://huggingface.co/datasets/rafalposwiata/plsc) | ['pol'] | Clustering | s2s | [Academic, Written] | None | None | | [PoemSentimentClassification](https://arxiv.org/abs/2011.02686) (Emily Sheng, 2020) | ['eng'] | Classification | s2s | [Reviews, Written] | None | None | -| [PolEmo2.0-IN](https://aclanthology.org/K19-1092.pdf) | ['pol'] | Classification | s2s | [Written, Social] | None | None | -| [PolEmo2.0-OUT](https://aclanthology.org/K19-1092.pdf) | ['pol'] | Classification | s2s | [Written, Social] | None | None | -| [PpcPC](https://arxiv.org/pdf/2207.12759.pdf) (Sławomir Dadas, 2022) | ['pol'] | PairClassification | s2s | [Fiction, Non-fiction, Web, Written, Spoken, Social, News] | None | None | -| [PublicHealthQA](https://huggingface.co/datasets/xhluca/publichealth-qa) | ['ara', 'eng', 'fra', 'kor', 'rus', 'spa', 'vie', 'zho'] | Retrieval | s2p | [Medical, Government, Web, Written] | None | None | +| [PolEmo2.0-IN](https://aclanthology.org/K19-1092.pdf) | ['pol'] | Classification | s2s | [Social, Written] | None | None | +| [PolEmo2.0-OUT](https://aclanthology.org/K19-1092.pdf) | ['pol'] | Classification | s2s | [Social, Written] | None | None | +| [PpcPC](https://arxiv.org/pdf/2207.12759.pdf) (Sławomir Dadas, 2022) | ['pol'] | PairClassification | s2s | [Fiction, News, Non-fiction, Social, Spoken, Web, Written] | None | None | +| [PubChemAISentenceParaphrasePC](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | PairClassification | s2s | [Chemistry] | None | None | +| [PubChemSMILESBitextMining](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | BitextMining | s2s | [Chemistry] | None | None | +| [PubChemSMILESPC](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | PairClassification | s2s | [Chemistry] | None | None | +| [PubChemSynonymPC](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | PairClassification | s2s | [Chemistry] | None | None | +| [PubChemWikiPairClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['ces', 'deu', 'eng', 'fra', 'hin', 'jpn', 'kor', 'msa', 'nld', 'por', 'spa', 'tur', 'zho'] | PairClassification | s2s | [Chemistry] | None | None | +| [PubChemWikiParagraphsPC](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | PairClassification | p2p | [Chemistry] | None | None | +| [PublicHealthQA](https://huggingface.co/datasets/xhluca/publichealth-qa) | ['ara', 'eng', 'fra', 'kor', 'rus', 'spa', 'vie', 'zho'] | Retrieval | s2p | [Government, Medical, Web, Written] | None | None | | [PunjabiNewsClassification](https://github.com/goru001/nlp-for-punjabi/) (Anoop Kunchukuttan, 2020) | ['pan'] | Classification | s2s | [News, Written] | None | None | | [QBQTC](https://github.com/CLUEbenchmark/QBQTC/tree/main/dataset) | ['cmn'] | STS | s2s | | None | None | -<<<<<<< HEAD -| [Quail](https://text-machine.cs.uml.edu/lab2/projects/quail/) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 2720} | {'test': {'average_document_length': 27.50788422240522, 'average_query_length': 1957.3632352941177, 'num_documents': 32787, 'num_queries': 2720, 'average_relevant_docs_per_query': 1.0}} | -| [Quora-PL](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2s | | None | {'validation': {'average_document_length': 65.82473022253414, 'average_query_length': 54.6006, 'num_documents': 522931, 'num_queries': 5000, 'average_relevant_docs_per_query': 1.5252}, 'test': {'average_document_length': 65.82473022253414, 'average_query_length': 54.5354, 'num_documents': 522931, 'num_queries': 10000, 'average_relevant_docs_per_query': 1.5675}} | -| [Quora-PLHardNegatives](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2s | | {'test': 1000} | {'test': {'average_document_length': 67.77529631287385, 'average_query_length': 53.846, 'num_documents': 172031, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.641}} | -| [QuoraRetrieval](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (DataCanary et al., 2017) | ['eng'] | Retrieval | s2s | | None | {'dev': {'average_document_length': 62.158154708747425, 'average_query_length': 51.5342, 'num_documents': 522931, 'num_queries': 5000, 'average_relevant_docs_per_query': 1.5252}, 'test': {'average_document_length': 62.158154708747425, 'average_query_length': 51.5396, 'num_documents': 522931, 'num_queries': 10000, 'average_relevant_docs_per_query': 1.5675}} | -| [QuoraRetrievalHardNegatives](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (DataCanary et al., 2017) | ['eng'] | Retrieval | s2s | | {'test': 1000} | {'test': {'average_document_length': 58.96963812985781, 'average_query_length': 51.228, 'num_documents': 177163, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.641}} | -| [RARbCode](https://arxiv.org/abs/2404.06347) (Xiao et al., 2024) | ['eng'] | Retrieval | s2p | [Programming, Written] | {'test': 1484} | {'test': {'average_document_length': 793.6813076734267, 'average_query_length': 375.7506738544474, 'num_documents': 301482, 'num_queries': 1484, 'average_relevant_docs_per_query': 1.0}} | -| [RARbMath](https://arxiv.org/abs/2404.06347) (Xiao et al., 2024) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 6319} | {'test': {'average_document_length': 504.0197829347469, 'average_query_length': 210.30732710871973, 'num_documents': 389376, 'num_queries': 6319, 'average_relevant_docs_per_query': 1.0}} | -| [RTE3](https://aclanthology.org/W07-1401/) | ['deu', 'eng', 'fra', 'ita'] | PairClassification | s2s | [News, Web, Encyclopaedic, Written] | {'test': 1923} | {'test': 124.79} | -| [RUParaPhraserSTS](https://aclanthology.org/2020.ngt-1.6) (Pivovarova et al., 2017) | ['rus'] | STS | s2s | [News, Written] | {'test': 1924} | {'test': 61.25} | -| [RedditClustering.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | s2s | [Web, Social, Written] | {'test': 32768} | {'test': 64.7} | -| [RedditClusteringP2P.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | p2p | [Web, Social, Written] | {'test': 18375} | {'test': 727.7} | -| [RestaurantReviewSentimentClassification](https://link.springer.com/chapter/10.1007/978-3-319-18117-2_2) (ElSahar et al., 2015) | ['ara'] | Classification | s2s | [Reviews, Written] | {'train': 2048} | {'train': 231.4} | -| [RiaNewsRetrieval](https://arxiv.org/abs/1901.07786) (Gavrilov et al., 2019) | ['rus'] | Retrieval | s2p | [News, Written] | {'test': 10000} | {'test': {'average_document_length': 1165.6429557148213, 'average_query_length': 62.4029, 'num_documents': 704344, 'num_queries': 10000, 'average_relevant_docs_per_query': 1.0}} | -| [RiaNewsRetrievalHardNegatives](https://arxiv.org/abs/1901.07786) (Gavrilov et al., 2019) | ['rus'] | Retrieval | s2p | [News, Written] | {'test': 1000} | {'test': {'average_document_length': 1225.7253146619116, 'average_query_length': 62.338, 'num_documents': 191237, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}} | -| [Robust04InstructionRetrieval](https://arxiv.org/abs/2403.15246) (Orion Weller, 2024) | ['eng'] | InstructionRetrieval | s2p | [News, Written] | {'eng': 95088} | {'eng': 2471.0398058252426} | -| [RomaTalesBitextMining](https://idoc.pub/documents/idocpub-zpnxm9g35ylv) | ['hun', 'rom'] | BitextMining | s2s | [Fiction, Written] | {'test': 215} | {'test': 316.8046511627907} | -| [RomaniBibleClustering](https://romani.global.bible/info) | ['rom'] | Clustering | p2p | [Religious, Written] | {'test': 2048} | {'test': 132.2} | -| [RomanianReviewsSentiment](https://arxiv.org/abs/2101.04197) (Anca Maria Tache, 2021) | ['ron'] | Classification | s2s | [Reviews, Written] | {'test': 2048} | {'test': 588.6} | -| [RomanianSentimentClassification](https://arxiv.org/abs/2009.08712) (Dumitrescu et al., 2020) | ['ron'] | Classification | s2s | [Reviews, Written] | {'test': 2048} | {'test': 67.6} | -| [RonSTS](https://openreview.net/forum?id=JH61CD7afTv) (Dumitrescu et al., 2021) | ['ron'] | STS | s2s | [News, Social, Web, Written] | {'test': 1379} | {'test': 60.5} | -| [RuBQReranking](https://openreview.net/pdf?id=P5UQFFoQ4PJ) (Ivan Rybin, 2021) | ['rus'] | Reranking | s2p | [Encyclopaedic, Written] | {'test': 1551} | {'test': 499.9} | -| [RuBQRetrieval](https://openreview.net/pdf?id=P5UQFFoQ4PJ) (Ivan Rybin, 2021) | ['rus'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 2845} | {'test': {'average_document_length': 448.94659134903037, 'average_query_length': 45.29609929078014, 'num_documents': 56826, 'num_queries': 1692, 'average_relevant_docs_per_query': 1.6814420803782506}} | -| [RuReviewsClassification](https://github.com/sismetanin/rureviews) (Sergey Smetanin, 2019) | ['rus'] | Classification | p2p | [Reviews, Written] | {'test': 2048} | {'test': 133.2} | -| [RuSTSBenchmarkSTS](https://github.com/PhilipMay/stsb-multi-mt/) (Philip May, 2021) | ['rus'] | STS | s2s | [News, Social, Web, Written] | {'test': 1264} | {'test': 54.2} | -| [RuSciBenchGRNTIClassification](https://github.com/mlsa-iai-msu-lab/ru_sci_bench/) | ['rus'] | Classification | p2p | [Academic, Written] | {'test': 2048} | {'test': 890.1} | -| [RuSciBenchGRNTIClusteringP2P](https://github.com/mlsa-iai-msu-lab/ru_sci_bench/) | ['rus'] | Clustering | p2p | [Academic, Written] | {'test': 2048} | {'test': {'num_samples': 2048, 'average_text_length': 889.81396484375, 'average_labels_per_text': 1.0, 'unique_labels': 28, 'labels': {'3': {'count': 73}, '4': {'count': 73}, '20': {'count': 73}, '9': {'count': 73}, '21': {'count': 73}, '15': {'count': 73}, '16': {'count': 74}, '2': {'count': 73}, '8': {'count': 73}, '23': {'count': 73}, '6': {'count': 73}, '24': {'count': 73}, '10': {'count': 73}, '1': {'count': 73}, '17': {'count': 74}, '14': {'count': 74}, '18': {'count': 73}, '27': {'count': 73}, '19': {'count': 73}, '22': {'count': 73}, '12': {'count': 73}, '25': {'count': 73}, '5': {'count': 74}, '0': {'count': 73}, '26': {'count': 73}, '11': {'count': 73}, '13': {'count': 73}, '7': {'count': 73}}}} | -| [RuSciBenchOECDClassification](https://github.com/mlsa-iai-msu-lab/ru_sci_bench/) | ['rus'] | Classification | p2p | [Academic, Written] | {'test': 2048} | {'test': 838.9} | -| [RuSciBenchOECDClusteringP2P](https://github.com/mlsa-iai-msu-lab/ru_sci_bench/) | ['rus'] | Clustering | p2p | [Academic, Written] | {'test': 2048} | {'test': 838.9} | -| [SCDBPAccountabilityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 379} | {'test': 3520} | -| [SCDBPAuditsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 379} | {'test': 3507} | -| [SCDBPCertificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 378} | {'test': 3507} | -| [SCDBPTrainingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 379} | {'test': 3506} | -| [SCDBPVerificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 379} | {'test': 3498} | -| [SCDDAccountabilityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 378} | {'test': 3522} | -| [SCDDAuditsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 379} | {'test': 3506} | -| [SCDDCertificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 378} | {'test': 3518} | -| [SCDDTrainingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 379} | {'test': 3499} | -| [SCDDVerificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 379} | {'test': 3503} | -| [SCIDOCS](https://allenai.org/data/scidocs) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | [Academic, Written, Non-fiction] | None | {'test': {'average_document_length': 1203.3659819932182, 'average_query_length': 71.632, 'num_documents': 25657, 'num_queries': 1000, 'average_relevant_docs_per_query': 4.928}} | -| [SCIDOCS-PL](https://allenai.org/data/scidocs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1270.0791986592353, 'average_query_length': 80.671, 'num_documents': 25657, 'num_queries': 1000, 'average_relevant_docs_per_query': 4.928}} | -| [SIB200Classification](https://arxiv.org/abs/2309.07445) (Adelani et al., 2023) | ['ace', 'acm', 'acq', 'aeb', 'afr', 'ajp', 'aka', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'ast', 'awa', 'ayr', 'azb', 'azj', 'bak', 'bam', 'ban', 'bel', 'bem', 'ben', 'bho', 'bjn', 'bod', 'bos', 'bug', 'bul', 'cat', 'ceb', 'ces', 'cjk', 'ckb', 'crh', 'cym', 'dan', 'deu', 'dik', 'dyu', 'dzo', 'ell', 'eng', 'epo', 'est', 'eus', 'ewe', 'fao', 'fij', 'fin', 'fon', 'fra', 'fur', 'fuv', 'gaz', 'gla', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hne', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kac', 'kam', 'kan', 'kas', 'kat', 'kaz', 'kbp', 'kea', 'khk', 'khm', 'kik', 'kin', 'kir', 'kmb', 'kmr', 'knc', 'kon', 'kor', 'lao', 'lij', 'lim', 'lin', 'lit', 'lmo', 'ltg', 'ltz', 'lua', 'lug', 'luo', 'lus', 'lvs', 'mag', 'mai', 'mal', 'mar', 'min', 'mkd', 'mlt', 'mni', 'mos', 'mri', 'mya', 'nld', 'nno', 'nob', 'npi', 'nqo', 'nso', 'nus', 'nya', 'oci', 'ory', 'pag', 'pan', 'pap', 'pbt', 'pes', 'plt', 'pol', 'por', 'prs', 'quy', 'ron', 'run', 'rus', 'sag', 'san', 'sat', 'scn', 'shn', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'ssw', 'sun', 'swe', 'swh', 'szl', 'tam', 'taq', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tpi', 'tsn', 'tso', 'tuk', 'tum', 'tur', 'twi', 'tzm', 'uig', 'ukr', 'umb', 'urd', 'uzn', 'vec', 'vie', 'war', 'wol', 'xho', 'ydd', 'yor', 'yue', 'zho', 'zsm', 'zul'] | Classification | s2s | [News, Written] | {'train': 701, 'validation': 99, 'test': 204} | {'train': 111.24, 'validation': 97.11, 'test': 135.53} | -| [SIB200ClusteringS2S](https://arxiv.org/abs/2309.07445) (Adelani et al., 2023) | ['ace', 'acm', 'acq', 'aeb', 'afr', 'ajp', 'aka', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'ast', 'awa', 'ayr', 'azb', 'azj', 'bak', 'bam', 'ban', 'bel', 'bem', 'ben', 'bho', 'bjn', 'bod', 'bos', 'bug', 'bul', 'cat', 'ceb', 'ces', 'cjk', 'ckb', 'crh', 'cym', 'dan', 'deu', 'dik', 'dyu', 'dzo', 'ell', 'eng', 'epo', 'est', 'eus', 'ewe', 'fao', 'fij', 'fin', 'fon', 'fra', 'fur', 'fuv', 'gaz', 'gla', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hne', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kac', 'kam', 'kan', 'kas', 'kat', 'kaz', 'kbp', 'kea', 'khk', 'khm', 'kik', 'kin', 'kir', 'kmb', 'kmr', 'knc', 'kon', 'kor', 'lao', 'lij', 'lim', 'lin', 'lit', 'lmo', 'ltg', 'ltz', 'lua', 'lug', 'luo', 'lus', 'lvs', 'mag', 'mai', 'mal', 'mar', 'min', 'mkd', 'mlt', 'mni', 'mos', 'mri', 'mya', 'nld', 'nno', 'nob', 'npi', 'nqo', 'nso', 'nus', 'nya', 'oci', 'ory', 'pag', 'pan', 'pap', 'pbt', 'pes', 'plt', 'pol', 'por', 'prs', 'quy', 'ron', 'run', 'rus', 'sag', 'san', 'sat', 'scn', 'shn', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'ssw', 'sun', 'swe', 'swh', 'szl', 'tam', 'taq', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tpi', 'tsn', 'tso', 'tuk', 'tum', 'tur', 'twi', 'tzm', 'uig', 'ukr', 'umb', 'urd', 'uzn', 'vec', 'vie', 'war', 'wol', 'xho', 'ydd', 'yor', 'yue', 'zho', 'zsm', 'zul'] | Clustering | s2s | [News, Written] | {'test': 1004} | {'test': 114.78} | -| [SICK-BR-PC](https://linux.ime.usp.br/~thalen/SICK_PT.pdf) | ['por'] | PairClassification | s2s | [Web, Written] | {'test': 1000} | {'test': 54.89} | -| [SICK-BR-STS](https://linux.ime.usp.br/~thalen/SICK_PT.pdf) | ['por'] | STS | s2s | [Web, Written] | {'test': 1000} | {'test': 54.89} | -======= | [Quail](https://text-machine.cs.uml.edu/lab2/projects/quail/) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | +| [Query2Query](https://mcinext.com/) | ['fas'] | STS | s2s | | None | None | | [Quora-PL](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2s | | None | None | | [Quora-PLHardNegatives](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2s | | None | None | -| [QuoraRetrieval](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (DataCanary et al., 2017) | ['eng'] | Retrieval | s2s | | None | None | +| [QuoraRetrieval](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (DataCanary et al., 2017) | ['eng'] | Retrieval | s2s | [Blog, Web, Written] | None | None | +| [QuoraRetrieval-Fa](https://huggingface.co/datasets/MCINext/quora-fa) | ['fas'] | Retrieval | s2s | [Web] | None | None | | [QuoraRetrievalHardNegatives](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (DataCanary et al., 2017) | ['eng'] | Retrieval | s2s | | None | None | | [RARbCode](https://arxiv.org/abs/2404.06347) (Xiao et al., 2024) | ['eng'] | Retrieval | s2p | [Programming, Written] | None | None | | [RARbMath](https://arxiv.org/abs/2404.06347) (Xiao et al., 2024) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | -| [RTE3](https://aclanthology.org/W07-1401/) | ['deu', 'eng', 'fra', 'ita'] | PairClassification | s2s | [News, Web, Encyclopaedic, Written] | None | None | +| [RESISC45](https://ieeexplore.ieee.org/abstract/document/7891544) (Cheng et al., 2017) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [RESISC45ZeroShot](https://ieeexplore.ieee.org/abstract/document/7891544) (Cheng et al., 2017) | ['eng'] | ZeroShotClassification | i2t | [Encyclopaedic] | None | None | +| [ROxfordEasyI2IMultiChoice](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyMultiChoice | i2i | [Web] | None | None | +| [ROxfordEasyI2IRetrieval](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyRetrieval | i2i | [Web] | None | None | +| [ROxfordHardI2IMultiChoice](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyMultiChoice | i2i | [Web] | None | None | +| [ROxfordHardI2IRetrieval](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyRetrieval | i2i | [Web] | None | None | +| [ROxfordMediumI2IMultiChoice](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyMultiChoice | i2i | [Web] | None | None | +| [ROxfordMediumI2IRetrieval](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyRetrieval | i2i | [Web] | None | None | +| [RP2kI2IRetrieval](https://arxiv.org/abs/2006.12634) (Peng et al., 2020) | ['eng'] | Any2AnyRetrieval | i2i | [Web] | None | None | +| [RParisEasyI2IMultiChoice](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyMultiChoice | i2i | [Web] | None | None | +| [RParisEasyI2IRetrieval](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyRetrieval | i2i | [Web] | None | None | +| [RParisHardI2IMultiChoice](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyMultiChoice | i2i | [Web] | None | None | +| [RParisHardI2IRetrieval](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyRetrieval | i2i | [Web] | None | None | +| [RParisMediumI2IMultiChoice](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyMultiChoice | i2i | [Web] | None | None | +| [RParisMediumI2IRetrieval](https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html) (Radenovi{'c, 2018) | ['eng'] | Any2AnyRetrieval | i2i | [Web] | None | None | +| [RTE3](https://aclanthology.org/W07-1401/) | ['deu', 'eng', 'fra', 'ita'] | PairClassification | s2s | [Encyclopaedic, News, Web, Written] | None | None | | [RUParaPhraserSTS](https://aclanthology.org/2020.ngt-1.6) (Pivovarova et al., 2017) | ['rus'] | STS | s2s | [News, Written] | None | None | -| [RedditClustering.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | s2s | [Web, Social, Written] | None | None | -| [RedditClusteringP2P.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | p2p | [Web, Social, Written] | {'test': 459389} | {'test': {'num_samples': 459389, 'number_of_characters': 334286895, 'min_text_length': 79, 'average_text_length': 727.68, 'max_text_length': 4359, 'min_labels_per_text': 2, 'average_labels_per_text': 1.0, 'max_labels_per_text': 77908, 'unique_labels': 440, 'labels': {'FortNiteBR': {'count': 436}, 'buildapc': {'count': 8484}, 'offmychest': {'count': 570}, 'nus': {'count': 45}, 'relationship_advice': {'count': 16651}, 'premed': {'count': 201}, 'dogecoin': {'count': 8108}, 'GamingLaptops': {'count': 183}, 'asktransgender': {'count': 326}, 'MachineLearning': {'count': 61}, 'puppy101': {'count': 1597}, 'GunAccessoriesForSale': {'count': 2619}, 'Random_Acts_Of_Amazon': {'count': 1115}, 'Catholicism': {'count': 183}, 'MonsterHunter': {'count': 218}, 'tipofmypenis': {'count': 87}, 'samsung': {'count': 69}, 'PersonalFinanceCanada': {'count': 341}, 'Dyson_Sphere_Program': {'count': 55}, 'bleach': {'count': 41}, 'AmItheAsshole': {'count': 3730}, 'WallStreetbetsELITE': {'count': 328}, 'GlobalPowers': {'count': 35}, 'ABraThatFits': {'count': 159}, 'PokemonGoFriends': {'count': 1165}, 'NoMansSkyTheGame': {'count': 259}, 'masseffect': {'count': 233}, 'dating_advice': {'count': 559}, 'yoga': {'count': 50}, 'depression': {'count': 515}, 'COVID19positive': {'count': 180}, 'generationology': {'count': 37}, 'feedthebeast': {'count': 192}, 'EliteDangerous': {'count': 270}, 'alcoholicsanonymous': {'count': 93}, 'GoRVing': {'count': 35}, 'thedivision': {'count': 111}, 'breakingmom': {'count': 105}, 'AskAnAmerican': {'count': 80}, 'HypnoFair': {'count': 5}, 'JustUnsubbed': {'count': 13}, 'socialanxiety': {'count': 123}, 'dirtykikpals': {'count': 202}, 'askTO': {'count': 126}, 'AskCulinary': {'count': 108}, 'Bogleheads': {'count': 71}, 'dragonquest': {'count': 45}, 'NoContract': {'count': 30}, 'gorillaz': {'count': 14}, 'MondoGore': {'count': 8}, 'comicswap': {'count': 56}, 'VirtualYoutubers': {'count': 92}, 'Gta5Modding': {'count': 28}, 'obs': {'count': 61}, 'vcu': {'count': 9}, 'KingkillerChronicle': {'count': 17}, 'AmongUs': {'count': 41}, 'wireshark': {'count': 3}, 'Dodocodes': {'count': 46}, 'Aliexpress': {'count': 40}, 'LearnerDriverUK': {'count': 12}, 'PanicAttack': {'count': 23}, 'KassadinMains': {'count': 10}, 'islam': {'count': 93}, 'chronotrigger': {'count': 4}, 'skincareexchange': {'count': 13}, 'PokemonHome': {'count': 21}, 'survivinginfidelity': {'count': 71}, 'igcse': {'count': 21}, 'C25K': {'count': 21}, 'aorus': {'count': 2}, 'idleon': {'count': 19}, 'photography': {'count': 22}, 'cryptocoins': {'count': 7}, 'CanaryWharfBets': {'count': 7}, 'KillingEve': {'count': 7}, 'GameBuilderGarage': {'count': 16}, 'SauceSharingCommunity': {'count': 7}, 'turo': {'count': 9}, 'foodscience': {'count': 14}, 'HIMYM': {'count': 20}, 'HauntingOfHillHouse': {'count': 4}, 'GoodNotes': {'count': 8}, 'RedditWritesSeinfeld': {'count': 6}, 'AirReps': {'count': 2}, 'ADHD': {'count': 3811}, 'BuddyCrossing': {'count': 446}, 'libraryofruina': {'count': 98}, 'SluttyConfessions': {'count': 2787}, 'tipofmytongue': {'count': 7145}, 'fleshlight': {'count': 128}, 'amcstock': {'count': 13910}, 'teenagers': {'count': 77908}, 'suggestmeabook': {'count': 1540}, 'dirtypenpals': {'count': 5587}, 'MinecraftServer': {'count': 177}, 'CreditCards': {'count': 669}, 'Guitar': {'count': 10952}, 'rpg': {'count': 529}, 'NoFap': {'count': 14853}, 'lfg': {'count': 1093}, 'MarsWallStreet': {'count': 935}, 'SummonSign': {'count': 931}, 'AssassinsCreedValhala': {'count': 295}, 'hoi4': {'count': 432}, 'Coins4Sale': {'count': 260}, 'xbox': {'count': 459}, 'TooAfraidToAsk': {'count': 7404}, 'NBA2k': {'count': 553}, 'KGBTR': {'count': 943}, 'roblox': {'count': 220}, 'salesforce': {'count': 214}, 'TwoXChromosomes': {'count': 1736}, 'mechmarket': {'count': 4863}, 'Gaming_Headsets': {'count': 103}, 'pittsburgh': {'count': 189}, 'CryptoMars': {'count': 1606}, 'FridayNightFunkin': {'count': 378}, 'vaginismus': {'count': 122}, 'transpositive': {'count': 10}, 'comicbooks': {'count': 274}, 'BDSMcommunity': {'count': 185}, 'aliens': {'count': 201}, 'Scotch': {'count': 64}, 'KikRoleplay': {'count': 141}, 'Kayaking': {'count': 91}, '196': {'count': 47}, 'digimon': {'count': 140}, 'Evernote': {'count': 42}, 'logh': {'count': 22}, 'arlington': {'count': 15}, 'Adopted': {'count': 8}, 'DissonautUniverse': {'count': 4}, 'Midsommar': {'count': 12}, 'SofiawithanF': {'count': 83}, 'xmpp': {'count': 6}, 'ZombsRoyale': {'count': 16}, 'accesscontrol': {'count': 8}, 'WetlanderHumor': {'count': 2}, 'PoonamPandeyFanatics': {'count': 2}, 'screenplaychallenge': {'count': 2}, 'scatstories': {'count': 2}, 'techsupport': {'count': 290}, 'whatcarshouldIbuy': {'count': 79}, 'Stormlight_Archive': {'count': 15}, 'deadbydaylight': {'count': 126}, 'bicycling': {'count': 27}, 'oculus': {'count': 64}, 'Cartalk': {'count': 33}, 'Sims4': {'count': 43}, 'NoFeeAC': {'count': 95}, 'Crypto_com': {'count': 37}, 'ITCareerQuestions': {'count': 259}, 'aromantic': {'count': 18}, 'Revu': {'count': 3}, 'exalted': {'count': 2}, 'HilariaBaldwin': {'count': 20}, 'Testosterone': {'count': 35}, 'Screenwriting': {'count': 170}, 'LifeProTips': {'count': 49}, 'steinsgate': {'count': 13}, 'Baystreetbets': {'count': 10}, 'AskGirls': {'count': 7}, 'idlechampions': {'count': 7}, 'facebook': {'count': 17}, 'tf2trade': {'count': 4}, 'mfdoom': {'count': 3}, 'FiddlesticksMains': {'count': 2}, 'HFY': {'count': 10}, 'FiestaST': {'count': 2}, 'whatsthatbook': {'count': 994}, 'GearsOfWar': {'count': 879}, 'KazuhaMains': {'count': 175}, 'RepTime': {'count': 211}, 'AstroGaming': {'count': 141}, 'metalgearsolid': {'count': 152}, 'qBittorrent': {'count': 39}, 'ELLIPAL_Official': {'count': 24}, 'raisedbynarcissists': {'count': 4895}, 'unpopularopinion': {'count': 14901}, 'ACTrade': {'count': 5679}, 'askcarsales': {'count': 1339}, 'AskVet': {'count': 1357}, 'whowouldwin': {'count': 4493}, 'playstation': {'count': 1362}, 'anime': {'count': 6531}, 'GME': {'count': 12577}, 'DotA2': {'count': 2004}, 'cryptostreetbets': {'count': 2241}, 'MonsterHunterWorld': {'count': 698}, 'Market76': {'count': 14274}, 'DnD': {'count': 5092}, 'leagueoflegends': {'count': 3683}, 'doordash_drivers': {'count': 1626}, 'theta_network': {'count': 489}, 'exmuslim': {'count': 1369}, 'gonewildaudio': {'count': 2998}, 'conspiracy': {'count': 3587}, 'heroesofthestorm': {'count': 535}, 'FanFiction': {'count': 2782}, 'Doom': {'count': 1251}, 'texas': {'count': 269}, 'Vent': {'count': 1738}, 'selfimprovement': {'count': 1284}, 'youtubers': {'count': 706}, 'askseddit': {'count': 237}, 'boardgames': {'count': 1237}, 'bravelydefault': {'count': 347}, 'ConquerorsBlade': {'count': 238}, 'ChronicPain': {'count': 527}, 'teenagersnew': {'count': 256}, 'brasil': {'count': 1092}, 'MatthiasSubmissions': {'count': 921}, 'MarylandUnemployment': {'count': 314}, 'SaltLakeCity': {'count': 411}, 'BokunoheroFanfiction': {'count': 155}, 'BenignExistence': {'count': 125}, 'GayYoungOldDating': {'count': 156}, 'Bible': {'count': 202}, 'haskell': {'count': 154}, 'seduction': {'count': 400}, 'fantasywriters': {'count': 262}, 'HiveOS': {'count': 100}, 'PerkByDaylight': {'count': 15}, 'Hedgehog': {'count': 73}, 'xmen': {'count': 263}, 'HyperRP': {'count': 122}, 'emotestories': {'count': 3}, 'tutanota': {'count': 135}, 'CultoftheFranklin': {'count': 46}, 'langrisser': {'count': 62}, 'CozyGrove': {'count': 61}, 'Sverigesforsvarsmakt': {'count': 12}, 'silverbugbets': {'count': 21}, 'WreckingBallMains': {'count': 5}, 'capitalism_in_decay': {'count': 8}, 'paintdotnet': {'count': 11}, 'u_mawadom118': {'count': 4}, 'xboxfindfriends': {'count': 2}, 'CPTSD': {'count': 540}, 'destiny2': {'count': 318}, 'Wallstreetsilver': {'count': 1013}, 'DestinyTheGame': {'count': 1107}, 'blackopscoldwar': {'count': 400}, 'InstacartShoppers': {'count': 202}, 'RocketLeagueExchange': {'count': 832}, 'apexlegends': {'count': 3265}, 'kansascity': {'count': 53}, 'namenerds': {'count': 235}, 'help': {'count': 152}, 'Kengan_Ashura': {'count': 132}, 'thetagang': {'count': 165}, 'GameSale': {'count': 262}, 'Reduction': {'count': 109}, 'sex': {'count': 906}, 'bostonr4r': {'count': 75}, 'LegendsOfRuneterra': {'count': 231}, 'overlord': {'count': 48}, 'madisonwi': {'count': 53}, 'steelseries': {'count': 79}, 'ClashOfClansRecruit': {'count': 214}, 'CharacterRant': {'count': 55}, 'AirForce': {'count': 94}, 'sexstories': {'count': 92}, 'NameThatSong': {'count': 162}, 'depressed': {'count': 74}, 'ibs': {'count': 150}, '40kLore': {'count': 269}, 'podcasts': {'count': 88}, 'miraculousladybug': {'count': 150}, 'ask': {'count': 224}, 'EverMerge': {'count': 31}, 'TMJ': {'count': 54}, 'BitLifeApp': {'count': 39}, 'FireEmblemHeroes': {'count': 100}, 'software': {'count': 62}, 'ShieldAndroidTV': {'count': 70}, 'GriefSupport': {'count': 125}, 'onewheel': {'count': 37}, 'MensRights': {'count': 80}, 'nhl': {'count': 22}, 'ClashOfClans': {'count': 107}, 'ps3homebrew': {'count': 33}, 'LightNovels': {'count': 77}, 'redsox': {'count': 34}, 'CryptoMarkets': {'count': 44}, 'ugly': {'count': 47}, 'GCXRep': {'count': 12}, 'cscareerquestionsEU': {'count': 65}, 'MindHunter': {'count': 6}, 'starcraft2coop': {'count': 15}, 'nanocurrency': {'count': 1421}, 'ModelCars': {'count': 8}, 'UKJobs': {'count': 30}, 'Netherlands': {'count': 44}, 'clonewars': {'count': 8}, 'Julia': {'count': 11}, 'Prolactinoma': {'count': 9}, 'sofi': {'count': 11}, 'royalfamily': {'count': 6}, 'ConnecticutR4R': {'count': 8}, 'weather': {'count': 5}, 'oneui': {'count': 7}, 'KTM': {'count': 5}, 'Aerials': {'count': 3}, 'seoul': {'count': 2}, 'exjw': {'count': 3281}, 'ModernMagic': {'count': 699}, 'Paladins': {'count': 1242}, 'kdramarecommends': {'count': 1611}, 'hitbtc': {'count': 330}, 'endocrinology': {'count': 75}, 'Bath': {'count': 43}, 'NassauCountyHookups': {'count': 5}, 'feminineboys': {'count': 1248}, 'dreamsmp': {'count': 2018}, 'SquaredCircle': {'count': 2255}, 'Minecraft': {'count': 8753}, 'spirituality': {'count': 1809}, 'Eldenring': {'count': 1471}, 'Sat': {'count': 1172}, 'bonnaroo': {'count': 194}, 'gardening': {'count': 1892}, 'Unemployment': {'count': 6185}, 'mac': {'count': 1847}, 'Bestbuy': {'count': 437}, 'quittingkratom': {'count': 1081}, 'lawschooladmissions': {'count': 3436}, 'NiceHash': {'count': 2135}, 'McMaster': {'count': 815}, 'covidlonghaulers': {'count': 1299}, 'stalker': {'count': 758}, 'MLBTheShow': {'count': 2721}, 'FortniteCompetitive': {'count': 998}, 'dpdr': {'count': 514}, 'appliancerepair': {'count': 720}, 'thomasthetankengine': {'count': 207}, 'delhi': {'count': 217}, 'Huel': {'count': 300}, 'leafs': {'count': 203}, 'HotWheels': {'count': 170}, '90dayfianceuncensored': {'count': 550}, 'Throwers': {'count': 142}, 'Wavyhair': {'count': 270}, 'CryptoHorde': {'count': 128}, 'ShuumatsuNoValkyrie': {'count': 453}, 'TeensMeetTeens': {'count': 432}, 'dbrand': {'count': 108}, 'SLFmeetups': {'count': 18}, '1200isplentyketo': {'count': 48}, 'passive_income': {'count': 211}, 'BroadCity': {'count': 16}, 'RevenantMain': {'count': 71}, 'extrarfl': {'count': 25}, 'AgonGame': {'count': 5}, 'FitnessDE': {'count': 3}, 'gaming': {'count': 1277}, 'livesound': {'count': 91}, 'IBO': {'count': 1896}, 'EscapefromTarkov': {'count': 1300}, 'amex': {'count': 145}, 'DMAcademy': {'count': 1411}, 'VinylCollectors': {'count': 556}, 'cardano': {'count': 716}, 'brave_browser': {'count': 159}, 'dating': {'count': 952}, 'OculusQuest': {'count': 942}, 'Superstonk': {'count': 3089}, 'MtF': {'count': 957}, 'findaleague': {'count': 207}, 'Nioh': {'count': 398}, 'IRS': {'count': 715}, 'transgendercirclejerk': {'count': 353}, 'learnmath': {'count': 489}, 'piano': {'count': 263}, 'LeagueConnect': {'count': 216}, 'eu4': {'count': 561}, 'Wordpress': {'count': 345}, 'RoleplayingForReddit': {'count': 31}, 'LOONA': {'count': 89}, 'newtothenavy': {'count': 167}, 'HaircareScience': {'count': 118}, 'appletv': {'count': 167}, 'sissypersonals': {'count': 102}, 'raleigh': {'count': 168}, 'realonlyfansreviews': {'count': 21}, 'AskGames': {'count': 49}, 'PokemonTCG': {'count': 325}, 'controlgame': {'count': 109}, 'GoogleDataStudio': {'count': 16}, 'WhiteWolfRPG': {'count': 139}, 'MECoOp': {'count': 31}, 'snuffrp': {'count': 46}, 'lockpicking': {'count': 103}, 'wicked_edge': {'count': 105}, 'BMW': {'count': 99}, 'choiceofgames': {'count': 24}, 'hisdarkmaterials': {'count': 12}, 'SakuraGakuin': {'count': 24}, 'detrans': {'count': 55}, 'Smallville': {'count': 37}, 'kingofqueens': {'count': 7}, 'JamesHoffmann': {'count': 22}, 'stashinvest': {'count': 16}, 'ABA': {'count': 79}, 'ladybusiness': {'count': 10}, 'gamegrumps': {'count': 32}, 'GodEater': {'count': 21}, 'tomorrow': {'count': 39}, 'Tomorrowland': {'count': 9}, 'BlackCountryNewRoad': {'count': 5}, 'STAYC': {'count': 3}, 'SatoshiStreetBets': {'count': 3828}, 'AskLosAngeles': {'count': 1036}, 'buildapcforme': {'count': 1689}, 'ApplyingToCollege': {'count': 10675}, 'watercooling': {'count': 1209}, 'BreakUps': {'count': 4914}, 'FIFA': {'count': 3811}, 'emacs': {'count': 712}, 'trakstocks': {'count': 691}, 'Shittyaskflying': {'count': 147}, 'AmazonFC': {'count': 1178}, 'stocks': {'count': 4610}, 'BangaloreMains': {'count': 26}, 'pokemon': {'count': 3953}, 'religion': {'count': 684}, 'cuboulder': {'count': 269}, 'self': {'count': 1688}, 'tarot': {'count': 912}, 'turtles': {'count': 49}, 'TheMagnusArchives': {'count': 300}, 'Superhero_Ideas': {'count': 34}, 'NTU': {'count': 308}, 'touhou': {'count': 623}, 'JoJolion': {'count': 50}, 'lasers': {'count': 27}, 'popperpigs': {'count': 67}, 'aggretsuko': {'count': 20}, 'Library': {'count': 5}}}} | +| [ReMuQIT2TRetrieval](https://github.com/luomancs/ReMuQ) | ['eng'] | Any2AnyRetrieval | it2t | [Encyclopaedic] | None | None | +| [RedditClustering.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | s2s | [Social, Web, Written] | None | None | +| [RedditClusteringP2P.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | p2p | [Social, Web, Written] | {'test': 459389} | {'test': {'num_samples': 459389, 'number_of_characters': 334286895, 'min_text_length': 79, 'average_text_length': 727.68, 'max_text_length': 4359, 'min_labels_per_text': 2, 'average_labels_per_text': 1.0, 'max_labels_per_text': 77908, 'unique_labels': 440, 'labels': {'FortNiteBR': {'count': 436}, 'buildapc': {'count': 8484}, 'offmychest': {'count': 570}, 'nus': {'count': 45}, 'relationship_advice': {'count': 16651}, 'premed': {'count': 201}, 'dogecoin': {'count': 8108}, 'GamingLaptops': {'count': 183}, 'asktransgender': {'count': 326}, 'MachineLearning': {'count': 61}, 'puppy101': {'count': 1597}, 'GunAccessoriesForSale': {'count': 2619}, 'Random_Acts_Of_Amazon': {'count': 1115}, 'Catholicism': {'count': 183}, 'MonsterHunter': {'count': 218}, 'tipofmypenis': {'count': 87}, 'samsung': {'count': 69}, 'PersonalFinanceCanada': {'count': 341}, 'Dyson_Sphere_Program': {'count': 55}, 'bleach': {'count': 41}, 'AmItheAsshole': {'count': 3730}, 'WallStreetbetsELITE': {'count': 328}, 'GlobalPowers': {'count': 35}, 'ABraThatFits': {'count': 159}, 'PokemonGoFriends': {'count': 1165}, 'NoMansSkyTheGame': {'count': 259}, 'masseffect': {'count': 233}, 'dating_advice': {'count': 559}, 'yoga': {'count': 50}, 'depression': {'count': 515}, 'COVID19positive': {'count': 180}, 'generationology': {'count': 37}, 'feedthebeast': {'count': 192}, 'EliteDangerous': {'count': 270}, 'alcoholicsanonymous': {'count': 93}, 'GoRVing': {'count': 35}, 'thedivision': {'count': 111}, 'breakingmom': {'count': 105}, 'AskAnAmerican': {'count': 80}, 'HypnoFair': {'count': 5}, 'JustUnsubbed': {'count': 13}, 'socialanxiety': {'count': 123}, 'dirtykikpals': {'count': 202}, 'askTO': {'count': 126}, 'AskCulinary': {'count': 108}, 'Bogleheads': {'count': 71}, 'dragonquest': {'count': 45}, 'NoContract': {'count': 30}, 'gorillaz': {'count': 14}, 'MondoGore': {'count': 8}, 'comicswap': {'count': 56}, 'VirtualYoutubers': {'count': 92}, 'Gta5Modding': {'count': 28}, 'obs': {'count': 61}, 'vcu': {'count': 9}, 'KingkillerChronicle': {'count': 17}, 'AmongUs': {'count': 41}, 'wireshark': {'count': 3}, 'Dodocodes': {'count': 46}, 'Aliexpress': {'count': 40}, 'LearnerDriverUK': {'count': 12}, 'PanicAttack': {'count': 23}, 'KassadinMains': {'count': 10}, 'islam': {'count': 93}, 'chronotrigger': {'count': 4}, 'skincareexchange': {'count': 13}, 'PokemonHome': {'count': 21}, 'survivinginfidelity': {'count': 71}, 'igcse': {'count': 21}, 'C25K': {'count': 21}, 'aorus': {'count': 2}, 'idleon': {'count': 19}, 'photography': {'count': 22}, 'cryptocoins': {'count': 7}, 'CanaryWharfBets': {'count': 7}, 'KillingEve': {'count': 7}, 'GameBuilderGarage': {'count': 16}, 'SauceSharingCommunity': {'count': 7}, 'turo': {'count': 9}, 'foodscience': {'count': 14}, 'HIMYM': {'count': 20}, 'HauntingOfHillHouse': {'count': 4}, 'GoodNotes': {'count': 8}, 'RedditWritesSeinfeld': {'count': 6}, 'AirReps': {'count': 2}, 'ADHD': {'count': 3811}, 'BuddyCrossing': {'count': 446}, 'libraryofruina': {'count': 98}, 'SluttyConfessions': {'count': 2787}, 'tipofmytongue': {'count': 7145}, 'fleshlight': {'count': 128}, 'amcstock': {'count': 13910}, 'teenagers': {'count': 77908}, 'suggestmeabook': {'count': 1540}, 'dirtypenpals': {'count': 5587}, 'MinecraftServer': {'count': 177}, 'CreditCards': {'count': 669}, 'Guitar': {'count': 10952}, 'rpg': {'count': 529}, 'NoFap': {'count': 14853}, 'lfg': {'count': 1093}, 'MarsWallStreet': {'count': 935}, 'SummonSign': {'count': 931}, 'AssassinsCreedValhala': {'count': 295}, 'hoi4': {'count': 432}, 'Coins4Sale': {'count': 260}, 'xbox': {'count': 459}, 'TooAfraidToAsk': {'count': 7404}, 'NBA2k': {'count': 553}, 'KGBTR': {'count': 943}, 'roblox': {'count': 220}, 'salesforce': {'count': 214}, 'TwoXChromosomes': {'count': 1736}, 'mechmarket': {'count': 4863}, 'Gaming_Headsets': {'count': 103}, 'pittsburgh': {'count': 189}, 'CryptoMars': {'count': 1606}, 'FridayNightFunkin': {'count': 378}, 'vaginismus': {'count': 122}, 'transpositive': {'count': 10}, 'comicbooks': {'count': 274}, 'BDSMcommunity': {'count': 185}, 'aliens': {'count': 201}, 'Scotch': {'count': 64}, 'KikRoleplay': {'count': 141}, 'Kayaking': {'count': 91}, '196': {'count': 47}, 'digimon': {'count': 140}, 'Evernote': {'count': 42}, 'logh': {'count': 22}, 'arlington': {'count': 15}, 'Adopted': {'count': 8}, 'DissonautUniverse': {'count': 4}, 'Midsommar': {'count': 12}, 'SofiawithanF': {'count': 83}, 'xmpp': {'count': 6}, 'ZombsRoyale': {'count': 16}, 'accesscontrol': {'count': 8}, 'WetlanderHumor': {'count': 2}, 'PoonamPandeyFanatics': {'count': 2}, 'screenplaychallenge': {'count': 2}, 'scatstories': {'count': 2}, 'techsupport': {'count': 290}, 'whatcarshouldIbuy': {'count': 79}, 'Stormlight_Archive': {'count': 15}, 'deadbydaylight': {'count': 126}, 'bicycling': {'count': 27}, 'oculus': {'count': 64}, 'Cartalk': {'count': 33}, 'Sims4': {'count': 43}, 'NoFeeAC': {'count': 95}, 'Crypto_com': {'count': 37}, 'ITCareerQuestions': {'count': 259}, 'aromantic': {'count': 18}, 'Revu': {'count': 3}, 'exalted': {'count': 2}, 'HilariaBaldwin': {'count': 20}, 'Testosterone': {'count': 35}, 'Screenwriting': {'count': 170}, 'LifeProTips': {'count': 49}, 'steinsgate': {'count': 13}, 'Baystreetbets': {'count': 10}, 'AskGirls': {'count': 7}, 'idlechampions': {'count': 7}, 'facebook': {'count': 17}, 'tf2trade': {'count': 4}, 'mfdoom': {'count': 3}, 'FiddlesticksMains': {'count': 2}, 'HFY': {'count': 10}, 'FiestaST': {'count': 2}, 'whatsthatbook': {'count': 994}, 'GearsOfWar': {'count': 879}, 'KazuhaMains': {'count': 175}, 'RepTime': {'count': 211}, 'AstroGaming': {'count': 141}, 'metalgearsolid': {'count': 152}, 'qBittorrent': {'count': 39}, 'ELLIPAL_Official': {'count': 24}, 'raisedbynarcissists': {'count': 4895}, 'unpopularopinion': {'count': 14901}, 'ACTrade': {'count': 5679}, 'askcarsales': {'count': 1339}, 'AskVet': {'count': 1357}, 'whowouldwin': {'count': 4493}, 'playstation': {'count': 1362}, 'anime': {'count': 6531}, 'GME': {'count': 12577}, 'DotA2': {'count': 2004}, 'cryptostreetbets': {'count': 2241}, 'MonsterHunterWorld': {'count': 698}, 'Market76': {'count': 14274}, 'DnD': {'count': 5092}, 'leagueoflegends': {'count': 3683}, 'doordash_drivers': {'count': 1626}, 'theta_network': {'count': 489}, 'exmuslim': {'count': 1369}, 'gonewildaudio': {'count': 2998}, 'conspiracy': {'count': 3587}, 'heroesofthestorm': {'count': 535}, 'FanFiction': {'count': 2782}, 'Doom': {'count': 1251}, 'texas': {'count': 269}, 'Vent': {'count': 1738}, 'selfimprovement': {'count': 1284}, 'youtubers': {'count': 706}, 'askseddit': {'count': 237}, 'boardgames': {'count': 1237}, 'bravelydefault': {'count': 347}, 'ConquerorsBlade': {'count': 238}, 'ChronicPain': {'count': 527}, 'teenagersnew': {'count': 256}, 'brasil': {'count': 1092}, 'MatthiasSubmissions': {'count': 921}, 'MarylandUnemployment': {'count': 314}, 'SaltLakeCity': {'count': 411}, 'BokunoheroFanfiction': {'count': 155}, 'BenignExistence': {'count': 125}, 'GayYoungOldDating': {'count': 156}, 'Bible': {'count': 202}, 'haskell': {'count': 154}, 'seduction': {'count': 400}, 'fantasywriters': {'count': 262}, 'HiveOS': {'count': 100}, 'PerkByDaylight': {'count': 15}, 'Hedgehog': {'count': 73}, 'xmen': {'count': 263}, 'HyperRP': {'count': 122}, 'emotestories': {'count': 3}, 'tutanota': {'count': 135}, 'CultoftheFranklin': {'count': 46}, 'langrisser': {'count': 62}, 'CozyGrove': {'count': 61}, 'Sverigesforsvarsmakt': {'count': 12}, 'silverbugbets': {'count': 21}, 'WreckingBallMains': {'count': 5}, 'capitalism_in_decay': {'count': 8}, 'paintdotnet': {'count': 11}, 'u_mawadom118': {'count': 4}, 'xboxfindfriends': {'count': 2}, 'CPTSD': {'count': 540}, 'destiny2': {'count': 318}, 'Wallstreetsilver': {'count': 1013}, 'DestinyTheGame': {'count': 1107}, 'blackopscoldwar': {'count': 400}, 'InstacartShoppers': {'count': 202}, 'RocketLeagueExchange': {'count': 832}, 'apexlegends': {'count': 3265}, 'kansascity': {'count': 53}, 'namenerds': {'count': 235}, 'help': {'count': 152}, 'Kengan_Ashura': {'count': 132}, 'thetagang': {'count': 165}, 'GameSale': {'count': 262}, 'Reduction': {'count': 109}, 'sex': {'count': 906}, 'bostonr4r': {'count': 75}, 'LegendsOfRuneterra': {'count': 231}, 'overlord': {'count': 48}, 'madisonwi': {'count': 53}, 'steelseries': {'count': 79}, 'ClashOfClansRecruit': {'count': 214}, 'CharacterRant': {'count': 55}, 'AirForce': {'count': 94}, 'sexstories': {'count': 92}, 'NameThatSong': {'count': 162}, 'depressed': {'count': 74}, 'ibs': {'count': 150}, '40kLore': {'count': 269}, 'podcasts': {'count': 88}, 'miraculousladybug': {'count': 150}, 'ask': {'count': 224}, 'EverMerge': {'count': 31}, 'TMJ': {'count': 54}, 'BitLifeApp': {'count': 39}, 'FireEmblemHeroes': {'count': 100}, 'software': {'count': 62}, 'ShieldAndroidTV': {'count': 70}, 'GriefSupport': {'count': 125}, 'onewheel': {'count': 37}, 'MensRights': {'count': 80}, 'nhl': {'count': 22}, 'ClashOfClans': {'count': 107}, 'ps3homebrew': {'count': 33}, 'LightNovels': {'count': 77}, 'redsox': {'count': 34}, 'CryptoMarkets': {'count': 44}, 'ugly': {'count': 47}, 'GCXRep': {'count': 12}, 'cscareerquestionsEU': {'count': 65}, 'MindHunter': {'count': 6}, 'starcraft2coop': {'count': 15}, 'nanocurrency': {'count': 1421}, 'ModelCars': {'count': 8}, 'UKJobs': {'count': 30}, 'Netherlands': {'count': 44}, 'clonewars': {'count': 8}, 'Julia': {'count': 11}, 'Prolactinoma': {'count': 9}, 'sofi': {'count': 11}, 'royalfamily': {'count': 6}, 'ConnecticutR4R': {'count': 8}, 'weather': {'count': 5}, 'oneui': {'count': 7}, 'KTM': {'count': 5}, 'Aerials': {'count': 3}, 'seoul': {'count': 2}, 'exjw': {'count': 3281}, 'ModernMagic': {'count': 699}, 'Paladins': {'count': 1242}, 'kdramarecommends': {'count': 1611}, 'hitbtc': {'count': 330}, 'endocrinology': {'count': 75}, 'Bath': {'count': 43}, 'NassauCountyHookups': {'count': 5}, 'feminineboys': {'count': 1248}, 'dreamsmp': {'count': 2018}, 'SquaredCircle': {'count': 2255}, 'Minecraft': {'count': 8753}, 'spirituality': {'count': 1809}, 'Eldenring': {'count': 1471}, 'Sat': {'count': 1172}, 'bonnaroo': {'count': 194}, 'gardening': {'count': 1892}, 'Unemployment': {'count': 6185}, 'mac': {'count': 1847}, 'Bestbuy': {'count': 437}, 'quittingkratom': {'count': 1081}, 'lawschooladmissions': {'count': 3436}, 'NiceHash': {'count': 2135}, 'McMaster': {'count': 815}, 'covidlonghaulers': {'count': 1299}, 'stalker': {'count': 758}, 'MLBTheShow': {'count': 2721}, 'FortniteCompetitive': {'count': 998}, 'dpdr': {'count': 514}, 'appliancerepair': {'count': 720}, 'thomasthetankengine': {'count': 207}, 'delhi': {'count': 217}, 'Huel': {'count': 300}, 'leafs': {'count': 203}, 'HotWheels': {'count': 170}, '90dayfianceuncensored': {'count': 550}, 'Throwers': {'count': 142}, 'Wavyhair': {'count': 270}, 'CryptoHorde': {'count': 128}, 'ShuumatsuNoValkyrie': {'count': 453}, 'TeensMeetTeens': {'count': 432}, 'dbrand': {'count': 108}, 'SLFmeetups': {'count': 18}, '1200isplentyketo': {'count': 48}, 'passive_income': {'count': 211}, 'BroadCity': {'count': 16}, 'RevenantMain': {'count': 71}, 'extrarfl': {'count': 25}, 'AgonGame': {'count': 5}, 'FitnessDE': {'count': 3}, 'gaming': {'count': 1277}, 'livesound': {'count': 91}, 'IBO': {'count': 1896}, 'EscapefromTarkov': {'count': 1300}, 'amex': {'count': 145}, 'DMAcademy': {'count': 1411}, 'VinylCollectors': {'count': 556}, 'cardano': {'count': 716}, 'brave_browser': {'count': 159}, 'dating': {'count': 952}, 'OculusQuest': {'count': 942}, 'Superstonk': {'count': 3089}, 'MtF': {'count': 957}, 'findaleague': {'count': 207}, 'Nioh': {'count': 398}, 'IRS': {'count': 715}, 'transgendercirclejerk': {'count': 353}, 'learnmath': {'count': 489}, 'piano': {'count': 263}, 'LeagueConnect': {'count': 216}, 'eu4': {'count': 561}, 'Wordpress': {'count': 345}, 'RoleplayingForReddit': {'count': 31}, 'LOONA': {'count': 89}, 'newtothenavy': {'count': 167}, 'HaircareScience': {'count': 118}, 'appletv': {'count': 167}, 'sissypersonals': {'count': 102}, 'raleigh': {'count': 168}, 'realonlyfansreviews': {'count': 21}, 'AskGames': {'count': 49}, 'PokemonTCG': {'count': 325}, 'controlgame': {'count': 109}, 'GoogleDataStudio': {'count': 16}, 'WhiteWolfRPG': {'count': 139}, 'MECoOp': {'count': 31}, 'snuffrp': {'count': 46}, 'lockpicking': {'count': 103}, 'wicked_edge': {'count': 105}, 'BMW': {'count': 99}, 'choiceofgames': {'count': 24}, 'hisdarkmaterials': {'count': 12}, 'SakuraGakuin': {'count': 24}, 'detrans': {'count': 55}, 'Smallville': {'count': 37}, 'kingofqueens': {'count': 7}, 'JamesHoffmann': {'count': 22}, 'stashinvest': {'count': 16}, 'ABA': {'count': 79}, 'ladybusiness': {'count': 10}, 'gamegrumps': {'count': 32}, 'GodEater': {'count': 21}, 'tomorrow': {'count': 39}, 'Tomorrowland': {'count': 9}, 'BlackCountryNewRoad': {'count': 5}, 'STAYC': {'count': 3}, 'SatoshiStreetBets': {'count': 3828}, 'AskLosAngeles': {'count': 1036}, 'buildapcforme': {'count': 1689}, 'ApplyingToCollege': {'count': 10675}, 'watercooling': {'count': 1209}, 'BreakUps': {'count': 4914}, 'FIFA': {'count': 3811}, 'emacs': {'count': 712}, 'trakstocks': {'count': 691}, 'Shittyaskflying': {'count': 147}, 'AmazonFC': {'count': 1178}, 'stocks': {'count': 4610}, 'BangaloreMains': {'count': 26}, 'pokemon': {'count': 3953}, 'religion': {'count': 684}, 'cuboulder': {'count': 269}, 'self': {'count': 1688}, 'tarot': {'count': 912}, 'turtles': {'count': 49}, 'TheMagnusArchives': {'count': 300}, 'Superhero_Ideas': {'count': 34}, 'NTU': {'count': 308}, 'touhou': {'count': 623}, 'JoJolion': {'count': 50}, 'lasers': {'count': 27}, 'popperpigs': {'count': 67}, 'aggretsuko': {'count': 20}, 'Library': {'count': 5}}}} | +| [RenderedSST2](https://huggingface.co/datasets/clip-benchmark/wds_renderedsst2) | ['eng'] | ZeroShotClassification | i2t | [Reviews] | None | None | | [RestaurantReviewSentimentClassification](https://link.springer.com/chapter/10.1007/978-3-319-18117-2_2) (ElSahar et al., 2015) | ['ara'] | Classification | s2s | [Reviews, Written] | None | None | | [RiaNewsRetrieval](https://arxiv.org/abs/1901.07786) (Gavrilov et al., 2019) | ['rus'] | Retrieval | s2p | [News, Written] | None | None | | [RiaNewsRetrievalHardNegatives](https://arxiv.org/abs/1901.07786) (Gavrilov et al., 2019) | ['rus'] | Retrieval | s2p | [News, Written] | None | None | @@ -834,6 +601,7 @@ The following tables give you an overview of the tasks in MTEB. | [RuSciBenchGRNTIClusteringP2P](https://github.com/mlsa-iai-msu-lab/ru_sci_bench/) | ['rus'] | Clustering | p2p | [Academic, Written] | {'test': 2048} | {'test': {'num_samples': 2048, 'number_of_characters': 1822339, 'min_text_length': 84, 'average_text_length': 889.81, 'max_text_length': 3143, 'min_labels_per_text': 73, 'average_labels_per_text': 1.0, 'max_labels_per_text': 74, 'unique_labels': 28, 'labels': {'3': {'count': 73}, '4': {'count': 73}, '20': {'count': 73}, '9': {'count': 73}, '21': {'count': 73}, '15': {'count': 73}, '16': {'count': 74}, '2': {'count': 73}, '8': {'count': 73}, '23': {'count': 73}, '6': {'count': 73}, '24': {'count': 73}, '10': {'count': 73}, '1': {'count': 73}, '17': {'count': 74}, '14': {'count': 74}, '18': {'count': 73}, '27': {'count': 73}, '19': {'count': 73}, '22': {'count': 73}, '12': {'count': 73}, '25': {'count': 73}, '5': {'count': 74}, '0': {'count': 73}, '26': {'count': 73}, '11': {'count': 73}, '13': {'count': 73}, '7': {'count': 73}}}} | | [RuSciBenchOECDClassification](https://github.com/mlsa-iai-msu-lab/ru_sci_bench/) | ['rus'] | Classification | p2p | [Academic, Written] | None | None | | [RuSciBenchOECDClusteringP2P](https://github.com/mlsa-iai-msu-lab/ru_sci_bench/) | ['rus'] | Clustering | p2p | [Academic, Written] | None | None | +| [SAMSumFa](https://huggingface.co/datasets/MCINext/samsum-fa) | ['fas'] | BitextMining | s2p | [Spoken] | None | None | | [SCDBPAccountabilityLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [SCDBPAuditsLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [SCDBPCertificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | @@ -844,87 +612,73 @@ The following tables give you an overview of the tasks in MTEB. | [SCDDCertificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [SCDDTrainingLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [SCDDVerificationLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | -| [SCIDOCS](https://allenai.org/data/scidocs) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | [Academic, Written, Non-fiction] | None | None | +| [SCIDOCS](https://allenai.org/data/scidocs) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | [Academic, Non-fiction, Written] | None | None | +| [SCIDOCS-Fa](https://huggingface.co/datasets/MCINext/scidocs-fa) | ['fas'] | Retrieval | s2p | [Academic] | None | None | | [SCIDOCS-PL](https://allenai.org/data/scidocs) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | None | +| [SDSEyeProtectionClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2p | [Chemistry] | None | None | +| [SDSGlovesClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2p | [Chemistry] | None | None | | [SIB200Classification](https://arxiv.org/abs/2309.07445) (Adelani et al., 2023) | ['ace', 'acm', 'acq', 'aeb', 'afr', 'ajp', 'aka', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'ast', 'awa', 'ayr', 'azb', 'azj', 'bak', 'bam', 'ban', 'bel', 'bem', 'ben', 'bho', 'bjn', 'bod', 'bos', 'bug', 'bul', 'cat', 'ceb', 'ces', 'cjk', 'ckb', 'crh', 'cym', 'dan', 'deu', 'dik', 'dyu', 'dzo', 'ell', 'eng', 'epo', 'est', 'eus', 'ewe', 'fao', 'fij', 'fin', 'fon', 'fra', 'fur', 'fuv', 'gaz', 'gla', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hne', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kac', 'kam', 'kan', 'kas', 'kat', 'kaz', 'kbp', 'kea', 'khk', 'khm', 'kik', 'kin', 'kir', 'kmb', 'kmr', 'knc', 'kon', 'kor', 'lao', 'lij', 'lim', 'lin', 'lit', 'lmo', 'ltg', 'ltz', 'lua', 'lug', 'luo', 'lus', 'lvs', 'mag', 'mai', 'mal', 'mar', 'min', 'mkd', 'mlt', 'mni', 'mos', 'mri', 'mya', 'nld', 'nno', 'nob', 'npi', 'nqo', 'nso', 'nus', 'nya', 'oci', 'ory', 'pag', 'pan', 'pap', 'pbt', 'pes', 'plt', 'pol', 'por', 'prs', 'quy', 'ron', 'run', 'rus', 'sag', 'san', 'sat', 'scn', 'shn', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'ssw', 'sun', 'swe', 'swh', 'szl', 'tam', 'taq', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tpi', 'tsn', 'tso', 'tuk', 'tum', 'tur', 'twi', 'tzm', 'uig', 'ukr', 'umb', 'urd', 'uzn', 'vec', 'vie', 'war', 'wol', 'xho', 'ydd', 'yor', 'yue', 'zho', 'zsm', 'zul'] | Classification | s2s | [News, Written] | None | None | | [SIB200ClusteringS2S](https://arxiv.org/abs/2309.07445) (Adelani et al., 2023) | ['ace', 'acm', 'acq', 'aeb', 'afr', 'ajp', 'aka', 'als', 'amh', 'apc', 'arb', 'ars', 'ary', 'arz', 'asm', 'ast', 'awa', 'ayr', 'azb', 'azj', 'bak', 'bam', 'ban', 'bel', 'bem', 'ben', 'bho', 'bjn', 'bod', 'bos', 'bug', 'bul', 'cat', 'ceb', 'ces', 'cjk', 'ckb', 'crh', 'cym', 'dan', 'deu', 'dik', 'dyu', 'dzo', 'ell', 'eng', 'epo', 'est', 'eus', 'ewe', 'fao', 'fij', 'fin', 'fon', 'fra', 'fur', 'fuv', 'gaz', 'gla', 'gle', 'glg', 'grn', 'guj', 'hat', 'hau', 'heb', 'hin', 'hne', 'hrv', 'hun', 'hye', 'ibo', 'ilo', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kac', 'kam', 'kan', 'kas', 'kat', 'kaz', 'kbp', 'kea', 'khk', 'khm', 'kik', 'kin', 'kir', 'kmb', 'kmr', 'knc', 'kon', 'kor', 'lao', 'lij', 'lim', 'lin', 'lit', 'lmo', 'ltg', 'ltz', 'lua', 'lug', 'luo', 'lus', 'lvs', 'mag', 'mai', 'mal', 'mar', 'min', 'mkd', 'mlt', 'mni', 'mos', 'mri', 'mya', 'nld', 'nno', 'nob', 'npi', 'nqo', 'nso', 'nus', 'nya', 'oci', 'ory', 'pag', 'pan', 'pap', 'pbt', 'pes', 'plt', 'pol', 'por', 'prs', 'quy', 'ron', 'run', 'rus', 'sag', 'san', 'sat', 'scn', 'shn', 'sin', 'slk', 'slv', 'smo', 'sna', 'snd', 'som', 'sot', 'spa', 'srd', 'srp', 'ssw', 'sun', 'swe', 'swh', 'szl', 'tam', 'taq', 'tat', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tpi', 'tsn', 'tso', 'tuk', 'tum', 'tur', 'twi', 'tzm', 'uig', 'ukr', 'umb', 'urd', 'uzn', 'vec', 'vie', 'war', 'wol', 'xho', 'ydd', 'yor', 'yue', 'zho', 'zsm', 'zul'] | Clustering | s2s | [News, Written] | None | None | | [SICK-BR-PC](https://linux.ime.usp.br/~thalen/SICK_PT.pdf) | ['por'] | PairClassification | s2s | [Web, Written] | None | None | | [SICK-BR-STS](https://linux.ime.usp.br/~thalen/SICK_PT.pdf) | ['por'] | STS | s2s | [Web, Written] | None | None | ->>>>>>> main | [SICK-E-PL](https://aclanthology.org/2020.lrec-1.207) | ['pol'] | PairClassification | s2s | | None | None | | [SICK-R](https://aclanthology.org/L14-1314/) | ['eng'] | STS | s2s | [Web, Written] | None | None | | [SICK-R-PL](https://aclanthology.org/2020.lrec-1.207) | ['pol'] | STS | s2s | [Web, Written] | None | None | | [SICKFr](https://huggingface.co/datasets/Lajavaness/SICK-fr) | ['fra'] | STS | s2s | | None | None | -<<<<<<< HEAD -| [SIQA](https://leaderboard.allenai.org/socialiqa/submissions/get-started) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 0} | {'test': {'average_document_length': 22.967085695044617, 'average_query_length': 127.75383828045035, 'num_documents': 71276, 'num_queries': 1954, 'average_relevant_docs_per_query': 1.0}} | -| [SKQuadRetrieval](https://huggingface.co/datasets/TUKE-KEMT/retrieval-skquad) | ['slk'] | Retrieval | s2s | [Encyclopaedic] | {'test': 1134} | {'test': {'average_document_length': 1180.5071792496526, 'average_query_length': 53.63403880070547, 'num_documents': 6477, 'num_queries': 1134, 'average_relevant_docs_per_query': 11}} | -| [SNLHierarchicalClusteringP2P](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | p2p | [Encyclopaedic, Non-fiction, Written] | {'test': 1300} | {'test': 1986.9453846153847} | -| [SNLHierarchicalClusteringS2S](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | s2s | [Encyclopaedic, Non-fiction, Written] | {'test': 1300} | {'test': 242.22384615384615} | -| [SNLRetrieval](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Written] | {'test': 2048} | {'test': {'average_document_length': 1986.9453846153847, 'average_query_length': 14.906153846153845, 'num_documents': 1300, 'num_queries': 1300, 'average_relevant_docs_per_query': 1.0}} | -| [SRNCorpusBitextMining](https://arxiv.org/abs/2212.06383) (Zwennicker et al., 2022) | ['nld', 'srn'] | BitextMining | s2s | [Social, Web, Written] | {'test': 256} | {'test': 55} | -| [STS12](https://www.aclweb.org/anthology/S12-1051.pdf) (Agirre et al., 2012) | ['eng'] | STS | s2s | [Encyclopaedic, News, Written] | {'test': 6216} | {'test': {'num_samples': 3108, 'average_sentence1_len': 63.78893178893179, 'average_sentence2_len': 65.5926640926641, 'avg_score': 3.5060643500643507}} | -| [STS13](https://www.aclweb.org/anthology/S13-1004/) (Eneko Agirre, 2013) | ['eng'] | STS | s2s | [Web, News, Non-fiction, Written] | {'test': 3000} | {'test': 54.0} | -| [STS14](https://www.aclweb.org/anthology/S14-1002) | ['eng'] | STS | s2s | [Blog, Web, Spoken] | {'test': 7500} | {'test': 54.3} | -| [STS15](https://www.aclweb.org/anthology/S15-2010) | ['eng'] | STS | s2s | [Blog, News, Web, Written, Spoken] | {'test': 6000} | {'test': 57.7} | -| [STS16](https://www.aclweb.org/anthology/S16-1001) | ['eng'] | STS | s2s | [Blog, Web, Spoken] | {'test': 2372} | {'test': 65.3} | -| [STS17](https://alt.qcri.org/semeval2017/task1/) | ['ara', 'deu', 'eng', 'fra', 'ita', 'kor', 'nld', 'spa', 'tur'] | STS | s2s | [News, Web, Written] | {'test': 500} | {'test': {'num_samples': 5346, 'average_sentence1_len': 38.14665170220726, 'average_sentence2_len': 36.72502805836139, 'avg_score': 2.3554804214989464, 'hf_subset_descriptive_stats': {'ko-ko': {'num_samples': 2846, 'average_sentence1_len': 31.991918482080113, 'average_sentence2_len': 32.44483485593816, 'avg_score': 2.469359920356055}, 'ar-ar': {'num_samples': 250, 'average_sentence1_len': 32.208, 'average_sentence2_len': 32.78, 'avg_score': 2.216800000000001}, 'en-ar': {'num_samples': 250, 'average_sentence1_len': 42.36, 'average_sentence2_len': 32.696, 'avg_score': 2.1423999999999994}, 'en-de': {'num_samples': 250, 'average_sentence1_len': 43.952, 'average_sentence2_len': 44.756, 'avg_score': 2.2776000000000014}, 'en-en': {'num_samples': 250, 'average_sentence1_len': 43.952, 'average_sentence2_len': 42.724, 'avg_score': 2.2776000000000014}, 'en-tr': {'num_samples': 250, 'average_sentence1_len': 41.916, 'average_sentence2_len': 41.6, 'avg_score': 2.1335999999999986}, 'es-en': {'num_samples': 250, 'average_sentence1_len': 50.84, 'average_sentence2_len': 42.024, 'avg_score': 2.1464000000000003}, 'es-es': {'num_samples': 250, 'average_sentence1_len': 49.836, 'average_sentence2_len': 51.224, 'avg_score': 2.2312000000000007}, 'fr-en': {'num_samples': 250, 'average_sentence1_len': 49.624, 'average_sentence2_len': 42.724, 'avg_score': 2.2776000000000014}, 'it-en': {'num_samples': 250, 'average_sentence1_len': 50.028, 'average_sentence2_len': 42.724, 'avg_score': 2.2776000000000014}, 'nl-en': {'num_samples': 250, 'average_sentence1_len': 46.816, 'average_sentence2_len': 42.724, 'avg_score': 2.2776000000000014}}}} | -| [STS22.v2](https://competitions.codalab.org/competitions/33835) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'ita', 'pol', 'rus', 'spa', 'tur'] | STS | p2p | [News, Written] | {'test': 3958} | {'test': 1993.6} | -======= +| [SIDClassification](https://mcinext.com/) | ['fas'] | Classification | p2p | [Academic] | None | None | +| [SIDClustring](https://www.sid.com/) | ['fas'] | Clustering | p2p | [Academic] | None | None | | [SIQA](https://leaderboard.allenai.org/socialiqa/submissions/get-started) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | | [SKQuadRetrieval](https://huggingface.co/datasets/TUKE-KEMT/retrieval-skquad) | ['slk'] | Retrieval | s2s | [Encyclopaedic] | None | None | | [SNLHierarchicalClusteringP2P](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | p2p | [Encyclopaedic, Non-fiction, Written] | None | None | | [SNLHierarchicalClusteringS2S](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | s2s | [Encyclopaedic, Non-fiction, Written] | None | None | | [SNLRetrieval](https://huggingface.co/datasets/navjordj/SNL_summarization) (Navjord et al., 2023) | ['nob'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Written] | None | None | +| [SOPI2IRetrieval](https://paperswithcode.com/dataset/stanford-online-products) (Oh Song et al., 2016) | ['eng'] | Any2AnyRetrieval | i2i | [Encyclopaedic] | None | None | | [SRNCorpusBitextMining](https://arxiv.org/abs/2212.06383) (Zwennicker et al., 2022) | ['nld', 'srn'] | BitextMining | s2s | [Social, Web, Written] | None | None | +| [STL10](https://cs.stanford.edu/~acoates/stl10/) (Coates et al., 2011) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [STL10ZeroShot](https://cs.stanford.edu/~acoates/stl10/) (Coates et al., 2011) | ['eng'] | ZeroShotClassification | i2t | [Encyclopaedic] | None | None | | [STS12](https://www.aclweb.org/anthology/S12-1051.pdf) (Agirre et al., 2012) | ['eng'] | STS | s2s | [Encyclopaedic, News, Written] | {'test': 3108} | {'test': {'num_samples': 3108, 'number_of_characters': 402118, 'min_sentence1_length': 3, 'average_sentence1_len': 63.79, 'max_sentence1_length': 220, 'unique_sentence1': 2236, 'min_sentence2_length': 7, 'average_sentence2_len': 65.59, 'max_sentence2_length': 204, 'unique_sentence2': 2797, 'min_score': 0.0, 'avg_score': 3.51, 'max_score': 5.0}} | -| [STS13](https://www.aclweb.org/anthology/S13-1004/) (Eneko Agirre, 2013) | ['eng'] | STS | s2s | [Web, News, Non-fiction, Written] | None | None | -| [STS14](https://www.aclweb.org/anthology/S14-1002) | ['eng'] | STS | s2s | [Blog, Web, Spoken] | None | None | -| [STS15](https://www.aclweb.org/anthology/S15-2010) | ['eng'] | STS | s2s | [Blog, News, Web, Written, Spoken] | None | None | -| [STS16](https://www.aclweb.org/anthology/S16-1001) | ['eng'] | STS | s2s | [Blog, Web, Spoken] | None | None | +| [STS12VisualSTS](https://arxiv.org/abs/2402.08183/) (Xiao et al., 2024) | ['eng'] | VisualSTS | i2i | [Encyclopaedic, News, Written] | None | None | +| [STS13](https://www.aclweb.org/anthology/S13-1004/) (Eneko Agirre, 2013) | ['eng'] | STS | s2s | [News, Non-fiction, Web, Written] | None | None | +| [STS13VisualSTS](https://arxiv.org/abs/2402.08183/) (Xiao et al., 2024) | ['eng'] | VisualSTS | i2i | [News, Non-fiction, Web, Written] | None | None | +| [STS14](https://www.aclweb.org/anthology/S14-1002) | ['eng'] | STS | s2s | [Blog, Spoken, Web] | None | None | +| [STS14VisualSTS](https://arxiv.org/abs/2402.08183/) (Xiao et al., 2024) | ['eng'] | VisualSTS | i2i | [Blog, Spoken, Web] | None | None | +| [STS15](https://www.aclweb.org/anthology/S15-2010) | ['eng'] | STS | s2s | [Blog, News, Spoken, Web, Written] | None | None | +| [STS15VisualSTS](https://arxiv.org/abs/2402.08183/) (Xiao et al., 2024) | ['eng'] | VisualSTS | i2i | [Blog, News, Spoken, Web, Written] | None | None | +| [STS16](https://www.aclweb.org/anthology/S16-1001) | ['eng'] | STS | s2s | [Blog, Spoken, Web] | None | None | +| [STS16VisualSTS](https://arxiv.org/abs/2402.08183/) (Xiao et al., 2024) | ['eng'] | VisualSTS | i2i | [Blog, Spoken, Web] | None | None | | [STS17](https://alt.qcri.org/semeval2017/task1/) | ['ara', 'deu', 'eng', 'fra', 'ita', 'kor', 'nld', 'spa', 'tur'] | STS | s2s | [News, Web, Written] | {'test': 5346} | {'test': {'num_samples': 5346, 'number_of_characters': 400264, 'min_sentence1_length': 6, 'average_sentence1_len': 38.15, 'max_sentence1_length': 976, 'unique_sentence1': 4900, 'min_sentence2_length': 6, 'average_sentence2_len': 36.73, 'max_sentence2_length': 1007, 'unique_sentence2': 4470, 'min_score': 0.0, 'avg_score': 2.36, 'max_score': 5.0, 'hf_subset_descriptive_stats': {'ko-ko': {'num_samples': 2846, 'number_of_characters': 183387, 'min_sentence1_length': 6, 'average_sentence1_len': 31.99, 'max_sentence1_length': 976, 'unique_sentence1': 2650, 'min_sentence2_length': 6, 'average_sentence2_len': 32.44, 'max_sentence2_length': 1007, 'unique_sentence2': 2720, 'min_score': 0.0, 'avg_score': 2.47, 'max_score': 5.0}, 'ar-ar': {'num_samples': 250, 'number_of_characters': 16247, 'min_sentence1_length': 11, 'average_sentence1_len': 32.21, 'max_sentence1_length': 99, 'unique_sentence1': 250, 'min_sentence2_length': 9, 'average_sentence2_len': 32.78, 'max_sentence2_length': 83, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.22, 'max_score': 5.0}, 'en-ar': {'num_samples': 250, 'number_of_characters': 18764, 'min_sentence1_length': 13, 'average_sentence1_len': 42.36, 'max_sentence1_length': 105, 'unique_sentence1': 250, 'min_sentence2_length': 10, 'average_sentence2_len': 32.7, 'max_sentence2_length': 104, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.14, 'max_score': 5.0}, 'en-de': {'num_samples': 250, 'number_of_characters': 22177, 'min_sentence1_length': 12, 'average_sentence1_len': 43.95, 'max_sentence1_length': 94, 'unique_sentence1': 250, 'min_sentence2_length': 15, 'average_sentence2_len': 44.76, 'max_sentence2_length': 104, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.28, 'max_score': 5.0}, 'en-en': {'num_samples': 250, 'number_of_characters': 21669, 'min_sentence1_length': 12, 'average_sentence1_len': 43.95, 'max_sentence1_length': 94, 'unique_sentence1': 250, 'min_sentence2_length': 15, 'average_sentence2_len': 42.72, 'max_sentence2_length': 101, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.28, 'max_score': 5.0}, 'en-tr': {'num_samples': 250, 'number_of_characters': 20879, 'min_sentence1_length': 15, 'average_sentence1_len': 41.92, 'max_sentence1_length': 101, 'unique_sentence1': 250, 'min_sentence2_length': 10, 'average_sentence2_len': 41.6, 'max_sentence2_length': 107, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.13, 'max_score': 5.0}, 'es-en': {'num_samples': 250, 'number_of_characters': 23216, 'min_sentence1_length': 12, 'average_sentence1_len': 50.84, 'max_sentence1_length': 160, 'unique_sentence1': 250, 'min_sentence2_length': 14, 'average_sentence2_len': 42.02, 'max_sentence2_length': 117, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.15, 'max_score': 5.0}, 'es-es': {'num_samples': 250, 'number_of_characters': 25265, 'min_sentence1_length': 18, 'average_sentence1_len': 49.84, 'max_sentence1_length': 136, 'unique_sentence1': 250, 'min_sentence2_length': 13, 'average_sentence2_len': 51.22, 'max_sentence2_length': 129, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.23, 'max_score': 5.0}, 'fr-en': {'num_samples': 250, 'number_of_characters': 23087, 'min_sentence1_length': 19, 'average_sentence1_len': 49.62, 'max_sentence1_length': 115, 'unique_sentence1': 250, 'min_sentence2_length': 15, 'average_sentence2_len': 42.72, 'max_sentence2_length': 101, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.28, 'max_score': 5.0}, 'it-en': {'num_samples': 250, 'number_of_characters': 23188, 'min_sentence1_length': 15, 'average_sentence1_len': 50.03, 'max_sentence1_length': 113, 'unique_sentence1': 250, 'min_sentence2_length': 15, 'average_sentence2_len': 42.72, 'max_sentence2_length': 101, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.28, 'max_score': 5.0}, 'nl-en': {'num_samples': 250, 'number_of_characters': 22385, 'min_sentence1_length': 14, 'average_sentence1_len': 46.82, 'max_sentence1_length': 123, 'unique_sentence1': 250, 'min_sentence2_length': 15, 'average_sentence2_len': 42.72, 'max_sentence2_length': 101, 'unique_sentence2': 250, 'min_score': 0.0, 'avg_score': 2.28, 'max_score': 5.0}}}} | +| [STS17MultilingualVisualSTS](https://arxiv.org/abs/2402.08183/) (Xiao et al., 2024) | ['ara', 'deu', 'eng', 'fra', 'ita', 'kor', 'nld', 'spa', 'tur'] | VisualSTS | i2i | [News, Social, Spoken, Web, Written] | None | None | | [STS22.v2](https://competitions.codalab.org/competitions/33835) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'ita', 'pol', 'rus', 'spa', 'tur'] | STS | p2p | [News, Written] | None | None | ->>>>>>> main | [STSB](https://aclanthology.org/2021.emnlp-main.357) (Shitao Xiao, 2024) | ['cmn'] | STS | s2s | | None | None | -| [STSBenchmark](https://github.com/PhilipMay/stsb-multi-mt/) (Philip May, 2021) | ['eng'] | STS | s2s | | None | None | -| [STSBenchmarkMultilingualSTS](https://github.com/PhilipMay/stsb-multi-mt/) (Philip May, 2021) | ['cmn', 'deu', 'eng', 'fra', 'ita', 'nld', 'pol', 'por', 'rus', 'spa'] | STS | s2s | [News, Social, Web, Spoken, Written] | None | None | +| [STSBenchmark](https://github.com/PhilipMay/stsb-multi-mt/) (Philip May, 2021) | ['eng'] | STS | s2s | [Blog, News, Written] | None | None | +| [STSBenchmarkMultilingualSTS](https://github.com/PhilipMay/stsb-multi-mt/) (Philip May, 2021) | ['cmn', 'deu', 'eng', 'fra', 'ita', 'nld', 'pol', 'por', 'rus', 'spa'] | STS | s2s | [News, Social, Spoken, Web, Written] | None | None | +| [STSBenchmarkMultilingualVisualSTS](https://arxiv.org/abs/2402.08183/) (Xiao et al., 2024) | ['cmn', 'deu', 'eng', 'fra', 'ita', 'nld', 'pol', 'por', 'rus', 'spa'] | VisualSTS | i2i | [News, Social, Spoken, Web, Written] | None | None | | [STSES](https://huggingface.co/datasets/PlanTL-GOB-ES/sts-es) (Agirre et al., 2015) | ['spa'] | STS | s2s | [Written] | None | None | -<<<<<<< HEAD -| [SadeemQuestionRetrieval](https://huggingface.co/datasets/sadeem-ai/sadeem-ar-eval-retrieval-questions) | ['ara'] | Retrieval | s2p | [Written, Written] | {'test': 22979} | {'test': 500.0} | -| [SanskritShlokasClassification](https://github.com/goru001/nlp-for-sanskrit) | ['san'] | Classification | s2s | [Religious, Written] | {'train': 383, 'validation': 96} | {'train': 98.415, 'validation': 96.635} | -| [ScalaClassification](https://aclanthology.org/2023.nodalida-1.20/) | ['dan', 'nno', 'nob', 'swe'] | Classification | s2s | [Fiction, News, Non-fiction, Blog, Spoken, Web, Written] | {'test': 4096} | {'test': 102.72} | -| [SciDocsRR](https://allenai.org/data/scidocs) | ['eng'] | Reranking | s2s | [Academic, Non-fiction, Written] | {'test': 19599} | {'test': 69.0} | -| [SciFact](https://github.com/allenai/scifact) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | | None | {'train': {'average_document_length': 1498.4152035500674, 'average_query_length': 88.58838071693448, 'num_documents': 5183, 'num_queries': 809, 'average_relevant_docs_per_query': 1.1359703337453646}, 'test': {'average_document_length': 1498.4152035500674, 'average_query_length': 90.34666666666666, 'num_documents': 5183, 'num_queries': 300, 'average_relevant_docs_per_query': 1.13}} | -| [SciFact-PL](https://github.com/allenai/scifact) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1553.5178468068686, 'average_query_length': 95.44, 'num_documents': 5183, 'num_queries': 300, 'average_relevant_docs_per_query': 1.13}} | -| [SemRel24STS](https://huggingface.co/datasets/SemRel/SemRel2024) (Nedjma Ousidhoum, 2024) | ['afr', 'amh', 'arb', 'arq', 'ary', 'eng', 'hau', 'hin', 'ind', 'kin', 'mar', 'tel'] | STS | s2s | [Spoken, Written] | {'dev': 2089, 'test': 7498} | {'dev': 163.1, 'test': 145.9} | -| [SensitiveTopicsClassification](https://aclanthology.org/2021.bsnlp-1.4) | ['rus'] | MultilabelClassification | s2s | [Web, Social, Written] | {'test': 2048} | {'test': 95.3} | -| [SentimentAnalysisHindi](https://huggingface.co/datasets/OdiaGenAI/sentiment_analysis_hindi) (Shantipriya Parida, 2023) | ['hin'] | Classification | s2s | [Reviews, Written] | {'train': 2497} | {'train': 81.29} | -| [SinhalaNewsClassification](https://huggingface.co/datasets/NLPC-UOM/Sinhala-News-Category-classification) (Nisansa de Silva, 2015) | ['sin'] | Classification | s2s | [News, Written] | {'train': 3327} | {'train': 148.04} | -| [SinhalaNewsSourceClassification](https://huggingface.co/datasets/NLPC-UOM/Sinhala-News-Source-classification) (Dhananjaya et al., 2022) | ['sin'] | Classification | s2s | [News, Written] | {'train': 24094} | {'train': 56.08} | -| [SiswatiNewsClassification](https://huggingface.co/datasets/dsfsi/za-isizulu-siswati-news) (Madodonga et al., 2023) | ['ssw'] | Classification | s2s | [News, Written] | {'train': 80} | {'train': 354.2} | -| [SlovakHateSpeechClassification](https://huggingface.co/datasets/TUKE-KEMT/hate_speech_slovak) | ['slk'] | Classification | s2s | [Social, Written] | {'test': 1319} | {'test': 92.71} | -| [SlovakMovieReviewSentimentClassification](https://arxiv.org/pdf/2304.01922) ({ {S, 2023) | ['svk'] | Classification | s2s | [Reviews, Written] | {'test': 2048} | {'test': 366.17} | -| [SlovakSumRetrieval](https://huggingface.co/datasets/NaiveNeuron/slovaksum) | ['slk'] | Retrieval | s2s | [News, Social, Web, Written] | {'test': 600} | {'test': {'average_document_length': 2156.445, 'average_query_length': 143.59833333333333, 'num_documents': 600, 'num_queries': 600, 'average_relevant_docs_per_query': 1.0}} | -| [SouthAfricanLangClassification](https://www.kaggle.com/competitions/south-african-language-identification/) (ExploreAI Academy et al., 2022) | ['afr', 'eng', 'nbl', 'nso', 'sot', 'ssw', 'tsn', 'tso', 'ven', 'xho', 'zul'] | Classification | s2s | [Web, Non-fiction, Written] | {'test': 2048} | {'test': 247.49} | -| [SpanishNewsClassification](https://huggingface.co/datasets/MarcOrfilaCarreras/spanish-news) | ['spa'] | Classification | s2s | [News, Written] | {'train': 2048} | {'train': 4218.2} | -======= +| [SUN397](https://ieeexplore.ieee.org/abstract/document/5539970) (Xiao et al., 2010) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [SUN397ZeroShot](https://ieeexplore.ieee.org/abstract/document/5539970) (Xiao et al., 2010) | ['eng'] | ZeroShotClassification | i2t | [Encyclopaedic] | None | None | | [SadeemQuestionRetrieval](https://huggingface.co/datasets/sadeem-ai/sadeem-ar-eval-retrieval-questions) | ['ara'] | Retrieval | s2p | [Written, Written] | None | None | | [SanskritShlokasClassification](https://github.com/goru001/nlp-for-sanskrit) | ['san'] | Classification | s2s | [Religious, Written] | None | None | -| [ScalaClassification](https://aclanthology.org/2023.nodalida-1.20/) | ['dan', 'nno', 'nob', 'swe'] | Classification | s2s | [Fiction, News, Non-fiction, Blog, Spoken, Web, Written] | None | None | +| [ScalaClassification](https://aclanthology.org/2023.nodalida-1.20/) | ['dan', 'nno', 'nob', 'swe'] | Classification | s2s | [Blog, Fiction, News, Non-fiction, Spoken, Web, Written] | None | None | | [SciDocsRR](https://allenai.org/data/scidocs) | ['eng'] | Reranking | s2s | [Academic, Non-fiction, Written] | None | None | | [SciFact](https://github.com/allenai/scifact) (Arman Cohan, 2020) | ['eng'] | Retrieval | s2p | [Academic, Medical, Written] | None | None | +| [SciFact-Fa](https://huggingface.co/datasets/MCINext/scifact-fa) | ['fas'] | Retrieval | s2p | [Academic] | None | None | | [SciFact-PL](https://github.com/allenai/scifact) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Academic, Medical, Written] | None | None | +| [SciMMIR](https://huggingface.co/datasets/m-a-p/SciMMIR) (Siwei Wu, 2024) | ['eng'] | ZeroShotClassification | i2t | [Academic] | None | None | +| [SciMMIRI2TRetrieval](https://aclanthology.org/2024.findings-acl.746/) (Wu et al., 2024) | ['eng'] | Any2AnyRetrieval | i2t | [Academic] | None | None | +| [SciMMIRT2IRetrieval](https://aclanthology.org/2024.findings-acl.746/) (Wu et al., 2024) | ['eng'] | Any2AnyRetrieval | t2i | [Academic] | None | None | | [SemRel24STS](https://huggingface.co/datasets/SemRel/SemRel2024) (Nedjma Ousidhoum, 2024) | ['afr', 'amh', 'arb', 'arq', 'ary', 'eng', 'hau', 'hin', 'ind', 'kin', 'mar', 'tel'] | STS | s2s | [Spoken, Written] | None | None | -| [SensitiveTopicsClassification](https://aclanthology.org/2021.bsnlp-1.4) | ['rus'] | MultilabelClassification | s2s | [Web, Social, Written] | None | None | +| [SensitiveTopicsClassification](https://aclanthology.org/2021.bsnlp-1.4) | ['rus'] | MultilabelClassification | s2s | [Social, Web, Written] | None | None | | [SentimentAnalysisHindi](https://huggingface.co/datasets/OdiaGenAI/sentiment_analysis_hindi) (Shantipriya Parida, 2023) | ['hin'] | Classification | s2s | [Reviews, Written] | None | None | +| [SentimentDKSF](https://github.com/hezarai/hezar) | ['fas'] | Classification | s2p | [Reviews] | None | None | | [SinhalaNewsClassification](https://huggingface.co/datasets/NLPC-UOM/Sinhala-News-Category-classification) (Nisansa de Silva, 2015) | ['sin'] | Classification | s2s | [News, Written] | None | None | | [SinhalaNewsSourceClassification](https://huggingface.co/datasets/NLPC-UOM/Sinhala-News-Source-classification) (Dhananjaya et al., 2022) | ['sin'] | Classification | s2s | [News, Written] | None | None | | [SiswatiNewsClassification](https://huggingface.co/datasets/dsfsi/za-isizulu-siswati-news) (Madodonga et al., 2023) | ['ssw'] | Classification | s2s | [News, Written] | None | None | +| [SketchyI2IRetrieval](https://arxiv.org/abs/2202.01747) (Ypsilantis et al., 2021) | ['eng'] | Any2AnyRetrieval | i2i | [Encyclopaedic] | None | None | | [SlovakHateSpeechClassification](https://huggingface.co/datasets/TUKE-KEMT/hate_speech_slovak) | ['slk'] | Classification | s2s | [Social, Written] | {'test': 1319, 'train': 11870} | {'test': {'num_samples': 1319, 'number_of_characters': 122279, 'num_texts_in_train': 46, 'min_text_length': 8, 'average_text_length': 92.71, 'max_text_length': 1584, 'unique_text': 1315, 'unique_labels': 2, 'labels': {'1': {'count': 360}, '0': {'count': 959}}}, 'train': {'num_samples': 11870, 'number_of_characters': 1130860, 'num_texts_in_train': None, 'min_text_length': 7, 'average_text_length': 95.27, 'max_text_length': 2112, 'unique_text': 11655, 'unique_labels': 2, 'labels': {'1': {'count': 3245}, '0': {'count': 8625}}}} | | [SlovakMovieReviewSentimentClassification](https://arxiv.org/pdf/2304.01922) ({ {S, 2023) | ['svk'] | Classification | s2s | [Reviews, Written] | None | None | | [SlovakSumRetrieval](https://huggingface.co/datasets/NaiveNeuron/slovaksum) | ['slk'] | Retrieval | s2s | [News, Social, Web, Written] | None | None | -| [SouthAfricanLangClassification](https://www.kaggle.com/competitions/south-african-language-identification/) (ExploreAI Academy et al., 2022) | ['afr', 'eng', 'nbl', 'nso', 'sot', 'ssw', 'tsn', 'tso', 'ven', 'xho', 'zul'] | Classification | s2s | [Web, Non-fiction, Written] | None | None | +| [SouthAfricanLangClassification](https://www.kaggle.com/competitions/south-african-language-identification/) (ExploreAI Academy et al., 2022) | ['afr', 'eng', 'nbl', 'nso', 'sot', 'ssw', 'tsn', 'tso', 'ven', 'xho', 'zul'] | Classification | s2s | [Non-fiction, Web, Written] | None | None | | [SpanishNewsClassification](https://huggingface.co/datasets/MarcOrfilaCarreras/spanish-news) | ['spa'] | Classification | s2s | [News, Written] | None | None | ->>>>>>> main | [SpanishNewsClusteringP2P](https://www.kaggle.com/datasets/kevinmorgado/spanish-news-classification) | ['spa'] | Clustering | p2p | | None | None | | [SpanishPassageRetrievalS2P](https://mklab.iti.gr/results/spanish-passage-retrieval-dataset/) | ['spa'] | Retrieval | s2p | | None | None | | [SpanishPassageRetrievalS2S](https://mklab.iti.gr/results/spanish-passage-retrieval-dataset/) | ['spa'] | Retrieval | s2s | | None | None | @@ -933,9 +687,13 @@ The following tables give you an overview of the tasks in MTEB. | [SprintDuplicateQuestions](https://www.aclweb.org/anthology/D18-1131/) | ['eng'] | PairClassification | s2s | [Programming, Written] | None | None | | [StackExchangeClustering.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | s2s | [Web, Written] | None | None | | [StackExchangeClusteringP2P.v2](https://arxiv.org/abs/2104.07081) (Gregor Geigle, 2021) | ['eng'] | Clustering | p2p | [Web, Written] | None | None | -| [StackOverflowDupQuestions](https://www.microsoft.com/en-us/research/uploads/prod/2019/03/nl4se18LinkSO.pdf) (Xueqing Liu, 2018) | ['eng'] | Reranking | s2s | | None | None | +| [StackOverflowDupQuestions](https://www.microsoft.com/en-us/research/uploads/prod/2019/03/nl4se18LinkSO.pdf) (Xueqing Liu, 2018) | ['eng'] | Reranking | s2s | [Blog, Programming, Written] | None | None | | [StackOverflowQA](https://arxiv.org/abs/2407.02883) (Xiangyang Li, 2024) | ['eng'] | Retrieval | p2p | [Programming, Written] | {'test': 21925} | {'test': {'number_of_characters': 26584028, 'num_samples': 21925, 'num_queries': 1994, 'num_documents': 19931, 'min_document_length': 61, 'average_document_length': 130.32, 'max_document_length': 22234, 'unique_documents': 19931, 'min_query_length': 5, 'average_query_length': 12029.38, 'max_query_length': 46028, 'unique_queries': 1994, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 1994}} | +| [StanfordCars](https://pure.mpg.de/rest/items/item_2029263/component/file_2029262/content) (Jonathan Krause, 2013) | ['eng'] | ImageClassification | i2i | [Encyclopaedic] | None | None | +| [StanfordCarsI2IRetrieval](https://pure.mpg.de/rest/items/item_2029263/component/file_2029262/content) (Jonathan Krause, 2013) | ['eng'] | Any2AnyRetrieval | i2i | [Encyclopaedic] | None | None | +| [StanfordCarsZeroShot](https://pure.mpg.de/rest/items/item_2029263/component/file_2029262/content) (Jonathan Krause, 2013) | ['eng'] | ZeroShotClassification | i2t | [Scene] | None | None | | [StatcanDialogueDatasetRetrieval](https://mcgill-nlp.github.io/statcan-dialogue-dataset/) | ['eng', 'fra'] | Retrieval | s2p | [Government, Web, Written] | None | None | +| [SugarCrepe](https://proceedings.neurips.cc/paper_files/paper/2023/hash/63461de0b4cb760fc498e85b18a7fe81-Abstract-Datasets_and_Benchmarks.html) (Hsieh et al., 2024) | ['eng'] | ImageTextPairClassification | i2t | [Encyclopaedic] | None | None | | [SummEvalFrSummarization.v2](https://github.com/Yale-LILY/SummEval) (Fabbri et al., 2020) | ['fra'] | Summarization | p2p | [News, Written] | None | None | | [SummEvalSummarization.v2](https://github.com/Yale-LILY/SummEval) (Fabbri et al., 2020) | ['eng'] | Summarization | p2p | [News, Written] | None | None | | [SwahiliNewsClassification](https://huggingface.co/datasets/Mollel/SwahiliNewsClassification) | ['swa'] | Classification | s2s | [News, Written] | None | None | @@ -946,6 +704,33 @@ The following tables give you an overview of the tasks in MTEB. | [SwednClusteringS2S](https://spraakbanken.gu.se/en/resources/swedn) (Monsen et al., 2021) | ['swe'] | Clustering | s2s | [News, Non-fiction, Written] | None | None | | [SwednRetrieval](https://spraakbanken.gu.se/en/resources/swedn) (Monsen et al., 2021) | ['swe'] | Retrieval | p2p | [News, Non-fiction, Written] | None | None | | [SwissJudgementClassification](https://aclanthology.org/2021.nllp-1.3/) (Joel Niklaus, 2022) | ['deu', 'fra', 'ita'] | Classification | s2s | [Legal, Written] | None | None | +| [SynPerChatbotConvSAAnger](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| SynPerChatbotConvSAClassification | ['fas'] | Classification | None | [Spoken] | None | None | +| [SynPerChatbotConvSAFear](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotConvSAFriendship](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotConvSAHappiness](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotConvSAJealousy](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotConvSALove](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotConvSASadness](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotConvSASatisfaction](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotConvSASurprise](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotConvSAToneChatbotClassification](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotConvSAToneUserClassification](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotRAGFAQPC](https://mcinext.com/) | ['fas'] | PairClassification | s2p | [Spoken] | None | None | +| [SynPerChatbotRAGFAQRetrieval](https://huggingface.co/datasets/MCINext/synthetic-persian-chatbot-rag-faq-retrieval) | ['fas'] | Retrieval | s2p | [Spoken] | None | None | +| [SynPerChatbotRAGSumSRetrieval](https://huggingface.co/datasets/MCINext/synthetic-persian-chatbot-rag-summary-retrieval) | ['fas'] | BitextMining | p2p | [Spoken] | None | None | +| [SynPerChatbotRAGToneChatbotClassification](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotRAGToneUserClassification](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotRAGTopicsRetrieval](https://huggingface.co/datasets/MCINext/synthetic-persian-chatbot-rag-topics-retrieval) | ['fas'] | Retrieval | s2p | [Spoken] | None | None | +| [SynPerChatbotSatisfactionLevelClassification](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotSumSRetrieval](https://huggingface.co/datasets/MCINext/synthetic-persian-chatbot-summary-retrieval) | ['fas'] | BitextMining | p2p | [Spoken] | None | None | +| [SynPerChatbotToneChatbotClassification](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotToneUserClassification](https://mcinext.com/) | ['fas'] | Classification | p2p | [Spoken] | None | None | +| [SynPerChatbotTopicsRetrieval](https://huggingface.co/datasets/MCINext/synthetic-persian-chatbot-topics-retrieval) | ['fas'] | Retrieval | s2p | [Spoken] | None | None | +| [SynPerQAPC](https://mcinext.com/) | ['fas'] | PairClassification | s2p | [Blog, News, Religious, Web] | None | None | +| [SynPerQARetrieval](https://huggingface.co/datasets/MCINext/synthetic-persian-qa-retrieval/settings) | ['fas'] | Retrieval | s2p | [Web] | None | None | +| [SynPerSTS](https://mcinext.com/) | ['fas'] | STS | s2s | [Blog, News, Religious, Web] | None | None | +| [SynPerTextKeywordsPC](https://mcinext.com/) | ['fas'] | PairClassification | s2p | [Blog, News, Religious, Web] | None | None | | [SyntecReranking](https://huggingface.co/datasets/lyon-nlp/mteb-fr-reranking-syntec-s2p) (Mathieu Ciancone, 2024) | ['fra'] | Reranking | s2p | [Legal, Written] | None | None | | [SyntecRetrieval](https://huggingface.co/datasets/lyon-nlp/mteb-fr-retrieval-syntec-s2p) (Mathieu Ciancone, 2024) | ['fra'] | Retrieval | s2p | [Legal, Written] | None | None | | [SyntheticText2SQL](https://huggingface.co/datasets/gretelai/synthetic_text_to_sql) (Meyer et al., 2024) | ['eng', 'sql'] | Retrieval | p2p | [Programming, Written] | {'test': 111702} | {'test': {'number_of_characters': 14041553, 'num_samples': 111702, 'num_queries': 5851, 'num_documents': 105851, 'min_document_length': 13, 'average_document_length': 4.58, 'max_document_length': 281, 'unique_documents': 105851, 'min_query_length': 17, 'average_query_length': 2316.95, 'max_query_length': 762, 'unique_queries': 5851, 'min_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 1.0, 'max_relevant_docs_per_query': 1, 'unique_relevant_docs': 5851}} | @@ -953,63 +738,10 @@ The following tables give you an overview of the tasks in MTEB. | [T2Retrieval](https://arxiv.org/abs/2304.03679) (Xiaohui Xie, 2023) | ['cmn'] | Retrieval | s2p | | None | None | | [TERRa](https://arxiv.org/pdf/2010.15925) (Shavrina et al., 2020) | ['rus'] | PairClassification | s2s | [News, Web, Written] | None | None | | [TNews](https://www.cluebenchmarks.com/introduce.html) | ['cmn'] | Classification | s2s | | None | None | -<<<<<<< HEAD -| [TRECCOVID](https://ir.nist.gov/covidSubmit/index.html) (Kirk Roberts, 2021) | ['eng'] | Retrieval | s2p | | None | {'test': {'average_document_length': 1116.7434221277986, 'average_query_length': 69.24, 'num_documents': 171332, 'num_queries': 50, 'average_relevant_docs_per_query': 493.5}} | -| [TRECCOVID-PL](https://ir.nist.gov/covidSubmit/index.html) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Academic, Non-fiction, Written] | None | {'test': {'average_document_length': 1159.8020276422385, 'average_query_length': 69.42, 'num_documents': 171332, 'num_queries': 50, 'average_relevant_docs_per_query': 493.5}} | -| [TV2Nordretrieval](https://huggingface.co/datasets/alexandrainst/nordjylland-news-summarization) | ['dan'] | Retrieval | p2p | [News, Non-fiction, Written] | {'test': 4096} | {'test': {'average_document_length': 1440.66552734375, 'average_query_length': 126.552734375, 'num_documents': 2048, 'num_queries': 2048, 'average_relevant_docs_per_query': 1.0}} | -| [TamilNewsClassification](https://github.com/vanangamudi/tamil-news-classification) (Anoop Kunchukuttan, 2020) | ['tam'] | Classification | s2s | [News, Written] | {'train': 14521, 'test': 3631} | {'train': 56.5, 'test': 56.52} | -| [Tatoeba](https://github.com/facebookresearch/LASER/tree/main/data/tatoeba/v1) (Tatoeba community, 2021) | ['afr', 'amh', 'ang', 'ara', 'arq', 'arz', 'ast', 'awa', 'aze', 'bel', 'ben', 'ber', 'bos', 'bre', 'bul', 'cat', 'cbk', 'ceb', 'ces', 'cha', 'cmn', 'cor', 'csb', 'cym', 'dan', 'deu', 'dsb', 'dtp', 'ell', 'eng', 'epo', 'est', 'eus', 'fao', 'fin', 'fra', 'fry', 'gla', 'gle', 'glg', 'gsw', 'heb', 'hin', 'hrv', 'hsb', 'hun', 'hye', 'ido', 'ile', 'ina', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kat', 'kaz', 'khm', 'kor', 'kur', 'kzj', 'lat', 'lfn', 'lit', 'lvs', 'mal', 'mar', 'max', 'mhr', 'mkd', 'mon', 'nds', 'nld', 'nno', 'nob', 'nov', 'oci', 'orv', 'pam', 'pes', 'pms', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'spa', 'sqi', 'srp', 'swe', 'swg', 'swh', 'tam', 'tat', 'tel', 'tgl', 'tha', 'tuk', 'tur', 'tzl', 'uig', 'ukr', 'urd', 'uzb', 'vie', 'war', 'wuu', 'xho', 'yid', 'yue', 'zsm'] | BitextMining | s2s | [Written] | {'test': 2000} | {'test': 39.4} | -| [TbilisiCityHallBitextMining](https://huggingface.co/datasets/jupyterjazz/tbilisi-city-hall-titles) | ['eng', 'kat'] | BitextMining | s2s | [News, Written] | {'test': 1820} | {'test': 78} | -| [TelemarketingSalesRuleLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 47} | {'test': 348.29} | -| [TeluguAndhraJyotiNewsClassification](https://github.com/AnushaMotamarri/Telugu-Newspaper-Article-Dataset) | ['tel'] | Classification | s2s | [News, Written] | {'test': 4329} | {'test': 1428.28} | -| [TempReasonL1](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 4000} | {'test': {'average_document_length': 8.989843250159948, 'average_query_length': 50.22375, 'num_documents': 12504, 'num_queries': 4000, 'average_relevant_docs_per_query': 1.0}} | -| [TempReasonL2Context](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 0} | {'test': {'average_document_length': 19.823525685690758, 'average_query_length': 11919.25792106726, 'num_documents': 15787, 'num_queries': 5397, 'average_relevant_docs_per_query': 1.0}} | -| [TempReasonL2Fact](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 5397} | {'test': {'average_document_length': 19.823525685690758, 'average_query_length': 830.7268853066519, 'num_documents': 15787, 'num_queries': 5397, 'average_relevant_docs_per_query': 1.0}} | -| [TempReasonL2Pure](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 5397} | {'test': {'average_document_length': 19.823525685690758, 'average_query_length': 55.94089308875301, 'num_documents': 15787, 'num_queries': 5397, 'average_relevant_docs_per_query': 1.0}} | -| [TempReasonL3Context](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 4426} | {'test': {'average_document_length': 19.80534984678243, 'average_query_length': 13424.633077270673, 'num_documents': 15664, 'num_queries': 4426, 'average_relevant_docs_per_query': 1.0}} | -| [TempReasonL3Fact](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 4426} | {'test': {'average_document_length': 19.80534984678243, 'average_query_length': 896.0754631721645, 'num_documents': 15664, 'num_queries': 4426, 'average_relevant_docs_per_query': 1.0}} | -| [TempReasonL3Pure](https://github.com/DAMO-NLP-SG/TempReason) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 4426} | {'test': {'average_document_length': 19.80534984678243, 'average_query_length': 74.44012652507908, 'num_documents': 15664, 'num_queries': 4426, 'average_relevant_docs_per_query': 1.0}} | -| [TenKGnadClassification](https://tblock.github.io/10kGNAD/) | ['deu'] | Classification | p2p | [News, Written] | {'test': 1028} | {'test': 2627.31} | -| [TenKGnadClusteringP2P.v2](https://tblock.github.io/10kGNAD/) | ['deu'] | Clustering | p2p | [News, Non-fiction, Written] | {'test': 10275} | {'test': 2641.03} | -| [TenKGnadClusteringS2S.v2](https://tblock.github.io/10kGNAD/) | ['deu'] | Clustering | s2s | [News, Non-fiction, Written] | {'test': 10267} | {'test': 50.96} | -| [TextualismToolDictionariesLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 107} | {'test': 943.23} | -| [TextualismToolPlainLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 165} | {'test': 997.97} | -| [ThuNewsClusteringP2P.v2](http://thuctc.thunlp.org/) (Sun et al., 2016) | ['cmn'] | Clustering | p2p | [News, Written] | {'test': 2048} | {} | -| [ThuNewsClusteringS2S.v2](http://thuctc.thunlp.org/) (Sun et al., 2016) | ['cmn'] | Clustering | s2s | [News, Written] | {'test': 2048} | {} | -| [TopiOCQA](https://mcgill-nlp.github.io/topiocqa) (Vaibhav Adlakha, 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | {'dev': 2514} | {'validation': {'average_document_length': 478.8968086416064, 'average_query_length': 12.579952267303103, 'num_documents': 25700592, 'num_queries': 2514, 'average_relevant_docs_per_query': 1.0}} | -| [TopiOCQAHardNegatives](https://mcgill-nlp.github.io/topiocqa) (Vaibhav Adlakha, 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | {'test': 1000} | {'validation': {'average_document_length': 538.7586536643946, 'average_query_length': 12.85, 'num_documents': 89933, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}} | -| [Touche2020Retrieval.v3](https://github.com/castorini/touche-error-analysis) | ['eng'] | Retrieval | s2p | [Academic] | | | -| [ToxicChatClassification](https://aclanthology.org/2023.findings-emnlp.311/) (Zi Lin, 2023) | ['eng'] | Classification | s2s | [Constructed, Written] | {'test': 1427} | {'test': 189.4} | -| [ToxicConversationsClassification](https://www.kaggle.com/competitions/jigsaw-unintended-bias-in-toxicity-classification/overview) (cjadams, 2019) | ['eng'] | Classification | s2s | [Social, Written] | {'test': 50000} | {'test': 296.6} | -| [TswanaNewsClassification](https://link.springer.com/chapter/10.1007/978-3-031-49002-6_17) (Vukosi Marivate, 2023) | ['tsn'] | Classification | s2s | [News, Written] | {'validation': 487, 'test': 487} | {'validation': 2417.72, 'test': 2369.52} | -| [TurHistQuadRetrieval](https://github.com/okanvk/Turkish-Reading-Comprehension-Question-Answering-Dataset) (Soygazi et al., 2021) | ['tur'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Academic, Written] | {'test': 1330} | {'test': {'average_document_length': 172.12118713932398, 'average_query_length': 62.5302734375, 'num_documents': 1213, 'num_queries': 1024, 'average_relevant_docs_per_query': 2.0}} | -| [TurkicClassification](https://huggingface.co/datasets/Electrotubbie/classification_Turkic_languages/) | ['bak', 'kaz', 'kir'] | Classification | s2s | [News, Written] | {'train': 193056} | {'train': 1103.13} | -| [TurkishMovieSentimentClassification](https://www.win.tue.nl/~mpechen/publications/pubs/MT_WISDOM2013.pdf) (Erkin Demirtas, 2013) | ['tur'] | Classification | s2s | [Reviews, Written] | {'test': 2644} | {'test': 141.5} | -| [TurkishProductSentimentClassification](https://www.win.tue.nl/~mpechen/publications/pubs/MT_WISDOM2013.pdf) (Erkin Demirtas, 2013) | ['tur'] | Classification | s2s | [Reviews, Written] | {'test': 800} | {'test': 246.85} | -| [TweetEmotionClassification](https://link.springer.com/chapter/10.1007/978-3-319-77116-8_8) (Al-Khatib et al., 2018) | ['ara'] | Classification | s2s | [Social, Written] | {'train': 2048} | {'train': 78.8} | -| [TweetSarcasmClassification](https://aclanthology.org/2020.osact-1.5/) | ['ara'] | Classification | s2s | [Social, Written] | {'test': 2110} | {'test': 102.1} | -| [TweetSentimentClassification](https://aclanthology.org/2022.lrec-1.27) | ['ara', 'deu', 'eng', 'fra', 'hin', 'ita', 'por', 'spa'] | Classification | s2s | [Social, Written] | {'test': 2048} | {'test': 83.51} | -| [TweetSentimentExtractionClassification](https://www.kaggle.com/competitions/tweet-sentiment-extraction/overview) (Maggie et al., 2020) | ['eng'] | Classification | s2s | [Social, Written] | {'test': 3534} | {'test': 67.8} | -| [TweetTopicSingleClassification](https://arxiv.org/abs/2209.09824) | ['eng'] | Classification | s2s | [Social, News, Written] | {'test_2021': 1693} | {'test_2021': 167.66} | -| [TwentyNewsgroupsClustering.v2](https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html) (Ken Lang, 1995) | ['eng'] | Clustering | s2s | [News, Written] | {'test': 2381} | {'test': 32.0} | -| [TwitterHjerneRetrieval](https://huggingface.co/datasets/sorenmulli/da-hashtag-twitterhjerne) (Holm et al., 2024) | ['dan'] | Retrieval | p2p | [Social, Written] | {'train': 340} | {'train': {'average_document_length': 128.85114503816794, 'average_query_length': 166.3846153846154, 'num_documents': 262, 'num_queries': 78, 'average_relevant_docs_per_query': 3.358974358974359}} | -| [TwitterSemEval2015](https://alt.qcri.org/semeval2015/task1/) | ['eng'] | PairClassification | s2s | | {'test': 16777} | {'test': 38.3} | -| [TwitterURLCorpus](https://languagenet.github.io/) | ['eng'] | PairClassification | s2s | | {'test': 51534} | {'test': {'num_samples': 51534, 'avg_sentence1_len': 79.48919160166103, 'avg_sentence2_len': 88.5540419916948, 'unique_labels': 2, 'labels': {'0': {'count': 38546}, '1': {'count': 12988}}}} | -| [UCCVCommonLawLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 94} | {'test': 114.127} | -| [UkrFormalityClassification](https://huggingface.co/datasets/ukr-detect/ukr-formality-dataset-translated-gyafc) | ['ukr'] | Classification | s2s | [News, Written] | {'train': 2048, 'test': 2048} | {'train': 52.1, 'test': 53.07} | -| [UnfairTOSLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | {'test': 2048} | {'test': 184.69} | -| [UrduRomanSentimentClassification](https://archive.ics.uci.edu/dataset/458/roman+urdu+data+set) (Sharf,Zareen, 2018) | ['urd'] | Classification | s2s | [Social, Written] | {'train': 2048} | {'train': 68.248} | -| [VGHierarchicalClusteringP2P](https://huggingface.co/datasets/navjordj/VG_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | p2p | [News, Non-fiction, Written] | {'test': 2048} | {'test': 2670.3243084794544} | -| [VGHierarchicalClusteringS2S](https://huggingface.co/datasets/navjordj/VG_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | p2p | [News, Non-fiction, Written] | {'test': 2048} | {'test': 139.31247668283325} | -| [VideoRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | {'dev': {'average_document_length': 31.048855642524522, 'average_query_length': 7.365, 'num_documents': 100930, 'num_queries': 1000, 'average_relevant_docs_per_query': 1.0}} | -| [VieMedEVBitextMining](https://aclanthology.org/2015.iwslt-evaluation.11/) (Nhu Vo, 2024) | ['eng', 'vie'] | BitextMining | s2s | [Medical, Written] | {'test': 2048} | {'test': 139.23} | -| [VieQuADRetrieval](https://aclanthology.org/2020.coling-main.233.pdf) | ['vie'] | Retrieval | s2p | [Encyclopaedic, Non-fiction, Written] | {'validation': 2048} | {'validation': {'average_document_length': 222.61244979919678, 'average_query_length': 65.51513671875, 'num_documents': 2490, 'num_queries': 2048, 'average_relevant_docs_per_query': 2.0}} | -| [VieStudentFeedbackClassification](https://ieeexplore.ieee.org/document/8573337) (Nguyen et al., 2018) | ['vie'] | Classification | s2s | [Reviews, Written] | {'test': 2048} | {'test': 14.22} | -| [VoyageMMarcoReranking](https://arxiv.org/abs/2312.16144) (Benjamin Clavié, 2023) | ['jpn'] | Reranking | s2s | [Academic, Non-fiction, Written] | {'test': 2048} | {'test': 162} | -| [WRIMEClassification](https://aclanthology.org/2021.naacl-main.169/) | ['jpn'] | Classification | s2s | [Social, Written] | {'test': 2048} | {'test': 47.78} | -======= -| [TRECCOVID](https://ir.nist.gov/covidSubmit/index.html) (Kirk Roberts, 2021) | ['eng'] | Retrieval | s2p | [Medical, Academic, Written] | None | None | +| [TRECCOVID](https://ir.nist.gov/covidSubmit/index.html) (Kirk Roberts, 2021) | ['eng'] | Retrieval | s2p | [Academic, Medical, Written] | None | None | +| [TRECCOVID-Fa](https://huggingface.co/datasets/MCINext/trec-covid-fa) | ['fas'] | Retrieval | s2p | [Medical] | None | None | | [TRECCOVID-PL](https://ir.nist.gov/covidSubmit/index.html) (Konrad Wojtasik, 2024) | ['pol'] | Retrieval | s2p | [Academic, Medical, Non-fiction, Written] | None | None | +| [TUBerlinT2IRetrieval](https://dl.acm.org/doi/pdf/10.1145/2185520.2185540?casa_token=tq-eUx5UROYAAAAA:_694nPzE7tali6LCkxQc0M-mlo9xslasPMcVnFPMy9tDfvt7lg7p1RTe-k8VWCjuv9gmkQqasKUZ) (Eitz et al., 2012) | ['eng'] | Any2AnyRetrieval | t2i | [Encyclopaedic] | None | None | | [TV2Nordretrieval](https://huggingface.co/datasets/alexandrainst/nordjylland-news-summarization) | ['dan'] | Retrieval | p2p | [News, Non-fiction, Written] | None | None | | [TamilNewsClassification](https://github.com/vanangamudi/tamil-news-classification) (Anoop Kunchukuttan, 2020) | ['tam'] | Classification | s2s | [News, Written] | None | None | | [Tatoeba](https://github.com/facebookresearch/LASER/tree/main/data/tatoeba/v1) (Tatoeba community, 2021) | ['afr', 'amh', 'ang', 'ara', 'arq', 'arz', 'ast', 'awa', 'aze', 'bel', 'ben', 'ber', 'bos', 'bre', 'bul', 'cat', 'cbk', 'ceb', 'ces', 'cha', 'cmn', 'cor', 'csb', 'cym', 'dan', 'deu', 'dsb', 'dtp', 'ell', 'eng', 'epo', 'est', 'eus', 'fao', 'fin', 'fra', 'fry', 'gla', 'gle', 'glg', 'gsw', 'heb', 'hin', 'hrv', 'hsb', 'hun', 'hye', 'ido', 'ile', 'ina', 'ind', 'isl', 'ita', 'jav', 'jpn', 'kab', 'kat', 'kaz', 'khm', 'kor', 'kur', 'kzj', 'lat', 'lfn', 'lit', 'lvs', 'mal', 'mar', 'max', 'mhr', 'mkd', 'mon', 'nds', 'nld', 'nno', 'nob', 'nov', 'oci', 'orv', 'pam', 'pes', 'pms', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'spa', 'sqi', 'srp', 'swe', 'swg', 'swh', 'tam', 'tat', 'tel', 'tgl', 'tha', 'tuk', 'tur', 'tzl', 'uig', 'ukr', 'urd', 'uzb', 'vie', 'war', 'wuu', 'xho', 'yid', 'yue', 'zsm'] | BitextMining | s2s | [Written] | None | None | @@ -1030,13 +762,15 @@ The following tables give you an overview of the tasks in MTEB. | [TextualismToolPlainLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [ThuNewsClusteringP2P.v2](http://thuctc.thunlp.org/) (Sun et al., 2016) | ['cmn'] | Clustering | p2p | [News, Written] | None | None | | [ThuNewsClusteringS2S.v2](http://thuctc.thunlp.org/) (Sun et al., 2016) | ['cmn'] | Clustering | s2s | [News, Written] | None | None | +| [TinyImageNetClustering](https://huggingface.co/datasets/zh-plus/tiny-imagenet/viewer/default/valid) | ['eng'] | ImageClustering | i2i | [Reviews] | None | None | | [TopiOCQA](https://mcgill-nlp.github.io/topiocqa) (Vaibhav Adlakha, 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | | [TopiOCQAHardNegatives](https://mcgill-nlp.github.io/topiocqa) (Vaibhav Adlakha, 2022) | ['eng'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [Touche2020-Fa](https://huggingface.co/datasets/MCINext/touche2020-fa) | ['fas'] | Retrieval | s2p | [Spoken] | None | None | | [Touche2020Retrieval.v3](https://github.com/castorini/touche-error-analysis) | ['eng'] | Retrieval | s2p | [Academic] | {'test': 303781} | {'test': {'number_of_characters': 637047138, 'num_samples': 303781, 'num_queries': 49, 'num_documents': 303732, 'min_document_length': 16, 'average_document_length': 0.01, 'max_document_length': 83, 'unique_documents': 303732, 'min_query_length': 41, 'average_query_length': 13000918.57, 'max_query_length': 105983, 'unique_queries': 49, 'min_relevant_docs_per_query': 40, 'average_relevant_docs_per_query': 58.14, 'max_relevant_docs_per_query': 87, 'unique_relevant_docs': 2732}} | | [ToxicChatClassification](https://aclanthology.org/2023.findings-emnlp.311/) (Zi Lin, 2023) | ['eng'] | Classification | s2s | [Constructed, Written] | None | None | | [ToxicConversationsClassification](https://www.kaggle.com/competitions/jigsaw-unintended-bias-in-toxicity-classification/overview) (cjadams, 2019) | ['eng'] | Classification | s2s | [Social, Written] | None | None | | [TswanaNewsClassification](https://link.springer.com/chapter/10.1007/978-3-031-49002-6_17) (Vukosi Marivate, 2023) | ['tsn'] | Classification | s2s | [News, Written] | None | None | -| [TurHistQuadRetrieval](https://github.com/okanvk/Turkish-Reading-Comprehension-Question-Answering-Dataset) (Soygazi et al., 2021) | ['tur'] | Retrieval | p2p | [Encyclopaedic, Non-fiction, Academic, Written] | None | None | +| [TurHistQuadRetrieval](https://github.com/okanvk/Turkish-Reading-Comprehension-Question-Answering-Dataset) (Soygazi et al., 2021) | ['tur'] | Retrieval | p2p | [Academic, Encyclopaedic, Non-fiction, Written] | None | None | | [TurkicClassification](https://huggingface.co/datasets/Electrotubbie/classification_Turkic_languages/) | ['bak', 'kaz', 'kir'] | Classification | s2s | [News, Written] | None | None | | [TurkishMovieSentimentClassification](https://www.win.tue.nl/~mpechen/publications/pubs/MT_WISDOM2013.pdf) (Erkin Demirtas, 2013) | ['tur'] | Classification | s2s | [Reviews, Written] | None | None | | [TurkishProductSentimentClassification](https://www.win.tue.nl/~mpechen/publications/pubs/MT_WISDOM2013.pdf) (Erkin Demirtas, 2013) | ['tur'] | Classification | s2s | [Reviews, Written] | None | None | @@ -1044,64 +778,83 @@ The following tables give you an overview of the tasks in MTEB. | [TweetSarcasmClassification](https://aclanthology.org/2020.osact-1.5/) | ['ara'] | Classification | s2s | [Social, Written] | None | None | | [TweetSentimentClassification](https://aclanthology.org/2022.lrec-1.27) | ['ara', 'deu', 'eng', 'fra', 'hin', 'ita', 'por', 'spa'] | Classification | s2s | [Social, Written] | None | None | | [TweetSentimentExtractionClassification](https://www.kaggle.com/competitions/tweet-sentiment-extraction/overview) (Maggie et al., 2020) | ['eng'] | Classification | s2s | [Social, Written] | None | None | -| [TweetTopicSingleClassification](https://arxiv.org/abs/2209.09824) | ['eng'] | Classification | s2s | [Social, News, Written] | None | None | +| [TweetTopicSingleClassification](https://arxiv.org/abs/2209.09824) | ['eng'] | Classification | s2s | [News, Social, Written] | None | None | | [TwentyNewsgroupsClustering.v2](https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html) (Ken Lang, 1995) | ['eng'] | Clustering | s2s | [News, Written] | {'test': 59545} | {'test': {'num_samples': 59545, 'number_of_characters': 1907719, 'min_text_length': 11, 'average_text_length': 32.04, 'max_text_length': 120, 'min_labels_per_text': 2082, 'average_labels_per_text': 1.0, 'max_labels_per_text': 3236, 'unique_labels': 20, 'labels': {'12': {'count': 3137}, '6': {'count': 3070}, '0': {'count': 2613}, '2': {'count': 3155}, '10': {'count': 3220}, '17': {'count': 2986}, '14': {'count': 3106}, '13': {'count': 3055}, '1': {'count': 3056}, '16': {'count': 2911}, '9': {'count': 2984}, '3': {'count': 3070}, '15': {'count': 3090}, '7': {'count': 3036}, '5': {'count': 3124}, '11': {'count': 3236}, '18': {'count': 2483}, '8': {'count': 3090}, '19': {'count': 2082}, '4': {'count': 3041}}}} | | [TwitterHjerneRetrieval](https://huggingface.co/datasets/sorenmulli/da-hashtag-twitterhjerne) (Holm et al., 2024) | ['dan'] | Retrieval | p2p | [Social, Written] | None | None | -| [TwitterSemEval2015](https://alt.qcri.org/semeval2015/task1/) | ['eng'] | PairClassification | s2s | | None | None | -| [TwitterURLCorpus](https://languagenet.github.io/) | ['eng'] | PairClassification | s2s | | {'test': 51534} | {'test': {'num_samples': 51534, 'number_of_characters': 8659940, 'min_sentence1_length': 24, 'avg_sentence1_length': 79.49, 'max_sentence1_length': 126, 'unique_sentence1': 4329, 'min_sentence2_length': 6, 'avg_sentence2_length': 88.55, 'max_sentence2_length': 608, 'unique_sentence2': 41304, 'unique_labels': 2, 'labels': {'0': {'count': 38546}, '1': {'count': 12988}}}} | +| [TwitterSemEval2015](https://alt.qcri.org/semeval2015/task1/) | ['eng'] | PairClassification | s2s | [Social, Written] | None | None | +| [TwitterURLCorpus](https://languagenet.github.io/) | ['eng'] | PairClassification | s2s | [Social, Written] | {'test': 51534} | {'test': {'num_samples': 51534, 'number_of_characters': 8659940, 'min_sentence1_length': 24, 'avg_sentence1_length': 79.49, 'max_sentence1_length': 126, 'unique_sentence1': 4329, 'min_sentence2_length': 6, 'avg_sentence2_length': 88.55, 'max_sentence2_length': 608, 'unique_sentence2': 41304, 'unique_labels': 2, 'labels': {'0': {'count': 38546}, '1': {'count': 12988}}}} | | [UCCVCommonLawLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | +| [UCF101](https://huggingface.co/datasets/flwrlabs/ucf101) (Khurram Soomro, 2012) | ['eng'] | ImageClassification | i2i | [Scene] | None | None | +| [UCF101ZeroShot](https://huggingface.co/datasets/flwrlabs/ucf101) (Khurram Soomro, 2012) | ['eng'] | ZeroShotClassification | i2t | [Scene] | None | None | | [UkrFormalityClassification](https://huggingface.co/datasets/ukr-detect/ukr-formality-dataset-translated-gyafc) | ['ukr'] | Classification | s2s | [News, Written] | None | None | | [UnfairTOSLegalBenchClassification](https://huggingface.co/datasets/nguha/legalbench) (Neel Guha, 2023) | ['eng'] | Classification | s2s | [Legal, Written] | None | None | | [UrduRomanSentimentClassification](https://archive.ics.uci.edu/dataset/458/roman+urdu+data+set) (Sharf,Zareen, 2018) | ['urd'] | Classification | s2s | [Social, Written] | None | None | | [VGHierarchicalClusteringP2P](https://huggingface.co/datasets/navjordj/VG_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | p2p | [News, Non-fiction, Written] | None | None | | [VGHierarchicalClusteringS2S](https://huggingface.co/datasets/navjordj/VG_summarization) (Navjord et al., 2023) | ['nob'] | Clustering | p2p | [News, Non-fiction, Written] | None | None | +| [VOC2007](http://host.robots.ox.ac.uk/pascal/VOC/) | ['eng'] | ImageMultilabelClassification | i2i | [Encyclopaedic] | None | None | +| [VQA2IT2TRetrieval](https://openaccess.thecvf.com/content_cvpr_2017/html/Goyal_Making_the_v_CVPR_2017_paper.html) (Goyal et al., 2017) | ['eng'] | Any2AnyRetrieval | it2t | [Web] | None | None | | [VideoRetrieval](https://arxiv.org/abs/2203.03367) | ['cmn'] | Retrieval | s2p | | None | None | +| [VidoreArxivQARetrieval](https://arxiv.org/pdf/2407.01449) (Faysse et al., 2024) | ['eng'] | Any2AnyRetrieval | t2i | [Academic] | None | None | +| [VidoreDocVQARetrieval](https://arxiv.org/pdf/2407.01449) (Faysse et al., 2024) | ['eng'] | Any2AnyRetrieval | t2i | [Academic] | None | None | +| [VidoreInfoVQARetrieval](https://arxiv.org/pdf/2407.01449) (Faysse et al., 2024) | ['eng'] | Any2AnyRetrieval | t2i | [Academic] | None | None | +| [VidoreShiftProjectRetrieval](https://arxiv.org/pdf/2407.01449) (Faysse et al., 2024) | ['eng'] | Any2AnyRetrieval | t2i | [Academic] | None | None | +| [VidoreSyntheticDocQAAIRetrieval](https://arxiv.org/pdf/2407.01449) (Faysse et al., 2024) | ['eng'] | Any2AnyRetrieval | t2i | [Academic] | None | None | +| [VidoreSyntheticDocQAEnergyRetrieval](https://arxiv.org/pdf/2407.01449) (Faysse et al., 2024) | ['eng'] | Any2AnyRetrieval | t2i | [Academic] | None | None | +| [VidoreSyntheticDocQAGovernmentReportsRetrieval](https://arxiv.org/pdf/2407.01449) (Faysse et al., 2024) | ['eng'] | Any2AnyRetrieval | t2i | [Academic] | None | None | +| [VidoreSyntheticDocQAHealthcareIndustryRetrieval](https://arxiv.org/pdf/2407.01449) (Faysse et al., 2024) | ['eng'] | Any2AnyRetrieval | t2i | [Academic] | None | None | +| [VidoreTabfquadRetrieval](https://arxiv.org/pdf/2407.01449) (Faysse et al., 2024) | ['eng'] | Any2AnyRetrieval | t2i | [Academic] | None | None | +| [VidoreTatdqaRetrieval](https://arxiv.org/pdf/2407.01449) (Faysse et al., 2024) | ['eng'] | Any2AnyRetrieval | t2i | [Academic] | None | None | | [VieMedEVBitextMining](https://aclanthology.org/2015.iwslt-evaluation.11/) (Nhu Vo, 2024) | ['eng', 'vie'] | BitextMining | s2s | [Medical, Written] | {'test': 2048} | {'test': {'num_samples': 2048, 'number_of_characters': 575910, 'unique_pairs': 2048, 'min_sentence1_length': 11, 'average_sentence1_length': 139.23, 'max_sentence1_length': 1291, 'unique_sentence1': 2048, 'min_sentence2_length': 11, 'average_sentence2_length': 141.98, 'max_sentence2_length': 1217, 'unique_sentence2': 2047}} | | [VieQuADRetrieval](https://aclanthology.org/2020.coling-main.233.pdf) | ['vie'] | Retrieval | s2p | [Encyclopaedic, Non-fiction, Written] | None | None | | [VieStudentFeedbackClassification](https://ieeexplore.ieee.org/document/8573337) (Nguyen et al., 2018) | ['vie'] | Classification | s2s | [Reviews, Written] | None | None | +| [VisualNewsI2TRetrieval](https://aclanthology.org/2021.emnlp-main.542/) (Liu et al., 2021) | ['eng'] | Any2AnyRetrieval | i2t | [Encyclopaedic] | None | None | +| [VisualNewsT2IRetrieval](https://aclanthology.org/2021.emnlp-main.542/) (Liu et al., 2021) | ['eng'] | Any2AnyRetrieval | t2i | [Encyclopaedic] | None | None | +| [VizWizIT2TRetrieval](https://openaccess.thecvf.com/content_cvpr_2018/papers/Gurari_VizWiz_Grand_Challenge_CVPR_2018_paper.pdf) (Gurari et al., 2018) | ['eng'] | Any2AnyRetrieval | it2t | [Web] | None | None | | [VoyageMMarcoReranking](https://arxiv.org/abs/2312.16144) (Benjamin Clavié, 2023) | ['jpn'] | Reranking | s2s | [Academic, Non-fiction, Written] | None | None | +| [WITT2IRetrieval](https://proceedings.mlr.press/v162/bugliarello22a/bugliarello22a.pdf) (Bugliarello et al., 2022) | ['ara', 'bul', 'dan', 'ell', 'eng', 'est', 'ind', 'jpn', 'kor', 'tur', 'vie'] | Any2AnyRetrieval | t2i | [Encyclopaedic, Written] | None | None | | [WRIMEClassification](https://aclanthology.org/2021.naacl-main.169/) | ['jpn'] | Classification | s2s | [Social, Written] | None | None | ->>>>>>> main | [Waimai](https://aclanthology.org/2023.nodalida-1.20/) (Xiao et al., 2023) | ['cmn'] | Classification | s2s | | None | None | | [WebLINXCandidatesReranking](https://mcgill-nlp.github.io/weblinx) (Xing Han Lù, 2024) | ['eng'] | Reranking | p2p | [Academic, Web, Written] | None | None | +| [WebQAT2ITRetrieval](https://openaccess.thecvf.com/content/CVPR2022/html/Chang_WebQA_Multihop_and_Multimodal_QA_CVPR_2022_paper.html) (Chang et al., 2022) | ['eng'] | Any2AnyRetrieval | t2it | [Encyclopaedic] | None | None | +| [WebQAT2TRetrieval](https://openaccess.thecvf.com/content/CVPR2022/html/Chang_WebQA_Multihop_and_Multimodal_QA_CVPR_2022_paper.html) (Chang et al., 2022) | ['eng'] | Any2AnyRetrieval | t2t | [Encyclopaedic] | None | None | | [WikiCitiesClustering](https://huggingface.co/datasets/wikipedia) | ['eng'] | Clustering | p2p | [Encyclopaedic, Written] | None | None | -<<<<<<< HEAD -| [WikiClusteringP2P.v2](https://github.com/Rysias/wiki-clustering) | ['bos', 'cat', 'ces', 'dan', 'eus', 'glv', 'ilo', 'kur', 'lav', 'min', 'mlt', 'sco', 'sqi', 'wln'] | Clustering | p2p | [Encyclopaedic, Written] | {'test': 2048} | {'test': {'num_samples': 28672, 'average_text_length': 629.7426409040179, 'average_labels_per_text': 1.0, 'unique_labels': 39, 'labels': {'16': {'count': 541}, '3': {'count': 1607}, '12': {'count': 846}, '0': {'count': 2410}, '15': {'count': 878}, '11': {'count': 864}, '6': {'count': 787}, '9': {'count': 654}, '14': {'count': 966}, '8': {'count': 1389}, '2': {'count': 2428}, '10': {'count': 839}, '1': {'count': 1370}, '4': {'count': 2942}, '7': {'count': 2514}, '5': {'count': 1490}, '13': {'count': 918}, '19': {'count': 315}, '17': {'count': 711}, '20': {'count': 345}, '18': {'count': 800}, '24': {'count': 467}, '25': {'count': 928}, '21': {'count': 62}, '26': {'count': 270}, '22': {'count': 186}, '23': {'count': 36}, '27': {'count': 465}, '28': {'count': 62}, '36': {'count': 139}, '32': {'count': 57}, '38': {'count': 43}, '30': {'count': 52}, '34': {'count': 80}, '33': {'count': 75}, '35': {'count': 62}, '31': {'count': 63}, '37': {'count': 8}, '29': {'count': 3}}, 'hf_subset_descriptive_stats': {'bs': {'num_samples': 2048, 'average_text_length': 1046.25732421875, 'average_labels_per_text': 1.0, 'unique_labels': 17, 'labels': {'16': {'count': 268}, '3': {'count': 89}, '12': {'count': 597}, '0': {'count': 202}, '15': {'count': 113}, '11': {'count': 11}, '6': {'count': 142}, '9': {'count': 181}, '14': {'count': 179}, '8': {'count': 33}, '2': {'count': 172}, '10': {'count': 12}, '1': {'count': 7}, '4': {'count': 25}, '7': {'count': 6}, '5': {'count': 9}, '13': {'count': 2}}}, 'ca': {'num_samples': 2048, 'average_text_length': 600.73291015625, 'average_labels_per_text': 1.0, 'unique_labels': 8, 'labels': {'6': {'count': 257}, '1': {'count': 737}, '2': {'count': 284}, '4': {'count': 394}, '0': {'count': 162}, '7': {'count': 151}, '5': {'count': 55}, '3': {'count': 8}}}, 'cs': {'num_samples': 2048, 'average_text_length': 659.2294921875, 'average_labels_per_text': 1.0, 'unique_labels': 21, 'labels': {'19': {'count': 35}, '5': {'count': 624}, '17': {'count': 126}, '10': {'count': 155}, '1': {'count': 231}, '7': {'count': 215}, '11': {'count': 128}, '0': {'count': 57}, '13': {'count': 75}, '2': {'count': 83}, '3': {'count': 38}, '9': {'count': 8}, '6': {'count': 14}, '12': {'count': 9}, '16': {'count': 16}, '20': {'count': 73}, '18': {'count': 38}, '4': {'count': 60}, '15': {'count': 14}, '14': {'count': 38}, '8': {'count': 11}}}, 'da': {'num_samples': 2048, 'average_text_length': 767.58935546875, 'average_labels_per_text': 1.0, 'unique_labels': 20, 'labels': {'14': {'count': 212}, '4': {'count': 74}, '15': {'count': 16}, '8': {'count': 165}, '13': {'count': 115}, '0': {'count': 79}, '1': {'count': 34}, '9': {'count': 114}, '7': {'count': 364}, '10': {'count': 32}, '17': {'count': 66}, '18': {'count': 32}, '12': {'count': 129}, '11': {'count': 159}, '2': {'count': 66}, '3': {'count': 185}, '19': {'count': 103}, '16': {'count': 33}, '5': {'count': 56}, '6': {'count': 14}}}, 'eu': {'num_samples': 2048, 'average_text_length': 405.16015625, 'average_labels_per_text': 1.0, 'unique_labels': 5, 'labels': {'4': {'count': 383}, '0': {'count': 995}, '3': {'count': 282}, '2': {'count': 344}, '1': {'count': 44}}}, 'gv': {'num_samples': 2048, 'average_text_length': 368.01123046875, 'average_labels_per_text': 1.0, 'unique_labels': 28, 'labels': {'6': {'count': 32}, '1': {'count': 83}, '24': {'count': 13}, '17': {'count': 152}, '2': {'count': 534}, '25': {'count': 76}, '5': {'count': 198}, '15': {'count': 100}, '21': {'count': 22}, '26': {'count': 188}, '13': {'count': 230}, '20': {'count': 11}, '3': {'count': 107}, '19': {'count': 88}, '16': {'count': 55}, '22': {'count': 29}, '14': {'count': 12}, '8': {'count': 61}, '0': {'count': 5}, '10': {'count': 4}, '4': {'count': 9}, '23': {'count': 6}, '7': {'count': 3}, '9': {'count': 20}, '18': {'count': 4}, '12': {'count': 3}, '27': {'count': 1}, '11': {'count': 2}}}, 'ilo': {'num_samples': 2048, 'average_text_length': 617.90771484375, 'average_labels_per_text': 1.0, 'unique_labels': 29, 'labels': {'3': {'count': 562}, '0': {'count': 373}, '18': {'count': 521}, '8': {'count': 129}, '13': {'count': 123}, '11': {'count': 54}, '25': {'count': 8}, '27': {'count': 5}, '17': {'count': 13}, '15': {'count': 4}, '4': {'count': 28}, '7': {'count': 83}, '10': {'count': 15}, '1': {'count': 11}, '24': {'count': 15}, '14': {'count': 8}, '16': {'count': 4}, '19': {'count': 9}, '23': {'count': 10}, '26': {'count': 4}, '28': {'count': 8}, '12': {'count': 29}, '21': {'count': 12}, '6': {'count': 5}, '20': {'count': 6}, '5': {'count': 4}, '22': {'count': 2}, '9': {'count': 2}, '2': {'count': 1}}}, 'ku': {'num_samples': 2048, 'average_text_length': 421.17333984375, 'average_labels_per_text': 1.0, 'unique_labels': 39, 'labels': {'14': {'count': 14}, '36': {'count': 139}, '20': {'count': 108}, '22': {'count': 27}, '15': {'count': 102}, '32': {'count': 55}, '8': {'count': 431}, '17': {'count': 210}, '38': {'count': 43}, '30': {'count': 51}, '4': {'count': 60}, '2': {'count': 111}, '6': {'count': 95}, '34': {'count': 70}, '27': {'count': 15}, '5': {'count': 174}, '26': {'count': 37}, '0': {'count': 11}, '25': {'count': 50}, '16': {'count': 2}, '12': {'count': 16}, '24': {'count': 2}, '11': {'count': 17}, '21': {'count': 9}, '13': {'count': 20}, '1': {'count': 7}, '33': {'count': 33}, '35': {'count': 28}, '10': {'count': 11}, '31': {'count': 51}, '18': {'count': 4}, '3': {'count': 4}, '28': {'count': 8}, '37': {'count': 8}, '23': {'count': 2}, '19': {'count': 7}, '7': {'count': 6}, '9': {'count': 8}, '29': {'count': 2}}}, 'lv': {'num_samples': 2048, 'average_text_length': 770.67138671875, 'average_labels_per_text': 1.0, 'unique_labels': 16, 'labels': {'15': {'count': 288}, '2': {'count': 110}, '6': {'count': 74}, '12': {'count': 50}, '0': {'count': 171}, '14': {'count': 188}, '10': {'count': 351}, '5': {'count': 142}, '4': {'count': 300}, '13': {'count': 60}, '11': {'count': 48}, '1': {'count': 165}, '8': {'count': 53}, '7': {'count': 5}, '3': {'count': 9}, '9': {'count': 34}}}, 'min': {'num_samples': 2048, 'average_text_length': 631.74072265625, 'average_labels_per_text': 1.0, 'unique_labels': 15, 'labels': {'7': {'count': 1595}, '9': {'count': 9}, '4': {'count': 48}, '3': {'count': 83}, '2': {'count': 160}, '0': {'count': 19}, '5': {'count': 74}, '6': {'count': 12}, '10': {'count': 12}, '13': {'count': 10}, '8': {'count': 5}, '11': {'count': 13}, '12': {'count': 2}, '1': {'count': 5}, '14': {'count': 1}}}, 'mt': {'num_samples': 2048, 'average_text_length': 821.22265625, 'average_labels_per_text': 1.0, 'unique_labels': 27, 'labels': {'12': {'count': 8}, '10': {'count': 147}, '14': {'count': 180}, '17': {'count': 117}, '25': {'count': 654}, '19': {'count': 35}, '0': {'count': 77}, '3': {'count': 12}, '16': {'count': 44}, '15': {'count': 108}, '24': {'count': 267}, '6': {'count': 43}, '26': {'count': 32}, '4': {'count': 79}, '22': {'count': 67}, '9': {'count': 16}, '8': {'count': 16}, '2': {'count': 55}, '5': {'count': 6}, '11': {'count': 30}, '18': {'count': 12}, '21': {'count': 12}, '20': {'count': 15}, '23': {'count': 7}, '13': {'count': 6}, '7': {'count': 1}, '1': {'count': 2}}}, 'sco': {'num_samples': 2048, 'average_text_length': 1065.21044921875, 'average_labels_per_text': 1.0, 'unique_labels': 23, 'labels': {'18': {'count': 178}, '6': {'count': 92}, '9': {'count': 28}, '15': {'count': 106}, '8': {'count': 432}, '2': {'count': 95}, '11': {'count': 104}, '1': {'count': 42}, '13': {'count': 248}, '16': {'count': 118}, '20': {'count': 130}, '3': {'count': 171}, '22': {'count': 57}, '7': {'count': 83}, '10': {'count': 74}, '5': {'count': 6}, '4': {'count': 17}, '17': {'count': 24}, '14': {'count': 14}, '0': {'count': 7}, '19': {'count': 18}, '21': {'count': 3}, '12': {'count': 1}}}, 'sq': {'num_samples': 2048, 'average_text_length': 425.486328125, 'average_labels_per_text': 1.0, 'unique_labels': 36, 'labels': {'27': {'count': 444}, '9': {'count': 234}, '14': {'count': 120}, '0': {'count': 128}, '15': {'count': 27}, '11': {'count': 298}, '24': {'count': 170}, '28': {'count': 46}, '19': {'count': 20}, '25': {'count': 140}, '3': {'count': 47}, '2': {'count': 87}, '35': {'count': 34}, '8': {'count': 53}, '31': {'count': 12}, '17': {'count': 3}, '23': {'count': 11}, '20': {'count': 2}, '33': {'count': 42}, '10': {'count': 26}, '34': {'count': 10}, '7': {'count': 2}, '13': {'count': 29}, '4': {'count': 4}, '6': {'count': 7}, '26': {'count': 9}, '5': {'count': 16}, '30': {'count': 1}, '21': {'count': 4}, '22': {'count': 4}, '18': {'count': 11}, '32': {'count': 2}, '12': {'count': 2}, '16': {'count': 1}, '1': {'count': 1}, '29': {'count': 1}}}, 'wa': {'num_samples': 2048, 'average_text_length': 216.00390625, 'average_labels_per_text': 1.0, 'unique_labels': 6, 'labels': {'5': {'count': 126}, '4': {'count': 1461}, '0': {'count': 124}, '2': {'count': 326}, '3': {'count': 10}, '1': {'count': 1}}}}}} | -| [WikipediaRerankingMultilingual](https://huggingface.co/datasets/ellamind/wikipedia-2023-11-reranking-multilingual) | ['ben', 'bul', 'ces', 'dan', 'deu', 'eng', 'fas', 'fin', 'hin', 'ita', 'nld', 'nor', 'por', 'ron', 'srp', 'swe'] | Reranking | s2p | [Encyclopaedic, Written] | {'en': 1500, 'de': 1500, 'it': 1500, 'pt': 1500, 'nl': 1500, 'cs': 1500, 'ro': 1500, 'bg': 1500, 'sr': 1500, 'fi': 1500, 'da': 1500, 'fa': 1500, 'hi': 1500, 'bn': 1500, 'no': 1500, 'sv': 1500} | {'test': {'num_samples': 24000, 'num_positive': 24000, 'num_negative': 24000, 'avg_query_len': 59.091208333333334, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0, 'hf_subset_descriptive_stats': {'bg': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 60.82666666666667, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'bn': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 47.266666666666666, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'cs': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 56.272, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'da': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 56.75066666666667, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'de': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 70.004, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'en': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 68.372, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'fa': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 48.66733333333333, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'fi': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 55.343333333333334, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'hi': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 50.77733333333333, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'it': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 70.05466666666666, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'nl': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 65.34466666666667, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'pt': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 65.11933333333333, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'ro': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 61.973333333333336, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'sr': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 55.669333333333334, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'no': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 55.288, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}, 'sv': {'num_samples': 1500, 'num_positive': 1500, 'num_negative': 1500, 'avg_query_len': 57.73, 'avg_positive_len': 1.0, 'avg_negative_len': 8.0}}}} | -| [WikipediaRetrievalMultilingual](https://huggingface.co/datasets/ellamind/wikipedia-2023-11-retrieval-multilingual-queries) | ['ben', 'bul', 'ces', 'dan', 'deu', 'eng', 'fas', 'fin', 'hin', 'ita', 'nld', 'nor', 'por', 'ron', 'srp', 'swe'] | Retrieval | s2p | [Encyclopaedic, Written] | {'en': 1500, 'de': 1500, 'it': 1500, 'pt': 1500, 'nl': 1500, 'cs': 1500, 'ro': 1500, 'bg': 1500, 'sr': 1500, 'fi': 1500, 'da': 1500, 'fa': 1500, 'hi': 1500, 'bn': 1500, 'no': 1500, 'sv': 1500} | {'test': {'bg': {'average_document_length': 374.376, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'bn': {'average_document_length': 394.05044444444445, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'cs': {'average_document_length': 369.9831111111111, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'da': {'average_document_length': 345.2597037037037, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'de': {'average_document_length': 398.4137777777778, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'en': {'average_document_length': 452.9871111111111, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'fa': {'average_document_length': 345.1568888888889, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'fi': {'average_document_length': 379.71237037037037, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'hi': {'average_document_length': 410.72540740740743, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'it': {'average_document_length': 393.73437037037036, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'nl': {'average_document_length': 375.6695555555556, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'pt': {'average_document_length': 398.27237037037037, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'ro': {'average_document_length': 348.3817037037037, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'sr': {'average_document_length': 384.3131851851852, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'no': {'average_document_length': 366.93733333333336, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}, 'sv': {'average_document_length': 369.340962962963, 'average_query_length': 1.0, 'num_documents': 13500, 'num_queries': 1500, 'average_relevant_docs_per_query': 1.0}}} | -| [WinoGrande](https://winogrande.allenai.org/) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | {'test': 0} | {'test': {'average_document_length': 7.68243375858685, 'average_query_length': 111.78216258879242, 'num_documents': 5095, 'num_queries': 1267, 'average_relevant_docs_per_query': 1.0}} | -| [WisesightSentimentClassification](https://github.com/PyThaiNLP/wisesight-sentiment) | ['tha'] | Classification | s2s | [Social, News, Written] | {'train': 2048} | {'train': 103.42} | -| XMarket (Bonab et al., 2021) | ['deu', 'eng', 'spa'] | Retrieval | s2p | | None | {'test': {'de': {'average_document_length': 187.4061197288943, 'average_query_length': 15.717612088184294, 'num_documents': 70526, 'num_queries': 4037, 'average_relevant_docs_per_query': 54.3522417636859}, 'en': {'average_document_length': 452.792089662076, 'average_query_length': 15.881635344543357, 'num_documents': 218777, 'num_queries': 9099, 'average_relevant_docs_per_query': 85.43719090009891}, 'es': {'average_document_length': 279.67909262759923, 'average_query_length': 19.97062937062937, 'num_documents': 39675, 'num_queries': 3575, 'average_relevant_docs_per_query': 36.01006993006993}}} | -| [XNLI](https://aclanthology.org/D18-1269/) (Conneau et al., 2018) | ['ara', 'bul', 'deu', 'ell', 'eng', 'fra', 'hin', 'rus', 'spa', 'swa', 'tha', 'tur', 'vie', 'zho'] | PairClassification | s2s | [Non-fiction, Fiction, Government, Written] | {'validation': 2163, 'test': 2460} | {'test': {'num_samples': 19110, 'avg_sentence1_len': 103.23793825222397, 'avg_sentence2_len': 48.88895866038723, 'unique_labels': 2, 'labels': {'0': {'count': 9562}, '1': {'count': 9548}}, 'hf_subset_descriptive_stats': {'ar': {'num_samples': 1365, 'avg_sentence1_len': 89.57362637362637, 'avg_sentence2_len': 41.99487179487179, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'bg': {'num_samples': 1365, 'avg_sentence1_len': 110.01611721611722, 'avg_sentence2_len': 51.62930402930403, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'de': {'num_samples': 1365, 'avg_sentence1_len': 119.92600732600732, 'avg_sentence2_len': 56.794871794871796, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'el': {'num_samples': 1365, 'avg_sentence1_len': 119.05421245421246, 'avg_sentence2_len': 56.93260073260073, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'en': {'num_samples': 1365, 'avg_sentence1_len': 105.67032967032966, 'avg_sentence2_len': 49.8043956043956, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'es': {'num_samples': 1365, 'avg_sentence1_len': 115.43296703296703, 'avg_sentence2_len': 54.68205128205128, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'fr': {'num_samples': 1365, 'avg_sentence1_len': 121.0967032967033, 'avg_sentence2_len': 58.58021978021978, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'hi': {'num_samples': 1365, 'avg_sentence1_len': 104.63443223443224, 'avg_sentence2_len': 50.17289377289377, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'ru': {'num_samples': 1365, 'avg_sentence1_len': 110.76923076923077, 'avg_sentence2_len': 52.452014652014654, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'sw': {'num_samples': 1365, 'avg_sentence1_len': 104.43956043956044, 'avg_sentence2_len': 49.48205128205128, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'th': {'num_samples': 1365, 'avg_sentence1_len': 96.6923076923077, 'avg_sentence2_len': 44.544322344322346, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'tr': {'num_samples': 1365, 'avg_sentence1_len': 103.67765567765568, 'avg_sentence2_len': 49.18534798534799, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'vi': {'num_samples': 1365, 'avg_sentence1_len': 111.31208791208792, 'avg_sentence2_len': 52.46007326007326, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'zh': {'num_samples': 1365, 'avg_sentence1_len': 33.03589743589744, 'avg_sentence2_len': 15.73040293040293, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}}}, 'validation': {'num_samples': 19110, 'avg_sentence1_len': 103.20790162218734, 'avg_sentence2_len': 49.01909994767138, 'unique_labels': 2, 'labels': {'0': {'count': 9562}, '1': {'count': 9548}}, 'hf_subset_descriptive_stats': {'ar': {'num_samples': 1365, 'avg_sentence1_len': 88.31868131868131, 'avg_sentence2_len': 41.61172161172161, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'bg': {'num_samples': 1365, 'avg_sentence1_len': 109.196336996337, 'avg_sentence2_len': 51.967032967032964, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'de': {'num_samples': 1365, 'avg_sentence1_len': 119.81172161172161, 'avg_sentence2_len': 57.36923076923077, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'el': {'num_samples': 1365, 'avg_sentence1_len': 119.87545787545787, 'avg_sentence2_len': 56.88278388278388, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'en': {'num_samples': 1365, 'avg_sentence1_len': 105.71648351648352, 'avg_sentence2_len': 49.87619047619047, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'es': {'num_samples': 1365, 'avg_sentence1_len': 115.17289377289377, 'avg_sentence2_len': 55.120879120879124, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'fr': {'num_samples': 1365, 'avg_sentence1_len': 121.75897435897436, 'avg_sentence2_len': 59.08864468864469, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'hi': {'num_samples': 1365, 'avg_sentence1_len': 105.06446886446886, 'avg_sentence2_len': 50.44395604395604, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'ru': {'num_samples': 1365, 'avg_sentence1_len': 109.74725274725274, 'avg_sentence2_len': 52.26886446886447, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'sw': {'num_samples': 1365, 'avg_sentence1_len': 104.32234432234432, 'avg_sentence2_len': 49.87692307692308, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'th': {'num_samples': 1365, 'avg_sentence1_len': 97.28498168498169, 'avg_sentence2_len': 43.843223443223444, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'tr': {'num_samples': 1365, 'avg_sentence1_len': 102.96630036630036, 'avg_sentence2_len': 49.63809523809524, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'vi': {'num_samples': 1365, 'avg_sentence1_len': 112.26373626373626, 'avg_sentence2_len': 52.432967032967035, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'zh': {'num_samples': 1365, 'avg_sentence1_len': 33.41098901098901, 'avg_sentence2_len': 15.846886446886447, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}}}} | -| [XNLIV2](https://arxiv.org/pdf/2301.06527) (Upadhyay et al., 2023) | ['asm', 'ben', 'bho', 'ell', 'guj', 'kan', 'mar', 'ory', 'pan', 'rus', 'san', 'tam', 'tur'] | PairClassification | s2s | [Non-fiction, Fiction, Government, Written] | {'test': 5010} | {'test': 80.06} | -| [XPQARetrieval](https://arxiv.org/abs/2305.09249) (Shen et al., 2023) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'jpn', 'kor', 'pol', 'por', 'spa', 'tam'] | Retrieval | s2p | [Reviews, Written] | {'test': 19801} | {'test': {'ara-ara': {'average_document_length': 61.88361204013378, 'average_query_length': 29.688, 'num_documents': 1495, 'num_queries': 750, 'average_relevant_docs_per_query': 2.004}, 'eng-ara': {'average_document_length': 125.26940639269407, 'average_query_length': 29.688, 'num_documents': 1533, 'num_queries': 750, 'average_relevant_docs_per_query': 2.058666666666667}, 'ara-eng': {'average_document_length': 61.88361204013378, 'average_query_length': 39.5188679245283, 'num_documents': 1495, 'num_queries': 742, 'average_relevant_docs_per_query': 2.024258760107817}, 'deu-deu': {'average_document_length': 69.54807692307692, 'average_query_length': 55.51827676240209, 'num_documents': 1248, 'num_queries': 766, 'average_relevant_docs_per_query': 1.6318537859007833}, 'eng-deu': {'average_document_length': 115.77118078719145, 'average_query_length': 55.51827676240209, 'num_documents': 1499, 'num_queries': 766, 'average_relevant_docs_per_query': 1.9634464751958225}, 'deu-eng': {'average_document_length': 69.54807692307692, 'average_query_length': 51.88903394255875, 'num_documents': 1248, 'num_queries': 766, 'average_relevant_docs_per_query': 1.6318537859007833}, 'spa-spa': {'average_document_length': 68.27511591962906, 'average_query_length': 46.711223203026485, 'num_documents': 1941, 'num_queries': 793, 'average_relevant_docs_per_query': 2.4489281210592684}, 'eng-spa': {'average_document_length': 123.43698347107438, 'average_query_length': 46.711223203026485, 'num_documents': 1936, 'num_queries': 793, 'average_relevant_docs_per_query': 2.472887767969735}, 'spa-eng': {'average_document_length': 68.27511591962906, 'average_query_length': 47.21059268600252, 'num_documents': 1941, 'num_queries': 793, 'average_relevant_docs_per_query': 2.4489281210592684}, 'fra-fra': {'average_document_length': 76.99354005167959, 'average_query_length': 56.0520694259012, 'num_documents': 1548, 'num_queries': 749, 'average_relevant_docs_per_query': 2.069425901201602}, 'eng-fra': {'average_document_length': 137.31242532855435, 'average_query_length': 56.0520694259012, 'num_documents': 1674, 'num_queries': 749, 'average_relevant_docs_per_query': 2.248331108144192}, 'fra-eng': {'average_document_length': 76.99354005167959, 'average_query_length': 49.58744993324433, 'num_documents': 1548, 'num_queries': 749, 'average_relevant_docs_per_query': 2.069425901201602}, 'hin-hin': {'average_document_length': 47.20783373301359, 'average_query_length': 33.47783783783784, 'num_documents': 1251, 'num_queries': 925, 'average_relevant_docs_per_query': 1.3902702702702703}, 'eng-hin': {'average_document_length': 106.67662682602922, 'average_query_length': 33.47783783783784, 'num_documents': 1506, 'num_queries': 925, 'average_relevant_docs_per_query': 1.8054054054054054}, 'hin-eng': {'average_document_length': 47.20783373301359, 'average_query_length': 34.98574561403509, 'num_documents': 1251, 'num_queries': 912, 'average_relevant_docs_per_query': 1.4100877192982457}, 'ita-ita': {'average_document_length': 59.778301886792455, 'average_query_length': 49.14932126696833, 'num_documents': 1272, 'num_queries': 663, 'average_relevant_docs_per_query': 1.9245852187028658}, 'eng-ita': {'average_document_length': 123.07302075326672, 'average_query_length': 49.14932126696833, 'num_documents': 1301, 'num_queries': 663, 'average_relevant_docs_per_query': 1.9849170437405732}, 'ita-eng': {'average_document_length': 59.778301886792455, 'average_query_length': 49.040723981900456, 'num_documents': 1272, 'num_queries': 663, 'average_relevant_docs_per_query': 1.9245852187028658}, 'jpn-jpn': {'average_document_length': 41.030605871330415, 'average_query_length': 23.296969696969697, 'num_documents': 1601, 'num_queries': 825, 'average_relevant_docs_per_query': 1.9406060606060607}, 'eng-jpn': {'average_document_length': 126.2647564469914, 'average_query_length': 23.296969696969697, 'num_documents': 1745, 'num_queries': 825, 'average_relevant_docs_per_query': 2.1187878787878787}, 'jpn-eng': {'average_document_length': 41.030605871330415, 'average_query_length': 51.416058394160586, 'num_documents': 1601, 'num_queries': 822, 'average_relevant_docs_per_query': 1.9476885644768855}, 'kor-kor': {'average_document_length': 31.22722159730034, 'average_query_length': 21.81804281345566, 'num_documents': 889, 'num_queries': 654, 'average_relevant_docs_per_query': 1.5642201834862386}, 'eng-kor': {'average_document_length': 112.41231822070145, 'average_query_length': 21.81804281345566, 'num_documents': 1169, 'num_queries': 654, 'average_relevant_docs_per_query': 1.952599388379205}, 'kor-eng': {'average_document_length': 31.22722159730034, 'average_query_length': 43.9527687296417, 'num_documents': 889, 'num_queries': 614, 'average_relevant_docs_per_query': 1.6661237785016287}, 'pol-pol': {'average_document_length': 50.66814439518683, 'average_query_length': 53.72101910828025, 'num_documents': 1579, 'num_queries': 785, 'average_relevant_docs_per_query': 2.080254777070064}, 'eng-pol': {'average_document_length': 112.96919566457501, 'average_query_length': 53.72101910828025, 'num_documents': 1753, 'num_queries': 785, 'average_relevant_docs_per_query': 2.385987261146497}, 'pol-eng': {'average_document_length': 50.66814439518683, 'average_query_length': 54.1994851994852, 'num_documents': 1579, 'num_queries': 777, 'average_relevant_docs_per_query': 2.101673101673102}, 'por-por': {'average_document_length': 75.9845869297164, 'average_query_length': 42.58875, 'num_documents': 1622, 'num_queries': 800, 'average_relevant_docs_per_query': 2.14}, 'eng-por': {'average_document_length': 111.42525930445393, 'average_query_length': 42.58875, 'num_documents': 1639, 'num_queries': 800, 'average_relevant_docs_per_query': 2.21875}, 'por-eng': {'average_document_length': 75.9845869297164, 'average_query_length': 46.57967377666248, 'num_documents': 1622, 'num_queries': 797, 'average_relevant_docs_per_query': 2.148055207026349}, 'tam-tam': {'average_document_length': 64.89019607843137, 'average_query_length': 33.267263427109974, 'num_documents': 1275, 'num_queries': 782, 'average_relevant_docs_per_query': 1.6994884910485935}, 'eng-tam': {'average_document_length': 96.96361185983828, 'average_query_length': 33.267263427109974, 'num_documents': 1484, 'num_queries': 782, 'average_relevant_docs_per_query': 2.0255754475703327}, 'tam-eng': {'average_document_length': 64.89019607843137, 'average_query_length': 34.777633289986994, 'num_documents': 1275, 'num_queries': 769, 'average_relevant_docs_per_query': 1.728218465539662}, 'cmn-cmn': {'average_document_length': 20.958944281524925, 'average_query_length': 12.21116504854369, 'num_documents': 1705, 'num_queries': 824, 'average_relevant_docs_per_query': 2.0716019417475726}, 'eng-cmn': {'average_document_length': 108.31593874078276, 'average_query_length': 12.21116504854369, 'num_documents': 1763, 'num_queries': 824, 'average_relevant_docs_per_query': 2.2633495145631066}, 'cmn-eng': {'average_document_length': 20.958944281524925, 'average_query_length': 41.24390243902439, 'num_documents': 1705, 'num_queries': 820, 'average_relevant_docs_per_query': 2.0817073170731706}}} | -| [XQuADRetrieval](https://huggingface.co/datasets/xquad) (Mikel Artetxe, 2019) | ['arb', 'deu', 'ell', 'eng', 'hin', 'ron', 'rus', 'spa', 'tha', 'tur', 'vie', 'zho'] | Retrieval | s2p | [Web, Written] | {'test': 1190} | {'validation': {'ar': {'average_document_length': 683.4666666666667, 'average_query_length': 53.327993254637434, 'num_documents': 240, 'num_queries': 1186, 'average_relevant_docs_per_query': 1.0}, 'de': {'average_document_length': 894.0666666666667, 'average_query_length': 69.04318374259103, 'num_documents': 240, 'num_queries': 1181, 'average_relevant_docs_per_query': 1.0}, 'el': {'average_document_length': 894.3791666666667, 'average_query_length': 68.61317567567568, 'num_documents': 240, 'num_queries': 1184, 'average_relevant_docs_per_query': 1.0}, 'en': {'average_document_length': 784.8333333333334, 'average_query_length': 61.25063291139241, 'num_documents': 240, 'num_queries': 1185, 'average_relevant_docs_per_query': 1.0}, 'es': {'average_document_length': 883.8041666666667, 'average_query_length': 68.23817567567568, 'num_documents': 240, 'num_queries': 1184, 'average_relevant_docs_per_query': 1.0}, 'hi': {'average_document_length': 764.9416666666667, 'average_query_length': 59.684699915469146, 'num_documents': 240, 'num_queries': 1183, 'average_relevant_docs_per_query': 1.0}, 'ro': {'average_document_length': 878.4458333333333, 'average_query_length': 67.17229729729729, 'num_documents': 240, 'num_queries': 1184, 'average_relevant_docs_per_query': 1.0}, 'ru': {'average_document_length': 850.1875, 'average_query_length': 64.94261603375527, 'num_documents': 240, 'num_queries': 1185, 'average_relevant_docs_per_query': 1.0}, 'th': {'average_document_length': 736.7583333333333, 'average_query_length': 55.103389830508476, 'num_documents': 240, 'num_queries': 1180, 'average_relevant_docs_per_query': 1.0}, 'tr': {'average_document_length': 788.3, 'average_query_length': 60.876689189189186, 'num_documents': 240, 'num_queries': 1184, 'average_relevant_docs_per_query': 1.0}, 'vi': {'average_document_length': 803.9083333333333, 'average_query_length': 61.62859560067682, 'num_documents': 240, 'num_queries': 1182, 'average_relevant_docs_per_query': 1.0}, 'zh': {'average_document_length': 252.4, 'average_query_length': 18.460626587637595, 'num_documents': 240, 'num_queries': 1181, 'average_relevant_docs_per_query': 1.0}}} | -| [XStance](https://github.com/ZurichNLP/xstance) | ['deu', 'fra', 'ita'] | PairClassification | s2s | [Social, Written] | {'test': 2048} | {'test': 152.41} | -| [YahooAnswersTopicsClassification](https://huggingface.co/datasets/yahoo_answers_topics) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [Web, Written] | {'test': 60000} | {'test': 346.35} | -| [YelpReviewFullClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [Reviews, Written] | {'test': 50000} | {} | -| [YueOpenriceReviewClassification](https://github.com/Christainx/Dataset_Cantonese_Openrice) (Xiang et al., 2019) | ['yue'] | Classification | s2s | [Reviews, Spoken] | {'test': 6161} | {'test': 173.0} | -| [indonli](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) | ['ind'] | PairClassification | s2s | [Encyclopaedic, Web, News, Written] | {'test_expert': 2040} | {'test_expert': 145.88} | -| [mFollowIRCrossLingualInstructionRetrieval](https://neuclir.github.io/) (Weller et al., 2024) | ['eng', 'fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'eng-fas': 80, 'eng-rus': 80, 'eng-zho': 86} | {'test': {'num_docs': 121635, 'num_queries': 123, 'average_document_length': 2331.0777818884367, 'average_query_length': 81.8780487804878, 'average_instruction_length': 389.9512195121951, 'average_changed_instruction_length': 450.5528455284553, 'average_relevant_docs_per_query': 10.30952380952381, 'average_top_ranked_per_query': 1024.3902439024391, 'hf_subset_descriptive_stats': {'eng-fas': {'num_docs': 41189, 'num_queries': 40, 'average_document_length': 3145.4990895627475, 'average_query_length': 80.075, 'average_instruction_length': 396.875, 'average_changed_instruction_length': 463.175, 'average_relevant_docs_per_query': 10.465116279069768, 'average_top_ranked_per_query': 1075}, 'eng-rus': {'num_docs': 39326, 'num_queries': 40, 'average_document_length': 2784.0813456746173, 'average_query_length': 81.875, 'average_instruction_length': 371.125, 'average_changed_instruction_length': 431.8, 'average_relevant_docs_per_query': 9.775, 'average_top_ranked_per_query': 1000}, 'eng-zho': {'num_docs': 41120, 'num_queries': 43, 'average_document_length': 1082.0501215953307, 'average_query_length': 83.55813953488372, 'average_instruction_length': 401.0232558139535, 'average_changed_instruction_length': 456.25581395348837, 'average_relevant_docs_per_query': 10.651162790697674, 'average_top_ranked_per_query': 1000}}}} | -| [mFollowIRInstructionRetrieval](https://neuclir.github.io/) (Weller et al., 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'fas': 80, 'rus': 80, 'zho': 86} | {'test': {'num_docs': 121635, 'num_queries': 123, 'average_document_length': 2331.0777818884367, 'average_query_length': 57.113821138211385, 'average_instruction_length': 281.0650406504065, 'average_changed_instruction_length': 326.9430894308943, 'average_relevant_docs_per_query': 10.30952380952381, 'average_top_ranked_per_query': 1024.3902439024391, 'hf_subset_descriptive_stats': {'fas': {'num_docs': 41189, 'num_queries': 40, 'average_document_length': 3145.4990895627475, 'average_query_length': 72.65, 'average_instruction_length': 358.925, 'average_changed_instruction_length': 415.325, 'average_relevant_docs_per_query': 10.465116279069768, 'average_top_ranked_per_query': 1075}, 'rus': {'num_docs': 39326, 'num_queries': 40, 'average_document_length': 2784.0813456746173, 'average_query_length': 77.5, 'average_instruction_length': 387, 'average_changed_instruction_length': 458, 'average_relevant_docs_per_query': 9.775, 'average_top_ranked_per_query': 1000}, 'zho': {'num_docs': 41120, 'num_queries': 43, 'average_document_length': 1082.0501215953307, 'average_query_length': 23.697674418604652, 'average_instruction_length': 110.09302325581395, 'average_changed_instruction_length': 122.81395348837209, 'average_relevant_docs_per_query': 10.651162790697674, 'average_top_ranked_per_query': 1000}}}} | -======= | [WikiClusteringP2P.v2](https://github.com/Rysias/wiki-clustering) | ['bos', 'cat', 'ces', 'dan', 'eus', 'glv', 'ilo', 'kur', 'lav', 'min', 'mlt', 'sco', 'sqi', 'wln'] | Clustering | p2p | [Encyclopaedic, Written] | None | None | +| [WikipediaBioMetChemClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaBiolumNeurochemClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaChemEngSpecialtiesClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaChemFieldsClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaChemistryTopicsClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaChemistryTopicsClustering](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Clustering | s2p | [Chemistry] | None | None | +| [WikipediaCompChemSpectroscopyClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaCryobiologySeparationClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaCrystallographyAnalyticalClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaGreenhouseEnantiopureClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaIsotopesFissionClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaLuminescenceClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaOrganicInorganicClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | | [WikipediaRerankingMultilingual](https://huggingface.co/datasets/ellamind/wikipedia-2023-11-reranking-multilingual) | ['ben', 'bul', 'ces', 'dan', 'deu', 'eng', 'fas', 'fin', 'hin', 'ita', 'nld', 'nor', 'por', 'ron', 'srp', 'swe'] | Reranking | s2p | [Encyclopaedic, Written] | {'test': 24000} | {'test': {'num_samples': 24000, 'number_of_characters': 83866932, 'num_positive': 24000, 'num_negative': 192000, 'min_query_length': 7, 'avg_query_length': 59.09, 'max_query_length': 180, 'unique_query': 23997, 'min_positive_length': 100, 'avg_positive_length': 385.45, 'max_positive_length': 3515, 'unique_positive': 23993, 'min_negative_length': 100, 'avg_negative_length': 381.24, 'max_negative_length': 9461, 'unique_negative': 191783, 'hf_subset_descriptive_stats': {'bg': {'num_samples': 1500, 'number_of_characters': 5145316, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 18, 'avg_query_length': 60.83, 'max_query_length': 166, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 375.89, 'max_positive_length': 2241, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 374.19, 'max_negative_length': 4869, 'unique_negative': 11996}, 'bn': {'num_samples': 1500, 'number_of_characters': 5390581, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 7, 'avg_query_length': 47.27, 'max_query_length': 123, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 394.59, 'max_positive_length': 2338, 'unique_positive': 1499, 'min_negative_length': 100, 'avg_negative_length': 393.98, 'max_negative_length': 5104, 'unique_negative': 11996}, 'cs': {'num_samples': 1500, 'number_of_characters': 5079180, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 17, 'avg_query_length': 56.27, 'max_query_length': 137, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 383.84, 'max_positive_length': 2300, 'unique_positive': 1499, 'min_negative_length': 100, 'avg_negative_length': 368.25, 'max_negative_length': 3487, 'unique_negative': 11982}, 'da': {'num_samples': 1500, 'number_of_characters': 4746132, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 17, 'avg_query_length': 56.75, 'max_query_length': 137, 'unique_query': 1499, 'min_positive_length': 100, 'avg_positive_length': 351.68, 'max_positive_length': 2159, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 344.46, 'max_negative_length': 2563, 'unique_negative': 11972}, 'de': {'num_samples': 1500, 'number_of_characters': 5483592, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 20, 'avg_query_length': 70.0, 'max_query_length': 180, 'unique_query': 1499, 'min_positive_length': 100, 'avg_positive_length': 391.54, 'max_positive_length': 2674, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 399.27, 'max_negative_length': 3083, 'unique_negative': 12000}, 'en': {'num_samples': 1500, 'number_of_characters': 6217884, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 18, 'avg_query_length': 68.37, 'max_query_length': 162, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 451.73, 'max_positive_length': 3515, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 453.14, 'max_negative_length': 3662, 'unique_negative': 12000}, 'fa': {'num_samples': 1500, 'number_of_characters': 4732619, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 12, 'avg_query_length': 48.67, 'max_query_length': 119, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 347.7, 'max_positive_length': 2571, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 344.84, 'max_negative_length': 4707, 'unique_negative': 11978}, 'fi': {'num_samples': 1500, 'number_of_characters': 5209132, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 14, 'avg_query_length': 55.34, 'max_query_length': 132, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 394.71, 'max_positive_length': 2129, 'unique_positive': 1498, 'min_negative_length': 100, 'avg_negative_length': 377.84, 'max_negative_length': 2574, 'unique_negative': 11972}, 'hi': {'num_samples': 1500, 'number_of_characters': 5620959, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 13, 'avg_query_length': 50.78, 'max_query_length': 125, 'unique_query': 1499, 'min_positive_length': 100, 'avg_positive_length': 420.38, 'max_positive_length': 2361, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 409.52, 'max_negative_length': 5912, 'unique_negative': 11996}, 'it': {'num_samples': 1500, 'number_of_characters': 5420496, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 23, 'avg_query_length': 70.05, 'max_query_length': 156, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 396.97, 'max_positive_length': 2082, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 393.33, 'max_negative_length': 9461, 'unique_negative': 11993}, 'nl': {'num_samples': 1500, 'number_of_characters': 5169556, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 18, 'avg_query_length': 65.34, 'max_query_length': 136, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 380.79, 'max_positive_length': 1864, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 375.03, 'max_negative_length': 3641, 'unique_negative': 11985}, 'pt': {'num_samples': 1500, 'number_of_characters': 5474356, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 18, 'avg_query_length': 65.12, 'max_query_length': 176, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 404.02, 'max_positive_length': 3057, 'unique_positive': 1499, 'min_negative_length': 100, 'avg_negative_length': 397.55, 'max_negative_length': 2877, 'unique_negative': 11991}, 'ro': {'num_samples': 1500, 'number_of_characters': 4796113, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 14, 'avg_query_length': 61.97, 'max_query_length': 169, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 346.71, 'max_positive_length': 1917, 'unique_positive': 1499, 'min_negative_length': 100, 'avg_negative_length': 348.59, 'max_negative_length': 4213, 'unique_negative': 11971}, 'sr': {'num_samples': 1500, 'number_of_characters': 5271732, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 15, 'avg_query_length': 55.67, 'max_query_length': 146, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 386.35, 'max_positive_length': 2421, 'unique_positive': 1499, 'min_negative_length': 100, 'avg_negative_length': 384.06, 'max_negative_length': 3668, 'unique_negative': 11974}, 'no': {'num_samples': 1500, 'number_of_characters': 5036586, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 14, 'avg_query_length': 55.29, 'max_query_length': 129, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 367.72, 'max_positive_length': 1450, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 366.84, 'max_negative_length': 2841, 'unique_negative': 11996}, 'sv': {'num_samples': 1500, 'number_of_characters': 5072698, 'num_positive': 1500, 'num_negative': 12000, 'min_query_length': 17, 'avg_query_length': 57.73, 'max_query_length': 133, 'unique_query': 1500, 'min_positive_length': 100, 'avg_positive_length': 372.59, 'max_positive_length': 2493, 'unique_positive': 1500, 'min_negative_length': 100, 'avg_negative_length': 368.94, 'max_negative_length': 3680, 'unique_negative': 11999}}}} | | [WikipediaRetrievalMultilingual](https://huggingface.co/datasets/ellamind/wikipedia-2023-11-retrieval-multilingual-queries) | ['ben', 'bul', 'ces', 'dan', 'deu', 'eng', 'fas', 'fin', 'hin', 'ita', 'nld', 'nor', 'por', 'ron', 'srp', 'swe'] | Retrieval | s2p | [Encyclopaedic, Written] | None | None | +| [WikipediaSaltsSemiconductorsClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaSolidStateColloidalClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | +| [WikipediaSpecialtiesInChemistryClustering](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Clustering | s2p | [Chemistry] | None | None | +| [WikipediaTheoreticalAppliedClassification](https://arxiv.org/abs/2412.00532) (Kasmaee et al., 2024) | ['eng'] | Classification | s2s | [Chemistry] | None | None | | [WinoGrande](https://winogrande.allenai.org/) (Xiao et al., 2024) | ['eng'] | Retrieval | s2s | [Encyclopaedic, Written] | None | None | -| [WisesightSentimentClassification](https://github.com/PyThaiNLP/wisesight-sentiment) | ['tha'] | Classification | s2s | [Social, News, Written] | None | None | +| [Winoground](https://openaccess.thecvf.com/content/CVPR2022/html/Thrush_Winoground_Probing_Vision_and_Language_Models_for_Visio-Linguistic_Compositionality_CVPR_2022_paper) (Tristan Thrush, 2022) | ['eng'] | ImageTextPairClassification | i2t | [Social] | None | None | +| [WisesightSentimentClassification](https://github.com/PyThaiNLP/wisesight-sentiment) | ['tha'] | Classification | s2s | [News, Social, Written] | None | None | +| [XFlickr30kCoT2IRetrieval](https://proceedings.mlr.press/v162/bugliarello22a/bugliarello22a.pdf) (Bugliarello et al., 2022) | ['deu', 'eng', 'ind', 'jpn', 'rus', 'spa', 'tur', 'zho'] | Any2AnyRetrieval | t2i | [Encyclopaedic, Written] | None | None | +| [XM3600T2IRetrieval](https://aclanthology.org/2022.emnlp-main.45/) (Thapliyal et al., 2022) | ['ara', 'ben', 'ces', 'dan', 'deu', 'ell', 'eng', 'fas', 'fil', 'fin', 'fra', 'heb', 'hin', 'hrv', 'hun', 'ind', 'ita', 'jpn', 'kor', 'mri', 'nld', 'nor', 'pol', 'por', 'quz', 'ron', 'rus', 'spa', 'swa', 'swe', 'tel', 'tha', 'tur', 'ukr', 'vie', 'zho'] | Any2AnyRetrieval | t2i | [Encyclopaedic, Written] | None | None | | XMarket (Bonab et al., 2021) | ['deu', 'eng', 'spa'] | Retrieval | s2p | | None | None | -| [XNLI](https://aclanthology.org/D18-1269/) (Conneau et al., 2018) | ['ara', 'bul', 'deu', 'ell', 'eng', 'fra', 'hin', 'rus', 'spa', 'swa', 'tha', 'tur', 'vie', 'zho'] | PairClassification | s2s | [Non-fiction, Fiction, Government, Written] | {'test': 19110, 'validation': 19110} | {'test': {'num_samples': 19110, 'number_of_characters': 2907145, 'min_sentence1_length': 3, 'avg_sentence1_length': 103.24, 'max_sentence1_length': 401, 'unique_sentence1': 15328, 'min_sentence2_length': 2, 'avg_sentence2_length': 48.89, 'max_sentence2_length': 187, 'unique_sentence2': 19104, 'unique_labels': 2, 'labels': {'0': {'count': 9562}, '1': {'count': 9548}}, 'hf_subset_descriptive_stats': {'ar': {'num_samples': 1365, 'number_of_characters': 179591, 'min_sentence1_length': 11, 'avg_sentence1_length': 89.57, 'max_sentence1_length': 242, 'unique_sentence1': 1095, 'min_sentence2_length': 8, 'avg_sentence2_length': 41.99, 'max_sentence2_length': 115, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'bg': {'num_samples': 1365, 'number_of_characters': 220646, 'min_sentence1_length': 14, 'avg_sentence1_length': 110.02, 'max_sentence1_length': 303, 'unique_sentence1': 1095, 'min_sentence2_length': 8, 'avg_sentence2_length': 51.63, 'max_sentence2_length': 150, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'de': {'num_samples': 1365, 'number_of_characters': 241224, 'min_sentence1_length': 3, 'avg_sentence1_length': 119.93, 'max_sentence1_length': 301, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 56.79, 'max_sentence2_length': 187, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'el': {'num_samples': 1365, 'number_of_characters': 240222, 'min_sentence1_length': 13, 'avg_sentence1_length': 119.05, 'max_sentence1_length': 344, 'unique_sentence1': 1095, 'min_sentence2_length': 13, 'avg_sentence2_length': 56.93, 'max_sentence2_length': 172, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'en': {'num_samples': 1365, 'number_of_characters': 212223, 'min_sentence1_length': 19, 'avg_sentence1_length': 105.67, 'max_sentence1_length': 268, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 49.8, 'max_sentence2_length': 137, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'es': {'num_samples': 1365, 'number_of_characters': 232207, 'min_sentence1_length': 11, 'avg_sentence1_length': 115.43, 'max_sentence1_length': 385, 'unique_sentence1': 1094, 'min_sentence2_length': 8, 'avg_sentence2_length': 54.68, 'max_sentence2_length': 163, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'fr': {'num_samples': 1365, 'number_of_characters': 245259, 'min_sentence1_length': 9, 'avg_sentence1_length': 121.1, 'max_sentence1_length': 327, 'unique_sentence1': 1095, 'min_sentence2_length': 10, 'avg_sentence2_length': 58.58, 'max_sentence2_length': 169, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'hi': {'num_samples': 1365, 'number_of_characters': 211312, 'min_sentence1_length': 16, 'avg_sentence1_length': 104.63, 'max_sentence1_length': 401, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 50.17, 'max_sentence2_length': 162, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'ru': {'num_samples': 1365, 'number_of_characters': 222797, 'min_sentence1_length': 11, 'avg_sentence1_length': 110.77, 'max_sentence1_length': 306, 'unique_sentence1': 1095, 'min_sentence2_length': 8, 'avg_sentence2_length': 52.45, 'max_sentence2_length': 167, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'sw': {'num_samples': 1365, 'number_of_characters': 210103, 'min_sentence1_length': 10, 'avg_sentence1_length': 104.44, 'max_sentence1_length': 266, 'unique_sentence1': 1094, 'min_sentence2_length': 2, 'avg_sentence2_length': 49.48, 'max_sentence2_length': 146, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'th': {'num_samples': 1365, 'number_of_characters': 192788, 'min_sentence1_length': 12, 'avg_sentence1_length': 96.69, 'max_sentence1_length': 262, 'unique_sentence1': 1095, 'min_sentence2_length': 6, 'avg_sentence2_length': 44.54, 'max_sentence2_length': 129, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'tr': {'num_samples': 1365, 'number_of_characters': 208658, 'min_sentence1_length': 15, 'avg_sentence1_length': 103.68, 'max_sentence1_length': 255, 'unique_sentence1': 1095, 'min_sentence2_length': 6, 'avg_sentence2_length': 49.19, 'max_sentence2_length': 140, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'vi': {'num_samples': 1365, 'number_of_characters': 223549, 'min_sentence1_length': 14, 'avg_sentence1_length': 111.31, 'max_sentence1_length': 265, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 52.46, 'max_sentence2_length': 143, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'zh': {'num_samples': 1365, 'number_of_characters': 66566, 'min_sentence1_length': 4, 'avg_sentence1_length': 33.04, 'max_sentence1_length': 112, 'unique_sentence1': 1095, 'min_sentence2_length': 3, 'avg_sentence2_length': 15.73, 'max_sentence2_length': 59, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}}}, 'validation': {'num_samples': 19110, 'number_of_characters': 2909058, 'min_sentence1_length': 5, 'avg_sentence1_length': 103.21, 'max_sentence1_length': 323, 'unique_sentence1': 11171, 'min_sentence2_length': 3, 'avg_sentence2_length': 49.02, 'max_sentence2_length': 172, 'unique_sentence2': 19101, 'unique_labels': 2, 'labels': {'0': {'count': 9562}, '1': {'count': 9548}}, 'hf_subset_descriptive_stats': {'ar': {'num_samples': 1365, 'number_of_characters': 177355, 'min_sentence1_length': 13, 'avg_sentence1_length': 88.32, 'max_sentence1_length': 214, 'unique_sentence1': 798, 'min_sentence2_length': 6, 'avg_sentence2_length': 41.61, 'max_sentence2_length': 137, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'bg': {'num_samples': 1365, 'number_of_characters': 219988, 'min_sentence1_length': 16, 'avg_sentence1_length': 109.2, 'max_sentence1_length': 316, 'unique_sentence1': 798, 'min_sentence2_length': 10, 'avg_sentence2_length': 51.97, 'max_sentence2_length': 151, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'de': {'num_samples': 1365, 'number_of_characters': 241852, 'min_sentence1_length': 20, 'avg_sentence1_length': 119.81, 'max_sentence1_length': 298, 'unique_sentence1': 798, 'min_sentence2_length': 12, 'avg_sentence2_length': 57.37, 'max_sentence2_length': 162, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'el': {'num_samples': 1365, 'number_of_characters': 241275, 'min_sentence1_length': 16, 'avg_sentence1_length': 119.88, 'max_sentence1_length': 302, 'unique_sentence1': 798, 'min_sentence2_length': 6, 'avg_sentence2_length': 56.88, 'max_sentence2_length': 171, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'en': {'num_samples': 1365, 'number_of_characters': 212384, 'min_sentence1_length': 20, 'avg_sentence1_length': 105.72, 'max_sentence1_length': 271, 'unique_sentence1': 798, 'min_sentence2_length': 8, 'avg_sentence2_length': 49.88, 'max_sentence2_length': 139, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'es': {'num_samples': 1365, 'number_of_characters': 232451, 'min_sentence1_length': 14, 'avg_sentence1_length': 115.17, 'max_sentence1_length': 265, 'unique_sentence1': 798, 'min_sentence2_length': 7, 'avg_sentence2_length': 55.12, 'max_sentence2_length': 148, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'fr': {'num_samples': 1365, 'number_of_characters': 246857, 'min_sentence1_length': 19, 'avg_sentence1_length': 121.76, 'max_sentence1_length': 323, 'unique_sentence1': 798, 'min_sentence2_length': 11, 'avg_sentence2_length': 59.09, 'max_sentence2_length': 172, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'hi': {'num_samples': 1365, 'number_of_characters': 212269, 'min_sentence1_length': 18, 'avg_sentence1_length': 105.06, 'max_sentence1_length': 277, 'unique_sentence1': 798, 'min_sentence2_length': 7, 'avg_sentence2_length': 50.44, 'max_sentence2_length': 152, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'ru': {'num_samples': 1365, 'number_of_characters': 221152, 'min_sentence1_length': 15, 'avg_sentence1_length': 109.75, 'max_sentence1_length': 310, 'unique_sentence1': 798, 'min_sentence2_length': 8, 'avg_sentence2_length': 52.27, 'max_sentence2_length': 140, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'sw': {'num_samples': 1365, 'number_of_characters': 210482, 'min_sentence1_length': 13, 'avg_sentence1_length': 104.32, 'max_sentence1_length': 264, 'unique_sentence1': 798, 'min_sentence2_length': 8, 'avg_sentence2_length': 49.88, 'max_sentence2_length': 153, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'th': {'num_samples': 1365, 'number_of_characters': 192640, 'min_sentence1_length': 7, 'avg_sentence1_length': 97.28, 'max_sentence1_length': 255, 'unique_sentence1': 798, 'min_sentence2_length': 3, 'avg_sentence2_length': 43.84, 'max_sentence2_length': 140, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'tr': {'num_samples': 1365, 'number_of_characters': 208305, 'min_sentence1_length': 15, 'avg_sentence1_length': 102.97, 'max_sentence1_length': 269, 'unique_sentence1': 798, 'min_sentence2_length': 10, 'avg_sentence2_length': 49.64, 'max_sentence2_length': 139, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'vi': {'num_samples': 1365, 'number_of_characters': 224811, 'min_sentence1_length': 18, 'avg_sentence1_length': 112.26, 'max_sentence1_length': 323, 'unique_sentence1': 798, 'min_sentence2_length': 9, 'avg_sentence2_length': 52.43, 'max_sentence2_length': 159, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'zh': {'num_samples': 1365, 'number_of_characters': 67237, 'min_sentence1_length': 5, 'avg_sentence1_length': 33.41, 'max_sentence1_length': 135, 'unique_sentence1': 798, 'min_sentence2_length': 3, 'avg_sentence2_length': 15.85, 'max_sentence2_length': 66, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}}}} | -| [XNLIV2](https://arxiv.org/pdf/2301.06527) (Upadhyay et al., 2023) | ['asm', 'ben', 'bho', 'ell', 'guj', 'kan', 'mar', 'ory', 'pan', 'rus', 'san', 'tam', 'tur'] | PairClassification | s2s | [Non-fiction, Fiction, Government, Written] | None | None | +| [XNLI](https://aclanthology.org/D18-1269/) (Conneau et al., 2018) | ['ara', 'bul', 'deu', 'ell', 'eng', 'fra', 'hin', 'rus', 'spa', 'swa', 'tha', 'tur', 'vie', 'zho'] | PairClassification | s2s | [Fiction, Government, Non-fiction, Written] | {'test': 19110, 'validation': 19110} | {'test': {'num_samples': 19110, 'number_of_characters': 2907145, 'min_sentence1_length': 3, 'avg_sentence1_length': 103.24, 'max_sentence1_length': 401, 'unique_sentence1': 15328, 'min_sentence2_length': 2, 'avg_sentence2_length': 48.89, 'max_sentence2_length': 187, 'unique_sentence2': 19104, 'unique_labels': 2, 'labels': {'0': {'count': 9562}, '1': {'count': 9548}}, 'hf_subset_descriptive_stats': {'ar': {'num_samples': 1365, 'number_of_characters': 179591, 'min_sentence1_length': 11, 'avg_sentence1_length': 89.57, 'max_sentence1_length': 242, 'unique_sentence1': 1095, 'min_sentence2_length': 8, 'avg_sentence2_length': 41.99, 'max_sentence2_length': 115, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'bg': {'num_samples': 1365, 'number_of_characters': 220646, 'min_sentence1_length': 14, 'avg_sentence1_length': 110.02, 'max_sentence1_length': 303, 'unique_sentence1': 1095, 'min_sentence2_length': 8, 'avg_sentence2_length': 51.63, 'max_sentence2_length': 150, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'de': {'num_samples': 1365, 'number_of_characters': 241224, 'min_sentence1_length': 3, 'avg_sentence1_length': 119.93, 'max_sentence1_length': 301, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 56.79, 'max_sentence2_length': 187, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'el': {'num_samples': 1365, 'number_of_characters': 240222, 'min_sentence1_length': 13, 'avg_sentence1_length': 119.05, 'max_sentence1_length': 344, 'unique_sentence1': 1095, 'min_sentence2_length': 13, 'avg_sentence2_length': 56.93, 'max_sentence2_length': 172, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'en': {'num_samples': 1365, 'number_of_characters': 212223, 'min_sentence1_length': 19, 'avg_sentence1_length': 105.67, 'max_sentence1_length': 268, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 49.8, 'max_sentence2_length': 137, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'es': {'num_samples': 1365, 'number_of_characters': 232207, 'min_sentence1_length': 11, 'avg_sentence1_length': 115.43, 'max_sentence1_length': 385, 'unique_sentence1': 1094, 'min_sentence2_length': 8, 'avg_sentence2_length': 54.68, 'max_sentence2_length': 163, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'fr': {'num_samples': 1365, 'number_of_characters': 245259, 'min_sentence1_length': 9, 'avg_sentence1_length': 121.1, 'max_sentence1_length': 327, 'unique_sentence1': 1095, 'min_sentence2_length': 10, 'avg_sentence2_length': 58.58, 'max_sentence2_length': 169, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'hi': {'num_samples': 1365, 'number_of_characters': 211312, 'min_sentence1_length': 16, 'avg_sentence1_length': 104.63, 'max_sentence1_length': 401, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 50.17, 'max_sentence2_length': 162, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'ru': {'num_samples': 1365, 'number_of_characters': 222797, 'min_sentence1_length': 11, 'avg_sentence1_length': 110.77, 'max_sentence1_length': 306, 'unique_sentence1': 1095, 'min_sentence2_length': 8, 'avg_sentence2_length': 52.45, 'max_sentence2_length': 167, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'sw': {'num_samples': 1365, 'number_of_characters': 210103, 'min_sentence1_length': 10, 'avg_sentence1_length': 104.44, 'max_sentence1_length': 266, 'unique_sentence1': 1094, 'min_sentence2_length': 2, 'avg_sentence2_length': 49.48, 'max_sentence2_length': 146, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'th': {'num_samples': 1365, 'number_of_characters': 192788, 'min_sentence1_length': 12, 'avg_sentence1_length': 96.69, 'max_sentence1_length': 262, 'unique_sentence1': 1095, 'min_sentence2_length': 6, 'avg_sentence2_length': 44.54, 'max_sentence2_length': 129, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'tr': {'num_samples': 1365, 'number_of_characters': 208658, 'min_sentence1_length': 15, 'avg_sentence1_length': 103.68, 'max_sentence1_length': 255, 'unique_sentence1': 1095, 'min_sentence2_length': 6, 'avg_sentence2_length': 49.19, 'max_sentence2_length': 140, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'vi': {'num_samples': 1365, 'number_of_characters': 223549, 'min_sentence1_length': 14, 'avg_sentence1_length': 111.31, 'max_sentence1_length': 265, 'unique_sentence1': 1095, 'min_sentence2_length': 9, 'avg_sentence2_length': 52.46, 'max_sentence2_length': 143, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'zh': {'num_samples': 1365, 'number_of_characters': 66566, 'min_sentence1_length': 4, 'avg_sentence1_length': 33.04, 'max_sentence1_length': 112, 'unique_sentence1': 1095, 'min_sentence2_length': 3, 'avg_sentence2_length': 15.73, 'max_sentence2_length': 59, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}}}, 'validation': {'num_samples': 19110, 'number_of_characters': 2909058, 'min_sentence1_length': 5, 'avg_sentence1_length': 103.21, 'max_sentence1_length': 323, 'unique_sentence1': 11171, 'min_sentence2_length': 3, 'avg_sentence2_length': 49.02, 'max_sentence2_length': 172, 'unique_sentence2': 19101, 'unique_labels': 2, 'labels': {'0': {'count': 9562}, '1': {'count': 9548}}, 'hf_subset_descriptive_stats': {'ar': {'num_samples': 1365, 'number_of_characters': 177355, 'min_sentence1_length': 13, 'avg_sentence1_length': 88.32, 'max_sentence1_length': 214, 'unique_sentence1': 798, 'min_sentence2_length': 6, 'avg_sentence2_length': 41.61, 'max_sentence2_length': 137, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'bg': {'num_samples': 1365, 'number_of_characters': 219988, 'min_sentence1_length': 16, 'avg_sentence1_length': 109.2, 'max_sentence1_length': 316, 'unique_sentence1': 798, 'min_sentence2_length': 10, 'avg_sentence2_length': 51.97, 'max_sentence2_length': 151, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'de': {'num_samples': 1365, 'number_of_characters': 241852, 'min_sentence1_length': 20, 'avg_sentence1_length': 119.81, 'max_sentence1_length': 298, 'unique_sentence1': 798, 'min_sentence2_length': 12, 'avg_sentence2_length': 57.37, 'max_sentence2_length': 162, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'el': {'num_samples': 1365, 'number_of_characters': 241275, 'min_sentence1_length': 16, 'avg_sentence1_length': 119.88, 'max_sentence1_length': 302, 'unique_sentence1': 798, 'min_sentence2_length': 6, 'avg_sentence2_length': 56.88, 'max_sentence2_length': 171, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'en': {'num_samples': 1365, 'number_of_characters': 212384, 'min_sentence1_length': 20, 'avg_sentence1_length': 105.72, 'max_sentence1_length': 271, 'unique_sentence1': 798, 'min_sentence2_length': 8, 'avg_sentence2_length': 49.88, 'max_sentence2_length': 139, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'es': {'num_samples': 1365, 'number_of_characters': 232451, 'min_sentence1_length': 14, 'avg_sentence1_length': 115.17, 'max_sentence1_length': 265, 'unique_sentence1': 798, 'min_sentence2_length': 7, 'avg_sentence2_length': 55.12, 'max_sentence2_length': 148, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'fr': {'num_samples': 1365, 'number_of_characters': 246857, 'min_sentence1_length': 19, 'avg_sentence1_length': 121.76, 'max_sentence1_length': 323, 'unique_sentence1': 798, 'min_sentence2_length': 11, 'avg_sentence2_length': 59.09, 'max_sentence2_length': 172, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'hi': {'num_samples': 1365, 'number_of_characters': 212269, 'min_sentence1_length': 18, 'avg_sentence1_length': 105.06, 'max_sentence1_length': 277, 'unique_sentence1': 798, 'min_sentence2_length': 7, 'avg_sentence2_length': 50.44, 'max_sentence2_length': 152, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'ru': {'num_samples': 1365, 'number_of_characters': 221152, 'min_sentence1_length': 15, 'avg_sentence1_length': 109.75, 'max_sentence1_length': 310, 'unique_sentence1': 798, 'min_sentence2_length': 8, 'avg_sentence2_length': 52.27, 'max_sentence2_length': 140, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'sw': {'num_samples': 1365, 'number_of_characters': 210482, 'min_sentence1_length': 13, 'avg_sentence1_length': 104.32, 'max_sentence1_length': 264, 'unique_sentence1': 798, 'min_sentence2_length': 8, 'avg_sentence2_length': 49.88, 'max_sentence2_length': 153, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'th': {'num_samples': 1365, 'number_of_characters': 192640, 'min_sentence1_length': 7, 'avg_sentence1_length': 97.28, 'max_sentence1_length': 255, 'unique_sentence1': 798, 'min_sentence2_length': 3, 'avg_sentence2_length': 43.84, 'max_sentence2_length': 140, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'tr': {'num_samples': 1365, 'number_of_characters': 208305, 'min_sentence1_length': 15, 'avg_sentence1_length': 102.97, 'max_sentence1_length': 269, 'unique_sentence1': 798, 'min_sentence2_length': 10, 'avg_sentence2_length': 49.64, 'max_sentence2_length': 139, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'vi': {'num_samples': 1365, 'number_of_characters': 224811, 'min_sentence1_length': 18, 'avg_sentence1_length': 112.26, 'max_sentence1_length': 323, 'unique_sentence1': 798, 'min_sentence2_length': 9, 'avg_sentence2_length': 52.43, 'max_sentence2_length': 159, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}, 'zh': {'num_samples': 1365, 'number_of_characters': 67237, 'min_sentence1_length': 5, 'avg_sentence1_length': 33.41, 'max_sentence1_length': 135, 'unique_sentence1': 798, 'min_sentence2_length': 3, 'avg_sentence2_length': 15.85, 'max_sentence2_length': 66, 'unique_sentence2': 1365, 'unique_labels': 2, 'labels': {'0': {'count': 683}, '1': {'count': 682}}}}}} | +| [XNLIV2](https://arxiv.org/pdf/2301.06527) (Upadhyay et al., 2023) | ['asm', 'ben', 'bho', 'ell', 'guj', 'kan', 'mar', 'ory', 'pan', 'rus', 'san', 'tam', 'tur'] | PairClassification | s2s | [Fiction, Government, Non-fiction, Written] | None | None | | [XPQARetrieval](https://arxiv.org/abs/2305.09249) (Shen et al., 2023) | ['ara', 'cmn', 'deu', 'eng', 'fra', 'hin', 'ita', 'jpn', 'kor', 'pol', 'por', 'spa', 'tam'] | Retrieval | s2p | [Reviews, Written] | None | None | | [XQuADRetrieval](https://huggingface.co/datasets/xquad) (Mikel Artetxe, 2019) | ['arb', 'deu', 'ell', 'eng', 'hin', 'ron', 'rus', 'spa', 'tha', 'tur', 'vie', 'zho'] | Retrieval | s2p | [Web, Written] | None | None | | [XStance](https://github.com/ZurichNLP/xstance) | ['deu', 'fra', 'ita'] | PairClassification | s2s | [Social, Written] | None | None | | [YahooAnswersTopicsClassification](https://huggingface.co/datasets/yahoo_answers_topics) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [Web, Written] | None | None | | [YelpReviewFullClassification](https://arxiv.org/abs/1509.01626) (Zhang et al., 2015) | ['eng'] | Classification | s2s | [Reviews, Written] | None | None | | [YueOpenriceReviewClassification](https://github.com/Christainx/Dataset_Cantonese_Openrice) (Xiang et al., 2019) | ['yue'] | Classification | s2s | [Reviews, Spoken] | None | None | -| [indonli](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) | ['ind'] | PairClassification | s2s | [Encyclopaedic, Web, News, Written] | None | None | +| [indonli](https://link.springer.com/chapter/10.1007/978-3-030-41505-1_39) | ['ind'] | PairClassification | s2s | [Encyclopaedic, News, Web, Written] | None | None | | [mFollowIRCrossLingualInstructionRetrieval](https://neuclir.github.io/) (Weller et al., 2024) | ['eng', 'fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'test': 121758} | {'test': {'num_samples': 121758, 'num_docs': 121635, 'num_queries': 123, 'number_of_characters': 283654099, 'min_document_length': 74, 'average_document_length': 2331.08, 'max_document_length': 24179, 'unique_docs': 121635, 'min_query_length': 32, 'average_query_length': 81.88, 'max_query_length': 173, 'unique_queries': 75, 'min_instruction_length': 93, 'average_instruction_length': 389.95, 'max_instruction_length': 887, 'unique_instructions': 75, 'min_changed_instruction_length': 180, 'average_changed_instruction_length': 450.55, 'max_changed_instruction_length': 974, 'unique_changed_instructions': 123, 'min_average_relevant_docs_per_query': 0, 'average_relevant_docs_per_query': 10.43, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000, 'hf_subset_descriptive_stats': {'eng-fas': {'num_samples': 41229, 'num_docs': 41189, 'num_queries': 40, 'number_of_characters': 129597567, 'min_document_length': 99, 'average_document_length': 3145.5, 'max_document_length': 24179, 'unique_docs': 41189, 'min_query_length': 34, 'average_query_length': 80.08, 'max_query_length': 124, 'unique_queries': 40, 'min_instruction_length': 150, 'average_instruction_length': 396.88, 'max_instruction_length': 887, 'unique_instructions': 40, 'min_changed_instruction_length': 205, 'average_changed_instruction_length': 463.18, 'max_changed_instruction_length': 974, 'unique_changed_instructions': 40, 'min_average_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 10.85, 'max_average_relevant_docs_per_query': 22, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}, 'eng-rus': {'num_samples': 39366, 'num_docs': 39326, 'num_queries': 40, 'number_of_characters': 109522175, 'min_document_length': 75, 'average_document_length': 2784.08, 'max_document_length': 24061, 'unique_docs': 39326, 'min_query_length': 32, 'average_query_length': 81.88, 'max_query_length': 173, 'unique_queries': 40, 'min_instruction_length': 93, 'average_instruction_length': 371.12, 'max_instruction_length': 887, 'unique_instructions': 40, 'min_changed_instruction_length': 180, 'average_changed_instruction_length': 431.8, 'max_changed_instruction_length': 957, 'unique_changed_instructions': 40, 'min_average_relevant_docs_per_query': 0, 'average_relevant_docs_per_query': 9.78, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}, 'eng-zho': {'num_samples': 41163, 'num_docs': 41120, 'num_queries': 43, 'number_of_characters': 44534357, 'min_document_length': 74, 'average_document_length': 1082.05, 'max_document_length': 23840, 'unique_docs': 41120, 'min_query_length': 32, 'average_query_length': 83.56, 'max_query_length': 159, 'unique_queries': 43, 'min_instruction_length': 157, 'average_instruction_length': 401.02, 'max_instruction_length': 731, 'unique_instructions': 43, 'min_changed_instruction_length': 209, 'average_changed_instruction_length': 456.26, 'max_changed_instruction_length': 822, 'unique_changed_instructions': 43, 'min_average_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 10.65, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}}}} | | [mFollowIRInstructionRetrieval](https://neuclir.github.io/) (Weller et al., 2024) | ['fas', 'rus', 'zho'] | Retrieval | s2p | [News, Written] | {'test': 121758} | {'test': {'num_samples': 121758, 'num_docs': 121635, 'num_queries': 123, 'number_of_characters': 283622456, 'min_document_length': 74, 'average_document_length': 2331.08, 'max_document_length': 24179, 'unique_docs': 121635, 'min_query_length': 10, 'average_query_length': 57.11, 'max_query_length': 136, 'unique_queries': 123, 'min_instruction_length': 37, 'average_instruction_length': 281.07, 'max_instruction_length': 1009, 'unique_instructions': 123, 'min_changed_instruction_length': 44, 'average_changed_instruction_length': 326.94, 'max_changed_instruction_length': 1083, 'unique_changed_instructions': 123, 'min_average_relevant_docs_per_query': 0, 'average_relevant_docs_per_query': 10.43, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000, 'hf_subset_descriptive_stats': {'fas': {'num_samples': 41229, 'num_docs': 41189, 'num_queries': 40, 'number_of_characters': 129593838, 'min_document_length': 99, 'average_document_length': 3145.5, 'max_document_length': 24179, 'unique_docs': 41189, 'min_query_length': 34, 'average_query_length': 72.65, 'max_query_length': 124, 'unique_queries': 40, 'min_instruction_length': 121, 'average_instruction_length': 358.93, 'max_instruction_length': 759, 'unique_instructions': 40, 'min_changed_instruction_length': 163, 'average_changed_instruction_length': 415.32, 'max_changed_instruction_length': 842, 'unique_changed_instructions': 40, 'min_average_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 10.85, 'max_average_relevant_docs_per_query': 22, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}, 'rus': {'num_samples': 39366, 'num_docs': 39326, 'num_queries': 40, 'number_of_characters': 109523683, 'min_document_length': 75, 'average_document_length': 2784.08, 'max_document_length': 24061, 'unique_docs': 39326, 'min_query_length': 26, 'average_query_length': 77.5, 'max_query_length': 136, 'unique_queries': 40, 'min_instruction_length': 78, 'average_instruction_length': 387.0, 'max_instruction_length': 1009, 'unique_instructions': 40, 'min_changed_instruction_length': 187, 'average_changed_instruction_length': 458.0, 'max_changed_instruction_length': 1083, 'unique_changed_instructions': 40, 'min_average_relevant_docs_per_query': 0, 'average_relevant_docs_per_query': 9.78, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}, 'zho': {'num_samples': 41163, 'num_docs': 41120, 'num_queries': 43, 'number_of_characters': 44504935, 'min_document_length': 74, 'average_document_length': 1082.05, 'max_document_length': 23840, 'unique_docs': 41120, 'min_query_length': 10, 'average_query_length': 23.7, 'max_query_length': 44, 'unique_queries': 43, 'min_instruction_length': 37, 'average_instruction_length': 110.09, 'max_instruction_length': 209, 'unique_instructions': 43, 'min_changed_instruction_length': 44, 'average_changed_instruction_length': 122.81, 'max_changed_instruction_length': 229, 'unique_changed_instructions': 43, 'min_average_relevant_docs_per_query': 1, 'average_relevant_docs_per_query': 10.65, 'max_average_relevant_docs_per_query': 24, 'min_average_top_ranked_per_query': 1000, 'average_top_ranked_per_query': 1000.0, 'max_average_top_ranked_per_query': 1000}}}} | ->>>>>>> main @@ -1113,2117 +866,1061 @@ The following tables give you an overview of the tasks in MTEB.
-<<<<<<< HEAD -| Language | BitextMining | Classification | Clustering | InstructionRetrieval | MultilabelClassification | PairClassification | Reranking | Retrieval | STS | Speed | Summarization | -|---|------|------|------|------|------|------|------|------|------|------|---| -| aai | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aak | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aau | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aaz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| abs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| abt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| abx | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aby | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ace | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| acf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| acm | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| acq | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| acr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| acu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| adz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aeb | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aer | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aey | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| afr | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | -| agd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| agg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| agm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| agn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| agr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| agt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| agu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aia | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aii | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ajp | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aka | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ake | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| alp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| alq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| als | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| aly | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ame | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| amf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| amh | 3 | 6 | 3 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | -| amk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| amm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| amn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| amo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| amp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| amr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| amu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| amx | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ang | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| anh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| anp | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| anv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aoi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aoj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aom | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aon | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| apb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| apc | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| ape | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| apn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| apr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| apu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| apw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| apz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ara | 2 | 12 | 0 | 0 | 0 | 2 | 1 | 9 | 2 | 0 | 0 | -| arb | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 2 | 1 | 0 | 0 | -| are | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| arl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| arn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| arp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| arq | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | -| ars | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| ary | 1 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | -| arz | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| asm | 5 | 3 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | -| aso | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ast | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ata | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| atb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| atd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| atg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| att | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| auc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aui | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| auy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| avt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| awa | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| awb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| awk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| awx | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ayr | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| azb | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| aze | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| azg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| azj | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| azz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bak | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bam | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| ban | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bao | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bba | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bbb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bbc | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bbr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bch | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bco | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bdd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bef | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bel | 4 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bem | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ben | 7 | 9 | 2 | 0 | 0 | 1 | 2 | 6 | 1 | 0 | 0 | -| beo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ber | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| beu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bew | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bgc | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bgs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bgt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bhb | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bhd | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bhg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bhl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bho | 2 | 2 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | -| bhp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| big | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bjj | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bjk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bjn | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bjp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bjr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bjv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bjz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bkd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bki | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bkq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bkx | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| blw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| blz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bmh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bmk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bmr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bmu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bnp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bns | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| boa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bod | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| boj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bon | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bos | 3 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| box | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| boy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bpr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bps | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bqc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bqp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bra | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bre | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| brx | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bsj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bsn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bsp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bss | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bug | 2 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| buk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bul | 3 | 4 | 1 | 0 | 1 | 1 | 1 | 2 | 0 | 0 | 0 | -| bus | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bvd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bvr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bxh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| byr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| byx | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bzd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bzh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| bzj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| caa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cab | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cac | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| caf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cak | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cao | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cap | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| car | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cat | 3 | 2 | 2 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| cav | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cax | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cbc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cbi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cbk | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cbr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cbs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cbt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cbu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cbv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cco | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ceb | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| cek | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ces | 4 | 5 | 2 | 0 | 1 | 1 | 1 | 2 | 0 | 0 | 0 | -| cgc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cha | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| chd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| chf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| chk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| chq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| chv | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| chz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cjk | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cjo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cjv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ckb | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| cle | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| clu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cme | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cmn | 4 | 10 | 4 | 0 | 0 | 3 | 4 | 10 | 9 | 0 | 0 | -| cmo | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cni | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cnl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cnt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| code | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 37 | 0 | 0 | 0 | -| cof | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| con | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cop | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cor | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cot | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cpa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cpb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cpc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cpu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cpy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| crh | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| crn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| crx | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| csb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cso | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| csy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cta | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cth | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ctp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ctu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cub | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cuc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cui | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cuk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cut | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cux | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cwe | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cya | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| cym | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| daa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dad | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dah | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dan | 5 | 9 | 2 | 0 | 1 | 0 | 1 | 5 | 0 | 0 | 0 | -| ded | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| deu | 6 | 14 | 7 | 0 | 1 | 6 | 2 | 18 | 4 | 0 | 0 | -| dgc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dgr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dgz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dhg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dif | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dik | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| div | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dji | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| djk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| djr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dob | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| doi | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dop | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dov | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dsb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dtp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dwr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dww | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dwy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dyu | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dza | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| dzo | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ebk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| eko | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ell | 3 | 6 | 1 | 0 | 1 | 2 | 0 | 3 | 0 | 0 | 0 | -| emi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| emp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| eng | 16 | 143 | 16 | 3 | 1 | 8 | 8 | 91 | 13 | 2 | 1 | -| enq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| epo | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| eri | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ese | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| esk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| est | 2 | 2 | 1 | 0 | 1 | 0 | 0 | 2 | 0 | 0 | 0 | -| etr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| eus | 3 | 2 | 2 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| ewe | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| faa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fai | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fao | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | -| far | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fas | 1 | 4 | 0 | 0 | 0 | 1 | 2 | 9 | 0 | 0 | 0 | -| ffm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fij | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fil | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fin | 3 | 5 | 1 | 0 | 1 | 1 | 2 | 5 | 1 | 0 | 0 | -| fon | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| for | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fra | 7 | 13 | 8 | 0 | 1 | 5 | 3 | 14 | 4 | 0 | 1 | -| fry | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fuc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fue | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fuf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fuh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fur | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| fuv | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| gah | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gai | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gam | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gaw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gaz | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| gbm | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gdn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gdr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| geb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gfk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ghs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gla | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gle | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| glg | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| glk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| glv | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gmv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gng | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gnn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gnw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gof | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gom | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| grc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| grn | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| gsw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gub | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| guh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gui | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| guj | 6 | 6 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | -| gul | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gum | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gun | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| guo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gup | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gux | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gvc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gvf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gvn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gvs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gwi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gym | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| gyr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hat | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| hau | 4 | 5 | 3 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | -| haw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hbo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hch | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| heb | 4 | 5 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| heg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hin | 9 | 12 | 2 | 0 | 0 | 1 | 2 | 10 | 2 | 0 | 0 | -| hix | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hla | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hlt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hmn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hmo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hne | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hns | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hop | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hot | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hrv | 4 | 3 | 1 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | -| hsb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hto | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hub | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hui | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hun | 5 | 3 | 1 | 0 | 1 | 0 | 0 | 2 | 0 | 0 | 0 | -| hus | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| huu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| huv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hvn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hye | 3 | 3 | 1 | 0 | 0 | 1 | 0 | 1 | 0 | 0 | 0 | -| ian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ibo | 3 | 5 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| ido | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ign | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ikk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ikw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ile | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ilo | 2 | 1 | 2 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| imo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ina | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| inb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ind | 6 | 7 | 1 | 0 | 0 | 1 | 1 | 4 | 1 | 0 | 0 | -| ino | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| iou | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ipi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| isl | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| isn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ita | 5 | 9 | 1 | 0 | 1 | 2 | 1 | 5 | 3 | 0 | 0 | -| iws | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ixl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| jac | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| jae | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| jao | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| jav | 4 | 7 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| jic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| jid | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| jiv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| jni | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| jpn | 5 | 8 | 3 | 0 | 0 | 1 | 3 | 13 | 2 | 0 | 0 | -| jvn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kab | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kac | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| kam | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kan | 6 | 7 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | -| kaq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kas | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kat | 4 | 3 | 1 | 0 | 0 | 0 | 0 | 2 | 0 | 0 | 0 | -| kaz | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| kbc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kbh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kbm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kbp | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kbq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kdc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kde | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kdl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kea | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| kek | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ken | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kew | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kfg | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kfy | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kgf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kgk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kgp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| khk | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| khm | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| khs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| khz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kik | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kin | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | -| kir | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| kiw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kiz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kje | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kjs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kkc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kkl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| klt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| klv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kmb | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kmg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kmh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kmk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kmo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kmr | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kms | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kmu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| knc | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kne | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| knf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| knj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| knv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kon | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kor | 4 | 8 | 1 | 0 | 1 | 2 | 1 | 8 | 3 | 0 | 0 | -| kos | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kpf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kpg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kpj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kpr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kpw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kpx | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kqa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kqc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kqf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kql | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kqw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| krc | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ksd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ksj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ksr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ktm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kto | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kud | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kue | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kup | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kur | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kvg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kvn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kwd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kwf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kwi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kwj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kyc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kyf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kyg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kyq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kyz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kze | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| kzj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lac | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lao | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| lat | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lav | 1 | 2 | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | -| lbb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lbk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lcm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| leu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lex | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lfn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lgl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lid | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lif | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lij | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lim | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lin | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| lit | 4 | 1 | 1 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | -| llg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lmo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ltg | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ltz | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lua | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lug | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| luo | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| lus | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| lvs | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| lww | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| maa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mad | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mag | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mai | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| maj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mak | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mal | 7 | 7 | 2 | 0 | 0 | 0 | 0 | 2 | 1 | 0 | 0 | -| mam | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| maq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mar | 7 | 6 | 2 | 0 | 0 | 1 | 0 | 2 | 2 | 0 | 0 | -| mau | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mav | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| max | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| maz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mbb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mbc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mbh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mbj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mbl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mbs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mbt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mca | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mcb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mcd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mcf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mco | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mcp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mcq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mcr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mdy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| med | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mee | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mek | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| meq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| met | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| meu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mey | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mgc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mgh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mgw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mhl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mhr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mib | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mie | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mig | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mih | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mil | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| min | 3 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mio | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mir | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| miz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mjc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mkd | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| mkj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mkl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mkn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mks | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mle | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mlg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mlh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mlp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mlt | 2 | 2 | 2 | 0 | 2 | 0 | 0 | 1 | 0 | 0 | 0 | -| mmo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mmx | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mna | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mni | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mon | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mop | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mos | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mox | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mph | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mpj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mpm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mpp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mps | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mpt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mpx | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mqb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mqj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mri | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| msa | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| msb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| msc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| msk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| msm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| msy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mti | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mto | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mui | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mup | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mux | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| muy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mva | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mvn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mwc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mwe | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mwf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mwp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mwr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mxb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mxp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mxq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mxt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mya | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| myk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| myu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| myw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| myy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| mzz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nab | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| naf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nak | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nas | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nbl | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nbq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nca | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nch | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ncj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ncl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ncu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nde | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ndg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ndj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nds | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nep | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nfa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ngp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ngu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nhe | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nhg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nhi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nho | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nhr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nhu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nhw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nhy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nif | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nii | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nij | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nin | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nko | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nld | 6 | 6 | 1 | 0 | 1 | 0 | 1 | 2 | 2 | 0 | 0 | -| nlg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nna | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nno | 4 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nnq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| noa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nob | 4 | 7 | 5 | 0 | 0 | 0 | 0 | 3 | 0 | 0 | 0 | -| noe | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nop | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nor | 0 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | -| not | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nou | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nov | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| npi | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| npl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nqo | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nsn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nso | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| nss | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ntj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ntp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ntu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nus | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nuy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nvm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nwi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nya | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| nys | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| nyu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| obo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| oci | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| okv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| omw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ong | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ons | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ood | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| opm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ori | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| orm | 1 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| orv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ory | 5 | 4 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | -| ote | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| otm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| otn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| otq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ots | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pab | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pad | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pag | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pah | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pam | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pan | 6 | 6 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | -| pao | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pap | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pbt | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| pcm | 1 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pes | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| pib | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pio | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pir | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| piu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pjt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pls | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| plt | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| plu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pma | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pms | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| poe | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| poh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| poi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pol | 4 | 11 | 4 | 0 | 1 | 4 | 0 | 18 | 4 | 0 | 0 | -| pon | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| por | 4 | 9 | 1 | 0 | 2 | 2 | 1 | 5 | 3 | 0 | 0 | -| poy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ppo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| prf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pri | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| prs | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ptp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ptu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pus | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| pwg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qub | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| quc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| quf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| quh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qul | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qup | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| quy | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qvc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qve | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qvh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qvm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qvn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qvs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qvw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qvz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qwh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qxh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qxn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| qxo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rai | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| raj | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| reg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rej | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rgu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rkb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rmc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rmy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rom | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ron | 5 | 6 | 1 | 0 | 1 | 0 | 1 | 3 | 1 | 0 | 0 | -| roo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rop | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| row | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rro | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ruf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rug | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| run | 1 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| rus | 5 | 13 | 6 | 0 | 2 | 4 | 2 | 16 | 4 | 0 | 0 | -| rwo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sab | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sag | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sah | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| san | 5 | 3 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | -| sat | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sbe | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sbk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sbs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| scn | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sco | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| seh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sey | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sgb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sgz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| shi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| shj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| shn | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| shp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sim | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sin | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| sja | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| slk | 3 | 4 | 1 | 0 | 1 | 0 | 0 | 3 | 0 | 0 | 0 | -| sll | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| slv | 3 | 4 | 1 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | -| smk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| smo | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sna | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| snc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| snd | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| snn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| snp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| snx | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sny | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| som | 3 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| soq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sot | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| soy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| spa | 4 | 13 | 4 | 0 | 1 | 2 | 2 | 12 | 4 | 0 | 0 | -| spl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| spm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| spp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sps | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| spy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sqi | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| srd | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sri | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| srm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| srn | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| srp | 4 | 1 | 1 | 0 | 0 | 0 | 1 | 2 | 0 | 0 | 0 | -| srq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ssd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ssg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ssw | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| ssx | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| stp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sua | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sue | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sun | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| sus | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| suz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| svk | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| swa | 1 | 7 | 2 | 0 | 0 | 1 | 1 | 3 | 0 | 0 | 0 | -| swe | 4 | 8 | 3 | 0 | 1 | 1 | 1 | 4 | 0 | 0 | 0 | -| swg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| swh | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| swp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| sxb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| szl | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tac | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tah | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| taj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tam | 7 | 7 | 2 | 0 | 0 | 1 | 0 | 3 | 1 | 0 | 0 | -| taq | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tat | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tav | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| taw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tbc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tbf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tbg | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tbo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tbz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tca | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tcs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tcz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tdt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tee | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tel | 7 | 7 | 2 | 0 | 0 | 0 | 1 | 5 | 2 | 0 | 0 | -| ter | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tet | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tew | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tfr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tgk | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| tgl | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| tgo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tgp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tha | 4 | 8 | 1 | 0 | 0 | 1 | 1 | 6 | 0 | 0 | 0 | -| tif | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tim | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tir | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| tiw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tiy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tke | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tku | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tlf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tmd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tna | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tnc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tnk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tnn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tnp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| toc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tod | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tof | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| toj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ton | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| too | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| top | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tos | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tpa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tpi | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tpt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tpz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| trc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tsn | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| tso | 1 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| tsw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ttc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tte | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tuc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tue | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tuf | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tuk | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tum | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tuo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tur | 4 | 7 | 1 | 0 | 0 | 2 | 0 | 3 | 2 | 0 | 0 | -| tvk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| twi | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| txq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| txu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tyv | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tzj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tzl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tzm | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| tzo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ubr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ubu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| udu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| uig | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ukr | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| uli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ulk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| umb | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| upv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ura | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| urb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| urd | 7 | 8 | 2 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | -| uri | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| urt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| urw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| usa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| usp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| uvh | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| uvl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| uzb | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| uzn | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| vec | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ven | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| vid | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| vie | 5 | 6 | 1 | 0 | 0 | 1 | 0 | 5 | 0 | 0 | 0 | -| viv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| vmy | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| waj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wal | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wap | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| war | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| wat | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wbi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wbp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wed | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wer | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wim | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wiu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wiv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wln | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wmt | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wmw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wnc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wnu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wol | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| wos | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wrk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wro | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wrs | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wsk | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wuu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| wuv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| xav | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| xbi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| xed | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| xho | 3 | 3 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| xla | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| xnn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| xon | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| xsi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| xtd | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| xtm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yaa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yad | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yal | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yap | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yaq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yby | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ycn | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ydd | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yid | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yka | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yle | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yml | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yon | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yor | 4 | 5 | 3 | 0 | 0 | 0 | 1 | 3 | 0 | 0 | 0 | -| yrb | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yre | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yss | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yue | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yuj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yut | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yuw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| yva | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zaa | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zab | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zac | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zad | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zai | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zaj | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zam | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zao | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zap | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zar | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zas | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zat | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zav | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zaw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zca | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zga | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zho | 2 | 2 | 1 | 0 | 0 | 1 | 1 | 13 | 0 | 0 | 0 | -| zia | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ziw | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zlm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zos | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zpc | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zpl | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zpm | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zpo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zpq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zpu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zpv | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zpz | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zsm | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| zsr | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| ztq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zty | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| zul | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | -| zyp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| Total | 1394 | 795 | 304 | 3 | 28 | 67 | 50 | 456 | 85 | 2 | 2 | -======= -| ISO Code | Language | Family | BitextMining | Classification | Clustering | InstructionRetrieval | MultilabelClassification | PairClassification | Reranking | Retrieval | STS | Speed | Summarization | Sum | -|---|------|------|------|------|------|------|------|------|------|------|------|---| -| aai | Arifama-Miniafia | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aak | Ankave | Angan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aau | Abau | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aaz | Amarasi | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| abs | Ambonese Malay | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| abt | Ambulas | Ndu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| abx | Inabaknon | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aby | Aneme Wake | Yareban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ace | Achinese | Austronesian | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| acf | Saint Lucian Creole French | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| acm | Mesopotamian Arabic | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| acq | Ta'izzi-Adeni Arabic | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| acr | Achi | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| acu | Achuar-Shiwiar | Chicham | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| adz | Adzera | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aeb | Tunisian Arabic | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| aer | Eastern Arrernte | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aey | Amele | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| afr | Afrikaans | Indo-European | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 10 | -| agd | Agarabi | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| agg | Angor | Senagi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| agm | Angaataha | Angan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| agn | Agutaynen | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| agr | Aguaruna | Chicham | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| agt | Central Cagayan Agta | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| agu | Aguacateco | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aia | Arosi | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aii | Assyrian Neo-Aramaic | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ajp | South Levantine Arabic | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| aka | Akan | Atlantic-Congo | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| ake | Akawaio | Cariban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| alp | Alune | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| alq | Algonquin | Algic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| als | Tosk Albanian | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | -| aly | Alyawarr | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ame | Yanesha' | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| amf | Hamer-Banna | South Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| amh | Amharic | Afro-Asiatic | 3 | 6 | 3 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 14 | -| amk | Ambai | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| amm | Ama (Papua New Guinea) | Left May | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| amn | Amanab | Border | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| amo | Amo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| amp | Alamblak | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| amr | Amarakaeri | Harakmbut | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| amu | Guerrero Amuzgo | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| amx | Anmatyerre | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ang | Old English (ca. 450-1100) | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| anh | Nend | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| anp | Angika | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| anv | Denya | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aoi | Anindilyakwa | Gunwinyguan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aoj | Mufian | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aom | Ömie | Koiarian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aon | Bumbita Arapesh | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| apb | Sa'a | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| apc | Levantine Arabic | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| ape | Bukiyip | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| apn | Apinayé | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| apr | Arop-Lokep | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| apu | Apurinã | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| apw | Western Apache | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| apz | Safeyoka | Angan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ara | Arabic | Unclassified | 2 | 12 | 0 | 0 | 0 | 2 | 2 | 9 | 2 | 0 | 0 | 29 | -| arb | Standard Arabic | Afro-Asiatic | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 2 | 1 | 0 | 0 | 8 | -| are | Western Arrarnta | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| arl | Arabela | Zaparoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| arn | Mapudungun | Araucanian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| arp | Arapaho | Algic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| arq | Algerian Arabic | Afro-Asiatic | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 4 | -| ars | Najdi Arabic | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| ary | Moroccan Arabic | Afro-Asiatic | 1 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 7 | -| arz | Egyptian Arabic | Afro-Asiatic | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | -| asm | Assamese | Indo-European | 5 | 3 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 14 | -| aso | Dano | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ast | Asturian | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| ata | Pele-Ata | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| atb | Zaiwa | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| atd | Ata Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| atg | Ivbie North-Okpela-Arhe | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| att | Pamplona Atta | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| auc | Waorani | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| aui | Anuki | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| auy | Awiyaana | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| avt | Au | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| awa | Awadhi | Indo-European | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| awb | Awa (Papua New Guinea) | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| awk | Awabakal | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| awx | Awara | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ayr | Central Aymara | Aymaran | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| azb | South Azerbaijani | Turkic | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| aze | Azerbaijani | Unclassified | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| azg | San Pedro Amuzgos Amuzgo | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| azj | North Azerbaijani | Turkic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| azz | Highland Puebla Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bak | Bashkir | Turkic | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| bam | Bambara | Mande | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | -| ban | Balinese | Austronesian | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| bao | Waimaha | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bba | Baatonum | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bbb | Barai | Koiarian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bbc | Batak Toba | Austronesian | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| bbr | Girawa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bch | Bariai | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bco | Kaluli | Bosavi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bdd | Bunama | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bea | Beaver | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bef | Benabena | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bel | Belarusian | Indo-European | 4 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| bem | Bemba (Zambia) | Atlantic-Congo | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| ben | Bengali | Indo-European | 7 | 9 | 2 | 0 | 0 | 1 | 2 | 6 | 1 | 0 | 0 | 28 | -| beo | Beami | Bosavi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ber | Berber (Other) | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| beu | Blagar | Timor-Alor-Pantar | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bew | Betawi | Austronesian | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| bgc | Haryanvi | Indo-European | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| bgs | Tagabawa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bgt | Bughotu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bhb | Bhili | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bhd | Bhadrawahi | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bhg | Binandere | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bhl | Bimin | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bho | Bhojpuri | Indo-European | 2 | 2 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | -| bhp | Bima | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| big | Biangai | Kunimaipan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bjj | Kanauji | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bjk | Barok | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bjn | Banjar | Austronesian | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| bjp | Fanamaket | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bjr | Binumarien | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bjv | Bedjond | Central Sudanic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bjz | Baruga | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bkd | Binukid | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bki | Baki | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bkq | Bakairí | Cariban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bkx | Baikeno | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| blw | Balangao | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| blz | Balantak | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bmh | Kein | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bmk | Ghayavi | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bmr | Muinane | Boran | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bmu | Somba-Siawari | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bnp | Bola | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bns | Bundeli | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| boa | Bora | Boran | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bod | Tibetan | Sino-Tibetan | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | -| boj | Anjam | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bon | Bine | Eastern Trans-Fly | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bos | Bosnian | Indo-European | 3 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| box | Buamu | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| boy | Bodo (Central African Republic) | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bpr | Koronadal Blaan | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bps | Sarangani Blaan | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bqc | Boko (Benin) | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bqp | Busa | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bra | Braj | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bre | Breton | Indo-European | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| brx | Bodo (India) | Sino-Tibetan | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| bsj | Bangwinji | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bsn | Barasana-Eduria | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bsp | Baga Sitemu | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bss | Akoose | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bug | Buginese | Austronesian | 2 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | -| buk | Bugawac | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bul | Bulgarian | Indo-European | 3 | 4 | 1 | 0 | 1 | 1 | 1 | 2 | 0 | 0 | 0 | 13 | -| bus | Bokobaru | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bvd | Baeggu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bvr | Burarra | Maningrida | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bxh | Buhutu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| byr | Baruya | Angan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| byx | Qaqet | Baining | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bzd | Bribri | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bzh | Mapos Buang | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| bzj | Belize Kriol English | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| caa | Chortí | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cab | Garifuna | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cac | Chuj | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| caf | Southern Carrier | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cak | Kaqchikel | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cao | Chácobo | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cap | Chipaya | Uru-Chipaya | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| car | Galibi Carib | Cariban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cat | Catalan | Indo-European | 3 | 2 | 2 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| cav | Cavineña | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cax | Chiquitano | Chiquitano | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cbc | Carapana | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cbi | Chachi | Barbacoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cbk | Chavacano | Indo-European | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| cbr | Cashibo-Cacataibo | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cbs | Cashinahua | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cbt | Chayahuita | Cahuapanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cbu | Candoshi-Shapra | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cbv | Cacua | Kakua-Nukak | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cco | Comaltepec Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ceb | Cebuano | Austronesian | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | -| cek | Eastern Khumi Chin | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ces | Czech | Indo-European | 4 | 5 | 2 | 0 | 1 | 1 | 1 | 2 | 0 | 0 | 0 | 16 | -| cgc | Kagayanen | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cha | Chamorro | Austronesian | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| chd | Highland Oaxaca Chontal | Tequistlatecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| chf | Tabasco Chontal | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| chk | Chuukese | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| chq | Quiotepec Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| chv | Chuvash | Turkic | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| chz | Ozumacín Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cjk | Chokwe | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| cjo | Ashéninka Pajonal | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cjv | Chuave | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ckb | Central Kurdish | Indo-European | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | -| cle | Lealao Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| clu | Caluyanun | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cme | Cerma | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cmn | Mandarin Chinese | Sino-Tibetan | 4 | 10 | 4 | 0 | 0 | 3 | 4 | 10 | 9 | 0 | 0 | 44 | -| cmo | Central Mnong | Austroasiatic | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| cni | Asháninka | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cnl | Lalana Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cnt | Tepetotutla Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| code | unknown | Unclassified | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 37 | 0 | 0 | 0 | 37 | -| cof | Colorado | Barbacoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| con | Cofán | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cop | Coptic | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cor | Cornish | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cot | Caquinte | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cpa | Palantla Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cpb | Ucayali-Yurúa Ashéninka | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cpc | Ajyíninka Apurucayali | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cpu | Pichis Ashéninka | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cpy | South Ucayali Ashéninka | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| crh | Crimean Tatar | Turkic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| crn | El Nayar Cora | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| crx | Carrier | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| csb | Kashubian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cso | Sochiapam Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| csy | Siyin Chin | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cta | Tataltepec Chatino | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cth | Thaiphum Chin | Bookkeeping | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ctp | Western Highland Chatino | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ctu | Chol | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cub | Cubeo | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cuc | Usila Chinantec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cui | Cuiba | Guahiboan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cuk | San Blas Kuna | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cut | Teutila Cuicatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cux | Tepeuxila Cuicatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cwe | Kwere | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cya | Nopala Chatino | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| cym | Welsh | Indo-European | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 8 | -| daa | Dangaléat | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dad | Marik | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dah | Gwahatike | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dan | Danish | Indo-European | 5 | 9 | 2 | 0 | 1 | 0 | 1 | 5 | 0 | 0 | 0 | 23 | -| ded | Dedua | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| deu | German | Indo-European | 6 | 14 | 7 | 0 | 1 | 6 | 2 | 18 | 4 | 0 | 0 | 58 | -| dgc | Casiguran Dumagat Agta | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dgr | Dogrib | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dgz | Daga | Dagan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dhg | Dhangu-Djangu | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dif | Dieri | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dik | Southwestern Dinka | Nilotic | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| div | Dhivehi | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dji | Djinang | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| djk | Eastern Maroon Creole | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| djr | Djambarrpuyngu | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dob | Dobu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| doi | Dogri (macrolanguage) | Unclassified | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| dop | Lukpa | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dov | Dombe | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dsb | Lower Sorbian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dtp | Kadazan Dusun | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dwr | Dawro | Ta-Ne-Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dww | Dawawa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dwy | Dhuwaya | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dyu | Dyula | Mande | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| dza | Tunzu | Atlantic-Congo | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| dzo | Dzongkha | Sino-Tibetan | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| ebk | Eastern Bontok | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| eko | Koti | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ell | Modern Greek (1453-) | Indo-European | 3 | 6 | 1 | 0 | 1 | 2 | 0 | 3 | 0 | 0 | 0 | 16 | -| emi | Mussau-Emira | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| emp | Northern Emberá | Chocoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| eng | English | Indo-European | 16 | 143 | 16 | 3 | 1 | 8 | 8 | 105 | 13 | 2 | 1 | 316 | -| enq | Enga | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| epo | Esperanto | Artificial Language | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| eri | Ogea | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ese | Ese Ejja | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| esk | Northwest Alaska Inupiatun | Eskimo-Aleut | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| est | Estonian | Uralic | 2 | 2 | 1 | 0 | 1 | 0 | 0 | 2 | 0 | 0 | 0 | 8 | -| etr | Edolo | Bosavi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| eus | Basque | Unclassified | 3 | 2 | 2 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| ewe | Ewe | Atlantic-Congo | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| faa | Fasu | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| fai | Faiwol | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| fao | Faroese | Indo-European | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 7 | -| far | Fataleka | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| fas | Persian | Indo-European | 1 | 4 | 0 | 0 | 0 | 1 | 2 | 9 | 0 | 0 | 0 | 17 | -| ffm | Maasina Fulfulde | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| fij | Fijian | Austronesian | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| fil | Filipino | Austronesian | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| fin | Finnish | Uralic | 3 | 5 | 1 | 0 | 1 | 1 | 2 | 5 | 1 | 0 | 0 | 19 | -| fon | Fon | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| for | Fore | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| fra | French | Indo-European | 7 | 13 | 8 | 0 | 1 | 5 | 3 | 15 | 4 | 0 | 1 | 57 | -| fry | Western Frisian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| fuc | Pulaar | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| fue | Borgu Fulfulde | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| fuf | Pular | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| fuh | Western Niger Fulfulde | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| fur | Friulian | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| fuv | Nigerian Fulfulde | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| gah | Alekano | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gai | Borei | Ramu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gam | Kandawo | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gaw | Nobonob | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gaz | West Central Oromo | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| gbm | Garhwali | Indo-European | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| gdn | Umanakaina | Dagan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gdr | Wipi | Eastern Trans-Fly | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| geb | Kire | Ramu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gfk | Patpatar | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ghs | Guhu-Samane | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gla | Scottish Gaelic | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| gle | Irish | Indo-European | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| glg | Galician | Indo-European | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| glk | Gilaki | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| glv | Manx | Indo-European | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gmv | Gamo | Ta-Ne-Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gng | Ngangam | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gnn | Gumatj | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gnw | Western Bolivian Guaraní | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gof | Gofa | Ta-Ne-Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gom | Goan Konkani | Indo-European | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| grc | Ancient Greek (to 1453) | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| grn | Guarani | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| gsw | Swiss German | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gub | Guajajára | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| guh | Guahibo | Guahiboan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gui | Eastern Bolivian Guaraní | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| guj | Gujarati | Indo-European | 6 | 6 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 18 | -| gul | Sea Island Creole English | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gum | Guambiano | Barbacoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gun | Mbyá Guaraní | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| guo | Guayabero | Guahiboan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gup | Gunwinggu | Gunwinyguan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gux | Gourmanchéma | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gvc | Guanano | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gvf | Golin | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gvn | Kuku-Yalanji | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gvs | Gumawana | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gwi | Gwichʼin | Athabaskan-Eyak-Tlingit | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gym | Ngäbere | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| gyr | Guarayu | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hat | Haitian | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | -| hau | Hausa | Afro-Asiatic | 4 | 5 | 3 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 14 | -| haw | Hawaiian | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hbo | Ancient Hebrew | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hch | Huichol | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| heb | Hebrew | Afro-Asiatic | 4 | 5 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 11 | -| heg | Helong | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hin | Hindi | Indo-European | 9 | 12 | 2 | 0 | 0 | 1 | 2 | 10 | 2 | 0 | 0 | 38 | -| hix | Hixkaryána | Cariban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hla | Halia | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hlt | Matu Chin | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hmn | Hmong | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hmo | Hiri Motu | Pidgin | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hne | Chhattisgarhi | Indo-European | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| hns | Caribbean Hindustani | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hop | Hopi | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hot | Hote | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hrv | Croatian | Indo-European | 4 | 3 | 1 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 10 | -| hsb | Upper Sorbian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hto | Minica Huitoto | Huitotoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hub | Huambisa | Chicham | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hui | Huli | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hun | Hungarian | Uralic | 5 | 3 | 1 | 0 | 1 | 0 | 0 | 2 | 0 | 0 | 0 | 12 | -| hus | Huastec | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| huu | Murui Huitoto | Huitotoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| huv | San Mateo Del Mar Huave | Huavean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hvn | Sabu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| hye | Armenian | Indo-European | 3 | 3 | 1 | 0 | 0 | 1 | 0 | 1 | 0 | 0 | 0 | 9 | -| ian | Iatmul | Ndu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ibo | Igbo | Atlantic-Congo | 3 | 5 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 12 | -| ido | Ido | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ign | Ignaciano | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ikk | Ika | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ikw | Ikwere | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ile | Interlingue | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ilo | Iloko | Austronesian | 2 | 1 | 2 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | -| imo | Imbongu | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ina | Interlingua (International Auxiliary Language Association) | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| inb | Inga | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ind | Indonesian | Austronesian | 6 | 7 | 1 | 0 | 0 | 1 | 1 | 4 | 1 | 0 | 0 | 21 | -| ino | Inoke-Yate | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| iou | Tuma-Irumu | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ipi | Ipili | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| isl | Icelandic | Indo-European | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 9 | -| isn | Isanzu | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ita | Italian | Indo-European | 5 | 9 | 1 | 0 | 1 | 2 | 1 | 5 | 3 | 0 | 0 | 27 | -| iws | Sepik Iwam | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ixl | Ixil | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| jac | Popti' | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| jae | Yabem | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| jao | Yanyuwa | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| jav | Javanese | Austronesian | 4 | 7 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 13 | -| jic | Tol | Jicaquean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| jid | Bu (Kaduna State) | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| jiv | Shuar | Chicham | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| jni | Janji | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| jpn | Japanese | Japonic | 5 | 8 | 3 | 0 | 0 | 1 | 3 | 13 | 2 | 0 | 0 | 35 | -| jvn | Caribbean Javanese | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kab | Kabyle | Afro-Asiatic | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| kac | Kachin | Sino-Tibetan | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| kam | Kamba (Kenya) | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| kan | Kannada | Dravidian | 6 | 7 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 19 | -| kaq | Capanahua | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kas | Kashmiri | Indo-European | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| kat | Georgian | Kartvelian | 4 | 3 | 1 | 0 | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 10 | -| kaz | Kazakh | Turkic | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| kbc | Kadiwéu | Guaicuruan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kbh | Camsá | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kbm | Iwal | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kbp | Kabiyè | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| kbq | Kamano | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kdc | Kutu | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kde | Makonde | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kdl | Tsikimba | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kea | Kabuverdianu | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| kek | Kekchí | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ken | Kenyang | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kew | West Kewa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kfg | Kudiya | Dravidian | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kfy | Kumaoni | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kgf | Kube | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kgk | Kaiwá | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kgp | Kaingang | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| khk | Halh Mongolian | Mongolic-Khitan | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| khm | Khmer | Austroasiatic | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| khs | Kasua | Bosavi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| khz | Keapara | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kik | Kikuyu | Atlantic-Congo | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| kin | Kinyarwanda | Atlantic-Congo | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 8 | -| kir | Kirghiz | Turkic | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | -| kiw | Northeast Kiwai | Kiwaian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kiz | Kisi | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kje | Kisar | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kjs | East Kewa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kkc | Odoodee | East Strickland | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kkl | Kosarek Yale | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| klt | Nukna | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| klv | Maskelynes | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kmb | Kimbundu | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| kmg | Kâte | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kmh | Kalam | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kmk | Limos Kalinga | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kmo | Kwoma | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kmr | Northern Kurdish | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| kms | Kamasau | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kmu | Kanite | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| knc | Central Kanuri | Saharan | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| kne | Kankanaey | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| knf | Mankanya | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| knj | Western Kanjobal | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| knv | Tabo | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kon | Kongo | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| kor | Korean | Koreanic | 4 | 8 | 1 | 0 | 1 | 2 | 1 | 9 | 3 | 0 | 0 | 29 | -| kos | Kosraean | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kpf | Komba | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kpg | Kapingamarangi | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kpj | Karajá | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kpr | Korafe-Yegha | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kpw | Kobon | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kpx | Mountain Koiali | Koiarian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kqa | Mum | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kqc | Doromu-Koki | Manubaran | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kqf | Kakabai | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kql | Kyenele | Yuat | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kqw | Kandas | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| krc | Karachay-Balkar | Turkic | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ksd | Kuanua | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ksj | Uare | Kwalean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ksr | Borong | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ktm | Kurti | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kto | Kuot | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kud | 'Auhelawa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kue | Kuman (Papua New Guinea) | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kup | Kunimaipa | Kunimaipan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kur | Kurdish | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| kvg | Kuni-Boazi | Anim | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kvn | Border Kuna | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kwd | Kwaio | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kwf | Kwara'ae | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kwi | Awa-Cuaiquer | Barbacoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kwj | Kwanga | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kyc | Kyaka | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kyf | Kouya | Kru | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kyg | Keyagana | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kyq | Kenga | Central Sudanic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kyz | Kayabí | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kze | Kosena | Bookkeeping | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| kzj | Coastal Kadazan | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| lac | Lacandon | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| lao | Lao | Tai-Kadai | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | -| lat | Latin | Indo-European | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| lav | Latvian | Indo-European | 1 | 2 | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| lbb | Label | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| lbk | Central Bontok | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| lcm | Tungag | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| leu | Kara (Papua New Guinea) | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| lex | Luang | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| lfn | Lingua Franca Nova | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| lgl | Wala | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| lid | Nyindrou | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| lif | Limbu | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| lij | Ligurian | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| lim | Limburgan | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| lin | Lingala | Atlantic-Congo | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| lit | Lithuanian | Indo-European | 4 | 1 | 1 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| llg | Lole | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| lmo | Lombard | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| ltg | Latgalian | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| ltz | Luxembourgish | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| lua | Luba-Lulua | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| lug | Ganda | Atlantic-Congo | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| luo | Luo (Kenya and Tanzania) | Nilotic | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | -| lus | Lushai | Sino-Tibetan | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| lvs | Standard Latvian | Unclassified | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | -| lww | Lewo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| maa | San Jerónimo Tecóatl Mazatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mad | Madurese | Austronesian | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| mag | Magahi | Indo-European | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| mai | Maithili | Indo-European | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | -| maj | Jalapa De Díaz Mazatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mak | Makasar | Austronesian | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| mal | Malayalam | Dravidian | 7 | 7 | 2 | 0 | 0 | 0 | 0 | 2 | 1 | 0 | 0 | 19 | -| mam | Mam | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| maq | Chiquihuitlán Mazatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mar | Marathi | Indo-European | 7 | 6 | 2 | 0 | 0 | 1 | 0 | 2 | 2 | 0 | 0 | 20 | -| mau | Huautla Mazatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mav | Sateré-Mawé | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| max | North Moluccan Malay | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| maz | Central Mazahua | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mbb | Western Bukidnon Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mbc | Macushi | Cariban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mbh | Mangseng | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mbj | Nadëb | Naduhup | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mbl | Maxakalí | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mbs | Sarangani Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mbt | Matigsalug Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mca | Maca | Mataguayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mcb | Machiguenga | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mcd | Sharanahua | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mcf | Matsés | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mco | Coatlán Mixe | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mcp | Makaa | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mcq | Ese | Koiarian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mcr | Menya | Angan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mdy | Male (Ethiopia) | Ta-Ne-Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| med | Melpa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mee | Mengen | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mek | Mekeo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| meq | Merey | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| met | Mato | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| meu | Motu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mey | Hassaniyya | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mgc | Morokodo | Central Sudanic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mgh | Makhuwa-Meetto | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mgw | Matumbi | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mhl | Mauwake | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mhr | Eastern Mari | Uralic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mib | Atatláhuca Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mic | Mi'kmaq | Algic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mie | Ocotepec Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mig | San Miguel El Grande Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mih | Chayuco Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mil | Peñoles Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| min | Minangkabau | Austronesian | 3 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 9 | -| mio | Pinotepa Nacional Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mir | Isthmus Mixe | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mit | Southern Puebla Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| miz | Coatzospan Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mjc | San Juan Colorado Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mkd | Macedonian | Indo-European | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | -| mkj | Mokilese | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mkl | Mokole | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mkn | Kupang Malay | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mks | Silacayoapan Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mle | Manambu | Ndu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mlg | Malagasy | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mlh | Mape | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mlp | Bargam | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mlt | Maltese | Afro-Asiatic | 2 | 2 | 2 | 0 | 2 | 0 | 0 | 1 | 0 | 0 | 0 | 9 | -| mmo | Mangga Buang | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mmx | Madak | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mna | Mbula | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mni | Manipuri | Sino-Tibetan | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | -| mon | Mongolian | Unclassified | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| mop | Mopán Maya | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mos | Mossi | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| mox | Molima | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mph | Maung | Iwaidjan Proper | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mpj | Martu Wangka | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mpm | Yosondúa Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mpp | Migabac | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mps | Dadibi | Teberan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mpt | Mian | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mpx | Misima-Panaeati | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mqb | Mbuko | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mqj | Mamasa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mri | Maori | Austronesian | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | -| msa | Malay (macrolanguage) | Unclassified | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| msb | Masbatenyo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| msc | Sankaran Maninka | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| msk | Mansaka | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| msm | Agusan Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| msy | Aruamu | Ramu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mti | Maiwa (Papua New Guinea) | Dagan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mto | Totontepec Mixe | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mui | Musi | Austronesian | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| mup | Malvi | Indo-European | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| mux | Bo-Ung | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| muy | Muyang | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mva | Manam | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mvn | Minaveha | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mwc | Are | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mwe | Mwera (Chimwera) | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mwf | Murrinh-Patha | Southern Daly | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mwp | Kala Lagaw Ya | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mwr | Marwari | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mxb | Tezoatlán Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mxp | Tlahuitoltepec Mixe | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mxq | Juquila Mixe | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mxt | Jamiltepec Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mya | Burmese | Sino-Tibetan | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 9 | -| myk | Mamara Senoufo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| myu | Mundurukú | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| myw | Muyuw | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| myy | Macuna | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| mzz | Maiadomu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nab | Southern Nambikuára | Nambiquaran | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| naf | Nabak | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nak | Nakanai | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nas | Naasioi | South Bougainville | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nbl | South Ndebele | Unclassified | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nbq | Nggem | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nca | Iyo | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nch | Central Huasteca Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ncj | Northern Puebla Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ncl | Michoacán Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ncu | Chumburung | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nde | North Ndebele | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ndg | Ndengereko | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ndj | Ndamba | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nds | Low German | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nep | Nepali (macrolanguage) | Unclassified | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| nfa | Dhao | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ngp | Ngulu | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ngu | Guerrero Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nhe | Eastern Huasteca Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nhg | Tetelcingo Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nhi | Zacatlán-Ahuacatlán-Tepetzintla Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nho | Takuu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nhr | Naro | Khoe-Kwadi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nhu | Noone | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nhw | Western Huasteca Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nhy | Northern Oaxaca Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nif | Nek | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nii | Nii | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nij | Ngaju | Austronesian | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| nin | Ninzo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nko | Nkonya | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nld | Dutch | Indo-European | 6 | 6 | 1 | 0 | 1 | 0 | 1 | 2 | 2 | 0 | 0 | 19 | -| nlg | Gela | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nna | Nyangumarta | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nno | Norwegian Nynorsk | Unclassified | 4 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 8 | -| nnq | Ngindo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| noa | Woun Meu | Chocoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nob | Norwegian Bokmål | Unclassified | 4 | 7 | 5 | 0 | 0 | 0 | 0 | 3 | 0 | 0 | 0 | 19 | -| noe | Nimadi | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nop | Numanggang | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nor | Norwegian | Indo-European | 0 | 1 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 3 | -| not | Nomatsiguenga | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nou | Ewage-Notu | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nov | Novial | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| npi | Nepali (individual language) | Indo-European | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| npl | Southeastern Puebla Nahuatl | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nqo | N'Ko | Artificial Language | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| nsn | Nehan | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nso | Pedi | Atlantic-Congo | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | -| nss | Nali | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ntj | Ngaanyatjarra | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ntp | Northern Tepehuan | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ntu | Natügu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nus | Nuer | Nilotic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| nuy | Nunggubuyu | Gunwinyguan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nvm | Namiae | Koiarian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nwi | Southwest Tanna | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nya | Nyanja | Atlantic-Congo | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | -| nys | Nyungar | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| nyu | Nyungwe | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| obo | Obo Manobo | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| oci | Occitan (post 1500) | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| okv | Orokaiva | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| omw | South Tairora | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ong | Olo | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ons | Ono | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ood | Tohono O'odham | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| opm | Oksapmin | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ori | Oriya (macrolanguage) | Unclassified | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| orm | Oromo | Unclassified | 1 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| orv | Old Russian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ory | Odia | Indo-European | 5 | 4 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 15 | -| ote | Mezquital Otomi | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| otm | Eastern Highland Otomi | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| otn | Tenango Otomi | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| otq | Querétaro Otomi | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ots | Estado de México Otomi | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pab | Parecís | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pad | Paumarí | Arawan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pag | Pangasinan | Austronesian | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| pah | Tenharim | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pam | Pampanga | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pan | Panjabi | Indo-European | 6 | 6 | 2 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 18 | -| pao | Northern Paiute | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pap | Papiamento | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| pbt | Southern Pashto | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| pcm | Nigerian Pidgin | Indo-European | 1 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | -| pes | Iranian Persian | Indo-European | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | -| pib | Yine | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pio | Piapoco | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pir | Piratapuyo | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| piu | Pintupi-Luritja | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pjt | Pitjantjatjara | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pls | San Marcos Tlacoyalco Popoloca | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| plt | Plateau Malagasy | Austronesian | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| plu | Palikúr | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pma | Paama | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pms | Piemontese | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| poe | San Juan Atzingo Popoloca | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| poh | Poqomchi' | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| poi | Highland Popoluca | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pol | Polish | Indo-European | 4 | 11 | 4 | 0 | 1 | 4 | 0 | 18 | 4 | 0 | 0 | 46 | -| pon | Pohnpeian | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| por | Portuguese | Indo-European | 4 | 9 | 1 | 0 | 2 | 2 | 1 | 5 | 3 | 0 | 0 | 27 | -| poy | Pogolo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ppo | Folopa | Teberan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| prf | Paranan | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pri | Paicî | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| prs | Dari | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| ptp | Patep | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ptu | Bambam | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| pus | Pushto | Unclassified | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| pwg | Gapapaiwa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qub | Huallaga Huánuco Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| quc | K'iche' | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| quf | Lambayeque Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| quh | South Bolivian Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qul | North Bolivian Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qup | Southern Pastaza Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| quy | Ayacucho Quechua | Quechuan | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| qvc | Cajamarca Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qve | Eastern Apurímac Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qvh | Huamalíes-Dos de Mayo Huánuco Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qvm | Margos-Yarowilca-Lauricocha Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qvn | North Junín Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qvs | San Martín Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qvw | Huaylla Wanca Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qvz | Northern Pastaza Quichua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qwh | Huaylas Ancash Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qxh | Panao Huánuco Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qxn | Northern Conchucos Ancash Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| qxo | Southern Conchucos Ancash Quechua | Quechuan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| rai | Ramoaaina | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| raj | Rajasthani | Unclassified | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| reg | Kara (Tanzania) | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| rej | Rejang | Austronesian | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| rgu | Ringgou | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| rkb | Rikbaktsa | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| rmc | Carpathian Romani | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| rmy | Vlax Romani | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| rom | Romany | Unclassified | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| ron | Romanian | Indo-European | 5 | 6 | 1 | 0 | 1 | 0 | 1 | 3 | 1 | 0 | 0 | 18 | -| roo | Rotokas | North Bougainville | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| rop | Kriol | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| row | Dela-Oenale | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| rro | Waima | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ruf | Luguru | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| rug | Roviana | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| run | Rundi | Atlantic-Congo | 1 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| rus | Russian | Indo-European | 5 | 13 | 6 | 0 | 2 | 4 | 2 | 16 | 4 | 0 | 0 | 52 | -| rwo | Rawa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sab | Buglere | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sag | Sango | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| sah | Yakut | Turkic | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| san | Sanskrit | Indo-European | 5 | 3 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 10 | -| sat | Santali | Austroasiatic | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | -| sbe | Saliba | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sbk | Safwa | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sbs | Subiya | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| scn | Sicilian | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| sco | Scots | Indo-European | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| seh | Sena | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sey | Secoya | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sgb | Mag-antsi Ayta | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sgz | Sursurunga | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| shi | Tachelhit | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| shj | Shatt | Dajuic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| shn | Shan | Tai-Kadai | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| shp | Shipibo-Conibo | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sim | Mende (Papua New Guinea) | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sin | Sinhala | Indo-European | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | -| sja | Epena | Chocoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| slk | Slovak | Indo-European | 3 | 4 | 1 | 0 | 1 | 0 | 0 | 3 | 0 | 0 | 0 | 12 | -| sll | Salt-Yui | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| slv | Slovenian | Indo-European | 3 | 4 | 1 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 10 | -| smk | Bolinao | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| smo | Samoan | Austronesian | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| sna | Shona | Atlantic-Congo | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| snc | Sinaugoro | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| snd | Sindhi | Indo-European | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| snn | Siona | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| snp | Siane | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| snx | Sam | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sny | Saniyo-Hiyewe | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| som | Somali | Afro-Asiatic | 3 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 9 | -| soq | Kanasi | Dagan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sot | Southern Sotho | Atlantic-Congo | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | -| soy | Miyobe | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| spa | Spanish | Indo-European | 4 | 13 | 4 | 0 | 1 | 2 | 2 | 13 | 4 | 0 | 0 | 43 | -| spl | Selepet | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| spm | Akukem | Ramu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| spp | Supyire Senoufo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sps | Saposa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| spy | Sabaot | Nilotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sqi | Albanian | Unclassified | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| srd | Sardinian | Unclassified | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| sri | Siriano | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| srm | Saramaccan | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| srn | Sranan Tongo | Indo-European | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| srp | Serbian | Indo-European | 4 | 1 | 1 | 0 | 0 | 0 | 1 | 2 | 0 | 0 | 0 | 9 | -| srq | Sirionó | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ssd | Siroi | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ssg | Seimat | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ssw | Swati | Atlantic-Congo | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | -| ssx | Samberigi | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| stp | Southeastern Tepehuan | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sua | Sulka | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sue | Suena | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sun | Sundanese | Austronesian | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 9 | -| sus | Susu | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| suz | Sunwar | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| svk | Slovakian Sign Language | Sign Language | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| swa | Swahili (macrolanguage) | Atlantic-Congo | 1 | 7 | 2 | 0 | 0 | 1 | 1 | 3 | 0 | 0 | 0 | 15 | -| swe | Swedish | Indo-European | 4 | 8 | 3 | 0 | 1 | 1 | 1 | 4 | 0 | 0 | 0 | 22 | -| swg | Swabian | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| swh | Swahili (individual language) | Atlantic-Congo | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | -| swp | Suau | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| sxb | Suba | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| szl | Silesian | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| tac | Lowland Tarahumara | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tah | Tahitian | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| taj | Eastern Tamang | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tam | Tamil | Dravidian | 7 | 7 | 2 | 0 | 0 | 1 | 0 | 3 | 1 | 0 | 0 | 21 | -| taq | Tamasheq | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| tat | Tatar | Turkic | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| tav | Tatuyo | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| taw | Tai | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tbc | Takia | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tbf | Mandara | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tbg | North Tairora | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tbo | Tawala | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tbz | Ditammari | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tca | Ticuna | Ticuna-Yuri | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tcs | Torres Strait Creole | Indo-European | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tcz | Thado Chin | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tdt | Tetun Dili | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tee | Huehuetla Tepehua | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tel | Telugu | Dravidian | 7 | 7 | 2 | 0 | 0 | 0 | 1 | 5 | 2 | 0 | 0 | 24 | -| ter | Tereno | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tet | Tetum | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tew | Tewa (USA) | Kiowa-Tanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tfr | Teribe | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tgk | Tajik | Indo-European | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | -| tgl | Tagalog | Austronesian | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| tgo | Sudest | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tgp | Tangoa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tha | Thai | Tai-Kadai | 4 | 8 | 1 | 0 | 0 | 1 | 1 | 6 | 0 | 0 | 0 | 21 | -| tif | Tifal | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tim | Timbe | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tir | Tigrinya | Afro-Asiatic | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| tiw | Tiwi | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tiy | Tiruray | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tke | Takwane | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tku | Upper Necaxa Totonac | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tlf | Telefol | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tmd | Haruai | Piawi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tna | Tacana | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tnc | Tanimuca-Retuarã | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tnk | Kwamera | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tnn | North Tanna | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tnp | Whitesands | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| toc | Coyutla Totonac | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tod | Toma | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tof | Gizrra | Eastern Trans-Fly | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| toj | Tojolabal | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ton | Tonga (Tonga Islands) | Austronesian | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| too | Xicotepec De Juárez Totonac | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| top | Papantla Totonac | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tos | Highland Totonac | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tpa | Taupota | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tpi | Tok Pisin | Indo-European | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | -| tpt | Tlachichilco Tepehua | Totonacan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tpz | Tinputz | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| trc | Copala Triqui | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tsn | Tswana | Atlantic-Congo | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | -| tso | Tsonga | Atlantic-Congo | 1 | 4 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | -| tsw | Tsishingini | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ttc | Tektiteko | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tte | Bwanabwana | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tuc | Mutu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tue | Tuyuca | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tuf | Central Tunebo | Chibchan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tuk | Turkmen | Turkic | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | -| tum | Tumbuka | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| tuo | Tucano | Tucanoan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tur | Turkish | Turkic | 4 | 7 | 1 | 0 | 0 | 2 | 0 | 3 | 2 | 0 | 0 | 19 | -| tvk | Southeast Ambrym | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| twi | Twi | Unclassified | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| txq | Tii | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| txu | Kayapó | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tyv | Tuvinian | Turkic | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tzj | Tz'utujil | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tzl | Talossan | Artificial Language | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| tzm | Central Atlas Tamazight | Afro-Asiatic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| tzo | Tzotzil | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ubr | Ubir | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ubu | Umbu-Ungu | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| udu | Uduk | Koman | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| uig | Uighur | Turkic | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | -| ukr | Ukrainian | Indo-European | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 8 | -| uli | Ulithian | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ulk | Meriam Mir | Eastern Trans-Fly | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| umb | Umbundu | Atlantic-Congo | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| upv | Uripiv-Wala-Rano-Atchin | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ura | Urarina | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| urb | Urubú-Kaapor | Tupian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| urd | Urdu | Indo-European | 7 | 8 | 2 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 19 | -| uri | Urim | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| urt | Urat | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| urw | Sop | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| usa | Usarufa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| usp | Uspanteco | Mayan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| uvh | Uri | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| uvl | Lote | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| uzb | Uzbek | Unclassified | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| uzn | Northern Uzbek | Turkic | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 4 | -| vec | Venetian | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| ven | Venda | Atlantic-Congo | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | -| vid | Vidunda | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| vie | Vietnamese | Austroasiatic | 5 | 6 | 1 | 0 | 0 | 1 | 0 | 5 | 0 | 0 | 0 | 18 | -| viv | Iduna | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| vmy | Ayautla Mazatec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| waj | Waffa | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wal | Wolaytta | Ta-Ne-Omotic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wap | Wapishana | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| war | Waray (Philippines) | Austronesian | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | -| wat | Kaninuwa | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wbi | Vwanji | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wbp | Warlpiri | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wed | Wedau | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wer | Weri | Kunimaipan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wim | Wik-Mungkan | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wiu | Wiru | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wiv | Vitu | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wln | Walloon | Indo-European | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wmt | Walmajarri | Pama-Nyungan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wmw | Mwani | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wnc | Wantoat | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wnu | Usan | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wol | Wolof | Atlantic-Congo | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 6 | -| wos | Hanga Hundi | Ndu | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wrk | Garrwa | Garrwan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wro | Worrorra | Worrorran | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wrs | Waris | Border | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wsk | Waskia | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wuu | Wu Chinese | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| wuv | Wuvulu-Aua | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| xav | Xavánte | Nuclear-Macro-Je | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| xbi | Kombio | Nuclear Torricelli | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| xed | Hdi | Afro-Asiatic | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| xho | Xhosa | Atlantic-Congo | 3 | 3 | 3 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 10 | -| xla | Kamula | Kamula-Elevala | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| xnn | Northern Kankanay | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| xon | Konkomba | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| xsi | Sio | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| xtd | Diuxi-Tilantongo Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| xtm | Magdalena Peñasco Mixtec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yaa | Yaminahua | Pano-Tacanan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yad | Yagua | Peba-Yagua | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yal | Yalunka | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yap | Yapese | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yaq | Yaqui | Uto-Aztecan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yby | Yaweyuha | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ycn | Yucuna | Arawakan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ydd | Eastern Yiddish | Indo-European | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | -| yid | Yiddish | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yka | Yakan | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yle | Yele | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yml | Iamalele | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yon | Yongkom | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yor | Yoruba | Atlantic-Congo | 4 | 5 | 3 | 0 | 0 | 0 | 1 | 3 | 0 | 0 | 0 | 16 | -| yrb | Yareba | Yareban | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yre | Yaouré | Mande | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yss | Yessan-Mayo | Sepik | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yue | Yue Chinese | Sino-Tibetan | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | -| yuj | Karkar-Yuri | Pauwasi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yut | Yopno | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yuw | Yau (Morobe Province) | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| yva | Yawa | Yawa-Saweru | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zaa | Sierra de Juárez Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zab | Western Tlacolula Valley Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zac | Ocotlán Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zad | Cajonos Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zai | Isthmus Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zaj | Zaramo | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zam | Miahuatlán Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zao | Ozolotepec Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zap | Zapotec | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zar | Rincón Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zas | Santo Domingo Albarradas Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zat | Tabaa Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zav | Yatzachi Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zaw | Mitla Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zca | Coatecas Altas Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zga | Kinga | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zho | Chinese | Unclassified | 2 | 2 | 1 | 0 | 0 | 1 | 1 | 13 | 0 | 0 | 0 | 20 | -| zia | Zia | Nuclear Trans New Guinea | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ziw | Zigula | Atlantic-Congo | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zlm | Malay (individual language) | Austronesian | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zos | Francisco León Zoque | Mixe-Zoque | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zpc | Choapan Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zpl | Lachixío Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zpm | Mixtepec Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zpo | Amatlán Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zpq | Zoogocho Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zpu | Yalálag Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zpv | Chichicapan Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zpz | Texmelucan Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zsm | Standard Malay | Austronesian | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 5 | -| zsr | Southern Rincon Zapotec | Unclassified | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| ztq | Quioquitani-Quierí Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zty | Yatee Zapotec | Otomanguean | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| zul | Zulu | Atlantic-Congo | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 7 | -| zyp | Zyphe Chin | Sino-Tibetan | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | -| Total | None | None | None | 1394 | 795 | 304 | 3 | 28 | 67 | 51 | 473 | 85 | 2 | 2 | ->>>>>>> main +| ISO Code | Language | Family | Any2AnyMultiChoice | Any2AnyRetrieval | Any2TextMutipleChoice | BitextMining | Classification | Clustering | ImageClassification | ImageClustering | ImageMultilabelClassification | ImageTextPairClassification | InstructionRetrieval | MultilabelClassification | PairClassification | Reranking | Retrieval | STS | Speed | Summarization | VisualSTS | ZeroShotClassification | Sum | +|---|------|------|------|------|------|------|------|------|------|------|------|------|------|------|------|------|------|------|------|------|---| +| aai | Arifama-Miniafia | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aak | Ankave | Angan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aau | Abau | Sepik | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aaz | Amarasi | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| abs | Ambonese Malay | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| abt | Ambulas | Ndu | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| abx | Inabaknon | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aby | Aneme Wake | Yareban | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ace | Achinese | Austronesian | 0 | 0 | 0 | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| acf | Saint Lucian Creole French | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| acm | Mesopotamian Arabic | Afro-Asiatic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| acq | Ta'izzi-Adeni Arabic | Afro-Asiatic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| acr | Achi | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| acu | Achuar-Shiwiar | Chicham | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| adz | Adzera | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aeb | Tunisian Arabic | Afro-Asiatic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| aer | Eastern Arrernte | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aey | Amele | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| afr | Afrikaans | Indo-European | 0 | 0 | 0 | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 10 | +| agd | Agarabi | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agg | Angor | Senagi | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agm | Angaataha | Angan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agn | Agutaynen | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agr | Aguaruna | Chicham | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agt | Central Cagayan Agta | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| agu | Aguacateco | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aia | Arosi | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aii | Assyrian Neo-Aramaic | Afro-Asiatic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ajp | South Levantine Arabic | Unclassified | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| aka | Akan | Atlantic-Congo | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| ake | Akawaio | Cariban | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| alp | Alune | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| alq | Algonquin | Algic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| als | Tosk Albanian | Indo-European | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 5 | +| aly | Alyawarr | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ame | Yanesha' | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amf | Hamer-Banna | South Omotic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amh | Amharic | Afro-Asiatic | 0 | 0 | 0 | 3 | 6 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 14 | +| amk | Ambai | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amm | Ama (Papua New Guinea) | Left May | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amn | Amanab | Border | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amo | Amo | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amp | Alamblak | Sepik | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amr | Amarakaeri | Harakmbut | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amu | Guerrero Amuzgo | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| amx | Anmatyerre | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ang | Old English (ca. 450-1100) | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| anh | Nend | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| anp | Angika | Indo-European | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| anv | Denya | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aoi | Anindilyakwa | Gunwinyguan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aoj | Mufian | Nuclear Torricelli | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aom | Ömie | Koiarian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aon | Bumbita Arapesh | Nuclear Torricelli | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apb | Sa'a | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apc | Levantine Arabic | Afro-Asiatic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| ape | Bukiyip | Nuclear Torricelli | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apn | Apinayé | Nuclear-Macro-Je | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apr | Arop-Lokep | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apu | Apurinã | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apw | Western Apache | Athabaskan-Eyak-Tlingit | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| apz | Safeyoka | Angan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ara | Arabic | Unclassified | 0 | 2 | 0 | 2 | 12 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | 2 | 9 | 2 | 0 | 0 | 1 | 0 | 32 | +| arb | Standard Arabic | Afro-Asiatic | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | 1 | 0 | 0 | 0 | 0 | 8 | +| are | Western Arrarnta | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| arl | Arabela | Zaparoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| arn | Mapudungun | Araucanian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| arp | Arapaho | Algic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| arq | Algerian Arabic | Afro-Asiatic | 0 | 0 | 0 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 4 | +| ars | Najdi Arabic | Afro-Asiatic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| ary | Moroccan Arabic | Afro-Asiatic | 0 | 0 | 0 | 1 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 7 | +| arz | Egyptian Arabic | Afro-Asiatic | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 5 | +| asm | Assamese | Indo-European | 0 | 0 | 0 | 5 | 3 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 0 | 0 | 14 | +| aso | Dano | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ast | Asturian | Indo-European | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| ata | Pele-Ata | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| atb | Zaiwa | Sino-Tibetan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| atd | Ata Manobo | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| atg | Ivbie North-Okpela-Arhe | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| att | Pamplona Atta | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| auc | Waorani | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| aui | Anuki | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| auy | Awiyaana | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| avt | Au | Nuclear Torricelli | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| awa | Awadhi | Indo-European | 0 | 0 | 0 | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| awb | Awa (Papua New Guinea) | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| awk | Awabakal | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| awx | Awara | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ayr | Central Aymara | Aymaran | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| azb | South Azerbaijani | Turkic | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| aze | Azerbaijani | Unclassified | 0 | 0 | 0 | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| azg | San Pedro Amuzgos Amuzgo | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| azj | North Azerbaijani | Turkic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| azz | Highland Puebla Nahuatl | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bak | Bashkir | Turkic | 0 | 0 | 0 | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| bam | Bambara | Mande | 0 | 0 | 0 | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 5 | +| ban | Balinese | Austronesian | 0 | 0 | 0 | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| bao | Waimaha | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bba | Baatonum | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bbb | Barai | Koiarian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bbc | Batak Toba | Austronesian | 0 | 0 | 0 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| bbr | Girawa | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bch | Bariai | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bco | Kaluli | Bosavi | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bdd | Bunama | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bea | Beaver | Athabaskan-Eyak-Tlingit | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bef | Benabena | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bel | Belarusian | Indo-European | 0 | 0 | 0 | 4 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| bem | Bemba (Zambia) | Atlantic-Congo | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| ben | Bengali | Indo-European | 0 | 1 | 0 | 7 | 9 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 2 | 6 | 1 | 0 | 0 | 0 | 0 | 29 | +| beo | Beami | Bosavi | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ber | Berber (Other) | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| beu | Blagar | Timor-Alor-Pantar | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bew | Betawi | Austronesian | 0 | 0 | 0 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| bgc | Haryanvi | Indo-European | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| bgs | Tagabawa | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bgt | Bughotu | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bhb | Bhili | Indo-European | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bhd | Bhadrawahi | Indo-European | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bhg | Binandere | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bhl | Bimin | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bho | Bhojpuri | Indo-European | 0 | 0 | 0 | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| bhp | Bima | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| big | Biangai | Kunimaipan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjj | Kanauji | Indo-European | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjk | Barok | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjn | Banjar | Austronesian | 0 | 0 | 0 | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| bjp | Fanamaket | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjr | Binumarien | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjv | Bedjond | Central Sudanic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bjz | Baruga | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bkd | Binukid | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bki | Baki | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bkq | Bakairí | Cariban | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bkx | Baikeno | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| blw | Balangao | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| blz | Balantak | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bmh | Kein | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bmk | Ghayavi | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bmr | Muinane | Boran | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bmu | Somba-Siawari | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bnp | Bola | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bns | Bundeli | Indo-European | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| boa | Bora | Boran | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bod | Tibetan | Sino-Tibetan | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| boj | Anjam | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bon | Bine | Eastern Trans-Fly | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bos | Bosnian | Indo-European | 0 | 0 | 0 | 3 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| box | Buamu | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| boy | Bodo (Central African Republic) | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bpr | Koronadal Blaan | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bps | Sarangani Blaan | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bqc | Boko (Benin) | Mande | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bqp | Busa | Mande | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bra | Braj | Indo-European | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bre | Breton | Indo-European | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| brx | Bodo (India) | Sino-Tibetan | 0 | 0 | 0 | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| bsj | Bangwinji | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bsn | Barasana-Eduria | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bsp | Baga Sitemu | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bss | Akoose | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bug | Buginese | Austronesian | 0 | 0 | 0 | 2 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| buk | Bugawac | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bul | Bulgarian | Indo-European | 0 | 1 | 0 | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 14 | +| bus | Bokobaru | Mande | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bvd | Baeggu | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bvr | Burarra | Maningrida | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bxh | Buhutu | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| byr | Baruya | Angan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| byx | Qaqet | Baining | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bzd | Bribri | Chibchan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bzh | Mapos Buang | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| bzj | Belize Kriol English | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| caa | Chortí | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cab | Garifuna | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cac | Chuj | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| caf | Southern Carrier | Athabaskan-Eyak-Tlingit | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cak | Kaqchikel | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cao | Chácobo | Pano-Tacanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cap | Chipaya | Uru-Chipaya | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| car | Galibi Carib | Cariban | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cat | Catalan | Indo-European | 0 | 0 | 0 | 3 | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| cav | Cavineña | Pano-Tacanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cax | Chiquitano | Chiquitano | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbc | Carapana | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbi | Chachi | Barbacoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbk | Chavacano | Indo-European | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| cbr | Cashibo-Cacataibo | Pano-Tacanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbs | Cashinahua | Pano-Tacanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbt | Chayahuita | Cahuapanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbu | Candoshi-Shapra | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cbv | Cacua | Kakua-Nukak | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cco | Comaltepec Chinantec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ceb | Cebuano | Austronesian | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| cek | Eastern Khumi Chin | Sino-Tibetan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ces | Czech | Indo-European | 0 | 1 | 0 | 4 | 5 | 2 | 0 | 0 | 0 | 0 | 0 | 1 | 2 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 18 | +| cgc | Kagayanen | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cha | Chamorro | Austronesian | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| chd | Highland Oaxaca Chontal | Tequistlatecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| chf | Tabasco Chontal | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| chk | Chuukese | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| chq | Quiotepec Chinantec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| chv | Chuvash | Turkic | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| chz | Ozumacín Chinantec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cjk | Chokwe | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| cjo | Ashéninka Pajonal | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cjv | Chuave | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ckb | Central Kurdish | Indo-European | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| cle | Lealao Chinantec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| clu | Caluyanun | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cme | Cerma | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cmn | Mandarin Chinese | Sino-Tibetan | 0 | 0 | 0 | 4 | 10 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | 4 | 10 | 9 | 0 | 0 | 1 | 0 | 45 | +| cmo | Central Mnong | Austroasiatic | 0 | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| cni | Asháninka | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cnl | Lalana Chinantec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cnt | Tepetotutla Chinantec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| code | unknown | Unclassified | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | 37 | 0 | 0 | 0 | 0 | 0 | 41 | +| cof | Colorado | Barbacoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| con | Cofán | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cop | Coptic | Afro-Asiatic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cor | Cornish | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cot | Caquinte | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cpa | Palantla Chinantec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cpb | Ucayali-Yurúa Ashéninka | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cpc | Ajyíninka Apurucayali | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cpu | Pichis Ashéninka | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cpy | South Ucayali Ashéninka | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| crh | Crimean Tatar | Turkic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| crn | El Nayar Cora | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| crx | Carrier | Athabaskan-Eyak-Tlingit | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| csb | Kashubian | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cso | Sochiapam Chinantec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| csy | Siyin Chin | Sino-Tibetan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cta | Tataltepec Chatino | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cth | Thaiphum Chin | Bookkeeping | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ctp | Western Highland Chatino | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ctu | Chol | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cub | Cubeo | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cuc | Usila Chinantec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cui | Cuiba | Guahiboan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cuk | San Blas Kuna | Chibchan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cut | Teutila Cuicatec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cux | Tepeuxila Cuicatec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cwe | Kwere | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cya | Nopala Chatino | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| cym | Welsh | Indo-European | 0 | 0 | 0 | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 8 | +| daa | Dangaléat | Afro-Asiatic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dad | Marik | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dah | Gwahatike | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dan | Danish | Indo-European | 0 | 2 | 0 | 5 | 9 | 2 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 1 | 5 | 0 | 0 | 0 | 0 | 0 | 25 | +| ded | Dedua | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| deu | German | Indo-European | 0 | 2 | 0 | 6 | 14 | 7 | 0 | 0 | 0 | 0 | 0 | 1 | 7 | 2 | 18 | 4 | 0 | 0 | 2 | 0 | 63 | +| dgc | Casiguran Dumagat Agta | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dgr | Dogrib | Athabaskan-Eyak-Tlingit | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dgz | Daga | Dagan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dhg | Dhangu-Djangu | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dif | Dieri | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dik | Southwestern Dinka | Nilotic | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| div | Dhivehi | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dji | Djinang | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| djk | Eastern Maroon Creole | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| djr | Djambarrpuyngu | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dob | Dobu | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| doi | Dogri (macrolanguage) | Unclassified | 0 | 0 | 0 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| dop | Lukpa | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dov | Dombe | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dsb | Lower Sorbian | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dtp | Kadazan Dusun | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dwr | Dawro | Ta-Ne-Omotic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dww | Dawawa | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dwy | Dhuwaya | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dyu | Dyula | Mande | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| dza | Tunzu | Atlantic-Congo | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| dzo | Dzongkha | Sino-Tibetan | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| ebk | Eastern Bontok | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| eko | Koti | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ell | Modern Greek (1453-) | Indo-European | 0 | 2 | 0 | 3 | 6 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 2 | 0 | 3 | 0 | 0 | 0 | 0 | 0 | 18 | +| emi | Mussau-Emira | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| emp | Northern Emberá | Chocoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| eng | English | Indo-European | 9 | 62 | 4 | 17 | 160 | 18 | 21 | 5 | 1 | 6 | 3 | 1 | 13 | 8 | 108 | 13 | 2 | 1 | 7 | 24 | 483 | +| enq | Enga | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| epo | Esperanto | Artificial Language | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| eri | Ogea | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ese | Ese Ejja | Pano-Tacanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| esk | Northwest Alaska Inupiatun | Eskimo-Aleut | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| est | Estonian | Uralic | 0 | 1 | 0 | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 9 | +| etr | Edolo | Bosavi | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| eus | Basque | Unclassified | 0 | 0 | 0 | 3 | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| ewe | Ewe | Atlantic-Congo | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| faa | Fasu | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fai | Faiwol | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fao | Faroese | Indo-European | 0 | 0 | 0 | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 7 | +| far | Fataleka | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fas | Persian | Indo-European | 0 | 1 | 0 | 4 | 28 | 5 | 0 | 0 | 0 | 0 | 0 | 0 | 8 | 2 | 40 | 3 | 0 | 0 | 0 | 0 | 91 | +| ffm | Maasina Fulfulde | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fij | Fijian | Austronesian | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| fil | Filipino | Austronesian | 0 | 1 | 0 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| fin | Finnish | Uralic | 0 | 1 | 0 | 3 | 5 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 2 | 5 | 1 | 0 | 0 | 0 | 0 | 20 | +| fon | Fon | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| for | Fore | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fra | French | Indo-European | 0 | 1 | 0 | 7 | 13 | 8 | 0 | 0 | 0 | 0 | 0 | 1 | 6 | 3 | 15 | 4 | 0 | 1 | 2 | 0 | 61 | +| fry | Western Frisian | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fuc | Pulaar | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fue | Borgu Fulfulde | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fuf | Pular | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fuh | Western Niger Fulfulde | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| fur | Friulian | Indo-European | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| fuv | Nigerian Fulfulde | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| gah | Alekano | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gai | Borei | Ramu | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gam | Kandawo | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gaw | Nobonob | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gaz | West Central Oromo | Afro-Asiatic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| gbm | Garhwali | Indo-European | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| gdn | Umanakaina | Dagan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gdr | Wipi | Eastern Trans-Fly | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| geb | Kire | Ramu | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gfk | Patpatar | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ghs | Guhu-Samane | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gla | Scottish Gaelic | Indo-European | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| gle | Irish | Indo-European | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| glg | Galician | Indo-European | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| glk | Gilaki | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| glv | Manx | Indo-European | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gmv | Gamo | Ta-Ne-Omotic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gng | Ngangam | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gnn | Gumatj | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gnw | Western Bolivian Guaraní | Tupian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gof | Gofa | Ta-Ne-Omotic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gom | Goan Konkani | Indo-European | 0 | 0 | 0 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| grc | Ancient Greek (to 1453) | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| grn | Guarani | Unclassified | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| gsw | Swiss German | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gub | Guajajára | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| guh | Guahibo | Guahiboan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gui | Eastern Bolivian Guaraní | Tupian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| guj | Gujarati | Indo-European | 0 | 0 | 0 | 6 | 6 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 0 | 0 | 18 | +| gul | Sea Island Creole English | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gum | Guambiano | Barbacoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gun | Mbyá Guaraní | Tupian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| guo | Guayabero | Guahiboan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gup | Gunwinggu | Gunwinyguan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gux | Gourmanchéma | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gvc | Guanano | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gvf | Golin | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gvn | Kuku-Yalanji | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gvs | Gumawana | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gwi | Gwichʼin | Athabaskan-Eyak-Tlingit | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gym | Ngäbere | Chibchan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| gyr | Guarayu | Tupian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hat | Haitian | Indo-European | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 5 | +| hau | Hausa | Afro-Asiatic | 0 | 0 | 0 | 4 | 5 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 14 | +| haw | Hawaiian | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hbo | Ancient Hebrew | Afro-Asiatic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hch | Huichol | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| heb | Hebrew | Afro-Asiatic | 0 | 1 | 0 | 4 | 5 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 12 | +| heg | Helong | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hin | Hindi | Indo-European | 0 | 1 | 0 | 9 | 12 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | 2 | 10 | 2 | 0 | 0 | 0 | 0 | 40 | +| hix | Hixkaryána | Cariban | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hla | Halia | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hlt | Matu Chin | Sino-Tibetan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hmn | Hmong | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hmo | Hiri Motu | Pidgin | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hne | Chhattisgarhi | Indo-European | 0 | 0 | 0 | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| hns | Caribbean Hindustani | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hop | Hopi | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hot | Hote | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hrv | Croatian | Indo-European | 0 | 1 | 0 | 4 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 11 | +| hsb | Upper Sorbian | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hto | Minica Huitoto | Huitotoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hub | Huambisa | Chicham | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hui | Huli | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hun | Hungarian | Uralic | 0 | 1 | 0 | 5 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 13 | +| hus | Huastec | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| huu | Murui Huitoto | Huitotoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| huv | San Mateo Del Mar Huave | Huavean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hvn | Sabu | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| hye | Armenian | Indo-European | 0 | 0 | 0 | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 9 | +| ian | Iatmul | Ndu | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ibo | Igbo | Atlantic-Congo | 0 | 0 | 0 | 3 | 5 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 12 | +| ido | Ido | Artificial Language | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ign | Ignaciano | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ikk | Ika | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ikw | Ikwere | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ile | Interlingue | Artificial Language | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ilo | Iloko | Austronesian | 0 | 0 | 0 | 2 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| imo | Imbongu | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ina | Interlingua (International Auxiliary Language Association) | Artificial Language | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| inb | Inga | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ind | Indonesian | Austronesian | 0 | 3 | 0 | 6 | 7 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 4 | 1 | 0 | 0 | 0 | 0 | 24 | +| ino | Inoke-Yate | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| iou | Tuma-Irumu | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ipi | Ipili | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| isl | Icelandic | Indo-European | 0 | 0 | 0 | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 9 | +| isn | Isanzu | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ita | Italian | Indo-European | 0 | 1 | 0 | 5 | 9 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 2 | 1 | 5 | 3 | 0 | 0 | 2 | 0 | 30 | +| iws | Sepik Iwam | Sepik | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ixl | Ixil | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jac | Popti' | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jae | Yabem | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jao | Yanyuwa | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jav | Javanese | Austronesian | 0 | 0 | 0 | 4 | 7 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 13 | +| jic | Tol | Jicaquean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jid | Bu (Kaduna State) | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jiv | Shuar | Chicham | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jni | Janji | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| jpn | Japanese | Japonic | 0 | 3 | 0 | 5 | 8 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | 3 | 13 | 2 | 0 | 0 | 0 | 0 | 39 | +| jvn | Caribbean Javanese | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kab | Kabyle | Afro-Asiatic | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| kac | Kachin | Sino-Tibetan | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| kam | Kamba (Kenya) | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kan | Kannada | Dravidian | 0 | 0 | 0 | 6 | 7 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 0 | 0 | 19 | +| kaq | Capanahua | Pano-Tacanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kas | Kashmiri | Indo-European | 0 | 0 | 0 | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| kat | Georgian | Kartvelian | 0 | 0 | 0 | 4 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 10 | +| kaz | Kazakh | Turkic | 0 | 0 | 0 | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| kbc | Kadiwéu | Guaicuruan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kbh | Camsá | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kbm | Iwal | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kbp | Kabiyè | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kbq | Kamano | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kdc | Kutu | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kde | Makonde | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kdl | Tsikimba | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kea | Kabuverdianu | Indo-European | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| kek | Kekchí | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ken | Kenyang | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kew | West Kewa | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kfg | Kudiya | Dravidian | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kfy | Kumaoni | Indo-European | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kgf | Kube | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kgk | Kaiwá | Tupian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kgp | Kaingang | Nuclear-Macro-Je | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| khk | Halh Mongolian | Mongolic-Khitan | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| khm | Khmer | Austroasiatic | 0 | 0 | 0 | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| khs | Kasua | Bosavi | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| khz | Keapara | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kik | Kikuyu | Atlantic-Congo | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| kin | Kinyarwanda | Atlantic-Congo | 0 | 0 | 0 | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 8 | +| kir | Kirghiz | Turkic | 0 | 0 | 0 | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 7 | +| kiw | Northeast Kiwai | Kiwaian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kiz | Kisi | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kje | Kisar | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kjs | East Kewa | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kkc | Odoodee | East Strickland | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kkl | Kosarek Yale | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| klt | Nukna | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| klv | Maskelynes | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmb | Kimbundu | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kmg | Kâte | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmh | Kalam | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmk | Limos Kalinga | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmo | Kwoma | Sepik | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmr | Northern Kurdish | Indo-European | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| kms | Kamasau | Nuclear Torricelli | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kmu | Kanite | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| knc | Central Kanuri | Saharan | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kne | Kankanaey | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| knf | Mankanya | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| knj | Western Kanjobal | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| knv | Tabo | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kon | Kongo | Unclassified | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kor | Korean | Koreanic | 0 | 2 | 0 | 4 | 8 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 3 | 1 | 9 | 3 | 0 | 0 | 1 | 0 | 33 | +| kos | Kosraean | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpf | Komba | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpg | Kapingamarangi | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpj | Karajá | Nuclear-Macro-Je | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpr | Korafe-Yegha | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpw | Kobon | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kpx | Mountain Koiali | Koiarian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kqa | Mum | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kqc | Doromu-Koki | Manubaran | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kqf | Kakabai | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kql | Kyenele | Yuat | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kqw | Kandas | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| krc | Karachay-Balkar | Turkic | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ksd | Kuanua | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ksj | Uare | Kwalean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ksr | Borong | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ktm | Kurti | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kto | Kuot | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kud | 'Auhelawa | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kue | Kuman (Papua New Guinea) | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kup | Kunimaipa | Kunimaipan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kur | Kurdish | Unclassified | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| kvg | Kuni-Boazi | Anim | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kvn | Border Kuna | Chibchan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kwd | Kwaio | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kwf | Kwara'ae | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kwi | Awa-Cuaiquer | Barbacoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kwj | Kwanga | Sepik | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kyc | Kyaka | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kyf | Kouya | Kru | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kyg | Keyagana | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kyq | Kenga | Central Sudanic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kyz | Kayabí | Tupian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kze | Kosena | Bookkeeping | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| kzj | Coastal Kadazan | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lac | Lacandon | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lao | Lao | Tai-Kadai | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 5 | +| lat | Latin | Indo-European | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| lav | Latvian | Indo-European | 0 | 0 | 0 | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| lbb | Label | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lbk | Central Bontok | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lcm | Tungag | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| leu | Kara (Papua New Guinea) | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lex | Luang | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lfn | Lingua Franca Nova | Artificial Language | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lgl | Wala | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lid | Nyindrou | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lif | Limbu | Sino-Tibetan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lij | Ligurian | Indo-European | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| lim | Limburgan | Indo-European | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| lin | Lingala | Atlantic-Congo | 0 | 0 | 0 | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| lit | Lithuanian | Indo-European | 0 | 0 | 0 | 4 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| llg | Lole | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| lmo | Lombard | Indo-European | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| ltg | Latgalian | Unclassified | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| ltz | Luxembourgish | Indo-European | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| lua | Luba-Lulua | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| lug | Ganda | Atlantic-Congo | 0 | 0 | 0 | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| luo | Luo (Kenya and Tanzania) | Nilotic | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 5 | +| lus | Lushai | Sino-Tibetan | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| lvs | Standard Latvian | Unclassified | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 5 | +| lww | Lewo | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| maa | San Jerónimo Tecóatl Mazatec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mad | Madurese | Austronesian | 0 | 0 | 0 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| mag | Magahi | Indo-European | 0 | 0 | 0 | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| mai | Maithili | Indo-European | 0 | 0 | 0 | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| maj | Jalapa De Díaz Mazatec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mak | Makasar | Austronesian | 0 | 0 | 0 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| mal | Malayalam | Dravidian | 0 | 0 | 0 | 7 | 7 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | 1 | 0 | 0 | 0 | 0 | 19 | +| mam | Mam | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| maq | Chiquihuitlán Mazatec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mar | Marathi | Indo-European | 0 | 0 | 0 | 7 | 6 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 2 | 2 | 0 | 0 | 0 | 0 | 20 | +| mau | Huautla Mazatec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mav | Sateré-Mawé | Tupian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| max | North Moluccan Malay | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| maz | Central Mazahua | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbb | Western Bukidnon Manobo | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbc | Macushi | Cariban | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbh | Mangseng | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbj | Nadëb | Naduhup | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbl | Maxakalí | Nuclear-Macro-Je | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbs | Sarangani Manobo | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mbt | Matigsalug Manobo | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mca | Maca | Mataguayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcb | Machiguenga | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcd | Sharanahua | Pano-Tacanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcf | Matsés | Pano-Tacanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mco | Coatlán Mixe | Mixe-Zoque | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcp | Makaa | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcq | Ese | Koiarian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mcr | Menya | Angan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mdy | Male (Ethiopia) | Ta-Ne-Omotic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| med | Melpa | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mee | Mengen | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mek | Mekeo | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| meq | Merey | Afro-Asiatic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| met | Mato | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| meu | Motu | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mey | Hassaniyya | Afro-Asiatic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mgc | Morokodo | Central Sudanic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mgh | Makhuwa-Meetto | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mgw | Matumbi | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mhl | Mauwake | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mhr | Eastern Mari | Uralic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mib | Atatláhuca Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mic | Mi'kmaq | Algic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mie | Ocotepec Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mig | San Miguel El Grande Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mih | Chayuco Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mil | Peñoles Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| min | Minangkabau | Austronesian | 0 | 0 | 0 | 3 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 9 | +| mio | Pinotepa Nacional Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mir | Isthmus Mixe | Mixe-Zoque | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mit | Southern Puebla Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| miz | Coatzospan Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mjc | San Juan Colorado Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mkd | Macedonian | Indo-European | 0 | 0 | 0 | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 7 | +| mkj | Mokilese | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mkl | Mokole | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mkn | Kupang Malay | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mks | Silacayoapan Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mle | Manambu | Ndu | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mlg | Malagasy | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mlh | Mape | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mlp | Bargam | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mlt | Maltese | Afro-Asiatic | 0 | 0 | 0 | 2 | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 2 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 9 | +| mmo | Mangga Buang | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mmx | Madak | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mna | Mbula | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mni | Manipuri | Sino-Tibetan | 0 | 0 | 0 | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| mon | Mongolian | Unclassified | 0 | 0 | 0 | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| mop | Mopán Maya | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mos | Mossi | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| mox | Molima | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mph | Maung | Iwaidjan Proper | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mpj | Martu Wangka | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mpm | Yosondúa Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mpp | Migabac | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mps | Dadibi | Teberan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mpt | Mian | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mpx | Misima-Panaeati | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mqb | Mbuko | Afro-Asiatic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mqj | Mamasa | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mri | Maori | Austronesian | 0 | 1 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| msa | Malay (macrolanguage) | Unclassified | 0 | 0 | 0 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| msb | Masbatenyo | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| msc | Sankaran Maninka | Mande | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| msk | Mansaka | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| msm | Agusan Manobo | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| msy | Aruamu | Ramu | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mti | Maiwa (Papua New Guinea) | Dagan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mto | Totontepec Mixe | Mixe-Zoque | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mui | Musi | Austronesian | 0 | 0 | 0 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| mup | Malvi | Indo-European | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| mux | Bo-Ung | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| muy | Muyang | Afro-Asiatic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mva | Manam | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mvn | Minaveha | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mwc | Are | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mwe | Mwera (Chimwera) | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mwf | Murrinh-Patha | Southern Daly | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mwp | Kala Lagaw Ya | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mwr | Marwari | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mxb | Tezoatlán Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mxp | Tlahuitoltepec Mixe | Mixe-Zoque | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mxq | Juquila Mixe | Mixe-Zoque | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mxt | Jamiltepec Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mya | Burmese | Sino-Tibetan | 0 | 0 | 0 | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 9 | +| myk | Mamara Senoufo | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| myu | Mundurukú | Tupian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| myw | Muyuw | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| myy | Macuna | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| mzz | Maiadomu | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nab | Southern Nambikuára | Nambiquaran | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| naf | Nabak | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nak | Nakanai | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nas | Naasioi | South Bougainville | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nbl | South Ndebele | Unclassified | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nbq | Nggem | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nca | Iyo | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nch | Central Huasteca Nahuatl | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ncj | Northern Puebla Nahuatl | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ncl | Michoacán Nahuatl | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ncu | Chumburung | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nde | North Ndebele | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ndg | Ndengereko | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ndj | Ndamba | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nds | Low German | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nep | Nepali (macrolanguage) | Unclassified | 0 | 0 | 0 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| nfa | Dhao | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ngp | Ngulu | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ngu | Guerrero Nahuatl | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhe | Eastern Huasteca Nahuatl | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhg | Tetelcingo Nahuatl | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhi | Zacatlán-Ahuacatlán-Tepetzintla Nahuatl | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nho | Takuu | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhr | Naro | Khoe-Kwadi | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhu | Noone | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhw | Western Huasteca Nahuatl | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nhy | Northern Oaxaca Nahuatl | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nif | Nek | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nii | Nii | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nij | Ngaju | Austronesian | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| nin | Ninzo | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nko | Nkonya | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nld | Dutch | Indo-European | 0 | 1 | 0 | 6 | 6 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 2 | 2 | 0 | 0 | 2 | 0 | 23 | +| nlg | Gela | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nna | Nyangumarta | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nno | Norwegian Nynorsk | Unclassified | 0 | 0 | 0 | 4 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 8 | +| nnq | Ngindo | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| noa | Woun Meu | Chocoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nob | Norwegian Bokmål | Unclassified | 0 | 0 | 0 | 4 | 7 | 5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | 0 | 0 | 0 | 0 | 0 | 19 | +| noe | Nimadi | Indo-European | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nop | Numanggang | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nor | Norwegian | Indo-European | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| not | Nomatsiguenga | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nou | Ewage-Notu | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nov | Novial | Artificial Language | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| npi | Nepali (individual language) | Indo-European | 0 | 0 | 0 | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| npl | Southeastern Puebla Nahuatl | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nqo | N'Ko | Artificial Language | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| nsn | Nehan | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nso | Pedi | Atlantic-Congo | 0 | 0 | 0 | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| nss | Nali | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ntj | Ngaanyatjarra | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ntp | Northern Tepehuan | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ntu | Natügu | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nus | Nuer | Nilotic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| nuy | Nunggubuyu | Gunwinyguan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nvm | Namiae | Koiarian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nwi | Southwest Tanna | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nya | Nyanja | Atlantic-Congo | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| nys | Nyungar | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| nyu | Nyungwe | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| obo | Obo Manobo | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| oci | Occitan (post 1500) | Indo-European | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| okv | Orokaiva | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| omw | South Tairora | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ong | Olo | Nuclear Torricelli | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ons | Ono | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ood | Tohono O'odham | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| opm | Oksapmin | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ori | Oriya (macrolanguage) | Unclassified | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| orm | Oromo | Unclassified | 0 | 0 | 0 | 1 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| orv | Old Russian | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ory | Odia | Indo-European | 0 | 0 | 0 | 5 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 0 | 0 | 15 | +| ote | Mezquital Otomi | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| otm | Eastern Highland Otomi | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| otn | Tenango Otomi | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| otq | Querétaro Otomi | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ots | Estado de México Otomi | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pab | Parecís | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pad | Paumarí | Arawan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pag | Pangasinan | Austronesian | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| pah | Tenharim | Tupian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pam | Pampanga | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pan | Panjabi | Indo-European | 0 | 0 | 0 | 6 | 6 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 2 | 1 | 0 | 0 | 0 | 0 | 18 | +| pao | Northern Paiute | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pap | Papiamento | Indo-European | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| pbt | Southern Pashto | Indo-European | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| pcm | Nigerian Pidgin | Indo-European | 0 | 0 | 0 | 1 | 4 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| pes | Iranian Persian | Indo-European | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| pib | Yine | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pio | Piapoco | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pir | Piratapuyo | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| piu | Pintupi-Luritja | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pjt | Pitjantjatjara | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pls | San Marcos Tlacoyalco Popoloca | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| plt | Plateau Malagasy | Austronesian | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| plu | Palikúr | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pma | Paama | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pms | Piemontese | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| poe | San Juan Atzingo Popoloca | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| poh | Poqomchi' | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| poi | Highland Popoluca | Mixe-Zoque | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pol | Polish | Indo-European | 0 | 1 | 0 | 4 | 11 | 4 | 0 | 0 | 0 | 0 | 0 | 1 | 4 | 0 | 18 | 4 | 0 | 0 | 1 | 0 | 48 | +| pon | Pohnpeian | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| por | Portuguese | Indo-European | 0 | 1 | 0 | 4 | 9 | 1 | 0 | 0 | 0 | 0 | 0 | 2 | 3 | 1 | 5 | 3 | 0 | 0 | 1 | 0 | 30 | +| poy | Pogolo | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ppo | Folopa | Teberan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| prf | Paranan | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pri | Paicî | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| prs | Dari | Indo-European | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| ptp | Patep | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ptu | Bambam | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| pus | Pushto | Unclassified | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| pwg | Gapapaiwa | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qub | Huallaga Huánuco Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| quc | K'iche' | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| quf | Lambayeque Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| quh | South Bolivian Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qul | North Bolivian Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qup | Southern Pastaza Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| quy | Ayacucho Quechua | Quechuan | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| quz | Cusco Quechua | Quechuan | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvc | Cajamarca Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qve | Eastern Apurímac Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvh | Huamalíes-Dos de Mayo Huánuco Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvm | Margos-Yarowilca-Lauricocha Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvn | North Junín Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvs | San Martín Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvw | Huaylla Wanca Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qvz | Northern Pastaza Quichua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qwh | Huaylas Ancash Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qxh | Panao Huánuco Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qxn | Northern Conchucos Ancash Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| qxo | Southern Conchucos Ancash Quechua | Quechuan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rai | Ramoaaina | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| raj | Rajasthani | Unclassified | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| reg | Kara (Tanzania) | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rej | Rejang | Austronesian | 0 | 0 | 0 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| rgu | Ringgou | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rkb | Rikbaktsa | Nuclear-Macro-Je | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rmc | Carpathian Romani | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rmy | Vlax Romani | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rom | Romany | Unclassified | 0 | 0 | 0 | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| ron | Romanian | Indo-European | 0 | 1 | 0 | 5 | 6 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 1 | 3 | 1 | 0 | 0 | 0 | 0 | 19 | +| roo | Rotokas | North Bougainville | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rop | Kriol | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| row | Dela-Oenale | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rro | Waima | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ruf | Luguru | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| rug | Roviana | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| run | Rundi | Atlantic-Congo | 0 | 0 | 0 | 1 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| rus | Russian | Indo-European | 0 | 2 | 0 | 5 | 13 | 6 | 0 | 0 | 0 | 0 | 0 | 2 | 4 | 2 | 16 | 4 | 0 | 0 | 1 | 0 | 55 | +| rwo | Rawa | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sab | Buglere | Chibchan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sag | Sango | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| sah | Yakut | Turkic | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| san | Sanskrit | Indo-European | 0 | 0 | 0 | 5 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 10 | +| sat | Santali | Austroasiatic | 0 | 0 | 0 | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| sbe | Saliba | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sbk | Safwa | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sbs | Subiya | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| scn | Sicilian | Indo-European | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| sco | Scots | Indo-European | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| seh | Sena | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sey | Secoya | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sgb | Mag-antsi Ayta | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sgz | Sursurunga | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| shi | Tachelhit | Afro-Asiatic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| shj | Shatt | Dajuic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| shn | Shan | Tai-Kadai | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| shp | Shipibo-Conibo | Pano-Tacanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sim | Mende (Papua New Guinea) | Sepik | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sin | Sinhala | Indo-European | 0 | 0 | 0 | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 7 | +| sja | Epena | Chocoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| slk | Slovak | Indo-European | 0 | 0 | 0 | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 3 | 0 | 0 | 0 | 0 | 0 | 12 | +| sll | Salt-Yui | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| slv | Slovenian | Indo-European | 0 | 0 | 0 | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 10 | +| smk | Bolinao | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| smo | Samoan | Austronesian | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| sna | Shona | Atlantic-Congo | 0 | 0 | 0 | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| snc | Sinaugoro | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| snd | Sindhi | Indo-European | 0 | 0 | 0 | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| snn | Siona | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| snp | Siane | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| snx | Sam | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sny | Saniyo-Hiyewe | Sepik | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| som | Somali | Afro-Asiatic | 0 | 0 | 0 | 3 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 9 | +| soq | Kanasi | Dagan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sot | Southern Sotho | Atlantic-Congo | 0 | 0 | 0 | 1 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 5 | +| soy | Miyobe | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| spa | Spanish | Indo-European | 0 | 2 | 0 | 4 | 13 | 4 | 0 | 0 | 0 | 0 | 0 | 1 | 3 | 2 | 13 | 4 | 0 | 0 | 2 | 0 | 48 | +| spl | Selepet | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| spm | Akukem | Ramu | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| spp | Supyire Senoufo | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sps | Saposa | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| spy | Sabaot | Nilotic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sqi | Albanian | Unclassified | 0 | 0 | 0 | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| srd | Sardinian | Unclassified | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| sri | Siriano | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| srm | Saramaccan | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| srn | Sranan Tongo | Indo-European | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| srp | Serbian | Indo-European | 0 | 0 | 0 | 4 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 2 | 0 | 0 | 0 | 0 | 0 | 9 | +| srq | Sirionó | Tupian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ssd | Siroi | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ssg | Seimat | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ssw | Swati | Atlantic-Congo | 0 | 0 | 0 | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 7 | +| ssx | Samberigi | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| stp | Southeastern Tepehuan | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sua | Sulka | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sue | Suena | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sun | Sundanese | Austronesian | 0 | 0 | 0 | 3 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 9 | +| sus | Susu | Mande | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| suz | Sunwar | Sino-Tibetan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| svk | Slovakian Sign Language | Sign Language | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| swa | Swahili (macrolanguage) | Atlantic-Congo | 0 | 1 | 0 | 1 | 7 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 3 | 0 | 0 | 0 | 0 | 0 | 16 | +| swe | Swedish | Indo-European | 0 | 1 | 0 | 4 | 8 | 3 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 4 | 0 | 0 | 0 | 0 | 0 | 23 | +| swg | Swabian | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| swh | Swahili (individual language) | Atlantic-Congo | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| swp | Suau | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| sxb | Suba | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| szl | Silesian | Indo-European | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| tac | Lowland Tarahumara | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tah | Tahitian | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| taj | Eastern Tamang | Sino-Tibetan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tam | Tamil | Dravidian | 0 | 0 | 0 | 7 | 7 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 3 | 1 | 0 | 0 | 0 | 0 | 21 | +| taq | Tamasheq | Afro-Asiatic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| tat | Tatar | Turkic | 0 | 0 | 0 | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| tav | Tatuyo | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| taw | Tai | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tbc | Takia | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tbf | Mandara | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tbg | North Tairora | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tbo | Tawala | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tbz | Ditammari | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tca | Ticuna | Ticuna-Yuri | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tcs | Torres Strait Creole | Indo-European | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tcz | Thado Chin | Sino-Tibetan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tdt | Tetun Dili | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tee | Huehuetla Tepehua | Totonacan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tel | Telugu | Dravidian | 0 | 1 | 0 | 7 | 7 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 5 | 2 | 0 | 0 | 0 | 0 | 25 | +| ter | Tereno | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tet | Tetum | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tew | Tewa (USA) | Kiowa-Tanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tfr | Teribe | Chibchan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tgk | Tajik | Indo-European | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| tgl | Tagalog | Austronesian | 0 | 0 | 0 | 3 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| tgo | Sudest | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tgp | Tangoa | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tha | Thai | Tai-Kadai | 0 | 1 | 0 | 4 | 8 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 6 | 0 | 0 | 0 | 0 | 0 | 22 | +| tif | Tifal | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tim | Timbe | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tir | Tigrinya | Afro-Asiatic | 0 | 0 | 0 | 2 | 2 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 8 | +| tiw | Tiwi | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tiy | Tiruray | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tke | Takwane | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tku | Upper Necaxa Totonac | Totonacan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tlf | Telefol | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tmd | Haruai | Piawi | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tna | Tacana | Pano-Tacanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tnc | Tanimuca-Retuarã | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tnk | Kwamera | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tnn | North Tanna | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tnp | Whitesands | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| toc | Coyutla Totonac | Totonacan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tod | Toma | Mande | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tof | Gizrra | Eastern Trans-Fly | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| toj | Tojolabal | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ton | Tonga (Tonga Islands) | Austronesian | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| too | Xicotepec De Juárez Totonac | Totonacan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| top | Papantla Totonac | Totonacan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tos | Highland Totonac | Totonacan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tpa | Taupota | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tpi | Tok Pisin | Indo-European | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4 | +| tpt | Tlachichilco Tepehua | Totonacan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tpz | Tinputz | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| trc | Copala Triqui | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tsn | Tswana | Atlantic-Congo | 0 | 0 | 0 | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 7 | +| tso | Tsonga | Atlantic-Congo | 0 | 0 | 0 | 1 | 4 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 7 | +| tsw | Tsishingini | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ttc | Tektiteko | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tte | Bwanabwana | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tuc | Mutu | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tue | Tuyuca | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tuf | Central Tunebo | Chibchan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tuk | Turkmen | Turkic | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5 | +| tum | Tumbuka | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| tuo | Tucano | Tucanoan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tur | Turkish | Turkic | 0 | 3 | 0 | 4 | 7 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | 0 | 3 | 2 | 0 | 0 | 1 | 0 | 24 | +| tvk | Southeast Ambrym | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| twi | Twi | Unclassified | 0 | 0 | 0 | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| txq | Tii | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| txu | Kayapó | Nuclear-Macro-Je | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tyv | Tuvinian | Turkic | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tzj | Tz'utujil | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tzl | Talossan | Artificial Language | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| tzm | Central Atlas Tamazight | Afro-Asiatic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| tzo | Tzotzil | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ubr | Ubir | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ubu | Umbu-Ungu | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| udu | Uduk | Koman | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| uig | Uighur | Turkic | 0 | 0 | 0 | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 7 | +| ukr | Ukrainian | Indo-European | 0 | 1 | 0 | 4 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 9 | +| uli | Ulithian | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ulk | Meriam Mir | Eastern Trans-Fly | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| umb | Umbundu | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| upv | Uripiv-Wala-Rano-Atchin | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ura | Urarina | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| urb | Urubú-Kaapor | Tupian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| urd | Urdu | Indo-European | 0 | 0 | 0 | 7 | 8 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 19 | +| uri | Urim | Nuclear Torricelli | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| urt | Urat | Nuclear Torricelli | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| urw | Sop | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| usa | Usarufa | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| usp | Uspanteco | Mayan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| uvh | Uri | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| uvl | Lote | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| uzb | Uzbek | Unclassified | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| uzn | Northern Uzbek | Turkic | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 4 | +| vec | Venetian | Indo-European | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| ven | Venda | Atlantic-Congo | 0 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | +| vid | Vidunda | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| vie | Vietnamese | Austroasiatic | 0 | 2 | 0 | 5 | 6 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 5 | 0 | 0 | 0 | 0 | 0 | 20 | +| viv | Iduna | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| vmy | Ayautla Mazatec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| waj | Waffa | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wal | Wolaytta | Ta-Ne-Omotic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wap | Wapishana | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| war | Waray (Philippines) | Austronesian | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 5 | +| wat | Kaninuwa | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wbi | Vwanji | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wbp | Warlpiri | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wed | Wedau | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wer | Weri | Kunimaipan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wim | Wik-Mungkan | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wiu | Wiru | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wiv | Vitu | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wln | Walloon | Indo-European | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wmt | Walmajarri | Pama-Nyungan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wmw | Mwani | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wnc | Wantoat | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wnu | Usan | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wol | Wolof | Atlantic-Congo | 0 | 0 | 0 | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 6 | +| wos | Hanga Hundi | Ndu | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wrk | Garrwa | Garrwan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wro | Worrorra | Worrorran | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wrs | Waris | Border | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wsk | Waskia | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wuu | Wu Chinese | Sino-Tibetan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| wuv | Wuvulu-Aua | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xav | Xavánte | Nuclear-Macro-Je | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xbi | Kombio | Nuclear Torricelli | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xed | Hdi | Afro-Asiatic | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xho | Xhosa | Atlantic-Congo | 0 | 0 | 0 | 3 | 3 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 10 | +| xla | Kamula | Kamula-Elevala | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xnn | Northern Kankanay | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xon | Konkomba | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xsi | Sio | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xtd | Diuxi-Tilantongo Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| xtm | Magdalena Peñasco Mixtec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yaa | Yaminahua | Pano-Tacanan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yad | Yagua | Peba-Yagua | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yal | Yalunka | Mande | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yap | Yapese | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yaq | Yaqui | Uto-Aztecan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yby | Yaweyuha | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ycn | Yucuna | Arawakan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ydd | Eastern Yiddish | Indo-European | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3 | +| yid | Yiddish | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yka | Yakan | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yle | Yele | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yml | Iamalele | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yon | Yongkom | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yor | Yoruba | Atlantic-Congo | 0 | 0 | 0 | 4 | 5 | 3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 3 | 0 | 0 | 0 | 0 | 0 | 16 | +| yrb | Yareba | Yareban | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yre | Yaouré | Mande | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yss | Yessan-Mayo | Sepik | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yue | Yue Chinese | Sino-Tibetan | 0 | 0 | 0 | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 6 | +| yuj | Karkar-Yuri | Pauwasi | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yut | Yopno | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yuw | Yau (Morobe Province) | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| yva | Yawa | Yawa-Saweru | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zaa | Sierra de Juárez Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zab | Western Tlacolula Valley Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zac | Ocotlán Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zad | Cajonos Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zai | Isthmus Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zaj | Zaramo | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zam | Miahuatlán Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zao | Ozolotepec Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zap | Zapotec | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zar | Rincón Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zas | Santo Domingo Albarradas Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zat | Tabaa Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zav | Yatzachi Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zaw | Mitla Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zca | Coatecas Altas Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zga | Kinga | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zho | Chinese | Unclassified | 0 | 2 | 0 | 2 | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | 1 | 13 | 0 | 0 | 0 | 0 | 0 | 23 | +| zia | Zia | Nuclear Trans New Guinea | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ziw | Zigula | Atlantic-Congo | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zlm | Malay (individual language) | Austronesian | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zos | Francisco León Zoque | Mixe-Zoque | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpc | Choapan Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpl | Lachixío Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpm | Mixtepec Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpo | Amatlán Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpq | Zoogocho Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpu | Yalálag Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpv | Chichicapan Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zpz | Texmelucan Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zsm | Standard Malay | Austronesian | 0 | 0 | 0 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 5 | +| zsr | Southern Rincon Zapotec | Unclassified | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| ztq | Quioquitani-Quierí Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zty | Yatee Zapotec | Otomanguean | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| zul | Zulu | Atlantic-Congo | 0 | 0 | 0 | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 7 | +| zyp | Zyphe Chin | Sino-Tibetan | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | +| Total | None | None | None | 9 | 114 | 4 | 1398 | 836 | 311 | 21 | 5 | 1 | 6 | 3 | 28 | 91 | 55 | 507 | 88 | 2 | 2 | 24 | 24 |
diff --git a/mteb/abstasks/AbsTask.py b/mteb/abstasks/AbsTask.py index 1ec1ebc4fc..c0368dcc54 100644 --- a/mteb/abstasks/AbsTask.py +++ b/mteb/abstasks/AbsTask.py @@ -63,7 +63,7 @@ class AbsTask(ABC): dataset: dict[HFSubset, DatasetDict] | None = None # type: ignore data_loaded: bool = False is_multilingual: bool = False - hf_subsets: list[HFSubset] | None = None + hf_subsets: list[HFSubset] def __init__(self, seed: int = 42, **kwargs: Any): self.save_suffix = kwargs.get("save_suffix", "") @@ -73,6 +73,7 @@ def __init__(self, seed: int = 42, **kwargs: Any): np.random.seed(self.seed) torch.manual_seed(self.seed) torch.cuda.manual_seed_all(self.seed) + self.hf_subsets = list(self.metadata.hf_subsets_to_langscripts.keys()) def check_if_dataset_is_superseded(self): """Check if the dataset is superseded by a newer version""" diff --git a/mteb/abstasks/AbsTaskBitextMining.py b/mteb/abstasks/AbsTaskBitextMining.py index 1c373cc2f7..b8105dc141 100644 --- a/mteb/abstasks/AbsTaskBitextMining.py +++ b/mteb/abstasks/AbsTaskBitextMining.py @@ -71,7 +71,7 @@ def evaluate( subsets_to_run: list[HFSubset] | None = None, *, encode_kwargs: dict[str, Any] = {}, - **kwargs, + **kwargs: Any, ) -> dict[HFSubset, ScoresDict]: if not self.data_loaded: self.load_data() diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 67aa9fbccd..1bebc79930 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -82,6 +82,8 @@ "Web", "Written", "Programming", + "Chemistry", + "Financial", ] SAMPLE_CREATION_METHOD = Literal[ @@ -94,6 +96,7 @@ "machine-translated and localized", "LM-generated and verified", "rendered", + "multiple", ] TASK_TYPE = Literal[ @@ -119,6 +122,7 @@ "ZeroShotClassification", ] + TASK_CATEGORY = Literal[ "s2s", # Sentence-to-sentence "s2p", # Sentence-to-paragraph @@ -199,6 +203,8 @@ "gpl-3.0", "cdla-sharing-1.0", "mpl-2.0", + "msr-la-nc", + "multiple", ] ) @@ -258,7 +264,7 @@ class TaskMetadata(BaseModel): bibtex_citation: The BibTeX citation for the dataset. Should be an empty string if no citation is available. """ - dataset: dict + dataset: dict[str, Any] name: str description: str @@ -365,6 +371,15 @@ def _check_language_code(code): f"Invalid script code: {script}, you can find valid ISO 15924 codes in {path_to_lang_scripts}" ) + @property + def bcp47_codes(self) -> list[ISO_LANGUAGE_SCRIPT]: + """Return the languages and script codes of the dataset formatting in accordance with the BCP-47 standard.""" + if isinstance(self.eval_langs, dict): + return sorted( + {lang for langs in self.eval_langs.values() for lang in langs} + ) + return sorted(set(self.eval_langs)) + @property def languages(self) -> list[str]: """Return the languages of the dataset as iso639-3 codes.""" @@ -451,8 +466,12 @@ def n_samples(self) -> dict[str, int] | None: for subset, subset_value in stats.items(): if subset == "hf_subset_descriptive_stats": continue - n_samples[subset] = subset_value["num_samples"] + n_samples[subset] = subset_value["num_samples"] # type: ignore return n_samples def __hash__(self) -> int: return hash(self.model_dump_json()) + + @property + def revision(self) -> str: + return self.dataset["revision"] diff --git a/mteb/abstasks/aggregate_task_metadata.py b/mteb/abstasks/aggregate_task_metadata.py new file mode 100644 index 0000000000..106419b752 --- /dev/null +++ b/mteb/abstasks/aggregate_task_metadata.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import logging +from datetime import datetime +from typing import Any + +from pydantic import ConfigDict, model_validator + +from mteb.abstasks.AbsTask import AbsTask +from mteb.abstasks.TaskMetadata import ( + ANNOTATOR_TYPE, + LANGUAGES, + LICENSES, + MODALITIES, + SAMPLE_CREATION_METHOD, + STR_DATE, + TASK_DOMAIN, + TASK_SUBTYPE, + TASK_TYPE, + HFSubset, + TaskMetadata, +) +from mteb.languages import ISO_LANGUAGE_SCRIPT + +logger = logging.getLogger(__name__) + + +class AggregateTaskMetadata(TaskMetadata): + """Metadata for an aggregation of tasks. This description only covers exceptions to the TaskMetadata. Many of the field if not filled out will be + autofilled from its tasks. + + Attributes: + name: The name of the aggregated task. + description: A description of the task. Should explain the aggregation. + prompt: An aggregate task does not have a prompt, thus this value is always None. + dataset: The dataset for the aggregated task is specified in its tasks. The aggregate task thus only specified the revision and uses a + placeholder path. + tasks: A list of tasks, the majority of the metadata is described within its tasks. + eval_splits: The splits of the tasks used for evaluation. + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + name: str + description: str + dataset: dict[str, Any] = { + "path": "aggregate tasks do not have a path", # just a place holder + "revision": "1", + } + + tasks: list[AbsTask] + main_score: str + type: TASK_TYPE + eval_splits: list[str] + eval_langs: LANGUAGES = [] + prompt: None = None + reference: str | None = None + bibtex_citation: str | None = None + + @property + def hf_subsets_to_langscripts(self) -> dict[HFSubset, list[ISO_LANGUAGE_SCRIPT]]: + """Return a dictionary mapping huggingface subsets to languages.""" + return {"default": self.eval_langs} # type: ignore + + @model_validator(mode="after") # type: ignore + def compute_unfilled_cases(self) -> AggregateTaskMetadata: + if not self.eval_langs: + self.eval_langs = self.compute_eval_langs() + if not self.date: + self.date = self.compute_date() + if not self.domains: + self.domains = self.compute_domains() + if not self.task_subtypes: + self.task_subtypes = self.compute_task_subtypes() + if not self.license: + self.license = self.compute_license() + if not self.annotations_creators: + self.annotations_creators = self.compute_annotations_creators() + if not self.dialect: + self.dialect = self.compute_dialect() + if not self.sample_creation: + self.sample_creation = self.compute_sample_creation() + if not self.modalities: + self.modalities = self.compute_modalities() + + return self + + def compute_eval_langs(self) -> list[ISO_LANGUAGE_SCRIPT]: + langs = set() + for task in self.tasks: + langs.update(set(task.metadata.bcp47_codes)) + return list(langs) + + def compute_date(self) -> tuple[STR_DATE, STR_DATE] | None: + # get min max date from tasks + dates = [] + for task in self.tasks: + if task.metadata.date: + dates.append(datetime.fromisoformat(task.metadata.date[0])) + dates.append(datetime.fromisoformat(task.metadata.date[1])) + + if not dates: + return None + + min_date = min(dates) + max_date = max(dates) + return min_date.isoformat(), max_date.isoformat() + + def compute_domains(self) -> list[TASK_DOMAIN] | None: + domains = set() + for task in self.tasks: + if task.metadata.domains: + domains.update(set(task.metadata.domains)) + if domains: + return list(domains) + return None + + def compute_task_subtypes(self) -> list[TASK_SUBTYPE] | None: + subtypes = set() + for task in self.tasks: + if task.metadata.task_subtypes: + subtypes.update(set(task.metadata.task_subtypes)) + if subtypes: + return list(subtypes) + return None + + def compute_license(self) -> LICENSES | None: + licenses = set() + for task in self.tasks: + if task.metadata.license: + licenses.add(task.metadata.license) + if len(licenses) > 1: + return "multiple" + return None + + def compute_annotations_creators(self) -> ANNOTATOR_TYPE | None: + creators = set() + for task in self.tasks: + if task.metadata.annotations_creators: + creators.add(task.metadata.annotations_creators) + if len(creators) > 1: + logger.warning( + f"Multiple annotations_creators found for tasks in {self.name}. Using None as annotations_creators." + ) + return None + + def compute_dialect(self) -> list[str] | None: + dialects = set() + for task in self.tasks: + if task.metadata.dialect: + dialects.update(set(task.metadata.dialect)) + if dialects: + return list(dialects) + return None + + def compute_sample_creation(self) -> SAMPLE_CREATION_METHOD | None: + sample_creations = set() + for task in self.tasks: + if task.metadata.sample_creation: + sample_creations.add(task.metadata.sample_creation) + if len(sample_creations) > 1: + return "multiple" + return None + + def compute_modalities(self) -> list[MODALITIES]: + modalities = set() + for task in self.tasks: + if task.metadata.modalities: + modalities.update(set(task.metadata.modalities)) + if modalities: + return list(modalities) + return None diff --git a/mteb/abstasks/aggregated_task.py b/mteb/abstasks/aggregated_task.py new file mode 100644 index 0000000000..255df2000f --- /dev/null +++ b/mteb/abstasks/aggregated_task.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +import numpy as np + +from mteb.abstasks.AbsTask import AbsTask +from mteb.abstasks.aggregate_task_metadata import AggregateTaskMetadata + +if TYPE_CHECKING: + from datasets import Dataset, DatasetDict + + from mteb.abstasks.TaskMetadata import DescriptiveStatistics, HFSubset + from mteb.encoder_interface import Encoder + from mteb.load_results.task_results import TaskResult + + from .AbsTask import ScoresDict + +logger = logging.getLogger(__name__) + + +class AbsTaskAggregate(AbsTask): + metadata: AggregateTaskMetadata + superseded_by: None | str = None + hf_subset = "default" # since there is no subset we use the "default" naming scheme + _eval_splits: list[str] | None = None + + def __init__(self, **kwargs: Any): + self.tasks = self.metadata.tasks + self.taskname_to_task = {task.metadata.name: task for task in self.tasks} + + def task_results_to_scores( + self, task_results: list[TaskResult] + ) -> dict[str, dict[HFSubset, ScoresDict]]: + """The function that aggregated scores. Can be redefined to allow for custom aggregations.""" + scores = {} + for split in self.metadata.eval_splits: + main_scores = [] + for task_res in task_results: + main_scores.append( + task_res.get_score_fast( + languages=None, + splits=self.metadata.eval_splits, + ) + ) + main_score = np.mean(main_scores) + scores[split] = { + "default": { + self.metadata.main_score: main_score, + "main_score": main_score, + } + } + return scores + + def combine_task_results(self, task_results: list[TaskResult]) -> TaskResult: + """Combined the task results for using `task_results_to_scores`. Do not redefine this function if you want to implement a custom aggregation. + Instead redefin `task_results_to_scores`. + """ + from mteb.load_results.task_results import ( + TaskResult, # to prevent circular imports, # TODO: can potentially likely be out of function in in v2.0.0 + ) + + eval_times = [tr.evaluation_time for tr in task_results if tr.evaluation_time] + if len(eval_times) != len(task_results): + logger.info( + f"Loaded results does not include runtime. Therefor evaluation of {self.metadata.name} " + + "can't be computed. Setting it to None." + ) + eval_time = np.nan + else: + eval_time = sum(eval_times) + + kg_co2_emissions_ = [ + tr.kg_co2_emissions for tr in task_results if tr.kg_co2_emissions + ] + if len(kg_co2_emissions_) != len(task_results): + logger.info( + f"Loaded results does not include co2-eq emissions. Therefor evaluation of {self.metadata.name} " + + "can't be computed. Setting it to None." + ) + kg_co2_emissions = np.nan + else: + kg_co2_emissions = sum(kg_co2_emissions_) + + task_res = TaskResult.from_task_results( + self, + scores=self.task_results_to_scores(task_results), + evaluation_time=eval_time, + kg_co2_emissions=kg_co2_emissions, + ) + mteb_versions = {tr.mteb_version for tr in task_results} + if len(mteb_versions) != 1: + logger.warning( + f"All tasks of {self.metadata.name} is not run using the same version." + ) + task_res.mteb_version = None + task_res.mteb_version = task_results[0].mteb_version + return task_res + + def check_if_dataset_is_superseded(self): + """Check if the dataset is superseded by a newer version""" + if self.superseded_by: + logger.warning( + f"Dataset '{self.metadata.name}' is superseded by '{self.superseded_by}', you might consider using the newer version of the dataset." + ) + + def filter_eval_splits(self, eval_splits: list[str] | None) -> AbsTaskAggregate: + """Filter the evaluation splits of the task.""" + self._eval_splits = eval_splits + return self + + def evaluate( + self, + model: Encoder, + split: str = "test", + subsets_to_run: list[HFSubset] | None = None, + *, + encode_kwargs: dict[str, Any] = {}, + **kwargs: Any, + ) -> dict[HFSubset, ScoresDict]: + # TODO: If we refactor the runner to at least have a subfunction mteb.run_task(model, task) we could use that here + raise NotImplementedError( + "Aggregate tasks can't be evaluated directly. Instead run it using the MTEB class." + ) + + def _evaluate_subset( + self, + model: Encoder, + data_split: DatasetDict | Dataset, + encode_kwargs: dict[str, Any], + **kwargs: Any, + ) -> ScoresDict: + raise NotImplementedError( + "Aggregate tasks does not implement a _evaluate_subset. Instead use the individual tasks." + ) + + def _calculate_metrics_from_split( + self, split: str, hf_subset: str | None = None, compute_overall: bool = False + ) -> DescriptiveStatistics: + raise NotImplementedError( + "Aggregate tasks does not implement a _calculate_metrics_from_split. Instead use the individual tasks." + ) + + @property + def eval_splits(self) -> list[str]: + if self._eval_splits: + return self._eval_splits + return self.metadata.eval_splits diff --git a/mteb/benchmarks/benchmarks.py b/mteb/benchmarks/benchmarks.py index 233c7a79b3..600981a77d 100644 --- a/mteb/benchmarks/benchmarks.py +++ b/mteb/benchmarks/benchmarks.py @@ -154,18 +154,7 @@ def load_results( "Banking77Classification", "BiorxivClusteringP2P", "BiorxivClusteringS2S", - "CQADupstackAndroidRetrieval", - "CQADupstackEnglishRetrieval", - "CQADupstackGamingRetrieval", - "CQADupstackGisRetrieval", - "CQADupstackMathematicaRetrieval", - "CQADupstackPhysicsRetrieval", - "CQADupstackProgrammersRetrieval", - "CQADupstackStatsRetrieval", - "CQADupstackTexRetrieval", - "CQADupstackUnixRetrieval", - "CQADupstackWebmastersRetrieval", - "CQADupstackWordpressRetrieval", + "CQADupstackRetrieval", "ClimateFEVER", "DBPedia", "EmotionClassification", @@ -432,13 +421,12 @@ def load_results( ), description="A curated selection of tasks coverering the Scandinavian languages; Danish, Swedish and Norwegian, including Bokmål and Nynorsk.", reference="https://kennethenevoldsen.github.io/scandinavian-embedding-benchmark/", - citation="""@misc{enevoldsen2024scandinavian, - title={The Scandinavian Embedding Benchmarks: Comprehensive Assessment of Multilingual and Monolingual Text Embedding}, - author={Kenneth Enevoldsen and Márton Kardos and Niklas Muennighoff and Kristoffer Laigaard Nielbo}, - year={2024}, - eprint={2406.02396}, - archivePrefix={arXiv}, - primaryClass={cs.CL} + citation="""@inproceedings{enevoldsen2024scandinavian, + title={The Scandinavian Embedding Benchmarks: Comprehensive Assessment of Multilingual and Monolingual Text Embedding}, + author={Enevoldsen, Kenneth and Kardos, M{\'a}rton and Muennighoff, Niklas and Nielbo, Kristoffer}, + booktitle={Advances in Neural Information Processing Systems}, + year={2024}, + url={https://nips.cc/virtual/2024/poster/97869} }""", contacts=["KennethEnevoldsen", "x-tabdeveloping", "Samoed"], ) @@ -1153,6 +1141,30 @@ def load_results( }""", ) + +CODE_RAG = Benchmark( + name="CodeRAG", + tasks=get_tasks( + tasks=[ + "CodeRAGLibraryDocumentationSolutions", + "CodeRAGOnlineTutorials", + "CodeRAGProgrammingSolutions", + "CodeRAGStackoverflowPosts", + ], + ), + description="A benchmark for evaluating code retrieval augmented generation, testing models' ability to retrieve relevant programming solutions, tutorials and documentation.", + reference="https://arxiv.org/abs/2406.14497", + citation="""@misc{wang2024coderagbenchretrievalaugmentcode, + title={CodeRAG-Bench: Can Retrieval Augment Code Generation?}, + author={Zora Zhiruo Wang and Akari Asai and Xinyan Velocity Yu and Frank F. Xu and Yiqing Xie and Graham Neubig and Daniel Fried}, + year={2024}, + eprint={2406.14497}, + archivePrefix={arXiv}, + primaryClass={cs.SE}, + url={https://arxiv.org/abs/2406.14497}, + }""", +) + NANOBEIR = Benchmark( name="NanoBEIR", tasks=get_tasks( @@ -1232,3 +1244,126 @@ def load_results( primaryClass={cs.CL} }""", ) + +FA_MTEB = Benchmark( + name="FaMTEB(fas, beta)", + tasks=get_tasks( + languages=["fas"], + tasks=[ + # Classification + "PersianFoodSentimentClassification", + "SynPerChatbotConvSAClassification", + "SynPerChatbotConvSAToneChatbotClassification", + "SynPerChatbotConvSAToneUserClassification", + "SynPerChatbotSatisfactionLevelClassification", + "SynPerChatbotRAGToneChatbotClassification", + "SynPerChatbotRAGToneUserClassification", + "SynPerChatbotToneChatbotClassification", + "SynPerChatbotToneUserClassification", + "PersianTextTone", + "SIDClassification", + "DeepSentiPers", + "PersianTextEmotion", + "SentimentDKSF", + "NLPTwitterAnalysisClassification", + "DigikalamagClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + # Clustering + "BeytooteClustering", + "DigikalamagClustering", + "HamshahriClustring", + "NLPTwitterAnalysisClustering", + "SIDClustring", + # PairClassification + "FarsTail", + "CExaPPC", + "SynPerChatbotRAGFAQPC", + "FarsiParaphraseDetection", + "SynPerTextKeywordsPC", + "SynPerQAPC", + "ParsinluEntail", + "ParsinluQueryParaphPC", + # Reranking + "MIRACLReranking", + "WikipediaRerankingMultilingual", + # Retrieval + "SynPerQARetrieval", + "SynPerChatbotTopicsRetrieval", + "SynPerChatbotRAGTopicsRetrieval", + "SynPerChatbotRAGFAQRetrieval", + "PersianWebDocumentRetrieval", + "WikipediaRetrievalMultilingual", + "MIRACLRetrieval", + "ClimateFEVER-Fa", + "DBPedia-Fa", + "HotpotQA-Fa", + "MSMARCO-Fa", + "NQ-Fa", + "ArguAna-Fa", + "CQADupstackRetrieval-Fa", + "FiQA2018-Fa", + "NFCorpus-Fa", + "QuoraRetrieval-Fa", + "SCIDOCS-Fa", + "SciFact-Fa", + "TRECCOVID-Fa", + "Touche2020-Fa", + # STS + "Farsick", + "SynPerSTS", + "Query2Query", + # SummaryRetrieval + "SAMSumFa", + "SynPerChatbotSumSRetrieval", + "SynPerChatbotRAGSumSRetrieval", + ], + ), + description="Main Persian (Farsi) benchmarks from MTEB", + reference=None, + citation=None, + contacts=["mehran-sarmadi", "ERfun", "morteza20"], +) + +CHEMTEB = Benchmark( + name="ChemTEB", + tasks=get_tasks( + tasks=[ + "PubChemSMILESBitextMining", + "SDSEyeProtectionClassification", + "SDSGlovesClassification", + "WikipediaBioMetChemClassification", + "WikipediaGreenhouseEnantiopureClassification", + "WikipediaSolidStateColloidalClassification", + "WikipediaOrganicInorganicClassification", + "WikipediaCryobiologySeparationClassification", + "WikipediaChemistryTopicsClassification", + "WikipediaTheoreticalAppliedClassification", + "WikipediaChemFieldsClassification", + "WikipediaLuminescenceClassification", + "WikipediaIsotopesFissionClassification", + "WikipediaSaltsSemiconductorsClassification", + "WikipediaBiolumNeurochemClassification", + "WikipediaCrystallographyAnalyticalClassification", + "WikipediaCompChemSpectroscopyClassification", + "WikipediaChemEngSpecialtiesClassification", + "WikipediaChemistryTopicsClustering", + "WikipediaSpecialtiesInChemistryClustering", + "PubChemAISentenceParaphrasePC", + "PubChemSMILESPC", + "PubChemSynonymPC", + "PubChemWikiParagraphsPC", + "PubChemWikiPairClassification", + "ChemNQRetrieval", + "ChemHotpotQARetrieval", + ], + ), + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + citation="""@article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} +}""", +) diff --git a/mteb/create_meta.py b/mteb/create_meta.py index e810751a08..ea4bf9c952 100644 --- a/mteb/create_meta.py +++ b/mteb/create_meta.py @@ -8,7 +8,6 @@ import mteb from mteb import TaskResult -from mteb.load_results.task_results import CQADupstackRetrievalDummy def generate_readme(results_folder: Path, from_existing: Path | None = None) -> str: @@ -46,12 +45,7 @@ def load_model_name(results_folder: Path) -> str: def process_task_result(task_result: TaskResult) -> list[dict[str, Any]]: - # CQADupstackRetrieval is a combined dataset (special case atm.) - task = ( - CQADupstackRetrievalDummy() - if task_result.task_name == "CQADupstackRetrieval" - else mteb.get_task(task_result.task_name) - ) + task = mteb.get_task(task_result.task_name) yaml_results = [] for split, hf_subset_scores in task_result.scores.items(): diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py index 70378931c2..bcef789174 100644 --- a/mteb/evaluation/MTEB.py +++ b/mteb/evaluation/MTEB.py @@ -4,7 +4,7 @@ import logging import os import traceback -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from copy import copy, deepcopy from datetime import datetime from itertools import chain @@ -16,6 +16,7 @@ from sentence_transformers import CrossEncoder, SentenceTransformer from mteb.abstasks.AbsTask import ScoresDict +from mteb.abstasks.aggregated_task import AbsTaskAggregate from mteb.encoder_interface import Encoder from mteb.model_meta import ModelMeta from mteb.models import model_meta_from_sentence_transformers @@ -31,9 +32,12 @@ class MTEB: + _tasks: Iterable[str | AbsTask] | None + tasks: list[AbsTask] + def __init__( self, - tasks: Iterable[str | AbsTask] | None = None, + tasks: Sequence[str | AbsTask] | None = None, *, task_types: list[str] | None = None, task_categories: list[str] | None = None, @@ -61,12 +65,11 @@ def __init__( self.deprecation_warning( task_types, task_categories, task_langs, tasks, version ) - if tasks is not None: self._tasks = tasks if isinstance(tasks[0], Benchmark): self.benchmarks = tasks - self._tasks = list(chain.from_iterable(tasks)) + self._tasks = self._tasks = list(chain.from_iterable(tasks)) # type: ignore assert ( task_types is None and task_categories is None ), "Cannot specify both `tasks` and `task_types`/`task_categories`" @@ -253,7 +256,7 @@ def select_tasks(self, **kwargs): f"WARNING: Unknown tasks: {unknown_str}. Known tasks: {known_str}." ) # add task if subclass of mteb.tasks - self.tasks.extend([x for x in self._tasks if isinstance(x, AbsTask)]) + self.tasks.extend([x for x in self._tasks if isinstance(x, (AbsTask))]) return # Otherwise use filters to select tasks @@ -463,6 +466,29 @@ def run( f"\n\n********************** Evaluating {task.metadata.name} **********************" ) + if isinstance(task, AbsTaskAggregate): + self_ = MTEB(tasks=task.metadata.tasks) + task_results = self_.run( + model, + verbosity=verbosity - 1, + output_folder=output_folder, + eval_splits=eval_splits, + eval_subsets=eval_subsets, + overwrite_results=overwrite_results, + raise_error=raise_error, + co2_tracker=co2_tracker, + encode_kwargs=encode_kwargs, + **kwargs, + ) + new_results = task.combine_task_results(task_results) + evaluation_results.append(new_results) + + if output_path: + save_path = output_path / f"{task.metadata.name}.json" + new_results.to_disk(save_path) + del self.tasks[0] + continue + if "bm25s" in meta.name and task.metadata.type != "Retrieval": logger.warning( f"bm25s only supports Retrieval tasks, but the task type is {task.metadata.type}. Skipping task." @@ -473,7 +499,11 @@ def run( task_eval_splits = ( eval_splits if eval_splits is not None else task.eval_splits ) - task_subsets = list(task.metadata.hf_subsets_to_langscripts.keys()) + task_subsets = ( + task.hf_subsets + if task.hf_subsets + else list(task.metadata.hf_subsets_to_langscripts.keys()) + ) existing_results = None save_path = None diff --git a/mteb/leaderboard/app.py b/mteb/leaderboard/app.py index 5ee5a6b9da..94bea19a83 100644 --- a/mteb/leaderboard/app.py +++ b/mteb/leaderboard/app.py @@ -5,6 +5,7 @@ import logging import tempfile import time +import typing from pathlib import Path from typing import Literal from urllib.parse import urlencode @@ -14,21 +15,41 @@ from gradio_rangeslider import RangeSlider import mteb +from mteb.abstasks.TaskMetadata import TASK_TYPE from mteb.caching import json_cache from mteb.leaderboard.figures import performance_size_plot, radar_chart from mteb.leaderboard.table import scores_to_tables logger = logging.getLogger(__name__) +acknowledgment_md = """ +### Acknowledgment +We thank [ServiceNow](https://www.servicenow.com/), [Contextual AI](https://contextual.ai/) and [Hugging Face](https://huggingface.co/) for their generous sponsorship. If you'd like to sponsor us, please get in [touch](mailto:n.muennighoff@gmail.com). + + + +We also thank the following companies which provide API credits to evaluate their models: [OpenAI](https://openai.com/), [Voyage AI](https://www.voyageai.com/) +""" + +ALL_MODELS = {meta.name for meta in mteb.get_model_metas()} + def load_results(): results_cache_path = Path(__file__).parent.joinpath("__cached_results.json") if not results_cache_path.exists(): - all_results = ( - mteb.load_results(only_main_score=True, require_model_meta=False) - .join_revisions() - .filter_models() - ) + all_results = mteb.load_results( + only_main_score=True, require_model_meta=False, models=ALL_MODELS + ).filter_models() all_results.to_disk(results_cache_path) return all_results else: @@ -168,7 +189,7 @@ def filter_models( benchmarks = mteb.get_benchmarks() all_benchmark_results = { - benchmark.name: benchmark.load_results(base_results=all_results) + benchmark.name: benchmark.load_results(base_results=all_results).join_revisions() for benchmark in benchmarks } default_benchmark = mteb.get_benchmark(DEFAULT_BENCHMARK_NAME) @@ -206,7 +227,7 @@ def filter_models( ) type_select = gr.Dropdown( all_results.task_types, - value=sorted(default_results.task_types), + value=sorted(typing.get_args(TASK_TYPE)), multiselect=True, label="Task Type", info="Select task types to include.", @@ -232,6 +253,12 @@ def filter_models( """ with gr.Blocks(fill_width=True, theme=gr.themes.Base(), head=head) as demo: + gr.Markdown(""" + ## MMTEB: Massive Multilingual Text Embedding Benchmark + + The MMTEB leaderboard compares text embedding models on 1000+ languages. Check out the [paper](https://openreview.net/pdf?id=zl3pfz4VCV) for details on datasets, languages and tasks. And you can contribute! 🤗 To add a model, please refer to the documentation in the [GitHub repository](https://github.com/embeddings-benchmark/mteb/blob/main/docs/adding_a_model.md). Also check out [MTEB Arena](https://huggingface.co/spaces/mteb/arena) ⚔️ + """) + with gr.Row(): with gr.Column(scale=5): gr.Markdown( @@ -632,6 +659,7 @@ def update_tables( outputs=[summary_table, per_task_table], ) + gr.Markdown(acknowledgment_md, elem_id="ack_markdown") if __name__ == "__main__": demo.launch() diff --git a/mteb/load_results/benchmark_results.py b/mteb/load_results/benchmark_results.py index e1632a3dec..caece1b2b4 100644 --- a/mteb/load_results/benchmark_results.py +++ b/mteb/load_results/benchmark_results.py @@ -3,7 +3,7 @@ import json import warnings from collections import defaultdict -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from pathlib import Path from typing import Any, Callable, Literal @@ -69,7 +69,7 @@ def filter_tasks( task_results=new_task_results, ) - def select_tasks(self, tasks: list[AbsTask]) -> ModelResult: + def select_tasks(self, tasks: Sequence[AbsTask]) -> ModelResult: task_name_to_task = {task.metadata.name: task for task in tasks} new_task_results = [ task_res.validate_and_filter_scores(task_name_to_task[task_res.task_name]) @@ -105,15 +105,15 @@ def get_scores( try: if use_fast: scores[res.task_name] = res.get_score_fast( - splits=splits, - languages=languages, + splits=splits, # type: ignore + languages=languages, # type: ignore ) else: scores[res.task_name] = res.get_score( splits=splits, languages=languages, - aggregation=aggregation, - getter=getter, + aggregation=aggregation, # type: ignore + getter=getter, # type: ignore scripts=scripts, ) except Exception as e: @@ -216,7 +216,7 @@ def filter_tasks( model_results=[res for res in model_results if res.task_results] ) - def select_tasks(self, tasks: list[AbsTask]) -> BenchmarkResults: + def select_tasks(self, tasks: Sequence[AbsTask]) -> BenchmarkResults: new_model_results = [ model_res.select_tasks(tasks) for model_res in self.model_results ] @@ -259,6 +259,8 @@ def parse_version(version_str: str) -> Version | None: return None def keep_best(group: pd.DataFrame) -> pd.DataFrame: + # Filtering out task_results where no scores are present + group = group[group["has_scores"]] is_main_revision = group["revision"] == group["main_revision"] # If the main revision is present we select that if is_main_revision.sum() > 0: @@ -286,6 +288,7 @@ def keep_best(group: pd.DataFrame) -> pd.DataFrame: task_name=task_result.task_name, mteb_version=task_result.mteb_version, task_result=task_result, + has_scores=bool(task_result.scores), ) ) task_df = pd.DataFrame.from_records(records) @@ -314,8 +317,8 @@ def get_scores( splits: list[Split] | None = None, languages: list[ISO_LANGUAGE | ISO_LANGUAGE_SCRIPT] | None = None, scripts: list[ISO_LANGUAGE_SCRIPT] | None = None, - getter: Callable[[ScoresDict], Score] = None, - aggregation: Callable[[list[Score]], Any] = None, + getter: Callable[[ScoresDict], Score] | None = None, + aggregation: Callable[[list[Score]], Any] | None = None, format: Literal["wide", "long"] = "wide", ) -> list[dict]: entries = [] @@ -390,7 +393,7 @@ def to_dict(self) -> dict: return self.model_dump() @classmethod - def from_dict(cls, data: dict) -> TaskResult: + def from_dict(cls, data: dict) -> BenchmarkResults: return cls.model_validate(data) def to_disk(self, path: Path | str) -> None: diff --git a/mteb/load_results/task_results.py b/mteb/load_results/task_results.py index 72cae5a93d..4ff2406934 100644 --- a/mteb/load_results/task_results.py +++ b/mteb/load_results/task_results.py @@ -4,6 +4,7 @@ import logging from argparse import Namespace from collections import defaultdict +from collections.abc import Iterable from functools import cached_property from importlib.metadata import version from pathlib import Path @@ -23,24 +24,6 @@ logger = logging.getLogger(__name__) -# Tasks that were completely removed from the MTEB (we generally don't do this anymore instead we supersede tasks) -class CQADupstackRetrievalDummy: - """A dummy task for loading historic results from before v1.11.0""" - - metadata = Namespace( # type: ignore - name="CQADupstackRetrieval", - main_score="ndcg_at_10", - type="Retrieval", - hf_subsets_to_langscripts={ - "default": ["eng-Latn"], - }, - dataset={ - "revision": "revision not applicable", - "path": "CQADupstackRetrieval_is_a_combined_dataset", - }, - ) - - class ScalaNbClassificationDummy: """A dummy task for loading historic results from before v1.11.0""" @@ -52,6 +35,7 @@ class ScalaNbClassificationDummy: "default": ["nob-Latn"], }, dataset={"revision": "revision_not_applicable"}, + revision="revision_not_applicable", ) @@ -66,6 +50,7 @@ class ScalaNnClassificationDummy: "default": ["nno-Latn"], }, dataset={"revision": "revision_not_applicable"}, + revision="revision_not_applicable", ) @@ -80,6 +65,7 @@ class ScalaDaClassificationDummy: "default": ["dan-Latn"], }, dataset={"revision": "revision_not_applicable"}, + revision="revision_not_applicable", ) @@ -94,11 +80,11 @@ class ScalaSvClassificationDummy: "default": ["swe-Latn"], }, dataset={"revision": "revision_not_applicable"}, + revision="revision_not_applicable", ) outdated_tasks = { - "CQADupstackRetrieval": CQADupstackRetrievalDummy, "ScalaNbClassification": ScalaNbClassificationDummy, "ScalaNnClassification": ScalaNnClassificationDummy, "ScalaDaClassification": ScalaDaClassificationDummy, @@ -183,7 +169,7 @@ def from_task_results( flat_scores[split].append(_scores) return TaskResult( - dataset_revision=task.metadata.dataset["revision"], + dataset_revision=task.metadata.revision, task_name=task.metadata.name, mteb_version=version("mteb"), scores=flat_scores, @@ -471,10 +457,12 @@ def get_score( return aggregation(values) - def get_score_fast(self, splits: str | None, languages: str | None) -> float: + def get_score_fast( + self, splits: Iterable[str] | None = None, languages: str | None = None + ) -> float: """Sped up version of get_score that will be used if no aggregation, script or getter needs to be specified.""" if splits is None: - splits = self.scores + splits = self.scores.keys() val_sum = 0 n_val = 0 for split in splits: @@ -536,14 +524,11 @@ def validate_and_filter_scores(self, task: AbsTask | None = None) -> TaskResult: if task is None: task = get_task(self.task_name) + splits = task.metadata.eval_splits - if task.is_multilingual: - hf_subsets = getattr( - task, "hf_subsets", task.metadata.hf_subsets_to_langscripts.keys() - ) - hf_subsets = set(hf_subsets) - else: - hf_subsets = {"default"} + hf_subsets = task.hf_subsets + hf_subsets = set(hf_subsets) + new_scores = {} seen_splits = set() for split in self.scores: diff --git a/mteb/models/arctic_models.py b/mteb/models/arctic_models.py index f765b01bff..e92c1ca098 100644 --- a/mteb/models/arctic_models.py +++ b/mteb/models/arctic_models.py @@ -110,7 +110,8 @@ # in MTEB "NQ": ["test"], "NQHardNegatives": ["test"], - "HotPotQA": ["test"], + "NQ-PL": ["test"], + "HotPotQA": ["test"], # translated, not trained on "HotPotQAHardNegatives": ["test"], "HotPotQA-PL": ["test"], # translated from hotpotQA (not trained on) "FEVER": ["test"], diff --git a/mteb/models/bedrock_models.py b/mteb/models/bedrock_models.py new file mode 100644 index 0000000000..4616209df1 --- /dev/null +++ b/mteb/models/bedrock_models.py @@ -0,0 +1,264 @@ +from __future__ import annotations + +import json +import logging +import re +from functools import partial +from typing import Any + +import numpy as np +import tqdm + +from mteb.encoder_interface import PromptType +from mteb.model_meta import ModelMeta +from mteb.models.cohere_models import model_prompts as cohere_model_prompts +from mteb.models.cohere_models import supported_languages as cohere_supported_languages +from mteb.requires_package import requires_package + +from .wrapper import Wrapper + +logger = logging.getLogger(__name__) + + +class BedrockWrapper(Wrapper): + def __init__( + self, + model_id: str, + provider: str, + max_tokens: int, + model_prompts: dict[str, str] | None = None, + **kwargs, + ) -> None: + requires_package(self, "boto3", "The AWS SDK for Python") + import boto3 + + boto3_session = boto3.session.Session() + region_name = boto3_session.region_name + self._client = boto3.client("bedrock-runtime", region_name) + + self._model_id = model_id + self._provider = provider.lower() + + if self._provider == "cohere": + self.model_prompts = ( + self.validate_task_to_prompt_name(model_prompts) + if model_prompts + else None + ) + self._max_batch_size = 96 + self._max_sequence_length = max_tokens * 4 + else: + self._max_tokens = max_tokens + + def encode( + self, + sentences: list[str], + *, + task_name: str | None = None, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + requires_package(self, "boto3", "Amazon Bedrock") + show_progress_bar = ( + False + if "show_progress_bar" not in kwargs + else kwargs.pop("show_progress_bar") + ) + if self._provider == "amazon": + return self._encode_amazon(sentences, show_progress_bar) + elif self._provider == "cohere": + prompt_name = self.get_prompt_name( + self.model_prompts, task_name, prompt_type + ) + cohere_task_type = self.model_prompts.get(prompt_name, "search_document") + return self._encode_cohere(sentences, cohere_task_type, show_progress_bar) + else: + raise ValueError( + f"Unknown provider '{self._provider}'. Must be 'amazon' or 'cohere'." + ) + + def _encode_amazon( + self, sentences: list[str], show_progress_bar: bool = False + ) -> np.ndarray: + from botocore.exceptions import ValidationError + + all_embeddings = [] + # https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html + max_sequence_length = int(self._max_tokens * 4.5) + + for sentence in tqdm.tqdm( + sentences, leave=False, disable=not show_progress_bar + ): + if len(sentence) > max_sequence_length: + truncated_sentence = sentence[:max_sequence_length] + else: + truncated_sentence = sentence + + try: + embedding = self._embed_amazon(truncated_sentence) + all_embeddings.append(embedding) + + except ValidationError as e: + error_str = str(e) + pattern = r"request input token count:\s*(\d+)" + match = re.search(pattern, error_str) + if match: + num_tokens = int(match.group(1)) + + ratio = 0.9 * (self._max_tokens / num_tokens) + dynamic_cutoff = int(len(truncated_sentence) * ratio) + + embedding = self._embed_amazon(truncated_sentence[:dynamic_cutoff]) + all_embeddings.append(embedding) + else: + raise e + + return np.array(all_embeddings) + + def _encode_cohere( + self, + sentences: list[str], + cohere_task_type: str, + show_progress_bar: bool = False, + ) -> np.ndarray: + batches = [ + sentences[i : i + self._max_batch_size] + for i in range(0, len(sentences), self._max_batch_size) + ] + + all_embeddings = [] + + for batch in tqdm.tqdm(batches, leave=False, disable=not show_progress_bar): + response = self._client.invoke_model( + body=json.dumps( + { + "texts": [sent[: self._max_sequence_length] for sent in batch], + "input_type": cohere_task_type, + } + ), + modelId=self._model_id, + accept="*/*", + contentType="application/json", + ) + all_embeddings.extend(self._to_numpy(response)) + + return np.array(all_embeddings) + + def _embed_amazon(self, sentence: str) -> np.ndarray: + response = self._client.invoke_model( + body=json.dumps({"inputText": sentence}), + modelId=self._model_id, + accept="application/json", + contentType="application/json", + ) + return self._to_numpy(response) + + def _to_numpy(self, embedding_response) -> np.ndarray: + response = json.loads(embedding_response.get("body").read()) + key = "embedding" if self._provider == "amazon" else "embeddings" + return np.array(response[key]) + + +amazon_titan_embed_text_v1 = ModelMeta( + name="bedrock/amazon-titan-embed-text-v1", + revision="1", + release_date="2023-09-27", + languages=None, # not specified + loader=partial( + BedrockWrapper, + model_id="amazon.titan-embed-text-v1", + provider="amazon", + max_tokens=8192, + ), + max_tokens=8192, + embed_dim=1536, + open_weights=False, + n_parameters=None, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, + license=None, + reference="https://aws.amazon.com/about-aws/whats-new/2023/09/amazon-titan-embeddings-generally-available/", + similarity_fn_name="cosine", + framework=["API"], + use_instructions=False, +) + +amazon_titan_embed_text_v2 = ModelMeta( + name="bedrock/amazon-titan-embed-text-v2", + revision="1", + release_date="2024-04-30", + languages=None, # not specified + loader=partial( + BedrockWrapper, + model_id="amazon.titan-embed-text-v2:0", + provider="amazon", + max_tokens=8192, + ), + max_tokens=8192, + embed_dim=1024, + open_weights=False, + n_parameters=None, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, + license=None, + reference="https://aws.amazon.com/about-aws/whats-new/2024/04/amazon-titan-text-embeddings-v2-amazon-bedrock/", + similarity_fn_name="cosine", + framework=["API"], + use_instructions=False, +) +# Note: For the original Cohere API implementation, refer to: +# https://github.com/embeddings-benchmark/mteb/blob/main/mteb/models/cohere_models.py +# This implementation uses the Amazon Bedrock endpoint for Cohere models. +cohere_embed_english_v3 = ModelMeta( + loader=partial( + BedrockWrapper, + model_id="cohere.embed-english-v3", + provider="cohere", + max_tokens=512, + model_prompts=cohere_model_prompts, + ), + name="bedrock/cohere-embed-english-v3", + languages=["eng-Latn"], + open_weights=False, + reference="https://cohere.com/blog/introducing-embed-v3", + revision="1", + release_date="2023-11-02", + n_parameters=None, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, + max_tokens=512, + embed_dim=1024, + license=None, + similarity_fn_name="cosine", + framework=["API"], + use_instructions=True, +) + +cohere_embed_multilingual_v3 = ModelMeta( + loader=partial( + BedrockWrapper, + model_id="cohere.embed-multilingual-v3", + provider="cohere", + max_tokens=512, + model_prompts=cohere_model_prompts, + ), + name="bedrock/cohere-embed-multilingual-v3", + languages=cohere_supported_languages, + open_weights=False, + reference="https://cohere.com/blog/introducing-embed-v3", + revision="1", + release_date="2023-11-02", + n_parameters=None, + public_training_code=None, + public_training_data=None, # assumed + training_datasets=None, + max_tokens=512, + embed_dim=1024, + license=None, + similarity_fn_name="cosine", + framework=["API"], + use_instructions=True, +) diff --git a/mteb/models/e5_instruct.py b/mteb/models/e5_instruct.py index 3eed189d33..3c18f9c27a 100644 --- a/mteb/models/e5_instruct.py +++ b/mteb/models/e5_instruct.py @@ -19,7 +19,6 @@ **E5_TRAINING_DATA, "FEVER": ["train"], "FEVERHardNegatives": ["train"], - "FEVER-PL": ["train"], # translation not trained on "HotpotQA": ["train"], "HotpotQAHardNegatives": ["train"], "HotpotQA-PL": ["train"], # translation not trained on diff --git a/mteb/models/e5_models.py b/mteb/models/e5_models.py index 0ad15e7320..94d04ee483 100644 --- a/mteb/models/e5_models.py +++ b/mteb/models/e5_models.py @@ -130,7 +130,6 @@ **E5_TRAINING_DATA, "FEVER": ["train"], "FEVERHardNegatives": ["train"], - "FEVER-PL": ["train"], # translation not trained on "HotpotQA": ["train"], "HotpotQAHardNegatives": ["train"], "HotpotQA-PL": ["train"], # translation not trained on diff --git a/mteb/models/gritlm_models.py b/mteb/models/gritlm_models.py index 35d0543811..440779787b 100644 --- a/mteb/models/gritlm_models.py +++ b/mteb/models/gritlm_models.py @@ -16,7 +16,6 @@ # also uses medi2 which contains fever and hotpotqa: "FEVER": ["train"], "FEVERHardNegatives": ["train"], - "FEVER-PL": ["train"], # translation not trained on "HotpotQA": ["train"], "HotpotQAHardNegatives": ["train"], "HotpotQA-PL": ["train"], # translation not trained on diff --git a/mteb/models/instruct_wrapper.py b/mteb/models/instruct_wrapper.py index 2ee3a09b56..cc6e814629 100644 --- a/mteb/models/instruct_wrapper.py +++ b/mteb/models/instruct_wrapper.py @@ -6,6 +6,7 @@ import numpy as np import torch +from sentence_transformers import SentenceTransformer from mteb.encoder_interface import PromptType @@ -78,3 +79,87 @@ def encode( return embeddings return InstructWrapper(model_name_or_path, mode, instruction_template, **kwargs) + + +class InstructSentenceTransformerWrapper(Wrapper): + def __init__( + self, + model_name: str, + revision: str, + instruction_template: str | Callable[[str], str] | None = None, + max_seq_length: int | None = None, + apply_instruction_to_passages: bool = True, + padding_side: str | None = None, + add_eos_token: bool = False, + **kwargs: Any, + ): + """Instruct Sentence Transformer Wrapper. Wrapper that passes instructions to the Sentence Transformer model. + Applied for models like NV-Embed, gte-Qwen, e5-mistral, etc. + + Arguments: + model_name: Model name of the sentence transformers model. + revision: Revision of the sentence transformers model. + instruction_template: Model template. Should contain the string '{instruction}'. + max_seq_length: Maximum sequence length. If None, the maximum sequence length will be read from the model config. + apply_instruction_to_passages: Whether to apply the instruction template to the passages. + padding_side: Padding side. If None, the padding side will be read from the model config. + add_eos_token: Whether to add the eos token to each input example. + **kwargs: Kwargs for Sentence Transformer model. + """ + if ( + isinstance(instruction_template, str) + and "{instruction}" not in instruction_template + ): + raise ValueError( + "Instruction template must contain the string '{instruction}'." + ) + if instruction_template is None: + logger.warning( + "No instruction template provided. Instructions will be used as-is." + ) + + self.model_name = model_name + self.model = SentenceTransformer(model_name, revision=revision, **kwargs) + self.instruction_template = instruction_template + self.apply_instruction_to_passages = apply_instruction_to_passages + self.add_eos_token = add_eos_token + if max_seq_length is not None: + self.model.max_seq_length = max_seq_length + if padding_side is not None: + self.model.tokenizer.padding_side = padding_side + + def encode( + self, + sentences: Sequence[str], + *, + task_name: str, + prompt_type: PromptType | None = None, + **kwargs: Any, + ) -> np.ndarray: + if self.add_eos_token: + sentences = [ + example + self.model.tokenizer.eos_token for example in sentences + ] + + instruction = self.get_task_instruction(task_name, prompt_type) + + # to passage prompts won't be applied to passages + if not self.apply_instruction_to_passages and prompt_type == PromptType.passage: + instruction = None + logger.info( + f"No instruction used, because prompt type = {prompt_type.passage}" + ) + + if instruction: + logger.info(f"Using instruction: '{instruction}' for task: '{task_name}'") + + embeddings = self.model.encode( + sentences, + prompt=instruction, + **kwargs, + ) + + if isinstance(embeddings, torch.Tensor): + # sometimes in kwargs can be return_tensors=True + embeddings = embeddings.cpu().detach().float().numpy() + return embeddings diff --git a/mteb/models/instructions.py b/mteb/models/instructions.py deleted file mode 100644 index ef439e42bb..0000000000 --- a/mteb/models/instructions.py +++ /dev/null @@ -1,430 +0,0 @@ -"""This specifies the default instructions for tasks within MTEB. These are optional to use and some models might want to use their own instructions.""" - -from __future__ import annotations - -import mteb - -# Prompts from -# SEB: https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/blob/c8376f967d1294419be1d3eb41217d04cd3a65d3/src/seb/registered_models/e5_instruct_models.py -# E5: https://github.com/microsoft/unilm/blob/9c0f1ff7ca53431fe47d2637dfe253643d94185b/e5/utils.py#L106 -DEFAULT_PROMPTS = { - "STS": "Retrieve semantically similar text.", - "Summarization": "Given a news summary, retrieve other semantically similar summaries", - "BitextMining": "Retrieve parallel sentences.", - "Classification": "Classify user passages", - "Clustering": "Identify categories in user passages", - "Reranking": "Retrieve text based on user query.", - "Retrieval": "Retrieve text based on user query.", - "InstructionRetrieval": "Retrieve text based on user query.", - "PairClassification": "Retrieve text that are semantically similar to the given text", -} - - -# This list is NOT comprehensive even for the tasks within MTEB -# TODO: We should probably move this prompt to the task object -TASKNAME2INSTRUCTIONS = { - # BitextMining - "BornholmBitextMining": "Retrieve parallel sentences in Danish and Bornholmsk", - "NorwegianCourtsBitextMining ": "Retrieve parallel sentences in Norwegian Bokmål and Nynorsk", - # Classification - "AngryTweetsClassification": "Classify Danish tweets by sentiment. (positive, negative, neutral)", - "DKHateClassification": "Classify Danish tweets based on offensiveness (offensive, not offensive)", - "DanishPoliticalCommentsClassification": "Classify Danish political comments for sentiment", - "DalajClassification": "Classify texts based on linguistic acceptability in Swedish", - "LccSentimentClassification": "Classify texts based on sentiment", - "NordicLangClassification": "Classify texts based on language", - "MassiveIntentClassification": "Given a user utterance as query, find the user intents", - "Massive Scenario": "Given a user utterance as query, find the user scenarios", - "NoRecClassification": "Classify Norwegian reviews by sentiment", - "SweRecClassification": "Classify Swedish reviews by sentiment", - "Norwegian parliament": "Classify parliament speeches in Norwegian based on political affiliation", - "ScalaClassification": "Classify passages in Scandinavian Languages based on linguistic acceptability", - "AmazonCounterfactualClassification": "Classify a given Amazon customer review text as either counterfactual or not-counterfactual", - "AmazonPolarityClassification": "Classify Amazon reviews into positive or negative sentiment", - "AmazonReviewsClassification": "Classify the given Amazon review into its appropriate rating category", - "Banking77Classification": "Given a online banking query, find the corresponding intents", - "EmotionClassification": "Classify the emotion expressed in the given Twitter message into one of the six emotions: anger, fear, joy, love, sadness, and surprise", - "ImdbClassification": "Classify the sentiment expressed in the given movie review text from the IMDB dataset", - "MassiveScenarioClassification": "Given a user utterance as query, find the user scenarios", - "MTOPDomainClassification": "Classify the intent domain of the given utterance in task-oriented conversation", - "MTOPIntentClassification": "Classify the intent of the given utterance in task-oriented conversation", - "ToxicConversationsClassification": "Classify the given comments as either toxic or not toxic", - "TweetSentimentExtractionClassification": "Classify the sentiment of a given tweet as either positive, negative, or neutral", - "TNews": "Classify the fine-grained category of the given news title", - "IFlyTek": "Given an App description text, find the appropriate fine-grained category", - "MultilingualSentiment": "Classify sentiment of the customer review into positive, neutral, or negative", - "JDReview": "Classify the customer review for iPhone on e-commerce platform into positive or negative", - "OnlineShopping": "Classify the customer review for online shopping into positive or negative", - "Waimai": "Classify the customer review from a food takeaway platform into positive or negative", - "RuReviewsClassification": "Classify product reviews into positive, negative or neutral sentiment", - "KinopoiskClassification": "Classify the sentiment expressed in the given movie review text", - "HeadlineClassification": "Classify the topic or theme of the given news headline", - "CEDRClassification": "Given a comment as query, find expressed emotions (joy, sadness, surprise, fear, and anger)", - "GeoreviewClassification": "Classify the organization rating based on the reviews", - "InappropriatenessClassification": "Classify the given message as either sensitive topic or not", - "RuSciBenchGRNTIClassification": "Classify the category of scientific papers based on the titles and abstracts", - "RuSciBenchOECDClassification": "Classify the category of scientific papers based on the titles and abstracts", - "SensitiveTopicsClassification": "Given a sentence as query, find sensitive topics", - # Clustering - "VGHierarchicalClusteringP2P": "Identify the categories (e.g. sports) of given articles in Norwegian", - "VGHierarchicalClusteringS2S": "Identify the categories (e.g. sports) of given articles in Norwegian", - "SNLHierarchicalClusteringP2P": "Identify categories in a Norwegian lexicon", - "SNLHierarchicalClusteringS2S": "Identify categories in a Norwegian lexicon", - "SwednClusteringP2P": "Identify news categories in Swedish passages", - "SwednClusteringS2S": "Identify news categories in Swedish passages", - "ArxivClusteringP2P": "Identify the main and secondary category of Arxiv papers based on the titles and abstracts", - "ArxivClusteringS2S": "Identify the main and secondary category of Arxiv papers based on the titles", - "BiorxivClusteringP2P": "Identify the main category of Biorxiv papers based on the titles and abstracts", - "BiorxivClusteringS2S": "Identify the main category of Biorxiv papers based on the titles", - "MedrxivClusteringP2P": "Identify the main category of Medrxiv papers based on the titles and abstracts", - "MedrxivClusteringS2S": "Identify the main category of Medrxiv papers based on the titles", - "RedditClustering": "Identify the topic or theme of Reddit posts based on the titles", - "RedditClusteringP2P": "Identify the topic or theme of Reddit posts based on the titles and posts", - "StackExchangeClustering": "Identify the topic or theme of StackExchange posts based on the titles", - "StackExchangeClusteringP2P": "Identify the topic or theme of StackExchange posts based on the given paragraphs", - "TwentyNewsgroupsClustering": "Identify the topic or theme of the given news articles", - "CLSClusteringS2S": "Identify the main category of scholar papers based on the titles", - "CLSClusteringP2P": "Identify the main category of scholar papers based on the titles and abstracts", - "ThuNewsClusteringS2S": "Identify the topic or theme of the given news articles based on the titles", - "ThuNewsClusteringP2P": "Identify the topic or theme of the given news articles based on the titles and contents", - "GeoreviewClusteringP2P": "Identify the organization category based on the reviews", - "RuSciBenchOECDClusteringP2P": "Identify the category of scientific papers based on the titles and abstracts", - "RuSciBenchGRNTIClusteringP2P": "Identify the category of scientific papers based on the titles and abstracts", - # Reranking and pair classification - "AskUbuntuDupQuestions": "Retrieve duplicate questions from AskUbuntu forum", - "MindSmallReranking": "Retrieve relevant news articles based on user browsing history", - "SciDocsRR": "Given a title of a scientific paper, retrieve the titles of other relevant papers", - "StackOverflowDupQuestions": "Retrieve duplicate questions from StackOverflow forum", - "SprintDuplicateQuestions": "Retrieve duplicate questions from Sprint forum", - "TwitterSemEval2015": "Retrieve tweets that are semantically similar to the given tweet", - "TwitterURLCorpus": "Retrieve tweets that are semantically similar to the given tweet", - "T2Reranking": "Given a Chinese search query, retrieve web passages that answer the question", - "MMarcoReranking": "Given a Chinese search query, retrieve web passages that answer the question", - "VoyageMMarcoReranking": "Given a Japanese search query, retrieve web passages that answer the question", - "CMedQAv1": "Given a Chinese community medical question, retrieve replies that best answer the question", - "CMedQAv2": "Given a Chinese community medical question, retrieve replies that best answer the question", - "Ocnli": "Retrieve semantically similar text.", - "Cmnli": "Retrieve semantically similar text.", - "TERRa": "Given a premise, retrieve a hypothesis that is entailed by the premise", - "RuBQReranking": ( - "Given a question, retrieve Wikipedia passages that answer the question", - "", - ), - "MIRACLReranking": ( - "Given a question, retrieve Wikipedia passages that answer the question", - "", - ), - # Retrieval - 1st item is query instruction; 2nd is corpus instruction - "TwitterHjerneRetrieval": ( - "Retrieve answers to questions asked in Danish tweets", - "", - ), - "SwednRetrieval": ( - "Given a Swedish news headline retrieve summaries or news articles", - "", - ), - "TV2Nordretrieval": ( - "Given a summary of a Danish news article retrieve the corresponding news article", - "", - ), - "DanFEVER": ( - "Given a claim in Danish, retrieve documents that support the claim", - "", - ), - "SNLRetrieval": ("Given a lexicon headline in Norwegian, retrieve its article", ""), - "NorQuadRetrieval": ( - "Given a question in Norwegian, retrieve the answer from Wikipedia articles", - "", - ), - "SweFaqRetrieval": ("Retrieve answers given questions in Swedish", ""), - "ArguAna": ("Given a claim, find documents that refute the claim", ""), - "ClimateFEVER": ( - "Given a claim about climate change, retrieve documents that support or refute the claim", - "", - ), - "DBPedia": ( - "Given a query, retrieve relevant entity descriptions from DBPedia", - "", - ), - "FEVER": ("Given a claim, retrieve documents that support or refute the claim", ""), - "FiQA2018": ( - "Given a financial question, retrieve user replies that best answer the question", - "", - ), - "HotpotQA": ( - "Given a multi-hop question, retrieve documents that can help answer the question", - "", - ), - "MSMARCO": ( - "Given a web search query, retrieve relevant passages that answer the query", - "", - ), - "NFCorpus": ( - "Given a question, retrieve relevant documents that best answer the question", - "", - ), - "NQ": ( - "Given a question, retrieve Wikipedia passages that answer the question", - "", - ), - "QuoraRetrieval": ( - "Given a question, retrieve questions that are semantically equivalent to the given question", - "", - ), - "SCIDOCS": ( - "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper", - "", - ), - "SciFact": ( - "Given a scientific claim, retrieve documents that support or refute the claim", - "", - ), - "Touche2020": ( - "Given a question, retrieve detailed and persuasive arguments that answer the question", - "", - ), - "TRECCOVID": ( - "Given a query on COVID-19, retrieve documents that answer the query", - "", - ), - "T2Retrieval": ( - "Given a Chinese search query, retrieve web passages that answer the question", - "", - ), - "MMarcoRetrieval": ( - "Given a web search query, retrieve relevant passages that answer the query", - "", - ), - "DuRetrieval": ( - "Given a Chinese search query, retrieve web passages that answer the question", - "", - ), - "CovidRetrieval": ( - "Given a question on COVID-19, retrieve news articles that answer the question", - "", - ), - "CmedqaRetrieval": ( - "Given a Chinese community medical question, retrieve replies that best answer the question", - "", - ), - "EcomRetrieval": ( - "Given a user query from an e-commerce website, retrieve description sentences of relevant products", - "", - ), - "MedicalRetrieval": ( - "Given a medical question, retrieve user replies that best answer the question", - "", - ), - "VideoRetrieval": ( - "Given a video search query, retrieve the titles of relevant videos", - "", - ), - "ARCChallenge": ( - "Retrieve the answer to the question.", - "", - ), - "AlphaNLI": ( - "Given the following start and end of a story, retrieve a possible reason that leads to the end.", - "", - ), - "HellaSwag": ( - "Given the following unfinished context, retrieve the most plausible ending to finish it.", - "", - ), - "PIQA": ( - "Given the following goal, retrieve a possible solution.", - "", - ), - "Quail": ( - "Given the following context and question, retrieve the correct answer.", - "", - ), - "SIQA": ( - "Given the following context and question, retrieve the correct answer.", - "", - ), - "RARbCode": ( - "Retrieve the answer for the following coding problem.", - "", - ), - "RARbMath": ( - "Retrieve the answer for the following math problem.", - "", - ), - "SpartQA": ( - "Given the following spatial reasoning question, retrieve the right answer.", - "", - ), - "TempReasonL1": ( - "Given the following question about time, retrieve the correct answer.", - "", - ), - "TempReasonL2Pure": ( - "Given the following question, retrieve the correct answer.", - "", - ), - "TempReasonL2Fact": ( - "Given the following question and facts, retrieve the correct answer.", - "", - ), - "TempReasonL2Context": ( - "Given the following question, facts and contexts, retrieve the correct answer.", - "", - ), - "TempReasonL3Pure": ( - "Given the following question, retrieve the correct answer.", - "", - ), - "TempReasonL3Fact": ( - "Given the following question and facts, retrieve the correct answer.", - "", - ), - "TempReasonL3Context": ( - "Given the following question, facts and contexts, retrieve the correct answer.", - "", - ), - "WinoGrande": ( - "Given the following sentence, retrieve an appropriate answer to fill in the missing underscored part.", - "", - ), - "RuBQRetrieval": ( - "Given a question, retrieve Wikipedia passages that answer the question", - "", - ), - "MIRACLRetrieval": ( - "Given a question, retrieve Wikipedia passages that answer the question", - "", - ), - "RiaNewsRetrieval": ("Given a news title, retrieve relevant news article", ""), - # Any2Any Retrieval - "WebQAT2TRetrieval": ( - "Retrieve passages from Wikipedia that provide answers to the following question.", - "", - ), - "NIGHTSI2IRetrieval": ( - "Find a day-to-day image that looks similar to the provided image.", - "", - ), - "VisualNewsT2IRetrieval": ( - "Identify the news-related image in line with the described event.", - "", - ), - "Fashion200kT2IRetrieval": ( - "Based on the following fashion description, retrieve the best matching image.", - "", - ), - "MSCOCOT2IRetrieval": ( - "Identify the image showcasing the described everyday scene.", - "", - ), - "Flickr30kT2IRetrieval": ("Find an image that matches the given caption.", ""), - "VidoreTatdqaRetrieval": ( - "Find a screenshot that relevant to the user's question.", - "", - ), - "VidoreArxivQARetrieval": ( - "Find a screenshot that relevant to the user's question.", - "", - ), - "VidoreDocVQARetrieval": ( - "Find a screenshot that relevant to the user's question.", - "", - ), - "VidoreInfoVQARetrieval": ( - "Find a screenshot that relevant to the user's question.", - "", - ), - "VidoreShiftProjectRetrieval": ( - "Find a screenshot that relevant to the user's question.", - "", - ), - "VidoreSyntheticDocQAAIRetrieval": ( - "Find a screenshot that relevant to the user's question.", - "", - ), - "VidoreSyntheticDocQAGovernmentReportsRetrieval": ( - "Find a screenshot that relevant to the user's question.", - "", - ), - "VidoreSyntheticDocQAHealthcareIndustryRetrieval": ( - "Find a screenshot that relevant to the user's question.", - "", - ), - "VidoreSyntheticDocQAEnergyRetrieval": ( - "Find a screenshot that relevant to the user's question.", - "", - ), - "VidoreTabfquadRetrieval": ( - "Find a screenshot that relevant to the user's question.", - "", - ), - "VisualNewsI2TRetrieval": ("Find a caption for the news in the given photo.", ""), - "Fashion200kI2TRetrieval": ( - "Based on the following fashion description, retrieve the best matching image.", - "", - ), - "MSCOCOI2TRetrieval": ( - "Find an image caption describing the following everyday image.", - "", - ), - "Flickr30kI2TRetrieval": ( - "Find an image caption describing the following image.", - "", - ), - "WebQAT2ITRetrieval": ("Find a Wikipedia image that answers this question.", ""), - "EDIST2ITRetrieval": ("Identify the news photo for the given caption.", ""), - "OVENIT2TRetrieval": ( - "Retrieve a Wikipedia paragraph that provides an answer to the given query about the image.", - "", - ), - "InfoSeekIT2TRetrieval": ( - "Find a paragraph from Wikipedia that answers my question about this image.", - "", - ), - "ReMuQIT2TRetrieval": ( - "Retrieve a fact-based paragraph that provides an answer to the given query about the image.", - "", - ), - "OKVQAIT2TRetrieval": ( - "Retrieve documents that provide an answer to the question alongside the image.", - "", - ), - "LLaVAIT2TRetrieval": ( - "Provide a specific decription of the image along with the following question.", - "", - ), - "FashionIQIT2IRetrieval": ( - "Find a fashion image that aligns with the reference image and style note.", - "", - ), - "CIRRIT2IRetrieval": ( - "Retrieve a day-to-day image that aligns with the modification instructions of the provided image.", - "", - ), - "OVENIT2ITRetrieval": ( - "Retrieve a Wikipedia image-description pair that provides evidence for the question of this image.", - "", - ), - "InfoSeekIT2ITRetrieval": ( - "Find an image and subject description from Wikipedia that answers my question about this image.", - "", - ), - "EncyclopediaVQAIT2ITRetrieval": ( - "Obtain illustrated documents that correspond to the inquiry alongside the provided image.", - "", - ), -} - - -def task_to_instruction(task_name: str, is_query: bool = True) -> str: - if task_name in TASKNAME2INSTRUCTIONS: - if isinstance(TASKNAME2INSTRUCTIONS[task_name], tuple): - return ( - TASKNAME2INSTRUCTIONS[task_name][0] - if is_query - else TASKNAME2INSTRUCTIONS[task_name][1] - ) - return TASKNAME2INSTRUCTIONS[task_name] - - meta = mteb.get_task(task_name).metadata - return DEFAULT_PROMPTS.get(meta.type, "") diff --git a/mteb/models/jasper_models.py b/mteb/models/jasper_models.py index dbd1615ad8..d0ff4ab681 100644 --- a/mteb/models/jasper_models.py +++ b/mteb/models/jasper_models.py @@ -90,8 +90,17 @@ def encode( use_instructions=True, adapted_from=None, superseded_by=None, - training_datasets=nvidia_training_datasets, # "In jasper model the teacher model is nvidia/NV-Embed-v2", source https://huggingface.co/infgrad/jasper_en_vision_language_v1 - # "non_mteb": ["BAAI/Infinity-MM", "HuggingFaceFW/fineweb-edu"], - public_training_code=None, - public_training_data=None, + training_datasets={ + # stage 1, 2, 3 + # "In jasper model the teacher model is nvidia/NV-Embed-v2", source https://huggingface.co/infgrad/jasper_en_vision_language_v1 + **nvidia_training_datasets, + # fineweb-edu + # https://huggingface.co/datasets/sentence-transformers/embedding-training-data + # stage 4 + # BAAI/Infinity-MM + }, + # training logs https://api.wandb.ai/links/dunnzhang0/z8jqoqpb + # more codes https://huggingface.co/NovaSearch/jasper_en_vision_language_v1/commit/da9b77d56c23d9398fa8f93af449102784f74e1d + public_training_code="https://github.com/NovaSearch-Team/RAG-Retrieval/blob/c40f4638b705eb77d88305d2056901ed550f9f4b/rag_retrieval/train/embedding/README.md", + public_training_data="https://huggingface.co/datasets/infgrad/jasper_text_distill_dataset", ) diff --git a/mteb/models/jina_models.py b/mteb/models/jina_models.py index e855ad3c7a..00641e9c89 100644 --- a/mteb/models/jina_models.py +++ b/mteb/models/jina_models.py @@ -245,6 +245,12 @@ def encode( jina_embeddings_v2_base_en = ModelMeta( + loader=partial( + SentenceTransformerWrapper, + model_name="jinaai/jina-embeddings-v2-base-en", + revision="6e85f575bc273f1fd840a658067d0157933c83f0", + trust_remote_code=True, + ), name="jinaai/jina-embeddings-v2-base-en", languages=["eng-Latn"], open_weights=True, @@ -266,6 +272,12 @@ def encode( ) jina_embeddings_v2_small_en = ModelMeta( + loader=partial( + SentenceTransformerWrapper, + model_name="jinaai/jina-embeddings-v2-small-en", + revision="796cff318cdd4e5fbe8b7303a1ef8cbec36996ef", + trust_remote_code=True, + ), name="jinaai/jina-embeddings-v2-small-en", languages=["eng-Latn"], open_weights=True, @@ -287,6 +299,12 @@ def encode( ) jina_embedding_b_en_v1 = ModelMeta( + loader=partial( + SentenceTransformerWrapper, + model_name="jinaai/jina-embedding-b-en-v1", + revision="aa0645035294a8c0607ce5bb700aba982cdff32c", + trust_remote_code=True, + ), name="jinaai/jina-embedding-b-en-v1", languages=["eng-Latn"], open_weights=True, @@ -308,6 +326,12 @@ def encode( ) jina_embedding_s_en_v1 = ModelMeta( + loader=partial( + SentenceTransformerWrapper, + model_name="jinaai/jina-embedding-s-en-v1", + revision="c1fed70aa4823a640f1a7150a276e4d3b08dce08", + trust_remote_code=True, + ), name="jinaai/jina-embedding-s-en-v1", languages=["eng-Latn"], open_weights=True, diff --git a/mteb/models/lens_models.py b/mteb/models/lens_models.py index 2cf055abd4..380724e53e 100644 --- a/mteb/models/lens_models.py +++ b/mteb/models/lens_models.py @@ -2,6 +2,8 @@ from mteb.model_meta import ModelMeta +from .bge_models import bge_full_data + lens_d4000 = ModelMeta( loader=None, # TODO: implement this in the future name="yibinlei/LENS-d4000", @@ -17,8 +19,8 @@ framework=["PyTorch"], use_instructions=True, public_training_code=None, - public_training_data=None, - training_datasets=None, + public_training_data="https://huggingface.co/datasets/cfli/bge-full-data", + training_datasets=bge_full_data, max_tokens=32768, ) @@ -37,7 +39,7 @@ framework=["PyTorch"], use_instructions=True, public_training_code=None, - public_training_data=None, - training_datasets=None, + public_training_data="https://huggingface.co/datasets/cfli/bge-full-data", + training_datasets=bge_full_data, max_tokens=32768, ) diff --git a/mteb/models/nvidia_models.py b/mteb/models/nvidia_models.py index 1997a85274..f3b313356a 100644 --- a/mteb/models/nvidia_models.py +++ b/mteb/models/nvidia_models.py @@ -1,17 +1,11 @@ from __future__ import annotations import logging -from collections.abc import Sequence from functools import partial -from typing import Any - -import numpy as np -import torch -from sentence_transformers import CrossEncoder, SentenceTransformer from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta -from mteb.models.sentence_transformer_wrapper import SentenceTransformerWrapper +from mteb.models.instruct_wrapper import InstructSentenceTransformerWrapper logger = logging.getLogger(__name__) @@ -22,56 +16,6 @@ def instruction_template( return f"Instruct: {instruction}\nQuery: " if instruction else "" -class NvEmbedWrapper(SentenceTransformerWrapper): - def __init__( - self, - model: str | SentenceTransformer | CrossEncoder, - revision: str | None = None, - model_prompts: dict[str, str] | None = None, - **kwargs, - ) -> None: - super().__init__(model, revision, model_prompts, **kwargs) - self.model.max_seq_length = 32768 - self.model.tokenizer.padding_side = "right" - logger.warning( - "Instructions are used in both query and docs, which may cause performance discrepancies from the original implementation." - ) - - def encode( - self, - sentences: Sequence[str], - *, - task_name: str, - prompt_type: PromptType | None = None, - **kwargs: Any, - ) -> np.ndarray: - # Add eos token to each input example - sentences = [example + self.model.tokenizer.eos_token for example in sentences] - - instruction = "" - if prompt_type == PromptType.query: - instruction = self.get_instruction(task_name, prompt_type) - - prompt = instruction_template(instruction) - - if prompt: - logger.info(f"Using {prompt=} for task={task_name} {prompt_type=}") - else: - logger.info(f"No model prompts found for task={task_name} {prompt_type=}") - - logger.info(f"Encoding {len(sentences)} sentences.") - - embeddings = self.model.encode( - sentences, - prompt=prompt, - normalize_embeddings=True, - **kwargs, - ) - if isinstance(embeddings, torch.Tensor): - embeddings = embeddings.cpu().detach().float().numpy() - return embeddings - - nvidia_training_datasets = { # source: https://arxiv.org/pdf/2405.17428 "ArguAna": ["train"], @@ -120,11 +64,18 @@ def encode( "STSBenchmark": ["train"], "STSBenchmarkMultilingualSTS": ["train"], # translated, not trained on } + NV_embed_v2 = ModelMeta( loader=partial( # type: ignore - NvEmbedWrapper, + InstructSentenceTransformerWrapper, model="nvidia/NV-Embed-v2", + revision="7604d305b621f14095a1aa23d351674c2859553a", + instruction_template=instruction_template, trust_remote_code=True, + max_seq_length=32768, + padding_side="right", + # for nv-embed, we add eos token to each input example + add_eos_token=True, ), name="nvidia/NV-Embed-v2", languages=["eng_Latn"], @@ -146,9 +97,15 @@ def encode( NV_embed_v1 = ModelMeta( loader=partial( # type: ignore - NvEmbedWrapper, + InstructSentenceTransformerWrapper, model="nvidia/NV-Embed-v1", + revision="7604d305b621f14095a1aa23d351674c2859553a", + instruction_template=instruction_template, trust_remote_code=True, + max_seq_length=32768, + padding_side="right", + # for nv-embed, we add eos token to each input example + add_eos_token=True, ), name="nvidia/NV-Embed-v1", languages=["eng_Latn"], diff --git a/mteb/models/overview.py b/mteb/models/overview.py index 7535c8939a..9137da2a79 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -14,6 +14,7 @@ from mteb.models import ( align_models, arctic_models, + bedrock_models, bge_models, blip2_models, blip_models, @@ -74,6 +75,7 @@ model_modules = [ align_models, arctic_models, + bedrock_models, bge_models, blip2_models, blip_models, diff --git a/mteb/models/salesforce_models.py b/mteb/models/salesforce_models.py index 235057a6f8..8c72265cc9 100644 --- a/mteb/models/salesforce_models.py +++ b/mteb/models/salesforce_models.py @@ -22,7 +22,6 @@ def instruction_template( "FiQA2018-PL": ["train"], "FEVER": ["train"], "FEVERHardNegatives": ["train"], - "FEVER-PL": ["train"], # translation not trained on "HotpotQA": ["train"], "HotpotQAHardNegatives": ["train"], "HotpotQA-PL": ["train"], # translation not trained on diff --git a/mteb/models/stella_models.py b/mteb/models/stella_models.py index 92d5db7c8a..9cc45a6e02 100644 --- a/mteb/models/stella_models.py +++ b/mteb/models/stella_models.py @@ -29,8 +29,7 @@ framework=["Sentence Transformers", "PyTorch", "GritLM"], reference="https://huggingface.co/dunzhang/stella_en_400M_v5", training_datasets=None, - # will be at https://github.com/NLPJCL/RAG-Retrieval - public_training_code=None, + public_training_code="https://github.com/NovaSearch-Team/RAG-Retrieval/blob/c40f4638b705eb77d88305d2056901ed550f9f4b/rag_retrieval/train/embedding/README.md", public_training_data=None, ) @@ -56,9 +55,8 @@ similarity_fn_name="cosine", framework=["Sentence Transformers", "PyTorch", "GritLM"], reference="https://huggingface.co/dunzhang/stella_en_1.5B_v5", - # will be at https://github.com/NLPJCL/RAG-Retrieval training_datasets=None, - public_training_code=None, + public_training_code="https://github.com/NovaSearch-Team/RAG-Retrieval/blob/c40f4638b705eb77d88305d2056901ed550f9f4b/rag_retrieval/train/embedding/README.md", public_training_data=None, ) diff --git a/mteb/models/voyage_models.py b/mteb/models/voyage_models.py index a637dee36a..3bcfb997bf 100644 --- a/mteb/models/voyage_models.py +++ b/mteb/models/voyage_models.py @@ -361,17 +361,17 @@ def _batched_encode( voyage_3_exp = ModelMeta( name="voyageai/voyage-3-m-exp", revision="1", - release_date=None, # not released - languages=None, # supported languages not specified + release_date="2025-01-08", + languages=["eng-Latn"], loader=partial( VoyageWrapper, model_name="voyage-3-m-exp", model_prompts=model_prompts, ), max_tokens=32000, - embed_dim=512, + embed_dim=2048, open_weights=False, - n_parameters=None, + n_parameters=int(6918 * 1e6), license=None, reference="https://huggingface.co/voyageai/voyage-3-m-exp", similarity_fn_name="cosine", diff --git a/mteb/tasks/BitextMining/__init__.py b/mteb/tasks/BitextMining/__init__.py index c176077215..1cec5d5ddc 100644 --- a/mteb/tasks/BitextMining/__init__.py +++ b/mteb/tasks/BitextMining/__init__.py @@ -1,6 +1,7 @@ from __future__ import annotations from .dan.BornholmskBitextMining import * +from .eng.PubChemSMILESBitextMining import * from .kat.TbilisiCityHallBitextMining import * from .multilingual.BibleNLPBitextMining import * from .multilingual.BUCCBitextMining import * diff --git a/mteb/tasks/BitextMining/eng/PubChemSMILESBitextMining.py b/mteb/tasks/BitextMining/eng/PubChemSMILESBitextMining.py new file mode 100644 index 0000000000..4951d8c596 --- /dev/null +++ b/mteb/tasks/BitextMining/eng/PubChemSMILESBitextMining.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining +from mteb.abstasks.MultilingualTask import MultilingualTask +from mteb.abstasks.TaskMetadata import TaskMetadata + +COL_MAPPING = { + "iso-title": {"title": "sentence1", "isomeric_smiles": "sentence2"}, + "iso-desc": {"description": "sentence1", "isomeric_smiles": "sentence2"}, + "canon-title": {"title": "sentence1", "canonical_smiles": "sentence2"}, + "canon-desc": {"description": "sentence1", "canonical_smiles": "sentence2"}, +} + +EVAL_LANGS = { + "iso-title": ["eng-Latn", "eng-Latn"], + "iso-desc": ["eng-Latn", "eng-Latn"], + "canon-title": ["eng-Latn", "eng-Latn"], + "canon-desc": ["eng-Latn", "eng-Latn"], +} + + +class PubChemSMILESBitextMining(MultilingualTask, AbsTaskBitextMining): + metadata = TaskMetadata( + name="PubChemSMILESBitextMining", + dataset={ + "path": "BASF-AI/PubChemSMILESBitextMining", + "revision": "36700ea628118312ebf2f90ad2353a9a8f188dc9", + }, + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + type="BitextMining", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=EVAL_LANGS, + main_score="f1", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + @article{kim2023pubchem, + title={PubChem 2023 update}, + author={Kim, Sunghwan and Chen, Jie and Cheng, Tiejun and Gindulyte, Asta and He, Jia and He, Siqian and Li, Qingliang and Shoemaker, Benjamin A and Thiessen, Paul A and Yu, Bo and others}, + journal={Nucleic acids research}, + volume={51}, + number={D1}, + pages={D1373--D1380}, + year={2023}, + publisher={Oxford University Press} + } + """, + ) + + def dataset_transform(self): + for subset in self.hf_subsets: + self.dataset[subset] = self.dataset[subset].rename_columns( + COL_MAPPING[subset] + ) diff --git a/mteb/tasks/BitextMining/eng/__init__.py b/mteb/tasks/BitextMining/eng/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Classification/__init__.py b/mteb/tasks/Classification/__init__.py index 3e80ae2181..b2aab22714 100644 --- a/mteb/tasks/Classification/__init__.py +++ b/mteb/tasks/Classification/__init__.py @@ -33,13 +33,31 @@ from .eng.NewsClassification import * from .eng.PatentClassification import * from .eng.PoemSentimentClassification import * +from .eng.SDSEyeProtectionClassification import * +from .eng.SDSGlovesClassification import * from .eng.ToxicChatClassification import * from .eng.ToxicConversationsClassification import * from .eng.TweetSentimentExtractionClassification import * from .eng.TweetTopicSingleClassification import * +from .eng.WikipediaBiolumNeurochemClassification import * +from .eng.WikipediaBioMetChemClassification import * +from .eng.WikipediaChemEngSpecialtiesClassification import * +from .eng.WikipediaChemFieldsClassification import * +from .eng.WikipediaChemistryTopicsClassification import * +from .eng.WikipediaCompChemSpectroscopyClassification import * +from .eng.WikipediaCryobiologySeparationClassification import * +from .eng.WikipediaCrystallographyAnalyticalClassification import * +from .eng.WikipediaGreenhouseEnantiopureClassification import * +from .eng.WikipediaIsotopesFissionClassification import * +from .eng.WikipediaLuminescenceClassification import * +from .eng.WikipediaOrganicInorganicClassification import * +from .eng.WikipediaSaltsSemiconductorsClassification import * +from .eng.WikipediaSolidStateColloidalClassification import * +from .eng.WikipediaTheoreticalAppliedClassification import * from .eng.YahooAnswersTopicsClassification import * from .eng.YelpReviewFullClassification import * from .est.estonian_valence import * +from .fas.FaMTEBClassification import * from .fas.PersianFoodSentimentClassification import * from .fil.FilipinoHateSpeechClassification import * from .fil.FilipinoShopeeReviewsClassification import * diff --git a/mteb/tasks/Classification/eng/FinancialPhrasebankClassification.py b/mteb/tasks/Classification/eng/FinancialPhrasebankClassification.py index 6ddb37c42a..b9abb5445a 100644 --- a/mteb/tasks/Classification/eng/FinancialPhrasebankClassification.py +++ b/mteb/tasks/Classification/eng/FinancialPhrasebankClassification.py @@ -22,7 +22,7 @@ class FinancialPhrasebankClassification(AbsTaskClassification): eval_langs=["eng-Latn"], main_score="accuracy", date=("2013-11-01", "2013-11-01"), - domains=["News", "Written"], + domains=["News", "Written", "Financial"], task_subtypes=["Sentiment/Hate speech"], license="cc-by-nc-sa-3.0", annotations_creators="expert-annotated", diff --git a/mteb/tasks/Classification/eng/SDSEyeProtectionClassification.py b/mteb/tasks/Classification/eng/SDSEyeProtectionClassification.py new file mode 100644 index 0000000000..197060ba0c --- /dev/null +++ b/mteb/tasks/Classification/eng/SDSEyeProtectionClassification.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class SDSEyeProtectionClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="SDSEyeProtectionClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/SDSEyeProtectionClassification", + "revision": "35cbe5ee544dd26e343238a333de4568e6f77819", + }, + type="Classification", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="LM-generated and reviewed", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + @inproceedings{pereira2020msds, + title={MSDS-OPP: Operator Procedures Prediction in Material Safety Data Sheets}, + author={Pereira, Eliseu}, + booktitle={15th Doctoral Symposium}, + pages={42}, + year={2020} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/SDSGlovesClassification.py b/mteb/tasks/Classification/eng/SDSGlovesClassification.py new file mode 100644 index 0000000000..ac471d58e9 --- /dev/null +++ b/mteb/tasks/Classification/eng/SDSGlovesClassification.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class SDSGlovesClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="SDSGlovesClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/SDSGlovesClassification", + "revision": "c723236c5ec417d79512e6104aca9d2cd88168f6", + }, + type="Classification", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="LM-generated and reviewed", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + @inproceedings{pereira2020msds, + title={MSDS-OPP: Operator Procedures Prediction in Material Safety Data Sheets}, + author={Pereira, Eliseu}, + booktitle={15th Doctoral Symposium}, + pages={42}, + year={2020} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaBioMetChemClassification.py b/mteb/tasks/Classification/eng/WikipediaBioMetChemClassification.py new file mode 100644 index 0000000000..3b494f46f6 --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaBioMetChemClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaBioMetChemClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaBioMetChemClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaEasy2GeneExpressionVsMetallurgyClassification", + "revision": "6ac491e5de9070c6dd434b31e76d3d379123dcff", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaBiolumNeurochemClassification.py b/mteb/tasks/Classification/eng/WikipediaBiolumNeurochemClassification.py new file mode 100644 index 0000000000..623ec8fc66 --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaBiolumNeurochemClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaBiolumNeurochemClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaBiolumNeurochemClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaMedium2BioluminescenceVsNeurochemistryClassification", + "revision": "2f68b7d34c2be896e46b14533573b366e59e5aae", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaChemEngSpecialtiesClassification.py b/mteb/tasks/Classification/eng/WikipediaChemEngSpecialtiesClassification.py new file mode 100644 index 0000000000..c95abcd4f2 --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaChemEngSpecialtiesClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaChemEngSpecialtiesClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaChemEngSpecialtiesClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaMedium5Classification", + "revision": "f81a76a2fb690e5d5bd7a26dd07e85cdf8405dfb", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaChemFieldsClassification.py b/mteb/tasks/Classification/eng/WikipediaChemFieldsClassification.py new file mode 100644 index 0000000000..7c0179fb1e --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaChemFieldsClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaChemFieldsClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaChemFieldsClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaEZ10Classification", + "revision": "a75fae77759acc115f015f2b856baa47776d733d", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaChemistryTopicsClassification.py b/mteb/tasks/Classification/eng/WikipediaChemistryTopicsClassification.py new file mode 100644 index 0000000000..02751b1a32 --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaChemistryTopicsClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaChemistryTopicsClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaChemistryTopicsClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaEasy10Classification", + "revision": "d8fb355db2248f95df8ea410a43aa1db1ee96ba4", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaCompChemSpectroscopyClassification.py b/mteb/tasks/Classification/eng/WikipediaCompChemSpectroscopyClassification.py new file mode 100644 index 0000000000..28a42ac044 --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaCompChemSpectroscopyClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaCompChemSpectroscopyClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaCompChemSpectroscopyClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaMedium2ComputationalVsSpectroscopistsClassification", + "revision": "474d706a22b0451b5846d623aa4b4234ba5b0513", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaCryobiologySeparationClassification.py b/mteb/tasks/Classification/eng/WikipediaCryobiologySeparationClassification.py new file mode 100644 index 0000000000..0e01454298 --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaCryobiologySeparationClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaCryobiologySeparationClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaCryobiologySeparationClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaEasy5Classification", + "revision": "858633e882dadd1ec6a0d220f7549bcafd379236", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaCrystallographyAnalyticalClassification.py b/mteb/tasks/Classification/eng/WikipediaCrystallographyAnalyticalClassification.py new file mode 100644 index 0000000000..724ffc4249 --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaCrystallographyAnalyticalClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaCrystallographyAnalyticalClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaCrystallographyAnalyticalClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaMedium2CrystallographyVsChromatographyTitrationpHClassification", + "revision": "740565a6a853aaed1114a13bdfd5fd46857b4f11", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaGreenhouseEnantiopureClassification.py b/mteb/tasks/Classification/eng/WikipediaGreenhouseEnantiopureClassification.py new file mode 100644 index 0000000000..b701584a70 --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaGreenhouseEnantiopureClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaGreenhouseEnantiopureClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaGreenhouseEnantiopureClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaEasy2GreenhouseVsEnantiopureClassification", + "revision": "0cfc1a83b6ed832454e8f4f93f7a0e26208274d9", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaIsotopesFissionClassification.py b/mteb/tasks/Classification/eng/WikipediaIsotopesFissionClassification.py new file mode 100644 index 0000000000..252ad85ed9 --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaIsotopesFissionClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaIsotopesFissionClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaIsotopesFissionClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaHard2IsotopesVsFissionProductsNuclearFissionClassification", + "revision": "897743346c7c794264f7dbfadc3978aa2895e8e2", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaLuminescenceClassification.py b/mteb/tasks/Classification/eng/WikipediaLuminescenceClassification.py new file mode 100644 index 0000000000..8e115b59d4 --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaLuminescenceClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaLuminescenceClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaLuminescenceClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaHard2BioluminescenceVsLuminescenceClassification", + "revision": "21c4dcebe2c5b36a35292e6441e7a10b59bf4896", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaOrganicInorganicClassification.py b/mteb/tasks/Classification/eng/WikipediaOrganicInorganicClassification.py new file mode 100644 index 0000000000..0ad784b69b --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaOrganicInorganicClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaOrganicInorganicClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaOrganicInorganicClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaEasy2SpecialClassification", + "revision": "96d1d9b37c4693f74c46c83d63a290573f78d511", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaSaltsSemiconductorsClassification.py b/mteb/tasks/Classification/eng/WikipediaSaltsSemiconductorsClassification.py new file mode 100644 index 0000000000..a409f87c8d --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaSaltsSemiconductorsClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaSaltsSemiconductorsClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaSaltsSemiconductorsClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaHard2SaltsVsSemiconductorMaterialsClassification", + "revision": "9e5415a096012fa2d1f3a929952cf9859e4550e7", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaSolidStateColloidalClassification.py b/mteb/tasks/Classification/eng/WikipediaSolidStateColloidalClassification.py new file mode 100644 index 0000000000..43f95c50f3 --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaSolidStateColloidalClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaSolidStateColloidalClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaSolidStateColloidalClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaEasy2SolidStateVsColloidalClassification", + "revision": "7d8df44e588b6143d4856c781f72f919fa0599a7", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/eng/WikipediaTheoreticalAppliedClassification.py b/mteb/tasks/Classification/eng/WikipediaTheoreticalAppliedClassification.py new file mode 100644 index 0000000000..f33b02f4bb --- /dev/null +++ b/mteb/tasks/Classification/eng/WikipediaTheoreticalAppliedClassification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaTheoreticalAppliedClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="WikipediaTheoreticalAppliedClassification", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaEZ2Classification", + "revision": "7896906653d31d7102a143d7f55d67cd688e3147", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Classification/fas/FaMTEBClassification.py b/mteb/tasks/Classification/fas/FaMTEBClassification.py new file mode 100644 index 0000000000..43c7971429 --- /dev/null +++ b/mteb/tasks/Classification/fas/FaMTEBClassification.py @@ -0,0 +1,635 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClassification import AbsTaskClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class SynPerChatbotConvSAAnger(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotConvSAAnger", + description="Synthetic Persian Chatbot Conversational Sentiment Analysis Anger", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-conversational-sentiment-analysis-anger", + "revision": "5cae68b7fc094cb2fa6890a464e4d836e8107f5e", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotConvSASatisfaction(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotConvSASatisfaction", + description="Synthetic Persian Chatbot Conversational Sentiment Analysis Satisfaction", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-conversational-sentiment-analysis-satisfaction", + "revision": "50fd9d5d09edd53af89af765636be5db6f983f0e", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotConvSAFriendship(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotConvSAFriendship", + description="Synthetic Persian Chatbot Conversational Sentiment Analysis Friendship", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-conversational-sentiment-analysis-friendship", + "revision": "9dae119101e9b4e9bb40d5b9d29ffd7a621f9942", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotConvSAFear(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotConvSAFear", + description="Synthetic Persian Chatbot Conversational Sentiment Analysis Fear", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-conversational-sentiment-analysis-fear", + "revision": "3c22f7e6bf4e366c86d69293c9164bf9e9d80aac", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotConvSAJealousy(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotConvSAJealousy", + description="Synthetic Persian Chatbot Conversational Sentiment Analysis Jealousy", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-conversational-sentiment-analysis-jealousy", + "revision": "0d5104ecaa109d2448afe1f40dbf860f5e4458a8", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotConvSASurprise(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotConvSASurprise", + description="Synthetic Persian Chatbot Conversational Sentiment Analysis Surprise", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-conversational-sentiment-analysis-surprise", + "revision": "62dad66fc2837b0ac5e5175fe7c265d2d502a386", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotConvSALove(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotConvSALove", + description="Synthetic Persian Chatbot Conversational Sentiment Analysis Love", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-conversational-sentiment-analysis-love", + "revision": "0e000b2f73e9bb74ec8fc6da10011c52725b8469", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotConvSASadness(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotConvSASadness", + description="Synthetic Persian Chatbot Conversational Sentiment Analysis Sadness", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-conversational-sentiment-analysis-sadness", + "revision": "e9c678325565a5e4dadc43fd6693a8ccff1dd6b2", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotConvSAHappiness(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotConvSAHappiness", + description="Synthetic Persian Chatbot Conversational Sentiment Analysis Happiness", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-conversational-sentiment-analysis-happiness", + "revision": "e60893b7a8d01c9b8c12fadfe8f0a06e9d548a63", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotConvSAToneChatbotClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotConvSAToneChatbotClassification", + description="Synthetic Persian Chatbot Conversational Sentiment Analysis Tone Chatbot Classification", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-conversational-sentiment-analysis-tone-chatbot-classification", + "revision": "1f403cfadb85004fbf7e2480334fffc4c999b4ab", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotConvSAToneUserClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotConvSAToneUserClassification", + description="Synthetic Persian Chatbot Conversational Sentiment Analysis Tone User", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/chatbot-conversational-sentiment-analysis-tone-user-classification", + "revision": "dd0f76661bef69819cc38c8a455b10af86ac3571", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotSatisfactionLevelClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotSatisfactionLevelClassification", + description="Synthetic Persian Chatbot Satisfaction Level Classification", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-satisfaction-level-classification", + "revision": "e72db473602d750f1bcdc9f0436e1e3b967e088f", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotRAGToneChatbotClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotRAGToneChatbotClassification", + description="Synthetic Persian Chatbot RAG Tone Chatbot Classification", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-rag-tone-chatbot-classification", + "revision": "76f15a203fc13bd98a8f0fdddab1b68c28d7d674", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotRAGToneUserClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotRAGToneUserClassification", + description="Synthetic Persian Chatbot RAG Tone User Classification", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-rag-tone-user-classification", + "revision": "f1f6ad83bb135dc94fbf1ca05c3ba164f5619369", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotToneChatbotClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotToneChatbotClassification", + description="Synthetic Persian Chatbot Tone Chatbot Classification", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-tone-chatbot-classification", + "revision": "a5a739a036fa7bb8ae0be91bc081fdd260d4bdab", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SynPerChatbotToneUserClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="SynPerChatbotToneUserClassification", + description="Synthetic Persian Chatbot Tone User Classification", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-tone-user-classification", + "revision": "780d629437f7be127c6b287a61776372f9f333b9", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class PersianTextTone(AbsTaskClassification): + metadata = TaskMetadata( + name="PersianTextTone", + description="Persian Text Tone", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/persian-text-tone", + "revision": "7144f4c6bdd77911df0dfc5a8bd44dba17e27e3a", + }, + type="Classification", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=[], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SIDClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="SIDClassification", + description="SID Classification", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/sid-classification", + "revision": "29bed651bb980395f5aa473607154d93226945e1", + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Academic"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class DeepSentiPers(AbsTaskClassification): + metadata = TaskMetadata( + name="DeepSentiPers", + description="Persian Sentiment Analysis Dataset", + reference="https://github.com/JoyeBright/DeepSentiPers", + dataset={ + "path": "PartAI/DeepSentiPers", + "revision": "ee4f09f404051761cfe14d68127532c82de41cb3", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Reviews"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + def dataset_transform(self): + self.dataset = self.dataset.rename_column("review", "text") + + +class PersianTextEmotion(AbsTaskClassification): + metadata = TaskMetadata( + name="PersianTextEmotion", + description="Emotion is a Persian dataset with six basic emotions: anger, fear, joy, love, sadness, and surprise.", + reference="https://huggingface.co/datasets/SeyedAli/Persian-Text-Emotion", + dataset={ + "path": "SeyedAli/Persian-Text-Emotion", + "revision": "518fcd2c8b89917c7696770672688217a2eabf88", + }, + type="Classification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=[], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class SentimentDKSF(AbsTaskClassification): + metadata = TaskMetadata( + name="SentimentDKSF", + description="The Sentiment DKSF (Digikala/Snappfood comments) is a dataset for sentiment analysis.", + reference="https://github.com/hezarai/hezar", + dataset={ + "path": "hezarai/sentiment-dksf", + "revision": "b4d5a8dd501db610b5ad89e9aa13f863b842b395", + }, + type="Classification", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Reviews"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + +class NLPTwitterAnalysisClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="NLPTwitterAnalysisClassification", + description="Twitter Analysis Classification", + reference="https://huggingface.co/datasets/hamedhf/nlp_twitter_analysis/tree/main", + dataset={ + "path": "hamedhf/nlp_twitter_analysis", + "revision": "4ceb1312583fd2c7c73ad2d550b726124dcd39a0", + }, + type="Classification", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Social"], + task_subtypes=["Sentiment/Hate speech"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + def dataset_transform(self): + self.dataset = self.dataset.rename_column("tweet", "text") + + +class DigikalamagClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="DigikalamagClassification", + description="A total of 8,515 articles scraped from Digikala Online Magazine. This dataset includes seven different classes.", + reference="https://hooshvare.github.io/docs/datasets/tc", + dataset={ + "path": "PNLPhub/DigiMag", + "revision": "969b335c9f50eda5c384460be4eb2b55505c2c53", + "trust_remote_code": True, + }, + type="Classification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="accuracy", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + samples_per_label = 32 + + def dataset_transform(self): + self.dataset = self.dataset.rename_column("content", "text") diff --git a/mteb/tasks/Classification/kor/KorFin.py b/mteb/tasks/Classification/kor/KorFin.py index a22b7d5cfe..1fdfb47694 100644 --- a/mteb/tasks/Classification/kor/KorFin.py +++ b/mteb/tasks/Classification/kor/KorFin.py @@ -25,7 +25,7 @@ class KorFin(AbsTaskClassification): "2022-01-01", "2022-12-31", ), # Assumed date based on the citations in the paper - domains=["News", "Written"], + domains=["News", "Written", "Financial"], task_subtypes=["Sentiment/Hate speech"], license="cc-by-sa-4.0", annotations_creators="expert-annotated", diff --git a/mteb/tasks/Clustering/__init__.py b/mteb/tasks/Clustering/__init__.py index 014796a4cb..65d8b01246 100644 --- a/mteb/tasks/Clustering/__init__.py +++ b/mteb/tasks/Clustering/__init__.py @@ -18,6 +18,9 @@ from .eng.StackExchangeClusteringP2P import * from .eng.TwentyNewsgroupsClustering import * from .eng.WikiCitiesClustering import * +from .eng.WikipediaChemistrySpecialtiesClustering import * +from .eng.WikipediaChemistryTopicsClustering import * +from .fas.FaMTEBClustering import * from .fra.AlloProfClusteringP2P import * from .fra.AlloProfClusteringS2S import * from .fra.HALClusteringS2S import * diff --git a/mteb/tasks/Clustering/eng/ArxivClusteringS2S.py b/mteb/tasks/Clustering/eng/ArxivClusteringS2S.py index c74766061d..8b4beb0e26 100644 --- a/mteb/tasks/Clustering/eng/ArxivClusteringS2S.py +++ b/mteb/tasks/Clustering/eng/ArxivClusteringS2S.py @@ -21,13 +21,13 @@ class ArxivClusteringS2S(AbsTaskClustering): eval_splits=["test"], eval_langs=["eng-Latn"], main_score="v_measure", - date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + date=("1991-01-01", "2021-01-01"), # 1991-01-01 is the first arxiv paper + domains=["Academic", "Written"], + task_subtypes=[], + license="cc0-1.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@misc{arxiv_org_submitters_2024, title={arXiv Dataset}, url={https://www.kaggle.com/dsv/7548853}, diff --git a/mteb/tasks/Clustering/eng/RedditClustering.py b/mteb/tasks/Clustering/eng/RedditClustering.py index c9efbe954a..84c6602c63 100644 --- a/mteb/tasks/Clustering/eng/RedditClustering.py +++ b/mteb/tasks/Clustering/eng/RedditClustering.py @@ -85,14 +85,13 @@ class RedditClustering(AbsTaskClustering): eval_splits=["test"], eval_langs=["eng-Latn"], main_score="v_measure", - date=None, - form=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + date=("2021-01-01", "2021-04-14"), + domains=["Web", "Social", "Written"], + task_subtypes=["Thematic clustering"], + license="not specified", # derived from pushshift + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@article{geigle:2021:arxiv, author = {Gregor Geigle and Nils Reimers and diff --git a/mteb/tasks/Clustering/eng/RedditClusteringP2P.py b/mteb/tasks/Clustering/eng/RedditClusteringP2P.py index 1e8d51cdfa..fc74844a2e 100644 --- a/mteb/tasks/Clustering/eng/RedditClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/RedditClusteringP2P.py @@ -29,14 +29,13 @@ class RedditClusteringP2P(AbsTaskClustering): eval_splits=["test"], eval_langs=["eng-Latn"], main_score="v_measure", - date=None, - form=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + date=("2021-01-01", "2021-04-14"), + domains=["Web", "Social", "Written"], + task_subtypes=["Thematic clustering"], + license="not specified", # derived from pushshift + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@article{geigle:2021:arxiv, author = {Gregor Geigle and Nils Reimers and diff --git a/mteb/tasks/Clustering/eng/StackExchangeClustering.py b/mteb/tasks/Clustering/eng/StackExchangeClustering.py index b123ab5bd1..c495b10de4 100644 --- a/mteb/tasks/Clustering/eng/StackExchangeClustering.py +++ b/mteb/tasks/Clustering/eng/StackExchangeClustering.py @@ -87,14 +87,13 @@ class StackExchangeClustering(AbsTaskClustering): eval_splits=["test"], eval_langs=["eng-Latn"], main_score="v_measure", - date=None, - form=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + date=("2021-01-01", "2021-04-14"), + domains=["Web", "Written"], + task_subtypes=["Thematic clustering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@article{geigle:2021:arxiv, author = {Gregor Geigle and Nils Reimers and diff --git a/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py b/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py index c411138e9f..a06eb82ae9 100644 --- a/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py +++ b/mteb/tasks/Clustering/eng/StackExchangeClusteringP2P.py @@ -91,13 +91,13 @@ class StackExchangeClusteringP2P(AbsTaskClustering): eval_splits=["test"], eval_langs=["eng-Latn"], main_score="v_measure", - date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + date=("2021-01-01", "2021-04-14"), + domains=["Web", "Written"], + task_subtypes=["Thematic clustering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@article{geigle:2021:arxiv, author = {Gregor Geigle and Nils Reimers and diff --git a/mteb/tasks/Clustering/eng/WikipediaChemistrySpecialtiesClustering.py b/mteb/tasks/Clustering/eng/WikipediaChemistrySpecialtiesClustering.py new file mode 100644 index 0000000000..a4e4082a69 --- /dev/null +++ b/mteb/tasks/Clustering/eng/WikipediaChemistrySpecialtiesClustering.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClustering import AbsTaskClustering +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaChemistrySpecialtiesClustering(AbsTaskClustering): + metadata = TaskMetadata( + name="WikipediaSpecialtiesInChemistryClustering", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaMedium5Clustering", + "revision": "7754d8d296f9f4c3af1c6426fab36304730ccddf", + }, + type="Clustering", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="v_measure", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Clustering/eng/WikipediaChemistryTopicsClustering.py b/mteb/tasks/Clustering/eng/WikipediaChemistryTopicsClustering.py new file mode 100644 index 0000000000..bfa5e1fcf3 --- /dev/null +++ b/mteb/tasks/Clustering/eng/WikipediaChemistryTopicsClustering.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskClustering import AbsTaskClustering +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class WikipediaChemistryTopicsClustering(AbsTaskClustering): + metadata = TaskMetadata( + name="WikipediaChemistryTopicsClustering", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/WikipediaEasy10Clustering", + "revision": "0a0886b06acbfc735bca6a71b21ce1e5cb92a37b", + }, + type="Clustering", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="v_measure", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + """, + ) diff --git a/mteb/tasks/Clustering/fas/FaMTEBClustering.py b/mteb/tasks/Clustering/fas/FaMTEBClustering.py new file mode 100644 index 0000000000..da0b8b53f3 --- /dev/null +++ b/mteb/tasks/Clustering/fas/FaMTEBClustering.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import numpy as np +from datasets import Dataset, DatasetDict + +from mteb.abstasks.AbsTaskClusteringFast import ( + AbsTaskClusteringFast, + check_label_distribution, +) +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class BeytooteClustering(AbsTaskClusteringFast): + metadata = TaskMetadata( + name="BeytooteClustering", + description="Beytoote Website Articles Clustering", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/beytoote-clustering", + "revision": "62ca5aecb9414214162569f2f1bfb07aa219a70e", + }, + type="Clustering", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="v_measure", + date=("2024-09-01", "2024-12-31"), + domains=["News"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + self.dataset = self.stratified_subsampling( + self.dataset, + seed=self.seed, + splits=["test"], + label="labels", + ) + + +class DigikalamagClustering(AbsTaskClusteringFast): + metadata = TaskMetadata( + name="DigikalamagClustering", + description="A total of 8,515 articles scraped from Digikala Online Magazine. This dataset includes seven different classes.", + reference="https://hooshvare.github.io/docs/datasets/tc", + dataset={ + "path": "PNLPhub/DigiMag", + "revision": "969b335c9f50eda5c384460be4eb2b55505c2c53", + "trust_remote_code": True, + }, + type="Clustering", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="v_measure", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + self.dataset = self.dataset.rename_columns( + {"label": "labels", "content": "sentences"} + ) + + self.dataset = self.stratified_subsampling( + self.dataset, + seed=self.seed, + splits=["test"], + label="labels", + ) + + +class HamshahriClustring(AbsTaskClusteringFast): + metadata = TaskMetadata( + name="HamshahriClustring", + description="These datasets have been extracted from the RSS feed of two Farsi news agency websites.", + reference="https://github.com/mallahyari/Farsi-datasets", + dataset={ + "path": "community-datasets/farsi_news", + "revision": "ca93dc707cea06cdb2e3ede3b547a1092053aca6", + }, + type="Clustering", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="v_measure", + date=("2024-09-01", "2024-12-31"), + domains=["News"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + self.dataset = self.dataset.map( + lambda x: {"sentences": f"{x['title']}\n: {x['summary']}"} + ) + self.dataset = self.dataset.map(lambda x: {"labels": x["tags"][0]}) + self.dataset = DatasetDict({"test": self.dataset["hamshahri"]}) + + ds = {} + for split in self.metadata.eval_splits: + labels = self.dataset[split]["labels"] + sentences = self.dataset[split]["sentences"] + + check_label_distribution(self.dataset[split]) + + # Remove sentences and labels with only 1 label example. + unique_labels, counts = np.unique(labels, return_counts=True) + solo_label_idx = np.where(counts == 1) + solo_labels = unique_labels[solo_label_idx] + is_solo = np.isin(labels, solo_labels) + split_ds = Dataset.from_dict({"labels": labels, "sentences": sentences}) + if is_solo.any(): + split_ds = split_ds.select(np.nonzero(is_solo == False)[0]) # noqa: E712 + ds[split] = split_ds + self.dataset = DatasetDict(ds) + + self.dataset = self.stratified_subsampling( + self.dataset, + seed=self.seed, + splits=["test"], + label="labels", + ) + + +class NLPTwitterAnalysisClustering(AbsTaskClusteringFast): + metadata = TaskMetadata( + name="NLPTwitterAnalysisClustering", + description="Clustering of tweets from twitter across 26 categories.", + reference="https://huggingface.co/datasets/hamedhf/nlp_twitter_analysis/commits/main", + dataset={ + "path": "hamedhf/nlp_twitter_analysis", + "revision": "4ceb1312583fd2c7c73ad2d550b726124dcd39a0", + }, + type="Clustering", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="v_measure", + date=("2024-09-01", "2024-12-31"), + domains=["Social"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + self.dataset = self.dataset.rename_column("tweet", "sentences") + self.dataset = self.dataset.rename_column("label", "labels") + self.dataset = self.stratified_subsampling( + self.dataset, + seed=self.seed, + splits=["test"], + label="labels", + ) + + +class SIDClustring(AbsTaskClusteringFast): + metadata = TaskMetadata( + name="SIDClustring", + description="Clustering of summariesfrom SIDClustring across categories.", + reference="https://www.sid.com/", + dataset={ + "path": "MCINext/sid-clustering", + "revision": "d361bb18535d592e845aeaaa8ac47a239aa2f87a", + }, + type="Clustering", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="v_measure", + date=("2024-09-01", "2024-12-31"), + domains=["Academic"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + self.dataset = self.stratified_subsampling( + self.dataset, + seed=self.seed, + splits=["test"], + label="labels", + ) diff --git a/mteb/tasks/Clustering/fas/__init__.py b/mteb/tasks/Clustering/fas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py index 78291c0f37..ed0172ae79 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/CIRRIT2IRetrieval.py @@ -34,6 +34,9 @@ class CIRRIT2IRetrieval(AbsTaskAny2AnyRetrieval): pages={2125--2134}, year={2021} }""", + prompt={ + "query": "Retrieve a day-to-day image that aligns with the modification instructions of the provided image." + }, descriptive_stats={ "n_samples": {"test": 4170}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/EDIST2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/EDIST2ITRetrieval.py index e1f8309066..ac7b310998 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/EDIST2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/EDIST2ITRetrieval.py @@ -33,6 +33,7 @@ class EDIST2ITRetrieval(AbsTaskAny2AnyRetrieval): pages={4877--4894}, year={2023} }""", + prompt={"query": "Identify the news photo for the given caption."}, descriptive_stats={ "n_samples": {"test": 3241}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/EncyclopediaVQAIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/EncyclopediaVQAIT2ITRetrieval.py index 4e24d13f7d..01f2e6a980 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/EncyclopediaVQAIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/EncyclopediaVQAIT2ITRetrieval.py @@ -33,6 +33,9 @@ class EncyclopediaVQAIT2ITRetrieval(AbsTaskAny2AnyRetrieval): pages={3113--3124}, year={2023} }""", + prompt={ + "query": "Obtain illustrated documents that correspond to the inquiry alongside the provided image." + }, descriptive_stats={ "n_samples": {"test": 3743}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py index cb67b9ad0b..5ba43daf1d 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kI2TRetrieval.py @@ -33,6 +33,9 @@ class Fashion200kI2TRetrieval(AbsTaskAny2AnyRetrieval): pages={1463--1471}, year={2017} }""", + prompt={ + "query": "Based on the following fashion description, retrieve the best matching image." + }, descriptive_stats={ "n_samples": {"test": 4889}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py index 2648fb8ee1..1511de7aa4 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Fashion200kT2IRetrieval.py @@ -34,6 +34,9 @@ class Fashion200kT2IRetrieval(AbsTaskAny2AnyRetrieval): pages={1463--1471}, year={2017} }""", + prompt={ + "query": "Based on the following fashion description, retrieve the best matching image." + }, descriptive_stats={ "n_samples": {"test": 1719}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py index d6099d1e0c..4e1209c23c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FashionIQIT2IRetrieval.py @@ -34,6 +34,9 @@ class FashionIQIT2IRetrieval(AbsTaskAny2AnyRetrieval): pages={11307--11317}, year={2021} }""", + prompt={ + "query": "Find a fashion image that aligns with the reference image and style note." + }, descriptive_stats={ "n_samples": {"test": 6003}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py index 9818f4de54..43aeea20d4 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py @@ -35,6 +35,7 @@ class Flickr30kI2TRetrieval(AbsTaskAny2AnyRetrieval): pages={67-78}, url={https://api.semanticscholar.org/CorpusID:3104920} }""", + prompt={"query": "Find an image caption describing the following image."}, descriptive_stats={ "n_samples": {"test": 1000}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py index 6e889f9c54..cb87cfcf86 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py @@ -35,6 +35,7 @@ class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval): pages={67-78}, url={https://api.semanticscholar.org/CorpusID:3104920} }""", + prompt={"query": "Find an image that matches the given caption."}, descriptive_stats={ "n_samples": {"test": 5000}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py index 38caf36f1a..f695de1d19 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2ITRetrieval.py @@ -34,6 +34,9 @@ class InfoSeekIT2ITRetrieval(AbsTaskAny2AnyRetrieval): pages={14948--14968}, year={2023} }""", + prompt={ + "query": "Find an image and subject description from Wikipedia that answers my question about this image." + }, descriptive_stats={ "n_samples": {"test": 17593}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py index ac8861f0aa..e5cecd8591 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/InfoSeekIT2TRetrieval.py @@ -34,6 +34,9 @@ class InfoSeekIT2TRetrieval(AbsTaskAny2AnyRetrieval): pages={14948--14968}, year={2023} }""", + prompt={ + "query": "Find a paragraph from Wikipedia that answers my question about this image." + }, descriptive_stats={ "n_samples": {"test": 11323}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/LLaVAIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/LLaVAIT2TRetrieval.py index e1c4d9ba2a..9a0ded2203 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/LLaVAIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/LLaVAIT2TRetrieval.py @@ -44,6 +44,9 @@ class LLaVAIT2TRetrieval(AbsTaskAny2AnyRetrieval): doi = "10.18653/v1/2024.acl-long.289", pages = "5294--5316", }""", + prompt={ + "query": "Provide a specific decription of the image along with the following question." + }, descriptive_stats={ "n_samples": {"test": 5120}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py index 2e84d22ee7..bc4ce63c72 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOI2TRetrieval.py @@ -35,6 +35,9 @@ class MSCOCOI2TRetrieval(AbsTaskAny2AnyRetrieval): year={2014}, organization={Springer} }""", + prompt={ + "query": "Find an image caption describing the following everyday image." + }, descriptive_stats={ "n_samples": {"test": 5000}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py index 1ad8aa7a04..4885e236c2 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/MSCOCOT2IRetrieval.py @@ -35,6 +35,7 @@ class MSCOCOT2IRetrieval(AbsTaskAny2AnyRetrieval): year={2014}, organization={Springer} }""", + prompt={"query": "Identify the image showcasing the described everyday scene."}, descriptive_stats={ "n_samples": {"test": 24809}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py index aed1805aae..aa05ac6494 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/NIGHTSI2IRetrieval.py @@ -33,6 +33,9 @@ class NIGHTSI2IRetrieval(AbsTaskAny2AnyRetrieval): volume={36}, year={2024} }""", + prompt={ + "query": "Find a day-to-day image that looks similar to the provided image." + }, descriptive_stats={ "n_samples": {"test": 2120}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OKVQAIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OKVQAIT2TRetrieval.py index a072f896e2..65b1c3b202 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OKVQAIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OKVQAIT2TRetrieval.py @@ -33,6 +33,9 @@ class OKVQAIT2TRetrieval(AbsTaskAny2AnyRetrieval): pages={3195--3204}, year={2019} }""", + prompt={ + "query": "Retrieve documents that provide an answer to the question alongside the image." + }, descriptive_stats={ "n_samples": {"test": 5046}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py index f355c43ecf..c6d1ef6baa 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2ITRetrieval.py @@ -33,6 +33,9 @@ class OVENIT2ITRetrieval(AbsTaskAny2AnyRetrieval): pages={12065--12075}, year={2023} }""", + prompt={ + "query": "Retrieve a Wikipedia image-description pair that provides evidence for the question of this image." + }, descriptive_stats={ "n_samples": {"test": 14741}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py index 308283454c..94898f4819 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/OVENIT2TRetrieval.py @@ -33,6 +33,9 @@ class OVENIT2TRetrieval(AbsTaskAny2AnyRetrieval): pages={12065--12075}, year={2023} }""", + prompt={ + "query": "Retrieve a Wikipedia paragraph that provides an answer to the given query about the image." + }, descriptive_stats={ "n_samples": {"test": 50004}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ReMuQIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ReMuQIT2TRetrieval.py index 00577d4ce8..648d2d2e44 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/ReMuQIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ReMuQIT2TRetrieval.py @@ -45,6 +45,9 @@ class ReMuQIT2TRetrieval(AbsTaskAny2AnyRetrieval): doi = "10.18653/v1/2023.acl-long.478", pages = "8573--8589", }""", + prompt={ + "query": "Retrieve a fact-based paragraph that provides an answer to the given query about the image." + }, descriptive_stats={ "n_samples": {"test": 3609}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py index f85e1dadd7..44d0d36cb0 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py @@ -98,6 +98,7 @@ class VidoreArxivQARetrieval(AbsTaskAny2AnyRetrieval): journal={arXiv preprint arXiv:2407.01449}, year={2024} }""", + prompt={"query": "Find a screenshot that relevant to the user's question."}, descriptive_stats={ "n_samples": None, "avg_character_length": { @@ -151,6 +152,7 @@ class VidoreDocVQARetrieval(AbsTaskAny2AnyRetrieval): journal={arXiv preprint arXiv:2407.01449}, year={2024} }""", + prompt={"query": "Find a screenshot that relevant to the user's question."}, descriptive_stats={ "n_samples": None, "avg_character_length": { @@ -204,6 +206,7 @@ class VidoreInfoVQARetrieval(AbsTaskAny2AnyRetrieval): journal={arXiv preprint arXiv:2407.01449}, year={2024} }""", + prompt={"query": "Find a screenshot that relevant to the user's question."}, descriptive_stats={ "n_samples": None, "avg_character_length": { @@ -257,6 +260,7 @@ class VidoreTabfquadRetrieval(AbsTaskAny2AnyRetrieval): journal={arXiv preprint arXiv:2407.01449}, year={2024} }""", + prompt={"query": "Find a screenshot that relevant to the user's question."}, descriptive_stats={ "n_samples": None, "avg_character_length": { @@ -310,6 +314,7 @@ class VidoreTatdqaRetrieval(AbsTaskAny2AnyRetrieval): journal={arXiv preprint arXiv:2407.01449}, year={2024} }""", + prompt={"query": "Find a screenshot that relevant to the user's question."}, descriptive_stats={ "n_samples": None, "avg_character_length": { @@ -363,6 +368,7 @@ class VidoreShiftProjectRetrieval(AbsTaskAny2AnyRetrieval): journal={arXiv preprint arXiv:2407.01449}, year={2024} }""", + prompt={"query": "Find a screenshot that relevant to the user's question."}, descriptive_stats={ "n_samples": None, "avg_character_length": { @@ -416,6 +422,7 @@ class VidoreSyntheticDocQAAIRetrieval(AbsTaskAny2AnyRetrieval): journal={arXiv preprint arXiv:2407.01449}, year={2024} }""", + prompt={"query": "Find a screenshot that relevant to the user's question."}, descriptive_stats={ "n_samples": None, "avg_character_length": { @@ -469,6 +476,7 @@ class VidoreSyntheticDocQAEnergyRetrieval(AbsTaskAny2AnyRetrieval): journal={arXiv preprint arXiv:2407.01449}, year={2024} }""", + prompt={"query": "Find a screenshot that relevant to the user's question."}, descriptive_stats={ "n_samples": None, "avg_character_length": { @@ -522,6 +530,7 @@ class VidoreSyntheticDocQAGovernmentReportsRetrieval(AbsTaskAny2AnyRetrieval): journal={arXiv preprint arXiv:2407.01449}, year={2024} }""", + prompt={"query": "Find a screenshot that relevant to the user's question."}, descriptive_stats={ "n_samples": None, "avg_character_length": { @@ -575,6 +584,7 @@ class VidoreSyntheticDocQAHealthcareIndustryRetrieval(AbsTaskAny2AnyRetrieval): journal={arXiv preprint arXiv:2407.01449}, year={2024} }""", + prompt={"query": "Find a screenshot that relevant to the user's question."}, descriptive_stats={ "n_samples": None, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py index ecce9f1e9a..2f79bfe9eb 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsI2TRetrieval.py @@ -33,6 +33,7 @@ class VisualNewsI2TRetrieval(AbsTaskAny2AnyRetrieval): pages={6761--6771}, year={2021} }""", + prompt={"query": "Find a caption for the news in the given photo."}, descriptive_stats={ "n_samples": {"test": 20000}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py index c700a5ab3c..1c5fa7fdbe 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VisualNewsT2IRetrieval.py @@ -33,6 +33,9 @@ class VisualNewsT2IRetrieval(AbsTaskAny2AnyRetrieval): pages={6761--6771}, year={2021} }""", + prompt={ + "query": "Identify the news-related image in line with the described event." + }, descriptive_stats={ "n_samples": {"test": 19995}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py index 583cae54dd..e3235c4912 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2ITRetrieval.py @@ -33,6 +33,7 @@ class WebQAT2ITRetrieval(AbsTaskAny2AnyRetrieval): pages={16495--16504}, year={2022} }""", + prompt={"query": "Find a Wikipedia image that answers this question."}, descriptive_stats={ "n_samples": {"test": 2511}, "avg_character_length": { diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py index eddc6e0fc4..4583e61221 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/WebQAT2TRetrieval.py @@ -33,6 +33,9 @@ class WebQAT2TRetrieval(AbsTaskAny2AnyRetrieval): pages={16495--16504}, year={2022} }""", + prompt={ + "query": "Retrieve passages from Wikipedia that provide answers to the following question." + }, descriptive_stats={ "n_samples": {"test": 2455}, "avg_character_length": { diff --git a/mteb/tasks/PairClassification/__init__.py b/mteb/tasks/PairClassification/__init__.py index c2057a4952..6cd75ea144 100644 --- a/mteb/tasks/PairClassification/__init__.py +++ b/mteb/tasks/PairClassification/__init__.py @@ -4,15 +4,21 @@ from .ces.CTKFactsNLI import * from .deu.FalseFriendsDeEnPC import * from .eng.LegalBenchPC import * +from .eng.PubChemAISentenceParaphrasePC import * +from .eng.PubChemSMILESPC import * +from .eng.PubChemSynonymPC import * +from .eng.PubChemWikiParagraphsPC import * from .eng.SprintDuplicateQuestionsPC import * from .eng.TwitterSemEval2015PC import * from .eng.TwitterURLCorpusPC import * +from .fas.FaMTEBPairClassification import * from .fas.FarsTail import * from .hye.ArmenianParaphrasePC import * from .ind.IndoNLI import * from .kor.KlueNLI import * from .multilingual.OpusparcusPC import * from .multilingual.PawsXPairClassification import * +from .multilingual.PubChemWikiPairClassification import * from .multilingual.RTE3 import * from .multilingual.XNLI import * from .multilingual.XStance import * diff --git a/mteb/tasks/PairClassification/eng/PubChemAISentenceParaphrasePC.py b/mteb/tasks/PairClassification/eng/PubChemAISentenceParaphrasePC.py new file mode 100644 index 0000000000..f453ebee31 --- /dev/null +++ b/mteb/tasks/PairClassification/eng/PubChemAISentenceParaphrasePC.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskPairClassification import AbsTaskPairClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class PubChemAISentenceParaphrasePC(AbsTaskPairClassification): + metadata = TaskMetadata( + name="PubChemAISentenceParaphrasePC", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/PubChemAISentenceParaphrasePC", + "revision": "f33a205966ce032f957c3a22f4f9e378f89a2c56", + }, + type="PairClassification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="max_ap", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="LM-generated", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + @article{kim2023pubchem, + title={PubChem 2023 update}, + author={Kim, Sunghwan and Chen, Jie and Cheng, Tiejun and Gindulyte, Asta and He, Jia and He, Siqian and Li, Qingliang and Shoemaker, Benjamin A and Thiessen, Paul A and Yu, Bo and others}, + journal={Nucleic acids research}, + volume={51}, + number={D1}, + pages={D1373--D1380}, + year={2023}, + publisher={Oxford University Press} + } + """, + ) + + def dataset_transform(self): + _dataset = {} + for split in self.metadata.eval_splits: + hf_dataset = self.dataset[split] + _dataset[split] = [ + { + "sentence1": hf_dataset["sent1"], + "sentence2": hf_dataset["sent2"], + "labels": hf_dataset["labels"], + } + ] + self.dataset = _dataset diff --git a/mteb/tasks/PairClassification/eng/PubChemSMILESPC.py b/mteb/tasks/PairClassification/eng/PubChemSMILESPC.py new file mode 100644 index 0000000000..b3e297e043 --- /dev/null +++ b/mteb/tasks/PairClassification/eng/PubChemSMILESPC.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import datasets + +from mteb.abstasks.AbsTaskPairClassification import AbsTaskPairClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + +_DATASET_COLUMN_MAP = [ + { + "name": "iso-desc", + "sent1": "description", + "sent2": "isomeric_smiles", + "labels": "labels", + }, + { + "name": "iso-title", + "sent1": "title", + "sent2": "isomeric_smiles", + "labels": "labels", + }, + { + "name": "canon-desc", + "sent1": "description", + "sent2": "canonical_smiles", + "labels": "labels", + }, + { + "name": "canon-title", + "sent1": "title", + "sent2": "canonical_smiles", + "labels": "labels", + }, +] + + +class PubChemSMILESPC(AbsTaskPairClassification): + metadata = TaskMetadata( + name="PubChemSMILESPC", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/PubChemSMILESPairClassification", + "revision": "7ba40b69f5fe6ffe4cc189aac9e1710913c73c8a", + }, + type="PairClassification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="max_ap", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + @article{kim2023pubchem, + title={PubChem 2023 update}, + author={Kim, Sunghwan and Chen, Jie and Cheng, Tiejun and Gindulyte, Asta and He, Jia and He, Siqian and Li, Qingliang and Shoemaker, Benjamin A and Thiessen, Paul A and Yu, Bo and others}, + journal={Nucleic acids research}, + volume={51}, + number={D1}, + pages={D1373--D1380}, + year={2023}, + publisher={Oxford University Press} + } + """, + ) + + def load_data(self): + """Load dataset from HuggingFace hub""" + if self.data_loaded: + return + + _hf_dataset = None + for dataset_col_map in _DATASET_COLUMN_MAP: + _dataset = datasets.load_dataset( + self.metadata.dataset["path"], + dataset_col_map["name"], + revision=self.metadata.dataset["revision"], + ) + + _dataset = _dataset.rename_columns( + { + dataset_col_map["sent1"]: "sentence1", + dataset_col_map["sent2"]: "sentence2", + dataset_col_map["labels"]: "labels", + } + ) + + if _hf_dataset is None: + _hf_dataset = _dataset + else: + _hf_dataset["test"] = datasets.concatenate_datasets( + [_hf_dataset["test"], _dataset["test"]] + ) + + self.dataset = _hf_dataset + self.dataset_transform() + self.data_loaded = True + + def dataset_transform(self): + self.dataset = self.stratified_subsampling( + self.dataset, + seed=self.seed, + splits=self.metadata["eval_splits"], + label="labels", + ) + + _dataset = {} + for split in self.metadata.eval_splits: + hf_dataset = self.dataset[split] + _dataset[split] = [ + { + "sentence1": hf_dataset["sentence1"], + "sentence2": hf_dataset["sentence2"], + "labels": hf_dataset["labels"], + } + ] + self.dataset = _dataset diff --git a/mteb/tasks/PairClassification/eng/PubChemSynonymPC.py b/mteb/tasks/PairClassification/eng/PubChemSynonymPC.py new file mode 100644 index 0000000000..6b6dfd81c8 --- /dev/null +++ b/mteb/tasks/PairClassification/eng/PubChemSynonymPC.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskPairClassification import AbsTaskPairClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class PubChemSynonymPC(AbsTaskPairClassification): + metadata = TaskMetadata( + name="PubChemSynonymPC", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/PubChemSynonymPC", + "revision": "5037d69d177c9628fb79cb57eea1299178b28c1b", + }, + type="PairClassification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="max_ap", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + @article{kim2023pubchem, + title={PubChem 2023 update}, + author={Kim, Sunghwan and Chen, Jie and Cheng, Tiejun and Gindulyte, Asta and He, Jia and He, Siqian and Li, Qingliang and Shoemaker, Benjamin A and Thiessen, Paul A and Yu, Bo and others}, + journal={Nucleic acids research}, + volume={51}, + number={D1}, + pages={D1373--D1380}, + year={2023}, + publisher={Oxford University Press} + } + """, + ) + + def dataset_transform(self): + _dataset = {} + + for split in self.metadata.eval_splits: + hf_dataset = self.dataset[split] + _dataset[split] = [ + { + "sentence1": hf_dataset["title"], + "sentence2": hf_dataset["synonyms"], + "labels": hf_dataset["labels"], + } + ] + self.dataset = _dataset diff --git a/mteb/tasks/PairClassification/eng/PubChemWikiParagraphsPC.py b/mteb/tasks/PairClassification/eng/PubChemWikiParagraphsPC.py new file mode 100644 index 0000000000..679580f28c --- /dev/null +++ b/mteb/tasks/PairClassification/eng/PubChemWikiParagraphsPC.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskPairClassification import AbsTaskPairClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class PubChemWikiParagraphsPC(AbsTaskPairClassification): + metadata = TaskMetadata( + name="PubChemWikiParagraphsPC", + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + dataset={ + "path": "BASF-AI/PubChemWikiParagraphsPC", + "revision": "7fb14716e4106b72f51a16e682e5cd2d67e9bd70", + }, + type="PairClassification", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="max_ap", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + @article{kim2023pubchem, + title={PubChem 2023 update}, + author={Kim, Sunghwan and Chen, Jie and Cheng, Tiejun and Gindulyte, Asta and He, Jia and He, Siqian and Li, Qingliang and Shoemaker, Benjamin A and Thiessen, Paul A and Yu, Bo and others}, + journal={Nucleic acids research}, + volume={51}, + number={D1}, + pages={D1373--D1380}, + year={2023}, + publisher={Oxford University Press} + } + """, + ) + + def dataset_transform(self): + _dataset = {} + for split in self.metadata.eval_splits: + hf_dataset = self.dataset[split] + _dataset[split] = [ + { + "sentence1": hf_dataset["sent1"], + "sentence2": hf_dataset["sent2"], + "labels": hf_dataset["labels"], + } + ] + self.dataset = _dataset diff --git a/mteb/tasks/PairClassification/eng/TwitterSemEval2015PC.py b/mteb/tasks/PairClassification/eng/TwitterSemEval2015PC.py index b8bc686d87..9da7c1072e 100644 --- a/mteb/tasks/PairClassification/eng/TwitterSemEval2015PC.py +++ b/mteb/tasks/PairClassification/eng/TwitterSemEval2015PC.py @@ -21,12 +21,12 @@ class TwitterSemEval2015PC(AbsTaskPairClassification): eval_langs=["eng-Latn"], main_score="max_ap", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Social", "Written"], + task_subtypes=[], + license="not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{xu-etal-2015-semeval, title = "{S}em{E}val-2015 Task 1: Paraphrase and Semantic Similarity in {T}witter ({PIT})", author = "Xu, Wei and diff --git a/mteb/tasks/PairClassification/eng/TwitterURLCorpusPC.py b/mteb/tasks/PairClassification/eng/TwitterURLCorpusPC.py index 24839e5938..85432b1d97 100644 --- a/mteb/tasks/PairClassification/eng/TwitterURLCorpusPC.py +++ b/mteb/tasks/PairClassification/eng/TwitterURLCorpusPC.py @@ -21,12 +21,12 @@ class TwitterURLCorpusPC(AbsTaskPairClassification): eval_langs=["eng-Latn"], main_score="max_ap", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Social", "Written"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{lan-etal-2017-continuously, title = "A Continuously Growing Dataset of Sentential Paraphrases", author = "Lan, Wuwei and diff --git a/mteb/tasks/PairClassification/fas/FaMTEBPairClassification.py b/mteb/tasks/PairClassification/fas/FaMTEBPairClassification.py new file mode 100644 index 0000000000..6deba76d8d --- /dev/null +++ b/mteb/tasks/PairClassification/fas/FaMTEBPairClassification.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskPairClassification import AbsTaskPairClassification +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class CExaPPC(AbsTaskPairClassification): + metadata = TaskMetadata( + name="CExaPPC", + description="ExaPPC is a large paraphrase corpus consisting of monolingual sentence-level paraphrases using different sources.", + reference="https://github.com/exaco/exappc", + dataset={ + "path": "PNLPhub/C-ExaPPC", + "revision": "68a0ff474739367a36c8066ee04802a65aefc117", + }, + type="PairClassification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="max_ap", + date=("2024-09-01", "2024-12-31"), + domains=["Social", "Web"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + _dataset = {} + self.dataset = self.dataset.map( + lambda example: {"label": 1 if example["label"] == "paraphrase" else 0} + ) + for split in self.metadata.eval_splits: + _dataset[split] = [ + { + "sentence1": self.dataset[split]["sentence1"], + "sentence2": self.dataset[split]["sentence2"], + "labels": self.dataset[split]["label"], + } + ] + self.dataset = _dataset + + +class SynPerChatbotRAGFAQPC(AbsTaskPairClassification): + metadata = TaskMetadata( + name="SynPerChatbotRAGFAQPC", + description="Synthetic Persian Chatbot RAG FAQ Pair Classification", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-rag-faq-pair-classification", + "revision": "2128d809e27ab8528906e2231f8e824516fb8e5a", + }, + type="PairClassification", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="max_ap", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=[], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + _dataset = {} + for split in self.metadata.eval_splits: + _dataset[split] = [ + { + "sentence1": self.dataset[split]["sent1"][0], + "sentence2": self.dataset[split]["sent2"][0], + "labels": self.dataset[split]["labels"][0], + } + ] + self.dataset = _dataset + + +class FarsiParaphraseDetection(AbsTaskPairClassification): + metadata = TaskMetadata( + name="FarsiParaphraseDetection", + description="Farsi Paraphrase Detection", + reference="https://huggingface.co/datasets/alighasemi/farsi_paraphrase_detection", + dataset={ + "path": "alighasemi/farsi_paraphrase_detection", + "revision": "c8129741af418d9ae43cfc1fc4f285704e26035f", + }, + type="PairClassification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="max_ap", + date=("2024-09-01", "2024-12-31"), + domains=[], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + _dataset = {} + for split in self.metadata.eval_splits: + _dataset[split] = [ + { + "sentence1": self.dataset[split]["sentence1"], + "sentence2": self.dataset[split]["sentence2"], + "labels": self.dataset[split]["label"], + } + ] + self.dataset = _dataset + + +class SynPerTextKeywordsPC(AbsTaskPairClassification): + metadata = TaskMetadata( + name="SynPerTextKeywordsPC", + description="Synthetic Persian Text Keywords Pair Classification", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-text-keyword-pair-classification", + "revision": "ea9a840cb163b415cc70b2f7adf2554feae159dc", + }, + type="PairClassification", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="max_ap", + date=("2024-09-01", "2024-12-31"), + domains=["Web", "News", "Religious", "Blog"], + task_subtypes=[], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + _dataset = {} + for split in self.metadata.eval_splits: + _dataset[split] = [ + { + "sentence1": self.dataset[split]["sent1"][0], + "sentence2": self.dataset[split]["sent2"][0], + "labels": self.dataset[split]["labels"][0], + } + ] + self.dataset = _dataset + + +class SynPerQAPC(AbsTaskPairClassification): + metadata = TaskMetadata( + name="SynPerQAPC", + description="Synthetic Persian QA Pair Classification", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-qa-pair-classification", + "revision": "d1b62ef31bebbb48ae01867993a1e583c2ce7d93", + }, + type="PairClassification", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="max_ap", + date=("2024-09-01", "2024-12-31"), + domains=["Web", "News", "Religious", "Blog"], + task_subtypes=[], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + _dataset = {} + for split in self.metadata.eval_splits: + _dataset[split] = [ + { + "sentence1": self.dataset[split]["sent1"][0], + "sentence2": self.dataset[split]["sent2"][0], + "labels": self.dataset[split]["labels"][0], + } + ] + self.dataset = _dataset + + +class ParsinluEntail(AbsTaskPairClassification): + metadata = TaskMetadata( + name="ParsinluEntail", + description="A Persian textual entailment task (deciding sent1 entails sent2). The questions are partially translated from the SNLI dataset and partially generated by expert annotators.", + reference="https://github.com/persiannlp/parsinlu", + dataset={ + "path": "persiannlp/parsinlu_entailment", + "revision": "c49b2d8fa0d6476520695c52207690b7ec854043", + "trust_remote_code": True, + }, + type="PairClassification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="max_ap", + date=("2024-09-01", "2024-12-31"), + domains=[], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + _dataset = {} + self.dataset = self.dataset.filter(lambda x: x["label"] != "n") + self.dataset = self.dataset.map( + lambda example: {"label": 1 if example["label"] == "e" else 0} + ) + for split in self.metadata.eval_splits: + _dataset[split] = [ + { + "sentence1": self.dataset[split]["sent1"], + "sentence2": self.dataset[split]["sent2"], + "labels": self.dataset[split]["label"], + } + ] + self.dataset = _dataset + + +class ParsinluQueryParaphPC(AbsTaskPairClassification): + metadata = TaskMetadata( + name="ParsinluQueryParaphPC", + description="A Persian query paraphrasng task (deciding whether two questions are paraphrases of each other). The questions are partially generated from Google auto-complete, and partially translated from the Quora paraphrasing dataset.", + reference="https://huggingface.co/datasets/persiannlp/parsinlu_query_paraphrasing", + dataset={ + "path": "persiannlp/parsinlu_query_paraphrasing", + "revision": "ec675bb3ac50c1a52317c101fe1d724b4601f47a", + "trust_remote_code": True, + }, + type="PairClassification", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="max_ap", + date=("2024-09-01", "2024-12-31"), + domains=[], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + _dataset = {} + self.dataset = self.dataset.map( + lambda example: {"label": 1 if example["label"] == "1" else 0} + ) + for split in self.metadata.eval_splits: + _dataset[split] = [ + { + "sentence1": self.dataset[split]["q1"], + "sentence2": self.dataset[split]["q2"], + "labels": self.dataset[split]["label"], + } + ] + self.dataset = _dataset diff --git a/mteb/tasks/PairClassification/multilingual/PubChemWikiPairClassification.py b/mteb/tasks/PairClassification/multilingual/PubChemWikiPairClassification.py new file mode 100644 index 0000000000..59a0605a82 --- /dev/null +++ b/mteb/tasks/PairClassification/multilingual/PubChemWikiPairClassification.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskPairClassification import AbsTaskPairClassification +from mteb.abstasks.MultilingualTask import MultilingualTask +from mteb.abstasks.TaskMetadata import TaskMetadata + +_LANGUAGES = { + "de": ["deu-Latn", "eng-Latn"], + "nl": ["nld-Latn", "eng-Latn"], + "zh": ["zho-Hans", "eng-Latn"], + "fr": ["fra-Latn", "eng-Latn"], + "es": ["spa-Latn", "eng-Latn"], + "pt": ["por-Latn", "eng-Latn"], + "ms": ["msa-Latn", "eng-Latn"], + "ko": ["kor-Hang", "eng-Latn"], + "tr": ["tur-Latn", "eng-Latn"], + "hi": ["hin-Deva", "eng-Latn"], + "cs": ["ces-Latn", "eng-Latn"], + "ja": ["jpn-Jpan", "eng-Latn"], +} + + +class PubChemWikiPairClassification(AbsTaskPairClassification, MultilingualTask): + metadata = TaskMetadata( + name="PubChemWikiPairClassification", + dataset={ + "path": "BASF-AI/PubChemWikiMultilingualPC", + "revision": "3412b208896a37e4ebb5ff7b96f6cc313ee9d2e3", + }, + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + category="s2s", + modalities=["text"], + type="PairClassification", + eval_splits=["test"], + eval_langs=_LANGUAGES, + main_score="max_ap", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="created", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + @article{kim2023pubchem, + title={PubChem 2023 update}, + author={Kim, Sunghwan and Chen, Jie and Cheng, Tiejun and Gindulyte, Asta and He, Jia and He, Siqian and Li, Qingliang and Shoemaker, Benjamin A and Thiessen, Paul A and Yu, Bo and others}, + journal={Nucleic acids research}, + volume={51}, + number={D1}, + pages={D1373--D1380}, + year={2023}, + publisher={Oxford University Press} + } + """, + ) + + def dataset_transform(self) -> None: + _dataset = {} + for lang in self.hf_subsets: + _dataset[lang] = {} + hf_dataset = self.dataset[lang][self.metadata.eval_splits[0]] + _dataset[lang]["test"] = [ + { + "sentence1": hf_dataset["sent1"], + "sentence2": hf_dataset["sent2"], + "labels": hf_dataset["labels"], + } + ] + self.dataset = _dataset diff --git a/mteb/tasks/Reranking/eng/AskUbuntuDupQuestions.py b/mteb/tasks/Reranking/eng/AskUbuntuDupQuestions.py index 90fe689cdd..b9dfde0055 100644 --- a/mteb/tasks/Reranking/eng/AskUbuntuDupQuestions.py +++ b/mteb/tasks/Reranking/eng/AskUbuntuDupQuestions.py @@ -21,12 +21,12 @@ class AskUbuntuDupQuestions(AbsTaskReranking): eval_langs=["eng-Latn"], main_score="map", date=None, - domains=None, + domains=["Programming", "Web"], task_subtypes=None, license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", prompt="Retrieve duplicate questions from AskUbuntu forum", bibtex_citation="""@article{wang-2021-TSDAE, title = "TSDAE: Using Transformer-based Sequential Denoising Auto-Encoderfor Unsupervised Sentence Embedding Learning", diff --git a/mteb/tasks/Reranking/eng/StackOverflowDupQuestions.py b/mteb/tasks/Reranking/eng/StackOverflowDupQuestions.py index 9e47461620..897f9d7bc9 100644 --- a/mteb/tasks/Reranking/eng/StackOverflowDupQuestions.py +++ b/mteb/tasks/Reranking/eng/StackOverflowDupQuestions.py @@ -20,13 +20,13 @@ class StackOverflowDupQuestions(AbsTaskReranking): eval_splits=["test"], eval_langs=["eng-Latn"], main_score="map", - date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + date=("2014-01-21", "2018-01-01"), + domains=["Written", "Blog", "Programming"], + task_subtypes=["Question answering"], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", prompt="Retrieve duplicate questions from StackOverflow forum", bibtex_citation="""@article{Liu2018LinkSOAD, title={LinkSO: a dataset for learning to retrieve similar question answer pairs on software development forums}, diff --git a/mteb/tasks/Retrieval/__init__.py b/mteb/tasks/Retrieval/__init__.py index d83df7ec5e..06414da081 100644 --- a/mteb/tasks/Retrieval/__init__.py +++ b/mteb/tasks/Retrieval/__init__.py @@ -5,6 +5,7 @@ from .code.CodeEditSearchRetrieval import * from .code.CodeFeedbackMTRetrieval import * from .code.CodeFeedbackSTRetrieval import * +from .code.CodeRAG import * from .code.CodeSearchNetCCRetrieval import * from .code.CodeSearchNetRetrieval import * from .code.CodeTransOceanContestRetrieval import * @@ -29,6 +30,8 @@ from .eng.ARCChallengeRetrieval import * from .eng.ArguAnaRetrieval import * from .eng.BrightRetrieval import * +from .eng.ChemHotpotQARetrieval import * +from .eng.ChemNQRetrieval import * from .eng.ClimateFEVERRetrieval import * from .eng.CQADupstackAndroidRetrieval import * from .eng.CQADupstackEnglishRetrieval import * @@ -101,6 +104,8 @@ from .eng.TRECCOVIDRetrieval import * from .eng.WinoGrandeRetrieval import * from .est.estqa import * +from .fas.BEIRFa import * +from .fas.FaMTEBRetrieval import * from .fra.AlloprofRetrieval import * from .fra.BSARDRetrieval import * from .fra.FQuADRetrieval import * diff --git a/mteb/tasks/Retrieval/code/CodeRAG.py b/mteb/tasks/Retrieval/code/CodeRAG.py new file mode 100644 index 0000000000..3724f44eca --- /dev/null +++ b/mteb/tasks/Retrieval/code/CodeRAG.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +import datasets + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +def split_by_first_newline(s): + # Split the string by the first newline + parts = s.split("\n", 1) + # Return parts or (s, '') if no newline + return parts if len(parts) > 1 else (s, "") + + +common_args = { + "reference": "https://arxiv.org/pdf/2406.14497", + "type": "Reranking", + "category": "s2s", + "modalities": ["text"], + "eval_splits": ["train"], + "eval_langs": ["python-Code"], + "main_score": "ndcg_at_10", + "date": ("2024-06-02", "2024-06-02"), # best guess + "domains": ["Programming"], + "task_subtypes": ["Code retrieval"], + "license": "cc-by-sa-4.0", + "annotations_creators": "derived", + "dialect": [], + "sample_creation": "found", + "bibtex_citation": """ + @misc{wang2024coderagbenchretrievalaugmentcode, + title={CodeRAG-Bench: Can Retrieval Augment Code Generation?}, + author={Zora Zhiruo Wang and Akari Asai and Xinyan Velocity Yu and Frank F. Xu and Yiqing Xie and Graham Neubig and Daniel Fried}, + year={2024}, + eprint={2406.14497}, + archivePrefix={arXiv}, + primaryClass={cs.SE}, + url={https://arxiv.org/abs/2406.14497}, + } + """, +} + + +class CodeRAGProgrammingSolutionsRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CodeRAGProgrammingSolutions", + description="Evaluation of programming solution retrieval using CodeRAG-Bench. Tests the ability to retrieve relevant programming solutions given code-related queries.", + dataset={ + "path": "code-rag-bench/programming-solutions", + "revision": "1064f7bba54d5400d4836f5831fe4c2332a566a6", + }, + **common_args, # type: ignore + ) + + def load_data(self, **kwargs): + """Load dataset from HuggingFace hub""" + if self.data_loaded: + return + self.dataset = datasets.load_dataset(**self.metadata.dataset) # type: ignore + self.dataset_transform() + self.data_loaded = True + + def dataset_transform(self) -> None: + """And transform to a retrieval datset, which have the following attributes + + self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text + self.queries = Dict[query_id, str] #id => query + self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + """ + self.corpus = {} + self.relevant_docs = {} + self.queries = {} + + split = self.metadata.eval_splits[0] + ds: datasets.Dataset = self.dataset[split] # type: ignore + ds = ds.shuffle(seed=42) + + self.queries[split] = {} + self.relevant_docs[split] = {} + self.corpus[split] = {} + + texts = ds["text"] + meta = ds["meta"] + for text, mt in zip(texts, meta): + # in code-rag-bench, + # text = query + "\n" + doc(code) + query, doc = split_by_first_newline(text) + + id = mt["task_id"] + + query_id = id + doc_id = f"doc_{id}" + self.queries[split][query_id] = query + self.corpus[split][doc_id] = {"title": "", "text": doc} + + self.relevant_docs[split][query_id] = { + doc_id: 1 + } # only one correct matches + + +class CodeRAGOnlineTutorialsRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CodeRAGOnlineTutorials", + description="Evaluation of online programming tutorial retrieval using CodeRAG-Bench. Tests the ability to retrieve relevant tutorials from online platforms given code-related queries.", + dataset={ + "path": "code-rag-bench/online-tutorials", + "revision": "095bb77130082e4690d6c3a031997b03487bf6e2", + }, + **common_args, # type: ignore + ) + + def load_data(self, **kwargs): + """Load dataset from HuggingFace hub""" + if self.data_loaded: + return + self.dataset = datasets.load_dataset(**self.metadata.dataset) # type: ignore + self.dataset_transform() + self.data_loaded = True + + def dataset_transform(self) -> None: + """And transform to a retrieval datset, which have the following attributes + + self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text + self.queries = Dict[query_id, str] #id => query + self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + """ + self.corpus = {} + self.relevant_docs = {} + self.queries = {} + + split = self.metadata.eval_splits[0] + ds: datasets.Dataset = self.dataset[split] # type: ignore + ds = ds.shuffle(seed=42) + + self.queries[split] = {} + self.relevant_docs[split] = {} + self.corpus[split] = {} + + titles = ds["title"] + texts = ds["text"] + parsed = ds["parsed"] + id = 0 + for title, text, mt in zip(titles, texts, parsed): + # in code-rag-bench, + # query=doc(code) + # text=query+doc(code) + query, doc = title, text + + query_id = str(id) + doc_id = f"doc_{id}" + self.queries[split][query_id] = query + self.corpus[split][doc_id] = {"title": "", "text": doc} + + self.relevant_docs[split][query_id] = { + doc_id: 1 + } # only one correct matches + + id += 1 + + +class CodeRAGLibraryDocumentationSolutionsRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CodeRAGLibraryDocumentationSolutions", + description="Evaluation of code library documentation retrieval using CodeRAG-Bench. Tests the ability to retrieve relevant Python library documentation sections given code-related queries.", + dataset={ + "path": "code-rag-bench/library-documentation", + "revision": "b530d3b5a25087d2074e731b76232db85b9e9107", + }, + **common_args, # type: ignore + ) + + def load_data(self, **kwargs): + """Load dataset from HuggingFace hub""" + if self.data_loaded: + return + self.dataset = datasets.load_dataset(**self.metadata.dataset) # type: ignore + self.dataset_transform() + self.data_loaded = True + + def dataset_transform(self) -> None: + """And transform to a retrieval datset, which have the following attributes + + self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text + self.queries = Dict[query_id, str] #id => query + self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + """ + self.corpus = {} + self.relevant_docs = {} + self.queries = {} + + split = self.metadata.eval_splits[0] + ds: datasets.Dataset = self.dataset[split] # type: ignore + ds = ds.shuffle(seed=42) + + self.queries[split] = {} + self.relevant_docs[split] = {} + self.corpus[split] = {} + + texts = ds["doc_content"] + + id = 0 + for text in texts: + # text format "document title \n document content" + query, doc = split_by_first_newline(text) + + # some library documents doesn't have query-doc pair + if not doc: + continue + query_id = str(id) + doc_id = f"doc_{id}" + self.queries[split][query_id] = query + self.corpus[split][doc_id] = {"title": "", "text": doc} + # only one correct match + self.relevant_docs[split][query_id] = {doc_id: 1} + id += 1 + + +class CodeRAGStackoverflowPostsRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CodeRAGStackoverflowPosts", + description="Evaluation of StackOverflow post retrieval using CodeRAG-Bench. Tests the ability to retrieve relevant StackOverflow posts given code-related queries.", + dataset={ + "path": "code-rag-bench/stackoverflow-posts", + "revision": "04e05d86cb0ac467b29a5d87f4c56eac99dfc0a4", + }, + **common_args, # type: ignore + ) + + def load_data(self, **kwargs): + """Load dataset from HuggingFace hub""" + if self.data_loaded: + return + self.dataset = datasets.load_dataset(**self.metadata.dataset) # type: ignore + self.dataset_transform() + self.data_loaded = True + + def dataset_transform(self) -> None: + """And transform to a retrieval datset, which have the following attributes + + self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text + self.queries = Dict[query_id, str] #id => query + self.relevant_docs = Dict[query_id, Dict[[doc_id, score]] + """ + self.corpus = {} + self.relevant_docs = {} + self.queries = {} + + split = self.metadata.eval_splits[0] + ds: datasets.Dataset = self.dataset[split] # type: ignore + ds = ds.shuffle(seed=42) + + self.queries[split] = {} + self.relevant_docs[split] = {} + self.corpus[split] = {} + + texts = ds["text"] + id = 0 + for text in texts: + # in code-rag-bench, + # text = query + "\n" + doc + query, doc = split_by_first_newline(text) + + query_id = str(id) + doc_id = f"doc_{id}" + self.queries[split][query_id] = query + self.corpus[split][doc_id] = {"title": "", "text": doc} + + self.relevant_docs[split][query_id] = { + doc_id: 1 + } # only one correct matches + id += 1 diff --git a/mteb/tasks/Retrieval/eng/CQADupstackAndroidRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackAndroidRetrieval.py index b95c61af47..156395a077 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackAndroidRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackAndroidRetrieval.py @@ -21,12 +21,12 @@ class CQADupstackAndroidRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Programming", "Web", "Written", "Non-fiction"], + task_subtypes=["Question answering", "Duplicate Detection"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{hoogeveen2015, author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, diff --git a/mteb/tasks/Retrieval/eng/CQADupstackEnglishRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackEnglishRetrieval.py index d9f1c1f344..af47eda5c4 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackEnglishRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackEnglishRetrieval.py @@ -21,12 +21,12 @@ class CQADupstackEnglishRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written"], + task_subtypes=["Question answering", "Duplicate Detection"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{hoogeveen2015, author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, diff --git a/mteb/tasks/Retrieval/eng/CQADupstackGamingRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackGamingRetrieval.py index 8c89299957..b51a3e64b5 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackGamingRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackGamingRetrieval.py @@ -21,12 +21,12 @@ class CQADupstackGamingRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Web", "Written"], + task_subtypes=["Question answering", "Duplicate Detection"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{hoogeveen2015, author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, diff --git a/mteb/tasks/Retrieval/eng/CQADupstackGisRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackGisRetrieval.py index 8ed296b003..da38284f2d 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackGisRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackGisRetrieval.py @@ -21,12 +21,12 @@ class CQADupstackGisRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Non-fiction"], + task_subtypes=["Question answering", "Duplicate Detection"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{hoogeveen2015, author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, diff --git a/mteb/tasks/Retrieval/eng/CQADupstackMathematicaRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackMathematicaRetrieval.py index 0d1804e5e7..b29d166129 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackMathematicaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackMathematicaRetrieval.py @@ -21,12 +21,12 @@ class CQADupstackMathematicaRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Academic", "Non-fiction"], + task_subtypes=["Question answering", "Duplicate Detection"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{hoogeveen2015, author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, diff --git a/mteb/tasks/Retrieval/eng/CQADupstackPhysicsRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackPhysicsRetrieval.py index 77402252f9..3dd0fdc4a5 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackPhysicsRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackPhysicsRetrieval.py @@ -21,12 +21,12 @@ class CQADupstackPhysicsRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Academic", "Non-fiction"], + task_subtypes=["Question answering", "Duplicate Detection"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{hoogeveen2015, author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, diff --git a/mteb/tasks/Retrieval/eng/CQADupstackProgrammersRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackProgrammersRetrieval.py index 1fa63dd20a..f84b1b17e4 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackProgrammersRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackProgrammersRetrieval.py @@ -23,7 +23,7 @@ class CQADupstackProgrammersRetrieval(AbsTaskRetrieval): date=None, domains=["Programming", "Written", "Non-fiction"], task_subtypes=[], - license="cc-by-sa-4.0", + license="apache-2.0", annotations_creators="derived", dialect=[], sample_creation="found", diff --git a/mteb/tasks/Retrieval/eng/CQADupstackStatsRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackStatsRetrieval.py index 8b2ee5950a..1fd18f8d84 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackStatsRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackStatsRetrieval.py @@ -21,12 +21,12 @@ class CQADupstackStatsRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Academic", "Non-fiction"], + task_subtypes=["Question answering", "Duplicate Detection"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{hoogeveen2015, author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, diff --git a/mteb/tasks/Retrieval/eng/CQADupstackTexRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackTexRetrieval.py index 2e87f49710..c4447442be 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackTexRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackTexRetrieval.py @@ -21,12 +21,12 @@ class CQADupstackTexRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Non-fiction"], + task_subtypes=["Question answering", "Duplicate Detection"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{hoogeveen2015, author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, diff --git a/mteb/tasks/Retrieval/eng/CQADupstackUnixRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackUnixRetrieval.py index f86d886519..57c9964b15 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackUnixRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackUnixRetrieval.py @@ -21,12 +21,12 @@ class CQADupstackUnixRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Web", "Programming"], + task_subtypes=["Question answering", "Duplicate Detection"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{hoogeveen2015, author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, diff --git a/mteb/tasks/Retrieval/eng/CQADupstackWebmastersRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackWebmastersRetrieval.py index eedacec19a..2e9bd63e08 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackWebmastersRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackWebmastersRetrieval.py @@ -21,12 +21,12 @@ class CQADupstackWebmastersRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Web"], + task_subtypes=["Question answering"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{hoogeveen2015, author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, diff --git a/mteb/tasks/Retrieval/eng/CQADupstackWordpressRetrieval.py b/mteb/tasks/Retrieval/eng/CQADupstackWordpressRetrieval.py index e70255c371..3b11866f82 100644 --- a/mteb/tasks/Retrieval/eng/CQADupstackWordpressRetrieval.py +++ b/mteb/tasks/Retrieval/eng/CQADupstackWordpressRetrieval.py @@ -21,12 +21,12 @@ class CQADupstackWordpressRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Web", "Programming"], + task_subtypes=["Question answering"], + license="apache-2.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{hoogeveen2015, author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, diff --git a/mteb/tasks/Retrieval/eng/ChemHotpotQARetrieval.py b/mteb/tasks/Retrieval/eng/ChemHotpotQARetrieval.py new file mode 100644 index 0000000000..88fbc50df4 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/ChemHotpotQARetrieval.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class ChemHotpotQARetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="ChemHotpotQARetrieval", + dataset={ + "path": "BASF-AI/ChemHotpotQARetrieval", + "revision": "1840e8a5ac6ec752bbdd97d543ead0189bc7c25b", + }, + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train", "dev", "test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + @inproceedings{yang-etal-2018-hotpotqa, + title = "{H}otpot{QA}: A Dataset for Diverse, Explainable Multi-hop Question Answering", + author = "Yang, Zhilin and + Qi, Peng and + Zhang, Saizheng and + Bengio, Yoshua and + Cohen, William and + Salakhutdinov, Ruslan and + Manning, Christopher D.", + editor = "Riloff, Ellen and + Chiang, David and + Hockenmaier, Julia and + Tsujii, Jun{'}ichi", + booktitle = "Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing", + month = oct # "-" # nov, + year = "2018", + address = "Brussels, Belgium", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/D18-1259", + doi = "10.18653/v1/D18-1259", + pages = "2369--2380", + abstract = "Existing question answering (QA) datasets fail to train QA systems to perform complex reasoning and provide explanations for answers. We introduce HotpotQA, a new dataset with 113k Wikipedia-based question-answer pairs with four key features: (1) the questions require finding and reasoning over multiple supporting documents to answer; (2) the questions are diverse and not constrained to any pre-existing knowledge bases or knowledge schemas; (3) we provide sentence-level supporting facts required for reasoning, allowing QA systems to reason with strong supervision and explain the predictions; (4) we offer a new type of factoid comparison questions to test QA systems{'} ability to extract relevant facts and perform necessary comparison. We show that HotpotQA is challenging for the latest QA systems, and the supporting facts enable models to improve performance and make explainable predictions.", + } +""", + ) diff --git a/mteb/tasks/Retrieval/eng/ChemNQRetrieval.py b/mteb/tasks/Retrieval/eng/ChemNQRetrieval.py new file mode 100644 index 0000000000..1e77971331 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/ChemNQRetrieval.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class ChemNQRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="ChemNQRetrieval", + dataset={ + "path": "BASF-AI/ChemNQRetrieval", + "revision": "5d958fb6b31055495347724d46431ba41309b03a", + }, + description="ChemTEB evaluates the performance of text embedding models on chemical domain data.", + reference="https://arxiv.org/abs/2412.00532", + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2024-06-01", "2024-11-30"), + domains=["Chemistry"], + task_subtypes=[], + license="cc-by-nc-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" + @article{kasmaee2024chemteb, + title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain}, + author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila}, + journal={arXiv preprint arXiv:2412.00532}, + year={2024} + } + @article{47761, + title = {Natural Questions: a Benchmark for Question Answering Research}, + author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh + and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee + and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le + and Slav Petrov}, + year = {2019}, + journal = {Transactions of the Association of Computational Linguistics}} + """, + ) diff --git a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py index d60b7a3817..b87e5223e0 100644 --- a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py @@ -21,12 +21,12 @@ class ClimateFEVER(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Encyclopaedic", "Written"], + task_subtypes=["Claim verification"], + license="cc-by-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", bibtex_citation="""@misc{diggelmann2021climatefever, title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold}, @@ -57,12 +57,12 @@ class ClimateFEVERHardNegatives(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Encyclopaedic", "Written"], + task_subtypes=["Claim verification"], + license="cc-by-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", bibtex_citation="""@misc{diggelmann2021climatefever, title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold}, diff --git a/mteb/tasks/Retrieval/eng/FEVERRetrieval.py b/mteb/tasks/Retrieval/eng/FEVERRetrieval.py index 776fd2fbe6..fff60a54d2 100644 --- a/mteb/tasks/Retrieval/eng/FEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/FEVERRetrieval.py @@ -27,12 +27,12 @@ class FEVER(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Encyclopaedic", "Written"], + task_subtypes=["Claim verification"], + license="cc-by-nc-sa-3.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{thorne-etal-2018-fever, title = "{FEVER}: a Large-scale Dataset for Fact Extraction and {VER}ification", author = "Thorne, James and diff --git a/mteb/tasks/Retrieval/eng/FiQA2018Retrieval.py b/mteb/tasks/Retrieval/eng/FiQA2018Retrieval.py index 1489cd168c..7a99d48a95 100644 --- a/mteb/tasks/Retrieval/eng/FiQA2018Retrieval.py +++ b/mteb/tasks/Retrieval/eng/FiQA2018Retrieval.py @@ -23,12 +23,12 @@ class FiQA2018(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Financial"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{ thakur2021beir, title={{BEIR}: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models}, diff --git a/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py b/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py index 5ada0cf887..6ebb5d7277 100644 --- a/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py +++ b/mteb/tasks/Retrieval/eng/MSMARCORetrieval.py @@ -23,12 +23,23 @@ class MSMARCO(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=[ + "Encyclopaedic", + "Academic", + "Blog", + "News", + "Medical", + "Government", + "Reviews", + "Non-fiction", + "Social", + "Web", + ], + task_subtypes=["Question answering"], + license="msr-la-nc", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@article{DBLP:journals/corr/NguyenRSGTMD16, author = {Tri Nguyen and Mir Rosenberg and @@ -73,12 +84,23 @@ class MSMARCOHardNegatives(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=[ + "Encyclopaedic", + "Academic", + "Blog", + "News", + "Medical", + "Government", + "Reviews", + "Non-fiction", + "Social", + "Web", + ], + task_subtypes=["Question answering"], + license="msr-la-nc", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@article{DBLP:journals/corr/NguyenRSGTMD16, author = {Tri Nguyen and Mir Rosenberg and diff --git a/mteb/tasks/Retrieval/eng/MSMARCOv2Retrieval.py b/mteb/tasks/Retrieval/eng/MSMARCOv2Retrieval.py index d3b10738cf..7487abb887 100644 --- a/mteb/tasks/Retrieval/eng/MSMARCOv2Retrieval.py +++ b/mteb/tasks/Retrieval/eng/MSMARCOv2Retrieval.py @@ -21,12 +21,23 @@ class MSMARCOv2(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=[ + "Encyclopaedic", + "Academic", + "Blog", + "News", + "Medical", + "Government", + "Reviews", + "Non-fiction", + "Social", + "Web", + ], + task_subtypes=["Question answering"], + license="msr-la-nc", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@article{DBLP:journals/corr/NguyenRSGTMD16, author = {Tri Nguyen and Mir Rosenberg and diff --git a/mteb/tasks/Retrieval/eng/NQRetrieval.py b/mteb/tasks/Retrieval/eng/NQRetrieval.py index 661bf3e0e2..e81018dbc4 100644 --- a/mteb/tasks/Retrieval/eng/NQRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NQRetrieval.py @@ -21,12 +21,12 @@ class NQ(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Encyclopaedic"], + task_subtypes=["Question answering"], + license="cc-by-nc-sa-3.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", bibtex_citation="""@article{47761,title = {Natural Questions: a Benchmark for Question Answering Research}, author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee @@ -67,4 +67,7 @@ class NQHardNegatives(AbsTaskRetrieval): and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational Linguistics}}""", + prompt={ + "query": "Given a question, retrieve Wikipedia passages that answer the question" + }, ) diff --git a/mteb/tasks/Retrieval/eng/QuoraRetrieval.py b/mteb/tasks/Retrieval/eng/QuoraRetrieval.py index 73660fb573..52e6cca4b1 100644 --- a/mteb/tasks/Retrieval/eng/QuoraRetrieval.py +++ b/mteb/tasks/Retrieval/eng/QuoraRetrieval.py @@ -26,12 +26,12 @@ class QuoraRetrieval(AbsTaskRetrieval): eval_langs=["eng-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Web", "Blog"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", bibtex_citation="""@misc{quora-question-pairs, author = {DataCanary, hilfialkaff, Lili Jiang, Meg Risdal, Nikhil Dandekar, tomtung}, title = {Quora Question Pairs}, diff --git a/mteb/tasks/Retrieval/eng/SciFactRetrieval.py b/mteb/tasks/Retrieval/eng/SciFactRetrieval.py index 8caa0c2af5..a44eb052bd 100644 --- a/mteb/tasks/Retrieval/eng/SciFactRetrieval.py +++ b/mteb/tasks/Retrieval/eng/SciFactRetrieval.py @@ -22,8 +22,8 @@ class SciFact(AbsTaskRetrieval): main_score="ndcg_at_10", date=None, domains=["Academic", "Medical", "Written"], - task_subtypes=None, - license=None, + task_subtypes=[], + license="not specified", annotations_creators=None, dialect=None, sample_creation=None, diff --git a/mteb/tasks/Retrieval/fas/BEIRFa.py b/mteb/tasks/Retrieval/fas/BEIRFa.py new file mode 100644 index 0000000000..0952eefff9 --- /dev/null +++ b/mteb/tasks/Retrieval/fas/BEIRFa.py @@ -0,0 +1,662 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval + + +class ArguAnaFa(AbsTaskRetrieval): + ignore_identical_ids = True + metadata = TaskMetadata( + name="ArguAna-Fa", + description="ArguAna-Fa", + reference="https://huggingface.co/datasets/MCINext/arguana-fa", + dataset={ + "path": "MCINext/arguana-fa", + "revision": "fa97814be356fe4d18caadb457b4469bd34019ca", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Blog"], + task_subtypes=["Article retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class ClimateFEVERFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="ClimateFEVER-Fa", + description="ClimateFEVER-Fa", + reference="https://huggingface.co/datasets/MCINext/climate-fever-fa", + dataset={ + "path": "MCINext/climate-fever-fa", + "revision": "45d9176b6fcba33abc58494ee82f18ee7e8ddbae", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Article retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackAndroidRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackAndroidRetrieval-Fa", + description="CQADupstackAndroidRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-android-fa", + dataset={ + "path": "MCINext/cqadupstack-android-fa", + "revision": "bcdaf4e30477eea9b9b17ecbb153ca403e5c3ebd", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackEnglishRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackEnglishRetrieval-Fa", + description="CQADupstackEnglishRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-english-fa", + dataset={ + "path": "MCINext/cqadupstack-english-fa", + "revision": "029a2e69e7d9e68b6bdc471073606104417a5be7", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackGamingRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackGamingRetrieval-Fa", + description="CQADupstackGamingRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-gaming-fa", + dataset={ + "path": "MCINext/cqadupstack-gaming-fa", + "revision": "e9c7ad03f29a55ab14eae730146961b8cdc14227", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackGisRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackGisRetrieval-Fa", + description="CQADupstackGisRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-gis-fa", + dataset={ + "path": "MCINext/cqadupstack-gis-fa", + "revision": "e907c4144dc27bc8a035d78d69e15f39c56623a9", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackMathematicaRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackMathematicaRetrieval-Fa", + description="CQADupstackMathematicaRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-mathematica-fa", + dataset={ + "path": "MCINext/cqadupstack-mathematica-fa", + "revision": "b92e24fab42ab599535a19ee744de5485ec92f64", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackPhysicsRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackPhysicsRetrieval-Fa", + description="CQADupstackPhysicsRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-physics-fa", + dataset={ + "path": "MCINext/cqadupstack-physics-fa", + "revision": "816ad7473d6813f77a0ca5e72b1ff7a52752d370", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackProgrammersRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackProgrammersRetrieval-Fa", + description="CQADupstackProgrammersRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-programmers-fa", + dataset={ + "path": "MCINext/cqadupstack-programmers-fa", + "revision": "be6460df57ab7c1b2c9fe295a31660dbd077ecf0", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackStatsRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackStatsRetrieval-Fa", + description="CQADupstackStatsRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-stats-fa", + dataset={ + "path": "MCINext/cqadupstack-stats-fa", + "revision": "c6e2c8b6153958118ec04352dd82a30ea2e2cad5", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackTexRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackTexRetrieval-Fa", + description="CQADupstackTexRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-tex-fa", + dataset={ + "path": "MCINext/cqadupstack-tex-fa", + "revision": "860d152c86fda27229270b6bf4e832ff374ac01b", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackUnixRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackUnixRetrieval-Fa", + description="CQADupstackUnixRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-unix-fa", + dataset={ + "path": "MCINext/cqadupstack-unix-fa", + "revision": "c2a326387954aad66ff00d324a9278067b1e3bb6", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackWebmastersRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackWebmastersRetrieval-Fa", + description="CQADupstackWebmastersRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-webmasters-fa", + dataset={ + "path": "MCINext/cqadupstack-webmasters-fa", + "revision": "506f29f8ce59648efe99afee736b0b158eced516", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class CQADupstackWordpressRetrievalFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="CQADupstackWordpressRetrieval-Fa", + description="CQADupstackWordpressRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/cqadupstack-wordpress-fa", + dataset={ + "path": "MCINext/cqadupstack-wordpress-fa", + "revision": "7f755e88647b4023df52da04d4e3d31a7de5fcb0", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class DBPediaFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="DBPedia-Fa", + description="DBPedia-Fa", + reference="https://huggingface.co/datasets/MCINext/dbpedia-fa", + dataset={ + "path": "MCINext/dbpedia-fa", + "revision": "13529e6e301e9d72f86def882cfbca04791d83f9", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Encyclopaedic"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class FiQA2018Fa(AbsTaskRetrieval): + ignore_identical_ids = True + + metadata = TaskMetadata( + name="FiQA2018-Fa", + description="FiQA2018-Fa", + reference="https://huggingface.co/datasets/MCINext/fiqa-fa", + dataset={ + "path": "MCINext/fiqa-fa", + "revision": "e683ce7ecd0b47edc3e29fda7cfd75335be4a997", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Article retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class HotpotQAFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="HotpotQA-Fa", + description="HotpotQA-Fa", + reference="https://huggingface.co/datasets/MCINext/hotpotqa-fa", + dataset={ + "path": "MCINext/hotpotqa-fa", + "revision": "1cafde1306aa56b5dfdce0b14633ae9ee1a63ddb", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Encyclopaedic"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class MSMARCOFa(AbsTaskRetrieval): + ignore_identical_ids = True + + metadata = TaskMetadata( + name="MSMARCO-Fa", + description="MSMARCO-Fa", + reference="https://huggingface.co/datasets/MCINext/msmarco-fa", + dataset={ + "path": "MCINext/msmarco-fa", + "revision": "88f90b0b04f91778ba5341095b0a9f1d7799ce10", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["dev"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class NFCorpusFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NFCorpus-Fa", + description="NFCorpus-Fa", + reference="https://huggingface.co/datasets/MCINext/nfcorpus-fa", + dataset={ + "path": "MCINext/nfcorpus-fa", + "revision": "70aa71825a791e87210c0355a01f538aa611feae", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Medical"], + task_subtypes=["Article retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class NQFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NQ-Fa", + description="NQ-Fa", + reference="https://huggingface.co/datasets/MCINext/nq-fa", + dataset={ + "path": "MCINext/nq-fa", + "revision": "d4ea898b644c8d5f608b60947cb637bebbf1ac66", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Encyclopaedic"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class QuoraRetrievalFa(AbsTaskRetrieval): + ignore_identical_ids = True + + metadata = TaskMetadata( + name="QuoraRetrieval-Fa", + description="QuoraRetrieval-Fa", + reference="https://huggingface.co/datasets/MCINext/quora-fa", + dataset={ + "path": "MCINext/quora-fa", + "revision": "1a43f4f5dcd71e6b14b275ae82c3237cdd4fd5fd", + }, + type="Retrieval", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class SCIDOCSFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="SCIDOCS-Fa", + description="SCIDOCS-Fa", + reference="https://huggingface.co/datasets/MCINext/scidocs-fa", + dataset={ + "path": "MCINext/scidocs-fa", + "revision": "6611ebf4b4c1aaf8b021e4728440db2188291b8b", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Academic"], + task_subtypes=["Article retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class SciFactFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="SciFact-Fa", + description="SciFact-Fa", + reference="https://huggingface.co/datasets/MCINext/scifact-fa", + dataset={ + "path": "MCINext/scifact-fa", + "revision": "7723397096199c4d6f367b445fccaf282c518abe", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Academic"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class TRECCOVIDFa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="TRECCOVID-Fa", + description="TRECCOVID-Fa", + reference="https://huggingface.co/datasets/MCINext/trec-covid-fa", + dataset={ + "path": "MCINext/trec-covid-fa", + "revision": "98e6c2d33dfa166ee326e8b1bc7ea82c7e6898dd", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Medical"], + task_subtypes=["Article retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + +class Touche2020Fa(AbsTaskRetrieval): + metadata = TaskMetadata( + name="Touche2020-Fa", + description="Touche2020-Fa", + reference="https://huggingface.co/datasets/MCINext/touche2020-fa", + dataset={ + "path": "MCINext/touche2020-fa", + "revision": "0f464636f91641cc6ef6f6f8f249c73f4a609982", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) diff --git a/mteb/tasks/Retrieval/fas/FaMTEBRetrieval.py b/mteb/tasks/Retrieval/fas/FaMTEBRetrieval.py new file mode 100644 index 0000000000..875f7ea7db --- /dev/null +++ b/mteb/tasks/Retrieval/fas/FaMTEBRetrieval.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval + + +class SynPerQARetrieval(AbsTaskRetrieval): + ignore_identical_ids = True + metadata = TaskMetadata( + name="SynPerQARetrieval", + description="Synthetic Persian QA Retrieval", + reference="https://huggingface.co/datasets/MCINext/synthetic-persian-qa-retrieval/settings", + dataset={ + "path": "MCINext/synthetic-persian-qa-retrieval", + "revision": "e85114f13f42dc1edc456d58931cc38d44d697cf", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation="""""", + ) + + +class SynPerChatbotTopicsRetrieval(AbsTaskRetrieval): + ignore_identical_ids = True + metadata = TaskMetadata( + name="SynPerChatbotTopicsRetrieval", + description="Synthetic Persian Chatbot Topics Retrieval", + reference="https://huggingface.co/datasets/MCINext/synthetic-persian-chatbot-topics-retrieval", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-topics-retrieval", + "revision": "086995ca4cea33f37a407c2fa5282f74913740ee", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=[], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation="""""", + ) + + +class SynPerChatbotRAGTopicsRetrieval(AbsTaskRetrieval): + ignore_identical_ids = True + metadata = TaskMetadata( + name="SynPerChatbotRAGTopicsRetrieval", + description="Synthetic Persian Chatbot RAG Topics Retrieval", + reference="https://huggingface.co/datasets/MCINext/synthetic-persian-chatbot-rag-topics-retrieval", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-rag-topics-retrieval", + "revision": "da8f36a723da155738f5e3d8d84d543589bd5083", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=[], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation="""""", + ) + + +class SynPerChatbotRAGFAQRetrieval(AbsTaskRetrieval): + ignore_identical_ids = True + metadata = TaskMetadata( + name="SynPerChatbotRAGFAQRetrieval", + description="Synthetic Persian Chatbot RAG FAQ Retrieval", + reference="https://huggingface.co/datasets/MCINext/synthetic-persian-chatbot-rag-faq-retrieval", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-rag-faq-retrieval", + "revision": "9d32af6540970e2845028cbfffe6b0d0e8f52428", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation="""""", + ) + + +class PersianWebDocumentRetrieval(AbsTaskRetrieval): + ignore_identical_ids = True + metadata = TaskMetadata( + name="PersianWebDocumentRetrieval", + description="Persian dataset designed specifically for the task of text information retrieval through the web.", + reference="https://ieeexplore.ieee.org/document/10553090", + dataset={ + "path": "MCINext/persian-web-document-retrieval", + "revision": "b3dc818368a867b30ccb55a42ff287d253512c36", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="ndcg_at_10", + date=("2024-09-01", "2024-12-31"), + domains=["Web"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation="""""", + ) diff --git a/mteb/tasks/Retrieval/fas/__init__.py b/mteb/tasks/Retrieval/fas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/Retrieval/kor/AutoRAGRetrieval.py b/mteb/tasks/Retrieval/kor/AutoRAGRetrieval.py index 4a24e04e9c..6eec67aad2 100644 --- a/mteb/tasks/Retrieval/kor/AutoRAGRetrieval.py +++ b/mteb/tasks/Retrieval/kor/AutoRAGRetrieval.py @@ -22,7 +22,7 @@ class AutoRAGRetrieval(AbsTaskRetrieval): eval_langs=["kor-Hang"], main_score="ndcg_at_10", date=("2024-08-03", "2024-08-03"), - domains=["Government", "Medical", "Legal", "Social"], + domains=["Government", "Medical", "Legal", "Social", "Financial"], task_subtypes=["Article retrieval"], license="mit", annotations_creators="human-annotated", diff --git a/mteb/tasks/Retrieval/pol/ArguAnaPLRetrieval.py b/mteb/tasks/Retrieval/pol/ArguAnaPLRetrieval.py index 342f727144..ada5c4ca8e 100644 --- a/mteb/tasks/Retrieval/pol/ArguAnaPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/ArguAnaPLRetrieval.py @@ -24,11 +24,11 @@ class ArguAnaPL(AbsTaskRetrieval): eval_langs=["pol-Latn"], main_score="ndcg_at_10", date=None, - domains=None, + domains=["Medical", "Written"], task_subtypes=None, - license=None, + license="cc-by-sa-4.0", annotations_creators=None, - dialect=None, + dialect=[], sample_creation=None, bibtex_citation="""@misc{wojtasik2024beirpl, title={BEIR-PL: Zero Shot Information Retrieval Benchmark for the Polish Language}, diff --git a/mteb/tasks/Retrieval/pol/FiQAPLRetrieval.py b/mteb/tasks/Retrieval/pol/FiQAPLRetrieval.py index 0a125f5e4f..b54f4ae4ed 100644 --- a/mteb/tasks/Retrieval/pol/FiQAPLRetrieval.py +++ b/mteb/tasks/Retrieval/pol/FiQAPLRetrieval.py @@ -24,12 +24,12 @@ class FiQAPLRetrieval(AbsTaskRetrieval): eval_langs=["pol-Latn"], main_score="ndcg_at_10", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Written", "Financial"], + task_subtypes=["Question answering"], + license="not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", bibtex_citation="""@inproceedings{ thakur2021beir, title={{BEIR}: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models}, diff --git a/mteb/tasks/STS/__init__.py b/mteb/tasks/STS/__init__.py index b61b79b293..471789f1c9 100644 --- a/mteb/tasks/STS/__init__.py +++ b/mteb/tasks/STS/__init__.py @@ -10,6 +10,7 @@ from .eng.STS16STS import * from .eng.STSBenchmarkSTS import * from .fao.FaroeseSTS import * +from .fas.FaMTEBSTS import * from .fin.FinParaSTS import * from .fra.SickFrSTS import * from .jpn.JSICK import * diff --git a/mteb/tasks/STS/eng/BiossesSTS.py b/mteb/tasks/STS/eng/BiossesSTS.py index ce54e37789..1fc1d5a1d0 100644 --- a/mteb/tasks/STS/eng/BiossesSTS.py +++ b/mteb/tasks/STS/eng/BiossesSTS.py @@ -21,12 +21,12 @@ class BiossesSTS(AbsTaskSTS): eval_langs=["eng-Latn"], main_score="cosine_spearman", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Medical"], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", bibtex_citation="""@article{10.1093/bioinformatics/btx238, author = {Soğancıoğlu, Gizem and Öztürk, Hakime and Özgür, Arzucan}, title = "{BIOSSES: a semantic sentence similarity estimation system for the biomedical domain}", diff --git a/mteb/tasks/STS/eng/STSBenchmarkSTS.py b/mteb/tasks/STS/eng/STSBenchmarkSTS.py index 099fba6773..e600711d34 100644 --- a/mteb/tasks/STS/eng/STSBenchmarkSTS.py +++ b/mteb/tasks/STS/eng/STSBenchmarkSTS.py @@ -21,12 +21,12 @@ class STSBenchmarkSTS(AbsTaskSTS): eval_langs=["eng-Latn"], main_score="cosine_spearman", date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + domains=["Blog", "News", "Written"], + task_subtypes=[], + license="not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="machine-translated and verified", bibtex_citation="""@InProceedings{huggingface:dataset:stsb_multi_mt, title = {Machine translated multilingual STS benchmark dataset.}, author={Philip May}, diff --git a/mteb/tasks/STS/fas/FaMTEBSTS.py b/mteb/tasks/STS/fas/FaMTEBSTS.py new file mode 100644 index 0000000000..2ce9522cd4 --- /dev/null +++ b/mteb/tasks/STS/fas/FaMTEBSTS.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from ....abstasks.AbsTaskSTS import AbsTaskSTS + + +class Farsick(AbsTaskSTS): + metadata = TaskMetadata( + name="Farsick", + description="A Persian Semantic Textual Similarity And Natural Language Inference Dataset", + reference="https://github.com/ZahraGhasemi-AI/FarSick", + dataset={ + "path": "MCINext/farsick-sts", + "revision": "f8b8d630f631c6c16b7bc3cb924bdf62a51bed06", + }, + type="STS", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="cosine_spearman", + date=("2024-09-01", "2024-12-31"), + domains=[], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 1 + metadata_dict["max_score"] = 5 + return metadata_dict + + +class SynPerSTS(AbsTaskSTS): + metadata = TaskMetadata( + name="SynPerSTS", + description="Synthetic Persian Semantic Textual Similarity Dataset", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/synthetic-persian-sts", + "revision": "914047db08928b5326d8b106583dc563b73d1ecf", + }, + type="STS", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="cosine_spearman", + date=("2024-09-01", "2024-12-31"), + domains=["Web", "News", "Religious", "Blog"], + task_subtypes=[], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 1 + metadata_dict["max_score"] = 5 + return metadata_dict + + +class Query2Query(AbsTaskSTS): + metadata = TaskMetadata( + name="Query2Query", + description="Query to Query Datasets.", + reference="https://mcinext.com/", + dataset={ + "path": "MCINext/query-to-query-sts", + "revision": "52602079f9032fcf181775a310d79d2f197534e4", + }, + type="STS", + category="s2s", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="cosine_spearman", + date=("2024-09-01", "2024-12-31"), + domains=[], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" """, + ) + + @property + def metadata_dict(self) -> dict[str, str]: + metadata_dict = super().metadata_dict + metadata_dict["min_score"] = 0 + metadata_dict["max_score"] = 2 + return metadata_dict diff --git a/mteb/tasks/STS/fas/__init__.py b/mteb/tasks/STS/fas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/SummaryRetrieval/__init__.py b/mteb/tasks/SummaryRetrieval/__init__.py new file mode 100644 index 0000000000..d000983be9 --- /dev/null +++ b/mteb/tasks/SummaryRetrieval/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from .fas.FaMTEBSummaryRetrieval import * diff --git a/mteb/tasks/SummaryRetrieval/fas/FaMTEBSummaryRetrieval.py b/mteb/tasks/SummaryRetrieval/fas/FaMTEBSummaryRetrieval.py new file mode 100644 index 0000000000..f0797068c3 --- /dev/null +++ b/mteb/tasks/SummaryRetrieval/fas/FaMTEBSummaryRetrieval.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class SAMSumFa(AbsTaskBitextMining): + metadata = TaskMetadata( + name="SAMSumFa", + description="Translated Version of SAMSum Dataset for summary retrieval.", + reference="https://huggingface.co/datasets/MCINext/samsum-fa", + dataset={ + "path": "MCINext/samsum-fa", + "revision": "fd981d78a0ab82c20d2e693a8b3929c5d71b0743", + }, + type="BitextMining", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="f1", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=[], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="machine-translated", + bibtex_citation="", + ) + + def dataset_transform(self): + self.dataset = self.dataset.rename_columns( + {"text": "sentence1", "summary": "sentence2"} + ) + + +class SynPerChatbotSumSRetrieval(AbsTaskBitextMining): + metadata = TaskMetadata( + name="SynPerChatbotSumSRetrieval", + description="Synthetic Persian Chatbot Summary Dataset for summary retrieval.", + reference="https://huggingface.co/datasets/MCINext/synthetic-persian-chatbot-summary-retrieval", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-summary-retrieval", + "revision": "9002f5e9de4ef61f1f5c34831d2a5ed855bac0ae", + }, + type="BitextMining", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="f1", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=[], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + self.dataset = self.dataset.rename_columns( + {"text": "sentence1", "summary": "sentence2"} + ) + + +class SynPerChatbotRAGSumSRetrieval(AbsTaskBitextMining): + metadata = TaskMetadata( + name="SynPerChatbotRAGSumSRetrieval", + description="Synthetic Persian Chatbot RAG Summary Dataset for summary retrieval.", + reference="https://huggingface.co/datasets/MCINext/synthetic-persian-chatbot-rag-summary-retrieval", + dataset={ + "path": "MCINext/synthetic-persian-chatbot-rag-summary-retrieval", + "revision": "f77746f286bbf2177ee7b5a803da8be440d5d4c1", + }, + type="BitextMining", + category="p2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fas-Arab"], + main_score="f1", + date=("2024-09-01", "2024-12-31"), + domains=["Spoken"], + task_subtypes=[], + license="not specified", + annotations_creators="LM-generated", + dialect=[], + sample_creation="LM-generated and verified", + bibtex_citation=""" """, + ) + + def dataset_transform(self): + self.dataset = self.dataset.rename_columns( + {"text": "sentence1", "summary": "sentence2"} + ) diff --git a/mteb/tasks/SummaryRetrieval/fas/__init__.py b/mteb/tasks/SummaryRetrieval/fas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mteb/tasks/__init__.py b/mteb/tasks/__init__.py index 8d49517136..e00f091174 100644 --- a/mteb/tasks/__init__.py +++ b/mteb/tasks/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +from .aggregated_tasks import * from .BitextMining import * from .Classification import * from .Clustering import * @@ -19,3 +20,4 @@ from .SpeedTask import * from .STS import * from .Summarization import * +from .SummaryRetrieval import * diff --git a/mteb/tasks/aggregated_tasks/CQADupStackRetrieval.py b/mteb/tasks/aggregated_tasks/CQADupStackRetrieval.py new file mode 100644 index 0000000000..917a667eb3 --- /dev/null +++ b/mteb/tasks/aggregated_tasks/CQADupStackRetrieval.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTask +from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata +from mteb.tasks.Retrieval import ( + CQADupstackAndroidRetrieval, + CQADupstackEnglishRetrieval, + CQADupstackGamingRetrieval, + CQADupstackGisRetrieval, + CQADupstackMathematicaRetrieval, + CQADupstackPhysicsRetrieval, + CQADupstackProgrammersRetrieval, + CQADupstackStatsRetrieval, + CQADupstackTexRetrieval, + CQADupstackUnixRetrieval, + CQADupstackWebmastersRetrieval, + CQADupstackWordpressRetrieval, +) + +task_list_cqa: list[AbsTask] = [ + CQADupstackAndroidRetrieval(), + CQADupstackEnglishRetrieval(), + CQADupstackGamingRetrieval(), + CQADupstackGisRetrieval(), + CQADupstackMathematicaRetrieval(), + CQADupstackPhysicsRetrieval(), + CQADupstackProgrammersRetrieval(), + CQADupstackStatsRetrieval(), + CQADupstackTexRetrieval(), + CQADupstackUnixRetrieval(), + CQADupstackWebmastersRetrieval(), + CQADupstackWordpressRetrieval(), +] + + +class CQADupstackRetrieval(AbsTaskAggregate): + metadata = AggregateTaskMetadata( + name="CQADupstackRetrieval", + description="CQADupStack: A Benchmark Data Set for Community Question-Answering Research", + reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/", + tasks=task_list_cqa, + main_score="ndcg_at_10", + type="Retrieval", # since everything is retrieval - otherwise it would be "Aggregated" + eval_splits=["test"], + bibtex_citation="""@inproceedings{hoogeveen2015, +author = {Hoogeveen, Doris and Verspoor, Karin M. and Baldwin, Timothy}, +title = {CQADupStack: A Benchmark Data Set for Community Question-Answering Research}, +booktitle = {Proceedings of the 20th Australasian Document Computing Symposium (ADCS)}, +series = {ADCS '15}, +year = {2015}, +isbn = {978-1-4503-4040-3}, +location = {Parramatta, NSW, Australia}, +pages = {3:1--3:8}, +articleno = {3}, +numpages = {8}, +url = {http://doi.acm.org/10.1145/2838931.2838934}, +doi = {10.1145/2838931.2838934}, +acmid = {2838934}, +publisher = {ACM}, +address = {New York, NY, USA}, +}""", + ) diff --git a/mteb/tasks/aggregated_tasks/CQADupStackRetrievalFa.py b/mteb/tasks/aggregated_tasks/CQADupStackRetrievalFa.py new file mode 100644 index 0000000000..6a60f4b000 --- /dev/null +++ b/mteb/tasks/aggregated_tasks/CQADupStackRetrievalFa.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTask +from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata +from mteb.tasks.Retrieval import ( + CQADupstackAndroidRetrievalFa, + CQADupstackEnglishRetrievalFa, + CQADupstackGamingRetrievalFa, + CQADupstackGisRetrievalFa, + CQADupstackMathematicaRetrievalFa, + CQADupstackPhysicsRetrievalFa, + CQADupstackProgrammersRetrievalFa, + CQADupstackStatsRetrievalFa, + CQADupstackTexRetrievalFa, + CQADupstackUnixRetrievalFa, + CQADupstackWebmastersRetrievalFa, + CQADupstackWordpressRetrievalFa, +) + +task_list_cqa: list[AbsTask] = [ + CQADupstackAndroidRetrievalFa(), + CQADupstackEnglishRetrievalFa(), + CQADupstackGamingRetrievalFa(), + CQADupstackGisRetrievalFa(), + CQADupstackMathematicaRetrievalFa(), + CQADupstackPhysicsRetrievalFa(), + CQADupstackProgrammersRetrievalFa(), + CQADupstackStatsRetrievalFa(), + CQADupstackTexRetrievalFa(), + CQADupstackUnixRetrievalFa(), + CQADupstackWebmastersRetrievalFa(), + CQADupstackWordpressRetrievalFa(), +] + + +class CQADupstackRetrievalFa(AbsTaskAggregate): + metadata = AggregateTaskMetadata( + name="CQADupstackRetrieval-Fa", + description="CQADupstackRetrieval-Fa", + reference="", + tasks=task_list_cqa, + main_score="ndcg_at_10", + type="Retrieval", # since everything is retrieval - otherwise it would be "Aggregated" + eval_splits=["test"], + bibtex_citation=""" """, + ) diff --git a/mteb/tasks/aggregated_tasks/SynPerChatbotConvSAClassification.py b/mteb/tasks/aggregated_tasks/SynPerChatbotConvSAClassification.py new file mode 100644 index 0000000000..46c6ed9600 --- /dev/null +++ b/mteb/tasks/aggregated_tasks/SynPerChatbotConvSAClassification.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from mteb.abstasks import AbsTask +from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata +from mteb.tasks.Classification import ( + SynPerChatbotConvSAAnger, + SynPerChatbotConvSAFear, + SynPerChatbotConvSAFriendship, + SynPerChatbotConvSAHappiness, + SynPerChatbotConvSAJealousy, + SynPerChatbotConvSALove, + SynPerChatbotConvSASadness, + SynPerChatbotConvSASatisfaction, + SynPerChatbotConvSASurprise, +) + +task_list_cqa: list[AbsTask] = [ + SynPerChatbotConvSAAnger(), + SynPerChatbotConvSASatisfaction(), + SynPerChatbotConvSAFriendship(), + SynPerChatbotConvSAFear(), + SynPerChatbotConvSAJealousy(), + SynPerChatbotConvSASurprise(), + SynPerChatbotConvSALove(), + SynPerChatbotConvSASadness(), + SynPerChatbotConvSAHappiness(), +] + + +class SynPerChatbotConvSAClassification(AbsTaskAggregate): + metadata = AggregateTaskMetadata( + name="SynPerChatbotConvSAClassification", + description="SynPerChatbotConvSAClassification", + reference="", + tasks=task_list_cqa, + main_score="accuracy", + type="Classification", + eval_splits=["test"], + bibtex_citation=""" """, + ) diff --git a/mteb/tasks/aggregated_tasks/__init__.py b/mteb/tasks/aggregated_tasks/__init__.py new file mode 100644 index 0000000000..5333db7916 --- /dev/null +++ b/mteb/tasks/aggregated_tasks/__init__.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from .CQADupStackRetrieval import CQADupstackRetrieval +from .CQADupStackRetrievalFa import CQADupstackRetrievalFa +from .SynPerChatbotConvSAClassification import SynPerChatbotConvSAClassification + +__all__ = [ + "CQADupstackRetrieval", + "CQADupstackRetrievalFa", + "SynPerChatbotConvSAClassification", +] diff --git a/pyproject.toml b/pyproject.toml index 2c1040047c..43811556c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mteb" -version = "1.29.14" +version = "1.31.8" description = "Massive Text Embedding Benchmark" readme = "README.md" authors = [ @@ -58,7 +58,7 @@ dev = ["ruff==0.6.4", # locked so we don't get PRs which fail only due to a lint codecarbon = ["codecarbon"] speedtask = ["GPUtil>=1.4.0", "psutil>=5.9.8"] peft = ["peft>=0.11.0"] -leaderboard = ["gradio>=5.7.1", "gradio_rangeslider>=0.0.8", "plotly>=5.24.0"] +leaderboard = ["gradio>=5.7.1", "gradio_rangeslider>=0.0.8", "plotly>=5.24.0,<6.0.0"] flagembedding = ["FlagEmbedding"] jina = ["einops>=0.8.0"] flash_attention = ["flash-attn>=2.6.3"] diff --git a/scripts/extract_model_names.py b/scripts/extract_model_names.py index 36cfc572e9..84a81fca26 100644 --- a/scripts/extract_model_names.py +++ b/scripts/extract_model_names.py @@ -28,6 +28,7 @@ def get_changed_files(base_branch="main"): and f.endswith(".py") and "overview" not in f and "init" not in f + and "instructions" not in f ] diff --git a/tests/test_TaskMetadata.py b/tests/test_TaskMetadata.py index 2b606c2c19..f7ac92a697 100644 --- a/tests/test_TaskMetadata.py +++ b/tests/test_TaskMetadata.py @@ -59,6 +59,7 @@ "AILAStatutes", "ArguAna", "ClimateFEVER", + "CQADupstackRetrieval", "CQADupstackAndroidRetrieval", "CQADupstackEnglishRetrieval", "CQADupstackGamingRetrieval", @@ -178,6 +179,8 @@ "TamilNewsClassification", "TenKGnadClusteringP2P.v2", "TenKGnadClusteringS2S.v2", + "SynPerChatbotConvSAClassification", + "CQADupstackRetrieval-Fa", ] diff --git a/tests/test_overview.py b/tests/test_overview.py index 127e54f279..7041328a59 100644 --- a/tests/test_overview.py +++ b/tests/test_overview.py @@ -20,7 +20,7 @@ def test_get_tasks_size_differences(): ) -@pytest.mark.parametrize("task_name", ["BornholmBitextMining"]) +@pytest.mark.parametrize("task_name", ["BornholmBitextMining", "CQADupstackRetrieval"]) @pytest.mark.parametrize("eval_splits", [["test"], None]) def test_get_task(task_name: str, eval_splits: list[str] | None): task = get_task(task_name, eval_splits=eval_splits) diff --git a/tests/test_tasks/test_all_abstasks.py b/tests/test_tasks/test_all_abstasks.py index 902c541390..7a87914f0a 100644 --- a/tests/test_tasks/test_all_abstasks.py +++ b/tests/test_tasks/test_all_abstasks.py @@ -13,6 +13,7 @@ from mteb.abstasks.AbsTaskInstructionRetrieval import AbsTaskInstructionRetrieval from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.AbsTaskSpeedTask import AbsTaskSpeedTask +from mteb.abstasks.aggregated_task import AbsTaskAggregate from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval from mteb.abstasks.MultiSubsetLoader import MultiSubsetLoader @@ -93,6 +94,8 @@ async def check_datasets_are_available_on_hf(tasks): def test_dataset_availability(): """Checks if the datasets are available on Hugging Face using both their name and revision.""" tasks = MTEB().tasks_cls + # do not check aggregated tasks as they don't have a dataset + tasks = [t for t in tasks if not isinstance(t, AbsTaskAggregate)] tasks = [ t for t in tasks From 0ef9ce16b9548c4ab6716ab3a28a1fd461415ec7 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 3 Feb 2025 12:36:19 +0000 Subject: [PATCH 147/154] fix merge conflict error in task metadata --- mteb/abstasks/TaskMetadata.py | 101 +--------------------------------- 1 file changed, 1 insertion(+), 100 deletions(-) diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 1179cc0321..9ba9839ecb 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -1,26 +1,12 @@ from __future__ import annotations -import json import json import logging from collections.abc import Mapping -from collections.abc import Mapping from datetime import date from pathlib import Path from typing import Annotated, Any, Union -from pathlib import Path -from typing import Annotated, Any, Union - -from pydantic import ( - AnyUrl, - BaseModel, - BeforeValidator, - TypeAdapter, - field_validator, -) -from typing_extensions import Literal, TypedDict -from ..encoder_interface import PromptType from pydantic import ( AnyUrl, BaseModel, @@ -244,25 +230,8 @@ class DescriptiveStatistics(TypedDict): pass -METRIC_VALUE = Union[int, float, dict[str, Any]] - -class PromptDict(TypedDict, total=False): - """A dictionary containing the prompt used for the task. - - Args: - query: The prompt used for the queries in the task. - passage: The prompt used for the passages in the task. - """ - - query: str - passage: str - - -class DescriptiveStatistics(TypedDict): - """Class for descriptive statistics.""" - - pass +METRIC_VALUE = Union[int, float, dict[str, Any]] logger = logging.getLogger(__name__) @@ -309,9 +278,6 @@ class TaskMetadata(BaseModel): prompt: str | PromptDict | None = None type: TASK_TYPE modalities: list[MODALITIES] = ["text"] - prompt: str | PromptDict | None = None - type: TASK_TYPE - modalities: list[MODALITIES] = ["text"] category: TASK_CATEGORY | None = None reference: STR_URL | None = None @@ -323,7 +289,6 @@ class TaskMetadata(BaseModel): domains: list[TASK_DOMAIN] | None = None task_subtypes: list[TASK_SUBTYPE] | None = None license: LICENSES | STR_URL | None = None - license: LICENSES | STR_URL | None = None annotations_creators: ANNOTATOR_TYPE | None = None dialect: list[str] | None = None @@ -362,18 +327,6 @@ def _check_prompt_is_valid( ) return prompt - @field_validator("prompt") - def _check_prompt_is_valid( - cls, prompt: str | PromptDict | None - ) -> str | PromptDict | None: - if isinstance(prompt, dict): - for key in prompt: - if key not in [e.value for e in PromptType]: - raise ValueError( - "The prompt dictionary should only contain the keys 'query' and 'passage'." - ) - return prompt - @staticmethod def dataset_path_is_specified(dataset: dict[str, Any]) -> None: """This method checks that the dataset path is specified.""" @@ -434,15 +387,6 @@ def bcp47_codes(self) -> list[ISO_LANGUAGE_SCRIPT]: ) return sorted(set(self.eval_langs)) - @property - def bcp47_codes(self) -> list[ISO_LANGUAGE_SCRIPT]: - """Return the languages and script codes of the dataset formatting in accordance with the BCP-47 standard.""" - if isinstance(self.eval_langs, dict): - return sorted( - {lang for langs in self.eval_langs.values() for lang in langs} - ) - return sorted(set(self.eval_langs)) - @property def languages(self) -> list[str]: """Return the languages of the dataset as iso639-3 codes.""" @@ -475,9 +419,6 @@ def is_filled(self) -> bool: getattr(self, field_name) is not None for field_name in self.model_fields if field_name != "prompt" - getattr(self, field_name) is not None - for field_name in self.model_fields - if field_name != "prompt" ) @property @@ -541,43 +482,3 @@ def __hash__(self) -> int: @property def revision(self) -> str: return self.dataset["revision"] - - @property - def descriptive_stats(self) -> dict[str, DescriptiveStatistics] | None: - """Return the descriptive statistics for the dataset.""" - if self.descriptive_stat_path.exists(): - with self.descriptive_stat_path.open("r") as f: - return json.load(f) - return None - - @property - def descriptive_stat_path(self) -> Path: - """Return the path to the descriptive statistics file.""" - descriptive_stat_base_dir = Path(__file__).parent.parent / "descriptive_stats" - if not descriptive_stat_base_dir.exists(): - descriptive_stat_base_dir.mkdir() - task_type_dir = descriptive_stat_base_dir / self.type - if not task_type_dir.exists(): - task_type_dir.mkdir() - return task_type_dir / f"{self.name}.json" - - @property - def n_samples(self) -> dict[str, int] | None: - """Returns the number of samples in the dataset""" - stats = self.descriptive_stats - if not stats: - return None - - n_samples = {} - for subset, subset_value in stats.items(): - if subset == "hf_subset_descriptive_stats": - continue - n_samples[subset] = subset_value["num_samples"] # type: ignore - return n_samples - - def __hash__(self) -> int: - return hash(self.model_dump_json()) - - @property - def revision(self) -> str: - return self.dataset["revision"] From a9a53dfd96c2def9804cd6b9dfe6a0090627fb43 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 3 Feb 2025 13:12:32 +0000 Subject: [PATCH 148/154] remove old file --- tasks_per_language.pdf | Bin 18297 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tasks_per_language.pdf diff --git a/tasks_per_language.pdf b/tasks_per_language.pdf deleted file mode 100644 index 6f6f267870386d17de1ff52c11af2cbbe683ecd4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18297 zcmb`v2Rzo__W&LuTZj_zAY?v^hbJRDTOP8q_nwatO7`BnP*!D=Rc1C(XjvgEL{@gj z|NWrvr%(C)KfV61|J&=j_kG^y-h0lu=bm%k_rCALazj#@8_L5=$WrhLlwVB<1wlae zMplGELLji(Z3j~jSi%r(Xk%{<0^cw+H+2TV00b0BM1;`P&IE%f@Jj`0dpk4;eux0p zR5r3QHAaK@k8Y*h(brYchGcD_ZVdHKrT}$Ex~IO@RgAicj2&svUr8JZ zLy&rVYVirHqO5sQZb`gheOweAK3uU;Oc}{v^{=s|E_)FxjhUF6NPc0QKt?XrMdFW@ zjKwwR9o+rCu)}a|&wanjJ3W1EEPW(xYL|1rUSRlp@z#$yu9=6+o2{J_RsC$Gm7jx# ze=ww8-nF=P@M9J2ao1xN$A1a5`i@=NSQp7=FL$N4kdJnAx_`K*;p%+2TZISKnwIg) zp(p#U^w$dT%rDoJZ5fG^%V;x9Z11gaH}u%Q`6#L3C!@qTnPlWlpy##0W}O$8Eayb? z7EGTgCy?ZBaBP8cno1CT``4uz689Vf@dAn*)vh=1qjRQ|3MG8IsEuVluuq1q zo?2?=$(pgMb4k-}H0@~p@pjNfhEN0?9OWV5HjOnJq`UGw>$0RWS&C>@AZkvmMig6` zfI*`yw*Cs)vdBUn99eB*Y|f@0PK3anTf>i2Bj!mCi`bC5b8mB;E%|Kr79Az|xiERg z7jGqHlP^P&1HBgdX0?$H)N znyoU)+&?9cc7qeHN4KZwQp~`yr#i<#*gHuo09GaWq ze)lvES0OGOFj_0o)Iu0S7P`-sv*W|> z*w3(h=fYK5tZ}C3s_R=ezW!%BhbdVn#40r{Tfkq5?OT*HSD@GH?(hp)0Ze}k_(yH1~Ovw`$rS> zomfM!Q!DMlU?5HrXk%mqam{`avEgkJMw8XmA`p)6>n4#B<3kxWGi?^7+%d>~eEA{8 zkC(@o`TO3lQeb)-hE@Ss=WHl3Qg{Gc`*?B2IVa@(t(JJ6kCJ!OA*BQ_4By`zy@g`< zoVIl{j)IQvwnx+>8XBIp5QFx5CilLvgbX!5u8!NQ>!YTW@{S>1!pGxsIgk86V~cgY z!{;=E=xBOLj3F?Fj9kS8dJk$d$U1y7TZLtbkA+V>xb@ZNtc2!GLgI|nq$5;AwKDOQ zc-XeTkc>{})DpovDNkA}s{UmCwKHDKf$TKrnPYDnCC3m37+l)+Y?2Pt_0|YP-dBU6 z@QgIEyMrxSelkxHA>*h}!bJ0|Q*dMRQvTOdX?|f1EVk18MjC!0SSs+_KMZ1s#uK}z zz7f#Qx~vOt=Z;i>(pZ!)w4^+OO%V{hdlpKE+IVQ`t#0nle0L&34)2wRAEm0*>vONr z76$zZ{H8N>)zSz_J^E*D%pGe}(*(3zF6)Zhx1X2|ii*(Ny_}k|VPsGqcv+lp{|X%3 ziI8IF6$)R%d{|$LKu2 z8&(p^MvfKMa@>U+lx6T0t`3$eWBSuqR671Xmaa1Hv<*d=?yaDt$t+9sG2PQf0gsbN zNfxSC(-$&tmI+bN8L@S^OX{#JDP5ps1Xzk1f)?5$rVtanJqlOZ(jL~f8&K6(G zhqd}^B^*CG)^GWf>{jnQ-t~g1qrQCQ)*+*`?a--{I%Z0e^7GKvMtFyt7ne?r)PU$x zW*vJ?a5RSo=hQIs-PPwzK3;-`9UtnHp-OIq zn+h=3pGxON=oAOsQmCSlD(YuC;^cgoENLI~c0+vh^1!QNdqLik`Bx8~$+sFT6ZKo= zgrnI8nkux9Md~E2QSX$a-o&d}R@PKsXMDtTZ200U^ujRb>ZN&>Qv0PNK^L=5;@o8! zizMziM@F*x?XZFL-DeT3Gjv-ywuuHhp^P#1Y~dl%z(`%bXfDsy{Mgvu!7!G%CK7zq zLJuy^`Lq|}oSv73c+87>kI9|!D;=aR>S_9BX2g{+Uxt6A^0@YyF0(;fQqT%*(l=L+ zSO|l3d9qK9IEzpTbOPHJQo}sP*Le_AU9vRc^E!S-R9wbw0?yT_dTp z>R6TEDfYH*%hlIynGnX}%0(Qr@_q2!N(|>SIniok7EJ3$a*vK6Vsw;J_qiFI0M{@% zCOWM1WrpQantmuAyT^DaR}{G*l5z;N^H$78koj3Z-zIfC>xz2oSwA4or<0eR0HEEf)H>}dwOKPK!Cc);E26y#F z>~s2j!nc=lYwwgvDEP(mo3f1x>_yOv1Siwke)0I({~H=1;e`3p-ePie&${NZHVIXu zs|O>ta7_d=b5>f4kF^QJD$VW3c9(oksdoYixG5cj?4GwEFItG)plI^Vtxk!*|T^3SvsP_ zjQM=?TLtf;*pS{=_gtTC1vlXH#^g+~W;=PTg@2zhFIK*`@8z+|;6&}VzkOTKerrZw z>A-pa9k`$2z-xc(dylu^-jALq4G-@Vun8G!jp;b8%LmafYP)4482hrU#${{W=yA3j zHhaeO?+n*PCXhA%NF!Qxn5ouMGcBkr>#v%^PO6yQ-D|J07i{07aq$?xQY%5`rtc7h zU@U(+(w?*!SuC?A|7thJdUG%U_xosDnpaCT>rXiiPcGUC3WD$*U}DccwHM z+^eZtVjIcQ;AC7OLbp-l&88y~o>^=cw9CXlyNjo{r^P_^{G!aBrIn&9>uHA>32A~` zTB^`f(G<2;5%1R>bY7bon+QI(5N%5GD+$1I_{tsc$nn(8n7FHPYud{7ntG#Oc*ngb zmobD=7qR9PjzsYkviXzs#~}x`te+J}j?KX;OGK-*_}F=gu~?s8e6@P<)1B&cO%V0C zAIa{=gwh~x2X*UHH{~8^8<1=B;C=qOUhu;zmdApu9qww=SGTB&$_-DXhcb`WZkxVs z$Zz}V3umB{8p!k>xt@)Z8m9T4MW0qti>s%vm@fV##%wi!WVSccTjpNB(3QQlnmw9$sBTA(&6rW-So5zK z7@Xt=nzHi}6r$xUHjzZBZBB_^-#6x}sNHF`uXZfCUGN;hJ{>AuB^FTdjeN5uEUH0i zr@zeHXG@oxKz(9^vv0xN4R_ryd|En zn{*9d&kWD7>BAEBXSa*t+zR4^llS8hcjOA_K2bJ|DF{6w!w-o`c7${eyz&i=KaV2i zwp@!QZF(|OX4AxcMX@7ftKyU~5l*?1eoieSt-7DGwLz~w4gK&erE&GRi)S14$n~ib z4yxWfrE@_9BW3fl;w1Bm3FN@=%a1&WC!8n}QefLpmg=Q9v}mPllOGG`ChpZ< zV-5K7=DF=F73te|1Rd4!DTo8#e4pSoCL)p(Z^=Xr(evTij~zfuPx)9BiL{Jo=S6aJ zJ?(tX8bHmk4iag&{|T?r2<5`q>5#0}PG1SKXswXbvQ6fUN>VRcWL4pE63c!9{Z7FJ zv?r;`D`UB^CU9nb)#_OhwT`rJsHJ~YHps@%he*0XUySUk!_9s-Qs)=_U^cwZ?2*1} z@yh;8Hiq{SsrYj@Ohr4GNu6_Rl;cFHEjeRRx;iXG__s9A_L|4N-N0wqTbrCxAtoPp zC>2@BmGkG`iQ3WA&KVl4*#Xgw1UBgHiIkpNF>g$5BB}}J=IWi$N~pfZS*#nAHM~68 z!V(Zpdm$;R-;L&$=A-FC4j1py*T(8d`*fvR)Rmna&3fYNJvxu14y4dDy8H$6>Fdq7 zMM?c}5U85uON5%#BX2UXp-CrBxzV;Q`UzaB&gl?q<>ude?!Hcc>jt113Bw z8aK&RQVu`LW6ojSi-JY6I~RAGuy_YEspqM064GPSBxr*y>)M=uSQ*{Bt9bKSO41#g zTNggBr(MjBMevEPJs4Eq>siF|f8YQK=m#FHF$~#iIPqv-EtLN-pR#>m7Go8T??yhi zT+h`;y=QHx7x!$le&xyy@4faGIOE3;U@f}QDi@3OdhSL_|Bd?B)zs+N)jT4W0*fZ@tBA*x8p;y8$~+ea^P5&!1gA z&P)2gkg0OdNh%G%)t<~ksSj&K8C>=Ff%b&gki-_nwzqKBgcpt;OdhDo$)8$f7GE2V z0<-R!2Zh6_^oC4rK6vw0`(H?{Z0Gt`Uz2k48mPnN#p+ugsov(c_DN~e+=Kpwp|RM{ zhP?FS-&1pw&O7P7t`(P#PARYS;a8-rECJz~RB{ARYg;%$*y>YG&mfsQKJ*7SSB^z; z@AQmXAoC)I=o3KoDIfhC)8?FZOTbmYNTE5O$gz}fC~ReEymTu|>ZK7g`*7i5Pb`_)iURp{&MOhT$PnK}Z%yIEF;!lj?QjVqq zZUI}j*cLUUt=p@D-r2QUJ;#(MmY2c|9)ndrdvw22XuNMx-|Ui5K=I=$gL7$bzn9>K z0rMbZwxnuRMquT7?b<_0Gu%sRKE~4|$^M0Ta3YwNa3pPEAvvA&OtHQqE7oO!w(FnT zYo%gK0&v+Myvf+0RJf%cjnWvnK?R(YkkW6S>HyKLTmrrHnS0a`?f zi;YUEHFD_U=Ci+s|Oj$`v9a;roiiUI;@R5pfMrl_bIPj-f#7xlK#&=p!qX>KpmpE+uDZrGAq5-mtD?aJDknVmi8~hO$8VnPQI&THo z-RRW5$;(UgjJLSl)xtLxK6LxiMO%2J;{i-P#2s%?E_{2XYID15ck7Gm#@TbH%XeNI z)vAV+NNq|*w*)#JAR4&i=|3W0tLJe}bmooiwo;NC5)tJ|QP2(#!~5ShGlRG6xaPLL zWixy~zxVP6`h#w$-Zg@;0_2kw=l$AXL;J5}7@FeX&9->!Tu{ozPjTYddtgY(V|Gh*6nojmvs6N7>i z&1x6q_8q?r%80~w&r|EGM-Ay%YZ|xKn1>sP$r&d0FJ-kbtlfSTsJ!D^z$G3sphQRb zF)5Q)Y2K*ZBxwBW*xa7!^Q~L;4aU#6{6xt5mu1OFlk9pO9Ykp&71bxFD=&bHwFI$O z?s}82-C=HHSU2qm?D25#=4FwPOw^6np*H?fb)if%#%MO>eiCC1XpG-zO-vsz{My|3 zO6b*ALE*Cs`mI7jdWK;uzSmrf-YaZs2G!fQY-QCG4=!twMWxi&S4@lWyb6PcxvrI# z-w|ur&gb6wItb6uws%>k-zYm+Us-=Qzd2LY({pY9mDPKytHX%%x+8TLbzTS_kZ9{` z77I_$jHS6~&pp!5>2F&mCtY7V7<$pei&@e?UXngu+&^5u#;j@c@goF&uJ|7=d}DyO zfYtGzh?r&m-xj|SP+-CPKNh~>e0&;^8ztqZC>}qSCj26uNXQ9$l9v#CoCAeM ze1U))8@YKRZ}%FwCA}8l_zND-9I_QCH^{ww_EXp13fw5~P@u6GDytv3{K7AZIiBtQ z+kB(-87Z<6hR*a0#3vAS!nObc@!v$j;k;^)8@+tk6lQ^}|B?(YWfxXiw?g+lE@M^I z%ZXUg2pFH=yR$TpupICtz40_IoT33s$^{c9;n>uhsTGqJPiFL%@&L z$TDT;HO~9t{nm3g9QN5tLDvSA@5vyfCwX&r_RRGVCgY^@Gm~Njb8<}<+QA8KQaECZ ztL3je2^NP(tva5a$xVJE$Zl3Lk4Ev>oeyH9?^u1hJe@UQE#9+L!pX8GGY|-OVq3Im z`|`eHB6@!`;Dxmla10+SZpA9U3-~RJKzh>J8a79%M=m|AqkWWFfZ^d%&Q6} zxO{am#X zXOfHPyZA-a_(hvrzUS8E3fR&f@y?RJT6T-K_HvXo-!x*!#}Q zCc{D%*Lq`B!-i;#elu?tOZcVCLn4k|l@uc@=b8bYmlW<7&*$x80Ygmx|Qj zO)e3X~bjw zTyHA;EPKu;X zg*^05^jDh7dGOSiD%_te$*{~%LhR3kq@V3ydK$F5p4Qg|qVO(R&)D~o?UAQh~X@pDR6IIRYkPKpf zbY*iq$XP9Wtuqq#WmX`DEj?^$W0=bod6#sa2ob?ZHM7Ox;fwIgT)4f2ebGtmKzC$| zUaDr61w>0g{ZYkeF?HHszs?m+UvY)PS^T~j4<+O4IgZC?@82jW%X)-TamW>UGF$M_ zds<(I`viGm_N0H87ccU^6-@YsV5B4?@jNmBynK0;*@0dX>USL`O{{hC1e5-89c53*qN!ec}X<<(26@N0(r#*r_jpm?cs z=*PlmSdl-P2>qRDO&8Q<*h2D{BEP~T+0!{ZAG%X>ed|EnVv7}?0*#|kA)Pr7<6m8~ zeHBl5)vWI{7sZRuCX|W-T{W_(aGH47SNg}UOm_-|I-byWh2|$NPrK#-tEb=98)VD0U;pWMzkk5LLPecDGgIsC$m_R$Nak-1X$&C^nzx zcBACZDNKr^v46gpw0S+jHiks1hnq;e-zIh&=5xmDLh^N&Nz0t}kqw*TjonjkY-vL+E3)SrP% zcq@j{nSYP=ZsuT(mPc&o;6d^Xld8%V>M4(rxNb0QPX_t2aE3L7R%zV|k}E64x&918 z!1L4a3@qb2valCVza%hHDiOQTk!oi*J-0S$p3v?Siuh#rL)y%g)t@BFMEui*d%>W2 z>}MCa;t>c+nARJdZzP>0+6Mi(v?^h~obewC(3$EjdEKAOzI=4$m*8Ig{PfPsj)i)X zK|r7Ux~98+fJky~itA#mxW~AnAX7IzVKz*Vo3p~R<*S{~&b2Po@?PiW)}>;y*Sk8Y zCrI#wkP!Jd9xtMi{3Qtg^nscf0U_a#GHV=Qvm?&JybNYY7yWUGDsM|O;3lx@QE8r$ zGyZfVO0;g=iM#U?E|~+H_fW4|YT`RL%NW_4?4PnJZXb}EvqnF?-DjM(No4$kS?B~R zPly@eynkUTrBL#Al2H8N22}?inen9P@8x-znCR96g%7Vk6ByXIcd0%$c~$a)h52%S zyz(6;iGs(wyiS*y9&gDK;JPM6wOh*{hSa0{Ix(r6ONWU&9h>9=~s< zu=fW$LDV`+cQ%dMdG1jc%+xoMo_hL(vK+UAW@r(?ZgJ{nb(m~Z)~bh{Yo<-s&ADy5 zQ*cs<)5~C%TbJtlMo%T)(Vdlw2;JipRnZdN?fBr@I125QaiIQ!7g^^Z)SBWTFVKc9 z8d^7T!DE?a;0suzDER&|flIXsxV9;M=*x_F9KKuODN;R?=ED4}r-|R@gnT#5rJSV0 z--iiYWsKE+i}am%`-on9Qis2E!j`%8g~r$-!+0uKXJh`n{DObW38QmD@C-%#jS@w- zJK_Q%>rx&z3*;W*dO`blvibeuz_oW`9v8%kjg*^8bTk|d(alj++;9@_>okjPJ22zg z51JshSsdb+ko>;v9oJTxytSzhMnzefI^e5J6y33t`J&8TSD&@VRaV0H(0U9Dj7yXU z92_%thFCm#i=x{ohL!hGgDPL|XGC#uTGLw%Y541s%MFY0Jb@{V)Zu84*Cpf&4=r7z zcfGp$C$sPq;&cALh;rbtsvE#N6fYJFybN*6E37i&d*v*t zzQN}Al0mj-TS3&4r70mal9y7kV!A%d;hbU_rtUm}o)cnLDEu#`hGHaO(;10zmTH+8 zM44=4Iy@`2zZw2Gu)c)KOBaF-I9AC9{(JjgVR zh3!W!M0EYUU|eUFs%p=w;P=a`jiMtUZrP|S!X0>MTsDy`^0=?b@^nQ@x^4#DZ%V~p zs7Q(JpV@Xzn#ak%0LDInmlHxr0r+1y_--V{JqSK-Z@TI(vwsKErXyXL`++*7505_I z;}N5PUv*>QxWTJxH06@&}+U>)+Q~e6E@QBL5-e+UuV4X=F8* zXhu#T?Sx>B_b=YcP8{~cv+6wO6}ZlHWA`OG5Hh7VCg7cVVNSw!LboTyJbVKGEfT31 zd8gALe96spU*>zeQ$lfarqCA$2$DA@2-l>id;W6Fk14oj&fSLG>0EtnQaR^_R2QVh zJtuV~x%1|1DR#b~YDD2Rgf_Sa8Z*-BlCXQL*E^00u4hlU9>nbDLu3GVT zu9KP)4iq@k*?fLw+kAmd`a$6$>1GH>nN9V2>o?Hg@)@>>I|=!m4rw+Ueew{}(|h8S z$x~sX^Ta-FuT|HkC>ACsU1xj@>~E9H44&+ zTk=QOc73B*Drc;)PwY=;ta-HiX|9gO!P}=E{|J2Z!XT-}Phn~M+Q*)qeH?mrvz-$} zc~bC$_zUfliWZ~*CW?foQU>!DGmc1)8H^pVX>UAWgL?T+6XaJz>HB`jH)Pf~sFGy_ zcd78Sx$*8C!j6w(>a@!}_3TNDc+{O2t(FPs6$^wjCs85gB{Ncw_$-=7XRlI?e#@M4 zdc62%Tf$*of4#wI1oc$DYo)i~B|3YnqZ@h>&4JGc<4Y;WJ(&JmMIdz!)OaAF%iot;vEz@ zMLpCDqZMghkvJC;@zJ5Y4HJ22JoG$i$Ewip>DGvp?C7+mL z&r5CYdo@*h`?TfE)x@G3AuW#gQIQ!9)($y6;AuA1ffGb@Lg4c^wni$v2D1z$>G{~; z^j+2s5taC+(FT&UEOyV?Muc@P5u~u&!J8%U(uuhObsc@y5+Ou7*Kb6oEo2#fsQ-F8 zV*N4d`x04v`a;1(;5-(#_(ii5NIN0gfx-XcMMv>Qr!6pgKWYg1ATMX>YUs~|2|e5M zmUu-Fn`9wBk+B+oEAf@-CNt8$vwB)0NGZW`)vot-ap`^*F}CZ$4Yjj{ETwYYI&bYt zd1b+a%>ofg))kR6xt{)7S?<=7eF5W@U9qNWSnJ(rBgUZN*>T2_{^|38{|$ zE}la;Pye1JGm3LK8I`8QJ$)hR;jE`uBbY|V8JUg3NpsAGL`gx%=b0RbdyumILc7t<(92Y z4TIVrIwT_;B6dUO@YHsdzv{#l5Tw+p&E%g(fc2i?dxH{Sc3GETPA3>i|j8&XYambftHdAh4+H|Jg+m<&4* zX?5m4&Fv-KdryDrVR7W=?O1Bd&1BnmMII#zf_d2`bT-P)daU}|<}a?&_nhh6C>kI~ z)nK0scmaRoJbvY!x^y|UD?-Lf$HqQ#PgAZpz)nXxlsWogE|bus?9Ogh!ilI{*?&`z{B9iOCk^F0Wwg3FrC*UOZqG~Lz}DBCAiHLJMI*5j1~E zm#Afden459?cdX;q^wl00iS0?QW4}DX=^lEo=Y(z7^cuyRZ+d~L9_p3b#X=T?miZm zvzPM;!*Wt6_7_GMjwrtYxz~J7qKNm@r7t+B6IwnYFoMGW#&<=@lb&V-!hqDw$d~O5 zGmfyi0Ox~m_tNYrjfDf$d|vow2=iFuq){<2(p0WgOfIq4{8~7`pC+p> zb>w`k8O;o<_s+(G5}d%z39%F$(CPo&hu}jY?Ew2WzejinbkA;hX%Hk6v)e$DJ9rMq z5siu$)K+ut3JOrGSU_)=Ck#>WPq~=}v=DUWQ+z`O$~>)AXZ)~~0xPcKGe6yd%4wR) zo^tH;z1E!bgs6SHvb1=*<5Cp8CKYprPE03;6+}XIGrU~l%MknhK9&#h*o~zH03^CG^ z%T|N$n6N7Ylvc@ zycFLtMoN<5txum7o{AX7{*rmkmg)@YL|L=m3m+=E^VYGOSh)(bK63}fD;4$q{S24B z-D^8xa88Uk|KfduTSe9hM-0EFhjY!}?8%sM5@*oVK|lVhGk7FnqG;`jN8t}aTpG30 zG5HqFV{oYN7;eyA&VF0XDN!|$tnm%L4A&KpHTnU^WP^$DuUl zRsE*#&o1Z|3#W*67U;CFP@Y%#V90DvdMyRUQ|>Ix51NtIOfrnNQ@;qOh~y2BVGPK7 zkp1Sg7)5xIbOgfb+>ab^6gu!`#8c>kGqQp?vt|JfId2j*U+ByQ~ zyozPPs;JD%Jv*{>=Vx(a?(#ElsNYR`zV2LYQZs0M&~GP-_%Jb68$9_Vym)42dlyIF z;8ySnGCV2Nv{!}hPiK;F)(-+Qs8M1)iT7soT~-*~kM<2dcH*K8eL|xE0JoV<)hELe$C6_utbRUcrlk+<+i}?L{+b z!pE#%e%W3$AvBwIpwYU|uBmy7hSluL4aVd^Iv!(H=SNtgQf-e7F6Lbe;`HMrIh7Di zv2+4SCq;RG5lQg@QQkcWv81Q+IQc}w;2B23freID2jj7kI6c&a==4?c$j3fxV(Te- zpussTb*Tvz+3o|yx?CyMp2y$v6;HFlBiCK%p5A}w(W`yMHjaAK;m)blOW#f)@Pt@T z;BN?2d_fKnI4>M<{biI)Em^yY|K6RF^56i@BEbQmJg6Sttj!6tc?1y4DK6+56E~UmJ;}BRqk?lY*+hS$Y$WY9wuj z;7crR6CQ|W53px=DL=6aT0AIbmaif5z%~vK=(4jACSJ{AanG|r)$-G0N6MB>bWfFb@l<{lm@MKMbWFb_ld*q zhc}RnjVSgLJr+2B!p<+>3E=|_`WN$uPX&ot$ywN>+84bX4t7`0yz+X#;dOYQI(?%~ zV0k=a$|@@EDdV)Nh(`)fYW1i7ZY$oI#By%ud(SIt!G*CeL(GEx8ZDEbW8D|RlIZVy zsv>cozW05O>XnjDr$s`m?8rupD}0hF8D~O3MptOK#9rr0QBvFXux0Z#0IzDJN#8~~ zRCC%AFZkeLsSD8c&-O1T2rIMpp{Zc+^ZB|i5Ku}n32jF{{y;ge&HENc@xG0`zzU~U za1Ww%>J+787q-bmx%jfF7W>k*GCYl)eXNJq)e`?4Ge8voEcPFrdOSSJ2o^VVHa#vW zs;Ee+b4r?88LGRe8rnHyjE|EuTEfE638)NKFgz~9`1lCHm^zZC&c;rb4rqHP5ESG2 zP$00XixK+pOeLldAjVQMv;|HkivQm9QGM~F(}>(q2n-71<^|Rhp!~r4ITQv&XSzV| zKo^RJXeZ0VBa1u`2xg3dzyFmmCl>+YZU84Kfy0W2=PC)oQg*=EL(DnLf6jOQ1QCKI zEzQhKfm4#0W0AK&m2_yh`BsCBcSj93V1RH_C#vrf>2#mo8fq|2x79g-C2y6ucTZ6zhAh0b6 zYzMRr0y}`9fT+Mgi-361Ag~Jv>*6L=4^~PPl|*ACKb~I=7t-F+47PwbdWK% zG`B#5AcSDepGDxbDsYb0(pb#S+{P56F{)@&TXjr9(a`;nBY+`Z*sn)fe+IL+KnMur z|BpoY|Fs=Bu)@O&Xc#X70uq1=fDmv5Ari23`~naV3~2b&QFNR zF+^aP!XyNsaG1J$0OLn(0JuQEhh^X$83yP+Ch_qj4|60CmmvT+0ty2dJnEMh=$8+e z&u}EB9xsRw$OQmlVe-QS!{8wfg@DO1{T~h*Mt}effItZ`eewZy4u$fQI81)1KbW3) zF*gv5h%jVhSi|H%Ivfv7cSnhk4`A`ICPu#h6aWH`nt%ZuW1e8B0w~8+$Ed;2;rf{a zLk6r1{n7$2KY#>$Jcc13qX+*Y6$3m}CZHuut^c)xAqoN^JbZTC01|-&{Y)4|IMM-( z9{f8U${NrMj86QL4)p<}6F<|Tt{m$GMjw7n7`hJcfi^I@@iSpKM?m<1F$8GE=)egH zDFBQKMlS$Y@@x936F+l|P=IkKJj^lWU((@NA3ew5{4Zhj0yC9{8}2r!?AxJS;3_n0q? zzXCqBI(*^+;yv!q6)6+a0ry2 zj~5E(guvJ!5H|K-0PE|A?Z42nQvH1RhW zKLil-?=Xx${{|BPK27}t2AHtlVMxq8{#RY#2Jw3z5C|MIKDf&KU2VVEHNcNh|g zi++P4;eW^!g2cFz-|_hQ{@@LX{8RQ2OlbV?y1<(O#-d`b|H%&u;RC*p_#GwyVEhJy zLXf}H0)@gbHW73E(;k!$*cAR9#{Xws81zrwg~0^=s0(~V^#^@0B=8yXZ+*aFf9Nw1 z-vHs&?|8u1&ws%9fiUrRJS4`-W3GRA&Dy8!~D?CmiI@6ct)+L_q{G1K7 Date: Mon, 3 Feb 2025 13:25:36 +0000 Subject: [PATCH 149/154] add mieb to readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 59cc5da9e2..32e95932e0 100644 --- a/README.md +++ b/README.md @@ -483,6 +483,7 @@ evaluation.run(model, ...) | 👩‍💻 [Adding a benchmark] | How to add a new benchmark to MTEB and to the leaderboard | | 🤝 [Contributing] | How to contribute to MTEB and set it up for development | | 🌐 [MMTEB] | An open-source effort to extend MTEB to cover a broad set of languages | +| 🖼️ [MIEB] | Extension of MTEB to image embeddings | [Tasks]: docs/tasks.md [Benchmarks]: docs/benchmarks.md @@ -492,6 +493,7 @@ evaluation.run(model, ...) [Adding a benchmark]: docs/adding_a_benchmark.md [Leaderboard]: https://huggingface.co/spaces/mteb/leaderboard [MMTEB]: docs/mmteb/readme.md +[MIEB]: docs/mieb.md [Reproducible workflows]: docs/reproducible_workflow.md ## Citing From b042e79cb7441a6dc8a5f90725bcead058c96b5d Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 3 Feb 2025 13:27:20 +0000 Subject: [PATCH 150/154] add comments to mieb task categories --- mteb/abstasks/TaskMetadata.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 9ba9839ecb..24b3c9fa23 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -129,14 +129,14 @@ "s2p", # Sentence-to-paragraph "p2p", # Paragraph-to-paragraph "t2t", # specifically for text-only tasks in mieb - "i2i", - "i2t", - "t2i", - "it2t", - "it2i", - "i2it", - "t2it", - "it2it", + "i2i", # image-to-image + "i2t", # image-to-text + "t2i", # text-to-image + "it2t", # image+text-to-text + "it2i", # image+text-to-image + "i2it", # image-to-image+text + "t2it", # text-to-image+text + "it2it", # image+text-to-image+text ] ANNOTATOR_TYPE = Literal[ From c00f079e03d41d9919d86c326cf01309eec443bc Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 3 Feb 2025 13:28:40 +0000 Subject: [PATCH 151/154] remove commented out code --- mteb/abstasks/Image/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mteb/abstasks/Image/__init__.py b/mteb/abstasks/Image/__init__.py index 9f816d4322..e69de29bb2 100644 --- a/mteb/abstasks/Image/__init__.py +++ b/mteb/abstasks/Image/__init__.py @@ -1,6 +0,0 @@ -# from .AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval -# from .AbsTaskImageClassification import AbsTaskImageClassification -# from .AbsTaskImageClustering import AbsTaskImageClustering -# from .AbsTaskImageMultilabelClassification import AbsTaskImageMultilabelClassification -# from .AbsTaskImageTextPairClassification import AbsTaskImageTextPairClassification -# from .AbsTaskZeroshotClassification import AbsTaskZeroshotClassification From 6eb46a4ea441dac52e4428f6f076dfd1919d1bde Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 3 Feb 2025 13:30:43 +0000 Subject: [PATCH 152/154] use logger.info in abstasks --- mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py | 14 +++++++------- mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py b/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py index 66a25c6619..64ca450c4d 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py @@ -388,7 +388,7 @@ def calculate_metadata_metrics(self) -> None: ) for split in pbar_split: pbar_split.set_postfix_str(f"Split: {split}") - print(f"Processing metadata for split {split}") + logger.info(f"Processing metadata for split {split}") all_details[split] = {} if self.is_multilingual: pbar_lang = tqdm.tqdm( @@ -396,7 +396,7 @@ def calculate_metadata_metrics(self) -> None: ) for lang in pbar_lang: pbar_lang.set_postfix_str(f"Language: {lang}") - print(f"Processing metadata for language {lang}") + logger.info(f"Processing metadata for language {lang}") split_details = process_language( self.relevant_docs[lang][split], self.queries[lang][split], @@ -431,11 +431,11 @@ def process_language(relevant_docs, queries, corpus, lang=None): qrels_per_doc = num_qrels_non_zero / num_queries if num_queries else 0 language_description = f" for language {lang}" if lang else "" - print(f"Average document character length{language_description} is {doc_len}") - print(f"Average query character length{language_description} is {query_len}") - print(f"Number of documents{language_description} is {num_documents}") - print(f"Number of queries{language_description} is {num_queries}") - print( + logger.info(f"Average document character length{language_description} is {doc_len}") + logger.info(f"Average query character length{language_description} is {query_len}") + logger.info(f"Number of documents{language_description} is {num_documents}") + logger.info(f"Number of queries{language_description} is {num_queries}") + logger.info( f"Average number of relevant documents per query{language_description} is {qrels_per_doc}" ) return { diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py index e288f3e37d..9913370666 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py @@ -383,7 +383,7 @@ def calculate_metadata_metrics(self) -> None: ) for split in pbar_split: pbar_split.set_postfix_str(f"Split: {split}") - print(f"Processing metadata for split {split}") + logger.info(f"Processing metadata for split {split}") all_details[split] = {} if self.is_multilingual: pbar_lang = tqdm.tqdm( @@ -391,7 +391,7 @@ def calculate_metadata_metrics(self) -> None: ) for lang in pbar_lang: pbar_lang.set_postfix_str(f"Language: {lang}") - print(f"Processing metadata for language {lang}") + logger.info(f"Processing metadata for language {lang}") split_details = process_language( self.relevant_docs[lang][split], self.queries[lang][split], @@ -426,11 +426,11 @@ def process_language(relevant_docs, queries, corpus, lang=None): qrels_per_doc = num_qrels_non_zero / num_queries if num_queries else 0 language_description = f" for language {lang}" if lang else "" - print(f"Average document character length{language_description} is {doc_len}") - print(f"Average query character length{language_description} is {query_len}") - print(f"Number of documents{language_description} is {num_documents}") - print(f"Number of queries{language_description} is {num_queries}") - print( + logger.info(f"Average document character length{language_description} is {doc_len}") + logger.info(f"Average query character length{language_description} is {query_len}") + logger.info(f"Number of documents{language_description} is {num_documents}") + logger.info(f"Number of queries{language_description} is {num_queries}") + logger.info( f"Average number of relevant documents per query{language_description} is {qrels_per_doc}" ) return { From 78e9d6ef25b6ff5fea07c698cc0ede229a1709be Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 3 Feb 2025 13:37:32 +0000 Subject: [PATCH 153/154] add blip2 dependency to pyproject --- mteb/models/blip2_models.py | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mteb/models/blip2_models.py b/mteb/models/blip2_models.py index 9cb914b15f..e7a5185d4b 100644 --- a/mteb/models/blip2_models.py +++ b/mteb/models/blip2_models.py @@ -20,7 +20,7 @@ def blip2_loader(**kwargs): ) except ImportError: raise ImportError( - "Please install `pip install salesforce-lavis` to use BLIP-2 models." + "Please install `pip install mteb[blip2]` to use BLIP-2 models." ) class BLIP2ModelWrapper: diff --git a/pyproject.toml b/pyproject.toml index 43811556c1..c91d510165 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ pylate = ["pylate>=1.1.4"] bm25s = ["bm25s>=0.2.6", "PyStemmer>=2.2.0.3"] gritlm = ["gritlm>=1.0.2"] xformers = ["xformers>=0.0.29"] +blip2 = ["salesforce-lavis>=1.0.2"] [tool.coverage.report] From 87ea21c9f41e3b19236179dd513011c9260eb435 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Mon, 3 Feb 2025 13:41:29 +0000 Subject: [PATCH 154/154] remove test code --- mteb/models/align_models.py | 6 ----- mteb/models/blip2_models.py | 40 ------------------------------ mteb/models/blip_models.py | 15 ----------- mteb/models/clip_models.py | 6 ----- mteb/models/cohere_v.py | 5 ---- mteb/models/dino_models.py | 5 ---- mteb/models/e5_v.py | 6 ----- mteb/models/jina_clip.py | 7 ------ mteb/models/nomic_models_vision.py | 6 ----- mteb/models/siglip_models.py | 8 ------ mteb/models/vista_models.py | 6 ----- mteb/models/voyage_v.py | 5 ---- 12 files changed, 115 deletions(-) diff --git a/mteb/models/align_models.py b/mteb/models/align_models.py index b190a5410a..95fb6fda25 100644 --- a/mteb/models/align_models.py +++ b/mteb/models/align_models.py @@ -158,9 +158,3 @@ def get_fused_embeddings( # COYO-700M }, ) - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(align_base.name, align_base.revision) - emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/models/blip2_models.py b/mteb/models/blip2_models.py index e7a5185d4b..e396cf39f0 100644 --- a/mteb/models/blip2_models.py +++ b/mteb/models/blip2_models.py @@ -268,43 +268,3 @@ def get_fused_embeddings( use_instructions=False, training_datasets=blip2_training_datasets, ) - - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(blip2_opt_2_7b.name, blip2_opt_2_7b.revision, device="cpu") - emb = mdl.get_text_embeddings(["Hello, world!"]) - emb2 = mdl.get_text_embeddings(["Hello there, world!"]) - emb3 = mdl.get_text_embeddings(["Goodbye, person!"]) - - sim = torch.nn.functional.cosine_similarity(emb, emb2) - print(sim) - - sim = torch.nn.functional.cosine_similarity(emb, emb3) - print(sim) - - cat_img = Image.open("cat.jpg") - cat_text = "An image of a cat" - - multi_cat_emb = mdl.get_fused_embeddings( - ["A photo of an animal"], [cat_img], fusion_mode="multimodal" - ) - multi_conflicting_emb = mdl.get_fused_embeddings( - ["A photo of a dog"], [cat_img], fusion_mode="multimodal" - ) - image_cat_emb = mdl.get_image_embeddings([cat_img]) - text_cat_emb = mdl.get_text_embeddings(["An photo of a cat"]) - text_dog_emb = mdl.get_text_embeddings(["An image of a dog"]) - - print(multi_cat_emb.shape) - - sim1 = torch.nn.functional.cosine_similarity(image_cat_emb, text_cat_emb) - sim2 = torch.nn.functional.cosine_similarity(image_cat_emb, text_dog_emb) - sim3 = torch.nn.functional.cosine_similarity(multi_cat_emb, text_cat_emb) - sim4 = torch.nn.functional.cosine_similarity(multi_cat_emb, text_dog_emb) - sim5 = torch.nn.functional.cosine_similarity(multi_conflicting_emb, text_cat_emb) - - print(sim1, sim2) - - print(sim3, sim4, sim5) diff --git a/mteb/models/blip_models.py b/mteb/models/blip_models.py index da23a2be3d..43822465dc 100644 --- a/mteb/models/blip_models.py +++ b/mteb/models/blip_models.py @@ -374,18 +374,3 @@ def get_fused_embeddings( # LAION115M }, ) - - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(blip_itm_base_coco.name, blip_itm_base_coco.revision) - emb = mdl.get_text_embeddings(["Hello, world!"]) - emb2 = mdl.get_text_embeddings(["Hello there, world!"]) - emb3 = mdl.get_text_embeddings(["Goodbye, person!"]) - - sim = torch.nn.functional.cosine_similarity(emb, emb2) - print(sim) - - sim = torch.nn.functional.cosine_similarity(emb, emb3) - print(sim) diff --git a/mteb/models/clip_models.py b/mteb/models/clip_models.py index f323756b44..faee0e7c9d 100644 --- a/mteb/models/clip_models.py +++ b/mteb/models/clip_models.py @@ -208,9 +208,3 @@ def get_fused_embeddings( use_instructions=False, training_datasets=None, ) - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(clip_vit_base_patch16.name, clip_vit_base_patch16.revision) - emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/models/cohere_v.py b/mteb/models/cohere_v.py index 0b22994373..c84d5ff640 100644 --- a/mteb/models/cohere_v.py +++ b/mteb/models/cohere_v.py @@ -13,7 +13,6 @@ from torchvision import transforms from tqdm import tqdm -import mteb from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -222,7 +221,3 @@ def get_fused_embeddings( use_instructions=False, training_datasets=None, ) - -if __name__ == "__main__": - mdl = mteb.get_model(cohere_mult_3.name, cohere_mult_3.revision) - emb = mdl.encode(["Hello, world!"]) diff --git a/mteb/models/dino_models.py b/mteb/models/dino_models.py index a123fcb52f..c2cd4db5fe 100644 --- a/mteb/models/dino_models.py +++ b/mteb/models/dino_models.py @@ -221,8 +221,3 @@ def get_fused_embeddings( use_instructions=False, training_datasets=dinov2_training_datasets, ) - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(dinov2_base.name, dinov2_base.revision) diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index dcbe0a1d71..909cfcbab7 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -213,9 +213,3 @@ def get_fused_embeddings( # princeton-nlp/datasets-for-simcse }, ) - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(e5_v.name, e5_v.revision) - emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/models/jina_clip.py b/mteb/models/jina_clip.py index 1f9a597803..551c82c101 100644 --- a/mteb/models/jina_clip.py +++ b/mteb/models/jina_clip.py @@ -178,10 +178,3 @@ def encode( # type: ignore # Natural Language Inference (NLI) dataset (Bowman et al., 2015) }, ) - - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(jina_clip_v1.name, jina_clip_v1.revision) - emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/models/nomic_models_vision.py b/mteb/models/nomic_models_vision.py index 7ea87bd38e..4eb00316ae 100644 --- a/mteb/models/nomic_models_vision.py +++ b/mteb/models/nomic_models_vision.py @@ -187,9 +187,3 @@ def get_fused_embeddings( # DFN-2B }, ) - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(nomic_embed_vision_v1_5.name, nomic_embed_vision_v1_5.revision) - emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/models/siglip_models.py b/mteb/models/siglip_models.py index 3bca2e508e..b7543afc68 100644 --- a/mteb/models/siglip_models.py +++ b/mteb/models/siglip_models.py @@ -398,11 +398,3 @@ def get_fused_embeddings( use_instructions=False, training_datasets=siglip_training_datasets, ) - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model( - siglip_so400m_patch14_384.name, siglip_so400m_patch14_384.revision - ) - emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/models/vista_models.py b/mteb/models/vista_models.py index 2a887fb48d..1344ec87cd 100644 --- a/mteb/models/vista_models.py +++ b/mteb/models/vista_models.py @@ -287,9 +287,3 @@ def calculate_probs(self, text_embeddings, image_embeddings): use_instructions=False, training_datasets=vista_training_datasets, ) - -if __name__ == "__main__": - import mteb - - mdl = mteb.get_model(visualized_bge_base.name, visualized_bge_base.name.revision) - emb = mdl.get_text_embeddings(["Hello, world!"]) diff --git a/mteb/models/voyage_v.py b/mteb/models/voyage_v.py index 3b08a7b564..6968fec03c 100644 --- a/mteb/models/voyage_v.py +++ b/mteb/models/voyage_v.py @@ -11,7 +11,6 @@ from torchvision import transforms from tqdm import tqdm -import mteb from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta @@ -261,7 +260,3 @@ def get_fused_embeddings( use_instructions=None, training_datasets=None, ) - -if __name__ == "__main__": - mdl = mteb.get_model(voyage_v.name, voyage_v.revision) - emb = mdl.encode(["Hello, world!"])